diff --git a/.ci/README.md b/.ci/README.md
index 2a9d5c09b15..c32a0fd6400 100644
--- a/.ci/README.md
+++ b/.ci/README.md
@@ -1,16 +1,4 @@
-# Open MPI Continuous Integration (CI) Services
-## Mellanox Open MPI CI
-### Scope
-[Mellanox](https://www.mellanox.com/) Open MPI CI is intended to verify Open MPI with recent Mellanox SW components ([Mellanox OFED](https://www.mellanox.com/page/products_dyn?product_family=26), [UCX](https://www.mellanox.com/page/products_dyn?product_family=281&mtag=ucx) and other [HPC-X](https://www.mellanox.com/page/products_dyn?product_family=189&mtag=hpc-x) components) in the Mellanox lab environment.
+Top-level directory for CI tests.
 
-CI is managed by [Azure Pipelines](https://docs.microsoft.com/en-us/azure/devops/pipelines/?view=azure-devops) service.
-
-Mellanox Open MPI CI includes:
-* Open MPI building with internal stable engineering versions of UCX and HCOLL. The building is run in Docker-based environment.
-* Sanity functional testing.
-### How to Run CI
-Mellanox Open MPI CI is triggered upon the following events:
-* Create a pull request (PR). CI status is visible in the PR status. CI is restarted automatically upon each new commit within the PR. CI status and log files are also available on the Azure DevOps server.
-* Trigger CI with special PR comments (for example, `/azp run`). Comment triggers are available only if the comment author has write permission to the PR target repo. Detailed information about comment triggers is available in the official Azure DevOps [documentation](https://docs.microsoft.com/en-us/azure/devops/pipelines/repos/github?view=azure-devops&tabs=yaml#comment-triggers).
-### Support
-In case of any issues, questions or suggestions please contact to [Mellanox Open MPI CI support team](mailto:swx-azure-svc@mellanox.com).
+Feel free to make your own subdirectory (e.g., for your organization)
+and put CI tests and supporting infrastructure here.
diff --git a/.ci/community-jenkins/Jenkinsfile b/.ci/community-jenkins/Jenkinsfile
new file mode 100644
index 00000000000..f4b305f1d66
--- /dev/null
+++ b/.ci/community-jenkins/Jenkinsfile
@@ -0,0 +1,107 @@
+// -*- groovy -*-
+//
+// Copyright (c) 2022-2023 Amazon.com, Inc. or its affiliates.  All rights
+//                         reserved.
+// Copyright (c) 2022-2023 Joe Downs.  All rights reserved.
+// Copyright (c) 2023      Cisco Systems, Inc.  All rights reserved.
+// $COPYRIGHT$
+//
+// Additional copyrights may follow
+//
+// $HEADER$
+//
+// Build an Open MPI Pull Request
+//
+//
+// WORKSPACE Layout:
+//   autotools-install/    Autotools install for the builder
+//   ompi/                 Open MPI source tree
+
+// We if we push changes to a PR, we don't need to keep old jobs running, so
+// we'll use the milestone step in Jenkins. Using an example from
+// https://stackoverflow.com/questions/40760716/jenkins-abort-running-build-if-new-one-is-started:
+//
+// - Build 1 runs and creates milestone 1.
+// - While build 1 is running, build 2 fires. It has milestone 1 and milestone
+//   2. It passes milestone 1, which causes build 1 to abort.
+def buildNumber = env.BUILD_NUMBER as int
+if (buildNumber > 1) {
+    milestone(buildNumber - 1)
+}
+milestone(buildNumber)
+
+// Add build description linking back to PR. This is redundant to the "GitHub"
+// link on the Pull Request page, but the Build page does not have a direct link
+// back to the PR. The "Details" link at the bottom of the GitHub PR page brings
+// you to the Jenkins Build page, so we're adding the link back to the GitHub PR
+// page.
+if (env.CHANGE_URL) {
+    currentBuild.description = "This is a build of <a href=\"${CHANGE_URL}\"}\">Open MPI PR #${CHANGE_ID}</a>"
+} else {
+    currentBuild.description = "Build of ${BRANCH_NAME}"
+}
+
+check_stages = prepare_check_stages()
+println("Initialized Pipeline")
+
+// Today, we only expect to have one stage (do everything), but allow that
+// we may split build and test stages in the future.
+for (check_stage in check_stages) {
+    parallel(check_stage)
+}
+
+println('Tests Completed')
+
+// Returns a list of build stages ("build Open MPI", "Build Tests", etc.),
+// although currently we only support the one stage of "everything", where each
+// build stage is a map of different configurations to test.
+def prepare_check_stages() {
+    def configure_options = ["--disable-dlopen", "--disable-oshmem", "--enable-builtin-atomic", "--enable-ipv6"]
+    def compilers = ["clang10", "gcc7", "gcc8", "gcc9", "gcc10"]
+    def platforms = ["amazon_linux_2", "amazon_linux_2-arm64", "rhel8"]
+    def check_stages_list = []
+
+    // Build everything stage
+    def build_parallel_map = [:]
+    for (platform in platforms) {
+        def name = "Platform: ${platform}".replaceAll("-", "")
+        build_parallel_map.put(name, prepare_build(name, platform, ""))
+    }
+
+    for (compiler in compilers) {
+        def name = "Compiler: ${compiler}".replaceAll("-", "")
+        build_parallel_map.put(name, prepare_build(name, compiler, "--compiler \\\"${compiler}\\\""))
+    }
+
+      for (configure_option in configure_options) {
+        def name = "Configure: ${configure_option}".replaceAll("-", "")
+        build_parallel_map.put(name, prepare_build(name, "amazon_linux_2", "--configure-args \\\"${configure_option}\\\""))
+    }
+
+    build_parallel_map.put("distcheck", prepare_build("distcheck", "tarball_build", "--distcheck"))
+
+    check_stages_list.add(build_parallel_map)
+
+    return check_stages_list
+}
+
+def prepare_build(build_name, label, build_arg) {
+    return {
+        stage("${build_name}") {
+            node(label) {
+                checkout(changelog: false, poll: false, scm: scm)
+                // If pr-builder.sh fails, the sh step will throw an exception,
+                // which we catch so that the job doesn't abort and continues on
+                // to other steps - such as cleanup. Because we catch the
+                // exception, we need to tell Jenkins the overall job has
+                // failed.
+                try {
+                    sh "/bin/bash -x .ci/community-jenkins/pr-builder.sh ${build_arg} ompi"
+                } catch (Exception e) {
+                    currentBuild.result = "FAILURE"
+                }
+                cleanWs(notFailBuild: true)
+            }
+        }
+    }
+}
diff --git a/.ci/community-jenkins/pr-builder.sh b/.ci/community-jenkins/pr-builder.sh
new file mode 100755
index 00000000000..eb88b4c1538
--- /dev/null
+++ b/.ci/community-jenkins/pr-builder.sh
@@ -0,0 +1,352 @@
+#!/bin/sh
+#
+# Copyright (c) 2022-2023 Amazon.com, Inc. or its affiliates.  All rights
+#                         reserved.
+# Copyright (c) 2022-2023 Joe Downs.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+
+# Abort on error
+set -euo pipefail
+
+BUILD_32BIT=0
+COMPILER=
+DISTCHECK=0
+AUTOGEN_ARGS=
+CONFIGURE_ARGS=
+MAKE_ARGS=
+MAKE_J="-j 8"
+PREFIX="${WORKSPACE}/install"
+MPIRUN_MODE=${MPIRUN_MODE:-runall}
+
+#
+# Options Parsing
+#
+# For each option, we need to remove the quotes from their arguments. Without
+# quotes, the command-line options for later commands (such as
+# --disable-oshmem), are interpreted (in the following switch statement) as
+# options for this script.
+
+strip_quotes() {
+    echo `echo "$1" | sed -e "s/\(\"\)\([[:alnum:]|_|-]*\)\(\"\)/\2/"`
+}
+
+PARAMS=""
+while (( "$#" )); do
+    case "$1" in
+        --distcheck)
+            DISTCHECK=1
+            shift
+            ;;
+        --autogen-args)
+            if [ -n "$2" ] && [ ${2:0:1} != "-" ]; then
+                AUTOGEN_ARGS=$(strip_quotes $2)
+                shift 2
+            else
+                echo "Error: Argument for $1 is missing" >&2
+                exit 1
+            fi
+            ;;
+        --configure-args)
+            if [ -n "$2" ] && [ ${2:0:1} != "-" ]; then
+                CONFIGURE_ARGS=$(strip_quotes $2)
+                shift 2
+            else
+                echo "Error: Argument for $1 is missing" >&2
+                exit 1
+            fi
+            ;;
+        --compiler)
+            if [ -n "$2" ] && [ ${2:0:1} != "-" ]; then
+                COMPILER=$(strip_quotes $2)
+                shift 2
+            else
+                echo "Error: Argument for $1 is missing" >&2
+                exit 1
+            fi
+            ;;
+        --mpirun-mode)
+            if [ -n "$2" ] && [ ${2:0:1} != "-" ]; then
+                MPIRUN_MODE=$(strip_quotes $2)
+                shift 2
+            else
+                echo "Error: Argument for $1 is missing" >&2
+                exit 1
+            fi
+            ;;
+        -*|--*=) # Unsupported flags
+            echo "Error: Unsupported flag $1" >&2
+            exit 1
+            ;;
+        *) # Preserve positional arguments
+            PARAMS="$PARAMS $1"
+            shift
+            ;;
+    esac
+done
+# Set positional arguments in their proper place.
+eval set -- "$PARAMS"
+
+#
+# Start by figuring out what we are...
+#
+os=`uname -s`
+if test "${os}" = "Linux"; then
+    eval "PLATFORM_ID=`sed -n 's/^ID=//p' /etc/os-release`"
+    eval "VERSION_ID=`sed -n 's/^VERSION_ID=//p' /etc/os-release`"
+else
+    PLATFORM_ID=`uname -s`
+    VERSION_ID=`uname -r`
+fi
+
+echo "--> platform: $PLATFORM_ID"
+echo "--> version: $VERSION_ID"
+
+#
+# See if builder provided a compiler we should use, and translate it to
+# CONFIGURE_ARGS.
+#
+case ${PLATFORM_ID} in
+    rhel)
+        case "$COMPILER" in
+            gcc48|"")
+                echo "--> Using default compilers"
+                ;;
+            *)
+                echo "Unsupported compiler ${COMPILER}.  Aborting"
+                exit 1
+                ;;
+        esac
+        ;;
+    amzn)
+        case "$COMPILER" in
+            "")
+                echo "--> Using default compilers"
+                ;;
+            gcc44)
+                CONFIGURE_ARGS="$CONFIGURE_ARGS CC=gcc44 CXX=g++44 FC=gfortran44"
+                ;;
+            gcc48)
+                CONFIGURE_ARGS="$CONFIGURE_ARGS CC=gcc48 CXX=g++48 FC=gfortran48"
+                ;;
+            clang36)
+                CONFIGURE_ARGS="$CONFIGURE_ARGS CC=clang CXX=clang++ --disable-mpi-fortran"
+                ;;
+            *)
+                echo "Unsupported compiler ${COMPILER}.  Aborting"
+                exit 1
+                ;;
+        esac
+        ;;
+    ubuntu)
+        case "$COMPILER" in
+            "")
+                echo "--> Using default compilers"
+                ;;
+            gcc4*)
+                version=`echo "$COMPILER" | sed -e 's/gcc4\([0-9]*\)/4.\1/'`
+                CONFIGURE_ARGS="CC=gcc-${version} CXX=g++-${version} FC=gfortran-${version}"
+                ;;
+            gcc*)
+                version=`echo "$COMPILER" | sed -e 's/gcc\([0-9]*\)/\1/'`
+                CONFIGURE_ARGS="CC=gcc-${version} CXX=g++-${version} FC=gfortran-${version}"
+                ;;
+            clang3*|clang4*|clang5*|clang6*)
+                version=`echo "$COMPILER" |  sed -e 's/clang\([0-9]\)\([0-9]*\)/\1.\2/'`
+                CONFIGURE_ARGS="CC=clang-${version} CXX=clang++-${version} --disable-mpi-fortran"
+                ;;
+            clang*)
+                version=`echo "$COMPILER" | sed -e 's/clang\([0-9]*\)/\1/'`
+                CONFIGURE_ARGS="CC=clang-${version} CXX=clang++-${version} --disable-mpi-fortran"
+                ;;
+            *)
+                echo "Unsupported compiler ${COMPILER}.  Aborting"
+                exit 1
+                ;;
+        esac
+        ;;
+    sles)
+        case "$COMPILER" in
+            "")
+                echo "--> Using default compilers"
+                ;;
+            gcc48)
+                CONFIGURE_ARGS="$CONFIGURE_ARGS CC=gcc-48 CXX=g++-48 FC=gfortran-48"
+                ;;
+            gcc5)
+                CONFIGURE_ARGS="$CONFIGURE_ARGS CC=gcc-5 CXX=g++-5 FC=gfortran-5"
+                ;;
+            gcc6)
+                CONFIGURE_ARGS="$CONFIGURE_ARGS CC=gcc-6 CXX=g++-6 FC=gfortran-6"
+                ;;
+            *)
+                echo "Unsupported compiler ${COMPILER}.  Aborting"
+                exit 1
+                ;;
+        esac
+        ;;
+    FreeBSD)
+        CONFIGURE_ARGS="$CONFIGURE_ARGS LDFLAGS=-Wl,-rpath,/usr/local/lib/gcc5 --with-wrapper-ldflags=-Wl,-rpath,/usr/local/lib/gcc5"
+        ;;
+esac
+CONFIGURE_ARGS="$CONFIGURE_ARGS --disable-silent-rules"
+
+echo "--> Compiler setup: $CONFIGURE_ARGS"
+
+#
+# Add any Autogen or Configure arguments provided by the builder job.
+#
+if test "$AUTOGEN_ARGS" != ""; then
+    # Special case, to work around the fact that Open MPI can't build when
+    # there's a space in the build path name. (sigh)
+    if test "$AUTOGEN_ARGS" = "--no-orte"; then
+        AUTOGEN_ARGS="--no-orte --no-ompi"
+    fi
+fi
+
+echo "--> Autogen arguments: $AUTOGEN_ARGS"
+echo "--> Configure arguments: $CONFIGURE_ARGS"
+
+# Build
+sha1=`git rev-parse HEAD`
+echo "--> Building commit ${sha1}"
+
+if test -f autogen.pl; then
+    echo "--> running ./autogen.pl ${AUTOGEN_ARGS}"
+    ./autogen.pl ${AUTOGEN_ARGS}
+else
+    if test "${AUTOGEN_ARGS}" != ""; then
+        echo "--> Being a coward and not running with special autogen arguments and autogen.sh"
+        exit 1
+    else
+        echo "--> running ./atogen.sh"
+        ./autogen.sh
+    fi
+fi
+
+echo "--> running ./configure --prefix=\"${PREFIX}\" ${CONFIGURE_ARGS}"
+if ! ./configure --prefix="${PREFIX}" ${CONFIGURE_ARGS}; then
+    echo "./configure --prefix=\"${PREFIX}\" ${CONFIGURE_ARGS} failed, ABORTING !"
+    if test -f config.log; then
+        echo "config.log content :"
+        cat config.log
+    else
+        echo "no config.log was generated"
+    fi
+    exit 1
+fi
+
+# Shortcut for the distcheck case, as it won't run any tests beyond the built-in
+# make check tests. We need to install the requirements (Sphinx) so we can build
+# the docs.
+if test "${DISTCHECK}" = "1"; then
+    echo "--> running make ${MAKE_ARGS} distcheck"
+    make ${MAKE_ARGS} distcheck
+    exit $?
+fi
+
+echo "--> running make ${MAKE_J} ${MAKE_ARGS} all"
+make ${MAKE_J} ${MAKE_ARGS} all
+echo "--> running make check"
+make ${MAKE_ARGS} check
+echo "--> running make install"
+make ${MAKE_ARGS} install
+
+export PATH="${PREFIX}/bin":${PATH}
+
+case "$AUTOGEN_ARGS" in
+    *--no-ompi*)
+        echo "--> Skipping MPI tests due to --no-ompi"
+        exit 0
+        ;;
+esac
+
+echo "--> running ompi_info"
+ompi_info
+
+echo "--> running make all in examples"
+cd "examples"
+make ${MAKE_ARGS} all
+cd ..
+
+# It's hard to determine what the failure was and there's no printing of error
+# code with set -e, so for the tests, we do per-command checking...
+set +e
+
+run_example() {
+    example=`basename ${2}`
+    echo "--> Running example: $example"
+    ${1} ${2}
+    ret=$?
+    if test ${ret} -ne 0 ; then
+        echo "Example failed: ${ret}"
+        echo "Command was: ${1} ${2}"
+        exit ${ret}
+    fi
+}
+
+if test "${MPIRUN_MODE}" != "none"; then
+    echo "--> running examples"
+    echo "localhost cpu=2" > "${WORKSPACE}/hostfile"
+    # Note: using perl here because figuring out a portable sed regexp
+    # proved to be a little challenging.
+    mpirun_version=`"${WORKSPACE}/install/bin/mpirun" --version | perl -wnE 'say $1 if /mpirun [^\d]*(\d+.\d+)/'`
+    echo "--> mpirun version: ${mpirun_version}"
+    case ${mpirun_version} in
+        1.*|2.0*)
+            exec="timeout -s SIGSEGV 3m mpirun -hostfile ${WORKSPACE}/hostfile -np 2 "
+            ;;
+        *)
+            exec="timeout -s SIGSEGV 4m mpirun --get-stack-traces --timeout 180 --hostfile ${WORKSPACE}/hostfile -np 2 --bind-to none "
+            ;;
+    esac
+    singleton="timeout -s SIGSEGV 1m "
+    run_example "${exec}" ./examples/hello_c
+    run_example "${singleton}" ./examples/hello_c
+    run_example "${exec}" ./examples/ring_c
+    run_example "${singleton}" ./examples/ring_c
+    run_example "${exec}" ./examples/connectivity_c
+    if ompi_info --parsable | grep -q bindings:cxx:yes >/dev/null; then
+        echo "--> running C++ examples"
+        run_example "${exec}" ./examples/hello_cxx
+        run_example "${singleton}" ./examples/hello_cxx
+        run_example "${exec}" ./examples/ring_cxx
+        run_example "${singleton}" ./examples/ring_cxx
+    else
+        echo "--> skipping C++ examples"
+    fi
+    if ompi_info --parsable | grep -q bindings:mpif.h:yes >/dev/null; then
+        echo "--> running mpif examples"
+        run_example "${exec}" ./examples/hello_mpifh
+        run_example "${singleton}" ./examples/hello_mpifh
+        run_example "${exec}" ./examples/ring_mpifh
+        run_example "${singleton}" ./examples/ring_mpifh
+    else
+        echo "--> skipping mpif examples"
+    fi
+    if ompi_info --parsable | egrep -q bindings:use_mpi:\"\?yes >/dev/null; then
+        echo "--> running usempi examples"
+        run_example "${exec}" ./examples/hello_usempi
+        run_example "${singleton}" ./examples/hello_usempi
+        run_example "${exec}" ./examples/ring_usempi
+        run_example "${singleton}" ./examples/ring_usempi
+    else
+        echo "--> skipping usempi examples"
+    fi
+    if ompi_info --parsable | grep -q bindings:use_mpi_f08:yes >/dev/null; then
+        echo "--> running usempif08 examples"
+        run_example "${exec}" ./examples/hello_usempif08
+        run_example "${singleton}" ./examples/hello_usempif08
+        run_example "${exec}" ./examples/ring_usempif08
+        run_example "${singleton}" ./examples/ring_usempif08
+    else
+        echo "--> skipping usempif08 examples"
+    fi
+else
+    echo "--> Skipping examples (MPIRUN_MODE = none)"
+fi
+
+echo "--> All done!"
diff --git a/.ci/lanl/gitlab-darwin-ci.yml b/.ci/lanl/gitlab-darwin-ci.yml
new file mode 100644
index 00000000000..decc4ddd0e3
--- /dev/null
+++ b/.ci/lanl/gitlab-darwin-ci.yml
@@ -0,0 +1,182 @@
+variables:
+  SCHEDULER_PARAMETERS: "-pgeneral -t 4:00:00 -N 1 --ntasks-per-node=16"
+  GIT_STRATEGY: clone
+  NPROCS: 4
+
+stages:
+  - build
+  - test
+
+build:intel:
+  id_tokens:
+    SITE_ID_TOKEN:
+      aud: https://re-git.lanl.gov
+  stage: build
+  tags: [darwin-slurm-shared]
+  script:
+    - module load intel/2022.0.1
+    - rm .gitmodules
+    - cp $GITSUBMODULEPATCH .gitmodules
+    - git submodule update --init --recursive
+    - ./autogen.pl
+    - ./configure CC=icx FC=ifx CXX=icpx --prefix=$PWD/install_test --with-libevent=internal
+    - make -j 8 install
+    - make check
+    - export PATH=$PWD/install_test/bin:$PATH
+    - cd examples
+    - make
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    untracked: true
+    paths:
+      - examples
+      - install_test
+    expire_in: 1 week
+
+build:amd:
+  id_tokens:
+    SITE_ID_TOKEN:
+      aud: https://re-git.lanl.gov
+  stage: build
+  tags: [darwin-slurm-shared]
+  variables:
+    SCHEDULER_PARAMETERS: "-pamd-rome -t 4:00:00 -N 1 --ntasks-per-node=16"
+  script:
+    - module load aocc/3.0.0
+    - rm .gitmodules
+    - cp $GITSUBMODULEPATCH .gitmodules
+    - git submodule update --init --recursive
+    - ./autogen.pl
+    - ./configure CC=clang FC=flang CXX=clang++ --prefix=$PWD/install_test --with-libevent=internal LIBS="-lucm -lucs"
+    - make -j 8 install
+    - make check
+    - export PATH=$PWD/install_test/bin:$PATH
+    - cd examples
+    - make
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    untracked: true
+    paths:
+      - examples
+      - install_test
+    expire_in: 1 week
+
+build:gnu:
+  id_tokens:
+    SITE_ID_TOKEN:
+      aud: https://re-git.lanl.gov
+  stage: build
+  tags: [darwin-slurm-shared]
+  script:
+    - module load gcc
+    - rm .gitmodules
+    - cp $GITSUBMODULEPATCH .gitmodules
+    - git submodule update --init --recursive
+    - ./autogen.pl
+    - ./configure --prefix=$PWD/install_test --with-libevent=internal
+    - make -j 8 install
+    - make check
+    - export PATH=$PWD/install_test/bin:$PATH
+    - cd examples
+    - make
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    untracked: true
+    paths:
+      - examples
+      - install_test
+    expire_in: 1 week
+
+test:intel:
+  id_tokens:
+    SITE_ID_TOKEN:
+      aud: https://re-git.lanl.gov
+  stage: test
+  tags: [darwin-slurm-shared]
+  dependencies:
+    - build:intel
+  needs: ["build:intel"]
+  script:
+    - pwd
+    - ls
+    - module load intel/2022.0.1
+    - export PATH=$PWD/install_test/bin:$PATH
+    - which mpirun
+    - cd examples
+    - mpirun -np 4 hostname
+    - mpirun -np 4 ./hello_c
+    - mpirun -np 4 ./ring_c
+    - mpirun -np 4 ./hello_mpifh
+    - mpirun -np 4 ./ring_mpifh
+    - mpirun -np 4 ./hello_usempi
+    - mpirun -np 4 ./ring_usempi
+    - mpirun -np 4 ./hello_usempif08
+    - mpirun -np 4 ./ring_usempif08
+    - mpirun -np 4 ./connectivity_c
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    expire_in: 1 week
+
+test:amd:
+  id_tokens:
+    SITE_ID_TOKEN:
+      aud: https://re-git.lanl.gov
+  stage: test
+  tags: [darwin-slurm-shared]
+  variables:
+    SCHEDULER_PARAMETERS: "-pamd-rome -t 2:00:00 -N 1 --ntasks-per-node=16"
+  dependencies:
+    - build:amd
+  needs: ["build:amd"]
+  script:
+    - pwd
+    - ls
+    - module load aocc/3.0.0
+    - export PATH=$PWD/install_test/bin:$PATH
+    - export LD_LIBRARY_PATH=$PWD/install_test/lib:$LD_LIBRARY_PATH
+    - which mpirun
+    - cd examples
+    - mpirun -np 4 hostname
+    - mpirun -np 4 ./hello_c
+    - mpirun -np 4 ./ring_c
+    - mpirun -np 4 ./hello_mpifh
+    - mpirun -np 4 ./ring_mpifh
+    - mpirun -np 4 ./hello_usempi
+    - mpirun -np 4 ./ring_usempi
+    - mpirun -np 4 ./hello_usempif08
+    - mpirun -np 4 ./ring_usempif08
+    - mpirun -np 4 ./connectivity_c
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    expire_in: 1 week
+
+test:gnu:
+  id_tokens:
+    SITE_ID_TOKEN:
+      aud: https://re-git.lanl.gov
+  stage: test
+  tags: [darwin-slurm-shared]
+  dependencies:
+    - build:gnu
+  needs: ["build:gnu"]
+  script:
+    - pwd
+    - ls
+    - module load gcc
+    - export PATH=$PWD/install_test/bin:$PATH
+    - which mpirun
+    - cd examples
+    - mpirun -np 4 hostname
+    - mpirun -np 4 ./hello_c
+    - mpirun -np 4 ./ring_c
+    - mpirun -np 4 ./hello_mpifh
+    - mpirun -np 4 ./ring_mpifh
+    - mpirun -np 4 ./hello_usempi
+    - mpirun -np 4 ./ring_usempi
+    - mpirun -np 4 ./hello_usempif08
+    - mpirun -np 4 ./ring_usempif08
+    - mpirun -np 4 ./connectivity_c
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    expire_in: 1 week
+
diff --git a/.ci/mellanox/README.md b/.ci/mellanox/README.md
new file mode 100644
index 00000000000..2a9d5c09b15
--- /dev/null
+++ b/.ci/mellanox/README.md
@@ -0,0 +1,16 @@
+# Open MPI Continuous Integration (CI) Services
+## Mellanox Open MPI CI
+### Scope
+[Mellanox](https://www.mellanox.com/) Open MPI CI is intended to verify Open MPI with recent Mellanox SW components ([Mellanox OFED](https://www.mellanox.com/page/products_dyn?product_family=26), [UCX](https://www.mellanox.com/page/products_dyn?product_family=281&mtag=ucx) and other [HPC-X](https://www.mellanox.com/page/products_dyn?product_family=189&mtag=hpc-x) components) in the Mellanox lab environment.
+
+CI is managed by [Azure Pipelines](https://docs.microsoft.com/en-us/azure/devops/pipelines/?view=azure-devops) service.
+
+Mellanox Open MPI CI includes:
+* Open MPI building with internal stable engineering versions of UCX and HCOLL. The building is run in Docker-based environment.
+* Sanity functional testing.
+### How to Run CI
+Mellanox Open MPI CI is triggered upon the following events:
+* Create a pull request (PR). CI status is visible in the PR status. CI is restarted automatically upon each new commit within the PR. CI status and log files are also available on the Azure DevOps server.
+* Trigger CI with special PR comments (for example, `/azp run`). Comment triggers are available only if the comment author has write permission to the PR target repo. Detailed information about comment triggers is available in the official Azure DevOps [documentation](https://docs.microsoft.com/en-us/azure/devops/pipelines/repos/github?view=azure-devops&tabs=yaml#comment-triggers).
+### Support
+In case of any issues, questions or suggestions please contact to [Mellanox Open MPI CI support team](mailto:swx-azure-svc@mellanox.com).
diff --git a/.ci/mellanox/azure-pipelines.yml b/.ci/mellanox/azure-pipelines.yml
index 187bcfb7752..c676cf7d1bc 100644
--- a/.ci/mellanox/azure-pipelines.yml
+++ b/.ci/mellanox/azure-pipelines.yml
@@ -1,6 +1,6 @@
 trigger: none
 pr:
-  - master
+  - main
   - v*.*.x
 
 pool:
@@ -10,7 +10,7 @@ pool:
   - MLNX_IB_DEVICE -equals yes
 
 variables:
-  ompi_jenkins_scripts_git_repo_url: https://github.com/mellanox-hpc/jenkins_scripts.git
+  ompi_jenkins_scripts_git_repo_url: https://github.com/Mellanox/jenkins_scripts.git
   ompi_jenkins_scripts_git_branch: master
   # Enable debug information, supported values: true, false
   debug: true
@@ -25,7 +25,7 @@ jobs:
       --ulimit memlock=-1 --security-opt seccomp=unconfined --cap-add=SYS_ADMIN --device=/dev/infiniband/
   steps:
   - checkout: self
-    submodules: true
+    submodules: recursive
     path: ompi
     clean: true
   - bash: |
diff --git a/.clang-format b/.clang-format
new file mode 100644
index 00000000000..d4249d80c2d
--- /dev/null
+++ b/.clang-format
@@ -0,0 +1,173 @@
+# This file represents the coding style enforced by Open MPI. This file
+# is based on the long-held, but not enforced, guidelines from the
+# beginning of the project. We will be requiring that all code going
+# forward uses this style. To check your code before attempting to open
+# a PR install clang-format and run your commits through clang-format.
+#
+# To install clang-format:
+#
+# macOS:
+#   Homebrew: brew install clang-format
+#   Mac Ports: port install clang
+#
+# Linux:
+#   debian/ubuntu/rasbian: apt-get install clang-format
+#   redhat/fedora: yum install clang-format
+#
+# To run against your code changes:
+#
+#   unstaged changes: git clang-format --style file -f
+#   staged changes: git clang-format --style file
+#
+#   For interactive add the -p option.
+#
+# To run against all of Open MPI:
+#
+#   ./contrib/clang-format-ompi.sh
+#
+#   This command is intended to be run only once.
+---
+Language:        Cpp
+# BasedOnStyle:  LLVM
+AccessModifierOffset: -2
+AlignAfterOpenBracket: Align
+AlignConsecutiveMacros: true
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlines: Left
+AlignOperands: true
+AlignTrailingComments: true
+AllowAllArgumentsOnNextLine: false
+AllowAllConstructorInitializersOnNextLine: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortBlocksOnASingleLine: Never
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: None
+AllowShortLambdasOnASingleLine: All
+AllowShortIfStatementsOnASingleLine: Never
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: false
+AlwaysBreakTemplateDeclarations: MultiLine
+BinPackArguments: true
+BinPackParameters: true
+BraceWrapping:
+  AfterCaseLabel:  false
+  AfterClass:      false
+  AfterControlStatement: Never
+  AfterEnum:       false
+  AfterFunction:   true
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  AfterExternBlock: false
+  BeforeCatch:     false
+  BeforeElse:      false
+  IndentBraces:    false
+  SplitEmptyFunction: true
+  SplitEmptyRecord: true
+  SplitEmptyNamespace: true
+BreakBeforeBinaryOperators: true
+BreakBeforeBraces: Custom
+BreakBeforeInheritanceComma: false
+BreakInheritanceList: BeforeColon
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakConstructorInitializers: BeforeColon
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+ColumnLimit:     100
+CommentPragmas:  '^ IWYU pragma:'
+CompactNamespaces: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: false
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DeriveLineEnding: true
+DerivePointerAlignment: false
+DisableFormat:   false
+FixNamespaceComments: true
+ForEachMacros:
+  - foreach
+  - Q_FOREACH
+  - BOOST_FOREACH
+  - BOOST_FOREACH
+  - OPAL_LIST_FOREACH
+  - OPAL_LIST_FOREACH_DECL
+  - OPAL_LIST_FOREACH_SAFE
+  - OPAL_LIST_FOREACH_REV
+  - OPAL_LIST_FOREACH_SAFE_REV
+  - OPAL_HASH_TABLE_FOREACH
+  - OPAL_HASH_TABLE_FOREACH_PTR
+IncludeBlocks:   Preserve
+IncludeCategories:
+  # Ensure config includes always come first (opal_config.h, ompi_config.h, etc)
+  - Regex:           '^".*_config\.h"'
+    Priority:        -1
+  # In-tree includes come next (after main include)
+  - Regex:           '^".*"'
+    Priority:        2
+  # System includes come last
+  - Regex:           '^<.*>'
+    Priority:        3
+IncludeIsMainRegex: '(Test)?$'
+IncludeIsMainSourceRegex: ''
+IndentCaseLabels: false
+IndentGotoLabels: true
+IndentPPDirectives: AfterHash
+IndentWidth:     4
+IndentWrappedFunctionNames: false
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: true
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBinPackProtocolList: Auto
+ObjCBlockIndentWidth: 4
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: true
+PenaltyBreakAssignment: 250
+PenaltyBreakBeforeFirstCallParameter: 301
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyBreakTemplateDeclaration: 10
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 60
+PointerAlignment: Right
+ReflowComments:  true
+SortIncludes:    true
+SortUsingDeclarations: true
+SpaceAfterCStyleCast: true
+SpaceAfterLogicalNot: false
+SpaceAfterTemplateKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeCpp11BracedList: false
+SpaceBeforeCtorInitializerColon: true
+SpaceBeforeInheritanceColon: true
+SpaceBeforeParens: ControlStatements
+SpaceBeforeRangeBasedForLoopColon: true
+SpaceInEmptyBlock: false
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 1
+SpacesInAngles:  false
+SpacesInConditionalStatement: false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+SpaceBeforeSquareBrackets: false
+Standard:        Latest
+StatementMacros:
+  - Q_UNUSED
+  - QT_REQUIRE_VERSION
+  - BEGIN_C_DECLS
+  - END_C_DECLS
+TabWidth:        8
+UseCRLF:         false
+UseTab:          Never
+...
diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md
index 69945cf0b6d..0d17ee08ec1 100644
--- a/.github/CONTRIBUTING.md
+++ b/.github/CONTRIBUTING.md
@@ -5,85 +5,7 @@ Open MPI!
 
 ![You're awesome!](https://www.open-mpi.org/images/youre-awesome.jpg)
 
-General information about contributing to the Open MPI project can be found at the [Contributing to Open MPI webpage](https://www.open-mpi.org/community/contribute/).
-The instructions below are specifically for opening issues and pull requests against Open MPI.
-
-## Content
-
-We love getting contributions from anyone.  But keep in mind that Open
-MPI is used in production environments all around the world.
-
-If you're contributing a small bug fix, awesome!
-
-If you're contributing a large new piece of functionality, that will
-be best viewed if you — or someone, anyone — is also stepping up to
-help maintain that functionality over time.  We love new ideas and new
-features, but we do need to be realistic in what we can reliably test
-and deliver to our users.
-
-## Contributor's Declaration
-
-In order to ensure that we can keep distributing Open MPI under our
-[open source license](/LICENSE), we need to ensure that all
-contributions are compatible with that license.
-
-To that end, we require that all Git commits contributed to Open MPI
-have a "Signed-off-by" token indicating that the commit author agrees
-with [Open MPI's Contributor's
-Declaration](https://github.com/open-mpi/ompi/wiki/Administrative-rules#contributors-declaration).
-
-If you have not already done so, please ensure that:
-
-1. Every commit contains exactly the "Signed-off-by" token.  You can
-add this token via `git commit -s`.
-1. The email address after "Signed-off-by" must match the Git commit
-email address.
-
-## **Did you find a bug?**
-
-* **Ensure the bug was not already reported** by searching on GitHub under [Issues](https://github.com/open-mpi/ompi/issues).
-
-* If you're unable to find an open issue addressing the problem, [open a new one](https://github.com/open-mpi/ompi/issues/new).
-
-* For more detailed information on submitting a bug report and creating an issue, visit our [Bug Tracking webpage](https://www.open-mpi.org/community/help/bugs.php).
-
-## **Did you write a patch that fixes a bug?**
-
-* Open a new GitHub pull request with the patch.
-
-* Ensure the PR description clearly describes the problem and solution. If there is an existing GitHub issue open describing this bug, please include it in the description so we can close it.
-
-* Before submitting, please read the [Contributing to the Open MPI Project FAQ](https://www.open-mpi.org/faq/?category=contributing) web page, and the [SubmittingPullRequests](https://github.com/open-mpi/ompi/wiki/SubmittingPullRequests) wiki page.  In particular, note that all Git commits contributed to Open MPI require a "Signed-off-by" line.
-
-## **Do you intend to add a new feature or change an existing one?**
-
-* Suggest your change on the [devel mail list](https://www.open-mpi.org/community/lists/ompi.php) and start writing code.  The [developer level technical information on the internals of Open MPI](https://www.open-mpi.org/faq/?category=developers) may also be useful for large scale features.
-
-* Do not open an issue on GitHub until you have collected positive feedback about the change. GitHub issues are primarily intended for bug reports and fixes.
-
-## **Do you have questions about the source code?**
-
-* First checkout the [developer level technical information on the internals of Open MPI](https://www.open-mpi.org/faq/?category=developers).  A paper describing the [multi-component architecture](https://www.open-mpi.org/papers/ics-2004/ics-2004.pdf)  of Open MPI may also be helpful.  The [devel mail list](https://www.open-mpi.org/community/lists/ompi.php) is a good place to post questions about the source code as well.
-
-## Style
-
-There are a small number of style rules for Open MPI:
-
-1. For all code:
-    * 4 space tabs.  No more, no less.
-    * No tab characters *at all*.  2 indentations are 8 spaces — not a tab.
-    * m4 code is a bit weird in terms of indentation: we don't have a
-      good, consistent indentation style in our existing code.  But
-      still: no tab characters at all.
-1. For C code:
-    * We prefer if all blocks are enclosed in `{}` (even 1-line
-      blocks).
-    * We prefer that if you are testing equality with a constant, put
-      the constant on the *left* of the `==`.  E.g., `if (NULL ==
-      ptr)`.
-    * If there are no parameters to a C function, declare it with
-      `(void)` (vs. `()`).
-
-That's about it.  Thank you!
-
-— The Open MPI Team
+Open MPI is hosted on GitHub, and we gladly accept pull requests.
+Please see the [Contributing
+guidelines](https://docs.open-mpi.org/en/main/contributing.html) for
+details on how to contribute to Open MPI.
diff --git a/.github/ISSUE_TEMPLATE/ask-a-question.md b/.github/ISSUE_TEMPLATE/ask-a-question.md
new file mode 100644
index 00000000000..21583e901d7
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/ask-a-question.md
@@ -0,0 +1,45 @@
+---
+name: Ask a question
+about: Got a problem or question?  This is the place to ask.
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+Please submit all the information below so that we can understand the working environment that is the context for your question.
+
+* If you have a problem building or installing Open MPI, [be sure to read this](https://docs.open-mpi.org/en/main/getting-help.html#for-problems-building-or-installing-open-mpi).
+* If you have a problem launching MPI or OpenSHMEM applications, [be sure to read this](https://docs.open-mpi.org/en/main/getting-help.html#for-problems-launching-mpi-or-openshmem-applications).
+* If you have a problem running MPI or OpenSHMEM applications (i.e., after launching them), [be sure to read this](https://docs.open-mpi.org/en/main/getting-help.html#for-problems-running-mpi-or-openshmem-applications).
+
+## Background information
+
+### What version of Open MPI are you using? (e.g., v4.1.6, v5.0.1, git branch name and hash, etc.)
+
+
+
+### Describe how Open MPI was installed (e.g., from a source/distribution tarball, from a git clone, from an operating system distribution package, etc.)
+
+
+
+### If you are building/installing from a git clone, please copy-n-paste the output from `git submodule status`.
+
+
+
+### Please describe the system on which you are running
+
+* Operating system/version:
+* Computer hardware:
+* Network type:
+
+-----------------------------
+
+## Details of the problem
+
+Please describe, in detail, the problem that you are having, including the behavior you expect to see, the actual behavior that you are seeing, steps to reproduce the problem, etc.  It is most helpful if you can attach a small program that a developer can use to reproduce your problem.
+
+**Note**: If you include verbatim output (or a code block), please use a [GitHub Markdown](https://help.github.com/articles/creating-and-highlighting-code-blocks/) code block like below:
+```shell
+shell$ mpirun -n 2 ./hello_world
+```
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
new file mode 100644
index 00000000000..e1c899554ad
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -0,0 +1,41 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+Thank you for taking the time to submit an issue!
+
+## Background information
+
+### What version of Open MPI are you using? (e.g., v4.1.6, v5.0.1, git branch name and hash, etc.)
+
+
+
+### Describe how Open MPI was installed (e.g., from a source/distribution tarball, from a git clone, from an operating system distribution package, etc.)
+
+
+
+### If you are building/installing from a git clone, please copy-n-paste the output from `git submodule status`.
+
+
+
+### Please describe the system on which you are running
+
+* Operating system/version:
+* Computer hardware:
+* Network type:
+
+-----------------------------
+
+## Details of the problem
+
+Please describe, in detail, the problem that you are having, including the behavior you expect to see, the actual behavior that you are seeing, steps to reproduce the problem, etc.  It is most helpful if you can attach a small program that a developer can use to reproduce your problem.
+
+**Note**: If you include verbatim output (or a code block), please use a [GitHub Markdown](https://help.github.com/articles/creating-and-highlighting-code-blocks/) code block like below:
+```shell
+shell$ mpirun -n 2 ./hello_world
+```
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
new file mode 100644
index 00000000000..bbcbbe7d615
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.
diff --git a/.github/issue_template.md b/.github/issue_template.md
deleted file mode 100644
index 4e2101b5751..00000000000
--- a/.github/issue_template.md
+++ /dev/null
@@ -1,33 +0,0 @@
-Thank you for taking the time to submit an issue!
-
-## Background information
-
-### What version of Open MPI are you using? (e.g., v3.0.5, v4.0.2, git branch name and hash, etc.)
-
-
-
-### Describe how Open MPI was installed (e.g., from a source/distribution tarball, from a git clone, from an operating system distribution package, etc.)
-
-
-
-### If you are building/installing from a git clone, please copy-n-paste the output from `git submodule status`.
-
-
-
-### Please describe the system on which you are running
-
-* Operating system/version: 
-* Computer hardware: 
-* Network type: 
-
------------------------------
-
-## Details of the problem
-
-Please describe, in detail, the problem that you are having, including the behavior you expect to see, the actual behavior that you are seeing, steps to reproduce the problem, etc.  It is most helpful if you can attach a small program that a developer can use to reproduce your problem.
-
-**Note**: If you include verbatim output (or a code block), please use a [GitHub Markdown](https://help.github.com/articles/creating-and-highlighting-code-blocks/) code block like below:
-```shell
-shell$ mpirun -np 2 ./hello_world
-```
-
diff --git a/.github/workflows/close-stale-issues.yaml b/.github/workflows/close-stale-issues.yaml
new file mode 100644
index 00000000000..e576df6ed9f
--- /dev/null
+++ b/.github/workflows/close-stale-issues.yaml
@@ -0,0 +1,65 @@
+# The idea behind this Action is to prevent the situation where a user
+# files a Github Issue, someone asks for clarification / more
+# information, but the original poster never provides the information.
+# The issue then becomes forgotten and abondoned.
+#
+# Instead of that scenario, Open MPI community members can assign a
+# label to Github Issues indicating that we're waiting for the user to
+# reply.  If too much time elapses with no reply, mark the Issue as
+# stale and emit a warning that we'll close the issue if we continue
+# to receive no reply.  If we timeout again with no reply after the
+# warning, close the Issue and emit a comment explaining why.
+#
+# If the user *does* reply, the label is removed, and this bot won't
+# touch the Issue.  Specifically: this bot will never mark stale /
+# close an Issue that doesn't have the specified label.
+#
+# Additionally, we are *only* marking stale / auto-closing Github
+# Issues -- not Pull Requests.
+#
+# This is a cron-based Action that runs a few times a day, just so
+# that we don't mark stale / close a bunch of issues all at once.
+#
+# While the actions/stale bot Action used here is capable of removing
+# the label when a user replies to the Issue, we actually use a 2nd
+# Action (removing-awaiting-user-info-label.yaml) to remove the label.
+# We do this because that 2nd Action runs whenever a comment is
+# created -- not via cron.  Hence, the 2nd Action will remove the
+# label effectively immediately when the user replies (vs. up to
+# several hours later).
+
+name: Close stale issues
+on:
+  schedule:
+    # Run it a few times a day so as not to necessarily mark stale /
+    # close a bunch of issues at once.
+    - cron: '0 1,5,9,13,17,21 * * *'
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      # https://github.com/marketplace/actions/close-stale-issues
+      - uses: actions/stale@v9
+        with:
+          # If there are no replies for 14 days, mark the issue as
+          # "stale" (and emit a warning).
+          days-before-stale: 14
+          # If there are no replies for 14 days after becoming stale,
+          # then close the issue (and emit a message explaining why).
+          days-before-close: 14
+
+          # Never close PRs
+          days-before-pr-close: -1
+
+          # We only close issues with this label
+          only-labels: State-Awaiting user information
+          close-issue-label: Closed due to no reply
+
+          # Messages that we put in comments on issues
+          stale-issue-message: |
+            It looks like this issue is expecting a response, but hasn't gotten one yet.  If there are no responses in the next 2 weeks, we'll assume that the issue has been abandoned and will close it.
+          close-issue-message: |
+            Per the above comment, it has been a month with no reply on this issue.  It looks like this issue has been abandoned.
+
+            I'm going to close this issue.  If I'm wrong and this issue is *not* abandoned, please feel free to re-open it.  Thank you!
diff --git a/.github/workflows/compile-cuda.yaml b/.github/workflows/compile-cuda.yaml
new file mode 100644
index 00000000000..97d89372ce0
--- /dev/null
+++ b/.github/workflows/compile-cuda.yaml
@@ -0,0 +1,28 @@
+name: CUDA
+
+on: [pull_request]
+
+env:
+  CUDA_PATH: /usr/local/cuda
+jobs:
+  compile-cuda:
+    runs-on: ubuntu-22.04
+    steps:
+    - name: Install dependencies
+      run: |
+        sudo apt update
+        sudo apt install -y --no-install-recommends wget
+    - name: Install extra dependencies
+      run: |
+        wget https://developer.download.nvidia.com/compute/cuda/repos/wsl-ubuntu/x86_64/cuda-keyring_1.1-1_all.deb
+        sudo dpkg -i cuda-keyring_1.1-1_all.deb
+        sudo apt update
+        sudo apt install -y cuda-toolkit
+    - uses: actions/checkout@v4
+      with:
+            submodules: recursive
+    - name: Build Open MPI
+      run: |
+        ./autogen.pl
+        ./configure --prefix=${PWD}/install --with-cuda=${CUDA_PATH} --with-cuda-libdir=${CUDA_PATH}/lib64/stubs --disable-silent-rules
+        make -j
diff --git a/.github/workflows/compile-rocm.yaml b/.github/workflows/compile-rocm.yaml
new file mode 100644
index 00000000000..22af463a164
--- /dev/null
+++ b/.github/workflows/compile-rocm.yaml
@@ -0,0 +1,31 @@
+name: ROCM
+
+on: [pull_request]
+
+env:
+  ROCM_VER: 6.2.2
+jobs:
+  compile-rocm:
+    runs-on: ubuntu-22.04
+    steps:
+    - name: Install dependencies
+      run: |
+        sudo apt update
+        sudo apt install -y --no-install-recommends wget lsb-core software-properties-common gpg curl
+    - name: Install extra dependencies
+      run: |
+        sudo mkdir --parents --mode=0755 /etc/apt/keyrings
+        wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
+        echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/amdgpu/${ROCM_VER}/ubuntu jammy main" | sudo tee /etc/apt/sources.list.d/amdgpu.list
+        echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/${ROCM_VER} jammy main" | sudo tee --append /etc/apt/sources.list.d/rocm.list
+        echo -e 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' | sudo tee /etc/apt/preferences.d/rocm-pin-600
+        sudo apt update
+        sudo apt install -y rocm-hip-runtime hip-dev
+    - uses: actions/checkout@v4
+      with:
+            submodules: recursive
+    - name: Build Open MPI
+      run: |
+        ./autogen.pl
+        ./configure --prefix=${PWD}/install --with-rocm=/opt/rocm --disable-mpi-fortran --disable-silent-rules
+        LD_LIBRARY_PATH=/opt/rocm/lib make -j
diff --git a/.github/workflows/compile-ze.yaml b/.github/workflows/compile-ze.yaml
new file mode 100644
index 00000000000..cd18cb4719a
--- /dev/null
+++ b/.github/workflows/compile-ze.yaml
@@ -0,0 +1,28 @@
+name: OneAPI ZE
+
+on: [pull_request]
+
+jobs:
+  compile-ze:
+    runs-on: ubuntu-22.04
+    steps:
+    - name: Install dependencies
+      run: |
+        sudo apt update
+        sudo apt install -y --no-install-recommends wget lsb-core software-properties-common gpg curl cmake git
+    - name: Build OneAPI ZE  
+      run: |
+        git clone https://github.com/oneapi-src/level-zero.git
+        cd level-zero
+        mkdir build
+        cd build
+        cmake ../ -DCMAKE_INSTALL_PREFIX=/opt/ze
+        sudo make -j install
+    - uses: actions/checkout@v4
+      with:
+            submodules: recursive
+    - name: Build Open MPI
+      run: |
+        ./autogen.pl
+        ./configure --prefix=${PWD}/install --disable-mpi-fortran  --disable-oshmem --with-ze --disable-silent-rules
+        make -j
diff --git a/.github/workflows/git-commit-checks.json b/.github/workflows/git-commit-checks.json
deleted file mode 100644
index e587b11fbd4..00000000000
--- a/.github/workflows/git-commit-checks.json
+++ /dev/null
@@ -1,3 +0,0 @@
-{
-    "cherry pick required" : 0
-}
diff --git a/.github/workflows/git-commit-checks.py b/.github/workflows/git-commit-checks.py
deleted file mode 100755
index d6ebfbfd1d2..00000000000
--- a/.github/workflows/git-commit-checks.py
+++ /dev/null
@@ -1,336 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-
-Sanity tests on git commits in a Github Pull Request.
-
-This script is designed to run as a Github Action.  It assumes environment
-variables that are available in the Github Action environment.  Specifically:
-
-* GITHUB_WORKSPACE: directory where the git clone is located
-* GITHUB_SHA: the git commit SHA of the artificial Github PR test merge commit
-* GITHUB_BASE_REF: the git ref for the base branch
-* GITHUB_TOKEN: token authorizing Github API usage
-* GITHUB_REPOSITORY: "org/repo" name of the Github repository of this PR
-* GITHUB_REF: string that includes this Github PR number
-
-This script tests each git commit between (and not including) GITHUB_SHA and
-GITHUB_BASE_REF multiple ways:
-
-1. Ensure that the committer and author do not match any bad patterns (e.g.,
-"root@", "localhost", etc.).
-
-2. Ensure that a proper "Signed-off-by" line exists in the commit message.
-    - Merge commits and reverts are exempted from this check.
-
-3. If required (by the git-commit-checks.json config file), ensure that a
-"(cherry picked from commit ...)" line exists in the commit message.
-    - Commits that are solely comprised of submodule updates are exempted from
-      this check.
-    - This check can also be disabled by adding "bot:notacherrypick" in the
-      Pull Request description.
-
-4. If a "(cherry picked from commit ...)" message exists, ensure that the commit
-hash it mentions exists in the git repository.
-
-If all checks pass, the script exits with status 0.  Otherwise, it exits with
-status 1.
-
-"""
-
-import os
-import re
-import git
-import json
-import copy
-import argparse
-
-from github import Github
-
-GOOD = "good"
-BAD  = "bad"
-
-GITHUB_WORKSPACE  = os.environ.get('GITHUB_WORKSPACE')
-GITHUB_SHA        = os.environ.get('GITHUB_SHA')
-GITHUB_BASE_REF   = os.environ.get('GITHUB_BASE_REF')
-GITHUB_TOKEN      = os.environ.get('GITHUB_TOKEN')
-GITHUB_REPOSITORY = os.environ.get('GITHUB_REPOSITORY')
-GITHUB_REF        = os.environ.get('GITHUB_REF')
-
-# Sanity check
-if (GITHUB_WORKSPACE is None or
-    GITHUB_SHA is None or
-    GITHUB_BASE_REF is None or
-    GITHUB_TOKEN is None or
-    GITHUB_REPOSITORY is None or
-    GITHUB_REF is None):
-    print("Error: this script is designed to run as a Github Action")
-    exit(1)
-
-#----------------------------------------------------------------------------
-
-"""
-Simple helper to make a 1-line git commit message summary.
-"""
-def make_commit_message(repo, hash):
-    commit  = repo.commit(hash)
-    lines   = commit.message.split('\n')
-    message = lines[0][:50]
-    if len(lines[0]) > 50:
-        message += "..."
-
-    return message
-
-#----------------------------------------------------------------------------
-
-"""
-The results dictionary is in the following format:
-
-    results[GOOD or BAD][commit hash][check name] = message
-
-If the message is None, there's nothing to print.
-
-A git commit hash will be in either the GOOD or the BAD results -- not both.
-"""
-def print_results(results, repo, hashes):
-    def _print_list(entries, prefix=""):
-        for hash, entry in entries.items():
-            print(f"{prefix}* {hash[:8]}: {make_commit_message(repo, hash)}")
-            for check_name, message in entry.items():
-                if message is not None:
-                    print(f"{prefix}    * {check_name}: {message}")
-
-    # First, print all the commits that have only-good results
-    if len(results[GOOD]) > 0:
-        print("\nThe following commits passed all tests:\n")
-        _print_list(results[GOOD])
-
-    # Now print all the results that are bad
-    if len(results[BAD]) > 0:
-        # The "::error ::" token will cause Github to highlight these
-        # lines as errors
-        print(f"\n::error ::The following commits caused this test to fail\n")
-        _print_list(results[BAD], "::error ::")
-
-#----------------------------------------------------------------------------
-
-"""
-Global regexp, because we use it every time we call
-check_signed_off() (i.e., for each commit in this PR)
-"""
-prog_sob = re.compile(r'Signed-off-by: (.+) <(.+)>')
-
-def check_signed_off(config, repo, commit):
-    # If the message starts with "Revert" or if the commit is a
-    # merge, don't require a signed-off-by
-    if commit.message.startswith("Revert "):
-        return GOOD, "skipped (revert)"
-    elif len(commit.parents) == 2:
-        return GOOD, "skipped (merge)"
-
-    matches = prog_sob.search(commit.message)
-    if not matches:
-        return BAD, "does not contain a valid Signed-off-by line"
-
-    return GOOD, None
-
-#----------------------------------------------------------------------------
-
-def check_email(config, repo, commit):
-    emails = {
-        "author"    : commit.author.email.lower(),
-        "committer" : commit.committer.email.lower(),
-    }
-
-    for id, email in emails.items():
-        for pattern in config['bad emails']:
-            match = re.search(pattern, email)
-            if match:
-                return BAD, f"{id} email address ({email}) contains '{pattern}'"
-
-    return GOOD, None
-
-#----------------------------------------------------------------------------
-
-"""
-Global regexp, because we use it every time we call check_cherry_pick()
-(i.e., for each commit in this PR)
-"""
-prog_cp = re.compile(r'\(cherry picked from commit ([a-z0-9]+)\)')
-
-def check_cherry_pick(config, repo, commit):
-    def _is_entirely_submodule_updates(repo, commit):
-        # If it's a merge commit, that doesn't fit our definition of
-        # "entirely submodule updates"
-        if len(commit.parents) == 2:
-            return False
-
-        # Check the diffs of this commit compared to the prior commit,
-        # and see if all the changes are updates to submodules.
-        submodule_paths = [ x.path for x in repo.submodules ]
-        diffs = repo.commit(f"{commit}~1").tree.diff(commit)
-        for diff in diffs:
-            if diff.a_path not in submodule_paths:
-                # If we get here, we found a diff that was not exclusively
-                # a submodule update.
-                return False
-
-        # If we get here, then all the diffs were submodule updates.
-        return True
-
-    # If this commit is solely comprised of submodule updates, don't
-    # require a cherry pick message.
-    if len(repo.submodules) > 0 and _is_entirely_submodule_updates(repo, commit):
-        return GOOD, "skipped (submodules updates)"
-
-    non_existent = dict()
-    found_cherry_pick_line = False
-    for match in prog_cp.findall(commit.message):
-        found_cherry_pick_line = True
-        try:
-            c = repo.commit(match)
-        except ValueError as e:
-            # These errors mean that the git library recognized the
-            # hash as a valid commit, but the GitHub Action didn't
-            # fetch the entire repo, so we don't have all the meta
-            # data about this commit.  Bottom line: it's a good hash.
-            # So -- no error here.
-            pass
-        except git.BadName as e:
-            # Use a dictionary to track the non-existent hashes, just
-            # on the off chance that the same non-existent hash exists
-            # more than once in a single commit message (i.e., the
-            # dictionary will effectively give us de-duplication for
-            # free).
-            non_existent[match] = True
-
-    # Process the results for this commit
-    if found_cherry_pick_line:
-        if len(non_existent) == 0:
-            return GOOD, None
-        else:
-            str = f"contains a cherry pick message that refers to non-existent commit"
-            if len(non_existent) > 1:
-                str += "s"
-            str += ": "
-            str += ", ".join(non_existent)
-            return BAD, str
-
-    else:
-        if config['cherry pick required']:
-            return BAD, "does not include a cherry pick message"
-        else:
-            return GOOD, None
-
-#----------------------------------------------------------------------------
-
-def check_all_commits(config, repo):
-    # Get a list of commits that we'll be examining.  Use the progromatic form
-    # of "git log GITHUB_BASE_REF..GITHUB_SHA" (i.e., "git log ^GITHUB_BASE_REF
-    # GITHUB_SHA") to do the heavy lifting to find that set of commits.
-    git_cli = git.cmd.Git(GITHUB_WORKSPACE)
-    hashes  = git_cli.log(f"--pretty=format:%h", f"origin/{GITHUB_BASE_REF}..{GITHUB_SHA}").splitlines()
-
-    # The first entry in the list will be the artificial Github merge commit for
-    # this PR. We don't want to examine this commit.
-    del hashes[0]
-
-    #------------------------------------------------------------------------
-
-    # Make an empty set of nested dictionaries to fill in, below. We initially
-    # create a "full" template dictionary (with all the hashes for both GOOD and
-    # BAD results), but will trim some of them later.
-    template = { hash : dict() for hash in hashes }
-    results  = {
-        GOOD : copy.deepcopy(template),
-        BAD  : copy.deepcopy(template),
-    }
-
-    for hash in hashes:
-        overall = GOOD
-
-        # Do the checks on this commit
-        commit  = repo.commit(hash)
-        for check_fn in [check_signed_off, check_email, check_cherry_pick]:
-            result, message = check_fn(config, repo, commit)
-            overall         = BAD if result == BAD else overall
-
-            results[result][hash][check_fn.__name__] = message
-
-        # Trim the results dictionary so that a hash only appears in GOOD *or*
-        # BAD -- not both. Specifically:
-        #
-        # 1. If a hash has BAD results, delete all of its results from GOOD.
-        # 2. If a hash has only GOOD results, delete its empty entry from BAD.
-        if overall == BAD:
-            del results[GOOD][hash]
-        else:
-            del results[BAD][hash]
-
-    return results, hashes
-
-#----------------------------------------------------------------------------
-
-"""
-If "bot:notacherrypick" is in the PR description, then disable the
-cherry-pick message requirement.
-"""
-def check_github_pr_description(config):
-    g      = Github(GITHUB_TOKEN)
-    repo   = g.get_repo(GITHUB_REPOSITORY)
-
-    # Extract the PR number from GITHUB_REF
-    match  = re.search("/(\d+)/", GITHUB_REF)
-    pr_num = int(match.group(1))
-    pr     = repo.get_pull(pr_num)
-
-    if "bot:notacherrypick" in pr.body:
-        config['cherry pick required'] = False
-
-#----------------------------------------------------------------------------
-
-def load_config():
-    # Defaults
-    config = {
-        'cherry pick required' : False,
-        'permit empty' : False,
-        'bad emails' : [
-            '^root@',
-            'localhost',
-            'localdomain',
-        ],
-    }
-
-    # If the config file exists, read it in and replace default values
-    # with the values from the file.
-    filename = os.path.join(GITHUB_WORKSPACE, '.github',
-                            'workflows', 'git-commit-checks.json')
-    if os.path.exists(filename):
-        with open(filename) as fp:
-            new_config = json.load(fp)
-        for key in new_config:
-            config[key] = new_config[key]
-
-    return config
-
-#----------------------------------------------------------------------------
-
-def main():
-    config = load_config()
-    check_github_pr_description(config)
-
-    repo = git.Repo(GITHUB_WORKSPACE)
-    results, hashes = check_all_commits(config, repo)
-    print_results(results, repo, hashes)
-
-    if len(results[BAD]) == 0:
-        print("\nTest passed: everything was good!")
-        exit(0)
-    else:
-        print("\nTest failed: sad panda")
-        exit(1)
-
-#----------------------------------------------------------------------------
-
-if __name__ == "__main__":
-    main()
diff --git a/.github/workflows/git-commit-checks.yml b/.github/workflows/git-commit-checks.yml
deleted file mode 100644
index 4cf1036878e..00000000000
--- a/.github/workflows/git-commit-checks.yml
+++ /dev/null
@@ -1,34 +0,0 @@
-name: GitHub Action CI
-
-on:
-    pull_request:
-        # We don't need this to be run on all types of PR behavior
-        # See https://docs.github.com/en/actions/reference/events-that-trigger-workflows#pull_request
-        types:
-          - opened
-          - synchronize
-          - edited
-
-jobs:
-    ci:
-        name: Git commit checker
-        runs-on: ubuntu-latest
-        steps:
-          - name: Check out the code
-            uses: actions/checkout@v2
-            with:
-                # Get all branches and history
-                fetch-depth: 0
-
-          - name: Setup Python
-            uses: actions/setup-python@v2
-            with:
-              python-version: '3.x'
-
-          - name: Get the GitPython and PyGithub modules
-            run: pip install gitpython PyGithub
-
-          - name: Check all git commits
-            run: $GITHUB_WORKSPACE/.github/workflows/git-commit-checks.py
-            env:
-                GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/hdf5-tests.yaml b/.github/workflows/hdf5-tests.yaml
new file mode 100644
index 00000000000..42417033a88
--- /dev/null
+++ b/.github/workflows/hdf5-tests.yaml
@@ -0,0 +1,45 @@
+name: HDF5
+
+on: [pull_request]
+
+jobs:
+  hdf5-testsuite:
+    runs-on: ubuntu-22.04
+    steps:
+    - name: Install dependencies
+      run: |
+        sudo apt update
+        sudo apt install -y --no-install-recommends wget
+    - uses: actions/checkout@v4
+      with:
+            submodules: recursive
+    - name: Build Open MPI
+      run: |
+        ./autogen.pl
+        ./configure --prefix=/opt/openmpi --with-pmix=internal --with-prrte=internal --with-hwloc=internal --with-libevent=internal --disable-mpi-fortran --disable-oshmem
+        make -j 8 && make install
+    - name: Install HDF5
+      run: |
+         wget --progress=dot:giga https://github.com/HDFGroup/hdf5/releases/latest/download/hdf5.tar.gz
+         tar -xzf hdf5.tar.gz
+         mv hdf5-1* hdf5
+         cd hdf5
+         export PATH=/opt/openmpi/bin:${PATH}
+         export LD_LIBRARY_PATH=/opt/openmpi/lib:${LD_LIBRARY_PATH}
+         echo ${PATH}
+         echo ${LD_LIBRARY_PATH}
+         ./configure --enable-parallel
+         make -j 8
+    - name: Tweak MPI
+      run:  |
+         # Tweak MPI
+         mca_params="$HOME/.prte/mca-params.conf"
+         mkdir -p "$(dirname "$mca_params")"
+         echo rmaps_default_mapping_policy = :oversubscribe >> "$mca_params"
+    - name: Run testsuite (ompio)
+      run: |
+         export PATH=/opt/openmpi/bin:${PATH}
+         export LD_LIBRARY_PATH=/opt/openmpi/lib:${LD_LIBRARY_PATH}
+         HDF5TestExpress=0 mpirun --mca io ompio -np 4 ./hdf5/testpar/t_shapesame
+         mpirun --mca io ompio -np 4 ./hdf5/testpar/t_filters_parallel
+         mpirun --mca io ompio -np 4 ./hdf5/testpar/testphdf5
diff --git a/.github/workflows/macos-checks.yaml b/.github/workflows/macos-checks.yaml
new file mode 100644
index 00000000000..5c50a8fe114
--- /dev/null
+++ b/.github/workflows/macos-checks.yaml
@@ -0,0 +1,46 @@
+name: macOS
+
+on: [pull_request]
+
+jobs:
+  macOS:
+    runs-on: macos-latest
+    steps:
+    - name: Setup macOS
+      run: |
+        # Copied from mpi4py/mpi-publish
+        # create gfortran symlink
+        cd $(brew --prefix)/bin
+        gfortran=$(ls gfortran-* | sort | head -n 1)
+        sudo ln -s $gfortran gfortran
+        # install autotools
+        brew install autoconf
+        brew install automake
+        brew install libtool
+        # unlink libevent
+        brew unlink libevent || true
+    - uses: actions/checkout@v4
+      with:
+            submodules: recursive
+    - name: Build Open MPI
+      run: |
+        ./autogen.pl
+        ./configure --prefix=/opt/openmpi --disable-silent-rules
+        make -j $(sysctl -n hw.logicalcpu)
+    - name: Run unit tests
+      run: |
+        make check
+    - name: Install Open MPI
+      run: |
+        sudo make install
+    - name: Add Open MPI to PATH
+      run: echo /opt/openmpi/bin >> $GITHUB_PATH
+    - name: Build examples
+      run: |
+        pushd examples
+        make
+        popd
+    - name: Test ring and sessions sanity check
+      run: |
+        mpirun --map-by ppr:1:core examples/ring_c
+        mpirun --map-by ppr:1:core examples/hello_sessions_c
diff --git a/.github/workflows/ompi_mpi4py.yaml b/.github/workflows/ompi_mpi4py.yaml
new file mode 100644
index 00000000000..8e3c450b4be
--- /dev/null
+++ b/.github/workflows/ompi_mpi4py.yaml
@@ -0,0 +1,157 @@
+name: mpi4py
+
+on:
+  pull_request:
+  workflow_dispatch:
+    inputs:
+      repository:
+        description: 'mpi4py repository'
+        default: 'mpi4py/mpi4py'
+        required: false
+        type: string
+      ref:
+        description: 'mpi4py branch/tag/SHA'
+        default: 'master'
+        required: false
+        type: string
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    env:
+      MPI4PY_TEST_SPAWN: true
+    steps:
+    - name: Configure hostname
+      run:  echo 127.0.0.1 `hostname` | sudo tee -a /etc/hosts > /dev/null
+      if:   ${{ runner.os == 'Linux' || runner.os == 'macOS' }}
+
+    - name: Install depencencies
+      run:  sudo apt-get install -y -q
+              libnuma-dev
+      if:   ${{ runner.os == 'Linux' }}
+
+    - name: Checkout Open MPI
+      uses: actions/checkout@v4
+      with:
+       path: mpi-build
+       submodules: recursive
+
+    - name: Bootstrap Open MPI
+      run:  ./autogen.pl
+      working-directory: mpi-build
+
+    # Install into a separate directory (/opt/openmpi) so that we can
+    # bundle up that tree into an artifact to share with other jobs in
+    # this github action.  Specifically don't use /usr/local, because
+    # there's a bunch of other stuff already installed in /usr/local,
+    # and we don't need to include that in our artifact.
+    - name: Configure Open MPI
+      run:  ./configure
+              --disable-dependency-tracking
+              --enable-debug
+              --enable-mem-debug
+              --disable-sphinx
+              --disable-mpi-fortran
+              --disable-oshmem
+              --disable-silent-rules
+              --prefix=/opt/openmpi
+              LDFLAGS=-Wl,-rpath,/opt/openmpi/lib
+      working-directory: mpi-build
+
+    - name: Build MPI
+      run:  make -j $(nproc)
+      working-directory: mpi-build
+
+    - name: Install MPI
+      run:  sudo make install
+      working-directory: mpi-build
+
+    - name: Add Open MPI to PATH
+      run: echo /opt/openmpi/bin >> $GITHUB_PATH
+
+    - name: Tweak MPI
+      run:  |
+        # Tweak MPI
+        mca_params="$HOME/.openmpi/mca-params.conf"
+        mkdir -p "$(dirname "$mca_params")"
+        echo mpi_param_check = true >> "$mca_params"
+        echo mpi_show_handle_leaks = true >> "$mca_params"
+        mca_params="$HOME/.prte/mca-params.conf"
+        mkdir -p "$(dirname "$mca_params")"
+        echo rmaps_default_mapping_policy = :oversubscribe >> "$mca_params"
+
+    - name: Show MPI
+      run:  ompi_info
+
+    - name: Show MPICC
+      run:  mpicc -show
+
+    - name: Use Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3
+        architecture: x64
+
+    - name: Install Python packages (build)
+      run:  python -m pip install --upgrade
+              setuptools pip wheel
+
+    - name: Install Python packages (test)
+      run:  python -m pip install --upgrade
+              numpy cffi pyyaml
+
+    - name: Checkout mpi4py
+      uses: actions/checkout@v4
+      with:
+        repository: ${{ inputs.repository || 'mpi4py/mpi4py' }}
+        ref: ${{ inputs.ref }}
+
+    - name: Install mpi4py
+      run:  python -m pip install .
+      env:
+        CFLAGS: "-O0"
+
+    - name: Test mpi4py (singleton)
+      run:  python test/main.py -v
+      if:   ${{ true }}
+      timeout-minutes: 10
+    - name: Test mpi4py (np=1)
+      run:  mpiexec -n 1 python test/main.py -v
+      if:   ${{ true }}
+      timeout-minutes: 10
+    - name: Test mpi4py (np=2)
+      run:  mpiexec -n 2 python test/main.py -v -f
+      if:   ${{ true }}
+      timeout-minutes: 10
+    - name: Test mpi4py (np=3)
+      run:  mpiexec -n 3 python test/main.py -v -f
+      if:   ${{ true }}
+      timeout-minutes: 10
+    - name: Test mpi4py (np=4)
+      run:  mpiexec -n 4 python test/main.py -v -f
+      if:   ${{ true }}
+      timeout-minutes: 10
+    - name: Test mpi4py (np=5)
+      run:  mpiexec -n 5 python test/main.py -v -f
+      if:   ${{ true }}
+      timeout-minutes: 10
+
+    - name: Test mpi4py.run
+      run:  python demo/test-run/test_run.py -v
+      if:   ${{ true }}
+      timeout-minutes: 10
+
+    - name: Relocate Open MPI installation
+      run:  mv /opt/openmpi /opt/ompi
+    - name: Update PATH and set OPAL_PREFIX and LD_LIBRARY_PATH
+      run: |
+        sed -i '\|/opt/openmpi/bin|d' $GITHUB_PATH
+        echo OPAL_PREFIX=/opt/ompi >> $GITHUB_ENV
+        echo LD_LIBRARY_PATH=/opt/ompi/lib >> $GITHUB_ENV
+
+    - name: Test mpi4py (singleton)
+      run:  python test/main.py -v
+      if:   ${{ true }}
+      timeout-minutes: 10
+
diff --git a/.github/workflows/ompi_nvidia.yaml b/.github/workflows/ompi_nvidia.yaml
new file mode 100644
index 00000000000..6a3201a3648
--- /dev/null
+++ b/.github/workflows/ompi_nvidia.yaml
@@ -0,0 +1,42 @@
+name: ompi_NVIDIA CI
+on: [pull_request]
+jobs:
+
+  deployment:
+    if: github.repository == 'open-mpi/ompi'
+    runs-on: [self-hosted, linux, x64, nvidia]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        submodules: recursive
+    - name: Checkout CI scripts
+      uses: actions/checkout@v4
+      with:
+        repository: Mellanox/jenkins_scripts
+        path: ompi_ci
+    - name: Deployment infrastructure
+      run: /start deploy
+  build:
+    needs: [deployment]
+    runs-on: [self-hosted, linux, x64, nvidia]
+    steps:
+    - name: Building OMPI,UCX and tests
+      run: /start build
+  test:
+    needs: [deployment, build]
+    runs-on: [self-hosted, linux, x64, nvidia]
+    steps:
+    - name: Running tests
+      run: /start test
+  clean:
+# always() should be used to run "clean" even when the workflow was canceled 
+#  ( in case of the right repository name)
+# The second condition doesn't work when the workflow was canceled
+
+    if: always() && (github.repository == 'open-mpi/ompi')
+    needs: [deployment, build, test]
+    runs-on: [self-hosted, linux, x64, nvidia]
+    steps:
+    - name: Cleaning
+      run: /start clean
diff --git a/.github/workflows/pr-checks.yaml b/.github/workflows/pr-checks.yaml
new file mode 100644
index 00000000000..5c390424560
--- /dev/null
+++ b/.github/workflows/pr-checks.yaml
@@ -0,0 +1,50 @@
+name: Git commit checks
+
+# We're using pull_request_target here instead of just pull_request so that the
+# action runs in the context of the base of the pull request, rather than in the
+# context of the merge commit. For more detail about the differences, see:
+# https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target
+on:
+    pull_request_target:
+        # We don't need this to be run on all types of PR behavior
+        # See https://docs.github.com/en/actions/reference/events-that-trigger-workflows#pull_request
+        types:
+          - opened
+          - synchronize
+          - edited
+
+permissions: {} # none
+
+jobs:
+  check:
+    permissions:
+      pull-requests: write
+    name: Check Commits
+    runs-on: ubuntu-latest
+    steps:
+      - name: Pull Request Commit Checker
+        uses: open-mpi/pr-git-commit-checker@v1.0.1
+        with:
+          token: "${{ secrets.GITHUB_TOKEN}}"
+  label:
+    permissions:
+      pull-requests: write
+    name: Label Pull Request
+    runs-on: ubuntu-latest
+    steps:
+      - name: Pull Request Labeler
+        uses: open-mpi/pr-labeler@v1.0.1
+        with:
+          token: "${{ secrets.GITHUB_TOKEN }}"
+
+  milestone:
+    permissions:
+      issues: write
+      pull-requests: write
+    name: Milestone Pull Request
+    runs-on: ubuntu-latest
+    steps:
+      - name: Pull Request Milestoner
+        uses: open-mpi/pr-milestoner@v1.0.1
+        with:
+          token: "${{ secrets.GITHUB_TOKEN }}"
diff --git a/.github/workflows/remove-awaiting-user-info-label.yaml b/.github/workflows/remove-awaiting-user-info-label.yaml
new file mode 100644
index 00000000000..4b9e9072798
--- /dev/null
+++ b/.github/workflows/remove-awaiting-user-info-label.yaml
@@ -0,0 +1,30 @@
+# This Action is run in conjunction with close-stale-issues.yaml.  See
+# that file for a more complete description of how they work together.
+
+name: 'Remove "State: Awaiting user information" label when there has been a reply'
+on:
+  issue_comment:
+    types:
+      - created
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    # From
+    # https://github.com/marketplace/actions/close-issues-after-no-reply:
+    # only remove the label if someone replies to an issue who is not
+    # an owner or collaborator on the repo.
+    if: |
+      github.event.comment.author_association != 'OWNER' &&
+      github.event.comment.author_association != 'COLLABORATOR'
+    steps:
+      - name: 'Remove "State: Awaiting user information" label'
+        uses: octokit/request-action@v2.x
+        continue-on-error: true
+        with:
+          route: DELETE /repos/:repository/issues/:issue/labels/:label
+          repository: ${{ github.repository }}
+          issue: ${{ github.event.issue.number }}
+          label: State-Awaiting user information
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.gitignore b/.gitignore
index 25f6d27908a..dfa2e7c86d0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,11 +8,6 @@
 .gdb*
 .idea
 
-.hgrc
-.hgignore
-.hg
-.hgignore_local
-
 *.la
 *.lo
 *.o
@@ -105,6 +100,7 @@ ltoptions.m4
 !3rd-party/libevent-*.tar.*
 3rd-party/hwloc-*
 !3rd-party/hwloc-*.tar.*
+3rd-party/treematch/config.h
 
 config/project_list.m4
 config/autogen_found_items.m4
@@ -139,6 +135,7 @@ examples/hello_cxx
 examples/hello_mpifh
 examples/hello_usempi
 examples/hello_usempif08
+examples/hello_sessions_c
 examples/ring_c
 examples/ring_cxx
 examples/ring_mpifh
@@ -188,33 +185,11 @@ ompi/mca/coll/ml/coll_ml_lex.c
 ompi/mca/coll/self/coll-self-version.h*
 ompi/mca/coll/sm/coll-sm-version.h*
 
-ompi/mca/io/romio321/romio/adio/include/romioconf.h
-ompi/mca/io/romio321/romio/include/mpio.h
-ompi/mca/io/romio321/romio/localdefs
-ompi/mca/io/romio321/romio/test/fcoll_test.f
-ompi/mca/io/romio321/romio/test/fmisc.f
-ompi/mca/io/romio321/romio/test/fperf.f
-ompi/mca/io/romio321/romio/test/large_file.c
-ompi/mca/io/romio321/romio/test/misc.c
-ompi/mca/io/romio321/romio/test/pfcoll_test.f
-ompi/mca/io/romio321/romio/test/runtests
-ompi/mca/io/romio321/romio/util/romioinstall
-ompi/mca/io/romio321/romio/test/syshints.c
-
 ompi/mca/osc/monitoring/osc_monitoring_template_gen.h
 
 ompi/mca/pml/v/autogen.vprotocols
 ompi/mca/pml/v/mca_vprotocol_config_output
 
-ompi/mca/rte/orte/ompi-ps.1
-ompi/mca/rte/orte/ompi-clean.1
-ompi/mca/rte/orte/mpiexec.1
-ompi/mca/rte/orte/ompi-top.1
-ompi/mca/rte/orte/ompi-server.1
-ompi/mca/rte/orte/ompi-restart.1
-ompi/mca/rte/orte/ompi-checkpoint.1
-ompi/mca/rte/orte/mpirun.1
-
 ompi/mca/sharedfp/addproc/mca_sharedfp_addproc_control
 
 ompi/mca/topo/treematch/config.h
@@ -226,6 +201,7 @@ ompi/mpi/fortran/mpiext/mpi-ext-module.F90
 ompi/mpi/fortran/mpiext/mpi-f08-ext-module.F90
 ompi/mpi/fortran/mpiext-use-mpi/mpi-ext-module.F90
 ompi/mpi/fortran/mpiext-use-mpi-f08/mpi-f08-ext-module.F90
+ompi/mpi/fortran/use-mpi-f08/psizeof_f08.f90
 
 ompi/mpi/fortran/mpif-h/sizeof_f.f90
 ompi/mpi/fortran/mpif-h/profile/p*.c
@@ -257,13 +233,8 @@ ompi/mpi/java/java/*.jar
 ompi/mpi/java/java/*.h
 ompi/mpi/java/java/doc
 
-ompi/mpi/man/man3/MPI*.3
-ompi/mpi/man/man3/OpenMPI.3
-ompi/mpi/man/man3/.dir-stamp
-
 ompi/mpi/tool/profile/*.c
 
-ompi/mpiext/affinity/c/OMPI_Affinity_str.3
 ompi/mpiext/affinity/c/example
 
 ompi/mpiext/ftmpi/c/profile/pcomm_agree.c
@@ -273,64 +244,19 @@ ompi/mpiext/ftmpi/c/profile/pcomm_iagree.c
 ompi/mpiext/ftmpi/c/profile/pcomm_is_revoked.c
 ompi/mpiext/ftmpi/c/profile/pcomm_revoke.c
 ompi/mpiext/ftmpi/c/profile/pcomm_shrink.c
+ompi/mpiext/ftmpi/c/profile/pcomm_ack_failed.c
+ompi/mpiext/ftmpi/c/profile/pcomm_get_failed.c
+ompi/mpiext/ftmpi/c/profile/pcomm_ishrink.c
 
 ompi/mpiext/example/tests/progress_c
 ompi/mpiext/example/tests/progress_mpifh
 ompi/mpiext/example/tests/progress_usempi
 ompi/mpiext/example/tests/progress_usempif08
 
-ompi/mpiext/cuda/c/MPIX_Query_cuda_support.3
 ompi/mpiext/cuda/c/mpiext_cuda_c.h
 ompi/mpiext/cuda/c/cuda_c.h
 
-ompi/mpiext/pcollreq/c/MPIX_*.3
-ompi/mpiext/pcollreq/c/profile/pallgather_init.c
-ompi/mpiext/pcollreq/c/profile/pallgatherv_init.c
-ompi/mpiext/pcollreq/c/profile/pallreduce_init.c
-ompi/mpiext/pcollreq/c/profile/palltoall_init.c
-ompi/mpiext/pcollreq/c/profile/palltoallv_init.c
-ompi/mpiext/pcollreq/c/profile/palltoallw_init.c
-ompi/mpiext/pcollreq/c/profile/pbarrier_init.c
-ompi/mpiext/pcollreq/c/profile/pbcast_init.c
-ompi/mpiext/pcollreq/c/profile/pexscan_init.c
-ompi/mpiext/pcollreq/c/profile/pgather_init.c
-ompi/mpiext/pcollreq/c/profile/pgatherv_init.c
-ompi/mpiext/pcollreq/c/profile/pmpiext_pcollreq_c.h
-ompi/mpiext/pcollreq/c/profile/pneighbor_allgather_init.c
-ompi/mpiext/pcollreq/c/profile/pneighbor_allgatherv_init.c
-ompi/mpiext/pcollreq/c/profile/pneighbor_alltoall_init.c
-ompi/mpiext/pcollreq/c/profile/pneighbor_alltoallv_init.c
-ompi/mpiext/pcollreq/c/profile/pneighbor_alltoallw_init.c
-ompi/mpiext/pcollreq/c/profile/preduce_init.c
-ompi/mpiext/pcollreq/c/profile/preduce_scatter_block_init.c
-ompi/mpiext/pcollreq/c/profile/preduce_scatter_init.c
-ompi/mpiext/pcollreq/c/profile/pscan_init.c
-ompi/mpiext/pcollreq/c/profile/pscatter_init.c
-ompi/mpiext/pcollreq/c/profile/pscatterv_init.c
-ompi/mpiext/pcollreq/c/profile/ppcollreq_c.h
-
-ompi/mpiext/pcollreq/mpif-h/profile/pallgather_init_f.c
-ompi/mpiext/pcollreq/mpif-h/profile/pallgatherv_init_f.c
-ompi/mpiext/pcollreq/mpif-h/profile/pallreduce_init_f.c
-ompi/mpiext/pcollreq/mpif-h/profile/palltoall_init_f.c
-ompi/mpiext/pcollreq/mpif-h/profile/palltoallv_init_f.c
-ompi/mpiext/pcollreq/mpif-h/profile/palltoallw_init_f.c
-ompi/mpiext/pcollreq/mpif-h/profile/pbarrier_init_f.c
-ompi/mpiext/pcollreq/mpif-h/profile/pbcast_init_f.c
-ompi/mpiext/pcollreq/mpif-h/profile/pexscan_init_f.c
-ompi/mpiext/pcollreq/mpif-h/profile/pgather_init_f.c
-ompi/mpiext/pcollreq/mpif-h/profile/pgatherv_init_f.c
-ompi/mpiext/pcollreq/mpif-h/profile/pneighbor_allgather_init_f.c
-ompi/mpiext/pcollreq/mpif-h/profile/pneighbor_allgatherv_init_f.c
-ompi/mpiext/pcollreq/mpif-h/profile/pneighbor_alltoall_init_f.c
-ompi/mpiext/pcollreq/mpif-h/profile/pneighbor_alltoallv_init_f.c
-ompi/mpiext/pcollreq/mpif-h/profile/pneighbor_alltoallw_init_f.c
-ompi/mpiext/pcollreq/mpif-h/profile/preduce_init_f.c
-ompi/mpiext/pcollreq/mpif-h/profile/preduce_scatter_block_init_f.c
-ompi/mpiext/pcollreq/mpif-h/profile/preduce_scatter_init_f.c
-ompi/mpiext/pcollreq/mpif-h/profile/pscan_init_f.c
-ompi/mpiext/pcollreq/mpif-h/profile/pscatter_init_f.c
-ompi/mpiext/pcollreq/mpif-h/profile/pscatterv_init_f.c
+ompi/mpiext/rocm/c/mpiext_rocm_c.h
 
 ompi/mpiext/shortfloat/c/mpiext_shortfloat_c.h
 ompi/mpiext/shortfloat/mpif-h/mpiext_shortfloat_mpifh.h
@@ -339,33 +265,23 @@ ompi/mpiext/shortfloat/use-mpi-f08/mpiext_shortfloat_usempif08.h
 ompi/tools/mpisync/mpisync
 ompi/tools/mpisync/mpirun_prof
 ompi/tools/mpisync/ompi_timing_post
-ompi/tools/mpisync/mpisync.1
+ompi/tools/mpirun/mpirun
 
 ompi/tools/ompi_info/ompi_info
-ompi/tools/ompi_info/ompi_info.1
 
 ompi/tools/wrappers/mpic++-wrapper-data.txt
 ompi/tools/wrappers/mpicc-wrapper-data.txt
 ompi/tools/wrappers/mpifort-wrapper-data.txt
-ompi/tools/wrappers/mpicc.1
-ompi/tools/wrappers/mpic++.1
-ompi/tools/wrappers/mpicxx.1
-ompi/tools/wrappers/mpifort.1
-ompi/tools/wrappers/mpijavac.1
 ompi/tools/wrappers/ompi_wrapper_script
 ompi/tools/wrappers/ompi.pc
 ompi/tools/wrappers/ompi-c.pc
 ompi/tools/wrappers/ompi-cxx.pc
 ompi/tools/wrappers/ompi-fort.pc
 ompi/tools/wrappers/mpijavac.pl
-ompi/tools/wrappers/mpif90.1
-ompi/tools/wrappers/mpif77.1
 ompi/tools/wrappers/mpicxx-wrapper-data.txt
 ompi/tools/wrappers/mpif77-wrapper-data.txt
 ompi/tools/wrappers/mpif90-wrapper-data.txt
 
-opal/libltdl
-
 opal/asm/atomic-asm.S
 opal/asm/atomic-test
 opal/asm/generated/atomic-*.s
@@ -381,14 +297,8 @@ opal/include/opal/sys/powerpc/atomic-32-64.s
 
 opal/mca/base/mca_base_parse_paramfile_lex.c
 
-opal/mca/common/libfabric/libfabric/config.h
-
-opal/mca/btl/openib/btl_openib_lex.c
-
 opal/mca/btl/usnic/usnic_btl_run_tests
 
-opal/mca/crs/opal_crs.7
-
 opal/mca/event/libevent*/libevent/config.h.in
 opal/mca/event/libevent*/libevent/config.h
 opal/mca/event/libevent*/libevent/libevent.pc
@@ -398,80 +308,18 @@ opal/mca/event/libevent*/libevent/include/event2/event-config.h
 
 opal/mca/installdirs/config/install_dirs.h
 
-!opal/mca/pmix/pmix*/openpmix/AUTHORS
-!opal/mca/pmix/pmix*/openpmix/contrib/perf_tools/Makefile
-opal/mca/pmix/pmix*/openpmix/include/pmix/autogen/config.h
-opal/mca/pmix/pmix*/openpmix/include/pmix/autogen/config.h.in
-opal/mca/pmix/pmix*/openpmix/src/include/private/autogen/config.h.in
-opal/mca/pmix/pmix*/openpmix/src/include/private/autogen/config.h
-opal/mca/pmix/pmix*/openpmix/src/include/frameworks.h
-opal/mca/pmix/pmix*/openpmix/src/mca/pinstalldirs/config/pinstall_dirs.h
-opal/mca/pmix/pmix*/openpmix/config/autogen_found_items.m4
-opal/mca/pmix/pmix*/openpmix/config/mca_library_paths.txt
-opal/mca/pmix/pmix*/openpmix/config/test-driver
-opal/mca/pmix/pmix*/openpmix/src/include/pmix_config.h
-opal/mca/pmix/pmix*/openpmix/src/include/pmix_config.h.in
-opal/mca/pmix/pmix*/openpmix/include/pmix_common.h
-opal/mca/pmix/pmix*/openpmix/include/pmix_rename.h
-opal/mca/pmix/pmix*/openpmix/include/pmix_version.h
-opal/mca/pmix/pmix*/openpmix/src/util/keyval/keyval_lex.c
-opal/mca/pmix/pmix*/openpmix/src/util/show_help_lex.c
-opal/mca/pmix/pmix*/openpmix/examples/alloc
-opal/mca/pmix/pmix*/openpmix/examples/client
-opal/mca/pmix/pmix*/openpmix/examples/debugger
-opal/mca/pmix/pmix*/openpmix/examples/debuggerd
-opal/mca/pmix/pmix*/openpmix/examples/dmodex
-opal/mca/pmix/pmix*/openpmix/examples/dynamic
-opal/mca/pmix/pmix*/openpmix/examples/fault
-opal/mca/pmix/pmix*/openpmix/examples/jctrl
-opal/mca/pmix/pmix*/openpmix/examples/pub
-opal/mca/pmix/pmix*/openpmix/examples/server
-opal/mca/pmix/pmix*/openpmix/examples/tool
-opal/mca/pmix/pmix*/openpmix/test/run_tests00.pl
-opal/mca/pmix/pmix*/openpmix/test/run_tests01.pl
-opal/mca/pmix/pmix*/openpmix/test/run_tests02.pl
-opal/mca/pmix/pmix*/openpmix/test/run_tests03.pl
-opal/mca/pmix/pmix*/openpmix/test/run_tests04.pl
-opal/mca/pmix/pmix*/openpmix/test/run_tests05.pl
-opal/mca/pmix/pmix*/openpmix/test/run_tests06.pl
-opal/mca/pmix/pmix*/openpmix/test/run_tests07.pl
-opal/mca/pmix/pmix*/openpmix/test/run_tests08.pl
-opal/mca/pmix/pmix*/openpmix/test/run_tests09.pl
-opal/mca/pmix/pmix*/openpmix/test/run_tests10.pl
-opal/mca/pmix/pmix*/openpmix/test/run_tests11.pl
-opal/mca/pmix/pmix*/openpmix/test/run_tests12.pl
-opal/mca/pmix/pmix*/openpmix/test/run_tests13.pl
-opal/mca/pmix/pmix*/openpmix/test/run_tests14.pl
-opal/mca/pmix/pmix*/openpmix/test/run_tests15.pl
-opal/mca/pmix/pmix*/openpmix/src/tools/wrapper/pmix.pc
-opal/mca/pmix/pmix*/openpmix/src/tools/wrapper/pmixcc-wrapper-data.txt
-
-
-opal/mca/pmix/ext4x/ext4x.c
-opal/mca/pmix/ext4x/ext4x.h
-opal/mca/pmix/ext4x/ext4x_client.c
-opal/mca/pmix/ext4x/ext4x_component.c
-opal/mca/pmix/ext4x/ext4x_server_north.c
-opal/mca/pmix/ext4x/ext4x_server_south.c
-
-opal/tools/opal-checkpoint/opal-checkpoint
-opal/tools/opal-checkpoint/opal-checkpoint.1
-opal/tools/opal-restart/opal-restart
-opal/tools/opal-restart/opal-restart.1
-
 opal/tools/wrappers/opalcc-wrapper-data.txt
 opal/tools/wrappers/opalc++-wrapper-data.txt
 opal/tools/wrappers/opalCC-wrapper-data.txt
 opal/tools/wrappers/opal_wrapper
-opal/tools/wrappers/opalcc.1
-opal/tools/wrappers/opalc++.1
-opal/tools/wrappers/generic_wrapper.1
-opal/tools/wrappers/opal_wrapper.1
 opal/tools/wrappers/opal.pc
 
-opal/util/show_help_lex.c
+opal/util/show_help_content.c
 opal/util/keyval/keyval_lex.c
 
+test/monitoring/aggregate_profile.pl
+test/monitoring/profile2mat.pl
+
 test/simple/abort
 test/simple/accept
 test/simple/attach
@@ -561,44 +409,15 @@ oshmem/shmem/fortran/profile/pshpalloc_f.c
 oshmem/shmem/fortran/profile/pnum_pes_f.c
 oshmem/shmem/fortran/profile/pstart_pes_f.c
 
-oshmem/shmem/java/java/shmem.jar
-oshmem/shmem/java/java/doc
-oshmem/shmem/java/java/shmem_Constant.h
-oshmem/shmem/java/java/shmem_ShMem.h
-oshmem/shmem/java/java/shmem
-oshmem/shmem/java/java/shmem_Addr.h
-oshmem/shmem/java/java/shmem_PSync.h
-
-oshmem/shmem/man/man3/shmem_*.3
-oshmem/shmem/man/man3/OpenSHMEM.3
-oshmem/shmem/man/man3/intro_shmem.3
-oshmem/shmem/man/man3/_my_pe.3
-oshmem/shmem/man/man3/_num_pes.3
-oshmem/shmem/man/man3/shfree.3
-oshmem/shmem/man/man3/shmalloc.3
-oshmem/shmem/man/man3/shmemalign.3
-oshmem/shmem/man/man3/shrealloc.3
-oshmem/shmem/man/man3/start_pes.3
-oshmem/shmem/man/man3/.dir-stamp
-
 oshmem/tools/oshmem_info/oshmem_info
-oshmem/tools/oshmem_info/oshmem_info.1
-
-oshmem/tools/wrappers/oshcc.1
-oshmem/tools/wrappers/oshfort.1
-oshmem/tools/wrappers/oshrun.1
-oshmem/tools/wrappers/shmemcc.1
-oshmem/tools/wrappers/shmemfort.1
-oshmem/tools/wrappers/shmemrun.1
+
+oshmem/tools/wrappers/oshmem-c.pc
+oshmem/tools/wrappers/oshmem-cxx.pc
+oshmem/tools/wrappers/oshmem-fort.pc
+oshmem/tools/wrappers/oshmem.pc
 oshmem/tools/wrappers/shmemcc-wrapper-data.txt
 oshmem/tools/wrappers/shmemfort-wrapper-data.txt
-oshmem/tools/wrappers/oshCC.1
-oshmem/tools/wrappers/oshc++.1
-oshmem/tools/wrappers/oshcxx.1
-oshmem/tools/wrappers/shmemCC.1
 oshmem/tools/wrappers/shmemc++-wrapper-data.txt
-oshmem/tools/wrappers/shmemc++.1
-oshmem/tools/wrappers/shmemcxx.1
 
 test/asm/atomic_math_noinline
 test/asm/atomic_barrier
@@ -621,6 +440,7 @@ test/class/ompi_rb_tree
 test/class/ompi_bitmap
 test/class/opal_bitmap
 test/class/opal_fifo
+test/class/opal_cstring
 test/class/opal_hash_table
 test/class/opal_lifo
 test/class/opal_list
@@ -651,7 +471,6 @@ test/monitoring/example_reduce_count
 test/monitoring/test_overhead
 test/monitoring/test_pvar_access
 
-
 test/mpi/environment/chello
 
 test/runtime/parse_context
@@ -686,8 +505,37 @@ test/util/opal_path_nfs
 test/util/opal_path_nfs.out
 test/util/opal_bit_ops
 test/util/bipartite_graph
+test/util/opal_sha256
 
 opal/test/reachable/reachable_netlink
 opal/test/reachable/reachable_weighted
 opal/mca/threads/argobots/threads_argobots.h
 opal/mca/threads/qthreads/threads_qthreads.h
+
+docs/_build
+docs/_static
+docs/_static/css/custom.css
+docs/_templates
+docs/man-openmpi/man3/bindings
+
+# Common Python virtual environment and cache directory names
+venv
+py??
+__pycache__/
+
+# Copies of PRRTE RST files (i.e., not source controlled in this tree)
+docs/prrte-rst-content
+docs/schizo-ompi-rst-content
+
+# Copies of the built HTML docs and man pages (for distribution
+# tarballs)
+docs/html
+docs/man
+
+# Generated C Bindings
+ompi/mpi/c/*_generated*.c
+
+# Generated Fortran Bindings
+ompi/mpi/fortran/use-mpi-f08/*_generated.F90
+ompi/mpi/fortran/use-mpi-f08/base/*_generated.c
+ompi/mpi/fortran/use-mpi-f08/mod/mpi-f08-interfaces-generated.h
diff --git a/.gitmodules b/.gitmodules
index 81400e0d6de..5646d8fedc5 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,8 +1,14 @@
 [submodule "prrte"]
 	path = 3rd-party/prrte
-	url = https://github.com/openpmix/prrte
+	url = ../../open-mpi/prrte
 	branch = master
 [submodule "openpmix"]
 	path = 3rd-party/openpmix
-	url = https://github.com/openpmix/openpmix.git
+	url = ../../openpmix/openpmix.git
 	branch = master
+[submodule "oac"]
+	path = config/oac
+	url = ../../open-mpi/oac
+[submodule "3rd-party/pympistandard"]
+	path = 3rd-party/pympistandard
+	url = ../../mpi-forum/pympistandard
diff --git a/.mailmap b/.mailmap
index 4ad3e4cbedb..b463497a038 100644
--- a/.mailmap
+++ b/.mailmap
@@ -32,6 +32,7 @@
 Jeff Squyres <jsquyres@cisco.com> <jsquyres@users.noreply.github.com>
 Jeff Squyres <jsquyres@cisco.com> --quiet <--quiet>
 Jeff Squyres <no-author@open-mpi.org>
+Jeff Squyres <jeff@squyres.com>
 
 George Bosilca <bosilca@icl.utk.edu> <bosilca@users.noreply.github.com>
 
@@ -44,6 +45,8 @@ Devendar Bureddy <devendar@mellanox.com> <bureddy@users.noreply.github.com>
 
 Edgar Gabriel <egabriel@central.uh.edu> <edgargabriel@users.noreply.github.com>
 Edgar Gabriel <gabriel@cs.uh.edu> <gabriel@Peggys-MacBook-Air.local>
+Edgar Gabriel <edgar.gabriel1@outlook.com> <edgar.gabriel1@outlook.com>
+Edgar Gabriel <edgar.gabriel@amd.com> <edgar.gabriel@amd.com>
 
 Gilles Gouaillardet <gilles@rist.or.jp> <ggouaillardet@users.noreply.github.com>
 
@@ -113,3 +116,24 @@ Anandhi S Jayakumar <anandhi.s.jayakumar@intel.com>
 Mohan Gandhi <mohgan@amazon.com>
 
 Harumi Kuno <harumi.kuno@hpe.com>
+
+Nick Papior <nickpapior@gmail.com>
+Nick Papior <nickpapior@gmail.com> <nicpa@dtu.dk>
+Nick Papior <nickpapior@gmail.com> <zerothi@users.noreply.github.com>
+
+Matthew G. F. Dosanjh <mdosanj@sandia.gov>
+
+Wei-keng Liao <wkliao@users.noreply.github.com>
+
+Samuel K. Gutierrez <samuel@lanl.gov> <samuelkgutierrez@users.noreply.github.com>
+Samuel K. Gutierrez <samuel@lanl.gov> <samuel@lanl.gov>
+
+Tomislav Janjusic <tomislavj@nvidia.com> Tomislavj Janjusic <tomislavj@nvidia.com>
+
+William P. LePera <lepera@us.ibm.com>
+
+George Katevenis <gkatev@ics.forth.gr>
+
+Brian Barrett <brian@bbarrett.org>
+
+Andrii Bilokur <abilokur@nvidia.com> B-a-S <abilokur@nvidia.com>
diff --git a/.readthedocs-pre-create-environment.sh b/.readthedocs-pre-create-environment.sh
new file mode 100755
index 00000000000..cc461ac2b13
--- /dev/null
+++ b/.readthedocs-pre-create-environment.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+# The ReadTheDocs build process does not run autogen/configure/make.
+# Hence, we have to copy the PRRTE RST files (from the 3rd-party/prrte
+# tree) to our docs/ tree manually.
+
+# Ensure that we're in the RTD CI environment
+
+if [[ "${READTHEDOCS:-no}" == "no" ]]; then
+    echo "This script is only intended to be run in the ReadTheDocs CI environment"
+    exit 1
+fi
+
+SCHIZO_SRC_DIR=3rd-party/prrte/src/mca/schizo/ompi
+SCHIZO_TARGET_DIR=docs/schizo-ompi-rst-content
+
+PRRTE_RST_SRC_DIR=3rd-party/prrte/src/docs/prrte-rst-content
+PRRTE_RST_TARGET_DIR=docs/prrte-rst-content
+
+# Copy the OMPI schizo file from PRRTE
+#
+# See lengthy comment in docs/Makefile.am about copying in RST files
+# from PRRTE for a longer explanation of what is happening here.
+
+cp -rp $SCHIZO_SRC_DIR $SCHIZO_TARGET_DIR
+cp -rp $PRRTE_RST_SRC_DIR $PRRTE_RST_TARGET_DIR
+
+cd docs
+python3 ./generate-mpi-man3-bindings.py --srcdir . --builddir .
diff --git a/.readthedocs.yaml b/.readthedocs.yaml
new file mode 100644
index 00000000000..2ba1fc07842
--- /dev/null
+++ b/.readthedocs.yaml
@@ -0,0 +1,31 @@
+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Currently, RTD needs to select an OS with OpenSSL>=1.1.1 because of
+# urllib3's dependence on that system library.  (alternately, pin urllib3<2
+# See https://github.com/urllib3/urllib3/issues/2168
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.10"
+  jobs:
+    # RTD doesn't run configure or make.  So we have to manually copy
+    # in the PRRTE RST files to docs/.
+    pre_create_environment:
+      - ./.readthedocs-pre-create-environment.sh
+
+python:
+  install:
+    - requirements: docs/requirements.txt
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+  fail_on_warning: true
+
+submodules:
+  include: all
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index c4069c3cfae..00000000000
--- a/.travis.yml
+++ /dev/null
@@ -1,82 +0,0 @@
-# Use "required" for sudo, because we want to use the "trusty" Debian
-# distro, which is (currently) only available in the legacy Travis
-# infrastructure (i.e., if we put "sudo: false" to use the new container-
-# based Travis infrastructure, then "trusty" is not available).  We
-# need the "trusty" distro because it has more recent versions of the
-# GNU Autotools (i.e., autogen.pl will fail if you use the regular
-# distro because the GNU Autotools are too old).
-sudo: required
-dist: trusty
-language: c
-
-# Iterate over 2 different compilers
-compiler:
-    - gcc
-    - clang
-
-# Test only linux now
-os:
-    - linux
-
-addons:
-    # For Linux, make sure we have some extra packages that we like to
-    # build with
-    apt:
-        packages:
-            - autoconf
-            - automake
-            - libtool
-            - libnl-3-200
-            - libnl-3-dev
-            - libnl-route-3-200
-            - libnl-route-3-dev
-            - libibverbs-dev
-            - librdmacm-dev
-            - libhwloc-dev
-        sources:
-            - ubuntu-toolchain-r-test
-
-env:
-    global:
-        - AM_MAKEFLAGS="-j4"
-        - LD_LIBRARY_PATH="$HOME/bogus/lib"
-    matrix:
-        - GCC_VERSION=default
-        - GCC_VERSION=6
-
-# Install dependencies for the verbs and usnic providers.  Open MPI is
-# not currently using the verbs provider in Libfabric, so we might as
-# well not build it.
-before_install:
-    - if [[ "GCC_VERSION" == "6" ]]; then COMPILERS="CC=gcc-6 CXX=g++-6 FC=gfortran-6"; fi
-    - export CONFIGURE_ARGS="--prefix=$HOME/bogus $COMPILERS CPPFLAGS=-I$HOME/bogus/include LDFLAGS=-L$HOME/bogus/lib" DISTCHECK_CONFIGURE_FLAGS="$CONFIGURE_ARGS"
-    - export DISTCHECK_CONFIGURE_FLAGS="$CONFIGURE_ARGS"
-    - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then git clone https://github.com/ofiwg/libfabric.git ; fi
-    - if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$GCC_VERSION" == "6" ]] ; then sudo apt-get --assume-yes install gcc-6 g++-6 gfortran-6; fi
-    - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then cd libfabric && ./autogen.sh && ./configure --prefix=$HOME/bogus --enable-usnic --disable-verbs $COMPILERS && make install && cd .. ; fi
-    - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update; brew upgrade automake || true; brew upgrade libtool || true; fi
-    - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew tap homebrew/science || true; brew install hwloc || true; brew upgrade hwloc || true ; fi
-    - if [[ "$TRAVIS_OS_NAME" == "osx" ]] && [[ "$GCC_VERSION" == "6" ]] ; then brew install gcc || true; brew upgrade gcc || true ; fi
-    - mkdir -p $HOME/bogus/include $HOME/bogus/lib
-
-# Note that we use "make -k" to do the entire build, even if there was a
-# build error in there somewhere.  This prevents us from needing to submit
-# to Travis, see the first error, fix that first error, submit again, ...etc.
-install:
-    - m4 --version
-    - autoconf --version
-    - automake --version
-    - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then libtool --version; else glibtool --version; fi
-    - ./autogen.pl
-    - if [[ "$TRAVIS_OS_NAME" == "linux" && "$CC" == "gcc" ]]; then ./configure $CONFIGURE_ARGS --with-libfabric=$HOME/bogus --with-usnic --with-verbs; else ./configure $CONFIGURE_ARGS; fi
-    - make -k
-
-# We only need to distcheck on one OS / compiler combination (this is just
-# a minor optimization to make the overall set of builds faster).
-script:
-    - if [[ "$TRAVIS_OS_NAME" == "linux" && "$CC" == "gcc" ]]; then make distcheck; else make check; fi
-
-matrix:
-  exclude:
-    - env: GCC_VERSION=6
-      compiler: clang
diff --git a/3rd-party/Makefile.am b/3rd-party/Makefile.am
index 0fdb5d43d3d..41ace54f31a 100644
--- a/3rd-party/Makefile.am
+++ b/3rd-party/Makefile.am
@@ -1,6 +1,7 @@
 #
 # Copyright (c) 2020      Amazon.com, Inc. or its affiliates.
 #                         All Rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -16,7 +17,7 @@
 
 SUBDIRS = $(OPAL_3RDPARTY_SUBDIRS)
 DIST_SUBDIRS = $(OPAL_3RDPARTY_DIST_SUBDIRS)
-EXTRA_DIST =  $(OPAL_3RDPARTY_EXTRA_DIST) autogen.subdirs
+EXTRA_DIST =  $(OPAL_3RDPARTY_EXTRA_DIST) autogen.subdirs pympistandard
 
 distclean-local:
 	rm -rf $(OPAL_3RDPARTY_DISTCLEAN_DIRS)
@@ -26,3 +27,12 @@ check:
 
 check-recursive:
 	@echo "auto-recursing into 3rd-party packages for check disabled"
+
+# We recursively copied the pympystandard directory.  As suggested by
+# the Automake docs (see section 14.3, "The dist Hook"), use the
+# dist-hook target to delete a bunch of extra stuff that we might have
+# copied.
+dist-hook:
+	@find $(distdir)/pympistandard -name __pycache__ -exec rm -rf {} \; || :
+	@find $(distdir)/pympistandard -name \*.pyc -exec rm -rf {} \; || :
+	@rm -rf $(distdir)/pympistandard/.git
diff --git a/3rd-party/autogen.subdirs b/3rd-party/autogen.subdirs
index 01e147cbb6e..e69de29bb2d 100644
--- a/3rd-party/autogen.subdirs
+++ b/3rd-party/autogen.subdirs
@@ -1 +0,0 @@
-romio321
diff --git a/3rd-party/hwloc-2.4.0.tar.gz b/3rd-party/hwloc-2.4.0.tar.gz
deleted file mode 100644
index da43f233082..00000000000
Binary files a/3rd-party/hwloc-2.4.0.tar.gz and /dev/null differ
diff --git a/3rd-party/hwloc-2.7.1.tar.gz b/3rd-party/hwloc-2.7.1.tar.gz
new file mode 100644
index 00000000000..d627caba9e4
Binary files /dev/null and b/3rd-party/hwloc-2.7.1.tar.gz differ
diff --git a/3rd-party/libevent-2.1.12-stable-ompi.tar.gz b/3rd-party/libevent-2.1.12-stable-ompi.tar.gz
new file mode 100644
index 00000000000..9a2e9694a3a
Binary files /dev/null and b/3rd-party/libevent-2.1.12-stable-ompi.tar.gz differ
diff --git a/3rd-party/libevent-2.1.12-stable.tar.gz b/3rd-party/libevent-2.1.12-stable.tar.gz
deleted file mode 100644
index 4fcefff24f8..00000000000
Binary files a/3rd-party/libevent-2.1.12-stable.tar.gz and /dev/null differ
diff --git a/3rd-party/openpmix b/3rd-party/openpmix
index 6d4bb1634f9..08e41ed5629 160000
--- a/3rd-party/openpmix
+++ b/3rd-party/openpmix
@@ -1 +1 @@
-Subproject commit 6d4bb1634f992ea34c4e8dd6eba1541020c6830c
+Subproject commit 08e41ed5629b51832f5708181af6d89218c7a74e
diff --git a/3rd-party/prrte b/3rd-party/prrte
index fed9394d227..30cadc6746e 160000
--- a/3rd-party/prrte
+++ b/3rd-party/prrte
@@ -1 +1 @@
-Subproject commit fed9394d227f974b27992677982ca6808b8e60bd
+Subproject commit 30cadc6746ebddd69ea42ca78b964398f782e4e3
diff --git a/3rd-party/pympistandard b/3rd-party/pympistandard
new file mode 160000
index 00000000000..7bc6bb0ff96
--- /dev/null
+++ b/3rd-party/pympistandard
@@ -0,0 +1 @@
+Subproject commit 7bc6bb0ff9630542fb0030ac4d976bef1f1cb026
diff --git a/3rd-party/romio321/.codingcheck b/3rd-party/romio321/.codingcheck
deleted file mode 100644
index 9b52b35ce42..00000000000
--- a/3rd-party/romio321/.codingcheck
+++ /dev/null
@@ -1,55 +0,0 @@
-# Here are names that at least at one point were used within ROMIO.
-# We should look at these and decide which we wish to allow and which 
-# should be replaced with something more ROMIO-specific.
-%romioDefines = ( 'ROMIO_[A-Za-z0-9_]+' => romio, 
-	'PROFILE' => romio, 
-	'PRINT_ERR_MSG' => romio, 
-	'HPUX' => romio,
-	'SPPUX'=> romio,
-	'SX4'=> romio,
-	'AIO_SUN'=> romio,
-	'AIO_HANDLE_IN_AIOCB'=> romio,
-	'NO_FD_IN_AIOCB'=> romio,
-	'NO_AIO'=> romio,
-	'AIO_PRIORITY_DEFAULT'=> romio,
-	'AIO_SIGNOTIFY_NONE'=> romio,
-	'MPISGI'=> romio,
-	'CRAY'=> romio,
-	'PARAGON'=> romio,
-	'FREEBSD'=> romio,
-	'LINUX'=> romio,
-	'tflops'=> romio,
-	'NFS'=> romio,
-	'XFS'=> romio,
-	'CB_CONFIG_LIST_DEBUG'=> romio,
-	'SFS'=> romio,
-	'HFS'=> romio,
-	'UFS'=> romio,
-	'PVFS_.+' => romio,
-	'MPI_hpux'=> romio,
-	'FORTRANCAPS'=> romio,
-	'MPILAM'=> romio,
-	'NEEDS_ADIOCB_T'=> romio,
-	'AGG_DEBUG'=> romio,
-	'SOLARIS'=> romio,
-	'IRIX'=> romio,
-	'AIX'=> romio,
-	'DEC'=> romio,
-	'NEEDS_MPI_TEST'=> romio,
-	'PFS'=> romio,
-	'PIOFS'=> romio,
-	'MPICH'=> romio,
-	'MPICH' => romio,
-	'MPI_OFFSET_IS_INT'=> romio,
-	'MPI_COMBINER_NAMED'=> romio,
-	'_UNICOS'=> romio,
-	'MPIHP'=> romio,
-	);
-
-# Only invoke this function if the function is defined (in case the
-# user removed the cpp defines check with -rmchecks=cppdefines)
-if (defined(&PushDefinesNames)) {
-   &PushDefinesNames( "romioDefines", "tree", "add" );
-}
-	
-1;
diff --git a/3rd-party/romio321/.config_params b/3rd-party/romio321/.config_params
deleted file mode 100644
index fcc2f9146d1..00000000000
--- a/3rd-party/romio321/.config_params
+++ /dev/null
@@ -1,39 +0,0 @@
-__sun4_
-__rs6000_
-__paragon_
-__solaris_
-__solaris86_
-__tflop_
-__tflops_
-__hpux_
-__sppux_
-__SX4_
-__sgi_
-__sgi5_
-__IRIX_
-__IRIX32_
-__IRIXN32_
-__IRIX64_
-__alpha_
-__ALPHA_
-__freebsd_
-__netbsd_
-__LINUX_
-__LINUX_ALPHA_
-__CRAY_
-__Darwin_
-__nfs_
-__ufs_
-__pfs_
-__piofs_
-__pvfs_
-__testfs_
-__xfs_
-__hfs_
-__sfs_
-__mpich_mpi
-__sgi_mpi
-__hp_mpi
-__cray_mpi
-__lam_mpi
-__open_mpi
diff --git a/3rd-party/romio321/.gitignore b/3rd-party/romio321/.gitignore
deleted file mode 100644
index 28f1e98219d..00000000000
--- a/3rd-party/romio321/.gitignore
+++ /dev/null
@@ -1,13 +0,0 @@
-/Makefile
-/.deps
-/*.bb
-/*.bbg
-/*.gcda
-/*.gcno
-/.libs
-/.libstamp*
-/*.lo
-/.*-cache
-.state-cache
-version.m4
-confdb/config.rpath
diff --git a/3rd-party/romio321/COPYRIGHT b/3rd-party/romio321/COPYRIGHT
deleted file mode 100644
index 609bcfa4e84..00000000000
--- a/3rd-party/romio321/COPYRIGHT
+++ /dev/null
@@ -1,41 +0,0 @@
-                               COPYRIGHT
-
-The following is a notice of limited availability of the code and
-disclaimer, which must be included in the prologue of the code and in
-all source listings of the code.
-
-Copyright (C) 1997 University of Chicago
-
-Permission is hereby granted to use, reproduce, prepare derivative
-works, and to redistribute to others. 
-
-The University of Chicago makes no representations as to the suitability, 
-operability, accuracy, or correctness of this software for any purpose. 
-It is provided "as is" without express or implied warranty. 
-
-This software was authored by:
-Rajeev Thakur: (630) 252-1682; thakur@mcs.anl.gov
-Mathematics and Computer Science Division
-Argonne National Laboratory, Argonne IL 60439, USA
-
-
-                           GOVERNMENT LICENSE
-
-Portions of this material resulted from work developed under a U.S.
-Government Contract and are subject to the following license: the
-Government is granted for itself and others acting on its behalf a
-paid-up, nonexclusive, irrevocable worldwide license in this computer
-software to reproduce, prepare derivative works, and perform publicly
-and display publicly. 
-
-                              DISCLAIMER
-
-This computer code material was prepared, in part, as an account of
-work sponsored by an agency of the United States Government.  Neither
-the United States Government, nor the University of Chicago, nor any
-of their employees, makes any warranty express or implied, or assumes
-any legal liability or responsibility for the accuracy, completeness,
-or usefulness of any information, apparatus, product, or process
-disclosed, or represents that its use would not infringe privately
-owned rights.  
-
diff --git a/3rd-party/romio321/Makefile.am b/3rd-party/romio321/Makefile.am
deleted file mode 100644
index 71d82a9fa73..00000000000
--- a/3rd-party/romio321/Makefile.am
+++ /dev/null
@@ -1,195 +0,0 @@
-# -*- Mode: Makefile; -*-
-# Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-# (C) 2011 by Argonne National Laboratory.
-#     See COPYRIGHT in top-level directory.
-#
-
-# OMPI: include a top level makefile with some options
-include $(top_srcdir)/Makefile.options
-
-## TODO: need to write an automakefile that handles two primary cases:
-## 1) that ROMIO is being embedded within the MPI library, as in MPICH or Open
-##    MPI
-## 2) that ROMIO is being built standalone, old-school style.  This case is
-##    basically unused in modern practice.
-
-# help autoreconf and friends realize where the macros live
-ACLOCAL_AMFLAGS = -I confdb
-
-# empty variable initializations so that later code can append (+=)
-include_HEADERS = 
-nodist_include_HEADERS =
-noinst_HEADERS = 
-EXTRA_DIST =
-SUFFIXES = 
-doc1_src_txt =
-
-# ------------------------------------------------------------------------
-# variables to be populated by the included Makefile.mk fragments:
-
-# These are files that contain MPI routines (e.g., MPI_File_open).
-# In MPICH these will have an MPI_ and a PMPI_ version.  Other implementations
-# (like OMPI) only want these to be MPI_ routines, possibly with some
-# name-shifting prefix.
-romio_mpi_sources = 
-
-# regular old source files that implement ROMIO, such as ADIO code
-romio_other_sources =
-
-# code that may need to be "up" called from the MPI library and/or is
-# MPI-implementation-specific in some way
-glue_sources = 
-
-# ------------------------------------------------------------------------
-# when building under MPICH we must be able to find mpi.h
-AM_CPPFLAGS += $(MPI_H_INCLUDE)
-
-# ------------------------------------------------------------------------
-# handle the "include" directory here
-AM_CPPFLAGS += -I$(top_builddir)/include -I$(top_srcdir)/include
-# nodist_ b/c these are created by config.status and should not be distributed
-# Open MPI: do not install mpio.h
-noinst_HEADERS += include/mpio.h
-noinst_HEADERS += include/io_romio_conv.h
-
-# Included for Open MPI's --enable-grequest-extensions feature.
-noinst_HEADERS += include/ompi_grequestx.h
-
-# ------------------------------------------------------------------------
-
-SUBDIRS =
-DIST_SUBDIRS = test test-internal
-
-# for the sake of parallel make and avoiding an excessive number of convenience
-# libs, we use a subdir automake fragment strategy
-include mpi-io/Makefile.mk
-include adio/Makefile.mk
-
-EXTRA_DIST += autogen.sh
-
-if BUILD_ROMIO_EMBEDDED
-# Build a libtool convenience library that the enclosing MPI implementation can
-# use by adding it to the right _LIBADD variable.
-noinst_LTLIBRARIES = libromio_dist.la
-libromio_dist_la_SOURCES = $(romio_mpi_sources) $(romio_other_sources) $(glue_sources)
-
-## NOTE: ROMIO's old build system builds a bunch of _foo.o objects that contain
-## PMPI_ implementations as well as calls to only other PMPI routines.  In
-## MPICH, these are the objects that need to go into libmpi, while the foo.o
-## objects should go into libpmpi.  Furthermore, the -D option for ROMIO's
-## source files is different and inverted (in the boolean sense) compared with
-## MPICH's defintion.  And ROMIO was dumping all of the symbols into the main
-## libmpi library, regardless of the separate profiling library's existence.
-##
-## Annoying, right?
-if BUILD_PROFILING_LIB
-# The current best strategy for now is to build the PMPI symbols as a separate
-# convenience lib to permit adding the special "-D..." argument for all objects.
-# MPICH will then link in both convenience library into libmpi, since it
-# won't work very well the other way around.
-noinst_LTLIBRARIES += libpromio.la
-libpromio_la_SOURCES = $(romio_mpi_sources)
-libpromio_la_CPPFLAGS = $(AM_CPPFLAGS) -DMPIO_BUILD_PROFILING 
-endif BUILD_PROFILING_LIB
-
-else !BUILD_ROMIO_EMBEDDED
-lib_LTLIBRARIES = libromio.la
-libromio_la_SOURCES = $(romio_mpi_sources) $(romio_other_sources) $(glue_sources)
-if BUILD_PROFILING_LIB
-libpromio_la_SOURCES = $(romio_mpi_sources)
-libpromio_la_CPPFLAGS = $(AM_CPPFLAGS) -DMPIO_BUILD_PROFILING
-endif BUILD_PROFILING_LIB
-
-endif
-
-# --------------------------------------------------------------------------
-.PHONY: coverage
-gcov_sources = $(libmpl_la_SOURCES)
-# assumes that these sources were compiled appropriately ("-fprofile-arcs"
-# and "-ftest-coverage")
-coverage:
-	@for file in $(gcov_sources) ; do \
-	    dir=`dirname $$file` ; \
-	    bname=`basename $$file` ; \
-	    aux=`echo $$bname | sed -e 's,\.*$$,,'` ; \
-	    echo "( $(GCOV) -b -f -o $$file $$file && mv $${bname}.gcov $$dir )" ; \
-	    ( $(GCOV) -b -f -o $$file $$file && mv $${bname}.gcov $$dir ) ; \
-	    rm -f *.gcov ; \
-	done
-	for subdir in $(SUBDIRS) - ; do \
-	    if test $$subdir = "-" ; then break ; fi ; \
-	    ( cd $$subdir && $(MAKE) $(AM_MAKEFLAGS) coverage ) ; \
-	done
-# --------------------------------------------------------------------------
-.PHONY: mandoc mandoc-local htmldoc htmldoc-local
-SUFFIXES += .man-phony .html-phony .man1-phony .html1-phony .txt
-
-# "make V=1" support for our documentation recipes
-doctextman_verbose = $(doctextman_verbose_$(V))
-doctextman_verbose_ = $(doctextman_verbose_$(AM_DEFAULT_VERBOSITY))
-doctextman_verbose_0 = @echo "   DOCTEXTMAN  " $@;
-doctexthtml_verbose = $(doctexthtml_verbose_$(V))
-doctexthtml_verbose_ = $(doctexthtml_verbose_$(AM_DEFAULT_VERBOSITY))
-doctexthtml_verbose_0 = @echo "   DOCTEXTHTML  " $@;
-
-# Build dir paths where the man pages will be created.  Will usually be
-# overridden by MPICH make.
-mandoc_path1=$(abs_top_builddir)/man/man1
-mandoc_path3=$(abs_top_builddir)/man/man3
-htmldoc_path1=$(abs_top_builddir)/www/www1
-htmldoc_path3=$(abs_top_builddir)/www/www3
-doctext_docnotes=
-# Provide an easily replaced url root for the generated index file.
-# You can override this with URL desired in the index file generated by doctext.
-# You can ignore this if you don't use mapnames or tohtml to add links
-# to the MPI manual pages to documents.
-htmldoc_root3="--your-url-here--"
-
-.c.man-phony:
-	$(doctextman_verbose)$(DOCTEXT) -man -mpath $(mandoc_path3) -ext 3 \
-	    -heading MPI -quotefmt -nolocation $(doctext_docnotes) $<
-.c.html-phony:
-	$(doctexthtml_verbose)$(DOCTEXT) -html -mpath $(htmldoc_path3) \
-	    -heading MPI -quotefmt -nolocation \
-	    -index $(htmldoc_path3)/mpi.cit -indexdir $(htmldoc_root3) \
-		$(doctext_docnotes) $<
-
-.txt.man1-phony:
-	$(doctextman_verbose)$(DOCTEXT) -man -mpath $(mandoc_path1) -ext 1 \
-	    -heading MPI -quotefmt -nolocation $(doctext_docnotes) $<
-.txt.html1-phony:
-	$(doctexthtml_verbose)$(DOCTEXT) -html -mpath $(htmldoc_path1) \
-	    -heading MPI -quotefmt -nolocation $(doctext_docnotes) $<
-
-# use mandoc-local target to force directory creation before running DOCTEXT
-mandoc:
-	test -d $(mandoc_path1) || $(MKDIR_P) $(mandoc_path1)
-	test -d $(mandoc_path3) || $(MKDIR_P) $(mandoc_path3)
-	$(MAKE) $(AM_MAKEFLAGS) mandoc-local
-mandoc-local: $(romio_mpi_sources:.c=.man-phony) $(doc1_src_txt:.txt=.man1-phony)
-
-# use htmldoc-local target to force directory creation before running DOCTEXT
-htmldoc:
-	test -d $(top_builddir)/www/www1 || $(MKDIR_P) $(top_builddir)/www/www1
-	test -d $(top_builddir)/www/www3 || $(MKDIR_P) $(top_builddir)/www/www3
-	$(MAKE) $(AM_MAKEFLAGS) htmldoc-local
-htmldoc-local: $(romio_mpi_sources:.c=.html-phony) $(doc1_src_txt:.txt=.html1-phony)
-
-# --------------------------------------------------------------------------
-
diff --git a/3rd-party/romio321/Makefile.options b/3rd-party/romio321/Makefile.options
deleted file mode 100644
index 0b72829e152..00000000000
--- a/3rd-party/romio321/Makefile.options
+++ /dev/null
@@ -1,36 +0,0 @@
-# -*- makefile -*-
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-AUTOMAKE_OPTIONS	= foreign dist-bzip2
-
-# $(OMPI_TOP_SRCDIR)                - mca_base_param.h
-# $(OMPI_TOP_SRCDIR)/opal/include   - opal_config_bottom.h
-# $(OMPI_TOP_BUILDDIR)/opal/include - opal_config.h
-# $(OMPI_TOP_BUILDDIR)/ompi/include - mpi.h
-# $(top_srcdir)/include             - vpath support
-# $(top_srcdir)/adio/include        - vpath support
-
-AM_CPPFLAGS = \
-        -DOMPI_BUILDING=1 \
-        -I$(OMPI_TOP_SRCDIR) \
-        -I$(OMPI_TOP_SRCDIR)/opal/include \
-        -I$(OMPI_TOP_BUILDDIR)/opal/include \
-        -I$(OMPI_TOP_BUILDDIR)/ompi/include \
-	-I$(top_srcdir)/include \
-	-I$(top_srcdir)/adio/include
diff --git a/3rd-party/romio321/README b/3rd-party/romio321/README
deleted file mode 100644
index a6fb25a09a5..00000000000
--- a/3rd-party/romio321/README
+++ /dev/null
@@ -1,660 +0,0 @@
-          ROMIO: A High-Performance, Portable MPI-IO Implementation
-
-                      Version 2008-03-09
-
-Major Changes in this version:
-------------------------------
-* Fixed performance problems with the darray and subarray datatypes
-  when using MPICH.
-
-* Better support for building against existing MPICH and MPICH versions.
-
-  When building against an existing MPICH installation, use the
-  "--with-mpi=mpich" option to ROMIO configure.  For MPICH, use the
-  "--with-mpi=mpich" option.  These will allow ROMIO to take advantage
-  of internal features of these implementations.
-
-* Deprecation of SFS, HFS, and PIOFS implementations.
-
-  These are no longer actively supported, although the code will continue
-  to be distributed for now.
-
-* Initial support for the Panasas PanFS filesystem.
-
-  PanFS allows users to specify the layout of a file at file-creation time.
-  Layout information includes the number of StorageBlades (SB)
-  across which the data is stored, the number of SBs across which a 
-  parity stripe is written, and the number of consecutive stripes that 
-  are placed on the same set of SBs.   The panfs_layout_* hints are only 
-  used if supplied at file-creation time.
- 
-  panfs_layout_type - Specifies the layout of a file:
-                      2 = RAID0
-                      3 = RAID5 Parity Stripes
-  panfs_layout_stripe_unit - The size of the stripe unit in bytes
-  panfs_layout_total_num_comps - The total number of StorageBlades a file
-                                 is striped across.
-  panfs_layout_parity_stripe_width - If the layout type is RAID5 Parity
-                                     Stripes, this hint specifies the 
-                                     number of StorageBlades in a parity
-                                     stripe.
-  panfs_layout_parity_stripe_depth - If the layout type is RAID5 Parity
-                                     Stripes, this hint specifies the
-                                     number of contiguous parity stripes written 
-                                     across the same set of SBs.
-  panfs_layout_visit_policy - If the layout type is RAID5 Parity Stripes, 
-                              the policy used to determine the parity 
-                              stripe a given file offset is written to:
-                              1 = Round Robin
-
-  PanFS supports the "concurrent write" (CW) mode, where groups of cooperating 
-  clients can disable the PanFS consistency mechanisms and use their own 
-  consistency protocol.  Clients participating in concurrent write mode use 
-  application specific information to improve performance while maintaining 
-  file consistency.  All clients accessing the file(s) must enable concurrent 
-  write mode.  If any client does not enable concurrent write mode, then the 
-  PanFS consistency protocol will be invoked.  Once a file is opened in CW mode 
-  on a machine, attempts to open a file in non-CW mode will fail with 
-  EACCES.  If a file is already opened in non-CW mode, attempts to open 
-  the file in CW mode will fail with EACCES.  The following hint is 
-  used to enable concurrent write mode.
-
-  panfs_concurrent_write - If set to 1 at file open time, the file 
-                           is opened using the PanFS concurrent write 
-                           mode flag.  Concurrent write mode is not a
-                           persistent attribute of the file.
-
-  Below is an example PanFS layout using the following parameters:
- 
-  - panfs_layout_type                = 3
-  - panfs_layout_total_num_comps     = 100
-  - panfs_layout_parity_stripe_width = 10
-  - panfs_layout_parity_stripe_depth = 8
-  - panfs_layout_visit_policy        = 1
-
-   Parity Stripe Group 1     Parity Stripe Group 2  . . . Parity Stripe Group 10
-  ----------------------    ----------------------        --------------------
-   SB1    SB2  ... SB10     SB11    SB12 ...  SB20  ...   SB91   SB92 ... SB100
-  -----------------------   -----------------------       ---------------------
-   D1     D2   ...  D10      D91    D92  ...  D100        D181   D182  ... D190
-   D11    D12       D20      D101   D102      D110        D191   D192      D193
-   D21    D22       D30      . . .                        . . .
-   D31    D32       D40
-   D41    D42       D50
-   D51    D52       D60
-   D61    D62       D70
-   D71    D72       D80
-   D81    D82       D90      D171   D172      D180        D261   D262   D270
-   D271   D272      D273     . . .                        . . .
-   ...
-
-* Initial support for the Globus GridFTP filesystem.  Work contributed by Troy
-  Baer (troy@osc.edu).  
-
-Major Changes in Version 1.2.5:
-------------------------------
-
-* Initial support for MPICH-2
-
-* fix for a bug in which ROMIO would get confused for some permutations
-  of the aggregator list
-
-* direct io on IRIX's XFS should work now
-
-* fixed an issue with the Fortran bindings that would cause them to fail
-   when some compilers tried to build them.
-
-* Initial support for deferred opens
-
-Major Changes in Version 1.2.4:
-------------------------------
-* Added section describing ROMIO MPI_FILE_SYNC and MPI_FILE_CLOSE behavior to 
-  User's Guide
-
-* Bug removed from PVFS ADIO implementation regarding resize operations
-
-* Added support for PVFS listio operations, including hints to control use
-
-
-Major Changes in Version 1.2.3:
--------------------------------
-* Enhanced aggregation control via cb_config_list, romio_cb_read,
-  and romio_cb_write hints
-
-* Asynchronous IO can be enabled under Linux with the --enable-aio argument
-  to configure
-
-* Additional PVFS support
-
-* Additional control over data sieving with romio_ds_read hint
-
-* NTFS ADIO implementation integrated into source tree
-
-* testfs ADIO implementation added for debugging purposes
-
-
-Major Changes in Version 1.0.3:
--------------------------------
-
-* When used with MPICH 1.2.1, the MPI-IO functions return proper error codes 
-  and classes, and the status object is filled in.
-
-* On SGI's XFS file system, ROMIO can use direct I/O even if the
-  user's request does not meet the various restrictions needed to use
-  direct I/O. ROMIO does this by doing part of the request with
-  buffered I/O (until all the restrictions are met) and doing the rest
-  with direct I/O. (This feature hasn't been tested rigorously. Please
-  check for errors.) 
-
-  By default, ROMIO will use only buffered I/O. Direct I/O can be
-  enabled either by setting the environment variables MPIO_DIRECT_READ
-  and/or MPIO_DIRECT_WRITE to TRUE, or on a per-file basis by using
-  the info keys "direct_read" and "direct_write". 
-
-  Direct I/O will result in higher performance only if you are
-  accessing a high-bandwidth disk system. Otherwise, buffered I/O is
-  better and is therefore used as the default.
-
-* Miscellaneous bug fixes.
-
-
-Major Changes Version 1.0.2:
----------------------------
-
-* Implemented the shared file pointer functions and
-  split collective I/O functions. Therefore, the main
-   components of the MPI I/O chapter not yet implemented are
-  file interoperability and error handling.
-
-* Added support for using "direct I/O" on SGI's XFS file system. 
-  Direct I/O is an optional feature of XFS in which data is moved
-  directly between the user's buffer and the storage devices, bypassing 
-  the file-system cache. This can improve performance significantly on 
-  systems with high disk bandwidth. Without high disk bandwidth,
-  regular I/O (that uses the file-system cache) perfoms better.
-  ROMIO, therefore, does not use direct I/O by default. The user can
-  turn on direct I/O (separately for reading and writing) either by
-  using environment variables or by using MPI's hints mechanism (info). 
-  To use the environment-variables method, do
-       setenv MPIO_DIRECT_READ TRUE
-       setenv MPIO_DIRECT_WRITE TRUE
-  To use the hints method, the two keys are "direct_read" and "direct_write". 
-  By default their values are "false". To turn on direct I/O, set the values 
-  to "true". The environment variables have priority over the info keys.
-  In other words, if the environment variables are set to TRUE, direct I/O
-  will be used even if the info keys say "false", and vice versa. 
-  Note that direct I/O must be turned on separately for reading 
-  and writing.
-  The environment-variables method assumes that the environment
-  variables can be read by each process in the MPI job. This is
-  not guaranteed by the MPI Standard, but it works with SGI's MPI
-  and the ch_shmem device of MPICH.
-
-* Added support (new ADIO device, ad_pvfs) for the PVFS parallel 
-  file system for Linux clusters, developed at Clemson University
-  (see http://www.parl.clemson.edu/pvfs ). To use it, you must first install
-  PVFS and then when configuring ROMIO, specify "-file_system=pvfs" in 
-  addition to any other options to "configure". (As usual, you can configure
-  for multiple file systems by using "+"; for example, 
-  "-file_system=pvfs+ufs+nfs".)  You will need to specify the path 
-  to the PVFS include files via the "-cflags" option to configure, 
-  for example, "configure -cflags=-I/usr/pvfs/include". You
-  will also need to specify the full path name of the PVFS library.
-  The best way to do this is via the "-lib" option to MPICH's 
-  configure script (assuming you are using ROMIO from within MPICH). 
-
-* Uses weak symbols (where available) for building the profiling version,
-  i.e., the PMPI routines. As a result, the size of the library is reduced
-  considerably. 
-
-* The Makefiles use "virtual paths" if supported by the make utility. GNU make
-  supports it, for example. This feature allows you to untar the
-  distribution in some directory, say a slow NFS directory,
-  and compile the library (the .o files) in another 
-  directory, say on a faster local disk. For example, if the tar file
-  has been untarred in an NFS directory called /home/thakur/romio,
-  one can compile it in a different directory, say /tmp/thakur, as follows:
-        cd /tmp/thakur
-        /home/thakur/romio/configure
-        make
-  The .o files will be created in /tmp/thakur; the library will be created in
-  /home/thakur/romio/lib/$ARCH/libmpio.a .
-  This method works only if the make utility supports virtual paths.
-  If the default make does not, you can install GNU make which does,
-  and specify it to configure as
-       /home/thakur/romio/configure -make=/usr/gnu/bin/gmake (or whatever)
-
-* Lots of miscellaneous bug fixes and other enhancements.
-
-* This version is included in MPICH 1.2.0. If you are using MPICH, you
-  need not download ROMIO separately; it gets built as part of MPICH.
-  The previous version of ROMIO is included in LAM, HP MPI, SGI MPI, and 
-  NEC MPI. NEC has also implemented the MPI-IO functions missing 
-  in ROMIO, and therefore NEC MPI has a complete implementation
-  of MPI-IO.
-
-
-Major Changes in Version 1.0.1:
-------------------------------
-
-* This version is included in MPICH 1.1.1 and HP MPI 1.4.
-
-* Added support for NEC SX-4 and created a new device ad_sfs for
-  NEC SFS file system.
-
-* New devices ad_hfs for HP/Convex HFS file system and ad_xfs for 
-  SGI XFS file system.
-
-* Users no longer need to prefix the filename with the type of 
-  file system; ROMIO determines the file-system type on its own.
-
-* Added support for 64-bit file sizes on IBM PIOFS, SGI XFS,
-  HP/Convex HFS, and NEC SFS file systems.
-
-* MPI_Offset is an 8-byte integer on machines that support 8-byte integers.
-  It is of type "long long" in C and "integer*8" in Fortran.
-  With a Fortran 90 compiler, you can use either integer*8 or
-  integer(kind=MPI_OFFSET_KIND). 
-  If you printf an MPI_Offset in C, remember to use %lld 
-  or %ld as required by your compiler. (See what is used in the test 
-  program romio/test/misc.c.)
-
-* On some machines, ROMIO detects at configure time that "long long" is 
-  either not supported by the C compiler or it doesn't work properly.
-  In such cases, configure sets MPI_Offset to long in C and integer in
-  Fortran. This happens on Intel Paragon, Sun4, and FreeBSD.
-
-* Added support for passing hints to the implementation via the MPI_Info 
-  parameter. ROMIO understands the following hints (keys in MPI_Info object):
-
-  /* on all file systems */ 
-     cb_buffer_size - buffer size for collective I/O
-     cb_nodes - no. of processes that actually perform I/O in collective I/O
-     ind_rd_buffer_size - buffer size for data sieving in independent reads
-
-  /* on all file systems except IBM PIOFS */
-     ind_wr_buffer_size - buffer size for data sieving in independent writes
-      /* ind_wr_buffer_size is ignored on PIOFS because data sieving 
-         cannot be done for writes since PIOFS doesn't support file locking */
-
-  /* on Intel PFS and IBM PIOFS only. These hints are understood only if
-     supplied at file-creation time. */
-     striping_factor - no. of I/O devices to stripe the file across
-     striping_unit - the striping unit in bytes
-     start_iodevice - the number of the I/O device from which to start
-                      striping (between 0 and (striping_factor-1))
-
-  /* on Intel PFS only. */
-     pfs_svr_buf - turn on or off PFS server buffering by setting the value 
-               to "true" or "false", case-sensitive.
-      
-  If ROMIO doesn't understand a hint, or if the value is invalid, the hint
-  will be ignored. The values of hints being used by ROMIO at any time 
-  can be obtained via MPI_File_get_info.
-
-
-
-General Information 
--------------------
-
-ROMIO is a high-performance, portable implementation of MPI-IO (the
-I/O chapter in MPI). ROMIO's home page is at
-http://www.mcs.anl.gov/romio .  The MPI standard is available at
-http://www.mpi-forum.org/docs/docs.html .
-
-This version of ROMIO includes everything defined in the MPI I/O
-chapter except support for file interoperability and
-user-defined error handlers for files.  The subarray and
-distributed array datatype constructor functions from Chapter 4
-(Sec. 4.14.4 & 4.14.5) have been implemented. They are useful for
-accessing arrays stored in files. The functions MPI_File_f2c and
-MPI_File_c2f (Sec. 4.12.4) are also implemented.
-
-C, Fortran, and profiling interfaces are provided for all functions
-that have been implemented. 
-
-Please read the limitations of this version of ROMIO that are listed
-below (e.g., MPIO_Request object, restriction to homogeneous
-environments).
-
-This version of ROMIO runs on at least the following machines: IBM SP;
-Intel Paragon; HP Exemplar; SGI Origin2000; Cray T3E; NEC SX-4; other
-symmetric multiprocessors from HP, SGI, DEC, Sun, and IBM; and networks of
-workstations (Sun, SGI, HP, IBM, DEC, Linux, and FreeBSD).  Supported
-file systems are IBM PIOFS, Intel PFS, HP/Convex HFS, SGI XFS, NEC
-SFS, PVFS, NFS, and any Unix file system (UFS).
-
-This version of ROMIO is included in MPICH 1.2.3; an earlier version
-is included in at least the following MPI implementations: LAM, HP
-MPI, SGI MPI, and NEC MPI.  
-
-Note that proper I/O error codes and classes are returned and the
-status variable is filled only when used with MPICH 1.2.1 or later.
-
-You can open files on multiple file systems in the same program. The
-only restriction is that the directory where the file is to be opened
-must be accessible from the process opening the file. For example, a
-process running on one workstation may not be able to access a
-directory on the local disk of another workstation, and therefore
-ROMIO will not be able to open a file in such a directory. NFS-mounted
-files can be accessed.
-
-An MPI-IO file created by ROMIO is no different than any other file
-created by the underlying file system. Therefore, you may use any of
-the commands provided by the file system to access the file, e.g., ls,
-mv, cp, rm, ftp.
-
-
-Using ROMIO on NFS
-------------------
-
-To use ROMIO on NFS, file locking with fcntl must work correctly on
-the NFS installation. On some installations, fcntl locks don't work.
-To get them to work, you need to use Version 3 of NFS, ensure that the
-lockd daemon is running on all the machines, and have the system
-administrator mount the NFS file system with the "noac" option (no
-attribute caching). Turning off attribute caching may reduce
-performance, but it is necessary for correct behavior.
-
-The following are some instructions we received from Ian Wells of HP
-for setting the noac option on NFS. We have not tried them
-ourselves. We are including them here because you may find 
-them useful. Note that some of the steps may be specific to HP
-systems, and you may need root permission to execute some of the
-commands. 
-   
-   >1. first confirm you are running nfs version 3
-   >
-   >rpcnfo -p `hostname` | grep nfs
-   >
-   >ie 
-   >    goedel >rpcinfo -p goedel | grep nfs
-   >    100003    2   udp   2049  nfs
-   >    100003    3   udp   2049  nfs
-   >
-   >
-   >2. then edit /etc/fstab for each nfs directory read/written by MPIO
-   >   on each  machine used for multihost MPIO.
-   >
-   >    Here is an example of a correct fstab entry for /epm1:
-   >
-   >   ie grep epm1 /etc/fstab
-   > 
-   >      ROOOOT 11>grep epm1 /etc/fstab
-   >      gershwin:/epm1 /rmt/gershwin/epm1 nfs bg,intr,noac 0 0
-   >
-   >   if the noac option is not present, add it 
-   >   and then remount this directory
-   >   on each of the machines that will be used to share MPIO files
-   >
-   >ie
-   >
-   >ROOOOT >umount /rmt/gershwin/epm1
-   >ROOOOT >mount  /rmt/gershwin/epm1
-   >
-   >3. Confirm that the directory is mounted noac:
-   >
-   >ROOOOT >grep gershwin /etc/mnttab 
-   >gershwin:/epm1 /rmt/gershwin/epm1 nfs
-   >noac,acregmin=0,acregmax=0,acdirmin=0,acdirmax=0 0 0 899911504
-
-
-
-
-ROMIO Installation Instructions
--------------------------------
-
-Since ROMIO is included in MPICH, LAM, HP MPI, SGI MPI, and NEC MPI,
-you don't need to install it separately if you are using any of these
-MPI implementations.  If you are using some other MPI, you can
-configure and build ROMIO as follows:
-
-Untar the tar file as
-
-    gunzip -c romio.tar.gz | tar xvf -
-
-OR
-
-    zcat romio.tar.Z | tar xvf -
-
-THEN
-
-    cd romio
-    ./configure
-    make
-
-Some example programs and a Makefile are provided in the romio/test directory.
-Run the examples the way you would run any MPI program. Each program takes 
-the filename as a command-line argument "-fname filename".  
-
-The configure script by default configures ROMIO for the file systems
-most likely to be used on the given machine. If you wish, you can
-explicitly specify the file systems by using the "-file_system" option
-to configure. Multiple file systems can be specified by using "+" as a
-separator. For example,
-
-    ./configure -file_system=xfs+nfs
-
-For the entire list of options to configure do
-
-    ./configure -h | more
-
-After building a specific version as above, you can install it in a
-particular directory with 
-
-    make install PREFIX=/usr/local/romio    (or whatever directory you like)
-
-or just
-
-    make install           (if you used -prefix at configure time)
-
-If you intend to leave ROMIO where you built it, you should NOT install it 
-(install is used only to move the necessary parts of a built ROMIO to 
-another location). The installed copy will have the include files,
-libraries, man pages, and a few other odds and ends, but not the whole
-source tree.  It will have a test directory for testing the
-installation and a location-independent Makefile built during
-installation, which users can copy and modify to compile and link
-against the installed copy. 
-
-To rebuild ROMIO with a different set of configure options, do
-
-    make distclean
-
-to clean everything including the Makefiles created by configure.
-Then run configure again with the new options, followed by make.
-
-
-
-Testing ROMIO
--------------
-
-To test if the installation works, do
-
-     make testing 
-
-in the romio/test directory. This calls a script that runs the test
-programs and compares the results with what they should be. By
-default, "make testing" causes the test programs to create files in
-the current directory and use whatever file system that corresponds
-to. To test with other file systems, you need to specify a filename in
-a directory corresponding to that file system as follows:
-
-     make testing TESTARGS="-fname=/foo/piofs/test"
-
-
-
-Compiling and Running MPI-IO Programs
--------------------------------------
-
-If ROMIO is not already included in the MPI implementation, you need
-to include the file mpio.h for C or mpiof.h for Fortran in your MPI-IO
-program.  
-
-Note that on HP machines running HPUX and on NEC SX-4, you need to
-compile Fortran programs with mpifort, because the f77 compilers on
-these machines don't support 8-byte integers. 
-
-With MPICH, HP MPI, or NEC MPI, you can compile MPI-IO programs as 
-   mpicc foo.c
-or 
-   mpif77 foo.f
-or
-   mpifort foo.f
-
-As mentioned above, mpifort is preferred over mpif77 on HPUX and NEC
-because the f77 compilers on those machines do not support 8-byte integers.
-
-With SGI MPI, you can compile MPI-IO programs as 
-   cc foo.c -lmpi
-or
-   f77 foo.f -lmpi
-or
-   f90 foo.f -lmpi
-
-With LAM, you can compile MPI-IO programs as 
-   hcc foo.c -lmpi
-or
-   hf77 foo.f -lmpi
-
-If you have built ROMIO with some other MPI implementation, you can
-compile MPI-IO programs by explicitly giving the path to the include
-file mpio.h or mpiof.h and explicitly specifying the path to the
-library libmpio.a, which is located in $(ROMIO_HOME)/lib/$(ARCH)/libmpio.a .
-
-
-Run the program as you would run any MPI program on the machine. If
-you use mpirun, make sure you use the correct mpirun for the MPI
-implementation you are using. For example, if you are using MPICH on
-an SGI machine, make sure that you use MPICH's mpirun and not SGI's
-mpirun.
-
-The Makefile in the romio/test directory illustrates how to compile
-and link MPI-IO programs. 
-
-
-
-Limitations of this version of ROMIO
-------------------------------------
-
-* When used with any MPI implementation other than MPICH 1.2.1 (or later),
-the "status" argument is not filled in any MPI-IO function. Consequently,
-MPI_Get_count and MPI_Get_elements will not work when passed the status
-object from an MPI-IO operation.
-
-* All nonblocking I/O functions use a ROMIO-defined "MPIO_Request"
-object instead of the usual "MPI_Request" object. Accordingly, two
-functions, MPIO_Test and MPIO_Wait, are provided to wait and test on
-these MPIO_Request objects. They have the same semantics as MPI_Test
-and MPI_Wait.
-
-int MPIO_Test(MPIO_Request *request, int *flag, MPI_Status *status);
-int MPIO_Wait(MPIO_Request *request, MPI_Status *status);
-
-The usual functions MPI_Test, MPI_Wait, MPI_Testany, etc., will not
-work for nonblocking I/O. 
-
-* This version works only on a homogeneous cluster of machines,
-and only the "native" file data representation is supported.
-
-* When used with any MPI implementation other than MPICH 1.2.1 (or later),
-all MPI-IO functions return only two possible error codes---MPI_SUCCESS
-on success and MPI_ERR_UNKNOWN on failure.
-
-* Shared file pointers are not supported on PVFS and IBM PIOFS file
-systems because they don't support fcntl file locks, and ROMIO uses
-that feature to implement shared file pointers.
-
-* On HP machines running HPUX and on NEC SX-4, you need to compile
-Fortran programs with mpifort instead of mpif77, because the f77
-compilers on these machines don't support 8-byte integers.
-
-* The file-open mode MPI_MODE_EXCL does not work on Intel PFS file system,
-due to a bug in PFS.
-
-
-
-Usage Tips
-----------
-
-* When using ROMIO with SGI MPI, you may sometimes get an error
-message from SGI MPI: ``MPI has run out of internal datatype
-entries. Please set the environment variable MPI_TYPE_MAX for
-additional space.'' If you get this error message, add this line to
-your .cshrc file:
-    setenv MPI_TYPE_MAX 65536 
-Use a larger number if you still get the error message.
-
-* If a Fortran program uses a file handle created using ROMIO's C
-interface, or vice-versa, you must use the functions MPI_File_c2f 
-or MPI_File_f2c. Such a situation occurs,
-for example, if a Fortran program uses an I/O library written in C
-with MPI-IO calls. Similar functions MPIO_Request_f2c and
-MPIO_Request_c2f are also provided.
-
-* For Fortran programs on the Intel Paragon, you may need
-to provide the complete path to mpif.h in the include statement, e.g.,
-        include '/usr/local/mpich/include/mpif.h'
-instead of 
-        include 'mpif.h'
-This is because the -I option to the Paragon Fortran compiler if77
-doesn't work correctly. It always looks in the default directories first 
-and, therefore, picks up Intel's mpif.h, which is actually the
-mpif.h of an older version of MPICH. 
-
-
-
-ROMIO Users Mailing List
-------------------------
-
-Please register your copy of ROMIO with us by sending email
-to majordomo@mcs.anl.gov with the message
-
-subscribe romio-users
-
-This will enable us to notify you of new releases of ROMIO as well as
-bug fixes.
-
-
-
-Reporting Bugs
---------------
-
-If you have trouble, first check the users guide (in
-romio/doc/users-guide.ps.gz). Then check the on-line list of known
-bugs and patches at http://www.mcs.anl.gov/romio .
-Finally, if you still have problems, send a detailed message containing:
-
-   The type of system (often, uname -a)
-   The output of configure
-   The output of make
-   Any programs or tests
-
-to romio-maint@mcs.anl.gov .
-
-
-
-ROMIO Internals
----------------
-
-A key component of ROMIO that enables such a portable MPI-IO
-implementation is an internal abstract I/O device layer called
-ADIO. Most users of ROMIO will not need to deal with the ADIO layer at
-all. However, ADIO is useful to those who want to port ROMIO to some
-other file system. The ROMIO source code and the ADIO paper
-(see doc/README) will help you get started.
-
-MPI-IO implementation issues are discussed in our IOPADS '99 paper,
-"On Implementing MPI-IO Portably and with High Performance."
-All ROMIO-related papers are available online from
-http://www.mcs.anl.gov/romio.
-
-
-Learning MPI-IO
----------------
-
-The book "Using MPI-2: Advanced Features of the Message-Passing
-Interface," published by MIT Press, provides a tutorial introduction to
-all aspects of MPI-2, including parallel I/O. It has lots of example
-programs. See http://www.mcs.anl.gov/mpi/usingmpi2 for further
-information about the book.
diff --git a/3rd-party/romio321/README_OMPI b/3rd-party/romio321/README_OMPI
deleted file mode 100644
index 83f1ffb2233..00000000000
--- a/3rd-party/romio321/README_OMPI
+++ /dev/null
@@ -1,11 +0,0 @@
-Please note that this is *NOT* a vanilla MPICH v3.2b1
-distribution of the ROMIO package from Argonne National Labs.
-Various customizations had to be applied to the configuration process.
-More to the point -- if replace this copy of ROMIO with a newer version,
-it will likely not work.  :-(
-
-- The Open MPI Team
-
------------------------------------------------------------------------------
-
-Local modifications are in ompi.patch
diff --git a/3rd-party/romio321/adio/Makefile.mk b/3rd-party/romio321/adio/Makefile.mk
deleted file mode 100644
index ffc05cb4151..00000000000
--- a/3rd-party/romio321/adio/Makefile.mk
+++ /dev/null
@@ -1,47 +0,0 @@
-## -*- Mode: Makefile; -*-
-## vim: set ft=automake :
-##
-## (C) 2011 by Argonne National Laboratory.
-##     See COPYRIGHT in top-level directory.
-##
-
-AM_CPPFLAGS += -I$(top_builddir)/adio/include -I$(top_srcdir)/adio/include
-
-noinst_HEADERS +=                      \
-    adio/include/adio.h                \
-    adio/include/adio_cb_config_list.h \
-    adio/include/adio_extern.h         \
-    adio/include/adioi.h               \
-    adio/include/adioi_errmsg.h        \
-    adio/include/adioi_error.h         \
-    adio/include/adioi_fs_proto.h      \
-    adio/include/heap-sort.h           \
-    adio/include/mpio_error.h          \
-    adio/include/mpipr.h               \
-    adio/include/mpiu_greq.h           \
-    adio/include/nopackage.h           \
-    adio/include/romioconf-undefs.h    \
-    adio/include/mpiu_external32.h     \
-    adio/include/hint_fns.h
-
-include $(top_srcdir)/adio/ad_gpfs/Makefile.mk
-include $(top_srcdir)/adio/ad_gpfs/bg/Makefile.mk
-include $(top_srcdir)/adio/ad_gpfs/pe/Makefile.mk
-include $(top_srcdir)/adio/ad_gridftp/Makefile.mk
-include $(top_srcdir)/adio/ad_hfs/Makefile.mk
-include $(top_srcdir)/adio/ad_lustre/Makefile.mk
-include $(top_srcdir)/adio/ad_nfs/Makefile.mk
-## NTFS builds are handled entirely by the separate Windows build system
-##include $(top_srcdir)/adio/ad_ntfs/Makefile.mk
-include $(top_srcdir)/adio/ad_panfs/Makefile.mk
-include $(top_srcdir)/adio/ad_pfs/Makefile.mk
-include $(top_srcdir)/adio/ad_piofs/Makefile.mk
-include $(top_srcdir)/adio/ad_pvfs/Makefile.mk
-include $(top_srcdir)/adio/ad_pvfs2/Makefile.mk
-include $(top_srcdir)/adio/ad_sfs/Makefile.mk
-include $(top_srcdir)/adio/ad_testfs/Makefile.mk
-include $(top_srcdir)/adio/ad_ufs/Makefile.mk
-include $(top_srcdir)/adio/ad_xfs/Makefile.mk
-include $(top_srcdir)/adio/ad_zoidfs/Makefile.mk
-include $(top_srcdir)/adio/common/Makefile.mk
-
diff --git a/3rd-party/romio321/adio/ad_gpfs/.gitignore b/3rd-party/romio321/adio/ad_gpfs/.gitignore
deleted file mode 100644
index 509a693e927..00000000000
--- a/3rd-party/romio321/adio/ad_gpfs/.gitignore
+++ /dev/null
@@ -1,11 +0,0 @@
-/Makefile
-/.deps
-/*.bb
-/*.bbg
-/*.gcda
-/*.gcno
-/.libs
-/.libstamp*
-/*.lo
-/.*-cache
-/.state-cache
diff --git a/3rd-party/romio321/adio/ad_gpfs/Makefile.mk b/3rd-party/romio321/adio/ad_gpfs/Makefile.mk
deleted file mode 100644
index db8737be5c7..00000000000
--- a/3rd-party/romio321/adio/ad_gpfs/Makefile.mk
+++ /dev/null
@@ -1,26 +0,0 @@
-## -*- Mode: Makefile; -*-
-## vim: set ft=automake :
-##
-## (C) 2012 by Argonne National Laboratory.
-##     See COPYRIGHT in top-level directory.
-##
-
-if BUILD_AD_GPFS
-
-noinst_HEADERS +=                                                    \
-    adio/ad_gpfs/ad_gpfs_aggrs.h                                         \
-    adio/ad_gpfs/ad_gpfs.h                                               \
-    adio/ad_gpfs/ad_gpfs_tuning.h
-
-romio_other_sources +=                                               \
-    adio/ad_gpfs/ad_gpfs_aggrs.c                                         \
-    adio/ad_gpfs/ad_gpfs_close.c                                         \
-    adio/ad_gpfs/ad_gpfs_flush.c                                         \
-    adio/ad_gpfs/ad_gpfs_tuning.c                                        \
-    adio/ad_gpfs/ad_gpfs.c                                               \
-    adio/ad_gpfs/ad_gpfs_open.c                                          \
-    adio/ad_gpfs/ad_gpfs_hints.c                                         \
-    adio/ad_gpfs/ad_gpfs_rdcoll.c                                        \
-    adio/ad_gpfs/ad_gpfs_wrcoll.c
-
-endif BUILD_AD_GPFS
diff --git a/3rd-party/romio321/adio/ad_gpfs/ad_gpfs.c b/3rd-party/romio321/adio/ad_gpfs/ad_gpfs.c
deleted file mode 100644
index f376150f0ba..00000000000
--- a/3rd-party/romio321/adio/ad_gpfs/ad_gpfs.c
+++ /dev/null
@@ -1,63 +0,0 @@
-/* ---------------------------------------------------------------- */
-/* (C)Copyright IBM Corp.  2007, 2008                               */
-/* ---------------------------------------------------------------- */
-/**
- * \file ad_gpfs.c
- * \brief ???
- */
-
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/* 
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-#include "ad_gpfs.h"
-
-/* adioi.h has the ADIOI_Fns_struct define */
-#include "adioi.h"
-
-struct ADIOI_Fns_struct ADIO_GPFS_operations = {
-    ADIOI_GPFS_Open, /* Open */
-    ADIOI_GEN_OpenColl, /* Collective open */
-    ADIOI_GEN_ReadContig, /* ReadContig */
-    ADIOI_GEN_WriteContig, /* WriteContig */
-    ADIOI_GPFS_ReadStridedColl, /* ReadStridedColl */
-    ADIOI_GPFS_WriteStridedColl, /* WriteStridedColl */
-    ADIOI_GEN_SeekIndividual, /* SeekIndividual */
-    ADIOI_GEN_Fcntl, /* Fcntl */
-#if defined(BGQPLATFORM) || defined(PEPLATFORM)
-    ADIOI_GPFS_SetInfo, /* SetInfo for BlueGene or PE */
-#else
-    ADIOI_GEN_SetInfo, /* SetInfo for any platform besides BlueGene or PE */
-#endif
-    ADIOI_GEN_ReadStrided, /* ReadStrided */
-    ADIOI_GEN_WriteStrided, /* WriteStrided */
-    ADIOI_GPFS_Close, /* Close */
-#ifdef ROMIO_HAVE_WORKING_AIO
-#warning Consider BG support for NFS before enabling this.
-    ADIOI_GEN_IreadContig, /* IreadContig */
-    ADIOI_GEN_IwriteContig, /* IwriteContig */
-#else
-    ADIOI_FAKE_IreadContig, /* IreadContig */
-    ADIOI_FAKE_IwriteContig, /* IwriteContig */
-#endif
-    ADIOI_GEN_IODone, /* ReadDone */
-    ADIOI_GEN_IODone, /* WriteDone */
-    ADIOI_GEN_IOComplete, /* ReadComplete */
-    ADIOI_GEN_IOComplete, /* WriteComplete */
-    ADIOI_GEN_IreadStrided, /* IreadStrided */
-    ADIOI_GEN_IwriteStrided, /* IwriteStrided */
-    ADIOI_GPFS_Flush, /* Flush */
-    ADIOI_GEN_Resize, /* Resize */
-    ADIOI_GEN_Delete, /* Delete */
-    ADIOI_GEN_Feature, /* Features */
-#ifdef BGQPLATFORM
-    "GPFS+BGQ: IBM GPFS for Blue Gene",
-#elif PEPLATFORM
-    "GPFS+PE: IBM GPFS for PE",
-#else
-    "GPFS: IBM GPFS",
-#endif
-    ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
-    ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
-};
diff --git a/3rd-party/romio321/adio/ad_gpfs/ad_gpfs.h b/3rd-party/romio321/adio/ad_gpfs/ad_gpfs.h
deleted file mode 100644
index d64dcaeb998..00000000000
--- a/3rd-party/romio321/adio/ad_gpfs/ad_gpfs.h
+++ /dev/null
@@ -1,73 +0,0 @@
-/* ---------------------------------------------------------------- */
-/* (C)Copyright IBM Corp.  2007, 2008                               */
-/* ---------------------------------------------------------------- */
-/**
- * \file ad_gpfs.h
- * \brief ???
- */
-
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#ifndef AD_GPFS_INCLUDE
-#define AD_GPFS_INCLUDE
-
-#include <unistd.h>
-#include <stdlib.h>
-#include <sys/types.h>
-#include <fcntl.h>
-#include "adio.h"
-
-#ifdef HAVE_SIGNAL_H
-#include <signal.h>
-#endif
-#ifdef HAVE_AIO_LITE_H
-#include <aio-lite.h>
-#elif defined  HAVE_AIO_H
-#include <aio.h>
-#endif
-
-
-void ADIOI_GPFS_Open(ADIO_File fd, int *error_code);
-
-void ADIOI_GPFS_Close(ADIO_File fd, int *error_code);
-
-void ADIOI_GPFS_ReadContig(ADIO_File fd, void *buf, int count,
-                      MPI_Datatype datatype, int file_ptr_type,
-                     ADIO_Offset offset, ADIO_Status *status, int
-		     *error_code);
-void ADIOI_GPFS_WriteContig(ADIO_File fd, const void *buf, int count,
-                      MPI_Datatype datatype, int file_ptr_type,
-                      ADIO_Offset offset, ADIO_Status *status, int
-		      *error_code);
-
-void ADIOI_GPFS_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code);
-
-void ADIOI_GPFS_WriteStrided(ADIO_File fd, const void *buf, int count,
-		       MPI_Datatype datatype, int file_ptr_type,
-		       ADIO_Offset offset, ADIO_Status *status, int
-		       *error_code);
-void ADIOI_GPFS_ReadStrided(ADIO_File fd, void *buf, int count,
-		       MPI_Datatype datatype, int file_ptr_type,
-		       ADIO_Offset offset, ADIO_Status *status, int
-		       *error_code);
-
-void ADIOI_GPFS_ReadStridedColl(ADIO_File fd, void *buf, int count,
-                               MPI_Datatype datatype, int file_ptr_type,
-                               ADIO_Offset offset, ADIO_Status *status, int
-                               *error_code);
-
-void ADIOI_GPFS_WriteStridedColl(ADIO_File fd, const void *buf, int count,
-                       MPI_Datatype datatype, int file_ptr_type,
-                       ADIO_Offset offset, ADIO_Status *status, int
-                       *error_code);
-
-void ADIOI_GPFS_Flush(ADIO_File fd, int *error_code);
-
-#include "ad_gpfs_tuning.h"
-
-
-#endif
diff --git a/3rd-party/romio321/adio/ad_gpfs/ad_gpfs_aggrs.c b/3rd-party/romio321/adio/ad_gpfs/ad_gpfs_aggrs.c
deleted file mode 100644
index f6df24748f0..00000000000
--- a/3rd-party/romio321/adio/ad_gpfs/ad_gpfs_aggrs.c
+++ /dev/null
@@ -1,817 +0,0 @@
-/* ---------------------------------------------------------------- */
-/* (C)Copyright IBM Corp.  2007, 2008, 2019                         */
-/* ---------------------------------------------------------------- */
-/**
- * \file ad_gpfs_aggrs.c
- * \brief The externally used function from this file is is declared in ad_gpfs_aggrs.h
- */
-
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/*
- *   Copyright (C) 1997-2001 University of Chicago.
- *   See COPYRIGHT notice in top-level directory.
- */
-
-
-#include "adio.h"
-#include "adio_cb_config_list.h"
-#include "ad_gpfs.h"
-#include "ad_gpfs_aggrs.h"
-
-#ifdef AGGREGATION_PROFILE
-#include "mpe.h"
-#endif
-
-
-#ifdef USE_DBG_LOGGING
-  #define AGG_DEBUG 1
-#endif
-
-#ifndef TRACE_ERR
-#  define TRACE_ERR(format...)
-#endif
-
-/* Comments copied from common:
- * This file contains four functions:
- *
- * ADIOI_Calc_aggregator()
- * ADIOI_Calc_file_domains()
- * ADIOI_Calc_my_req()
- * ADIOI_Calc_others_req()
- *
- * The last three of these were originally in ad_read_coll.c, but they are
- * also shared with ad_write_coll.c.  I felt that they were better kept with
- * the rest of the shared aggregation code.
- */
-
-/* Discussion of values available from above:
- *
- * ADIO_Offset st_offsets[0..nprocs-1]
- * ADIO_Offset end_offsets[0..nprocs-1]
- *    These contain a list of start and end offsets for each process in
- *    the communicator.  For example, an access at loc 10, size 10 would
- *    have a start offset of 10 and end offset of 19.
- * int nprocs
- *    number of processors in the collective I/O communicator
- * ADIO_Offset min_st_offset
- * ADIO_Offset fd_start[0..nprocs_for_coll-1]
- *    starting location of "file domain"; region that a given process will
- *    perform aggregation for (i.e. actually do I/O)
- * ADIO_Offset fd_end[0..nprocs_for_coll-1]
- *    start + size - 1 roughly, but it can be less, or 0, in the case of
- *    uneven distributions
- */
-
-/* Description from common/ad_aggregate.c.  (Does it completely apply to bg?)
- * ADIOI_Calc_aggregator()
- *
- * The intention here is to implement a function which provides basically
- * the same functionality as in Rajeev's original version of
- * ADIOI_Calc_my_req().  He used a ceiling division approach to assign the
- * file domains, and we use the same approach here when calculating the
- * location of an offset/len in a specific file domain.  Further we assume
- * this same distribution when calculating the rank_index, which is later
- *  used to map to a specific process rank in charge of the file domain.
- *
- * A better (i.e. more general) approach would be to use the list of file
- * domains only.  This would be slower in the case where the
- * original ceiling division was used, but it would allow for arbitrary
- * distributions of regions to aggregators.  We'd need to know the
- * nprocs_for_coll in that case though, which we don't have now.
- *
- * Note a significant difference between this function and Rajeev's old code:
- * this code doesn't necessarily return a rank in the range
- * 0..nprocs_for_coll; instead you get something in 0..nprocs.  This is a
- * result of the rank mapping; any set of ranks in the communicator could be
- * used now.
- *
- * Returns an integer representing a rank in the collective I/O communicator.
- *
- * The "len" parameter is also modified to indicate the amount of data
- * actually available in this file domain.
- */
-/*
- * This is more general aggregator search function which does not base on the assumption
- * that each aggregator hosts the file domain with the same size
- */
-int ADIOI_GPFS_Calc_aggregator(ADIO_File fd,
-			      ADIO_Offset off,
-			      ADIO_Offset min_off,
-			      ADIO_Offset *len,
-			      ADIO_Offset fd_size,
-			      ADIO_Offset *fd_start,
-			      ADIO_Offset *fd_end)
-{
-    int rank_index, rank;
-    ADIO_Offset avail_bytes;
-    TRACE_ERR("Entering ADIOI_GPFS_Calc_aggregator\n");
-
-    ADIOI_Assert ( (off <= fd_end[fd->hints->cb_nodes-1] && off >= min_off && fd_start[0] >= min_off ) );
-
-    /* binary search --> rank_index is returned */
-    int ub = fd->hints->cb_nodes;
-    int lb = 0;
-    /* get an index into our array of aggregators */
-    /* Common code for striping - bg doesn't use it but it's
-       here to make diff'ing easier.
-    rank_index = (int) ((off - min_off + fd_size)/ fd_size - 1);
-
-    if (fd->hints->striping_unit > 0) {
-        * wkliao: implementation for file domain alignment
-           fd_start[] and fd_end[] have been aligned with file lock
-	   boundaries when returned from ADIOI_Calc_file_domains() so cannot
-	   just use simple arithmatic as above *
-        rank_index = 0;
-        while (off > fd_end[rank_index]) rank_index++;
-    }
-    bg does it's own striping below
-    */
-    rank_index = fd->hints->cb_nodes / 2;
-    while ( off < fd_start[rank_index] || off > fd_end[rank_index] ) {
-	if ( off > fd_end  [rank_index] ) {
-	    lb = rank_index;
-	    rank_index = (rank_index + ub) / 2;
-	}
-	else
-	if ( off < fd_start[rank_index] ) {
-	    ub = rank_index;
-	    rank_index = (rank_index + lb) / 2;
-	}
-    }
-    /* we index into fd_end with rank_index, and fd_end was allocated to be no
-     * bigger than fd->hins->cb_nodes.   If we ever violate that, we're
-     * overrunning arrays.  Obviously, we should never ever hit this abort */
-    if (rank_index >= fd->hints->cb_nodes || rank_index < 0) {
-        FPRINTF(stderr, "Error in ADIOI_Calc_aggregator(): rank_index(%d) >= fd->hints->cb_nodes (%d) fd_size=%lld off=%lld\n",
-			rank_index,fd->hints->cb_nodes,fd_size,off);
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-    /* DBG_FPRINTF ("ADIOI_GPFS_Calc_aggregator: rank_index = %d\n",
-       rank_index ); */
-
-    /*
-     * remember here that even in Rajeev's original code it was the case that
-     * different aggregators could end up with different amounts of data to
-     * aggregate.  here we use fd_end[] to make sure that we know how much
-     * data this aggregator is working with.
-     *
-     * the +1 is to take into account the end vs. length issue.
-     */
-    avail_bytes = fd_end[rank_index] + 1 - off;
-    if (avail_bytes < *len && avail_bytes > 0) {
-        /* this file domain only has part of the requested contig. region */
-
-        *len = avail_bytes;
-    }
-
-    /* map our index to a rank */
-    /* NOTE: FOR NOW WE DON'T HAVE A MAPPING...JUST DO 0..NPROCS_FOR_COLL */
-    rank = fd->hints->ranklist[rank_index];
-    TRACE_ERR("Leaving ADIOI_GPFS_Calc_aggregator\n");
-
-    return rank;
-}
-
-/*
- * Compute a dynamic access range based file domain partition among I/O aggregators,
- * which align to the GPFS block size
- * Divide the I/O workload among "nprocs_for_coll" processes. This is
- * done by (logically) dividing the file into file domains (FDs); each
- * process may directly access only its own file domain.
- * Additional effort is to make sure that each I/O aggregator get
- * a file domain that aligns to the GPFS block size.  So, there will
- * not be any false sharing of GPFS file blocks among multiple I/O nodes.
- *
- * The common version of this now accepts a min_fd_size and striping_unit.
- * It doesn't seem necessary here (using GPFS block sizes) but keep it in mind
- * (e.g. we could pass striping unit instead of using fs_ptr->blksize).
- */
-void ADIOI_GPFS_Calc_file_domains(ADIO_File fd,
-	                              ADIO_Offset *st_offsets,
-                                      ADIO_Offset *end_offsets,
-                                      int          nprocs,
-                                      int          nprocs_for_coll,
-                                      ADIO_Offset *min_st_offset_ptr,
-                                      ADIO_Offset **fd_start_ptr,
-                                      ADIO_Offset **fd_end_ptr,
-                                      ADIO_Offset *fd_size_ptr,
-                                      void        *fs_ptr)
-{
-    ADIO_Offset min_st_offset, max_end_offset, *fd_start, *fd_end, *fd_size;
-    int i, aggr;
-    TRACE_ERR("Entering ADIOI_GPFS_Calc_file_domains\n");
-    blksize_t blksize;
-
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5004, 0, NULL);
-#endif
-
-#   if AGG_DEBUG
-    static char myname[] = "ADIOI_GPFS_Calc_file_domains";
-    DBG_FPRINTF(stderr, "%s(%d): %d aggregator(s)\n",
-	    myname,__LINE__,nprocs_for_coll);
-#   endif
-    if (fd->blksize <= 0)
-	/* default to 1M if blksize unset */
-	fd->blksize = 1048576;
-    blksize = fd->blksize;
-
-#   if AGG_DEBUG
-    DBG_FPRINTF(stderr,"%s(%d): Blocksize=%ld\n",myname,__LINE__,blksize);
-#   endif
-/* find min of start offsets and max of end offsets of all processes */
-    min_st_offset  = st_offsets [0];
-    max_end_offset = end_offsets[0];
-    for (i=1; i<nprocs; i++) {
-        min_st_offset = ADIOI_MIN(min_st_offset, st_offsets[i]);
-        max_end_offset = ADIOI_MAX(max_end_offset, end_offsets[i]);
-    }
-
-    /* DBG_FPRINTF(stderr, "_calc_file_domains, min_st_offset, max_
-       = %qd, %qd\n", min_st_offset, max_end_offset );*/
-
-    /* determine the "file domain (FD)" of each process, i.e., the portion of
-       the file that will be "owned" by each process */
-
-    ADIO_Offset gpfs_ub       = (max_end_offset +blksize-1) / blksize * blksize - 1;
-    ADIO_Offset gpfs_lb       = min_st_offset / blksize * blksize;
-    ADIO_Offset gpfs_ub_rdoff = (max_end_offset +blksize-1) / blksize * blksize - 1 - max_end_offset;
-    ADIO_Offset gpfs_lb_rdoff = min_st_offset - min_st_offset / blksize * blksize;
-    ADIO_Offset fd_gpfs_range = gpfs_ub - gpfs_lb + 1;
-
-    int         naggs    = nprocs_for_coll;
-
-    /* Tweak the file domains so that no fd is smaller than a threshold.  We
-     * have to strike a balance between efficency and parallelism: somewhere
-     * between 10k processes sending 32-byte requests and one process sending a
-     * 320k request is a (system-dependent) sweet spot
-
-    This is from the common code - the new min_fd_size parm that we didn't implement.
-    (And common code uses a different declaration of fd_size so beware)
-
-    if (fd_size < min_fd_size)
-        fd_size = min_fd_size;
-    */
-    fd_size              = (ADIO_Offset *) ADIOI_Malloc(nprocs_for_coll * sizeof(ADIO_Offset));
-    *fd_start_ptr        = (ADIO_Offset *) ADIOI_Malloc(nprocs_for_coll * sizeof(ADIO_Offset));
-    *fd_end_ptr          = (ADIO_Offset *) ADIOI_Malloc(nprocs_for_coll * sizeof(ADIO_Offset));
-    fd_start             = *fd_start_ptr;
-    fd_end               = *fd_end_ptr;
-
-    /* each process will have a file domain of some number of gpfs blocks, but
-     * the division of blocks is not likely to be even.  Some file domains will
-     * be "large" and others "small"
-     *
-     * Example: consider  17 blocks distributed over 3 aggregators.
-     * nb_cn_small = 17/3 = 5
-     * naggs_large = 17 - 3*(17/3) = 17 - 15  = 2
-     * naggs_small = 3 - 2 = 1
-     *
-     * and you end up with file domains of {5-blocks, 6-blocks, 6-blocks}
-     *
-     * what about (relatively) small files?  say, a file of 1000 blocks
-     * distributed over 2064 aggregators:
-     * nb_cn_small = 1000/2064 = 0
-     * naggs_large = 1000 - 2064*(1000/2064) = 1000
-     * naggs_small = 2064 - 1000 = 1064
-     * and you end up with domains of {0, 0, 0, ... 1, 1, 1 ...}
-     *
-     * it might be a good idea instead of having all the zeros up front, to
-     * "mix" those zeros into the fd_size array.  that way, no pset/bridge-set
-     * is left with zero work.  In fact, even if the small file domains aren't
-     * zero, it's probably still a good idea to mix the "small" file domains
-     * across the fd_size array to keep the io nodes in balance */
-
-
-    ADIO_Offset n_gpfs_blk    = fd_gpfs_range / blksize;
-    ADIO_Offset nb_cn_small   = n_gpfs_blk/naggs;
-    ADIO_Offset naggs_large   = n_gpfs_blk - naggs * (n_gpfs_blk/naggs);
-    ADIO_Offset naggs_small   = naggs - naggs_large;
-
-#ifdef BGQPLATFORM
-    if (gpfsmpio_balancecontig == 1) {
-	/* File domains blocks are assigned to aggregators in a breadth-first
-	 * fashion relative to the ions - additionally, file domains on the
-	 * aggregators sharing the same bridgeset and ion have contiguous
-	 * offsets. */
-
-	// initialize everything to small
-	for (i=0; i<naggs; i++)
-	    fd_size[i] = nb_cn_small     * blksize;
-
-	// go thru and distribute the large across the bridges
-
-	/* bridelistoffset: agg rank list offsets using the bridgelist - each
-	 * entry is created by adding up the indexes for the aggs from all
-	 * previous bridges */
-	int *bridgelistoffset =
-	    (int *) ADIOI_Malloc(fd->hints->fs_hints.bg.numbridges*sizeof(int));
-	/* tmpbridgelistnum: copy of the bridgelistnum whose entries can be
-	 * decremented to keep track of bridge assignments during the actual
-	 * large block assignments to the agg rank list*/
-	int *tmpbridgelistnum =
-	    (int *) ADIOI_Malloc(fd->hints->fs_hints.bg.numbridges*sizeof(int));
-
-	int j;
-	for (j=0;j<fd->hints->fs_hints.bg.numbridges;j++) {
-	    int k, bridgerankoffset = 0;
-	    for (k=0;k<j;k++) {
-		bridgerankoffset += fd->hints->fs_hints.bg.bridgelistnum[k];
-	    }
-	    bridgelistoffset[j] = bridgerankoffset;
-	}
-
-	for (j=0;j<fd->hints->fs_hints.bg.numbridges;j++)
-	    tmpbridgelistnum[j] = fd->hints->fs_hints.bg.bridgelistnum[j];
-	int bridgeiter = 0;
-
-	/* distribute the large blocks across the aggs going breadth-first
-	 * across the bridgelist - this distributes the fd sizes across the
-	 * ions, so later in the file domain assignment when it iterates thru
-	 * the ranklist the offsets will be contiguous within the bridge and
-	 * ion as well */
-	for (j=0;j<naggs_large;j++) {
-	    int foundbridge = 0;
-	    int numbridgelistpasses = 0;
-	    while (!foundbridge) {
-		if (tmpbridgelistnum[bridgeiter] > 0) {
-		    foundbridge = 1;
-		    /*
-		       printf("bridgeiter is %d tmpbridgelistnum[bridgeiter] is %d bridgelistoffset[bridgeiter] is %d\n",bridgeiter,tmpbridgelistnum[bridgeiter],bridgelistoffset[bridgeiter]);
-		       printf("naggs is %d bridgeiter is %d bridgelistoffset[bridgeiter] is %d tmpbridgelistnum[bridgeiter] is %d\n",naggs, bridgeiter,bridgelistoffset[bridgeiter],tmpbridgelistnum[bridgeiter]);
-		       printf("naggs is %d bridgeiter is %d setting fd_size[%d]\n",naggs, bridgeiter,bridgelistoffset[bridgeiter]+(fd->hints->bridgelistnum[bridgeiter]-tmpbridgelistnum[bridgeiter]));
-		     */
-		    int currentbridgelistnum =
-			(fd->hints->fs_hints.bg.bridgelistnum[bridgeiter]-
-			 tmpbridgelistnum[bridgeiter]);
-		    int currentfdsizeindex = bridgelistoffset[bridgeiter] +
-			currentbridgelistnum;
-		    fd_size[currentfdsizeindex] = (nb_cn_small+1) * blksize;
-		    tmpbridgelistnum[bridgeiter]--;
-		}
-		if (bridgeiter == (fd->hints->fs_hints.bg.numbridges-1)) {
-		    /* guard against infinite loop - should only ever make 1 pass
-		     * thru bridgelist */
-		    ADIOI_Assert(numbridgelistpasses == 0);
-		    numbridgelistpasses++;
-		    bridgeiter = 0;
-		}
-		else
-		    bridgeiter++;
-	    }
-	}
-	ADIOI_Free(tmpbridgelistnum);
-	ADIOI_Free(bridgelistoffset);
-
-    } else {
-	/* BG/L- and BG/P-style distribution of file domains: simple allocation of
-	 * file domins to each aggregator */
-	for (i=0; i<naggs; i++) {
-	    if (i < naggs_large) {
-		fd_size[i] = (nb_cn_small+1) * blksize;
-	    } else {
-		fd_size[i] = nb_cn_small     * blksize;
-	    }
-	}
-    }
-#ifdef balancecontigtrace
-    int myrank;
-    MPI_Comm_rank(fd->comm,&myrank);
-    if (myrank == 0) {
-      fprintf(stderr,"naggs_small is %d nb_cn_small is %d\n",naggs_small,nb_cn_small);
-	for (i=0; i<naggs; i++) {
-	    fprintf(stderr,"fd_size[%d] set to %d agg rank is %d\n",i,fd_size[i],fd->hints->ranklist[i]);
-	}
-    }
-#endif
-
-#else // not BGQ platform
-	for (i=0; i<naggs; i++) {
-	    if (i < naggs_large) {
-		fd_size[i] = (nb_cn_small+1) * blksize;
-	    } else {
-		fd_size[i] = nb_cn_small     * blksize;
-	    }
-    }
-
-#endif
-
-
-#   if AGG_DEBUG
-     DBG_FPRINTF(stderr,"%s(%d): "
-                   "gpfs_ub       %llu, "
-                   "gpfs_lb       %llu, "
-                   "gpfs_ub_rdoff %llu, "
-                   "gpfs_lb_rdoff %llu, "
-                   "fd_gpfs_range %llu, "
-                   "n_gpfs_blk    %llu, "
-                   "nb_cn_small   %llu, "
-                   "naggs_large   %llu, "
-                   "naggs_small   %llu, "
-                   "\n",
-                   myname,__LINE__,
-                   gpfs_ub      ,
-                   gpfs_lb      ,
-                   gpfs_ub_rdoff,
-                   gpfs_lb_rdoff,
-                   fd_gpfs_range,
-                   n_gpfs_blk   ,
-                   nb_cn_small  ,
-                   naggs_large  ,
-                   naggs_small
-                   );
-#   endif
-
-    fd_size[0]       -= gpfs_lb_rdoff;
-    fd_size[naggs-1] -= gpfs_ub_rdoff;
-
-    /* compute the file domain for each aggr */
-    ADIO_Offset offset = min_st_offset;
-    for (aggr=0; aggr<naggs; aggr++) {
-        fd_start[aggr] = offset;
-        fd_end  [aggr] = offset + fd_size[aggr] - 1;
-        offset += fd_size[aggr];
-    }
-
-    *fd_size_ptr = fd_size[0];
-    *min_st_offset_ptr = min_st_offset;
-
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5005, 0, NULL);
-#endif
-    ADIOI_Free (fd_size);
-    TRACE_ERR("Leaving ADIOI_GPFS_Calc_file_domains\n");
-}
-
-/*
- * ADIOI_GPFS_Calc_my_req() overrides ADIOI_Calc_my_req for the default implementation
- * is specific for static file domain partitioning.
- *
- * ADIOI_Calc_my_req() - calculate what portions of the access requests
- * of this process are located in the file domains of various processes
- * (including this one)
- */
-void ADIOI_GPFS_Calc_my_req(ADIO_File fd, ADIO_Offset *offset_list, ADIO_Offset *len_list,
-			   int contig_access_count, ADIO_Offset
-			   min_st_offset, ADIO_Offset *fd_start,
-			   ADIO_Offset *fd_end, ADIO_Offset fd_size,
-			   int nprocs,
-			   int *count_my_req_procs_ptr,
-			   int **count_my_req_per_proc_ptr,
-			   ADIOI_Access **my_req_ptr,
-			   int **buf_idx_ptr)
-/* Possibly reconsider if buf_idx's are ok as int's, or should they be aints/offsets?
-   They are used as memory buffer indices so it seems like the 2G limit is in effect */
-{
-    int *count_my_req_per_proc, count_my_req_procs, *buf_idx;
-    int i, l, proc;
-    ADIO_Offset fd_len, rem_len, curr_idx, off;
-    ADIOI_Access *my_req;
-    TRACE_ERR("Entering ADIOI_GPFS_Calc_my_req\n");
-
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5024, 0, NULL);
-#endif
-    *count_my_req_per_proc_ptr = (int *) ADIOI_Calloc(nprocs,sizeof(int));
-    count_my_req_per_proc = *count_my_req_per_proc_ptr;
-/* count_my_req_per_proc[i] gives the no. of contig. requests of this
-   process in process i's file domain. calloc initializes to zero.
-   I'm allocating memory of size nprocs, so that I can do an
-   MPI_Alltoall later on.*/
-
-    buf_idx = (int *) ADIOI_Malloc(nprocs*sizeof(int));
-/* buf_idx is relevant only if buftype_is_contig.
-   buf_idx[i] gives the index into user_buf where data received
-   from proc. i should be placed. This allows receives to be done
-   without extra buffer. This can't be done if buftype is not contig. */
-
-    /* initialize buf_idx to -1 */
-    for (i=0; i < nprocs; i++) buf_idx[i] = -1;
-
-    /* one pass just to calculate how much space to allocate for my_req;
-     * contig_access_count was calculated way back in ADIOI_Calc_my_off_len()
-     */
-    for (i=0; i < contig_access_count; i++) {
-	/* short circuit offset/len processing if len == 0
-	 * 	(zero-byte  read/write */
-	if (len_list[i] == 0)
-		continue;
-	off = offset_list[i];
-	fd_len = len_list[i];
-	/* note: we set fd_len to be the total size of the access.  then
-	 * ADIOI_Calc_aggregator() will modify the value to return the
-	 * amount that was available from the file domain that holds the
-	 * first part of the access.
-	 */
-  /* BES */
-	proc = ADIOI_GPFS_Calc_aggregator(fd, off, min_st_offset, &fd_len, fd_size,
-				     fd_start, fd_end);
-	count_my_req_per_proc[proc]++;
-
-	/* figure out how much data is remaining in the access (i.e. wasn't
-	 * part of the file domain that had the starting byte); we'll take
-	 * care of this data (if there is any) in the while loop below.
-	 */
-	rem_len = len_list[i] - fd_len;
-
-	while (rem_len > 0) {
-	    off += fd_len; /* point to first remaining byte */
-	    fd_len = rem_len; /* save remaining size, pass to calc */
-	    proc = ADIOI_GPFS_Calc_aggregator(fd, off, min_st_offset, &fd_len,
-					 fd_size, fd_start, fd_end);
-
-	    count_my_req_per_proc[proc]++;
-	    rem_len -= fd_len; /* reduce remaining length by amount from fd */
-	}
-    }
-
-/* now allocate space for my_req, offset, and len */
-
-    *my_req_ptr = (ADIOI_Access *)
-	ADIOI_Malloc(nprocs*sizeof(ADIOI_Access));
-    my_req = *my_req_ptr;
-
-    count_my_req_procs = 0;
-    for (i=0; i < nprocs; i++) {
-	if (count_my_req_per_proc[i]) {
-	    my_req[i].offsets = (ADIO_Offset *)
-		ADIOI_Malloc(count_my_req_per_proc[i] * sizeof(ADIO_Offset));
-	    my_req[i].lens =
-		ADIOI_Malloc(count_my_req_per_proc[i] * sizeof(ADIO_Offset));
-	    count_my_req_procs++;
-	}
-	my_req[i].count = 0;  /* will be incremented where needed
-				      later */
-    }
-
-/* now fill in my_req */
-    curr_idx = 0;
-    for (i=0; i<contig_access_count; i++) {
-	/* short circuit offset/len processing if len == 0
-	 * 	(zero-byte  read/write */
-	if (len_list[i] == 0)
-		continue;
-	off = offset_list[i];
-	fd_len = len_list[i];
-	proc = ADIOI_GPFS_Calc_aggregator(fd, off, min_st_offset, &fd_len, fd_size,
-				     fd_start, fd_end);
-
-	/* for each separate contiguous access from this process */
-	if (buf_idx[proc] == -1)
-  {
-    ADIOI_Assert(curr_idx == (int) curr_idx);
-    buf_idx[proc] = (int) curr_idx;
-  }
-
-	l = my_req[proc].count;
-	curr_idx += fd_len;
-
-	rem_len = len_list[i] - fd_len;
-
-	/* store the proc, offset, and len information in an array
-         * of structures, my_req. Each structure contains the
-         * offsets and lengths located in that process's FD,
-	 * and the associated count.
-	 */
-	my_req[proc].offsets[l] = off;
-	my_req[proc].lens[l] = fd_len;
-	my_req[proc].count++;
-
-	while (rem_len > 0) {
-	    off += fd_len;
-	    fd_len = rem_len;
-	    proc = ADIOI_GPFS_Calc_aggregator(fd, off, min_st_offset, &fd_len,
-					 fd_size, fd_start, fd_end);
-
-	    if (buf_idx[proc] == -1)
-      {
-        ADIOI_Assert(curr_idx == (int) curr_idx);
-        buf_idx[proc] = (int) curr_idx;
-      }
-
-	    l = my_req[proc].count;
-	    curr_idx += fd_len;
-	    rem_len -= fd_len;
-
-	    my_req[proc].offsets[l] = off;
-	    my_req[proc].lens[l] = fd_len;
-	    my_req[proc].count++;
-	}
-    }
-
-
-
-#ifdef AGG_DEBUG
-    for (i=0; i<nprocs; i++) {
-	if (count_my_req_per_proc[i] > 0) {
-	    DBG_FPRINTF(stderr, "data needed from %d (count = %d):\n", i,
-		    my_req[i].count);
-	    for (l=0; l < my_req[i].count; l++) {
-		DBG_FPRINTF(stderr, "   off[%d] = %lld, len[%d] = %lld\n", l,
-			my_req[i].offsets[l], l, my_req[i].lens[l]);
-	    }
-	}
-	DBG_FPRINTF(stderr, "buf_idx[%d] = 0x%x\n", i, buf_idx[i]);
-    }
-#endif
-
-    *count_my_req_procs_ptr = count_my_req_procs;
-    *buf_idx_ptr = buf_idx;
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5025, 0, NULL);
-#endif
-    TRACE_ERR("Leaving ADIOI_GPFS_Calc_my_req\n");
-}
-
-/*
- * ADIOI_Calc_others_req (copied to bg and switched to all to all for performance)
- *
- * param[in]  count_my_req_procs        Number of processes whose file domain my
- *                                        request touches.
- * param[in]  count_my_req_per_proc     count_my_req_per_proc[i] gives the no. of
- *                                        contig. requests of this process in
- *                                        process i's file domain.
- * param[in]  my_req                    A structure defining my request
- * param[in]  nprocs                    Number of nodes in the block
- * param[in]  myrank                    Rank of this node
- * param[out] count_others_req_proc_ptr Number of processes whose requests lie in
- *                                        my process's file domain (including my
- *                                        process itself)
- * param[out] others_req_ptr            Array of other process' requests that lie
- *                                        in my process's file domain
- */
-void ADIOI_GPFS_Calc_others_req(ADIO_File fd, int count_my_req_procs,
-				int *count_my_req_per_proc,
-				ADIOI_Access *my_req,
-				int nprocs, int myrank,
-				int *count_others_req_procs_ptr,
-				ADIOI_Access **others_req_ptr)
-{
-    TRACE_ERR("Entering ADIOI_GPFS_Calc_others_req\n");
-/* determine what requests of other processes lie in this process's
-   file domain */
-
-/* count_others_req_procs = number of processes whose requests lie in
-   this process's file domain (including this process itself)
-   count_others_req_per_proc[i] indicates how many separate contiguous
-   requests of proc. i lie in this process's file domain. */
-
-    int *count_others_req_per_proc, count_others_req_procs;
-    int i;
-    ADIOI_Access *others_req;
-
-    /* Parameters for MPI_Alltoallv */
-    int *scounts, *sdispls, *rcounts, *rdispls;
-
-/* first find out how much to send/recv and from/to whom */
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5026, 0, NULL);
-#endif
-    /* Send 1 int to each process.  count_my_req_per_proc[i] is the number of
-     * requests that my process will do to the file domain owned by process[i].
-     * Receive 1 int from each process.  count_others_req_per_proc[i] is the number of
-     * requests that process[i] will do to the file domain owned by my process.
-     */
-    count_others_req_per_proc = (int *) ADIOI_Malloc(nprocs*sizeof(int));
-/*     cora2a1=timebase(); */
-/*for(i=0;i<nprocs;i++) ?*/
-    MPI_Alltoall(count_my_req_per_proc, 1, MPI_INT,
-		 count_others_req_per_proc, 1, MPI_INT, fd->comm);
-
-/*     total_cora2a+=timebase()-cora2a1; */
-
-    /* Allocate storage for an array of other nodes' accesses of our
-     * node's file domain.  Also allocate storage for the alltoallv
-     * parameters.
-     */
-    *others_req_ptr = (ADIOI_Access *)
-	ADIOI_Malloc(nprocs*sizeof(ADIOI_Access));
-    others_req = *others_req_ptr;
-
-    scounts = ADIOI_Malloc(nprocs*sizeof(int));
-    sdispls = ADIOI_Malloc(nprocs*sizeof(int));
-    rcounts = ADIOI_Malloc(nprocs*sizeof(int));
-    rdispls = ADIOI_Malloc(nprocs*sizeof(int));
-
-    /* If process[i] has any requests in my file domain,
-     *   initialize an ADIOI_Access structure that will describe each request
-     *   from process[i].  The offsets, lengths, and buffer pointers still need
-     *   to be obtained to complete the setting of this structure.
-     */
-    count_others_req_procs = 0;
-    for (i=0; i<nprocs; i++) {
-	if (count_others_req_per_proc[i])
-  {
-	    others_req[i].count = count_others_req_per_proc[i];
-
-	    others_req[i].offsets = (ADIO_Offset *)
-		ADIOI_Malloc(count_others_req_per_proc[i]*sizeof(ADIO_Offset));
-	    others_req[i].lens =
-		ADIOI_Malloc(count_others_req_per_proc[i]*sizeof(ADIO_Offset));
-
-	    others_req[i].mem_ptrs = (MPI_Aint *)
-		ADIOI_Malloc(count_others_req_per_proc[i]*sizeof(MPI_Aint));
-
-	    count_others_req_procs++;
-	}
-	else
-	{
-	    others_req[i].count = 0;
-	    others_req[i].offsets = NULL;
-	    others_req[i].lens    = NULL;
-	}
-    }
-
-    /* Now send the calculated offsets and lengths to respective processes */
-
-    /************************/
-    /* Exchange the offsets */
-    /************************/
-
-    // Figure out the layout for the sendbuf and recvbuf.
-    // scounts[] and sdisps[]  /  rcounts[] and rdisps[] define the layout,
-    // and the data for each section will come from from my_req[i].offsets
-    // or others_req[i].offsets.
-
-    int scount_total = 0;
-    int rcount_total = 0;
-    for (i=0; i<nprocs; i++)
-    {
-	/* Send these offsets to process i.*/
-	scounts[i] = count_my_req_per_proc[i];
-	sdispls[i] = scount_total;
-        scount_total += scounts[i];
-
-	/* Receive these offsets from process i.*/
-	rcounts[i] = count_others_req_per_proc[i];
-	rdispls[i] = rcount_total;
-        rcount_total += rcounts[i];
-    }
-
-    void *sbuf_copy_of_req_info;
-    void *rbuf_copy_of_req_info;
-
-    sbuf_copy_of_req_info = (ADIO_Offset *) ADIOI_Malloc(scount_total * sizeof(ADIO_Offset));
-    rbuf_copy_of_req_info = (ADIO_Offset *) ADIOI_Malloc(rcount_total * sizeof(ADIO_Offset));
-    for (i=0; i<nprocs; i++)
-    {
-        // I haven't timed it, I'm just assuming a memcpy(,,0) is fast for
-        // the entries that don't have data to contribute so I didn't bother
-        // with an 'if' statement
-        memcpy(sbuf_copy_of_req_info + sdispls[i] * sizeof(ADIO_Offset),
-            my_req[i].offsets,
-            scounts[i] * sizeof(ADIO_Offset));
-    }
-
-    /* Exchange the offsets */
-    MPI_Alltoallv(sbuf_copy_of_req_info,
-		  scounts, sdispls, ADIO_OFFSET,
-		  rbuf_copy_of_req_info,
-		  rcounts, rdispls, ADIO_OFFSET,
-		  fd->comm);
-    for (i=0; i<nprocs; i++)
-    {
-        memcpy(others_req[i].offsets,
-            rbuf_copy_of_req_info + rdispls[i] * sizeof(ADIO_Offset),
-            rcounts[i] * sizeof(ADIO_Offset));
-    }
-
-    /************************/
-    /* Exchange the lengths */
-    /************************/
-
-    for (i=0; i<nprocs; i++)
-    {
-        memcpy(sbuf_copy_of_req_info + sdispls[i] * sizeof(ADIO_Offset),
-            my_req[i].lens,
-            scounts[i] * sizeof(ADIO_Offset));
-    }
-
-    /* Exchange the lengths */
-    MPI_Alltoallv(sbuf_copy_of_req_info,
-		  scounts, sdispls, ADIO_OFFSET,
-		  rbuf_copy_of_req_info,
-		  rcounts, rdispls, ADIO_OFFSET,
-		  fd->comm);
-    for (i=0; i<nprocs; i++)
-    {
-        memcpy(others_req[i].lens,
-            rbuf_copy_of_req_info + rdispls[i] * sizeof(ADIO_Offset),
-            rcounts[i] * sizeof(ADIO_Offset));
-    }
-
-    /* Clean up */
-    ADIOI_Free(sbuf_copy_of_req_info);
-    ADIOI_Free(rbuf_copy_of_req_info);
-    ADIOI_Free(count_others_req_per_proc);
-    ADIOI_Free (scounts);
-    ADIOI_Free (sdispls);
-    ADIOI_Free (rcounts);
-    ADIOI_Free (rdispls);
-
-    *count_others_req_procs_ptr = count_others_req_procs;
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5027, 0, NULL);
-#endif
-    TRACE_ERR("Leaving ADIOI_GPFS_Calc_others_req\n");
-}
diff --git a/3rd-party/romio321/adio/ad_gpfs/ad_gpfs_aggrs.h b/3rd-party/romio321/adio/ad_gpfs/ad_gpfs_aggrs.h
deleted file mode 100644
index cd8dce90614..00000000000
--- a/3rd-party/romio321/adio/ad_gpfs/ad_gpfs_aggrs.h
+++ /dev/null
@@ -1,86 +0,0 @@
-/* ---------------------------------------------------------------- */
-/* (C)Copyright IBM Corp.  2007, 2008                               */
-/* ---------------------------------------------------------------- */
-/**
- * \file ad_gpfs_aggrs.h
- * \brief ???
- */
-
-/*
- * File: ad_gpfs_aggrs.h
- *
- * Declares functions optimized specifically for GPFS parallel I/O solution.
- *
- */
-
-#ifndef AD_GPFS_AGGRS_H_
-#define AD_GPFS_AGGRS_H_
-
-#include "adio.h"
-#include <sys/stat.h>
-
-#ifdef HAVE_GPFS_H
-#include <gpfs.h>
-#endif
-
-
-    /* overriding ADIOI_Calc_file_domains() to apply 'aligned file domain partitioning'. */
-    void ADIOI_GPFS_Calc_file_domains(ADIO_File fd,
-	                                  ADIO_Offset *st_offsets,
-				          ADIO_Offset *end_offsets,
-				          int          nprocs,
-				          int          nprocs_for_coll,
-				          ADIO_Offset *min_st_offset_ptr,
-				          ADIO_Offset **fd_start_ptr,
-				          ADIO_Offset **fd_end_ptr,
-				          ADIO_Offset *fd_size_ptr,
-                  void        *fs_ptr);
-
-    /* overriding ADIOI_Calc_aggregator() for the default implementation is specific for
-       static file domain partitioning */
-    int ADIOI_GPFS_Calc_aggregator(ADIO_File fd,
-				  ADIO_Offset off,
-				  ADIO_Offset min_off,
-				  ADIO_Offset *len,
-				  ADIO_Offset fd_size,
-				  ADIO_Offset *fd_start,
-				  ADIO_Offset *fd_end);
-
-    /* overriding ADIOI_Calc_my_req for the default implementation is specific for
-       static file domain partitioning */
-    void ADIOI_GPFS_Calc_my_req ( ADIO_File fd, ADIO_Offset *offset_list, ADIO_Offset *len_list,
-				 int contig_access_count, ADIO_Offset
-				 min_st_offset, ADIO_Offset *fd_start,
-				 ADIO_Offset *fd_end, ADIO_Offset fd_size,
-				 int nprocs,
-				 int *count_my_req_procs_ptr,
-				 int **count_my_req_per_proc_ptr,
-				 ADIOI_Access **my_req_ptr,
-				 int **buf_idx_ptr);
-
-    /*
-     * ADIOI_Calc_others_req
-     *
-     * param[in]  count_my_req_procs        Number of processes whose file domain my
-     *                                        request touches.
-     * param[in]  count_my_req_per_proc     count_my_req_per_proc[i] gives the no. of
-     *                                        contig. requests of this process in
-     *                                        process i's file domain.
-     * param[in]  my_req                    A structure defining my request
-     * param[in]  nprocs                    Number of nodes in the block
-     * param[in]  myrank                    Rank of this node
-     * param[out] count_others_req_proc_ptr Number of processes whose requests lie in
-     *                                        my process's file domain (including my
-     *                                        process itself)
-     * param[out] others_req_ptr            Array of other process' requests that lie
-     *                                        in my process's file domain
-     */
-     void ADIOI_GPFS_Calc_others_req(ADIO_File fd, int count_my_req_procs,
-				    int *count_my_req_per_proc,
-				    ADIOI_Access *my_req,
-				    int nprocs, int myrank,
-				    int *count_others_req_procs_ptr,
-				    ADIOI_Access **others_req_ptr);
-
-
-#endif  /* AD_GPFS_AGGRS_H_ */
diff --git a/3rd-party/romio321/adio/ad_gpfs/ad_gpfs_close.c b/3rd-party/romio321/adio/ad_gpfs/ad_gpfs_close.c
deleted file mode 100644
index f8a41671a21..00000000000
--- a/3rd-party/romio321/adio/ad_gpfs/ad_gpfs_close.c
+++ /dev/null
@@ -1,57 +0,0 @@
-/* ---------------------------------------------------------------- */
-/* (C)Copyright IBM Corp.  2007, 2008                               */
-/* ---------------------------------------------------------------- */
-/**
- * \file ad_gpfs_close.c
- * \brief ???
- */
-
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_gpfs.h"
-#include "ad_gpfs_tuning.h"
-#include <unistd.h>
-
-void ADIOI_GPFS_Close(ADIO_File fd, int *error_code)
-{
-  int err, derr=0;
-  static char myname[] = "ADIOI_GPFS_CLOSE";
-
-#ifdef PROFILE
-  MPE_Log_event(9, 0, "start close");
-#endif
-
-  if (fd->null_fd >= 0)
-      close(fd->null_fd);
-
-  err = close(fd->fd_sys);
-  if (fd->fd_direct >= 0)
-  {
-    derr = close(fd->fd_direct);
-  }
-
-#ifdef PROFILE
-  MPE_Log_event(10, 0, "end close");
-#endif
-
-/*  FPRINTF(stderr,"%s(%d):'%s'. Free %#X\n",myname,__LINE__,fd->filename,(int)fd->fs_ptr);*/
-  if (fd->fs_ptr != NULL) {
-	  ADIOI_Free(fd->fs_ptr);
-	  fd->fs_ptr = NULL;
-  }
-  fd->fd_sys    = -1;
-  fd->fd_direct = -1;
-
-  if (err == -1 || derr == -1)
-  {
-    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-                                       myname, __LINE__, MPI_ERR_IO,
-                                       "**io",
-                                       "**io %s", strerror(errno));
-  }
-  else *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_gpfs/ad_gpfs_flush.c b/3rd-party/romio321/adio/ad_gpfs/ad_gpfs_flush.c
deleted file mode 100644
index 555002f639d..00000000000
--- a/3rd-party/romio321/adio/ad_gpfs/ad_gpfs_flush.c
+++ /dev/null
@@ -1,68 +0,0 @@
-/* ---------------------------------------------------------------- */
-/* (C)Copyright IBM Corp.  2007, 2008                               */
-/* ---------------------------------------------------------------- */
-/**
- * \file ad_gpfs_flush.c
- * \brief Scalable flush for GPFS
- */
-
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_gpfs.h"
-
-void ADIOI_GPFS_Flush(ADIO_File fd, int *error_code)
-{
-    int err=0;
-    static char myname[] = "ADIOI_GPFS_FLUSH";
-
-    int rank;
-
-    MPI_Comm_rank(fd->comm, &rank);
-
-    /* the old logic about who is an fsync aggregator and who is not fell down
-     * when deferred open was enabled.  Instead, make this look more like
-     * ad_pvfs2_flush.  If one day the I/O aggregators have something they need
-     * to flush, we can consult the 'fd->hints->ranklist[]' array.  For now, a
-     * flush from one process should suffice */
-
-    /* ensure all other proceses are done writing. On many platforms MPI_Reduce
-     * is fastest because it has the lightest constraints. On Blue Gene, BARRIER
-     * is optimized  */
-    MPI_Barrier(fd->comm);
-
-    if (rank == fd->hints->ranklist[0]) {
-	err = fsync(fd->fd_sys);
-	DBG_FPRINTF(stderr,"aggregation:fsync %s, err=%#X, errno=%#X\n",fd->filename, err, errno);
-	/* We want errno, not the return code if it failed */
-	if (err == -1) err = errno;
-	else err = 0;
-    }
-    MPI_Bcast(&err, 1, MPI_UNSIGNED, fd->hints->ranklist[0], fd->comm);
-    DBGV_FPRINTF(stderr,"aggregation result:fsync %s, errno %#X,\n",fd->filename, err);
-
-    if (err) /* if it's non-zero, it must be an errno */
-    {
-	errno = err;
-	err = -1;
-    }
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (err == -1)
-    {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-		myname, __LINE__, MPI_ERR_IO,
-		"**io",
-		"**io %s", strerror(errno));
-	DBGT_FPRINTF(stderr,"fsync %s, err=%#X, errno=%#X\n",fd->filename, err, errno);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-    *error_code = MPI_SUCCESS;
-}
-
diff --git a/3rd-party/romio321/adio/ad_gpfs/ad_gpfs_hints.c b/3rd-party/romio321/adio/ad_gpfs/ad_gpfs_hints.c
deleted file mode 100644
index 7af0a0c67a0..00000000000
--- a/3rd-party/romio321/adio/ad_gpfs/ad_gpfs_hints.c
+++ /dev/null
@@ -1,288 +0,0 @@
-/* ---------------------------------------------------------------- */
-/* (C)Copyright IBM Corp.  2007, 2008                               */
-/* ---------------------------------------------------------------- */
-/**
- * \file ad_gpfs_hints.c
- * \brief GPFS hint processing - for now, only used for BlueGene and PE platforms
- */
-
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/*
- *   Copyright (C) 1997 University of Chicago.
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-#include "hint_fns.h"
-
-#include "ad_gpfs.h"
-
-#define   ADIOI_GPFS_CB_BUFFER_SIZE_DFLT      	"16777216"
-#define	  ADIOI_GPFS_IND_RD_BUFFER_SIZE_DFLT	"4194304"
-#define   ADIOI_GPFS_IND_WR_BUFFER_SIZE_DFLT	"4194304"
-
-#ifdef BGQPLATFORM
-#define   ADIOI_BG_NAGG_IN_PSET_HINT_NAME	"bg_nodes_pset"
-#endif
-
-/** \page mpiio_vars MPIIO Configuration
- *
- * GPFS MPIIO configuration and performance tuning. Used by ad_gpfs ADIO.
- *
- * Used for BlueGene and PE platforms, which each have their own aggregator selection
- * algorithms that ignore user provided cb_config_list.
- *
- * \section hint_sec Hints
- * - bg_nodes_pset - BlueGene only - specify how many aggregators to use per pset.
- *   This hint will override the cb_nodes hint based on BlueGene psets.
- *   - N - Use N nodes per pset as aggregators.
- *   - Default is based on partition configuration and cb_nodes.
- *
- *   The following default key/value pairs may differ from other platform defaults.
- *
- *     - key = cb_buffer_size     value = 16777216
- *     - key = romio_cb_read      value = enable
- *     - key = romio_cb_write     value = enable
- *     - key = ind_rd_buffer_size value = 4194304
- *     - key = ind_wr_buffer_size value = 4194304
- */
-
-#ifdef BGQPLATFORM
-/* Compute the aggregator-related parameters that are required in 2-phase collective IO of ADIO. */
-extern int
-ADIOI_BG_gen_agg_ranklist(ADIO_File fd, int n_proxy_per_pset);
-#elif PEPLATFORM
-extern int
-ADIOI_PE_gen_agg_ranklist(ADIO_File fd);
-#endif
-
-void ADIOI_GPFS_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code)
-{
-/* if fd->info is null, create a new info object.
-   Initialize fd->info to default values.
-   Initialize fd->hints to default values.
-   Examine the info object passed by the user. If it contains values that
-   ROMIO understands, override the default. */
-
-    MPI_Info info;
-    char *value;
-    int flag, intval, nprocs=0, nprocs_is_valid = 0;
-    static char myname[] = "ADIOI_GPFS_SETINFO";
-
-    int did_anything = 0;
-
-    if (fd->info == MPI_INFO_NULL) MPI_Info_create(&(fd->info));
-    info = fd->info;
-
-    /* Note that fd->hints is allocated at file open time; thus it is
-     * not necessary to allocate it, or check for allocation, here.
-     */
-
-    value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
-    ADIOI_Assert ((value != NULL));
-
-    /* initialize info and hints to default values if they haven't been
-     * previously initialized
-     */
-    if (!fd->hints->initialized) {
-
-	ad_gpfs_get_env_vars();
-	did_anything = 1;
-
-	/* buffer size for collective I/O */
-	ADIOI_Info_set(info, "cb_buffer_size", ADIOI_GPFS_CB_BUFFER_SIZE_DFLT);
-	fd->hints->cb_buffer_size = atoi(ADIOI_GPFS_CB_BUFFER_SIZE_DFLT);
-
-	/* default is to let romio automatically decide when to use
-	 * collective buffering
-	 */
-	ADIOI_Info_set(info, "romio_cb_read", "enable");
-	fd->hints->cb_read = ADIOI_HINT_ENABLE;
-	ADIOI_Info_set(info, "romio_cb_write", "enable");
-	fd->hints->cb_write = ADIOI_HINT_ENABLE;
-
-   	if ( fd->hints->cb_config_list != NULL ) ADIOI_Free (fd->hints->cb_config_list);
-	fd->hints->cb_config_list = NULL;
-
-	/* number of processes that perform I/O in collective I/O */
-	MPI_Comm_size(fd->comm, &nprocs);
-	nprocs_is_valid = 1;
-	ADIOI_Snprintf(value, MPI_MAX_INFO_VAL+1, "%d", nprocs);
-	ADIOI_Info_set(info, "cb_nodes", value);
-	fd->hints->cb_nodes = -1;
-
-	/* hint indicating that no indep. I/O will be performed on this file */
-	ADIOI_Info_set(info, "romio_no_indep_rw", "false");
-	fd->hints->no_indep_rw = 0;
-
-	/* gpfs is not implementing file realms (ADIOI_IOStridedColl),
-	   initialize to disabled it. 	   */
-	/* hint instructing the use of persistent file realms */
-	ADIOI_Info_set(info, "romio_cb_pfr", "disable");
-	fd->hints->cb_pfr = ADIOI_HINT_DISABLE;
-
-	/* hint guiding the assignment of persistent file realms */
-	ADIOI_Info_set(info, "romio_cb_fr_types", "aar");
-	fd->hints->cb_fr_type = ADIOI_FR_AAR;
-
-	/* hint to align file realms with a certain byte value */
-	ADIOI_Info_set(info, "romio_cb_fr_alignment", "1");
-	fd->hints->cb_fr_alignment = 1;
-
-	/* hint to set a threshold percentage for a datatype's size/extent at
-	 * which data sieving should be done in collective I/O */
-	ADIOI_Info_set(info, "romio_cb_ds_threshold", "0");
-	fd->hints->cb_ds_threshold = 0;
-
-	/* hint to switch between point-to-point or all-to-all for two-phase */
-	ADIOI_Info_set(info, "romio_cb_alltoall", "automatic");
-	fd->hints->cb_alltoall = ADIOI_HINT_AUTO;
-
-	 /* deferred_open derived from no_indep_rw and cb_{read,write} */
-	fd->hints->deferred_open = 0;
-
-	/* buffer size for data sieving in independent reads */
-	ADIOI_Info_set(info, "ind_rd_buffer_size", ADIOI_GPFS_IND_RD_BUFFER_SIZE_DFLT);
-	fd->hints->ind_rd_buffer_size = atoi(ADIOI_GPFS_IND_RD_BUFFER_SIZE_DFLT);
-
-	/* buffer size for data sieving in independent writes */
-	ADIOI_Info_set(info, "ind_wr_buffer_size", ADIOI_GPFS_IND_WR_BUFFER_SIZE_DFLT);
-	fd->hints->ind_wr_buffer_size = atoi(ADIOI_GPFS_IND_WR_BUFFER_SIZE_DFLT);
-
-
-    ADIOI_Info_set(info, "romio_ds_read", "automatic");
-    fd->hints->ds_read = ADIOI_HINT_AUTO;
-    ADIOI_Info_set(info, "romio_ds_write", "automatic");
-    fd->hints->ds_write = ADIOI_HINT_AUTO;
-
-    /* still to do: tune this a bit for a variety of file systems. there's
-	 * no good default value so just leave it unset */
-    fd->hints->min_fdomain_size = 0;
-    fd->hints->striping_unit = 0;
-
-    fd->hints->initialized = 1;
-    }
-
-    /* add in user's info if supplied */
-    if (users_info != MPI_INFO_NULL) {
-	ADIOI_Info_check_and_install_int(fd, users_info, "cb_buffer_size",
-		&(fd->hints->cb_buffer_size), myname, error_code);
-	/* new hints for enabling/disabling coll. buffering on
-	 * reads/writes
-	 */
-	ADIOI_Info_check_and_install_enabled(fd, users_info, "romio_cb_read",
-		&(fd->hints->cb_read), myname, error_code);
-	if (fd->hints->cb_read == ADIOI_HINT_DISABLE) {
-	    /* romio_cb_read overrides no_indep_rw */
-	    ADIOI_Info_set(info, "romio_no_indep_rw", "false");
-	    fd->hints->no_indep_rw = ADIOI_HINT_DISABLE;
-	}
-	ADIOI_Info_check_and_install_enabled(fd, users_info, "romio_cb_write",
-		&(fd->hints->cb_write), myname, error_code);
-	if (fd->hints->cb_write == ADIOI_HINT_DISABLE) {
-	    /* romio_cb_write overrides no_indep_rw */
-	    ADIOI_Info_set(info, "romio_no_indep_rw", "false");
-	    fd->hints->no_indep_rw = ADIOI_HINT_DISABLE;
-	}
-	/* Has the user indicated all I/O will be done collectively? */
-	ADIOI_Info_check_and_install_true(fd, users_info, "romio_no_indep_rw",
-		&(fd->hints->no_indep_rw), myname, error_code);
-	if (fd->hints->no_indep_rw == 1) {
-	    /* if 'no_indep_rw' set, also hint that we will do
-	     * collective buffering: if we aren't doing independent io,
-	     * then we have to do collective  */
-	    ADIOI_Info_set(info, "romio_cb_write", "enable");
-	    ADIOI_Info_set(info, "romio_cb_read", "enable");
-	    fd->hints->cb_read = 1;
-	    fd->hints->cb_write = 1;
-	}
-
-	/* new hints for enabling/disabling data sieving on
-	 * reads/writes
-	 */
-	ADIOI_Info_check_and_install_enabled(fd, users_info, "romio_ds_read",
-		&(fd->hints->ds_read), myname, error_code);
-	ADIOI_Info_check_and_install_enabled(fd, users_info, "romio_ds_write",
-		&(fd->hints->ds_write), myname, error_code);
-
-	ADIOI_Info_check_and_install_int(fd, users_info, "ind_wr_buffer_size",
-		&(fd->hints->ind_wr_buffer_size), myname, error_code);
-	ADIOI_Info_check_and_install_int(fd, users_info, "ind_rd_buffer_size",
-		&(fd->hints->ind_rd_buffer_size), myname, error_code);
-
-	memset( value, 0, MPI_MAX_INFO_VAL+1 );
-	ADIOI_Info_get(users_info, "romio_min_fdomain_size", MPI_MAX_INFO_VAL,
-			value, &flag);
-	if ( flag && ((intval = atoi(value)) > 0) ) {
-		ADIOI_Info_set(info, "romio_min_fdomain_size", value);
-		fd->hints->min_fdomain_size = intval;
-	}
-  /* Now we use striping unit in common code so we should
-     process hints for it. */
-	ADIOI_Info_check_and_install_int(fd, users_info, "striping_unit",
-		&(fd->hints->striping_unit), myname, error_code);
-
-#ifdef BGQPLATFORM
-	memset( value, 0, MPI_MAX_INFO_VAL+1 );
-        ADIOI_Info_get(users_info, ADIOI_BG_NAGG_IN_PSET_HINT_NAME, MPI_MAX_INFO_VAL,
-		     value, &flag);
-	if (flag && ((intval = atoi(value)) > 0)) {
-
-	    did_anything = 1;
-	    ADIOI_Info_set(info, ADIOI_BG_NAGG_IN_PSET_HINT_NAME, value);
-	    fd->hints->cb_nodes = intval;
-	}
-#endif
-    }
-
-    /* special CB aggregator assignment */
-    if (did_anything) {
-#ifdef BGQPLATFORM
-	ADIOI_BG_gen_agg_ranklist(fd, fd->hints->cb_nodes);
-#elif PEPLATFORM
-	ADIOI_PE_gen_agg_ranklist(fd);
-#endif
-    }
-
-    /* deferred_open won't be set by callers, but if the user doesn't
-     * explicitly disable collecitve buffering (two-phase) and does hint that
-     * io w/o independent io is going on, we'll set this internal hint as a
-     * convenience */
-    if ( ( (fd->hints->cb_read != ADIOI_HINT_DISABLE) \
-			    && (fd->hints->cb_write != ADIOI_HINT_DISABLE)\
-			    && fd->hints->no_indep_rw ) ) {
-	    fd->hints->deferred_open = 1;
-    } else {
-	    /* setting romio_no_indep_rw enable and romio_cb_{read,write}
-	     * disable at the same time doesn't make sense. honor
-	     * romio_cb_{read,write} and force the no_indep_rw hint to
-	     * 'disable' */
-	    ADIOI_Info_set(info, "romio_no_indep_rw", "false");
-	    fd->hints->no_indep_rw = 0;
-	    fd->hints->deferred_open = 0;
-    }
-
-    /* BobC commented this out, but since hint processing runs on both bg and
-     * bglockless, we need to keep DS writes enabled on gpfs and disabled on
-     * PVFS */
-    if (ADIO_Feature(fd, ADIO_DATA_SIEVING_WRITES) == 0) {
-    /* disable data sieving for fs that do not
-       support file locking */
-       	ADIOI_Info_get(info, "ind_wr_buffer_size", MPI_MAX_INFO_VAL,
-		     value, &flag);
-	if (flag) {
-	    /* get rid of this value if it is set */
-	    ADIOI_Info_delete(info, "ind_wr_buffer_size");
-	}
-	/* note: leave ind_wr_buffer_size alone; used for other cases
-	 * as well. -- Rob Ross, 04/22/2003
-	 */
-	ADIOI_Info_set(info, "romio_ds_write", "disable");
-	fd->hints->ds_write = ADIOI_HINT_DISABLE;
-    }
-
-    ADIOI_Free(value);
-
-    *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_gpfs/ad_gpfs_open.c b/3rd-party/romio321/adio/ad_gpfs/ad_gpfs_open.c
deleted file mode 100644
index f4fef37c85c..00000000000
--- a/3rd-party/romio321/adio/ad_gpfs/ad_gpfs_open.c
+++ /dev/null
@@ -1,156 +0,0 @@
-/* ---------------------------------------------------------------- */
-/* (C)Copyright IBM Corp.  2007, 2008                               */
-/* ---------------------------------------------------------------- */
-/**
- * \file ad_gpfs_open.c
- * \brief ???
- */
-
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_gpfs.h"
-#include "ad_gpfs_tuning.h"
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#include <fcntl.h>
-
-
-#ifdef HAVE_GPFS_H
-#include <gpfs.h>
-#endif
-#ifdef HAVE_GPFS_FCNTL_H
-#include <gpfs_fcntl.h>
-#endif
-
-#ifdef HAVE_GPFS_FCNTL_H
-static void gpfs_free_all_locks(int fd)
-{
-    int rc;
-    struct {
-	gpfsFcntlHeader_t header;
-	gpfsFreeRange_t release;
-    } release_all;
-
-    release_all.header.totalLength = sizeof(release_all);
-    release_all.header.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION;
-    release_all.header.fcntlReserved = 0;
-
-    release_all.release.structLen = sizeof(release_all.release);
-    release_all.release.structType = GPFS_FREE_RANGE;
-    release_all.release.start = 0;
-    release_all.release.length = 0;
-
-    rc = gpfs_fcntl(fd, &release_all);
-    if (rc != 0) {
-	DBGV_FPRINTF(stderr,"GPFS fcntl release failed with rc=%d, errno=%d\n",
-		rc,errno);
-    }
-}
-#endif
-
-
-void ADIOI_GPFS_Open(ADIO_File fd, int *error_code)
-{
-  int perm, old_mask, amode, rank, rc;
-  static char myname[] = "ADIOI_GPFS_OPEN";
-
-  /* set internal variables for tuning environment variables */
-  ad_gpfs_get_env_vars();
-
-  if (fd->perm == ADIO_PERM_NULL)  {
-    old_mask = umask(022);
-    umask(old_mask);
-    perm = old_mask ^ 0666;
-  }
-  else perm = fd->perm;
-
-    amode = 0;
-    if (fd->access_mode & ADIO_CREATE)
-	amode = amode | O_CREAT;
-    if (fd->access_mode & ADIO_RDONLY)
-	amode = amode | O_RDONLY;
-    if (fd->access_mode & ADIO_WRONLY)
-	amode = amode | O_WRONLY;
-    if (fd->access_mode & ADIO_RDWR)
-	amode = amode | O_RDWR;
-    if (fd->access_mode & ADIO_EXCL)
-	amode = amode | O_EXCL;
-#ifdef ADIOI_MPE_LOGGING
-    MPE_Log_event(ADIOI_MPE_open_a, 0, NULL);
-#endif
-    fd->fd_sys = open(fd->filename, amode, perm);
-#ifdef ADIOI_MPE_LOGGING
-    MPE_Log_event(ADIOI_MPE_open_b, 0, NULL);
-#endif
-  DBG_FPRINTF(stderr,"open('%s',%#X,%#X) rc=%d, errno=%d\n",fd->filename,amode,perm,fd->fd_sys,errno);
-  fd->fd_direct = -1;
-
-  if (gpfsmpio_devnullio == 1) {
-      fd->null_fd = open("/dev/null", O_RDWR);
-  } else {
-      fd->null_fd = -1;
-  }
-
-  if ((fd->fd_sys != -1) && (fd->access_mode & ADIO_APPEND))
-    fd->fp_ind = fd->fp_sys_posn = lseek(fd->fd_sys, 0, SEEK_END);
-
-    if(fd->fd_sys != -1)
-    {
-
-        fd->blksize = 1048576; /* default to 1M */
-
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event(ADIOI_MPE_stat_a, 0, NULL);
-#endif
-	/* in this fs-specific routine, we might not be called over entire
-	 * communicator (deferred open).  Collect statistics on one process.
-	 * ADIOI_GEN_Opencoll (common-code caller) will take care of the
-	 * broadcast */
-
-	MPI_Comm_rank(fd->comm, &rank);
-	if ((rank == fd->hints->ranklist[0]) || (fd->comm == MPI_COMM_SELF)) {
-	    struct stat64 gpfs_statbuf;
-	    /* Get the (real) underlying file system block size */
-	    rc = stat64(fd->filename, &gpfs_statbuf);
-	    if (rc >= 0)
-	    {
-		fd->blksize = gpfs_statbuf.st_blksize;
-		DBGV_FPRINTF(stderr,"Successful stat '%s'.  Blocksize=%ld\n",
-			fd->filename,gpfs_statbuf.st_blksize);
-	    }
-	    else
-	    {
-		DBGV_FPRINTF(stderr,"Stat '%s' failed with rc=%d, errno=%d\n",
-			fd->filename,rc,errno);
-	    }
-	}
-	/* all other ranks have incorrect fd->blocksize, but ADIOI_GEN_Opencoll
-	 * will take care of that in both standard and deferred-open case */
-
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event(ADIOI_MPE_stat_b, 0, NULL);
-#endif
-
-#ifdef HAVE_GPFS_FCNTL_H
-	/* in parallel workload, might be helpful to immediately release block
-	 * tokens.  Or, system call overhead will outweigh any benefits... */
-	if (getenv("ROMIO_GPFS_FREE_LOCKS")!=NULL)
-	    gpfs_free_all_locks(fd->fd_sys);
-
-#endif
-    }
-
-  if (fd->fd_sys == -1)  {
-      *error_code = ADIOI_Err_create_code(myname, fd->filename, errno);
-  }
-  else *error_code = MPI_SUCCESS;
-}
-/* 
- *vim: ts=8 sts=4 sw=4 noexpandtab 
- */
diff --git a/3rd-party/romio321/adio/ad_gpfs/ad_gpfs_rdcoll.c b/3rd-party/romio321/adio/ad_gpfs/ad_gpfs_rdcoll.c
deleted file mode 100644
index f449acb158a..00000000000
--- a/3rd-party/romio321/adio/ad_gpfs/ad_gpfs_rdcoll.c
+++ /dev/null
@@ -1,1255 +0,0 @@
-/* ---------------------------------------------------------------- */
-/* (C)Copyright IBM Corp.  2007, 2008                               */
-/* ---------------------------------------------------------------- */
-/**
- * \file ad_gpfs_rdcoll.c
- * \brief ???
- */
-
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-#include "ad_gpfs.h"
-#include "ad_gpfs_aggrs.h"
-
-#ifdef PROFILE
-#include "mpe.h"
-#endif
-
-#ifdef USE_DBG_LOGGING
-  #define RDCOLL_DEBUG 1
-#endif
-#ifdef AGGREGATION_PROFILE
-#include "mpe.h"
-#endif
-
-/* prototypes of functions used for collective reads only. */
-static void ADIOI_Read_and_exch(ADIO_File fd, void *buf, MPI_Datatype
-				datatype, int nprocs,
-				int myrank, ADIOI_Access
-				*others_req, ADIO_Offset *offset_list,
-				ADIO_Offset *len_list, int contig_access_count, 
-				ADIO_Offset
-				min_st_offset, ADIO_Offset fd_size,
-				ADIO_Offset *fd_start, ADIO_Offset *fd_end,
-				int *buf_idx, int *error_code);
-static void ADIOI_R_Exchange_data(ADIO_File fd, void *buf, ADIOI_Flatlist_node
-				  *flat_buf, ADIO_Offset *offset_list, ADIO_Offset
-				  *len_list, int *send_size, int *recv_size,
-				  int *count, int *start_pos, 
-				  int *partial_send, 
-				  int *recd_from_proc, int nprocs, 
-				  int myrank, int
-				  buftype_is_contig, int contig_access_count,
-				  ADIO_Offset min_st_offset, 
-				  ADIO_Offset fd_size,
-				  ADIO_Offset *fd_start, ADIO_Offset *fd_end, 
-				  ADIOI_Access *others_req, 
-				  int iter, 
-				  MPI_Aint buftype_extent, int *buf_idx);
-static void ADIOI_R_Exchange_data_alltoallv(ADIO_File fd, void *buf, ADIOI_Flatlist_node
-                                  *flat_buf, ADIO_Offset *offset_list, ADIO_Offset
-                                  *len_list, int *send_size, int *recv_size,
-                                  int *count, int *start_pos,
-                                  int *partial_send,
-                                  int *recd_from_proc, int nprocs,
-                                  int myrank, int
-                                  buftype_is_contig, int contig_access_count,
-                                  ADIO_Offset min_st_offset,
-                                  ADIO_Offset fd_size,
-                                  ADIO_Offset *fd_start, ADIO_Offset *fd_end,
-                                  ADIOI_Access *others_req,
-                                  int iter,
-                                  MPI_Aint buftype_extent, int *buf_idx);
-static void ADIOI_Fill_user_buffer(ADIO_File fd, void *buf, ADIOI_Flatlist_node
-				   *flat_buf, char **recv_buf, ADIO_Offset 
-				   *offset_list, ADIO_Offset *len_list, 
-				   unsigned *recv_size, 
-				   MPI_Request *requests, MPI_Status *statuses,
-				   int *recd_from_proc, int nprocs,
-				   int contig_access_count, 
-				   ADIO_Offset min_st_offset, 
-				   ADIO_Offset fd_size, ADIO_Offset *fd_start, 
-				   ADIO_Offset *fd_end,
-				   MPI_Aint buftype_extent);
-
-extern void ADIOI_Calc_my_off_len(ADIO_File fd, int bufcount, MPI_Datatype
-			    datatype, int file_ptr_type, ADIO_Offset
-			    offset, ADIO_Offset **offset_list_ptr, ADIO_Offset
-			    **len_list_ptr, ADIO_Offset *start_offset_ptr,
-			    ADIO_Offset *end_offset_ptr, int
-			   *contig_access_count_ptr);
-
-
-
-void ADIOI_GPFS_ReadStridedColl(ADIO_File fd, void *buf, int count,
-			       MPI_Datatype datatype, int file_ptr_type,
-			       ADIO_Offset offset, ADIO_Status *status, int
-			       *error_code)
-{
-/* Uses a generalized version of the extended two-phase method described
-   in "An Extended Two-Phase Method for Accessing Sections of 
-   Out-of-Core Arrays", Rajeev Thakur and Alok Choudhary,
-   Scientific Programming, (5)4:301--317, Winter 1996. 
-   http://www.mcs.anl.gov/home/thakur/ext2ph.ps */
-
-    ADIOI_Access *my_req; 
-    /* array of nprocs structures, one for each other process in
-       whose file domain this process's request lies */
-    
-    ADIOI_Access *others_req;
-    /* array of nprocs structures, one for each other process
-       whose request lies in this process's file domain. */
-
-    int i, filetype_is_contig, nprocs, nprocs_for_coll, myrank;
-    int contig_access_count=0, interleave_count = 0, buftype_is_contig;
-    int *count_my_req_per_proc, count_my_req_procs, count_others_req_procs;
-    ADIO_Offset start_offset, end_offset, orig_fp, fd_size, min_st_offset, off;
-    ADIO_Offset *offset_list = NULL, *st_offsets = NULL, *fd_start = NULL,
-	*fd_end = NULL, *end_offsets = NULL;
-    ADIO_Offset *gpfs_offsets0 = NULL, *gpfs_offsets = NULL;
-    ADIO_Offset *count_sizes;
-    int  ii;
-    ADIO_Offset *len_list = NULL;
-    int *buf_idx = NULL;
-
-    GPFSMPIO_T_CIO_RESET( r);
-
-#ifdef HAVE_STATUS_SET_BYTES
-    MPI_Count bufsize, size;
-#endif
-
-#if 0
-/*   From common code - not implemented for bg. */
-    if (fd->hints->cb_pfr != ADIOI_HINT_DISABLE) {
-        ADIOI_IOStridedColl (fd, buf, count, ADIOI_READ, datatype, 
-			file_ptr_type, offset, status, error_code);
-        return;
-    } */
-#endif
-#ifdef PROFILE
-        MPE_Log_event(13, 0, "start computation");
-#endif
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-
-    /* number of aggregators, cb_nodes, is stored in the hints */
-    nprocs_for_coll = fd->hints->cb_nodes;
-    orig_fp = fd->fp_ind;
-
-    GPFSMPIO_T_CIO_SET_GET( r, 1, 0, GPFSMPIO_CIO_T_MPIO_CRW, GPFSMPIO_CIO_LAST)
-    GPFSMPIO_T_CIO_SET_GET( r, 1, 0, GPFSMPIO_CIO_T_LCOMP, GPFSMPIO_CIO_LAST )
-
-    /* only check for interleaving if cb_read isn't disabled */
-    if (fd->hints->cb_read != ADIOI_HINT_DISABLE) {
-	/* For this process's request, calculate the list of offsets and
-	   lengths in the file and determine the start and end offsets. */
-
-	/* Note: end_offset points to the last byte-offset that will be accessed.
-	   e.g., if start_offset=0 and 100 bytes to be read, end_offset=99*/
-
-	ADIOI_Calc_my_off_len(fd, count, datatype, file_ptr_type, offset,
-			      &offset_list, &len_list, &start_offset,
-			      &end_offset, &contig_access_count);
-
-    GPFSMPIO_T_CIO_SET_GET( r, 1, 1, GPFSMPIO_CIO_T_GATHER, GPFSMPIO_CIO_T_LCOMP )
-
-#ifdef RDCOLL_DEBUG
-    for (i=0; i<contig_access_count; i++) {
-	      DBG_FPRINTF(stderr, "rank %d  off %lld  len %lld\n", 
-			      myrank, offset_list[i], len_list[i]);
-    }
-#endif
-
-	/* each process communicates its start and end offsets to other 
-	   processes. The result is an array each of start and end offsets
-	   stored in order of process rank. */ 
-    
-	st_offsets   = (ADIO_Offset *) ADIOI_Malloc(nprocs*sizeof(ADIO_Offset));
-	end_offsets  = (ADIO_Offset *) ADIOI_Malloc(nprocs*sizeof(ADIO_Offset));
-
-    ADIO_Offset my_count_size=0;
-    /* One-sided aggregation needs the amount of data per rank as well because the difference in
-     * starting and ending offsets for 1 byte is 0 the same as 0 bytes so it cannot be distiguished.
-     */
-    if ((gpfsmpio_read_aggmethod == 1) || (gpfsmpio_read_aggmethod == 2)) {
-        count_sizes = (ADIO_Offset *) ADIOI_Malloc(nprocs*sizeof(ADIO_Offset));
-        MPI_Count buftype_size;
-        MPI_Type_size_x(datatype, &buftype_size);
-        my_count_size = (ADIO_Offset) count  * (ADIO_Offset)buftype_size;
-    }
-    if (gpfsmpio_tunegather) {
-      if ((gpfsmpio_read_aggmethod == 1) || (gpfsmpio_read_aggmethod == 2)) {
-        gpfs_offsets0 = (ADIO_Offset *) ADIOI_Malloc(3*nprocs*sizeof(ADIO_Offset));
-        gpfs_offsets  = (ADIO_Offset *) ADIOI_Malloc(3*nprocs*sizeof(ADIO_Offset));
-        for (ii=0; ii<nprocs; ii++)  {
-          gpfs_offsets0[ii*3]   = 0;
-          gpfs_offsets0[ii*3+1] = 0;
-          gpfs_offsets0[ii*3+2] = 0;
-        }
-        gpfs_offsets0[myrank*3]   = start_offset;
-        gpfs_offsets0[myrank*3+1] =   end_offset;
-        gpfs_offsets0[myrank*3+2] =   my_count_size;
-        MPI_Allreduce( gpfs_offsets0, gpfs_offsets, nprocs*3, ADIO_OFFSET, MPI_MAX, fd->comm );
-        for (ii=0; ii<nprocs; ii++)  {
-          st_offsets [ii] = gpfs_offsets[ii*3]  ;
-          end_offsets[ii] = gpfs_offsets[ii*3+1];
-          count_sizes[ii] = gpfs_offsets[ii*3+2];
-        }
-      }
-      else {
-	    gpfs_offsets0 = (ADIO_Offset *) ADIOI_Malloc(2*nprocs*sizeof(ADIO_Offset));
-	    gpfs_offsets  = (ADIO_Offset *) ADIOI_Malloc(2*nprocs*sizeof(ADIO_Offset));
-	    for (ii=0; ii<nprocs; ii++)  {
-		gpfs_offsets0[ii*2]   = 0;
-		gpfs_offsets0[ii*2+1] = 0;
-	    }
-	    gpfs_offsets0[myrank*2]   = start_offset;
-	    gpfs_offsets0[myrank*2+1] =   end_offset;
-
-	MPI_Allreduce( gpfs_offsets0, gpfs_offsets, nprocs*2, ADIO_OFFSET, MPI_MAX, fd->comm );
-
-	    for (ii=0; ii<nprocs; ii++)  {
-		st_offsets [ii] = gpfs_offsets[ii*2]  ;
-		end_offsets[ii] = gpfs_offsets[ii*2+1];
-	    }
-      }
-	    ADIOI_Free( gpfs_offsets0 );
-	    ADIOI_Free( gpfs_offsets  );
-    } else {
-        MPI_Allgather(&start_offset, 1, ADIO_OFFSET, st_offsets, 1,
-                      ADIO_OFFSET, fd->comm);
-        MPI_Allgather(&end_offset, 1, ADIO_OFFSET, end_offsets, 1,
-                      ADIO_OFFSET, fd->comm);
-        if ((gpfsmpio_read_aggmethod == 1) || (gpfsmpio_read_aggmethod == 2)) {
-	      MPI_Allgather(&count_sizes, 1, ADIO_OFFSET, count_sizes, 1,
-                        ADIO_OFFSET, fd->comm);
-        }
-    }
-
-    GPFSMPIO_T_CIO_SET_GET( r, 1, 1, GPFSMPIO_CIO_T_PATANA, GPFSMPIO_CIO_T_GATHER )
-
-	/* are the accesses of different processes interleaved? */
-	for (i=1; i<nprocs; i++)
-	    if ((st_offsets[i] < end_offsets[i-1]) && 
-                (st_offsets[i] <= end_offsets[i]))
-                interleave_count++;
-	/* This is a rudimentary check for interleaving, but should suffice
-	   for the moment. */
-    }
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-
-    if (fd->hints->cb_read == ADIOI_HINT_DISABLE
-	|| (!interleave_count && (fd->hints->cb_read == ADIOI_HINT_AUTO))) 
-    {
-	/* don't do aggregation */
-	if (fd->hints->cb_read != ADIOI_HINT_DISABLE) {
-	    ADIOI_Free(offset_list);
-	    ADIOI_Free(len_list);
-	    ADIOI_Free(st_offsets);
-	    ADIOI_Free(end_offsets);
-	}
-
-	fd->fp_ind = orig_fp;
-	ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-
-	if (buftype_is_contig && filetype_is_contig) {
-	    if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-		off = fd->disp + (ADIO_Offset)(fd->etype_size) * offset;
-		ADIO_ReadContig(fd, buf, count, datatype, ADIO_EXPLICIT_OFFSET,
-                       off, status, error_code);
-	    }
-	    else ADIO_ReadContig(fd, buf, count, datatype, ADIO_INDIVIDUAL,
-                       0, status, error_code);
-	}
-	else ADIO_ReadStrided(fd, buf, count, datatype, file_ptr_type,
-                       offset, status, error_code);
-
-	return;
-    }
-
-    GPFSMPIO_T_CIO_SET_GET( r, 1, 1, GPFSMPIO_CIO_T_FD_PART, GPFSMPIO_CIO_T_PATANA )
-
-    /* We're going to perform aggregation of I/O.  Here we call
-     * ADIOI_Calc_file_domains() to determine what processes will handle I/O
-     * to what regions.  We pass nprocs_for_coll into this function; it is
-     * used to determine how many processes will perform I/O, which is also
-     * the number of regions into which the range of bytes must be divided.
-     * These regions are called "file domains", or FDs.
-     *
-     * When this function returns, fd_start, fd_end, fd_size, and
-     * min_st_offset will be filled in.  fd_start holds the starting byte
-     * location for each file domain.  fd_end holds the ending byte location.
-     * min_st_offset holds the minimum byte location that will be accessed.
-     *
-     * Both fd_start[] and fd_end[] are indexed by an aggregator number; this
-     * needs to be mapped to an actual rank in the communicator later.
-     *
-     */
-    int currentNonZeroDataIndex = 0;
-    if ((gpfsmpio_read_aggmethod == 1) || (gpfsmpio_read_aggmethod == 2)) {
-      /* Take out the 0-data offsets by shifting the indexes with data to the
-       * front and keeping track of the non-zero data index for use as the
-       * length.  By doing this we will optimally use all available aggs
-       * and spread the actual data across them instead of having offsets
-       * with empty data potentially dilute the file domains and create
-       * problems for the one-sided aggregation.
-       */
-      for (i=0; i<nprocs; i++) {
-        if (count_sizes[i] > 0) {
-          st_offsets[currentNonZeroDataIndex] = st_offsets[i];
-          end_offsets[currentNonZeroDataIndex] = end_offsets[i];
-          currentNonZeroDataIndex++;
-        }
-      }
-    }
-    if (gpfsmpio_tuneblocking) {
-    if ((gpfsmpio_read_aggmethod == 1) || (gpfsmpio_read_aggmethod == 2)) {
-    ADIOI_GPFS_Calc_file_domains(fd, st_offsets, end_offsets, currentNonZeroDataIndex,
-			    nprocs_for_coll, &min_st_offset,
-			    &fd_start, &fd_end, &fd_size, fd->fs_ptr);
-    }
-    else {
-    ADIOI_GPFS_Calc_file_domains(fd, st_offsets, end_offsets, nprocs,
-			    nprocs_for_coll, &min_st_offset,
-			    &fd_start, &fd_end, &fd_size, fd->fs_ptr);
-    }
-    }
-    else {
-    if ((gpfsmpio_read_aggmethod == 1) || (gpfsmpio_read_aggmethod == 2)) {
-    ADIOI_Calc_file_domains(st_offsets, end_offsets, currentNonZeroDataIndex,
-			    nprocs_for_coll, &min_st_offset,
-			    &fd_start, &fd_end,
-			    fd->hints->min_fdomain_size, &fd_size,
-			    fd->hints->striping_unit);
-	}
-    else {
-    ADIOI_Calc_file_domains(st_offsets, end_offsets, nprocs,
-			    nprocs_for_coll, &min_st_offset,
-			    &fd_start, &fd_end,
-			    fd->hints->min_fdomain_size, &fd_size, 
-			    fd->hints->striping_unit);
-	}
-    }
-
-    GPFSMPIO_T_CIO_SET_GET( r, 1, 1, GPFSMPIO_CIO_T_MYREQ, GPFSMPIO_CIO_T_FD_PART );
-    if ((gpfsmpio_read_aggmethod == 1) || (gpfsmpio_read_aggmethod == 2)) {
-    /* If the user has specified to use a one-sided aggregation method then do that at
-     * this point instead of the two-phase I/O.
-     */
-      ADIOI_OneSidedReadAggregation(fd, offset_list, len_list, contig_access_count, buf,
-		    datatype,error_code, st_offsets, end_offsets, currentNonZeroDataIndex, fd_start, fd_end);
-      GPFSMPIO_T_CIO_REPORT( 0, fd, myrank, nprocs)
-      ADIOI_Free(offset_list);
-      ADIOI_Free(len_list);
-      ADIOI_Free(st_offsets);
-      ADIOI_Free(end_offsets);
-      ADIOI_Free(fd_start);
-      ADIOI_Free(fd_end);
-      ADIOI_Free(count_sizes);
-	  goto fn_exit;
-    }
-    if (gpfsmpio_p2pcontig==1) {
-	/* For some simple yet common(?) workloads, full-on two-phase I/O is
-	 * overkill.  We can establish sub-groups of processes and their
-	 * aggregator, and then these sub-groups will carry out a simplified
-	 * two-phase over that sub-group.
-	 *
-	 * First verify that the filetype is contig and the offsets are
-	 * increasing in rank order*/
-	int x, inOrderAndNoGaps = 1;
-	for (x=0;x<(nprocs-1);x++) {
-	    if (end_offsets[x] != (st_offsets[x+1]-1))
-		inOrderAndNoGaps = 0;
-	}
-	if (inOrderAndNoGaps && buftype_is_contig) {
-	    /* if these conditions exist then execute the P2PContig code else
-	     * execute the original code */
-	    ADIOI_P2PContigReadAggregation(fd, buf,
-		    error_code, st_offsets, end_offsets, fd_start, fd_end);
-
-	    /* NOTE: we are skipping the rest of two-phase in this path */
-            GPFSMPIO_T_CIO_REPORT( 0, fd, myrank, nprocs)
-
-            ADIOI_Free(offset_list);
-            ADIOI_Free(len_list);
-            ADIOI_Free(st_offsets);
-            ADIOI_Free(end_offsets);
-            ADIOI_Free(fd_start);
-            ADIOI_Free(fd_end);
-	    goto fn_exit;
-
-	}
-    }
-
-    /* calculate where the portions of the access requests of this process 
-     * are located in terms of the file domains.  this could be on the same
-     * process or on other processes.  this function fills in:
-     * count_my_req_procs - number of processes (including this one) for which
-     *     this process has requests in their file domain
-     * count_my_req_per_proc - count of requests for each process, indexed
-     *     by rank of the process
-     * my_req[] - array of data structures describing the requests to be
-     *     performed by each process (including self).  indexed by rank.
-     * buf_idx[] - array of locations into which data can be directly moved;
-     *     this is only valid for contiguous buffer case
-     */
-    if (gpfsmpio_tuneblocking)
-    ADIOI_GPFS_Calc_my_req(fd, offset_list, len_list, contig_access_count,
-		      min_st_offset, fd_start, fd_end, fd_size,
-		      nprocs, &count_my_req_procs, 
-		      &count_my_req_per_proc, &my_req,
-		      &buf_idx);
-    else
-    ADIOI_Calc_my_req(fd, offset_list, len_list, contig_access_count,
-		      min_st_offset, fd_start, fd_end, fd_size,
-		      nprocs, &count_my_req_procs, 
-		      &count_my_req_per_proc, &my_req,
-		      &buf_idx);
-
-    GPFSMPIO_T_CIO_SET_GET( r, 1, 1, GPFSMPIO_CIO_T_OTHREQ, GPFSMPIO_CIO_T_MYREQ )
-
-    /* perform a collective communication in order to distribute the
-     * data calculated above.  fills in the following:
-     * count_others_req_procs - number of processes (including this
-     *     one) which have requests in this process's file domain.
-     * count_others_req_per_proc[] - number of separate contiguous
-     *     requests from proc i lie in this process's file domain.
-     */
-    if (gpfsmpio_tuneblocking)
-    ADIOI_GPFS_Calc_others_req(fd, count_my_req_procs,
-			  count_my_req_per_proc, my_req,
-			  nprocs, myrank, &count_others_req_procs,
-			  &others_req);
-
-    else
-    ADIOI_Calc_others_req(fd, count_my_req_procs, 
-			  count_my_req_per_proc, my_req, 
-			  nprocs, myrank, &count_others_req_procs, 
-			  &others_req); 
-
-    GPFSMPIO_T_CIO_SET_GET( r, 1, 1, GPFSMPIO_CIO_T_DEXCH, GPFSMPIO_CIO_T_OTHREQ )
-
-    /* my_req[] and count_my_req_per_proc aren't needed at this point, so 
-     * let's free the memory 
-     */
-    ADIOI_Free(count_my_req_per_proc);
-    for (i=0; i<nprocs; i++) {
-	if (my_req[i].count) {
-	    ADIOI_Free(my_req[i].offsets);
-	    ADIOI_Free(my_req[i].lens);
-	}
-    }
-    ADIOI_Free(my_req);
-
-
-    /* read data in sizes of no more than ADIOI_Coll_bufsize, 
-     * communicate, and fill user buf. 
-     */
-    ADIOI_Read_and_exch(fd, buf, datatype, nprocs, myrank,
-                        others_req, offset_list,
-			len_list, contig_access_count, min_st_offset,
-			fd_size, fd_start, fd_end, buf_idx, error_code);
-
-    GPFSMPIO_T_CIO_SET_GET( r, 0, 1, GPFSMPIO_CIO_LAST, GPFSMPIO_CIO_T_DEXCH )
-    GPFSMPIO_T_CIO_SET_GET( r, 0, 1, GPFSMPIO_CIO_LAST, GPFSMPIO_CIO_T_MPIO_CRW )
-
-    GPFSMPIO_T_CIO_REPORT( 0, fd, myrank, nprocs)
-
-    if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
-
-    /* free all memory allocated for collective I/O */
-    for (i=0; i<nprocs; i++) {
-	if (others_req[i].count) {
-	    ADIOI_Free(others_req[i].offsets);
-	    ADIOI_Free(others_req[i].lens);
-	    ADIOI_Free(others_req[i].mem_ptrs);
-	}
-    }
-    ADIOI_Free(others_req);
-
-    ADIOI_Free(buf_idx);
-    ADIOI_Free(offset_list);
-    ADIOI_Free(len_list);
-    ADIOI_Free(st_offsets);
-    ADIOI_Free(end_offsets);
-    ADIOI_Free(fd_start);
-    ADIOI_Free(fd_end);
-
-fn_exit:
-#ifdef HAVE_STATUS_SET_BYTES
-    MPI_Type_size_x(datatype, &size);
-    bufsize = size * count;
-    MPIR_Status_set_bytes(status, datatype, bufsize);
-/* This is a temporary way of filling in status. The right way is to 
-   keep track of how much data was actually read and placed in buf 
-   during collective I/O. */
-#endif
-
-    fd->fp_sys_posn = -1;   /* set it to null. */
-}
-
-static void ADIOI_Read_and_exch(ADIO_File fd, void *buf, MPI_Datatype
-			 datatype, int nprocs,
-			 int myrank, ADIOI_Access
-			 *others_req, ADIO_Offset *offset_list,
-			 ADIO_Offset *len_list, int contig_access_count, ADIO_Offset
-                         min_st_offset, ADIO_Offset fd_size,
-			 ADIO_Offset *fd_start, ADIO_Offset *fd_end,
-                         int *buf_idx, int *error_code)
-{
-/* Read in sizes of no more than coll_bufsize, an info parameter.
-   Send data to appropriate processes. 
-   Place recd. data in user buf.
-   The idea is to reduce the amount of extra memory required for
-   collective I/O. If all data were read all at once, which is much
-   easier, it would require temp space more than the size of user_buf,
-   which is often unacceptable. For example, to read a distributed
-   array from a file, where each local array is 8Mbytes, requiring
-   at least another 8Mbytes of temp space is unacceptable. */
-
-    int i, j, m, ntimes, max_ntimes, buftype_is_contig;
-    ADIO_Offset st_loc=-1, end_loc=-1, off, done, real_off, req_off;
-    char *read_buf = NULL, *tmp_buf;
-    int *curr_offlen_ptr, *count, *send_size, *recv_size;
-    int *partial_send, *recd_from_proc, *start_pos;
-    /* Not convinced end_loc-st_loc couldn't be > int, so make these offsets*/
-    ADIO_Offset real_size, size, for_curr_iter, for_next_iter;
-    int req_len, flag, rank;
-    MPI_Status status;
-    ADIOI_Flatlist_node *flat_buf=NULL;
-    MPI_Aint buftype_extent, buftype_lb;
-    int coll_bufsize;
-#ifdef RDCOLL_DEBUG
-    int iii;
-#endif
-    *error_code = MPI_SUCCESS;  /* changed below if error */
-    /* only I/O errors are currently reported */
-    
-/* calculate the number of reads of size coll_bufsize
-   to be done by each process and the max among all processes.
-   That gives the no. of communication phases as well.
-   coll_bufsize is obtained from the hints object. */
-
-    coll_bufsize = fd->hints->cb_buffer_size;
-
-    /* grab some initial values for st_loc and end_loc */
-    for (i=0; i < nprocs; i++) {
-	if (others_req[i].count) {
-	    st_loc = others_req[i].offsets[0];
-	    end_loc = others_req[i].offsets[0];
-	    break;
-	}
-    }
-
-    /* now find the real values */
-    for (i=0; i < nprocs; i++)
-	for (j=0; j<others_req[i].count; j++) {
-	    st_loc = ADIOI_MIN(st_loc, others_req[i].offsets[j]);
-	    end_loc = ADIOI_MAX(end_loc, (others_req[i].offsets[j]
-					  + others_req[i].lens[j] - 1));
-	}
-
-    /* calculate ntimes, the number of times this process must perform I/O
-     * operations in order to complete all the requests it has received.
-     * the need for multiple I/O operations comes from the restriction that
-     * we only use coll_bufsize bytes of memory for internal buffering.
-     */
-    if ((st_loc==-1) && (end_loc==-1)) {
-	/* this process does no I/O. */
-	ntimes = 0;
-    }
-    else {
-	/* ntimes=ceiling_div(end_loc - st_loc + 1, coll_bufsize)*/
-	ntimes = (int) ((end_loc - st_loc + coll_bufsize)/coll_bufsize);
-    }
-
-    MPI_Allreduce(&ntimes, &max_ntimes, 1, MPI_INT, MPI_MAX, fd->comm); 
-
-    read_buf = fd->io_buf;
-
-    curr_offlen_ptr = (int *) ADIOI_Calloc(nprocs, sizeof(int)); 
-    /* its use is explained below. calloc initializes to 0. */
-
-    count = (int *) ADIOI_Malloc(nprocs * sizeof(int));
-    /* to store count of how many off-len pairs per proc are satisfied
-       in an iteration. */
-
-    partial_send = (int *) ADIOI_Calloc(nprocs, sizeof(int));
-    /* if only a portion of the last off-len pair is sent to a process 
-       in a particular iteration, the length sent is stored here.
-       calloc initializes to 0. */
-
-    send_size = (int *) ADIOI_Malloc(nprocs * sizeof(int));
-    /* total size of data to be sent to each proc. in an iteration */
-
-    recv_size = (int *) ADIOI_Malloc(nprocs * sizeof(int));
-    /* total size of data to be recd. from each proc. in an iteration.
-       Of size nprocs so that I can use MPI_Alltoall later. */
-
-    recd_from_proc = (int *) ADIOI_Calloc(nprocs, sizeof(int));
-    /* amount of data recd. so far from each proc. Used in
-       ADIOI_Fill_user_buffer. initialized to 0 here. */
-
-    start_pos = (int *) ADIOI_Malloc(nprocs*sizeof(int));
-    /* used to store the starting value of curr_offlen_ptr[i] in 
-       this iteration */
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-    if (!buftype_is_contig) {
-	flat_buf = ADIOI_Flatten_and_find(datatype);
-    }
-    MPI_Type_get_extent(datatype, &buftype_lb, &buftype_extent);
-
-    done = 0;
-    off = st_loc;
-    for_curr_iter = for_next_iter = 0;
-
-    MPI_Comm_rank(fd->comm, &rank);
-
-#ifdef PROFILE
-        MPE_Log_event(14, 0, "end computation");
-#endif
-
-    for (m=0; m<ntimes; m++) {
-       /* read buf of size coll_bufsize (or less) */
-       /* go through all others_req and check if any are satisfied
-          by the current read */
-
-       /* since MPI guarantees that displacements in filetypes are in 
-          monotonically nondecreasing order, I can maintain a pointer
-	  (curr_offlen_ptr) to 
-          current off-len pair for each process in others_req and scan
-          further only from there. There is still a problem of filetypes
-          such as:  (1, 2, 3 are not process nos. They are just numbers for
-          three chunks of data, specified by a filetype.)
-
-                   1  -------!--
-                   2    -----!----
-                   3       --!-----
-
-          where ! indicates where the current read_size limitation cuts 
-          through the filetype.  I resolve this by reading up to !, but
-          filling the communication buffer only for 1. I copy the portion
-          left over for 2 into a tmp_buf for use in the next
-	  iteration. i.e., 2 and 3 will be satisfied in the next
-	  iteration. This simplifies filling in the user's buf at the
-	  other end, as only one off-len pair with incomplete data
-	  will be sent. I also don't need to send the individual
-	  offsets and lens along with the data, as the data is being
-	  sent in a particular order. */ 
-
-          /* off = start offset in the file for the data actually read in 
-                   this iteration 
-             size = size of data read corresponding to off
-             real_off = off minus whatever data was retained in memory from
-                  previous iteration for cases like 2, 3 illustrated above
-             real_size = size plus the extra corresponding to real_off
-             req_off = off in file for a particular contiguous request 
-                       minus what was satisfied in previous iteration
-             req_size = size corresponding to req_off */
-
-#ifdef PROFILE
-        MPE_Log_event(13, 0, "start computation");
-#endif
-	size = ADIOI_MIN((unsigned)coll_bufsize, end_loc-st_loc+1-done); 
-	real_off = off - for_curr_iter;
-	real_size = size + for_curr_iter;
-
-	for (i=0; i<nprocs; i++) count[i] = send_size[i] = 0;
-	for_next_iter = 0;
-
-	for (i=0; i<nprocs; i++) {
-#ifdef RDCOLL_DEBUG
-	    DBG_FPRINTF(stderr, "rank %d, i %d, others_count %d\n", rank, i, others_req[i].count); 
-#endif
-	    if (others_req[i].count) {
-		start_pos[i] = curr_offlen_ptr[i];
-		for (j=curr_offlen_ptr[i]; j<others_req[i].count;
-		     j++) {
-		    if (partial_send[i]) {
-			/* this request may have been partially
-			   satisfied in the previous iteration. */
-			req_off = others_req[i].offsets[j] +
-			    partial_send[i]; 
-                        req_len = others_req[i].lens[j] -
-			    partial_send[i];
-			partial_send[i] = 0;
-			/* modify the off-len pair to reflect this change */
-			others_req[i].offsets[j] = req_off;
-			others_req[i].lens[j] = req_len;
-		    }
-		    else {
-			req_off = others_req[i].offsets[j];
-                        req_len = others_req[i].lens[j];
-		    }
-		    if (req_off < real_off + real_size) {
-			count[i]++;
-      ADIOI_Assert((((ADIO_Offset)(MPIU_Upint)read_buf)+req_off-real_off) == (ADIO_Offset)(MPIU_Upint)(read_buf+req_off-real_off));
-			MPI_Get_address(read_buf+req_off-real_off, 
-                               &(others_req[i].mem_ptrs[j]));
-      ADIOI_Assert((real_off + real_size - req_off) == (int)(real_off + real_size - req_off));
-			send_size[i] += (int)(ADIOI_MIN(real_off + real_size - req_off, 
-                                      (ADIO_Offset)(unsigned)req_len)); 
-
-			if (real_off+real_size-req_off < (ADIO_Offset)(unsigned)req_len) {
-			    partial_send[i] = (int) (real_off + real_size - req_off);
-			    if ((j+1 < others_req[i].count) && 
-                                 (others_req[i].offsets[j+1] < 
-                                     real_off+real_size)) { 
-				/* this is the case illustrated in the
-				   figure above. */
-				for_next_iter = ADIOI_MAX(for_next_iter,
-					  real_off + real_size - others_req[i].offsets[j+1]); 
-				/* max because it must cover requests 
-				   from different processes */
-			    }
-			    break;
-			}
-		    }
-		    else break;
-		}
-		curr_offlen_ptr[i] = j;
-	    }
-	}
-
-	flag = 0;
-	for (i=0; i<nprocs; i++)
-	    if (count[i]) flag = 1;
-
-#ifdef PROFILE
-        MPE_Log_event(14, 0, "end computation");
-#endif
-	if (flag) {
-	    char round[50];
-	    sprintf(round, "two-phase-round=%d", m);
-	    setenv("LIBIOLOG_EXTRA_INFO", round, 1);
-      ADIOI_Assert(size == (int)size);
-	    ADIO_ReadContig(fd, read_buf+for_curr_iter, (int)size, MPI_BYTE,
-			    ADIO_EXPLICIT_OFFSET, off, &status, error_code);
-#ifdef RDCOLL_DEBUG
-	    DBG_FPRINTF(stderr, "\tread_coll: 700, data read [%lld] = ", size );
-	    for (iii=0; iii<size && iii<80; iii++) { DBGV_FPRINTF(stderr, "%3d,", *((unsigned char *)read_buf + for_curr_iter + iii) ); }
-	    DBG_FPRINTF(stderr, "\n" );
-#endif
-
-	    if (*error_code != MPI_SUCCESS) return;
-	}
-	
-	for_curr_iter = for_next_iter;
-	
-#ifdef PROFILE
-        MPE_Log_event(7, 0, "start communication");
-#endif
-	if (gpfsmpio_comm == 1)
-	ADIOI_R_Exchange_data(fd, buf, flat_buf, offset_list, len_list,
-			    send_size, recv_size, count, 
-       			    start_pos, partial_send, recd_from_proc, nprocs,
-			    myrank, 
-			    buftype_is_contig, contig_access_count,
-			    min_st_offset, fd_size, fd_start, fd_end,
-			    others_req,
-                            m, buftype_extent, buf_idx);
-        else
-	if (gpfsmpio_comm == 0) {
-        ADIOI_R_Exchange_data_alltoallv(fd, buf, flat_buf, offset_list, len_list,
-                            send_size, recv_size, count,
-                            start_pos, partial_send, recd_from_proc, nprocs,
-                            myrank,
-                            buftype_is_contig, contig_access_count,
-                            min_st_offset, fd_size, fd_start, fd_end,
-                            others_req,
-                            m, buftype_extent, buf_idx);
-	}
-
-
-#ifdef PROFILE
-        MPE_Log_event(8, 0, "end communication");
-#endif
-
-	if (for_next_iter) {
-	    tmp_buf = (char *) ADIOI_Malloc(for_next_iter);
-      ADIOI_Assert((((ADIO_Offset)(MPIU_Upint)read_buf)+real_size-for_next_iter) == (ADIO_Offset)(MPIU_Upint)(read_buf+real_size-for_next_iter));
-      ADIOI_Assert((for_next_iter+coll_bufsize) == (size_t)(for_next_iter+coll_bufsize));
-	    memcpy(tmp_buf, read_buf+real_size-for_next_iter, for_next_iter);
-	    ADIOI_Free(fd->io_buf);
-	    fd->io_buf = (char *) ADIOI_Malloc(for_next_iter+coll_bufsize);
-	    memcpy(fd->io_buf, tmp_buf, for_next_iter);
-	    read_buf = fd->io_buf;
-	    ADIOI_Free(tmp_buf);
-	}
-
-	off += size;
-	done += size;
-    }
-
-    for (i=0; i<nprocs; i++) count[i] = send_size[i] = 0;
-#ifdef PROFILE
-        MPE_Log_event(7, 0, "start communication");
-#endif
-    for (m=ntimes; m<max_ntimes; m++) 
-/* nothing to send, but check for recv. */
-
-	if (gpfsmpio_comm == 1)
-	ADIOI_R_Exchange_data(fd, buf, flat_buf, offset_list, len_list,
-			    send_size, recv_size, count, 
-			    start_pos, partial_send, recd_from_proc, nprocs,
-			    myrank, 
-			    buftype_is_contig, contig_access_count,
-			    min_st_offset, fd_size, fd_start, fd_end,
-			    others_req, m,
-                            buftype_extent, buf_idx); 
-        else    /* strncmp( env_switch, "alltoall", 8 ) == 0 */
-	if (gpfsmpio_comm == 0)
-        ADIOI_R_Exchange_data_alltoallv(fd, buf, flat_buf, offset_list, len_list,
-                            send_size, recv_size, count, 
-                            start_pos, partial_send, recd_from_proc, nprocs,
-                            myrank, 
-                            buftype_is_contig, contig_access_count,
-                            min_st_offset, fd_size, fd_start, fd_end,
-                            others_req, 
-                            m, buftype_extent, buf_idx);
-
-#ifdef PROFILE
-        MPE_Log_event(8, 0, "end communication");
-#endif
-
-    ADIOI_Free(curr_offlen_ptr);
-    ADIOI_Free(count);
-    ADIOI_Free(partial_send);
-    ADIOI_Free(send_size);
-    ADIOI_Free(recv_size);
-    ADIOI_Free(recd_from_proc);
-    ADIOI_Free(start_pos);
-
-    unsetenv("LIBIOLOG_EXTRA_INFO");
-}
-
-static void ADIOI_R_Exchange_data(ADIO_File fd, void *buf, ADIOI_Flatlist_node
-			 *flat_buf, ADIO_Offset *offset_list, ADIO_Offset
-                         *len_list, int *send_size, int *recv_size,
-			 int *count, int *start_pos, int *partial_send, 
-			 int *recd_from_proc, int nprocs, 
-			 int myrank, int
-			 buftype_is_contig, int contig_access_count,
-			 ADIO_Offset min_st_offset, ADIO_Offset fd_size,
-			 ADIO_Offset *fd_start, ADIO_Offset *fd_end, 
-			 ADIOI_Access *others_req, 
-                         int iter, MPI_Aint buftype_extent, int *buf_idx)
-{
-    int i, j, k=0, tmp=0, nprocs_recv, nprocs_send;
-    char **recv_buf = NULL; 
-    MPI_Request *requests;
-    MPI_Datatype send_type;
-    MPI_Status *statuses;
-
-/* exchange send_size info so that each process knows how much to
-   receive from whom and how much memory to allocate. */
-
-    MPI_Alltoall(send_size, 1, MPI_INT, recv_size, 1, MPI_INT, fd->comm);
-
-    nprocs_recv = 0;
-    for (i=0; i < nprocs; i++) if (recv_size[i]) nprocs_recv++;
-
-    nprocs_send = 0;
-    for (i=0; i<nprocs; i++) if (send_size[i]) nprocs_send++;
-
-    requests = (MPI_Request *)
-	ADIOI_Malloc((nprocs_send+nprocs_recv+1)*sizeof(MPI_Request));
-/* +1 to avoid a 0-size malloc */
-
-/* post recvs. if buftype_is_contig, data can be directly recd. into
-   user buf at location given by buf_idx. else use recv_buf. */
-
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5032, 0, NULL);
-#endif
-
-    if (buftype_is_contig) {
-	j = 0;
-	for (i=0; i < nprocs; i++) 
-	    if (recv_size[i]) { 
-		MPI_Irecv(((char *) buf) + buf_idx[i], recv_size[i], 
-		  MPI_BYTE, i, myrank+i+100*iter, fd->comm, requests+j);
-		j++;
-		buf_idx[i] += recv_size[i];
-	    }
-    }
-    else {
-/* allocate memory for recv_buf and post receives */
-	recv_buf = (char **) ADIOI_Malloc(nprocs * sizeof(char*));
-	for (i=0; i < nprocs; i++) 
-	    if (recv_size[i]) recv_buf[i] = 
-                                  (char *) ADIOI_Malloc(recv_size[i]);
-
-	    j = 0;
-	    for (i=0; i < nprocs; i++) 
-		if (recv_size[i]) {
-		    MPI_Irecv(recv_buf[i], recv_size[i], MPI_BYTE, i, 
-			      myrank+i+100*iter, fd->comm, requests+j);
-		    j++;
-#ifdef RDCOLL_DEBUG
-		    DBG_FPRINTF(stderr, "node %d, recv_size %d, tag %d \n", 
-		       myrank, recv_size[i], myrank+i+100*iter); 
-#endif
-		}
-    }
-
-/* create derived datatypes and send data */
-
-    j = 0;
-    for (i=0; i<nprocs; i++) {
-	if (send_size[i]) {
-/* take care if the last off-len pair is a partial send */
-	    if (partial_send[i]) {
-		k = start_pos[i] + count[i] - 1;
-		tmp = others_req[i].lens[k];
-		others_req[i].lens[k] = partial_send[i];
-	    }
-	    ADIOI_Type_create_hindexed_x(count[i],
-		  &(others_req[i].lens[start_pos[i]]),
-	            &(others_req[i].mem_ptrs[start_pos[i]]), 
-			 MPI_BYTE, &send_type);
-	    /* absolute displacement; use MPI_BOTTOM in send */
-	    MPI_Type_commit(&send_type);
-	    MPI_Isend(MPI_BOTTOM, 1, send_type, i, myrank+i+100*iter,
-		      fd->comm, requests+nprocs_recv+j);
-	    MPI_Type_free(&send_type);
-	    if (partial_send[i]) others_req[i].lens[k] = tmp;
-	    j++;
-	}
-    }
-
-    statuses = (MPI_Status *) ADIOI_Malloc((nprocs_send+nprocs_recv+1) * \
-                                     sizeof(MPI_Status)); 
-     /* +1 to avoid a 0-size malloc */
-
-    /* wait on the receives */
-    if (nprocs_recv) {
-#ifdef NEEDS_MPI_TEST
-	j = 0;
-	while (!j) MPI_Testall(nprocs_recv, requests, &j, statuses);
-#else
-	MPI_Waitall(nprocs_recv, requests, statuses);
-#endif
-
-	/* if noncontiguous, to the copies from the recv buffers */
-	if (!buftype_is_contig) 
-	    ADIOI_Fill_user_buffer(fd, buf, flat_buf, recv_buf,
-				   offset_list, len_list, (unsigned*)recv_size, 
-				   requests, statuses, recd_from_proc, 
-				   nprocs, contig_access_count,
-				   min_st_offset, fd_size, fd_start, fd_end,
-				   buftype_extent);
-    }
-
-    /* wait on the sends*/
-    MPI_Waitall(nprocs_send, requests+nprocs_recv, statuses+nprocs_recv);
-
-    ADIOI_Free(statuses);
-    ADIOI_Free(requests);
-
-    if (!buftype_is_contig) {
-	for (i=0; i < nprocs; i++) 
-	    if (recv_size[i]) ADIOI_Free(recv_buf[i]);
-	ADIOI_Free(recv_buf);
-    }
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5033, 0, NULL);
-#endif
-}
-
-#define ADIOI_BUF_INCR \
-{ \
-    while (buf_incr) { \
-	size_in_buf = ADIOI_MIN(buf_incr, flat_buf_sz); \
-	user_buf_idx += size_in_buf; \
-	flat_buf_sz -= size_in_buf; \
-	if (!flat_buf_sz) { \
-            if (flat_buf_idx < (flat_buf->count - 1)) flat_buf_idx++; \
-            else { \
-                flat_buf_idx = 0; \
-                n_buftypes++; \
-            } \
-            user_buf_idx = flat_buf->indices[flat_buf_idx] + \
-                              (ADIO_Offset)n_buftypes*(ADIO_Offset)buftype_extent; \
-	    flat_buf_sz = flat_buf->blocklens[flat_buf_idx]; \
-	} \
-	buf_incr -= size_in_buf; \
-    } \
-}
-
-
-#define ADIOI_BUF_COPY \
-{ \
-    while (size) { \
-	size_in_buf = ADIOI_MIN(size, flat_buf_sz); \
-  ADIOI_Assert((((ADIO_Offset)(MPIU_Upint)buf) + user_buf_idx) == (ADIO_Offset)(MPIU_Upint)(buf + user_buf_idx)); \
-  ADIOI_Assert(size_in_buf == (size_t)size_in_buf); \
-	memcpy(((char *) buf) + user_buf_idx, \
-	       &(recv_buf[p][recv_buf_idx[p]]), size_in_buf); \
-	recv_buf_idx[p] += size_in_buf; /* already tested (size_t)size_in_buf*/ \
-	user_buf_idx += size_in_buf; \
-	flat_buf_sz -= size_in_buf; \
-	if (!flat_buf_sz) { \
-            if (flat_buf_idx < (flat_buf->count - 1)) flat_buf_idx++; \
-            else { \
-                flat_buf_idx = 0; \
-                n_buftypes++; \
-            } \
-            user_buf_idx = flat_buf->indices[flat_buf_idx] + \
-                              (ADIO_Offset)n_buftypes*(ADIO_Offset)buftype_extent; \
-	    flat_buf_sz = flat_buf->blocklens[flat_buf_idx]; \
-	} \
-	size -= size_in_buf; \
-	buf_incr -= size_in_buf; \
-    } \
-    ADIOI_BUF_INCR \
-}
-
-static void ADIOI_Fill_user_buffer(ADIO_File fd, void *buf, ADIOI_Flatlist_node
-				   *flat_buf, char **recv_buf, ADIO_Offset 
-				   *offset_list, ADIO_Offset *len_list, 
-				   unsigned *recv_size, 
-				   MPI_Request *requests, MPI_Status *statuses,
-				   int *recd_from_proc, int nprocs,
-				   int contig_access_count, 
-				   ADIO_Offset min_st_offset, 
-				   ADIO_Offset fd_size, ADIO_Offset *fd_start, 
-				   ADIO_Offset *fd_end,
-				   MPI_Aint buftype_extent)
-{
-
-/* this function is only called if buftype is not contig */
-
-    int i, p, flat_buf_idx;
-    ADIO_Offset flat_buf_sz, size_in_buf, buf_incr, size;
-    int n_buftypes;
-    ADIO_Offset off, len, rem_len, user_buf_idx;
-    /* Not sure unsigned is necessary, but it makes the math safer */
-    unsigned *curr_from_proc, *done_from_proc, *recv_buf_idx;
-
-    ADIOI_UNREFERENCED_ARG(requests);
-    ADIOI_UNREFERENCED_ARG(statuses);
-
-/*  curr_from_proc[p] = amount of data recd from proc. p that has already
-                        been accounted for so far
-    done_from_proc[p] = amount of data already recd from proc. p and 
-                        filled into user buffer in previous iterations
-    user_buf_idx = current location in user buffer 
-    recv_buf_idx[p] = current location in recv_buf of proc. p  */
-    curr_from_proc = (unsigned *) ADIOI_Malloc(nprocs * sizeof(unsigned));
-    done_from_proc = (unsigned *) ADIOI_Malloc(nprocs * sizeof(unsigned));
-    recv_buf_idx   = (unsigned *) ADIOI_Malloc(nprocs * sizeof(unsigned));
-
-    for (i=0; i < nprocs; i++) {
-	recv_buf_idx[i] = curr_from_proc[i] = 0;
-	done_from_proc[i] = recd_from_proc[i];
-    }
-
-    user_buf_idx = flat_buf->indices[0];
-    flat_buf_idx = 0;
-    n_buftypes = 0;
-    flat_buf_sz = flat_buf->blocklens[0];
-
-    /* flat_buf_idx = current index into flattened buftype
-       flat_buf_sz = size of current contiguous component in 
-                flattened buf */
-
-    for (i=0; i<contig_access_count; i++) { 
-	off     = offset_list[i];
-	rem_len = len_list[i];
-
-	/* this request may span the file domains of more than one process */
-	while (rem_len > 0) {
-	    len = rem_len;
-	    /* NOTE: len value is modified by ADIOI_Calc_aggregator() to be no
-	     * longer than the single region that processor "p" is responsible
-	     * for.
-	     */
-	    p = ADIOI_GPFS_Calc_aggregator(fd,
-				      off,
-				      min_st_offset,
-				      &len,
-				      fd_size,
-				      fd_start,
-				      fd_end);
-
-	    if (recv_buf_idx[p] < recv_size[p]) {
-		if (curr_from_proc[p]+len > done_from_proc[p]) {
-		    if (done_from_proc[p] > curr_from_proc[p]) {
-			size = ADIOI_MIN(curr_from_proc[p] + len - 
-			      done_from_proc[p], recv_size[p]-recv_buf_idx[p]);
-			buf_incr = done_from_proc[p] - curr_from_proc[p];
-			ADIOI_BUF_INCR
-			buf_incr = curr_from_proc[p]+len-done_from_proc[p];
-      ADIOI_Assert((done_from_proc[p] + size) == (unsigned)((ADIO_Offset)done_from_proc[p] + size));
-			curr_from_proc[p] = done_from_proc[p] + size;
-			ADIOI_BUF_COPY
-		    }
-		    else {
-			size = ADIOI_MIN(len,recv_size[p]-recv_buf_idx[p]);
-			buf_incr = len;
-      ADIOI_Assert((curr_from_proc[p] + size) == (unsigned)((ADIO_Offset)curr_from_proc[p] + size));
-			curr_from_proc[p] += (unsigned) size;
-			ADIOI_BUF_COPY
-		    }
-		}
-		else {
-        ADIOI_Assert((curr_from_proc[p] + len) == (unsigned)((ADIO_Offset)curr_from_proc[p] + len));
-		    curr_from_proc[p] += (unsigned) len;
-		    buf_incr = len;
-		    ADIOI_BUF_INCR
-		}
-	    }
-	    else {
-		buf_incr = len;
-		ADIOI_BUF_INCR
-	    }
-	    off     += len;
-	    rem_len -= len;
-	}
-    }
-    for (i=0; i < nprocs; i++) 
-	if (recv_size[i]) recd_from_proc[i] = curr_from_proc[i];
-
-    ADIOI_Free(curr_from_proc);
-    ADIOI_Free(done_from_proc);
-    ADIOI_Free(recv_buf_idx);
-}
-
-static void ADIOI_R_Exchange_data_alltoallv(
-                ADIO_File fd, void *buf, ADIOI_Flatlist_node
-                *flat_buf, ADIO_Offset *offset_list, ADIO_Offset
-                *len_list, int *send_size, int *recv_size, 
-                int *count, int *start_pos, int *partial_send,
-                int *recd_from_proc, int nprocs,
-                int myrank, int
-                buftype_is_contig, int contig_access_count,
-                ADIO_Offset min_st_offset, ADIO_Offset fd_size,
-                ADIO_Offset *fd_start, ADIO_Offset *fd_end, 
-                ADIOI_Access *others_req,
-                int iter, MPI_Aint buftype_extent, int *buf_idx)
-{   
-    int i, j, k=0, tmp=0, nprocs_recv, nprocs_send;
-    char **recv_buf = NULL;
-    MPI_Request *requests=NULL;
-    MPI_Status *statuses=NULL;
-    int rtail, stail;
-    char *sbuf_ptr, *from_ptr;
-    int  len;
-    int  *sdispls, *rdispls;
-    char *all_recv_buf, *all_send_buf;
-
-  /* exchange send_size info so that each process knows how much to
-     receive from whom and how much memory to allocate. */
-    MPI_Alltoall(send_size, 1, MPI_INT, recv_size, 1, MPI_INT, fd->comm);
-    
-    nprocs_recv = 0;
-    for (i=0; i<nprocs; i++) if (recv_size[i]) { nprocs_recv++; break; }
-    
-    nprocs_send = 0;
-    for (i=0; i<nprocs; i++) if (send_size[i]) { nprocs_send++; break; }
-    
-  /* receiver side data structures */
-    rdispls = (int *) ADIOI_Malloc( nprocs * sizeof(int) );
-    rtail = 0;
-    for (i=0; i<nprocs; i++) { rdispls[i] = rtail; rtail += recv_size[i]; }
-
-        /* data buffer */
-    all_recv_buf = (char *) ADIOI_Malloc( rtail );
-    recv_buf = (char **) ADIOI_Malloc(nprocs * sizeof(char *));
-    for (i=0; i<nprocs; i++) { recv_buf[i] = all_recv_buf + rdispls[i]; }
-
-  /* sender side data structures */
-    sdispls = (int *) ADIOI_Malloc( nprocs * sizeof(int) );
-    stail = 0;
-    for (i=0; i<nprocs; i++) { sdispls[i] = stail; stail += send_size[i]; }
-
-        /* data buffer */
-    all_send_buf = (char *) ADIOI_Malloc( stail );
-    for (i=0; i<nprocs; i++)
-    {
-        if (send_size[i]) {
-	    if (partial_send[i]) {
-		k = start_pos[i] + count[i] - 1;
-		tmp = others_req[i].lens[k];
-		others_req[i].lens[k] = partial_send[i];
-	    }
-            sbuf_ptr = all_send_buf + sdispls[i];
-            for (j=0; j<count[i]; j++) {
-                ADIOI_ENSURE_AINT_FITS_IN_PTR( others_req[i].mem_ptrs[ start_pos[i]+j ]);
-                from_ptr = (char *) ADIOI_AINT_CAST_TO_VOID_PTR ( others_req[i].mem_ptrs[ start_pos[i]+j ] );
-                len      =           others_req[i].lens[     start_pos[i]+j ]  ;
-                memcpy( sbuf_ptr, from_ptr, len );
-                sbuf_ptr += len;
-            }
-	    if (partial_send[i]) others_req[i].lens[k] = tmp;
-        }
-    }
-
-#if RDCOLL_DEBUG
-    DBG_FPRINTF(stderr, "\tsend_size = [%d]%2d,",0,send_size[0]);
-    for (i=1; i<nprocs; i++) if(send_size[i-1]!=send_size[i]){ DBG_FPRINTF(stderr, "\t\t[%d]%2d,", i,send_size[i] ); }
-    DBG_FPRINTF(stderr, "\trecv_size =  [%d]%2d,",0,recv_size[0]);
-    for (i=1; i<nprocs; i++) if(recv_size[i-1]!=recv_size[i]){ DBG_FPRINTF(stderr, "\t\t[%d]%2d,", i,recv_size[i] ); }
-    DBG_FPRINTF(stderr, "\tsdispls   =  [%d]%2d,",0,sdispls[0]);
-    for (i=1; i<nprocs; i++) if(sdispls[i-1]!=sdispls[i]){ DBG_FPRINTF(stderr, "\t\t[%d]%2d,", i,sdispls  [i] ); }
-    DBG_FPRINTF(stderr, "\trdispls   =  [%d]%2d,",0,rdispls[0]);
-    for (i=1; i<nprocs; i++) if(rdispls[i-1]!=rdispls[i]){ DBG_FPRINTF(stderr, "\t\t[%d]%2d,", i,rdispls  [i] ); }
-    DBG_FPRINTF(stderr, "\ttails = %4d, %4d\n", stail, rtail );
-    if (nprocs_send) {
-    DBG_FPRINTF(stderr, "\tall_send_buf =  [%d]%2d,",0,all_send_buf[0]);
-    /* someone at some point found it useful to look at the 128th kilobyte of data from each processor, but this segfaults in many situations if "all debugging" enabled */
-    //for (i=1; i<nprocs; i++) if(all_send_buf[(i-1)*131072]!=all_send_buf[i*131072]){ DBG_FPRINTF(stderr, "\t\t[%d]%2d,", i, all_send_buf  [i*131072] ); }
-    }
-#endif
-    
-  /* alltoallv */
-    MPI_Alltoallv( 
-            all_send_buf, send_size, sdispls, MPI_BYTE,
-            all_recv_buf, recv_size, rdispls, MPI_BYTE,
-            fd->comm ); 
-
-#if 0
-    DBG_FPRINTF(stderr, "\tall_recv_buf = " );
-    for (i=131072; i<131073; i++) { DBG_FPRINTF(stderr, "%2d,", all_recv_buf  [i] ); }
-    DBG_FPRINTF(stderr, "\n" );
-#endif
-    
-  /* unpack at the receiver side */
-    if (nprocs_recv) { 
-        if (!buftype_is_contig)
-            ADIOI_Fill_user_buffer(fd, buf, flat_buf, recv_buf,
-                                   offset_list, len_list, (unsigned*)recv_size,
-                                   requests, statuses,          /* never used inside */
-                                   recd_from_proc,
-                                   nprocs, contig_access_count,
-                                   min_st_offset, fd_size, fd_start, fd_end,
-                               buftype_extent);
-        else {
-	    rtail = 0;
-            for (i=0; i < nprocs; i++)
-                if (recv_size[i]) {
-                    memcpy( (char *)buf + buf_idx[i], all_recv_buf + rtail, recv_size[i] );
-		    buf_idx[i] += recv_size[i];
-		    rtail += recv_size[i];
-		}
-        }
-    }
-    
-    ADIOI_Free( all_send_buf );
-    ADIOI_Free( all_recv_buf );
-    ADIOI_Free( recv_buf  );
-    ADIOI_Free( sdispls );
-    ADIOI_Free( rdispls );
-    return; 
-}   
diff --git a/3rd-party/romio321/adio/ad_gpfs/ad_gpfs_tuning.c b/3rd-party/romio321/adio/ad_gpfs/ad_gpfs_tuning.c
deleted file mode 100644
index 333612b0da1..00000000000
--- a/3rd-party/romio321/adio/ad_gpfs/ad_gpfs_tuning.c
+++ /dev/null
@@ -1,338 +0,0 @@
-/* ---------------------------------------------------------------- */
-/* (C)Copyright IBM Corp.  2007, 2008                               */
-/* ---------------------------------------------------------------- */
-/**
- * \file ad_gpfs_tuning.c
- * \brief Defines ad_gpfs performance tuning
- */
-
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/*
- *   Copyright (C) 2008 University of Chicago.
- *   See COPYRIGHT notice in top-level directory.
- */
-
-/*---------------------------------------------------------------------
- * ad_gpfs_tuning.c
- *
- * defines global variables and functions for performance tuning and
- * functional debugging.
- *---------------------------------------------------------------------*/
-
-#include "ad_gpfs_tuning.h"
-#include "mpi.h"
-
-#if !defined(PVFS2_SUPER_MAGIC)
-  #define PVFS2_SUPER_MAGIC (0x20030528)
-#endif
-
-
-int 	gpfsmpio_timing;
-int 	gpfsmpio_timing2;
-int     gpfsmpio_timing_cw_level;
-int 	gpfsmpio_comm;
-int 	gpfsmpio_tunegather;
-int 	gpfsmpio_tuneblocking;
-long    bglocklessmpio_f_type;
-int     gpfsmpio_bg_nagg_pset;
-int     gpfsmpio_pthreadio;
-int     gpfsmpio_p2pcontig;
-int     gpfsmpio_write_aggmethod;
-int     gpfsmpio_read_aggmethod;
-int	gpfsmpio_balancecontig;
-int     gpfsmpio_devnullio;
-int     gpfsmpio_bridgeringagg;
-int     gpfsmpio_onesided_no_rmw;
-int     gpfsmpio_onesided_always_rmw;
-int     gpfsmpio_onesided_inform_rmw;
-
-double	gpfsmpio_prof_cw    [GPFSMPIO_CIO_LAST+1];
-double	gpfsmpio_prof_cr    [GPFSMPIO_CIO_LAST+1];
-
-/* set internal variables for tuning environment variables */
-/** \page mpiio_vars MPIIO Configuration
-  \section env_sec Environment Variables
- * - GPFSMPIO_COMM - Define how data is exchanged on collective
- *   reads and writes.  Possible values:
- *   - 0 - Use MPI_Alltoallv.
- *   - 1 - Use MPI_Isend/MPI_Irecv.
- *   - Default is 0.
- *
- * - GPFSMPIO_TIMING - collect timing breakdown for MPI I/O collective calls.
- *   Possible values:
- *   - 0 - Do not collect/report timing.
- *   - 1 - Collect/report timing.
- *   - Default is 0.
- *
- * - GPFSMPIO_TUNEGATHER - Tune how starting and ending offsets are communicated
- *   for aggregator collective i/o.  Possible values:
- *   - 0 - Use two MPI_Allgather's to collect starting and ending offsets.
- *   - 1 - Use MPI_Allreduce(MPI_MAX) to collect starting and ending offsets.
- *   - Default is 1.
- *
- * - GPFSMPIO_TUNEBLOCKING - Tune how aggregate file domains are
- *   calculated (block size).  Possible values:
- *   - 0 - Evenly calculate file domains across aggregators.  Also use
- *   MPI_Isend/MPI_Irecv to exchange domain information.
- *   - 1 - Align file domains with the underlying file system's block size.  Also use
- *   MPI_Alltoallv to exchange domain information.
- *   - Default is 1.
- *
- * - BGLOCKLESSMPIO_F_TYPE - Specify a filesystem type that should run
- *   the ad_bglockless driver.   NOTE: Using romio prefixes (such as
- *   "bg:" or "bglockless:") on a file name will override this environment
- *   variable.  Possible values:
- *   - 0xnnnnnnnn - Any valid file system type (or "magic number") from
- *                  statfs() field f_type.
- *   - The default is 0x20030528 (PVFS2_SUPER_MAGIC)
- *
- * - GPFSMPIO_NAGG_PSET - Specify a ratio of "I/O aggregators" to use for each
- *   compute group (compute nodes + i/o nodes).    Possible values:
- *   - any integer
- *   - Default is 8
- *
- * - GPFSMPIO_PTHREADIO - Enables a very simple form of asyncronous io where a
- *   pthread is spawned to do the posix writes while the main thread does the
- *   data aggregation - useful for large files where multiple rounds are
- *   required (more that the cb_buffer_size of data per aggregator).   User
- *   must ensure there is hw resource available for the thread to run.  I
- *   am sure there is a better way to do this involving comm threads - this is
- *   just a start.  NOTE: For some reason the stats collected when this is
- *   enabled misses some of the data so the data sizes are off a bit - this is
- *   a statistical issue only, the data is still accurately written out
- *
- * - GPFSMPIO_P2PCONTIG -  Does simple point-to-point communication between the
- *   aggregator and the procs that feed it.  Performance could be enhanced by a
- *   one-sided put algorithm.  Current implementation allows only 1 round of
- *   data.  Useful/allowed only when:
- * 1.) The datatype is contiguous.
- * 2.) The offsets are increasing in rank-order.
- * 3.) There are no gaps between the offsets.
- * 4.) No single rank has a data size which spans multiple file domains.
- *
- * - GPFSMPIO_WRITE_AGGMETHOD/GPFSMPIO_READ_AGGMETHOD -  Replaces the two-phase
- *   collective IO aggregation
- *   with a one-sided algorithm, significantly reducing communication and
- *   memory overhead.  Fully
- *   supports all datasets and datatypes, the only caveat is that any holes in the data
- *   when writing to a pre-existing file are ignored -- there is no read-modify-write
- *   support to maintain the correctness of regions of pre-existing data so every byte
- *   must be explicitly written to maintain correctness.  Users must beware of middle-ware
- *   libraries like PNETCDF which may count on read-modify-write functionality for certain
- *   features (like fill values).  Possible values:
- *   - 0 - Normal two-phase collective IO is used.
- *   - 1 - A separate one-sided MPI_Put or MPI_Get is used for each contigous chunk of data
- *         for a compute to write to or read from the collective buffer on the aggregator.
- *   - 2 - An MPI derived datatype is created using all the contigous chunks and just one
- *         call to MPI_Put or MPI_Get is done with the derived datatype.  On Blue Gene /Q
- *         optimal performance for this is achieved when paired with PAMID_TYPED_ONESIDED=1.
- *   - Default is 0
- *
- * - GPFSMPIO_ONESIDED_NO_RMW - For one-sided aggregation (GPFSMPIO_WRITE_AGGMETHOD = 1 or 2)
- *   disable the detection of holes in the data when writing to a pre-existing
- *   file requiring a read-modify-write, thereby avoiding the communication
- *   overhead for this detection.
- *   - 0 (hole detection enabled) or 1 (hole detection disabled)
- *   - Default is 0
- *
- * - GPFSMPIO_ONESIDED_INFORM_RMW - For one-sided aggregation
- *   (GPFSMPIO_AGGMETHOD = 1 or 2) generate an informational message informing
- *   the user whether holes exist in the data when writing to a pre-existing
- *   file requiring a read-modify-write, thereby educating the user to set
- *   GPFSMPIO_ONESIDED_NO_RMW=1 on a future run to avoid the communication
- *   overhead for this detection.
- *   - 0 (disabled) or 1 (enabled)
- *   - Default is 0
- *
- * - GPFSMPIO_BALANCECONTIG -  Relevant only to BGQ.  File domain blocks are assigned
- *   to aggregators in a breadth-first fashion relative to the ions - additionally,
- *   file domains on the aggregators sharing the same bridgeset and ion have contiguous
- *   offsets.  The breadth-first assignment improves performance in the case of
- *   a relatively small file of size less than the gpfs block size multiplied
- *   by the number of ions. Files: ad_gpfs_aggrs.c ad_bg_aggrs.c.  Possible Values
- *   - 0 - assign file domain blocks in the traditional manner
- *   - 1 - if there are variable sized file domain blocks, spread them out
- *         (balance) across bridge nodes
- *
- * - GPFSMPIO_DEVNULLIO - do everything *except* write to / read from the file
- *   system. When experimenting with different two-phase I/O strategies, it's
- *   helpful to remove the highly variable file system from the experiment.
- *   - 0 (disabled) or 1 (enabled)
- *   - Default is 0
- *
- * - GPFSMPIO_BRIDGERINGAGG - Relevant only to BGQ.  Aggregator placement
- *   optimization whch forms a 5-d ring around the bridge node starting at
- *   GPFSMPIO_BRIDGERINGAGG hops away.  Experimental performance results
- *   suggest best value is 1 and only in conjunction with GPFSMPIO_P2PCONTIG
- *   and GPFSMPIO_BALANCECONTIG.  The number of aggregators selected is still
- *   GPFSMPIO_NAGG_PSET however the bridge node itself is NOT selected.
- *
- */
-
-void ad_gpfs_get_env_vars() {
-    char *x, *dummy;
-
-    gpfsmpio_comm   = 0;
-	x = getenv( "GPFSMPIO_COMM"         );
-	if (x) gpfsmpio_comm         = atoi(x);
-    gpfsmpio_timing = 0;
-	x = getenv( "GPFSMPIO_TIMING"       );
-	if (x) gpfsmpio_timing       = atoi(x);
-    gpfsmpio_tunegather = 1;
-	x = getenv( "GPFSMPIO_TUNEGATHER"   );
-	if (x) gpfsmpio_tunegather   = atoi(x);
-    gpfsmpio_tuneblocking = 1;
-    x = getenv( "GPFSMPIO_TUNEBLOCKING" );
-    if (x) gpfsmpio_tuneblocking = atoi(x);
-    bglocklessmpio_f_type = PVFS2_SUPER_MAGIC;
-    x = getenv( "BGLOCKLESSMPIO_F_TYPE" );
-    if (x) bglocklessmpio_f_type = strtol(x,&dummy,0);
-    DBG_FPRINTF(stderr,"BGLOCKLESSMPIO_F_TYPE=%ld/%#lX\n",
-            bglocklessmpio_f_type,bglocklessmpio_f_type);
-    /* note: this value will be 'sanity checked' in ADIOI_BG_persInfo_init(),
-     * when we know a bit more about what "largest possible value" and
-     * "smallest possible value" should be */
-    gpfsmpio_bg_nagg_pset = ADIOI_BG_NAGG_PSET_DFLT;
-    x = getenv("GPFSMPIO_NAGG_PSET");
-    if (x) gpfsmpio_bg_nagg_pset = atoi(x);
-
-    gpfsmpio_pthreadio = 0;
-    x = getenv( "GPFSMPIO_PTHREADIO" );
-    if (x) gpfsmpio_pthreadio = atoi(x);
-
-    gpfsmpio_p2pcontig = 0;
-    x = getenv( "GPFSMPIO_P2PCONTIG" );
-    if (x) gpfsmpio_p2pcontig = atoi(x);
-
-    gpfsmpio_write_aggmethod = 0;
-    x = getenv( "GPFSMPIO_WRITE_AGGMETHOD" );
-    if (x) gpfsmpio_write_aggmethod = atoi(x);
-
-    gpfsmpio_read_aggmethod = 0;
-    x = getenv( "GPFSMPIO_READ_AGGMETHOD" );
-    if (x) gpfsmpio_read_aggmethod = atoi(x);
-
-    gpfsmpio_balancecontig = 0;
-    x = getenv( "GPFSMPIO_BALANCECONTIG" );
-    if (x) gpfsmpio_balancecontig = atoi(x);
-
-    gpfsmpio_devnullio = 0;
-    x = getenv( "GPFSMPIO_DEVNULLIO" );
-    if (x) gpfsmpio_devnullio = atoi(x);
-
-    gpfsmpio_bridgeringagg = 0;
-    x = getenv( "GPFSMPIO_BRIDGERINGAGG" );
-    if (x) gpfsmpio_bridgeringagg = atoi(x);
-
-    gpfsmpio_onesided_no_rmw = 0;
-    x = getenv( "GPFSMPIO_ONESIDED_NO_RMW" );
-    if (x) gpfsmpio_onesided_no_rmw = atoi(x);
-
-    gpfsmpio_onesided_always_rmw = 0;
-    x = getenv( "GPFSMPIO_ONESIDED_ALWAYS_RMW" );
-    if (x) gpfsmpio_onesided_always_rmw = atoi(x);
-    if (gpfsmpio_onesided_always_rmw)
-      gpfsmpio_onesided_no_rmw = 1;
-
-    gpfsmpio_onesided_inform_rmw = 0;
-    x = getenv( "GPFSMPIO_ONESIDED_INFORM_RMW" );
-    if (x) gpfsmpio_onesided_inform_rmw = atoi(x);
-}
-
-/* report timing breakdown for MPI I/O collective call */
-void ad_gpfs_timing_crw_report( int rw, ADIO_File fd, int myrank, int nprocs )
-{
-    int i;
-
-    if (gpfsmpio_timing) {
-	/* Timing across the whole communicator is a little bit interesting,
-	 * but what is *more* interesting is if we single out the aggregators
-	 * themselves.  non-aggregators spend a lot of time in "exchange" not
-	 * exchanging data, but blocked because they are waiting for
-	 * aggregators to finish writing.  If we focus on just the aggregator
-	 * processes we will get a more clear picture about the data exchange
-	 * vs. i/o time breakdown */
-
-	/* if deferred open enabled, we could use the aggregator communicator */
-	MPI_Comm agg_comm;
-	int nr_aggs, agg_rank;
-	MPI_Comm_split(fd->comm, (fd->is_agg ? 1 : MPI_UNDEFINED), 0, &agg_comm);
-	if(agg_comm != MPI_COMM_NULL) {
-	    MPI_Comm_size(agg_comm, &nr_aggs);
-	    MPI_Comm_rank(agg_comm, &agg_rank);
-	}
-
-	double *gpfsmpio_prof_org = gpfsmpio_prof_cr;
-	if (rw) gpfsmpio_prof_org = gpfsmpio_prof_cw;
-
-	double gpfsmpio_prof_avg[ GPFSMPIO_CIO_LAST ];
-	double gpfsmpio_prof_max[ GPFSMPIO_CIO_LAST ];
-
-	if( agg_comm != MPI_COMM_NULL) {
-	    MPI_Reduce( gpfsmpio_prof_org, gpfsmpio_prof_avg, GPFSMPIO_CIO_LAST, MPI_DOUBLE, MPI_SUM, 0, agg_comm);
-	    MPI_Reduce( gpfsmpio_prof_org, gpfsmpio_prof_max, GPFSMPIO_CIO_LAST, MPI_DOUBLE, MPI_MAX, 0, agg_comm);
-	}
-	if (agg_comm != MPI_COMM_NULL && agg_rank == 0) {
-
-	    for (i=0; i<GPFSMPIO_CIO_LAST; i++) gpfsmpio_prof_avg[i] /= nr_aggs;
-
-	    gpfsmpio_prof_avg[ GPFSMPIO_CIO_B_POSI_RW  ] =
-		gpfsmpio_prof_avg[ GPFSMPIO_CIO_DATA_SIZE ] * nr_aggs /
-		gpfsmpio_prof_max[ GPFSMPIO_CIO_T_POSI_RW  ];
-	    gpfsmpio_prof_avg[ GPFSMPIO_CIO_B_MPIO_RW  ] =
-		gpfsmpio_prof_avg[ GPFSMPIO_CIO_DATA_SIZE ] * nr_aggs /
-		gpfsmpio_prof_max[ GPFSMPIO_CIO_T_MPIO_RW  ];
-
-	    gpfsmpio_prof_avg[ GPFSMPIO_CIO_B_MPIO_CRW ] =
-		gpfsmpio_prof_avg[ GPFSMPIO_CIO_DATA_SIZE ] * nr_aggs /
-		gpfsmpio_prof_max[ GPFSMPIO_CIO_T_MPIO_CRW ];
-
-	    fprintf(stderr,"TIMING-%1s,", (rw ? "W" : "R") );
-	    fprintf(stderr,"SIZE: %12.4lld , ", (long long int)(gpfsmpio_prof_avg[ GPFSMPIO_CIO_DATA_SIZE ] * nr_aggs));
-	    fprintf(stderr,"SEEK-avg: %10.3f , ",
-		    gpfsmpio_prof_avg[ GPFSMPIO_CIO_T_SEEK ]     );
-	    fprintf(stderr,"SEEK-max: %10.3f , ",
-		    gpfsmpio_prof_max[ GPFSMPIO_CIO_T_SEEK ]     );
-	    fprintf(stderr,"LOCAL-avg: %10.3f , ",
-		    gpfsmpio_prof_avg[ GPFSMPIO_CIO_T_LCOMP ]    );
-	    fprintf(stderr,"GATHER-max: %10.3f , ",
-		    gpfsmpio_prof_max[ GPFSMPIO_CIO_T_GATHER ]   );
-	    fprintf(stderr,"PATTERN-avg: %10.3f , ",
-		    gpfsmpio_prof_avg[ GPFSMPIO_CIO_T_PATANA ]   );
-	    fprintf(stderr,"FILEDOMAIN-avg: %10.3f , ",
-		    gpfsmpio_prof_avg[ GPFSMPIO_CIO_T_FD_PART ]  );
-	    fprintf(stderr,"MYREQ-avg: %10.3f , ",
-		    gpfsmpio_prof_avg[ GPFSMPIO_CIO_T_MYREQ ]    );
-	    fprintf(stderr,"OTHERREQ-max: %10.3f , ",
-		    gpfsmpio_prof_max[ GPFSMPIO_CIO_T_OTHREQ ]   );
-	    fprintf(stderr,"EXCHANGE-max: %10.3f , ",
-		    gpfsmpio_prof_max[ GPFSMPIO_CIO_T_DEXCH ]    );
-	    fprintf(stderr, "EXCHANGE-RECV_EXCH-max: %10.3f , ",
-		    gpfsmpio_prof_max[ GPFSMPIO_CIO_T_DEXCH_RECV_EXCH]  );
-	    fprintf(stderr, "EXCHANGE-SETUP-max: %10.3f , ",
-		    gpfsmpio_prof_max[ GPFSMPIO_CIO_T_DEXCH_SETUP]  );
-	    fprintf(stderr, "EXCHANGE-NET-max: %10.3f , ",
-		    gpfsmpio_prof_max[ GPFSMPIO_CIO_T_DEXCH_NET]  );
-	    fprintf(stderr, "EXCHANGE-SORT-max: %10.3f , ",
-		    gpfsmpio_prof_max[ GPFSMPIO_CIO_T_DEXCH_SORT]  );
-	    fprintf(stderr, "EXCHANGE-SIEVE-max: %10.3f , ",
-		    gpfsmpio_prof_max[ GPFSMPIO_CIO_T_DEXCH_SIEVE]  );
-	    fprintf(stderr,"POSIX-TIME-avg: %10.3f , ",
-		    gpfsmpio_prof_avg[ GPFSMPIO_CIO_T_POSI_RW ]  );
-	    fprintf(stderr,"POSIX-TIME-max: %10.3f , ",
-		    gpfsmpio_prof_max[ GPFSMPIO_CIO_T_POSI_RW ]  );
-	    fprintf(stderr,"MPIIO-CONTIG-TIME-avg: %10.3f , ",
-		    gpfsmpio_prof_avg[ GPFSMPIO_CIO_T_MPIO_RW ]  );
-	    fprintf(stderr,"MPIIO-STRIDED-TIME-avg: %10.3f , ",
-		    gpfsmpio_prof_avg[ GPFSMPIO_CIO_T_MPIO_CRW ] );
-	    fprintf(stderr,"POSIX-BW-avg: %10.3f , ",
-		    gpfsmpio_prof_avg[ GPFSMPIO_CIO_B_POSI_RW ]  );
-	    fprintf(stderr,"MPI-BW-avg: %10.3f , ",
-		    gpfsmpio_prof_avg[ GPFSMPIO_CIO_B_MPIO_RW ]  );
-	    fprintf(stderr,"MPI-BW-collective-avg: %10.3f\n ",
-		    gpfsmpio_prof_avg[ GPFSMPIO_CIO_B_MPIO_CRW ] );
-	}
-	if (agg_comm != MPI_COMM_NULL) MPI_Comm_free(&agg_comm);
-    }
-
-}
diff --git a/3rd-party/romio321/adio/ad_gpfs/ad_gpfs_tuning.h b/3rd-party/romio321/adio/ad_gpfs/ad_gpfs_tuning.h
deleted file mode 100644
index 35fdc8d8c36..00000000000
--- a/3rd-party/romio321/adio/ad_gpfs/ad_gpfs_tuning.h
+++ /dev/null
@@ -1,119 +0,0 @@
-/* ---------------------------------------------------------------- */
-/* (C)Copyright IBM Corp.  2007, 2008                               */
-/* ---------------------------------------------------------------- */
-/**
- * \file ad_gpfs_tuning.h
- * \brief ???
- */
-
-/*---------------------------------------------------------------------
- * ad_gpfs_tuning.h
- *
- * declares global variables and macros for performance tuning and
- * functional debugging.
- *---------------------------------------------------------------------*/
-
-#ifndef AD_GPFS_TUNING_H_
-#define AD_GPFS_TUNING_H_
-
-#include "adio.h"
-
-
-/*-----------------------------------------
- *  Global variables for the control of
- *  1.  timing
- *  2.  select specific optimizations
- *  3.  global flags for certain optimizations
- *-----------------------------------------*/
-
-/* timing fields */
-enum {
-    GPFSMPIO_CIO_DATA_SIZE=0,
-    GPFSMPIO_CIO_T_SEEK,
-    GPFSMPIO_CIO_T_LCOMP,	/* time for ADIOI_Calc_my_off_len(), local */
-    GPFSMPIO_CIO_T_GATHER,	/* time for previous MPI_Allgather, now Allreduce */
-    GPFSMPIO_CIO_T_PATANA,	/* time for a quick test if access is contiguous or not, local */
-    GPFSMPIO_CIO_T_FD_PART,	/* time for file domain partitioning, local */
-    GPFSMPIO_CIO_T_MYREQ,	/* time for ADIOI_Calc_my_req(), local */
-    GPFSMPIO_CIO_T_OTHREQ,	/* time for ADIOI_Calc_others_req(), short Alltoall */
-    GPFSMPIO_CIO_T_DEXCH,	/* time for I/O data exchange */
-    /* the next DEXCH_* timers capture finer-grained portions of T_DEXCH */
-    GPFSMPIO_CIO_T_DEXCH_RECV_EXCH,/* time for each process to exchange recieve
-				    size info with everyone else */
-    GPFSMPIO_CIO_T_DEXCH_SETUP,	/* time for setup portion of I/O data exchange */
-    GPFSMPIO_CIO_T_DEXCH_NET,	/* time for network portion of I/O data exchange */
-    GPFSMPIO_CIO_T_DEXCH_SORT, 	/* time to sort requesst in I/O data exchange */
-    GPFSMPIO_CIO_T_DEXCH_SIEVE, 	/* time for read portion of RMW in two phase */
-    GPFSMPIO_CIO_T_POSI_RW,
-    GPFSMPIO_CIO_B_POSI_RW,
-    GPFSMPIO_CIO_T_MPIO_RW,	/* time for ADIOI_WriteContig() */
-    GPFSMPIO_CIO_B_MPIO_RW,
-    GPFSMPIO_CIO_T_MPIO_CRW,	/* time for ADIOI_GPFS_WriteStridedColl() */
-    GPFSMPIO_CIO_B_MPIO_CRW,
-    GPFSMPIO_CIO_LAST
-};
-
-/* +1 because GPFSMPIO_CIO_LAST is actually used to say "zero this counter"" */
-extern double 	gpfsmpio_prof_cw    [GPFSMPIO_CIO_LAST+1];
-extern double 	gpfsmpio_prof_cr    [GPFSMPIO_CIO_LAST+1];
-
-/* corresponds to environment variables to select optimizations and timing level */
-extern int 	gpfsmpio_timing;
-extern int      gpfsmpio_timing_cw_level;
-extern int 	gpfsmpio_comm;
-extern int 	gpfsmpio_tunegather;
-extern int 	gpfsmpio_tuneblocking;
-extern long bglocklessmpio_f_type;
-extern int      gpfsmpio_pthreadio;
-extern int      gpfsmpio_p2pcontig;
-extern int      gpfsmpio_write_aggmethod;
-extern int      gpfsmpio_read_aggmethod;
-extern int  gpfsmpio_balancecontig;
-extern int      gpfsmpio_devnullio;
-extern int      gpfsmpio_bridgeringagg;
-extern int      gpfsmpio_onesided_no_rmw;
-extern int      gpfsmpio_onesided_always_rmw;
-extern int      gpfsmpio_onesided_inform_rmw;
-
-/* Default is, well, kind of complicated. Blue Gene /L and /P had "psets": one
- * i/o node and all compute nodes wired to it.  On Blue Gene /Q that
- * relationship is a lot more fluid.  There are still I/O nodes, and compute
- * nodes are assigned to an i/o node, but there are two routes to the i/o node,
- * via compute nodes designated as "bridge nodes".  In this code, what we used
- * to call a "pset" is actually "compute nodes associated with and including a
- * bridge node".  So, "nAgg" is roughly "number of aggregators per bridge", but
- * look closely at ADIOI_BG_persInfo_init() for the details */
-
-#define ADIOI_BG_NAGG_PSET_DFLT 16
-
-extern int     gpfsmpio_bg_nagg_pset;
-
-
-/* set internal variables for tuning environment variables */
-void ad_gpfs_get_env_vars(void);
-
-/* report timing breakdown for MPI I/O collective call */
-void ad_gpfs_timing_crw_report( int rw, ADIO_File fd, int myrank, int nprocs );
-
-/* note:
- *   T := timing;
- * CIO := collective I/O
- */
-#define GPFSMPIO_T_CIO_RESET( RW ) \
-	{ \
-	  int _i; \
-	  for ( _i = 0; _i < GPFSMPIO_CIO_LAST; _i ++ ) \
-	    gpfsmpio_prof_c##RW [ _i ] = 0; \
-	}
-
-#define GPFSMPIO_T_CIO_REPORT( RW, FD, MYRANK, NPROCS ) \
-	ad_gpfs_timing_crw_report ( RW, FD, MYRANK, NPROCS ); \
-
-#define GPFSMPIO_T_CIO_SET_GET(RW, ISSET, ISGET, VAR1, VAR2 ) \
-         {\
-	 double temp = MPI_Wtime(); \
-	 if ( ISSET ) gpfsmpio_prof_c##RW [ VAR1 ] = temp; \
-	 if ( ISGET ) gpfsmpio_prof_c##RW [ VAR2 ] = temp - gpfsmpio_prof_c##RW [ VAR2 ] ;\
-	 }
-
-#endif  /* AD_GPFS_TUNING_H_ */
diff --git a/3rd-party/romio321/adio/ad_gpfs/ad_gpfs_wrcoll.c b/3rd-party/romio321/adio/ad_gpfs/ad_gpfs_wrcoll.c
deleted file mode 100644
index 301f78dc931..00000000000
--- a/3rd-party/romio321/adio/ad_gpfs/ad_gpfs_wrcoll.c
+++ /dev/null
@@ -1,1814 +0,0 @@
-/* ---------------------------------------------------------------- */
-/* (C)Copyright IBM Corp.  2007, 2008                               */
-/* ---------------------------------------------------------------- */
-/**
- * \file ad_gpfs_wrcoll.c
- * \brief ???
- */
-
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-#include "ad_gpfs.h"
-#include "ad_gpfs_aggrs.h"
-
-#ifdef BGQPLATFORM
-#include "bg/ad_bg_pset.h"
-#endif
-
-#ifdef AGGREGATION_PROFILE
-#include "mpe.h"
-#endif
-#ifdef PROFILE
-#include "mpe.h"
-#endif
-
-#include <pthread.h>
-
-#ifdef HAVE_GPFS_H
-#include <gpfs.h>
-#endif
-#ifdef HAVE_GPFS_FCNTL_H
-#include <gpfs_fcntl.h>
-#endif
-
-#include <limits.h>
-/* prototypes of functions used for collective writes only. */
-static void ADIOI_Exch_and_write(ADIO_File fd, const void *buf, MPI_Datatype
-                         datatype, int nprocs, int myrank, ADIOI_Access
-                         *others_req, ADIO_Offset *offset_list,
-                         ADIO_Offset *len_list, int contig_access_count, ADIO_Offset
-                         min_st_offset, ADIO_Offset fd_size,
-                         ADIO_Offset *fd_start, ADIO_Offset *fd_end,
-                         int *buf_idx, int *error_code);
-static void ADIOI_W_Exchange_data(ADIO_File fd, const void *buf, char *write_buf,
-                         ADIOI_Flatlist_node *flat_buf, ADIO_Offset 
-                         *offset_list, ADIO_Offset *len_list, int *send_size, 
-                         int *recv_size, ADIO_Offset off, int size,
-                         int *count, int *start_pos, int *partial_recv, 
-                         int *sent_to_proc, int nprocs, 
-                         int myrank, int
-                         buftype_is_contig, int contig_access_count,
-                         ADIO_Offset min_st_offset, ADIO_Offset fd_size,
-                         ADIO_Offset *fd_start, ADIO_Offset *fd_end, 
-                         ADIOI_Access *others_req, 
-                         int *send_buf_idx, int *curr_to_proc,
-                         int *done_to_proc, int *hole, int iter, 
-                         MPI_Aint buftype_extent, int *buf_idx, int *error_code);
-static void ADIOI_W_Exchange_data_alltoallv(
-		ADIO_File fd, const void *buf, 
-		char *write_buf,					/* 1 */
-		ADIOI_Flatlist_node *flat_buf, 
-		ADIO_Offset *offset_list, 
-		ADIO_Offset *len_list, int *send_size, int *recv_size, 
-		ADIO_Offset off, int size,				/* 2 */
-		int *count, int *start_pos, int *partial_recv,
-		int *sent_to_proc, int nprocs, int myrank, 
-		int buftype_is_contig, int contig_access_count,
-		ADIO_Offset min_st_offset,
-		ADIO_Offset fd_size,
-		ADIO_Offset *fd_start, 
-		ADIO_Offset *fd_end,
-		ADIOI_Access *others_req,
-		int *send_buf_idx, int *curr_to_proc,			/* 3 */
-		int *done_to_proc, int *hole, 				/* 4 */
-		int iter, MPI_Aint buftype_extent, int *buf_idx,
-		int *error_code);
-static void ADIOI_Fill_send_buffer(ADIO_File fd, const void *buf, ADIOI_Flatlist_node
-                           *flat_buf, char **send_buf, ADIO_Offset 
-                           *offset_list, ADIO_Offset *len_list, int *send_size, 
-                           MPI_Request *requests, int *sent_to_proc, 
-                           int nprocs, int myrank, 
-                           int contig_access_count, ADIO_Offset
-                           min_st_offset, ADIO_Offset fd_size,
-                           ADIO_Offset *fd_start, ADIO_Offset *fd_end, 
-                           int *send_buf_idx, int *curr_to_proc, 
-                           int *done_to_proc, int iter, 
-                           MPI_Aint buftype_extent);
-static void ADIOI_Fill_send_buffer_nosend(ADIO_File fd, const void *buf, ADIOI_Flatlist_node
-                           *flat_buf, char **send_buf, ADIO_Offset 
-                           *offset_list, ADIO_Offset *len_list, int *send_size, 
-                           MPI_Request *requests, int *sent_to_proc, 
-                           int nprocs, int myrank, 
-                           int contig_access_count, ADIO_Offset
-                           min_st_offset, ADIO_Offset fd_size,
-                           ADIO_Offset *fd_start, ADIO_Offset *fd_end, 
-                           int *send_buf_idx, int *curr_to_proc, 
-                           int *done_to_proc, int iter, 
-                           MPI_Aint buftype_extent);
-static void ADIOI_Heap_merge(ADIOI_Access *others_req, int *count, 
-                      ADIO_Offset *srt_off, int *srt_len, int *start_pos,
-                      int nprocs, int nprocs_recv, int total_elements);
-
-
-void ADIOI_GPFS_WriteStridedColl(ADIO_File fd, const void *buf, int count,
-                       MPI_Datatype datatype, int file_ptr_type,
-                       ADIO_Offset offset, ADIO_Status *status, int
-                       *error_code)
-{
-/* Uses a generalized version of the extended two-phase method described
-   in "An Extended Two-Phase Method for Accessing Sections of 
-   Out-of-Core Arrays", Rajeev Thakur and Alok Choudhary,
-   Scientific Programming, (5)4:301--317, Winter 1996. 
-   http://www.mcs.anl.gov/home/thakur/ext2ph.ps */
-
-    ADIOI_Access *my_req; 
-    /* array of nprocs access structures, one for each other process in
-       whose file domain this process's request lies */
-    
-    ADIOI_Access *others_req;
-    /* array of nprocs access structures, one for each other process
-       whose request lies in this process's file domain. */
-
-    int i, filetype_is_contig, nprocs, nprocs_for_coll, myrank;
-    int contig_access_count=0, interleave_count = 0, buftype_is_contig;
-    int *count_my_req_per_proc, count_my_req_procs, count_others_req_procs;
-    ADIO_Offset orig_fp, start_offset, end_offset, fd_size, min_st_offset, off;
-    ADIO_Offset *offset_list = NULL, *st_offsets = NULL, *fd_start = NULL,
-	*fd_end = NULL, *end_offsets = NULL;
-    ADIO_Offset *gpfs_offsets0 = NULL, *gpfs_offsets = NULL;
-    ADIO_Offset *count_sizes;
-    int  ii;
-
-    int *buf_idx = NULL;
-    ADIO_Offset *len_list = NULL;
-    GPFSMPIO_T_CIO_RESET( w )
-#ifdef PROFILE
-	MPE_Log_event(13, 0, "start computation");
-#endif
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-
-/* the number of processes that actually perform I/O, nprocs_for_coll,
- * is stored in the hints off the ADIO_File structure
- */
-    nprocs_for_coll = fd->hints->cb_nodes;
-    orig_fp = fd->fp_ind;
-
-    GPFSMPIO_T_CIO_SET_GET( w, 1, 0, GPFSMPIO_CIO_T_MPIO_CRW, GPFSMPIO_CIO_LAST)
-    GPFSMPIO_T_CIO_SET_GET( w, 1, 0, GPFSMPIO_CIO_T_LCOMP, GPFSMPIO_CIO_LAST )
-
-
-    /* only check for interleaving if cb_write isn't disabled */
-    if (fd->hints->cb_write != ADIOI_HINT_DISABLE) {
-	/* For this process's request, calculate the list of offsets and
-	   lengths in the file and determine the start and end offsets. */
-
-	/* Note: end_offset points to the last byte-offset that will be accessed.
-	   e.g., if start_offset=0 and 100 bytes to be read, end_offset=99*/
-
-	ADIOI_Calc_my_off_len(fd, count, datatype, file_ptr_type, offset,
-			      &offset_list, &len_list, &start_offset,
-			      &end_offset, &contig_access_count); 
-
-    GPFSMPIO_T_CIO_SET_GET( w, 1, 1, GPFSMPIO_CIO_T_GATHER, GPFSMPIO_CIO_T_LCOMP )
-
-	/* each process communicates its start and end offsets to other 
-	   processes. The result is an array each of start and end offsets stored
-	   in order of process rank. */ 
-
-	st_offsets = (ADIO_Offset *) ADIOI_Malloc(nprocs*sizeof(ADIO_Offset));
-	end_offsets = (ADIO_Offset *) ADIOI_Malloc(nprocs*sizeof(ADIO_Offset));
-
-    ADIO_Offset my_count_size=0;
-    /* One-sided aggregation needs the amount of data per rank as well because
-     * the difference in starting and ending offsets for 1 byte is 0 the same
-     * as 0 bytes so it cannot be distiguished.
-     */
-    if ((gpfsmpio_write_aggmethod == 1) || (gpfsmpio_write_aggmethod == 2)) {
-        count_sizes = (ADIO_Offset *) ADIOI_Malloc(nprocs*sizeof(ADIO_Offset));
-        MPI_Count buftype_size;
-        MPI_Type_size_x(datatype, &buftype_size);
-        my_count_size = (ADIO_Offset) count  * (ADIO_Offset)buftype_size;
-    }
-    if (gpfsmpio_tunegather) {
-      if ((gpfsmpio_write_aggmethod == 1) || (gpfsmpio_write_aggmethod == 2)) {
-        gpfs_offsets0 = (ADIO_Offset *) ADIOI_Malloc(3*nprocs*sizeof(ADIO_Offset));
-        gpfs_offsets  = (ADIO_Offset *) ADIOI_Malloc(3*nprocs*sizeof(ADIO_Offset));
-        for (ii=0; ii<nprocs; ii++)  {
-          gpfs_offsets0[ii*3]   = 0;
-          gpfs_offsets0[ii*3+1] = 0;
-          gpfs_offsets0[ii*3+2] = 0;
-        }
-        gpfs_offsets0[myrank*3]   = start_offset;
-        gpfs_offsets0[myrank*3+1] =   end_offset;
-        gpfs_offsets0[myrank*3+2] =   my_count_size;
-        MPI_Allreduce( gpfs_offsets0, gpfs_offsets, nprocs*3, ADIO_OFFSET, MPI_MAX, fd->comm );
-        for (ii=0; ii<nprocs; ii++)  {
-          st_offsets [ii] = gpfs_offsets[ii*3]  ;
-          end_offsets[ii] = gpfs_offsets[ii*3+1];
-          count_sizes[ii] = gpfs_offsets[ii*3+2];
-        }
-      }
-      else {
-            gpfs_offsets0 = (ADIO_Offset *) ADIOI_Malloc(2*nprocs*sizeof(ADIO_Offset));
-            gpfs_offsets  = (ADIO_Offset *) ADIOI_Malloc(2*nprocs*sizeof(ADIO_Offset));
-            for (ii=0; ii<nprocs; ii++)  {
-                gpfs_offsets0[ii*2]   = 0;
-                gpfs_offsets0[ii*2+1] = 0;
-            }
-            gpfs_offsets0[myrank*2]   = start_offset;
-            gpfs_offsets0[myrank*2+1] =   end_offset;
-
-        MPI_Allreduce( gpfs_offsets0, gpfs_offsets, nprocs*2, ADIO_OFFSET, MPI_MAX, fd->comm );
-
-            for (ii=0; ii<nprocs; ii++)  {
-                st_offsets [ii] = gpfs_offsets[ii*2]  ;
-                end_offsets[ii] = gpfs_offsets[ii*2+1];
-            }
-      }
-            ADIOI_Free( gpfs_offsets0 );
-            ADIOI_Free( gpfs_offsets  );
-    } else {
-	MPI_Allgather(&start_offset, 1, ADIO_OFFSET, st_offsets, 1,
-		      ADIO_OFFSET, fd->comm);
-	MPI_Allgather(&end_offset, 1, ADIO_OFFSET, end_offsets, 1,
-		      ADIO_OFFSET, fd->comm);
-        if ((gpfsmpio_write_aggmethod == 1) || (gpfsmpio_write_aggmethod == 2)) {
-	    MPI_Allgather(&count_sizes, 1, ADIO_OFFSET, count_sizes, 1,
-                     ADIO_OFFSET, fd->comm);
-        }
-    }
-
-    GPFSMPIO_T_CIO_SET_GET(w, 1, 1, GPFSMPIO_CIO_T_PATANA, GPFSMPIO_CIO_T_GATHER )
-
-	/* are the accesses of different processes interleaved? */
-	for (i=1; i<nprocs; i++)
-      if ((st_offsets[i] < end_offsets[i-1]) &&
-                (st_offsets[i] <= end_offsets[i]))
-                interleave_count++;
-	/* This is a rudimentary check for interleaving, but should suffice
-	   for the moment. */
-    }
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-
-    if (fd->hints->cb_write == ADIOI_HINT_DISABLE ||
-	(!interleave_count && (fd->hints->cb_write == ADIOI_HINT_AUTO)))
-    {
-	/* use independent accesses */
-	if (fd->hints->cb_write != ADIOI_HINT_DISABLE) {
-	    ADIOI_Free(offset_list);
-	    ADIOI_Free(len_list);
-	    ADIOI_Free(st_offsets);
-	    ADIOI_Free(end_offsets);
-	}
-
-	fd->fp_ind = orig_fp;
-        ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-
-        if (buftype_is_contig && filetype_is_contig) {
-
-            if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-                off = fd->disp + (ADIO_Offset)(fd->etype_size) * offset;
-                ADIO_WriteContig(fd, buf, count, datatype,
-				 ADIO_EXPLICIT_OFFSET,
-				 off, status, error_code);
-            }
-            else ADIO_WriteContig(fd, buf, count, datatype, ADIO_INDIVIDUAL,
-				  0, status, error_code);
-        }
-	else ADIO_WriteStrided(fd, buf, count, datatype, file_ptr_type,
-			       offset, status, error_code);
-
-	return;
-    }
-
-    GPFSMPIO_T_CIO_SET_GET( w, 1, 1, GPFSMPIO_CIO_T_FD_PART, GPFSMPIO_CIO_T_PATANA )
-
-/* Divide the I/O workload among "nprocs_for_coll" processes. This is
-   done by (logically) dividing the file into file domains (FDs); each
-   process may directly access only its own file domain. */
-
-    int currentValidDataIndex = 0;
-    if ((gpfsmpio_write_aggmethod == 1) || (gpfsmpio_write_aggmethod == 2)) {
-      /* Take out the 0-data offsets by shifting the indexes with data to the front
-       * and keeping track of the valid data index for use as the length.
-       */
-      for (i=0; i<nprocs; i++) {
-        if (count_sizes[i] > 0) {
-          st_offsets[currentValidDataIndex] = st_offsets[i];
-          end_offsets[currentValidDataIndex] = end_offsets[i];
-          currentValidDataIndex++;
-        }
-      }
-    }
-
-    if (gpfsmpio_tuneblocking) {
-	if ((gpfsmpio_write_aggmethod == 1) || (gpfsmpio_write_aggmethod == 2)) {
-	    ADIOI_GPFS_Calc_file_domains(fd, st_offsets, end_offsets,
-		    currentValidDataIndex,
-		    nprocs_for_coll, &min_st_offset,
-		    &fd_start, &fd_end, &fd_size, fd->fs_ptr);
-	}
-	else {
-
-    ADIOI_GPFS_Calc_file_domains(fd, st_offsets, end_offsets, nprocs,
-			    nprocs_for_coll, &min_st_offset,
-			    &fd_start, &fd_end, &fd_size, fd->fs_ptr);   
-	}
-    }
-    else {
-	if ((gpfsmpio_write_aggmethod == 1) || (gpfsmpio_write_aggmethod == 2)) {
-	    ADIOI_Calc_file_domains(st_offsets, end_offsets, currentValidDataIndex,
-		    nprocs_for_coll, &min_st_offset,
-		    &fd_start, &fd_end,
-		    fd->hints->min_fdomain_size, &fd_size,
-		    fd->hints->striping_unit);
-	}
-	else {
-    ADIOI_Calc_file_domains(st_offsets, end_offsets, nprocs,
-			    nprocs_for_coll, &min_st_offset,
-			    &fd_start, &fd_end,
-			    fd->hints->min_fdomain_size, &fd_size,
-			    fd->hints->striping_unit);   
-	}
-    }
-
-    GPFSMPIO_T_CIO_SET_GET( w, 1, 1, GPFSMPIO_CIO_T_MYREQ, GPFSMPIO_CIO_T_FD_PART );
-
-    if ((gpfsmpio_write_aggmethod == 1) || (gpfsmpio_write_aggmethod == 2)) {
-    /* If the user has specified to use a one-sided aggregation method then do that at
-     * this point instead of the two-phase I/O.
-     */
-      int holeFound = 0;
-      ADIOI_OneSidedWriteAggregation(fd, offset_list, len_list, contig_access_count,
-	      buf, datatype, error_code, st_offsets, end_offsets,
-	      currentValidDataIndex, fd_start, fd_end, &holeFound);
-      int anyHolesFound = 0;
-      if (!gpfsmpio_onesided_no_rmw)
-        MPI_Allreduce(&holeFound, &anyHolesFound, 1, MPI_INT, MPI_MAX, fd->comm);
-      if (anyHolesFound == 0) {
-      GPFSMPIO_T_CIO_REPORT( 1, fd, myrank, nprocs)
-      ADIOI_Free(offset_list);
-      ADIOI_Free(len_list);
-      ADIOI_Free(st_offsets);
-      ADIOI_Free(end_offsets);
-      ADIOI_Free(fd_start);
-      ADIOI_Free(fd_end);
-      ADIOI_Free(count_sizes);
-	  goto fn_exit;
-	  }
-      else {
-	/* Holes are found in the data and the user has not set
-	 * gpfsmpio_onesided_no_rmw --- set gpfsmpio_onesided_always_rmw to 1
-	 * and re-call ADIOI_OneSidedWriteAggregation and if the user has
-	 * gpfsmpio_onesided_inform_rmw set then inform him of this condition
-	 * and behavior.
-         */
-
-        if (gpfsmpio_onesided_inform_rmw && (myrank ==0))
-          FPRINTF(stderr,"Information: Holes found during one-sided "
-		  "write aggregation algorithm --- re-running one-sided "
-		  "write aggregation with GPFSMPIO_ONESIDED_ALWAYS_RMW set to 1.\n");
-          gpfsmpio_onesided_always_rmw = 1;
-          int prev_gpfsmpio_onesided_no_rmw = gpfsmpio_onesided_no_rmw;
-          gpfsmpio_onesided_no_rmw = 1;
-          ADIOI_OneSidedWriteAggregation(fd, offset_list, len_list, contig_access_count, buf, datatype, error_code, st_offsets, end_offsets, currentValidDataIndex, fd_start, fd_end, &holeFound);
-          gpfsmpio_onesided_no_rmw = prev_gpfsmpio_onesided_no_rmw;
-          GPFSMPIO_T_CIO_REPORT( 1, fd, myrank, nprocs)
-          ADIOI_Free(offset_list);
-          ADIOI_Free(len_list);
-          ADIOI_Free(st_offsets);
-          ADIOI_Free(end_offsets);
-          ADIOI_Free(fd_start);
-          ADIOI_Free(fd_end);
-          ADIOI_Free(count_sizes);
-          goto fn_exit;
-      }
-    }
-    if (gpfsmpio_p2pcontig==1) {
-	/* For some simple yet common(?) workloads, full-on two-phase I/O is overkill.  We can establish sub-groups of processes and their aggregator, and then these sub-groups will carry out a simplified two-phase over that sub-group.
-	 *
-	 * First verify that the filetype is contig and the offsets are
-	 * increasing in rank order*/
-	int inOrderAndNoGaps = 1;
-	for (i=0;i<(nprocs-1);i++) {
-	    if (end_offsets[i] != (st_offsets[i+1]-1))
-		inOrderAndNoGaps = 0;
-	}
-	if (inOrderAndNoGaps && buftype_is_contig) {
-	    /* if these conditions exist then execute the P2PContig code else
-	     * execute the original code */
-	    ADIOI_P2PContigWriteAggregation(fd, buf,
-		    error_code, st_offsets, end_offsets, fd_start, fd_end);
-	    /* NOTE: we are skipping the rest of two-phase in this path */
-            GPFSMPIO_T_CIO_REPORT( 1, fd, myrank, nprocs)
-
-            ADIOI_Free(offset_list);
-            ADIOI_Free(len_list);
-            ADIOI_Free(st_offsets);
-            ADIOI_Free(end_offsets);
-            ADIOI_Free(fd_start);
-            ADIOI_Free(fd_end);
-
-	    goto fn_exit;
-	}
-    }
-
-/* calculate what portions of the access requests of this process are
-   located in what file domains */
-
-    if (gpfsmpio_tuneblocking)
-    ADIOI_GPFS_Calc_my_req(fd, offset_list, len_list, contig_access_count,
-		      min_st_offset, fd_start, fd_end, fd_size,
-		      nprocs, &count_my_req_procs, 
-		      &count_my_req_per_proc, &my_req,
-		      &buf_idx); 
-    else
-    ADIOI_Calc_my_req(fd, offset_list, len_list, contig_access_count,
-		      min_st_offset, fd_start, fd_end, fd_size,
-		      nprocs, &count_my_req_procs, 
-		      &count_my_req_per_proc, &my_req,
-		      &buf_idx);
-
-    GPFSMPIO_T_CIO_SET_GET( w, 1, 1, GPFSMPIO_CIO_T_OTHREQ, GPFSMPIO_CIO_T_MYREQ )
-
-/* based on everyone's my_req, calculate what requests of other
-   processes lie in this process's file domain.
-   count_others_req_procs = number of processes whose requests lie in
-   this process's file domain (including this process itself) 
-   count_others_req_per_proc[i] indicates how many separate contiguous
-   requests of proc. i lie in this process's file domain. */
-
-    if (gpfsmpio_tuneblocking)
-	ADIOI_GPFS_Calc_others_req(fd, count_my_req_procs,
-			      count_my_req_per_proc, my_req,
-			      nprocs, myrank,
-			      &count_others_req_procs, &others_req);
-    else
-    ADIOI_Calc_others_req(fd, count_my_req_procs, 
-			  count_my_req_per_proc, my_req, 
-			  nprocs, myrank,
-			  &count_others_req_procs, &others_req);
-
-    GPFSMPIO_T_CIO_SET_GET( w, 1, 1, GPFSMPIO_CIO_T_DEXCH, GPFSMPIO_CIO_T_OTHREQ )
-
-    ADIOI_Free(count_my_req_per_proc);
-    for (i=0; i < nprocs; i++) {
-	if (my_req[i].count) {
-	    ADIOI_Free(my_req[i].offsets);
-	    ADIOI_Free(my_req[i].lens);
-	}
-    }
-    ADIOI_Free(my_req);
-
-/* exchange data and write in sizes of no more than coll_bufsize. */
-    ADIOI_Exch_and_write(fd, buf, datatype, nprocs, myrank,
-                        others_req, offset_list,
-			len_list, contig_access_count, min_st_offset,
-			fd_size, fd_start, fd_end, buf_idx, error_code);
-
-    GPFSMPIO_T_CIO_SET_GET( w, 0, 1, GPFSMPIO_CIO_LAST, GPFSMPIO_CIO_T_DEXCH )
-    GPFSMPIO_T_CIO_SET_GET( w, 0, 1, GPFSMPIO_CIO_LAST, GPFSMPIO_CIO_T_MPIO_CRW )
-
-    GPFSMPIO_T_CIO_REPORT( 1, fd, myrank, nprocs)
-
-/* free all memory allocated for collective I/O */
-    if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
-
-    for (i=0; i<nprocs; i++) {
-	if (others_req[i].count) {
-	    ADIOI_Free(others_req[i].offsets);
-	    ADIOI_Free(others_req[i].lens);
-	    ADIOI_Free(others_req[i].mem_ptrs);
-	}
-    }
-    ADIOI_Free(others_req);
-
-    ADIOI_Free(buf_idx);
-    ADIOI_Free(offset_list);
-    ADIOI_Free(len_list);
-    ADIOI_Free(st_offsets);
-    ADIOI_Free(end_offsets);
-    ADIOI_Free(fd_start);
-    ADIOI_Free(fd_end);
-
-fn_exit:
-#ifdef HAVE_STATUS_SET_BYTES
-    if (status) {
-      MPI_Count bufsize, size;
-      /* Don't set status if it isn't needed */
-      MPI_Type_size_x(datatype, &size);
-      bufsize = size * count;
-      MPIR_Status_set_bytes(status, datatype, bufsize);
-    }
-/* This is a temporary way of filling in status. The right way is to 
-   keep track of how much data was actually written during collective I/O. */
-#endif
-
-    fd->fp_sys_posn = -1;   /* set it to null. */
-#ifdef AGGREGATION_PROFILE
-	MPE_Log_event (5013, 0, NULL);
-#endif
-}
-
-static void gpfs_wr_access_start(int fd, ADIO_Offset offset, ADIO_Offset length)
-{
-        int rc=0;
-#ifdef HAVE_GPFS_FCNTL_H
-        struct {
-                gpfsFcntlHeader_t header;
-                gpfsAccessRange_t access;
-        } take_locks;
-
-        take_locks.header.totalLength = sizeof(take_locks);
-        take_locks.header.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION;
-        take_locks.header.fcntlReserved = 0;
-
-        take_locks.access.structLen = sizeof(take_locks.access);
-        take_locks.access.structType = GPFS_ACCESS_RANGE;
-        take_locks.access.start = offset;
-        take_locks.access.length = length;
-        take_locks.access.isWrite = 1;
-
-        rc = gpfs_fcntl(fd, &take_locks);
-#endif
-        ADIOI_Assert(rc == 0);
-}
-
-static void gpfs_wr_access_end(int fd, ADIO_Offset offset, ADIO_Offset length)
-{
-        int rc=0;
-#ifdef HAVE_GPFS_FCNTL_H
-        struct {
-                gpfsFcntlHeader_t header;
-                gpfsFreeRange_t free;
-        } free_locks;
-
-
-        free_locks.header.totalLength = sizeof(free_locks);
-        free_locks.header.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION;
-        free_locks.header.fcntlReserved = 0;
-
-        free_locks.free.structLen = sizeof(free_locks.free);
-        free_locks.free.structType = GPFS_FREE_RANGE;
-        free_locks.free.start = offset;
-        free_locks.free.length = length;
-
-        rc = gpfs_fcntl(fd, &free_locks);
-#endif
-        ADIOI_Assert(rc == 0);
-}
-
-#ifdef BGQPLATFORM
-/* my_start, my_end: this processes file domain.  coudd be -1,-1 for "no i/o"
- * fd_start, fd_end: arrays of length fd->hints->cb_nodes specifying all file domains */
-static int gpfs_find_access_for_ion(ADIO_File fd,
-	ADIO_Offset my_start, ADIO_Offset my_end,
-	ADIO_Offset *fd_start, ADIO_Offset *fd_end,
-	ADIO_Offset *start, ADIO_Offset *end)
-{
-    int my_ionode = BGQ_IO_node_id();
-    int *rank_to_ionode;
-    int i, nprocs, rank;
-    ADIO_Offset group_start=LLONG_MAX, group_end=0;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &rank);
-
-    rank_to_ionode = ADIOI_Calloc(nprocs, sizeof(int));
-    MPI_Allgather(&my_ionode, 1, MPI_INT,  rank_to_ionode, 1, MPI_INT, fd->comm);
-
-    /* rank_to_ionode now contains a mapping from MPI rank to IO node */
-    /* fd->hints->ranklist[] contains a list of MPI ranks that are aggregators */
-    /* fd_start[] and fd_end[] contain a list of file domains. */
-
-    /* what we really want to do is take all the file domains associated
-     * with a given i/o node and find the begin/end of that range.
-     *
-     * Because gpfs_fcntl hints are expected to be released, we'll pass this
-     * start/end back to the caller, who will both declare and free this range
-     */
-    if (my_start == -1 || my_end == -1) {
-	ADIOI_Free(rank_to_ionode);
-	return 0; /* no work to do */
-    }
-
-    for (i=0; i<fd->hints->cb_nodes; i++ ){
-	if (my_ionode == rank_to_ionode[fd->hints->ranklist[i]] ) {
-	    group_start = ADIOI_MIN(fd_start[i], group_start);
-	    group_end = ADIOI_MAX(fd_end[i], group_end);
-	}
-    }
-    *start = group_start;
-    *end = group_end;
-    ADIOI_Free(rank_to_ionode);
-    return 1;
-}
-#endif // BGQPLATFORM
-
-
-/* If successful, error_code is set to MPI_SUCCESS.  Otherwise an error
- * code is created and returned in error_code.
- */
-static void ADIOI_Exch_and_write(ADIO_File fd, const void *buf, MPI_Datatype
-				 datatype, int nprocs, 
-				 int myrank,
-				 ADIOI_Access
-				 *others_req, ADIO_Offset *offset_list,
-				 ADIO_Offset *len_list, int contig_access_count,
-				 ADIO_Offset min_st_offset, ADIO_Offset fd_size,
-				 ADIO_Offset *fd_start, ADIO_Offset *fd_end,
-				 int *buf_idx, int *error_code)
-{
-/* Send data to appropriate processes and write in sizes of no more
-   than coll_bufsize.    
-   The idea is to reduce the amount of extra memory required for
-   collective I/O. If all data were written all at once, which is much
-   easier, it would require temp space more than the size of user_buf,
-   which is often unacceptable. For example, to write a distributed
-   array to a file, where each local array is 8Mbytes, requiring
-   at least another 8Mbytes of temp space is unacceptable. */
-
-    /* Not convinced end_loc-st_loc couldn't be > int, so make these offsets*/
-    ADIO_Offset size=0;
-    int hole, i, j, m, ntimes, max_ntimes, buftype_is_contig;
-    ADIO_Offset st_loc=-1, end_loc=-1, off, done, req_off;
-    char *write_buf=NULL, *write_buf2=NULL;
-    int *curr_offlen_ptr, *count, *send_size, req_len, *recv_size;
-    int *partial_recv, *sent_to_proc, *start_pos, flag;
-    int *send_buf_idx, *curr_to_proc, *done_to_proc;
-    MPI_Status status;
-    ADIOI_Flatlist_node *flat_buf=NULL;
-    MPI_Aint buftype_extent, buftype_lb;
-    int info_flag, coll_bufsize;
-    char *value;
-    static char myname[] = "ADIOI_EXCH_AND_WRITE";
-    pthread_t io_thread;
-    void *thread_ret;
-    ADIOI_IO_ThreadFuncData io_thread_args;
-
-    *error_code = MPI_SUCCESS;  /* changed below if error */
-    /* only I/O errors are currently reported */
-
-/* calculate the number of writes of size coll_bufsize
-   to be done by each process and the max among all processes.
-   That gives the no. of communication phases as well. */
-
-    value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
-    ADIOI_Info_get(fd->info, "cb_buffer_size", MPI_MAX_INFO_VAL, value, 
-                 &info_flag);
-    coll_bufsize = atoi(value);
-    ADIOI_Free(value);
-
-    if (gpfsmpio_pthreadio == 1){
-	/* ROMIO will spawn an additional thread. both threads use separate
-	 * halves of the collective buffer*/
-	coll_bufsize = coll_bufsize/2;
-    }
-
-    for (i=0; i < nprocs; i++) {
-	if (others_req[i].count) {
-	    st_loc = others_req[i].offsets[0];
-	    end_loc = others_req[i].offsets[0];
-	    break;
-	}
-    }
-
-    for (i=0; i < nprocs; i++)
-	for (j=0; j < others_req[i].count; j++) {
-	    st_loc = ADIOI_MIN(st_loc, others_req[i].offsets[j]);
-	    end_loc = ADIOI_MAX(end_loc, (others_req[i].offsets[j]
-				       + others_req[i].lens[j] - 1));
-	}
-
-/* ntimes=ceiling_div(end_loc - st_loc + 1, coll_bufsize)*/
-
-    ntimes = (int) ((end_loc - st_loc + coll_bufsize)/coll_bufsize);
-
-    if ((st_loc==-1) && (end_loc==-1)) {
-	ntimes = 0; /* this process does no writing. */
-    }
-    if (ntimes > 0) { /* only set the gpfs hint if we have io - ie this rank is
-			 an aggregator -- otherwise will fail for deferred open */
-      if (getenv("ROMIO_GPFS_DECLARE_ACCESS")!=NULL) {
-	gpfs_wr_access_start(fd->fd_sys, st_loc, end_loc - st_loc);
-      }
-    }
-
-    ADIO_Offset st_loc_ion=0, end_loc_ion=0, needs_gpfs_access_cleanup=0;
-#ifdef BGQPLATFORM
-    if (ntimes > 0) { /* only set the gpfs hint if we have io - ie this rank is
-			 an aggregator -- otherwise will fail for deferred open */
-
-      if (getenv("ROMIO_GPFS_DECLARE_ION_ACCESS")!=NULL) {
-	if (gpfs_find_access_for_ion(fd, st_loc, end_loc, fd_start, fd_end,
-		    &st_loc_ion, &end_loc_ion)) {
-	    gpfs_wr_access_start(fd->fd_sys, st_loc_ion, end_loc_ion-st_loc_ion);
-	    needs_gpfs_access_cleanup=1;
-	}
-      }
-    }
-#endif
-
-    MPI_Allreduce(&ntimes, &max_ntimes, 1, MPI_INT, MPI_MAX,
-		  fd->comm); 
-
-    write_buf = fd->io_buf;
-    if (gpfsmpio_pthreadio == 1) {
-	write_buf2 = fd->io_buf + coll_bufsize;
-    }
-
-    curr_offlen_ptr = (int *) ADIOI_Calloc(nprocs, sizeof(int)); 
-    /* its use is explained below. calloc initializes to 0. */
-
-    count = (int *) ADIOI_Malloc(nprocs*sizeof(int));
-    /* to store count of how many off-len pairs per proc are satisfied
-       in an iteration. */
-
-    partial_recv = (int *) ADIOI_Calloc(nprocs, sizeof(int));
-    /* if only a portion of the last off-len pair is recd. from a process
-       in a particular iteration, the length recd. is stored here.
-       calloc initializes to 0. */
-
-    send_size = (int *) ADIOI_Malloc(nprocs*sizeof(int));
-    /* total size of data to be sent to each proc. in an iteration.
-       Of size nprocs so that I can use MPI_Alltoall later. */
-
-    recv_size = (int *) ADIOI_Malloc(nprocs*sizeof(int));
-    /* total size of data to be recd. from each proc. in an iteration.*/
-
-    sent_to_proc = (int *) ADIOI_Calloc(nprocs, sizeof(int));
-    /* amount of data sent to each proc so far. Used in
-       ADIOI_Fill_send_buffer. initialized to 0 here. */
-
-    send_buf_idx = (int *) ADIOI_Malloc(nprocs*sizeof(int));
-    curr_to_proc = (int *) ADIOI_Malloc(nprocs*sizeof(int));
-    done_to_proc = (int *) ADIOI_Malloc(nprocs*sizeof(int));
-    /* Above three are used in ADIOI_Fill_send_buffer*/
-
-    start_pos = (int *) ADIOI_Malloc(nprocs*sizeof(int));
-    /* used to store the starting value of curr_offlen_ptr[i] in 
-       this iteration */
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-    if (!buftype_is_contig) {
-	flat_buf = ADIOI_Flatten_and_find(datatype);
-    }
-    MPI_Type_get_extent(datatype, &buftype_lb, &buftype_extent);
-
-
-/* I need to check if there are any outstanding nonblocking writes to
-   the file, which could potentially interfere with the writes taking
-   place in this collective write call. Since this is not likely to be
-   common, let me do the simplest thing possible here: Each process
-   completes all pending nonblocking operations before completing. */
-
-    /*ADIOI_Complete_async(error_code);
-    if (*error_code != MPI_SUCCESS) return;
-    MPI_Barrier(fd->comm);
-    */
-
-    done = 0;
-    off = st_loc;
-
-    if(gpfsmpio_pthreadio == 1)
-	io_thread = pthread_self();
-
-#ifdef PROFILE
-	MPE_Log_event(14, 0, "end computation");
-#endif
-
-    for (m=0; m < ntimes; m++) {
-       /* go through all others_req and check which will be satisfied
-          by the current write */
-
-       /* Note that MPI guarantees that displacements in filetypes are in 
-          monotonically nondecreasing order and that, for writes, the
-	  filetypes cannot specify overlapping regions in the file. This
-	  simplifies implementation a bit compared to reads. */
-
-          /* off = start offset in the file for the data to be written in 
-                   this iteration 
-             size = size of data written (bytes) corresponding to off
-             req_off = off in file for a particular contiguous request 
-                       minus what was satisfied in previous iteration
-             req_size = size corresponding to req_off */
-
-	/* first calculate what should be communicated */
-
-#ifdef PROFILE
-	MPE_Log_event(13, 0, "start computation");
-#endif
-	for (i=0; i < nprocs; i++) count[i] = recv_size[i] = 0;
-
-	size = ADIOI_MIN((unsigned)coll_bufsize, end_loc-st_loc+1-done); 
-
-	for (i=0; i < nprocs; i++) {
-	    if (others_req[i].count) {
-		start_pos[i] = curr_offlen_ptr[i];
-		for (j=curr_offlen_ptr[i]; j<others_req[i].count; j++) {
-		    if (partial_recv[i]) {
-			/* this request may have been partially
-			   satisfied in the previous iteration. */
-			req_off = others_req[i].offsets[j] +
-			    partial_recv[i]; 
-                        req_len = others_req[i].lens[j] -
-			    partial_recv[i];
-			partial_recv[i] = 0;
-			/* modify the off-len pair to reflect this change */
-			others_req[i].offsets[j] = req_off;
-			others_req[i].lens[j] = req_len;
-		    }
-		    else {
-			req_off = others_req[i].offsets[j];
-                        req_len = others_req[i].lens[j];
-		    }
-		    if (req_off < off + size) {
-			count[i]++;
-      ADIOI_Assert((((ADIO_Offset)(MPIU_Upint)write_buf)+req_off-off) == (ADIO_Offset)(MPIU_Upint)(write_buf+req_off-off));
-			MPI_Get_address(write_buf+req_off-off, 
-                               &(others_req[i].mem_ptrs[j]));
-      ADIOI_Assert((off + size - req_off) == (int)(off + size - req_off));
-			recv_size[i] += (int)(ADIOI_MIN(off + size - req_off, 
-                                      (unsigned)req_len));
-
-			if (off+size-req_off < (unsigned)req_len)
-			{
-			    partial_recv[i] = (int) (off + size - req_off);
-
-			    /* --BEGIN ERROR HANDLING-- */
-			    if ((j+1 < others_req[i].count) && 
-                                 (others_req[i].offsets[j+1] < off+size))
-			    { 
-				*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-								   MPIR_ERR_RECOVERABLE,
-								   myname,
-								   __LINE__,
-								   MPI_ERR_ARG,
-								   "Filetype specifies overlapping write regions (which is illegal according to the MPI-2 specification)", 0);
-				/* allow to continue since additional
-				 * communication might have to occur
-				 */
-			    }
-			    /* --END ERROR HANDLING-- */
-			    break;
-			}
-		    }
-		    else break;
-		}
-		curr_offlen_ptr[i] = j;
-	    }
-	}
-	
-#ifdef PROFILE
-	MPE_Log_event(14, 0, "end computation");
-	MPE_Log_event(7, 0, "start communication");
-#endif
-        if (gpfsmpio_comm == 1)
-	ADIOI_W_Exchange_data(fd, buf, write_buf, flat_buf, offset_list,
-                            len_list, send_size, recv_size, off, size, count,
-                            start_pos, partial_recv,
-                            sent_to_proc, nprocs, myrank,
-			    buftype_is_contig, contig_access_count,
-			    min_st_offset, fd_size, fd_start, fd_end,
-			    others_req, send_buf_idx, curr_to_proc,
-                            done_to_proc, &hole, m, buftype_extent, buf_idx,
-			    error_code); 
-	else
-        if (gpfsmpio_comm == 0)
-	ADIOI_W_Exchange_data_alltoallv(fd, buf, write_buf, flat_buf, offset_list,
-                            len_list, send_size, recv_size, off, size, count,
-                            start_pos, partial_recv,
-                            sent_to_proc, nprocs, myrank,
-			    buftype_is_contig, contig_access_count,
-			    min_st_offset, fd_size, fd_start, fd_end,
-			    others_req, send_buf_idx, curr_to_proc,
-                            done_to_proc, &hole, m, buftype_extent, buf_idx,
-			    error_code); 
-        if (*error_code != MPI_SUCCESS) return;
-#ifdef PROFILE
-	MPE_Log_event(8, 0, "end communication");
-#endif
-
-	flag = 0;
-	for (i=0; i<nprocs; i++)
-	    if (count[i]) flag = 1;
-
-	if (flag) {
-	    char round[50];
-	    sprintf(round, "two-phase-round=%d", m);
-	    setenv("LIBIOLOG_EXTRA_INFO", round, 1);
-      ADIOI_Assert(size == (int)size);
-	    if (gpfsmpio_pthreadio == 1) {
-		/* there is no such thing as "invalid pthread identifier", so
-		 * we'll use pthread_self() instead.  Before we do I/O we want
-		 * to complete I/O from any previous iteration -- but only a
-		 * previous iteration that had I/O work to do (i.e. set 'flag')
-		 */
-		if(!pthread_equal(io_thread, pthread_self())) {
-		    pthread_join(io_thread, &thread_ret);
-		    *error_code = *(int *)thread_ret;
-		    if (*error_code != MPI_SUCCESS) return;
-		    io_thread = pthread_self();
-
-		}
-		io_thread_args.fd = fd;
-		/* do a little pointer shuffling: background I/O works from one
-		 * buffer while two-phase machinery fills up another */
-		io_thread_args.buf = write_buf;
-		ADIOI_SWAP(write_buf, write_buf2, char*);
-		io_thread_args.io_kind = ADIOI_WRITE;
-		io_thread_args.size = size;
-		io_thread_args.offset = off;
-		io_thread_args.status = &status;
-		io_thread_args.error_code = *error_code;
-		if ( (pthread_create(&io_thread, NULL,
-			ADIOI_IO_Thread_Func, &(io_thread_args))) != 0)
-		    io_thread = pthread_self();
-	    } else {
-		ADIO_WriteContig(fd, write_buf, (int)size, MPI_BYTE,
-			ADIO_EXPLICIT_OFFSET, off, &status, error_code);
-		if (*error_code != MPI_SUCCESS) return;
-	    }
-	}
-
-	off += size;
-	done += size;
-    }
-    if (gpfsmpio_pthreadio == 1) {
-	if ( !pthread_equal(io_thread, pthread_self()) ) {
-	    pthread_join(io_thread, &thread_ret);
-	    *error_code = *(int *)thread_ret;
-	}
-    }
-
-    for (i=0; i<nprocs; i++) count[i] = recv_size[i] = 0;
-#ifdef PROFILE
-	MPE_Log_event(7, 0, "start communication");
-#endif
-    for (m=ntimes; m<max_ntimes; m++) 
-	/* nothing to recv, but check for send. */
-        if (gpfsmpio_comm == 1)
-	ADIOI_W_Exchange_data(fd, buf, write_buf, flat_buf, offset_list,
-                            len_list, send_size, recv_size, off, size, count,
-                            start_pos, partial_recv,
-                            sent_to_proc, nprocs, myrank,
-			    buftype_is_contig, contig_access_count,
-			    min_st_offset, fd_size, fd_start, fd_end,
-			    others_req, send_buf_idx, 
-                            curr_to_proc, done_to_proc, &hole, m, 
-                            buftype_extent, buf_idx, error_code); 
-	else
-        if (gpfsmpio_comm == 0)
-	ADIOI_W_Exchange_data_alltoallv(fd, buf, write_buf, flat_buf, offset_list,
-                            len_list, send_size, recv_size, off, size, count,
-                            start_pos, partial_recv,
-                            sent_to_proc, nprocs, myrank,
-			    buftype_is_contig, contig_access_count,
-			    min_st_offset, fd_size, fd_start, fd_end,
-			    others_req, send_buf_idx, 
-                            curr_to_proc, done_to_proc, &hole, m, 
-                            buftype_extent, buf_idx, error_code); 
-        if (*error_code != MPI_SUCCESS) return;
-#ifdef PROFILE
-	MPE_Log_event(8, 0, "end communication");
-#endif
-
-    ADIOI_Free(curr_offlen_ptr);
-    ADIOI_Free(count);
-    ADIOI_Free(partial_recv);
-    ADIOI_Free(send_size);
-    ADIOI_Free(recv_size);
-    ADIOI_Free(sent_to_proc);
-    ADIOI_Free(start_pos);
-    ADIOI_Free(send_buf_idx);
-    ADIOI_Free(curr_to_proc);
-    ADIOI_Free(done_to_proc);
-
-    if (ntimes != 0 && getenv("ROMIO_GPFS_DECLARE_ACCESS")!=NULL) {
-	gpfs_wr_access_end(fd->fd_sys, st_loc, end_loc-st_loc);
-    }
-
-    if (needs_gpfs_access_cleanup) {
-	gpfs_wr_access_end(fd->fd_sys, st_loc_ion, end_loc_ion-st_loc_ion);
-	needs_gpfs_access_cleanup=0;
-    }
-
-    unsetenv("LIBIOLOG_EXTRA_INFO");
-}
-
-
-/* Sets error_code to MPI_SUCCESS if successful, or creates an error code
- * in the case of error.
- */
-static void ADIOI_W_Exchange_data(ADIO_File fd, const void *buf, char *write_buf,
-				  ADIOI_Flatlist_node *flat_buf, ADIO_Offset 
-				  *offset_list, ADIO_Offset *len_list, int *send_size, 
-				  int *recv_size, ADIO_Offset off, int size,
-				  int *count, int *start_pos,
-				  int *partial_recv,
-				  int *sent_to_proc, int nprocs, 
-				  int myrank, int
-				  buftype_is_contig, int contig_access_count,
-				  ADIO_Offset min_st_offset,
-				  ADIO_Offset fd_size,
-				  ADIO_Offset *fd_start, ADIO_Offset *fd_end, 
-				  ADIOI_Access *others_req, 
-				  int *send_buf_idx, int *curr_to_proc,
-				  int *done_to_proc, int *hole, int iter, 
-				  MPI_Aint buftype_extent, int *buf_idx,
-				  int *error_code)
-{
-    int i, j, k, *tmp_len, nprocs_recv, nprocs_send, err;
-    char **send_buf = NULL; 
-    MPI_Request *requests, *send_req;
-    MPI_Datatype *recv_types;
-    MPI_Status *statuses, status;
-    int *srt_len, sum;
-    ADIO_Offset *srt_off;
-    static char myname[] = "ADIOI_W_EXCHANGE_DATA";
-
-/* exchange recv_size info so that each process knows how much to
-   send to whom. */
-
-    MPI_Alltoall(recv_size, 1, MPI_INT, send_size, 1, MPI_INT, fd->comm);
-
-    /* create derived datatypes for recv */
-
-    nprocs_recv = 0;
-    for (i=0; i<nprocs; i++) if (recv_size[i]) nprocs_recv++;
-
-    recv_types = (MPI_Datatype *)
-	ADIOI_Malloc((nprocs_recv+1)*sizeof(MPI_Datatype)); 
-/* +1 to avoid a 0-size malloc */
-
-    tmp_len = (int *) ADIOI_Malloc(nprocs*sizeof(int));
-    j = 0;
-    for (i=0; i<nprocs; i++) {
-	if (recv_size[i]) {
-/* take care if the last off-len pair is a partial recv */
-	    if (partial_recv[i]) {
-		k = start_pos[i] + count[i] - 1;
-		tmp_len[i] = others_req[i].lens[k];
-		others_req[i].lens[k] = partial_recv[i];
-	    }
-	    ADIOI_Type_create_hindexed_x(count[i],
-		  &(others_req[i].lens[start_pos[i]]),
-	             &(others_req[i].mem_ptrs[start_pos[i]]), 
-			 MPI_BYTE, recv_types+j);
-	    /* absolute displacements; use MPI_BOTTOM in recv */
-	    MPI_Type_commit(recv_types+j);
-	    j++;
-	}
-    }
-
-    /* To avoid a read-modify-write, check if there are holes in the 
-       data to be written. For this, merge the (sorted) offset lists
-       others_req using a heap-merge. */
-
-    sum = 0;
-    for (i=0; i<nprocs; i++) sum += count[i];
-    srt_off = (ADIO_Offset *) ADIOI_Malloc((sum+1)*sizeof(ADIO_Offset));
-    srt_len = (int *) ADIOI_Malloc((sum+1)*sizeof(int));
-    /* +1 to avoid a 0-size malloc */
-
-    ADIOI_Heap_merge(others_req, count, srt_off, srt_len, start_pos,
-                     nprocs, nprocs_recv, sum);
-
-/* for partial recvs, restore original lengths */
-    for (i=0; i<nprocs; i++) 
-        if (partial_recv[i]) {
-            k = start_pos[i] + count[i] - 1;
-            others_req[i].lens[k] = tmp_len[i];
-        }
-    ADIOI_Free(tmp_len);
-
-    /* check if there are any holes. If yes, must do read-modify-write.
-     * holes can be in three places.  'middle' is what you'd expect: the
-     * processes are operating on noncontigous data.  But holes can also show
-     * up at the beginning or end of the file domain (see John Bent ROMIO REQ
-     * #835). Missing these holes would result in us writing more data than
-     * recieved by everyone else. */
-    *hole = 0;
-    if (off != srt_off[0]) /* hole at the front */
-        *hole = 1;
-    else { /* coalesce the sorted offset-length pairs */
-        for (i=1; i<sum; i++) {
-            if (srt_off[i] <= srt_off[0] + srt_len[0]) {
-		int new_len = srt_off[i] + srt_len[i] - srt_off[0];
-		if (new_len > srt_len[0]) srt_len[0] = new_len;
-	    }
-            else
-                break;
-        }
-        if (i < sum || size != srt_len[0]) /* hole in middle or end */
-            *hole = 1;
-	}
-
-    ADIOI_Free(srt_off);
-    ADIOI_Free(srt_len);
-
-    if (nprocs_recv) {
-	if (*hole) {
-	    const char * stuff = "data-sieve-in-two-phase";
-	    setenv("LIBIOLOG_EXTRA_INFO", stuff, 1);
-	    ADIO_ReadContig(fd, write_buf, size, MPI_BYTE, 
-			    ADIO_EXPLICIT_OFFSET, off, &status, &err);
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (err != MPI_SUCCESS) {
-		*error_code = MPIO_Err_create_code(err,
-						   MPIR_ERR_RECOVERABLE, myname,
-						   __LINE__, MPI_ERR_IO,
-						   "**ioRMWrdwr", 0);
-		return;
-	    } 
-	    /* --END ERROR HANDLING-- */
-	    unsetenv("LIBIOLOG_EXTRA_INFO");
-	}
-    }
-
-    nprocs_send = 0;
-    for (i=0; i < nprocs; i++) if (send_size[i]) nprocs_send++;
-
-    if (fd->atomicity) {
-        /* bug fix from Wei-keng Liao and Kenin Coloma */
-        requests = (MPI_Request *)
-	    ADIOI_Malloc((nprocs_send+1)*sizeof(MPI_Request)); 
-        send_req = requests;
-    }
-    else {
-        requests = (MPI_Request *) 	
-            ADIOI_Malloc((nprocs_send+nprocs_recv+1)*sizeof(MPI_Request)); 
-        /* +1 to avoid a 0-size malloc */
-
-        /* post receives */
-        j = 0;
-        for (i=0; i<nprocs; i++) {
-            if (recv_size[i]) {
-                MPI_Irecv(MPI_BOTTOM, 1, recv_types[j], i, myrank+i+100*iter,
-                          fd->comm, requests+j);
-                j++;
-            }
-        }
-	send_req = requests + nprocs_recv;
-    }
-
-/* post sends. if buftype_is_contig, data can be directly sent from
-   user buf at location given by buf_idx. else use send_buf. */
-
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5032, 0, NULL);
-#endif
-    if (buftype_is_contig) {
-	j = 0;
-	for (i=0; i < nprocs; i++) 
-	    if (send_size[i]) {
-		MPI_Isend(((char *) buf) + buf_idx[i], send_size[i], 
-  		            MPI_BYTE, i,  myrank+i+100*iter, fd->comm, 
-                                  send_req+j);
-		j++;
-                buf_idx[i] += send_size[i];
-	    }
-    }
-    else if (nprocs_send) {
-	/* buftype is not contig */
-	send_buf = (char **) ADIOI_Malloc(nprocs*sizeof(char*));
-	for (i=0; i < nprocs; i++) 
-	    if (send_size[i]) 
-		send_buf[i] = (char *) ADIOI_Malloc(send_size[i]);
-
-	ADIOI_Fill_send_buffer(fd, buf, flat_buf, send_buf,
-                           offset_list, len_list, send_size, 
-			   send_req,
-                           sent_to_proc, nprocs, myrank, 
-                           contig_access_count,
-                           min_st_offset, fd_size, fd_start, fd_end, 
-                           send_buf_idx, curr_to_proc, done_to_proc, iter,
-                           buftype_extent);
-        /* the send is done in ADIOI_Fill_send_buffer */
-    }
-
-    if (fd->atomicity) {
-        /* bug fix from Wei-keng Liao and Kenin Coloma */
-        j = 0;
-        for (i=0; i<nprocs; i++) {
-            MPI_Status wkl_status;
-	    if (recv_size[i]) {
-	        MPI_Recv(MPI_BOTTOM, 1, recv_types[j], i, myrank+i+100*iter,
-		          fd->comm, &wkl_status);
-	        j++;
-	    }
-        }
-    }
-
-    for (i=0; i<nprocs_recv; i++) MPI_Type_free(recv_types+i);
-    ADIOI_Free(recv_types);
-    
-    if (fd->atomicity) {
-        /* bug fix from Wei-keng Liao and Kenin Coloma */
-        statuses = (MPI_Status *) ADIOI_Malloc((nprocs_send+1) * \
-                                         sizeof(MPI_Status)); 
-         /* +1 to avoid a 0-size malloc */
-    }
-    else {
-        statuses = (MPI_Status *) ADIOI_Malloc((nprocs_send+nprocs_recv+1) * \
-                                     sizeof(MPI_Status)); 
-        /* +1 to avoid a 0-size malloc */
-    }
-
-#ifdef NEEDS_MPI_TEST
-    i = 0;
-    if (fd->atomicity) {
-        /* bug fix from Wei-keng Liao and Kenin Coloma */
-        while (!i) MPI_Testall(nprocs_send, send_req, &i, statuses);
-    }
-    else {
-        while (!i) MPI_Testall(nprocs_send+nprocs_recv, requests, &i, statuses);
-    }
-#else
-    if (fd->atomicity)
-        /* bug fix from Wei-keng Liao and Kenin Coloma */
-        MPI_Waitall(nprocs_send, send_req, statuses);
-    else
-        MPI_Waitall(nprocs_send+nprocs_recv, requests, statuses);
-#endif
-
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5033, 0, NULL);
-#endif
-    ADIOI_Free(statuses);
-    ADIOI_Free(requests);
-    if (!buftype_is_contig && nprocs_send) {
-	for (i=0; i < nprocs; i++) 
-	    if (send_size[i]) ADIOI_Free(send_buf[i]);
-	ADIOI_Free(send_buf);
-    }
-}
-
-
-#define ADIOI_BUF_INCR \
-{ \
-    while (buf_incr) { \
-        size_in_buf = ADIOI_MIN(buf_incr, flat_buf_sz); \
-        user_buf_idx += size_in_buf; \
-        flat_buf_sz -= size_in_buf; \
-        if (!flat_buf_sz) { \
-            if (flat_buf_idx < (flat_buf->count - 1)) flat_buf_idx++; \
-            else { \
-                flat_buf_idx = 0; \
-                n_buftypes++; \
-            } \
-            user_buf_idx = flat_buf->indices[flat_buf_idx] + \
-                              (ADIO_Offset)n_buftypes*(ADIO_Offset)buftype_extent; \
-            flat_buf_sz = flat_buf->blocklens[flat_buf_idx]; \
-        } \
-        buf_incr -= size_in_buf; \
-    } \
-}
-
-
-#define ADIOI_BUF_COPY \
-{ \
-    while (size) { \
-        size_in_buf = ADIOI_MIN(size, flat_buf_sz); \
-  ADIOI_Assert((((ADIO_Offset)(MPIU_Upint)buf) + user_buf_idx) == (ADIO_Offset)(MPIU_Upint)((MPIU_Upint)buf + user_buf_idx)); \
-  ADIOI_Assert(size_in_buf == (size_t)size_in_buf); \
-        memcpy(&(send_buf[p][send_buf_idx[p]]), \
-               ((char *) buf) + user_buf_idx, size_in_buf); \
-        send_buf_idx[p] += size_in_buf; \
-        user_buf_idx += size_in_buf; \
-        flat_buf_sz -= size_in_buf; \
-        if (!flat_buf_sz) { \
-            if (flat_buf_idx < (flat_buf->count - 1)) flat_buf_idx++; \
-            else { \
-                flat_buf_idx = 0; \
-                n_buftypes++; \
-            } \
-            user_buf_idx = flat_buf->indices[flat_buf_idx] + \
-                              (ADIO_Offset)n_buftypes*(ADIO_Offset)buftype_extent; \
-            flat_buf_sz = flat_buf->blocklens[flat_buf_idx]; \
-        } \
-        size -= size_in_buf; \
-        buf_incr -= size_in_buf; \
-    } \
-    ADIOI_BUF_INCR \
-}
-
-static void ADIOI_Fill_send_buffer(ADIO_File fd, const void *buf, ADIOI_Flatlist_node
-                           *flat_buf, char **send_buf, ADIO_Offset 
-                           *offset_list, ADIO_Offset *len_list, int *send_size, 
-                           MPI_Request *requests, int *sent_to_proc, 
-                           int nprocs, int myrank, 
-                           int contig_access_count, 
-                           ADIO_Offset min_st_offset, ADIO_Offset fd_size,
-                           ADIO_Offset *fd_start, ADIO_Offset *fd_end, 
-                           int *send_buf_idx, int *curr_to_proc, 
-                           int *done_to_proc, int iter,
-                           MPI_Aint buftype_extent)
-{
-/* this function is only called if buftype is not contig */
-
-    int i, p, flat_buf_idx;
-    ADIO_Offset flat_buf_sz, size_in_buf, buf_incr, size;
-    int jj, n_buftypes;
-    ADIO_Offset off, len, rem_len, user_buf_idx;
-
-/*  curr_to_proc[p] = amount of data sent to proc. p that has already
-    been accounted for so far
-    done_to_proc[p] = amount of data already sent to proc. p in 
-    previous iterations
-    user_buf_idx = current location in user buffer 
-    send_buf_idx[p] = current location in send_buf of proc. p  */
-
-    for (i=0; i < nprocs; i++) {
-	send_buf_idx[i] = curr_to_proc[i] = 0;
-	done_to_proc[i] = sent_to_proc[i];
-    }
-    jj = 0;
-
-    user_buf_idx = flat_buf->indices[0];
-    flat_buf_idx = 0;
-    n_buftypes = 0;
-    flat_buf_sz = flat_buf->blocklens[0];
-
-    /* flat_buf_idx = current index into flattened buftype
-       flat_buf_sz = size of current contiguous component in 
-	                 flattened buf */
-
-    for (i=0; i<contig_access_count; i++) { 
-	off     = offset_list[i];
-	rem_len = len_list[i];
-
-	/*this request may span the file domains of more than one process*/
-  while (rem_len != 0) {
-	    len = rem_len;
-	    /* NOTE: len value is modified by ADIOI_Calc_aggregator() to be no
-	     * longer than the single region that processor "p" is responsible
-	     * for.
-	     */
-	    p = ADIOI_GPFS_Calc_aggregator(fd,
-				      off,
-				      min_st_offset,
-				      &len,
-				      fd_size,
-				      fd_start,
-				      fd_end);
-
-	    if (send_buf_idx[p] < send_size[p]) {
-		if (curr_to_proc[p]+len > done_to_proc[p]) {
-		    if (done_to_proc[p] > curr_to_proc[p]) {
-			size = ADIOI_MIN(curr_to_proc[p] + len - 
-                                done_to_proc[p], send_size[p]-send_buf_idx[p]);
-			buf_incr = done_to_proc[p] - curr_to_proc[p];
-			ADIOI_BUF_INCR
-      ADIOI_Assert((curr_to_proc[p] + len - done_to_proc[p]) == (unsigned)(curr_to_proc[p] + len - done_to_proc[p]));
-		        buf_incr = curr_to_proc[p] + len - done_to_proc[p];
-      ADIOI_Assert((done_to_proc[p] + size) == (unsigned)(done_to_proc[p] + size));
-			curr_to_proc[p] = done_to_proc[p] + size;
-		        ADIOI_BUF_COPY
-		    }
-		    else {
-			size = ADIOI_MIN(len,send_size[p]-send_buf_idx[p]);
-			buf_incr = len;
-      ADIOI_Assert((curr_to_proc[p] + size) == (unsigned)((ADIO_Offset)curr_to_proc[p] + size));
-			curr_to_proc[p] += size;
-			ADIOI_BUF_COPY
-		    }
-		    if (send_buf_idx[p] == send_size[p]) {
-			MPI_Isend(send_buf[p], send_size[p], MPI_BYTE, p, 
-				myrank+p+100*iter, fd->comm, requests+jj);
-			jj++;
-		    }
-		}
-		else {
-        ADIOI_Assert((curr_to_proc[p] + len) == (unsigned)((ADIO_Offset)curr_to_proc[p] + len));
-		    curr_to_proc[p] += len;
-		    buf_incr = len;
-		    ADIOI_BUF_INCR
-		}
-	    }
-	    else {
-		buf_incr = len;
-		ADIOI_BUF_INCR
-            }
-	    off     += len;
-	    rem_len -= len;
-	}
-    }
-    for (i=0; i < nprocs; i++) 
-	if (send_size[i]) sent_to_proc[i] = curr_to_proc[i];
-}
-
-
-
-static void ADIOI_Heap_merge(ADIOI_Access *others_req, int *count, 
-		      ADIO_Offset *srt_off, int *srt_len, int *start_pos,
-		      int nprocs, int nprocs_recv, int total_elements)
-{
-    typedef struct {
-	ADIO_Offset *off_list;
-	ADIO_Offset *len_list;
-	int nelem;
-    } heap_struct;
-
-    heap_struct *a, tmp;
-    int i, j, heapsize, l, r, k, smallest;
-
-    a = (heap_struct *) ADIOI_Malloc((nprocs_recv+1)*sizeof(heap_struct));
-
-    j = 0;
-    for (i=0; i<nprocs; i++)
-	if (count[i]) {
-	    a[j].off_list = &(others_req[i].offsets[start_pos[i]]);
-	    a[j].len_list = &(others_req[i].lens[start_pos[i]]);
-	    a[j].nelem = count[i];
-	    j++;
-	}
-
-    /* build a heap out of the first element from each list, with
-       the smallest element of the heap at the root */
-
-    heapsize = nprocs_recv;
-    for (i=heapsize/2 - 1; i>=0; i--) {
-	/* Heapify(a, i, heapsize); Algorithm from Cormen et al. pg. 143
-           modified for a heap with smallest element at root. I have 
-           removed the recursion so that there are no function calls.
-           Function calls are too expensive. */
-	k = i;
-	while (1) {
-	    l = 2*(k+1) - 1;
-	    r = 2*(k+1);
-
-	    if ((l < heapsize) && 
-		(*(a[l].off_list) < *(a[k].off_list)))
-		smallest = l;
-	    else smallest = k;
-
-	    if ((r < heapsize) && 
-		(*(a[r].off_list) < *(a[smallest].off_list)))
-		smallest = r;
-
-	    if (smallest != k) {
-		tmp.off_list = a[k].off_list;
-		tmp.len_list = a[k].len_list;
-		tmp.nelem = a[k].nelem;
-
-		a[k].off_list = a[smallest].off_list;
-		a[k].len_list = a[smallest].len_list;
-		a[k].nelem = a[smallest].nelem;
-		
-		a[smallest].off_list = tmp.off_list;
-		a[smallest].len_list = tmp.len_list;
-		a[smallest].nelem = tmp.nelem;
-	    
-		k = smallest;
-	    }
-	    else break;
-	}
-    }
-
-    for (i=0; i<total_elements; i++) {
-        /* extract smallest element from heap, i.e. the root */
-	srt_off[i] = *(a[0].off_list);
-	srt_len[i] = *(a[0].len_list);
-	(a[0].nelem)--;
-
-	if (!a[0].nelem) {
-	    a[0].off_list = a[heapsize-1].off_list;
-	    a[0].len_list = a[heapsize-1].len_list;
-	    a[0].nelem = a[heapsize-1].nelem;
-	    heapsize--;
-	}
-	else {
-	    (a[0].off_list)++;
-	    (a[0].len_list)++;
-	}
-
-	/* Heapify(a, 0, heapsize); */
-	k = 0;
-	while (1) {
-	    l = 2*(k+1) - 1;
-	    r = 2*(k+1);
-
-	    if ((l < heapsize) && 
-		(*(a[l].off_list) < *(a[k].off_list)))
-		smallest = l;
-	    else smallest = k;
-
-	    if ((r < heapsize) && 
-		(*(a[r].off_list) < *(a[smallest].off_list)))
-		smallest = r;
-
-	    if (smallest != k) {
-		tmp.off_list = a[k].off_list;
-		tmp.len_list = a[k].len_list;
-		tmp.nelem = a[k].nelem;
-
-		a[k].off_list = a[smallest].off_list;
-		a[k].len_list = a[smallest].len_list;
-		a[k].nelem = a[smallest].nelem;
-		
-		a[smallest].off_list = tmp.off_list;
-		a[smallest].len_list = tmp.len_list;
-		a[smallest].nelem = tmp.nelem;
-	    
-		k = smallest;
-	    }
-	    else break;
-	}
-    }
-
-    ADIOI_Free(a);
-}
-
-
-static void ADIOI_W_Exchange_data_alltoallv(
-		ADIO_File fd, const void *buf, 
-		char *write_buf,					/* 1 */
-		ADIOI_Flatlist_node *flat_buf, 
-		ADIO_Offset *offset_list, 
-		ADIO_Offset *len_list, int *send_size, int *recv_size, 
-		ADIO_Offset off, int size,				/* 2 */
-		int *count, int *start_pos, int *partial_recv,
-		int *sent_to_proc, int nprocs, int myrank, 
-		int buftype_is_contig, int contig_access_count,
-		ADIO_Offset min_st_offset,
-		ADIO_Offset fd_size,
-		ADIO_Offset *fd_start, 
-		ADIO_Offset *fd_end,
-		ADIOI_Access *others_req,
-		int *send_buf_idx, int *curr_to_proc,			/* 3 */
-		int *done_to_proc, int *hole, 				/* 4 */
-		int iter, MPI_Aint buftype_extent, int *buf_idx,
-		int *error_code)
-{   
-    int i, j, k=0, nprocs_recv, nprocs_send, *tmp_len, err;
-    char **send_buf = NULL;
-    MPI_Request *send_req=NULL;
-    MPI_Status status;
-    int rtail, stail;
-    char *sbuf_ptr, *to_ptr;
-    int  len;
-    int  *sdispls, *rdispls;
-    char *all_recv_buf, *all_send_buf;
-    int *srt_len, sum;
-    ADIO_Offset *srt_off;
-    static char myname[] = "ADIOI_W_EXCHANGE_DATA";
-    double io_time;
-
-    io_time = MPI_Wtime();
-  /* exchange recv_size info so that each process knows how much to
-     send to whom. */
-    MPI_Alltoall(recv_size, 1, MPI_INT, send_size, 1, MPI_INT, fd->comm);
-
-    gpfsmpio_prof_cw[GPFSMPIO_CIO_T_DEXCH_RECV_EXCH] += MPI_Wtime() - io_time;
-    io_time = MPI_Wtime();
-    
-    nprocs_recv = 0;
-    for (i=0; i<nprocs; i++) if (recv_size[i]) { nprocs_recv++; }
-    nprocs_send = 0;
-    for (i=0; i<nprocs; i++) if (send_size[i]) { nprocs_send++; }
-    
-  /* receiver side data structures */
-    rdispls = (int *) ADIOI_Malloc( nprocs * sizeof(int) );
-    rtail = 0;
-    for (i=0; i<nprocs; i++) { rdispls[i] = rtail; rtail += recv_size[i]; }
-
-        /* data buffer */
-    all_recv_buf = (char *) ADIOI_Malloc( rtail );
-
-  /* sender side data structures */
-    sdispls = (int *) ADIOI_Malloc( nprocs * sizeof(int) );
-    stail = 0;
-    for (i=0; i<nprocs; i++) { sdispls[i] = stail; stail += send_size[i]; }
-
-        /* data buffer */
-    all_send_buf = (char *) ADIOI_Malloc( stail );
-    if (buftype_is_contig) {
-	for (i=0; i<nprocs; i++)
-	{
-	    if (send_size[i]) {
-		sbuf_ptr = all_send_buf + sdispls[i];
-		memcpy( sbuf_ptr, buf + buf_idx[i], send_size[i] );
-		buf_idx[i] += send_size[i];
-	    }
-	} 
-    } else {
-	send_buf = (char **) ADIOI_Malloc( nprocs * sizeof(char *) );
-	for (i=0; i<nprocs; i++)
-	    send_buf[i] = all_send_buf + sdispls[i];
-	ADIOI_Fill_send_buffer_nosend(fd, buf, flat_buf, send_buf,
-                           offset_list, len_list, send_size, 
-			   send_req,
-                           sent_to_proc, nprocs, myrank, 
-                           contig_access_count,
-                           min_st_offset, fd_size, fd_start, fd_end, 
-                           send_buf_idx, curr_to_proc, done_to_proc, iter,
-                           buftype_extent);
-	ADIOI_Free(send_buf);
-    }
-
-    gpfsmpio_prof_cw[GPFSMPIO_CIO_T_DEXCH_SETUP] += MPI_Wtime() - io_time;
-
-    io_time = MPI_Wtime();
-  /* alltoallv */
-    MPI_Alltoallv( 
-            all_send_buf, send_size, sdispls, MPI_BYTE,
-            all_recv_buf, recv_size, rdispls, MPI_BYTE,
-            fd->comm ); 
-
-    ADIOI_Free( all_send_buf );
-    ADIOI_Free(sdispls);
-
-    gpfsmpio_prof_cw[GPFSMPIO_CIO_T_DEXCH_NET] += MPI_Wtime() - io_time;
-    io_time = MPI_Wtime();
-  /* data sieving pre-read */
-  /* To avoid a read-modify-write, check if there are holes in the 
-     data to be written. For this, merge the (sorted) offset lists
-     others_req using a heap-merge. */
-
-    sum = 0;
-    for (i=0; i<nprocs; i++) sum += count[i];
-    srt_off = (ADIO_Offset *) ADIOI_Malloc((sum+1)*sizeof(ADIO_Offset));
-    srt_len = (int *) ADIOI_Malloc((sum+1)*sizeof(int));
-
-    ADIOI_Heap_merge(others_req, count, srt_off, srt_len, start_pos,
-                     nprocs, nprocs_recv, sum);
-
-    /* check if there are any holes */
-    *hole = 0;
-    /* See if there are holes before the first request or after the last request*/
-    if((srt_off[0] > off) || 
-       ((srt_off[sum-1] + srt_len[sum-1]) < (off + size)))
-    {
-       *hole = 1;
-    }
-    else /* See if there are holes between the requests, if there are more than one */
-      for (i=0; i<sum-1; i++)
-        if (srt_off[i]+srt_len[i] < srt_off[i+1]) {
-            *hole = 1;
-            break;
-        }
-
-    ADIOI_Free(srt_off);
-    ADIOI_Free(srt_len);
-
-    gpfsmpio_prof_cw[GPFSMPIO_CIO_T_DEXCH_SORT] += MPI_Wtime() - io_time;
-    io_time = MPI_Wtime();
-    if (nprocs_recv) {
-        if (*hole) {
-            ADIO_ReadContig(fd, write_buf, size, MPI_BYTE,
-                            ADIO_EXPLICIT_OFFSET, off, &status, &err);
-            /* --BEGIN ERROR HANDLING-- */
-            if (err != MPI_SUCCESS) {
-                *error_code = MPIO_Err_create_code(err,
-                                                   MPIR_ERR_RECOVERABLE, myname,
-                                                   __LINE__, MPI_ERR_IO,
-                                                   "**ioRMWrdwr", 0);
-                return;
-            }
-            /* --END ERROR HANDLING-- */
-        }
-    }
-    gpfsmpio_prof_cw[GPFSMPIO_CIO_T_DEXCH_SIEVE] += MPI_Wtime() - io_time;
-
-  /* scater all_recv_buf into 4M cb_buffer */
-    tmp_len = (int *) ADIOI_Malloc(nprocs*sizeof(int));
-    for (i=0; i<nprocs; i++)
-    {
-        if (recv_size[i]) {
-            if (partial_recv[i]) {
-                k = start_pos[i] + count[i] - 1;
-                tmp_len[i] = others_req[i].lens[k];
-                others_req[i].lens[k] = partial_recv[i];
-            }
-
-            sbuf_ptr = all_recv_buf + rdispls[i];
-            for (j=0; j<count[i]; j++) {
-                ADIOI_ENSURE_AINT_FITS_IN_PTR(others_req[i].mem_ptrs[ start_pos[i]+j ]);
-                to_ptr = (char *) ADIOI_AINT_CAST_TO_VOID_PTR ( others_req[i].mem_ptrs[ start_pos[i]+j ] );
-                len    =           others_req[i].lens[     start_pos[i]+j ]  ;
-                memcpy( to_ptr, sbuf_ptr, len );
-                sbuf_ptr += len;
-            }
-
-	    /* restore */
-            if (partial_recv[i]) {
-                k = start_pos[i] + count[i] - 1;
-                others_req[i].lens[k] = tmp_len[i];
-            }
-	    
-        }
-    }
-    
-    ADIOI_Free( tmp_len );
-    ADIOI_Free( all_recv_buf );
-    ADIOI_Free(rdispls);
-    return; 
-}   
-
-static void ADIOI_Fill_send_buffer_nosend(ADIO_File fd, const void *buf, ADIOI_Flatlist_node
-                           *flat_buf, char **send_buf, ADIO_Offset 
-                           *offset_list, ADIO_Offset *len_list, int *send_size, 
-                           MPI_Request *requests, int *sent_to_proc, 
-                           int nprocs, int myrank, 
-                           int contig_access_count, 
-                           ADIO_Offset min_st_offset, ADIO_Offset fd_size,
-                           ADIO_Offset *fd_start, ADIO_Offset *fd_end, 
-                           int *send_buf_idx, int *curr_to_proc, 
-                           int *done_to_proc, int iter,
-                           MPI_Aint buftype_extent)
-{
-/* this function is only called if buftype is not contig */
-
-    int i, p, flat_buf_idx;
-    ADIO_Offset flat_buf_sz, size_in_buf, buf_incr, size;
-    int jj, n_buftypes;
-    ADIO_Offset off, len, rem_len, user_buf_idx;
-
-/*  curr_to_proc[p] = amount of data sent to proc. p that has already
-    been accounted for so far
-    done_to_proc[p] = amount of data already sent to proc. p in 
-    previous iterations
-    user_buf_idx = current location in user buffer 
-    send_buf_idx[p] = current location in send_buf of proc. p  */
-
-    for (i=0; i < nprocs; i++) {
-	send_buf_idx[i] = curr_to_proc[i] = 0;
-	done_to_proc[i] = sent_to_proc[i];
-    }
-    jj = 0;
-
-    user_buf_idx = flat_buf->indices[0];
-    flat_buf_idx = 0;
-    n_buftypes = 0;
-    flat_buf_sz = flat_buf->blocklens[0];
-
-    /* flat_buf_idx = current index into flattened buftype
-       flat_buf_sz = size of current contiguous component in 
-	                 flattened buf */
-
-    for (i=0; i<contig_access_count; i++) { 
-	off     = offset_list[i];
-	rem_len = len_list[i];
-
-	/*this request may span the file domains of more than one process*/
-  while (rem_len != 0) {
-	    len = rem_len;
-	    /* NOTE: len value is modified by ADIOI_Calc_aggregator() to be no
-	     * longer than the single region that processor "p" is responsible
-	     * for.
-	     */
-	    p = ADIOI_GPFS_Calc_aggregator(fd,
-				      off,
-				      min_st_offset,
-				      &len,
-				      fd_size,
-				      fd_start,
-				      fd_end);
-
-	    if (send_buf_idx[p] < send_size[p]) {
-		if (curr_to_proc[p]+len > done_to_proc[p]) {
-		    if (done_to_proc[p] > curr_to_proc[p]) {
-			size = ADIOI_MIN(curr_to_proc[p] + len - 
-                                done_to_proc[p], send_size[p]-send_buf_idx[p]);
-			buf_incr = done_to_proc[p] - curr_to_proc[p];
-			ADIOI_BUF_INCR
-      ADIOI_Assert((curr_to_proc[p] + len - done_to_proc[p]) == (unsigned)(curr_to_proc[p] + len - done_to_proc[p]));
-		        buf_incr = curr_to_proc[p] + len - done_to_proc[p];
-      ADIOI_Assert((done_to_proc[p] + size) == (unsigned)(done_to_proc[p] + size));
-			curr_to_proc[p] = done_to_proc[p] + size;
-		        ADIOI_BUF_COPY
-		    }
-		    else {
-			size = ADIOI_MIN(len,send_size[p]-send_buf_idx[p]);
-			buf_incr = len;
-      ADIOI_Assert((curr_to_proc[p] + size) == (unsigned)((ADIO_Offset)curr_to_proc[p] + size));
-			curr_to_proc[p] += size;
-			ADIOI_BUF_COPY
-		    }
-		    /* moved to alltoallv */
-		    /*
-		    if (send_buf_idx[p] == send_size[p]) {
-			MPI_Isend(send_buf[p], send_size[p], MPI_BYTE, p, 
-				myrank+p+100*iter, fd->comm, requests+jj);
-			jj++;
-		    }
-		    */
-		}
-		else {
-        ADIOI_Assert((curr_to_proc[p] + len) == (unsigned)((ADIO_Offset)curr_to_proc[p] + len));
-		    curr_to_proc[p] += (int)len;
-		    buf_incr = len;
-		    ADIOI_BUF_INCR
-		}
-	    }
-	    else {
-		buf_incr = len;
-		ADIOI_BUF_INCR
-            }
-	    off     += len;
-	    rem_len -= len;
-	}
-    }
-    for (i=0; i < nprocs; i++) 
-	if (send_size[i]) sent_to_proc[i] = curr_to_proc[i];
-}
diff --git a/3rd-party/romio321/adio/ad_gpfs/bg/Makefile.mk b/3rd-party/romio321/adio/ad_gpfs/bg/Makefile.mk
deleted file mode 100644
index 1d957ef8f6f..00000000000
--- a/3rd-party/romio321/adio/ad_gpfs/bg/Makefile.mk
+++ /dev/null
@@ -1,18 +0,0 @@
-## -*- Mode: Makefile; -*-
-## vim: set ft=automake :
-##
-## (C) 2012 by Argonne National Laboratory.
-##     See COPYRIGHT in top-level directory.
-##
-
-if BUILD_AD_BG
-
-noinst_HEADERS +=                                                    \
-    adio/ad_gpfs/bg/ad_bg_aggrs.h                                         \
-    adio/ad_gpfs/bg/ad_bg_pset.h
-
-romio_other_sources +=                                               \
-    adio/ad_gpfs/bg/ad_bg_aggrs.c                                         \
-    adio/ad_gpfs/bg/ad_bg_pset.c 
-
-endif BUILD_AD_BG
diff --git a/3rd-party/romio321/adio/ad_gpfs/bg/ad_bg_aggrs.c b/3rd-party/romio321/adio/ad_gpfs/bg/ad_bg_aggrs.c
deleted file mode 100644
index f81e460fe94..00000000000
--- a/3rd-party/romio321/adio/ad_gpfs/bg/ad_bg_aggrs.c
+++ /dev/null
@@ -1,682 +0,0 @@
-/* ---------------------------------------------------------------- */
-/* (C)Copyright IBM Corp.  2007, 2008                               */
-/* ---------------------------------------------------------------- */
-/**
- * \file ad_bg_aggrs.c
- * \brief The externally used function from this file is is declared in ad_bg_aggrs.h
- */
-
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/* 
- *   Copyright (C) 1997-2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-/*#define TRACE_ON */
-
-// Uncomment this line to turn tracing on for the gpfsmpio_balancecontig aggr selection optimization
-// #define balancecontigtrace 1
-// #define bridgeringaggtrace 1
-
-#include "adio.h"
-#include "adio_cb_config_list.h"
-#include "../ad_gpfs.h"
-#include "ad_bg_pset.h"
-#include "ad_bg_aggrs.h"
-#ifdef AGGREGATION_PROFILE
-#include "mpe.h"
-#endif
-
-
-#ifdef USE_DBG_LOGGING
-  #define AGG_DEBUG 1
-#endif
-
-#ifndef TRACE_ERR
-#  define TRACE_ERR(format...)
-#endif
-
-/* Comments copied from common:
- * This file contains four functions:
- *
- * ADIOI_Calc_aggregator()
- * ADIOI_Calc_file_domains()
- * ADIOI_Calc_my_req()
- * ADIOI_Calc_others_req()
- *
- * The last three of these were originally in ad_read_coll.c, but they are
- * also shared with ad_write_coll.c.  I felt that they were better kept with
- * the rest of the shared aggregation code.  
- */
-
-/* Discussion of values available from above:
- *
- * ADIO_Offset st_offsets[0..nprocs-1]
- * ADIO_Offset end_offsets[0..nprocs-1]
- *    These contain a list of start and end offsets for each process in 
- *    the communicator.  For example, an access at loc 10, size 10 would
- *    have a start offset of 10 and end offset of 19.
- * int nprocs
- *    number of processors in the collective I/O communicator
- * ADIO_Offset min_st_offset
- * ADIO_Offset fd_start[0..nprocs_for_coll-1]
- *    starting location of "file domain"; region that a given process will
- *    perform aggregation for (i.e. actually do I/O)
- * ADIO_Offset fd_end[0..nprocs_for_coll-1]
- *    start + size - 1 roughly, but it can be less, or 0, in the case of 
- *    uneven distributions
- */
-
-/* forward declaration */
-static void 
-ADIOI_BG_compute_agg_ranklist_serial ( ADIO_File fd, 
-					const ADIOI_BG_ConfInfo_t *confInfo, 
-					ADIOI_BG_ProcInfo_t *all_procInfo);
-
-/*
- * Compute the aggregator-related parameters that are required in 2-phase collective IO of ADIO.
- * The parameters are 
- * 	. the number of aggregators (proxies) : fd->hints->cb_nodes
- *	. the ranks of the aggregators :        fd->hints->ranklist
- * By compute these two parameters in a BG-PSET-aware way, the default 2-phase collective IO of 
- *	ADIO can work more efficiently.
- */
-int 
-ADIOI_BG_gen_agg_ranklist(ADIO_File fd, int n_aggrs_per_pset) 
-{
-    int r, s;
-    ADIOI_BG_ProcInfo_t  *procInfo, *all_procInfo;
-    ADIOI_BG_ConfInfo_t  *confInfo;
-    TRACE_ERR("Entering ADIOI_BG_gen_agg_ranklist\n");
-
-    MPI_Comm_size( fd->comm, &s );
-    MPI_Comm_rank( fd->comm, &r );
-
-  /* Collect individual BG personality information */
-    confInfo = ADIOI_BG_ConfInfo_new ();
-    procInfo = ADIOI_BG_ProcInfo_new ();
-    ADIOI_BG_persInfo_init( confInfo, procInfo, s, r, n_aggrs_per_pset, fd->comm);
-
-  /* Gather BG personality infomation onto process 0 */
-    /* if (r == 0) */
-    all_procInfo  = ADIOI_BG_ProcInfo_new_n  (s);
-
-    MPI_Gather( (void *)procInfo,     sizeof(ADIOI_BG_ProcInfo_t), MPI_BYTE, 
-		(void *)all_procInfo, sizeof(ADIOI_BG_ProcInfo_t), MPI_BYTE, 
-		0, 
-		fd->comm );
-
-  /* Compute a list of the ranks of chosen IO proxy CN on process 0 */
-    if (r == 0) { 
-	ADIOI_BG_compute_agg_ranklist_serial (fd, confInfo, all_procInfo);
-	/* ADIOI_BG_ProcInfo_free (all_procInfo);*/
-    }
-    ADIOI_BG_ProcInfo_free (all_procInfo);
-
-  /* Send the info of IO proxy CN to all processes and keep the info in fd->hints struct.
-     Declared in adio_cb_config_list.h */
-    ADIOI_cb_bcast_rank_map(fd);
-    if (gpfsmpio_balancecontig == 1) { /* additionally need to send bridgelist,
-					bridgelistnum and numbridges to all
-					ranks */
-	if (r != 0) {
-	    fd->hints->fs_hints.bg.bridgelist =
-		ADIOI_Malloc(fd->hints->cb_nodes*sizeof(int));
-	    if (fd->hints->fs_hints.bg.bridgelist == NULL) {
-		/* NEED TO HANDLE ENOMEM */
-	    }
-	}
-	MPI_Bcast(fd->hints->fs_hints.bg.bridgelist, fd->hints->cb_nodes, MPI_INT, 0,
-		fd->comm);
-
-	if (r != 0) {
-	    fd->hints->fs_hints.bg.bridgelistnum =
-		ADIOI_Malloc(fd->hints->cb_nodes*sizeof(int));
-	    if (fd->hints->fs_hints.bg.bridgelistnum == NULL) {
-		/* NEED TO HANDLE ENOMEM */
-	    }
-	}
-	MPI_Bcast(fd->hints->fs_hints.bg.bridgelistnum, fd->hints->cb_nodes,
-		MPI_INT, 0, fd->comm);
-
-	MPI_Bcast(&fd->hints->fs_hints.bg.numbridges, 1, MPI_INT, 0,
-		fd->comm);
-
-    }
-
-
-    ADIOI_BG_persInfo_free( confInfo, procInfo );
-    TRACE_ERR("Leaving ADIOI_BG_gen_agg_ranklist\n");
-    return 0;
-}
-
-
-/* There are some number of bridge nodes (randomly) distributed through the job
- * We need to split the nodes among the bridge nodes */
-/* Maybe find which bridge node is closer (manhattan distance) and try to
- * distribute evenly.
- */
-/* 
- * Pick IO aggregators based on the under PSET organization and stores the ranks of the proxy CNs in tmp_ranklist.
- * The first order of tmp_ranklist is : PSET number
- * The secondary order of the list is determined in ADIOI_BG_select_agg_in_pset() and thus adjustable.
- */
-typedef struct
-{
-   int rank;
-   int bridge;
-} sortstruct;
-
-typedef struct
-{
-   int bridgeRank;
-   int numAggsAssigned;
-} bridgeAggAssignment;
-
-static int intsort(const void *p1, const void *p2)
-{
-   sortstruct *i1, *i2;
-   i1 = (sortstruct *)p1;
-   i2 = (sortstruct *)p2;
-   return(i1->bridge - i2->bridge);
-}
-
-static int 
-ADIOI_BG_compute_agg_ranklist_serial_do (const ADIOI_BG_ConfInfo_t *confInfo, 
-					  ADIOI_BG_ProcInfo_t       *all_procInfo, 
-					  int *tmp_ranklist)
-{
-    TRACE_ERR("Entering ADIOI_BG_compute_agg_ranklist_serial_do\n");
-   /* BES: This should be done in the init routines probably. */
-    int i, j;
-    int aggTotal;
-    int *aggList;
-
-    if (gpfsmpio_bridgeringagg > 0) {
-
-      int numAggs = confInfo->aggRatio * confInfo->ioMinSize /*virtualPsetSize*/;
-        /* the number of aggregators is (numAggs per bridgenode) */
-      if(numAggs == 1)
-        aggTotal = 1;
-      else
-        aggTotal = confInfo->numBridgeRanks * numAggs;
-
-      aggList = (int *)ADIOI_Malloc(aggTotal * sizeof(int));
-      if(aggTotal == 1) { /* special case when we only have one bridge node */
-
-        sortstruct *bridgelist = (sortstruct *)ADIOI_Malloc(confInfo->nProcs * sizeof(sortstruct));
-        for(i=0; i < confInfo->nProcs; i++)
-        {
-          bridgelist[i].bridge = all_procInfo[i].bridgeRank;
-          bridgelist[i].rank = i;
-          TRACE_ERR("bridgelist[%d].bridge: %d .rank: %d\n", i, bridgelist[i].bridge, i);
-        }
-
-        /* This list contains rank->bridge info. Now, we need to sort this list. */
-        qsort(bridgelist, confInfo->nProcs, sizeof(sortstruct), intsort);
-
-        aggList[0] = bridgelist[0].bridge;
-        ADIOI_Free(bridgelist);
-
-      }
-      else { // aggTotal > 1
-
-        int currentAggListSize = 0;
-        int numBridgesWithAggAssignments = 0;
-        bridgeAggAssignment *aggAssignments = (bridgeAggAssignment *)ADIOI_Malloc(confInfo->numBridgeRanks * sizeof(bridgeAggAssignment));
-
-        int partitionSize = all_procInfo[0].numNodesInPartition;
-        int *nodesAssigned = (int *)ADIOI_Malloc(partitionSize * sizeof(int));
-        for (i=0;i<partitionSize;i++)
-          nodesAssigned[i] = 0;
-
-        int currentNumHops = gpfsmpio_bridgeringagg;
-        int allAggsAssigned = 0;
-
-        /* Iterate thru the process infos and select aggregators starting at currentNumHops
-           away.  Increase the currentNumHops until all bridges have numAggs assigned to them.
-        */
-        while (!allAggsAssigned) {
-          /* track whether any aggs are selected durng this round */
-          int startingCurrentAggListSize = currentAggListSize;
-          int numIterForHopsWithNoAggs = 0;
-          for (i=0;i<confInfo->nProcs;i++) {
-          if (all_procInfo[i].manhattanDistanceToBridge == currentNumHops) {
-            if (nodesAssigned[all_procInfo[i].nodeRank] == 0) { // node is not assigned as an agg yet
-              int foundBridge = 0;
-              for (j=0;(j<numBridgesWithAggAssignments && !foundBridge);j++) {
-                if (aggAssignments[j].bridgeRank == all_procInfo[i].bridgeRank) {
-                  foundBridge = 1;
-                  if (aggAssignments[j].numAggsAssigned < numAggs) {
-                    aggAssignments[j].numAggsAssigned++;
-                    nodesAssigned[all_procInfo[i].nodeRank] = 1;
-                    aggList[currentAggListSize] = all_procInfo[i].rank;
-                    currentAggListSize++;
-#ifdef bridgeringaggtrace
-                printf("Assigned agg rank %d at nodeRank %d to bridge rank %d at a distance of %d hops\n",all_procInfo[i].rank,all_procInfo[i].nodeRank,all_procInfo[i].bridgeRank,currentNumHops);
-#endif
-                  }
-                }
-              }
-              if (!foundBridge) {
-                aggAssignments[numBridgesWithAggAssignments].bridgeRank = all_procInfo[i].bridgeRank;
-                aggAssignments[numBridgesWithAggAssignments].numAggsAssigned = 1;
-                numBridgesWithAggAssignments++;
-                nodesAssigned[all_procInfo[i].nodeRank] = 1;
-                aggList[currentAggListSize] = all_procInfo[i].rank;
-                currentAggListSize++;
-#ifdef bridgeringaggtrace
-                printf("Assigned agg rank %d at nodeRank %d to bridge rank %d at a distance of %d hops\n",all_procInfo[i].rank,all_procInfo[i].nodeRank,all_procInfo[i].bridgeRank,currentNumHops);
-#endif
-              }
-            }
-          }
-        }
-
-        if (numBridgesWithAggAssignments == confInfo->numBridgeRanks) {
-          allAggsAssigned = 1;
-          for (i=0;(i<numBridgesWithAggAssignments && allAggsAssigned);i++) {
-            if (aggAssignments[i].numAggsAssigned < numAggs)
-              allAggsAssigned = 0;
-          }
-        }
-        currentNumHops++;
-        /* Handle the case where the numAggs is more than exists starting
-         * at gpfsmpio_bridgeringagg hops, wrap back and restart at 0 to
-         * assign the overrun - it is up to the user to realize this
-         * situation and adjust numAggs and gpfsmpio_bridgeringagg
-         * accordingly.
-         */
-        if (currentNumHops > 16)
-          currentNumHops = 0;
-        /* If 3 rounds go by without selecting an agg abort to avoid
-           infinite loop.
-        */
-        if (startingCurrentAggListSize == currentAggListSize)
-          numIterForHopsWithNoAggs++;
-        else
-          numIterForHopsWithNoAggs = 0;
-        ADIOI_Assert(numIterForHopsWithNoAggs <= 3);
-        }
-
-        ADIOI_Free(aggAssignments);
-        ADIOI_Free(nodesAssigned);
-
-      } // else aggTotal  > 1
-
-       memcpy(tmp_ranklist, aggList, aggTotal*sizeof(int));
-    } // gpfsmpio_bridgeringagg > 0
-
-    else { // gpfsmpio_bridgeringagg unset - default code
-
-    int distance, numAggs;
-
-    /* Aggregators will be midpoints between sorted MPI rank lists of who shares a given
-     * bridge node */
-
-   sortstruct *bridgelist = (sortstruct *)ADIOI_Malloc(confInfo->nProcs * sizeof(sortstruct));
-   for(i=0; i < confInfo->nProcs; i++)
-   {
-      bridgelist[i].bridge = all_procInfo[i].bridgeRank;
-      bridgelist[i].rank = i;
-      TRACE_ERR("bridgelist[%d].bridge: %d .rank: %d\n", i, bridgelist[i].bridge, i);
-   }
-   
-   /* This list contains rank->bridge info. Now, we need to sort this list. */
-   qsort(bridgelist, confInfo->nProcs, sizeof(sortstruct), intsort);
-
-   /* In this array, we can pick an appropriate number of midpoints based on
-    * our bridgenode index and the number of aggregators */
-
-   numAggs = confInfo->aggRatio * confInfo->ioMinSize /*virtualPsetSize*/;
-   if(numAggs == 1)
-      aggTotal = 1;
-   else
-   /* the number of aggregators is (numAggs per bridgenode) plus each 
-    * bridge node is an aggregator */
-      aggTotal = confInfo->numBridgeRanks * (numAggs+1);
-
-   if(aggTotal>confInfo->nProcs) aggTotal=confInfo->nProcs;
-
-   TRACE_ERR("numBridgeRanks: %d, aggRatio: %f numBridge: %d pset size: %d/%d numAggs: %d, aggTotal: %d\n", confInfo->numBridgeRanks, confInfo->aggRatio, confInfo->numBridgeRanks,  confInfo->ioMinSize, confInfo->ioMaxSize /*virtualPsetSize*/, numAggs, aggTotal);
-   aggList = (int *)ADIOI_Malloc(aggTotal * sizeof(int));
-
-
-   /* For each bridge node, determine who the aggregators will be */
-   /* basically, the n*distance and bridge node */
-   if(aggTotal == 1) /* special case when we only have one bridge node */
-      aggList[0] = bridgelist[0].bridge;
-   else
-   {
-     int lastBridge = bridgelist[confInfo->nProcs-1].bridge;
-     int nextBridge = 0, nextAggr = confInfo->numBridgeRanks;
-     int psetSize = 0;
-     int procIndex;
-     for(procIndex=confInfo->nProcs-1; procIndex>=0; procIndex--)
-     {
-       TRACE_ERR("bridgelist[%d].bridge %u/rank %u\n",procIndex,  bridgelist[procIndex].bridge, bridgelist[procIndex].rank);
-       if(lastBridge == bridgelist[procIndex].bridge)
-       {
-         psetSize++;
-         if(procIndex) continue; 
-         else procIndex--;/* procIndex == 0 */
-       }
-       /* Sets up a list of nodes which will act as aggregators. numAggs
-        * per bridge node total. The list of aggregators is
-        * bridgeNode 0
-        * bridgeNode 1
-        * bridgeNode ...
-        * bridgeNode N
-        * bridgeNode[0]aggr[0]
-        * bridgeNode[0]aggr[1]...
-        * bridgeNode[0]aggr[N]...
-        * ...
-        * bridgeNode[N]aggr[0]..
-        * bridgeNode[N]aggr[N]
-        */
-       aggList[nextBridge]=lastBridge;
-       distance = psetSize/numAggs;
-       TRACE_ERR("nextBridge %u is bridge %u, distance %u, size %u\n",nextBridge, aggList[nextBridge],distance,psetSize);
-       if(numAggs>1)
-       {
-         for(j = 0; j < numAggs; j++)
-         {
-           ADIOI_Assert(nextAggr<aggTotal);
-           aggList[nextAggr] = bridgelist[procIndex+j*distance+1].rank;
-           TRACE_ERR("agglist[%d] -> bridgelist[%d] = %d\n", nextAggr, procIndex+j*distance+1,aggList[nextAggr]);
-           if(aggList[nextAggr]==lastBridge) /* can't have bridge in the list twice */
-           {  
-             aggList[nextAggr] = bridgelist[procIndex+psetSize].rank; /* take the last one in the pset */
-             TRACE_ERR("replacement agglist[%d] -> bridgelist[%d] = %d\n", nextAggr, procIndex+psetSize,aggList[nextAggr]);
-           }
-           nextAggr++;
-         }
-       }
-       if(procIndex<0) break;
-       lastBridge = bridgelist[procIndex].bridge;
-       psetSize = 1;
-       nextBridge++;
-     }
-   }
-
-   TRACE_ERR("memcpy(tmp_ranklist, aggList, (numAggs(%u)*confInfo->numBridgeRanks(%u)+numAggs(%u)) (%u) %u*sizeof(int))\n",numAggs,confInfo->numBridgeRanks,numAggs,(numAggs*confInfo->numBridgeRanks+numAggs),aggTotal);
-   memcpy(tmp_ranklist, aggList, aggTotal*sizeof(int));
-   for(i=0;i<aggTotal;i++)
-   {
-      TRACE_ERR("tmp_ranklist[%d]: %d\n", i, tmp_ranklist[i]);
-   }
-
-
-   ADIOI_Free (bridgelist);
-
-   TRACE_ERR("Leaving ADIOI_BG_compute_agg_ranklist_serial_do\n");
-   }
-
-   ADIOI_Free (aggList);
-   return aggTotal;
-
-}
-
-/* 
- * compute aggregators ranklist and put it into fd->hints struct
- */ 
-static void 
-ADIOI_BG_compute_agg_ranklist_serial ( ADIO_File fd, 
-					const ADIOI_BG_ConfInfo_t *confInfo, 
-					ADIOI_BG_ProcInfo_t *all_procInfo)
-{
-    TRACE_ERR("Entering ADIOI_BG_compute_agg_ranklist_serial\n");
-    int i; 
-    int naggs; 
-    int size;
-    int *tmp_ranklist;
-
-  /* compute the ranklist of IO aggregators and put into tmp_ranklist */
-    tmp_ranklist = (int *) ADIOI_Malloc (confInfo->nProcs * sizeof(int));
-
-#   if AGG_DEBUG
-    for (i=0; i<confInfo->nProcs; i++) {
-      DBG_FPRINTF(stderr, "\trank = %6d\n", all_procInfo[i].rank );
-    }
-#   endif
-
-    naggs= 
-    ADIOI_BG_compute_agg_ranklist_serial_do (confInfo, all_procInfo, tmp_ranklist);
-
-#   define VERIFY 1
-#   if VERIFY
-    DBG_FPRINTF(stderr, "\tconfInfo = min: %3d, max: %3d, naggrs: %3d, bridge: %3d, nprocs: %3d, vpset: %3d, ratio: %.4f; naggs = %d\n",
-	    confInfo->ioMinSize        ,
-	    confInfo->ioMaxSize        ,
-	    confInfo->nAggrs           ,
-	    confInfo->numBridgeRanks ,
-	    confInfo->nProcs          ,
-	    confInfo->ioMaxSize /*virtualPsetSize*/          ,
-	    confInfo->aggRatio        ,
-	    naggs );
-#   endif
-    MPI_Comm_size( fd->comm, &size );
-    /* This fix is for when the bridgenode rnk is not part of the particular
-     * subcomm associated with this MPI File operation. I don't know if
-     * this is the best/right answer but it passes the test cases at least.
-     * I don't know how common file IO in subcomms is anyway... */
-    for(i=0;i<naggs;i++)
-    {
-      if(tmp_ranklist[i] > size)
-      {
-         TRACE_ERR("Using 0 as tmp_ranklist[%d] instead of %d for comm %x\n",
-               i, tmp_ranklist[i], fd->comm);
-         tmp_ranklist[i] = 0;
-      }
-   }
-         
-#   if AGG_DEBUG
-    for (i=0; i<naggs; i++) {
-      DBG_FPRINTF(stderr, "\taggr %-4d = %6d\n", i, tmp_ranklist[i] );
-    }
-#   endif
-    if (gpfsmpio_balancecontig == 1) {
-	/* what comes out of this code block is the agg ranklist sorted by
-	 * bridge set and ion id with associated bridge info stored in the
-	 * hints structure for later access during file domain assignment */
-
-	// sort the agg ranklist by ions and bridges
-
-	int *interleavedbridgeranklist = (int *) ADIOI_Malloc (naggs * sizeof(int)); // resorted agg rank list
-	/* list of all bridge ranks */
-	int *bridgelist = (int *) ADIOI_Malloc (naggs * sizeof(int));
-
-	/* each entry here is the number of aggregators associated with the
-	 * bridge rank of the same index in bridgelist */
-	int *bridgelistnum = (int *) ADIOI_Malloc (naggs * sizeof(int));
-	/* list of all ion IDs corresponding with bridgelist entries of same index */
-	int *ionlist = (int *) ADIOI_Malloc (naggs * sizeof(int));
-
-	int numbridges = 0;
-
-	for (i=0;i<naggs;i++)
-	    bridgelistnum[i] = 0;
-
-	/* Each entry in this list corresponds with the bridgelist and will contain the lowest bridge
-	 * agg rank on that ion. */
-	int *summarybridgeminionaggrank = (int *) ADIOI_Malloc (naggs * sizeof(int));
-	for (i=0;i<naggs;i++)
-	    summarybridgeminionaggrank[i] = -1;
-
-	/* build the bridgelist, ionlist and bridgelistnum data by going thru each agg
-	 * entry and find the associated bridge list index - at the end we will
-	 * know how many aggs belong to each bridge in each ion */
-	for (i=0;i<naggs;i++) {
-	    int aggbridgerank = all_procInfo[tmp_ranklist[i]].bridgeRank;
-	    int aggionid = all_procInfo[tmp_ranklist[i]].ionID;
-	    int foundrank = 0;
-	    int summaryranklistbridgeindex = 0;
-	    int j;
-	    for (j=0;(j<numbridges && !foundrank);j++) {
-		if (bridgelist[j] == aggbridgerank) {
-		    foundrank = 1;
-		    summaryranklistbridgeindex = j;
-		}
-		else
-		    summaryranklistbridgeindex++;
-	    }
-	    if (!foundrank) {
-		bridgelist[summaryranklistbridgeindex] = aggbridgerank;
-		ionlist[summaryranklistbridgeindex] = aggionid;
-
-		if (summarybridgeminionaggrank[summaryranklistbridgeindex] == -1)
-		    summarybridgeminionaggrank[summaryranklistbridgeindex] = aggbridgerank;
-		else if (summarybridgeminionaggrank[summaryranklistbridgeindex] > aggbridgerank)
-		    summarybridgeminionaggrank[summaryranklistbridgeindex] = aggbridgerank;
-		numbridges++;
-	    }
-
-	    bridgelistnum[summaryranklistbridgeindex]++;
-	}
-
-    /* at this point summarybridgeminionaggrank has the agg rank of the bridge for entries,
-     * need to make each entry the minimum bridge rank for the entire ion. */
-    for (i=0;i<numbridges;i++) {
-        int aggIonId = ionlist[i];
-        int j;
-        for (j=0;j<numbridges;j++) {
-          if (ionlist[j] == aggIonId) {
-            if (summarybridgeminionaggrank[j] < summarybridgeminionaggrank[i])
-              summarybridgeminionaggrank[i] = summarybridgeminionaggrank[j];
-          }
-        }
-    }
-
-	// resort by io node minimum bridge rank
-	int x;
-	for (x=0;x<numbridges;x++) {
-	    for (i=0;i<(numbridges-1);i++) {
-		if (summarybridgeminionaggrank[i] > summarybridgeminionaggrank[i+1]) {
-		    int tmpminionaggrank = summarybridgeminionaggrank[i];
-		    summarybridgeminionaggrank[i] = summarybridgeminionaggrank[i+1];
-		    summarybridgeminionaggrank[i+1] = tmpminionaggrank;
-		    int tmpionid = ionlist[i];
-		    ionlist[i] = ionlist[i+1];
-		    ionlist[i+1] = tmpionid;
-		    int tmpbridgerank = bridgelist[i];
-		    bridgelist[i] = bridgelist[i+1];
-		    bridgelist[i+1] = tmpbridgerank;
-		    int tmpbridgeranknum = bridgelistnum[i];
-		    bridgelistnum[i] = bridgelistnum[i+1];
-		    bridgelistnum[i+1] = tmpbridgeranknum;
-		  }
-	    }
-	}
-
-	// for each io node make sure bridgelist is in rank order
-	int startSortIndex = -1;
-	int endSortIndex = -1;
-	int currentBridgeIndex = 0;
-
-	while (currentBridgeIndex < numbridges) {
-	    int currentIonId = ionlist[currentBridgeIndex];
-	    startSortIndex = currentBridgeIndex;
-	    while (ionlist[currentBridgeIndex] == currentIonId)
-		  currentBridgeIndex++;
-	    endSortIndex = currentBridgeIndex-1;
-	    for (x=startSortIndex;x<=endSortIndex;x++) {
-		  for (i=startSortIndex;i<endSortIndex;i++) {
-		    if (bridgelist[i] > bridgelist[i+1]) {
-			  int tmpbridgerank = bridgelist[i];
-			  bridgelist[i] = bridgelist[i+1];
-			  bridgelist[i+1] = tmpbridgerank;
-			  int tmpbridgeranknum = bridgelistnum[i];
-			  bridgelistnum[i] = bridgelistnum[i+1];
-			  bridgelistnum[i+1] = tmpbridgeranknum;
-		    }
-		  }
-	    }
-	}
-
-
-	/* populate interleavedbridgeranklist - essentially the agg rank list
-	 * is now sorted by the ion minimum bridge rank and bridge node */
-	int currentrankoffset = 0;
-	for (i=0;i<numbridges;i++) {
-	    int *thisBridgeAggList = (int *) ADIOI_Malloc (naggs * sizeof(int));
-	    int numAggsForThisBridge = 0;
-
-	    int k;
-	    for (k=0;k<naggs;k++) {
-		int aggbridgerank = all_procInfo[tmp_ranklist[k]].bridgeRank;
-		if (aggbridgerank == bridgelist[i]) {
-		    thisBridgeAggList[numAggsForThisBridge] = tmp_ranklist[k];
-		    numAggsForThisBridge++;
-		}
-	    }
-
-	    // sort thisBridgeAggList
-	    for (x=0;x<numAggsForThisBridge;x++) {
-		int n;
-		for (n=0;n<(numAggsForThisBridge-1);n++) {
-		    if (thisBridgeAggList[n] > thisBridgeAggList[n+1]) {
-			int tmpthisBridgeAggList = thisBridgeAggList[n];
-			thisBridgeAggList[n] = thisBridgeAggList[n+1];
-			thisBridgeAggList[n+1] = tmpthisBridgeAggList;
-		    }
-		}
-	    }
-	    int n;
-	    for (n=0;n<numAggsForThisBridge;n++) {
-		interleavedbridgeranklist[currentrankoffset] = thisBridgeAggList[n];
-		currentrankoffset++;
-	    }
-	    ADIOI_Free(thisBridgeAggList);
-	}
-
-#ifdef balancecontigtrace
-	fprintf(stderr,"Interleaved aggregator list:\n");
-	for (i=0;i<naggs;i++) {
-	    fprintf(stderr,"Agg: %d Agg rank: %d with bridge rank %d and ion ID %d\n",i,interleavedbridgeranklist[i],all_procInfo[interleavedbridgeranklist[i]].bridgeRank,all_procInfo[interleavedbridgeranklist[i]].ionID);
-	}
-	fprintf(stderr,"Bridges list:\n");
-	for (i=0;i<numbridges;i++) {
-	    fprintf(stderr,"bridge %d ion min rank %d rank %d number of aggs %d ion id %d\n",i,summarybridgeminionaggrank[i],bridgelist[i],bridgelistnum[i],ionlist[i]);
-	}
-
-#endif
-	/* copy the ranklist of IO aggregators to fd->hints */
-	if(fd->hints->ranklist != NULL)
-	    ADIOI_Free (fd->hints->ranklist);
-	if(fd->hints->fs_hints.bg.bridgelist != NULL)
-	    ADIOI_Free (fd->hints->fs_hints.bg.bridgelist);
-	if(fd->hints->fs_hints.bg.bridgelistnum != NULL)
-	    ADIOI_Free (fd->hints->fs_hints.bg.bridgelistnum);
-
-	fd->hints->cb_nodes = naggs;
-	fd->hints->fs_hints.bg.numbridges = numbridges;
-	fd->hints->ranklist = (int *) ADIOI_Malloc (naggs * sizeof(int));
-	memcpy( fd->hints->ranklist, interleavedbridgeranklist, naggs*sizeof(int) );
-
-	fd->hints->fs_hints.bg.bridgelist = (int *) ADIOI_Malloc (naggs * sizeof(int));
-	memcpy( fd->hints->fs_hints.bg.bridgelist, bridgelist, naggs*sizeof(int) );
-
-	fd->hints->fs_hints.bg.bridgelistnum = (int *) ADIOI_Malloc (naggs * sizeof(int));
-	memcpy( fd->hints->fs_hints.bg.bridgelistnum, bridgelistnum, naggs*sizeof(int) );
-
-	ADIOI_Free(summarybridgeminionaggrank);
-	ADIOI_Free( tmp_ranklist );
-	ADIOI_Free( bridgelistnum );
-	ADIOI_Free( bridgelist );
-	ADIOI_Free( interleavedbridgeranklist );
-	ADIOI_Free(ionlist);
-
-    }  else {
-	/* classic topology-agnostic copy of the ranklist of IO aggregators to
-	 * fd->hints */
-	if(fd->hints->ranklist != NULL) ADIOI_Free (fd->hints->ranklist);
-
-	fd->hints->cb_nodes = naggs;
-	fd->hints->ranklist = (int *) ADIOI_Malloc (naggs * sizeof(int));
-	memcpy( fd->hints->ranklist, tmp_ranklist, naggs*sizeof(int) );
-
-	ADIOI_Free( tmp_ranklist );
-    }
-    TRACE_ERR("Leaving ADIOI_BG_compute_agg_ranklist_serial\n");
-    return;
-}
diff --git a/3rd-party/romio321/adio/ad_gpfs/bg/ad_bg_aggrs.h b/3rd-party/romio321/adio/ad_gpfs/bg/ad_bg_aggrs.h
deleted file mode 100644
index b154722850c..00000000000
--- a/3rd-party/romio321/adio/ad_gpfs/bg/ad_bg_aggrs.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/* ---------------------------------------------------------------- */
-/* (C)Copyright IBM Corp.  2007, 2008                               */
-/* ---------------------------------------------------------------- */
-/**
- * \file ad_bg_aggrs.h
- * \brief ???
- */
-
-/* 
- *
- * Declares functions specific for the BlueGene platform within the GPFS
- * parallel I/O solution.  Implements aligned file-domain partitioning
- * (7/28/2005);  persistent file doamin work not implemented
- *
- */
-
-#ifndef AD_BG_AGGRS_H_
-#define AD_BG_AGGRS_H_
-
-#include "adio.h"
-#include <sys/stat.h>
-
-#ifdef HAVE_GPFS_H
-#include <gpfs.h>
-#endif
-#if !defined(GPFS_SUPER_MAGIC)
-  #define GPFS_SUPER_MAGIC (0x47504653)
-#endif
-
-    /* generate a list of I/O aggregators that utilizes BG-PSET orginization. */
-    int ADIOI_BG_gen_agg_ranklist(ADIO_File fd, int n_aggrs_per_pset);
-
-#endif  /* AD_BG_AGGRS_H_ */
diff --git a/3rd-party/romio321/adio/ad_gpfs/bg/ad_bg_pset.c b/3rd-party/romio321/adio/ad_gpfs/bg/ad_bg_pset.c
deleted file mode 100644
index 58b051af89a..00000000000
--- a/3rd-party/romio321/adio/ad_gpfs/bg/ad_bg_pset.c
+++ /dev/null
@@ -1,414 +0,0 @@
-/* ---------------------------------------------------------------- */
-/* (C)Copyright IBM Corp.  2007, 2008                               */
-/* ---------------------------------------------------------------- */
-/**
- * \file ad_bg_pset.c
- * \brief Definition of functions associated to structs ADIOI_BG_ProcInfo_t and ADIOI_BG_ConfInfo_t 
- */
-
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-/* #define TRACE_ON */
-// #define bridgeringaggtrace 1
-
-#include <stdlib.h>
-#include <stdbool.h>
-#include "../ad_gpfs.h"
-#include "ad_bg_pset.h"
-#include <spi/include/kernel/process.h>
-#include <firmware/include/personality.h>
-
-#define BGQ_TORUS_MAX_DIMS 5
-#define BGQ_FULL_TORUS_SIZE 512
-
-#ifndef TRACE_ERR
-#  define TRACE_ERR(fmt...)
-#endif
-
-ADIOI_BG_ProcInfo_t *
-ADIOI_BG_ProcInfo_new()
-{
-    ADIOI_BG_ProcInfo_t *p = (ADIOI_BG_ProcInfo_t *) ADIOI_Malloc (sizeof(ADIOI_BG_ProcInfo_t));
-    ADIOI_Assert ((p != NULL));
-    return p;
-}
-
-ADIOI_BG_ProcInfo_t *
-ADIOI_BG_ProcInfo_new_n( int n )
-{
-    ADIOI_BG_ProcInfo_t *p = (ADIOI_BG_ProcInfo_t *) ADIOI_Malloc (n * sizeof(ADIOI_BG_ProcInfo_t));
-    ADIOI_Assert ((p != NULL));
-    return p;
-}
-
-void
-ADIOI_BG_ProcInfo_free( ADIOI_BG_ProcInfo_t *info )
-{
-    if (info != NULL) ADIOI_Free (info);
-}
-
-ADIOI_BG_ConfInfo_t *
-ADIOI_BG_ConfInfo_new ()
-{
-    ADIOI_BG_ConfInfo_t *p = (ADIOI_BG_ConfInfo_t *) ADIOI_Malloc (sizeof(ADIOI_BG_ConfInfo_t));
-    ADIOI_Assert ((p != NULL));
-    return p;
-}
-
-
-void
-ADIOI_BG_ConfInfo_free( ADIOI_BG_ConfInfo_t *info )
-{
-    if (info != NULL) ADIOI_Free (info);
-}
-
-
-typedef struct
-{
-   int rank;
-   int bridgeCoord;
-} sortstruct;
-
-static int intsort(const void *p1, const void *p2)
-{
-   sortstruct *i1, *i2;
-   i1 = (sortstruct *)p1;
-   i2 = (sortstruct *)p2;
-   return(i1->bridgeCoord - i2->bridgeCoord);
-}
-
-unsigned torusSize[BGQ_TORUS_MAX_DIMS];
-bool dimTorus[BGQ_TORUS_MAX_DIMS];
-
-/* This function computes the number of hops between the torus coordinates of the
- * aggCoords and bridgeCoords parameters.
-*/
-static unsigned procManhattanDistance(unsigned *aggCoords, unsigned *bridgeCoords) {
-
-  unsigned totalDistance = 0;
-  int i;
-  for (i=0;i<BGQ_TORUS_MAX_DIMS;i++) {
-    unsigned dimDistance = abs((int)aggCoords[i] - (int)bridgeCoords[i]);
-    if (dimDistance > 0) { // could torus make it closer?
-      if (dimTorus[i]) {
-        if (aggCoords[i] == torusSize[i]) { // is wrap-around closer
-          if ((bridgeCoords[i]+1) < dimDistance) // assume will use torus link
-            dimDistance = bridgeCoords[i]+1;
-        }
-        else if (bridgeCoords[i] == torusSize[i]) { // is wrap-around closer
-          if ((aggCoords[i]+1) < dimDistance) // assume will use torus link
-            dimDistance = aggCoords[i]+1;
-        }
-      }
-    } /* else: dimDistance == 0, meaning aggCoords[i] and bridgeCoords[i] are
-	 the same and there's no closer point to pick */
-    totalDistance += dimDistance;
-  }
-  return totalDistance;
-}
-
-int BGQ_IO_node_id ()
-{
-  static unsigned long IO_node_id = ULONG_MAX;
-
-  if (IO_node_id != ULONG_MAX)
-    return (int)(IO_node_id>>32);
-
-  int rc;
-  int fd;
-  char* uci_str;
-  char buffer[4096];
-
-  fd = open("/dev/bgpers", O_RDONLY, 0);
-  assert(fd>=0);
-  rc = read(fd, buffer, sizeof(buffer));
-  assert(rc>0);
-  close(fd);
-
-  uci_str = strstr(buffer, "BG_UCI=");
-  assert(uci_str);
-  uci_str += sizeof("BG_UCI=")-1;
-
-  IO_node_id = strtoul(uci_str, NULL, 16);
-  return (int)(IO_node_id>>32);
-}
-
-void 
-ADIOI_BG_persInfo_init(ADIOI_BG_ConfInfo_t *conf, 
-			ADIOI_BG_ProcInfo_t *proc, 
-			int size, int rank, int n_aggrs, MPI_Comm comm)
-{
-   int i, iambridge=0, bridgerank = -1, bridgeIndex;
-   int countPset;
-   sortstruct *bridges;
-   int commsize;
-
-   TRACE_ERR("Entering BG_persInfo_init, size: %d, rank: %d, n_aggrs: %d, comm: %d\n", size, rank, n_aggrs, (int)comm);
-
-   Personality_t pers;
-
-
-   Kernel_GetPersonality(&pers, sizeof(pers));
-   Personality_Networks_t *net = &pers.Network_Config;
-
-   TRACE_ERR("BG_persInfo_init, my coords{%u,%u,%u,%u,%u}\n",net->Acoord,net->Bcoord,net->Ccoord,net->Dcoord,net->Ecoord);
-   proc->rank = rank;
-
-   if (gpfsmpio_bridgeringagg > 0) {
-#ifdef bridgeringaggtrace
-     if (rank == 0)
-       fprintf(stderr,"Block dimensions:\n");
-#endif
-
-     /* Set the numNodesInPartition and nodeRank for this proc
-     */
-     unsigned dimMaxArray[BGQ_TORUS_MAX_DIMS];
-     dimMaxArray[0] = net->Anodes;
-     dimMaxArray[1] = net->Bnodes;
-     dimMaxArray[2] = net->Cnodes;
-     dimMaxArray[3] = net->Dnodes;
-     dimMaxArray[4] = net->Enodes;
-
-     unsigned hwCoordsArray[BGQ_TORUS_MAX_DIMS];
-     hwCoordsArray[0] = net->Acoord;
-     hwCoordsArray[1] = net->Bcoord;
-     hwCoordsArray[2] = net->Ccoord;
-     hwCoordsArray[3] = net->Dcoord;
-     hwCoordsArray[4] = net->Ecoord;
-     proc->numNodesInPartition = net->Anodes * net->Bnodes * net->Cnodes * net->Dnodes * net->Enodes;
-     proc->nodeRank = 0;
-     /* Set the indicator for if a dimension in the partitions is a torus or not.
-      */
-     dimTorus[0] = (bool) (ND_ENABLE_TORUS_DIM_A & net->NetFlags);
-     dimTorus[1] = (bool) (ND_ENABLE_TORUS_DIM_B & net->NetFlags);
-     dimTorus[2] = (bool) (ND_ENABLE_TORUS_DIM_C & net->NetFlags);
-     dimTorus[3] = (bool) (ND_ENABLE_TORUS_DIM_D & net->NetFlags);
-     dimTorus[4] = (bool) (ND_ENABLE_TORUS_DIM_E & net->NetFlags);
-     for (i=0;i<BGQ_TORUS_MAX_DIMS;i++) {
-       torusSize[i] = dimMaxArray[i];
-         int baseNum = 1, j;
-         for (j=0;j<i;j++)
-           baseNum *= dimMaxArray[j];
-         proc->nodeRank += (hwCoordsArray[i] * baseNum);
-#ifdef bridgeringaggtrace
-       if (rank == 0)
-         fprintf(stderr,"numNodesInPartition is %d Dimension %d has %d elements wrap-around value is %d\n",proc->numNodesInPartition,i,torusSize[i],dimTorus[i]);
-#endif
-     }
-   }
-
-   MPI_Comm_size(comm, &commsize);
-
-   proc->ionID = BGQ_IO_node_id ();
-
-   if(size == 1)
-   {
-      proc->iamBridge = 1;
-      proc->bridgeRank = rank;
-      if (gpfsmpio_bridgeringagg > 0) {
-        proc->manhattanDistanceToBridge = 0;
-      }
-
-      /* Set up the other parameters */
-      proc->myIOSize = size;
-      proc->ioNodeIndex = 0;
-      conf->ioMinSize = size;
-      conf->ioMaxSize = size;
-      conf->numBridgeRanks = 1;
-      conf->nProcs = size;
-      conf->nAggrs = 1;
-      conf->aggRatio = 1. * conf->nAggrs / conf->ioMinSize /*virtualPsetSize*/;
-      if(conf->aggRatio > 1) conf->aggRatio = 1.;
-      TRACE_ERR("I am (single) Bridge rank\n");
-      return;
-   }
-
-   /* Find the nearest bridge node coords.  We don't know the
-      rank in our comm so we will collective find/pick a bridge
-      rank later.
-   */ 
-   int32_t bridgeCoords;
-   bridgeCoords = pers.Network_Config.cnBridge_A << 24 | 
-                  pers.Network_Config.cnBridge_B << 18 | 
-                  pers.Network_Config.cnBridge_C << 12 | 
-                  pers.Network_Config.cnBridge_D << 6 | 
-                  pers.Network_Config.cnBridge_E << 2;
-   ADIOI_Assert((bridgeCoords >= 0)); /* A dim is < 6 bits or sorting won't work */
-
-   if((net->Acoord == pers.Network_Config.cnBridge_A) &&
-      (net->Bcoord == pers.Network_Config.cnBridge_B) &&
-      (net->Ccoord == pers.Network_Config.cnBridge_C) &&
-      (net->Dcoord == pers.Network_Config.cnBridge_D) &&
-      (net->Ecoord == pers.Network_Config.cnBridge_E)) {
-      iambridge = 1;      /* I am bridge */
-      if (gpfsmpio_bridgeringagg > 0) {
-        proc->manhattanDistanceToBridge = 0;
-      }
-    }
-    else {  // calculate manhattan distance to bridge if gpfsmpio_bridgeringagg is set
-      if (gpfsmpio_bridgeringagg > 0) {
-        unsigned aggCoords[BGQ_TORUS_MAX_DIMS],manhattanBridgeCoords[BGQ_TORUS_MAX_DIMS];
-        aggCoords[0] = net->Acoord;
-        manhattanBridgeCoords[0] = pers.Network_Config.cnBridge_A;
-        aggCoords[1] = net->Bcoord;
-        manhattanBridgeCoords[1] = pers.Network_Config.cnBridge_B;
-        aggCoords[2] = net->Ccoord;
-        manhattanBridgeCoords[2] = pers.Network_Config.cnBridge_C;
-        aggCoords[3] = net->Dcoord;
-        manhattanBridgeCoords[3] = pers.Network_Config.cnBridge_D;
-        aggCoords[4] = net->Ecoord;
-        manhattanBridgeCoords[4] = pers.Network_Config.cnBridge_E;
-
-        proc->manhattanDistanceToBridge= procManhattanDistance(aggCoords, manhattanBridgeCoords);
-#ifdef bridgeringaggtrace
-        fprintf(stderr,"agg coords are %u %u %u %u %u bridge coords are %u %u %u %u %u distance is %u\n",aggCoords[0],aggCoords[1],aggCoords[2],aggCoords[3],aggCoords[4],manhattanBridgeCoords[0],manhattanBridgeCoords[1],manhattanBridgeCoords[2],manhattanBridgeCoords[3],manhattanBridgeCoords[4], proc->manhattanDistanceToBridge);
-#endif
-      }
-    }
-
-   TRACE_ERR("Bridge coords(%8.8X): %d %d %d %d %d, %d. iambridge %d\n",bridgeCoords, pers.Network_Config.cnBridge_A,pers.Network_Config.cnBridge_B,pers.Network_Config.cnBridge_C,pers.Network_Config.cnBridge_D,pers.Network_Config.cnBridge_E,0, iambridge);
-
-   /* Allgather the ranks and bridgeCoords to determine the bridge
-      rank and how many ranks belong to each bridge rank*/
-   bridges = (sortstruct *) ADIOI_Malloc(sizeof(sortstruct) * size);
-
-   /* We're going to sort this structure by bridgeCoord:
-    
-   typedef struct
-   {
-      int rank;
-      int bridgeCoord;
-   } sortstruct; 
-    
-   and I want the rank that IS the bridge to sort first, so 
-   OR in '1' on non-bridge ranks that use a bridge coord. 
-   */ 
-
-   /* My input to the collective */
-   bridges[rank].rank = rank;
-   bridges[rank].bridgeCoord = bridgeCoords;
-   if(!iambridge)
-      bridges[rank].bridgeCoord |= 1;  /* I am not bridge, turn on bit */
-
-
-   MPI_Allgather(MPI_IN_PLACE, 2, MPI_INT, bridges, 2, MPI_INT, comm);
-
-   qsort(bridges, size, sizeof(sortstruct), intsort);
-
-   /* Once the list is sorted walk through it to setup bridge
-      info and find bridge ranks, etc. */
-
-   int tempCoords, tempRank, mincompute, maxcompute;
-   tempCoords = bridges[0].bridgeCoord & ~1;
-   tempRank   = bridges[0].rank;
-
-   countPset=1;
-   bridgeIndex = 0; 
-   mincompute = size+1;
-   maxcompute = 1;
-
-   for(i=1; i<size; i++)
-   {
-      if((bridges[i].bridgeCoord  & ~1) == tempCoords) 
-            countPset++; /* same bridge (pset), count it */
-      else /* new bridge found */
-      {
-#ifdef TRACE_ON
-         if(rank == 0) 
-            TRACE_ERR("Bridge set %u, bridge rank %d (%#8.8X) has %d ranks\n",
-                      bridgeIndex, tempRank, tempCoords, countPset);
-#endif
-         if(countPset > maxcompute)
-            maxcompute = countPset;
-         if(countPset < mincompute)
-            mincompute = countPset;
-
-         /* Was this my bridge we finished? */
-         if(tempCoords == bridgeCoords)
-         {
-            /* Am I the bridge rank? */
-            if(tempRank == rank)
-               iambridge = 1;
-            else 
-               iambridge = 0; /* Another rank on my node may have taken over */
-            TRACE_ERR("Rank %u, bridge set %u, bridge rank %d (%#8.8X) has %d ranks, iambridge %u\n",
-                      rank, bridgeIndex, tempRank, tempCoords, countPset,iambridge);
-            bridgerank = tempRank;
-            proc->myIOSize = countPset;
-            proc->ioNodeIndex = bridgeIndex;
-         }
-         /* Setup next bridge */
-         tempCoords = bridges[i].bridgeCoord & ~1;
-         tempRank   = bridges[i].rank;
-         bridgeIndex++;
-         countPset = 1;
-      }
-   }
-   /* Process last bridge */
-
-#ifdef TRACE_ON
-   if(rank == 0) 
-      TRACE_ERR("Bridge set %u, bridge rank %d (%#8.8X) has %d ranks\n",
-                bridgeIndex, tempRank, tempCoords, countPset);
-#endif
-   if(countPset > maxcompute)
-      maxcompute = countPset;
-   if(countPset < mincompute)
-      mincompute = countPset;
-
-   /* Was this my bridge? */
-   if(tempCoords == bridgeCoords)
-   {
-      /* Am I the bridge rank? */
-      if(tempRank == rank)
-         iambridge = 1;
-      else 
-         iambridge = 0; /* Another rank on my node may have taken over */
-      bridgerank = tempRank;
-      proc->myIOSize = countPset;
-      proc->ioNodeIndex = bridgeIndex;
-   }
-   
-   
-   if(rank == 0) 
-   {
-      /* Only rank 0 has a conf structure, fill in stuff as appropriate */
-      conf->ioMinSize = mincompute;
-      conf->ioMaxSize = maxcompute; /* equivalent to pset size */
-      conf->numBridgeRanks = bridgeIndex+1;
-      conf->nProcs = size;
-            
-      conf->nAggrs = n_aggrs;
-      /*    First pass gets nAggrs = -1 */
-      if(conf->nAggrs <=0)
-         conf->nAggrs = gpfsmpio_bg_nagg_pset;
-      if(conf->ioMinSize <= conf->nAggrs)
-        conf->nAggrs = ADIOI_MAX(1,conf->ioMinSize-1); /* not including bridge itself */
-/*      if(conf->nAggrs > conf->numBridgeRanks) 
-         conf->nAggrs = conf->numBridgeRanks; 
-*/
-      conf->aggRatio = 1. * conf->nAggrs / conf->ioMinSize /*virtualPsetSize*/;
-/*    if(conf->aggRatio > 1) conf->aggRatio = 1.; */
-      TRACE_ERR("n_aggrs %zd, conf->nProcs %zu, conf->ioMaxSize %zu, ADIOI_BG_NAGG_PSET_DFLT %zu,conf->numBridgeRanks %zu,conf->nAggrs %zu\n",(size_t)n_aggrs, (size_t)conf->nProcs, (size_t)conf->ioMaxSize, (size_t)ADIOI_BG_NAGG_PSET_DFLT,(size_t)conf->numBridgeRanks,(size_t)conf->nAggrs);
-      TRACE_ERR("Maximum ranks under a bridge rank: %d, minimum: %d, nAggrs: %d, numBridgeRanks: %d pset dflt: %d naggrs: %d ratio: %f\n", maxcompute, mincompute, conf->nAggrs, conf->numBridgeRanks, ADIOI_BG_NAGG_PSET_DFLT, conf->nAggrs, conf->aggRatio);
-   }
-
-   ADIOI_Assert((bridgerank != -1));
-   proc->bridgeRank = bridgerank;
-   proc->iamBridge = iambridge;
-   TRACE_ERR("Rank %d has bridge set index %d (bridge rank: %d) with %d other ranks, ioNodeIndex: %d\n", rank,  proc->ioNodeIndex, bridgerank, proc->myIOSize, proc->ioNodeIndex);
-
-   ADIOI_Free(bridges);
-
-}
-
-void 
-ADIOI_BG_persInfo_free( ADIOI_BG_ConfInfo_t *conf, ADIOI_BG_ProcInfo_t *proc )
-{
-    ADIOI_BG_ConfInfo_free( conf );
-    ADIOI_BG_ProcInfo_free( proc );
-}
diff --git a/3rd-party/romio321/adio/ad_gpfs/bg/ad_bg_pset.h b/3rd-party/romio321/adio/ad_gpfs/bg/ad_bg_pset.h
deleted file mode 100644
index 3f251e8275e..00000000000
--- a/3rd-party/romio321/adio/ad_gpfs/bg/ad_bg_pset.h
+++ /dev/null
@@ -1,80 +0,0 @@
-/* ---------------------------------------------------------------- */
-/* (C)Copyright IBM Corp.  2007, 2008                               */
-/* ---------------------------------------------------------------- */
-/**
- * \file ad_bg_pset.h
- * \brief ???
- */
-
-/* File: ad_bg_pset.h
- *
- * Defines two structures that keep BlueGene PSET specific information and their public interfaces:
- * 	. ADIOI_BG_ProcInfo_t object keeps specific information to each process
- * 	. ADIOI_BG_ConfInfo_t object keeps general information for the whole communicator, only kept
- *	  on process 0.
- */
-
-#ifndef AD_BG_PSET_H_
-#define AD_BG_PSET_H_
-
-
-/* Keeps specific information to each process, will be exchanged among processes */
-typedef struct {
-   int ioNodeIndex; /* similar to psetNum on BGL/BGP */
-   int rank; /* my rank */
-   int ionID;  /* ion id this cn is using */
-/*   int myCoords[5]; */
-   int bridgeRank; /* my bridge node (or proxy) rank */
-   unsigned char threadID; /* unlikely to be useful but better than just padding */
-   unsigned char __cpad[2];
-   int myIOSize;  /* number of ranks sharing my bridge/IO
-      node, i.e. psetsize*/
-   int iamBridge; /* am *I* the bridge rank? */
-   int __ipad[2];
-   unsigned nodeRank; /* torus coords converted to an integer for use with gpfsmpio_bridgeringagg */
-   unsigned numNodesInPartition; /* number of physical nodes in the job partition */
-   unsigned manhattanDistanceToBridge; /* number of hops between this rank and the bridge node */
-} ADIOI_BG_ProcInfo_t __attribute__((aligned(16)));
-
-/* Keeps general information for the whole communicator, only on process 0 */
-typedef struct {
-   int ioMinSize; /* Smallest number of ranks shareing 1 bridge node */
-   int ioMaxSize; /* Largest number of ranks sharing 1 bridge node */
-   /* ioMaxSize will be the "psetsize" */
-   int nAggrs;
-   int numBridgeRanks;
-   /*int virtualPsetSize; ppn * pset size */
-   int nProcs;
-   int cpuIDsize; /* num ppn */
-   float aggRatio;
-
-} ADIOI_BG_ConfInfo_t __attribute__((aligned(16)));
-
-
-#undef MIN
-#define MIN(a,b) (((a)<(b) ? (a) : (b)))
-
-
-
-
-/* public funcs for ADIOI_BG_ProcInfo_t objects */
-    ADIOI_BG_ProcInfo_t * ADIOI_BG_ProcInfo_new();
-    ADIOI_BG_ProcInfo_t * ADIOI_BG_ProcInfo_new_n( int n );
-    void ADIOI_BG_ProcInfo_free( ADIOI_BG_ProcInfo_t *info );
-
-
-/* public funcs for ADIOI_BG_ConfInfo_t objects */
-    ADIOI_BG_ConfInfo_t * ADIOI_BG_ConfInfo_new ();
-    void ADIOI_BG_ConfInfo_free( ADIOI_BG_ConfInfo_t *info );
-
-
-/* public funcs for a pair of ADIOI_BG_ConfInfo_t and ADIOI_BG_ProcInfo_t objects */
-    int BGQ_IO_node_id ();
-    void ADIOI_BG_persInfo_init( ADIOI_BG_ConfInfo_t *conf,
-				  ADIOI_BG_ProcInfo_t *proc,
-				  int s, int r, int n_aggrs, MPI_Comm comm);
-    void ADIOI_BG_persInfo_free( ADIOI_BG_ConfInfo_t *conf,
-				  ADIOI_BG_ProcInfo_t *proc );
-
-
-#endif  /* AD_BG_PSET_H_ */
diff --git a/3rd-party/romio321/adio/ad_gpfs/pe/Makefile.mk b/3rd-party/romio321/adio/ad_gpfs/pe/Makefile.mk
deleted file mode 100644
index 6173bd7fe0a..00000000000
--- a/3rd-party/romio321/adio/ad_gpfs/pe/Makefile.mk
+++ /dev/null
@@ -1,16 +0,0 @@
-## -*- Mode: Makefile; -*-
-## vim: set ft=automake :
-##
-## (C) 2012 by Argonne National Laboratory.
-##     See COPYRIGHT in top-level directory.
-##
-
-if BUILD_AD_PE
-
-noinst_HEADERS +=                                                    \
-    adio/ad_gpfs/pe/ad_pe_aggrs.h
-
-romio_other_sources +=                                               \
-    adio/ad_gpfs/pe/ad_pe_aggrs.c
-
-endif BUILD_AD_PE
diff --git a/3rd-party/romio321/adio/ad_gpfs/pe/ad_pe_aggrs.c b/3rd-party/romio321/adio/ad_gpfs/pe/ad_pe_aggrs.c
deleted file mode 100644
index 8453238a833..00000000000
--- a/3rd-party/romio321/adio/ad_gpfs/pe/ad_pe_aggrs.c
+++ /dev/null
@@ -1,276 +0,0 @@
-/* ---------------------------------------------------------------- */
-/* (C)Copyright IBM Corp.  2007, 2008                               */
-/* ---------------------------------------------------------------- */
-/**
- * \file ad_pe_aggrs.c
- * \brief The externally used function from this file is is declared in ad_pe_aggrs.h
- */
-
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/*
- *   Copyright (C) 1997-2001 University of Chicago.
- *   See COPYRIGHT notice in top-level directory.
- */
-
-/*#define TRACE_ON */
-
-#include "adio.h"
-#include "adio_cb_config_list.h"
-#include "../ad_gpfs.h"
-#include "ad_pe_aggrs.h"
-#include "mpiimpl.h"
-
-#ifdef AGGREGATION_PROFILE
-#include "mpe.h"
-#endif
-
-#ifdef USE_DBG_LOGGING
-  #define AGG_DEBUG 1
-#endif
-
-#ifndef TRACE_ERR
-#  define TRACE_ERR(format...)
-#endif
-
-/*
- * Compute the aggregator-related parameters that are required in 2-phase
- * collective IO of ADIO.
- * The parameters are
- * 	. the number of aggregators (proxies) : fd->hints->cb_nodes
- *	. the ranks of the aggregators :        fd->hints->ranklist
- * If MP_IONODEFILE is defined, POE determines all tasks on every node listed
- * in the node file and defines MP_IOTASKLIST with them, making them all
- * aggregators.  Alternatively, the user can explictly set MP_IOTASKLIST
- * themselves.  The format of the MP_IOTASKLIST is a colon-delimited list of
- * task ids, the first entry being the total number of aggregators, for example
- * to specify 4 aggregators on task ids 0,8,16,24  the value would be:
- * 4:0:8:16:24.  If there is no MP_IONODEFILE, or MP_IOTASKLIST, then the
- * default aggregator selection is 1 task per node for every node of the job -
- * additionally, an environment variable MP_IOAGGR_CNT  can be specified, which
- * defines the total number of aggregators, spread evenly across all the nodes.
- * The romio_cb_nodes and romio_cb_config_list hint user settings are ignored.
- */
-int
-ADIOI_PE_gen_agg_ranklist(ADIO_File fd)
-{
-
-    int numAggs = 0;
-    char *ioTaskList = getenv( "MP_IOTASKLIST" );
-    char *ioAggrCount = getenv("MP_IOAGGR_CNT");
-    int i,j;
-    int inTERcommFlag = 0;
-
-    int myRank,commSize;
-    MPI_Comm_rank(fd->comm, &myRank);
-    MPI_Comm_size(fd->comm, &commSize);
-
-    MPI_Comm_test_inter(fd->comm, &inTERcommFlag);
-    if (inTERcommFlag) {
-      FPRINTF(stderr,"ERROR: ATTENTION: inTERcomms are not supported in MPI-IO - aborting....\n");
-      perror("ADIOI_PE_gen_agg_ranklist:");
-      MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    if (ioTaskList) {
-      int ioTaskListLen = strlen(ioTaskList);
-      int ioTaskListPos = 0;
-      char tmpBuf[8];   /* Big enough for 1M tasks (7 digits task ID). */
-      tmpBuf[7] = '\0';
-      for (i=0; i<7; i++) {
-         tmpBuf[i] = *ioTaskList++;      /* Maximum is 7 digits for 1 million. */
-         ioTaskListPos++;
-         if (*ioTaskList == ':') {       /* If the next char is a ':' ends it. */
-             tmpBuf[i+1] = '\0';
-             break;
-         }
-      }
-      numAggs = atoi(tmpBuf);
-      if (numAggs == 0)
-        FPRINTF(stderr,"ERROR: ATTENTION: Number of aggregators specified in MP_IOTASKLIST set at 0 - default aggregator selection will be used.\n");
-      else if (!((numAggs > 0 ) && (numAggs <= commSize))) {
-        FPRINTF(stderr,"ERROR: ATTENTION: The number of aggregators (%s) specified in MP_IOTASKLIST is outside the communicator task range of %d.\n",tmpBuf,commSize);
-        numAggs = commSize;
-      }
-      fd->hints->ranklist = (int *) ADIOI_Malloc (numAggs * sizeof(int));
-
-      int aggIndex = 0;
-      while (aggIndex < numAggs) {
-         ioTaskList++;                /* Advance past the ':' */
-         ioTaskListPos++;
-         int allDigits=1;
-         for (i=0; i<7; i++) {
-            if (*ioTaskList < '0' || *ioTaskList > '9')
-              allDigits=0;
-            tmpBuf[i] = *ioTaskList++;
-            ioTaskListPos++;
-            if ( (*ioTaskList == ':') || (*ioTaskList == '\0') ) {
-                tmpBuf[i+1] = '\0';
-                break;
-            }
-         }
-         if (allDigits) {
-           int newAggRank = atoi(tmpBuf);
-           if (!((newAggRank >= 0 ) && (newAggRank < commSize))) {
-             FPRINTF(stderr,"ERROR: ATTENTION: The aggregator '%s' specified in MP_IOTASKLIST is not within the communicator task range of 0 to %d  - it will be ignored.\n",tmpBuf,commSize-1);
-           }
-           else {
-             int aggAlreadyAdded = 0;
-             for (i=0;i<aggIndex;i++)
-               if (fd->hints->ranklist[i] == newAggRank) {
-                 aggAlreadyAdded = 1;
-                 break;
-               }
-             if (!aggAlreadyAdded)
-               fd->hints->ranklist[aggIndex++] = newAggRank;
-             else
-               FPRINTF(stderr,"ERROR: ATTENTION: The aggregator '%d' is specified multiple times in MP_IOTASKLIST - duplicates are ignored.\n",newAggRank);
-           }
-         }
-         else {
-           FPRINTF(stderr,"ERROR: ATTENTION: The aggregator '%s' specified in MP_IOTASKLIST is not a valid integer task id  - it will be ignored.\n",tmpBuf);
-         }
-
-         /* At the end check whether the list is shorter than specified. */
-         if (ioTaskListPos == ioTaskListLen) {
-           if (aggIndex == 0) {
-             FPRINTF(stderr,"ERROR: ATTENTION: No aggregators were correctly specified in MP_IOTASKLIST - default aggregator selection will be used.\n");
-             ADIOI_Free(fd->hints->ranklist);
-           }
-           else if (aggIndex < numAggs)
-             FPRINTF(stderr,"ERROR: ATTENTION: %d aggregators were specified in MP_IOTASKLIST but only %d were correctly specified - setting the number of aggregators to %d.\n",numAggs, aggIndex,aggIndex);
-           numAggs = aggIndex;
-         }
-      }
-    }
-    if (numAggs == 0)  {
-      MPID_Comm *mpidCommData;
-
-      MPID_Comm_get_ptr(fd->comm,mpidCommData);
-      int localSize = mpidCommData->local_size;
-
-      // get my node rank
-      int myNodeRank = mpidCommData->intranode_table[mpidCommData->rank];
-
-      int *allNodeRanks = (int *) ADIOI_Malloc (localSize * sizeof(int));
-
-      allNodeRanks[myRank] = myNodeRank;
-      MPI_Allgather(MPI_IN_PLACE, 1, MPI_INT, allNodeRanks, 1, MPI_INT, fd->comm);
-
-#ifdef AGG_DEBUG
-      printf("MPID_Comm data: local_size is %d\nintranode_table entries:\n",mpidCommData->local_size);
-      for (i=0;i<localSize;i++) {
-        printf("%d ",mpidCommData->intranode_table[i]);
-      }
-      printf("\ninternode_table entries:\n");
-      for (i=0;i<localSize;i++) {
-        printf("%d ",mpidCommData->internode_table[i]);
-      }
-      printf("\n");
-
-      printf("\nallNodeRanks entries:\n");
-      for (i=0;i<localSize;i++) {
-        printf("%d ",allNodeRanks[i]);
-      }
-      printf("\n");
-
-#endif
-
-      if (ioAggrCount) {
-        int cntType = -1;
-
-        if ( strcasecmp(ioAggrCount, "ALL") ) {
-           if ( (cntType = atoi(ioAggrCount)) <= 0 ) {
-              /* Input is other non-digit or less than 1 the  assume */
-              /* 1 aggregator per node.  Note: atoi(-1) reutns -1.   */
-              /* No warning message given here -- done earlier.      */
-              cntType = -1;
-           }
-        }
-        else {
-           /* ALL is specified set aggr count to localSize */
-           cntType = -2;
-        }
-        switch(cntType) {
-           case -1:
-              /* 1 aggr/node case */
-            {
-             int rankListIndex = 0;
-             fd->hints->ranklist = (int *) ADIOI_Malloc (localSize * sizeof(int));
-             for (i=0;i<localSize;i++) {
-               if (allNodeRanks[i] == 0) {
-                 fd->hints->ranklist[rankListIndex++] = i;
-                 numAggs++;
-               }
-             }
-            }
-              break;
-           case -2:
-              /* ALL tasks case */
-             fd->hints->ranklist = (int *) ADIOI_Malloc (localSize * sizeof(int));
-             for (i=0;i<localSize;i++) {
-               fd->hints->ranklist[i] = i;
-               numAggs++;
-             }
-              break;
-           default:
-              /* Specific aggr count case -- MUST be less than localSize, otherwise set to localSize */
-             if (cntType > localSize)
-               cntType = localSize;
-
-             numAggs = cntType;
-             // Round-robin thru allNodeRanks - pick the 0's, then the 1's, etc
-             int currentNodeRank = 0;  // node rank currently being selected as aggregator
-             int rankListIndex = 0;
-             int currentAllNodeIndex = 0;
-
-             fd->hints->ranklist = (int *) ADIOI_Malloc (numAggs * sizeof(int));
-
-             while (rankListIndex < numAggs) {
-               int foundEntry = 0;
-               while (!foundEntry && (currentAllNodeIndex < localSize)) {
-                 if (allNodeRanks[currentAllNodeIndex] == currentNodeRank) {
-                   fd->hints->ranklist[rankListIndex++] = currentAllNodeIndex;
-                   foundEntry = 1;
-                 }
-                 currentAllNodeIndex++;
-               }
-               if (!foundEntry) {
-                 currentNodeRank++;
-                 currentAllNodeIndex = 0;
-               }
-             } // while
-          break;
-        } // switch(cntType)
-      } // if (ioAggrCount)
-
-      else { // default is 1 aggregator per node
-        // take the 0 entries from allNodeRanks
-        int rankListIndex = 0;
-        fd->hints->ranklist = (int *) ADIOI_Malloc (localSize * sizeof(int));
-        for (i=0;i<localSize;i++) {
-          if (allNodeRanks[i] == 0) {
-            fd->hints->ranklist[rankListIndex++] = i;
-            numAggs++;
-          }
-        }
-      }
-
-      ADIOI_Free(allNodeRanks);
-
-    }
-
-    if ( getenv("MP_I_SHOW_AGGRS") ) {
-      if (myRank == 0) {
-        printf("Agg rank list of %d generated:\n", numAggs);
-        for (i=0;i<numAggs;i++) {
-          printf("%d ",fd->hints->ranklist[i]);
-        }
-        printf("\n");
-      }
-    }
-
-    fd->hints->cb_nodes = numAggs;
-
-    return 0;
-}
-
diff --git a/3rd-party/romio321/adio/ad_gpfs/pe/ad_pe_aggrs.h b/3rd-party/romio321/adio/ad_gpfs/pe/ad_pe_aggrs.h
deleted file mode 100644
index f779d182a5d..00000000000
--- a/3rd-party/romio321/adio/ad_gpfs/pe/ad_pe_aggrs.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/* ---------------------------------------------------------------- */
-/* (C)Copyright IBM Corp.  2007, 2008                               */
-/* ---------------------------------------------------------------- */
-/**
- * \file ad_pe_aggrs.h
- * \brief ???
- */
-
-/*
- *
- * Declares functions specific for the PE platform within the GPFS
- * parallel I/O solution.  For now simply processes the MP_IOTASKLIST
- * env var.
- *
- */
-
-#ifndef AD_PE_AGGRS_H_
-#define AD_PE_AGGRS_H_
-
-#include "adio.h"
-#include <sys/stat.h>
-
-#if !defined(GPFS_SUPER_MAGIC)
-  #define GPFS_SUPER_MAGIC (0x47504653)
-#endif
-
-    /* generate a list of I/O aggregators following a methodology specific for PE */
-    int ADIOI_PE_gen_agg_ranklist(ADIO_File fd);
-
-#endif  /* AD_PE_AGGRS_H_ */
diff --git a/3rd-party/romio321/adio/ad_gridftp/Makefile.mk b/3rd-party/romio321/adio/ad_gridftp/Makefile.mk
deleted file mode 100644
index 2cd83a2ad28..00000000000
--- a/3rd-party/romio321/adio/ad_gridftp/Makefile.mk
+++ /dev/null
@@ -1,27 +0,0 @@
-## -*- Mode: Makefile; -*-
-## vim: set ft=automake :
-##
-## (C) 2011 by Argonne National Laboratory.
-##     See COPYRIGHT in top-level directory.
-##
-
-if BUILD_AD_GRIDFTP
-
-noinst_HEADERS += adio/ad_gridftp/ad_gridftp.h
-
-romio_other_sources +=                    \
-    adio/ad_gridftp/ad_gridftp_close.c    \
-    adio/ad_gridftp/ad_gridftp_open.c     \
-    adio/ad_gridftp/ad_gridftp_read.c     \
-    adio/ad_gridftp/ad_gridftp_write.c    \
-    adio/ad_gridftp/ad_gridftp_fcntl.c    \
-    adio/ad_gridftp/ad_gridftp_flush.c    \
-    adio/ad_gridftp/ad_gridftp_resize.c   \
-    adio/ad_gridftp/ad_gridftp_hints.c    \
-    adio/ad_gridftp/ad_gridftp_delete.c   \
-    adio/ad_gridftp/ad_gridftp.c          \
-    adio/ad_gridftp/globus_routines.c     \
-    adio/ad_gridftp/ad_gridftp_features.c
-
-endif BUILD_AD_GRIDFTP
-
diff --git a/3rd-party/romio321/adio/ad_gridftp/ad_gridftp.c b/3rd-party/romio321/adio/ad_gridftp/ad_gridftp.c
deleted file mode 100644
index f3767dcda21..00000000000
--- a/3rd-party/romio321/adio/ad_gridftp/ad_gridftp.c
+++ /dev/null
@@ -1,39 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2003 University of Chicago, Ohio Supercomputer Center. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_gridftp.h"
-
-/* adioi.h has the ADIOI_Fns_struct define */
-#include "adioi.h"
-
-struct ADIOI_Fns_struct ADIO_GRIDFTP_operations = {
-    ADIOI_GRIDFTP_Open, /* Open */
-    ADIOI_GRIDFTP_ReadContig, /* ReadContig */
-    ADIOI_GRIDFTP_WriteContig, /* WriteContig */
-    ADIOI_GEN_ReadStridedColl, /* ReadStridedColl */
-    ADIOI_GEN_WriteStridedColl, /* WriteStridedColl */
-    ADIOI_GEN_SeekIndividual, /* SeekIndividual */
-    ADIOI_GRIDFTP_Fcntl, /* Fcntl */
-    ADIOI_GRIDFTP_SetInfo, /* SetInfo */
-    ADIOI_GRIDFTP_ReadStrided, /* ReadStrided */
-    ADIOI_GRIDFTP_WriteStrided, /* WriteStrided */
-    ADIOI_GRIDFTP_Close, /* Close */
-    ADIOI_FAKE_IreadContig, /* IreadContig */
-    ADIOI_FAKE_IwriteContig, /* IwriteContig */
-    ADIOI_FAKE_IODone, /* ReadDone */
-    ADIOI_FAKE_IODone, /* WriteDone */
-    ADIOI_FAKE_IOComplete, /* ReadComplete */
-    ADIOI_FAKE_IOComplete, /* WriteComplete */
-    ADIOI_FAKE_IreadStrided, /* IreadStrided */
-    ADIOI_FAKE_IwriteStrided, /* IwriteStrided */
-    ADIOI_GRIDFTP_Flush, /* Flush */
-    ADIOI_GRIDFTP_Resize, /* Resize */
-    ADIOI_GRIDFTP_Delete, /* Delete */
-    ADIOI_GRIDFTP_Feature, /* Features */
-    ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
-    ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
-};
diff --git a/3rd-party/romio321/adio/ad_gridftp/ad_gridftp.h b/3rd-party/romio321/adio/ad_gridftp/ad_gridftp.h
deleted file mode 100644
index 0b94c780eaa..00000000000
--- a/3rd-party/romio321/adio/ad_gridftp/ad_gridftp.h
+++ /dev/null
@@ -1,96 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 2003 University of Chicago, Ohio Supercomputer Center. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#ifndef AD_GRIDFTP_INCLUDE
-#define AD_GRIDFTP_INCLUDE
-
-#include <unistd.h>
-#include <sys/types.h>
-#include <sys/uio.h>
-#include <fcntl.h>
-#include "adio.h"
-#include <globus_ftp_client.h>
-
-/* These live in globus_routines.c */
-extern int num_gridftp_handles;
-#ifndef ADIO_GRIDFTP_HANDLES_MAX
-#define ADIO_GRIDFTP_HANDLES_MAX 200
-#endif /* ! ADIO_GRIDFTP_HANDLES_MAX */
-extern globus_ftp_client_handle_t gridftp_fh[ADIO_GRIDFTP_HANDLES_MAX];
-extern globus_ftp_client_operationattr_t oattr[ADIO_GRIDFTP_HANDLES_MAX];
-
-
-/* TODO: weed out the now-unused prototypes  */
-void ADIOI_GRIDFTP_Open(ADIO_File fd, int *error_code);
-void ADIOI_GRIDFTP_Close(ADIO_File fd, int *error_code);
-void ADIOI_GRIDFTP_ReadContig(ADIO_File fd, void *buf, int count, 
-			     MPI_Datatype datatype, int file_ptr_type,
-			     ADIO_Offset offset, ADIO_Status *status, int
-			     *error_code);
-void ADIOI_GRIDFTP_WriteContig(ADIO_File fd, void *buf, int count, 
-			      MPI_Datatype datatype, int file_ptr_type,
-			      ADIO_Offset offset, ADIO_Status *status, int
-			      *error_code);   
-void ADIOI_GRIDFTP_IwriteContig(ADIO_File fd, void *buf, int count, 
-			       MPI_Datatype datatype, int file_ptr_type,
-			       ADIO_Offset offset, ADIO_Request *request, int
-			       *error_code);   
-void ADIOI_GRIDFTP_IreadContig(ADIO_File fd, void *buf, int count, 
-			      MPI_Datatype datatype, int file_ptr_type,
-			      ADIO_Offset offset, ADIO_Request *request, int
-			      *error_code);   
-int ADIOI_GRIDFTP_ReadDone(ADIO_Request *request, ADIO_Status *status, int
-			  *error_code);
-int ADIOI_GRIDFTP_WriteDone(ADIO_Request *request, ADIO_Status *status, int
-			   *error_code);
-void ADIOI_GRIDFTP_ReadComplete(ADIO_Request *request, ADIO_Status *status, int
-			       *error_code); 
-void ADIOI_GRIDFTP_WriteComplete(ADIO_Request *request, ADIO_Status *status,
-				int *error_code); 
-void ADIOI_GRIDFTP_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct, 
-			int *error_code); 
-void ADIOI_GRIDFTP_WriteStrided(ADIO_File fd, void *buf, int count,
-			       MPI_Datatype datatype, int file_ptr_type,
-			       ADIO_Offset offset, ADIO_Status *status,
-			       int *error_code);
-void ADIOI_GRIDFTP_ReadStrided(ADIO_File fd, void *buf, int count,
-			      MPI_Datatype datatype, int file_ptr_type,
-			      ADIO_Offset offset, ADIO_Status *status, int
-			      *error_code);
-void ADIOI_GRIDFTP_WriteStridedColl(ADIO_File fd, void *buf, int count,
-				   MPI_Datatype datatype, int file_ptr_type,
-				   ADIO_Offset offset, ADIO_Status *status, int
-				   *error_code);
-void ADIOI_GRIDFTP_ReadStridedColl(ADIO_File fd, void *buf, int count,
-				  MPI_Datatype datatype, int file_ptr_type,
-				  ADIO_Offset offset, ADIO_Status *status, int
-				  *error_code);
-void ADIOI_GRIDFTP_IreadStrided(ADIO_File fd, void *buf, int count,
-			       MPI_Datatype datatype, int file_ptr_type,
-			       ADIO_Offset offset, ADIO_Request *request, int
-			       *error_code);
-void ADIOI_GRIDFTP_IwriteStrided(ADIO_File fd, void *buf, int count,
-				MPI_Datatype datatype, int file_ptr_type,
-				ADIO_Offset offset, ADIO_Request *request, int
-				*error_code);
-void ADIOI_GRIDFTP_Flush(ADIO_File fd, int *error_code);
-void ADIOI_GRIDFTP_Resize(ADIO_File fd, ADIO_Offset size, int *error_code);
-void ADIOI_GRIDFTP_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code);
-void ADIOI_GRIDFTP_Get_shared_fp(ADIO_File fd, int size, 
-				ADIO_Offset *shared_fp, 
-				int *error_code);
-void ADIOI_GRIDFTP_Set_shared_fp(ADIO_File fd, ADIO_Offset offset, 
-				int *error_code);
-void ADIOI_GRIDFTP_Delete(char *filename, int *error_code);
-
-void globus_err_handler(const char *routine, const char *caller,
-			globus_result_t result);
-
-#endif
-
-
-
-
diff --git a/3rd-party/romio321/adio/ad_gridftp/ad_gridftp_close.c b/3rd-party/romio321/adio/ad_gridftp/ad_gridftp_close.c
deleted file mode 100644
index c1693d65ce0..00000000000
--- a/3rd-party/romio321/adio/ad_gridftp/ad_gridftp_close.c
+++ /dev/null
@@ -1,50 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2003 University of Chicago, Ohio Supercomputer Center. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_gridftp.h"
-#include "adioi.h"
-
-void ADIOI_GRIDFTP_Close(ADIO_File fd, int *error_code)
-{
-    int err;
-    static char myname[]="ADIOI_GRIDFTP_Close";
-
-    globus_result_t result;
-
-    MPI_Barrier(fd->comm);
-
-    /* Destroy the ftp handle  and opattr */
-    result = globus_ftp_client_operationattr_destroy(&(oattr[fd->fd_sys]));
-    if (result != GLOBUS_SUCCESS )
-    {
-	    globus_err_handler("globus_ftp_client_operationattr_destroy",
-		    myname,result);
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-		    myname, __LINE__, MPI_ERR_IO,
-		    "**io",
-		    "**io %s",globus_object_printable_to_string(globus_error_get(result)));
-	    return;
-    }
-    result=globus_ftp_client_handle_destroy(&(gridftp_fh[fd->fd_sys]));
-    if (result != GLOBUS_SUCCESS )
-    {
-	    globus_err_handler("globus_ftp_client_handle_destroy",
-		    myname,result);
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-		    myname, __LINE__, MPI_ERR_IO,
-		    "**io",
-		    "**io %s", globus_object_printable_to_string(globus_error_get(result)));
-	    return;
-    }
-
-    fd->fd_sys = -1;
-    fd->fp_ind=0;
-    fd->fp_sys_posn=0;
-    num_gridftp_handles--;
-
-    *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_gridftp/ad_gridftp_delete.c b/3rd-party/romio321/adio/ad_gridftp/ad_gridftp_delete.c
deleted file mode 100644
index 54eb7144295..00000000000
--- a/3rd-party/romio321/adio/ad_gridftp/ad_gridftp_delete.c
+++ /dev/null
@@ -1,95 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2003 University of Chicago, Ohio Supercomputer Center. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_gridftp.h"
-#include "adioi.h"
-
-static globus_mutex_t lock;
-static globus_cond_t cond;
-static globus_bool_t delete_done, delete_success;
-static void delete_cb(void *myarg, globus_ftp_client_handle_t *handle, globus_object_t *error)
-{
-    
-    if (error)
-	{
-	    FPRINTF(stderr, "%s\n", globus_object_printable_to_string(error));
-	}
-    else
-	{
-	    delete_success=GLOBUS_TRUE;
-	}
-    delete_done=GLOBUS_TRUE;
-}
-
-void ADIOI_GRIDFTP_Delete(char *filename, int *error_code)
-{
-    char myname[]="ADIOI_GRIDFTP_Delete";
-    int myrank, nprocs;
-    globus_ftp_client_handle_t handle;
-    globus_result_t result;
-
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
-    MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
-
-    globus_module_activate(GLOBUS_FTP_CLIENT_MODULE);
-    result=globus_ftp_client_handle_init(&handle,GLOBUS_NULL);
-   
-    if (result != GLOBUS_SUCCESS )
-    {
-	    globus_err_handler("globus_ftp_client_handle_init",myname,result);
-	    *error_code= MPIO_Err_create_code(MPI_SUCCESS,
-			    MPIR_ERR_RECOVERABLE,
-			    myname, __LINE__,
-			    MPI_ERR_IO,
-			    "**io", "**io %s", 
-			    globus_object_printable_to_string(globus_error_get(result)));
-	    return; 
-    }
-    
-    delete_done=GLOBUS_FALSE;
-    delete_success=GLOBUS_FALSE;
-    result=globus_ftp_client_delete(&handle,filename,GLOBUS_NULL,delete_cb,GLOBUS_NULL);
-    if (result != GLOBUS_SUCCESS )
-	{
-	    globus_err_handler("globus_ftp_client_delete",myname,result);
-	    *error_code= MPIO_Err_create_code(MPI_SUCCESS,
-			    MPIR_ERR_RECOVERABLE,
-			    myname, __LINE__,
-			    MPI_ERR_IO,
-			    "**io", "**io %s",
-			    globus_object_printable_to_string(globus_error_get(result)));
-	    return;
-	}
-    globus_mutex_lock(&lock);
-    while ( delete_done!=GLOBUS_TRUE )
-	globus_cond_wait(&cond,&lock);
-    globus_mutex_unlock(&lock);
-    result=globus_ftp_client_handle_destroy(&handle);
-    if (result != GLOBUS_SUCCESS )
-	{
-	    globus_err_handler("globus_ftp_client_handle_destroy",myname,result);
-	    *error_code= MPIO_Err_create_code(MPI_SUCCESS,
-			    MPIR_ERR_RECOVERABLE,
-			    myname, __LINE__,
-			    MPI_ERR_IO,
-			    "**io", "**io %s", 
-			    globus_object_printable_to_string(globus_error_get(result)));
-	    return;
-	}
-
-    if ( delete_success!=GLOBUS_TRUE )
-	{
-	    *error_code= MPIO_Err_create_code(MPI_SUCCESS,
-			    MPIR_ERR_RECOVERABLE,
-			    myname, __LINE__,
-			    MPI_ERR_IO,
-			    "**io", "**io %s", 
-			    globus_object_printable_to_string(globus_error_get(result)));
-	}
-}
diff --git a/3rd-party/romio321/adio/ad_gridftp/ad_gridftp_fcntl.c b/3rd-party/romio321/adio/ad_gridftp/ad_gridftp_fcntl.c
deleted file mode 100644
index dd9cb5ee098..00000000000
--- a/3rd-party/romio321/adio/ad_gridftp/ad_gridftp_fcntl.c
+++ /dev/null
@@ -1,91 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2003 University of Chicago, Ohio Supercomputer Center. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_gridftp.h"
-#include "adioi.h"
-#include "adio_extern.h"
-
-globus_mutex_t fcntl_size_lock;
-globus_cond_t fcntl_size_cond;
-globus_bool_t fcntl_size_done;
-
-void fcntl_size_cb(void *myargs, globus_ftp_client_handle_t *handle,
-		  globus_object_t *error)
-{
-    if (error)
-	{
-	    FPRINTF(stderr, "%s\n", globus_object_printable_to_string(error));
-	}
-    globus_mutex_lock(&fcntl_size_lock);
-    fcntl_size_done=GLOBUS_TRUE;
-    globus_cond_signal(&fcntl_size_cond);
-    globus_mutex_unlock(&fcntl_size_lock);
-}
-
-void ADIOI_GRIDFTP_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct, 
-			int *error_code)
-{
-    MPI_Datatype copy_etype, copy_filetype;
-    int combiner, i, j, k, filetype_is_contig, err;
-    ADIOI_Flatlist_node *flat_file;
-    char myname[]="ADIOI_GRIDFTP_Fcntl";
-
-    int myrank, nprocs;
-
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-
-    switch(flag) {
-    case ADIO_FCNTL_GET_FSIZE:
-	{
-	    globus_result_t result;
-	    globus_off_t fsize=0;
-	    
-	    globus_mutex_init(&fcntl_size_lock,GLOBUS_NULL);
-	    globus_cond_init(&fcntl_size_cond,GLOBUS_NULL);
-	    fcntl_size_done=GLOBUS_FALSE;
-	    if ( (result=globus_ftp_client_size(&(gridftp_fh[fd->fd_sys]),
-						fd->filename,
-						&(oattr[fd->fd_sys]),
-						&(fsize),
-						fcntl_size_cb,
-						GLOBUS_NULL))!=GLOBUS_SUCCESS )
-		{
-		    globus_err_handler("globus_ftp_client_size",myname,result);
-		    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-				    MPIR_ERR_RECOVERABLE,
-                                    myname, __LINE__, MPI_ERR_IO,
-				    "**io", "**io %s", 
-				    globus_object_printable_to_string(globus_error_get(result)));
-		    return;
-		}
-	    globus_mutex_lock(&fcntl_size_lock);
-	    while ( fcntl_size_done!=GLOBUS_TRUE )
-		globus_cond_wait(&fcntl_size_lock,&fcntl_size_cond);
-	    globus_mutex_unlock(&fcntl_size_lock);
-	    globus_mutex_destroy(&fcntl_size_lock);
-	    globus_cond_destroy(&fcntl_size_cond);
-	    fcntl_struct->fsize=fsize;
-	}
-	*error_code = MPI_SUCCESS;
-	break;
-
-    case ADIO_FCNTL_SET_DISKSPACE:
-	ADIOI_GEN_Prealloc(fd, fcntl_struct->diskspace, error_code);
-	break;
-
-    case ADIO_FCNTL_SET_ATOMICITY:
-    default:
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-			MPIR_ERR_RECOVERABLE,
-			myname, __LINE__,
-			MPI_ERR_ARG,
-			"**flag", "**flag %d", flag);
-    }
-}
diff --git a/3rd-party/romio321/adio/ad_gridftp/ad_gridftp_features.c b/3rd-party/romio321/adio/ad_gridftp/ad_gridftp_features.c
deleted file mode 100644
index cbdc39586f8..00000000000
--- a/3rd-party/romio321/adio/ad_gridftp/ad_gridftp_features.c
+++ /dev/null
@@ -1,18 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *
- *  (C) 2008 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-int ADIOI_GRIDFTP_Feature (ADIO_File fd, int flag)
-{
-	switch(flag) {
-		case ADIO_SCALABLE_OPEN:
-		case ADIO_SHARED_FP:
-		case ADIO_LOCKS:
-		case ADIO_SEQUENTIAL:
-		case ADIO_DATA_SIEVING_WRITES:
-		default:
-			return 0;
-	}
-}
diff --git a/3rd-party/romio321/adio/ad_gridftp/ad_gridftp_flush.c b/3rd-party/romio321/adio/ad_gridftp/ad_gridftp_flush.c
deleted file mode 100644
index 795341e8889..00000000000
--- a/3rd-party/romio321/adio/ad_gridftp/ad_gridftp_flush.c
+++ /dev/null
@@ -1,19 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2003 University of Chicago, Ohio Supercomputer Center. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_gridftp.h"
-#include "adioi.h"
-
-/* GridFTP doesn't give you a way to cache writes on the client side, so
-   is essentially a no-op */
-/* if there is a mechanism where we can ask the server to flush data to disk we
- * should do it here.  I'll leave that up to Troy */
-
-void ADIOI_GRIDFTP_Flush(ADIO_File fd, int *error_code)
-{
-	return;
-}
diff --git a/3rd-party/romio321/adio/ad_gridftp/ad_gridftp_hints.c b/3rd-party/romio321/adio/ad_gridftp/ad_gridftp_hints.c
deleted file mode 100644
index c0b0a40ebb2..00000000000
--- a/3rd-party/romio321/adio/ad_gridftp/ad_gridftp_hints.c
+++ /dev/null
@@ -1,68 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2003 University of Chicago, Ohio Supercomputer Center. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-/* 
-
-Valid hints for ftp:// and gsiftp:// URLs (aside from the std. ones):
-
-  ftp_control_mode   extended|block|stream|compressed
-                     (default extended for gsiftp:// URLs and stream for ftp:// URLs)
-
-  parallelism        integer number of simultaneous threads connecting to
-                     ftp server (default 1)
-
-  striped_ftp        true|false or enable|disable; enables gsiftp striped data transfer
-
-  tcp_buffer         integer size of tcp stream buffers in bytes
-
-  transfer_type      ascii or binary (default binary)  
-
-These *must* be specified at open time currently.
-*/
-
-#include "ad_gridftp.h"
-#include "adioi.h"
-
-void ADIOI_GRIDFTP_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code)
-{
-    
-    if (!(fd->info))
-	{
-	    if ( users_info==MPI_INFO_NULL )
-		{
-		    /* This must be part of the open call. */ 
-		    MPI_Info_create(&(fd->info));
-		}
-	    else
-		{
-		    MPI_Info_dup(users_info,&(fd->info));
-		}
-	}
-    else
-	{
-	    int i,nkeys,valuelen,flag;
-	    char key[MPI_MAX_INFO_KEY], value[MPI_MAX_INFO_VAL];
-	    
-	    if ( users_info!=MPI_INFO_NULL )
-		{
-		    MPI_Info_get_nkeys(users_info,&nkeys);
-		    for (i=0;i<nkeys;i++)
-			{
-			    MPI_Info_get_nthkey(users_info,i,key);
-			    MPI_Info_get_valuelen(users_info,key,&valuelen,&flag);
-			    if (flag)
-				{
-				    ADIOI_Info_get(users_info,key,valuelen,value,&flag);
-				    if (flag) ADIOI_Info_set(fd->info,key,value);
-				}
-			}
-		}
-	}
-    
-    /* let the generic ROMIO and MPI-I/O stuff happen... */
-    ADIOI_GEN_SetInfo(fd, users_info, error_code); 
-}
diff --git a/3rd-party/romio321/adio/ad_gridftp/ad_gridftp_open.c b/3rd-party/romio321/adio/ad_gridftp/ad_gridftp_open.c
deleted file mode 100644
index 45aab921051..00000000000
--- a/3rd-party/romio321/adio/ad_gridftp/ad_gridftp_open.c
+++ /dev/null
@@ -1,343 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 2003 University of Chicago, Ohio Supercomputer Center.
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_gridftp.h"
-#include "adioi.h"
-
-static globus_mutex_t lock;
-static globus_cond_t cond;
-
-static globus_bool_t file_exists,exists_done;
-static void exists_cb(void *myargs, globus_ftp_client_handle_t *handle, globus_object_t *error)
-{    
-    if (error)
-	{
-	    FPRINTF(stderr, "%s\n", globus_object_printable_to_string(error));
-	}
-    else
-	{
-	    file_exists=GLOBUS_TRUE;
-	}
-    exists_done=GLOBUS_TRUE;
-}
-
-static globus_bool_t touch_ctl_done;
-static void touch_ctl_cb(void *myargs, globus_ftp_client_handle_t *handle, globus_object_t *error)
-{
-    if (error)
-	{
-	    FPRINTF(stderr, "%s\n", globus_object_printable_to_string(error));
-	}
-    globus_mutex_lock(&lock);
-    touch_ctl_done=GLOBUS_TRUE;
-    globus_cond_signal(&cond);
-    globus_mutex_unlock(&lock);
-}
-
-static void touch_data_cb(void *myargs, globus_ftp_client_handle_t *handle, globus_object_t *error,
-			  globus_byte_t *buffer, globus_size_t length, globus_off_t offset,
-			  globus_bool_t eof)
-{
-    if (error)
-	FPRINTF(stderr, "%s\n", globus_object_printable_to_string(error));
-    globus_ftp_client_register_read(handle,buffer,length,touch_data_cb,myargs);
-    return;
-}
-
-void ADIOI_GRIDFTP_Open(ADIO_File fd, int *error_code)
-{
-    static char myname[]="ADIOI_GRIDFTP_Open";
-    int myrank, nprocs, keyfound;
-    char hintval[MPI_MAX_INFO_VAL+1];
-    globus_ftp_client_handleattr_t hattr;
-    globus_result_t result;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-
-    /* activate Globus ftp client module -- can be called multiple times, so
-       it's safest to call once per file/connection */
-    globus_module_activate(GLOBUS_FTP_CLIENT_MODULE);
-    fd->fd_sys = num_gridftp_handles;
-    /* No shared file pointers for now */
-    fd->shared_fp_fname = NULL;
-    *error_code = MPI_SUCCESS;
-
-    /* Access modes here mean something very different here than they
-       would on a "real" filesystem...  As a result, the amode and hint
-       processing here is intermingled and a little weird because many
-       of them have to do with the connection rather than the file itself.
-       The thing that sucks about this is that read and write ops will
-       have to check themselves if the file is being accessed rdonly, rdwr,
-       or wronly.
-       */
-    result=globus_ftp_client_handleattr_init(&hattr);
-    if ( result != GLOBUS_SUCCESS )
-	{
-	    
-
-	    globus_err_handler("globus_ftp_client_handleattr_init",
-			       myname,result);
-	    fd->fd_sys = -1;
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-		    myname, __LINE__, MPI_ERR_IO,
-		    "**io",
-		    "**io %s", globus_object_printable_to_string(globus_error_get(result)));
-	    return;
-	}
-    result = globus_ftp_client_operationattr_init(&(oattr[fd->fd_sys]));
-    if ( result != GLOBUS_SUCCESS )
-	{
-	    globus_err_handler("globus_ftp_client_operationattr_init",
-			       myname,result);
-	    fd->fd_sys = -1;
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-		    myname, __LINE__, MPI_ERR_IO,
-		    "**io",
-		    "**io %s", globus_object_printable_to_string(globus_error_get(result)));
-	    return;
-	}
-
-
-    /* Always use connection caching unless told otherwise */
-    result=globus_ftp_client_handleattr_set_cache_all(&hattr,GLOBUS_TRUE);
-    if ( result !=GLOBUS_SUCCESS )
-	globus_err_handler("globus_ftp_client_handleattr_set_cache_all",myname,result);
-
-    /* Assume that it's safe to cache a file if it's read-only */
-    if ( (fd->access_mode&ADIO_RDONLY) &&
-	 (result=globus_ftp_client_handleattr_add_cached_url(&hattr,fd->filename))!=GLOBUS_SUCCESS )
-	globus_err_handler("globus_ftp_client_handleattr_add_cached_url",myname,result);
-
-    /* Since we're (almost by definition) doing things that FTP S (stream)
-       control mode can't handle, default to E (extended block) control mode
-       for gsiftp:// URLs.  ftp:// URLs use standard stream control mode 
-       by default.  This behavior can be overridden by the ftp_control_mode
-       hint. */
-
-    /*
-    if ( !strncmp(fd->filename,"gsiftp:",7) && 
-	 (result=globus_ftp_client_operationattr_set_mode(&(oattr[fd->fd_sys]),GLOBUS_FTP_CONTROL_MODE_EXTENDED_BLOCK))!=GLOBUS_SUCCESS )
-	globus_err_handler("globus_ftp_client_operationattr_set_mode",myname,result);
-    else if ( !strncmp(fd->filename,"ftp:",4) && 
-	      (result=globus_ftp_client_operationattr_set_mode(&(oattr[fd->fd_sys]),GLOBUS_FTP_CONTROL_MODE_STREAM))!=GLOBUS_SUCCESS )
-	globus_err_handler("globus_ftp_client_operationattr_set_mode",myname,result);
-    */
-
-    /* Set append mode if necessary */
-    if ( (fd->access_mode&ADIO_APPEND) && 
-	 ((result=globus_ftp_client_operationattr_set_append(&(oattr[fd->fd_sys]),GLOBUS_TRUE))!=GLOBUS_SUCCESS) )
-	globus_err_handler("globus_ftp_client_operationattr_set_append",myname,result);
-
-    /* Other hint and amode processing that would affect hattr and/or 
-       oattr[] (eg. parallelism, striping, etc.) goes here */
-    if ( fd->info!=MPI_INFO_NULL )
-	{
-	    ADIOI_Info_get(fd->info,"ftp_control_mode",MPI_MAX_INFO_VAL,hintval,&keyfound);
-	    if ( keyfound )
-		{
-		    if ( ( !strcmp(hintval,"extended") || !strcmp(hintval,"extended_block") ) && 
-			 (result=globus_ftp_client_operationattr_set_mode(&(oattr[fd->fd_sys]),GLOBUS_FTP_CONTROL_MODE_EXTENDED_BLOCK))!=GLOBUS_SUCCESS )
-			globus_err_handler("globus_ftp_client_operationattr_set_mode",myname,result);
-		    else if ( !strcmp(hintval,"block") && 
-			      (result=globus_ftp_client_operationattr_set_mode(&(oattr[fd->fd_sys]),GLOBUS_FTP_CONTROL_MODE_BLOCK))!=GLOBUS_SUCCESS )
-			globus_err_handler("globus_ftp_client_operationattr_set_mode",myname,result);
-		    else if ( !strcmp(hintval,"compressed") && 
-			      (result=globus_ftp_client_operationattr_set_mode(&(oattr[fd->fd_sys]),GLOBUS_FTP_CONTROL_MODE_COMPRESSED))!=GLOBUS_SUCCESS )
-			globus_err_handler("globus_ftp_client_operationattr_set_mode",myname,result);
-		    else if ( !strcmp(hintval,"stream") && 
-			      (result=globus_ftp_client_operationattr_set_mode(&(oattr[fd->fd_sys]),GLOBUS_FTP_CONTROL_MODE_STREAM))!=GLOBUS_SUCCESS )
-			globus_err_handler("globus_ftp_client_operationattr_set_mode",myname,result);
-		}
-
-	    ADIOI_Info_get(fd->info,"parallelism",MPI_MAX_INFO_VAL,hintval,&keyfound);
-	    if ( keyfound )
-		{
-		    int nftpthreads;
-		    
-		    if ( sscanf(hintval,"%d",&nftpthreads)==1 )
-			{
-			    globus_ftp_control_parallelism_t parallelism;
-
-			    parallelism.mode = GLOBUS_FTP_CONTROL_PARALLELISM_FIXED;
-			    parallelism.fixed.size = nftpthreads;
-			    if ( (result=globus_ftp_client_operationattr_set_parallelism(&(oattr[fd->fd_sys]),
-											 &parallelism))!=GLOBUS_SUCCESS )
-				globus_err_handler("globus_ftp_client_operationattr_set_parallelism",myname,result);
-			}
-		}
-
-	    ADIOI_Info_get(fd->info,"striped_ftp",MPI_MAX_INFO_VAL,hintval,&keyfound);
-	    if ( keyfound )
-		{
-		    /* if set to "true" or "enable", set up round-robin block layout */
-		    if ( !strncmp("true",hintval,4) || !strncmp("TRUE",hintval,4) ||
-			 !strncmp("enable",hintval,4) || !strncmp("ENABLE",hintval,4) )
-			{
-			    ADIOI_Info_get(fd->info,"striping_factor",MPI_MAX_INFO_VAL,hintval,&keyfound);
-			    if ( keyfound )
-				{
-				    int striping_factor;
-
-				    if ( sscanf(hintval,"%d",&striping_factor)==1 )
-					{
-					    globus_ftp_control_layout_t layout;
-
-					    layout.mode = GLOBUS_FTP_CONTROL_STRIPING_BLOCKED_ROUND_ROBIN;
-					    layout.round_robin.block_size = striping_factor;
-					    if ( (result=globus_ftp_client_operationattr_set_layout(&(oattr[fd->fd_sys]),
-												    &layout))!=GLOBUS_SUCCESS  )
-						globus_err_handler("globus_ftp_client_operationattr_set_layout",
-								   myname,result);
-					}
-				}
-			}
-		}
-
-	    ADIOI_Info_get(fd->info,"tcp_buffer",MPI_MAX_INFO_VAL,hintval,&keyfound);
-	    if ( keyfound )
-		{
-		    /* set tcp buffer size */
-		    int buffer_size;
-		    if ( sscanf(hintval,"%d",&buffer_size)==1 )
-			{
-			    globus_ftp_control_tcpbuffer_t tcpbuf;
-
-			    tcpbuf.mode = GLOBUS_FTP_CONTROL_TCPBUFFER_FIXED;
-			    tcpbuf.fixed.size = buffer_size;
-			    if ( (result=globus_ftp_client_operationattr_set_tcp_buffer(&(oattr[fd->fd_sys]),
-											&tcpbuf))!=GLOBUS_SUCCESS )
-				globus_err_handler("globus_ftp_client_operationattr_set_tcp_buffer",myname,result);
-			}
-		}
-
-	    ADIOI_Info_get(fd->info,"transfer_type",MPI_MAX_INFO_VAL,hintval,&keyfound);
-	    if ( keyfound )
-		{
-		    globus_ftp_control_type_t filetype;
-		    /* set transfer type (i.e. ASCII or binary) */
-		    if ( !strcmp("ascii",hintval) || !strcmp("ASCII",hintval) )
-			{
-			    filetype=GLOBUS_FTP_CONTROL_TYPE_ASCII;
-			}
-		    else
-			{
-			    filetype=GLOBUS_FTP_CONTROL_TYPE_IMAGE;
-			}
-		    if ( (result=globus_ftp_client_operationattr_set_type(&(oattr[fd->fd_sys]),filetype))!=GLOBUS_SUCCESS )
-			globus_err_handler("globus_ftp_client_operationattr_set_type",myname,result);
-		}
-	}
-    else
-	FPRINTF(stderr,"no MPI_Info object associated with %s\n",fd->filename);
-
-    /* Create the ftp handle */
-    result=globus_ftp_client_handle_init(&(gridftp_fh[fd->fd_sys]),&hattr);
-    if ( result != GLOBUS_SUCCESS )
-	{
-	    globus_err_handler("globus_ftp_client_handle_init",myname,result);
-	    fd->fd_sys = -1;
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-		    myname, __LINE__, MPI_ERR_IO,
-		    "**io",
-		    "**io %s", globus_object_printable_to_string(globus_error_get(result)));
-	    return;
-	}
-
-    /* Check for existence of the file */
-    globus_mutex_init(&lock, GLOBUS_NULL);
-    globus_cond_init(&cond, GLOBUS_NULL);
-    file_exists=GLOBUS_FALSE;
-    exists_done=GLOBUS_FALSE;
-    if ( myrank==0 )
-	{
-	    if ( (result=globus_ftp_client_exists(&(gridftp_fh[fd->fd_sys]),
-						  fd->filename,
-						  &(oattr[fd->fd_sys]),
-						  exists_cb,
-						  GLOBUS_NULL))!=GLOBUS_SUCCESS )
-		{
-		    globus_err_handler("globus_ftp_client_exists",myname,result);
-		    fd->fd_sys = -1; 
-		    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-				    myname, __LINE__, MPI_ERR_IO,
-				    "**io", "**io %s", 
-				    globus_object_printable_to_string(globus_error_get(result)));
-		    return;
-		}
-	    /* wait till the callback completes */
-	    globus_mutex_lock(&lock);
-	    while ( exists_done!=GLOBUS_TRUE )
-		globus_cond_wait(&cond,&lock);
-	    globus_mutex_unlock(&lock);
-	}
-    MPI_Barrier(fd->comm);
-    MPI_Bcast(&file_exists,1,MPI_INT,0,fd->comm);
-
-    /* It turns out that this is handled by MPI_File_open() directly */
-    if ( (file_exists!=GLOBUS_TRUE) && (fd->access_mode&ADIO_CREATE) &&
-	 !(fd->access_mode&ADIO_EXCL) && !(fd->access_mode&ADIO_RDONLY) )
-	{
-	    if ( myrank==0 )
-		{
-		    /* if the file doesn't exist, write a single NULL to it */
-		    globus_byte_t touchbuf=(globus_byte_t)'\0';
-		    touch_ctl_done=GLOBUS_FALSE;
-		    if ( (result=globus_ftp_client_put(&(gridftp_fh[fd->fd_sys]),
-						       fd->filename,
-						       &(oattr[fd->fd_sys]),
-						       GLOBUS_NULL,
-						       touch_ctl_cb,
-						       GLOBUS_NULL))!=GLOBUS_SUCCESS )
-			{
-			    globus_err_handler("globus_ftp_client_put",myname,result);
-			    fd->fd_sys = -1;
-			    *error_code = MPIO_Err_create_code(MPI_SUCCESS, 
-				MPIR_ERR_RECOVERABLE,
-				myname, __LINE__, MPI_ERR_IO,
-				"**io", "**io %s", 
-				globus_object_printable_to_string(globus_error_get(result)));
-			    return;
-			}
-		    result=globus_ftp_client_register_write(&(gridftp_fh[fd->fd_sys]),
-				  (globus_byte_t *)&touchbuf, 0,
-				  (globus_off_t)0, GLOBUS_TRUE,
-				  touch_data_cb, GLOBUS_NULL);
-
-		    if ( result != GLOBUS_SUCCESS )
-			{
-			    globus_err_handler("globus_ftp_client_register_write",myname,result);
-			    *error_code = MPIO_Err_create_code(MPI_SUCCESS, 
-				MPIR_ERR_RECOVERABLE,
-				myname, __LINE__, MPI_ERR_IO,
-				"**io", "**io %s", 
-				globus_object_printable_to_string(globus_error_get(result)));
-			    return;
-			}
-		    globus_mutex_lock(&lock);
-		    while ( touch_ctl_done!=GLOBUS_TRUE )
-			globus_cond_wait(&cond,&lock);
-		    globus_mutex_unlock(&lock);
-		}
-	    MPI_Barrier(fd->comm);
-	}
-    else if ( (fd->access_mode&ADIO_EXCL) && (file_exists==GLOBUS_TRUE) )
-	{
-	    fd->fd_sys = -1;
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-			    myname, __LINE__, MPI_ERR_IO, 
-			    "**io", 0);
-	    return;
-	}
-    else if ( (fd->access_mode&ADIO_RDONLY) && (file_exists!=GLOBUS_TRUE) )
-	{
-	    if ( myrank==0 )
-		{
-		    FPRINTF(stderr,"WARNING:  read-only file %s does not exist!\n",fd->filename);
-		}
-	}
-    num_gridftp_handles++;
-}
diff --git a/3rd-party/romio321/adio/ad_gridftp/ad_gridftp_read.c b/3rd-party/romio321/adio/ad_gridftp/ad_gridftp_read.c
deleted file mode 100644
index 91ed544c807..00000000000
--- a/3rd-party/romio321/adio/ad_gridftp/ad_gridftp_read.c
+++ /dev/null
@@ -1,465 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2003 University of Chicago, Ohio Supercomputer Center. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_gridftp.h"
-#include "adioi.h"
-#include "adio_extern.h"
-
-static globus_mutex_t readcontig_ctl_lock;
-static globus_cond_t readcontig_ctl_cond;
-static globus_bool_t readcontig_ctl_done;
-static void readcontig_ctl_cb(void *myargs, globus_ftp_client_handle_t *handle, globus_object_t *error)
-{
-    if (error)
-	{
-	    FPRINTF(stderr, "%s\n", globus_object_printable_to_string(error));
-	}
-    globus_mutex_lock(&readcontig_ctl_lock);
-    if ( readcontig_ctl_done!=GLOBUS_TRUE )
-	readcontig_ctl_done=GLOBUS_TRUE;
-    globus_cond_signal(&readcontig_ctl_cond);
-    globus_mutex_unlock(&readcontig_ctl_lock);
-    return;
-}
-
-static void readcontig_data_cb(void *myargs, globus_ftp_client_handle_t *handle, globus_object_t *error,
-			       globus_byte_t *buffer, globus_size_t length, globus_off_t offset,
-			       globus_bool_t eof)
-{
-   globus_size_t *bytes_read;
-
-    bytes_read=(globus_size_t *)myargs;
-    if (error)
-	{
-	    FPRINTF(stderr, "%s\n", globus_object_printable_to_string(error));
-	}
-    *bytes_read+=length;
-    /* I don't understand why the data callback has to keep recalling register_read,
-       but everything I've done and all the examples I've seen seem to require
-       that behavior to work... */
-    /*
-     * Using buffer+length seems to work, but is probably not the correct
-     * solution.  A big read of 256kB chunks will have lines like this:
-	readcontig_data_cb: buffer 0x404e0008 length 0 offset 31719424 eof 1
-	readcontig_data_cb: buffer 0x404a0008 length 65536 offset 31981568 eof 0
-	readcontig_data_cb: buffer 0x404b0008 length 65536 offset 32047104 eof 0
-	readcontig_data_cb: buffer 0x404c0008 length 65536 offset 32112640 eof 0
-	readcontig_data_cb: buffer 0x404d0008 length 65536 offset 32178176 eof 0
-     */
-    if ( !eof )
-	    globus_ftp_client_register_read(handle,
-					    buffer+length,
-					    length,
-					    readcontig_data_cb,
-					    (void *)(bytes_read));
-    return;
-}
-
-static globus_mutex_t readdiscontig_ctl_lock;
-static globus_cond_t readdiscontig_ctl_cond;
-static globus_bool_t readdiscontig_ctl_done;
-static void readdiscontig_ctl_cb(void *myargs, globus_ftp_client_handle_t *handle, globus_object_t *error)
-{
-    if (error)
-	{
-	    FPRINTF(stderr, "%s\n", globus_object_printable_to_string(error));
-	}
-    globus_mutex_lock(&readdiscontig_ctl_lock);
-    if ( readdiscontig_ctl_done!=GLOBUS_TRUE )
-	readdiscontig_ctl_done=GLOBUS_TRUE;
-    globus_cond_signal(&readdiscontig_ctl_cond);
-    globus_mutex_unlock(&readdiscontig_ctl_lock);
-    return;
-}
-
-static void readdiscontig_data_cb(void *myargs, globus_ftp_client_handle_t *handle, globus_object_t *error,
-			       globus_byte_t *buffer, globus_size_t length, globus_off_t offset,
-			       globus_bool_t eof)
-{
-   globus_size_t *bytes_read;
-
-    bytes_read=(globus_size_t *)myargs;
-    if (error)
-	{
-	    FPRINTF(stderr, "%s\n", globus_object_printable_to_string(error));
-	}
-    *bytes_read+=length;
-    /* I don't understand why the data callback has to keep recalling register_read,
-       but everything I've done and all the examples I've seen seem to require
-       that behavior to work... */
-    if ( !eof )
-	    globus_ftp_client_register_read(handle,
-					    buffer,
-					    length,
-					    readdiscontig_data_cb,
-					    (void *)(bytes_read));
-    return;
-}
-
-void ADIOI_GRIDFTP_ReadContig(ADIO_File fd, void *buf, int count, 
-			     MPI_Datatype datatype, int file_ptr_type,
-			     ADIO_Offset offset, ADIO_Status *status, int
-			     *error_code)
-{
-    static char myname[]="ADIOI_GRIDFTP_ReadContig";
-    int myrank, nprocs;
-    MPI_Count datatype_size;
-    globus_size_t len,bytes_read=0;
-    globus_off_t goff;
-    globus_result_t result;
-
-    if ( fd->access_mode&ADIO_WRONLY )
-	{
-	    *error_code=MPIR_ERR_MODE_WRONLY;
-	    return;
-	}
-
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-    MPI_Type_size_x(datatype, &datatype_size);
-
-    if (file_ptr_type != ADIO_EXPLICIT_OFFSET)
-    {
-	offset = fd->fp_ind;
-    }
-
-    /* Do the gridftp I/O transfer */
-    goff = (globus_off_t)offset;
-    len = ((globus_size_t)datatype_size)*((globus_size_t)count);
-
-    globus_mutex_init(&readcontig_ctl_lock, GLOBUS_NULL);
-    globus_cond_init(&readcontig_ctl_cond, GLOBUS_NULL);
-    readcontig_ctl_done=GLOBUS_FALSE;
-    if ( (result=globus_ftp_client_partial_get(&(gridftp_fh[fd->fd_sys]),
-					       fd->filename,
-					       &(oattr[fd->fd_sys]),
-					       GLOBUS_NULL,
-					       goff,
-					       goff+(globus_off_t)len,
-					       readcontig_ctl_cb,
-					       GLOBUS_NULL))!=GLOBUS_SUCCESS )
-	{
-	    globus_err_handler("globus_ftp_client_partial_get",myname,result);
-	    *error_code=MPI_ERR_IO;
-	    ADIOI_Error(fd,*error_code,myname);
-	    return;
-	}
-    result=globus_ftp_client_register_read(&(gridftp_fh[fd->fd_sys]),
-		    (globus_byte_t *)buf, len, readcontig_data_cb,
-		    (void *)(&bytes_read));
-    if ( result != GLOBUS_SUCCESS )
-	{
-	    globus_err_handler("globus_ftp_client_register_read",myname,result);
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, 
-			    MPIR_ERR_RECOVERABLE, myname, __LINE__, 
-			    MPI_ERR_IO, "**io", "**io %s", 
-			    globus_object_printable_to_string(globus_error_get(result)));
-	    return;
-	}  
-
-
-    /* The ctl callback won't start till the data callbacks complete, so it's
-       safe to wait on just the ctl callback */
-    globus_mutex_lock(&readcontig_ctl_lock);
-    while ( readcontig_ctl_done!=GLOBUS_TRUE )
-	globus_cond_wait(&readcontig_ctl_cond,&readcontig_ctl_lock);
-    globus_mutex_unlock(&readcontig_ctl_lock);
-
-    globus_mutex_destroy(&readcontig_ctl_lock);
-    globus_cond_destroy(&readcontig_ctl_cond);
-
-#ifdef HAVE_STATUS_SET_BYTES
-    MPIR_Status_set_bytes(status, datatype, bytes_read);
-#endif
-    if (file_ptr_type != ADIO_EXPLICIT_OFFSET)
-    {
-	fd->fp_ind += bytes_read;
-	fd->fp_sys_posn = fd->fp_ind;
-    }
-    else {
-	fd->fp_sys_posn = offset + bytes_read;
-    }
-}
-
-void ADIOI_GRIDFTP_ReadDiscontig(ADIO_File fd, void *buf, int count,
-				 MPI_Datatype datatype, int file_ptr_type,
-				 ADIO_Offset offset, ADIO_Status *status, int
-				 *error_code)
-{
-    char myname[]="ADIOI_GRIDFTP_ReadDiscontig";
-    int myrank,nprocs;
-    /* size and extent of buffer in memory */
-    MPI_Aint btype_size,btype_extent,btype_lb;
-    /* size and extent of file record layout */
-    MPI_Aint ftype_size,ftype_extent,ftype_lb;
-    /* size of file elemental type; seeks are done in units of this */
-    MPI_Aint etype_size;
-    MPI_Aint extent;
-    ADIOI_Flatlist_node *flat_file;
-    int i,buf_contig,boff,nblks;
-    globus_off_t start,end,goff;
-    globus_size_t bytes_read;
-    globus_result_t result;
-    globus_byte_t *tmp;
-
-    if ( fd->access_mode&ADIO_WRONLY )
-	{
-	    *error_code=MPIR_ERR_MODE_WRONLY;
-	    return;
-	}
-
-    *error_code=MPI_SUCCESS;
-
-    MPI_Comm_rank(fd->comm,&myrank);
-    MPI_Comm_size(fd->comm,&nprocs);
-
-    etype_size=fd->etype_size;
-    MPI_Type_size_x(fd->filetype,&ftype_size);
-    MPI_Type_get_extent(fd->filetype,&ftype_lb,&ftype_extent);
-    /* This is arguably unnecessary, as this routine assumes that the
-       buffer in memory is contiguous */
-    MPI_Type_size_x(datatype,&btype_size);
-    MPI_Type_get_extent(datatype,&btype_lb,&btype_extent);
-    ADIOI_Datatype_iscontig(datatype,&buf_contig);
-    
-    if ( ( btype_extent!=btype_size ) || ( ! buf_contig ) )
-	{
-	    FPRINTF(stderr,"[%d/%d] %s called with discontigous memory buffer\n",
-		    myrank,nprocs,myname);
-	    fflush(stderr);
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, 
-			    MPIR_ERR_RECOVERABLE, myname, __LINE__, 
-			    MPI_ERR_IO, "**io", 0 );
-	    return;
-	}
-    /* from here we can assume btype_extent==btype_size */
-
-    /* Flatten out fd->filetype so we know which blocks to skip */
-    flat_file = ADIOI_Flatten_and_find(fd->filetype);
-
-    /* Figure out how big the area to read is */
-    start=(globus_off_t)(offset*etype_size);
-    goff=start;
-    boff=0;
-    extent=0;
-    nblks=0;
-    while ( boff < (count*btype_size) )
-	{
-	    int blklen=0;
-
-	    for (i=0;i<flat_file->count;i++)
-		{
-		    /* find the length of the next block */
-		    if ( (boff+flat_file->blocklens[i]) < (count*btype_size) )
-			blklen=flat_file->blocklens[i];
-		    else
-			blklen=(count*btype_size)-boff;
-		    /* increment buffer size to be used */
-		    boff+=blklen;
-		    /* compute extent -- the nblks*ftype_extent bit is
-		       there so we remember how many ftypes we've already
-		       been through */
-		    extent=MAX(extent,nblks*ftype_extent+flat_file->indices[i]+blklen);
-		    if ( boff>=(count*btype_size) )
-			break;
-		}
-	    nblks++;
-	}
-    if ( extent < count*btype_size )
-	{
-	    FPRINTF(stderr,"[%d/%d] %s error in computing extent -- extent %d is smaller than total bytes requested %d!\n",
-		    myrank,nprocs,myname,extent,count*btype_size);
-	    fflush(stderr);
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, 
-			    MPIR_ERR_RECOVERABLE, myname, __LINE__, 
-			    MPI_ERR_IO, "**io", 0);
-	    return;
-	}
-    end=start+(globus_off_t)extent;
-    tmp=(globus_byte_t *)ADIOI_Malloc((size_t)extent*sizeof(globus_byte_t));
-
-    /* start up the globus partial read */
-    globus_mutex_init(&readdiscontig_ctl_lock, GLOBUS_NULL);
-    globus_cond_init(&readdiscontig_ctl_cond, GLOBUS_NULL);
-    readdiscontig_ctl_done=GLOBUS_FALSE;
-    if ( (result=globus_ftp_client_partial_get(&(gridftp_fh[fd->fd_sys]),
-					       fd->filename,
-					       &(oattr[fd->fd_sys]),
-					       GLOBUS_NULL,
-					       start,
-					       end,
-					       readdiscontig_ctl_cb,
-					       GLOBUS_NULL))!=GLOBUS_SUCCESS )
-	{
-	    globus_err_handler("globus_ftp_client_partial_get",myname,result);
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, 
-			    MPIR_ERR_RECOVERABLE, myname, __LINE__, 
-			    MPI_ERR_IO, "**io", "**io %s", 
-			    globus_object_printable_to_string(globus_error_get(result)));
-	    return;
-	}
-
-    /* Do all the actual I/Os */
-    /* Since globus_ftp_client_register_read() is brain-dead and doesn't
-       let you specify an offset, we have to slurp the entire extent into
-       memory and then parse out the pieces we want...  Sucks, doesn't it?
-
-       This should probably be done in chunks (preferably of a size
-       set using a file hint), but that'll have to come later.
-       --TB */
-    if ( (result=globus_ftp_client_register_read(&(gridftp_fh[fd->fd_sys]),
-						 tmp,
-						 (globus_size_t)extent,
-						 readdiscontig_data_cb,
-						 (void *)(&bytes_read)))!=GLOBUS_SUCCESS )
-	{
-	    globus_err_handler("globus_ftp_client_register_read",myname,result);
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-		    myname, __LINE__, MPI_ERR_IO,
-		    "**io",
-		    "**io %s", globus_object_printable_to_string(globus_error_get(result)));
-	    return;
-	}
-    /* The ctl callback won't start till the data callbacks complete, so it's
-       safe to wait on just the ctl callback */
-    globus_mutex_lock(&readdiscontig_ctl_lock);
-    while ( readdiscontig_ctl_done!=GLOBUS_TRUE )
-	globus_cond_wait(&readdiscontig_ctl_cond,&readdiscontig_ctl_lock);
-    globus_mutex_unlock(&readdiscontig_ctl_lock);
-
-    globus_mutex_destroy(&readdiscontig_ctl_lock);
-    globus_cond_destroy(&readdiscontig_ctl_cond);
-
-    boff=0;
-    nblks=0;
-    goff=0;
-    while ( boff < (count*btype_size) )
-	{
-	    int i,blklen;
-
-	    for (i=0;i<flat_file->count;i++)
-		{
-		    if ( (boff+flat_file->blocklens[i]) < (count*btype_size) )
-			blklen=flat_file->blocklens[i];
-		    else
-			blklen=(count*btype_size)-boff;
-		    if ( blklen > 0 )
-			{
-			    goff=nblks*ftype_extent+flat_file->indices[i];
-			    memcpy((globus_byte_t *)buf+boff,tmp+goff,(size_t)blklen);
-			    boff+=blklen;
-			    if ( boff>=(count*btype_size) )
-				break;
-			}
-		}
-	    nblks++;
-	}
-    ADIOI_Free(tmp);
-
-#ifdef HAVE_STATUS_SET_BYTES
-    MPIR_Status_set_bytes(status, datatype, bytes_read);
-#endif
-    if (file_ptr_type != ADIO_EXPLICIT_OFFSET)
-    {
-	fd->fp_ind += extent;
-	fd->fp_sys_posn = fd->fp_ind;
-    }
-    else {
-	fd->fp_sys_posn = offset + extent;
-    }
-}
-
-void ADIOI_GRIDFTP_ReadStrided(ADIO_File fd, void *buf, int count,
-			      MPI_Datatype datatype, int file_ptr_type,
-			      ADIO_Offset offset, ADIO_Status *status, int
-			      *error_code)
-{
-    /*
-    int myrank, nprocs;
-
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-#ifdef PRINT_ERR_MSG
-    FPRINTF(stdout, "[%d/%d] ADIOI_GRIDFTP_ReadStrided called on %s\n", myrank, 
-	    nprocs, fd->filename);
-    FPRINTF(stdout, "[%d/%d]    calling ADIOI_GEN_ReadStrided\n", myrank, 
-	    nprocs);
-#endif
-
-    ADIOI_GEN_ReadStrided(fd, buf, count, datatype, file_ptr_type, offset,
-			  status, error_code);
-    
-    */
-
-    char myname[]="ADIOI_GRIDFTP_ReadStrided";
-    int myrank, nprocs;
-    int i,j;
-    int buf_contig,file_contig;
-    MPI_Aint btype_size,bufsize;
-    globus_off_t start,disp;
-    globus_size_t bytes_read;
-    globus_byte_t *intermediate;
-
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-
-    MPI_Type_size_x(datatype,&btype_size);
-    bufsize=count*btype_size;
-    ADIOI_Datatype_iscontig(fd->filetype,&file_contig);
-    ADIOI_Datatype_iscontig(datatype,&buf_contig);
-    if ( buf_contig && !file_contig )
-	{
-	    /* Contiguous in memory, discontig in file */
-	    ADIOI_GRIDFTP_ReadDiscontig(fd, buf, count, datatype,
-					file_ptr_type, offset, status, error_code);
-	}
-    else if ( !buf_contig && file_contig )
-	{
-	    /* Discontiguous in mem, contig in file -- comparatively easy */
-	    int posn=0;
-
-	    /* read contiguous data into intermediate buffer */
-	    intermediate=(globus_byte_t *)ADIOI_Malloc((size_t)bufsize);
-	    ADIOI_GRIDFTP_ReadContig(fd, intermediate, bufsize, MPI_BYTE,
-				     file_ptr_type, offset, status, error_code);
-
-	    /* explode contents of intermediate buffer into main buffer */
-	    MPI_Unpack(intermediate,bufsize,&posn,buf,count,datatype,fd->comm);
-
-	    ADIOI_Free(intermediate);
-	}
-    else if ( !buf_contig && !file_contig )
-	{
-	    /* Discontig in both mem and file -- the hardest case */
-	    int posn=0;
-
-	    /* Read discontiguous data into intermediate buffer */
-	    intermediate=(globus_byte_t *)ADIOI_Malloc((size_t)bufsize);
-	    ADIOI_GRIDFTP_ReadDiscontig(fd, intermediate, bufsize, MPI_BYTE,
-					file_ptr_type, offset, status, error_code);
-
-	    /* explode contents of intermediate buffer into main buffer */
-	    posn=0;
-	    MPI_Unpack(intermediate,bufsize,&posn,buf,count,datatype,fd->comm);
-
-	    ADIOI_Free(intermediate);
-	}
-    else 
-	{
-	    /* Why did you bother calling ReadStrided?!?!?! */
-	    ADIOI_GRIDFTP_ReadContig(fd, buf, count, datatype,
-				     file_ptr_type, offset, status, error_code);
-	}
-
-}
-
diff --git a/3rd-party/romio321/adio/ad_gridftp/ad_gridftp_resize.c b/3rd-party/romio321/adio/ad_gridftp/ad_gridftp_resize.c
deleted file mode 100644
index 96c0460c42f..00000000000
--- a/3rd-party/romio321/adio/ad_gridftp/ad_gridftp_resize.c
+++ /dev/null
@@ -1,241 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2003 University of Chicago, Ohio Supercomputer Center. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_gridftp.h"
-#include "adioi.h"
-
-static globus_mutex_t resize_lock;
-static globus_cond_t resize_cond;
-static globus_bool_t resize_done;
-static globus_bool_t resize_success;
-
-void resize_cb(void *myargs, globus_ftp_client_handle_t *handle,
-		  globus_object_t *error)
-{
-    if (error)
-	{
-	    FPRINTF(stderr, "%s\n", globus_object_printable_to_string(error));
-	    globus_mutex_lock(&resize_lock);
-	    resize_success=GLOBUS_FALSE;
-	    globus_mutex_unlock(&resize_lock);
-	}
-    else
-	{
-	    globus_mutex_lock(&resize_lock);
-	    resize_success=GLOBUS_TRUE;
-	    globus_mutex_unlock(&resize_lock);
-	}
-    globus_mutex_lock(&resize_lock);
-    resize_done=GLOBUS_TRUE;
-    globus_cond_signal(&resize_cond);
-    globus_mutex_unlock(&resize_lock);
-}
-
-
-static void resize_wrdata_cb(void *myargs, globus_ftp_client_handle_t *handle, globus_object_t *error,
-			     globus_byte_t *buffer, globus_size_t length, globus_off_t offset,
-			     globus_bool_t eof)
-{
-    if (error)
-	FPRINTF(stderr, "%s\n", globus_object_printable_to_string(error));
-    if (!eof)
-	globus_ftp_client_register_read(handle,
-					buffer,
-					length,
-					resize_wrdata_cb,
-					myargs);
-    return;
-}
-
-
-void ADIOI_GRIDFTP_Resize(ADIO_File fd, ADIO_Offset size, int *error_code)
-{
-    int myrank, nprocs;
-    char myname[]="ADIOI_GRIDFTP_Resize";
-    globus_off_t fsize;
-    globus_result_t result;
-
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-
-    /* Sanity check */
-    if ( fd->access_mode&ADIO_RDONLY )
-	{
-	    FPRINTF(stderr,"%s:  attempt to resize read-only file %s!\n",
-		    myname,fd->filename);
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-		    myname, __LINE__, MPI_ERR_IO,
-		    "**io", 0);
-	    return;
-	}
-
-    /* This routine is supposed to do the moral equivalent of truncate(),
-       but there's not an equivalent operation in the globus_ftp_client API. */
-    globus_mutex_init(&resize_lock,GLOBUS_NULL);
-    globus_cond_init(&resize_cond,GLOBUS_NULL);
-    resize_done=GLOBUS_FALSE;
-    if ( (result=globus_ftp_client_size(&(gridftp_fh[fd->fd_sys]),
-					fd->filename,
-					&(oattr[fd->fd_sys]),
-					&(fsize),
-					resize_cb,
-					GLOBUS_NULL))!=GLOBUS_SUCCESS )
-	{
-	    globus_err_handler("globus_ftp_client_size",myname,result);
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-		    myname, __LINE__, MPI_ERR_IO,
-		    "**io",
-		    "**io %s", globus_object_printable_to_string(globus_error_get(result)));
-	    return;
-	}
-    globus_mutex_lock(&resize_lock);
-    while ( resize_done!=GLOBUS_TRUE )
-	globus_cond_wait(&resize_lock,&resize_cond);
-    if ( fsize < (globus_off_t)size )
-	{
-	    /* The file is smaller than the requested size, so
-	       do a zero-byte write to where the new EOF should be. */
-	    globus_byte_t touchbuf=(globus_byte_t)'\0';
-	    resize_done=GLOBUS_FALSE;
-	    if ( (result=globus_ftp_client_partial_put(&(gridftp_fh[fd->fd_sys]),
-						       fd->filename,
-						       &(oattr[fd->fd_sys]),
-						       GLOBUS_NULL,
-						       (globus_off_t)size,
-						       (globus_off_t)size,
-						       resize_cb,
-						       GLOBUS_NULL))!=GLOBUS_SUCCESS )
-		{
-		    globus_err_handler("globus_ftp_client_partial_put",myname,result);
-		    *error_code = MPIO_Err_create_code(MPI_SUCCESS, 
-				    MPIR_ERR_RECOVERABLE, myname, __LINE__, 
-				    MPI_ERR_IO, "**io", "**io %s", 
-				    globus_object_printable_to_string(globus_error_get(result)));
-		    return;
-		}
-
-	    if ( (result=globus_ftp_client_register_write(&(gridftp_fh[fd->fd_sys]),
-							  (globus_byte_t *)&touchbuf,
-							  0,
-							  (globus_off_t)0,
-							  GLOBUS_TRUE,
-							  resize_wrdata_cb,
-							  GLOBUS_NULL))!=GLOBUS_SUCCESS )
-		{
-		    globus_err_handler("globus_ftp_client_register_write",myname,result);
-		    *error_code = MPIO_Err_create_code(MPI_SUCCESS, 
-				    MPIR_ERR_RECOVERABLE, myname, __LINE__, 
-				    MPI_ERR_IO, "**io", "**io %s", 
-				    globus_object_printable_to_string(globus_error_get(result)));
-		    return;
-		}
-	    globus_mutex_lock(&resize_lock);
-	    while ( resize_done!=GLOBUS_TRUE )
-		globus_cond_wait(&resize_cond,&resize_lock);
-	    globus_mutex_unlock(&resize_lock);
-	}
-    else if ( fsize > (globus_off_t)size )
-	{
-	    /* The file is bigger than the requested size, so
-	       we'll abuse globus_ftp_client_third_party_partial_put()
-	       into truncating it for us. */
-	    char *urlold;
-	    size_t urllen;
-
-	    urllen=strlen(fd->filename);
-	    urlold=(char *)ADIOI_Malloc(urllen+5);
-	    ADIOI_Snprintf(urlold,urllen+5,"%s.old",fd->filename);
-	    resize_done=GLOBUS_FALSE;
-	    resize_success=GLOBUS_FALSE;
-	    if ( (result=globus_ftp_client_move(&(gridftp_fh[fd->fd_sys]),
-						fd->filename,
-						urlold,
-						&(oattr[fd->fd_sys]),
-						resize_cb,
-						GLOBUS_NULL))!=GLOBUS_SUCCESS )
-		{
-		    globus_err_handler("globus_ftp_client_move",myname,result);
-		    *error_code = MPIO_Err_create_code(MPI_SUCCESS, 
-				    MPIR_ERR_RECOVERABLE, myname, __LINE__, 
-				    MPI_ERR_IO, "**io", "**io %s", 
-				    globus_object_printable_to_string(globus_error_get(result)));
-		    return;
-		}
-	    globus_mutex_lock(&resize_lock);
-	    while ( resize_done!=GLOBUS_TRUE )
-		globus_cond_wait(&resize_cond,&resize_lock);
-	    globus_mutex_unlock(&resize_lock);
-	    if ( resize_success!=GLOBUS_TRUE )
-		{
-		    *error_code = MPI_ERR_IO;
-		    return;
-		}
-	    resize_done=GLOBUS_FALSE;
-	    if ( (result=globus_ftp_client_partial_third_party_transfer(&(gridftp_fh[fd->fd_sys]),
-						urlold,
-						&(oattr[fd->fd_sys]),
-						fd->filename,
-						&(oattr[fd->fd_sys]),
-						GLOBUS_NULL,
-						0,
-						(globus_off_t)size,
-						resize_cb,
-						GLOBUS_NULL))!=GLOBUS_SUCCESS )
-		{
-		    globus_err_handler("globus_ftp_client_partial_third_party_transfer",myname,result);
-		    *error_code = MPIO_Err_create_code(MPI_SUCCESS, 
-				    MPIR_ERR_RECOVERABLE, myname, __LINE__, 
-				    MPI_ERR_IO, "**io", "**io %s", 
-				    globus_object_printable_to_string(globus_error_get(result)));
-		    return;
-		}
-	    globus_mutex_lock(&resize_lock);
-	    while ( resize_done!=GLOBUS_TRUE )
-		globus_cond_wait(&resize_cond,&resize_lock);
-	    globus_mutex_unlock(&resize_lock);
-	    if ( resize_success!=GLOBUS_TRUE )
-		{
-		    *error_code = MPI_ERR_IO;
-		    ADIOI_Error(fd,*error_code,myname);
-		    return;
-		}
-	    resize_done=GLOBUS_FALSE;
-	    if ( (result=globus_ftp_client_delete(&(gridftp_fh[fd->fd_sys]),
-						  urlold,
-						  &(oattr[fd->fd_sys]),
-						  resize_cb,
-						  GLOBUS_NULL))!=GLOBUS_SUCCESS )
-		{
-		    globus_err_handler("globus_ftp_client_delete",myname,result);
-		    *error_code = MPIO_Err_create_code(MPI_SUCCESS, 
-				    MPIR_ERR_RECOVERABLE, myname, __LINE__, 
-				    MPI_ERR_IO, "**io", "**io %s", 
-				    globus_object_printable_to_string(globus_error_get(result)));
-		    return;
-		}
-	    globus_mutex_lock(&resize_lock);
-	    while ( resize_done!=GLOBUS_TRUE )
-		globus_cond_wait(&resize_cond,&resize_lock);
-	    globus_mutex_unlock(&resize_lock);
-	    if ( resize_success!=GLOBUS_TRUE )
-		{
-		    *error_code = MPI_ERR_IO;
-		    ADIOI_Error(fd,*error_code,myname);
-		    return;
-		}
-	    ADIOI_Free(urlold);
-	}
-    globus_mutex_destroy(&resize_lock);
-    globus_cond_destroy(&resize_cond);
-}
-
-
-
-
-
diff --git a/3rd-party/romio321/adio/ad_gridftp/ad_gridftp_write.c b/3rd-party/romio321/adio/ad_gridftp/ad_gridftp_write.c
deleted file mode 100644
index d349d5c2318..00000000000
--- a/3rd-party/romio321/adio/ad_gridftp/ad_gridftp_write.c
+++ /dev/null
@@ -1,470 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2003 University of Chicago, Ohio Supercomputer Center. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_gridftp.h"
-#include "adioi.h"
-#include "adio_extern.h"
-
-static globus_mutex_t writecontig_ctl_lock;
-static globus_cond_t writecontig_ctl_cond;
-static globus_bool_t writecontig_ctl_done;
-static void writecontig_ctl_cb(void *myargs, globus_ftp_client_handle_t *handle, globus_object_t *error)
-{
-    if (error)
-	{
-	    FPRINTF(stderr, "%s\n", globus_object_printable_to_string(error));
-	}
-    globus_mutex_lock(&writecontig_ctl_lock);
-    if ( writecontig_ctl_done!=GLOBUS_TRUE )
-	writecontig_ctl_done=GLOBUS_TRUE;
-    globus_cond_signal(&writecontig_ctl_cond);
-    globus_mutex_unlock(&writecontig_ctl_lock);
-#ifdef PRINT_ERR_MSG
-    FPRINTF(stderr,"finished with contig write transaction\n");
-#endif /* PRINT_ERR_MSG */
-    return;
-}
-
-static void writecontig_data_cb(void *myargs, globus_ftp_client_handle_t *handle, globus_object_t *error,
-			       globus_byte_t *buffer, globus_size_t length, globus_off_t offset,
-			       globus_bool_t eof)
-{
-   globus_size_t *bytes_written;
-
-    bytes_written=(globus_size_t *)myargs;
-    if (error)
-	{
-	    FPRINTF(stderr, "%s\n", globus_object_printable_to_string(error));
-	}
-    *bytes_written+=length;
-    /* I don't understand why the data callback has to keep recalling register_write,
-       but everything I've done and all the examples I've seen seem to require
-       that behavior to work... */
-    if ( !eof )
-	{
-	    globus_ftp_client_register_write(handle,
-					     buffer,
-					     length,
-					     offset,
-					     GLOBUS_TRUE,
-					     writecontig_data_cb,
-					     (void *)(bytes_written));
-	}
-#ifdef PRINT_ERR_MSG
-    FPRINTF(stderr,"wrote %Ld bytes...",(long long)length);
-#endif /* PRINT_ERR_MSG */
-    return;
-}
-
-
-static globus_mutex_t writediscontig_ctl_lock;
-static globus_cond_t writediscontig_ctl_cond;
-static globus_bool_t writediscontig_ctl_done;
-static void writediscontig_ctl_cb(void *myargs, globus_ftp_client_handle_t *handle, globus_object_t *error)
-{
-    if (error)
-	{
-	    FPRINTF(stderr, "%s\n", globus_object_printable_to_string(error));
-	}
-    globus_mutex_lock(&writediscontig_ctl_lock);
-    if ( writediscontig_ctl_done!=GLOBUS_TRUE )
-	writediscontig_ctl_done=GLOBUS_TRUE;
-    globus_cond_signal(&writediscontig_ctl_cond);
-    globus_mutex_unlock(&writediscontig_ctl_lock);
-    return;
-}
-
-static void writediscontig_data_cb(void *myargs, globus_ftp_client_handle_t *handle, globus_object_t *error,
-			       globus_byte_t *buffer, globus_size_t length, globus_off_t offset,
-			       globus_bool_t eof)
-{
-   globus_size_t *bytes_written;
-
-    bytes_written=(globus_size_t *)myargs;
-    if (error)
-	{
-	    FPRINTF(stderr, "%s\n", globus_object_printable_to_string(error));
-	}
-    *bytes_written+=length;
-    /* I don't understand why the data callback has to keep recalling register_read,
-       but everything I've done and all the examples I've seen seem to require
-       that behavior to work... */
-    if ( !eof )
-	globus_ftp_client_register_write(handle,
-					 buffer,
-					 length,
-					 offset,
-					 eof,
-					 writediscontig_data_cb,
-					 (void *)(bytes_written));
-    FPRINTF(stderr,"wrote %Ld bytes...",(long long)length); 
-    return;
-}
-
-
-void ADIOI_GRIDFTP_WriteContig(ADIO_File fd, void *buf, int count, 
-			     MPI_Datatype datatype, int file_ptr_type,
-			     ADIO_Offset offset, ADIO_Status *status, int
-			     *error_code)
-{
-    char myname[]="ADIOI_GRIDFTP_WriteContig";
-    int myrank, nprocs;
-    MPI_Count datatype_size;
-    globus_size_t len,bytes_written=0;
-    globus_off_t goff;
-    globus_result_t result;
-
-    if ( fd->access_mode&ADIO_RDONLY )
-	{
-	    *error_code=MPI_ERR_AMODE;
-	    return;
-	}
-
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-    MPI_Type_size_x(datatype, &datatype_size);
-
-    if (file_ptr_type != ADIO_EXPLICIT_OFFSET)
-    {
-	offset = fd->fp_ind;
-    }
-
-    /* Do the gridftp I/O transfer */
-    goff = (globus_off_t)offset;
-    len = ((globus_size_t)datatype_size)*((globus_size_t)count);
-
-    globus_mutex_init(&writecontig_ctl_lock, GLOBUS_NULL);
-    globus_cond_init(&writecontig_ctl_cond, GLOBUS_NULL);
-    writecontig_ctl_done=GLOBUS_FALSE;
-    if ( (result=globus_ftp_client_partial_put(&(gridftp_fh[fd->fd_sys]),
-					       fd->filename,
-					       &(oattr[fd->fd_sys]),
-					       GLOBUS_NULL,
-					       goff,
-					       goff+(globus_off_t)len,
-					       writecontig_ctl_cb,
-					       GLOBUS_NULL))!=GLOBUS_SUCCESS )
-	{
-	    globus_err_handler("globus_ftp_client_partial_put",myname,result);
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-		    myname, __LINE__, MPI_ERR_IO,
-		    "**io",
-		    "**io %s", globus_object_printable_to_string(globus_error_get(result)));
-	    return;
-	}
-    if ( (result=globus_ftp_client_register_write(&(gridftp_fh[fd->fd_sys]),
-						  (globus_byte_t *)buf,
-						  len,
-						  goff,
-						  GLOBUS_TRUE,
-						  writecontig_data_cb,
-						  (void *)(&bytes_written)))!=GLOBUS_SUCCESS )
-	{
-	    globus_err_handler("globus_ftp_client_register_write",myname,result);
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-		    myname, __LINE__, MPI_ERR_IO,
-		    "**io",
-		    "**io %s", globus_object_printable_to_string(globus_error_get(result)));
-	    return;
-	}
-
-
-    /* The ctl callback won't start till the data callbacks complete, so it's
-       safe to wait on just the ctl callback */
-    globus_mutex_lock(&writecontig_ctl_lock);
-    while ( writecontig_ctl_done!=GLOBUS_TRUE )
-	globus_cond_wait(&writecontig_ctl_cond,&writecontig_ctl_lock);
-    globus_mutex_unlock(&writecontig_ctl_lock);
-
-    globus_mutex_destroy(&writecontig_ctl_lock);
-    globus_cond_destroy(&writecontig_ctl_cond);
-
-#ifdef HAVE_STATUS_SET_BYTES
-    MPIR_Status_set_bytes(status, datatype, bytes_written);
-#endif
-    if (file_ptr_type != ADIO_EXPLICIT_OFFSET)
-    {
-	offset = fd->fp_ind;
-	fd->fp_ind += bytes_written;
-	fd->fp_sys_posn = fd->fp_ind;
-    }
-    else {
-	fd->fp_sys_posn = offset + bytes_written;
-    }
-}
-
-
-void ADIOI_GRIDFTP_WriteDiscontig(ADIO_File fd, void *buf, int count,
-				 MPI_Datatype datatype, int file_ptr_type,
-				 ADIO_Offset offset, ADIO_Status *status, int
-				 *error_code)
-{
-    char myname[]="ADIOI_GRIDFTP_WriteDiscontig";
-    int myrank,nprocs;
-    MPI_Aint btype_size,btype_extent,btype_lb;
-    MPI_Aint ftype_size,ftype_extent,ftype_lb;
-    MPI_Aint etype_size;
-    MPI_Aint extent;
-    ADIOI_Flatlist_node *flat_file;
-    int buf_contig,boff,i,nblks;
-    globus_off_t start,end,goff;
-    globus_size_t bytes_written;
-    globus_result_t result;
-
-    MPI_Comm_rank(fd->comm,&myrank);
-    MPI_Comm_size(fd->comm,&nprocs);
-    etype_size=fd->etype_size;
-    MPI_Type_size_x(fd->filetype,&ftype_size);
-    MPI_Type_get_extent(fd->filetype,&ftype_lb,&ftype_extent);
-    /* This is arguably unnecessary, as this routine assumes that the
-       buffer in memory is contiguous */
-    MPI_Type_size_x(datatype,&btype_size);
-    MPI_Type_get_extent(datatype,&btype_lb,&btype_extent);
-    ADIOI_Datatype_iscontig(datatype,&buf_contig);
-    
-    if ( ( btype_extent!=btype_size ) || ( ! buf_contig ) )
-	{
-	    FPRINTF(stderr,"[%d/%d] %s called with discontigous memory buffer\n",
-		    myrank,nprocs,myname);
-	    fflush(stderr);
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-		    myname, __LINE__, MPI_ERR_IO,
-		    "**io",
-		    "**io %s", globus_object_printable_to_string(globus_error_get(result)));
-	    return;
-	}
-    /* from here we can assume btype_extent==btype_size */
-
-    /* Flatten out fd->filetype so we know which blocks to skip */
-    flat_file = ADIOI_Flatten_and_find(fd->filetype);
-
-    /* Figure out how big the area to write is */
-    /* ASSUMPTION: ftype_size is an integer multiple of btype_size or vice versa. */
-    start=(globus_off_t)(offset*etype_size);
-    goff=start;
-    boff=0;
-    extent=0;
-    nblks=0;
-    while ( boff < (count*btype_size) )
-	{
-	    int blklen;
-
-	    for (i=0;i<flat_file->count;i++)
-		{
-		    if ( (boff+flat_file->blocklens[i]) < (count*btype_size) )
-			blklen=flat_file->blocklens[i];
-		    else
-			blklen=(count*btype_size)-boff;
-		    boff+=blklen;
-		    extent=MAX(extent,nblks*ftype_extent+flat_file->indices[i]+blklen);
-		    if ( boff>=(count*btype_size) )
-			break;
-		}
-	    nblks++;
-	}
-    if ( extent < count*btype_size )
-	{
-	    FPRINTF(stderr,"[%d/%d] %s error in computing extent -- extent %d is smaller than total bytes requested %d!\n",
-		    myrank,nprocs,myname,extent,count*btype_size);
-	    fflush(stderr);
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-		    myname, __LINE__, MPI_ERR_IO,
-		    "**io",
-		    "**io %s", globus_object_printable_to_string(globus_error_get(result)));
-	    return;
-	}
-    end=start+(globus_off_t)extent;
-    FPRINTF(stderr,"[%d/%d] %s writing %d bytes into extent of %d bytes starting at offset %Ld\n",
-	    myrank,nprocs,myname,count*btype_size,extent,(long long)start);
-    fflush(stderr);
-
-    /* start up the globus partial write */
-    globus_mutex_init(&writediscontig_ctl_lock, GLOBUS_NULL);
-    globus_cond_init(&writediscontig_ctl_cond, GLOBUS_NULL);
-    writediscontig_ctl_done=GLOBUS_FALSE;
-    if ( (result=globus_ftp_client_partial_put(&(gridftp_fh[fd->fd_sys]),
-					       fd->filename,
-					       &(oattr[fd->fd_sys]),
-					       GLOBUS_NULL,
-					       start,
-					       end,
-					       writediscontig_ctl_cb,
-					       GLOBUS_NULL))!=GLOBUS_SUCCESS )
-	{
-	    globus_err_handler("globus_ftp_client_partial_get",myname,result);
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-		    myname, __LINE__, MPI_ERR_IO,
-		    "**io",
-		    "**io %s", globus_object_printable_to_string(globus_error_get(result)));
-	    return;
-	}
-
-    /* Do all the actual I/Os */
-    boff=0;
-    nblks=0;
-    while ( boff < (count*btype_size) )
-	{
-	    int i,blklen;
-
-	    for (i=0;i<flat_file->count;i++)
-		{
-		    if ( (boff+flat_file->blocklens[i]) < (count*btype_size) )
-			blklen=flat_file->blocklens[i];
-		    else
-			blklen=(count*btype_size)-boff;
-		    if ( blklen > 0 )
-			{
-			    goff=start+nblks*ftype_extent+((globus_off_t)flat_file->indices[i]);
-			    /*
-			    FPRINTF(stderr,"[%d/%d] %s writing %d bytes from boff=%d at goff=%Ld\n",myrank,nprocs,myname,blklen,boff,goff);
-			    */
-			    if ( (result=globus_ftp_client_register_write(&(gridftp_fh[fd->fd_sys]),
-									  ((globus_byte_t *)buf)+boff,
-									  (globus_size_t)blklen,
-									  goff,
-									  GLOBUS_TRUE,
-									  writediscontig_data_cb,
-									  (void *)(&bytes_written)))!=GLOBUS_SUCCESS )
-				{
-				    globus_err_handler("globus_ftp_client_register_write",myname,result);
-				    *error_code=MPI_ERR_IO;
-				    ADIOI_Error(fd,*error_code,myname);
-				    return;
-				}
-			    boff+=blklen;
-			    if ( boff>=(count*btype_size) )
-				break;
-			}
-		}
-	    nblks++;
-	}
-
-    
-    /* The ctl callback won't start till the data callbacks complete, so it's
-       safe to wait on just the ctl callback */
-    globus_mutex_lock(&writediscontig_ctl_lock);
-    while ( writediscontig_ctl_done!=GLOBUS_TRUE )
-	globus_cond_wait(&writediscontig_ctl_cond,&writediscontig_ctl_lock);
-    globus_mutex_unlock(&writediscontig_ctl_lock);
-    globus_mutex_destroy(&writediscontig_ctl_lock);
-    globus_cond_destroy(&writediscontig_ctl_cond);
-
-#ifdef HAVE_STATUS_SET_BYTES
-    MPIR_Status_set_bytes(status, datatype, bytes_written);
-#endif
-    if (file_ptr_type != ADIO_EXPLICIT_OFFSET)
-    {
-	fd->fp_ind += extent;
-	fd->fp_sys_posn = fd->fp_ind;
-    }
-    else {
-	fd->fp_sys_posn = offset + extent;
-    }
-}
-
-
-#define GRIDFTP_USE_GENERIC_STRIDED
-void ADIOI_GRIDFTP_WriteStrided(ADIO_File fd, void *buf, int count,
-			       MPI_Datatype datatype, int file_ptr_type,
-			       ADIO_Offset offset, ADIO_Status *status,
-			       int *error_code)
-{
-#ifdef GRIDFTP_USE_GENERIC_STRIDED
-    int myrank, nprocs;
-
-    if ( fd->access_mode&ADIO_RDONLY )
-	{
-	    *error_code=MPI_ERR_AMODE;
-	    return;
-	}
-
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-
-    ADIOI_GEN_WriteStrided(fd, buf, count, datatype, file_ptr_type, offset, 
-			   status, error_code);
-    return;
-#else
-    char myname[]="ADIOI_GRIDFTP_WriteStrided";
-    int myrank, nprocs;
-    int buf_contig,file_contig;
-    MPI_Aint btype_size,bufsize;
-    globus_byte_t *intermediate;
-
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-
-    MPI_Type_size_x(datatype,&btype_size);
-    bufsize=count*btype_size;
-    ADIOI_Datatype_iscontig(fd->filetype,&file_contig);
-    ADIOI_Datatype_iscontig(datatype,&buf_contig);
-    if ( buf_contig && !file_contig )
-	{
-	    /* Contiguous in memory, discontig in file */
-	    FPRINTF(stderr,"[%d/%d] %s called w/ contig mem, discontig file\n",
-		    myrank,nprocs,myname);
-	    fflush(stderr);
-
-	    ADIOI_GRIDFTP_WriteDiscontig(fd, buf, count, datatype,
-					file_ptr_type, offset, status, error_code);
-	}
-    else if ( !buf_contig && file_contig )
-	{
-	    /* Discontiguous in mem, contig in file -- comparatively easy */
-	    int posn=0;
-
-	    FPRINTF(stderr,"[%d/%d] %s called w/ discontig mem, contig file\n",
-		    myrank,nprocs,myname);
-	    fflush(stderr);
-
-
-	    /* squeeze contents of main buffer into intermediate buffer*/
-	    intermediate=(globus_byte_t *)ADIOI_Malloc((size_t)bufsize);
-	    MPI_Pack(buf,count,datatype,intermediate,bufsize,&posn,fd->comm);
-
-	    /* write contiguous data from intermediate buffer */
-	    ADIOI_GRIDFTP_WriteContig(fd, intermediate, bufsize, MPI_BYTE,
-				     file_ptr_type, offset, status, error_code);
-
-	    ADIOI_Free(intermediate);
-	}
-    else if ( !buf_contig && !file_contig )
-	{
-	    /* Discontig in both mem and file -- the hardest case */
-	    int posn=0;
-
-	    FPRINTF(stderr,"[%d/%d] %s called w/ discontig mem, discontig file\n",
-		    myrank,nprocs,myname);
-	    fflush(stderr);
-
-	    /* squeeze contents of main buffer into intermediate buffer*/
-	    intermediate=(globus_byte_t *)ADIOI_Malloc((size_t)bufsize);
-	    MPI_Pack(buf,count,datatype,intermediate,bufsize,&posn,fd->comm);
-
-	    /* write contiguous data from intermediate buffer */
-	    ADIOI_GRIDFTP_WriteDiscontig(fd, intermediate, bufsize, MPI_BYTE,
-				     file_ptr_type, offset, status, error_code);
-
-	    ADIOI_Free(intermediate);
-	}
-    else 
-	{
-	    /* Why did you bother calling WriteStrided?!?!?! */
-	    FPRINTF(stderr,"[%d/%d] Why the heck did you call %s with contiguous buffer *and* file types?\n",
-		    myrank,nprocs,myname);
-	    ADIOI_GRIDFTP_WriteContig(fd, buf, count, datatype,
-				      file_ptr_type, offset, status, error_code);
-	}
-#endif /* ! GRIDFTP_USE_GENERIC_STRIDED */
-}
-
diff --git a/3rd-party/romio321/adio/ad_gridftp/globus_routines.c b/3rd-party/romio321/adio/ad_gridftp/globus_routines.c
deleted file mode 100644
index 1cca367a3f7..00000000000
--- a/3rd-party/romio321/adio/ad_gridftp/globus_routines.c
+++ /dev/null
@@ -1,36 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2003 University of Chicago, Ohio Supercomputer Center. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include <unistd.h>
-#include <sys/types.h>
-#include <sys/uio.h>
-#include <fcntl.h>
-#include "adio.h"
-#include <globus_ftp_client.h>
-
-/* Here are the canonical definitions of the extern's referenced by
-   ad_gridftp.h */
-int num_gridftp_handles=0;
-#ifndef ADIO_GRIDFTP_HANDLES_MAX
-#define ADIO_GRIDFTP_HANDLES_MAX 200
-#endif /* ! ADIO_GRIDFTP_HANDLES_MAX */
-/* having to keep not one but two big global tables sucks... */
-globus_ftp_client_handle_t gridftp_fh[ADIO_GRIDFTP_HANDLES_MAX];
-globus_ftp_client_operationattr_t oattr[ADIO_GRIDFTP_HANDLES_MAX];
-
-void globus_err_handler(const char *routine, const char *caller,
-			globus_result_t result)
-{
-  int myrank,nprocs;
-  globus_object_t *err;
-
-  MPI_Comm_rank(MPI_COMM_WORLD,&myrank);
-  MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
-  err = globus_error_get(result);
-  FPRINTF(stderr, "[%d/%d] %s error \"%s\", called from %s\n",
-	  myrank,nprocs,routine,globus_object_printable_to_string(err),caller);
-}
diff --git a/3rd-party/romio321/adio/ad_hfs/Makefile.mk b/3rd-party/romio321/adio/ad_hfs/Makefile.mk
deleted file mode 100644
index 318b5076396..00000000000
--- a/3rd-party/romio321/adio/ad_hfs/Makefile.mk
+++ /dev/null
@@ -1,21 +0,0 @@
-## -*- Mode: Makefile; -*-
-## vim: set ft=automake :
-##
-## (C) 2011 by Argonne National Laboratory.
-##     See COPYRIGHT in top-level directory.
-##
-
-if BUILD_AD_HFS
-
-noinst_HEADERS += adio/ad_hfs/ad_hfs.h
-
-romio_other_sources +=          \
-    adio/ad_hfs/ad_hfs_read.c   \
-    adio/ad_hfs/ad_hfs_open.c   \
-    adio/ad_hfs/ad_hfs_write.c  \
-    adio/ad_hfs/ad_hfs_fcntl.c  \
-    adio/ad_hfs/ad_hfs_resize.c \
-    adio/ad_hfs/ad_hfs.c
-
-endif BUILD_AD_HFS
-
diff --git a/3rd-party/romio321/adio/ad_hfs/README b/3rd-party/romio321/adio/ad_hfs/README
deleted file mode 100644
index 933677b1770..00000000000
--- a/3rd-party/romio321/adio/ad_hfs/README
+++ /dev/null
@@ -1 +0,0 @@
-This code is no longer supported.
diff --git a/3rd-party/romio321/adio/ad_hfs/ad_hfs.c b/3rd-party/romio321/adio/ad_hfs/ad_hfs.c
deleted file mode 100644
index 5b34354cabe..00000000000
--- a/3rd-party/romio321/adio/ad_hfs/ad_hfs.c
+++ /dev/null
@@ -1,38 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_hfs.h"
-
-/* adioi.h has the ADIOI_Fns_struct define */
-#include "adioi.h"
-
-struct ADIOI_Fns_struct ADIO_HFS_operations = {
-    ADIOI_HFS_Open, /* Open */
-    ADIOI_HFS_ReadContig, /* ReadContig */
-    ADIOI_HFS_WriteContig, /* WriteContig */
-    ADIOI_GEN_ReadStridedColl, /* ReadStridedColl */
-    ADIOI_GEN_WriteStridedColl, /* WriteStridedColl */
-    ADIOI_GEN_SeekIndividual, /* SeekIndividual */
-    ADIOI_HFS_Fcntl, /* Fcntl */
-    ADIOI_GEN_SetInfo, /* SetInfo */
-    ADIOI_GEN_ReadStrided, /* ReadStrided */
-    ADIOI_GEN_WriteStrided, /* WriteStrided */
-    ADIOI_GEN_Close, /* Close */
-    ADIOI_FAKE_IreadContig, /* IreadContig */
-    ADIOI_FAKE_IwriteContig, /* IwriteContig */
-    ADIOI_FAKE_IODone, /* ReadDone */
-    ADIOI_FAKE_IODone, /* WriteDone */
-    ADIOI_FAKE_IOComplete, /* ReadComplete */
-    ADIOI_FAKE_IOComplete, /* WriteComplete */
-    ADIOI_FAKE_IreadStrided, /* IreadStrided */
-    ADIOI_FAKE_IwriteStrided, /* IwriteStrided */
-    ADIOI_GEN_Flush, /* Flush */
-    ADIOI_HFS_Resize, /* Resize */
-    ADIOI_GEN_Delete, /* Delete */
-    ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
-    ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
-};
diff --git a/3rd-party/romio321/adio/ad_hfs/ad_hfs.h b/3rd-party/romio321/adio/ad_hfs/ad_hfs.h
deleted file mode 100644
index 2950aa50fcf..00000000000
--- a/3rd-party/romio321/adio/ad_hfs/ad_hfs.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#ifndef AD_HFS_INCLUDE
-#define AD_HFS_INCLUDE
-
-#include <unistd.h>
-#include <sys/types.h>
-#include <fcntl.h>
-#include "adio.h"
-#ifdef SPPUX
-#  include <sys/cnx_types.h>
-#  include <sys/puio.h>
-#  include <cnx_unistd.h>
-#endif
-
-void ADIOI_HFS_Open(ADIO_File fd, int *error_code);
-void ADIOI_HFS_ReadContig(ADIO_File fd, void *buf, int count, 
-                      MPI_Datatype datatype, int file_ptr_type,
-                     ADIO_Offset offset, ADIO_Status *status, int
-		     *error_code);
-void ADIOI_HFS_WriteContig(ADIO_File fd, void *buf, int count, 
-                      MPI_Datatype datatype, int file_ptr_type,
-                      ADIO_Offset offset, ADIO_Status *status, int
-		      *error_code);   
-void ADIOI_HFS_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct, int
-		*error_code); 
-void ADIOI_HFS_Resize(ADIO_File fd, ADIO_Offset size, int *error_code);
-
-#endif
diff --git a/3rd-party/romio321/adio/ad_hfs/ad_hfs_fcntl.c b/3rd-party/romio321/adio/ad_hfs/ad_hfs_fcntl.c
deleted file mode 100644
index 460c73666fe..00000000000
--- a/3rd-party/romio321/adio/ad_hfs/ad_hfs_fcntl.c
+++ /dev/null
@@ -1,113 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_hfs.h"
-#include "adio_extern.h"
-
-#ifndef HAVE_LSEEK64
-#define lseek64 lseek
-#endif
-void ADIOI_HFS_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct, int *error_code)
-{
-    int  i, ntimes, err;
-    ADIO_Offset curr_fsize, alloc_size, size, len, done;
-    ADIO_Status status;
-    char *buf;
-#ifndef PRINT_ERR_MSG
-    static char myname[] = "ADIOI_HFS_FCNTL";
-#endif
-
-    switch(flag) {
-    case ADIO_FCNTL_GET_FSIZE:
-	fcntl_struct->fsize = lseek64(fd->fd_sys, 0, SEEK_END);
-#ifdef HPUX
-	if (fd->fp_sys_posn != -1) 
-	     lseek64(fd->fd_sys, fd->fp_sys_posn, SEEK_SET);
-/* not required in SPPUX since there we use pread/pwrite */
-#endif
-	if (fcntl_struct->fsize == -1) {
-#ifdef MPICH
-	    *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", 
-		"**io %s", strerror(errno));
-#elif defined(PRINT_ERR_MSG)
-		*error_code = MPI_ERR_UNKNOWN;
-#else /* MPICH-1 */
-		*error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,
-				myname, "I/O Error", "%s", strerror(errno));
-		ADIOI_Error(fd, *error_code, myname);	    
-#endif
-	}
-	else *error_code = MPI_SUCCESS;
-	break;
-
-    case ADIO_FCNTL_SET_DISKSPACE:
-	/* will be called by one process only */
-
-#ifdef HPUX
-	err = prealloc64(fd->fd_sys, fcntl_struct->diskspace);
-	/* prealloc64 works only if file is of zero length */
-	if (err && (errno != ENOTEMPTY)) {
-#ifdef MPICH
-	    *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", 
-		"**io %s", strerror(errno));
-#elif defined(PRINT_ERR_MSG)
-	    *error_code = MPI_ERR_UNKNOWN;
-#else
-	    *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,
-			      myname, "I/O Error", "%s", strerror(errno));
-	    ADIOI_Error(fd, *error_code, myname);
-#endif
-	    return;
-	}
-	if (err && (errno == ENOTEMPTY)) {
-#endif
-
-#ifdef SPPUX
-	/* SPPUX has no prealloc64. therefore, use prealloc
-           if size < (2GB - 1), otherwise use long method. */
-        if (fcntl_struct->diskspace <= 2147483647) {
-	    err = prealloc(fd->fd_sys, (off_t) fcntl_struct->diskspace);
-	    if (err && (errno != ENOTEMPTY)) {
-#ifdef MPICH
-		*error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io",
-		    "**io %s", strerror(errno));
-#elif defined(PRINT_ERR_MSG)
-    	        *error_code = MPI_ERR_UNKNOWN;
-#else
-		*error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,
-			      myname, "I/O Error", "%s", strerror(errno));
-		ADIOI_Error(fd, *error_code, myname);
-#endif
-	        return;
-	    }
-	}    
-
-	if ((fcntl_struct->diskspace > 2147483647) || 
-	    (err && (errno == ENOTEMPTY))) {
-#endif
-		ADIOI_GEN_Prealloc(fd,fcntl_struct->diskspace, error_code);
-	    }
-	    ADIOI_Free(buf);
-#ifdef HPUX
-	    if (fd->fp_sys_posn != -1) 
-		lseek64(fd->fd_sys, fd->fp_sys_posn, SEEK_SET);
-	    /* not required in SPPUX since there we use pread/pwrite */
-#endif
-	}
-	*error_code = MPI_SUCCESS;
-	break;
-
-    case ADIO_FCNTL_SET_ATOMICITY:
-	fd->atomicity = (fcntl_struct->atomicity == 0) ? 0 : 1;
-	*error_code = MPI_SUCCESS;
-	break;
-
-    default:
-	FPRINTF(stderr, "Unknown flag passed to ADIOI_HFS_Fcntl\n");
-	MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-}
diff --git a/3rd-party/romio321/adio/ad_hfs/ad_hfs_open.c b/3rd-party/romio321/adio/ad_hfs/ad_hfs_open.c
deleted file mode 100644
index 46ee2848e56..00000000000
--- a/3rd-party/romio321/adio/ad_hfs/ad_hfs_open.c
+++ /dev/null
@@ -1,67 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_hfs.h"
-
-#ifndef HAVE_LSEEK64
-#define lseek64 lseek
-#endif
-
-void ADIOI_HFS_Open(ADIO_File fd, int *error_code)
-{
-    int perm, old_mask, amode;
-#ifndef PRINT_ERR_MSG
-    static char myname[] = "ADIOI_HFS_OPEN";
-#endif
-
-    if (fd->perm == ADIO_PERM_NULL) {
-	old_mask = umask(022);
-	umask(old_mask);
-	perm = old_mask ^ 0666;
-    }
-    else perm = fd->perm;
-
-    amode = 0;
-    if (fd->access_mode & ADIO_CREATE)
-	amode = amode | O_CREAT;
-    if (fd->access_mode & ADIO_RDONLY)
-	amode = amode | O_RDONLY;
-    if (fd->access_mode & ADIO_WRONLY)
-	amode = amode | O_WRONLY;
-    if (fd->access_mode & ADIO_RDWR)
-	amode = amode | O_RDWR;
-    if (fd->access_mode & ADIO_EXCL)
-	amode = amode | O_EXCL;
-
-    fd->fd_sys = open64(fd->filename, amode, perm);
-    fd->fd_direct = -1;
-
-    if ((fd->fd_sys != -1) && (fd->access_mode & ADIO_APPEND)) {
-	fd->fp_ind = lseek64(fd->fd_sys, 0, SEEK_END);
-#ifdef HPUX
-	fd->fp_sys_posn = fd->fp_ind;
-#endif
-    }
-
-#ifdef SPPUX
-	fd->fp_sys_posn = -1;  /* set it to null bec. we use pread, pwrite*/
-#endif
-
-	if (fd->fd_sys == -1 ) {
-#ifdef MPICH
-	    *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", 
-		"**io %s", strerror(errno));
-#elif defined(PRINT_ERR_MSG)
-	    *error_code = MPI_ERR_UNKNOWN;
-#else /* MPICH-1 */
-	*error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,
-			      myname, "I/O Error", "%s", strerror(errno));
-	ADIOI_Error(ADIO_FILE_NULL, *error_code, myname);	    
-#endif
-    }
-    else *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_hfs/ad_hfs_read.c b/3rd-party/romio321/adio/ad_hfs/ad_hfs_read.c
deleted file mode 100644
index eac03ab6f80..00000000000
--- a/3rd-party/romio321/adio/ad_hfs/ad_hfs_read.c
+++ /dev/null
@@ -1,71 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_hfs.h"
-
-#ifndef HAVE_LSEEK64
-#define lseek64 lseek
-#endif
-
-void ADIOI_HFS_ReadContig(ADIO_File fd, void *buf, int count, 
-                     MPI_Datatype datatype, int file_ptr_type,
-		     ADIO_Offset offset, ADIO_Status *status, int *error_code)
-{
-    MPI_Count err=-1, datatype_size, len;
-#ifndef PRINT_ERR_MSG
-    static char myname[] = "ADIOI_HFS_READCONTIG";
-#endif
-
-    MPI_Type_size_x(datatype, &datatype_size);
-    len = datatype_size * count;
-
-#ifdef SPPUX
-    fd->fp_sys_posn = -1; /* set it to null, since we are using pread */
-
-    if (file_ptr_type == ADIO_EXPLICIT_OFFSET) 
-	err = pread64(fd->fd_sys, buf, len, offset);
-    else {    /* read from curr. location of ind. file pointer */
-	err = pread64(fd->fd_sys, buf, len, fd->fp_ind);
-	fd->fp_ind += err;
-    }
-#endif
-
-#ifdef HPUX
-    if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-	if (fd->fp_sys_posn != offset)
-	    lseek64(fd->fd_sys, offset, SEEK_SET);
-	err = read(fd->fd_sys, buf, len);
-	fd->fp_sys_posn = offset + err;
-	/* individual file pointer not updated */        
-    }
-    else {  /* read from curr. location of ind. file pointer */
-	if (fd->fp_sys_posn != fd->fp_ind)
-	    lseek64(fd->fd_sys, fd->fp_ind, SEEK_SET);
-	err = read(fd->fd_sys, buf, len);
-	fd->fp_ind += err; 
-	fd->fp_sys_posn = fd->fp_ind;
-    }         
-#endif
-
-#ifdef HAVE_STATUS_SET_BYTES
-    if (err != -1) MPIR_Status_set_bytes(status, datatype, err);
-#endif
-
-	if (err == -1 ) {
-#ifdef MPICH
-	    *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io",
-		"**io %s", strerror(errno));
-#elif defined(PRINT_ERR_MSG)
-	    *error_code = (err == -1) ? MPI_ERR_UNKNOWN : MPI_SUCCESS;
-#else /* MPICH-1 */
-	*error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,
-			      myname, "I/O Error", "%s", strerror(errno));
-	ADIOI_Error(fd, *error_code, myname);	    
-#endif
-    }
-    else *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_hfs/ad_hfs_resize.c b/3rd-party/romio321/adio/ad_hfs/ad_hfs_resize.c
deleted file mode 100644
index de24ad672ae..00000000000
--- a/3rd-party/romio321/adio/ad_hfs/ad_hfs_resize.c
+++ /dev/null
@@ -1,31 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_hfs.h"
-
-void ADIOI_HFS_Resize(ADIO_File fd, ADIO_Offset size, int *error_code)
-{
-    int err;
-#ifndef PRINT_ERR_MSG
-    static char myname[] = "ADIOI_HFS_RESIZE";
-#endif
-    
-    err = ftruncate64(fd->fd_sys, size);
-    if (err == -1) {
-#ifdef MPICH
-	*error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io",
-	    "**io %s", strerror(errno));
-#elif defined(PRINT_ERR_MSG)
-	*error_code = MPI_ERR_UNKNOWN;
-#else /* MPICH-1 */
-	*error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,
-			      myname, "I/O Error", "%s", strerror(errno));
-	ADIOI_Error(fd, *error_code, myname);	    
-#endif
-    }
-    else *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_hfs/ad_hfs_write.c b/3rd-party/romio321/adio/ad_hfs/ad_hfs_write.c
deleted file mode 100644
index 4bffa82a2f9..00000000000
--- a/3rd-party/romio321/adio/ad_hfs/ad_hfs_write.c
+++ /dev/null
@@ -1,70 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_hfs.h"
-
-#ifndef HAVE_LSEEK64
-#define lseek64 lseek
-#endif
-
-void ADIOI_HFS_WriteContig(ADIO_File fd, void *buf, int count, 
-                     MPI_Datatype datatype, int file_ptr_type,
-		     ADIO_Offset offset, ADIO_Status *status, int *error_code)
-{
-    MPI_Count err=-1, datatype_size, len;
-#ifndef PRINT_ERR_MSG
-    static char myname[] = "ADIOI_HFS_WRITECONTIG";
-#endif
-
-    MPI_Type_size_x(datatype, &datatype_size);
-    len = datatype_size * count;
-
-#ifdef SPPUX
-    fd->fp_sys_posn = -1; /* set it to null, since we are using pwrite */
-    if (file_ptr_type == ADIO_EXPLICIT_OFFSET) 
-	err = pwrite64(fd->fd_sys, buf, len, offset);
-    else {    /* write from curr. location of ind. file pointer */
-	err = pwrite64(fd->fd_sys, buf, len, fd->fp_ind);
-	fd->fp_ind += err;
-    }
-#endif
-
-#ifdef HPUX
-    if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-	if (fd->fp_sys_posn != offset)
-	    lseek64(fd->fd_sys, offset, SEEK_SET);
-	err = write(fd->fd_sys, buf, len);
-	fd->fp_sys_posn = offset + err;
-	/* individual file pointer not updated */        
-    }
-    else { /* write from curr. location of ind. file pointer */
-	if (fd->fp_sys_posn != fd->fp_ind)
-	    lseek64(fd->fd_sys, fd->fp_ind, SEEK_SET);
-	err = write(fd->fd_sys, buf, len);
-	fd->fp_ind += err;
-	fd->fp_sys_posn = fd->fp_ind;
-    }
-#endif
-
-#ifdef HAVE_STATUS_SET_BYTES
-    if (err != -1) MPIR_Status_set_bytes(status, datatype, err);
-#endif
-
-    if (err == -1) {
-#ifdef MPICH
-	*error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io",
-	    "**io %s", strerror(errno));
-#elif defined(PRINT_ERR_MSG)
-    *error_code =  MPI_SUCCESS;
-#else /* MPICH-1 */
-	*error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,
-			      myname, "I/O Error", "%s", strerror(errno));
-	ADIOI_Error(fd, *error_code, myname);
-#endif
-    }
-    else *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_lustre/Makefile.mk b/3rd-party/romio321/adio/ad_lustre/Makefile.mk
deleted file mode 100644
index 43eaa025b94..00000000000
--- a/3rd-party/romio321/adio/ad_lustre/Makefile.mk
+++ /dev/null
@@ -1,22 +0,0 @@
-## -*- Mode: Makefile; -*-
-## vim: set ft=automake :
-##
-## (C) 2011 by Argonne National Laboratory.
-##     See COPYRIGHT in top-level directory.
-##
-
-if BUILD_AD_LUSTRE
-
-noinst_HEADERS += adio/ad_lustre/ad_lustre.h
-
-romio_other_sources +=                   \
-    adio/ad_lustre/ad_lustre.c           \
-    adio/ad_lustre/ad_lustre_open.c      \
-    adio/ad_lustre/ad_lustre_rwcontig.c  \
-    adio/ad_lustre/ad_lustre_wrcoll.c    \
-    adio/ad_lustre/ad_lustre_wrstr.c     \
-    adio/ad_lustre/ad_lustre_hints.c     \
-    adio/ad_lustre/ad_lustre_aggregate.c
-
-endif BUILD_AD_LUSTRE
-
diff --git a/3rd-party/romio321/adio/ad_lustre/README b/3rd-party/romio321/adio/ad_lustre/README
deleted file mode 100644
index a217c0f8fe5..00000000000
--- a/3rd-party/romio321/adio/ad_lustre/README
+++ /dev/null
@@ -1,55 +0,0 @@
-Upcoming soon: 
-  o Hierarchical striping as described in the paper from CCGrid2007
-    http://ft.ornl.gov/projects/io/pubs/CCGrid-2007-file-joining.pdf
-Further out:
-  o To post the code for ParColl (Partitioned collective IO)
- 
------------------------------------------------------
-V05: 
------------------------------------------------------
-Improved data redistribution
-  o Improve I/O pattern identification. Besides checking interleaving,
-    if request I/O size is small, collective I/O will be performed.
-    The hint bigsize can be used to define the req size value.
-  o Provide hint CO for load balancing to control the number of
-    IO clients for each OST
-  o Produce stripe-contiguous I/O pattern that Lustre prefers
-  o Control read-modify-write in data sieving in collective IO
-    by hint ds_in_coll.
-  o Reduce extent lock conflicts by make each OST accessed by one or
-    more constant clients.
-
------------------------------------------------------
-V04: 
------------------------------------------------------
-  o Direct IO and Lockless IO support
-
------------------------------------------------------
-V03: 
------------------------------------------------------
-  o Correct detection of fs_type when lustre: prefix is not given
-  o Further fix on stripe alignment
-  o Tested/Enabled striping hints over Cray XT (Catamount and CNL)
-
------------------------------------------------------
-V02:
------------------------------------------------------
-The Lustre ADIO driver has been cleaned up quite a lot. Compared 
-to the intital posting, here are the changes:
-  o Removal of dead/redundant code
-  o Removal of asynchronous IO piece as it appears outdated
-  o Bug fixes for setting Lustre Hints
-  o Bug fixes for data sieving 
-  o Improved Setsize operation with one process calling ftruncate 
-  o Improved collective IO with domain partitioning on 
-    Lustre stripe boundary
-
-Contributing:
-  o You may contribute via many different ways, such as 
-    testing results, bug reports, and new feature patches.
-  o We appreciate any courtesy reference of this work.
-  o Disclaimer: you are welcome to try the code, but at your own risk. 
-
-Contact info:
-    For more info, visit http://ft.ornl.gov/projects/io/
-
diff --git a/3rd-party/romio321/adio/ad_lustre/ad_lustre.c b/3rd-party/romio321/adio/ad_lustre/ad_lustre.c
deleted file mode 100644
index f3bc3d091b0..00000000000
--- a/3rd-party/romio321/adio/ad_lustre/ad_lustre.c
+++ /dev/null
@@ -1,46 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *   Copyright (C) 2001 University of Chicago.
- *   See COPYRIGHT notice in top-level directory.
- *
- *   Copyright (C) 2007 Oak Ridge National Laboratory
- *
- *   Copyright (C) 2008 Sun Microsystems, Lustre group
- */
-
-#include "ad_lustre.h"
-
-struct ADIOI_Fns_struct ADIO_LUSTRE_operations = {
-    ADIOI_LUSTRE_Open, /* Open */
-    ADIOI_GEN_OpenColl, /* OpenColl */
-    ADIOI_LUSTRE_ReadContig, /* ReadContig */
-    ADIOI_LUSTRE_WriteContig, /* WriteContig */
-    ADIOI_GEN_ReadStridedColl, /* ReadStridedColl */
-    ADIOI_LUSTRE_WriteStridedColl, /* WriteStridedColl */
-    ADIOI_GEN_SeekIndividual, /* SeekIndividual */
-    ADIOI_GEN_Fcntl, /* Fcntl */
-    ADIOI_LUSTRE_SetInfo, /* SetInfo */
-    ADIOI_GEN_ReadStrided, /* ReadStrided */
-    ADIOI_LUSTRE_WriteStrided, /* WriteStrided */
-    ADIOI_GEN_Close, /* Close */
-#if defined(ROMIO_HAVE_WORKING_AIO) && !defined(CRAY_XT_LUSTRE)
-    ADIOI_GEN_IreadContig, /* IreadContig */
-    ADIOI_GEN_IwriteContig, /* IwriteContig */
-#else
-    ADIOI_FAKE_IreadContig, /* IreadContig */
-    ADIOI_FAKE_IwriteContig, /* IwriteContig */
-#endif
-    ADIOI_GEN_IODone, /* ReadDone */
-    ADIOI_GEN_IODone, /* WriteDone */
-    ADIOI_GEN_IOComplete, /* ReadComplete */
-    ADIOI_GEN_IOComplete, /* WriteComplete */
-    ADIOI_GEN_IreadStrided, /* IreadStrided */
-    ADIOI_GEN_IwriteStrided, /* IwriteStrided */
-    ADIOI_GEN_Flush, /* Flush */
-    ADIOI_GEN_Resize, /* Resize */
-    ADIOI_GEN_Delete, /* Delete */
-    ADIOI_GEN_Feature, /* Features */
-    "LUSTRE:",
-    ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
-    ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
-};
diff --git a/3rd-party/romio321/adio/ad_lustre/ad_lustre.h b/3rd-party/romio321/adio/ad_lustre/ad_lustre.h
deleted file mode 100644
index edd2bd35efd..00000000000
--- a/3rd-party/romio321/adio/ad_lustre/ad_lustre.h
+++ /dev/null
@@ -1,95 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *   Copyright (C) 1997 University of Chicago.
- *   See COPYRIGHT notice in top-level directory.
- *
- *   Copyright (C) 2007 Oak Ridge National Laboratory
- *
- *   Copyright (C) 2008 Sun Microsystems, Lustre group
- */
-
-#ifndef AD_UNIX_INCLUDE
-#define AD_UNIX_INCLUDE
-
-/* temp*/
-#define HAVE_ASM_TYPES_H 1
-
-#include <unistd.h>
-#include <linux/types.h>
-
-#ifdef __linux__
-#  include <sys/ioctl.h>                            /* necessary for: */
-#  include <time.h>
-#  define __USE_GNU                                 /* O_DIRECT and */
-#  include <fcntl.h>                                /* IO operations */
-#  undef __USE_GNU
-#endif /* __linux__ */
-
-/*#include <fcntl.h>*/
-#include <sys/ioctl.h>
-#include <lustre/lustre_user.h>
-#include "adio.h"
-/*#include "adioi.h"*/
-
-#ifdef HAVE_SIGNAL_H
-#include <signal.h>
-#endif
-
-#ifdef HAVE_AIO_LITE_H
-#include <aio-lite.h>
-#else
- #ifdef  HAVE_AIO_H
- #include <aio.h>
- #endif
- #ifdef HAVE_SYS_AIO_H
- #include <sys/aio.h>
- #endif
-#endif /* End of HAVE_AIO_LITE_H */
-
-void ADIOI_LUSTRE_Open(ADIO_File fd, int *error_code);
-void ADIOI_LUSTRE_Close(ADIO_File fd, int *error_code);
-void ADIOI_LUSTRE_ReadContig(ADIO_File fd, void *buf, int count,
-                             MPI_Datatype datatype, int file_ptr_type,
-                             ADIO_Offset offset, ADIO_Status *status,
-                             int *error_code);
-void ADIOI_LUSTRE_WriteContig(ADIO_File fd, const void *buf, int count,
-                              MPI_Datatype datatype, int file_ptr_type,
-                              ADIO_Offset offset, ADIO_Status *status,
-                              int *error_code);
-void ADIOI_LUSTRE_WriteStrided(ADIO_File fd, const void *buf, int count,
-			       MPI_Datatype datatype, int file_ptr_type,
-			       ADIO_Offset offset, ADIO_Status *status,
-			       int *error_code);
-void ADIOI_LUSTRE_WriteStridedColl(ADIO_File fd, const void *buf, int count,
-		                   MPI_Datatype datatype, int file_ptr_type,
-		                   ADIO_Offset offset, ADIO_Status *status,
-                                   int *error_code);
-void ADIOI_LUSTRE_ReadStridedColl(ADIO_File fd, void *buf, int count,
-		                  MPI_Datatype datatype, int file_ptr_type,
-		                  ADIO_Offset offset, ADIO_Status *status,
-                                  int *error_code);
-void ADIOI_LUSTRE_ReadStrided(ADIO_File fd, void *buf, int count,
-			      MPI_Datatype datatype, int file_ptr_type,
-			      ADIO_Offset offset, ADIO_Status *status,
-                              int *error_code);
-void ADIOI_LUSTRE_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct,
-	               int *error_code);
-void ADIOI_LUSTRE_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code);
-
-/* the lustre utilities: */
-int ADIOI_LUSTRE_Docollect(ADIO_File fd, int contig_access_count,
-			   ADIO_Offset *len_list, int nprocs);
-
-void ADIOI_LUSTRE_Get_striping_info(ADIO_File fd, int **striping_info_ptr,
-				    int mode);
-void ADIOI_LUSTRE_Calc_my_req(ADIO_File fd, ADIO_Offset *offset_list,
-			      ADIO_Offset *len_list, int contig_access_count,
-			      int *striping_info, int nprocs,
-                              int *count_my_req_procs_ptr,
-			      int **count_my_req_per_proc_ptr,
-			      ADIOI_Access **my_req_ptr,
-			      int ***buf_idx_ptr);
-
-int ADIOI_LUSTRE_Calc_aggregator(ADIO_File fd, ADIO_Offset off,
-                                 ADIO_Offset *len, int *striping_info);
-#endif /* End of AD_UNIX_INCLUDE */
diff --git a/3rd-party/romio321/adio/ad_lustre/ad_lustre_aggregate.c b/3rd-party/romio321/adio/ad_lustre/ad_lustre_aggregate.c
deleted file mode 100644
index da54d71cf59..00000000000
--- a/3rd-party/romio321/adio/ad_lustre/ad_lustre_aggregate.c
+++ /dev/null
@@ -1,327 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *   Copyright (C) 1997 University of Chicago.
- *   See COPYRIGHT notice in top-level directory.
- *
- *   Copyright (C) 2007 Oak Ridge National Laboratory
- *
- *   Copyright (C) 2008 Sun Microsystems, Lustre group
- */
-
-#include "ad_lustre.h"
-#include "adio_extern.h"
-
-#undef AGG_DEBUG
-
-void ADIOI_LUSTRE_Get_striping_info(ADIO_File fd, int **striping_info_ptr,
-				    int mode)
-{
-    int *striping_info = NULL;
-    /* get striping information:
-     *  striping_info[0]: stripe_size
-     *  striping_info[1]: stripe_count
-     *  striping_info[2]: avail_cb_nodes
-     */
-    int stripe_size, stripe_count, CO = 1;
-    int avail_cb_nodes, divisor, nprocs_for_coll = fd->hints->cb_nodes;
-
-    /* Get hints value */
-    /* stripe size */
-    stripe_size = fd->hints->striping_unit;
-    /* stripe count */
-    /* stripe_size and stripe_count have been validated in ADIOI_LUSTRE_Open() */
-    stripe_count = fd->hints->striping_factor;
-
-    /* Calculate the available number of I/O clients */
-    if (!mode) {
-        /* for collective read,
-	 * if "CO" clients access the same OST simultaneously,
-	 * the OST disk seek time would be much. So, to avoid this,
-	 * it might be better if 1 client only accesses 1 OST.
-	 * So, we set CO = 1 to meet the above requirement.
-	 */
-	CO = 1;
-	/*XXX: maybe there are other better way for collective read */
-    } else {
-        /* CO also has been validated in ADIOI_LUSTRE_Open(), >0 */
-	CO = fd->hints->fs_hints.lustre.co_ratio;
-    }
-    /* Calculate how many IO clients we need */
-    /* Algorithm courtesy Pascal Deveze (pascal.deveze@bull.net) */
-    /* To avoid extent lock conflicts,
-     * avail_cb_nodes should either 
-     * 	- be a multiple of stripe_count,
-     *  - or divide stripe_count exactly
-     * so that each OST is accessed by a maximum of CO constant clients. */
-    if (nprocs_for_coll >= stripe_count)
-	/* avail_cb_nodes should be a multiple of stripe_count and the number
-	 * of procs per OST should be limited to the minimum between
-	 * nprocs_for_coll/stripe_count and CO 
-	 * 
-	 * e.g. if stripe_count=20, nprocs_for_coll=42 and CO=3 then 
-	 * avail_cb_nodes should be equal to 40 */
-        avail_cb_nodes = 
-		stripe_count * ADIOI_MIN(nprocs_for_coll/stripe_count, CO);
-    else {
-        /* nprocs_for_coll is less than stripe_count */
-        /* avail_cb_nodes should divide stripe_count */
-        /* e.g. if stripe_count=60 and nprocs_for_coll=8 then 
-	 * avail_cb_nodes should be egal to 6 */
-        /* This could be done with :
-                while (stripe_count % avail_cb_nodes != 0) avail_cb_nodes--;
-	   but this can be optimized for large values of nprocs_for_coll and
-	   stripe_count */
-        divisor = 2;
-        avail_cb_nodes = 1;
-        /* try to divise */
-        while (stripe_count >= divisor*divisor) {
-            if ((stripe_count % divisor) == 0) {
-                 if (stripe_count/divisor <= nprocs_for_coll) {
-                     /* The value is found ! */
-                     avail_cb_nodes = stripe_count/divisor;
-                     break;
-		}
-		/* if divisor is less than nprocs_for_coll, divisor is a
-		 * solution, but it is not sure that it is the best one */
-                else if (divisor <= nprocs_for_coll) 
-			avail_cb_nodes = divisor;
-	    }
-	    divisor++;
-        }
-    }
-
-    *striping_info_ptr = (int *) ADIOI_Malloc(3 * sizeof(int));
-    striping_info = *striping_info_ptr;
-    striping_info[0] = stripe_size;
-    striping_info[1] = stripe_count;
-    striping_info[2] = avail_cb_nodes;
-}
-
-int ADIOI_LUSTRE_Calc_aggregator(ADIO_File fd, ADIO_Offset off,
-                                 ADIO_Offset *len, int *striping_info)
-{
-    int rank_index, rank;
-    ADIO_Offset avail_bytes;
-    int stripe_size = striping_info[0];
-    int avail_cb_nodes = striping_info[2];
-
-    /* Produce the stripe-contiguous pattern for Lustre */
-    rank_index = (int)((off / stripe_size) % avail_cb_nodes);
-
-    /* we index into fd_end with rank_index, and fd_end was allocated to be no
-     * bigger than fd->hins->cb_nodes.   If we ever violate that, we're
-     * overrunning arrays.  Obviously, we should never ever hit this abort
-     */
-    if (rank_index >= fd->hints->cb_nodes)
-	    MPI_Abort(MPI_COMM_WORLD, 1);
-
-    avail_bytes = (off / (ADIO_Offset)stripe_size + 1) *
-                  (ADIO_Offset)stripe_size - off;
-    if (avail_bytes < *len) {
-	/* this proc only has part of the requested contig. region */
-	*len = avail_bytes;
-    }
-    /* map our index to a rank */
-    /* NOTE: FOR NOW WE DON'T HAVE A MAPPING...JUST DO 0..NPROCS_FOR_COLL */
-    rank = fd->hints->ranklist[rank_index];
-
-    return rank;
-}
-
-/* ADIOI_LUSTRE_Calc_my_req() - calculate what portions of the access requests
- * of this process are located in the file domains of various processes
- * (including this one)
- */
-
-
-void ADIOI_LUSTRE_Calc_my_req(ADIO_File fd, ADIO_Offset *offset_list,
-			      ADIO_Offset *len_list, int contig_access_count,
-			      int *striping_info, int nprocs,
-                              int *count_my_req_procs_ptr,
-			      int **count_my_req_per_proc_ptr,
-			      ADIOI_Access **my_req_ptr,
-			      int ***buf_idx_ptr)
-{
-    /* Nothing different from ADIOI_Calc_my_req(), except calling
-     * ADIOI_Lustre_Calc_aggregator() instead of the old one */
-    int *count_my_req_per_proc, count_my_req_procs, **buf_idx;
-    int i, l, proc;
-    ADIO_Offset avail_len, rem_len, curr_idx, off;
-    ADIOI_Access *my_req;
-
-    *count_my_req_per_proc_ptr = (int *) ADIOI_Calloc(nprocs, sizeof(int));
-    count_my_req_per_proc = *count_my_req_per_proc_ptr;
-    /* count_my_req_per_proc[i] gives the no. of contig. requests of this
-     * process in process i's file domain. calloc initializes to zero.
-     * I'm allocating memory of size nprocs, so that I can do an
-     * MPI_Alltoall later on.
-     */
-
-    buf_idx = (int **) ADIOI_Malloc(nprocs * sizeof(int*));
-
-    /* one pass just to calculate how much space to allocate for my_req;
-     * contig_access_count was calculated way back in ADIOI_Calc_my_off_len()
-     */
-    for (i = 0; i < contig_access_count; i++) {
-	/* short circuit offset/len processing if len == 0
-	 * (zero-byte  read/write
-	 */
-	if (len_list[i] == 0)
-	    continue;
-	off = offset_list[i];
-	avail_len = len_list[i];
-	/* note: we set avail_len to be the total size of the access.
-	 * then ADIOI_LUSTRE_Calc_aggregator() will modify the value to return
-	 * the amount that was available.
-	 */
-	proc = ADIOI_LUSTRE_Calc_aggregator(fd, off, &avail_len, striping_info);
-	count_my_req_per_proc[proc]++;
-
-	/* figure out how many data is remaining in the access
-	 * we'll take care of this data (if there is any)
-	 * in the while loop below.
-	 */
-	rem_len = len_list[i] - avail_len;
-
-	while (rem_len != 0) {
-	    off += avail_len;	/* point to first remaining byte */
-	    avail_len = rem_len;	/* save remaining size, pass to calc */
-	    proc = ADIOI_LUSTRE_Calc_aggregator(fd, off, &avail_len, striping_info);
-	    count_my_req_per_proc[proc]++;
-	    rem_len -= avail_len;	/* reduce remaining length by amount from fd */
-	}
-    }
-
-    /* buf_idx is relevant only if buftype_is_contig.
-     * buf_idx[i] gives the index into user_buf where data received
-     * from proc 'i' should be placed. This allows receives to be done
-     * without extra buffer. This can't be done if buftype is not contig.
-     */
-
-    /* initialize buf_idx vectors */
-    for (i = 0; i < nprocs; i++) {
-	/* add one to count_my_req_per_proc[i] to avoid zero size malloc */
-	buf_idx[i] = (int *) ADIOI_Malloc((count_my_req_per_proc[i] + 1)
-			                   * sizeof(int)); 
-    }
-
-    /* now allocate space for my_req, offset, and len */
-    *my_req_ptr = (ADIOI_Access *) ADIOI_Malloc(nprocs * sizeof(ADIOI_Access));
-    my_req = *my_req_ptr;
-
-    count_my_req_procs = 0;
-    for (i = 0; i < nprocs; i++) {
-	if (count_my_req_per_proc[i]) {
-	    my_req[i].offsets = (ADIO_Offset *)
-		                ADIOI_Malloc(count_my_req_per_proc[i] *
-                                             sizeof(ADIO_Offset));
-	    my_req[i].lens = ADIOI_Malloc(count_my_req_per_proc[i] *
-				                  sizeof(ADIO_Offset));
-	    count_my_req_procs++;
-	}
-	my_req[i].count = 0;	/* will be incremented where needed later */
-    }
-
-    /* now fill in my_req */
-    curr_idx = 0;
-    for (i = 0; i < contig_access_count; i++) {
-	/* short circuit offset/len processing if len == 0
-	 *	(zero-byte  read/write */
-	if (len_list[i] == 0)
-	    continue;
-	off = offset_list[i];
-	avail_len = len_list[i];
-	proc = ADIOI_LUSTRE_Calc_aggregator(fd, off, &avail_len, striping_info);
-
-	l = my_req[proc].count;
-
-	ADIOI_Assert(curr_idx == (int) curr_idx);
-	ADIOI_Assert(l < count_my_req_per_proc[proc]);
-	buf_idx[proc][l] = (int) curr_idx;
-	curr_idx += avail_len;
-
-	rem_len = len_list[i] - avail_len;
-
-	/* store the proc, offset, and len information in an array
-	 * of structures, my_req. Each structure contains the
-	 * offsets and lengths located in that process's FD,
-	 * and the associated count.
-	 */
-	my_req[proc].offsets[l] = off;
-	ADIOI_Assert(avail_len == (int) avail_len);
-	my_req[proc].lens[l] = (int) avail_len;
-	my_req[proc].count++;
-
-	while (rem_len != 0) {
-	    off += avail_len;
-	    avail_len = rem_len;
-	    proc = ADIOI_LUSTRE_Calc_aggregator(fd, off, &avail_len,
-                                                striping_info);
-
-	    l = my_req[proc].count;
-	    ADIOI_Assert(curr_idx == (int) curr_idx);
-	    ADIOI_Assert(l < count_my_req_per_proc[proc]);
-	    buf_idx[proc][l] = (int) curr_idx;
-
-	    curr_idx += avail_len;
-	    rem_len -= avail_len;
-
-	    my_req[proc].offsets[l] = off;
-	    ADIOI_Assert(avail_len == (int) avail_len);
-	    my_req[proc].lens[l] = (int) avail_len;
-	    my_req[proc].count++;
-	}
-    }
-
-#ifdef AGG_DEBUG
-    for (i = 0; i < nprocs; i++) {
-	if (count_my_req_per_proc[i] > 0) {
-	    FPRINTF(stdout, "data needed from %d (count = %d):\n",
-		            i, my_req[i].count);
-	    for (l = 0; l < my_req[i].count; l++) {
-		FPRINTF(stdout, "   off[%d] = %lld, len[%d] = %d\n",
-			        l, my_req[i].offsets[l], l, my_req[i].lens[l]);
-	    }
-	}
-    }
-#endif
-
-    *count_my_req_procs_ptr = count_my_req_procs;
-    *buf_idx_ptr = buf_idx;
-}
-
-int ADIOI_LUSTRE_Docollect(ADIO_File fd, int contig_access_count,
-			   ADIO_Offset *len_list, int nprocs)
-{
-    /* If the processes are non-interleaved, we will check the req_size.
-     *   if (avg_req_size > big_req_size) {
-     *       docollect = 0;
-     *   }
-     */
-
-    int i, docollect = 1, big_req_size = 0;
-    ADIO_Offset req_size = 0, total_req_size;
-    int avg_req_size, total_access_count;
-
-    /* calculate total_req_size and total_access_count */
-    for (i = 0; i < contig_access_count; i++)
-        req_size += len_list[i];
-    MPI_Allreduce(&req_size, &total_req_size, 1, MPI_LONG_LONG_INT, MPI_SUM,
-               fd->comm);
-    MPI_Allreduce(&contig_access_count, &total_access_count, 1, MPI_INT, MPI_SUM,
-               fd->comm);
-    /* avoid possible divide-by-zero) */
-    if (total_access_count != 0) {
-	/* estimate average req_size */
-	avg_req_size = (int)(total_req_size / total_access_count);
-    } else {
-	avg_req_size = 0;
-    }
-    /* get hint of big_req_size */
-    big_req_size = fd->hints->fs_hints.lustre.coll_threshold;
-    /* Don't perform collective I/O if there are big requests */
-    if ((big_req_size > 0) && (avg_req_size > big_req_size))
-        docollect = 0;
-
-    return docollect;
-}
diff --git a/3rd-party/romio321/adio/ad_lustre/ad_lustre_fcntl.c b/3rd-party/romio321/adio/ad_lustre/ad_lustre_fcntl.c
deleted file mode 100644
index a4bd6fc6d17..00000000000
--- a/3rd-party/romio321/adio/ad_lustre/ad_lustre_fcntl.c
+++ /dev/null
@@ -1,97 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- *
- *   Copyright (C) 2007 Oak Ridge National Laboratory
- */
-
-#include "ad_lustre.h"
-#include "adio_extern.h"
-
-void ADIOI_LUSTRE_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct, int *error_code)
-{
-    int i, ntimes;
-    ADIO_Offset curr_fsize, alloc_size, size, len, done;
-    ADIO_Status status;
-    char *buf;
-#if defined(MPICH) || !defined(PRINT_ERR_MSG)
-    static char myname[] = "ADIOI_LUSTRE_FCNTL";
-#endif
-
-    switch(flag) {
-    case ADIO_FCNTL_GET_FSIZE:
-	fcntl_struct->fsize = lseek(fd->fd_sys, 0, SEEK_END);
-	if (fd->fp_sys_posn != -1) 
-	     lseek(fd->fd_sys, fd->fp_sys_posn, SEEK_SET);
-	if (fcntl_struct->fsize == -1) {
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, 
-		    MPIR_ERR_RECOVERABLE, myname, __LINE__, 
-		    MPI_ERR_IO, "**io", "**io %s", strerror(errno));
-	}
-	else *error_code = MPI_SUCCESS;
-	break;
-
-    case ADIO_FCNTL_SET_DISKSPACE:
-	/* will be called by one process only */
-	/* On file systems with no preallocation function, I have to 
-           explicitly write 
-           to allocate space. Since there could be holes in the file, 
-           I need to read up to the current file size, write it back, 
-           and then write beyond that depending on how much 
-           preallocation is needed.
-           read/write in sizes of no more than ADIOI_PREALLOC_BUFSZ */
-
-	curr_fsize = lseek(fd->fd_sys, 0, SEEK_END);
-	alloc_size = fcntl_struct->diskspace;
-
-	size = ADIOI_MIN(curr_fsize, alloc_size);
-	
-	ntimes = (size + ADIOI_PREALLOC_BUFSZ - 1)/ADIOI_PREALLOC_BUFSZ;
-	buf = (char *) ADIOI_Malloc(ADIOI_PREALLOC_BUFSZ);
-	done = 0;
-
-	for (i=0; i<ntimes; i++) {
-	    len = ADIOI_MIN(size-done, ADIOI_PREALLOC_BUFSZ);
-	    ADIO_ReadContig(fd, buf, len, MPI_BYTE, ADIO_EXPLICIT_OFFSET, done,
-			    &status, error_code);
-	    if (*error_code != MPI_SUCCESS) {
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS, 
-			MPIR_ERR_RECOVERABLE, myname, __LINE__, 
-			MPI_ERR_IO, "**io", "**io %s", strerror(errno));
-                return;  
-	    }
-	    ADIO_WriteContig(fd, buf, len, MPI_BYTE, ADIO_EXPLICIT_OFFSET, 
-                             done, &status, error_code);
-	    if (*error_code != MPI_SUCCESS) return;
-	    done += len;
-	}
-
-	if (alloc_size > curr_fsize) {
-	    memset(buf, 0, ADIOI_PREALLOC_BUFSZ); 
-	    size = alloc_size - curr_fsize;
-	    ntimes = (size + ADIOI_PREALLOC_BUFSZ - 1)/ADIOI_PREALLOC_BUFSZ;
-	    for (i=0; i<ntimes; i++) {
-		len = ADIOI_MIN(alloc_size-done, ADIOI_PREALLOC_BUFSZ);
-		ADIO_WriteContig(fd, buf, len, MPI_BYTE, ADIO_EXPLICIT_OFFSET, 
-				 done, &status, error_code);
-		if (*error_code != MPI_SUCCESS) return;
-		done += len;  
-	    }
-	}
-	ADIOI_Free(buf);
-	if (fd->fp_sys_posn != -1) 
-	    lseek(fd->fd_sys, fd->fp_sys_posn, SEEK_SET);
-	*error_code = MPI_SUCCESS;
-	break;
-
-    case ADIO_FCNTL_SET_ATOMICITY:
-	fd->atomicity = (fcntl_struct->atomicity == 0) ? 0 : 1;
-	*error_code = MPI_SUCCESS;
-	break;
-
-    default:
-	FPRINTF(stderr, "Unknown flag passed to ADIOI_LUSTRE_Fcntl\n");
-	MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-}
diff --git a/3rd-party/romio321/adio/ad_lustre/ad_lustre_hints.c b/3rd-party/romio321/adio/ad_lustre/ad_lustre_hints.c
deleted file mode 100644
index 1d40c86f26a..00000000000
--- a/3rd-party/romio321/adio/ad_lustre/ad_lustre_hints.c
+++ /dev/null
@@ -1,140 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *   Copyright (C) 1997 University of Chicago.
- *   See COPYRIGHT notice in top-level directory.
- *
- *   Copyright (C) 2007 Oak Ridge National Laboratory
- *
- *   Copyright (C) 2008 Sun Microsystems, Lustre group
- */
-
-#include "ad_lustre.h"
-#include "adio_extern.h"
-#include "hint_fns.h"
-#ifdef HAVE_LIMITS_H
-#include <limits.h>
-#endif
-
-void ADIOI_LUSTRE_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code)
-{
-    char *value;
-    int flag;
-    ADIO_Offset stripe_val[3], str_factor = -1, str_unit=0, start_iodev=-1;
-    int myrank;
-    static char myname[] = "ADIOI_LUSTRE_SETINFO";
-
-    value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
-    if ( (fd->info) == MPI_INFO_NULL) {
-	/* This must be part of the open call. can set striping parameters
-           if necessary. */
-	MPI_Info_create(&(fd->info));
-
-	ADIOI_Info_set(fd->info, "direct_read", "false");
-	ADIOI_Info_set(fd->info, "direct_write", "false");
-	fd->direct_read = fd->direct_write = 0;
-        /* initialize lustre hints */
-	ADIOI_Info_set(fd->info, "romio_lustre_co_ratio", "1");
-        fd->hints->fs_hints.lustre.co_ratio = 1;
-	ADIOI_Info_set(fd->info, "romio_lustre_coll_threshold", "0");
-        fd->hints->fs_hints.lustre.coll_threshold = 0;
-	ADIOI_Info_set(fd->info, "romio_lustre_ds_in_coll", "enable");
-        fd->hints->fs_hints.lustre.ds_in_coll = ADIOI_HINT_ENABLE;
-
-	/* has user specified striping or server buffering parameters
-           and do they have the same value on all processes? */
-	if (users_info != MPI_INFO_NULL) {
-            /* striping information */
-	    ADIOI_Info_get(users_info, "striping_unit", MPI_MAX_INFO_VAL,
-			 value, &flag);
-	    if (flag) {
-		ADIOI_Info_set(fd->info, "striping_unit", value);
-		str_unit=atoll(value);
-	    }
-
-	    ADIOI_Info_get(users_info, "striping_factor", MPI_MAX_INFO_VAL,
-			 value, &flag);
-	    if (flag) {
-		ADIOI_Info_set(fd->info, "striping_factor", value);
-		str_factor=atoll(value);
-	    }
-
-	    ADIOI_Info_get(users_info, "romio_lustre_start_iodevice",
-                         MPI_MAX_INFO_VAL, value, &flag);
-	    if (flag) {
-		ADIOI_Info_set(fd->info, "romio_lustre_start_iodevice", value);
-		start_iodev=atoll(value);
-	    }
-
-
-            /* direct read and write */
-	    ADIOI_Info_get(users_info, "direct_read", MPI_MAX_INFO_VAL,
-			 value, &flag);
-	    if (flag && (!strcmp(value, "true") || !strcmp(value, "TRUE"))) {
-		ADIOI_Info_set(fd->info, "direct_read", "true");
-		fd->direct_read = 1;
-	    }
-	    ADIOI_Info_get(users_info, "direct_write", MPI_MAX_INFO_VAL,
-			     value, &flag);
-	    if (flag && (!strcmp(value, "true") || !strcmp(value, "TRUE"))) {
-		ADIOI_Info_set(fd->info, "direct_write", "true");
-		fd->direct_write = 1;
-	    }
-	}
-
-        /* set striping information with ioctl */
-	MPI_Comm_rank(fd->comm, &myrank);
-	if (myrank == 0) {
-	    stripe_val[0] = str_factor;
-	    stripe_val[1] = str_unit;
-	    stripe_val[2] = start_iodev;
-	}
-	MPI_Bcast(stripe_val, 3, MPI_OFFSET, 0, fd->comm);
-
-	/* do not open file in hint processing.   Open file in open routines,
-	 * where we can better deal with EXCL flag .  Continue to check the
-	 * "all processors set a value" condition holds.  */
-	if (stripe_val[0] != str_factor
-		|| stripe_val[1] != str_unit
-		|| stripe_val[2] != start_iodev) {
-		   MPIO_ERR_CREATE_CODE_INFO_NOT_SAME("ADIOI_LUSTRE_SetInfo",
-					       "str_factor or str_unit or start_iodev",
-					       error_code);
-		   ADIOI_Free(value);
-		   return;
-	}
-    }
-
-    /* get other hint */
-    if (users_info != MPI_INFO_NULL) {
-        /* CO: IO Clients/OST,
-         * to keep the load balancing between clients and OSTs */
-	ADIOI_Info_check_and_install_int(fd, users_info, "romio_lustre_co_ratio", 
-		&(fd->hints->fs_hints.lustre.co_ratio), myname, error_code );
-
-        /* coll_threshold:
-         * if the req size is bigger than this, collective IO may not be performed.
-         */
-	ADIOI_Info_check_and_install_int(fd, users_info, "romio_lustre_coll_threshold",
-		&(fd->hints->fs_hints.lustre.coll_threshold), myname, error_code );
-
-        /* ds_in_coll: disable data sieving in collective IO */
-	ADIOI_Info_check_and_install_enabled(fd, users_info, "romio_lustre_ds_in_coll",
-		&(fd->hints->fs_hints.lustre.ds_in_coll), myname, error_code );
-
-    }
-    /* set the values for collective I/O and data sieving parameters */
-    ADIOI_GEN_SetInfo(fd, users_info, error_code);
-
-    /* generic hints might step on striping_unit */
-    if (users_info != MPI_INFO_NULL) {
-	ADIOI_Info_check_and_install_int(fd, users_info, "striping_unit",
-		NULL, myname, error_code);
-    }
-
-    if (ADIOI_Direct_read) fd->direct_read = 1;
-    if (ADIOI_Direct_write) fd->direct_write = 1;
-
-    ADIOI_Free(value);
-
-    *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_lustre/ad_lustre_open.c b/3rd-party/romio321/adio/ad_lustre/ad_lustre_open.c
deleted file mode 100644
index a3b03be9eaf..00000000000
--- a/3rd-party/romio321/adio/ad_lustre/ad_lustre_open.c
+++ /dev/null
@@ -1,172 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *   Copyright (C) 1997 University of Chicago.
- *   See COPYRIGHT notice in top-level directory.
- *
- *   Copyright (C) 2007 Oak Ridge National Laboratory
- *
- *   Copyright (C) 2008 Sun Microsystems, Lustre group
- */
-
-#include "ad_lustre.h"
-
-/* what is the basis for this define?
- * what happens if there are more than 1k UUIDs? */
-
-#define MAX_LOV_UUID_COUNT      1000
-
-void ADIOI_LUSTRE_Open(ADIO_File fd, int *error_code)
-{
-    int perm, old_mask, amode, amode_direct;
-    int lumlen, myrank, flag, set_layout=0, err;
-    struct lov_user_md *lum = NULL;
-    char *value;
-    ADIO_Offset str_factor = -1, str_unit=0, start_iodev=-1;
-
-#if defined(MPICH) || !defined(PRINT_ERR_MSG)
-    static char myname[] = "ADIOI_LUSTRE_OPEN";
-#endif
-
-    MPI_Comm_rank(fd->comm, &myrank);
-
-    if (fd->perm == ADIO_PERM_NULL) {
-	old_mask = umask(022);
-	umask(old_mask);
-	perm = old_mask ^ 0666;
-    }
-    else perm = fd->perm;
-
-    amode = 0;
-    if (fd->access_mode & ADIO_CREATE)
-	amode = amode | O_CREAT;
-    if (fd->access_mode & ADIO_RDONLY)
-	amode = amode | O_RDONLY;
-    if (fd->access_mode & ADIO_WRONLY)
-	amode = amode | O_WRONLY;
-    if (fd->access_mode & ADIO_RDWR)
-	amode = amode | O_RDWR;
-    if (fd->access_mode & ADIO_EXCL)
-	amode = amode | O_EXCL;
-
-    amode_direct = amode | O_DIRECT;
-
-    /* odd length here because lov_user_md contains some fixed data and
-     * then a list of 'lmm_objects' representing stripe */
-    lumlen = sizeof(struct lov_user_md) +
-	    MAX_LOV_UUID_COUNT * sizeof(struct lov_user_ost_data);
-    lum = (struct lov_user_md *)ADIOI_Calloc(1,lumlen);
-
-     value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
-    /* we already validated in LUSTRE_SetInfo that these are going to be the same */
-    if (fd->info != MPI_INFO_NULL) {
-	/* striping information */
-	ADIOI_Info_get(fd->info, "striping_unit", MPI_MAX_INFO_VAL,
-		value, &flag);
-	if (flag)
-	    str_unit=atoll(value);
-
-	ADIOI_Info_get(fd->info, "striping_factor", MPI_MAX_INFO_VAL,
-		value, &flag);
-	if (flag)
-	    str_factor=atoll(value);
-
-	ADIOI_Info_get(fd->info, "romio_lustre_start_iodevice",
-		MPI_MAX_INFO_VAL, value, &flag);
-	if (flag)
-	    start_iodev=atoll(value);
-    }
-    if ((str_factor > 0) || (str_unit > 0) || (start_iodev >= 0))
-	set_layout = 1;
-
-    /* if hints were set, we need to delay creation of any lustre objects.
-     * However, if we open the file with O_LOV_DELAY_CREATE and don't call the
-     * follow-up ioctl, subsequent writes will fail */
-    if (myrank == 0 && set_layout)
-	amode = amode | O_LOV_DELAY_CREATE;
-
-    fd->fd_sys = open(fd->filename, amode, perm);
-    if (fd->fd_sys == -1) goto fn_exit;
-
-    /* we can only set these hints on new files */
-    /* It was strange and buggy to open the file in the hint path.  Instead,
-     * we'll apply the file tunings at open time */
-    if ((amode & O_CREAT) && set_layout ) {
-	/* if user has specified striping info, process 0 tries to set it */
-	if (!myrank) {
-	    lum->lmm_magic = LOV_USER_MAGIC;
-	    lum->lmm_pattern = 0;
-	    /* crude check for overflow of lustre internal datatypes.
-		 * Silently cap to large value if user provides a value
-		 * larger than lustre supports */
-	    if (str_unit > UINT_MAX)
-	            lum->lmm_stripe_size = UINT_MAX;
-	    else
-	            lum->lmm_stripe_size = str_unit;
-
-	    if (str_factor > USHRT_MAX)
-	            lum->lmm_stripe_count = USHRT_MAX;
-	    else
-	            lum->lmm_stripe_count = str_factor;
-
-	    if (start_iodev > USHRT_MAX)
-	             lum->lmm_stripe_offset = USHRT_MAX;
-	    else
-	            lum->lmm_stripe_offset = start_iodev;
-	    err = ioctl(fd->fd_sys, LL_IOC_LOV_SETSTRIPE, lum);
-	    if (err == -1 && errno != EEXIST) {
-		fprintf(stderr, "Failure to set stripe info %s \n", strerror(errno));
-		/* not a fatal error, but user might care to know */
-	    }
-	} /* End of striping parameters validation */
-    }
-
-    /* Pascal Deveze reports that, even though we pass a
-     * "GETSTRIPE" (read) flag to the ioctl, if some of the values of this
-     * struct are uninitialzed, the call can give an error.  zero it out in case
-     * there are other members that must be initialized and in case
-     * lov_user_md struct changes in future */
-    memset(lum, 0, lumlen);
-    lum->lmm_magic = LOV_USER_MAGIC;
-    err = ioctl(fd->fd_sys, LL_IOC_LOV_GETSTRIPE, (void *)lum);
-    if (!err) {
-
-	fd->hints->striping_unit = lum->lmm_stripe_size;
-	sprintf(value, "%d", lum->lmm_stripe_size);
-	ADIOI_Info_set(fd->info, "striping_unit", value);
-
-	fd->hints->striping_factor = lum->lmm_stripe_count;
-	sprintf(value, "%d", lum->lmm_stripe_count);
-	ADIOI_Info_set(fd->info, "striping_factor", value);
-
-	fd->hints->fs_hints.lustre.start_iodevice = lum->lmm_stripe_offset;
-	sprintf(value, "%d", lum->lmm_stripe_offset);
-	ADIOI_Info_set(fd->info, "romio_lustre_start_iodevice", value);
-
-    }
-
-    if (fd->access_mode & ADIO_APPEND)
-	fd->fp_ind = fd->fp_sys_posn = lseek(fd->fd_sys, 0, SEEK_END);
-
-    fd->fd_direct = -1;
-    if (fd->direct_write || fd->direct_read) {
-	fd->fd_direct = open(fd->filename, amode_direct, perm);
-	if (fd->fd_direct != -1) {
-	    fd->d_mem = fd->d_miniosz = (1<<12);
-	} else {
-	    perror("cannot open file with O_Direct");
-	    fd->direct_write = fd->direct_read = 0;
-	}
-    }
-
-fn_exit:
-    ADIOI_Free(lum);
-    ADIOI_Free(value);
-    /* --BEGIN ERROR HANDLING-- */
-    if (fd->fd_sys == -1 || ((fd->fd_direct == -1) && 
-		(fd->direct_write || fd->direct_read))) {
-	*error_code = ADIOI_Err_create_code(myname, fd->filename, errno);
-    }
-    /* --END ERROR HANDLING-- */
-    else *error_code = MPI_SUCCESS;
-
-}
diff --git a/3rd-party/romio321/adio/ad_lustre/ad_lustre_rwcontig.c b/3rd-party/romio321/adio/ad_lustre/ad_lustre_rwcontig.c
deleted file mode 100644
index cb187a39e9c..00000000000
--- a/3rd-party/romio321/adio/ad_lustre/ad_lustre_rwcontig.c
+++ /dev/null
@@ -1,203 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *   Copyright (C) 1997 University of Chicago.
- *   See COPYRIGHT notice in top-level directory.
- *
- *   Copyright (C) 2007 Oak Ridge National Laboratory
- *
- *   Copyright (C) 2008 Sun Microsystems, Lustre group
- */
-
-#include <unistd.h>
-
-#include <stdlib.h>
-#include <malloc.h>
-#include "ad_lustre.h"
-
-#define LUSTRE_MEMALIGN (1<<12)  /* to use page_shift */
-
-static void ADIOI_LUSTRE_Aligned_Mem_File_Write(ADIO_File fd, const void *buf, int len,
-              ADIO_Offset offset, int *err);
-static void ADIOI_LUSTRE_Aligned_Mem_File_Write(ADIO_File fd, const void *buf, int len,
-              ADIO_Offset offset, int *err)
-{
-    int rem, size, nbytes;
-    if (!(len % fd->d_miniosz) && (len >= fd->d_miniosz)) {
-	*err = pwrite(fd->fd_direct, buf, len, offset);
-    } else if (len < fd->d_miniosz) {
-	*err = pwrite(fd->fd_sys, buf, len, offset);
-    } else {
-	rem = len % fd->d_miniosz;
-	size = len - rem;
-	nbytes = pwrite(fd->fd_direct, buf, size, offset);
-	nbytes += pwrite(fd->fd_sys, ((char *)buf) + size, rem, offset+size);
-	*err = nbytes;
-    }
-}
-
-static void ADIOI_LUSTRE_Aligned_Mem_File_Read(ADIO_File fd, const void *buf, int len,
-              ADIO_Offset offset, int *err);
-static void ADIOI_LUSTRE_Aligned_Mem_File_Read(ADIO_File fd, const void *buf, int len,
-              ADIO_Offset offset, int *err)
-{
-    int rem, size, nbytes;
-    if (!(len % fd->d_miniosz) && (len >= fd->d_miniosz))
-	*err = pread(fd->fd_direct, (void *)buf, len, offset);
-    else if (len < fd->d_miniosz)
-	*err = pread(fd->fd_sys, (void *)buf, len, offset);
-    else {
-	rem = len % fd->d_miniosz;
-	size = len - rem;
-	nbytes = pread(fd->fd_direct, (void *)buf, size, offset);
-	nbytes += pread(fd->fd_sys, ((char *)buf) + size, rem, offset+size);
-	*err = nbytes;
-    }
-}
-
-
-static int ADIOI_LUSTRE_Directio(ADIO_File fd, const void *buf, int len,
-			   off_t offset, int rw);
-static int ADIOI_LUSTRE_Directio(ADIO_File fd, const void *buf, int len,
-			   off_t offset, int rw)
-{
-    int err=-1, diff, size=len, nbytes = 0;
-    void *newbuf;
-
-    if (offset % fd->d_miniosz) {
-	diff = fd->d_miniosz - (offset % fd->d_miniosz);
-	diff = ADIOI_MIN(diff, len);
-	if (rw)
-	    nbytes = pwrite(fd->fd_sys, (void *)buf, diff, offset);
-	else
-	    nbytes = pread(fd->fd_sys, (void *)buf, diff, offset);
-	buf = ((char *) buf) + diff;
-	offset += diff;
-	size = len - diff;
-    }
-
-    if (!size) {
-	return diff;
-    }
-
-    if (rw) { /* direct I/O enabled */
-	if (!(((long) buf) % fd->d_mem)) {
-	    ADIOI_LUSTRE_Aligned_Mem_File_Write(fd, buf, size, offset, &err);
-	    nbytes += err;
-	} else {
-	    newbuf = (void *) memalign(LUSTRE_MEMALIGN, size);
-	    if (newbuf) {
-		memcpy(newbuf, buf, size);
-		ADIOI_LUSTRE_Aligned_Mem_File_Write(fd, newbuf, size, offset, &err);
-		nbytes += err;
-		ADIOI_Free(newbuf);
-	    }
-	    else nbytes += pwrite(fd->fd_sys, buf, size, offset);
-	}
-	err = nbytes;
-    } else {       
-	if (!(((long) buf) % fd->d_mem)) {
-	    ADIOI_LUSTRE_Aligned_Mem_File_Read(fd, buf, size, offset, &err);
-	    nbytes += err;
-	} else {
-	    newbuf = (void *) memalign(LUSTRE_MEMALIGN, size);
-	    if (newbuf) {
-		ADIOI_LUSTRE_Aligned_Mem_File_Read(fd, newbuf, size, offset, &err);
-		if (err > 0) memcpy((void *)buf, newbuf, err);
-		nbytes += err;
-		ADIOI_Free(newbuf);
-	    }
-	    else nbytes += pread(fd->fd_sys, (void *)buf, size, offset);
-	}
-	err = nbytes;
-    }
-    return err;
-}
-
-static void ADIOI_LUSTRE_IOContig(ADIO_File fd, const void *buf, int count,
-                   MPI_Datatype datatype, int file_ptr_type,
-	           ADIO_Offset offset, ADIO_Status *status, 
-		   int io_mode, int *error_code);
-static void ADIOI_LUSTRE_IOContig(ADIO_File fd, const void *buf, int count,
-                   MPI_Datatype datatype, int file_ptr_type,
-	           ADIO_Offset offset, ADIO_Status *status, 
-		   int io_mode, int *error_code)
-{
-    int err=-1;
-    MPI_Count datatype_size, len;
-    static char myname[] = "ADIOI_LUSTRE_IOCONTIG";
-
-    MPI_Type_size_x(datatype, &datatype_size);
-    len = datatype_size * count;
-
-    if (file_ptr_type == ADIO_INDIVIDUAL) {
-	offset = fd->fp_ind;
-    }
-
-    if (!(fd->direct_read || fd->direct_write)) {
-	if (fd->fp_sys_posn != offset) {
-	    err = lseek(fd->fd_sys, offset, SEEK_SET);
-	    if (err == -1) goto ioerr;
-	}
-	
-	if (io_mode) {
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event(ADIOI_MPE_write_a, 0, NULL);
-#endif
-	    err = write(fd->fd_sys, buf, len);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event(ADIOI_MPE_write_b, 0, NULL);
-#endif
-        } else {
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event(ADIOI_MPE_read_a, 0, NULL);
-#endif
-	    err = read(fd->fd_sys, (void *)buf, len);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event(ADIOI_MPE_read_b, 0, NULL);
-#endif
-        }
-    } else {
-	err = ADIOI_LUSTRE_Directio(fd, buf, len, offset, io_mode);
-    }
-
-    if (err == -1) goto ioerr;
-    fd->fp_sys_posn = offset + err;
-
-    if (file_ptr_type == ADIO_INDIVIDUAL) {
-	fd->fp_ind += err; 
-    }
-
-#ifdef HAVE_STATUS_SET_BYTES
-    if (status) MPIR_Status_set_bytes(status, datatype, err);
-#endif
-    *error_code = MPI_SUCCESS;
-
-ioerr:
-    /* --BEGIN ERROR HANDLING-- */
-    if (err == -1) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   MPI_ERR_IO, "**io",
-					   "**io %s", strerror(errno));
-	fd->fp_sys_posn = -1;
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-}
-
-void ADIOI_LUSTRE_WriteContig(ADIO_File fd, const void *buf, int count,
-                   MPI_Datatype datatype, int file_ptr_type,
-	           ADIO_Offset offset, ADIO_Status *status, int *error_code)
-{
-    ADIOI_LUSTRE_IOContig(fd, buf, count, datatype, file_ptr_type,
-	           offset, status, 1, error_code);
-}
-
-void ADIOI_LUSTRE_ReadContig(ADIO_File fd, void *buf, int count, 
-                   MPI_Datatype datatype, int file_ptr_type,
-	           ADIO_Offset offset, ADIO_Status *status, int *error_code)
-{
-    ADIOI_LUSTRE_IOContig(fd, buf, count, datatype, file_ptr_type,
-	           offset, status, 0, error_code);
-}
diff --git a/3rd-party/romio321/adio/ad_lustre/ad_lustre_wrcoll.c b/3rd-party/romio321/adio/ad_lustre/ad_lustre_wrcoll.c
deleted file mode 100644
index c034d42827a..00000000000
--- a/3rd-party/romio321/adio/ad_lustre/ad_lustre_wrcoll.c
+++ /dev/null
@@ -1,986 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *   Copyright (C) 1997 University of Chicago.
- *   See COPYRIGHT notice in top-level directory.
- *
- *   Copyright (C) 2007 Oak Ridge National Laboratory
- *
- *   Copyright (C) 2008 Sun Microsystems, Lustre group
- */
-
-#include "ad_lustre.h"
-#include "adio_extern.h"
-
-/* prototypes of functions used for collective writes only. */
-static void ADIOI_LUSTRE_Exch_and_write(ADIO_File fd, const void *buf,
-					MPI_Datatype datatype, int nprocs,
-					int myrank,
-					ADIOI_Access *others_req,
-					ADIOI_Access *my_req,
-					ADIO_Offset *offset_list,
-					ADIO_Offset *len_list,
-					int contig_access_count,
-					int *striping_info,
-                                        int **buf_idx, int *error_code);
-static void ADIOI_LUSTRE_Fill_send_buffer(ADIO_File fd, const void *buf,
-					  ADIOI_Flatlist_node *flat_buf,
-					  char **send_buf,
-					  ADIO_Offset *offset_list,
-					  ADIO_Offset *len_list, int *send_size,
-					  MPI_Request *requests,
-					  int *sent_to_proc, int nprocs,
-					  int myrank, int contig_access_count,
-					  int *striping_info,
-					  int *send_buf_idx,
-                                          int *curr_to_proc,
-					  int *done_to_proc, int iter,
-					  MPI_Aint buftype_extent);
-static void ADIOI_LUSTRE_W_Exchange_data(ADIO_File fd, const void *buf,
-					 char *write_buf,
-					 ADIOI_Flatlist_node *flat_buf,
-					 ADIO_Offset *offset_list,
-					 ADIO_Offset *len_list, int *send_size,
-					 int *recv_size, ADIO_Offset off,
-					 int size, int *count,
-					 int *start_pos, 
-					 int *sent_to_proc, int nprocs,
-					 int myrank, int buftype_is_contig,
-					 int contig_access_count,
-					 int *striping_info,
-					 ADIOI_Access *others_req,
-					 int *send_buf_idx,
-					 int *curr_to_proc,
-					 int *done_to_proc, int *hole,
-					 int iter, MPI_Aint buftype_extent,
-					 int *buf_idx,
-					 ADIO_Offset **srt_off, int **srt_len, int *srt_num,
-					 int *error_code);
-void ADIOI_Heap_merge(ADIOI_Access *others_req, int *count,
-                      ADIO_Offset *srt_off, int *srt_len, int *start_pos,
-                      int nprocs, int nprocs_recv, int total_elements);
-
-void ADIOI_LUSTRE_WriteStridedColl(ADIO_File fd, const void *buf, int count,
-				   MPI_Datatype datatype,
-				   int file_ptr_type, ADIO_Offset offset,
-				   ADIO_Status *status, int *error_code)
-{
-    /* Uses a generalized version of the extended two-phase method described
-     * in "An Extended Two-Phase Method for Accessing Sections of
-     * Out-of-Core Arrays", Rajeev Thakur and Alok Choudhary,
-     * Scientific Programming, (5)4:301--317, Winter 1996.
-     * http://www.mcs.anl.gov/home/thakur/ext2ph.ps
-     */
-
-    ADIOI_Access *my_req;
-    /* array of nprocs access structures, one for each other process has
-       this process's request */
-
-    ADIOI_Access *others_req;
-    /* array of nprocs access structures, one for each other process
-       whose request is written by this process. */
-
-    int i, filetype_is_contig, nprocs, myrank, do_collect = 0;
-    int contig_access_count = 0, buftype_is_contig, interleave_count = 0;
-    int *count_my_req_per_proc, count_my_req_procs, count_others_req_procs;
-    ADIO_Offset orig_fp, start_offset, end_offset, off;
-    ADIO_Offset *offset_list = NULL, *st_offsets = NULL, *end_offsets = NULL;
-    ADIO_Offset *len_list = NULL;
-    int **buf_idx = NULL, *striping_info = NULL;
-    int old_error, tmp_error;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-
-    orig_fp = fd->fp_ind;
-
-    /* IO patten identification if cb_write isn't disabled */
-    if (fd->hints->cb_write != ADIOI_HINT_DISABLE) {
-	/* For this process's request, calculate the list of offsets and
-	   lengths in the file and determine the start and end offsets. */
-
-	/* Note: end_offset points to the last byte-offset that will be accessed.
-         * e.g., if start_offset=0 and 100 bytes to be read, end_offset=99
-         */
-
-	ADIOI_Calc_my_off_len(fd, count, datatype, file_ptr_type, offset,
-	                      &offset_list, &len_list, &start_offset,
-	                      &end_offset, &contig_access_count);
-
-	/* each process communicates its start and end offsets to other
-         * processes. The result is an array each of start and end offsets
-         * stored in order of process rank.
-         */
-	st_offsets = (ADIO_Offset *) ADIOI_Malloc(nprocs * sizeof(ADIO_Offset));
-	end_offsets = (ADIO_Offset *) ADIOI_Malloc(nprocs * sizeof(ADIO_Offset));
-	MPI_Allgather(&start_offset, 1, ADIO_OFFSET, st_offsets, 1,
-		      ADIO_OFFSET, fd->comm);
-	MPI_Allgather(&end_offset, 1, ADIO_OFFSET, end_offsets, 1,
-		      ADIO_OFFSET, fd->comm);
-	/* are the accesses of different processes interleaved? */
-	for (i = 1; i < nprocs; i++)
-	    if ((st_offsets[i] < end_offsets[i-1]) &&
-                (st_offsets[i] <= end_offsets[i]))
-                interleave_count++;
-	/* This is a rudimentary check for interleaving, but should suffice
-	   for the moment. */
-
-	/* Two typical access patterns can benefit from collective write.
-         *   1) the processes are interleaved, and
-         *   2) the req size is small.
-         */
-        if (interleave_count > 0) {
-	    do_collect = 1;
-        } else {
-            do_collect = ADIOI_LUSTRE_Docollect(fd, contig_access_count,
-			                        len_list, nprocs);
-        }
-    }
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-
-    /* Decide if collective I/O should be done */
-    if ((!do_collect && fd->hints->cb_write == ADIOI_HINT_AUTO) ||
-        fd->hints->cb_write == ADIOI_HINT_DISABLE) {
-
-	/* use independent accesses */
-	if (fd->hints->cb_write != ADIOI_HINT_DISABLE) {
-	    ADIOI_Free(offset_list);
-	    ADIOI_Free(len_list);
-            ADIOI_Free(st_offsets);
-            ADIOI_Free(end_offsets);
-	}
-
-	fd->fp_ind = orig_fp;
-	ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-	if (buftype_is_contig && filetype_is_contig) {
-	    if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-                off = fd->disp + (ADIO_Offset)(fd->etype_size) * offset;
-		ADIO_WriteContig(fd, buf, count, datatype,
-				 ADIO_EXPLICIT_OFFSET,
-				 off, status, error_code);
-	    } else
-		ADIO_WriteContig(fd, buf, count, datatype, ADIO_INDIVIDUAL,
-				 0, status, error_code);
-	} else {
-	    ADIO_WriteStrided(fd, buf, count, datatype, file_ptr_type,
-			      offset, status, error_code);
-	}
-	return;
-    }
-
-    /* Get Lustre hints information */
-    ADIOI_LUSTRE_Get_striping_info(fd, &striping_info, 1);
-
-    /* calculate what portions of the access requests of this process are
-     * located in which process
-     */
-    ADIOI_LUSTRE_Calc_my_req(fd, offset_list, len_list, contig_access_count,
-                             striping_info, nprocs, &count_my_req_procs,
-                             &count_my_req_per_proc, &my_req,
-                             &buf_idx);
-
-    /* based on everyone's my_req, calculate what requests of other processes
-     * will be accessed by this process.
-     * count_others_req_procs = number of processes whose requests (including
-     * this process itself) will be accessed by this process
-     * count_others_req_per_proc[i] indicates how many separate contiguous
-     * requests of proc. i will be accessed by this process.
-     */
-
-    ADIOI_Calc_others_req(fd, count_my_req_procs, count_my_req_per_proc,
-                          my_req, nprocs, myrank, &count_others_req_procs,
-                          &others_req);
-    ADIOI_Free(count_my_req_per_proc);
-
-    /* exchange data and write in sizes of no more than stripe_size. */
-    ADIOI_LUSTRE_Exch_and_write(fd, buf, datatype, nprocs, myrank,
-                                others_req, my_req, offset_list, len_list,
-                                contig_access_count, striping_info,
-                                buf_idx, error_code);
-
-    /* If this collective write is followed by an independent write,
-     * it's possible to have those subsequent writes on other processes
-     * race ahead and sneak in before the read-modify-write completes.
-     * We carry out a collective communication at the end here so no one
-     * can start independent i/o before collective I/O completes.
-     *
-     * need to do some gymnastics with the error codes so that if something
-     * went wrong, all processes report error, but if a process has a more
-     * specific error code, we can still have that process report the
-     * additional information */
-
-    old_error = *error_code;
-    if (*error_code != MPI_SUCCESS)
-	*error_code = MPI_ERR_IO;
-
-    /* optimization: if only one process performing i/o, we can perform
-     * a less-expensive Bcast  */
-#ifdef ADIOI_MPE_LOGGING
-    MPE_Log_event(ADIOI_MPE_postwrite_a, 0, NULL);
-#endif
-    if (fd->hints->cb_nodes == 1)
-	MPI_Bcast(error_code, 1, MPI_INT,
-		  fd->hints->ranklist[0], fd->comm);
-    else {
-	tmp_error = *error_code;
-	MPI_Allreduce(&tmp_error, error_code, 1, MPI_INT,
-		      MPI_MAX, fd->comm);
-    }
-#ifdef ADIOI_MPE_LOGGING
-    MPE_Log_event(ADIOI_MPE_postwrite_b, 0, NULL);
-#endif
-
-    if ((old_error != MPI_SUCCESS) && (old_error != MPI_ERR_IO))
-	*error_code = old_error;
-
-
-    if (!buftype_is_contig)
-	ADIOI_Delete_flattened(datatype);
-
-    /* free all memory allocated for collective I/O */
-    /* free others_req */
-    for (i = 0; i < nprocs; i++) {
-	if (others_req[i].count) {
-	    ADIOI_Free(others_req[i].offsets);
-	    ADIOI_Free(others_req[i].lens);
-	    ADIOI_Free(others_req[i].mem_ptrs);
-	}
-    }
-    ADIOI_Free(others_req);
-    /* free my_req here */
-    for (i = 0; i < nprocs; i++) {
-	if (my_req[i].count) {
-	    ADIOI_Free(my_req[i].offsets);
-	    ADIOI_Free(my_req[i].lens);
-	}
-    }
-    ADIOI_Free(my_req);
-    for (i = 0; i < nprocs; i++) {
-        ADIOI_Free(buf_idx[i]);
-    }
-    ADIOI_Free(buf_idx);
-    ADIOI_Free(offset_list);
-    ADIOI_Free(len_list);
-    ADIOI_Free(st_offsets);
-    ADIOI_Free(end_offsets);
-    ADIOI_Free(striping_info);
-
-#ifdef HAVE_STATUS_SET_BYTES
-    if (status) {
-	MPI_Count bufsize, size;
-	/* Don't set status if it isn't needed */
-	MPI_Type_size_x(datatype, &size);
-	bufsize = size * count;
-	MPIR_Status_set_bytes(status, datatype, bufsize);
-    }
-    /* This is a temporary way of filling in status. The right way is to
-     * keep track of how much data was actually written during collective I/O.
-     */
-#endif
-
-    fd->fp_sys_posn = -1;	/* set it to null. */
-}
-
-/* If successful, error_code is set to MPI_SUCCESS.  Otherwise an error
- * code is created and returned in error_code.
- */
-static void ADIOI_LUSTRE_Exch_and_write(ADIO_File fd, const void *buf,
-					MPI_Datatype datatype, int nprocs,
-					int myrank, ADIOI_Access *others_req,
-                                        ADIOI_Access *my_req,
-					ADIO_Offset *offset_list,
-                                        ADIO_Offset *len_list, 
-					int contig_access_count,
-                                        int *striping_info, int **buf_idx,
-                                        int *error_code)
-{
-    /* Send data to appropriate processes and write in sizes of no more
-     * than lustre stripe_size.
-     * The idea is to reduce the amount of extra memory required for
-     * collective I/O. If all data were written all at once, which is much
-     * easier, it would require temp space more than the size of user_buf,
-     * which is often unacceptable. For example, to write a distributed
-     * array to a file, where each local array is 8Mbytes, requiring
-     * at least another 8Mbytes of temp space is unacceptable.
-     */
-
-    int hole, i, j, m, flag, ntimes = 1 , max_ntimes, buftype_is_contig;
-    ADIO_Offset st_loc = -1, end_loc = -1, min_st_loc, max_end_loc;
-    ADIO_Offset off, req_off, send_off, iter_st_off, *off_list;
-    ADIO_Offset max_size, step_size = 0;
-    int real_size, req_len, send_len;
-    int *recv_curr_offlen_ptr, *recv_count, *recv_size;
-    int *send_curr_offlen_ptr, *send_size;
-    int *sent_to_proc, *recv_start_pos;
-    int *send_buf_idx, *curr_to_proc, *done_to_proc;
-    int *this_buf_idx;
-    char *write_buf = NULL;
-    MPI_Status status;
-    ADIOI_Flatlist_node *flat_buf = NULL;
-    MPI_Aint buftype_extent, buftype_lb;
-    int stripe_size = striping_info[0], avail_cb_nodes = striping_info[2];
-    int data_sieving = 0;
-    ADIO_Offset *srt_off = NULL;
-    int *srt_len = NULL;
-    int srt_num = 0;
-    ADIO_Offset block_offset;
-    int block_len;
-
-    *error_code = MPI_SUCCESS;	/* changed below if error */
-    /* only I/O errors are currently reported */
-
-    /* calculate the number of writes of stripe size to be done.
-     * That gives the no. of communication phases as well.
-     * Note:
-     *   Because we redistribute data in stripe-contiguous pattern for Lustre,
-     *   each process has the same no. of communication phases.
-     */
-
-    for (i = 0; i < nprocs; i++) {
-	if (others_req[i].count) {
-	    st_loc = others_req[i].offsets[0];
-	    end_loc = others_req[i].offsets[0];
-	    break;
-	}
-    }
-    for (i = 0; i < nprocs; i++) {
-	for (j = 0; j < others_req[i].count; j++) {
-	    st_loc = ADIOI_MIN(st_loc, others_req[i].offsets[j]);
-	    end_loc = ADIOI_MAX(end_loc, (others_req[i].offsets[j] +
-                                          others_req[i].lens[j] - 1));
-	}
-    }
-    /* this process does no writing. */
-    if ((st_loc == -1) && (end_loc == -1))
-	ntimes = 0;
-    MPI_Allreduce(&end_loc, &max_end_loc, 1, MPI_LONG_LONG_INT, MPI_MAX, fd->comm);
-    /* avoid min_st_loc be -1 */
-    if (st_loc == -1)
-        st_loc = max_end_loc;
-    MPI_Allreduce(&st_loc, &min_st_loc, 1, MPI_LONG_LONG_INT, MPI_MIN, fd->comm);
-    /* align downward */
-    min_st_loc -= min_st_loc % (ADIO_Offset)stripe_size;
-
-    /* Each time, only avail_cb_nodes number of IO clients perform IO,
-     * so, step_size=avail_cb_nodes*stripe_size IO will be performed at most,
-     * and ntimes=whole_file_portion/step_size
-     */
-    step_size = (ADIO_Offset) avail_cb_nodes * stripe_size;
-    max_ntimes = (max_end_loc - min_st_loc + 1) / step_size
-        + (((max_end_loc - min_st_loc + 1) % step_size) ? 1 : 0);
-/*     max_ntimes = (int)((max_end_loc - min_st_loc) / step_size + 1); */
-    if (ntimes)
-	write_buf = (char *) ADIOI_Malloc(stripe_size);
-
-    /* calculate the start offset for each iteration */
-    off_list = (ADIO_Offset *) ADIOI_Malloc(max_ntimes * sizeof(ADIO_Offset));
-    for (m = 0; m < max_ntimes; m ++)
-        off_list[m] = max_end_loc;
-    for (i = 0; i < nprocs; i++) {
-        for (j = 0; j < others_req[i].count; j ++) {
-            req_off = others_req[i].offsets[j];
-            m = (int)((req_off - min_st_loc) / step_size);
-            off_list[m] = ADIOI_MIN(off_list[m], req_off);
-        }
-    }
-
-    recv_curr_offlen_ptr = (int *) ADIOI_Calloc(nprocs, sizeof(int));
-    send_curr_offlen_ptr = (int *) ADIOI_Calloc(nprocs, sizeof(int));
-    /* their use is explained below. calloc initializes to 0. */
-
-    recv_count = (int *) ADIOI_Malloc(nprocs * sizeof(int));
-    /* to store count of how many off-len pairs per proc are satisfied
-       in an iteration. */
-
-    send_size = (int *) ADIOI_Malloc(nprocs * sizeof(int));
-    /* total size of data to be sent to each proc. in an iteration.
-       Of size nprocs so that I can use MPI_Alltoall later. */
-
-    recv_size = (int *) ADIOI_Malloc(nprocs * sizeof(int));
-    /* total size of data to be recd. from each proc. in an iteration. */
-
-    sent_to_proc = (int *) ADIOI_Calloc(nprocs, sizeof(int));
-    /* amount of data sent to each proc so far. Used in
-       ADIOI_Fill_send_buffer. initialized to 0 here. */
-
-    send_buf_idx = (int *) ADIOI_Malloc(nprocs * sizeof(int));
-    curr_to_proc = (int *) ADIOI_Malloc(nprocs * sizeof(int));
-    done_to_proc = (int *) ADIOI_Malloc(nprocs * sizeof(int));
-    /* Above three are used in ADIOI_Fill_send_buffer */
-
-    this_buf_idx = (int *) ADIOI_Malloc(nprocs * sizeof(int));
-
-    recv_start_pos = (int *) ADIOI_Malloc(nprocs * sizeof(int));
-    /* used to store the starting value of recv_curr_offlen_ptr[i] in
-       this iteration */
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-    if (!buftype_is_contig) {
-	flat_buf = ADIOI_Flatten_and_find(datatype);
-    }
-    MPI_Type_get_extent(datatype, &buftype_lb, &buftype_extent);
-    /* I need to check if there are any outstanding nonblocking writes to
-     * the file, which could potentially interfere with the writes taking
-     * place in this collective write call. Since this is not likely to be
-     * common, let me do the simplest thing possible here: Each process
-     * completes all pending nonblocking operations before completing.
-     */
-    /*ADIOI_Complete_async(error_code);
-    if (*error_code != MPI_SUCCESS) return;
-    MPI_Barrier(fd->comm);
-    */
-
-    iter_st_off = min_st_loc;
-
-    /* Although we have recognized the data according to OST index,
-     * a read-modify-write will be done if there is a hole between the data.
-     * For example: if blocksize=60, xfersize=30 and stripe_size=100,
-     * then rank0 will collect data [0, 30] and [60, 90] then write. There
-     * is a hole in [30, 60], which will cause a read-modify-write in [0, 90].
-     *
-     * To reduce its impact on the performance, we can disable data sieving
-     * by hint "ds_in_coll".
-     */
-    /* check the hint for data sieving */
-    data_sieving = fd->hints->fs_hints.lustre.ds_in_coll;
-
-    for (m = 0; m < max_ntimes; m++) {
-	/* go through all others_req and my_req to check which will be received
-         * and sent in this iteration.
-         */
-
-	/* Note that MPI guarantees that displacements in filetypes are in
-	   monotonically nondecreasing order and that, for writes, the
-	   filetypes cannot specify overlapping regions in the file. This
-	   simplifies implementation a bit compared to reads. */
-
-	/*
-           off         = start offset in the file for the data to be written in
-                         this iteration
-           iter_st_off = start offset of this iteration
-           real_size   = size of data written (bytes) corresponding to off
-           max_size    = possible maximum size of data written in this iteration
-           req_off     = offset in the file for a particular contiguous request minus
-                         what was satisfied in previous iteration
-           send_off    = offset the request needed by other processes in this iteration
-           req_len     = size corresponding to req_off
-           send_len    = size corresponding to send_off
-         */
-
-	/* first calculate what should be communicated */
-	for (i = 0; i < nprocs; i++)
-	    recv_count[i] = recv_size[i] = send_size[i] = 0;
-
-        off = off_list[m];
-        max_size = ADIOI_MIN(step_size, max_end_loc - iter_st_off + 1);
-        real_size = (int) ADIOI_MIN((off / stripe_size + 1) * stripe_size -
-                                    off,
-                                    end_loc - off + 1);
-
-	for (i = 0; i < nprocs; i++) {
-            if (my_req[i].count) {
-                this_buf_idx[i] = buf_idx[i][send_curr_offlen_ptr[i]];
-                for (j = send_curr_offlen_ptr[i]; j < my_req[i].count; j++) {
-                    send_off = my_req[i].offsets[j];
-                    send_len = my_req[i].lens[j];
-                    if (send_off < iter_st_off + max_size) {
-                        send_size[i] += send_len;
-                    } else {
-                        break;
-                    }
-                }
-                send_curr_offlen_ptr[i] = j;
-            }
-	    if (others_req[i].count) {
-		recv_start_pos[i] = recv_curr_offlen_ptr[i];
-		for (j = recv_curr_offlen_ptr[i]; j < others_req[i].count; j++) {
-                    req_off = others_req[i].offsets[j];
-                    req_len = others_req[i].lens[j];
-		    if (req_off < iter_st_off + max_size) {
-			recv_count[i]++;
-                        ADIOI_Assert((((ADIO_Offset)(MPIU_Upint)write_buf)+req_off-off) == (ADIO_Offset)(MPIU_Upint)(write_buf+req_off-off));
-			MPI_Get_address(write_buf + req_off - off,
-				        &(others_req[i].mem_ptrs[j]));
-                        recv_size[i] += req_len;
-		    } else {
-			break;
-                    }
-		}
-		recv_curr_offlen_ptr[i] = j;
-	    }
-	}
-        /* use variable "hole" to pass data_sieving flag into W_Exchange_data */
-        hole = data_sieving;
-	ADIOI_LUSTRE_W_Exchange_data(fd, buf, write_buf, flat_buf, offset_list,
-                                     len_list, send_size, recv_size, off, real_size,
-                                     recv_count, recv_start_pos,
-                                     sent_to_proc, nprocs, myrank,
-                                     buftype_is_contig, contig_access_count,
-                                     striping_info, others_req, send_buf_idx,
-                                     curr_to_proc, done_to_proc, &hole, m,
-                                  buftype_extent, this_buf_idx,
-                                  &srt_off, &srt_len, &srt_num, error_code);
-
-	if (*error_code != MPI_SUCCESS)
-            goto over;
-
-	flag = 0;
-	for (i = 0; i < nprocs; i++)
-	    if (recv_count[i]) {
-		flag = 1;
-		break;
-	    }
-	if (flag) {
-            /* check whether to do data sieving */
-            if(data_sieving == ADIOI_HINT_ENABLE) {
-	        ADIO_WriteContig(fd, write_buf, real_size, MPI_BYTE,
-			         ADIO_EXPLICIT_OFFSET, off, &status,
-			         error_code);
-            } else {
-                /* if there is no hole, write data in one time;
-                 * otherwise, write data in several times */
-                if (!hole) {
-                    ADIO_WriteContig(fd, write_buf, real_size, MPI_BYTE,
-                                     ADIO_EXPLICIT_OFFSET, off, &status,
-                                     error_code);
-                } else {
-                    block_offset = -1;
-                    block_len = 0;
-                    for (i = 0; i < srt_num; ++i) {
-                        if (srt_off[i] < off + real_size &&
-                            srt_off[i] >= off) {
-                            if (block_offset == -1) {
-                                block_offset = srt_off[i];
-                                block_len = srt_len[i];
-                            } else {
-                                if (srt_off[i] == block_offset + block_len) {
-                                    block_len += srt_len[i];
-                                } else {
-                                    ADIO_WriteContig(fd,
-                                                     write_buf + block_offset - off,
-                                                     block_len,
-                                                     MPI_BYTE, ADIO_EXPLICIT_OFFSET,
-                                                     block_offset, &status,
-                                                     error_code);
-	                            if (*error_code != MPI_SUCCESS)
-		                        goto over;
-                                    block_offset = srt_off[i];
-                                    block_len = srt_len[i];
-                                }
-                            }
-                        }
-                    }
-                    if (block_offset != -1) {
-                        ADIO_WriteContig(fd,
-                                         write_buf + block_offset - off,
-                                         block_len,
-                                         MPI_BYTE, ADIO_EXPLICIT_OFFSET,
-                                         block_offset, &status,
-                                         error_code);
-                        if (*error_code != MPI_SUCCESS)
-                            goto over;
-                    }
-                }
-            }
-	    if (*error_code != MPI_SUCCESS)
-		goto over;
-	}
-        iter_st_off += max_size;
-    }
-over:
-    if (srt_off)
-        ADIOI_Free(srt_off);
-    if (srt_len)
-        ADIOI_Free(srt_len);
-    if (ntimes)
-	ADIOI_Free(write_buf);
-    ADIOI_Free(recv_curr_offlen_ptr);
-    ADIOI_Free(send_curr_offlen_ptr);
-    ADIOI_Free(recv_count);
-    ADIOI_Free(send_size);
-    ADIOI_Free(recv_size);
-    ADIOI_Free(sent_to_proc);
-    ADIOI_Free(recv_start_pos);
-    ADIOI_Free(send_buf_idx);
-    ADIOI_Free(curr_to_proc);
-    ADIOI_Free(done_to_proc);
-    ADIOI_Free(this_buf_idx);
-    ADIOI_Free(off_list);
-}
-
-/* Sets error_code to MPI_SUCCESS if successful, or creates an error code
- * in the case of error.
- */
-static void ADIOI_LUSTRE_W_Exchange_data(ADIO_File fd, const void *buf,
-					 char *write_buf,
-					 ADIOI_Flatlist_node *flat_buf,
-					 ADIO_Offset *offset_list,
-					 ADIO_Offset *len_list, int *send_size,
-					 int *recv_size, ADIO_Offset off,
-					 int size, int *count,
-					 int *start_pos, 
-					 int *sent_to_proc, int nprocs,
-					 int myrank, int buftype_is_contig,
-					 int contig_access_count,
-					 int *striping_info,
-					 ADIOI_Access *others_req,
-					 int *send_buf_idx,
-					 int *curr_to_proc, int *done_to_proc,
-                                         int *hole, int iter,
-                                         MPI_Aint buftype_extent,
-					 int *buf_idx,
-                          ADIO_Offset **srt_off, int **srt_len, int *srt_num,
-                          int *error_code)
-{
-    int i, j, nprocs_recv, nprocs_send, err;
-    char **send_buf = NULL;
-    MPI_Request *requests, *send_req;
-    MPI_Datatype *recv_types;
-    MPI_Status *statuses, status;
-    int sum_recv;
-    int data_sieving = *hole;
-    static char myname[] = "ADIOI_W_EXCHANGE_DATA";
-
-    /* create derived datatypes for recv */
-    nprocs_recv = 0;
-    for (i = 0; i < nprocs; i++)
-	if (recv_size[i])
-	    nprocs_recv++;
-
-    recv_types = (MPI_Datatype *) ADIOI_Malloc((nprocs_recv + 1) *
-					       sizeof(MPI_Datatype));
-    /* +1 to avoid a 0-size malloc */
-
-    j = 0;
-    for (i = 0; i < nprocs; i++) {
-	if (recv_size[i]) {
-	    ADIOI_Type_create_hindexed_x(count[i],
-			      &(others_req[i].lens[start_pos[i]]),
-			      &(others_req[i].mem_ptrs[start_pos[i]]),
-			      MPI_BYTE, recv_types + j);
-	    /* absolute displacements; use MPI_BOTTOM in recv */
-	    MPI_Type_commit(recv_types + j);
-	    j++;
-	}
-    }
-
-    /* To avoid a read-modify-write,
-     * check if there are holes in the data to be written.
-     * For this, merge the (sorted) offset lists others_req using a heap-merge.
-     */
-
-    *srt_num = 0;
-    for (i = 0; i < nprocs; i++)
-        *srt_num += count[i];
-    if (*srt_off)
-        *srt_off = (ADIO_Offset *) ADIOI_Realloc(*srt_off, (*srt_num + 1) * sizeof(ADIO_Offset));
-    else
-        *srt_off = (ADIO_Offset *) ADIOI_Malloc((*srt_num + 1) * sizeof(ADIO_Offset));
-    if (*srt_len)
-        *srt_len = (int *) ADIOI_Realloc(*srt_len, (*srt_num + 1) * sizeof(int));
-    else
-        *srt_len = (int *) ADIOI_Malloc((*srt_num + 1) * sizeof(int));
-    /* +1 to avoid a 0-size malloc */
-
-    ADIOI_Heap_merge(others_req, count, *srt_off, *srt_len, start_pos,
-		     nprocs, nprocs_recv, *srt_num);
-
-    /* check if there are any holes */
-    *hole = 0;
-    for (i = 0; i < *srt_num - 1; i++) {
-        if ((*srt_off)[i] + (*srt_len)[i] < (*srt_off)[i + 1]) {
-            *hole = 1;
-	    break;
-	}
-    }
-    /* In some cases (see John Bent ROMIO REQ # 835), an odd interaction
-     * between aggregation, nominally contiguous regions, and cb_buffer_size
-     * should be handled with a read-modify-write (otherwise we will write out
-     * more data than we receive from everyone else (inclusive), so override
-     * hole detection
-     */
-    if (*hole == 0) {
-        sum_recv = 0;
-        for (i = 0; i < nprocs; i++)
-            sum_recv += recv_size[i];
-	if (size > sum_recv)
-	    *hole = 1;
-    }
-    /* check the hint for data sieving */
-    if (data_sieving == ADIOI_HINT_ENABLE && nprocs_recv && *hole) {
-        ADIO_ReadContig(fd, write_buf, size, MPI_BYTE,
-                        ADIO_EXPLICIT_OFFSET, off, &status, &err);
-        // --BEGIN ERROR HANDLING--
-        if (err != MPI_SUCCESS) {
-            *error_code = MPIO_Err_create_code(err,
-                                               MPIR_ERR_RECOVERABLE,
-                                               myname, __LINE__,
-                                               MPI_ERR_IO,
-                                               "**ioRMWrdwr", 0);
-            ADIOI_Free(recv_types);
-            return;
-        }
-        // --END ERROR HANDLING--
-    }
-
-    nprocs_send = 0;
-    for (i = 0; i < nprocs; i++)
-	if (send_size[i])
-	    nprocs_send++;
-
-    if (fd->atomicity) {
-	/* bug fix from Wei-keng Liao and Kenin Coloma */
-	requests = (MPI_Request *) ADIOI_Malloc((nprocs_send + 1) *
-                                                sizeof(MPI_Request));
-	send_req = requests;
-    } else {
-	requests = (MPI_Request *) ADIOI_Malloc((nprocs_send + nprocs_recv + 1)*
-                                                sizeof(MPI_Request));
-	/* +1 to avoid a 0-size malloc */
-
-	/* post receives */
-	j = 0;
-	for (i = 0; i < nprocs; i++) {
-	    if (recv_size[i]) {
-		MPI_Irecv(MPI_BOTTOM, 1, recv_types[j], i,
-			  myrank + i + 100 * iter, fd->comm, requests + j);
-		j++;
-	    }
-	}
-	send_req = requests + nprocs_recv;
-    }
-
-    /* post sends.
-     * if buftype_is_contig, data can be directly sent from
-     * user buf at location given by buf_idx. else use send_buf.
-     */
-    if (buftype_is_contig) {
-	j = 0;
-	for (i = 0; i < nprocs; i++)
-	    if (send_size[i]) {
-                ADIOI_Assert(buf_idx[i] != -1);
-		MPI_Isend(((char *) buf) + buf_idx[i], send_size[i],
-			  MPI_BYTE, i, myrank + i + 100 * iter, fd->comm,
-			  send_req + j);
-		j++;
-	    }
-    } else
-        if (nprocs_send) {
-	/* buftype is not contig */
-	send_buf = (char **) ADIOI_Malloc(nprocs * sizeof(char *));
-	for (i = 0; i < nprocs; i++)
-	    if (send_size[i])
-		send_buf[i] = (char *) ADIOI_Malloc(send_size[i]);
-
-	ADIOI_LUSTRE_Fill_send_buffer(fd, buf, flat_buf, send_buf, offset_list,
-                                      len_list, send_size, send_req,
-                                      sent_to_proc, nprocs, myrank,
-                                      contig_access_count, striping_info,
-                                      send_buf_idx, curr_to_proc, done_to_proc,
-                                      iter, buftype_extent);
-	/* the send is done in ADIOI_Fill_send_buffer */
-    }
-
-	/* bug fix from Wei-keng Liao and Kenin Coloma */
-    if (fd->atomicity) {
-	j = 0;
-	for (i = 0; i < nprocs; i++) {
-	    MPI_Status wkl_status;
-	    if (recv_size[i]) {
-		MPI_Recv(MPI_BOTTOM, 1, recv_types[j], i,
-			 myrank + i + 100 * iter, fd->comm, &wkl_status);
-		j++;
-	    }
-	}
-    }
-
-    for (i = 0; i < nprocs_recv; i++)
-	MPI_Type_free(recv_types + i);
-    ADIOI_Free(recv_types);
-
-	/* bug fix from Wei-keng Liao and Kenin Coloma */
-	/* +1 to avoid a 0-size malloc */
-    if (fd->atomicity) {
-	statuses = (MPI_Status *) ADIOI_Malloc((nprocs_send + 1) *
-					       sizeof(MPI_Status));
-    } else {
-	statuses = (MPI_Status *) ADIOI_Malloc((nprocs_send + nprocs_recv + 1) *
-					       sizeof(MPI_Status));
-    }
-
-#ifdef NEEDS_MPI_TEST
-    i = 0;
-    if (fd->atomicity) {
-	/* bug fix from Wei-keng Liao and Kenin Coloma */
-	while (!i)
-	    MPI_Testall(nprocs_send, send_req, &i, statuses);
-    } else {
-	while (!i)
-	    MPI_Testall(nprocs_send + nprocs_recv, requests, &i, statuses);
-    }
-#else
-	/* bug fix from Wei-keng Liao and Kenin Coloma */
-    if (fd->atomicity)
-	MPI_Waitall(nprocs_send, send_req, statuses);
-    else
-	MPI_Waitall(nprocs_send + nprocs_recv, requests, statuses);
-#endif
-    ADIOI_Free(statuses);
-    ADIOI_Free(requests);
-    if (!buftype_is_contig && nprocs_send) {
-	for (i = 0; i < nprocs; i++)
-	    if (send_size[i])
-		ADIOI_Free(send_buf[i]);
-	ADIOI_Free(send_buf);
-    }
-}
-
-#define ADIOI_BUF_INCR \
-{ \
-    while (buf_incr) { \
-        size_in_buf = ADIOI_MIN(buf_incr, flat_buf_sz); \
-        user_buf_idx += size_in_buf; \
-        flat_buf_sz -= size_in_buf; \
-        if (!flat_buf_sz) { \
-            if (flat_buf_idx < (flat_buf->count - 1)) flat_buf_idx++; \
-            else { \
-                flat_buf_idx = 0; \
-                n_buftypes++; \
-            } \
-            user_buf_idx = flat_buf->indices[flat_buf_idx] + \
-                (ADIO_Offset)n_buftypes*(ADIO_Offset)buftype_extent;  \
-            flat_buf_sz = flat_buf->blocklens[flat_buf_idx]; \
-        } \
-        buf_incr -= size_in_buf; \
-    } \
-}
-
-
-#define ADIOI_BUF_COPY \
-{ \
-    while (size) { \
-        size_in_buf = ADIOI_MIN(size, flat_buf_sz); \
-        ADIOI_Assert((((ADIO_Offset)(MPIU_Upint)buf) + user_buf_idx) == (ADIO_Offset)(MPIU_Upint)((MPIU_Upint)buf + user_buf_idx)); \
-        ADIOI_Assert(size_in_buf == (size_t)size_in_buf);               \
-        memcpy(&(send_buf[p][send_buf_idx[p]]), \
-               ((char *) buf) + user_buf_idx, size_in_buf); \
-        send_buf_idx[p] += size_in_buf; \
-        user_buf_idx += size_in_buf; \
-        flat_buf_sz -= size_in_buf; \
-        if (!flat_buf_sz) { \
-            if (flat_buf_idx < (flat_buf->count - 1)) flat_buf_idx++; \
-            else { \
-                flat_buf_idx = 0; \
-                n_buftypes++; \
-            } \
-            user_buf_idx = flat_buf->indices[flat_buf_idx] + \
-                (ADIO_Offset)n_buftypes*(ADIO_Offset)buftype_extent;    \
-            flat_buf_sz = flat_buf->blocklens[flat_buf_idx]; \
-        } \
-        size -= size_in_buf; \
-        buf_incr -= size_in_buf; \
-    } \
-    ADIOI_BUF_INCR \
-}
-
-static void ADIOI_LUSTRE_Fill_send_buffer(ADIO_File fd, const void *buf,
-					  ADIOI_Flatlist_node *flat_buf,
-					  char **send_buf,
-					  ADIO_Offset *offset_list,
-					  ADIO_Offset *len_list, int *send_size,
-					  MPI_Request *requests,
-					  int *sent_to_proc, int nprocs,
-					  int myrank,
-					  int contig_access_count,
-					  int *striping_info,
-					  int *send_buf_idx,
-					  int *curr_to_proc,
-					  int *done_to_proc, int iter,
-					  MPI_Aint buftype_extent)
-{
-    /* this function is only called if buftype is not contig */
-    int i, p, flat_buf_idx, size;
-    int flat_buf_sz, buf_incr, size_in_buf, jj, n_buftypes;
-    ADIO_Offset off, len, rem_len, user_buf_idx;
-
-    /* curr_to_proc[p] = amount of data sent to proc. p that has already
-     * been accounted for so far
-     * done_to_proc[p] = amount of data already sent to proc. p in
-     * previous iterations
-     * user_buf_idx = current location in user buffer
-     * send_buf_idx[p] = current location in send_buf of proc. p
-     */
-
-    for (i = 0; i < nprocs; i++) {
-	send_buf_idx[i] = curr_to_proc[i] = 0;
-	done_to_proc[i] = sent_to_proc[i];
-    }
-    jj = 0;
-
-    user_buf_idx = flat_buf->indices[0];
-    flat_buf_idx = 0;
-    n_buftypes = 0;
-    flat_buf_sz = flat_buf->blocklens[0];
-
-    /* flat_buf_idx = current index into flattened buftype
-     * flat_buf_sz = size of current contiguous component in flattened buf
-     */
-    for (i = 0; i < contig_access_count; i++) {
-	off = offset_list[i];
-	rem_len = (ADIO_Offset) len_list[i];
-
-	/*this request may span to more than one process */
-	while (rem_len != 0) {
-	    len = rem_len;
-	    /* NOTE: len value is modified by ADIOI_Calc_aggregator() to be no
-	     * longer than the single region that processor "p" is responsible
-	     * for.
-	     */
-	    p = ADIOI_LUSTRE_Calc_aggregator(fd, off, &len, striping_info);
-
-	    if (send_buf_idx[p] < send_size[p]) {
-		if (curr_to_proc[p] + len > done_to_proc[p]) {
-		    if (done_to_proc[p] > curr_to_proc[p]) {
-			size = (int) ADIOI_MIN(curr_to_proc[p] + len -
-					       done_to_proc[p],
-					       send_size[p] -
-					       send_buf_idx[p]);
-			buf_incr = done_to_proc[p] - curr_to_proc[p];
-			ADIOI_BUF_INCR
-                        ADIOI_Assert((curr_to_proc[p] + len - done_to_proc[p]) == (unsigned)(curr_to_proc[p] + len - done_to_proc[p]));
-			    buf_incr = (int) (curr_to_proc[p] + len -
-					      done_to_proc[p]);
-                        ADIOI_Assert((done_to_proc[p] + size) == (unsigned)(done_to_proc[p] + size));
-			curr_to_proc[p] = done_to_proc[p] + size;
-		        ADIOI_BUF_COPY
-                    } else {
-			size = (int) ADIOI_MIN(len, send_size[p] -
-					       send_buf_idx[p]);
-			buf_incr = (int) len;
-                        ADIOI_Assert((curr_to_proc[p] + size) == (unsigned)((ADIO_Offset)curr_to_proc[p] + size));
-			curr_to_proc[p] += size;
-		        ADIOI_BUF_COPY
-                    }
-		    if (send_buf_idx[p] == send_size[p]) {
-			MPI_Isend(send_buf[p], send_size[p], MPI_BYTE, p,
-				  myrank + p + 100 * iter, fd->comm,
-				  requests + jj);
-			jj++;
-		    }
-		} else {
-                    ADIOI_Assert((curr_to_proc[p] + len) == (unsigned)((ADIO_Offset)curr_to_proc[p] + len));
-		    curr_to_proc[p] += (int) len;
-		    buf_incr = (int) len;
-		    ADIOI_BUF_INCR
-                }
-	    } else {
-		buf_incr = (int) len;
-	        ADIOI_BUF_INCR
-            }
-	    off += len;
-	    rem_len -= len;
-	}
-    }
-    for (i = 0; i < nprocs; i++)
-	if (send_size[i])
-	    sent_to_proc[i] = curr_to_proc[i];
-}
diff --git a/3rd-party/romio321/adio/ad_lustre/ad_lustre_wrstr.c b/3rd-party/romio321/adio/ad_lustre/ad_lustre_wrstr.c
deleted file mode 100644
index ce538d4a6b6..00000000000
--- a/3rd-party/romio321/adio/ad_lustre/ad_lustre_wrstr.c
+++ /dev/null
@@ -1,527 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *   Copyright (C) 1997 University of Chicago.
- *   See COPYRIGHT notice in top-level directory.
- *
- *   Copyright (C) 2007 Oak Ridge National Laboratory
- *
- *   Copyright (C) 2008 Sun Microsystems, Lustre group
- */
-
-#include "ad_lustre.h"
-#include "adio_extern.h"
-
-#define ADIOI_BUFFERED_WRITE \
-{ \
-    if (req_off >= writebuf_off + writebuf_len) { \
-        if (writebuf_len) { \
-           ADIO_WriteContig(fd, writebuf, writebuf_len, MPI_BYTE, \
-                             ADIO_EXPLICIT_OFFSET, writebuf_off,        \
-                             &status1, error_code);                     \
-           if (!(fd->atomicity)) \
-                ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
-           if (*error_code != MPI_SUCCESS) { \
-               *error_code = MPIO_Err_create_code(*error_code, \
-                                                   MPIR_ERR_RECOVERABLE, \
-                                                   myname,              \
-                                                  __LINE__, MPI_ERR_IO, \
-                                                  "**iowswc", 0); \
-               ADIOI_Free(writebuf); \
-               return; \
-           } \
-        } \
-	writebuf_off = req_off; \
-        /* stripe_size alignment */ \
-        writebuf_len = (unsigned) ADIOI_MIN(end_offset - writebuf_off + 1, \
-                                       (writebuf_off / stripe_size + 1) * \
-                                            stripe_size - writebuf_off); \
-	if (!(fd->atomicity)) \
-            ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
-        ADIO_ReadContig(fd, writebuf, writebuf_len, MPI_BYTE,           \
-                        ADIO_EXPLICIT_OFFSET,                           \
-                        writebuf_off, &status1, error_code); \
-	if (*error_code != MPI_SUCCESS) { \
-	    *error_code = MPIO_Err_create_code(*error_code, \
-                                               MPIR_ERR_RECOVERABLE,    \
-                                               myname,                  \
-					       __LINE__, MPI_ERR_IO, \
-					       "**iowsrc", 0); \
-            ADIOI_Free(writebuf); \
-	    return; \
-	} \
-    } \
-    write_sz = (unsigned) (ADIOI_MIN(req_len,                           \
-                                     writebuf_off + writebuf_len - req_off)); \
-    ADIOI_Assert((ADIO_Offset)write_sz ==                               \
-                 ADIOI_MIN(req_len, writebuf_off + writebuf_len - req_off)); \
-    memcpy(writebuf + req_off - writebuf_off, (char *)buf +userbuf_off, write_sz); \
-    while (write_sz != req_len) {                                       \
-        ADIO_WriteContig(fd, writebuf, writebuf_len, MPI_BYTE, \
-                         ADIO_EXPLICIT_OFFSET, writebuf_off, &status1, error_code); \
-        if (!(fd->atomicity)) \
-            ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
-        if (*error_code != MPI_SUCCESS) { \
-            *error_code = MPIO_Err_create_code(*error_code, \
-                                               MPIR_ERR_RECOVERABLE, myname, \
-                                               __LINE__, MPI_ERR_IO, \
-                                               "**iowswc", 0); \
-            ADIOI_Free(writebuf); \
-            return; \
-        } \
-        req_len -= write_sz; \
-        userbuf_off += write_sz; \
-        writebuf_off += writebuf_len; \
-        /* stripe_size alignment */ \
-        writebuf_len = (unsigned) ADIOI_MIN(end_offset - writebuf_off + 1, \
-                                       (writebuf_off / stripe_size + 1) * \
-                                            stripe_size - writebuf_off); \
-	if (!(fd->atomicity)) \
-            ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
-        ADIO_ReadContig(fd, writebuf, writebuf_len, MPI_BYTE,           \
-                        ADIO_EXPLICIT_OFFSET,                           \
-                        writebuf_off, &status1, error_code); \
-	if (*error_code != MPI_SUCCESS) { \
-	    *error_code = MPIO_Err_create_code(*error_code, \
-	                                       MPIR_ERR_RECOVERABLE, myname, \
-		                               __LINE__, MPI_ERR_IO, \
-				               "**iowsrc", 0); \
-            ADIOI_Free(writebuf); \
-	    return; \
-	} \
-        write_sz = ADIOI_MIN(req_len, writebuf_len); \
-        memcpy(writebuf, (char *)buf + userbuf_off, write_sz);          \
-    } \
-}
-
-
-/* this macro is used when filetype is contig and buftype is not contig.
-   it does not do a read-modify-write and does not lock*/
-#define ADIOI_BUFFERED_WRITE_WITHOUT_READ \
-{ \
-    if (req_off >= writebuf_off + writebuf_len) { \
-        ADIO_WriteContig(fd, writebuf, writebuf_len, MPI_BYTE,          \
-                         ADIO_EXPLICIT_OFFSET, writebuf_off, &status1,  \
-                         error_code);                                   \
-        if (*error_code != MPI_SUCCESS) {                               \
-            *error_code = MPIO_Err_create_code(*error_code,             \
-                                               MPIR_ERR_RECOVERABLE,    \
-                                               myname,                  \
-                                               __LINE__, MPI_ERR_IO,    \
-                                               "**iowswc", 0);          \
-            ADIOI_Free(writebuf);                                       \
-            return;                                                     \
-        }                                                               \
-	writebuf_off = req_off; \
-        /* stripe_size alignment */ \
-        writebuf_len = (unsigned) ADIOI_MIN(end_offset - writebuf_off + 1, \
-                                       (writebuf_off / stripe_size + 1) * \
-                                            stripe_size - writebuf_off); \
-    } \
-    write_sz = (unsigned) ADIOI_MIN(req_len, writebuf_off + writebuf_len - req_off); \
-    ADIOI_Assert((ADIO_Offset)write_sz == ADIOI_MIN(req_len, writebuf_off + writebuf_len - req_off)); \
-    memcpy(writebuf + req_off - writebuf_off,                           \
-           (char *)buf + userbuf_off, write_sz);                        \
-    while (write_sz != req_len) {                                       \
-        ADIO_WriteContig(fd, writebuf, writebuf_len, MPI_BYTE, \
-                         ADIO_EXPLICIT_OFFSET, writebuf_off, &status1, error_code); \
-        if (*error_code != MPI_SUCCESS) { \
-            *error_code = MPIO_Err_create_code(*error_code, \
-                                               MPIR_ERR_RECOVERABLE, myname, \
-                                               __LINE__, MPI_ERR_IO, \
-                                               "**iowswc", 0); \
-            ADIOI_Free(writebuf); \
-            return; \
-        } \
-        req_len -= write_sz; \
-        userbuf_off += write_sz; \
-        writebuf_off += writebuf_len; \
-        /* stripe_size alignment */ \
-        writebuf_len = (unsigned) ADIOI_MIN(end_offset - writebuf_off + 1, \
-                                       (writebuf_off / stripe_size + 1) * \
-                                            stripe_size - writebuf_off); \
-        write_sz = ADIOI_MIN(req_len, writebuf_len); \
-        memcpy(writebuf, (char *)buf + userbuf_off, write_sz);          \
-    } \
-}
-
-void ADIOI_LUSTRE_WriteStrided(ADIO_File fd, const void *buf, int count,
-			       MPI_Datatype datatype, int file_ptr_type,
-			       ADIO_Offset offset, ADIO_Status * status,
-			       int *error_code)
-{
-    /* offset is in units of etype relative to the filetype. */
-    ADIOI_Flatlist_node *flat_buf, *flat_file;
-    ADIO_Offset i_offset, sum, size_in_filetype;
-    int i, j, k, st_index=0;
-    int n_etypes_in_filetype;
-    ADIO_Offset num, size, n_filetypes, etype_in_filetype, st_n_filetypes;
-    ADIO_Offset abs_off_in_filetype=0;
-    MPI_Count filetype_size, etype_size, buftype_size;
-    MPI_Aint filetype_extent, buftype_extent, filetype_lb, buftype_lb;
-    int buf_count, buftype_is_contig, filetype_is_contig;
-    ADIO_Offset userbuf_off;
-    ADIO_Offset off, req_off, disp, end_offset=0, writebuf_off, start_off;
-    char *writebuf;
-    unsigned bufsize, writebuf_len, write_sz;
-    ADIO_Status status1;
-    ADIO_Offset new_bwr_size, new_fwr_size, st_fwr_size, fwr_size=0, bwr_size, req_len;
-    int stripe_size;
-    static char myname[] = "ADIOI_LUSTRE_WriteStrided";
-
-    if (fd->hints->ds_write == ADIOI_HINT_DISABLE) {
-	/* if user has disabled data sieving on writes, use naive
-	 * approach instead.
-	 */
-	ADIOI_GEN_WriteStrided_naive(fd,
-				     buf,
-				     count,
-				     datatype,
-				     file_ptr_type,
-				     offset, status, error_code);
-	return;
-    }
-
-    *error_code = MPI_SUCCESS;	/* changed below if error */
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-
-    MPI_Type_size_x(fd->filetype, &filetype_size);
-    if (!filetype_size) {
-#ifdef HAVE_STATUS_SET_BYTES
-	MPIR_Status_set_bytes(status, datatype, 0);
-#endif
-	*error_code = MPI_SUCCESS;
-	return;
-    }
-
-    MPI_Type_get_extent(fd->filetype, &filetype_lb, &filetype_extent);
-    MPI_Type_size_x(datatype, &buftype_size);
-    MPI_Type_get_extent(datatype, &buftype_lb, &buftype_extent);
-    etype_size = fd->etype_size;
-
-    ADIOI_Assert((buftype_size * count) == ((ADIO_Offset)(unsigned)buftype_size * (ADIO_Offset)count));
-    bufsize = buftype_size * count;
-
-    /* get striping info */
-    stripe_size = fd->hints->striping_unit;
-
-    /* Different buftype to different filetype */
-    if (!buftype_is_contig && filetype_is_contig) {
-        /* noncontiguous in memory, contiguous in file. */
-	flat_buf = ADIOI_Flatten_and_find(datatype);
-
-	off = (file_ptr_type == ADIO_INDIVIDUAL) ? fd->fp_ind :
-            fd->disp + (ADIO_Offset)etype_size * offset;
-
-	start_off = off;
-	end_offset = start_off + bufsize - 1;
-        /* write stripe size buffer each time */
-	writebuf = (char *) ADIOI_Malloc(ADIOI_MIN(bufsize, stripe_size));
-        writebuf_off = 0;
-        writebuf_len = 0;
-
-        /* if atomicity is true, lock the region to be accessed */
-	if (fd->atomicity)
-	    ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, bufsize);
-
-	for (j = 0; j < count; j++) {
-	    for (i = 0; i < flat_buf->count; i++) {
-                userbuf_off = (ADIO_Offset)j * (ADIO_Offset)buftype_extent +
-                    flat_buf->indices[i];
-		req_off = off;
-		req_len = flat_buf->blocklens[i];
-		ADIOI_BUFFERED_WRITE_WITHOUT_READ
-		off += flat_buf->blocklens[i];
-	    }
-        }
-
-	/* write the buffer out finally */
-	ADIO_WriteContig(fd, writebuf, writebuf_len, MPI_BYTE,
-			 ADIO_EXPLICIT_OFFSET, writebuf_off, &status1,
-			 error_code);
-
-	if (fd->atomicity)
-	    ADIOI_UNLOCK(fd, start_off, SEEK_SET, bufsize);
-	if (*error_code != MPI_SUCCESS) {
-            ADIOI_Free(writebuf);
-	    return;
-        }
-	ADIOI_Free(writebuf);
-	if (file_ptr_type == ADIO_INDIVIDUAL)
-	    fd->fp_ind = off;
-    } else {
-        /* noncontiguous in file */
-        /* filetype already flattened in ADIO_Open */
-	flat_file = ADIOI_Flatlist;
-	while (flat_file->type != fd->filetype)
-	    flat_file = flat_file->next;
-	disp = fd->disp;
-
-	if (file_ptr_type == ADIO_INDIVIDUAL) {
-            /* Wei-keng reworked type processing to be a bit more efficient */
-            offset       = fd->fp_ind - disp;
-            n_filetypes  = (offset - flat_file->indices[0]) / filetype_extent;
-            offset      -= (ADIO_Offset)n_filetypes * filetype_extent;
-            /* now offset is local to this extent */
-
-            /* find the block where offset is located, skip blocklens[i]==0 */
-            for (i=0; i<flat_file->count; i++) {
-                ADIO_Offset dist;
-                if (flat_file->blocklens[i] == 0) continue;
-                dist = flat_file->indices[i] + flat_file->blocklens[i] - offset;
-                /* fwr_size is from offset to the end of block i */
-                if (dist == 0) {
-                    i++;
-                    offset   = flat_file->indices[i];
-                    fwr_size = flat_file->blocklens[i];
-			break;
-		    }
-                if (dist > 0) {
-                    fwr_size = dist;
-                    break;
-		}
-	    }
-            st_index = i;  /* starting index in flat_file->indices[] */
-            offset += disp + (ADIO_Offset)n_filetypes*filetype_extent;
-        }
-        else {
-            n_etypes_in_filetype = filetype_size/etype_size;
-            n_filetypes = offset / n_etypes_in_filetype;
-            etype_in_filetype = offset % n_etypes_in_filetype;
-	    size_in_filetype = etype_in_filetype * etype_size;
-
-	    sum = 0;
-	    for (i = 0; i < flat_file->count; i++) {
-		sum += flat_file->blocklens[i];
-		if (sum > size_in_filetype) {
-		    st_index = i;
-		    fwr_size = sum - size_in_filetype;
-		    abs_off_in_filetype = flat_file->indices[i] +
-			size_in_filetype - (sum - flat_file->blocklens[i]);
-		    break;
-		}
-	    }
-
-	    /* abs. offset in bytes in the file */
-	    offset = disp + (ADIO_Offset) n_filetypes *filetype_extent +
-		     abs_off_in_filetype;
-	}
-
-	start_off = offset;
-
-        /* Wei-keng Liao:write request is within single flat_file
-         * contig block*/
-        /* this could happen, for example, with subarray types that are
-         * actually fairly contiguous */
-        if (buftype_is_contig && bufsize <= fwr_size) {
-            req_off = start_off;
-            req_len = bufsize;
-            end_offset = start_off + bufsize - 1;
-	    writebuf = (char *) ADIOI_Malloc(ADIOI_MIN(bufsize, stripe_size));
-	    memset(writebuf, -1, ADIOI_MIN(bufsize, stripe_size));
-            writebuf_off = 0;
-            writebuf_len = 0;
-            userbuf_off = 0;
-            ADIOI_BUFFERED_WRITE_WITHOUT_READ
-            /* write the buffer out finally */
-            ADIO_WriteContig(fd, writebuf, writebuf_len, MPI_BYTE,
-                             ADIO_EXPLICIT_OFFSET, writebuf_off, &status1,
-                             error_code);
-
-            if (file_ptr_type == ADIO_INDIVIDUAL) {
-                /* update MPI-IO file pointer to point to the first byte
-                 * that can be accessed in the fileview. */
-                fd->fp_ind = offset + bufsize;
-                if (bufsize == fwr_size) {
-                    do {
-                        st_index++;
-                        if (st_index == flat_file->count) {
-                            st_index = 0;
-                            n_filetypes++;
-                        }
-                    } while (flat_file->blocklens[st_index] == 0);
-                    fd->fp_ind = disp + flat_file->indices[st_index]
-                        + (ADIO_Offset)n_filetypes*filetype_extent;
-                }
-            }
-            fd->fp_sys_posn = -1;   /* set it to null. */
-#ifdef HAVE_STATUS_SET_BYTES
-            MPIR_Status_set_bytes(status, datatype, bufsize);
-#endif
-            ADIOI_Free(writebuf);
-            return;
-        }
-
-	    /* Calculate end_offset, the last byte-offset that will be accessed.
-           e.g., if start_offset=0 and 100 bytes to be write, end_offset=99*/
-
-	    st_fwr_size = fwr_size;
-	    st_n_filetypes = n_filetypes;
-        i_offset = 0;
-	    j = st_index;
-	    off = offset;
-	    fwr_size = ADIOI_MIN(st_fwr_size, bufsize);
-        while (i_offset < bufsize) {
-            i_offset += fwr_size;
-		end_offset = off + fwr_size - 1;
-
-            j = (j+1) % flat_file->count;
-            n_filetypes += (j == 0) ? 1 : 0;
-            while (flat_file->blocklens[j]==0) {
-                j = (j+1) % flat_file->count;
-                n_filetypes += (j == 0) ? 1 : 0;
-		}
-
-		off = disp + flat_file->indices[j] +
-                n_filetypes*(ADIO_Offset)filetype_extent;
-            fwr_size = ADIOI_MIN(flat_file->blocklens[j], bufsize-i_offset);
-	    }
-
-/* if atomicity is true, lock the region to be accessed */
-        if (fd->atomicity)
-            ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
-
-	    writebuf_off = 0;
-	    writebuf_len = 0;
-	    writebuf = (char *) ADIOI_Malloc(stripe_size);
-	    memset(writebuf, -1, stripe_size);
-
-	    if (buftype_is_contig && !filetype_is_contig) {
-
-/* contiguous in memory, noncontiguous in file. should be the most
-		   common case. */
-
-            i_offset = 0;
-		j = st_index;
-		off = offset;
-		n_filetypes = st_n_filetypes;
-		fwr_size = ADIOI_MIN(st_fwr_size, bufsize);
-            while (i_offset < bufsize) {
-		    if (fwr_size) {
-			/* TYPE_UB and TYPE_LB can result in
-			   fwr_size = 0. save system call in such cases */
-                    /* lseek(fd->fd_sys, off, SEEK_SET);
-                       err = write(fd->fd_sys, ((char *) buf) + i_offset, fwr_size);*/
-
-			req_off = off;
-			req_len = fwr_size;
-                    userbuf_off = i_offset;
-			ADIOI_BUFFERED_WRITE
-                    }
-                i_offset += fwr_size;
-
-		    if (off + fwr_size < disp + flat_file->indices[j] +
-		                         flat_file->blocklens[j] +
-                    n_filetypes*(ADIO_Offset)filetype_extent)
-		        off += fwr_size;
-		    /* did not reach end of contiguous block in filetype.
-		    no more I/O needed. off is incremented by fwr_size. */
-		    else {
-                    j = (j+1) % flat_file->count;
-                    n_filetypes += (j == 0) ? 1 : 0;
-                    while (flat_file->blocklens[j]==0) {
-                        j = (j+1) % flat_file->count;
-                        n_filetypes += (j == 0) ? 1 : 0;
-			}
-			off = disp + flat_file->indices[j] +
-                        n_filetypes*(ADIO_Offset)filetype_extent;
-			fwr_size = ADIOI_MIN(flat_file->blocklens[j],
-                                         bufsize-i_offset);
-		    }
-		}
-        }
-        else {
-/* noncontiguous in memory as well as in file */
-	    flat_buf = ADIOI_Flatten_and_find(datatype);
-
-		k = num = buf_count = 0;
-            i_offset = flat_buf->indices[0];
-		j = st_index;
-		off = offset;
-		n_filetypes = st_n_filetypes;
-		fwr_size = st_fwr_size;
-		bwr_size = flat_buf->blocklens[0];
-
-		while (num < bufsize) {
-		    size = ADIOI_MIN(fwr_size, bwr_size);
-		    if (size) {
-                    /* lseek(fd->fd_sys, off, SEEK_SET);
-                       err = write(fd->fd_sys, ((char *) buf) + i_offset, size); */
-
-		        req_off = off;
-		        req_len = size;
-                    userbuf_off = i_offset;
-		        ADIOI_BUFFERED_WRITE
-                    }
-
-	            new_fwr_size = fwr_size;
-		    new_bwr_size = bwr_size;
-
-		    if (size == fwr_size) {
-/* reached end of contiguous block in file */
-                    j = (j+1) % flat_file->count;
-                    n_filetypes += (j == 0) ? 1 : 0;
-                    while (flat_file->blocklens[j]==0) {
-                        j = (j+1) % flat_file->count;
-                        n_filetypes += (j == 0) ? 1 : 0;
-			}
-
-			off = disp + flat_file->indices[j] +
-                        n_filetypes*(ADIO_Offset)filetype_extent;
-
-                        new_fwr_size = flat_file->blocklens[j];
-			if (size != bwr_size) {
-                        i_offset += size;
-			    new_bwr_size -= size;
-			}
-		    }
-
-		    if (size == bwr_size) {
-/* reached end of contiguous block in memory */
-
-                    k = (k + 1)%flat_buf->count;
-		        buf_count++;
-                    i_offset = (ADIO_Offset)buftype_extent *
-                        (ADIO_Offset)(buf_count/flat_buf->count) +
-                        flat_buf->indices[k];
-			new_bwr_size = flat_buf->blocklens[k];
-			if (size != fwr_size) {
-			    off += size;
-			    new_fwr_size -= size;
-			}
-		    }
-		    num += size;
-		    fwr_size = new_fwr_size;
-		    bwr_size = new_bwr_size;
-		}
-            }
-
-	    /* write the buffer out finally */
-	    if (writebuf_len) {
-            ADIO_WriteContig(fd, writebuf, writebuf_len, MPI_BYTE,
-                             ADIO_EXPLICIT_OFFSET,
-	                         writebuf_off, &status1, error_code);
-		if (!(fd->atomicity))
-		    ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len);
-            if (*error_code != MPI_SUCCESS) return;
-	    }
-	    if (fd->atomicity)
-            ADIOI_UNLOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
-
-        ADIOI_Free(writebuf);
-
-        if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
-    }
-
-    fd->fp_sys_posn = -1;	/* set it to null. */
-
-#ifdef HAVE_STATUS_SET_BYTES
-    MPIR_Status_set_bytes(status, datatype, bufsize);
-/* This is a temporary way of filling in status. The right way is to
-    keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
-#endif
-
-    if (!buftype_is_contig)
-        ADIOI_Delete_flattened(datatype);
-}
diff --git a/3rd-party/romio321/adio/ad_nfs/Makefile.mk b/3rd-party/romio321/adio/ad_nfs/Makefile.mk
deleted file mode 100644
index ca25e20cbea..00000000000
--- a/3rd-party/romio321/adio/ad_nfs/Makefile.mk
+++ /dev/null
@@ -1,28 +0,0 @@
-## -*- Mode: Makefile; -*-
-## vim: set ft=automake :
-##
-## (C) 2011 by Argonne National Laboratory.
-##     See COPYRIGHT in top-level directory.
-##
-
-if BUILD_AD_NFS
-
-noinst_HEADERS += adio/ad_nfs/ad_nfs.h
-
-romio_other_sources +=            \
-    adio/ad_nfs/ad_nfs_read.c     \
-    adio/ad_nfs/ad_nfs_open.c     \
-    adio/ad_nfs/ad_nfs_write.c    \
-    adio/ad_nfs/ad_nfs_done.c     \
-    adio/ad_nfs/ad_nfs_fcntl.c    \
-    adio/ad_nfs/ad_nfs_iread.c    \
-    adio/ad_nfs/ad_nfs_iwrite.c   \
-    adio/ad_nfs/ad_nfs_wait.c     \
-    adio/ad_nfs/ad_nfs_setsh.c    \
-    adio/ad_nfs/ad_nfs_getsh.c    \
-    adio/ad_nfs/ad_nfs.c          \
-    adio/ad_nfs/ad_nfs_resize.c   \
-    adio/ad_nfs/ad_nfs_features.c
-
-endif BUILD_AD_NFS
-
diff --git a/3rd-party/romio321/adio/ad_nfs/ad_nfs.c b/3rd-party/romio321/adio/ad_nfs/ad_nfs.c
deleted file mode 100644
index 763a1b4c7e5..00000000000
--- a/3rd-party/romio321/adio/ad_nfs/ad_nfs.c
+++ /dev/null
@@ -1,43 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_nfs.h"
-
-/* adioi.h has the ADIOI_Fns_struct define */
-#include "adioi.h"
-
-struct ADIOI_Fns_struct ADIO_NFS_operations = {
-    ADIOI_NFS_Open, /* Open */
-    ADIOI_FAILSAFE_OpenColl, /* OpenColl */
-    ADIOI_NFS_ReadContig, /* ReadContig */
-    ADIOI_NFS_WriteContig, /* WriteContig */
-    ADIOI_GEN_ReadStridedColl, /* ReadStridedColl */
-    ADIOI_GEN_WriteStridedColl, /* WriteStridedColl */
-    ADIOI_GEN_SeekIndividual, /* SeekIndividual */
-    ADIOI_NFS_Fcntl, /* Fcntl */
-    ADIOI_GEN_SetInfo, /* SetInfo */
-    ADIOI_NFS_ReadStrided, /* ReadStrided */
-    ADIOI_NFS_WriteStrided, /* WriteStrided */
-    ADIOI_GEN_Close, /* Close */
-    /* Even with lockd running and NFS mounted 'noac', we have been unable to
-     * gaurantee correct behavior over NFS with asyncronous I/O operations */
-    ADIOI_FAKE_IreadContig, /* IreadContig */
-    ADIOI_FAKE_IwriteContig, /* IwriteContig */
-    ADIOI_NFS_ReadDone, /* ReadDone */
-    ADIOI_NFS_WriteDone, /* WriteDone */
-    ADIOI_NFS_ReadComplete, /* ReadComplete */
-    ADIOI_NFS_WriteComplete, /* WriteComplete */
-    ADIOI_GEN_IreadStrided, /* IreadStrided */
-    ADIOI_GEN_IwriteStrided, /* IwriteStrided */
-    ADIOI_GEN_Flush, /* Flush */
-    ADIOI_NFS_Resize, /* Resize */
-    ADIOI_GEN_Delete, /* Delete */
-    ADIOI_NFS_Feature, /* Features */
-    "NFS:",  /* fsname: just a string */
-    ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
-    ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
-};
diff --git a/3rd-party/romio321/adio/ad_nfs/ad_nfs.h b/3rd-party/romio321/adio/ad_nfs/ad_nfs.h
deleted file mode 100644
index dd02bf4a2c3..00000000000
--- a/3rd-party/romio321/adio/ad_nfs/ad_nfs.h
+++ /dev/null
@@ -1,88 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#ifndef AD_NFS_INCLUDE
-#define AD_NFS_INCLUDE
-
-#include "adio.h"
-
-#include <unistd.h>
-#include <sys/types.h>
-#include <fcntl.h>
-
-#ifdef HAVE_SIGNAL_H
-#include <signal.h>
-#endif
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-#ifdef HAVE_AIO_LITE_H
-#include <aio-lite.h>
-#else
- #ifdef  HAVE_AIO_H
- #include <aio.h>
- #endif
- #ifdef HAVE_SYS_AIO_H
- #include <sys/aio.h>
- #endif
-#endif
-
-/* Workaround for incomplete set of definitions if __REDIRECT is not 
-   defined and large file support is used in aio.h */
-#if !defined(__REDIRECT) && defined(__USE_FILE_OFFSET64)
-#define aiocb aiocb64
-#endif
-
-int ADIOI_NFS_aio(ADIO_File fd, void *buf, int len, ADIO_Offset offset,
-                  int wr, MPI_Request *request);
-
-#ifdef SX4
-#define lseek llseek
-#endif
-
-void ADIOI_NFS_Open(ADIO_File fd, int *error_code);
-void ADIOI_NFS_ReadContig(ADIO_File fd, void *buf, int count, 
-                      MPI_Datatype datatype, int file_ptr_type,
-                     ADIO_Offset offset, ADIO_Status *status, int
-		     *error_code);
-void ADIOI_NFS_WriteContig(ADIO_File fd, const void *buf, int count,
-                      MPI_Datatype datatype, int file_ptr_type,
-                      ADIO_Offset offset, ADIO_Status *status, int
-		      *error_code);   
-void ADIOI_NFS_IwriteContig(ADIO_File fd, void *buf, int count, 
-                      MPI_Datatype datatype, int file_ptr_type,
-                      ADIO_Offset offset, ADIO_Request *request, int
-		      *error_code);   
-void ADIOI_NFS_IreadContig(ADIO_File fd, void *buf, int count, 
-                      MPI_Datatype datatype, int file_ptr_type,
-                      ADIO_Offset offset, ADIO_Request *request, int
-		      *error_code);   
-int ADIOI_NFS_ReadDone(ADIO_Request *request, ADIO_Status *status, int
-		       *error_code);
-int ADIOI_NFS_WriteDone(ADIO_Request *request, ADIO_Status *status, int
-		       *error_code);
-void ADIOI_NFS_ReadComplete(ADIO_Request *request, ADIO_Status *status, int
-		       *error_code); 
-void ADIOI_NFS_WriteComplete(ADIO_Request *request, ADIO_Status *status,
-			int *error_code); 
-void ADIOI_NFS_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct, int
-		*error_code); 
-void ADIOI_NFS_WriteStrided(ADIO_File fd, const void *buf, int count,
-		       MPI_Datatype datatype, int file_ptr_type,
-		       ADIO_Offset offset, ADIO_Status *status, int
-		       *error_code);
-void ADIOI_NFS_ReadStrided(ADIO_File fd, void *buf, int count,
-		       MPI_Datatype datatype, int file_ptr_type,
-		       ADIO_Offset offset, ADIO_Status *status, int
-		       *error_code);
-void ADIOI_NFS_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code);
-void ADIOI_NFS_Get_shared_fp(ADIO_File fd, ADIO_Offset size, ADIO_Offset *shared_fp,
-			 int *error_code);
-void ADIOI_NFS_Set_shared_fp(ADIO_File fd, ADIO_Offset offset, int *error_code);
-void ADIOI_NFS_Resize(ADIO_File fd, ADIO_Offset size, int *error_code);
-int ADIOI_NFS_Feature(ADIO_File fd, int feature_flag);
-
-#endif
diff --git a/3rd-party/romio321/adio/ad_nfs/ad_nfs_done.c b/3rd-party/romio321/adio/ad_nfs/ad_nfs_done.c
deleted file mode 100644
index ff688546cab..00000000000
--- a/3rd-party/romio321/adio/ad_nfs/ad_nfs_done.c
+++ /dev/null
@@ -1,19 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_nfs.h"
-
-int ADIOI_NFS_ReadDone(ADIO_Request *request, ADIO_Status *status,
-		       int *error_code)
-{
-	*error_code = MPI_SUCCESS;
-	return 1;
-}
-int ADIOI_NFS_WriteDone(ADIO_Request *request, ADIO_Status *status,
-			int *error_code)
-{
-    return ADIOI_NFS_ReadDone(request, status, error_code);
-} 
diff --git a/3rd-party/romio321/adio/ad_nfs/ad_nfs_fcntl.c b/3rd-party/romio321/adio/ad_nfs/ad_nfs_fcntl.c
deleted file mode 100644
index c73006e6cd9..00000000000
--- a/3rd-party/romio321/adio/ad_nfs/ad_nfs_fcntl.c
+++ /dev/null
@@ -1,65 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_nfs.h"
-#include "adio_extern.h"
-/* #ifdef MPISGI
-#include "mpisgi2.h"
-#endif */
-
-void ADIOI_NFS_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct, int *error_code)
-{
-    static char myname[] = "ADIOI_NFS_FCNTL";
-
-    switch(flag) {
-    case ADIO_FCNTL_GET_FSIZE:
-	ADIOI_READ_LOCK(fd, 0, SEEK_SET, 1);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-	fcntl_struct->fsize = lseek(fd->fd_sys, 0, SEEK_END);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-	ADIOI_UNLOCK(fd, 0, SEEK_SET, 1);
-	if (fd->fp_sys_posn != -1) {
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-	    lseek(fd->fd_sys, fd->fp_sys_posn, SEEK_SET);
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-	}
-	if (fcntl_struct->fsize == -1) {
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE, myname,
-					       __LINE__, MPI_ERR_IO, "**io",
-					       "**io %s", strerror(errno));
-	}
-	else *error_code = MPI_SUCCESS;
-	break;
-
-    case ADIO_FCNTL_SET_DISKSPACE:
-	ADIOI_GEN_Prealloc(fd, fcntl_struct->diskspace, error_code);
-	break;
-
-    case ADIO_FCNTL_SET_ATOMICITY:
-	fd->atomicity = (fcntl_struct->atomicity == 0) ? 0 : 1;
-	*error_code = MPI_SUCCESS;
-	break;
-
-    default:
-	/* --BEGIN ERROR HANDLING-- */
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   MPI_ERR_ARG, 
-					   "**flag", "**flag %d", flag);
-	return;
-	/* --END ERROR HANDLING-- */
-    }
-}
diff --git a/3rd-party/romio321/adio/ad_nfs/ad_nfs_features.c b/3rd-party/romio321/adio/ad_nfs/ad_nfs_features.c
deleted file mode 100644
index 05b061acf47..00000000000
--- a/3rd-party/romio321/adio/ad_nfs/ad_nfs_features.c
+++ /dev/null
@@ -1,24 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *
- *  (C) 2008 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include "adio.h"
-#include "ad_nfs.h"
-
-int ADIOI_NFS_Feature(ADIO_File fd, int flag)
-{       
-        switch(flag) {
-                case ADIO_SHARED_FP:
-                case ADIO_LOCKS:
-                case ADIO_SEQUENTIAL:
-                case ADIO_DATA_SIEVING_WRITES:
-			return 1;
-                case ADIO_SCALABLE_OPEN:
-		case ADIO_UNLINK_AFTER_CLOSE:
-		case ADIO_SCALABLE_RESIZE:
-                default:
-                        return 0;
-        }
-}
diff --git a/3rd-party/romio321/adio/ad_nfs/ad_nfs_getsh.c b/3rd-party/romio321/adio/ad_nfs/ad_nfs_getsh.c
deleted file mode 100644
index 974d547cc13..00000000000
--- a/3rd-party/romio321/adio/ad_nfs/ad_nfs_getsh.c
+++ /dev/null
@@ -1,105 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_nfs.h"
-
-/* returns the current location of the shared_fp in terms of the
-   no. of etypes relative to the current view, and also increments the
-   shared_fp by the number of etypes to be accessed (incr) in the read
-   or write following this function. */
-
-void ADIOI_NFS_Get_shared_fp(ADIO_File fd, ADIO_Offset incr, ADIO_Offset *shared_fp,
-			 int *error_code)
-{
-    ADIO_Offset new_fp;
-    ssize_t err;
-    MPI_Comm dupcommself;
-    static char myname[] = "ADIOI_NFS_GET_SHARED_FP";
-
-    if (fd->shared_fp_fd == ADIO_FILE_NULL) {
-	MPI_Comm_dup(MPI_COMM_SELF, &dupcommself);
-	fd->shared_fp_fd = ADIO_Open(MPI_COMM_SELF, dupcommself,
-				     fd->shared_fp_fname, 
-				     fd->file_system,
-				     fd->fns,
-				     ADIO_CREATE | ADIO_RDWR | ADIO_DELETE_ON_CLOSE, 
-				     0, MPI_BYTE, MPI_BYTE, MPI_INFO_NULL, 
-				     ADIO_PERM_NULL, error_code);
-	if (*error_code != MPI_SUCCESS) return;
-	*shared_fp = 0;
-	ADIOI_WRITE_LOCK(fd->shared_fp_fd, 0, SEEK_SET, sizeof(ADIO_Offset));
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
-#endif
-	err = read(fd->shared_fp_fd->fd_sys, shared_fp, sizeof(ADIO_Offset));
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
-#endif
-        /* if the file is empty, the above read may return error
-           (reading beyond end of file). In that case, shared_fp = 0, 
-           set above, is the correct value. */
-    }
-    else {
-	ADIOI_WRITE_LOCK(fd->shared_fp_fd, 0, SEEK_SET, sizeof(ADIO_Offset));
-
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-	err = lseek(fd->shared_fp_fd->fd_sys, 0, SEEK_SET);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-	if (err == 0) {
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
-#endif
-	    err = read(fd->shared_fp_fd->fd_sys, shared_fp,
-		       sizeof(ADIO_Offset));
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
-#endif
-	}
-	if (err == -1) {
-	    ADIOI_UNLOCK(fd->shared_fp_fd, 0, SEEK_SET, sizeof(ADIO_Offset));
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE, myname,
-					       __LINE__, MPI_ERR_IO, "**io",
-					       "**io %s", strerror(errno));
-	    return;
-	}
-    }
-
-    if (incr == 0) {goto done;}
-
-    new_fp = *shared_fp + incr;
-
-#ifdef ADIOI_MPE_LOGGING
-    MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-    err = lseek(fd->shared_fp_fd->fd_sys, 0, SEEK_SET);
-#ifdef ADIOI_MPE_LOGGING
-    MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-    if (err == 0) {
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
-#endif
-	err = write(fd->shared_fp_fd->fd_sys, &new_fp, sizeof(ADIO_Offset));
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
-#endif
-    }
-done:
-    ADIOI_UNLOCK(fd->shared_fp_fd, 0, SEEK_SET, sizeof(ADIO_Offset));
-    if (err == -1) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io",
-					   "**io %s", strerror(errno));
-    }
-    else *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_nfs/ad_nfs_hints.c b/3rd-party/romio321/adio/ad_nfs/ad_nfs_hints.c
deleted file mode 100644
index 0e5386d22bb..00000000000
--- a/3rd-party/romio321/adio/ad_nfs/ad_nfs_hints.c
+++ /dev/null
@@ -1,13 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_nfs.h"
-
-void ADIOI_NFS_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code)
-{
-    ADIOI_GEN_SetInfo(fd, users_info, error_code); 
-}
diff --git a/3rd-party/romio321/adio/ad_nfs/ad_nfs_iread.c b/3rd-party/romio321/adio/ad_nfs/ad_nfs_iread.c
deleted file mode 100644
index 28d20281059..00000000000
--- a/3rd-party/romio321/adio/ad_nfs/ad_nfs_iread.c
+++ /dev/null
@@ -1,37 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_nfs.h"
-
-#ifdef ROMIO_HAVE_WORKING_AIO
-/* nearly identical to ADIOI_GEN_IreadContig, except we lock around I/O */
-void ADIOI_NFS_IreadContig(ADIO_File fd, void *buf, int count, 
-			   MPI_Datatype datatype, int file_ptr_type,
-			   ADIO_Offset offset, ADIO_Request *request,
-			   int *error_code)  
-{
-    MPI_Count len, typesize;
-    int aio_errno = 0;
-    static char myname[] = "ADIOI_NFS_IREADCONTIG";
-
-    MPI_Type_size_x(datatype, &typesize);
-    len = count * typesize;
-
-    if (file_ptr_type == ADIO_INDIVIDUAL) offset = fd->fp_ind;
-    aio_errno = ADIOI_NFS_aio(fd, buf, len, offset, 0, request);
-    if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind += len;
-
-    fd->fp_sys_posn = -1;
-
-    if (aio_errno != 0) {
-	/* --BEGIN ERROR HANDLING-- */
-	MPIO_ERR_CREATE_CODE_ERRNO(myname, aio_errno, error_code);
-	return;
-	/* --END ERROR HANDLING-- */
-    }
-    else *error_code = MPI_SUCCESS;
-}
-#endif
diff --git a/3rd-party/romio321/adio/ad_nfs/ad_nfs_iwrite.c b/3rd-party/romio321/adio/ad_nfs/ad_nfs_iwrite.c
deleted file mode 100644
index a5f988cf205..00000000000
--- a/3rd-party/romio321/adio/ad_nfs/ad_nfs_iwrite.c
+++ /dev/null
@@ -1,130 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_nfs.h"
-
-#include "../../mpi-io/mpioimpl.h"
-#include "../../mpi-io/mpioprof.h"
-#include "mpiu_greq.h"
-
-#include <string.h>
-
-#ifdef ROMIO_HAVE_WORKING_AIO
-static MPIX_Grequest_class ADIOI_GEN_greq_class = 0;
-/* this routine is nearly identical to ADIOI_GEN_IwriteContig, except we lock
- * around I/O */
-void ADIOI_NFS_IwriteContig(ADIO_File fd, void *buf, int count, 
-                MPI_Datatype datatype, int file_ptr_type,
-                ADIO_Offset offset, ADIO_Request *request, int *error_code)  
-{
-    MPI_Count len, typesize;
-    int aio_errno = 0;
-    static char myname[] = "ADIOI_NFS_IWRITECONTIG";
-
-    MPI_Type_size_x(datatype, &typesize);
-    len = count * typesize;
-
-    if (file_ptr_type == ADIO_INDIVIDUAL) offset = fd->fp_ind;
-    aio_errno = ADIOI_NFS_aio(fd, buf, len, offset, 1, request);
-    if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind += len;
-
-    fd->fp_sys_posn = -1;
-
-    if (aio_errno != 0) {
-	/* --BEGIN ERROR HANDLING-- */
-	MPIO_ERR_CREATE_CODE_ERRNO(myname, aio_errno, error_code);
-	return;
-	/* --END ERROR HANDLING-- */
-    }
-    else *error_code = MPI_SUCCESS;
-    return;
-}
-#endif
-
-/* This function is for implementation convenience. It is not user-visible.
- * It takes care of the differences in the interface for nonblocking I/O
- * on various Unix machines! If wr==1 write, wr==0 read.
- *
- * Returns 0 on success, -errno on failure.
- */
-#ifdef ROMIO_HAVE_WORKING_AIO
-int ADIOI_NFS_aio(ADIO_File fd, void *buf, int len, ADIO_Offset offset,
-		  int wr, MPI_Request *request)
-{
-    int err=-1, fd_sys;
-    int error_code, this_errno;
-
-    struct aiocb *aiocbp;
-    ADIOI_AIO_Request *aio_req;
-    MPI_Status status;
-
-    fd_sys = fd->fd_sys;
-
-    aio_req = (ADIOI_AIO_Request*)ADIOI_Calloc(sizeof(ADIOI_AIO_Request), 1);
-    aiocbp = (struct aiocb *) ADIOI_Calloc(sizeof(struct aiocb), 1);
-    aiocbp->aio_offset = offset;
-    aiocbp->aio_buf    = buf;
-    aiocbp->aio_nbytes = len;
-
-#ifdef HAVE_STRUCT_AIOCB_AIO_WHENCE
-    aiocbp->aio_whence = SEEK_SET;
-#endif
-#ifdef HAVE_STRUCT_AIOCB_AIO_FILDES
-    aiocbp->aio_fildes = fd_sys;
-#endif
-#ifdef HAVE_STRUCT_AIOCB_AIO_SIGEVENT
-# ifdef AIO_SIGNOTIFY_NONE
-    aiocbp->aio_sigevent.sigev_notify = SIGEV_NONE;
-# endif
-    aiocbp->aio_sigevent.sigev_signo = 0;
-#endif
-#ifdef HAVE_STRUCT_AIOCB_AIO_REQPRIO
-# ifdef AIO_PRIO_DFL
-    aiocbp->aio_reqprio = AIO_PRIO_DFL;   /* not needed in DEC Unix 4.0 */
-# else
-    aiocbp->aio_reqprio = 0;
-# endif
-#endif
-
-    if (wr) ADIOI_WRITE_LOCK(fd, offset, SEEK_SET, len);
-    else ADIOI_READ_LOCK(fd, offset, SEEK_SET, len);
-
-#ifndef ROMIO_HAVE_AIO_CALLS_NEED_FILEDES
-    if (wr) err = aio_write(aiocbp);
-    else err = aio_read(aiocbp);
-#else
-    /* Broken IBM interface */
-    if (wr) err = aio_write(fd_sys, aiocbp);
-    else err = aio_read(fd_sys, aiocbp);
-#endif
-
-    this_errno = errno;
-    ADIOI_UNLOCK(fd, offset, SEEK_SET, len);
-
-    if (err == -1) {
-	if (this_errno == EAGAIN) {
-        /* exceeded the max. no. of outstanding requests.
-           complete all previous async. requests and try again. */
-	    ADIO_WriteContig(fd, buf, len, MPI_BYTE, ADIO_EXPLICIT_OFFSET,
-			    offset, &status, &error_code);
-	    MPIO_Completed_request_create(&fd, len, &error_code, request);
-	    return 0;
-	} else {
-	    return -this_errno;
-	}
-    }
-    aio_req->aiocbp = aiocbp;
-    if (ADIOI_GEN_greq_class == 0) {
-	    MPIX_Grequest_class_create(ADIOI_GEN_aio_query_fn, 
-			    ADIOI_GEN_aio_free_fn, MPIU_Greq_cancel_fn, 
-			    ADIOI_GEN_aio_poll_fn, ADIOI_GEN_aio_wait_fn, 
-			    &ADIOI_GEN_greq_class);
-    }
-    MPIX_Grequest_class_allocate(ADIOI_GEN_greq_class, aio_req, request);
-    memcpy(&(aio_req->req), request, sizeof(MPI_Request));
-    return 0;
-}
-#endif
diff --git a/3rd-party/romio321/adio/ad_nfs/ad_nfs_open.c b/3rd-party/romio321/adio/ad_nfs/ad_nfs_open.c
deleted file mode 100644
index d8763292b91..00000000000
--- a/3rd-party/romio321/adio/ad_nfs/ad_nfs_open.c
+++ /dev/null
@@ -1,58 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_nfs.h"
-
-void ADIOI_NFS_Open(ADIO_File fd, int *error_code)
-{
-    int perm, amode;
-    mode_t old_mask;
-    static char myname[] = "ADIOI_NFS_OPEN";
-
-    if (fd->perm == ADIO_PERM_NULL) {
-	old_mask = umask(022);
-	umask(old_mask);
-	perm = old_mask ^ 0666;
-    }
-    else perm = fd->perm;
-    
-    amode = 0;
-    if (fd->access_mode & ADIO_CREATE)
-	amode = amode | O_CREAT;
-    if (fd->access_mode & ADIO_RDONLY)
-	amode = amode | O_RDONLY;
-    if (fd->access_mode & ADIO_WRONLY)
-	amode = amode | O_WRONLY;
-    if (fd->access_mode & ADIO_RDWR)
-	amode = amode | O_RDWR;
-    if (fd->access_mode & ADIO_EXCL)
-	amode = amode | O_EXCL;
-
-#ifdef ADIOI_MPE_LOGGING
-    MPE_Log_event( ADIOI_MPE_open_a, 0, NULL );
-#endif
-    fd->fd_sys = open(fd->filename, amode, perm);
-#ifdef ADIOI_MPE_LOGGING
-    MPE_Log_event( ADIOI_MPE_open_b, 0, NULL );
-#endif
-    fd->fd_direct = -1;
-
-    if ((fd->fd_sys != -1) && (fd->access_mode & ADIO_APPEND)) {
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-        fd->fp_ind = fd->fp_sys_posn = lseek(fd->fd_sys, 0, SEEK_END);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-    }
-
-    if (fd->fd_sys == -1) {
-	*error_code = ADIOI_Err_create_code(myname, fd->filename, errno);
-    }
-    else *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_nfs/ad_nfs_read.c b/3rd-party/romio321/adio/ad_nfs/ad_nfs_read.c
deleted file mode 100644
index 6dce2893118..00000000000
--- a/3rd-party/romio321/adio/ad_nfs/ad_nfs_read.c
+++ /dev/null
@@ -1,552 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_nfs.h"
-#include "adio_extern.h"
-
-void ADIOI_NFS_ReadContig(ADIO_File fd, void *buf, int count, 
-                     MPI_Datatype datatype, int file_ptr_type,
-		     ADIO_Offset offset, ADIO_Status *status, int *error_code)
-{
-    int err=-1;
-    MPI_Count datatype_size, len;
-    static char myname[] = "ADIOI_NFS_READCONTIG";
-
-    MPI_Type_size_x(datatype, &datatype_size);
-    len = datatype_size * count;
-
-    if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-	if (fd->fp_sys_posn != offset) {
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-	    lseek(fd->fd_sys, offset, SEEK_SET);
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-        }
-	if (fd->atomicity)
-	    ADIOI_WRITE_LOCK(fd, offset, SEEK_SET, len);
-	else ADIOI_READ_LOCK(fd, offset, SEEK_SET, len);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
-#endif
-	err = read(fd->fd_sys, buf, len);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
-#endif
-	ADIOI_UNLOCK(fd, offset, SEEK_SET, len);
-	fd->fp_sys_posn = offset + err;
-	/* individual file pointer not updated */        
-    }
-    else {  /* read from curr. location of ind. file pointer */
-	offset = fd->fp_ind;
-	if (fd->fp_sys_posn != fd->fp_ind) {
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-	    lseek(fd->fd_sys, fd->fp_ind, SEEK_SET);
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-        }
-	if (fd->atomicity)
-	    ADIOI_WRITE_LOCK(fd, offset, SEEK_SET, len);
-	else ADIOI_READ_LOCK(fd, offset, SEEK_SET, len);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
-#endif
-	err = read(fd->fd_sys, buf, len);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
-#endif
-	ADIOI_UNLOCK(fd, offset, SEEK_SET, len);
-	fd->fp_ind += err;
-	fd->fp_sys_posn = fd->fp_ind;
-    }
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (err == -1) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io", "**io %s", strerror(errno));
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-#ifdef HAVE_STATUS_SET_BYTES
-    MPIR_Status_set_bytes(status, datatype, err);
-#endif
-
-    *error_code = MPI_SUCCESS;
-}
-
-
-
-#ifdef ADIOI_MPE_LOGGING
-#define ADIOI_BUFFERED_READ \
-{ \
-    if (req_off >= readbuf_off + readbuf_len) { \
-	readbuf_off = req_off; \
-	readbuf_len = (int) (ADIOI_MIN(max_bufsize, end_offset-readbuf_off+1));\
-        MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL ); \
-	lseek(fd->fd_sys, readbuf_off, SEEK_SET);\
-        MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL ); \
-        if (!(fd->atomicity)) ADIOI_READ_LOCK(fd, readbuf_off, SEEK_SET, readbuf_len);\
-        MPE_Log_event( ADIOI_MPE_read_a, 0, NULL ); \
-        err = read(fd->fd_sys, readbuf, readbuf_len);\
-        MPE_Log_event( ADIOI_MPE_read_b, 0, NULL ); \
-        if (!(fd->atomicity)) ADIOI_UNLOCK(fd, readbuf_off, SEEK_SET, readbuf_len);\
-        if (err == -1) err_flag = 1; \
-    } \
-    while (req_len > readbuf_off + readbuf_len - req_off) { \
-	partial_read = (int) (readbuf_off + readbuf_len - req_off); \
-	tmp_buf = (char *) ADIOI_Malloc(partial_read); \
-	memcpy(tmp_buf, readbuf+readbuf_len-partial_read, partial_read); \
-	ADIOI_Free(readbuf); \
-	readbuf = (char *) ADIOI_Malloc(partial_read + max_bufsize); \
-	memcpy(readbuf, tmp_buf, partial_read); \
-	ADIOI_Free(tmp_buf); \
-	readbuf_off += readbuf_len-partial_read; \
-	readbuf_len = (int) (partial_read + ADIOI_MIN(max_bufsize, \
-				       end_offset-readbuf_off+1)); \
-        MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL ); \
-	lseek(fd->fd_sys, readbuf_off+partial_read, SEEK_SET);\
-        MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL ); \
-        if (!(fd->atomicity)) ADIOI_READ_LOCK(fd, readbuf_off+partial_read, SEEK_SET, readbuf_len-partial_read);\
-        MPE_Log_event( ADIOI_MPE_read_a, 0, NULL ); \
-        err = read(fd->fd_sys, readbuf+partial_read, readbuf_len-partial_read);\
-        MPE_Log_event( ADIOI_MPE_read_b, 0, NULL ); \
-        if (!(fd->atomicity)) ADIOI_UNLOCK(fd, readbuf_off+partial_read, SEEK_SET, readbuf_len-partial_read);\
-        if (err == -1) err_flag = 1; \
-    } \
-    memcpy((char *)buf + userbuf_off, readbuf+req_off-readbuf_off, req_len); \
-}
-#else
-#define ADIOI_BUFFERED_READ \
-{ \
-    if (req_off >= readbuf_off + readbuf_len) { \
-	readbuf_off = req_off; \
-	readbuf_len = (int) (ADIOI_MIN(max_bufsize, end_offset-readbuf_off+1));\
-	lseek(fd->fd_sys, readbuf_off, SEEK_SET);\
-        if (!(fd->atomicity)) ADIOI_READ_LOCK(fd, readbuf_off, SEEK_SET, readbuf_len);\
-        err = read(fd->fd_sys, readbuf, readbuf_len);\
-        if (!(fd->atomicity)) ADIOI_UNLOCK(fd, readbuf_off, SEEK_SET, readbuf_len);\
-        if (err == -1) err_flag = 1; \
-    } \
-    while (req_len > readbuf_off + readbuf_len - req_off) { \
-	partial_read = (int) (readbuf_off + readbuf_len - req_off); \
-	tmp_buf = (char *) ADIOI_Malloc(partial_read); \
-	memcpy(tmp_buf, readbuf+readbuf_len-partial_read, partial_read); \
-	ADIOI_Free(readbuf); \
-	readbuf = (char *) ADIOI_Malloc(partial_read + max_bufsize); \
-	memcpy(readbuf, tmp_buf, partial_read); \
-	ADIOI_Free(tmp_buf); \
-	readbuf_off += readbuf_len-partial_read; \
-	readbuf_len = (int) (partial_read + ADIOI_MIN(max_bufsize, \
-				       end_offset-readbuf_off+1)); \
-	lseek(fd->fd_sys, readbuf_off+partial_read, SEEK_SET);\
-        if (!(fd->atomicity)) ADIOI_READ_LOCK(fd, readbuf_off+partial_read, SEEK_SET, readbuf_len-partial_read);\
-        err = read(fd->fd_sys, readbuf+partial_read, readbuf_len-partial_read);\
-        if (!(fd->atomicity)) ADIOI_UNLOCK(fd, readbuf_off+partial_read, SEEK_SET, readbuf_len-partial_read);\
-        if (err == -1) err_flag = 1; \
-    } \
-    memcpy((char *)buf + userbuf_off, readbuf+req_off-readbuf_off, req_len); \
-}
-#endif
-
-
-void ADIOI_NFS_ReadStrided(ADIO_File fd, void *buf, int count,
-                       MPI_Datatype datatype, int file_ptr_type,
-                       ADIO_Offset offset, ADIO_Status *status, int
-                       *error_code)
-{
-/* offset is in units of etype relative to the filetype. */
-
-    ADIOI_Flatlist_node *flat_buf, *flat_file;
-    int i, j, k, err=-1, brd_size, st_index=0;
-    int num, size, sum, n_etypes_in_filetype, size_in_filetype;
-    MPI_Count bufsize;
-    int n_filetypes, etype_in_filetype;
-    ADIO_Offset abs_off_in_filetype=0;
-    int req_len, partial_read;
-    MPI_Count filetype_size, etype_size, buftype_size;
-    MPI_Aint filetype_extent, buftype_extent, lb; 
-    int buf_count, buftype_is_contig, filetype_is_contig;
-    ADIO_Offset userbuf_off;
-    ADIO_Offset off, req_off, disp, end_offset=0, readbuf_off, start_off;
-    char *readbuf, *tmp_buf, *value;
-    int st_n_filetypes, readbuf_len;
-    ADIO_Offset frd_size=0, new_frd_size, st_frd_size;
-    int new_brd_size, err_flag=0, info_flag, max_bufsize;
-
-    static char myname[] = "ADIOI_NFS_READSTRIDED";
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-
-    MPI_Type_size_x(fd->filetype, &filetype_size);
-    if ( ! filetype_size ) {
-#ifdef HAVE_STATUS_SET_BYTES
-	MPIR_Status_set_bytes(status, datatype, 0);
-#endif
-	*error_code = MPI_SUCCESS; 
-	return;
-    }
-
-    MPI_Type_get_extent(fd->filetype, &lb, &filetype_extent);
-    MPI_Type_size_x(datatype, &buftype_size);
-    MPI_Type_get_extent(datatype, &lb, &buftype_extent);
-    etype_size = fd->etype_size;
-
-    bufsize = buftype_size * count;
-
-/* get max_bufsize from the info object. */
-
-    value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
-    ADIOI_Info_get(fd->info, "ind_rd_buffer_size", MPI_MAX_INFO_VAL, value, 
-                 &info_flag);
-    max_bufsize = atoi(value);
-    ADIOI_Free(value);
-
-    if (!buftype_is_contig && filetype_is_contig) {
-
-/* noncontiguous in memory, contiguous in file. */
-
-	flat_buf = ADIOI_Flatten_and_find(datatype);
-
-        off = (file_ptr_type == ADIO_INDIVIDUAL) ? fd->fp_ind : 
-                 fd->disp + etype_size * offset;
-
-	start_off = off;
-	end_offset = off + bufsize - 1;
-        readbuf_off = off;
-        readbuf = (char *) ADIOI_Malloc(max_bufsize);
-        readbuf_len = (int) (ADIOI_MIN(max_bufsize, end_offset-readbuf_off+1));
-
-/* if atomicity is true, lock (exclusive) the region to be accessed */
-        if (fd->atomicity)
-            ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
-
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-	lseek(fd->fd_sys, readbuf_off, SEEK_SET);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-        if (!(fd->atomicity)) ADIOI_READ_LOCK(fd, readbuf_off, SEEK_SET, readbuf_len);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
-#endif
-        err = read(fd->fd_sys, readbuf, readbuf_len);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
-#endif
-        if (!(fd->atomicity)) ADIOI_UNLOCK(fd, readbuf_off, SEEK_SET, readbuf_len);
-        if (err == -1) err_flag = 1;
-
-        for (j=0; j<count; j++) 
-            for (i=0; i<flat_buf->count; i++) {
-                userbuf_off = j*buftype_extent + flat_buf->indices[i];
-		req_off = off;
-		req_len = flat_buf->blocklens[i];
-		ADIOI_BUFFERED_READ
-                off += flat_buf->blocklens[i];
-            }
-
-        if (fd->atomicity)
-            ADIOI_UNLOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
-
-        if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
-
-	ADIOI_Free(readbuf); /* malloced in the buffered_read macro */
-
-	if (err_flag) {
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE, myname,
-					       __LINE__, MPI_ERR_IO, "**io",
-					       "**io %s", strerror(errno));
-	}
-	else *error_code = MPI_SUCCESS;
-    }
-
-    else {  /* noncontiguous in file */
-
-/* filetype already flattened in ADIO_Open */
-	flat_file = ADIOI_Flatlist;
-	while (flat_file->type != fd->filetype) flat_file = flat_file->next;
-	disp = fd->disp;
-
-	if (file_ptr_type == ADIO_INDIVIDUAL) {
-          /* Wei-keng reworked type processing to be a bit more efficient */
-           offset       = fd->fp_ind - disp;
-           n_filetypes  = (offset - flat_file->indices[0]) / filetype_extent;
-          offset -= (ADIO_Offset)n_filetypes * filetype_extent;
-          /* now offset is local to this extent */
- 
-           /* find the block where offset is located, skip blocklens[i]==0 */
-           for (i=0; i<flat_file->count; i++) {
-               ADIO_Offset dist;
-               if (flat_file->blocklens[i] == 0) continue;
-               dist = flat_file->indices[i] + flat_file->blocklens[i] - offset;
-               /* frd_size is from offset to the end of block i */
-              if (dist == 0) {
-                  i++;
-                  offset   = flat_file->indices[i];
-                  frd_size = flat_file->blocklens[i];
-                  break;
-              }
-              if (dist > 0 ) { 
-                   frd_size = dist;
-		   break;
-              }
-          }
-           st_index = i;  /* starting index in flat_file->indices[] */
-           offset += disp + (ADIO_Offset)n_filetypes*filetype_extent;
-       }
-	else {
-	    n_etypes_in_filetype = filetype_size/etype_size;
-	    n_filetypes = (int) (offset / n_etypes_in_filetype);
-	    etype_in_filetype = (int) (offset % n_etypes_in_filetype);
-	    size_in_filetype = etype_in_filetype * etype_size;
- 
-	    sum = 0;
-	    for (i=0; i<flat_file->count; i++) {
-		sum += flat_file->blocklens[i];
-		if (sum > size_in_filetype) {
-		    st_index = i;
-		    frd_size = sum - size_in_filetype;
-		    abs_off_in_filetype = flat_file->indices[i] +
-			size_in_filetype - (sum - flat_file->blocklens[i]);
-		    break;
-		}
-	    }
-
-	    /* abs. offset in bytes in the file */
-	    offset = disp + (ADIO_Offset) n_filetypes*filetype_extent + 
-		    abs_off_in_filetype;
-	}
-
-        start_off = offset;
-
-       /* Wei-keng Liao: read request is within a single flat_file contig
-        * block e.g. with subarray types that actually describe the whole
-        * array */
-       if (buftype_is_contig && bufsize <= frd_size) {
-            ADIO_ReadContig(fd, buf, bufsize, MPI_BYTE, ADIO_EXPLICIT_OFFSET,
-                             offset, status, error_code);
-
-           if (file_ptr_type == ADIO_INDIVIDUAL) {
-                /* update MPI-IO file pointer to point to the first byte that 
-                * can be accessed in the fileview. */
-               fd->fp_ind = offset + bufsize;
-               if (bufsize == frd_size) {
-                   do {
-                       st_index++;
-                       if (st_index == flat_file->count) {
-                           st_index = 0;
-                           n_filetypes++;
-                       }
-                    } while (flat_file->blocklens[st_index] == 0);
-                   fd->fp_ind = disp + flat_file->indices[st_index]
-                               + n_filetypes*filetype_extent;
-               }
-           }
-           fd->fp_sys_posn = -1;   /* set it to null. */ 
-#ifdef HAVE_STATUS_SET_BYTES
-           MPIR_Status_set_bytes(status, datatype, bufsize);
-#endif 
-            return;
-       }
-
-       /* Calculate end_offset, the last byte-offset that will be accessed.
-         e.g., if start_offset=0 and 100 bytes to be read, end_offset=99*/
-
-	st_frd_size = frd_size;
-	st_n_filetypes = n_filetypes;
-	i = 0;
-	j = st_index;
-	off = offset;
-	frd_size = ADIOI_MIN(st_frd_size, bufsize);
-	while (i < bufsize) {
-	    i += frd_size;
-	    end_offset = off + frd_size - 1;
-            j = (j+1) % flat_file->count;
-            n_filetypes += (j == 0) ? 1 : 0;
-            while (flat_file->blocklens[j]==0) {
-               j = (j+1) % flat_file->count;
-               n_filetypes += (j == 0) ? 1 : 0;
-	    }
-
-	    off = disp + flat_file->indices[j] + (ADIO_Offset) n_filetypes*filetype_extent;
-	    frd_size = ADIOI_MIN(flat_file->blocklens[j], bufsize-i);
-	}
-
-/* if atomicity is true, lock (exclusive) the region to be accessed */
-        if (fd->atomicity)
-            ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
-
-        /* initial read into readbuf */
-	readbuf_off = offset;
-	readbuf = (char *) ADIOI_Malloc(max_bufsize);
-	readbuf_len = (int) (ADIOI_MIN(max_bufsize, end_offset-readbuf_off+1));
-
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-	lseek(fd->fd_sys, offset, SEEK_SET);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-        if (!(fd->atomicity)) ADIOI_READ_LOCK(fd, offset, SEEK_SET, readbuf_len);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
-#endif
-        err = read(fd->fd_sys, readbuf, readbuf_len);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
-#endif
-        if (!(fd->atomicity)) ADIOI_UNLOCK(fd, offset, SEEK_SET, readbuf_len);
-
-        if (err == -1) err_flag = 1;
-
-	if (buftype_is_contig && !filetype_is_contig) {
-
-/* contiguous in memory, noncontiguous in file. should be the most
-   common case. */
-
-	    i = 0;
-	    j = st_index;
-	    off = offset;
-	    n_filetypes = st_n_filetypes;
-	    frd_size = ADIOI_MIN(st_frd_size, bufsize);
-	    while (i < bufsize) {
-                if (frd_size) { 
-                    /* TYPE_UB and TYPE_LB can result in 
-                       frd_size = 0. save system call in such cases */ 
-		    /* lseek(fd->fd_sys, off, SEEK_SET);
-		    err = read(fd->fd_sys, ((char *) buf) + i, frd_size);*/
-
-		    req_off = off;
-		    req_len = frd_size;
-		    userbuf_off = i;
-		    ADIOI_BUFFERED_READ
-		}
-		i += frd_size;
-
-                if (off + frd_size < disp + flat_file->indices[j] +
-                   flat_file->blocklens[j] + (ADIO_Offset) n_filetypes*filetype_extent)
-                       off += frd_size;
-                /* did not reach end of contiguous block in filetype.
-                   no more I/O needed. off is incremented by frd_size. */
-                else {
-                    j = (j+1) % flat_file->count;
-                    n_filetypes += (j == 0) ? 1 : 0;
-                    while (flat_file->blocklens[j]==0) {
-                        j = (j+1) % flat_file->count;
-                        n_filetypes += (j == 0) ? 1 : 0;
-                    }
-		    off = disp + flat_file->indices[j] + 
-                                        (ADIO_Offset) n_filetypes*filetype_extent;
-		    frd_size = ADIOI_MIN(flat_file->blocklens[j], bufsize-i);
-		}
-	    }
-	}
-	else {
-/* noncontiguous in memory as well as in file */
-
-            ADIO_Offset i;
-	    flat_buf = ADIOI_Flatten_and_find(datatype);
-
-	    k = num = buf_count = 0;
-	    i = flat_buf->indices[0];
-	    j = st_index;
-	    off = offset;
-	    n_filetypes = st_n_filetypes;
-	    frd_size = st_frd_size;
-	    brd_size = flat_buf->blocklens[0];
-
-	    while (num < bufsize) {
-		size = ADIOI_MIN(frd_size, brd_size);
-		if (size) {
-		    /* lseek(fd->fd_sys, off, SEEK_SET);
-		    err = read(fd->fd_sys, ((char *) buf) + i, size); */
-
-		    req_off = off;
-		    req_len = size;
-		    userbuf_off = i;
-		    ADIOI_BUFFERED_READ
-		}
-
-		new_frd_size = frd_size;
-		new_brd_size = brd_size;
-
-		if (size == frd_size) {
-/* reached end of contiguous block in file */
-                    j = (j+1) % flat_file->count;
-                    n_filetypes += (j == 0) ? 1 : 0;
-                    while (flat_file->blocklens[j]==0) {
-                        j = (j+1) % flat_file->count;
-                        n_filetypes += (j == 0) ? 1 : 0;
-                    }
-		    off = disp + flat_file->indices[j] + 
-                                              (ADIO_Offset) n_filetypes*filetype_extent;
-
-		    new_frd_size = flat_file->blocklens[j];
-		    if (size != brd_size) {
-			i += size;
-			new_brd_size -= size;
-		    }
-		}
-
-		if (size == brd_size) {
-/* reached end of contiguous block in memory */
-
-		    k = (k + 1)%flat_buf->count;
-		    buf_count++;
-		    i = buftype_extent*(buf_count/flat_buf->count) +
-			flat_buf->indices[k];
-		    new_brd_size = flat_buf->blocklens[k];
-		    if (size != frd_size) {
-			off += size;
-			new_frd_size -= size;
-		    }
-		}
-		num += size;
-		frd_size = new_frd_size;
-                brd_size = new_brd_size;
-	    }
-	}
-	
-        if (fd->atomicity)
-            ADIOI_UNLOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
-
-	if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
-
-	ADIOI_Free(readbuf); /* malloced in the buffered_read macro */
-
-	if (err_flag) {
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE, myname,
-					       __LINE__, MPI_ERR_IO, "**io",
-					       "**io %s", strerror(errno));
-	}
-	else *error_code = MPI_SUCCESS;
-    }
-
-    fd->fp_sys_posn = -1;   /* set it to null. */
-
-#ifdef HAVE_STATUS_SET_BYTES
-    MPIR_Status_set_bytes(status, datatype, bufsize);
-/* This is a temporary way of filling in status. The right way is to 
-   keep track of how much data was actually read and placed in buf 
-   by ADIOI_BUFFERED_READ. */
-#endif
-
-    if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
-}
diff --git a/3rd-party/romio321/adio/ad_nfs/ad_nfs_resize.c b/3rd-party/romio321/adio/ad_nfs/ad_nfs_resize.c
deleted file mode 100644
index d86dfcc77ed..00000000000
--- a/3rd-party/romio321/adio/ad_nfs/ad_nfs_resize.c
+++ /dev/null
@@ -1,35 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2004 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_nfs.h"
-
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-
-/* NFS resize
- *
- * Note: we resize on all processors to guarantee that all processors
- * will have updated cache values.  This used to be the generic
- * implementation used by the majority of the ADIO implementations.
- */
-void ADIOI_NFS_Resize(ADIO_File fd, ADIO_Offset size, int *error_code)
-{
-    int err;
-    static char myname[] = "ADIOI_NFS_RESIZE";
-
-    err = ftruncate(fd->fd_sys, size);
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (err == -1) {
-	*error_code = ADIOI_Err_create_code(myname, fd->filename, errno);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-    *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_nfs/ad_nfs_setsh.c b/3rd-party/romio321/adio/ad_nfs/ad_nfs_setsh.c
deleted file mode 100644
index 42e558cb476..00000000000
--- a/3rd-party/romio321/adio/ad_nfs/ad_nfs_setsh.c
+++ /dev/null
@@ -1,74 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_nfs.h"
-
-/* set the shared file pointer to "offset" etypes relative to the current 
-   view */
-
-/*
-This looks very similar to ADIOI_GEN_Set_shared_fp, except this 
-function avoids locking the file twice.  The generic version does
-
-Write lock
-ADIO_WriteContig
-Unlock
-
-For NFS, ADIOI_NFS_WriteContig does a lock before writing to disable
-caching. To avoid the lock being called twice, this version for NFS does
-
-Write lock
-Lseek
-Write
-Unlock 
-
-*/
-
-void ADIOI_NFS_Set_shared_fp(ADIO_File fd, ADIO_Offset offset, int *error_code)
-{
-    ssize_t err;
-    MPI_Comm dupcommself;
-    static char myname[] = "ADIOI_NFS_SET_SHARED_FP";
-
-    if (fd->shared_fp_fd == ADIO_FILE_NULL) {
-	MPI_Comm_dup(MPI_COMM_SELF, &dupcommself);
-	fd->shared_fp_fd = ADIO_Open(MPI_COMM_SELF, dupcommself,
-				     fd->shared_fp_fname, 
-				     fd->file_system, fd->fns,
-				     ADIO_CREATE | ADIO_RDWR | ADIO_DELETE_ON_CLOSE, 
-				     0, MPI_BYTE, MPI_BYTE, MPI_INFO_NULL, 
-				     ADIO_PERM_NULL, error_code);
-    }
-
-    if (*error_code != MPI_SUCCESS) return;
-
-    ADIOI_WRITE_LOCK(fd->shared_fp_fd, 0, SEEK_SET, sizeof(ADIO_Offset));
-#ifdef ADIOI_MPE_LOGGING
-    MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-    lseek(fd->shared_fp_fd->fd_sys, 0, SEEK_SET);
-#ifdef ADIOI_MPE_LOGGING
-    MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-#ifdef ADIOI_MPE_LOGGING
-    MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
-#endif
-    err = write(fd->shared_fp_fd->fd_sys, &offset, sizeof(ADIO_Offset));
-#ifdef ADIOI_MPE_LOGGING
-    MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
-#endif
-    ADIOI_UNLOCK(fd->shared_fp_fd, 0, SEEK_SET, sizeof(ADIO_Offset));
-
-    if (err == -1) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io",
-					   "**io %s", strerror(errno));
-    }
-    else *error_code = MPI_SUCCESS;
-}
-
diff --git a/3rd-party/romio321/adio/ad_nfs/ad_nfs_wait.c b/3rd-party/romio321/adio/ad_nfs/ad_nfs_wait.c
deleted file mode 100644
index e1037fc5513..00000000000
--- a/3rd-party/romio321/adio/ad_nfs/ad_nfs_wait.c
+++ /dev/null
@@ -1,20 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_nfs.h"
-
-void ADIOI_NFS_ReadComplete(ADIO_Request *request, ADIO_Status *status,
-			    int *error_code)
-{
-	return;
-}
-
-
-void ADIOI_NFS_WriteComplete(ADIO_Request *request, ADIO_Status *status,
-			     int *error_code)
-{
-    ADIOI_NFS_ReadComplete(request, status, error_code);
-}
diff --git a/3rd-party/romio321/adio/ad_nfs/ad_nfs_write.c b/3rd-party/romio321/adio/ad_nfs/ad_nfs_write.c
deleted file mode 100644
index 06dca92ef84..00000000000
--- a/3rd-party/romio321/adio/ad_nfs/ad_nfs_write.c
+++ /dev/null
@@ -1,678 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_nfs.h"
-#include "adio_extern.h"
-
-void ADIOI_NFS_WriteContig(ADIO_File fd, const void *buf, int count,
-                     MPI_Datatype datatype, int file_ptr_type,
-		     ADIO_Offset offset, ADIO_Status *status, int *error_code)
-{
-    int err=-1;
-    MPI_Count datatype_size, len;
-    static char myname[] = "ADIOI_NFS_WRITECONTIG";
-
-    MPI_Type_size_x(datatype, &datatype_size);
-    len = datatype_size * count;
-
-    if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-	if (fd->fp_sys_posn != offset) {
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-	    lseek(fd->fd_sys, offset, SEEK_SET);
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-        }
-	ADIOI_WRITE_LOCK(fd, offset, SEEK_SET, len);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
-#endif
-	err = write(fd->fd_sys, buf, len);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
-#endif
-	ADIOI_UNLOCK(fd, offset, SEEK_SET, len);
-	fd->fp_sys_posn = offset + err;
-	/* individual file pointer not updated */        
-    }
-    else { /* write from curr. location of ind. file pointer */
-	offset = fd->fp_ind;
-	if (fd->fp_sys_posn != fd->fp_ind) {
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-	    lseek(fd->fd_sys, fd->fp_ind, SEEK_SET);
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-        }
-	ADIOI_WRITE_LOCK(fd, offset, SEEK_SET, len);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
-#endif
-	err = write(fd->fd_sys, buf, len);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
-#endif
-	ADIOI_UNLOCK(fd, offset, SEEK_SET, len);
-	fd->fp_ind += err;
-	fd->fp_sys_posn = fd->fp_ind;
-    }
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (err == -1) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io",
-					   "**io %s", strerror(errno));
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-#ifdef HAVE_STATUS_SET_BYTES
-    MPIR_Status_set_bytes(status, datatype, err);
-#endif
-
-    *error_code = MPI_SUCCESS;
-}
-
-
-
-
-#ifdef ADIOI_MPE_LOGGING
-#define ADIOI_BUFFERED_WRITE \
-{ \
-    if (req_off >= writebuf_off + writebuf_len) { \
-        MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL ); \
-	lseek(fd->fd_sys, writebuf_off, SEEK_SET); \
-        MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL ); \
-        MPE_Log_event( ADIOI_MPE_write_a, 0, NULL ); \
-	err = write(fd->fd_sys, writebuf, writebuf_len); \
-        MPE_Log_event( ADIOI_MPE_write_b, 0, NULL ); \
-        if (!(fd->atomicity)) ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
-        if (err == -1) err_flag = 1; \
-	writebuf_off = req_off; \
-        writebuf_len = (int) (ADIOI_MIN(max_bufsize,end_offset-writebuf_off+1));\
-	if (!(fd->atomicity)) ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
-        MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL ); \
-	lseek(fd->fd_sys, writebuf_off, SEEK_SET); \
-        MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL ); \
-        MPE_Log_event( ADIOI_MPE_read_a, 0, NULL ); \
-	err = read(fd->fd_sys, writebuf, writebuf_len); \
-        MPE_Log_event( ADIOI_MPE_read_b, 0, NULL ); \
-        if (err == -1) { \
-            *error_code = MPIO_Err_create_code(MPI_SUCCESS, \
-					       MPIR_ERR_RECOVERABLE, myname, \
-					       __LINE__, MPI_ERR_IO, \
-					       "**ioRMWrdwr", 0); \
-	    goto fn_exit; \
-        } \
-    } \
-    write_sz = (int) (ADIOI_MIN(req_len, writebuf_off + writebuf_len - req_off)); \
-    memcpy(writebuf+req_off-writebuf_off, (char *)buf +userbuf_off, write_sz);\
-    while (write_sz != req_len) { \
-        MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL ); \
-	lseek(fd->fd_sys, writebuf_off, SEEK_SET); \
-        MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL ); \
-        MPE_Log_event( ADIOI_MPE_write_a, 0, NULL ); \
-	err = write(fd->fd_sys, writebuf, writebuf_len); \
-        MPE_Log_event( ADIOI_MPE_write_b, 0, NULL ); \
-        if (!(fd->atomicity)) ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
-        if (err == -1) err_flag = 1; \
-        req_len -= write_sz; \
-        userbuf_off += write_sz; \
-        writebuf_off += writebuf_len; \
-        writebuf_len = (int) (ADIOI_MIN(max_bufsize,end_offset-writebuf_off+1));\
-	if (!(fd->atomicity)) ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
-        MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL ); \
-	lseek(fd->fd_sys, writebuf_off, SEEK_SET); \
-        MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL ); \
-        MPE_Log_event( ADIOI_MPE_read_a, 0, NULL ); \
-	err = read(fd->fd_sys, writebuf, writebuf_len); \
-        MPE_Log_event( ADIOI_MPE_read_b, 0, NULL ); \
-        if (err == -1) { \
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, \
-					       MPIR_ERR_RECOVERABLE, myname, \
-					       __LINE__, MPI_ERR_IO, \
-					       "**ioRMWrdwr", 0); \
-	    goto fn_exit; \
-        } \
-        write_sz = ADIOI_MIN(req_len, writebuf_len); \
-        memcpy(writebuf, (char *)buf + userbuf_off, write_sz);\
-    } \
-}
-#else
-#define ADIOI_BUFFERED_WRITE \
-{ \
-    if (req_off >= writebuf_off + writebuf_len) { \
-	lseek(fd->fd_sys, writebuf_off, SEEK_SET); \
-	err = write(fd->fd_sys, writebuf, writebuf_len); \
-        if (!(fd->atomicity)) ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
-        if (err == -1) err_flag = 1; \
-	writebuf_off = req_off; \
-        writebuf_len = (int) (ADIOI_MIN(max_bufsize,end_offset-writebuf_off+1));\
-	if (!(fd->atomicity)) ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
-	lseek(fd->fd_sys, writebuf_off, SEEK_SET); \
-	err = read(fd->fd_sys, writebuf, writebuf_len); \
-        if (err == -1) { \
-            *error_code = MPIO_Err_create_code(MPI_SUCCESS, \
-					       MPIR_ERR_RECOVERABLE, myname, \
-					       __LINE__, MPI_ERR_IO, \
-					       "**ioRMWrdwr", 0); \
-	    goto fn_exit; \
-        } \
-    } \
-    write_sz = (int) (ADIOI_MIN(req_len, writebuf_off + writebuf_len - req_off)); \
-    memcpy(writebuf+req_off-writebuf_off, (char *)buf +userbuf_off, write_sz);\
-    while (write_sz != req_len) { \
-	lseek(fd->fd_sys, writebuf_off, SEEK_SET); \
-	err = write(fd->fd_sys, writebuf, writebuf_len); \
-        if (!(fd->atomicity)) ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
-        if (err == -1) err_flag = 1; \
-        req_len -= write_sz; \
-        userbuf_off += write_sz; \
-        writebuf_off += writebuf_len; \
-        writebuf_len = (int) (ADIOI_MIN(max_bufsize,end_offset-writebuf_off+1));\
-	if (!(fd->atomicity)) ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
-	lseek(fd->fd_sys, writebuf_off, SEEK_SET); \
-	err = read(fd->fd_sys, writebuf, writebuf_len); \
-        if (err == -1) { \
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, \
-					       MPIR_ERR_RECOVERABLE, myname, \
-					       __LINE__, MPI_ERR_IO, \
-					       "**ioRMWrdwr", 0); \
-	    goto fn_exit; \
-        } \
-        write_sz = ADIOI_MIN(req_len, writebuf_len); \
-        memcpy(writebuf, (char *)buf + userbuf_off, write_sz);\
-    } \
-}
-#endif
-
-/* this macro is used when filetype is contig and buftype is not contig.
-   it does not do a read-modify-write and does not lock*/
-#ifdef ADIOI_MPE_LOGGING
-#define ADIOI_BUFFERED_WRITE_WITHOUT_READ \
-{ \
-    if (req_off >= writebuf_off + writebuf_len) { \
-        MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL ); \
-	lseek(fd->fd_sys, writebuf_off, SEEK_SET); \
-        MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL ); \
-	if (!(fd->atomicity)) ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
-        MPE_Log_event( ADIOI_MPE_write_a, 0, NULL ); \
-	err = write(fd->fd_sys, writebuf, writebuf_len); \
-        MPE_Log_event( ADIOI_MPE_write_b, 0, NULL ); \
-        if (!(fd->atomicity)) ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
-        if (err == -1) err_flag = 1; \
-	writebuf_off = req_off; \
-        writebuf_len = (int) (ADIOI_MIN(max_bufsize,end_offset-writebuf_off+1));\
-    } \
-    write_sz = (int) (ADIOI_MIN(req_len, writebuf_off + writebuf_len - req_off)); \
-    memcpy(writebuf+req_off-writebuf_off, (char *)buf +userbuf_off, write_sz);\
-    while (write_sz != req_len) { \
-        MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL ); \
-	lseek(fd->fd_sys, writebuf_off, SEEK_SET); \
-        MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL ); \
-	if (!(fd->atomicity)) ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
-        MPE_Log_event( ADIOI_MPE_write_a, 0, NULL ); \
-	err = write(fd->fd_sys, writebuf, writebuf_len); \
-        MPE_Log_event( ADIOI_MPE_write_b, 0, NULL ); \
-        if (!(fd->atomicity)) ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
-        if (err == -1) err_flag = 1; \
-        req_len -= write_sz; \
-        userbuf_off += write_sz; \
-        writebuf_off += writebuf_len; \
-        writebuf_len = (int) (ADIOI_MIN(max_bufsize,end_offset-writebuf_off+1));\
-        write_sz = ADIOI_MIN(req_len, writebuf_len); \
-        memcpy(writebuf, (char *)buf + userbuf_off, write_sz);\
-    } \
-}
-#else
-#define ADIOI_BUFFERED_WRITE_WITHOUT_READ \
-{ \
-    if (req_off >= writebuf_off + writebuf_len) { \
-	lseek(fd->fd_sys, writebuf_off, SEEK_SET); \
-	if (!(fd->atomicity)) ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
-	err = write(fd->fd_sys, writebuf, writebuf_len); \
-        if (!(fd->atomicity)) ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
-        if (err == -1) err_flag = 1; \
-	writebuf_off = req_off; \
-        writebuf_len = (int) (ADIOI_MIN(max_bufsize,end_offset-writebuf_off+1));\
-    } \
-    write_sz = (int) (ADIOI_MIN(req_len, writebuf_off + writebuf_len - req_off)); \
-    memcpy(writebuf+req_off-writebuf_off, (char *)buf +userbuf_off, write_sz);\
-    while (write_sz != req_len) { \
-	lseek(fd->fd_sys, writebuf_off, SEEK_SET); \
-	if (!(fd->atomicity)) ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
-	err = write(fd->fd_sys, writebuf, writebuf_len); \
-        if (!(fd->atomicity)) ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
-        if (err == -1) err_flag = 1; \
-        req_len -= write_sz; \
-        userbuf_off += write_sz; \
-        writebuf_off += writebuf_len; \
-        writebuf_len = (int) (ADIOI_MIN(max_bufsize,end_offset-writebuf_off+1));\
-        write_sz = ADIOI_MIN(req_len, writebuf_len); \
-        memcpy(writebuf, (char *)buf + userbuf_off, write_sz);\
-    } \
-}
-#endif
-
-
-void ADIOI_NFS_WriteStrided(ADIO_File fd, const void *buf, int count,
-                       MPI_Datatype datatype, int file_ptr_type,
-                       ADIO_Offset offset, ADIO_Status *status, int
-                       *error_code)
-{
-/* offset is in units of etype relative to the filetype. */
-
-    ADIOI_Flatlist_node *flat_buf, *flat_file;
-    int i, j, k, err=-1, bwr_size, st_index=0;
-    int num, size, sum, n_etypes_in_filetype, size_in_filetype;
-    MPI_Count bufsize;
-    int n_filetypes, etype_in_filetype;
-    ADIO_Offset abs_off_in_filetype=0;
-    int req_len;
-    MPI_Count filetype_size, etype_size, buftype_size;
-    MPI_Aint filetype_extent, buftype_extent, lb; 
-    int buf_count, buftype_is_contig, filetype_is_contig;
-    ADIO_Offset userbuf_off;
-    ADIO_Offset off, req_off, disp, end_offset=0, writebuf_off, start_off;
-    char *writebuf=NULL, *value;
-    int st_n_filetypes, writebuf_len, write_sz;
-    ADIO_Offset fwr_size = 0, new_fwr_size, st_fwr_size;
-    int new_bwr_size, err_flag=0, info_flag, max_bufsize;
-    static char myname[] = "ADIOI_NFS_WRITESTRIDED";
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-
-    MPI_Type_size_x(fd->filetype, &filetype_size);
-    if ( ! filetype_size ) {
-#ifdef HAVE_STATUS_SET_BYTES
-	MPIR_Status_set_bytes(status, datatype, 0);
-#endif
-	*error_code = MPI_SUCCESS; 
-	return;
-    }
-
-    MPI_Type_get_extent(fd->filetype, &lb, &filetype_extent);
-    MPI_Type_size_x(datatype, &buftype_size);
-    MPI_Type_get_extent(datatype, &lb, &buftype_extent);
-    etype_size = fd->etype_size;
-
-    bufsize = buftype_size * count;
-
-/* get max_bufsize from the info object. */
-
-    value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
-    ADIOI_Info_get(fd->info, "ind_wr_buffer_size", MPI_MAX_INFO_VAL, value, 
-                 &info_flag);
-    max_bufsize = atoi(value);
-    ADIOI_Free(value);
-
-    if (!buftype_is_contig && filetype_is_contig) {
-
-/* noncontiguous in memory, contiguous in file. */
-
-	flat_buf = ADIOI_Flatten_and_find(datatype);
-
-        off = (file_ptr_type == ADIO_INDIVIDUAL) ? fd->fp_ind : 
-                 fd->disp + etype_size * offset;
-
-        start_off = off;
-	end_offset = off + bufsize - 1;
-        writebuf_off = off;
-        writebuf = (char *) ADIOI_Malloc(max_bufsize);
-        writebuf_len = (int) (ADIOI_MIN(max_bufsize,end_offset-writebuf_off+1));
-
-/* if atomicity is true, lock the region to be accessed */
-        if (fd->atomicity) 
-            ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
-
-        for (j=0; j<count; j++) 
-            for (i=0; i<flat_buf->count; i++) {
-                userbuf_off = j*buftype_extent + flat_buf->indices[i];
-		req_off = off;
-		req_len = flat_buf->blocklens[i];
-		ADIOI_BUFFERED_WRITE_WITHOUT_READ
-                off += flat_buf->blocklens[i];
-            }
-
-        /* write the buffer out finally */
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-	lseek(fd->fd_sys, writebuf_off, SEEK_SET); 
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-	if (!(fd->atomicity)) ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
-#endif
-	err = write(fd->fd_sys, writebuf, writebuf_len); 
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
-#endif
-        if (!(fd->atomicity)) ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len);
-        if (err == -1) err_flag = 1; 
-
-        if (fd->atomicity) 
-            ADIOI_UNLOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
-
-        if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
-	if (err_flag) {
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE, myname,
-					       __LINE__, MPI_ERR_IO, "**io",
-					       "**io %s", strerror(errno));
-	}
-	else *error_code = MPI_SUCCESS;
-    }
-
-    else {  /* noncontiguous in file */
-
-/* filetype already flattened in ADIO_Open */
-	flat_file = ADIOI_Flatlist;
-	while (flat_file->type != fd->filetype) flat_file = flat_file->next;
-	disp = fd->disp;
-
-	if (file_ptr_type == ADIO_INDIVIDUAL) {
-       /* Wei-keng reworked type processing to be a bit more efficient */
-            offset       = fd->fp_ind - disp;
-            n_filetypes  = (offset - flat_file->indices[0]) / filetype_extent;
-            offset      -= (ADIO_Offset)n_filetypes * filetype_extent;
-            /* now offset is local to this extent */
-
-            /* find the block where offset is located, skip blocklens[i]==0 */
-            for (i=0; i<flat_file->count; i++) {
-                ADIO_Offset dist;
-                if (flat_file->blocklens[i] == 0) continue;
-                dist = flat_file->indices[i] + flat_file->blocklens[i] - offset;
-                /* fwr_size is from offset to the end of block i */
-                if (dist == 0) {
-                    i++;
-                    offset   = flat_file->indices[i];
-                    fwr_size = flat_file->blocklens[i];
-                    break;
-                }
-                if (dist > 0) {
-                    fwr_size = dist;
-                    break;
-                }
-            }
-            st_index = i;  /* starting index in flat_file->indices[] */
-            offset += disp + (ADIO_Offset)n_filetypes*filetype_extent;
-        }
-	else {
-	    n_etypes_in_filetype = filetype_size/etype_size;
-	    n_filetypes = (int) (offset / n_etypes_in_filetype);
-	    etype_in_filetype = (int) (offset % n_etypes_in_filetype);
-	    size_in_filetype = etype_in_filetype * etype_size;
- 
-	    sum = 0;
-	    for (i=0; i<flat_file->count; i++) {
-		sum += flat_file->blocklens[i];
-		if (sum > size_in_filetype) {
-		    st_index = i;
-		    fwr_size = sum - size_in_filetype;
-		    abs_off_in_filetype = flat_file->indices[i] +
-			size_in_filetype - (sum - flat_file->blocklens[i]);
-		    break;
-		}
-	    }
-
-	    /* abs. offset in bytes in the file */
-	    offset = disp + (ADIO_Offset) n_filetypes*filetype_extent + 
-		abs_off_in_filetype;
-	}
-
-        start_off = offset;
-       /* Wei-keng Liao:write request is within single flat_file contig block*/
-       /* this could happen, for example, with subarray types that are
-        * actually fairly contiguous */
-        if (buftype_is_contig && bufsize <= fwr_size) {
-            ADIO_WriteContig(fd, buf, bufsize, MPI_BYTE, ADIO_EXPLICIT_OFFSET,
-                             offset, status, error_code);
-
-           if (file_ptr_type == ADIO_INDIVIDUAL) {
-                /* update MPI-IO file pointer to point to the first byte 
-                * that can be accessed in the fileview. */
-                fd->fp_ind = offset + bufsize;
-                if (bufsize == fwr_size) {
-                    do {
-                        st_index++;
-                        if (st_index == flat_file->count) {
-                            st_index = 0;
-                            n_filetypes++;
-                        }
-                    } while (flat_file->blocklens[st_index] == 0);
-                    fd->fp_ind = disp + flat_file->indices[st_index]
-                               + (ADIO_Offset)n_filetypes*filetype_extent;
-                }
-            }
-           fd->fp_sys_posn = -1;   /* set it to null. */ 
-#ifdef HAVE_STATUS_SET_BYTES
-           MPIR_Status_set_bytes(status, datatype, bufsize);
-#endif 
-            return;
-        }
-
-       /* Calculate end_offset, the last byte-offset that will be accessed.
-         e.g., if start_offset=0 and 100 bytes to be write, end_offset=99*/
-
-	st_fwr_size = fwr_size;
-	st_n_filetypes = n_filetypes;
-	i = 0;
-	j = st_index;
-	off = offset;
-	fwr_size = ADIOI_MIN(st_fwr_size, bufsize);
-	while (i < bufsize) {
-	    i += fwr_size;
-	    end_offset = off + fwr_size - 1;
-            j = (j+1) % flat_file->count;
-            n_filetypes += (j == 0) ? 1 : 0;
-            while (flat_file->blocklens[j]==0) {
-                j = (j+1) % flat_file->count;
-                n_filetypes += (j == 0) ? 1 : 0;
-	    }
-
-	    off = disp + flat_file->indices[j] + 
-		    (ADIO_Offset) n_filetypes*filetype_extent;
-	    fwr_size = ADIOI_MIN(flat_file->blocklens[j], bufsize-i);
-	}
-
-/* if atomicity is true, lock the region to be accessed */
-        if (fd->atomicity) 
-            ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
-
-        /* initial read for the read-modify-write */
-        writebuf_off = offset;
-        writebuf = (char *) ADIOI_Malloc(max_bufsize);
-        writebuf_len = (int)(ADIOI_MIN(max_bufsize,end_offset-writebuf_off+1));
-	if (!(fd->atomicity)) ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-	lseek(fd->fd_sys, writebuf_off, SEEK_SET); 
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
-#endif
-	err = read(fd->fd_sys, writebuf, writebuf_len); 
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
-#endif
-        if (err == -1) {
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE,
-					       myname, __LINE__,
-					       MPI_ERR_IO,
-					       "ADIOI_NFS_WriteStrided: ROMIO tries to optimize this access by doing a read-modify-write, but is unable to read the file. Please give the file read permission and open it with MPI_MODE_RDWR.", 0);
-	    goto fn_exit;
-        }
-
-	if (buftype_is_contig && !filetype_is_contig) {
-
-/* contiguous in memory, noncontiguous in file. should be the most
-   common case. */
-
-	    i = 0;
-	    j = st_index;
-	    off = offset;
-	    n_filetypes = st_n_filetypes;
-	    fwr_size = ADIOI_MIN(st_fwr_size, bufsize);
-	    while (i < bufsize) {
-                if (fwr_size) { 
-                    /* TYPE_UB and TYPE_LB can result in 
-                       fwr_size = 0. save system call in such cases */ 
-		    /* lseek(fd->fd_sys, off, SEEK_SET);
-		    err = write(fd->fd_sys, ((char *) buf) + i, fwr_size);*/
-
-		    req_off = off;
-		    req_len = fwr_size;
-		    userbuf_off = i;
-		    ADIOI_BUFFERED_WRITE
-		}
-		i += fwr_size;
-
-                if (off + fwr_size < disp + flat_file->indices[j] +
-                   flat_file->blocklens[j] + (ADIO_Offset) n_filetypes*filetype_extent)
-                       off += fwr_size;
-                /* did not reach end of contiguous block in filetype.
-                   no more I/O needed. off is incremented by fwr_size. */
-                else {
-                    j = (j+1) % flat_file->count;
-                    n_filetypes += (j == 0) ? 1 : 0;
-                    while (flat_file->blocklens[j]==0) {
-                        j = (j+1) % flat_file->count;
-                        n_filetypes += (j == 0) ? 1 : 0;
-                    }
-		    off = disp + flat_file->indices[j] + 
-                                      (ADIO_Offset) n_filetypes*filetype_extent;
-		    fwr_size = ADIOI_MIN(flat_file->blocklens[j], bufsize-i);
-		}
-	    }
-	}
-	else {
-/* noncontiguous in memory as well as in file */
-
-	    ADIO_Offset i;
-	    flat_buf = ADIOI_Flatten_and_find(datatype);
-
-	    k = num = buf_count = 0;
-	    i = flat_buf->indices[0];
-	    j = st_index;
-	    off = offset;
-	    n_filetypes = st_n_filetypes;
-	    fwr_size = st_fwr_size;
-	    bwr_size = flat_buf->blocklens[0];
-
-	    while (num < bufsize) {
-		size = ADIOI_MIN(fwr_size, bwr_size);
-		if (size) {
-		    /* lseek(fd->fd_sys, off, SEEK_SET);
-		    err = write(fd->fd_sys, ((char *) buf) + i, size); */
-
-		    req_off = off;
-		    req_len = size;
-		    userbuf_off = i;
-		    ADIOI_BUFFERED_WRITE
-		}
-
-		new_fwr_size = fwr_size;
-		new_bwr_size = bwr_size;
-
-		if (size == fwr_size) {
-/* reached end of contiguous block in file */
-                   j = (j+1) % flat_file->count;
-                   n_filetypes += (j == 0) ? 1 : 0;
-                   while (flat_file->blocklens[j]==0) {
-                       j = (j+1) % flat_file->count;
-                       n_filetypes += (j == 0) ? 1 : 0;
-		    }
-
-		    off = disp + flat_file->indices[j] + 
-                                  (ADIO_Offset) n_filetypes*filetype_extent;
-
-		    new_fwr_size = flat_file->blocklens[j];
-		    if (size != bwr_size) {
-			i += size;
-			new_bwr_size -= size;
-		    }
-		}
-
-		if (size == bwr_size) {
-/* reached end of contiguous block in memory */
-
-		    k = (k + 1)%flat_buf->count;
-		    buf_count++;
-		    i = buftype_extent*(buf_count/flat_buf->count) +
-			flat_buf->indices[k];
-		    new_bwr_size = flat_buf->blocklens[k];
-		    if (size != fwr_size) {
-			off += size;
-			new_fwr_size -= size;
-		    }
-		}
-		num += size;
-		fwr_size = new_fwr_size;
-                bwr_size = new_bwr_size;
-	    }
-	}
-
-        /* write the buffer out finally */	
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-	lseek(fd->fd_sys, writebuf_off, SEEK_SET); 
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-	if (!(fd->atomicity)) ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
-#endif
-	err = write(fd->fd_sys, writebuf, writebuf_len); 
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
-#endif
-
-        if (!(fd->atomicity))
-	    ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len);
-	else ADIOI_UNLOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
-
-        if (err == -1) err_flag = 1; 
-
-	if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
-	if (err_flag) {
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE, myname,
-					       __LINE__, MPI_ERR_IO, "**io",
-					       "**io %s", strerror(errno));
-	}
-	else *error_code = MPI_SUCCESS;
-    }
-
-    fd->fp_sys_posn = -1;   /* set it to null. */
-
-#ifdef HAVE_STATUS_SET_BYTES
-    MPIR_Status_set_bytes(status, datatype, bufsize);
-/* This is a temporary way of filling in status. The right way is to 
-   keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
-#endif
-
-    if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
-fn_exit:
-    if (writebuf != NULL) ADIOI_Free(writebuf);
-
-    return;
-}
diff --git a/3rd-party/romio321/adio/ad_ntfs/ad_ntfs.c b/3rd-party/romio321/adio/ad_ntfs/ad_ntfs.c
deleted file mode 100644
index 97882741217..00000000000
--- a/3rd-party/romio321/adio/ad_ntfs/ad_ntfs.c
+++ /dev/null
@@ -1,40 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_ntfs.h"
-
-/* adioi.h has the ADIOI_Fns_struct define */
-#include "adioi.h"
-
-struct ADIOI_Fns_struct ADIO_NTFS_operations = {
-    ADIOI_NTFS_Open, /* Open */
-    ADIOI_FAILSAFE_OpenColl, /* OpenColl */
-    ADIOI_NTFS_ReadContig, /* ReadContig */
-    ADIOI_NTFS_WriteContig, /* WriteContig */
-    ADIOI_GEN_ReadStridedColl, /* ReadStridedColl */
-    ADIOI_GEN_WriteStridedColl, /* WriteStridedColl */
-    ADIOI_GEN_SeekIndividual, /* SeekIndividual */
-    ADIOI_NTFS_Fcntl, /* Fcntl */
-    ADIOI_GEN_SetInfo, /* SetInfo */
-    ADIOI_GEN_ReadStrided, /* ReadStrided */
-    ADIOI_GEN_WriteStrided, /* WriteStrided */
-    ADIOI_NTFS_Close, /* Close */
-    ADIOI_NTFS_IreadContig, /* IreadContig */
-    ADIOI_NTFS_IwriteContig, /* IwriteContig */
-    ADIOI_NTFS_ReadDone, /* ReadDone */
-    ADIOI_NTFS_WriteDone, /* WriteDone */
-    ADIOI_NTFS_ReadComplete, /* ReadComplete */
-    ADIOI_NTFS_WriteComplete, /* WriteComplete */
-    ADIOI_FAKE_IreadStrided, /* IreadStrided */
-    ADIOI_FAKE_IwriteStrided, /* IwriteStrided */
-    ADIOI_NTFS_Flush, /* Flush */
-    ADIOI_NTFS_Resize, /* Resize */
-    ADIOI_GEN_Delete, /* Delete */
-    ADIOI_NTFS_Feature, /* Features */
-    ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
-    ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
-};
diff --git a/3rd-party/romio321/adio/ad_ntfs/ad_ntfs.h b/3rd-party/romio321/adio/ad_ntfs/ad_ntfs.h
deleted file mode 100644
index aed6168bd8b..00000000000
--- a/3rd-party/romio321/adio/ad_ntfs/ad_ntfs.h
+++ /dev/null
@@ -1,68 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#ifndef AD_NTFS_INCLUDE
-#define AD_NTFS_INCLUDE
-
-#include <sys/types.h>
-#include <fcntl.h>
-#include "adio.h"
-
-#ifdef HAVE_INT64
-#define DWORDLOW(x)        ( (DWORD) ( x & (__int64) 0xFFFFFFFF ) )
-#define DWORDHIGH(x)       ( (DWORD) ( (x >> 32) & (__int64) 0xFFFFFFFF ) )
-#define DWORDTOINT64(x,y)  ( (__int64) ( ( (__int64 x) << 32 ) + (__int64) y ) )
-#else
-#define DWORDLOW(x)         x
-#define DWORDHIGH(x)        0
-#define DWORDTOINT64(x,y)   x
-#endif
-
-int ADIOI_NTFS_aio(ADIO_File fd, void *buf, int len, ADIO_Offset offset,
-		  int wr, void *handle);
-
-void ADIOI_NTFS_Open(ADIO_File fd, int *error_code);
-void ADIOI_NTFS_Close(ADIO_File fd, int *error_code);
-void ADIOI_NTFS_ReadContig(ADIO_File fd, void *buf, int count, 
-                      MPI_Datatype datatype, int file_ptr_type,
-                     ADIO_Offset offset, ADIO_Status *status, int
-		     *error_code);
-void ADIOI_NTFS_WriteContig(ADIO_File fd, void *buf, int count, 
-                      MPI_Datatype datatype, int file_ptr_type,
-                      ADIO_Offset offset, ADIO_Status *status, int
-		      *error_code);   
-void ADIOI_NTFS_IwriteContig(ADIO_File fd, void *buf, int count, 
-                      MPI_Datatype datatype, int file_ptr_type,
-                      ADIO_Offset offset, ADIO_Request *request, int
-		      *error_code);   
-void ADIOI_NTFS_IreadContig(ADIO_File fd, void *buf, int count, 
-                      MPI_Datatype datatype, int file_ptr_type,
-                      ADIO_Offset offset, ADIO_Request *request, int
-		      *error_code);   
-int ADIOI_NTFS_ReadDone(ADIO_Request *request, ADIO_Status *status, int
-		       *error_code);
-int ADIOI_NTFS_WriteDone(ADIO_Request *request, ADIO_Status *status, int
-		       *error_code);
-void ADIOI_NTFS_ReadComplete(ADIO_Request *request, ADIO_Status *status, int
-		       *error_code); 
-void ADIOI_NTFS_WriteComplete(ADIO_Request *request, ADIO_Status *status,
-			int *error_code); 
-void ADIOI_NTFS_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct, int
-		*error_code); 
-void ADIOI_NTFS_IwriteStrided(ADIO_File fd, void *buf, int count,
-		       MPI_Datatype datatype, int file_ptr_type,
-		       ADIO_Offset offset, ADIO_Request *request, int
-		       *error_code);
-void ADIOI_NTFS_Flush(ADIO_File fd, int *error_code);
-void ADIOI_NTFS_Resize(ADIO_File fd, ADIO_Offset size, int *error_code);
-
-#define FORMAT_MESSAGE_MIN_SIZE 100
-#define ADIOI_NTFS_ERR_MSG_MAX    FORMAT_MESSAGE_MIN_SIZE
-void ADIOI_NTFS_Strerror(int error, char *errMsg, int errMsgLen);
-
-#endif
diff --git a/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_close.c b/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_close.c
deleted file mode 100644
index 95022dca77b..00000000000
--- a/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_close.c
+++ /dev/null
@@ -1,30 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_ntfs.h"
-
-void ADIOI_NTFS_Close(ADIO_File fd, int *error_code)
-{
-    int err;
-    static char myname[] = "ADIOI_NTFS_Close";
-
-    err = CloseHandle(fd->fd_sys);
-    /* --BEGIN ERROR HANDLING-- */
-    if (err == FALSE)
-    {
-    char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-	err = GetLastError();
-    ADIOI_NTFS_Strerror(err, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io",
-					   "**io %s", errMsg);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-    *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_done.c b/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_done.c
deleted file mode 100644
index 9f4967b48ba..00000000000
--- a/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_done.c
+++ /dev/null
@@ -1,20 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_ntfs.h"
-
-int ADIOI_NTFS_ReadDone(ADIO_Request *request, ADIO_Status *status,
-			int *error_code)
-{
-    return 0;
-}
-
-
-int ADIOI_NTFS_WriteDone(ADIO_Request *request, ADIO_Status *status,
-			 int *error_code)
-{
-    return 0;
-}
diff --git a/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_fcntl.c b/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_fcntl.c
deleted file mode 100644
index 4c7d66cd074..00000000000
--- a/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_fcntl.c
+++ /dev/null
@@ -1,76 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_ntfs.h"
-#include "adio_extern.h"
-
-void ADIOI_NTFS_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct, int *error_code)
-{
-    DWORD err;
-    LONG dwTemp;
-    static char myname[] = "ADIOI_NTFS_FCNTL";
-
-    switch(flag)
-    {
-    case ADIO_FCNTL_GET_FSIZE:
-	fcntl_struct->fsize = SetFilePointer(fd->fd_sys, 0, 0, FILE_END);
-	if (fd->fp_sys_posn != -1) 
-	{
-	    dwTemp = DWORDHIGH(fd->fp_sys_posn);
-	    if (SetFilePointer(fd->fd_sys, DWORDLOW(fd->fp_sys_posn), &dwTemp, FILE_BEGIN) == INVALID_SET_FILE_POINTER)
-	    {
-        char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-		err = GetLastError();
-        ADIOI_NTFS_Strerror(err, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-		if (err != NO_ERROR)
-		{
-		    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-			myname, __LINE__, MPI_ERR_IO,
-			"**io", "**io %s", errMsg);
-		    return;
-		}
-	    }
-	}
-	/* --BEGIN ERROR HANDLING-- */
-	if (fcntl_struct->fsize == INVALID_SET_FILE_POINTER)
-	{
-        char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-	    dwTemp = GetLastError();
-        ADIOI_NTFS_Strerror(dwTemp, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE, myname,
-					       __LINE__, MPI_ERR_IO, "**io",
-					       "**io %s", errMsg);
-	    return;
-	}
-	/* --END ERROR HANDLING-- */
-	*error_code = MPI_SUCCESS;
-	break;
-
-    case ADIO_FCNTL_SET_DISKSPACE:
-	ADIOI_GEN_Prealloc(fd, fcntl_struct->diskspace, error_code);
-	break;
-
-    case ADIO_FCNTL_SET_ATOMICITY:
-	fd->atomicity = (fcntl_struct->atomicity == 0) ? 0 : 1;
-	*error_code = MPI_SUCCESS;
-	/*
-	fd->atomicity = 0;
-	*error_code = MPI_ERR_UNSUPPORTED_OPERATION;
-	*/
-	break;
-
-    default:
-	/* --BEGIN ERROR HANDLING-- */
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   MPI_ERR_ARG,
-					   "**flag", "**flag %d", flag);
-	return;
-	/* --END ERROR HANDLING-- */
-    }
-}
diff --git a/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_feature.c b/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_feature.c
deleted file mode 100644
index 335b5aed57e..00000000000
--- a/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_feature.c
+++ /dev/null
@@ -1,26 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *
- *  (C) 2008 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include "adio.h"
-
-int ADIOI_NTFS_Feature(ADIO_File fd, int flag)
-{
-	switch(flag) {
-	    /* supported features */
-		case ADIO_LOCKS:
-		case ADIO_SHARED_FP:
-		case ADIO_ATOMIC_MODE:
-		case ADIO_DATA_SIEVING_WRITES:
-			return 1;
-			break;
-	    /* unsupported features */
-		case ADIO_SCALABLE_OPEN:
-		case ADIO_UNLINK_AFTER_CLOSE:
-		default:
-			return 0;
-			break;
-	}
-}
diff --git a/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_flush.c b/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_flush.c
deleted file mode 100644
index 7656d60fb66..00000000000
--- a/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_flush.c
+++ /dev/null
@@ -1,32 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_ntfs.h"
-
-void ADIOI_NTFS_Flush(ADIO_File fd, int *error_code)
-{
-    int err;
-    static char myname[] = "ADIOI_NTFS_Flush";
-
-    err = (fd->access_mode & ADIO_RDONLY) ? TRUE :
-	FlushFileBuffers(fd->fd_sys);
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (err == FALSE)
-    {
-    char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-	err = GetLastError();
-    ADIOI_NTFS_Strerror(err, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io",
-					   "**io %s", errMsg);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-    *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_iread.c b/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_iread.c
deleted file mode 100644
index 845401ad44a..00000000000
--- a/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_iread.c
+++ /dev/null
@@ -1,42 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_ntfs.h"
-
-void ADIOI_NTFS_IreadContig(ADIO_File fd, void *buf, int count, 
-                MPI_Datatype datatype, int file_ptr_type,
-                ADIO_Offset offset, ADIO_Request *request, int *error_code)  
-{
-    MPI_Count len, typesize;
-    int err;
-    static char myname[] = "ADIOI_NTFS_IreadContig";
-
-    MPI_Type_size_x(datatype, &typesize);
-    len = count * typesize;
-
-    if (file_ptr_type == ADIO_INDIVIDUAL)
-    {
-	offset = fd->fp_ind;
-    }
-    err = ADIOI_NTFS_aio(fd, buf, len, offset, 0, request);
-    if (file_ptr_type == ADIO_INDIVIDUAL)
-    {
-	fd->fp_ind += len;
-    }
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (err != MPI_SUCCESS)
-    {
-	*error_code = MPIO_Err_create_code(err, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io", 0);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-    *error_code = MPI_SUCCESS;
-
-    fd->fp_sys_posn = -1;   /* set it to null. */
-}
diff --git a/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_iwrite.c b/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_iwrite.c
deleted file mode 100644
index 40a567e564b..00000000000
--- a/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_iwrite.c
+++ /dev/null
@@ -1,303 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_ntfs.h"
-
-#include "../../mpi-io/mpioimpl.h"
-#include "../../mpi-io/mpioprof.h"
-#include "mpiu_greq.h"
-
-static MPIX_Grequest_class ADIOI_NTFS_greq_class = 0;
-
-/* Fills the input buffer, errMsg, with the error message 
-   corresponding to error code, error */
-void ADIOI_NTFS_Strerror(int error, char *errMsg, int errMsgLen)
-{
-    LPTSTR str;
-    int num_bytes;
-    num_bytes = FormatMessage(
-	FORMAT_MESSAGE_FROM_SYSTEM |
-	FORMAT_MESSAGE_ALLOCATE_BUFFER,
-	NULL,
-	error,
-	0,
-	&str,
-	FORMAT_MESSAGE_MIN_SIZE,
-    0);
-    if (num_bytes == 0)
-    {
-	strncpy(errMsg, "\0", errMsgLen);
-    }
-    else
-    {
-	strncpy(errMsg, str, errMsgLen);
-	LocalFree(str);
-    }
-}
-
-/* poll for completion of a single outstanding AIO request */
-int ADIOI_NTFS_aio_poll_fn(void *extra_state, MPI_Status *status)
-{
-    ADIOI_AIO_Request *aio_req;
-    int mpi_errno = MPI_SUCCESS;
-
-    /* FIXME: Validate the args -- has it already been done by the 
-       caller ? */
-
-    aio_req = (ADIOI_AIO_Request *)extra_state;
-    
-    /* XXX: test for AIO completion here */
-    if(!GetOverlappedResult( aio_req->fd, aio_req->lpOvl, 
-                            &(aio_req->nbytes), FALSE)){
-        if(GetLastError() == ERROR_IO_INCOMPLETE){
-        /* IO in progress */
-	    /* TODO: need to diddle with status somehow */
-        }else{
-        /* Error occured */
-        /* TODO: unsure how to handle this */    
-        }
-    }else{
-        mpi_errno = MPI_Grequest_complete(aio_req->req);
-	    if (mpi_errno != MPI_SUCCESS) {
-		    mpi_errno = MPIO_Err_create_code(MPI_SUCCESS,
-				    MPIR_ERR_RECOVERABLE,
-				    "ADIOI_NTFS_aio_poll_fn", __LINE__,
-				    MPI_ERR_IO, "**mpi_grequest_complete",
-				    0);
-	    }
-    }
-    return mpi_errno;
-}
-
-
-/* Wait for completion of one of the outstanding AIO requests */
-int ADIOI_NTFS_aio_wait_fn(int count, void **array_of_states,
-		double timeout, MPI_Status *status)
-{
-	int i, mpi_errno = MPI_SUCCESS;
-	ADIOI_AIO_Request **aio_reqlist;
-    LPHANDLE lpHandles;
-    DWORD retObject=0;
-
-    /* FIXME: Validate the args -- has it already been done by the 
-       caller ? */
-	aio_reqlist = (ADIOI_AIO_Request **)array_of_states;
-    lpHandles = (LPHANDLE) ADIOI_Calloc(count, sizeof(HANDLE));
-    if (lpHandles == NULL)
-    {
-	mpi_errno = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-	    "ADIOI_NTFS_aio_wait_fn", __LINE__, MPI_ERR_IO,
-	    "**nomem", "**nomem %s", "Event handles");
-	return mpi_errno;
-    }
-	/* XXX: set-up arrays of outstanding requests */
-    for(i=0; i<count; i++){
-        lpHandles[i] = (aio_reqlist[i])->lpOvl->hEvent;
-    }
-
-	/* XXX: wait for one request to complete */
-    /* FIXME: Is the timeout in seconds ? */
-    timeout = (timeout <= 0) ? INFINITE : (timeout * 1000);
-    
-    if((retObject = WaitForMultipleObjects(count, lpHandles,
-                    FALSE, timeout)) != WAIT_FAILED){
-        retObject = retObject - WAIT_OBJECT_0;
-        if(GetOverlappedResult( aio_reqlist[retObject]->fd, 
-                aio_reqlist[retObject]->lpOvl, &(aio_reqlist[retObject]->nbytes), 
-                FALSE)){
-        	/* XXX: mark completed requests as 'done'*/
-            mpi_errno = MPI_Grequest_complete(aio_reqlist[retObject]->req);
-    	    if (mpi_errno != MPI_SUCCESS) {
-	    	    mpi_errno = MPIO_Err_create_code(MPI_SUCCESS,
-				    MPIR_ERR_RECOVERABLE,
-				    "ADIOI_NTFS_aio_wait_fn", __LINE__,
-				    MPI_ERR_IO, "**mpi_grequest_complete",
-				    0);
-            }
-        }else{
-            if(GetLastError() == ERROR_IO_INCOMPLETE){
-            /* IO in progress */
-	        /* TODO: need to diddle with status somehow */
-            }else{
-            /* Error occured */
-            /* TODO: not sure how to handle this */    
-            }
-        }
-    }else{
-        /* TODO: How to handle error while waiting ? */
-    }
-    ADIOI_Free(lpHandles);
-	return mpi_errno;
-}
-
-int ADIOI_NTFS_aio_query_fn(void *extra_state, MPI_Status *status) 
-{
-	ADIOI_AIO_Request *aio_req;
-
-	aio_req = (ADIOI_AIO_Request *)extra_state;
-
-
-	MPI_Status_set_elements(status, MPI_BYTE, aio_req->nbytes); 
-
-	/* can never cancel so always true */ 
-	MPI_Status_set_cancelled(status, 0); 
-
-	/* choose not to return a value for this */ 
-	status->MPI_SOURCE = MPI_UNDEFINED; 
-	/* tag has no meaning for this generalized request */ 
-	status->MPI_TAG = MPI_UNDEFINED; 
-	/* this generalized request never fails */ 
-	return MPI_SUCCESS; 
-}
-
-
-int ADIOI_NTFS_aio_free_fn(void *extra_state)
-{
-	ADIOI_AIO_Request *aio_req;
-    /* FIXME: Validate the args -- has it already been done by the 
-       caller ? */
-	aio_req = (ADIOI_AIO_Request*)extra_state;
-    CloseHandle(aio_req->lpOvl->hEvent);
-    ADIOI_Free(aio_req->lpOvl);
-    ADIOI_Free(aio_req);
-	return MPI_SUCCESS;    
-}
-
-void ADIOI_NTFS_IwriteContig(ADIO_File fd, void *buf, int count, 
-			     MPI_Datatype datatype, int file_ptr_type,
-			     ADIO_Offset offset, ADIO_Request *request,
-			     int *error_code)  
-{
-    MPI_Count len, typesize;
-    int err;
-    static char myname[] = "ADIOI_NTFS_IwriteContig";
-
-    MPI_Type_size_x(datatype, &typesize);
-    len = count * typesize;
-
-    if (file_ptr_type == ADIO_INDIVIDUAL)
-    {
-	offset = fd->fp_ind;
-    }
-    err = ADIOI_NTFS_aio(fd, buf, len, offset, 1, request);
-    if (file_ptr_type == ADIO_INDIVIDUAL)
-    {
-	fd->fp_ind += len;
-    }
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (err != MPI_SUCCESS)
-    {
-	*error_code = MPIO_Err_create_code(err, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io", 0);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-    *error_code = MPI_SUCCESS;
-
-    fd->fp_sys_posn = -1;   /* set it to null. */
-}
-
-
-/* This function is for implementation convenience. It is not user-visible.
- * If wr==1 write, wr==0 read.
- *
- * Returns MPI_SUCCESS on success, mpi_errno on failure.
- */
-int ADIOI_NTFS_aio(ADIO_File fd, void *buf, int len, ADIO_Offset offset,
-		   int wr, MPI_Request *request)
-{
-    static char myname[] = "ADIOI_NTFS_aio";
-
-    ADIOI_AIO_Request *aio_req;
-    static DWORD dwNumWritten, dwNumRead;
-    BOOL ret_val = FALSE;
-    FDTYPE fd_sys;
-    int mpi_errno = MPI_SUCCESS;
-    DWORD err;
-
-    fd_sys = fd->fd_sys;
-
-    aio_req = (ADIOI_AIO_Request *)ADIOI_Calloc(sizeof(ADIOI_AIO_Request), 1);
-    if (aio_req == NULL)
-    {
-	mpi_errno = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-	    myname, __LINE__, MPI_ERR_IO,
-	    "**nomem", "**nomem %s", "AIO_REQ");
-	return mpi_errno;
-    }
-    aio_req->lpOvl = (LPOVERLAPPED ) ADIOI_Calloc(sizeof(OVERLAPPED), 1);
-    if (aio_req->lpOvl == NULL)
-    {
-	mpi_errno = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-	    myname, __LINE__, MPI_ERR_IO,
-	    "**nomem", "**nomem %s", "OVERLAPPED");
-    ADIOI_Free(aio_req);
-	return mpi_errno;
-    }
-    aio_req->lpOvl->hEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
-    if (aio_req->lpOvl->hEvent == NULL)
-    {
-    char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-	err = GetLastError();
-    ADIOI_NTFS_Strerror(err, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-	mpi_errno = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-	    myname, __LINE__, MPI_ERR_IO,
-	    "**io", "**io %s", errMsg);
-    ADIOI_Free(aio_req->lpOvl);
-    ADIOI_Free(aio_req);
-	return mpi_errno;
-    }
-    aio_req->lpOvl->Offset = DWORDLOW(offset);
-    aio_req->lpOvl->OffsetHigh = DWORDHIGH(offset);
-    aio_req->fd = fd_sys;
-    
-    /* XXX: initiate async I/O  */
-    if (wr)
-    {
-	ret_val = WriteFile(fd_sys, buf, len, &dwNumWritten, aio_req->lpOvl);
-    }
-    else
-    {
-	ret_val = ReadFile(fd_sys, buf, len, &dwNumRead, aio_req->lpOvl);
-    }
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (ret_val == FALSE) 
-    {
-	mpi_errno = GetLastError();
-	if (mpi_errno != ERROR_IO_PENDING)
-	{
-        char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-        ADIOI_NTFS_Strerror(mpi_errno, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-	    mpi_errno = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-		myname, __LINE__, MPI_ERR_IO,
-		"**io",
-		"**io %s", errMsg);
-	    return mpi_errno;
-	}
-	mpi_errno = MPI_SUCCESS;
-    }
-    /* --END ERROR HANDLING-- */
-
-    /* XXX: set up generalized request class and request */
-    if (ADIOI_NTFS_greq_class == 0) {
-	    mpi_errno = MPIX_Grequest_class_create(ADIOI_NTFS_aio_query_fn,
-			    ADIOI_NTFS_aio_free_fn, MPIU_Greq_cancel_fn,
-			    ADIOI_NTFS_aio_poll_fn, ADIOI_NTFS_aio_wait_fn,
-			    &ADIOI_NTFS_greq_class);
-        if(mpi_errno != MPI_SUCCESS){
-        /* FIXME: Pass appropriate error code to user */
-        }
-    }
-    mpi_errno = MPIX_Grequest_class_allocate(ADIOI_NTFS_greq_class, aio_req, request);
-    if(mpi_errno != MPI_SUCCESS){
-    /* FIXME: Pass appropriate error code to user */
-    }
-    memcpy(&(aio_req->req), request, sizeof(MPI_Request));
-    return mpi_errno;
-}
diff --git a/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_open.c b/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_open.c
deleted file mode 100644
index 3a49418dbaa..00000000000
--- a/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_open.c
+++ /dev/null
@@ -1,101 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_ntfs.h"
-
-void ADIOI_NTFS_Open(ADIO_File fd, int *error_code)
-{
-    int err;
-    int cmode, amode, attrib;
-    static char myname[] = "ADIOI_NTFS_Open";
-
-    amode = 0;
-    cmode = OPEN_EXISTING;
-#ifdef USE_WIN_THREADED_IO
-    attrib = FILE_FLAG_OVERLAPPED;
-#else
-    attrib = FILE_ATTRIBUTE_NORMAL;
-#endif
-
-    if (fd->access_mode & ADIO_CREATE)
-    {
-	cmode = OPEN_ALWAYS;
-    }
-    if (fd->access_mode & ADIO_EXCL)
-    {
-	cmode = CREATE_NEW;
-    }
-
-    if (fd->access_mode & ADIO_RDONLY)
-    {
-	amode = GENERIC_READ;
-    }
-    if (fd->access_mode & ADIO_WRONLY)
-    {
-	amode = GENERIC_WRITE;
-    }
-    if (fd->access_mode & ADIO_RDWR)
-    {
-	amode = GENERIC_READ | GENERIC_WRITE;
-    }
-
-    if (fd->access_mode & ADIO_DELETE_ON_CLOSE)
-    {
-	attrib = attrib | FILE_FLAG_DELETE_ON_CLOSE;
-    }
-    if (fd->access_mode & ADIO_SEQUENTIAL)
-    {
-	attrib = attrib | FILE_FLAG_SEQUENTIAL_SCAN;
-    }
-    else
-    {
-	attrib = attrib | FILE_FLAG_RANDOM_ACCESS;
-    }
-
-    fd->fd_sys = CreateFile(fd->filename, 
-	amode,
-	FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, 
-	NULL, 
-	cmode, 
-	attrib, 
-	NULL);
-    fd->fd_direct = -1;
-
-    if ((fd->fd_sys != INVALID_HANDLE_VALUE) && (fd->access_mode & ADIO_APPEND))
-    {
-	fd->fp_ind = fd->fp_sys_posn = SetFilePointer(fd->fd_sys, 0, NULL, FILE_END);
-	if (fd->fp_ind == INVALID_SET_FILE_POINTER)
-	{
-        char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-	    err = GetLastError();
-        ADIOI_NTFS_Strerror(err, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-	    if (err != NO_ERROR)
-	    {
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-		    myname, __LINE__, MPI_ERR_IO,
-		    "**io", "**io %s", errMsg);
-		return;
-	    }
-	}
-    }
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (fd->fd_sys == INVALID_HANDLE_VALUE)
-    {
-    char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-	err = GetLastError();
-    ADIOI_NTFS_Strerror(err, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io",
-					   "**io %s", errMsg);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-    *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_read.c b/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_read.c
deleted file mode 100644
index 7c55c1bc376..00000000000
--- a/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_read.c
+++ /dev/null
@@ -1,259 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_ntfs.h"
-
-void ADIOI_NTFS_ReadContig(ADIO_File fd, void *buf, int count, 
-			   MPI_Datatype datatype, int file_ptr_type,
-			   ADIO_Offset offset, ADIO_Status *status,
-			   int *error_code)
-{
-    LONG dwTemp;
-    DWORD dwNumRead = 0;
-    int err=-1;
-    MPI_Count datatype_size, len;
-    static char myname[] = "ADIOI_NTFS_ReadContig";
-    OVERLAPPED *pOvl;
-
-    /* If file pointer is of type ADIO_INDIVIDUAL ignore the offset
-       and use the current location of file pointer */
-    if(file_ptr_type == ADIO_INDIVIDUAL){
-	offset = fd->fp_ind;
-    }
-
-    MPI_Type_size_x(datatype, &datatype_size);
-    len = datatype_size * count;
-
-    pOvl = (OVERLAPPED *) ADIOI_Calloc(sizeof(OVERLAPPED), 1);
-    if (pOvl == NULL)
-    {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-	    myname, __LINE__, MPI_ERR_IO,
-	    "**nomem", "**nomem %s", "OVERLAPPED");
-	return;
-    }
-    pOvl->hEvent = CreateEvent(NULL, TRUE, TRUE, NULL);
-    if (pOvl->hEvent == NULL)
-    {
-    char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-	err = GetLastError();
-    ADIOI_NTFS_Strerror(err, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-	    myname, __LINE__, MPI_ERR_IO,
-	    "**io", "**io %s", errMsg);
-	ADIOI_Free(pOvl);
-	return;
-    }
-    pOvl->Offset = DWORDLOW(offset);
-    pOvl->OffsetHigh = DWORDHIGH(offset);
-
-    if (file_ptr_type == ADIO_EXPLICIT_OFFSET)
-    {
-	if (fd->fp_sys_posn != offset)
-	{
-	    dwTemp = DWORDHIGH(offset);
-	    if (SetFilePointer(fd->fd_sys, DWORDLOW(offset), &dwTemp, FILE_BEGIN) == INVALID_SET_FILE_POINTER)
-	    {
-        char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-		err = GetLastError();
-        ADIOI_NTFS_Strerror(err, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-		if (err != NO_ERROR)
-		{
-		    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-			myname, __LINE__, MPI_ERR_IO,
-			"**io", "**io %s", errMsg);
-		    CloseHandle(pOvl->hEvent);
-		    ADIOI_Free(pOvl);
-		    return;
-		}
-	    }
-	}
-	/*
-	{
-	    ADIO_Fcntl_t fcntl_struct;
-	    int error_code;
-	    ADIO_Fcntl(fd, ADIO_FCNTL_GET_FSIZE, &fcntl_struct, &error_code);
-	    printf("File size b: %d\n", fcntl_struct.fsize);
-	}
-	printf("ReadFile(%d bytes)\n", len);fflush(stdout);
-	*/
-	err = ReadFile(fd->fd_sys, buf, len, &dwNumRead, pOvl);
-	/* --BEGIN ERROR HANDLING-- */
-	if (err == FALSE)
-	{
-        char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-	    err = GetLastError();
-        ADIOI_NTFS_Strerror(err, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-	    switch (err)
-	    {
-	    case ERROR_IO_PENDING:
-		break;
-	    case ERROR_HANDLE_EOF:
-		/*printf("EOF error\n");fflush(stdout);*/
-		SetEvent(pOvl->hEvent);
-		break;
-	    default:
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-		    myname, __LINE__, MPI_ERR_IO,
-		    "**io",
-		    "**io %s", errMsg);
-		CloseHandle(pOvl->hEvent);
-		ADIOI_Free(pOvl);
-		return;
-	    }
-	}
-	/* --END ERROR HANDLING-- */
-	err = GetOverlappedResult(fd->fd_sys, pOvl, &dwNumRead, TRUE);
-	/* --BEGIN ERROR HANDLING-- */
-	if (err == FALSE)
-	{
-        char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-	    err = GetLastError();
-        ADIOI_NTFS_Strerror(err, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-	    if (err != ERROR_HANDLE_EOF) /* Ignore EOF errors */
-	    {
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-		    MPIR_ERR_RECOVERABLE, myname,
-		    __LINE__, MPI_ERR_IO, "**io",
-		    "**io %s", errMsg);
-		CloseHandle(pOvl->hEvent);
-		ADIOI_Free(pOvl);
-		return;
-	    }
-	}
-	/* --END ERROR HANDLING-- */
-	if (!CloseHandle(pOvl->hEvent))
-	{
-        char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-	    err = GetLastError();
-        ADIOI_NTFS_Strerror(err, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-		myname, __LINE__, MPI_ERR_IO,
-		"**io", "**io %s", errMsg);
-	    CloseHandle(pOvl->hEvent);
-	    ADIOI_Free(pOvl);
-	    return;
-	}
-	ADIOI_Free(pOvl);
-
-	fd->fp_sys_posn = offset + (ADIO_Offset)dwNumRead;
-	/* individual file pointer not updated */        
-    }
-    else
-    {
-	/* read from curr. location of ind. file pointer */
-	if (fd->fp_sys_posn != fd->fp_ind)
-	{
-	    dwTemp = DWORDHIGH(fd->fp_ind);
-	    if (SetFilePointer(fd->fd_sys, DWORDLOW(fd->fp_ind), &dwTemp, FILE_BEGIN) == INVALID_SET_FILE_POINTER)
-	    {
-        char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-		err = GetLastError();
-        ADIOI_NTFS_Strerror(err, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-		if (err != NO_ERROR)
-		{
-		    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-			myname, __LINE__, MPI_ERR_IO,
-			"**io", "**io %s", errMsg);
-		    CloseHandle(pOvl->hEvent);
-		    ADIOI_Free(pOvl);
-		    return;
-		}
-	    }
-	}
-	/*
-	{
-	    ADIO_Fcntl_t fcntl_struct;
-	    int error_code;
-	    ADIO_Fcntl(fd, ADIO_FCNTL_GET_FSIZE, &fcntl_struct, &error_code);
-	    printf("File size c: %d\n", fcntl_struct.fsize);
-	}
-	printf("ReadFile(%d bytes)\n", len);fflush(stdout);
-	*/
-	err = ReadFile(fd->fd_sys, buf, len, &dwNumRead, pOvl);
-	/* --BEGIN ERROR HANDLING-- */
-	if (err == FALSE)
-	{
-        char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-	    err = GetLastError();
-        ADIOI_NTFS_Strerror(err, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-	    switch (err)
-	    {
-	    case ERROR_IO_PENDING:
-		break;
-	    case ERROR_HANDLE_EOF:
-		/*printf("EOF error\n");fflush(stdout);*/
-		SetEvent(pOvl->hEvent);
-		break;
-	    default:
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-		    myname, __LINE__, MPI_ERR_IO,
-		    "**io",
-		    "**io %s", errMsg);
-		CloseHandle(pOvl->hEvent);
-		ADIOI_Free(pOvl);
-		return;
-	    }
-	}
-	/* --END ERROR HANDLING-- */
-	err = GetOverlappedResult(fd->fd_sys, pOvl, &dwNumRead, TRUE);
-	/* --BEGIN ERROR HANDLING-- */
-	if (err == FALSE)
-	{
-        char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-	    err = GetLastError();
-        ADIOI_NTFS_Strerror(err, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-	    if (err != ERROR_HANDLE_EOF) /* Ignore EOF errors */
-	    {
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-		    MPIR_ERR_RECOVERABLE, myname,
-		    __LINE__, MPI_ERR_IO, "**io",
-		    "**io %s", errMsg);
-		CloseHandle(pOvl->hEvent);
-		ADIOI_Free(pOvl);
-		return;
-	    }
-	}
-	/* --END ERROR HANDLING-- */
-	if (!CloseHandle(pOvl->hEvent))
-	{
-        char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-	    err = GetLastError();
-        ADIOI_NTFS_Strerror(err, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-		myname, __LINE__, MPI_ERR_IO,
-		"**io", "**io %s", errMsg);
-	    ADIOI_Free(pOvl);
-	    return;
-	}
-	ADIOI_Free(pOvl);
-
-	fd->fp_ind = fd->fp_ind + (ADIO_Offset)dwNumRead; 
-	fd->fp_sys_posn = fd->fp_ind;
-    }         
-
-#ifdef HAVE_STATUS_SET_BYTES
-    if (err != FALSE)
-    {
-	MPIR_Status_set_bytes(status, datatype, dwNumRead);
-    }
-#endif
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (err == FALSE)
-    {
-    char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-	err = GetLastError();
-    ADIOI_NTFS_Strerror(err, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io",
-					   "**io %s", errMsg);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-    *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_resize.c b/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_resize.c
deleted file mode 100644
index 0fbeaaf9728..00000000000
--- a/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_resize.c
+++ /dev/null
@@ -1,51 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_ntfs.h"
-
-void ADIOI_NTFS_Resize(ADIO_File fd, ADIO_Offset size, int *error_code)
-{
-    LONG dwTemp;
-    DWORD err;
-    BOOL result;
-    static char myname[] = "ADIOI_NTFS_Resize";
-
-    dwTemp = DWORDHIGH(size);
-    err = SetFilePointer(fd->fd_sys, DWORDLOW(size), &dwTemp, FILE_BEGIN);
-    /* --BEGIN ERROR HANDLING-- */
-    if (err == INVALID_SET_FILE_POINTER)
-    {
-	err = GetLastError();
-	if (err != NO_ERROR)
-	{
-        char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-        ADIOI_NTFS_Strerror(err, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io",
-					   "**io %s", errMsg);
-	    return;
-	}
-    }
-    /*printf("setting file length to %d\n", size);fflush(stdout);*/
-    /* --END ERROR HANDLING-- */
-    result = SetEndOfFile(fd->fd_sys);
-    /* --BEGIN ERROR HANDLING-- */
-    if (result == FALSE)
-    {
-    char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-	err = GetLastError();
-    ADIOI_NTFS_Strerror(err, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io",
-					   "**io %s", errMsg);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-    *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_wait.c b/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_wait.c
deleted file mode 100644
index 80dfa4d3349..00000000000
--- a/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_wait.c
+++ /dev/null
@@ -1,20 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_ntfs.h"
-
-void ADIOI_NTFS_ReadComplete(ADIO_Request *request, ADIO_Status *status,
-			     int *error_code)  
-{
-    return;
-}
-
-
-void ADIOI_NTFS_WriteComplete(ADIO_Request *request, ADIO_Status *status,
-			      int *error_code)
-{
-    return;
-}
diff --git a/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_write.c b/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_write.c
deleted file mode 100644
index 389e8669362..00000000000
--- a/3rd-party/romio321/adio/ad_ntfs/ad_ntfs_write.c
+++ /dev/null
@@ -1,222 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_ntfs.h"
-
-void ADIOI_NTFS_WriteContig(ADIO_File fd, void *buf, int count, 
-			    MPI_Datatype datatype, int file_ptr_type,
-			    ADIO_Offset offset, ADIO_Status *status,
-			    int *error_code)
-{
-    static char myname[] = "ADIOI_NTFS_WriteContig";
-    LONG dwTemp;
-    DWORD dwNumWritten = 0;
-    MPI_Count err=-1, datatype_size, len;
-    OVERLAPPED *pOvl;
-    
-    /* If file pointer type in ADIO_INDIVIDUAL then offset should be
-	ignored and the current location of file pointer should be used */
-    if(file_ptr_type == ADIO_INDIVIDUAL){
-	offset = fd->fp_ind;
-    }
-
-    MPI_Type_size_x(datatype, &datatype_size);
-    len = datatype_size * count;
-
-    pOvl = (OVERLAPPED *) ADIOI_Calloc(sizeof(OVERLAPPED), 1);
-    if (pOvl == NULL)
-    {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-	    myname, __LINE__, MPI_ERR_IO,
-	    "**nomem", "**nomem %s", "OVERLAPPED");
-	return;
-    }
-    pOvl->hEvent = CreateEvent(NULL, TRUE, TRUE, NULL);
-    if (pOvl->hEvent == NULL)
-    {
-    char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-	err = GetLastError();
-    ADIOI_NTFS_Strerror(err, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-	    myname, __LINE__, MPI_ERR_IO,
-	    "**io", "**io %s", errMsg);
-	ADIOI_Free(pOvl);
-	return;
-    }
-    pOvl->Offset = DWORDLOW(offset);
-    pOvl->OffsetHigh = DWORDHIGH(offset);
-
-    if (file_ptr_type == ADIO_EXPLICIT_OFFSET)
-    {
-	if (fd->fp_sys_posn != offset)
-	{
-	    dwTemp = DWORDHIGH(offset);
-	    if (SetFilePointer(fd->fd_sys, DWORDLOW(offset), &dwTemp, FILE_BEGIN) == INVALID_SET_FILE_POINTER)
-	    {
-		err = GetLastError();
-		if (err != NO_ERROR)
-		{
-            char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-            ADIOI_NTFS_Strerror(err, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-		    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-			myname, __LINE__, MPI_ERR_IO,
-			"**io", "**io %s", errMsg);
-		    CloseHandle(pOvl->hEvent);
-		    ADIOI_Free(pOvl);
-		    return;
-		}
-	    }
-	}
-	/*printf("WriteFile(%d bytes)\n", len);fflush(stdout);*/
-	err = WriteFile(fd->fd_sys, buf, len, &dwNumWritten, pOvl);
-	/* --BEGIN ERROR HANDLING-- */
-	if (err == FALSE)
-	{
-	    err = GetLastError();
-	    if (err != ERROR_IO_PENDING)
-	    {
-        char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-        ADIOI_NTFS_Strerror(err, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-		    myname, __LINE__, MPI_ERR_IO,
-		    "**io",
-		    "**io %s", errMsg);
-		CloseHandle(pOvl->hEvent);
-		ADIOI_Free(pOvl);
-		return;
-	    }
-	}
-	/* --END ERROR HANDLING-- */
-	err = GetOverlappedResult(fd->fd_sys, pOvl, &dwNumWritten, TRUE);
-	/* --BEGIN ERROR HANDLING-- */
-	if (err == FALSE)
-	{
-        char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-	    err = GetLastError();
-        ADIOI_NTFS_Strerror(err, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-		MPIR_ERR_RECOVERABLE, myname,
-		__LINE__, MPI_ERR_IO, "**io",
-		"**io %s", errMsg);
-	    CloseHandle(pOvl->hEvent);
-	    ADIOI_Free(pOvl);
-	    return;
-	}
-	/* --END ERROR HANDLING-- */
-	if (!CloseHandle(pOvl->hEvent))
-	{
-        char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-	    err = GetLastError();
-        ADIOI_NTFS_Strerror(err, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-		myname, __LINE__, MPI_ERR_IO,
-		"**io", "**io %s", errMsg);
-	    CloseHandle(pOvl->hEvent);
-	    ADIOI_Free(pOvl);
-	    return;
-	}
-	ADIOI_Free(pOvl);
-
-	fd->fp_sys_posn = offset + dwNumWritten;
-	/* individual file pointer not updated */        
-    }
-    else
-    {
-	/* write from curr. location of ind. file pointer */
-	if (fd->fp_sys_posn != fd->fp_ind)
-	{
-	    dwTemp = DWORDHIGH(fd->fp_ind);
-	    if (SetFilePointer(fd->fd_sys, DWORDLOW(fd->fp_ind), &dwTemp, FILE_BEGIN) == INVALID_SET_FILE_POINTER)
-	    {
-		err = GetLastError();
-		if (err != NO_ERROR)
-		{
-            char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-            ADIOI_NTFS_Strerror(err, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-		    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-			myname, __LINE__, MPI_ERR_IO,
-			"**io", "**io %s", errMsg);
-		    CloseHandle(pOvl->hEvent);
-		    ADIOI_Free(pOvl);
-		    return;
-		}
-	    }
-	}
-	/*printf("WriteFile(%d bytes)\n", len);fflush(stdout);*/
-	err = WriteFile(fd->fd_sys, buf, len, &dwNumWritten, pOvl);
-	/* --BEGIN ERROR HANDLING-- */
-	if (err == FALSE)
-	{
-	    err = GetLastError();
-	    if (err != ERROR_IO_PENDING)
-	    {
-        char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-        ADIOI_NTFS_Strerror(err, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-		    myname, __LINE__, MPI_ERR_IO,
-		    "**io",
-		    "**io %s", errMsg);
-		CloseHandle(pOvl->hEvent);
-		ADIOI_Free(pOvl);
-		return;
-	    }
-	}
-	/* --END ERROR HANDLING-- */
-	err = GetOverlappedResult(fd->fd_sys, pOvl, &dwNumWritten, TRUE);
-	/* --BEGIN ERROR HANDLING-- */
-	if (err == FALSE)
-	{
-        char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-	    err = GetLastError();
-        ADIOI_NTFS_Strerror(err, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-		MPIR_ERR_RECOVERABLE, myname,
-		__LINE__, MPI_ERR_IO, "**io",
-		"**io %s", errMsg);
-	    CloseHandle(pOvl->hEvent);
-	    ADIOI_Free(pOvl);
-	    return;
-	}
-	/* --END ERROR HANDLING-- */
-	if (!CloseHandle(pOvl->hEvent))
-	{
-        char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-	    err = GetLastError();
-        ADIOI_NTFS_Strerror(err, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-		myname, __LINE__, MPI_ERR_IO,
-		"**io", "**io %s", errMsg);
-	    ADIOI_Free(pOvl);
-	    return;
-	}
-	ADIOI_Free(pOvl);
-
-	fd->fp_ind = fd->fp_ind + dwNumWritten;
-	fd->fp_sys_posn = fd->fp_ind;
-    }
-
-#ifdef HAVE_STATUS_SET_BYTES
-    if (err != FALSE)
-    {
-	MPIR_Status_set_bytes(status, datatype, dwNumWritten);
-    }
-#endif
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (err == FALSE)
-    {
-    char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-	err = GetLastError();
-    ADIOI_NTFS_Strerror(err, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io",
-					   "**io %s", errMsg);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-    *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_panfs/Makefile.mk b/3rd-party/romio321/adio/ad_panfs/Makefile.mk
deleted file mode 100644
index ad412d9bb08..00000000000
--- a/3rd-party/romio321/adio/ad_panfs/Makefile.mk
+++ /dev/null
@@ -1,28 +0,0 @@
-## -*- Mode: Makefile; -*-
-## vim: set ft=automake :
-##
-## (C) 2011 by Argonne National Laboratory.
-##     See COPYRIGHT in top-level directory.
-##
-
-if BUILD_PANFS_OPEN6
-# override open with one that uses newer features
-panfs_open = adio/ad_panfs/ad_panfs_open6.c
-else
-panfs_open = adio/ad_panfs/ad_panfs_open.c
-endif
-
-if BUILD_AD_PANFS
-
-noinst_HEADERS += adio/ad_panfs/ad_panfs.h
-
-romio_other_sources +=              \
-    adio/ad_panfs/ad_panfs.c        \
-    $(panfs_open)                   \
-    adio/ad_panfs/ad_panfs_hints.c  \
-    adio/ad_panfs/ad_panfs_read.c   \
-    adio/ad_panfs/ad_panfs_resize.c \
-    adio/ad_panfs/ad_panfs_write.c
-
-endif BUILD_AD_PANFS
-
diff --git a/3rd-party/romio321/adio/ad_panfs/ad_panfs.c b/3rd-party/romio321/adio/ad_panfs/ad_panfs.c
deleted file mode 100644
index 8bc14fccdf2..00000000000
--- a/3rd-party/romio321/adio/ad_panfs/ad_panfs.c
+++ /dev/null
@@ -1,57 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   ad_panfs.c
- *
- *   Copyright (C) 2001 University of Chicago.
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_panfs.h"
-
-/* adioi.h has the ADIOI_Fns_struct define */
-#include "adioi.h"
-
-#ifndef ROMIOCONF_H_INCLUDED
-#include "romioconf.h"
-#define ROMIOCONF_H_INCLUDED
-#endif
-
-
-struct ADIOI_Fns_struct ADIO_PANFS_operations = {
-#ifdef HAVE_PAN_FS_CLIENT_RAIDN_ENCODING_T
-    ADIOI_PANFS_Open6, /* Open, using newer Panasas features */
-#else
-    ADIOI_PANFS_Open,  /* open, but using Panasas5 and earlier features */
-#endif
-    ADIOI_GEN_OpenColl,
-    ADIOI_PANFS_ReadContig, /* ReadContig */
-    ADIOI_PANFS_WriteContig, /* WriteContig */
-    ADIOI_GEN_ReadStridedColl, /* ReadStridedColl */
-    ADIOI_GEN_WriteStridedColl, /* WriteStridedColl */
-    ADIOI_GEN_SeekIndividual, /* SeekIndividual */
-    ADIOI_GEN_Fcntl, /* Fcntl */
-    ADIOI_PANFS_SetInfo, /* SetInfo */
-    ADIOI_GEN_ReadStrided, /* ReadStrided */
-    ADIOI_GEN_WriteStrided, /* WriteStrided */
-    ADIOI_GEN_Close, /* Close */
-#ifdef ROMIO_HAVE_WORKING_AIO
-    ADIOI_GEN_IreadContig, /* IreadContig */
-    ADIOI_GEN_IwriteContig, /* IwriteContig */
-#else
-    ADIOI_FAKE_IreadContig, /* IreadContig */
-    ADIOI_FAKE_IwriteContig, /* IwriteContig */
-#endif
-    ADIOI_GEN_IODone, /* ReadDone */
-    ADIOI_GEN_IODone, /* WriteDone */
-    ADIOI_GEN_IOComplete, /* ReadComplete */
-    ADIOI_GEN_IOComplete, /* WriteComplete */
-    ADIOI_GEN_IreadStrided, /* IreadStrided */
-    ADIOI_GEN_IwriteStrided, /* IwriteStrided */
-    ADIOI_GEN_Flush, /* Flush */
-    ADIOI_PANFS_Resize, /* Resize */
-    ADIOI_GEN_Delete, /* Delete */
-    ADIOI_GEN_Feature,
-    "PANFS: Panasas PanFS",
-    ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
-    ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
-};
diff --git a/3rd-party/romio321/adio/ad_panfs/ad_panfs.h b/3rd-party/romio321/adio/ad_panfs/ad_panfs.h
deleted file mode 100644
index 31a0f3d8a8b..00000000000
--- a/3rd-party/romio321/adio/ad_panfs/ad_panfs.h
+++ /dev/null
@@ -1,70 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   ad_panfs.h
- *
- *   Copyright (C) 2001 University of Chicago.
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#ifndef AD_PANFS_INCLUDE
-#define AD_PANFS_INCLUDE
-
-#include <unistd.h>
-#include <sys/types.h>
-#include <fcntl.h>
-#include "adio.h"
-
-#ifndef NO_AIO
-# ifdef AIO_SUN
-# include <sys/asynch.h>
-# else
-  #ifdef HAVE_AIO_LITE_H
-  #include <aio-lite.h>
-  #else
-   #ifdef  HAVE_AIO_H
-   #include <aio.h>
-   #endif
-   #ifdef HAVE_SYS_AIO_H
-   #include <sys/aio.h>
-   #endif
-  #endif
-# endif
-#endif
-
-void ADIOI_PANFS_Open(ADIO_File fd, int *error_code);
-/* Panasas 6 introduced some new features */
-void ADIOI_PANFS_Open6(ADIO_File fd, int *error_code);
-void ADIOI_PANFS_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code);
-void ADIOI_PANFS_ReadContig(ADIO_File fd, void *buf, int count, 
-			  MPI_Datatype datatype, int file_ptr_type,
-			  ADIO_Offset offset, ADIO_Status *status,
-			  int *error_code);
-void ADIOI_PANFS_Resize(ADIO_File fd, ADIO_Offset size, int *error_code);
-void ADIOI_PANFS_WriteContig(ADIO_File fd, const void *buf, int count,
-			   MPI_Datatype datatype, int file_ptr_type,
-			   ADIO_Offset offset, ADIO_Status *status,
-			   int *error_code);
-
-/* TODO: move this to common code and have all routines retry. */
-/* TODO: also check for EWOULDBLOCK */
-#if defined(NEEDS_USLEEP_DECL)
-int usleep(useconds_t usec);
-#endif
-
-/* Delay 1 ms */
-#define AD_PANFS_RETRY_DELAY 1000
-
-#define AD_PANFS_RETRY(_op_,_rc_) \
-{ \
-    _rc_ = (_op_); \
-    while(_rc_ == -1 && errno == EAGAIN) \
-    { \
-        if(usleep(AD_PANFS_RETRY_DELAY) == -1) \
-        { \
-            break; \
-        } \
-        _rc_ = (_op_); \
-    } \
-}
-
-#endif
diff --git a/3rd-party/romio321/adio/ad_panfs/ad_panfs_hints.c b/3rd-party/romio321/adio/ad_panfs/ad_panfs_hints.c
deleted file mode 100644
index 94178ab0cfe..00000000000
--- a/3rd-party/romio321/adio/ad_panfs/ad_panfs_hints.c
+++ /dev/null
@@ -1,72 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   ad_panfs_hints.c
- *
- *   Copyright (C) 2001 University of Chicago.
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_panfs.h"
-#include <pan_fs_client_cw_mode.h>
-#include "hint_fns.h"
-
-void ADIOI_PANFS_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code)
-{
-#if defined(MPICH) || !defined(PRINT_ERR_MSG)
-    static char myname[] = "ADIOI_PANFS_SETINFO";
-#endif
-    int gen_error_code;
-
-    *error_code = MPI_SUCCESS;
-
-    if (fd->info == MPI_INFO_NULL) {
-	    /* This must be part of the open call. can set striping parameters 
-         * if necessary. 
-         */ 
-	    MPI_Info_create(&(fd->info));
-
-	/* anticipate concurrent writes in an MPI-IO application */
-	ADIOI_Info_set (fd->info, "panfs_concurrent_write", "1");
-
-        /* has user specified striping parameters 
-               and do they have the same value on all processes? */
-        if (users_info != MPI_INFO_NULL) {
-
-	    ADIOI_Info_check_and_install_int(fd, users_info, "panfs_concurrent_write",
-		    NULL, myname, error_code);
-
-	    ADIOI_Info_check_and_install_int(fd, users_info, "panfs_layout_type",
-		    NULL, myname, error_code);
-
-	    ADIOI_Info_check_and_install_int(fd, users_info, "panfs_layout_stripe_unit",
-		    NULL, myname, error_code);
-
-	    /* strange: there was a check "layout_type ==
-	     * PAN_FS_CLIENT_LAYOUT_TYPE__RAID1_5_PARITY_STRIPE, but
-	     * nothing ever touched layout_type  */
-	    ADIOI_Info_check_and_install_int(fd, users_info,
-		    "panfs_layout_parity_stripe_width", NULL, myname, error_code);
-
-	    ADIOI_Info_check_and_install_int(fd, users_info,
-		    "panfs_layout_parity_stripe_depth", NULL, myname, error_code);
-
-	    ADIOI_Info_check_and_install_int(fd, users_info,
-		    "panfs_layout_total_num_comps", NULL, myname, error_code);
-	    /* this hint used to check for
-	     * PAN_FS_CLIENT_LAYOUT_TYPE__RAID1_5_PARITY_STRIPE or
-	     * PAN_FS_CLIENT_LAYOUT_TYPE__RAID10, but again, layout_type never
-	     * gets updated */
-	    ADIOI_Info_check_and_install_int(fd, users_info,
-		    "panfs_layout_visit_policy", NULL, myname, error_code);
-        }
-    }
-
-    ADIOI_GEN_SetInfo(fd, users_info, &gen_error_code); 
-    /* If this function is successful, use the error code returned from ADIOI_GEN_SetInfo
-     * otherwise use the error_code generated by this function
-     */
-    if(*error_code == MPI_SUCCESS)
-    {
-        *error_code = gen_error_code;
-    }
-}
diff --git a/3rd-party/romio321/adio/ad_panfs/ad_panfs_open.c b/3rd-party/romio321/adio/ad_panfs/ad_panfs_open.c
deleted file mode 100644
index d5374ebf448..00000000000
--- a/3rd-party/romio321/adio/ad_panfs/ad_panfs_open.c
+++ /dev/null
@@ -1,348 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   ad_panfs_open.c
- *
- *   Copyright (C) 2001 University of Chicago.
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_panfs.h"
-#include <string.h>
-#include <pan_fs_client_cw_mode.h>
-#define TEMP_BUFFER_SIZE 64
-
-void ADIOI_PANFS_Open(ADIO_File fd, int *error_code)
-{
-    char* value;
-    int perm, old_mask, amode, flag;
-    static char myname[] = "ADIOI_PANFS_OPEN";
-
-    if (fd->perm == ADIO_PERM_NULL) {
-        old_mask = umask(022);
-        umask(old_mask);
-        perm = ~old_mask & 0666;
-    }
-    else perm = fd->perm;
-
-    amode = 0;
-    if (fd->access_mode & ADIO_CREATE)
-    {
-        pan_fs_client_layout_agg_type_t layout_type = PAN_FS_CLIENT_LAYOUT_TYPE__DEFAULT;
-        unsigned long int layout_stripe_unit = 0;
-        unsigned long int layout_parity_stripe_width = 0;
-        unsigned long int layout_parity_stripe_depth = 0; 
-        unsigned long int layout_total_num_comps = 0;
-        pan_fs_client_layout_visit_t layout_visit_policy  = PAN_FS_CLIENT_LAYOUT_VISIT__ROUND_ROBIN;
-        int myrank;
-
-        MPI_Comm_rank(fd->comm, &myrank);
-
-        *error_code = MPI_SUCCESS;
-        value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
-        ADIOI_Info_get(fd->info, "panfs_layout_type", MPI_MAX_INFO_VAL, 
-                 value, &flag);
-        if (flag) {
-            layout_type = strtoul(value,NULL,10);
-        }
-        ADIOI_Info_get(fd->info, "panfs_layout_stripe_unit", MPI_MAX_INFO_VAL, 
-                 value, &flag);
-        if (flag) {
-            layout_stripe_unit = strtoul(value,NULL,10);
-        }
-        ADIOI_Info_get(fd->info, "panfs_layout_total_num_comps", MPI_MAX_INFO_VAL, 
-                 value, &flag);
-        if (flag) {
-            layout_total_num_comps = strtoul(value,NULL,10);
-        }
-        ADIOI_Info_get(fd->info, "panfs_layout_parity_stripe_width", MPI_MAX_INFO_VAL, 
-                 value, &flag);
-        if (flag) {
-            layout_parity_stripe_width = strtoul(value,NULL,10);
-        }
-        ADIOI_Info_get(fd->info, "panfs_layout_parity_stripe_depth", MPI_MAX_INFO_VAL, 
-                 value, &flag);
-        if (flag) {
-            layout_parity_stripe_depth = strtoul(value,NULL,10);
-        }
-        ADIOI_Info_get(fd->info, "panfs_layout_visit_policy", MPI_MAX_INFO_VAL, 
-                 value, &flag);
-        if (flag) {
-            layout_visit_policy = strtoul(value,NULL,10);
-        }
-        ADIOI_Free(value);
-
-        amode = amode | O_CREAT;
-        /* Check for valid set of hints */
-        if ((layout_type < PAN_FS_CLIENT_LAYOUT_TYPE__DEFAULT) ||
-           (layout_type > PAN_FS_CLIENT_LAYOUT_TYPE__RAID10))
-        {
-            FPRINTF(stderr, "%s: panfs_layout_type is not a valid value: %u.\n", myname, layout_type);
-            MPI_Abort(MPI_COMM_WORLD, 1);
-        }
-        if ((layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAID0) &&
-           ((layout_stripe_unit == 0) || (layout_total_num_comps == 0)))
-        {
-            if(layout_stripe_unit == 0)
-            {
-                FPRINTF(stderr, "%s: MPI_Info does not contain the panfs_layout_stripe_unit hint which is necessary to specify a valid RAID0 layout to the PAN_FS_CLIENT_LAYOUT_CREATE_FILE ioctl.\n", myname);
-            }
-            if(layout_total_num_comps == 0)
-            {
-                FPRINTF(stderr, "%s: MPI_Info does not contain the panfs_layout_total_num_comps hint which is necessary to specify a valid RAID0 layout to the PAN_FS_CLIENT_LAYOUT_CREATE_FILE ioctl.\n", myname);
-            }
-            MPI_Abort(MPI_COMM_WORLD, 1);
-        }
-        if (layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAID1_5_PARITY_STRIPE)
-        {
-            if ((layout_stripe_unit == 0) ||
-               (layout_parity_stripe_width == 0) ||
-               (layout_parity_stripe_depth == 0) ||
-               (layout_total_num_comps == 0))
-            {
-                if(layout_stripe_unit == 0)
-                {
-                    FPRINTF(stderr, "%s: MPI_Info does not contain the panfs_layout_stripe_unit hint which is necessary to specify a valid RAID5 parity stripe layout to the PAN_FS_CLIENT_LAYOUT_CREATE_FILE ioctl.\n", myname);
-                }
-                if(layout_total_num_comps == 0)
-                {
-                    FPRINTF(stderr, "%s: MPI_Info does not contain the panfs_layout_total_num_comps hint which is necessary to specify a valid RAID5 parity stripe layout to the PAN_FS_CLIENT_LAYOUT_CREATE_FILE ioctl.\n", myname);
-                }
-                if(layout_parity_stripe_width == 0)
-                {
-                    FPRINTF(stderr, "%s: MPI_Info does not contain the panfs_layout_parity_stripe_width hint which is necessary to specify a valid RAID5 parity stripe layout to the PAN_FS_CLIENT_LAYOUT_CREATE_FILE ioctl.\n", myname);
-                }
-                if(layout_parity_stripe_depth == 0)
-                {
-                    FPRINTF(stderr, "%s: MPI_Info does not contain the panfs_layout_parity_stripe_depth hint which is necessary to specify a valid RAID5 parity stripe layout to the PAN_FS_CLIENT_LAYOUT_CREATE_FILE ioctl.\n", myname);
-                }
-                MPI_Abort(MPI_COMM_WORLD, 1);
-           }
-           if ((layout_visit_policy < PAN_FS_CLIENT_LAYOUT_VISIT__ROUND_ROBIN) ||
-              (layout_visit_policy > PAN_FS_CLIENT_LAYOUT_VISIT__ROUND_ROBIN_WITH_HASHED_OFFSET))
-           {
-                FPRINTF(stderr, "%s: panfs_layout_visit_policy is not a valid value: %u.\n", myname, layout_visit_policy);
-                MPI_Abort(MPI_COMM_WORLD, 1);
-           }
-        }
-        if (layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAID10)
-        {
-            if ((layout_stripe_unit == 0) || (layout_total_num_comps == 0))
-            {
-                if(layout_stripe_unit == 0)
-                {
-                    FPRINTF(stderr, "%s: MPI_Info does not contain the panfs_layout_stripe_unit hint which is necessary to specify a valid RAID10 layout to the PAN_FS_CLIENT_LAYOUT_CREATE_FILE ioctl.\n", myname);
-                }
-                if(layout_total_num_comps == 0)
-                {
-                    FPRINTF(stderr, "%s: MPI_Info does not contain the panfs_layout_total_num_comps hint which is necessary to specify a valid RAID10 layout to the PAN_FS_CLIENT_LAYOUT_CREATE_FILE ioctl.\n", myname);
-                }
-                MPI_Abort(MPI_COMM_WORLD, 1);
-            }
-            if ((layout_visit_policy < PAN_FS_CLIENT_LAYOUT_VISIT__ROUND_ROBIN) ||
-              (layout_visit_policy > PAN_FS_CLIENT_LAYOUT_VISIT__ROUND_ROBIN_WITH_HASHED_OFFSET))
-            {
-                FPRINTF(stderr, "%s: panfs_layout_visit_policy is not a valid value: %u.\n", myname, layout_visit_policy);
-                MPI_Abort(MPI_COMM_WORLD, 1);
-            }
-        }
-        /* Create the file via ioctl() or open(). ADIOI_PANFS_Open's caller 
-         * already optimizes performance by only calling this function with
-         * ADIO_CREATE on rank 0.  Therefore, we don't need to worry about 
-         * implementing that optimization here. */
-        if((layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAID0) || (layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAID1_5_PARITY_STRIPE) 
-                || (layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAID10)) {
-            pan_fs_client_layout_create_args_t file_create_args;    
-            int fd_dir;
-            char* slash;
-            struct stat stat_buf;
-            int err;
-            char *path;
-
-            /* Check that the file does not exist before
-             * trying to create it.  The ioctl itself should
-             * be able to handle this condition.  Currently,
-             * the ioctl will return successfully if the file
-             * has been previously created.  Filed bug 33862
-             * to track the problem.
-             */
-            err = stat(fd->filename,&stat_buf);
-            if((err == -1) && (errno != ENOENT))
-            {
-                FPRINTF(stderr,"%s: Unexpected I/O Error calling stat() on PanFS file: %s.\n", myname, strerror(errno));
-                MPI_Abort(MPI_COMM_WORLD, 1);
-            }
-            else if (err == 0)
-            {
-                FPRINTF(stderr,"%s: Cannot create PanFS file with ioctl when file already exists.\n", myname);
-                MPI_Abort(MPI_COMM_WORLD, 1);
-            }
-            else
-            {
-                /* (err == -1) && (errno == ENOENT) */
-                /* File does not exist */
-                path = ADIOI_Strdup(fd->filename);
-                slash = strrchr(path, '/');
-                if (!slash)
-                    ADIOI_Strncpy(path, ".", 2);
-                else {
-                    if (slash == path) 
-                        *(path + 1) = '\0';
-                    else *slash = '\0';
-                }
-
-                /* create PanFS object */
-                memset(&file_create_args,0,sizeof(pan_fs_client_layout_create_args_t));
-                /* open directory */
-                fd_dir = open(path, O_RDONLY);
-                if (fd_dir < 0) {
-                    FPRINTF(stderr, "%s: I/O Error opening parent directory to create PanFS file using ioctl: %s.\n", myname, strerror(errno));
-                    MPI_Abort(MPI_COMM_WORLD, 1);
-                }
-                else
-                {
-                    char *file_name_ptr = fd->filename;
-                    slash = strrchr(fd->filename, '/');
-                    if (slash)
-                    {
-                        file_name_ptr = slash + 1;
-                    }
-                    /* create file in the directory */
-                    file_create_args.mode = perm;
-                    file_create_args.version = PAN_FS_CLIENT_LAYOUT_VERSION;
-                    file_create_args.flags = PAN_FS_CLIENT_LAYOUT_CREATE_F__NONE;
-                    ADIOI_Strncpy(file_create_args.filename, file_name_ptr, strlen(fd->filename)+1); 
-                    file_create_args.layout.agg_type = layout_type;
-                    file_create_args.layout.layout_is_valid = 1;
-                    if(layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAID1_5_PARITY_STRIPE)
-                    {
-                        file_create_args.layout.u.raid1_5_parity_stripe.total_num_comps = layout_total_num_comps;
-                        file_create_args.layout.u.raid1_5_parity_stripe.parity_stripe_width   = layout_parity_stripe_width;
-                        file_create_args.layout.u.raid1_5_parity_stripe.parity_stripe_depth   = layout_parity_stripe_depth;
-                        file_create_args.layout.u.raid1_5_parity_stripe.stripe_unit     = layout_stripe_unit;
-                        file_create_args.layout.u.raid1_5_parity_stripe.layout_visit_policy   = layout_visit_policy;
-                    }
-                    else if(layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAID0)
-                    {
-                        file_create_args.layout.u.raid0.total_num_comps = layout_total_num_comps;
-                        file_create_args.layout.u.raid0.stripe_unit     = layout_stripe_unit;
-                    }
-                    else if(layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAID10)
-                    {
-                        file_create_args.layout.u.raid10.total_num_comps     = layout_total_num_comps;
-                        file_create_args.layout.u.raid10.stripe_unit         = layout_stripe_unit;
-                        file_create_args.layout.u.raid10.layout_visit_policy = layout_visit_policy;
-                    }
-                    err = ioctl(fd_dir, PAN_FS_CLIENT_LAYOUT_CREATE_FILE, &file_create_args);
-                    if (err < 0) {
-                        FPRINTF(stderr, "%s: I/O Error doing ioctl on parent directory to create PanFS file using ioctl: %s.\n", myname, strerror(errno));
-                        MPI_Abort(MPI_COMM_WORLD, 1);
-                    }
-                    err = close(fd_dir);
-                }
-                ADIOI_Free(path);
-            }
-        }
-        else
-        {
-            int create_fd = open(fd->filename,amode,perm);
-            if(create_fd != -1)
-            {
-                close(create_fd);
-            }
-            else
-            {
-                FPRINTF(stderr, "%s: I/O Error creating PanFS file using open: %s.\n", myname, strerror(errno));
-                MPI_Abort(MPI_COMM_WORLD, 1);
-            }
-        }
-    }
-    if (fd->access_mode & ADIO_RDONLY)
-	amode = amode | O_RDONLY;
-    if (fd->access_mode & ADIO_WRONLY)
-	amode = amode | O_WRONLY;
-    if (fd->access_mode & ADIO_RDWR)
-	amode = amode | O_RDWR;
-    if (fd->access_mode & ADIO_EXCL)
-	amode = amode | O_EXCL;
-
-	value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
-	ADIOI_Info_get(fd->info, "panfs_concurrent_write", MPI_MAX_INFO_VAL, 
-		     value, &flag);
-	if (flag) {
-        unsigned long int concurrent_write = strtoul(value,NULL,10);
-        if(concurrent_write == 1)
-        {
-            amode = amode | O_CONCURRENT_WRITE;
-        }
-	}
-	ADIOI_Free(value);
-
-    fd->fd_sys = open(fd->filename, amode, perm);
-    fd->fd_direct = -1;
-
-    if (fd->fd_sys != -1)
-    {
-        int rc;
-        char temp_buffer[TEMP_BUFFER_SIZE];
-        pan_fs_client_layout_query_args_t file_query_args;
-        memset(&file_query_args,0,sizeof(pan_fs_client_layout_query_args_t));
-        file_query_args.version = PAN_FS_CLIENT_LAYOUT_VERSION;
-        rc = ioctl(fd->fd_sys, PAN_FS_CLIENT_LAYOUT_QUERY_FILE, &file_query_args);
-        if (rc < 0)
-        {
-            /* Error - set layout type to unknown */
-	        ADIOI_Info_set(fd->info, "panfs_layout_type", "PAN_FS_CLIENT_LAYOUT_TYPE__INVALID");
-        }
-        else 
-        {
-            ADIOI_Snprintf(temp_buffer,TEMP_BUFFER_SIZE,"%u",file_query_args.layout.agg_type);
-            ADIOI_Info_set(fd->info, "panfs_layout_type", temp_buffer);
-            if (file_query_args.layout.layout_is_valid == 1)
-            {
-                switch (file_query_args.layout.agg_type)
-                {
-                    case PAN_FS_CLIENT_LAYOUT_TYPE__RAID0:
-                        ADIOI_Snprintf(temp_buffer,TEMP_BUFFER_SIZE,"%u",file_query_args.layout.u.raid0.stripe_unit);
-                        ADIOI_Info_set(fd->info, "panfs_layout_stripe_unit", temp_buffer);
-                        ADIOI_Snprintf(temp_buffer,TEMP_BUFFER_SIZE,"%u",file_query_args.layout.u.raid0.total_num_comps);
-                        ADIOI_Info_set(fd->info, "panfs_layout_total_num_comps", temp_buffer);
-                        break;
-                    case PAN_FS_CLIENT_LAYOUT_TYPE__RAID1_5_PARITY_STRIPE:
-                        ADIOI_Snprintf(temp_buffer,TEMP_BUFFER_SIZE,"%u",file_query_args.layout.u.raid1_5_parity_stripe.stripe_unit);
-                        ADIOI_Info_set(fd->info, "panfs_layout_stripe_unit", temp_buffer);
-                        ADIOI_Snprintf(temp_buffer,TEMP_BUFFER_SIZE,"%u",file_query_args.layout.u.raid1_5_parity_stripe.parity_stripe_width);
-                        ADIOI_Info_set(fd->info, "panfs_layout_parity_stripe_width", temp_buffer);
-                        ADIOI_Snprintf(temp_buffer,TEMP_BUFFER_SIZE,"%u",file_query_args.layout.u.raid1_5_parity_stripe.parity_stripe_depth);
-                        ADIOI_Info_set(fd->info, "panfs_layout_parity_stripe_depth", temp_buffer);
-                        ADIOI_Snprintf(temp_buffer,TEMP_BUFFER_SIZE,"%u",file_query_args.layout.u.raid1_5_parity_stripe.total_num_comps);
-                        ADIOI_Info_set(fd->info, "panfs_layout_total_num_comps", temp_buffer);
-                        ADIOI_Snprintf(temp_buffer,TEMP_BUFFER_SIZE,"%u",file_query_args.layout.u.raid1_5_parity_stripe.layout_visit_policy);
-                        ADIOI_Info_set(fd->info, "panfs_layout_visit_policy", temp_buffer);
-                        break;
-                    case PAN_FS_CLIENT_LAYOUT_TYPE__RAID10:
-                        ADIOI_Snprintf(temp_buffer,TEMP_BUFFER_SIZE,"%u",file_query_args.layout.u.raid10.stripe_unit);
-                        ADIOI_Info_set(fd->info, "panfs_layout_stripe_unit", temp_buffer);
-                        ADIOI_Snprintf(temp_buffer,TEMP_BUFFER_SIZE,"%u",file_query_args.layout.u.raid10.total_num_comps);
-                        ADIOI_Info_set(fd->info, "panfs_layout_total_num_comps", temp_buffer);
-                        ADIOI_Snprintf(temp_buffer,TEMP_BUFFER_SIZE,"%u",file_query_args.layout.u.raid10.layout_visit_policy);
-                        ADIOI_Info_set(fd->info, "panfs_layout_visit_policy", temp_buffer);
-                        break;
-                    case PAN_FS_CLIENT_LAYOUT_TYPE__INVALID:
-                    case PAN_FS_CLIENT_LAYOUT_TYPE__DEFAULT:
-                        MPI_Info_set(fd->info, "panfs_layout_type",
-                                "PAN_FS_CLIENT_LAYOUT_TYPE__INVALID");
-		  default:
-			  break;
-                }
-            }
-        }
-    }
-
-    if ((fd->fd_sys != -1) && (fd->access_mode & ADIO_APPEND))
-	fd->fp_ind = fd->fp_sys_posn = lseek(fd->fd_sys, 0, SEEK_END);
-
-    if (fd->fd_sys == -1) {
-	*error_code = ADIOI_Err_create_code(myname, fd->filename, errno);
-    }
-    else *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_panfs/ad_panfs_open6.c b/3rd-party/romio321/adio/ad_panfs/ad_panfs_open6.c
deleted file mode 100644
index 6906ff08d7e..00000000000
--- a/3rd-party/romio321/adio/ad_panfs/ad_panfs_open6.c
+++ /dev/null
@@ -1,440 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *   ad_panfs_open.c
- *
- *   Copyright (C) 2001 University of Chicago.
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_panfs.h"
-#include <string.h>
-#include <pan_fs_client_cw_mode.h>
-#define TEMP_BUFFER_SIZE 64
-
-void ADIOI_PANFS_Open6(ADIO_File fd, int *error_code)
-{
-    char *value;
-    int perm, old_mask, amode, flag;
-    static char myname[] = "ADIOI_PANFS_OPEN6";
-
-    if (fd->perm == ADIO_PERM_NULL) {
-        old_mask = umask(022);
-        umask(old_mask);
-        perm = ~old_mask & 0666;
-    }
-    else
-        perm = fd->perm;
-
-    amode = 0;
-    if (fd->access_mode & ADIO_CREATE) {
-        pan_fs_client_layout_agg_type_t layout_type = PAN_FS_CLIENT_LAYOUT_TYPE__DEFAULT;
-        unsigned long int layout_stripe_unit = 0;
-        unsigned long int layout_parity_stripe_width = 0;
-        unsigned long int layout_parity_stripe_depth = 0;
-        unsigned long int layout_total_num_comps = 0;
-        unsigned long int layout_max_faults = 2;
-        pan_fs_client_layout_visit_t layout_visit_policy = PAN_FS_CLIENT_LAYOUT_VISIT__ROUND_ROBIN;
-        pan_fs_client_raidn_encoding_t layout_encoding = PAN_FS_CLIENT_LAYOUT_RAIDN_ENCODING_RS;
-        int myrank;
-
-        MPI_Comm_rank(fd->comm, &myrank);
-
-        *error_code = MPI_SUCCESS;
-        value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL + 1) * sizeof(char));
-        ADIOI_Info_get(fd->info, "panfs_layout_type", MPI_MAX_INFO_VAL, value, &flag);
-        if (flag) {
-            layout_type = strtoul(value, NULL, 10);
-        }
-        ADIOI_Info_get(fd->info, "panfs_layout_stripe_unit", MPI_MAX_INFO_VAL, value, &flag);
-        if (flag) {
-            layout_stripe_unit = strtoul(value, NULL, 10);
-        }
-        ADIOI_Info_get(fd->info, "panfs_layout_total_num_comps", MPI_MAX_INFO_VAL, value, &flag);
-        if (flag) {
-            layout_total_num_comps = strtoul(value, NULL, 10);
-        }
-        ADIOI_Info_get(fd->info, "panfs_layout_parity_stripe_width", MPI_MAX_INFO_VAL,
-                       value, &flag);
-        if (flag) {
-            layout_parity_stripe_width = strtoul(value, NULL, 10);
-        }
-        ADIOI_Info_get(fd->info, "panfs_layout_parity_stripe_depth", MPI_MAX_INFO_VAL,
-                       value, &flag);
-        if (flag) {
-            layout_parity_stripe_depth = strtoul(value, NULL, 10);
-        }
-        ADIOI_Info_get(fd->info, "panfs_layout_max_faults", MPI_MAX_INFO_VAL, value, &flag);
-        if (flag) {
-            layout_max_faults = strtoul(value, NULL, 10);
-        }
-        ADIOI_Info_get(fd->info, "panfs_layout_visit_policy", MPI_MAX_INFO_VAL, value, &flag);
-        if (flag) {
-            layout_visit_policy = strtoul(value, NULL, 10);
-        }
-        ADIOI_Info_get(fd->info, "panfs_layout_encoding", MPI_MAX_INFO_VAL, value, &flag);
-        if (flag) {
-            layout_encoding = strtoul(value, NULL, 10);
-        }
-        ADIOI_Free(value);
-
-        amode = amode | O_CREAT;
-        /* Check for valid set of hints
-         *
-         * Note that RAID0 has been dropped.  In the event PAN_FS_CLIENT_LAYOUT_TYPE__RAID0
-         * enumeraion no longer exists, the following check will still be correct.
-         *
-         * The enumerations looks as follows:
-         *
-         * enum pan_fs_client_layout_agg_type_e {
-         * PAN_FS_CLIENT_LAYOUT_TYPE__INVALID                 = 0,    - *INVALID
-         * PAN_FS_CLIENT_LAYOUT_TYPE__DEFAULT                 = 1,    - VALID
-         * PAN_FS_CLIENT_LAYOUT_TYPE__RAID0                   = 2,    - *INVALID
-         * PAN_FS_CLIENT_LAYOUT_TYPE__RAID1_5_PARITY_STRIPE   = 3,    - VALID
-         * PAN_FS_CLIENT_LAYOUT_TYPE__RAID10                  = 4,    - VALID
-         * PAN_FS_CLIENT_LAYOUT_TYPE__RAIDN_PARITY_STRIPE     = 5     - VALID
-         * };
-         */
-        if (((layout_type < PAN_FS_CLIENT_LAYOUT_TYPE__RAID1_5_PARITY_STRIPE) &&
-             (layout_type != PAN_FS_CLIENT_LAYOUT_TYPE__DEFAULT)) ||
-            (layout_type > PAN_FS_CLIENT_LAYOUT_TYPE__RAIDN_PARITY_STRIPE)) {
-            FPRINTF(stderr, "%s: panfs_layout_type is not a valid value: %u.\n", myname,
-                    layout_type);
-            MPI_Abort(MPI_COMM_WORLD, 1);
-        }
-        if (layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAIDN_PARITY_STRIPE) {
-            if ((layout_stripe_unit == 0) ||
-                (layout_parity_stripe_width == 0) ||
-                (layout_parity_stripe_depth == 0) || (layout_total_num_comps == 0)) {
-                if (layout_stripe_unit == 0) {
-                    FPRINTF(stderr,
-                            "%s: MPI_Info does not contain the panfs_layout_stripe_unit hint which is necessary to specify a valid RAIDN parity stripe layout to the PAN_FS_CLIENT_LAYOUT_CREATE_FILE ioctl.\n",
-                            myname);
-                }
-                if (layout_total_num_comps == 0) {
-                    FPRINTF(stderr,
-                            "%s: MPI_Info does not contain the panfs_layout_total_num_comps hint which is necessary to specify a valid RAIDN parity stripe layout to the PAN_FS_CLIENT_LAYOUT_CREATE_FILE ioctl.\n",
-                            myname);
-                }
-                if (layout_parity_stripe_width == 0) {
-                    FPRINTF(stderr,
-                            "%s: MPI_Info does not contain the panfs_layout_parity_stripe_width hint which is necessary to specify a valid RAIDN parity stripe layout to the PAN_FS_CLIENT_LAYOUT_CREATE_FILE ioctl.\n",
-                            myname);
-                }
-                if (layout_parity_stripe_depth == 0) {
-                    FPRINTF(stderr,
-                            "%s: MPI_Info does not contain the panfs_layout_parity_stripe_depth hint which is necessary to specify a valid RAIDN parity stripe layout to the PAN_FS_CLIENT_LAYOUT_CREATE_FILE ioctl.\n",
-                            myname);
-                }
-                MPI_Abort(MPI_COMM_WORLD, 1);
-            }
-            /* as of 6.0.x release, we only support max_faults == 2 */
-            if (layout_max_faults != 2) {
-                FPRINTF(stderr,
-                        "%s: panfs_layout_max_faults is not a valid value. Setting default of 2\n",
-                        myname);
-                layout_max_faults = 2;
-            }
-            /* as of 6.0.x release, we only support RS enconding */
-            if (layout_encoding != PAN_FS_CLIENT_LAYOUT_RAIDN_ENCODING_RS) {
-                FPRINTF(stderr,
-                        "%s: panfs_layout_encoding is not a valid value: %u. Setting to default of %u\n",
-                        myname, layout_encoding, PAN_FS_CLIENT_LAYOUT_RAIDN_ENCODING_RS);
-                layout_encoding = PAN_FS_CLIENT_LAYOUT_RAIDN_ENCODING_RS;
-            }
-        }
-        if (layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAID1_5_PARITY_STRIPE) {
-            if ((layout_stripe_unit == 0) ||
-                (layout_parity_stripe_width == 0) ||
-                (layout_parity_stripe_depth == 0) || (layout_total_num_comps == 0)) {
-                if (layout_stripe_unit == 0) {
-                    FPRINTF(stderr,
-                            "%s: MPI_Info does not contain the panfs_layout_stripe_unit hint which is necessary to specify a valid RAID5 parity stripe layout to the PAN_FS_CLIENT_LAYOUT_CREATE_FILE ioctl.\n",
-                            myname);
-                }
-                if (layout_total_num_comps == 0) {
-                    FPRINTF(stderr,
-                            "%s: MPI_Info does not contain the panfs_layout_total_num_comps hint which is necessary to specify a valid RAID5 parity stripe layout to the PAN_FS_CLIENT_LAYOUT_CREATE_FILE ioctl.\n",
-                            myname);
-                }
-                if (layout_parity_stripe_width == 0) {
-                    FPRINTF(stderr,
-                            "%s: MPI_Info does not contain the panfs_layout_parity_stripe_width hint which is necessary to specify a valid RAID5 parity stripe layout to the PAN_FS_CLIENT_LAYOUT_CREATE_FILE ioctl.\n",
-                            myname);
-                }
-                if (layout_parity_stripe_depth == 0) {
-                    FPRINTF(stderr,
-                            "%s: MPI_Info does not contain the panfs_layout_parity_stripe_depth hint which is necessary to specify a valid RAID5 parity stripe layout to the PAN_FS_CLIENT_LAYOUT_CREATE_FILE ioctl.\n",
-                            myname);
-                }
-                MPI_Abort(MPI_COMM_WORLD, 1);
-            }
-            if ((layout_visit_policy < PAN_FS_CLIENT_LAYOUT_VISIT__ROUND_ROBIN) ||
-                (layout_visit_policy > PAN_FS_CLIENT_LAYOUT_VISIT__ROUND_ROBIN_WITH_HASHED_OFFSET))
-            {
-                FPRINTF(stderr, "%s: panfs_layout_visit_policy is not a valid value: %u.\n", myname,
-                        layout_visit_policy);
-                MPI_Abort(MPI_COMM_WORLD, 1);
-            }
-        }
-        if (layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAID10) {
-            if ((layout_stripe_unit == 0) || (layout_total_num_comps == 0)) {
-                if (layout_stripe_unit == 0) {
-                    FPRINTF(stderr,
-                            "%s: MPI_Info does not contain the panfs_layout_stripe_unit hint which is necessary to specify a valid RAID10 layout to the PAN_FS_CLIENT_LAYOUT_CREATE_FILE ioctl.\n",
-                            myname);
-                }
-                if (layout_total_num_comps == 0) {
-                    FPRINTF(stderr,
-                            "%s: MPI_Info does not contain the panfs_layout_total_num_comps hint which is necessary to specify a valid RAID10 layout to the PAN_FS_CLIENT_LAYOUT_CREATE_FILE ioctl.\n",
-                            myname);
-                }
-                MPI_Abort(MPI_COMM_WORLD, 1);
-            }
-            if ((layout_visit_policy < PAN_FS_CLIENT_LAYOUT_VISIT__ROUND_ROBIN) ||
-                (layout_visit_policy > PAN_FS_CLIENT_LAYOUT_VISIT__ROUND_ROBIN_WITH_HASHED_OFFSET))
-            {
-                FPRINTF(stderr, "%s: panfs_layout_visit_policy is not a valid value: %u.\n", myname,
-                        layout_visit_policy);
-                MPI_Abort(MPI_COMM_WORLD, 1);
-            }
-        }
-        /* Create the file via ioctl() or open(). ADIOI_PANFS_Open's caller
-         * already optimizes performance by only calling this function with
-         * ADIO_CREATE on rank 0.  Therefore, we don't need to worry about
-         * implementing that optimization here. */
-        if ((layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAID1_5_PARITY_STRIPE) ||
-            (layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAID10) ||
-            (layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAIDN_PARITY_STRIPE)) {
-            pan_fs_client_layout_create_args_t file_create_args;
-            int fd_dir;
-            char *slash;
-            struct stat stat_buf;
-            int err;
-            char *path;
-
-            /* Check that the file does not exist before
-             * trying to create it.  The ioctl itself should
-             * be able to handle this condition.  Currently,
-             * the ioctl will return successfully if the file
-             * has been previously created.  Filed bug 33862
-             * to track the problem.
-             */
-            err = stat(fd->filename, &stat_buf);
-            if ((err == -1) && (errno != ENOENT)) {
-                FPRINTF(stderr, "%s: Unexpected I/O Error calling stat() on PanFS file: %s.\n",
-                        myname, strerror(errno));
-                MPI_Abort(MPI_COMM_WORLD, 1);
-            }
-            else if (err == 0) {
-                /* ensure that we have the same semantics here and in the call to creat(). In the latter, we do not
-                 * use O_EXCL so a create on an existing file should not fail.
-                 */
-                FPRINTF(stderr,
-                        "%s: Cannot create PanFS file with ioctl when file already exists, using open() syscall.\n",
-                        myname);
-                goto use_open_syscall;
-            }
-            else {
-                /* (err == -1) && (errno == ENOENT) */
-                /* File does not exist */
-                path = ADIOI_Strdup(fd->filename);
-                slash = strrchr(path, '/');
-                if (!slash)
-                    ADIOI_Strncpy(path, ".", 2);
-                else {
-                    if (slash == path)
-                        *(path + 1) = '\0';
-                    else
-                        *slash = '\0';
-                }
-
-                /* create PanFS object */
-                memset(&file_create_args, 0, sizeof(pan_fs_client_layout_create_args_t));
-                /* open directory */
-                fd_dir = open(path, O_RDONLY);
-                if (fd_dir < 0) {
-                    FPRINTF(stderr,
-                            "%s: I/O Error opening parent directory to create PanFS file using ioctl: %s.\n",
-                            myname, strerror(errno));
-                    MPI_Abort(MPI_COMM_WORLD, 1);
-                }
-                else {
-                    char *file_name_ptr = fd->filename;
-                    slash = strrchr(fd->filename, '/');
-                    if (slash) {
-                        file_name_ptr = slash + 1;
-                    }
-                    /* create file in the directory */
-                    file_create_args.mode = perm;
-                    file_create_args.version = PAN_FS_CLIENT_LAYOUT_VERSION;
-                    file_create_args.flags = PAN_FS_CLIENT_LAYOUT_CREATE_F__NONE;
-                    ADIOI_Strncpy(file_create_args.filename, file_name_ptr,
-                                  strlen(fd->filename) + 1);
-                    file_create_args.layout.agg_type = layout_type;
-                    file_create_args.layout.layout_is_valid = 1;
-                    if (layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAIDN_PARITY_STRIPE) {
-                        file_create_args.layout.u.raidn_parity_stripe.total_num_comps =
-                            layout_total_num_comps;
-                        file_create_args.layout.u.raidn_parity_stripe.parity_stripe_width =
-                            layout_parity_stripe_width;
-                        file_create_args.layout.u.raidn_parity_stripe.parity_stripe_depth =
-                            layout_parity_stripe_depth;
-                        file_create_args.layout.u.raidn_parity_stripe.stripe_unit =
-                            layout_stripe_unit;
-                        file_create_args.layout.u.raidn_parity_stripe.max_faults =
-                            layout_max_faults;
-                        file_create_args.layout.u.raidn_parity_stripe.encoding = layout_encoding;
-                    }
-                    else if (layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAID1_5_PARITY_STRIPE) {
-                        file_create_args.layout.u.raid1_5_parity_stripe.total_num_comps =
-                            layout_total_num_comps;
-                        file_create_args.layout.u.raid1_5_parity_stripe.parity_stripe_width =
-                            layout_parity_stripe_width;
-                        file_create_args.layout.u.raid1_5_parity_stripe.parity_stripe_depth =
-                            layout_parity_stripe_depth;
-                        file_create_args.layout.u.raid1_5_parity_stripe.stripe_unit =
-                            layout_stripe_unit;
-                        file_create_args.layout.u.raid1_5_parity_stripe.layout_visit_policy =
-                            layout_visit_policy;
-                    }
-                    else if (layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAID10) {
-                        file_create_args.layout.u.raid10.total_num_comps = layout_total_num_comps;
-                        file_create_args.layout.u.raid10.stripe_unit = layout_stripe_unit;
-                        file_create_args.layout.u.raid10.layout_visit_policy = layout_visit_policy;
-                    }
-                    err = ioctl(fd_dir, PAN_FS_CLIENT_LAYOUT_CREATE_FILE, &file_create_args);
-                    if (err < 0) {
-                        FPRINTF(stderr,
-                                "%s: I/O Error doing ioctl on parent directory to create PanFS file using ioctl: %s.\n",
-                                myname, strerror(errno));
-                        MPI_Abort(MPI_COMM_WORLD, 1);
-                    }
-                    err = close(fd_dir);
-                }
-                ADIOI_Free(path);
-            }
-        }
-        else {
-          use_open_syscall:;
-            int create_fd = open(fd->filename, amode, perm);
-            if (create_fd != -1) {
-                close(create_fd);
-            }
-            else {
-                FPRINTF(stderr, "%s: I/O Error creating PanFS file using open: %s.\n", myname,
-                        strerror(errno));
-                MPI_Abort(MPI_COMM_WORLD, 1);
-            }
-        }
-    }
-    if (fd->access_mode & ADIO_RDONLY)
-        amode = amode | O_RDONLY;
-    if (fd->access_mode & ADIO_WRONLY)
-        amode = amode | O_WRONLY;
-    if (fd->access_mode & ADIO_RDWR)
-        amode = amode | O_RDWR;
-    if (fd->access_mode & ADIO_EXCL)
-        amode = amode | O_EXCL;
-
-    value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL + 1) * sizeof(char));
-    ADIOI_Info_get(fd->info, "panfs_concurrent_write", MPI_MAX_INFO_VAL, value, &flag);
-    if (flag) {
-        unsigned long int concurrent_write = strtoul(value, NULL, 10);
-        if (concurrent_write == 1) {
-            amode = amode | O_CONCURRENT_WRITE;
-        }
-    }
-    ADIOI_Free(value);
-
-    fd->fd_sys = open(fd->filename, amode, perm);
-    fd->fd_direct = -1;
-
-    if (fd->fd_sys != -1) {
-        int rc;
-        char temp_buffer[TEMP_BUFFER_SIZE];
-        pan_fs_client_layout_query_args_t file_query_args;
-        memset(&file_query_args, 0, sizeof(pan_fs_client_layout_query_args_t));
-        file_query_args.version = PAN_FS_CLIENT_LAYOUT_VERSION;
-        rc = ioctl(fd->fd_sys, PAN_FS_CLIENT_LAYOUT_QUERY_FILE, &file_query_args);
-        if (rc < 0) {
-            /* Error - set layout type to unknown */
-            ADIOI_Info_set(fd->info, "panfs_layout_type", "PAN_FS_CLIENT_LAYOUT_TYPE__INVALID");
-        }
-        else {
-            ADIOI_Snprintf(temp_buffer, TEMP_BUFFER_SIZE, "%u", file_query_args.layout.agg_type);
-            ADIOI_Info_set(fd->info, "panfs_layout_type", temp_buffer);
-            if (file_query_args.layout.layout_is_valid == 1) {
-                switch (file_query_args.layout.agg_type) {
-                case PAN_FS_CLIENT_LAYOUT_TYPE__RAIDN_PARITY_STRIPE:
-                    ADIOI_Snprintf(temp_buffer, TEMP_BUFFER_SIZE, "%u",
-                                   file_query_args.layout.u.raidn_parity_stripe.stripe_unit);
-                    ADIOI_Info_set(fd->info, "panfs_layout_stripe_unit", temp_buffer);
-                    ADIOI_Snprintf(temp_buffer, TEMP_BUFFER_SIZE, "%u",
-                                   file_query_args.layout.u.raidn_parity_stripe.
-                                   parity_stripe_width);
-                    ADIOI_Info_set(fd->info, "panfs_layout_parity_stripe_width", temp_buffer);
-                    ADIOI_Snprintf(temp_buffer, TEMP_BUFFER_SIZE, "%u",
-                                   file_query_args.layout.u.raidn_parity_stripe.
-                                   parity_stripe_depth);
-                    ADIOI_Info_set(fd->info, "panfs_layout_parity_stripe_depth", temp_buffer);
-                    ADIOI_Snprintf(temp_buffer, TEMP_BUFFER_SIZE, "%u",
-                                   file_query_args.layout.u.raidn_parity_stripe.total_num_comps);
-                    ADIOI_Info_set(fd->info, "panfs_layout_total_num_comps", temp_buffer);
-                    ADIOI_Snprintf(temp_buffer, TEMP_BUFFER_SIZE, "%u",
-                                   file_query_args.layout.u.raidn_parity_stripe.max_faults);
-                    ADIOI_Info_set(fd->info, "panfs_layout_max_faults", temp_buffer);
-                    ADIOI_Snprintf(temp_buffer, TEMP_BUFFER_SIZE, "%u",
-                                   file_query_args.layout.u.raidn_parity_stripe.encoding);
-                    ADIOI_Info_set(fd->info, "panfs_layout_encoding", temp_buffer);
-                    break;
-                case PAN_FS_CLIENT_LAYOUT_TYPE__RAID1_5_PARITY_STRIPE:
-                    ADIOI_Snprintf(temp_buffer, TEMP_BUFFER_SIZE, "%u",
-                                   file_query_args.layout.u.raid1_5_parity_stripe.stripe_unit);
-                    ADIOI_Info_set(fd->info, "panfs_layout_stripe_unit", temp_buffer);
-                    ADIOI_Snprintf(temp_buffer, TEMP_BUFFER_SIZE, "%u",
-                                   file_query_args.layout.u.raid1_5_parity_stripe.
-                                   parity_stripe_width);
-                    ADIOI_Info_set(fd->info, "panfs_layout_parity_stripe_width", temp_buffer);
-                    ADIOI_Snprintf(temp_buffer, TEMP_BUFFER_SIZE, "%u",
-                                   file_query_args.layout.u.raid1_5_parity_stripe.
-                                   parity_stripe_depth);
-                    ADIOI_Info_set(fd->info, "panfs_layout_parity_stripe_depth", temp_buffer);
-                    ADIOI_Snprintf(temp_buffer, TEMP_BUFFER_SIZE, "%u",
-                                   file_query_args.layout.u.raid1_5_parity_stripe.total_num_comps);
-                    ADIOI_Info_set(fd->info, "panfs_layout_total_num_comps", temp_buffer);
-                    ADIOI_Snprintf(temp_buffer, TEMP_BUFFER_SIZE, "%u",
-                                   file_query_args.layout.u.raid1_5_parity_stripe.
-                                   layout_visit_policy);
-                    ADIOI_Info_set(fd->info, "panfs_layout_visit_policy", temp_buffer);
-                    break;
-                case PAN_FS_CLIENT_LAYOUT_TYPE__RAID10:
-                    ADIOI_Snprintf(temp_buffer, TEMP_BUFFER_SIZE, "%u",
-                                   file_query_args.layout.u.raid10.stripe_unit);
-                    ADIOI_Info_set(fd->info, "panfs_layout_stripe_unit", temp_buffer);
-                    ADIOI_Snprintf(temp_buffer, TEMP_BUFFER_SIZE, "%u",
-                                   file_query_args.layout.u.raid10.total_num_comps);
-                    ADIOI_Info_set(fd->info, "panfs_layout_total_num_comps", temp_buffer);
-                    ADIOI_Snprintf(temp_buffer, TEMP_BUFFER_SIZE, "%u",
-                                   file_query_args.layout.u.raid10.layout_visit_policy);
-                    ADIOI_Info_set(fd->info, "panfs_layout_visit_policy", temp_buffer);
-                    break;
-                case PAN_FS_CLIENT_LAYOUT_TYPE__INVALID:
-                case PAN_FS_CLIENT_LAYOUT_TYPE__DEFAULT:
-                    MPI_Info_set(fd->info, "panfs_layout_type",
-                                 "PAN_FS_CLIENT_LAYOUT_TYPE__INVALID");
-                default:
-                    break;
-                }
-            }
-        }
-    }
-
-    if ((fd->fd_sys != -1) && (fd->access_mode & ADIO_APPEND))
-        fd->fp_ind = fd->fp_sys_posn = lseek(fd->fd_sys, 0, SEEK_END);
-
-    if (fd->fd_sys == -1) {
-        *error_code = ADIOI_Err_create_code(myname, fd->filename, errno);
-    }
-    else
-        *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_panfs/ad_panfs_read.c b/3rd-party/romio321/adio/ad_panfs/ad_panfs_read.c
deleted file mode 100644
index 237e4929de3..00000000000
--- a/3rd-party/romio321/adio/ad_panfs/ad_panfs_read.c
+++ /dev/null
@@ -1,68 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_panfs.h"
-
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-
-void ADIOI_PANFS_ReadContig(ADIO_File fd, void *buf, int count, 
-			  MPI_Datatype datatype, int file_ptr_type,
-			  ADIO_Offset offset, ADIO_Status *status,
-			  int *error_code)
-{
-    MPI_Count err = -1, datatype_size, len;
-    static char myname[] = "ADIOI_PANFS_READCONTIG";
-
-    MPI_Type_size_x(datatype, &datatype_size);
-    len = datatype_size * count;
-
-    if (file_ptr_type == ADIO_INDIVIDUAL) {
-	offset = fd->fp_ind;
-    }
-
-    if (fd->fp_sys_posn != offset) {
-	err = lseek(fd->fd_sys, offset, SEEK_SET);
-	/* --BEGIN ERROR HANDLING-- */
-	if (err == -1) {
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE,
-					       myname, __LINE__,
-					       MPI_ERR_IO, "**io",
-					       "**io %s", strerror(errno));
-	    fd->fp_sys_posn = -1;
-	    return;
-	}
-	/* --END ERROR HANDLING-- */
-    }
-
-    AD_PANFS_RETRY(read(fd->fd_sys, buf, len),err)
-    /* --BEGIN ERROR HANDLING-- */
-    if (err == -1) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   MPI_ERR_IO, "**io",
-					   "**io %s", strerror(errno));
-	fd->fp_sys_posn = -1;
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-    fd->fp_sys_posn = offset + err;
-
-    if (file_ptr_type == ADIO_INDIVIDUAL) {
-	fd->fp_ind += err; 
-    }
-
-#ifdef HAVE_STATUS_SET_BYTES
-    if (err != -1) MPIR_Status_set_bytes(status, datatype, err);
-#endif
-
-    *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_panfs/ad_panfs_resize.c b/3rd-party/romio321/adio/ad_panfs/ad_panfs_resize.c
deleted file mode 100644
index 5c41126c4de..00000000000
--- a/3rd-party/romio321/adio/ad_panfs/ad_panfs_resize.c
+++ /dev/null
@@ -1,49 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2004 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_panfs.h"
-
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-
-void ADIOI_PANFS_Resize(ADIO_File fd, ADIO_Offset size, int *error_code)
-{
-    int err;
-    int myrank;
-    struct stat stat_buf;
-    static char myname[] = "ADIOI_PANFS_RESIZE";
-
-    MPI_Comm_rank(fd->comm, &myrank);
-    if (!myrank)
-    {
-        AD_PANFS_RETRY(ftruncate(fd->fd_sys,size),err);
-        MPI_Barrier(fd->comm);
-    }
-    else
-    {
-        MPI_Barrier(fd->comm);
-        AD_PANFS_RETRY(fstat(fd->fd_sys,&stat_buf),err);
-        if(((ADIO_Offset)stat_buf.st_size) != size)
-        {
-            /* This should never happen otherwise there is a coherency problem. */
-            FPRINTF(stderr, "%s: Rank %d: Resize failed: requested=%llu actual=%llu.\n",myname,myrank,size,(unsigned long long)stat_buf.st_size);
-            MPI_Abort(MPI_COMM_WORLD, 1);
-        }
-    }
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (err == -1) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io", "**io %s", strerror(errno));
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-    *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_panfs/ad_panfs_write.c b/3rd-party/romio321/adio/ad_panfs/ad_panfs_write.c
deleted file mode 100644
index 920d2f473e7..00000000000
--- a/3rd-party/romio321/adio/ad_panfs/ad_panfs_write.c
+++ /dev/null
@@ -1,68 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2004 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_panfs.h"
-
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-
-void ADIOI_PANFS_WriteContig(ADIO_File fd, const void *buf, int count,
-			   MPI_Datatype datatype, int file_ptr_type,
-			   ADIO_Offset offset, ADIO_Status *status,
-			   int *error_code)
-{
-    MPI_Count err = -1, datatype_size, len;
-    static char myname[] = "ADIOI_PANFS_WRITECONTIG";
-
-    MPI_Type_size_x(datatype, &datatype_size);
-    len = datatype_size * count;
-
-    if (file_ptr_type == ADIO_INDIVIDUAL) {
-	offset = fd->fp_ind;
-    }
-
-    if (fd->fp_sys_posn != offset) {
-	err = lseek(fd->fd_sys, offset, SEEK_SET);
-	/* --BEGIN ERROR HANDLING-- */
-	if (err == -1) {
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE,
-					       myname, __LINE__,
-					       MPI_ERR_IO, "**io",
-					       "**io %s", strerror(errno));
-	    fd->fp_sys_posn = -1;
-	    return;
-	}
-	/* --END ERROR HANDLING-- */
-    }
-    
-    AD_PANFS_RETRY(write(fd->fd_sys, buf, len),err)
-    /* --BEGIN ERROR HANDLING-- */
-    if (err == -1) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   MPI_ERR_IO, "**io",
-					   "**io %s", strerror(errno));
-	fd->fp_sys_posn = -1;
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-    fd->fp_sys_posn = offset + err;
-
-    if (file_ptr_type == ADIO_INDIVIDUAL) {
-	fd->fp_ind += err; 
-    }
-
-#ifdef HAVE_STATUS_SET_BYTES
-    if (err != -1 && status) MPIR_Status_set_bytes(status, datatype, err);
-#endif
-
-    *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_pfs/Makefile.mk b/3rd-party/romio321/adio/ad_pfs/Makefile.mk
deleted file mode 100644
index 3521c07c863..00000000000
--- a/3rd-party/romio321/adio/ad_pfs/Makefile.mk
+++ /dev/null
@@ -1,26 +0,0 @@
-## -*- Mode: Makefile; -*-
-## vim: set ft=automake :
-##
-## (C) 2011 by Argonne National Laboratory.
-##     See COPYRIGHT in top-level directory.
-##
-
-if BUILD_AD_PFS
-
-noinst_HEADERS += adio/ad_pfs/ad_pfs.h
-
-romio_other_sources +=          \
-    adio/ad_pfs/ad_pfs_read.c   \
-    adio/ad_pfs/ad_pfs_open.c   \
-    adio/ad_pfs/ad_pfs_write.c  \
-    adio/ad_pfs/ad_pfs_done.c   \
-    adio/ad_pfs/ad_pfs_fcntl.c  \
-    adio/ad_pfs/ad_pfs_iread.c  \
-    adio/ad_pfs/ad_pfs_iwrite.c \
-    adio/ad_pfs/ad_pfs_wait.c   \
-    adio/ad_pfs/ad_pfs_flush.c  \
-    adio/ad_pfs/ad_pfs_hints.c  \
-    adio/ad_pfs/ad_pfs.c
-
-endif BUILD_AD_PFS
-
diff --git a/3rd-party/romio321/adio/ad_pfs/ad_pfs.c b/3rd-party/romio321/adio/ad_pfs/ad_pfs.c
deleted file mode 100644
index cc480b01114..00000000000
--- a/3rd-party/romio321/adio/ad_pfs/ad_pfs.c
+++ /dev/null
@@ -1,38 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_pfs.h"
-
-/* adioi.h has the ADIOI_Fns_struct define */
-#include "adioi.h"
-
-struct ADIOI_Fns_struct ADIO_PFS_operations = {
-    ADIOI_PFS_Open, /* Open */
-    ADIOI_PFS_ReadContig, /* ReadContig */
-    ADIOI_PFS_WriteContig, /* WriteContig */
-    ADIOI_GEN_ReadStridedColl, /* ReadStridedColl */
-    ADIOI_GEN_WriteStridedColl, /* WriteStridedColl */
-    ADIOI_GEN_SeekIndividual, /* SeekIndividual */
-    ADIOI_PFS_Fcntl, /* Fcntl */
-    ADIOI_PFS_SetInfo, /* SetInfo */
-    ADIOI_GEN_ReadStrided, /* ReadStrided */
-    ADIOI_GEN_WriteStrided, /* WriteStrided */
-    ADIOI_GEN_Close, /* Close */
-    ADIOI_PFS_IreadContig, /* IreadContig */
-    ADIOI_PFS_IwriteContig, /* IwriteContig */
-    ADIOI_PFS_ReadDone, /* ReadDone */
-    ADIOI_PFS_WriteDone, /* WriteDone */
-    ADIOI_PFS_ReadComplete, /* ReadComplete */
-    ADIOI_PFS_WriteComplete, /* WriteComplete */
-    ADIOI_FAKE_IreadStrided, /* IreadStrided */
-    ADIOI_FAKE_IwriteStrided, /* IwriteStrided */
-    ADIOI_PFS_Flush, /* Flush */
-    ADIOI_GEN_Resize, /* Resize */
-    ADIOI_GEN_Delete, /* Delete */
-    ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
-    ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
-};
diff --git a/3rd-party/romio321/adio/ad_pfs/ad_pfs.h b/3rd-party/romio321/adio/ad_pfs/ad_pfs.h
deleted file mode 100644
index fbe055ccf75..00000000000
--- a/3rd-party/romio321/adio/ad_pfs/ad_pfs.h
+++ /dev/null
@@ -1,62 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-/* contains definitions, declarations, and macros specific to the
-   implementation of ADIO on PFS */
-
-#ifndef AD_PFS_INCLUDE
-#define AD_PFS_INCLUDE
-
-#include <unistd.h>
-#include <sys/types.h>
-#include <fcntl.h>
-#include <nx.h>
-#include <sys/uio.h>
-#include "adio.h"
-
-#ifdef tflops
-#define lseek eseek
-#define _gopen(n,m,i,p) open(n,m,p)
-#endif
-
-/* PFS file-pointer modes (removed most of them because they are unused) */
-#ifndef M_ASYNC 
-#define M_UNIX                    0
-#define M_ASYNC                   5
-#endif
-
-void ADIOI_PFS_Open(ADIO_File fd, int *error_code);
-void ADIOI_PFS_ReadContig(ADIO_File fd, void *buf, int count, 
-                      MPI_Datatype datatype, int file_ptr_type,
-                     ADIO_Offset offset, ADIO_Status *status, int
-		     *error_code);
-void ADIOI_PFS_WriteContig(ADIO_File fd, void *buf, int count, 
-                      MPI_Datatype datatype, int file_ptr_type,
-                      ADIO_Offset offset, ADIO_Status *status, int
-		      *error_code);   
-void ADIOI_PFS_IwriteContig(ADIO_File fd, void *buf, int count, 
-                      MPI_Datatype datatype, int file_ptr_type,
-                      ADIO_Offset offset, ADIO_Request *request, int
-		      *error_code);   
-void ADIOI_PFS_IreadContig(ADIO_File fd, void *buf, int count, 
-                      MPI_Datatype datatype, int file_ptr_type,
-                      ADIO_Offset offset, ADIO_Request *request, int
-		      *error_code);   
-int ADIOI_PFS_ReadDone(ADIO_Request *request, ADIO_Status *status, int
-		       *error_code);
-int ADIOI_PFS_WriteDone(ADIO_Request *request, ADIO_Status *status, int
-		       *error_code);
-void ADIOI_PFS_ReadComplete(ADIO_Request *request, ADIO_Status *status, int
-		       *error_code); 
-void ADIOI_PFS_WriteComplete(ADIO_Request *request, ADIO_Status *status,
-			int *error_code); 
-void ADIOI_PFS_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct, int
-		*error_code); 
-void ADIOI_PFS_Flush(ADIO_File fd, int *error_code);
-void ADIOI_PFS_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code);
-
-#endif
diff --git a/3rd-party/romio321/adio/ad_pfs/ad_pfs_done.c b/3rd-party/romio321/adio/ad_pfs/ad_pfs_done.c
deleted file mode 100644
index 60e2d7da298..00000000000
--- a/3rd-party/romio321/adio/ad_pfs/ad_pfs_done.c
+++ /dev/null
@@ -1,57 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_pfs.h"
-
-int ADIOI_PFS_ReadDone(ADIO_Request *request, ADIO_Status *status,
-		       int *error_code)  
-{
-    int done=0;
-    static char myname[] = "ADIOI_PFS_READDONE";
-
-    if (*request == ADIO_REQUEST_NULL) {
-        *error_code = MPI_SUCCESS;
-        return 1;
-    }
-
-    if ((*request)->queued)
-	done = _iodone(*((long *) (*request)->handle));
-    else done = 1; /* ADIOI_Complete_Async completed this request, 
-                      but request object was not freed. */
-
-#ifdef HAVE_STATUS_SET_BYTES
-    if ((done >= 0) && ((*request)->nbytes != -1))
-	MPIR_Status_set_bytes(status, (*request)->datatype, (*request)->nbytes);
-#endif
-
-    if (done >= 0) {
-        /* if request is still queued in the system, it is also there
-           on ADIOI_Async_list. Delete it from there. */
-        if ((*request)->queued) ADIOI_Del_req_from_list(request);
-
-        (*request)->fd->async_count--;
-        if ((*request)->handle) ADIOI_Free((*request)->handle);
-        ADIOI_Free_request((ADIOI_Req_node *) (*request));
-        *request = ADIO_REQUEST_NULL;
-    }
-    
-    if (done == -1 && errno != 0) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io",
-					   "**io %s", strerror(errno));
-    }
-    else *error_code = MPI_SUCCESS;
-    return done;
-}
-
-
-int ADIOI_PFS_WriteDone(ADIO_Request *request, ADIO_Status *status,
-			int *error_code)
-{
-    return ADIOI_PFS_ReadDone(request, status, error_code);
-} 
diff --git a/3rd-party/romio321/adio/ad_pfs/ad_pfs_fcntl.c b/3rd-party/romio321/adio/ad_pfs/ad_pfs_fcntl.c
deleted file mode 100644
index 4a2c0fd3827..00000000000
--- a/3rd-party/romio321/adio/ad_pfs/ad_pfs_fcntl.c
+++ /dev/null
@@ -1,81 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_pfs.h"
-#include "adio_extern.h"
-
-void ADIOI_PFS_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct,
-		     int *error_code)
-{
-    int i, err;
-    int iomod, np_total, np_comm;
-    static char myname[] = "ADIOI_PFS_FCNTL";
-
-    switch(flag) {
-    case ADIO_FCNTL_GET_FSIZE:
-	if (!(fd->atomicity)) {
-          /* in M_ASYNC mode, all processes are not aware of changes 
-             in file size (although the manual says otherwise). Therefore, 
-             temporarily change to M_UNIX and then change 
-             back to M_ASYNC.*/ 
-	    MPI_Comm_size(MPI_COMM_WORLD, &np_total);
-	    MPI_Comm_size(fd->comm, &np_comm);
-	    if (np_total == np_comm) {
-		err = _setiomode(fd->fd_sys, M_UNIX);
-		err = _setiomode(fd->fd_sys, M_ASYNC);
-	    }
-            /* else it is M_UNIX anyway, so no problem */
-	}
-	fcntl_struct->fsize = lseek(fd->fd_sys, 0, SEEK_END);
-	if (fd->fp_sys_posn != -1) 
-	    lseek(fd->fd_sys, fd->fp_sys_posn, SEEK_SET);
-	*error_code = MPI_SUCCESS;
-	break;
-
-    case ADIO_FCNTL_SET_DISKSPACE:
-	err = _lsize(fd->fd_sys, fcntl_struct->diskspace, SEEK_SET);
-	if (err == -1) {
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE, myname,
-					       __LINE__, MPI_ERR_IO, "**io",
-					       "**io %s", strerror(errno));
-	}
-	else *error_code = MPI_SUCCESS;
-	break;
-
-    case ADIO_FCNTL_SET_ATOMICITY:
-	MPI_Comm_size(MPI_COMM_WORLD, &np_total);
-	MPI_Comm_size(fd->comm, &np_comm);
-	if (np_total == np_comm) {
-	    iomod = (fcntl_struct->atomicity == 0) ? M_ASYNC : M_UNIX;
-	    err = _setiomode(fd->fd_sys, iomod);
-	}
-        /* else can't do anything because setiomode is global. but
-           the file will have been opened with M_UNIX anyway, because
-           gopen is also global. */
-
-	fd->atomicity = (fcntl_struct->atomicity == 0) ? 0 : 1;
-	if (err == -1) {
-	    /* --BEGIN ERROR HANDLING-- */
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE, myname,
-					       __LINE__, MPI_ERR_IO, "**io",
-					       "**io %s", strerror(errno));
-	    /* --END ERROR HANDLING-- */
-	}
-	else *error_code = MPI_SUCCESS;
-	break;
-
-    default:
-	/* --BEGIN ERROR HANDLING-- */
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPI_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_ARG,
-					   "**flag", "**flag %d", flag);
-	return;
-	/* --END ERROR HANDLING-- */
-    }
-}
diff --git a/3rd-party/romio321/adio/ad_pfs/ad_pfs_flush.c b/3rd-party/romio321/adio/ad_pfs/ad_pfs_flush.c
deleted file mode 100644
index 98dedc099c3..00000000000
--- a/3rd-party/romio321/adio/ad_pfs/ad_pfs_flush.c
+++ /dev/null
@@ -1,38 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_pfs.h"
-
-void ADIOI_PFS_Flush(ADIO_File fd, int *error_code)
-{
-    int err, np_total, np_comm;
-    static char myname[] = "ADIOI_PFS_FLUSH";
-
-/* fsync is not actually needed in PFS, because it uses something
-   called fast-path I/O. However, it doesn't do any harm either. */
-    err = fsync(fd->fd_sys);
-    if (err == -1) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io",
-					   "**io %s", strerror(errno));
-    }
-    else *error_code = MPI_SUCCESS;
-
-/* MPI-IO requires that after an fsync all processes must see the same
-   file size. In PFS M_ASYNC mode, this doesn't automatically happen.
-   Therefore, if M_ASYNC mode, temporarily change it to M_UNIX mode
-   and then switch back to M_ASYNC. That updates the file size! */
-
-    MPI_Comm_size(MPI_COMM_WORLD, &np_total);
-    MPI_Comm_size(fd->comm, &np_comm);
-    if ((np_total == np_comm) && (!(fd->atomicity))) {
-	err = _setiomode(fd->fd_sys, M_UNIX);
-	err = _setiomode(fd->fd_sys, M_ASYNC);
-    }
-    /* else it is M_UNIX anyway. don't do anything. */
-}
diff --git a/3rd-party/romio321/adio/ad_pfs/ad_pfs_hints.c b/3rd-party/romio321/adio/ad_pfs/ad_pfs_hints.c
deleted file mode 100644
index 407a0eb7758..00000000000
--- a/3rd-party/romio321/adio/ad_pfs/ad_pfs_hints.c
+++ /dev/null
@@ -1,174 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_pfs.h"
-
-void ADIOI_PFS_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code)
-{
-    char *value, *value_in_fd;
-    int flag, tmp_val, str_factor=-1, str_unit=-1, start_iodev=-1;
-    struct sattr attr;
-    int err, myrank, fd_sys, perm, amode, old_mask;
-
-    if ( (fd->info) == MPI_INFO_NULL) {
-	/* This must be part of the open call. can set striping parameters 
-           if necessary. */ 
-	MPI_Info_create(&(fd->info));
-	
-	/* has user specified striping or server buffering parameters 
-           and do they have the same value on all processes? */
-	if (users_info != MPI_INFO_NULL) {
-	    value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
-
-	    ADIOI_Info_get(users_info, "striping_factor", MPI_MAX_INFO_VAL, 
-			 value, &flag);
-	    if (flag) {
-		str_factor=atoi(value);
-		tmp_val = str_factor;
-		MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
-		/* --BEGIN ERROR HANDLING-- */
-		if (tmp_val != str_factor) {
-		    MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
-						       "striping_factor",
-						       error_code);
-		    return;
-		}
-		/* --END ERROR HANDLING-- */
-	    }
-
-	    ADIOI_Info_get(users_info, "striping_unit", MPI_MAX_INFO_VAL, 
-			 value, &flag);
-	    if (flag) {
-		str_unit=atoi(value);
-		tmp_val = str_unit;
-		MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
-		/* --BEGIN ERROR HANDLING-- */
-		if (tmp_val != str_unit) {
-		    MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
-						       "striping_unit",
-						       error_code);
-		    return;
-		}
-		/* --END ERROR HANDLING-- */
-	    }
-
-	    ADIOI_Info_get(users_info, "start_iodevice", MPI_MAX_INFO_VAL, 
-			 value, &flag);
-	    if (flag) {
-		start_iodev=atoi(value);
-		tmp_val = start_iodev;
-		MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
-		/* --BEGIN ERROR HANDLING-- */
-		if (tmp_val != start_iodev) {
-		    MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
-						       "start_iodevice",
-						       error_code);
-		    return;
-		}
-		/* --END ERROR HANDLING-- */
-	    }
-
-         /* if user has specified striping info, process 0 tries to set it */
-	    if ((str_factor > 0) || (str_unit > 0) || (start_iodev >= 0)) {
-		MPI_Comm_rank(fd->comm, &myrank);
-		if (!myrank) {
-		    if (fd->perm == ADIO_PERM_NULL) {
-			old_mask = umask(022);
-			umask(old_mask);
-			perm = old_mask ^ 0666;
-		    }
-		    else perm = fd->perm;
-
-		    amode = 0;
-		    if (fd->access_mode & ADIO_CREATE)
-			amode = amode | O_CREAT;
-		    if (fd->access_mode & ADIO_RDONLY)
-			amode = amode | O_RDONLY;
-		    if (fd->access_mode & ADIO_WRONLY)
-			amode = amode | O_WRONLY;
-		    if (fd->access_mode & ADIO_RDWR)
-			amode = amode | O_RDWR;
-		    if (fd->access_mode & ADIO_EXCL)
-			amode = amode | O_EXCL;
-
-		    fd_sys = open(fd->filename, amode, perm);
-		    err = fcntl(fd_sys, F_GETSATTR, &attr);
-
-		    if (!err) {
-			if (str_unit > 0) attr.s_sunitsize = str_unit;
-			if ((start_iodev >= 0) && 
-			    (start_iodev < attr.s_sfactor))
-			    attr.s_start_sdir = start_iodev;
-			if ((str_factor > 0) && (str_factor < attr.s_sfactor))
-			    attr.s_sfactor = str_factor;
-
-			err = fcntl(fd_sys, F_SETSATTR, &attr);
-		    }
-
-		    close(fd_sys);
-		}
-
-		MPI_Barrier(fd->comm);
-	    }
-
-	    /* Has user asked for pfs server buffering to be turned on?
-	       If so, mark it as true in fd->info and turn it on in 
-	       ADIOI_PFS_Open after the file is opened */
-
-	    ADIOI_Info_get(users_info, "pfs_svr_buf", MPI_MAX_INFO_VAL, 
-			 value, &flag);
-	    if (flag && (!strcmp(value, "true")))
-		ADIOI_Info_set(fd->info, "pfs_svr_buf", "true");
-	    else ADIOI_Info_set(fd->info, "pfs_svr_buf", "false");
-
-	    ADIOI_Free(value);
-	}
-	else ADIOI_Info_set(fd->info, "pfs_svr_buf", "false");
-	
-	/* set the values for collective I/O and data sieving parameters */
-	ADIOI_GEN_SetInfo(fd, users_info, error_code);
-    }
-    
-    else {
-	/* The file has been opened previously and fd->fd_sys is a valid
-           file descriptor. cannot set striping parameters now. */
-	
-	/* set the values for collective I/O and data sieving parameters */
-	ADIOI_GEN_SetInfo(fd, users_info, error_code);
-
-	/* has user specified value for pfs_svr_buf? */
-	if (users_info != MPI_INFO_NULL) {
-	    value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
-
-	    ADIOI_Info_get(users_info, "pfs_svr_buf", MPI_MAX_INFO_VAL, 
-			 value, &flag);
-	    if (flag && (!strcmp(value, "true") || !strcmp(value, "false"))) {
-		value_in_fd = (char *) 
-                          ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
-		ADIOI_Info_get(fd->info, "pfs_svr_buf", MPI_MAX_INFO_VAL, 
-			 value_in_fd, &flag);
-		if (strcmp(value, value_in_fd)) {
-		    if (!strcmp(value, "true")) {
-			err = fcntl(fd->fd_sys, F_PFS_SVR_BUF, TRUE);
-			if (!err) 
-			    ADIOI_Info_set(fd->info, "pfs_svr_buf", "true");
-		    }
-		    else {
-			err = fcntl(fd->fd_sys, F_PFS_SVR_BUF, FALSE);
-			if (!err) 
-			    ADIOI_Info_set(fd->info, "pfs_svr_buf", "false");
-		    }
-		}
-		ADIOI_Free(value_in_fd);
-	    }
-	    ADIOI_Free(value);
-	}
-
-    }
-    
-    *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_pfs/ad_pfs_iread.c b/3rd-party/romio321/adio/ad_pfs/ad_pfs_iread.c
deleted file mode 100644
index 78b3c592fb7..00000000000
--- a/3rd-party/romio321/adio/ad_pfs/ad_pfs_iread.c
+++ /dev/null
@@ -1,81 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_pfs.h"
-
-void ADIOI_PFS_IreadContig(ADIO_File fd, void *buf, int count, 
-			   MPI_Datatype datatype, int file_ptr_type,
-			   ADIO_Offset offset, ADIO_Request *request,
-			   int *error_code)
-{
-    long *id_sys;
-    int err=-1;
-    MPI_Count len, typesize;
-    ADIO_Offset off;
-    static char myname[] = "ADIOI_PFS_IREADCONTIG";
-
-    *request = ADIOI_Malloc_request();
-    (*request)->optype = ADIOI_READ;
-    (*request)->fd = fd;
-    (*request)->datatype = datatype;
-
-    MPI_Type_size_x(datatype, &typesize);
-    len = count * typesize;
-
-    id_sys = (long *) ADIOI_Malloc(sizeof(long));
-    (*request)->handle = (void *) id_sys;
-
-    off = (file_ptr_type == ADIO_INDIVIDUAL) ? fd->fp_ind : offset;
-
-    lseek(fd->fd_sys, off, SEEK_SET);
-    *id_sys = _iread(fd->fd_sys, buf, len);
-
-    if ((*id_sys == -1) && (errno == EQNOMID)) {
-     /* the man pages say EMREQUEST, but in reality errno is set to EQNOMID! */
-
-        /* exceeded the max. no. of outstanding requests. */
-
-        /* complete all previous async. requests */
-        /*ADIOI_Complete_async(error_code); */
-	if (*error_code != MPI_SUCCESS) return;
-
-        /* try again */
-        *id_sys = _iread(fd->fd_sys, buf, len);
-
-        if ((*id_sys == -1) && (errno == EQNOMID)) {
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE, myname,
-					       __LINE__, MPI_ERR_IO, "**io",
-					       "**io %s", strerror(errno));
-	    return;
-        }
-    }
-    else if (*id_sys == -1) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io",
-					   "**io %s", strerror(errno));
-	return;
-    }
-
-    if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind += len; 
-
-    (*request)->queued = 1;
-    (*request)->nbytes = len;
-    ADIOI_Add_req_to_list(request);
-    fd->async_count++;
-
-    fd->fp_sys_posn = -1;   /* set it to null. */
-
-    if (*id_sys == -1) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io",
-					   "**io %s", strerror(errno));
-    }
-    else *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_pfs/ad_pfs_iwrite.c b/3rd-party/romio321/adio/ad_pfs/ad_pfs_iwrite.c
deleted file mode 100644
index 5dda2fbdeb6..00000000000
--- a/3rd-party/romio321/adio/ad_pfs/ad_pfs_iwrite.c
+++ /dev/null
@@ -1,80 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_pfs.h"
-
-void ADIOI_PFS_IwriteContig(ADIO_File fd, void *buf, int count, 
-                MPI_Datatype datatype, int file_ptr_type,
-                ADIO_Offset offset, ADIO_Request *request, int *error_code)  
-{
-    long *id_sys;
-    ADIO_Offset off;
-    int err;
-    MPI_Count len, typesize;
-    static char myname[] = "ADIOI_PFS_IWRITECONTIG";
-
-    *request = ADIOI_Malloc_request();
-    (*request)->optype = ADIOI_WRITE;
-    (*request)->fd = fd;
-    (*request)->datatype = datatype;
-
-    MPI_Type_size_x(datatype, &typesize);
-    len = count * typesize;
-
-    id_sys = (long *) ADIOI_Malloc(sizeof(long));
-    (*request)->handle = (void *) id_sys;
-
-    off = (file_ptr_type == ADIO_INDIVIDUAL) ? fd->fp_ind : offset;
-
-    lseek(fd->fd_sys, off, SEEK_SET);
-    *id_sys = _iwrite(fd->fd_sys, buf, len);
-
-    if ((*id_sys == -1) && (errno == EQNOMID)) {
-     /* the man pages say EMREQUEST, but in reality errno is set to EQNOMID! */
-
-        /* exceeded the max. no. of outstanding requests. */
-
-        /* complete all previous async. requests */
-        ADIOI_Complete_async(error_code);
-	if (error_code != MPI_SUCCESS) return;
-
-        /* try again */
-	*id_sys = _iwrite(fd->fd_sys, buf, len);
-
-        if ((*id_sys == -1) && (errno == EQNOMID)) {
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE, myname,
-					       __LINE__, MPI_ERR_IO, "**io",
-					       "**io %s", strerror(errno));
-	    return;
-        }
-    }
-    else if (*id_sys == -1) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io",
-					   "**io %s", strerror(errno));
-	return;
-    }
-
-    if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind += len; 
-
-    (*request)->queued = 1;
-    (*request)->nbytes = len;
-    ADIOI_Add_req_to_list(request);
-    fd->async_count++;
-
-    fd->fp_sys_posn = -1;   /* set it to null. */
-
-    if (*id_sys == -1) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io",
-					   "**io %s", strerror(errno));
-    }
-    else *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_pfs/ad_pfs_open.c b/3rd-party/romio321/adio/ad_pfs/ad_pfs_open.c
deleted file mode 100644
index f814b7c0a14..00000000000
--- a/3rd-party/romio321/adio/ad_pfs/ad_pfs_open.c
+++ /dev/null
@@ -1,85 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_pfs.h"
-
-void ADIOI_PFS_Open(ADIO_File fd, int *error_code)
-{
-    int perm, amode, old_mask, np_comm, np_total, err, flag;
-    char *value;
-    struct sattr attr;
-    static char myname[] = "ADIOI_PFS_OPEN";
-
-    if (fd->perm == ADIO_PERM_NULL) {
-	old_mask = umask(022);
-	umask(old_mask);
-	perm = old_mask ^ 0666;
-    }
-    else perm = fd->perm;
-
-    amode = 0;
-    if (fd->access_mode & ADIO_CREATE)
-	amode = amode | O_CREAT;
-    if (fd->access_mode & ADIO_RDONLY)
-	amode = amode | O_RDONLY;
-    if (fd->access_mode & ADIO_WRONLY)
-	amode = amode | O_WRONLY;
-    if (fd->access_mode & ADIO_RDWR)
-	amode = amode | O_RDWR;
-    if (fd->access_mode & ADIO_EXCL)
-	amode = amode | O_EXCL;
-
-    MPI_Comm_size(MPI_COMM_WORLD, &np_total);
-    MPI_Comm_size(fd->comm, &np_comm);
-
-    if (np_total == np_comm) 
-	fd->fd_sys = _gopen(fd->filename, amode, M_ASYNC, perm);
-    else fd->fd_sys = open(fd->filename, amode, perm);
-    fd->fd_direct = -1;
-
-    if (fd->fd_sys != -1) {
-	value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
-
-        /* if user has asked for pfs server buffering to be turned on,
-           it will be set to true in fd->info in the earlier call
-           to ADIOI_PFS_SetInfo. Turn it on now, since we now have a 
-           valid file descriptor. */
-
-	ADIOI_Info_get(fd->info, "pfs_svr_buf", MPI_MAX_INFO_VAL, 
-		     value, &flag);
-	if (flag && (!strcmp(value, "true"))) {
-	    err = fcntl(fd->fd_sys, F_PFS_SVR_BUF, TRUE);
-	    if (err) ADIOI_Info_set(fd->info, "pfs_svr_buf", "false");
-	}
-
-        /* get file striping information and set it in info */
-	err = fcntl(fd->fd_sys, F_GETSATTR, &attr);
-
-	if (!err) {
-	    ADIOI_Snprintf(value, MPI_MAX_INFO_VAL+1, "%d", attr.s_sunitsize);
-	    ADIOI_Info_set(fd->info, "striping_unit", value);
-
-	    ADIOI_Snprintf(value, MPI_MAX_INFO_VAL+1, "%d", attr.s_sfactor);
-	    ADIOI_Info_set(fd->info, "striping_factor", value);
-
-	    ADIOI_Snprintf(value, MPI_MAX_INFO_VAL+1, "%d", attr.s_start_sdir);
-	    ADIOI_Info_set(fd->info, "start_iodevice", value);
-	}
-	ADIOI_Free(value);
-
-	if (fd->access_mode & ADIO_APPEND) 
-	    fd->fp_ind = fd->fp_sys_posn = lseek(fd->fd_sys, 0, SEEK_END);
-    }
-
-    if (fd->fd_sys == -1) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io",
-					   "**io %s", strerror(errno));
-    }
-    else *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_pfs/ad_pfs_read.c b/3rd-party/romio321/adio/ad_pfs/ad_pfs_read.c
deleted file mode 100644
index bd3b7e70e47..00000000000
--- a/3rd-party/romio321/adio/ad_pfs/ad_pfs_read.c
+++ /dev/null
@@ -1,48 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_pfs.h"
-
-void ADIOI_PFS_ReadContig(ADIO_File fd, void *buf, int count, 
-                     MPI_Datatype datatype, int file_ptr_type,
-		     ADIO_Offset offset, ADIO_Status *status, int *error_code)
-{
-    MPI_Count err=-1, datatype_size, len;
-    static char myname[] = "ADIOI_PFS_READCONTIG";
-
-    MPI_Type_size_x(datatype, &datatype_size);
-    len = datatype_size * count;
-
-    if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-        if (fd->fp_sys_posn != offset) {
-	    lseek(fd->fd_sys, offset, SEEK_SET);
-	}
-	err = _cread(fd->fd_sys, buf, len);
-        fd->fp_sys_posn = offset + err;
-	/* individual file pointer not updated */        
-    }
-    else {  /* read from curr. location of ind. file pointer */
-        if (fd->fp_sys_posn != fd->fp_ind) {
-	    lseek(fd->fd_sys, fd->fp_ind, SEEK_SET);
-	}
-	err = _cread(fd->fd_sys, buf, len);
-	fd->fp_ind += err; 
-        fd->fp_sys_posn = fd->fp_ind;
-    }         
-
-#ifdef HAVE_STATUS_SET_BYTES
-    if (err != -1) MPIR_Status_set_bytes(status, datatype, err);
-#endif
-
-    if (err == -1) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io",
-					   "**io %s", strerror(errno));
-    }
-    else *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_pfs/ad_pfs_wait.c b/3rd-party/romio321/adio/ad_pfs/ad_pfs_wait.c
deleted file mode 100644
index e14159521a5..00000000000
--- a/3rd-party/romio321/adio/ad_pfs/ad_pfs_wait.c
+++ /dev/null
@@ -1,62 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_pfs.h"
-
-void ADIOI_PFS_ReadComplete(ADIO_Request *request, ADIO_Status *status,
-			    int *error_code)
-{
-    int err=0;
-    static char myname[] = "ADIOI_PFS_READCOMPLETE";
-
-    if (*request == ADIO_REQUEST_NULL) {
-        *error_code = MPI_SUCCESS;
-        return;
-    }
-
-    if ((*request)->queued) {
-	err = _iowait(*((long *) (*request)->handle));
-	if (err == -1) {
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE, myname,
-					       __LINE__, MPI_ERR_IO, "**io",
-					       "**io %s", strerror(errno));
-	}
-	else *error_code = MPI_SUCCESS;
-    } /* if ((*request)->queued) ... */
-    else *error_code = MPI_SUCCESS;
-#ifdef HAVE_STATUS_SET_BYTES
-    if ((*request)->nbytes != -1)
-	MPIR_Status_set_bytes(status, (*request)->datatype, (*request)->nbytes);
-#endif
-
-    if ((*request)->queued != -1) {
-
-        /* queued = -1 is an internal hack used when the request must
-           be completed, but the request object should not be
-           freed. This is used in ADIOI_Complete_async, because the user
-           will call MPI_Wait later, which would require status to
-           be filled. Ugly but works. queued = -1 should be used only
-           in ADIOI_Complete_async. 
-           This should not affect the user in any way. */
-
-        /* if request is still queued in the system, it is also there
-           on ADIOI_Async_list. Delete it from there. */
-        if ((*request)->queued) ADIOI_Del_req_from_list(request);
-
-        (*request)->fd->async_count--;
-        if ((*request)->handle) ADIOI_Free((*request)->handle);
-        ADIOI_Free_request((ADIOI_Req_node *) (*request));
-        *request = ADIO_REQUEST_NULL;
-    }
-}
-
-
-void ADIOI_PFS_WriteComplete(ADIO_Request *request, ADIO_Status *status, int *error_code)  
-{
-    ADIOI_PFS_ReadComplete(request, status, error_code);
-}
diff --git a/3rd-party/romio321/adio/ad_pfs/ad_pfs_write.c b/3rd-party/romio321/adio/ad_pfs/ad_pfs_write.c
deleted file mode 100644
index c64e976a2b4..00000000000
--- a/3rd-party/romio321/adio/ad_pfs/ad_pfs_write.c
+++ /dev/null
@@ -1,49 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_pfs.h"
-
-void ADIOI_PFS_WriteContig(ADIO_File fd, void *buf, int count, 
-			   MPI_Datatype datatype, int file_ptr_type,
-			   ADIO_Offset offset, ADIO_Status *status,
-			   int *error_code)
-{
-    MPI_Count err=-1, datatype_size, len;
-    static char myname[] = "ADIOI_PFS_WRITECONTIG";
-
-    MPI_Type_size_x(datatype, &datatype_size);
-    len = datatype_size * count;
-
-    if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-        if (fd->fp_sys_posn != offset) {
-            lseek(fd->fd_sys, offset, SEEK_SET);
-	}
-        err = _cwrite(fd->fd_sys, buf, len);
-        fd->fp_sys_posn = offset + err;
-         /* individual file pointer not updated */        
-    }
-    else { /* write from curr. location of ind. file pointer */
-        if (fd->fp_sys_posn != fd->fp_ind) {
-            lseek(fd->fd_sys, fd->fp_ind, SEEK_SET);
-	}
-        err = _cwrite(fd->fd_sys, buf, len);
-        fd->fp_ind += err;
-        fd->fp_sys_posn = fd->fp_ind;
-    }
-
-#ifdef HAVE_STATUS_SET_BYTES
-    if (err != -1) MPIR_Status_set_bytes(status, datatype, err);
-#endif
-
-    if (err == -1) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io",
-					   "**io %s", strerror(errno));
-    }
-    else *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_piofs/Makefile.mk b/3rd-party/romio321/adio/ad_piofs/Makefile.mk
deleted file mode 100644
index 4bcbd61f243..00000000000
--- a/3rd-party/romio321/adio/ad_piofs/Makefile.mk
+++ /dev/null
@@ -1,21 +0,0 @@
-## -*- Mode: Makefile; -*-
-## vim: set ft=automake :
-##
-## (C) 2011 by Argonne National Laboratory.
-##     See COPYRIGHT in top-level directory.
-##
-
-if BUILD_AD_PIOFS
-
-noinst_HEADERS += adio/ad_piofs/ad_piofs.h
-
-romio_other_sources +=             \
-    adio/ad_piofs/ad_piofs_read.c  \
-    adio/ad_piofs/ad_piofs_open.c  \
-    adio/ad_piofs/ad_piofs_write.c \
-    adio/ad_piofs/ad_piofs_fcntl.c \
-    adio/ad_piofs/ad_piofs_hints.c \
-    adio/ad_piofs/ad_piofs.c
-
-endif BUILD_AD_PIOFS
-
diff --git a/3rd-party/romio321/adio/ad_piofs/README b/3rd-party/romio321/adio/ad_piofs/README
deleted file mode 100644
index 933677b1770..00000000000
--- a/3rd-party/romio321/adio/ad_piofs/README
+++ /dev/null
@@ -1 +0,0 @@
-This code is no longer supported.
diff --git a/3rd-party/romio321/adio/ad_piofs/ad_piofs.c b/3rd-party/romio321/adio/ad_piofs/ad_piofs.c
deleted file mode 100644
index 726bbf15f87..00000000000
--- a/3rd-party/romio321/adio/ad_piofs/ad_piofs.c
+++ /dev/null
@@ -1,39 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_piofs.h"
-
-/* adioi.h has the ADIOI_Fns_struct define */
-#include "adioi.h"
-
-struct ADIOI_Fns_struct ADIO_PIOFS_operations = {
-    ADIOI_PIOFS_Open, /* Open */
-    ADIOI_PIOFS_ReadContig, /* ReadContig */
-    ADIOI_PIOFS_WriteContig, /* WriteContig */
-    ADIOI_GEN_ReadStridedColl, /* ReadStridedColl */
-    ADIOI_GEN_WriteStridedColl, /* WriteStridedColl */
-    ADIOI_GEN_SeekIndividual, /* SeekIndividual */
-    ADIOI_PIOFS_Fcntl, /* Fcntl */
-    ADIOI_PIOFS_SetInfo, /* SetInfo */
-    ADIOI_GEN_ReadStrided, /* ReadStrided */
-    ADIOI_PIOFS_WriteStrided, /* WriteStrided */
-    ADIOI_GEN_Close, /* Close */
-    ADIOI_FAKE_IreadContig, /* IreadContig */
-    ADIOI_FAKE_IwriteContig, /* IwriteContig */
-    ADIOI_FAKE_IODone, /* ReadDone */
-    ADIOI_FAKE_IODone, /* WriteDone */
-    ADIOI_FAKE_IOComplete, /* ReadComplete */
-    ADIOI_FAKE_IOComplete, /* WriteComplete */
-    ADIOI_FAKE_IreadStrided, /* IreadStrided */
-    ADIOI_FAKE_IwriteStrided, /* IwriteStrided */
-    ADIOI_GEN_Flush, /* Flush */
-    ADIOI_GEN_Resize, /* Resize */
-    ADIOI_GEN_Delete, /* Delete */
-    ADIOI_PIOFS_Feature, 
-    ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
-    ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
-};
diff --git a/3rd-party/romio321/adio/ad_piofs/ad_piofs.h b/3rd-party/romio321/adio/ad_piofs/ad_piofs.h
deleted file mode 100644
index e9b74c9e872..00000000000
--- a/3rd-party/romio321/adio/ad_piofs/ad_piofs.h
+++ /dev/null
@@ -1,40 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-/* contains definitions, declarations, and macros specific to the
-   implementation of ADIO on PIOFS */
-
-#ifndef AD_PIOFS_INCLUDE
-#define AD_PIOFS_INCLUDE
-
-#include <unistd.h>
-#include <sys/uio.h>
-#include <sys/types.h>
-#include <fcntl.h>
-#include <piofs/piofs_ioctl.h>
-#include "adio.h"
-
-void ADIOI_PIOFS_Open(ADIO_File fd, int *error_code);
-void ADIOI_PIOFS_ReadContig(ADIO_File fd, void *buf, int count, 
-                      MPI_Datatype datatype, int file_ptr_type,
-                     ADIO_Offset offset, ADIO_Status *status, int
-		     *error_code);
-void ADIOI_PIOFS_WriteContig(ADIO_File fd, void *buf, int count, 
-                      MPI_Datatype datatype, int file_ptr_type,
-                      ADIO_Offset offset, ADIO_Status *status, int
-		      *error_code);   
-void ADIOI_PIOFS_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct, int
-		*error_code); 
-void ADIOI_PIOFS_WriteStrided(ADIO_File fd, void *buf, int count,
-		       MPI_Datatype datatype, int file_ptr_type,
-		       ADIO_Offset offset, ADIO_Status *status, int
-		       *error_code);
-void ADIOI_PIOFS_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code);
-
-void ADIOI_PIOFS_Feature(ADIO_File fd, int flag);
-
-#endif
diff --git a/3rd-party/romio321/adio/ad_piofs/ad_piofs_fcntl.c b/3rd-party/romio321/adio/ad_piofs/ad_piofs_fcntl.c
deleted file mode 100644
index 7d4a37cbfd5..00000000000
--- a/3rd-party/romio321/adio/ad_piofs/ad_piofs_fcntl.c
+++ /dev/null
@@ -1,77 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_piofs.h"
-#include "adio_extern.h"
-
-void ADIOI_PIOFS_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct, int *error_code)
-{
-    MPI_Datatype copy_etype, copy_filetype;
-    int i, ntimes, err;
-    ADIO_Offset curr_fsize, alloc_size, size, len, done;
-    ADIO_Status status;
-    char *buf;
-    piofs_change_view_t *piofs_change_view;
-#ifndef PRINT_ERR_MSG
-    static char myname[] = "ADIOI_PIOFS_FCNTL";
-#endif
-
-    switch(flag) {
-    case ADIO_FCNTL_GET_FSIZE:
-	fcntl_struct->fsize = llseek(fd->fd_sys, 0, SEEK_END);
-	if (fd->fp_sys_posn != -1) 
-	     llseek(fd->fd_sys, fd->fp_sys_posn, SEEK_SET);
-	if (fcntl_struct->fsize == -1) {
-#ifdef MPICH
-	    *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io",
-		"**io %s", strerror(errno));
-#elif defined(PRINT_ERR_MSG)
-	    *error_code =  MPI_ERR_UNKNOWN;
-#else /* MPICH-1 */
-	    *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,
-			      myname, "I/O Error", "%s", strerror(errno));
-	    ADIOI_Error(fd, *error_code, myname);	    
-#endif
-	}
-	else *error_code = MPI_SUCCESS;
-	break;
-
-    case ADIO_FCNTL_SET_DISKSPACE:
-	ADIOI_GEN_Prealloc(fd, fcntl_struct->diskspace, error_code);
-	break;
-
-    case ADIO_FCNTL_SET_ATOMICITY:
-	piofs_change_view = (piofs_change_view_t *) 
-                                 ADIOI_Malloc(sizeof(piofs_change_view_t));
-	piofs_change_view->Vbs = piofs_change_view->Vn = 
-             piofs_change_view->Hbs = piofs_change_view->Hn = 1;
-	piofs_change_view->subfile = 0;
-	piofs_change_view->flags = (fcntl_struct->atomicity == 0) 
-                             ? (ACTIVE | NORMAL) : (ACTIVE | CAUTIOUS);
-	err = piofsioctl(fd->fd_sys, PIOFS_CHANGE_VIEW, piofs_change_view);
-	ADIOI_Free(piofs_change_view);
-	fd->atomicity = (fcntl_struct->atomicity == 0) ? 0 : 1;
-	if (err == -1) {
-#ifdef MPICH
-	    *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io",
-		"**io %s", strerror(errno));
-#elif defined(PRINT_ERR_MSG)
-	    *error_code =  MPI_ERR_UNKNOWN;
-#else /* MPICH-1 */
-	    *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,
-			      myname, "I/O Error", "%s", strerror(errno));
-	    ADIOI_Error(fd, *error_code, myname);	    
-#endif
-	}
-	else *error_code = MPI_SUCCESS;
-	break;
-
-    default:
-	FPRINTF(stderr, "Unknown flag passed to ADIOI_PIOFS_Fcntl\n");
-	MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-}
diff --git a/3rd-party/romio321/adio/ad_piofs/ad_piofs_features.c b/3rd-party/romio321/adio/ad_piofs/ad_piofs_features.c
deleted file mode 100644
index f3b0df6b21a..00000000000
--- a/3rd-party/romio321/adio/ad_piofs/ad_piofs_features.c
+++ /dev/null
@@ -1,19 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *
- *  (C) 2008 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-int ADIOI_PIOFS_Features(int flag)
-{
-	switch(flag) {
-		case ADIO_LOCKS:
-		case ADIO_SHARED_FP:
-		case ADIO_ATOMIC_MODE:
-		case ADIO_DATA_SIEVING_WRITES:
-		case ADIO_SCALABLE_OPEN:
-		default:
-			return 0;
-			break;
-	}
-}
diff --git a/3rd-party/romio321/adio/ad_piofs/ad_piofs_hints.c b/3rd-party/romio321/adio/ad_piofs/ad_piofs_hints.c
deleted file mode 100644
index 242ebb3f07d..00000000000
--- a/3rd-party/romio321/adio/ad_piofs/ad_piofs_hints.c
+++ /dev/null
@@ -1,118 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_piofs.h"
-
-void ADIOI_PIOFS_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code)
-{
-    piofs_create_t piofs_create;
-    piofs_statfs_t piofs_statfs;
-    char *value, *path, *slash;
-    int flag, tmp_val, str_factor=-1, str_unit=-1, start_iodev=-1;
-    int err, myrank, perm, old_mask, nioservers;
-
-    if ((fd->info) == MPI_INFO_NULL) {
-	/* This must be part of the open call. can set striping parameters 
-           if necessary. */ 
-	MPI_Info_create(&(fd->info));
-	
-	/* has user specified striping parameters 
-           and do they have the same value on all processes? */
-	if (users_info != MPI_INFO_NULL) {
-	    value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
-
-	    ADIOI_Info_get(users_info, "striping_factor", MPI_MAX_INFO_VAL, 
-			 value, &flag);
-	    if (flag) {
-		str_factor=atoi(value);
-		tmp_val = str_factor;
-		MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
-		if (tmp_val != str_factor) {
-		    FPRINTF(stderr, "ADIOI_PIOFS_SetInfo: the value for key \"striping_factor\" must be the same on all processes\n");
-		    MPI_Abort(MPI_COMM_WORLD, 1);
-		}
-	    }
-
-	    ADIOI_Info_get(users_info, "striping_unit", MPI_MAX_INFO_VAL, 
-			 value, &flag);
-	    if (flag) {
-		str_unit=atoi(value);
-		tmp_val = str_unit;
-		MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
-		if (tmp_val != str_unit) {
-		    FPRINTF(stderr, "ADIOI_PIOFS_SetInfo: the value for key \"striping_unit\" must be the same on all processes\n");
-		    MPI_Abort(MPI_COMM_WORLD, 1);
-		}
-	    }
-
-	    ADIOI_Info_get(users_info, "start_iodevice", MPI_MAX_INFO_VAL, 
-			 value, &flag);
-	    if (flag) {
-		start_iodev=atoi(value);
-		tmp_val = start_iodev;
-		MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
-		if (tmp_val != start_iodev) {
-		    FPRINTF(stderr, "ADIOI_PIOFS_SetInfo: the value for key \"start_iodevice\" must be the same on all processes\n");
-		    MPI_Abort(MPI_COMM_WORLD, 1);
-		}
-	    }
-
-	    ADIOI_Free(value);
-
-         /* if user has specified striping info, process 0 tries to set it */
-	    if ((str_factor > 0) || (str_unit > 0) || (start_iodev >= 0)) {
-		MPI_Comm_rank(fd->comm, &myrank);
-		if (!myrank) {
-		    int len;
-
-		    if (fd->perm == ADIO_PERM_NULL) {
-			old_mask = umask(022);
-			umask(old_mask);
-			perm = old_mask ^ 0666;
-		    }
-		    else perm = fd->perm;
-
-		    /* to find out the number of I/O servers, I need
-                       the path to the directory containing the file */
-
-		    path = ADIOI_Strdup(fd->filename);
-		    len = strlen(path) + 1;
-		    slash = strrchr(path, '/');
-		    if (!slash) ADIOI_Strncpy(path, ".", len);
-		    else {
-			if (slash == path) *(path + 1) = '\0';
-			else *slash = '\0';
-		    }
-		    ADIOI_Strncpy(piofs_statfs.name, path, len);
-		    err = piofsioctl(0, PIOFS_STATFS, &piofs_statfs);
-		    nioservers = (err) ? -1 : piofs_statfs.f_nodes;
-
-		    ADIOI_Free(path);
-
-		    str_factor = ADIOI_MIN(nioservers, str_factor);
-		    if (start_iodev >= nioservers) start_iodev = -1;
-
-		    ADIOI_Strncpy(piofs_create.name, fd->filename, len);
-		    piofs_create.bsu = (str_unit > 0) ? str_unit : -1;
-		    piofs_create.cells = (str_factor > 0) ? str_factor : -1;
-		    piofs_create.permissions = perm;
-		    piofs_create.base_node = (start_iodev >= 0) ? 
-                                                     start_iodev : -1;
-		    piofs_create.flags = 0;
-
-		    err = piofsioctl(0, PIOFS_CREATE, &piofs_create);
-		}
-		MPI_Barrier(fd->comm);
-	    }
-	}
-    }	
-	
-    /* set the values for collective I/O and data sieving parameters */
-    ADIOI_GEN_SetInfo(fd, users_info, error_code);
-
-    *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_piofs/ad_piofs_open.c b/3rd-party/romio321/adio/ad_piofs/ad_piofs_open.c
deleted file mode 100644
index e02e90cf325..00000000000
--- a/3rd-party/romio321/adio/ad_piofs/ad_piofs_open.c
+++ /dev/null
@@ -1,79 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_piofs.h"
-
-void ADIOI_PIOFS_Open(ADIO_File fd, int *error_code)
-{
-    int amode, perm, old_mask, err;
-    piofs_fstat_t piofs_fstat;
-    char *value;
-#ifndef PRINT_ERR_MSG
-    static char myname[] = "ADIOI_PIOFS_OPEN";
-#endif
-
-    if (fd->perm == ADIO_PERM_NULL) {
-	old_mask = umask(022);
-	umask(old_mask);
-	perm = old_mask ^ 0666;
-    }
-    else perm = fd->perm;
-
-    amode = 0;
-    if (fd->access_mode & ADIO_CREATE)
-	amode = amode | O_CREAT;
-    if (fd->access_mode & ADIO_RDONLY)
-	amode = amode | O_RDONLY;
-    if (fd->access_mode & ADIO_WRONLY)
-	amode = amode | O_WRONLY;
-    if (fd->access_mode & ADIO_RDWR)
-	amode = amode | O_RDWR;
-    if (fd->access_mode & ADIO_EXCL)
-	amode = amode | O_EXCL;
-
-    fd->fd_sys = open(fd->filename, amode, perm);
-    fd->fd_direct = -1;
-
-    llseek(fd->fd_sys, 0, SEEK_SET);
-/* required to initiate use of 64-bit offset */
-
-    if (fd->fd_sys != -1) {
-	value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
-
-        /* get file striping information and set it in info */
-	err = piofsioctl(fd->fd_sys, PIOFS_FSTAT, &piofs_fstat);
-
-	if (!err) {
-	    ADIOI_Snprintf(value, MPI_MAX_INFO_VAL+1, "%d", piofs_fstat.st_bsu);
-	    ADIOI_Info_set(fd->info, "striping_unit", value);
-
-	    ADIOI_Snprintf(value, MPI_MAX_INFO_VAL+1, "%d", piofs_fstat.st_cells);
-	    ADIOI_Info_set(fd->info, "striping_factor", value);
-
-	    ADIOI_Snprintf(value, MPI_MAX_INFO_VAL+1, "%d", piofs_fstat.st_base_node);
-	    ADIOI_Info_set(fd->info, "start_iodevice", value);
-	}
-	ADIOI_Free(value);
-
-	if (fd->access_mode & ADIO_APPEND)
-	    fd->fp_ind = fd->fp_sys_posn = llseek(fd->fd_sys, 0, SEEK_END);
-    }
-
-    if (fd->fd_sys == -1) {
-#ifdef MPICH
-	*error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io",
-	    "**io %s", strerror(errno));
-#elif defined(PRINT_ERR_MSG)
-	*error_code =  MPI_ERR_UNKNOWN;
-#else /* MPICH-1 */
-	*error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,
-			      myname, "I/O Error", "%s", strerror(errno));
-	ADIOI_Error(ADIO_FILE_NULL, *error_code, myname);	    
-#endif
-    }
-    else *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_piofs/ad_piofs_read.c b/3rd-party/romio321/adio/ad_piofs/ad_piofs_read.c
deleted file mode 100644
index 278548656e3..00000000000
--- a/3rd-party/romio321/adio/ad_piofs/ad_piofs_read.c
+++ /dev/null
@@ -1,56 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_piofs.h"
-
-void ADIOI_PIOFS_ReadContig(ADIO_File fd, void *buf, int count, 
-                     MPI_Datatype datatype, int file_ptr_type,
-		     ADIO_Offset offset, ADIO_Status *status, int *error_code)
-{
-    MPI_Count err=-1, datatype_size, len;
-#ifndef PRINT_ERR_MSG
-    static char myname[] = "ADIOI_PIOFS_READCONTIG";
-#endif
-
-    MPI_Type_size_x(datatype, &datatype_size);
-    len = datatype_size * count;
-
-    if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-	if (fd->fp_sys_posn != offset) {
-	    llseek(fd->fd_sys, offset, SEEK_SET);
-	}
-	err = read(fd->fd_sys, buf, len);
-	fd->fp_sys_posn = offset + err;
-         /* individual file pointer not updated */        
-    }
-    else {  /* read from curr. location of ind. file pointer */
-	if (fd->fp_sys_posn != fd->fp_ind) {
-	    llseek(fd->fd_sys, fd->fp_ind, SEEK_SET);
-	}
-	err = read(fd->fd_sys, buf, len);
-	fd->fp_ind += err; 
-	fd->fp_sys_posn = fd->fp_ind;
-    }         
-
-#ifdef HAVE_STATUS_SET_BYTES
-    if (err != -1) MPIR_Status_set_bytes(status, datatype, err);
-#endif
-
-    if (err == -1) {
-#ifdef MPICH
-	*error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io",
-	    "**io %s", strerror(errno));
-#elif defined(PRINT_ERR_MSG)
-	*error_code =  MPI_ERR_UNKNOWN;
-#else
-	*error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,
-			      myname, "I/O Error", "%s", strerror(errno));
-	ADIOI_Error(fd, *error_code, myname);	    
-#endif
-    }
-    else *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_piofs/ad_piofs_write.c b/3rd-party/romio321/adio/ad_piofs/ad_piofs_write.c
deleted file mode 100644
index 31836a82406..00000000000
--- a/3rd-party/romio321/adio/ad_piofs/ad_piofs_write.c
+++ /dev/null
@@ -1,340 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_piofs.h"
-#include "adio_extern.h"
-
-void ADIOI_PIOFS_WriteContig(ADIO_File fd, void *buf, int count, 
-                     MPI_Datatype datatype, int file_ptr_type,
-		     ADIO_Offset offset, ADIO_Status *status, int *error_code)
-{
-    MPI_Count err=-1, datatype_size, len;
-#ifndef PRINT_ERR_MSG
-    static char myname[] = "ADIOI_PIOFS_WRITECONTIG";
-#endif
-
-    MPI_Type_size_x(datatype, &datatype_size);
-    len = datatype_size * count;
-
-    if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-	if (fd->fp_sys_posn != offset) {
-	    llseek(fd->fd_sys, offset, SEEK_SET);
-	}
-	err = write(fd->fd_sys, buf, len);
-	fd->fp_sys_posn = offset + err;
-	/* individual file pointer not updated */        
-    }
-    else { /* write from curr. location of ind. file pointer */
-	if (fd->fp_sys_posn != fd->fp_ind) {
-	    llseek(fd->fd_sys, fd->fp_ind, SEEK_SET);
-	}
-	err = write(fd->fd_sys, buf, len);
-	fd->fp_ind += err;
-	fd->fp_sys_posn = fd->fp_ind;
-    }
-
-#ifdef HAVE_STATUS_SET_BYTES
-    if (err != -1) MPIR_Status_set_bytes(status, datatype, err);
-#endif
-
-    if (err == -1) {
-#ifdef MPICH
-	*error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io",
-	    "**io %s", strerror(errno));
-#elif defined(PRINT_ERR_MSG)
-	*error_code =  MPI_ERR_UNKNOWN;
-#else
-	*error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,
-			      myname, "I/O Error", "%s", strerror(errno));
-	ADIOI_Error(fd, *error_code, myname);
-#endif
-    }
-    else *error_code = MPI_SUCCESS;
-}
-
-
-
-void ADIOI_PIOFS_WriteStrided(ADIO_File fd, void *buf, int count,
-                       MPI_Datatype datatype, int file_ptr_type,
-                       ADIO_Offset offset, ADIO_Status *status, int
-                       *error_code)
-{
-/* Since PIOFS does not support file locking, can't do buffered writes
-   as on Unix */
-
-/* offset is in units of etype relative to the filetype. */
-
-    ADIOI_Flatlist_node *flat_buf, *flat_file;
-    struct iovec *iov;
-    int i, j, k, err=-1, bwr_size, fwr_size=0, st_index=0;
-    int num, size, sum, n_etypes_in_filetype, size_in_filetype;
-    MPI_Count bufsize;
-    int n_filetypes, etype_in_filetype;
-    ADIO_Offset abs_off_in_filetype=0;
-    MPI_Count filetype_size, etype_size, buftype_size;
-    MPI_Aint filetype_extent, buftype_extent, indx, filetype_lb, buftype_lb;
-    int buf_count, buftype_is_contig, filetype_is_contig;
-    ADIO_Offset off, disp;
-    int flag, new_bwr_size, new_fwr_size, err_flag=0;
-#ifndef PRINT_ERR_MSG
-    static char myname[] = "ADIOI_PIOFS_WRITESTRIDED";
-#endif
-
-    if (fd->atomicity) {
-	FPRINTF(stderr, "ROMIO cannot guarantee atomicity of noncontiguous accesses in atomic mode, as PIOFS doesn't support file locking. Use nonatomic mode and its associated semantics.\n");
-	MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-
-    MPI_Type_size_x(fd->filetype, &filetype_size);
-    if ( ! filetype_size ) {
-#ifdef HAVE_STATUS_SET_BYTES
-	MPIR_Status_set_bytes(status, datatype, 0);
-#endif
-	*error_code = MPI_SUCCESS; 
-	return;
-    }
-
-    MPI_Type_get_extent(fd->filetype, &filetype_lb, &filetype_extent);
-    MPI_Type_size_x(datatype, &buftype_size);
-    MPI_Type_get_extent(datatype, &buftype_lb, &buftype_extent);
-    etype_size = fd->etype_size;
-    
-    bufsize = buftype_size * count;
-
-    if (!buftype_is_contig && filetype_is_contig) {
-
-/* noncontiguous in memory, contiguous in file. use writev */
-
-	flat_buf = ADIOI_Flatten_and_find(datatype);
-
-/* There is a limit of 16 on the number of iovecs for readv/writev! */
-
-	iov = (struct iovec *) ADIOI_Malloc(16*sizeof(struct iovec));
-
-	if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-	    off = fd->disp + etype_size * offset;
-	    llseek(fd->fd_sys, off, SEEK_SET);
-	}
-	else off = llseek(fd->fd_sys, fd->fp_ind, SEEK_SET);
-
-	k = 0;
-	for (j=0; j<count; j++) 
-	    for (i=0; i<flat_buf->count; i++) {
-		iov[k].iov_base = ((char *) buf) + j*buftype_extent +
-		    flat_buf->indices[i]; 
-		iov[k].iov_len = flat_buf->blocklens[i];
-		/*FPRINTF(stderr, "%d %d\n", iov[k].iov_base, iov[k].iov_len);*/
-
-		off += flat_buf->blocklens[i];
-		k = (k+1)%16;
-
-		if (!k) {
-		    err = writev(fd->fd_sys, iov, 16);
-		    if (err == -1) err_flag = 1;
-		}
-	    }
-
-	if (k) {
-	    err = writev(fd->fd_sys, iov, k);
-	    if (err == -1) err_flag = 1;
-	}
-
-	if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
-
-	ADIOI_Free(iov);
-	if (err_flag) {
-#ifdef MPICH
-	    *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io",
-		"**io %s", strerror(errno));
-#elif defined(PRINT_ERR_MSG) 
-	    *error_code =  MPI_ERR_UNKNOWN;
-#else /* MPICH-1 */
-	    *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,
-			      myname, "I/O Error", "%s", strerror(errno));
-	    ADIOI_Error(fd, *error_code, myname);
-#endif
-	}
-	else *error_code = MPI_SUCCESS;
-    } /* if (!buftype_is_contig && filetype_is_contig) ... */
-
-    else {  /* noncontiguous in file */
-
-/* split up into several contiguous writes */
-
-/* find starting location in the file */
-
-/* filetype already flattened in ADIO_Open */
-	flat_file = ADIOI_Flatlist;
-	while (flat_file->type != fd->filetype) flat_file = flat_file->next;
-        disp = fd->disp;
-
-	if (file_ptr_type == ADIO_INDIVIDUAL) {
-	    offset = fd->fp_ind; /* in bytes */
-            n_filetypes = -1;
-            flag = 0;
-            while (!flag) {
-                n_filetypes++;
-                for (i=0; i<flat_file->count; i++) {
-                    if (disp + flat_file->indices[i] + 
-                        (ADIO_Offset) n_filetypes*filetype_extent + flat_file->blocklens[i] 
-                            >= offset) {
-                        st_index = i;
-                        fwr_size = disp + flat_file->indices[i] + 
-                                (ADIO_Offset) n_filetypes*filetype_extent
-                                 + flat_file->blocklens[i] - offset;
-                        flag = 1;
-                        break;
-                    }
-                }
-            }
-	}
-	else {
-	    n_etypes_in_filetype = filetype_size/etype_size;
-	    n_filetypes = (int) (offset / n_etypes_in_filetype);
-	    etype_in_filetype = (int) (offset % n_etypes_in_filetype);
-	    size_in_filetype = etype_in_filetype * etype_size;
- 
-	    sum = 0;
-	    for (i=0; i<flat_file->count; i++) {
-		sum += flat_file->blocklens[i];
-		if (sum > size_in_filetype) {
-		    st_index = i;
-		    fwr_size = sum - size_in_filetype;
-		    abs_off_in_filetype = flat_file->indices[i] +
-			size_in_filetype - (sum - flat_file->blocklens[i]);
-		    break;
-		}
-	    }
-
-	    /* abs. offset in bytes in the file */
-            offset = disp + (ADIO_Offset) n_filetypes*filetype_extent + abs_off_in_filetype;
-	}
-
-	if (buftype_is_contig && !filetype_is_contig) {
-
-/* contiguous in memory, noncontiguous in file. should be the most
-   common case. */
-
-	    i = 0;
-	    j = st_index;
-	    off = offset;
-	    fwr_size = ADIOI_MIN(fwr_size, bufsize);
-	    while (i < bufsize) {
-                if (fwr_size) { 
-                    /* TYPE_UB and TYPE_LB can result in 
-                       fwr_size = 0. save system call in such cases */ 
-		    llseek(fd->fd_sys, off, SEEK_SET);
-		    err = write(fd->fd_sys, ((char *) buf) + i, fwr_size);
-		    if (err == -1) err_flag = 1;
-		}
-		i += fwr_size;
-
-                if (off + fwr_size < disp + flat_file->indices[j] +
-                   flat_file->blocklens[j] + (ADIO_Offset) n_filetypes*filetype_extent)
-                       off += fwr_size;
-                /* did not reach end of contiguous block in filetype.
-                   no more I/O needed. off is incremented by fwr_size. */
-                else {
-		    if (j < (flat_file->count - 1)) j++;
-		    else {
-			j = 0;
-			n_filetypes++;
-		    }
-		    off = disp + flat_file->indices[j] + 
-                                        (ADIO_Offset) n_filetypes*filetype_extent;
-		    fwr_size = ADIOI_MIN(flat_file->blocklens[j], bufsize-i);
-		}
-	    }
-	}
-	else {
-/* noncontiguous in memory as well as in file */
-
-	    flat_buf = ADIOI_Flatten_and_find(datatype);
-
-	    k = num = buf_count = 0;
-	    indx = flat_buf->indices[0];
-	    j = st_index;
-	    off = offset;
-	    bwr_size = flat_buf->blocklens[0];
-
-	    while (num < bufsize) {
-		size = ADIOI_MIN(fwr_size, bwr_size);
-		if (size) {
-		    llseek(fd->fd_sys, off, SEEK_SET);
-		    err = write(fd->fd_sys, ((char *) buf) + indx, size);
-		    if (err == -1) err_flag = 1;
-		}
-
-		new_fwr_size = fwr_size;
-		new_bwr_size = bwr_size;
-
-		if (size == fwr_size) {
-/* reached end of contiguous block in file */
-                    if (j < (flat_file->count - 1)) j++;
-                    else {
-                        j = 0;
-                        n_filetypes++;
-                    }
-
-                    off = disp + flat_file->indices[j] + 
-                                              (ADIO_Offset) n_filetypes*filetype_extent;
-
-		    new_fwr_size = flat_file->blocklens[j];
-		    if (size != bwr_size) {
-			indx += size;
-			new_bwr_size -= size;
-		    }
-		}
-
-		if (size == bwr_size) {
-/* reached end of contiguous block in memory */
-
-		    k = (k + 1)%flat_buf->count;
-		    buf_count++;
-		    indx = buftype_extent*(buf_count/flat_buf->count) +
-			flat_buf->indices[k]; 
-		    new_bwr_size = flat_buf->blocklens[k];
-		    if (size != fwr_size) {
-			off += size;
-			new_fwr_size -= size;
-		    }
-		}
-		num += size;
-		fwr_size = new_fwr_size;
-                bwr_size = new_bwr_size;
-	    }
-	}
-
-        if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
-	if (err_flag) {
-#ifdef MPICH
-	    *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io",
-		"**io %s", strerror(errno));
-#elif defined(PRINT_ERR_MSG)
-	    *error_code = MPI_ERR_UNKNOWN;
-#else /* MPICH-1 */
-	    *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,
-			      myname, "I/O Error", "%s", strerror(errno));
-	    ADIOI_Error(fd, *error_code, myname);
-#endif
-	}
-	else *error_code = MPI_SUCCESS;
-    } 
-
-    fd->fp_sys_posn = -1;   /* set it to null. */
-
-#ifdef HAVE_STATUS_SET_BYTES
-    MPIR_Status_set_bytes(status, datatype, bufsize);
-/* This is a temporary way of filling in status. The right way is to 
-   keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
-#endif
-
-    if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
-}
diff --git a/3rd-party/romio321/adio/ad_pvfs/Makefile.mk b/3rd-party/romio321/adio/ad_pvfs/Makefile.mk
deleted file mode 100644
index 50e7bd0ae60..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs/Makefile.mk
+++ /dev/null
@@ -1,26 +0,0 @@
-## -*- Mode: Makefile; -*-
-## vim: set ft=automake :
-##
-## (C) 2011 by Argonne National Laboratory.
-##     See COPYRIGHT in top-level directory.
-##
-
-if BUILD_AD_PVFS
-
-noinst_HEADERS += adio/ad_pvfs/ad_pvfs.h
-
-romio_other_sources +=            \
-    adio/ad_pvfs/ad_pvfs_close.c  \
-    adio/ad_pvfs/ad_pvfs_read.c   \
-    adio/ad_pvfs/ad_pvfs_open.c   \
-    adio/ad_pvfs/ad_pvfs_write.c  \
-    adio/ad_pvfs/ad_pvfs_fcntl.c  \
-    adio/ad_pvfs/ad_pvfs_flush.c  \
-    adio/ad_pvfs/ad_pvfs_resize.c \
-    adio/ad_pvfs/ad_pvfs_hints.c  \
-    adio/ad_pvfs/ad_pvfs_delete.c \
-    adio/ad_pvfs/ad_pvfs.c
-
-endif BUILD_AD_PVFS
-
-
diff --git a/3rd-party/romio321/adio/ad_pvfs/ad_pvfs.c b/3rd-party/romio321/adio/ad_pvfs/ad_pvfs.c
deleted file mode 100644
index 27a3df8af4a..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs/ad_pvfs.c
+++ /dev/null
@@ -1,39 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_pvfs.h"
-
-/* adioi.h has the ADIOI_Fns_struct define */
-#include "adioi.h"
-
-struct ADIOI_Fns_struct ADIO_PVFS_operations = {
-    ADIOI_PVFS_Open, /* Open */
-    ADIOI_PVFS_ReadContig, /* ReadContig */
-    ADIOI_PVFS_WriteContig, /* WriteContig */
-    ADIOI_GEN_ReadStridedColl, /* ReadStridedColl */
-    ADIOI_GEN_WriteStridedColl, /* WriteStridedColl */
-    ADIOI_GEN_SeekIndividual, /* SeekIndividual */
-    ADIOI_PVFS_Fcntl, /* Fcntl */
-    ADIOI_PVFS_SetInfo, /* SetInfo */
-    ADIOI_PVFS_ReadStrided, /* ReadStrided */
-    ADIOI_PVFS_WriteStrided, /* WriteStrided */
-    ADIOI_PVFS_Close, /* Close */
-    ADIOI_FAKE_IreadContig, /* IreadContig */
-    ADIOI_FAKE_IwriteContig, /* IwriteContig */
-    ADIOI_FAKE_IODone, /* ReadDone */
-    ADIOI_FAKE_IODone, /* WriteDone */
-    ADIOI_FAKE_IOComplete, /* ReadComplete */
-    ADIOI_FAKE_IOComplete, /* WriteComplete */
-    ADIOI_FAKE_IreadStrided, /* IreadStrided */
-    ADIOI_FAKE_IwriteStrided, /* IwriteStrided */
-    ADIOI_PVFS_Flush, /* Flush */
-    ADIOI_PVFS_Resize, /* Resize */
-    ADIOI_PVFS_Delete, /* Delete */
-    ADIOI_PVFS_Feature, /* Features */
-    ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
-    ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
-};
diff --git a/3rd-party/romio321/adio/ad_pvfs/ad_pvfs.h b/3rd-party/romio321/adio/ad_pvfs/ad_pvfs.h
deleted file mode 100644
index 88e1a9f2253..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs/ad_pvfs.h
+++ /dev/null
@@ -1,54 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#ifndef AD_PVFS_INCLUDE
-#define AD_PVFS_INCLUDE
-
-#ifndef ROMIOCONF_H_INCLUDED
-#include "romioconf.h"
-#define ROMIOCONF_H_INCLUDED
-#endif
-#ifdef ROMIO_PVFS_NEEDS_INT64_DEFINITION
-typedef long long int int64_t;
-#endif
-
-#include <unistd.h>
-#include <sys/types.h>
-#include <sys/uio.h>
-#include <fcntl.h>
-#ifdef HAVE_PVFS_H
-#include <pvfs.h>
-#endif
-#include "adio.h"
-
-void ADIOI_PVFS_Open(ADIO_File fd, int *error_code);
-void ADIOI_PVFS_Close(ADIO_File fd, int *error_code);
-void ADIOI_PVFS_ReadContig(ADIO_File fd, void *buf, int count, 
-                      MPI_Datatype datatype, int file_ptr_type,
-                     ADIO_Offset offset, ADIO_Status *status, int
-		     *error_code);
-void ADIOI_PVFS_WriteContig(ADIO_File fd, void *buf, int count, 
-                      MPI_Datatype datatype, int file_ptr_type,
-                      ADIO_Offset offset, ADIO_Status *status, int
-		      *error_code);   
-void ADIOI_PVFS_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct, int
-		*error_code); 
-void ADIOI_PVFS_WriteStrided(ADIO_File fd, void *buf, int count,
-		       MPI_Datatype datatype, int file_ptr_type,
-		       ADIO_Offset offset, ADIO_Status *status, int
-		       *error_code);
-void ADIOI_PVFS_ReadStrided(ADIO_File fd, void *buf, int count,
-		       MPI_Datatype datatype, int file_ptr_type,
-		       ADIO_Offset offset, ADIO_Status *status, int
-		       *error_code);
-void ADIOI_PVFS_Flush(ADIO_File fd, int *error_code);
-void ADIOI_PVFS_Delete(char *filename, int *error_code);
-void ADIOI_PVFS_Resize(ADIO_File fd, ADIO_Offset size, int *error_code);
-void ADIOI_PVFS_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code);
-
-
-#endif
diff --git a/3rd-party/romio321/adio/ad_pvfs/ad_pvfs_close.c b/3rd-party/romio321/adio/ad_pvfs/ad_pvfs_close.c
deleted file mode 100644
index c2da2e360ef..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs/ad_pvfs_close.c
+++ /dev/null
@@ -1,31 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_pvfs.h"
-
-void ADIOI_PVFS_Close(ADIO_File fd, int *error_code)
-{
-    int err;
-    static char myname[] = "ADIOI_PVFS_CLOSE";
-
-#ifdef ADIOI_MPE_LOGGING
-    MPE_Log_event( ADIOI_MPE_close_a, 0, NULL );
-#endif
-    err = pvfs_close(fd->fd_sys);
-#ifdef ADIOI_MPE_LOGGING
-    MPE_Log_event( ADIOI_MPE_close_b, 0, NULL );
-#endif
-    fd->fd_sys = -1;
-
-    if (err == -1) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io",
-					   "**io %s", strerror(errno));
-    }
-    else *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_pvfs/ad_pvfs_delete.c b/3rd-party/romio321/adio/ad_pvfs/ad_pvfs_delete.c
deleted file mode 100644
index 0e322ad32ae..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs/ad_pvfs_delete.c
+++ /dev/null
@@ -1,24 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_pvfs.h"
-#include "adio.h"
-
-void ADIOI_PVFS_Delete(char *filename, int *error_code)
-{
-    int err;
-    static char myname[] = "ADIOI_PVFS_DELETE";
-
-    err = pvfs_unlink(filename);
-    if (err == -1) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io",
-					   "**io %s", strerror(errno));
-    }
-    else *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_pvfs/ad_pvfs_fcntl.c b/3rd-party/romio321/adio/ad_pvfs/ad_pvfs_fcntl.c
deleted file mode 100644
index eeff2507189..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs/ad_pvfs_fcntl.c
+++ /dev/null
@@ -1,72 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_pvfs.h"
-#include "adio_extern.h"
-
-void ADIOI_PVFS_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct,
-		      int *error_code)
-{
-    static char myname[] = "ADIOI_PVFS_FCNTL";
-
-    switch(flag) {
-    case ADIO_FCNTL_GET_FSIZE:
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-	fcntl_struct->fsize = pvfs_lseek64(fd->fd_sys, 0, SEEK_END);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-	if (fd->fp_sys_posn != -1) {
-#ifdef ADIOI_MPE_LOGGING
-             MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-	     pvfs_lseek64(fd->fd_sys, fd->fp_sys_posn, SEEK_SET);
-#ifdef ADIOI_MPE_LOGGING
-             MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-        }
-	if (fcntl_struct->fsize == -1) {
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE, myname,
-					       __LINE__, MPI_ERR_IO, "**io",
-					       "**io %s", strerror(errno));
-	}
-	else *error_code = MPI_SUCCESS;
-	break;
-
-    case ADIO_FCNTL_SET_DISKSPACE:
-	ADIOI_GEN_Prealloc(fd, fcntl_struct->diskspace, error_code);
-	break;
-
-    case ADIO_FCNTL_SET_ATOMICITY:
-	fd->atomicity = 0;
-	/* --BEGIN ERROR HANDLING-- */
-	if (fcntl_struct->atomicity != 0) {
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE,
-					       myname, __LINE__,
-					       MPI_ERR_UNSUPPORTED_OPERATION,
-					       "PVFS does not support atomic mode",
-					       0);
-	    return;
-	}
-	/* --END ERROR HANDLING-- */
-	break;
-
-    default:
-	/* --BEGIN ERROR HANDLING-- */
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   MPI_ERR_ARG,
-					   "**flag", "**flag %d", flag);
-	return;  
-	/* --END ERROR HANDLING-- */
-    }
-}
diff --git a/3rd-party/romio321/adio/ad_pvfs/ad_pvfs_flush.c b/3rd-party/romio321/adio/ad_pvfs/ad_pvfs_flush.c
deleted file mode 100644
index 340f0cb3395..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs/ad_pvfs_flush.c
+++ /dev/null
@@ -1,36 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_pvfs.h"
-
-void ADIOI_PVFS_Flush(ADIO_File fd, int *error_code)
-{
-    int err, rank, dummy=0, dummy_in=0;
-    static char myname[] = "ADIOI_PVFS_FLUSH";
-
-    /* a collective routine: because we do not cache data in PVFS1, one process
-     * can initiate the fsync operation and broadcast the result to the others.
-     * One catch: MPI_File_sync has special meaning with respect to file system
-     * consistency.  Ensure no clients have outstanding write operations.
-     */
-
-    MPI_Comm_rank(fd->comm, &rank);
-    MPI_Reduce(&dummy_in, &dummy, 1, MPI_INT, MPI_SUM, 
-		    fd->hints->ranklist[0], fd->comm);
-    if (rank == fd->hints->ranklist[0]) {
-	    err = pvfs_fsync(fd->fd_sys);
-    }
-    MPI_Bcast(&err, 1, MPI_INT, fd->hints->ranklist[0], fd->comm);
-
-    if (err == -1) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io",
-					   "**io %s", strerror(errno));
-    }
-    else *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_pvfs/ad_pvfs_hints.c b/3rd-party/romio321/adio/ad_pvfs/ad_pvfs_hints.c
deleted file mode 100644
index fdc06ed8465..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs/ad_pvfs_hints.c
+++ /dev/null
@@ -1,146 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_pvfs.h"
-
-void ADIOI_PVFS_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code)
-{
-    char *value;
-    int flag, tmp_val, str_factor=-1, str_unit=-1, start_iodev=-1; 
-    static char myname[] = "ADIOI_PVFS_SETINFO";
-
-    if ((fd->info) == MPI_INFO_NULL) {
-	/* This must be part of the open call. can set striping parameters 
-           if necessary. */ 
-	MPI_Info_create(&(fd->info));
-	ADIOI_Info_set(fd->info, "romio_pvfs_listio_read", "disable");
-	ADIOI_Info_set(fd->info, "romio_pvfs_listio_write", "disable");
-	fd->hints->fs_hints.pvfs.listio_read = ADIOI_HINT_DISABLE;
-	fd->hints->fs_hints.pvfs.listio_write = ADIOI_HINT_DISABLE;
-	
-	/* has user specified any pvfs-specific hints (striping params, listio)
-           and do they have the same value on all processes? */
-	if (users_info != MPI_INFO_NULL) {
-	    value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
-
-	    ADIOI_Info_get(users_info, "striping_factor", MPI_MAX_INFO_VAL, 
-			 value, &flag);
-	    if (flag) {
-		str_factor=atoi(value);
-		tmp_val = str_factor;
-		MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
-		if (tmp_val != str_factor) {
-		    /* --BEGIN ERROR HANDLING-- */
-		    MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
-						       "striping_factor",
-						       error_code);
-		    return;
-		    /* --END ERROR HANDLING-- */
-		}
-		else ADIOI_Info_set(fd->info, "striping_factor", value);
-	    }
-
-	    ADIOI_Info_get(users_info, "striping_unit", MPI_MAX_INFO_VAL, 
-			 value, &flag);
-	    if (flag) {
-		str_unit=atoi(value);
-		tmp_val = str_unit;
-		MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
-		if (tmp_val != str_unit) {
-		    /* --BEGIN ERROR HANDLING-- */
-		    MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
-						       "striping_unit",
-						       error_code);
-		    return;
-		    /* --END ERROR HANDLING-- */
-		}
-		else ADIOI_Info_set(fd->info, "striping_unit", value);
-	    }
-
-	    ADIOI_Info_get(users_info, "start_iodevice", MPI_MAX_INFO_VAL, 
-			 value, &flag);
-	    if (flag) {
-		start_iodev=atoi(value);
-		tmp_val = start_iodev;
-		MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
-		if (tmp_val != start_iodev) {
-		    /* --BEGIN ERROR HANDLING-- */
-		    MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
-						       "start_iodevice",
-						       error_code);
-		    return;
-		    /* --END ERROR HANDLING-- */
-		}
-		else ADIOI_Info_set(fd->info, "start_iodevice", value);
-	    }
-
-	    ADIOI_Info_get(users_info, "romio_pvfs_listio_read",
-			 MPI_MAX_INFO_VAL,
-			 value, &flag);
-	    if (flag) {
-		if ( !strcmp(value, "enable") || !strcmp(value, "ENABLE")) 
-		{
-		    ADIOI_Info_set(fd->info, "romio_pvfs_listio_read", value);
-		    fd->hints->fs_hints.pvfs.listio_read = ADIOI_HINT_ENABLE;
-		} else if ( !strcmp(value, "disable") || !strcmp(value, "DISABLE")) 
-		{
-		    ADIOI_Info_set(fd->info , "romio_pvfs_listio_read", value);
-		    fd->hints->fs_hints.pvfs.listio_read = ADIOI_HINT_DISABLE;
-		}
-		else if ( !strcmp(value, "automatic") || !strcmp(value, "AUTOMATIC")) 
-		{
-		    ADIOI_Info_set(fd->info, "romio_pvfs_listio_read", value);
-		    fd->hints->fs_hints.pvfs.listio_read = ADIOI_HINT_AUTO;
-		}
-		tmp_val = fd->hints->fs_hints.pvfs.listio_read;
-		MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
-		if (tmp_val != fd->hints->fs_hints.pvfs.listio_read) {
-		    /* --BEGIN ERROR HANDLING-- */
-		    MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
-						       "romio_pvfs_listio_read",
-						       error_code);
-		    return;
-		    /* --END ERROR HANDLING-- */
-		}
-	    }
-	    ADIOI_Info_get(users_info, "romio_pvfs_listio_write", MPI_MAX_INFO_VAL,
-			 value, &flag);
-	    if (flag) {
-		if ( !strcmp(value, "enable") || !strcmp(value, "ENABLE")) 
-		{
-		    ADIOI_Info_set(fd->info, "romio_pvfs_listio_write", value);
-		    fd->hints->fs_hints.pvfs.listio_write = ADIOI_HINT_ENABLE;
-		} else if ( !strcmp(value, "disable") || !strcmp(value, "DISABLE")) 
-		{
-		    ADIOI_Info_set(fd->info, "romio_pvfs_listio_write", value);
-		    fd->hints->fs_hints.pvfs.listio_write = ADIOI_HINT_DISABLE;
-		}
-		else if ( !strcmp(value, "automatic") || !strcmp(value, "AUTOMATIC")) 
-		{
-		    ADIOI_Info_set(fd->info, "romio_pvfs_listio_write", value);
-		    fd->hints->fs_hints.pvfs.listio_write = ADIOI_HINT_AUTO;
-		}
-		tmp_val = fd->hints->fs_hints.pvfs.listio_write;
-		MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
-		if (tmp_val != fd->hints->fs_hints.pvfs.listio_write) {
-		    /* --BEGIN ERROR HANDLING-- */
-		    MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
-						       "romio_pvfs_listio_write",
-						       error_code);
-		    return;
-		    /* --END ERROR HANDLING-- */
-		}
-	    }		    
-	    ADIOI_Free(value);
-	}
-    }	
-
-    /* set the values for collective I/O and data sieving parameters */
-    ADIOI_GEN_SetInfo(fd, users_info, error_code);
-
-    *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_pvfs/ad_pvfs_open.c b/3rd-party/romio321/adio/ad_pvfs/ad_pvfs_open.c
deleted file mode 100644
index c4fa28805da..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs/ad_pvfs_open.c
+++ /dev/null
@@ -1,90 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_pvfs.h"
-
-void ADIOI_PVFS_Open(ADIO_File fd, int *error_code)
-{
-    int perm, amode, old_mask, flag;
-    char *value;
-    /* some really old versions of pvfs may not have a release nr */
-    /* we changed the structure of pvfs_filestat in pvfs-1.5.7 */
-    struct pvfs_filestat pstat = {-1,-1,-1};
-    static char myname[] = "ADIOI_PVFS_OPEN";
-
-    if (fd->perm == ADIO_PERM_NULL) {
-	old_mask = umask(022);
-	umask(old_mask);
-	perm = old_mask ^ 0666;
-    }
-    else perm = fd->perm;
-
-    amode = O_META;
-    if (fd->access_mode & ADIO_CREATE)
-	amode = amode | O_CREAT;
-    if (fd->access_mode & ADIO_RDONLY)
-	amode = amode | O_RDONLY;
-    if (fd->access_mode & ADIO_WRONLY)
-	amode = amode | O_WRONLY;
-    if (fd->access_mode & ADIO_RDWR)
-	amode = amode | O_RDWR;
-    if (fd->access_mode & ADIO_EXCL)
-	amode = amode | O_EXCL;
-
-    value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
-
-    ADIOI_Info_get(fd->info, "striping_factor", MPI_MAX_INFO_VAL, 
-		 value, &flag);
-    if (flag && (atoi(value) > 0)) pstat.pcount = atoi(value);
-
-    ADIOI_Info_get(fd->info, "striping_unit", MPI_MAX_INFO_VAL, 
-		 value, &flag);
-    if (flag && (atoi(value) > 0)) pstat.ssize = atoi(value);
-
-    ADIOI_Info_get(fd->info, "start_iodevice", MPI_MAX_INFO_VAL, 
-		 value, &flag);
-    if (flag && (atoi(value) >= 0)) pstat.base = atoi(value);
-
-#ifdef ADIOI_MPE_LOGGING
-    MPE_Log_event( ADIOI_MPE_open_a, 0, NULL );
-#endif
-    fd->fd_sys = pvfs_open64(fd->filename, amode, perm, &pstat, NULL);
-#ifdef ADIOI_MPE_LOGGING
-    MPE_Log_event( ADIOI_MPE_open_b, 0, NULL );
-#endif
-    fd->fd_direct = -1;
-
-    if ((fd->fd_sys != -1) && (fd->access_mode & ADIO_APPEND)) {
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-	fd->fp_ind = fd->fp_sys_posn = pvfs_lseek64(fd->fd_sys, 0, SEEK_END);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-    }
-
-    if (fd->fd_sys != -1) {
-	pvfs_ioctl(fd->fd_sys, GETMETA, &pstat);
-	ADIOI_Snprintf(value, MPI_MAX_INFO_VAL+1, "%d", pstat.pcount);
-	ADIOI_Info_set(fd->info, "striping_factor", value);
-	ADIOI_Snprintf(value, MPI_MAX_INFO_VAL+1, "%d", pstat.ssize);
-	ADIOI_Info_set(fd->info, "striping_unit", value);
-	ADIOI_Snprintf(value, MPI_MAX_INFO_VAL+1, "%d", pstat.base);
-	ADIOI_Info_set(fd->info, "start_iodevice", value);
-    }
-
-    ADIOI_Free(value);
-
-    if (fd->fd_sys == -1) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io",
-					   "**io %s", strerror(errno));
-    }
-    else *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_pvfs/ad_pvfs_read.c b/3rd-party/romio321/adio/ad_pvfs/ad_pvfs_read.c
deleted file mode 100644
index 7ea249f28a4..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs/ad_pvfs_read.c
+++ /dev/null
@@ -1,784 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-#include "ad_pvfs.h"
-
-#ifdef HAVE_PVFS_LISTIO
-void ADIOI_PVFS_ReadStridedListIO(ADIO_File fd, void *buf, int count,
-                       MPI_Datatype datatype, int file_ptr_type,
-                       ADIO_Offset offset, ADIO_Status *status, int
-                       *error_code);
-#endif
-
-void ADIOI_PVFS_ReadContig(ADIO_File fd, void *buf, int count, 
-                     MPI_Datatype datatype, int file_ptr_type,
-		     ADIO_Offset offset, ADIO_Status *status, int *error_code)
-{
-    MPI_Count err=-1, datatype_size, len;
-    static char myname[] = "ADIOI_PVFS_READCONTIG";
-
-    MPI_Type_size_x(datatype, &datatype_size);
-    len = datatype_size * count;
-
-    if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-	if (fd->fp_sys_posn != offset) {
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-	    pvfs_lseek64(fd->fd_sys, offset, SEEK_SET);
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-        }
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
-#endif
-	err = pvfs_read(fd->fd_sys, buf, len);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
-#endif
-	if (err>0)
-		fd->fp_sys_posn = offset + err;
-	/* individual file pointer not updated */        
-    }
-    else {  /* read from curr. location of ind. file pointer */
-	if (fd->fp_sys_posn != fd->fp_ind) {
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-	    pvfs_lseek64(fd->fd_sys, fd->fp_ind, SEEK_SET);
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-        }
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
-#endif
-	err = pvfs_read(fd->fd_sys, buf, len);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
-#endif
-	if (err > 0)
-		fd->fp_ind += err; 
-	fd->fp_sys_posn = fd->fp_ind;
-    }         
-
-#ifdef HAVE_STATUS_SET_BYTES
-    if (err != -1) MPIR_Status_set_bytes(status, datatype, err);
-#endif
-
-    if (err == -1) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io",
-					   "**io %s", strerror(errno));
-    }
-    else *error_code = MPI_SUCCESS;
-}
-
-
-void ADIOI_PVFS_ReadStrided(ADIO_File fd, void *buf, int count,
-			    MPI_Datatype datatype, int file_ptr_type,
-			    ADIO_Offset offset, ADIO_Status *status, int
-			    *error_code)
-{
-#ifdef HAVE_PVFS_LISTIO
-    if ( fd->hints->fs_hints.pvfs.listio_read == ADIOI_HINT_ENABLE) {
-	    ADIOI_PVFS_ReadStridedListIO(fd, buf, count, datatype, file_ptr_type,
-			    offset, status, error_code);
-	    return;
-    }
-#endif
-/* If hint set to DISABLE or AUTOMATIC, don't use listio */
-    ADIOI_GEN_ReadStrided(fd, buf, count, datatype, file_ptr_type,
-		    offset, status, error_code);
-}
-
-#ifdef HAVE_PVFS_LISTIO
-void ADIOI_PVFS_ReadStridedListIO(ADIO_File fd, void *buf, int count,
-                       MPI_Datatype datatype, int file_ptr_type,
-                       ADIO_Offset offset, ADIO_Status *status, int
-                       *error_code)
-{
-/* offset is in units of etype relative to the filetype. */
-
-    ADIOI_Flatlist_node *flat_buf, *flat_file;
-    int i, j, k, l, brd_size, frd_size=0, st_index=0;
-    int sum, n_etypes_in_filetype, size_in_filetype;
-    MPI_Count bufsize;
-    int n_filetypes, etype_in_filetype;
-    ADIO_Offset abs_off_in_filetype=0;
-    MPI_Count filetype_size, etype_size, buftype_size;
-    MPI_Aint filetype_extent, buftype_extent; 
-    int buf_count, buftype_is_contig, filetype_is_contig;
-    ADIO_Offset userbuf_off;
-    ADIO_Offset off, disp, start_off;
-    int flag, st_frd_size, st_n_filetypes;
-    int new_brd_size, new_frd_size;
-
-    int mem_list_count, file_list_count;
-    char **mem_offsets;
-    int64_t *file_offsets;
-    int *mem_lengths;
-    int32_t *file_lengths;
-    int total_blks_to_read;
-
-    int max_mem_list, max_file_list;
-
-    int b_blks_read;
-    int f_data_read;
-    int size_read=0, n_read_lists, extra_blks;
-
-    int end_brd_size, end_frd_size;
-    int start_k, start_j, new_file_read, new_buffer_read;
-    int start_mem_offset;
-
-#define MAX_ARRAY_SIZE 1024
-
-#ifndef PRINT_ERR_MESG
-  static char myname[] = "ADIOI_PVFS_ReadStrided";
-#endif
-
-    *error_code = MPI_SUCCESS;  /* changed below if error */
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-    MPI_Type_size_x(fd->filetype, &filetype_size);
-    if ( ! filetype_size ) {
-#ifdef HAVE_STATUS_SET_BYTES
-	MPIR_Status_set_bytes(status, datatype, 0);
-#endif
-	*error_code = MPI_SUCCESS; 
-	return;
-    }
-
-    MPI_Type_get_extent(fd->filetype, &filetype_lb, &filetype_extent);
-    MPI_Type_size_x(datatype, &buftype_size);
-    MPI_Type_get_extent(datatype, &buftype_lb, &buftype_extent);
-    etype_size = fd->etype_size;
-
-    bufsize = buftype_size * count;
-
-    if (!buftype_is_contig && filetype_is_contig) {
-
-/* noncontiguous in memory, contiguous in file. */
-        int64_t file_offsets;
-	int32_t file_lengths;
-
-	flat_buf = ADIOI_Flatten_and_find(datatype);
-
-	off = (file_ptr_type == ADIO_INDIVIDUAL) ? fd->fp_ind : 
-	    fd->disp + etype_size * offset;
-
-	file_list_count = 1;
-	file_offsets = off;
-	file_lengths = 0;
-	total_blks_to_read = count*flat_buf->count;
-	b_blks_read = 0;
-
-	/* allocate arrays according to max usage */
-	if (total_blks_to_read > MAX_ARRAY_SIZE)
-	    mem_list_count = MAX_ARRAY_SIZE;
-	else mem_list_count = total_blks_to_read;
-	mem_offsets = (char**)ADIOI_Malloc(mem_list_count*sizeof(char*));
-	mem_lengths = (int*)ADIOI_Malloc(mem_list_count*sizeof(int));
-
-	j = 0;
-	/* step through each block in memory, filling memory arrays */
-	while (b_blks_read < total_blks_to_read) {
-	    for (i=0; i<flat_buf->count; i++) {
-		mem_offsets[b_blks_read % MAX_ARRAY_SIZE] = 
-		    (char*)((char *)buf + j*buftype_extent + flat_buf->indices[i]);
-		mem_lengths[b_blks_read % MAX_ARRAY_SIZE] = 
-		    flat_buf->blocklens[i];
-		file_lengths += flat_buf->blocklens[i];
-		b_blks_read++;
-		if (!(b_blks_read % MAX_ARRAY_SIZE) ||
-		    (b_blks_read == total_blks_to_read)) {
-
-		    /* in the case of the last read list call,
-		       adjust mem_list_count */
-		    if (b_blks_read == total_blks_to_read) {
-		        mem_list_count = total_blks_to_read % MAX_ARRAY_SIZE;
-			/* in case last read list call fills max arrays */
-			if (!mem_list_count) mem_list_count = MAX_ARRAY_SIZE;
-		    }
-
-		    pvfs_read_list(fd->fd_sys ,mem_list_count, mem_offsets,
-				   mem_lengths, file_list_count,
-				   &file_offsets, &file_lengths);
-		  
-		    /* in the case of the last read list call, leave here */
-		    if (b_blks_read == total_blks_to_read) break;
-
-		    file_offsets += file_lengths;
-		    file_lengths = 0;
-		} 
-	    } /* for (i=0; i<flat_buf->count; i++) */
-	    j++;
-	} /* while (b_blks_read < total_blks_to_read) */
-	ADIOI_Free(mem_offsets);
-	ADIOI_Free(mem_lengths);
-
-        if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
-
-	fd->fp_sys_posn = -1;  /* set it to null. */
-
-#ifdef HAVE_STATUS_SET_BYTES
-	MPIR_Status_set_bytes(status, datatype, bufsize);
-	/* This isa temporary way of filling in status.  The right way is to
-	   keep tracke of how much data was actually read adn placed in buf
-	   by ADIOI_BUFFERED_READ. */
-#endif
-	ADIOI_Delete_flattened(datatype);
-
-	return;
-    } /* if (!buftype_is_contig && filetype_is_contig) */
-
-    /* know file is noncontiguous from above */
-    /* noncontiguous in file */
-
-    /* filetype already flattened in ADIO_Open */
-    flat_file = ADIOI_Flatlist;
-    while (flat_file->type != fd->filetype) flat_file = flat_file->next;
-
-    disp = fd->disp;
-
-    /* for each case - ADIO_Individual pointer or explicit, find the file
-       offset in bytes (offset), n_filetypes (how many filetypes into
-       file to start), frd_size (remaining amount of data in present
-       file block), and st_index (start point in terms of blocks in
-       starting filetype) */
-    if (file_ptr_type == ADIO_INDIVIDUAL) {
-        offset = fd->fp_ind; /* in bytes */
-	n_filetypes = -1;
-	flag = 0;
-	while (!flag) {
-	    n_filetypes++;
-	    for (i=0; i<flat_file->count; i++) {
-	        if (disp + flat_file->indices[i] + 
-		    (ADIO_Offset) n_filetypes*filetype_extent +
-		    flat_file->blocklens[i]  >= offset) {
-		    st_index = i;
-		    frd_size = (int) (disp + flat_file->indices[i] + 
-				      (ADIO_Offset) n_filetypes*filetype_extent
-				      + flat_file->blocklens[i] - offset);
-		    flag = 1;
-		    break;
-		}
-	    }
-	} /* while (!flag) */
-    } /* if (file_ptr_type == ADIO_INDIVIDUAL) */
-    else {
-        n_etypes_in_filetype = filetype_size/etype_size;
-	n_filetypes = (int) (offset / n_etypes_in_filetype);
-	etype_in_filetype = (int) (offset % n_etypes_in_filetype);
-	size_in_filetype = etype_in_filetype * etype_size;
-	
-	sum = 0;
-	for (i=0; i<flat_file->count; i++) {
-	    sum += flat_file->blocklens[i];
-	    if (sum > size_in_filetype) {
-	        st_index = i;
-		frd_size = sum - size_in_filetype;
-		abs_off_in_filetype = flat_file->indices[i] +
-		    size_in_filetype - (sum - flat_file->blocklens[i]);
-		break;
-	    }
-	}
-	
-	/* abs. offset in bytes in the file */
-	offset = disp + (ADIO_Offset) n_filetypes*filetype_extent + 
-	    abs_off_in_filetype;
-    } /* else [file_ptr_type != ADIO_INDIVIDUAL] */
-
-    start_off = offset;
-    st_frd_size = frd_size;
-    st_n_filetypes = n_filetypes;
-    
-    if (buftype_is_contig && !filetype_is_contig) {
-
-/* contiguous in memory, noncontiguous in file. should be the most
-   common case. */
-
-        int mem_lengths;
-	char *mem_offsets;
-	
-	i = 0;
-	j = st_index;
-	n_filetypes = st_n_filetypes;
-	
-	mem_list_count = 1;
-	
-	/* determine how many blocks in file to read */
-	f_data_read = ADIOI_MIN(st_frd_size, bufsize);
-	total_blks_to_read = 1;
-	j++;
-	while (f_data_read < bufsize) {
-	    f_data_read += flat_file->blocklens[j];
-	    total_blks_to_read++;
-	    if (j<(flat_file->count-1)) j++;
-	    else j = 0;	
-	}
-      
-	j = st_index;
-	n_filetypes = st_n_filetypes;
-	n_read_lists = total_blks_to_read/MAX_ARRAY_SIZE;
-	extra_blks = total_blks_to_read%MAX_ARRAY_SIZE;
-	
-	mem_offsets = buf;
-	mem_lengths = 0;
-	
-	/* if at least one full readlist, allocate file arrays
-	   at max array size and don't free until very end */
-	if (n_read_lists) {
-	    file_offsets = (int64_t*)ADIOI_Malloc(MAX_ARRAY_SIZE*
-						  sizeof(int64_t));
-	    file_lengths = (int32_t*)ADIOI_Malloc(MAX_ARRAY_SIZE*
-						  sizeof(int32_t));
-	}
-	/* if there's no full readlist allocate file arrays according
-	   to needed size (extra_blks) */
-	else {
-	    file_offsets = (int64_t*)ADIOI_Malloc(extra_blks*
-						  sizeof(int64_t));
-	    file_lengths = (int32_t*)ADIOI_Malloc(extra_blks*
-						  sizeof(int32_t));
-	}
-	
-	/* for file arrays that are of MAX_ARRAY_SIZE, build arrays */
-	for (i=0; i<n_read_lists; i++) {
-	    file_list_count = MAX_ARRAY_SIZE;
-	    if(!i) {
-	        file_offsets[0] = offset;
-		file_lengths[0] = st_frd_size;
-		mem_lengths = st_frd_size;
-	    }
-	    for (k=0; k<MAX_ARRAY_SIZE; k++) {
-	        if (i || k) {
-		    file_offsets[k] = disp + n_filetypes*filetype_extent
-		      + flat_file->indices[j];
-		    file_lengths[k] = flat_file->blocklens[j];
-		    mem_lengths += file_lengths[k];
-		}
-		if (j<(flat_file->count - 1)) j++;
-		else {
-		    j = 0;
-		    n_filetypes++;
-		}
-	    } /* for (k=0; k<MAX_ARRAY_SIZE; k++) */
-	    pvfs_read_list(fd->fd_sys, mem_list_count,
-			   &mem_offsets, &mem_lengths,
-			   file_list_count, file_offsets,
-			   file_lengths);
-	    mem_offsets += mem_lengths;
-	    mem_lengths = 0;
-	} /* for (i=0; i<n_read_lists; i++) */
-
-	/* for file arrays smaller than MAX_ARRAY_SIZE (last read_list call) */
-	if (extra_blks) {
-	    file_list_count = extra_blks;
-	    if(!i) {
-	        file_offsets[0] = offset;
-		file_lengths[0] = st_frd_size;
-	    }
-	    for (k=0; k<extra_blks; k++) {
-	        if(i || k) {
-		    file_offsets[k] = disp + n_filetypes*filetype_extent +
-		      flat_file->indices[j];
-		    if (k == (extra_blks - 1)) {
-		        file_lengths[k] = bufsize - (int32_t) mem_lengths
-			  - (int32_t) mem_offsets + (int32_t)  buf;
-		    }
-		    else file_lengths[k] = flat_file->blocklens[j];
-		} /* if(i || k) */
-		mem_lengths += file_lengths[k];
-		if (j<(flat_file->count - 1)) j++;
-		else {
-		    j = 0;
-		    n_filetypes++;
-		}
-	    } /* for (k=0; k<extra_blks; k++) */
-	    pvfs_read_list(fd->fd_sys, mem_list_count, &mem_offsets,
-			   &mem_lengths, file_list_count, file_offsets,
-			   file_lengths);
-	}
-    }
-    else {
-/* noncontiguous in memory as well as in file */
-      
-	flat_buf = ADIOI_Flatten_and_find(datatype);
-
-	size_read = 0;
-	n_filetypes = st_n_filetypes;
-	frd_size = st_frd_size;
-	brd_size = flat_buf->blocklens[0];
-	buf_count = 0;
-	start_mem_offset = 0;
-	start_k = k = 0;
-	start_j = st_index;
-	max_mem_list = 0;
-	max_file_list = 0;
-
-	/* run through and file max_file_list and max_mem_list so that you 
-	   can allocate the file and memory arrays less than MAX_ARRAY_SIZE
-	   if possible */
-
-	while (size_read < bufsize) {
-	    k = start_k;
-	    new_buffer_read = 0;
-	    mem_list_count = 0;
-	    while ((mem_list_count < MAX_ARRAY_SIZE) && 
-		   (new_buffer_read < bufsize-size_read)) {
-	        /* find mem_list_count and file_list_count such that both are
-		   less than MAX_ARRAY_SIZE, the sum of their lengths are
-		   equal, and the sum of all the data read and data to be
-		   read in the next immediate read list is less than
-		   bufsize */
-	        if(mem_list_count) {
-		    if((new_buffer_read + flat_buf->blocklens[k] + 
-			size_read) > bufsize) {
-		        end_brd_size = new_buffer_read + 
-			    flat_buf->blocklens[k] - (bufsize - size_read);
-			new_buffer_read = bufsize - size_read;
-		    }
-		    else {
-		        new_buffer_read += flat_buf->blocklens[k];
-			end_brd_size = flat_buf->blocklens[k];
-		    }
-		}
-		else {
-		    if (brd_size > (bufsize - size_read)) {
-		        new_buffer_read = bufsize - size_read;
-			brd_size = new_buffer_read;
-		    }
-		    else new_buffer_read = brd_size;
-		}
-		mem_list_count++;
-		k = (k + 1)%flat_buf->count;
-	     } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
-	       (new_buffer_read < bufsize-size_read)) */
-	    j = start_j;
-	    new_file_read = 0;
-	    file_list_count = 0;
-	    while ((file_list_count < MAX_ARRAY_SIZE) && 
-		   (new_file_read < new_buffer_read)) {
-	        if(file_list_count) {
-		    if((new_file_read + flat_file->blocklens[j]) > 
-		       new_buffer_read) {
-		        end_frd_size = new_buffer_read - new_file_read;
-			new_file_read = new_buffer_read;
-			j--;
-		    }
-		    else {
-		        new_file_read += flat_file->blocklens[j];
-			end_frd_size = flat_file->blocklens[j];
-		    }
-		}
-		else {
-		    if (frd_size > new_buffer_read) {
-		        new_file_read = new_buffer_read;
-			frd_size = new_file_read;
-		    }
-		    else new_file_read = frd_size;
-		}
-		file_list_count++;
-		if (j < (flat_file->count - 1)) j++;
-		else j = 0;
-		
-		k = start_k;
-		if ((new_file_read < new_buffer_read) && 
-		    (file_list_count == MAX_ARRAY_SIZE)) {
-		    new_buffer_read = 0;
-		    mem_list_count = 0;
-		    while (new_buffer_read < new_file_read) {
-		        if(mem_list_count) {
-			    if((new_buffer_read + flat_buf->blocklens[k]) >
-			       new_file_read) {
-			        end_brd_size = new_file_read - new_buffer_read;
-				new_buffer_read = new_file_read;
-				k--;
-			    }
-			    else {
-			        new_buffer_read += flat_buf->blocklens[k];
-				end_brd_size = flat_buf->blocklens[k];
-			    }
-			}
-			else {
-			    new_buffer_read = brd_size;
-			    if (brd_size > (bufsize - size_read)) {
-			        new_buffer_read = bufsize - size_read;
-				brd_size = new_buffer_read;
-			    }
-			}
-			mem_list_count++;
-			k = (k + 1)%flat_buf->count;
-		    } /* while (new_buffer_read < new_file_read) */
-		} /* if ((new_file_read < new_buffer_read) && (file_list_count
-		     == MAX_ARRAY_SIZE)) */
-	    } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
-		 (new_buffer_read < bufsize-size_read)) */
-
-	    /*  fakes filling the readlist arrays of lengths found above  */
-	    k = start_k;
-	    j = start_j;
-	    for (i=0; i<mem_list_count; i++) {	     
-		if(i) {
-		    if (i == (mem_list_count - 1)) {
-			if (flat_buf->blocklens[k] == end_brd_size)
-			    brd_size = flat_buf->blocklens[(k+1)%
-							  flat_buf->count];
-			else {
-			    brd_size = flat_buf->blocklens[k] - end_brd_size;
-			    k--;
-			    buf_count--;
-			}
-		    }
-		}
-		buf_count++;
-		k = (k + 1)%flat_buf->count;
-	    } /* for (i=0; i<mem_list_count; i++) */
-	    for (i=0; i<file_list_count; i++) {
-		if (i) {
-		    if (i == (file_list_count - 1)) {
-			if (flat_file->blocklens[j] == end_frd_size)
-			    frd_size = flat_file->blocklens[(j+1)%
-							  flat_file->count];   
-			else {
-			    frd_size = flat_file->blocklens[j] - end_frd_size;
-			    j--;
-			}
-		    }
-		}
-		if (j < flat_file->count - 1) j++;
-		else {
-		    j = 0;
-		    n_filetypes++;
-		}
-	    } /* for (i=0; i<file_list_count; i++) */
-	    size_read += new_buffer_read;
-	    start_k = k;
-	    start_j = j;
-	    if (max_mem_list < mem_list_count)
-	        max_mem_list = mem_list_count;
-	    if (max_file_list < file_list_count)
-	        max_file_list = file_list_count;
-	} /* while (size_read < bufsize) */
-
-	mem_offsets = (char **)ADIOI_Malloc(max_mem_list*sizeof(char *));
-	mem_lengths = (int *)ADIOI_Malloc(max_mem_list*sizeof(int));
-	file_offsets = (int64_t *)ADIOI_Malloc(max_file_list*sizeof(int64_t));
-	file_lengths = (int32_t *)ADIOI_Malloc(max_file_list*sizeof(int32_t));
-	    
-	size_read = 0;
-	n_filetypes = st_n_filetypes;
-	frd_size = st_frd_size;
-	brd_size = flat_buf->blocklens[0];
-	buf_count = 0;
-	start_mem_offset = 0;
-	start_k = k = 0;
-	start_j = st_index;
-
-	/*  this section calculates mem_list_count and file_list_count
-	    and also finds the possibly odd sized last array elements
-	    in new_frd_size and new_brd_size  */
-	
-	while (size_read < bufsize) {
-	    k = start_k;
-	    new_buffer_read = 0;
-	    mem_list_count = 0;
-	    while ((mem_list_count < MAX_ARRAY_SIZE) && 
-		   (new_buffer_read < bufsize-size_read)) {
-	        /* find mem_list_count and file_list_count such that both are
-		   less than MAX_ARRAY_SIZE, the sum of their lengths are
-		   equal, and the sum of all the data read and data to be
-		   read in the next immediate read list is less than
-		   bufsize */
-	        if(mem_list_count) {
-		    if((new_buffer_read + flat_buf->blocklens[k] + 
-			size_read) > bufsize) {
-		        end_brd_size = new_buffer_read + 
-			    flat_buf->blocklens[k] - (bufsize - size_read);
-			new_buffer_read = bufsize - size_read;
-		    }
-		    else {
-		        new_buffer_read += flat_buf->blocklens[k];
-			end_brd_size = flat_buf->blocklens[k];
-		    }
-		}
-		else {
-		    if (brd_size > (bufsize - size_read)) {
-		        new_buffer_read = bufsize - size_read;
-			brd_size = new_buffer_read;
-		    }
-		    else new_buffer_read = brd_size;
-		}
-		mem_list_count++;
-		k = (k + 1)%flat_buf->count;
-	     } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
-	       (new_buffer_read < bufsize-size_read)) */
-	    j = start_j;
-	    new_file_read = 0;
-	    file_list_count = 0;
-	    while ((file_list_count < MAX_ARRAY_SIZE) && 
-		   (new_file_read < new_buffer_read)) {
-	        if(file_list_count) {
-		    if((new_file_read + flat_file->blocklens[j]) > 
-		       new_buffer_read) {
-		        end_frd_size = new_buffer_read - new_file_read;
-			new_file_read = new_buffer_read;
-			j--;
-		    }
-		    else {
-		        new_file_read += flat_file->blocklens[j];
-			end_frd_size = flat_file->blocklens[j];
-		    }
-		}
-		else {
-		    if (frd_size > new_buffer_read) {
-		        new_file_read = new_buffer_read;
-			frd_size = new_file_read;
-		    }
-		    else new_file_read = frd_size;
-		}
-		file_list_count++;
-		if (j < (flat_file->count - 1)) j++;
-		else j = 0;
-		
-		k = start_k;
-		if ((new_file_read < new_buffer_read) && 
-		    (file_list_count == MAX_ARRAY_SIZE)) {
-		    new_buffer_read = 0;
-		    mem_list_count = 0;
-		    while (new_buffer_read < new_file_read) {
-		        if(mem_list_count) {
-			    if((new_buffer_read + flat_buf->blocklens[k]) >
-			       new_file_read) {
-			        end_brd_size = new_file_read - new_buffer_read;
-				new_buffer_read = new_file_read;
-				k--;
-			    }
-			    else {
-			        new_buffer_read += flat_buf->blocklens[k];
-				end_brd_size = flat_buf->blocklens[k];
-			    }
-			}
-			else {
-			    new_buffer_read = brd_size;
-			    if (brd_size > (bufsize - size_read)) {
-			        new_buffer_read = bufsize - size_read;
-				brd_size = new_buffer_read;
-			    }
-			}
-			mem_list_count++;
-			k = (k + 1)%flat_buf->count;
-		    } /* while (new_buffer_read < new_file_read) */
-		} /* if ((new_file_read < new_buffer_read) && (file_list_count
-		     == MAX_ARRAY_SIZE)) */
-	    } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
-		 (new_buffer_read < bufsize-size_read)) */
-
-	    /*  fills the allocated readlist arrays  */
-	    k = start_k;
-	    j = start_j;
-	    for (i=0; i<mem_list_count; i++) {	     
-	        mem_offsets[i] = (char*)((char *)buf + buftype_extent*
-					 (buf_count/flat_buf->count) +
-					 (int)flat_buf->indices[k]);
-		if(!i) {
-		    mem_lengths[0] = brd_size;
-		    mem_offsets[0] += flat_buf->blocklens[k] - brd_size;
-		}
-		else {
-		    if (i == (mem_list_count - 1)) {
-		        mem_lengths[i] = end_brd_size;
-			if (flat_buf->blocklens[k] == end_brd_size)
-			    brd_size = flat_buf->blocklens[(k+1)%
-							  flat_buf->count];
-			else {
-			    brd_size = flat_buf->blocklens[k] - end_brd_size;
-			    k--;
-			    buf_count--;
-			}
-		    }
-		    else {
-		        mem_lengths[i] = flat_buf->blocklens[k];
-		    }
-		}
-		buf_count++;
-		k = (k + 1)%flat_buf->count;
-	    } /* for (i=0; i<mem_list_count; i++) */
-	    for (i=0; i<file_list_count; i++) {
-	        file_offsets[i] = disp + flat_file->indices[j] + n_filetypes *
-		    filetype_extent;
-	        if (!i) {
-		    file_lengths[0] = frd_size;
-		    file_offsets[0] += flat_file->blocklens[j] - frd_size;
-		}
-		else {
-		    if (i == (file_list_count - 1)) {
-		        file_lengths[i] = end_frd_size;
-			if (flat_file->blocklens[j] == end_frd_size)
-			    frd_size = flat_file->blocklens[(j+1)%
-							  flat_file->count];   
-			else {
-			    frd_size = flat_file->blocklens[j] - end_frd_size;
-			    j--;
-			}
-		    }
-		    else file_lengths[i] = flat_file->blocklens[j];
-		}
-		if (j < flat_file->count - 1) j++;
-		else {
-		    j = 0;
-		    n_filetypes++;
-		}
-	    } /* for (i=0; i<file_list_count; i++) */
-
-	    /* 
-	    printf("about to call read_list in noncontig/noncontig\n");
-	    printf("offsets and lengths in terms of integers\n");
-	    printf("\nmem_list_count = %d\n", mem_list_count);
-	    for (i=0; i<mem_list_count; i++) {
-	      printf("mem_offsets[%2d] = %2d   ", i, (int)(mem_offsets[i] - (int)buf)/4);
-	      printf("mem_lengths[%2d] = %2d\n", i, mem_lengths[i]/4);
-	    }
-	    printf("\nfile_list_count = %d\n", file_list_count);
-	    for (i=0; i<file_list_count; i++) {
-	      printf("file_offsets[%2d] = %2d   ", i, (int)file_offsets[i]/4);
-	      printf("file_lengths[%2d] = %2d\n", i, file_lengths[i]/4);
-	    }
-	    printf("\n\n");
-	    */
-	    pvfs_read_list(fd->fd_sys,mem_list_count, mem_offsets,
-			   mem_lengths, file_list_count, file_offsets,
-			   file_lengths);
-	    size_read += new_buffer_read;
-	    start_k = k;
-	    start_j = j;
-	} /* while (size_read < bufsize) */
-	ADIOI_Free(mem_offsets);
-	ADIOI_Free(mem_lengths);
-    }
-    ADIOI_Free(file_offsets);
-    ADIOI_Free(file_lengths);
-    
-    if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
-    fd->fp_sys_posn = -1;   /* set it to null. */
-    
-#ifdef HAVE_STATUS_SET_BYTES
-    MPIR_Status_set_bytes(status, datatype, bufsize);
-    /* This is a temporary way of filling in status. The right way is to 
-       keep track of how much data was actually read and placed in buf 
-       by ADIOI_BUFFERED_READ. */
-#endif
-    
-    if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
-}
-#endif /* HAVE_PVFS_LISTIO */
diff --git a/3rd-party/romio321/adio/ad_pvfs/ad_pvfs_resize.c b/3rd-party/romio321/adio/ad_pvfs/ad_pvfs_resize.c
deleted file mode 100644
index b4b9553633f..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs/ad_pvfs_resize.c
+++ /dev/null
@@ -1,32 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_pvfs.h"
-
-void ADIOI_PVFS_Resize(ADIO_File fd, ADIO_Offset size, int *error_code)
-{
-    int err;
-    int rank;
-    static char myname[] = "ADIOI_PVFS_RESIZE";
-
-    /* because MPI_File_set_size is a collective operation, and PVFS1 clients
-     * do not cache metadata locally, one client can resize and broadcast the
-     * result to the others */
-    MPI_Comm_rank(fd->comm, &rank);
-    if (rank == fd->hints->ranklist[0]) {
-	err = pvfs_ftruncate64(fd->fd_sys, size);
-    }
-    MPI_Bcast(&err, 1, MPI_INT, fd->hints->ranklist[0], fd->comm);
-
-    if (err == -1) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io",
-					   "**io %s", strerror(errno));
-    }
-    else *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_pvfs/ad_pvfs_write.c b/3rd-party/romio321/adio/ad_pvfs/ad_pvfs_write.c
deleted file mode 100644
index 5f8d7e0f12e..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs/ad_pvfs_write.c
+++ /dev/null
@@ -1,1167 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_pvfs.h"
-#include "adio_extern.h"
-
-#ifdef HAVE_PVFS_LISTIO
-void ADIOI_PVFS_WriteStridedListIO(ADIO_File fd, void *buf, int count,
-                       MPI_Datatype datatype, int file_ptr_type,
-                       ADIO_Offset offset, ADIO_Status *status, int
-                       *error_code);
-#endif
-
-void ADIOI_PVFS_WriteContig(ADIO_File fd, void *buf, int count, 
-			    MPI_Datatype datatype, int file_ptr_type,
-			    ADIO_Offset offset, ADIO_Status *status,
-			    int *error_code)
-{
-    MPI_Count err=-1, datatype_size, len;
-    static char myname[] = "ADIOI_PVFS_WRITECONTIG";
-
-    MPI_Type_size_x(datatype, &datatype_size);
-    len = datatype_size * count;
-
-    if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-	if (fd->fp_sys_posn != offset) {
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-	    pvfs_lseek64(fd->fd_sys, offset, SEEK_SET);
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-        }
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
-#endif
-	err = pvfs_write(fd->fd_sys, buf, len);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
-#endif
-	if (err > 0)
-		fd->fp_sys_posn = offset + err;
-	/* individual file pointer not updated */        
-    }
-    else { /* write from curr. location of ind. file pointer */
-	if (fd->fp_sys_posn != fd->fp_ind) {
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-	    pvfs_lseek64(fd->fd_sys, fd->fp_ind, SEEK_SET);
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-        }
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
-#endif
-	err = pvfs_write(fd->fd_sys, buf, len);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
-#endif
-	if (err > 0)
-		fd->fp_ind += err;
-	fd->fp_sys_posn = fd->fp_ind;
-    }
-
-#ifdef HAVE_STATUS_SET_BYTES
-    if (err != -1) MPIR_Status_set_bytes(status, datatype, err);
-#endif
-
-    if (err == -1) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io",
-					   "**io %s", strerror(errno));
-    }
-    else *error_code = MPI_SUCCESS;
-}
-
-
-
-void ADIOI_PVFS_WriteStrided(ADIO_File fd, void *buf, int count,
-			     MPI_Datatype datatype, int file_ptr_type,
-			     ADIO_Offset offset, ADIO_Status *status, int
-			     *error_code)
-{
-/* Since PVFS does not support file locking, can't do buffered writes
-   as on Unix */
-
-/* offset is in units of etype relative to the filetype. */
-
-    ADIOI_Flatlist_node *flat_buf, *flat_file;
-    int i, j, k, err=-1, bwr_size, fwr_size=0, st_index=0;
-    int num, size, sum, n_etypes_in_filetype, size_in_filetype;
-    MPI_Count bufsize;
-    int n_filetypes, etype_in_filetype;
-    ADIO_Offset abs_off_in_filetype=0;
-    MPI_Count filetype_size, etype_size, buftype_size;
-    MPI_Aint filetype_extent, buftype_extent, indx, filetype_lb, buftype_lb;
-    int buf_count, buftype_is_contig, filetype_is_contig;
-    ADIO_Offset off, disp;
-    int flag, new_bwr_size, new_fwr_size, err_flag=0;
-    static char myname[] = "ADIOI_PVFS_WRITESTRIDED";
-
-#ifdef HAVE_PVFS_LISTIO
-    if ( fd->hints->fs_hints.pvfs.listio_write == ADIOI_HINT_ENABLE ) {
-	    ADIOI_PVFS_WriteStridedListIO(fd, buf, count, datatype, 
-			    file_ptr_type, offset, status, error_code);
-	    return;
-    }
-#endif
-    /* if hint set to DISABLE or AUTOMATIC, don't use listio */
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (fd->atomicity) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   MPI_ERR_INTERN,
-					   "Atomic mode set in PVFS I/O function", 0);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-
-    MPI_Type_size_x(fd->filetype, &filetype_size);
-    if ( ! filetype_size ) {
-#ifdef HAVE_STATUS_SET_BYTES
-	MPIR_Status_set_bytes(status, datatype, 0);
-#endif
-	*error_code = MPI_SUCCESS; 
-	return;
-    }
-
-    MPI_Type_get_extent(fd->filetype, &filetype_lb, &filetype_extent);
-    MPI_Type_size_x(datatype, &buftype_size);
-    MPI_Type_get_extent(datatype, &buftype_lb, &buftype_extent);
-    etype_size = fd->etype_size;
-    
-    bufsize = buftype_size * count;
-
-    if (!buftype_is_contig && filetype_is_contig) {
-	char *combine_buf, *combine_buf_ptr;
-	ADIO_Offset combine_buf_remain;
-/* noncontiguous in memory, contiguous in file. use writev */
-
-	flat_buf = ADIOI_Flatten_and_find(datatype);
-
-	/* allocate our "combine buffer" to pack data into before writing */
-	combine_buf = (char *) ADIOI_Malloc(fd->hints->ind_wr_buffer_size);
-	combine_buf_ptr = combine_buf;
-	combine_buf_remain = fd->hints->ind_wr_buffer_size;
-
-	/* seek to the right spot in the file */
-	if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-	    off = fd->disp + etype_size * offset;
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-	    pvfs_lseek64(fd->fd_sys, off, SEEK_SET);
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-	}
-	else {
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-            off = pvfs_lseek64(fd->fd_sys, fd->fp_ind, SEEK_SET);
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-        }
-
-	/* loop through all the flattened pieces.  combine into buffer until
-	 * no more will fit, then write.
-	 *
-	 * special case of a given piece being bigger than the combine buffer
-	 * is also handled.
-	 */
-	for (j=0; j<count; j++) {
-	    for (i=0; i<flat_buf->count; i++) {
-		if (flat_buf->blocklens[i] > combine_buf_remain && combine_buf != combine_buf_ptr) {
-		    /* there is data in the buffer; write out the buffer so far */
-#ifdef ADIOI_MPE_LOGGING
-                    MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
-#endif
-		    err = pvfs_write(fd->fd_sys,
-				     combine_buf,
-				     fd->hints->ind_wr_buffer_size - combine_buf_remain);
-#ifdef ADIOI_MPE_LOGGING
-                    MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
-#endif
-		    if (err == -1) err_flag = 1;
-
-		    /* reset our buffer info */
-		    combine_buf_ptr = combine_buf;
-		    combine_buf_remain = fd->hints->ind_wr_buffer_size;
-		}
-
-		/* TODO: heuristic for when to not bother to use combine buffer? */
-		if (flat_buf->blocklens[i] >= combine_buf_remain) {
-		    /* special case: blocklen is as big as or bigger than the combine buf;
-		     * write directly
-		     */
-#ifdef ADIOI_MPE_LOGGING
-                    MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
-#endif
-		    err = pvfs_write(fd->fd_sys,
-				     ((char *) buf) + j*buftype_extent + flat_buf->indices[i],
-				     flat_buf->blocklens[i]);
-#ifdef ADIOI_MPE_LOGGING
-                    MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
-#endif
-		    if (err == -1) err_flag = 1;
-		    off += flat_buf->blocklens[i]; /* keep up with the final file offset too */
-		}
-		else {
-		    /* copy more data into combine buffer */
-		    memcpy(combine_buf_ptr,
-			   ((char *) buf) + j*buftype_extent + flat_buf->indices[i],
-			   flat_buf->blocklens[i]);
-		    combine_buf_ptr += flat_buf->blocklens[i];
-		    combine_buf_remain -= flat_buf->blocklens[i];
-		    off += flat_buf->blocklens[i]; /* keep up with the final file offset too */
-		}
-	    }
-	}
-
-	if (combine_buf_ptr != combine_buf) {
-	    /* data left in buffer to write */
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
-#endif
-	    err = pvfs_write(fd->fd_sys,
-			     combine_buf,
-			     fd->hints->ind_wr_buffer_size - combine_buf_remain);
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
-#endif
-	    if (err == -1) err_flag = 1;
-	}
-
-	if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
-
-	ADIOI_Free(combine_buf);
-
-	if (err_flag) {
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE, myname,
-					       __LINE__, MPI_ERR_IO, "**io",
-					       "**io %s", strerror(errno));
-	}
-	else *error_code = MPI_SUCCESS;
-    } /* if (!buftype_is_contig && filetype_is_contig)  ... */
-
-    else {  /* noncontiguous in file */
-
-/* split up into several contiguous writes */
-
-/* find starting location in the file */
-
-/* filetype already flattened in ADIO_Open */
-	flat_file = ADIOI_Flatlist;
-	while (flat_file->type != fd->filetype) flat_file = flat_file->next;
-        disp = fd->disp;
-
-	if (file_ptr_type == ADIO_INDIVIDUAL) {
-	    offset = fd->fp_ind; /* in bytes */
-            n_filetypes = -1;
-            flag = 0;
-            while (!flag) {
-                n_filetypes++;
-                for (i=0; i<flat_file->count; i++) {
-                    if (disp + flat_file->indices[i] + 
-                        (ADIO_Offset) n_filetypes*filetype_extent + flat_file->blocklens[i] 
-                            >= offset) {
-                        st_index = i;
-                        fwr_size = disp + flat_file->indices[i] + 
-                                (ADIO_Offset) n_filetypes*filetype_extent
-                                 + flat_file->blocklens[i] - offset;
-                        flag = 1;
-                        break;
-                    }
-                }
-            }
-	}
-	else {
-	    n_etypes_in_filetype = filetype_size/etype_size;
-	    n_filetypes = (int) (offset / n_etypes_in_filetype);
-	    etype_in_filetype = (int) (offset % n_etypes_in_filetype);
-	    size_in_filetype = etype_in_filetype * etype_size;
- 
-	    sum = 0;
-	    for (i=0; i<flat_file->count; i++) {
-		sum += flat_file->blocklens[i];
-		if (sum > size_in_filetype) {
-		    st_index = i;
-		    fwr_size = sum - size_in_filetype;
-		    abs_off_in_filetype = flat_file->indices[i] +
-			size_in_filetype - (sum - flat_file->blocklens[i]);
-		    break;
-		}
-	    }
-
-	    /* abs. offset in bytes in the file */
-            offset = disp + (ADIO_Offset) n_filetypes*filetype_extent + abs_off_in_filetype;
-	}
-
-	if (buftype_is_contig && !filetype_is_contig) {
-
-/* contiguous in memory, noncontiguous in file. should be the most
-   common case. */
-
-	    i = 0;
-	    j = st_index;
-	    off = offset;
-	    fwr_size = ADIOI_MIN(fwr_size, bufsize);
-	    while (i < bufsize) {
-                if (fwr_size) { 
-                    /* TYPE_UB and TYPE_LB can result in 
-                       fwr_size = 0. save system call in such cases */ 
-#ifdef ADIOI_MPE_LOGGING
-                    MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-		    pvfs_lseek64(fd->fd_sys, off, SEEK_SET);
-#ifdef ADIOI_MPE_LOGGING
-                    MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-#ifdef ADIOI_MPE_LOGGING
-                    MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
-#endif
-		    err = pvfs_write(fd->fd_sys, ((char *) buf) + i, fwr_size);
-#ifdef ADIOI_MPE_LOGGING
-                    MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
-#endif
-		    if (err == -1) err_flag = 1;
-		}
-		i += fwr_size;
-
-                if (off + fwr_size < disp + flat_file->indices[j] +
-                   flat_file->blocklens[j] + (ADIO_Offset) n_filetypes*filetype_extent)
-                       off += fwr_size;
-                /* did not reach end of contiguous block in filetype.
-                   no more I/O needed. off is incremented by fwr_size. */
-                else {
-		    if (j < (flat_file->count - 1)) j++;
-		    else {
-			j = 0;
-			n_filetypes++;
-		    }
-		    off = disp + flat_file->indices[j] + 
-                                        (ADIO_Offset) n_filetypes*filetype_extent;
-		    fwr_size = ADIOI_MIN(flat_file->blocklens[j], bufsize-i);
-		}
-	    }
-	}
-	else {
-/* noncontiguous in memory as well as in file */
-
-	    flat_buf = ADIOI_Flatten_and_find(datatype);
-
-	    k = num = buf_count = 0;
-	    indx = flat_buf->indices[0];
-	    j = st_index;
-	    off = offset;
-	    bwr_size = flat_buf->blocklens[0];
-
-	    while (num < bufsize) {
-		size = ADIOI_MIN(fwr_size, bwr_size);
-		if (size) {
-#ifdef ADIOI_MPE_LOGGING
-                    MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-		    pvfs_lseek64(fd->fd_sys, off, SEEK_SET);
-#ifdef ADIOI_MPE_LOGGING
-                    MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-#ifdef ADIOI_MPE_LOGGING
-                    MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
-#endif
-		    err = pvfs_write(fd->fd_sys, ((char *) buf) + indx, size);
-#ifdef ADIOI_MPE_LOGGING
-                    MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
-#endif
-		    if (err == -1) err_flag = 1;
-		}
-
-		new_fwr_size = fwr_size;
-		new_bwr_size = bwr_size;
-
-		if (size == fwr_size) {
-/* reached end of contiguous block in file */
-                    if (j < (flat_file->count - 1)) j++;
-                    else {
-                        j = 0;
-                        n_filetypes++;
-                    }
-
-                    off = disp + flat_file->indices[j] + 
-                                   (ADIO_Offset) n_filetypes*filetype_extent;
-
-		    new_fwr_size = flat_file->blocklens[j];
-		    if (size != bwr_size) {
-			indx += size;
-			new_bwr_size -= size;
-		    }
-		}
-
-		if (size == bwr_size) {
-/* reached end of contiguous block in memory */
-
-		    k = (k + 1)%flat_buf->count;
-		    buf_count++;
-		    indx = buftype_extent*(buf_count/flat_buf->count) +
-			flat_buf->indices[k]; 
-		    new_bwr_size = flat_buf->blocklens[k];
-		    if (size != fwr_size) {
-			off += size;
-			new_fwr_size -= size;
-		    }
-		}
-		num += size;
-		fwr_size = new_fwr_size;
-                bwr_size = new_bwr_size;
-	    }
-	}
-
-        if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
-	if (err_flag) {
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE, myname,
-					       __LINE__, MPI_ERR_IO, "**io",
-					       "**io %s", strerror(errno));
-	}
-	else *error_code = MPI_SUCCESS;
-    }
-
-    fd->fp_sys_posn = -1;   /* set it to null. */
-
-#ifdef HAVE_STATUS_SET_BYTES
-    MPIR_Status_set_bytes(status, datatype, bufsize);
-/* This is a temporary way of filling in status. The right way is to 
-   keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
-#endif
-
-    if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
-}
-
-#ifdef HAVE_PVFS_LISTIO
-void ADIOI_PVFS_WriteStridedListIO(ADIO_File fd, void *buf, int count,
-                       MPI_Datatype datatype, int file_ptr_type,
-                       ADIO_Offset offset, ADIO_Status *status, int
-                       *error_code) 
-{
-/* Since PVFS does not support file locking, can't do buffered writes
-   as on Unix */
-
-/* offset is in units of etype relative to the filetype. */
-
-    ADIOI_Flatlist_node *flat_buf, *flat_file;
-    int i, j, k, err=-1, bwr_size, fwr_size=0, st_index=0;
-    int size, sum, n_etypes_in_filetype, size_in_filetype;
-    MPI_Count bufsize;
-    int n_filetypes, etype_in_filetype;
-    ADIO_Offset abs_off_in_filetype=0;
-    MPI_Count filetype_size, etype_size, buftype_size;
-    MPI_Aint filetype_extent, buftype_extent, filetype_lb, buftype_lb;
-    int buf_count, buftype_is_contig, filetype_is_contig;
-    ADIO_Offset userbuf_off;
-    ADIO_Offset off, disp, start_off;
-    int flag, st_fwr_size, st_n_filetypes;
-    int new_bwr_size, new_fwr_size, err_flag=0;
-
-    int mem_list_count, file_list_count;
-    char ** mem_offsets;
-    int64_t *file_offsets;
-    int *mem_lengths;
-    int32_t *file_lengths;
-    int total_blks_to_write;
-
-    int max_mem_list, max_file_list;
-
-    int b_blks_wrote;
-    int f_data_wrote;
-    int size_wrote=0, n_write_lists, extra_blks;
-
-    int end_bwr_size, end_fwr_size;
-    int start_k, start_j, new_file_write, new_buffer_write;
-    int start_mem_offset;
-#define MAX_ARRAY_SIZE 1024
-    static char myname[] = "ADIOI_PVFS_WRITESTRIDED";
-
-/* PFS file pointer modes are not relevant here, because PFS does
-   not support strided accesses. */
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (fd->atomicity) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   MPI_ERR_INTERN,
-					   "Atomic mode set in PVFS I/O function", 0);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-
-    MPI_Type_size_x(fd->filetype, &filetype_size);
-    if ( ! filetype_size ) {
-#ifdef HAVE_STATUS_SET_BYTES
-	MPIR_Status_set_bytes(status, datatype, 0);
-#endif
-	*error_code = MPI_SUCCESS; 
-	return;
-    }
-
-    MPI_Type_get_extent(fd->filetype, &filetype_lb, &filetype_extent);
-    MPI_Type_size_x(datatype, &buftype_size);
-    MPI_Type_get_extent(datatype, &buftype_lb, &buftype_extent);
-    etype_size = fd->etype_size;
-    
-    bufsize = buftype_size * count;
-
-    if (!buftype_is_contig && filetype_is_contig) {
-
-/* noncontiguous in memory, contiguous in file.  */
-        int64_t file_offsets;
-	int32_t file_lengths;
-
-	flat_buf = ADIOI_Flatten_and_find(datatype);
-	
-	if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-	    off = fd->disp + etype_size * offset;
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-	    pvfs_lseek64(fd->fd_sys, fd->fp_ind, SEEK_SET);
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-	}
-	else {
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-            off = pvfs_lseek64(fd->fd_sys, fd->fp_ind, SEEK_SET);
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-        }
-
-	file_list_count = 1;
-	file_offsets = off;
-	file_lengths = 0;
-	total_blks_to_write = count*flat_buf->count;
-	b_blks_wrote = 0;
-
-	/* allocate arrays according to max usage */
-	if (total_blks_to_write > MAX_ARRAY_SIZE)
-	    mem_list_count = MAX_ARRAY_SIZE;
-	else mem_list_count = total_blks_to_write;
-	mem_offsets = (char**)ADIOI_Malloc(mem_list_count*sizeof(char*));
-	mem_lengths = (int*)ADIOI_Malloc(mem_list_count*sizeof(int));
-
-	j = 0;
-	/* step through each block in memory, filling memory arrays */
-	while (b_blks_wrote < total_blks_to_write) {
-	    for (i=0; i<flat_buf->count; i++) {
-		mem_offsets[b_blks_wrote % MAX_ARRAY_SIZE] = 
-		    ((char*)buf + j*buftype_extent + flat_buf->indices[i]);
-		mem_lengths[b_blks_wrote % MAX_ARRAY_SIZE] = 
-		    flat_buf->blocklens[i];
-		file_lengths += flat_buf->blocklens[i];
-		b_blks_wrote++;
-		if (!(b_blks_wrote % MAX_ARRAY_SIZE) ||
-		    (b_blks_wrote == total_blks_to_write)) {
-
-		    /* in the case of the last read list call,
-		       adjust mem_list_count */
-		    if (b_blks_wrote == total_blks_to_write) {
-		        mem_list_count = total_blks_to_write % MAX_ARRAY_SIZE;
-			/* in case last read list call fills max arrays */
-			if (!mem_list_count) mem_list_count = MAX_ARRAY_SIZE;
-		    }
-
-		    pvfs_write_list(fd->fd_sys ,mem_list_count, mem_offsets,
-				   mem_lengths, file_list_count,
-				   &file_offsets, &file_lengths);
-		  
-		    /* in the case of the last read list call, leave here */
-		    if (b_blks_wrote == total_blks_to_write) break;
-
-		    file_offsets += file_lengths;
-		    file_lengths = 0;
-		} 
-	    } /* for (i=0; i<flat_buf->count; i++) */
-	    j++;
-	} /* while (b_blks_wrote < total_blks_to_write) */
-	ADIOI_Free(mem_offsets);
-	ADIOI_Free(mem_lengths);
-
-	if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
-
-	if (err_flag) {
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE, myname,
-					       __LINE__, MPI_ERR_IO, "**io",
-					       "**io %s", strerror(errno));
-	}
-	else *error_code = MPI_SUCCESS;
-
-	fd->fp_sys_posn = -1;   /* clear this. */
-
-#ifdef HAVE_STATUS_SET_BYTES
-	MPIR_Status_set_bytes(status, datatype, bufsize);
-/* This is a temporary way of filling in status. The right way is to 
-   keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
-#endif
-
-	ADIOI_Delete_flattened(datatype);
-	return;
-    } /* if (!buftype_is_contig && filetype_is_contig) */
-
-    /* already know that file is noncontiguous from above */
-    /* noncontiguous in file */
-
-/* filetype already flattened in ADIO_Open */
-    flat_file = ADIOI_Flatlist;
-    while (flat_file->type != fd->filetype) flat_file = flat_file->next;
-
-    disp = fd->disp;
-
-    /* for each case - ADIO_Individual pointer or explicit, find offset
-       (file offset in bytes), n_filetypes (how many filetypes into file 
-       to start), fwr_size (remaining amount of data in present file
-       block), and st_index (start point in terms of blocks in starting
-       filetype) */
-    if (file_ptr_type == ADIO_INDIVIDUAL) {
-        offset = fd->fp_ind; /* in bytes */
-	n_filetypes = -1;
-	flag = 0;
-	while (!flag) {
-	    n_filetypes++;
-	    for (i=0; i<flat_file->count; i++) {
-	        if (disp + flat_file->indices[i] + 
-		    (ADIO_Offset) n_filetypes*filetype_extent +
-		      flat_file->blocklens[i] >= offset) {
-		  st_index = i;
-		  fwr_size = disp + flat_file->indices[i] + 
-		    (ADIO_Offset) n_filetypes*filetype_extent
-		    + flat_file->blocklens[i] - offset;
-		  flag = 1;
-		  break;
-		}
-	    }
-	} /* while (!flag) */
-    } /* if (file_ptr_type == ADIO_INDIVIDUAL) */
-    else {
-        n_etypes_in_filetype = filetype_size/etype_size;
-	n_filetypes = (int) (offset / n_etypes_in_filetype);
-	etype_in_filetype = (int) (offset % n_etypes_in_filetype);
-	size_in_filetype = etype_in_filetype * etype_size;
-	
-	sum = 0;
-	for (i=0; i<flat_file->count; i++) {
-	    sum += flat_file->blocklens[i];
-	    if (sum > size_in_filetype) {
-	        st_index = i;
-		fwr_size = sum - size_in_filetype;
-		abs_off_in_filetype = flat_file->indices[i] +
-		    size_in_filetype - (sum - flat_file->blocklens[i]);
-		break;
-	    }
-	}
-
-	/* abs. offset in bytes in the file */
-	offset = disp + (ADIO_Offset) n_filetypes*filetype_extent +
-	    abs_off_in_filetype;
-    } /* else [file_ptr_type != ADIO_INDIVIDUAL] */
-
-    start_off = offset;
-    st_fwr_size = fwr_size;
-    st_n_filetypes = n_filetypes;
-    
-    if (buftype_is_contig && !filetype_is_contig) {
-
-/* contiguous in memory, noncontiguous in file. should be the most
-   common case. */
-
-        int mem_lengths;
-	char *mem_offsets;
-        
-	i = 0;
-	j = st_index;
-	off = offset;
-	n_filetypes = st_n_filetypes;
-        
-	mem_list_count = 1;
-        
-	/* determine how many blocks in file to read */
-	f_data_wrote = ADIOI_MIN(st_fwr_size, bufsize);
-	total_blks_to_write = 1;
-	j++;
-	while (f_data_wrote < bufsize) {
-	    f_data_wrote += flat_file->blocklens[j];
-	    total_blks_to_write++;
-	    if (j<(flat_file->count-1)) j++;
-	    else j = 0; 
-	}
-	    
-	j = st_index;
-	n_filetypes = st_n_filetypes;
-	n_write_lists = total_blks_to_write/MAX_ARRAY_SIZE;
-	extra_blks = total_blks_to_write%MAX_ARRAY_SIZE;
-        
-	mem_offsets = buf;
-	mem_lengths = 0;
-        
-	/* if at least one full readlist, allocate file arrays
-	   at max array size and don't free until very end */
-	if (n_write_lists) {
-	    file_offsets = (int64_t*)ADIOI_Malloc(MAX_ARRAY_SIZE*
-						  sizeof(int64_t));
-	    file_lengths = (int32_t*)ADIOI_Malloc(MAX_ARRAY_SIZE*
-						  sizeof(int32_t));
-	}
-	/* if there's no full readlist allocate file arrays according
-	   to needed size (extra_blks) */
-	else {
-	    file_offsets = (int64_t*)ADIOI_Malloc(extra_blks*
-                                                  sizeof(int64_t));
-            file_lengths = (int32_t*)ADIOI_Malloc(extra_blks*
-                                                  sizeof(int32_t));
-        }
-        
-        /* for file arrays that are of MAX_ARRAY_SIZE, build arrays */
-        for (i=0; i<n_write_lists; i++) {
-            file_list_count = MAX_ARRAY_SIZE;
-            if(!i) {
-                file_offsets[0] = offset;
-                file_lengths[0] = st_fwr_size;
-                mem_lengths = st_fwr_size;
-            }
-            for (k=0; k<MAX_ARRAY_SIZE; k++) {
-                if (i || k) {
-                    file_offsets[k] = disp + n_filetypes*filetype_extent
-                      + flat_file->indices[j];
-                    file_lengths[k] = flat_file->blocklens[j];
-                    mem_lengths += file_lengths[k];
-                }
-                if (j<(flat_file->count - 1)) j++;
-                else {
-                    j = 0;
-                    n_filetypes++;
-                }
-            } /* for (k=0; k<MAX_ARRAY_SIZE; k++) */
-            pvfs_write_list(fd->fd_sys, mem_list_count,
-                           &mem_offsets, &mem_lengths,
-                           file_list_count, file_offsets,
-                           file_lengths);
-            mem_offsets += mem_lengths;
-            mem_lengths = 0;
-        } /* for (i=0; i<n_write_lists; i++) */
-
-        /* for file arrays smaller than MAX_ARRAY_SIZE (last read_list call) */
-        if (extra_blks) {
-            file_list_count = extra_blks;
-            if(!i) {
-                file_offsets[0] = offset;
-                file_lengths[0] = st_fwr_size;
-            }
-            for (k=0; k<extra_blks; k++) {
-                if(i || k) {
-                    file_offsets[k] = disp + n_filetypes*filetype_extent +
-                      flat_file->indices[j];
-                    if (k == (extra_blks - 1)) {
-                        file_lengths[k] = bufsize - (int32_t) mem_lengths
-                          - (int32_t) mem_offsets + (int32_t)  buf;
-                    }
-                    else file_lengths[k] = flat_file->blocklens[j];
-                } /* if(i || k) */
-                mem_lengths += file_lengths[k];
-                if (j<(flat_file->count - 1)) j++;
-                else {
-                    j = 0;
-                    n_filetypes++;
-                }
-            } /* for (k=0; k<extra_blks; k++) */
-            pvfs_write_list(fd->fd_sys, mem_list_count, &mem_offsets,
-                           &mem_lengths, file_list_count, file_offsets,
-                           file_lengths);
-        }
-    } 
-    else {
-        /* noncontiguous in memory as well as in file */
-
-	flat_buf = ADIOI_Flatten_and_find(datatype);
-
-	size_wrote = 0;
-	n_filetypes = st_n_filetypes;
-	fwr_size = st_fwr_size;
-	bwr_size = flat_buf->blocklens[0];
-	buf_count = 0;
-	start_mem_offset = 0;
-	start_k = k = 0;
-	start_j = st_index;
-	max_mem_list = 0;
-	max_file_list = 0;
-
-	/* run through and file max_file_list and max_mem_list so that you 
-	   can allocate the file and memory arrays less than MAX_ARRAY_SIZE
-	   if possible */
-
-	while (size_wrote < bufsize) {
-	    k = start_k;
-	    new_buffer_write = 0;
-	    mem_list_count = 0;
-	    while ((mem_list_count < MAX_ARRAY_SIZE) && 
-		   (new_buffer_write < bufsize-size_wrote)) {
-	        /* find mem_list_count and file_list_count such that both are
-		   less than MAX_ARRAY_SIZE, the sum of their lengths are
-		   equal, and the sum of all the data read and data to be
-		   read in the next immediate read list is less than
-		   bufsize */
-	        if(mem_list_count) {
-		    if((new_buffer_write + flat_buf->blocklens[k] + 
-			size_wrote) > bufsize) {
-		        end_bwr_size = new_buffer_write + 
-			    flat_buf->blocklens[k] - (bufsize - size_wrote);
-			new_buffer_write = bufsize - size_wrote;
-		    }
-		    else {
-		        new_buffer_write += flat_buf->blocklens[k];
-			end_bwr_size = flat_buf->blocklens[k];
-		    }
-		}
-		else {
-		    if (bwr_size > (bufsize - size_wrote)) {
-		        new_buffer_write = bufsize - size_wrote;
-			bwr_size = new_buffer_write;
-		    }
-		    else new_buffer_write = bwr_size;
-		}
-		mem_list_count++;
-		k = (k + 1)%flat_buf->count;
-	     } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
-	       (new_buffer_write < bufsize-size_wrote)) */
-	    j = start_j;
-	    new_file_write = 0;
-	    file_list_count = 0;
-	    while ((file_list_count < MAX_ARRAY_SIZE) && 
-		   (new_file_write < new_buffer_write)) {
-	        if(file_list_count) {
-		    if((new_file_write + flat_file->blocklens[j]) > 
-		       new_buffer_write) {
-		        end_fwr_size = new_buffer_write - new_file_write;
-			new_file_write = new_buffer_write;
-			j--;
-		    }
-		    else {
-		        new_file_write += flat_file->blocklens[j];
-			end_fwr_size = flat_file->blocklens[j];
-		    }
-		}
-		else {
-		    if (fwr_size > new_buffer_write) {
-		        new_file_write = new_buffer_write;
-			fwr_size = new_file_write;
-		    }
-		    else new_file_write = fwr_size;
-		}
-		file_list_count++;
-		if (j < (flat_file->count - 1)) j++;
-		else j = 0;
-		
-		k = start_k;
-		if ((new_file_write < new_buffer_write) && 
-		    (file_list_count == MAX_ARRAY_SIZE)) {
-		    new_buffer_write = 0;
-		    mem_list_count = 0;
-		    while (new_buffer_write < new_file_write) {
-		        if(mem_list_count) {
-			    if((new_buffer_write + flat_buf->blocklens[k]) >
-			       new_file_write) {
-			        end_bwr_size = new_file_write - 
-				    new_buffer_write;
-				new_buffer_write = new_file_write;
-				k--;
-			    }
-			    else {
-			        new_buffer_write += flat_buf->blocklens[k];
-				end_bwr_size = flat_buf->blocklens[k];
-			    }
-			}
-			else {
-			    new_buffer_write = bwr_size;
-			    if (bwr_size > (bufsize - size_wrote)) {
-			        new_buffer_write = bufsize - size_wrote;
-				bwr_size = new_buffer_write;
-			    }
-			}
-			mem_list_count++;
-			k = (k + 1)%flat_buf->count;
-		    } /* while (new_buffer_write < new_file_write) */
-		} /* if ((new_file_write < new_buffer_write) &&
-		     (file_list_count == MAX_ARRAY_SIZE)) */
-	    } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
-		 (new_buffer_write < bufsize-size_wrote)) */
-
-	    /*  fakes filling the writelist arrays of lengths found above  */
-	    k = start_k;
-	    j = start_j;
-	    for (i=0; i<mem_list_count; i++) {	     
-		if(i) {
-		    if (i == (mem_list_count - 1)) {
-			if (flat_buf->blocklens[k] == end_bwr_size)
-			    bwr_size = flat_buf->blocklens[(k+1)%
-							  flat_buf->count];
-			else {
-			    bwr_size = flat_buf->blocklens[k] - end_bwr_size;
-			    k--;
-			    buf_count--;
-			}
-		    }
-		}
-		buf_count++;
-		k = (k + 1)%flat_buf->count;
-	    } /* for (i=0; i<mem_list_count; i++) */
-	    for (i=0; i<file_list_count; i++) {
-		if (i) {
-		    if (i == (file_list_count - 1)) {
-			if (flat_file->blocklens[j] == end_fwr_size)
-			    fwr_size = flat_file->blocklens[(j+1)%
-							  flat_file->count];   
-			else {
-			    fwr_size = flat_file->blocklens[j] - end_fwr_size;
-			    j--;
-			}
-		    }
-		}
-		if (j < flat_file->count - 1) j++;
-		else {
-		    j = 0;
-		    n_filetypes++;
-		}
-	    } /* for (i=0; i<file_list_count; i++) */
-	    size_wrote += new_buffer_write;
-	    start_k = k;
-	    start_j = j;
-	    if (max_mem_list < mem_list_count)
-	        max_mem_list = mem_list_count;
-	    if (max_file_list < file_list_count)
-	        max_file_list = file_list_count;
-	} /* while (size_wrote < bufsize) */
-
-	mem_offsets = (char **)ADIOI_Malloc(max_mem_list*sizeof(char *));
-	mem_lengths = (int *)ADIOI_Malloc(max_mem_list*sizeof(int));
-	file_offsets = (int64_t *)ADIOI_Malloc(max_file_list*sizeof(int64_t));
-	file_lengths = (int32_t *)ADIOI_Malloc(max_file_list*sizeof(int32_t));
-	    
-	size_wrote = 0;
-	n_filetypes = st_n_filetypes;
-	fwr_size = st_fwr_size;
-	bwr_size = flat_buf->blocklens[0];
-	buf_count = 0;
-	start_mem_offset = 0;
-	start_k = k = 0;
-	start_j = st_index;
-
-	/*  this section calculates mem_list_count and file_list_count
-	    and also finds the possibly odd sized last array elements
-	    in new_fwr_size and new_bwr_size  */
-	
-	while (size_wrote < bufsize) {
-	    k = start_k;
-	    new_buffer_write = 0;
-	    mem_list_count = 0;
-	    while ((mem_list_count < MAX_ARRAY_SIZE) && 
-		   (new_buffer_write < bufsize-size_wrote)) {
-	        /* find mem_list_count and file_list_count such that both are
-		   less than MAX_ARRAY_SIZE, the sum of their lengths are
-		   equal, and the sum of all the data read and data to be
-		   read in the next immediate read list is less than
-		   bufsize */
-	        if(mem_list_count) {
-		    if((new_buffer_write + flat_buf->blocklens[k] + 
-			size_wrote) > bufsize) {
-		        end_bwr_size = new_buffer_write + 
-			    flat_buf->blocklens[k] - (bufsize - size_wrote);
-			new_buffer_write = bufsize - size_wrote;
-		    }
-		    else {
-		        new_buffer_write += flat_buf->blocklens[k];
-			end_bwr_size = flat_buf->blocklens[k];
-		    }
-		}
-		else {
-		    if (bwr_size > (bufsize - size_wrote)) {
-		        new_buffer_write = bufsize - size_wrote;
-			bwr_size = new_buffer_write;
-		    }
-		    else new_buffer_write = bwr_size;
-		}
-		mem_list_count++;
-		k = (k + 1)%flat_buf->count;
-	     } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
-	       (new_buffer_write < bufsize-size_wrote)) */
-	    j = start_j;
-	    new_file_write = 0;
-	    file_list_count = 0;
-	    while ((file_list_count < MAX_ARRAY_SIZE) && 
-		   (new_file_write < new_buffer_write)) {
-	        if(file_list_count) {
-		    if((new_file_write + flat_file->blocklens[j]) > 
-		       new_buffer_write) {
-		        end_fwr_size = new_buffer_write - new_file_write;
-			new_file_write = new_buffer_write;
-			j--;
-		    }
-		    else {
-		        new_file_write += flat_file->blocklens[j];
-			end_fwr_size = flat_file->blocklens[j];
-		    }
-		}
-		else {
-		    if (fwr_size > new_buffer_write) {
-		        new_file_write = new_buffer_write;
-			fwr_size = new_file_write;
-		    }
-		    else new_file_write = fwr_size;
-		}
-		file_list_count++;
-		if (j < (flat_file->count - 1)) j++;
-		else j = 0;
-		
-		k = start_k;
-		if ((new_file_write < new_buffer_write) && 
-		    (file_list_count == MAX_ARRAY_SIZE)) {
-		    new_buffer_write = 0;
-		    mem_list_count = 0;
-		    while (new_buffer_write < new_file_write) {
-		        if(mem_list_count) {
-			    if((new_buffer_write + flat_buf->blocklens[k]) >
-			       new_file_write) {
-			        end_bwr_size = new_file_write -
-				  new_buffer_write;
-				new_buffer_write = new_file_write;
-				k--;
-			    }
-			    else {
-			        new_buffer_write += flat_buf->blocklens[k];
-				end_bwr_size = flat_buf->blocklens[k];
-			    }
-			}
-			else {
-			    new_buffer_write = bwr_size;
-			    if (bwr_size > (bufsize - size_wrote)) {
-			        new_buffer_write = bufsize - size_wrote;
-				bwr_size = new_buffer_write;
-			    }
-			}
-			mem_list_count++;
-			k = (k + 1)%flat_buf->count;
-		    } /* while (new_buffer_write < new_file_write) */
-		} /* if ((new_file_write < new_buffer_write) &&
-		     (file_list_count == MAX_ARRAY_SIZE)) */
-	    } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
-		 (new_buffer_write < bufsize-size_wrote)) */
-
-	    /*  fills the allocated readlist arrays  */
-	    k = start_k;
-	    j = start_j;
-	    for (i=0; i<mem_list_count; i++) {	     
-	        mem_offsets[i] = ((char*)buf + buftype_extent*
-					 (buf_count/flat_buf->count) +
-					 (int)flat_buf->indices[k]);
-		
-		if(!i) {
-		    mem_lengths[0] = bwr_size;
-		    mem_offsets[0] += flat_buf->blocklens[k] - bwr_size;
-		}
-		else {
-		    if (i == (mem_list_count - 1)) {
-		        mem_lengths[i] = end_bwr_size;
-			if (flat_buf->blocklens[k] == end_bwr_size)
-			    bwr_size = flat_buf->blocklens[(k+1)%
-							  flat_buf->count];
-			else {
-			    bwr_size = flat_buf->blocklens[k] - end_bwr_size;
-			    k--;
-			    buf_count--;
-			}
-		    }
-		    else {
-		        mem_lengths[i] = flat_buf->blocklens[k];
-		    }
-		}
-		buf_count++;
-		k = (k + 1)%flat_buf->count;
-	    } /* for (i=0; i<mem_list_count; i++) */
-	    for (i=0; i<file_list_count; i++) {
-	        file_offsets[i] = disp + flat_file->indices[j] + n_filetypes *
-		    filetype_extent;
-	        if (!i) {
-		    file_lengths[0] = fwr_size;
-		    file_offsets[0] += flat_file->blocklens[j] - fwr_size;
-		}
-		else {
-		    if (i == (file_list_count - 1)) {
-		        file_lengths[i] = end_fwr_size;
-			if (flat_file->blocklens[j] == end_fwr_size)
-			    fwr_size = flat_file->blocklens[(j+1)%
-							  flat_file->count];   
-			else {
-			    fwr_size = flat_file->blocklens[j] - end_fwr_size;
-			    j--;
-			}
-		    }
-		    else file_lengths[i] = flat_file->blocklens[j];
-		}
-		if (j < flat_file->count - 1) j++;
-		else {
-		    j = 0;
-		    n_filetypes++;
-		}
-	    } /* for (i=0; i<file_list_count; i++) */
-	    pvfs_write_list(fd->fd_sys,mem_list_count, mem_offsets,
-			   mem_lengths, file_list_count, file_offsets,
-			   file_lengths);
-	    size_wrote += new_buffer_write;
-	    start_k = k;
-	    start_j = j;
-	} /* while (size_wrote < bufsize) */
-	ADIOI_Free(mem_offsets);
-	ADIOI_Free(mem_lengths);
-    }
-    ADIOI_Free(file_offsets);
-    ADIOI_Free(file_lengths);
-
-    if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
-    if (err_flag) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**io",
-					   "**io %s", strerror(errno));
-    }
-    else *error_code = MPI_SUCCESS;
-
-    fd->fp_sys_posn = -1;   /* set it to null. */
-
-#ifdef HAVE_STATUS_SET_BYTES
-    MPIR_Status_set_bytes(status, datatype, bufsize);
-/* This is a temporary way of filling in status. The right way is to 
-   keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
-#endif
-
-    if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
-}
-#endif /* HAVE_PVFS_LISTIO */
diff --git a/3rd-party/romio321/adio/ad_pvfs2/Makefile.mk b/3rd-party/romio321/adio/ad_pvfs2/Makefile.mk
deleted file mode 100644
index 4ee9d0fd362..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs2/Makefile.mk
+++ /dev/null
@@ -1,34 +0,0 @@
-## -*- Mode: Makefile; -*-
-## vim: set ft=automake :
-##
-## (C) 2011 by Argonne National Laboratory.
-##     See COPYRIGHT in top-level directory.
-##
-
-if BUILD_AD_PVFS2
-
-noinst_HEADERS += adio/ad_pvfs2/ad_pvfs2.h \
-		  adio/ad_pvfs2/ad_pvfs2_io.h \
-		  adio/ad_pvfs2/ad_pvfs2_common.h
-
-romio_other_sources +=                          \
-    adio/ad_pvfs2/ad_pvfs2_close.c              \
-    adio/ad_pvfs2/ad_pvfs2_read.c               \
-    adio/ad_pvfs2/ad_pvfs2_open.c               \
-    adio/ad_pvfs2/ad_pvfs2_write.c              \
-    adio/ad_pvfs2/ad_pvfs2_fcntl.c              \
-    adio/ad_pvfs2/ad_pvfs2_flush.c              \
-    adio/ad_pvfs2/ad_pvfs2_resize.c             \
-    adio/ad_pvfs2/ad_pvfs2_hints.c              \
-    adio/ad_pvfs2/ad_pvfs2_delete.c             \
-    adio/ad_pvfs2/ad_pvfs2.c                    \
-    adio/ad_pvfs2/ad_pvfs2_common.c             \
-    adio/ad_pvfs2/ad_pvfs2_aio.c                \
-    adio/ad_pvfs2/ad_pvfs2_read_list_classic.c  \
-    adio/ad_pvfs2/ad_pvfs2_io_list.c            \
-    adio/ad_pvfs2/ad_pvfs2_io_dtype.c           \
-    adio/ad_pvfs2/ad_pvfs2_write_list_classic.c \
-    adio/ad_pvfs2/ad_pvfs2_features.c
-
-endif BUILD_AD_PVFS2
-
diff --git a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2.c b/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2.c
deleted file mode 100644
index bdebe19d37a..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2.c
+++ /dev/null
@@ -1,49 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2003 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_pvfs2.h"
-
-#include "adio.h"
-
-struct ADIOI_Fns_struct ADIO_PVFS2_operations = {
-    ADIOI_PVFS2_Open, /* Open */
-    ADIOI_SCALEABLE_OpenColl, /* OpenColl */
-    ADIOI_PVFS2_ReadContig, /* ReadContig */
-    ADIOI_PVFS2_WriteContig, /* WriteContig */
-    ADIOI_GEN_ReadStridedColl, /* ReadStridedColl */
-    ADIOI_GEN_WriteStridedColl, /* WriteStridedColl */
-    ADIOI_GEN_SeekIndividual, /* SeekIndividual */
-    ADIOI_PVFS2_Fcntl, /* Fcntl */
-    ADIOI_PVFS2_SetInfo, /* SetInfo */
-    ADIOI_PVFS2_ReadStrided, /* ReadStrided */
-    ADIOI_PVFS2_WriteStrided, /* WriteStrided */
-    ADIOI_PVFS2_Close, /* Close */
-#ifdef HAVE_MPI_GREQUEST_EXTENSIONS
-    ADIOI_PVFS2_IReadContig, /* IreadContig */
-    ADIOI_PVFS2_IWriteContig, /* IwriteContig */
-#else
-    ADIOI_FAKE_IreadContig, /* IreadContig */
-    ADIOI_FAKE_IwriteContig, /* IwriteContig */
-#endif
-    ADIOI_FAKE_IODone, /* ReadDone */
-    ADIOI_FAKE_IODone, /* WriteDone */
-    ADIOI_FAKE_IOComplete, /* ReadComplete */
-    ADIOI_FAKE_IOComplete, /* WriteComplete */
-    ADIOI_FAKE_IreadStrided, /* IreadStrided */
-    ADIOI_FAKE_IwriteStrided, /* IwriteStrided */
-    ADIOI_PVFS2_Flush, /* Flush */
-    ADIOI_PVFS2_Resize, /* Resize */
-    ADIOI_PVFS2_Delete, /* Delete */
-    ADIOI_PVFS2_Feature,
-    "PVFS2: the PVFS v2 or OrangeFS file systems",
-    ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
-    ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
-};
-
-/* 
- * vim: ts=8 sts=4 sw=4 noexpandtab 
- */
diff --git a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2.h b/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2.h
deleted file mode 100644
index 95c5912376b..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2.h
+++ /dev/null
@@ -1,76 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#ifndef AD_PVFS2_INCLUDE
-#define AD_PVFS2_INCLUDE
-
-#include "adio.h"
-#ifdef HAVE_PVFS2_H
-#include "pvfs2.h"
-#endif
-
-#ifdef PVFS2_VERSION_MAJOR
-#include "pvfs2-compat.h"
-#endif
-
-void ADIOI_PVFS2_Open(ADIO_File fd, int *error_code);
-void ADIOI_PVFS2_Close(ADIO_File fd, int *error_code);
-void ADIOI_PVFS2_ReadContig(ADIO_File fd, void *buf, int count, 
-                      MPI_Datatype datatype, int file_ptr_type,
-                     ADIO_Offset offset, ADIO_Status *status, int
-		     *error_code);
-void ADIOI_PVFS2_WriteContig(ADIO_File fd, const void *buf, int count,
-                      MPI_Datatype datatype, int file_ptr_type,
-                      ADIO_Offset offset, ADIO_Status *status, int
-		      *error_code);   
-void ADIOI_PVFS2_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct, int
-		*error_code); 
-void ADIOI_PVFS2_WriteStrided(ADIO_File fd, const void *buf, int count,
-		       MPI_Datatype datatype, int file_ptr_type,
-		       ADIO_Offset offset, ADIO_Status *status, int
-		       *error_code);
-void ADIOI_PVFS2_ReadStrided(ADIO_File fd, void *buf, int count,
-		       MPI_Datatype datatype, int file_ptr_type,
-		       ADIO_Offset offset, ADIO_Status *status, int
-		       *error_code);
-void ADIOI_PVFS2_Flush(ADIO_File fd, int *error_code);
-void ADIOI_PVFS2_Delete(const char *filename, int *error_code);
-void ADIOI_PVFS2_Resize(ADIO_File fd, ADIO_Offset size, int *error_code);
-void ADIOI_PVFS2_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code);
-int ADIOI_PVFS2_Feature(ADIO_File fd, int flag);
-
-void ADIOI_PVFS2_IReadContig(ADIO_File fd, void *buf, int count, 
-			    MPI_Datatype datatype, int file_ptr_type,
-			    ADIO_Offset offset, MPI_Request *request,
-			    int *error_code);
-void ADIOI_PVFS2_IWriteContig(ADIO_File fd, const void *buf, int count,
-			    MPI_Datatype datatype, int file_ptr_type,
-			    ADIO_Offset offset, MPI_Request *request,
-			    int *error_code);
-void ADIOI_PVFS2_AIO_contig(ADIO_File fd, void *buf, int count, 
-			    MPI_Datatype datatype, int file_ptr_type,
-			    ADIO_Offset offset, MPI_Request *request,
-			    int flag, int *error_code);
-void ADIOI_PVFS2_OldWriteStrided(ADIO_File fd, const void *buf, int count,
-		       MPI_Datatype datatype, int file_ptr_type,
-		       ADIO_Offset offset, ADIO_Status *status, int
-		       *error_code);
-void ADIOI_PVFS2_OldReadStrided(ADIO_File fd, void *buf, int count,
-		       MPI_Datatype datatype, int file_ptr_type,
-		       ADIO_Offset offset, ADIO_Status *status, int
-		       *error_code);
-
-int ADIOI_PVFS2_WriteStridedListIO(ADIO_File fd, const void *buf, int count,
-				   MPI_Datatype datatype, int file_ptr_type,
-				   ADIO_Offset offset, ADIO_Status *status,
-				   int *error_code);
-int ADIOI_PVFS2_WriteStridedDtypeIO(ADIO_File fd, const void *buf, int count,
-				    MPI_Datatype datatype, int file_ptr_type,
-				    ADIO_Offset offset, ADIO_Status *status,
-				    int *error_code);
-
-#endif
diff --git a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_aio.c b/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_aio.c
deleted file mode 100644
index 88be6f1eaff..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_aio.c
+++ /dev/null
@@ -1,220 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- 
- *     vim: ts=8 sts=4 sw=4 noexpandtab 
- * 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-#include "ad_pvfs2.h"
-#include <string.h>
-
-#include "ad_pvfs2_common.h"
-#include "mpiu_greq.h"
-#include "../../mpi-io/mpioimpl.h"
-
-#define READ 0
-#define WRITE 1
-
-static int ADIOI_PVFS2_greq_class = 0;
-int ADIOI_PVFS2_aio_free_fn(void *extra_state);
-int ADIOI_PVFS2_aio_poll_fn(void *extra_state, MPI_Status *status);
-int ADIOI_PVFS2_aio_wait_fn(int count, void ** array_of_states, 
-		double timeout, MPI_Status *status);
-
-void ADIOI_PVFS2_IReadContig(ADIO_File fd, void *buf, int count, 
-			    MPI_Datatype datatype, int file_ptr_type,
-			    ADIO_Offset offset, MPI_Request *request,
-			    int *error_code)
-{
-    ADIOI_PVFS2_AIO_contig(fd, buf, count, datatype, file_ptr_type,
-	    offset, request, READ, error_code);
-}
-
-void ADIOI_PVFS2_IWriteContig(ADIO_File fd, const void *buf, int count,
-			    MPI_Datatype datatype, int file_ptr_type,
-			    ADIO_Offset offset, MPI_Request *request,
-			    int *error_code)
-{
-    ADIOI_PVFS2_AIO_contig(fd, (void *)buf, count, datatype, file_ptr_type,
-	    offset, request, WRITE, error_code);
-}
-
-void ADIOI_PVFS2_AIO_contig(ADIO_File fd, void *buf, int count, 
-			    MPI_Datatype datatype, int file_ptr_type,
-			    ADIO_Offset offset, MPI_Request *request,
-			    int flag, int *error_code)
-{
-
-    int ret;
-    MPI_Count datatype_size, len;
-    ADIOI_PVFS2_fs *pvfs_fs;
-    ADIOI_AIO_Request *aio_req;
-    static char myname[] = "ADIOI_PVFS2_AIO_contig";
-
-    pvfs_fs = (ADIOI_PVFS2_fs*)fd->fs_ptr;
-
-    aio_req = (ADIOI_AIO_Request*)ADIOI_Calloc(sizeof(ADIOI_AIO_Request), 1);
-
-    MPI_Type_size_x(datatype, &datatype_size);
-    len = datatype_size * count;
-
-    ret = PVFS_Request_contiguous(len, PVFS_BYTE, &(aio_req->mem_req));
-    /* --BEGIN ERROR HANDLING-- */
-    if (ret != 0) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   ADIOI_PVFS2_error_convert(ret),
-					   "Error in pvfs_request_contig (memory)", 0);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-    ret = PVFS_Request_contiguous(len, PVFS_BYTE, &(aio_req->file_req));
-    /* --BEGIN ERROR HANDLING-- */
-    if (ret != 0) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   ADIOI_PVFS2_error_convert(ret),
-					   "Error in pvfs_request_contig (file)", 0);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-    if (file_ptr_type == ADIO_INDIVIDUAL) {
-	/* copy individual file pointer into offset variable, continue */
-	offset = fd->fp_ind;
-    } 
-    if (flag == READ) {
-#ifdef ADIOI_MPE_LOGGING
-	MPE_Log_event( ADIOI_MPE_iread_a, 0, NULL );
-#endif
-	ret = PVFS_isys_read(pvfs_fs->object_ref, aio_req->file_req, offset, 
-		buf, aio_req->mem_req, &(pvfs_fs->credentials), 
-		&(aio_req->resp_io), &(aio_req->op_id), NULL);
-#ifdef ADIOI_MPE_LOGGING
-	MPE_Log_event( ADIOI_MPE_iread_b, 0, NULL );
-#endif
-    } else if (flag == WRITE) {
-#ifdef ADIOI_MPE_LOGGING
-	MPE_Log_event( ADIOI_MPE_iwrite_a, 0, NULL );
-#endif
-	ret = PVFS_isys_write(pvfs_fs->object_ref, aio_req->file_req, offset, 
-		buf, aio_req->mem_req, &(pvfs_fs->credentials), 
-		&(aio_req->resp_io), &(aio_req->op_id), NULL);
-#ifdef ADIOI_MPE_LOGGING
-	MPE_Log_event( ADIOI_MPE_iwrite_b, 0, NULL );
-#endif 
-    } 
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (ret < 0 ) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   ADIOI_PVFS2_error_convert(ret),
-					   "Error in PVFS_isys_io", 0);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    /* posted. defered completion */
-    if (ret == 0) { 
-	if (ADIOI_PVFS2_greq_class == 0) {
-	    MPIX_Grequest_class_create(ADIOI_GEN_aio_query_fn, 
-		    ADIOI_PVFS2_aio_free_fn, MPIU_Greq_cancel_fn,
-		    ADIOI_PVFS2_aio_poll_fn, ADIOI_PVFS2_aio_wait_fn,
-		    &ADIOI_PVFS2_greq_class);
-	}
-	MPIX_Grequest_class_allocate(ADIOI_PVFS2_greq_class, aio_req, request);
-	memcpy(&(aio_req->req), request, sizeof(*request));
-    }
-
-    /* immediate completion */
-    if (ret == 1) {
-	MPIO_Completed_request_create(&fd, len, error_code, request);
-    }
-
-    if (file_ptr_type == ADIO_INDIVIDUAL) {
-	fd->fp_ind += len;
-    }
-    fd->fp_sys_posn = offset + len;
-
-    *error_code = MPI_SUCCESS;
-fn_exit:
-    return;
-}
-
-int ADIOI_PVFS2_aio_free_fn(void *extra_state)
-{
-    ADIOI_AIO_Request *aio_req;
-    aio_req = (ADIOI_AIO_Request*)extra_state;
-
-    PVFS_Request_free(&(aio_req->mem_req));
-    PVFS_Request_free(&(aio_req->file_req));
-    ADIOI_Free(aio_req);
-
-    return MPI_SUCCESS;
-}
-
-int ADIOI_PVFS2_aio_poll_fn(void *extra_state, MPI_Status *status)
-{
-    ADIOI_AIO_Request *aio_req;
-    int ret, error;
-
-    aio_req = (ADIOI_AIO_Request *)extra_state;
-
-    /* BUG: cannot PVFS_sys_testsome: does not work for a specific request */
-    ret = PVFS_sys_wait(aio_req->op_id, "ADIOI_PVFS2_aio_poll_fn", &error);
-    if (ret == 0) {
-	aio_req->nbytes = aio_req->resp_io.total_completed;
-	MPI_Grequest_complete(aio_req->req);
-	return MPI_SUCCESS;
-    } else
-	return MPI_UNDEFINED; /* TODO: what's this error? */
-}
-
-/* wait for multiple requests to complete */
-int ADIOI_PVFS2_aio_wait_fn(int count, void ** array_of_states, 
-		double timeout, MPI_Status *status)
-{
-
-    ADIOI_AIO_Request **aio_reqlist;
-    PVFS_sys_op_id *op_id_array;
-    int i,j, greq_count, completed_count=0;
-    int *error_array;
-
-    aio_reqlist = (ADIOI_AIO_Request **)array_of_states;
-
-    op_id_array = (PVFS_sys_op_id*)ADIOI_Calloc(count, sizeof(PVFS_sys_op_id));
-    error_array = (int *)ADIOI_Calloc(count, sizeof(int));
-    greq_count = count;
-
-
-    /* PVFS-2.6: testsome actually tests all requests and fills in op_id_array
-     * with the ones that have completed.  count is an in/out parameter.
-     * returns with the number of completed operations.  what a mess! */
-    while (completed_count < greq_count ) {
-	count = greq_count;
-	PVFS_sys_testsome(op_id_array, &count, NULL, error_array, INT_MAX);
-	completed_count += count;
-	for (i=0; i< count; i++) {
-	    for (j=0; j<greq_count; j++) {
-		if (op_id_array[i] == aio_reqlist[j]->op_id) {
-		    aio_reqlist[j]->nbytes = 
-			aio_reqlist[j]->resp_io.total_completed;
-		    MPI_Grequest_complete(aio_reqlist[j]->req);
-		}
-	    }
-	}
-    }
-    return MPI_SUCCESS; /* TODO: no idea how to deal with errors */
-}
-
-
-/*
- * vim: ts=8 sts=4 sw=4 noexpandtab 
- */
diff --git a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_close.c b/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_close.c
deleted file mode 100644
index 847dd8341cd..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_close.c
+++ /dev/null
@@ -1,28 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_pvfs2.h"
-
-void ADIOI_PVFS2_Close(ADIO_File fd, int *error_code)
-{
-    ADIOI_Free(fd->fs_ptr);
-    fd->fs_ptr = NULL;
-    /* PVFS2 doesn't have a 'close', but MPI-IO semantics dictate that we
-     * ensure all data has been flushed.
-     */
-
-    /* At some point or another it was decided that ROMIO would not
-     * explicitly flush (other than any local cache) on close, because
-     * there is no way to *avoid* that overhead if you implement it here
-     * and don't actually want it.
-     */
-
-    *error_code = MPI_SUCCESS;
-}
-/* 
- * vim: ts=8 sts=4 sw=4 noexpandtab 
- */
diff --git a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_common.c b/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_common.c
deleted file mode 100644
index ccd3c1916c6..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_common.c
+++ /dev/null
@@ -1,150 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 2003 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_pvfs2.h"
-#include "ad_pvfs2_common.h"
-#include <unistd.h>
-#include <sys/types.h>
-#include <time.h>
-#include <stdlib.h>
-
-/* maybe give romio access to the globalconfig struct */
-/* keyval hack to both tell us if we've already initialized pvfs2 and also
- * close it down when mpi exits */
-int ADIOI_PVFS2_Initialized = MPI_KEYVAL_INVALID;
-
-void ADIOI_PVFS2_End(int *error_code)
-{
-    int ret;
-    static char myname[] = "ADIOI_PVFS2_END";
-
-    ret = PVFS_sys_finalize();
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (ret != 0 ) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   ADIOI_PVFS2_error_convert(ret),
-					   "Error in PVFS_sys_finalize", 0);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-    *error_code = MPI_SUCCESS;
-}
-
-int ADIOI_PVFS2_End_call(MPI_Comm comm, int keyval, 
-			 void *attribute_val, void *extra_state)
-{
-    int error_code;
-    ADIOI_PVFS2_End(&error_code);
-    MPI_Keyval_free(&keyval);
-    return error_code;
-}
-
-void ADIOI_PVFS2_Init(int *error_code )
-{
-    int ret;
-    static char myname[] = "ADIOI_PVFS2_INIT";
-    char * ncache_timeout;
-
-    /* do nothing if we've already fired up the pvfs2 interface */
-    if (ADIOI_PVFS2_Initialized != MPI_KEYVAL_INVALID) {
-	*error_code = MPI_SUCCESS;
-	return;
-    }
-
-    /* for consistency, we should disable the pvfs2 ncache.  If the
-     * environtment variable is already set, assume a  user knows it
-     * won't be a problem */
-    ncache_timeout = getenv("PVFS2_NCACHE_TIMEOUT");
-    if (ncache_timeout == NULL )
-	setenv("PVFS2_NCACHE_TIMEOUT", "0", 1);
-
-    ret = PVFS_util_init_defaults();
-    if (ret < 0 ) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   ADIOI_PVFS2_error_convert(ret),
-					   "Error in PVFS_util_init_defaults",
-					   0);
-	PVFS_perror("PVFS_util_init_defaults", ret);
-
-	return;
-    }
-    
-    MPI_Keyval_create(MPI_NULL_COPY_FN, ADIOI_PVFS2_End_call,
-		      &ADIOI_PVFS2_Initialized, (void *)0); 
-    /* just like romio does, we make a dummy attribute so we 
-     * get cleaned up */
-    MPI_Attr_put(MPI_COMM_SELF, ADIOI_PVFS2_Initialized, (void *)0);
-}
-
-void ADIOI_PVFS2_makeattribs(PVFS_sys_attr * attribs)
-{
-    memset(attribs, 0, sizeof(PVFS_sys_attr));
-    
-    attribs->owner = geteuid();
-    attribs->group = getegid();
-    attribs->perms = 0644;
-    attribs->mask =  PVFS_ATTR_SYS_ALL_SETABLE;
-    attribs->atime = time(NULL);
-    attribs->mtime = attribs->atime;
-    attribs->ctime = attribs->atime;
-}
-
-
-void ADIOI_PVFS2_makecredentials(PVFS_credentials * credentials)
-{
-    memset(credentials, 0, sizeof(PVFS_credentials));
-
-    PVFS_util_gen_credentials(credentials);
-}
-
-int ADIOI_PVFS2_error_convert(int pvfs_error)
-{
-    switch(pvfs_error)
-    {
-	case PVFS_EPERM:
-	case PVFS_EACCES:
-	    return MPI_ERR_ACCESS;
-	case PVFS_ENOENT:
-	case PVFS_ENXIO:
-	case PVFS_ENODEV:
-	    return MPI_ERR_NO_SUCH_FILE;
-	case PVFS_EIO:
-	    return MPI_ERR_IO;
-	case PVFS_EEXIST:
-	    return MPI_ERR_FILE_EXISTS;
-	case PVFS_ENOTDIR: /* ??? */
-	case PVFS_EISDIR: /* ??? */
-	case PVFS_ENAMETOOLONG:
-	    return MPI_ERR_BAD_FILE;
-	case PVFS_EINVAL:
-	    return MPI_ERR_FILE;
-	case PVFS_EFBIG: /* ??? */
-	case PVFS_ENOSPC:
-	    return MPI_ERR_NO_SPACE;
-	case PVFS_EROFS:
-	    return MPI_ERR_READ_ONLY;
-	case PVFS_ENOSYS:
-	    return MPI_ERR_UNSUPPORTED_OPERATION;
-	    /* PVFS does not support quotas */
-	case EDQUOT:
-	    return MPI_ERR_QUOTA;
-	case PVFS_ENOMEM:
-	    return MPI_ERR_INTERN;
-	default:
-	    return MPI_UNDEFINED;
-    }
-
-}
-
-/* 
- * vim: ts=8 sts=4 sw=4 noexpandtab 
- */
diff --git a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_common.h b/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_common.h
deleted file mode 100644
index f63f84a27bd..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_common.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*-
- * vim: ts=8 sts=4 sw=4 noexpandtab
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#ifndef _AD_PVFS2_COMMON_H
-#define _AD_PVFS2_COMMON_H
-#include "ad_pvfs2.h"
-
-/* useful values:
- *  0:		    no debugging
- *  CLIENT_DEBUG:   debug client state machines
- */
-#define ADIOI_PVFS2_DEBUG_MASK 0
-
-
-struct ADIOI_PVFS2_fs_s {
-    PVFS_object_ref object_ref;
-    PVFS_credentials credentials;
-} ADIOI_PVFS2_fs_s;
-
-typedef struct ADIOI_PVFS2_fs_s ADIOI_PVFS2_fs;
-
-
-void ADIOI_PVFS2_Init(int *error_code );
-void ADIOI_PVFS2_makeattribs(PVFS_sys_attr * attribs);
-void ADIOI_PVFS2_makecredentials(PVFS_credentials * credentials);
-void ADIOI_PVFS2_End(int *error_code);
-int ADIOI_PVFS2_End_call(MPI_Comm comm, int keyval, 
-	void *attribute_val, void *extra_state);
-int ADIOI_PVFS2_error_convert(int pvfs_error);
-
-#endif
diff --git a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_delete.c b/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_delete.c
deleted file mode 100644
index f5770bb393c..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_delete.c
+++ /dev/null
@@ -1,69 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2003 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_pvfs2.h"
-#include "adio.h"
-
-#include "ad_pvfs2_common.h"
-
-void ADIOI_PVFS2_Delete(const char *filename, int *error_code)
-{
-    PVFS_credentials credentials;
-    PVFS_sysresp_getparent resp_getparent;
-    int ret;
-    PVFS_fs_id cur_fs;
-    static char myname[] = "ADIOI_PVFS2_DELETE";
-    char pvfs_path[PVFS_NAME_MAX] = {0};
-
-    ADIOI_PVFS2_Init(error_code);
-    /* --BEGIN ERROR HANDLING-- */
-    if (*error_code != MPI_SUCCESS) 
-    {
-	/* ADIOI_PVFS2_INIT handles creating error codes itself */
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-    /* in most cases we'll store the credentials in the fs struct, but we don't
-     * have one of those in Delete  */
-    ADIOI_PVFS2_makecredentials(&credentials);
-
-    /* given the filename, figure out which pvfs filesystem it is on */
-    ret = PVFS_util_resolve(filename, &cur_fs, pvfs_path, PVFS_NAME_MAX);
-    /* --BEGIN ERROR HANDLING-- */
-    if (ret != 0) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   ADIOI_PVFS2_error_convert(ret),
-					   "Error in PVFS_util_resolve", 0);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-    ret = PVFS_sys_getparent(cur_fs, pvfs_path, &credentials, &resp_getparent);
-
-    ret = PVFS_sys_remove(resp_getparent.basename, 
-			  resp_getparent.parent_ref, &credentials);
-    /* --BEGIN ERROR HANDLING-- */
-    if (ret != 0) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   ADIOI_PVFS2_error_convert(ret),
-					   "Error in PVFS_sys_remove", 0);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-    *error_code = MPI_SUCCESS;
-    return;
-}
-
-/* 
- * vim: ts=8 sts=4 sw=4 noexpandtab 
- */
diff --git a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_fcntl.c b/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_fcntl.c
deleted file mode 100644
index e07291f7575..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_fcntl.c
+++ /dev/null
@@ -1,59 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_pvfs2.h"
-#include "adio_extern.h"
-#include "ad_pvfs2_common.h"
-
-void ADIOI_PVFS2_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct,
-		       int *error_code)
-{
-    int ret;
-    ADIOI_PVFS2_fs *pvfs_fs;
-    PVFS_sysresp_getattr resp_getattr;
-    static char myname[] = "ADIOI_PVFS2_FCNTL";
-
-    pvfs_fs = (ADIOI_PVFS2_fs*)fd->fs_ptr;
-
-    switch(flag) {
-    case ADIO_FCNTL_GET_FSIZE:
-	ret = PVFS_sys_getattr(pvfs_fs->object_ref, PVFS_ATTR_SYS_SIZE, 
-		&(pvfs_fs->credentials), &resp_getattr);
-	if (ret != 0 ) {
-	    /* --BEGIN ERROR HANDLING-- */
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE,
-					       myname, __LINE__,
-					       ADIOI_PVFS2_error_convert(ret),
-					       "Error in PVFS_sys_getattr", 0);
-	    /* --END ERROR HANDLING-- */
-	}
-	else {
-	    *error_code = MPI_SUCCESS;
-	}
-	fcntl_struct->fsize = resp_getattr.attr.size;
-	return;
-
-    case ADIO_FCNTL_SET_DISKSPACE:
-	ADIOI_GEN_Prealloc(fd, fcntl_struct->diskspace, error_code);
-	break;
-
-    /* --BEGIN ERROR HANDLING-- */
-    case ADIO_FCNTL_SET_ATOMICITY:
-    default:
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   MPI_ERR_ARG,
-					   "**flag", "**flag %d", flag);
-    /* --END ERROR HANDLING-- */
-    }
-}
-
-/* 
- * vim: ts=8 sts=4 sw=4 noexpandtab 
- */
diff --git a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_features.c b/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_features.c
deleted file mode 100644
index 89a82c129fe..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_features.c
+++ /dev/null
@@ -1,23 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *
- *  (C) 2008 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include "adio.h"
-#include "ad_pvfs2.h"
-
-int ADIOI_PVFS2_Feature(ADIO_File fd, int flag)
-{
-	switch(flag) {
-		case ADIO_SCALABLE_OPEN:
-		case ADIO_SCALABLE_RESIZE:
-			return 1;
-		case ADIO_SHARED_FP:
-		case ADIO_LOCKS:
-		case ADIO_SEQUENTIAL:
-		case ADIO_DATA_SIEVING_WRITES:
-		default:
-			return 0;
-	}
-}
diff --git a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_flush.c b/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_flush.c
deleted file mode 100644
index 18061084a85..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_flush.c
+++ /dev/null
@@ -1,55 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_pvfs2.h"
-#include "ad_pvfs2_common.h"
-
-/* we want to be a bit clever here:  at scale, if every client sends a
- * flush request, it will stress the PVFS2 servers with redundant
- * PVFS_sys_flush requests.  Instead, one process should wait for
- * everyone to catch up, do the sync, then broadcast the result.  We can
- * get away with this thanks to PVFS2's stateless design 
- */
-
-void ADIOI_PVFS2_Flush(ADIO_File fd, int *error_code) 
-{ 
-    int ret, rank, dummy=0, dummy_in=0; 
-    ADIOI_PVFS2_fs *pvfs_fs;
-    static char myname[] = "ADIOI_PVFS2_FLUSH";
-
-    *error_code = MPI_SUCCESS;
-
-    pvfs_fs = (ADIOI_PVFS2_fs*)fd->fs_ptr;
-
-    MPI_Comm_rank(fd->comm, &rank);
-
-
-    /* unlike ADIOI_PVFS2_Resize, MPI_File_sync() does not perform any
-     * syncronization */
-    MPI_Reduce(&dummy_in, &dummy, 1, MPI_INT, MPI_SUM, 
-	    fd->hints->ranklist[0], fd->comm);
-
-    /* io_worker computed in ADIO_Open */
-    if (rank == fd->hints->ranklist[0]) {
-	ret = PVFS_sys_flush(pvfs_fs->object_ref, &(pvfs_fs->credentials));
-    }
-    MPI_Bcast(&ret, 1, MPI_INT, fd->hints->ranklist[0], fd->comm);
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (ret != 0) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   ADIOI_PVFS2_error_convert(ret),
-					   "Error in PVFS_sys_flush", 0);
-    }
-    /* --END ERROR HANDLING-- */
-}
-
-/* 
- * vim: ts=8 sts=4 sw=4 noexpandtab 
- */
diff --git a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_hints.c b/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_hints.c
deleted file mode 100644
index 47522064e30..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_hints.c
+++ /dev/null
@@ -1,120 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include <stdlib.h>
-#include "ad_pvfs2.h"
-
-#include "hint_fns.h"
-
-void ADIOI_PVFS2_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code)
-{
-    char *value;
-    int flag, tmp_value;
-    static char myname[] = "ADIOI_PVFS_SETINFO";
-
-    if ((fd->info) == MPI_INFO_NULL) {
-	/* part of the open call */
-	MPI_Info_create(&(fd->info));
-	ADIOI_Info_set(fd->info, "romio_pvfs2_debugmask", "0");
-	fd->hints->fs_hints.pvfs2.debugmask = 0;
-
-	ADIOI_Info_set(fd->info, "striping_factor", "0");
-	fd->hints->striping_factor = 0;
-
-	ADIOI_Info_set(fd->info, "striping_unit", "0");
-	fd->hints->striping_unit = 0;
-
-	/* disable the aggressive strided optimizations by default */
-        ADIOI_Info_set(fd->info, "romio_pvfs2_posix_read", "disable");
-        ADIOI_Info_set(fd->info, "romio_pvfs2_posix_write", "disable");
-        fd->hints->fs_hints.pvfs2.posix_read = ADIOI_HINT_DISABLE;
-        fd->hints->fs_hints.pvfs2.posix_write = ADIOI_HINT_DISABLE;
-
-        ADIOI_Info_set(fd->info, "romio_pvfs2_dtype_read", "disable");
-        ADIOI_Info_set(fd->info, "romio_pvfs2_dtype_write", "disable");
-        fd->hints->fs_hints.pvfs2.dtype_read = ADIOI_HINT_DISABLE;
-        fd->hints->fs_hints.pvfs2.dtype_write = ADIOI_HINT_DISABLE;
-
-        ADIOI_Info_set(fd->info, "romio_pvfs2_listio_read", "disable");
-        ADIOI_Info_set(fd->info, "romio_pvfs2_listio_write", "disable");
-        fd->hints->fs_hints.pvfs2.listio_read = ADIOI_HINT_DISABLE;
-        fd->hints->fs_hints.pvfs2.listio_write = ADIOI_HINT_DISABLE;
-
-	
-	/* any user-provided hints? */
-	if (users_info != MPI_INFO_NULL) {
-	    value = (char *) ADIOI_Malloc( (MPI_MAX_INFO_VAL+1)*sizeof(char));
-	    /* pvfs2 debugging */
-	    ADIOI_Info_get(users_info, "romio_pvfs2_debugmask", 
-		    MPI_MAX_INFO_VAL, value, &flag);
-	    if (flag) {
-		tmp_value = fd->hints->fs_hints.pvfs2.debugmask = 
-		    PVFS_debug_eventlog_to_mask(value);
-
-		MPI_Bcast(&tmp_value, 1, MPI_INT, 0, fd->comm);
-		/* --BEGIN ERROR HANDLING-- */
-		if (tmp_value != fd->hints->fs_hints.pvfs2.debugmask) {
-		    MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
-						       "romio_pvfs2_debugmask",
-						       error_code);
-		    return;
-		}
-		/* --END ERROR HANDLING-- */
-		
-		ADIOI_Info_set(fd->info, "romio_pvfs2_debugmask", value);
-	    }
-
-	    /* the striping factor */
-	    ADIOI_Info_check_and_install_int(fd, users_info, "striping_factor",
-		    &(fd->hints->striping_factor), myname, error_code);
-
-
-	    /* the striping unit */
-	    ADIOI_Info_check_and_install_int(fd, users_info, "striping_unit",
-		    &(fd->hints->striping_unit), myname, error_code);
-
-	    /* distribution name */
-	    ADIOI_Info_get(users_info, "romio_pvfs2_distribution_name",
-		    MPI_MAX_INFO_VAL, value, &flag);
-	    if (flag) {
-	    }
-
-	    /* POSIX read */
-	    ADIOI_Info_check_and_install_enabled(fd, users_info, "romio_pvfs2_posix_read",
-		    &(fd->hints->fs_hints.pvfs2.posix_read), myname, error_code);
-
-            /* POSIX write */
-	    ADIOI_Info_check_and_install_enabled(fd, users_info, "romio_pvfs2_posix_write",
-		    &(fd->hints->fs_hints.pvfs2.posix_write), myname, error_code);
-
-	    /* Datatype read */
-	    ADIOI_Info_check_and_install_enabled(fd, users_info, "romio_pvfs2_dtype_read",
-		    &(fd->hints->fs_hints.pvfs2.dtype_read), myname, error_code);
-
-            /* Datatype write */
-	    ADIOI_Info_check_and_install_enabled(fd, users_info, "romio_pvfs2_dtype_write",
-		    &(fd->hints->fs_hints.pvfs2.dtype_write), myname, error_code);
-
-	    /* Listio read */
-	    ADIOI_Info_check_and_install_enabled(fd, users_info, "romio_pvfs2_listio_read",
-		    &(fd->hints->fs_hints.pvfs2.listio_read), myname, error_code);
-
-            /* Datatype write */
-	    ADIOI_Info_check_and_install_enabled(fd, users_info, "romio_pvfs2_listio_write",
-		    &(fd->hints->fs_hints.pvfs2.listio_write), myname, error_code);
-            ADIOI_Free(value);
-	}
-    }
-    /* set the values for collective I/O and data sieving parameters */
-    ADIOI_GEN_SetInfo(fd, users_info, error_code);
-
-    *error_code = MPI_SUCCESS;
-}
-
-/*
- * vim: ts=8 sts=4 sw=4 noexpandtab
- */
diff --git a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_io.h b/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_io.h
deleted file mode 100644
index aefe0653ecb..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_io.h
+++ /dev/null
@@ -1,79 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*-
- * vim: ts=8 sts=4 sw=4 noexpandtab
- *
- *   Copyright (C) 2006 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-/* Contig I/O helper prototypes */
-
-#define READ 0
-#define WRITE 1
-
-/* #define DEBUG_CONTIG */
-/* #define DEBUG_LIST */
-/* #define DEBUG_DTYPE */
-
-/* Contig I/O helper prototypes */
-int ADIOI_PVFS2_Contig(ADIO_File fd, void *buf, int count,
-		       MPI_Datatype datatype, int file_ptr_type,
-		       ADIO_Offset offset, ADIO_Status *status,
-		       int *error_code, int rw_type);
-
-/* List I/O helper prototypes */
-int ADIOI_PVFS2_StridedListIO(ADIO_File fd, void *buf, int count,
-			      MPI_Datatype datatype, int file_ptr_type,
-			      ADIO_Offset offset, ADIO_Status *status,
-			      int *error_code, int rw_type);
-
-int gen_listio_arr(ADIOI_Flatlist_node *flat_buf,
-                   int *flat_buf_index_p,
-                   int64_t *cur_flat_buf_reg_off_p,
-                   int flat_buf_size,
-                   int flat_buf_extent,
-                   ADIOI_Flatlist_node *flat_file,
-                   int *flat_file_index_p,
-                   int64_t *cur_flat_file_reg_off_p,
-                   int flat_file_size,
-                   int flat_file_extent,
-                   int max_ol_count,
-                   ADIO_Offset disp,
-                   int bytes_into_filetype,
-                   int64_t *bytes_completed,
-                   int64_t total_io_size,
-                   int64_t buf_off_arr[],
-                   int32_t buf_len_arr[],
-                   int32_t *buf_ol_count_p,
-                   int64_t file_off_arr[],
-                   int32_t file_len_arr[],
-                   int32_t *file_ol_count_p);
-
-void print_buf_file_ol_pairs(int64_t buf_off_arr[],
-			     int32_t buf_len_arr[],
-			     int32_t buf_ol_count,
-			     int64_t file_off_arr[],
-			     int32_t file_len_arr[],
-			     int32_t file_ol_count,
-			     void *buf,
-			     int rw_type);
-
-/* Datatype I/O helper prototypes */
-int ADIOI_PVFS2_StridedDtypeIO(ADIO_File fd, void *buf, int count,
-			       MPI_Datatype datatype, int file_ptr_type,
-			       ADIO_Offset offset, ADIO_Status *status, 
-			       int *error_code, int rw_type);
-
-int convert_named(MPI_Datatype *mpi_dtype,
-                  PVFS_Request *pvfs_dtype, int combiner);
-
-void print_dtype_info(int combiner,
-                      int num_int,
-                      int num_addr,
-                      int num_dtype,
-                      int *arr_int,
-                      MPI_Aint *arr_addr,
-                      MPI_Datatype *arr_dtype);
-
-int convert_mpi_pvfs2_dtype(MPI_Datatype *mpi_dtype,
-                            PVFS_Request *pvfs_dtype);
-
diff --git a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_io_dtype.c b/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_io_dtype.c
deleted file mode 100644
index 02219293cad..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_io_dtype.c
+++ /dev/null
@@ -1,739 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*-
- * vim: ts=8 sts=4 sw=4 noexpandtab
- *
- *   Copyright (C) 2006 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include <assert.h>
-#include "adio.h"
-#include "adio_extern.h"
-#include "ad_pvfs2.h"
-#include "ad_pvfs2_io.h"
-#include "ad_pvfs2_common.h"
-
-int ADIOI_PVFS2_StridedDtypeIO(ADIO_File fd, void *buf, int count,
-			       MPI_Datatype datatype, int file_ptr_type,
-			       ADIO_Offset offset, ADIO_Status *status, int
-			       *error_code,
-			       int rw_type)
-{
-    int ret = -1, filetype_is_contig = -1;
-    MPI_Count filetype_size = -1;
-    int num_filetypes = 0, cur_flat_file_reg_off = 0;
-    PVFS_Request tmp_mem_req, mem_req, tmp_file_req, file_req;
-    PVFS_sysresp_io resp_io;
-    ADIO_Offset off = -1, bytes_into_filetype = 0;
-    MPI_Aint filetype_extent = -1, filetype_lb = -1;
-    int i = -1;
-    MPI_Count etype_size;
-    PVFS_size pvfs_disp = -1;
-    ADIOI_Flatlist_node *flat_file_p = ADIOI_Flatlist;
-
-    /* Use for offseting the PVFS2 filetype */
-    int pvfs_blk = 1;
-    ADIOI_PVFS2_fs *pvfs_fs;
-    static char myname[] = "ADIOI_PVFS2_STRIDED_DTYPE";
-
-    memset(&tmp_mem_req, 0, sizeof(PVFS_Request));
-    memset(&mem_req, 0, sizeof(PVFS_Request));
-    memset(&tmp_file_req, 0, sizeof(PVFS_Request));
-    memset(&file_req, 0, sizeof(PVFS_Request));
-
-    pvfs_fs = (ADIOI_PVFS2_fs*)fd->fs_ptr;
-
-    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-
-    /* changed below if error */
-    *error_code = MPI_SUCCESS;  
-
-    /* datatype is the memory type 
-     * fd->filetype is the file type */
-    MPI_Type_size_x(fd->filetype, &filetype_size);
-    if (filetype_size == 0) {
-        *error_code = MPI_SUCCESS;
-        return -1;
-    }
-    MPI_Type_get_extent(fd->filetype, &filetype_lb, &filetype_extent);
-    MPI_Type_size_x(fd->etype, &etype_size);
-    if (filetype_size == 0) {
-        *error_code = MPI_SUCCESS;
-        return -1;
-    }
-
-    /* offset is in units of etype relative to the filetype.  We
-     * convert this to off in terms of actual data bytes (the offset
-     * minus the number of bytes that are not used).  We are allowed
-     * to do this since PVFS2 handles offsets with respect to a
-     * file_req in bytes, otherwise we would have to convert into a
-     * pure byte offset as is done in other methods.  Explicit offset
-     * case is handled by using fd->disp and byte-converted off. */
-
-    pvfs_disp = fd->disp;
-    if (file_ptr_type == ADIO_INDIVIDUAL) 
-    {
-	if (filetype_is_contig) 
-	{
-	    off = fd->fp_ind - fd->disp;
-	}
-	else 
-	{
-	    int flag = 0;
-	    /* Should have already been flattened in ADIO_Open*/
-	    while (flat_file_p->type != fd->filetype) 
-	    {
-		flat_file_p = flat_file_p->next;
-	    }
-	    num_filetypes = -1;
-	    while (!flag) 
-	    {
-		num_filetypes++;
-		for (i = 0; i < flat_file_p->count; i++) 
-		{
-		    /* Start on a non zero-length region */
-		    if (flat_file_p->blocklens[i]) 
-		    {
-			if (fd->disp + flat_file_p->indices[i] +
-			    (num_filetypes * filetype_extent) +
-			    flat_file_p->blocklens[i] > fd->fp_ind &&
-			    fd->disp + flat_file_p->indices[i] <= 
-			    fd->fp_ind) 
-			{
-			    cur_flat_file_reg_off = fd->fp_ind -
-				(fd->disp + flat_file_p->indices[i] +
-				 (num_filetypes * filetype_extent));
-			    flag = 1;
-			    break;
-			}
-			else
-			    bytes_into_filetype += flat_file_p->blocklens[i];
-		    }
-		}
-	    }
-	    /* Impossible that we don't find it in this datatype */
-	    assert(i != flat_file_p->count);
-	    off = bytes_into_filetype + cur_flat_file_reg_off;
-	}
-    }
-    else /* ADIO_EXPLICIT */
-    { 
-	off = etype_size * offset;
-    }
-
-#ifdef DEBUG_DTYPE
-    fprintf(stderr, "ADIOI_PVFS2_StridedDtypeIO: (fd->fp_ind=%Ld,fd->disp=%Ld,"
-	    " offset=%Ld),(pvfs_disp=%Ld,off=%Ld)\n",
-	    fd->fp_ind, fd->disp, offset, pvfs_disp, off);
-#endif
-
-
-    /* Convert the MPI memory and file datatypes into
-     * PVFS2 datatypes */
-    ret = convert_mpi_pvfs2_dtype(&datatype, &tmp_mem_req);
-    if (ret < 0)
-    {
-	goto error_state;
-    }
-    ret = convert_mpi_pvfs2_dtype(&(fd->filetype), &tmp_file_req);
-    if (ret < 0)
-    {
-	goto error_state;
-    }
-
-    ret = PVFS_Request_contiguous(count, tmp_mem_req, &mem_req);
-    if (ret != 0) /* TODO: convert this to MPIO error handling */
-        fprintf(stderr, "ADIOI_PVFS2_stridedDtypeIO: error in final"
-		" CONTIG memory type\n");
-    PVFS_Request_free(&tmp_mem_req);    
-
-    /* pvfs_disp is used to offset the filetype */
-    ret = PVFS_Request_hindexed(1, &pvfs_blk, &pvfs_disp,
-                                tmp_file_req, &file_req);
-    if (ret != 0)
-        fprintf(stderr, "ADIOI_PVFS2_StridedDtypeIO: error in final"
-			" HINDEXED file type\n");
-    PVFS_Request_free(&tmp_file_req);
-
-    if (rw_type == READ)
-	ret = PVFS_sys_read(pvfs_fs->object_ref, file_req, off, buf,
-			    mem_req, &(pvfs_fs->credentials), &resp_io);
-    else
-	ret = PVFS_sys_write(pvfs_fs->object_ref, file_req, off, buf,
-			     mem_req, &(pvfs_fs->credentials), &resp_io);
-
-    if (ret != 0) {
-	fprintf(stderr, "ADIOI_PVFS2_StridedDtypeIO: Warning - PVFS_sys_"
-		"read/write returned %d and completed %Ld bytes.\n", 
-		ret, (long long)resp_io.total_completed);
-        *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-                                           MPIR_ERR_RECOVERABLE,
-                                           myname, __LINE__,
-                                           ADIOI_PVFS2_error_convert(ret),
-                                           "Error in PVFS_sys_io \n", 0);
-        goto error_state;
-    }
-
-    if (file_ptr_type == ADIO_INDIVIDUAL)
-    {
-        fd->fp_ind = off += resp_io.total_completed;
-    }
-    
-  error_state:
-    fd->fp_sys_posn = -1;   /* set it to null. */
-
-    PVFS_Request_free(&mem_req);
-    PVFS_Request_free(&file_req);    
-
-#ifdef DEBUG_DTYPE
-    fprintf(stderr, "ADIOI_PVFS2_StridedDtypeIO: "
-            "resp_io.total_completed=%Ld,ret=%d\n", 
-	    resp_io.total_completed, ret);
-#endif
-
-#ifdef HAVE_STATUS_SET_BYTES
-    MPIR_Status_set_bytes(status, datatype, resp_io.total_completed);
-    /* This is a temporary way of filling in status. The right way is to
-     * keep track of how much data was actually acccessed by 
-     * ADIOI_BUFFERED operations */
-#endif
-    return ret;
-}
-
-/* convert_mpi_pvfs2_dtype - Convert a MPI datatype into
- * a PVFS2 datatype so that we can natively use the PVFS2 
- * datatypes in the PVFS2 I/O calls instead of converting 
- * all datatypes to the hindexed method 
- * return 1  - a leaf node
- * return 0  - normal return 
- * return -1 - problems */
-
-int convert_mpi_pvfs2_dtype(MPI_Datatype *mpi_dtype, 
-			    PVFS_Request *pvfs_dtype)
-{
-    int num_int = -1, num_addr = -1, num_dtype = -1, 
-	combiner = -1, i = -1, ret = -1, leaf = -1;
-    int *arr_int = NULL;
-    MPI_Aint *arr_addr = NULL;
-    MPI_Datatype *arr_dtype = NULL;
-    PVFS_Request *old_pvfs_dtype = NULL;
-    PVFS_Request *old_pvfs_dtype_arr = NULL;
-    int arr_count = -1;
-    PVFS_size *pvfs_arr_disp = NULL;
-    int *pvfs_arr_len = NULL;
-
-    MPI_Type_get_envelope(*mpi_dtype,
-			  &num_int,
-			  &num_addr,
-			  &num_dtype,
-			  &combiner);
-
-    /* Depending on type of datatype do the following 
-     * operations */
-    
-    if (combiner == MPI_COMBINER_NAMED)
-    {
-	convert_named(mpi_dtype, pvfs_dtype, combiner);
-	return 1;
-    }
-
-    /* Allocate space for the arrays necessary for 
-     * MPI_Type_get_contents */
-
-    if ((arr_int = ADIOI_Malloc(sizeof(int)*num_int)) == NULL)
-    {
-	fprintf(stderr, "Failed to allocate array_int\n");
-	return -1;
-    }
-    if ((arr_addr = ADIOI_Malloc(sizeof(int)*num_addr)) == NULL)
-    {
-	ADIOI_Free(arr_int);
-	fprintf(stderr, "Failed to allocate array_addr\n");
-	return -1;
-    }
-    if ((arr_dtype = ADIOI_Malloc(sizeof(MPI_Datatype)*num_dtype)) == NULL)
-    {
-	ADIOI_Free(arr_int);
-	ADIOI_Free(arr_addr);
-	fprintf(stderr, "Failed to allocate array_dtypes\n");
-	return -1;
-    }
-
-    MPI_Type_get_contents(*mpi_dtype,
-			  num_int,
-			  num_addr,
-			  num_dtype,
-			  arr_int,
-			  arr_addr,
-			  arr_dtype);
-
-    /* If it's not a predefined datatype, it is either a 
-     * derived datatype or a structured datatype */
-
-    if (combiner != MPI_COMBINER_STRUCT)
-    {
-	if ((old_pvfs_dtype = ADIOI_Malloc(sizeof(PVFS_Request))) == NULL)
-	    fprintf(stderr, "convert_mpi_pvfs2_dtype: "
-		    "Failed to allocate PVFS_Request\n");
-	switch (combiner)
-	{
-	    case MPI_COMBINER_CONTIGUOUS:
-		leaf = convert_mpi_pvfs2_dtype(&arr_dtype[0], old_pvfs_dtype);
-		ret = PVFS_Request_contiguous(arr_int[0], 
-					      *old_pvfs_dtype, pvfs_dtype);
-		break;
-	    case MPI_COMBINER_VECTOR:
-		leaf = convert_mpi_pvfs2_dtype(&arr_dtype[0], old_pvfs_dtype);
-		ret = PVFS_Request_vector(arr_int[0], arr_int[1],
-					  arr_int[2], *old_pvfs_dtype, 
-					  pvfs_dtype);
-		break;
-	    case MPI_COMBINER_HVECTOR:
-		leaf = convert_mpi_pvfs2_dtype(&arr_dtype[0], old_pvfs_dtype);
-		ret = PVFS_Request_hvector(arr_int[0], arr_int[1],
-					   arr_addr[0], *old_pvfs_dtype, 
-					   pvfs_dtype);
-		break;
-		/* Both INDEXED and HINDEXED types require PVFS_size 
-		 * address arrays.  Therefore, we need to copy and 
-		 * convert the data from MPI_get_contents() into 
-		 * a PVFS_size buffer */
-	    case MPI_COMBINER_INDEXED:
-		leaf = convert_mpi_pvfs2_dtype(&arr_dtype[0], old_pvfs_dtype);
-		if ((pvfs_arr_disp = 
-			    ADIOI_Malloc(arr_int[0]*sizeof(PVFS_size))) == 0)
-		{
-		    fprintf(stderr, "convert_mpi_pvfs2_dtype: "
-			    "Failed to allocate pvfs_arr_disp\n");
-		}
-		for (i = 0; i < arr_int[0]; i++)
-		{
-		    pvfs_arr_disp[i] = 
-			(PVFS_size) arr_int[arr_int[0]+1+i];
-		}
-		ret = PVFS_Request_indexed(arr_int[0], &arr_int[1], 
-				     pvfs_arr_disp,
-				     *old_pvfs_dtype, pvfs_dtype);
-		ADIOI_Free(pvfs_arr_disp);
-		break;
-	    case MPI_COMBINER_HINDEXED:
-		leaf = convert_mpi_pvfs2_dtype(&arr_dtype[0], old_pvfs_dtype);
-		if ((pvfs_arr_disp = 
-			    ADIOI_Malloc(arr_int[0]*sizeof(PVFS_size))) == 0)
-		{
-		    fprintf(stderr, "convert_mpi_pvfs2_dtype: "
-			    "Failed to allocate pvfs_arr_disp\n");
-		}
-		for (i = 0; i < arr_int[0]; i++)
-		{
-		    pvfs_arr_disp[i] = 
-			(PVFS_size) arr_addr[i];
-		}
-		ret = PVFS_Request_hindexed(arr_int[0], &arr_int[1], 
-				      (int64_t *)&arr_addr[0],
-				      *old_pvfs_dtype, pvfs_dtype);
-		ADIOI_Free(pvfs_arr_disp);		
-		break;
-	    case MPI_COMBINER_DUP:
-                leaf = convert_mpi_pvfs2_dtype(&arr_dtype[0], old_pvfs_dtype);
-		ret = PVFS_Request_contiguous(1, 
-					      *old_pvfs_dtype, pvfs_dtype);
-
-                break;
-	    case MPI_COMBINER_INDEXED_BLOCK:
-		/* No native PVFS2 support for this operation currently */
-		ADIOI_Free(old_pvfs_dtype);
-		fprintf(stderr, "convert_mpi_pvfs2_dtype: "
-			"INDEXED_BLOCK is unsupported\n"); 
-		break;
-	    case MPI_COMBINER_HINDEXED_BLOCK:
-		/* No native PVFS2 support for this operation currently */
-		ADIOI_Free(old_pvfs_dtype);
-		fprintf(stderr, "convert_mpi_pvfs2_dtype: "
-			"HINDEXED_BLOCK is unsupported\n");
-		break;
-	    case MPI_COMBINER_HINDEXED_INTEGER:
-		ADIOI_Free(old_pvfs_dtype);
-		fprintf(stderr, "convert_mpi_pvfs2_dtype: "
-			"HINDEXED_INTEGER is unsupported\n"); 
-		break;
-	    case MPI_COMBINER_STRUCT_INTEGER:
-		ADIOI_Free(old_pvfs_dtype);
-		fprintf(stderr, "convert_mpi_pvfs2_dtype: "
-			"STRUCT_INTEGER is unsupported\n"); 
-		break;
-	    case MPI_COMBINER_SUBARRAY:
-		ADIOI_Free(old_pvfs_dtype);
-		fprintf(stderr, "convert_mpi_pvfs2_dtype: "
-			"SUBARRAY is unsupported\n"); 
-		break;
-	    case MPI_COMBINER_DARRAY:
-		ADIOI_Free(old_pvfs_dtype);
-		fprintf(stderr, "convert_mpi_pvfs2_dtype: "
-			"DARRAY is unsupported\n"); 
-		break;
-	    case MPI_COMBINER_F90_REAL:
-		ADIOI_Free(old_pvfs_dtype);
-		fprintf(stderr, "convert_mpi_pvfs2_dtype: "
-			"F90_REAL is unsupported\n"); 
-		break;
-	    case MPI_COMBINER_F90_COMPLEX:
-		ADIOI_Free(old_pvfs_dtype);
-		fprintf(stderr, "convert_mpi_pvfs2_dtype: "
-			"F90_COMPLEX is unsupported\n"); 
-		break;
-	    case MPI_COMBINER_F90_INTEGER:
-		ADIOI_Free(old_pvfs_dtype);
-		fprintf(stderr, "convert_mpi_pvfs2_dtype: "
-			"F90_INTEGER is unsupported\n"); 
-		break;
-	    case MPI_COMBINER_RESIZED:
-		ADIOI_Free(old_pvfs_dtype);
-		fprintf(stderr, "convert_mpi_pvfs2_dtype: "
-			"RESIZED is unsupported\n"); 
-		break;
-	    default:
-		break;
-	}
-
-	if (ret != 0)
-	    fprintf(stderr, "Error in PVFS_Request_* "
-		    "for a derived datatype\n");
-
-#ifdef DEBUG_DTYPE
-	print_dtype_info(combiner,    
-			 num_int,
-			 num_addr,
-			 num_dtype,
-			 arr_int,
-			 arr_addr,
-			 arr_dtype);
-#endif
-
-	if (leaf != 1 && combiner != MPI_COMBINER_DUP)
-	    MPI_Type_free(&arr_dtype[0]);
-
-	ADIOI_Free(arr_int);
-	ADIOI_Free(arr_addr);
-	ADIOI_Free(arr_dtype);
-
-	PVFS_Request_free(old_pvfs_dtype);
-	ADIOI_Free(old_pvfs_dtype);
-	
-	return ret;
-    }
-    else /* MPI_COMBINER_STRUCT */
-    {
-	MPI_Aint mpi_lb = -1, mpi_extent = -1;
-	PVFS_offset pvfs_lb = -1;
-	PVFS_size pvfs_extent = -1;
-	int has_lb_ub = 0;
-
-	/* When converting into a PVFS_Request_struct, we no longer
-	 * can use MPI_LB and MPI_UB.  Therfore, we have to do the
-	 * following.  
-	 * We simply ignore all the MPI_LB and MPI_UB types and 
-	 * get the lb and extent and pass it on through a 
-	 * PVFS resized_req */
-
-	arr_count = 0;
-	for (i = 0; i < arr_int[0]; i++)
-	{
-	    if (arr_dtype[i] != MPI_LB &&
-		arr_dtype[i] != MPI_UB)
-	    {
-		arr_count++;
-	    }
-	}
-
-	if (arr_int[0] != arr_count)
-	{
-	    MPI_Type_get_extent(*mpi_dtype, &mpi_lb, &mpi_extent);
-	    pvfs_lb = mpi_lb;
-	    pvfs_extent = mpi_extent;
-	    if ((pvfs_arr_len = ADIOI_Malloc(arr_count*sizeof(int))) 
-		== NULL)
-	    {
-		fprintf(stderr, "convert_mpi_pvfs2_dtype: "
-			"Failed to allocate pvfs_arr_len\n");
-	    }
-	    has_lb_ub = 1;
-	}
-
-	if ((old_pvfs_dtype_arr
-	     = ADIOI_Malloc(arr_count*sizeof(PVFS_Request))) == NULL)
-	    fprintf(stderr, "convert_mpi_pvfs2_dtype: "
-		    "Failed to allocate PVFS_Requests\n");
-
-	if ((pvfs_arr_disp = ADIOI_Malloc(arr_count*sizeof(PVFS_size))) 
-	    == NULL)
-	{
-	    fprintf(stderr, "convert_mpi_pvfs2_dtype: "
-		    "Failed to allocate pvfs_arr_disp\n");
-	}
-
-	arr_count = 0;
-	for (i = 0; i < arr_int[0]; i++)
-	{
-	    if (arr_dtype[i] != MPI_LB &&
-		arr_dtype[i] != MPI_UB)
-	    {
-		leaf = convert_mpi_pvfs2_dtype(
-		    &arr_dtype[i], &old_pvfs_dtype_arr[arr_count]);
-		if (leaf != 1)
-		    MPI_Type_free(&arr_dtype[i]); 
-		pvfs_arr_disp[arr_count] = 
-		    (PVFS_size) arr_addr[i];
-		if (has_lb_ub)
-		{
-		    pvfs_arr_len[arr_count] = 
-			arr_int[i+1];
-		}
-		arr_count++;
-	    }
-	}
-
-	/* If a MPI_UB or MPI_LB did exist, we have to
-	 * resize the datatype */
-	if (has_lb_ub)
-	{
-	    PVFS_Request *tmp_pvfs_dtype = NULL;
-	    if ((tmp_pvfs_dtype = ADIOI_Malloc(sizeof(PVFS_Request))) == NULL)
-		fprintf(stderr, "convert_mpi_pvfs2_dtype: "
-			"Failed to allocate PVFS_Request\n");
-	    
-	    ret = PVFS_Request_struct(arr_count, pvfs_arr_len, 
-				      pvfs_arr_disp,
-				      old_pvfs_dtype_arr, tmp_pvfs_dtype);
-	    if (ret != 0)
-		fprintf(stderr, "Error in PVFS_Request_struct\n");
-
-	    arr_count = 0;
-	    for (i = 0; i < arr_int[0]; i++)
-	    {
-		if (arr_dtype[i] != MPI_LB &&
-		    arr_dtype[i] != MPI_UB)
-		{
-		    PVFS_Request_free(&old_pvfs_dtype_arr[arr_count]);
-		    arr_count++;
-		}
-	    }
-	    
-#ifdef DEBUG_DTYPE
-	    fprintf(stderr, "STRUCT(WITHOUT %d LB or UB)(%d,[",
-		    arr_int[0] - arr_count, arr_count);
-	    for (i = 0; i < arr_count; i++)
-		fprintf(stderr, "(%d,%Ld) ",
-			pvfs_arr_len[i],
-			pvfs_arr_disp[i]);
-	    fprintf(stderr, "]\n");
-	    fprintf(stderr, "RESIZED(LB = %Ld, EXTENT = %Ld)\n",
-		    pvfs_lb, pvfs_extent);
-#endif 
-	    ret = PVFS_Request_resized(*tmp_pvfs_dtype, 
-				       pvfs_lb, pvfs_extent, pvfs_dtype);
-	    if (ret != 0)
-		fprintf(stderr, "Error in PVFS_Request_resize\n");
-
-	    PVFS_Request_free(tmp_pvfs_dtype);
-	    ADIOI_Free(tmp_pvfs_dtype);
-	}
-	else /* No MPI_LB or MPI_UB datatypes */
-	{
-	    ret = PVFS_Request_struct(arr_int[0], &arr_int[1], 
-				      pvfs_arr_disp,
-				      old_pvfs_dtype_arr, pvfs_dtype);
-	    if (ret != 0)
-		fprintf(stderr, "Error in PVFS_Request_struct\n");
-
-	    for (i = 0; i < arr_int[0]; i++)
-	    {
-		if (arr_dtype[i] != MPI_LB &&
-		    arr_dtype[i] != MPI_UB)
-		    PVFS_Request_free(&old_pvfs_dtype_arr[i]);
-	    }
-
-#ifdef DEBUG_DTYPE
-	    print_dtype_info(combiner,    
-			     num_int,
-			     num_addr,
-			     num_dtype,
-			     arr_int,
-			     arr_addr,
-			     arr_dtype);
-#endif 
-	}
-
-	ADIOI_Free(arr_int);
-	ADIOI_Free(arr_addr);
-	ADIOI_Free(arr_dtype);
-
-	ADIOI_Free(old_pvfs_dtype_arr);
-	ADIOI_Free(pvfs_arr_disp);
-	ADIOI_Free(pvfs_arr_len);
-
-	return ret;
-    }
-
-    /* Shouldn't have gotten here */
-    fprintf(stderr, "convert_mpi_pvfs2_dtype:  SERIOUS ERROR\n");
-    return -1;
-}
-
-int convert_named(MPI_Datatype *mpi_dtype, 
-		  PVFS_Request *pvfs_dtype, int combiner)
-{ 
-    int ret = -1;
-#ifdef DEBUG_DTYPE
-    fprintf(stderr, "NAMED");
-#endif
-
-    if (MPI_CHAR == *mpi_dtype)
-    {
-	    ret = PVFS_Request_contiguous(1, PVFS_CHAR, pvfs_dtype);
-#ifdef DEBUG_DTYPE
-	    fprintf(stderr, "-MPI_CHAR\n");
-#endif
-    }
-    else if ( MPI_BYTE == *mpi_dtype )
-    {
-            ret = PVFS_Request_contiguous(1, PVFS_BYTE, pvfs_dtype);
-#ifdef DEBUG_DTYPE
-            fprintf(stderr, "-MPI_BYTE\n");
-#endif
-    }
-    else if ( MPI_SHORT == *mpi_dtype )
-    {
-            ret = PVFS_Request_contiguous(1, PVFS_SHORT, pvfs_dtype);
-#ifdef DEBUG_DTYPE
-            fprintf(stderr, "-MPI_SHORT\n");
-#endif
-    }
-    else if ( MPI_INT == *mpi_dtype )
-    {
-            ret = PVFS_Request_contiguous(1, PVFS_INT, pvfs_dtype);
-#ifdef DEBUG_DTYPE
-            fprintf(stderr, "-MPI_INT\n");
-#endif
-    }
-    else if ( MPI_LONG == *mpi_dtype )
-    {
-            ret = PVFS_Request_contiguous(1, PVFS_LONG, pvfs_dtype);
-#ifdef DEBUG_DTYPE
-            fprintf(stderr, "-MPI_LONG\n");
-#endif
-    }
-    else if ( MPI_FLOAT == *mpi_dtype )
-    {
-	    ret  = PVFS_Request_contiguous(1, PVFS_FLOAT, pvfs_dtype);
-#ifdef DEBUG_DTYPE
-	    fprintf(stderr, "-MPI_FLOAT\n");
-#endif
-    }
-    else if ( MPI_DOUBLE == *mpi_dtype )
-    {
-	    ret = PVFS_Request_contiguous(1, PVFS_DOUBLE, pvfs_dtype);
-#ifdef DEBUG_DTYPE
-	    fprintf(stderr, "-MPI_DOUBLE\n");
-#endif
-    }
-    else if ( MPI_UNSIGNED_CHAR == *mpi_dtype )
-    {
-	    ret = PVFS_Request_contiguous(1, PVFS_UNSIGNED_CHAR, pvfs_dtype);
-#ifdef DEBUG_DTYPE
-	    fprintf(stderr, "-MPI_UNSIGNED_CHAR\n");
-#endif
-    }
-    else if ( MPI_UNSIGNED_SHORT == *mpi_dtype )
-    {
-	    ret = PVFS_Request_contiguous(1, PVFS_UNSIGNED, pvfs_dtype);
-#ifdef DEBUG_DTYPE
-	    fprintf(stderr, "-MPI_UNSIGNED_SHORT\n");
-#endif
-    }
-    else if ( MPI_UNSIGNED == *mpi_dtype )
-    {
-	    ret = PVFS_Request_contiguous(1, PVFS_UNSIGNED, pvfs_dtype);
-#ifdef DEBUG_DTYPE
-	    fprintf(stderr, "-MPI_SHORT\n");
-#endif
-    }
-    else if ( MPI_UNSIGNED_LONG == *mpi_dtype )
-    {
-	    ret = PVFS_Request_contiguous(1, PVFS_UNSIGNED_LONG, pvfs_dtype);
-#ifdef DEBUG_DTYPE
-	    fprintf(stderr, "-MPI_UNSIGNED_LONG\n");
-#endif
-    }
-    else if ( MPI_LONG_DOUBLE == *mpi_dtype )
-    {
-	    ret = PVFS_Request_contiguous(1, PVFS_LONG_DOUBLE, pvfs_dtype);
-#ifdef DEBUG_DTYPE
-	    fprintf(stderr, "-MPI_LONG_DOUBLE\n");
-#endif
-    }
-    else
-    {
-	    fprintf(stderr, "convert_named: predefined type not found");
-	    return -1;
-    }
-    if (ret != 0)
-	fprintf(stderr, "convert_named: Datatype creation failed\n");
-    return ret;
-}
-
-void print_dtype_info(int combiner,    
-		      int num_int,
-		      int num_addr,
-		      int num_dtype,
-		      int *arr_int,
-		      MPI_Aint *arr_addr,
-		      MPI_Datatype *arr_dtype)
-{
-    int i = -1;
-    switch (combiner)
-    {
-	case MPI_COMBINER_CONTIGUOUS:
-	    fprintf(stderr, "CONTIG(%d)\n", arr_int[0]);
-	    break;
-	case MPI_COMBINER_VECTOR:
-	    fprintf(stderr, "VECTOR(%d,%d,%d)\n", 
-		    arr_int[0], arr_int[1], arr_int[2]);
-	    break;
-	case MPI_COMBINER_HVECTOR:
-	    fprintf(stderr, "HVECTOR(%d,%d,%ld)\n",
-		    arr_int[0], arr_int[1],arr_addr[0]);
-	    break;
-	case MPI_COMBINER_INDEXED:
-	    fprintf(stderr, "INDEXED(%d,[",
-		    arr_int[0]);
-	    for (i = 0; i < arr_int[0]; i++)
-		fprintf(stderr, "(%d,%d) ",
-			arr_int[1+i],
-			arr_int[arr_int[0]+1+i]);
-	    fprintf(stderr, "]\n");
-	    break;
-	case MPI_COMBINER_HINDEXED:
-	    fprintf(stderr, "HINDEXED(%d,[",
-		    arr_int[0]);
-	    for (i = 0; i < arr_int[0]; i++)
-		fprintf(stderr, "(%d,%lld) ",
-			arr_int[1+i],
-			(long long)arr_addr[i]);
-	    fprintf(stderr, "]\n");
-	    break;
-	case MPI_COMBINER_STRUCT:
-	    fprintf(stderr, "STRUCT(%d,[",
-		    arr_int[0]);
-	    for (i = 0; i < arr_int[0]; i++)
-		fprintf(stderr, "(%d,%lld) ",
-			arr_int[1+i],
-			(long long) arr_addr[i]);
-	    fprintf(stderr, "]\n");
-	    break;
-	case MPI_COMBINER_DUP:
-	    fprintf(stderr, "DUP\n");
-	    break;
-	default:
-	    fprintf(stderr, "no available information on this datatype");
-    }
-}
diff --git a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_io_list.c b/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_io_list.c
deleted file mode 100644
index a88aaf39faa..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_io_list.c
+++ /dev/null
@@ -1,660 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- 
- *   vim: ts=8 sts=4 sw=4 noexpandtab
- *
- *   Copyright (C) 2006 Unknown (TODO: fix this)
- */
-
-#include <assert.h>
-#include "adio.h"
-#include "adio_extern.h"
-#include "ad_pvfs2.h"
-#include "ad_pvfs2_io.h"
-#include "ad_pvfs2_common.h"
-
-#define COALESCE_REGIONS  /* TODO: would we ever want to *not* coalesce? */
-#define MAX_OL_COUNT 64
-int ADIOI_PVFS2_StridedListIO(ADIO_File fd, void *buf, int count,
-			      MPI_Datatype datatype, int file_ptr_type,
-			      ADIO_Offset offset, ADIO_Status *status,
-			      int *error_code, int rw_type)
-{
-    /* list I/O parameters */
-    int i = -1, ret = -1;
-    int tmp_filetype_size = -1;
-    int64_t cur_io_size = 0, io_size = 0;
-    int etype_size = -1;
-    int num_etypes_in_filetype = -1, num_filetypes = -1;
-    int etypes_in_filetype = -1, size_in_filetype = -1;
-    int bytes_into_filetype = 0;
-    MPI_Offset total_bytes_accessed = 0;
-    
-    /* parameters for offset-length pairs arrays */
-    int64_t buf_off_arr[MAX_OL_COUNT];
-    int32_t buf_len_arr[MAX_OL_COUNT];
-    int64_t file_off_arr[MAX_OL_COUNT];
-    int32_t file_len_arr[MAX_OL_COUNT];
-    int32_t buf_ol_count = 0;
-    int32_t file_ol_count = 0;
-    
-    /* parameters for flattened memory and file datatypes*/
-    int flat_buf_index = 0;
-    int flat_file_index = 0;
-    int64_t cur_flat_buf_reg_off = 0;
-    int64_t cur_flat_file_reg_off = 0;
-    ADIOI_Flatlist_node *flat_buf_p, *flat_file_p;
-    MPI_Count buftype_size = -1, filetype_size = -1;
-    MPI_Aint filetype_extent = -1, buftype_extent = -1;;
-    MPI_Aint filetype_lb = -1, buftype_lb = -1;;
-    int buftype_is_contig = -1, filetype_is_contig = -1;
-    
-    /* PVFS2 specific parameters */
-    PVFS_Request mem_req, file_req;
-    ADIOI_PVFS2_fs * pvfs_fs;
-    PVFS_sysresp_io resp_io;
-    static char myname[] = "ADIOI_PVFS2_STRIDED_LISTIO";
-
-    if (fd->atomicity) {
-        *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-                                           MPIR_ERR_RECOVERABLE,
-                                           myname, __LINE__,
-                                           MPI_ERR_ARG,
-                                           "Atomic noncontiguous writes"
-					   " are not supported by PVFS2", 0);
-        return -1;
-    }
-
-    MPI_Type_size_x(fd->filetype, &filetype_size);
-    if (filetype_size == 0) {
-        *error_code = MPI_SUCCESS;
-        return -1;
-    }
-    MPI_Type_get_extent(fd->filetype, &filetype_lb, &filetype_extent);
-    MPI_Type_size_x(datatype, &buftype_size);
-    MPI_Type_get_extent(datatype, &buftype_lb, &buftype_extent);
-    io_size = buftype_size*count;
-
-    pvfs_fs = (ADIOI_PVFS2_fs*)fd->fs_ptr;
-    
-    /* Flatten the memory datatype
-     * (file datatype has already been flattened in ADIO open
-     * unless it is contibuous, then we need to flatten it manually)
-     * and set the correct buffers for flat_buf and flat_file */
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-    if (buftype_is_contig == 0)
-    {
-	flat_buf_p = ADIOI_Flatten_and_find(datatype);
-    }
-    else 
-    {
-	/* flatten and add to the list */
-	flat_buf_p = (ADIOI_Flatlist_node *) ADIOI_Malloc
-	    (sizeof(ADIOI_Flatlist_node));
-	flat_buf_p->blocklens = (ADIO_Offset*)ADIOI_Malloc(sizeof(ADIO_Offset));
-	flat_buf_p->indices = 
-		(ADIO_Offset *) ADIOI_Malloc(sizeof(ADIO_Offset));
-	/* For the buffer, we can optimize the buftype, this is not 
-	 * possible with the filetype since it is tiled */
-	buftype_size = buftype_size*count;
-	buftype_extent = buftype_size*count;
-	flat_buf_p->blocklens[0] = buftype_size;
-	flat_buf_p->indices[0] = 0;
-	flat_buf_p->count = 1;
-    }
-    if (filetype_is_contig == 0)
-    {
-	    /* TODO: why does avery say this should already have been
-	     * flattened in Open, but also says contig types don't get
-	     * flattened */
-	flat_file_p = ADIOI_Flatten_and_find(fd->filetype);
-    }
-    else
-    {
-        /* flatten and add to the list */
-        flat_file_p = (ADIOI_Flatlist_node *) ADIOI_Malloc
-            (sizeof(ADIOI_Flatlist_node));
-        flat_file_p->blocklens =(ADIO_Offset*)ADIOI_Malloc(sizeof(ADIO_Offset));
-        flat_file_p->indices = 
-		(ADIO_Offset *) ADIOI_Malloc(sizeof(ADIO_Offset));
-        flat_file_p->blocklens[0] = filetype_size;
-        flat_file_p->indices[0] = 0;
-        flat_file_p->count = 1;
-    }
-        
-    /* Find out where we are in the flattened filetype (the block index, 
-     * how far into the block, and how many bytes_into_filetype)
-     * If the file_ptr_type == ADIO_INDIVIDUAL we will use disp, fp_ind 
-     * to figure this out (offset should always be zero) 
-     * If file_ptr_type == ADIO_EXPLICIT, we will use disp and offset 
-     * to figure this out. */
-
-    etype_size = fd->etype_size;
-    num_etypes_in_filetype = filetype_size / etype_size;
-    if (file_ptr_type == ADIO_INDIVIDUAL)
-    {
-	int flag = 0;
-	/* Should have already been flattened in ADIO_Open*/
-	num_filetypes = -1;
-	while (!flag)
-	{
-	    num_filetypes++;
-	    for (i = 0; i < flat_file_p->count; i++)
-	    {
-		/* Start on a non zero-length region */
-		if (flat_file_p->blocklens[i])
-		{
-		    if (fd->disp + flat_file_p->indices[i] +
-			(num_filetypes * filetype_extent) +
-			flat_file_p->blocklens[i] > fd->fp_ind &&
-			fd->disp + flat_file_p->indices[i] <=
-			fd->fp_ind)
-		    {
-			flat_file_index = i;
-			cur_flat_file_reg_off = fd->fp_ind -
-			    (fd->disp + flat_file_p->indices[i] +
-			     (num_filetypes * filetype_extent));
-			flag = 1;
-			break;
-		    }
-		    else
-			bytes_into_filetype += flat_file_p->blocklens[i];
-		}
-	    }
-	}
-	/* Impossible that we don't find it in this datatype */
-	assert(i != flat_file_p->count);
-    }
-    else
-    { 
-	num_filetypes = (int) (offset / num_etypes_in_filetype);
-	etypes_in_filetype = (int) (offset % num_etypes_in_filetype);
-	size_in_filetype = etypes_in_filetype * etype_size;
-
-	tmp_filetype_size = 0;
-	for (i=0; i<flat_file_p->count; i++) {
-	    tmp_filetype_size += flat_file_p->blocklens[i];
-	    if (tmp_filetype_size > size_in_filetype) 
-	    {
-		flat_file_index = i;
-		cur_flat_file_reg_off = flat_file_p->blocklens[i] - 
-		    (tmp_filetype_size - size_in_filetype);
-		bytes_into_filetype = offset * filetype_size -
-		    flat_file_p->blocklens[i];
-		break;
-	    }
-	}
-    }
-#ifdef DEBUG_LIST
-    fprintf(stderr, "ADIOI_PVFS2_StridedListIO: (fd->fp_ind=%Ld,fd->disp=%Ld,"
-            " offset=%Ld)\n(flat_file_index=%d,cur_flat_file_reg_off=%Ld,"
-	    "bytes_into_filetype=%d)\n",
-            fd->fp_ind, fd->disp, offset, flat_file_index, 
-	    cur_flat_file_reg_off, bytes_into_filetype);
-#endif
-#ifdef DEBUG_LIST2
-    fprintf(stderr, "flat_buf:\n");
-    for (i = 0; i < flat_buf_p->count; i++)
-	fprintf(stderr, "(offset, length) = (%Ld, %d)\n",
-	       flat_buf_p->indices[i],
-	       flat_buf_p->blocklens[i]);
-    fprintf(stderr, "flat_file:\n");
-    for (i = 0; i < flat_file_p->count; i++)
-	fprintf(stderr, "(offset, length) = (%Ld, %d)\n",
-	       flat_file_p->indices[i],
-	       flat_file_p->blocklens[i]);
-#endif    
-
-    /* total data written */
-    cur_io_size = 0;
-    while (cur_io_size != io_size)
-    {
-	/* Initialize the temporarily unrolling lists and 
-	 * and associated variables */
-	buf_ol_count = 0;
-	file_ol_count = 0;
-	for (i = 0; i < MAX_OL_COUNT; i++)
-	{
-	    buf_off_arr[i] = 0;
-	    buf_len_arr[i] = 0;
-	    file_off_arr[i] = 0;
-	    file_len_arr[i] = 0;
-	}
-
-        /* Generate the offset-length pairs for a
-         * list I/O operation */
-	gen_listio_arr(flat_buf_p,
-		       &flat_buf_index,
-		       &cur_flat_buf_reg_off,
-		       buftype_size,
-		       buftype_extent,
-		       flat_file_p,
-		       &flat_file_index,
-		       &cur_flat_file_reg_off,
-		       filetype_size,
-		       filetype_extent,
-		       MAX_OL_COUNT,
-		       fd->disp,
-		       bytes_into_filetype,
-		       &cur_io_size,
-		       io_size,
-		       buf_off_arr,
-		       buf_len_arr,
-		       &buf_ol_count,
-		       file_off_arr,
-		       file_len_arr,
-		       &file_ol_count);
-
-	assert(buf_ol_count <= MAX_OL_COUNT);
-	assert(file_ol_count <= MAX_OL_COUNT);
-#ifdef DEBUG_LIST2
-	print_buf_file_ol_pairs(buf_off_arr,
-				buf_len_arr,
-				buf_ol_count,
-				file_off_arr,
-				file_len_arr,
-				file_ol_count,
-				buf,
-				rw_type);
-#endif
-#ifdef DEBUG_LIST2
-	do {
-	    int y, z;
-	    fprintf(stderr, "ad_pvfs2_io_list.c::\n");
-	    for (y = 0; y < buf_ol_count; y++)
-	    {
-		for (z = 0; z < buf_len_arr[y]; z++)
-		{
-		    fprintf(stderr, "buf[%d][%d]=%c\n",
-			    y, z, ((char *) buf + buf_off_arr[y])[z]);
-		}
-	    }
-	} while (0);
-#endif
-	
-	/* Run list I/O operation */
-	ret = PVFS_Request_hindexed(buf_ol_count, buf_len_arr,
-				    buf_off_arr, PVFS_BYTE, &mem_req);
-
-	ret = PVFS_Request_hindexed(file_ol_count, file_len_arr,
-				    file_off_arr, PVFS_BYTE, &file_req);
-	if (rw_type == READ)
-	{
-	    ret = PVFS_sys_read(pvfs_fs->object_ref, file_req, 0,
-				buf, mem_req, 
-				&(pvfs_fs->credentials), &resp_io);
-	}
-	else 
-	{
-	    ret = PVFS_sys_write(pvfs_fs->object_ref, file_req, 0,
-				 buf, mem_req, 
-				 &(pvfs_fs->credentials), &resp_io);
-	}
-	if (ret != 0) 
-	{
-	    fprintf(stderr, "ADIOI_PVFS2_StridedListIO: Warning - PVFS_sys_"
-		    "read/write returned %d and completed %lld bytes.\n",
-		    ret, (long long)resp_io.total_completed);
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE,
-					       myname, __LINE__,
-					       ADIOI_PVFS2_error_convert(ret),
-					       "Error in PVFS_sys_io \n", 0);
-	    PVFS_Request_free(&mem_req);
-	    PVFS_Request_free(&file_req);
-	    goto error_state;
-	}
-	total_bytes_accessed += resp_io.total_completed;
-
-	PVFS_Request_free(&mem_req);
-	PVFS_Request_free(&file_req);
-    }
-    
-#ifdef DEBUG_LIST
-    fprintf(stderr, "ADIOI_PVFS2_StridedListIO: "
-            "total_bytes_accessed=%Ld,ret=%d\n", 
-	    total_bytes_accessed, ret);
-#endif
-
-    if (file_ptr_type == ADIO_INDIVIDUAL) 
-	fd->fp_ind += total_bytes_accessed;
-    *error_code = MPI_SUCCESS;
-
-error_state:
-#ifdef HAVE_STATUS_SET_BYTES
-    /* TODO: why the cast? */
-    MPIR_Status_set_bytes(status, datatype, total_bytes_accessed);
-/* This is a temporary way of filling in status. The right way is to
-   keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
-#endif
-    if (buftype_is_contig == 0)
-	ADIOI_Delete_flattened(datatype);
-    else
-    {
-	ADIOI_Free(flat_buf_p->blocklens);
-	ADIOI_Free(flat_buf_p->indices);
-	ADIOI_Free(flat_buf_p);
-    }
-
-    if (filetype_is_contig == 0)
-	ADIOI_Delete_flattened(fd->filetype);
-    else
-    {
-	ADIOI_Free(flat_file_p->blocklens);
-	ADIOI_Free(flat_file_p->indices);
-	ADIOI_Free(flat_file_p);
-    }
-
-    return 0;
-}
-
-/* To do: Fix the code to coalesce the offset-length pairs for memory
- * and file. */
-
-/* gen_listio_arr - fills in offset-length pairs for memory and file
- * for list I/O */
-int gen_listio_arr(ADIOI_Flatlist_node *flat_buf_p,
-		   int *flat_buf_index_p,
-		   int64_t *cur_flat_buf_reg_off_p,
-		   int flat_buf_size,
-		   int flat_buf_extent,
-		   ADIOI_Flatlist_node *flat_file_p,
-		   int *flat_file_index_p,
-		   int64_t *cur_flat_file_reg_off_p,
-		   int flat_file_size,
-		   int flat_file_extent,
-		   int max_ol_count,
-		   ADIO_Offset disp,
-		   int bytes_into_filetype,
-		   int64_t *bytes_completed,
-		   int64_t total_io_size,
-		   int64_t buf_off_arr[],
-		   int32_t buf_len_arr[],
-		   int32_t *buf_ol_count_p,
-		   int64_t file_off_arr[],
-		   int32_t file_len_arr[],
-		   int32_t *file_ol_count_p)
-{
-    int region_size = -1;
-    
-    /* parameters for flattened memory and file datatypes*/
-    int64_t cur_flat_buf_reg_left = 0;
-    int64_t cur_flat_file_reg_left = 0;
-    
-#ifdef DEBUG_LIST2
-    fprintf(stderr, "gen_list_arr:\n");
-#endif
-
-    if ((*buf_ol_count_p) != 0 ||(*file_ol_count_p) != 0)
-    {
-	fprintf(stderr, "buf_ol_count != 0 || file_ol_count != 0\n");
-	return -1;
-    }
-    
-    /* Start on a non-zero memory and file region 
-     * Note this does not affect the bytes_completed 
-     * since no data is in these regions.  Initialize the 
-     * first memory and file offsets. */
-    while (flat_buf_p->blocklens[(*flat_buf_index_p)] == 0)
-    {
-	(*flat_buf_index_p) = ((*flat_buf_index_p) + 1) % 
-	    flat_buf_p->count;
-    }
-    buf_off_arr[*buf_ol_count_p] =
-	(*bytes_completed / flat_buf_size) * 
-	flat_buf_extent + 
-	flat_buf_p->indices[*flat_buf_index_p] +
-	*cur_flat_buf_reg_off_p;
-    buf_len_arr[*buf_ol_count_p] = 0;
-
-    while (flat_file_p->blocklens[(*flat_file_index_p)] == 0)
-    {
-	(*flat_file_index_p) = ((*flat_file_index_p) + 1) % 
-	    flat_file_p->count;
-    }
-    file_off_arr[*file_ol_count_p] = disp + 
-	(((bytes_into_filetype + *bytes_completed) / flat_file_size) * 
-	 flat_file_extent) + 
-	flat_file_p->indices[*flat_file_index_p] +
-	*cur_flat_file_reg_off_p;
-    file_len_arr[*file_ol_count_p] = 0;
-
-#ifdef DEBUG_LIST2
-    fprintf(stderr, "initial buf_off_arr[%d] = %Ld\n", *buf_ol_count_p,
-	    buf_off_arr[*buf_ol_count_p]);
-    fprintf(stderr, "initial file_off_arr[%d] = %Ld\n", *file_ol_count_p,
-	    file_off_arr[*file_ol_count_p]);
-#endif
-
-    while (*bytes_completed != total_io_size
-	   && (*buf_ol_count_p) < max_ol_count
-	   && (*file_ol_count_p) < max_ol_count)
-    {
-	/* How much data is left in the current piece in
-	 * the flattened datatypes */
-	cur_flat_buf_reg_left = flat_buf_p->blocklens[*flat_buf_index_p]
-	    - *cur_flat_buf_reg_off_p;
-	cur_flat_file_reg_left = flat_file_p->blocklens[*flat_file_index_p]
-	    - *cur_flat_file_reg_off_p;
-
-#ifdef DEBUG_LIST2
-	fprintf(stderr, 
-		"flat_buf_index=%d flat_buf->blocklens[%d]=%d\n"
-		"cur_flat_buf_reg_left=%Ld "
-		"*cur_flat_buf_reg_off_p=%Ld\n" 
-		"flat_file_index=%d flat_file->blocklens[%d]=%d\n"
-		"cur_flat_file_reg_left=%Ld "
-		"*cur_flat_file_reg_off_p=%Ld\n" 
-		"bytes_completed=%Ld\n"
-		"buf_ol_count=%d file_ol_count=%d\n"
-		"buf_len_arr[%d]=%d file_len_arr[%d]=%d\n\n",
-		*flat_buf_index_p, *flat_buf_index_p, 
-		flat_buf_p->blocklens[*flat_buf_index_p],
-		cur_flat_buf_reg_left,
-		*cur_flat_buf_reg_off_p,
-		*flat_file_index_p, *flat_file_index_p, 
-		flat_file_p->blocklens[*flat_file_index_p],
-		cur_flat_file_reg_left,
-		*cur_flat_file_reg_off_p,
-		*bytes_completed,
-		*buf_ol_count_p, *file_ol_count_p,
-		*buf_ol_count_p,
-		buf_len_arr[*buf_ol_count_p],
-		*file_ol_count_p,
-		file_len_arr[*file_ol_count_p]);
-#endif
-
-	/* What is the size of the next contiguous region agreed
-	 * upon by both memory and file regions that does not
-	 * surpass the file size */
-	if (cur_flat_buf_reg_left > cur_flat_file_reg_left)
-	    region_size = cur_flat_file_reg_left;
-	else
-	    region_size = cur_flat_buf_reg_left;
-	
-	if (region_size > total_io_size - *bytes_completed)
-	    region_size = total_io_size - *bytes_completed;
-	
-	/* Add this piece to both the mem and file arrays
-	 * coalescing offset-length pairs if possible and advance 
-	 * the pointers through the flatten mem and file datatypes
-	 * as well Note: no more than a single piece can be done 
-	 * since we take the smallest one possible */
-	
-	if (cur_flat_buf_reg_left == region_size)
-	{
-#ifdef DEBUG_LIST2
-	    fprintf(stderr, "reached end of memory block...\n");
-#endif
-	    (*flat_buf_index_p) = ((*flat_buf_index_p) + 1) % 
-		flat_buf_p->count;
-	    while (flat_buf_p->blocklens[(*flat_buf_index_p)] == 0)
-	    {
-		(*flat_buf_index_p) = ((*flat_buf_index_p) + 1) % 
-		    flat_buf_p->count;
-	    }
-	    *cur_flat_buf_reg_off_p = 0;
-
-#ifdef COALESCE_REGIONS
-	    if (*buf_ol_count_p != 0)
-	    {
-		if (buf_off_arr[(*buf_ol_count_p) - 1] +
-		    buf_len_arr[(*buf_ol_count_p) - 1] ==
-		    buf_off_arr[*buf_ol_count_p])
-		{
-		    buf_len_arr[(*buf_ol_count_p) - 1] +=
-			region_size;
-		}
-		else
-		{
-		    buf_len_arr[*buf_ol_count_p] += region_size;
-		    (*buf_ol_count_p)++;
-		}
-	    }
-	    else
-	    {
-#endif
-		buf_len_arr[*buf_ol_count_p] += region_size;
-		(*buf_ol_count_p)++;
-#ifdef COALESCE_REGIONS
-	    }
-#endif
-
-	    /* Don't prepare for the next piece if we have reached 
-	     * the limit or else it will segment fault. */
-	    if ((*buf_ol_count_p) != max_ol_count)
-	    {
-		buf_off_arr[*buf_ol_count_p] = 
-		    ((*bytes_completed + region_size) / flat_buf_size) * 
-		    flat_buf_extent + 
-		    flat_buf_p->indices[*flat_buf_index_p] +
-		    (*cur_flat_buf_reg_off_p);
-		buf_len_arr[*buf_ol_count_p] = 0;
-	    }
-	}
-	else if (cur_flat_buf_reg_left > region_size)
-	{
-#ifdef DEBUG_LIST2
-	    fprintf(stderr, "advanced %d in memory block...\n",
-		    region_size);
-#endif
-	    (*cur_flat_buf_reg_off_p) += region_size;
-	    buf_len_arr[*buf_ol_count_p] += region_size;
-	}
-	else
-	{
-	    fprintf(stderr, "gen_listio_arr: Error\n");
-	}
-	
-	/* To calculate the absolute file offset we need to 
-	 * add the disp, how many filetypes we have gone through,
-	 * the relative block offset in the filetype and how far
-	 * into the block we have gone. */
-	if (cur_flat_file_reg_left == region_size)
-	{
-#ifdef DEBUG_LIST2
-	    fprintf(stderr, "reached end of file block...\n");
-#endif
-	    (*flat_file_index_p) = ((*flat_file_index_p) + 1) % 
-		flat_file_p->count;
-	    while (flat_file_p->blocklens[(*flat_file_index_p)] == 0)
-	    {
-		(*flat_file_index_p) = ((*flat_file_index_p) + 1) % 
-		    flat_file_p->count;
-	    }
-	    (*cur_flat_file_reg_off_p) = 0;
-
-#ifdef COALESCE_REGIONS
-            if (*file_ol_count_p != 0)
-            {
-                if (file_off_arr[(*file_ol_count_p) - 1] +
-                    file_len_arr[(*file_ol_count_p) - 1] ==
-                    file_off_arr[*file_ol_count_p])
-                {
-                    file_len_arr[(*file_ol_count_p) - 1] +=
-                        region_size;
-                }
-                else
-                {
-                    file_len_arr[*file_ol_count_p] += region_size;
-                    (*file_ol_count_p)++;
-                }
-            }
-            else
-            {
-#endif
-                file_len_arr[*file_ol_count_p] += region_size;
-                (*file_ol_count_p)++;
-#ifdef COALESCE_REGIONS
-            }
-#endif
-
-	    /* Don't prepare for the next piece if we have reached
-             * the limit or else it will segment fault. */
-            if ((*file_ol_count_p) != max_ol_count)
-	    {
-		file_off_arr[*file_ol_count_p] = disp + 
-		    (((bytes_into_filetype + *bytes_completed + region_size) 
-		      / flat_file_size) * 
-		     flat_file_extent) + 
-		    flat_file_p->indices[*flat_file_index_p] +
-		    (*cur_flat_file_reg_off_p);
-		file_len_arr[*file_ol_count_p] = 0;
-	    }
-	}
-	else if (cur_flat_file_reg_left > region_size)
-	{
-#ifdef DEBUG_LIST2
-	    fprintf(stderr, "advanced %d in file block...\n",
-		    region_size);
-#endif
-	    (*cur_flat_file_reg_off_p) += region_size;
-	    file_len_arr[*file_ol_count_p] += region_size;
-	}
-	else
-	{
-	    fprintf(stderr, "gen_listio_arr: Error\n");
-	}
-#ifdef DEBUG_LIST2
-	fprintf(stderr, 
-		"------------------------------\n\n");
-#endif
-	*bytes_completed += region_size;
-    }
-    /* Increment the count if we stopped in the middle of a 
-     * memory or file region */
-    if (*cur_flat_buf_reg_off_p != 0)
-	(*buf_ol_count_p)++;
-    if (*cur_flat_file_reg_off_p != 0)
-	(*file_ol_count_p)++;
-
-    return 0;
-}
-
-void print_buf_file_ol_pairs(int64_t buf_off_arr[],
-			     int32_t buf_len_arr[],
-			     int32_t buf_ol_count,
-			     int64_t file_off_arr[],
-			     int32_t file_len_arr[],
-			     int32_t file_ol_count,
-			     void *buf,
-			     int rw_type)
-{
-    int i = -1;
-    
-    fprintf(stderr, "buf_ol_pairs(offset,length) count = %d\n",
-	    buf_ol_count);
-    for (i = 0; i < buf_ol_count; i++)
-    {
-	fprintf(stderr, "(%lld, %d) ", (long long)buf_off_arr[i], buf_len_arr[i]);
-    }
-    fprintf(stderr, "\n");
-
-    fprintf(stderr, "file_ol_pairs(offset,length) count = %d\n",
-	    file_ol_count);
-    for (i = 0; i < file_ol_count; i++)
-    {
-	fprintf(stderr, "(%lld, %d) ", (long long)file_off_arr[i], file_len_arr[i]);
-    }
-    fprintf(stderr, "\n\n");
-
-}
diff --git a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_open.c b/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_open.c
deleted file mode 100644
index f266c88e5c9..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_open.c
+++ /dev/null
@@ -1,246 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*-
- * vim: ts=8 sts=4 sw=4 noexpandtab
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_pvfs2.h"
-#include "ad_pvfs2_common.h"
-
-/* open_status is helpful for bcasting values around */
-struct open_status_s {
-    int error;
-    PVFS_object_ref object_ref;
-};
-typedef struct open_status_s open_status;
-    
-    /* steps for getting a handle:  (it gets a little convoluted, but at least
-     * it's deterministic) 
-     * . lookup the file.  
-     * . if lookup succeeds, but we were passed MPI_MODE_EXCL, that's an error
-     * . if lookup fails, the file might not exist. 
-     *		in that case, create the file if we were passed MPI_MODE_CREATE 
-     * . if the create fails, that means someone else created the file between
-     *    our call to lookup and our call to create (like if N processors all
-     *    open the same file with MPI_COMM_SELF).  Then we can just look up the
-     *    file (which now exists).
-     *
-     * the good news is that only one processor does this and broadcasts the
-     * handle to everyone else in the communicator
-     */
-static void fake_an_open(PVFS_fs_id fs_id, char *pvfs_name, int access_mode,
-	                 int nr_datafiles, PVFS_size strip_size,
-                         ADIOI_PVFS2_fs *pvfs2_fs, 
-			 open_status *o_status)
-{
-    int ret;
-    PVFS_sysresp_lookup resp_lookup;
-    PVFS_sysresp_getparent resp_getparent;
-    PVFS_sysresp_create resp_create;
-    PVFS_sys_attr attribs;
-    PVFS_sys_dist* dist;
-
-    ADIOI_PVFS2_makeattribs(&attribs);
-    if (nr_datafiles > 0 ) {
-	attribs.dfile_count = nr_datafiles;
-	attribs.mask |= PVFS_ATTR_SYS_DFILE_COUNT;
-    }
-
-    dist = NULL;
-    
-    memset(&resp_lookup, 0, sizeof(resp_lookup));
-    memset(&resp_getparent, 0, sizeof(resp_getparent));
-    memset(&resp_create, 0, sizeof(resp_create));
-
-
-    ret = PVFS_sys_lookup(fs_id, pvfs_name,
-	    &(pvfs2_fs->credentials), &resp_lookup, PVFS2_LOOKUP_LINK_FOLLOW);
-    if ( ret == (-PVFS_ENOENT)) {
-	if (access_mode & ADIO_CREATE)  {
-	    ret = PVFS_sys_getparent(fs_id, pvfs_name,
-		    &(pvfs2_fs->credentials), &resp_getparent); 
-	    if (ret < 0) {
-		FPRINTF(stderr, "pvfs_sys_getparent returns with %d\n", ret);
-		o_status->error = ret;
-		return;
-	    }
-            
-            /* Set the distribution strip size if specified */
-            if (0 < strip_size) {
-                /* Note that the distribution is hardcoded here */
-                dist = PVFS_sys_dist_lookup("simple_stripe");
-                ret = PVFS_sys_dist_setparam(dist,
-                                             "strip_size",
-                                             &strip_size);
-                if (ret < 0)
-                {
-                    FPRINTF(stderr,
-                            "pvfs_sys_dist_setparam returns with %d\n", ret);
-                    o_status->error = ret;
-                }
-            }
-
-            /* Perform file creation */
-#ifdef HAVE_PVFS2_CREATE_WITHOUT_LAYOUT
-            ret = PVFS_sys_create(resp_getparent.basename, 
-		    resp_getparent.parent_ref, attribs, 
-		    &(pvfs2_fs->credentials), dist, &resp_create); 
-#else 
-            ret = PVFS_sys_create(resp_getparent.basename, 
-		    resp_getparent.parent_ref, attribs, 
-		    &(pvfs2_fs->credentials), dist, NULL, &resp_create); 
-#endif
-
-	    /* if many creates are happening in this directory, the earlier
-	     * sys_lookup may have returned ENOENT, but the sys_create could
-	     * return EEXISTS.  That means the file has been created anyway, so
-	     * less work for us and we can just open it up and return the
-	     * handle */
-	    if (ret == (-PVFS_EEXIST)) {
-		ret = PVFS_sys_lookup(fs_id, pvfs_name,
-			&(pvfs2_fs->credentials), &resp_lookup, 
-			PVFS2_LOOKUP_LINK_FOLLOW);
-		if ( ret < 0 ) {
-		    o_status->error = ret;
-		    return;
-		}
-		o_status->error = ret;
-		o_status->object_ref = resp_lookup.ref;
-		return;
-	    }
-	    o_status->object_ref = resp_create.ref;
-	} else {
-	    FPRINTF(stderr, "cannot create file without MPI_MODE_CREATE\n");
-	    o_status->error = ret;
-	    return;
-	}
-    } else if (access_mode & ADIO_EXCL) {
-	/* lookup should not succeed if opened with EXCL */
-	o_status->error = -PVFS_EEXIST;
-	return;
-    } else {
-	o_status->object_ref = resp_lookup.ref;
-    }
-    o_status->error = ret;
-    return;
-
-}
-
-
-/* ADIOI_PVFS2_Open:
- *  one process opens (or creates) the file, then broadcasts the result to the
- *  remaining processors. 
- *
- *  ADIO_Open used to perform an optimization when MPI_MODE_CREATE (and before
- * that, MPI_MODE_EXCL) was set.  Because PVFS2 handles file lookup and
- * creation more scalably than other file systems, ADIO_Open now skips any
- * special handling when CREATE is set.  */
-void ADIOI_PVFS2_Open(ADIO_File fd, int *error_code)
-{
-    int rank, ret;
-    PVFS_fs_id cur_fs;
-    static char myname[] = "ADIOI_PVFS2_OPEN";
-    char pvfs_path[PVFS_NAME_MAX] = {0};
-
-    ADIOI_PVFS2_fs *pvfs2_fs;
-
-    /* since one process is doing the open, that means one process is also
-     * doing the error checking.  define a struct for both the object reference
-     * and the error code to broadcast to all the processors */
-
-    open_status o_status = {0, {0, 0}};
-    MPI_Datatype open_status_type;
-    MPI_Datatype types[2] = {MPI_INT, MPI_BYTE};
-    int lens[2] = {1, sizeof(PVFS_object_ref)};
-    MPI_Aint offsets[2];
-    
-    pvfs2_fs = (ADIOI_PVFS2_fs *) ADIOI_Malloc(sizeof(ADIOI_PVFS2_fs));
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (pvfs2_fs == NULL) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   MPI_ERR_UNKNOWN,
-					   "Error allocating memory", 0);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-    MPI_Comm_rank(fd->comm, &rank);
-
-    ADIOI_PVFS2_Init(error_code);
-    if (*error_code != MPI_SUCCESS)
-    {
-	/* ADIOI_PVFS2_INIT handles creating error codes on its own */
-	return;
-    }
-
-    /* currently everyone gets their own credentials */
-    ADIOI_PVFS2_makecredentials(&(pvfs2_fs->credentials));
-
-    /* one process resolves name and will later bcast to others */
-#ifdef ADIOI_MPE_LOGGING
-    MPE_Log_event( ADIOI_MPE_open_a, 0, NULL );
-#endif
-    if (rank == fd->hints->ranklist[0] && fd->fs_ptr == NULL) {
-	/* given the filename, figure out which pvfs filesystem it is on */
-	ret = PVFS_util_resolve(fd->filename, &cur_fs, 
-		pvfs_path, PVFS_NAME_MAX);
-	if (ret < 0 ) {
-	    PVFS_perror("PVFS_util_resolve", ret);
-	    /* TODO: pick a good error for this */
-	    o_status.error = -1;
-	} else  {
-	    fake_an_open(cur_fs, pvfs_path,
-                         fd->access_mode, fd->hints->striping_factor,
-                         fd->hints->striping_unit,
-                         pvfs2_fs, &o_status);
-	}
-
-	/* store credentials and object reference in fd */
-	pvfs2_fs->object_ref = o_status.object_ref;
-	fd->fs_ptr = pvfs2_fs;
-    }
-#ifdef ADIOI_MPE_LOGGING
-    MPE_Log_event( ADIOI_MPE_open_b, 0, NULL );
-#endif
-
-    /* broadcast status and (possibly valid) object reference */
-    MPI_Get_address(&o_status.error, &offsets[0]);
-    MPI_Get_address(&o_status.object_ref, &offsets[1]);
-
-    MPI_Type_struct(2, lens, offsets, types, &open_status_type);
-    MPI_Type_commit(&open_status_type);
-
-    /* Assertion: if we hit this Bcast, then all processes collectively
-     *            called this open.
-     *
-     * That's because deferred open never happens with PVFS2.
-     */
-    MPI_Bcast(MPI_BOTTOM, 1, open_status_type, fd->hints->ranklist[0],
-	      fd->comm);
-    MPI_Type_free(&open_status_type);
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (o_status.error != 0)
-    { 
-	ADIOI_Free(pvfs2_fs);
-	fd->fs_ptr = NULL;
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   ADIOI_PVFS2_error_convert(o_status.error),
-					   "Unknown error", 0);
-	/* TODO: FIX STRING */
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-    pvfs2_fs->object_ref = o_status.object_ref;
-    fd->fs_ptr = pvfs2_fs;
-
-    *error_code = MPI_SUCCESS;
-    return;
-}
diff --git a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_read.c b/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_read.c
deleted file mode 100644
index 007b4dc3933..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_read.c
+++ /dev/null
@@ -1,169 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- 
- *     vim: ts=8 sts=4 sw=4 noexpandtab 
- * 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-#include "ad_pvfs2.h"
-#include "ad_pvfs2_io.h"
-#include "ad_pvfs2_common.h"
-
-void ADIOI_PVFS2_ReadContig(ADIO_File fd, void *buf, int count, 
-			    MPI_Datatype datatype, int file_ptr_type,
-			    ADIO_Offset offset, ADIO_Status *status,
-			    int *error_code)
-{
-    int ret;
-    MPI_Count datatype_size, len;
-    PVFS_Request file_req, mem_req;
-    PVFS_sysresp_io resp_io;
-    ADIOI_PVFS2_fs *pvfs_fs;
-    static char myname[] = "ADIOI_PVFS2_READCONTIG";
-
-    pvfs_fs = (ADIOI_PVFS2_fs*)fd->fs_ptr;
-
-    MPI_Type_size_x(datatype, &datatype_size);
-    len = datatype_size * count;
-
-    ret = PVFS_Request_contiguous(len, PVFS_BYTE, &mem_req);
-    /* --BEGIN ERROR HANDLING-- */
-    if (ret != 0) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   ADIOI_PVFS2_error_convert(ret),
-					   "Error in pvfs_request_contig (memory)", 0);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-    ret = PVFS_Request_contiguous(len, PVFS_BYTE, &file_req);
-    /* --BEGIN ERROR HANDLING-- */
-    if (ret != 0) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   ADIOI_PVFS2_error_convert(ret),
-					   "Error in pvfs_request_contig (file)", 0);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-    if (file_ptr_type == ADIO_INDIVIDUAL) {
-	/* copy individual file pointer into offset variable, continue */
-	offset = fd->fp_ind;
-    }
-
-#ifdef ADIOI_MPE_LOGGING
-    MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
-#endif
-    ret = PVFS_sys_read(pvfs_fs->object_ref, file_req, offset, buf, 
-			mem_req, &(pvfs_fs->credentials), &resp_io);
-#ifdef ADIOI_MPE_LOGGING
-    MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
-#endif
-    /* --BEGIN ERROR HANDLING-- */
-    if (ret != 0 ) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   ADIOI_PVFS2_error_convert(ret),
-					   "Error in PVFS_sys_read", 0);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    if (file_ptr_type == ADIO_INDIVIDUAL) {
-	fd->fp_ind += (int) resp_io.total_completed;
-	/* TODO: WHY THE INT CAST? */
-    }
-    fd->fp_sys_posn = offset + (int)resp_io.total_completed;
-
-#ifdef HAVE_STATUS_SET_BYTES
-    MPIR_Status_set_bytes(status, datatype, resp_io.total_completed);
-#endif
-
-    *error_code = MPI_SUCCESS;
-fn_exit:
-    PVFS_Request_free(&mem_req);
-    PVFS_Request_free(&file_req);
-    return;
-}
-
-static int ADIOI_PVFS2_ReadStridedListIO(ADIO_File fd, void *buf, int count,
-				  MPI_Datatype datatype, int file_ptr_type,
-				  ADIO_Offset offset, ADIO_Status *status,
-				  int *error_code)
-{
-    return ADIOI_PVFS2_StridedListIO(fd, buf, count,
-				     datatype, file_ptr_type,
-				     offset, status,
-				     error_code, READ);
-}
-
-static int ADIOI_PVFS2_ReadStridedDtypeIO(ADIO_File fd, void *buf, int count,
-				   MPI_Datatype datatype, int file_ptr_type,
-				   ADIO_Offset offset, ADIO_Status *status, 
-				   int *error_code)
-{
-    return ADIOI_PVFS2_StridedDtypeIO(fd, buf, count,
-				      datatype, file_ptr_type,
-				      offset, status, error_code,
-				      READ);
-}
-
-void ADIOI_PVFS2_ReadStrided(ADIO_File fd, void *buf, int count,
-			     MPI_Datatype datatype, int file_ptr_type,
-			     ADIO_Offset offset, ADIO_Status *status, int
-			     *error_code)
-{
-    /* four ways (to date) that we can carry out strided i/o accesses:
-     * - naive posix
-     * - 'true' Datatype (from avery)
-     * - new List I/O (from avery)
-     * - classic List I/O  (the one that's always been in ROMIO)
-     * I imagine we'll keep Datatype as an optional optimization, and afer a
-     * release or two promote it to the default 
-     */
-    int ret = -1;
-
-    if (fd->hints->fs_hints.pvfs2.posix_read == ADIOI_HINT_ENABLE) {
-	ADIOI_GEN_ReadStrided(fd, buf, count, datatype, 
-		file_ptr_type, offset, status, error_code);
-	return;
-    }
-    if (fd->hints->fs_hints.pvfs2.dtype_read == ADIOI_HINT_ENABLE) {
-        ret = ADIOI_PVFS2_ReadStridedDtypeIO(fd, buf, count,
-                                             datatype, file_ptr_type,
-                                             offset, status, error_code);
-
-        /* Fall back to list I/O if datatype I/O didn't work */
-        if (ret != 0)
-        {
-            fprintf(stderr,
-                    "Falling back to list I/O since datatype I/O failed\n");
-            ret = ADIOI_PVFS2_ReadStridedListIO(fd, buf, count,
-                                                datatype, file_ptr_type,
-                                                offset, status, error_code);
-        }
-        return;
-    }
-    if (fd->hints->fs_hints.pvfs2.listio_read == ADIOI_HINT_ENABLE) {
-	ret = ADIOI_PVFS2_ReadStridedListIO(fd, buf, count, datatype,
-		file_ptr_type, offset, status, error_code);
-	return;
-    }
-    /* Use classic list I/O if no hints given base case */
-
-    ADIOI_PVFS2_OldReadStrided(fd, buf, count, datatype, 
-	    file_ptr_type, offset, status, error_code);
-    return;
-}
-
-
-/*
- * vim: ts=8 sts=4 sw=4 noexpandtab 
- */
diff --git a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_read_list_classic.c b/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_read_list_classic.c
deleted file mode 100644
index 856c5b8c2fb..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_read_list_classic.c
+++ /dev/null
@@ -1,909 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- 
- * vim: ts=8 sts=4 sw=4 noexpandtab 
- * 
- *   Copyright (C) 2008 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-#include "ad_pvfs2.h"
-
-#include "ad_pvfs2_common.h"
-
-void ADIOI_PVFS2_OldReadStrided(ADIO_File fd, void *buf, int count,
-			     MPI_Datatype datatype, int file_ptr_type,
-			     ADIO_Offset offset, ADIO_Status *status, int
-			     *error_code)
-{
-    /* offset is in units of etype relative to the filetype. */
-    ADIOI_Flatlist_node *flat_buf, *flat_file;
-    int i, j, k,  brd_size, frd_size=0, st_index=0;
-    int sum, n_etypes_in_filetype, size_in_filetype;
-    MPI_Count bufsize;
-    int n_filetypes, etype_in_filetype;
-    ADIO_Offset abs_off_in_filetype=0;
-    MPI_Count filetype_size, etype_size, buftype_size;
-    MPI_Aint filetype_extent, buftype_extent, filetype_lb, buftype_lb; 
-    int buf_count, buftype_is_contig, filetype_is_contig;
-    ADIO_Offset off, disp, start_off, initial_off;
-    int flag, st_frd_size, st_n_filetypes;
-
-    int mem_list_count, file_list_count;
-    PVFS_size *mem_offsets;
-    int64_t *file_offsets;
-    int *mem_lengths;
-    int32_t *file_lengths;
-    int total_blks_to_read;
-
-    int max_mem_list, max_file_list;
-
-    int b_blks_read;
-    int f_data_read;
-    int size_read=0, n_read_lists, extra_blks;
-
-    int end_brd_size, end_frd_size;
-    int start_k, start_j, new_file_read, new_buffer_read;
-    int start_mem_offset;
-    PVFS_Request mem_req, file_req;
-    ADIOI_PVFS2_fs * pvfs_fs;
-    PVFS_sysresp_io resp_io;
-    int err_flag=0;
-    MPI_Offset total_bytes_read = 0;
-    static char myname[] = "ADIOI_PVFS2_ReadStrided";
-
-#define MAX_ARRAY_SIZE 64
-
-    *error_code = MPI_SUCCESS;  /* changed below if error */
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-
-    /* the HDF5 tests showed a bug in this list processing code (see many many
-     * lines down below).  We added a workaround, but common HDF5 file types
-     * are actually contiguous and do not need the expensive workarond */
-    if (!filetype_is_contig) {
-	flat_file = ADIOI_Flatlist;
-	while (flat_file->type != fd->filetype) flat_file = flat_file->next;
-	if (flat_file->count == 1 && !buftype_is_contig)
-	    filetype_is_contig = 1;
-    }
-
-    MPI_Type_size_x(fd->filetype, &filetype_size);
-    if ( ! filetype_size ) {
-#ifdef HAVE_STATUS_SET_BYTES
-	MPIR_Status_set_bytes(status, datatype, 0);
-#endif
-	*error_code = MPI_SUCCESS; 
-	return;
-    }
-
-    MPI_Type_get_extent(fd->filetype, &filetype_lb, &filetype_extent);
-    MPI_Type_size_x(datatype, &buftype_size);
-    MPI_Type_get_extent(datatype, &buftype_lb, &buftype_extent);
-    etype_size = fd->etype_size;
-
-    bufsize = buftype_size * count;
-    
-    pvfs_fs = (ADIOI_PVFS2_fs*)fd->fs_ptr;
-
-    if (!buftype_is_contig && filetype_is_contig) {
-
-/* noncontiguous in memory, contiguous in file. */
-       int64_t file_offset;
-	int32_t file_length;
-
-	flat_buf = ADIOI_Flatten_and_find(datatype);
-
-	off = (file_ptr_type == ADIO_INDIVIDUAL) ? fd->fp_ind : 
-	    fd->disp + etype_size * offset;
-
-	file_list_count = 1;
-	file_offset = off;
-	file_length = 0;
-	total_blks_to_read = count*flat_buf->count;
-	b_blks_read = 0;
-
-	/* allocate arrays according to max usage */
-	if (total_blks_to_read > MAX_ARRAY_SIZE)
-	    mem_list_count = MAX_ARRAY_SIZE;
-	else mem_list_count = total_blks_to_read;
-	mem_offsets = (PVFS_size*)ADIOI_Malloc(mem_list_count*sizeof(PVFS_size));
-	mem_lengths = (int*)ADIOI_Malloc(mem_list_count*sizeof(int));
-
-	/* TODO: CHECK RESULTS OF MEMORY ALLOCATION */
-
-	j = 0;
-	/* step through each block in memory, filling memory arrays */
-	while (b_blks_read < total_blks_to_read) {
-	    for (i=0; i<flat_buf->count; i++) {
-		mem_offsets[b_blks_read % MAX_ARRAY_SIZE] = 
-		    /* TODO: fix this compiler warning */
-		    ((PVFS_size)buf + j*buftype_extent + flat_buf->indices[i]);
-		mem_lengths[b_blks_read % MAX_ARRAY_SIZE] = 
-		    flat_buf->blocklens[i];
-		file_length += flat_buf->blocklens[i];
-		b_blks_read++;
-		if (!(b_blks_read % MAX_ARRAY_SIZE) ||
-		    (b_blks_read == total_blks_to_read)) {
-
-		    /* in the case of the last read list call,
-		       adjust mem_list_count */
-		    if (b_blks_read == total_blks_to_read) {
-		        mem_list_count = total_blks_to_read % MAX_ARRAY_SIZE;
-			/* in case last read list call fills max arrays */
-			if (!mem_list_count) mem_list_count = MAX_ARRAY_SIZE;
-		    }
-		    err_flag = PVFS_Request_hindexed(mem_list_count, 
-			    mem_lengths, mem_offsets, PVFS_BYTE, &mem_req);
-		    if (err_flag < 0) break;
-		    err_flag = PVFS_Request_contiguous(file_length,
-			    PVFS_BYTE, &file_req);
-		    if (err_flag < 0) break;
-#ifdef ADIOI_MPE_LOGGING
-                    MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
-#endif
-		    err_flag = PVFS_sys_read(pvfs_fs->object_ref, file_req, 
-			    file_offset, PVFS_BOTTOM, mem_req,
-			    &(pvfs_fs->credentials), &resp_io);
-#ifdef ADIOI_MPE_LOGGING
-                    MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
-#endif
-		    /* --BEGIN ERROR HANDLING-- */
-		    if (err_flag != 0) {
-			*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-							   MPIR_ERR_RECOVERABLE,
-							   myname, __LINE__,
-							   ADIOI_PVFS2_error_convert(err_flag),
-							   "Error in PVFS_sys_read", 0);
-			goto error_state;
-		    }
-		    PVFS_Request_free(&mem_req);
-		    PVFS_Request_free(&file_req);
-		    total_bytes_read += resp_io.total_completed;
-		    /* --END ERROR HANDLING-- */
-		  
-		    /* in the case of error or the last read list call, 
-		     * leave here */
-		    if (err_flag || b_blks_read == total_blks_to_read) break;
-
-		    file_offset += file_length;
-		    file_length = 0;
-		} 
-	    } /* for (i=0; i<flat_buf->count; i++) */
-	    j++;
-	} /* while (b_blks_read < total_blks_to_read) */
-	ADIOI_Free(mem_offsets);
-	ADIOI_Free(mem_lengths);
-
-        if (file_ptr_type == ADIO_INDIVIDUAL) 
-	    fd->fp_ind += total_bytes_read;
-
-	fd->fp_sys_posn = -1;  /* set it to null. */
-
-#ifdef HAVE_STATUS_SET_BYTES
-	MPIR_Status_set_bytes(status, datatype, bufsize);
-	/* This isa temporary way of filling in status.  The right way is to
-	   keep tracke of how much data was actually read adn placed in buf
-	   by ADIOI_BUFFERED_READ. */
-#endif
-	ADIOI_Delete_flattened(datatype);
-
-	return;
-    } /* if (!buftype_is_contig && filetype_is_contig) */
-
-    /* know file is noncontiguous from above */
-    /* noncontiguous in file */
-
-    /* filetype already flattened in ADIO_Open */
-    flat_file = ADIOI_Flatlist;
-    while (flat_file->type != fd->filetype) flat_file = flat_file->next;
-
-    disp = fd->disp;
-    initial_off = offset;
-
-
-    /* for each case - ADIO_Individual pointer or explicit, find the file
-       offset in bytes (offset), n_filetypes (how many filetypes into
-       file to start), frd_size (remaining amount of data in present
-       file block), and st_index (start point in terms of blocks in
-       starting filetype) */
-    if (file_ptr_type == ADIO_INDIVIDUAL) {
-        offset = fd->fp_ind; /* in bytes */
-	n_filetypes = -1;
-	flag = 0;
-	while (!flag) {
-	    n_filetypes++;
-	    for (i=0; i<flat_file->count; i++) {
-	        if (disp + flat_file->indices[i] + 
-		    ((ADIO_Offset) n_filetypes)*filetype_extent +
-		    flat_file->blocklens[i]  >= offset) {
-		    st_index = i;
-		    frd_size = (int) (disp + flat_file->indices[i] + 
-				    ((ADIO_Offset) n_filetypes)*filetype_extent
-				      + flat_file->blocklens[i] - offset);
-		    flag = 1;
-		    break;
-		}
-	    }
-	} /* while (!flag) */
-    } /* if (file_ptr_type == ADIO_INDIVIDUAL) */
-    else {
-        n_etypes_in_filetype = filetype_size/etype_size;
-	n_filetypes = (int) (offset / n_etypes_in_filetype);
-	etype_in_filetype = (int) (offset % n_etypes_in_filetype);
-	size_in_filetype = etype_in_filetype * etype_size;
-	
-	sum = 0;
-	for (i=0; i<flat_file->count; i++) {
-	    sum += flat_file->blocklens[i];
-	    if (sum > size_in_filetype) {
-	        st_index = i;
-		frd_size = sum - size_in_filetype;
-		abs_off_in_filetype = flat_file->indices[i] +
-		    size_in_filetype - (sum - flat_file->blocklens[i]);
-		break;
-	    }
-	}
-	
-	/* abs. offset in bytes in the file */
-	offset = disp + ((ADIO_Offset) n_filetypes)*filetype_extent + 
-	    abs_off_in_filetype;
-    } /* else [file_ptr_type != ADIO_INDIVIDUAL] */
-
-    start_off = offset;
-    st_frd_size = frd_size;
-    st_n_filetypes = n_filetypes;
-    
-    if (buftype_is_contig && !filetype_is_contig) {
-
-/* contiguous in memory, noncontiguous in file. should be the most
-   common case. */
-
-       int mem_length=0;
-	intptr_t mem_offset;
-	
-	i = 0;
-	j = st_index;
-	n_filetypes = st_n_filetypes;
-	
-	mem_list_count = 1;
-	
-	/* determine how many blocks in file to read */
-	f_data_read = ADIOI_MIN(st_frd_size, bufsize);
-	total_blks_to_read = 1;
-	if (j < (flat_file->count-1)) j++;
-	else {
-	    j = 0;
-	    n_filetypes++;
-	}
-	while (f_data_read < bufsize) {
-	    f_data_read += flat_file->blocklens[j];
-	    total_blks_to_read++;
-	    if (j<(flat_file->count-1)) j++;
-	    else j = 0;	
-	}
-      
-	j = st_index;
-	n_filetypes = st_n_filetypes;
-	n_read_lists = total_blks_to_read/MAX_ARRAY_SIZE;
-	extra_blks = total_blks_to_read%MAX_ARRAY_SIZE;
-	
-	mem_offset = (intptr_t)buf;
-	mem_lengths = 0;
-	
-	/* if at least one full readlist, allocate file arrays
-	   at max array size and don't free until very end */
-	if (n_read_lists) {
-	    file_offsets = (int64_t*)ADIOI_Malloc(MAX_ARRAY_SIZE*
-						  sizeof(int64_t));
-	    file_lengths = (int32_t*)ADIOI_Malloc(MAX_ARRAY_SIZE*
-						  sizeof(int32_t));
-	}
-	/* if there's no full readlist allocate file arrays according
-	   to needed size (extra_blks) */
-	else {
-	    file_offsets = (int64_t*)ADIOI_Malloc(extra_blks*
-						  sizeof(int64_t));
-	    file_lengths = (int32_t*)ADIOI_Malloc(extra_blks*
-						  sizeof(int32_t));
-	}
-	
-	/* for file arrays that are of MAX_ARRAY_SIZE, build arrays */
-	for (i=0; i<n_read_lists; i++) {
-	    file_list_count = MAX_ARRAY_SIZE;
-	    if(!i) {
-	        file_offsets[0] = offset;
-		file_lengths[0] = st_frd_size;
-		mem_length = st_frd_size;
-	    }
-	    for (k=0; k<MAX_ARRAY_SIZE; k++) {
-	        if (i || k) {
-		    file_offsets[k] = disp + 
-			((ADIO_Offset)n_filetypes)*filetype_extent
-		      + flat_file->indices[j];
-		    file_lengths[k] = flat_file->blocklens[j];
-		    mem_lengths += file_lengths[k];
-		}
-		if (j<(flat_file->count - 1)) j++;
-		else {
-		    j = 0;
-		    n_filetypes++;
-		}
-	    } /* for (k=0; k<MAX_ARRAY_SIZE; k++) */
-	    err_flag = PVFS_Request_contiguous(mem_length,
-					       PVFS_BYTE, &mem_req);
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (err_flag != 0) {
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-						   MPIR_ERR_RECOVERABLE,
-						   myname, __LINE__,
-						   ADIOI_PVFS2_error_convert(err_flag),
-						   "Error in PVFS_Request_contiguous (memory)", 0);
-		goto error_state;
-	    }
-	    /* --END ERROR HANDLING-- */
-
-	    err_flag = PVFS_Request_hindexed(file_list_count, file_lengths, 
-					     file_offsets, PVFS_BYTE,
-					     &file_req);
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (err_flag != 0) {
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-						   MPIR_ERR_RECOVERABLE,
-						   myname, __LINE__,
-						   ADIOI_PVFS2_error_convert(err_flag),
-						   "Error in PVFS_Request_hindexed (file)", 0);
-		goto error_state;
-	    }
-	    /* --END ERROR HANDLING-- */
-
-	    /* PVFS_Request_hindexed already expresses the offsets into the
-	     * file, so we should not pass in an offset if we are using
-	     * hindexed for the file type */
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
-#endif
-	    err_flag = PVFS_sys_read(pvfs_fs->object_ref, file_req, 0, 
-				     (void *)mem_offset, mem_req,
-				     &(pvfs_fs->credentials), &resp_io);
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
-#endif
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (err_flag != 0) {
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-						   MPIR_ERR_RECOVERABLE,
-						   myname, __LINE__,
-						   ADIOI_PVFS2_error_convert(err_flag),
-						   "Error in PVFS_sys_read", 0);
-		goto error_state;
-	    }
-	    /* --END ERROR HANDING-- */
-	    PVFS_Request_free(&mem_req);
-	    PVFS_Request_free(&file_req);
-
-	    total_bytes_read += resp_io.total_completed;
-
-	    mem_offset += mem_length;
-	    mem_lengths = 0;
-	} /* for (i=0; i<n_read_lists; i++) */
-
-	/* for file arrays smaller than MAX_ARRAY_SIZE (last read_list call) */
-	if (extra_blks) {
-	    file_list_count = extra_blks;
-	    if(!i) {
-	        file_offsets[0] = offset;
-		file_lengths[0] = ADIOI_MIN(st_frd_size, bufsize);
-	    }
-	    for (k=0; k<extra_blks; k++) {
-	        if(i || k) {
-		    file_offsets[k] = disp + 
-			((ADIO_Offset)n_filetypes)*filetype_extent +
-			flat_file->indices[j];
-		    if (k == (extra_blks - 1)) {
-		        file_lengths[k] = bufsize - (int32_t) mem_lengths
-			  - mem_offset + (int32_t)  buf;
-		    }
-		    else file_lengths[k] = flat_file->blocklens[j];
-		} /* if(i || k) */
-		mem_lengths += file_lengths[k];
-		if (j<(flat_file->count - 1)) j++;
-		else {
-		    j = 0;
-		    n_filetypes++;
-		}
-	    } /* for (k=0; k<extra_blks; k++) */
-	    err_flag = PVFS_Request_contiguous(mem_length,
-					       PVFS_BYTE, &mem_req);
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (err_flag != 0) {
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-						   MPIR_ERR_RECOVERABLE,
-						   myname, __LINE__,
-						   ADIOI_PVFS2_error_convert(err_flag),
-						   "Error in PVFS_Request_contiguous (memory)", 0);
-		goto error_state;
-	    }
-	    /* --END ERROR HANDLING-- */
-
-	    err_flag = PVFS_Request_hindexed(file_list_count, file_lengths, 
-		    file_offsets, PVFS_BYTE, &file_req);
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (err_flag != 0) {
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-						   MPIR_ERR_RECOVERABLE,
-						   myname, __LINE__,
-						   ADIOI_PVFS2_error_convert(err_flag),
-						   "Error in PVFS_Request_hindexed (file)", 0);
-		goto error_state;
-	    }
-	    /* --END ERROR HANDLING-- */
-
-	    /* as above, use 0 for 'offset' when using hindexed file type */
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
-#endif
-	    err_flag = PVFS_sys_read(pvfs_fs->object_ref, file_req, 0, 
-		    (void *)mem_offset, mem_req, &(pvfs_fs->credentials), &resp_io);
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
-#endif
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (err_flag != 0) {
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-						   MPIR_ERR_RECOVERABLE,
-						   myname, __LINE__,
-						   ADIOI_PVFS2_error_convert(err_flag),
-						   "Error in PVFS_sys_read", 0);		
-		goto error_state;
-	    }
-	    /* --END ERROR HANDLING-- */
-	    PVFS_Request_free(&mem_req);
-	    PVFS_Request_free(&file_req);
-	    total_bytes_read += resp_io.total_completed;
-	}
-    }
-    else {
-/* noncontiguous in memory as well as in file */
-      
-	flat_buf = ADIOI_Flatten_and_find(datatype);
-
-	size_read = 0;
-	n_filetypes = st_n_filetypes;
-	frd_size = st_frd_size;
-	brd_size = flat_buf->blocklens[0];
-	buf_count = 0;
-	start_mem_offset = 0;
-	start_k = k = 0;
-	start_j = st_index;
-	max_mem_list = 0;
-	max_file_list = 0;
-
-	/* run through and file max_file_list and max_mem_list so that you 
-	   can allocate the file and memory arrays less than MAX_ARRAY_SIZE
-	   if possible */
-
-	while (size_read < bufsize) {
-	    k = start_k;
-	    new_buffer_read = 0;
-	    mem_list_count = 0;
-	    while ((mem_list_count < MAX_ARRAY_SIZE) && 
-		   (new_buffer_read < bufsize-size_read)) {
-	        /* find mem_list_count and file_list_count such that both are
-		   less than MAX_ARRAY_SIZE, the sum of their lengths are
-		   equal, and the sum of all the data read and data to be
-		   read in the next immediate read list is less than
-		   bufsize */
-	        if(mem_list_count) {
-		    if((new_buffer_read + flat_buf->blocklens[k] + 
-			size_read) > bufsize) {
-		        end_brd_size = new_buffer_read + 
-			    flat_buf->blocklens[k] - (bufsize - size_read);
-			new_buffer_read = bufsize - size_read;
-		    }
-		    else {
-		        new_buffer_read += flat_buf->blocklens[k];
-			end_brd_size = flat_buf->blocklens[k];
-		    }
-		}
-		else {
-		    if (brd_size > (bufsize - size_read)) {
-		        new_buffer_read = bufsize - size_read;
-			brd_size = new_buffer_read;
-		    }
-		    else new_buffer_read = brd_size;
-		}
-		mem_list_count++;
-		k = (k + 1)%flat_buf->count;
-	     } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
-	       (new_buffer_read < bufsize-size_read)) */
-	    j = start_j;
-	    new_file_read = 0;
-	    file_list_count = 0;
-	    while ((file_list_count < MAX_ARRAY_SIZE) && 
-		   (new_file_read < new_buffer_read)) {
-	        if(file_list_count) {
-		    if((new_file_read + flat_file->blocklens[j]) > 
-		       new_buffer_read) {
-		        end_frd_size = new_buffer_read - new_file_read;
-			new_file_read = new_buffer_read;
-			j--;
-		    }
-		    else {
-		        new_file_read += flat_file->blocklens[j];
-			end_frd_size = flat_file->blocklens[j];
-		    }
-		}
-		else {
-		    if (frd_size > new_buffer_read) {
-		        new_file_read = new_buffer_read;
-			frd_size = new_file_read;
-		    }
-		    else new_file_read = frd_size;
-		}
-		file_list_count++;
-		if (j < (flat_file->count - 1)) j++;
-		else j = 0;
-		
-		k = start_k;
-		if ((new_file_read < new_buffer_read) && 
-		    (file_list_count == MAX_ARRAY_SIZE)) {
-		    new_buffer_read = 0;
-		    mem_list_count = 0;
-		    while (new_buffer_read < new_file_read) {
-		        if(mem_list_count) {
-			    if((new_buffer_read + flat_buf->blocklens[k]) >
-			       new_file_read) {
-			        end_brd_size = new_file_read - new_buffer_read;
-				new_buffer_read = new_file_read;
-				k--;
-			    }
-			    else {
-			        new_buffer_read += flat_buf->blocklens[k];
-				end_brd_size = flat_buf->blocklens[k];
-			    }
-			}
-			else {
-			    new_buffer_read = brd_size;
-			    if (brd_size > (bufsize - size_read)) {
-			        new_buffer_read = bufsize - size_read;
-				brd_size = new_buffer_read;
-			    }
-			}
-			mem_list_count++;
-			k = (k + 1)%flat_buf->count;
-		    } /* while (new_buffer_read < new_file_read) */
-		} /* if ((new_file_read < new_buffer_read) && (file_list_count
-		     == MAX_ARRAY_SIZE)) */
-	    } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
-		 (new_buffer_read < bufsize-size_read)) */
-
-	    /*  fakes filling the readlist arrays of lengths found above  */
-	    k = start_k;
-	    j = start_j;
-	    for (i=0; i<mem_list_count; i++) {	     
-		if(i) {
-		    if (i == (mem_list_count - 1)) {
-			if (flat_buf->blocklens[k] == end_brd_size)
-			    brd_size = flat_buf->blocklens[(k+1)%
-							  flat_buf->count];
-			else {
-			    brd_size = flat_buf->blocklens[k] - end_brd_size;
-			    k--;
-			    buf_count--;
-			}
-		    }
-		}
-		buf_count++;
-		k = (k + 1)%flat_buf->count;
-	    } /* for (i=0; i<mem_list_count; i++) */
-	    for (i=0; i<file_list_count; i++) {
-		if (i) {
-		    if (i == (file_list_count - 1)) {
-			if (flat_file->blocklens[j] == end_frd_size)
-			    frd_size = flat_file->blocklens[(j+1)%
-							  flat_file->count];   
-			else {
-			    frd_size = flat_file->blocklens[j] - end_frd_size;
-			    j--;
-			}
-		    }
-		}
-		if (j < flat_file->count - 1) j++;
-		else {
-		    j = 0;
-		    n_filetypes++;
-		}
-	    } /* for (i=0; i<file_list_count; i++) */
-	    size_read += new_buffer_read;
-	    start_k = k;
-	    start_j = j;
-	    if (max_mem_list < mem_list_count)
-	        max_mem_list = mem_list_count;
-	    if (max_file_list < file_list_count)
-	        max_file_list = file_list_count;
-	} /* while (size_read < bufsize) */
-
-	/* one last check before we actually carry out the operation:
-	 * this code has hard-to-fix bugs when a noncontiguous file type has
-	 * such large pieces that the sum of the lengths of the memory type is
-	 * not larger than one of those pieces (and vice versa for large memory
-	 * types and many pices of file types.  In these cases, give up and
-	 * fall back to naive reads and writes.  The testphdf5 test created a
-	 * type with two very large memory regions and 600 very small file
-	 * regions.  The same test also created a type with one very large file
-	 * region and many (700) very small memory regions.  both cases caused
-	 * problems for this code */
-
-	if ( ( (file_list_count == 1) && 
-		    (new_file_read < flat_file->blocklens[0] ) ) ||
-		((mem_list_count == 1) && 
-		    (new_buffer_read < flat_buf->blocklens[0]) ) ||
-		((file_list_count == MAX_ARRAY_SIZE) && 
-		    (new_file_read < flat_buf->blocklens[0]) ) ||
-		( (mem_list_count == MAX_ARRAY_SIZE) &&
-		    (new_buffer_read < flat_file->blocklens[0])) )
-	{
-
-	    ADIOI_Delete_flattened(datatype);
-	    ADIOI_GEN_ReadStrided_naive(fd, buf, count, datatype,
-		    file_ptr_type, initial_off, status, error_code);
-	    return;
-	}
-
-	mem_offsets = (PVFS_size*)ADIOI_Malloc(max_mem_list*sizeof(PVFS_size));
-	mem_lengths = (int *)ADIOI_Malloc(max_mem_list*sizeof(int));
-	file_offsets = (int64_t *)ADIOI_Malloc(max_file_list*sizeof(int64_t));
-	file_lengths = (int32_t *)ADIOI_Malloc(max_file_list*sizeof(int32_t));
-	    
-	size_read = 0;
-	n_filetypes = st_n_filetypes;
-	frd_size = st_frd_size;
-	brd_size = flat_buf->blocklens[0];
-	buf_count = 0;
-	start_mem_offset = 0;
-	start_k = k = 0;
-	start_j = st_index;
-
-	/*  this section calculates mem_list_count and file_list_count
-	    and also finds the possibly odd sized last array elements
-	    in new_frd_size and new_brd_size  */
-	
-	while (size_read < bufsize) {
-	    k = start_k;
-	    new_buffer_read = 0;
-	    mem_list_count = 0;
-	    while ((mem_list_count < MAX_ARRAY_SIZE) && 
-		   (new_buffer_read < bufsize-size_read)) {
-	        /* find mem_list_count and file_list_count such that both are
-		   less than MAX_ARRAY_SIZE, the sum of their lengths are
-		   equal, and the sum of all the data read and data to be
-		   read in the next immediate read list is less than
-		   bufsize */
-	        if(mem_list_count) {
-		    if((new_buffer_read + flat_buf->blocklens[k] + 
-			size_read) > bufsize) {
-		        end_brd_size = new_buffer_read + 
-			    flat_buf->blocklens[k] - (bufsize - size_read);
-			new_buffer_read = bufsize - size_read;
-		    }
-		    else {
-		        new_buffer_read += flat_buf->blocklens[k];
-			end_brd_size = flat_buf->blocklens[k];
-		    }
-		}
-		else {
-		    if (brd_size > (bufsize - size_read)) {
-		        new_buffer_read = bufsize - size_read;
-			brd_size = new_buffer_read;
-		    }
-		    else new_buffer_read = brd_size;
-		}
-		mem_list_count++;
-		k = (k + 1)%flat_buf->count;
-	     } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
-	       (new_buffer_read < bufsize-size_read)) */
-	    j = start_j;
-	    new_file_read = 0;
-	    file_list_count = 0;
-	    while ((file_list_count < MAX_ARRAY_SIZE) && 
-		   (new_file_read < new_buffer_read)) {
-	        if(file_list_count) {
-		    if((new_file_read + flat_file->blocklens[j]) > 
-		       new_buffer_read) {
-		        end_frd_size = new_buffer_read - new_file_read;
-			new_file_read = new_buffer_read;
-			j--;
-		    }
-		    else {
-		        new_file_read += flat_file->blocklens[j];
-			end_frd_size = flat_file->blocklens[j];
-		    }
-		}
-		else {
-		    if (frd_size > new_buffer_read) {
-		        new_file_read = new_buffer_read;
-			frd_size = new_file_read;
-		    }
-		    else new_file_read = frd_size;
-		}
-		file_list_count++;
-		if (j < (flat_file->count - 1)) j++;
-		else j = 0;
-		
-		k = start_k;
-		if ((new_file_read < new_buffer_read) && 
-		    (file_list_count == MAX_ARRAY_SIZE)) {
-		    new_buffer_read = 0;
-		    mem_list_count = 0;
-		    while (new_buffer_read < new_file_read) {
-		        if(mem_list_count) {
-			    if((new_buffer_read + flat_buf->blocklens[k]) >
-			       new_file_read) {
-			        end_brd_size = new_file_read - new_buffer_read;
-				new_buffer_read = new_file_read;
-				k--;
-			    }
-			    else {
-			        new_buffer_read += flat_buf->blocklens[k];
-				end_brd_size = flat_buf->blocklens[k];
-			    }
-			}
-			else {
-			    new_buffer_read = brd_size;
-			    if (brd_size > (bufsize - size_read)) {
-			        new_buffer_read = bufsize - size_read;
-				brd_size = new_buffer_read;
-			    }
-			}
-			mem_list_count++;
-			k = (k + 1)%flat_buf->count;
-		    } /* while (new_buffer_read < new_file_read) */
-		} /* if ((new_file_read < new_buffer_read) && (file_list_count
-		     == MAX_ARRAY_SIZE)) */
-	    } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
-		 (new_buffer_read < bufsize-size_read)) */
-
-	    /*  fills the allocated readlist arrays  */
-	    k = start_k;
-	    j = start_j;
-	    for (i=0; i<mem_list_count; i++) {	     
-	        mem_offsets[i] = ((PVFS_size)buf + buftype_extent*
-					 (buf_count/flat_buf->count) +
-					 (int)flat_buf->indices[k]);
-		if(!i) {
-		    mem_lengths[0] = brd_size;
-		    mem_offsets[0] += flat_buf->blocklens[k] - brd_size;
-		}
-		else {
-		    if (i == (mem_list_count - 1)) {
-		        mem_lengths[i] = end_brd_size;
-			if (flat_buf->blocklens[k] == end_brd_size)
-			    brd_size = flat_buf->blocklens[(k+1)%
-							  flat_buf->count];
-			else {
-			    brd_size = flat_buf->blocklens[k] - end_brd_size;
-			    k--;
-			    buf_count--;
-			}
-		    }
-		    else {
-		        mem_lengths[i] = flat_buf->blocklens[k];
-		    }
-		}
-		buf_count++;
-		k = (k + 1)%flat_buf->count;
-	    } /* for (i=0; i<mem_list_count; i++) */
-	    for (i=0; i<file_list_count; i++) {
-	        file_offsets[i] = disp + flat_file->indices[j] + 
-		    ((ADIO_Offset)n_filetypes) * filetype_extent;
-	        if (!i) {
-		    file_lengths[0] = frd_size;
-		    file_offsets[0] += flat_file->blocklens[j] - frd_size;
-		}
-		else {
-		    if (i == (file_list_count - 1)) {
-		        file_lengths[i] = end_frd_size;
-			if (flat_file->blocklens[j] == end_frd_size)
-			    frd_size = flat_file->blocklens[(j+1)%
-							  flat_file->count];   
-			else {
-			    frd_size = flat_file->blocklens[j] - end_frd_size;
-			    j--;
-			}
-		    }
-		    else file_lengths[i] = flat_file->blocklens[j];
-		}
-		if (j < flat_file->count - 1) j++;
-		else {
-		    j = 0;
-		    n_filetypes++;
-		}
-	    } /* for (i=0; i<file_list_count; i++) */
-	    err_flag = PVFS_Request_hindexed(mem_list_count, mem_lengths, 
-		    mem_offsets, PVFS_BYTE, &mem_req);
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (err_flag != 0 ) {
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-						   MPIR_ERR_RECOVERABLE,
-						   myname, __LINE__,
-						   ADIOI_PVFS2_error_convert(err_flag),
-						   "Error in PVFS_Request_hindexed (memory)", 0);
-		goto error_state;
-	    }
-	    /* -- END ERROR HANDLING-- */
-	    err_flag = PVFS_Request_hindexed(file_list_count, file_lengths, 
-		    file_offsets, PVFS_BYTE, &file_req);
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (err_flag != 0) {
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-						   MPIR_ERR_RECOVERABLE,
-						   myname, __LINE__,
-						   ADIOI_PVFS2_error_convert(err_flag),
-						   "Error in PVFS_Request_hindexed (file)", 0);
-		goto error_state;
-	    }
-	    /* --END ERROR HANDLING-- */
-
-	    /* offset will be expressed in memory and file datatypes */
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
-#endif
-	    err_flag = PVFS_sys_read(pvfs_fs->object_ref, file_req, 0, 
-		    PVFS_BOTTOM, mem_req, &(pvfs_fs->credentials), &resp_io);
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
-#endif
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (err_flag != 0) {
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-						   MPIR_ERR_RECOVERABLE,
-						   myname, __LINE__,
-						   ADIOI_PVFS2_error_convert(err_flag),
-						   "Error in PVFS_sys_read", 0);
-	    }
-	    /* --END ERROR HANDLING-- */
-	    PVFS_Request_free(&mem_req);
-	    PVFS_Request_free(&file_req);
-	    total_bytes_read += resp_io.total_completed;
-	    size_read += new_buffer_read;
-	    start_k = k;
-	    start_j = j;
-	} /* while (size_read < bufsize) */
-	ADIOI_Free(mem_offsets);
-	ADIOI_Free(mem_lengths);
-    }
-    /* Other ADIO routines will convert absolute bytes into counts of datatypes */
-    /* when incrementing fp_ind, need to also take into account the file type:
-     * consider an N-element 1-d subarray with a lb and ub: ( |---xxxxx-----|
-     * if we wrote N elements, offset needs to point at beginning of type, not
-     * at empty region at offset N+1) 
-     *
-     * As we discussed on mpich-discuss in may/june 2009, the code below might
-     * look wierd, but by putting fp_ind at the last byte written, the next
-     * time we run through the strided code we'll update the fp_ind to the
-     * right location. */
-    if (file_ptr_type == ADIO_INDIVIDUAL) {
-	fd->fp_ind = file_offsets[file_list_count-1]+
-	    file_lengths[file_list_count-1];
-    }
-    
-    ADIOI_Free(file_offsets);
-    ADIOI_Free(file_lengths);
-    
-    if (err_flag == 0) *error_code = MPI_SUCCESS;
-
-error_state:
-    fd->fp_sys_posn = -1;   /* set it to null. */
-
-#ifdef HAVE_STATUS_SET_BYTES
-    MPIR_Status_set_bytes(status, datatype, bufsize);
-    /* This is a temporary way of filling in status. The right way is to 
-       keep track of how much data was actually read and placed in buf 
-       by ADIOI_BUFFERED_READ. */
-#endif
-    
-    if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
-}
-
diff --git a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_resize.c b/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_resize.c
deleted file mode 100644
index db219db0fde..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_resize.c
+++ /dev/null
@@ -1,56 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_pvfs2.h"
-#include "ad_pvfs2_common.h"
-
-/* as with ADIOI_PVFS2_Flush, implement the resize operation in a scalable
- * manner. one process does the work, then broadcasts the result to everyone
- * else.  fortunately, this operation is defined to be collective */
-void ADIOI_PVFS2_Resize(ADIO_File fd, ADIO_Offset size, int *error_code)
-{
-    int ret, rank;
-    ADIOI_PVFS2_fs *pvfs_fs;
-    static char myname[] = "ADIOI_PVFS2_RESIZE";
-
-    *error_code = MPI_SUCCESS;
-
-    pvfs_fs = (ADIOI_PVFS2_fs*)fd->fs_ptr;
-
-    MPI_Comm_rank(fd->comm, &rank);
-
-    /* We desginate one node in the communicator to be an 'io_worker' in 
-     * ADIO_Open.  This node can perform operations on files and then 
-     * inform the other nodes of the result */
-
-    /* MPI-IO semantics treat conflicting MPI_File_set_size requests the
-     * same as conflicting write requests. Thus, a resize from one
-     * process does not have to be visible to the other processes until a
-     * syncronization point is reached */
-
-    if (rank == fd->hints->ranklist[0]) {
-	ret = PVFS_sys_truncate(pvfs_fs->object_ref, 
-		size, &(pvfs_fs->credentials));
-	MPI_Bcast(&ret, 1, MPI_INT, fd->hints->ranklist[0], fd->comm);
-    } else  {
-	MPI_Bcast(&ret, 1, MPI_INT, fd->hints->ranklist[0], fd->comm);
-    }
-    /* --BEGIN ERROR HANDLING-- */
-    if (ret != 0) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   ADIOI_PVFS2_error_convert(ret),
-					   "Error in PVFS_sys_truncate", 0);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-}
-
-/*
- * vim: ts=8 sts=4 sw=4 noexpandtab 
- */
diff --git a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_write.c b/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_write.c
deleted file mode 100644
index 93e142bb687..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_write.c
+++ /dev/null
@@ -1,182 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- 
- *   vim: ts=8 sts=4 sw=4 noexpandtab
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_pvfs2.h"
-#include "adio_extern.h"
-#include "ad_pvfs2_io.h"
-#include "ad_pvfs2_common.h"
-
-void ADIOI_PVFS2_WriteContig(ADIO_File fd, const void *buf, int count,
-			     MPI_Datatype datatype, int file_ptr_type,
-			     ADIO_Offset offset, ADIO_Status *status,
-			     int *error_code)
-{
-    int ret;
-    MPI_Count datatype_size, len;
-    PVFS_Request file_req, mem_req;
-    PVFS_sysresp_io resp_io;
-    ADIOI_PVFS2_fs *pvfs_fs;
-    static char myname[] = "ADIOI_PVFS2_WRITECONTIG";
-
-    pvfs_fs = (ADIOI_PVFS2_fs*)fd->fs_ptr;
-
-    MPI_Type_size_x(datatype, &datatype_size);
-    len = datatype_size * count;
-
-    ret = PVFS_Request_contiguous(len, PVFS_BYTE, &mem_req);
-    /* --BEGIN ERROR HANDLING-- */
-    if (ret != 0) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   ADIOI_PVFS2_error_convert(ret),
-					   "Error in PVFS_Request_contiguous (memory)", 0);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-    ret = PVFS_Request_contiguous(len, PVFS_BYTE, &file_req);
-    /* --BEGIN ERROR HANDLING-- */
-    if (ret != 0) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   ADIOI_PVFS2_error_convert(ret),
-					   "Error in PVFS_Request_contiguous (file)", 0);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-    if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
-#endif
-	ret = PVFS_sys_write(pvfs_fs->object_ref, file_req, offset,  (void *)buf,
-			     mem_req, &(pvfs_fs->credentials), &resp_io);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
-#endif
-	/* --BEGIN ERROR HANDLING-- */
-	if (ret != 0) {
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE,
-					       myname, __LINE__,
-					       ADIOI_PVFS2_error_convert(ret),
-					       "Error in PVFS_sys_write", 0);
-	    goto fn_exit;
-	}
-	/* --END ERROR HANDLING-- */
-
-	fd->fp_sys_posn = offset + (int) resp_io.total_completed;
-    }
-    else {
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
-#endif
-	ret = PVFS_sys_write(pvfs_fs->object_ref, file_req, fd->fp_ind, (void *)buf,
-			     mem_req, &(pvfs_fs->credentials), &resp_io);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
-#endif
-	/* --BEGIN ERROR HANDLING-- */
-	if (ret != 0) {
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE,
-					       myname, __LINE__,
-					       ADIOI_PVFS2_error_convert(ret),
-					       "Error in PVFS_sys_write", 0);
-	    goto fn_exit;
-	}
-	/* --END ERROR HANDLING-- */
-	fd->fp_ind += (int)resp_io.total_completed;
-	fd->fp_sys_posn = fd->fp_ind;
-    }
-#ifdef HAVE_STATUS_SET_BYTES
-    MPIR_Status_set_bytes(status, datatype, resp_io.total_completed);
-#endif
-    *error_code = MPI_SUCCESS;
-fn_exit:
-    PVFS_Request_free(&file_req);
-    PVFS_Request_free(&mem_req);
-    return;
-}
-
-int ADIOI_PVFS2_WriteStridedListIO(ADIO_File fd, const void *buf, int count,
-				   MPI_Datatype datatype, int file_ptr_type,
-				   ADIO_Offset offset, ADIO_Status *status,
-				   int *error_code)
-{
-    return ADIOI_PVFS2_StridedListIO(fd, (void *)buf, count,
-				     datatype, file_ptr_type,
-				     offset, status,
-				     error_code, WRITE);
-}
-
-int ADIOI_PVFS2_WriteStridedDtypeIO(ADIO_File fd, const void *buf, int count,
-				    MPI_Datatype datatype, int file_ptr_type,
-				    ADIO_Offset offset, ADIO_Status *status, 
-				    int *error_code)
-{
-    return ADIOI_PVFS2_StridedDtypeIO(fd, (void *)buf, count,
-				      datatype, file_ptr_type,
-				      offset, status, error_code,
-				      WRITE);
-}
-
-
-void ADIOI_PVFS2_WriteStrided(ADIO_File fd, const void *buf, int count,
-			      MPI_Datatype datatype, int file_ptr_type,
-			      ADIO_Offset offset, ADIO_Status *status,
-			      int *error_code)
-{
-    /* four ways (to date) that we can carry out strided i/o accesses:
-     * - naive posix
-     * - 'true' Datatype (from avery)
-     * - new List I/O (from avery)
-     * - classic List I/O  (the one that's always been in ROMIO)
-     * I imagine we'll keep Datatype as an optional optimization, and afer a
-     * release or two promote it to the default 
-     */
-
-    /* a lot of near-duplication from ADIOI_PVFS2_ReadStrided: for
-     * debugging/testing it's helpful to be able to turn on and off these
-     * optimizations separately for the read and write cases */
-    int ret = -1;
-    if ( fd->hints->fs_hints.pvfs2.posix_write == ADIOI_HINT_ENABLE) {
-        ADIOI_GEN_WriteStrided_naive(fd, buf, count,
-                                     datatype, file_ptr_type,
-                                     offset, status, error_code);
-        return;
-    }
-    if ( fd->hints->fs_hints.pvfs2.dtype_write == ADIOI_HINT_ENABLE) {
-        ret = ADIOI_PVFS2_WriteStridedDtypeIO(fd, buf, count,
-                                              datatype, file_ptr_type,
-                                              offset, status, error_code);
-
-        /* Fall back to list I/O if datatype I/O didn't work */
-        if (ret != 0)
-        {
-            fprintf(stderr,
-                    "Falling back to list I/O since datatype I/O failed\n");
-            ret = ADIOI_PVFS2_WriteStridedListIO(fd, buf, count,
-                                                 datatype, file_ptr_type,
-                                                 offset, status, error_code);
-        }
-        return;
-    }
-    /* Use list I/O in the base case */
-    if (fd->hints->fs_hints.pvfs2.listio_write == ADIOI_HINT_ENABLE) {
-	ret = ADIOI_PVFS2_WriteStridedListIO(fd, buf, count, datatype, 
-			file_ptr_type, offset, status, error_code);
-	return;
-    }
-
-    /* Use classic list I/O if no hints given base case */
-    ADIOI_PVFS2_OldWriteStrided(fd, buf, count, datatype,
-	    file_ptr_type, offset, status, error_code);
-    return;
-}
diff --git a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_write_list_classic.c b/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_write_list_classic.c
deleted file mode 100644
index 7a9758c30e3..00000000000
--- a/3rd-party/romio321/adio/ad_pvfs2/ad_pvfs2_write_list_classic.c
+++ /dev/null
@@ -1,963 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- 
- * vim: ts=8 sts=4 sw=4 noexpandtab 
- * 
- *   Copyright (C) 2008 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-#include "ad_pvfs2.h"
-
-#include "ad_pvfs2_common.h"
-
-void ADIOI_PVFS2_OldWriteStrided(ADIO_File fd, const void *buf, int count,
-			MPI_Datatype datatype, int file_ptr_type,
-			ADIO_Offset offset, ADIO_Status *status,
-			int *error_code)
-{
-    /* as with all the other WriteStrided functions, offset is in units of
-     * etype relative to the filetype */
-
-    /* Since PVFS2 does not support file locking, can't do buffered writes
-       as on Unix */
-
-    ADIOI_Flatlist_node *flat_buf, *flat_file;
-    int i, j, k, bwr_size, fwr_size=0, st_index=0;
-    int sum, n_etypes_in_filetype, size_in_filetype;
-    MPI_Count bufsize;
-    int n_filetypes, etype_in_filetype;
-    ADIO_Offset abs_off_in_filetype=0;
-    MPI_Count filetype_size, etype_size, buftype_size;
-    MPI_Aint filetype_extent, buftype_extent, filetype_lb, buftype_lb;
-    int buf_count, buftype_is_contig, filetype_is_contig;
-    ADIO_Offset off, disp, start_off, initial_off;
-    int flag, st_fwr_size, st_n_filetypes;
-    int err_flag=0;
-
-    int mem_list_count, file_list_count;
-    PVFS_size * mem_offsets;
-    int64_t *file_offsets;
-    int *mem_lengths;
-    int32_t *file_lengths;
-    int total_blks_to_write;
-
-    int max_mem_list, max_file_list;
-
-    int b_blks_wrote;
-    int f_data_wrote;
-    int size_wrote=0, n_write_lists, extra_blks;
-
-    int end_bwr_size, end_fwr_size;
-    int start_k, start_j, new_file_write, new_buffer_write;
-    int start_mem_offset;
-    PVFS_Request mem_req, file_req;
-    ADIOI_PVFS2_fs * pvfs_fs;
-    PVFS_sysresp_io resp_io;
-    MPI_Offset total_bytes_written=0;
-    static char myname[] = "ADIOI_PVFS2_WRITESTRIDED";
-
-    /* note: don't increase this: several parts of PVFS2 now 
-     * assume this limit*/
-#define MAX_ARRAY_SIZE 64
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (fd->atomicity) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   MPI_ERR_ARG,
-					   "Atomic noncontiguous writes are not supported by PVFS2", 0);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-
-    /* the HDF5 tests showed a bug in this list processing code (see many many
-     * lines down below).  We added a workaround, but common HDF5 file types
-     * are actually contiguous and do not need the expensive workarond */
-    if (!filetype_is_contig) {
-	flat_file = ADIOI_Flatlist;
-	while (flat_file->type != fd->filetype) flat_file = flat_file->next;
-	if (flat_file->count == 1 && !buftype_is_contig)
-	    filetype_is_contig = 1;
-    }
-
-    MPI_Type_size_x(fd->filetype, &filetype_size);
-    if ( ! filetype_size ) {
-#ifdef HAVE_STATUS_SET_BYTES
-	MPIR_Status_set_bytes(status, datatype, 0);
-#endif
-	*error_code = MPI_SUCCESS; 
-	return;
-    }
-
-    MPI_Type_get_extent(fd->filetype, &filetype_lb, &filetype_extent);
-    MPI_Type_size_x(datatype, &buftype_size);
-    MPI_Type_get_extent(datatype, &buftype_lb, &buftype_extent);
-    etype_size = fd->etype_size;
-    
-    bufsize = buftype_size * count;
-
-    pvfs_fs = (ADIOI_PVFS2_fs*)fd->fs_ptr;
-
-    if (!buftype_is_contig && filetype_is_contig) {
-
-/* noncontiguous in memory, contiguous in file.  */
-       int64_t file_offset;
-	int32_t file_length;
-
-	flat_buf = ADIOI_Flatten_and_find(datatype);
-	
-	if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-	    off = fd->disp + etype_size * offset;
-	}
-	else off = fd->fp_ind;
-
-	file_list_count = 1;
-	file_offset = off;
-	file_length = 0;
-	total_blks_to_write = count*flat_buf->count;
-	b_blks_wrote = 0;
-
-	/* allocate arrays according to max usage */
-	if (total_blks_to_write > MAX_ARRAY_SIZE)
-	    mem_list_count = MAX_ARRAY_SIZE;
-	else mem_list_count = total_blks_to_write;
-	mem_offsets = (PVFS_size*)ADIOI_Malloc(mem_list_count*sizeof(PVFS_size));
-	mem_lengths = (int*)ADIOI_Malloc(mem_list_count*sizeof(int));
-
-	j = 0;
-	/* step through each block in memory, filling memory arrays */
-	while (b_blks_wrote < total_blks_to_write) {
-	    for (i=0; i<flat_buf->count; i++) {
-		mem_offsets[b_blks_wrote % MAX_ARRAY_SIZE] = 
-		    /* TODO: fix this warning by casting to an integer that's
-		     * the same size as a char * and /then/ casting to
-		     * PVFS_size */
-		    ((PVFS_size)buf + j*buftype_extent + flat_buf->indices[i]);
-		mem_lengths[b_blks_wrote % MAX_ARRAY_SIZE] = 
-		    flat_buf->blocklens[i];
-		file_length += flat_buf->blocklens[i];
-		b_blks_wrote++;
-		if (!(b_blks_wrote % MAX_ARRAY_SIZE) ||
-		    (b_blks_wrote == total_blks_to_write)) {
-
-		    /* in the case of the last write list call,
-		       adjust mem_list_count */
-		    if (b_blks_wrote == total_blks_to_write) {
-		        mem_list_count = total_blks_to_write % MAX_ARRAY_SIZE;
-			/* in case last write list call fills max arrays */
-			if (!mem_list_count) mem_list_count = MAX_ARRAY_SIZE;
-		    }
-		    err_flag = PVFS_Request_hindexed(mem_list_count, 
-						     mem_lengths, mem_offsets,
-						     PVFS_BYTE, &mem_req);
-		    /* --BEGIN ERROR HANDLING-- */
-		    if (err_flag != 0) {
-			*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-							   MPIR_ERR_RECOVERABLE,
-							   myname, __LINE__,
-							   ADIOI_PVFS2_error_convert(err_flag),
-							   "Error in PVFS_Request_hindexed (memory)", 0);
-			break;
-		    }
-		    /* --END ERROR HANDLING-- */
-
-		    err_flag = PVFS_Request_contiguous(file_length,
-						       PVFS_BYTE, &file_req);
-		    /* --BEGIN ERROR HANDLING-- */
-		    if (err_flag != 0) {
-			*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-							   MPIR_ERR_RECOVERABLE,
-							   myname, __LINE__,
-							   ADIOI_PVFS2_error_convert(err_flag),
-							   "Error in PVFS_Request_contiguous (file)", 0);
-			break;
-		    }
-		    /* --END ERROR HANDLING-- */
-
-#ifdef ADIOI_MPE_LOGGING
-                    MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
-#endif
-		    err_flag = PVFS_sys_write(pvfs_fs->object_ref, file_req, 
-					      file_offset, PVFS_BOTTOM,
-					      mem_req, 
-					      &(pvfs_fs->credentials),
-					      &resp_io);
-#ifdef ADIOI_MPE_LOGGING
-                    MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
-#endif
-		    total_bytes_written += resp_io.total_completed;
-		  
-		    /* in the case of error or the last write list call, 
-		     * leave here */
-		    /* --BEGIN ERROR HANDLING-- */
-		    if (err_flag) {
-			*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-							   MPIR_ERR_RECOVERABLE,
-							   myname, __LINE__,
-							   ADIOI_PVFS2_error_convert(err_flag),
-							   "Error in PVFS_sys_write", 0);
-			break;
-		    }
-		    /* --END ERROR HANDLING-- */
-		    if (b_blks_wrote == total_blks_to_write) break;
-
-		    file_offset += file_length;
-		    file_length = 0;
-		    PVFS_Request_free(&mem_req);
-		    PVFS_Request_free(&file_req);
-		} 
-	    } /* for (i=0; i<flat_buf->count; i++) */
-	    j++;
-	} /* while (b_blks_wrote < total_blks_to_write) */
-	ADIOI_Free(mem_offsets);
-	ADIOI_Free(mem_lengths);
-
-	if (file_ptr_type == ADIO_INDIVIDUAL) 
-	    fd->fp_ind += total_bytes_written;
-
-	if (!err_flag)  *error_code = MPI_SUCCESS;
-
-	fd->fp_sys_posn = -1;   /* clear this. */
-
-#ifdef HAVE_STATUS_SET_BYTES
-	MPIR_Status_set_bytes(status, datatype, bufsize);
-/* This is a temporary way of filling in status. The right way is to 
-   keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
-#endif
-
-	ADIOI_Delete_flattened(datatype);
-	return;
-    } /* if (!buftype_is_contig && filetype_is_contig) */
-
-    /* already know that file is noncontiguous from above */
-    /* noncontiguous in file */
-
-/* filetype already flattened in ADIO_Open */
-    flat_file = ADIOI_Flatlist;
-    while (flat_file->type != fd->filetype) flat_file = flat_file->next;
-
-    disp = fd->disp;
-    initial_off = offset;
-
-    /* for each case - ADIO_Individual pointer or explicit, find offset
-       (file offset in bytes), n_filetypes (how many filetypes into file 
-       to start), fwr_size (remaining amount of data in present file
-       block), and st_index (start point in terms of blocks in starting
-       filetype) */
-    if (file_ptr_type == ADIO_INDIVIDUAL) {
-        offset = fd->fp_ind; /* in bytes */
-	n_filetypes = -1;
-	flag = 0;
-	while (!flag) {
-	    n_filetypes++;
-	    for (i=0; i<flat_file->count; i++) {
-	        if (disp + flat_file->indices[i] + 
-		    ((ADIO_Offset) n_filetypes)*filetype_extent +
-		      flat_file->blocklens[i] >= offset) {
-		  st_index = i;
-		  fwr_size = disp + flat_file->indices[i] + 
-		    ((ADIO_Offset) n_filetypes)*filetype_extent
-		    + flat_file->blocklens[i] - offset;
-		  flag = 1;
-		  break;
-		}
-	    }
-	} /* while (!flag) */
-    } /* if (file_ptr_type == ADIO_INDIVIDUAL) */
-    else {
-        n_etypes_in_filetype = filetype_size/etype_size;
-	n_filetypes = (int) (offset / n_etypes_in_filetype);
-	etype_in_filetype = (int) (offset % n_etypes_in_filetype);
-	size_in_filetype = etype_in_filetype * etype_size;
-	
-	sum = 0;
-	for (i=0; i<flat_file->count; i++) {
-	    sum += flat_file->blocklens[i];
-	    if (sum > size_in_filetype) {
-	        st_index = i;
-		fwr_size = sum - size_in_filetype;
-		abs_off_in_filetype = flat_file->indices[i] +
-		    size_in_filetype - (sum - flat_file->blocklens[i]);
-		break;
-	    }
-	}
-
-	/* abs. offset in bytes in the file */
-	offset = disp + ((ADIO_Offset) n_filetypes)*filetype_extent +
-	    abs_off_in_filetype;
-    } /* else [file_ptr_type != ADIO_INDIVIDUAL] */
-
-    start_off = offset;
-    st_fwr_size = fwr_size;
-    st_n_filetypes = n_filetypes;
-    
-    if (buftype_is_contig && !filetype_is_contig) {
-
-/* contiguous in memory, noncontiguous in file. should be the most
-   common case. */
-
-       int mem_length;
-	intptr_t mem_offset;
-        
-	i = 0;
-	j = st_index;
-	off = offset;
-	n_filetypes = st_n_filetypes;
-        
-	mem_list_count = 1;
-        
-	/* determine how many blocks in file to write */
-	f_data_wrote = ADIOI_MIN(st_fwr_size, bufsize);
-	total_blks_to_write = 1;
-	if (j < (flat_file->count -1)) j++;
-	else {
-	    j = 0;
-	    n_filetypes++;
-	}
-	while (f_data_wrote < bufsize) {
-	    f_data_wrote += flat_file->blocklens[j];
-	    total_blks_to_write++;
-	    if (j<(flat_file->count-1)) j++;
-	    else j = 0; 
-	}
-	    
-	j = st_index;
-	n_filetypes = st_n_filetypes;
-	n_write_lists = total_blks_to_write/MAX_ARRAY_SIZE;
-	extra_blks = total_blks_to_write%MAX_ARRAY_SIZE;
-        
-	mem_offset = (intptr_t) buf;
-	mem_length = 0;
-        
-	/* if at least one full writelist, allocate file arrays
-	   at max array size and don't free until very end */
-	if (n_write_lists) {
-	    file_offsets = (int64_t*)ADIOI_Malloc(MAX_ARRAY_SIZE*
-						  sizeof(int64_t));
-	    file_lengths = (int32_t*)ADIOI_Malloc(MAX_ARRAY_SIZE*
-						  sizeof(int32_t));
-	}
-	/* if there's no full writelist allocate file arrays according
-	   to needed size (extra_blks) */
-	else {
-	    file_offsets = (int64_t*)ADIOI_Malloc(extra_blks*
-                                                  sizeof(int64_t));
-            file_lengths = (int32_t*)ADIOI_Malloc(extra_blks*
-                                                  sizeof(int32_t));
-        }
-        
-        /* for file arrays that are of MAX_ARRAY_SIZE, build arrays */
-        for (i=0; i<n_write_lists; i++) {
-            file_list_count = MAX_ARRAY_SIZE;
-            if(!i) {
-                file_offsets[0] = offset;
-                file_lengths[0] = st_fwr_size;
-                mem_length = st_fwr_size;
-            }
-            for (k=0; k<MAX_ARRAY_SIZE; k++) {
-                if (i || k) {
-                    file_offsets[k] = disp + 
-			((ADIO_Offset)n_filetypes)*filetype_extent
-			+ flat_file->indices[j];
-                    file_lengths[k] = flat_file->blocklens[j];
-                    mem_length += file_lengths[k];
-                }
-                if (j<(flat_file->count - 1)) j++;
-                else {
-                    j = 0;
-                    n_filetypes++;
-                }
-            } /* for (k=0; k<MAX_ARRAY_SIZE; k++) */
-
-	    err_flag = PVFS_Request_contiguous(mem_length,
-					       PVFS_BYTE, &mem_req);
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (err_flag != 0) {
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-						   MPIR_ERR_RECOVERABLE,
-						   myname, __LINE__,
-						   ADIOI_PVFS2_error_convert(err_flag),
-						   "Error in PVFS_Request_contiguous (memory)", 0);
-		goto error_state;
-	    }
-	    /* --END ERROR HANDLING-- */
-
-	    err_flag = PVFS_Request_hindexed(file_list_count, file_lengths, 
-					     file_offsets, PVFS_BYTE,
-					     &file_req);
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (err_flag != 0) {
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-						   MPIR_ERR_RECOVERABLE,
-						   myname, __LINE__,
-						   ADIOI_PVFS2_error_convert(err_flag),
-						   "Error in PVFS_Request_hindexed (file)", 0);
-		goto error_state;
-	    }
-	    /* --END ERROR HANDLING-- */
-
-	    /* PVFS_Request_hindexed already expresses the offsets into the
-	     * file, so we should not pass in an offset if we are using
-	     * hindexed for the file type */
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
-#endif
-	    err_flag = PVFS_sys_write(pvfs_fs->object_ref, file_req, 0, 
-				      (void *)mem_offset, mem_req,
-				      &(pvfs_fs->credentials), &resp_io);
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
-#endif
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (err_flag != 0) {
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-						   MPIR_ERR_RECOVERABLE,
-						   myname, __LINE__,
-						   ADIOI_PVFS2_error_convert(err_flag),
-						   "Error in PVFS_sys_write", 0);
-		goto error_state;
-	    }
-	    /* --END ERROR HANDLING-- */
-	    total_bytes_written += resp_io.total_completed;
-
-            mem_offset += mem_length;
-            mem_lengths = 0;
-	    PVFS_Request_free(&file_req);
-	    PVFS_Request_free(&mem_req);
-
-        } /* for (i=0; i<n_write_lists; i++) */
-
-        /* for file arrays smaller than MAX_ARRAY_SIZE (last write_list call) */
-        if (extra_blks) {
-            file_list_count = extra_blks;
-            if(!i) {
-                file_offsets[0] = offset;
-                file_lengths[0] = ADIOI_MIN(st_fwr_size, bufsize);
-            }
-            for (k=0; k<extra_blks; k++) {
-                if(i || k) {
-                    file_offsets[k] = disp + 
-			((ADIO_Offset)n_filetypes)*filetype_extent +
-			flat_file->indices[j];
-                    if (k == (extra_blks - 1)) {
-                        file_lengths[k] = bufsize - (int32_t) mem_lengths
-                          - mem_offset + (int32_t)  buf;
-                    }
-                    else file_lengths[k] = flat_file->blocklens[j];
-                } /* if(i || k) */
-                mem_lengths += file_lengths[k];
-                if (j<(flat_file->count - 1)) j++;
-                else {
-                    j = 0;
-                    n_filetypes++;
-                }
-            } /* for (k=0; k<extra_blks; k++) */
-
-	    err_flag = PVFS_Request_contiguous(mem_length,
-					       PVFS_BYTE, &mem_req);
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (err_flag != 0) {
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-						   MPIR_ERR_RECOVERABLE,
-						   myname, __LINE__,
-						   ADIOI_PVFS2_error_convert(err_flag),
-						   "Error in PVFS_Request_contiguous (memory)", 0);
-		goto error_state;
-	    }
-	    /* --END ERROR HANDLING-- */
-
-	    err_flag = PVFS_Request_hindexed(file_list_count, file_lengths, 
-					     file_offsets, PVFS_BYTE,
-					     &file_req);
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (err_flag != 0) {
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-						   MPIR_ERR_RECOVERABLE,
-						   myname, __LINE__,
-						   ADIOI_PVFS2_error_convert(err_flag),
-						   "Error in PVFS_Request_hindexed(file)", 0);
-		goto error_state;
-	    }
-	    /* --END ERROR HANDLING-- */
-
-	    /* as above, use 0 for 'offset' when using hindexed file type*/
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
-#endif
-	    err_flag = PVFS_sys_write(pvfs_fs->object_ref, file_req, 0, 
-				      (void *)mem_offset, mem_req,
-				      &(pvfs_fs->credentials), &resp_io);
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
-#endif
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (err_flag != 0) {
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-						   MPIR_ERR_RECOVERABLE,
-						   myname, __LINE__,
-						   ADIOI_PVFS2_error_convert(err_flag),
-						   "Error in PVFS_sys_write", 0);
-		goto error_state;
-	    }
-	    /* --END ERROR HANDLING-- */
-	    total_bytes_written += resp_io.total_completed;
-	    PVFS_Request_free(&mem_req);
-	    PVFS_Request_free(&file_req);
-        }
-    } 
-    else {
-        /* noncontiguous in memory as well as in file */
-
-	flat_buf = ADIOI_Flatten_and_find(datatype);
-
-	size_wrote = 0;
-	n_filetypes = st_n_filetypes;
-	fwr_size = st_fwr_size;
-	bwr_size = flat_buf->blocklens[0];
-	buf_count = 0;
-	start_mem_offset = 0;
-	start_k = k = 0;
-	start_j = st_index;
-	max_mem_list = 0;
-	max_file_list = 0;
-
-	/* run through and file max_file_list and max_mem_list so that you 
-	   can allocate the file and memory arrays less than MAX_ARRAY_SIZE
-	   if possible */
-
-	while (size_wrote < bufsize) {
-	    k = start_k;
-	    new_buffer_write = 0;
-	    mem_list_count = 0;
-	    while ((mem_list_count < MAX_ARRAY_SIZE) && 
-		   (new_buffer_write < bufsize-size_wrote)) {
-	        /* find mem_list_count and file_list_count such that both are
-		   less than MAX_ARRAY_SIZE, the sum of their lengths are
-		   equal, and the sum of all the data written and data to be
-		   written in the next immediate write list is less than
-		   bufsize */
-	        if(mem_list_count) {
-		    if((new_buffer_write + flat_buf->blocklens[k] + 
-			size_wrote) > bufsize) {
-		        end_bwr_size = new_buffer_write + 
-			    flat_buf->blocklens[k] - (bufsize - size_wrote);
-			new_buffer_write = bufsize - size_wrote;
-		    }
-		    else {
-		        new_buffer_write += flat_buf->blocklens[k];
-			end_bwr_size = flat_buf->blocklens[k];
-		    }
-		}
-		else {
-		    if (bwr_size > (bufsize - size_wrote)) {
-		        new_buffer_write = bufsize - size_wrote;
-			bwr_size = new_buffer_write;
-		    }
-		    else new_buffer_write = bwr_size;
-		}
-		mem_list_count++;
-		k = (k + 1)%flat_buf->count;
-	     } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
-	       (new_buffer_write < bufsize-size_wrote)) */
-	    j = start_j;
-	    new_file_write = 0;
-	    file_list_count = 0;
-	    while ((file_list_count < MAX_ARRAY_SIZE) && 
-		   (new_file_write < new_buffer_write)) { 
-	        if(file_list_count) {
-		    if((new_file_write + flat_file->blocklens[j]) > 
-		       new_buffer_write) {
-		        end_fwr_size = new_buffer_write - new_file_write;
-			new_file_write = new_buffer_write;
-			j--;
-		    }
-		    else {
-		        new_file_write += flat_file->blocklens[j];
-			end_fwr_size = flat_file->blocklens[j];
-		    }
-		}
-		else {
-		    if (fwr_size > new_buffer_write) {
-		        new_file_write = new_buffer_write;
-			fwr_size = new_file_write;
-		    }
-		    else new_file_write = fwr_size;
-		}
-		file_list_count++;
-		if (j < (flat_file->count - 1)) j++;
-		else j = 0;
-		
-		k = start_k;
-		if ((new_file_write < new_buffer_write) && 
-		    (file_list_count == MAX_ARRAY_SIZE)) {
-		    new_buffer_write = 0;
-		    mem_list_count = 0;
-		    while (new_buffer_write < new_file_write) {
-		        if(mem_list_count) {
-			    if((new_buffer_write + flat_buf->blocklens[k]) >
-			       new_file_write) {
-			        end_bwr_size = new_file_write - 
-				    new_buffer_write;
-				new_buffer_write = new_file_write;
-				k--;
-			    }
-			    else {
-			        new_buffer_write += flat_buf->blocklens[k];
-				end_bwr_size = flat_buf->blocklens[k];
-			    }
-			}
-			else {
-			    new_buffer_write = bwr_size;
-			    if (bwr_size > (bufsize - size_wrote)) {
-			        new_buffer_write = bufsize - size_wrote;
-				bwr_size = new_buffer_write;
-			    }
-			}
-			mem_list_count++;
-			k = (k + 1)%flat_buf->count;
-		    } /* while (new_buffer_write < new_file_write) */
-		} /* if ((new_file_write < new_buffer_write) &&
-		     (file_list_count == MAX_ARRAY_SIZE)) */
-	    } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
-		 (new_buffer_write < bufsize-size_wrote)) */
-
-	    /*  fakes filling the writelist arrays of lengths found above  */
-	    k = start_k;
-	    j = start_j;
-	    for (i=0; i<mem_list_count; i++) {	     
-		if(i) {
-		    if (i == (mem_list_count - 1)) {
-			if (flat_buf->blocklens[k] == end_bwr_size)
-			    bwr_size = flat_buf->blocklens[(k+1)%
-							  flat_buf->count];
-			else {
-			    bwr_size = flat_buf->blocklens[k] - end_bwr_size;
-			    k--;
-			    buf_count--;
-			}
-		    }
-		}
-		buf_count++;
-		k = (k + 1)%flat_buf->count;
-	    } /* for (i=0; i<mem_list_count; i++) */
-	    for (i=0; i<file_list_count; i++) {
-		if (i) {
-		    if (i == (file_list_count - 1)) {
-			if (flat_file->blocklens[j] == end_fwr_size)
-			    fwr_size = flat_file->blocklens[(j+1)%
-							  flat_file->count];   
-			else {
-			    fwr_size = flat_file->blocklens[j] - end_fwr_size;
-			    j--;
-			}
-		    }
-		}
-		if (j < flat_file->count - 1) j++;
-		else {
-		    j = 0;
-		    n_filetypes++;
-		}
-	    } /* for (i=0; i<file_list_count; i++) */
-	    size_wrote += new_buffer_write;
-	    start_k = k;
-	    start_j = j;
-	    if (max_mem_list < mem_list_count)
-	        max_mem_list = mem_list_count;
-	    if (max_file_list < file_list_count)
-	        max_file_list = file_list_count;
-	} /* while (size_wrote < bufsize) */
-
-	/* one last check before we actually carry out the operation:
-	 * this code has hard-to-fix bugs when a noncontiguous file type has
-	 * such large pieces that the sum of the lengths of the memory type is
-	 * not larger than one of those pieces (and vice versa for large memory
-	 * types and many pices of file types.  In these cases, give up and
-	 * fall back to naive reads and writes.  The testphdf5 test created a
-	 * type with two very large memory regions and 600 very small file
-	 * regions.  The same test also created a type with one very large file
-	 * region and many (700) very small memory regions.  both cases caused
-	 * problems for this code */
-
-	if ( ( (file_list_count == 1) && 
-		    (new_file_write < flat_file->blocklens[0] ) ) ||
-		((mem_list_count == 1) && 
-		    (new_buffer_write < flat_buf->blocklens[0]) ) ||
-		((file_list_count == MAX_ARRAY_SIZE) && 
-		    (new_file_write < flat_buf->blocklens[0]) ) ||
-		( (mem_list_count == MAX_ARRAY_SIZE) &&
-		    (new_buffer_write < flat_file->blocklens[0])) )
-	{
-	    ADIOI_Delete_flattened(datatype);
-	    ADIOI_GEN_WriteStrided_naive(fd, buf, count, datatype,
-		    file_ptr_type, initial_off, status, error_code);
-	    return;
-	}
-
-
-	mem_offsets = (PVFS_size*)ADIOI_Malloc(max_mem_list*sizeof(PVFS_size));
-	mem_lengths = (int *)ADIOI_Malloc(max_mem_list*sizeof(int));
-	file_offsets = (int64_t *)ADIOI_Malloc(max_file_list*sizeof(int64_t));
-	file_lengths = (int32_t *)ADIOI_Malloc(max_file_list*sizeof(int32_t));
-	    
-	size_wrote = 0;
-	n_filetypes = st_n_filetypes;
-	fwr_size = st_fwr_size;
-	bwr_size = flat_buf->blocklens[0];
-	buf_count = 0;
-	start_mem_offset = 0;
-	start_k = k = 0;
-	start_j = st_index;
-
-	/*  this section calculates mem_list_count and file_list_count
-	    and also finds the possibly odd sized last array elements
-	    in new_fwr_size and new_bwr_size  */
-	
-	while (size_wrote < bufsize) {
-	    k = start_k;
-	    new_buffer_write = 0;
-	    mem_list_count = 0;
-	    while ((mem_list_count < MAX_ARRAY_SIZE) && 
-		   (new_buffer_write < bufsize-size_wrote)) {
-	        /* find mem_list_count and file_list_count such that both are
-		   less than MAX_ARRAY_SIZE, the sum of their lengths are
-		   equal, and the sum of all the data written and data to be
-		   written in the next immediate write list is less than
-		   bufsize */
-	        if(mem_list_count) {
-		    if((new_buffer_write + flat_buf->blocklens[k] + 
-			size_wrote) > bufsize) {
-		        end_bwr_size = new_buffer_write + 
-			    flat_buf->blocklens[k] - (bufsize - size_wrote);
-			new_buffer_write = bufsize - size_wrote;
-		    }
-		    else {
-		        new_buffer_write += flat_buf->blocklens[k];
-			end_bwr_size = flat_buf->blocklens[k];
-		    }
-		}
-		else {
-		    if (bwr_size > (bufsize - size_wrote)) {
-		        new_buffer_write = bufsize - size_wrote;
-			bwr_size = new_buffer_write;
-		    }
-		    else new_buffer_write = bwr_size;
-		}
-		mem_list_count++;
-		k = (k + 1)%flat_buf->count;
-	     } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
-	       (new_buffer_write < bufsize-size_wrote)) */
-	    j = start_j;
-	    new_file_write = 0;
-	    file_list_count = 0;
-	    while ((file_list_count < MAX_ARRAY_SIZE) && 
-		   (new_file_write < new_buffer_write)) {
-	        if(file_list_count) {
-		    if((new_file_write + flat_file->blocklens[j]) > 
-		       new_buffer_write) {
-		        end_fwr_size = new_buffer_write - new_file_write;
-			new_file_write = new_buffer_write;
-			j--;
-		    }
-		    else {
-		        new_file_write += flat_file->blocklens[j];
-			end_fwr_size = flat_file->blocklens[j];
-		    }
-		}
-		else {
-		    if (fwr_size > new_buffer_write) {
-		        new_file_write = new_buffer_write;
-			fwr_size = new_file_write;
-		    }
-		    else new_file_write = fwr_size;
-		}
-		file_list_count++;
-		if (j < (flat_file->count - 1)) j++;
-		else j = 0;
-		
-		k = start_k;
-		if ((new_file_write < new_buffer_write) && 
-		    (file_list_count == MAX_ARRAY_SIZE)) {
-		    new_buffer_write = 0;
-		    mem_list_count = 0;
-		    while (new_buffer_write < new_file_write) {
-		        if(mem_list_count) {
-			    if((new_buffer_write + flat_buf->blocklens[k]) >
-			       new_file_write) {
-			        end_bwr_size = new_file_write -
-				  new_buffer_write;
-				new_buffer_write = new_file_write;
-				k--;
-			    }
-			    else {
-			        new_buffer_write += flat_buf->blocklens[k];
-				end_bwr_size = flat_buf->blocklens[k];
-			    }
-			}
-			else {
-			    new_buffer_write = bwr_size;
-			    if (bwr_size > (bufsize - size_wrote)) {
-			        new_buffer_write = bufsize - size_wrote;
-				bwr_size = new_buffer_write;
-			    }
-			}
-			mem_list_count++;
-			k = (k + 1)%flat_buf->count;
-		    } /* while (new_buffer_write < new_file_write) */
-		} /* if ((new_file_write < new_buffer_write) &&
-		     (file_list_count == MAX_ARRAY_SIZE)) */
-	    } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
-		 (new_buffer_write < bufsize-size_wrote)) */
-
-	    /*  fills the allocated writelist arrays  */
-	    k = start_k;
-	    j = start_j;
-	    for (i=0; i<mem_list_count; i++) {	     
-		/* TODO: fix this warning by casting to an integer that's the
-		 * same size as a char * and /then/ casting to PVFS_size */
-	        mem_offsets[i] = ((PVFS_size)buf + buftype_extent*
-				  (buf_count/flat_buf->count) +
-				  (int)flat_buf->indices[k]);
-		
-		if(!i) {
-		    mem_lengths[0] = bwr_size;
-		    mem_offsets[0] += flat_buf->blocklens[k] - bwr_size;
-		}
-		else {
-		    if (i == (mem_list_count - 1)) {
-		        mem_lengths[i] = end_bwr_size;
-			if (flat_buf->blocklens[k] == end_bwr_size)
-			    bwr_size = flat_buf->blocklens[(k+1)%
-							  flat_buf->count];
-			else {
-			    bwr_size = flat_buf->blocklens[k] - end_bwr_size;
-			    k--;
-			    buf_count--;
-			}
-		    }
-		    else {
-		        mem_lengths[i] = flat_buf->blocklens[k];
-		    }
-		}
-		buf_count++;
-		k = (k + 1)%flat_buf->count;
-	    } /* for (i=0; i<mem_list_count; i++) */
-	    for (i=0; i<file_list_count; i++) {
-	        file_offsets[i] = disp + flat_file->indices[j] + 
-		    ((ADIO_Offset)n_filetypes) * filetype_extent;
-	        if (!i) {
-		    file_lengths[0] = fwr_size;
-		    file_offsets[0] += flat_file->blocklens[j] - fwr_size;
-		}
-		else {
-		    if (i == (file_list_count - 1)) {
-		        file_lengths[i] = end_fwr_size;
-			if (flat_file->blocklens[j] == end_fwr_size)
-			    fwr_size = flat_file->blocklens[(j+1)%
-							  flat_file->count];   
-			else {
-			    fwr_size = flat_file->blocklens[j] - end_fwr_size;
-			    j--;
-			}
-		    }
-		    else file_lengths[i] = flat_file->blocklens[j];
-		}
-		if (j < flat_file->count - 1) j++;
-		else {
-		    j = 0;
-		    n_filetypes++;
-		}
-	    } /* for (i=0; i<file_list_count; i++) */
-
-	    err_flag = PVFS_Request_hindexed(mem_list_count, mem_lengths, 
-					     mem_offsets, PVFS_BYTE, &mem_req);
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (err_flag != 0 ) {
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-						   MPIR_ERR_RECOVERABLE,
-						   myname, __LINE__,
-						   ADIOI_PVFS2_error_convert(err_flag),
-						   "Error in PVFS_Request_hindexed (memory)", 0);
-		goto error_state;
-	    }
-	    /* --END ERROR HANDLING-- */
-
-	    err_flag = PVFS_Request_hindexed(file_list_count, file_lengths, 
-					     file_offsets, PVFS_BYTE,
-					     &file_req);
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (err_flag != 0) {
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-						   MPIR_ERR_RECOVERABLE,
-						   myname, __LINE__,
-						   ADIOI_PVFS2_error_convert(err_flag),
-						   "Error in PVFS_Request_hindexed", 0);
-		goto error_state;
-	    }
-	    /* --END ERROR HANDLING-- */
-
-	    /* offset will be expressed in memory and file datatypes */
-
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
-#endif
-	    err_flag = PVFS_sys_write(pvfs_fs->object_ref, file_req, 0, 
-				      PVFS_BOTTOM, mem_req,
-				      &(pvfs_fs->credentials), &resp_io);
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
-#endif
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (err_flag != 0) {
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-						   MPIR_ERR_RECOVERABLE,
-						   myname, __LINE__,
-						   ADIOI_PVFS2_error_convert(err_flag),
-						   "Error in PVFS_sys_write", 0);
-		goto error_state;
-	    }
-	    /* --END ERROR HANDLING-- */
-
-	    size_wrote += new_buffer_write;
-	    total_bytes_written += resp_io.total_completed;
-	    start_k = k;
-	    start_j = j;
-	    PVFS_Request_free(&mem_req);
-	    PVFS_Request_free(&file_req);
-	} /* while (size_wrote < bufsize) */
-	ADIOI_Free(mem_offsets);
-	ADIOI_Free(mem_lengths);
-    }
-    /* when incrementing fp_ind, need to also take into account the file type:
-     * consider an N-element 1-d subarray with a lb and ub: ( |---xxxxx-----|
-     * if we wrote N elements, offset needs to point at beginning of type, not
-     * at empty region at offset N+1).  
-     *
-     * As we discussed on mpich-discuss in may/june 2009, the code below might
-     * look wierd, but by putting fp_ind at the last byte written, the next
-     * time we run through the strided code we'll update the fp_ind to the
-     * right location. */
-    if (file_ptr_type == ADIO_INDIVIDUAL) {
-	fd->fp_ind = file_offsets[file_list_count-1]+
-	    file_lengths[file_list_count-1];
-    }
-    ADIOI_Free(file_offsets);
-    ADIOI_Free(file_lengths);
-
-    *error_code = MPI_SUCCESS;
-
-error_state:
-    fd->fp_sys_posn = -1;   /* set it to null. */
-
-#ifdef HAVE_STATUS_SET_BYTES
-    MPIR_Status_set_bytes(status, datatype, bufsize);
-/* This is a temporary way of filling in status. The right way is to 
-   keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
-#endif
-
-    if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
-}
diff --git a/3rd-party/romio321/adio/ad_sfs/Makefile.mk b/3rd-party/romio321/adio/ad_sfs/Makefile.mk
deleted file mode 100644
index 2db1ac26cbe..00000000000
--- a/3rd-party/romio321/adio/ad_sfs/Makefile.mk
+++ /dev/null
@@ -1,19 +0,0 @@
-## -*- Mode: Makefile; -*-
-## vim: set ft=automake :
-##
-## (C) 2011 by Argonne National Laboratory.
-##     See COPYRIGHT in top-level directory.
-##
-
-if BUILD_AD_SFS
-
-noinst_HEADERS += adio/ad_sfs/ad_sfs.h
-
-romio_other_sources +=         \
-    adio/ad_sfs/ad_sfs_open.c  \
-    adio/ad_sfs/ad_sfs_fcntl.c \
-    adio/ad_sfs/ad_sfs_flush.c \
-    adio/ad_sfs/ad_sfs.c
-
-endif BUILD_AD_SFS
-
diff --git a/3rd-party/romio321/adio/ad_sfs/README b/3rd-party/romio321/adio/ad_sfs/README
deleted file mode 100644
index 933677b1770..00000000000
--- a/3rd-party/romio321/adio/ad_sfs/README
+++ /dev/null
@@ -1 +0,0 @@
-This code is no longer supported.
diff --git a/3rd-party/romio321/adio/ad_sfs/ad_sfs.c b/3rd-party/romio321/adio/ad_sfs/ad_sfs.c
deleted file mode 100644
index c4c1609dd29..00000000000
--- a/3rd-party/romio321/adio/ad_sfs/ad_sfs.c
+++ /dev/null
@@ -1,38 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_sfs.h"
-
-/* adioi.h has the ADIOI_Fns_struct define */
-#include "adioi.h"
-
-struct ADIOI_Fns_struct ADIO_SFS_operations = {
-    ADIOI_SFS_Open, /* Open */
-    ADIOI_GEN_ReadContig, /* ReadContig */
-    ADIOI_GEN_WriteContig, /* WriteContig */
-    ADIOI_GEN_ReadStridedColl, /* ReadStridedColl */
-    ADIOI_GEN_WriteStridedColl, /* WriteStridedColl */
-    ADIOI_GEN_SeekIndividual, /* SeekIndividual */
-    ADIOI_SFS_Fcntl, /* Fcntl */
-    ADIOI_GEN_SetInfo, /* SetInfo */
-    ADIOI_GEN_ReadStrided, /* ReadStrided */
-    ADIOI_GEN_WriteStrided, /* WriteStrided */
-    ADIOI_GEN_Close, /* Close */
-    ADIOI_FAKE_IreadContig, /* IreadContig */
-    ADIOI_FAKE_IwriteContig, /* IwriteContig */
-    ADIOI_FAKE_IODone, /* ReadDone */
-    ADIOI_FAKE_IODone, /* WriteDone */
-    ADIOI_FAKE_IOComplete, /* ReadComplete */
-    ADIOI_FAKE_IOComplete, /* WriteComplete */
-    ADIOI_FAKE_IreadStrided, /* IreadStrided */
-    ADIOI_FAKE_IwriteStrided, /* IwriteStrided */
-    ADIOI_SFS_Flush, /* Flush */
-    ADIOI_GEN_Resize, /* Resize */
-    ADIOI_GEN_Delete, /* Delete */
-    ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
-    ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
-};
diff --git a/3rd-party/romio321/adio/ad_sfs/ad_sfs.h b/3rd-party/romio321/adio/ad_sfs/ad_sfs.h
deleted file mode 100644
index 9f029f05580..00000000000
--- a/3rd-party/romio321/adio/ad_sfs/ad_sfs.h
+++ /dev/null
@@ -1,21 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#ifndef AD_SFS_INCLUDE
-#define AD_SFS_INCLUDE
-
-#include <unistd.h>
-#include <sys/types.h>
-#include <fcntl.h>
-#include "adio.h"
-
-void ADIOI_SFS_Open(ADIO_File fd, int *error_code);
-void ADIOI_SFS_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct, int
-		*error_code); 
-void ADIOI_SFS_Flush(ADIO_File fd, int *error_code);
-
-#endif
diff --git a/3rd-party/romio321/adio/ad_sfs/ad_sfs_fcntl.c b/3rd-party/romio321/adio/ad_sfs/ad_sfs_fcntl.c
deleted file mode 100644
index 533e26a77b6..00000000000
--- a/3rd-party/romio321/adio/ad_sfs/ad_sfs_fcntl.c
+++ /dev/null
@@ -1,64 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_sfs.h"
-#include "adio_extern.h"
-
-void ADIOI_SFS_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct, int *error_code)
-{
-    int  i, ntimes, len;
-    ADIO_Offset curr_fsize, alloc_size, size, done;
-    ADIO_Status status;
-    char *buf;
-#ifndef PRINT_ERR_MSG
-    static char myname[] = "ADIOI_SFS_FCNTL";
-#endif
-
-    switch(flag) {
-    case ADIO_FCNTL_GET_FSIZE:
-        /* On SFS, I find that a write from one process, which changes
-           the file size, does not automatically make the new file size 
-           visible to other processes. Therefore, a sync-barrier-sync is 
-           needed. (Other processes are able to read the data written 
-           though; only file size is returned incorrectly.) */
-
-	fsync(fd->fd_sys);
-	MPI_Barrier(fd->comm);
-	fsync(fd->fd_sys);
-	
-	fcntl_struct->fsize = llseek(fd->fd_sys, 0, SEEK_END);
-	if (fd->fp_sys_posn != -1) 
-	     llseek(fd->fd_sys, fd->fp_sys_posn, SEEK_SET);
-	if (fcntl_struct->fsize == -1) {
-#ifdef MPICH
-	    *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io",
-		"**io %s", strerror(errno));
-#elif defined(PRINT_ERR_MSG)
-	    *error_code = MPI_ERR_UNKNOWN;
-#else /* MPICH-1 */
-	    *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,
-			      myname, "I/O Error", "%s", strerror(errno));
-	    ADIOI_Error(fd, *error_code, myname);	    
-#endif
-	}
-	else *error_code = MPI_SUCCESS;
-	break;
-
-    case ADIO_FCNTL_SET_DISKSPACE:
-	ADIOI_GEN_Prealloc(fd, fcntl_struct->diskspace, error_code);
-	break;
-
-    case ADIO_FCNTL_SET_ATOMICITY:
-	fd->atomicity = (fcntl_struct->atomicity == 0) ? 0 : 1;
-	*error_code = MPI_SUCCESS;
-	break;
-
-    default:
-	FPRINTF(stderr, "Unknown flag passed to ADIOI_SFS_Fcntl\n");
-	MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-}
diff --git a/3rd-party/romio321/adio/ad_sfs/ad_sfs_flush.c b/3rd-party/romio321/adio/ad_sfs/ad_sfs_flush.c
deleted file mode 100644
index 8fc7e358cf6..00000000000
--- a/3rd-party/romio321/adio/ad_sfs/ad_sfs_flush.c
+++ /dev/null
@@ -1,27 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_sfs.h"
-
-void ADIOI_SFS_Flush(ADIO_File fd, int *error_code)
-{
-#ifndef PRINT_ERR_MSG
-    static char myname[] = "ADIOI_SFS_FLUSH";
-#endif
-
-     /* there is no fsync on SX-4 */
-#ifdef MPICH
-    *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io",
-	"**io %s", strerror(errno));
-#elif defined(PRINT_ERR_MSG)
-     *error_code = MPI_ERR_UNKNOWN; 
-#else /* MPICH-1 */
-     *error_code = MPIR_Err_setmsg(MPI_ERR_UNSUPPORTED_OPERATION, 1,
-			      myname, (char *) 0, (char *) 0);
-     ADIOI_Error(fd, *error_code, myname);	    
-#endif
-}
diff --git a/3rd-party/romio321/adio/ad_sfs/ad_sfs_open.c b/3rd-party/romio321/adio/ad_sfs/ad_sfs_open.c
deleted file mode 100644
index ebeefdcbcba..00000000000
--- a/3rd-party/romio321/adio/ad_sfs/ad_sfs_open.c
+++ /dev/null
@@ -1,57 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_sfs.h"
-
-void ADIOI_SFS_Open(ADIO_File fd, int *error_code)
-{
-    int perm, old_mask, amode;
-#ifndef PRINT_ERR_MSG
-    static char myname[] = "ADIOI_SFS_OPEN";
-#endif
-
-    if (fd->perm == ADIO_PERM_NULL) {
-	old_mask = umask(022);
-	umask(old_mask);
-	perm = old_mask ^ 0666;
-    }
-    else perm = fd->perm;
-
-    amode = 0;
-    if (fd->access_mode & ADIO_CREATE)
-	amode = amode | O_CREAT;
-    if (fd->access_mode & ADIO_RDONLY)
-	amode = amode | O_RDONLY;
-    if (fd->access_mode & ADIO_WRONLY)
-	amode = amode | O_WRONLY;
-    if (fd->access_mode & ADIO_RDWR)
-	amode = amode | O_RDWR;
-    if (fd->access_mode & ADIO_EXCL)
-	amode = amode | O_EXCL;
-
-    fd->fd_sys = open(fd->filename, amode, perm);
-    fd->fd_direct = -1;
-
-    if ((fd->fd_sys != -1) && (fd->access_mode & ADIO_APPEND)) {
-	fd->fp_ind = llseek(fd->fd_sys, 0, SEEK_END);
-	fd->fp_sys_posn = fd->fp_ind;
-    }
-
-    if (fd->fd_sys == -1) {
-#ifdef MPICH
-	*error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io",
-	    "**io %s", strerror(errno));
-#elif defined(PRINT_ERR_MSG)
-	*error_code = MPI_ERR_UNKNOWN;
-#else /* MPICH-1 */
-	*error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,
-			      myname, "I/O Error", "%s", strerror(errno));
-	ADIOI_Error(ADIO_FILE_NULL, *error_code, myname);	    
-#endif
-    }
-    else *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_testfs/Makefile.mk b/3rd-party/romio321/adio/ad_testfs/Makefile.mk
deleted file mode 100644
index 7f9c6172171..00000000000
--- a/3rd-party/romio321/adio/ad_testfs/Makefile.mk
+++ /dev/null
@@ -1,32 +0,0 @@
-## -*- Mode: Makefile; -*-
-## vim: set ft=automake :
-##
-## (C) 2011 by Argonne National Laboratory.
-##     See COPYRIGHT in top-level directory.
-##
-
-if BUILD_AD_TESTFS
-
-noinst_HEADERS += adio/ad_testfs/ad_testfs.h
-
-romio_other_sources +=                \
-    adio/ad_testfs/ad_testfs_close.c  \
-    adio/ad_testfs/ad_testfs_read.c   \
-    adio/ad_testfs/ad_testfs_rdcoll.c \
-    adio/ad_testfs/ad_testfs_wrcoll.c \
-    adio/ad_testfs/ad_testfs_open.c   \
-    adio/ad_testfs/ad_testfs_write.c  \
-    adio/ad_testfs/ad_testfs_done.c   \
-    adio/ad_testfs/ad_testfs_fcntl.c  \
-    adio/ad_testfs/ad_testfs_iread.c  \
-    adio/ad_testfs/ad_testfs_iwrite.c \
-    adio/ad_testfs/ad_testfs_wait.c   \
-    adio/ad_testfs/ad_testfs_flush.c  \
-    adio/ad_testfs/ad_testfs_seek.c   \
-    adio/ad_testfs/ad_testfs_resize.c \
-    adio/ad_testfs/ad_testfs_hints.c  \
-    adio/ad_testfs/ad_testfs_delete.c \
-    adio/ad_testfs/ad_testfs.c
-
-endif BUILD_AD_TESTFS
-
diff --git a/3rd-party/romio321/adio/ad_testfs/ad_testfs.c b/3rd-party/romio321/adio/ad_testfs/ad_testfs.c
deleted file mode 100644
index 6696847c3e1..00000000000
--- a/3rd-party/romio321/adio/ad_testfs/ad_testfs.c
+++ /dev/null
@@ -1,41 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_testfs.h"
-
-/* adioi.h has the ADIOI_Fns_struct define */
-#include "adioi.h"
-
-struct ADIOI_Fns_struct ADIO_TESTFS_operations = {
-    ADIOI_TESTFS_Open, /* Open */
-    ADIOI_GEN_OpenColl, /* OpenColl */
-    ADIOI_TESTFS_ReadContig, /* ReadContig */
-    ADIOI_TESTFS_WriteContig, /* WriteContig */
-    ADIOI_TESTFS_ReadStridedColl, /* ReadStridedColl */
-    ADIOI_TESTFS_WriteStridedColl, /* WriteStridedColl */
-    ADIOI_TESTFS_SeekIndividual, /* SeekIndividual */
-    ADIOI_TESTFS_Fcntl, /* Fcntl */
-    ADIOI_TESTFS_SetInfo, /* SetInfo */
-    ADIOI_TESTFS_ReadStrided, /* ReadStrided */
-    ADIOI_TESTFS_WriteStrided, /* WriteStrided */
-    ADIOI_TESTFS_Close, /* Close */
-    ADIOI_TESTFS_IreadContig, /* IreadContig */
-    ADIOI_TESTFS_IwriteContig, /* IwriteContig */
-    ADIOI_TESTFS_ReadDone, /* ReadDone */
-    ADIOI_TESTFS_WriteDone, /* WriteDone */
-    ADIOI_TESTFS_ReadComplete, /* ReadComplete */
-    ADIOI_TESTFS_WriteComplete, /* WriteComplete */
-    ADIOI_TESTFS_IreadStrided, /* IreadStrided */
-    ADIOI_TESTFS_IwriteStrided, /* IwriteStrided */
-    ADIOI_TESTFS_Flush, /* Flush */
-    ADIOI_TESTFS_Resize, /* Resize */
-    ADIOI_TESTFS_Delete, /* Delete */
-    ADIOI_GEN_Feature, /* Features */
-    "TESTFS: the logging-only file system",
-    ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
-    ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
-};
diff --git a/3rd-party/romio321/adio/ad_testfs/ad_testfs.h b/3rd-party/romio321/adio/ad_testfs/ad_testfs.h
deleted file mode 100644
index 7f51ced0d2d..00000000000
--- a/3rd-party/romio321/adio/ad_testfs/ad_testfs.h
+++ /dev/null
@@ -1,85 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#ifndef AD_TESTFS_INCLUDE
-#define AD_TESTFS_INCLUDE
-
-#include "adio.h"
-#include <unistd.h>
-#include <sys/types.h>
-#include <sys/uio.h>
-#include <fcntl.h>
-
-void ADIOI_TESTFS_Open(ADIO_File fd, int *error_code);
-void ADIOI_TESTFS_Close(ADIO_File fd, int *error_code);
-void ADIOI_TESTFS_ReadContig(ADIO_File fd, void *buf, int count,
-			     MPI_Datatype datatype, int file_ptr_type,
-			     ADIO_Offset offset, ADIO_Status *status, int
-			     *error_code);
-void ADIOI_TESTFS_WriteContig(ADIO_File fd, const void *buf, int count,
-			      MPI_Datatype datatype, int file_ptr_type,
-			      ADIO_Offset offset, ADIO_Status *status, int
-			      *error_code);   
-void ADIOI_TESTFS_IwriteContig(ADIO_File fd, const void *buf, int count,
-			       MPI_Datatype datatype, int file_ptr_type,
-			       ADIO_Offset offset, ADIO_Request *request, int
-			       *error_code);   
-void ADIOI_TESTFS_IreadContig(ADIO_File fd, void *buf, int count, 
-			      MPI_Datatype datatype, int file_ptr_type,
-			      ADIO_Offset offset, ADIO_Request *request, int
-			      *error_code);   
-int ADIOI_TESTFS_ReadDone(ADIO_Request *request, ADIO_Status *status, int
-			  *error_code);
-int ADIOI_TESTFS_WriteDone(ADIO_Request *request, ADIO_Status *status, int
-			   *error_code);
-void ADIOI_TESTFS_ReadComplete(ADIO_Request *request, ADIO_Status *status, int
-			       *error_code); 
-void ADIOI_TESTFS_WriteComplete(ADIO_Request *request, ADIO_Status *status,
-				int *error_code); 
-void ADIOI_TESTFS_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct, 
-			int *error_code); 
-void ADIOI_TESTFS_WriteStrided(ADIO_File fd, const void *buf, int count,
-			       MPI_Datatype datatype, int file_ptr_type,
-			       ADIO_Offset offset, ADIO_Status *status,
-			       int *error_code);
-void ADIOI_TESTFS_ReadStrided(ADIO_File fd, void *buf, int count,
-			      MPI_Datatype datatype, int file_ptr_type,
-			      ADIO_Offset offset, ADIO_Status *status, int
-			      *error_code);
-void ADIOI_TESTFS_WriteStridedColl(ADIO_File fd, const void *buf, int count,
-				   MPI_Datatype datatype, int file_ptr_type,
-				   ADIO_Offset offset, ADIO_Status *status, int
-				   *error_code);
-void ADIOI_TESTFS_ReadStridedColl(ADIO_File fd, void *buf, int count,
-				  MPI_Datatype datatype, int file_ptr_type,
-				  ADIO_Offset offset, ADIO_Status *status, int
-				  *error_code);
-void ADIOI_TESTFS_IreadStrided(ADIO_File fd, void *buf, int count,
-			       MPI_Datatype datatype, int file_ptr_type,
-			       ADIO_Offset offset, ADIO_Request *request, int
-			       *error_code);
-void ADIOI_TESTFS_IwriteStrided(ADIO_File fd, const void *buf, int count,
-				MPI_Datatype datatype, int file_ptr_type,
-				ADIO_Offset offset, ADIO_Request *request, int
-				*error_code);
-void ADIOI_TESTFS_Flush(ADIO_File fd, int *error_code);
-void ADIOI_TESTFS_Resize(ADIO_File fd, ADIO_Offset size, int *error_code);
-ADIO_Offset ADIOI_TESTFS_SeekIndividual(ADIO_File fd, ADIO_Offset offset, 
-					int whence, int *error_code);
-void ADIOI_TESTFS_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code);
-void ADIOI_TESTFS_Get_shared_fp(ADIO_File fd, int size, 
-				ADIO_Offset *shared_fp, 
-				int *error_code);
-void ADIOI_TESTFS_Set_shared_fp(ADIO_File fd, ADIO_Offset offset, 
-				int *error_code);
-void ADIOI_TESTFS_Delete(const char *filename, int *error_code);
-
-#endif
-
-
-
-
diff --git a/3rd-party/romio321/adio/ad_testfs/ad_testfs_close.c b/3rd-party/romio321/adio/ad_testfs/ad_testfs_close.c
deleted file mode 100644
index a1b85e600d0..00000000000
--- a/3rd-party/romio321/adio/ad_testfs/ad_testfs_close.c
+++ /dev/null
@@ -1,22 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_testfs.h"
-#include "adioi.h"
-
-void ADIOI_TESTFS_Close(ADIO_File fd, int *error_code)
-{
-    int myrank, nprocs;
-
-    fd->fd_sys = -1;
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-    FPRINTF(stdout, "[%d/%d] ADIOI_TESTFS_Close called on %s\n", myrank, 
-	    nprocs, fd->filename);
-}
diff --git a/3rd-party/romio321/adio/ad_testfs/ad_testfs_delete.c b/3rd-party/romio321/adio/ad_testfs/ad_testfs_delete.c
deleted file mode 100644
index 9a1b6f37e12..00000000000
--- a/3rd-party/romio321/adio/ad_testfs/ad_testfs_delete.c
+++ /dev/null
@@ -1,21 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_testfs.h"
-#include "adioi.h"
-
-void ADIOI_TESTFS_Delete(const char *filename, int *error_code)
-{
-    int myrank, nprocs;
-
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
-    MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
-    FPRINTF(stdout, "[%d/%d] ADIOI_TESTFS_Delete called on %s\n", 
-	    myrank, nprocs, filename);
-}
diff --git a/3rd-party/romio321/adio/ad_testfs/ad_testfs_done.c b/3rd-party/romio321/adio/ad_testfs/ad_testfs_done.c
deleted file mode 100644
index 2ee3111587e..00000000000
--- a/3rd-party/romio321/adio/ad_testfs/ad_testfs_done.c
+++ /dev/null
@@ -1,39 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_testfs.h"
-#include "adioi.h"
-
-int ADIOI_TESTFS_ReadDone(ADIO_Request *request, ADIO_Status *status, int
-			  *error_code)
-{
-    int myrank, nprocs;
-
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size( MPI_COMM_WORLD, &nprocs );
-    MPI_Comm_rank( MPI_COMM_WORLD, &myrank );
-    FPRINTF(stdout, "[%d/%d] ADIOI_TESTFS_ReadDone called on ADIO_REQUEST_NULL\n", 
-	myrank, nprocs);
-    return 1;
-}
-
-
-int ADIOI_TESTFS_WriteDone(ADIO_Request *request, ADIO_Status *status, int
-			   *error_code)
-{
-    int myrank, nprocs;
-
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size( MPI_COMM_WORLD, &nprocs );
-    MPI_Comm_rank( MPI_COMM_WORLD, &myrank );
-    FPRINTF(stdout, 
-		"[%d/%d] ADIOI_TESTFS_WriteDone called on ADIO_REQUEST_NULL\n", 
-		myrank, nprocs);
-    return 1;
-}
diff --git a/3rd-party/romio321/adio/ad_testfs/ad_testfs_fcntl.c b/3rd-party/romio321/adio/ad_testfs/ad_testfs_fcntl.c
deleted file mode 100644
index 8b1e516d9a2..00000000000
--- a/3rd-party/romio321/adio/ad_testfs/ad_testfs_fcntl.c
+++ /dev/null
@@ -1,49 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_testfs.h"
-#include "adioi.h"
-#include "adio_extern.h"
-
-void ADIOI_TESTFS_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct, 
-			int *error_code)
-{
-    int myrank, nprocs;
-    static char myname[] = "ADIOI_TESTFS_FCNTL";
-
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-    FPRINTF(stdout, "[%d/%d] ADIOI_TESTFS_Fcntl called on %s\n", 
-	    myrank, nprocs, fd->filename);
-
-    switch(flag) {
-    case ADIO_FCNTL_GET_FSIZE:
-	fcntl_struct->fsize = 0;
-	*error_code = MPI_SUCCESS;
-	break;
-
-    case ADIO_FCNTL_SET_DISKSPACE:
-	*error_code = MPI_SUCCESS;
-	break;
-
-    case ADIO_FCNTL_SET_ATOMICITY:
-	fd->atomicity = (fcntl_struct->atomicity == 0) ? 0 : 1;
-	*error_code = MPI_SUCCESS;
-	break;
-
-    default:
-	/* --BEGIN ERROR HANDLING-- */
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, 
-					   MPI_ERR_ARG,
-					   "**flag", "**flag %d", flag);
-	return;
-	/* --END ERROR HANDLING-- */
-    }
-}
diff --git a/3rd-party/romio321/adio/ad_testfs/ad_testfs_flush.c b/3rd-party/romio321/adio/ad_testfs/ad_testfs_flush.c
deleted file mode 100644
index 23d559787d3..00000000000
--- a/3rd-party/romio321/adio/ad_testfs/ad_testfs_flush.c
+++ /dev/null
@@ -1,21 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_testfs.h"
-#include "adioi.h"
-
-void ADIOI_TESTFS_Flush(ADIO_File fd, int *error_code)
-{
-    int myrank, nprocs;
-
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-    FPRINTF(stdout, "[%d/%d] ADIOI_TESTFS_Flush called on %s\n", 
-	    myrank, nprocs, fd->filename);
-}
diff --git a/3rd-party/romio321/adio/ad_testfs/ad_testfs_getsh.c b/3rd-party/romio321/adio/ad_testfs/ad_testfs_getsh.c
deleted file mode 100644
index 2bdb3dceb80..00000000000
--- a/3rd-party/romio321/adio/ad_testfs/ad_testfs_getsh.c
+++ /dev/null
@@ -1,23 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_testfs.h"
-#include "adioi.h"
-
-void ADIOI_TESTFS_Get_shared_fp(ADIO_File fd, int size, 
-				ADIO_Offset *shared_fp, 
-				int *error_code)
-{
-    int myrank, nprocs;
-
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-    FPRINTF(stdout, "[%d/%d] ADIOI_TESTFS_Get_shared_fp called on %s\n", 
-	    myrank, nprocs, fd->filename);
-}
diff --git a/3rd-party/romio321/adio/ad_testfs/ad_testfs_hints.c b/3rd-party/romio321/adio/ad_testfs/ad_testfs_hints.c
deleted file mode 100644
index a6c1be9e9fa..00000000000
--- a/3rd-party/romio321/adio/ad_testfs/ad_testfs_hints.c
+++ /dev/null
@@ -1,27 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_testfs.h"
-#include "adioi.h"
-#ifdef ROMIO_BGL
-#include "../ad_bgl/ad_bgl.h"
-#endif
-void ADIOI_TESTFS_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code)
-{
-    int myrank, nprocs;
-
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-    FPRINTF(stdout, "[%d/%d] ADIOI_TESTFS_SetInfo called on %s\n", 
-	    myrank, nprocs, fd->filename);
-    FPRINTF(stdout, "[%d/%d]    calling ADIOI_GEN_SetInfo\n", 
-	    myrank, nprocs);
-
-    ADIOI_GEN_SetInfo(fd, users_info, error_code);
-}
diff --git a/3rd-party/romio321/adio/ad_testfs/ad_testfs_iread.c b/3rd-party/romio321/adio/ad_testfs/ad_testfs_iread.c
deleted file mode 100644
index 77e862438c6..00000000000
--- a/3rd-party/romio321/adio/ad_testfs/ad_testfs_iread.c
+++ /dev/null
@@ -1,62 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_testfs.h"
-#include "adioi.h"
-
-/* ADIOI_TESTFS_IreadContig()
- *
- * Implemented by immediately calling ReadContig()
- */
-void ADIOI_TESTFS_IreadContig(ADIO_File fd, void *buf, int count, 
-			      MPI_Datatype datatype, int file_ptr_type,
-			      ADIO_Offset offset, ADIO_Request *request, int
-			      *error_code)
-{
-    ADIO_Status status;
-    int myrank, nprocs;
-    MPI_Count typesize, len;
-
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-    MPI_Type_size_x(datatype, &typesize);
-    FPRINTF(stdout, "[%d/%d] ADIOI_TESTFS_IreadContig called on %s\n", 
-	    myrank, nprocs, fd->filename);
-    FPRINTF(stdout, "[%d/%d]    calling ADIOI_TESTFS_ReadContig\n", 
-	    myrank, nprocs);
-
-    len = count * typesize;
-    ADIOI_TESTFS_ReadContig(fd, buf, len, MPI_BYTE, file_ptr_type, 
-			    offset, &status, error_code);
-
-    MPIO_Completed_request_create(&fd, len, error_code, request);
-}
-
-void ADIOI_TESTFS_IreadStrided(ADIO_File fd, void *buf, int count,
-			       MPI_Datatype datatype, int file_ptr_type,
-			       ADIO_Offset offset, ADIO_Request *request, int
-			       *error_code)
-{
-    ADIO_Status status;
-    int myrank, nprocs;
-    MPI_Count typesize;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-    MPI_Type_size_x(datatype, &typesize);
-    FPRINTF(stdout, "[%d/%d] ADIOI_TESTFS_IreadStrided called on %s\n", 
-	    myrank, nprocs, fd->filename);
-    FPRINTF(stdout, "[%d/%d]    calling ADIOI_TESTFS_ReadStrided\n", 
-	    myrank, nprocs);
-
-    ADIOI_TESTFS_ReadStrided(fd, buf, count, datatype, file_ptr_type, 
-			     offset, &status, error_code); 
-    MPIO_Completed_request_create(&fd, count*typesize, error_code, request);
-}
-
diff --git a/3rd-party/romio321/adio/ad_testfs/ad_testfs_iwrite.c b/3rd-party/romio321/adio/ad_testfs/ad_testfs_iwrite.c
deleted file mode 100644
index e29c9f6ee03..00000000000
--- a/3rd-party/romio321/adio/ad_testfs/ad_testfs_iwrite.c
+++ /dev/null
@@ -1,68 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_testfs.h"
-#include "adioi.h"
-
-#include "mpiu_greq.h"
-#include "../../mpi-io/mpioimpl.h"
-
-/* ADIOI_TESTFS_IwriteContig()
- *
- * Implemented by immediately calling WriteContig()
- */
-void ADIOI_TESTFS_IwriteContig(ADIO_File fd, const void *buf, int count,
-			       MPI_Datatype datatype, int file_ptr_type,
-			       ADIO_Offset offset, ADIO_Request *request, int
-			       *error_code)
-{
-    ADIO_Status status;
-    int myrank, nprocs;
-    MPI_Count typesize, len;
-
-    *error_code = MPI_SUCCESS;
-
-    MPI_Type_size_x(datatype, &typesize);
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-    FPRINTF(stdout, "[%d/%d] ADIOI_TESTFS_IwriteContig called on %s\n", 
-	    myrank, nprocs, fd->filename);
-    FPRINTF(stdout, "[%d/%d]    calling ADIOI_TESTFS_WriteContig\n", 
-	    myrank, nprocs);
-
-    len = count * typesize;
-    ADIOI_TESTFS_WriteContig(fd, buf, len, MPI_BYTE, file_ptr_type, 
-			     offset, &status, error_code);
-    MPIO_Completed_request_create(&fd, len, error_code, request);
-
-}
-
-void ADIOI_TESTFS_IwriteStrided(ADIO_File fd, const void *buf, int count,
-				MPI_Datatype datatype, int file_ptr_type,
-				ADIO_Offset offset, ADIO_Request *request, int
-				*error_code)
-{
-    ADIO_Status status;
-    int myrank, nprocs;
-    MPI_Count typesize;
-
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-    MPI_Type_size_x(datatype, &typesize);
-
-    FPRINTF(stdout, "[%d/%d] ADIOI_TESTFS_IwriteStrided called on %s\n", 
-	    myrank, nprocs, fd->filename);
-    FPRINTF(stdout, "[%d/%d]    calling ADIOI_TESTFS_WriteStrided\n", 
-	    myrank, nprocs);
-
-    ADIOI_TESTFS_WriteStrided(fd, buf, count, datatype, file_ptr_type, 
-			      offset, &status, error_code);
-    MPIO_Completed_request_create(&fd, count*typesize, error_code, request);
-
-}
diff --git a/3rd-party/romio321/adio/ad_testfs/ad_testfs_open.c b/3rd-party/romio321/adio/ad_testfs/ad_testfs_open.c
deleted file mode 100644
index 6b1595b99ca..00000000000
--- a/3rd-party/romio321/adio/ad_testfs/ad_testfs_open.c
+++ /dev/null
@@ -1,23 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_testfs.h"
-#include "adioi.h"
-
-void ADIOI_TESTFS_Open(ADIO_File fd, int *error_code)
-{
-    int myrank, nprocs;
-
-    fd->fd_sys = 1;
-    fd->fd_direct = -1;
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-    FPRINTF(stdout, "[%d/%d] ADIOI_TESTFS_Open called on %s\n", myrank, 
-	    nprocs, fd->filename);
-}
diff --git a/3rd-party/romio321/adio/ad_testfs/ad_testfs_rdcoll.c b/3rd-party/romio321/adio/ad_testfs/ad_testfs_rdcoll.c
deleted file mode 100644
index 5df94458fdd..00000000000
--- a/3rd-party/romio321/adio/ad_testfs/ad_testfs_rdcoll.c
+++ /dev/null
@@ -1,29 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_testfs.h"
-#include "adioi.h"
-
-void ADIOI_TESTFS_ReadStridedColl(ADIO_File fd, void *buf, int count,
-				  MPI_Datatype datatype, int file_ptr_type,
-				  ADIO_Offset offset, ADIO_Status *status, 
-				  int *error_code)
-{
-    int myrank, nprocs;
-
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-    FPRINTF(stdout, "[%d/%d] ADIOI_TESTFS_ReadStridedColl called on %s\n", 
-	    myrank, nprocs, fd->filename);    
-    FPRINTF(stdout, "[%d/%d]    calling ADIOI_GEN_ReadStridedColl\n", 
-	    myrank, nprocs);
-
-    ADIOI_GEN_ReadStridedColl(fd, buf, count, datatype, file_ptr_type,
-			      offset, status, error_code);
-}
diff --git a/3rd-party/romio321/adio/ad_testfs/ad_testfs_read.c b/3rd-party/romio321/adio/ad_testfs/ad_testfs_read.c
deleted file mode 100644
index f60a9920f76..00000000000
--- a/3rd-party/romio321/adio/ad_testfs/ad_testfs_read.c
+++ /dev/null
@@ -1,62 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_testfs.h"
-#include "adioi.h"
-
-void ADIOI_TESTFS_ReadContig(ADIO_File fd, void *buf, int count, 
-			     MPI_Datatype datatype, int file_ptr_type,
-			     ADIO_Offset offset, ADIO_Status *status, int
-			     *error_code)
-{
-    int myrank, nprocs;
-    MPI_Count datatype_size;
-
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-    MPI_Type_size_x(datatype, &datatype_size);
-    FPRINTF(stdout, "[%d/%d] ADIOI_TESTFS_ReadContig called on %s\n", myrank, 
-	    nprocs, fd->filename);
-    if (file_ptr_type != ADIO_EXPLICIT_OFFSET)
-    {
-	offset = fd->fp_ind;
-	fd->fp_ind += datatype_size * count;
-	fd->fp_sys_posn = fd->fp_ind;
-    }
-    else {
-	fd->fp_sys_posn = offset + datatype_size * count;
-    }
-
-    FPRINTF(stdout, "[%d/%d]    reading (buf = %p, loc = %lld, sz = %lld)\n",
-	    myrank, nprocs, buf, (long long) offset, 
-	    (long long) datatype_size * count);
-
-#ifdef HAVE_STATUS_SET_BYTES
-    MPIR_Status_set_bytes(status, datatype, datatype_size * count);
-#endif
-}
-
-void ADIOI_TESTFS_ReadStrided(ADIO_File fd, void *buf, int count,
-			      MPI_Datatype datatype, int file_ptr_type,
-			      ADIO_Offset offset, ADIO_Status *status, int
-			      *error_code)
-{
-    int myrank, nprocs;
-
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-    FPRINTF(stdout, "[%d/%d] ADIOI_TESTFS_ReadStrided called on %s\n", myrank, 
-	    nprocs, fd->filename);
-    FPRINTF(stdout, "[%d/%d]    calling ADIOI_GEN_ReadStrided\n", myrank, 
-	    nprocs);
-
-    ADIOI_GEN_ReadStrided(fd, buf, count, datatype, file_ptr_type, offset,
-			  status, error_code);
-}
diff --git a/3rd-party/romio321/adio/ad_testfs/ad_testfs_resize.c b/3rd-party/romio321/adio/ad_testfs/ad_testfs_resize.c
deleted file mode 100644
index a4a37eb94bb..00000000000
--- a/3rd-party/romio321/adio/ad_testfs/ad_testfs_resize.c
+++ /dev/null
@@ -1,21 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_testfs.h"
-#include "adioi.h"
-
-void ADIOI_TESTFS_Resize(ADIO_File fd, ADIO_Offset size, int *error_code)
-{
-    int myrank, nprocs;
-
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-    FPRINTF(stdout, "[%d/%d] ADIOI_TESTFS_Resize called on %s\n", 
-	    myrank, nprocs, fd->filename);
-}
diff --git a/3rd-party/romio321/adio/ad_testfs/ad_testfs_seek.c b/3rd-party/romio321/adio/ad_testfs/ad_testfs_seek.c
deleted file mode 100644
index 9366a1215e5..00000000000
--- a/3rd-party/romio321/adio/ad_testfs/ad_testfs_seek.c
+++ /dev/null
@@ -1,82 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_testfs.h"
-#include "adioi.h"
-#include "adio_extern.h"
-
-/* ADIOI_TESTFS_SeekIndividual()
- *
- * Implements SEEK_SET only (and doesn't test for whence type); all
- * other types of whence must be converted before calling this.
- *
- * Returns an absolute offset in bytes.  The offset passed into the call is in
- * terms of the etype relative to the filetype, so some calculations are
- * necessary.  
- */
-ADIO_Offset ADIOI_TESTFS_SeekIndividual(ADIO_File fd, ADIO_Offset offset, 
-					int whence, int *error_code)
-{
-    int myrank, nprocs;
-
-    ADIO_Offset off;
-    ADIOI_Flatlist_node *flat_file;
-    int i, n_etypes_in_filetype, n_filetypes, etype_in_filetype;
-    ADIO_Offset abs_off_in_filetype=0, sum;
-    int size_in_filetype;
-    int filetype_is_contig;
-    MPI_Count filetype_size;
-    MPI_Aint etype_size, filetype_extent, lb;
-
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-    FPRINTF(stdout, "[%d/%d] ADIOI_TESTFS_SeekIndividual called on %s\n", 
-	    myrank, nprocs, fd->filename);
-
-    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-    etype_size = fd->etype_size;
-
-    if (filetype_is_contig) off = fd->disp + etype_size * offset;
-    else {
-        flat_file = ADIOI_Flatlist;
-        while (flat_file->type != fd->filetype) flat_file = flat_file->next;
-
-	MPI_Type_get_extent(fd->filetype, &lb, &filetype_extent);
-	MPI_Type_size_x(fd->filetype, &filetype_size);
-	if ( ! filetype_size ) {
-	    *error_code = MPI_SUCCESS; 
-	    return 0;
-	}
-
-	n_etypes_in_filetype = filetype_size/etype_size;
-  ADIOI_Assert((offset / n_etypes_in_filetype) == (int) (offset / n_etypes_in_filetype));
-	n_filetypes = (int) (offset / n_etypes_in_filetype);
-	etype_in_filetype = (int) (offset % n_etypes_in_filetype);
-	size_in_filetype = etype_in_filetype * etype_size;
- 
-	sum = 0;
-	for (i=0; i<flat_file->count; i++) {
-	    sum += flat_file->blocklens[i];
-
-	    if (sum > size_in_filetype) {
-		abs_off_in_filetype = flat_file->indices[i] +
-		    size_in_filetype - (sum - flat_file->blocklens[i]);
-		break;
-	    }
-	}
-
-	/* abs. offset in bytes in the file */
-	off = fd->disp + (ADIO_Offset)n_filetypes * (ADIO_Offset)filetype_extent +
-                abs_off_in_filetype;
-    }
-
-    fd->fp_ind = off;
-
-    return off;
-}
diff --git a/3rd-party/romio321/adio/ad_testfs/ad_testfs_setsh.c b/3rd-party/romio321/adio/ad_testfs/ad_testfs_setsh.c
deleted file mode 100644
index 6fc8a04be9f..00000000000
--- a/3rd-party/romio321/adio/ad_testfs/ad_testfs_setsh.c
+++ /dev/null
@@ -1,22 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_testfs.h"
-#include "adioi.h"
-
-void ADIOI_TESTFS_Set_shared_fp(ADIO_File fd, ADIO_Offset offset, 
-				int *error_code)
-{
-    int myrank, nprocs;
-
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-    FPRINTF(stdout, "[%d/%d] ADIOI_TESTFS_Set_shared_fp called on %s\n", 
-	    myrank, nprocs, fd->filename);
-}
diff --git a/3rd-party/romio321/adio/ad_testfs/ad_testfs_wait.c b/3rd-party/romio321/adio/ad_testfs/ad_testfs_wait.c
deleted file mode 100644
index 6eef9ef7060..00000000000
--- a/3rd-party/romio321/adio/ad_testfs/ad_testfs_wait.c
+++ /dev/null
@@ -1,39 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_testfs.h"
-#include "adioi.h"
-
-void ADIOI_TESTFS_ReadComplete(ADIO_Request *request, ADIO_Status *status, int
-			       *error_code)
-{
-    int myrank, nprocs;
-
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
-    MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
-    FPRINTF(stdout, "[%d/%d] ADIOI_TESTFS_ReadComplete called \n", 
-	    myrank, nprocs);
-
-    /* do something with status set bytes? */
-}
-
-void ADIOI_TESTFS_WriteComplete(ADIO_Request *request, ADIO_Status *status, int
-				*error_code)
-{
-    int myrank, nprocs;
-
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
-    MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
-    FPRINTF(stdout, "[%d/%d] ADIOI_TESTFS_WriteComplete called\n", 
-	    myrank, nprocs);
-
-    /* do something with status_set_bytes? */
-}
diff --git a/3rd-party/romio321/adio/ad_testfs/ad_testfs_wrcoll.c b/3rd-party/romio321/adio/ad_testfs/ad_testfs_wrcoll.c
deleted file mode 100644
index ac11f5847f3..00000000000
--- a/3rd-party/romio321/adio/ad_testfs/ad_testfs_wrcoll.c
+++ /dev/null
@@ -1,29 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_testfs.h"
-#include "adioi.h"
-
-void ADIOI_TESTFS_WriteStridedColl(ADIO_File fd, const void *buf, int count,
-				   MPI_Datatype datatype, int file_ptr_type,
-				   ADIO_Offset offset, ADIO_Status *status, 
-				   int *error_code)
-{
-    int myrank, nprocs;
-
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-    FPRINTF(stdout, "[%d/%d] ADIOI_TESTFS_WriteStridedColl called on %s\n", 
-	    myrank, nprocs, fd->filename);
-    FPRINTF(stdout, "[%d/%d]    calling ADIOI_GEN_WriteStridedColl\n", 
-	    myrank, nprocs);
-
-    ADIOI_GEN_WriteStridedColl(fd, buf, count, datatype, file_ptr_type,
-			       offset, status, error_code);
-}
diff --git a/3rd-party/romio321/adio/ad_testfs/ad_testfs_write.c b/3rd-party/romio321/adio/ad_testfs/ad_testfs_write.c
deleted file mode 100644
index e1a59ff0101..00000000000
--- a/3rd-party/romio321/adio/ad_testfs/ad_testfs_write.c
+++ /dev/null
@@ -1,63 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_testfs.h"
-#include "adioi.h"
-
-void ADIOI_TESTFS_WriteContig(ADIO_File fd, const void *buf, int count,
-			      MPI_Datatype datatype, int file_ptr_type,
-			      ADIO_Offset offset, ADIO_Status *status, int
-			      *error_code)
-{
-    int myrank, nprocs;
-    MPI_Count datatype_size;
-
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-    MPI_Type_size_x(datatype, &datatype_size);
-    FPRINTF(stdout, "[%d/%d] ADIOI_TESTFS_WriteContig called on %s\n", myrank, 
-	    nprocs, fd->filename);
-    FPRINTF(stdout, "[%d/%d]    writing (buf = %p, loc = %lld, sz = %lld)\n",
-	    myrank, nprocs, buf, (long long) offset, 
-	    (long long)datatype_size * (long long)count);
-
-    if (file_ptr_type != ADIO_EXPLICIT_OFFSET)
-    {
-	fd->fp_ind += datatype_size * count;
-	fd->fp_sys_posn = fd->fp_ind;
-	FPRINTF(stdout, "[%d/%d]    new file position is %lld\n", myrank, 
-		nprocs, (long long) fd->fp_ind);
-    }
-    else {
-	fd->fp_sys_posn = offset + datatype_size * count;
-    }
-
-#ifdef HAVE_STATUS_SET_BYTES
-    MPIR_Status_set_bytes(status, datatype, datatype_size * count);
-#endif
-}
-
-void ADIOI_TESTFS_WriteStrided(ADIO_File fd, const void *buf, int count,
-			       MPI_Datatype datatype, int file_ptr_type,
-			       ADIO_Offset offset, ADIO_Status *status,
-			       int *error_code)
-{
-    int myrank, nprocs;
-
-    *error_code = MPI_SUCCESS;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-    FPRINTF(stdout, "[%d/%d] ADIOI_TESTFS_WriteStrided called on %s\n", 
-	    myrank, nprocs, fd->filename);
-    FPRINTF(stdout, "[%d/%d]    calling ADIOI_GEN_WriteStrided\n", 
-	    myrank, nprocs);
-
-    ADIOI_GEN_WriteStrided(fd, buf, count, datatype, file_ptr_type, offset, 
-			   status, error_code);
-}
diff --git a/3rd-party/romio321/adio/ad_ufs/Makefile.mk b/3rd-party/romio321/adio/ad_ufs/Makefile.mk
deleted file mode 100644
index 40ecdb4ae4e..00000000000
--- a/3rd-party/romio321/adio/ad_ufs/Makefile.mk
+++ /dev/null
@@ -1,17 +0,0 @@
-## -*- Mode: Makefile; -*-
-## vim: set ft=automake :
-##
-## (C) 2011 by Argonne National Laboratory.
-##     See COPYRIGHT in top-level directory.
-##
-
-if BUILD_AD_UFS
-
-noinst_HEADERS += adio/ad_ufs/ad_ufs.h
-
-romio_other_sources +=        \
-    adio/ad_ufs/ad_ufs.c      \
-    adio/ad_ufs/ad_ufs_open.c
-
-endif BUILD_AD_UFS
-
diff --git a/3rd-party/romio321/adio/ad_ufs/ad_ufs.c b/3rd-party/romio321/adio/ad_ufs/ad_ufs.c
deleted file mode 100644
index a7134dce0b0..00000000000
--- a/3rd-party/romio321/adio/ad_ufs/ad_ufs.c
+++ /dev/null
@@ -1,46 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_ufs.h"
-
-/* adioi.h has the ADIOI_Fns_struct define */
-#include "adioi.h"
-
-struct ADIOI_Fns_struct ADIO_UFS_operations = {
-    ADIOI_UFS_Open, /* Open */
-    ADIOI_GEN_OpenColl, /* OpenColl */
-    ADIOI_GEN_ReadContig, /* ReadContig */
-    ADIOI_GEN_WriteContig, /* WriteContig */
-    ADIOI_GEN_ReadStridedColl, /* ReadStridedColl */
-    ADIOI_GEN_WriteStridedColl, /* WriteStridedColl */
-    ADIOI_GEN_SeekIndividual, /* SeekIndividual */
-    ADIOI_GEN_Fcntl, /* Fcntl */
-    ADIOI_GEN_SetInfo, /* SetInfo */
-    ADIOI_GEN_ReadStrided, /* ReadStrided */
-    ADIOI_GEN_WriteStrided, /* WriteStrided */
-    ADIOI_GEN_Close, /* Close */
-#ifdef ROMIO_HAVE_WORKING_AIO
-    ADIOI_GEN_IreadContig, /* IreadContig */
-    ADIOI_GEN_IwriteContig, /* IwriteContig */
-#else
-    ADIOI_FAKE_IreadContig, /* IreadContig */
-    ADIOI_FAKE_IwriteContig, /* IwriteContig */
-#endif
-    ADIOI_GEN_IODone, /* ReadDone */
-    ADIOI_GEN_IODone, /* WriteDone */
-    ADIOI_GEN_IOComplete, /* ReadComplete */
-    ADIOI_GEN_IOComplete, /* WriteComplete */
-    ADIOI_GEN_IreadStrided, /* IreadStrided */
-    ADIOI_GEN_IwriteStrided, /* IwriteStrided */
-    ADIOI_GEN_Flush, /* Flush */
-    ADIOI_GEN_Resize, /* Resize */
-    ADIOI_GEN_Delete, /* Delete */
-    ADIOI_GEN_Feature, /* Features */
-    "UFS: Generic ROMIO driver for all UNIX-like file systems",
-    ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
-    ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
-};
diff --git a/3rd-party/romio321/adio/ad_ufs/ad_ufs.h b/3rd-party/romio321/adio/ad_ufs/ad_ufs.h
deleted file mode 100644
index 72d66f853bf..00000000000
--- a/3rd-party/romio321/adio/ad_ufs/ad_ufs.h
+++ /dev/null
@@ -1,62 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#ifndef AD_UNIX_INCLUDE
-#define AD_UNIX_INCLUDE
-
-#include "adio.h"
-#include <unistd.h>
-#include <sys/types.h>
-#include <fcntl.h>
-
-#ifdef HAVE_SIGNAL_H
-#include <signal.h>
-#endif
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-
-#ifdef HAVE_AIO_LITE_H
-#include <aio-lite.h>
-#else
- #ifdef  HAVE_AIO_H
- #include <aio.h>
- #endif
- #ifdef HAVE_SYS_AIO_H
- #include <sys/aio.h>
- #endif
-#endif
-
-/* Workaround for incomplete set of definitions if __REDIRECT is not 
-   defined and large file support is used in aio.h */
-#if !defined(__REDIRECT) && defined(__USE_FILE_OFFSET64)
-#define aiocb aiocb64
-#endif
-
-int ADIOI_UFS_aio(ADIO_File fd, void *buf, int len, ADIO_Offset offset,
-		  int wr, void *handle);
-
-void ADIOI_UFS_Open(ADIO_File fd, int *error_code);
-void ADIOI_UFS_IwriteContig(ADIO_File fd, void *buf, int count, 
-                      MPI_Datatype datatype, int file_ptr_type,
-                      ADIO_Offset offset, ADIO_Request *request, int
-		      *error_code);   
-void ADIOI_UFS_IreadContig(ADIO_File fd, void *buf, int count, 
-                      MPI_Datatype datatype, int file_ptr_type,
-                      ADIO_Offset offset, ADIO_Request *request, int
-		      *error_code);   
-int ADIOI_UFS_ReadDone(ADIO_Request *request, ADIO_Status *status, int
-		       *error_code);
-int ADIOI_UFS_WriteDone(ADIO_Request *request, ADIO_Status *status, int
-		       *error_code);
-void ADIOI_UFS_ReadComplete(ADIO_Request *request, ADIO_Status *status, int
-		       *error_code); 
-void ADIOI_UFS_WriteComplete(ADIO_Request *request, ADIO_Status *status,
-			int *error_code); 
-void ADIOI_UFS_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct, int
-		*error_code); 
-
-#endif
diff --git a/3rd-party/romio321/adio/ad_ufs/ad_ufs_open.c b/3rd-party/romio321/adio/ad_ufs/ad_ufs_open.c
deleted file mode 100644
index 9d5a2a11741..00000000000
--- a/3rd-party/romio321/adio/ad_ufs/ad_ufs_open.c
+++ /dev/null
@@ -1,58 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_ufs.h"
-
-void ADIOI_UFS_Open(ADIO_File fd, int *error_code)
-{
-    int perm, old_mask, amode;
-    static char myname[] = "ADIOI_UFS_OPEN";
-
-    if (fd->perm == ADIO_PERM_NULL) {
-	old_mask = umask(022);
-	umask(old_mask);
-	perm = old_mask ^ 0666;
-    }
-    else perm = fd->perm;
-
-    amode = 0;
-    if (fd->access_mode & ADIO_CREATE)
-	amode = amode | O_CREAT;
-    if (fd->access_mode & ADIO_RDONLY)
-	amode = amode | O_RDONLY;
-    if (fd->access_mode & ADIO_WRONLY)
-	amode = amode | O_WRONLY;
-    if (fd->access_mode & ADIO_RDWR)
-	amode = amode | O_RDWR;
-    if (fd->access_mode & ADIO_EXCL)
-	amode = amode | O_EXCL;
-
-    
-#ifdef ADIOI_MPE_LOGGING
-    MPE_Log_event( ADIOI_MPE_open_a, 0, NULL );
-#endif
-    fd->fd_sys = open(fd->filename, amode, perm);
-#ifdef ADIOI_MPE_LOGGING
-    MPE_Log_event( ADIOI_MPE_open_b, 0, NULL );
-#endif
-    fd->fd_direct = -1;
-
-    if ((fd->fd_sys != -1) && (fd->access_mode & ADIO_APPEND)) {
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-	fd->fp_ind = fd->fp_sys_posn = lseek(fd->fd_sys, 0, SEEK_END);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-    }
-
-    if (fd->fd_sys == -1) {
-	*error_code = ADIOI_Err_create_code(myname, fd->filename, errno);
-    }
-    else *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_xfs/Makefile.mk b/3rd-party/romio321/adio/ad_xfs/Makefile.mk
deleted file mode 100644
index 9a2d7078a3e..00000000000
--- a/3rd-party/romio321/adio/ad_xfs/Makefile.mk
+++ /dev/null
@@ -1,22 +0,0 @@
-## -*- Mode: Makefile; -*-
-## vim: set ft=automake :
-##
-## (C) 2011 by Argonne National Laboratory.
-##     See COPYRIGHT in top-level directory.
-##
-
-if BUILD_AD_XFS
-
-noinst_HEADERS += adio/ad_xfs/ad_xfs.h
-
-romio_other_sources +=          \
-    adio/ad_xfs/ad_xfs.c        \
-    adio/ad_xfs/ad_xfs_fcntl.c  \
-    adio/ad_xfs/ad_xfs_hints.c  \
-    adio/ad_xfs/ad_xfs_open.c   \
-    adio/ad_xfs/ad_xfs_read.c   \
-    adio/ad_xfs/ad_xfs_resize.c \
-    adio/ad_xfs/ad_xfs_write.c
-
-endif BUILD_AD_XFS
-
diff --git a/3rd-party/romio321/adio/ad_xfs/ad_xfs.c b/3rd-party/romio321/adio/ad_xfs/ad_xfs.c
deleted file mode 100644
index f43e0e86b7c..00000000000
--- a/3rd-party/romio321/adio/ad_xfs/ad_xfs.c
+++ /dev/null
@@ -1,46 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_xfs.h"
-
-/* adioi.h has the ADIOI_Fns_struct define */
-#include "adioi.h"
-
-struct ADIOI_Fns_struct ADIO_XFS_operations = {
-    ADIOI_XFS_Open, /* Open */
-    ADIOI_GEN_OpenColl, /* OpenColl */
-    ADIOI_XFS_ReadContig, /* ReadContig */
-    ADIOI_XFS_WriteContig, /* WriteContig */
-    ADIOI_GEN_ReadStridedColl, /* ReadStridedColl */
-    ADIOI_GEN_WriteStridedColl, /* WriteStridedColl */
-    ADIOI_GEN_SeekIndividual, /* SeekIndividual */
-    ADIOI_XFS_Fcntl, /* Fcntl */
-    ADIOI_XFS_SetInfo, /* SetInfo */
-    ADIOI_GEN_ReadStrided, /* ReadStrided */
-    ADIOI_GEN_WriteStrided, /* WriteStrided */
-    ADIOI_GEN_Close, /* Close */
-#if defined(ROMIO_HAVE_WORKING_AIO)
-    ADIOI_GEN_IreadContig, /* IreadContig */
-    ADIOI_GEN_IwriteContig, /* IwriteContig */
-#else
-    ADIOI_FAKE_IreadContig, /* IreadContig */
-    ADIOI_FAKE_IwriteContig, /* IwriteContig */
-#endif /* ROMIO_HAVE_WORKING_AIO */
-    ADIOI_GEN_IODone, /* ReadDone */
-    ADIOI_GEN_IODone, /* WriteDone */
-    ADIOI_GEN_IOComplete, /* ReadComplete */
-    ADIOI_GEN_IOComplete, /* WriteComplete */
-    ADIOI_GEN_IreadStrided, /* IreadStrided */
-    ADIOI_GEN_IwriteStrided, /* IwriteStrided */
-    ADIOI_GEN_Flush, /* Flush */
-    ADIOI_XFS_Resize, /* Resize */
-    ADIOI_GEN_Delete, /* Delete */
-    ADIOI_GEN_Feature, /* Features */
-    "XFS: SGI XFS",
-    ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
-    ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
-};
diff --git a/3rd-party/romio321/adio/ad_xfs/ad_xfs.h b/3rd-party/romio321/adio/ad_xfs/ad_xfs.h
deleted file mode 100644
index d14858d4aa5..00000000000
--- a/3rd-party/romio321/adio/ad_xfs/ad_xfs.h
+++ /dev/null
@@ -1,39 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#ifndef AD_XFS_INCLUDE
-#define AD_XFS_INCLUDE
-
-#include <unistd.h>
-#include <sys/types.h>
-#include <fcntl.h>
-#include "adio.h"
-
-#if defined(MPISGI)
-#include "xfs/xfs_fs.h"
-#ifndef  __USE_LARGEFILE64
-#define  __USE_LARGEFILE64
-#endif
-typedef struct aiocb64 aiocb64_t;
-#endif
-
-void ADIOI_XFS_Open(ADIO_File fd, int *error_code);
-void ADIOI_XFS_Close(ADIO_File fd, int *error_code);
-void ADIOI_XFS_ReadContig(ADIO_File fd, void *buf, int count, 
-                      MPI_Datatype datatype, int file_ptr_type,
-                     ADIO_Offset offset, ADIO_Status *status, int
-		     *error_code);
-void ADIOI_XFS_WriteContig(ADIO_File fd, void *buf, int count, 
-                      MPI_Datatype datatype, int file_ptr_type,
-                      ADIO_Offset offset, ADIO_Status *status, int
-		      *error_code);   
-void ADIOI_XFS_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct, int
-		*error_code);
-void ADIOI_XFS_Resize(ADIO_File fd, ADIO_Offset size, int *error_code);
-void ADIOI_XFS_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code);
-
-#endif
diff --git a/3rd-party/romio321/adio/ad_xfs/ad_xfs_fcntl.c b/3rd-party/romio321/adio/ad_xfs/ad_xfs_fcntl.c
deleted file mode 100644
index 1f19081afca..00000000000
--- a/3rd-party/romio321/adio/ad_xfs/ad_xfs_fcntl.c
+++ /dev/null
@@ -1,81 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_xfs.h"
-#include "adio_extern.h"
-#include <sys/ioctl.h>
-
-#ifndef HAVE_LSEEK64
-#define lseek64 lseek
-#endif
-
-void ADIOI_XFS_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct, int *error_code)
-{
-    int i, err;
-#if defined(LINUX) && defined(MPISGI)
-    struct xfs_flock64 fl;
-#else
-    struct flock64 fl;
-#endif
-    static char myname[] = "ADIOI_XFS_FCNTL";
-
-    switch(flag) {
-    case ADIO_FCNTL_GET_FSIZE:
-	fcntl_struct->fsize = lseek64(fd->fd_sys, 0, SEEK_END);
-	if (fcntl_struct->fsize == -1) {
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE, myname,
-					       __LINE__, MPI_ERR_IO, "**io",
-					       "**io %s", strerror(errno));
-	}
-	else *error_code = MPI_SUCCESS;
-	break;
-
-    case ADIO_FCNTL_SET_DISKSPACE:
-	i = 0;
-	fl.l_start = 0;
-	fl.l_whence = SEEK_SET;
-	fl.l_len = fcntl_struct->diskspace;
-
-#if defined(LINUX) && defined(MPISGI)
-	err = ioctl(fd->fd_sys, XFS_IOC_RESVSP64, &fl);
-#else
-	err = fcntl(fd->fd_sys, F_RESVSP64, &fl);
-#endif
-
-	if (err) i = 1;
-	if (fcntl_struct->diskspace > lseek64(fd->fd_sys, 0, SEEK_END)) {
-	    /* also need to set the file size */
-	    err = ftruncate64(fd->fd_sys, fcntl_struct->diskspace);
-	    if (err) i = 1;
-	}
-
-	if (i == 1) {
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE, myname,
-					       __LINE__, MPI_ERR_IO, "**io",
-					       "**io %s", strerror(errno));
-	}
-	else *error_code = MPI_SUCCESS;
-	break;
-
-    case ADIO_FCNTL_SET_ATOMICITY:
-	fd->atomicity = (fcntl_struct->atomicity == 0) ? 0 : 1;
-	*error_code = MPI_SUCCESS;
-	break;
-
-    default:
-	/* --BEGIN ERROR HANDLING-- */
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   MPI_ERR_ARG,
-					   "**flag", "**flag %d", flag);
-	return;
-	/* --END ERROR HANDLING-- */
-    }
-}
diff --git a/3rd-party/romio321/adio/ad_xfs/ad_xfs_hints.c b/3rd-party/romio321/adio/ad_xfs/ad_xfs_hints.c
deleted file mode 100644
index 0fe0e832f4c..00000000000
--- a/3rd-party/romio321/adio/ad_xfs/ad_xfs_hints.c
+++ /dev/null
@@ -1,97 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_xfs.h"
-#include "adio_extern.h"
-
-static unsigned xfs_direct_read_chunk_size;
-static unsigned xfs_direct_write_chunk_size;
-
-void ADIOI_XFS_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code)
-{
-    char *value, * c;
-    int flag;
-    static char xfs_initialized = 0;
-
-    if (fd->info == MPI_INFO_NULL) MPI_Info_create(&(fd->info));
-
-    ADIOI_Info_set(fd->info, "direct_read", "false");
-    ADIOI_Info_set(fd->info, "direct_write", "false");
-    fd->direct_read = fd->direct_write = 0;
-
-	if (!xfs_initialized) {
-		xfs_initialized = 1;
-		c = getenv("MPIO_DIRECT_READ_CHUNK_SIZE");
-		if (c) {
-			int io;
-			io = atoi(c);
-			if (io <= 0) {
-				fprintf(stderr,
-"MPI: Ignoring an invalid setting for MPIO_DIRECT_READ_CHUNK_SIZE.\n"
-"     It must be set to a positive integer value.\n");
-			} else {
-				xfs_direct_read_chunk_size = io;
-			}
-		} else {
-			xfs_direct_read_chunk_size = 0;
-		}
-
-		c = getenv("MPIO_DIRECT_WRITE_CHUNK_SIZE");
-		if (c) {
-			int io;
-			io = atoi(c);
-			if (io <= 0) {
-				fprintf(stderr,
-"MPI: Ignoring an invalid setting for MPIO_DIRECT_WRITE_CHUNK_SIZE.\n"
-"     It must be set to a positive integer value.\n");
-			} else {
-				xfs_direct_write_chunk_size = io;
-			}
-		} else {
-			xfs_direct_write_chunk_size = 0;
-		}
-	}
-
-	if (!fd->hints->initialized) {
-		fd->hints->fs_hints.xfs.read_chunk_sz =
-			xfs_direct_read_chunk_size;
-		fd->hints->fs_hints.xfs.write_chunk_sz =
-			xfs_direct_write_chunk_size;
-	}
-
-    /* has user specified values for keys "direct_read" and "direct write"? */
-    if (users_info != MPI_INFO_NULL) {
-	value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
-
-	ADIOI_Info_get(users_info, "direct_read", MPI_MAX_INFO_VAL, 
-			 value, &flag);
-	if (flag && !strcmp(value, "true")) {
-	    ADIOI_Info_set(fd->info, "direct_read", "true");
-	    fd->direct_read = 1;
-	}
-
-	ADIOI_Info_get(users_info, "direct_write", MPI_MAX_INFO_VAL, 
-			 value, &flag);
-	if (flag && !strcmp(value, "true")) {
-	    ADIOI_Info_set(fd->info, "direct_write", "true");
-	    fd->direct_write = 1;
-	}
-
-	ADIOI_Free(value);
-    }
-    
-    /* set the values for collective I/O and data sieving parameters */
-    ADIOI_GEN_SetInfo(fd, users_info, error_code);
-
-    /* Environment variables override MPI_Info hints */
-    if (ADIOI_Direct_read) fd->direct_read = 1;
-    if (ADIOI_Direct_write) fd->direct_write = 1;
-
-    /* environment variables checked in ADIO_Init */
-
-    *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_xfs/ad_xfs_open.c b/3rd-party/romio321/adio/ad_xfs/ad_xfs_open.c
deleted file mode 100644
index 5c0a9a2b8ae..00000000000
--- a/3rd-party/romio321/adio/ad_xfs/ad_xfs_open.c
+++ /dev/null
@@ -1,111 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#define _GNU_SOURCE          // for O_DIRECT
-
-#include "ad_xfs.h"
-#include <sys/ioctl.h>
-#ifdef HAVE_STDDEF_H
-#include <stddef.h>
-#endif
-
-#ifndef HAVE_LSEEK64
-#define lseek64 lseek
-#endif
-
-void ADIOI_XFS_Open(ADIO_File fd, int *error_code)
-{
-    int perm, amode, amode_direct, factor;
-    unsigned int old_mask;
-    struct dioattr st;
-    static char myname[] = "ADIOI_XFS_OPEN";
-    unsigned read_chunk_sz = fd->hints->fs_hints.xfs.read_chunk_sz;
-    unsigned write_chunk_sz = fd->hints->fs_hints.xfs.write_chunk_sz;
-
-    if (fd->perm == ADIO_PERM_NULL) {
-	old_mask = umask(022);
-	umask(old_mask);
-	perm = old_mask ^ 0666;
-    }
-    else perm = fd->perm;
-
-    amode = 0;
-    if (fd->access_mode & ADIO_CREATE)
-	amode = amode | O_CREAT;
-    if (fd->access_mode & ADIO_RDONLY)
-	amode = amode | O_RDONLY;
-    if (fd->access_mode & ADIO_WRONLY)
-	amode = amode | O_WRONLY;
-    if (fd->access_mode & ADIO_RDWR)
-	amode = amode | O_RDWR;
-
-    amode_direct = amode | O_DIRECT;
-
-    if (fd->access_mode & ADIO_EXCL)
-	amode = amode | O_EXCL;
-
-    fd->fd_sys = open(fd->filename, amode, perm);
-
-    fd->fd_direct = open(fd->filename, amode_direct, perm);
-    if (fd->fd_direct != -1) {
-
-#if defined(MPISGI)
-	ioctl(fd->fd_direct, XFS_IOC_DIOINFO, &st);
-#else
-	fcntl(fd->fd_direct, F_DIOINFO, &st);
-#endif
-
-	fd->d_mem = st.d_mem;
-	fd->d_miniosz = st.d_miniosz;
-
-	if (read_chunk_sz == 0) {
-		fd->hints->fs_hints.xfs.read_chunk_sz = st.d_maxiosz;
-	} else {
-		/*
-		 * MPIO_DIRECT_READ_CHUNK_SIZE was set.
-		 * Make read_chunk_sz a multiple of d_miniosz.
-		 */
-		factor = read_chunk_sz / fd->d_miniosz;
-		if (factor == 0 || read_chunk_sz != fd->d_miniosz * factor) {
-			fd->hints->fs_hints.xfs.read_chunk_sz =
-				fd->d_miniosz * (factor + 1);
-		}
-	}
-
-	if (write_chunk_sz == 0) {
-		fd->hints->fs_hints.xfs.write_chunk_sz = st.d_maxiosz;
-	} else {
-		/*
-		 * MPIO_DIRECT_WRITE_CHUNK_SIZE was set. 
-		 * Make write_chunk_sz a multiple of d_miniosz.
-		 */
-		factor = write_chunk_sz / fd->d_miniosz;
-		if (factor == 0 || write_chunk_sz != fd->d_miniosz * factor) {
-			fd->hints->fs_hints.xfs.write_chunk_sz =
-				fd->d_miniosz * (factor + 1);
-		}
-	}
-
-	if (fd->d_mem > XFS_MEMALIGN) {
-	    FPRINTF(stderr, "MPI: Run-time Direct-IO memory alignment, %d, does not match compile-time value, %d.\n",
-		fd->d_mem, XFS_MEMALIGN);
-	    FPRINTF(stderr, "MPI: Report this error and rerun with Direct-IO disabled.\n");
-	    close(fd->fd_direct);
-	    fd->fd_direct = -1;
-	}
-    }
-
-    if ((fd->fd_sys != -1) && (fd->access_mode & ADIO_APPEND))
-	fd->fp_ind = lseek64(fd->fd_sys, 0, SEEK_END);
-
-    fd->fp_sys_posn = -1; /* set it to null because we use pread/pwrite */
-
-    if ((fd->fd_sys == -1) || (fd->fd_direct == -1)) {
-	*error_code = ADIOI_Err_create_code(mymame, fd->filename, errno);
-    }
-    else *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_xfs/ad_xfs_read.c b/3rd-party/romio321/adio/ad_xfs/ad_xfs_read.c
deleted file mode 100644
index c3c237cc15d..00000000000
--- a/3rd-party/romio321/adio/ad_xfs/ad_xfs_read.c
+++ /dev/null
@@ -1,152 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_xfs.h"
-#ifdef HAVE_MALLOC_H
-#include <malloc.h>
-#endif
-
-/* style: allow:free:2 sig:0 */
-
-static void ADIOI_XFS_Aligned_Mem_File_Read(ADIO_File fd, void *buf, int len, 
-					     ADIO_Offset offset, int *err);
-
-void ADIOI_XFS_ReadContig(ADIO_File fd, void *buf, int count, 
-                     MPI_Datatype datatype, int file_ptr_type,
-		     ADIO_Offset offset, ADIO_Status *status, int *error_code)
-{
-    int diff, size, nbytes;
-    MPI_Count err=-1, datatype_size, len;
-    void *newbuf;
-    static char myname[] = "ADIOI_XFS_READCONTIG";
-
-    MPI_Type_size_x(datatype, &datatype_size);
-    len = datatype_size * count;
-
-    fd->fp_sys_posn = -1; /* set it to null, since we are using pread */
-
-    if (file_ptr_type == ADIO_INDIVIDUAL) offset = fd->fp_ind;
-
-    if (!(fd->direct_read))     /* direct I/O not enabled */
-	err = pread(fd->fd_sys, buf, len, offset);
-    else {       /* direct I/O enabled */
-
-	/* (1) if mem_aligned && file_aligned 
-                    use direct I/O to read up to correct io_size
-                    use buffered I/O for remaining  */
-
-	if (!(((long) buf) % fd->d_mem) && !(offset % fd->d_miniosz)) 
-	    ADIOI_XFS_Aligned_Mem_File_Read(fd, buf, len, offset, &err);
-
-        /* (2) if !file_aligned
-                    use buffered I/O to read up to file_aligned
-                    At that point, if still mem_aligned, use (1)
-   		        else copy into aligned buf and then use (1) */
-	else if (offset % fd->d_miniosz) {
-	    diff = fd->d_miniosz - (offset % fd->d_miniosz);
-	    diff = ADIOI_MIN(diff, len);
-	    nbytes = pread(fd->fd_sys, buf, diff, offset);
-
-	    buf = ((char *) buf) + diff;
-	    offset += diff;
-	    size = len - diff;
-	    if (!(((long) buf) % fd->d_mem)) {
-		ADIOI_XFS_Aligned_Mem_File_Read(fd, buf, size, offset, &err);
-		nbytes += err;
-	    }
-	    else {
-		newbuf = (void *) memalign(XFS_MEMALIGN, size);
-		if (newbuf) {
-		    ADIOI_XFS_Aligned_Mem_File_Read(fd, newbuf, size, offset, &err);
-		    if (err > 0) memcpy(buf, newbuf, err);
-		    nbytes += err;
-		    ADIOI_Free(newbuf);
-		}
-		else nbytes += pread(fd->fd_sys, buf, size, offset);
-	    }
-	    err = nbytes;
-	}
-
-        /* (3) if !mem_aligned && file_aligned
-    	            copy into aligned buf, then use (1)  */
-	else {
-	    newbuf = (void *) memalign(XFS_MEMALIGN, len);
-	    if (newbuf) {
-		ADIOI_XFS_Aligned_Mem_File_Read(fd, newbuf, len, offset, &err);
-		if (err > 0) memcpy(buf, newbuf, err);
-		ADIOI_Free(newbuf);
-	    }
-	    else err = pread(fd->fd_sys, buf, len, offset);
-	}
-    }
-
-    if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind += err;
-
-#ifdef HAVE_STATUS_SET_BYTES
-    if (err != -1) MPIR_Status_set_bytes(status, datatype, err);
-#endif
-
-    if (err == -1) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO, "**io",
-					   "**io %s", strerror(errno));
-    }
-    else *error_code = MPI_SUCCESS;
-}
-
-
-void ADIOI_XFS_Aligned_Mem_File_Read(ADIO_File fd, void *buf, int len, 
-              ADIO_Offset offset, int *err)
-{
-    int ntimes, rem, newrem, i, size, nbytes;
-    unsigned read_chunk_sz = fd->hints->fs_hints.xfs.read_chunk_sz;
-
-    /* memory buffer is aligned, offset in file is aligned,
-       io_size may or may not be of the right size.
-       use direct I/O to read up to correct io_size,
-       use buffered I/O for remaining. */
-
-    if (!(len % fd->d_miniosz) && 
-	(len >= fd->d_miniosz) && (len <= read_chunk_sz))
-	*err = pread(fd->fd_direct, buf, len, offset);
-    else if (len < fd->d_miniosz)
-	*err = pread(fd->fd_sys, buf, len, offset);
-    else if (len > read_chunk_sz) {
-	ntimes = len/(read_chunk_sz);
-	rem = len - ntimes * read_chunk_sz;
-	nbytes = 0;
-	for (i=0; i<ntimes; i++) {
-	    nbytes += pread(fd->fd_direct, ((char *)buf) + i * read_chunk_sz,
-			 read_chunk_sz, offset);
-	    offset += read_chunk_sz;
-	}
-	if (rem) {
-	    if (!(rem % fd->d_miniosz))
-		nbytes += pread(fd->fd_direct, 
-		     ((char *)buf) + ntimes * read_chunk_sz, rem, offset);
-	    else {
-		newrem = rem % fd->d_miniosz;
-		size = rem - newrem;
-		if (size) {
-		    nbytes += pread(fd->fd_direct, 
-		         ((char *)buf) + ntimes * read_chunk_sz, size, offset);
-		    offset += size;
-		}
-		nbytes += pread(fd->fd_sys, 
-	              ((char *)buf) + ntimes * read_chunk_sz + size, newrem, offset);
-	    }
-	}
-	*err = nbytes;
-    }
-    else {
-	rem = len % fd->d_miniosz;
-	size = len - rem;
-	nbytes = pread(fd->fd_direct, buf, size, offset);
-	nbytes += pread(fd->fd_sys, (char *)buf + size, rem, offset+size);
-	*err = nbytes;
-    }
-}
diff --git a/3rd-party/romio321/adio/ad_xfs/ad_xfs_resize.c b/3rd-party/romio321/adio/ad_xfs/ad_xfs_resize.c
deleted file mode 100644
index 8caf8b48bd7..00000000000
--- a/3rd-party/romio321/adio/ad_xfs/ad_xfs_resize.c
+++ /dev/null
@@ -1,22 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_xfs.h"
-
-void ADIOI_XFS_Resize(ADIO_File fd, ADIO_Offset size, int *error_code)
-{
-    int err;
-    static char myname[] = "ADIOI_XFS_RESIZE";
-    
-    err = ftruncate64(fd->fd_sys, size);
-    if (err == -1) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO, "**io",
-					   "**io %s", strerror(errno));
-    }
-    else *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/ad_xfs/ad_xfs_write.c b/3rd-party/romio321/adio/ad_xfs/ad_xfs_write.c
deleted file mode 100644
index 9ab82768f76..00000000000
--- a/3rd-party/romio321/adio/ad_xfs/ad_xfs_write.c
+++ /dev/null
@@ -1,171 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_xfs.h"
-
-#ifdef HAVE_MALLOC_H
-#include <malloc.h>
-#endif
-
-/* style: allow:free:2 sig:0 */
-
-static int ADIOI_XFS_Aligned_Mem_File_Write(ADIO_File fd, void *buf,
-						  ADIO_Offset len, ADIO_Offset offset);
-
-void ADIOI_XFS_WriteContig(ADIO_File fd, void *buf, int count, 
-                     MPI_Datatype datatype, int file_ptr_type,
-		     ADIO_Offset offset, ADIO_Status *status, int *error_code)
-{
-    int diff, size;
-    MPI_Count err=-1, datatype_size;
-    ssize_t len;
-    void *newbuf;
-    static char myname[] = "ADIOI_XFS_WRITECONTIG";
-
-    MPI_Type_size_x(datatype, &datatype_size);
-    len = datatype_size * count;
-
-    fd->fp_sys_posn = -1; /* set it to null, since we are using pwrite */
-
-    if (file_ptr_type == ADIO_INDIVIDUAL) offset = fd->fp_ind;
-
-    if (!(fd->direct_write)) {    /* direct I/O not enabled */
-	err = pwrite(fd->fd_sys, buf, len, offset);
-	if (err < 0) {goto leaving;}
-    } else {       /* direct I/O enabled */
-
-	/* (1) if mem_aligned && file_aligned 
-                    use direct I/O to write up to correct io_size
-                    use buffered I/O for remaining  */
-
-	if (!(((long) buf) % fd->d_mem) && !(offset % fd->d_miniosz)) {
-	    err = ADIOI_XFS_Aligned_Mem_File_Write(fd, buf, len, offset);
-	    if (err < 0) {goto leaving;}
-
-        /* (2) if !file_aligned
-                    use buffered I/O to write up to file_aligned
-                    At that point, if still mem_aligned, use (1)
-   		        else copy into aligned buf and then use (1) */
-	} else if (offset % fd->d_miniosz) {
-	    diff = fd->d_miniosz - (offset % fd->d_miniosz);
-	    diff = ADIOI_MIN(diff, len);
-	    err = pwrite(fd->fd_sys, buf, diff, offset);
-	    if (err < 0) {goto leaving;}
-
-	    buf = ((char *) buf) + diff;
-	    offset += diff;
-	    size = len - diff;
-	    if (!(((long) buf) % fd->d_mem)) {
-		err = ADIOI_XFS_Aligned_Mem_File_Write(fd, buf, size, offset);
-		if (err < 0) {goto leaving;}
-	    }
-	    else {
-		newbuf = (void *) memalign(XFS_MEMALIGN, size);
-		if (newbuf) {
-		    memcpy(newbuf, buf, size);
-		    err = ADIOI_XFS_Aligned_Mem_File_Write(fd, newbuf, size, offset);
-		    ADIOI_Free(newbuf);
-		    if (err < 0) {goto leaving;}
-		} else {
-		    err = pwrite(fd->fd_sys, buf, size, offset);
-		    if (err < 0) {goto leaving;}
-		}
-	    }
-	}
-
-        /* (3) if !mem_aligned && file_aligned
-    	            copy into aligned buf, then use (1)  */
-	else {
-	    newbuf = (void *) memalign(XFS_MEMALIGN, len);
-	    if (newbuf) {
-		memcpy(newbuf, buf, len);
-		err = ADIOI_XFS_Aligned_Mem_File_Write(fd, newbuf, len, offset);
-		ADIOI_Free(newbuf);
-	    } else {
-		 err = pwrite(fd->fd_sys, buf, len, offset);
-	    }
-
-	    if (err < 0) {goto leaving;}
-	}
-    }
-
-    if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind += len;
-
-#ifdef HAVE_STATUS_SET_BYTES
-    if (err != -1) MPIR_Status_set_bytes(status, datatype, len);
-#endif
-leaving:
-    if (err == -1) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO, "**io",
-					   "**io %s", strerror(errno));
-    }
-    else *error_code = MPI_SUCCESS;
-}
-
-
-static int
-ADIOI_XFS_Aligned_Mem_File_Write(ADIO_File fd, void *buf, ADIO_Offset len, 
-              ADIO_Offset offset)
-{
-    unsigned write_chunk_sz = fd->hints->fs_hints.xfs.write_chunk_sz;
-    ADIO_Offset nbytes, rem, newrem, size;
-    int ntimes, i;
-
-    /* memory buffer is aligned, offset in file is aligned,
-       io_size may or may not be of the right size.
-       use direct I/O to write up to correct io_size,
-       use buffered I/O for remaining. */
-
-    if (!(len % fd->d_miniosz) && 
-	 (len >= fd->d_miniosz) && (len <= write_chunk_sz)) {
-	nbytes = pwrite(fd->fd_direct, buf, len, offset);
-	if (nbytes < 0) {return -1;}
-    } else if (len < fd->d_miniosz) {
-	nbytes = pwrite(fd->fd_sys, buf, len, offset);
-	if (nbytes < 0) {return -1;}
-    } else if (len > write_chunk_sz) {
-	ntimes = len/(write_chunk_sz);
-	rem = len - ntimes * write_chunk_sz;
-	nbytes = 0;
-	for (i=0; i<ntimes; i++) {
-	    nbytes = pwrite(fd->fd_direct, ((char *)buf) + i * write_chunk_sz,
-			 write_chunk_sz, offset);
-	    offset += write_chunk_sz;
-	    if (nbytes < 0) {return -1;}
-	}
-	if (rem) {
-	    if (!(rem % fd->d_miniosz)) {
-		nbytes = pwrite(fd->fd_direct, 
-		             ((char *)buf) + ntimes * write_chunk_sz, rem, offset);
-		if (nbytes < 0) {return -1;}
-	    } else {
-		newrem = rem % fd->d_miniosz;
-		size = rem - newrem;
-		if (size) {
-		    nbytes = pwrite(fd->fd_direct, 
-		            ((char *)buf) + ntimes * write_chunk_sz, size, offset);
-		    offset += size;
-		    if (nbytes < 0) {return -1;}
-		}
-		nbytes = pwrite(fd->fd_sys, 
-	              ((char *)buf) + ntimes * write_chunk_sz + size, newrem, offset);
-		if (nbytes < 0) {return -1;}
-	    }
-	}
-    }
-    else {
-	rem = len % fd->d_miniosz;
-	size = len - rem;
-	nbytes = pwrite(fd->fd_direct, buf, size, offset);
-	if (nbytes < 0) {return -1;}
-	nbytes = pwrite(fd->fd_sys, (char *)buf + size, rem, offset+size);
-	if (nbytes < 0) {return -1;}
-    }
-
-    return 0;
-}
diff --git a/3rd-party/romio321/adio/ad_zoidfs/Makefile.mk b/3rd-party/romio321/adio/ad_zoidfs/Makefile.mk
deleted file mode 100644
index a26e59c8e0e..00000000000
--- a/3rd-party/romio321/adio/ad_zoidfs/Makefile.mk
+++ /dev/null
@@ -1,27 +0,0 @@
-## -*- Mode: Makefile; -*-
-## vim: set ft=automake :
-##
-## (C) 2011 by Argonne National Laboratory.
-##     See COPYRIGHT in top-level directory.
-##
-
-if BUILD_AD_ZOIDFS
-
-noinst_HEADERS += adio/ad_zoidfs/ad_zoidfs.h adio/ad_zoidfs/ad_zoidfs_common.h
-
-romio_other_sources +=                    \
-    adio/ad_zoidfs/ad_zoidfs.c            \
-    adio/ad_zoidfs/ad_zoidfs_close.c      \
-    adio/ad_zoidfs/ad_zoidfs_common.c     \
-    adio/ad_zoidfs/ad_zoidfs_delete.c     \
-    adio/ad_zoidfs/ad_zoidfs_fcntl.c      \
-    adio/ad_zoidfs/ad_zoidfs_flush.c      \
-    adio/ad_zoidfs/ad_zoidfs_io.c         \
-    adio/ad_zoidfs/ad_zoidfs_open.c       \
-    adio/ad_zoidfs/ad_zoidfs_resize.c     \
-    adio/ad_zoidfs/ad_zoidfs_features.c   \
-    adio/ad_zoidfs/ad_zoidfs_read_list.c  \
-    adio/ad_zoidfs/ad_zoidfs_write_list.c
-
-endif BUILD_AD_ZOIDFS
-
diff --git a/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs.c b/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs.c
deleted file mode 100644
index 3c9131aeac4..00000000000
--- a/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs.c
+++ /dev/null
@@ -1,44 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2003 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_zoidfs.h"
-
-/* adioi.h has the ADIOI_Fns_struct define */
-#include "adioi.h"
-
-struct ADIOI_Fns_struct ADIO_ZOIDFS_operations = {
-    ADIOI_ZOIDFS_Open, /* Open */
-    ADIOI_SCALEABLE_OpenColl, /* OpenColl */
-    ADIOI_ZOIDFS_ReadContig, /* ReadContig */
-    ADIOI_ZOIDFS_WriteContig, /* WriteContig */
-    ADIOI_GEN_ReadStridedColl, /* ReadStridedColl */
-    ADIOI_GEN_WriteStridedColl, /* WriteStridedColl */
-    ADIOI_GEN_SeekIndividual, /* SeekIndividual */
-    ADIOI_ZOIDFS_Fcntl, /* Fcntl */
-    ADIOI_GEN_SetInfo, /* SetInfo */
-    ADIOI_GEN_ReadStrided, /* ReadStrided */
-    ADIOI_ZOIDFS_WriteStrided, /* WriteStrided */
-    ADIOI_ZOIDFS_Close, /* Close */
-    ADIOI_FAKE_IreadContig, /* IreadContig */
-    ADIOI_FAKE_IwriteContig, /* IwriteContig */
-    ADIOI_FAKE_IODone, /* ReadDone */
-    ADIOI_FAKE_IODone, /* WriteDone */
-    ADIOI_FAKE_IOComplete, /* ReadComplete */
-    ADIOI_FAKE_IOComplete, /* WriteComplete */
-    ADIOI_FAKE_IreadStrided, /* IreadStrided */
-    ADIOI_FAKE_IwriteStrided, /* IwriteStrided */
-    ADIOI_ZOIDFS_Flush, /* Flush */
-    ADIOI_ZOIDFS_Resize, /* Resize */
-    ADIOI_ZOIDFS_Delete, /* Delete */
-    ADIOI_ZOIDFS_Feature,
-    ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
-    ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
-};
-
-/* 
- * vim: ts=8 sts=4 sw=4 noexpandtab 
- */
diff --git a/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs.h b/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs.h
deleted file mode 100644
index d4999b66228..00000000000
--- a/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#ifndef AD_ZOIDFS_INCLUDE
-#define AD_ZOIDFS_INCLUDE
-
-#include "adio.h"
-#ifdef HAVE_ZOIDFS_H
-#include "zoidfs.h"
-#endif
-
-
-typedef zoidfs_handle_t ADIOI_ZOIDFS_object;
-
-void ADIOI_ZOIDFS_Open(ADIO_File fd, int *error_code);
-void ADIOI_ZOIDFS_Close(ADIO_File fd, int *error_code);
-void ADIOI_ZOIDFS_ReadContig(ADIO_File fd, void *buf, int count, 
-                      MPI_Datatype datatype, int file_ptr_type,
-                     ADIO_Offset offset, ADIO_Status *status, int
-		     *error_code);
-void ADIOI_ZOIDFS_WriteContig(ADIO_File fd, void *buf, int count, 
-                      MPI_Datatype datatype, int file_ptr_type,
-                      ADIO_Offset offset, ADIO_Status *status, int
-		      *error_code);   
-void ADIOI_ZOIDFS_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct, int
-		*error_code); 
-void ADIOI_ZOIDFS_WriteStrided(ADIO_File fd, void *buf, int count,
-		       MPI_Datatype datatype, int file_ptr_type,
-		       ADIO_Offset offset, ADIO_Status *status, int
-		       *error_code);
-void ADIOI_ZOIDFS_ReadStrided(ADIO_File fd, void *buf, int count,
-		       MPI_Datatype datatype, int file_ptr_type,
-		       ADIO_Offset offset, ADIO_Status *status, int
-		       *error_code);
-void ADIOI_ZOIDFS_Flush(ADIO_File fd, int *error_code);
-void ADIOI_ZOIDFS_Delete(char *filename, int *error_code);
-void ADIOI_ZOIDFS_Resize(ADIO_File fd, ADIO_Offset size, int *error_code);
-void ADIOI_ZOIDFS_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code);
-int  ADIOI_ZOIDFS_Feature(ADIO_File fd, int flag);
-#endif
diff --git a/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_close.c b/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_close.c
deleted file mode 100644
index 0126783aa37..00000000000
--- a/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_close.c
+++ /dev/null
@@ -1,25 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_zoidfs.h"
-
-void ADIOI_ZOIDFS_Close(ADIO_File fd, int *error_code)
-{
-    ADIOI_Free(fd->fs_ptr);
-    fd->fs_ptr = NULL;
-
-    /* At some point or another it was decided that ROMIO would not
-     * explicitly flush (other than any local cache) on close, because
-     * there is no way to *avoid* that overhead if you implement it here
-     * and don't actually want it.
-     */
-
-    *error_code = MPI_SUCCESS;
-}
-/* 
- * vim: ts=8 sts=4 sw=4 noexpandtab 
- */
diff --git a/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_common.c b/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_common.c
deleted file mode 100644
index 018d439aaa2..00000000000
--- a/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_common.c
+++ /dev/null
@@ -1,126 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 2003 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_zoidfs.h"
-#include "ad_zoidfs_common.h"
-#include <unistd.h>
-#include <sys/types.h>
-
-/* keyval hack to both tell us if we've already initialized zoidfs and also
- * close it down when mpi exits */
-int ADIOI_ZOIDFS_Initialized = MPI_KEYVAL_INVALID;
-
-void ADIOI_ZOIDFS_End(int *error_code)
-{
-    int ret;
-    static char myname[] = "ADIOI_ZOIDFS_END";
-
-    ret = zoidfs_finalize();
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (ret != 0 ) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   ADIOI_ZOIDFS_error_convert(ret),
-					   "Error in zoidfs_finalize", 0);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-    *error_code = MPI_SUCCESS;
-}
-
-int ADIOI_ZOIDFS_End_call(MPI_Comm comm, int keyval, 
-			 void *attribute_val, void *extra_state)
-{
-    int error_code;
-    ADIOI_ZOIDFS_End(&error_code);
-    MPI_Keyval_free(&keyval);
-    return error_code;
-}
-
-void ADIOI_ZOIDFS_Init(int rank, int *error_code )
-{
-    int ret;
-    static char myname[] = "ADIOI_ZOIDFS_INIT";
-
-    /* do nothing if we've already fired up the zoidfs interface */
-    if (ADIOI_ZOIDFS_Initialized != MPI_KEYVAL_INVALID) {
-	*error_code = MPI_SUCCESS;
-	return;
-    }
-
-    ret = zoidfs_init();
-    if (ret < 0 ) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   ADIOI_ZOIDFS_error_convert(ret),
-					   "Error in zoidfs_init",
-					   0);
-	return;
-    }
-    
-    MPI_Keyval_create(MPI_NULL_COPY_FN, ADIOI_ZOIDFS_End_call,
-		      &ADIOI_ZOIDFS_Initialized, (void *)0); 
-    /* just like romio does, we make a dummy attribute so we 
-     * get cleaned up */
-    MPI_Attr_put(MPI_COMM_SELF, ADIOI_ZOIDFS_Initialized, (void *)0);
-}
-
-void ADIOI_ZOIDFS_makeattribs(zoidfs_sattr_t * attribs)
-{
-    memset(attribs, 0, sizeof(zoidfs_sattr_t));
-
-    attribs->mask = ZOIDFS_ATTR_MODE;
-    attribs->mode = 0644;
-}
-
-int ADIOI_ZOIDFS_error_convert(int error)
-{
-    switch (error)
-    {
-	case ZFSERR_PERM: /* ??? */
-	case ZFSERR_ACCES:
-	    return MPI_ERR_ACCESS;
-	case ZFSERR_NOENT:
-	case ZFSERR_NXIO: /* ??? */
-	case ZFSERR_NODEV: /* ??? */
-	    return MPI_ERR_NO_SUCH_FILE;
-	case ZFSERR_IO:
-	    return MPI_ERR_IO;
-	case ZFSERR_EXIST:
-	    return MPI_ERR_FILE_EXISTS;
-	case ZFSERR_NOTDIR: /* ??? */
-	case ZFSERR_ISDIR: /* ??? */
-	case ZFSERR_NAMETOOLONG:
-	    return MPI_ERR_BAD_FILE;
-	case ZFSERR_INVAL:
-	case ZFSERR_STALE:
-	    return MPI_ERR_FILE;
-	case ZFSERR_FBIG: /* ??? */
-	case ZFSERR_NOSPC:
-	    return MPI_ERR_NO_SPACE;
-	case ZFSERR_ROFS:
-	    return MPI_ERR_READ_ONLY;
-	case ZFSERR_NOTIMPL:
-	    return MPI_ERR_UNSUPPORTED_OPERATION;
-	case ZFSERR_DQUOT:
-	    return MPI_ERR_QUOTA;
-	/* case ZFSERR_NOTEMPTY: */
-	/* case ZFSERR_WFLUSH: */
-	/* case ZFSERR_OTHER: */
-	case ZFSERR_NOMEM:
-	    return MPI_ERR_INTERN;
-	default:
-	    return MPI_UNDEFINED;
-    }
-}
-
-/* 
- * vim: ts=8 sts=4 sw=4 noexpandtab 
- */
diff --git a/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_common.h b/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_common.h
deleted file mode 100644
index ba985b49226..00000000000
--- a/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_common.h
+++ /dev/null
@@ -1,43 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*-
- * vim: ts=8 sts=4 sw=4 noexpandtab
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#ifndef _AD_ZOIDFS_COMMON_H
-#define _AD_ZOIDFS_COMMON_H
-#include "ad_zoidfs.h"
-
-/* The ESTALE problem: 
- * The IO forwarding protocol can respond to any call with ESTALE, which means
- * the handle upon which that call operates has expired from the metadata
- * cache.  We thus wrap any zoidfs routine (expr) in this macro.  
- *
- * ROMIO stores the filename in the ADIOI_File structrue (fd), so we can always
- * re-lookup in response to ESTALE */
-
-#define NO_STALE(ret, fd, handle_p, expr)                               \
-    do {                                                                \
-        (ret) = (expr);                                                 \
-        while ((ret) == ZFSERR_STALE) {                                 \
-            /* lookup again */                                          \
-            (ret) = zoidfs_lookup(NULL, NULL, (fd)->filename,           \
-                                  (zoidfs_handle_t*)((fd)->fs_ptr), ZOIDFS_NO_OP_HINT);    \
-            if ((ret) == ZFS_OK) {                                      \
-                *((ADIOI_ZOIDFS_object*)handle_p)                       \
-                    = *((ADIOI_ZOIDFS_object*)((fd)->fs_ptr));          \
-                /* re-execute the expr with new handle */               \
-                (ret) = (expr);                                         \
-            }                                                           \
-        }                                                               \
-    } while (0)
-
-void ADIOI_ZOIDFS_Init(int rank, int *error_code );
-void ADIOI_ZOIDFS_makeattribs(zoidfs_sattr_t * attribs);
-void ADIOI_ZOIDFS_End(int *error_code);
-int ADIOI_ZOIDFS_End_call(MPI_Comm comm, int keyval, 
-	void *attribute_val, void *extra_state);
-int ADIOI_ZOIDFS_error_convert(int error);
-
-#endif
diff --git a/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_delete.c b/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_delete.c
deleted file mode 100644
index 87193147a56..00000000000
--- a/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_delete.c
+++ /dev/null
@@ -1,45 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2003 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_zoidfs.h"
-#include "adio.h"
-
-#include "ad_zoidfs_common.h"
-
-void ADIOI_ZOIDFS_Delete(char *filename, int *error_code)
-{
-    int ret;
-    static char myname[] = "ADIOI_ZOIDFS_DELETE";
-
-    ADIOI_ZOIDFS_Init(0, error_code);
-    /* --BEGIN ERROR HANDLING-- */
-    if (*error_code != MPI_SUCCESS) 
-    {
-	/* ADIOI_ZOIDFS_INIT handles creating error codes itself */
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-    ret = zoidfs_remove(NULL, NULL, filename, NULL, ZOIDFS_NO_OP_HINT);
-    /* --BEGIN ERROR HANDLING-- */
-    if (ret != ZFS_OK) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   ADIOI_ZOIDFS_error_convert(ret),
-					   "Error in zoidfs_remove", 0);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-    *error_code = MPI_SUCCESS;
-    return;
-}
-
-/* 
- * vim: ts=8 sts=4 sw=4 noexpandtab 
- */
diff --git a/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_fcntl.c b/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_fcntl.c
deleted file mode 100644
index 22c26714d80..00000000000
--- a/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_fcntl.c
+++ /dev/null
@@ -1,60 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_zoidfs.h"
-#include "adio_extern.h"
-#include "ad_zoidfs_common.h"
-
-void ADIOI_ZOIDFS_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct,
-		       int *error_code)
-{
-    int ret;
-    zoidfs_attr_t attr;
-    ADIOI_ZOIDFS_object *zoidfs_obj_ptr;
-    static char myname[] = "ADIOI_ZOIDFS_FCNTL";
-
-    zoidfs_obj_ptr = (ADIOI_ZOIDFS_object*)fd->fs_ptr;
-
-    switch(flag) {
-    case ADIO_FCNTL_GET_FSIZE:
-	attr.mask = ZOIDFS_ATTR_SIZE;
-        NO_STALE(ret, fd, zoidfs_obj_ptr,
-                 zoidfs_getattr(zoidfs_obj_ptr, &attr, ZOIDFS_NO_OP_HINT));
-	if ( !(attr.mask & ZOIDFS_ATTR_SIZE) || (ret != ZFS_OK ) ) {
-	    /* --BEGIN ERROR HANDLING-- */
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE,
-					       myname, __LINE__,
-					       ADIOI_ZOIDFS_error_convert(ret),
-					       "Error in zoidfs_getattr", 0);
-	    /* --END ERROR HANDLING-- */
-	}
-	else {
-	    *error_code = MPI_SUCCESS;
-	}
-	fcntl_struct->fsize = attr.size;
-	return;
-
-    case ADIO_FCNTL_SET_DISKSPACE:
-	ADIOI_GEN_Prealloc(fd, fcntl_struct->diskspace, error_code);
-	break;
-
-    /* --BEGIN ERROR HANDLING-- */
-    case ADIO_FCNTL_SET_ATOMICITY:
-    default:
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   MPI_ERR_ARG,
-					   "**flag", "**flag %d", flag);
-    /* --END ERROR HANDLING-- */
-    }
-}
-
-/* 
- * vim: ts=8 sts=4 sw=4 noexpandtab 
- */
diff --git a/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_features.c b/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_features.c
deleted file mode 100644
index a3f5c1467e4..00000000000
--- a/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_features.c
+++ /dev/null
@@ -1,22 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *
- *  (C) 2008 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include "adio.h"
-#include "ad_zoidfs.h"
-
-int ADIOI_ZOIDFS_Feature(ADIO_File fd, int flag)
-{
-	switch(flag) {
-		case ADIO_SCALABLE_OPEN:
-			return 1;
-		case ADIO_SHARED_FP:
-		case ADIO_LOCKS:
-		case ADIO_SEQUENTIAL:
-		case ADIO_DATA_SIEVING_WRITES:
-		default:
-			return 0;
-	}
-}
diff --git a/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_flush.c b/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_flush.c
deleted file mode 100644
index 8ec0b8d5518..00000000000
--- a/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_flush.c
+++ /dev/null
@@ -1,52 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_zoidfs.h"
-#include "ad_zoidfs_common.h"
-
-/* we want to be a bit clever here:  at scale, if every client sends a
- * flush request, it will stress the file system with redundant
- * commit requests.  Instead, one process should wait for
- * everyone to catch up, do the sync, then broadcast the result.  
- */
-
-void ADIOI_ZOIDFS_Flush(ADIO_File fd, int *error_code) 
-{ 
-    int ret, rank, dummy=0, dummy_in=0; 
-    ADIOI_ZOIDFS_object *zoidfs_obj_ptr;
-    static char myname[] = "ADIOI_ZOIDFS_FLUSH";
-
-    *error_code = MPI_SUCCESS;
-
-    zoidfs_obj_ptr = (ADIOI_ZOIDFS_object*)fd->fs_ptr;
-
-    MPI_Comm_rank(fd->comm, &rank);
-
-    /* collective call to ensure no outstanding write requests. reduce is
-     * slightly less expensvie than barrier */
-    MPI_Reduce(&dummy_in, &dummy, 1, MPI_INT, MPI_SUM, 
-	    fd->hints->ranklist[0], fd->comm);
-
-    if (rank == fd->hints->ranklist[0]) {
-	ret = zoidfs_commit(zoidfs_obj_ptr, ZOIDFS_NO_OP_HINT);
-    }
-    MPI_Bcast(&ret, 1, MPI_INT, fd->hints->ranklist[0], fd->comm);
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (ret != 0) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   ADIOI_ZOIDFS_error_convert(ret),
-					   "Error in zoidfs_commit", 0);
-    }
-    /* --END ERROR HANDLING-- */
-}
-
-/* 
- * vim: ts=8 sts=4 sw=4 noexpandtab 
- */
diff --git a/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_io.c b/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_io.c
deleted file mode 100644
index 337c92f4101..00000000000
--- a/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_io.c
+++ /dev/null
@@ -1,96 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- 
- * 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-#include "ad_zoidfs.h"
-
-#include "ad_zoidfs_common.h"
-
-#define ZOIDFS_READ 0
-#define ZOIDFS_WRITE 1
-
-static void ZOIDFS_IOContig(ADIO_File fd, void * buf, int count,
-	    MPI_Datatype datatype, int file_ptr_type,
-	    ADIO_Offset offset, ADIO_Status *status,
-	    int flag, int *error_code)
-{
-    int ret;
-    MPI_Count datatype_size;
-    uint64_t file_len;
-    size_t mem_len;
-    ADIOI_ZOIDFS_object *zoidfs_obj_ptr;
-    uint64_t file_offset = offset;
-    static char myname[] = "ADIOI_ZOIDFS_IOCONTIG";
-
-    zoidfs_obj_ptr = (ADIOI_ZOIDFS_object*)fd->fs_ptr;
-
-    MPI_Type_size_x(datatype, &datatype_size);
-    file_len = mem_len = datatype_size * count;
-
-    if (file_ptr_type == ADIO_INDIVIDUAL) {
-	file_offset = fd->fp_ind;
-    }
-
-    if (flag == ZOIDFS_READ) {
-	NO_STALE(ret, fd, zoidfs_obj_ptr,
-                 zoidfs_read(zoidfs_obj_ptr, 
-                             1, &buf, &mem_len,
-                             1, &file_offset, &file_len, ZOIDFS_NO_OP_HINT));
-    } else {
-        NO_STALE(ret, fd, zoidfs_obj_ptr,
-                 zoidfs_write(zoidfs_obj_ptr, 
-                              1, (const void **)&buf, &mem_len,
-                              1, &file_offset, &file_len, ZOIDFS_NO_OP_HINT));
-    }
-    /* --BEGIN ERROR HANDLING-- */
-    if (ret != ZFS_OK ) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   ADIOI_ZOIDFS_error_convert(ret),
-					   "Error in ZOIDFS I/O", 0);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    if (file_ptr_type == ADIO_INDIVIDUAL) {
-	fd->fp_ind += file_len;
-    }
-    fd->fp_sys_posn = file_offset + file_len;
-
-#ifdef HAVE_STATUS_SET_BYTES
-    MPIR_Status_set_bytes(status, datatype, file_len);
-#endif
-
-    *error_code = MPI_SUCCESS;
-
-fn_exit:
-    return;
-}
-
-void ADIOI_ZOIDFS_ReadContig(ADIO_File fd, void *buf, int count, 
-			    MPI_Datatype datatype, int file_ptr_type,
-			    ADIO_Offset offset, ADIO_Status *status,
-			    int *error_code)
-{
-    ZOIDFS_IOContig(fd, buf, count, datatype, file_ptr_type, 
-	    offset, status, ZOIDFS_READ, error_code);
-}
-
-void ADIOI_ZOIDFS_WriteContig(ADIO_File fd, void *buf, int count,
-			   MPI_Datatype datatype, int file_ptr_type,
-			   ADIO_Offset offset, ADIO_Status *status,
-			   int *error_code)
-{
-    ZOIDFS_IOContig(fd, buf, count, datatype, file_ptr_type,
-	    offset, status, ZOIDFS_WRITE, error_code);
-}
-
-    
-/*
- * vim: ts=8 sts=4 sw=4 noexpandtab 
- */
diff --git a/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_open.c b/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_open.c
deleted file mode 100644
index 3c22cb54e1f..00000000000
--- a/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_open.c
+++ /dev/null
@@ -1,153 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*-
- * vim: ts=8 sts=4 sw=4 noexpandtab
- *
- *   Copyright (C) 2007 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_zoidfs.h"
-#include "ad_zoidfs_common.h"
-
-/* open_status is helpful for bcasting values around */
-struct open_status_s {
-    int error;
-    zoidfs_handle_t handle;
-};
-typedef struct open_status_s open_status;
-    
-static void fake_an_open(char *fname, int access_mode,
-	                 int nr_datafiles, MPI_Offset strip_size,
-                         ADIOI_ZOIDFS_object *zoidfs_ptr, 
-			 open_status *o_status)
-{
-    int ret, created;
-    zoidfs_sattr_t attribs;
-    zoidfs_handle_t handle;
-
-    ADIOI_ZOIDFS_makeattribs(&attribs);
-
-    /* zoidfs_create succeeds even if a file already exists, so we can do
-     * our job with fewer calls than in other cases.  However, we need to
-     * be careful with ADIO_EXCL.
-     */
-    if (access_mode & ADIO_CREATE) {
-	ret = zoidfs_create(NULL, NULL, 
-			    fname, &attribs, &handle, &created, ZOIDFS_NO_OP_HINT);
-	if ((ret == ZFS_OK) && !created && (access_mode & ADIO_EXCL)) {
-	    /* lookup should not succeed if opened with EXCL */
-	    o_status->error = ZFSERR_EXIST;
-	    return;
-	}
-    }
-    else {
-	ret = zoidfs_lookup(NULL, NULL, fname, &handle, ZOIDFS_NO_OP_HINT);
-    }
-
-    o_status->error = ret;
-    o_status->handle = handle;
-    return;
-}
-
-
-/* ADIOI_ZOIDFS_Open:
- *  one process opens (or creates) the file, then broadcasts the result to the
- *  remaining processors. 
- *
- * ADIO_Open used to perform an optimization when MPI_MODE_CREATE (and before
- * that, MPI_MODE_EXCL) was set.  Because ZoidFS handles file lookup and
- * creation more scalably than traditional file systems, ADIO_Open now skips any
- * special handling when CREATE is set.  */
-void ADIOI_ZOIDFS_Open(ADIO_File fd, int *error_code)
-{
-    int rank;
-    static char myname[] = "ADIOI_ZOIDFS_OPEN";
-    ADIOI_ZOIDFS_object *zoidfs_obj_ptr;
-
-    /* since one process is doing the open, that means one process is also
-     * doing the error checking.  define a struct for both the object reference
-     * and the error code to broadcast to all the processors */
-
-    open_status o_status;
-    MPI_Datatype open_status_type;
-    MPI_Datatype types[2] = {MPI_INT, MPI_BYTE};
-    int lens[2] = {1, sizeof(ADIOI_ZOIDFS_object)};
-    MPI_Aint offsets[2];
-    
-    memset(&o_status, 0, sizeof(o_status));
-    zoidfs_obj_ptr = (ADIOI_ZOIDFS_object *) 
-	ADIOI_Malloc(sizeof(ADIOI_ZOIDFS_object));
-    /* --BEGIN ERROR HANDLING-- */
-    if (zoidfs_obj_ptr == NULL) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   MPI_ERR_UNKNOWN,
-					   "Error allocating memory", 0);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-    MPI_Comm_rank(fd->comm, &rank);
-
-    ADIOI_ZOIDFS_Init(rank, error_code);
-    if (*error_code != MPI_SUCCESS)
-    {
-	/* ADIOI_ZOIDFS_INIT handles creating error codes on its own */
-	ADIOI_Free(zoidfs_obj_ptr);
-	return;
-    }
-
-    /* one process resolves name and will later bcast to others */
-#ifdef ADIOI_MPE_LOGGING
-    MPE_Log_event( ADIOI_MPE_open_a, 0, NULL );
-#endif
-    if (rank == fd->hints->ranklist[0] && fd->fs_ptr == NULL) {
-	    fake_an_open(fd->filename, fd->access_mode, 
-		    fd->hints->striping_factor,
-		    fd->hints->striping_unit,
-		    zoidfs_obj_ptr, &o_status); 
-	    /* store credentials and object reference in fd */
-	    *zoidfs_obj_ptr = o_status.handle;
-	    fd->fs_ptr = zoidfs_obj_ptr;
-    }
-#ifdef ADIOI_MPE_LOGGING
-    MPE_Log_event( ADIOI_MPE_open_b, 0, NULL );
-#endif
-
-    /* broadcast status and (possibly valid) object reference */
-    MPI_Get_address(&o_status.error, &offsets[0]);
-    MPI_Get_address(&o_status.handle, &offsets[1]);
-
-    MPI_Type_struct(2, lens, offsets, types, &open_status_type);
-    MPI_Type_commit(&open_status_type);
-
-    /* Assertion: if we hit this Bcast, then all processes collectively
-     *            called this open.
-     *
-     * That's because deferred open never happens with this fs.
-     */
-    MPI_Bcast(MPI_BOTTOM, 1, open_status_type, fd->hints->ranklist[0],
-	      fd->comm);
-    MPI_Type_free(&open_status_type);
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (o_status.error != ZFS_OK)
-    { 
-	ADIOI_Free(zoidfs_obj_ptr);
-	fd->fs_ptr = NULL;
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   ADIOI_ZOIDFS_error_convert(o_status.error),
-					   "Unknown error", 0);
-	/* TODO: FIX STRING */
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-    *zoidfs_obj_ptr = o_status.handle;
-    fd->fs_ptr = zoidfs_obj_ptr;
-
-    *error_code = MPI_SUCCESS;
-    return;
-}
diff --git a/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_read_list.c b/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_read_list.c
deleted file mode 100644
index 3537e63e832..00000000000
--- a/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_read_list.c
+++ /dev/null
@@ -1,826 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- 
- * vim: ts=8 sts=4 sw=4 noexpandtab 
- * 
- *   Copyright (C) 2008 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-#include "ad_zoidfs.h"
-
-#include "ad_zoidfs_common.h"
-
-/* Copied from ADIOI_PVFS2_OldReadStrided.  It would be good to have fewer
- * copies of this code... */
-void ADIOI_ZOIDFS_ReadStrided(ADIO_File fd, void *buf, int count,
-			     MPI_Datatype datatype, int file_ptr_type,
-			     ADIO_Offset offset, ADIO_Status *status, int
-			     *error_code)
-{
-    /* offset is in units of etype relative to the filetype. */
-    ADIOI_Flatlist_node *flat_buf, *flat_file;
-    int i, j, k,  brd_size, frd_size=0, st_index=0;
-    int sum, n_etypes_in_filetype, size_in_filetype;
-    MPI_Count bufsize;
-    int n_filetypes, etype_in_filetype;
-    ADIO_Offset abs_off_in_filetype=0;
-    MPI_Count filetype_size, etype_size, buftype_size;
-    MPI_Aint filetype_extent, buftype_extent, filetype_lb, buftype_lb; 
-    int buf_count, buftype_is_contig, filetype_is_contig;
-    ADIO_Offset off, disp, start_off, initial_off;
-    int flag, st_frd_size, st_n_filetypes;
-
-    size_t mem_list_count, file_list_count;
-    void ** mem_offsets;
-    uint64_t *file_offsets;
-    size_t *mem_lengths;
-    uint64_t *file_lengths;
-    int total_blks_to_read;
-
-    int max_mem_list, max_file_list;
-
-    int b_blks_read;
-    int f_data_read;
-    int size_read=0, n_read_lists, extra_blks;
-
-    int end_brd_size, end_frd_size;
-    int start_k, start_j, new_file_read, new_buffer_read;
-    int start_mem_offset;
-    ADIOI_ZOIDFS_object * zoidfs_obj_ptr;
-    int err_flag=0;
-    MPI_Offset total_bytes_read = 0;
-    static char myname[] = "ADIOI_ZOIDFS_ReadStrided";
-
-    /* note: I don't know what zoidfs will do if you pass it a super-long list,
-     * so let's keep with the PVFS limit for now */
-#define MAX_ARRAY_SIZE 64
-
-    *error_code = MPI_SUCCESS;  /* changed below if error */
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-
-    /* the HDF5 tests showed a bug in this list processing code (see many many
-     * lines down below).  We added a workaround, but common HDF5 file types
-     * are actually contiguous and do not need the expensive workarond */
-    if (!filetype_is_contig) {
-	flat_file = ADIOI_Flatlist;
-	while (flat_file->type != fd->filetype) flat_file = flat_file->next;
-	if (flat_file->count == 1 && !buftype_is_contig)
-	    filetype_is_contig = 1;
-    }
-
-    MPI_Type_size_x(fd->filetype, &filetype_size);
-    if ( ! filetype_size ) {
-#ifdef HAVE_STATUS_SET_BYTES
-	MPIR_Status_set_bytes(status, datatype, 0);
-#endif
-	*error_code = MPI_SUCCESS; 
-	return;
-    }
-
-    MPI_Type_get_extent(fd->filetype, &filetype_lb, &filetype_extent);
-    MPI_Type_size_x(datatype, &buftype_size);
-    MPI_Type_get_extent(datatype, &buftype_lb, &buftype_extent);
-    etype_size = fd->etype_size;
-
-    bufsize = buftype_size * count;
-    
-    zoidfs_obj_ptr = (ADIOI_ZOIDFS_object *)fd->fs_ptr;
-
-    if (!buftype_is_contig && filetype_is_contig) {
-
-/* noncontiguous in memory, contiguous in file. */
-        uint64_t file_offsets;
-	uint64_t file_lengths;
-
-	flat_buf = ADIOI_Flatten_and_find(datatype);
-
-	off = (file_ptr_type == ADIO_INDIVIDUAL) ? fd->fp_ind : 
-	    fd->disp + etype_size * offset;
-
-	file_list_count = 1;
-	file_offsets = off;
-	file_lengths = 0;
-	total_blks_to_read = count*flat_buf->count;
-	b_blks_read = 0;
-
-	/* allocate arrays according to max usage */
-	if (total_blks_to_read > MAX_ARRAY_SIZE)
-	    mem_list_count = MAX_ARRAY_SIZE;
-	else mem_list_count = total_blks_to_read;
-	mem_offsets = (void*)ADIOI_Malloc(mem_list_count*sizeof(void*));
-	mem_lengths = (size_t*)ADIOI_Malloc(mem_list_count*sizeof(size_t));
-
-	/* TODO: CHECK RESULTS OF MEMORY ALLOCATION */
-
-	j = 0;
-	/* step through each block in memory, filling memory arrays */
-	while (b_blks_read < total_blks_to_read) {
-	    for (i=0; i<flat_buf->count; i++) {
-		mem_offsets[b_blks_read % MAX_ARRAY_SIZE] = 
-		    buf + j*buftype_extent + flat_buf->indices[i];
-		mem_lengths[b_blks_read % MAX_ARRAY_SIZE] = 
-		    flat_buf->blocklens[i];
-		file_lengths += flat_buf->blocklens[i];
-		b_blks_read++;
-		if (!(b_blks_read % MAX_ARRAY_SIZE) ||
-		    (b_blks_read == total_blks_to_read)) {
-
-		    /* in the case of the last read list call,
-		       adjust mem_list_count */
-		    if (b_blks_read == total_blks_to_read) {
-		        mem_list_count = total_blks_to_read % MAX_ARRAY_SIZE;
-			/* in case last read list call fills max arrays */
-			if (!mem_list_count) mem_list_count = MAX_ARRAY_SIZE;
-		    }
-#ifdef ADIOI_MPE_LOGGING
-                    MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
-#endif
-		    NO_STALE(err_flag, fd, zoidfs_obj_ptr,
-				    zoidfs_read(zoidfs_obj_ptr,
-					    mem_list_count,
-					    mem_offsets, mem_lengths,
-					    1, &file_offsets, &file_lengths, ZOIDFS_NO_OP_HINT));
-#ifdef ADIOI_MPE_LOGGING
-                    MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
-#endif
-		    /* --BEGIN ERROR HANDLING-- */
-		    if (err_flag != ZFS_OK) {
-			*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-							   MPIR_ERR_RECOVERABLE,
-							   myname, __LINE__,
-							   ADIOI_ZOIDFS_error_convert(err_flag),
-							   "Error in zoidfs_read", 0);
-			goto error_state;
-		    }
-		    total_bytes_read += file_lengths;
-		    /* --END ERROR HANDLING-- */
-		  
-		    /* in the case of error or the last read list call, 
-		     * leave here */
-		    if (err_flag || b_blks_read == total_blks_to_read) break;
-
-		    file_offsets += file_lengths;
-		    file_lengths = 0;
-		} 
-	    } /* for (i=0; i<flat_buf->count; i++) */
-	    j++;
-	} /* while (b_blks_read < total_blks_to_read) */
-	ADIOI_Free(mem_offsets);
-	ADIOI_Free(mem_lengths);
-
-        if (file_ptr_type == ADIO_INDIVIDUAL) 
-	    fd->fp_ind += total_bytes_read;
-
-	fd->fp_sys_posn = -1;  /* set it to null. */
-
-#ifdef HAVE_STATUS_SET_BYTES
-	MPIR_Status_set_bytes(status, datatype, bufsize);
-	/* This isa temporary way of filling in status.  The right way is to
-	   keep tracke of how much data was actually read adn placed in buf
-	   by ADIOI_BUFFERED_READ. */
-#endif
-	ADIOI_Delete_flattened(datatype);
-
-	return;
-    } /* if (!buftype_is_contig && filetype_is_contig) */
-
-    /* know file is noncontiguous from above */
-    /* noncontiguous in file */
-
-    /* filetype already flattened in ADIO_Open */
-    flat_file = ADIOI_Flatlist;
-    while (flat_file->type != fd->filetype) flat_file = flat_file->next;
-
-    disp = fd->disp;
-    initial_off = offset;
-
-
-    /* for each case - ADIO_Individual pointer or explicit, find the file
-       offset in bytes (offset), n_filetypes (how many filetypes into
-       file to start), frd_size (remaining amount of data in present
-       file block), and st_index (start point in terms of blocks in
-       starting filetype) */
-    if (file_ptr_type == ADIO_INDIVIDUAL) {
-        offset = fd->fp_ind; /* in bytes */
-	n_filetypes = -1;
-	flag = 0;
-	while (!flag) {
-	    n_filetypes++;
-	    for (i=0; i<flat_file->count; i++) {
-	        if (disp + flat_file->indices[i] + 
-		    ((ADIO_Offset) n_filetypes)*filetype_extent +
-		    flat_file->blocklens[i]  >= offset) {
-		    st_index = i;
-		    frd_size = disp + flat_file->indices[i] + 
-				    ((ADIO_Offset) n_filetypes)*filetype_extent
-				      + flat_file->blocklens[i] - offset;
-		    flag = 1;
-		    break;
-		}
-	    }
-	} /* while (!flag) */
-    } /* if (file_ptr_type == ADIO_INDIVIDUAL) */
-    else {
-        n_etypes_in_filetype = filetype_size/etype_size;
-	n_filetypes = (int) (offset / n_etypes_in_filetype);
-	etype_in_filetype = (int) (offset % n_etypes_in_filetype);
-	size_in_filetype = etype_in_filetype * etype_size;
-	
-	sum = 0;
-	for (i=0; i<flat_file->count; i++) {
-	    sum += flat_file->blocklens[i];
-	    if (sum > size_in_filetype) {
-	        st_index = i;
-		frd_size = sum - size_in_filetype;
-		abs_off_in_filetype = flat_file->indices[i] +
-		    size_in_filetype - (sum - flat_file->blocklens[i]);
-		break;
-	    }
-	}
-	
-	/* abs. offset in bytes in the file */
-	offset = disp + ((ADIO_Offset) n_filetypes)*filetype_extent + 
-	    abs_off_in_filetype;
-    } /* else [file_ptr_type != ADIO_INDIVIDUAL] */
-
-    start_off = offset;
-    st_frd_size = frd_size;
-    st_n_filetypes = n_filetypes;
-    
-    if (buftype_is_contig && !filetype_is_contig) {
-
-/* contiguous in memory, noncontiguous in file. should be the most
-   common case. */
-
-	/* only one memory off-len pair, so no array here */
-        size_t mem_lengths;
-	size_t mem_offsets;
-	
-	i = 0;
-	j = st_index;
-	n_filetypes = st_n_filetypes;
-	
-	mem_list_count = 1;
-	
-	/* determine how many blocks in file to read */
-	f_data_read = ADIOI_MIN(st_frd_size, bufsize);
-	total_blks_to_read = 1;
-	if (j < (flat_file->count-1)) j++;
-	else {
-	    j = 0;
-	    n_filetypes++;
-	}
-	while (f_data_read < bufsize) {
-	    f_data_read += flat_file->blocklens[j];
-	    total_blks_to_read++;
-	    if (j<(flat_file->count-1)) j++;
-	    else j = 0;	
-	}
-      
-	j = st_index;
-	n_filetypes = st_n_filetypes;
-	n_read_lists = total_blks_to_read/MAX_ARRAY_SIZE;
-	extra_blks = total_blks_to_read%MAX_ARRAY_SIZE;
-	
-	mem_offsets = (size_t)buf;
-	mem_lengths = 0;
-	
-	/* if at least one full readlist, allocate file arrays
-	   at max array size and don't free until very end */
-	if (n_read_lists) {
-	    file_offsets = (int64_t*)ADIOI_Malloc(MAX_ARRAY_SIZE*
-						  sizeof(int64_t));
-	    file_lengths = (uint64_t*)ADIOI_Malloc(MAX_ARRAY_SIZE*
-						  sizeof(uint64_t));
-	}
-	/* if there's no full readlist allocate file arrays according
-	   to needed size (extra_blks) */
-	else {
-	    file_offsets = (int64_t*)ADIOI_Malloc(extra_blks*
-						  sizeof(int64_t));
-	    file_lengths = (uint64_t*)ADIOI_Malloc(extra_blks*
-						  sizeof(uint64_t));
-	}
-	
-	/* for file arrays that are of MAX_ARRAY_SIZE, build arrays */
-	for (i=0; i<n_read_lists; i++) {
-	    file_list_count = MAX_ARRAY_SIZE;
-	    if(!i) {
-	        file_offsets[0] = offset;
-		file_lengths[0] = st_frd_size;
-		mem_lengths = st_frd_size;
-	    }
-	    for (k=0; k<MAX_ARRAY_SIZE; k++) {
-	        if (i || k) {
-		    file_offsets[k] = disp + 
-			((ADIO_Offset)n_filetypes)*filetype_extent
-		      + flat_file->indices[j];
-		    file_lengths[k] = flat_file->blocklens[j];
-		    mem_lengths += file_lengths[k];
-		}
-		if (j<(flat_file->count - 1)) j++;
-		else {
-		    j = 0;
-		    n_filetypes++;
-		}
-	    } /* for (k=0; k<MAX_ARRAY_SIZE; k++) */
-	    /* --END ERROR HANDLING-- */
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
-#endif
-	    NO_STALE(err_flag, fd, zoidfs_obj_ptr,
-			    zoidfs_read(zoidfs_obj_ptr,
-				    1, buf, &mem_lengths,
-				    file_list_count,
-				    file_offsets, file_lengths, ZOIDFS_NO_OP_HINT));
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
-#endif
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (err_flag != ZFS_OK) {
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-						   MPIR_ERR_RECOVERABLE,
-						   myname, __LINE__,
-						   ADIOI_ZOIDFS_error_convert(err_flag),
-						   "Error in zoidfs_read", 0);
-		goto error_state;
-	    }
-	    /* --END ERROR HANDING-- */
-	    total_bytes_read += mem_lengths;
-
-	    mem_offsets += mem_lengths;
-	    mem_lengths = 0;
-	} /* for (i=0; i<n_read_lists; i++) */
-
-	/* for file arrays smaller than MAX_ARRAY_SIZE (last read_list call) */
-	if (extra_blks) {
-	    file_list_count = extra_blks;
-	    if(!i) {
-	        file_offsets[0] = offset;
-		file_lengths[0] = ADIOI_MIN(st_frd_size, bufsize);
-	    }
-	    for (k=0; k<extra_blks; k++) {
-	        if(i || k) {
-		    file_offsets[k] = disp + 
-			((ADIO_Offset)n_filetypes)*filetype_extent +
-			flat_file->indices[j];
-		    if (k == (extra_blks - 1)) {
-		        file_lengths[k] = bufsize - mem_lengths
-			  - mem_offsets + (size_t)buf;
-		    }
-		    else file_lengths[k] = flat_file->blocklens[j];
-		} /* if(i || k) */
-		mem_lengths += file_lengths[k];
-		if (j<(flat_file->count - 1)) j++;
-		else {
-		    j = 0;
-		    n_filetypes++;
-		}
-	    } /* for (k=0; k<extra_blks; k++) */
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
-#endif
-	    NO_STALE(err_flag, fd, zoidfs_obj_ptr,
-			    zoidfs_read(zoidfs_obj_ptr, 1,
-				   (void **)&mem_offsets,
-				   &mem_lengths,
-				   file_list_count,
-				   file_offsets, file_lengths, ZOIDFS_NO_OP_HINT));
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
-#endif
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (err_flag != 0) {
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-						   MPIR_ERR_RECOVERABLE,
-						   myname, __LINE__,
-						   ADIOI_ZOIDFS_error_convert(err_flag),
-						   "Error in zoidfs_read", 0);		
-		goto error_state;
-	    }
-	    /* --END ERROR HANDLING-- */
-	    total_bytes_read += mem_lengths;
-	}
-    }
-    else {
-/* noncontiguous in memory as well as in file */
-      
-	flat_buf = ADIOI_Flatten_and_find(datatype);
-
-	size_read = 0;
-	n_filetypes = st_n_filetypes;
-	frd_size = st_frd_size;
-	brd_size = flat_buf->blocklens[0];
-	buf_count = 0;
-	start_mem_offset = 0;
-	start_k = k = 0;
-	start_j = st_index;
-	max_mem_list = 0;
-	max_file_list = 0;
-
-	/* run through and file max_file_list and max_mem_list so that you 
-	   can allocate the file and memory arrays less than MAX_ARRAY_SIZE
-	   if possible */
-
-	while (size_read < bufsize) {
-	    k = start_k;
-	    new_buffer_read = 0;
-	    mem_list_count = 0;
-	    while ((mem_list_count < MAX_ARRAY_SIZE) && 
-		   (new_buffer_read < bufsize-size_read)) {
-	        /* find mem_list_count and file_list_count such that both are
-		   less than MAX_ARRAY_SIZE, the sum of their lengths are
-		   equal, and the sum of all the data read and data to be
-		   read in the next immediate read list is less than
-		   bufsize */
-	        if(mem_list_count) {
-		    if((new_buffer_read + flat_buf->blocklens[k] + 
-			size_read) > bufsize) {
-		        end_brd_size = new_buffer_read + 
-			    flat_buf->blocklens[k] - (bufsize - size_read);
-			new_buffer_read = bufsize - size_read;
-		    }
-		    else {
-		        new_buffer_read += flat_buf->blocklens[k];
-			end_brd_size = flat_buf->blocklens[k];
-		    }
-		}
-		else {
-		    if (brd_size > (bufsize - size_read)) {
-		        new_buffer_read = bufsize - size_read;
-			brd_size = new_buffer_read;
-		    }
-		    else new_buffer_read = brd_size;
-		}
-		mem_list_count++;
-		k = (k + 1)%flat_buf->count;
-	     } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
-	       (new_buffer_read < bufsize-size_read)) */
-	    j = start_j;
-	    new_file_read = 0;
-	    file_list_count = 0;
-	    while ((file_list_count < MAX_ARRAY_SIZE) && 
-		   (new_file_read < new_buffer_read)) {
-	        if(file_list_count) {
-		    if((new_file_read + flat_file->blocklens[j]) > 
-		       new_buffer_read) {
-		        end_frd_size = new_buffer_read - new_file_read;
-			new_file_read = new_buffer_read;
-			j--;
-		    }
-		    else {
-		        new_file_read += flat_file->blocklens[j];
-			end_frd_size = flat_file->blocklens[j];
-		    }
-		}
-		else {
-		    if (frd_size > new_buffer_read) {
-		        new_file_read = new_buffer_read;
-			frd_size = new_file_read;
-		    }
-		    else new_file_read = frd_size;
-		}
-		file_list_count++;
-		if (j < (flat_file->count - 1)) j++;
-		else j = 0;
-		
-		k = start_k;
-		if ((new_file_read < new_buffer_read) && 
-		    (file_list_count == MAX_ARRAY_SIZE)) {
-		    new_buffer_read = 0;
-		    mem_list_count = 0;
-		    while (new_buffer_read < new_file_read) {
-		        if(mem_list_count) {
-			    if((new_buffer_read + flat_buf->blocklens[k]) >
-			       new_file_read) {
-			        end_brd_size = new_file_read - new_buffer_read;
-				new_buffer_read = new_file_read;
-				k--;
-			    }
-			    else {
-			        new_buffer_read += flat_buf->blocklens[k];
-				end_brd_size = flat_buf->blocklens[k];
-			    }
-			}
-			else {
-			    new_buffer_read = brd_size;
-			    if (brd_size > (bufsize - size_read)) {
-			        new_buffer_read = bufsize - size_read;
-				brd_size = new_buffer_read;
-			    }
-			}
-			mem_list_count++;
-			k = (k + 1)%flat_buf->count;
-		    } /* while (new_buffer_read < new_file_read) */
-		} /* if ((new_file_read < new_buffer_read) && (file_list_count
-		     == MAX_ARRAY_SIZE)) */
-	    } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
-		 (new_buffer_read < bufsize-size_read)) */
-
-	    /*  fakes filling the readlist arrays of lengths found above  */
-	    k = start_k;
-	    j = start_j;
-	    for (i=0; i<mem_list_count; i++) {	     
-		if(i) {
-		    if (i == (mem_list_count - 1)) {
-			if (flat_buf->blocklens[k] == end_brd_size)
-			    brd_size = flat_buf->blocklens[(k+1)%
-							  flat_buf->count];
-			else {
-			    brd_size = flat_buf->blocklens[k] - end_brd_size;
-			    k--;
-			    buf_count--;
-			}
-		    }
-		}
-		buf_count++;
-		k = (k + 1)%flat_buf->count;
-	    } /* for (i=0; i<mem_list_count; i++) */
-	    for (i=0; i<file_list_count; i++) {
-		if (i) {
-		    if (i == (file_list_count - 1)) {
-			if (flat_file->blocklens[j] == end_frd_size)
-			    frd_size = flat_file->blocklens[(j+1)%
-							  flat_file->count];   
-			else {
-			    frd_size = flat_file->blocklens[j] - end_frd_size;
-			    j--;
-			}
-		    }
-		}
-		if (j < flat_file->count - 1) j++;
-		else {
-		    j = 0;
-		    n_filetypes++;
-		}
-	    } /* for (i=0; i<file_list_count; i++) */
-	    size_read += new_buffer_read;
-	    start_k = k;
-	    start_j = j;
-	    if (max_mem_list < mem_list_count)
-	        max_mem_list = mem_list_count;
-	    if (max_file_list < file_list_count)
-	        max_file_list = file_list_count;
-	} /* while (size_read < bufsize) */
-
-	/* one last check before we actually carry out the operation:
-	 * this code has hard-to-fix bugs when a noncontiguous file type has
-	 * such large pieces that the sum of the lengths of the memory type is
-	 * not larger than one of those pieces (and vice versa for large memory
-	 * types and many pices of file types.  In these cases, give up and
-	 * fall back to naive reads and writes.  The testphdf5 test created a
-	 * type with two very large memory regions and 600 very small file
-	 * regions.  The same test also created a type with one very large file
-	 * region and many (700) very small memory regions.  both cases caused
-	 * problems for this code */
-
-	if ( ( (file_list_count == 1) && 
-		    (new_file_read < flat_file->blocklens[0] ) ) ||
-		((mem_list_count == 1) && 
-		    (new_buffer_read < flat_buf->blocklens[0]) ) ||
-		((file_list_count == MAX_ARRAY_SIZE) && 
-		    (new_file_read < flat_buf->blocklens[0]) ) ||
-		( (mem_list_count == MAX_ARRAY_SIZE) &&
-		    (new_buffer_read < flat_file->blocklens[0])) )
-	{
-
-	    ADIOI_Delete_flattened(datatype);
-	    ADIOI_GEN_ReadStrided_naive(fd, buf, count, datatype,
-		    file_ptr_type, initial_off, status, error_code);
-	    return;
-	}
-
-	mem_offsets = (void *)ADIOI_Malloc(max_mem_list*sizeof(void *));
-	mem_lengths = (size_t*)ADIOI_Malloc(max_mem_list*sizeof(size_t));
-	file_offsets = (uint64_t *)ADIOI_Malloc(max_file_list*sizeof(uint64_t));
-	file_lengths = (uint64_t *)ADIOI_Malloc(max_file_list*sizeof(uint64_t));
-	    
-	size_read = 0;
-	n_filetypes = st_n_filetypes;
-	frd_size = st_frd_size;
-	brd_size = flat_buf->blocklens[0];
-	buf_count = 0;
-	start_mem_offset = 0;
-	start_k = k = 0;
-	start_j = st_index;
-
-	/*  this section calculates mem_list_count and file_list_count
-	    and also finds the possibly odd sized last array elements
-	    in new_frd_size and new_brd_size  */
-	
-	while (size_read < bufsize) {
-	    k = start_k;
-	    new_buffer_read = 0;
-	    mem_list_count = 0;
-	    while ((mem_list_count < MAX_ARRAY_SIZE) && 
-		   (new_buffer_read < bufsize-size_read)) {
-	        /* find mem_list_count and file_list_count such that both are
-		   less than MAX_ARRAY_SIZE, the sum of their lengths are
-		   equal, and the sum of all the data read and data to be
-		   read in the next immediate read list is less than
-		   bufsize */
-	        if(mem_list_count) {
-		    if((new_buffer_read + flat_buf->blocklens[k] + 
-			size_read) > bufsize) {
-		        end_brd_size = new_buffer_read + 
-			    flat_buf->blocklens[k] - (bufsize - size_read);
-			new_buffer_read = bufsize - size_read;
-		    }
-		    else {
-		        new_buffer_read += flat_buf->blocklens[k];
-			end_brd_size = flat_buf->blocklens[k];
-		    }
-		}
-		else {
-		    if (brd_size > (bufsize - size_read)) {
-		        new_buffer_read = bufsize - size_read;
-			brd_size = new_buffer_read;
-		    }
-		    else new_buffer_read = brd_size;
-		}
-		mem_list_count++;
-		k = (k + 1)%flat_buf->count;
-	     } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
-	       (new_buffer_read < bufsize-size_read)) */
-	    j = start_j;
-	    new_file_read = 0;
-	    file_list_count = 0;
-	    while ((file_list_count < MAX_ARRAY_SIZE) && 
-		   (new_file_read < new_buffer_read)) {
-	        if(file_list_count) {
-		    if((new_file_read + flat_file->blocklens[j]) > 
-		       new_buffer_read) {
-		        end_frd_size = new_buffer_read - new_file_read;
-			new_file_read = new_buffer_read;
-			j--;
-		    }
-		    else {
-		        new_file_read += flat_file->blocklens[j];
-			end_frd_size = flat_file->blocklens[j];
-		    }
-		}
-		else {
-		    if (frd_size > new_buffer_read) {
-		        new_file_read = new_buffer_read;
-			frd_size = new_file_read;
-		    }
-		    else new_file_read = frd_size;
-		}
-		file_list_count++;
-		if (j < (flat_file->count - 1)) j++;
-		else j = 0;
-		
-		k = start_k;
-		if ((new_file_read < new_buffer_read) && 
-		    (file_list_count == MAX_ARRAY_SIZE)) {
-		    new_buffer_read = 0;
-		    mem_list_count = 0;
-		    while (new_buffer_read < new_file_read) {
-		        if(mem_list_count) {
-			    if((new_buffer_read + flat_buf->blocklens[k]) >
-			       new_file_read) {
-			        end_brd_size = new_file_read - new_buffer_read;
-				new_buffer_read = new_file_read;
-				k--;
-			    }
-			    else {
-			        new_buffer_read += flat_buf->blocklens[k];
-				end_brd_size = flat_buf->blocklens[k];
-			    }
-			}
-			else {
-			    new_buffer_read = brd_size;
-			    if (brd_size > (bufsize - size_read)) {
-			        new_buffer_read = bufsize - size_read;
-				brd_size = new_buffer_read;
-			    }
-			}
-			mem_list_count++;
-			k = (k + 1)%flat_buf->count;
-		    } /* while (new_buffer_read < new_file_read) */
-		} /* if ((new_file_read < new_buffer_read) && (file_list_count
-		     == MAX_ARRAY_SIZE)) */
-	    } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
-		 (new_buffer_read < bufsize-size_read)) */
-
-	    /*  fills the allocated readlist arrays  */
-	    k = start_k;
-	    j = start_j;
-	    for (i=0; i<mem_list_count; i++) {	     
-	        mem_offsets[i] = buf + 
-			buftype_extent* (buf_count/flat_buf->count) +
-					 flat_buf->indices[k];
-		if(!i) {
-		    mem_lengths[0] = brd_size;
-		    mem_offsets[0] += flat_buf->blocklens[k] - brd_size;
-		}
-		else {
-		    if (i == (mem_list_count - 1)) {
-		        mem_lengths[i] = end_brd_size;
-			if (flat_buf->blocklens[k] == end_brd_size)
-			    brd_size = flat_buf->blocklens[(k+1)%
-							  flat_buf->count];
-			else {
-			    brd_size = flat_buf->blocklens[k] - end_brd_size;
-			    k--;
-			    buf_count--;
-			}
-		    }
-		    else {
-		        mem_lengths[i] = flat_buf->blocklens[k];
-		    }
-		}
-		buf_count++;
-		k = (k + 1)%flat_buf->count;
-	    } /* for (i=0; i<mem_list_count; i++) */
-	    for (i=0; i<file_list_count; i++) {
-	        file_offsets[i] = disp + flat_file->indices[j] + 
-		    ((ADIO_Offset)n_filetypes) * filetype_extent;
-	        if (!i) {
-		    file_lengths[0] = frd_size;
-		    file_offsets[0] += flat_file->blocklens[j] - frd_size;
-		}
-		else {
-		    if (i == (file_list_count - 1)) {
-		        file_lengths[i] = end_frd_size;
-			if (flat_file->blocklens[j] == end_frd_size)
-			    frd_size = flat_file->blocklens[(j+1)%
-							  flat_file->count];   
-			else {
-			    frd_size = flat_file->blocklens[j] - end_frd_size;
-			    j--;
-			}
-		    }
-		    else file_lengths[i] = flat_file->blocklens[j];
-		}
-		if (j < flat_file->count - 1) j++;
-		else {
-		    j = 0;
-		    n_filetypes++;
-		}
-	    } /* for (i=0; i<file_list_count; i++) */
-
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
-#endif
-	    NO_STALE(err_flag, fd, zoidfs_obj_ptr,
-			    zoidfs_read(zoidfs_obj_ptr,
-				    mem_list_count, mem_offsets, mem_lengths,
-				    file_list_count,
-				    file_offsets, file_lengths, ZOIDFS_NO_OP_HINT));
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
-#endif
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (err_flag != ZFS_OK) {
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-						   MPIR_ERR_RECOVERABLE,
-						   myname, __LINE__,
-						   ADIOI_ZOIDFS_error_convert(err_flag),
-						   "Error in zoidfs_read", 0);
-	    }
-	    /* --END ERROR HANDLING-- */
-	    size_read += new_buffer_read;
-	    total_bytes_read += new_buffer_read; /* XXX: is this right? */
-	    start_k = k;
-	    start_j = j;
-	} /* while (size_read < bufsize) */
-	ADIOI_Free(mem_offsets);
-	ADIOI_Free(mem_lengths);
-    }
-    /* Other ADIO routines will convert absolute bytes into counts of datatypes */
-    /* when incrementing fp_ind, need to also take into account the file type:
-     * consider an N-element 1-d subarray with a lb and ub: ( |---xxxxx-----|
-     * if we wrote N elements, offset needs to point at beginning of type, not
-     * at empty region at offset N+1) 
-     *
-     * As we discussed on mpich-discuss in may/june 2009, the code below might
-     * look wierd, but by putting fp_ind at the last byte written, the next
-     * time we run through the strided code we'll update the fp_ind to the
-     * right location. */
-    if (file_ptr_type == ADIO_INDIVIDUAL) {
-	fd->fp_ind = file_offsets[file_list_count-1]+
-	    file_lengths[file_list_count-1];
-    }
-    
-    ADIOI_Free(file_offsets);
-    ADIOI_Free(file_lengths);
-    
-    if (err_flag == 0) *error_code = MPI_SUCCESS;
-
-error_state:
-    fd->fp_sys_posn = -1;   /* set it to null. */
-
-#ifdef HAVE_STATUS_SET_BYTES
-    MPIR_Status_set_bytes(status, datatype, bufsize);
-    /* This is a temporary way of filling in status. The right way is to 
-       keep track of how much data was actually read and placed in buf 
-       by ADIOI_BUFFERED_READ. */
-#endif
-    
-    if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
-}
-
diff --git a/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_resize.c b/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_resize.c
deleted file mode 100644
index 60d2fcaba20..00000000000
--- a/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_resize.c
+++ /dev/null
@@ -1,53 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ad_zoidfs.h"
-#include "ad_zoidfs_common.h"
-
-/* as with flush, implement the resize operation in a scalable
- * manner. one process does the work, then broadcasts the result to everyone
- * else.  fortunately, this operation is defined to be collective */
-void ADIOI_ZOIDFS_Resize(ADIO_File fd, ADIO_Offset size, int *error_code)
-{
-    int ret, rank;
-    ADIOI_ZOIDFS_object *zoidfs_obj_ptr;
-    static char myname[] = "ADIOI_ZOIDFS_RESIZE";
-
-    *error_code = MPI_SUCCESS;
-
-    zoidfs_obj_ptr = (ADIOI_ZOIDFS_object *)fd->fs_ptr;
-
-    MPI_Comm_rank(fd->comm, &rank);
-
-
-    /* MPI-IO semantics treat conflicting MPI_File_set_size requests the
-     * same as conflicting write requests. Thus, a resize from one
-     * process does not have to be visible to the other processes until a
-     * syncronization point is reached */
-
-    if (rank == fd->hints->ranklist[0]) {
-        NO_STALE(ret, fd, zoidfs_obj_ptr,
-                 zoidfs_resize(zoidfs_obj_ptr, size, ZOIDFS_NO_OP_HINT));
-	MPI_Bcast(&ret, 1, MPI_INT, fd->hints->ranklist[0], fd->comm);
-    } else  {
-	MPI_Bcast(&ret, 1, MPI_INT, fd->hints->ranklist[0], fd->comm);
-    }
-    /* --BEGIN ERROR HANDLING-- */
-    if (ret != ZFS_OK) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   ADIOI_ZOIDFS_error_convert(ret),
-					   "Error in zoidfs_resize", 0);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-}
-
-/*
- * vim: ts=8 sts=4 sw=4 noexpandtab 
- */
diff --git a/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_write_list.c b/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_write_list.c
deleted file mode 100644
index 44376316a64..00000000000
--- a/3rd-party/romio321/adio/ad_zoidfs/ad_zoidfs_write_list.c
+++ /dev/null
@@ -1,857 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- 
- * vim: ts=8 sts=4 sw=4 noexpandtab 
- * 
- *   Copyright (C) 2008 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-#include "ad_zoidfs.h"
-
-#include "ad_zoidfs_common.h"
-
-/* Copied from ADIOI_PVFS2_OldWriteStrided.  It would be good to have fewer
- * copies of this code... */
-void ADIOI_ZOIDFS_WriteStrided(ADIO_File fd, void *buf, int count,
-			MPI_Datatype datatype, int file_ptr_type,
-			ADIO_Offset offset, ADIO_Status *status,
-			int *error_code)
-{
-    /* as with all the other WriteStrided functions, offset is in units of
-     * etype relative to the filetype */
-
-    /* Since zoidfs does not support file locking, can't do buffered writes
-       as on Unix */
-
-    ADIOI_Flatlist_node *flat_buf, *flat_file;
-    int i, j, k, bwr_size, fwr_size=0, st_index=0;
-    int sum, n_etypes_in_filetype, size_in_filetype;
-    MPI_Count bufsize;
-    int n_filetypes, etype_in_filetype;
-    ADIO_Offset abs_off_in_filetype=0;
-    MPI_Count filetype_size, etype_size, buftype_size;
-    MPI_Aint filetype_extent, buftype_extent, filetype_lb, buftype_lb;
-    int buf_count, buftype_is_contig, filetype_is_contig;
-    ADIO_Offset off, disp, start_off, initial_off;
-    int flag, st_fwr_size, st_n_filetypes;
-    int err_flag=0;
-
-    size_t mem_list_count, file_list_count;
-    const void ** mem_offsets;
-    uint64_t *file_offsets;
-    size_t *mem_lengths;
-    uint64_t *file_lengths;
-    int total_blks_to_write;
-
-    int max_mem_list, max_file_list;
-
-    int b_blks_wrote;
-    int f_data_wrote;
-    int size_wrote=0, n_write_lists, extra_blks;
-
-    int end_bwr_size, end_fwr_size;
-    int start_k, start_j, new_file_write, new_buffer_write;
-    int start_mem_offset;
-    ADIOI_ZOIDFS_object *zoidfs_obj_ptr;
-    MPI_Offset total_bytes_written=0;
-    static char myname[] = "ADIOI_ZOIDFS_WRITESTRIDED";
-
-    /* note: I don't know what zoidfs will do if you pass it a super-long list,
-     * so let's keep with the PVFS limit for now */
-#define MAX_ARRAY_SIZE 64
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (fd->atomicity) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   MPI_ERR_ARG,
-					   "Atomic noncontiguous writes are not supported by ZOIDFS", 0);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-
-    /* the HDF5 tests showed a bug in this list processing code (see many many
-     * lines down below).  We added a workaround, but common HDF5 file types
-     * are actually contiguous and do not need the expensive workarond */
-    if (!filetype_is_contig) {
-	flat_file = ADIOI_Flatlist;
-	while (flat_file->type != fd->filetype) flat_file = flat_file->next;
-	if (flat_file->count == 1 && !buftype_is_contig)
-	    filetype_is_contig = 1;
-    }
-
-    MPI_Type_size_x(fd->filetype, &filetype_size);
-    if ( ! filetype_size ) {
-#ifdef HAVE_STATUS_SET_BYTES
-	MPIR_Status_set_bytes(status, datatype, 0);
-#endif
-	*error_code = MPI_SUCCESS; 
-	return;
-    }
-
-    MPI_Type_get_extent(fd->filetype, &filetype_lb, &filetype_extent);
-    MPI_Type_size_x(datatype, &buftype_size);
-    MPI_Type_get_extent(datatype, &buftype_lb, &buftype_extent);
-    etype_size = fd->etype_size;
-    
-    bufsize = buftype_size * count;
-
-    zoidfs_obj_ptr = (ADIOI_ZOIDFS_object*)fd->fs_ptr;
-
-    if (!buftype_is_contig && filetype_is_contig) {
-
-/* noncontiguous in memory, contiguous in file.  */
-        uint64_t file_offsets;
-	uint64_t file_lengths;
-
-	flat_buf = ADIOI_Flatten_and_find(datatype);
-	
-	if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-	    off = fd->disp + etype_size * offset;
-	}
-	else off = fd->fp_ind;
-
-	file_list_count = 1;
-	file_offsets = off;
-	file_lengths = 0;
-	total_blks_to_write = count*flat_buf->count;
-	b_blks_wrote = 0;
-
-	/* allocate arrays according to max usage */
-	if (total_blks_to_write > MAX_ARRAY_SIZE)
-	    mem_list_count = MAX_ARRAY_SIZE;
-	else mem_list_count = total_blks_to_write;
-	mem_offsets = (void*)ADIOI_Malloc(mem_list_count*sizeof(void*));
-	mem_lengths = (size_t*)ADIOI_Malloc(mem_list_count*sizeof(size_t));
-
-	j = 0;
-	/* step through each block in memory, filling memory arrays */
-	while (b_blks_wrote < total_blks_to_write) {
-	    for (i=0; i<flat_buf->count; i++) {
-		mem_offsets[b_blks_wrote % MAX_ARRAY_SIZE] = 
-		    buf + 
-		     j*buftype_extent + 
-		     flat_buf->indices[i];
-		mem_lengths[b_blks_wrote % MAX_ARRAY_SIZE] = 
-		    flat_buf->blocklens[i];
-		file_lengths += flat_buf->blocklens[i];
-		b_blks_wrote++;
-		if (!(b_blks_wrote % MAX_ARRAY_SIZE) ||
-		    (b_blks_wrote == total_blks_to_write)) {
-
-		    /* in the case of the last write list call,
-		       adjust mem_list_count */
-		    if (b_blks_wrote == total_blks_to_write) {
-		        mem_list_count = total_blks_to_write % MAX_ARRAY_SIZE;
-			/* in case last write list call fills max arrays */
-			if (!mem_list_count) mem_list_count = MAX_ARRAY_SIZE;
-		    }
-#ifdef ADIOI_MPE_LOGGING
-                    MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
-#endif
-		    NO_STALE(err_flag, fd, zoidfs_obj_ptr,
-				    zoidfs_write(zoidfs_obj_ptr, 
-					    mem_list_count,
-					    mem_offsets, mem_lengths, 
-					    1, &file_offsets, &file_lengths, ZOIDFS_NO_OP_HINT));
-
-		    /* --BEGIN ERROR HANDLING-- */
-		    if (err_flag != ZFS_OK) {
-			*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-							   MPIR_ERR_RECOVERABLE,
-							   myname, __LINE__,
-							   ADIOI_ZOIDFS_error_convert(err_flag),
-							   "Error in zoidfs_write", 0);
-			break;
-		    }
-#ifdef ADIOI_MPE_LOGGING
-                    MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
-#endif
-		    total_bytes_written += file_lengths;
-		  
-		    /* in the case of error or the last write list call, 
-		     * leave here */
-		    /* --BEGIN ERROR HANDLING-- */
-		    if (err_flag) {
-			*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-							   MPIR_ERR_RECOVERABLE,
-							   myname, __LINE__,
-							   ADIOI_ZOIDFS_error_convert(err_flag),
-							   "Error in zoidfs_write", 0);
-			break;
-		    }
-		    /* --END ERROR HANDLING-- */
-		    if (b_blks_wrote == total_blks_to_write) break;
-
-		    file_offsets += file_lengths;
-		    file_lengths = 0;
-		} 
-	    } /* for (i=0; i<flat_buf->count; i++) */
-	    j++;
-	} /* while (b_blks_wrote < total_blks_to_write) */
-	ADIOI_Free(mem_offsets);
-	ADIOI_Free(mem_lengths);
-
-	if (file_ptr_type == ADIO_INDIVIDUAL) 
-	    fd->fp_ind += total_bytes_written;
-
-	if (!err_flag)  *error_code = MPI_SUCCESS;
-
-	fd->fp_sys_posn = -1;   /* clear this. */
-
-#ifdef HAVE_STATUS_SET_BYTES
-	MPIR_Status_set_bytes(status, datatype, bufsize);
-/* This is a temporary way of filling in status. The right way is to 
-   keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
-#endif
-
-	ADIOI_Delete_flattened(datatype);
-	return;
-    } /* if (!buftype_is_contig && filetype_is_contig) */
-
-    /* already know that file is noncontiguous from above */
-    /* noncontiguous in file */
-
-/* filetype already flattened in ADIO_Open */
-    flat_file = ADIOI_Flatlist;
-    while (flat_file->type != fd->filetype) flat_file = flat_file->next;
-
-    disp = fd->disp;
-    initial_off = offset;
-
-    /* for each case - ADIO_Individual pointer or explicit, find offset
-       (file offset in bytes), n_filetypes (how many filetypes into file 
-       to start), fwr_size (remaining amount of data in present file
-       block), and st_index (start point in terms of blocks in starting
-       filetype) */
-    if (file_ptr_type == ADIO_INDIVIDUAL) {
-        offset = fd->fp_ind; /* in bytes */
-	n_filetypes = -1;
-	flag = 0;
-	while (!flag) {
-	    n_filetypes++;
-	    for (i=0; i<flat_file->count; i++) {
-	        if (disp + flat_file->indices[i] + 
-		    ((ADIO_Offset) n_filetypes)*filetype_extent +
-		      flat_file->blocklens[i] >= offset) {
-		  st_index = i;
-		  fwr_size = disp + flat_file->indices[i] + 
-		    ((ADIO_Offset) n_filetypes)*filetype_extent
-		    + flat_file->blocklens[i] - offset;
-		  flag = 1;
-		  break;
-		}
-	    }
-	} /* while (!flag) */
-    } /* if (file_ptr_type == ADIO_INDIVIDUAL) */
-    else {
-        n_etypes_in_filetype = filetype_size/etype_size;
-	n_filetypes = (int) (offset / n_etypes_in_filetype);
-	etype_in_filetype = (int) (offset % n_etypes_in_filetype);
-	size_in_filetype = etype_in_filetype * etype_size;
-	
-	sum = 0;
-	for (i=0; i<flat_file->count; i++) {
-	    sum += flat_file->blocklens[i];
-	    if (sum > size_in_filetype) {
-	        st_index = i;
-		fwr_size = sum - size_in_filetype;
-		abs_off_in_filetype = flat_file->indices[i] +
-		    size_in_filetype - (sum - flat_file->blocklens[i]);
-		break;
-	    }
-	}
-
-	/* abs. offset in bytes in the file */
-	offset = disp + ((ADIO_Offset) n_filetypes)*filetype_extent +
-	    abs_off_in_filetype;
-    } /* else [file_ptr_type != ADIO_INDIVIDUAL] */
-
-    start_off = offset;
-    st_fwr_size = fwr_size;
-    st_n_filetypes = n_filetypes;
-    
-    if (buftype_is_contig && !filetype_is_contig) {
-
-/* contiguous in memory, noncontiguous in file. should be the most
-   common case. */
-
-	/* only one memory off-len pair, so no array */
-        size_t mem_lengths;
-	size_t mem_offsets;
-        
-	i = 0;
-	j = st_index;
-	off = offset;
-	n_filetypes = st_n_filetypes;
-        
-	mem_list_count = 1;
-        
-	/* determine how many blocks in file to write */
-	f_data_wrote = ADIOI_MIN(st_fwr_size, bufsize);
-	total_blks_to_write = 1;
-	if (j < (flat_file->count -1)) j++;
-	else {
-	    j = 0;
-	    n_filetypes++;
-	}
-	while (f_data_wrote < bufsize) {
-	    f_data_wrote += flat_file->blocklens[j];
-	    total_blks_to_write++;
-	    if (j<(flat_file->count-1)) j++;
-	    else j = 0; 
-	}
-	    
-	j = st_index;
-	n_filetypes = st_n_filetypes;
-	n_write_lists = total_blks_to_write/MAX_ARRAY_SIZE;
-	extra_blks = total_blks_to_write%MAX_ARRAY_SIZE;
-        
-	mem_offsets = (size_t)buf;
-	mem_lengths = 0;
-        
-	/* if at least one full writelist, allocate file arrays
-	   at max array size and don't free until very end */
-	if (n_write_lists) {
-	    file_offsets = (int64_t*)ADIOI_Malloc(MAX_ARRAY_SIZE*
-						  sizeof(int64_t));
-	    file_lengths = (uint64_t*)ADIOI_Malloc(MAX_ARRAY_SIZE*
-						  sizeof(uint64_t));
-	}
-	/* if there's no full writelist allocate file arrays according
-	   to needed size (extra_blks) */
-	else {
-	    file_offsets = (int64_t*)ADIOI_Malloc(extra_blks*
-                                                  sizeof(int64_t));
-            file_lengths = (uint64_t*)ADIOI_Malloc(extra_blks*
-                                                  sizeof(uint64_t));
-        }
-        
-        /* for file arrays that are of MAX_ARRAY_SIZE, build arrays */
-        for (i=0; i<n_write_lists; i++) {
-            file_list_count = MAX_ARRAY_SIZE;
-            if(!i) {
-                file_offsets[0] = offset;
-                file_lengths[0] = st_fwr_size;
-                mem_lengths = st_fwr_size;
-            }
-            for (k=0; k<MAX_ARRAY_SIZE; k++) {
-                if (i || k) {
-                    file_offsets[k] = disp + 
-			((ADIO_Offset)n_filetypes)*filetype_extent
-			+ flat_file->indices[j];
-                    file_lengths[k] = flat_file->blocklens[j];
-                    mem_lengths += file_lengths[k];
-                }
-                if (j<(flat_file->count - 1)) j++;
-                else {
-                    j = 0;
-                    n_filetypes++;
-                }
-            } /* for (k=0; k<MAX_ARRAY_SIZE; k++) */
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
-#endif
-	    NO_STALE(err_flag, fd, zoidfs_obj_ptr,
-			    zoidfs_write(zoidfs_obj_ptr,
-				    1, buf, &mem_lengths,
-				    file_list_count, 
-				    file_offsets, file_lengths, ZOIDFS_NO_OP_HINT));
-
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
-#endif
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (err_flag != ZFS_OK) {
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-						   MPIR_ERR_RECOVERABLE,
-						   myname, __LINE__,
-						   ADIOI_ZOIDFS_error_convert(err_flag),
-						   "Error in zoidfs_write", 0);
-		goto error_state;
-	    }
-	    /* --END ERROR HANDLING-- */
-	    total_bytes_written += mem_lengths;
-
-            mem_offsets += mem_lengths;
-            mem_lengths = 0;
-
-        } /* for (i=0; i<n_write_lists; i++) */
-
-        /* for file arrays smaller than MAX_ARRAY_SIZE (last write_list call) */
-        if (extra_blks) {
-            file_list_count = extra_blks;
-            if(!i) {
-                file_offsets[0] = offset;
-                file_lengths[0] = ADIOI_MIN(st_fwr_size, bufsize);
-            }
-            for (k=0; k<extra_blks; k++) {
-                if(i || k) {
-                    file_offsets[k] = disp + 
-			((ADIO_Offset)n_filetypes)*filetype_extent +
-			flat_file->indices[j];
-		    /* XXX: double-check these casts  */
-                    if (k == (extra_blks - 1)) {
-                        file_lengths[k] = bufsize 
-				- mem_lengths - mem_offsets +  (size_t)buf;
-                    }
-                    else file_lengths[k] = flat_file->blocklens[j];
-                } /* if(i || k) */
-                mem_lengths += file_lengths[k];
-                if (j<(flat_file->count - 1)) j++;
-                else {
-                    j = 0;
-                    n_filetypes++;
-                }
-            } /* for (k=0; k<extra_blks; k++) */
-
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
-#endif
-	    NO_STALE(err_flag, fd, zoidfs_obj_ptr, 
-			    zoidfs_write(zoidfs_obj_ptr, 1, 
-				    (const void **)&mem_offsets, 
-				    &mem_lengths,
-				    file_list_count, 
-				    file_offsets, file_lengths, ZOIDFS_NO_OP_HINT));
-
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
-#endif
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (err_flag != 0) {
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-						   MPIR_ERR_RECOVERABLE,
-						   myname, __LINE__,
-						   ADIOI_ZOIDFS_error_convert(err_flag),
-						   "Error in zoidfs_write", 0);
-		goto error_state;
-	    }
-	    /* --END ERROR HANDLING-- */
-	    total_bytes_written += mem_lengths;
-        }
-    } 
-    else {
-        /* noncontiguous in memory as well as in file */
-
-	flat_buf = ADIOI_Flatten_and_find(datatype);
-
-	size_wrote = 0;
-	n_filetypes = st_n_filetypes;
-	fwr_size = st_fwr_size;
-	bwr_size = flat_buf->blocklens[0];
-	buf_count = 0;
-	start_mem_offset = 0;
-	start_k = k = 0;
-	start_j = st_index;
-	max_mem_list = 0;
-	max_file_list = 0;
-
-	/* run through and file max_file_list and max_mem_list so that you 
-	   can allocate the file and memory arrays less than MAX_ARRAY_SIZE
-	   if possible */
-
-	while (size_wrote < bufsize) {
-	    k = start_k;
-	    new_buffer_write = 0;
-	    mem_list_count = 0;
-	    while ((mem_list_count < MAX_ARRAY_SIZE) && 
-		   (new_buffer_write < bufsize-size_wrote)) {
-	        /* find mem_list_count and file_list_count such that both are
-		   less than MAX_ARRAY_SIZE, the sum of their lengths are
-		   equal, and the sum of all the data written and data to be
-		   written in the next immediate write list is less than
-		   bufsize */
-	        if(mem_list_count) {
-		    if((new_buffer_write + flat_buf->blocklens[k] + 
-			size_wrote) > bufsize) {
-		        end_bwr_size = new_buffer_write + 
-			    flat_buf->blocklens[k] - (bufsize - size_wrote);
-			new_buffer_write = bufsize - size_wrote;
-		    }
-		    else {
-		        new_buffer_write += flat_buf->blocklens[k];
-			end_bwr_size = flat_buf->blocklens[k];
-		    }
-		}
-		else {
-		    if (bwr_size > (bufsize - size_wrote)) {
-		        new_buffer_write = bufsize - size_wrote;
-			bwr_size = new_buffer_write;
-		    }
-		    else new_buffer_write = bwr_size;
-		}
-		mem_list_count++;
-		k = (k + 1)%flat_buf->count;
-	     } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
-	       (new_buffer_write < bufsize-size_wrote)) */
-	    j = start_j;
-	    new_file_write = 0;
-	    file_list_count = 0;
-	    while ((file_list_count < MAX_ARRAY_SIZE) && 
-		   (new_file_write < new_buffer_write)) { 
-	        if(file_list_count) {
-		    if((new_file_write + flat_file->blocklens[j]) > 
-		       new_buffer_write) {
-		        end_fwr_size = new_buffer_write - new_file_write;
-			new_file_write = new_buffer_write;
-			j--;
-		    }
-		    else {
-		        new_file_write += flat_file->blocklens[j];
-			end_fwr_size = flat_file->blocklens[j];
-		    }
-		}
-		else {
-		    if (fwr_size > new_buffer_write) {
-		        new_file_write = new_buffer_write;
-			fwr_size = new_file_write;
-		    }
-		    else new_file_write = fwr_size;
-		}
-		file_list_count++;
-		if (j < (flat_file->count - 1)) j++;
-		else j = 0;
-		
-		k = start_k;
-		if ((new_file_write < new_buffer_write) && 
-		    (file_list_count == MAX_ARRAY_SIZE)) {
-		    new_buffer_write = 0;
-		    mem_list_count = 0;
-		    while (new_buffer_write < new_file_write) {
-		        if(mem_list_count) {
-			    if((new_buffer_write + flat_buf->blocklens[k]) >
-			       new_file_write) {
-			        end_bwr_size = new_file_write - 
-				    new_buffer_write;
-				new_buffer_write = new_file_write;
-				k--;
-			    }
-			    else {
-			        new_buffer_write += flat_buf->blocklens[k];
-				end_bwr_size = flat_buf->blocklens[k];
-			    }
-			}
-			else {
-			    new_buffer_write = bwr_size;
-			    if (bwr_size > (bufsize - size_wrote)) {
-			        new_buffer_write = bufsize - size_wrote;
-				bwr_size = new_buffer_write;
-			    }
-			}
-			mem_list_count++;
-			k = (k + 1)%flat_buf->count;
-		    } /* while (new_buffer_write < new_file_write) */
-		} /* if ((new_file_write < new_buffer_write) &&
-		     (file_list_count == MAX_ARRAY_SIZE)) */
-	    } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
-		 (new_buffer_write < bufsize-size_wrote)) */
-
-	    /*  fakes filling the writelist arrays of lengths found above  */
-	    k = start_k;
-	    j = start_j;
-	    for (i=0; i<mem_list_count; i++) {	     
-		if(i) {
-		    if (i == (mem_list_count - 1)) {
-			if (flat_buf->blocklens[k] == end_bwr_size)
-			    bwr_size = flat_buf->blocklens[(k+1)%
-							  flat_buf->count];
-			else {
-			    bwr_size = flat_buf->blocklens[k] - end_bwr_size;
-			    k--;
-			    buf_count--;
-			}
-		    }
-		}
-		buf_count++;
-		k = (k + 1)%flat_buf->count;
-	    } /* for (i=0; i<mem_list_count; i++) */
-	    for (i=0; i<file_list_count; i++) {
-		if (i) {
-		    if (i == (file_list_count - 1)) {
-			if (flat_file->blocklens[j] == end_fwr_size)
-			    fwr_size = flat_file->blocklens[(j+1)%
-							  flat_file->count];   
-			else {
-			    fwr_size = flat_file->blocklens[j] - end_fwr_size;
-			    j--;
-			}
-		    }
-		}
-		if (j < flat_file->count - 1) j++;
-		else {
-		    j = 0;
-		    n_filetypes++;
-		}
-	    } /* for (i=0; i<file_list_count; i++) */
-	    size_wrote += new_buffer_write;
-	    start_k = k;
-	    start_j = j;
-	    if (max_mem_list < mem_list_count)
-	        max_mem_list = mem_list_count;
-	    if (max_file_list < file_list_count)
-	        max_file_list = file_list_count;
-	} /* while (size_wrote < bufsize) */
-
-	/* one last check before we actually carry out the operation:
-	 * this code has hard-to-fix bugs when a noncontiguous file type has
-	 * such large pieces that the sum of the lengths of the memory type is
-	 * not larger than one of those pieces (and vice versa for large memory
-	 * types and many pices of file types.  In these cases, give up and
-	 * fall back to naive reads and writes.  The testphdf5 test created a
-	 * type with two very large memory regions and 600 very small file
-	 * regions.  The same test also created a type with one very large file
-	 * region and many (700) very small memory regions.  both cases caused
-	 * problems for this code */
-
-	if ( ( (file_list_count == 1) && 
-		    (new_file_write < flat_file->blocklens[0] ) ) ||
-		((mem_list_count == 1) && 
-		    (new_buffer_write < flat_buf->blocklens[0]) ) ||
-		((file_list_count == MAX_ARRAY_SIZE) && 
-		    (new_file_write < flat_buf->blocklens[0]) ) ||
-		( (mem_list_count == MAX_ARRAY_SIZE) &&
-		    (new_buffer_write < flat_file->blocklens[0])) )
-	{
-	    ADIOI_Delete_flattened(datatype);
-	    ADIOI_GEN_WriteStrided_naive(fd, buf, count, datatype,
-		    file_ptr_type, initial_off, status, error_code);
-	    return;
-	}
-
-
-	mem_offsets = (void *)ADIOI_Malloc(max_mem_list*sizeof(void *));
-	mem_lengths = (size_t*)ADIOI_Malloc(max_mem_list*sizeof(size_t));
-	file_offsets = (uint64_t *)ADIOI_Malloc(max_file_list*sizeof(uint64_t));
-	file_lengths = (uint64_t*)ADIOI_Malloc(max_file_list*sizeof(uint64_t));
-	    
-	size_wrote = 0;
-	n_filetypes = st_n_filetypes;
-	fwr_size = st_fwr_size;
-	bwr_size = flat_buf->blocklens[0];
-	buf_count = 0;
-	start_mem_offset = 0;
-	start_k = k = 0;
-	start_j = st_index;
-
-	/*  this section calculates mem_list_count and file_list_count
-	    and also finds the possibly odd sized last array elements
-	    in new_fwr_size and new_bwr_size  */
-	
-	while (size_wrote < bufsize) {
-	    k = start_k;
-	    new_buffer_write = 0;
-	    mem_list_count = 0;
-	    while ((mem_list_count < MAX_ARRAY_SIZE) && 
-		   (new_buffer_write < bufsize-size_wrote)) {
-	        /* find mem_list_count and file_list_count such that both are
-		   less than MAX_ARRAY_SIZE, the sum of their lengths are
-		   equal, and the sum of all the data written and data to be
-		   written in the next immediate write list is less than
-		   bufsize */
-	        if(mem_list_count) {
-		    if((new_buffer_write + flat_buf->blocklens[k] + 
-			size_wrote) > bufsize) {
-		        end_bwr_size = new_buffer_write + 
-			    flat_buf->blocklens[k] - (bufsize - size_wrote);
-			new_buffer_write = bufsize - size_wrote;
-		    }
-		    else {
-		        new_buffer_write += flat_buf->blocklens[k];
-			end_bwr_size = flat_buf->blocklens[k];
-		    }
-		}
-		else {
-		    if (bwr_size > (bufsize - size_wrote)) {
-		        new_buffer_write = bufsize - size_wrote;
-			bwr_size = new_buffer_write;
-		    }
-		    else new_buffer_write = bwr_size;
-		}
-		mem_list_count++;
-		k = (k + 1)%flat_buf->count;
-	     } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
-	       (new_buffer_write < bufsize-size_wrote)) */
-	    j = start_j;
-	    new_file_write = 0;
-	    file_list_count = 0;
-	    while ((file_list_count < MAX_ARRAY_SIZE) && 
-		   (new_file_write < new_buffer_write)) {
-	        if(file_list_count) {
-		    if((new_file_write + flat_file->blocklens[j]) > 
-		       new_buffer_write) {
-		        end_fwr_size = new_buffer_write - new_file_write;
-			new_file_write = new_buffer_write;
-			j--;
-		    }
-		    else {
-		        new_file_write += flat_file->blocklens[j];
-			end_fwr_size = flat_file->blocklens[j];
-		    }
-		}
-		else {
-		    if (fwr_size > new_buffer_write) {
-		        new_file_write = new_buffer_write;
-			fwr_size = new_file_write;
-		    }
-		    else new_file_write = fwr_size;
-		}
-		file_list_count++;
-		if (j < (flat_file->count - 1)) j++;
-		else j = 0;
-		
-		k = start_k;
-		if ((new_file_write < new_buffer_write) && 
-		    (file_list_count == MAX_ARRAY_SIZE)) {
-		    new_buffer_write = 0;
-		    mem_list_count = 0;
-		    while (new_buffer_write < new_file_write) {
-		        if(mem_list_count) {
-			    if((new_buffer_write + flat_buf->blocklens[k]) >
-			       new_file_write) {
-			        end_bwr_size = new_file_write -
-				  new_buffer_write;
-				new_buffer_write = new_file_write;
-				k--;
-			    }
-			    else {
-			        new_buffer_write += flat_buf->blocklens[k];
-				end_bwr_size = flat_buf->blocklens[k];
-			    }
-			}
-			else {
-			    new_buffer_write = bwr_size;
-			    if (bwr_size > (bufsize - size_wrote)) {
-			        new_buffer_write = bufsize - size_wrote;
-				bwr_size = new_buffer_write;
-			    }
-			}
-			mem_list_count++;
-			k = (k + 1)%flat_buf->count;
-		    } /* while (new_buffer_write < new_file_write) */
-		} /* if ((new_file_write < new_buffer_write) &&
-		     (file_list_count == MAX_ARRAY_SIZE)) */
-	    } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
-		 (new_buffer_write < bufsize-size_wrote)) */
-
-	    /*  fills the allocated writelist arrays  */
-	    k = start_k;
-	    j = start_j;
-	    for (i=0; i<mem_list_count; i++) {	     
-	        mem_offsets[i] = buf + 
-			buftype_extent* (buf_count/flat_buf->count) +
-				  flat_buf->indices[k];
-		
-		if(!i) {
-		    mem_lengths[0] = bwr_size;
-		    mem_offsets[0] += flat_buf->blocklens[k] - bwr_size;
-		}
-		else {
-		    if (i == (mem_list_count - 1)) {
-		        mem_lengths[i] = end_bwr_size;
-			if (flat_buf->blocklens[k] == end_bwr_size)
-			    bwr_size = flat_buf->blocklens[(k+1)%
-							  flat_buf->count];
-			else {
-			    bwr_size = flat_buf->blocklens[k] - end_bwr_size;
-			    k--;
-			    buf_count--;
-			}
-		    }
-		    else {
-		        mem_lengths[i] = flat_buf->blocklens[k];
-		    }
-		}
-		buf_count++;
-		k = (k + 1)%flat_buf->count;
-	    } /* for (i=0; i<mem_list_count; i++) */
-	    for (i=0; i<file_list_count; i++) {
-	        file_offsets[i] = disp + flat_file->indices[j] + 
-		    ((ADIO_Offset)n_filetypes) * filetype_extent;
-	        if (!i) {
-		    file_lengths[0] = fwr_size;
-		    file_offsets[0] += flat_file->blocklens[j] - fwr_size;
-		}
-		else {
-		    if (i == (file_list_count - 1)) {
-		        file_lengths[i] = end_fwr_size;
-			if (flat_file->blocklens[j] == end_fwr_size)
-			    fwr_size = flat_file->blocklens[(j+1)%
-							  flat_file->count];   
-			else {
-			    fwr_size = flat_file->blocklens[j] - end_fwr_size;
-			    j--;
-			}
-		    }
-		    else file_lengths[i] = flat_file->blocklens[j];
-		}
-		if (j < flat_file->count - 1) j++;
-		else {
-		    j = 0;
-		    n_filetypes++;
-		}
-	    } /* for (i=0; i<file_list_count; i++) */
-
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
-#endif
-	    NO_STALE(err_flag, fd, zoidfs_obj_ptr,
-			    zoidfs_write(zoidfs_obj_ptr, 
-				    mem_list_count, mem_offsets, mem_lengths, 
-				    file_list_count, 
-				    file_offsets, file_lengths, ZOIDFS_NO_OP_HINT));
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (err_flag != ZFS_OK) {
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-						   MPIR_ERR_RECOVERABLE,
-						   myname, __LINE__,
-						   ADIOI_ZOIDFS_error_convert(err_flag),
-						   "Error in zoidfs_write", 0);
-		goto error_state;
-	    }
-	    /* --END ERROR HANDLING-- */
-#ifdef ADIOI_MPE_LOGGING
-            MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
-#endif
-	    size_wrote += new_buffer_write;
-	    total_bytes_written += new_buffer_write; /* XXX: is this right? */
-	    start_k = k;
-	    start_j = j;
-	} /* while (size_wrote < bufsize) */
-	ADIOI_Free(mem_offsets);
-	ADIOI_Free(mem_lengths);
-    }
-    /* when incrementing fp_ind, need to also take into account the file type:
-     * consider an N-element 1-d subarray with a lb and ub: ( |---xxxxx-----|
-     * if we wrote N elements, offset needs to point at beginning of type, not
-     * at empty region at offset N+1).  
-     *
-     * As we discussed on mpich-discuss in may/june 2009, the code below might
-     * look wierd, but by putting fp_ind at the last byte written, the next
-     * time we run through the strided code we'll update the fp_ind to the
-     * right location. */
-    if (file_ptr_type == ADIO_INDIVIDUAL) {
-	fd->fp_ind = file_offsets[file_list_count-1]+
-	    file_lengths[file_list_count-1];
-    }
-    ADIOI_Free(file_offsets);
-    ADIOI_Free(file_lengths);
-
-    *error_code = MPI_SUCCESS;
-
-error_state:
-    fd->fp_sys_posn = -1;   /* set it to null. */
-
-#ifdef HAVE_STATUS_SET_BYTES
-    MPIR_Status_set_bytes(status, datatype, bufsize);
-/* This is a temporary way of filling in status. The right way is to 
-   keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
-#endif
-
-    if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
-}
diff --git a/3rd-party/romio321/adio/common/Makefile.mk b/3rd-party/romio321/adio/common/Makefile.mk
deleted file mode 100644
index 80194efe483..00000000000
--- a/3rd-party/romio321/adio/common/Makefile.mk
+++ /dev/null
@@ -1,76 +0,0 @@
-## -*- Mode: Makefile; -*-
-## vim: set ft=automake :
-##
-## (C) 2011 by Argonne National Laboratory.
-##     See COPYRIGHT in top-level directory.
-##
-
-romio_other_sources +=                  \
-    adio/common/ad_aggregate.c          \
-    adio/common/ad_aggregate_new.c      \
-    adio/common/ad_close.c              \
-    adio/common/ad_coll_build_req_new.c \
-    adio/common/ad_coll_exch_new.c      \
-    adio/common/ad_darray.c             \
-    adio/common/ad_delete.c             \
-    adio/common/ad_done.c               \
-    adio/common/ad_done_fake.c          \
-    adio/common/ad_end.c                \
-    adio/common/ad_fcntl.c              \
-    adio/common/ad_features.c           \
-    adio/common/ad_flush.c              \
-    adio/common/ad_fstype.c             \
-    adio/common/ad_get_sh_fp.c          \
-    adio/common/ad_hints.c              \
-    adio/common/ad_init.c               \
-    adio/common/ad_io_coll.c            \
-    adio/common/ad_iopen.c              \
-    adio/common/ad_iread.c              \
-    adio/common/ad_iread_coll.c         \
-    adio/common/ad_iread_fake.c         \
-    adio/common/ad_iwrite.c             \
-    adio/common/ad_iwrite_coll.c        \
-    adio/common/ad_iwrite_fake.c        \
-    adio/common/ad_open.c               \
-    adio/common/ad_opencoll.c           \
-    adio/common/ad_opencoll_failsafe.c  \
-    adio/common/ad_opencoll_scalable.c  \
-    adio/common/ad_prealloc.c           \
-    adio/common/ad_read.c               \
-    adio/common/ad_read_coll.c          \
-    adio/common/ad_read_str.c           \
-    adio/common/ad_read_str_naive.c     \
-    adio/common/ad_resize.c             \
-    adio/common/ad_seek.c               \
-    adio/common/ad_set_sh_fp.c          \
-    adio/common/ad_set_view.c           \
-    adio/common/ad_subarray.c           \
-    adio/common/ad_wait.c               \
-    adio/common/ad_wait_fake.c          \
-    adio/common/ad_write.c              \
-    adio/common/ad_write_coll.c         \
-    adio/common/ad_write_nolock.c       \
-    adio/common/ad_write_str.c          \
-    adio/common/ad_write_str_naive.c    \
-    adio/common/adi_close.c             \
-    adio/common/byte_offset.c           \
-    adio/common/cb_config_list.c        \
-    adio/common/eof_offset.c            \
-    adio/common/error.c                 \
-    adio/common/flatten.c               \
-    adio/common/get_fp_posn.c           \
-    adio/common/greq_fns.c              \
-    adio/common/heap-sort.c             \
-    adio/common/iscontig.c              \
-    adio/common/lock.c                  \
-    adio/common/malloc.c                \
-    adio/common/shfp_fname.c            \
-    adio/common/status_setb.c           \
-    adio/common/strfns.c                \
-    adio/common/system_hints.c          \
-    adio/common/hint_fns.c              \
-    adio/common/ad_threaded_io.c        \
-    adio/common/p2p_aggregation.c       \
-    adio/common/onesided_aggregation.c  \
-    adio/common/utils.c
-
diff --git a/3rd-party/romio321/adio/common/ad_aggregate.c b/3rd-party/romio321/adio/common/ad_aggregate.c
deleted file mode 100644
index 77c69bdad66..00000000000
--- a/3rd-party/romio321/adio/common/ad_aggregate.c
+++ /dev/null
@@ -1,653 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 1997-2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-
-#ifdef AGGREGATION_PROFILE
-#include "mpe.h"
-#endif
-
-#undef AGG_DEBUG
-
-/* This file contains four functions:
- *
- * ADIOI_Calc_aggregator()
- * ADIOI_Calc_file_domains()
- * ADIOI_Calc_my_req()
- * ADIOI_Calc_others_req()
- *
- * The last three of these were originally in ad_read_coll.c, but they are
- * also shared with ad_write_coll.c.  I felt that they were better kept with
- * the rest of the shared aggregation code.  
- */
-
-/* Discussion of values available from above:
- *
- * ADIO_Offset st_offsets[0..nprocs-1]
- * ADIO_Offset end_offsets[0..nprocs-1]
- *    These contain a list of start and end offsets for each process in 
- *    the communicator.  For example, an access at loc 10, size 10 would
- *    have a start offset of 10 and end offset of 19.
- * int nprocs
- *    number of processors in the collective I/O communicator
- * ADIO_Offset min_st_offset
- * ADIO_Offset fd_start[0..nprocs_for_coll-1]
- *    starting location of "file domain"; region that a given process will
- *    perform aggregation for (i.e. actually do I/O)
- * ADIO_Offset fd_end[0..nprocs_for_coll-1]
- *    start + size - 1 roughly, but it can be less, or 0, in the case of 
- *    uneven distributions
- */
-
-/* ADIOI_Calc_aggregator()
- *
- * The intention here is to implement a function which provides basically 
- * the same functionality as in Rajeev's original version of 
- * ADIOI_Calc_my_req().  He used a ceiling division approach to assign the 
- * file domains, and we use the same approach here when calculating the
- * location of an offset/len in a specific file domain.  Further we assume
- * this same distribution when calculating the rank_index, which is later
- *  used to map to a specific process rank in charge of the file domain.
- *
- * A better (i.e. more general) approach would be to use the list of file
- * domains only.  This would be slower in the case where the
- * original ceiling division was used, but it would allow for arbitrary
- * distributions of regions to aggregators.  We'd need to know the 
- * nprocs_for_coll in that case though, which we don't have now.
- *
- * Note a significant difference between this function and Rajeev's old code:
- * this code doesn't necessarily return a rank in the range
- * 0..nprocs_for_coll; instead you get something in 0..nprocs.  This is a
- * result of the rank mapping; any set of ranks in the communicator could be
- * used now.
- *
- * Returns an integer representing a rank in the collective I/O communicator.
- *
- * The "len" parameter is also modified to indicate the amount of data
- * actually available in this file domain.
- */
-int ADIOI_Calc_aggregator(ADIO_File fd,
-			 ADIO_Offset off, 
-			 ADIO_Offset min_off, 
-			 ADIO_Offset *len, 
-			 ADIO_Offset fd_size,
-			 ADIO_Offset *fd_start,
-			 ADIO_Offset *fd_end)
-{
-    int rank_index, rank;
-    ADIO_Offset avail_bytes;
-
-    ADIOI_UNREFERENCED_ARG(fd_start);
-
-    /* get an index into our array of aggregators */
-    rank_index = (int) ((off - min_off + fd_size)/ fd_size - 1);
-
-    if (fd->hints->striping_unit > 0) {
-        /* wkliao: implementation for file domain alignment
-           fd_start[] and fd_end[] have been aligned with file lock
-	   boundaries when returned from ADIOI_Calc_file_domains() so cannot
-	   just use simple arithmatic as above */
-        rank_index = 0;
-        while (off > fd_end[rank_index]) rank_index++;
-    }
-
-    /* we index into fd_end with rank_index, and fd_end was allocated to be no
-     * bigger than fd->hins->cb_nodes.   If we ever violate that, we're
-     * overrunning arrays.  Obviously, we should never ever hit this abort */
-    if (rank_index >= fd->hints->cb_nodes || rank_index < 0) {
-        FPRINTF(stderr, "Error in ADIOI_Calc_aggregator(): rank_index(%d) >= fd->hints->cb_nodes (%d) fd_size=%lld off=%lld\n",
-			rank_index,fd->hints->cb_nodes,fd_size,off);
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    /* remember here that even in Rajeev's original code it was the case that
-     * different aggregators could end up with different amounts of data to
-     * aggregate.  here we use fd_end[] to make sure that we know how much
-     * data this aggregator is working with.  
-     *
-     * the +1 is to take into account the end vs. length issue.
-     */
-    avail_bytes = fd_end[rank_index] + 1 - off;
-    if (avail_bytes < *len) {
-	/* this file domain only has part of the requested contig. region */
-	*len = avail_bytes;
-    }
-
-    /* map our index to a rank */
-    /* NOTE: FOR NOW WE DON'T HAVE A MAPPING...JUST DO 0..NPROCS_FOR_COLL */
-    rank = fd->hints->ranklist[rank_index];
-
-    return rank;
-}
-
-void ADIOI_Calc_file_domains(ADIO_Offset *st_offsets, ADIO_Offset
-			     *end_offsets, int nprocs, int nprocs_for_coll,
-			     ADIO_Offset *min_st_offset_ptr,
-			     ADIO_Offset **fd_start_ptr, ADIO_Offset 
-			     **fd_end_ptr, int min_fd_size, 
-			     ADIO_Offset *fd_size_ptr,
-			     int striping_unit)
-{
-/* Divide the I/O workload among "nprocs_for_coll" processes. This is
-   done by (logically) dividing the file into file domains (FDs); each
-   process may directly access only its own file domain. */
-
-    ADIO_Offset min_st_offset, max_end_offset, *fd_start, *fd_end, fd_size;
-    int i;
-
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5004, 0, NULL);
-#endif
-
-#ifdef AGG_DEBUG
-    FPRINTF(stderr, "ADIOI_Calc_file_domains: %d aggregator(s)\n", 
-	    nprocs_for_coll);
-#endif
-
-/* find min of start offsets and max of end offsets of all processes */
-
-    min_st_offset = st_offsets[0];
-    max_end_offset = end_offsets[0];
-
-    for (i=1; i<nprocs; i++) {
-	min_st_offset = ADIOI_MIN(min_st_offset, st_offsets[i]);
-	max_end_offset = ADIOI_MAX(max_end_offset, end_offsets[i]);
-    }
-
-/* determine the "file domain (FD)" of each process, i.e., the portion of
-   the file that will be "owned" by each process */
-
-/* partition the total file access range equally among nprocs_for_coll
-   processes */ 
-    fd_size = ((max_end_offset - min_st_offset + 1) + nprocs_for_coll -
-	       1)/nprocs_for_coll; 
-    /* ceiling division as in HPF block distribution */
-
-    /* Tweak the file domains so that no fd is smaller than a threshold.  We
-     * have to strike a balance between efficency and parallelism: somewhere
-     * between 10k processes sending 32-byte requests and one process sending a
-     * 320k request is a (system-dependent) sweet spot */
-
-    if (fd_size < min_fd_size)
-	fd_size = min_fd_size;
-
-    *fd_start_ptr = (ADIO_Offset *)
-	ADIOI_Malloc(nprocs_for_coll*sizeof(ADIO_Offset)); 
-    *fd_end_ptr = (ADIO_Offset *)
-	ADIOI_Malloc(nprocs_for_coll*sizeof(ADIO_Offset)); 
-
-    fd_start = *fd_start_ptr;
-    fd_end = *fd_end_ptr;
-
-    /* Wei-keng Liao: implementation for fild domain alignment to nearest file
-     * lock boundary (as specified by striping_unit hint).  Could also
-     * experiment with other alignment strategies here */
-    if (striping_unit > 0) {
-        ADIO_Offset end_off;
-        int         rem_front, rem_back;
-
-        /* align fd_end[0] to the nearest file lock boundary */
-        fd_start[0] = min_st_offset;
-        end_off     = fd_start[0] + fd_size;
-        rem_front   = end_off % striping_unit;
-        rem_back    = striping_unit - rem_front;
-        if (rem_front < rem_back) 
-		end_off -= rem_front;
-        else                      
-		end_off += rem_back;
-        fd_end[0] = end_off - 1;
-    
-        /* align fd_end[i] to the nearest file lock boundary */
-        for (i=1; i<nprocs_for_coll; i++) {
-            fd_start[i] = fd_end[i-1] + 1;
-            end_off     = min_st_offset + fd_size * (i+1);
-            rem_front   = end_off % striping_unit;
-            rem_back    = striping_unit - rem_front;
-            if (rem_front < rem_back) 
-		    end_off -= rem_front;
-            else                      
-		    end_off += rem_back;
-            fd_end[i] = end_off - 1;
-        }
-        fd_end[nprocs_for_coll-1] = max_end_offset;
-    }
-    else { /* no hints set: do things the 'old' way */
-        fd_start[0] = min_st_offset;
-        fd_end[0] = min_st_offset + fd_size - 1;
-
-        for (i=1; i<nprocs_for_coll; i++) {
-            fd_start[i] = fd_end[i-1] + 1;
-            fd_end[i] = fd_start[i] + fd_size - 1;
-        }
-    }
-
-/* take care of cases in which the total file access range is not
-   divisible by the number of processes. In such cases, the last
-   process, or the last few processes, may have unequal load (even 0).
-   For example, a range of 97 divided among 16 processes.
-   Note that the division is ceiling division. */
-
-    for (i=0; i<nprocs_for_coll; i++) {
-	if (fd_start[i] > max_end_offset)
-	    fd_start[i] = fd_end[i] = -1;
-	if (fd_end[i] > max_end_offset)
-	    fd_end[i] = max_end_offset;
-    }
-
-    *fd_size_ptr = fd_size;
-    *min_st_offset_ptr = min_st_offset;
-
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5005, 0, NULL);
-#endif
-}
-
-
-/* ADIOI_Calc_my_req() - calculate what portions of the access requests
- * of this process are located in the file domains of various processes
- * (including this one)
- */
-void ADIOI_Calc_my_req(ADIO_File fd, ADIO_Offset *offset_list, ADIO_Offset *len_list, 
-		       int contig_access_count, ADIO_Offset 
-		       min_st_offset, ADIO_Offset *fd_start,
-		       ADIO_Offset *fd_end, ADIO_Offset fd_size,
-                       int nprocs,
-                       int *count_my_req_procs_ptr,
-		       int **count_my_req_per_proc_ptr,
-		       ADIOI_Access **my_req_ptr,
-		       int **buf_idx_ptr)
-/* Possibly reconsider if buf_idx's are ok as int's, or should they be aints/offsets? 
-   They are used as memory buffer indices so it seems like the 2G limit is in effect */
-{
-    int *count_my_req_per_proc, count_my_req_procs, *buf_idx;
-    int i, l, proc;
-    ADIO_Offset fd_len, rem_len, curr_idx, off;
-    ADIOI_Access *my_req;
-
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5024, 0, NULL);
-#endif
-
-    *count_my_req_per_proc_ptr = (int *) ADIOI_Calloc(nprocs,sizeof(int)); 
-    count_my_req_per_proc = *count_my_req_per_proc_ptr;
-/* count_my_req_per_proc[i] gives the no. of contig. requests of this
-   process in process i's file domain. calloc initializes to zero.
-   I'm allocating memory of size nprocs, so that I can do an 
-   MPI_Alltoall later on.*/
-
-    buf_idx = (int *) ADIOI_Malloc(nprocs*sizeof(int));
-/* buf_idx is relevant only if buftype_is_contig.
-   buf_idx[i] gives the index into user_buf where data received
-   from proc. i should be placed. This allows receives to be done
-   without extra buffer. This can't be done if buftype is not contig. */
-   
-    /* initialize buf_idx to -1 */
-    for (i=0; i < nprocs; i++) buf_idx[i] = -1;
-
-    /* one pass just to calculate how much space to allocate for my_req;
-     * contig_access_count was calculated way back in ADIOI_Calc_my_off_len()
-     */
-    for (i=0; i < contig_access_count; i++) {
-	/* short circuit offset/len processing if len == 0 
-	 * 	(zero-byte  read/write */
-	if (len_list[i] == 0) 
-		continue;
-	off = offset_list[i];
-	fd_len = len_list[i];
-	/* note: we set fd_len to be the total size of the access.  then
-	 * ADIOI_Calc_aggregator() will modify the value to return the 
-	 * amount that was available from the file domain that holds the
-	 * first part of the access.
-	 */
-	proc = ADIOI_Calc_aggregator(fd, off, min_st_offset, &fd_len, fd_size, 
-				     fd_start, fd_end);
-	count_my_req_per_proc[proc]++;
-
-	/* figure out how much data is remaining in the access (i.e. wasn't 
-	 * part of the file domain that had the starting byte); we'll take 
-	 * care of this data (if there is any) in the while loop below.
-	 */
-	rem_len = len_list[i] - fd_len;
-
-	while (rem_len != 0) {
-	    off += fd_len; /* point to first remaining byte */
-	    fd_len = rem_len; /* save remaining size, pass to calc */
-	    proc = ADIOI_Calc_aggregator(fd, off, min_st_offset, &fd_len, 
-					 fd_size, fd_start, fd_end);
-
-	    count_my_req_per_proc[proc]++;
-	    rem_len -= fd_len; /* reduce remaining length by amount from fd */
-	}
-    }
-
-/* now allocate space for my_req, offset, and len */
-
-    *my_req_ptr = (ADIOI_Access *)
-	ADIOI_Malloc(nprocs*sizeof(ADIOI_Access)); 
-    my_req = *my_req_ptr;
-
-    count_my_req_procs = 0;
-    for (i=0; i < nprocs; i++) {
-	if (count_my_req_per_proc[i]) {
-	    my_req[i].offsets = (ADIO_Offset *)
-		ADIOI_Malloc(count_my_req_per_proc[i] * sizeof(ADIO_Offset));
-	    my_req[i].lens =
-		ADIOI_Malloc(count_my_req_per_proc[i] * sizeof(ADIO_Offset));
-	    count_my_req_procs++;
-	}	    
-	my_req[i].count = 0;  /* will be incremented where needed
-				      later */
-    }
-
-/* now fill in my_req */
-    curr_idx = 0;
-    for (i=0; i<contig_access_count; i++) { 
-	/* short circuit offset/len processing if len == 0 
-	 * 	(zero-byte  read/write */
-	if (len_list[i] == 0)
-		continue;
-	off = offset_list[i];
-	fd_len = len_list[i];
-	proc = ADIOI_Calc_aggregator(fd, off, min_st_offset, &fd_len, fd_size, 
-				     fd_start, fd_end);
-
-	/* for each separate contiguous access from this process */
-	if (buf_idx[proc] == -1) 
-  {
-    ADIOI_Assert(curr_idx == (int) curr_idx);
-    buf_idx[proc] = (int) curr_idx;
-  }
-
-	l = my_req[proc].count;
-	curr_idx += fd_len; 
-
-	rem_len = len_list[i] - fd_len;
-
-	/* store the proc, offset, and len information in an array
-         * of structures, my_req. Each structure contains the 
-         * offsets and lengths located in that process's FD, 
-	 * and the associated count. 
-	 */
-	my_req[proc].offsets[l] = off;
-	my_req[proc].lens[l] = fd_len;
-	my_req[proc].count++;
-
-	while (rem_len != 0) {
-	    off += fd_len;
-	    fd_len = rem_len;
-	    proc = ADIOI_Calc_aggregator(fd, off, min_st_offset, &fd_len, 
-					 fd_size, fd_start, fd_end);
-
-	    if (buf_idx[proc] == -1) 
-      {
-        ADIOI_Assert(curr_idx == (int) curr_idx);
-        buf_idx[proc] = (int) curr_idx;
-      }
-
-	    l = my_req[proc].count;
-	    curr_idx += fd_len;
-	    rem_len -= fd_len;
-
-	    my_req[proc].offsets[l] = off;
-	    my_req[proc].lens[l] = fd_len;
-	    my_req[proc].count++;
-	}
-    }
-
-#ifdef AGG_DEBUG
-    for (i=0; i<nprocs; i++) {
-	if (count_my_req_per_proc[i] > 0) {
-	    FPRINTF(stdout, "data needed from %d (count = %d):\n", i, 
-		    my_req[i].count);
-	    for (l=0; l < my_req[i].count; l++) {
-		FPRINTF(stdout, "   off[%d] = %lld, len[%d] = %d\n", l,
-			my_req[i].offsets[l], l, my_req[i].lens[l]);
-	    }
-	FPRINTF(stdout, "buf_idx[%d] = 0x%x\n", i, buf_idx[i]);
-	}
-    }
-#endif
-
-    *count_my_req_procs_ptr = count_my_req_procs;
-    *buf_idx_ptr = buf_idx;
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5025, 0, NULL);
-#endif
-}
-
-
-
-void ADIOI_Calc_others_req(ADIO_File fd, int count_my_req_procs, 
-				int *count_my_req_per_proc,
-				ADIOI_Access *my_req, 
-				int nprocs, int myrank,
-				int *count_others_req_procs_ptr,
-				ADIOI_Access **others_req_ptr)  
-{
-/* determine what requests of other processes lie in this process's
-   file domain */
-
-/* count_others_req_procs = number of processes whose requests lie in
-   this process's file domain (including this process itself) 
-   count_others_req_per_proc[i] indicates how many separate contiguous
-   requests of proc. i lie in this process's file domain. */
-
-    int *count_others_req_per_proc, count_others_req_procs;
-    int i, j;
-    MPI_Request *requests;
-    MPI_Status *statuses;
-    ADIOI_Access *others_req;
-
-/* first find out how much to send/recv and from/to whom */
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5026, 0, NULL);
-#endif
-    count_others_req_per_proc = (int *) ADIOI_Malloc(nprocs*sizeof(int));
-
-    MPI_Alltoall(count_my_req_per_proc, 1, MPI_INT,
-		 count_others_req_per_proc, 1, MPI_INT, fd->comm);
-
-    *others_req_ptr = (ADIOI_Access *)
-	ADIOI_Malloc(nprocs*sizeof(ADIOI_Access)); 
-    others_req = *others_req_ptr;
-
-    count_others_req_procs = 0;
-    for (i=0; i<nprocs; i++) {
-	if (count_others_req_per_proc[i]) {
-	    others_req[i].count = count_others_req_per_proc[i];
-	    others_req[i].offsets = (ADIO_Offset *)
-		ADIOI_Malloc(count_others_req_per_proc[i]*sizeof(ADIO_Offset));
-	    others_req[i].lens =
-		ADIOI_Malloc(count_others_req_per_proc[i]*sizeof(ADIO_Offset));
-	    others_req[i].mem_ptrs = (MPI_Aint *)
-		ADIOI_Malloc(count_others_req_per_proc[i]*sizeof(MPI_Aint)); 
-	    count_others_req_procs++;
-	}
-	else others_req[i].count = 0;
-    }
-    
-/* now send the calculated offsets and lengths to respective processes */
-
-    requests = (MPI_Request *)
-	ADIOI_Malloc(1+2*(count_my_req_procs+count_others_req_procs)*sizeof(MPI_Request)); 
-/* +1 to avoid a 0-size malloc */
-
-    j = 0;
-    for (i=0; i<nprocs; i++) {
-	if (others_req[i].count) {
-	    MPI_Irecv(others_req[i].offsets, others_req[i].count, 
-                      ADIO_OFFSET, i, i+myrank, fd->comm, &requests[j]);
-	    j++;
-	    MPI_Irecv(others_req[i].lens, others_req[i].count, 
-                      ADIO_OFFSET, i, i+myrank+1, fd->comm, &requests[j]);
-	    j++;
-	}
-    }
-
-    for (i=0; i < nprocs; i++) {
-	if (my_req[i].count) {
-	    MPI_Isend(my_req[i].offsets, my_req[i].count, 
-                      ADIO_OFFSET, i, i+myrank, fd->comm, &requests[j]);
-	    j++;
-	    MPI_Isend(my_req[i].lens, my_req[i].count, 
-                      ADIO_OFFSET, i, i+myrank+1, fd->comm, &requests[j]);
-	    j++;
-	}
-    }
-
-    if (j) {
-	statuses = (MPI_Status *) ADIOI_Malloc(j * sizeof(MPI_Status));
-	MPI_Waitall(j, requests, statuses);
-	ADIOI_Free(statuses);
-    }
-
-    ADIOI_Free(requests);
-    ADIOI_Free(count_others_req_per_proc);
-
-    *count_others_req_procs_ptr = count_others_req_procs;
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5027, 0, NULL);
-#endif
-}
-
-
-/* Nonblocking version of ADIOI_Calc_others_req().
-   It consists of three functions - ADIOI_Icalc_others_req(),
-   ADIOI_Icalc_others_req_main(), and ADIOI_Icalc_others_req_fini(). */
-void ADIOI_Icalc_others_req(ADIOI_NBC_Request *nbc_req, int *error_code)
-{
-    ADIOI_Icalc_others_req_vars *vars = nbc_req->cor_vars;
-
-    /* count_others_req_per_proc[i] indicates how many separate contiguous
-       requests of proc. i lie in this process's file domain. */
-
-    /* first find out how much to send/recv and from/to whom */
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event(5026, 0, NULL);
-#endif
-    vars->count_others_req_per_proc =
-        (int *)ADIOI_Malloc(vars->nprocs * sizeof(int));
-
-    *error_code = MPI_Ialltoall(vars->count_my_req_per_proc, 1, MPI_INT,
-            vars->count_others_req_per_proc, 1, MPI_INT, vars->fd->comm,
-            &vars->req1);
-
-    if (nbc_req->rdwr == ADIOI_READ) {
-        nbc_req->data.rd.state = ADIOI_IRC_STATE_ICALC_OTHERS_REQ;
-    } else {
-        ADIOI_Assert(nbc_req->rdwr == ADIOI_WRITE);
-        nbc_req->data.wr.state = ADIOI_IWC_STATE_ICALC_OTHERS_REQ;
-    }
-}
-
-void ADIOI_Icalc_others_req_main(ADIOI_NBC_Request *nbc_req, int *error_code)
-{
-    ADIOI_Icalc_others_req_vars *vars = nbc_req->cor_vars;
-    ADIO_File fd = vars->fd;
-    int count_my_req_procs = vars->count_my_req_procs;
-    ADIOI_Access *my_req = vars->my_req;
-    int nprocs = vars->nprocs;
-    int myrank = vars->myrank;
-    ADIOI_Access **others_req_ptr = vars->others_req_ptr;
-
-    /* determine what requests of other processes lie in this process's
-       file domain */
-
-    /* count_others_req_procs = number of processes whose requests lie in
-       this process's file domain (including this process itself)
-       count_others_req_per_proc[i] indicates how many separate contiguous
-       requests of proc. i lie in this process's file domain. */
-
-    int *count_others_req_per_proc = vars->count_others_req_per_proc;
-    int count_others_req_procs;
-    int i, j;
-    ADIOI_Access *others_req;
-
-    *others_req_ptr = (ADIOI_Access *)ADIOI_Malloc(nprocs*sizeof(ADIOI_Access));
-    others_req = *others_req_ptr;
-
-    count_others_req_procs = 0;
-    for (i = 0; i < nprocs; i++) {
-        if (count_others_req_per_proc[i]) {
-            others_req[i].count = count_others_req_per_proc[i];
-            others_req[i].offsets = (ADIO_Offset *)
-                ADIOI_Malloc(count_others_req_per_proc[i]*sizeof(ADIO_Offset));
-            others_req[i].lens =
-                ADIOI_Malloc(count_others_req_per_proc[i]*sizeof(ADIO_Offset));
-            others_req[i].mem_ptrs = (MPI_Aint *)
-                ADIOI_Malloc(count_others_req_per_proc[i]*sizeof(MPI_Aint));
-            count_others_req_procs++;
-        }
-        else others_req[i].count = 0;
-    }
-    vars->count_others_req_procs = count_others_req_procs;
-
-    /* now send the calculated offsets and lengths to respective processes */
-
-    vars->req2 = (MPI_Request *)
-        ADIOI_Malloc(1+2*(count_my_req_procs+count_others_req_procs)
-                     *sizeof(MPI_Request));
-    /* +1 to avoid a 0-size malloc */
-
-    j = 0;
-    for (i = 0; i < nprocs; i++) {
-        if (others_req[i].count) {
-            MPI_Irecv(others_req[i].offsets, others_req[i].count,
-                    ADIO_OFFSET, i, i+myrank, fd->comm, &vars->req2[j]);
-            j++;
-            MPI_Irecv(others_req[i].lens, others_req[i].count,
-                    ADIO_OFFSET, i, i+myrank+1, fd->comm, &vars->req2[j]);
-            j++;
-        }
-    }
-
-    for (i=0; i < nprocs; i++) {
-        if (my_req[i].count) {
-            MPI_Isend(my_req[i].offsets, my_req[i].count,
-                    ADIO_OFFSET, i, i+myrank, fd->comm, &vars->req2[j]);
-            j++;
-            MPI_Isend(my_req[i].lens, my_req[i].count,
-                    ADIO_OFFSET, i, i+myrank+1, fd->comm, &vars->req2[j]);
-            j++;
-        }
-    }
-
-    /* keep the number of requests */
-    vars->num_req2 = j;
-
-    if (nbc_req->rdwr == ADIOI_READ) {
-        nbc_req->data.rd.state = ADIOI_IRC_STATE_ICALC_OTHERS_REQ_MAIN;
-    } else {
-        ADIOI_Assert(nbc_req->rdwr == ADIOI_WRITE);
-        nbc_req->data.wr.state = ADIOI_IWC_STATE_ICALC_OTHERS_REQ_MAIN;
-    }
-}
-
-void ADIOI_Icalc_others_req_fini(ADIOI_NBC_Request *nbc_req, int *error_code)
-{
-    ADIOI_Icalc_others_req_vars *vars = nbc_req->cor_vars;
-    void (*next_fn)(ADIOI_NBC_Request *, int *);
-
-    ADIOI_Free(vars->req2);
-    ADIOI_Free(vars->count_others_req_per_proc);
-
-    *vars->count_others_req_procs_ptr = vars->count_others_req_procs;
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event(5027, 0, NULL);
-#endif
-    /* end of the calculation */
-
-    next_fn = vars->next_fn;
-
-    /* free the struct for parameters and variables */
-    ADIOI_Free(vars);
-    nbc_req->cor_vars = NULL;
-
-    /* move to the next function */
-    next_fn(nbc_req, error_code);
-}
-
diff --git a/3rd-party/romio321/adio/common/ad_aggregate_new.c b/3rd-party/romio321/adio/common/ad_aggregate_new.c
deleted file mode 100644
index aca60b27023..00000000000
--- a/3rd-party/romio321/adio/common/ad_aggregate_new.c
+++ /dev/null
@@ -1,284 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 2008 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-#ifdef AGGREGATION_PROFILE
-#include "mpe.h"
-#endif
-#include <assert.h>
-/* #define DEBUG */
-
-void ADIOI_Calc_file_realms_user_size (ADIO_File fd, int fr_size,
-				       int nprocs_for_coll,
-				       ADIO_Offset *file_realm_st_offs,
-				       MPI_Datatype *file_realm_types);
-void ADIOI_Calc_file_realms_aar (ADIO_File fd, int nprocs_for_coll,
-				 int pfr_enabled,
-				 ADIO_Offset min_st_offset,
-				 ADIO_Offset max_end_offset,
-				 ADIO_Offset *file_realm_st_offs,
-				 MPI_Datatype *file_realm_types);
-void ADIOI_Calc_file_realms_fsize (ADIO_File fd,
-				   int nprocs_for_coll,
-				   ADIO_Offset max_end_offset,
-				   ADIO_Offset *file_realm_st_offs,
-				   MPI_Datatype *file_realm_types);
-void ADIOI_Create_fr_simpletype (int size, int nprocs_for_coll,
-			      MPI_Datatype *simpletype);
-static void align_fr (int fr_size, ADIO_Offset fr_off, int alignment,
-	       int *aligned_fr_size, ADIO_Offset *aligned_fr_off);
-void ADIOI_Verify_fr (int nprocs_for_coll, ADIO_Offset *file_realm_st_offs,
-		      MPI_Datatype *file_realm_types);
-
-void ADIOI_Calc_file_realms (ADIO_File fd, ADIO_Offset min_st_offset,
-			     ADIO_Offset max_end_offset)
-{
-    int nprocs_for_coll;
-    int file_realm_calc_type;
-    
-    MPI_Datatype *file_realm_types = NULL;
-    ADIO_Offset *file_realm_st_offs = NULL;
-
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5004, 0, NULL);
-#endif
-#ifdef DEBUG
-    printf ("ADIOI_Calc_file_realms\n");
-#endif
-    
-    nprocs_for_coll = fd->hints->cb_nodes;    
-    file_realm_calc_type = fd->hints->cb_fr_type;
-
-    /* If PFRs are disabled we know these pointers are not allocated */
-    if (fd->hints->cb_pfr != ADIOI_HINT_ENABLE) {
-	fd->file_realm_st_offs = NULL;
-	fd->file_realm_types = NULL;
-    }
-
-    if (nprocs_for_coll == 1) {
-	/* if there's only one aggregator, we can reset the file
-	 * realms every single time */
-	if (fd->file_realm_st_offs == NULL)
-	{
-	    file_realm_st_offs = (ADIO_Offset *)
-		ADIOI_Malloc (sizeof(ADIO_Offset));
-	    file_realm_types   = (MPI_Datatype *)
-		ADIOI_Malloc (sizeof(MPI_Datatype));
-	}
-	else
-	{
-	    file_realm_st_offs = fd->file_realm_st_offs;
-	    file_realm_types   = fd->file_realm_types;
-	}
-	*file_realm_st_offs = min_st_offset;
-	MPI_Type_contiguous ((max_end_offset - min_st_offset + 1), MPI_BYTE,
-			     file_realm_types);
-	MPI_Type_commit (file_realm_types);
-	ADIOI_Add_contig_flattened (*file_realm_types);
-    }
-    else if (fd->file_realm_st_offs == NULL) {
-	file_realm_st_offs = (ADIO_Offset *)
-	    ADIOI_Malloc (nprocs_for_coll * sizeof(ADIO_Offset));
-	file_realm_types   = (MPI_Datatype *)
-	    ADIOI_Malloc (nprocs_for_coll * sizeof(MPI_Datatype));
-	
-	if (file_realm_calc_type == ADIOI_FR_AAR) {
-	    ADIOI_Calc_file_realms_aar (fd, nprocs_for_coll,
-					fd->hints->cb_pfr,
-					min_st_offset, max_end_offset,
-					file_realm_st_offs, file_realm_types);
-	    /* flatten file realm datatype for future use - only one
-	     * because all are the same*/
-	    ADIOI_Flatten_datatype (file_realm_types[0]);
-	}
-	else if (file_realm_calc_type == ADIOI_FR_FSZ) {
-	    ADIOI_Calc_file_realms_fsize (fd, nprocs_for_coll, max_end_offset,
-					  file_realm_st_offs,
-					  file_realm_types);
-	    /* flatten file realm datatype for future use - only one
-	     * because all are the same*/
-	    ADIOI_Flatten_datatype (file_realm_types[0]);
-	}
-	else if (file_realm_calc_type == ADIOI_FR_USR_REALMS) {
-	    /* copy user provided realm datatypes and realm offsets in
-	     * hints to file descriptor. may also want to verify that
-	     * the provided file realms are covering (for pfr at
-	     * least) and non-overlapping */
-	}
-	else if (file_realm_calc_type > 0) {
-	    ADIOI_Calc_file_realms_user_size (fd, file_realm_calc_type,
-					      nprocs_for_coll,
-					      file_realm_st_offs,
-					      file_realm_types);
-	    /* flatten file realm datatype for future use - only one
-	     * because all are the same */
-	    ADIOI_Flatten_datatype (file_realm_types[0]);
-	}
-    }
-    fd->file_realm_st_offs = file_realm_st_offs;
-    fd->file_realm_types   = file_realm_types;
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5005, 0, NULL);
-#endif
-}
-
-void ADIOI_Calc_file_realms_user_size (ADIO_File fd, int fr_size,
-				       int nprocs_for_coll,
-				       ADIO_Offset *file_realm_st_offs,
-				       MPI_Datatype *file_realm_types)
-{
-    int i;
-    int aligned_fr_size;
-    ADIO_Offset aligned_fr_off;
-    MPI_Datatype simpletype;
-
-    align_fr(fr_size, 0, fd->hints->cb_fr_alignment, &aligned_fr_size,
-	     &aligned_fr_off);
-    fr_size = aligned_fr_size;
-    ADIOI_Create_fr_simpletype (fr_size, nprocs_for_coll, &simpletype);
-
-    if (fd->hints->cb_pfr == ADIOI_HINT_ENABLE)
-	file_realm_st_offs[0] = 0;
-    else
-	file_realm_st_offs[0] = aligned_fr_off;
-    file_realm_types[0]   = simpletype;
-#ifdef DEBUG
-    printf ("file_realm[0] = (%lld, %d)\n", file_realm_st_offs[0],
-	    fr_size);
-#endif
-
-    for (i=1; i < nprocs_for_coll; i++)
-    {
-	file_realm_st_offs[i] = file_realm_st_offs[i-1] + fr_size;
-	file_realm_types[i]   = simpletype;
-#ifdef DEBUG
-	printf ("file_realm[%d] = (%lld, %d)\n", i, file_realm_st_offs[i],
-		aligned_fr_size);
-#endif
-    }
-}
-
-/* takes an extra romio_cb_pfr param to decide whether file realms
- * should start at byte 0 of the file*/
-void ADIOI_Calc_file_realms_aar (ADIO_File fd, int nprocs_for_coll, int cb_pfr,
-				 ADIO_Offset min_st_offset,
-				 ADIO_Offset max_end_offset,
-				 ADIO_Offset *file_realm_st_offs,
-				 MPI_Datatype *file_realm_types)
-{
-    int fr_size, aligned_fr_size, i;
-    MPI_Datatype simpletype;
-    ADIO_Offset aligned_start_off;
-    char value[9];
-
-    fr_size = (max_end_offset - min_st_offset + nprocs_for_coll) /
-	nprocs_for_coll;
-    align_fr(fr_size, min_st_offset, fd->hints->cb_fr_alignment,
-	     &aligned_fr_size, &aligned_start_off);
-    fr_size = aligned_fr_size;
-    ADIOI_Create_fr_simpletype (fr_size, nprocs_for_coll, &simpletype);
-    if (cb_pfr == ADIOI_HINT_ENABLE)
-	file_realm_st_offs[0] = 0;
-    else
-	file_realm_st_offs[0] = aligned_start_off;
-    file_realm_types[0]   = simpletype;
-
-#ifdef DEBUG
-    printf ("file_realm[0] = (%lld, %d)\n", file_realm_st_offs[0],
-	    fr_size);
-#endif
-    for (i=1; i < nprocs_for_coll; i++)
-    {
-	file_realm_st_offs[i] = file_realm_st_offs[i-1] + fr_size;
-	file_realm_types[i]   = simpletype;
-#ifdef DEBUG
-	printf ("file_realm[%d] = (%lld, %d)\n", i, file_realm_st_offs[i],
-		fr_size);
-#endif
-    }
-    if (fd->hints->cb_pfr == ADIOI_HINT_ENABLE) {
-	sprintf (value, "%d", fr_size);
-	ADIOI_Info_set (fd->info, "romio_cb_fr_type", value);
-    }
-}
-
-void ADIOI_Calc_file_realms_fsize (ADIO_File fd, int nprocs_for_coll,
-				   ADIO_Offset max_end_offset,
-				   ADIO_Offset *file_realm_st_offs,
-				   MPI_Datatype *file_realm_types)
-{
-    int fr_size, aligned_fr_size, error_code, i;
-    int fsize;
-    ADIO_Offset aligned_fr_off;
-    ADIO_Fcntl_t fcntl_struct;
-    MPI_Datatype simpletype;
-
-    ADIO_Fcntl (fd, ADIO_FCNTL_GET_FSIZE, &fcntl_struct, &error_code);
-    
-    /* use impending file size since a write call may lengthen the file */
-    fsize = ADIOI_MAX (fcntl_struct.fsize, max_end_offset+1);
-    fr_size = (fsize + nprocs_for_coll - 1) / nprocs_for_coll;
-    align_fr(fr_size, 0, fd->hints->cb_fr_alignment,
-	     &aligned_fr_size, &aligned_fr_off);
-    ADIOI_Create_fr_simpletype (fr_size, nprocs_for_coll, &simpletype);
-
-    for (i=0; i < nprocs_for_coll; i++)
-    {
-	file_realm_st_offs[i] = fr_size * i;
-	file_realm_types[i]   = simpletype;
-    }
-}
-
-/* creates a datatype with an empty trailing edge */
-void ADIOI_Create_fr_simpletype (int size, int nprocs_for_coll,
-			      MPI_Datatype *simpletype)
-{
-    int count=1, blocklens[1];
-    MPI_Aint indices[1];
-    MPI_Datatype old_types[1];
-    MPI_Datatype inttype;
-
-    blocklens[0] = size;
-    indices[0]   = 0;
-    old_types[0] = MPI_BYTE;
-
-    MPI_Type_create_struct (count, blocklens, indices, old_types, &inttype);
-    MPI_Type_create_resized (inttype, 0, size*nprocs_for_coll, simpletype);
-    MPI_Type_free (&inttype);
-
-    MPI_Type_commit (simpletype);
-}
-
-/* Verify that file realms are covering (PFRs) and non-overlapping */
-void ADIOI_Verify_fr (int nprocs_for_coll, ADIO_Offset *file_realm_st_offs,
-		      MPI_Datatype *file_realm_types)
-{
-}
-
-int ADIOI_Agg_idx (int rank, ADIO_File fd) {
-    int i, cb_nodes, *ranklist;
-    cb_nodes = fd->hints->cb_nodes;
-    ranklist = fd->hints->ranklist;
-
-    for (i=0; i<cb_nodes; i++) {
-	if (ranklist[i])
-	    return i;
-    }
-    return -1;
-}
-
-static void align_fr (int fr_size, ADIO_Offset fr_off, int alignment,
-	       int *aligned_fr_size, ADIO_Offset *aligned_fr_off) {
-    *aligned_fr_off = fr_off - (fr_off % alignment);
-    *aligned_fr_size = ((fr_off + fr_size) / alignment) * alignment - 
-	*aligned_fr_off;
-    if ((fr_off + fr_size) % alignment)
-	*aligned_fr_size += alignment;
-
-    assert(!((*aligned_fr_off % alignment) ||
-	     (*aligned_fr_size % alignment)));
-}
diff --git a/3rd-party/romio321/adio/common/ad_close.c b/3rd-party/romio321/adio/common/ad_close.c
deleted file mode 100644
index 0c39d9e9471..00000000000
--- a/3rd-party/romio321/adio/common/ad_close.c
+++ /dev/null
@@ -1,118 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-
-void ADIO_Close(ADIO_File fd, int *error_code)
-{
-    int i, j, k, combiner, myrank, err, is_contig;
-    static char myname[] = "ADIO_CLOSE";
-
-    if (fd->async_count) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO, "**io",
-					   "**io %s", strerror(errno));
-	return;
-    }
-
-    /* because of deferred open, this warants a bit of explaining.  First, if
-     * we've done aggregation,
-     * then close the file.  Then, if any process left has done independent
-     * i/o, close the file.  Otherwise, we'll skip the fs-specific close and
-     * just say everything is a-ok.
-     *
-     * XXX: is it ok for those processes with a "real" communicator and those
-     * with "MPI_COMM_SELF" to both call ADIOI_xxx_Close at the same time ?
-     * everyone who ever opened the file will close it. Is order important? Is
-     * timing important?
-     */
-    if (fd->hints->deferred_open && fd->is_agg) {
-	    (*(fd->fns->ADIOI_xxx_Close))(fd, error_code);
-    } else {
-	    if(fd->is_open)  {
-		    (*(fd->fns->ADIOI_xxx_Close))(fd, error_code);
-	    } else {
-		    *error_code = MPI_SUCCESS;
-	    }
-	    
-    }
-
-    if (fd->access_mode & ADIO_DELETE_ON_CLOSE) {
-	/* if we are doing aggregation and deferred open, then it's possible
-	 * that rank 0 does not have access to the file. make sure only an
-	 * aggregator deletes the file.*/
-	MPI_Comm_rank(fd->comm, &myrank);
-	if (myrank == fd->hints->ranklist[0]) {
-		ADIO_Delete(fd->filename, &err);
-	}
-	MPI_Barrier(fd->comm);
-    }
-
-    if (fd->fortran_handle != -1) {
-	ADIOI_Ftable[fd->fortran_handle] = MPI_FILE_NULL;
-    }
-
-    if (fd->hints) ADIOI_Free(fd->hints->ranklist);
-    if (fd->hints && fd->hints->cb_config_list) ADIOI_Free(fd->hints->cb_config_list);
-
-    /* This BlueGene platform-specific free must be done in the common code
-     * because the malloc's for these hint data structures are done at the
-     * scope of ADIO_Open within the SetInfo call (ADIOI_GPFS_SetInfo which
-     * calls ADIOI_BG_gen_agg_ranklist).  They cannot be done in the
-     * ADIOI_GPFS_Close because of the file creation case where the
-     * ADIOI_GPFS_Close and re-open via ADIOI_GPFS_Open are done which results
-     * in a double-free - ADIOI_GPFS_Open does not redo the SetInfo...  */
-#ifdef BGQPLATFORM
-    if (fd->hints && fd->hints->fs_hints.bg.bridgelist) ADIOI_Free(fd->hints->fs_hints.bg.bridgelist);
-    if (fd->hints && fd->hints->fs_hints.bg.bridgelistnum) ADIOI_Free(fd->hints->fs_hints.bg.bridgelistnum);
-#endif
-
-    /* Persistent File Realms */
-    if (fd->hints->cb_pfr == ADIOI_HINT_ENABLE) {
-	/* AAR, FSIZE, and User provided uniform File realms */
-	if (1) {
-	    ADIOI_Delete_flattened (fd->file_realm_types[0]);
-	    MPI_Type_free (&fd->file_realm_types[0]);
-	}
-	else {
-	    for (i=0; i<fd->hints->cb_nodes; i++) {
-		ADIOI_Datatype_iscontig(fd->file_realm_types[i], &is_contig);
-		if (!is_contig)
-		    ADIOI_Delete_flattened(fd->file_realm_types[i]);
-		MPI_Type_free (&fd->file_realm_types[i]);
-	    }
-	}
-	ADIOI_Free(fd->file_realm_st_offs);
-	ADIOI_Free(fd->file_realm_types);
-    }
-    ADIOI_Free(fd->hints);
-
-
-
-    MPI_Comm_free(&(fd->comm));
-    ADIOI_Free(fd->filename); 
-
-    MPI_Type_get_envelope(fd->etype, &i, &j, &k, &combiner);
-    if (combiner != MPI_COMBINER_NAMED) MPI_Type_free(&(fd->etype));
-
-    ADIOI_Datatype_iscontig(fd->filetype, &is_contig);
-    if (!is_contig) ADIOI_Delete_flattened(fd->filetype);
-
-    MPI_Type_get_envelope(fd->filetype, &i, &j, &k, &combiner);
-    if (combiner != MPI_COMBINER_NAMED) MPI_Type_free(&(fd->filetype));
-
-    MPI_Info_free(&(fd->info));
-
-    ADIOI_Free(fd->io_buf);
-    ADIOI_OneSidedCleanup(fd);
-
-    /* memory for fd is freed in MPI_File_close */
-}
diff --git a/3rd-party/romio321/adio/common/ad_coll_build_req_new.c b/3rd-party/romio321/adio/common/ad_coll_build_req_new.c
deleted file mode 100644
index a15d61cc329..00000000000
--- a/3rd-party/romio321/adio/common/ad_coll_build_req_new.c
+++ /dev/null
@@ -1,2084 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include <assert.h>
-#include "adio.h"
-#include "adio_extern.h"
-#ifdef AGGREGATION_PROFILE
-#include "mpe.h"
-#endif
-#include "heap-sort.h"
-
-/*
-#define DEBUG1
-#define DEBUG2
-#define DEBUG3
-*/
-/* #define DEBUG_HEAP */
-
-#define DTYPE_SKIP
-
-#ifdef DEBUG3
-static char *off_type_name[MAX_OFF_TYPE] = {"TEMP_OFFSETS",
-				     "REAL_OFFSETS"};
-#endif
-
-/* Simple function to return the size of the view_state. */
-static inline ADIO_Offset view_state_get_cur_sz(view_state *tmp_view_state_p,
-                                                int op_type)
-{
-    flatten_state *tmp_state_p = NULL;
-    switch(op_type)
-    {   
-        case TEMP_OFF:
-            tmp_state_p = &(tmp_view_state_p->tmp_state);
-            break;
-        case REAL_OFF:
-            tmp_state_p = &(tmp_view_state_p->cur_state);
-            break;
-        default:
-            fprintf(stderr, "op_type invalid\n");
-    }
-    return tmp_state_p->cur_sz;
-}
-
-/* Simple function to return the len of the next piece of the view_state. */
-static inline ADIO_Offset view_state_get_next_len(view_state *tmp_view_state_p,
-						  int op_type)
-{
-    flatten_state *tmp_state_p = NULL;
-    switch(op_type)
-    {
-	case TEMP_OFF:
-	    tmp_state_p = &(tmp_view_state_p->tmp_state);
-	    break;
-	case REAL_OFF:
-	    tmp_state_p = &(tmp_view_state_p->cur_state);
-	    break;
-	default:
-	    fprintf(stderr, "op_type invalid\n");
-    }
-    return (ADIO_Offset) 
-	tmp_view_state_p->flat_type_p->blocklens[tmp_state_p->idx] -
-	tmp_state_p->cur_reg_off;
-}
-
-/* Add up to a region of a file view and no larger than a max size.
- * The view_state is always consistent with the abs_off and where the
- * index and cur_reg_off point to.  The regions should be coalesced if
- * possible later on. */
-static inline int view_state_add_region(
-    ADIO_Offset max_sz,
-    view_state *tmp_view_state_p, 
-    ADIO_Offset *st_reg_p,
-    ADIO_Offset *tmp_reg_sz_p,
-    int op_type)
-{
-    ADIOI_Flatlist_node *tmp_flat_type_p = NULL;
-    flatten_state *tmp_state_p = NULL;
-    int64_t data_sz = 0;
-
-#ifdef AGGREGATION_PROFILE
-    /* MPE_Log_event (5020, 0, NULL); */
-#endif
-
-    switch(op_type)
-    {
-	case TEMP_OFF:
-	    tmp_state_p = &(tmp_view_state_p->tmp_state);
-	    break;
-	case REAL_OFF:
-	    tmp_state_p = &(tmp_view_state_p->cur_state);
-	    break;
-	default:
-	    fprintf(stderr, "op_type invalid\n");
-    }
-
-    tmp_flat_type_p = tmp_view_state_p->flat_type_p;
-
-    *st_reg_p = tmp_state_p->abs_off;
-
-    /* Should be looking at some data (or it's a zero len blocklens
-     * (i.e. placeholder). */
-    assert(tmp_state_p->cur_reg_off != 
-	   tmp_flat_type_p->blocklens[tmp_state_p->idx]);
-    /* Shouldn't have been called if the view_state is done. */
-    assert(tmp_state_p->cur_sz != tmp_view_state_p->sz);
-
-    /* Make sure we are not in a non-zero region in the flat_type */
-    assert(tmp_flat_type_p->blocklens[tmp_state_p->idx] != 0);
-    
-#ifdef DEBUG3
-    fprintf(stderr, "view_state:(blocklens[%Ld]=%d,cur_reg_off=%Ld,"
-	    "max_sz=%Ld)\n", tmp_state_p->idx, 
-	    tmp_flat_type_p->blocklens[tmp_state_p->idx], 
-	    tmp_state_p->cur_reg_off, max_sz);
-#endif
-
-    /* Can it add the whole piece? */
-    if (tmp_flat_type_p->blocklens[tmp_state_p->idx] - 
-	tmp_state_p->cur_reg_off <= max_sz)
-    {
-	data_sz = tmp_flat_type_p->blocklens[tmp_state_p->idx] -
-            tmp_state_p->cur_reg_off;
-
-	tmp_state_p->cur_sz += data_sz;
-
-	/* Advance the abs_off to the beginning of the next piece */
-	if (tmp_flat_type_p->count == 1)
-	{
-	    assert(tmp_flat_type_p->blocklens[tmp_state_p->idx] != 0);
-	    tmp_state_p->abs_off += data_sz;
-#ifdef DEBUG3 
-	    fprintf(stderr, "view_state_add_region: %s contig type "
-		    "(old abs_off=%Ld,abs_off=%Ld,cur_sz=%Ld,reg size=%Ld)\n", 
-		    off_type_name[op_type], tmp_state_p->abs_off - data_sz, 
-		    tmp_state_p->abs_off, tmp_state_p->cur_sz, data_sz);
-#endif
-	}
-	else
-	{ 
-	    /* Is this the last region in the datatype? */
-	    if (tmp_state_p->idx == (tmp_flat_type_p->count - 1))
-	    {
-		tmp_state_p->abs_off += data_sz -
-		    tmp_flat_type_p->indices[tmp_flat_type_p->count-1] -
-		    tmp_flat_type_p->blocklens[tmp_flat_type_p->count-1] +
-		    tmp_view_state_p->ext;
-#ifdef DEBUG3
-	    fprintf(stderr, "view_state_add_region: %s last region for type "
-		    "(old abs_off=%Ld,abs_off=%Ld,cur_sz=%Ld,reg size=%Ld)\n", 
-		    off_type_name[op_type], tmp_state_p->abs_off - data_sz, 
-		    tmp_state_p->abs_off, tmp_state_p->cur_sz, data_sz);
-#endif
-	    }
-	    else
-	    {
-		tmp_state_p->abs_off += 
-		    tmp_flat_type_p->indices[tmp_state_p->idx + 1] -
-		    (tmp_flat_type_p->indices[tmp_state_p->idx] +
-		     tmp_state_p->cur_reg_off);
-#ifdef DEBUG3
-	    fprintf(stderr, "view_state_add_region: %s inner region type "
-		    "(old abs_off=%Ld,abs_off=%Ld,cur_sz=%Ld,reg size=%Ld)\n", 
-		    off_type_name[op_type], tmp_state_p->abs_off - 
-		    (tmp_flat_type_p->indices[tmp_state_p->idx + 1] -
-                    (tmp_flat_type_p->indices[tmp_state_p->idx] +
-                     tmp_state_p->cur_reg_off)), tmp_state_p->abs_off, 
-		    tmp_state_p->cur_sz, data_sz);
-#endif
-	    }
-	    /* Increment idx to next non-zero region in the flat_type */
-	    do {
-		tmp_state_p->idx = 
-		    (tmp_state_p->idx + 1) % tmp_flat_type_p->count;
-	    } while (tmp_flat_type_p->blocklens[tmp_state_p->idx] == 0);
-	}
-	tmp_state_p->cur_reg_off = 0;
-    }
-    else /* Add part of the piece */
-    {
-	data_sz = max_sz;
-	tmp_state_p->cur_reg_off += data_sz;
-	tmp_state_p->abs_off += data_sz;
-	tmp_state_p->cur_sz += data_sz;
-#ifdef DEBUG3 
-	    fprintf(stderr, "view_state_add_region: %s partial region type "
-		    "(cur_reg_off=%Ld,abs_off=%Ld,cur_sz=%Ld,reg size=%Ld\n", 
-		    off_type_name[op_type], tmp_state_p->cur_reg_off, 
-		    tmp_state_p->abs_off, tmp_state_p->cur_sz, data_sz);
-#endif
-    }
-
-    *tmp_reg_sz_p = data_sz;
-#ifdef AGGREGATION_PROFILE
-    /* MPE_Log_event (5021, 0, NULL); */
-#endif
-    return 0;
-}
-
-/* Set up the abs_off, idx, and cur_reg_off of a view_state for the
- * tmp_state or the cur_state. */
-int ADIOI_init_view_state(int file_ptr_type,
-		    int nprocs, 
-		    view_state *view_state_arr,
-		    int op_type)
-{
-    ADIOI_Flatlist_node *tmp_flat_type_p = NULL;
-    ADIO_Offset tmp_off_used = 0, st_reg = 0, tmp_reg_sz = 0;
-    int i;
-    flatten_state *tmp_state_p = NULL;
-    view_state *tmp_view_p = NULL;
-
-    for (i = 0; i < nprocs; i++)
-    {
-	switch(op_type)
-	{
-	    case TEMP_OFF:
-		tmp_state_p = &(view_state_arr[i].tmp_state);
-		break;
-	    case REAL_OFF:
-		tmp_state_p = &(view_state_arr[i].cur_state);
-		break;
-	    default:
-		fprintf(stderr, "op_type invalid\n");
-	}
-	
-	tmp_view_p = &(view_state_arr[i]);
-	tmp_flat_type_p = tmp_view_p->flat_type_p;
-
-	if (file_ptr_type == ADIO_INDIVIDUAL)
-	    tmp_state_p->abs_off = tmp_view_p->fp_ind;
-	else
-	    tmp_state_p->abs_off = tmp_view_p->disp;
-	
-	tmp_off_used = 0;
-
-	/* initialize tmp_state idx */
-	while (tmp_flat_type_p->blocklens[tmp_state_p->idx] == 0)
-	    tmp_state_p->idx = (tmp_state_p->idx + 1) % tmp_flat_type_p->count;
-	if (file_ptr_type == ADIO_EXPLICIT_OFFSET)
-	    tmp_state_p->abs_off += tmp_flat_type_p->indices[tmp_state_p->idx];
-
-	/* Initialize the abs_off by moving into the datatype 
-	 * byte_off bytes.  Since we only do this in the beginning, we
-	 * make the assumption that pieces are added whole until the last
-	 * piece which MAY be partial. */
-	while (tmp_off_used != tmp_view_p->byte_off)
-	{
-	    view_state_add_region(
-		tmp_view_p->byte_off - tmp_off_used,
-		&(view_state_arr[i]), &st_reg, &tmp_reg_sz, 
-		op_type);
-	}
-	
-	/* Re-initialize the cur_size so that the abs_off was set to
-	 * the proper position while the actual size = 0.*/
-	tmp_state_p->cur_sz = 0;
-#ifdef DEBUG1
-	fprintf(stderr, "init_view_state: %s (idx=%d,byte_off=%Ld,"
-		"abs_off=%Ld,reg_off=%Ld,sz=%Ld)\n", off_type_name[op_type], 
-		i, tmp_view_p->byte_off, tmp_state_p->abs_off,
-		tmp_state_p->cur_reg_off, tmp_view_p->sz);
-#endif	
-
-    }
-    return 0;
-}
-
-/* Return the next file realm offset and length for this datatype state
- * within a particular file realm. */
-static inline int get_next_fr_off(ADIO_File fd,
-				  ADIO_Offset off,
-				  ADIO_Offset fr_st_off,
-				  MPI_Datatype *fr_type_p,
-				  ADIO_Offset *fr_next_off_p,
-				  ADIO_Offset *fr_max_len_p) 
-{
-    MPI_Aint fr_extent = -1, lb;
-    ADIO_Offset tmp_off, off_rem;
-    ADIOI_Flatlist_node *fr_node_p = ADIOI_Flatlist;
-    int i = -1, fr_dtype_ct = 0;
-
-    /* Should have already been flattened in calc_file_realms() */
-    while (fr_node_p->type != (*fr_type_p))
-	fr_node_p = fr_node_p->next;
-    assert(fr_node_p != NULL);
-
-    /* Did we get to the first region of the file realm? */
-    if (off - fr_st_off < 0)
-    {
-	*fr_next_off_p = fr_st_off + fr_node_p->indices[0];
-	*fr_max_len_p = fr_node_p->blocklens[0];
-	return 0;
-    }
-
-    /* Calculate how many times to loop through the fr_type 
-     * and where the next fr_off is. */
-    MPI_Type_get_extent(*fr_type_p, &lb, &fr_extent);
-    tmp_off = off - fr_st_off;
-    fr_dtype_ct = tmp_off / fr_extent;
-    off_rem = tmp_off % fr_extent;
-    for (i = 0; i < fr_node_p->count; i++)
-    {
-	if (off_rem < fr_node_p->indices[i])
-	{
-	    *fr_next_off_p = fr_st_off +
-		(fr_dtype_ct * fr_extent) + fr_node_p->indices[i];
-	    *fr_max_len_p = fr_node_p->blocklens[i];
-	    return 0;
-	}
-	else if (off_rem < fr_node_p->indices[i] + fr_node_p->blocklens[i])
-	{
-	    *fr_next_off_p = off;
-	    *fr_max_len_p = fr_node_p->blocklens[i] - 
-		(off_rem - fr_node_p->indices[i]);
-	    return off;
-	}
-    }
-    
-    /* Shouldn't get here. */
-    fprintf(stderr, "get_next_fr_off: Couldn't find the correct "
-	    "location of the next offset for this file realm.\n");
-    return -1;
-}
-
-/* Look in all the view states for the first offset within a given
- * file realm.  Report the end of a contiguous region within the file
- * realm (possibly more than the actual view state may be able to
- * process contiguously). */
-static inline int find_next_off(ADIO_File fd,
-				view_state *view_state_p,
-				ADIO_Offset fr_st_off,
-				MPI_Datatype *fr_type_p,
-				int op_type,
-				ADIO_Offset *cur_off_p,
-				ADIO_Offset *cur_reg_max_len_p)
-{
-    ADIOI_Flatlist_node *tmp_flat_type_p = NULL;
-    ADIO_Offset tmp_off = -1, fr_next_off = -1, fr_max_len = -1, 
-	tmp_fr_max_len = -1;
-    int ret = 0;
-    flatten_state *tmp_state_p = NULL;
-    ADIO_Offset tmp_st_off = 0, tmp_reg_sz = 0;
-#ifdef DTYPE_SKIP
-    int skip_type_ct;
-#endif
-
-#ifdef AGGREGATION_PROFILE
-    /* MPE_Log_event (5022, 0, NULL); */
-#endif
-
-    switch(op_type)
-    {
-	case TEMP_OFF:
-	    tmp_state_p = &(view_state_p->tmp_state);
-	    break;
-	case REAL_OFF:
-	    tmp_state_p = &(view_state_p->cur_state);
-	    break;
-	default:
-	    fprintf(stderr, "op_type invalid\n");
-    }
-	
-    tmp_flat_type_p = view_state_p->flat_type_p;
-
-    /* Can we use this proc? */
-    if (tmp_state_p->cur_sz < view_state_p->sz) {
-	tmp_st_off = 0;
-	tmp_reg_sz = 0;
-	/* If the current region is not within the file realm, advance
-	 * the state until it is and calculate the end of the next file 
-	 * realm in fr_max_len. */
-	ret = get_next_fr_off(fd,
-			      tmp_state_p->abs_off, 
-			      fr_st_off,
-			      fr_type_p,
-			      &fr_next_off,
-			      &fr_max_len);
-	
-	while ((tmp_state_p->abs_off < fr_next_off) &&
-	       (tmp_state_p->cur_sz != view_state_p->sz))
-	{
-	    
-	/* While this might appear to be erroneous at first,
-	 * view_state_add_region can only add a single piece at a
-	 * time.  Therefore, it will never overshoot the beginning
-	 * of the next file realm.  When it finally does enter the
-	 * next file realm it will not be able to go beyond its
-	 * first piece. */
-	    
-#ifdef DTYPE_SKIP
-	    if (tmp_flat_type_p->count > 1) {
-		/* let's see if we can skip whole datatypes */
-		skip_type_ct = (fr_next_off - tmp_state_p->abs_off) /
-		    view_state_p->ext;
-		if (skip_type_ct > 0) {
-		    /* before we go on, let's check if we've actually
-		     * finished up already */
-		    tmp_state_p->cur_sz += skip_type_ct *
-			view_state_p->type_sz;
-		    if (tmp_state_p->cur_sz >= view_state_p->sz) {
-			tmp_state_p->cur_sz = view_state_p->sz;
-			break;
-		    }
-		    tmp_state_p->abs_off += skip_type_ct * view_state_p->ext;
-		}
-	    }
-#endif
-	    view_state_add_region(
-		fr_next_off - tmp_state_p->abs_off,
-		view_state_p,
-		&tmp_st_off,
-		&tmp_reg_sz,
-		op_type);
-
-	    ret = get_next_fr_off(fd,
-				  tmp_state_p->abs_off, 
-				  fr_st_off,
-				  fr_type_p,
-				  &fr_next_off,
-				  &fr_max_len);
-	}
-
-	if (tmp_state_p->cur_sz != view_state_p->sz) {
-	    tmp_off = tmp_state_p->abs_off;
-	    /* Calculate how much of the remaining file realm there is from the
-	     * current offset */
-	    tmp_fr_max_len = fr_next_off + fr_max_len - tmp_off;
-	}
-    }
-
-    *cur_off_p = tmp_off;
-    *cur_reg_max_len_p = tmp_fr_max_len;
-#ifdef AGGREGATION_PROFILE
-    /* MPE_Log_event (5023, 0, NULL); */
-#endif
-    return ret;
-}
-
-/* Upon completion of a full collective buffer, end of a file realm
- * region (data sieving), or the end of all I/O for an aggregator, we
- * should return a list of MPI_Datatypes that correspond to client
- * communication into a collective buffer, a list of corresponding
- * sizes, and an aggregate MPI_Datatype which will be used as a
- * filetype in MPI_File_write/read on the aggregator. */ 
-int ADIOI_Build_agg_reqs(ADIO_File fd, int rw_type, int nprocs,
-			 view_state *client_file_view_state_arr,
-			 MPI_Datatype *client_comm_dtype_arr,
-			 ADIO_Offset *client_comm_sz_arr,
-			 ADIO_Offset *agg_dtype_offset_p,
-			 MPI_Datatype *agg_dtype_p)
-{
-    MPI_Aint **client_disp_arr = NULL, *agg_disp_arr = NULL;
-    int **client_blk_arr = NULL, *agg_blk_arr = NULL;
-    ADIO_Offset tmp_coll_buf_sz = 0, st_reg = 0, act_reg_sz = 0;
-    ADIO_Offset cur_off = -1, cur_reg_max_len = -1;
-    ADIO_Offset ds_fr_end = -1;
-    ADIO_Offset *fr_st_off_arr = fd->file_realm_st_offs;
-    MPI_Datatype *fr_type_arr = fd->file_realm_types;
-    int *client_ol_ct_arr = NULL;
-    int *client_ol_cur_ct_arr = NULL;
-    int agg_ol_ct = 0, agg_ol_cur_ct = 0;
-    int cur_off_proc = -1;
-    int next_off_idx = -1;
-    int i = 0, j = 0, all_done = -1;
-    int agg_idx = fd->my_cb_nodes_index;
-    heap_t offset_heap;
-    ADIO_Offset next_off = -1, next_reg_max_len = -1;
-
-    /* Used for coalescing ol pairs next to each other. */
-    ADIO_Offset *client_comm_next_off_arr = NULL;
-    ADIO_Offset agg_next_off = -1;
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5016, 0, NULL);
-#endif
-
-    memset(client_comm_sz_arr, 0, nprocs*sizeof(ADIO_Offset));
-
-    if ((client_comm_next_off_arr = (ADIO_Offset *) 
-	 ADIOI_Malloc(nprocs*sizeof(ADIO_Offset))) == NULL)
-    {
-	fprintf(stderr, "ADIOI_Build_agg_reqs: malloc client_next_off_arr "
-		"failed\n");
-	return -1;
-    }
-    
-    if ((client_ol_ct_arr = (int *) ADIOI_Calloc(nprocs, sizeof(int))) == NULL)
-    {
-	fprintf(stderr, "ADIOI_Build_agg_reqs: "
-		"malloc client_ol_ct_arr failed\n");
-	return -1;
-    }
-    if ((client_ol_cur_ct_arr = 
-	 (int *) ADIOI_Calloc(nprocs, sizeof(int))) == NULL)
-    {
-	fprintf(stderr, "ADIOI_Build_agg_reqs: "
-		"malloc client_ol_cur_ct_arr failed\n");
-	return -1;
-    }
-
-    /* On the first pass see how many offset-length pairs are
-     * necessary for each client.  Then allocate the correct amount of
-     * offset-length pairs for describing the collective buffer.  All
-     * data is processed in order by the aggregator's file realm.  On
-     * the second pass, set the offset-length pairs to the correct
-     * values. */
-    for (i = 0; i < MAX_OFF_TYPE; i++)
-    {
-	memset(client_comm_next_off_arr, -1, nprocs*sizeof(ADIO_Offset));
-	tmp_coll_buf_sz = 0;
-	ds_fr_end = -1;
-
-	/* initialize heap */
-	ADIOI_Heap_create(&offset_heap, nprocs);
-	offset_heap.size = 0;
-	
-	for (j=0; j<nprocs; j++) {
-	    find_next_off(fd, 
-			  &client_file_view_state_arr[j],
-			  fr_st_off_arr[agg_idx],
-			  &(fr_type_arr[agg_idx]),
-			  i,
-			  &cur_off,
-			  &cur_reg_max_len);
-	    if ((cur_off != -1) && (cur_reg_max_len > 0)) {
-		ADIOI_Heap_insert(&offset_heap, cur_off, j, cur_reg_max_len);
-#ifdef DEBUG_HEAP
-		printf ("initial: inserting offset %lld with "
-			"cur_reg_max_len = %lld for p%d\n",
-			cur_off, cur_reg_max_len, j);
-#endif
-	    }
-
-	}
-	if (!offset_heap.size)
-	    ADIOI_Heap_insert(&offset_heap, -1, -1, -1);
-
-	while (tmp_coll_buf_sz < fd->hints->cb_buffer_size)
-	{
-	    /* Find the next process with the next region within the
-	     * file realm and the maximum amount that can be added for
-	     * this particular file realm as a contiguous region. */
-	    ADIOI_Heap_extract_min(&offset_heap, &cur_off, &cur_off_proc,
-			     &cur_reg_max_len);
-#ifdef DEBUG_HEAP
-	    printf ("extracted cur_off %lld from proc %d\n",
-		    cur_off, cur_off_proc);
-#endif
-
-	    if (cur_off == -1)
-		break;
-	    
-#ifdef DEBUG3
-	    fprintf(stderr, "ADIOI_Build_agg_reqs: %s proc %d start/add to"
-		    " list (max_reg_fr=%Ld,tmp_coll_buf_sz=%Ld,"
-		    "cb_buffer_size=%d)\n", off_type_name[i], cur_off_proc,
-		    cur_reg_max_len, tmp_coll_buf_sz, 
-		    fd->hints->cb_buffer_size);
-#endif
-	    
-	    /* We process only contiguous file realm regions if we are
-	     * using data sieving. Note that we only do this for
-	     * writes since reads can be data sieved across each other
-	     * without consistency issues. */
-	    if ((fd->hints->ds_write == ADIOI_HINT_ENABLE ||
-		 fd->hints->ds_write == ADIOI_HINT_AUTO) &&
-		rw_type == ADIOI_WRITE && fd->hints->cb_nodes > 1)
-	    {
-#ifdef DEBUG2
-		fprintf(stderr, "ADIOI_Build_agg_reqs: "
-			"Warning - Data sieving writes on\n");
-#endif
-		if (ds_fr_end == -1)
-		{
-		    ds_fr_end = cur_off + cur_reg_max_len;
-#ifdef DEBUG1
-		fprintf(stderr, "ADIOI_Build_agg_reqs: "
-			"cur_off=%Ld, cur_reg_max_len=%Ld\n"
-			"Data sieving file realm end initialized to %Ld\n",
-			cur_off,
-			cur_reg_max_len,
-			ds_fr_end);
-#endif
-		}
-		else
-		{
-		    /* The next off switched file realms, so we will stop
-		     * here. */
-		    if (ds_fr_end != cur_off + cur_reg_max_len)
-		    {
-#ifdef DEBUG1
-			fprintf(stderr, "ADIOI_Build_agg_reqs: "
-				"Data sieving file realm end changed from "
-				"%Ld to %Ld\n", ds_fr_end, 
-				cur_off + cur_reg_max_len);
-#endif
-			break;
-		    }
-		}
-	    }
-	    
-	    /* Add up to the end of the file realm or the collective
-	     * buffer. */
-	    if (cur_reg_max_len > (fd->hints->cb_buffer_size - 
-				   tmp_coll_buf_sz))
-		cur_reg_max_len = fd->hints->cb_buffer_size - tmp_coll_buf_sz;
-
-	    view_state_add_region(
-		cur_reg_max_len,
-		&(client_file_view_state_arr[cur_off_proc]), 
-		&st_reg, &act_reg_sz, i);
-
-	    switch(i)
-	    {
-		case TEMP_OFF:
-		    /* Increment the ol list count for each proc and
-		     * the used part of the collective buffer if the
-		     * next region is not adjacent to the previous
-		     * region. */
-		    if (client_comm_next_off_arr[cur_off_proc] != 
-			tmp_coll_buf_sz)
-		    {
-			(client_ol_ct_arr[cur_off_proc])++;
-		    }
-		    client_comm_next_off_arr[cur_off_proc] = 
-			tmp_coll_buf_sz + act_reg_sz;
-		    
-		    if (agg_next_off != st_reg)
-			agg_ol_ct++;
-		    agg_next_off = st_reg + act_reg_sz;
-		    break;
-		case REAL_OFF:
-		    /* Add this region to the proper client ol list if
-		     * the next region is not adjacent to the previous
-		     * region. */
-		    next_off_idx = client_ol_cur_ct_arr[cur_off_proc];
-		    if (client_comm_next_off_arr[cur_off_proc] != 
-			tmp_coll_buf_sz)
-		    {
-			client_disp_arr[cur_off_proc][next_off_idx] =
-			    tmp_coll_buf_sz;
-			client_blk_arr[cur_off_proc][next_off_idx] = 
-			    act_reg_sz;
-			(client_ol_cur_ct_arr[cur_off_proc])++;
-		    }
-		    else
-		    {
-			client_blk_arr[cur_off_proc][next_off_idx - 1] 
-			    += act_reg_sz;
-		    }
-		    client_comm_sz_arr[cur_off_proc] += act_reg_sz;
-		    client_comm_next_off_arr[cur_off_proc] =
-			tmp_coll_buf_sz + act_reg_sz;
-		    
-		    /* Add to the aggregator filetype if the next
-		     * region is not adjacent to the previous
-		     * region. */
-		    if (agg_next_off != st_reg)
-		    {
-			/* this will enable initial offsets much further into
-			 * the file than an MPI_Aint */
-			if (!agg_ol_cur_ct)
-			    *agg_dtype_offset_p = st_reg;
-			agg_disp_arr[agg_ol_cur_ct] = st_reg -
-			    (MPI_Aint) *agg_dtype_offset_p;
-			agg_blk_arr[agg_ol_cur_ct] = act_reg_sz;	
-			agg_ol_cur_ct++;
-		    }
-		    else
-		    {
-			agg_blk_arr[agg_ol_cur_ct - 1] += act_reg_sz;
-		    }
-		    agg_next_off = st_reg + act_reg_sz;
-		    
-		    break;
-		default:
-		    fprintf(stderr, "ADIOI_Build_agg_reqs: Impossible type\n");
-	    }
-	    tmp_coll_buf_sz += act_reg_sz;
-
-	    find_next_off(fd,
-			  &client_file_view_state_arr[cur_off_proc],
-			  fr_st_off_arr[agg_idx],
-			  &(fr_type_arr[agg_idx]),
-			  i,
-			  &next_off,
-			  &next_reg_max_len);
-
-	    if ((next_off != -1) || (!offset_heap.size)) {
-		ADIOI_Heap_insert(&offset_heap, next_off, cur_off_proc,
-			    next_reg_max_len);
-#ifdef DEBUG_HEAP
-		printf ("inserting offset %lld for p%d\n", next_off,
-			cur_off_proc);
-#endif
-	    }
-	}
-	
-	if (i == TEMP_OFF)
-	{
-	    /* Allocate offset-length pairs for creating hindexed
-	     * MPI_Datatypes for both the client and the aggregator. */
-	    if ((client_disp_arr = (MPI_Aint **) 
-		 ADIOI_Malloc(nprocs*sizeof(MPI_Aint *))) == NULL)
-	    {
-		fprintf(stderr, "ADIOI_Build_agg_reqs: malloc "
-			"client_disp_arr failed\n");
-		return -1;
-	    }
-	    if ((client_blk_arr = (int **) ADIOI_Malloc(
-		     nprocs*sizeof(int *))) == NULL)
-	    {
-		ADIOI_Free(client_disp_arr);
-		fprintf(stderr, "ADIOI_Build_agg_reqs: malloc "
-			"client_blk_arr failed\n");
-		return -1;
-	    }    
-	    for (j = 0; j < nprocs; j++)
-	    {
-		if ((client_disp_arr[j] = (MPI_Aint *) ADIOI_Malloc(
-			 client_ol_ct_arr[j]*sizeof(MPI_Aint))) == NULL)
-		{
-		    fprintf(stderr, "ADIOI_Build_agg_reqs: malloc "
-			    "client_disp_arr[%d] failed\n", j);
-		    return -1;
-		}
-		if ((client_blk_arr[j] = (int *) 
-		     ADIOI_Malloc(client_ol_ct_arr[j]*sizeof(int))) == NULL)
-		{
-		    ADIOI_Free(client_disp_arr[j]);
-		    fprintf(stderr, "ADIOI_Build_agg_reqs: malloc "
-			    "client_blk_arr[%d] failed\n", j);
-		    return -1;
-		}
-	    }
-	    
-	    if (agg_ol_ct > 0) 
-	    {
-		if ((agg_disp_arr = (MPI_Aint *) ADIOI_Malloc(
-			 agg_ol_ct*sizeof(MPI_Aint))) == NULL)
-		{
-		    fprintf(stderr, 
-			    "ADIOI_Build_agg_reqs: malloc disp_arr failed\n");
-		    return -1;
-		}
-		if ((agg_blk_arr = (int *) 
-		     ADIOI_Malloc(agg_ol_ct*sizeof(int))) == NULL)
-		{
-		    ADIOI_Free(agg_disp_arr);
-		    fprintf(stderr, 
-			    "ADIOI_Build_agg_reqs: malloc blk_arr failed\n");
-		    return -1;
-		}
-	    }
-	}
-	ADIOI_Heap_free(&offset_heap);
-    }
-    
-    /* Let the clients know if this aggregator is totally finished
-     * with all possible client requests. */
-    all_done = 1;
-    for (i = 0; i < nprocs; i++)
-    {
-	if ((client_file_view_state_arr[i].cur_state.cur_sz !=
-            client_file_view_state_arr[i].sz) ||
-            client_comm_sz_arr[i] != 0)
-	{
-	    all_done = 0;
-	    break;
-	}
-    }
-    if (all_done == 1)
-    {
-	for (i = 0; i < nprocs; i++)
-	{
-	    client_comm_sz_arr[i] = -1;
-	}
-    }
-
-    /* Quick check to make sure we found all the ol pairs we thought
-     * we did */
-    for (i = 0; i < nprocs; i++)
-    {
-	if (client_ol_cur_ct_arr[i] != client_ol_ct_arr[i])
-	{
-	    fprintf(stderr, "ADIOI_Build_agg_reqs: ERROR Process %d "
-		    "processed only %d out of %d ol pairs\n", i, 
-		    client_ol_cur_ct_arr[i],
-		    client_ol_ct_arr[i]);
-	    return -1;
-	}
-    }
-#ifdef DEBUG1
-    fprintf(stderr, "ADIOI_Build_agg_reqs:(client,ol_pairs,size_req)=");
-    for (i = 0; i < nprocs; i++)
-    {
-	fprintf(stderr, "(%d,%d,%Ld)", i, client_ol_ct_arr[i],
-		client_comm_sz_arr[i]);
-	if (i != nprocs - 1)
-	    fprintf(stderr, ",");
-    }
-    fprintf(stderr, "\n");
-#endif
-#ifdef DEBUG1
-    fprintf(stderr, "ADIOI_Build_agg_reqs: Generated %d of %d "
-	    "aggregate offset-length pairs\n", agg_ol_cur_ct, agg_ol_ct);
-#endif
-#ifdef DEBUG2
-    for (i = 0; i < nprocs; i++)
-    {
-	if (client_ol_ct_arr[i] > 0)
-	{
-	    fprintf(stderr, "ADIOI_Build_agg_reqs: p %d (off,len) = ", i);
-	    for (j = 0; j < client_ol_ct_arr[i]; j++)
-	    {
-		fprintf(stderr, "[%d](%d,%d) ", j, 
-			client_disp_arr[i][j],
-			client_blk_arr[i][j]);
-	    }
-	    fprintf(stderr, "\n");
-	}
-    }    
-    if (agg_ol_ct) {
-	fprintf(stderr, "ADIOI_Build_agg_reqs:agg_type(off,len)=");
-	for (i = 0; i < agg_ol_ct; i++)
-	    {
-		fprintf(stderr, "[%d](%d,%d)",
-			i, agg_disp_arr[i], agg_blk_arr[i]);
-		if (i != agg_ol_ct - 1)
-		    fprintf(stderr, ",");
-	    }
-	fprintf(stderr, "\n");
-    }
-#endif
-
-    assert(agg_ol_cur_ct == agg_ol_ct);
-
-    /* Create all the client and aggregate MPI_Datatypes */
-    for (i = 0; i < nprocs; i++)
-    {
-	if (client_comm_sz_arr[i] > 0)
-	{
-	    MPI_Type_create_hindexed(client_ol_ct_arr[i], client_blk_arr[i],
-			             client_disp_arr[i], MPI_BYTE, 
-			             &(client_comm_dtype_arr[i]));
-	    MPI_Type_commit(&(client_comm_dtype_arr[i]));
-	}
-	else
-	{
-	    client_comm_dtype_arr[i] = MPI_BYTE;
-	}
-	ADIOI_Free(client_blk_arr[i]);
-	ADIOI_Free(client_disp_arr[i]);
-    }
-    ADIOI_Free(client_blk_arr);
-    ADIOI_Free(client_disp_arr);
-
-    if (agg_ol_ct > 0) {
-	if (agg_ol_ct == 1)
-	    MPI_Type_contiguous (agg_blk_arr[0], MPI_BYTE, agg_dtype_p);
-	else if (agg_ol_ct > 1)
-	    MPI_Type_create_hindexed(agg_ol_ct, agg_blk_arr, agg_disp_arr, MPI_BYTE,
-			             agg_dtype_p);    
-
-	MPI_Type_commit(agg_dtype_p);
-
-	ADIOI_Free(agg_disp_arr);
-	ADIOI_Free(agg_blk_arr);
-    }
-    ADIOI_Free(client_ol_ct_arr);
-    ADIOI_Free(client_ol_cur_ct_arr);
-    ADIOI_Free(client_comm_next_off_arr);
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5017, 0, NULL);
-#endif
-    return 0;
-}
-
-/* All sizes from all aggregators are gathered on the clients, which
- * then call this function, which will generate the comm datatypes for
- * each aggregator (agg_comm_dtype_arr) in the upcoming
- * MPI_Alltoallw() */
-int ADIOI_Build_client_reqs(ADIO_File fd, 
-			    int nprocs,
-			    view_state *my_mem_view_state_arr,
-			    view_state *agg_file_view_state_arr,
-			    ADIO_Offset *agg_comm_sz_arr,
-			    MPI_Datatype *agg_comm_dtype_arr)
-{
-    MPI_Aint **agg_disp_arr = NULL;
-    int **agg_blk_arr = NULL;
-    view_state *tmp_mem_state_p = NULL, *tmp_file_state_p = NULL;
-    ADIO_Offset total_agg_comm_sz = 0, cur_total_agg_comm_sz = 0;
-    ADIO_Offset st_reg = 0, act_reg_sz = 0, tmp_reg_sz = 0;
-    ADIO_Offset cur_off = -1, cur_reg_max_len = -1;
-    ADIO_Offset tmp_cur_off = -1, tmp_cur_reg_max_len = -1;
-    ADIO_Offset agg_mem_st_reg = 0, agg_mem_act_reg_sz = 0;
-    ADIO_Offset *fr_st_off_arr = fd->file_realm_st_offs;
-    ADIO_Offset *agg_comm_cur_sz_arr = NULL;
-    MPI_Datatype *fr_type_arr = fd->file_realm_types;
-    int *agg_ol_ct_arr = NULL;
-    int *agg_ol_cur_ct_arr = NULL;
-    int tmp_agg_fr_idx = -1;
-    int cur_off_proc = -1;
-    int i = 0, j = 0;
-    int agg_next_off_idx = -1;
-    /* Used for coalescing ol pairs next to each other. */
-    ADIO_Offset *agg_mem_next_off_arr = NULL;
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5018, 0, NULL);
-#endif
-
-#ifdef DEBUG
-    fprintf(stderr, "ADIOI_Build_client_reqs:(agg,size_req)=");
-    for (i = 0; i < nprocs; i++)
-    {
-	int tmp_agg_idx = ADIOI_Agg_idx(i, fd);
-        if (tmp_agg_idx >= 0)
-        {
-	    fprintf(stderr, "(%d,%Ld)", i, agg_comm_sz_arr[i]);
-	    if (i != fd->hints->cb_nodes - 1)
-		fprintf(stderr, ",");
-	}
-	fprintf(stderr, "\n");
-    }
-#endif
-    
-    if ((agg_mem_next_off_arr = (ADIO_Offset *) ADIOI_Malloc(
-	     nprocs*sizeof(ADIO_Offset))) == NULL)
-    {
-	fprintf(stderr, "ADIOI_Build_client_reqs: malloc agg_mem_next_off_arr"
-		"failed\n");
-	return -1;
-    }
-
-    if ((agg_comm_cur_sz_arr = (ADIO_Offset *) 
-	 ADIOI_Malloc(nprocs*sizeof(ADIO_Offset))) == NULL)
-    {
-	fprintf(stderr, "ADIOI_Build_client_reqs: malloc agg_comm_cur_sz_arr"
-		" failed\n");
-	return -1;
-    }
-    if ((agg_ol_ct_arr = (int *) ADIOI_Calloc(nprocs, sizeof(int)))
-	== NULL)
-    {
-	fprintf(stderr, "ADIOI_Build_client_reqs: "
-		"malloc agg_ol_ct_arr failed\n");
-	return -1;
-    }
-    if ((agg_ol_cur_ct_arr = (int *) ADIOI_Calloc(nprocs, sizeof(int)))
-	== NULL)
-    {
-	fprintf(stderr, "ADIOI_Build_client_reqs: "
-		"malloc agg_ol_cur_ct_arr failed\n");
-	return -1;
-    }
-
-    for (i = 0; i < nprocs; i++)
-    {
-	if (agg_comm_sz_arr[i] > 0)
-	    total_agg_comm_sz += agg_comm_sz_arr[i];
-    }
-    
-    /* On the first pass see how many offset-length pairs are
-     * necessary for each aggregator.  Then allocate the correct
-     * amount of offset-length pairs for handling each aggregator's
-     * particular data size.  On the last pass, we actually create the
-     * offset-length pairs. */
-    for (i = 0; i < MAX_OFF_TYPE; i++)
-    {
-	cur_total_agg_comm_sz = 0;
-	memset(agg_comm_cur_sz_arr, 0, nprocs*sizeof(ADIO_Offset));
-	memset(agg_mem_next_off_arr, -1, nprocs*sizeof(ADIO_Offset));
-	while (total_agg_comm_sz > cur_total_agg_comm_sz)
-	{
-	    /* Look for the next aggregator offset among all the
-	     * aggregators and their respective file realms. */
-	    cur_off = -1;
-	    for (j = 0; j < nprocs; j++)
-	    {
-		tmp_agg_fr_idx = ADIOI_Agg_idx(j, fd);
-                assert(tmp_agg_fr_idx < fd->hints->cb_nodes);
-		
-		/* If this process is not an aggregator or we have
-		 * finished all the bytes for this aggregator, move
-		 * along. */
-		if (tmp_agg_fr_idx < 0 || 
-		    agg_comm_cur_sz_arr[j] == agg_comm_sz_arr[j])
-		{
-		    continue;
-		}
-
-		find_next_off(fd,
-			      &(agg_file_view_state_arr[j]),
-			      fr_st_off_arr[tmp_agg_fr_idx],
-			      &(fr_type_arr[tmp_agg_fr_idx]),
-			      i,
-			      &tmp_cur_off,
-			      &tmp_cur_reg_max_len);
-		if (tmp_cur_off == -1)
-		    continue;	       
-
-		if ((cur_off == -1) || 
-		    (cur_off > tmp_cur_off))
-		{
-		    cur_off_proc = j;
-		    cur_off = tmp_cur_off;
-		    cur_reg_max_len = tmp_cur_reg_max_len;
-		}
-	    }
-
-	    assert(cur_off_proc != -1);
-	    
-	    /* Add up to the end of the file realm or as many bytes
-	     * are left for this particular aggregator in the client's
-	     * filetype */
-	    if (cur_reg_max_len > agg_comm_sz_arr[cur_off_proc] - 
-		agg_comm_cur_sz_arr[cur_off_proc])
-	    {
-		cur_reg_max_len = agg_comm_sz_arr[cur_off_proc] - 
-		    agg_comm_cur_sz_arr[cur_off_proc];
-	    }
-	    assert(cur_reg_max_len > 0);
-	    
-	    view_state_add_region(
-		cur_reg_max_len,
-		&(agg_file_view_state_arr[cur_off_proc]),
-		&st_reg, &act_reg_sz, i);
-	    
-#ifdef DEBUG2
-	    fprintf(stderr, "ADIOI_Build_client_reqs: %s File region"
-		    " (proc=%d,off=%Ld,sz=%Ld)\n",
-		    off_type_name[i], cur_off_proc,
-		    cur_off, act_reg_sz);
-#endif
-
-	    /* Before translating the file regions to memory regions,
-	     * we first must advance to the proper point in the
-	     * mem_view_state for this aggregator to match the
-	     * file_view_state. */
-	    tmp_file_state_p = &(agg_file_view_state_arr[cur_off_proc]);
-	    tmp_mem_state_p = &(my_mem_view_state_arr[cur_off_proc]);
-	    assert(view_state_get_cur_sz(tmp_file_state_p, i) - act_reg_sz >=
-		   view_state_get_cur_sz(tmp_mem_state_p, i));
-	    while (view_state_get_cur_sz(tmp_file_state_p, i) - act_reg_sz != 
-		   view_state_get_cur_sz(tmp_mem_state_p, i))
-	    {
-		ADIO_Offset fill_st_reg = -1, fill_reg_sz = -1;
-		view_state_add_region(
-		    view_state_get_cur_sz(tmp_file_state_p, i) - act_reg_sz -
-		    view_state_get_cur_sz(tmp_mem_state_p, i),
-		    tmp_mem_state_p,
-		    &fill_st_reg,
-		    &fill_reg_sz, i);
-	    }
-	    
-	    /* Based on how large the act_reg_sz 1. Figure out how
-	     * many memory offset-length pairs are necessary. 2. Set
-	     * the offset-length pairs. */
-	    tmp_reg_sz = 0;
-	    while (tmp_reg_sz != act_reg_sz)
-	    {
-		view_state_add_region(
-		    act_reg_sz - tmp_reg_sz,
-		    tmp_mem_state_p,
-		    &agg_mem_st_reg, &agg_mem_act_reg_sz, 
-		    i);
-		tmp_reg_sz += agg_mem_act_reg_sz;
-
-#ifdef DEBUG2
-		fprintf(stderr, "ADIOI_Build_client_reqs: Mem region %s"
-			"(proc=%d,off=%Ld,sz=%Ld)\n",
-			off_type_name[i], cur_off_proc,
-			agg_mem_st_reg, agg_mem_act_reg_sz);
-#endif
-		agg_comm_cur_sz_arr[cur_off_proc] += agg_mem_act_reg_sz;
-		cur_total_agg_comm_sz += agg_mem_act_reg_sz;	    
-		switch(i)
-		{
-		    case TEMP_OFF:
-			/* Increment the ol list count a particular
-			 * aggregator if next region is not adjacent
-			 * to the previous region. */
-			if (agg_mem_next_off_arr[cur_off_proc] != 
-			    agg_mem_st_reg)
-			{
-			    agg_ol_ct_arr[cur_off_proc]++;
-			}
-			agg_mem_next_off_arr[cur_off_proc] = 
-			    agg_mem_st_reg + agg_mem_act_reg_sz;
-			break;
-		    case REAL_OFF:
-			/* Set the ol list for the memtypes that will
-			 * map to each aggregator, coaslescing if
-			 * possible. */
-			agg_next_off_idx = agg_ol_cur_ct_arr[cur_off_proc];
-			if (agg_mem_next_off_arr[cur_off_proc] != 
-			    agg_mem_st_reg)
-			{
-			    agg_disp_arr[cur_off_proc][agg_next_off_idx] = 
-				agg_mem_st_reg;
-			    agg_blk_arr[cur_off_proc][agg_next_off_idx] = 
-				agg_mem_act_reg_sz;
-			    (agg_ol_cur_ct_arr[cur_off_proc])++;
-			}
-			else
-			{
-			    agg_blk_arr[cur_off_proc][agg_next_off_idx - 1]
-				+= agg_mem_act_reg_sz;
-			}
-			agg_mem_next_off_arr[cur_off_proc] = 
-			    agg_mem_st_reg + agg_mem_act_reg_sz;
-			break;
-		    default:
-			fprintf(stderr, "ADIOI_Build_client_reqs: "
-				"Impossible type\n");
-		}
-	    }
-	}
-	
-	/* On the first pass, allocate the memory structures for
-	 * creating the MPI_hindexed type. */
-	if (i == TEMP_OFF)
-	{	    
-	    /* Allocate offset-length pairs for creating hindexed
-	     * MPI_Datatypes for each aggregator */
-	    if ((agg_disp_arr = (MPI_Aint **) 
-		 ADIOI_Malloc(nprocs*sizeof(MPI_Aint *))) == NULL)
-	    {
-		fprintf(stderr, 
-			"ADIOI_Build_client_reqs: malloc agg_disp_arr failed\n");
-		return -1;
-	    }
-	    if ((agg_blk_arr = (int **) ADIOI_Malloc(nprocs*sizeof(int *))) 
-		== NULL)
-	    {
-		ADIOI_Free(agg_disp_arr);
-		fprintf(stderr, 
-			"ADIOI_Build_client_reqs: malloc agg_blk_arr failed\n");
-		return -1;
-	    }    
-	    for (j = 0; j < nprocs; j++)
-	    {
-		if ((agg_disp_arr[j] = (MPI_Aint *) 
-		     ADIOI_Malloc(agg_ol_ct_arr[j]*sizeof(MPI_Aint))) == NULL)
-		{
-		    fprintf(stderr, "ADIOI_Build_client_reqs: malloc "
-			    "agg_disp_arr[%d] failed\n", j);
-		    return -1;
-		}
-		if ((agg_blk_arr[j] = (int *) 
-		     ADIOI_Malloc(agg_ol_ct_arr[j]*sizeof(int))) == NULL)
-		{
-		    ADIOI_Free(agg_disp_arr[j]);
-		    fprintf(stderr, "ADIOI_Build_client_reqs: malloc "
-			    "agg_blk_arr[%d] failed\n", j);
-		    return -1;
-		}
-	    }
-	}
-    }
-
-#ifdef DEBUG
-    fprintf(stderr, "ADIOI_Build_client_reqs:(agg,cur_ol_count=ol_count)=");
-    for (i = 0; i < nprocs; i++)
-    {
-	int tmp_agg_idx = ADIOI_Agg_idx(i, fd);
-	if (tmp_agg_idx >= 0)
-	{
-	    fprintf(stderr, "(%d,%d=%d)", i, agg_ol_cur_ct_arr[i],
-		    agg_ol_ct_arr[i]);
-	    assert(agg_ol_ct_arr[i] == agg_ol_cur_ct_arr[i]);
-	    if (tmp_agg_idx != fd->hints->cb_nodes - 1)
-		fprintf(stderr, ",");
-	}
-    }
-    fprintf(stderr, "\n");
-#endif
-
-#ifdef DEBUG2
-    for (i = 0; i < nprocs; i++)
-    {
-	if (agg_ol_ct_arr[i] > 0)
-	{
-	    fprintf(stderr, "ADIOI_Build_client_reqs: p %d (off,len) = ", i);
-	    for (j = 0; j < agg_ol_ct_arr[i]; j++)
-	    {
-		fprintf(stderr, "[%d](%d,%d) ", j,
-			agg_disp_arr[i][j],
-			agg_blk_arr[i][j]);
-	    }
-	    fprintf(stderr, "\n");
-	}
-    }
-#endif
-
-    /* Create all the aggregator MPI_Datatypes */
-    for (i = 0; i < nprocs; i++)
-    {
-	if (agg_comm_sz_arr[i] > 0)
-	{
-	    MPI_Type_create_hindexed(agg_ol_ct_arr[i], agg_blk_arr[i],
-                                     agg_disp_arr[i], MPI_BYTE,
-                                     &(agg_comm_dtype_arr[i]));
-            MPI_Type_commit(&(agg_comm_dtype_arr[i]));
-	}
-	else
-	{
-	    agg_comm_dtype_arr[i] = MPI_BYTE;
-	}
-	ADIOI_Free(agg_blk_arr[i]);
-	ADIOI_Free(agg_disp_arr[i]);
-    }
-    ADIOI_Free(agg_blk_arr);
-    ADIOI_Free(agg_disp_arr);
-
-    ADIOI_Free(agg_mem_next_off_arr);
-    ADIOI_Free(agg_comm_cur_sz_arr);
-    ADIOI_Free(agg_ol_ct_arr);
-    ADIOI_Free(agg_ol_cur_ct_arr);
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5019, 0, NULL);
-#endif    
-    return 0;
-}
-/* ADIOI_Build_client_pre_req allows a client to calculate the memtype
- * offset-length pairs up (up to a limit - max_pre_req_sz or max
- * ol_ct). It basically allows ADIOI_Build_client_req to do less work.
- * If it called and there already exist some preprocessed memtype
- * offset-length pairs, it will exit immediately if a limit has been
- * reached or if will add on the old limites to reach the new
- * limits. */
-
-int ADIOI_Build_client_pre_req(ADIO_File fd,
-			       int agg_rank, int agg_idx,
-			       view_state *my_mem_view_state_p,
-			       view_state *agg_file_view_state_p,
-			       ADIO_Offset max_pre_req_sz,
-			       int max_ol_ct)
-{
-    ADIO_Offset act_reg_sz = 0, tmp_reg_sz = 0;
-    ADIO_Offset cur_off = -1, cur_reg_max_len = -1;
-    ADIO_Offset agg_mem_st_reg = 0, agg_mem_act_reg_sz = 0;
-    int agg_ol_ct = 0, agg_ol_cur_ct = 0;
-    int i, agg_next_off_idx = -1;
-
-    ADIO_Offset cur_sz = 0, max_sz = 0, agg_mem_next_off = -1;
-    ADIO_Offset fill_st_reg = -1, fill_reg_sz = -1;
-    ADIO_Offset *fr_st_off_arr = fd->file_realm_st_offs;
-    MPI_Datatype *fr_type_arr = fd->file_realm_types;
-    MPI_Aint *tmp_disp_arr = NULL;
-    int *tmp_blk_arr = NULL, exit_loop = -1;
-    flatten_state *tmp_mem_state_p = NULL, *tmp_file_state_p = NULL;
-#ifdef DTYPE_SKIP
-    int skip_type_ct;
-#endif
-    if (agg_idx < 0 || agg_idx >= fd->hints->cb_nodes)
-    {
-        fprintf(stderr, "ADIOI_Build_client_pre_req: Invalid agg_idx %d\n",
-		agg_idx);
-        return -1;
-    }
-
-    if (agg_file_view_state_p->cur_state.cur_sz == 
-	agg_file_view_state_p->sz || max_pre_req_sz <= 0 ||
-	max_ol_ct <= 0)
-    {
-#ifdef DEBUG1
-	fprintf(stderr, 
-		"ADIOI_Build_client_pre_req: Nothing to preprocess\n");
-#endif
-	return 0;
-    }
-
-    /* The new limits have already been surpassed by what already
-     * exists.  Otherwise we will use the next restrictions */
-    if ((my_mem_view_state_p->pre_sz >= max_pre_req_sz) ||
-	(my_mem_view_state_p->pre_ol_ct >= max_ol_ct))
-    {
-#ifdef DEBUG1
-	fprintf(stderr, 
-		"ADIOI_Build_client_pre_req:  Old values surpass new "
-		"pre_req values\n");
-#endif
-	return 0;
-    }
-    
-    /* General idea is to first advance the filetype to the file realm
-     * and then the memtype to the filetype.  The memtype is advanced
-     * further by peeking at the filetype and then the filetype is
-     * advanced. */
-    for (i = 0; i < MAX_OFF_TYPE; i++)
-    {
-	switch(i)
-	{
-	    case TEMP_OFF:
-		tmp_mem_state_p  = &(my_mem_view_state_p->tmp_state);
-		tmp_file_state_p = &(agg_file_view_state_p->tmp_state);
-		break;
-	    case REAL_OFF:
-		tmp_mem_state_p  = &(my_mem_view_state_p->cur_state);
-		tmp_file_state_p = &(agg_file_view_state_p->cur_state);
-		break;
-	    default:
-		fprintf(stderr, "ADIOI_Build_client_pre_req: "
-			"Invalid off type %d\n", i);
-	}
-
-	if (i == TEMP_OFF && my_mem_view_state_p->pre_sz > 0)
-	{
-	    cur_sz = my_mem_view_state_p->pre_sz;
-            agg_ol_ct = my_mem_view_state_p->pre_ol_ct;
-	    /* Save the old arrays */
-	    tmp_disp_arr = my_mem_view_state_p->pre_disp_arr;
-	    tmp_blk_arr  = my_mem_view_state_p->pre_blk_arr;
-	    my_mem_view_state_p->pre_disp_arr = NULL;
-	    my_mem_view_state_p->pre_blk_arr  = NULL;
-            agg_mem_next_off =
-		tmp_disp_arr[agg_ol_ct - 1] + tmp_blk_arr[agg_ol_ct - 1];
-	}
-	else if (i == REAL_OFF && my_mem_view_state_p->pre_sz > 0)
-	{
-	    cur_sz = my_mem_view_state_p->pre_sz;
-	    agg_ol_cur_ct = my_mem_view_state_p->pre_ol_ct;
-	    
-	    /* Copy the old data to the new data, freeing the old
-	     * arrays */
-	    memcpy(my_mem_view_state_p->pre_disp_arr, tmp_disp_arr, 
-		   my_mem_view_state_p->pre_ol_ct * sizeof(MPI_Aint));
-	    memcpy(my_mem_view_state_p->pre_blk_arr, tmp_blk_arr, 
-		   my_mem_view_state_p->pre_ol_ct * sizeof(int));
-
-	    ADIOI_Free(tmp_disp_arr);
-	    ADIOI_Free(tmp_blk_arr);
-
-	    agg_mem_next_off = 
-		my_mem_view_state_p->pre_disp_arr[agg_ol_cur_ct - 1] +
-		my_mem_view_state_p->pre_blk_arr[agg_ol_cur_ct - 1];
-	}
-	else
-	{
-	    cur_sz = 0;
-	}
-	
-	/* Max_pre_req_sz may be larger than the amount of data left
-	 * to preprocess */
-	if (max_pre_req_sz - cur_sz > 
-	    agg_file_view_state_p->sz - tmp_file_state_p->cur_sz)
-	{
-	    max_sz = cur_sz +
-		agg_file_view_state_p->sz - tmp_file_state_p->cur_sz;
-	}
-	else
-	    max_sz = max_pre_req_sz;
-	
-	assert(cur_sz != max_sz);
-#ifdef DEBUG1
-	fprintf(stderr, 
-		"ADIOI_Build_client_pre_req: (cur_sz=%Ld,agg_ol_ct=%d,"
-		"agg_mem_next_off=%Ld,max_sz=%Ld,max_ol_ct=%d)\n", 
-		cur_sz, agg_ol_ct, agg_mem_next_off, max_sz, max_ol_ct);
-#endif
-	while (cur_sz < max_sz)
-	{
-	    find_next_off(fd, agg_file_view_state_p,
-			  fr_st_off_arr[agg_rank],
-			  &(fr_type_arr[agg_rank]),
-			  i,
-			  &cur_off,
-			  &cur_reg_max_len);
-	    
-	    /* find_next_off may show that the file_view_state is done
-	     * even if cur_sz != max_sz since find_next_off may
-	     * advance the file_view_state to the end here and realize
-	     * that it is done. */
-	    if (cur_off == -1)
-		break;
-
-	    assert(cur_off != -1);
-	    
-	    /* Before translating the file regions to memory regions,
-	     * we first must advance to the proper point in the
-	     * mem_view_state for this aggregator to match the
-	     * file_view_state. */
-	    while (tmp_file_state_p->cur_sz != tmp_mem_state_p->cur_sz)
-	    {
-#ifdef DTYPE_SKIP
-		if (my_mem_view_state_p->flat_type_p->count > 1) {
-		    /* let's see if we can skip whole memory datatypes */
-		    skip_type_ct =
-			(tmp_file_state_p->cur_sz - tmp_mem_state_p->cur_sz) /
-			my_mem_view_state_p->type_sz;
-		    if (skip_type_ct > 0) {
-			tmp_mem_state_p->cur_sz +=
-			    skip_type_ct * my_mem_view_state_p->type_sz;
-			tmp_mem_state_p->abs_off +=
-			    skip_type_ct * my_mem_view_state_p->ext;
-			if (tmp_mem_state_p->cur_sz ==
-			    tmp_file_state_p->cur_sz)
-			    break;
-		    }
-		}
-#endif
-		view_state_add_region(
-		    tmp_file_state_p->cur_sz - tmp_mem_state_p->cur_sz,
-		    my_mem_view_state_p,
-		    &fill_st_reg,
-		    &fill_reg_sz, i);
-	    }
-
-	    /* Now that the filetype and memtype are advanced to the
-	     * same position, add memtype ol-pairs while we have not
-	     * overstepped the min(end of the current piece in the
-	     * file view, end of the file realm, data left in
-	     * max_sz) */
-	    
-	    if (cur_reg_max_len >  
-		view_state_get_next_len(agg_file_view_state_p, i))
-		cur_reg_max_len =  
-		    view_state_get_next_len(agg_file_view_state_p, i);
-
-	    if (cur_reg_max_len > max_sz - cur_sz)
-		cur_reg_max_len = max_sz - cur_sz;
-
-	    assert(cur_reg_max_len > 0);
-
-	    /* Add memtype ol pairs while we have not passed
-	     * cur_reg_max_len or the max number of ol pairs
-	     * allowed */
-	    act_reg_sz = 0;
-	    exit_loop = 0;
-	    while ((act_reg_sz < cur_reg_max_len) && 
-		   (exit_loop == 0))
-	    {
-		view_state_add_region(
-		    cur_reg_max_len - act_reg_sz,
-		    my_mem_view_state_p,
-		    &agg_mem_st_reg, &agg_mem_act_reg_sz, 
-		    i);
-		act_reg_sz += agg_mem_act_reg_sz;
-		
-#ifdef DEBUG2
-		fprintf(stderr, "ADIOI_Build_client_pre_req: %s Mem region"
-			"(proc=%d,off=%Ld,sz=%Ld)\n",
-			off_type_name[i], agg_rank, agg_mem_st_reg, 
-			agg_mem_act_reg_sz);
-#endif
-		switch(i)
-		{
-		    case TEMP_OFF:
-			/* Increment the ol list count if the next
-			 * region is not adjacent to the previous
-			 * region. */
-			if (agg_mem_next_off != agg_mem_st_reg)
-			{
-			    agg_ol_ct++;
-			    if (agg_ol_ct == max_ol_ct)
-				exit_loop = 1;
-			}
-			agg_mem_next_off = 
-			    agg_mem_st_reg + agg_mem_act_reg_sz;
-			break;
-		    case REAL_OFF:
-			/* Set the ol list for the memtype that
-			 * will map to our aggregator, coaslescing
-			 * if possible. */
-			agg_next_off_idx = agg_ol_cur_ct;
-			if (agg_mem_next_off != agg_mem_st_reg)
-			{
-			    my_mem_view_state_p->
-				pre_disp_arr[agg_next_off_idx] = 
-				agg_mem_st_reg;
-			    my_mem_view_state_p->
-				pre_blk_arr[agg_next_off_idx] = 
-				agg_mem_act_reg_sz;
-			    agg_ol_cur_ct++;
-			    if (agg_ol_cur_ct == agg_ol_ct)
-				exit_loop = 1;
-			}
-			else
-			{
-			    my_mem_view_state_p->
-				pre_blk_arr[agg_next_off_idx - 1]
-				+= agg_mem_act_reg_sz;
-			}
-			agg_mem_next_off = 
-			    agg_mem_st_reg + agg_mem_act_reg_sz;
-			break;
-		    default:
-			fprintf(stderr, "ADIOI_Build_client_pre_req: "
-				"Impossible type\n");
-		}
-	    }
-
-	    /* Advance the filetype flatten state appropriately to
-	     * match the data advanced in the memtype flatten state.
-	     * Should only take at most a single view_state_add_region
-	     * call since the memtype cannot proceed beyond the end of
-	     * a contig piece in the file type. */
-	    view_state_add_region(act_reg_sz - tmp_reg_sz,
-				  agg_file_view_state_p,
-				  &fill_st_reg, &fill_reg_sz, i);
-#ifdef DEBUG2
-	    fprintf(stderr, "ADIOI_Build_client_pre_req: %s File region"
-		    " (proc=%d,off=%Ld,sz=%Ld)\n",
-		    off_type_name[i], agg_rank, fill_st_reg, fill_reg_sz);
-#endif
-	    if (fill_reg_sz != act_reg_sz)
-	    {
-		fprintf(stderr, "ADIOI_Build_client_pre_req: "
-			"view_state_add_region failed to match the memtype\n");
-		return -1;
-	    }
-	    
-	    cur_sz += act_reg_sz;
-	}
-	
-	/* On the first pass, allocate the memory structures for
-	 * storing the preprocessed information */
-	if (i == TEMP_OFF)
-	{
-	    if ((my_mem_view_state_p->pre_disp_arr = (MPI_Aint *)
-		 ADIOI_Malloc(agg_ol_ct * sizeof(MPI_Aint))) == NULL)
-	    {
-		fprintf(stderr, "ADIOI_Build_client_pre_req: malloc "
-                        "pre_disp_arr of size %ld failed\n",
-                        (long int)agg_ol_ct * sizeof(MPI_Aint));
-                return -1;
-	    }
-	    if ((my_mem_view_state_p->pre_blk_arr = (int *) 
-		 ADIOI_Malloc(agg_ol_ct * sizeof(int))) == NULL)
-	    {
-		ADIOI_Free(my_mem_view_state_p->pre_disp_arr);
-		fprintf(stderr, "ADIOI_Build_client_pre_req: malloc "
-			"agg_blk_arr of size %ld failed\n",
-			(long int)agg_ol_ct * sizeof(int));
-		return -1;
-	    }
-	}
-    }
-
-    my_mem_view_state_p->pre_sz = cur_sz;
-    my_mem_view_state_p->pre_ol_ct = agg_ol_ct;
-
-#ifdef DEBUG1
-    fprintf(stderr, "ADIOI_Build_client_pre_req:(agg=%d,cur_ol_count=%d"
-	    "=ol_count=%d)\n",
-	    agg_rank, my_mem_view_state_p->pre_ol_ct, agg_ol_ct);
-#endif
-
-#ifdef DEBUG2
-    if (agg_ol_ct > 0)
-    {
-	fprintf(stderr, "ADIOI_Build_client_pre_req: agg=%d,pre_sz=%Ld "
-		"(off,len) = \n", agg_rank, my_mem_view_state_p->pre_sz);
-	for (i = 0; i < my_mem_view_state_p->pre_ol_ct; i++)
-	{
-	    fprintf(stderr, "[%d](%d,%d) ", i, 
-		    my_mem_view_state_p->pre_disp_arr[i], 
-		    my_mem_view_state_p->pre_blk_arr[i]);
-	    if (i % 5 == 0 && i != 0)
-		fprintf(stderr, "\n");
-	}
-	fprintf(stderr, "\n");
-    }
-#endif
-
-    return 0;
-}
-
-/* process_pre_req() allows ADIOI_Build_client_req to use the pre_req
- * information. */
-
-static int process_pre_req(ADIO_File fd,
-                           int agg_rank,
-                           int agg_idx,
-                           view_state *my_mem_view_state_p,
-                           view_state *agg_file_view_state_p,
-                           ADIO_Offset agg_comm_sz,
-			   int off_type,
-			   MPI_Aint *agg_disp_arr,
-			   int *agg_blk_arr,
-			   ADIO_Offset *agg_comm_pre_sz_p,
-			   ADIO_Offset *agg_comm_cur_sz_p,
-			   ADIO_Offset *agg_comm_sz_p,
-			   int *agg_ol_cur_ct_p,
-			   int *agg_ol_ct_p,
-			   ADIO_Offset *agg_mem_next_off_p)
-{
-    int i, has_partial = 0;
-    MPI_Aint partial_disp = 0;
-    int partial_len = 0;
-    ADIO_Offset tmp_agg_comm_pre_sz = 0;
-
-    assert (my_mem_view_state_p->pre_sz > 0);
-    switch(off_type)
-    {
-	case TEMP_OFF:
-	    /* Use only some of the precalculated data */
-	    if (my_mem_view_state_p->pre_sz > *agg_comm_sz_p)
-	    {
-		for (i = 0; i < my_mem_view_state_p->pre_ol_ct; i++)
-		{
-		    if ((my_mem_view_state_p->pre_blk_arr[i] + 
-			 *agg_comm_pre_sz_p) > *agg_comm_sz_p)
-		    {
-			has_partial = 1;
-			partial_len = *agg_comm_sz_p - *agg_comm_pre_sz_p;
-			*agg_comm_pre_sz_p = *agg_comm_sz_p;
-			i++;
-			break;
-		    }
-		    else if ((my_mem_view_state_p->pre_blk_arr[i] +
-			      *agg_comm_pre_sz_p) == *agg_comm_sz_p)
-		    {
-			*agg_comm_pre_sz_p += 
-			    my_mem_view_state_p->pre_blk_arr[i];
-			i++;
-			break;
-		    }
-		    else
-			*agg_comm_pre_sz_p += 
-			    my_mem_view_state_p->pre_blk_arr[i];
-		}
-		
-		if (has_partial == 1)
-		{
-		    *agg_mem_next_off_p = 
-			my_mem_view_state_p->pre_disp_arr[i - 1] + 
-			partial_len;
-		}
-		else
-		{
-		    *agg_mem_next_off_p = 
-			my_mem_view_state_p->pre_disp_arr[i - 1] + 
-			my_mem_view_state_p->pre_blk_arr[i - 1];
-		}
-		
-		*agg_comm_cur_sz_p = *agg_comm_pre_sz_p;
-		*agg_ol_ct_p = i;
-		
-	    }
-	    else /* Use all the precalculated data */
-	    {
-		*agg_comm_pre_sz_p = my_mem_view_state_p->pre_sz;
-		*agg_comm_cur_sz_p = *agg_comm_pre_sz_p;
-		*agg_ol_ct_p = my_mem_view_state_p->pre_ol_ct;
-		*agg_mem_next_off_p = 
-		    my_mem_view_state_p->pre_disp_arr[
-			my_mem_view_state_p->pre_ol_ct - 1] +
-		    my_mem_view_state_p->pre_blk_arr[
-			my_mem_view_state_p->pre_ol_ct - 1];
-	    }
-#ifdef DEBUG1
-	    fprintf(stderr, "process_pre_req: TEMP_OFF "
-		    "agg_comm_pre_sz=%Ld,agg_comm_cur_sz=%Ld,agg_ol_ct=%d\n",
-		    *agg_comm_pre_sz_p, *agg_comm_cur_sz_p, *agg_ol_ct_p);
-#endif
-	    assert(*agg_comm_cur_sz_p <= *agg_comm_sz_p);
-	    break;
-	case REAL_OFF:
-	    /* Set the ol list for the memtype that will map to our
-	     * aggregator, coaslescing if possible. */
-	    for (i = 0; i < my_mem_view_state_p->pre_ol_ct; i++)
-	    {
-		agg_disp_arr[i] = my_mem_view_state_p->pre_disp_arr[i];
-		agg_blk_arr[i]  = my_mem_view_state_p->pre_blk_arr[i];
-		
-		if ((my_mem_view_state_p->pre_blk_arr[i] + 
-		     tmp_agg_comm_pre_sz) > *agg_comm_pre_sz_p)
-		{
-		    has_partial = 1;
-		    agg_blk_arr[i] = *agg_comm_pre_sz_p - tmp_agg_comm_pre_sz;
-		    tmp_agg_comm_pre_sz = *agg_comm_pre_sz_p;
-		    partial_disp = my_mem_view_state_p->pre_disp_arr[i] +
-			agg_blk_arr[i];
-		    partial_len  = my_mem_view_state_p->pre_blk_arr[i] - 
-			agg_blk_arr[i];
-		    i++;
-		    break;
-		}
-		else if ((my_mem_view_state_p->pre_blk_arr[i] +
-			  tmp_agg_comm_pre_sz) == *agg_comm_pre_sz_p)
-		{
-		    tmp_agg_comm_pre_sz +=  
-			my_mem_view_state_p->pre_blk_arr[i];
-		    i++;
-		    break;
-		}
-		else
-		    tmp_agg_comm_pre_sz +=
-			my_mem_view_state_p->pre_blk_arr[i];
-	    }
-	    *agg_mem_next_off_p = agg_disp_arr[i - 1] + agg_blk_arr[i - 1];
-	    *agg_ol_cur_ct_p = i;
-	    *agg_comm_cur_sz_p = *agg_comm_pre_sz_p;
-	    
-	    /* Clean up the ol pairs we used */	    
-	    if ((i < my_mem_view_state_p->pre_ol_ct) || (has_partial == 1))
-	    {
-		int remain_ol_ct = 
-		    my_mem_view_state_p->pre_ol_ct - i + has_partial;
-		MPI_Aint *new_pre_disp_arr = NULL;
-		int *new_pre_blk_arr = NULL;
-		
-		if ((new_pre_disp_arr = (MPI_Aint *)
-		     ADIOI_Malloc(remain_ol_ct * sizeof(MPI_Aint))) == NULL)
-		{
-		    fprintf(stderr, "process_pre_req: malloc "
-			    "new_pre_disp_arr failed\n");
-		    return -1;
-		}
-		if ((new_pre_blk_arr = (int *)
-		     ADIOI_Malloc(remain_ol_ct * sizeof(int))) == NULL)
-                {
-                    fprintf(stderr, "process_pre_req: malloc "
-                            "new_pre_blk_arr failed\n");
-                    return -1;
-                }
-		
-		memcpy(new_pre_disp_arr, 
-		       &(my_mem_view_state_p->pre_disp_arr[i - has_partial]),
-		       remain_ol_ct * sizeof(MPI_Aint));
-		memcpy(new_pre_blk_arr, 
-		       &(my_mem_view_state_p->pre_blk_arr[i - has_partial]),
-		       remain_ol_ct * sizeof(int));
-		
-		/* Set the partial len of the first piece */
-		if (has_partial == 1)
-		{
-		    /* new_pre_disp_arr[remain_ol_ct - 1] = partial_disp;
-		       new_pre_blk_arr[remain_ol_ct - 1]  = partial_len; */
-		    new_pre_disp_arr[0] = partial_disp;
-		    new_pre_blk_arr[0]  = partial_len;
-		}
-		
-		ADIOI_Free(my_mem_view_state_p->pre_disp_arr);
-		ADIOI_Free(my_mem_view_state_p->pre_blk_arr);
-		
-		my_mem_view_state_p->pre_disp_arr = new_pre_disp_arr;
-		my_mem_view_state_p->pre_blk_arr  = new_pre_blk_arr;
-		my_mem_view_state_p->pre_ol_ct = remain_ol_ct;
-		my_mem_view_state_p->pre_sz -= *agg_comm_pre_sz_p;
-	    }
-	    else /* Used all the precalculated ol pairs */
-	    {
-		ADIOI_Free(my_mem_view_state_p->pre_disp_arr);
-		ADIOI_Free(my_mem_view_state_p->pre_blk_arr);
-		
-		my_mem_view_state_p->pre_disp_arr = NULL;
-		my_mem_view_state_p->pre_blk_arr = NULL;
-		my_mem_view_state_p->pre_ol_ct = 0;
-		my_mem_view_state_p->pre_sz = 0;
-	    }
-#ifdef DEBUG1
-	    fprintf(stderr, "process_pre_req: REAL_OFF "
-		    "agg_comm_pre_sz=%Ld,agg_comm_cur_sz=%Ld,agg_ol_ct=%d,"
-		    "agg_ol_cur_ct=%d\n",
-		    *agg_comm_pre_sz_p, *agg_comm_cur_sz_p, *agg_ol_ct_p, 
-		    *agg_ol_cur_ct_p);
-#endif
-	    break;
-	default:
-	    fprintf(stderr, "process_pre_req: Invalid off_type %d\n",
-		    off_type);
-    }
-    return 0;
-}
-
-/* ADIOI_Build_client_req() creates a memory datatype to transfer data
- * to/from a particular aggregator. */
-
-int ADIOI_Build_client_req(ADIO_File fd,
-			   int agg_rank,
-			   int agg_idx,
-			   view_state *my_mem_view_state_p,
-			   view_state *agg_file_view_state_p,
-			   ADIO_Offset agg_comm_sz,
-			   MPI_Datatype *agg_comm_dtype_p)
-{
-    MPI_Aint *agg_disp_arr = NULL;
-    int *agg_blk_arr = NULL;
-    ADIO_Offset st_reg = 0, act_reg_sz = 0, tmp_reg_sz = 0;
-    ADIO_Offset cur_off = -1, cur_reg_max_len = -1;
-    ADIO_Offset agg_mem_st_reg = 0, agg_mem_act_reg_sz = 0;
-    int agg_ol_ct = 0, agg_ol_cur_ct = 0;
-    int i = 0, agg_next_off_idx = -1;
-    ADIO_Offset agg_mem_next_off = 0, agg_comm_cur_sz = 0, agg_comm_pre_sz = 0;
-    ADIO_Offset *fr_st_off_arr = fd->file_realm_st_offs;
-    MPI_Datatype *fr_type_arr = fd->file_realm_types;
-    flatten_state *tmp_mem_state_p = NULL, *tmp_file_state_p = NULL;
-#ifdef DTYPE_SKIP
-    int skip_type_ct;
-#endif
-
-    if (agg_idx < 0 || agg_idx >= fd->hints->cb_nodes)
-    {
-#ifdef DEBUG1
-	fprintf(stderr, "ADIOI_Build_client_req: agg_rank %d does not map "
-		"to a valid node in cb_node\n", agg_rank);
-#endif
-	return 0;
-    }
-
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5018, 0, NULL);
-#endif
-
-#ifdef DEBUG1
-    fprintf(stderr, "ADIOI_Build_client_req:(agg=%d,size_req=%Ld)\n",
-	    agg_idx, agg_comm_sz);
-#endif
-    
-    /* On the first pass see how many offset-length pairs are
-     * necessary for each aggregator.  Then allocate the correct
-     * amount of offset-length pairs for handling each aggregator's
-     * particular data size.  On the last pass, we actually create the
-     * offset-length pairs. */
-    for (i = 0; i < MAX_OFF_TYPE; i++)
-    {
-	switch(i)
-	{
-	    case TEMP_OFF:
-		tmp_mem_state_p  = &(my_mem_view_state_p->tmp_state);
-		tmp_file_state_p = &(agg_file_view_state_p->tmp_state);
-		break;
-	    case REAL_OFF:
-		tmp_mem_state_p  = &(my_mem_view_state_p->cur_state);
-		tmp_file_state_p = &(agg_file_view_state_p->cur_state);
-		break;
-	    default:
-		fprintf(stderr, "ADIOI_Build_client_pre_req: "
-			"Invalid off type %d\n", i);
-	}
-
-	agg_comm_cur_sz = 0;
-	agg_mem_next_off = -1;
-
-	/* First try to preprocess anything we can */
-	if (my_mem_view_state_p->pre_sz > 0)
-	{
-	    process_pre_req(fd,
-			    agg_rank,
-			    agg_idx,
-			    my_mem_view_state_p,
-			    agg_file_view_state_p,
-			    agg_comm_sz,
-			    i,
-			    agg_disp_arr,
-			    agg_blk_arr,
-			    &agg_comm_pre_sz,
-			    &agg_comm_cur_sz,
-			    &agg_comm_sz,
-			    &agg_ol_cur_ct,
-			    &agg_ol_ct,
-			    &agg_mem_next_off);
-	}
-	
-	while (agg_comm_cur_sz < agg_comm_sz)
-	{	
-	    find_next_off(fd, agg_file_view_state_p,
-			  fr_st_off_arr[agg_idx],
-			  &(fr_type_arr[agg_idx]),
-			  i,
-			  &cur_off,
-			  &cur_reg_max_len);
-	    
-	    assert(cur_off != -1);
-	    
-	    /* Add up to the end of the file realm or as many bytes
-	     * are left for this particular aggregator in the client's
-	     * filetype */
-	    if (cur_reg_max_len > (agg_comm_sz - agg_comm_cur_sz))
-	    {
-		cur_reg_max_len = agg_comm_sz - agg_comm_cur_sz;
-	    }
-	    assert(cur_reg_max_len > 0);
-	
-	    view_state_add_region(
-		cur_reg_max_len,
-		agg_file_view_state_p,
-		&st_reg, &act_reg_sz, i);
-	    
-#ifdef DEBUG2
-	    fprintf(stderr, "ADIOI_Build_client_req: %s File region"
-		    " (proc=%d,off=%Ld,sz=%Ld)\n",
-		    off_type_name[i], agg_rank, cur_off, act_reg_sz);
-#endif
-	    
-	    /* Before translating the file regions to memory regions,
-	     * we first must advance to the proper point in the
-	     * mem_view_state for this aggregator to match the
-	     * file_view_state. */
-	    
-	    assert(tmp_file_state_p->cur_sz - act_reg_sz >= 
-		   tmp_mem_state_p->cur_sz);
-	    
-	    while (tmp_file_state_p->cur_sz - act_reg_sz != 
-		   tmp_mem_state_p->cur_sz)
-	    {
-		ADIO_Offset fill_st_reg = -1, fill_reg_sz = -1;
-#ifdef DTYPE_SKIP
-		if (my_mem_view_state_p->flat_type_p->count > 1) {
-		    /* let's see if we can skip whole memory datatypes */
-		    skip_type_ct =
-			(tmp_file_state_p->cur_sz - act_reg_sz -
-			 tmp_mem_state_p->cur_sz) /
-			my_mem_view_state_p->type_sz;
-		    if (skip_type_ct > 0) {
-			tmp_mem_state_p->cur_sz +=
-			    skip_type_ct * my_mem_view_state_p->type_sz;
-			tmp_mem_state_p->abs_off +=
-			    skip_type_ct * my_mem_view_state_p->ext;
-			if ((tmp_mem_state_p->cur_sz - act_reg_sz) ==
-			    tmp_file_state_p->cur_sz)
-			    break;
-		    }
-		}
-#endif
-		view_state_add_region(
-		    tmp_file_state_p->cur_sz - 
-		    act_reg_sz - tmp_mem_state_p->cur_sz,
-		    my_mem_view_state_p,
-		    &fill_st_reg,
-		    &fill_reg_sz, i);
-	    }
-	    
-	    /* Based on how large the act_reg_sz is, first figure
-	     * out how many memory offset-length pairs are
-	     * necessary and then set the offset-length pairs. */
-	    tmp_reg_sz = 0;
-	    while (tmp_reg_sz != act_reg_sz)
-	    {
-		view_state_add_region(
-		    act_reg_sz - tmp_reg_sz,
-		    my_mem_view_state_p,
-		    &agg_mem_st_reg, &agg_mem_act_reg_sz, 
-		    i);
-		tmp_reg_sz += agg_mem_act_reg_sz;
-		
-#ifdef DEBUG2
-		fprintf(stderr, "ADIOI_Build_client_req: %s Mem region"
-			"(off=%Ld,sz=%Ld)\n",
-			off_type_name[i], agg_mem_st_reg, 
-			agg_mem_act_reg_sz);
-#endif
-		agg_comm_cur_sz += agg_mem_act_reg_sz;
-		switch(i)
-		{
-		    case TEMP_OFF:
-			/* Increment the ol list count if the next
-			 * region is not adjacent to the previous
-			 * region. */
-			if (agg_mem_next_off != agg_mem_st_reg)
-			{
-			    agg_ol_ct++;
-			}
-			agg_mem_next_off = 
-			    agg_mem_st_reg + agg_mem_act_reg_sz;
-			break;
-		    case REAL_OFF:
-			/* Set the ol list for the memtype that
-			 * will map to our aggregator, coaslescing
-			 * if possible. */
-			agg_next_off_idx = agg_ol_cur_ct;
-			if (agg_mem_next_off != agg_mem_st_reg)
-			{
-			    agg_disp_arr[agg_next_off_idx] = 
-				agg_mem_st_reg;
-			    agg_blk_arr[agg_next_off_idx] = 
-				agg_mem_act_reg_sz;
-			    agg_ol_cur_ct++;
-			}
-			else
-			{
-			    agg_blk_arr[agg_next_off_idx - 1]
-				+= agg_mem_act_reg_sz;
-			}
-			agg_mem_next_off = 
-			    agg_mem_st_reg + agg_mem_act_reg_sz;
-			break;
-		    default:
-			fprintf(stderr, "ADIOI_Build_client_req: "
-				"Impossible type\n");
-		}
-	    }
-	}
-	
-	/* On the first pass, allocate the memory structures for
-	 * creating the MPI_hindexed type. */
-	if (i == TEMP_OFF)
-	{	    
-	    /* Allocate offset-length pairs for creating hindexed
-	     * MPI_Datatypes for each aggregator */
-	    if ((agg_disp_arr = (MPI_Aint *) 
-		 ADIOI_Malloc(agg_ol_ct * sizeof(MPI_Aint))) == NULL)
-	    {
-		fprintf(stderr, "ADIOI_Build_client_req: malloc "
-			"agg_disp_arr of size %ld failed\n",
-			(long int)agg_ol_ct * sizeof(MPI_Aint));
-		return -1;
-	    }
-	    if ((agg_blk_arr = (int *) 
-		 ADIOI_Malloc(agg_ol_ct * sizeof(int))) == NULL)
-	    {
-		ADIOI_Free(agg_disp_arr);
-		fprintf(stderr, "ADIOI_Build_client_req: malloc "
-			"agg_blk_arr of size %ld failed\n",
-			(long int)agg_ol_ct * sizeof(int));
-		return -1;
-	    }
-	}
-    }
-
-    assert(agg_ol_ct == agg_ol_cur_ct);
-#ifdef DEBUG1
-    fprintf(stderr, 
-	    "ADIOI_Build_client_req:(agg=%d,cur_ol_count=%d=ol_count=%d)\n",
-	    agg_rank, agg_ol_cur_ct, agg_ol_ct);
-#endif
-
-#ifdef DEBUG2
-    if (agg_ol_ct > 0)
-    {
-	fprintf(stderr, "ADIOI_Build_client_req: p %d (off,len) = ", agg_rank);
-	for (i = 0; i < agg_ol_ct; i++)
-	{
-	    fprintf(stderr, "[%d](%d,%d) ", i, 
-		    agg_disp_arr[i], agg_blk_arr[i]);
-	    if (i % 5 == 0 && i != 0)
-		fprintf(stderr, "\n");
-	}
-	fprintf(stderr, "\n");
-    }
-#endif
-#ifdef DEBUG1
-    fprintf(stderr, 
-	    "ADIOI_Build_client_req:(agg=%d,pre_ol_count=%d)\n",
-	    agg_idx, my_mem_view_state_p->pre_ol_ct);
-#endif
-
-#ifdef DEBUG2
-    if (my_mem_view_state_p->pre_sz > 0)
-    {
-	fprintf(stderr, "ADIOI_Build_client_req: p %d pre(off,len) = ", 
-		agg_idx);
-	for (i = 0; i < my_mem_view_state_p->pre_ol_ct; i++)
-	{
-	    fprintf(stderr, "[%d](%d,%d) ", i, 
-		    my_mem_view_state_p->pre_disp_arr[i], 
-		    my_mem_view_state_p->pre_blk_arr[i]);
-	    if (i % 5 == 0 && i != 0)
-		fprintf(stderr, "\n");
-	}
-	fprintf(stderr, "\n");
-    }
-#endif
-
-    /* Create the aggregator MPI_Datatype */
-    if (agg_comm_sz > 0)
-    {
-	MPI_Type_create_hindexed(agg_ol_ct, agg_blk_arr, agg_disp_arr, MPI_BYTE,
-			         agg_comm_dtype_p);
-	MPI_Type_commit(agg_comm_dtype_p);
-    }
-    else
-    {
-	*agg_comm_dtype_p = MPI_BYTE;
-    }
-
-    ADIOI_Free(agg_blk_arr);
-    ADIOI_Free(agg_disp_arr);
-
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5019, 0, NULL);
-#endif    
-    return 0;
-}
-
-
diff --git a/3rd-party/romio321/adio/common/ad_coll_exch_new.c b/3rd-party/romio321/adio/common/ad_coll_exch_new.c
deleted file mode 100644
index 7013a3a8554..00000000000
--- a/3rd-party/romio321/adio/common/ad_coll_exch_new.c
+++ /dev/null
@@ -1,506 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include <assert.h>
-#include "adio.h"
-#include "adio_extern.h"
-#ifdef AGGREGATION_PROFILE
-#include "mpe.h"
-#endif
-
-/*
-#define DEBUG
-#define DEBUG2
-*/
-
-#define COUNT_EXCH          0
-#define BLOCK_LENS          1
-#define INDICES             2
-#define FPIND_DISP_OFF_SZ   3
-
-
-typedef struct {
-    int count;
-    ADIO_Offset fp_ind;
-    ADIO_Offset disp;
-    ADIO_Offset byte_off;
-    ADIO_Offset sz;
-    ADIO_Offset ext;
-    ADIO_Offset type_sz;
-} amount_and_extra_data_t;
-
-/* Debugging function to print out an ADIOI_Flatlist_node. */
-void ADIOI_Print_flatlist_node(ADIOI_Flatlist_node *flatlist_node_p)
-{
-    int i;
-    if (flatlist_node_p == NULL)
-    {
-	fprintf(stderr, "print flatlist node of NULL ptr\n");
-	return;
-    }
-    fprintf(stderr, "print flatlist node count = %d (idx,blocklen)\n", 
-	    (int)flatlist_node_p->count);
-    for (i = 0; i < flatlist_node_p->count; i++)
-    {
-	if (i % 5 == 0 && i != 0)
-	{
-	    fprintf(stderr, "%d=(%lld,%lld)\n", i, (long long)flatlist_node_p->indices[i],
-		    (long long)flatlist_node_p->blocklens[i]);
-	}
-	else
-	    fprintf(stderr, "%d=(%lld,%lld) ", i, (long long)flatlist_node_p->indices[i],
-		    (long long)flatlist_node_p->blocklens[i]);
-    }
-    fprintf(stderr, "\n");
-}
-
-/* Since ADIOI_Flatten_datatype won't add a contig datatype to the
- * ADIOI_Flatlist, we can force it to do so with this function. */
-ADIOI_Flatlist_node * ADIOI_Add_contig_flattened(MPI_Datatype contig_type)
-{
-    MPI_Count contig_type_sz = -1;
-    ADIOI_Flatlist_node *flat_node_p = ADIOI_Flatlist;
-    
-    /* Add contig type to the end of the list if it doesn't already
-     * exist. */
-    while (flat_node_p->next)
-    {
-	if (flat_node_p->type == contig_type)
-	    return flat_node_p;
-	flat_node_p = flat_node_p->next;
-    }
-    if (flat_node_p->type == contig_type)
-	return flat_node_p;
-
-    MPI_Type_size_x(contig_type, &contig_type_sz);
-    if ((flat_node_p->next = (ADIOI_Flatlist_node *) ADIOI_Malloc
-	 (sizeof(ADIOI_Flatlist_node))) == NULL)
-    {
-	fprintf(stderr, "ADIOI_Add_contig_flattened: malloc next failed\n");
-    }
-    flat_node_p = flat_node_p->next;
-    flat_node_p->type = contig_type;
-    if ((flat_node_p->blocklens = (ADIO_Offset *) ADIOI_Malloc(sizeof(ADIO_Offset))) == NULL)
-    {
-	fprintf(stderr, "ADIOI_Flatlist_node: malloc blocklens failed\n");
-    }
-    if ((flat_node_p->indices = (ADIO_Offset *) 
-	 ADIOI_Malloc(sizeof(ADIO_Offset))) == NULL)
-    {
-	fprintf(stderr, "ADIOI_Flatlist_node: malloc indices failed\n");
-    }
-    flat_node_p->blocklens[0] = contig_type_sz;
-    flat_node_p->indices[0] = 0;
-    flat_node_p->count = 1;
-    flat_node_p->next = NULL;
-    return flat_node_p;
-}
-
-/* ADIOI_Exchange_file_views - Sends all the aggregators the file
- * views and file view states of the clients.  It fills in the
- * client_file_view_state_arr for the aggregators and the
- * my_mem_view_state for the client.  It also initializes the
- * agg_file_view_state for all clients, which is the view for each
- * aggregator of a client's filetype. */
-void ADIOI_Exch_file_views(int myrank, int nprocs, int file_ptr_type,
-			   ADIO_File fd, int count,
-			   MPI_Datatype datatype, ADIO_Offset off,
-			   view_state *my_mem_view_state_arr,
-			   view_state *agg_file_view_state_arr,
-			   view_state *client_file_view_state_arr)
-{
-    /* Convert my own fileview to an ADIOI_Flattened type and a
-     * disp. MPI_Alltoall the count of ADIOI_Flatlist nodes.
-     * MPI_Isend/Irecv the block_lens, indices of ADIOI_Flatlist node
-     * to/from each of the aggregators with the rest of the file view
-     * state. */
-
-    int i = -1, j = -1;
-    amount_and_extra_data_t *send_count_arr = NULL;
-    amount_and_extra_data_t *recv_count_arr = NULL;
-    int send_req_arr_sz = 0;
-    int recv_req_arr_sz = 0;
-    MPI_Request *send_req_arr = NULL, *recv_req_arr = NULL;
-    MPI_Status *statuses = NULL;
-    ADIO_Offset disp_off_sz_ext_typesz[6];
-    MPI_Aint memtype_extent, filetype_extent, lb;
-    int ret = -1;
-
-    /* parameters for datatypes */
-    ADIOI_Flatlist_node *flat_mem_p = NULL, *flat_file_p = NULL;
-    MPI_Count memtype_sz = -1;
-    int memtype_is_contig = -1;
-    ADIO_Offset filetype_sz = -1;
-
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5014, 0, NULL);
-#endif
-    /* The memtype will be freed after the call.  The filetype will be
-     * freed in the close and should have been flattened in the file
-     * view. */
-    MPI_Type_size_x(datatype, &memtype_sz);
-    MPI_Type_get_extent(datatype, &lb, &memtype_extent);
-    if (memtype_sz == memtype_extent) {
-	memtype_is_contig = 1;
-	flat_mem_p = ADIOI_Add_contig_flattened(datatype);
-	flat_mem_p->blocklens[0] = memtype_sz*count;
-    }
-    else {
-	flat_mem_p = ADIOI_Flatten_and_find(datatype);
-    }
-
-    MPI_Type_get_extent(fd->filetype, &lb, &filetype_extent);
-    MPI_Type_size_x(fd->filetype, &filetype_sz);
-    if (filetype_extent == filetype_sz) {
-	flat_file_p = ADIOI_Add_contig_flattened(fd->filetype);
-	flat_file_p->blocklens[0] = memtype_sz*count;
-	filetype_extent = memtype_sz*count;
-	filetype_sz = filetype_extent;
-    }
-    else {
-        flat_file_p = ADIOI_Flatlist;
-        while (flat_file_p->type != fd->filetype)
-            flat_file_p = flat_file_p->next; 
-    }
-
-    disp_off_sz_ext_typesz[0] = fd->fp_ind;
-    disp_off_sz_ext_typesz[1] = fd->disp;
-    disp_off_sz_ext_typesz[2] = off;
-    disp_off_sz_ext_typesz[3] = memtype_sz*count;
-    disp_off_sz_ext_typesz[4] = (ADIO_Offset) filetype_extent;
-    disp_off_sz_ext_typesz[5] = (ADIO_Offset) filetype_sz;
-
-    if (fd->hints->cb_alltoall != ADIOI_HINT_DISABLE) {
-        recv_count_arr = ADIOI_Calloc(nprocs, sizeof(amount_and_extra_data_t));
-        send_count_arr = ADIOI_Calloc(nprocs, sizeof(amount_and_extra_data_t));
-    } else {
-        send_count_arr = ADIOI_Calloc(fd->hints->cb_nodes,
-				  sizeof(amount_and_extra_data_t));
-
-        /* only aggregators receive data */
-        if (fd->is_agg) {
-	    recv_count_arr = ADIOI_Calloc(nprocs, 
-			    sizeof(amount_and_extra_data_t));
-	    recv_req_arr = ADIOI_Malloc (nprocs * sizeof(MPI_Request));
-	    for (i=0; i < nprocs; i++)
-	        MPI_Irecv (&recv_count_arr[i], sizeof(amount_and_extra_data_t),
-		       MPI_BYTE, i, COUNT_EXCH, fd->comm, &recv_req_arr[i]);
-        }
-    
-        /* only send data to aggregators */
-        send_req_arr = ADIOI_Calloc (fd->hints->cb_nodes, sizeof(MPI_Request));
-        for (i=0; i < fd->hints->cb_nodes; i++) {
-	    send_count_arr[i].count    = flat_file_p->count;
-	    send_count_arr[i].fp_ind   = disp_off_sz_ext_typesz[0];
-	    send_count_arr[i].disp     = disp_off_sz_ext_typesz[1];
-	    send_count_arr[i].byte_off = disp_off_sz_ext_typesz[2];
-	    send_count_arr[i].sz       = disp_off_sz_ext_typesz[3];
-	    send_count_arr[i].ext      = disp_off_sz_ext_typesz[4];
-	    send_count_arr[i].type_sz  = disp_off_sz_ext_typesz[5];
-	    MPI_Isend (&send_count_arr[i], sizeof(amount_and_extra_data_t),
-		   MPI_BYTE, fd->hints->ranklist[i], COUNT_EXCH, fd->comm,
-		   &send_req_arr[i]);
-        }
-    }
-
- 
-    /* Every client has to build mem and file view_states for each aggregator.
-     * We initialize their values here.  and we also initialize
-     * send_count_arr */
-
-    if (memtype_is_contig) {
-	/* if memory is contigous, we now replace memtype_sz and
-	 * memtype_extent with the full access size */
-	memtype_sz *= count;
-	memtype_extent = memtype_sz;
-    }
-
-    for (i = 0; i < fd->hints->cb_nodes; i++)
-    {
-	int tmp_agg_idx = fd->hints->ranklist[i];
-	memset(&(my_mem_view_state_arr[tmp_agg_idx]), 0, sizeof(view_state));
-	my_mem_view_state_arr[tmp_agg_idx].sz          =
-	    disp_off_sz_ext_typesz[3];
-	my_mem_view_state_arr[tmp_agg_idx].ext         =
-	    (ADIO_Offset) memtype_extent;
-	my_mem_view_state_arr[tmp_agg_idx].type_sz     =
-	    (ADIO_Offset) memtype_sz;
-	my_mem_view_state_arr[tmp_agg_idx].flat_type_p = flat_mem_p;
-	ADIOI_init_view_state(file_ptr_type,
-			1,
-			&(my_mem_view_state_arr[tmp_agg_idx]),
-			TEMP_OFF);
-	ADIOI_init_view_state(file_ptr_type,
-			1,
-			&(my_mem_view_state_arr[tmp_agg_idx]),
-			REAL_OFF);
-	
-	memset(&(agg_file_view_state_arr[tmp_agg_idx]), 0, sizeof(view_state));
-	agg_file_view_state_arr[tmp_agg_idx].fp_ind    =
-	    disp_off_sz_ext_typesz[0];
-	agg_file_view_state_arr[tmp_agg_idx].disp      =
-	    disp_off_sz_ext_typesz[1];
-	agg_file_view_state_arr[tmp_agg_idx].byte_off  =
-	    disp_off_sz_ext_typesz[2];
-	agg_file_view_state_arr[tmp_agg_idx].sz        =
-	    disp_off_sz_ext_typesz[3];
-	agg_file_view_state_arr[tmp_agg_idx].ext       =
-	    disp_off_sz_ext_typesz[4];
-	agg_file_view_state_arr[tmp_agg_idx].type_sz   =
-	    disp_off_sz_ext_typesz[5];
-	agg_file_view_state_arr[tmp_agg_idx].flat_type_p = flat_file_p;
-
-	ADIOI_init_view_state(file_ptr_type,
-			1,
-			&(agg_file_view_state_arr[tmp_agg_idx]),
-			TEMP_OFF);
-	ADIOI_init_view_state(file_ptr_type,
-			1,
-			&(agg_file_view_state_arr[tmp_agg_idx]),
-			REAL_OFF);
-
-	if (fd->hints->cb_alltoall != ADIOI_HINT_DISABLE) {
-	    send_count_arr[tmp_agg_idx].count    = flat_file_p->count;
-	    send_count_arr[tmp_agg_idx].fp_ind   = disp_off_sz_ext_typesz[0];
-	    send_count_arr[tmp_agg_idx].disp     = disp_off_sz_ext_typesz[1];
-	    send_count_arr[tmp_agg_idx].byte_off = disp_off_sz_ext_typesz[2];
-	    send_count_arr[tmp_agg_idx].sz       = disp_off_sz_ext_typesz[3];
-	    send_count_arr[tmp_agg_idx].ext      = disp_off_sz_ext_typesz[4];
-	    send_count_arr[tmp_agg_idx].type_sz  = disp_off_sz_ext_typesz[5];
-	}
-    }
-
-#ifdef DEBUG2
-    fprintf(stderr, "my own flattened memtype: ");
-    ADIOI_Print_flatlist_node(flat_mem_p);
-    fprintf(stderr, "my own flattened filetype: ");
-    ADIOI_Print_flatlist_node(flat_file_p);
-#endif
-	
-    if (fd->hints->cb_alltoall != ADIOI_HINT_DISABLE) {
-        ret = MPI_Alltoall(send_count_arr, sizeof(amount_and_extra_data_t),
-		       MPI_BYTE, 
-		       recv_count_arr, sizeof(amount_and_extra_data_t),
-		       MPI_BYTE, fd->comm);
-        if (ret != MPI_SUCCESS)
-        {
-	    fprintf(stderr, "ADIOI_Exchange_file_views: MPI_Alltoall failed "
-		"with error %d", ret);
-	    return;
-        }
-    } else {
-        statuses = (MPI_Status *) ADIOI_Malloc(1 + nprocs * sizeof(MPI_Status));
-        if (fd->is_agg) {
-	    MPI_Waitall(nprocs, recv_req_arr, statuses);
-	    ADIOI_Free(recv_req_arr);
-        }
-        MPI_Waitall(fd->hints->cb_nodes, send_req_arr, statuses);
-        ADIOI_Free(statuses);
-        ADIOI_Free(send_req_arr);
-    }
-#ifdef DEBUG2
-    if (fd->hints->cb_alltoall != ADIOI_HINT_DISABLE) {
-        fprintf(stderr, "send_count_arr:");
-        for (i = 0; i < nprocs; i++)
-        {
-	    fprintf(stderr, "[%d]=%d ", i, send_count_arr[i].count);
-        }
-        fprintf(stderr, "\n");
-        fprintf(stderr, "recv_count_arr:");
-        for (i = 0; i < nprocs; i++)
-	{
-	    fprintf(stderr, "[%d]=%d ", i, recv_count_arr[i].count);
-	}
-        fprintf(stderr, "\n");
-    } else {
-        fprintf(stderr, "send_count_arr:");
-        for (i = 0; i < fd->hints->cb_nodes; i++)
-        {
-	    fprintf(stderr, "[%d]=%d ", i, send_count_arr[i].count);
-        }
-        fprintf(stderr, "\n");
-        if (fd->is_agg) {
-	    fprintf(stderr, "recv_count_arr:");
-	    for (i = 0; i < nprocs; i++)
-	    {
-	        fprintf(stderr, "[%d]=%d ", i, recv_count_arr[i].count);
-	    }
-	    fprintf(stderr, "\n");
-        }
-    }
-#endif
-
-    if (fd->hints->cb_alltoall == ADIOI_HINT_DISABLE) {
-        for (i=0; i < fd->hints->cb_nodes; i++)
-	    if (send_count_arr[i].count > 0)
-	        send_req_arr_sz++;
-    }
-    /* Figure out how many counts to send/recv */
-    for (i = 0; i < nprocs; i++)
-    {
-        if (fd->hints->cb_alltoall != ADIOI_HINT_DISABLE) {
-	    if (send_count_arr[i].count > 0)
-	        send_req_arr_sz++;
-	}
-	/* Only aggregators should recv*/
-	if (fd->is_agg) {
-	    if (recv_count_arr[i].count > 0)
-	    {
-		if ((client_file_view_state_arr[i].flat_type_p = 
-		     (ADIOI_Flatlist_node *) ADIOI_Malloc(
-			 sizeof(ADIOI_Flatlist_node))) == NULL)
-		{
-		    fprintf(stderr, "ADIOI_Exchange_file_views: malloc "
-			    "flat_type_p failed\n");
-		}
-		client_file_view_state_arr[i].flat_type_p->count = 
-		    recv_count_arr[i].count;
-		client_file_view_state_arr[i].flat_type_p->indices = 
-		    (ADIO_Offset *) ADIOI_Calloc(recv_count_arr[i].count, 
-						 sizeof(ADIO_Offset));
-		client_file_view_state_arr[i].flat_type_p->blocklens =
-		    (ADIO_Offset *) ADIOI_Calloc(recv_count_arr[i].count, 
-				    sizeof(ADIO_Offset));
-		
-		/* Copy the extra data out of the stuff we Alltoall'd */
-		memcpy (&client_file_view_state_arr[i].fp_ind,
-			&recv_count_arr[i].fp_ind,
-			6*sizeof(ADIO_Offset));
-
-		recv_req_arr_sz++;
-	    }
-	}
-    }
-
-    /* Since ADIOI_Calloc may do other things we add the +1 
-     * to avoid a 0-size malloc */
-    send_req_arr = (MPI_Request *) ADIOI_Calloc(2*(send_req_arr_sz)+1,
-						sizeof(MPI_Request));
-    
-    j = 0;
-    if (recv_req_arr_sz > 0) {
-	assert (fd->is_agg);
-	recv_req_arr = (MPI_Request *) ADIOI_Calloc(2*(recv_req_arr_sz),
-						    sizeof(MPI_Request));
-    	for (i = 0; i < nprocs; i++) {
-	    if (recv_count_arr[i].count > 0) {
-		MPI_Irecv(client_file_view_state_arr[i].flat_type_p->indices,
-			  recv_count_arr[i].count, ADIO_OFFSET, i, 
-			  INDICES, fd->comm, &recv_req_arr[j]);
-		j++;
-		MPI_Irecv(client_file_view_state_arr[i].flat_type_p->blocklens,
-			  recv_count_arr[i].count, ADIO_OFFSET, i,
-			  BLOCK_LENS, fd->comm, &recv_req_arr[j]);
-		j++;
-	    }
-	}
-    }
-
-    if (fd->hints->cb_alltoall != ADIOI_HINT_DISABLE) {
-        j = 0;
-        for (i = 0; i < nprocs; i++) {
-	    if (send_count_arr[i].count > 0) {
-	        MPI_Isend(flat_file_p->indices,
-		      send_count_arr[i].count, ADIO_OFFSET, i,
-                      INDICES, fd->comm, &send_req_arr[j]);
-	        j++;
-	        MPI_Isend(flat_file_p->blocklens,         
-		      send_count_arr[i].count, ADIO_OFFSET, i,
-                      BLOCK_LENS, fd->comm, &send_req_arr[j]);
-	        j++;
-	    }
-        }
-    } else {
-        j = 0;
-        for (i = 0; i < fd->hints->cb_nodes; i++) {
-	    if (send_count_arr[i].count > 0) {
-	        MPI_Isend(flat_file_p->indices,
-		      send_count_arr[i].count, ADIO_OFFSET,
-		      fd->hints->ranklist[i], INDICES, fd->comm,
-		      &send_req_arr[j]);
-	        j++;
-	        MPI_Isend(flat_file_p->blocklens,         
-		      send_count_arr[i].count, ADIO_OFFSET,
-		      fd->hints->ranklist[i], BLOCK_LENS, fd->comm,
-		      &send_req_arr[j]);
-	        j++;
-	    }
-        }
-    }
-
-    /* Since ADIOI_Malloc may do other things we add the +1 
-     * to avoid a 0-size malloc */    
-    statuses = (MPI_Status *) 
-	ADIOI_Malloc(1 + 2 * ADIOI_MAX(send_req_arr_sz,recv_req_arr_sz)
-		     * sizeof(MPI_Status));
-
-    if (send_req_arr_sz > 0) {
-	MPI_Waitall(2 * send_req_arr_sz, send_req_arr, statuses);
-	ADIOI_Free(send_count_arr);
-	ADIOI_Free(send_req_arr);
-    }
-    if (recv_req_arr_sz > 0) {
-	MPI_Waitall(2 * recv_req_arr_sz, recv_req_arr, statuses);
-	ADIOI_Free(recv_count_arr);
-	ADIOI_Free(recv_req_arr);
-    }
-    ADIOI_Free(statuses);
-
-    if (fd->is_agg == 1)
-    {
-	ADIOI_init_view_state(file_ptr_type,
-			nprocs,
-			client_file_view_state_arr,
-			TEMP_OFF);
-	ADIOI_init_view_state(file_ptr_type,
-			nprocs,
-			client_file_view_state_arr,
-			REAL_OFF);
-    }
-
-#ifdef DEBUG
-    if (fd->is_agg == 1)
-    {
-	ADIOI_Flatlist_node *fr_node_p = ADIOI_Flatlist;
-	for (i = 0; i < nprocs; i++)
-	{
-	    fprintf(stderr, "client_file_view_state_arr[%d]=(fp_ind=%Ld,"
-		    "disp=%Ld,byte_off=%Ld,sz=%Ld,ext=%Ld\n", i,
-		    client_file_view_state_arr[i].fp_ind,
-		    client_file_view_state_arr[i].disp,
-		    client_file_view_state_arr[i].byte_off,
-		    client_file_view_state_arr[i].sz,
-		    client_file_view_state_arr[i].ext);
-	}
-	
-	while (fr_node_p->type != 
-	       fd->file_realm_types[fd->my_cb_nodes_index])
-	    fr_node_p = fr_node_p->next;
-	assert(fr_node_p != NULL);
-	
-	fprintf(stderr, "my file realm (idx=%d,st_off=%Ld) ", 
-		fd->my_cb_nodes_index,
-		fd->file_realm_st_offs[fd->my_cb_nodes_index]);
-	ADIOI_Print_flatlist_node(fr_node_p);
-    }
-#endif
-    
-#ifdef DEBUG2
-    if (fd->is_agg == 1)
-    {
-	for (i = 0; i < nprocs; i++)
-	{
-	    fprintf(stderr, "client_file_view_state_arr[%d]: ", i);
-	    ADIOI_Print_flatlist_node(
-		client_file_view_state_arr[i].flat_type_p);
-	}
-    }
-#endif
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5015, 0, NULL);
-#endif
-}
diff --git a/3rd-party/romio321/adio/common/ad_darray.c b/3rd-party/romio321/adio/common/ad_darray.c
deleted file mode 100644
index 3c9ca36a8e1..00000000000
--- a/3rd-party/romio321/adio/common/ad_darray.c
+++ /dev/null
@@ -1,291 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-
-static int MPIOI_Type_block(int *array_of_gsizes, int dim, int ndims, int nprocs,
-		     int rank, int darg, int order, MPI_Aint orig_extent,
-		     MPI_Datatype type_old, MPI_Datatype *type_new,
-		     MPI_Aint *st_offset);
-static int MPIOI_Type_cyclic(int *array_of_gsizes, int dim, int ndims, int nprocs,
-		      int rank, int darg, int order, MPI_Aint orig_extent,
-		      MPI_Datatype type_old, MPI_Datatype *type_new,
-		      MPI_Aint *st_offset);
-
-
-int ADIO_Type_create_darray(int size, int rank, int ndims, 
-			    int *array_of_gsizes, int *array_of_distribs, 
-			    int *array_of_dargs, int *array_of_psizes, 
-			    int order, MPI_Datatype oldtype, 
-			    MPI_Datatype *newtype) 
-{
-    MPI_Datatype type_old, type_new=MPI_DATATYPE_NULL, inttype;
-    int procs, tmp_rank, i, tmp_size, blklen, *coords;
-    MPI_Aint *st_offsets, orig_extent, disp, ub, lb;
-
-    MPI_Type_get_extent(oldtype, &lb, &orig_extent);
-
-/* calculate position in Cartesian grid as MPI would (row-major
-   ordering) */
-    coords = (int *) ADIOI_Malloc(ndims*sizeof(int));
-    procs = size;
-    tmp_rank = rank;
-    for (i=0; i<ndims; i++) {
-	procs = procs/array_of_psizes[i];
-	coords[i] = tmp_rank/procs;
-	tmp_rank = tmp_rank % procs;
-    }
-
-    st_offsets = (MPI_Aint *) ADIOI_Malloc(ndims*sizeof(MPI_Aint));
-    type_old = oldtype;
-
-    if (order == MPI_ORDER_FORTRAN) {
-      /* dimension 0 changes fastest */
-	for (i=0; i<ndims; i++) {
-	    switch(array_of_distribs[i]) {
-	    case MPI_DISTRIBUTE_BLOCK:
-		MPIOI_Type_block(array_of_gsizes, i, ndims,
-				 array_of_psizes[i],
-				 coords[i], array_of_dargs[i],
-				 order, orig_extent, 
-				 type_old, &type_new,
-				 st_offsets+i); 
-		break;
-	    case MPI_DISTRIBUTE_CYCLIC:
-		MPIOI_Type_cyclic(array_of_gsizes, i, ndims, 
-				  array_of_psizes[i], coords[i],
-				  array_of_dargs[i], order,
-				  orig_extent, type_old,
-				  &type_new, st_offsets+i);
-		break;
-	    case MPI_DISTRIBUTE_NONE:
-		/* treat it as a block distribution on 1 process */
-		MPIOI_Type_block(array_of_gsizes, i, ndims, 1, 0, 
-				 MPI_DISTRIBUTE_DFLT_DARG, order,
-				 orig_extent, 
-				 type_old, &type_new,
-				 st_offsets+i); 
-		break;
-	    }
-	    if (i) MPI_Type_free(&type_old);
-	    type_old = type_new;
-	}
-
-	/* add displacement and UB */
-	disp = st_offsets[0];
-	tmp_size = 1;
-	for (i=1; i<ndims; i++) {
-	    tmp_size *= array_of_gsizes[i-1];
-	    disp += (MPI_Aint)tmp_size*st_offsets[i];
-	}
-        /* rest done below for both Fortran and C order */
-    }
-
-    else /* order == MPI_ORDER_C */ {
-        /* dimension ndims-1 changes fastest */
-	for (i=ndims-1; i>=0; i--) {
-	    switch(array_of_distribs[i]) {
-	    case MPI_DISTRIBUTE_BLOCK:
-		MPIOI_Type_block(array_of_gsizes, i, ndims, array_of_psizes[i],
-				 coords[i], array_of_dargs[i], order,
-				 orig_extent, type_old, &type_new,
-				 st_offsets+i); 
-		break;
-	    case MPI_DISTRIBUTE_CYCLIC:
-		MPIOI_Type_cyclic(array_of_gsizes, i, ndims, 
-				  array_of_psizes[i], coords[i],
-				  array_of_dargs[i], order, 
-				  orig_extent, type_old, &type_new,
-				  st_offsets+i);
-		break;
-	    case MPI_DISTRIBUTE_NONE:
-		/* treat it as a block distribution on 1 process */
-		MPIOI_Type_block(array_of_gsizes, i, ndims, array_of_psizes[i],
-		      coords[i], MPI_DISTRIBUTE_DFLT_DARG, order, orig_extent, 
-                           type_old, &type_new, st_offsets+i); 
-		break;
-	    }
-	    if (i != ndims-1) MPI_Type_free(&type_old);
-	    type_old = type_new;
-	}
-
-	/* add displacement and UB */
-	disp = st_offsets[ndims-1];
-	tmp_size = 1;
-	for (i=ndims-2; i>=0; i--) {
-	    tmp_size *= array_of_gsizes[i+1];
-	    disp += (MPI_Aint)tmp_size*st_offsets[i];
-	}
-    }
-
-    disp *= orig_extent;
-
-    ub = orig_extent;
-    for (i=0; i<ndims; i++) ub *= (MPI_Aint)array_of_gsizes[i];
-	
-    blklen = 1;
-    
-    MPI_Type_create_struct(1, &blklen, &disp, &type_new, &inttype);
-    MPI_Type_create_resized (inttype, 0, ub, newtype);
-    MPI_Type_free (&inttype);
-
-    MPI_Type_free(&type_new);
-    ADIOI_Free(st_offsets);
-    ADIOI_Free(coords);
-    return MPI_SUCCESS;
-}
-
-
-/* Returns MPI_SUCCESS on success, an MPI error code on failure.  Code above
- * needs to call MPIO_Err_return_xxx.
- */
-static int MPIOI_Type_block(int *array_of_gsizes, int dim, int ndims, int nprocs,
-		     int rank, int darg, int order, MPI_Aint orig_extent,
-		     MPI_Datatype type_old, MPI_Datatype *type_new,
-		     MPI_Aint *st_offset) 
-{
-/* nprocs = no. of processes in dimension dim of grid
-   rank = coordinate of this process in dimension dim */
-    int blksize, global_size, mysize, i, j;
-    MPI_Aint stride;
-    
-    global_size = array_of_gsizes[dim];
-
-    if (darg == MPI_DISTRIBUTE_DFLT_DARG)
-	blksize = (global_size + nprocs - 1)/nprocs;
-    else {
-	blksize = darg;
-
-	/* --BEGIN ERROR HANDLING-- */
-	if (blksize <= 0) {
-	    return MPI_ERR_ARG;
-	}
-
-	if (blksize * nprocs < global_size) {
-	    return MPI_ERR_ARG;
-	}
-	/* --END ERROR HANDLING-- */
-    }
-
-    j = global_size - blksize*rank;
-    mysize = ADIOI_MIN(blksize, j);
-    if (mysize < 0) mysize = 0;
-
-    stride = orig_extent;
-    if (order == MPI_ORDER_FORTRAN) {
-	if (dim == 0) 
-	    MPI_Type_contiguous(mysize, type_old, type_new);
-	else {
-	    for (i=0; i<dim; i++) stride *= (MPI_Aint)array_of_gsizes[i];
-	    MPI_Type_create_hvector(mysize, 1, stride, type_old, type_new);
-	}
-    }
-    else {
-	if (dim == ndims-1) 
-	    MPI_Type_contiguous(mysize, type_old, type_new);
-	else {
-	    for (i=ndims-1; i>dim; i--) stride *= (MPI_Aint)array_of_gsizes[i];
-	    MPI_Type_create_hvector(mysize, 1, stride, type_old, type_new);
-	}
-
-    }
-
-    *st_offset = (MPI_Aint)blksize * (MPI_Aint)rank;
-     /* in terms of no. of elements of type oldtype in this dimension */
-    if (mysize == 0) *st_offset = 0;
-
-    return MPI_SUCCESS;
-}
-
-
-/* Returns MPI_SUCCESS on success, an MPI error code on failure.  Code above
- * needs to call MPIO_Err_return_xxx.
- */
-static int MPIOI_Type_cyclic(int *array_of_gsizes, int dim, int ndims, int nprocs,
-		      int rank, int darg, int order, MPI_Aint orig_extent,
-		      MPI_Datatype type_old, MPI_Datatype *type_new,
-		      MPI_Aint *st_offset) 
-{
-/* nprocs = no. of processes in dimension dim of grid
-   rank = coordinate of this process in dimension dim */
-    int blksize, i, blklens[3], st_index, end_index, local_size, rem, count;
-    MPI_Aint stride, disps[3];
-    MPI_Datatype type_tmp, type_tmp1, types[3];
-
-    if (darg == MPI_DISTRIBUTE_DFLT_DARG) blksize = 1;
-    else blksize = darg;
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (blksize <= 0) {
-	return MPI_ERR_ARG;
-    }
-    /* --END ERROR HANDLING-- */
-    
-    st_index = rank*blksize;
-    end_index = array_of_gsizes[dim] - 1;
-
-    if (end_index < st_index) local_size = 0;
-    else {
-	local_size = ((end_index - st_index + 1)/(nprocs*blksize))*blksize;
-	rem = (end_index - st_index + 1) % (nprocs*blksize);
-	local_size += ADIOI_MIN(rem, blksize);
-    }
-
-    count = local_size/blksize;
-    rem = local_size % blksize;
-    
-    stride = (MPI_Aint)nprocs*(MPI_Aint)blksize*orig_extent;
-    if (order == MPI_ORDER_FORTRAN)
-	for (i=0; i<dim; i++) stride *= (MPI_Aint)array_of_gsizes[i];
-    else for (i=ndims-1; i>dim; i--) stride *= (MPI_Aint)array_of_gsizes[i];
-
-    MPI_Type_create_hvector(count, blksize, stride, type_old, type_new);
-
-    if (rem) {
-	/* if the last block is of size less than blksize, include
-	   it separately using MPI_Type_struct */
-
-	types[0] = *type_new;
-	types[1] = type_old;
-	disps[0] = 0;
-	disps[1] = (MPI_Aint)count*stride;
-	blklens[0] = 1;
-	blklens[1] = rem;
-
-	MPI_Type_create_struct(2, blklens, disps, types, &type_tmp);
-
-	MPI_Type_free(type_new);
-	*type_new = type_tmp;
-    }
-
-    /* In the first iteration, we need to set the displacement in that
-       dimension correctly. */ 
-    if ( ((order == MPI_ORDER_FORTRAN) && (dim == 0)) ||
-         ((order == MPI_ORDER_C) && (dim == ndims-1)) ) {
-        types[0] = *type_new;
-        disps[0] = (MPI_Aint)rank * (MPI_Aint)blksize * orig_extent;
-        blklens[0] = 1;
-        MPI_Type_create_struct(1, blklens, disps, types, &type_tmp1);
-        MPI_Type_create_resized (type_tmp1, 0, orig_extent * (MPI_Aint)array_of_gsizes[dim], &type_tmp);
-        MPI_Type_free(&type_tmp1);
-        MPI_Type_free(type_new);
-        *type_new = type_tmp;
-
-        *st_offset = 0;  /* set it to 0 because it is taken care of in
-                            the struct above */
-    }
-    else {
-        *st_offset = (MPI_Aint)rank * (MPI_Aint)blksize; 
-        /* st_offset is in terms of no. of elements of type oldtype in
-         * this dimension */ 
-    }
-
-    if (local_size == 0) *st_offset = 0;
-
-    return MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/common/ad_delete.c b/3rd-party/romio321/adio/common/ad_delete.c
deleted file mode 100644
index ae78d7e0a7e..00000000000
--- a/3rd-party/romio321/adio/common/ad_delete.c
+++ /dev/null
@@ -1,24 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-
-void ADIOI_GEN_Delete(const char *filename, int *error_code)
-{
-    int err;
-    static char myname[] = "ADIOI_GEN_DELETE";
-
-    err = unlink(filename);
-    if (err == -1) {
-	*error_code = ADIOI_Err_create_code(myname, filename, errno);
-    }
-    else *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/common/ad_done.c b/3rd-party/romio321/adio/common/ad_done.c
deleted file mode 100644
index 50764b7bb59..00000000000
--- a/3rd-party/romio321/adio/common/ad_done.c
+++ /dev/null
@@ -1,52 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2004 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_SIGNAL_H
-#include <signal.h>
-#endif
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-
-#ifdef HAVE_AIO_LITE_H
-#include <aio-lite.h>
-#else
- #ifdef  HAVE_AIO_H
- #include <aio.h>
- #endif
- #ifdef HAVE_SYS_AIO_H
- #include <sys/aio.h>
- #endif
-#endif
-
-/* Workaround for incomplete set of definitions if __REDIRECT is not 
-   defined and large file support is used in aio.h */
-#if !defined(__REDIRECT) && defined(__USE_FILE_OFFSET64)
-#define aiocb aiocb64
-#endif
-
-/* ADIOI_GEN_IODone
- *
- * This code handles two distinct cases.  If ROMIO_HAVE_WORKING_AIO is not
- * defined, then I/O was done as a blocking call earlier.  In that case
- * we have nothing much to do other than set the bytes transferred and
- * free the request.
- *
- * If ROMIO_HAVE_WORKING_AIO is defined, then we may need to wait for I/O
- * to complete.
- */
-int ADIOI_GEN_IODone(ADIO_Request *request, ADIO_Status *status,
-		     int *error_code)  
-{
-	return 0;
-
-}
diff --git a/3rd-party/romio321/adio/common/ad_done_fake.c b/3rd-party/romio321/adio/common/ad_done_fake.c
deleted file mode 100644
index 59da5755e0b..00000000000
--- a/3rd-party/romio321/adio/common/ad_done_fake.c
+++ /dev/null
@@ -1,20 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2004 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-
-/* Generic implementation of ReadDone/WriteDone simply sets the
- * bytes field in the status structure and frees the request.
- *
- * Same function is used for both reads and writes.
- */
-int ADIOI_FAKE_IODone(ADIO_Request *request, ADIO_Status *status,
-		     int *error_code)
-{
-	/* should not ever get called now */
-	return 1;
-}
diff --git a/3rd-party/romio321/adio/common/ad_end.c b/3rd-party/romio321/adio/common/ad_end.c
deleted file mode 100644
index a4d186f9272..00000000000
--- a/3rd-party/romio321/adio/common/ad_end.c
+++ /dev/null
@@ -1,85 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-
-void ADIO_End(int *error_code)
-{
-    ADIOI_Flatlist_node *curr, *next;
-    ADIOI_Datarep *datarep, *datarep_next;
-    
-/*    FPRINTF(stderr, "reached end\n"); */
-
-    /* if a default errhandler was set on MPI_FILE_NULL then we need to ensure
-     * that our reference to that errhandler is released */
-/* Open MPI: The call to PMPI_File_set_errhandler has to be done in romio/src/io_romio_file_open.c
-   in routine mca_io_romio_file_close()
-*/
-#if 0
-    PMPI_File_set_errhandler(MPI_FILE_NULL, MPI_ERRORS_RETURN);
-#endif
-
-/* delete the flattened datatype list */
-    curr = ADIOI_Flatlist;
-    while (curr) {
-	if (curr->blocklens) ADIOI_Free(curr->blocklens);
-	if (curr->indices) ADIOI_Free(curr->indices);
-	next = curr->next;
-	ADIOI_Free(curr);
-	curr = next;
-    }
-    ADIOI_Flatlist = NULL;
-
-/* free file and info tables used for Fortran interface */
-    if (ADIOI_Ftable) ADIOI_Free(ADIOI_Ftable);
-#ifndef HAVE_MPI_INFO
-    if (MPIR_Infotable) ADIOI_Free(MPIR_Infotable);
-#endif
-
-
-/* free the memory allocated for a new data representation, if any */
-    datarep = ADIOI_Datarep_head;
-    while (datarep) {
-        datarep_next = datarep->next;
-        ADIOI_Free(datarep->name);
-        ADIOI_Free(datarep);
-        datarep = datarep_next;
-    }
-
-    if( ADIOI_syshints != MPI_INFO_NULL)
-	    MPI_Info_free(&ADIOI_syshints);
-
-    MPI_Op_free(&ADIO_same_amode);
-
-    *error_code = MPI_SUCCESS;
-}
-
-
-
-/* This is the delete callback function associated with
-   ADIO_Init_keyval when MPI_COMM_SELF is freed */
-
-int ADIOI_End_call(MPI_Comm comm, int keyval, void *attribute_val, void
-		  *extra_state)
-{
-    int error_code;
-
-    ADIOI_UNREFERENCED_ARG(comm);
-    ADIOI_UNREFERENCED_ARG(attribute_val);
-    ADIOI_UNREFERENCED_ARG(extra_state);
-
-    MPI_Comm_free_keyval (&keyval);
-
-    /* The end call will be called after all possible uses of this keyval, even
-     * if a file was opened with MPI_COMM_SELF.  Note, this assumes LIFO
-     * MPI_COMM_SELF attribute destruction behavior mandated by MPI-2.2. */
-    if (ADIOI_cb_config_list_keyval != MPI_KEYVAL_INVALID)
-        MPI_Comm_free_keyval (&ADIOI_cb_config_list_keyval);
-
-    ADIO_End(&error_code);
-    return error_code;
-}
diff --git a/3rd-party/romio321/adio/common/ad_fcntl.c b/3rd-party/romio321/adio/common/ad_fcntl.c
deleted file mode 100644
index 62067573dd1..00000000000
--- a/3rd-party/romio321/adio/common/ad_fcntl.c
+++ /dev/null
@@ -1,64 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2005 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-
-void ADIOI_GEN_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct,
-		     int *error_code)
-{
-    static char myname[] = "ADIOI_GEN_FCNTL";
-
-    switch(flag) {
-    case ADIO_FCNTL_GET_FSIZE:
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-	fcntl_struct->fsize = lseek(fd->fd_sys, 0, SEEK_END);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-	if (fd->fp_sys_posn != -1) {
-#ifdef ADIOI_MPE_LOGGING
-             MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-	     lseek(fd->fd_sys, fd->fp_sys_posn, SEEK_SET);
-#ifdef ADIOI_MPE_LOGGING
-             MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-        }
-	if (fcntl_struct->fsize == -1) {
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE, myname,
-					       __LINE__, MPI_ERR_IO, "**io",
-					       "**io %s", strerror(errno));
-	}
-	else *error_code = MPI_SUCCESS;
-	break;
-
-    case ADIO_FCNTL_SET_DISKSPACE:
-	ADIOI_GEN_Prealloc(fd, fcntl_struct->diskspace, error_code);
-	break;
-
-    case ADIO_FCNTL_SET_ATOMICITY:
-	fd->atomicity = (fcntl_struct->atomicity == 0) ? 0 : 1;
-	*error_code = MPI_SUCCESS;
-	break;
-
-	/* --BEGIN ERROR HANDLING-- */
-    default:
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					   MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   MPI_ERR_ARG,
-					   "**flag", "**flag %d", flag);
-	/* --END ERROR HANDLING-- */
-    }
-}
diff --git a/3rd-party/romio321/adio/common/ad_features.c b/3rd-party/romio321/adio/common/ad_features.c
deleted file mode 100644
index 480a42c6fe4..00000000000
--- a/3rd-party/romio321/adio/common/ad_features.c
+++ /dev/null
@@ -1,26 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *
- *  (C) 2008 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include "adio.h"
-
-int ADIOI_GEN_Feature(ADIO_File fd, int flag)
-{
-	switch(flag) {
-		case ADIO_LOCKS:
-		case ADIO_SHARED_FP:
-		case ADIO_ATOMIC_MODE:
-		case ADIO_DATA_SIEVING_WRITES:
-		case ADIO_UNLINK_AFTER_CLOSE:
-		case ADIO_TWO_PHASE:
-		case ADIO_SCALABLE_RESIZE:
-			return 1;
-			break;
-		case ADIO_SCALABLE_OPEN:
-		default:
-			return 0;
-			break;
-	}
-}
diff --git a/3rd-party/romio321/adio/common/ad_flush.c b/3rd-party/romio321/adio/common/ad_flush.c
deleted file mode 100644
index 3ace6778bf4..00000000000
--- a/3rd-party/romio321/adio/common/ad_flush.c
+++ /dev/null
@@ -1,35 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-
-void ADIOI_GEN_Flush(ADIO_File fd, int *error_code)
-{
-    int err;
-    static char myname[] = "ADIOI_GEN_FLUSH";
-
-    /* the deferred-open optimization may mean that a file has not been opened
-     * on this processor */
-    if (fd->is_open > 0) {
-	err = fsync(fd->fd_sys);
-	/* --BEGIN ERROR HANDLING-- */
-	if (err == -1) {
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-		    myname, __LINE__, MPI_ERR_IO,
-		    "**io",
-		    "**io %s", strerror(errno));
-	    return;
-	}
-	/* --END ERROR HANDLING-- */
-    }
-
-    *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/common/ad_fstype.c b/3rd-party/romio321/adio/common/ad_fstype.c
deleted file mode 100644
index 75fb24fee66..00000000000
--- a/3rd-party/romio321/adio/common/ad_fstype.c
+++ /dev/null
@@ -1,872 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-/* This file is quickly becoming the single one, outside the ADIO
- * implementations, which has "what ADIO components are built in" code in it.
- */
-
-#include "adio.h"
-
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-
-#ifdef HAVE_SYS_PARAM_H
-#include <sys/param.h>
-#endif
-
-#ifdef HAVE_PVFS_H
-#include "pvfs.h"
-#endif
-
-#ifdef HAVE_PVFS2_H
-#include "pvfs2.h"
-#endif
-
-#ifdef HAVE_ZOIDFS_H
-#include "zoidfs.h"
-#endif
-
-#ifdef HAVE_GPFS_H
-#include "gpfs.h"
-#endif
-
-/* Notes on detection process:
- *
- * There are three more "general" mechanisms that we use for detecting
- * file system type:
- * - struct statfs's f_type field
- * - struct statvfs's f_basetype field
- * - struct stat's st_fstype field
- *
- * Otherwise we'll fall back on some OS-specific approach.
- */
-
-#ifdef HAVE_STRUCT_STATFS
-# ifdef HAVE_SYS_VFS_H
-# include <sys/vfs.h>
-# endif
-# ifdef HAVE_SYS_STATVFS_H
-# include <sys/statvfs.h>
-# endif
-# ifdef HAVE_SYS_PARAM_H
-# include <sys/param.h>
-# endif
-# ifdef HAVE_SYS_MOUNT_H
-# include <sys/mount.h>
-# endif
- /* On Linux platforms, linux/nfs_fs.h is all messed up and cannot be
-  * reliably included.
-  */
-# if defined(ROMIO_NFS) && !defined(NFS_SUPER_MAGIC)
-# define NFS_SUPER_MAGIC 0x6969
-# endif
-
-# if defined(ROMIO_PANFS) && !defined(PAN_KERNEL_FS_CLIENT_SUPER_MAGIC)
-# define PAN_KERNEL_FS_CLIENT_SUPER_MAGIC 0xAAD7AAEA
-# endif
-#endif
-
-# if defined(ROMIO_XFS) && !defined(XFS_SUPER_MAGIC)
-# define XFS_SUPER_MAGIC 0x58465342
-# endif
-
-#if !defined(PVFS2_SUPER_MAGIC)
-#define PVFS2_SUPER_MAGIC (0x20030528)
-#endif
-
-#if defined(ROMIO_GPFS) && !defined(GPFS_SUPER_MAGIC)
-# define GPFS_SUPER_MAGIC 0x47504653
-#endif
-
-#ifdef ROMIO_HAVE_STRUCT_STATVFS_WITH_F_BASETYPE
-# ifdef HAVE_SYS_STATVFS_H
-# include <sys/statvfs.h>
-# endif
-# ifdef HAVE_SYS_VFS_H
-# include <sys/vfs.h>
-# endif
-# ifdef HAVE_SYS_PARAM_H
-# include <sys/param.h>
-# endif
-# ifdef HAVE_SYS_MOUNT_H
-# include <sys/mount.h>
-# endif
-#endif
-
-#ifdef ROMIO_HAVE_STRUCT_STAT_WITH_ST_FSTYPE
-# ifdef HAVE_SYS_TYPES_H
-# include <sys/types.h>
-# endif
-# ifdef HAVE_SYS_STAT_H
-# include <sys/stat.h>
-# endif
-#endif
-
-/* ADIO_FileSysType_parentdir is only used if one of these is defined.
-   By including this test, we avoid warnings about unused static functions
-   from the compiler */
-#if defined(ROMIO_HAVE_STRUCT_STATVFS_WITH_F_BASETYPE) || \
-    defined(HAVE_STRUCT_STATFS) || \
-    defined(ROMIO_HAVE_STRUCT_STAT_WITH_ST_FSTYPE) 
-#ifndef ROMIO_NTFS
-#define ROMIO_NEEDS_ADIOPARENTDIR
-static void ADIO_FileSysType_parentdir(const char *filename, char **dirnamep);
-#endif
-#endif 
-static void ADIO_FileSysType_prefix(const char *filename, int *fstype,
-				    int *error_code);
-static void ADIO_FileSysType_fncall(const char *filename, int *fstype,
-				    int *error_code);
-
-/*
- ADIO_FileSysType_parentdir - determines a string pathname for the
- parent directory of a given filename.
-
-Input Parameters:
-. filename - pointer to file name character array
-
-Output Parameters:
-. dirnamep - pointer to location in which to store a pointer to a string
-
- Note that the caller should free the memory located at the pointer returned
- after the string is no longer needed.
-*/
-#ifdef ROMIO_NEEDS_ADIOPARENTDIR
-
-/* In a strict ANSI environment, S_ISLNK may not be defined.  Fix that
-   here.  We assume that S_ISLNK is *always* defined as a macro.  If
-   that is not universally true, then add a test to the romio
-   configure that trys to link a program that references S_ISLNK */
-#if !defined(S_ISLNK) 
-#    if defined(S_IFLNK)
-     /* Check for the link bit */
-#    define S_ISLNK(mode) ((mode) & S_IFLNK)
-#    else
-     /* no way to check if it is a link, so say false */
-#    define S_ISLNK(mode) 0   
-#    endif
-#endif /* !(S_ISLNK) */
-
-/* ADIO_FileSysType_parentdir
- *
- * Returns pointer to string in dirnamep; that string is allocated with
- * strdup and must be free()'d.
- */
-static void ADIO_FileSysType_parentdir(const char *filename, char **dirnamep)
-{
-    int err;
-    char *dir = NULL, *slash;
-    struct stat statbuf;
-    
-    err = lstat(filename, &statbuf);
-
-    if (err || (!S_ISLNK(statbuf.st_mode))) {
-	/* no such file, or file is not a link; these are the "normal"
-	 * cases where we can just return the parent directory.
-	 */
-	dir = ADIOI_Strdup(filename);
-    }
-    else {
-	/* filename is a symlink.  we've presumably already tried
-	 * to stat it and found it to be missing (dangling link),
-	 * but this code doesn't care if the target is really there
-	 * or not.
-	 */
-	ssize_t namelen;
-	char *linkbuf;
-
-	linkbuf = ADIOI_Malloc(PATH_MAX+1);
-	namelen = readlink(filename, linkbuf, PATH_MAX+1);
-	if (namelen == -1) {
-	    /* something strange has happened between the time that
-	     * we determined that this was a link and the time that
-	     * we attempted to read it; punt and use the old name.
-	     */
-	    dir = ADIOI_Strdup(filename);
-	}
-	else {
-	    /* successfully read the link */
-	    linkbuf[namelen] = '\0'; /* readlink doesn't null terminate */
-	    dir = ADIOI_Strdup(linkbuf);
-	}
-	ADIOI_Free(linkbuf);
-    }
-
-    slash = strrchr(dir, '/');
-    if (!slash) ADIOI_Strncpy(dir, ".", 2);
-    else {
-	if (slash == dir) *(dir + 1) = '\0';
-	else *slash = '\0';
-    }
-
-    *dirnamep = dir;
-    return;
-}
-#endif /* ROMIO_NTFS */
-
-/*
- ADIO_FileSysType_fncall - determines the file system type for a given file
- using a system-dependent function call
-
-Input Parameters:
-. filename - pointer to file name character array
-
-Output Parameters:
-. fstype - location in which to store file system type (ADIO_XXX)
-. error_code - location in which to store error code
-
- MPI_SUCCESS is stored in the location pointed to by error_code on success.
-
- This function is used by MPI_File_open() and MPI_File_delete() to determine 
- file system type.  Most other functions use the type which is stored when the 
- file is opened.
- */
-static void ADIO_FileSysType_fncall(const char *filename, int *fstype, int *error_code)
-{
-#if defined (ROMIO_HAVE_STRUCT_STATVFS_WITH_F_BASETYPE) || defined (HAVE_STRUCT_STATFS) || defined (ROMIO_HAVE_STRUCT_STAT_WITH_ST_FSTYPE)
-    int err;
-#endif
-
-#ifdef ROMIO_HAVE_STRUCT_STATVFS_WITH_F_BASETYPE
-    struct statvfs vfsbuf;
-#endif
-#ifdef HAVE_STRUCT_STATFS
-    struct statfs fsbuf;
-#endif
-#ifdef ROMIO_HAVE_STRUCT_STAT_WITH_ST_FSTYPE
-    struct stat sbuf;
-#endif
-    static char myname[] = "ADIO_RESOLVEFILETYPE_FNCALL";
-
-/* NFS can get stuck and end up returing ESTALE "forever" */
-#define MAX_ESTALE_RETRY 10000
-    int retry_cnt;
-
-    *error_code = MPI_SUCCESS;
-
-#ifdef ROMIO_HAVE_STRUCT_STATVFS_WITH_F_BASETYPE
-    /* rare: old solaris machines */
-    retry_cnt=0;
-    do {
-	err = statvfs(filename, &vfsbuf);
-    } while (err && (errno == ESTALE) && retry_cnt++ < MAX_ESTALE_RETRY);
-
-    if (err) {
-	/* ENOENT may be returned in two cases:
-	 * 1) no directory entry for "filename"
-	 * 2) "filename" is a dangling symbolic link
-	 *
-	 * ADIO_FileSysType_parentdir tries to deal with both cases.
-	 */
-	if (errno == ENOENT) {
-	    char *dir;
-	    ADIO_FileSysType_parentdir(filename, &dir);
-	    err = statvfs(dir, &vfsbuf);
-
-	    ADIOI_Free(dir);
-	}
-	else {
-	    *error_code = ADIOI_Err_create_code(myname, filename, errno);
-	    if(*error_code != MPI_SUCCESS) return;
-	}
-    }
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (err) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_NO_SUCH_FILE,
-					   "**filename", "**filename %s", filename);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-    /* FPRINTF(stderr, "%s\n", vfsbuf.f_basetype); */
-    if (!strncmp(vfsbuf.f_basetype, "nfs", 3)) {
-	*fstype = ADIO_NFS;
-	return;
-    }
-    if (!strncmp(vfsbuf.f_basetype, "xfs", 3)) {
-	*fstype = ADIO_XFS;
-	return;
-    }
-
-# ifdef ROMIO_UFS
-    /* if UFS support is enabled, default to that */
-    *fstype = ADIO_UFS;
-    return;
-# endif
-
-    /* --BEGIN ERROR HANDLING-- */
-    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-				       myname, __LINE__, MPI_ERR_NO_SUCH_FILE,
-				       "**filename", "**filename %s", filename);
-    /* --END ERROR HANDLING-- */
-#endif /* STATVFS APPROACH */
-
-#if defined(HAVE_STRUCT_STATFS) && defined(HAVE_STATFS)
-    /* common automagic fs-detection logic for any modern POSX-compliant
-     * environment */
-    retry_cnt = 0;
-    do {
-	err = statfs(filename, &fsbuf);
-    } while (err && (errno == ESTALE) && retry_cnt++ < MAX_ESTALE_RETRY);
-
-    if (err) {
-	if(errno == ENOENT) {
-	    char *dir;
-	    ADIO_FileSysType_parentdir(filename, &dir);
-	    err = statfs(dir, &fsbuf);
-	    ADIOI_Free(dir);
-	}
-	else {
-	    *error_code = ADIOI_Err_create_code(myname, filename, errno);
-	    if(*error_code != MPI_SUCCESS) return;
-	}
-    }
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (err) {
-    	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_NO_SUCH_FILE,
-					   "**filename", "**filename %s", filename);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-# ifdef ROMIO_HAVE_STRUCT_STATFS_WITH_F_FSTYPENAME
-    /* uncommon: maybe only on Darwin ? */
-    if ( !strncmp("nfs",fsbuf.f_fstypename,3) ) {
-	*fstype = ADIO_NFS;
-	return;
-    }
-# endif
-
-
-#ifdef ROMIO_GPFS
-    if (fsbuf.f_type == GPFS_SUPER_MAGIC) {
-	*fstype = ADIO_GPFS;
-	return;
-    }
-#endif
-
-    /* FPRINTF(stderr, "%d\n", fsbuf.f_type);*/
-# ifdef NFS_SUPER_MAGIC
-    if (fsbuf.f_type == NFS_SUPER_MAGIC) {
-	*fstype = ADIO_NFS;
-	return;
-    }
-# endif
-
-#ifdef ROMIO_LUSTRE
-# ifndef LL_SUPER_MAGIC
-#  define LL_SUPER_MAGIC 0x0BD00BD0
-# endif
-    if (fsbuf.f_type == LL_SUPER_MAGIC) {
-	*fstype = ADIO_LUSTRE;
-	return;
-    }
-#endif
-
-# ifdef PAN_KERNEL_FS_CLIENT_SUPER_MAGIC
-    if (fsbuf.f_type == PAN_KERNEL_FS_CLIENT_SUPER_MAGIC) {
-	*fstype = ADIO_PANFS;
-	return;
-    }
-# endif
-
-# ifdef MOUNT_NFS
-    if (fsbuf.f_type == MOUNT_NFS) {
-	*fstype = ADIO_NFS;
-	return;
-    }
-# endif
-
-# ifdef MOUNT_PFS
-    if (fsbuf.f_type == MOUNT_PFS) {
-	*fstype = ADIO_PFS;
-	return;
-    }
-# endif
-
-# ifdef PVFS_SUPER_MAGIC
-    if (fsbuf.f_type == PVFS_SUPER_MAGIC) {
-	*fstype = ADIO_PVFS;
-	return;
-    }
-# endif
-
-# ifdef PVFS2_SUPER_MAGIC
-    if (fsbuf.f_type == PVFS2_SUPER_MAGIC) {
-	*fstype = ADIO_PVFS2;
-	return;
-    }
-# endif
-
-# ifdef XFS_SUPER_MAGIC
-    if (fsbuf.f_type == XFS_SUPER_MAGIC) {
-	    *fstype = ADIO_XFS;
-	    return;
-    }
-# endif
-
-# ifdef ROMIO_UFS
-    /* if UFS support is enabled, default to that */
-    *fstype = ADIO_UFS;
-    return;
-# endif
-    /* --BEGIN ERROR HANDLING-- */
-    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-				       myname, __LINE__, MPI_ERR_NO_SUCH_FILE,
-				       "**filename", "**filename %s", filename);
-    /* --END ERROR HANDLING-- */
-#endif /* STATFS APPROACH */
-
-#ifdef ROMIO_HAVE_STRUCT_STAT_WITH_ST_FSTYPE
-    /* rare: maybe old NEC SX or SGI IRIX machines */
-    retry_cnt = 0;
-    do {
-	err = stat(filename, &sbuf);
-    } while (err && (errno == ESTALE) && retry_cnt++ < MAX_ESTALE_RETRY);
-
-    if (err) {
-	if(errno == ENOENT) {
-	    char *dir;
-	    ADIO_FileSysType_parentdir(filename, &dir);
-	    err = stat(dir, &sbuf);
-	    ADIOI_Free(dir);
-	}
-	else{
-	    *error_code = ADIOI_Err_create_code(myname, filename, errno);
-	    if(*error_code != MPI_SUCCESS) return;
-	}
-    }
-    
-    if (err) {
-    	/* --BEGIN ERROR HANDLING-- */
-    	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-				           myname, __LINE__, MPI_ERR_NO_SUCH_FILE,
-				           "**filename", "**filename %s", filename);
-    	/* --END ERROR HANDLING-- */
-	return;
-    }
-    else {
-	if (!strcmp(sbuf.st_fstype, "nfs")) *fstype = ADIO_NFS;
-	else *fstype = ADIO_SFS; /* assuming SX4 for now */
-    }
-#endif /* STAT APPROACH */
-
-#ifdef ROMIO_NTFS
-    ADIOI_UNREFERENCED_ARG(filename);
-    ADIOI_UNREFERENCED_ARG(error_code);
-    *fstype = ADIO_NTFS; /* only supported FS on Windows */
-#elif defined(ROMIO_NFS)
-    *fstype = ADIO_NFS;
-#elif defined(ROMIO_UFS)
-    *fstype = ADIO_UFS;
-#else
-    /* --BEGIN ERROR HANDLING-- */
-    *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-				       myname, __LINE__, MPI_ERR_NO_SUCH_FILE,
-				       "**filename", "**filename %s", filename);
-    /* --END ERROR HANDLING-- */
-#endif
-}
-
-/* all proceeses opening, creating, or deleting a file end up invoking several
- * stat system calls (unless a fs prefix is given).  Cary out this file system
- * detection in a more scalable way by having rank 0 stat the file and broadcast the result (fs type and error code) to the other mpi processes */
-
-static void ADIO_FileSysType_fncall_scalable(MPI_Comm comm, const char *filename, int * file_system, int * error_code)
-{
-    int rank;
-    int buf[2];
-    MPI_Comm_rank(comm, &rank);
-
-    if (rank == 0) {
-	ADIO_FileSysType_fncall(filename, file_system, error_code);
-	buf[0] = *file_system;
-	buf[1] = *error_code;
-    }
-    MPI_Bcast(buf, 2, MPI_INT, 0, comm);
-    *file_system = buf[0];
-    *error_code = buf[1];
-}
-
-
-
-/*
-  ADIO_FileSysType_prefix - determines file system type for a file using 
-  a prefix on the file name.  upper layer should have already determined
-  that a prefix is present.
-
-Input Parameters:
-. filename - path to file, including prefix (xxx:)
-
-Output Parameters:
-. fstype - pointer to integer in which to store file system type (ADIO_XXX)
-. error_code - pointer to integer in which to store error code
-
-  Returns MPI_SUCCESS in error_code on success.  Filename not having a prefix
-  is considered an error. Except for on Windows systems where the default is NTFS.
-
- */
-static void ADIO_FileSysType_prefix(const char *filename, int *fstype, int *error_code)
-{
-    static char myname[] = "ADIO_RESOLVEFILETYPE_PREFIX";
-    *error_code = MPI_SUCCESS;
-
-    if (!strncmp(filename, "pfs:", 4) || !strncmp(filename, "PFS:", 4)) {
-	*fstype = ADIO_PFS;
-    }
-    else if (!strncmp(filename, "piofs:", 6) || !strncmp(filename, "PIOFS:", 6)) {
-	*fstype = ADIO_PIOFS;
-    }
-    else if (!strncmp(filename, "ufs:", 4) || !strncmp(filename, "UFS:", 4)) {
-	*fstype = ADIO_UFS;
-    }
-    else if (!strncmp(filename, "nfs:", 4) || !strncmp(filename, "NFS:", 4)) {
-	*fstype = ADIO_NFS;
-    }
-    else if (!strncmp(filename, "panfs:", 6) || !strncmp(filename, "PANFS:", 6)) {
-	*fstype = ADIO_PANFS;
-    }
-    else if (!strncmp(filename, "hfs:", 4) || !strncmp(filename, "HFS:", 4)) {
-	*fstype = ADIO_HFS;
-    }
-    else if (!strncmp(filename, "xfs:", 4) || !strncmp(filename, "XFS:", 4)) {
-	*fstype = ADIO_XFS;
-    }
-    else if (!strncmp(filename, "sfs:", 4) || !strncmp(filename, "SFS:", 4)) {
-	*fstype = ADIO_SFS;
-    }
-    else if (!strncmp(filename, "pvfs:", 5) || !strncmp(filename, "PVFS:", 5)) {
-	*fstype = ADIO_PVFS;
-    }
-    else if (!strncmp(filename, "pvfs2:", 6)||!strncmp(filename, "PVFS2:", 6)) {
-	*fstype = ADIO_PVFS2;
-    }
-    else if (!strncmp(filename, "zoidfs:", 7)||
-		    !strncmp(filename, "ZOIDFS:", 7)) {
-	    *fstype = ADIO_ZOIDFS;
-    } 
-    else if (!strncmp(filename, "testfs:", 7) 
-	     || !strncmp(filename, "TESTFS:", 7))
-    {
-	*fstype = ADIO_TESTFS;
-    }
-    else if (!strncmp(filename, "ftp:", 4) 
-		    || !strncmp(filename, "gsiftp:", 7))
-    {
-	*fstype = ADIO_GRIDFTP;
-    }
-    else if (!strncmp(filename, "lustre:", 7) 
-	     || !strncmp(filename, "LUSTRE:", 7))
-    {
-	*fstype = ADIO_LUSTRE;
-    }
-    else if (!strncmp(filename, "gpfs:", 5) || !strncmp(filename, "GPFS:", 5)) {
-	*fstype = ADIO_GPFS;
-    }
-    else {
-#ifdef ROMIO_NTFS
-	*fstype = ADIO_NTFS;
-#else
-	*fstype = 0;
-        /* --BEGIN ERROR HANDLING-- */
-        *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-				           myname, __LINE__, MPI_ERR_NO_SUCH_FILE,
-				           "**filename", "**filename %s", filename);
-        /* --END ERROR HANDLING-- */
-#endif
-    }
-}
-
-/*@
-    ADIO_ResolveFileType - determines file system type and operations from
-                           file name string; this is a collective call
-
-Input Parameters:
-. comm - communicator across which collective open is performed
-. filename - name of file (string)
-
-Output Parameters:
-. fstype - (pointer to) int holding file system type
-. ops - (address of) pointer to table of valid file operations
-. error_code - (pointer to) int holding error code
-
-Notes:
-This code used to be in MPI_File_open(), but it has been moved into here in 
-order to clean things up.  The goal is to separate all this "did we compile
-for this fs type" code from the MPI layer and also to introduce the ADIOI_Fns
-tables in a reasonable way. -- Rob, 06/06/2001
-@*/
-void ADIO_ResolveFileType(MPI_Comm comm, const char *filename, int *fstype,
-			  ADIOI_Fns **ops, int *error_code)
-{
-    int myerrcode, file_system, min_code, max_code;
-    char *tmp;
-    static char myname[] = "ADIO_RESOLVEFILETYPE";
-    char * p;
-
-    file_system = -1;
-    if (filename == NULL) {
-	    *error_code = ADIOI_Err_create_code(myname, filename, ENOENT);
-	    return;
-    }
-    tmp = strchr(filename, ':');
-    if (!tmp) {
-	int have_nfs_enabled=0;
-	*error_code = MPI_SUCCESS;
-	/* no prefix; use system-dependent function call to determine type */
-	/* Optimization: we can reduce the 'storm of stats' that result from
-	 * thousands of mpi processes determinig file type this way.  Let us
-	 * have just one process stat the file and broadcast the result to
-	 * everyone else.  
-	 * - Note that we will not catch cases like
-	 * http://www.mcs.anl.gov/web-mail-archive/lists/mpich-discuss/2007/08/msg00042.html
-	 * (edit: now http://lists.mcs.anl.gov/pipermail/mpich-discuss/2007-August/002648.html)
-	 *
-	 * where file systems are not mounted or available on other processes,
-	 * but we'll catch those a few functions later in ADIO_Open 
-	 * - Note that if we have NFS enabled, we might have a situation where,
-	 *   for example, /home/user/data.out is UFS on one process but NFS on
-	 *   others, so we won't perform this optimization if NFS is enabled.
-	 * - Another point: error codes and file system types are broadcast to
-	 *   all members of the communicator, so we get to skip the allreduce
-	 *   steps*/
-
-#ifdef ROMIO_NFS
-	have_nfs_enabled=1;
-#endif
-	if (!have_nfs_enabled) {
-	    ADIO_FileSysType_fncall_scalable(comm, filename, &file_system, &myerrcode);
-	    if (myerrcode != MPI_SUCCESS) {
-		*error_code = myerrcode;
-		return;
-	    }
-	} else {
-	    ADIO_FileSysType_fncall(filename, &file_system, &myerrcode);
-
-	    /* the check for file system type will hang if any process got
-	     * an error in ADIO_FileSysType_fncall.  Processes encountering
-	     * an error will return early, before the collective file
-	     * system type check below.  This case could happen if a full
-	     * path exists on one node but not on others, and no prefix
-	     * like ufs: was provided.  see discussion at
-	     * http://www.mcs.anl.gov/web-mail-archive/lists/mpich-discuss/2007/08/msg00042.html
-	     * (edit: now
-	     * http://lists.mcs.anl.gov/pipermail/mpich-discuss/2007-August/002648.html)
-	     */
-
-	    MPI_Allreduce(&myerrcode, &max_code, 1, MPI_INT, MPI_MAX, comm);
-	    if (max_code != MPI_SUCCESS)  {
-		*error_code = max_code;
-		return;
-	    }
-	    /* ensure everyone came up with the same file system type */
-	    MPI_Allreduce(&file_system, &min_code, 1, MPI_INT,
-		    MPI_MIN, comm);
-	    if (min_code == ADIO_NFS) file_system = ADIO_NFS;
-	}
-    }
-    else {
-	/* prefix specified; just match via prefix and assume everyone got 
-	 * the same thing.
-	 *
-	 * perhaps we should have this code go through the allreduce as well?
-	 */
-	ADIO_FileSysType_prefix(filename, &file_system, &myerrcode);
-	if (myerrcode != MPI_SUCCESS) {
-	    *error_code = myerrcode;
-	    return;
-	}
-    }
-
-    /* lastly, there may be situations where one cannot override the file
-     * system detection with a prefix -- maybe the file name is passed to both
-     * posix and MPI-IO routines, or maybe the file name is hard-coded into an
-     * application.
-     * Assumes all processes set the same environment varialble.
-     * Values: the same prefix you would stick on a file path. e.g. pvfs2: --
-     * including the colon! */
-    p = getenv("ROMIO_FSTYPE_FORCE");
-    if (p != NULL) {
-	ADIO_FileSysType_prefix(p, &file_system, &myerrcode);
-	if (myerrcode != MPI_SUCCESS) {
-	    *error_code = myerrcode;
-	    return;
-	}
-    }
-
-    /* verify that we support this file system type and set ops pointer */
-    if (file_system == ADIO_PFS) {
-#ifndef ROMIO_PFS
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**iofstypeunsupported", 0);
-	return;
-#else
-	*ops = &ADIO_PFS_operations;
-#endif
-    }
-    if (file_system == ADIO_PIOFS) {
-#ifndef ROMIO_PIOFS
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**iofstypeunsupported", 0);
-	return;
-#else
-	*ops = &ADIO_PIOFS_operations;
-#endif
-    }
-    if (file_system == ADIO_UFS) {
-#ifndef ROMIO_UFS
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**iofstypeunsupported", 0);
-	return;
-#else
-	*ops = &ADIO_UFS_operations;
-#endif
-    }
-    if (file_system == ADIO_NFS) {
-#ifndef ROMIO_NFS
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**iofstypeunsupported", 0);
-	return;
-#else
-	*ops = &ADIO_NFS_operations;
-#endif
-    }
-    if (file_system == ADIO_PANFS) {
-#ifndef ROMIO_PANFS
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**iofstypeunsupported", 0);
-	return;
-#else
-	*ops = &ADIO_PANFS_operations;
-#endif
-    }
-    if (file_system == ADIO_HFS) {
-#ifndef ROMIO_HFS
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**iofstypeunsupported", 0);
-	return;
-#else
-	*ops = &ADIO_HFS_operations;
-#endif
-    }
-    if (file_system == ADIO_XFS) {
-#ifndef ROMIO_XFS
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**iofstypeunsupported", 0);
-	return;
-#else
-	*ops = &ADIO_XFS_operations;
-#endif
-    }
-    if (file_system == ADIO_SFS) {
-#ifndef ROMIO_SFS
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**iofstypeunsupported", 0);
-	return;
-#else
-	*ops = &ADIO_SFS_operations;
-#endif
-    }
-    if (file_system == ADIO_PVFS) {
-#ifndef ROMIO_PVFS
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**iofstypeunsupported", 0);
-	return;
-#else
-	*ops = &ADIO_PVFS_operations;
-#endif
-    }
-    if (file_system == ADIO_PVFS2) {
-#ifndef ROMIO_PVFS2
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**iofstypeunsupported", 0);
-	return;
-#else
-	*ops = &ADIO_PVFS2_operations;
-#endif
-    }
-    if (file_system == ADIO_NTFS) {
-#ifndef ROMIO_NTFS
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**iofstypeunsupported", 0);
-	return;
-#else
-	*ops = &ADIO_NTFS_operations;
-#endif
-    }
-    if (file_system == ADIO_TESTFS) {
-#ifndef ROMIO_TESTFS
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**iofstypeunsupported", 0);
-	return;
-#else
-	*ops = &ADIO_TESTFS_operations;
-#endif
-    }
-
-    if (file_system == ADIO_GPFS) {
-#ifndef ROMIO_GPFS
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					myname, __LINE__, MPI_ERR_IO,
-					"**iofstypeunsupported", 0);
-	return;
-#else
-	*ops = &ADIO_GPFS_operations;
-#endif
-    }
-
-    if (file_system == ADIO_GRIDFTP) {
-#ifndef ROMIO_GRIDFTP
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**iofstypeunsupported", 0);
-	return;
-#else
-	*ops = &ADIO_GRIDFTP_operations;
-#endif
-    }
-    if (file_system == ADIO_LUSTRE) {
-#ifndef ROMIO_LUSTRE 
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**iofstypeunsupported", 0);
-	return;
-#else
-	*ops = &ADIO_LUSTRE_operations;
-#endif
-    }
-    if (file_system == ADIO_ZOIDFS) {
-#ifndef ROMIO_ZOIDFS
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**iofstypeunsupported", 0);
-	return;
-#else
-	*ops = &ADIO_ZOIDFS_operations;
-#endif
-    }
-    *error_code = MPI_SUCCESS;
-    *fstype = file_system;
-    return;
-}
-/* 
- * vim: ts=8 sts=4 sw=4 noexpandtab 
- */
diff --git a/3rd-party/romio321/adio/common/ad_get_sh_fp.c b/3rd-party/romio321/adio/common/ad_get_sh_fp.c
deleted file mode 100644
index 12133277903..00000000000
--- a/3rd-party/romio321/adio/common/ad_get_sh_fp.c
+++ /dev/null
@@ -1,74 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-
-/* returns the current location of the shared_fp in terms of the
-   no. of etypes relative to the current view, and also increments the
-   shared_fp by the number of etypes to be accessed (incr) in the read
-   or write following this function. */
-
-void ADIOI_NFS_Get_shared_fp(ADIO_File fd, ADIO_Offset incr, ADIO_Offset *shared_fp,
-                             int *error_code);
-
-void ADIO_Get_shared_fp(ADIO_File fd, ADIO_Offset incr, ADIO_Offset *shared_fp,
-			 int *error_code)
-{
-    ADIO_Status status;
-    ADIO_Offset new_fp;
-    MPI_Comm dupcommself;
-
-    /* Set the shared_fp in case this comes from an uninitialized stack variable
-       The read routines will not read into the address of this variable if the file
-       size of a shared pointer is 0, and if incr is always zero, this value will remain
-       uninitialized.  Initialize it here to prevent incorrect values
-    */
-    *shared_fp = 0;
-
-#ifdef ROMIO_NFS
-    if (fd->file_system == ADIO_NFS) {
-	ADIOI_NFS_Get_shared_fp(fd, incr, shared_fp, error_code);
-	return;
-    }
-#endif
-
-    if (fd->shared_fp_fd == ADIO_FILE_NULL) {
-	MPI_Comm_dup(MPI_COMM_SELF, &dupcommself);
-	fd->shared_fp_fd = ADIO_Open(MPI_COMM_SELF, dupcommself, 
-				     fd->shared_fp_fname, 
-				     fd->file_system,
-				     fd->fns,
-				     ADIO_CREATE | ADIO_RDWR | ADIO_DELETE_ON_CLOSE, 
-				     0, MPI_BYTE, MPI_BYTE,
-				     MPI_INFO_NULL, 
-				     ADIO_PERM_NULL, error_code);
-	if (*error_code != MPI_SUCCESS) return;
-	ADIOI_WRITE_LOCK(fd->shared_fp_fd, 0, SEEK_SET, sizeof(ADIO_Offset));
-	ADIO_ReadContig(fd->shared_fp_fd, shared_fp, sizeof(ADIO_Offset), 
-		       MPI_BYTE, ADIO_EXPLICIT_OFFSET, 0, &status, error_code);
-        /* if the file is empty, the above function may return error
-           (reading beyond end of file). In that case, shared_fp = 0, 
-           set above, is the correct value. */
-    }
-    else {
-	ADIOI_WRITE_LOCK(fd->shared_fp_fd, 0, SEEK_SET, sizeof(ADIO_Offset));
-	ADIO_ReadContig(fd->shared_fp_fd, shared_fp, sizeof(ADIO_Offset), 
-		       MPI_BYTE, ADIO_EXPLICIT_OFFSET, 0, &status, error_code);
-	if (*error_code != MPI_SUCCESS) {
-	    ADIOI_UNLOCK(fd->shared_fp_fd, 0, SEEK_SET, sizeof(ADIO_Offset));
-	    return;
-	}
-    }
-
-    if (incr == 0) {goto done;}
-
-    new_fp = *shared_fp + incr;
-
-    ADIO_WriteContig(fd->shared_fp_fd, &new_fp, sizeof(ADIO_Offset), 
-		    MPI_BYTE, ADIO_EXPLICIT_OFFSET, 0, &status, error_code);
-done:
-    ADIOI_UNLOCK(fd->shared_fp_fd, 0, SEEK_SET, sizeof(ADIO_Offset));
-}
diff --git a/3rd-party/romio321/adio/common/ad_hints.c b/3rd-party/romio321/adio/common/ad_hints.c
deleted file mode 100644
index 6e89e76a3f5..00000000000
--- a/3rd-party/romio321/adio/common/ad_hints.c
+++ /dev/null
@@ -1,310 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-#include "hint_fns.h"
-
-void ADIOI_GEN_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code)
-{
-/* if fd->info is null, create a new info object. 
-   Initialize fd->info to default values.
-   Initialize fd->hints to default values.
-   Examine the info object passed by the user. If it contains values that
-   ROMIO understands, override the default. */
-
-    MPI_Info info;
-    char *value;
-    int flag, nprocs=0, len;
-    int ok_to_override_cb_nodes=0;
-    static char myname[] = "ADIOI_GEN_SETINFO";
-
-
-    /* if we've already set up default hints and the user has not asked us to
-     * process any hints (MPI_INFO_NULL), then we can short-circuit hint
-     * processing */
-    if (fd->hints->initialized && fd->info == MPI_INFO_NULL) {
-	    *error_code = MPI_SUCCESS;
-	    return;
-    }
-
-    if (fd->info == MPI_INFO_NULL) MPI_Info_create(&(fd->info));
-    info = fd->info;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-
-    /* Note that fd->hints is allocated at file open time; thus it is
-     * not necessary to allocate it, or check for allocation, here.
-     */
-
-    value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
-    if (value == NULL) {
-        *error_code = MPIO_Err_create_code(*error_code,
-                                           MPIR_ERR_RECOVERABLE,
-                                           myname,
-                                           __LINE__,
-                                           MPI_ERR_OTHER,
-                                           "**nomem2",0);
-        return;
-    }
-
-    /* initialize info and hints to default values if they haven't been
-     * previously initialized
-     */
-    if (!fd->hints->initialized) {
-
-	/* buffer size for collective I/O */
-	ADIOI_Info_set(info, "cb_buffer_size", ADIOI_CB_BUFFER_SIZE_DFLT); 
-	fd->hints->cb_buffer_size = atoi(ADIOI_CB_BUFFER_SIZE_DFLT);
-
-	/* default is to let romio automatically decide when to use
-	 * collective buffering
-	 */
-	ADIOI_Info_set(info, "romio_cb_read", "automatic"); 
-	fd->hints->cb_read = ADIOI_HINT_AUTO;
-	ADIOI_Info_set(info, "romio_cb_write", "automatic"); 
-	fd->hints->cb_write = ADIOI_HINT_AUTO;
-
-	fd->hints->cb_config_list = NULL;
-
-	/* number of processes that perform I/O in collective I/O */
-	ADIOI_Snprintf(value, MPI_MAX_INFO_VAL+1, "%d", nprocs);
-	ADIOI_Info_set(info, "cb_nodes", value);
-	fd->hints->cb_nodes = nprocs;
-
-	/* hint indicating that no indep. I/O will be performed on this file */
-	ADIOI_Info_set(info, "romio_no_indep_rw", "false");
-	fd->hints->no_indep_rw = 0;
-
-	/* hint instructing the use of persistent file realms */
-	ADIOI_Info_set(info, "romio_cb_pfr", "disable");
-	fd->hints->cb_pfr = ADIOI_HINT_DISABLE;
-	
-	/* hint guiding the assignment of persistent file realms */
-	ADIOI_Info_set(info, "romio_cb_fr_types", "aar");
-	fd->hints->cb_fr_type = ADIOI_FR_AAR;
-
-	/* hint to align file realms with a certain byte value */
-	ADIOI_Info_set(info, "romio_cb_fr_alignment", "1");
-	fd->hints->cb_fr_alignment = 1;
-
-	/* hint to set a threshold percentage for a datatype's size/extent at
-	 * which data sieving should be done in collective I/O */
-	ADIOI_Info_set(info, "romio_cb_ds_threshold", "0");
-	fd->hints->cb_ds_threshold = 0;
-
-	/* hint to switch between point-to-point or all-to-all for two-phase */
-	ADIOI_Info_set(info, "romio_cb_alltoall", "automatic");
-	fd->hints->cb_alltoall = ADIOI_HINT_AUTO;
-
-	 /* deferred_open derived from no_indep_rw and cb_{read,write} */
-	fd->hints->deferred_open = 0;
-
-	/* buffer size for data sieving in independent reads */
-	ADIOI_Info_set(info, "ind_rd_buffer_size", ADIOI_IND_RD_BUFFER_SIZE_DFLT);
-	fd->hints->ind_rd_buffer_size = atoi(ADIOI_IND_RD_BUFFER_SIZE_DFLT);
-
-	/* buffer size for data sieving in independent writes */
-	ADIOI_Info_set(info, "ind_wr_buffer_size", ADIOI_IND_WR_BUFFER_SIZE_DFLT);
-	fd->hints->ind_wr_buffer_size = atoi(ADIOI_IND_WR_BUFFER_SIZE_DFLT);
-
-	/* default is to let romio automatically decide when to use data
-	 * sieving
-	 */
-	ADIOI_Info_set(info, "romio_ds_read", "automatic"); 
-	fd->hints->ds_read = ADIOI_HINT_AUTO;
-	ADIOI_Info_set(info, "romio_ds_write", "automatic"); 
-	fd->hints->ds_write = ADIOI_HINT_AUTO;
-
-	/* still to do: tune this a bit for a variety of file systems. there's
-	 * no good default value so just leave it unset */
-	fd->hints->min_fdomain_size = 0;
-  fd->hints->striping_unit = 0;
-
-	fd->hints->initialized = 1;
-
-	/* ADIO_Open sets up collective buffering arrays.  If we are in this
-	 * path from say set_file_view, then we've don't want to adjust the
-	 * array: we'll get a segfault during collective i/o.  We only want to
-	 * look at the users cb_nodes if it's open time  */
-	ok_to_override_cb_nodes = 1;
-
-    }
-
-    /* add in user's info if supplied */
-    if (users_info != MPI_INFO_NULL) {
-	ADIOI_Info_check_and_install_int(fd, users_info, "cb_buffer_size", 
-		&(fd->hints->cb_buffer_size), myname, error_code);
-
-	/* aligning file realms to certain sizes (e.g. stripe sizes)
-	 * may benefit I/O performance */
-	ADIOI_Info_check_and_install_int(fd, users_info, "romio_cb_fr_alignment", 
-		&(fd->hints->cb_fr_alignment), myname, error_code);
-
-	/* for collective I/O, try to be smarter about when to do data sieving
-	 * using a specific threshold for the datatype size/extent
-	 * (percentage 0-100%) */
-	ADIOI_Info_check_and_install_int(fd, users_info, "romio_cb_ds_threshold", 
-		&(fd->hints->cb_ds_threshold), myname, error_code);
-
-	ADIOI_Info_check_and_install_enabled(fd, users_info, "romio_cb_alltoall",
-		&(fd->hints->cb_alltoall), myname, error_code);
-
-	/* new hints for enabling/disabling coll. buffering on
-	 * reads/writes
-	 */
-	ADIOI_Info_check_and_install_enabled(fd, users_info, "romio_cb_read",
-		&(fd->hints->cb_read), myname, error_code);
-	if (fd->hints->cb_read == ADIOI_HINT_DISABLE) {
-	    /* romio_cb_read overrides no_indep_rw */
-	    ADIOI_Info_set(info, "romio_no_indep_rw", "false");
-	    fd->hints->no_indep_rw = ADIOI_HINT_DISABLE;
-	}
-
-	ADIOI_Info_check_and_install_enabled(fd, users_info, "romio_cb_write",
-		&(fd->hints->cb_write), myname, error_code);
-	if (fd->hints->cb_write == ADIOI_HINT_DISABLE) {
-	    /* romio_cb_write overrides no_indep_rw */
-	    ADIOI_Info_set(info, "romio_no_indep_rw", "false");
-	    fd->hints->no_indep_rw = ADIOI_HINT_DISABLE;
-	}
-
-	/* enable/disable persistent file realms for collective I/O */
-	/* may want to check for no_indep_rdwr hint as well */
-	ADIOI_Info_check_and_install_enabled(fd, users_info, "romio_cb_pfr",
-		&(fd->hints->cb_pfr), myname, error_code);
-
-	
-	/* file realm assignment types ADIOI_FR_AAR(0),
-	 ADIOI_FR_FSZ(-1), ADIOI_FR_USR_REALMS(-2), all others specify
-	 a regular fr size in bytes. probably not the best way... */
-	ADIOI_Info_check_and_install_int(fd, users_info, "romio_cb_fr_type",
-		&(fd->hints->cb_fr_type), myname, error_code);
-
-	/* Has the user indicated all I/O will be done collectively? */
-	ADIOI_Info_check_and_install_true(fd, users_info, "romio_no_indep_rw",
-		&(fd->hints->no_indep_rw), myname, error_code);
-	if (fd->hints->no_indep_rw == 1) {
-	    /* if 'no_indep_rw' set, also hint that we will do
-	     * collective buffering: if we aren't doing independent io,
-	     * then we have to do collective  */
-	    ADIOI_Info_set(info, "romio_cb_write", "enable");
-	    ADIOI_Info_set(info, "romio_cb_read", "enable");
-	    fd->hints->cb_read = 1;
-	    fd->hints->cb_write = 1;
-	} 
-	/* new hints for enabling/disabling data sieving on
-	 * reads/writes
-	 */
-	ADIOI_Info_check_and_install_enabled(fd, users_info, "romio_ds_read",
-		&(fd->hints->ds_read), myname, error_code);
-	ADIOI_Info_check_and_install_enabled(fd, users_info, "romio_ds_write",
-		&(fd->hints->ds_write), myname, error_code);
-
-	if (ok_to_override_cb_nodes) {
-		/* MPI_File_open path sets up some data structrues that don't
-		 * get resized in the MPI_File_set_view path, so ignore
-		 * cb_nodes in the set_view case */
-	    ADIOI_Info_check_and_install_int(fd, users_info, "cb_nodes",
-		    &(fd->hints->cb_nodes), myname, error_code);
-	    if ((fd->hints->cb_nodes <= 0) || (fd->hints->cb_nodes > nprocs)) {
-		/* can't ask for more aggregators than mpi processes, though it
-		 * might be interesting to think what such oversubscription
-		 * might mean... someday */
-		ADIOI_Snprintf(value, MPI_MAX_INFO_VAL+1, "%d", nprocs);
-		ADIOI_Info_set(info, "cb_nodes", value);
-		fd->hints->cb_nodes = nprocs;
-	    }
-	} /* if (ok_to_override_cb_nodes) */
-
-	ADIOI_Info_check_and_install_int(fd, users_info, "ind_wr_buffer_size",
-		&(fd->hints->ind_wr_buffer_size), myname, error_code);
-	ADIOI_Info_check_and_install_int(fd, users_info, "ind_rd_buffer_size",
-		&(fd->hints->ind_rd_buffer_size), myname, error_code);
-
-	if (fd->hints->cb_config_list == NULL) {
-	    /* only set cb_config_list if it isn't already set.  Note that
-	     * since we set it below, this ensures that the cb_config_list hint
-	     * will be set at file open time either by the user or to the
-	     * default */
-	    /* if it has been set already, we ignore it the second time.
-	     * otherwise we would get an error if someone used the same info
-	     * value with a cb_config_list value in it in a couple of calls,
-	     * which would be irritating. */
-	    ADIOI_Info_check_and_install_str(fd, users_info, "cb_config_list",
-		&(fd->hints->cb_config_list), myname, error_code);
-
-	}
-	ADIOI_Info_check_and_install_int(fd, users_info, "romio_min_fdomain_size",
-		&(fd->hints->min_fdomain_size), myname, error_code);
-
-	/* Now we use striping unit in common code so we should
-	   process hints for it. */
-	ADIOI_Info_check_and_install_int(fd, users_info, "striping_unit", 
-		&(fd->hints->striping_unit), myname, error_code);
-    }
-
-    /* Begin hint post-processig: some hints take precidence over or conflict
-     * with others, or aren't supported by some file systems */
-
-    /* handle cb_config_list default value here; avoids an extra
-     * free/alloc and insures it is always set
-     */
-    if (fd->hints->cb_config_list == NULL) {
-	ADIOI_Info_set(info, "cb_config_list", ADIOI_CB_CONFIG_LIST_DFLT);
-	len = (strlen(ADIOI_CB_CONFIG_LIST_DFLT)+1) * sizeof(char);
-	fd->hints->cb_config_list = ADIOI_Malloc(len);
-	if (fd->hints->cb_config_list == NULL) {
-	    ADIOI_Free(value);
-            *error_code = MPIO_Err_create_code(*error_code,
-                                               MPIR_ERR_RECOVERABLE,
-                                               myname,
-                                               __LINE__,
-                                               MPI_ERR_OTHER,
-                                               "**nomem2",0);
-            return;
-	}
-	ADIOI_Strncpy(fd->hints->cb_config_list, ADIOI_CB_CONFIG_LIST_DFLT, len);
-    }
-    /* deferred_open won't be set by callers, but if the user doesn't
-     * explicitly disable collecitve buffering (two-phase) and does hint that
-     * io w/o independent io is going on, we'll set this internal hint as a
-     * convenience */
-    if ( ( (fd->hints->cb_read != ADIOI_HINT_DISABLE) \
-			    && (fd->hints->cb_write != ADIOI_HINT_DISABLE)\
-			    && fd->hints->no_indep_rw ) ) {
-	    fd->hints->deferred_open = 1;
-    } else {
-	    /* setting romio_no_indep_rw enable and romio_cb_{read,write}
-	     * disable at the same time doesn't make sense. honor
-	     * romio_cb_{read,write} and force the no_indep_rw hint to
-	     * 'disable' */
-	    ADIOI_Info_set(info, "romio_no_indep_rw", "false");
-	    fd->hints->no_indep_rw = 0;
-	    fd->hints->deferred_open = 0;
-    }
-
-    if (ADIO_Feature(fd, ADIO_DATA_SIEVING_WRITES) == 0) {
-    /* disable data sieving for fs that do not
-       support file locking */
-       	ADIOI_Info_get(info, "ind_wr_buffer_size", MPI_MAX_INFO_VAL,
-		     value, &flag);
-	if (flag) {
-	    /* get rid of this value if it is set */
-	    ADIOI_Info_delete(info, "ind_wr_buffer_size");
-	}
-	/* note: leave ind_wr_buffer_size alone; used for other cases
-	 * as well. -- Rob Ross, 04/22/2003
-	 */
-	ADIOI_Info_set(info, "romio_ds_write", "disable");
-	fd->hints->ds_write = ADIOI_HINT_DISABLE;
-    }
-
-    ADIOI_Free(value);
-
-    *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/common/ad_init.c b/3rd-party/romio321/adio/common/ad_init.c
deleted file mode 100644
index 88e75a5251f..00000000000
--- a/3rd-party/romio321/adio/common/ad_init.c
+++ /dev/null
@@ -1,134 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-
-ADIOI_Flatlist_node *ADIOI_Flatlist = NULL;
-ADIOI_Datarep *ADIOI_Datarep_head = NULL;
-    /* list of datareps registered by the user */
-
-/* for f2c and c2f conversion */
-ADIO_File *ADIOI_Ftable = NULL;
-int ADIOI_Ftable_ptr = 0, ADIOI_Ftable_max = 0;
-ADIO_Request *ADIOI_Reqtable = NULL;
-int ADIOI_Reqtable_ptr = 0, ADIOI_Reqtable_max = 0;
-#ifndef HAVE_MPI_INFO
-MPI_Info *MPIR_Infotable = NULL;
-int MPIR_Infotable_ptr = 0, MPIR_Infotable_max = 0;
-#endif
-
-MPI_Info ADIOI_syshints = MPI_INFO_NULL;
-
-MPI_Op ADIO_same_amode=MPI_OP_NULL;
-
-#if defined(ROMIO_XFS) || defined(ROMIO_LUSTRE)
-int ADIOI_Direct_read = 0, ADIOI_Direct_write = 0;
-#endif
-
-int ADIO_Init_keyval=MPI_KEYVAL_INVALID;
-
-MPI_Errhandler ADIOI_DFLT_ERR_HANDLER = MPI_ERRORS_RETURN;
-
-
-static void my_consensus(void *invec, void *inoutvec, int *len, MPI_Datatype *datatype)
-{
-    int i, *in, *inout;
-    in = (int*)invec;
-    inout = (int*)inoutvec;
-
-    for (i=0; i< *len; i++) {
-        if (in[i] != inout[i])
-	    inout[i] = ADIO_AMODE_NOMATCH;
-    }
-    return;
-}
-
-void ADIO_Init(int *argc, char ***argv, int *error_code)
-{
-#if defined(ROMIO_XFS) || defined(ROMIO_LUSTRE)
-    char *c;
-#endif
-
-    ADIOI_UNREFERENCED_ARG(argc);
-    ADIOI_UNREFERENCED_ARG(argv);
-    
-#ifdef ROMIO_INSIDE_MPICH
-    MPIR_Ext_init();
-#endif
-
-/* initialize the linked list containing flattened datatypes */
-    ADIOI_Flatlist = (ADIOI_Flatlist_node *) ADIOI_Malloc(sizeof(ADIOI_Flatlist_node));
-    ADIOI_Flatlist->type = MPI_DATATYPE_NULL;
-    ADIOI_Flatlist->next = NULL;
-    ADIOI_Flatlist->blocklens = NULL;
-    ADIOI_Flatlist->indices = NULL;
-
-#if defined(ROMIO_XFS) || defined(ROMIO_LUSTRE)
-    c = getenv("MPIO_DIRECT_READ");
-    if (c && (!strcmp(c, "true") || !strcmp(c, "TRUE"))) 
-	ADIOI_Direct_read = 1;
-    else ADIOI_Direct_read = 0;
-    c = getenv("MPIO_DIRECT_WRITE");
-    if (c && (!strcmp(c, "true") || !strcmp(c, "TRUE"))) 
-	ADIOI_Direct_write = 1;
-    else ADIOI_Direct_write = 0;
-#endif
-
-
-#ifdef ADIOI_MPE_LOGGING
-    {
-        MPE_Log_get_state_eventIDs( &ADIOI_MPE_open_a, &ADIOI_MPE_open_b );
-        MPE_Log_get_state_eventIDs( &ADIOI_MPE_read_a, &ADIOI_MPE_read_b );
-        MPE_Log_get_state_eventIDs( &ADIOI_MPE_write_a, &ADIOI_MPE_write_b );
-        MPE_Log_get_state_eventIDs( &ADIOI_MPE_lseek_a, &ADIOI_MPE_lseek_b );
-        MPE_Log_get_state_eventIDs( &ADIOI_MPE_close_a, &ADIOI_MPE_close_b );
-        MPE_Log_get_state_eventIDs( &ADIOI_MPE_writelock_a,
-                                    &ADIOI_MPE_writelock_b );
-        MPE_Log_get_state_eventIDs( &ADIOI_MPE_readlock_a,
-                                    &ADIOI_MPE_readlock_b );
-        MPE_Log_get_state_eventIDs( &ADIOI_MPE_unlock_a, &ADIOI_MPE_unlock_b );
-        MPE_Log_get_state_eventIDs( &ADIOI_MPE_postwrite_a,
-                                    &ADIOI_MPE_postwrite_b );
-	MPE_Log_get_state_eventIDs( &ADIOI_MPE_openinternal_a, 
-			&ADIOI_MPE_openinternal_b);
-	MPE_Log_get_state_eventIDs( &ADIOI_MPE_stat_a, &ADIOI_MPE_stat_b);
-	MPE_Log_get_state_eventIDs( &ADIOI_MPE_iread_a, &ADIOI_MPE_iread_b);
-	MPE_Log_get_state_eventIDs( &ADIOI_MPE_iwrite_a, &ADIOI_MPE_iwrite_b);
-
-        int  comm_world_rank;
-        MPI_Comm_rank( MPI_COMM_WORLD, &comm_world_rank );
-
-        if ( comm_world_rank == 0 ) {
-            MPE_Describe_state( ADIOI_MPE_open_a, ADIOI_MPE_open_b,
-                                "open", "orange" );
-            MPE_Describe_state( ADIOI_MPE_read_a, ADIOI_MPE_read_b,
-                                "read", "green" );
-            MPE_Describe_state( ADIOI_MPE_write_a, ADIOI_MPE_write_b,
-                                "write", "blue" );
-            MPE_Describe_state( ADIOI_MPE_lseek_a, ADIOI_MPE_lseek_b,
-                                "lseek", "red" );
-            MPE_Describe_state( ADIOI_MPE_close_a, ADIOI_MPE_close_b,
-                                "close", "grey" );
-            MPE_Describe_state( ADIOI_MPE_writelock_a, ADIOI_MPE_writelock_b,
-                                "writelock", "plum" );
-            MPE_Describe_state( ADIOI_MPE_readlock_a, ADIOI_MPE_readlock_b,
-                                "readlock", "magenta" );
-            MPE_Describe_state( ADIOI_MPE_unlock_a, ADIOI_MPE_unlock_b,
-                                "unlock", "purple" );
-            MPE_Describe_state( ADIOI_MPE_postwrite_a, ADIOI_MPE_postwrite_b,
-                                "postwrite", "ivory" );
-	    MPE_Describe_state( ADIOI_MPE_openinternal_a, ADIOI_MPE_openinternal_b, "open system", "blue");
-	    MPE_Describe_state( ADIOI_MPE_stat_a, ADIOI_MPE_stat_b, "stat", "purple");
-	    MPE_Describe_state( ADIOI_MPE_iread_a, ADIOI_MPE_iread_b, "iread", "purple");
-	    MPE_Describe_state( ADIOI_MPE_iwrite_a, ADIOI_MPE_iwrite_b, "iwrite", "purple");
-        }
-    }
-#endif
-
-    *error_code = MPI_SUCCESS;
-    MPI_Op_create(my_consensus, 1, &ADIO_same_amode);
-}
diff --git a/3rd-party/romio321/adio/common/ad_io_coll.c b/3rd-party/romio321/adio/common/ad_io_coll.c
deleted file mode 100644
index 273c8a551b2..00000000000
--- a/3rd-party/romio321/adio/common/ad_io_coll.c
+++ /dev/null
@@ -1,1135 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 2008 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "assert.h"
-#include "adio.h"
-#include "adio_extern.h"
-#ifdef AGGREGATION_PROFILE
-#include "mpe.h"
-#endif
-
-/* #define ALLTOALL */
-
-/* #define DEBUG */
-/* #define DEBUG2 */  /* print buffers */
-
-#define USE_PRE_REQ
-
-static void Exch_data_amounts (ADIO_File fd, int nprocs,
-			ADIO_Offset *client_comm_sz_arr,
-			ADIO_Offset *agg_comm_sz_arr,
-			int *client_alltoallw_counts,
-			int *agg_alltoallw_counts,
-			int *aggregators_done);
-static void post_aggregator_comm (MPI_Comm comm, int rw_type, int nproc,
-			   void *cb_buf,
-			   MPI_Datatype *client_comm_dtype_arr,
-			   ADIO_Offset *client_comm_sz_arr,
-			   MPI_Request **requests,
-			   int *aggregators_client_count_p);
-
-static void post_client_comm (ADIO_File fd, int rw_type, 
-		       int agg_rank, void *buf,
-		       MPI_Datatype agg_comm_dtype,
-		       int agg_alltoallw_count,
-		       MPI_Request *request);
-
-/* Avery Ching and Kenin Columa's reworked two-phase algorithm.  Key features
- * - persistent file domains
- * - an option to use alltoall instead of point-to-point
- */
-void ADIOI_IOStridedColl (ADIO_File fd, void *buf, int count, int rdwr,
-			  MPI_Datatype datatype, int file_ptr_type,
-			  ADIO_Offset offset, ADIO_Status *status,
-			  int *error_code)
-{
-    ADIO_Offset min_st_offset=0, max_end_offset=0;
-    ADIO_Offset st_end_offset[2];
-    ADIO_Offset *all_st_end_offsets = NULL;
-    int filetype_is_contig, buftype_is_contig, is_contig;
-    ADIO_Offset off;
-    int interleave_count = 0, i, nprocs, myrank, nprocs_for_coll;
-    int cb_enable;
-    ADIO_Offset bufsize;
-    MPI_Aint extent, lb;
-#ifdef DEBUG2
-    MPI_Aint bufextent;
-#endif
-    MPI_Count size;
-    int agg_rank;
-
-    ADIO_Offset agg_disp; /* aggregated file offset */
-    MPI_Datatype agg_dtype; /* aggregated file datatype */
-
-    int aggregators_done = 0;
-    ADIO_Offset buffered_io_size = 0;
-
-    int *alltoallw_disps;
-
-    int *alltoallw_counts;
-    int *client_alltoallw_counts;
-    int *agg_alltoallw_counts;
-
-    char *cb_buf = NULL;
-
-    MPI_Datatype *client_comm_dtype_arr; /* aggregator perspective */
-    MPI_Datatype *agg_comm_dtype_arr;    /* client perspective */
-    ADIO_Offset *client_comm_sz_arr;     /* aggregator perspective */
-    ADIO_Offset *agg_comm_sz_arr;        /* client perspective */
-
-    /* file views for each client and aggregator */
-    view_state *client_file_view_state_arr = NULL;
-    view_state *agg_file_view_state_arr    = NULL;
-    /* mem views for local process */
-    view_state *my_mem_view_state_arr      = NULL;
-
-    MPI_Status *agg_comm_statuses     = NULL;
-    MPI_Request *agg_comm_requests    = NULL;
-    MPI_Status *client_comm_statuses  = NULL;
-    MPI_Request *client_comm_requests = NULL;
-    int aggs_client_count = 0;
-    int clients_agg_count = 0;
-
-    MPI_Comm_size (fd->comm, &nprocs);
-    MPI_Comm_rank (fd->comm, &myrank);
-#ifdef DEBUG
-    fprintf (stderr, "p%d: entering ADIOI_IOStridedColl\n", myrank);
-#endif
-#ifdef AGGREGATION_PROFILE
-    if (rdwr == ADIOI_READ)
-	MPE_Log_event (5010, 0, NULL);
-    else
-	MPE_Log_event (5012, 0, NULL);
-#endif
-
-    /* I need to check if there are any outstanding nonblocking writes
-       to the file, which could potentially interfere with the writes
-       taking place in this collective write call. Since this is not
-       likely to be common, let me do the simplest thing possible here:
-       Each process completes all pending nonblocking operations before
-       completing. */
-
-    nprocs_for_coll = fd->hints->cb_nodes;
-
-    if (rdwr == ADIOI_READ)
-	cb_enable = fd->hints->cb_read;
-    else
-	cb_enable = fd->hints->cb_write;
-
-    /* only check for interleaving if cb_read isn't disabled */
-    if (cb_enable != ADIOI_HINT_DISABLE) {
-	/* find the starting and ending byte of my I/O access */
-	ADIOI_Calc_bounds (fd, count, datatype, file_ptr_type, offset,
-			   &st_end_offset[0], &st_end_offset[1]);
-
-	/* allocate an array of start/end pairs */
-	all_st_end_offsets = (ADIO_Offset *)
-	    ADIOI_Malloc (2*nprocs*sizeof(ADIO_Offset));
-	MPI_Allgather (st_end_offset, 2, ADIO_OFFSET, all_st_end_offsets, 2,
-		       ADIO_OFFSET, fd->comm);
-
-	min_st_offset = all_st_end_offsets[0];
-	max_end_offset = all_st_end_offsets[1];
-
-	for (i=1; i<nprocs; i++) {
-	    /* are the accesses of different processes interleaved? */
-	    if ((all_st_end_offsets[i*2] < all_st_end_offsets[i*2-1]) &&
-		(all_st_end_offsets[i*2] <= all_st_end_offsets[i*2+1]))
-		interleave_count++;
-	    /* This is a rudimentary check for interleaving, but should
-	     * suffice for the moment. */
-	    
-	    min_st_offset = ADIOI_MIN(all_st_end_offsets[i*2],
-				      min_st_offset);
-	    max_end_offset = ADIOI_MAX(all_st_end_offsets[i*2+1],
-				       max_end_offset);
-	}
-    }
-
-    ADIOI_Datatype_iscontig (datatype, &buftype_is_contig);
-    ADIOI_Datatype_iscontig (fd->filetype, &filetype_is_contig);
-
-    if ((cb_enable == ADIOI_HINT_DISABLE
-	 || (!interleave_count && (cb_enable == ADIOI_HINT_AUTO)))
-	&& (fd->hints->cb_pfr != ADIOI_HINT_ENABLE)){
-	if (cb_enable != ADIOI_HINT_DISABLE) {
-	    ADIOI_Free (all_st_end_offsets);
-	}
-
-	if (buftype_is_contig && filetype_is_contig) {
-	    if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-		off = fd->disp + (fd->etype_size) * offset;
-		if (rdwr == ADIOI_READ)
-		    ADIO_ReadContig(fd, buf, count, datatype,
-				    ADIO_EXPLICIT_OFFSET, off, status,
-				    error_code);
-		else
-		    ADIO_WriteContig(fd, buf, count, datatype,
-				     ADIO_EXPLICIT_OFFSET, off, status,
-				     error_code);
-	    }
-	    else {
-		if (rdwr == ADIOI_READ)
-		    ADIO_ReadContig(fd, buf, count, datatype, ADIO_INDIVIDUAL,
-				    0, status, error_code);
-		else
-		    ADIO_WriteContig(fd, buf, count, datatype, ADIO_INDIVIDUAL,
-				     0, status, error_code);
-	    }
-	}
-	else {
-	    if (rdwr == ADIOI_READ)
-		ADIO_ReadStrided(fd, buf, count, datatype, file_ptr_type,
-				 offset, status, error_code);
-	    else
-		ADIO_WriteStrided(fd, buf, count, datatype, file_ptr_type,
-				  offset, status, error_code);
-	}
-	return;
-    }
-
-    MPI_Type_get_extent(datatype, &lb, &extent);
-#ifdef DEBUG2
-    bufextent = extent * count;
-#endif
-    MPI_Type_size_x(datatype, &size);
-    bufsize = size * (MPI_Count)count;
-
-    /* Calculate file realms */
-    if ((fd->hints->cb_pfr != ADIOI_HINT_ENABLE) ||
-	(fd->file_realm_types == NULL))
-	ADIOI_Calc_file_realms (fd, min_st_offset, max_end_offset);
-
-    my_mem_view_state_arr = (view_state *)
-	ADIOI_Calloc (1, nprocs * sizeof(view_state));
-    agg_file_view_state_arr = (view_state *)
-	ADIOI_Calloc (1, nprocs * sizeof(view_state));
-    client_comm_sz_arr = (ADIO_Offset *)
-	ADIOI_Calloc (1, nprocs * sizeof(ADIO_Offset));
-
-    if (fd->is_agg) {
-	client_file_view_state_arr = (view_state *)
-	    ADIOI_Calloc (1, nprocs * sizeof(view_state));
-    }
-    else {
-	client_file_view_state_arr = NULL;
-    }
-
-    /* Alltoallw doesn't like a null array even if the counts are
-     * zero.  If you do not include this code, it will fail. */
-    client_comm_dtype_arr = (MPI_Datatype *)
-	ADIOI_Calloc (1, nprocs * sizeof(MPI_Datatype));
-    if (!fd->is_agg)
-	for (i = 0; i < nprocs; i++)
-	    client_comm_dtype_arr[i] = MPI_BYTE;
-
-    ADIOI_Exch_file_views (myrank, nprocs, file_ptr_type, fd, count,
-			   datatype, offset, my_mem_view_state_arr,
-			   agg_file_view_state_arr,
-			   client_file_view_state_arr);
-
-    agg_comm_sz_arr = (ADIO_Offset *)
-	ADIOI_Calloc (1, nprocs * sizeof(ADIO_Offset));
-    agg_comm_dtype_arr = (MPI_Datatype *)
-	ADIOI_Malloc (nprocs * sizeof(MPI_Datatype));
-    if (fd->is_agg) {
-	ADIOI_Build_agg_reqs (fd, rdwr, nprocs,
-			      client_file_view_state_arr,
-			      client_comm_dtype_arr,
-			      client_comm_sz_arr,
-			      &agg_disp,
-			      &agg_dtype);
-	buffered_io_size = 0;
-	for (i=0; i <nprocs; i++) {
-	    if (client_comm_sz_arr[i] > 0)
-		buffered_io_size += client_comm_sz_arr[i];
-	}
-    }
-#ifdef USE_PRE_REQ
-    else 
-    {
-	/* Example use of ADIOI_Build_client_pre_req. to an
-	 * appropriate section */
-	
-	for (i = 0; i < fd->hints->cb_nodes; i++)
-	{
-	    agg_rank = fd->hints->ranklist[(i+myrank)%fd->hints->cb_nodes];
-#ifdef AGGREGATION_PROFILE
-	    MPE_Log_event (5040, 0, NULL);
-#endif
-	    ADIOI_Build_client_pre_req(
-		fd, agg_rank, (i+myrank)%fd->hints->cb_nodes,
-		&(my_mem_view_state_arr[agg_rank]),
-		&(agg_file_view_state_arr[agg_rank]),
-		2*1024*1024, 
-		64*1024);
-#ifdef AGGREGATION_PROFILE
-	    MPE_Log_event (5041, 0, NULL);
-#endif
-	}
-    }
-#endif
-
-
-    if (fd->is_agg)
-	cb_buf = (char *) ADIOI_Malloc (fd->hints->cb_buffer_size);
-    alltoallw_disps  = (int *) ADIOI_Calloc (nprocs, sizeof(int));
-    alltoallw_counts = client_alltoallw_counts = (int *)
-	ADIOI_Calloc (2*nprocs, sizeof(int));
-    agg_alltoallw_counts = &alltoallw_counts[nprocs];
-
-    if (fd->hints->cb_alltoall == ADIOI_HINT_DISABLE) {
-        /* aggregators pre-post all Irecv's for incoming data from clients */
-        if ((fd->is_agg) && (rdwr == ADIOI_WRITE))
-	    post_aggregator_comm(fd->comm, rdwr, nprocs, cb_buf,
-			     client_comm_dtype_arr,
-			     client_comm_sz_arr,
-			     &agg_comm_requests,
-			     &aggs_client_count);
-    }
-    /* Aggregators send amounts for data requested to clients */
-    Exch_data_amounts (fd, nprocs, client_comm_sz_arr, agg_comm_sz_arr,
-		       client_alltoallw_counts, agg_alltoallw_counts,
-		       &aggregators_done);
-
-#ifdef DEBUG
-    fprintf (stderr, "client_alltoallw_counts[ ");
-    for (i=0; i<nprocs; i++) {
-	fprintf (stderr, "%d ", client_alltoallw_counts[i]);
-    }
-    fprintf (stderr, "]\n");
-    fprintf (stderr, "agg_alltoallw_counts[ ");
-    for (i=0; i<nprocs; i++) {
-	fprintf (stderr,"%d ", agg_alltoallw_counts[i]);
-    }
-    fprintf (stderr, "]\n");
-#endif
-
-    /* keep looping while aggregators still have I/O to do */
-    while (aggregators_done != nprocs_for_coll) {
-	if (fd->hints->cb_alltoall == ADIOI_HINT_DISABLE) {
-	/* clients should build datatypes for local memory locations
-	   for data communication with aggregators and post
-	   communication as the datatypes are built */
-
-	client_comm_requests = (MPI_Request *)
-	    ADIOI_Calloc (fd->hints->cb_nodes, sizeof(MPI_Request));
-
-	for (i = 0; i < fd->hints->cb_nodes; i++)
-	{
-	    clients_agg_count = 0;
-	    agg_rank = fd->hints->ranklist[(i+myrank)%fd->hints->cb_nodes];
-	    if (agg_comm_sz_arr[agg_rank] > 0) {
-	        ADIOI_Build_client_req(fd, agg_rank,
-				       (i+myrank)%fd->hints->cb_nodes,
-				       &(my_mem_view_state_arr[agg_rank]),
-				       &(agg_file_view_state_arr[agg_rank]),
-				       agg_comm_sz_arr[agg_rank], 
-				       &(agg_comm_dtype_arr[agg_rank]));
-
-#ifdef AGGREGATION_PROFILE
-		if (i == 0)
-		    MPE_Log_event (5038, 0, NULL);
-#endif
-		post_client_comm (fd, rdwr, agg_rank, buf,
-				  agg_comm_dtype_arr[agg_rank],
-				  agg_alltoallw_counts[agg_rank],
-				  &client_comm_requests[clients_agg_count]);
-		clients_agg_count++;
-	    }
-	}
-#ifdef AGGREGATION_PROFILE
-	if (!clients_agg_count)
-	    MPE_Log_event(5039, 0, NULL);
-#endif
-
-	if (rdwr == ADIOI_READ) {
-	    if (fd->is_agg && buffered_io_size) {
-		ADIOI_IOFiletype (fd, cb_buf, buffered_io_size, MPI_BYTE,
-				  ADIO_EXPLICIT_OFFSET, agg_disp, agg_dtype,
-				  ADIOI_READ, status, error_code);
-		if (*error_code != MPI_SUCCESS) return;
-		MPI_Type_free (&agg_dtype);
-	    }
-
-#ifdef DEBUG
-	    fprintf (stderr, "expecting from [agg](disp,size,cnt)=");
-	    for (i=0; i < nprocs; i++) {
-		MPI_Type_size_x (agg_comm_dtype_arr[i], &size);
-		fprintf (stderr, "[%d](%d,%d,%d)", i, alltoallw_disps[i], 
-			 size, agg_alltoallw_counts[i]);
-		if (i != nprocs - 1)
-		    fprintf(stderr, ",");
-	    }
-	    fprintf (stderr, "]\n");
-	    if (fd->is_agg) {
-		fprintf (stderr, "sending to [client](disp,size,cnt)=");
-		for (i=0; i < nprocs; i++) {
-		    if (fd->is_agg)
-			MPI_Type_size_x (client_comm_dtype_arr[i], &size);
-		    else
-			size = -1;
-		    
-		    fprintf (stderr, "[%d](%d,%d,%d)", i, alltoallw_disps[i], 
-			     size, client_alltoallw_counts[i]);
-		    if (i != nprocs - 1)
-			fprintf(stderr, ",");
-		}
-		fprintf (stderr,"\n");
-	    }
-	    fflush (NULL);
-#endif
-	    /* aggregators post all Isends for outgoing data to clients */
-	    if (fd->is_agg)
-		post_aggregator_comm(fd->comm, rdwr, nprocs, cb_buf,
-				     client_comm_dtype_arr,
-				     client_comm_sz_arr,
-				     &agg_comm_requests,
-				     &aggs_client_count);
-
-	    if (fd->is_agg && aggs_client_count) {
-		agg_comm_statuses = ADIOI_Malloc(aggs_client_count *
-						 sizeof(MPI_Status));
-		MPI_Waitall(aggs_client_count, agg_comm_requests,
-			    agg_comm_statuses);
-#ifdef AGGREGATION_PROFILE
-		MPE_Log_event (5033, 0, NULL);
-#endif
-		ADIOI_Free (agg_comm_requests);
-		ADIOI_Free (agg_comm_statuses);
-	    }
-
-	    if (clients_agg_count) {
-		client_comm_statuses = ADIOI_Malloc(clients_agg_count *
-						    sizeof(MPI_Status));
-		MPI_Waitall(clients_agg_count, client_comm_requests,
-			    client_comm_statuses);
-#ifdef AGGREGATION_PROFILE
-		MPE_Log_event (5039, 0, NULL);
-#endif
-		ADIOI_Free (client_comm_requests);
-		ADIOI_Free (client_comm_statuses);
-	    }
-
-#ifdef DEBUG2
-	    fprintf (stderr, "buffered_io_size = %lld\n", buffered_io_size);
-	    if (fd->is_agg && buffered_io_size) {
-		fprintf (stderr, "buf = [");
-		for (i=0; i<bufextent; i++)
-		    fprintf (stderr, "%c", ((char *) buf)[i]);
-		fprintf (stderr, "]\n");
-		fprintf (stderr, "cb_buf = [");
-		for (i=0; i<buffered_io_size; i++)
-		    fprintf (stderr, "%c", cb_buf[i]);
-		fprintf (stderr, "]\n");
-		fflush (NULL);
-	    }
-#endif
-	}
-	else { /* Write Case */
-#ifdef DEBUG
-	    fprintf (stderr, "sending to [agg](disp,size,cnt)=");
-	    for (i=0; i < nprocs; i++) {
-		MPI_Type_size_x (agg_comm_dtype_arr[i], &size);
-		fprintf (stderr, "[%d](%d,%d,%d)", i, alltoallw_disps[i], 
-			 size, agg_alltoallw_counts[i]);
-		if (i != nprocs - 1)
-		    fprintf(stderr, ",");
-	    }
-	    fprintf (stderr, "]\n");
-	    fprintf (stderr, "expecting from [client](disp,size,cnt)=");
-	    for (i=0; i < nprocs; i++) {
-		if (fd->is_agg)
-		    MPI_Type_size_x (client_comm_dtype_arr[i], &size);
-		else
-		    size = -1;
-		
-		fprintf (stderr, "[%d](%d,%d,%d)", i, alltoallw_disps[i], 
-			 size, client_alltoallw_counts[i]);
-		if (i != nprocs - 1)
-		    fprintf(stderr, ",");
-	    }
-	    fprintf (stderr,"\n");
-	    fflush (NULL);
-#endif
-#ifdef DEBUG
-	    fprintf (stderr, "buffered_io_size = %lld\n", buffered_io_size);
-#endif
-	    
-	    if (clients_agg_count) {
-		client_comm_statuses = ADIOI_Malloc(clients_agg_count *
-						    sizeof(MPI_Status));
-		MPI_Waitall(clients_agg_count, client_comm_requests,
-			    client_comm_statuses);
-#ifdef AGGREGATION_PROFILE
-		MPE_Log_event (5039, 0, NULL);
-#endif
-		ADIOI_Free(client_comm_requests);
-		ADIOI_Free(client_comm_statuses);
-	    }
-#ifdef DEBUG2
-	    if (bufextent) {
-		fprintf (stderr, "buf = [");
-		for (i=0; i<bufextent; i++)
-		    fprintf (stderr, "%c", ((char *) buf)[i]);
-		fprintf (stderr, "]\n");
-	    }
-#endif
-
-	    if (fd->is_agg && buffered_io_size) {
-		assert (aggs_client_count != 0);
-		/* make sure we actually have the data to write out */
-		agg_comm_statuses = (MPI_Status *)
-		    ADIOI_Malloc (aggs_client_count*sizeof(MPI_Status));
-		
-		MPI_Waitall (aggs_client_count, agg_comm_requests,
-			     agg_comm_statuses);
-#ifdef AGGREGATION_PROFILE
-		MPE_Log_event (5033, 0, NULL);
-#endif
-		ADIOI_Free (agg_comm_requests);
-		ADIOI_Free (agg_comm_statuses);
-#ifdef DEBUG2
-		fprintf (stderr, "cb_buf = [");
-		for (i=0; i<buffered_io_size; i++)
-		    fprintf (stderr, "%c", cb_buf[i]);
-		fprintf (stderr, "]\n");
-		fflush (NULL);
-#endif
-		ADIOI_IOFiletype (fd, cb_buf, buffered_io_size, MPI_BYTE,
-				  ADIO_EXPLICIT_OFFSET, agg_disp, agg_dtype,
-				  ADIOI_WRITE, status, error_code);
-		if (*error_code != MPI_SUCCESS) return;
-		MPI_Type_free (&agg_dtype);
-	    }
-
-	}
-	} else {
-	/* Alltoallw version of everything */
-	ADIOI_Build_client_reqs(fd, nprocs, my_mem_view_state_arr,
-				agg_file_view_state_arr,
-				agg_comm_sz_arr, agg_comm_dtype_arr);
-
-	if (rdwr == ADIOI_READ) {
-	    if (fd->is_agg && buffered_io_size) {
-		ADIOI_IOFiletype (fd, cb_buf, buffered_io_size, MPI_BYTE,
-				  ADIO_EXPLICIT_OFFSET, agg_disp, agg_dtype,
-				  ADIOI_READ, status, error_code);
-		if (*error_code != MPI_SUCCESS) return;
-		MPI_Type_free (&agg_dtype);
-	    }
-
-#ifdef AGGREGATION_PROFILE
-	    MPE_Log_event (5032, 0, NULL);
-#endif
-	    MPI_Alltoallw (cb_buf, client_alltoallw_counts, alltoallw_disps,
-			   client_comm_dtype_arr,
-			   buf, agg_alltoallw_counts , alltoallw_disps,
-			   agg_comm_dtype_arr,
-			   fd->comm);
-#ifdef AGGREGATION_PROFILE
-	    MPE_Log_event (5033, 0, NULL);
-#endif
-	}
-	else { /* Write Case */
-#ifdef AGGREGATION_PROFILE
-	    MPE_Log_event (5032, 0, NULL);
-#endif
-	    MPI_Alltoallw (buf, agg_alltoallw_counts, alltoallw_disps,
-			   agg_comm_dtype_arr,
-			   cb_buf, client_alltoallw_counts, alltoallw_disps,
-			   client_comm_dtype_arr,
-			   fd->comm);
-#ifdef AGGREGATION_PROFILE
-	    MPE_Log_event (5033, 0, NULL);
-#endif
-	    if (fd->is_agg && buffered_io_size) {
-		ADIOI_IOFiletype (fd, cb_buf, buffered_io_size, MPI_BYTE,
-				  ADIO_EXPLICIT_OFFSET, agg_disp, agg_dtype,
-				  ADIOI_WRITE, status, error_code);
-		if (*error_code != MPI_SUCCESS) return;
-		MPI_Type_free (&agg_dtype);
-	    }
-	}
-	}
-
-	/* Free (uncommit) datatypes for reuse */
-	if (fd->is_agg) {
-	    if (buffered_io_size > 0) {
-		for (i=0; i<nprocs; i++) {
-		    if (client_comm_sz_arr[i] > 0)
-			MPI_Type_free (&client_comm_dtype_arr[i]);
-		}
-	    }
-	}
-	for (i=0; i<nprocs; i++) {
-	    if (agg_comm_sz_arr[i] > 0)
-		MPI_Type_free (&agg_comm_dtype_arr[i]);
-	}
-
-	/* figure out next set up requests */
-	if (fd->is_agg) {
-	    ADIOI_Build_agg_reqs (fd, rdwr, nprocs,
-				  client_file_view_state_arr,
-				  client_comm_dtype_arr,
-				  client_comm_sz_arr,
-				  &agg_disp,
-				  &agg_dtype);
-	    buffered_io_size = 0;
-	    for (i=0; i <nprocs; i++) {
-		if (client_comm_sz_arr[i] > 0)
-		    buffered_io_size += client_comm_sz_arr[i];
-	    }
-	}
-#ifdef USE_PRE_REQ
-	else {
-	    /* Example use of ADIOI_Build_client_pre_req. to an
-	     * appropriate section */
-	    for (i = 0; i < fd->hints->cb_nodes; i++)
-	    {
-		agg_rank = fd->hints->ranklist[(i+myrank)%fd->hints->cb_nodes];
-#ifdef AGGREGATION_PROFILE
-		MPE_Log_event (5040, 0, NULL);
-#endif
-		ADIOI_Build_client_pre_req(
-		    fd, agg_rank, (i+myrank)%fd->hints->cb_nodes,
-		    &(my_mem_view_state_arr[agg_rank]),
-		    &(agg_file_view_state_arr[agg_rank]),
-		    2*1024*1024, 
-		    64*1024);
-#ifdef AGGREGATION_PROFILE
-		MPE_Log_event (5041, 0, NULL);
-#endif
-	    }
-	}
-#endif
-	
-	/* aggregators pre-post all Irecv's for incoming data from
-	 * clients.  if nothing is needed, agg_comm_requests is not
-	 * allocated */
-	if (fd->hints->cb_alltoall == ADIOI_HINT_DISABLE) {
-	    if ((fd->is_agg) && (rdwr == ADIOI_WRITE))
-	        post_aggregator_comm(fd->comm, rdwr, nprocs, cb_buf,
-				 client_comm_dtype_arr,
-				 client_comm_sz_arr,
-				 &agg_comm_requests,
-				 &aggs_client_count);
-	}
-
-	/* Aggregators send amounts for data requested to clients */
-	Exch_data_amounts (fd, nprocs, client_comm_sz_arr, agg_comm_sz_arr,
-			   client_alltoallw_counts, agg_alltoallw_counts,
-			   &aggregators_done);
-
-    }
-
-    /* Clean up */
-	
-    if (fd->hints->cb_pfr != ADIOI_HINT_ENABLE) {
-	/* AAR, FSIZE, and User provided uniform File realms */
-	if (1) {
-	    ADIOI_Delete_flattened (fd->file_realm_types[0]);
-	    MPI_Type_free (&fd->file_realm_types[0]);
-	}
-	else {
-	    for (i=0; i<fd->hints->cb_nodes; i++) {
-		ADIOI_Datatype_iscontig(fd->file_realm_types[i], &is_contig);
-		if (!is_contig)
-		    ADIOI_Delete_flattened(fd->file_realm_types[i]);
-		MPI_Type_free (&fd->file_realm_types[i]);
-	    }
-	}
-	ADIOI_Free (fd->file_realm_types);
-	ADIOI_Free (fd->file_realm_st_offs);
-    }
-
-    /* This memtype must be deleted from the ADIOI_Flatlist or else it
-     * will match incorrectly with other datatypes which use this
-     * pointer. */
-    ADIOI_Delete_flattened(datatype);
-    ADIOI_Delete_flattened(fd->filetype);
-
-    if (fd->is_agg) {
-	if (buffered_io_size > 0)
-	    MPI_Type_free (&agg_dtype);
-	for (i=0; i<nprocs; i++) {
-	    MPI_Type_free (&client_comm_dtype_arr[i]);
-	    ADIOI_Free (client_file_view_state_arr[i].flat_type_p->indices);
-	    ADIOI_Free (client_file_view_state_arr[i].flat_type_p->blocklens);
-	    ADIOI_Free (client_file_view_state_arr[i].flat_type_p);
-	}
-	ADIOI_Free (client_file_view_state_arr);
-	ADIOI_Free (cb_buf);
-    } 
-    for (i = 0; i<nprocs; i++)
-	if (agg_comm_sz_arr[i] > 0)
-	    MPI_Type_free (&agg_comm_dtype_arr[i]);
-    
-    ADIOI_Free (client_comm_sz_arr);
-    ADIOI_Free (client_comm_dtype_arr);
-    ADIOI_Free (my_mem_view_state_arr);
-    ADIOI_Free (agg_file_view_state_arr);
-    ADIOI_Free (agg_comm_sz_arr);
-    ADIOI_Free (agg_comm_dtype_arr);
-    ADIOI_Free (alltoallw_disps);
-    ADIOI_Free (alltoallw_counts);
-    ADIOI_Free (all_st_end_offsets);
-
-#ifdef HAVE_STATUS_SET_BYTES
-    MPIR_Status_set_bytes(status, datatype, bufsize);
-    /* This is a temporary way of filling in status.  The right way is
-     * to keep track of how much data was actually read and placed in
-     * buf during collective I/O. */
-#endif
-    fd->fp_sys_posn = -1; /* set it to null. */
-#ifdef AGGREGATION_PROFILE
-    if (rdwr == ADIOI_READ)
-	MPE_Log_event (5011, 0, NULL);
-    else
-	MPE_Log_event (5013, 0, NULL);
-#endif
-}
-
-
-/* Some of this code is from the old Calc_my_off_len() function.
- * It calculates the 1st and last byte accessed */
-void ADIOI_Calc_bounds (ADIO_File fd, int count, MPI_Datatype buftype,
-			int file_ptr_type, ADIO_Offset offset,
-			ADIO_Offset *st_offset, ADIO_Offset *end_offset)
-{
-    MPI_Count filetype_size, buftype_size, etype_size;
-    int sum;
-    MPI_Aint filetype_extent, lb;
-    ADIO_Offset total_io;
-    int filetype_is_contig;
-    ADIO_Offset i, remainder;
-    ADIOI_Flatlist_node *flat_file;
-    
-    ADIO_Offset st_byte_off, end_byte_off;
-    
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5000, 0, NULL);
-#endif
-
-    if (!count) {
-	/* Max signed positive value for ADIO_Offset
-	 * (arch. dependent?).  is there a better way? */
-	memset (st_offset, 8, sizeof(ADIO_Offset));
-	*st_offset = *st_offset / 2;
-	*end_offset = -1;
-	return;
-    }
-
-    ADIOI_Datatype_iscontig (fd->filetype, &filetype_is_contig);
-    
-    MPI_Type_size_x (fd->filetype, &filetype_size);
-    MPI_Type_get_extent (fd->filetype, &lb, &filetype_extent);
-    MPI_Type_size_x (fd->etype, &etype_size);
-    MPI_Type_size_x (buftype, &buftype_size);
-    
-    total_io = buftype_size * count;
-
-    if (filetype_is_contig) {
-	if (file_ptr_type == ADIO_INDIVIDUAL)
-	    st_byte_off = fd->fp_ind;
-	else
-	    st_byte_off = fd->disp + etype_size * offset;
-
-	end_byte_off = st_byte_off + total_io - 1;
-    }
-    else {
-	flat_file = ADIOI_Flatlist;
-	while (flat_file->type != fd->filetype) flat_file = flat_file->next;
-
-	/* we need to take care of some weirdness since fd->fp_ind
-	   points at an accessible byte in file.  the first accessible
-	   byte in the file is not necessarily the first byte, nor is
-	   it necessarily the first off/len pair in the filetype. */
-	if (file_ptr_type == ADIO_INDIVIDUAL) {	
-	    st_byte_off = fd->fp_ind;
-	    /* find end byte of I/O (may be in middle of an etype) */
-
-	    /* calculate byte starting point of first filetype */
-	    end_byte_off = (ADIO_Offset)
-		((fd->fp_ind - fd->disp - flat_file->indices[0]) /
-		 filetype_extent) * filetype_extent + fd->disp +
-		flat_file->indices[0];
-	    /* number of absolute bytes into first filetype */
-	    remainder = (fd->fp_ind - fd->disp - flat_file->indices[0]) %
-		filetype_extent;
-	    if (remainder) {
-		/* find how many file viewable bytes into first filetype */
-		sum = 0;
-		for (i=0; i<flat_file->count; i++) {
-		    sum += flat_file->blocklens[i];
-		    if ((flat_file->indices[i] - flat_file->indices[0] +
-			 flat_file->blocklens[i]) >= remainder) {
-			sum -= (flat_file->blocklens[i] - (sum - remainder));
-			break;
-		    }
-		}
-		total_io += sum;
-	    }
-	    /* byte starting point of last filetype */
-	    end_byte_off += (total_io - 1) / filetype_size * filetype_extent;
-	    /* number of bytes into last filetype */
-	    remainder = total_io % filetype_size;
-	    if (!remainder) {
-		for (i=flat_file->count - 1; i>=0; i--) {
-		    if (flat_file->blocklens[i]) break;
-		}
-		assert (i > -1);
-		end_byte_off += flat_file->indices[i] +
-		    flat_file->blocklens[i] - 1;
-		end_byte_off -= flat_file->indices[0];
-	    }
-	    else {
-		sum = 0;
-		for (i=0; i<flat_file->count; i++) {
-		    sum += flat_file->blocklens[i];
-		    if (sum >= remainder) {
-			end_byte_off += flat_file->indices[i] + 
-			    flat_file->blocklens[i] - sum + remainder - 1;
-			break;
-		    }
-		}
-		end_byte_off -= flat_file->indices[0];
-	    }
-	}
-	else {
-	    /* find starting byte of I/O (must be aligned with an etype) */
-	    /* byte starting point of starting filetype */		    
-	    st_byte_off = fd->disp + ((offset * etype_size) / filetype_size) *
-		filetype_extent;
-	    /* number of file viewable bytes into starting filetype */
-	    remainder = (etype_size * offset) % filetype_size;
-	    
-	    sum = 0;
-	    for (i=0; i<flat_file->count; i++) {
-		sum += flat_file->blocklens[i];
-		if (sum >= remainder) {
-		    if (sum == remainder)
-			st_byte_off += flat_file->indices[i+1];
-		    else
-			st_byte_off += flat_file->indices[i] +
-			    flat_file->blocklens[i] - sum + remainder;
-		    break;
-		}
-	    }
-	    
-	    /* find end byte of I/O (may be in middle of an etype) */
-	    /* byte starting point of last filetype */
-	    end_byte_off = fd->disp + (offset * etype_size + total_io) /
-		filetype_size * filetype_extent;
-	    /* number of bytes into last filetype */
-	    remainder = (offset * etype_size + total_io) % filetype_size;
-	    
-	    if (!remainder) {
-		/* the last non-zero off/len pair */
-		for (i=flat_file->count-1; i>=0; i--) {
-		    if (flat_file->blocklens[i]) break;
-		}
-		assert (i >= 0);
-		/* back up a whole filetype, and put back up to the
-		 * last byte of the last non-zero offlen pair */
-		/* end_byte_off = (end_byte_off - filetype_extent) +
-		    flat_file->indices[i] +
-		    flat_file->blocklens[i] - 1; */
-		/* equivalent of above commented out equation */
-		end_byte_off -= filetype_extent - flat_file->indices[i] -
-		    flat_file->blocklens[i] + 1;
-	    }
-	    else {
-		sum = 0;
-		for (i=0; i<flat_file->count; i++) {
-		    sum += flat_file->blocklens[i];
-		    if (sum >= remainder) {
-			end_byte_off += flat_file->indices[i] + 
-			    flat_file->blocklens[i] - sum + remainder - 1;
-			break;
-		    }
-		}
-	    }
-	}
-    }
-    
-    *st_offset  = st_byte_off;
-    *end_offset = end_byte_off;
-#ifdef DEBUG
-    printf ("st_offset = %lld\nend_offset = %lld\n",
-	    st_byte_off, end_byte_off);
-#endif
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5001, 0, NULL);
-#endif
-}
-
-/* wrapper function for ADIO_WriteStrided and ADIO_ReadStrided.  Used
- * by new 2 phase code to pass an arbitrary file type directly to
- * WriteStrided call without affecting existing code.  For the new 2
- * phase code, we really only need to set a custom_ftype, and we can
- * assume that this uses MPI_BYTE for the etype, and disp is 0 */
-void ADIOI_IOFiletype(ADIO_File fd, void *buf, int count,
-		      MPI_Datatype datatype, int file_ptr_type,
-		      ADIO_Offset offset, MPI_Datatype custom_ftype, 
-		      int rdwr, ADIO_Status *status, int *error_code)
-{
-    MPI_Datatype user_filetype;
-    MPI_Datatype user_etype;
-    ADIO_Offset user_disp;
-    int user_ind_wr_buffer_size;
-    int user_ind_rd_buffer_size;
-    int f_is_contig, m_is_contig;
-    int user_ds_read, user_ds_write;
-    MPI_Aint f_extent, lb;
-    MPI_Count f_size;
-    int f_ds_percent; /* size/extent */
-
-#ifdef AGGREGATION_PROFILE
-    if (rdwr == ADIOI_READ)
-	MPE_Log_event(5006, 0, NULL);
-    else
-	MPE_Log_event(5008, 0, NULL);
-#endif
-    MPI_Type_get_extent(custom_ftype, &lb, &f_extent);
-    MPI_Type_size_x(custom_ftype, &f_size);
-    f_ds_percent = 100 * f_size / f_extent;
-
-    /* temporarily store file view information */
-    user_filetype           = fd->filetype;
-    user_etype              = fd->etype;
-    user_disp               = fd->disp;
-    user_ds_read            = fd->hints->ds_read;
-    user_ds_write           = fd->hints->ds_write;
-    /* temporarily override the independent I/O datasieve buffer size */
-    user_ind_wr_buffer_size = fd->hints->ind_wr_buffer_size;
-    user_ind_rd_buffer_size = fd->hints->ind_rd_buffer_size;
-
-    /* set new values for temporary file view */
-    fd->filetype = custom_ftype;
-    fd->etype    = MPI_BYTE;
-    /* set new values for independent I/O datasieve buffer size */
-    fd->hints->ind_wr_buffer_size = fd->hints->cb_buffer_size;
-    fd->hints->ind_rd_buffer_size = fd->hints->cb_buffer_size;
-    /* decide whether or not to do datasieving */
-#ifdef DEBUG
-    printf ("f_ds_percent = %d cb_ds_threshold = %d\n", f_ds_percent,
-	    fd->hints->cb_ds_threshold);
-#endif
-    if (f_ds_percent >= fd->hints->cb_ds_threshold) {
-	fd->hints->ds_read = ADIOI_HINT_ENABLE;
-	fd->hints->ds_write = ADIOI_HINT_ENABLE;
-    }
-    else {
-	fd->hints->ds_read = ADIOI_HINT_DISABLE;
-	fd->hints->ds_write = ADIOI_HINT_DISABLE;
-    }
-
-    /* flatten the new filetype since the strided calls expect it to
-     * have been flattened in set file view.  in the two phase code,
-     * the datatype passed down should always be MPI_BYTE, and
-     * therefore contiguous, but just for completeness sake, we'll
-     * check the memory datatype anyway */
-    ADIOI_Datatype_iscontig(custom_ftype, &f_is_contig);
-    ADIOI_Datatype_iscontig(datatype, &m_is_contig);
-    if (!f_is_contig)
-	ADIOI_Flatten_datatype (custom_ftype);
-
-    /* make appropriate Read/Write calls.  Let ROMIO figure out file
-     * system specific stuff. */
-    if (f_is_contig && m_is_contig) {
-	fd->disp = 0;
-	if (rdwr == ADIOI_READ)
-	    ADIO_ReadContig(fd, buf, count, datatype, file_ptr_type, offset,
-			    status, error_code);
-	else
-	    ADIO_WriteContig(fd, buf, count, datatype, file_ptr_type, offset,
-			     status, error_code);
-    }
-    else {
-	fd->disp = offset;
-	if (rdwr == ADIOI_READ)
-	    ADIO_ReadStrided(fd, buf, count, datatype, file_ptr_type, 0,
-			     status, error_code);
-	else
-	    ADIO_WriteStrided(fd, buf, count, datatype, file_ptr_type, 0,
-			      status, error_code);
-    }
-
-    /* Delete flattened temporary filetype */
-    if (!f_is_contig)
-	ADIOI_Delete_flattened (custom_ftype);
-
-    /* restore the user specified file view to cover our tracks */
-    fd->filetype                  = user_filetype;
-    fd->etype                     = user_etype;
-    fd->disp                      = user_disp;
-    fd->hints->ds_read            = user_ds_read;
-    fd->hints->ds_write           = user_ds_write;
-    fd->hints->ind_wr_buffer_size = user_ind_wr_buffer_size;
-    fd->hints->ind_rd_buffer_size = user_ind_rd_buffer_size;
-#ifdef AGGREGATION_PROFILE
-    if (rdwr == ADIOI_READ)
-	MPE_Log_event (5007, 0, NULL);
-    else
-	MPE_Log_event (5009, 0, NULL);
-#endif
-}
-
-static void Exch_data_amounts (ADIO_File fd, int nprocs,
-			ADIO_Offset *client_comm_sz_arr,
-			ADIO_Offset *agg_comm_sz_arr,
-			int *client_alltoallw_counts,
-			int *agg_alltoallw_counts,
-			int *aggregators_done)
-{
-    int i;
-    int recv_idx;
-    MPI_Request *recv_requests;
-    MPI_Request *send_requests;
-    MPI_Status status;
-    MPI_Status *send_statuses;
-    /* Aggregators send amounts for data requested to clients */
-    if (fd->hints->cb_alltoall != ADIOI_HINT_DISABLE) {
-        MPI_Alltoall (client_comm_sz_arr, sizeof(ADIO_Offset), MPI_BYTE,
-		  agg_comm_sz_arr, sizeof(ADIO_Offset), MPI_BYTE,
-		  fd->comm);
-
-        if (fd->is_agg) {
-	    for (i=0; i<nprocs; i++)
-	        if (client_comm_sz_arr[i] > 0)
-		    client_alltoallw_counts[i] = 1;
-	        else
-		    client_alltoallw_counts[i] = 0;
-        }
-        *aggregators_done = 0;
-        for (i=0; i<nprocs; i++) {
-	    if (agg_comm_sz_arr[i] == -1)
-	        *aggregators_done = *aggregators_done + 1;
-	    else if (agg_comm_sz_arr[i] > 0)
-	        agg_alltoallw_counts[i] = 1;
-	    else
-	        agg_alltoallw_counts[i] = 0;
-        }
-    } else {
-        /* let's see if we can't reduce some communication as well as
-         * overlap some communication and work */
-
-        recv_requests = ADIOI_Malloc (fd->hints->cb_nodes * sizeof(MPI_Request));
-        /* post all receives - only receive from aggregators */
-        for (i = 0; i < fd->hints->cb_nodes; i++)
-	    MPI_Irecv (&agg_comm_sz_arr[fd->hints->ranklist[i]],
-		   sizeof(ADIO_Offset), MPI_BYTE, fd->hints->ranklist[i],
-		   AMT_TAG, fd->comm, &recv_requests[i]);
-
-        /* Barrier is needed here if we're worried about unexpected
-         * messages being dropped */
-        /* MPI_Barrier (fd->comm); */
-        send_requests = NULL;
-        if (fd->is_agg) {
-	    /* only aggregators send data */
-	    send_requests = ADIOI_Malloc (nprocs * sizeof(MPI_Request)); 
-
-	    /* post all sends */
-	    for (i = 0; i < nprocs; i++) {
-	        MPI_Isend (&client_comm_sz_arr[i], sizeof(ADIO_Offset),
-		       MPI_BYTE, i, AMT_TAG, fd->comm, &send_requests[i]);
-
-	        if (client_comm_sz_arr[i] > 0)
-		    client_alltoallw_counts[i] = 1;
-	        else
-		    client_alltoallw_counts[i] = 0;
-	    }
-        }
-
-        *aggregators_done = 0;
-        for (i=0; i < fd->hints->cb_nodes; i++) {
-	    MPI_Waitany (fd->hints->cb_nodes, recv_requests, &recv_idx, &status);
-	    if (agg_comm_sz_arr[fd->hints->ranklist[recv_idx]] == -1)
-	        *aggregators_done = *aggregators_done + 1;
-	    else if (agg_comm_sz_arr[fd->hints->ranklist[recv_idx]] > 0)
-	        agg_alltoallw_counts[fd->hints->ranklist[recv_idx]] = 1;
-	    else
-	        agg_alltoallw_counts[fd->hints->ranklist[recv_idx]] = 0;
-        }
-
-        ADIOI_Free (recv_requests);
-        if (fd->is_agg) {
-	    /* wait for all sends to complete */
-	    send_statuses = ADIOI_Malloc (nprocs * sizeof (MPI_Status));
-	    MPI_Waitall (nprocs, send_requests, send_statuses);
-	    ADIOI_Free (send_requests);
-	    ADIOI_Free (send_statuses);
-        }
-    }
-}
-
-static void post_aggregator_comm (MPI_Comm comm, int rw_type, 
-		           int nproc, void *cb_buf,
-			   MPI_Datatype *client_comm_dtype_arr,
-			   ADIO_Offset *client_comm_sz_arr,
-			   MPI_Request **requests_p,
-			   int *aggs_client_count_p)
-{
-    int aggs_client_count = 0;
-    MPI_Request *requests;
-    int i;
-
-#ifdef DEBUG
-    printf ("posting aggregator communication\n");
-#endif
-
-    for (i=0; i < nproc; i++)
-	if (client_comm_sz_arr[i] > 0)
-	    aggs_client_count++;
-#ifdef DEBUG
-    printf ("aggregator needs to talk to %d clients\n",
-	aggs_client_count);
-#endif
-    *aggs_client_count_p = aggs_client_count;
-    if (aggs_client_count) {
-	requests = (MPI_Request *)
-	    ADIOI_Malloc (aggs_client_count * sizeof(MPI_Request));
-	aggs_client_count = 0;
-#ifdef AGGREGATION_PROFILE
-	MPE_Log_event (5032, 0, NULL);
-#endif
-	for (i=0; i < nproc; i++) {
-	    if (client_comm_sz_arr[i] > 0) {
-		if (rw_type == ADIOI_WRITE)
-		    MPI_Irecv (cb_buf, 1, client_comm_dtype_arr[i], i,
-			       DATA_TAG, comm,
-			       &requests[aggs_client_count]);
-		else
-		    MPI_Isend (cb_buf, 1, client_comm_dtype_arr[i], i,
-			       DATA_TAG, comm,
-			       &requests[aggs_client_count]);
-
-		aggs_client_count++;
-	    }
-	}
-	*requests_p = requests;
-    }
-}
-
-static void post_client_comm (ADIO_File fd, int rw_type, 
-		       int agg_rank, void *buf,
-		       MPI_Datatype agg_comm_dtype,
-		       int agg_alltoallw_count,
-		       MPI_Request *request)
-{
-    if (agg_alltoallw_count) {
-	if (rw_type == ADIOI_READ)
-	    MPI_Irecv (buf, 1, agg_comm_dtype, agg_rank, DATA_TAG, fd->comm,
-		       request);
-	else
-	    MPI_Isend (buf, 1, agg_comm_dtype, agg_rank, DATA_TAG, fd->comm,
-		       request);
-    }
-}
-
-
-
diff --git a/3rd-party/romio321/adio/common/ad_iopen.c b/3rd-party/romio321/adio/common/ad_iopen.c
deleted file mode 100644
index 5deeaa2cdeb..00000000000
--- a/3rd-party/romio321/adio/common/ad_iopen.c
+++ /dev/null
@@ -1,21 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2002 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-
-void ADIO_ImmediateOpen(ADIO_File fd, int *error_code)
-{ 
-	MPI_Comm tmp_comm;
-	tmp_comm = fd->comm;
-	/* some file systems might try to be clever inside their open routine.
-	 * e.g. Blue Gene does a stat-and-broadcast */
-	fd->comm = MPI_COMM_SELF;
-	(*(fd->fns->ADIOI_xxx_Open))(fd, error_code);
-	fd->is_open = 1;
-	fd->comm = tmp_comm;
-
-} 
diff --git a/3rd-party/romio321/adio/common/ad_iread.c b/3rd-party/romio321/adio/common/ad_iread.c
deleted file mode 100644
index 74c342ab939..00000000000
--- a/3rd-party/romio321/adio/common/ad_iread.c
+++ /dev/null
@@ -1,91 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2004 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_SIGNAL_H
-#include <signal.h>
-#endif
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-#ifdef HAVE_AIO_H
-#include <aio.h>
-#endif
-#ifdef HAVE_SYS_AIO_H
-#include <sys/aio.h>
-#endif
-
-#include "mpiu_greq.h"
-
-#ifdef ROMIO_HAVE_WORKING_AIO
-/* ADIOI_GEN_IreadContig
- *
- * This code handles two distinct cases.  If ROMIO_HAVE_WORKING_AIO is not
- * defined, then I/O is performed in a blocking manner.  Otherwise we post
- * an asynchronous I/O operation using the appropriate aio routines. 
- *
- * In the aio case we rely on ADIOI_GEN_aio(), which is implemented in
- * common/ad_iwrite.c.
- */
-void ADIOI_GEN_IreadContig(ADIO_File fd, void *buf, int count, 
-			   MPI_Datatype datatype, int file_ptr_type,
-			   ADIO_Offset offset, MPI_Request *request,
-			   int *error_code)  
-{
-    MPI_Count len, typesize;
-    int aio_errno = 0;
-    static char myname[] = "ADIOI_GEN_IREADCONTIG";
-
-    MPI_Type_size_x(datatype, &typesize);
-    ADIOI_Assert((count * typesize) == ((ADIO_Offset)(unsigned)count * (ADIO_Offset)typesize));
-    len = count * typesize;
-
-    if (file_ptr_type == ADIO_INDIVIDUAL) offset = fd->fp_ind;
-    aio_errno = ADIOI_GEN_aio(fd, buf, len, offset, 0, request);
-    if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind += len;
-
-    fd->fp_sys_posn = -1;
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (aio_errno != 0) {
-	MPIO_ERR_CREATE_CODE_ERRNO(myname, aio_errno, error_code);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-    
-    *error_code = MPI_SUCCESS;
-}
-#endif
-
-/* Generic implementation of IreadStrided calls the blocking ReadStrided
- * immediately.
- */
-void ADIOI_GEN_IreadStrided(ADIO_File fd, void *buf, int count, 
-			    MPI_Datatype datatype, int file_ptr_type,
-			    ADIO_Offset offset, ADIO_Request *request,
-			    int *error_code)
-{
-    ADIO_Status status;
-    MPI_Count typesize;
-    MPI_Offset nbytes=0;
-
-    /* Call the blocking function.  It will create an error code
-     * if necessary.
-     */
-    ADIO_ReadStrided(fd, buf, count, datatype, file_ptr_type, 
-		     offset, &status, error_code);  
-
-    if (*error_code == MPI_SUCCESS) {
-	MPI_Type_size_x(datatype, &typesize);
-	nbytes = (MPI_Offset)count*(MPI_Offset)typesize;
-    }
-    MPIO_Completed_request_create(&fd, nbytes, error_code, request);
-}
diff --git a/3rd-party/romio321/adio/common/ad_iread_coll.c b/3rd-party/romio321/adio/common/ad_iread_coll.c
deleted file mode 100644
index 21c21e434f0..00000000000
--- a/3rd-party/romio321/adio/common/ad_iread_coll.c
+++ /dev/null
@@ -1,1346 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *  (C) 2014 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-#include "mpiu_greq.h"
-#include "mpioimpl.h"
-
-#ifdef USE_DBG_LOGGING
-  #define RDCOLL_DEBUG 1
-#endif
-#ifdef AGGREGATION_PROFILE
-#include "mpe.h"
-#endif
-
-#ifdef HAVE_MPI_GREQUEST_EXTENSIONS
-
-/* ADIOI_GEN_IreadStridedColl */
-struct ADIOI_GEN_IreadStridedColl_vars {
-    /* requests */
-    MPI_Request req_offset[2];  /* ADIOI_IRC_STATE_GEN_IREADSTRIDEDCOLL */
-    MPI_Request req_ind_io;     /* ADIOI_IRC_STATE_GEN_IREADSTRIDEDCOLL_INDIO */
-
-    /* parameters */
-    ADIO_File fd;
-    void *buf;
-    int count;
-    MPI_Datatype datatype;
-    int file_ptr_type;
-    ADIO_Offset offset;
-
-    /* stack variables */
-    ADIOI_Access *my_req;
-    /* array of nprocs structures, one for each other process in
-       whose file domain this process's request lies */
-
-    ADIOI_Access *others_req;
-    /* array of nprocs structures, one for each other process
-       whose request lies in this process's file domain. */
-
-    int nprocs;
-    int nprocs_for_coll;
-    int myrank;
-    int contig_access_count;
-    int interleave_count;
-    int buftype_is_contig;
-    int *count_my_req_per_proc;
-    int count_my_req_procs;
-    int count_others_req_procs;
-    ADIO_Offset start_offset;
-    ADIO_Offset end_offset;
-    ADIO_Offset orig_fp;
-    ADIO_Offset fd_size;
-    ADIO_Offset min_st_offset;
-    ADIO_Offset *offset_list;
-    ADIO_Offset *st_offsets;
-    ADIO_Offset *fd_start;
-    ADIO_Offset *fd_end;
-    ADIO_Offset *end_offsets;
-    ADIO_Offset *len_list;
-    int *buf_idx;
-};
-
-/* ADIOI_Iread_and_exch */
-struct ADIOI_Iread_and_exch_vars {
-    /* requests */
-    MPI_Request req1;   /* ADIOI_IRC_STATE_IREAD_AND_EXCH */
-    MPI_Request req2;   /* ADIOI_IRC_STATE_IREAD_AND_EXCH_L1_BEGIN */
-
-    /* parameters */
-    ADIO_File fd;
-    void *buf;
-    MPI_Datatype datatype;
-    int nprocs;
-    int myrank;
-    ADIOI_Access *others_req;
-    ADIO_Offset *offset_list;
-    ADIO_Offset *len_list;
-    int contig_access_count;
-    ADIO_Offset min_st_offset;
-    ADIO_Offset fd_size;
-    ADIO_Offset *fd_start;
-    ADIO_Offset *fd_end;
-    int *buf_idx;
-
-    /* stack variables */
-    int m;
-    int ntimes;
-    int max_ntimes;
-    int buftype_is_contig;
-    ADIO_Offset st_loc;
-    ADIO_Offset end_loc;
-    ADIO_Offset off;
-    ADIO_Offset done;
-    char *read_buf;
-    int *curr_offlen_ptr;
-    int *count;
-    int *send_size;
-    int *recv_size;
-    int *partial_send;
-    int *recd_from_proc;
-    int *start_pos;
-    /* Not convinced end_loc-st_loc couldn't be > int, so make these offsets*/
-    ADIO_Offset size;
-    ADIO_Offset real_size;
-    ADIO_Offset for_curr_iter;
-    ADIO_Offset for_next_iter;
-    ADIOI_Flatlist_node *flat_buf;
-    MPI_Aint buftype_extent;
-    int coll_bufsize;
-
-    /* next function to be called */
-    void (*next_fn)(ADIOI_NBC_Request *, int *);
-};
-
-/* ADIOI_R_Iexchange_data */
-struct ADIOI_R_Iexchange_data_vars {
-    /* requests */
-    MPI_Request req1;   /* ADIOI_IRC_STATE_R_IEXCHANGE_DATA */
-    MPI_Request *req2;  /* ADIOI_IRC_STATE_R_IEXCHANGE_DATA_RECV & FILL */
-
-    /* parameters */
-    ADIO_File fd;
-    void *buf;
-    ADIOI_Flatlist_node *flat_buf;
-    ADIO_Offset *offset_list;
-    ADIO_Offset *len_list;
-    int *send_size;
-    int *recv_size;
-    int *count;
-    int *start_pos;
-    int *partial_send;
-    int *recd_from_proc;
-    int nprocs;
-    int myrank;
-    int buftype_is_contig;
-    int contig_access_count;
-    ADIO_Offset min_st_offset;
-    ADIO_Offset fd_size;
-    ADIO_Offset *fd_start;
-    ADIO_Offset *fd_end;
-    ADIOI_Access *others_req;
-    int iter;
-    MPI_Aint buftype_extent;
-    int *buf_idx;
-
-    /* stack variables */
-    int nprocs_recv;
-    int nprocs_send;
-    char **recv_buf;
-
-    /* next function to be called */
-    void (*next_fn)(ADIOI_NBC_Request *, int *);
-};
-
-
-void ADIOI_Fill_user_buffer(ADIO_File fd, void *buf, ADIOI_Flatlist_node
-                   *flat_buf, char **recv_buf, ADIO_Offset
-                   *offset_list, ADIO_Offset *len_list,
-                   unsigned *recv_size,
-                   MPI_Request *requests, MPI_Status *statuses,
-                   int *recd_from_proc, int nprocs,
-                   int contig_access_count,
-                   ADIO_Offset min_st_offset,
-                   ADIO_Offset fd_size, ADIO_Offset *fd_start,
-                   ADIO_Offset *fd_end,
-                   MPI_Aint buftype_extent);
-
-/* prototypes of functions used for nonblocking collective reads only. */
-static void ADIOI_GEN_IreadStridedColl_inter(ADIOI_NBC_Request *, int *);
-static void ADIOI_GEN_IreadStridedColl_indio(ADIOI_NBC_Request *, int *);
-static void ADIOI_GEN_IreadStridedColl_read(ADIOI_NBC_Request *, int *);
-static void ADIOI_GEN_IreadStridedColl_free(ADIOI_NBC_Request *, int *);
-static void ADIOI_GEN_IreadStridedColl_fini(ADIOI_NBC_Request *, int *);
-
-static void ADIOI_Iread_and_exch(ADIOI_NBC_Request *, int *);
-static void ADIOI_Iread_and_exch_l1_begin(ADIOI_NBC_Request *, int *);
-static void ADIOI_Iread_and_exch_l1_end(ADIOI_NBC_Request *, int *);
-static void ADIOI_Iread_and_exch_reset(ADIOI_NBC_Request *, int *);
-static void ADIOI_Iread_and_exch_l2_begin(ADIOI_NBC_Request *, int *);
-static void ADIOI_Iread_and_exch_l2_end(ADIOI_NBC_Request *, int *);
-static void ADIOI_Iread_and_exch_fini(ADIOI_NBC_Request *, int *);
-
-static void ADIOI_R_Iexchange_data(ADIOI_NBC_Request *, int *);
-static void ADIOI_R_Iexchange_data_recv(ADIOI_NBC_Request *, int *);
-static void ADIOI_R_Iexchange_data_fill(ADIOI_NBC_Request *, int *);
-static void ADIOI_R_Iexchange_data_fini(ADIOI_NBC_Request *, int *);
-
-static MPIX_Grequest_class ADIOI_GEN_greq_class = 0;
-static int ADIOI_GEN_irc_query_fn(void *extra_state, MPI_Status *status);
-static int ADIOI_GEN_irc_free_fn(void *extra_state);
-static int ADIOI_GEN_irc_poll_fn(void *extra_state, MPI_Status *status);
-static int ADIOI_GEN_irc_wait_fn(int count, void **array_of_states,
-                                 double timeout, MPI_Status *status);
-
-
-/* Nonblocking version of ADIOI_GEN_ReadStridedColl() */
-void ADIOI_GEN_IreadStridedColl(ADIO_File fd, void *buf, int count,
-                   MPI_Datatype datatype, int file_ptr_type,
-                   ADIO_Offset offset, MPI_Request *request,
-                   int *error_code)
-{
-    /* Uses a generalized version of the extended two-phase method described
-       in "An Extended Two-Phase Method for Accessing Sections of
-       Out-of-Core Arrays", Rajeev Thakur and Alok Choudhary,
-       Scientific Programming, (5)4:301--317, Winter 1996.
-       http://www.mcs.anl.gov/home/thakur/ext2ph.ps */
-
-    ADIOI_NBC_Request *nbc_req = NULL;
-    ADIOI_GEN_IreadStridedColl_vars *vars = NULL;
-    int nprocs, myrank;
-#ifdef RDCOLL_DEBUG
-    int i;
-#endif
-
-    /* FIXME: need an implementation of ADIOI_IOIstridedColl
-    if (fd->hints->cb_pfr != ADIOI_HINT_DISABLE) {
-        ADIOI_IOIstridedColl(fd, buf, count, ADIOI_READ, datatype,
-                             file_ptr_type, offset, request, error_code);
-        return;
-    }
-    */
-
-    /* top-level struct keeping the status of function progress */
-    nbc_req = (ADIOI_NBC_Request *)ADIOI_Calloc(1, sizeof(ADIOI_NBC_Request));
-    nbc_req->rdwr = ADIOI_READ;
-
-    /* create a generalized request */
-    if (ADIOI_GEN_greq_class == 0) {
-        MPIX_Grequest_class_create(ADIOI_GEN_irc_query_fn,
-                ADIOI_GEN_irc_free_fn, MPIU_Greq_cancel_fn,
-                ADIOI_GEN_irc_poll_fn, ADIOI_GEN_irc_wait_fn,
-                &ADIOI_GEN_greq_class);
-    }
-    MPIX_Grequest_class_allocate(ADIOI_GEN_greq_class, nbc_req, request);
-    memcpy(&nbc_req->req, request, sizeof(MPI_Request));
-
-    /* create a struct for parameters and variables */
-    vars = (ADIOI_GEN_IreadStridedColl_vars *)ADIOI_Calloc(
-            1, sizeof(ADIOI_GEN_IreadStridedColl_vars));
-    nbc_req->data.rd.rsc_vars = vars;
-
-    /* save the parameters */
-    vars->fd = fd;
-    vars->buf = buf;
-    vars->count = count;
-    vars->datatype = datatype;
-    vars->file_ptr_type = file_ptr_type;
-    vars->offset = offset;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-    vars->nprocs = nprocs;
-    vars->myrank = myrank;
-
-    /* number of aggregators, cb_nodes, is stored in the hints */
-    vars->nprocs_for_coll = fd->hints->cb_nodes;
-    vars->orig_fp = fd->fp_ind;
-
-    /* only check for interleaving if cb_read isn't disabled */
-    if (fd->hints->cb_read != ADIOI_HINT_DISABLE) {
-        /* For this process's request, calculate the list of offsets and
-           lengths in the file and determine the start and end offsets. */
-
-        /* Note: end_offset points to the last byte-offset that will be accessed.
-           e.g., if start_offset=0 and 100 bytes to be read, end_offset=99*/
-
-        ADIOI_Calc_my_off_len(fd, count, datatype, file_ptr_type, offset,
-                              &vars->offset_list, &vars->len_list,
-                              &vars->start_offset, &vars->end_offset,
-                              &vars->contig_access_count);
-
-#ifdef RDCOLL_DEBUG
-        for (i = 0; i < vars->contig_access_count; i++) {
-            DBG_FPRINTF(stderr, "rank %d  off %lld  len %lld\n",
-                        myrank, vars->offset_list[i], vars->len_list[i]);
-        }
-#endif
-
-        /* each process communicates its start and end offsets to other
-           processes. The result is an array each of start and end offsets
-           stored in order of process rank. */
-
-        vars->st_offsets = (ADIO_Offset *)ADIOI_Malloc(nprocs*sizeof(ADIO_Offset));
-        vars->end_offsets = (ADIO_Offset *)ADIOI_Malloc(nprocs*sizeof(ADIO_Offset));
-
-        *error_code = MPI_Iallgather(&vars->start_offset, 1, ADIO_OFFSET,
-                                     vars->st_offsets, 1, ADIO_OFFSET,
-                                     fd->comm, &vars->req_offset[0]);
-        if (*error_code != MPI_SUCCESS) return;
-        *error_code = MPI_Iallgather(&vars->end_offset, 1, ADIO_OFFSET,
-                                     vars->end_offsets, 1, ADIO_OFFSET,
-                                     fd->comm, &vars->req_offset[1]);
-
-        nbc_req->data.rd.state = ADIOI_IRC_STATE_GEN_IREADSTRIDEDCOLL;
-        return;
-    }
-
-    ADIOI_GEN_IreadStridedColl_indio(nbc_req, error_code);
-}
-
-static void ADIOI_GEN_IreadStridedColl_inter(ADIOI_NBC_Request *nbc_req,
-                                             int *error_code)
-{
-    ADIOI_GEN_IreadStridedColl_vars *vars = nbc_req->data.rd.rsc_vars;
-    int nprocs = vars->nprocs;
-    ADIO_Offset *st_offsets = vars->st_offsets;
-    ADIO_Offset *end_offsets = vars->end_offsets;
-    int i, interleave_count = 0;
-
-    /* are the accesses of different processes interleaved? */
-    for (i = 1; i < nprocs; i++)
-        if ((st_offsets[i] < end_offsets[i-1]) &&
-            (st_offsets[i] <= end_offsets[i]))
-            interleave_count++;
-    /* This is a rudimentary check for interleaving, but should suffice
-       for the moment. */
-
-    vars->interleave_count = interleave_count;
-
-    ADIOI_GEN_IreadStridedColl_indio(nbc_req, error_code);
-}
-
-static void ADIOI_GEN_IreadStridedColl_indio(ADIOI_NBC_Request *nbc_req,
-                                             int *error_code)
-{
-    ADIOI_GEN_IreadStridedColl_vars *vars = nbc_req->data.rd.rsc_vars;
-    ADIOI_Icalc_others_req_vars *cor_vars = NULL;
-    ADIO_File fd = vars->fd;
-    void *buf;
-    int count, file_ptr_type;
-    MPI_Datatype datatype = vars->datatype;
-    ADIO_Offset offset;
-    int filetype_is_contig;
-    ADIO_Offset off;
-    int nprocs;
-
-    ADIOI_Datatype_iscontig(datatype, &vars->buftype_is_contig);
-
-    if (fd->hints->cb_read == ADIOI_HINT_DISABLE
-    || (!vars->interleave_count && (fd->hints->cb_read == ADIOI_HINT_AUTO)))
-    {
-        buf = vars->buf;
-        count = vars->count;
-        file_ptr_type = vars->file_ptr_type;
-        offset = vars->offset;
-
-        /* don't do aggregation */
-        if (fd->hints->cb_read != ADIOI_HINT_DISABLE) {
-            ADIOI_Free(vars->offset_list);
-            ADIOI_Free(vars->len_list);
-            ADIOI_Free(vars->st_offsets);
-            ADIOI_Free(vars->end_offsets);
-        }
-
-        fd->fp_ind = vars->orig_fp;
-        ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-
-#if defined(ROMIO_RUN_ON_LINUX) && !defined(HAVE_AIO_LITE_H)
-        /* NOTE: This is currently a workaround to avoid weird errors, e.g.,
-         * stack fault, occurred on Linux. When the host OS is Linux and
-         * aio-lite is not used, a blocking ADIO function is used here.
-         * See https://trac.mpich.org/projects/mpich/ticket/2201. */
-        MPI_Status status;
-        if (vars->buftype_is_contig && filetype_is_contig) {
-            if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-                off = fd->disp + (fd->etype_size) * offset;
-                ADIO_ReadContig(fd, buf, count, datatype, ADIO_EXPLICIT_OFFSET,
-                                off, &status, error_code);
-            }
-            else ADIO_ReadContig(fd, buf, count, datatype, ADIO_INDIVIDUAL,
-                                 0, &status, error_code);
-        }
-        else {
-            ADIO_ReadStrided(fd, buf, count, datatype, file_ptr_type,
-                             offset, &status, error_code);
-        }
-        ADIOI_GEN_IreadStridedColl_fini(nbc_req, error_code);
-#else
-        if (vars->buftype_is_contig && filetype_is_contig) {
-            if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-                off = fd->disp + (fd->etype_size) * offset;
-                ADIO_IreadContig(fd, buf, count, datatype, ADIO_EXPLICIT_OFFSET,
-                                 off, &vars->req_ind_io, error_code);
-            }
-            else ADIO_IreadContig(fd, buf, count, datatype, ADIO_INDIVIDUAL,
-                                  0, &vars->req_ind_io, error_code);
-        }
-        else {
-            ADIO_IreadStrided(fd, buf, count, datatype, file_ptr_type,
-                              offset, &vars->req_ind_io, error_code);
-        }
-
-        nbc_req->data.rd.state = ADIOI_IRC_STATE_GEN_IREADSTRIDEDCOLL_INDIO;
-#endif
-        return;
-    }
-
-    nprocs = vars->nprocs;
-
-    /* We're going to perform aggregation of I/O.  Here we call
-     * ADIOI_Calc_file_domains() to determine what processes will handle I/O
-     * to what regions.  We pass nprocs_for_coll into this function; it is
-     * used to determine how many processes will perform I/O, which is also
-     * the number of regions into which the range of bytes must be divided.
-     * These regions are called "file domains", or FDs.
-     *
-     * When this function returns, fd_start, fd_end, fd_size, and
-     * min_st_offset will be filled in.  fd_start holds the starting byte
-     * location for each file domain.  fd_end holds the ending byte location.
-     * min_st_offset holds the minimum byte location that will be accessed.
-     *
-     * Both fd_start[] and fd_end[] are indexed by an aggregator number; this
-     * needs to be mapped to an actual rank in the communicator later.
-     *
-     */
-    ADIOI_Calc_file_domains(vars->st_offsets, vars->end_offsets, nprocs,
-                vars->nprocs_for_coll, &vars->min_st_offset,
-                &vars->fd_start, &vars->fd_end,
-                fd->hints->min_fdomain_size, &vars->fd_size,
-                fd->hints->striping_unit);
-
-    /* calculate where the portions of the access requests of this process
-     * are located in terms of the file domains.  this could be on the same
-     * process or on other processes.  this function fills in:
-     * count_my_req_procs - number of processes (including this one) for which
-     *     this process has requests in their file domain
-     * count_my_req_per_proc - count of requests for each process, indexed
-     *     by rank of the process
-     * my_req[] - array of data structures describing the requests to be
-     *     performed by each process (including self).  indexed by rank.
-     * buf_idx[] - array of locations into which data can be directly moved;
-     *     this is only valid for contiguous buffer case
-     */
-    ADIOI_Calc_my_req(fd, vars->offset_list, vars->len_list,
-              vars->contig_access_count, vars->min_st_offset,
-              vars->fd_start, vars->fd_end, vars->fd_size,
-              nprocs, &vars->count_my_req_procs,
-              &vars->count_my_req_per_proc, &vars->my_req,
-              &vars->buf_idx);
-
-    /* perform a collective communication in order to distribute the
-     * data calculated above.  fills in the following:
-     * count_others_req_procs - number of processes (including this
-     *     one) which have requests in this process's file domain.
-     * count_others_req_per_proc[] - number of separate contiguous
-     *     requests from proc i lie in this process's file domain.
-     */
-
-    cor_vars = (ADIOI_Icalc_others_req_vars *)ADIOI_Calloc(
-            1, sizeof(ADIOI_Icalc_others_req_vars));
-    nbc_req->cor_vars = cor_vars;
-    cor_vars->fd = vars->fd;
-    cor_vars->count_my_req_procs = vars->count_my_req_procs;
-    cor_vars->count_my_req_per_proc = vars->count_my_req_per_proc;
-    cor_vars->my_req = vars->my_req;
-    cor_vars->nprocs = vars->nprocs;
-    cor_vars->myrank = vars->myrank;
-    cor_vars->count_others_req_procs_ptr = &vars->count_others_req_procs;
-    cor_vars->others_req_ptr = &vars->others_req;
-    cor_vars->next_fn = ADIOI_GEN_IreadStridedColl_read;
-
-    ADIOI_Icalc_others_req(nbc_req, error_code);
-}
-
-static void ADIOI_GEN_IreadStridedColl_read(ADIOI_NBC_Request *nbc_req,
-                                            int *error_code)
-{
-    ADIOI_GEN_IreadStridedColl_vars *vars = nbc_req->data.rd.rsc_vars;
-    ADIOI_Iread_and_exch_vars *rae_vars = NULL;
-    ADIOI_Access *my_req = vars->my_req;
-    int nprocs = vars->nprocs;
-    int i;
-
-    /* my_req[] and count_my_req_per_proc aren't needed at this point, so
-     * let's free the memory
-     */
-    ADIOI_Free(vars->count_my_req_per_proc);
-    for (i = 0; i < nprocs; i++) {
-        if (my_req[i].count) {
-            ADIOI_Free(my_req[i].offsets);
-            ADIOI_Free(my_req[i].lens);
-        }
-    }
-    ADIOI_Free(my_req);
-
-    /* read data in sizes of no more than ADIOI_Coll_bufsize,
-     * communicate, and fill user buf.
-     */
-    rae_vars = (ADIOI_Iread_and_exch_vars *)ADIOI_Calloc(
-            1, sizeof(ADIOI_Iread_and_exch_vars));
-    nbc_req->data.rd.rae_vars = rae_vars;
-    rae_vars->fd = vars->fd;
-    rae_vars->buf = vars->buf;
-    rae_vars->datatype = vars->datatype;
-    rae_vars->nprocs = vars->nprocs;
-    rae_vars->myrank = vars->myrank;
-    rae_vars->others_req = vars->others_req;
-    rae_vars->offset_list = vars->offset_list;
-    rae_vars->len_list = vars->len_list;
-    rae_vars->contig_access_count = vars->contig_access_count;
-    rae_vars->min_st_offset = vars->min_st_offset;
-    rae_vars->fd_size = vars->fd_size;
-    rae_vars->fd_start = vars->fd_start;
-    rae_vars->fd_end = vars->fd_end;
-    rae_vars->buf_idx = vars->buf_idx;
-    rae_vars->next_fn = ADIOI_GEN_IreadStridedColl_free;
-
-    ADIOI_Iread_and_exch(nbc_req, error_code);
-}
-
-static void ADIOI_GEN_IreadStridedColl_free(ADIOI_NBC_Request *nbc_req,
-                                            int *error_code)
-{
-    ADIOI_GEN_IreadStridedColl_vars *vars = nbc_req->data.rd.rsc_vars;
-    ADIO_File fd = vars->fd;
-    MPI_Datatype datatype = vars->datatype;
-    ADIOI_Access *others_req = vars->others_req;
-    int nprocs = vars->nprocs;
-    int i;
-
-    if (!vars->buftype_is_contig) ADIOI_Delete_flattened(datatype);
-
-    /* free all memory allocated for collective I/O */
-    for (i = 0; i < nprocs; i++) {
-        if (others_req[i].count) {
-            ADIOI_Free(others_req[i].offsets);
-            ADIOI_Free(others_req[i].lens);
-            ADIOI_Free(others_req[i].mem_ptrs);
-        }
-    }
-    ADIOI_Free(others_req);
-
-    ADIOI_Free(vars->buf_idx);
-    ADIOI_Free(vars->offset_list);
-    ADIOI_Free(vars->len_list);
-    ADIOI_Free(vars->st_offsets);
-    ADIOI_Free(vars->end_offsets);
-    ADIOI_Free(vars->fd_start);
-    ADIOI_Free(vars->fd_end);
-
-    fd->fp_sys_posn = -1;   /* set it to null. */
-
-    ADIOI_GEN_IreadStridedColl_fini(nbc_req, error_code);
-}
-
-static void ADIOI_GEN_IreadStridedColl_fini(ADIOI_NBC_Request *nbc_req,
-                                            int *error_code)
-{
-    ADIOI_GEN_IreadStridedColl_vars *vars = nbc_req->data.rd.rsc_vars;
-    MPI_Count size;
-
-    /* This is a temporary way of filling in status. The right way is to
-       keep track of how much data was actually read and placed in buf
-       during collective I/O. */
-    MPI_Type_size_x(vars->datatype, &size);
-    nbc_req->nbytes = size * vars->count;
-
-    /* free the struct for parameters and variables */
-    if (nbc_req->data.rd.rsc_vars) {
-        ADIOI_Free(nbc_req->data.rd.rsc_vars);
-        nbc_req->data.rd.rsc_vars = NULL;
-    }
-
-    /* make the request complete */
-    *error_code = MPI_Grequest_complete(nbc_req->req);
-    nbc_req->data.rd.state = ADIOI_IRC_STATE_COMPLETE;
-}
-
-
-static void ADIOI_Iread_and_exch(ADIOI_NBC_Request *nbc_req, int *error_code)
-{
-    ADIOI_Iread_and_exch_vars *vars = nbc_req->data.rd.rae_vars;
-    ADIO_File fd = vars->fd;
-    MPI_Datatype datatype = vars->datatype;
-    int nprocs = vars->nprocs;
-    MPI_Aint lb;
-    ADIOI_Access *others_req = vars->others_req;
-
-    /* Read in sizes of no more than coll_bufsize, an info parameter.
-       Send data to appropriate processes.
-       Place recd. data in user buf.
-       The idea is to reduce the amount of extra memory required for
-       collective I/O. If all data were read all at once, which is much
-       easier, it would require temp space more than the size of user_buf,
-       which is often unacceptable. For example, to read a distributed
-       array from a file, where each local array is 8Mbytes, requiring
-       at least another 8Mbytes of temp space is unacceptable. */
-
-    int i, j;
-    ADIO_Offset st_loc = -1, end_loc = -1;
-    int coll_bufsize;
-
-    *error_code = MPI_SUCCESS;  /* changed below if error */
-    /* only I/O errors are currently reported */
-
-    /* calculate the number of reads of size coll_bufsize
-       to be done by each process and the max among all processes.
-       That gives the no. of communication phases as well.
-       coll_bufsize is obtained from the hints object. */
-
-    coll_bufsize = fd->hints->cb_buffer_size;
-    vars->coll_bufsize = coll_bufsize;
-
-    /* grab some initial values for st_loc and end_loc */
-    for (i = 0; i < nprocs; i++) {
-        if (others_req[i].count) {
-            st_loc = others_req[i].offsets[0];
-            end_loc = others_req[i].offsets[0];
-            break;
-        }
-    }
-
-    /* now find the real values */
-    for (i = 0; i < nprocs; i++)
-        for (j = 0; j < others_req[i].count; j++) {
-            st_loc = ADIOI_MIN(st_loc, others_req[i].offsets[j]);
-            end_loc = ADIOI_MAX(end_loc, (others_req[i].offsets[j]
-                          + others_req[i].lens[j] - 1));
-        }
-
-    vars->st_loc = st_loc;
-    vars->end_loc = end_loc;
-
-    /* calculate ntimes, the number of times this process must perform I/O
-     * operations in order to complete all the requests it has received.
-     * the need for multiple I/O operations comes from the restriction that
-     * we only use coll_bufsize bytes of memory for internal buffering.
-     */
-    if ((st_loc == -1) && (end_loc == -1)) {
-        /* this process does no I/O. */
-        vars->ntimes = 0;
-    }
-    else {
-        /* ntimes=ceiling_div(end_loc - st_loc + 1, coll_bufsize)*/
-        vars->ntimes = (int)((end_loc - st_loc + coll_bufsize) / coll_bufsize);
-    }
-
-    *error_code = MPI_Iallreduce(&vars->ntimes, &vars->max_ntimes, 1, MPI_INT,
-                                 MPI_MAX, fd->comm, &vars->req1);
-
-    vars->read_buf = fd->io_buf;  /* Allocated at open time */
-
-    vars->curr_offlen_ptr = (int *)ADIOI_Calloc(nprocs, sizeof(int));
-    /* its use is explained below. calloc initializes to 0. */
-
-    vars->count = (int *)ADIOI_Malloc(nprocs * sizeof(int));
-    /* to store count of how many off-len pairs per proc are satisfied
-       in an iteration. */
-
-    vars->partial_send = (int *)ADIOI_Calloc(nprocs, sizeof(int));
-    /* if only a portion of the last off-len pair is sent to a process
-       in a particular iteration, the length sent is stored here.
-       calloc initializes to 0. */
-
-    vars->send_size = (int *)ADIOI_Malloc(nprocs * sizeof(int));
-    /* total size of data to be sent to each proc. in an iteration */
-
-    vars->recv_size = (int *)ADIOI_Malloc(nprocs * sizeof(int));
-    /* total size of data to be recd. from each proc. in an iteration.
-       Of size nprocs so that I can use MPI_Alltoall later. */
-
-    vars->recd_from_proc = (int *)ADIOI_Calloc(nprocs, sizeof(int));
-    /* amount of data recd. so far from each proc. Used in
-       ADIOI_Fill_user_buffer. initialized to 0 here. */
-
-    vars->start_pos = (int *)ADIOI_Malloc(nprocs*sizeof(int));
-    /* used to store the starting value of curr_offlen_ptr[i] in
-       this iteration */
-
-    ADIOI_Datatype_iscontig(datatype, &vars->buftype_is_contig);
-    if (!vars->buftype_is_contig) {
-	vars->flat_buf = ADIOI_Flatten_and_find(datatype);
-    }
-    MPI_Type_get_extent(datatype, &lb, &vars->buftype_extent);
-
-    vars->done = 0;
-    vars->off = st_loc;
-    vars->for_curr_iter = vars->for_next_iter = 0;
-
-    /* set the state to wait until MPI_Ialltoall finishes. */
-    nbc_req->data.rd.state = ADIOI_IRC_STATE_IREAD_AND_EXCH;
-}
-
-static void ADIOI_Iread_and_exch_l1_begin(ADIOI_NBC_Request *nbc_req,
-                                          int *error_code)
-{
-    ADIOI_Iread_and_exch_vars *vars = nbc_req->data.rd.rae_vars;
-    ADIO_File fd;
-    int nprocs;
-    ADIOI_Access *others_req;
-
-    int i, j;
-    ADIO_Offset real_off, req_off;
-    char *read_buf;
-    int *curr_offlen_ptr, *count, *send_size;
-    int *partial_send, *start_pos;
-    ADIO_Offset size, real_size, for_next_iter;
-    int req_len, flag;
-
-    ADIOI_R_Iexchange_data_vars *red_vars = NULL;
-
-    /* loop exit condition */
-    if (vars->m >= vars->ntimes) {
-        ADIOI_Iread_and_exch_reset(nbc_req, error_code);
-        return;
-    }
-
-    fd = vars->fd;
-    nprocs = vars->nprocs;
-    others_req = vars->others_req;
-
-    read_buf = vars->read_buf;
-    curr_offlen_ptr = vars->curr_offlen_ptr;
-    count = vars->count;
-    send_size = vars->send_size;
-    partial_send = vars->partial_send;
-    start_pos = vars->start_pos;
-
-    /* read buf of size coll_bufsize (or less) */
-    /* go through all others_req and check if any are satisfied
-       by the current read */
-
-    /* since MPI guarantees that displacements in filetypes are in
-       monotonically nondecreasing order, I can maintain a pointer
-       (curr_offlen_ptr) to
-       current off-len pair for each process in others_req and scan
-       further only from there. There is still a problem of filetypes
-       such as:  (1, 2, 3 are not process nos. They are just numbers for
-       three chunks of data, specified by a filetype.)
-
-       1  -------!--
-       2    -----!----
-       3       --!-----
-
-       where ! indicates where the current read_size limitation cuts
-       through the filetype.  I resolve this by reading up to !, but
-       filling the communication buffer only for 1. I copy the portion
-       left over for 2 into a tmp_buf for use in the next
-       iteration. i.e., 2 and 3 will be satisfied in the next
-       iteration. This simplifies filling in the user's buf at the
-       other end, as only one off-len pair with incomplete data
-       will be sent. I also don't need to send the individual
-       offsets and lens along with the data, as the data is being
-       sent in a particular order. */
-
-    /* off = start offset in the file for the data actually read in
-             this iteration
-       size = size of data read corresponding to off
-       real_off = off minus whatever data was retained in memory from
-             previous iteration for cases like 2, 3 illustrated above
-       real_size = size plus the extra corresponding to real_off
-       req_off = off in file for a particular contiguous request
-                 minus what was satisfied in previous iteration
-       req_size = size corresponding to req_off */
-
-    size = ADIOI_MIN((unsigned)vars->coll_bufsize,
-                     vars->end_loc - vars->st_loc + 1 - vars->done);
-    real_off = vars->off - vars->for_curr_iter;
-    real_size = size + vars->for_curr_iter;
-
-    vars->size = size;
-    vars->real_size = real_size;
-
-    for (i = 0; i < nprocs; i++) count[i] = send_size[i] = 0;
-    for_next_iter = 0;
-
-    for (i = 0; i < nprocs; i++) {
-#ifdef RDCOLL_DEBUG
-        DBG_FPRINTF(stderr, "rank %d, i %d, others_count %d\n",
-                    vars->myrank, i, others_req[i].count);
-#endif
-        if (others_req[i].count) {
-            start_pos[i] = curr_offlen_ptr[i];
-            for (j = curr_offlen_ptr[i]; j < others_req[i].count; j++) {
-                if (partial_send[i]) {
-                    /* this request may have been partially
-                       satisfied in the previous iteration. */
-                    req_off = others_req[i].offsets[j] + partial_send[i];
-                    req_len = others_req[i].lens[j] - partial_send[i];
-                    partial_send[i] = 0;
-                    /* modify the off-len pair to reflect this change */
-                    others_req[i].offsets[j] = req_off;
-                    others_req[i].lens[j] = req_len;
-                }
-                else {
-                    req_off = others_req[i].offsets[j];
-                    req_len = others_req[i].lens[j];
-                }
-                if (req_off < real_off + real_size) {
-                    count[i]++;
-                    ADIOI_Assert((((ADIO_Offset)(MPIU_Upint)read_buf) + req_off - real_off) == (ADIO_Offset)(MPIU_Upint)(read_buf + req_off - real_off));
-                    MPI_Get_address(read_buf + req_off - real_off,
-                                &(others_req[i].mem_ptrs[j]));
-                    ADIOI_Assert((real_off + real_size - req_off) == (int)(real_off + real_size - req_off));
-                    send_size[i] += (int)(ADIOI_MIN(real_off + real_size - req_off,
-                                                    (ADIO_Offset)(unsigned)req_len));
-
-                    if (real_off + real_size - req_off < (ADIO_Offset)(unsigned)req_len) {
-                        partial_send[i] = (int)(real_off + real_size - req_off);
-                        if ((j+1 < others_req[i].count) &&
-                            (others_req[i].offsets[j+1] < real_off + real_size)) {
-                            /* this is the case illustrated in the
-                               figure above. */
-                            for_next_iter = ADIOI_MAX(for_next_iter,
-                                    real_off + real_size - others_req[i].offsets[j+1]);
-                            /* max because it must cover requests
-                               from different processes */
-                        }
-                        break;
-                    }
-                }
-                else break;
-            }
-            curr_offlen_ptr[i] = j;
-        }
-    }
-    vars->for_next_iter = for_next_iter;
-
-    flag = 0;
-    for (i = 0; i < nprocs; i++)
-        if (count[i]) flag = 1;
-
-    /* create a struct for ADIOI_R_Iexchange_data() */
-    red_vars = (ADIOI_R_Iexchange_data_vars *)ADIOI_Calloc(
-            1, sizeof(ADIOI_R_Iexchange_data_vars));
-    nbc_req->data.rd.red_vars = red_vars;
-    red_vars->fd = vars->fd;
-    red_vars->buf = vars->buf;
-    red_vars->flat_buf = vars->flat_buf;
-    red_vars->offset_list = vars->offset_list;
-    red_vars->len_list = vars->len_list;
-    red_vars->send_size = vars->send_size;
-    red_vars->recv_size = vars->recv_size;
-    red_vars->count = vars->count;
-    red_vars->start_pos = vars->start_pos;
-    red_vars->partial_send = vars->partial_send;
-    red_vars->recd_from_proc = vars->recd_from_proc;
-    red_vars->nprocs = vars->nprocs;
-    red_vars->myrank = vars->myrank;
-    red_vars->buftype_is_contig = vars->buftype_is_contig;
-    red_vars->contig_access_count = vars->contig_access_count;
-    red_vars->min_st_offset = vars->min_st_offset;
-    red_vars->fd_size = vars->fd_size;
-    red_vars->fd_start = vars->fd_start;
-    red_vars->fd_end = vars->fd_end;
-    red_vars->others_req = vars->others_req;
-    red_vars->iter = vars->m;
-    red_vars->buftype_extent = vars->buftype_extent;
-    red_vars->buf_idx = vars->buf_idx;
-    red_vars->next_fn = ADIOI_Iread_and_exch_l1_end;
-
-    if (flag) {
-        ADIOI_Assert(size == (int)size);
-#if defined(ROMIO_RUN_ON_LINUX) && !defined(HAVE_AIO_LITE_H)
-        MPI_Status status;
-        ADIO_ReadContig(fd, read_buf+vars->for_curr_iter, (int)size,
-                        MPI_BYTE, ADIO_EXPLICIT_OFFSET, vars->off,
-                        &status, error_code);
-#else
-        ADIO_IreadContig(fd, read_buf+vars->for_curr_iter, (int)size,
-                         MPI_BYTE, ADIO_EXPLICIT_OFFSET, vars->off,
-                         &vars->req2, error_code);
-
-        nbc_req->data.rd.state = ADIOI_IRC_STATE_IREAD_AND_EXCH_L1_BEGIN;
-        return;
-#endif
-    }
-
-    ADIOI_R_Iexchange_data(nbc_req, error_code);
-}
-
-static void ADIOI_Iread_and_exch_l1_end(ADIOI_NBC_Request *nbc_req,
-                                        int *error_code)
-{
-    ADIOI_Iread_and_exch_vars *vars = nbc_req->data.rd.rae_vars;
-    ADIO_File fd = vars->fd;
-    ADIO_Offset size = vars->size;
-    ADIO_Offset real_size = vars->real_size;
-    ADIO_Offset for_next_iter = vars->for_next_iter;
-    char *read_buf = vars->read_buf;
-    char *tmp_buf;
-
-    vars->for_curr_iter = for_next_iter;
-
-    if (for_next_iter) {
-        tmp_buf = (char *)ADIOI_Malloc(for_next_iter);
-        ADIOI_Assert((((ADIO_Offset)(MPIU_Upint)read_buf)+real_size-for_next_iter) == (ADIO_Offset)(MPIU_Upint)(read_buf+real_size-for_next_iter));
-        ADIOI_Assert((for_next_iter+vars->coll_bufsize) == (size_t)(for_next_iter+vars->coll_bufsize));
-        memcpy(tmp_buf, read_buf+real_size-for_next_iter, for_next_iter);
-        ADIOI_Free(fd->io_buf);
-        fd->io_buf = (char *)ADIOI_Malloc(for_next_iter+vars->coll_bufsize);
-        memcpy(fd->io_buf, tmp_buf, for_next_iter);
-        vars->read_buf = fd->io_buf;
-        ADIOI_Free(tmp_buf);
-    }
-
-    vars->off += size;
-    vars->done += size;
-
-    /* increment m and go back to the beginning of m loop */
-    vars->m++;
-    ADIOI_Iread_and_exch_l1_begin(nbc_req, error_code);
-}
-
-static void ADIOI_Iread_and_exch_reset(ADIOI_NBC_Request *nbc_req,
-                                       int *error_code)
-{
-    ADIOI_Iread_and_exch_vars *vars = nbc_req->data.rd.rae_vars;
-    int nprocs = vars->nprocs;
-    int *count = vars->count;
-    int *send_size = vars->send_size;
-    int i;
-
-    for (i = 0; i < nprocs; i++) count[i] = send_size[i] = 0;
-
-    vars->m = vars->ntimes;
-    ADIOI_Iread_and_exch_l2_begin(nbc_req, error_code);
-}
-
-static void ADIOI_Iread_and_exch_l2_begin(ADIOI_NBC_Request *nbc_req,
-                                          int *error_code)
-{
-    ADIOI_Iread_and_exch_vars *vars = nbc_req->data.rd.rae_vars;
-    ADIOI_R_Iexchange_data_vars *red_vars = NULL;
-
-    /* loop exit condition */
-    if (vars->m >= vars->max_ntimes) {
-        ADIOI_Iread_and_exch_fini(nbc_req, error_code);
-        return;
-    }
-
-    /* create a struct for ADIOI_R_Iexchange_data() */
-    red_vars = (ADIOI_R_Iexchange_data_vars *)ADIOI_Calloc(
-            1, sizeof(ADIOI_R_Iexchange_data_vars));
-    nbc_req->data.rd.red_vars = red_vars;
-    red_vars->fd = vars->fd;
-    red_vars->buf = vars->buf;
-    red_vars->flat_buf = vars->flat_buf;
-    red_vars->offset_list = vars->offset_list;
-    red_vars->len_list = vars->len_list;
-    red_vars->send_size = vars->send_size;
-    red_vars->recv_size = vars->recv_size;
-    red_vars->count = vars->count;
-    red_vars->start_pos = vars->start_pos;
-    red_vars->partial_send = vars->partial_send;
-    red_vars->recd_from_proc = vars->recd_from_proc;
-    red_vars->nprocs = vars->nprocs;
-    red_vars->myrank = vars->myrank;
-    red_vars->buftype_is_contig = vars->buftype_is_contig;
-    red_vars->contig_access_count = vars->contig_access_count;
-    red_vars->min_st_offset = vars->min_st_offset;
-    red_vars->fd_size = vars->fd_size;
-    red_vars->fd_start = vars->fd_start;
-    red_vars->fd_end = vars->fd_end;
-    red_vars->others_req = vars->others_req;
-    red_vars->iter = vars->m;
-    red_vars->buftype_extent = vars->buftype_extent;
-    red_vars->buf_idx = vars->buf_idx;
-    red_vars->next_fn = ADIOI_Iread_and_exch_l2_end;
-
-    ADIOI_R_Iexchange_data(nbc_req, error_code);
-}
-
-static void ADIOI_Iread_and_exch_l2_end(ADIOI_NBC_Request *nbc_req,
-                                        int *error_code)
-{
-    ADIOI_Iread_and_exch_vars *vars = nbc_req->data.rd.rae_vars;
-
-    vars->m++;
-    ADIOI_Iread_and_exch_l2_begin(nbc_req, error_code);
-}
-
-static void ADIOI_Iread_and_exch_fini(ADIOI_NBC_Request *nbc_req, int *error_code)
-{
-    ADIOI_Iread_and_exch_vars *vars = nbc_req->data.rd.rae_vars;
-    void (*next_fn)(ADIOI_NBC_Request *, int *);
-
-    ADIOI_Free(vars->curr_offlen_ptr);
-    ADIOI_Free(vars->count);
-    ADIOI_Free(vars->partial_send);
-    ADIOI_Free(vars->send_size);
-    ADIOI_Free(vars->recv_size);
-    ADIOI_Free(vars->recd_from_proc);
-    ADIOI_Free(vars->start_pos);
-
-    next_fn = vars->next_fn;
-
-    /* free the struct for parameters and variables */
-    ADIOI_Free(nbc_req->data.rd.rae_vars);
-    nbc_req->data.rd.rae_vars = NULL;
-
-    /* move to the next function */
-    next_fn(nbc_req, error_code);
-}
-
-
-static void ADIOI_R_Iexchange_data(ADIOI_NBC_Request *nbc_req, int *error_code)
-{
-    ADIOI_R_Iexchange_data_vars *vars = nbc_req->data.rd.red_vars;
-
-    /* exchange send_size info so that each process knows how much to
-       receive from whom and how much memory to allocate. */
-    *error_code = MPI_Ialltoall(vars->send_size, 1, MPI_INT, vars->recv_size, 1,
-                                MPI_INT, vars->fd->comm, &vars->req1);
-
-    nbc_req->data.rd.state = ADIOI_IRC_STATE_R_IEXCHANGE_DATA;
-}
-
-static void ADIOI_R_Iexchange_data_recv(ADIOI_NBC_Request *nbc_req,
-                                        int *error_code)
-{
-    ADIOI_R_Iexchange_data_vars *vars = nbc_req->data.rd.red_vars;
-    ADIO_File fd = vars->fd;
-    int *send_size = vars->send_size;
-    int *recv_size = vars->recv_size;
-    int *count = vars->count;
-    int *start_pos = vars->start_pos;
-    int *partial_send = vars->partial_send;
-    int nprocs = vars->nprocs;
-    int myrank = vars->myrank;
-    ADIOI_Access *others_req = vars->others_req;
-    int iter = vars->iter;
-    int *buf_idx = vars->buf_idx;
-
-    int i, j, k = 0, tmp = 0, nprocs_recv, nprocs_send;
-    char **recv_buf = NULL;
-    MPI_Datatype send_type;
-
-    nprocs_recv = 0;
-    for (i = 0; i < nprocs; i++) if (recv_size[i]) nprocs_recv++;
-    vars->nprocs_recv = nprocs_recv;
-
-    nprocs_send = 0;
-    for (i = 0; i < nprocs; i++) if (send_size[i]) nprocs_send++;
-    vars->nprocs_send = nprocs_send;
-
-    vars->req2 = (MPI_Request *)
-        ADIOI_Malloc((nprocs_send+nprocs_recv+1)*sizeof(MPI_Request));
-    /* +1 to avoid a 0-size malloc */
-
-    /* post recvs. if buftype_is_contig, data can be directly recd. into
-       user buf at location given by buf_idx. else use recv_buf. */
-
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5032, 0, NULL);
-#endif
-
-    if (vars->buftype_is_contig) {
-        j = 0;
-        for (i = 0; i < nprocs; i++)
-            if (recv_size[i]) {
-                MPI_Irecv(((char *)vars->buf) + buf_idx[i], recv_size[i],
-                          MPI_BYTE, i, myrank+i+100*iter, fd->comm,
-                          vars->req2 + j);
-                j++;
-                buf_idx[i] += recv_size[i];
-            }
-    }
-    else {
-        /* allocate memory for recv_buf and post receives */
-        recv_buf = (char **) ADIOI_Malloc(nprocs * sizeof(char*));
-        vars->recv_buf = recv_buf;
-        for (i = 0; i < nprocs; i++)
-            if (recv_size[i]) recv_buf[i] = (char *)ADIOI_Malloc(recv_size[i]);
-
-        j = 0;
-        for (i = 0; i < nprocs; i++)
-            if (recv_size[i]) {
-                MPI_Irecv(recv_buf[i], recv_size[i], MPI_BYTE, i,
-                          myrank+i+100*iter, fd->comm,
-                          vars->req2 + j);
-                j++;
-#ifdef RDCOLL_DEBUG
-                DBG_FPRINTF(stderr, "node %d, recv_size %d, tag %d \n",
-                            myrank, recv_size[i], myrank+i+100*iter);
-#endif
-            }
-    }
-
-    /* create derived datatypes and send data */
-
-    j = 0;
-    for (i = 0; i < nprocs; i++) {
-        if (send_size[i]) {
-            /* take care if the last off-len pair is a partial send */
-            if (partial_send[i]) {
-                k = start_pos[i] + count[i] - 1;
-                tmp = others_req[i].lens[k];
-                others_req[i].lens[k] = partial_send[i];
-            }
-            ADIOI_Type_create_hindexed_x(count[i],
-                    &(others_req[i].lens[start_pos[i]]),
-                    &(others_req[i].mem_ptrs[start_pos[i]]),
-                    MPI_BYTE, &send_type);
-            /* absolute displacement; use MPI_BOTTOM in send */
-            MPI_Type_commit(&send_type);
-            MPI_Isend(MPI_BOTTOM, 1, send_type, i, myrank+i+100*iter,
-                      fd->comm, vars->req2 + nprocs_recv + j);
-            MPI_Type_free(&send_type);
-            if (partial_send[i]) others_req[i].lens[k] = tmp;
-            j++;
-        }
-    }
-
-    /* wait on the receives */
-    if (nprocs_recv) {
-        nbc_req->data.rd.state = ADIOI_IRC_STATE_R_IEXCHANGE_DATA_RECV;
-        return;
-    }
-
-    ADIOI_R_Iexchange_data_fill(nbc_req, error_code);
-}
-
-static void ADIOI_R_Iexchange_data_fill(ADIOI_NBC_Request *nbc_req,
-                                        int *error_code)
-{
-    ADIOI_R_Iexchange_data_vars *vars = nbc_req->data.rd.red_vars;
-
-    if (vars->nprocs_recv) {
-        /* if noncontiguous, to the copies from the recv buffers */
-        if (!vars->buftype_is_contig)
-            ADIOI_Fill_user_buffer(vars->fd, vars->buf, vars->flat_buf,
-                    vars->recv_buf, vars->offset_list, vars->len_list,
-                    (unsigned*)vars->recv_size,
-                    vars->req2, NULL, vars->recd_from_proc,
-                    vars->nprocs, vars->contig_access_count,
-                    vars->min_st_offset, vars->fd_size, vars->fd_start,
-                    vars->fd_end, vars->buftype_extent);
-    }
-
-    nbc_req->data.rd.state = ADIOI_IRC_STATE_R_IEXCHANGE_DATA_FILL;
-}
-
-static void ADIOI_R_Iexchange_data_fini(ADIOI_NBC_Request *nbc_req, int *error_code)
-{
-    ADIOI_R_Iexchange_data_vars *vars = nbc_req->data.rd.red_vars;
-    void (*next_fn)(ADIOI_NBC_Request *, int *);
-    int i;
-
-    ADIOI_Free(vars->req2);
-
-    if (!vars->buftype_is_contig) {
-        for (i = 0; i < vars->nprocs; i++)
-            if (vars->recv_size[i]) ADIOI_Free(vars->recv_buf[i]);
-        ADIOI_Free(vars->recv_buf);
-    }
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5033, 0, NULL);
-#endif
-
-    next_fn = vars->next_fn;
-
-    /* free the structure for parameters and variables */
-    ADIOI_Free(vars);
-    nbc_req->data.rd.red_vars = NULL;
-
-    /* move to the next function */
-    next_fn(nbc_req, error_code);
-}
-
-
-static int ADIOI_GEN_irc_query_fn(void *extra_state, MPI_Status *status)
-{
-    ADIOI_NBC_Request *nbc_req;
-
-    nbc_req = (ADIOI_NBC_Request *)extra_state;
-
-    MPI_Status_set_elements_x(status, MPI_BYTE, nbc_req->nbytes);
-
-    /* can never cancel so always true */
-    MPI_Status_set_cancelled(status, 0);
-
-    /* choose not to return a value for this */
-    status->MPI_SOURCE = MPI_UNDEFINED;
-    /* tag has no meaning for this generalized request */
-    status->MPI_TAG = MPI_UNDEFINED;
-
-    /* this generalized request never fails */
-    return MPI_SUCCESS;
-}
-
-static int ADIOI_GEN_irc_free_fn(void *extra_state)
-{
-    ADIOI_NBC_Request *nbc_req;
-
-    nbc_req = (ADIOI_NBC_Request *)extra_state;
-    ADIOI_Free(nbc_req);
-
-    return MPI_SUCCESS;
-}
-
-static int ADIOI_GEN_irc_poll_fn(void *extra_state, MPI_Status *status)
-{
-    ADIOI_NBC_Request *nbc_req;
-    ADIOI_GEN_IreadStridedColl_vars *rsc_vars = NULL;
-    ADIOI_Icalc_others_req_vars     *cor_vars = NULL;
-    ADIOI_Iread_and_exch_vars       *rae_vars = NULL;
-    ADIOI_R_Iexchange_data_vars     *red_vars = NULL;
-    int errcode = MPI_SUCCESS;
-    int flag;
-
-    nbc_req = (ADIOI_NBC_Request *)extra_state;
-
-    switch (nbc_req->data.rd.state) {
-        case ADIOI_IRC_STATE_GEN_IREADSTRIDEDCOLL:
-            rsc_vars = nbc_req->data.rd.rsc_vars;
-            errcode = MPI_Testall(2, rsc_vars->req_offset, &flag,
-                                  MPI_STATUSES_IGNORE);
-            if (errcode == MPI_SUCCESS && flag) {
-                ADIOI_GEN_IreadStridedColl_inter(nbc_req, &errcode);
-            }
-            break;
-
-        case ADIOI_IRC_STATE_GEN_IREADSTRIDEDCOLL_INDIO:
-            rsc_vars = nbc_req->data.rd.rsc_vars;
-            errcode = MPI_Test(&rsc_vars->req_ind_io, &flag, MPI_STATUS_IGNORE);
-            if (errcode == MPI_SUCCESS && flag) {
-                /* call the last function */
-                ADIOI_GEN_IreadStridedColl_fini(nbc_req, &errcode);
-            }
-            break;
-
-        case ADIOI_IRC_STATE_ICALC_OTHERS_REQ:
-            cor_vars = nbc_req->cor_vars;
-            errcode = MPI_Test(&cor_vars->req1, &flag, MPI_STATUS_IGNORE);
-            if (errcode == MPI_SUCCESS && flag) {
-                ADIOI_Icalc_others_req_main(nbc_req, &errcode);
-            }
-            break;
-
-        case ADIOI_IRC_STATE_ICALC_OTHERS_REQ_MAIN:
-            cor_vars = nbc_req->cor_vars;
-            if (cor_vars->num_req2) {
-                errcode = MPI_Testall(cor_vars->num_req2, cor_vars->req2,
-                                      &flag, MPI_STATUSES_IGNORE);
-                if (errcode == MPI_SUCCESS && flag) {
-                    ADIOI_Icalc_others_req_fini(nbc_req, &errcode);
-                }
-            } else {
-                ADIOI_Icalc_others_req_fini(nbc_req, &errcode);
-            }
-            break;
-
-        case ADIOI_IRC_STATE_IREAD_AND_EXCH:
-            rae_vars = nbc_req->data.rd.rae_vars;
-            errcode = MPI_Test(&rae_vars->req1, &flag, MPI_STATUS_IGNORE);
-            if (errcode == MPI_SUCCESS && flag) {
-                rae_vars->m = 0;
-                ADIOI_Iread_and_exch_l1_begin(nbc_req, &errcode);
-            }
-            break;
-
-        case ADIOI_IRC_STATE_IREAD_AND_EXCH_L1_BEGIN:
-            rae_vars = nbc_req->data.rd.rae_vars;
-            errcode = MPI_Test(&rae_vars->req2, &flag, MPI_STATUS_IGNORE);
-            if (errcode == MPI_SUCCESS && flag) {
-                ADIOI_R_Iexchange_data(nbc_req, &errcode);
-            }
-            break;
-
-        case ADIOI_IRC_STATE_R_IEXCHANGE_DATA:
-            red_vars = nbc_req->data.rd.red_vars;
-            errcode = MPI_Test(&red_vars->req1, &flag, MPI_STATUS_IGNORE);
-            if (errcode == MPI_SUCCESS && flag) {
-                ADIOI_R_Iexchange_data_recv(nbc_req, &errcode);
-            }
-            break;
-
-        case ADIOI_IRC_STATE_R_IEXCHANGE_DATA_RECV:
-            red_vars = nbc_req->data.rd.red_vars;
-            errcode = MPI_Testall(red_vars->nprocs_recv, red_vars->req2, &flag,
-                                  MPI_STATUSES_IGNORE);
-            if (errcode == MPI_SUCCESS && flag) {
-                ADIOI_R_Iexchange_data_fill(nbc_req, &errcode);
-            }
-            break;
-
-        case ADIOI_IRC_STATE_R_IEXCHANGE_DATA_FILL:
-            red_vars = nbc_req->data.rd.red_vars;
-            errcode = MPI_Testall(red_vars->nprocs_send,
-                                  red_vars->req2 + red_vars->nprocs_recv,
-                                  &flag, MPI_STATUSES_IGNORE);
-            if (errcode == MPI_SUCCESS && flag) {
-                ADIOI_R_Iexchange_data_fini(nbc_req, &errcode);
-            }
-            break;
-
-        default:
-            break;
-    }
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (errcode != MPI_SUCCESS) {
-        errcode = MPIO_Err_create_code(MPI_SUCCESS,
-                MPIR_ERR_RECOVERABLE,
-                "ADIOI_GEN_irc_poll_fn", __LINE__,
-                MPI_ERR_IO, "**mpi_grequest_complete",
-                0);
-    }
-    /* --END ERROR HANDLING-- */
-
-    return errcode;
-}
-
-/* wait for multiple requests to complete */
-static int ADIOI_GEN_irc_wait_fn(int count, void **array_of_states,
-                                 double timeout, MPI_Status *status)
-{
-    int i, errcode = MPI_SUCCESS;
-    double starttime;
-    ADIOI_NBC_Request **nbc_reqlist;
-
-    nbc_reqlist = (ADIOI_NBC_Request **)array_of_states;
-
-    starttime = MPI_Wtime();
-    for (i = 0; i < count ; i++) {
-        while (nbc_reqlist[i]->data.rd.state != ADIOI_IRC_STATE_COMPLETE) {
-            errcode = ADIOI_GEN_irc_poll_fn(nbc_reqlist[i], MPI_STATUS_IGNORE);
-            /* --BEGIN ERROR HANDLING-- */
-            if (errcode != MPI_SUCCESS) {
-                errcode = MPIO_Err_create_code(MPI_SUCCESS,
-                        MPIR_ERR_RECOVERABLE,
-                        "ADIOI_GEN_irc_wait_fn",
-                        __LINE__, MPI_ERR_IO,
-                        "**mpi_grequest_complete", 0);
-            }
-            /* --END ERROR HANDLING-- */
-
-            if ((timeout > 0) && (timeout < (MPI_Wtime() - starttime)))
-                goto fn_exit;
-
-            /* If the progress engine is blocked, we have to yield for another
-             * thread to be able to unblock the progress engine. */
-            MPIR_Ext_cs_yield();
-        }
-    }
-
-  fn_exit:
-    return errcode;
-}
-
-#endif /* HAVE_MPI_GREQUEST_EXTENSIONS */
diff --git a/3rd-party/romio321/adio/common/ad_iread_coll.pdf b/3rd-party/romio321/adio/common/ad_iread_coll.pdf
deleted file mode 100644
index 2c646fb5a5a..00000000000
Binary files a/3rd-party/romio321/adio/common/ad_iread_coll.pdf and /dev/null differ
diff --git a/3rd-party/romio321/adio/common/ad_iread_fake.c b/3rd-party/romio321/adio/common/ad_iread_fake.c
deleted file mode 100644
index b5cbd1da271..00000000000
--- a/3rd-party/romio321/adio/common/ad_iread_fake.c
+++ /dev/null
@@ -1,61 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2004 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpiu_greq.h"
-
-/* Generic implementation of IreadContig calls the blocking ReadContig
- * immediately.
- */
-void ADIOI_FAKE_IreadContig(ADIO_File fd, void *buf, int count, 
-			   MPI_Datatype datatype, int file_ptr_type,
-			   ADIO_Offset offset, ADIO_Request *request,
-			   int *error_code)  
-{
-    ADIO_Status status;
-    MPI_Count typesize;
-    MPI_Offset len;
-
-    MPI_Type_size_x(datatype, &typesize);
-    len = (MPI_Offset)count * (MPI_Offset)typesize;
-
-    /* Call the blocking function.  It will create an error code
-     * if necessary.
-     */
-    ADIOI_Assert(len == (int) len); /* the count is an int parm */
-    ADIO_ReadContig(fd, buf, (int)len, MPI_BYTE, file_ptr_type, offset, 
-		    &status, error_code);  
-    if (*error_code != MPI_SUCCESS) {
-	    len=0;
-    }
-    MPIO_Completed_request_create(&fd, len, error_code, request);
-}
-
-
-/* Generic implementation of IreadStrided calls the blocking ReadStrided
- * immediately.
- */
-void ADIOI_FAKE_IreadStrided(ADIO_File fd, void *buf, int count, 
-			    MPI_Datatype datatype, int file_ptr_type,
-			    ADIO_Offset offset, ADIO_Request *request,
-			    int *error_code)
-{
-    ADIO_Status status;
-    MPI_Count typesize;
-    MPI_Offset nbytes=0;
-
-    /* Call the blocking function.  It will create an error code
-     * if necessary.
-     */
-    ADIO_ReadStrided(fd, buf, count, datatype, file_ptr_type, 
-		     offset, &status, error_code);  
-    if (*error_code == MPI_SUCCESS) {
-	MPI_Type_size_x(datatype, &typesize);
-	nbytes = (MPI_Offset)count*(MPI_Offset)typesize;
-    }
-    MPIO_Completed_request_create(&fd, nbytes, error_code, request);
-}
diff --git a/3rd-party/romio321/adio/common/ad_iwrite.c b/3rd-party/romio321/adio/common/ad_iwrite.c
deleted file mode 100644
index e72af5fe224..00000000000
--- a/3rd-party/romio321/adio/common/ad_iwrite.c
+++ /dev/null
@@ -1,344 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2004 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_SIGNAL_H
-#include <signal.h>
-#endif
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-#ifdef HAVE_AIO_H
-#include <aio.h>
-#endif
-#ifdef HAVE_SYS_AIO_H
-#include <sys/aio.h>
-#endif
-#include <time.h>
-
-#include "../../mpi-io/mpioimpl.h"
-#include "../../mpi-io/mpioprof.h"
-#include "mpiu_greq.h"
-/* Workaround for incomplete set of definitions if __REDIRECT is not 
-   defined and large file support is used in aio.h */
-#if !defined(__REDIRECT) && defined(__USE_FILE_OFFSET64)
-#define aiocb aiocb64
-#endif
-
-#ifdef ROMIO_HAVE_WORKING_AIO
-
-static MPIX_Grequest_class ADIOI_GEN_greq_class = 0;
-
-/* ADIOI_GEN_IwriteContig
- *
- * This code handles only the case where ROMIO_HAVE_WORKING_AIO is 
- * defined. We post an asynchronous I/O operations using the appropriate aio
- * routines.  Otherwise, the ADIOI_Fns_struct will point to the FAKE
- * version.
- */
-void ADIOI_GEN_IwriteContig(ADIO_File fd, const void *buf, int count,
-			    MPI_Datatype datatype, int file_ptr_type,
-			    ADIO_Offset offset, ADIO_Request *request,
-			    int *error_code)
-{
-    MPI_Count len, typesize;
-    int aio_errno = 0;
-    static char myname[] = "ADIOI_GEN_IWRITECONTIG";
-
-    MPI_Type_size_x(datatype, &typesize);
-    len = count * typesize;
-    ADIOI_Assert(len == (int)((ADIO_Offset)count * (ADIO_Offset)typesize)); /* the count is an int parm */
-
-    if (file_ptr_type == ADIO_INDIVIDUAL) offset = fd->fp_ind;
-    /* Cast away the const'ness of 'buf' as ADIOI_GEN_aio is used for
-     * both read and write calls */
-    aio_errno = ADIOI_GEN_aio(fd, (char *) buf, len, offset, 1, request);
-    if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind += len;
-
-    fd->fp_sys_posn = -1;
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (aio_errno != 0) {
-	MPIO_ERR_CREATE_CODE_ERRNO(myname, aio_errno, error_code);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-    *error_code = MPI_SUCCESS;
-}
-/* This function is for implementation convenience.
- * It takes care of the differences in the interface for nonblocking I/O
- * on various Unix machines! If wr==1 write, wr==0 read.
- *
- * Returns 0 on success, -errno on failure.
- */
-int ADIOI_GEN_aio(ADIO_File fd, void *buf, int len, ADIO_Offset offset,
-		  int wr, MPI_Request *request)
-{
-    int err=-1, fd_sys;
-
-    int error_code;
-    struct aiocb *aiocbp=NULL;
-    ADIOI_AIO_Request *aio_req=NULL;
-    MPI_Status status;
-#if defined(ROMIO_XFS)
-    unsigned maxiosz = wr ? fd->hints->fs_hints.xfs.write_chunk_sz :
-	    fd->hints->fs_hints.xfs.read_chunk_sz;
-#endif /* ROMIO_XFS */
-
-    fd_sys = fd->fd_sys;
-
-#if defined(ROMIO_XFS)
-    /* Use Direct I/O if desired and properly aligned */
-    if (fd->fns == &ADIO_XFS_operations &&
-	 ((wr && fd->direct_write) || (!wr && fd->direct_read)) &&
-	 !(((long) buf) % fd->d_mem) && !(offset % fd->d_miniosz) && 
-	 !(len % fd->d_miniosz) && (len >= fd->d_miniosz) && 
-	 (len <= maxiosz)) {
-	    fd_sys = fd->fd_direct;
-    }
-#endif /* ROMIO_XFS */
-
-    aio_req = (ADIOI_AIO_Request*)ADIOI_Calloc(sizeof(ADIOI_AIO_Request), 1);
-    aiocbp = (struct aiocb *) ADIOI_Calloc(sizeof(struct aiocb), 1);
-    aiocbp->aio_offset = offset;
-    aiocbp->aio_buf    = buf;
-    aiocbp->aio_nbytes = len;
-
-#ifdef HAVE_STRUCT_AIOCB_AIO_WHENCE
-    aiocbp->aio_whence = SEEK_SET;
-#endif
-#ifdef HAVE_STRUCT_AIOCB_AIO_FILDES
-    aiocbp->aio_fildes = fd_sys;
-#endif
-#ifdef HAVE_STRUCT_AIOCB_AIO_SIGEVENT
-# ifdef AIO_SIGNOTIFY_NONE
-    aiocbp->aio_sigevent.sigev_notify = SIGEV_NONE;
-# endif
-    aiocbp->aio_sigevent.sigev_signo = 0;
-#endif
-#ifdef HAVE_STRUCT_AIOCB_AIO_REQPRIO
-# ifdef AIO_PRIO_DFL
-    aiocbp->aio_reqprio = AIO_PRIO_DFL;   /* not needed in DEC Unix 4.0 */
-# else
-    aiocbp->aio_reqprio = 0;
-# endif
-#endif
-
-#ifndef ROMIO_HAVE_AIO_CALLS_NEED_FILEDES
-#ifndef HAVE_STRUCT_AIOCB_AIO_FILDES
-#error 'No fildes set for aio structure'
-#endif
-    if (wr) err = aio_write(aiocbp);
-    else err = aio_read(aiocbp);
-#else
-    /* Broken IBM interface */
-    if (wr) err = aio_write(fd_sys, aiocbp);
-    else err = aio_read(fd_sys, aiocbp);
-#endif
-
-    if (err == -1) {
-	if (errno == EAGAIN || errno == ENOSYS) { 
-	    /* exceeded the max. no. of outstanding requests.
-               or, aio routines are not actually implemented 
-	    treat this as a blocking request and return.  */
-	    if (wr) 
-		ADIO_WriteContig(fd, buf, len, MPI_BYTE, 
-			    ADIO_EXPLICIT_OFFSET, offset, &status, &error_code);  
-	    else
-		ADIO_ReadContig(fd, buf, len, MPI_BYTE,
-			    ADIO_EXPLICIT_OFFSET, offset, &status, &error_code);  
-		    
-	    MPIO_Completed_request_create(&fd, len, &error_code, request);
-	    if (aiocbp != NULL) ADIOI_Free(aiocbp);
-	    if (aio_req != NULL) ADIOI_Free(aio_req);
-	    return 0;
-	} else {
-	    ADIOI_Free(aio_req);
-	    ADIOI_Free(aiocbp);
-	    return errno;
-	}
-    }
-    aio_req->aiocbp = aiocbp;
-    if (ADIOI_GEN_greq_class == 0) {
-	    MPIX_Grequest_class_create(ADIOI_GEN_aio_query_fn, 
-			    ADIOI_GEN_aio_free_fn, MPIU_Greq_cancel_fn, 
-			    ADIOI_GEN_aio_poll_fn, ADIOI_GEN_aio_wait_fn, 
-			    &ADIOI_GEN_greq_class);
-    }
-    MPIX_Grequest_class_allocate(ADIOI_GEN_greq_class, aio_req, request);
-    memcpy(&(aio_req->req), request, sizeof(MPI_Request));
-    return 0;
-}
-#endif
-
-
-/* Generic implementation of IwriteStrided calls the blocking WriteStrided
- * immediately.
- */
-void ADIOI_GEN_IwriteStrided(ADIO_File fd, const void *buf, int count,
-			     MPI_Datatype datatype, int file_ptr_type,
-			     ADIO_Offset offset, MPI_Request *request,
-			     int *error_code)
-{
-    ADIO_Status status;
-    MPI_Count typesize;
-    MPI_Offset nbytes=0;
-
-    /* Call the blocking function.  It will create an error code 
-     * if necessary.
-     */
-    ADIO_WriteStrided(fd, buf, count, datatype, file_ptr_type, 
-		      offset, &status, error_code);  
-
-    if (*error_code == MPI_SUCCESS) {
-	MPI_Type_size_x(datatype, &typesize);
-	nbytes = (MPI_Offset)count * (MPI_Offset)typesize;
-    }
-    MPIO_Completed_request_create(&fd, nbytes, error_code, request);
-}
-
-#ifdef ROMIO_HAVE_WORKING_AIO
-/* generic POSIX aio completion test routine */
-int ADIOI_GEN_aio_poll_fn(void *extra_state, MPI_Status *status)
-{
-    ADIOI_AIO_Request *aio_req;
-    int errcode=MPI_SUCCESS;
-
-    aio_req = (ADIOI_AIO_Request *)extra_state;
-
-    /* aio_error returns an ERRNO value */
-    errno = aio_error(aio_req->aiocbp);
-    if (errno == EINPROGRESS) {
-	    /* TODO: need to diddle with status somehow */
-    }
-    else if (errno == ECANCELED) {
-	    /* TODO: unsure how to handle this */
-    } else if (errno == 0) {
-	    ssize_t n = aio_return(aio_req->aiocbp);
-	    aio_req->nbytes = n;
-	    errcode = MPI_Grequest_complete(aio_req->req);
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (errcode != MPI_SUCCESS) {
-		    errcode = MPIO_Err_create_code(MPI_SUCCESS,
-				    MPIR_ERR_RECOVERABLE,
-				    "ADIOI_GEN_aio_poll_fn", __LINE__,
-				    MPI_ERR_IO, "**mpi_grequest_complete",
-				    0);
-	    }
-	    /* --END ERROR HANDLING-- */
-    }
-    return errcode;
-}
-
-/* wait for multiple requests to complete */
-int ADIOI_GEN_aio_wait_fn(int count, void ** array_of_states, 
-		double timeout, MPI_Status *status)
-{
-	const struct aiocb **cblist;
-	int err, errcode=MPI_SUCCESS;
-	int nr_complete=0;
-	double starttime;
-	struct timespec aio_timer;
-	struct timespec *aio_timer_p = NULL;
-
-	ADIOI_AIO_Request **aio_reqlist;
-	int i;
-
-	aio_reqlist = (ADIOI_AIO_Request **)array_of_states;
-
-	cblist = (const struct aiocb**) ADIOI_Calloc(count, sizeof(struct aiocb*));
-
-	starttime = MPI_Wtime();
-	if (timeout >0) {
-	    aio_timer.tv_sec = (time_t)timeout;
-	    aio_timer.tv_nsec = timeout - aio_timer.tv_sec;
-	    aio_timer_p = &aio_timer;
-	}
-	for (i=0; i< count; i++)
-	{
-		cblist[i] = aio_reqlist[i]->aiocbp;
-	}
-
-	while(nr_complete < count) {
-	    do {
-		err = aio_suspend(cblist, count, aio_timer_p);
-	    } while (err < 0 && errno == EINTR);
-	    if (err == 0) 
-	    { /* run through the list of requests, and mark all the completed
-		 ones as done */
-		for (i=0; i< count; i++)
-		{
-		    /* aio_error returns an ERRNO value */
-		    if (aio_reqlist[i]->aiocbp == NULL) 
-			continue;
-		    errno = aio_error(aio_reqlist[i]->aiocbp);
-		    if (errno == 0) {
-			ssize_t n = aio_return(aio_reqlist[i]->aiocbp);
-			aio_reqlist[i]->nbytes = n;
-			errcode = MPI_Grequest_complete(aio_reqlist[i]->req);
-			if (errcode != MPI_SUCCESS) {
-			    errcode = MPIO_Err_create_code(MPI_SUCCESS,
-				    MPIR_ERR_RECOVERABLE,
-				    "ADIOI_GEN_aio_wait_fn", 
-				    __LINE__, MPI_ERR_IO, 
-				    "**mpi_grequest_complete", 0);
-			}
-			ADIOI_Free(aio_reqlist[i]->aiocbp);
-			aio_reqlist[i]->aiocbp = NULL;
-			cblist[i] = NULL;
-			nr_complete++;
-		    } 
-		    /* TODO: need to handle error conditions somehow*/
-		}
-	    } /* TODO: also need to handle errors here  */
-	    if ( (timeout > 0) && (timeout < (MPI_Wtime() - starttime) ))
-		break;
-	}
-
-	if (cblist != NULL) ADIOI_Free(cblist);
-        return errcode;
-}
-
-int ADIOI_GEN_aio_free_fn(void *extra_state)
-{
-	ADIOI_AIO_Request *aio_req;
-	aio_req = (ADIOI_AIO_Request*)extra_state;
-
-	if (aio_req->aiocbp != NULL)
-		ADIOI_Free(aio_req->aiocbp);
-	ADIOI_Free(aio_req);
-
-	return MPI_SUCCESS;
-}
-#endif /* working AIO */
-
-int ADIOI_GEN_aio_query_fn(void *extra_state, MPI_Status *status) 
-{
-	ADIOI_AIO_Request *aio_req;
-
-	aio_req = (ADIOI_AIO_Request *)extra_state;
-
-	MPI_Status_set_elements_x(status, MPI_BYTE, aio_req->nbytes);
-
-	/* can never cancel so always true */ 
-	MPI_Status_set_cancelled(status, 0); 
-
-	/* choose not to return a value for this */ 
-	status->MPI_SOURCE = MPI_UNDEFINED; 
-	/* tag has no meaning for this generalized request */ 
-	status->MPI_TAG = MPI_UNDEFINED; 
-	/* this generalized request never fails */ 
-	return MPI_SUCCESS; 
-}
-/* 
- * vim: ts=8 sts=4 sw=4 noexpandtab 
- */
diff --git a/3rd-party/romio321/adio/common/ad_iwrite_coll.c b/3rd-party/romio321/adio/common/ad_iwrite_coll.c
deleted file mode 100644
index d3167fe2039..00000000000
--- a/3rd-party/romio321/adio/common/ad_iwrite_coll.c
+++ /dev/null
@@ -1,1571 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *  (C) 2014 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-#include "mpiu_greq.h"
-#include "mpioimpl.h"
-
-#ifdef AGGREGATION_PROFILE
-#include "mpe.h"
-#endif
-
-#ifdef HAVE_MPI_GREQUEST_EXTENSIONS
-
-/* ADIOI_GEN_IwriteStridedColl */
-struct ADIOI_GEN_IwriteStridedColl_vars {
-    /* requests */
-    MPI_Request req_offset[2]; /* ADIOI_IWC_STATE_GEN_IWRITESTRIDEDCOLL */
-    MPI_Request req_ind_io;    /* ADIOI_IWC_STATE_GEN_IWRITESTRIDEDCOLL_INDIO */
-    MPI_Request req_err;       /* ADIOI_IWC_STATE_GEN_IWRITESTRIDEDCOLL_BCAST */
-
-    /* parameters */
-    ADIO_File fd;
-    const void *buf;
-    int count;
-    MPI_Datatype datatype;
-    int file_ptr_type;
-    ADIO_Offset offset;
-
-    /* stack variables */
-    ADIOI_Access *my_req;
-    /* array of nprocs access structures, one for each other process in
-       whose file domain this process's request lies */
-
-    ADIOI_Access *others_req;
-    /* array of nprocs access structures, one for each other process
-       whose request lies in this process's file domain. */
-
-    int nprocs;
-    int nprocs_for_coll;
-    int myrank;
-    int contig_access_count;
-    int interleave_count;
-    int buftype_is_contig;
-    int *count_my_req_per_proc;
-    int count_my_req_procs;
-    int count_others_req_procs;
-    ADIO_Offset start_offset;
-    ADIO_Offset end_offset;
-    ADIO_Offset orig_fp;
-    ADIO_Offset fd_size;
-    ADIO_Offset min_st_offset;
-    ADIO_Offset *offset_list;
-    ADIO_Offset *st_offsets;
-    ADIO_Offset *fd_start;
-    ADIO_Offset *fd_end;
-    ADIO_Offset *end_offsets;
-    int *buf_idx;
-    ADIO_Offset *len_list;
-    int old_error;
-    int tmp_error;
-    int error_code;
-};
-
-/* ADIOI_Iexch_and_write */
-struct ADIOI_Iexch_and_write_vars {
-    /* requests */
-    MPI_Request req1;       /* ADIOI_IWC_STATE_IEXCH_AND_WRITE */
-    MPI_Request req3;       /* ADIOI_IWC_STATE_IEXCH_AND_WRITE_L1_BODY */
-
-    /* parameters */
-    ADIO_File fd;
-    void *buf;
-    MPI_Datatype datatype;
-    int nprocs;
-    int myrank;
-    ADIOI_Access *others_req;
-    ADIO_Offset *offset_list;
-    ADIO_Offset *len_list;
-    int contig_access_count;
-    ADIO_Offset min_st_offset;
-    ADIO_Offset fd_size;
-    ADIO_Offset *fd_start;
-    ADIO_Offset *fd_end;
-    int *buf_idx;
-
-    /* stack variables */
-    /* Not convinced end_loc-st_loc couldn't be > int, so make these offsets*/
-    ADIO_Offset size;
-    int hole;
-    int m;
-    int ntimes;
-    int max_ntimes;
-    int buftype_is_contig;
-    ADIO_Offset st_loc;
-    ADIO_Offset end_loc;
-    ADIO_Offset off;
-    ADIO_Offset done;
-    char *write_buf;
-    int *curr_offlen_ptr;
-    int *count;
-    int *send_size;
-    int *recv_size;
-    int *partial_recv;
-    int *sent_to_proc;
-    int *start_pos;
-    int *send_buf_idx;
-    int *curr_to_proc;
-    int *done_to_proc;
-    ADIOI_Flatlist_node *flat_buf;
-    MPI_Aint buftype_extent;
-    int coll_bufsize;
-
-    /* next function to be called */
-    void (*next_fn)(ADIOI_NBC_Request *, int *);
-};
-
-/* ADIOI_W_Iexchange_data */
-struct ADIOI_W_Iexchange_data_vars {
-    /* requests */
-    MPI_Request req1;   /* ADIOI_IWC_STATE_W_IEXCHANGE_DATA */
-    MPI_Request req2;   /* ADIOI_IWC_STATE_W_IEXCHANGE_DATA_HOLE */
-    MPI_Request *req3;  /* ADIOI_IWC_STATE_W_IEXCHANGE_DATA_SEND */
-
-    /* parameters */
-    ADIO_File fd;
-    void *buf;
-    char *write_buf;
-    ADIOI_Flatlist_node *flat_buf;
-    ADIO_Offset *offset_list;
-    ADIO_Offset *len_list;
-    int *send_size;
-    int *recv_size;
-    ADIO_Offset off;
-    int size;
-    int *count;
-    int *start_pos;
-    int *partial_recv;
-    int *sent_to_proc;
-    int nprocs;
-    int myrank;
-    int buftype_is_contig;
-    int contig_access_count;
-    ADIO_Offset min_st_offset;
-    ADIO_Offset fd_size;
-    ADIO_Offset *fd_start;
-    ADIO_Offset *fd_end;
-    ADIOI_Access *others_req;
-    int *send_buf_idx;
-    int *curr_to_proc;
-    int *done_to_proc;
-    int *hole;
-    int iter;
-    MPI_Aint buftype_extent;
-    int *buf_idx;
-
-    /* stack variables */
-    int nprocs_recv;
-    int nprocs_send;
-    int err;
-    char **send_buf;
-    MPI_Request *requests;
-    MPI_Request *send_req;
-    MPI_Datatype *recv_types;
-    int sum;
-    ADIO_Offset *srt_off;
-
-    /* next function to be called */
-    void (*next_fn)(ADIOI_NBC_Request *, int *);
-};
-
-
-void ADIOI_Fill_send_buffer(ADIO_File fd, void *buf, ADIOI_Flatlist_node
-                           *flat_buf, char **send_buf, ADIO_Offset
-                           *offset_list, ADIO_Offset *len_list, int *send_size,
-                           MPI_Request *requests, int *sent_to_proc,
-                           int nprocs, int myrank,
-                           int contig_access_count, ADIO_Offset
-                           min_st_offset, ADIO_Offset fd_size,
-                           ADIO_Offset *fd_start, ADIO_Offset *fd_end,
-                           int *send_buf_idx, int *curr_to_proc,
-                           int *done_to_proc, int iter,
-                           MPI_Aint buftype_extent);
-void ADIOI_Heap_merge(ADIOI_Access *others_req, int *count,
-                      ADIO_Offset *srt_off, int *srt_len, int *start_pos,
-                      int nprocs, int nprocs_recv, int total_elements);
-
-
-/* prototypes of functions used for nonblocking collective writes only. */
-static void ADIOI_GEN_IwriteStridedColl_inter(ADIOI_NBC_Request *, int *);
-static void ADIOI_GEN_IwriteStridedColl_indio(ADIOI_NBC_Request *, int *);
-static void ADIOI_GEN_IwriteStridedColl_exch(ADIOI_NBC_Request *, int *);
-static void ADIOI_GEN_IwriteStridedColl_bcast(ADIOI_NBC_Request *, int *);
-static void ADIOI_GEN_IwriteStridedColl_free(ADIOI_NBC_Request *, int *);
-static void ADIOI_GEN_IwriteStridedColl_fini(ADIOI_NBC_Request *, int *);
-
-static void ADIOI_Iexch_and_write(ADIOI_NBC_Request *, int *);
-static void ADIOI_Iexch_and_write_l1_begin(ADIOI_NBC_Request *, int *);
-static void ADIOI_Iexch_and_write_l1_body(ADIOI_NBC_Request *, int *);
-static void ADIOI_Iexch_and_write_l1_end(ADIOI_NBC_Request *, int *);
-static void ADIOI_Iexch_and_write_reset(ADIOI_NBC_Request *, int *);
-static void ADIOI_Iexch_and_write_l2_begin(ADIOI_NBC_Request *, int *);
-static void ADIOI_Iexch_and_write_l2_end(ADIOI_NBC_Request *, int *);
-static void ADIOI_Iexch_and_write_fini(ADIOI_NBC_Request *, int *);
-
-static void ADIOI_W_Iexchange_data(ADIOI_NBC_Request *, int *);
-static void ADIOI_W_Iexchange_data_hole(ADIOI_NBC_Request *, int *);
-static void ADIOI_W_Iexchange_data_send(ADIOI_NBC_Request *, int *);
-static void ADIOI_W_Iexchange_data_wait(ADIOI_NBC_Request *, int *);
-static void ADIOI_W_Iexchange_data_fini(ADIOI_NBC_Request *, int *);
-
-static MPIX_Grequest_class ADIOI_GEN_greq_class = 0;
-static int ADIOI_GEN_iwc_query_fn(void *extra_state, MPI_Status *status);
-static int ADIOI_GEN_iwc_free_fn(void *extra_state);
-static int ADIOI_GEN_iwc_poll_fn(void *extra_state, MPI_Status *status);
-static int ADIOI_GEN_iwc_wait_fn(int count, void **array_of_states,
-                                 double timeout, MPI_Status *status);
-
-
-/* Non-blocking version of ADIOI_GEN_WriteStridedColl() */
-void ADIOI_GEN_IwriteStridedColl(ADIO_File fd, const void *buf, int count,
-                       MPI_Datatype datatype, int file_ptr_type,
-                       ADIO_Offset offset, MPI_Request *request,
-                       int *error_code)
-{
-    /* Uses a generalized version of the extended two-phase method described
-       in "An Extended Two-Phase Method for Accessing Sections of
-       Out-of-Core Arrays", Rajeev Thakur and Alok Choudhary,
-       Scientific Programming, (5)4:301--317, Winter 1996.
-       http://www.mcs.anl.gov/home/thakur/ext2ph.ps */
-
-    ADIOI_NBC_Request *nbc_req = NULL;
-    ADIOI_GEN_IwriteStridedColl_vars *vars = NULL;
-    int nprocs, myrank;
-
-#if 0
-    /* FIXME: need an implementation of ADIOI_IOIstridedColl */
-    if (fd->hints->cb_pfr != ADIOI_HINT_DISABLE) {
-        /* Cast away const'ness as the below function is used for read
-         * and write */
-        ADIOI_IOIstridedColl(fd, (char *) buf, count, ADIOI_WRITE, datatype,
-                             file_ptr_type, offset, request, error_code);
-        return;
-    }
-#endif
-
-    /* top-level struct keeping the status of function progress */
-    nbc_req = (ADIOI_NBC_Request *)ADIOI_Calloc(1, sizeof(ADIOI_NBC_Request));
-    nbc_req->rdwr = ADIOI_WRITE;
-
-    /* create a generalized request */
-    if (ADIOI_GEN_greq_class == 0) {
-        MPIX_Grequest_class_create(ADIOI_GEN_iwc_query_fn,
-                ADIOI_GEN_iwc_free_fn, MPIU_Greq_cancel_fn,
-                ADIOI_GEN_iwc_poll_fn, ADIOI_GEN_iwc_wait_fn,
-                &ADIOI_GEN_greq_class);
-    }
-    MPIX_Grequest_class_allocate(ADIOI_GEN_greq_class, nbc_req, request);
-    memcpy(&nbc_req->req, request, sizeof(MPI_Request));
-
-    /* create a struct for parameters and variables */
-    vars = (ADIOI_GEN_IwriteStridedColl_vars *)ADIOI_Calloc(
-            1, sizeof(ADIOI_GEN_IwriteStridedColl_vars));
-    nbc_req->data.wr.wsc_vars = vars;
-
-    /* save the parameters */
-    vars->fd = fd;
-    vars->buf = buf;
-    vars->count = count;
-    vars->datatype = datatype;
-    vars->file_ptr_type = file_ptr_type;
-    vars->offset = offset;
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-    vars->nprocs = nprocs;
-    vars->myrank = myrank;
-
-    /* the number of processes that actually perform I/O, nprocs_for_coll,
-     * is stored in the hints off the ADIO_File structure
-     */
-    vars->nprocs_for_coll = fd->hints->cb_nodes;
-    vars->orig_fp = fd->fp_ind;
-
-    /* only check for interleaving if cb_write isn't disabled */
-    if (fd->hints->cb_write != ADIOI_HINT_DISABLE) {
-        /* For this process's request, calculate the list of offsets and
-           lengths in the file and determine the start and end offsets. */
-
-        /* Note: end_offset points to the last byte-offset that will be accessed.
-           e.g., if start_offset=0 and 100 bytes to be read, end_offset=99*/
-
-        ADIOI_Calc_my_off_len(fd, count, datatype, file_ptr_type, offset,
-                              &vars->offset_list, &vars->len_list,
-                              &vars->start_offset, &vars->end_offset,
-                              &vars->contig_access_count);
-
-        /* each process communicates its start and end offsets to other
-           processes. The result is an array each of start and end offsets
-           stored in order of process rank. */
-
-        vars->st_offsets = (ADIO_Offset *)ADIOI_Malloc(nprocs*sizeof(ADIO_Offset));
-        vars->end_offsets = (ADIO_Offset *)ADIOI_Malloc(nprocs*sizeof(ADIO_Offset));
-
-        *error_code = MPI_Iallgather(&vars->start_offset, 1, ADIO_OFFSET,
-                                     vars->st_offsets, 1, ADIO_OFFSET,
-                                     fd->comm, &vars->req_offset[0]);
-        if (*error_code != MPI_SUCCESS) return;
-        *error_code = MPI_Iallgather(&vars->end_offset, 1, ADIO_OFFSET,
-                                     vars->end_offsets, 1, ADIO_OFFSET,
-                                     fd->comm, &vars->req_offset[1]);
-
-        nbc_req->data.wr.state = ADIOI_IWC_STATE_GEN_IWRITESTRIDEDCOLL;
-        return;
-    }
-
-    ADIOI_GEN_IwriteStridedColl_indio(nbc_req, error_code);
-}
-
-static void ADIOI_GEN_IwriteStridedColl_inter(ADIOI_NBC_Request *nbc_req,
-                                              int *error_code)
-{
-    ADIOI_GEN_IwriteStridedColl_vars *vars = nbc_req->data.wr.wsc_vars;
-    int nprocs = vars->nprocs;
-    ADIO_Offset *st_offsets = vars->st_offsets;
-    ADIO_Offset *end_offsets = vars->end_offsets;
-    int i, interleave_count = 0;
-
-    /* are the accesses of different processes interleaved? */
-    for (i = 1; i < nprocs; i++)
-        if ((st_offsets[i] < end_offsets[i-1]) &&
-            (st_offsets[i] <= end_offsets[i]))
-            interleave_count++;
-    /* This is a rudimentary check for interleaving, but should suffice
-       for the moment. */
-
-    vars->interleave_count = interleave_count;
-
-    ADIOI_GEN_IwriteStridedColl_indio(nbc_req, error_code);
-}
-
-static void ADIOI_GEN_IwriteStridedColl_indio(ADIOI_NBC_Request *nbc_req,
-                                              int *error_code)
-{
-    ADIOI_GEN_IwriteStridedColl_vars *vars = nbc_req->data.wr.wsc_vars;
-    ADIOI_Icalc_others_req_vars *cor_vars = NULL;
-    ADIO_File fd = vars->fd;
-    const void *buf;
-    int count, file_ptr_type;
-    MPI_Datatype datatype = vars->datatype;
-    ADIO_Offset offset;
-    int filetype_is_contig;
-    ADIO_Offset off;
-    int nprocs;
-
-    ADIOI_Datatype_iscontig(datatype, &vars->buftype_is_contig);
-
-    if (fd->hints->cb_write == ADIOI_HINT_DISABLE ||
-       (!vars->interleave_count && (fd->hints->cb_write == ADIOI_HINT_AUTO)))
-    {
-        buf = vars->buf;
-        count = vars->count;
-        file_ptr_type = vars->file_ptr_type;
-        offset = vars->offset;
-
-        /* use independent accesses */
-        if (fd->hints->cb_write != ADIOI_HINT_DISABLE) {
-            ADIOI_Free(vars->offset_list);
-            ADIOI_Free(vars->len_list);
-            ADIOI_Free(vars->st_offsets);
-            ADIOI_Free(vars->end_offsets);
-        }
-
-        fd->fp_ind = vars->orig_fp;
-        ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-
-#if defined(ROMIO_RUN_ON_LINUX) && !defined(HAVE_AIO_LITE_H)
-        /* NOTE: This is currently a workaround to avoid weird errors, e.g.,
-         * stack fault, occurred on Linux. When the host OS is Linux and
-         * aio-lite is not used, a blocking ADIO function is used here.
-         * See https://trac.mpich.org/projects/mpich/ticket/2201. */
-        MPI_Status status;
-        if (vars->buftype_is_contig && filetype_is_contig) {
-            if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-                off = fd->disp + (ADIO_Offset)(fd->etype_size) * offset;
-                ADIO_WriteContig(fd, buf, count, datatype,
-                                 ADIO_EXPLICIT_OFFSET,
-                                 off, &status, error_code);
-            }
-            else ADIO_WriteContig(fd, buf, count, datatype, ADIO_INDIVIDUAL,
-                                  0, &status, error_code);
-        }
-        else {
-            ADIO_WriteStrided(fd, buf, count, datatype, file_ptr_type,
-                              offset, &status, error_code);
-        }
-        ADIOI_GEN_IwriteStridedColl_fini(nbc_req, error_code);
-#else
-        if (vars->buftype_is_contig && filetype_is_contig) {
-            if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-                off = fd->disp + (ADIO_Offset)(fd->etype_size) * offset;
-                ADIO_IwriteContig(fd, buf, count, datatype,
-                                  ADIO_EXPLICIT_OFFSET,
-                                  off, &vars->req_ind_io, error_code);
-            }
-            else ADIO_IwriteContig(fd, buf, count, datatype, ADIO_INDIVIDUAL,
-                                   0, &vars->req_ind_io, error_code);
-        }
-        else {
-            ADIO_IwriteStrided(fd, buf, count, datatype, file_ptr_type,
-                               offset, &vars->req_ind_io, error_code);
-        }
-
-        nbc_req->data.wr.state = ADIOI_IWC_STATE_GEN_IWRITESTRIDEDCOLL_INDIO;
-#endif
-        return;
-    }
-
-    nprocs = vars->nprocs;
-
-    /* Divide the I/O workload among "nprocs_for_coll" processes. This is
-       done by (logically) dividing the file into file domains (FDs); each
-       process may directly access only its own file domain. */
-
-    ADIOI_Calc_file_domains(vars->st_offsets, vars->end_offsets, nprocs,
-            vars->nprocs_for_coll, &vars->min_st_offset,
-            &vars->fd_start, &vars->fd_end,
-            fd->hints->min_fdomain_size, &vars->fd_size,
-            fd->hints->striping_unit);
-
-    /* calculate what portions of the access requests of this process are
-       located in what file domains */
-
-    ADIOI_Calc_my_req(fd, vars->offset_list, vars->len_list,
-            vars->contig_access_count, vars->min_st_offset,
-            vars->fd_start, vars->fd_end, vars->fd_size,
-            nprocs, &vars->count_my_req_procs,
-            &vars->count_my_req_per_proc, &vars->my_req,
-            &vars->buf_idx);
-
-    /* based on everyone's my_req, calculate what requests of other
-       processes lie in this process's file domain.
-       count_others_req_procs = number of processes whose requests lie in
-       this process's file domain (including this process itself)
-       count_others_req_per_proc[i] indicates how many separate contiguous
-       requests of proc. i lie in this process's file domain. */
-
-    cor_vars = (ADIOI_Icalc_others_req_vars *)ADIOI_Calloc(
-            1, sizeof(ADIOI_Icalc_others_req_vars));
-    nbc_req->cor_vars = cor_vars;
-    cor_vars->fd = vars->fd;
-    cor_vars->count_my_req_procs = vars->count_my_req_procs;
-    cor_vars->count_my_req_per_proc = vars->count_my_req_per_proc;
-    cor_vars->my_req = vars->my_req;
-    cor_vars->nprocs = vars->nprocs;
-    cor_vars->myrank = vars->myrank;
-    cor_vars->count_others_req_procs_ptr = &vars->count_others_req_procs;
-    cor_vars->others_req_ptr = &vars->others_req;
-    cor_vars->next_fn = ADIOI_GEN_IwriteStridedColl_exch;
-
-    ADIOI_Icalc_others_req(nbc_req, error_code);
-}
-
-static void ADIOI_GEN_IwriteStridedColl_exch(ADIOI_NBC_Request *nbc_req,
-                                             int *error_code)
-{
-    ADIOI_GEN_IwriteStridedColl_vars *vars = nbc_req->data.wr.wsc_vars;
-    ADIOI_Iexch_and_write_vars *eaw_vars = NULL;
-    ADIOI_Access *my_req = vars->my_req;
-    int nprocs = vars->nprocs;
-    int i;
-
-    ADIOI_Free(vars->count_my_req_per_proc);
-    for (i = 0; i < nprocs; i++) {
-        if (my_req[i].count) {
-            ADIOI_Free(my_req[i].offsets);
-            ADIOI_Free(my_req[i].lens);
-        }
-    }
-    ADIOI_Free(my_req);
-
-    /* exchange data and write in sizes of no more than coll_bufsize. */
-    /* Cast away const'ness for the below function */
-    eaw_vars = (ADIOI_Iexch_and_write_vars *)ADIOI_Calloc(
-            1, sizeof(ADIOI_Iexch_and_write_vars));
-    nbc_req->data.wr.eaw_vars = eaw_vars;
-    eaw_vars->fd = vars->fd;
-    eaw_vars->buf = (char *)vars->buf;
-    eaw_vars->datatype = vars->datatype;
-    eaw_vars->nprocs = vars->nprocs;
-    eaw_vars->myrank = vars->myrank;
-    eaw_vars->others_req = vars->others_req;
-    eaw_vars->offset_list = vars->offset_list;
-    eaw_vars->len_list = vars->len_list;
-    eaw_vars->contig_access_count = vars->contig_access_count;
-    eaw_vars->min_st_offset = vars->min_st_offset;
-    eaw_vars->fd_size = vars->fd_size;
-    eaw_vars->fd_start = vars->fd_start;
-    eaw_vars->fd_end = vars->fd_end;
-    eaw_vars->buf_idx = vars->buf_idx;
-    eaw_vars->next_fn = ADIOI_GEN_IwriteStridedColl_bcast;
-
-    ADIOI_Iexch_and_write(nbc_req, error_code);
-}
-
-static void ADIOI_GEN_IwriteStridedColl_bcast(ADIOI_NBC_Request *nbc_req,
-                                              int *error_code)
-{
-    ADIOI_GEN_IwriteStridedColl_vars *vars = nbc_req->data.wr.wsc_vars;
-    ADIO_File fd = vars->fd;
-
-    /* If this collective write is followed by an independent write,
-     * it's possible to have those subsequent writes on other processes
-     * race ahead and sneak in before the read-modify-write completes.
-     * We carry out a collective communication at the end here so no one
-     * can start independent i/o before collective I/O completes.
-     *
-     * need to do some gymnastics with the error codes so that if something
-     * went wrong, all processes report error, but if a process has a more
-     * specific error code, we can still have that process report the
-     * additional information */
-
-    vars->old_error = *error_code;
-    if (*error_code != MPI_SUCCESS) *error_code = MPI_ERR_IO;
-
-    /* optimization: if only one process performing i/o, we can perform
-     * a less-expensive Bcast  */
-#ifdef ADIOI_MPE_LOGGING
-    MPE_Log_event( ADIOI_MPE_postwrite_a, 0, NULL );
-#endif
-    vars->error_code = *error_code;
-    if (fd->hints->cb_nodes == 1) {
-        *error_code = MPI_Ibcast(&vars->error_code, 1, MPI_INT,
-                                 fd->hints->ranklist[0], fd->comm,
-                                 &vars->req_err);
-    } else {
-        vars->tmp_error = *error_code;
-        *error_code  = MPI_Iallreduce(&vars->tmp_error, &vars->error_code, 1,
-                                      MPI_INT, MPI_MAX, fd->comm,
-                                      &vars->req_err);
-    }
-
-    nbc_req->data.wr.state = ADIOI_IWC_STATE_GEN_IWRITESTRIDEDCOLL_BCAST;
-}
-
-static void ADIOI_GEN_IwriteStridedColl_free(ADIOI_NBC_Request *nbc_req,
-                                             int *error_code)
-{
-    ADIOI_GEN_IwriteStridedColl_vars *vars = nbc_req->data.wr.wsc_vars;
-    ADIO_File fd = vars->fd;
-    MPI_Datatype datatype = vars->datatype;
-    ADIOI_Access *others_req = vars->others_req;
-    int nprocs = vars->nprocs;
-    int old_error = vars->old_error;
-    int i;
-
-#ifdef ADIOI_MPE_LOGGING
-    MPE_Log_event( ADIOI_MPE_postwrite_b, 0, NULL );
-#endif
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event(5012, 0, NULL);
-#endif
-
-    if ( (old_error != MPI_SUCCESS) && (old_error != MPI_ERR_IO) )
-        *error_code = old_error;
-
-
-    if (!vars->buftype_is_contig) ADIOI_Delete_flattened(datatype);
-
-    /* free all memory allocated for collective I/O */
-    for (i = 0; i < nprocs; i++) {
-        if (others_req[i].count) {
-            ADIOI_Free(others_req[i].offsets);
-            ADIOI_Free(others_req[i].lens);
-            ADIOI_Free(others_req[i].mem_ptrs);
-        }
-    }
-    ADIOI_Free(others_req);
-
-    ADIOI_Free(vars->buf_idx);
-    ADIOI_Free(vars->offset_list);
-    ADIOI_Free(vars->len_list);
-    ADIOI_Free(vars->st_offsets);
-    ADIOI_Free(vars->end_offsets);
-    ADIOI_Free(vars->fd_start);
-    ADIOI_Free(vars->fd_end);
-
-    fd->fp_sys_posn = -1;   /* set it to null. */
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5013, 0, NULL);
-#endif
-
-    ADIOI_GEN_IwriteStridedColl_fini(nbc_req, error_code);
-}
-
-static void ADIOI_GEN_IwriteStridedColl_fini(ADIOI_NBC_Request *nbc_req,
-                                             int *error_code)
-{
-    ADIOI_GEN_IwriteStridedColl_vars *vars = nbc_req->data.wr.wsc_vars;
-    MPI_Count size;
-
-    /* This is a temporary way of filling in status. The right way is to
-       keep track of how much data was actually written during collective I/O. */
-    MPI_Type_size_x(vars->datatype, &size);
-    nbc_req->nbytes = size * vars->count;
-
-    /* free the struct for parameters and variables */
-    if (nbc_req->data.wr.wsc_vars) {
-        ADIOI_Free(nbc_req->data.wr.wsc_vars);
-        nbc_req->data.wr.wsc_vars = NULL;
-    }
-
-    /* make the request complete */
-    *error_code = MPI_Grequest_complete(nbc_req->req);
-    nbc_req->data.wr.state = ADIOI_IWC_STATE_COMPLETE;
-}
-
-
-static void ADIOI_Iexch_and_write(ADIOI_NBC_Request *nbc_req, int *error_code)
-{
-    ADIOI_Iexch_and_write_vars *vars = nbc_req->data.wr.eaw_vars;
-    ADIO_File fd = vars->fd;
-    MPI_Datatype datatype = vars->datatype;
-    int nprocs = vars->nprocs;
-    ADIOI_Access *others_req = vars->others_req;
-    MPI_Aint lb;
-
-    /* Send data to appropriate processes and write in sizes of no more
-       than coll_bufsize.
-       The idea is to reduce the amount of extra memory required for
-       collective I/O. If all data were written all at once, which is much
-       easier, it would require temp space more than the size of user_buf,
-       which is often unacceptable. For example, to write a distributed
-       array to a file, where each local array is 8Mbytes, requiring
-       at least another 8Mbytes of temp space is unacceptable. */
-
-    int i, j;
-    ADIO_Offset st_loc = -1, end_loc = -1;
-    int info_flag, coll_bufsize;
-    char *value;
-
-    *error_code = MPI_SUCCESS;  /* changed below if error */
-    /* only I/O errors are currently reported */
-
-    /* calculate the number of writes of size coll_bufsize
-       to be done by each process and the max among all processes.
-       That gives the no. of communication phases as well. */
-
-    value = (char *)ADIOI_Malloc((MPI_MAX_INFO_VAL+1) * sizeof(char));
-    ADIOI_Info_get(fd->info, "cb_buffer_size", MPI_MAX_INFO_VAL, value,
-                   &info_flag);
-    coll_bufsize = atoi(value);
-    vars->coll_bufsize = coll_bufsize;
-    ADIOI_Free(value);
-
-    for (i = 0; i < nprocs; i++) {
-        if (others_req[i].count) {
-            st_loc = others_req[i].offsets[0];
-            end_loc = others_req[i].offsets[0];
-            break;
-        }
-    }
-
-    for (i = 0; i < nprocs; i++)
-        for (j = 0; j < others_req[i].count; j++) {
-            st_loc = ADIOI_MIN(st_loc, others_req[i].offsets[j]);
-            end_loc = ADIOI_MAX(end_loc, (others_req[i].offsets[j]
-                        + others_req[i].lens[j] - 1));
-        }
-
-    vars->st_loc = st_loc;
-    vars->end_loc = end_loc;
-
-    /* ntimes=ceiling_div(end_loc - st_loc + 1, coll_bufsize)*/
-
-    vars->ntimes = (int)((end_loc - st_loc + coll_bufsize) / coll_bufsize);
-
-    if ((st_loc==-1) && (end_loc==-1)) {
-        vars->ntimes = 0; /* this process does no writing. */
-    }
-
-    *error_code = MPI_Iallreduce(&vars->ntimes, &vars->max_ntimes, 1, MPI_INT,
-                                 MPI_MAX, fd->comm, &vars->req1);
-
-    vars->write_buf = fd->io_buf;
-
-    vars->curr_offlen_ptr = (int *)ADIOI_Calloc(nprocs, sizeof(int));
-    /* its use is explained below. calloc initializes to 0. */
-
-    vars->count = (int *)ADIOI_Malloc(nprocs*sizeof(int));
-    /* to store count of how many off-len pairs per proc are satisfied
-       in an iteration. */
-
-    vars->partial_recv = (int *)ADIOI_Calloc(nprocs, sizeof(int));
-    /* if only a portion of the last off-len pair is recd. from a process
-       in a particular iteration, the length recd. is stored here.
-       calloc initializes to 0. */
-
-    vars->send_size = (int *)ADIOI_Malloc(nprocs*sizeof(int));
-    /* total size of data to be sent to each proc. in an iteration.
-       Of size nprocs so that I can use MPI_Alltoall later. */
-
-    vars->recv_size = (int *)ADIOI_Malloc(nprocs*sizeof(int));
-    /* total size of data to be recd. from each proc. in an iteration.*/
-
-    vars->sent_to_proc = (int *)ADIOI_Calloc(nprocs, sizeof(int));
-    /* amount of data sent to each proc so far. Used in
-       ADIOI_Fill_send_buffer. initialized to 0 here. */
-
-    vars->send_buf_idx = (int *)ADIOI_Malloc(nprocs*sizeof(int));
-    vars->curr_to_proc = (int *)ADIOI_Malloc(nprocs*sizeof(int));
-    vars->done_to_proc = (int *)ADIOI_Malloc(nprocs*sizeof(int));
-    /* Above three are used in ADIOI_Fill_send_buffer*/
-
-    vars->start_pos = (int *)ADIOI_Malloc(nprocs*sizeof(int));
-    /* used to store the starting value of curr_offlen_ptr[i] in
-       this iteration */
-
-    ADIOI_Datatype_iscontig(datatype, &vars->buftype_is_contig);
-    if (!vars->buftype_is_contig) {
-	vars->flat_buf = ADIOI_Flatten_and_find(datatype);
-    }
-    MPI_Type_get_extent(datatype, &lb, &vars->buftype_extent);
-
-
-    /* I need to check if there are any outstanding nonblocking writes to
-       the file, which could potentially interfere with the writes taking
-       place in this collective write call. Since this is not likely to be
-       common, let me do the simplest thing possible here: Each process
-       completes all pending nonblocking operations before completing. */
-
-    /*ADIOI_Complete_async(error_code);
-      if (*error_code != MPI_SUCCESS) return;
-      MPI_Barrier(fd->comm);
-     */
-
-    vars->done = 0;
-    vars->off = st_loc;
-
-    /* set the state to wait until MPI_Ialltoall finishes. */
-    nbc_req->data.wr.state = ADIOI_IWC_STATE_IEXCH_AND_WRITE;
-}
-
-static void ADIOI_Iexch_and_write_l1_begin(ADIOI_NBC_Request *nbc_req,
-                                           int *error_code)
-{
-    ADIOI_Iexch_and_write_vars *vars = nbc_req->data.wr.eaw_vars;
-    int nprocs;
-    ADIOI_Access *others_req;
-
-    int i, j;
-    ADIO_Offset off, req_off;
-    char *write_buf;
-    int *curr_offlen_ptr, *count, req_len, *recv_size;
-    int *partial_recv, *start_pos;
-    ADIO_Offset size;
-    static char myname[] = "ADIOI_IEXCH_AND_WRITE_L1_BEGIN";
-
-    ADIOI_W_Iexchange_data_vars *wed_vars = NULL;
-
-    /* loop exit condition */
-    if (vars->m >= vars->ntimes) {
-        ADIOI_Iexch_and_write_reset(nbc_req, error_code);
-        return;
-    }
-
-    nprocs = vars->nprocs;
-    others_req = vars->others_req;
-
-    off = vars->off;
-    write_buf = vars->write_buf;
-    curr_offlen_ptr = vars->curr_offlen_ptr;
-    count = vars->count;
-    recv_size = vars->recv_size;
-    partial_recv = vars->partial_recv;
-    start_pos = vars->start_pos;
-
-    /* go through all others_req and check which will be satisfied
-       by the current write */
-
-    /* Note that MPI guarantees that displacements in filetypes are in
-       monotonically nondecreasing order and that, for writes, the
-       filetypes cannot specify overlapping regions in the file. This
-       simplifies implementation a bit compared to reads. */
-
-    /* off = start offset in the file for the data to be written in
-       this iteration
-       size = size of data written (bytes) corresponding to off
-       req_off = off in file for a particular contiguous request
-       minus what was satisfied in previous iteration
-       req_size = size corresponding to req_off */
-
-    /* first calculate what should be communicated */
-
-    for (i = 0; i < nprocs; i++) count[i] = recv_size[i] = 0;
-
-    size = ADIOI_MIN((unsigned)vars->coll_bufsize,
-                     vars->end_loc - vars->st_loc + 1 - vars->done);
-    vars->size = size;
-
-    for (i = 0; i < nprocs; i++) {
-        if (others_req[i].count) {
-            start_pos[i] = curr_offlen_ptr[i];
-            for (j = curr_offlen_ptr[i]; j < others_req[i].count; j++) {
-                if (partial_recv[i]) {
-                    /* this request may have been partially
-                       satisfied in the previous iteration. */
-                    req_off = others_req[i].offsets[j] +
-                        partial_recv[i];
-                    req_len = others_req[i].lens[j] -
-                        partial_recv[i];
-                    partial_recv[i] = 0;
-                    /* modify the off-len pair to reflect this change */
-                    others_req[i].offsets[j] = req_off;
-                    others_req[i].lens[j] = req_len;
-                }
-                else {
-                    req_off = others_req[i].offsets[j];
-                    req_len = others_req[i].lens[j];
-                }
-                if (req_off < off + size) {
-                    count[i]++;
-                    ADIOI_Assert((((ADIO_Offset)(MPIU_Upint)write_buf)+req_off-off) == (ADIO_Offset)(MPIU_Upint)(write_buf+req_off-off));
-                    MPI_Get_address(write_buf + req_off - off,
-                                    &(others_req[i].mem_ptrs[j]));
-                    ADIOI_Assert((off + size - req_off) == (int)(off + size - req_off));
-                    recv_size[i] += (int)(ADIOI_MIN(off + size - req_off,
-                                                    (unsigned)req_len));
-
-                    if (off+size-req_off < (unsigned)req_len)
-                    {
-                        partial_recv[i] = (int)(off + size - req_off);
-
-                        /* --BEGIN ERROR HANDLING-- */
-                        if ((j+1 < others_req[i].count) &&
-                            (others_req[i].offsets[j+1] < off+size))
-                        {
-                            *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-                                    MPIR_ERR_RECOVERABLE,
-                                    myname,
-                                    __LINE__,
-                                    MPI_ERR_ARG,
-                                    "Filetype specifies overlapping write regions (which is illegal according to the MPI-2 specification)", 0);
-                            /* allow to continue since additional
-                             * communication might have to occur
-                             */
-                        }
-                        /* --END ERROR HANDLING-- */
-                        break;
-                    }
-                }
-                else break;
-            }
-            curr_offlen_ptr[i] = j;
-        }
-    }
-
-    /* create a struct for ADIOI_W_Iexchange_data() */
-    wed_vars = (ADIOI_W_Iexchange_data_vars *)ADIOI_Calloc(
-            1, sizeof(ADIOI_W_Iexchange_data_vars));
-    nbc_req->data.wr.wed_vars = wed_vars;
-
-    wed_vars->fd = vars->fd;
-    wed_vars->buf = vars->buf;
-    wed_vars->write_buf = vars->write_buf;
-    wed_vars->flat_buf = vars->flat_buf;
-    wed_vars->offset_list = vars->offset_list;
-    wed_vars->len_list = vars->len_list;
-    wed_vars->send_size = vars->send_size;
-    wed_vars->recv_size = vars->recv_size;
-    wed_vars->off = vars->off;
-    wed_vars->size = vars->size;
-    wed_vars->count = vars->count;
-    wed_vars->start_pos = vars->start_pos;
-    wed_vars->partial_recv = vars->partial_recv;
-    wed_vars->sent_to_proc = vars->sent_to_proc;
-    wed_vars->nprocs = vars->nprocs;
-    wed_vars->myrank = vars->myrank;
-    wed_vars->buftype_is_contig = vars->buftype_is_contig;
-    wed_vars->contig_access_count = vars->contig_access_count;
-    wed_vars->min_st_offset = vars->min_st_offset;
-    wed_vars->fd_size = vars->fd_size;
-    wed_vars->fd_start = vars->fd_start;
-    wed_vars->fd_end = vars->fd_end;
-    wed_vars->others_req = vars->others_req;
-    wed_vars->send_buf_idx = vars->send_buf_idx;
-    wed_vars->curr_to_proc = vars->curr_to_proc;
-    wed_vars->done_to_proc = vars->done_to_proc;
-    wed_vars->hole = &vars->hole;
-    wed_vars->iter = vars->m;
-    wed_vars->buftype_extent = vars->buftype_extent;
-    wed_vars->buf_idx = vars->buf_idx;
-    wed_vars->next_fn = ADIOI_Iexch_and_write_l1_body;
-
-    ADIOI_W_Iexchange_data(nbc_req, error_code);
-}
-
-static void ADIOI_Iexch_and_write_l1_body(ADIOI_NBC_Request *nbc_req,
-                                          int *error_code)
-{
-    ADIOI_Iexch_and_write_vars *vars = nbc_req->data.wr.eaw_vars;
-    ADIO_File fd = vars->fd;
-    int nprocs = vars->nprocs;
-    ADIO_Offset size = vars->size;
-    char *write_buf = vars->write_buf;
-    int *count = vars->count;
-    int flag, i;
-
-    flag = 0;
-    for (i = 0; i < nprocs; i++)
-        if (count[i]) flag = 1;
-
-    if (flag) {
-        ADIOI_Assert(size == (int)size);
-#if defined(ROMIO_RUN_ON_LINUX) && !defined(HAVE_AIO_LITE_H)
-        MPI_Status status;
-        ADIO_WriteContig(fd, write_buf, (int)size, MPI_BYTE,
-                         ADIO_EXPLICIT_OFFSET, vars->off, &status,
-                         error_code);
-#else
-        ADIO_IwriteContig(fd, write_buf, (int)size, MPI_BYTE,
-                          ADIO_EXPLICIT_OFFSET, vars->off, &vars->req3,
-                          error_code);
-
-        nbc_req->data.wr.state = ADIOI_IWC_STATE_IEXCH_AND_WRITE_L1_BODY;
-        return;
-#endif
-    }
-
-    ADIOI_Iexch_and_write_l1_end(nbc_req, error_code);
-}
-
-static void ADIOI_Iexch_and_write_l1_end(ADIOI_NBC_Request *nbc_req,
-                                         int *error_code)
-{
-    ADIOI_Iexch_and_write_vars *vars = nbc_req->data.wr.eaw_vars;
-    ADIO_Offset size = vars->size;
-
-    vars->off += size;
-    vars->done += size;
-
-    /* increment m and go back to the beginning of m loop */
-    vars->m++;
-    ADIOI_Iexch_and_write_l1_begin(nbc_req, error_code);
-}
-
-static void ADIOI_Iexch_and_write_reset(ADIOI_NBC_Request *nbc_req,
-                                        int *error_code)
-{
-    ADIOI_Iexch_and_write_vars *vars = nbc_req->data.wr.eaw_vars;
-    int nprocs = vars->nprocs;
-    int *count = vars->count;
-    int *recv_size = vars->recv_size;
-    int i;
-
-    for (i = 0; i < nprocs; i++) count[i] = recv_size[i] = 0;
-
-    vars->m = vars->ntimes;
-    ADIOI_Iexch_and_write_l2_begin(nbc_req, error_code);
-}
-
-static void ADIOI_Iexch_and_write_l2_begin(ADIOI_NBC_Request *nbc_req,
-                                           int *error_code)
-{
-    ADIOI_Iexch_and_write_vars *vars = nbc_req->data.wr.eaw_vars;
-    ADIO_Offset size = vars->size;
-    ADIOI_W_Iexchange_data_vars *wed_vars = NULL;
-
-    /* loop exit condition */
-    if (vars->m >= vars->max_ntimes) {
-        ADIOI_Iexch_and_write_fini(nbc_req, error_code);
-        return;
-    }
-
-    ADIOI_Assert(size == (int)size);
-
-    /* create a struct for ADIOI_W_Iexchange_data() */
-    wed_vars = (ADIOI_W_Iexchange_data_vars *)ADIOI_Calloc(
-            1, sizeof(ADIOI_W_Iexchange_data_vars));
-    nbc_req->data.wr.wed_vars = wed_vars;
-
-    wed_vars->fd = vars->fd;
-    wed_vars->buf = vars->buf;
-    wed_vars->write_buf = vars->write_buf;
-    wed_vars->flat_buf = vars->flat_buf;
-    wed_vars->offset_list = vars->offset_list;
-    wed_vars->len_list = vars->len_list;
-    wed_vars->send_size = vars->send_size;
-    wed_vars->recv_size = vars->recv_size;
-    wed_vars->off = vars->off;
-    wed_vars->size = (int)vars->size;
-    wed_vars->count = vars->count;
-    wed_vars->start_pos = vars->start_pos;
-    wed_vars->partial_recv = vars->partial_recv;
-    wed_vars->sent_to_proc = vars->sent_to_proc;
-    wed_vars->nprocs = vars->nprocs;
-    wed_vars->myrank = vars->myrank;
-    wed_vars->buftype_is_contig = vars->buftype_is_contig;
-    wed_vars->contig_access_count = vars->contig_access_count;
-    wed_vars->min_st_offset = vars->min_st_offset;
-    wed_vars->fd_size = vars->fd_size;
-    wed_vars->fd_start = vars->fd_start;
-    wed_vars->fd_end = vars->fd_end;
-    wed_vars->others_req = vars->others_req;
-    wed_vars->send_buf_idx = vars->send_buf_idx;
-    wed_vars->curr_to_proc = vars->curr_to_proc;
-    wed_vars->done_to_proc = vars->done_to_proc;
-    wed_vars->hole = &vars->hole;
-    wed_vars->iter = vars->m;
-    wed_vars->buftype_extent = vars->buftype_extent;
-    wed_vars->buf_idx = vars->buf_idx;
-    wed_vars->next_fn = ADIOI_Iexch_and_write_l2_end;
-
-    /* nothing to recv, but check for send. */
-    ADIOI_W_Iexchange_data(nbc_req, error_code);
-}
-
-static void ADIOI_Iexch_and_write_l2_end(ADIOI_NBC_Request *nbc_req,
-                                         int *error_code)
-{
-    ADIOI_Iexch_and_write_vars *vars = nbc_req->data.wr.eaw_vars;
-
-    vars->m++;
-    ADIOI_Iexch_and_write_l2_begin(nbc_req, error_code);
-}
-
-static void ADIOI_Iexch_and_write_fini(ADIOI_NBC_Request *nbc_req, int *error_code)
-{
-    ADIOI_Iexch_and_write_vars *vars = nbc_req->data.wr.eaw_vars;
-    void (*next_fn)(ADIOI_NBC_Request *, int *);
-
-    ADIOI_Free(vars->curr_offlen_ptr);
-    ADIOI_Free(vars->count);
-    ADIOI_Free(vars->partial_recv);
-    ADIOI_Free(vars->send_size);
-    ADIOI_Free(vars->recv_size);
-    ADIOI_Free(vars->sent_to_proc);
-    ADIOI_Free(vars->start_pos);
-    ADIOI_Free(vars->send_buf_idx);
-    ADIOI_Free(vars->curr_to_proc);
-    ADIOI_Free(vars->done_to_proc);
-
-    next_fn = vars->next_fn;
-
-    /* free the struct for parameters and variables */
-    ADIOI_Free(nbc_req->data.wr.eaw_vars);
-    nbc_req->data.wr.eaw_vars = NULL;
-
-    /* move to the next function */
-    next_fn(nbc_req, error_code);
-}
-
-
-static void ADIOI_W_Iexchange_data(ADIOI_NBC_Request *nbc_req, int *error_code)
-{
-    ADIOI_W_Iexchange_data_vars *vars = nbc_req->data.wr.wed_vars;
-
-    /* exchange recv_size info so that each process knows how much to
-       send to whom. */
-
-    *error_code = MPI_Ialltoall(vars->recv_size, 1, MPI_INT, vars->send_size, 1,
-                                MPI_INT, vars->fd->comm, &vars->req1);
-
-    nbc_req->data.wr.state = ADIOI_IWC_STATE_W_IEXCHANGE_DATA;
-}
-
-static void ADIOI_W_Iexchange_data_hole(ADIOI_NBC_Request *nbc_req,
-                                        int *error_code)
-{
-    ADIOI_W_Iexchange_data_vars *vars = nbc_req->data.wr.wed_vars;
-    ADIO_File fd = vars->fd;
-    int *recv_size = vars->recv_size;
-    ADIO_Offset off = vars->off;
-    int size = vars->size;
-    int *count = vars->count;
-    int *start_pos = vars->start_pos;
-    int *partial_recv = vars->partial_recv;
-    int nprocs = vars->nprocs;
-    ADIOI_Access *others_req = vars->others_req;
-    int *hole = vars->hole;
-
-    int i, j, k, *tmp_len, nprocs_recv;
-    MPI_Datatype *recv_types;
-    int *srt_len = NULL, sum;
-    ADIO_Offset *srt_off = NULL;
-
-    /* create derived datatypes for recv */
-
-    nprocs_recv = 0;
-    for (i = 0; i < nprocs; i++) if (recv_size[i]) nprocs_recv++;
-    vars->nprocs_recv = nprocs_recv;
-
-    recv_types = (MPI_Datatype *)
-        ADIOI_Malloc((nprocs_recv+1)*sizeof(MPI_Datatype));
-    vars->recv_types = recv_types;
-    /* +1 to avoid a 0-size malloc */
-
-    tmp_len = (int *)ADIOI_Malloc(nprocs*sizeof(int));
-    j = 0;
-    for (i = 0; i < nprocs; i++) {
-        if (recv_size[i]) {
-            /* take care if the last off-len pair is a partial recv */
-            if (partial_recv[i]) {
-                k = start_pos[i] + count[i] - 1;
-                tmp_len[i] = others_req[i].lens[k];
-                others_req[i].lens[k] = partial_recv[i];
-            }
-            ADIOI_Type_create_hindexed_x(count[i],
-                    &(others_req[i].lens[start_pos[i]]),
-                    &(others_req[i].mem_ptrs[start_pos[i]]),
-                    MPI_BYTE, recv_types+j);
-            /* absolute displacements; use MPI_BOTTOM in recv */
-            MPI_Type_commit(recv_types+j);
-            j++;
-        }
-    }
-
-    /* To avoid a read-modify-write, check if there are holes in the
-       data to be written. For this, merge the (sorted) offset lists
-       others_req using a heap-merge. */
-
-    sum = 0;
-    for (i = 0; i < nprocs; i++) sum += count[i];
-    /* valgrind-detcted optimization: if there is no work on this process we do
-     * not need to search for holes */
-    if (sum) {
-        srt_off = (ADIO_Offset *)ADIOI_Malloc(sum*sizeof(ADIO_Offset));
-        srt_len = (int *)ADIOI_Malloc(sum*sizeof(int));
-
-        ADIOI_Heap_merge(others_req, count, srt_off, srt_len, start_pos,
-                         nprocs, nprocs_recv, sum);
-    }
-
-    /* for partial recvs, restore original lengths */
-    for (i = 0; i < nprocs; i++)
-        if (partial_recv[i]) {
-            k = start_pos[i] + count[i] - 1;
-            others_req[i].lens[k] = tmp_len[i];
-        }
-    ADIOI_Free(tmp_len);
-
-    /* check if there are any holes. If yes, must do read-modify-write.
-     * holes can be in three places.  'middle' is what you'd expect: the
-     * processes are operating on noncontigous data.  But holes can also show
-     * up at the beginning or end of the file domain (see John Bent ROMIO REQ
-     * #835). Missing these holes would result in us writing more data than
-     * recieved by everyone else. */
-
-    *hole = 0;
-    if (sum) {
-        if (off != srt_off[0]) /* hole at the front */
-            *hole = 1;
-        else { /* coalesce the sorted offset-length pairs */
-            for (i = 1; i < sum; i++) {
-                if (srt_off[i] <= srt_off[0] + srt_len[0]) {
-                    /* ok to cast: operating on cb_buffer_size chunks */
-                    int new_len = (int)srt_off[i] + srt_len[i] - (int)srt_off[0];
-                    if (new_len > srt_len[0]) srt_len[0] = new_len;
-                }
-                else
-                    break;
-            }
-            if (i < sum || size != srt_len[0]) /* hole in middle or end */
-                *hole = 1;
-        }
-
-        ADIOI_Free(srt_off);
-        ADIOI_Free(srt_len);
-    }
-
-    if (nprocs_recv) {
-        if (*hole) {
-            ADIO_IreadContig(fd, vars->write_buf, size, MPI_BYTE,
-                             ADIO_EXPLICIT_OFFSET, off, &vars->req2,
-                             &vars->err);
-            nbc_req->data.wr.state = ADIOI_IWC_STATE_W_IEXCHANGE_DATA_HOLE;
-            return;
-        }
-    }
-
-    ADIOI_W_Iexchange_data_send(nbc_req, error_code);
-}
-
-static void ADIOI_W_Iexchange_data_send(ADIOI_NBC_Request *nbc_req,
-                                        int *error_code)
-{
-    ADIOI_W_Iexchange_data_vars *vars = nbc_req->data.wr.wed_vars;
-    ADIO_File fd = vars->fd;
-    void *buf = vars->buf;
-    int *send_size = vars->send_size;
-    int *recv_size = vars->recv_size;
-    int nprocs = vars->nprocs;
-    int myrank = vars->myrank;
-    int iter = vars->iter;
-    int *buf_idx = vars->buf_idx;
-
-    int nprocs_recv = vars->nprocs_recv;
-    MPI_Datatype *recv_types = vars->recv_types;
-
-    int i, j;
-    int nprocs_send;
-    char **send_buf = NULL;
-
-    nprocs_send = 0;
-    for (i = 0; i < nprocs; i++) if (send_size[i]) nprocs_send++;
-    vars->nprocs_send = nprocs_send;
-
-    if (fd->atomicity) {
-        /* bug fix from Wei-keng Liao and Kenin Coloma */
-        vars->requests = (MPI_Request *)
-            ADIOI_Malloc((nprocs_send+1)*sizeof(MPI_Request));
-        vars->send_req = vars->requests;
-    }
-    else {
-        vars->requests = (MPI_Request *)
-            ADIOI_Malloc((nprocs_send+nprocs_recv+1)*sizeof(MPI_Request));
-        /* +1 to avoid a 0-size malloc */
-
-        /* post receives */
-        j = 0;
-        for (i = 0; i < nprocs; i++) {
-            if (recv_size[i]) {
-                MPI_Irecv(MPI_BOTTOM, 1, recv_types[j], i, myrank+i+100*iter,
-                          fd->comm, vars->requests+j);
-                j++;
-            }
-        }
-        vars->send_req = vars->requests + nprocs_recv;
-    }
-
-    /* post sends. if buftype_is_contig, data can be directly sent from
-       user buf at location given by buf_idx. else use send_buf. */
-
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5032, 0, NULL);
-#endif
-    if (vars->buftype_is_contig) {
-        j = 0;
-        for (i = 0; i < nprocs; i++)
-            if (send_size[i]) {
-                MPI_Isend(((char *) buf) + buf_idx[i], send_size[i],
-                          MPI_BYTE, i,  myrank+i+100*iter, fd->comm,
-                          vars->send_req+j);
-                j++;
-                buf_idx[i] += send_size[i];
-            }
-    }
-    else if (nprocs_send) {
-        /* buftype is not contig */
-        send_buf = (char **)ADIOI_Malloc(nprocs*sizeof(char*));
-        vars->send_buf = send_buf;
-        for (i = 0; i < nprocs; i++)
-            if (send_size[i])
-                send_buf[i] = (char *)ADIOI_Malloc(send_size[i]);
-
-        ADIOI_Fill_send_buffer(fd, buf, vars->flat_buf, send_buf,
-                               vars->offset_list, vars->len_list, send_size,
-                               vars->send_req,
-                               vars->sent_to_proc, nprocs, myrank,
-                               vars->contig_access_count,
-                               vars->min_st_offset, vars->fd_size,
-                               vars->fd_start, vars->fd_end,
-                               vars->send_buf_idx, vars->curr_to_proc,
-                               vars->done_to_proc, iter,
-                               vars->buftype_extent);
-        /* the send is done in ADIOI_Fill_send_buffer */
-    }
-
-    if (fd->atomicity) {
-        vars->req3 = (MPI_Request *)
-            ADIOI_Malloc((nprocs_recv+1)*sizeof(MPI_Request));
-        /* +1 to avoid a 0-size malloc */
-
-        /* bug fix from Wei-keng Liao and Kenin Coloma */
-        j = 0;
-        for (i = 0; i < nprocs; i++) {
-            if (recv_size[i]) {
-                MPI_Irecv(MPI_BOTTOM, 1, recv_types[j], i, myrank+i+100*iter,
-                          fd->comm, vars->req3+j);
-                j++;
-            }
-        }
-
-        nbc_req->data.wr.state = ADIOI_IWC_STATE_W_IEXCHANGE_DATA_SEND;
-        return;
-    }
-
-    ADIOI_W_Iexchange_data_wait(nbc_req, error_code);
-}
-
-static void ADIOI_W_Iexchange_data_wait(ADIOI_NBC_Request *nbc_req,
-                                        int *error_code)
-{
-    ADIOI_W_Iexchange_data_vars *vars = nbc_req->data.wr.wed_vars;
-    ADIO_File fd = vars->fd;
-    int nprocs_send = vars->nprocs_send;
-    int nprocs_recv = vars->nprocs_recv;
-    MPI_Datatype *recv_types = vars->recv_types;
-    int i;
-
-    for (i = 0; i < nprocs_recv; i++) MPI_Type_free(recv_types+i);
-    ADIOI_Free(recv_types);
-
-    i= 0;
-    if (fd->atomicity) {
-        /* bug fix from Wei-keng Liao and Kenin Coloma */
-        MPI_Testall(nprocs_send, vars->send_req, &i, MPI_STATUSES_IGNORE);
-    }
-    else {
-        MPI_Testall(nprocs_send+nprocs_recv, vars->requests, &i,
-                    MPI_STATUSES_IGNORE);
-    }
-
-    if (i) {
-        ADIOI_W_Iexchange_data_fini(nbc_req, error_code);
-    } else {
-        nbc_req->data.wr.state = ADIOI_IWC_STATE_W_IEXCHANGE_DATA_WAIT;
-    }
-}
-
-static void ADIOI_W_Iexchange_data_fini(ADIOI_NBC_Request *nbc_req, int *error_code)
-{
-    ADIOI_W_Iexchange_data_vars *vars = nbc_req->data.wr.wed_vars;
-    void (*next_fn)(ADIOI_NBC_Request *, int *);
-    ADIO_File fd = vars->fd;
-    int *send_size = vars->send_size;
-    int nprocs = vars->nprocs;
-    char **send_buf = vars->send_buf;
-    int i;
-
-    if (fd->atomicity) ADIOI_Free(vars->req3);
-
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5033, 0, NULL);
-#endif
-    ADIOI_Free(vars->requests);
-    if (!vars->buftype_is_contig && vars->nprocs_send) {
-        for (i = 0; i < nprocs; i++)
-            if (send_size[i]) ADIOI_Free(send_buf[i]);
-        ADIOI_Free(send_buf);
-    }
-
-    next_fn = vars->next_fn;
-
-    /* free the structure for parameters and variables */
-    ADIOI_Free(vars);
-    nbc_req->data.wr.wed_vars = NULL;
-
-    /* move to the next function */
-    next_fn(nbc_req, error_code);
-}
-
-
-static int ADIOI_GEN_iwc_query_fn(void *extra_state, MPI_Status *status)
-{
-    ADIOI_NBC_Request *nbc_req;
-
-    nbc_req = (ADIOI_NBC_Request *)extra_state;
-
-    MPI_Status_set_elements_x(status, MPI_BYTE, nbc_req->nbytes);
-
-    /* can never cancel so always true */
-    MPI_Status_set_cancelled(status, 0);
-
-    /* choose not to return a value for this */
-    status->MPI_SOURCE = MPI_UNDEFINED;
-    /* tag has no meaning for this generalized request */
-    status->MPI_TAG = MPI_UNDEFINED;
-
-    /* this generalized request never fails */
-    return MPI_SUCCESS;
-}
-
-static int ADIOI_GEN_iwc_free_fn(void *extra_state)
-{
-    ADIOI_NBC_Request *nbc_req;
-
-    nbc_req = (ADIOI_NBC_Request *)extra_state;
-    ADIOI_Free(nbc_req);
-
-    return MPI_SUCCESS;
-}
-
-static int ADIOI_GEN_iwc_poll_fn(void *extra_state, MPI_Status *status)
-{
-    ADIOI_NBC_Request *nbc_req;
-    ADIOI_GEN_IwriteStridedColl_vars *wsc_vars = NULL;
-    ADIOI_Icalc_others_req_vars      *cor_vars = NULL;
-    ADIOI_Iexch_and_write_vars       *eaw_vars = NULL;
-    ADIOI_W_Iexchange_data_vars      *wed_vars = NULL;
-    int errcode = MPI_SUCCESS;
-    int flag;
-
-    nbc_req = (ADIOI_NBC_Request *)extra_state;
-
-    switch (nbc_req->data.wr.state) {
-        case ADIOI_IWC_STATE_GEN_IWRITESTRIDEDCOLL:
-            wsc_vars = nbc_req->data.wr.wsc_vars;
-            errcode = MPI_Testall(2, wsc_vars->req_offset, &flag,
-                                  MPI_STATUSES_IGNORE);
-            if (errcode == MPI_SUCCESS && flag) {
-                ADIOI_GEN_IwriteStridedColl_inter(nbc_req, &errcode);
-            }
-            break;
-
-        case ADIOI_IWC_STATE_GEN_IWRITESTRIDEDCOLL_INDIO:
-            wsc_vars = nbc_req->data.wr.wsc_vars;
-            errcode = MPI_Test(&wsc_vars->req_ind_io, &flag, MPI_STATUS_IGNORE);
-            if (errcode == MPI_SUCCESS && flag) {
-                /* call the last function */
-                ADIOI_GEN_IwriteStridedColl_fini(nbc_req, &errcode);
-            }
-            break;
-
-        case ADIOI_IWC_STATE_GEN_IWRITESTRIDEDCOLL_BCAST:
-            wsc_vars = nbc_req->data.wr.wsc_vars;
-            errcode = MPI_Test(&wsc_vars->req_err, &flag, MPI_STATUS_IGNORE);
-            if (errcode == MPI_SUCCESS && flag) {
-                errcode = wsc_vars->error_code;
-                ADIOI_GEN_IwriteStridedColl_free(nbc_req, &errcode);
-            }
-            break;
-
-        case ADIOI_IWC_STATE_ICALC_OTHERS_REQ:
-            cor_vars = nbc_req->cor_vars;
-            errcode = MPI_Test(&cor_vars->req1, &flag, MPI_STATUS_IGNORE);
-            if (errcode == MPI_SUCCESS && flag) {
-                ADIOI_Icalc_others_req_main(nbc_req, &errcode);
-            }
-            break;
-
-        case ADIOI_IWC_STATE_ICALC_OTHERS_REQ_MAIN:
-            cor_vars = nbc_req->cor_vars;
-            if (cor_vars->num_req2) {
-                errcode = MPI_Testall(cor_vars->num_req2, cor_vars->req2,
-                                      &flag, MPI_STATUSES_IGNORE);
-                if (errcode == MPI_SUCCESS && flag) {
-                    ADIOI_Icalc_others_req_fini(nbc_req, &errcode);
-                }
-            } else {
-                ADIOI_Icalc_others_req_fini(nbc_req, &errcode);
-            }
-            break;
-
-        case ADIOI_IWC_STATE_IEXCH_AND_WRITE:
-            eaw_vars = nbc_req->data.wr.eaw_vars;
-            errcode = MPI_Test(&eaw_vars->req1, &flag, MPI_STATUS_IGNORE);
-            if (errcode == MPI_SUCCESS && flag) {
-                eaw_vars->m = 0;
-                ADIOI_Iexch_and_write_l1_begin(nbc_req, &errcode);
-            }
-            break;
-
-        case ADIOI_IWC_STATE_IEXCH_AND_WRITE_L1_BODY:
-            eaw_vars = nbc_req->data.wr.eaw_vars;
-            errcode = MPI_Test(&eaw_vars->req3, &flag, MPI_STATUS_IGNORE);
-            if (errcode == MPI_SUCCESS && flag) {
-                ADIOI_Iexch_and_write_l1_end(nbc_req, &errcode);
-            }
-            break;
-
-        case ADIOI_IWC_STATE_W_IEXCHANGE_DATA:
-            wed_vars = nbc_req->data.wr.wed_vars;
-            errcode = MPI_Test(&wed_vars->req1, &flag, MPI_STATUS_IGNORE);
-            if (errcode == MPI_SUCCESS && flag) {
-                ADIOI_W_Iexchange_data_hole(nbc_req, &errcode);
-            }
-            break;
-
-        case ADIOI_IWC_STATE_W_IEXCHANGE_DATA_HOLE:
-            wed_vars = nbc_req->data.wr.wed_vars;
-            errcode = MPI_Test(&wed_vars->req2, &flag, MPI_STATUSES_IGNORE);
-            if (errcode == MPI_SUCCESS && flag) {
-                /* --BEGIN ERROR HANDLING-- */
-                if (wed_vars->err != MPI_SUCCESS) {
-                    errcode = MPIO_Err_create_code(wed_vars->err,
-                            MPIR_ERR_RECOVERABLE,
-                            "ADIOI_W_EXCHANGE_DATA",
-                            __LINE__, MPI_ERR_IO,
-                            "**ioRMWrdwr", 0);
-                    break;;
-                }
-                /* --END ERROR HANDLING-- */
-                ADIOI_W_Iexchange_data_send(nbc_req, &errcode);
-            }
-            break;
-
-        case ADIOI_IWC_STATE_W_IEXCHANGE_DATA_SEND:
-            wed_vars = nbc_req->data.wr.wed_vars;
-            errcode = MPI_Testall(wed_vars->nprocs_recv, wed_vars->req3,
-                                  &flag, MPI_STATUSES_IGNORE);
-            if (errcode == MPI_SUCCESS && flag) {
-                ADIOI_W_Iexchange_data_wait(nbc_req, &errcode);
-            }
-            break;
-
-        case ADIOI_IWC_STATE_W_IEXCHANGE_DATA_WAIT:
-            wed_vars = nbc_req->data.wr.wed_vars;
-            if (wed_vars->fd->atomicity) {
-                /* bug fix from Wei-keng Liao and Kenin Coloma */
-                errcode = MPI_Testall(wed_vars->nprocs_send, wed_vars->send_req,
-                                      &flag, MPI_STATUSES_IGNORE);
-            } else {
-                errcode = MPI_Testall(wed_vars->nprocs_send +
-                                      wed_vars->nprocs_recv,
-                                      wed_vars->requests,
-                                      &flag, MPI_STATUSES_IGNORE);
-            }
-            if (errcode == MPI_SUCCESS && flag) {
-                ADIOI_W_Iexchange_data_fini(nbc_req, &errcode);
-            }
-            break;
-
-        default:
-            break;
-    }
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (errcode != MPI_SUCCESS) {
-        errcode = MPIO_Err_create_code(MPI_SUCCESS,
-                MPIR_ERR_RECOVERABLE,
-                "ADIOI_GEN_iwc_poll_fn", __LINE__,
-                MPI_ERR_IO, "**mpi_grequest_complete",
-                0);
-    }
-    /* --END ERROR HANDLING-- */
-
-    return errcode;
-}
-
-/* wait for multiple requests to complete */
-static int ADIOI_GEN_iwc_wait_fn(int count, void **array_of_states,
-                                 double timeout, MPI_Status *status)
-{
-    int i, errcode = MPI_SUCCESS;
-    double starttime;
-    ADIOI_NBC_Request **nbc_reqlist;
-
-    nbc_reqlist = (ADIOI_NBC_Request **)array_of_states;
-
-    starttime = MPI_Wtime();
-    for (i = 0; i < count ; i++) {
-        while (nbc_reqlist[i]->data.wr.state != ADIOI_IWC_STATE_COMPLETE) {
-            errcode = ADIOI_GEN_iwc_poll_fn(nbc_reqlist[i], MPI_STATUS_IGNORE);
-            /* --BEGIN ERROR HANDLING-- */
-            if (errcode != MPI_SUCCESS) {
-                errcode = MPIO_Err_create_code(MPI_SUCCESS,
-                        MPIR_ERR_RECOVERABLE,
-                        "ADIOI_GEN_iwc_wait_fn",
-                        __LINE__, MPI_ERR_IO,
-                        "**mpi_grequest_complete", 0);
-            }
-            /* --END ERROR HANDLING-- */
-
-            if ((timeout > 0) && (timeout < (MPI_Wtime() - starttime)))
-                goto fn_exit;
-
-            /* If the progress engine is blocked, we have to yield for another
-             * thread to be able to unblock the progress engine. */
-            MPIR_Ext_cs_yield();
-        }
-    }
-
-  fn_exit:
-    return errcode;
-}
-
-#endif /* HAVE_MPI_GREQUEST_EXTENSIONS */
diff --git a/3rd-party/romio321/adio/common/ad_iwrite_coll.pdf b/3rd-party/romio321/adio/common/ad_iwrite_coll.pdf
deleted file mode 100644
index 8efe138a1e3..00000000000
Binary files a/3rd-party/romio321/adio/common/ad_iwrite_coll.pdf and /dev/null differ
diff --git a/3rd-party/romio321/adio/common/ad_iwrite_fake.c b/3rd-party/romio321/adio/common/ad_iwrite_fake.c
deleted file mode 100644
index 6210942e663..00000000000
--- a/3rd-party/romio321/adio/common/ad_iwrite_fake.c
+++ /dev/null
@@ -1,65 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2004 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-
-#include "mpiu_greq.h"
-
-/* Generic implementation of IwriteContig calls the blocking WriteContig
- * immediately.
- */
-void ADIOI_FAKE_IwriteContig(ADIO_File fd, const void *buf, int count,
-			    MPI_Datatype datatype, int file_ptr_type,
-			    ADIO_Offset offset, ADIO_Request *request,
-			    int *error_code)
-{
-    ADIO_Status status;
-    MPI_Offset len;
-    MPI_Count typesize;
-    MPI_Offset nbytes=0;
-
-    MPI_Type_size_x(datatype, &typesize);
-    len = (MPI_Offset)count * (MPI_Offset)typesize;
-
-    /* Call the blocking function.  It will create an error code
-     * if necessary.
-     */
-    ADIOI_Assert(len == (int) len); /* the count is an int parm */
-    ADIO_WriteContig(fd, buf, (int)len, MPI_BYTE, file_ptr_type, offset,
-		     &status, error_code);  
-    if (*error_code == MPI_SUCCESS) {
-	MPI_Type_size_x(datatype, &typesize);
-	nbytes = (MPI_Offset)count*(MPI_Offset)typesize;
-    }
-    MPIO_Completed_request_create(&fd, nbytes, error_code, request);
-
-}
-
-
-/* Generic implementation of IwriteStrided calls the blocking WriteStrided
- * immediately.
- */
-void ADIOI_FAKE_IwriteStrided(ADIO_File fd, const void *buf, int count,
-                             MPI_Datatype datatype, int file_ptr_type,
-			     ADIO_Offset offset, ADIO_Request *request,
-			     int *error_code)
-{
-    ADIO_Status status;
-    MPI_Count typesize;
-    MPI_Offset nbytes=0;
-
-    /* Call the blocking function.  It will create an error code 
-     * if necessary.
-     */
-    ADIO_WriteStrided(fd, buf, count, datatype, file_ptr_type, 
-		      offset, &status, error_code);  
-    if (*error_code == MPI_SUCCESS) {
-	MPI_Type_size_x(datatype, &typesize);
-	nbytes = (MPI_Offset)count * (MPI_Offset)typesize;
-    }
-    MPIO_Completed_request_create(&fd, nbytes, error_code, request);
-}
diff --git a/3rd-party/romio321/adio/common/ad_open.c b/3rd-party/romio321/adio/common/ad_open.c
deleted file mode 100644
index ced256d3494..00000000000
--- a/3rd-party/romio321/adio/common/ad_open.c
+++ /dev/null
@@ -1,337 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-#include "adio_cb_config_list.h"
-
-#include "mpio.h"
-static int is_aggregator(int rank, ADIO_File fd);
-static int uses_generic_read(ADIO_File fd);
-static int uses_generic_write(ADIO_File fd);
-static int build_cb_config_list(ADIO_File fd, 
-	MPI_Comm orig_comm, MPI_Comm comm, 
-	int rank, int procs, int *error_code);
-
-MPI_File ADIO_Open(MPI_Comm orig_comm,
-		   MPI_Comm comm, const char *filename, int file_system,
-		   ADIOI_Fns *ops,
-		   int access_mode, ADIO_Offset disp, MPI_Datatype etype, 
-		   MPI_Datatype filetype,
-		   MPI_Info info, int perm, int *error_code)
-{
-    MPI_File mpi_fh;
-    ADIO_File fd;
-    int err, rank, procs;
-    static char myname[] = "ADIO_OPEN";
-    int  max_error_code;
-    MPI_Info dupinfo;
-    int syshints_processed, can_skip;
-    char *p;
-
-    *error_code = MPI_SUCCESS;
-
-    /* obtain MPI_File handle */
-    mpi_fh = MPIO_File_create(sizeof(struct ADIOI_FileD));
-    if (mpi_fh == MPI_FILE_NULL) {
-	fd = MPI_FILE_NULL;
-	*error_code = MPIO_Err_create_code(*error_code,
-					   MPIR_ERR_RECOVERABLE,
-					   myname,
-					   __LINE__,
-					   MPI_ERR_OTHER,
-					   "**nomem2",0);
-	goto fn_exit;
-
-    }
-    fd = MPIO_File_resolve(mpi_fh);
-
-    fd->cookie = ADIOI_FILE_COOKIE;
-    fd->fp_ind = disp;
-    fd->fp_sys_posn = 0;
-    fd->comm = comm;       /* dup'ed in MPI_File_open */
-    fd->filename = ADIOI_Strdup(filename);
-    fd->file_system = file_system;
-    fd->fs_ptr = NULL;
-
-    fd->fns = ops;
-
-    fd->disp = disp;
-    fd->split_coll_count = 0;
-    fd->shared_fp_fd = ADIO_FILE_NULL;
-    fd->atomicity = 0;
-    fd->etype = etype;          /* MPI_BYTE by default */
-    fd->filetype = filetype;    /* MPI_BYTE by default */
-    fd->etype_size = 1;  /* default etype is MPI_BYTE */
-
-    fd->file_realm_st_offs = NULL;
-    fd->file_realm_types = NULL;
-
-    fd->perm = perm;
-
-    fd->async_count = 0;
-
-    fd->fortran_handle = -1;
-
-    fd->err_handler = ADIOI_DFLT_ERR_HANDLER;
-
-    fd->io_buf_window = MPI_WIN_NULL;
-    fd->io_buf_put_amounts_window = MPI_WIN_NULL;
-
-    MPI_Comm_rank(comm, &rank);
-    MPI_Comm_size(comm, &procs);
-/* create and initialize info object */
-    fd->hints = (ADIOI_Hints *)ADIOI_Calloc(1, sizeof(struct ADIOI_Hints_struct));
-    if (fd->hints == NULL) {
-	*error_code = MPIO_Err_create_code(*error_code,
-					   MPIR_ERR_RECOVERABLE,
-					   myname,
-					   __LINE__,
-					   MPI_ERR_OTHER,
-					   "**nomem2",0);
-	goto fn_exit;
-    }
-    fd->hints->cb_config_list = NULL;
-    fd->hints->ranklist = NULL;
-    fd->hints->initialized = 0;
-    fd->info = MPI_INFO_NULL;
-
-    /* move system-wide hint processing *back* into open, but this time the
-     * hintfile reader will do a scalable read-and-broadcast.  The global
-     * ADIOI_syshints will get initialized at first open.  subsequent open
-     * calls will just use result from first open.
-     *
-     * We have two goals here:
-     * 1: avoid processing the hintfile multiple times
-     * 2: have all processes participate in hintfile processing (so we can read-and-broadcast)
-     *
-     * a code might do an "initialize from 0", so we can only skip hint
-     * processing once everyone has particpiated in hint processing */
-    if (ADIOI_syshints == MPI_INFO_NULL)
-	syshints_processed = 0;
-    else
-	syshints_processed = 1;
-
-    MPI_Allreduce(&syshints_processed, &can_skip, 1, MPI_INT, MPI_MIN, fd->comm);
-    if (!can_skip) {
-	if (ADIOI_syshints == MPI_INFO_NULL)
-	    MPI_Info_create(&ADIOI_syshints);
-	ADIOI_process_system_hints(fd, ADIOI_syshints);
-    }
-
-    ADIOI_incorporate_system_hints(info, ADIOI_syshints, &dupinfo);
-    ADIO_SetInfo(fd, dupinfo, &err);
-    if (dupinfo != MPI_INFO_NULL) {
-	*error_code = MPI_Info_free(&dupinfo);
-	if (*error_code != MPI_SUCCESS)
-	    goto fn_exit;
-    }
-    ADIOI_Info_set(fd->info, "romio_filesystem_type", fd->fns->fsname);
-
-    /* Instead of repeatedly allocating this buffer in collective read/write,
-     * allocating up-front might make memory management on small platforms
-     * (e.g. Blue Gene) more efficent */
-
-    fd->io_buf = ADIOI_Malloc(fd->hints->cb_buffer_size);
-     /* deferred open: 
-     * we can only do this optimization if 'fd->hints->deferred_open' is set
-     * (which means the user hinted 'no_indep_rw' and collective buffering).
-     * Furthermore, we only do this if our collective read/write routines use
-     * our generic function, and not an fs-specific routine (we can defer opens
-     * only if we use our aggreagation code). */
-    if (fd->hints->deferred_open && 
-		    !(uses_generic_read(fd) \
-			    && uses_generic_write(fd))) {
-	    fd->hints->deferred_open = 0;
-    }
-    if (ADIO_Feature(fd, ADIO_SCALABLE_OPEN))
-	    /* disable deferred open on these fs so that scalable broadcast
-	     * will always use the propper communicator */
-	    fd->hints->deferred_open = 0;
-
-
-    /* on BlueGene, the cb_config_list is built when hints are processed. No
-     * one else does that right now */
-    if (fd->hints->ranklist == NULL) {
-	build_cb_config_list(fd, orig_comm, comm, rank, procs, error_code);
-	if (*error_code != MPI_SUCCESS) 
-	    goto fn_exit;
-    }
-    fd->is_open = 0;
-    fd->my_cb_nodes_index = -2;
-    fd->is_agg = is_aggregator(rank, fd);
-    /* deferred open used to split the communicator to create an "aggregator
-     * communicator", but we only used it as a way to indicate that deferred
-     * open happened.  fd->is_open and fd->is_agg are sufficient */
-
-    /* actual opens start here */
-    /* generic open: one process opens to create the file, all others open */
-    /* nfs open: everybody opens or else you'll end up with "file not found"
-     * due to stupid nfs consistency semantics */
-    /* scalable open: one process opens and broadcasts results to everyone */
-
-    ADIOI_OpenColl(fd, rank, access_mode, error_code);
-
-    /* deferred open consideration: if an independent process lied about
-     * "no_indep_rw" and opens the file later (example: HDF5 uses independent
-     * i/o for metadata), that deferred open will use the access_mode provided
-     * by the user.  CREATE|EXCL only makes sense here -- exclusive access in
-     * the deferred open case is going to fail and surprise the user.  Turn off
-     * the excl amode bit. Save user's ammode for MPI_FILE_GET_AMODE */
-    fd->orig_access_mode = access_mode;
-    if (fd->access_mode & ADIO_EXCL) fd->access_mode ^= ADIO_EXCL;
-
-
-    /* for debugging, it can be helpful to see the hints selected. Some file
-     * systes set up the hints in the open call (e.g. lustre) */
-    p = getenv("ROMIO_PRINT_HINTS");
-    if (rank == 0 && p != NULL ) {
-	ADIOI_Info_print_keyvals(fd->info);
-    }
-
- fn_exit:
-    MPI_Allreduce(error_code, &max_error_code, 1, MPI_INT, MPI_MAX, comm);
-    if (max_error_code != MPI_SUCCESS) {
-
-        /* If the file was successfully opened, close it */
-        if (*error_code == MPI_SUCCESS) {
-        
-            /* in the deferred open case, only those who have actually
-               opened the file should close it */
-            if (fd->hints->deferred_open)  {
-                if (fd->is_agg) {
-                    (*(fd->fns->ADIOI_xxx_Close))(fd, error_code);
-                }
-            }
-            else {
-                (*(fd->fns->ADIOI_xxx_Close))(fd, error_code);
-            }
-        }
-	ADIOI_Free(fd->filename);
-	ADIOI_Free(fd->hints->ranklist);
-	if ( fd->hints->cb_config_list != NULL ) ADIOI_Free(fd->hints->cb_config_list);
-	ADIOI_Free(fd->hints);
-	if (fd->info != MPI_INFO_NULL) MPI_Info_free(&(fd->info));
-	ADIOI_Free(fd->io_buf);
-	ADIOI_Free(fd);
-        fd = ADIO_FILE_NULL;
-	if (*error_code == MPI_SUCCESS)
-	{
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE, myname,
-					       __LINE__, MPI_ERR_IO,
-					       "**oremote_fail", 0);
-	}
-    }
-
-    return fd;
-}
-
-/* a simple linear search. possible enancement: add a my_cb_nodes_index member
- * ( index into cb_nodes, else -1 if not aggregator ) for faster lookups 
- *
- * fd->hints->cb_nodes is the number of aggregators
- * fd->hints->ranklist[] is an array of the ranks of aggregators
- *
- * might want to move this to adio/common/cb_config_list.c 
- */
-int is_aggregator(int rank, ADIO_File fd ) {
-        int i;
-        
-	if (fd->my_cb_nodes_index == -2) {
-	    for (i=0; i< fd->hints->cb_nodes; i++ ) {
-		if ( rank == fd->hints->ranklist[i] ) {
-		    fd->my_cb_nodes_index = i;
-		    return 1;
-		}
-	    }
-	    fd->my_cb_nodes_index = -1;
-        }
-	else if (fd->my_cb_nodes_index != -1)
-	    return 1;
-
-        return 0;
-}
-
-/*
- * If file system implements some version of two-phase -- doesn't have to be
- * generic -- we can still carry out the defered open optimization
- */
-static int uses_generic_read(ADIO_File fd)
-{
-    if (ADIO_Feature(fd, ADIO_TWO_PHASE))
-        return 1;
-    return 0;
-}
-
-static int uses_generic_write(ADIO_File fd)
-{
-    if (ADIO_Feature(fd, ADIO_TWO_PHASE))
-        return 1;
-    return 0;
-}
-
-static int build_cb_config_list(ADIO_File fd, 
-	MPI_Comm orig_comm, MPI_Comm comm, 
-	int rank, int procs, int *error_code)
-{
-    ADIO_cb_name_array array;
-    int *tmp_ranklist;
-    int rank_ct;
-    char *value;
-    static char myname[] = "ADIO_OPEN cb_config_list";
-
-    /* gather the processor name array if we don't already have it */
-    /* this has to be done early in ADIO_Open so that we can cache the name
-     * array in both the dup'd communicator (in case we want it later) and the
-     * original communicator */
-    ADIOI_cb_gather_name_array(orig_comm, comm, &array);
-
-/* parse the cb_config_list and create a rank map on rank 0 */
-    if (rank == 0) {
-	tmp_ranklist = (int *) ADIOI_Malloc(sizeof(int) * procs);
-	if (tmp_ranklist == NULL) {
-	    *error_code = MPIO_Err_create_code(*error_code,
-					       MPIR_ERR_RECOVERABLE,
-					       myname,
-					       __LINE__,
-					       MPI_ERR_OTHER,
-					       "**nomem2",0);
-	    return 0;
-	}
-
-	rank_ct = ADIOI_cb_config_list_parse(fd->hints->cb_config_list, 
-					     array, tmp_ranklist,
-					     fd->hints->cb_nodes);
-
-	/* store the ranklist using the minimum amount of memory */
-	if (rank_ct > 0) {
-	    fd->hints->ranklist = (int *) ADIOI_Malloc(sizeof(int) * rank_ct);
-	    memcpy(fd->hints->ranklist, tmp_ranklist, sizeof(int) * rank_ct);
-	}
-	ADIOI_Free(tmp_ranklist);
-	fd->hints->cb_nodes = rank_ct;
-	/* TEMPORARY -- REMOVE WHEN NO LONGER UPDATING INFO FOR FS-INDEP. */
-	value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
-	ADIOI_Snprintf(value, MPI_MAX_INFO_VAL+1, "%d", rank_ct);
-	ADIOI_Info_set(fd->info, "cb_nodes", value);
-	ADIOI_Free(value);
-    }
-
-    ADIOI_cb_bcast_rank_map(fd);
-    if (fd->hints->cb_nodes <= 0) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__, MPI_ERR_IO,
-					   "**ioagnomatch", 0);
-	fd = ADIO_FILE_NULL;
-    }
-    return 0;
-}
-
-/* 
- * vim: ts=8 sts=4 sw=4 noexpandtab 
- */
diff --git a/3rd-party/romio321/adio/common/ad_opencoll.c b/3rd-party/romio321/adio/common/ad_opencoll.c
deleted file mode 100644
index 467a322a340..00000000000
--- a/3rd-party/romio321/adio/common/ad_opencoll.c
+++ /dev/null
@@ -1,129 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2007 UChicago/Argonne LLC
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-
-/* Generic version of a "collective open".  Assumes a "real" underlying
- * file system (meaning no wonky consistency semantics like NFS).
- *
- * optimization: by having just one process create a file, close it,
- * then have all N processes open it, we can possibly avoid contention
- * for write locks on a directory for some file systems.  
- *
- * Happy side-effect: exclusive create (error if file already exists)
- * just falls out 
- *
- * Note: this is not a "scalable open" (c.f. "The impact of file systems
- * on MPI-IO scalability").  
- */
-     
-void ADIOI_GEN_OpenColl(ADIO_File fd, int rank, 
-	int access_mode, int *error_code)
-{
-    int orig_amode_excl, orig_amode_wronly;
-    MPI_Comm tmp_comm;
-
-    orig_amode_excl = access_mode;
-
-    if (access_mode & ADIO_CREATE ){
-       if(rank == fd->hints->ranklist[0]) {
-	   /* remove delete_on_close flag if set */
-	   if (access_mode & ADIO_DELETE_ON_CLOSE)
-	       fd->access_mode = access_mode ^ ADIO_DELETE_ON_CLOSE;
-	   else 
-	       fd->access_mode = access_mode;
-	       
-	   tmp_comm = fd->comm;
-	   fd->comm = MPI_COMM_SELF;
-	   (*(fd->fns->ADIOI_xxx_Open))(fd, error_code);
-	   fd->comm = tmp_comm;
-	   MPI_Bcast(error_code, 1, MPI_INT, \
-		     fd->hints->ranklist[0], fd->comm);
-	   /* if no error, close the file and reopen normally below */
-	   if (*error_code == MPI_SUCCESS) 
-	       (*(fd->fns->ADIOI_xxx_Close))(fd, error_code);
-
-	   fd->access_mode = access_mode; /* back to original */
-       }
-       else MPI_Bcast(error_code, 1, MPI_INT, fd->hints->ranklist[0], fd->comm);
-
-       if (*error_code != MPI_SUCCESS) {
-	   return;
-       } 
-       else {
-           /* turn off CREAT (and EXCL if set) for real multi-processor open */
-           access_mode ^= ADIO_CREATE; 
-	   if (access_mode & ADIO_EXCL)
-		   access_mode ^= ADIO_EXCL;
-       }
-    }
-    fd->blksize = 1024*1024*4; /* this large default value should be good for
-				 most file systems.  any ROMIO driver is free
-				 to stat the file and find an optimial value */
-
-    /* if we are doing deferred open, non-aggregators should return now */
-    if (fd->hints->deferred_open ) {
-        if (!(fd->is_agg)) {
-            /* we might have turned off EXCL for the aggregators.
-             * restore access_mode that non-aggregators get the right
-             * value from get_amode */
-            fd->access_mode = orig_amode_excl;
-	    /* In file-system specific open, a driver might collect some
-	     * information via stat().  Deferred open means not every process
-	     * participates in fs-specific open, but they all participate in
-	     * this open call.  Broadcast a bit of information in case
-	     * lower-level file system driver (e.g. 'bluegene') collected it
-	     * (not all do)*/
-	    MPI_Bcast(&(fd->blksize), 1, MPI_LONG, fd->hints->ranklist[0], fd->comm);
-	    *error_code = MPI_SUCCESS;
-	    ADIOI_Assert(fd->blksize > 0);
-	    return;
-	}
-    }
-
-/* For writing with data sieving, a read-modify-write is needed. If 
-   the file is opened for write_only, the read will fail. Therefore,
-   if write_only, open the file as read_write, but record it as write_only
-   in fd, so that get_amode returns the right answer. */
-
-    /* observation from David Knaak: file systems that do not support data
-     * sieving do not need to change the mode */
-
-    orig_amode_wronly = access_mode;
-    if ( (access_mode & ADIO_WRONLY) &&
-	    ADIO_Feature(fd, ADIO_DATA_SIEVING_WRITES) ) {
-	access_mode = access_mode ^ ADIO_WRONLY;
-	access_mode = access_mode | ADIO_RDWR;
-    }
-    fd->access_mode = access_mode;
-
-    (*(fd->fns->ADIOI_xxx_Open))(fd, error_code);
-
-    /* if error, may be it was due to the change in amode above. 
-       therefore, reopen with access mode provided by the user.*/ 
-    fd->access_mode = orig_amode_wronly;  
-    if (*error_code != MPI_SUCCESS) 
-        (*(fd->fns->ADIOI_xxx_Open))(fd, error_code);
-
-    /* if we turned off EXCL earlier, then we should turn it back on */
-    if (fd->access_mode != orig_amode_excl) fd->access_mode = orig_amode_excl;
-
-    /* broadcast a bit of information (blocksize for now) to all proceses in
-     * communicator, not just those who participated in open */
-    MPI_Bcast(&(fd->blksize), 1, MPI_LONG, fd->hints->ranklist[0], fd->comm);
-    /* file domain code will get terribly confused in a hard-to-debug way if
-     * gpfs blocksize not sensible */
-    ADIOI_Assert( fd->blksize > 0);
-    /* for deferred open: this process has opened the file (because if we are
-     * not an aggregaor and we are doing deferred open, we returned earlier)*/
-    fd->is_open = 1;
-
-}
-
-/* 
- * vim: ts=8 sts=4 sw=4 noexpandtab 
- */
diff --git a/3rd-party/romio321/adio/common/ad_opencoll_failsafe.c b/3rd-party/romio321/adio/common/ad_opencoll_failsafe.c
deleted file mode 100644
index 962f191c1f9..00000000000
--- a/3rd-party/romio321/adio/common/ad_opencoll_failsafe.c
+++ /dev/null
@@ -1,97 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2007 UChicago/Argonne LLC
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-
-
-/* this "collective" open is useful for frankly broken file systems such
- * as NFS where a create from one client might not be immediately
- * visible on another */
-
-void ADIOI_FAILSAFE_OpenColl(ADIO_File fd, int rank, 
-	int access_mode, int *error_code)
-{
-    MPI_Comm tmp_comm;
-    int orig_amode_excl, orig_amode_wronly;
-
-    orig_amode_excl = access_mode;
-    if ((access_mode & ADIO_CREATE) && (access_mode & ADIO_EXCL)) {
-	/* the open should fail if the file exists. Only *1* process
-	 * should check this. Otherwise, if all processes try to check
-	 * and the file does not exist, one process will create the file
-	 * and others who reach later will return error. */ 
-	if(rank == fd->hints->ranklist[0]) {
-	    fd->access_mode = access_mode;
-
-	    /* if the lower-level file system tries to communicate, COMM_SELF
-	     * will ensure it doesn't get stuck waiting for non-existant
-	     * participants */
-	    tmp_comm = fd->comm;
-	    fd->comm = MPI_COMM_SELF;
-	    (*(fd->fns->ADIOI_xxx_Open))(fd, error_code);
-	    MPI_Bcast(error_code, 1, MPI_INT, \
-		    fd->hints->ranklist[0], tmp_comm);
-	    /* if no error, close the file and reopen normally below */
-	    if (*error_code == MPI_SUCCESS)
-		(*(fd->fns->ADIOI_xxx_Close))(fd, error_code);
-	    /* and put it all back the way we found it for subsequent code */
-	    fd->comm = tmp_comm;
-	}
-	else MPI_Bcast(error_code, 1, MPI_INT,
-		fd->hints->ranklist[0], fd->comm);
-	if (*error_code != MPI_SUCCESS) {
-	    return;
-	}
-	else {
-	    /* turn off EXCL for real open */
-	    access_mode = access_mode ^ ADIO_EXCL;
-	}
-    }
-    /* if we are doing deferred open, non-aggregators should return now */
-    if (fd->hints->deferred_open && !(fd->is_agg)) {
-	/* we might have turned off EXCL for the aggregators.
-	 * restore access_mode that non-aggregators get the right
-	 * value from get_amode */
-	fd->access_mode = orig_amode_excl;
-	*error_code = MPI_SUCCESS;
-	return;
-    }
-
-/* For writing with data sieving, a read-modify-write is needed. If 
-   the file is opened for write_only, the read will fail. Therefore,
-   if write_only, open the file as read_write, but record it as write_only
-   in fd, so that get_amode returns the right answer. */
-
-    orig_amode_wronly = access_mode;
-    if (access_mode & ADIO_WRONLY) {
-	access_mode = access_mode ^ ADIO_WRONLY;
-	access_mode = access_mode | ADIO_RDWR;
-    }
-    fd->access_mode = access_mode;
-
-    (*(fd->fns->ADIOI_xxx_Open))(fd, error_code);
-
-    /* if error, may be it was due to the change in amode above. 
-       therefore, reopen with access mode provided by the user.*/ 
-    fd->access_mode = orig_amode_wronly;  
-    if (*error_code != MPI_SUCCESS) 
-        (*(fd->fns->ADIOI_xxx_Open))(fd, error_code);
-
-    if(*error_code != MPI_SUCCESS) return;
-    /* if we turned off EXCL earlier, then we should turn it back on */
-    if (fd->access_mode != orig_amode_excl) fd->access_mode = orig_amode_excl;
-
-    /* for deferred open: this process has opened the file (because if we are
-     * not an aggregaor and we are doing deferred open, we returned earlier)*/
-    fd->is_open = 1;
-}
-
-
-
-/* 
- * vim: ts=8 sts=4 sw=4 noexpandtab 
- */
diff --git a/3rd-party/romio321/adio/common/ad_opencoll_scalable.c b/3rd-party/romio321/adio/common/ad_opencoll_scalable.c
deleted file mode 100644
index 1c472017f53..00000000000
--- a/3rd-party/romio321/adio/common/ad_opencoll_scalable.c
+++ /dev/null
@@ -1,55 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2007 UChicago/Argonne LLC
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-
-/* 
- * Scalable open:  for file systems capable of having one process
- * create/open a file and broadcast the result to everyone else. 
- * - Does not need one process to create the file
- * - Does not need special handling for CREATE|EXCL
- */
-void ADIOI_SCALEABLE_OpenColl(ADIO_File fd, int rank, 
-		int access_mode, int *error_code)
-{    
-    int orig_amode_wronly;
-
-    /* if we are doing deferred open, non-aggregators should return now */
-    if (fd->hints->deferred_open && !(fd->is_agg)) {
-	*error_code = MPI_SUCCESS;
-	return;
-    } 
-    
-    /* For writing with data sieving, a read-modify-write is needed. If 
-       the file is opened for write_only, the read will fail. Therefore,
-       if write_only, open the file as read_write, but record it as
-       write_only in fd, so that get_amode returns the right answer. */
-
-    orig_amode_wronly = access_mode;
-    if (access_mode & ADIO_WRONLY) {
-	access_mode = access_mode ^ ADIO_WRONLY;
-	access_mode = access_mode | ADIO_RDWR;
-    }
-    fd->access_mode = access_mode;
-
-    (*(fd->fns->ADIOI_xxx_Open))(fd, error_code);
-
-    /* if error, may be it was due to the change in amode above. 
-       therefore, reopen with access mode provided by the user.*/ 
-    fd->access_mode = orig_amode_wronly;  
-    if (*error_code != MPI_SUCCESS) 
-        (*(fd->fns->ADIOI_xxx_Open))(fd, error_code);
-
-    /* for deferred open: this process has opened the file (because if we are
-     * not an aggregaor and we are doing deferred open, we returned earlier)*/
-    fd->is_open = 1;
-
-}
-
-/* 
- * vim: ts=8 sts=4 sw=4 noexpandtab 
- */
diff --git a/3rd-party/romio321/adio/common/ad_prealloc.c b/3rd-party/romio321/adio/common/ad_prealloc.c
deleted file mode 100644
index 9462f775aa6..00000000000
--- a/3rd-party/romio321/adio/common/ad_prealloc.c
+++ /dev/null
@@ -1,91 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2004 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-
-/* this used to be implemented in every file system as an fcntl, but the code
- * is identical for all file systems without a real "preallocate" system call.
- * This naive approach will get the job done, but not in a terribly efficient
- * manner.
- */
-void ADIOI_GEN_Prealloc(ADIO_File fd, ADIO_Offset diskspace, int *error_code) 
-{
-	ADIO_Offset curr_fsize, alloc_size, size, len, done, ntimes;
-	ADIO_Status status;
-	int i;
-	char *buf;
-	ADIO_Fcntl_t *fcntl_struct;
-	static char myname[] = "ADIOI_GEN_PREALLOC";
-
-	/* will be called by one process only */
-	/* On file systems with no preallocation function, we have to 
-           explicitly write 
-           to allocate space. Since there could be holes in the file, 
-           we need to read up to the current file size, write it back, 
-           and then write beyond that depending on how much 
-           preallocation is needed.
-           read/write in sizes of no more than ADIOI_PREALLOC_BUFSZ */
-
-	/*curr_fsize = fd->fp_ind; */
-	fcntl_struct = (ADIO_Fcntl_t *) ADIOI_Malloc(sizeof(ADIO_Fcntl_t));
-	ADIO_Fcntl(fd, ADIO_FCNTL_GET_FSIZE, fcntl_struct, error_code);
-
-	curr_fsize = fcntl_struct->fsize; /* don't rely on fd->fp_ind: might be
-					    working on a pre-existing file */
-	alloc_size = diskspace;
-
-	size = ADIOI_MIN(curr_fsize, alloc_size);
-	
-	ntimes = (size + ADIOI_PREALLOC_BUFSZ - 1)/ADIOI_PREALLOC_BUFSZ;
-	buf = (char *) ADIOI_Malloc(ADIOI_PREALLOC_BUFSZ);
-	done = 0;
-
-	for (i=0; i<ntimes; i++) {
-	    len = ADIOI_MIN(size-done, ADIOI_PREALLOC_BUFSZ);
-	    ADIO_ReadContig(fd, buf, 
-                      (int)len, /* len is ADIO_Offset but is <= ADIOI_PREALLOC_BUFSZ (16M),
-                              so it fits in an int parameter */
-                      MPI_BYTE, ADIO_EXPLICIT_OFFSET, done,
-			    &status, error_code);
-	    if (*error_code != MPI_SUCCESS) {
-		*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-						   MPIR_ERR_RECOVERABLE,
-						   myname, __LINE__,
-						   MPI_ERR_IO, 
-						   "**iopreallocrdwr",
-						   0);
-                return;  
-	    }
-	    ADIO_WriteContig(fd, buf, 
-                       (int)len, /* len is ADIO_Offset but is <= ADIOI_PREALLOC_BUFSZ (16M),
-                               so it fits in an int parameter */
-                       MPI_BYTE, ADIO_EXPLICIT_OFFSET, 
-                             done, &status, error_code);
-	    if (*error_code != MPI_SUCCESS) return;
-	    done += len;
-	}
-
-	if (alloc_size > curr_fsize) {
-	    memset(buf, 0, ADIOI_PREALLOC_BUFSZ); 
-	    size = alloc_size - curr_fsize;
-	    ntimes = (size + ADIOI_PREALLOC_BUFSZ - 1)/ADIOI_PREALLOC_BUFSZ;
-	    for (i=0; i<ntimes; i++) {
-		len = ADIOI_MIN(alloc_size-done, ADIOI_PREALLOC_BUFSZ);
-		ADIO_WriteContig(fd, buf, 
-                     (int)len, /* len is ADIO_Offset but is <= ADIOI_PREALLOC_BUFSZ (16M),
-                             so it fits in an int parameter */
-                     MPI_BYTE, ADIO_EXPLICIT_OFFSET, 
-				 done, &status, error_code);
-		if (*error_code != MPI_SUCCESS) return;
-		done += len;  
-	    }
-	}
-	ADIOI_Free(fcntl_struct);
-	ADIOI_Free(buf);
-	*error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/common/ad_read.c b/3rd-party/romio321/adio/common/ad_read.c
deleted file mode 100644
index 75b6c4823bb..00000000000
--- a/3rd-party/romio321/adio/common/ad_read.c
+++ /dev/null
@@ -1,115 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-
-#include "adio.h"
-#include <unistd.h>
-
-#ifdef AGGREGATION_PROFILE
-#include "mpe.h"
-#endif
-#ifdef ROMIO_GPFS
-# include "adio/ad_gpfs/ad_gpfs_tuning.h"
-#endif
-
-#ifdef HAVE_LIMITS_H
-#include <limits.h>
-#endif
-
-void ADIOI_GEN_ReadContig(ADIO_File fd, void *buf, int count, 
-			  MPI_Datatype datatype, int file_ptr_type,
-			  ADIO_Offset offset, ADIO_Status *status,
-			  int *error_code)
-{
-    ssize_t err = -1;
-    MPI_Count datatype_size;
-    ADIO_Offset len, bytes_xfered=0;
-    size_t rd_count;
-    static char myname[] = "ADIOI_GEN_READCONTIG";
-#ifdef ROMIO_GPFS
-    double io_time=0;
-#endif
-    char *p;
-
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5034, 0, NULL);
-#endif
-    MPI_Type_size_x(datatype, &datatype_size);
-    len = datatype_size * (ADIO_Offset)count;
-
-#ifdef ROMIO_GPFS
-    io_time = MPI_Wtime();
-    if (gpfsmpio_timing) {
-	gpfsmpio_prof_cr[ GPFSMPIO_CIO_DATA_SIZE ] += len;
-    }
-#endif
-
-    if (file_ptr_type == ADIO_INDIVIDUAL) {
-	offset = fd->fp_ind;
-    }
-
-    p=buf;
-    while (bytes_xfered < len) {
-#ifdef ADIOI_MPE_LOGGING
-	MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
-#endif
-	rd_count = len - bytes_xfered;
-	/* stupid FreeBSD and Darwin do not like a count larger than a signed
-           int, even though size_t is eight bytes... */
-        if (rd_count > INT_MAX)
-            rd_count = INT_MAX;
-#ifdef ROMIO_GPFS
-	if (gpfsmpio_devnullio)
-	    err = pread(fd->null_fd, p, rd_count, offset+bytes_xfered);
-	else
-#endif
-	    err = pread(fd->fd_sys, p, rd_count, offset+bytes_xfered);
-	/* --BEGIN ERROR HANDLING-- */
-	if (err == -1) {
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-		    MPIR_ERR_RECOVERABLE,
-		    myname, __LINE__,
-		    MPI_ERR_IO, "**io",
-		    "**io %s", strerror(errno));
-	    fd->fp_sys_posn = -1;
-	    return;
-	}
-	/* --END ERROR HANDLING-- */
-	if (err == 0) {
-	    /* end of file */
-	    break;
-	}
-
-#ifdef ADIOI_MPE_LOGGING
-	MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
-#endif
-	bytes_xfered += err;
-	p += err;
-    }
-#ifdef ROMIO_GPFS
-    if (gpfsmpio_timing) gpfsmpio_prof_cr[ GPFSMPIO_CIO_T_POSI_RW ] += (MPI_Wtime() - io_time);
-#endif
-    fd->fp_sys_posn = offset + bytes_xfered;
-
-    if (file_ptr_type == ADIO_INDIVIDUAL) {
-	fd->fp_ind += bytes_xfered; 
-    }
-
-#ifdef HAVE_STATUS_SET_BYTES
-    /* what if we only read half a datatype? */
-    /* bytes_xfered could be larger than int */
-    if (err != -1) MPIR_Status_set_bytes(status, datatype, bytes_xfered);
-#endif
-
-    *error_code = MPI_SUCCESS;
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5035, 0, NULL);
-#endif
-#ifdef ROMIO_GPFS
-    if (gpfsmpio_timing) gpfsmpio_prof_cr[ GPFSMPIO_CIO_T_MPIO_RW ] += (MPI_Wtime() - io_time);
-#endif
-}
diff --git a/3rd-party/romio321/adio/common/ad_read_coll.c b/3rd-party/romio321/adio/common/ad_read_coll.c
deleted file mode 100644
index eaf99e6c2dc..00000000000
--- a/3rd-party/romio321/adio/common/ad_read_coll.c
+++ /dev/null
@@ -1,1063 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-
-#ifdef USE_DBG_LOGGING
-  #define RDCOLL_DEBUG 1
-#endif
-#ifdef AGGREGATION_PROFILE
-#include "mpe.h"
-#endif
-
-/* prototypes of functions used for collective reads only. */
-static void ADIOI_Read_and_exch(ADIO_File fd, void *buf, MPI_Datatype
-				datatype, int nprocs,
-				int myrank, ADIOI_Access
-				*others_req, ADIO_Offset *offset_list,
-				ADIO_Offset *len_list, int contig_access_count, 
-				ADIO_Offset
-				min_st_offset, ADIO_Offset fd_size,
-				ADIO_Offset *fd_start, ADIO_Offset *fd_end,
-				int *buf_idx, int *error_code);
-static void ADIOI_R_Exchange_data(ADIO_File fd, void *buf, ADIOI_Flatlist_node
-				  *flat_buf, ADIO_Offset *offset_list, ADIO_Offset
-				  *len_list, int *send_size, int *recv_size,
-				  int *count, int *start_pos, 
-				  int *partial_send, 
-				  int *recd_from_proc, int nprocs, 
-				  int myrank, int
-				  buftype_is_contig, int contig_access_count,
-				  ADIO_Offset min_st_offset, 
-				  ADIO_Offset fd_size,
-				  ADIO_Offset *fd_start, ADIO_Offset *fd_end, 
-				  ADIOI_Access *others_req, 
-				  int iter, 
-				  MPI_Aint buftype_extent, int *buf_idx);
-void ADIOI_Fill_user_buffer(ADIO_File fd, void *buf, ADIOI_Flatlist_node
-				   *flat_buf, char **recv_buf, ADIO_Offset 
-				   *offset_list, ADIO_Offset *len_list, 
-				   unsigned *recv_size, 
-				   MPI_Request *requests, MPI_Status *statuses,
-				   int *recd_from_proc, int nprocs,
-				   int contig_access_count, 
-				   ADIO_Offset min_st_offset, 
-				   ADIO_Offset fd_size, ADIO_Offset *fd_start, 
-				   ADIO_Offset *fd_end,
-				   MPI_Aint buftype_extent);
-
-
-void ADIOI_GEN_ReadStridedColl(ADIO_File fd, void *buf, int count,
-			       MPI_Datatype datatype, int file_ptr_type,
-			       ADIO_Offset offset, ADIO_Status *status, int
-			       *error_code)
-{
-/* Uses a generalized version of the extended two-phase method described
-   in "An Extended Two-Phase Method for Accessing Sections of 
-   Out-of-Core Arrays", Rajeev Thakur and Alok Choudhary,
-   Scientific Programming, (5)4:301--317, Winter 1996. 
-   http://www.mcs.anl.gov/home/thakur/ext2ph.ps */
-
-    ADIOI_Access *my_req; 
-    /* array of nprocs structures, one for each other process in
-       whose file domain this process's request lies */
-    
-    ADIOI_Access *others_req;
-    /* array of nprocs structures, one for each other process
-       whose request lies in this process's file domain. */
-
-    int i, filetype_is_contig, nprocs, nprocs_for_coll, myrank;
-    int contig_access_count=0, interleave_count = 0, buftype_is_contig;
-    int *count_my_req_per_proc, count_my_req_procs, count_others_req_procs;
-    ADIO_Offset start_offset, end_offset, orig_fp, fd_size, min_st_offset, off;
-    ADIO_Offset *offset_list = NULL, *st_offsets = NULL, *fd_start = NULL,
-	*fd_end = NULL, *end_offsets = NULL;
-    ADIO_Offset *len_list = NULL;
-    int *buf_idx = NULL;
-
-#ifdef HAVE_STATUS_SET_BYTES
-    MPI_Count bufsize, size;
-#endif
-
-    if (fd->hints->cb_pfr != ADIOI_HINT_DISABLE) {
-        ADIOI_IOStridedColl (fd, buf, count, ADIOI_READ, datatype, 
-			file_ptr_type, offset, status, error_code);
-        return;
-    }
-
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-
-    /* number of aggregators, cb_nodes, is stored in the hints */
-    nprocs_for_coll = fd->hints->cb_nodes;
-    orig_fp = fd->fp_ind;
-
-    /* only check for interleaving if cb_read isn't disabled */
-    if (fd->hints->cb_read != ADIOI_HINT_DISABLE) {
-    /* For this process's request, calculate the list of offsets and
-       lengths in the file and determine the start and end offsets. */
-
-    /* Note: end_offset points to the last byte-offset that will be accessed.
-       e.g., if start_offset=0 and 100 bytes to be read, end_offset=99*/
-
-	ADIOI_Calc_my_off_len(fd, count, datatype, file_ptr_type, offset,
-			      &offset_list, &len_list, &start_offset,
-			      &end_offset, &contig_access_count); 
-    
-#ifdef RDCOLL_DEBUG
-    for (i=0; i<contig_access_count; i++) {
-	      DBG_FPRINTF(stderr, "rank %d  off %lld  len %lld\n", 
-			      myrank, offset_list[i], len_list[i]);
-	      }
-#endif
-
-	/* each process communicates its start and end offsets to other 
-	   processes. The result is an array each of start and end offsets
-	   stored in order of process rank. */ 
-    
-	st_offsets = (ADIO_Offset *) ADIOI_Malloc(nprocs*sizeof(ADIO_Offset));
-	end_offsets = (ADIO_Offset *) ADIOI_Malloc(nprocs*sizeof(ADIO_Offset));
-
-	MPI_Allgather(&start_offset, 1, ADIO_OFFSET, st_offsets, 1,
-		      ADIO_OFFSET, fd->comm);
-	MPI_Allgather(&end_offset, 1, ADIO_OFFSET, end_offsets, 1,
-		      ADIO_OFFSET, fd->comm);
-
-	/* are the accesses of different processes interleaved? */
-	for (i=1; i<nprocs; i++)
-	    if ((st_offsets[i] < end_offsets[i-1]) && 
-                (st_offsets[i] <= end_offsets[i]))
-                interleave_count++;
-	/* This is a rudimentary check for interleaving, but should suffice
-	   for the moment. */
-    }
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-
-    if (fd->hints->cb_read == ADIOI_HINT_DISABLE
-	|| (!interleave_count && (fd->hints->cb_read == ADIOI_HINT_AUTO))) 
-    {
-	/* don't do aggregation */
-	if (fd->hints->cb_read != ADIOI_HINT_DISABLE) {
-	    ADIOI_Free(offset_list);
-	    ADIOI_Free(len_list);
-	    ADIOI_Free(st_offsets);
-	    ADIOI_Free(end_offsets);
-	}
-
-	fd->fp_ind = orig_fp;
-	ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-
-	if (buftype_is_contig && filetype_is_contig) {
-	    if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-		off = fd->disp + (fd->etype_size) * offset;
-		ADIO_ReadContig(fd, buf, count, datatype, ADIO_EXPLICIT_OFFSET,
-                       off, status, error_code);
-	    }
-	    else ADIO_ReadContig(fd, buf, count, datatype, ADIO_INDIVIDUAL,
-                       0, status, error_code);
-	}
-	else ADIO_ReadStrided(fd, buf, count, datatype, file_ptr_type,
-                       offset, status, error_code);
-
-	return;
-    }
-
-    /* We're going to perform aggregation of I/O.  Here we call
-     * ADIOI_Calc_file_domains() to determine what processes will handle I/O
-     * to what regions.  We pass nprocs_for_coll into this function; it is
-     * used to determine how many processes will perform I/O, which is also
-     * the number of regions into which the range of bytes must be divided.
-     * These regions are called "file domains", or FDs.
-     *
-     * When this function returns, fd_start, fd_end, fd_size, and
-     * min_st_offset will be filled in.  fd_start holds the starting byte
-     * location for each file domain.  fd_end holds the ending byte location.
-     * min_st_offset holds the minimum byte location that will be accessed.
-     *
-     * Both fd_start[] and fd_end[] are indexed by an aggregator number; this
-     * needs to be mapped to an actual rank in the communicator later.
-     *
-     */
-    ADIOI_Calc_file_domains(st_offsets, end_offsets, nprocs,
-			    nprocs_for_coll, &min_st_offset,
-			    &fd_start, &fd_end, 
-			    fd->hints->min_fdomain_size, &fd_size,
-			    fd->hints->striping_unit);
-
-    /* calculate where the portions of the access requests of this process 
-     * are located in terms of the file domains.  this could be on the same
-     * process or on other processes.  this function fills in:
-     * count_my_req_procs - number of processes (including this one) for which
-     *     this process has requests in their file domain
-     * count_my_req_per_proc - count of requests for each process, indexed
-     *     by rank of the process
-     * my_req[] - array of data structures describing the requests to be
-     *     performed by each process (including self).  indexed by rank.
-     * buf_idx[] - array of locations into which data can be directly moved;
-     *     this is only valid for contiguous buffer case
-     */
-    ADIOI_Calc_my_req(fd, offset_list, len_list, contig_access_count,
-		      min_st_offset, fd_start, fd_end, fd_size,
-		      nprocs, &count_my_req_procs, 
-		      &count_my_req_per_proc, &my_req,
-		      &buf_idx);
-
-    /* perform a collective communication in order to distribute the
-     * data calculated above.  fills in the following:
-     * count_others_req_procs - number of processes (including this
-     *     one) which have requests in this process's file domain.
-     * count_others_req_per_proc[] - number of separate contiguous
-     *     requests from proc i lie in this process's file domain.
-     */
-    ADIOI_Calc_others_req(fd, count_my_req_procs, 
-			  count_my_req_per_proc, my_req, 
-			  nprocs, myrank, &count_others_req_procs, 
-			  &others_req); 
-
-    /* my_req[] and count_my_req_per_proc aren't needed at this point, so 
-     * let's free the memory 
-     */
-    ADIOI_Free(count_my_req_per_proc);
-    for (i=0; i<nprocs; i++) {
-	if (my_req[i].count) {
-	    ADIOI_Free(my_req[i].offsets);
-	    ADIOI_Free(my_req[i].lens);
-	}
-    }
-    ADIOI_Free(my_req);
-
-
-    /* read data in sizes of no more than ADIOI_Coll_bufsize, 
-     * communicate, and fill user buf. 
-     */
-    ADIOI_Read_and_exch(fd, buf, datatype, nprocs, myrank,
-                        others_req, offset_list,
-			len_list, contig_access_count, min_st_offset,
-			fd_size, fd_start, fd_end, buf_idx, error_code);
-
-    if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
-
-    /* free all memory allocated for collective I/O */
-    for (i=0; i<nprocs; i++) {
-	if (others_req[i].count) {
-	    ADIOI_Free(others_req[i].offsets);
-	    ADIOI_Free(others_req[i].lens);
-	    ADIOI_Free(others_req[i].mem_ptrs);
-	}
-    }
-    ADIOI_Free(others_req);
-
-    ADIOI_Free(buf_idx);
-    ADIOI_Free(offset_list);
-    ADIOI_Free(len_list);
-    ADIOI_Free(st_offsets);
-    ADIOI_Free(end_offsets);
-    ADIOI_Free(fd_start);
-    ADIOI_Free(fd_end);
-
-#ifdef HAVE_STATUS_SET_BYTES
-    MPI_Type_size_x(datatype, &size);
-    bufsize = size * count;
-    MPIR_Status_set_bytes(status, datatype, bufsize);
-/* This is a temporary way of filling in status. The right way is to 
-   keep track of how much data was actually read and placed in buf 
-   during collective I/O. */
-#endif
-
-    fd->fp_sys_posn = -1;   /* set it to null. */
-}
-
-void ADIOI_Calc_my_off_len(ADIO_File fd, int bufcount, MPI_Datatype
-			    datatype, int file_ptr_type, ADIO_Offset
-			    offset, ADIO_Offset **offset_list_ptr, ADIO_Offset
-			    **len_list_ptr, ADIO_Offset *start_offset_ptr,
-			    ADIO_Offset *end_offset_ptr, int
-			   *contig_access_count_ptr)
-{
-    MPI_Count filetype_size, etype_size;
-    MPI_Count buftype_size;
-    int i, j, k;
-    ADIO_Offset i_offset;
-    ADIO_Offset frd_size=0, old_frd_size=0;
-    int st_index=0;
-    ADIO_Offset n_filetypes, etype_in_filetype;
-    ADIO_Offset abs_off_in_filetype=0;
-    ADIO_Offset bufsize;
-    ADIO_Offset sum, n_etypes_in_filetype, size_in_filetype;
-    int contig_access_count, filetype_is_contig;
-    ADIO_Offset *len_list;
-    MPI_Aint filetype_extent, filetype_lb;
-    ADIOI_Flatlist_node *flat_file;
-    ADIO_Offset *offset_list, off, end_offset=0, disp;
-
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5028, 0, NULL);
-#endif
-    
-/* For this process's request, calculate the list of offsets and
-   lengths in the file and determine the start and end offsets. */
-
-    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-
-    MPI_Type_size_x(fd->filetype, &filetype_size);
-    MPI_Type_get_extent(fd->filetype, &filetype_lb, &filetype_extent);
-    MPI_Type_size_x(datatype, &buftype_size);
-    etype_size = fd->etype_size;
-
-    if ( ! filetype_size ) {
-	*contig_access_count_ptr = 0;
-	*offset_list_ptr = (ADIO_Offset *) ADIOI_Malloc(2*sizeof(ADIO_Offset));
-	*len_list_ptr = (ADIO_Offset *) ADIOI_Malloc(2*sizeof(ADIO_Offset));
-        /* 2 is for consistency. everywhere I malloc one more than needed */
-
-	offset_list = *offset_list_ptr;
-	len_list = *len_list_ptr;
-        offset_list[0] = (file_ptr_type == ADIO_INDIVIDUAL) ? fd->fp_ind : 
-                 fd->disp + (ADIO_Offset)etype_size * offset;
-	len_list[0] = 0;
-	*start_offset_ptr = offset_list[0];
-	*end_offset_ptr = offset_list[0] + len_list[0] - 1;
-	
-	return;
-    }
-
-    if (filetype_is_contig) {
-	*contig_access_count_ptr = 1;        
-	*offset_list_ptr = (ADIO_Offset *) ADIOI_Malloc(2*sizeof(ADIO_Offset));
-	*len_list_ptr = (ADIO_Offset *) ADIOI_Malloc(2*sizeof(ADIO_Offset));
-        /* 2 is for consistency. everywhere I malloc one more than needed */
-
-	offset_list = *offset_list_ptr;
-	len_list = *len_list_ptr;
-        offset_list[0] = (file_ptr_type == ADIO_INDIVIDUAL) ? fd->fp_ind : 
-                 fd->disp + (ADIO_Offset)etype_size * offset;
-	len_list[0] = (ADIO_Offset)bufcount * (ADIO_Offset)buftype_size;
-	*start_offset_ptr = offset_list[0];
-	*end_offset_ptr = offset_list[0] + len_list[0] - 1;
-
-	/* update file pointer */
-	if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = *end_offset_ptr + 1;
-    }
-
-    else {
-
-       /* First calculate what size of offset_list and len_list to allocate */
-   
-       /* filetype already flattened in ADIO_Open or ADIO_Fcntl */
-	flat_file = ADIOI_Flatlist;
-	while (flat_file->type != fd->filetype) flat_file = flat_file->next;
-	disp = fd->disp;
-
-#ifdef RDCOLL_DEBUG 
-        {
-            int ii;
-            DBG_FPRINTF(stderr, "flattened %3lld : ", flat_file->count );
-            for (ii=0; ii<flat_file->count; ii++) {
-                DBG_FPRINTF(stderr, "%16lld:%-16lld", flat_file->indices[ii], flat_file->blocklens[ii] );
-            }
-            DBG_FPRINTF(stderr, "\n" );
-        }
-#endif
-	if (file_ptr_type == ADIO_INDIVIDUAL) {
-           /* Wei-keng reworked type processing to be a bit more efficient */
-            offset       = fd->fp_ind - disp;
-            n_filetypes  = (offset - flat_file->indices[0]) / filetype_extent;
-             offset     -= (ADIO_Offset)n_filetypes * filetype_extent;
-	     	/* now offset is local to this extent */
- 
-            /* find the block where offset is located, skip blocklens[i]==0 */
-            for (i=0; i<flat_file->count; i++) {
-                ADIO_Offset dist;
-                if (flat_file->blocklens[i] == 0) continue;
-                dist = flat_file->indices[i] + flat_file->blocklens[i] - offset;
-                /* frd_size is from offset to the end of block i */
-		if (dist == 0) {
-			i++;
-			offset   = flat_file->indices[i];
-			frd_size = flat_file->blocklens[i];
-			break;
-		}
-		if (dist > 0) {
-                    frd_size = dist;
-		    break;
-		}
-	    }
-            st_index = i;  /* starting index in flat_file->indices[] */
-            offset += disp + (ADIO_Offset)n_filetypes*filetype_extent;
-        }
-	else {
-	    n_etypes_in_filetype = filetype_size/etype_size;
-	    n_filetypes = offset / n_etypes_in_filetype;
-	    etype_in_filetype = offset % n_etypes_in_filetype;
-	    size_in_filetype = etype_in_filetype * etype_size;
- 
-	    sum = 0;
-	    for (i=0; i<flat_file->count; i++) {
-		sum += flat_file->blocklens[i];
-		if (sum > size_in_filetype) {
-		    st_index = i;
-		    frd_size = sum - size_in_filetype;
-		    abs_off_in_filetype = flat_file->indices[i] +
-			size_in_filetype - (sum - flat_file->blocklens[i]);
-		    break;
-		}
-	    }
-
-	    /* abs. offset in bytes in the file */
-	    offset = disp + n_filetypes* (ADIO_Offset)filetype_extent + 
-		abs_off_in_filetype;
-	}
-
-         /* calculate how much space to allocate for offset_list, len_list */
-
-	old_frd_size = frd_size;
-	contig_access_count = i_offset = 0;
-	j = st_index;
-	bufsize = (ADIO_Offset)buftype_size * (ADIO_Offset)bufcount;
-	frd_size = ADIOI_MIN(frd_size, bufsize);
-	while (i_offset < bufsize) {
-	    if (frd_size) contig_access_count++;
-	    i_offset += frd_size;
-	    j = (j + 1) % flat_file->count;
-	    frd_size = ADIOI_MIN(flat_file->blocklens[j], bufsize-i_offset);
-	}
-
-        /* allocate space for offset_list and len_list */
-
-	*offset_list_ptr = (ADIO_Offset *)
-	         ADIOI_Malloc((contig_access_count+1)*sizeof(ADIO_Offset));  
-	*len_list_ptr = (ADIO_Offset *) ADIOI_Malloc((contig_access_count+1)*sizeof(ADIO_Offset));
-        /* +1 to avoid a 0-size malloc */
-
-	offset_list = *offset_list_ptr;
-	len_list = *len_list_ptr;
-
-      /* find start offset, end offset, and fill in offset_list and len_list */
-
-	*start_offset_ptr = offset; /* calculated above */
-
-	i_offset = k = 0;
-	j = st_index;
-	off = offset;
-	frd_size = ADIOI_MIN(old_frd_size, bufsize);
-	while (i_offset < bufsize) {
-	    if (frd_size) {
-		offset_list[k] = off;
-		len_list[k] = frd_size;
-		k++;
-	    }
-	    i_offset += frd_size;
-	    end_offset = off + frd_size - 1;
-
-     /* Note: end_offset points to the last byte-offset that will be accessed.
-         e.g., if start_offset=0 and 100 bytes to be read, end_offset=99*/
-
-	    if (off + frd_size < disp + flat_file->indices[j] +
-		flat_file->blocklens[j] + 
-		 n_filetypes* (ADIO_Offset)filetype_extent)
-	    {
-		off += frd_size;
-		/* did not reach end of contiguous block in filetype.
-		 * no more I/O needed. off is incremented by frd_size. 
-		 */
-	    }
-	    else {
-		j = (j+1) % flat_file->count;
-                n_filetypes += (j == 0) ? 1 : 0;
-                while (flat_file->blocklens[j]==0) {
-			j = (j+1) % flat_file->count;
-                    n_filetypes += (j == 0) ? 1 : 0;
-                    /* hit end of flattened filetype; start at beginning 
-		     * again */
-		}
-		off = disp + flat_file->indices[j] + 
-		     n_filetypes* (ADIO_Offset)filetype_extent;
-		frd_size = ADIOI_MIN(flat_file->blocklens[j], bufsize-i_offset);
-	    }
-	}
-
-	/* update file pointer */
-	if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
-
-	*contig_access_count_ptr = contig_access_count;
-	 *end_offset_ptr = end_offset;
-    }
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5029, 0, NULL);
-#endif
-}
-
-static void ADIOI_Read_and_exch(ADIO_File fd, void *buf, MPI_Datatype
-			 datatype, int nprocs,
-			 int myrank, ADIOI_Access
-			 *others_req, ADIO_Offset *offset_list,
-			 ADIO_Offset *len_list, int contig_access_count, ADIO_Offset
-                         min_st_offset, ADIO_Offset fd_size,
-			 ADIO_Offset *fd_start, ADIO_Offset *fd_end,
-                         int *buf_idx, int *error_code)
-{
-/* Read in sizes of no more than coll_bufsize, an info parameter.
-   Send data to appropriate processes. 
-   Place recd. data in user buf.
-   The idea is to reduce the amount of extra memory required for
-   collective I/O. If all data were read all at once, which is much
-   easier, it would require temp space more than the size of user_buf,
-   which is often unacceptable. For example, to read a distributed
-   array from a file, where each local array is 8Mbytes, requiring
-   at least another 8Mbytes of temp space is unacceptable. */
-
-    int i, j, m, ntimes, max_ntimes, buftype_is_contig;
-    ADIO_Offset st_loc=-1, end_loc=-1, off, done, real_off, req_off;
-    char *read_buf = NULL, *tmp_buf;
-    int *curr_offlen_ptr, *count, *send_size, *recv_size;
-    int *partial_send, *recd_from_proc, *start_pos;
-    /* Not convinced end_loc-st_loc couldn't be > int, so make these offsets*/
-    ADIO_Offset real_size, size, for_curr_iter, for_next_iter;
-    int req_len, flag, rank;
-    MPI_Status status;
-    ADIOI_Flatlist_node *flat_buf=NULL;
-    MPI_Aint buftype_extent, lb;
-    int coll_bufsize;
-
-    *error_code = MPI_SUCCESS;  /* changed below if error */
-    /* only I/O errors are currently reported */
-    
-/* calculate the number of reads of size coll_bufsize
-   to be done by each process and the max among all processes.
-   That gives the no. of communication phases as well.
-   coll_bufsize is obtained from the hints object. */
-
-    coll_bufsize = fd->hints->cb_buffer_size;
-
-    /* grab some initial values for st_loc and end_loc */
-    for (i=0; i < nprocs; i++) {
-	if (others_req[i].count) {
-	    st_loc = others_req[i].offsets[0];
-	    end_loc = others_req[i].offsets[0];
-	    break;
-	}
-    }
-
-    /* now find the real values */
-    for (i=0; i < nprocs; i++)
-	for (j=0; j<others_req[i].count; j++) {
-	    st_loc = ADIOI_MIN(st_loc, others_req[i].offsets[j]);
-	    end_loc = ADIOI_MAX(end_loc, (others_req[i].offsets[j]
-					  + others_req[i].lens[j] - 1));
-	}
-
-    /* calculate ntimes, the number of times this process must perform I/O
-     * operations in order to complete all the requests it has received.
-     * the need for multiple I/O operations comes from the restriction that
-     * we only use coll_bufsize bytes of memory for internal buffering.
-     */
-    if ((st_loc==-1) && (end_loc==-1)) {
-	/* this process does no I/O. */
-	ntimes = 0;
-    }
-    else {
-	/* ntimes=ceiling_div(end_loc - st_loc + 1, coll_bufsize)*/
-	ntimes = (int) ((end_loc - st_loc + coll_bufsize)/coll_bufsize);
-    }
-
-    MPI_Allreduce(&ntimes, &max_ntimes, 1, MPI_INT, MPI_MAX, fd->comm); 
-
-    read_buf = fd->io_buf;  /* Allocated at open time */
-
-    curr_offlen_ptr = (int *) ADIOI_Calloc(nprocs, sizeof(int)); 
-    /* its use is explained below. calloc initializes to 0. */
-
-    count = (int *) ADIOI_Malloc(nprocs * sizeof(int));
-    /* to store count of how many off-len pairs per proc are satisfied
-       in an iteration. */
-
-    partial_send = (int *) ADIOI_Calloc(nprocs, sizeof(int));
-    /* if only a portion of the last off-len pair is sent to a process 
-       in a particular iteration, the length sent is stored here.
-       calloc initializes to 0. */
-
-    send_size = (int *) ADIOI_Malloc(nprocs * sizeof(int));
-    /* total size of data to be sent to each proc. in an iteration */
-
-    recv_size = (int *) ADIOI_Malloc(nprocs * sizeof(int));
-    /* total size of data to be recd. from each proc. in an iteration.
-       Of size nprocs so that I can use MPI_Alltoall later. */
-
-    recd_from_proc = (int *) ADIOI_Calloc(nprocs, sizeof(int));
-    /* amount of data recd. so far from each proc. Used in
-       ADIOI_Fill_user_buffer. initialized to 0 here. */
-
-    start_pos = (int *) ADIOI_Malloc(nprocs*sizeof(int));
-    /* used to store the starting value of curr_offlen_ptr[i] in 
-       this iteration */
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-    if (!buftype_is_contig) {
-	flat_buf = ADIOI_Flatten_and_find(datatype);
-    }
-    MPI_Type_get_extent(datatype, &lb, &buftype_extent);
-
-    done = 0;
-    off = st_loc;
-    for_curr_iter = for_next_iter = 0;
-
-    MPI_Comm_rank(fd->comm, &rank);
-
-    for (m=0; m<ntimes; m++) {
-       /* read buf of size coll_bufsize (or less) */
-       /* go through all others_req and check if any are satisfied
-          by the current read */
-
-       /* since MPI guarantees that displacements in filetypes are in 
-          monotonically nondecreasing order, I can maintain a pointer
-	  (curr_offlen_ptr) to 
-          current off-len pair for each process in others_req and scan
-          further only from there. There is still a problem of filetypes
-          such as:  (1, 2, 3 are not process nos. They are just numbers for
-          three chunks of data, specified by a filetype.)
-
-                   1  -------!--
-                   2    -----!----
-                   3       --!-----
-
-          where ! indicates where the current read_size limitation cuts 
-          through the filetype.  I resolve this by reading up to !, but
-          filling the communication buffer only for 1. I copy the portion
-          left over for 2 into a tmp_buf for use in the next
-	  iteration. i.e., 2 and 3 will be satisfied in the next
-	  iteration. This simplifies filling in the user's buf at the
-	  other end, as only one off-len pair with incomplete data
-	  will be sent. I also don't need to send the individual
-	  offsets and lens along with the data, as the data is being
-	  sent in a particular order. */ 
-
-          /* off = start offset in the file for the data actually read in 
-                   this iteration 
-             size = size of data read corresponding to off
-             real_off = off minus whatever data was retained in memory from
-                  previous iteration for cases like 2, 3 illustrated above
-             real_size = size plus the extra corresponding to real_off
-             req_off = off in file for a particular contiguous request 
-                       minus what was satisfied in previous iteration
-             req_size = size corresponding to req_off */
-
-	size = ADIOI_MIN((unsigned)coll_bufsize, end_loc-st_loc+1-done); 
-	real_off = off - for_curr_iter;
-	real_size = size + for_curr_iter;
-
-	for (i=0; i<nprocs; i++) count[i] = send_size[i] = 0;
-	for_next_iter = 0;
-
-	for (i=0; i<nprocs; i++) {
-#ifdef RDCOLL_DEBUG
-	    DBG_FPRINTF(stderr, "rank %d, i %d, others_count %d\n", rank, i, others_req[i].count); 
-#endif
-	    if (others_req[i].count) {
-		start_pos[i] = curr_offlen_ptr[i];
-		for (j=curr_offlen_ptr[i]; j<others_req[i].count;
-		     j++) {
-		    if (partial_send[i]) {
-			/* this request may have been partially
-			   satisfied in the previous iteration. */
-			req_off = others_req[i].offsets[j] +
-			    partial_send[i]; 
-                        req_len = others_req[i].lens[j] -
-			    partial_send[i];
-			partial_send[i] = 0;
-			/* modify the off-len pair to reflect this change */
-			others_req[i].offsets[j] = req_off;
-			others_req[i].lens[j] = req_len;
-		    }
-		    else {
-			req_off = others_req[i].offsets[j];
-                        req_len = others_req[i].lens[j];
-		    }
-		    if (req_off < real_off + real_size) {
-			count[i]++;
-      ADIOI_Assert((((ADIO_Offset)(MPIU_Upint)read_buf)+req_off-real_off) == (ADIO_Offset)(MPIU_Upint)(read_buf+req_off-real_off));
-			MPI_Get_address(read_buf+req_off-real_off, 
-                               &(others_req[i].mem_ptrs[j]));
-      ADIOI_Assert((real_off + real_size - req_off) == (int)(real_off + real_size - req_off));
-			send_size[i] += (int)(ADIOI_MIN(real_off + real_size - req_off, 
-                                      (ADIO_Offset)(unsigned)req_len)); 
-
-			if (real_off+real_size-req_off < (ADIO_Offset)(unsigned)req_len) {
-			    partial_send[i] = (int) (real_off + real_size - req_off);
-			    if ((j+1 < others_req[i].count) && 
-                                 (others_req[i].offsets[j+1] < 
-                                     real_off+real_size)) { 
-				/* this is the case illustrated in the
-				   figure above. */
-				for_next_iter = ADIOI_MAX(for_next_iter,
-					  real_off + real_size - others_req[i].offsets[j+1]); 
-				/* max because it must cover requests 
-				   from different processes */
-			    }
-			    break;
-			}
-		    }
-		    else break;
-		}
-		curr_offlen_ptr[i] = j;
-	    }
-	}
-
-	flag = 0;
-	for (i=0; i<nprocs; i++)
-	    if (count[i]) flag = 1;
-
-	if (flag) {
-      ADIOI_Assert(size == (int)size);
-	    ADIO_ReadContig(fd, read_buf+for_curr_iter, (int)size, MPI_BYTE,
-			    ADIO_EXPLICIT_OFFSET, off, &status, error_code);
-	    if (*error_code != MPI_SUCCESS) return;
-	}
-	
-	for_curr_iter = for_next_iter;
-	
-	ADIOI_R_Exchange_data(fd, buf, flat_buf, offset_list, len_list,
-			    send_size, recv_size, count, 
-       			    start_pos, partial_send, recd_from_proc, nprocs,
-			    myrank, 
-			    buftype_is_contig, contig_access_count,
-			    min_st_offset, fd_size, fd_start, fd_end,
-			    others_req, 
-                            m, buftype_extent, buf_idx); 
-
-
-	if (for_next_iter) {
-	    tmp_buf = (char *) ADIOI_Malloc(for_next_iter);
-      ADIOI_Assert((((ADIO_Offset)(MPIU_Upint)read_buf)+real_size-for_next_iter) == (ADIO_Offset)(MPIU_Upint)(read_buf+real_size-for_next_iter));
-      ADIOI_Assert((ADIO_Size) (for_next_iter+coll_bufsize) == (size_t)(for_next_iter+coll_bufsize));
-	    memcpy(tmp_buf, read_buf+real_size-for_next_iter, for_next_iter);
-	    ADIOI_Free(fd->io_buf);
-	    fd->io_buf = (char *) ADIOI_Malloc(for_next_iter+coll_bufsize);
-	    memcpy(fd->io_buf, tmp_buf, for_next_iter);
-	    read_buf = fd->io_buf;
-	    ADIOI_Free(tmp_buf);
-	}
-
-	off += size;
-	done += size;
-    }
-
-    for (i=0; i<nprocs; i++) count[i] = send_size[i] = 0;
-    for (m=ntimes; m<max_ntimes; m++) 
-/* nothing to send, but check for recv. */
-	ADIOI_R_Exchange_data(fd, buf, flat_buf, offset_list, len_list,
-			    send_size, recv_size, count, 
-			    start_pos, partial_send, recd_from_proc, nprocs,
-			    myrank, 
-			    buftype_is_contig, contig_access_count,
-			    min_st_offset, fd_size, fd_start, fd_end,
-			    others_req, m,
-                            buftype_extent, buf_idx); 
-
-    ADIOI_Free(curr_offlen_ptr);
-    ADIOI_Free(count);
-    ADIOI_Free(partial_send);
-    ADIOI_Free(send_size);
-    ADIOI_Free(recv_size);
-    ADIOI_Free(recd_from_proc);
-    ADIOI_Free(start_pos);
-}
-
-static void ADIOI_R_Exchange_data(ADIO_File fd, void *buf, ADIOI_Flatlist_node
-			 *flat_buf, ADIO_Offset *offset_list, ADIO_Offset
-                         *len_list, int *send_size, int *recv_size,
-			 int *count, int *start_pos, int *partial_send, 
-			 int *recd_from_proc, int nprocs, 
-			 int myrank, int
-			 buftype_is_contig, int contig_access_count,
-			 ADIO_Offset min_st_offset, ADIO_Offset fd_size,
-			 ADIO_Offset *fd_start, ADIO_Offset *fd_end, 
-			 ADIOI_Access *others_req, 
-                         int iter, MPI_Aint buftype_extent, int *buf_idx)
-{
-    int i, j, k=0, tmp=0, nprocs_recv, nprocs_send;
-    char **recv_buf = NULL; 
-    MPI_Request *requests;
-    MPI_Datatype send_type;
-    MPI_Status *statuses;
-
-/* exchange send_size info so that each process knows how much to
-   receive from whom and how much memory to allocate. */
-
-    MPI_Alltoall(send_size, 1, MPI_INT, recv_size, 1, MPI_INT, fd->comm);
-
-    nprocs_recv = 0;
-    for (i=0; i < nprocs; i++) if (recv_size[i]) nprocs_recv++;
-
-    nprocs_send = 0;
-    for (i=0; i<nprocs; i++) if (send_size[i]) nprocs_send++;
-
-    requests = (MPI_Request *)
-	ADIOI_Malloc((nprocs_send+nprocs_recv+1)*sizeof(MPI_Request));
-/* +1 to avoid a 0-size malloc */
-
-/* post recvs. if buftype_is_contig, data can be directly recd. into
-   user buf at location given by buf_idx. else use recv_buf. */
-
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5032, 0, NULL);
-#endif
-
-    if (buftype_is_contig) {
-	j = 0;
-	for (i=0; i < nprocs; i++) 
-	    if (recv_size[i]) {
-		MPI_Irecv(((char *) buf) + buf_idx[i], recv_size[i], 
-		  MPI_BYTE, i, myrank+i+100*iter, fd->comm, requests+j);
-		j++;
-		buf_idx[i] += recv_size[i];
-	    }
-    }
-    else {
-/* allocate memory for recv_buf and post receives */
-	recv_buf = (char **) ADIOI_Malloc(nprocs * sizeof(char*));
-	for (i=0; i < nprocs; i++) 
-	    if (recv_size[i]) recv_buf[i] = 
-                                  (char *) ADIOI_Malloc(recv_size[i]);
-
-        j = 0;
-        for (i=0; i < nprocs; i++)
-            if (recv_size[i]) {
-                MPI_Irecv(recv_buf[i], recv_size[i], MPI_BYTE, i,
-                          myrank+i+100*iter, fd->comm, requests+j);
-                j++;
-#ifdef RDCOLL_DEBUG
-                DBG_FPRINTF(stderr, "node %d, recv_size %d, tag %d \n",
-                            myrank, recv_size[i], myrank+i+100*iter);
-#endif
-            }
-    }
-
-/* create derived datatypes and send data */
-
-    j = 0;
-    for (i=0; i<nprocs; i++) {
-	if (send_size[i]) {
-/* take care if the last off-len pair is a partial send */
-	    if (partial_send[i]) {
-		k = start_pos[i] + count[i] - 1;
-		tmp = others_req[i].lens[k];
-		others_req[i].lens[k] = partial_send[i];
-	    }
-	    ADIOI_Type_create_hindexed_x(count[i],
-		  &(others_req[i].lens[start_pos[i]]),
-	            &(others_req[i].mem_ptrs[start_pos[i]]), 
-			 MPI_BYTE, &send_type);
-	    /* absolute displacement; use MPI_BOTTOM in send */
-	    MPI_Type_commit(&send_type);
-	    MPI_Isend(MPI_BOTTOM, 1, send_type, i, myrank+i+100*iter,
-		      fd->comm, requests+nprocs_recv+j);
-	    MPI_Type_free(&send_type);
-	    if (partial_send[i]) others_req[i].lens[k] = tmp;
-	    j++;
-	}
-    }
-
-    statuses = (MPI_Status *) ADIOI_Malloc((nprocs_send+nprocs_recv+1) * \
-                                     sizeof(MPI_Status)); 
-     /* +1 to avoid a 0-size malloc */
-
-    /* wait on the receives */
-    if (nprocs_recv) {
-#ifdef NEEDS_MPI_TEST
-	j = 0;
-	while (!j) MPI_Testall(nprocs_recv, requests, &j, statuses);
-#else
-	MPI_Waitall(nprocs_recv, requests, statuses);
-#endif
-
-	/* if noncontiguous, to the copies from the recv buffers */
-	if (!buftype_is_contig) 
-	    ADIOI_Fill_user_buffer(fd, buf, flat_buf, recv_buf,
-				   offset_list, len_list, (unsigned*)recv_size, 
-				   requests, statuses, recd_from_proc, 
-				   nprocs, contig_access_count,
-				   min_st_offset, fd_size, fd_start, fd_end,
-				   buftype_extent);
-    }
-
-    /* wait on the sends*/
-    MPI_Waitall(nprocs_send, requests+nprocs_recv, statuses+nprocs_recv);
-
-    ADIOI_Free(statuses);
-    ADIOI_Free(requests);
-
-    if (!buftype_is_contig) {
-	for (i=0; i < nprocs; i++) 
-	    if (recv_size[i]) ADIOI_Free(recv_buf[i]);
-	ADIOI_Free(recv_buf);
-    }
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5033, 0, NULL);
-#endif
-}
-
-#define ADIOI_BUF_INCR \
-{ \
-    while (buf_incr) { \
-	size_in_buf = ADIOI_MIN(buf_incr, flat_buf_sz); \
-	user_buf_idx += size_in_buf; \
-	flat_buf_sz -= size_in_buf; \
-	if (!flat_buf_sz) { \
-            if (flat_buf_idx < (flat_buf->count - 1)) flat_buf_idx++; \
-            else { \
-                flat_buf_idx = 0; \
-                n_buftypes++; \
-            } \
-            user_buf_idx = flat_buf->indices[flat_buf_idx] + \
-                              (ADIO_Offset)n_buftypes*(ADIO_Offset)buftype_extent; \
-	    flat_buf_sz = flat_buf->blocklens[flat_buf_idx]; \
-	} \
-	buf_incr -= size_in_buf; \
-    } \
-}
-
-
-#define ADIOI_BUF_COPY \
-{ \
-    while (size) { \
-	size_in_buf = ADIOI_MIN(size, flat_buf_sz); \
-  ADIOI_Assert((((ADIO_Offset)(MPIU_Upint)buf) + user_buf_idx) == (ADIO_Offset)(MPIU_Upint)((MPIU_Upint)buf + user_buf_idx)); \
-  ADIOI_Assert((ADIO_Size) size_in_buf == (size_t)size_in_buf);          \
-	memcpy(((char *) buf) + user_buf_idx, \
-	       &(recv_buf[p][recv_buf_idx[p]]), size_in_buf); \
-	recv_buf_idx[p] += size_in_buf; /* already tested (size_t)size_in_buf*/ \
-	user_buf_idx += size_in_buf; \
-	flat_buf_sz -= size_in_buf; \
-	if (!flat_buf_sz) { \
-            if (flat_buf_idx < (flat_buf->count - 1)) flat_buf_idx++; \
-            else { \
-                flat_buf_idx = 0; \
-                n_buftypes++; \
-            } \
-            user_buf_idx = flat_buf->indices[flat_buf_idx] + \
-                              (ADIO_Offset)n_buftypes*(ADIO_Offset)buftype_extent; \
-	    flat_buf_sz = flat_buf->blocklens[flat_buf_idx]; \
-	} \
-	size -= size_in_buf; \
-	buf_incr -= size_in_buf; \
-    } \
-    ADIOI_BUF_INCR \
-}
-
-void ADIOI_Fill_user_buffer(ADIO_File fd, void *buf, ADIOI_Flatlist_node
-				   *flat_buf, char **recv_buf, ADIO_Offset 
-				   *offset_list, ADIO_Offset *len_list, 
-				   unsigned *recv_size, 
-				   MPI_Request *requests, MPI_Status *statuses,
-				   int *recd_from_proc, int nprocs,
-				   int contig_access_count, 
-				   ADIO_Offset min_st_offset, 
-				   ADIO_Offset fd_size, ADIO_Offset *fd_start, 
-				   ADIO_Offset *fd_end,
-				   MPI_Aint buftype_extent)
-{
-
-/* this function is only called if buftype is not contig */
-
-    int i, p, flat_buf_idx;
-    ADIO_Offset flat_buf_sz, size_in_buf, buf_incr, size;
-    int n_buftypes;
-    ADIO_Offset off, len, rem_len, user_buf_idx;
-    /* Not sure unsigned is necessary, but it makes the math safer */
-    unsigned *curr_from_proc, *done_from_proc, *recv_buf_idx;
-
-    ADIOI_UNREFERENCED_ARG(requests);
-    ADIOI_UNREFERENCED_ARG(statuses);
-
-/*  curr_from_proc[p] = amount of data recd from proc. p that has already
-                        been accounted for so far
-    done_from_proc[p] = amount of data already recd from proc. p and 
-                        filled into user buffer in previous iterations
-    user_buf_idx = current location in user buffer 
-    recv_buf_idx[p] = current location in recv_buf of proc. p  */
-    curr_from_proc = (unsigned *) ADIOI_Malloc(nprocs * sizeof(unsigned));
-    done_from_proc = (unsigned *) ADIOI_Malloc(nprocs * sizeof(unsigned));
-    recv_buf_idx   = (unsigned *) ADIOI_Malloc(nprocs * sizeof(unsigned));
-
-    for (i=0; i < nprocs; i++) {
-	recv_buf_idx[i] = curr_from_proc[i] = 0;
-	done_from_proc[i] = recd_from_proc[i];
-    }
-
-    user_buf_idx = flat_buf->indices[0];
-    flat_buf_idx = 0;
-    n_buftypes = 0;
-    flat_buf_sz = flat_buf->blocklens[0];
-
-    /* flat_buf_idx = current index into flattened buftype
-       flat_buf_sz = size of current contiguous component in 
-                flattened buf */
-
-    for (i=0; i<contig_access_count; i++) { 
-	off     = offset_list[i];
-	rem_len = len_list[i];
-
-	/* this request may span the file domains of more than one process */
-	while (rem_len != 0) {
-	    len = rem_len;
-	    /* NOTE: len value is modified by ADIOI_Calc_aggregator() to be no
-	     * longer than the single region that processor "p" is responsible
-	     * for.
-	     */
-	    p = ADIOI_Calc_aggregator(fd,
-				      off,
-				      min_st_offset,
-				      &len,
-				      fd_size,
-				      fd_start,
-				      fd_end);
-
-	    if (recv_buf_idx[p] < recv_size[p]) {
-		if (curr_from_proc[p]+len > done_from_proc[p]) {
-		    if (done_from_proc[p] > curr_from_proc[p]) {
-			size = ADIOI_MIN(curr_from_proc[p] + len - 
-			      done_from_proc[p], recv_size[p]-recv_buf_idx[p]);
-			buf_incr = done_from_proc[p] - curr_from_proc[p];
-			ADIOI_BUF_INCR
-			buf_incr = curr_from_proc[p]+len-done_from_proc[p];
-      ADIOI_Assert((done_from_proc[p] + size) == (unsigned)((ADIO_Offset)done_from_proc[p] + size));
-			curr_from_proc[p] = done_from_proc[p] + size;
-			ADIOI_BUF_COPY
-		    }
-		    else {
-			size = ADIOI_MIN(len,recv_size[p]-recv_buf_idx[p]);
-			buf_incr = len;
-      ADIOI_Assert((curr_from_proc[p] + size) == (unsigned)((ADIO_Offset)curr_from_proc[p] + size));
-			curr_from_proc[p] += (unsigned) size;
-			ADIOI_BUF_COPY
-		    }
-		}
-		else {
-        ADIOI_Assert((curr_from_proc[p] + len) == (unsigned)((ADIO_Offset)curr_from_proc[p] + len));
-		    curr_from_proc[p] += (unsigned) len;
-		    buf_incr = len;
-		    ADIOI_BUF_INCR
-		}
-	    }
-	    else {
-		buf_incr = len;
-		ADIOI_BUF_INCR
-	    }
-	    off     += len;
-	    rem_len -= len;
-	}
-    }
-    for (i=0; i < nprocs; i++) 
-	if (recv_size[i]) recd_from_proc[i] = curr_from_proc[i];
-
-    ADIOI_Free(curr_from_proc);
-    ADIOI_Free(done_from_proc);
-    ADIOI_Free(recv_buf_idx);
-}
diff --git a/3rd-party/romio321/adio/common/ad_read_str.c b/3rd-party/romio321/adio/common/ad_read_str.c
deleted file mode 100644
index fbb4bbe3c1e..00000000000
--- a/3rd-party/romio321/adio/common/ad_read_str.c
+++ /dev/null
@@ -1,403 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-
-#define ADIOI_BUFFERED_READ \
-{ \
-    if (req_off >= readbuf_off + readbuf_len) { \
-	readbuf_off = req_off; \
-	readbuf_len = (unsigned) (ADIOI_MIN(max_bufsize, end_offset-readbuf_off+1));\
-	ADIO_ReadContig(fd, readbuf, readbuf_len, MPI_BYTE, \
-              ADIO_EXPLICIT_OFFSET, readbuf_off, &status1, error_code); \
-        if (*error_code != MPI_SUCCESS) return; \
-    } \
-    while (req_len > readbuf_off + readbuf_len - req_off) { \
-  ADIOI_Assert((readbuf_off + readbuf_len - req_off) == (int) (readbuf_off + readbuf_len - req_off));\
-	partial_read = (int) (readbuf_off + readbuf_len - req_off); \
-	tmp_buf = (char *) ADIOI_Malloc(partial_read); \
-	memcpy(tmp_buf, readbuf+readbuf_len-partial_read, partial_read); \
-	ADIOI_Free(readbuf); \
-	readbuf = (char *) ADIOI_Malloc(partial_read + max_bufsize); \
-	memcpy(readbuf, tmp_buf, partial_read); \
-	ADIOI_Free(tmp_buf); \
-	readbuf_off += readbuf_len-partial_read; \
-	readbuf_len = (unsigned) (partial_read + ADIOI_MIN(max_bufsize, \
-				       end_offset-readbuf_off+1)); \
-	ADIO_ReadContig(fd, readbuf+partial_read, readbuf_len-partial_read, \
-             MPI_BYTE, ADIO_EXPLICIT_OFFSET, readbuf_off+partial_read, \
-             &status1, error_code); \
-        if (*error_code != MPI_SUCCESS) return; \
-    } \
-    ADIOI_Assert((ADIO_Size) req_len == (size_t)req_len);                \
-    memcpy((char *)buf + userbuf_off, readbuf+req_off-readbuf_off, req_len); \
-}
-
-
-void ADIOI_GEN_ReadStrided(ADIO_File fd, void *buf, int count,
-                       MPI_Datatype datatype, int file_ptr_type,
-                       ADIO_Offset offset, ADIO_Status *status, int
-                       *error_code)
-{
-
-
-/* offset is in units of etype relative to the filetype. */
-
-    ADIOI_Flatlist_node *flat_buf, *flat_file;
-    ADIO_Offset i_offset, new_brd_size, brd_size, size;
-    int i, j, k, st_index=0;
-    MPI_Count num, bufsize;
-    int n_etypes_in_filetype;
-    ADIO_Offset n_filetypes, etype_in_filetype, st_n_filetypes, size_in_filetype;
-    ADIO_Offset abs_off_in_filetype=0, new_frd_size, frd_size=0, st_frd_size;
-    MPI_Count filetype_size, etype_size, buftype_size, partial_read;
-    MPI_Aint filetype_extent, buftype_extent, lb; 
-    int buf_count, buftype_is_contig, filetype_is_contig;
-    ADIO_Offset userbuf_off, req_len, sum;
-    ADIO_Offset off, req_off, disp, end_offset=0, readbuf_off, start_off;
-    char *readbuf, *tmp_buf, *value;
-    int info_flag;
-    unsigned max_bufsize, readbuf_len;
-    ADIO_Status status1;
-
-    if (fd->hints->ds_read == ADIOI_HINT_DISABLE) {
-    	/* if user has disabled data sieving on reads, use naive
-	 * approach instead.
-	 */
-	ADIOI_GEN_ReadStrided_naive(fd, 
-				    buf,
-				    count,
-				    datatype,
-				    file_ptr_type,
-				    offset,
-				    status,
-				    error_code);
-    	return;
-    }
-
-    *error_code = MPI_SUCCESS;  /* changed below if error */
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-
-    MPI_Type_size_x(fd->filetype, &filetype_size);
-    if ( ! filetype_size ) {
-#ifdef HAVE_STATUS_SET_BYTES
-	MPIR_Status_set_bytes(status, datatype, 0);
-#endif
-	*error_code = MPI_SUCCESS; 
-	return;
-    }
-
-    MPI_Type_get_extent(fd->filetype, &lb, &filetype_extent);
-    MPI_Type_size_x(datatype, &buftype_size);
-    MPI_Type_get_extent(datatype, &lb, &buftype_extent);
-    etype_size = fd->etype_size;
-
-    ADIOI_Assert((buftype_size * count) == ((ADIO_Offset)(MPI_Count)buftype_size * (ADIO_Offset)count));
-    bufsize = buftype_size * count;
-
-/* get max_bufsize from the info object. */
-
-    value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
-    ADIOI_Info_get(fd->info, "ind_rd_buffer_size", MPI_MAX_INFO_VAL, value, 
-                 &info_flag);
-    max_bufsize = atoi(value);
-    ADIOI_Free(value);
-
-
-    if (!buftype_is_contig && filetype_is_contig) {
-
-/* noncontiguous in memory, contiguous in file. */
-
-	flat_buf = ADIOI_Flatten_and_find(datatype);
-
-        off = (file_ptr_type == ADIO_INDIVIDUAL) ? fd->fp_ind : 
-                 fd->disp + (ADIO_Offset)etype_size * offset;
-
-	start_off = off;
-	end_offset = off + bufsize - 1;
-        readbuf_off = off;
-        readbuf = (char *) ADIOI_Malloc(max_bufsize);
-        readbuf_len = (unsigned) (ADIOI_MIN(max_bufsize, end_offset-readbuf_off+1));
-
-/* if atomicity is true, lock (exclusive) the region to be accessed */
-        if ((fd->atomicity) && ADIO_Feature(fd, ADIO_LOCKS))
-            ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
-
-        ADIO_ReadContig(fd, readbuf, readbuf_len, MPI_BYTE, 
-            ADIO_EXPLICIT_OFFSET, readbuf_off, &status1, error_code);
-	if (*error_code != MPI_SUCCESS) return;
-
-        for (j=0; j<count; j++) 
-        {
-              for (i=0; i<flat_buf->count; i++) {
-                  userbuf_off = (ADIO_Offset)j*(ADIO_Offset)buftype_extent + flat_buf->indices[i];
-      req_off = off;
-      req_len = flat_buf->blocklens[i];
-      ADIOI_BUFFERED_READ
-                  off += flat_buf->blocklens[i];
-              }
-        }
-
-        if ((fd->atomicity) && ADIO_Feature(fd, ADIO_LOCKS))
-            ADIOI_UNLOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
-
-        if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
-
-	ADIOI_Free(readbuf); 
-    }
-
-    else {  /* noncontiguous in file */
-
-/* filetype already flattened in ADIO_Open */
-	flat_file = ADIOI_Flatlist;
-	while (flat_file->type != fd->filetype) flat_file = flat_file->next;
-	disp = fd->disp;
-
-	if (file_ptr_type == ADIO_INDIVIDUAL) {
-	    /* Wei-keng reworked type processing to be a bit more efficient */
-            offset       = fd->fp_ind - disp;
-            n_filetypes  = (offset - flat_file->indices[0]) / filetype_extent;
-	    offset -= (ADIO_Offset)n_filetypes * filetype_extent;
-	    /* now offset is local to this extent */
-
-            /* find the block where offset is located, skip blocklens[i]==0 */
-            for (i=0; i<flat_file->count; i++) {
-                ADIO_Offset dist;
-                if (flat_file->blocklens[i] == 0) continue;
-                dist = flat_file->indices[i] + flat_file->blocklens[i] - offset;
-                /* frd_size is from offset to the end of block i */
-		if (dist == 0) {
-		    i++;
-		    offset   = flat_file->indices[i];
-		    frd_size = flat_file->blocklens[i];
-		    break;
-		}
-		if (dist > 0) {
-                    frd_size = dist;
-		    break;
-		}
-	    }
-            st_index = i;  /* starting index in flat_file->indices[] */
-            offset += disp + (ADIO_Offset)n_filetypes*filetype_extent;
-        }
-	else {
-	    n_etypes_in_filetype = filetype_size/etype_size;
-	    n_filetypes = offset / n_etypes_in_filetype;
-	    etype_in_filetype = offset % n_etypes_in_filetype;
-	    size_in_filetype = etype_in_filetype * etype_size;
- 
-	    sum = 0;
-	    for (i=0; i<flat_file->count; i++) {
-		sum += flat_file->blocklens[i];
-		if (sum > size_in_filetype) {
-		    st_index = i;
-		    frd_size = sum - size_in_filetype;
-		    abs_off_in_filetype = flat_file->indices[i] +
-			size_in_filetype - (sum - flat_file->blocklens[i]);
-		    break;
-		}
-	    }
-
-	    /* abs. offset in bytes in the file */
-	    offset = disp + (ADIO_Offset) n_filetypes*filetype_extent + 
-		    abs_off_in_filetype;
-	}
-
-        start_off = offset;
-
-	/* Wei-keng Liao: read request is within a single flat_file contig
-	 * block e.g. with subarray types that actually describe the whole
-	 * array */
-	if (buftype_is_contig && bufsize <= frd_size) {
-	    /* a count of bytes can overflow. operate on original type instead */
-            ADIO_ReadContig(fd, buf, count, datatype, ADIO_EXPLICIT_OFFSET,
-                             offset, status, error_code);
-
-	    if (file_ptr_type == ADIO_INDIVIDUAL) {
-                /* update MPI-IO file pointer to point to the first byte that 
-		 * can be accessed in the fileview. */
-		fd->fp_ind = offset + bufsize;
-		if (bufsize == frd_size) {
-		    do {
-			st_index++;
-			if (st_index == flat_file->count) {
-			    st_index = 0;
-			    n_filetypes++;
-			}
-                    } while (flat_file->blocklens[st_index] == 0);
-		    fd->fp_ind = disp + flat_file->indices[st_index]
-                               + n_filetypes*filetype_extent;
-		}
-	    }
-	    fd->fp_sys_posn = -1;   /* set it to null. */ 
-#ifdef HAVE_STATUS_SET_BYTES
-	    MPIR_Status_set_bytes(status, datatype, bufsize);
-#endif 
-            return;
-	}
-
-       /* Calculate end_offset, the last byte-offset that will be accessed.
-         e.g., if start_offset=0 and 100 bytes to be read, end_offset=99*/
-
-	st_frd_size = frd_size;
-	st_n_filetypes = n_filetypes;
-	i_offset = 0;
-	j = st_index;
-	off = offset;
-	frd_size = ADIOI_MIN(st_frd_size, bufsize);
-	while (i_offset < bufsize) {
-	    i_offset += frd_size;
-	    end_offset = off + frd_size - 1;
-
-	    j = (j+1) % flat_file->count;
-            n_filetypes += (j == 0) ? 1 : 0;
-            while (flat_file->blocklens[j]==0) {
-		j = (j+1) % flat_file->count;
-		n_filetypes += (j == 0) ? 1 : 0;
-	    }
-	    off = disp + flat_file->indices[j] + n_filetypes*(ADIO_Offset)filetype_extent;
-	    frd_size = ADIOI_MIN(flat_file->blocklens[j], bufsize-i_offset);
-	}
-
-/* if atomicity is true, lock (exclusive) the region to be accessed */
-        if ((fd->atomicity) && ADIO_Feature(fd, ADIO_LOCKS))
-            ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
-
-	readbuf_off = 0;
-	readbuf_len = 0;
-	readbuf = (char *) ADIOI_Malloc(max_bufsize);
-
-	if (buftype_is_contig && !filetype_is_contig) {
-
-/* contiguous in memory, noncontiguous in file. should be the most
-   common case. */
-
-	    i_offset = 0;
-	    j = st_index;
-	    off = offset;
-	    n_filetypes = st_n_filetypes;
-	    frd_size = ADIOI_MIN(st_frd_size, bufsize);
-	    while (i_offset < bufsize) {
-                if (frd_size) { 
-                    /* TYPE_UB and TYPE_LB can result in 
-                       frd_size = 0. save system call in such cases */ 
-		    /* lseek(fd->fd_sys, off, SEEK_SET);
-		    err = read(fd->fd_sys, ((char *) buf) + i, frd_size);*/
-
-		    req_off = off;
-		    req_len = frd_size;
-		    userbuf_off = i_offset;
-		    ADIOI_BUFFERED_READ
-		}
-		i_offset += frd_size;
-
-                if (off + frd_size < disp + flat_file->indices[j] +
-                   flat_file->blocklens[j] + n_filetypes*(ADIO_Offset)filetype_extent)
-                       off += frd_size;
-                /* did not reach end of contiguous block in filetype.
-                   no more I/O needed. off is incremented by frd_size. */
-                else {
-                    j = (j+1) % flat_file->count;
-                    n_filetypes += (j == 0) ? 1 : 0;
-                    while (flat_file->blocklens[j]==0) {
-                        j = (j+1) % flat_file->count;
-                        n_filetypes += (j == 0) ? 1 : 0;
-                    }
-		    off = disp + flat_file->indices[j] + 
-                                        n_filetypes*(ADIO_Offset)filetype_extent;
-		    frd_size = ADIOI_MIN(flat_file->blocklens[j], bufsize-i_offset);
-		}
-	    }
-	}
-	else {
-/* noncontiguous in memory as well as in file */
-
-	    flat_buf = ADIOI_Flatten_and_find(datatype);
-
-	    k = num = buf_count = 0;
-	    i_offset = flat_buf->indices[0];
-	    j = st_index;
-	    off = offset;
-	    n_filetypes = st_n_filetypes;
-	    frd_size = st_frd_size;
-	    brd_size = flat_buf->blocklens[0];
-
-	    while (num < bufsize) {
-		size = ADIOI_MIN(frd_size, brd_size);
-		if (size) {
-		    /* lseek(fd->fd_sys, off, SEEK_SET);
-		    err = read(fd->fd_sys, ((char *) buf) + i, size); */
-
-		    req_off = off;
-		    req_len = size;
-		    userbuf_off = i_offset;
-		    ADIOI_BUFFERED_READ
-		}
-
-		new_frd_size = frd_size;
-		new_brd_size = brd_size;
-
-		if (size == frd_size) {
-/* reached end of contiguous block in file */
-                    j = (j+1) % flat_file->count;
-                    n_filetypes += (j == 0) ? 1 : 0;
-                    while (flat_file->blocklens[j]==0) {
-                        j = (j+1) % flat_file->count;
-                        n_filetypes += (j == 0) ? 1 : 0;
-                    }
-		    off = disp + flat_file->indices[j] + 
-          n_filetypes*(ADIO_Offset)filetype_extent;
-
-		    new_frd_size = flat_file->blocklens[j];
-		    if (size != brd_size) {
-			i_offset += size;
-			new_brd_size -= size;
-		    }
-		}
-
-		if (size == brd_size) {
-/* reached end of contiguous block in memory */
-
-		    k = (k + 1)%flat_buf->count;
-		    buf_count++;
-		    i_offset = ((ADIO_Offset)buftype_extent*(ADIO_Offset)(buf_count/flat_buf->count) +
-			flat_buf->indices[k]);
-		    new_brd_size = flat_buf->blocklens[k];
-		    if (size != frd_size) {
-			off += size;
-			new_frd_size -= size;
-		    }
-		}
-    ADIOI_Assert(((ADIO_Offset)num + size) == (unsigned)(num + size));
-		num += size;
-		frd_size = new_frd_size;
-                brd_size = new_brd_size;
-	    }
-	}
-	
-        if ((fd->atomicity) && ADIO_Feature(fd, ADIO_LOCKS))
-            ADIOI_UNLOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
-
-	if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
-
-	ADIOI_Free(readbuf); /* malloced in the buffered_read macro */
-    }
-
-    fd->fp_sys_posn = -1;   /* set it to null. */
-
-#ifdef HAVE_STATUS_SET_BYTES
-    MPIR_Status_set_bytes(status, datatype, bufsize);
-/* This is a temporary way of filling in status. The right way is to 
-   keep track of how much data was actually read and placed in buf 
-   by ADIOI_BUFFERED_READ. */
-#endif
-
-    if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
-}
diff --git a/3rd-party/romio321/adio/common/ad_read_str_naive.c b/3rd-party/romio321/adio/common/ad_read_str_naive.c
deleted file mode 100644
index a65eede9558..00000000000
--- a/3rd-party/romio321/adio/common/ad_read_str_naive.c
+++ /dev/null
@@ -1,380 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-
-void ADIOI_GEN_ReadStrided_naive(ADIO_File fd, void *buf, int count,
-                       MPI_Datatype buftype, int file_ptr_type,
-                       ADIO_Offset offset, ADIO_Status *status, int
-                       *error_code)
-{
-    /* offset is in units of etype relative to the filetype. */
-
-    ADIOI_Flatlist_node *flat_buf, *flat_file;
-    ADIO_Offset size, brd_size, frd_size=0, req_len, sum;
-    int b_index;
-    int n_etypes_in_filetype;
-    ADIO_Offset n_filetypes, etype_in_filetype;
-    ADIO_Offset abs_off_in_filetype=0;
-    MPI_Count bufsize, filetype_size, buftype_size, size_in_filetype;
-    ADIO_Offset etype_size;
-    MPI_Aint filetype_extent, buftype_extent, lb; 
-    int buf_count, buftype_is_contig, filetype_is_contig;
-    ADIO_Offset userbuf_off;
-    ADIO_Offset off, req_off, disp, end_offset=0, start_off;
-    ADIO_Status status1;
-
-    *error_code = MPI_SUCCESS;  /* changed below if error */
-
-    ADIOI_Datatype_iscontig(buftype, &buftype_is_contig);
-    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-
-    MPI_Type_size_x(fd->filetype, &filetype_size);
-    if ( ! filetype_size ) {
-#ifdef HAVE_STATUS_SET_BYTES
-	MPIR_Status_set_bytes(status, buftype, 0);
-#endif
-	*error_code = MPI_SUCCESS; 
-	return;
-    }
-
-    MPI_Type_get_extent(fd->filetype, &lb, &filetype_extent);
-    MPI_Type_size_x(buftype, &buftype_size);
-    MPI_Type_get_extent(buftype, &lb, &buftype_extent);
-    etype_size = fd->etype_size;
-
-    ADIOI_Assert((buftype_size * count) == ((ADIO_Offset)buftype_size * (ADIO_Offset)count));
-    bufsize = buftype_size * count;
-
-    /* contiguous in buftype and filetype is handled elsewhere */
-
-    if (!buftype_is_contig && filetype_is_contig) {
-    	int b_count;
-	/* noncontiguous in memory, contiguous in file. */
-
-	flat_buf = ADIOI_Flatten_and_find(buftype);
-
-        off = (file_ptr_type == ADIO_INDIVIDUAL) ? fd->fp_ind : 
-              fd->disp + etype_size * offset;
-
-	start_off = off;
-	end_offset = off + bufsize - 1;
-
-	/* if atomicity is true, lock (exclusive) the region to be accessed */
-        if ((fd->atomicity) && ADIO_Feature(fd, ADIO_LOCKS))
-	{
-            ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
-	}
-
-	/* for each region in the buffer, grab the data and put it in
-	 * place
-	 */
-        for (b_count=0; b_count < count; b_count++) {
-            for (b_index=0; b_index < flat_buf->count; b_index++) {
-                userbuf_off = (ADIO_Offset)b_count*(ADIO_Offset)buftype_extent + 
-		              flat_buf->indices[b_index];
-		req_off = off;
-		req_len = flat_buf->blocklens[b_index];
-
-    ADIOI_Assert((((ADIO_Offset)(MPIU_Upint)buf) + userbuf_off) == (ADIO_Offset)(MPIU_Upint)((MPIU_Upint)buf + userbuf_off));
-    ADIOI_Assert(req_len == (int) req_len);
-		ADIO_ReadContig(fd, 
-				(char *) buf + userbuf_off,
-				req_len, 
-				MPI_BYTE, 
-		    		ADIO_EXPLICIT_OFFSET,
-				req_off,
-				&status1,
-				error_code);
-		if (*error_code != MPI_SUCCESS) return;
-
-		/* off is (potentially) used to save the final offset later */
-                off += flat_buf->blocklens[b_index];
-            }
-	}
-
-        if ((fd->atomicity) && ADIO_Feature(fd, ADIO_LOCKS))
-	{
-            ADIOI_UNLOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
-	}
-
-        if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
-
-    }
-
-    else {  /* noncontiguous in file */
-    	int f_index, st_index = 0; 
-      ADIO_Offset st_n_filetypes;
-      ADIO_Offset st_frd_size;
-	int flag;
-
-        /* First we're going to calculate a set of values for use in all
-	 * the noncontiguous in file cases:
-	 * start_off - starting byte position of data in file
-	 * end_offset - last byte offset to be acessed in the file
-	 * st_n_filetypes - how far into the file we start in terms of
-	 *                  whole filetypes
-	 * st_index - index of block in first filetype that we will be
-	 *            starting in (?)
-	 * st_frd_size - size of the data in the first filetype block
-	 *               that we will read (accounts for being part-way
-	 *               into reading this block of the filetype
-	 *
-	 */
-
-	/* filetype already flattened in ADIO_Open */
-	flat_file = ADIOI_Flatlist;
-	while (flat_file->type != fd->filetype) flat_file = flat_file->next;
-	disp = fd->disp;
-
-	if (file_ptr_type == ADIO_INDIVIDUAL) {
-	    start_off = fd->fp_ind; /* in bytes */
-	    n_filetypes = -1;
-	    flag = 0;
-	    while (!flag) {
-                n_filetypes++;
-		for (f_index=0; f_index < flat_file->count; f_index++) {
-		    if (disp + flat_file->indices[f_index] + 
-                       n_filetypes*(ADIO_Offset)filetype_extent + 
-		       flat_file->blocklens[f_index] >= start_off) 
-		    {
-		    	/* this block contains our starting position */
-
-			st_index = f_index;
-			frd_size = disp + flat_file->indices[f_index] + 
-		 	           n_filetypes*(ADIO_Offset)filetype_extent + 
-				   flat_file->blocklens[f_index] - start_off;
-			flag = 1;
-			break;
-		    }
-		}
-	    }
-	}
-	else {
-	    n_etypes_in_filetype = filetype_size/etype_size;
-	    n_filetypes = offset / n_etypes_in_filetype;
-	    etype_in_filetype = (int) (offset % n_etypes_in_filetype);
-	    size_in_filetype = (unsigned)etype_in_filetype * (unsigned)etype_size;
- 
-	    sum = 0;
-	    for (f_index=0; f_index < flat_file->count; f_index++) {
-		sum += flat_file->blocklens[f_index];
-		if (sum > size_in_filetype) {
-		    st_index = f_index;
-		    frd_size = sum - size_in_filetype;
-		    abs_off_in_filetype = flat_file->indices[f_index] +
-			                  size_in_filetype - 
-			                  (sum - flat_file->blocklens[f_index]);
-		    break;
-		}
-	    }
-
-	    /* abs. offset in bytes in the file */
-	    start_off = disp + n_filetypes*(ADIO_Offset)filetype_extent + 
-	    	        abs_off_in_filetype;
-	}
-
-	st_frd_size = frd_size;
-	st_n_filetypes = n_filetypes;
-
-	/* start_off, st_n_filetypes, st_index, and st_frd_size are 
-	 * all calculated at this point
-	 */
-
-        /* Calculate end_offset, the last byte-offset that will be accessed.
-         * e.g., if start_off=0 and 100 bytes to be read, end_offset=99
-	 */
-	userbuf_off = 0;
-	f_index = st_index;
-	off = start_off;
-	frd_size = ADIOI_MIN(st_frd_size, bufsize);
-	while (userbuf_off < bufsize) {
-	    userbuf_off += frd_size;
-	    end_offset = off + frd_size - 1;
-
-	    if (f_index < (flat_file->count - 1)) f_index++;
-	    else {
-		f_index = 0;
-		n_filetypes++;
-	    }
-
-	    off = disp + flat_file->indices[f_index] + 
-	          n_filetypes*(ADIO_Offset)filetype_extent;
-	    frd_size = ADIOI_MIN(flat_file->blocklens[f_index], 
-	                         bufsize-(unsigned)userbuf_off);
-	}
-
-	/* End of calculations.  At this point the following values have
-	 * been calculated and are ready for use:
-	 * - start_off
-	 * - end_offset
-	 * - st_n_filetypes
-	 * - st_index
-	 * - st_frd_size
-	 */
-
-	/* if atomicity is true, lock (exclusive) the region to be accessed */
-        if ((fd->atomicity) && ADIO_Feature(fd, ADIO_LOCKS))
-	{
-            ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
-	}
-
-	if (buftype_is_contig && !filetype_is_contig) {
-	    /* contiguous in memory, noncontiguous in file. should be the
-	     * most common case.
-	     */
-
-	    userbuf_off = 0;
-	    f_index = st_index;
-	    off = start_off;
-	    n_filetypes = st_n_filetypes;
-	    frd_size = ADIOI_MIN(st_frd_size, bufsize);
-
-	    /* while there is still space in the buffer, read more data */
-	    while (userbuf_off < bufsize) {
-                if (frd_size) { 
-                    /* TYPE_UB and TYPE_LB can result in 
-                       frd_size = 0. save system call in such cases */ 
-		    req_off = off;
-		    req_len = frd_size;
-
-        ADIOI_Assert((((ADIO_Offset)(MPIU_Upint)buf) + userbuf_off) == (ADIO_Offset)(MPIU_Upint)((MPIU_Upint)buf + userbuf_off));
-        ADIOI_Assert(req_len == (int) req_len);
-		    ADIO_ReadContig(fd, 
-				    (char *) buf + userbuf_off,
-				    req_len, 
-				    MPI_BYTE, 
-				    ADIO_EXPLICIT_OFFSET,
-				    req_off,
-				    &status1,
-				    error_code);
-		    if (*error_code != MPI_SUCCESS) return;
-		}
-		userbuf_off += frd_size;
-
-                if (off + frd_size < disp + flat_file->indices[f_index] +
-                   flat_file->blocklens[f_index] + 
-		   n_filetypes*(ADIO_Offset)filetype_extent)
-		{
-		    /* important that this value be correct, as it is
-		     * used to set the offset in the fd near the end of
-		     * this function.
-		     */
-                    off += frd_size;
-		}
-                /* did not reach end of contiguous block in filetype.
-                 * no more I/O needed. off is incremented by frd_size.
-		 */
-                else {
-		    if (f_index < (flat_file->count - 1)) f_index++;
-		    else {
-			f_index = 0;
-			n_filetypes++;
-		    }
-		    off = disp + flat_file->indices[f_index] + 
-                          n_filetypes*(ADIO_Offset)filetype_extent;
-		    frd_size = ADIOI_MIN(flat_file->blocklens[f_index], 
-		                         bufsize-(unsigned)userbuf_off);
-		}
-	    }
-	}
-	else {
-	    ADIO_Offset i_offset, tmp_bufsize = 0;
-	    /* noncontiguous in memory as well as in file */
-
-	    flat_buf = ADIOI_Flatten_and_find(buftype);
-
-	    b_index = buf_count = 0;
-	    i_offset = flat_buf->indices[0];
-	    f_index = st_index;
-	    off = start_off;
-	    n_filetypes = st_n_filetypes;
-	    frd_size = st_frd_size;
-	    brd_size = flat_buf->blocklens[0];
-
-	    /* while we haven't read size * count bytes, keep going */
-	    while (tmp_bufsize < bufsize) {
-    		ADIO_Offset new_brd_size = brd_size, new_frd_size = frd_size;
-
-		size = ADIOI_MIN(frd_size, brd_size);
-		if (size) {
-		    req_off = off;
-		    req_len = size;
-		    userbuf_off = i_offset;
-
-        ADIOI_Assert((((ADIO_Offset)(MPIU_Upint)buf) + userbuf_off) == (ADIO_Offset)(MPIU_Upint)((MPIU_Upint)buf + userbuf_off));
-        ADIOI_Assert(req_len == (int) req_len);
-		    ADIO_ReadContig(fd, 
-				    (char *) buf + userbuf_off,
-				    req_len, 
-				    MPI_BYTE, 
-				    ADIO_EXPLICIT_OFFSET,
-				    req_off,
-				    &status1,
-				    error_code);
-		    if (*error_code != MPI_SUCCESS) return;
-		}
-
-		if (size == frd_size) {
-		    /* reached end of contiguous block in file */
-		    if (f_index < (flat_file->count - 1)) f_index++;
-		    else {
-			f_index = 0;
-			n_filetypes++;
-		    }
-
-		    off = disp + flat_file->indices[f_index] + 
-                          n_filetypes*(ADIO_Offset)filetype_extent;
-
-		    new_frd_size = flat_file->blocklens[f_index];
-		    if (size != brd_size) {
-			i_offset += size;
-			new_brd_size -= size;
-		    }
-		}
-
-		if (size == brd_size) {
-		    /* reached end of contiguous block in memory */
-
-		    b_index = (b_index + 1)%flat_buf->count;
-		    buf_count++;
-		    i_offset = buftype_extent*(buf_count/flat_buf->count) +
-			flat_buf->indices[b_index];
-		    new_brd_size = flat_buf->blocklens[b_index];
-		    if (size != frd_size) {
-			off += size;
-			new_frd_size -= size;
-		    }
-		}
-		tmp_bufsize += size;
-		frd_size = new_frd_size;
-                brd_size = new_brd_size;
-	    }
-	}
-
-	/* unlock the file region if we locked it */
-        if ((fd->atomicity) && (fd->file_system != ADIO_PIOFS) && 
-	   (fd->file_system != ADIO_PVFS) && (fd->file_system != ADIO_PVFS2))
-	{
-            ADIOI_UNLOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
-	}
-
-	if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
-    } /* end of (else noncontiguous in file) */
-
-    fd->fp_sys_posn = -1;   /* mark it as invalid. */
-
-#ifdef HAVE_STATUS_SET_BYTES
-    MPIR_Status_set_bytes(status, buftype, bufsize);
-    /* This is a temporary way of filling in status. The right way is to 
-     * keep track of how much data was actually read and placed in buf 
-     */
-#endif
-
-    if (!buftype_is_contig) ADIOI_Delete_flattened(buftype);
-}
diff --git a/3rd-party/romio321/adio/common/ad_resize.c b/3rd-party/romio321/adio/common/ad_resize.c
deleted file mode 100644
index a6be2c17608..00000000000
--- a/3rd-party/romio321/adio/common/ad_resize.c
+++ /dev/null
@@ -1,38 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2004 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-
-void ADIOI_GEN_Resize(ADIO_File fd, ADIO_Offset size, int *error_code)
-{
-    int err, rank;
-    static char myname[] = "ADIOI_GEN_RESIZE";
-
-    MPI_Comm_rank(fd->comm, &rank);
-
-    /* first aggregator performs ftruncate() */
-    if (rank == fd->hints->ranklist[0]) {
-    ADIOI_Assert(size == (off_t) size); 
-	err = ftruncate(fd->fd_sys, (off_t)size);
-    }
-
-    /* bcast return value */
-    MPI_Bcast(&err, 1, MPI_INT, fd->hints->ranklist[0], fd->comm);
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (err == -1) {
-	*error_code = ADIOI_Err_create_code(myname, fd->filename, errno);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-    *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/common/ad_seek.c b/3rd-party/romio321/adio/common/ad_seek.c
deleted file mode 100644
index 2d6f75fd54e..00000000000
--- a/3rd-party/romio321/adio/common/ad_seek.c
+++ /dev/null
@@ -1,86 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-
-ADIO_Offset ADIOI_GEN_SeekIndividual(ADIO_File fd, ADIO_Offset offset, 
-				     int whence, int *error_code)
-{
-/* implemented for whence=SEEK_SET only. SEEK_CUR and SEEK_END must
-   be converted to the equivalent with SEEK_SET before calling this 
-   routine. */
-/* offset is in units of etype relative to the filetype */
-
-    ADIO_Offset off;
-    ADIOI_Flatlist_node *flat_file;
-
-    int i;
-    ADIO_Offset n_etypes_in_filetype, n_filetypes, etype_in_filetype;
-    ADIO_Offset abs_off_in_filetype=0;
-    ADIO_Offset size_in_filetype, sum;
-    MPI_Count filetype_size, etype_size;
-    int filetype_is_contig;
-    MPI_Aint filetype_extent, lb;
-
-    ADIOI_UNREFERENCED_ARG(whence);
-
-    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-    etype_size = fd->etype_size;
-
-    if (filetype_is_contig) off = fd->disp + etype_size * offset;
-    else {
-        flat_file = ADIOI_Flatlist;
-        while (flat_file->type != fd->filetype) flat_file = flat_file->next;
-
-	MPI_Type_get_extent(fd->filetype, &lb, &filetype_extent);
-	MPI_Type_size_x(fd->filetype, &filetype_size);
-	if ( ! filetype_size ) {
-	    /* Since offset relative to the filetype size, we can't
-	       do compute the offset when that result is zero.
-	       Return zero for the offset for now */
-	    *error_code = MPI_SUCCESS; 
-	    return 0;
-	}
-
-	n_etypes_in_filetype = filetype_size/etype_size;
-	n_filetypes = offset / n_etypes_in_filetype;
-	etype_in_filetype = offset % n_etypes_in_filetype;
-	size_in_filetype = etype_in_filetype * etype_size;
- 
-	sum = 0;
-	for (i=0; i<flat_file->count; i++) {
-	    sum += flat_file->blocklens[i];
-	    if (sum > size_in_filetype) {
-		abs_off_in_filetype = flat_file->indices[i] +
-		    size_in_filetype - (sum - flat_file->blocklens[i]);
-		break;
-	    }
-	}
-
-	/* abs. offset in bytes in the file */
-	off = fd->disp + n_filetypes * filetype_extent +
-                abs_off_in_filetype;
-    }
-
-/*
- * we used to call lseek here and update both fp_ind and fp_sys_posn, but now
- * we don't seek and only update fp_ind (ROMIO's idea of where we are in the
- * file).  We leave the system file descriptor and fp_sys_posn alone. 
- * The fs-specifc ReadContig and WriteContig will seek to the correct place in
- * the file before reading/writing if the 'offset' parameter doesn't match
- * fp_sys_posn
- */
-    fd->fp_ind = off;
-
-    *error_code = MPI_SUCCESS;
-
-    return off;
-}
diff --git a/3rd-party/romio321/adio/common/ad_set_sh_fp.c b/3rd-party/romio321/adio/common/ad_set_sh_fp.c
deleted file mode 100644
index c55f2efe5a9..00000000000
--- a/3rd-party/romio321/adio/common/ad_set_sh_fp.c
+++ /dev/null
@@ -1,45 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-
-/* set the shared file pointer to "offset" etypes relative to the current 
-   view */
-
-void ADIOI_NFS_Set_shared_fp(ADIO_File fd, ADIO_Offset offset, int *error_code);
-
-void ADIO_Set_shared_fp(ADIO_File fd, ADIO_Offset offset, int *error_code)
-{
-    ADIO_Status status;
-    MPI_Comm dupcommself;
-
-#ifdef ROMIO_NFS
-    if (fd->file_system == ADIO_NFS) {
-	ADIOI_NFS_Set_shared_fp(fd, offset, error_code);
-	return;
-    }
-#endif
-
-    if (fd->shared_fp_fd == ADIO_FILE_NULL) {
-	MPI_Comm_dup(MPI_COMM_SELF, &dupcommself);
-	fd->shared_fp_fd = ADIO_Open(MPI_COMM_SELF, dupcommself, 
-				     fd->shared_fp_fname, 
-				     fd->file_system,
-				     fd->fns,
-				     ADIO_CREATE | ADIO_RDWR | ADIO_DELETE_ON_CLOSE, 
-				     0, MPI_BYTE, MPI_BYTE,
-				     MPI_INFO_NULL, 
-				     ADIO_PERM_NULL, error_code);
-    }
-
-    if (*error_code != MPI_SUCCESS) return;
-
-    ADIOI_WRITE_LOCK(fd->shared_fp_fd, 0, SEEK_SET, sizeof(ADIO_Offset));
-    ADIO_WriteContig(fd->shared_fp_fd, &offset, sizeof(ADIO_Offset), 
-		     MPI_BYTE, ADIO_EXPLICIT_OFFSET, 0, &status, error_code);
-    ADIOI_UNLOCK(fd->shared_fp_fd, 0, SEEK_SET, sizeof(ADIO_Offset));
-}
-
diff --git a/3rd-party/romio321/adio/common/ad_set_view.c b/3rd-party/romio321/adio/common/ad_set_view.c
deleted file mode 100644
index 18becf269a9..00000000000
--- a/3rd-party/romio321/adio/common/ad_set_view.c
+++ /dev/null
@@ -1,77 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-
-/* this used to be implemented in every file system as an fcntl.  It makes
- * deferred open easier if we know ADIO_Fcntl will always need a file to really
- * be open. set_view doesn't modify anything related to the open files.
- */
-void ADIO_Set_view(ADIO_File fd, ADIO_Offset disp, MPI_Datatype etype, 
-		MPI_Datatype filetype, MPI_Info info,  int *error_code) 
-{
-	int combiner, i, j, k, err, filetype_is_contig;
-	MPI_Datatype copy_etype, copy_filetype;
-	ADIOI_Flatlist_node *flat_file;
-	/* free copies of old etypes and filetypes and delete flattened 
-       version of filetype if necessary */
-
-	MPI_Type_get_envelope(fd->etype, &i, &j, &k, &combiner);
-	if (combiner != MPI_COMBINER_NAMED) MPI_Type_free(&(fd->etype));
-
-	ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-	if (!filetype_is_contig) ADIOI_Delete_flattened(fd->filetype);
-
-	MPI_Type_get_envelope(fd->filetype, &i, &j, &k, &combiner);
-	if (combiner != MPI_COMBINER_NAMED) MPI_Type_free(&(fd->filetype));
-
-	/* set new info */
-	ADIO_SetInfo(fd, info, &err);
-
-        /* set new etypes and filetypes */
-
-	ADIOI_Type_get_envelope(etype, &i, &j, &k, &combiner);
-	if (combiner == MPI_COMBINER_NAMED) fd->etype = etype;
-	else {
-	    MPI_Type_contiguous(1, etype, &copy_etype);
-	    MPI_Type_commit(&copy_etype);
-	    fd->etype = copy_etype;
-	}
-	ADIOI_Type_get_envelope(filetype, &i, &j, &k, &combiner);
-	if (combiner == MPI_COMBINER_NAMED) 
-	    fd->filetype = filetype;
-	else {
-	    MPI_Type_contiguous(1, filetype, &copy_filetype);
-	    MPI_Type_commit(&copy_filetype);
-	    fd->filetype = copy_filetype;
-	    ADIOI_Flatten_datatype(fd->filetype);
-            /* this function will not flatten the filetype if it turns out
-               to be all contiguous. */
-	}
-
-	MPI_Type_size_x(fd->etype, &(fd->etype_size));
-	fd->disp = disp;
-
-        /* reset MPI-IO file pointer to point to the first byte that can
-           be accessed in this view. */
-
-        ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-	if (filetype_is_contig) fd->fp_ind = disp;
-	else {
-	    flat_file = ADIOI_Flatlist;
-	    while (flat_file->type != fd->filetype) 
-		flat_file = flat_file->next;
-	    for (i=0; i<flat_file->count; i++) {
-		if (flat_file->blocklens[i]) {
-		    fd->fp_ind = disp + flat_file->indices[i];
-		    break;
-		}
-	    }
-	}
-	*error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/common/ad_subarray.c b/3rd-party/romio321/adio/common/ad_subarray.c
deleted file mode 100644
index c6eb9450ce6..00000000000
--- a/3rd-party/romio321/adio/common/ad_subarray.c
+++ /dev/null
@@ -1,96 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-
-int ADIO_Type_create_subarray(int ndims,
-			      int *array_of_sizes, 
-			      int *array_of_subsizes,
-			      int *array_of_starts,
-			      int order,
-			      MPI_Datatype oldtype, 
-			      MPI_Datatype *newtype)
-{
-    MPI_Aint extent, disp, size, lb, ub;
-    int i, blklen;
-    MPI_Datatype tmp1, tmp2, inttype;
-
-    MPI_Type_get_extent(oldtype, &lb, &extent);
-
-    if (order == MPI_ORDER_FORTRAN) {
-	/* dimension 0 changes fastest */
-	if (ndims == 1) {
-	    MPI_Type_contiguous(array_of_subsizes[0], oldtype, &tmp1);
-	}
-	else {
-	    MPI_Type_vector(array_of_subsizes[1],
-			    array_of_subsizes[0],
-			    array_of_sizes[0], oldtype, &tmp1);
-	    
-	    size = (MPI_Aint)array_of_sizes[0]*extent;
-	    for (i=2; i<ndims; i++) {
-		size *= (MPI_Aint)array_of_sizes[i-1];
-		MPI_Type_create_hvector(array_of_subsizes[i], 1, size, tmp1, &tmp2);
-		MPI_Type_free(&tmp1);
-		tmp1 = tmp2;
-	    }
-	}
-	
-	/* add displacement and UB */
-	disp = array_of_starts[0];
-	size = 1;
-	for (i=1; i<ndims; i++) {
-	    size *= (MPI_Aint)array_of_sizes[i-1];
-	    disp += size*(MPI_Aint)array_of_starts[i];
-	}  
-        /* rest done below for both Fortran and C order */
-    }
-
-    else /* order == MPI_ORDER_C */ {
-	/* dimension ndims-1 changes fastest */
-	if (ndims == 1) {
-	    MPI_Type_contiguous(array_of_subsizes[0], oldtype, &tmp1);
-	}
-	else {
-	    MPI_Type_vector(array_of_subsizes[ndims-2],
-			    array_of_subsizes[ndims-1],
-			    array_of_sizes[ndims-1], oldtype, &tmp1);
-	    
-	    size = (MPI_Aint)array_of_sizes[ndims-1]*extent;
-	    for (i=ndims-3; i>=0; i--) {
-		size *= (MPI_Aint)array_of_sizes[i+1];
-		MPI_Type_create_hvector(array_of_subsizes[i], 1, size, tmp1, &tmp2);
-		MPI_Type_free(&tmp1);
-		tmp1 = tmp2;
-	    }
-	}
-	
-	/* add displacement and UB */
-	disp = array_of_starts[ndims-1];
-	size = 1;
-	for (i=ndims-2; i>=0; i--) {
-	    size *= (MPI_Aint)array_of_sizes[i+1];
-	    disp += size*(MPI_Aint)array_of_starts[i];
-	}
-    }
-    
-    disp *= extent;
-    
-    ub = extent;
-    for (i=0; i<ndims; i++) ub *= (MPI_Aint)array_of_sizes[i];
-    
-    blklen = 1;
- 
-    MPI_Type_create_struct(1, &blklen, &disp, &tmp1, &inttype);
-    MPI_Type_create_resized (inttype, 0, ub, newtype);
-    MPI_Type_free(&inttype);
-
-    MPI_Type_free(&tmp1);
-
-    return MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/common/ad_threaded_io.c b/3rd-party/romio321/adio/common/ad_threaded_io.c
deleted file mode 100644
index 2436a7a519a..00000000000
--- a/3rd-party/romio321/adio/common/ad_threaded_io.c
+++ /dev/null
@@ -1,32 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/*
- *   Copyright (C) 1997-2001 University of Chicago.
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-
-#include <pthread.h>
-/* Function for running in another thread for doing the file reading while the
- * main thread is doing data aggregation - useful only when multiple rounds are
- * needed due to file size relative to the read buffer size and number of
- * aggregators */
-
-void *ADIOI_IO_Thread_Func(void *vptr_args) {
-    ADIOI_IO_ThreadFuncData *args = (ADIOI_IO_ThreadFuncData*)vptr_args;
-
-    ADIOI_Assert(args->size == (int)(args->size));
-
-    if (args->io_kind == ADIOI_READ) {
-	ADIO_ReadContig(args->fd, args->buf, args->size, MPI_BYTE,
-		ADIO_EXPLICIT_OFFSET, args->offset,
-		args->status, &(args->error_code));
-    } else {
-	ADIO_WriteContig(args->fd, args->buf, args->size, MPI_BYTE,
-		ADIO_EXPLICIT_OFFSET, args->offset,
-		args->status, &(args->error_code));
-    }
-    pthread_exit(&(args->error_code));
-    return NULL;
-}
diff --git a/3rd-party/romio321/adio/common/ad_wait.c b/3rd-party/romio321/adio/common/ad_wait.c
deleted file mode 100644
index 2112ee8fb66..00000000000
--- a/3rd-party/romio321/adio/common/ad_wait.c
+++ /dev/null
@@ -1,47 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2004 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_SIGNAL_H
-#include <signal.h>
-#endif
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-#ifdef HAVE_AIO_H
-#include <aio.h>
-#endif
-#ifdef HAVE_SYS_AIO_H
-#include <sys/aio.h>
-#endif
-
-/* Workaround for incomplete set of definitions if __REDIRECT is not 
-   defined and large file support is used in aio.h */
-#if !defined(__REDIRECT) && defined(__USE_FILE_OFFSET64)
-#define aiocb aiocb64
-#endif
-
-/* ADIOI_GEN_IOComplete
- *
- * This code handles two distinct cases.  If ROMIO_HAVE_WORKING_AIO is
- * not defined, then I/O was performed by a blocking call already.  In
- * that case all we need to do is optionally set the bytes in the
- * status structure and free the request.
- *
- * If ROMIO_HAVE_WORKING_AIO is defined, then we may need to wait for I/O
- * to complete.
- */
-void ADIOI_GEN_IOComplete(ADIO_Request *request, ADIO_Status *status,
-			  int *error_code)  
-{
-	return;
- 
-}
diff --git a/3rd-party/romio321/adio/common/ad_wait_fake.c b/3rd-party/romio321/adio/common/ad_wait_fake.c
deleted file mode 100644
index 16947baa1fb..00000000000
--- a/3rd-party/romio321/adio/common/ad_wait_fake.c
+++ /dev/null
@@ -1,20 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2004 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-
-/* Generic implementation of ReadComplete/WriteComplete simply sets the
- * bytes field in the status structure and frees the request.
- *
- * Same function is used for both reads and writes.
- */
-void ADIOI_FAKE_IOComplete(ADIO_Request *request, ADIO_Status *status,
-			  int *error_code)
-{
-        *error_code = MPI_SUCCESS;
-	return;
-}
diff --git a/3rd-party/romio321/adio/common/ad_write.c b/3rd-party/romio321/adio/common/ad_write.c
deleted file mode 100644
index d1c24ac45ed..00000000000
--- a/3rd-party/romio321/adio/common/ad_write.c
+++ /dev/null
@@ -1,114 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2004 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-
-#include "adio.h"
-#include <unistd.h>
-
-#ifdef AGGREGATION_PROFILE
-#include "mpe.h"
-#endif
-
-#ifdef ROMIO_GPFS
-#include "adio/ad_gpfs/ad_gpfs_tuning.h"
-#endif
-
-#ifdef HAVE_LIMITS_H
-#include <limits.h>
-#endif
-
-
-void ADIOI_GEN_WriteContig(ADIO_File fd, const void *buf, int count,
-			   MPI_Datatype datatype, int file_ptr_type,
-			   ADIO_Offset offset, ADIO_Status *status,
-			   int *error_code)
-{
-    ssize_t err = -1;
-    MPI_Count datatype_size;
-    ADIO_Offset len, bytes_xfered=0;
-    size_t wr_count;
-    static char myname[] = "ADIOI_GEN_WRITECONTIG";
-#ifdef ROMIO_GPFS
-    double io_time=0;
-#endif
-    char * p;
-
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5036, 0, NULL);
-#endif
-
-    MPI_Type_size_x(datatype, &datatype_size);
-    len = (ADIO_Offset)datatype_size * (ADIO_Offset)count;
-
-#ifdef ROMIO_GPFS
-    io_time = MPI_Wtime();
-    if (gpfsmpio_timing) {
-	gpfsmpio_prof_cw[ GPFSMPIO_CIO_DATA_SIZE ] += len;
-    }
-#endif
-
-    if (file_ptr_type == ADIO_INDIVIDUAL) {
-	offset = fd->fp_ind;
-    }
-
-    p = (char *)buf;
-    while (bytes_xfered < len) {
-#ifdef ADIOI_MPE_LOGGING
-	MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
-#endif
-	wr_count = len - bytes_xfered;
-	/* Frustrating! FreeBSD and OS X do not like a count larger than 2^31 */
-        if (wr_count > INT_MAX)
-            wr_count = INT_MAX;
-
-#ifdef ROMIO_GPFS
-	if (gpfsmpio_devnullio)
-	    err = pwrite(fd->null_fd, p, wr_count, offset+bytes_xfered);
-	else
-#endif
-	    err = pwrite(fd->fd_sys, p, wr_count, offset+bytes_xfered);
-	/* --BEGIN ERROR HANDLING-- */
-	if (err == -1) {
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-		    MPIR_ERR_RECOVERABLE,
-		    myname, __LINE__,
-		    MPI_ERR_IO, "**io",
-		    "**io %s", strerror(errno));
-	    fd->fp_sys_posn = -1;
-	    return;
-	}
-    /* --END ERROR HANDLING-- */
-#ifdef ADIOI_MPE_LOGGING
-	MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
-#endif
-	bytes_xfered += err;
-	p += err;
-    }
-
-#ifdef ROMIO_GPFS
-    if (gpfsmpio_timing) gpfsmpio_prof_cw[ GPFSMPIO_CIO_T_POSI_RW ] += (MPI_Wtime() - io_time);
-#endif
-    fd->fp_sys_posn = offset + bytes_xfered;
-
-    if (file_ptr_type == ADIO_INDIVIDUAL) {
-	fd->fp_ind += bytes_xfered; 
-    }
-
-#ifdef ROMIO_GPFS
-    if (gpfsmpio_timing) gpfsmpio_prof_cw[ GPFSMPIO_CIO_T_MPIO_RW ] += (MPI_Wtime() - io_time);
-#endif
-
-#ifdef HAVE_STATUS_SET_BYTES
-    /* bytes_xfered could be larger than int */
-    if (err != -1 && status) MPIR_Status_set_bytes(status, datatype, bytes_xfered);
-#endif
-
-    *error_code = MPI_SUCCESS;
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5037, 0, NULL);
-#endif
-}
diff --git a/3rd-party/romio321/adio/common/ad_write_coll.c b/3rd-party/romio321/adio/common/ad_write_coll.c
deleted file mode 100644
index 6a2e09771bf..00000000000
--- a/3rd-party/romio321/adio/common/ad_write_coll.c
+++ /dev/null
@@ -1,1086 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-
-#ifdef AGGREGATION_PROFILE
-#include "mpe.h"
-#endif
-
-/* prototypes of functions used for collective writes only. */
-static void ADIOI_Exch_and_write(ADIO_File fd, void *buf, MPI_Datatype
-                         datatype, int nprocs, int myrank,
-			 ADIOI_Access
-                         *others_req, ADIO_Offset *offset_list,
-                         ADIO_Offset *len_list, int contig_access_count, ADIO_Offset
-                         min_st_offset, ADIO_Offset fd_size,
-                         ADIO_Offset *fd_start, ADIO_Offset *fd_end,
-                         int *buf_idx, int *error_code);
-static void ADIOI_W_Exchange_data(ADIO_File fd, void *buf, char *write_buf,
-                         ADIOI_Flatlist_node *flat_buf, ADIO_Offset 
-                         *offset_list, ADIO_Offset *len_list, int *send_size, 
-                         int *recv_size, ADIO_Offset off, int size,
-                         int *count, int *start_pos, int *partial_recv, 
-                         int *sent_to_proc, int nprocs, 
-                         int myrank, int
-                         buftype_is_contig, int contig_access_count,
-                         ADIO_Offset min_st_offset, ADIO_Offset fd_size,
-                         ADIO_Offset *fd_start, ADIO_Offset *fd_end, 
-                         ADIOI_Access *others_req, 
-                         int *send_buf_idx, int *curr_to_proc,
-                         int *done_to_proc, int *hole, int iter, 
-                         MPI_Aint buftype_extent, int *buf_idx, int *error_code);
-void ADIOI_Fill_send_buffer(ADIO_File fd, void *buf, ADIOI_Flatlist_node
-                           *flat_buf, char **send_buf, ADIO_Offset 
-                           *offset_list, ADIO_Offset *len_list, int *send_size, 
-                           MPI_Request *requests, int *sent_to_proc, 
-                           int nprocs, int myrank, 
-                           int contig_access_count, ADIO_Offset
-                           min_st_offset, ADIO_Offset fd_size,
-                           ADIO_Offset *fd_start, ADIO_Offset *fd_end, 
-                           int *send_buf_idx, int *curr_to_proc, 
-                           int *done_to_proc, int iter, 
-                           MPI_Aint buftype_extent);
-void ADIOI_Heap_merge(ADIOI_Access *others_req, int *count, 
-                      ADIO_Offset *srt_off, int *srt_len, int *start_pos,
-                      int nprocs, int nprocs_recv, int total_elements);
-
-
-void ADIOI_GEN_WriteStridedColl(ADIO_File fd, const void *buf, int count,
-                       MPI_Datatype datatype, int file_ptr_type,
-                       ADIO_Offset offset, ADIO_Status *status, int
-                       *error_code)
-{
-/* Uses a generalized version of the extended two-phase method described
-   in "An Extended Two-Phase Method for Accessing Sections of 
-   Out-of-Core Arrays", Rajeev Thakur and Alok Choudhary,
-   Scientific Programming, (5)4:301--317, Winter 1996. 
-   http://www.mcs.anl.gov/home/thakur/ext2ph.ps */
-
-    ADIOI_Access *my_req; 
-    /* array of nprocs access structures, one for each other process in
-       whose file domain this process's request lies */
-    
-    ADIOI_Access *others_req;
-    /* array of nprocs access structures, one for each other process
-       whose request lies in this process's file domain. */
-
-    int i, filetype_is_contig, nprocs, nprocs_for_coll, myrank;
-    int contig_access_count=0, interleave_count = 0, buftype_is_contig;
-    int *count_my_req_per_proc, count_my_req_procs, count_others_req_procs;
-    ADIO_Offset orig_fp, start_offset, end_offset, fd_size, min_st_offset, off;
-    ADIO_Offset *offset_list = NULL, *st_offsets = NULL, *fd_start = NULL,
-	*fd_end = NULL, *end_offsets = NULL;
-    int *buf_idx = NULL;
-    ADIO_Offset *len_list = NULL;
-    int old_error, tmp_error;
-
-    if (fd->hints->cb_pfr != ADIOI_HINT_DISABLE) { 
-        /* Cast away const'ness as the below function is used for read
-         * and write */
-	ADIOI_IOStridedColl (fd, (char *) buf, count, ADIOI_WRITE, datatype,
-			file_ptr_type, offset, status, error_code);
-	return;
-    }
-
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-
-/* the number of processes that actually perform I/O, nprocs_for_coll,
- * is stored in the hints off the ADIO_File structure
- */
-    nprocs_for_coll = fd->hints->cb_nodes;
-    orig_fp = fd->fp_ind;
-
-    /* only check for interleaving if cb_write isn't disabled */
-    if (fd->hints->cb_write != ADIOI_HINT_DISABLE) {
-	/* For this process's request, calculate the list of offsets and
-	   lengths in the file and determine the start and end offsets. */
-
-	/* Note: end_offset points to the last byte-offset that will be accessed.
-	   e.g., if start_offset=0 and 100 bytes to be read, end_offset=99*/
-
-	ADIOI_Calc_my_off_len(fd, count, datatype, file_ptr_type, offset,
-			      &offset_list, &len_list, &start_offset,
-			      &end_offset, &contig_access_count); 
-
-	/* each process communicates its start and end offsets to other 
-	   processes. The result is an array each of start and end offsets stored
-	   in order of process rank. */ 
-    
-	st_offsets = (ADIO_Offset *) ADIOI_Malloc(nprocs*sizeof(ADIO_Offset));
-	end_offsets = (ADIO_Offset *) ADIOI_Malloc(nprocs*sizeof(ADIO_Offset));
-
-	MPI_Allgather(&start_offset, 1, ADIO_OFFSET, st_offsets, 1,
-		      ADIO_OFFSET, fd->comm);
-	MPI_Allgather(&end_offset, 1, ADIO_OFFSET, end_offsets, 1,
-		      ADIO_OFFSET, fd->comm);
-
-	/* are the accesses of different processes interleaved? */
-	for (i=1; i<nprocs; i++)
-	    if ((st_offsets[i] < end_offsets[i-1]) && 
-                (st_offsets[i] <= end_offsets[i]))
-                interleave_count++;
-	/* This is a rudimentary check for interleaving, but should suffice
-	   for the moment. */
-    }
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-
-    if (fd->hints->cb_write == ADIOI_HINT_DISABLE ||
-	(!interleave_count && (fd->hints->cb_write == ADIOI_HINT_AUTO)))
-    {
-	/* use independent accesses */
-	if (fd->hints->cb_write != ADIOI_HINT_DISABLE) {
-	    ADIOI_Free(offset_list);
-	    ADIOI_Free(len_list);
-	    ADIOI_Free(st_offsets);
-	    ADIOI_Free(end_offsets);
-	}
-
-	fd->fp_ind = orig_fp;
-        ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-
-        if (buftype_is_contig && filetype_is_contig) {
-            if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-                off = fd->disp + (ADIO_Offset)(fd->etype_size) * offset;
-                ADIO_WriteContig(fd, buf, count, datatype,
-				 ADIO_EXPLICIT_OFFSET,
-				 off, status, error_code);
-            }
-            else ADIO_WriteContig(fd, buf, count, datatype, ADIO_INDIVIDUAL,
-				  0, status, error_code);
-        }
-	else ADIO_WriteStrided(fd, buf, count, datatype, file_ptr_type,
-			       offset, status, error_code);
-
-	return;
-    }
-
-/* Divide the I/O workload among "nprocs_for_coll" processes. This is
-   done by (logically) dividing the file into file domains (FDs); each
-   process may directly access only its own file domain. */
-
-    ADIOI_Calc_file_domains(st_offsets, end_offsets, nprocs,
-			    nprocs_for_coll, &min_st_offset,
-			    &fd_start, &fd_end, 
-			    fd->hints->min_fdomain_size, &fd_size,
-			    fd->hints->striping_unit);   
-
-
-/* calculate what portions of the access requests of this process are
-   located in what file domains */
-
-    ADIOI_Calc_my_req(fd, offset_list, len_list, contig_access_count,
-		      min_st_offset, fd_start, fd_end, fd_size,
-		      nprocs, &count_my_req_procs, 
-		      &count_my_req_per_proc, &my_req,
-		      &buf_idx); 
-
-/* based on everyone's my_req, calculate what requests of other
-   processes lie in this process's file domain.
-   count_others_req_procs = number of processes whose requests lie in
-   this process's file domain (including this process itself) 
-   count_others_req_per_proc[i] indicates how many separate contiguous
-   requests of proc. i lie in this process's file domain. */
-
-    ADIOI_Calc_others_req(fd, count_my_req_procs, 
-			  count_my_req_per_proc, my_req, 
-			  nprocs, myrank,
-			  &count_others_req_procs, &others_req); 
-    
-    ADIOI_Free(count_my_req_per_proc);
-    for (i=0; i < nprocs; i++) {
-	if (my_req[i].count) {
-	    ADIOI_Free(my_req[i].offsets);
-	    ADIOI_Free(my_req[i].lens);
-	}
-    }
-    ADIOI_Free(my_req);
-
-/* exchange data and write in sizes of no more than coll_bufsize. */
-    /* Cast away const'ness for the below function */
-    ADIOI_Exch_and_write(fd, (char *) buf, datatype, nprocs, myrank,
-                        others_req, offset_list,
-			len_list, contig_access_count, min_st_offset,
-			fd_size, fd_start, fd_end, buf_idx, error_code);
-
-    /* If this collective write is followed by an independent write,
-     * it's possible to have those subsequent writes on other processes
-     * race ahead and sneak in before the read-modify-write completes.
-     * We carry out a collective communication at the end here so no one
-     * can start independent i/o before collective I/O completes. 
-     *
-     * need to do some gymnastics with the error codes so that if something
-     * went wrong, all processes report error, but if a process has a more
-     * specific error code, we can still have that process report the
-     * additional information */
-
-    old_error = *error_code;
-    if (*error_code != MPI_SUCCESS) *error_code = MPI_ERR_IO;
-
-     /* optimization: if only one process performing i/o, we can perform
-     * a less-expensive Bcast  */
-#ifdef ADIOI_MPE_LOGGING
-    MPE_Log_event( ADIOI_MPE_postwrite_a, 0, NULL );
-#endif
-    if (fd->hints->cb_nodes == 1) 
-	    MPI_Bcast(error_code, 1, MPI_INT, 
-			    fd->hints->ranklist[0], fd->comm);
-    else {
-	    tmp_error = *error_code;
-	    MPI_Allreduce(&tmp_error, error_code, 1, MPI_INT, 
-			    MPI_MAX, fd->comm);
-    }
-#ifdef ADIOI_MPE_LOGGING
-    MPE_Log_event( ADIOI_MPE_postwrite_b, 0, NULL );
-#endif
-#ifdef AGGREGATION_PROFILE
-	MPE_Log_event (5012, 0, NULL);
-#endif
-
-    if ( (old_error != MPI_SUCCESS) && (old_error != MPI_ERR_IO) )
-	    *error_code = old_error;
-
-
-    if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
-
-/* free all memory allocated for collective I/O */
-
-    for (i=0; i<nprocs; i++) {
-	if (others_req[i].count) {
-	    ADIOI_Free(others_req[i].offsets);
-	    ADIOI_Free(others_req[i].lens);
-	    ADIOI_Free(others_req[i].mem_ptrs);
-	}
-    }
-    ADIOI_Free(others_req);
-
-    ADIOI_Free(buf_idx);
-    ADIOI_Free(offset_list);
-    ADIOI_Free(len_list);
-    ADIOI_Free(st_offsets);
-    ADIOI_Free(end_offsets);
-    ADIOI_Free(fd_start);
-    ADIOI_Free(fd_end);
-
-#ifdef HAVE_STATUS_SET_BYTES
-    if (status) {
-      MPI_Count bufsize, size;
-      /* Don't set status if it isn't needed */
-      MPI_Type_size_x(datatype, &size);
-      bufsize = size * count;
-      MPIR_Status_set_bytes(status, datatype, bufsize);
-    }
-/* This is a temporary way of filling in status. The right way is to 
-   keep track of how much data was actually written during collective I/O. */
-#endif
-
-    fd->fp_sys_posn = -1;   /* set it to null. */
-#ifdef AGGREGATION_PROFILE
-	MPE_Log_event (5013, 0, NULL);
-#endif
-}
-
-
-
-/* If successful, error_code is set to MPI_SUCCESS.  Otherwise an error
- * code is created and returned in error_code.
- */
-static void ADIOI_Exch_and_write(ADIO_File fd, void *buf, MPI_Datatype
-				 datatype, int nprocs, 
-				 int myrank,
-				 ADIOI_Access
-				 *others_req, ADIO_Offset *offset_list,
-				 ADIO_Offset *len_list, int contig_access_count,
-				 ADIO_Offset min_st_offset, ADIO_Offset fd_size,
-				 ADIO_Offset *fd_start, ADIO_Offset *fd_end,
-				 int *buf_idx, int *error_code)
-{
-/* Send data to appropriate processes and write in sizes of no more
-   than coll_bufsize.    
-   The idea is to reduce the amount of extra memory required for
-   collective I/O. If all data were written all at once, which is much
-   easier, it would require temp space more than the size of user_buf,
-   which is often unacceptable. For example, to write a distributed
-   array to a file, where each local array is 8Mbytes, requiring
-   at least another 8Mbytes of temp space is unacceptable. */
-
-    /* Not convinced end_loc-st_loc couldn't be > int, so make these offsets*/
-    ADIO_Offset size=0;
-    int hole, i, j, m, ntimes, max_ntimes, buftype_is_contig;
-    ADIO_Offset st_loc=-1, end_loc=-1, off, done, req_off;
-    char *write_buf=NULL;
-    int *curr_offlen_ptr, *count, *send_size, req_len, *recv_size;
-    int *partial_recv, *sent_to_proc, *start_pos, flag;
-    int *send_buf_idx, *curr_to_proc, *done_to_proc;
-    MPI_Status status;
-    ADIOI_Flatlist_node *flat_buf=NULL;
-    MPI_Aint buftype_extent, lb;
-    int info_flag, coll_bufsize;
-    char *value;
-    static char myname[] = "ADIOI_EXCH_AND_WRITE";
-
-    *error_code = MPI_SUCCESS;  /* changed below if error */
-    /* only I/O errors are currently reported */
-
-/* calculate the number of writes of size coll_bufsize
-   to be done by each process and the max among all processes.
-   That gives the no. of communication phases as well. */
-
-    value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
-    ADIOI_Info_get(fd->info, "cb_buffer_size", MPI_MAX_INFO_VAL, value, 
-                 &info_flag);
-    coll_bufsize = atoi(value);
-    ADIOI_Free(value);
-
-
-    for (i=0; i < nprocs; i++) {
-	if (others_req[i].count) {
-	    st_loc = others_req[i].offsets[0];
-	    end_loc = others_req[i].offsets[0];
-	    break;
-	}
-    }
-
-    for (i=0; i < nprocs; i++)
-	for (j=0; j < others_req[i].count; j++) {
-	    st_loc = ADIOI_MIN(st_loc, others_req[i].offsets[j]);
-	    end_loc = ADIOI_MAX(end_loc, (others_req[i].offsets[j]
-				       + others_req[i].lens[j] - 1));
-	}
-
-/* ntimes=ceiling_div(end_loc - st_loc + 1, coll_bufsize)*/
-
-    ntimes = (int) ((end_loc - st_loc + coll_bufsize)/coll_bufsize);
-
-    if ((st_loc==-1) && (end_loc==-1)) {
-	ntimes = 0; /* this process does no writing. */
-    }
-
-    MPI_Allreduce(&ntimes, &max_ntimes, 1, MPI_INT, MPI_MAX,
-		  fd->comm); 
-
-    write_buf = fd->io_buf;
-
-    curr_offlen_ptr = (int *) ADIOI_Calloc(nprocs, sizeof(int)); 
-    /* its use is explained below. calloc initializes to 0. */
-
-    count = (int *) ADIOI_Malloc(nprocs*sizeof(int));
-    /* to store count of how many off-len pairs per proc are satisfied
-       in an iteration. */
-
-    partial_recv = (int *) ADIOI_Calloc(nprocs, sizeof(int));
-    /* if only a portion of the last off-len pair is recd. from a process
-       in a particular iteration, the length recd. is stored here.
-       calloc initializes to 0. */
-
-    send_size = (int *) ADIOI_Malloc(nprocs*sizeof(int));
-    /* total size of data to be sent to each proc. in an iteration.
-       Of size nprocs so that I can use MPI_Alltoall later. */
-
-    recv_size = (int *) ADIOI_Malloc(nprocs*sizeof(int));
-    /* total size of data to be recd. from each proc. in an iteration.*/
-
-    sent_to_proc = (int *) ADIOI_Calloc(nprocs, sizeof(int));
-    /* amount of data sent to each proc so far. Used in
-       ADIOI_Fill_send_buffer. initialized to 0 here. */
-
-    send_buf_idx = (int *) ADIOI_Malloc(nprocs*sizeof(int));
-    curr_to_proc = (int *) ADIOI_Malloc(nprocs*sizeof(int));
-    done_to_proc = (int *) ADIOI_Malloc(nprocs*sizeof(int));
-    /* Above three are used in ADIOI_Fill_send_buffer*/
-
-    start_pos = (int *) ADIOI_Malloc(nprocs*sizeof(int));
-    /* used to store the starting value of curr_offlen_ptr[i] in 
-       this iteration */
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-    if (!buftype_is_contig) {
-	flat_buf = ADIOI_Flatten_and_find(datatype);
-    }
-    MPI_Type_get_extent(datatype, &lb, &buftype_extent);
-
-
-/* I need to check if there are any outstanding nonblocking writes to
-   the file, which could potentially interfere with the writes taking
-   place in this collective write call. Since this is not likely to be
-   common, let me do the simplest thing possible here: Each process
-   completes all pending nonblocking operations before completing. */
-
-    /*ADIOI_Complete_async(error_code);
-    if (*error_code != MPI_SUCCESS) return;
-    MPI_Barrier(fd->comm);
-    */
-
-    done = 0;
-    off = st_loc;
-
-    for (m=0; m < ntimes; m++) {
-       /* go through all others_req and check which will be satisfied
-          by the current write */
-
-       /* Note that MPI guarantees that displacements in filetypes are in 
-          monotonically nondecreasing order and that, for writes, the
-	  filetypes cannot specify overlapping regions in the file. This
-	  simplifies implementation a bit compared to reads. */
-
-          /* off = start offset in the file for the data to be written in 
-                   this iteration 
-             size = size of data written (bytes) corresponding to off
-             req_off = off in file for a particular contiguous request 
-                       minus what was satisfied in previous iteration
-             req_size = size corresponding to req_off */
-
-	/* first calculate what should be communicated */
-
-	for (i=0; i < nprocs; i++) count[i] = recv_size[i] = 0;
-
-	size = ADIOI_MIN((unsigned)coll_bufsize, end_loc-st_loc+1-done); 
-
-	for (i=0; i < nprocs; i++) {
-	    if (others_req[i].count) {
-		start_pos[i] = curr_offlen_ptr[i];
-		for (j=curr_offlen_ptr[i]; j<others_req[i].count; j++) {
-		    if (partial_recv[i]) {
-			/* this request may have been partially
-			   satisfied in the previous iteration. */
-			req_off = others_req[i].offsets[j] +
-			    partial_recv[i]; 
-                        req_len = others_req[i].lens[j] -
-			    partial_recv[i];
-			partial_recv[i] = 0;
-			/* modify the off-len pair to reflect this change */
-			others_req[i].offsets[j] = req_off;
-			others_req[i].lens[j] = req_len;
-		    }
-		    else {
-			req_off = others_req[i].offsets[j];
-                        req_len = others_req[i].lens[j];
-		    }
-		    if (req_off < off + size) {
-			count[i]++;
-      ADIOI_Assert((((ADIO_Offset)(MPIU_Upint)write_buf)+req_off-off) == (ADIO_Offset)(MPIU_Upint)(write_buf+req_off-off));
-			MPI_Get_address(write_buf+req_off-off, 
-                                        &(others_req[i].mem_ptrs[j]));
-      ADIOI_Assert((off + size - req_off) == (int)(off + size - req_off));
-			recv_size[i] += (int)(ADIOI_MIN(off + size - req_off, 
-                                      (unsigned)req_len));
-
-			if (off+size-req_off < (unsigned)req_len)
-			{
-			    partial_recv[i] = (int) (off + size - req_off);
-
-			    /* --BEGIN ERROR HANDLING-- */
-			    if ((j+1 < others_req[i].count) && 
-                                 (others_req[i].offsets[j+1] < off+size))
-			    { 
-				*error_code = MPIO_Err_create_code(MPI_SUCCESS,
-								   MPIR_ERR_RECOVERABLE,
-								   myname,
-								   __LINE__,
-								   MPI_ERR_ARG,
-								   "Filetype specifies overlapping write regions (which is illegal according to the MPI-2 specification)", 0);
-				/* allow to continue since additional
-				 * communication might have to occur
-				 */
-			    }
-			    /* --END ERROR HANDLING-- */
-			    break;
-			}
-		    }
-		    else break;
-		}
-		curr_offlen_ptr[i] = j;
-	    }
-	}
-	
-	ADIOI_W_Exchange_data(fd, buf, write_buf, flat_buf, offset_list, 
-                            len_list, send_size, recv_size, off, size, count, 
-                            start_pos, partial_recv, 
-                            sent_to_proc, nprocs, myrank, 
-			    buftype_is_contig, contig_access_count,
-			    min_st_offset, fd_size, fd_start, fd_end,
-			    others_req, send_buf_idx, curr_to_proc,
-                            done_to_proc, &hole, m, buftype_extent, buf_idx,
-			    error_code); 
-        if (*error_code != MPI_SUCCESS) return;
-
-	flag = 0;
-	for (i=0; i<nprocs; i++)
-	    if (count[i]) flag = 1;
-
-	if (flag) {
-      ADIOI_Assert(size == (int)size);
-	    ADIO_WriteContig(fd, write_buf, (int)size, MPI_BYTE, ADIO_EXPLICIT_OFFSET, 
-                        off, &status, error_code);
-	    if (*error_code != MPI_SUCCESS) return;
-	}
-
-	off += size;
-	done += size;
-    }
-
-    for (i=0; i<nprocs; i++) count[i] = recv_size[i] = 0;
-    for (m=ntimes; m<max_ntimes; m++) {
-	ADIOI_Assert(size == (int)size);
-	/* nothing to recv, but check for send. */
-	ADIOI_W_Exchange_data(fd, buf, write_buf, flat_buf, offset_list, 
-                            len_list, send_size, recv_size, off, (int)size, count,
-                            start_pos, partial_recv, 
-                            sent_to_proc, nprocs, myrank, 
-			    buftype_is_contig, contig_access_count,
-			    min_st_offset, fd_size, fd_start, fd_end,
-			    others_req, send_buf_idx, 
-                            curr_to_proc, done_to_proc, &hole, m, 
-                            buftype_extent, buf_idx, error_code); 
-        if (*error_code != MPI_SUCCESS) return;
-    }
-
-    ADIOI_Free(curr_offlen_ptr);
-    ADIOI_Free(count);
-    ADIOI_Free(partial_recv);
-    ADIOI_Free(send_size);
-    ADIOI_Free(recv_size);
-    ADIOI_Free(sent_to_proc);
-    ADIOI_Free(start_pos);
-    ADIOI_Free(send_buf_idx);
-    ADIOI_Free(curr_to_proc);
-    ADIOI_Free(done_to_proc);
-}
-
-
-/* Sets error_code to MPI_SUCCESS if successful, or creates an error code
- * in the case of error.
- */
-static void ADIOI_W_Exchange_data(ADIO_File fd, void *buf, char *write_buf,
-				  ADIOI_Flatlist_node *flat_buf, ADIO_Offset 
-				  *offset_list, ADIO_Offset *len_list, int *send_size, 
-				  int *recv_size, ADIO_Offset off, int size,
-				  int *count, int *start_pos,
-				  int *partial_recv,
-				  int *sent_to_proc, int nprocs, 
-				  int myrank, int
-				  buftype_is_contig, int contig_access_count,
-				  ADIO_Offset min_st_offset,
-				  ADIO_Offset fd_size,
-				  ADIO_Offset *fd_start, ADIO_Offset *fd_end, 
-				  ADIOI_Access *others_req, 
-				  int *send_buf_idx, int *curr_to_proc,
-				  int *done_to_proc, int *hole, int iter, 
-				  MPI_Aint buftype_extent, int *buf_idx,
-				  int *error_code)
-{
-    int i, j, k, *tmp_len, nprocs_recv, nprocs_send, err;
-    char **send_buf = NULL; 
-    MPI_Request *requests, *send_req;
-    MPI_Datatype *recv_types;
-    MPI_Status *statuses, status;
-    int *srt_len=NULL, sum;
-    ADIO_Offset *srt_off=NULL;
-    static char myname[] = "ADIOI_W_EXCHANGE_DATA";
-
-/* exchange recv_size info so that each process knows how much to
-   send to whom. */
-
-    MPI_Alltoall(recv_size, 1, MPI_INT, send_size, 1, MPI_INT, fd->comm);
-
-    /* create derived datatypes for recv */
-
-    nprocs_recv = 0;
-    for (i=0; i<nprocs; i++) if (recv_size[i]) nprocs_recv++;
-
-    recv_types = (MPI_Datatype *)
-	ADIOI_Malloc((nprocs_recv+1)*sizeof(MPI_Datatype)); 
-/* +1 to avoid a 0-size malloc */
-
-    tmp_len = (int *) ADIOI_Malloc(nprocs*sizeof(int));
-    j = 0;
-    for (i=0; i<nprocs; i++) {
-	if (recv_size[i]) {
-/* take care if the last off-len pair is a partial recv */
-	    if (partial_recv[i]) {
-		k = start_pos[i] + count[i] - 1;
-		tmp_len[i] = others_req[i].lens[k];
-		others_req[i].lens[k] = partial_recv[i];
-	    }
-	    ADIOI_Type_create_hindexed_x(count[i],
-		     &(others_req[i].lens[start_pos[i]]),
-	             &(others_req[i].mem_ptrs[start_pos[i]]), 
-			 MPI_BYTE, recv_types+j);
-	    /* absolute displacements; use MPI_BOTTOM in recv */
-	    MPI_Type_commit(recv_types+j);
-	    j++;
-	}
-    }
-
-    /* To avoid a read-modify-write, check if there are holes in the 
-       data to be written. For this, merge the (sorted) offset lists
-       others_req using a heap-merge. */
-
-    sum = 0;
-    for (i=0; i<nprocs; i++) sum += count[i];
-    /* valgrind-detcted optimization: if there is no work on this process we do
-     * not need to search for holes */
-    if (sum) {
-        srt_off = (ADIO_Offset *) ADIOI_Malloc(sum*sizeof(ADIO_Offset));
-        srt_len = (int *) ADIOI_Malloc(sum*sizeof(int));
-
-        ADIOI_Heap_merge(others_req, count, srt_off, srt_len, start_pos,
-                         nprocs, nprocs_recv, sum);
-    }
-
-/* for partial recvs, restore original lengths */
-    for (i=0; i<nprocs; i++) 
-        if (partial_recv[i]) {
-            k = start_pos[i] + count[i] - 1;
-            others_req[i].lens[k] = tmp_len[i];
-        }
-    ADIOI_Free(tmp_len);
-
-    /* check if there are any holes. If yes, must do read-modify-write.
-     * holes can be in three places.  'middle' is what you'd expect: the
-     * processes are operating on noncontigous data.  But holes can also show
-     * up at the beginning or end of the file domain (see John Bent ROMIO REQ
-     * #835). Missing these holes would result in us writing more data than
-     * recieved by everyone else. */
-
-    *hole = 0;
-    if (sum) {
-        if (off != srt_off[0]) /* hole at the front */
-            *hole = 1;
-        else { /* coalesce the sorted offset-length pairs */
-            for (i=1; i<sum; i++) {
-                if (srt_off[i] <= srt_off[0] + srt_len[0]) {
-		    /* ok to cast: operating on cb_buffer_size chunks */
-		    int new_len = (int)srt_off[i] + srt_len[i] - (int)srt_off[0];
-		    if (new_len > srt_len[0]) srt_len[0] = new_len;
-		}
-		else
-			break;
-	    }
-            if (i < sum || size != srt_len[0]) /* hole in middle or end */
-                *hole = 1;
-	}
-
-        ADIOI_Free(srt_off);
-        ADIOI_Free(srt_len);
-    }
-
-    if (nprocs_recv) {
-	if (*hole) {
-	    ADIO_ReadContig(fd, write_buf, size, MPI_BYTE, 
-			    ADIO_EXPLICIT_OFFSET, off, &status, &err);
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (err != MPI_SUCCESS) {
-		*error_code = MPIO_Err_create_code(err,
-						   MPIR_ERR_RECOVERABLE, myname,
-						   __LINE__, MPI_ERR_IO,
-						   "**ioRMWrdwr", 0);
-		return;
-	    } 
-	    /* --END ERROR HANDLING-- */
-	}
-    }
-
-    nprocs_send = 0;
-    for (i=0; i < nprocs; i++) if (send_size[i]) nprocs_send++;
-
-    if (fd->atomicity) {
-        /* bug fix from Wei-keng Liao and Kenin Coloma */
-        requests = (MPI_Request *)
-	    ADIOI_Malloc((nprocs_send+1)*sizeof(MPI_Request)); 
-        send_req = requests;
-    }
-    else {
-        requests = (MPI_Request *) 	
-            ADIOI_Malloc((nprocs_send+nprocs_recv+1)*sizeof(MPI_Request)); 
-        /* +1 to avoid a 0-size malloc */
-
-        /* post receives */
-        j = 0;
-        for (i=0; i<nprocs; i++) {
-            if (recv_size[i]) {
-                MPI_Irecv(MPI_BOTTOM, 1, recv_types[j], i, myrank+i+100*iter,
-                          fd->comm, requests+j);
-                j++;
-            }
-        }
-	send_req = requests + nprocs_recv;
-    }
-
-/* post sends. if buftype_is_contig, data can be directly sent from
-   user buf at location given by buf_idx. else use send_buf. */
-
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5032, 0, NULL);
-#endif
-    if (buftype_is_contig) {
-	j = 0;
-	for (i=0; i < nprocs; i++) 
-	    if (send_size[i]) {
-		MPI_Isend(((char *) buf) + buf_idx[i], send_size[i], 
-  		            MPI_BYTE, i,  myrank+i+100*iter, fd->comm, 
-                                  send_req+j);
-		j++;
-                buf_idx[i] += send_size[i];
-	    }
-    }
-    else if (nprocs_send) {
-	/* buftype is not contig */
-	send_buf = (char **) ADIOI_Malloc(nprocs*sizeof(char*));
-	for (i=0; i < nprocs; i++) 
-	    if (send_size[i]) 
-		send_buf[i] = (char *) ADIOI_Malloc(send_size[i]);
-
-	ADIOI_Fill_send_buffer(fd, buf, flat_buf, send_buf,
-                           offset_list, len_list, send_size, 
-			   send_req,
-                           sent_to_proc, nprocs, myrank, 
-                           contig_access_count,
-                           min_st_offset, fd_size, fd_start, fd_end, 
-                           send_buf_idx, curr_to_proc, done_to_proc, iter,
-                           buftype_extent);
-        /* the send is done in ADIOI_Fill_send_buffer */
-    }
-
-    if (fd->atomicity) {
-        /* bug fix from Wei-keng Liao and Kenin Coloma */
-        j = 0;
-        for (i=0; i<nprocs; i++) {
-            MPI_Status wkl_status;
-	    if (recv_size[i]) {
-	        MPI_Recv(MPI_BOTTOM, 1, recv_types[j], i, myrank+i+100*iter,
-		          fd->comm, &wkl_status);
-	        j++;
-	    }
-        }
-    }
-
-    for (i=0; i<nprocs_recv; i++) MPI_Type_free(recv_types+i);
-    ADIOI_Free(recv_types);
-    
-    if (fd->atomicity) {
-        /* bug fix from Wei-keng Liao and Kenin Coloma */
-        statuses = (MPI_Status *) ADIOI_Malloc((nprocs_send+1) * \
-                                         sizeof(MPI_Status)); 
-         /* +1 to avoid a 0-size malloc */
-    }
-    else {
-        statuses = (MPI_Status *) ADIOI_Malloc((nprocs_send+nprocs_recv+1) * \
-                                     sizeof(MPI_Status)); 
-        /* +1 to avoid a 0-size malloc */
-    }
-
-#ifdef NEEDS_MPI_TEST
-    i = 0;
-    if (fd->atomicity) {
-        /* bug fix from Wei-keng Liao and Kenin Coloma */
-        while (!i) MPI_Testall(nprocs_send, send_req, &i, statuses);
-    }
-    else {
-        while (!i) MPI_Testall(nprocs_send+nprocs_recv, requests, &i, statuses);
-    }
-#else
-    if (fd->atomicity)
-        /* bug fix from Wei-keng Liao and Kenin Coloma */
-        MPI_Waitall(nprocs_send, send_req, statuses);
-    else
-        MPI_Waitall(nprocs_send+nprocs_recv, requests, statuses);
-#endif
-
-#ifdef AGGREGATION_PROFILE
-    MPE_Log_event (5033, 0, NULL);
-#endif
-    ADIOI_Free(statuses);
-    ADIOI_Free(requests);
-    if (!buftype_is_contig && nprocs_send) {
-	for (i=0; i < nprocs; i++) 
-	    if (send_size[i]) ADIOI_Free(send_buf[i]);
-	ADIOI_Free(send_buf);
-    }
-}
-
-#define ADIOI_BUF_INCR \
-{ \
-    while (buf_incr) { \
-        size_in_buf = ADIOI_MIN(buf_incr, flat_buf_sz); \
-        user_buf_idx += size_in_buf; \
-        flat_buf_sz -= size_in_buf; \
-        if (!flat_buf_sz) { \
-            if (flat_buf_idx < (flat_buf->count - 1)) flat_buf_idx++; \
-            else { \
-                flat_buf_idx = 0; \
-                n_buftypes++; \
-            } \
-            user_buf_idx = flat_buf->indices[flat_buf_idx] + \
-                              (ADIO_Offset)n_buftypes*(ADIO_Offset)buftype_extent; \
-            flat_buf_sz = flat_buf->blocklens[flat_buf_idx]; \
-        } \
-        buf_incr -= size_in_buf; \
-    } \
-}
-
-
-#define ADIOI_BUF_COPY \
-{ \
-    while (size) { \
-        size_in_buf = ADIOI_MIN(size, flat_buf_sz); \
-  ADIOI_Assert((((ADIO_Offset)(MPIU_Upint)buf) + user_buf_idx) == (ADIO_Offset)(MPIU_Upint)((MPIU_Upint)buf + user_buf_idx)); \
-  ADIOI_Assert((ADIO_Size) size_in_buf == (size_t)size_in_buf);          \
-        memcpy(&(send_buf[p][send_buf_idx[p]]), \
-               ((char *) buf) + user_buf_idx, size_in_buf); \
-        send_buf_idx[p] += size_in_buf; \
-        user_buf_idx += size_in_buf; \
-        flat_buf_sz -= size_in_buf; \
-        if (!flat_buf_sz) { \
-            if (flat_buf_idx < (flat_buf->count - 1)) flat_buf_idx++; \
-            else { \
-                flat_buf_idx = 0; \
-                n_buftypes++; \
-            } \
-            user_buf_idx = flat_buf->indices[flat_buf_idx] + \
-                              (ADIO_Offset)n_buftypes*(ADIO_Offset)buftype_extent; \
-            flat_buf_sz = flat_buf->blocklens[flat_buf_idx]; \
-        } \
-        size -= size_in_buf; \
-        buf_incr -= size_in_buf; \
-    } \
-    ADIOI_BUF_INCR \
-}
-
-
-
-
-
-void ADIOI_Fill_send_buffer(ADIO_File fd, void *buf, ADIOI_Flatlist_node
-                           *flat_buf, char **send_buf, ADIO_Offset 
-                           *offset_list, ADIO_Offset *len_list, int *send_size, 
-                           MPI_Request *requests, int *sent_to_proc, 
-                           int nprocs, int myrank, 
-                           int contig_access_count, 
-                           ADIO_Offset min_st_offset, ADIO_Offset fd_size,
-                           ADIO_Offset *fd_start, ADIO_Offset *fd_end, 
-                           int *send_buf_idx, int *curr_to_proc, 
-                           int *done_to_proc, int iter,
-                           MPI_Aint buftype_extent)
-{
-/* this function is only called if buftype is not contig */
-
-    int i, p, flat_buf_idx;
-    ADIO_Offset flat_buf_sz, size_in_buf, buf_incr, size;
-    int jj, n_buftypes;
-    ADIO_Offset off, len, rem_len, user_buf_idx;
-
-/*  curr_to_proc[p] = amount of data sent to proc. p that has already
-    been accounted for so far
-    done_to_proc[p] = amount of data already sent to proc. p in 
-    previous iterations
-    user_buf_idx = current location in user buffer 
-    send_buf_idx[p] = current location in send_buf of proc. p  */
-
-    for (i=0; i < nprocs; i++) {
-	send_buf_idx[i] = curr_to_proc[i] = 0;
-	done_to_proc[i] = sent_to_proc[i];
-    }
-    jj = 0;
-
-    user_buf_idx = flat_buf->indices[0];
-    flat_buf_idx = 0;
-    n_buftypes = 0;
-    flat_buf_sz = flat_buf->blocklens[0];
-
-    /* flat_buf_idx = current index into flattened buftype
-       flat_buf_sz = size of current contiguous component in 
-	                 flattened buf */
-
-    for (i=0; i<contig_access_count; i++) { 
-	off     = offset_list[i];
-	rem_len = len_list[i];
-
-	/*this request may span the file domains of more than one process*/
-	while (rem_len != 0) {
-	    len = rem_len;
-	    /* NOTE: len value is modified by ADIOI_Calc_aggregator() to be no
-	     * longer than the single region that processor "p" is responsible
-	     * for.
-	     */
-	    p = ADIOI_Calc_aggregator(fd,
-				      off,
-				      min_st_offset,
-				      &len,
-				      fd_size,
-				      fd_start,
-				      fd_end);
-
-	    if (send_buf_idx[p] < send_size[p]) {
-		if (curr_to_proc[p]+len > done_to_proc[p]) {
-		    if (done_to_proc[p] > curr_to_proc[p]) {
-			size = ADIOI_MIN(curr_to_proc[p] + len - 
-                                done_to_proc[p], send_size[p]-send_buf_idx[p]);
-			buf_incr = done_to_proc[p] - curr_to_proc[p];
-			ADIOI_BUF_INCR
-      ADIOI_Assert((curr_to_proc[p] + len - done_to_proc[p]) == (unsigned)(curr_to_proc[p] + len - done_to_proc[p]));
-		        buf_incr = curr_to_proc[p] + len - done_to_proc[p];
-      ADIOI_Assert((done_to_proc[p] + size) == (unsigned)(done_to_proc[p] + size));
-			/* ok to cast: bounded by cb buffer size */
-			curr_to_proc[p] = done_to_proc[p] + (int)size;
-		        ADIOI_BUF_COPY
-		    }
-		    else {
-			size = ADIOI_MIN(len,send_size[p]-send_buf_idx[p]);
-			buf_incr = len;
-      ADIOI_Assert((curr_to_proc[p] + size) == (unsigned)((ADIO_Offset)curr_to_proc[p] + size));
-			curr_to_proc[p] += size;
-			ADIOI_BUF_COPY
-		    }
-		    if (send_buf_idx[p] == send_size[p]) {
-			MPI_Isend(send_buf[p], send_size[p], MPI_BYTE, p, 
-				myrank+p+100*iter, fd->comm, requests+jj);
-			jj++;
-		    }
-		}
-		else {
-        ADIOI_Assert((curr_to_proc[p] + len) == (unsigned)((ADIO_Offset)curr_to_proc[p] + len));
-		    curr_to_proc[p] += len;
-		    buf_incr = len;
-		    ADIOI_BUF_INCR
-		}
-	    }
-	    else {
-		buf_incr = len;
-		ADIOI_BUF_INCR
-            }
-	    off     += len;
-	    rem_len -= len;
-	}
-    }
-    for (i=0; i < nprocs; i++) 
-	if (send_size[i]) sent_to_proc[i] = curr_to_proc[i];
-}
-
-
-
-void ADIOI_Heap_merge(ADIOI_Access *others_req, int *count, 
-		      ADIO_Offset *srt_off, int *srt_len, int *start_pos,
-		      int nprocs, int nprocs_recv, int total_elements)
-{
-    typedef struct {
-	ADIO_Offset *off_list;
-	ADIO_Offset *len_list;
-	int nelem;
-    } heap_struct;
-
-    heap_struct *a, tmp;
-    int i, j, heapsize, l, r, k, smallest;
-
-    a = (heap_struct *) ADIOI_Malloc((nprocs_recv+1)*sizeof(heap_struct));
-
-    j = 0;
-    for (i=0; i<nprocs; i++)
-	if (count[i]) {
-	    a[j].off_list = &(others_req[i].offsets[start_pos[i]]);
-	    a[j].len_list = &(others_req[i].lens[start_pos[i]]);
-	    a[j].nelem = count[i];
-	    j++;
-	}
-
-    /* build a heap out of the first element from each list, with
-       the smallest element of the heap at the root */
-
-    heapsize = nprocs_recv;
-    for (i=heapsize/2 - 1; i>=0; i--) {
-	/* Heapify(a, i, heapsize); Algorithm from Cormen et al. pg. 143
-           modified for a heap with smallest element at root. I have 
-           removed the recursion so that there are no function calls.
-           Function calls are too expensive. */
-	k = i;
-	for(;;) {
-	    l = 2*(k+1) - 1;
-	    r = 2*(k+1);
-
-	    if ((l < heapsize) && 
-		(*(a[l].off_list) < *(a[k].off_list)))
-		smallest = l;
-	    else smallest = k;
-
-	    if ((r < heapsize) && 
-		(*(a[r].off_list) < *(a[smallest].off_list)))
-		smallest = r;
-
-	    if (smallest != k) {
-		tmp.off_list = a[k].off_list;
-		tmp.len_list = a[k].len_list;
-		tmp.nelem = a[k].nelem;
-
-		a[k].off_list = a[smallest].off_list;
-		a[k].len_list = a[smallest].len_list;
-		a[k].nelem = a[smallest].nelem;
-		
-		a[smallest].off_list = tmp.off_list;
-		a[smallest].len_list = tmp.len_list;
-		a[smallest].nelem = tmp.nelem;
-	    
-		k = smallest;
-	    }
-	    else break;
-	}
-    }
-
-    for (i=0; i<total_elements; i++) {
-        /* extract smallest element from heap, i.e. the root */
-	srt_off[i] = *(a[0].off_list);
-	srt_len[i] = *(a[0].len_list);
-	(a[0].nelem)--;
-
-	if (!a[0].nelem) {
-	    a[0].off_list = a[heapsize-1].off_list;
-	    a[0].len_list = a[heapsize-1].len_list;
-	    a[0].nelem = a[heapsize-1].nelem;
-	    heapsize--;
-	}
-	else {
-	    (a[0].off_list)++;
-	    (a[0].len_list)++;
-	}
-
-	/* Heapify(a, 0, heapsize); */
-	k = 0;
-	for (;;) {
-	    l = 2*(k+1) - 1;
-	    r = 2*(k+1);
-
-	    if ((l < heapsize) && 
-		(*(a[l].off_list) < *(a[k].off_list)))
-		smallest = l;
-	    else smallest = k;
-
-	    if ((r < heapsize) && 
-		(*(a[r].off_list) < *(a[smallest].off_list)))
-		smallest = r;
-
-	    if (smallest != k) {
-		tmp.off_list = a[k].off_list;
-		tmp.len_list = a[k].len_list;
-		tmp.nelem = a[k].nelem;
-
-		a[k].off_list = a[smallest].off_list;
-		a[k].len_list = a[smallest].len_list;
-		a[k].nelem = a[smallest].nelem;
-		
-		a[smallest].off_list = tmp.off_list;
-		a[smallest].len_list = tmp.len_list;
-		a[smallest].nelem = tmp.nelem;
-	    
-		k = smallest;
-	    }
-	    else break;
-	}
-    }
-    ADIOI_Free(a);
-}
diff --git a/3rd-party/romio321/adio/common/ad_write_nolock.c b/3rd-party/romio321/adio/common/ad_write_nolock.c
deleted file mode 100644
index 7532a499b9e..00000000000
--- a/3rd-party/romio321/adio/common/ad_write_nolock.c
+++ /dev/null
@@ -1,406 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-
-#include "adio.h"
-#include "adio_extern.h"
-#include <unistd.h>
-
-
-/* #define IO_DEBUG 1 */
-void ADIOI_NOLOCK_WriteStrided(ADIO_File fd, const void *buf, int count,
-			     MPI_Datatype datatype, int file_ptr_type,
-			     ADIO_Offset offset, ADIO_Status *status, int
-			     *error_code)
-{
-/* borrowed from old-school PVFS (v1) code. A driver for file systems that
- * cannot or do not support client-side buffering
- * Does not do data sieving optimization
- * Does contain write-combining optimization for noncontig in memory, contig in
- * file 
- */
-
-/* offset is in units of etype relative to the filetype. */
-
-    ADIOI_Flatlist_node *flat_buf, *flat_file;
-    int j, k, st_index=0;
-    off_t err_lseek=-1;
-    ssize_t err=-1;
-    ADIO_Offset fwr_size=0, bwr_size, new_bwr_size, new_fwr_size, i_offset, num;
-    ADIO_Offset bufsize, n_etypes_in_filetype;
-    ADIO_Offset n_filetypes, etype_in_filetype, size, sum;
-    ADIO_Offset abs_off_in_filetype=0, size_in_filetype;
-    MPI_Count filetype_size, etype_size, buftype_size;
-    MPI_Aint filetype_extent, buftype_extent, indx, lb;
-    int buf_count, buftype_is_contig, filetype_is_contig;
-    ADIO_Offset off, disp;
-    int flag, err_flag=0;
-    static char myname[] = "ADIOI_NOLOCK_WRITESTRIDED";
-#ifdef IO_DEBUG
-    int rank,nprocs;
-#endif
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (fd->atomicity) {
-	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					   myname, __LINE__,
-					   MPI_ERR_INTERN,
-					   "Atomic mode set in I/O function", 0);
-	return;
-    }
-    /* --END ERROR HANDLING-- */
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-
-    MPI_Type_size_x(fd->filetype, &filetype_size);
-    if ( ! filetype_size ) {
-#ifdef HAVE_STATUS_SET_BYTES
-	MPIR_Status_set_bytes(status, datatype, 0);
-#endif
-	*error_code = MPI_SUCCESS; 
-	return;
-    }
-
-#ifdef IO_DEBUG
-    MPI_Comm_rank(fd->comm, &rank);
-    MPI_Comm_size(fd->comm, &nprocs);
-#endif
-
-    MPI_Type_get_extent(fd->filetype, &lb, &filetype_extent);
-    MPI_Type_size_x(datatype, &buftype_size);
-    MPI_Type_get_extent(datatype, &lb, &buftype_extent);
-    etype_size = fd->etype_size;
-    
-    ADIOI_Assert((buftype_size * count) == ((ADIO_Offset)(unsigned)buftype_size * (ADIO_Offset)count));
-    bufsize = buftype_size * count;
-
-    if (!buftype_is_contig && filetype_is_contig) {
-	char *combine_buf, *combine_buf_ptr;
-	ADIO_Offset combine_buf_remain;
-/* noncontiguous in memory, contiguous in file. use writev */
-
-	flat_buf = ADIOI_Flatten_and_find(datatype);
-
-	/* allocate our "combine buffer" to pack data into before writing */
-	combine_buf = (char *) ADIOI_Malloc(fd->hints->ind_wr_buffer_size);
-	combine_buf_ptr = combine_buf;
-	combine_buf_remain = fd->hints->ind_wr_buffer_size;
-
-	/* seek to the right spot in the file */
-	if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-	    off = fd->disp + etype_size * offset;
-	    lseek(fd->fd_sys, off, SEEK_SET);
-	}
-	else off = lseek(fd->fd_sys, fd->fp_ind, SEEK_SET);
-
-	/* loop through all the flattened pieces.  combine into buffer until
-	 * no more will fit, then write.
-	 *
-	 * special case of a given piece being bigger than the combine buffer
-	 * is also handled.
-	 */
-	for (j=0; j<count; j++) {
-    int i;
-	    for (i=0; i<flat_buf->count; i++) {
-		if (flat_buf->blocklens[i] > combine_buf_remain && combine_buf != combine_buf_ptr) {
-		    /* there is data in the buffer; write out the buffer so far */
-#ifdef IO_DEBUG
-		    printf("[%d/%d] nc mem c file (0) writing loc = %Ld sz = %Ld\n", 
-				    rank, nprocs, off, 
-				    fd->hints->ind_wr_buffer_size-combine_buf_remain);
-#endif
-#ifdef ADIOI_MPE_LOGGING
-		    MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
-#endif
-		    err = write(fd->fd_sys,
-				     combine_buf,
-				     fd->hints->ind_wr_buffer_size - combine_buf_remain);
-#ifdef ADIOI_MPE_LOGGING
-		    MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
-#endif
-		    if (err == -1) err_flag = 1;
-
-		    /* reset our buffer info */
-		    combine_buf_ptr = combine_buf;
-		    combine_buf_remain = fd->hints->ind_wr_buffer_size;
-		}
-
-		/* TODO: heuristic for when to not bother to use combine buffer? */
-		if (flat_buf->blocklens[i] >= combine_buf_remain) {
-		    /* special case: blocklen is as big as or bigger than the combine buf;
-		     * write directly
-		     */
-#ifdef IO_DEBUG
-		    printf("[%d/%d] nc mem c file (1) writing loc = %Ld sz = %d\n", 
-				    rank, nprocs, off, 
-				    flat_buf->blocklens[i]);
-#endif
-        ADIOI_Assert(flat_buf->blocklens[i] == (unsigned)flat_buf->blocklens[i]);
-        ADIOI_Assert((((ADIO_Offset)(MPIU_Upint)buf) + (ADIO_Offset)j*(ADIO_Offset)buftype_extent + flat_buf->indices[i]) == (ADIO_Offset)((MPIU_Upint)buf + (ADIO_Offset)j*(ADIO_Offset)buftype_extent + flat_buf->indices[i]));
-#ifdef ADIOI_MPE_LOGGING
-		    MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
-#endif
-		    err = write(fd->fd_sys,
-				     ((char *) buf) + (ADIO_Offset)j*(ADIO_Offset)buftype_extent + flat_buf->indices[i],
-				     (unsigned)flat_buf->blocklens[i]);
-#ifdef ADIOI_MPE_LOGGING
-		    MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
-#endif
-		    if (err == -1) err_flag = 1;
-		    off += flat_buf->blocklens[i]; /* keep up with the final file offset too */
-		}
-		else {
-		    /* copy more data into combine buffer */
-		    memcpy(combine_buf_ptr,
-			   ((char *) buf) + j*buftype_extent + flat_buf->indices[i],
-			   flat_buf->blocklens[i]);
-		    combine_buf_ptr += flat_buf->blocklens[i];
-		    combine_buf_remain -= flat_buf->blocklens[i];
-		    off += flat_buf->blocklens[i]; /* keep up with the final file offset too */
-		}
-	    }
-	}
-
-	if (combine_buf_ptr != combine_buf) {
-	    /* data left in buffer to write */
-#ifdef IO_DEBUG
-	    printf("[%d/%d] nc mem c file (2) writing loc = %Ld sz = %Ld\n", 
-			    rank, nprocs, off, 
-			     fd->hints->ind_wr_buffer_size-combine_buf_remain);
-#endif
-#ifdef ADIOI_MPE_LOGGING
-	    MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
-#endif
-	    err = write(fd->fd_sys,
-			     combine_buf,
-			     fd->hints->ind_wr_buffer_size - combine_buf_remain);
-#ifdef ADIOI_MPE_LOGGING
-	    MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
-#endif
-	    if (err == -1) err_flag = 1;
-	}
-
-	if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
-
-	ADIOI_Free(combine_buf);
-
-	if (err_flag) {
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE, myname,
-					       __LINE__, MPI_ERR_IO, "**io",
-					       "**io %s", strerror(errno));
-	}
-	else *error_code = MPI_SUCCESS;
-    } /* if (!buftype_is_contig && filetype_is_contig)  ... */
-
-    else {  /* noncontiguous in file */
-
-/* split up into several contiguous writes */
-
-/* find starting location in the file */
-
-/* filetype already flattened in ADIO_Open */
-	flat_file = ADIOI_Flatlist;
-	while (flat_file->type != fd->filetype) flat_file = flat_file->next;
-        disp = fd->disp;
-
-	if (file_ptr_type == ADIO_INDIVIDUAL) {
-	    offset = fd->fp_ind; /* in bytes */
-            n_filetypes = -1;
-            flag = 0;
-            while (!flag) {
-                int i;
-                n_filetypes++;
-                for (i=0; i<flat_file->count; i++) {
-                    if (disp + flat_file->indices[i] + 
-                        n_filetypes*(ADIO_Offset)filetype_extent + flat_file->blocklens[i] 
-                            >= offset) {
-                        st_index = i;
-                        fwr_size = disp + flat_file->indices[i] + 
-                                n_filetypes*(ADIO_Offset)filetype_extent
-                                 + flat_file->blocklens[i] - offset;
-                        flag = 1;
-                        break;
-                    }
-                }
-            }
-	}
-	else {
-            int i;
-	    n_etypes_in_filetype = filetype_size/etype_size;
-	    n_filetypes = offset / n_etypes_in_filetype;
-	    etype_in_filetype = offset % n_etypes_in_filetype;
-	    size_in_filetype = etype_in_filetype * etype_size;
- 
-	    sum = 0;
-	    for (i=0; i<flat_file->count; i++) {
-		sum += flat_file->blocklens[i];
-		if (sum > size_in_filetype) {
-		    st_index = i;
-		    fwr_size = sum - size_in_filetype;
-		    abs_off_in_filetype = flat_file->indices[i] +
-			size_in_filetype - (sum - flat_file->blocklens[i]);
-		    break;
-		}
-	    }
-
-	    /* abs. offset in bytes in the file */
-            offset = disp + n_filetypes*(ADIO_Offset)filetype_extent + abs_off_in_filetype;
-	}
-
-	if (buftype_is_contig && !filetype_is_contig) {
-
-/* contiguous in memory, noncontiguous in file. should be the most
-   common case. */
-
-	    i_offset = 0;
-	    j = st_index;
-	    off = offset;
-	    fwr_size = ADIOI_MIN(fwr_size, bufsize);
-	    while (i_offset < bufsize) {
-                if (fwr_size) { 
-                    /* TYPE_UB and TYPE_LB can result in 
-                       fwr_size = 0. save system call in such cases */ 
-#ifdef ADIOI_MPE_LOGGING
-		    MPE_Log_event(ADIOI_MPE_lseek_a, 0, NULL);
-#endif
-#ifdef IO_DEBUG
-		    printf("[%d/%d] c mem nc file writing loc = %Ld sz = %d\n", 
-			    rank, nprocs, off, fwr_size);
-#endif
-		    err_lseek = lseek(fd->fd_sys, off, SEEK_SET);
-#ifdef ADIOI_MPE_LOGGING
-		    MPE_Log_event(ADIOI_MPE_lseek_b, 0, NULL);
-#endif
-		    if (err_lseek == -1) err_flag = 1;
-#ifdef ADIOI_MPE_LOGGING
-		    MPE_Log_event(ADIOI_MPE_write_a, 0, NULL);
-#endif
-		    err = write(fd->fd_sys, ((char *) buf) + i_offset, fwr_size);
-#ifdef ADIOI_MPE_LOGGING
-		    MPE_Log_event(ADIOI_MPE_write_b, 0, NULL);
-#endif
-		    if (err == -1) err_flag = 1;
-		}
-		i_offset += fwr_size;
-
-                if (off + fwr_size < disp + flat_file->indices[j] +
-                   flat_file->blocklens[j] + n_filetypes*(ADIO_Offset)filetype_extent)
-                       off += fwr_size;
-                /* did not reach end of contiguous block in filetype.
-                   no more I/O needed. off is incremented by fwr_size. */
-                else {
-		    if (j < (flat_file->count - 1)) j++;
-		    else {
-			j = 0;
-			n_filetypes++;
-		    }
-		    off = disp + flat_file->indices[j] + 
-                                        n_filetypes*(ADIO_Offset)filetype_extent;
-		    fwr_size = ADIOI_MIN(flat_file->blocklens[j], bufsize-i_offset);
-		}
-	    }
-	}
-	else {
-/* noncontiguous in memory as well as in file */
-
-	    flat_buf = ADIOI_Flatten_and_find(datatype);
-
-	    k = num = buf_count = 0;
-	    indx = flat_buf->indices[0];
-	    j = st_index;
-	    off = offset;
-	    bwr_size = flat_buf->blocklens[0];
-
-	    while (num < bufsize) {
-		size = ADIOI_MIN(fwr_size, bwr_size);
-		if (size) {
-#ifdef IO_DEBUG
-		    printf("[%d/%d] nc mem nc file writing loc = %Ld sz = %d\n", 
-				    rank, nprocs, off, size);
-#endif
-#ifdef ADIOI_MPE_LOGGING
-		    MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
-#endif
-		    lseek(fd->fd_sys, off, SEEK_SET);
-#ifdef ADIOI_MPE_LOGGING 
-		    MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
-#endif
-		    if (err == -1) err_flag = 1;
-#ifdef ADIOI_MPE_LOGGING 
-		    MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
-#endif
-                    ADIOI_Assert((ADIO_Size) size == (size_t) size);
-                    ADIOI_Assert(off == (off_t) off);
-		    err = write(fd->fd_sys, ((char *) buf) + indx, size);
-#ifdef ADIOI_MPE_LOGGING
-		    MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
-#endif
-		    if (err == -1) err_flag = 1;
-		}
-
-		new_fwr_size = fwr_size;
-		new_bwr_size = bwr_size;
-
-		if (size == fwr_size) {
-/* reached end of contiguous block in file */
-                    if (j < (flat_file->count - 1)) j++;
-                    else {
-                        j = 0;
-                        n_filetypes++;
-                    }
-
-                    off = disp + flat_file->indices[j] + 
-                                   n_filetypes*(ADIO_Offset)filetype_extent;
-
-		    new_fwr_size = flat_file->blocklens[j];
-		    if (size != bwr_size) {
-			indx += size;
-			new_bwr_size -= size;
-		    }
-		}
-
-		if (size == bwr_size) {
-/* reached end of contiguous block in memory */
-
-		    k = (k + 1)%flat_buf->count;
-		    buf_count++;
-		    indx = buftype_extent*(buf_count/flat_buf->count) +
-			flat_buf->indices[k]; 
-		    new_bwr_size = flat_buf->blocklens[k];
-		    if (size != fwr_size) {
-			off += size;
-			new_fwr_size -= size;
-		    }
-		}
-		num += size;
-		fwr_size = new_fwr_size;
-                bwr_size = new_bwr_size;
-	    }
-	}
-
-        if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
-	if (err_flag) {
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE, myname,
-					       __LINE__, MPI_ERR_IO, "**io",
-					       "**io %s", strerror(errno));
-	}
-	else *error_code = MPI_SUCCESS;
-    }
-
-    fd->fp_sys_posn = -1;   /* set it to null. */
-
-#ifdef HAVE_STATUS_SET_BYTES
-    MPIR_Status_set_bytes(status, datatype, bufsize);
-/* This is a temporary way of filling in status. The right way is to 
-   keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
-#endif
-
-    if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
-}
diff --git a/3rd-party/romio321/adio/common/ad_write_str.c b/3rd-party/romio321/adio/common/ad_write_str.c
deleted file mode 100644
index 83f2420ddc9..00000000000
--- a/3rd-party/romio321/adio/common/ad_write_str.c
+++ /dev/null
@@ -1,486 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-
-#define ADIOI_BUFFERED_WRITE \
-{ \
-    if (req_off >= writebuf_off + writebuf_len) { \
-        if (writebuf_len) { \
-           ADIO_WriteContig(fd, writebuf, writebuf_len, MPI_BYTE, \
-                  ADIO_EXPLICIT_OFFSET, writebuf_off, &status1, error_code); \
-           if (!(fd->atomicity)) ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
-           if (*error_code != MPI_SUCCESS) { \
-               *error_code = MPIO_Err_create_code(*error_code, \
-                                                  MPIR_ERR_RECOVERABLE, myname, \
-                                                  __LINE__, MPI_ERR_IO, \
-                                                  "**iowswc", 0); \
-               goto fn_exit; \
-           } \
-        } \
-	writebuf_off = req_off; \
-        writebuf_len = (unsigned) (ADIOI_MIN(max_bufsize,end_offset-writebuf_off+1));\
-	if (!(fd->atomicity)) ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
-	ADIO_ReadContig(fd, writebuf, writebuf_len, MPI_BYTE, \
-                 ADIO_EXPLICIT_OFFSET, writebuf_off, &status1, error_code); \
-	if (*error_code != MPI_SUCCESS) { \
-	    *error_code = MPIO_Err_create_code(*error_code, \
-					       MPIR_ERR_RECOVERABLE, myname, \
-					       __LINE__, MPI_ERR_IO, \
-					       "**iowsrc", 0); \
-	    goto fn_exit; \
-	} \
-    } \
-    write_sz = (unsigned) (ADIOI_MIN(req_len, writebuf_off + writebuf_len - req_off)); \
-    ADIOI_Assert((ADIO_Offset)write_sz == ADIOI_MIN(req_len, writebuf_off + writebuf_len - req_off));\
-    memcpy(writebuf+req_off-writebuf_off, (char *)buf +userbuf_off, write_sz);\
-    while (write_sz != req_len) { \
-        ADIO_WriteContig(fd, writebuf, writebuf_len, MPI_BYTE, \
-                  ADIO_EXPLICIT_OFFSET, writebuf_off, &status1, error_code); \
-        if (!(fd->atomicity)) ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
-        if (*error_code != MPI_SUCCESS) { \
-            *error_code = MPIO_Err_create_code(*error_code, \
-                                               MPIR_ERR_RECOVERABLE, myname, \
-                                               __LINE__, MPI_ERR_IO, \
-                                               "**iowswc", 0); \
-            goto fn_exit; \
-        } \
-        req_len -= write_sz; \
-        userbuf_off += write_sz; \
-        writebuf_off += writebuf_len; \
-        writebuf_len = (unsigned) (ADIOI_MIN(max_bufsize,end_offset-writebuf_off+1));\
-	if (!(fd->atomicity)) ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
-        ADIO_ReadContig(fd, writebuf, writebuf_len, MPI_BYTE, \
-                  ADIO_EXPLICIT_OFFSET, writebuf_off, &status1, error_code); \
-	if (*error_code != MPI_SUCCESS) { \
-	    *error_code = MPIO_Err_create_code(*error_code, \
-					       MPIR_ERR_RECOVERABLE, myname, \
-					       __LINE__, MPI_ERR_IO, \
-					       "**iowsrc", 0); \
-	    goto fn_exit; \
-	} \
-        write_sz = ADIOI_MIN(req_len, writebuf_len); \
-        memcpy(writebuf, (char *)buf + userbuf_off, write_sz);\
-    } \
-}
-
-
-/* this macro is used when filetype is contig and buftype is not contig.
-   it does not do a read-modify-write and does not lock*/
-#define ADIOI_BUFFERED_WRITE_WITHOUT_READ \
-{ \
-    if (req_off >= writebuf_off + writebuf_len) { \
-        ADIO_WriteContig(fd, writebuf, writebuf_len, MPI_BYTE, \
-                 ADIO_EXPLICIT_OFFSET, writebuf_off, &status1, error_code); \
-        if (*error_code != MPI_SUCCESS) { \
-            *error_code = MPIO_Err_create_code(*error_code, \
-                                               MPIR_ERR_RECOVERABLE, myname, \
-                                               __LINE__, MPI_ERR_IO, \
-                                               "**iowswc", 0); \
-            goto fn_exit; \
-        } \
-	writebuf_off = req_off; \
-        writebuf_len = (unsigned) (ADIOI_MIN(max_bufsize,end_offset-writebuf_off+1));\
-    } \
-    write_sz = (unsigned) (ADIOI_MIN(req_len, writebuf_off + writebuf_len - req_off)); \
-    ADIOI_Assert((ADIO_Offset)write_sz == ADIOI_MIN(req_len, writebuf_off + writebuf_len - req_off));\
-    memcpy(writebuf+req_off-writebuf_off, (char *)buf +userbuf_off, write_sz);\
-    while (write_sz != req_len) { \
-        ADIO_WriteContig(fd, writebuf, writebuf_len, MPI_BYTE, \
-                ADIO_EXPLICIT_OFFSET, writebuf_off, &status1, error_code); \
-        if (*error_code != MPI_SUCCESS) { \
-            *error_code = MPIO_Err_create_code(*error_code, \
-                                               MPIR_ERR_RECOVERABLE, myname, \
-                                               __LINE__, MPI_ERR_IO, \
-                                               "**iowswc", 0); \
-            goto fn_exit; \
-        } \
-        req_len -= write_sz; \
-        userbuf_off += write_sz; \
-        writebuf_off += writebuf_len; \
-        writebuf_len = (unsigned) (ADIOI_MIN(max_bufsize,end_offset-writebuf_off+1));\
-        write_sz = ADIOI_MIN(req_len, writebuf_len); \
-        memcpy(writebuf, (char *)buf + userbuf_off, write_sz);\
-    } \
-}
-void ADIOI_GEN_WriteStrided(ADIO_File fd, const void *buf, int count,
-                       MPI_Datatype datatype, int file_ptr_type,
-                       ADIO_Offset offset, ADIO_Status *status, int
-                       *error_code)
-{
-
-/* offset is in units of etype relative to the filetype. */
-
-    ADIOI_Flatlist_node *flat_buf, *flat_file;
-    ADIO_Offset i_offset, sum, size_in_filetype;
-    int i, j, k, st_index=0;
-    ADIO_Offset num, size, n_filetypes, etype_in_filetype, st_n_filetypes;
-    ADIO_Offset n_etypes_in_filetype, abs_off_in_filetype=0;
-    MPI_Count filetype_size, etype_size, buftype_size;
-    MPI_Aint filetype_extent, buftype_extent, lb; 
-    int buf_count, buftype_is_contig, filetype_is_contig;
-    ADIO_Offset userbuf_off;
-    ADIO_Offset off, req_off, disp, end_offset=0, writebuf_off, start_off;
-    char *writebuf=NULL;
-    unsigned writebuf_len, max_bufsize, write_sz;
-    MPI_Aint bufsize;
-    ADIO_Status status1;
-    ADIO_Offset new_bwr_size, new_fwr_size, st_fwr_size, fwr_size=0, bwr_size, req_len;
-    static char myname[] = "ADIOI_GEN_WriteStrided";
-
-    if (fd->hints->ds_write == ADIOI_HINT_DISABLE) {
-    	/* if user has disabled data sieving on reads, use naive
-	 * approach instead.
-	 */
-
-	ADIOI_GEN_WriteStrided_naive(fd, 
-				    buf,
-				    count,
-				    datatype,
-				    file_ptr_type,
-				    offset,
-				    status,
-				    error_code);
-    	return;
-    }
-
-
-    *error_code = MPI_SUCCESS;  /* changed below if error */
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-
-    MPI_Type_size_x(fd->filetype, &filetype_size);
-    if ( ! filetype_size ) {
-#ifdef HAVE_STATUS_SET_BYTES
-	MPIR_Status_set_bytes(status, datatype, 0);
-#endif
-	*error_code = MPI_SUCCESS; 
-	return;
-    }
-
-    MPI_Type_get_extent(fd->filetype, &lb, &filetype_extent);
-    MPI_Type_size_x(datatype, &buftype_size);
-    MPI_Type_get_extent(datatype, &lb, &buftype_extent);
-    etype_size = fd->etype_size;
-
-    ADIOI_Assert((buftype_size * count) == ((MPI_Count)buftype_size * (ADIO_Offset)count));
-    bufsize = buftype_size * count;
-
-/* get max_bufsize from the info object. */
-
-    max_bufsize = fd->hints->ind_wr_buffer_size;
-
-    if (!buftype_is_contig && filetype_is_contig) {
-
-/* noncontiguous in memory, contiguous in file. */
-
-	flat_buf = ADIOI_Flatten_and_find(datatype);
-
-        off = (file_ptr_type == ADIO_INDIVIDUAL) ? fd->fp_ind : 
-                 fd->disp + (ADIO_Offset)etype_size * offset;
-
-	start_off = off;
-	end_offset = off + bufsize - 1;
-        writebuf_off = off;
-        writebuf = (char *) ADIOI_Malloc(max_bufsize);
-        writebuf_len = (unsigned) (ADIOI_MIN(max_bufsize, end_offset-writebuf_off+1));
-
-/* if atomicity is true, lock the region to be accessed */
-	if (fd->atomicity) 
-	    ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
-
-        for (j=0; j<count; j++) 
-        {
-              for (i=0; i<flat_buf->count; i++) {
-                  userbuf_off = (ADIO_Offset)j*(ADIO_Offset)buftype_extent + flat_buf->indices[i];
-      req_off = off;
-      req_len = flat_buf->blocklens[i];
-      ADIOI_BUFFERED_WRITE_WITHOUT_READ
-                  off += flat_buf->blocklens[i];
-              }
-        }
-
-        /* write the buffer out finally */
-        ADIO_WriteContig(fd, writebuf, writebuf_len, MPI_BYTE, ADIO_EXPLICIT_OFFSET,
-			writebuf_off, &status1, error_code);
-
-	if (fd->atomicity) 
-	    ADIOI_UNLOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
-
-	if (*error_code != MPI_SUCCESS) goto fn_exit;
-
-        if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
-    }
-
-    else {  /* noncontiguous in file */
-
-/* filetype already flattened in ADIO_Open */
-	flat_file = ADIOI_Flatlist;
-	while (flat_file->type != fd->filetype) flat_file = flat_file->next;
-	disp = fd->disp;
-
-	if (file_ptr_type == ADIO_INDIVIDUAL) {
-	/* Wei-keng reworked type processing to be a bit more efficient */
-            offset       = fd->fp_ind - disp;
-            n_filetypes  = (offset - flat_file->indices[0]) / filetype_extent;
-            offset      -= (ADIO_Offset)n_filetypes * filetype_extent;
-            /* now offset is local to this extent */
-
-            /* find the block where offset is located, skip blocklens[i]==0 */
-            for (i=0; i<flat_file->count; i++) {
-                ADIO_Offset dist;
-                if (flat_file->blocklens[i] == 0) continue;
-                dist = flat_file->indices[i] + flat_file->blocklens[i] - offset;
-                /* fwr_size is from offset to the end of block i */
-                if (dist == 0) {
-                    i++;
-                    offset   = flat_file->indices[i];
-                    fwr_size = flat_file->blocklens[i];
-                    break;
-                }
-                if (dist > 0) {
-                    fwr_size = dist;
-                    break;
-                }
-            }
-            st_index = i;  /* starting index in flat_file->indices[] */
-            offset += disp + (ADIO_Offset)n_filetypes*filetype_extent;
-        }
-	else {
-	    n_etypes_in_filetype = filetype_size/etype_size;
-	    n_filetypes = offset / n_etypes_in_filetype;
-	    etype_in_filetype = offset % n_etypes_in_filetype;
-	    size_in_filetype = etype_in_filetype * etype_size;
- 
-	    sum = 0;
-	    for (i=0; i<flat_file->count; i++) {
-		sum += flat_file->blocklens[i];
-		if (sum > size_in_filetype) {
-		    st_index = i;
-		    fwr_size = sum - size_in_filetype;
-		    abs_off_in_filetype = flat_file->indices[i] +
-			size_in_filetype - (sum - flat_file->blocklens[i]);
-		    break;
-		}
-	    }
-
-	    /* abs. offset in bytes in the file */
-	    offset = disp + (ADIO_Offset) n_filetypes*filetype_extent + 
-		    abs_off_in_filetype;
-	}
-
-	start_off = offset;
-
-        /* Wei-keng Liao:write request is within single flat_file contig block*/
-	/* this could happen, for example, with subarray types that are
-	 * actually fairly contiguous */
-        if (buftype_is_contig && bufsize <= fwr_size) {
-	    /* though MPI api has an integer 'count' parameter, derived
-	     * datatypes might describe more bytes than can fit into an integer.
-	     * if we've made it this far, we can pass a count of original
-	     * datatypes, instead of a count of bytes (which might overflow)
-	     * Other WriteContig calls in this path are operating on data
-	     * sieving buffer */
-            ADIO_WriteContig(fd, buf, count, datatype, ADIO_EXPLICIT_OFFSET,
-                             offset, status, error_code);
-
-	    if (file_ptr_type == ADIO_INDIVIDUAL) {
-                /* update MPI-IO file pointer to point to the first byte 
-		 * that can be accessed in the fileview. */
-                fd->fp_ind = offset + bufsize;
-                if (bufsize == fwr_size) {
-                    do {
-                        st_index++;
-                        if (st_index == flat_file->count) {
-                            st_index = 0;
-                            n_filetypes++;
-                        }
-                    } while (flat_file->blocklens[st_index] == 0);
-                    fd->fp_ind = disp + flat_file->indices[st_index]
-                               + (ADIO_Offset)n_filetypes*filetype_extent;
-                }
-            }
-	    fd->fp_sys_posn = -1;   /* set it to null. */ 
-#ifdef HAVE_STATUS_SET_BYTES
-	    MPIR_Status_set_bytes(status, datatype, bufsize);
-#endif 
-            goto fn_exit;
-        }
-
-       /* Calculate end_offset, the last byte-offset that will be accessed.
-         e.g., if start_offset=0 and 100 bytes to be write, end_offset=99*/
-
-	st_fwr_size = fwr_size;
-	st_n_filetypes = n_filetypes;
-	i_offset = 0;
-	j = st_index;
-	off = offset;
-	fwr_size = ADIOI_MIN(st_fwr_size, bufsize);
-	while (i_offset < bufsize) {
-	    i_offset += fwr_size;
-	    end_offset = off + fwr_size - 1;
-
-            j = (j+1) % flat_file->count;
-            n_filetypes += (j == 0) ? 1 : 0;
-            while (flat_file->blocklens[j]==0) {
-                j = (j+1) % flat_file->count;
-                n_filetypes += (j == 0) ? 1 : 0;
-            }
-
-	    off = disp + flat_file->indices[j] + 
-		    n_filetypes*(ADIO_Offset)filetype_extent;
-	    fwr_size = ADIOI_MIN(flat_file->blocklens[j], bufsize-i_offset);
-	}
-
-/* if atomicity is true, lock the region to be accessed */
-	if (fd->atomicity) 
-	    ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
-
-        writebuf_off = 0;
-        writebuf_len = 0;
-        writebuf = (char *) ADIOI_Malloc(max_bufsize);
-	memset(writebuf, -1, max_bufsize);
-
-	if (buftype_is_contig && !filetype_is_contig) {
-
-/* contiguous in memory, noncontiguous in file. should be the most
-   common case. */
-
-	    i_offset = 0;
-	    j = st_index;
-	    off = offset;
-	    n_filetypes = st_n_filetypes;
-	    fwr_size = ADIOI_MIN(st_fwr_size, bufsize);
-	    while (i_offset < bufsize) {
-                if (fwr_size) { 
-                    /* TYPE_UB and TYPE_LB can result in 
-                       fwr_size = 0. save system call in such cases */ 
-		    /* lseek(fd->fd_sys, off, SEEK_SET);
-		    err = write(fd->fd_sys, ((char *) buf) + i_offset, fwr_size);*/
-
-		    req_off = off;
-		    req_len = fwr_size;
-		    userbuf_off = i_offset;
-		    ADIOI_BUFFERED_WRITE
-		}
-		i_offset += fwr_size;
-
-		if (off + fwr_size < disp + flat_file->indices[j] +
-	           flat_file->blocklens[j] + n_filetypes*(ADIO_Offset)filetype_extent)
-		       off += fwr_size;
-		/* did not reach end of contiguous block in filetype.
-                   no more I/O needed. off is incremented by fwr_size. */
-		else {
-                    j = (j+1) % flat_file->count;
-                    n_filetypes += (j == 0) ? 1 : 0;
-                    while (flat_file->blocklens[j]==0) {
-                        j = (j+1) % flat_file->count;
-                        n_filetypes += (j == 0) ? 1 : 0;
-                    }
-		    off = disp + flat_file->indices[j] + 
-                                    n_filetypes*(ADIO_Offset)filetype_extent;
-		    fwr_size = ADIOI_MIN(flat_file->blocklens[j], 
-				    bufsize-i_offset);
-		}
-	    }
-	}
-	else {
-/* noncontiguous in memory as well as in file */
-
-	    flat_buf = ADIOI_Flatten_and_find(datatype);
-
-	    k = num = buf_count = 0;
-	    i_offset = flat_buf->indices[0];
-	    j = st_index;
-	    off = offset;
-	    n_filetypes = st_n_filetypes;
-	    fwr_size = st_fwr_size;
-	    bwr_size = flat_buf->blocklens[0];
-
-	    while (num < bufsize) {
-		size = ADIOI_MIN(fwr_size, bwr_size);
-		if (size) {
-		    /* lseek(fd->fd_sys, off, SEEK_SET);
-		    err = write(fd->fd_sys, ((char *) buf) + i_offset, size); */
-
-		    req_off = off;
-		    req_len = size;
-		    userbuf_off = i_offset;
-		    ADIOI_BUFFERED_WRITE
-		}
-
-		new_fwr_size = fwr_size;
-		new_bwr_size = bwr_size;
-
-		if (size == fwr_size) {
-/* reached end of contiguous block in file */
- 		    j = (j+1) % flat_file->count;
- 		    n_filetypes += (j == 0) ? 1 : 0;
- 		    while (flat_file->blocklens[j]==0) {
- 			j = (j+1) % flat_file->count;
- 			n_filetypes += (j == 0) ? 1 : 0;
-		    }
-
-		    off = disp + flat_file->indices[j] + 
-                                      n_filetypes*(ADIO_Offset)filetype_extent;
-
-		    new_fwr_size = flat_file->blocklens[j];
-		    if (size != bwr_size) {
-			i_offset += size;
-			new_bwr_size -= size;
-		    }
-		}
-
-		if (size == bwr_size) {
-/* reached end of contiguous block in memory */
-
-		    k = (k + 1)%flat_buf->count;
-		    buf_count++;
-		    i_offset = (ADIO_Offset)buftype_extent*(ADIO_Offset)(buf_count/flat_buf->count) +
-			flat_buf->indices[k]; 
-		    new_bwr_size = flat_buf->blocklens[k];
-		    if (size != fwr_size) {
-			off += size;
-			new_fwr_size -= size;
-		    }
-		}
-		num += size;
-		fwr_size = new_fwr_size;
-                bwr_size = new_bwr_size;
-	    }
-	}
-
-        /* write the buffer out finally */	
-	if (writebuf_len) {
-	    ADIO_WriteContig(fd, writebuf, writebuf_len, MPI_BYTE, ADIO_EXPLICIT_OFFSET,
-			writebuf_off, &status1, error_code);
-	    if (!(fd->atomicity)) 
-		ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len);
-	    if (*error_code != MPI_SUCCESS) goto fn_exit;
-	}
-	if (fd->atomicity) 
-	    ADIOI_UNLOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
-
-	if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
-    }
-
-    fd->fp_sys_posn = -1;   /* set it to null. */
-
-#ifdef HAVE_STATUS_SET_BYTES
-    /* datatypes returning negagive values, probably related to tt 1893 */
-    MPIR_Status_set_bytes(status, datatype, bufsize);
-/* This is a temporary way of filling in status. The right way is to 
-   keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
-#endif
-
-    if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
-fn_exit:
-    if (writebuf != NULL) ADIOI_Free(writebuf);
-}
-
diff --git a/3rd-party/romio321/adio/common/ad_write_str_naive.c b/3rd-party/romio321/adio/common/ad_write_str_naive.c
deleted file mode 100644
index 4ab4920bc16..00000000000
--- a/3rd-party/romio321/adio/common/ad_write_str_naive.c
+++ /dev/null
@@ -1,379 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-
-void ADIOI_GEN_WriteStrided_naive(ADIO_File fd, const void *buf, int count,
-                       MPI_Datatype buftype, int file_ptr_type,
-                       ADIO_Offset offset, ADIO_Status *status, int
-                       *error_code)
-{
-    /* offset is in units of etype relative to the filetype. */
-
-    ADIOI_Flatlist_node *flat_buf, *flat_file;
-    /* bwr == buffer write; fwr == file write */
-    ADIO_Offset bwr_size, fwr_size=0, sum, size_in_filetype; 
-    int b_index;
-    MPI_Count bufsize;
-    ADIO_Offset n_etypes_in_filetype;
-    ADIO_Offset size, n_filetypes, etype_in_filetype;
-    ADIO_Offset abs_off_in_filetype=0, req_len;
-    MPI_Count filetype_size, etype_size, buftype_size;
-    MPI_Aint filetype_extent, buftype_extent, lb; 
-    int buf_count, buftype_is_contig, filetype_is_contig;
-    ADIO_Offset userbuf_off;
-    ADIO_Offset off, req_off, disp, end_offset=0, start_off;
-    ADIO_Status status1;
-
-    *error_code = MPI_SUCCESS;  /* changed below if error */
-
-    ADIOI_Datatype_iscontig(buftype, &buftype_is_contig);
-    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-
-    MPI_Type_size_x(fd->filetype, &filetype_size);
-    if ( ! filetype_size ) {
-#ifdef HAVE_STATUS_SET_BYTES
-	MPIR_Status_set_bytes(status, buftype, 0);
-#endif
-	*error_code = MPI_SUCCESS; 
-	return;
-    }
-
-    MPI_Type_get_extent(fd->filetype, &lb, &filetype_extent);
-    MPI_Type_size_x(buftype, &buftype_size);
-    MPI_Type_get_extent(buftype, &lb, &buftype_extent);
-    etype_size = fd->etype_size;
-
-    ADIOI_Assert((buftype_size * count) == ((ADIO_Offset)(unsigned)buftype_size * (ADIO_Offset)count));
-    bufsize = buftype_size * count;
-
-    /* contiguous in buftype and filetype is handled elsewhere */
-
-    if (!buftype_is_contig && filetype_is_contig) {
-    	int b_count;
-	/* noncontiguous in memory, contiguous in file. */
-
-	flat_buf = ADIOI_Flatten_and_find(buftype);
-
-        off = (file_ptr_type == ADIO_INDIVIDUAL) ? fd->fp_ind : 
-              fd->disp + (ADIO_Offset)etype_size * offset;
-
-	start_off = off;
-	end_offset = off + bufsize - 1;
-
-	/* if atomicity is true, lock (exclusive) the region to be accessed */
-        if ((fd->atomicity) && ADIO_Feature(fd, ADIO_LOCKS))
-	{
-            ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
-	}
-
-	/* for each region in the buffer, grab the data and put it in
-	 * place
-	 */
-        for (b_count=0; b_count < count; b_count++) {
-            for (b_index=0; b_index < flat_buf->count; b_index++) {
-                userbuf_off = (ADIO_Offset)b_count*(ADIO_Offset)buftype_extent + 
-		              flat_buf->indices[b_index];
-		req_off = off;
-		req_len = flat_buf->blocklens[b_index];
-
-    ADIOI_Assert(req_len == (int) req_len);
-    ADIOI_Assert((((ADIO_Offset)(MPIU_Upint)buf) + userbuf_off) == (ADIO_Offset)(MPIU_Upint)((MPIU_Upint)buf + userbuf_off));
-		ADIO_WriteContig(fd, 
-				(char *) buf + userbuf_off,
-				(int)req_len, 
-				MPI_BYTE, 
-		    		ADIO_EXPLICIT_OFFSET,
-				req_off,
-				&status1,
-				error_code);
-		if (*error_code != MPI_SUCCESS) return;
-
-		/* off is (potentially) used to save the final offset later */
-                off += flat_buf->blocklens[b_index];
-            }
-	}
-
-        if ((fd->atomicity) && ADIO_Feature(fd, ADIO_LOCKS))
-	{
-            ADIOI_UNLOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
-	}
-
-        if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
-
-    }
-
-    else {  /* noncontiguous in file */
-    	int f_index, st_index = 0;
-      ADIO_Offset st_fwr_size, st_n_filetypes;
-	int flag;
-
-        /* First we're going to calculate a set of values for use in all
-	 * the noncontiguous in file cases:
-	 * start_off - starting byte position of data in file
-	 * end_offset - last byte offset to be acessed in the file
-	 * st_n_filetypes - how far into the file we start in terms of
-	 *                  whole filetypes
-	 * st_index - index of block in first filetype that we will be
-	 *            starting in (?)
-	 * st_fwr_size - size of the data in the first filetype block
-	 *               that we will write (accounts for being part-way
-	 *               into writing this block of the filetype
-	 *
-	 */
-
-	/* filetype already flattened in ADIO_Open */
-	flat_file = ADIOI_Flatlist;
-	while (flat_file->type != fd->filetype) flat_file = flat_file->next;
-	disp = fd->disp;
-
-	if (file_ptr_type == ADIO_INDIVIDUAL) {
-	    start_off = fd->fp_ind; /* in bytes */
-	    n_filetypes = -1;
-	    flag = 0;
-	    while (!flag) {
-                n_filetypes++;
-		for (f_index=0; f_index < flat_file->count; f_index++) {
-		    if (disp + flat_file->indices[f_index] + 
-                       n_filetypes*(ADIO_Offset)filetype_extent + 
-		       flat_file->blocklens[f_index] >= start_off) 
-		    {
-		    	/* this block contains our starting position */
-
-			st_index = f_index;
-			fwr_size = disp + flat_file->indices[f_index] + 
-		 	           n_filetypes*(ADIO_Offset)filetype_extent + 
-				   flat_file->blocklens[f_index] - start_off;
-			flag = 1;
-			break;
-		    }
-		}
-	    }
-	}
-	else {
-	    n_etypes_in_filetype = filetype_size/etype_size;
-	    n_filetypes = offset / n_etypes_in_filetype;
-	    etype_in_filetype = offset % n_etypes_in_filetype;
-	    size_in_filetype = etype_in_filetype * etype_size;
- 
-	    sum = 0;
-	    for (f_index=0; f_index < flat_file->count; f_index++) {
-		sum += flat_file->blocklens[f_index];
-		if (sum > size_in_filetype) {
-		    st_index = f_index;
-		    fwr_size = sum - size_in_filetype;
-		    abs_off_in_filetype = flat_file->indices[f_index] +
-			                  size_in_filetype - 
-			                  (sum - flat_file->blocklens[f_index]);
-		    break;
-		}
-	    }
-
-	    /* abs. offset in bytes in the file */
-	    start_off = disp + n_filetypes*(ADIO_Offset)filetype_extent + 
-	    	        abs_off_in_filetype;
-	}
-
-	st_fwr_size = fwr_size;
-	st_n_filetypes = n_filetypes;
-
-	/* start_off, st_n_filetypes, st_index, and st_fwr_size are 
-	 * all calculated at this point
-	 */
-
-        /* Calculate end_offset, the last byte-offset that will be accessed.
-         * e.g., if start_off=0 and 100 bytes to be written, end_offset=99
-	 */
-	userbuf_off = 0;
-	f_index = st_index;
-	off = start_off;
-	fwr_size = ADIOI_MIN(st_fwr_size, bufsize);
-	while (userbuf_off < bufsize) {
-	    userbuf_off += fwr_size;
-	    end_offset = off + fwr_size - 1;
-
-	    if (f_index < (flat_file->count - 1)) f_index++;
-	    else {
-		f_index = 0;
-		n_filetypes++;
-	    }
-
-	    off = disp + flat_file->indices[f_index] + 
-	          n_filetypes*(ADIO_Offset)filetype_extent;
-	    fwr_size = ADIOI_MIN(flat_file->blocklens[f_index], 
-	                         bufsize-(unsigned)userbuf_off);
-	}
-
-	/* End of calculations.  At this point the following values have
-	 * been calculated and are ready for use:
-	 * - start_off
-	 * - end_offset
-	 * - st_n_filetypes
-	 * - st_index
-	 * - st_fwr_size
-	 */
-
-	/* if atomicity is true, lock (exclusive) the region to be accessed */
-        if ((fd->atomicity) && ADIO_Feature(fd, ADIO_LOCKS))
-	{
-            ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
-	}
-
-	if (buftype_is_contig && !filetype_is_contig) {
-	    /* contiguous in memory, noncontiguous in file. should be the
-	     * most common case.
-	     */
-
-	    userbuf_off = 0;
-	    f_index = st_index;
-	    off = start_off;
-	    n_filetypes = st_n_filetypes;
-	    fwr_size = ADIOI_MIN(st_fwr_size, bufsize);
-
-	    /* while there is still space in the buffer, write more data */
-	    while (userbuf_off < bufsize) {
-                if (fwr_size) { 
-                    /* TYPE_UB and TYPE_LB can result in 
-                       fwr_size = 0. save system call in such cases */ 
-		    req_off = off;
-		    req_len = fwr_size;
-
-        ADIOI_Assert(req_len == (int) req_len);
-        ADIOI_Assert((((ADIO_Offset)(MPIU_Upint)buf) + userbuf_off) == (ADIO_Offset)(MPIU_Upint)((MPIU_Upint)buf + userbuf_off));
-		    ADIO_WriteContig(fd, 
-				    (char *) buf + userbuf_off,
-				    (int)req_len, 
-				    MPI_BYTE, 
-				    ADIO_EXPLICIT_OFFSET,
-				    req_off,
-				    &status1,
-				    error_code);
-		    if (*error_code != MPI_SUCCESS) return;
-		}
-		userbuf_off += fwr_size;
-
-                if (off + fwr_size < disp + flat_file->indices[f_index] +
-                   flat_file->blocklens[f_index] + 
-		   n_filetypes*(ADIO_Offset)filetype_extent)
-		{
-		    /* important that this value be correct, as it is
-		     * used to set the offset in the fd near the end of
-		     * this function.
-		     */
-                    off += fwr_size;
-		}
-                /* did not reach end of contiguous block in filetype.
-                 * no more I/O needed. off is incremented by fwr_size.
-		 */
-                else {
-		    if (f_index < (flat_file->count - 1)) f_index++;
-		    else {
-			f_index = 0;
-			n_filetypes++;
-		    }
-		    off = disp + flat_file->indices[f_index] + 
-                          n_filetypes*(ADIO_Offset)filetype_extent;
-		    fwr_size = ADIOI_MIN(flat_file->blocklens[f_index], 
-		                         bufsize-(unsigned)userbuf_off);
-		}
-	    }
-	}
-	else {
-	    ADIO_Offset i_offset, tmp_bufsize = 0;
-	    /* noncontiguous in memory as well as in file */
-
-	    flat_buf = ADIOI_Flatten_and_find(buftype);
-
-	    b_index = buf_count = 0;
-	    i_offset = flat_buf->indices[0];
-	    f_index = st_index;
-	    off = start_off;
-	    n_filetypes = st_n_filetypes;
-	    fwr_size = st_fwr_size;
-	    bwr_size = flat_buf->blocklens[0];
-
-	    /* while we haven't read size * count bytes, keep going */
-	    while (tmp_bufsize < bufsize) {
-    		ADIO_Offset new_bwr_size = bwr_size, new_fwr_size = fwr_size;
-
-		size = ADIOI_MIN(fwr_size, bwr_size);
-		if (size) {
-		    req_off = off;
-		    req_len = size;
-		    userbuf_off = i_offset;
-
-        ADIOI_Assert(req_len == (int) req_len);
-        ADIOI_Assert((((ADIO_Offset)(MPIU_Upint)buf) + userbuf_off) == (ADIO_Offset)(MPIU_Upint)((MPIU_Upint)buf + userbuf_off));
-		    ADIO_WriteContig(fd, 
-				    (char *) buf + userbuf_off,
-				    (int)req_len, 
-				    MPI_BYTE, 
-				    ADIO_EXPLICIT_OFFSET,
-				    req_off,
-				    &status1,
-				    error_code);
-		    if (*error_code != MPI_SUCCESS) return;
-		}
-
-		if (size == fwr_size) {
-		    /* reached end of contiguous block in file */
-		    if (f_index < (flat_file->count - 1)) f_index++;
-		    else {
-			f_index = 0;
-			n_filetypes++;
-		    }
-
-		    off = disp + flat_file->indices[f_index] + 
-                          n_filetypes*(ADIO_Offset)filetype_extent;
-
-		    new_fwr_size = flat_file->blocklens[f_index];
-		    if (size != bwr_size) {
-			i_offset += size;
-			new_bwr_size -= size;
-		    }
-		}
-
-		if (size == bwr_size) {
-		    /* reached end of contiguous block in memory */
-
-		    b_index = (b_index + 1)%flat_buf->count;
-		    buf_count++;
-		    i_offset = (ADIO_Offset)buftype_extent*(ADIO_Offset)(buf_count/flat_buf->count) +
-			flat_buf->indices[b_index];
-		    new_bwr_size = flat_buf->blocklens[b_index];
-		    if (size != fwr_size) {
-			off += size;
-			new_fwr_size -= size;
-		    }
-		}
-		tmp_bufsize += size;
-		fwr_size = new_fwr_size;
-                bwr_size = new_bwr_size;
-	    }
-	}
-
-	/* unlock the file region if we locked it */
-        if ((fd->atomicity) && ADIO_Feature(fd, ADIO_LOCKS))
-	{
-            ADIOI_UNLOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
-	}
-
-	if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
-    } /* end of (else noncontiguous in file) */
-
-    fd->fp_sys_posn = -1;   /* mark it as invalid. */
-
-#ifdef HAVE_STATUS_SET_BYTES
-    MPIR_Status_set_bytes(status, buftype, bufsize);
-    /* This is a temporary way of filling in status. The right way is to 
-     * keep track of how much data was actually written and placed in buf 
-     */
-#endif
-
-    if (!buftype_is_contig) ADIOI_Delete_flattened(buftype);
-}
diff --git a/3rd-party/romio321/adio/common/adi_close.c b/3rd-party/romio321/adio/common/adi_close.c
deleted file mode 100644
index cdc18737850..00000000000
--- a/3rd-party/romio321/adio/common/adi_close.c
+++ /dev/null
@@ -1,45 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-
-
-
-void ADIOI_GEN_Close(ADIO_File fd, int *error_code)
-{
-    int err, derr=0;
-    static char myname[] = "ADIOI_GEN_CLOSE";
-
-#ifdef ADIOI_MPE_LOGGING
-    MPE_Log_event( ADIOI_MPE_close_a, 0, NULL );
-#endif
-    err = close(fd->fd_sys);
-#ifdef ADIOI_MPE_LOGGING
-    MPE_Log_event( ADIOI_MPE_close_b, 0, NULL );
-#endif
-    if (fd->fd_direct >= 0) {
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_close_a, 0, NULL );
-#endif
-	derr = close(fd->fd_direct);
-#ifdef ADIOI_MPE_LOGGING
-        MPE_Log_event( ADIOI_MPE_close_b, 0, NULL );
-#endif
-    }
-
-    fd->fd_sys    = -1;
-    fd->fd_direct = -1;
-
-    if (err == -1 || derr == -1) {
-	*error_code = ADIOI_Err_create_code(myname, fd->filename, errno);
-    }
-    else *error_code = MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/adio/common/async_list.c b/3rd-party/romio321/adio/common/async_list.c
deleted file mode 100644
index 239dbe587ba..00000000000
--- a/3rd-party/romio321/adio/common/async_list.c
+++ /dev/null
@@ -1,170 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-
-ADIOI_Async_node *ADIOI_Malloc_async_node(void)
-{
-/* returns a pointer to a new node that can be added to ADIOI_Async_list.
-   To reduce the number of system calls, mallocs NUM nodes at a time
-   and maintains list of available nodes. Supplies a node from this
-   list if available, else mallocs a new set of NUM and provides one
-   from that set. Is NUM=100 a good number? */
-
-#define NUM 100
-
-    ADIOI_Async_node *curr, *ptr;
-    int i;
-
-    if (!ADIOI_Async_avail_head) {
-	ADIOI_Async_avail_head = (ADIOI_Async_node *)
-	              ADIOI_Malloc(NUM*sizeof(ADIOI_Async_node));  
-	curr = ADIOI_Async_avail_head;
-	for (i=1; i<NUM; i++) {
-	    curr->next = ADIOI_Async_avail_head+i;
-	    curr = curr->next;
-	}
-	curr->next = NULL;
-	ADIOI_Async_avail_tail = curr;
-
-	/* keep track of malloced area that needs to be freed later */
-	if (!ADIOI_Malloc_async_tail) {
-	    ADIOI_Malloc_async_tail = (ADIOI_Malloc_async *)
-		ADIOI_Malloc(sizeof(ADIOI_Malloc_async)); 
-	    ADIOI_Malloc_async_head = ADIOI_Malloc_async_tail;
-	    ADIOI_Malloc_async_head->ptr = ADIOI_Async_avail_head;
-	    ADIOI_Malloc_async_head->next = NULL;
-	}
-	else {
-	    ADIOI_Malloc_async_tail->next = (ADIOI_Malloc_async *)
-		ADIOI_Malloc(sizeof(ADIOI_Malloc_async));
-	    ADIOI_Malloc_async_tail = ADIOI_Malloc_async_tail->next;
-	    ADIOI_Malloc_async_tail->ptr = ADIOI_Async_avail_head;
-	    ADIOI_Malloc_async_tail->next = NULL;
-	}
-    }
-
-    ptr = ADIOI_Async_avail_head;
-    ADIOI_Async_avail_head = ADIOI_Async_avail_head->next;
-    if (!ADIOI_Async_avail_head) ADIOI_Async_avail_tail = NULL;
-
-    return ptr;
-}
-
-
-void ADIOI_Free_async_node(ADIOI_Async_node *node)
-{
-/* moves this node to available pool. does not actually free it. */
-
-    if (!ADIOI_Async_avail_tail)
-	ADIOI_Async_avail_head = ADIOI_Async_avail_tail = node;
-    else {
-	ADIOI_Async_avail_tail->next = node;
-	ADIOI_Async_avail_tail = node;
-    }
-    node->next = NULL;
-}
-
-
-void ADIOI_Add_req_to_list(ADIO_Request *request)
-{
-/* add request to list of outstanding requests */
-
-    ADIOI_Async_node *curr;
-
-    if (!ADIOI_Async_list_head) {
-	ADIOI_Async_list_head = ADIOI_Malloc_async_node();
-	ADIOI_Async_list_head->request = request;
-	ADIOI_Async_list_head->prev = ADIOI_Async_list_head->next = NULL;
-	ADIOI_Async_list_tail = ADIOI_Async_list_head;
-	(*request)->ptr_in_async_list = ADIOI_Async_list_head;
-    }
-    else {
-	curr = ADIOI_Async_list_tail;
-	curr->next = ADIOI_Malloc_async_node();
-	ADIOI_Async_list_tail = curr->next;
-	ADIOI_Async_list_tail->request = request;
-	ADIOI_Async_list_tail->prev = curr;
-	ADIOI_Async_list_tail->next = NULL;
-	(*request)->ptr_in_async_list = ADIOI_Async_list_tail;
-    }
-}
-	
-/* Sets error_code to MPI_SUCCESS on success, creates an error code on
- * failure.
- */
-void ADIOI_Complete_async(int *error_code)
-{
-/* complete all outstanding async I/O operations so that new ones can be
-   initiated. Remove them all from async_list. */
-
-    ADIO_Status status;
-    ADIO_Request *request;
-    ADIOI_Async_node *tmp;
-    static char myname[] = "ADIOI_Complete_async";
-
-    *error_code = MPI_SUCCESS;
-
-    while (ADIOI_Async_list_head) {
-	request = ADIOI_Async_list_head->request;
-	(*request)->queued = -1; /* ugly internal hack that prevents
-                  ADIOI_xxxComplete from freeing the request object. 
-                  This is required, because the user will call MPI_Wait
-                  later, which would require status to be filled. */
-	switch ((*request)->optype) {
-	case ADIOI_READ:
-/*	    (*((*request)->fd->fns->ADIOI_xxx_ReadComplete))(request,
-						    &status,error_code);*/
-	    ADIO_ReadComplete(request, &status, error_code);
-	    break;
-	case ADIOI_WRITE:
-/*	    (*((*request)->fd->fns->ADIOI_xxx_WriteComplete))(request,
-						     &status, error_code);*/
-	    ADIO_WriteComplete(request, &status, error_code);
-	    break;
-	default:
-	    /* --BEGIN ERROR HANDLING-- */
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE,
-					       myname, __LINE__,
-					       MPI_ERR_INTERN,
-					       "Unknown request optype", 0);
-	    return;
-	    /* --END ERROR HANDLING-- */
-	}
-	(*request)->queued = 0;  /* dequeued, but request object not
-				    freed */
-
-	tmp = ADIOI_Async_list_head;
-	ADIOI_Async_list_head = ADIOI_Async_list_head->next;
-	ADIOI_Free_async_node(tmp);
-    }
-    ADIOI_Async_list_tail = NULL;
-}
-
-
-void ADIOI_Del_req_from_list(ADIO_Request *request)
-{
-/* Delete a request that has already been completed from the async
-   list and move it to the list of available nodes. Typically called
-   from within an ADIO_Test/ADIO_Wait. */ 
-
-    ADIOI_Async_node *curr, *prev, *next;
-
-    curr = (*request)->ptr_in_async_list;
-    prev = curr->prev;
-
-    if (prev) prev->next = curr->next;
-    else ADIOI_Async_list_head = curr->next;
-
-    next = curr->next;
-    if (next) next->prev = prev;
-    else ADIOI_Async_list_tail = prev;
-
-    ADIOI_Free_async_node(curr);
-}
diff --git a/3rd-party/romio321/adio/common/byte_offset.c b/3rd-party/romio321/adio/common/byte_offset.c
deleted file mode 100644
index 7c3d1fe1c7d..00000000000
--- a/3rd-party/romio321/adio/common/byte_offset.c
+++ /dev/null
@@ -1,52 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-
-/* Returns the absolute byte position in the file corresponding to 
-   "offset" etypes relative to the current view. */
-
-void ADIOI_Get_byte_offset(ADIO_File fd, ADIO_Offset offset, ADIO_Offset *disp)
-{
-    ADIOI_Flatlist_node *flat_file;
-    int i;
-    ADIO_Offset n_filetypes, etype_in_filetype, sum, abs_off_in_filetype=0, size_in_filetype;
-    MPI_Count n_etypes_in_filetype, filetype_size, etype_size;
-    int filetype_is_contig;
-    MPI_Aint filetype_extent, lb;
-
-    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-    etype_size = fd->etype_size;
-
-    if (filetype_is_contig) *disp = fd->disp + etype_size * offset;
-    else {
-/* filetype already flattened in ADIO_Open */
-        flat_file = ADIOI_Flatlist;
-        while (flat_file->type != fd->filetype) flat_file = flat_file->next;
-
-	MPI_Type_size_x(fd->filetype, &filetype_size);
-	n_etypes_in_filetype = filetype_size/etype_size;
-	n_filetypes = offset / n_etypes_in_filetype;
-	etype_in_filetype = offset % n_etypes_in_filetype;
-	size_in_filetype = etype_in_filetype * etype_size;
- 
-	sum = 0;
-	for (i=0; i<flat_file->count; i++) {
-	    sum += flat_file->blocklens[i];
-	    if (sum > size_in_filetype) {
-		abs_off_in_filetype = flat_file->indices[i] +
-		    size_in_filetype - (sum - flat_file->blocklens[i]);
-		break;
-	    }
-	}
-
-	/* abs. offset in bytes in the file */
-	MPI_Type_get_extent(fd->filetype, &lb, &filetype_extent);
-	*disp = fd->disp + n_filetypes * ADIOI_AINT_CAST_TO_OFFSET filetype_extent + abs_off_in_filetype;
-    }
-}
diff --git a/3rd-party/romio321/adio/common/cb_config_list.c b/3rd-party/romio321/adio/common/cb_config_list.c
deleted file mode 100644
index 1fd827d7d2b..00000000000
--- a/3rd-party/romio321/adio/common/cb_config_list.c
+++ /dev/null
@@ -1,756 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-/* cb_config_list.c
- *
- * The important, externally used functions from this file are:
- * ADIOI_cb_bcast_rank_map()
- * ADIOI_cb_gather_name_array()
- * ADIOI_cb_config_list_parse()
- * ADIOI_cb_copy_name_array()
- * ADIOI_cb_delete_name_array()
- *
- * Prototypes for these are in adio/include/adio_cb_config_list.h
- */
-
-#include "adio.h"
-#include "mpi.h"
-#include "adio_cb_config_list.h"
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-
-/* token types */
-#define AGG_WILDCARD 1
-#define AGG_STRING   2
-#define AGG_COMMA    3
-#define AGG_COLON    4
-#define AGG_ERROR   -1
-#define AGG_EOS      0
-
-#undef CB_CONFIG_LIST_DEBUG
-
-/* a couple of globals keep things simple */
-int ADIOI_cb_config_list_keyval = MPI_KEYVAL_INVALID;
-static char *yylval;
-static char *token_ptr;
-
-/* internal stuff */
-static int get_max_procs(int cb_nodes);
-static int match_procs(char *name, int max_per_proc, char *procnames[], 
-		       char used_procnames[],
-		       int nr_procnames, int ranks[], int nr_ranks, 
-		       int *nr_ranks_allocated);
-static int match_this_proc(char *name, int cur_proc, int max_matches,
-			   char *procnames[], char used_procnames[],
-			   int nr_procnames, int ranks[], 
-			   int nr_ranks, int nr_ranks_allocated);
-static int find_name(char *name, char *procnames[], char used_procnames[], 
-		     int nr_procnames, int start_ind);
-static int cb_config_list_lex(void);
-
-
-/* ADIOI_cb_bcast_rank_map() - broadcast the rank array
- *
- * Parameters:
- * fd - ADIO_File for which update is occurring.  cb_nodes and ranklist
- * parameters must be up-to-date on rank 0 of the fd->comm.
- *
- * should probably be a void fn.
- */
-int ADIOI_cb_bcast_rank_map(ADIO_File fd)
-{
-    int my_rank;
-    char *value;
-	int error_code = MPI_SUCCESS;
-	static char myname[] = "ADIOI_cb_bcast_rank_map";
-    char *p;
-    int i;
-
-    MPI_Bcast(&(fd->hints->cb_nodes), 1, MPI_INT, 0, fd->comm);
-    if (fd->hints->cb_nodes > 0) {
-	MPI_Comm_rank(fd->comm, &my_rank);
-	if (my_rank != 0) {
-	    fd->hints->ranklist = ADIOI_Malloc(fd->hints->cb_nodes*sizeof(int));
-	    if (fd->hints->ranklist == NULL) {
-                error_code = MPIO_Err_create_code(error_code,
-                                                  MPIR_ERR_RECOVERABLE,
-                                                  myname,
-                                                  __LINE__,
-                                                  MPI_ERR_OTHER,
-                                                  "**nomem2",0);
-                return error_code;
-	    }
-	}
-	MPI_Bcast(fd->hints->ranklist, fd->hints->cb_nodes, MPI_INT, 0, 
-		  fd->comm);
-    }
-    /* TEMPORARY -- REMOVE WHEN NO LONGER UPDATING INFO FOR
-     * FS-INDEP. */
-    value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
-    ADIOI_Snprintf(value, MPI_MAX_INFO_VAL+1, "%d", fd->hints->cb_nodes);
-    ADIOI_Info_set(fd->info, "cb_nodes", value);
-    p = value;
-    /* the (by MPI rank) list of aggregators can be larger than
-     * MPI_MAX_INFO_VAL, so we will simply truncate when we reach capacity. I
-     * wasn't clever enough to figure out how to rewind and put '...' at the
-     * end in the truncate case */
-    for (i=0; i< fd->hints->cb_nodes; i++) {
-        int incr, remain = (MPI_MAX_INFO_VAL) - (p-value);
-        incr = ADIOI_Snprintf(p, remain, "%d ", fd->hints->ranklist[i]);
-    if (incr >= remain) break;
-        p += incr;
-    }
-    ADIOI_Info_set(fd->info, "romio_aggregator_list", value);
-    ADIOI_Free(value);
-
-    return 0;
-}
-
-/* ADIOI_cb_gather_name_array() - gather a list of processor names from all processes
- *                          in a communicator and store them on rank 0.
- *
- * This is a collective call on the communicator(s) passed in.
- *
- * Obtains a rank-ordered list of processor names from the processes in
- * "dupcomm".
- *
- * Returns 0 on success, -1 on failure.
- *
- * NOTE: Needs some work to cleanly handle out of memory cases!  
- */
-int ADIOI_cb_gather_name_array(MPI_Comm comm,
-			       MPI_Comm dupcomm,
-			       ADIO_cb_name_array *arrayp)
-{
-    char my_procname[MPI_MAX_PROCESSOR_NAME], **procname = 0;
-    int *procname_len = NULL, my_procname_len, *disp = NULL, i;
-    int commsize, commrank, found;
-    ADIO_cb_name_array array = NULL;
-    int alloc_size;
-
-    if (ADIOI_cb_config_list_keyval == MPI_KEYVAL_INVALID) {
-        /* cleaned up by ADIOI_End_call */
-	MPI_Comm_create_keyval((MPI_Comm_copy_attr_function *) ADIOI_cb_copy_name_array, 
-			  (MPI_Comm_delete_attr_function *) ADIOI_cb_delete_name_array,
-			  &ADIOI_cb_config_list_keyval, NULL);
-    }
-    else {
-	MPI_Comm_get_attr(comm, ADIOI_cb_config_list_keyval, (void *) &array, &found);
-        if (found) {
-            ADIOI_Assert(array != NULL);
-	    *arrayp = array;
-	    return 0;
-	}
-    }
-
-    MPI_Comm_size(dupcomm, &commsize);
-    MPI_Comm_rank(dupcomm, &commrank);
-
-    MPI_Get_processor_name(my_procname, &my_procname_len);
-
-    /* allocate space for everything */
-    array = (ADIO_cb_name_array) ADIOI_Malloc(sizeof(*array));
-    if (array == NULL) {
-	return -1;
-    }
-    array->refct = 2; /* we're going to associate this with two comms */
-
-    if (commrank == 0) {
-	/* process 0 keeps the real list */
-	array->namect = commsize;
-
-	array->names = (char **) ADIOI_Malloc(sizeof(char *) * commsize);
-	if (array->names == NULL) {
-	    return -1;
-	}
-	procname = array->names; /* simpler to read */
-
-	procname_len = (int *) ADIOI_Malloc(commsize * sizeof(int));
-	if (procname_len == NULL) { 
-	    return -1;
-	}
-    }
-    else {
-	/* everyone else just keeps an empty list as a placeholder */
-	array->namect = 0;
-	array->names = NULL;
-    }
-    /* gather lengths first */
-    MPI_Gather(&my_procname_len, 1, MPI_INT, 
-	       procname_len, 1, MPI_INT, 0, dupcomm);
-
-    if (commrank == 0) {
-#ifdef CB_CONFIG_LIST_DEBUG
-	for (i=0; i < commsize; i++) {
-	    FPRINTF(stderr, "len[%d] = %d\n", i, procname_len[i]);
-	}
-#endif
-
-	alloc_size = 0;
-	for (i=0; i < commsize; i++) {
-	    /* add one to the lengths because we need to count the
-	     * terminator, and we are going to use this list of lengths
-	     * again in the gatherv.  
-	     */
-	    alloc_size += ++procname_len[i];
-	}
-	
-	procname[0] = ADIOI_Malloc(alloc_size);
-	if (procname[0] == NULL) {
-	    ADIOI_Free(array);
-	    return -1;
-	}
-
-	for (i=1; i < commsize; i++) {
-	    procname[i] = procname[i-1] + procname_len[i-1];
-	}
-	
-	/* create our list of displacements for the gatherv.  we're going
-	 * to do everything relative to the start of the region allocated
-	 * for procname[0]
-	 */
-	disp = ADIOI_Malloc(commsize * sizeof(int));
-	disp[0] = 0;
-	for (i=1; i < commsize; i++) {
-	    disp[i] = (int) (procname[i] - procname[0]);
-	}
-
-    }
-
-    /* now gather strings */
-    if (commrank == 0) {
-	MPI_Gatherv(my_procname, my_procname_len + 1, MPI_CHAR, 
-		    procname[0], procname_len, disp, MPI_CHAR,
-		    0, dupcomm);
-    }
-    else {
-	/* if we didn't do this, we would need to allocate procname[]
-	 * on all processes...which seems a little silly.
-	 */
-	MPI_Gatherv(my_procname, my_procname_len + 1, MPI_CHAR, 
-		    NULL, NULL, NULL, MPI_CHAR, 0, dupcomm);
-    }
-
-    if (commrank == 0) {
-	/* no longer need the displacements or lengths */
-	ADIOI_Free(disp);
-	ADIOI_Free(procname_len);
-
-#ifdef CB_CONFIG_LIST_DEBUG
-	for (i=0; i < commsize; i++) {
-	    FPRINTF(stderr, "name[%d] = %s\n", i, procname[i]);
-	}
-#endif
-    }
-
-    /* store the attribute; we want to store SOMETHING on all processes
-     * so that they can all tell if we have gone through this procedure 
-     * or not for the given communicator.
-     *
-     * specifically we put it on both the original comm, so we can find
-     * it next time an open is performed on this same comm, and on the
-     * dupcomm, so we can use it in I/O operations.
-     */
-    MPI_Comm_set_attr (comm, ADIOI_cb_config_list_keyval, array);
-    MPI_Comm_set_attr (dupcomm, ADIOI_cb_config_list_keyval, array);
-    *arrayp = array;
-    return 0;
-}
-
-
-/* ADIOI_cb_config_list_parse() - parse the cb_config_list and build the 
- * ranklist
- *
- * Parameters:
- * (pretty self explanatory)
- *
- * Returns number of ranks allocated in parsing, -1 on error.
- */
-int ADIOI_cb_config_list_parse(char *config_list, 
-			 ADIO_cb_name_array array,
-			 int ranklist[], 
-			 int cb_nodes)
-{
-    int token, max_procs, cur_rank = 0, nr_procnames;
-    char *cur_procname, *cur_procname_p, **procnames;
-    char *used_procnames;
-
-    nr_procnames = array->namect;
-    procnames = array->names;
-
-    /* nothing big goes on the stack */
-    /* we use info val here and for yylval because we know the string
-     * cannot be any bigger than this.
-     */
-    cur_procname = ADIOI_Malloc((MPI_MAX_INFO_VAL+1) * sizeof(char));
-    if (cur_procname == NULL) {
-	return -1;
-    }
-
-    yylval = ADIOI_Malloc((MPI_MAX_INFO_VAL+1) * sizeof(char));
-    if (yylval == NULL) {
-	ADIOI_Free(cur_procname);
-	return -1;
-    }
-
-    token_ptr = config_list;
-
-    /* right away let's make sure cb_nodes isn't too big */
-    if (cb_nodes > nr_procnames) cb_nodes = nr_procnames;
-
-    /* used_procnames is used as a mask so that we don't have to destroy
-     * our procnames array
-     */
-    used_procnames = ADIOI_Malloc(array->namect * sizeof(char));
-    if (used_procnames == NULL) {
-	ADIOI_Free(cur_procname);
-	ADIOI_Free(yylval);
-	yylval = NULL;
-	return -1;
-    }
-    memset(used_procnames, 0, array->namect);
-
-    /* optimization for "*:*"; arguably this could be done before we
-     * build the list of processor names...but that would make things
-     * messy.
-     */
-    if (strcmp(config_list, "*:*") == 0) {
-	for (cur_rank = 0; cur_rank < cb_nodes; cur_rank++) {
-	    ranklist[cur_rank] = cur_rank;
-	}
-	ADIOI_Free(cur_procname);
-	ADIOI_Free(yylval);
-	yylval = NULL;
-    	ADIOI_Free(used_procnames);
-	return cb_nodes;
-    }
-
-    while (cur_rank < cb_nodes) {
-	token = cb_config_list_lex();
-
-	if (token == AGG_EOS) {
-	    ADIOI_Free(cur_procname);
-	    ADIOI_Free(yylval);
-	    yylval = NULL;
-    	    ADIOI_Free(used_procnames);
-	    return cur_rank;
-	}
-
-	if (token != AGG_WILDCARD && token != AGG_STRING) {
-	    /* maybe ignore and try to keep going? */
-	    FPRINTF(stderr, "error parsing config list\n");
-	    ADIOI_Free(cur_procname);
-	    ADIOI_Free(yylval);
-	    yylval = NULL;
-    	    ADIOI_Free(used_procnames);
-	    return cur_rank;
-	}
-	
-	if (token == AGG_WILDCARD) {
-	    cur_procname_p = NULL;
-	}
-	else {
-	    /* AGG_STRING is the only remaining case */
-	    /* save procname (for now) */
-	    ADIOI_Strncpy(cur_procname, yylval, MPI_MAX_INFO_VAL+1);
-	    cur_procname_p = cur_procname;
-	}
-
-	/* after we have saved the current procname, we can grab max_procs */
-	max_procs = get_max_procs(cb_nodes);
-
-#ifdef CB_CONFIG_LIST_DEBUG
-	if (token == AGG_WILDCARD) {
-	    FPRINTF(stderr, "looking for *:%d\n", max_procs);
-	}
-	else {
-	    FPRINTF(stderr, "looking for %s:%d\n", cur_procname, max_procs);
-	}
-#endif
-
-	/* do the matching for this piece of the cb_config_list */
-	match_procs(cur_procname_p, max_procs, procnames, used_procnames,
-		    nr_procnames, ranklist, cb_nodes, &cur_rank);
-    }
-    ADIOI_Free(cur_procname);
-    ADIOI_Free(yylval);
-    yylval = NULL;
-    ADIOI_Free(used_procnames);
-    return cur_rank;
-}
-
-/* ADIOI_cb_copy_name_array() - attribute copy routine
- */
-int ADIOI_cb_copy_name_array(MPI_Comm comm, 
-		       int keyval, 
-		       void *extra, 
-		       void *attr_in,
-		       void **attr_out, 
-		       int *flag)
-{
-    ADIO_cb_name_array array;
-
-    ADIOI_UNREFERENCED_ARG(comm);
-    ADIOI_UNREFERENCED_ARG(keyval); 
-    ADIOI_UNREFERENCED_ARG(extra);
-
-    array = (ADIO_cb_name_array) attr_in;
-    if (array != NULL) array->refct++;
-
-    *attr_out = attr_in;
-    *flag = 1; /* make a copy in the new communicator */
-    
-    return MPI_SUCCESS;
-}
-
-/* ADIOI_cb_delete_name_array() - attribute destructor
- */
-int ADIOI_cb_delete_name_array(MPI_Comm comm, 
-			 int keyval, 
-			 void *attr_val, 
-			 void *extra)
-{
-    ADIO_cb_name_array array;
-
-    ADIOI_UNREFERENCED_ARG(comm);
-    ADIOI_UNREFERENCED_ARG(extra);
-
-    array = (ADIO_cb_name_array) attr_val;
-    ADIOI_Assert(array != NULL);
-    array->refct--;
-
-    if (array->refct <= 0) {
-	/* time to free the structures (names, array of ptrs to names, struct)
-	 */
-	if (array->namect) {
-	    /* Note that array->names[i], where i > 0, 
-	     * are just pointers into the allocated region array->names[0]
-	     */
-	    ADIOI_Free(array->names[0]);
-	}
-	if (array->names != NULL) ADIOI_Free(array->names);
-	ADIOI_Free(array);
-    }
-    return MPI_SUCCESS;
-}
-
-/* match_procs() - given a name (or NULL for wildcard) and a max. number
- *                 of aggregator processes (per processor name), this 
- *                 matches in the procnames[] array and puts appropriate
- *                 ranks in the ranks array.
- *
- * Parameters:
- * name - processor name (or NULL for wildcard)
- * max_per_proc - maximum # of processes to use for aggregation from a
- *                single processor
- * procnames - array of processor names
- * nr_procnames - length of procnames array
- * ranks - array of process ranks
- * nr_ranks - length of process ranks array (also max. # of aggregators)
- * nr_ranks_allocated - # of array entries which have been filled in,
- *                      which is also the index to the first empty entry
- *
- * Returns number of matches.
- */
-static int match_procs(char *name, 
-		       int max_per_proc, 
-		       char *procnames[],
-		       char used_procnames[],
-		       int nr_procnames,
-		       int ranks[],
-		       int nr_ranks,
-		       int *nr_ranks_allocated)
-{
-    int wildcard_proc, cur_proc, old_nr_allocated, ret;
-    
-    /* save this so we can report on progress */
-    old_nr_allocated = *nr_ranks_allocated;
-
-    if (name == NULL) {
-	/* wildcard case */
-
-	/* optimize for *:0 case */
-	if (max_per_proc == 0) {
-	    /* loop through procnames and mark them all as used */
-	    for (cur_proc = 0; cur_proc < nr_procnames; cur_proc++) {
-		used_procnames[cur_proc] = 1;
-	    }
-	    return 0;
-	}
-
-	/* the plan here is to start at the beginning of the procnames
-	 * array looking for processor names to apply the wildcard to.
-	 *
-	 * we set wildcard_proc to 0 here but do the search inside the
-	 * while loop so that we aren't starting our search from the
-	 * beginning of the procnames array each time.
-	 */
-	wildcard_proc = 0;
-
-	while (nr_ranks - *nr_ranks_allocated > 0) {
-	    /* find a name */
-	    while ((wildcard_proc < nr_procnames) && 
-		   (used_procnames[wildcard_proc] != 0)) 
-	    {
-		wildcard_proc++;
-	    }
-
-	    if (wildcard_proc == nr_procnames) {
-		/* we have used up the entire procnames list */
-		return *nr_ranks_allocated - old_nr_allocated;
-	    }
-
-#ifdef CB_CONFIG_LIST_DEBUG
-	    FPRINTF(stderr, "performing wildcard match (*:%d) starting with %s (%d)\n", 
-		   max_per_proc, procnames[wildcard_proc], wildcard_proc);
-#endif
-
-	    cur_proc = wildcard_proc;
-
-#ifdef CB_CONFIG_LIST_DEBUG
-	    FPRINTF(stderr, "  assigning name %s (%d) to rank %d in mapping\n",
-		   procnames[cur_proc], cur_proc, *nr_ranks_allocated);
-#endif
-
-	    /* alloc max_per_proc from this host; cur_proc points to
-	     * the first one.  We want to save this name for use in
-	     * our while loop.
-	     */
-	    ranks[*nr_ranks_allocated] = cur_proc;
-	    *nr_ranks_allocated = *nr_ranks_allocated + 1;	    
-	    cur_proc++;
-
-	    /* so, to accomplish this we use the match_this_proc() to
-	     * alloc max_per_proc-1.  we increment cur_proc so that the
-	     * procnames[] entry doesn't get trashed.  then AFTER the call
-	     * we clean up the first instance of the name.
-	     */
-	    ret = match_this_proc(procnames[wildcard_proc],  cur_proc,
-				  max_per_proc-1, procnames, used_procnames,
-				  nr_procnames,
-				  ranks, nr_ranks, *nr_ranks_allocated);
-	    if (ret > 0) *nr_ranks_allocated = *nr_ranks_allocated + ret;
-    
-	    /* clean up and point wildcard_proc to the next entry, since
-	     * we know that this one is NULL now.
-             */
-	    used_procnames[wildcard_proc] = 1;
-	    wildcard_proc++;
-	}
-    }
-    else {
-	/* specific host was specified; this one is easy */
-#ifdef CB_CONFIG_LIST_DEBUG
-	FPRINTF(stderr, "performing name match (%s:%d)\n", name, max_per_proc);
-#endif
-
-	ret = match_this_proc(name, 0, max_per_proc, procnames, used_procnames,
-			      nr_procnames, ranks, nr_ranks,
-			      *nr_ranks_allocated);
-	if (ret > 0) *nr_ranks_allocated = *nr_ranks_allocated + ret;
-    }
-    return *nr_ranks_allocated - old_nr_allocated;
-}
-
-/* match_this_proc() - find each instance of processor name "name" in 
- *                     the "procnames" array, starting with index "cur_proc"
- *                     and add the first "max_matches" into the "ranks"
- *                     array.  remove all instances of "name" from
- *                     "procnames".
- *
- * Parameters:
- * name - processor name to match
- * cur_proc - index into procnames[] at which to start matching
- * procnames - array of processor names
- * used_procnames - array of values indicating if a given procname has
- *                  been allocated or removed already
- * nr_procnames - length of procnames array
- * ranks - array of processor ranks
- * nr_ranks - length of ranks array
- * nr_ranks_allocated - number of ranks already filled in, or the next
- *                      entry to fill in (equivalent)
- *
- * Returns number of ranks filled in (allocated).
- */
-static int match_this_proc(char *name,
-			   int cur_proc,
-			   int max_matches,
-			   char *procnames[],
-			   char used_procnames[],
-			   int nr_procnames, 
-			   int ranks[], 
-			   int nr_ranks,
-			   int nr_ranks_allocated)
-{
-    int ranks_remaining, nr_to_alloc, old_nr_allocated;
-
-    old_nr_allocated = nr_ranks_allocated;
-
-    /* calculate how many ranks we want to allocate */
-    ranks_remaining = nr_ranks - nr_ranks_allocated;
-    nr_to_alloc = (max_matches < ranks_remaining) ? 
-	max_matches : ranks_remaining;
-
-    while (nr_to_alloc > 0) {
-	cur_proc = find_name(name, procnames, used_procnames, nr_procnames, 
-			     cur_proc);
-	if (cur_proc < 0) {
-	    /* didn't find it */
-	    return nr_ranks_allocated - old_nr_allocated;
-	}
-
-	/* need bounds checking on ranks */
-#ifdef CB_CONFIG_LIST_DEBUG
-	FPRINTF(stderr, "  assigning name %s (%d) to rank %d in mapping\n",
-	       procnames[cur_proc], cur_proc, nr_ranks_allocated);
-#endif
-
-	ranks[nr_ranks_allocated] = cur_proc;
-	nr_ranks_allocated++;
-	used_procnames[cur_proc] = 1;
-	    
-	cur_proc++;
-	nr_to_alloc--;
-    }
-	
-    /* take all other instances of this host out of the list */
-    while (cur_proc >= 0) {
-	cur_proc = find_name(name, procnames, used_procnames, nr_procnames, 
-		             cur_proc);
-	if (cur_proc >= 0) {
-#ifdef CB_CONFIG_LIST_DEBUG
-	    FPRINTF(stderr, "  taking name %s (%d) out of procnames\n",
-		   procnames[cur_proc], cur_proc);
-#endif
-	    used_procnames[cur_proc] = 1;
-	    cur_proc++;
-	}
-    }
-    return nr_ranks_allocated - old_nr_allocated;
-}
-  
-
-/* find_name() - finds the first entry in procnames[] which matches name,
- *               starting at index start_ind
- *
- * Returns an index [0..nr_procnames-1] on success, -1 if not found.
- */
-static int find_name(char *name, 
-		     char *procnames[], 
-		     char used_procnames[],
-		     int nr_procnames, 
-		     int start_ind)
-{
-    int i;
-
-    for (i=start_ind; i < nr_procnames; i++) {
-	if (!used_procnames[i] && !strcmp(name, procnames[i])) break;
-    }
-
-    if (i < nr_procnames) return i;
-    else return -1;
-}
-
-/* get_max_procs() - grab the maximum number of processes to use out of
- *                   the cb_config_list string
- *
- * Parameters:
- * cb_nodes - cb_nodes value.  this is returned when a "*" is encountered
- *            as the max_procs value.
- *
- * Returns # of processes, or -1 on error.
- */
-static int get_max_procs(int cb_nodes)
-{
-    int token, max_procs = -1;
-    char *errptr;
-
-    token = cb_config_list_lex();
-
-    switch(token) {
-    case AGG_EOS:
-    case AGG_COMMA:
-	return 1;
-    case AGG_COLON:
-	token = cb_config_list_lex();
-	if (token != AGG_WILDCARD && token != AGG_STRING) return -1;
-	if (token == AGG_WILDCARD) max_procs = cb_nodes;
-	else if (token == AGG_STRING) {
-	    max_procs = (int)strtol(yylval, &errptr, 10);
-	    if (*errptr != '\0') {
-		/* some garbage value; default to 1 */
-		max_procs = 1;
-	    }
-	}
-	/* strip off next comma (if there is one) */
-	token = cb_config_list_lex();
-	if (token != AGG_COMMA && token != AGG_EOS) return -1;
-	
-	/* return max_procs */
-	if (max_procs < 0) return -1;
-	else return max_procs;
-    }
-    return -1;
-}
-
-
-/* cb_config_list_lex() - lexical analyzer for cb_config_list language
- *
- * Returns a token of types defined at top of this file.
- */
-#if defined(BGQPLATFORM)
-/* On BlueGene, the ',' character shows up in get_processor_name, so we have to
- * use a different delimiter */
-#define COLON ':'
-#define COMMA ';'
-#define DELIMS ":;"
-#else 
-/* these tokens work for every other platform */
-#define COLON ':'
-#define COMMA ','
-#define DELIMS ":,"
-#endif
-
-static int cb_config_list_lex(void)
-{
-    int slen;
-
-    if (*token_ptr == '\0') return AGG_EOS;
-
-    slen = (int)strcspn(token_ptr, DELIMS);
-
-    if (*token_ptr == COLON) {
-	token_ptr++;
-	return AGG_COLON;
-    }
-    if (*token_ptr == COMMA) {
-	token_ptr++;
-	return AGG_COMMA;
-    }
-
-    if (*token_ptr == '*') {
-	/* make sure that we don't have characters after the '*' */
-	if (slen == 1) {
-	    token_ptr++;
-	    return AGG_WILDCARD;
-	}
-	else return AGG_ERROR;
-    }
-
-    /* last case: some kind of string.  for now we copy the string. */
-
-    /* it would be a good idea to look at the string and make sure that
-     * it doesn't have any illegal characters in it.  in particular we
-     * should ensure that no one tries to use wildcards with strings 
-     * (e.g. "ccn*").
-     */
-    ADIOI_Strncpy(yylval, token_ptr, slen);
-    yylval[slen] = '\0';
-    token_ptr += slen;
-    return AGG_STRING;
-}
diff --git a/3rd-party/romio321/adio/common/eof_offset.c b/3rd-party/romio321/adio/common/eof_offset.c
deleted file mode 100644
index ea3c51f6eab..00000000000
--- a/3rd-party/romio321/adio/common/eof_offset.c
+++ /dev/null
@@ -1,78 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-
-/* return the current end of file in etype units relative to the 
-   current view */
-
-void ADIOI_Get_eof_offset(ADIO_File fd, ADIO_Offset *eof_offset)
-{
-    MPI_Count filetype_size;
-    int error_code, filetype_is_contig;
-    ADIO_Offset fsize, disp, sum=0, size_in_file, n_filetypes, rem, etype_size;
-    int flag, i;
-    ADIO_Fcntl_t *fcntl_struct;
-    MPI_Aint filetype_extent, lb;
-    ADIOI_Flatlist_node *flat_file;
-
-    /* find the eof in bytes */
-    fcntl_struct = (ADIO_Fcntl_t *) ADIOI_Malloc(sizeof(ADIO_Fcntl_t));
-    ADIO_Fcntl(fd, ADIO_FCNTL_GET_FSIZE, fcntl_struct, &error_code);
-    fsize = fcntl_struct->fsize;
-    ADIOI_Free(fcntl_struct);
-	
-    /* Find the offset in etype units corresponding to eof.
-       The eof could lie in a hole in the current view, or in the 
-       middle of an etype. In that case the offset will be the offset
-       corresponding to the start of the next etype in the current view.*/
-
-    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-    etype_size = fd->etype_size;
-
-    if (filetype_is_contig) 
-	*eof_offset = (fsize - fd->disp + etype_size - 1)/etype_size;
-    /* ceiling division in case fsize is not a multiple of etype_size;*/
-    else {
-	/* filetype already flattened in ADIO_Open */
-	flat_file = ADIOI_Flatlist;
-	while (flat_file->type != fd->filetype) 
-	    flat_file = flat_file->next;
-	
-	MPI_Type_size_x(fd->filetype, &filetype_size);
-	MPI_Type_get_extent(fd->filetype, &lb, &filetype_extent);
-
-	disp = fd->disp;
-	n_filetypes = -1;
-	flag = 0;
-	while (!flag) {
-	    sum = 0;
-	    n_filetypes++;
-	    for (i=0; i<flat_file->count; i++) {
-		sum += flat_file->blocklens[i];
-		if (disp + flat_file->indices[i] + 
-		    n_filetypes* ADIOI_AINT_CAST_TO_OFFSET filetype_extent + 
-		       flat_file->blocklens[i] >= fsize) {
-		    if (disp + flat_file->indices[i] + 
-			   n_filetypes * ADIOI_AINT_CAST_TO_OFFSET filetype_extent >= fsize)
-			sum -= flat_file->blocklens[i];
-		    else {
-			rem = (disp + flat_file->indices[i] + 
-				n_filetypes* ADIOI_AINT_CAST_TO_OFFSET filetype_extent
-				+ flat_file->blocklens[i] - fsize);
-			sum -= rem;
-		    }
-		    flag = 1;
-		    break;
-		}
-	    }
-	}
-	size_in_file = n_filetypes*(ADIO_Offset)filetype_size + sum;
-	*eof_offset = (size_in_file+etype_size-1)/etype_size; /* ceiling division */
-    }
-}
diff --git a/3rd-party/romio321/adio/common/error.c b/3rd-party/romio321/adio/common/error.c
deleted file mode 100644
index 20e211d9fc1..00000000000
--- a/3rd-party/romio321/adio/common/error.c
+++ /dev/null
@@ -1,127 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-
-/* NOTE: THIS FUNCTION IS DEPRECATED AND ONLY EXISTS HERE BECAUSE
- * SOME DEPRECATED ADIO IMPLEMENTATIONS STILL CALL IT (SFS, HFS, PIOFS).
- */
-int ADIOI_Error(ADIO_File fd, int error_code, char *string)
-{
-    char buf[MPI_MAX_ERROR_STRING];
-    int myrank, result_len; 
-    MPI_Errhandler err_handler;
-
-    if (fd == ADIO_FILE_NULL) err_handler = ADIOI_DFLT_ERR_HANDLER;
-    else err_handler = fd->err_handler;
-
-    MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
-    if (err_handler == MPI_ERRORS_ARE_FATAL) {
-	MPI_Error_string(error_code, buf, &result_len);
-	FPRINTF(stderr, "[%d] - %s : %s\n", myrank, string, buf);
-	MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-    else if (err_handler != MPI_ERRORS_RETURN) {
-	/* MPI_File_call_errorhandler(fd, error_code); */
-
-	FPRINTF(stderr, "Only MPI_ERRORS_RETURN and MPI_ERRORS_ARE_FATAL are currently supported as error handlers for files\n");
-	MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    return error_code;
-}
-/* Check for special error codes for those MPI error
-   classes that relate to particular problems.
-   Returns an MPI error code corresponding to "my_errno", for function "myname"
- * and the given file, "filename".  */
-int ADIOI_Err_create_code(const char *myname, const char *filename, int my_errno)
-{
-    int error_code = MPI_SUCCESS;
-    if(!my_errno) return MPI_SUCCESS;
-
-    /* --BEGIN ERROR HANDLING-- */
-    switch(my_errno) {
-        case EACCES:
-            error_code = MPIO_Err_create_code(MPI_SUCCESS,
-                                              MPIR_ERR_RECOVERABLE, myname,
-                                              __LINE__, MPI_ERR_ACCESS,
-                                              "**fileaccess",
-                                              "**fileaccess %s",
-                                              filename );
-            break;
-        case ENAMETOOLONG:
-            error_code = MPIO_Err_create_code(MPI_SUCCESS,
-                                              MPIR_ERR_RECOVERABLE, myname,
-                                              __LINE__, MPI_ERR_BAD_FILE,
-                                              "**filenamelong",
-                                              "**filenamelong %s %d",
-                                              filename,
-                                              strlen(filename));
-            break;
-        case ENOENT:
-            error_code = MPIO_Err_create_code(MPI_SUCCESS,
-                                              MPIR_ERR_RECOVERABLE, myname,
-                                              __LINE__, MPI_ERR_NO_SUCH_FILE,
-                                              "**filenoexist",
-                                              "**filenoexist %s",
-                                              filename);
-            break;
-        case EISDIR:
-            error_code = MPIO_Err_create_code(MPI_SUCCESS,
-                                              MPIR_ERR_RECOVERABLE,
-                                              myname, __LINE__,
-                                              MPI_ERR_BAD_FILE,
-                                              "**filenamedir",
-                                              "**filenamedir %s",
-                                              filename);
-            break;
-        case EROFS:
-	    /* Read only file or file system and write access requested */
-            error_code = MPIO_Err_create_code(MPI_SUCCESS,
-                                              MPIR_ERR_RECOVERABLE, myname,
-                                              __LINE__, MPI_ERR_READ_ONLY,
-                                              "**ioneedrd", 0 );
-            break;
-        case EEXIST:
-            error_code = MPIO_Err_create_code(MPI_SUCCESS,
-                                              MPIR_ERR_RECOVERABLE, myname,
-                                              __LINE__, MPI_ERR_FILE_EXISTS,
-                                              "**fileexist", 0);
-            break;
-	case ENOTDIR:
-	case ELOOP:
-	    error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					       MPIR_ERR_RECOVERABLE,
-					       myname, __LINE__,
-					       MPI_ERR_BAD_FILE,
-					       "**filenamedir",
-					       "**filenamedir %s",
-					       filename);
-            break;
-	case ENOSPC:
-	    error_code = MPIO_Err_create_code(MPI_SUCCESS,
-		    MPIR_ERR_RECOVERABLE, myname, __LINE__,
-		    MPI_ERR_NO_SPACE,
-		    "**filenospace", 0);
-	    break;
-	case EDQUOT:
-	    error_code = MPIO_Err_create_code(MPI_SUCCESS,
-		    MPIR_ERR_RECOVERABLE, myname, __LINE__,
-		    MPI_ERR_QUOTA,
-		    "**filequota", 0);
-	    break;
-        default:
-            error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-                                              myname, __LINE__, MPI_ERR_IO, "**io",
-                                              "**io %s", strerror(my_errno));
-            break;
-    }
-    /* --END ERROR HANDLING-- */
-
-    return error_code;
-}
diff --git a/3rd-party/romio321/adio/common/flatten.c b/3rd-party/romio321/adio/common/flatten.c
deleted file mode 100644
index b468ec2f0f4..00000000000
--- a/3rd-party/romio321/adio/common/flatten.c
+++ /dev/null
@@ -1,1293 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-/* #ifdef MPISGI
-#include "mpisgi2.h"
-#endif */
-
-#ifdef USE_DBG_LOGGING
-  #define FLATTEN_DEBUG 1
-#endif
-
-struct adio_short_int {
-  short elem_s;
-  int elem_i;
-};
-
-struct adio_double_int {
-  double elem_d;
-  int elem_i;
-};
-
-struct adio_long_int {
-  long elem_l;
-  int elem_i;
-};
-
-struct adio_long_double_int {
-  long double elem_ld;
-  int elem_i;
-};
-
-int ADIOI_Type_get_envelope (MPI_Datatype datatype, int *num_integers,
-                             int *num_addresses, int *num_datatypes, int *combiner)
-{
-  int rc, is_contig;
-
-  ADIOI_Datatype_iscontig(datatype, &is_contig);
-
-  rc = MPI_Type_get_envelope (datatype, num_integers, num_addresses, num_datatypes, combiner);
-  if (MPI_SUCCESS != rc || MPI_COMBINER_NAMED != *combiner || is_contig) {
-    return rc;
-  }
-
-  if (MPI_SHORT_INT == datatype || MPI_DOUBLE_INT == datatype || MPI_LONG_DOUBLE_INT == datatype ||
-      MPI_LONG_INT == datatype) {
-      *num_integers = 2;
-      *num_addresses = 2;
-      *num_datatypes = 2;
-      *combiner = MPI_COMBINER_STRUCT;
-  }
-
-  return rc;
-}
-
-int ADIOI_Type_get_contents (MPI_Datatype datatype, int max_integers,
-                             int max_addresses, int max_datatypes, int array_of_integers[],
-                             MPI_Aint array_of_addresses[], MPI_Datatype array_of_datatypes[])
-{
-  int dontcare, combiner;
-  int rc;
-
-  rc = MPI_Type_get_envelope (datatype, &dontcare, &dontcare, &dontcare, &combiner);
-  if (MPI_SUCCESS != rc) {
-    return rc;
-  }
-
-  if (MPI_COMBINER_NAMED != combiner) {
-    return MPI_Type_get_contents (datatype, max_integers, max_addresses, max_datatypes,
-                                  array_of_integers, array_of_addresses, array_of_datatypes);
-  }
-
-  array_of_integers[0] = 1;
-  array_of_integers[1] = 1;
-  array_of_addresses[0] = 0;
-  array_of_datatypes[1] = MPI_INT;
-
-  if (MPI_SHORT_INT == datatype) {
-      array_of_datatypes[0] = MPI_SHORT;
-      array_of_addresses[1] = offsetof (struct adio_short_int, elem_i);
-  } else if (MPI_DOUBLE_INT == datatype) {
-      array_of_datatypes[0] = MPI_DOUBLE;
-      array_of_addresses[1] = offsetof (struct adio_double_int, elem_i);
-  } else if (MPI_LONG_DOUBLE_INT == datatype) {
-      array_of_datatypes[0] = MPI_LONG_DOUBLE;
-      array_of_addresses[1] = offsetof (struct adio_long_double_int, elem_i);
-  } else if (MPI_LONG_INT == datatype) {
-      array_of_datatypes[0] = MPI_LONG;
-      array_of_addresses[1] = offsetof (struct adio_long_int, elem_i);
-  } else {
-    rc = MPI_ERR_TYPE;
-  }
-
-  return rc;
-}
-
-void ADIOI_Optimize_flattened(ADIOI_Flatlist_node *flat_type);
-/* flatten datatype and add it to Flatlist */
-void ADIOI_Flatten_datatype(MPI_Datatype datatype)
-{
-#ifdef HAVE_MPIR_TYPE_FLATTEN
-    MPI_Aint flatten_idx;
-#endif
-    MPI_Count curr_index=0;
-    int is_contig;
-    ADIOI_Flatlist_node *flat, *prev=0;
-
-    /* check if necessary to flatten. */
- 
-    /* is it entirely contiguous? */
-    ADIOI_Datatype_iscontig(datatype, &is_contig);
-  #ifdef FLATTEN_DEBUG 
-  DBG_FPRINTF(stderr,"ADIOI_Flatten_datatype:: is_contig %#X\n",is_contig);
-  #endif
-    if (is_contig) return;
-
-    /* has it already been flattened? */
-    flat = ADIOI_Flatlist;
-    while (flat) {
-	if (flat->type == datatype) {
-      #ifdef FLATTEN_DEBUG 
-      DBG_FPRINTF(stderr,"ADIOI_Flatten_datatype:: found datatype %#X\n", datatype);
-      #endif
-		return;
-	}
-	else {
-	    prev = flat;
-	    flat = flat->next;
-	}
-    }
-
-    /* flatten and add to the list */
-    flat = prev;
-    flat->next = (ADIOI_Flatlist_node *)ADIOI_Malloc(sizeof(ADIOI_Flatlist_node));
-    flat = flat->next;
-
-    flat->type = datatype;
-    flat->next = NULL;
-    flat->blocklens = NULL;
-    flat->indices = NULL;
-    flat->lb_idx = flat->ub_idx = -1;
-
-    flat->count = ADIOI_Count_contiguous_blocks(datatype, &curr_index);
-#ifdef FLATTEN_DEBUG 
-    DBG_FPRINTF(stderr,"ADIOI_Flatten_datatype:: count %llX, cur_idx = %#llX\n",flat->count,curr_index);
-#endif
-/*    DBG_FPRINTF(stderr, "%d\n", flat->count);*/
-
-    if (flat->count) {
-	flat->blocklens = (ADIO_Offset *) ADIOI_Malloc(flat->count * sizeof(ADIO_Offset));
-	flat->indices = (ADIO_Offset *) ADIOI_Malloc(flat->count * sizeof(ADIO_Offset));
-    }
-	
-    curr_index = 0;
-#ifdef HAVE_MPIR_TYPE_FLATTEN
-    flatten_idx = (MPI_Aint) flat->count;
-    MPIR_Type_flatten(datatype, flat->indices, flat->blocklens, &flatten_idx);
-  #ifdef FLATTEN_DEBUG 
-  DBG_FPRINTF(stderr,"ADIOI_Flatten_datatype:: MPIR_Type_flatten\n");
-  #endif
-#else
-    ADIOI_Flatten(datatype, flat, 0, &curr_index);
-  #ifdef FLATTEN_DEBUG 
-  DBG_FPRINTF(stderr,"ADIOI_Flatten_datatype:: ADIOI_Flatten\n");
-  #endif
-
-    ADIOI_Optimize_flattened(flat);
-#endif
-/* debug */
-#ifdef FLATTEN_DEBUG
-    {
-	int i;
-	for (i=0; i<flat->count; i++) 
-      DBG_FPRINTF(stderr,"ADIOI_Flatten_datatype:: i %#X, blocklens %#llX, indices %#llX\n",
-              i,
-              flat->blocklens[i],
-              flat->indices[i]
-             );
-  }
-#endif
-}
-
-/* ADIOI_Flatten()
- *
- * Assumption: input datatype is not a basic!!!!
- */
-void ADIOI_Flatten(MPI_Datatype datatype, ADIOI_Flatlist_node *flat, 
-		  ADIO_Offset st_offset, MPI_Count *curr_index)
-{
-    int k, m, n, is_hindexed_block=0;
-    int lb_updated=0;
-    int combiner, old_combiner, old_is_contig;
-    int nints, nadds, ntypes, old_nints, old_nadds, old_ntypes;
-    /* By using ADIO_Offset we preserve +/- sign and 
-         avoid >2G integer arithmetic problems */
-    ADIO_Offset top_count;
-    MPI_Count i, j, old_size, prev_index, basic_num, num, nonzeroth;
-    MPI_Aint old_extent, lb;/* Assume extents are non-negative */
-    int *ints;
-    MPI_Aint *adds; /* Make no assumptions about +/- sign on these */
-    MPI_Datatype *types;
-    ADIOI_Type_get_envelope(datatype, &nints, &nadds, &ntypes, &combiner);
-    if (combiner == MPI_COMBINER_NAMED) {
-        return;  /* can't do anything else: calling get_contents on a builtin
-                    type is an error */
-    }
-    ints = (int *) ADIOI_Malloc((nints+1)*sizeof(int));
-    adds = (MPI_Aint *) ADIOI_Malloc((nadds+1)*sizeof(MPI_Aint));
-    types = (MPI_Datatype *) ADIOI_Malloc((ntypes+1)*sizeof(MPI_Datatype));
-    ADIOI_Type_get_contents(datatype, nints, nadds, ntypes, ints, adds, types);
-
-  #ifdef FLATTEN_DEBUG 
-  DBG_FPRINTF(stderr,"ADIOI_Flatten:: st_offset %#llX, curr_index %#llX\n",st_offset,*curr_index);
-  DBG_FPRINTF(stderr,"ADIOI_Flatten:: nints %#X, nadds %#X, ntypes %#X\n",nints, nadds, ntypes);
-  for(i=0; i< nints; ++i)
-  {
-    DBG_FPRINTF(stderr,"ADIOI_Flatten:: ints[%lld]=%#X\n",i,ints[i]);
-  }
-  for(i=0; i< nadds; ++i)
-  {
-    DBG_FPRINTF(stderr,"ADIOI_Flatten:: adds[%lld]="MPI_AINT_FMT_HEX_SPEC"\n",i,adds[i]);
-  }
-  for(i=0; i< ntypes; ++i)
-  {
-    DBG_FPRINTF(stderr,"ADIOI_Flatten:: types[%lld]=%#llX\n",i,(unsigned long long)(unsigned long)types[i]);
-  }
-  #endif
-  /* Chapter 4, page 83: when processing datatypes, note this item from the
-   * standard:
-	 Most datatype constructors have replication count or block length
-	 arguments.  Allowed values are non-negative integers. If the value is
-	 zero, no elements are generated in the type map and there is no effect
-	 on datatype bounds or extent.  */
-
-    switch (combiner) {
-#ifdef MPIIMPL_HAVE_MPI_COMBINER_DUP
-    case MPI_COMBINER_DUP:
-    #ifdef FLATTEN_DEBUG 
-    DBG_FPRINTF(stderr,"ADIOI_Flatten:: MPI_COMBINER_DUP\n");
-    #endif
-        ADIOI_Type_get_envelope(types[0], &old_nints, &old_nadds,
-			        &old_ntypes, &old_combiner); 
-        ADIOI_Datatype_iscontig(types[0], &old_is_contig);
-	if ((old_combiner != MPI_COMBINER_NAMED) && (!old_is_contig))
-            ADIOI_Flatten(types[0], flat, st_offset, curr_index);
-        break;
-#endif
-#ifdef MPIIMPL_HAVE_MPI_COMBINER_SUBARRAY
-    case MPI_COMBINER_SUBARRAY:
-        {
-	    int dims = ints[0];
-	    MPI_Datatype stype;
-      #ifdef FLATTEN_DEBUG 
-      DBG_FPRINTF(stderr,"ADIOI_Flatten:: MPI_COMBINER_SUBARRAY\n");
-      #endif
-
-	    ADIO_Type_create_subarray(dims,
-				      &ints[1],        /* sizes */
-				      &ints[dims+1],   /* subsizes */
-				      &ints[2*dims+1], /* starts */
-				      ints[3*dims+1],  /* order */
-				      types[0],        /* type */
-				      &stype);
-	    ADIOI_Flatten(stype, flat, st_offset, curr_index);
-	    MPI_Type_free(&stype);
-	}
-	break;
-#endif
-#ifdef MPIIMPL_HAVE_MPI_COMBINER_DARRAY
-    case MPI_COMBINER_DARRAY:
-	{
-	    int dims = ints[2];
-	    MPI_Datatype dtype;
-      #ifdef FLATTEN_DEBUG 
-      DBG_FPRINTF(stderr,"ADIOI_Flatten:: MPI_COMBINER_DARRAY\n");
-      #endif
-
-	    ADIO_Type_create_darray(ints[0],         /* size */
-				    ints[1],         /* rank */
-				    dims,
-				    &ints[3],        /* gsizes */
-				    &ints[dims+3],   /* distribs */
-				    &ints[2*dims+3], /* dargs */
-				    &ints[3*dims+3], /* psizes */
-				    ints[4*dims+3],  /* order */
-				    types[0],
-				    &dtype);
-      #ifdef FLATTEN_DEBUG 
-      DBG_FPRINTF(stderr,"ADIOI_Flatten:: MPI_COMBINER_DARRAY <ADIOI_Flatten(dtype, flat->indices[%#X] %#llX, flat->blocklens[%#X] %#llX, st_offset %#llX, curr_index %#llX);\n",
-              0, flat->indices[0], 0, flat->blocklens[0], st_offset, *curr_index);
-      #endif
-	    ADIOI_Flatten(dtype, flat, st_offset, curr_index);
-      #ifdef FLATTEN_DEBUG 
-      DBG_FPRINTF(stderr,"ADIOI_Flatten:: MPI_COMBINER_DARRAY >ADIOI_Flatten(dtype, flat->indices[%#X] %#llX, flat->blocklens[%#X] %#llX, st_offset %#llX, curr_index %#llX);\n",
-              0, flat->indices[0], 0, flat->blocklens[0], st_offset, *curr_index);
-      #endif
-	    MPI_Type_free(&dtype);
-	}
-	break;
-#endif
-    case MPI_COMBINER_CONTIGUOUS:
-    #ifdef FLATTEN_DEBUG 
-    DBG_FPRINTF(stderr,"ADIOI_Flatten:: MPI_COMBINER_CONTIGUOUS\n");
-    #endif
-	top_count = ints[0];
-        ADIOI_Type_get_envelope(types[0], &old_nints, &old_nadds,
-			        &old_ntypes, &old_combiner); 
-        ADIOI_Datatype_iscontig(types[0], &old_is_contig);
-
-	prev_index = *curr_index;
-	if ((old_combiner != MPI_COMBINER_NAMED) && (!old_is_contig))
-	    ADIOI_Flatten(types[0], flat, st_offset, curr_index);
-
-	if (prev_index == *curr_index) {
-/* simplest case, made up of basic or contiguous types */
-	    j = *curr_index;
-	    flat->indices[j] = st_offset;
-	    MPI_Type_size_x(types[0], &old_size);
-	    flat->blocklens[j] = top_count * old_size;
-      #ifdef FLATTEN_DEBUG 
-      DBG_FPRINTF(stderr,"ADIOI_Flatten:: simple flat->indices[%#llX] %#llX, flat->blocklens[%#llX] %#llX\n",j, flat->indices[j], j, flat->blocklens[j]);
-      #endif
-	    (*curr_index)++;
-	}
-	else {
-/* made up of noncontiguous derived types */
-	    j = *curr_index;
-	    num = *curr_index - prev_index;
-
-/* The noncontiguous types have to be replicated count times */
-	    MPI_Type_get_extent(types[0], &lb, &old_extent);
-	    for (m=1; m<top_count; m++) {
-		for (i=0; i<num; i++) {
-		    flat->indices[j] = flat->indices[j-num] + ADIOI_AINT_CAST_TO_OFFSET old_extent;
-		    flat->blocklens[j] = flat->blocklens[j-num];
-          #ifdef FLATTEN_DEBUG 
-          DBG_FPRINTF(stderr,"ADIOI_Flatten:: derived flat->indices[%#llX] %#llX, flat->blocklens[%#llX] %#llX\n",j, flat->indices[j], j, flat->blocklens[j]);
-          #endif
-		    j++;
-		}
-	    }
-	    *curr_index = j;
-	}
-	break;
-
-    case MPI_COMBINER_VECTOR: 
-    #ifdef FLATTEN_DEBUG 
-    DBG_FPRINTF(stderr,"ADIOI_Flatten:: MPI_COMBINER_VECTOR\n");
-    #endif
-	top_count = ints[0];
-        ADIOI_Type_get_envelope(types[0], &old_nints, &old_nadds,
-			        &old_ntypes, &old_combiner); 
-        ADIOI_Datatype_iscontig(types[0], &old_is_contig);
-
-	prev_index = *curr_index;
-	if ((old_combiner != MPI_COMBINER_NAMED) && (!old_is_contig))
-	    ADIOI_Flatten(types[0], flat, st_offset, curr_index);
-
-	if (prev_index == *curr_index) {
-/* simplest case, vector of basic or contiguous types */
-    /* By using ADIO_Offset we preserve +/- sign and 
-         avoid >2G integer arithmetic problems */
-    ADIO_Offset blocklength = ints[1], stride = ints[2];
-	    j = *curr_index;
-	    flat->indices[j] = st_offset;
-	    MPI_Type_size_x(types[0], &old_size);
-	    flat->blocklens[j] = blocklength * old_size;
-	    for (i=j+1; i<j+top_count; i++) {
-		flat->indices[i] = flat->indices[i-1] + stride * old_size;
-		flat->blocklens[i] = flat->blocklens[j];
-	    }
-	    *curr_index = i;
-	}
-	else {
-/* vector of noncontiguous derived types */
-    /* By using ADIO_Offset we preserve +/- sign and 
-         avoid >2G integer arithmetic problems */
-    ADIO_Offset blocklength = ints[1], stride = ints[2];
-
-	    j = *curr_index;
-	    num = *curr_index - prev_index;
-
-/* The noncontiguous types have to be replicated blocklen times
-   and then strided. Replicate the first one. */
-	    MPI_Type_get_extent(types[0], &lb, &old_extent);
-	    for (m=1; m<blocklength; m++) {
-		for (i=0; i<num; i++) {
-		    flat->indices[j] = flat->indices[j-num] + ADIOI_AINT_CAST_TO_OFFSET old_extent;
-		    flat->blocklens[j] = flat->blocklens[j-num];
-		    j++;
-		}
-	    }
-	    *curr_index = j;
-
-/* Now repeat with strides. */
-	    num = *curr_index - prev_index;
-	    for (i=1; i<top_count; i++) {
- 		for (m=0; m<num; m++) {
-		   flat->indices[j] =  flat->indices[j-num] + stride * ADIOI_AINT_CAST_TO_OFFSET old_extent;
-		   flat->blocklens[j] = flat->blocklens[j-num];
-		   j++;
-		}
-	    }
-	    *curr_index = j;
-	}
-	break;
-
-    case MPI_COMBINER_HVECTOR: 
-    case MPI_COMBINER_HVECTOR_INTEGER: 
-    #ifdef FLATTEN_DEBUG 
-    DBG_FPRINTF(stderr,"ADIOI_Flatten:: MPI_COMBINER_HVECTOR_INTEGER\n");
-    #endif
-	top_count = ints[0];
-        ADIOI_Type_get_envelope(types[0], &old_nints, &old_nadds,
-			        &old_ntypes, &old_combiner); 
-        ADIOI_Datatype_iscontig(types[0], &old_is_contig);
-
-	prev_index = *curr_index;
-	if ((old_combiner != MPI_COMBINER_NAMED) && (!old_is_contig))
-	    ADIOI_Flatten(types[0], flat, st_offset, curr_index);
-
-	if (prev_index == *curr_index) {
-/* simplest case, vector of basic or contiguous types */
-    /* By using ADIO_Offset we preserve +/- sign and 
-         avoid >2G integer arithmetic problems */
-    ADIO_Offset blocklength = ints[1];
-	    j = *curr_index;
-	    flat->indices[j] = st_offset;
-	    MPI_Type_size_x(types[0], &old_size);
-	    flat->blocklens[j] = blocklength * old_size;
-	    for (i=j+1; i<j+top_count; i++) {
-		flat->indices[i] = flat->indices[i-1] + adds[0];
-		flat->blocklens[i] = flat->blocklens[j];
-	    }
-	    *curr_index = i;
-	}
-	else {
-/* vector of noncontiguous derived types */
-    /* By using ADIO_Offset we preserve +/- sign and 
-         avoid >2G integer arithmetic problems */
-    ADIO_Offset blocklength = ints[1];
-
-	    j = *curr_index;
-	    num = *curr_index - prev_index;
-
-/* The noncontiguous types have to be replicated blocklen times
-   and then strided. Replicate the first one. */
-	    MPI_Type_get_extent(types[0], &lb, &old_extent);
-	    for (m=1; m<blocklength; m++) {
-		for (i=0; i<num; i++) {
-		    flat->indices[j] = flat->indices[j-num] + ADIOI_AINT_CAST_TO_OFFSET old_extent;
-		    flat->blocklens[j] = flat->blocklens[j-num];
-		    j++;
-		}
-	    }
-	    *curr_index = j;
-
-/* Now repeat with strides. */
-	    num = *curr_index - prev_index;
-	    for (i=1; i<top_count; i++) {
- 		for (m=0; m<num; m++) {
-		   flat->indices[j] =  flat->indices[j-num] + adds[0];
-		   flat->blocklens[j] = flat->blocklens[j-num];
-		   j++;
-		}
-	    }
-	    *curr_index = j;
-	}
-	break;
-
-    case MPI_COMBINER_INDEXED: 
-    #ifdef FLATTEN_DEBUG 
-    DBG_FPRINTF(stderr,"ADIOI_Flatten:: MPI_COMBINER_INDEXED\n");
-    #endif
-	top_count = ints[0];
-        ADIOI_Type_get_envelope(types[0], &old_nints, &old_nadds,
-			        &old_ntypes, &old_combiner); 
-        ADIOI_Datatype_iscontig(types[0], &old_is_contig);
-	MPI_Type_get_extent(types[0], &lb, &old_extent);
-
-	prev_index = *curr_index;
-	if ((old_combiner != MPI_COMBINER_NAMED) && (!old_is_contig))
-  {
-    /* By using ADIO_Offset we preserve +/- sign and 
-         avoid >2G integer arithmetic problems */
-    ADIO_Offset stride = ints[top_count+1];
-        ADIOI_Flatten(types[0], flat,
-         st_offset+stride* ADIOI_AINT_CAST_TO_OFFSET old_extent, curr_index);
-  }
-
-	if (prev_index == *curr_index) {
-/* simplest case, indexed type made up of basic or contiguous types */
-	    j = *curr_index;
-	    for (i=j, nonzeroth=i; i<j+top_count; i++) {
-    /* By using ADIO_Offset we preserve +/- sign and 
-         avoid >2G integer arithmetic problems */
-    ADIO_Offset blocklength = ints[1+i-j], stride = ints[top_count+1+i-j];
-		if (blocklength > 0) {
-		    flat->indices[nonzeroth] =
-			st_offset + stride* ADIOI_AINT_CAST_TO_OFFSET old_extent;
-		    flat->blocklens[nonzeroth] =
-			blocklength* ADIOI_AINT_CAST_TO_OFFSET old_extent;
-		    nonzeroth++;
-		} else {
-		    flat->count--; /* don't count/consider any zero-length blocklens */
-		}
-	    }
-	    *curr_index = i;
-	}
-	else {
-/* indexed type made up of noncontiguous derived types */
-
-	    j = *curr_index;
-	    num = *curr_index - prev_index;
-	    basic_num = num;
-
-/* The noncontiguous types have to be replicated blocklens[i] times
-   and then strided. Replicate the first one. */
-	    for (m=1; m<ints[1]; m++) {
-		for (i=0, nonzeroth = j; i<num; i++) {
-		    if (flat->blocklens[j-num] > 0) {
-			flat->indices[nonzeroth] =
-			    flat->indices[nonzeroth-num] + ADIOI_AINT_CAST_TO_OFFSET old_extent;
-			flat->blocklens[nonzeroth] =
-			    flat->blocklens[nonzeroth-num];
-			j++;
-			nonzeroth++;
-		    } else {
-			flat->count --;
-		    }
-		}
-	    }
-	    *curr_index = j;
-
-/* Now repeat with strides. */
-	    for (i=1; i<top_count; i++) {
-		num = *curr_index - prev_index;
-		prev_index = *curr_index;
-		for (m=0, nonzeroth=j; m<basic_num; m++) {
-      /* By using ADIO_Offset we preserve +/- sign and 
-         avoid >2G integer arithmetic problems */
-      ADIO_Offset stride = ints[top_count+1+i]-ints[top_count+i];
-		    if (flat->blocklens[j-num] > 0 ) {
-			flat->indices[nonzeroth] =
-			    flat->indices[j-num] + stride* ADIOI_AINT_CAST_TO_OFFSET old_extent;
-			flat->blocklens[nonzeroth] = flat->blocklens[j-num];
-			j++;
-			nonzeroth++;
-		    } else {
-			flat->count--;
-		    }
-		}
-		*curr_index = j;
-		for (m=1; m<ints[1+i]; m++) {
-                    for (k=0, nonzeroth=j; k<basic_num; k++) {
-			if (flat->blocklens[j-basic_num] > 0) {
-			    flat->indices[nonzeroth] =
-				flat->indices[j-basic_num] + ADIOI_AINT_CAST_TO_OFFSET old_extent;
-			    flat->blocklens[nonzeroth] = flat->blocklens[j-basic_num];
-			    j++;
-			    nonzeroth++;
-			} else {
-			    flat->count --;
-			}
-                    }
-                }
-		*curr_index = j;
-	    }
-	}
-	break;
-
-#if defined HAVE_DECL_MPI_COMBINER_HINDEXED_BLOCK && HAVE_DECL_MPI_COMBINER_HINDEXED_BLOCK
-    case MPI_COMBINER_HINDEXED_BLOCK:
-	is_hindexed_block=1;
-	/* deliberate fall-through */
-#endif
-    case MPI_COMBINER_INDEXED_BLOCK:
-    #ifdef FLATTEN_DEBUG 
-    DBG_FPRINTF(stderr,"ADIOI_Flatten:: MPI_COMBINER_INDEXED_BLOCK\n");
-    #endif
-	top_count = ints[0];
-        ADIOI_Type_get_envelope(types[0], &old_nints, &old_nadds,
-			        &old_ntypes, &old_combiner); 
-        ADIOI_Datatype_iscontig(types[0], &old_is_contig);
-	MPI_Type_get_extent(types[0], &lb, &old_extent);
-
-	prev_index = *curr_index;
-	if ((old_combiner != MPI_COMBINER_NAMED) && (!old_is_contig))
-  {
-      /* By using ADIO_Offset we preserve +/- sign and 
-         avoid >2G integer arithmetic problems */
-      ADIO_Offset stride = ints[1+1];
-	if (is_hindexed_block) {
-	    ADIOI_Flatten(types[0], flat,
-		    st_offset+adds[0], curr_index);
-	} else {
-	    ADIOI_Flatten(types[0], flat,
-		    st_offset+stride* ADIOI_AINT_CAST_TO_OFFSET old_extent, curr_index);
-	}
-  }
-
-	if (prev_index == *curr_index) {
-/* simplest case, indexed type made up of basic or contiguous types */
-	    j = *curr_index;
-	    for (i=j; i<j+top_count; i++) {
-      /* By using ADIO_Offset we preserve +/- sign and 
-         avoid >2G integer arithmetic problems */
-		ADIO_Offset blocklength = ints[1];
-		if (is_hindexed_block) {
-		    flat->indices[i] = st_offset + adds[i-j];
-		} else {
-		    ADIO_Offset stride = ints[1+1+i-j];
-		    flat->indices[i] = st_offset +
-			stride* ADIOI_AINT_CAST_TO_OFFSET old_extent;
-		}
-		flat->blocklens[i] = blocklength* ADIOI_AINT_CAST_TO_OFFSET old_extent;
-	    }
-	    *curr_index = i;
-	}
-	else {
-/* vector of noncontiguous derived types */
-
-	    j = *curr_index;
-	    num = *curr_index - prev_index;
-
-/* The noncontiguous types have to be replicated blocklens[i] times
-   and then strided. Replicate the first one. */
-	    for (m=1; m<ints[1]; m++) {
-		for (i=0; i<num; i++) {
-		    if (is_hindexed_block) {
-			/* this is the one place the hindexed case uses the
-			 * extent of a type */
-			MPI_Type_get_extent(types[0], &lb, &old_extent);
-		    }
-		    flat->indices[j] = flat->indices[j-num] +
-			ADIOI_AINT_CAST_TO_OFFSET old_extent;
-		    flat->blocklens[j] = flat->blocklens[j-num];
-		    j++;
-		}
-	    }
-	    *curr_index = j;
-
-/* Now repeat with strides. */
-	    num = *curr_index - prev_index;
-	    for (i=1; i<top_count; i++) {
-		for (m=0; m<num; m++) {
-		    if (is_hindexed_block) {
-			flat->indices[j] = flat->indices[j-num] +
-			    adds[i] - adds[i-1];
-		    } else {
-			/* By using ADIO_Offset we preserve +/- sign and
-			   avoid >2G integer arithmetic problems */
-			ADIO_Offset stride = ints[2+i]-ints[1+i];
-			flat->indices[j] = flat->indices[j-num] +
-			    stride* ADIOI_AINT_CAST_TO_OFFSET old_extent;
-		    }
-		    flat->blocklens[j] = flat->blocklens[j-num];
-		    j++;
-		}
-	    }
-	    *curr_index = j;
-	}
-	break;
-
-    case MPI_COMBINER_HINDEXED: 
-    case MPI_COMBINER_HINDEXED_INTEGER:
-    #ifdef FLATTEN_DEBUG 
-    DBG_FPRINTF(stderr,"ADIOI_Flatten:: MPI_COMBINER_HINDEXED_INTEGER\n");
-    #endif
-	top_count = ints[0];
-        ADIOI_Type_get_envelope(types[0], &old_nints, &old_nadds,
-			        &old_ntypes, &old_combiner); 
-        ADIOI_Datatype_iscontig(types[0], &old_is_contig);
-
-	prev_index = *curr_index;
-	if ((old_combiner != MPI_COMBINER_NAMED) && (!old_is_contig))
-  {
-        ADIOI_Flatten(types[0], flat, st_offset+adds[0], curr_index); 
-  }
-
-	if (prev_index == *curr_index) {
-/* simplest case, indexed type made up of basic or contiguous types */
-	    j = *curr_index;
-	    MPI_Type_size_x(types[0], &old_size);
-	    for (i=j, nonzeroth=j; i<j+top_count; i++) {
-		if (ints[1+i-j] > 0) {
-		    /* By using ADIO_Offset we preserve +/- sign and
-		       avoid >2G integer arithmetic problems */
-		    ADIO_Offset blocklength = ints[1+i-j];
-		    flat->indices[nonzeroth] = st_offset + adds[i-j];
-		    flat->blocklens[nonzeroth] = blocklength*old_size;
-		    nonzeroth++;
-		} else {
-		    flat->count--;
-		}
-	    }
-	    *curr_index = i;
-	}
-	else {
-/* indexed type made up of noncontiguous derived types */
-
-	    j = *curr_index;
-	    num = *curr_index - prev_index;
-	    basic_num = num;
-
-/* The noncontiguous types have to be replicated blocklens[i] times
-   and then strided. Replicate the first one. */
-	    MPI_Type_get_extent(types[0], &lb, &old_extent);
-	    for (m=1; m<ints[1]; m++) {
-		for (i=0, nonzeroth=j; i<num; i++) {
-		    if (flat->blocklens[j-num] > 0) {
-			flat->indices[nonzeroth] =
-			    flat->indices[j-num] + ADIOI_AINT_CAST_TO_OFFSET old_extent;
-			flat->blocklens[nonzeroth] = flat->blocklens[j-num];
-			j++;
-			nonzeroth++;
-		    } else {
-			flat->count--;
-		    }
-		}
-	    }
-	    *curr_index = j;
-
-/* Now repeat with strides. */
-	    for (i=1; i<top_count; i++) {
-		num = *curr_index - prev_index;
-		prev_index = *curr_index;
-		for (m=0, nonzeroth=j; m<basic_num; m++) {
-		    if (flat->blocklens[j-num] > 0) {
-			flat->indices[nonzeroth] =
-			    flat->indices[j-num] + adds[i] - adds[i-1];
-			flat->blocklens[nonzeroth] = flat->blocklens[j-num];
-			j++;
-			nonzeroth++;
-		    } else {
-			flat->count--;
-		    }
-		}
-		*curr_index = j;
-		for (m=1; m<ints[1+i]; m++) {
-		    for (k=0,nonzeroth=j; k<basic_num; k++) {
-			if (flat->blocklens[j-basic_num] >0) {
-			    flat->indices[nonzeroth] =
-				flat->indices[j-basic_num] + ADIOI_AINT_CAST_TO_OFFSET old_extent;
-			    flat->blocklens[nonzeroth] = flat->blocklens[j-basic_num];
-			    j++;
-			    nonzeroth++;
-			}
-		    }
-		}
-		*curr_index = j;
-	    }
-	}
-	break;
-
-    case MPI_COMBINER_STRUCT: 
-    case MPI_COMBINER_STRUCT_INTEGER: 
-    #ifdef FLATTEN_DEBUG 
-    DBG_FPRINTF(stderr,"ADIOI_Flatten:: MPI_COMBINER_STRUCT_INTEGER\n");
-    #endif
-	top_count = ints[0];
-	for (n=0; n<top_count; n++) {
-	    ADIOI_Type_get_envelope(types[n], &old_nints, &old_nadds,
-				    &old_ntypes, &old_combiner); 
-            ADIOI_Datatype_iscontig(types[n], &old_is_contig);
-
-	    prev_index = *curr_index;
-            if ((old_combiner != MPI_COMBINER_NAMED) && (!old_is_contig))
-		ADIOI_Flatten(types[n], flat, st_offset+adds[n], curr_index);
-
-	    if (prev_index == *curr_index) {
-/* simplest case, current type is basic or contiguous types */
-        /* By using ADIO_Offset we preserve +/- sign and 
-           avoid >2G integer arithmetic problems */
-		if (ints[1+n] > 0) {
-		    ADIO_Offset blocklength = ints[1+n];
-		    j = *curr_index;
-		    flat->indices[j] = st_offset + adds[n];
-		    MPI_Type_size_x(types[n], &old_size);
-		    flat->blocklens[j] = blocklength * old_size;
-#ifdef FLATTEN_DEBUG
-		    DBG_FPRINTF(stderr,"ADIOI_Flatten:: simple adds[%#X] "MPI_AINT_FMT_HEX_SPEC", flat->indices[%#llX] %#llX, flat->blocklens[%#llX] %#llX\n",n,adds[n],j, flat->indices[j], j, flat->blocklens[j]);
-#endif
-		    (*curr_index)++;
-		}
-	    }
-	    else {
-/* current type made up of noncontiguous derived types */
-
-		j = *curr_index;
-		num = *curr_index - prev_index;
-
-/* The current type has to be replicated blocklens[n] times */
-		MPI_Type_get_extent(types[n], &lb, &old_extent);
-		for (m=1; m<ints[1+n]; m++) {
-		    for (i=0; i<num; i++) {
-			flat->indices[j] =
-			    flat->indices[j-num] + ADIOI_AINT_CAST_TO_OFFSET old_extent;
-			flat->blocklens[j] = flat->blocklens[j-num];
-#ifdef FLATTEN_DEBUG
-			DBG_FPRINTF(stderr,"ADIOI_Flatten:: simple old_extent "MPI_AINT_FMT_HEX_SPEC", flat->indices[%#llX] %#llX, flat->blocklens[%#llX] %#llX\n",old_extent,j, flat->indices[j], j, flat->blocklens[j]);
-#endif
-			j++;
-		    }
-		}
-		*curr_index = j;
-	    }
-	}
- 	break;
-
-    case MPI_COMBINER_RESIZED: 
-    #ifdef FLATTEN_DEBUG 
-    DBG_FPRINTF(stderr,"ADIOI_Flatten:: MPI_COMBINER_RESIZED\n");
-    #endif
-
-    /* This is done similar to a type_struct with an lb, datatype, ub */
-
-    /* handle the Lb */
-	j = *curr_index;
-	/* when we process resized types, we (recursively) process the lower
-	 * bound, the type being resized, then the upper bound.  In the
-	 * resized-of-resized case, we might find ourselves updating the upper
-	 * bound based on the inner type, but the lower bound based on the
-	 * upper type.  check both lb and ub to prevent mixing updates */
-	if (flat->lb_idx == -1 && flat->ub_idx == -1) {
-	    flat->indices[j] = st_offset + adds[0];
-	    /* this zero-length blocklens[] element, unlike eleswhere in the
-	     * flattening code, is correct and is used to indicate a lower bound
-	     * marker */
-	    flat->blocklens[j] = 0;
-	    flat->lb_idx = *curr_index;
-	    lb_updated=1;
-
-	    #ifdef FLATTEN_DEBUG
-	    DBG_FPRINTF(stderr,"ADIOI_Flatten:: simple adds[%#X] "MPI_AINT_FMT_HEX_SPEC", flat->indices[%#llX] %#llX, flat->blocklens[%#llX] %#llX\n",0,adds[0],j, flat->indices[j], j, flat->blocklens[j]);
-	    #endif
-
-	    (*curr_index)++;
-	} else {
-	    /* skipped over this chunk because something else higher-up in the
-	     * type construction set this for us already */
-	    flat->count--;
-	    st_offset -= adds[0];
-	}
-
-	/* handle the datatype */
-
-	ADIOI_Type_get_envelope(types[0], &old_nints, &old_nadds,
-			        &old_ntypes, &old_combiner); 
-	ADIOI_Datatype_iscontig(types[0], &old_is_contig);
-
-	if ((old_combiner != MPI_COMBINER_NAMED) && (!old_is_contig)) {
-	    ADIOI_Flatten(types[0], flat, st_offset+adds[0], curr_index);
-	}
-	else {
-            /* current type is basic or contiguous */
-	    j = *curr_index;
-	    flat->indices[j] = st_offset;
-	    MPI_Type_size_x(types[0], &old_size);
-	    flat->blocklens[j] = old_size;
-
-            #ifdef FLATTEN_DEBUG 
-	    DBG_FPRINTF(stderr,"ADIOI_Flatten:: simple adds[%#X] "MPI_AINT_FMT_HEX_SPEC", flat->indices[%#llX] %#llX, flat->blocklens[%#llX] %#llX\n",0,adds[0],j, flat->indices[j], j, flat->blocklens[j]);
-            #endif
-
-	    (*curr_index)++;
-	}
-
-	/* take care of the extent as a UB */
-	/* see note above about mixing updates for why we check lb and ub */
-	if ((flat->lb_idx == -1 && flat->ub_idx == -1) || lb_updated) {
-	    j = *curr_index;
-	    flat->indices[j] = st_offset + adds[0] + adds[1];
-	    /* again, zero-element ok: an upper-bound marker explicitly set by the
-	     * constructor of this resized type */
-	    flat->blocklens[j] = 0;
-	    flat->ub_idx = *curr_index;
-	} else {
-	    /* skipped over this chunk because something else higher-up in the
-	     * type construction set this for us already */
-	    flat->count--;
-	    (*curr_index)--;
-	}
-
-        #ifdef FLATTEN_DEBUG 
-        DBG_FPRINTF(stderr,"ADIOI_Flatten:: simple adds[%#X] "MPI_AINT_FMT_HEX_SPEC", flat->indices[%#llX] %#llX, flat->blocklens[%#llX] %#llX\n",1,adds[1],j, flat->indices[j], j, flat->blocklens[j]);
-        #endif
-
-	(*curr_index)++;
-
- 	break;
-
-    default:
-	/* TODO: FIXME (requires changing prototypes to return errors...) */
-	DBG_FPRINTF(stderr, "Error: Unsupported datatype passed to ADIOI_Flatten\n");
-	MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-#ifndef MPISGI
-/* There is a bug in SGI's impl. of MPI_Type_get_contents. It doesn't
-   return new datatypes. Therefore no need to free. */
-    for (i=0; i<ntypes; i++) {
- 	MPI_Type_get_envelope(types[i], &old_nints, &old_nadds, &old_ntypes,
- 			      &old_combiner);
- 	if (old_combiner != MPI_COMBINER_NAMED) MPI_Type_free(types+i);
-    }
-#endif
-
-    ADIOI_Free(ints);
-    ADIOI_Free(adds);
-    ADIOI_Free(types);
-
-  #ifdef FLATTEN_DEBUG 
-  DBG_FPRINTF(stderr,"ADIOI_Flatten:: return st_offset %#llX, curr_index %#llX\n",st_offset,*curr_index);
-  #endif
-
-}
-
-/********************************************************/
-
-/* ADIOI_Count_contiguous_blocks
- *
- * Returns number of contiguous blocks in type, and also updates
- * curr_index to reflect the space for the additional blocks.
- *
- * ASSUMES THAT TYPE IS NOT A BASIC!!!
- */
-MPI_Count ADIOI_Count_contiguous_blocks(MPI_Datatype datatype, MPI_Count *curr_index)
-{
-    int i, n;
-    MPI_Count count=0, prev_index, num, basic_num;
-    int top_count, combiner, old_combiner, old_is_contig;
-    int nints, nadds, ntypes, old_nints, old_nadds, old_ntypes;
-    int *ints;
-    MPI_Aint *adds; /* Make no assumptions about +/- sign on these */
-    MPI_Datatype *types;
-
-    ADIOI_Type_get_envelope(datatype, &nints, &nadds, &ntypes, &combiner);
-    if (combiner == MPI_COMBINER_NAMED) {
-        return 1;  /* builtin types not supposed to be passed to this routine
-                    */
-    }
-    ints = (int *) ADIOI_Malloc((nints+1)*sizeof(int));
-    adds = (MPI_Aint *) ADIOI_Malloc((nadds+1)*sizeof(MPI_Aint));
-    types = (MPI_Datatype *) ADIOI_Malloc((ntypes+1)*sizeof(MPI_Datatype));
-    MPI_Type_get_contents(datatype, nints, nadds, ntypes, ints, adds, types);
-
-    switch (combiner) {
-#ifdef MPIIMPL_HAVE_MPI_COMBINER_DUP
-    case MPI_COMBINER_DUP:
-        ADIOI_Type_get_envelope(types[0], &old_nints, &old_nadds,
-                                &old_ntypes, &old_combiner); 
-	ADIOI_Datatype_iscontig(types[0], &old_is_contig);
-	if ((old_combiner != MPI_COMBINER_NAMED) && (!old_is_contig))
-	    count = ADIOI_Count_contiguous_blocks(types[0], curr_index);
-	else {
-		count = 1;
-		(*curr_index)++;
-	}
-        break;
-#endif
-#ifdef MPIIMPL_HAVE_MPI_COMBINER_SUBARRAY
-    case MPI_COMBINER_SUBARRAY:
-        {
-	    int dims = ints[0];
-	    MPI_Datatype stype;
-
-	    ADIO_Type_create_subarray(dims,
-				      &ints[1],        /* sizes */
-				      &ints[dims+1],   /* subsizes */
-				      &ints[2*dims+1], /* starts */
-				      ints[3*dims+1],  /* order */
-				      types[0],        /* type */
-				      &stype);
-	    count = ADIOI_Count_contiguous_blocks(stype, curr_index);
-	    /* curr_index will have already been updated; just pass
-	     * count back up.
-	     */
-	    MPI_Type_free(&stype);
-
-	}
-	break;
-#endif
-#ifdef MPIIMPL_HAVE_MPI_COMBINER_DARRAY
-    case MPI_COMBINER_DARRAY:
-	{
-	    int dims = ints[2];
-	    MPI_Datatype dtype;
-
-	    ADIO_Type_create_darray(ints[0],         /* size */
-				    ints[1],         /* rank */
-				    dims,
-				    &ints[3],        /* gsizes */
-				    &ints[dims+3],   /* distribs */
-				    &ints[2*dims+3], /* dargs */
-				    &ints[3*dims+3], /* psizes */
-				    ints[4*dims+3],  /* order */
-				    types[0],
-				    &dtype);
-	    count = ADIOI_Count_contiguous_blocks(dtype, curr_index);
-	    /* curr_index will have already been updated; just pass
-	     * count back up.
-	     */
-	    MPI_Type_free(&dtype);
-	}
-	break;
-#endif
-    case MPI_COMBINER_CONTIGUOUS:
-        top_count = ints[0];
-        ADIOI_Type_get_envelope(types[0], &old_nints, &old_nadds,
-                                &old_ntypes, &old_combiner); 
-	ADIOI_Datatype_iscontig(types[0], &old_is_contig);
-
-	prev_index = *curr_index;
-	if ((old_combiner != MPI_COMBINER_NAMED) && (!old_is_contig))
-	    count = ADIOI_Count_contiguous_blocks(types[0], curr_index);
-	else count = 1;
-
-	if (prev_index == *curr_index) 
-/* simplest case, made up of basic or contiguous types */
-	    (*curr_index)++;
-	else {
-/* made up of noncontiguous derived types */
-	    num = *curr_index - prev_index;
-	    count *= top_count;
-	    *curr_index += (top_count - 1)*num;
-	}
-	break;
-
-    case MPI_COMBINER_VECTOR:
-    case MPI_COMBINER_HVECTOR:
-    case MPI_COMBINER_HVECTOR_INTEGER: 
-        top_count = ints[0];
-        ADIOI_Type_get_envelope(types[0], &old_nints, &old_nadds,
-                                &old_ntypes, &old_combiner); 
-	ADIOI_Datatype_iscontig(types[0], &old_is_contig);
-
-	prev_index = *curr_index;
-	if ((old_combiner != MPI_COMBINER_NAMED) && (!old_is_contig))
-	    count = ADIOI_Count_contiguous_blocks(types[0], curr_index);
-	else count = 1;
-
-	if (prev_index == *curr_index) {
-/* simplest case, vector of basic or contiguous types */
-	    count = top_count;
-	    *curr_index += count;
-	}
-	else {
-/* vector of noncontiguous derived types */
-	    num = *curr_index - prev_index;
-
-/* The noncontiguous types have to be replicated blocklen times
-   and then strided. */
-	    count *= ints[1] * top_count;
-
-/* First one */
-	    *curr_index += (ints[1] - 1)*num;
-
-/* Now repeat with strides. */
-	    num = *curr_index - prev_index;
-	    *curr_index += (top_count - 1)*num;
-	}
-	break;
-
-    case MPI_COMBINER_INDEXED: 
-    case MPI_COMBINER_HINDEXED:
-    case MPI_COMBINER_HINDEXED_INTEGER:
-        top_count = ints[0];
-        ADIOI_Type_get_envelope(types[0], &old_nints, &old_nadds,
-                                &old_ntypes, &old_combiner); 
-	ADIOI_Datatype_iscontig(types[0], &old_is_contig);
-
-	prev_index = *curr_index;
-	if ((old_combiner != MPI_COMBINER_NAMED) && (!old_is_contig))
-	    count = ADIOI_Count_contiguous_blocks(types[0], curr_index);
-	else count = 1;
-
-	if (prev_index == *curr_index) {
-/* simplest case, indexed type made up of basic or contiguous types */
-	    count = top_count;
-	    *curr_index += count;
-	}
-	else {
-/* indexed type made up of noncontiguous derived types */
-	    basic_num = *curr_index - prev_index;
-
-/* The noncontiguous types have to be replicated blocklens[i] times
-   and then strided. */
-	    *curr_index += (ints[1]-1) * basic_num;
-	    count *= ints[1];
-
-/* Now repeat with strides. */
-	    for (i=1; i<top_count; i++) {
-		count += ints[1+i] * basic_num;
-		*curr_index += ints[1+i] * basic_num;
-	    }
-	}
-	break;
-
-#if defined HAVE_DECL_MPI_COMBINER_HINDEXED_BLOCK && HAVE_DECL_MPI_COMBINER_HINDEXED_BLOCK
-    case MPI_COMBINER_HINDEXED_BLOCK:
-#endif
-    case MPI_COMBINER_INDEXED_BLOCK:
-        top_count = ints[0];
-        ADIOI_Type_get_envelope(types[0], &old_nints, &old_nadds,
-                                &old_ntypes, &old_combiner); 
-	ADIOI_Datatype_iscontig(types[0], &old_is_contig);
-
-	prev_index = *curr_index;
-	if ((old_combiner != MPI_COMBINER_NAMED) && (!old_is_contig))
-	    count = ADIOI_Count_contiguous_blocks(types[0], curr_index);
-	else count = 1;
-
-	if (prev_index == *curr_index) {
-/* simplest case, indexed type made up of basic or contiguous types */
-	    count = top_count;
-	    *curr_index += count;
-	}
-	else {
-/* indexed type made up of noncontiguous derived types */
-	    basic_num = *curr_index - prev_index;
-
-/* The noncontiguous types have to be replicated blocklens[i] times
-   and then strided. */
-	    *curr_index += (ints[1]-1) * basic_num;
-	    count *= ints[1];
-
-/* Now repeat with strides. */
-	    *curr_index += (top_count-1) * count;
-	    count *= top_count;
-	}
-	break;
-
-    case MPI_COMBINER_STRUCT: 
-    case MPI_COMBINER_STRUCT_INTEGER: 
-        top_count = ints[0];
-	count = 0;
-	for (n=0; n<top_count; n++) {
-            ADIOI_Type_get_envelope(types[n], &old_nints, &old_nadds,
-                                    &old_ntypes, &old_combiner); 
-	    ADIOI_Datatype_iscontig(types[n], &old_is_contig);
-
-	    prev_index = *curr_index;
-	    if ((old_combiner != MPI_COMBINER_NAMED) && (!old_is_contig))
-	    count += ADIOI_Count_contiguous_blocks(types[n], curr_index);
-
-	    if (prev_index == *curr_index) {
-/* simplest case, current type is basic or contiguous types */
-		count++;
-		(*curr_index)++;
-	    }
-	    else {
-/* current type made up of noncontiguous derived types */
-/* The current type has to be replicated blocklens[n] times */
-
-		num = *curr_index - prev_index;
-		count += (ints[1+n]-1)*num;
-		(*curr_index) += (ints[1+n]-1)*num;
-	    }
-	}
-	break;
-
-    case MPI_COMBINER_RESIZED: 
-	/* treat it as a struct with lb, type, ub */
-
-	/* add 2 for lb and ub */
-	(*curr_index) += 2;
-	count += 2;
-
-	/* add for datatype */ 
-	ADIOI_Type_get_envelope(types[0], &old_nints, &old_nadds,
-                                &old_ntypes, &old_combiner); 
-	ADIOI_Datatype_iscontig(types[0], &old_is_contig);
-
-	if ((old_combiner != MPI_COMBINER_NAMED) && (!old_is_contig)) {
-	    count += ADIOI_Count_contiguous_blocks(types[0], curr_index);
-	}
-	else {
-        /* basic or contiguous type */
-	    count++;
-	    (*curr_index)++;
-	}
-	break;
-
-    default:
-	/* TODO: FIXME */
-	DBG_FPRINTF(stderr, "Error: Unsupported datatype passed to ADIOI_Count_contiguous_blocks, combiner = %d\n", combiner);
-	MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-#ifndef MPISGI
-/* There is a bug in SGI's impl. of MPI_Type_get_contents. It doesn't
-   return new datatypes. Therefore no need to free. */
-    for (i=0; i<ntypes; i++) {
- 	MPI_Type_get_envelope(types[i], &old_nints, &old_nadds, &old_ntypes,
- 			      &old_combiner);
- 	if (old_combiner != MPI_COMBINER_NAMED) MPI_Type_free(types+i);
-    }
-#endif
-
-    ADIOI_Free(ints);
-    ADIOI_Free(adds);
-    ADIOI_Free(types);
-    return count;
-}
-
-
-/****************************************************************/
-
-/* ADIOI_Optimize_flattened()
- *
- * Scans the blocks of a flattened type and merges adjacent blocks 
- * together, resulting in a shorter blocklist (and thus fewer
- * contiguous operations).
- *
- * NOTE: a further optimization would be to remove zero length blocks. However,
- * the first and last blocks must remain as zero length first or last block 
- * indicates UB and LB.  Furthermore, once the "zero length blocklen" fix
- * went in, the flattened representation should no longer have zero-length
- * blocks except for UB and LB markers.
- */
-void ADIOI_Optimize_flattened(ADIOI_Flatlist_node *flat_type)
-{
-    int i, j, opt_blocks;
-    ADIO_Offset *opt_blocklens;
-    ADIO_Offset *opt_indices;
-
-    opt_blocks = 1;
-    
-    /* save number of noncontiguous blocks in opt_blocks */
-    for (i=0; i < (flat_type->count - 1); i++) {
-        if ((flat_type->indices[i] + flat_type->blocklens[i] !=
-	     flat_type->indices[i + 1]))
-	    opt_blocks++;
-    }
-
-    /* if we can't reduce the number of blocks, quit now */
-    if (opt_blocks == flat_type->count) return;
-
-    opt_blocklens = (ADIO_Offset *) ADIOI_Malloc(opt_blocks * sizeof(ADIO_Offset));
-    opt_indices = (ADIO_Offset *)ADIOI_Malloc(opt_blocks*sizeof(ADIO_Offset));
-
-    /* fill in new blocklists */
-    opt_blocklens[0] = flat_type->blocklens[0];
-    opt_indices[0] = flat_type->indices[0];
-    j = 0;
-    for (i=0; i < (flat_type->count - 1); i++) {
-	if ((flat_type->indices[i] + flat_type->blocklens[i] ==
-	     flat_type->indices[i + 1]))
-	    opt_blocklens[j] += flat_type->blocklens[i + 1];
-	else {
-	    j++;
-	    opt_indices[j] = flat_type->indices[i + 1];
-	    opt_blocklens[j] = flat_type->blocklens[i + 1];
-	} 
-    }
-    flat_type->count = opt_blocks;
-    ADIOI_Free(flat_type->blocklens);
-    ADIOI_Free(flat_type->indices);
-    flat_type->blocklens = opt_blocklens;
-    flat_type->indices = opt_indices;
-    return;
-}
-
-void ADIOI_Delete_flattened(MPI_Datatype datatype)
-{
-    ADIOI_Flatlist_node *flat, *prev;
-
-    prev = flat = ADIOI_Flatlist;
-    while (flat && (flat->type != datatype)) {
-	prev = flat;
-	flat = flat->next;
-    }
-    if (flat) {
-	prev->next = flat->next;
-	if (flat->blocklens) ADIOI_Free(flat->blocklens);
-	if (flat->indices) ADIOI_Free(flat->indices);
-	ADIOI_Free(flat);
-    }
-}
-
-ADIOI_Flatlist_node * ADIOI_Flatten_and_find(MPI_Datatype datatype)
-{
-    ADIOI_Flatlist_node *node;
-    ADIOI_Flatten_datatype(datatype);
-    node = ADIOI_Flatlist;
-    while (node->type != datatype) node = node->next;
-    return node;
-}
diff --git a/3rd-party/romio321/adio/common/get_fp_posn.c b/3rd-party/romio321/adio/common/get_fp_posn.c
deleted file mode 100644
index f2e025d00f8..00000000000
--- a/3rd-party/romio321/adio/common/get_fp_posn.c
+++ /dev/null
@@ -1,60 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-#include "adioi.h"
-
-/* returns the current position of the individual file pointer
-   in etype units relative to the current view. */
-
-void ADIOI_Get_position(ADIO_File fd, ADIO_Offset *offset)
-{
-    ADIOI_Flatlist_node *flat_file;
-    int i, flag;
-    MPI_Count filetype_size, etype_size;
-    int filetype_is_contig;
-    MPI_Aint filetype_extent, lb;
-    ADIO_Offset disp, byte_offset, sum=0, size_in_file, n_filetypes, frd_size;
-    
-    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
-    etype_size = fd->etype_size;
-
-    if (filetype_is_contig) *offset = (fd->fp_ind - fd->disp)/etype_size;
-    else {
-/* filetype already flattened in ADIO_Open */
-        flat_file = ADIOI_Flatlist;
-        while (flat_file->type != fd->filetype) flat_file = flat_file->next;
-
-	MPI_Type_size_x(fd->filetype, &filetype_size);
-	MPI_Type_get_extent(fd->filetype, &lb, &filetype_extent);
-
-	disp = fd->disp;
-	byte_offset = fd->fp_ind;
-	n_filetypes = -1;
-	flag = 0;
-	while (!flag) {
-	    sum = 0;
-	    n_filetypes++;
-	    for (i=0; i<flat_file->count; i++) {
-		sum += flat_file->blocklens[i];
-		if (disp + flat_file->indices[i] + 
-	     	    n_filetypes* ADIOI_AINT_CAST_TO_OFFSET filetype_extent + flat_file->blocklens[i] 
-		    >= byte_offset) {
-		    frd_size = disp + flat_file->indices[i] + 
-			n_filetypes * ADIOI_AINT_CAST_TO_OFFSET filetype_extent
-			+ flat_file->blocklens[i] - byte_offset;
-		    sum -= frd_size;
-		    flag = 1;
-		    break;
-		}
-	    }
-	}
-	size_in_file = n_filetypes * (ADIO_Offset)filetype_size + sum;
-	*offset = size_in_file/etype_size;
-    }
-}
diff --git a/3rd-party/romio321/adio/common/greq_fns.c b/3rd-party/romio321/adio/common/greq_fns.c
deleted file mode 100644
index aff8ff77e5d..00000000000
--- a/3rd-party/romio321/adio/common/greq_fns.c
+++ /dev/null
@@ -1,31 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2004 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpiu_greq.h"
-
-/* In cases where nonblocking operation will carry out blocking version,
- * instantiate and complete a generalized request  */
-
-void MPIO_Completed_request_create(MPI_File *fh, MPI_Offset bytes,
-		int *error_code, MPI_Request *request)
-{
-	MPI_Status *status;
-	status = (MPI_Status *)ADIOI_Malloc(sizeof(MPI_Status));
-
-	status->MPI_ERROR = *error_code;
-#ifdef HAVE_STATUS_SET_BYTES
-	MPIR_Status_set_bytes(status, MPI_BYTE, bytes);
-#endif
-	/* --BEGIN ERROR HANDLING-- */
-	if (*error_code != MPI_SUCCESS)
-		*error_code = MPIO_Err_return_file(*fh, *error_code);
-	/* --END ERROR HANDLING-- */
-	MPI_Grequest_start(MPIU_Greq_query_fn, MPIU_Greq_free_fn, 
-			MPIU_Greq_cancel_fn, status, request);
-	MPI_Grequest_complete(*request);
-}
diff --git a/3rd-party/romio321/adio/common/heap-sort.c b/3rd-party/romio321/adio/common/heap-sort.c
deleted file mode 100644
index c398e41aeea..00000000000
--- a/3rd-party/romio321/adio/common/heap-sort.c
+++ /dev/null
@@ -1,133 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2008 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "heap-sort.h"
-#include <assert.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <math.h>
-
-#define NOEXP2
-
-static void heapify(heap_t *heap, int i);
-
-/* From Introduction To Algorithms by Cormen, Leiserson, and Rivest */
-
-static inline int parent(int i) {
-    return (i/2);
-}
-
-static inline int left(int i) {
-    return (2*i);
-}
-
-static inline int right(int i) {
-    return (2*i + 1);
-}
-
-int ADIOI_Heap_create(heap_t *heap, int size) {
-    heap->size = size;
-    heap->nodes = (heap_node_t *) ADIOI_Calloc (size, sizeof(heap_node_t));
-    if (heap->nodes == NULL)
-	return 1;
-    else
-	return 0;
-}
-
-void ADIOI_Heap_free(heap_t *heap) {
-    ADIOI_Free(heap->nodes);
-}
-
-/* should suppress unused warnings on GCC */
-static void build_heap(heap_t *heap) ATTRIBUTE((unused, used));
-
-static void build_heap(heap_t *heap)
-{
-    int i;
-    for (i=(heap->size/2-1); i >= 0; i--)
-	heapify(heap, i);
-}
-
-static void heapify(heap_t *heap, int i) {
-    int l, r, smallest;
-    heap_node_t *nodes;
-    heap_node_t tmp_node;
-
-    nodes = heap->nodes;
-
-    l = left(i);
-    r = right(i);
-
-    if ((l <= heap->size) && (nodes[l].offset < nodes[i].offset))
-	smallest = l;
-    else
-	smallest = i;
-
-    if ((r <= heap->size) && (nodes[r].offset < nodes[smallest].offset))
-	smallest = r;
-
-    if (smallest != i) {
-	tmp_node = nodes[i];
-	nodes[i] = nodes[smallest];
-	nodes[smallest] = tmp_node;
-	heapify(heap, smallest);
-    }
-}
-
-void ADIOI_Heap_insert(heap_t *heap, ADIO_Offset offset, int proc,
-		 ADIO_Offset reg_max_len) {
-    heap_node_t *nodes;
-    int i;
-    nodes = heap->nodes;
-    i = ++heap->size - 1;
-    while ((i > 0) && (nodes[parent(i)].offset > offset)) {
-	nodes[i] = nodes[parent(i)];
-	i = parent(i);
-    }
-    nodes[i].offset      = offset;
-    nodes[i].proc        = proc;
-    nodes[i].reg_max_len = reg_max_len;
-}
-
-void ADIOI_Heap_extract_min(heap_t *heap, ADIO_Offset* offset, int *proc,
-		      ADIO_Offset *reg_max_len) {
-    heap_node_t *nodes;
-    nodes = heap->nodes;
-    
-    assert (heap->size > 0);
-    *offset = nodes[0].offset;
-    *proc = nodes[0].proc;
-    *reg_max_len = nodes[0].reg_max_len;
-    nodes[0] = nodes[heap->size-1];
-    heap->size--;
-    heapify(heap, 0);
-}
-
-/* should suppress unused warnings on GCC */
-static void print_heap(heap_t *heap) ATTRIBUTE((unused, used));
-
-static void print_heap(heap_t *heap)
-{
-#ifndef NOEXP2
-    int i;
-    double level = 0;
-    int next_level_idx = 1;
-
-    printf ("heap->size = %d\n", heap->size);
-    printf ("offsets:\n");
-    for (i=0; i < heap->size; i++) {
-	printf ("%lld ", heap->nodes[i].offset);
-	
-	if ((i+1) == next_level_idx) {
-	    printf ("\n");
-	    next_level_idx += (int) exp2(level+1);
-	    level++;
-	}
-    }
-    printf ("\n");
-#endif
-}
diff --git a/3rd-party/romio321/adio/common/hint_fns.c b/3rd-party/romio321/adio/common/hint_fns.c
deleted file mode 100644
index 284e4a96272..00000000000
--- a/3rd-party/romio321/adio/common/hint_fns.c
+++ /dev/null
@@ -1,191 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2013 UChicago/Argonne, LLC
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "hint_fns.h"
-
-
-int ADIOI_Info_check_and_install_int(ADIO_File fd, MPI_Info info, const char *key, 
-	int *local_cache, char *funcname, int *error_code)
-{
-    int intval, tmp_val, flag, ret=0;
-    char *value;
-
-    value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
-    if (value == NULL) {
-        *error_code = MPIO_Err_create_code(*error_code,
-                                           MPIR_ERR_RECOVERABLE,
-                                           funcname,
-                                           __LINE__,
-                                           MPI_ERR_OTHER,
-                                           "**nomem2",0);
-        return -1;
-    }
-
-    ADIOI_Info_get(info, key, MPI_MAX_INFO_VAL, value, &flag);
-    if (flag) {
-	intval = atoi(value);
-	tmp_val = intval;
-
-	MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
-	/* --BEGIN ERROR HANDLING-- */
-	if (tmp_val != intval) {
-	    MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(funcname,
-		    key, 
-		    error_code);
-	    ret = -1;
-	    goto fn_exit;
-	}
-	/* --END ERROR HANDLING-- */
-
-	ADIOI_Info_set(fd->info, key, value);
-	/* some file systems do not cache hints in the fd struct */
-	if (local_cache != NULL) *local_cache = intval;
-    }
-fn_exit:
-    ADIOI_Free(value);
-    return ret;
-}
-
-int ADIOI_Info_check_and_install_enabled(ADIO_File fd, MPI_Info info, const char *key, 
-	int *local_cache, char *funcname, int *error_code)
-{
-    int tmp_val, flag, ret=0;
-    char *value;
-
-    value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
-    if (value == NULL) {
-        *error_code = MPIO_Err_create_code(*error_code,
-                                           MPIR_ERR_RECOVERABLE,
-                                           funcname,
-                                           __LINE__,
-                                           MPI_ERR_OTHER,
-                                           "**nomem2",0);
-        return -1;
-    }
-
-    ADIOI_Info_get(info, key, MPI_MAX_INFO_VAL, value, &flag);
-    if (flag) {
-	if (!strcmp(value, "enable") || !strcmp(value, "ENABLE")) {
-	    ADIOI_Info_set(fd->info, key, value);
-	    *local_cache = ADIOI_HINT_ENABLE;
-	}
-	else if (!strcmp(value, "disable") || !strcmp(value, "DISABLE")) {
-	    ADIOI_Info_set(fd->info, key, value);
-	    *local_cache = ADIOI_HINT_DISABLE;
-	}
-	else if (!strcmp(value, "automatic") || !strcmp(value, "AUTOMATIC"))
-	{
-	    ADIOI_Info_set(fd->info, key, value);
-	    *local_cache = ADIOI_HINT_AUTO;
-	}
-
-	tmp_val = *local_cache;
-
-	MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
-	/* --BEGIN ERROR HANDLING-- */
-	if (tmp_val != *local_cache) {
-	    MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(funcname,
-		    key,
-		    error_code);
-	    ret = -1;
-	    goto fn_exit;
-	}
-	/* --END ERROR HANDLING-- */
-    }
-fn_exit:
-    ADIOI_Free(value);
-    return ret;
-}
-int ADIOI_Info_check_and_install_true(ADIO_File fd, MPI_Info info, const char *key, 
-	int *local_cache, char *funcname, int *error_code)
-{
-    int flag, tmp_val, ret=0;
-    char *value;
-
-    value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
-    if (value == NULL) {
-        *error_code = MPIO_Err_create_code(*error_code,
-                                           MPIR_ERR_RECOVERABLE,
-                                           funcname,
-                                           __LINE__,
-                                           MPI_ERR_OTHER,
-                                           "**nomem2",0);
-        return -1;
-    }
-
-    ADIOI_Info_get(info, key, MPI_MAX_INFO_VAL, value, &flag);
-    if (flag) {
-	if (!strcmp(value, "true") || !strcmp(value, "TRUE")) {
-	    ADIOI_Info_set(fd->info, key, value);
-	    *local_cache = 1;
-	}
-	else if (!strcmp(value, "false") || !strcmp(value, "FALSE")) {
-	    ADIOI_Info_set(fd->info, key, value);
-	    *local_cache = 0;
-	}
-	tmp_val = *local_cache;
-
-	MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
-	/* --BEGIN ERROR HANDLING-- */
-	if (tmp_val != *local_cache) {
-	    MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(funcname,
-		    key,
-		    error_code);
-	    ret = -1;
-	    goto fn_exit;
-	}
-	/* --END ERROR HANDLING-- */
-    }
-fn_exit:
-    ADIOI_Free(value);
-    return ret;
-}
-int ADIOI_Info_check_and_install_str(ADIO_File fd, MPI_Info info, const char *key, 
-	char **local_cache, char *funcname, int *error_code)
-{
-    int flag, ret=0;
-    size_t len;
-    char *value;
-
-    value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
-    if (value == NULL) {
-        *error_code = MPIO_Err_create_code(*error_code,
-                                           MPIR_ERR_RECOVERABLE,
-                                           funcname,
-                                           __LINE__,
-                                           MPI_ERR_OTHER,
-                                           "**nomem2",0);
-        return -1;
-    }
-
-    ADIOI_Info_get(info, key, MPI_MAX_INFO_VAL,
-	    value, &flag);
-    if (flag) {
-	ADIOI_Info_set(fd->info, "cb_config_list", value);
-	len = (strlen(value)+1) * sizeof(char);
-	*local_cache = ADIOI_Malloc(len);
-	if (*local_cache == NULL) {
-	    *error_code = MPIO_Err_create_code(*error_code,
-		    MPIR_ERR_RECOVERABLE,
-		    funcname,
-		    __LINE__,
-		    MPI_ERR_OTHER,
-		    "**nomem2",0);
-	    ret = -1;
-	    goto fn_exit;
-	}
-	ADIOI_Strncpy(*local_cache, value, len);
-    }
-    /* if it has been set already, we ignore it the second time. 
-     * otherwise we would get an error if someone used the same
-     * info value with a cb_config_list value in it in a couple
-     * of calls, which would be irritating. */
-fn_exit:
-    ADIOI_Free(value);
-    return ret;
-}
diff --git a/3rd-party/romio321/adio/common/iscontig.c b/3rd-party/romio321/adio/common/iscontig.c
deleted file mode 100644
index 3c8aa712f4e..00000000000
--- a/3rd-party/romio321/adio/common/iscontig.c
+++ /dev/null
@@ -1,105 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-/* #ifdef MPISGI
-#include "mpisgi2.h"
-#endif */
-
-#if defined(MPICH)
-/* MPICH also provides this routine */
-void MPIR_Datatype_iscontig(MPI_Datatype datatype, int *flag);
-
-void ADIOI_Datatype_iscontig(MPI_Datatype datatype, int *flag)
-{
-    MPIR_Datatype_iscontig(datatype, flag);
-
-    /* if the datatype is reported as contigous, check if the true_lb is
-     * non-zero, and if so, mark the datatype as noncontiguous */
-    if (*flag) {
-        MPI_Aint true_extent, true_lb;
-
-        MPI_Type_get_true_extent(datatype, &true_lb, &true_extent);
-
-        if (true_lb > 0)
-            *flag = 0;
-    }
-}
-
-#elif (defined(MPIHP) && defined(HAVE_MPI_INFO))
-/* i.e. HPMPI 1.4 only */
-
-int hpmp_dtiscontig(MPI_Datatype datatype);
-
-void ADIOI_Datatype_iscontig(MPI_Datatype datatype, int *flag)
-{
-    *flag = hpmp_dtiscontig(datatype);
-}
-
-#elif (defined(MPISGI) && !defined(NO_MPI_SGI_type_is_contig))
-
-int MPI_SGI_type_is_contig(MPI_Datatype datatype);
-
-void ADIOI_Datatype_iscontig(MPI_Datatype datatype, int *flag)
-{
-    MPI_Aint displacement;
-    MPI_Type_lb(datatype, &distplacement);
-
-    /* SGI's MPI_SGI_type_is_contig() returns true for indexed
-     * datatypes with holes at the beginning, which causes
-     * problems with ROMIO's use of this function.
-     */
-    *flag = MPI_SGI_type_is_contig(datatype) && (displacement == 0);
-}
-
-#elif defined(OMPI_BUILDING) && OMPI_BUILDING
-
-/* void ADIOI_Datatype_iscontig(MPI_Datatype datatype, int *flag) is defined
- * and implemented in OpenMPI itself */
-
-#else
-
-void ADIOI_Datatype_iscontig(MPI_Datatype datatype, int *flag)
-{
-    int nints, nadds, ntypes, combiner;
-    int *ints, ni, na, nt, cb;
-    MPI_Aint *adds;
-    MPI_Datatype *types;
-
-    MPI_Type_get_envelope(datatype, &nints, &nadds, &ntypes, &combiner);
-
-    switch (combiner) {
-    case MPI_COMBINER_NAMED:
-	*flag = 1;
-	break;
-    case MPI_COMBINER_CONTIGUOUS:
-	ints = (int *) ADIOI_Malloc((nints+1)*sizeof(int));
-	adds = (MPI_Aint *) ADIOI_Malloc((nadds+1)*sizeof(MPI_Aint));
-	types = (MPI_Datatype *) ADIOI_Malloc((ntypes+1)*sizeof(MPI_Datatype));
-	MPI_Type_get_contents(datatype, nints, nadds, ntypes, ints,
-			      adds, types); 
-	ADIOI_Datatype_iscontig(types[0], flag);
-
-#ifndef MPISGI
-/* There is a bug in SGI's impl. of MPI_Type_get_contents. It doesn't
-   return new datatypes. Therefore no need to free. */
-	MPI_Type_get_envelope(types[0], &ni, &na, &nt, &cb);
-	if (cb != MPI_COMBINER_NAMED) MPI_Type_free(types);
-#endif
-
-	ADIOI_Free(ints);
-	ADIOI_Free(adds);
-	ADIOI_Free(types);
-	break;
-    default:
-	*flag = 0;
-	break;
-    }
-
-    /* This function needs more work. It should check for contiguity 
-       in other cases as well.*/
-}
-#endif
diff --git a/3rd-party/romio321/adio/common/lock.c b/3rd-party/romio321/adio/common/lock.c
deleted file mode 100644
index 2590d77433a..00000000000
--- a/3rd-party/romio321/adio/common/lock.c
+++ /dev/null
@@ -1,225 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-
-#ifdef ROMIO_NTFS
-/* This assumes that lock will always remain in the common directory and 
- * that the ntfs directory will always be called ad_ntfs. */
-#include "..\ad_ntfs\ad_ntfs.h"
-int ADIOI_Set_lock(FDTYPE fd, int cmd, int type, ADIO_Offset offset, int whence,
-	     ADIO_Offset len) 
-{
-    static char myname[] = "ADIOI_Set_lock";
-    int ret_val, error_code = MPI_SUCCESS;
-    OVERLAPPED Overlapped;
-    DWORD dwFlags;
-
-    ADIOI_UNREFERENCED_ARG(whence);
-
-    if (len == 0) return MPI_SUCCESS;
-
-    dwFlags = type;
-
-    Overlapped.hEvent = /*0;*/CreateEvent(NULL, TRUE, FALSE, NULL);
-#ifdef HAVE_INT64
-    Overlapped.Offset = ( (DWORD) ( offset & (__int64) 0xFFFFFFFF ) );
-    Overlapped.OffsetHigh = ( (DWORD) ( (offset >> 32) & (__int64) 0xFFFFFFFF ) );
-
-    if (cmd == ADIOI_LOCK_CMD)
-    {
-	/*printf("locking %d\n", (int)fd);fflush(stdout);*/
-	ret_val = LockFileEx(fd, dwFlags, 0, 
-	( (DWORD) ( len & (__int64) 0xFFFFFFFF ) ), 
-	( (DWORD) ( (len >> 32) & (__int64) 0xFFFFFFFF ) ), 
-	&Overlapped);
-    }
-    else
-    {
-	/*printf("unlocking %d\n", (int)fd);fflush(stdout);*/
-	ret_val = UnlockFileEx(fd, 0, 
-	( (DWORD) ( len & (__int64) 0xFFFFFFFF ) ), 
-	( (DWORD) ( (len >> 32) & (__int64) 0xFFFFFFFF ) ), 
-	&Overlapped);
-    }
-#else
-    Overlapped.Offset = offset;
-    Overlapped.OffsetHigh = 0;
-
-    if (cmd == ADIOI_LOCK_CMD)
-    {
-	/*printf("locking %d\n", (int)fd);fflush(stdout);*/
-	ret_val = LockFileEx(fd, dwFlags, 0, len, 0, &Overlapped);
-    }
-    else
-    {
-	/*printf("unlocking %d\n", (int)fd);fflush(stdout);*/
-	ret_val = UnlockFileEx(fd, 0, len, 0, &Overlapped);
-    }
-#endif
-
-    if (!ret_val)
-    {
-    char errMsg[ADIOI_NTFS_ERR_MSG_MAX];
-	/*
-	FPRINTF(stderr, "File locking failed in ADIOI_Set_lock.\n");
-	MPI_Abort(MPI_COMM_WORLD, 1);
-	*/
-	ret_val = GetLastError();
-	if (ret_val == ERROR_IO_PENDING)
-	{
-	    DWORD dummy;
-	    ret_val = GetOverlappedResult(fd, &Overlapped, &dummy, TRUE);
-	    if (ret_val)
-	    {
-		CloseHandle(Overlapped.hEvent);
-		return MPI_SUCCESS;
-	    }
-	    ret_val = GetLastError();
-	}
-    ADIOI_NTFS_Strerror(ret_val, errMsg, ADIOI_NTFS_ERR_MSG_MAX);
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__,
-	    MPI_ERR_IO, "**io", "**io %s", errMsg);
-    }
-    CloseHandle(Overlapped.hEvent);
-
-    return error_code;
-}
-#else
-int ADIOI_Set_lock(FDTYPE fd, int cmd, int type, ADIO_Offset offset, int whence,
-	     ADIO_Offset len) 
-{
-    int err, error_code, err_count = 0, sav_errno;
-    struct flock lock;
-
-    if (len == 0) return MPI_SUCCESS;
-
-
-    /* Depending on the compiler flags and options, struct flock 
-       may not be defined with types that are the same size as
-       ADIO_Offsets.  */
-/* FIXME: This is a temporary hack until we use flock64 where
-   available. It also doesn't fix the broken Solaris header sys/types.h
-   header file, which declars off_t as a UNION ! Configure tests to
-   see if the off64_t is a union if large file support is requested; 
-   if so, it does not select large file support.
-*/
-#ifdef NEEDS_INT_CAST_WITH_FLOCK
-    lock.l_type	  = type;
-    lock.l_start  = (int)offset;
-    lock.l_whence = whence;
-    lock.l_len	  = (int)len;
-#else
-    lock.l_type	  = type;
-    lock.l_whence = whence;
-    lock.l_start  = offset;
-    lock.l_len	  = len;
-#endif
-
-    sav_errno = errno; /* save previous errno in case we recover from retryable errors */
-    errno = 0;
-    do {
-	err = fcntl(fd, cmd, &lock);
-#ifdef USE_DBG_LOGGING
-/*      if (MPIU_DBG_SELECTED(ROMIO,TERSE)) */
-      {
-        if (err && ((errno == EINTR) || (errno == EINPROGRESS)))
-        {
-          if((err_count < 5) || (err_count > 9995))
-          {
-            fprintf(stderr, "File locking failed in ADIOI_Set_lock(fd %#X,cmd %s/%#X,type %s/%#X,whence %#X) with return value %#X and errno %#X.  Retry (%d).\n",                    
-                    fd,
-                    ((cmd == F_GETLK   )? "F_GETLK" :
-                    ((cmd == F_SETLK   )? "F_SETLK" :
-                    ((cmd == F_SETLKW  )? "F_SETLKW" : "UNEXPECTED"))),
-                    cmd, 
-                    ((type == F_RDLCK   )? "F_RDLCK" :
-                    ((type == F_WRLCK   )? "F_WRLCK" :
-                    ((type == F_UNLCK   )? "F_UNLOCK" : "UNEXPECTED"))),
-                    type, 
-                    whence, err, errno, err_count);
-          perror("ADIOI_Set_lock:");
-          fprintf(stderr,"ADIOI_Set_lock:offset %#llx, length %#llx\n",(unsigned long long)offset, (unsigned long long)len);
-          }
-        }
-      }
-#endif
-    } while (err && ((errno == EINTR) || ((errno == EINPROGRESS) && (++err_count < 10000))));
-
-    if (err && (errno != EBADF)) {
-	/* FIXME: This should use the error message system, 
-	   especially for MPICH */
-	FPRINTF(stderr, "This requires fcntl(2) to be implemented. As of 8/25/2011 it is not. Generic MPICH Message: File locking failed in ADIOI_Set_lock(fd %X,cmd %s/%X,type %s/%X,whence %X) with return value %X and errno %X.\n"
-                  "- If the file system is NFS, you need to use NFS version 3, ensure that the lockd daemon is running on all the machines, and mount the directory with the 'noac' option (no attribute caching).\n"
-                  "- If the file system is LUSTRE, ensure that the directory is mounted with the 'flock' option.\n",
-          fd,
-          ((cmd == F_GETLK   )? "F_GETLK" :
-          ((cmd == F_SETLK   )? "F_SETLK" :
-          ((cmd == F_SETLKW  )? "F_SETLKW" : "UNEXPECTED"))),
-          cmd, 
-          ((type == F_RDLCK   )? "F_RDLCK" :
-          ((type == F_WRLCK   )? "F_WRLCK" :
-          ((type == F_UNLCK   )? "F_UNLOCK" : "UNEXPECTED"))),
-          type, 
-          whence, err, errno);
-  perror("ADIOI_Set_lock:");
-  FPRINTF(stderr,"ADIOI_Set_lock:offset %llu, length %llu\n",(unsigned long long)offset, (unsigned long long)len);
-	MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    if(!err)             /* report fcntl failure errno's (EBADF), otherwise */
-      errno = sav_errno; /* restore previous errno in case we recovered from retryable errors */
-
-    error_code = (err == 0) ? MPI_SUCCESS : MPI_ERR_UNKNOWN;
-    return error_code;
-}
-#endif
-
-#if (defined(ROMIO_HFS) || defined(ROMIO_XFS))
-int ADIOI_Set_lock64(FDTYPE fd, int cmd, int type, ADIO_Offset offset,
-                     int whence,
-	             ADIO_Offset len) 
-{
-    int err, error_code;
-    struct flock64 lock;
-
-    if (len == 0) return MPI_SUCCESS;
-
-    lock.l_type = type;
-    lock.l_start = offset;
-    lock.l_whence = whence;
-    lock.l_len = len;
-
-    do {
-	err = fcntl(fd, cmd, &lock);
-    } while (err && (errno == EINTR));
-
-    if (err && (errno != EBADF)) {
-	FPRINTF(stderr, "File locking failed in ADIOI_Set_lock64(fd %X,cmd %s/%X,type %s/%X,whence %X) with return value %X and errno %X.\n"
-                  "If the file system is NFS, you need to use NFS version 3, ensure that the lockd daemon is running on all the machines, and mount the directory with the 'noac' option (no attribute caching).\n",
-          fd,
-          ((cmd == F_GETLK   )? "F_GETLK" :
-          ((cmd == F_SETLK   )? "F_SETLK" :
-          ((cmd == F_SETLKW  )? "F_SETLKW" :
-          ((cmd == F_GETLK64 )? "F_GETLK64" :
-          ((cmd == F_SETLK64 )? "F_SETLK64" :
-          ((cmd == F_SETLKW64)? "F_SETLKW64" : "UNEXPECTED")))))),
-          cmd, 
-          ((type == F_RDLCK   )? "F_RDLCK" :
-          ((type == F_WRLCK   )? "F_WRLCK" :
-          ((type == F_UNLCK   )? "F_UNLOCK" : "UNEXPECTED"))),
-          type, 
-          whence, err, errno);
-  perror("ADIOI_Set_lock64:");
-  FPRINTF(stderr,"ADIOI_Set_lock:offset %llu, length %llu\n",(unsigned long long)offset, (unsigned long long)len);
-	MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    error_code = (err == 0) ? MPI_SUCCESS : MPI_ERR_UNKNOWN;
-    return error_code;
-}
-#endif
diff --git a/3rd-party/romio321/adio/common/malloc.c b/3rd-party/romio321/adio/common/malloc.c
deleted file mode 100644
index 73c5a707050..00000000000
--- a/3rd-party/romio321/adio/common/malloc.c
+++ /dev/null
@@ -1,113 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-/* These are routines for allocating and deallocating memory.
-   They should be called as ADIOI_Malloc(size) and
-   ADIOI_Free(ptr). In adio.h, they are macro-replaced to 
-   ADIOI_Malloc(size,__LINE__,__FILE__) and 
-   ADIOI_Free(ptr,__LINE__,__FILE__).
-
-   Later on, add some tracing and error checking, similar to 
-   MPID_trmalloc. */
-
-#include "adio.h"
-#include "mpi.h"
-#include <stdlib.h>
-#include <stdio.h>
-#include "mpipr.h"
-
-#ifdef HAVE_MALLOC_H
-#include <malloc.h>
-#endif
-
-/* for the style checker */
-/* style: allow:malloc:1 sig:0 */
-/* style: allow:free:1 sig:0 */
-/* style: allow:calloc:1 sig:0 */
-/* style: allow:realloc:1 sig:0 */
-
-#define FPRINTF fprintf
-
-void *ADIOI_Malloc_fn(size_t size, int lineno, const char *fname);
-void *ADIOI_Calloc_fn(size_t nelem, size_t elsize, int lineno, const char *fname);
-void *ADIOI_Realloc_fn(void *ptr, size_t size, int lineno, const char *fname);
-void ADIOI_Free_fn(void *ptr, int lineno, const char *fname);
-
-void *ADIOI_Malloc_fn(size_t size, int lineno, const char *fname)
-{
-    void *new;
-
-#ifdef ROMIO_XFS
-    new = (void *) memalign(XFS_MEMALIGN, size);
-#else
-#ifdef HAVE_MPIU_FUNCS
-    new = (void *) MPIU_trmalloc(size, lineno, fname);
-#else
-    new = (void *) malloc(size);
-#endif
-#endif
-    if (!new && size) {
-	FPRINTF(stderr, "Out of memory in file %s, line %d\n", fname, lineno);
-	MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-    DBG_FPRINTF(stderr, "ADIOI_Malloc %s:<%d> %p (%#zX)\n", fname, lineno, new, size);
-    return new;
-}
-
-
-void *ADIOI_Calloc_fn(size_t nelem, size_t elsize, int lineno, const char *fname)
-{
-    void *new;
-
-#ifdef HAVE_MPIU_FUNCS
-    new = (void *) MPIU_trcalloc(nelem, elsize, lineno, fname);
-#else
-    new = (void *) calloc(nelem, elsize);
-#endif
-    if (!new && nelem) {
-	FPRINTF(stderr, "Out of memory in file %s, line %d\n", fname, lineno);
-	MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-    DBG_FPRINTF(stderr, "ADIOI_Calloc %s:<%d> %p\n", fname, lineno, new);
-    return new;
-}
-
-
-void *ADIOI_Realloc_fn(void *ptr, size_t size, int lineno, const char *fname)
-{
-    void *new;
-
-#ifdef HAVE_MPIU_FUNCS
-    new = (void *) MPIU_trrealloc(ptr, size, lineno, fname);
-#else
-    new = (void *) realloc(ptr, size);
-#endif
-    if (!new && size) {
-	FPRINTF(stderr, "realloc failed in file %s, line %d\n", fname, lineno);
-	MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-    DBG_FPRINTF(stderr, "ADIOI_Realloc %s:<%d> %p\n", fname, lineno, new);
-    return new;
-}
-
-
-void ADIOI_Free_fn(void *ptr, int lineno, const char *fname)
-{
-    DBG_FPRINTF(stderr, "ADIOI_Free %s:<%d> %p\n", fname, lineno, ptr);
-    if (!ptr) {
-	FPRINTF(stderr, "Attempt to free null pointer in file %s, line %d\n", fname, lineno);
-	MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-#ifdef HAVE_MPIU_FUNCS
-    MPIU_trfree(ptr, lineno, fname);
-#else
-    free(ptr);
-#endif
-}
-
-
diff --git a/3rd-party/romio321/adio/common/onesided_aggregation.c b/3rd-party/romio321/adio/common/onesided_aggregation.c
deleted file mode 100644
index 71a5b7bee6e..00000000000
--- a/3rd-party/romio321/adio/common/onesided_aggregation.c
+++ /dev/null
@@ -1,2174 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *  (C) 2015 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-#ifdef ROMIO_GPFS
-/* right now this is GPFS only but TODO: extend this to all file systems */
-#include "../ad_gpfs/ad_gpfs_tuning.h"
-#else
-int gpfsmpio_onesided_no_rmw = 0;
-int gpfsmpio_write_aggmethod = 0;
-int gpfsmpio_read_aggmethod = 0;
-int gpfsmpio_onesided_always_rmw = 0;
-#endif
-
-#include <pthread.h>
-
-//  #define onesidedtrace 1
-
-
-/* This data structure holds the number of extents, the index into the flattened buffer and the remnant length
- * beyond the flattened buffer index corresponding to the base buffer offset for non-contiguous source data
- * for the range to be written coresponding to the round and target agg.
- */
-typedef struct NonContigSourceBufOffset {
-  int dataTypeExtent;
-  int flatBufIndice;
-  ADIO_Offset indiceOffset;
-} NonContigSourceBufOffset;
-
-/* This data structure holds the access state of the source buffer for target
- * file domains within aggregators corresponding to the target data blocks.  It
- * is designed to be initialized with a starting point for a given file domain
- * with an aggregator, after which the data access for data written to a given
- * file domain from this compute is linear and uninterupted, and this serves as
- * a key optimization for feeding the target aggs.  For contigous source data
- * the starting point is a single-value offset, for non-contiguous data it is
- * the number of extents, the index into the flattened buffer and the remnant
- * length beyond the flattened buffer index.  The validity of the usage of this
- * structure relies on the requirement that only 1 aggregator can write to a
- * given file domain.  */
-typedef struct FDSourceBufferState {
-
-    ADIO_Offset indiceOffset;
-    MPI_Aint bufTypeExtent;
-    int dataTypeExtent;
-    int flatBufIndice;
-
-    ADIO_Offset sourceBufferOffset;
-
-} FDSourceBufferState;
-
-
-static int ADIOI_OneSidedSetup(ADIO_File fd, int procs) {
-    int ret = MPI_SUCCESS;
-
-    ret = MPI_Win_create(fd->io_buf,fd->hints->cb_buffer_size,1,
-	    MPI_INFO_NULL,fd->comm, &fd->io_buf_window);
-    if (ret != MPI_SUCCESS) goto fn_exit;
-    fd->io_buf_put_amounts = (int *) ADIOI_Malloc(procs*sizeof(int));
-    ret =MPI_Win_create(fd->io_buf_put_amounts,procs*sizeof(int),sizeof(int),
-	    MPI_INFO_NULL,fd->comm, &fd->io_buf_put_amounts_window);
-fn_exit:
-    return ret;
-}
-
-int ADIOI_OneSidedCleanup(ADIO_File fd)
-{
-    int ret = MPI_SUCCESS;
-    if (fd->io_buf_window != MPI_WIN_NULL)
-	ret = MPI_Win_free(&fd->io_buf_window);
-    if (fd->io_buf_put_amounts_window != MPI_WIN_NULL)
-	ret = MPI_Win_free(&fd->io_buf_put_amounts_window);
-    if (fd->io_buf_put_amounts != NULL)
-	ADIOI_Free(fd->io_buf_put_amounts);
-
-    return ret;
-}
-
-/* This funtion packs a contigous buffer of data from the non-contgious source
- * buffer for a specified chunk of data and advances the FDSourceBufferState
- * machinery, so subsequent calls with the FDSourceBufferState will return the
- * next linear chunk.
- * Parameters:
- * in:     sourceDataBuffer - pointer to source data buffer.
- * in:    flatBuf - pointer to flattened source data buffer
- * in:     targetNumBytes - number of bytes to return and advance.
- * in:     packing - whether data is being packed from the source buffer to the
- *         packed buffer (1) or unpacked from the packed buffer to the source
- *         buffer (0)
- * in/out: currentFDSourceBufferState - pointer to FDSourceBufferState structure, current
- *                                      data used as starting point, will be updated with
- *                                      the new state after targetNumBytes advance.
- * out:    packedDataBufer - pointer to the output packed data buffer.  If the
- *                           value is NULL then no data will be written.
- *
- */
-inline static void nonContigSourceDataBufferAdvance(char *sourceDataBuffer,
-	ADIOI_Flatlist_node *flatBuf, int targetNumBytes, int packing,
-	FDSourceBufferState *currentFDSourceBufferState, char *packedDataBufer)
-{
-    // make currentDataTypeExtent and bufTypeExtent ADIO_Offset since they are
-    // used in offset calculations
-    ADIO_Offset currentIndiceOffset = currentFDSourceBufferState->indiceOffset;
-    ADIO_Offset bufTypeExtent = (ADIO_Offset)currentFDSourceBufferState->bufTypeExtent;
-    ADIO_Offset currentDataTypeExtent =
-	(ADIO_Offset)currentFDSourceBufferState->dataTypeExtent;
-    int currentFlatBufIndice = currentFDSourceBufferState->flatBufIndice;
-
-    int targetSendDataIndex = 0;
-
-#ifdef onesidedtrace
-    printf("nonContigSourceDataBufferAdvance: currentFlatBufIndice is %d currentDataTypeExtent is %ld currentIndiceOffset is %ld\n",currentFlatBufIndice,currentDataTypeExtent,currentIndiceOffset);
-#endif
-
-    int remainingBytesToLoad = targetNumBytes;
-    while (remainingBytesToLoad > 0) {
-      if ((flatBuf->blocklens[currentFlatBufIndice] - currentIndiceOffset) >= remainingBytesToLoad) { // we can get the rest of our data from this indice
-        ADIO_Offset physicalSourceBufferOffset = (currentDataTypeExtent * bufTypeExtent) + flatBuf->indices[currentFlatBufIndice] + currentIndiceOffset;
-
-#ifdef onesidedtrace
-        printf("loading remainingBytesToLoad %d from src buffer offset %ld to targetSendDataIndex %d\n",remainingBytesToLoad,physicalSourceBufferOffset,targetSendDataIndex);
-#endif
-
-        if (packedDataBufer != NULL) {
-        if (packing)
-          memcpy(&(packedDataBufer[targetSendDataIndex]),&(sourceDataBuffer[physicalSourceBufferOffset]),remainingBytesToLoad);
-        else
-          memcpy(&(sourceDataBuffer[physicalSourceBufferOffset]),&(packedDataBufer[targetSendDataIndex]),remainingBytesToLoad);
-        }
-
-        targetSendDataIndex += remainingBytesToLoad;
-        currentIndiceOffset += (ADIO_Offset)remainingBytesToLoad;
-        if (currentIndiceOffset >= flatBuf->blocklens[currentFlatBufIndice]) {
-          currentIndiceOffset = (ADIO_Offset)0;
-          currentFlatBufIndice++;
-          if (currentFlatBufIndice == flatBuf->count) {
-            currentFlatBufIndice = 0;
-            currentDataTypeExtent++;
-          }
-        }
-        remainingBytesToLoad = 0;
-
-      }
-      else { // we can only get part of our data from this indice
-        int amountDataToLoad = (flatBuf->blocklens[currentFlatBufIndice] - currentIndiceOffset);
-        ADIO_Offset physicalSourceBufferOffset = (currentDataTypeExtent * bufTypeExtent) + flatBuf->indices[currentFlatBufIndice] + currentIndiceOffset;
-
-#ifdef onesidedtrace
-        printf("loading amountDataToLoad %d from src buffer offset %ld to targetSendDataIndex %d\n",amountDataToLoad,physicalSourceBufferOffset,targetSendDataIndex);
-#endif
-        if (packedDataBufer != NULL) {
-        if (packing)
-            memcpy(&(packedDataBufer[targetSendDataIndex]),&(sourceDataBuffer[physicalSourceBufferOffset]),amountDataToLoad);
-        else
-          memcpy(&(sourceDataBuffer[physicalSourceBufferOffset]),&(packedDataBufer[targetSendDataIndex]),amountDataToLoad);
-        }
-
-        targetSendDataIndex += amountDataToLoad;
-        currentIndiceOffset = (ADIO_Offset)0;
-        currentFlatBufIndice++;
-        if (currentFlatBufIndice == flatBuf->count) {
-          currentFlatBufIndice = 0;
-          currentDataTypeExtent++;
-        }
-        remainingBytesToLoad -= amountDataToLoad;
-      }
-    } // while
-
-    /* update machinery with new flatbuf position
-     */
-    currentFDSourceBufferState->indiceOffset = currentIndiceOffset;
-    currentFDSourceBufferState->dataTypeExtent = (int) currentDataTypeExtent;
-    currentFDSourceBufferState->flatBufIndice = currentFlatBufIndice;
-#ifdef onesidedtrace
-    printf("source buf advanced to currentFlatBufIndice %d currentDataTypeExtent %ld currentIndiceOffset %ld\n",currentFlatBufIndice,currentDataTypeExtent,currentIndiceOffset);
-#endif
-}
-
-
-void ADIOI_OneSidedWriteAggregation(ADIO_File fd,
-    ADIO_Offset *offset_list,
-    ADIO_Offset *len_list,
-    int contig_access_count,
-    const void *buf,
-    MPI_Datatype datatype,
-    int *error_code,
-    ADIO_Offset *st_offsets,
-    ADIO_Offset *end_offsets,
-    int numNonZeroDataOffsets,
-    ADIO_Offset *fd_start,
-    ADIO_Offset* fd_end,
-    int *hole_found)
-
-{
-    int i,j; /* generic iterators */
-
-#ifdef onesidedtrace
-    if (buf == NULL) {
-      printf("ADIOI_OneSidedWriteAggregation - buf is NULL contig_access_count is %d\n",contig_access_count);
-      for (i=0;i<contig_access_count;i++)
-        printf("offset_list[%d] is %ld len_list[%d] is %ld\n",
-		i,offset_list[i],i,len_list[i]);
-    }
-    if (contig_access_count < 0)
-      printf("ADIOI_OneSidedWriteAggregation - contig_access_count "
-	      "of %d is less than 0\n",contig_access_count);
-#endif
-
-    int lenListOverZero = 0;
-    for (i=0;((i<contig_access_count) && (!lenListOverZero));i++)
-      if (len_list[i] > 0)
-        lenListOverZero = 1;
-
-
-    *error_code = MPI_SUCCESS; /* initialize to success */
-
-#ifdef ROMIO_GPFS
-    double startTimeBase,endTimeBase;
-    startTimeBase = MPI_Wtime();
-#endif
-
-    MPI_Status status;
-    pthread_t io_thread;
-    void *thread_ret;
-    ADIOI_IO_ThreadFuncData io_thread_args;
-
-    int nprocs,myrank;
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-#ifdef onesidedtrace
-printf("ADIOI_OneSidedWriteAggregation started on rank %d\n",myrank);
-#endif
-
-
-    if (fd->io_buf_window == MPI_WIN_NULL ||
-	    fd->io_buf_put_amounts_window == MPI_WIN_NULL)
-    {
-	ADIOI_OneSidedSetup(fd, nprocs);
-    }
-
-    /* This flag denotes whether the source datatype is contiguous, which is referenced throughout the algorithm
-     * and defines how the source buffer offsets and data chunks are determined.  If the value is 1 (true - contiguous data)
-     * things are profoundly simpler in that the source buffer offset for a given target offset simply linearly increases
-     * by the chunk sizes being written.  If the value is 0 (non-contiguous) then these values are based on calculations
-     * from the flattened source datatype.
-     */
-    int bufTypeIsContig;
-
-    MPI_Aint bufTypeExtent, lb;
-    ADIOI_Flatlist_node *flatBuf=NULL;
-    ADIOI_Datatype_iscontig(datatype, &bufTypeIsContig);
-
-    if (!bufTypeIsContig) {
-   /* Flatten the non-contiguous source datatype and set the extent. */
-      flatBuf = ADIOI_Flatten_and_find(datatype);
-      MPI_Type_get_extent(datatype, &lb, &bufTypeExtent);
-#ifdef onesidedtrace
-      printf("flatBuf->count is %d bufTypeExtent is %d\n", flatBuf->count,bufTypeExtent);
-      for (i=0;i<flatBuf->count;i++)
-        printf("flatBuf->blocklens[%d] is %d flatBuf->indices[%d] is %ld\n",i,flatBuf->blocklens[i],i,flatBuf->indices[i]);
-#endif
-    }
-
-    int naggs = fd->hints->cb_nodes;
-
-    /* Track the state of the source buffer for feeding the target data blocks.
-     * For GPFS the number of file domains per agg is always 1 so we just need 1 agg
-     * dimension to track the data, in the case of lustre we will need 2 dimensions
-     * agg and file domain since aggs write to multiple file domains in the case of lustre.
-     * This structure will be modified as the data is written to reflect the current state
-     * of the offset.
-     */
-
-#ifdef onesidedtrace
-    printf("sizeof(FDSourceBufferState) is %d - make sure is 32 for 32-byte memalign optimal\n",sizeof(FDSourceBufferState));
-#endif
-    FDSourceBufferState *currentFDSourceBufferState;
-
-    currentFDSourceBufferState = (FDSourceBufferState *) ADIOI_Malloc(naggs * sizeof(FDSourceBufferState));
-    for (i=0;i<naggs;i++) {
-      /* initialize based on the bufType to indicate that it is unset.
-       */
-      if (bufTypeIsContig) {
-        currentFDSourceBufferState[i].sourceBufferOffset = -1;
-      }
-      else {
-        currentFDSourceBufferState[i].indiceOffset = -1;
-      }
-    }
-
-#ifdef onesidedtrace
-    printf(" ADIOI_OneSidedWriteAggregation bufTypeIsContig is %d contig_access_count is %d\n",bufTypeIsContig,contig_access_count);
-#endif
-
-    /* maxNumContigOperations keeps track of how many different chunks we will need to send
-     * for the purpose of pre-allocating the data structures to hold them.
-     */
-    int maxNumContigOperations = contig_access_count;
-
-    ADIO_Offset lastFileOffset = 0, firstFileOffset = -1;
-    /* Get the total range being written - in the case of just 1 byte the starting and ending offsets
-     * will match the same as they would for 0 bytes so to distinguish we need the actual data count.
-     */
-    for (j=0;j<numNonZeroDataOffsets;j++) {
-#ifdef onesidedtrace
-printf("end_offsets[%d] is %ld st_offsets[%d] is %ld\n",j,end_offsets[j],j,st_offsets[j]);
-#endif
-        lastFileOffset = ADIOI_MAX(lastFileOffset,end_offsets[j]);
-        if (firstFileOffset == -1)
-          firstFileOffset = st_offsets[j];
-        else
-          firstFileOffset = ADIOI_MIN(firstFileOffset,st_offsets[j]);
-    }
-
-    int myAggRank = -1; /* if I am an aggregor this is my index into fd->hints->ranklist */
-    int iAmUsedAgg = 0; /* whether or not this rank is used as an aggregator. */
-
-    /* Make coll_bufsize an ADIO_Offset since it is used in calculations with offsets.
-     */
-    ADIO_Offset coll_bufsize = (ADIO_Offset)(fd->hints->cb_buffer_size);
-#ifdef ROMIO_GPFS
-    if (gpfsmpio_pthreadio == 1) {
-      /* split buffer in half for a kind of double buffering with the threads*/
-      coll_bufsize = (ADIO_Offset)(fd->hints->cb_buffer_size/2);
-    }
-#endif
-
-    /* This logic defines values that are used later to determine what offsets define the portion
-     * of the file domain the agg is writing this round.
-     */
-    int greatestFileDomainAggRank = -1,smallestFileDomainAggRank = -1;
-    ADIO_Offset greatestFileDomainOffset = 0;
-    ADIO_Offset smallestFileDomainOffset = lastFileOffset;
-    for (j=0;j<naggs;j++) {
-      if (fd_end[j] > greatestFileDomainOffset) {
-        greatestFileDomainOffset = fd_end[j];
-        greatestFileDomainAggRank = j;
-      }
-      if (fd_start[j] < smallestFileDomainOffset) {
-        smallestFileDomainOffset = fd_start[j];
-        smallestFileDomainAggRank = j;
-      }
-      if (fd->hints->ranklist[j] == myrank) {
-        myAggRank = j;
-        if (fd_end[j] > fd_start[j]) {
-          iAmUsedAgg = 1;
-        }
-      }
-    }
-
-#ifdef onesidedtrace
-    printf("contig_access_count is %d lastFileOffset is %ld firstFileOffset is %ld\n",contig_access_count,lastFileOffset,firstFileOffset);
-    for (j=0;j<contig_access_count;j++) {
-      printf("offset_list[%d]: %ld , len_list[%d]: %ld\n",j,offset_list[j],j,len_list[j]);
-
-    }
-#endif
-
-    /* Compute number of rounds.
-     */
-    int numberOfRounds = 0;
-    for (j=0;j<naggs;j++) {
-	  int currentNumberOfRounds = (int)(((fd_end[j] - fd_start[j])+(ADIO_Offset)1)/coll_bufsize);
-      if (((ADIO_Offset)currentNumberOfRounds*coll_bufsize) < ((fd_end[j] - fd_start[j])+(ADIO_Offset)1))
-        currentNumberOfRounds++;
-	  if (currentNumberOfRounds > numberOfRounds)
-	    numberOfRounds = currentNumberOfRounds;
-    }
-
-    /* Data structures to track what data this compute needs to send to whom.
-     * For lustre they will all need another dimension for the file domain.
-     */
-    int *targetAggsForMyData = (int *)ADIOI_Malloc(naggs * sizeof(int));
-    ADIO_Offset *targetAggsForMyDataFDStart = (ADIO_Offset *)ADIOI_Malloc(naggs * sizeof(ADIO_Offset));
-    ADIO_Offset *targetAggsForMyDataFDEnd = (ADIO_Offset *)ADIOI_Malloc(naggs * sizeof(ADIO_Offset));
-    int numTargetAggs = 0;
-
-    /* This data structure holds the beginning offset and len list index for the range to be written
-     * coresponding to the round and target agg.  Initialize to -1 to denote being unset.
-     */
-    int **targetAggsForMyDataFirstOffLenIndex = (int **)ADIOI_Malloc(numberOfRounds * sizeof(int *));
-    for (i=0;i<numberOfRounds;i++) {
-      targetAggsForMyDataFirstOffLenIndex[i] = (int *)ADIOI_Malloc(naggs * sizeof(int));
-      for (j=0;j<naggs;j++)
-        targetAggsForMyDataFirstOffLenIndex[i][j] = -1;
-    }
-
-    /* This data structure holds the ending offset and len list index for the range to be written
-     * coresponding to the round and target agg.
-     */
-    int **targetAggsForMyDataLastOffLenIndex = (int **)ADIOI_Malloc(numberOfRounds * sizeof(int *));
-    for (i=0;i<numberOfRounds;i++)
-      targetAggsForMyDataLastOffLenIndex[i] = (int *)ADIOI_Malloc(naggs * sizeof(int));
-
-#ifdef onesidedtrace
-   printf("NumberOfRounds is %d\n",numberOfRounds);
-   for (i=0;i<naggs;i++)
-     printf("fd->hints->ranklist[%d]is %d fd_start is %ld fd_end is %ld\n",i,fd->hints->ranklist[i],fd_start[i],fd_end[i]);
-   for (j=0;j<contig_access_count;j++)
-     printf("offset_list[%d] is %ld len_list is %ld\n",j,offset_list[j],len_list[j]);
-#endif
-
-    int currentAggRankListIndex = 0;
-    int maxNumNonContigSourceChunks = 0;
-
-    ADIO_Offset currentSourceBufferOffset = 0;
-    int currentDataTypeExtent = 0;
-    int currentFlatBufIndice=0;
-    ADIO_Offset currentIndiceOffset = 0;
-
-    /* This denotes the coll_bufsize boundaries within the source buffer for writing for the same round.
-     */
-    ADIO_Offset intraRoundCollBufsizeOffset = 0;
-
-    /* This data structure tracks what target aggs need to be written to on what rounds.
-     */
-    int *targetAggsForMyDataCurrentRoundIter = (int *)ADIOI_Malloc(naggs * sizeof(int));
-    for (i=0;i<naggs;i++)
-      targetAggsForMyDataCurrentRoundIter[i] = 0;
-
-    /* This is the first of the two main loops in this algorithm.  The purpose of this loop is essentially to populate
-     * the data structures defined above for what source data blocks needs to go where (target agg and file domain) and when
-     * (round iter).  For lustre essentially an additional layer of nesting will be required for the multiple file domains
-     * within the target agg.
-     */
-    if ((contig_access_count > 0) && (buf != NULL) && lenListOverZero) {
-    int blockIter;
-    for (blockIter=0;blockIter<contig_access_count;blockIter++) {
-
-      /* Determine the starting source buffer offset for this block - for iter 0 skip it since that value is 0.
-       */
-      if (blockIter>0) {
-        if (bufTypeIsContig) {
-          currentSourceBufferOffset += len_list[blockIter-1];
-        }
-        else {
-
-          /* Non-contiguous source datatype, count up the extents and indices to this point
-           * in the blocks for use in computing the source starting buffer offset for target aggs
-           * and file domains.
-           */
-          ADIO_Offset sourceBlockTotal = 0;
-          int lastIndiceUsed = currentFlatBufIndice;
-          int numNonContigSourceChunks = 0;
-
-          while (sourceBlockTotal < len_list[blockIter-1]) {
-            numNonContigSourceChunks++;
-            sourceBlockTotal += (flatBuf->blocklens[currentFlatBufIndice] - currentIndiceOffset);
-            lastIndiceUsed = currentFlatBufIndice;
-            currentFlatBufIndice++;
-            if (currentFlatBufIndice == flatBuf->count) {
-              currentFlatBufIndice = 0;
-              currentDataTypeExtent++;
-            }
-            currentIndiceOffset = (ADIO_Offset)0;
-          }
-          if (sourceBlockTotal > len_list[blockIter-1]) {
-            currentFlatBufIndice--;
-            if (currentFlatBufIndice < 0 ) {
-              currentDataTypeExtent--;
-              currentFlatBufIndice = flatBuf->count-1;
-            }
-            currentIndiceOffset =  len_list[blockIter-1] - (sourceBlockTotal - flatBuf->blocklens[lastIndiceUsed]);
-            // ADIOI_Assert((currentIndiceOffset >= 0) && (currentIndiceOffset < flatBuf->blocklens[currentFlatBufIndice]));
-          }
-          else
-            currentIndiceOffset = (ADIO_Offset)0;
-          maxNumContigOperations += (numNonContigSourceChunks+2);
-          if (numNonContigSourceChunks > maxNumNonContigSourceChunks)
-            maxNumNonContigSourceChunks = numNonContigSourceChunks;
-
-#ifdef onesidedtrace
-          printf("blockiter %d currentFlatBufIndice is now %d currentDataTypeExtent is now %d currentIndiceOffset is now %ld maxNumContigOperations is now %d\n",blockIter,currentFlatBufIndice,currentDataTypeExtent,currentIndiceOffset,maxNumContigOperations);
-#endif
-        } // !bufTypeIsContig
-      } // blockIter > 0
-
-      /* For the last iteration we need to include these maxNumContigOperations and maxNumNonContigSourceChunks
-       * for non-contig case even though we did not need to compute the next starting offset.
-       */
-      if ((blockIter == (contig_access_count-1)) && (!bufTypeIsContig)) {
-        ADIO_Offset sourceBlockTotal = 0;
-        int tmpCurrentFlatBufIndice = currentFlatBufIndice;
-        int  lastNumNonContigSourceChunks = 0;
-        while (sourceBlockTotal < len_list[blockIter]) {
-          lastNumNonContigSourceChunks++;
-          sourceBlockTotal += flatBuf->blocklens[tmpCurrentFlatBufIndice];
-          tmpCurrentFlatBufIndice++;
-          if (tmpCurrentFlatBufIndice == flatBuf->count) {
-            tmpCurrentFlatBufIndice = 0;
-          }
-        }
-        maxNumContigOperations += (lastNumNonContigSourceChunks+2);
-        if (lastNumNonContigSourceChunks > maxNumNonContigSourceChunks)
-          maxNumNonContigSourceChunks = lastNumNonContigSourceChunks;
-
-      }
-
-      ADIO_Offset blockStart = offset_list[blockIter], blockEnd = offset_list[blockIter]+len_list[blockIter]-(ADIO_Offset)1;
-
-      /* Find the starting target agg for this block - normally it will be the current agg so guard the expensive
-       * while loop with a cheap if-check which for large numbers of small blocks will usually be false.
-       */
-      if (!((blockStart >= fd_start[currentAggRankListIndex]) && (blockStart <= fd_end[currentAggRankListIndex]))) {
-        while (!((blockStart >= fd_start[currentAggRankListIndex]) && (blockStart <= fd_end[currentAggRankListIndex])))
-          currentAggRankListIndex++;
-      };
-
-#ifdef onesidedtrace
-      printf("currentAggRankListIndex is %d blockStart %ld blockEnd %ld fd_start[currentAggRankListIndex] %ld fd_end[currentAggRankListIndex] %ld\n",currentAggRankListIndex,blockStart,blockEnd,fd_start[currentAggRankListIndex],fd_end[currentAggRankListIndex]);
-#endif
-
-      /* Determine if this is a new target agg.
-       */
-      if (blockIter>0) {
-        if ((offset_list[blockIter-1]+len_list[blockIter-1]-(ADIO_Offset)1) < fd_start[currentAggRankListIndex]) {
-          numTargetAggs++;
-        }
-      }
-
-       /* Determine which round to start writing - data is written coll_bufsize per round from the aggregator
-        * so if our starting offset in the file domain is multiple coll_bufsize that will correspond to the round.
-        */
-      if ((blockStart - fd_start[currentAggRankListIndex]) >= coll_bufsize) {
-        ADIO_Offset currentRoundBlockStart = fd_start[currentAggRankListIndex];
-        int startingRound = 0;
-        while (blockStart > (currentRoundBlockStart + coll_bufsize - (ADIO_Offset)1)) {
-          currentRoundBlockStart+=coll_bufsize;
-          startingRound++;
-        }
-        targetAggsForMyDataCurrentRoundIter[numTargetAggs] = startingRound;
-      }
-
-      /* Initialize the data structures if this is the first offset in the round/target agg.
-       */
-      if (targetAggsForMyDataFirstOffLenIndex[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] == -1) {
-        targetAggsForMyData[numTargetAggs] = fd->hints->ranklist[currentAggRankListIndex];
-        targetAggsForMyDataFDStart[numTargetAggs] = fd_start[currentAggRankListIndex];
-        /* Round up file domain to the first actual offset used if this is the first file domain.
-         */
-        if (currentAggRankListIndex == smallestFileDomainAggRank) {
-          if (targetAggsForMyDataFDStart[numTargetAggs] < firstFileOffset)
-            targetAggsForMyDataFDStart[numTargetAggs] = firstFileOffset;
-        }
-        targetAggsForMyDataFDEnd[numTargetAggs] = fd_end[currentAggRankListIndex];
-        /* Round down file domain to the last actual offset used if this is the last file domain.
-         */
-        if (currentAggRankListIndex == greatestFileDomainAggRank) {
-          if (targetAggsForMyDataFDEnd[numTargetAggs] > lastFileOffset)
-            targetAggsForMyDataFDEnd[numTargetAggs] = lastFileOffset;
-        }
-        targetAggsForMyDataFirstOffLenIndex[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] = blockIter;
-	/* Set the source buffer state starting point for data access for this
-           agg and file domain.  */
-
-        if (bufTypeIsContig) {
-          if (currentFDSourceBufferState[numTargetAggs].sourceBufferOffset == -1) {
-
-            currentFDSourceBufferState[numTargetAggs].sourceBufferOffset = currentSourceBufferOffset;
-#ifdef onesidedtrace
-            printf("For agg %d sourceBufferOffset initialized to %ld\n",currentAggRankListIndex,currentSourceBufferOffset);
-#endif
-          }
-        }
-        else {
-          if (currentFDSourceBufferState[numTargetAggs].indiceOffset == -1) {
-            currentFDSourceBufferState[numTargetAggs].indiceOffset = currentIndiceOffset;
-            currentFDSourceBufferState[numTargetAggs].bufTypeExtent = bufTypeExtent;
-            currentFDSourceBufferState[numTargetAggs].dataTypeExtent = currentDataTypeExtent;
-            currentFDSourceBufferState[numTargetAggs].flatBufIndice = currentFlatBufIndice;
-#ifdef onesidedtrace
-            printf("For agg %d dataTypeExtent initialized to %d flatBufIndice to %d indiceOffset to %ld\n",numTargetAggs,currentDataTypeExtent,currentFlatBufIndice,currentIndiceOffset);
-#endif
-          }
-        }
-
-        intraRoundCollBufsizeOffset = fd_start[currentAggRankListIndex] + ((ADIO_Offset)(targetAggsForMyDataCurrentRoundIter[numTargetAggs]+1) * coll_bufsize);
-
-#ifdef onesidedtrace
-        printf("Initial settings numTargetAggs %d offset_list[%d] with value %ld past fd border %ld with len %ld currentSourceBufferOffset set to %ld intraRoundCollBufsizeOffset set to %ld\n",numTargetAggs,blockIter,offset_list[blockIter],fd_start[currentAggRankListIndex],len_list[blockIter],currentSourceBufferOffset,intraRoundCollBufsizeOffset);
-#endif
-      }
-
-      /* Replace the last offset block iter with this one.
-       */
-      targetAggsForMyDataLastOffLenIndex[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] = blockIter;
-
-      /* If this blocks extends into the next file domain advance to the next target aggs and source buffer states.
-       */
-      if (blockEnd > fd_end[currentAggRankListIndex]) {
-#ifdef onesidedtrace
-      printf("block extends past current fd, blockEnd %ld >= fd_end[currentAggRankListIndex] %ld total block size is %ld blockStart was %ld\n",blockEnd,fd_end[currentAggRankListIndex], len_list[blockIter],blockStart);
-#endif
-        ADIO_Offset amountToAdvanceSBOffsetForFD = 0;
-        int additionalFDCounter = 0;
-
-        while (blockEnd >= fd_end[currentAggRankListIndex]) {
-          ADIO_Offset thisAggBlockEnd = fd_end[currentAggRankListIndex];
-          if (thisAggBlockEnd >= intraRoundCollBufsizeOffset) {
-            while (thisAggBlockEnd >= intraRoundCollBufsizeOffset) {
-              targetAggsForMyDataCurrentRoundIter[numTargetAggs]++;
-              intraRoundCollBufsizeOffset += coll_bufsize;
-              targetAggsForMyDataFirstOffLenIndex[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] = blockIter;
-              targetAggsForMyDataLastOffLenIndex[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] = blockIter;
-#ifdef onesidedtrace
-              printf("targetAggsForMyDataCurrentRoundI%d] is now %d intraRoundCollBufsizeOffset is now %ld\n",numTargetAggs,targetAggsForMyDataCurrentRoundIter[numTargetAggs],intraRoundCollBufsizeOffset);
-#endif
-            } // while (thisAggBlockEnd >= intraRoundCollBufsizeOffset)
-          } // if (thisAggBlockEnd >= intraRoundCollBufsizeOffset)
-
-          int prevAggRankListIndex = currentAggRankListIndex;
-          currentAggRankListIndex++;
-
-          /* Skip over unused aggs.
-           */
-          if (fd_start[currentAggRankListIndex] > fd_end[currentAggRankListIndex]) {
-            while (fd_start[currentAggRankListIndex] > fd_end[currentAggRankListIndex])
-              currentAggRankListIndex++;
-
-          } // (fd_start[currentAggRankListIndex] > fd_end[currentAggRankListIndex])
-
-          /* Start new target agg.
-           */
-          if (blockEnd >= fd_start[currentAggRankListIndex]) {
-            numTargetAggs++;
-            targetAggsForMyData[numTargetAggs] = fd->hints->ranklist[currentAggRankListIndex];
-            targetAggsForMyDataFDStart[numTargetAggs] = fd_start[currentAggRankListIndex];
-            /* Round up file domain to the first actual offset used if this is the first file domain.
-             */
-            if (currentAggRankListIndex == smallestFileDomainAggRank) {
-              if (targetAggsForMyDataFDStart[numTargetAggs] < firstFileOffset)
-                targetAggsForMyDataFDStart[numTargetAggs] = firstFileOffset;
-            }
-            targetAggsForMyDataFDEnd[numTargetAggs] = fd_end[currentAggRankListIndex];
-            /* Round down file domain to the last actual offset used if this is the last file domain.
-             */
-            if (currentAggRankListIndex == greatestFileDomainAggRank) {
-              if (targetAggsForMyDataFDEnd[numTargetAggs] > lastFileOffset)
-                targetAggsForMyDataFDEnd[numTargetAggs] = lastFileOffset;
-            }
-            targetAggsForMyDataFirstOffLenIndex[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] = blockIter;
-	    /* For the first additonal file domain the source buffer offset
-	     * will be incremented relative to the state of this first main
-	     * loop but for subsequent full file domains the offset will be
-	     * incremented by the size
-             * of the file domain.
-             */
-            if (additionalFDCounter == 0)
-              amountToAdvanceSBOffsetForFD = (fd_end[prevAggRankListIndex]
-		      - blockStart) + (ADIO_Offset)1;
-            else
-              amountToAdvanceSBOffsetForFD = (fd_end[prevAggRankListIndex]
-		      -fd_start[prevAggRankListIndex]) +(ADIO_Offset)1;
-
-            if (bufTypeIsContig) {
-              ADIOI_Assert(numTargetAggs > 0);
-              if (currentFDSourceBufferState[numTargetAggs].sourceBufferOffset == -1) {
-                if (additionalFDCounter == 0) { // first file domain, still use the current data counter
-                  currentFDSourceBufferState[numTargetAggs].sourceBufferOffset =
-		      currentSourceBufferOffset+amountToAdvanceSBOffsetForFD;
-                }
-                else { // 2nd file domain, advance full file domain from last source buffer state
-                  currentFDSourceBufferState[numTargetAggs].sourceBufferOffset =
-		      currentFDSourceBufferState[numTargetAggs-1].sourceBufferOffset+amountToAdvanceSBOffsetForFD;
-                }
-
-#ifdef onesidedtrace
-            printf("Crossed into new FD - for agg %d sourceBufferOffset initialized to %ld amountToAdvanceSBOffsetForFD is %ld\n",numTargetAggs,currentFDSourceBufferState[numTargetAggs].sourceBufferOffset,amountToAdvanceSBOffsetForFD);
-#endif
-              }
-            }
-            else if (currentFDSourceBufferState[numTargetAggs].indiceOffset == -1) {
-		// non-contiguos source buffer
-              ADIOI_Assert(numTargetAggs > 0);
-
-	      /* Initialize the source buffer state appropriately and then
-	       * advance it with the
-               * nonContigSourceDataBufferAdvance function.
-               */
-              if (additionalFDCounter == 0) {
-		  // first file domain, still use the current data counter
-                currentFDSourceBufferState[numTargetAggs].indiceOffset =
-		    currentIndiceOffset;
-                currentFDSourceBufferState[numTargetAggs].bufTypeExtent = bufTypeExtent;
-                currentFDSourceBufferState[numTargetAggs].dataTypeExtent =
-		    currentDataTypeExtent;
-                currentFDSourceBufferState[numTargetAggs].flatBufIndice =
-		    currentFlatBufIndice;
-              }
-              else {
-		  // 2nd file domain, advance full file domain from last source buffer state
-                currentFDSourceBufferState[numTargetAggs].indiceOffset =
-		    currentFDSourceBufferState[numTargetAggs-1].indiceOffset;
-                currentFDSourceBufferState[numTargetAggs].bufTypeExtent =
-		    currentFDSourceBufferState[numTargetAggs-1].bufTypeExtent;
-                currentFDSourceBufferState[numTargetAggs].dataTypeExtent =
-		    currentFDSourceBufferState[numTargetAggs-1].dataTypeExtent;
-                currentFDSourceBufferState[numTargetAggs].flatBufIndice =
-		    currentFDSourceBufferState[numTargetAggs-1].flatBufIndice;
-              }
-              nonContigSourceDataBufferAdvance(((char*)buf), flatBuf,
-		      (int)amountToAdvanceSBOffsetForFD, 1,
-		      &currentFDSourceBufferState[numTargetAggs], NULL);
-#ifdef onesidedtrace
-              printf("Crossed into new FD - for agg %d dataTypeExtent initialized to %d flatBufIndice to %d indiceOffset to %ld amountToAdvanceSBOffsetForFD is %d\n",numTargetAggs,currentFDSourceBufferState[numTargetAggs].dataTypeExtent,currentFDSourceBufferState[numTargetAggs].flatBufIndice,currentFDSourceBufferState[numTargetAggs].indiceOffset,amountToAdvanceSBOffsetForFD);
-#endif
-	    }
-	    additionalFDCounter++;
-
-#ifdef onesidedtrace
-            printf("block extended beyond fd init settings numTargetAggs %d offset_list[%d] with value %ld past fd border %ld with len %ld\n",numTargetAggs,i,offset_list[blockIter],fd_start[currentAggRankListIndex],len_list[blockIter]);
-#endif
-            intraRoundCollBufsizeOffset = fd_start[currentAggRankListIndex] + coll_bufsize;
-            targetAggsForMyDataLastOffLenIndex[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] = blockIter;
-          } // if (blockEnd >= fd_start[currentAggRankListIndex])
-        } // while (blockEnd >= fd_end[currentAggRankListIndex])
-      } // if (blockEnd > fd_end[currentAggRankListIndex])
-
-      /* If we are still in the same file domain / target agg but have gone
-       * past the coll_bufsize and need to advance to the next round -
-       * initialize tracking data appropriately.
-       */
-      if (blockEnd >= intraRoundCollBufsizeOffset) {
-        ADIO_Offset currentBlockEnd = blockEnd;
-        while (currentBlockEnd >= intraRoundCollBufsizeOffset) {
-          targetAggsForMyDataCurrentRoundIter[numTargetAggs]++;
-          intraRoundCollBufsizeOffset += coll_bufsize;
-          targetAggsForMyDataFirstOffLenIndex[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] = blockIter;
-          targetAggsForMyDataLastOffLenIndex[targetAggsForMyDataCurrentRoundIter[numTargetAggs]][numTargetAggs] = blockIter;
-#ifdef onesidedtrace
-        printf("smaller than fd currentBlockEnd is now %ld intraRoundCollBufsizeOffset is now %ld targetAggsForMyDataCurrentRoundIter[%d] is now %d\n",currentBlockEnd, intraRoundCollBufsizeOffset, numTargetAggs,targetAggsForMyDataCurrentRoundIter[numTargetAggs]);
-#endif
-        } // while (currentBlockEnd >= intraRoundCollBufsizeOffset)
-      } // if (blockEnd >= intraRoundCollBufsizeOffset)
-
-      /* Need to advance numTargetAggs if this is the last target offset to
-       * include this one.
-       */
-      if (blockIter == (contig_access_count-1))
-        numTargetAggs++;
-    }
-
-#ifdef onesidedtrace
-    printf("numTargetAggs is %d\n",numTargetAggs);
-    for (i=0;i<numTargetAggs;i++) {
-      for (j=0;j<=targetAggsForMyDataCurrentRoundIter[i];j++)
-        printf("targetAggsForMyData[%d] is %d targetAggsForMyDataFDStart[%d] is %ld targetAggsForMyDataFDEnd is %ld targetAggsForMyDataFirstOffLenIndex is %d with value %ld targetAggsForMyDataLastOffLenIndex is %d with value %ld\n",i,targetAggsForMyData[i],i,targetAggsForMyDataFDStart[i],targetAggsForMyDataFDEnd[i],targetAggsForMyDataFirstOffLenIndex[j][i],offset_list[targetAggsForMyDataFirstOffLenIndex[j][i]],targetAggsForMyDataLastOffLenIndex[j][i],offset_list[targetAggsForMyDataLastOffLenIndex[j][i]]);
-
-    }
-#endif
-
-    } // if ((contig_access_count > 0) && (buf != NULL) && lenListOverZero)
-
-    ADIOI_Free(targetAggsForMyDataCurrentRoundIter);
-
-    int currentWriteBuf = 0;
-    int useIOBuffer = 0;
-#ifdef ROMIO_GPFS
-    if (gpfsmpio_pthreadio && (numberOfRounds>1)) {
-    useIOBuffer = 1;
-    io_thread = pthread_self();
-    }
-#endif
-
-    /* use the write buffer allocated in the file_open */
-    char *write_buf0 = fd->io_buf;
-    char *write_buf1 = fd->io_buf + coll_bufsize;
-
-    /* start off pointing to the first buffer. If we use the 2nd buffer (threaded
-     * case) we'll swap later */
-    char *write_buf = write_buf0;
-    MPI_Win write_buf_window = fd->io_buf_window;
-
-    int *write_buf_put_amounts = fd->io_buf_put_amounts;
-    if(!gpfsmpio_onesided_no_rmw) {
-      *hole_found = 0;
-      for (i=0;i<nprocs;i++)
-        write_buf_put_amounts[i] = 0;
-    }
-
-    /* Counters to track the offset range being written by the used aggs.
-     */
-    ADIO_Offset currentRoundFDStart = 0;
-    ADIO_Offset currentRoundFDEnd = 0;
-
-    if (iAmUsedAgg) {
-      currentRoundFDStart = fd_start[myAggRank];
-      if (myAggRank == smallestFileDomainAggRank) {
-        if (currentRoundFDStart < firstFileOffset)
-          currentRoundFDStart = firstFileOffset;
-      }
-      else if (myAggRank == greatestFileDomainAggRank) {
-        if (currentRoundFDEnd > lastFileOffset)
-          currentRoundFDEnd = lastFileOffset;
-      }
-#ifdef onesidedtrace
-printf("iAmUsedAgg - currentRoundFDStart initialized to %ld currentRoundFDEnd to %ld\n",currentRoundFDStart,currentRoundFDEnd);
-#endif
-      if (gpfsmpio_onesided_always_rmw) { // read in the first buffer
-        ADIO_Offset tmpCurrentRoundFDEnd = 0;
-        if ((fd_end[myAggRank] - currentRoundFDStart) < coll_bufsize) {
-          if (myAggRank == greatestFileDomainAggRank) {
-            if (fd_end[myAggRank] > lastFileOffset)
-              tmpCurrentRoundFDEnd = lastFileOffset;
-            else
-              tmpCurrentRoundFDEnd = fd_end[myAggRank];
-          }
-          else
-            tmpCurrentRoundFDEnd = fd_end[myAggRank];
-        }
-        else
-        tmpCurrentRoundFDEnd = currentRoundFDStart + coll_bufsize - (ADIO_Offset)1;
-#ifdef onesidedtrace
-printf("gpfsmpio_onesided_always_rmw - first buffer pre-read for file offsets %ld to %ld total is %d\n",currentRoundFDStart,tmpCurrentRoundFDEnd,(int)(tmpCurrentRoundFDEnd - currentRoundFDStart)+1);
-#endif
-        ADIO_ReadContig(fd, write_buf, (int)(tmpCurrentRoundFDEnd - currentRoundFDStart)+1,
-          MPI_BYTE, ADIO_EXPLICIT_OFFSET,currentRoundFDStart, &status, error_code);
-
-      }
-    }
-    if (gpfsmpio_onesided_always_rmw) // wait until the first buffer is read
-      MPI_Barrier(fd->comm);
-
-#ifdef ROMIO_GPFS
-    endTimeBase = MPI_Wtime();
-    gpfsmpio_prof_cw[GPFSMPIO_CIO_T_DEXCH_SETUP] += (endTimeBase-startTimeBase);
-    startTimeBase = MPI_Wtime();
-#endif
-
-    /* This is the second main loop of the algorithm, actually nested loop of target aggs within rounds.  There are 2 flavors of this.
-     * For gpfsmpio_write_aggmethod of 1 each nested iteration for the target
-     * agg does an mpi_put on a contiguous chunk using a primative datatype
-     * determined using the data structures from the first main loop.  For
-     * gpfsmpio_write_aggmethod of 2 each nested iteration for the target agg
-     * builds up data to use in created a derived data type for 1 mpi_put that is done for the target agg for each round.
-     * To support lustre there will need to be an additional layer of nesting
-     * for the multiple file domains within target aggs.
-     */
-    int roundIter;
-
-    for (roundIter=0;roundIter<numberOfRounds;roundIter++) {
-	if ((contig_access_count > 0) && (buf != NULL) && lenListOverZero) {
-
-
-    int aggIter;
-    for (aggIter=0;aggIter<numTargetAggs;aggIter++) {
-
-    int numBytesPutThisAggRound = 0;
-    /* If we have data for the round/agg process it.
-     */
-    if (targetAggsForMyDataFirstOffLenIndex[roundIter][aggIter] != -1) {
-      ADIO_Offset currentRoundFDStartForMyTargetAgg = (ADIO_Offset)((ADIO_Offset)targetAggsForMyDataFDStart[aggIter] + (ADIO_Offset)((ADIO_Offset)roundIter*coll_bufsize));
-      ADIO_Offset currentRoundFDEndForMyTargetAgg = (ADIO_Offset)((ADIO_Offset)targetAggsForMyDataFDStart[aggIter] + (ADIO_Offset)((ADIO_Offset)(roundIter+1)*coll_bufsize) - (ADIO_Offset)1);
-
-      int targetAggContigAccessCount = 0;
-
-      /* These data structures are used for the derived datatype mpi_put
-       * in the gpfsmpio_write_aggmethod of 2 case.
-       */
-      int *targetAggBlockLengths=NULL;
-      MPI_Aint *targetAggDisplacements=NULL, *sourceBufferDisplacements=NULL;
-      MPI_Datatype *targetAggDataTypes=NULL;
-
-      char *derivedTypePackedSourceBuffer=NULL;
-      int derivedTypePackedSourceBufferOffset = 0;
-      int allocatedDerivedTypeArrays = 0;
-      ADIO_Offset amountOfDataWrittenThisRoundAgg = 0;
-
-#ifdef onesidedtrace
-      printf("roundIter %d processing targetAggsForMyData %d \n",roundIter,targetAggsForMyData[aggIter]);
-#endif
-
-      /* Process the range of offsets for this target agg.
-       */
-      int offsetIter;
-      int startingOffLenIndex = targetAggsForMyDataFirstOffLenIndex[roundIter][aggIter], endingOffLenIndex = targetAggsForMyDataLastOffLenIndex[roundIter][aggIter];
-      for (offsetIter=startingOffLenIndex;offsetIter<=endingOffLenIndex;offsetIter++) {
-        if (currentRoundFDEndForMyTargetAgg > targetAggsForMyDataFDEnd[aggIter])
-            currentRoundFDEndForMyTargetAgg = targetAggsForMyDataFDEnd[aggIter];
-
-        ADIO_Offset offsetStart = offset_list[offsetIter], offsetEnd = (offset_list[offsetIter]+len_list[offsetIter]-(ADIO_Offset)1);
-
-#ifdef onesidedtrace
-        printf("roundIter %d target iter %d targetAggsForMyData is %d offset_list[%d] is %ld len_list[%d] is %ld targetAggsForMyDataFDStart is %ld targetAggsForMyDataFDEnd is %ld currentRoundFDStartForMyTargetAgg is %ld currentRoundFDEndForMyTargetAgg is %ld targetAggsForMyDataFirstOffLenIndex is %ld\n",
-            roundIter,aggIter,targetAggsForMyData[aggIter],offsetIter,offset_list[offsetIter],offsetIter,len_list[offsetIter],
-            targetAggsForMyDataFDStart[aggIter],targetAggsForMyDataFDEnd[aggIter],
-            currentRoundFDStartForMyTargetAgg,currentRoundFDEndForMyTargetAgg, targetAggsForMyDataFirstOffLenIndex[roundIter][aggIter]);
-#endif
-
-        /* Determine the amount of data and exact source buffer offsets to use.
-         */
-        int bufferAmountToSend = 0;
-
-        if ((offsetStart >= currentRoundFDStartForMyTargetAgg) && (offsetStart <= currentRoundFDEndForMyTargetAgg)) {
-            if (offsetEnd > currentRoundFDEndForMyTargetAgg)
-            bufferAmountToSend = (currentRoundFDEndForMyTargetAgg - offsetStart) +1;
-            else
-            bufferAmountToSend = (offsetEnd - offsetStart) +1;
-        }
-        else if ((offsetEnd >= currentRoundFDStartForMyTargetAgg) && (offsetEnd <= currentRoundFDEndForMyTargetAgg)) {
-            if (offsetEnd > currentRoundFDEndForMyTargetAgg)
-            bufferAmountToSend = (currentRoundFDEndForMyTargetAgg - currentRoundFDStartForMyTargetAgg) +1;
-            else
-            bufferAmountToSend = (offsetEnd - currentRoundFDStartForMyTargetAgg) +1;
-            if (offsetStart < currentRoundFDStartForMyTargetAgg) {
-              offsetStart = currentRoundFDStartForMyTargetAgg;
-            }
-        }
-        else if ((offsetStart <= currentRoundFDStartForMyTargetAgg) && (offsetEnd >= currentRoundFDEndForMyTargetAgg)) {
-            bufferAmountToSend = (currentRoundFDEndForMyTargetAgg - currentRoundFDStartForMyTargetAgg) +1;
-            offsetStart = currentRoundFDStartForMyTargetAgg;
-        }
-
-        numBytesPutThisAggRound += bufferAmountToSend;
-#ifdef onesidedtrace
-        printf("bufferAmountToSend is %d\n",bufferAmountToSend);
-#endif
-        if (bufferAmountToSend > 0) { /* we have data to send this round */
-          if (gpfsmpio_write_aggmethod == 2) {
-            /* Only allocate these arrays if we are using method 2 and only do it once for this round/target agg.
-             */
-            if (!allocatedDerivedTypeArrays) {
-              targetAggBlockLengths = (int *)ADIOI_Malloc(maxNumContigOperations * sizeof(int));
-              targetAggDisplacements = (MPI_Aint *)ADIOI_Malloc(maxNumContigOperations * sizeof(MPI_Aint));
-              sourceBufferDisplacements = (MPI_Aint *)ADIOI_Malloc(maxNumContigOperations * sizeof(MPI_Aint));
-              targetAggDataTypes = (MPI_Datatype *)ADIOI_Malloc(maxNumContigOperations * sizeof(MPI_Datatype));
-              if (!bufTypeIsContig) {
-                int k;
-                for (k=targetAggsForMyDataFirstOffLenIndex[roundIter][aggIter];k<=targetAggsForMyDataLastOffLenIndex[roundIter][aggIter];k++)
-                  amountOfDataWrittenThisRoundAgg += len_list[k];
-
-#ifdef onesidedtrace
-                printf("derivedTypePackedSourceBuffer mallocing %ld\n",amountOfDataWrittenThisRoundAgg);
-#endif
-                if (amountOfDataWrittenThisRoundAgg > 0)
-                  derivedTypePackedSourceBuffer = (char *)ADIOI_Malloc(amountOfDataWrittenThisRoundAgg * sizeof(char));
-                else
-                  derivedTypePackedSourceBuffer = NULL;
-              }
-              allocatedDerivedTypeArrays = 1;
-            }
-          }
-
-          /* Determine the offset into the target window.
-           */
-          MPI_Aint targetDisplacementToUseThisRound = (MPI_Aint) (offsetStart - currentRoundFDStartForMyTargetAgg);
-
-          /* If using the thread writer select the appropriate side of the split window.
-           */
-          if (useIOBuffer && (write_buf == write_buf1)) {
-            targetDisplacementToUseThisRound += (MPI_Aint) coll_bufsize;
-          }
-
-          /* For gpfsmpio_write_aggmethod of 1 do the mpi_put using the primitive MPI_BYTE type for each contiguous
-           * chunk in the target, of source data is non-contiguous then pack the data first.
-           */
-
-          if (gpfsmpio_write_aggmethod == 1) {
-            MPI_Win_lock(MPI_LOCK_SHARED, targetAggsForMyData[aggIter], 0, write_buf_window);
-            if (bufTypeIsContig) {
-              MPI_Put(((char*)buf) + currentFDSourceBufferState[aggIter].sourceBufferOffset,bufferAmountToSend, MPI_BYTE,targetAggsForMyData[aggIter],targetDisplacementToUseThisRound, bufferAmountToSend,MPI_BYTE,write_buf_window);
-              currentFDSourceBufferState[aggIter].sourceBufferOffset += (ADIO_Offset)bufferAmountToSend;
-            }
-            else {
-              char *putSourceData = (char *) ADIOI_Malloc(bufferAmountToSend*sizeof(char));
-              nonContigSourceDataBufferAdvance(((char*)buf), flatBuf, bufferAmountToSend, 1, &currentFDSourceBufferState[aggIter], putSourceData);
-              MPI_Put(putSourceData,bufferAmountToSend, MPI_BYTE,targetAggsForMyData[aggIter],targetDisplacementToUseThisRound, bufferAmountToSend,MPI_BYTE,write_buf_window);
-              ADIOI_Free(putSourceData);
-            }
-            MPI_Win_unlock(targetAggsForMyData[aggIter], write_buf_window);
-          }
-
-          /* For gpfsmpio_write_aggmethod of 2 populate the data structures for this round/agg for this offset iter
-           * to be used subsequently when building the derived type for 1 mpi_put for all the data for this
-           * round/agg.
-           */
-          else if (gpfsmpio_write_aggmethod == 2) {
-
-            if (bufTypeIsContig) {
-              targetAggBlockLengths[targetAggContigAccessCount]= bufferAmountToSend;
-              targetAggDataTypes[targetAggContigAccessCount] = MPI_BYTE;
-              targetAggDisplacements[targetAggContigAccessCount] = targetDisplacementToUseThisRound;
-              sourceBufferDisplacements[targetAggContigAccessCount] = (MPI_Aint)currentFDSourceBufferState[aggIter].sourceBufferOffset;
-              currentFDSourceBufferState[aggIter].sourceBufferOffset += (ADIO_Offset)bufferAmountToSend;
-              targetAggContigAccessCount++;
-            }
-            else {
-              nonContigSourceDataBufferAdvance(((char*)buf), flatBuf, bufferAmountToSend, 1, &currentFDSourceBufferState[aggIter], &derivedTypePackedSourceBuffer[derivedTypePackedSourceBufferOffset]);
-              targetAggBlockLengths[targetAggContigAccessCount]= bufferAmountToSend;
-              targetAggDataTypes[targetAggContigAccessCount] = MPI_BYTE;
-              targetAggDisplacements[targetAggContigAccessCount] = targetDisplacementToUseThisRound;
-              sourceBufferDisplacements[targetAggContigAccessCount] = (MPI_Aint)derivedTypePackedSourceBufferOffset;
-              targetAggContigAccessCount++;
-              derivedTypePackedSourceBufferOffset += (ADIO_Offset)bufferAmountToSend;
-            }
-          }
-#ifdef onesidedtrace
-        printf("roundIter %d bufferAmountToSend is %d offsetStart is %ld currentRoundFDStartForMyTargetAgg is %ld targetDisplacementToUseThisRound is %ld targetAggsForMyDataFDStart[aggIter] is %ld\n",roundIter, bufferAmountToSend, offsetStart,currentRoundFDStartForMyTargetAgg,targetDisplacementToUseThisRound,targetAggsForMyDataFDStart[aggIter]);
-#endif
-
-        } // bufferAmountToSend > 0
-      } // contig list
-
-      /* For gpfsmpio_write_aggmethod of 2 now build the derived type using the data from this round/agg and do 1 single mpi_put.
-       */
-      if (gpfsmpio_write_aggmethod == 2) {
-
-        MPI_Datatype sourceBufferDerivedDataType, targetBufferDerivedDataType;
-        MPI_Type_create_struct(targetAggContigAccessCount, targetAggBlockLengths, sourceBufferDisplacements, targetAggDataTypes, &sourceBufferDerivedDataType);
-        MPI_Type_commit(&sourceBufferDerivedDataType);
-        MPI_Type_create_struct(targetAggContigAccessCount, targetAggBlockLengths, targetAggDisplacements, targetAggDataTypes, &targetBufferDerivedDataType);
-        MPI_Type_commit(&targetBufferDerivedDataType);
-
-#ifdef onesidedtrace
-        printf("mpi_put of derived type to agg %d targetAggContigAccessCount is %d\n",targetAggsForMyData[aggIter],targetAggContigAccessCount);
-#endif
-        if (targetAggContigAccessCount > 0) {
-        MPI_Win_lock(MPI_LOCK_SHARED, targetAggsForMyData[aggIter], 0, write_buf_window);
-        if (bufTypeIsContig) {
-          MPI_Put(((char*)buf),1, sourceBufferDerivedDataType,targetAggsForMyData[aggIter],0, 1,targetBufferDerivedDataType,write_buf_window);
-        }
-        else {
-          MPI_Put(derivedTypePackedSourceBuffer,1, sourceBufferDerivedDataType,targetAggsForMyData[aggIter],0, 1,targetBufferDerivedDataType,write_buf_window);
-        }
-
-
-        MPI_Win_unlock(targetAggsForMyData[aggIter], write_buf_window);
-        }
-
-        if (allocatedDerivedTypeArrays) {
-          ADIOI_Free(targetAggBlockLengths);
-          ADIOI_Free(targetAggDisplacements);
-          ADIOI_Free(targetAggDataTypes);
-          ADIOI_Free(sourceBufferDisplacements);
-          if (!bufTypeIsContig)
-            if (derivedTypePackedSourceBuffer != NULL)
-              ADIOI_Free(derivedTypePackedSourceBuffer);
-        }
-        if (targetAggContigAccessCount > 0) {
-        MPI_Type_free(&sourceBufferDerivedDataType);
-        MPI_Type_free(&targetBufferDerivedDataType);
-        }
-      }
-      if (!gpfsmpio_onesided_no_rmw) {
-        MPI_Win_lock(MPI_LOCK_SHARED, targetAggsForMyData[aggIter], 0, fd->io_buf_put_amounts_window);
-        MPI_Put(&numBytesPutThisAggRound,1, MPI_INT,targetAggsForMyData[aggIter],myrank, 1,MPI_INT,fd->io_buf_put_amounts_window);
-        MPI_Win_unlock(targetAggsForMyData[aggIter], fd->io_buf_put_amounts_window);
-      }
-      } // baseoffset != -1
-    } // target aggs
-	} /// contig_access_count > 0
-
-#ifdef onesidedtrace
-printf("first barrier roundIter %d\n",roundIter);
-#endif
-    MPI_Barrier(fd->comm);
-
-    if (iAmUsedAgg) {
-    /* Determine what offsets define the portion of the file domain the agg is writing this round.
-     */
-        if ((fd_end[myAggRank] - currentRoundFDStart) < coll_bufsize) {
-          if (myAggRank == greatestFileDomainAggRank) {
-            if (fd_end[myAggRank] > lastFileOffset)
-              currentRoundFDEnd = lastFileOffset;
-            else
-              currentRoundFDEnd = fd_end[myAggRank];
-          }
-          else
-            currentRoundFDEnd = fd_end[myAggRank];
-        }
-        else
-        currentRoundFDEnd = currentRoundFDStart + coll_bufsize - (ADIO_Offset)1;
-
-#ifdef onesidedtrace
-        printf("used agg about to writecontig - currentRoundFDStart is %ld currentRoundFDEnd is %ld within file domeain %ld to %ld\n",currentRoundFDStart,currentRoundFDEnd,fd_start[myAggRank],fd_end[myAggRank]);
-#endif
-
-        int doWriteContig = 1;
-        if (!gpfsmpio_onesided_no_rmw) {
-          int numBytesPutIntoBuf = 0;
-          for (i=0;i<nprocs;i++) {
-            numBytesPutIntoBuf += write_buf_put_amounts[i];
-            write_buf_put_amounts[i] = 0;
-          }
-          if (numBytesPutIntoBuf != ((int)(currentRoundFDEnd - currentRoundFDStart)+1)) {
-            doWriteContig = 0;
-            *hole_found = 1;
-          }
-        }
-
-        if (!useIOBuffer) {
-          if (doWriteContig)
-            ADIO_WriteContig(fd, write_buf, (int)(currentRoundFDEnd - currentRoundFDStart)+1,
-              MPI_BYTE, ADIO_EXPLICIT_OFFSET,currentRoundFDStart, &status, error_code);
-
-        } else { /* use the thread writer */
-
-        if(!pthread_equal(io_thread, pthread_self())) {
-            pthread_join(io_thread, &thread_ret);
-            *error_code = *(int *)thread_ret;
-            if (*error_code != MPI_SUCCESS) return;
-            io_thread = pthread_self();
-
-        }
-        io_thread_args.fd = fd;
-        /* do a little pointer shuffling: background I/O works from one
-         * buffer while two-phase machinery fills up another */
-
-        if (currentWriteBuf == 0) {
-            io_thread_args.buf = write_buf0;
-            currentWriteBuf = 1;
-            write_buf = write_buf1;
-        }
-        else {
-            io_thread_args.buf = write_buf1;
-            currentWriteBuf = 0;
-            write_buf = write_buf0;
-        }
-        if (doWriteContig) {
-        io_thread_args.io_kind = ADIOI_WRITE;
-        io_thread_args.size = (currentRoundFDEnd-currentRoundFDStart) + 1;
-        io_thread_args.offset = currentRoundFDStart;
-        io_thread_args.status = &status;
-        io_thread_args.error_code = *error_code;
-
-        if ( (pthread_create(&io_thread, NULL,
-                ADIOI_IO_Thread_Func, &(io_thread_args))) != 0)
-            io_thread = pthread_self();
-        }
-        } // useIOBuffer
-
-    } // iAmUsedAgg
-
-    if (!iAmUsedAgg && useIOBuffer) {
-        if (currentWriteBuf == 0) {
-            currentWriteBuf = 1;
-            write_buf = write_buf1;
-        }
-        else {
-            currentWriteBuf = 0;
-            write_buf = write_buf0;
-        }
-    }
-
-    if (iAmUsedAgg) {
-      currentRoundFDStart += coll_bufsize;
-
-      if (gpfsmpio_onesided_always_rmw && (roundIter<(numberOfRounds-1))) { // read in the buffer for the next round unless this is the last round
-        ADIO_Offset tmpCurrentRoundFDEnd = 0;
-        if ((fd_end[myAggRank] - currentRoundFDStart) < coll_bufsize) {
-          if (myAggRank == greatestFileDomainAggRank) {
-            if (fd_end[myAggRank] > lastFileOffset)
-              tmpCurrentRoundFDEnd = lastFileOffset;
-            else
-              tmpCurrentRoundFDEnd = fd_end[myAggRank];
-          }
-          else
-            tmpCurrentRoundFDEnd = fd_end[myAggRank];
-        }
-        else
-        tmpCurrentRoundFDEnd = currentRoundFDStart + coll_bufsize - (ADIO_Offset)1;
-#ifdef onesidedtrace
-printf("gpfsmpio_onesided_always_rmw - round %d buffer pre-read for file offsets %ld to %ld total is %d\n",roundIter, currentRoundFDStart,tmpCurrentRoundFDEnd,(int)(tmpCurrentRoundFDEnd - currentRoundFDStart)+1);
-#endif
-        ADIO_ReadContig(fd, write_buf, (int)(tmpCurrentRoundFDEnd - currentRoundFDStart)+1,
-          MPI_BYTE, ADIO_EXPLICIT_OFFSET,currentRoundFDStart, &status, error_code);
-
-      }
-    }
-
-    if (roundIter<(numberOfRounds-1)) {
-#ifdef onesidedtrace
-printf("second barrier roundIter %d\n",roundIter);
-#endif
-      MPI_Barrier(fd->comm);
-    }
-
-    } /* for-loop roundIter */
-
-
-#ifdef ROMIO_GPFS
-    endTimeBase = MPI_Wtime();
-    gpfsmpio_prof_cw[GPFSMPIO_CIO_T_DEXCH] += (endTimeBase-startTimeBase);
-#endif
-
-    if (useIOBuffer) { /* thread writer cleanup */
-
-    if ( !pthread_equal(io_thread, pthread_self()) ) {
-        pthread_join(io_thread, &thread_ret);
-        *error_code = *(int *)thread_ret;
-    }
-
-    }
-
-#ifdef onesidedtrace
-printf("freeing datastructures\n");
-#endif
-    ADIOI_Free(targetAggsForMyData);
-    ADIOI_Free(targetAggsForMyDataFDStart);
-    ADIOI_Free(targetAggsForMyDataFDEnd);
-
-    for (i=0;i<numberOfRounds;i++) {
-      ADIOI_Free(targetAggsForMyDataFirstOffLenIndex[i]);
-      ADIOI_Free(targetAggsForMyDataLastOffLenIndex[i]);
-    }
-    ADIOI_Free(targetAggsForMyDataFirstOffLenIndex);
-    ADIOI_Free(targetAggsForMyDataLastOffLenIndex);
-
-    ADIOI_Free(currentFDSourceBufferState);
-
-    if (!bufTypeIsContig)
-      ADIOI_Delete_flattened(datatype);
-    return;
-}
-
-
-void ADIOI_OneSidedReadAggregation(ADIO_File fd,
-    ADIO_Offset *offset_list,
-    ADIO_Offset *len_list,
-    int contig_access_count,
-    const void *buf,
-    MPI_Datatype datatype,
-    int *error_code,
-    ADIO_Offset *st_offsets,
-    ADIO_Offset *end_offsets,
-    int numNonZeroDataOffsets,
-    ADIO_Offset *fd_start,
-    ADIO_Offset* fd_end)
-{
-    int i,j; /* generic iterators */
-
-#ifdef onesidedtrace
-    if (buf == NULL) {
-      printf("ADIOI_OneSidedWriteAggregation - buf is NULL contig_access_count is %d\n",contig_access_count);
-      for (i=0;i<contig_access_count;i++)
-        printf("offset_list[%d] is %ld len_list[%d] is %ld\n",i,offset_list[i],i,len_list[i]);
-    }
-    if (contig_access_count < 0)
-      printf("ADIOI_OneSidedWriteAggregation - contig_access_count of %d is less than 0\n",contig_access_count);
-#endif
-
-    int lenListOverZero = 0;
-    for (i=0;((i<contig_access_count) && (!lenListOverZero));i++)
-      if (len_list[i] > 0)
-        lenListOverZero = 1;
-
-    *error_code = MPI_SUCCESS; /* initialize to success */
-
-#ifdef ROMIO_GPFS
-    double startTimeBase,endTimeBase;
-    startTimeBase = MPI_Wtime();
-#endif
-
-    MPI_Status status;
-    pthread_t io_thread;
-    void *thread_ret;
-    ADIOI_IO_ThreadFuncData io_thread_args;
-
-    int nprocs,myrank;
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-
-#ifdef onesidedtrace
-printf("ADIOI_OneSidedReadAggregation started on rank %d\n",myrank);
-#endif
-
-    if (fd->io_buf_window == MPI_WIN_NULL ||
-	    fd->io_buf_put_amounts_window == MPI_WIN_NULL)
-    {
-	ADIOI_OneSidedSetup(fd, nprocs);
-    }
-    /* This flag denotes whether the source datatype is contiguus, which is referenced throughout the algorithm
-     * and defines how the source buffer offsets and data chunks are determined.  If the value is 1 (true - contiguous data)
-     * things are profoundly simpler in that the source buffer offset for a given source offset simply linearly increases
-     * by the chunk sizes being read.  If the value is 0 (non-contiguous) then these values are based on calculations
-     * from the flattened source datatype.
-     */
-    int bufTypeIsContig;
-
-    MPI_Aint bufTypeExtent, lb;
-    ADIOI_Flatlist_node *flatBuf=NULL;
-    ADIOI_Datatype_iscontig(datatype, &bufTypeIsContig);
-
-    if (!bufTypeIsContig) {
-    /* Flatten the non-contiguous source datatype.
-     */
-      flatBuf = ADIOI_Flatten_and_find(datatype);
-      MPI_Type_get_extent(datatype, &lb, &bufTypeExtent);
-#ifdef onesidedtrace
-      printf("flatBuf->count is %d bufTypeExtent is %d\n", flatBuf->count,bufTypeExtent);
-      for (i=0;i<flatBuf->count;i++)
-        printf("flatBuf->blocklens[%d] is %d flatBuf->indices[%d] is %ld\n",i,flatBuf->blocklens[i],i,flatBuf->indices[i]);
-#endif
-    }
-#ifdef onesidedtrace
-      printf("ADIOI_OneSidedReadAggregation bufTypeIsContig is %d contig_access_count is %d\n",bufTypeIsContig,contig_access_count);
-#endif
-
-    int naggs = fd->hints->cb_nodes;
-
-    /* Track the state of the source buffer for feeding the target data blocks.
-     * For GPFS the number of file domains per agg is always 1 so we just need 1 agg
-     * dimension to track the data, in the case of lustre we will need 2 dimensions
-     * agg and file domain since aggs write to multiple file domains in the
-     * case of lustre.
-     * This structure will be modified as the data is written to reflect the
-     * current state of the offset.
-     */
-
-#ifdef onesidedtrace
-    printf("sizeof(FDSourceBufferState) is %d - make sure is 32 for 32-byte memalign optimal\n",sizeof(FDSourceBufferState));
-#endif
-    FDSourceBufferState *currentFDSourceBufferState;
-
-    currentFDSourceBufferState = (FDSourceBufferState *) ADIOI_Malloc(naggs * sizeof(FDSourceBufferState));
-    for (i=0;i<naggs;i++) {
-      /* initialize based on the bufType to indicate that it is unset.
-       */
-      if (bufTypeIsContig) {
-        currentFDSourceBufferState[i].sourceBufferOffset = -1;
-      }
-      else {
-        currentFDSourceBufferState[i].indiceOffset = -1;
-      }
-    }
-
-#ifdef onesidedtrace
-    printf(" ADIOI_OneSidedReadAggregation bufTypeIsContig is %d contig_access_count is %d\n",bufTypeIsContig,contig_access_count);
-#endif
-
-    /* maxNumContigOperations keeps track of how many different chunks we will
-     * need to recv for the purpose of pre-allocating the data structures to
-     * hold them.
-     */
-    int maxNumContigOperations = contig_access_count;
-
-
-    ADIO_Offset lastFileOffset = 0, firstFileOffset = -1;
-
-    /* Get the total range being read.
-     */
-    for (j=0;j<numNonZeroDataOffsets;j++) {
-#ifdef onesidedtrace
-printf("end_offsets[%d] is %ld st_offsets[%d] is %ld\n",j,end_offsets[j],j,st_offsets[j]);
-#endif
-        lastFileOffset = ADIOI_MAX(lastFileOffset,end_offsets[j]);
-        if (firstFileOffset == -1)
-          firstFileOffset = st_offsets[j];
-        else
-          firstFileOffset = ADIOI_MIN(firstFileOffset,st_offsets[j]);
-    }
-
-    int myAggRank = -1; /* if I am an aggregor this is my index into fd->hints->ranklist */
-    int iAmUsedAgg = 0; /* whether or not this rank is used as an aggregator. */
-
-    int coll_bufsize = fd->hints->cb_buffer_size;
-#ifdef ROMIO_GPFS
-    if (gpfsmpio_pthreadio == 1) {
-    /* split buffer in half for a kind of double buffering with the threads*/
-    coll_bufsize = fd->hints->cb_buffer_size/2;
-    }
-#endif
-
-    /* This logic defines values that are used later to determine what offsets define the portion
-     * of the file domain the agg is reading this round.
-     */
-    int greatestFileDomainAggRank = -1,smallestFileDomainAggRank = -1;
-    ADIO_Offset greatestFileDomainOffset = 0;
-    ADIO_Offset smallestFileDomainOffset = lastFileOffset;
-    for (j=0;j<naggs;j++) {
-      if (fd_end[j] > greatestFileDomainOffset) {
-        greatestFileDomainOffset = fd_end[j];
-        greatestFileDomainAggRank = j;
-      }
-      if (fd_start[j] < smallestFileDomainOffset) {
-        smallestFileDomainOffset = fd_start[j];
-        smallestFileDomainAggRank = j;
-      }
-      if (fd->hints->ranklist[j] == myrank) {
-        myAggRank = j;
-        if (fd_end[j] > fd_start[j]) {
-          iAmUsedAgg = 1;
-        }
-      }
-    }
-
-#ifdef onesidedtrace
-    printf("contig_access_count is %d lastFileOffset is %ld firstFileOffset is %ld\n",contig_access_count,lastFileOffset,firstFileOffset);
-    for (j=0;j<contig_access_count;j++) {
-      printf("offset_list[%d]: %ld , len_list[%d]: %ld\n",j,offset_list[j],j,len_list[j]);
-    }
-#endif
-
-    /* Compute number of rounds.
-     */
-    int numberOfRounds = 0;
-    for (j=0;j<naggs;j++) {
-	  int currentNumberOfRounds = (int)(((fd_end[j] - fd_start[j])+(ADIO_Offset)1)/coll_bufsize);
-      if (((ADIO_Offset)currentNumberOfRounds*coll_bufsize) < ((fd_end[j] - fd_start[j])+(ADIO_Offset)1))
-        currentNumberOfRounds++;
-	  if (currentNumberOfRounds > numberOfRounds)
-	    numberOfRounds = currentNumberOfRounds;
-    }
-
-    /* Data structures to track what data this compute needs to receive from whom.
-     * For lustre they will all need another dimension for the file domain.
-     */    int *sourceAggsForMyData = (int *)ADIOI_Malloc(naggs * sizeof(int));
-    ADIO_Offset *sourceAggsForMyDataFDStart = (ADIO_Offset *)ADIOI_Malloc(naggs * sizeof(ADIO_Offset));
-    ADIO_Offset *sourceAggsForMyDataFDEnd = (ADIO_Offset *)ADIOI_Malloc(naggs * sizeof(ADIO_Offset));
-    int numSourceAggs = 0;
-
-    /* This data structure holds the beginning offset and len list index for the range to be read
-     * coresponding to the round and source agg. Initialize to -1 to denote being unset.
-    */
-    int **sourceAggsForMyDataFirstOffLenIndex = (int **)ADIOI_Malloc(numberOfRounds * sizeof(int *));
-    for (i=0;i<numberOfRounds;i++) {
-      sourceAggsForMyDataFirstOffLenIndex[i] = (int *)ADIOI_Malloc(naggs * sizeof(int));
-      for (j=0;j<naggs;j++)
-        sourceAggsForMyDataFirstOffLenIndex[i][j] = -1;
-    }
-
-    /* This data structure holds the ending offset and len list index for the range to be read
-     * coresponding to the round and source agg.
-    */
-    int **sourceAggsForMyDataLastOffLenIndex = (int **)ADIOI_Malloc(numberOfRounds * sizeof(int *));
-    for (i=0;i<numberOfRounds;i++)
-      sourceAggsForMyDataLastOffLenIndex[i] = (int *)ADIOI_Malloc(naggs * sizeof(int));
-
-#ifdef onesidedtrace
-    printf("NumberOfRounds is %d\n",numberOfRounds);
-    for (i=0;i<naggs;i++)
-      printf("fd->hints->ranklist[%d]is %d fd_start is %ld fd_end is %ld\n",i,fd->hints->ranklist[i],fd_start[i],fd_end[i]);
-    for (j=0;j<contig_access_count;j++)
-      printf("offset_list[%d] is %ld len_list is %ld\n",j,offset_list[j],len_list[j]);
-#endif
-
-    int currentAggRankListIndex = 0;
-    int maxNumNonContigSourceChunks = 0;
-
-    ADIO_Offset currentRecvBufferOffset = 0;
-    int currentDataTypeExtent = 0;
-    int currentFlatBufIndice=0;
-    ADIO_Offset currentIndiceOffset = 0;
-
-    /* This denotes the coll_bufsize boundaries within the source buffer for reading for 1 round.
-     */
-    ADIO_Offset intraRoundCollBufsizeOffset = 0;
-
-    /* This data structure tracks what source aggs need to be read to on what rounds.
-     */
-    int *sourceAggsForMyDataCurrentRoundIter = (int *)ADIOI_Malloc(naggs * sizeof(int));
-    for (i=0;i<naggs;i++)
-      sourceAggsForMyDataCurrentRoundIter[i] = 0;
-
-    /* This is the first of the two main loops in this algorithm.  The purpose of this loop is essentially to populate
-     * the data structures defined above for what read data blocks needs to go where (source agg and file domain) and when
-     * (round iter).  For lustre essentially an additional layer of nesting will be required for the multiple file domains
-     * within the source agg.
-     */
-     if ((contig_access_count > 0) && (buf != NULL) && lenListOverZero) {
-    int blockIter;
-    for (blockIter=0;blockIter<contig_access_count;blockIter++) {
-
-      /* Determine the starting source buffer offset for this block - for iter 0 skip it since that value is 0.
-       */
-      if (blockIter>0) {
-        if (bufTypeIsContig) {
-          currentRecvBufferOffset += len_list[blockIter-1];
-        }
-        else {
-          /* Non-contiguous source datatype, count up the extents and indices to this point
-           * in the blocks.
-           */
-          ADIO_Offset sourceBlockTotal = 0;
-          int lastIndiceUsed = currentFlatBufIndice;
-          int numNonContigSourceChunks = 0;
-          while (sourceBlockTotal < len_list[blockIter-1]) {
-            numNonContigSourceChunks++;
-            sourceBlockTotal += (flatBuf->blocklens[currentFlatBufIndice] - currentIndiceOffset);
-            lastIndiceUsed = currentFlatBufIndice;
-            currentFlatBufIndice++;
-            if (currentFlatBufIndice == flatBuf->count) {
-              currentFlatBufIndice = 0;
-              currentDataTypeExtent++;
-            }
-            currentIndiceOffset = (ADIO_Offset)0;
-          }
-          if (sourceBlockTotal > len_list[blockIter-1]) {
-            currentFlatBufIndice--;
-            if (currentFlatBufIndice < 0 ) {
-              currentDataTypeExtent--;
-              currentFlatBufIndice = flatBuf->count-1;
-            }
-            currentIndiceOffset =  len_list[blockIter-1] - (sourceBlockTotal - flatBuf->blocklens[lastIndiceUsed]);
-            // ADIOI_Assert((currentIndiceOffset >= 0) && (currentIndiceOffset < flatBuf->blocklens[currentFlatBufIndice]));
-          }
-          else
-            currentIndiceOffset = 0;
-          maxNumContigOperations += (numNonContigSourceChunks+2);
-          if (numNonContigSourceChunks > maxNumNonContigSourceChunks)
-            maxNumNonContigSourceChunks = numNonContigSourceChunks;
-
-#ifdef onesidedtrace
-          printf("block iter %d currentFlatBufIndice is now %d currentDataTypeExtent is now %d currentIndiceOffset is now %ld maxNumContigOperations is now %d\n",blockIter,currentFlatBufIndice,currentDataTypeExtent,currentIndiceOffset,maxNumContigOperations);
-#endif
-        } // !bufTypeIsContig
-      } // blockIter > 0
-
-      /* For the last iteration we need to include these maxNumContigOperations and maxNumNonContigSourceChunks
-       * for non-contig case even though we did not need to compute the next starting offset.
-       */
-      if ((blockIter == (contig_access_count-1)) && (!bufTypeIsContig)) {
-        ADIO_Offset sourceBlockTotal = 0;
-        int tmpCurrentFlatBufIndice = currentFlatBufIndice;
-        int  lastNumNonContigSourceChunks = 0;
-        while (sourceBlockTotal < len_list[blockIter]) {
-          lastNumNonContigSourceChunks++;
-          sourceBlockTotal += flatBuf->blocklens[tmpCurrentFlatBufIndice];
-          tmpCurrentFlatBufIndice++;
-          if (tmpCurrentFlatBufIndice == flatBuf->count) {
-            tmpCurrentFlatBufIndice = 0;
-          }
-        }
-        maxNumContigOperations += (lastNumNonContigSourceChunks+2);
-        if (lastNumNonContigSourceChunks > maxNumNonContigSourceChunks)
-          maxNumNonContigSourceChunks = lastNumNonContigSourceChunks;
-
-      }
-
-      ADIO_Offset blockStart = offset_list[blockIter], blockEnd = offset_list[blockIter]+len_list[blockIter]-(ADIO_Offset)1;
-
-      /* Find the starting source agg for this block - normally it will be the current agg so guard the expensive
-       * while loop with a cheap if-check which for large numbers of small blocks will usually be false.
-       */
-      if (!((blockStart >= fd_start[currentAggRankListIndex]) && (blockStart <= fd_end[currentAggRankListIndex]))) {
-        while (!((blockStart >= fd_start[currentAggRankListIndex]) && (blockStart <= fd_end[currentAggRankListIndex])))
-          currentAggRankListIndex++;
-      };
-
-      /* Determine if this is a new source agg.
-       */
-      if (blockIter>0) {
-        if ((offset_list[blockIter-1]+len_list[blockIter-1]-(ADIO_Offset)1) < fd_start[currentAggRankListIndex])
-          numSourceAggs++;
-      }
-
-       /* Determine which round to start reading.
-        */
-      if ((blockStart - fd_start[currentAggRankListIndex]) >= coll_bufsize) {
-        ADIO_Offset currentRoundBlockStart = fd_start[currentAggRankListIndex];
-        int startingRound = 0;
-        while (blockStart > (currentRoundBlockStart + coll_bufsize - (ADIO_Offset)1)) {
-          currentRoundBlockStart+=coll_bufsize;
-          startingRound++;
-        }
-        sourceAggsForMyDataCurrentRoundIter[numSourceAggs] = startingRound;
-      }
-
-      /* Initialize the data structures if this is the first offset in the round/source agg.
-       */
-      if (sourceAggsForMyDataFirstOffLenIndex[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] == -1) {
-        sourceAggsForMyData[numSourceAggs] = fd->hints->ranklist[currentAggRankListIndex];
-        sourceAggsForMyDataFDStart[numSourceAggs] = fd_start[currentAggRankListIndex];
-        /* Round up file domain to the first actual offset used if this is the first file domain.
-         */
-        if (currentAggRankListIndex == smallestFileDomainAggRank) {
-          if (sourceAggsForMyDataFDStart[numSourceAggs] < firstFileOffset)
-            sourceAggsForMyDataFDStart[numSourceAggs] = firstFileOffset;
-        }
-        sourceAggsForMyDataFDEnd[numSourceAggs] = fd_end[currentAggRankListIndex];
-        /* Round down file domain to the last actual offset used if this is the last file domain.
-         */
-        if (currentAggRankListIndex == greatestFileDomainAggRank) {
-          if (sourceAggsForMyDataFDEnd[numSourceAggs] > lastFileOffset)
-            sourceAggsForMyDataFDEnd[numSourceAggs] = lastFileOffset;
-        }
-        sourceAggsForMyDataFirstOffLenIndex[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] = blockIter;
-
-        /* Set the source buffer state starting point for data access for this agg and file domain.
-         */
-        if (bufTypeIsContig) {
-          if (currentFDSourceBufferState[numSourceAggs].sourceBufferOffset == -1) {
-
-            currentFDSourceBufferState[numSourceAggs].sourceBufferOffset = currentRecvBufferOffset;
-#ifdef onesidedtrace
-            printf("For agg %d sourceBufferOffset initialized to %ld\n",currentAggRankListIndex,currentRecvBufferOffset);
-#endif
-          }
-        }
-	else {
-          if (currentFDSourceBufferState[numSourceAggs].indiceOffset == -1) {
-            currentFDSourceBufferState[numSourceAggs].indiceOffset = currentIndiceOffset;
-            currentFDSourceBufferState[numSourceAggs].bufTypeExtent = bufTypeExtent;
-            currentFDSourceBufferState[numSourceAggs].dataTypeExtent = currentDataTypeExtent;
-            currentFDSourceBufferState[numSourceAggs].flatBufIndice = currentFlatBufIndice;
-#ifdef onesidedtrace
-            printf("For agg %d dataTypeExtent initialized to %d flatBufIndice to %d indiceOffset to %ld\n",numSourceAggs,currentDataTypeExtent,currentFlatBufIndice,currentIndiceOffset);
-#endif
-          }
-	}
-        intraRoundCollBufsizeOffset = fd_start[currentAggRankListIndex] + ((ADIO_Offset)(sourceAggsForMyDataCurrentRoundIter[numSourceAggs]+1) * coll_bufsize);
-#ifdef onesidedtrace
-        printf("init settings numSourceAggs %d offset_list[%d] with value %ld past fd border %ld with len %ld currentRecvBufferOffset set to %ld intraRoundCollBufsizeOffset set to %ld\n",numSourceAggs,blockIter,offset_list[blockIter],fd_start[currentAggRankListIndex],len_list[blockIter],currentRecvBufferOffset,intraRoundCollBufsizeOffset);
-#endif
-
-      }
-
-      /* Replace the last offset block iter with this one.
-       */
-      sourceAggsForMyDataLastOffLenIndex[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] = blockIter;
-
-      /* If this blocks extends into the next file domain advance to the next source aggs and source buffer states.
-       */
-      if (blockEnd > fd_end[currentAggRankListIndex]) {
-#ifdef onesidedtrace
-      printf("block extends past current fd, blockEnd %ld >= fd_end[currentAggRankListIndex] %ld total block size is %ld blockStart was %ld\n",blockEnd,fd_end[currentAggRankListIndex], len_list[blockIter],blockStart);
-#endif
-        ADIO_Offset amountToAdvanceSBOffsetForFD = 0;
-        int additionalFDCounter = 0;
-        while (blockEnd >= fd_end[currentAggRankListIndex]) {
-          ADIO_Offset thisAggBlockEnd = fd_end[currentAggRankListIndex];
-          if (thisAggBlockEnd >= intraRoundCollBufsizeOffset) {
-            while (thisAggBlockEnd >= intraRoundCollBufsizeOffset) {
-              sourceAggsForMyDataCurrentRoundIter[numSourceAggs]++;
-              intraRoundCollBufsizeOffset += coll_bufsize;
-              sourceAggsForMyDataFirstOffLenIndex[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] = blockIter;
-              sourceAggsForMyDataLastOffLenIndex[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] = blockIter;
-#ifdef onesidedtrace
-              printf("sourceAggsForMyDataCurrentRoundI%d] is now %d intraRoundCollBufsizeOffset is now %ld\n",numSourceAggs,sourceAggsForMyDataCurrentRoundIter[numSourceAggs],intraRoundCollBufsizeOffset);
-#endif
-            } // while (thisAggBlockEnd >= intraRoundCollBufsizeOffset)
-          } // if (thisAggBlockEnd >= intraRoundCollBufsizeOffset)
-
-          int prevAggRankListIndex = currentAggRankListIndex;
-          currentAggRankListIndex++;
-
-          /* Skip over unused aggs.
-           */
-          if (fd_start[currentAggRankListIndex] > fd_end[currentAggRankListIndex]) {
-            while (fd_start[currentAggRankListIndex] > fd_end[currentAggRankListIndex])
-              currentAggRankListIndex++;
-
-          } // (fd_start[currentAggRankListIndex] > fd_end[currentAggRankListIndex])
-
-          /* Start new source agg.
-           */
-          if (blockEnd >= fd_start[currentAggRankListIndex]) {
-            numSourceAggs++;
-            sourceAggsForMyData[numSourceAggs] = fd->hints->ranklist[currentAggRankListIndex];
-            sourceAggsForMyDataFDStart[numSourceAggs] = fd_start[currentAggRankListIndex];
-            /* Round up file domain to the first actual offset used if this is the first file domain.
-             */
-            if (currentAggRankListIndex == smallestFileDomainAggRank) {
-              if (sourceAggsForMyDataFDStart[numSourceAggs] < firstFileOffset)
-                sourceAggsForMyDataFDStart[numSourceAggs] = firstFileOffset;
-            }
-            sourceAggsForMyDataFDEnd[numSourceAggs] = fd_end[currentAggRankListIndex];
-            /* Round down file domain to the last actual offset used if this is the last file domain.
-             */
-            if (currentAggRankListIndex == greatestFileDomainAggRank) {
-              if (sourceAggsForMyDataFDEnd[numSourceAggs] > lastFileOffset)
-                sourceAggsForMyDataFDEnd[numSourceAggs] = lastFileOffset;
-            }
-            sourceAggsForMyDataFirstOffLenIndex[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] = blockIter;
-
-
-	    /* For the first additonal file domain the source buffer offset
-	     * will be incremented relative to the state of this first main
-	     * loop but for subsequent full file domains the offset will be
-	     * incremented by the size of the file domain.
-             */
-            if (additionalFDCounter == 0)
-              amountToAdvanceSBOffsetForFD = (fd_end[prevAggRankListIndex] - blockStart) + (ADIO_Offset)1;
-            else
-              amountToAdvanceSBOffsetForFD = (fd_end[prevAggRankListIndex]-fd_start[prevAggRankListIndex]) +(ADIO_Offset)1;
-
-            if (bufTypeIsContig) {
-              ADIOI_Assert(numSourceAggs > 0);
-              if (currentFDSourceBufferState[numSourceAggs].sourceBufferOffset == -1) {
-                if (additionalFDCounter == 0) { // first file domain, still use the current data counter
-                  currentFDSourceBufferState[numSourceAggs].sourceBufferOffset = currentRecvBufferOffset+amountToAdvanceSBOffsetForFD;
-                }
-                else { // 2nd file domain, advance full file domain from last source buffer state
-                  currentFDSourceBufferState[numSourceAggs].sourceBufferOffset = currentFDSourceBufferState[numSourceAggs-1].sourceBufferOffset+amountToAdvanceSBOffsetForFD;
-                }
-
-#ifdef onesidedtrace
-            printf("Crossed into new FD - for agg %d sourceBufferOffset initialized to %ld amountToAdvanceSBOffsetForFD is %ld\n",numSourceAggs,currentFDSourceBufferState[numSourceAggs].sourceBufferOffset,amountToAdvanceSBOffsetForFD);
-#endif
-              }
-            }
-            else if (currentFDSourceBufferState[numSourceAggs].indiceOffset == -1) {
-		// non-contiguos source buffer
-              ADIOI_Assert(numSourceAggs > 0);
-
-	      /* Initialize the source buffer state appropriately and then
-	       * advance it with the nonContigSourceDataBufferAdvance function.
-               */
-              if (additionalFDCounter == 0) {
-		  // first file domain, still use the current data counter
-                currentFDSourceBufferState[numSourceAggs].indiceOffset = currentIndiceOffset;
-                currentFDSourceBufferState[numSourceAggs].bufTypeExtent = bufTypeExtent;
-                currentFDSourceBufferState[numSourceAggs].dataTypeExtent = currentDataTypeExtent;
-                currentFDSourceBufferState[numSourceAggs].flatBufIndice = currentFlatBufIndice;
-              }
-              else {
-		  // 2nd file domain, advance full file domain from last source
-		  // buffer state
-                currentFDSourceBufferState[numSourceAggs].indiceOffset = currentFDSourceBufferState[numSourceAggs-1].indiceOffset;
-                currentFDSourceBufferState[numSourceAggs].bufTypeExtent = currentFDSourceBufferState[numSourceAggs-1].bufTypeExtent;
-                currentFDSourceBufferState[numSourceAggs].dataTypeExtent = currentFDSourceBufferState[numSourceAggs-1].dataTypeExtent;
-                currentFDSourceBufferState[numSourceAggs].flatBufIndice = currentFDSourceBufferState[numSourceAggs-1].flatBufIndice;
-              }
-              nonContigSourceDataBufferAdvance(((char*)buf), flatBuf, (int)amountToAdvanceSBOffsetForFD, 0, &currentFDSourceBufferState[numSourceAggs], NULL);
-#ifdef onesidedtrace
-              printf("Crossed into new FD - for agg %d dataTypeExtent initialized to %d flatBufIndice to %d indiceOffset to %ld amountToAdvanceSBOffsetForFD is %d\n",numSourceAggs,currentFDSourceBufferState[numSourceAggs].dataTypeExtent,currentFDSourceBufferState[numSourceAggs].flatBufIndice,currentFDSourceBufferState[numSourceAggs].indiceOffset,amountToAdvanceSBOffsetForFD);
-#endif
-            }
-
-            additionalFDCounter++;
- 
-
-#ifdef onesidedtrace
-            printf("block extended beyond fd init settings numSourceAggs %d offset_list[%d] with value %ld past fd border %ld with len %ld\n",numSourceAggs,i,offset_list[blockIter],fd_start[currentAggRankListIndex],len_list[blockIter]);
-#endif
-            intraRoundCollBufsizeOffset = fd_start[currentAggRankListIndex] + coll_bufsize;
-            sourceAggsForMyDataLastOffLenIndex[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] = blockIter;
-          } // if (blockEnd >= fd_start[currentAggRankListIndex])
-        } // while (blockEnd >= fd_end[currentAggRankListIndex])
-      } // if (blockEnd > fd_end[currentAggRankListIndex])
-
-      /* If we are still in the same file domain / source agg but have gone past the coll_bufsize and need
-       * to advance to the next round handle this situation.
-       */
-      if (blockEnd >= intraRoundCollBufsizeOffset) {
-        ADIO_Offset currentBlockEnd = blockEnd;
-        while (currentBlockEnd >= intraRoundCollBufsizeOffset) {
-          sourceAggsForMyDataCurrentRoundIter[numSourceAggs]++;
-          intraRoundCollBufsizeOffset += coll_bufsize;
-          sourceAggsForMyDataFirstOffLenIndex[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] = blockIter;
-          sourceAggsForMyDataLastOffLenIndex[sourceAggsForMyDataCurrentRoundIter[numSourceAggs]][numSourceAggs] = blockIter;
-#ifdef onesidedtrace
-          printf("block less than fd currentBlockEnd is now %ld intraRoundCollBufsizeOffset is now %ld sourceAggsForMyDataCurrentRoundIter[%d] is now %d\n",currentBlockEnd, intraRoundCollBufsizeOffset, numSourceAggs,sourceAggsForMyDataCurrentRoundIter[numSourceAggs]);
-#endif
-        } // while (currentBlockEnd >= intraRoundCollBufsizeOffset)
-      } // if (blockEnd >= intraRoundCollBufsizeOffset)
-
-      /* Need to advance numSourceAggs if this is the last source offset to
-       * include this one.
-       */
-      if (blockIter == (contig_access_count-1))
-        numSourceAggs++;
-    }
-
-#ifdef onesidedtrace
-    printf("numSourceAggs is %d\n",numSourceAggs);
-    for (i=0;i<numSourceAggs;i++) {
-      for (j=0;j<=sourceAggsForMyDataCurrentRoundIter[i];j++)
-        printf("sourceAggsForMyData[%d] is %d sourceAggsForMyDataFDStart[%d] is %ld sourceAggsForMyDataFDEnd is %ld sourceAggsForMyDataFirstOffLenIndex is %d with value %ld sourceAggsForMyDataLastOffLenIndex is %d with value %ld\n",i,sourceAggsForMyData[i],i,sourceAggsForMyDataFDStart[i],sourceAggsForMyDataFDEnd[i],sourceAggsForMyDataFirstOffLenIndex[j][i],offset_list[sourceAggsForMyDataFirstOffLenIndex[j][i]],sourceAggsForMyDataLastOffLenIndex[j][i],offset_list[sourceAggsForMyDataLastOffLenIndex[j][i]]);
-
-    }
-#endif
-
-    } // if ((contig_access_count > 0) && (buf != NULL) && lenListOverZero)
-
-    ADIOI_Free(sourceAggsForMyDataCurrentRoundIter);
-
-    /* use the two-phase buffer allocated in the file_open - no app should ever
-     * be both reading and reading at the same time */
-    char *read_buf0 = fd->io_buf;
-    char *read_buf1 = fd->io_buf + coll_bufsize;
-    /* if threaded i/o selected, we'll do a kind of double buffering */
-    char *read_buf = read_buf0;
-
-    int currentReadBuf = 0;
-    int useIOBuffer = 0;
-#ifdef ROMIO_GPFS
-    if (gpfsmpio_pthreadio && (numberOfRounds>1)) {
-    useIOBuffer = 1;
-    io_thread = pthread_self();
-    }
-#endif
-
-    MPI_Win read_buf_window = fd->io_buf_window;
-
-    ADIO_Offset currentRoundFDStart = 0, nextRoundFDStart = 0;
-    ADIO_Offset currentRoundFDEnd = 0, nextRoundFDEnd = 0;
-
-    if (iAmUsedAgg) {
-      currentRoundFDStart = fd_start[myAggRank];
-      nextRoundFDStart = fd_start[myAggRank];
-      if (myAggRank == smallestFileDomainAggRank) {
-        if (currentRoundFDStart < firstFileOffset)
-          currentRoundFDStart = firstFileOffset;
-        if (nextRoundFDStart < firstFileOffset)
-          nextRoundFDStart = firstFileOffset;
-      }
-      else if (myAggRank == greatestFileDomainAggRank) {
-        if (currentRoundFDEnd > lastFileOffset)
-          currentRoundFDEnd = lastFileOffset;
-        if (nextRoundFDEnd > lastFileOffset)
-          nextRoundFDEnd = lastFileOffset;
-      }
-#ifdef onesidedtrace
-printf("iAmUsedAgg - currentRoundFDStart initialized "
-	"to %ld currentRoundFDEnd to %ld\n",
-	currentRoundFDStart,currentRoundFDEnd);
-#endif
-
-
-    }
-
-#ifdef ROMIO_GPFS
-    endTimeBase = MPI_Wtime();
-    gpfsmpio_prof_cw[GPFSMPIO_CIO_T_DEXCH_SETUP] += (endTimeBase-startTimeBase);
-    startTimeBase = MPI_Wtime();
-#endif
-
-
-    /* This is the second main loop of the algorithm, actually nested loop of target aggs within rounds.  There are 2 flavors of this.
-     * For gpfsmpio_read_aggmethod of 1 each nested iteration for the source agg does an mpi_put on a contiguous chunk using a primative datatype
-     * determined using the data structures from the first main loop.  For gpfsmpio_read_aggmethod of 2 each nested iteration for the source agg
-     * builds up data to use in created a derived data type for 1 mpi_put that is done for the target agg for each round.
-     * To support lustre there will need to be an additional layer of nesting for the multiple file domains
-     * within target aggs.
-     */
-    int roundIter;
-    for (roundIter=0;roundIter<numberOfRounds;roundIter++) {
-
-    if ((contig_access_count > 0) && (buf != NULL) && lenListOverZero)
-    {
-    /* determine what offsets define the portion of the file domain the agg is reading this round */
-    if (iAmUsedAgg) {
-
-        currentRoundFDStart = nextRoundFDStart;
-
-        if (!useIOBuffer || (roundIter == 0)) {
-        int amountDataToReadThisRound;
-        if ((fd_end[myAggRank] - currentRoundFDStart) < coll_bufsize) {
-            currentRoundFDEnd = fd_end[myAggRank];
-            amountDataToReadThisRound = ((currentRoundFDEnd-currentRoundFDStart)+1);
-        }
-        else {
-            currentRoundFDEnd = currentRoundFDStart + coll_bufsize - (ADIO_Offset)1;
-            amountDataToReadThisRound = coll_bufsize;
-        }
-
-        /* read currentRoundFDEnd bytes */
-        ADIO_ReadContig(fd, read_buf,amountDataToReadThisRound,
-            MPI_BYTE, ADIO_EXPLICIT_OFFSET, currentRoundFDStart,
-            &status, error_code);
-        currentReadBuf = 1;
-
-        }
-        if (useIOBuffer) { /* use the thread reader for the next round */
-        /* switch back and forth between the read buffers so that the data aggregation code is diseminating 1 buffer while the thread is reading into the other */
-
-        if (roundIter > 0)
-            currentRoundFDEnd = nextRoundFDEnd;
-
-        if (roundIter < (numberOfRounds-1)) {
-            nextRoundFDStart += coll_bufsize;
-            int amountDataToReadNextRound;
-            if ((fd_end[myAggRank] - nextRoundFDStart) < coll_bufsize) {
-            nextRoundFDEnd = fd_end[myAggRank];
-            amountDataToReadNextRound = ((nextRoundFDEnd-nextRoundFDStart)+1);
-            }
-            else {
-            nextRoundFDEnd = nextRoundFDStart + coll_bufsize - (ADIO_Offset)1;
-            amountDataToReadNextRound = coll_bufsize;
-            }
-
-            if(!pthread_equal(io_thread, pthread_self())) {
-            pthread_join(io_thread, &thread_ret);
-            *error_code = *(int *)thread_ret;
-            if (*error_code != MPI_SUCCESS) return;
-            io_thread = pthread_self();
-
-            }
-            io_thread_args.fd = fd;
-            /* do a little pointer shuffling: background I/O works from one
-             * buffer while two-phase machinery fills up another */
-
-            if (currentReadBuf == 0) {
-            io_thread_args.buf = read_buf0;
-            currentReadBuf = 1;
-            read_buf = read_buf1;
-            }
-            else {
-            io_thread_args.buf = read_buf1;
-            currentReadBuf = 0;
-            read_buf = read_buf0;
-            }
-            io_thread_args.io_kind = ADIOI_READ;
-            io_thread_args.size = amountDataToReadNextRound;
-            io_thread_args.offset = nextRoundFDStart;
-            io_thread_args.status = &status;
-            io_thread_args.error_code = *error_code;
-            if ( (pthread_create(&io_thread, NULL,
-                    ADIOI_IO_Thread_Func, &(io_thread_args))) != 0)
-            io_thread = pthread_self();
-
-        }
-        else { /* last round */
-
-            if(!pthread_equal(io_thread, pthread_self())) {
-            pthread_join(io_thread, &thread_ret);
-            *error_code = *(int *)thread_ret;
-            if (*error_code != MPI_SUCCESS) return;
-            io_thread = pthread_self();
-
-            }
-            if (currentReadBuf == 0) {
-            read_buf = read_buf0;
-            }
-            else {
-            read_buf = read_buf1;
-            }
-
-        }
-        } /* useIOBuffer */
-    } /* IAmUsedAgg */
-    else if (useIOBuffer) {
-      if (roundIter < (numberOfRounds-1)) {
-            if (currentReadBuf == 0) {
-            currentReadBuf = 1;
-            read_buf = read_buf1;
-            }
-            else {
-            currentReadBuf = 0;
-            read_buf = read_buf0;
-            }
-      }
-      else {
-            if (currentReadBuf == 0) {
-            read_buf = read_buf0;
-            }
-            else {
-            read_buf = read_buf1;
-            }
-      }
-
-    }
-    // wait until the read buffers are full before we start pulling from the source procs
-    MPI_Barrier(fd->comm);
-
-    int aggIter;
-    for (aggIter=0;aggIter<numSourceAggs;aggIter++) {
-
-    /* If we have data for the round/agg process it.
-     */
-    if (sourceAggsForMyDataFirstOffLenIndex[roundIter][aggIter] != -1)  {
-
-      ADIO_Offset currentRoundFDStartForMySourceAgg = (ADIO_Offset)((ADIO_Offset)sourceAggsForMyDataFDStart[aggIter] + (ADIO_Offset)((ADIO_Offset)roundIter*coll_bufsize));
-      ADIO_Offset currentRoundFDEndForMySourceAgg = (ADIO_Offset)((ADIO_Offset)sourceAggsForMyDataFDStart[aggIter] + (ADIO_Offset)((ADIO_Offset)(roundIter+1)*coll_bufsize) - (ADIO_Offset)1);
-
-      int sourceAggContigAccessCount = 0;
-
-      /* These data structures are used for the derived datatype mpi_get
-       * in the gpfsmpio_read_aggmethod of 2 case.
-       */
-      int *sourceAggBlockLengths=NULL;
-      MPI_Aint *sourceAggDisplacements=NULL, *recvBufferDisplacements=NULL;
-      MPI_Datatype *sourceAggDataTypes=NULL;
-      char *derivedTypePackedSourceBuffer=NULL;
-      int derivedTypePackedSourceBufferOffset = 0;
-      int allocatedDerivedTypeArrays = 0;
-      ADIO_Offset amountOfDataReadThisRoundAgg = 0;
-
-      /* Process the range of offsets for this source agg.
-       */
-      int offsetIter;
-      int startingOffLenIndex = sourceAggsForMyDataFirstOffLenIndex[roundIter][aggIter], endingOffLenIndex = sourceAggsForMyDataLastOffLenIndex[roundIter][aggIter];
-      for (offsetIter=startingOffLenIndex;offsetIter<=endingOffLenIndex;offsetIter++) {
-        if (currentRoundFDEndForMySourceAgg > sourceAggsForMyDataFDEnd[aggIter])
-            currentRoundFDEndForMySourceAgg = sourceAggsForMyDataFDEnd[aggIter];
-
-        ADIO_Offset offsetStart = offset_list[offsetIter], offsetEnd = (offset_list[offsetIter]+len_list[offsetIter]-(ADIO_Offset)1);
-
-        /* Determine the amount of data and exact source buffer offsets to use.
-         */
-        int bufferAmountToRecv = 0;
-
-        if ((offsetStart >= currentRoundFDStartForMySourceAgg) && (offsetStart <= currentRoundFDEndForMySourceAgg)) {
-            if (offsetEnd > currentRoundFDEndForMySourceAgg)
-            bufferAmountToRecv = (currentRoundFDEndForMySourceAgg - offsetStart) +1;
-            else
-            bufferAmountToRecv = (offsetEnd - offsetStart) +1;
-        }
-        else if ((offsetEnd >= currentRoundFDStartForMySourceAgg) && (offsetEnd <= currentRoundFDEndForMySourceAgg)) {
-            if (offsetEnd > currentRoundFDEndForMySourceAgg)
-            bufferAmountToRecv = (currentRoundFDEndForMySourceAgg - currentRoundFDStartForMySourceAgg) +1;
-            else
-            bufferAmountToRecv = (offsetEnd - currentRoundFDStartForMySourceAgg) +1;
-            if (offsetStart < currentRoundFDStartForMySourceAgg) {
-              offsetStart = currentRoundFDStartForMySourceAgg;
-            }
-        }
-        else if ((offsetStart <= currentRoundFDStartForMySourceAgg) && (offsetEnd >= currentRoundFDEndForMySourceAgg)) {
-            bufferAmountToRecv = (currentRoundFDEndForMySourceAgg - currentRoundFDStartForMySourceAgg) +1;
-            offsetStart = currentRoundFDStartForMySourceAgg;
-        }
-
-        if (bufferAmountToRecv > 0) { /* we have data to recv this round */
-          if (gpfsmpio_read_aggmethod == 2) {
-            /* Only allocate these arrays if we are using method 2 and only do it once for this round/source agg.
-             */
-            if (!allocatedDerivedTypeArrays) {
-              sourceAggBlockLengths = (int *)ADIOI_Malloc(maxNumContigOperations * sizeof(int));
-              sourceAggDisplacements = (MPI_Aint *)ADIOI_Malloc(maxNumContigOperations * sizeof(MPI_Aint));
-              recvBufferDisplacements = (MPI_Aint *)ADIOI_Malloc(maxNumContigOperations * sizeof(MPI_Aint));
-              sourceAggDataTypes = (MPI_Datatype *)ADIOI_Malloc(maxNumContigOperations * sizeof(MPI_Datatype));
-              if (!bufTypeIsContig) {
-                int k;
-                for (k=sourceAggsForMyDataFirstOffLenIndex[roundIter][aggIter];k<=sourceAggsForMyDataLastOffLenIndex[roundIter][aggIter];k++)
-                  amountOfDataReadThisRoundAgg += len_list[k];
-
-#ifdef onesidedtrace
-                printf("derivedTypePackedSourceBuffer mallocing %ld\n",amountOfDataReadThisRoundAgg);
-#endif
-                if (amountOfDataReadThisRoundAgg > 0)
-                  derivedTypePackedSourceBuffer = (char *)ADIOI_Malloc(amountOfDataReadThisRoundAgg * sizeof(char));
-                else
-                  derivedTypePackedSourceBuffer = NULL;
-              }
-              allocatedDerivedTypeArrays = 1;
-            }
-          }
-
-          /* Determine the offset into the source window.
-           */
-          MPI_Aint sourceDisplacementToUseThisRound = (MPI_Aint) (offsetStart - currentRoundFDStartForMySourceAgg);
-
-          /* If using the thread reader select the appropriate side of the split window.
-           */
-          if (useIOBuffer && (read_buf == read_buf1)) {
-            sourceDisplacementToUseThisRound += (MPI_Aint)coll_bufsize;
-          }
-
-          /* For gpfsmpio_read_aggmethod of 1 do the mpi_get using the primitive MPI_BYTE type from each
-           * contiguous chunk from the target, if the source is non-contiguous then unpack the data after
-           * the MPI_Win_unlock is done to make sure the data has arrived first.
-           */
-          if (gpfsmpio_read_aggmethod == 1) {
-            MPI_Win_lock(MPI_LOCK_SHARED, sourceAggsForMyData[aggIter], 0, read_buf_window);
-            char *getSourceData = NULL;
-            if (bufTypeIsContig) {
-              MPI_Get(((char*)buf) + currentFDSourceBufferState[aggIter].sourceBufferOffset,bufferAmountToRecv, MPI_BYTE,sourceAggsForMyData[aggIter],sourceDisplacementToUseThisRound, bufferAmountToRecv,MPI_BYTE,read_buf_window);
-              currentFDSourceBufferState[aggIter].sourceBufferOffset += (ADIO_Offset)bufferAmountToRecv;
-
-            }
-            else {
-              getSourceData = (char *) ADIOI_Malloc(bufferAmountToRecv*sizeof(char));
-              MPI_Get(getSourceData,bufferAmountToRecv, MPI_BYTE,sourceAggsForMyData[aggIter],sourceDisplacementToUseThisRound, bufferAmountToRecv,MPI_BYTE,read_buf_window);
-
-            }
-            MPI_Win_unlock(sourceAggsForMyData[aggIter], read_buf_window);
-            if (!bufTypeIsContig) {
-              nonContigSourceDataBufferAdvance(((char*)buf), flatBuf, bufferAmountToRecv, 0, &currentFDSourceBufferState[aggIter], getSourceData);
-              ADIOI_Free(getSourceData);
-            }
-          }
-
-          /* For gpfsmpio_read_aggmethod of 2 populate the data structures for this round/agg for this offset iter
-           * to be used subsequently when building the derived type for 1 mpi_put for all the data for this
-           * round/agg.
-           */
-          else if (gpfsmpio_read_aggmethod == 2) {
-            if (bufTypeIsContig) {
-              sourceAggBlockLengths[sourceAggContigAccessCount]= bufferAmountToRecv;
-              sourceAggDataTypes[sourceAggContigAccessCount] = MPI_BYTE;
-              sourceAggDisplacements[sourceAggContigAccessCount] = sourceDisplacementToUseThisRound;
-              recvBufferDisplacements[sourceAggContigAccessCount] = (MPI_Aint)currentFDSourceBufferState[aggIter].sourceBufferOffset;
-              currentFDSourceBufferState[aggIter].sourceBufferOffset += (ADIO_Offset)bufferAmountToRecv;
-              sourceAggContigAccessCount++;
-            }
-            else {
-              sourceAggBlockLengths[sourceAggContigAccessCount]= bufferAmountToRecv;
-              sourceAggDataTypes[sourceAggContigAccessCount] = MPI_BYTE;
-              sourceAggDisplacements[sourceAggContigAccessCount] = sourceDisplacementToUseThisRound;
-              recvBufferDisplacements[sourceAggContigAccessCount] = (MPI_Aint)derivedTypePackedSourceBufferOffset;
-              derivedTypePackedSourceBufferOffset += (ADIO_Offset)bufferAmountToRecv;
-              sourceAggContigAccessCount++;
-            }
-          }
-          } // bufferAmountToRecv > 0
-      } // contig list
-
-      /* For gpfsmpio_read_aggmethod of 2 now build the derived type using the data from this round/agg and do 1 single mpi_put.
-       */
-      if (gpfsmpio_read_aggmethod == 2) {
-        MPI_Datatype recvBufferDerivedDataType, sourceBufferDerivedDataType;
-
-        MPI_Type_create_struct(sourceAggContigAccessCount, sourceAggBlockLengths, recvBufferDisplacements, sourceAggDataTypes, &recvBufferDerivedDataType);
-        MPI_Type_commit(&recvBufferDerivedDataType);
-        MPI_Type_create_struct(sourceAggContigAccessCount, sourceAggBlockLengths, sourceAggDisplacements, sourceAggDataTypes, &sourceBufferDerivedDataType);
-        MPI_Type_commit(&sourceBufferDerivedDataType);
-
-        if (sourceAggContigAccessCount > 0) {
-
-        MPI_Win_lock(MPI_LOCK_SHARED, sourceAggsForMyData[aggIter], 0, read_buf_window);
-        if (bufTypeIsContig) {
-          MPI_Get(((char*)buf),1, recvBufferDerivedDataType,sourceAggsForMyData[aggIter],0, 1,sourceBufferDerivedDataType,read_buf_window);
-        }
-        else {
-          MPI_Get(derivedTypePackedSourceBuffer,1, recvBufferDerivedDataType,sourceAggsForMyData[aggIter],0, 1,sourceBufferDerivedDataType,read_buf_window);
-        }
-
-        MPI_Win_unlock(sourceAggsForMyData[aggIter], read_buf_window);
-        if (!bufTypeIsContig) {
-          nonContigSourceDataBufferAdvance(((char*)buf), flatBuf, derivedTypePackedSourceBufferOffset, 0, &currentFDSourceBufferState[aggIter], derivedTypePackedSourceBuffer);
-        }
-        }
-
-        if (allocatedDerivedTypeArrays) {
-          ADIOI_Free(sourceAggBlockLengths);
-          ADIOI_Free(sourceAggDisplacements);
-          ADIOI_Free(sourceAggDataTypes);
-          ADIOI_Free(recvBufferDisplacements);
-          if (!bufTypeIsContig)
-            if (derivedTypePackedSourceBuffer != NULL)
-              ADIOI_Free(derivedTypePackedSourceBuffer);
-        }
-        if (sourceAggContigAccessCount > 0) {
-        MPI_Type_free(&recvBufferDerivedDataType);
-        MPI_Type_free(&sourceBufferDerivedDataType);
-        }
-      }
-      } // baseoffset != -1
-    } // source aggs
-    } // contig_access_count > 0
-    /* the source procs recv the requested data to the aggs */
-
-    MPI_Barrier(fd->comm);
-
-    nextRoundFDStart = currentRoundFDStart + coll_bufsize;
-
-    } /* for-loop roundIter */
-
-#ifdef ROMIO_GPFS
-    endTimeBase = MPI_Wtime();
-    gpfsmpio_prof_cw[GPFSMPIO_CIO_T_DEXCH] += (endTimeBase-startTimeBase);
-#endif
-
-    if (useIOBuffer) { /* thread readr cleanup */
-
-    if ( !pthread_equal(io_thread, pthread_self()) ) {
-        pthread_join(io_thread, &thread_ret);
-        *error_code = *(int *)thread_ret;
-    }
-
-    }
-
-    ADIOI_Free(sourceAggsForMyData);
-    ADIOI_Free(sourceAggsForMyDataFDStart);
-    ADIOI_Free(sourceAggsForMyDataFDEnd);
-
-    for (i=0;i<numberOfRounds;i++) {
-      ADIOI_Free(sourceAggsForMyDataFirstOffLenIndex[i]);
-      ADIOI_Free(sourceAggsForMyDataLastOffLenIndex[i]);
-    }
-    ADIOI_Free(sourceAggsForMyDataFirstOffLenIndex);
-    ADIOI_Free(sourceAggsForMyDataLastOffLenIndex);
-
-    ADIOI_Free(currentFDSourceBufferState);
-
-    if (!bufTypeIsContig)
-      ADIOI_Delete_flattened(datatype);
-    return;
-}
diff --git a/3rd-party/romio321/adio/common/p2p_aggregation.c b/3rd-party/romio321/adio/common/p2p_aggregation.c
deleted file mode 100644
index ac77a8f876e..00000000000
--- a/3rd-party/romio321/adio/common/p2p_aggregation.c
+++ /dev/null
@@ -1,916 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *  (C) 2014 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-#include "../ad_gpfs/ad_gpfs_tuning.h"
-
-#include <pthread.h>
-
-/* #define p2pcontigtrace 1 */
-
-void ADIOI_P2PContigWriteAggregation(ADIO_File fd,
-	const void *buf,
-	int *error_code,
-	ADIO_Offset *st_offsets,
-	ADIO_Offset *end_offsets,
-	ADIO_Offset *fd_start,
-	ADIO_Offset* fd_end)
-{
-
-    *error_code = MPI_SUCCESS; /* initialize to success */
-
-#ifdef ROMIO_GPFS
-    double startTimeBase,endTimeBase;
-#endif
-
-    MPI_Status status;
-    pthread_t io_thread;
-    void *thread_ret;
-    ADIOI_IO_ThreadFuncData io_thread_args;
-
-    int nprocs,myrank;
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-
-	ADIO_Offset myOffsetStart = st_offsets[myrank], myOffsetEnd = end_offsets[myrank];
-
-    int myAggRank = -1; /* if I am an aggregor this is my index into fd->hints->ranklist */
-    int iAmUsedAgg = 0;
-
-#ifdef ROMIO_GPFS
-    startTimeBase = MPI_Wtime();
-#endif
-
-    int naggs = fd->hints->cb_nodes;
-    int coll_bufsize = fd->hints->cb_buffer_size;
-#ifdef ROMIO_GPFS
-    if (gpfsmpio_pthreadio == 1) {
-	/* split buffer in half for a kind of double buffering with the threads*/
-	coll_bufsize = fd->hints->cb_buffer_size/2;
-    }
-#endif
-
-    int j;
-    for (j=0;j<naggs;j++) {
-	if (fd->hints->ranklist[j] == myrank) {
-	    myAggRank = j;
-	    if (fd_end[j] > fd_start[j]) {
-		iAmUsedAgg = 1;
-	    }
-	}
-    }
-
-    /* Determine how much data and to whom I need to send.  For source proc
-     * targets, also determine the target file domain offsets locally to
-     * reduce communication overhead */
-    int *targetAggsForMyData = (int *)ADIOI_Malloc(naggs * sizeof(int));
-    ADIO_Offset *targetAggsForMyDataFDStart = (ADIO_Offset *)ADIOI_Malloc(naggs * sizeof(ADIO_Offset));
-    ADIO_Offset *targetAggsForMyDataFDEnd = (ADIO_Offset *)ADIOI_Malloc(naggs * sizeof(ADIO_Offset));
-    int numTargetAggs = 0;
-    int i;
-    for (i=0;i<naggs;i++) {
-	if ( ((myOffsetStart >= fd_start[i]) &&  (myOffsetStart <= fd_end[i])) || ((myOffsetEnd >= fd_start[i]) &&  (myOffsetEnd <= fd_end[i]))) {
-	    targetAggsForMyData[numTargetAggs] = fd->hints->ranklist[i];
-	    targetAggsForMyDataFDStart[numTargetAggs] = fd_start[i];
-	    targetAggsForMyDataFDEnd[numTargetAggs] = fd_end[i];
-	    numTargetAggs++;
-	}
-    }
-
-    /* these 3 arrays track info on the procs that feed an aggregtor */
-    int *sourceProcsForMyData=NULL;
-    int *remainingDataAmountToGetPerProc=NULL;
-    ADIO_Offset *remainingDataOffsetToGetPerProc=NULL;
-
-    int numSourceProcs = 0;
-
-    if (iAmUsedAgg) { /* for the used aggregators figure out how much data I
-			 need from what procs */
-
-	/* count numSourceProcs so we know how large to make the arrays */
-	for (i=0;i<nprocs;i++)
-	    if ( ((st_offsets[i] >= fd_start[myAggRank]) &&  (st_offsets[i] <= fd_end[myAggRank])) || ((end_offsets[i] >= fd_start[myAggRank]) &&  (end_offsets[i] <= fd_end[myAggRank])))
-		numSourceProcs++;
-
-	sourceProcsForMyData = (int *)ADIOI_Malloc(numSourceProcs * sizeof(int));
-	remainingDataAmountToGetPerProc = (int *)ADIOI_Malloc(numSourceProcs * sizeof(int));
-	remainingDataOffsetToGetPerProc = (ADIO_Offset *)ADIOI_Malloc(numSourceProcs * sizeof(ADIO_Offset));
-
-	/* everybody has the st_offsets and end_offsets for all ranks so if I am a
-	 * used aggregator go thru them and figure out which ranks have data that
-	 * falls into my file domain assigned to me */
-	numSourceProcs = 0;
-	for (i=0;i<nprocs;i++) {
-	    if ( ((st_offsets[i] >= fd_start[myAggRank]) &&  (st_offsets[i] <= fd_end[myAggRank])) || ((end_offsets[i] >= fd_start[myAggRank]) &&  (end_offsets[i] <= fd_end[myAggRank]))) {
-		sourceProcsForMyData[numSourceProcs] = i;
-		if ( ((st_offsets[i] >= fd_start[myAggRank]) &&  (st_offsets[i] <= fd_end[myAggRank])) && ((end_offsets[i] >= fd_start[myAggRank]) &&  (end_offsets[i] <= fd_end[myAggRank]))) {
-		    remainingDataAmountToGetPerProc[numSourceProcs] = (end_offsets[i] - st_offsets[i])+1;
-		    remainingDataOffsetToGetPerProc[numSourceProcs] = st_offsets[i];
-		}
-		else if ((st_offsets[i] >= fd_start[myAggRank]) &&  (st_offsets[i] <= fd_end[myAggRank])) {/* starts in this fd and goes past it */
-		    remainingDataAmountToGetPerProc[numSourceProcs] = (fd_end[myAggRank] - st_offsets[i]) +1;
-		    remainingDataOffsetToGetPerProc[numSourceProcs] = st_offsets[i];
-		}
-		else { /* starts in fd before this and ends in it */
-		    remainingDataAmountToGetPerProc[numSourceProcs] = (end_offsets[i] - fd_start[myAggRank]) +1;
-		    remainingDataOffsetToGetPerProc[numSourceProcs] = fd_start[myAggRank];
-		}
-#ifdef p2pcontigtrace
-		printf("getting %ld bytes from source proc %d in fd rank %d with borders %ld to %ld\n",remainingDataAmountToGetPerProc[numSourceProcs],i,fd->hints->ranklist[myAggRank],fd_start[myAggRank],fd_end[myAggRank]);
-#endif
-		numSourceProcs++;
-	    }
-	}
-    }
-
-    int *amountOfDataReqestedByTargetAgg = (int *)ADIOI_Malloc(naggs * sizeof(int));
-    for (i=0;i<numTargetAggs;i++) {
-	amountOfDataReqestedByTargetAgg[i] = 0;
-    }
-
-    int totalAmountDataReceived = 0;
-    MPI_Request *mpiSizeToSendRequest = (MPI_Request *) ADIOI_Malloc(numTargetAggs * sizeof(MPI_Request));
-    MPI_Request *mpiRecvDataRequest = (MPI_Request *) ADIOI_Malloc(numSourceProcs * sizeof(MPI_Request));
-    MPI_Request *mpiSendDataSizeRequest = (MPI_Request *) ADIOI_Malloc(numSourceProcs * sizeof(MPI_Request));
-
-    MPI_Request *mpiSendDataToTargetAggRequest = (MPI_Request *) ADIOI_Malloc(numTargetAggs * sizeof(MPI_Request));
-    MPI_Status mpiWaitAnyStatusFromTargetAggs,mpiWaitAnyStatusFromSourceProcs;
-    MPI_Status mpiIsendStatusForSize,  mpiIsendStatusForData;
-
-    /* use the write buffer allocated in the file_open */
-    char *write_buf0 = fd->io_buf;
-    char *write_buf1 = fd->io_buf + coll_bufsize;
-
-    /* start off pointing to the first buffer. If we use the 2nd buffer (threaded
-     * case) we'll swap later */
-    char *write_buf = write_buf0;
-
-    /* compute number of rounds */
-    ADIO_Offset numberOfRounds = (ADIO_Offset)((((ADIO_Offset)(end_offsets[nprocs-1]-st_offsets[0]))/((ADIO_Offset)((ADIO_Offset)coll_bufsize*(ADIO_Offset)naggs)))) + 1;
-
-    int currentWriteBuf = 0;
-    int useIOBuffer = 0;
-#ifdef ROMIO_GPFS
-    if (gpfsmpio_pthreadio && (numberOfRounds>1)) {
-	useIOBuffer = 1;
-	io_thread = pthread_self();
-    }
-#endif
-
-    ADIO_Offset currentRoundFDStart = 0;
-    ADIO_Offset currentRoundFDEnd = 0;
-
-    if (iAmUsedAgg) {
-	currentRoundFDStart = fd_start[myAggRank];
-    }
-
-    int *dataSizeGottenThisRoundPerProc = (int *)ADIOI_Malloc(numSourceProcs * sizeof(int));
-    int *mpiRequestMapPerProc = (int *)ADIOI_Malloc(numSourceProcs * sizeof(int));
-    int *targetAggIndexesForMyDataThisRound = (int *)ADIOI_Malloc(numTargetAggs * sizeof(int));
-    int *sendBufferOffsetsThisRound = (int *)ADIOI_Malloc(numTargetAggs * sizeof(int));
-    int *bufferAmountsToSendThisRound = (int *)ADIOI_Malloc(numTargetAggs * sizeof(int));
-
-#ifdef ROMIO_GPFS
-    endTimeBase = MPI_Wtime();
-    gpfsmpio_prof_cw[GPFSMPIO_CIO_T_MYREQ] += (endTimeBase-startTimeBase);
-    startTimeBase = MPI_Wtime();
-#endif
-
-    /* each iteration of this loop writes a coll_bufsize portion of the file
-     * domain */
-    int roundIter;
-    for (roundIter=0;roundIter<numberOfRounds;roundIter++) {
-
-	/* determine what target aggs I need to send data to this round */
-	int numTargetAggsThisRound = 0;
-	for (i=0;i<numTargetAggs;i++) {
-	    if ( ((myOffsetStart >= targetAggsForMyDataFDStart[i]) && (myOffsetStart <= targetAggsForMyDataFDEnd[i])) ||
-		    ((myOffsetEnd >= targetAggsForMyDataFDStart[i]) && (myOffsetEnd <= targetAggsForMyDataFDEnd[i]))) {
-		/* we know that we need to send data to this target agg at some point, now need to figure out how much this round */
-
-		/* here are the offsets currently being collected by the aggregator during this round */
-		ADIO_Offset currentRoundFDStartForMyTargetAgg = (ADIO_Offset)((ADIO_Offset)targetAggsForMyDataFDStart[i] + (ADIO_Offset)((ADIO_Offset)roundIter*(ADIO_Offset)coll_bufsize));
-		ADIO_Offset currentRoundFDEndForMyTargetAgg = (ADIO_Offset)((ADIO_Offset)targetAggsForMyDataFDStart[i] + (ADIO_Offset)((ADIO_Offset)(roundIter+1)*(ADIO_Offset)coll_bufsize) - (ADIO_Offset)1);
-		if (currentRoundFDEndForMyTargetAgg > targetAggsForMyDataFDEnd[i])
-		    currentRoundFDEndForMyTargetAgg = targetAggsForMyDataFDEnd[i];
-
-#ifdef p2pcontigtrace
-		printf("roundIter %d target iter %d targetAggsForMyData is %d myOffsetStart is %ld myOffsetEnd is %ld targetAggsForMyDataFDStart is %ld targetAggsForMyDataFDEnd is %ld currentRoundFDStartForMyTargetAgg is %ld currentRoundFDEndForMyTargetAgg is %ld\n",
-			roundIter,i,targetAggsForMyData[i],myOffsetStart,myOffsetEnd,
-			targetAggsForMyDataFDStart[i],targetAggsForMyDataFDEnd[i],
-			currentRoundFDStartForMyTargetAgg,currentRoundFDEndForMyTargetAgg);
-#endif
-
-		/* send the portion of my data that is within
-		 * currentRoundFDStartForMyTargetAgg to
-		 * currentRoundFDEndForMyTargetAgg */
-		/* find the offset into the send buffer and the amount
-		 * of data to send */
-		int sendBufferOffset = 0;
-		int bufferAmountToSend = 0;
-
-		if ((myOffsetStart >= currentRoundFDStartForMyTargetAgg) && (myOffsetStart <= currentRoundFDEndForMyTargetAgg)) {
-		    if (myOffsetEnd > currentRoundFDEndForMyTargetAgg)
-			bufferAmountToSend = (currentRoundFDEndForMyTargetAgg - myOffsetStart) +1;
-		    else
-			bufferAmountToSend = (myOffsetEnd - myOffsetStart) +1;
-		}
-		else if ((myOffsetEnd >= currentRoundFDStartForMyTargetAgg) && (myOffsetEnd <= currentRoundFDEndForMyTargetAgg)) {
-		    sendBufferOffset = (int) (currentRoundFDStartForMyTargetAgg - myOffsetStart);
-		    if (myOffsetEnd > currentRoundFDEndForMyTargetAgg)
-			bufferAmountToSend = (currentRoundFDEndForMyTargetAgg - currentRoundFDStartForMyTargetAgg) +1;
-		    else
-			bufferAmountToSend = (myOffsetEnd - currentRoundFDStartForMyTargetAgg) +1;
-		}
-		else if ((myOffsetStart <= currentRoundFDStartForMyTargetAgg) && (myOffsetEnd >= currentRoundFDEndForMyTargetAgg)) {
-		    sendBufferOffset = (int) (currentRoundFDStartForMyTargetAgg - myOffsetStart);
-		    bufferAmountToSend = (currentRoundFDEndForMyTargetAgg - currentRoundFDStartForMyTargetAgg) +1;
-		}
-
-		if (bufferAmountToSend > 0) { /* we have data to send this round */
-		    targetAggIndexesForMyDataThisRound[numTargetAggsThisRound] = i;
-		    sendBufferOffsetsThisRound[numTargetAggsThisRound] = sendBufferOffset;
-		    bufferAmountsToSendThisRound[numTargetAggsThisRound] = bufferAmountToSend;
-#ifdef p2pcontigtrace
-		    printf("bufferAmountToSend is %d sendBufferOffset is %d\n",bufferAmountToSend,sendBufferOffset);
-#endif
-		    /* only need to be pinged by the agg for rounds after the first one - for the first one just
-		     * send the data without being pinged */
-		    if (roundIter > 0)
-			MPI_Irecv(&amountOfDataReqestedByTargetAgg[numTargetAggsThisRound],1,
-				MPI_INT,targetAggsForMyData[i],0,
-				fd->comm,&mpiSizeToSendRequest[numTargetAggsThisRound]);
-		    numTargetAggsThisRound++;
-
-		}
-	    }
-	}
-
-	/* determine what offsets define the portion of the file domain the agg is writing this round */
-	if (iAmUsedAgg) {
-	    if ((fd_end[myAggRank] - currentRoundFDStart) < coll_bufsize) {
-		currentRoundFDEnd = fd_end[myAggRank];
-	    }
-	    else
-		currentRoundFDEnd = currentRoundFDStart + coll_bufsize - 1;
-#ifdef p2pcontigtrace
-	    printf("currentRoundFDStart is %ld currentRoundFDEnd is %ld within file domeain %ld to %ld\n",currentRoundFDStart,currentRoundFDEnd,fd_start[myAggRank],fd_end[myAggRank]);
-#endif
-	}
-
-	int irecv,isend;
-	int numSourceProcsSentData = 0;
-
-	/* the aggs send the amount of data they need to their source procs */
-	for (i=0;i<numSourceProcs;i++) {
-	    if ((remainingDataOffsetToGetPerProc[i] >= currentRoundFDStart) && (remainingDataOffsetToGetPerProc[i] <= currentRoundFDEnd)) {
-		if ((remainingDataOffsetToGetPerProc[i] + remainingDataAmountToGetPerProc[i]) <= currentRoundFDEnd)
-		    dataSizeGottenThisRoundPerProc[i] = remainingDataAmountToGetPerProc[i];
-		else
-		    dataSizeGottenThisRoundPerProc[i] = (currentRoundFDEnd - remainingDataOffsetToGetPerProc[i]) +1;
-	    }
-	    else if (((remainingDataOffsetToGetPerProc[i]+remainingDataAmountToGetPerProc[i]) >= currentRoundFDStart) && ((remainingDataOffsetToGetPerProc[i]+remainingDataAmountToGetPerProc[i]) <= currentRoundFDEnd)) {
-		if ((remainingDataOffsetToGetPerProc[i]) >= currentRoundFDStart)
-		    dataSizeGottenThisRoundPerProc[i] = remainingDataAmountToGetPerProc[i];
-		else
-		    dataSizeGottenThisRoundPerProc[i] = (remainingDataOffsetToGetPerProc[i]-currentRoundFDStart) +1;
-	    }
-	    else
-		dataSizeGottenThisRoundPerProc[i] = 0;
-
-#ifdef p2pcontigtrace
-	    printf("dataSizeGottenThisRoundPerProc[%d] set to %d - remainingDataOffsetToGetPerProc is %d remainingDataAmountToGetPerProc is %d currentRoundFDStart is %d currentRoundFDEnd is %d\n",i,dataSizeGottenThisRoundPerProc[i],remainingDataOffsetToGetPerProc[i],remainingDataAmountToGetPerProc[i],currentRoundFDStart,currentRoundFDEnd);
-#endif
-	    if (dataSizeGottenThisRoundPerProc[i] > 0) {
-		if (roundIter > 0) {
-		    MPI_Isend(&dataSizeGottenThisRoundPerProc[i],1,MPI_INT,
-			    sourceProcsForMyData[i],0,fd->comm,
-			    &mpiSendDataSizeRequest[numSourceProcsSentData]);
-		    numSourceProcsSentData++;
-		}
-	    }
-	}
-
-	int numDataSendToWaitFor = 0;
-	/* the source procs send the requested data to the aggs */
-	for (i = 0; i < numTargetAggsThisRound; i++) {
-
-		/* the source procs aren't pinged by the target aggs on the first round */
-	    if (roundIter > 0) {
-
-		MPI_Waitany(numTargetAggsThisRound,mpiSizeToSendRequest,
-			&irecv,&mpiWaitAnyStatusFromTargetAggs);
-
-#ifdef p2pcontigtrace
-		printf("irecv is %d amountOfDataReqestedByTargetAgg is %d bufferAmountsToSendThisRound is %d sendBufferOffsetsThisRound is %d targetAggsForMyData is %d\n",irecv,amountOfDataReqestedByTargetAgg[irecv], bufferAmountsToSendThisRound[irecv], sendBufferOffsetsThisRound[irecv],targetAggsForMyData[targetAggIndexesForMyDataThisRound[irecv]]);
-#endif
-		ADIOI_Assert(amountOfDataReqestedByTargetAgg[irecv] == bufferAmountsToSendThisRound[irecv]);
-		MPI_Isend(&((char*)buf)[sendBufferOffsetsThisRound[irecv]],
-			bufferAmountsToSendThisRound[irecv],MPI_BYTE,
-			targetAggsForMyData[targetAggIndexesForMyDataThisRound[irecv]],
-			0,fd->comm,&mpiSendDataToTargetAggRequest[irecv]);
-
-	    }
-	    else {
-#ifdef p2pcontigtrace
-		printf("i is %d bufferAmountsToSendThisRound is %d sendBufferOffsetsThisRound is %d targetAggsForMyData is %d\n",i, bufferAmountsToSendThisRound[i], sendBufferOffsetsThisRound[i],targetAggsForMyData[targetAggIndexesForMyDataThisRound[i]]);
-#endif
-		MPI_Isend(&((char*)buf)[sendBufferOffsetsThisRound[i]],bufferAmountsToSendThisRound[i],MPI_BYTE,
-			targetAggsForMyData[targetAggIndexesForMyDataThisRound[i]],0,fd->comm,&mpiSendDataToTargetAggRequest[i]);
-	    }
-        numDataSendToWaitFor++;
-	}
-
-#ifdef ROMIO_GPFS
-	gpfsmpio_prof_cw[GPFSMPIO_CIO_T_DEXCH_SETUP] += (endTimeBase-startTimeBase);
-	startTimeBase = MPI_Wtime();
-#endif
-
-	/* the aggs receive the data from the source procs */
-	int numDataRecvToWaitFor = 0;
-	for (i=0;i<numSourceProcs;i++) {
-
-	    int currentWBOffset = 0;
-	    for (j=0;j<i;j++)
-		currentWBOffset += dataSizeGottenThisRoundPerProc[j];
-
-	    /* only receive from source procs that will send > 0 count data */
-	    if (dataSizeGottenThisRoundPerProc[i] > 0) {
-#ifdef p2pcontigtrace
-		printf("receiving data from rank %d dataSizeGottenThisRoundPerProc is %d currentWBOffset is %d\n",sourceProcsForMyData[i],dataSizeGottenThisRoundPerProc[i],currentWBOffset);
-#endif
-		MPI_Irecv(&((char*)write_buf)[currentWBOffset],dataSizeGottenThisRoundPerProc[i],
-			MPI_BYTE,sourceProcsForMyData[i],0,
-			fd->comm,&mpiRecvDataRequest[numDataRecvToWaitFor]);
-		mpiRequestMapPerProc[numDataRecvToWaitFor] = i;
-		numDataRecvToWaitFor++;
-	    }
-
-#ifdef p2pcontigtrace
-	    printf("MPI_Irecv from rank %d\n",targetAggsForMyData[i]);
-#endif
-	}
-
-	int totalDataReceivedThisRound = 0;
-	for (i = 0; i < numDataRecvToWaitFor; i++) {
-	    MPI_Waitany(numDataRecvToWaitFor,mpiRecvDataRequest,
-		    &irecv,&mpiWaitAnyStatusFromSourceProcs);
-	    totalDataReceivedThisRound +=
-		dataSizeGottenThisRoundPerProc[mpiRequestMapPerProc[irecv]];
-	    totalAmountDataReceived +=
-		dataSizeGottenThisRoundPerProc[mpiRequestMapPerProc[irecv]];
-
-#ifdef p2pcontigtrace
-	    printf("numDataRecvToWaitFor is %d was sent %d bytes data for %d remaining bytes from rank %d irecv index %d\n",numDataRecvToWaitFor,dataSizeGottenThisRoundPerProc[mpiRequestMapPerProc[irecv]],remainingDataAmountToGetPerProc[mpiRequestMapPerProc[irecv]],sourceProcsForMyData[mpiRequestMapPerProc[irecv]],irecv);
-#endif
-	    remainingDataAmountToGetPerProc[mpiRequestMapPerProc[irecv]] -=
-		dataSizeGottenThisRoundPerProc[mpiRequestMapPerProc[irecv]];
-	    remainingDataOffsetToGetPerProc[mpiRequestMapPerProc[irecv]] +=
-		dataSizeGottenThisRoundPerProc[mpiRequestMapPerProc[irecv]];
-
-	}
-
-	/* clean up the MPI_Request object for the MPI_Isend which told the
-	 * source procs how much data to send */
-        for (i=0;i<numSourceProcsSentData;i++) {
-           MPI_Waitany(numSourceProcsSentData,mpiSendDataSizeRequest,
-		   &isend,&mpiIsendStatusForSize);
-        }
-
-
-#ifdef ROMIO_GPFS
-        endTimeBase = MPI_Wtime();
-	gpfsmpio_prof_cw[GPFSMPIO_CIO_T_DEXCH_NET] += (endTimeBase-startTimeBase);
-#endif
-	/* the aggs now write the data */
-	if (numDataRecvToWaitFor > 0) {
-
-#ifdef p2pcontigtrace
-	    printf("totalDataReceivedThisRound is %d\n",totalDataReceivedThisRound);
-#endif
-	    if (!useIOBuffer) {
-
-		ADIO_WriteContig(fd, write_buf, (int)totalDataReceivedThisRound,
-			MPI_BYTE, ADIO_EXPLICIT_OFFSET,
-			currentRoundFDStart, &status, error_code);
-	    } else { /* use the thread writer */
-
-		if(!pthread_equal(io_thread, pthread_self())) {
-		    pthread_join(io_thread, &thread_ret);
-		    *error_code = *(int *)thread_ret;
-		    if (*error_code != MPI_SUCCESS) return;
-		    io_thread = pthread_self();
-
-		}
-		io_thread_args.fd = fd;
-		/* do a little pointer shuffling: background I/O works from one
-		 * buffer while two-phase machinery fills up another */
-
-		if (currentWriteBuf == 0) {
-		    io_thread_args.buf = write_buf0;
-		    currentWriteBuf = 1;
-		    write_buf = write_buf1;
-		}
-		else {
-		    io_thread_args.buf = write_buf1;
-		    currentWriteBuf = 0;
-		    write_buf = write_buf0;
-		}
-		io_thread_args.io_kind = ADIOI_WRITE;
-		io_thread_args.size = totalDataReceivedThisRound;
-		io_thread_args.offset = currentRoundFDStart;
-		io_thread_args.status = &status;
-		io_thread_args.error_code = *error_code;
-		if ( (pthread_create(&io_thread, NULL,
-				ADIOI_IO_Thread_Func, &(io_thread_args))) != 0)
-		    io_thread = pthread_self();
-
-	    }
-
-	} /* numDataRecvToWaitFor > 0 */
-
-	if (iAmUsedAgg)
-	    currentRoundFDStart += coll_bufsize;
-        for (i = 0; i < numDataSendToWaitFor; i++) {
-          MPI_Wait(&mpiSendDataToTargetAggRequest[i],
-		  &mpiIsendStatusForData);
-        }
-
-    } /* for-loop roundIter */
-
-#ifdef ROMIO_GPFS
-    endTimeBase = MPI_Wtime();
-    gpfsmpio_prof_cw[GPFSMPIO_CIO_T_DEXCH] += (endTimeBase-startTimeBase);
-#endif
-
-    if (useIOBuffer) { /* thread writer cleanup */
-
-	if ( !pthread_equal(io_thread, pthread_self()) ) {
-	    pthread_join(io_thread, &thread_ret);
-	    *error_code = *(int *)thread_ret;
-	}
-
-    }
-
-
-
-    if (iAmUsedAgg) {
-	ADIOI_Free(sourceProcsForMyData);
-	ADIOI_Free(remainingDataAmountToGetPerProc);
-	ADIOI_Free(remainingDataOffsetToGetPerProc);
-    }
-
-    ADIOI_Free(targetAggsForMyData);
-    ADIOI_Free(targetAggsForMyDataFDStart);
-    ADIOI_Free(targetAggsForMyDataFDEnd);
-    ADIOI_Free(targetAggIndexesForMyDataThisRound);
-    ADIOI_Free(sendBufferOffsetsThisRound);
-    ADIOI_Free(bufferAmountsToSendThisRound);
-    ADIOI_Free(amountOfDataReqestedByTargetAgg);
-    ADIOI_Free(mpiSizeToSendRequest);
-    ADIOI_Free(mpiRecvDataRequest);
-    ADIOI_Free(mpiSendDataSizeRequest);
-    ADIOI_Free(mpiSendDataToTargetAggRequest);
-    ADIOI_Free(dataSizeGottenThisRoundPerProc);
-    ADIOI_Free(mpiRequestMapPerProc);
-
-    /* TODO: still need a barrier here? */
-    MPI_Barrier(fd->comm);
-    return;
-}
-
-void ADIOI_P2PContigReadAggregation(ADIO_File fd,
-	const void *buf,
-	int *error_code,
-	ADIO_Offset *st_offsets,
-	ADIO_Offset *end_offsets,
-	ADIO_Offset *fd_start,
-	ADIO_Offset* fd_end)
-{
-
-    *error_code = MPI_SUCCESS; /* initialize to success */
-
-#ifdef ROMIO_GPFS
-    double startTimeBase,endTimeBase;
-#endif
-
-    MPI_Status status;
-    pthread_t io_thread;
-    void *thread_ret;
-    ADIOI_IO_ThreadFuncData io_thread_args;
-
-#ifdef ROMIO_GPFS
-    startTimeBase = MPI_Wtime();
-#endif
-
-    int nprocs,myrank;
-    MPI_Comm_size(fd->comm, &nprocs);
-    MPI_Comm_rank(fd->comm, &myrank);
-
-    ADIO_Offset myOffsetStart = st_offsets[myrank], myOffsetEnd = end_offsets[myrank];
-
-    int myAggRank = -1; /* if I am an aggregor this is my index into fd->hints->ranklist */
-    int iAmUsedAgg = 0;
-
-    int naggs = fd->hints->cb_nodes;
-    int coll_bufsize = fd->hints->cb_buffer_size;
-#ifdef ROMIO_GPFS
-    if (gpfsmpio_pthreadio == 1)
-	/* share buffer between working threads */
-	coll_bufsize = coll_bufsize/2;
-#endif
-
-    int j;
-    for (j=0;j<naggs;j++) {
-	if (fd->hints->ranklist[j] == myrank) {
-	    myAggRank = j;
-	    if (fd_end[j] > fd_start[j]) {
-		iAmUsedAgg = 1;
-	    }
-	}
-    }
-
-    /* for my offset range determine how much data and from whom I need to get
-     * it.  For source ag targets, also determine the source file domain
-     * offsets locally to reduce communication overhead */
-    int *sourceAggsForMyData = (int *)ADIOI_Malloc(naggs * sizeof(int));
-    ADIO_Offset *sourceAggsForMyDataFDStart = (ADIO_Offset *)ADIOI_Malloc(naggs * sizeof(ADIO_Offset));
-    ADIO_Offset *sourceAggsForMyDataFDEnd = (ADIO_Offset *)ADIOI_Malloc(naggs * sizeof(ADIO_Offset));
-    int numSourceAggs = 0;
-    int i;
-    for (i=0;i<naggs;i++) {
-	if ( ((myOffsetStart >= fd_start[i]) && (myOffsetStart <= fd_end[i])) ||
-		((myOffsetEnd >= fd_start[i]) &&  (myOffsetEnd <= fd_end[i]))) {
-	    sourceAggsForMyData[numSourceAggs] = fd->hints->ranklist[i];
-	    sourceAggsForMyDataFDStart[numSourceAggs] = fd_start[i];
-	    sourceAggsForMyDataFDEnd[numSourceAggs] = fd_end[i];
-	    numSourceAggs++;
-	}
-    }
-
-    /* these 3 arrays track info on the procs that are fed from an aggregtor -
-     * to sacrifice some performance at setup to save on memory instead of
-     * using max size of nprocs for the arrays could determine exact size first
-     * and then allocate that size */
-    int *targetProcsForMyData=NULL;
-    int *remainingDataAmountToSendPerProc=NULL;
-    ADIO_Offset *remainingDataOffsetToSendPerProc=NULL;
-
-    int numTargetProcs = 0;
-
-    if (iAmUsedAgg) {
-	/* for the used aggregators figure out how much data I need from what procs */
-
-	/* count numTargetProcs so we know how large to make the arrays */
-	for (i=0;i<nprocs;i++)
-	    if ( ((st_offsets[i] >= fd_start[myAggRank]) &&
-			(st_offsets[i] <= fd_end[myAggRank])) ||
-		    ((end_offsets[i] >= fd_start[myAggRank]) &&
-		     (end_offsets[i] <= fd_end[myAggRank]))  )
-		numTargetProcs++;
-
-	targetProcsForMyData =
-	    (int *)ADIOI_Malloc(numTargetProcs * sizeof(int));
-	remainingDataAmountToSendPerProc =
-	    (int *)ADIOI_Malloc(numTargetProcs * sizeof(int));
-	remainingDataOffsetToSendPerProc =
-	    (ADIO_Offset *)ADIOI_Malloc(numTargetProcs * sizeof(ADIO_Offset));
-
-	/* everybody has the st_offsets and end_offsets for all ranks so if I am a
-	 * used aggregator go thru them and figure out which ranks have data that
-	 * falls into my file domain assigned to me */
-	numTargetProcs = 0;
-	for (i=0;i<nprocs;i++) {
-	    if ( ((st_offsets[i] >= fd_start[myAggRank]) &&  (st_offsets[i] <= fd_end[myAggRank])) || ((end_offsets[i] >= fd_start[myAggRank]) &&  (end_offsets[i] <= fd_end[myAggRank]))) {
-		targetProcsForMyData[numTargetProcs] = i;
-		if ( ((st_offsets[i] >= fd_start[myAggRank]) &&  (st_offsets[i] <= fd_end[myAggRank])) && ((end_offsets[i] >= fd_start[myAggRank]) &&  (end_offsets[i] <= fd_end[myAggRank]))) {
-		    remainingDataAmountToSendPerProc[numTargetProcs] = (end_offsets[i] - st_offsets[i])+1;
-		    remainingDataOffsetToSendPerProc[numTargetProcs] = st_offsets[i];
-		}
-		else if ((st_offsets[i] >= fd_start[myAggRank]) &&  (st_offsets[i] <= fd_end[myAggRank])) {/* starts in this fd and goes past it */
-		    remainingDataAmountToSendPerProc[numTargetProcs] = (fd_end[myAggRank] - st_offsets[i]) +1;
-		    remainingDataOffsetToSendPerProc[numTargetProcs] = st_offsets[i];
-		}
-		else { /* starts in fd before this and ends in it */
-		    remainingDataAmountToSendPerProc[numTargetProcs] = (end_offsets[i] - fd_start[myAggRank]) +1;
-		    remainingDataOffsetToSendPerProc[numTargetProcs] = fd_start[myAggRank];
-		}
-		numTargetProcs++;
-	    }
-	}
-    }
-
-
-    MPI_Request *mpiRecvDataFromSourceAggsRequest = (MPI_Request *) ADIOI_Malloc(numSourceAggs * sizeof(MPI_Request));
-    MPI_Request *mpiSendDataToTargetProcRequest = (MPI_Request *) ADIOI_Malloc(numTargetProcs * sizeof(MPI_Request));
-    MPI_Status mpiWaitAnyStatusFromSourceProcs,mpiIsendStatusForData;
-
-    /* use the two-phase buffer allocated in the file_open - no app should ever
-     * be both reading and writing at the same time */
-    char *read_buf0 = fd->io_buf;
-    char *read_buf1 = fd->io_buf + coll_bufsize;
-    /* if threaded i/o selected, we'll do a kind of double buffering */
-    char *read_buf = read_buf0;
-
-    /* compute number of rounds */
-    ADIO_Offset numberOfRounds = (ADIO_Offset)((((ADIO_Offset)(end_offsets[nprocs-1]-st_offsets[0]))/((ADIO_Offset)((ADIO_Offset)coll_bufsize*(ADIO_Offset)naggs)))) + 1;
-
-    ADIO_Offset currentRoundFDStart = 0, nextRoundFDStart = 0;
-    ADIO_Offset currentRoundFDEnd = 0, nextRoundFDEnd = 0;
-
-    if (iAmUsedAgg) {
-	currentRoundFDStart = fd_start[myAggRank];
-	nextRoundFDStart = fd_start[myAggRank];
-    }
-
-    int *dataSizeSentThisRoundPerProc = (int *)ADIOI_Malloc(numTargetProcs * sizeof(int));
-    int *sourceAggIndexesForMyDataThisRound = (int *)ADIOI_Malloc(numSourceAggs * sizeof(int));
-    int *recvBufferOffsetsThisRound = (int *)ADIOI_Malloc(numSourceAggs * sizeof(int));
-    int *bufferAmountsToGetThisRound = (int *)ADIOI_Malloc(numSourceAggs * sizeof(int));
-    *error_code = MPI_SUCCESS;
-
-    int currentReadBuf = 0;
-    int useIOBuffer = 0;
-#ifdef ROMIO_GPFS
-    if (gpfsmpio_pthreadio && (numberOfRounds>1)) {
-	useIOBuffer = 1;
-	io_thread = pthread_self();
-    }
-#endif
-
-#ifdef ROMIO_GPFS
-    endTimeBase = MPI_Wtime();
-    gpfsmpio_prof_cw[GPFSMPIO_CIO_T_MYREQ] += (endTimeBase-startTimeBase);
-#endif
-
-
-    /* each iteration of this loop reads a coll_bufsize portion of the file domain */
-    int roundIter;
-    for (roundIter=0;roundIter<numberOfRounds;roundIter++) {
-
-	int irecv,isend;
-	/* determine what offsets define the portion of the file domain the agg is reading this round */
-	if (iAmUsedAgg) {
-
-	    currentRoundFDStart = nextRoundFDStart;
-
-	    if (!useIOBuffer || (roundIter == 0)) {
-		int amountDataToReadThisRound;
-		if ((fd_end[myAggRank] - currentRoundFDStart) < coll_bufsize) {
-		    currentRoundFDEnd = fd_end[myAggRank];
-		    amountDataToReadThisRound = ((currentRoundFDEnd-currentRoundFDStart)+1);
-		}
-		else {
-		    currentRoundFDEnd = currentRoundFDStart + coll_bufsize - 1;
-		    amountDataToReadThisRound = coll_bufsize;
-		}
-
-		/* read currentRoundFDEnd bytes */
-		ADIO_ReadContig(fd, read_buf,amountDataToReadThisRound,
-			MPI_BYTE, ADIO_EXPLICIT_OFFSET, currentRoundFDStart,
-			&status, error_code);
-        currentReadBuf = 1;
-
-#ifdef ROMIO_GPFS
-		endTimeBase = MPI_Wtime();
-#endif
-	    }
-
-	    if (useIOBuffer) { /* use the thread reader for the next round */
-		/* switch back and forth between the read buffers so that the data aggregation code is diseminating 1 buffer while the thread is reading into the other */
-
-		if (roundIter > 0)
-		    currentRoundFDEnd = nextRoundFDEnd;
-
-		if (roundIter < (numberOfRounds-1)) {
-		    nextRoundFDStart += coll_bufsize;
-		    int amountDataToReadNextRound;
-		    if ((fd_end[myAggRank] - nextRoundFDStart) < coll_bufsize) {
-			nextRoundFDEnd = fd_end[myAggRank];
-			amountDataToReadNextRound = ((nextRoundFDEnd-nextRoundFDStart)+1);
-		    }
-		    else {
-			nextRoundFDEnd = nextRoundFDStart + coll_bufsize - 1;
-			amountDataToReadNextRound = coll_bufsize;
-		    }
-
-		    if(!pthread_equal(io_thread, pthread_self())) {
-			pthread_join(io_thread, &thread_ret);
-			*error_code = *(int *)thread_ret;
-			if (*error_code != MPI_SUCCESS) return;
-			io_thread = pthread_self();
-
-		    }
-		    io_thread_args.fd = fd;
-		    /* do a little pointer shuffling: background I/O works from one
-		     * buffer while two-phase machinery fills up another */
-
-		    if (currentReadBuf == 0) {
-			io_thread_args.buf = read_buf0;
-			currentReadBuf = 1;
-			read_buf = read_buf1;
-		    }
-		    else {
-			io_thread_args.buf = read_buf1;
-			currentReadBuf = 0;
-			read_buf = read_buf0;
-		    }
-		    io_thread_args.io_kind = ADIOI_READ;
-		    io_thread_args.size = amountDataToReadNextRound;
-		    io_thread_args.offset = nextRoundFDStart;
-		    io_thread_args.status = &status;
-		    io_thread_args.error_code = *error_code;
-		    if ( (pthread_create(&io_thread, NULL,
-				    ADIOI_IO_Thread_Func, &(io_thread_args))) != 0)
-			io_thread = pthread_self();
-
-		}
-		else { /* last round */
-
-		    if(!pthread_equal(io_thread, pthread_self())) {
-			pthread_join(io_thread, &thread_ret);
-			*error_code = *(int *)thread_ret;
-			if (*error_code != MPI_SUCCESS) return;
-			io_thread = pthread_self();
-
-		    }
-		    if (currentReadBuf == 0) {
-			read_buf = read_buf1;
-		    }
-		    else {
-			read_buf = read_buf0;
-		    }
-
-		}
-	    } /* useIOBuffer */
-	} /* IAmUsedAgg */
-
-	/* determine what source aggs I need to get data from this round and
-	 * recv only from them */
-	int numSourceAggsThisRound = 0;
-	for (i=0;i<numSourceAggs;i++) {
-	    if ( ((myOffsetStart >= sourceAggsForMyDataFDStart[i]) && (myOffsetStart <= sourceAggsForMyDataFDEnd[i]))
-		    || ((myOffsetEnd >= sourceAggsForMyDataFDStart[i]) && (myOffsetEnd <= sourceAggsForMyDataFDEnd[i])) ) {
-		/* we know that we need to get data from this source agg at
-		 * some point, now need to figure out how much this round */
-
-		/* here are the offsets currently being sent by the aggregator
-		 * during this round */
-		ADIO_Offset currentRoundFDStartForMySourceAgg =
-		    (ADIO_Offset)((ADIO_Offset)sourceAggsForMyDataFDStart[i] +
-			    (ADIO_Offset)((ADIO_Offset)roundIter*(ADIO_Offset)coll_bufsize));
-		ADIO_Offset currentRoundFDEndForMySourceAgg =
-		    (ADIO_Offset)((ADIO_Offset)sourceAggsForMyDataFDStart[i] +
-			    (ADIO_Offset)((ADIO_Offset)(roundIter+1)*(ADIO_Offset)coll_bufsize) - (ADIO_Offset)1);
-		if (currentRoundFDEndForMySourceAgg > sourceAggsForMyDataFDEnd[i])
-		    currentRoundFDEndForMySourceAgg = sourceAggsForMyDataFDEnd[i];
-
-#ifdef p2pcontigtrace
-		printf("roundIter %d source iter %d sourceAggsForMyData is %d myOffsetStart is %ld myOffsetEnd is %ld sourceAggsForMyDataFDStart is %ld sourceAggsForMyDataFDEnd is %ld currentRoundFDStartForMySourceAgg is %ld currentRoundFDEndForMySourceAgg is %ld\n",roundIter,i,sourceAggsForMyData[i],myOffsetStart,myOffsetEnd,sourceAggsForMyDataFDStart[i],sourceAggsForMyDataFDEnd[i],currentRoundFDStartForMySourceAgg,currentRoundFDEndForMySourceAgg);
-#endif
-
-		/* get the portion of my data that is within currentRoundFDStartForMySourceAgg to currentRoundFDEndForMySourceAgg */
-		/* find the offset into the recv buffer and the amount of data to get */
-		int recvBufferOffset = 0;
-		int bufferAmountToGet = 0;
-
-		if ((myOffsetStart >= currentRoundFDStartForMySourceAgg) && (myOffsetStart <= currentRoundFDEndForMySourceAgg)) {
-		    if (myOffsetEnd > currentRoundFDEndForMySourceAgg)
-			bufferAmountToGet = (currentRoundFDEndForMySourceAgg - myOffsetStart) +1;
-		    else
-			bufferAmountToGet = (myOffsetEnd - myOffsetStart) +1;
-		}
-		else if ((myOffsetEnd >= currentRoundFDStartForMySourceAgg) && (myOffsetEnd <= currentRoundFDEndForMySourceAgg)) {
-		    recvBufferOffset = (int) (currentRoundFDStartForMySourceAgg - myOffsetStart);
-		    if (myOffsetEnd > currentRoundFDEndForMySourceAgg)
-			bufferAmountToGet = (currentRoundFDEndForMySourceAgg - currentRoundFDStartForMySourceAgg) +1;
-		    else
-			bufferAmountToGet = (myOffsetEnd - currentRoundFDStartForMySourceAgg) +1;
-		}
-		else if ((myOffsetStart <= currentRoundFDStartForMySourceAgg) && (myOffsetEnd >= currentRoundFDEndForMySourceAgg)) {
-		    recvBufferOffset = (int) (currentRoundFDStartForMySourceAgg - myOffsetStart);
-		    bufferAmountToGet = (currentRoundFDEndForMySourceAgg - currentRoundFDStartForMySourceAgg) +1;
-		}
-
-
-		if (bufferAmountToGet > 0) { /* we have data to get this round */
-		    sourceAggIndexesForMyDataThisRound[numSourceAggsThisRound] = i;
-		    recvBufferOffsetsThisRound[numSourceAggsThisRound] = recvBufferOffset;
-		    bufferAmountsToGetThisRound[numSourceAggsThisRound] = bufferAmountToGet;
-#ifdef p2pcontigtrace
-		    printf("bufferAmountToGet is %d recvBufferOffset is %d\n",bufferAmountToGet,recvBufferOffset);
-#endif
-		    numSourceAggsThisRound++;
-		}
-	    }
-	}
-
-	/* the aggs determine the amount of data they will be sending to their
-	 * source procs */
-	for (i=0;i<numTargetProcs;i++) {
-	    if ((remainingDataOffsetToSendPerProc[i] >= currentRoundFDStart) &&
-		    (remainingDataOffsetToSendPerProc[i] <= currentRoundFDEnd)) {
-		if ((remainingDataOffsetToSendPerProc[i] +
-			    remainingDataAmountToSendPerProc[i]) <= currentRoundFDEnd)
-		    dataSizeSentThisRoundPerProc[i] = remainingDataAmountToSendPerProc[i];
-		else
-		    dataSizeSentThisRoundPerProc[i] =
-			(currentRoundFDEnd - remainingDataOffsetToSendPerProc[i]) +1;
-	    }
-	    else if (((remainingDataOffsetToSendPerProc[i]+
-			    remainingDataAmountToSendPerProc[i]) >=
-			currentRoundFDStart) &&
-		    ((remainingDataOffsetToSendPerProc[i]+
-		      remainingDataAmountToSendPerProc[i]) <= currentRoundFDEnd)) {
-		if ((remainingDataOffsetToSendPerProc[i]) >= currentRoundFDStart)
-		    dataSizeSentThisRoundPerProc[i] = remainingDataAmountToSendPerProc[i];
-		else
-		    dataSizeSentThisRoundPerProc[i] =
-			(remainingDataOffsetToSendPerProc[i]-currentRoundFDStart) +1;
-	    }
-	    else
-		dataSizeSentThisRoundPerProc[i] = 0;
-
-	}
-
-	/* the target procs get the data from the source aggs */
-	for (i = 0; i < numSourceAggsThisRound; i++) {
-	    MPI_Irecv(&((char*)buf)[recvBufferOffsetsThisRound[i]],
-		    bufferAmountsToGetThisRound[i],MPI_BYTE,
-		    sourceAggsForMyData[sourceAggIndexesForMyDataThisRound[i]],0,fd->comm,
-		    &mpiRecvDataFromSourceAggsRequest[i]);
-	}
-
-	/* the source aggs send the data to the target procs */
-	int numTargetProcsSentThisRound = 0;
-	for (i=0;i<numTargetProcs;i++) {
-
-	    int currentWBOffset = 0;
-	    for (j=0;j<i;j++)
-		currentWBOffset += dataSizeSentThisRoundPerProc[j];
-
-	    /* only send to target procs that will recv > 0 count data */
-	    if (dataSizeSentThisRoundPerProc[i] > 0) {
-		MPI_Isend(&((char*)read_buf)[currentWBOffset],
-			dataSizeSentThisRoundPerProc[i],
-			MPI_BYTE,targetProcsForMyData[i],0,
-			fd->comm,&mpiSendDataToTargetProcRequest[numTargetProcsSentThisRound]);
-		numTargetProcsSentThisRound++;
-		remainingDataAmountToSendPerProc[i] -= dataSizeSentThisRoundPerProc[i];
-		remainingDataOffsetToSendPerProc[i] += dataSizeSentThisRoundPerProc[i];
-	    }
-	}
-
-	/* wait for the target procs to get their data */
-	for (i = 0; i < numSourceAggsThisRound; i++) {
-	    MPI_Waitany(numSourceAggsThisRound,mpiRecvDataFromSourceAggsRequest,
-		    &irecv,&mpiWaitAnyStatusFromSourceProcs);
-	}
-
-	nextRoundFDStart = currentRoundFDStart + coll_bufsize;
-
-        /* clean up the MPI_Isend MPI_Requests */
-        for (i=0;i<numTargetProcsSentThisRound;i++) {
-          MPI_Waitany(numTargetProcsSentThisRound,mpiSendDataToTargetProcRequest,
-		  &isend,&mpiIsendStatusForData);
-        }
-
-	MPI_Barrier(fd->comm); /* need to sync up the source aggs which did the isend with the target procs which did the irecvs to give the target procs time to get the data before overwriting with next round readcontig */
-
-    } /* for-loop roundIter */
-
-    if (useIOBuffer) { /* thread reader cleanup */
-
-	if ( !pthread_equal(io_thread, pthread_self()) ) {
-	    pthread_join(io_thread, &thread_ret);
-	    *error_code = *(int *)thread_ret;
-	}
-    }
-
-    if (iAmUsedAgg) {
-	ADIOI_Free(targetProcsForMyData);
-	ADIOI_Free(remainingDataAmountToSendPerProc);
-	ADIOI_Free(remainingDataOffsetToSendPerProc);
-    }
-
-    ADIOI_Free(sourceAggsForMyData);
-    ADIOI_Free(sourceAggsForMyDataFDStart);
-    ADIOI_Free(sourceAggsForMyDataFDEnd);
-
-    ADIOI_Free(mpiRecvDataFromSourceAggsRequest);
-    ADIOI_Free(mpiSendDataToTargetProcRequest);
-    ADIOI_Free(dataSizeSentThisRoundPerProc);
-    ADIOI_Free(sourceAggIndexesForMyDataThisRound);
-    ADIOI_Free(recvBufferOffsetsThisRound);
-    ADIOI_Free(bufferAmountsToGetThisRound);
-
-    /* TODO: is Barrier here needed? */
-    MPI_Barrier(fd->comm);
-
-    return;
-
-}
diff --git a/3rd-party/romio321/adio/common/req_malloc.c b/3rd-party/romio321/adio/common/req_malloc.c
deleted file mode 100644
index 61018e1a684..00000000000
--- a/3rd-party/romio321/adio/common/req_malloc.c
+++ /dev/null
@@ -1,83 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "adio_extern.h"
-
-struct ADIOI_RequestD *ADIOI_Malloc_request(void)
-{
-/* returns a pointer to a new request object.
-   To reduce the number of system calls, mallocs NUM requests at a time
-   and maintains list of available requests. Supplies an object from this
-   list if available, else mallocs a new set of NUM and provides one
-   from that set. Is NUM=100 a good number? */
-
-#define NUM 100
-
-    ADIOI_Req_node *curr, *ptr;
-    int i;
-
-    if (!ADIOI_Req_avail_head) {
-	ADIOI_Req_avail_head = (ADIOI_Req_node *) 
-	              ADIOI_Malloc(NUM*sizeof(ADIOI_Req_node));
-	if (ADIOI_Req_avail_head == NULL)
-	{
-	    /* FIXME: Insert error here */
-	    return NULL;
-	}
-	curr = ADIOI_Req_avail_head;
-	for (i=1; i<NUM; i++) {
-	    curr->next = ADIOI_Req_avail_head+i;
-	    curr = curr->next;
-	}
-	curr->next = NULL;
-	ADIOI_Req_avail_tail = curr;
-
-	/* keep track of malloced area that needs to be freed later */
-	if (!ADIOI_Malloc_req_tail) {
-	    ADIOI_Malloc_req_tail = (ADIOI_Malloc_req *)
-		ADIOI_Malloc(sizeof(ADIOI_Malloc_req)); 
-	    ADIOI_Malloc_req_head = ADIOI_Malloc_req_tail;
-	    ADIOI_Malloc_req_head->ptr = ADIOI_Req_avail_head;
-	    ADIOI_Malloc_req_head->next = NULL;
-	}
-	else {
-	    ADIOI_Malloc_req_tail->next = (ADIOI_Malloc_req *)
-		ADIOI_Malloc(sizeof(ADIOI_Malloc_req));
-	    ADIOI_Malloc_req_tail = ADIOI_Malloc_req_tail->next;
-	    ADIOI_Malloc_req_tail->ptr = ADIOI_Req_avail_head;
-	    ADIOI_Malloc_req_tail->next = NULL;
-	}
-    }
-
-    ptr = ADIOI_Req_avail_head;
-    ADIOI_Req_avail_head = ADIOI_Req_avail_head->next;
-    if (!ADIOI_Req_avail_head) ADIOI_Req_avail_tail = NULL;
-    
-    (ptr->reqd).cookie = ADIOI_REQ_COOKIE;
-    return &(ptr->reqd);
-}
-
-
-void ADIOI_Free_request(ADIOI_Req_node *node)
-{
-/* This function could be called as ADIOI_Free_request(ADIO_Request request), 
-   because request would be a pointer to the first element of ADIOI_Req_node.*/
-
-/* moves this node to available pool. does not actually free it. */
-
-    (node->reqd).cookie = 0;
-
-    if (!ADIOI_Req_avail_tail)
-	ADIOI_Req_avail_head = ADIOI_Req_avail_tail = node;
-    else {
-	ADIOI_Req_avail_tail->next = node;
-	ADIOI_Req_avail_tail = node;
-    }
-    node->next = NULL;
-}
-
diff --git a/3rd-party/romio321/adio/common/shfp_fname.c b/3rd-party/romio321/adio/common/shfp_fname.c
deleted file mode 100644
index dfa5bafc2cf..00000000000
--- a/3rd-party/romio321/adio/common/shfp_fname.c
+++ /dev/null
@@ -1,102 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-
-#ifdef HAVE_TIME_H
-#include <time.h>
-#endif
-/* The following function selects the name of the file to be used to 
-   store the shared file pointer. The shared-file-pointer file is a 
-   hidden file in the same directory as the real file being accessed.
-   If the real file is /tmp/thakur/testfile, the shared-file-pointer
-   file will be /tmp/thakur/.testfile.shfp.yyy.xxxx, where yyy
-   is rank 0's process id and xxxx is a random number. If the
-   underlying file system supports shared file pointers
-   (PVFS does not, for example), the file name is always
-   constructed. This file is created only if the shared
-   file pointer functions are used and is deleted when the real
-   file is closed. */
-
-void ADIOI_Shfp_fname(ADIO_File fd, int rank, int *error_code)
-{
-    int i;
-    int len;
-    char *slash, *ptr, tmp[128];
-    int pid = 0;
-
-    fd->shared_fp_fname = (char *) ADIOI_Malloc(PATH_MAX);
-
-    if (!rank) {
-        srand(time(NULL));
-        i = rand();
-	pid = (int)getpid();
-	
-	if (ADIOI_Strncpy(fd->shared_fp_fname, fd->filename, PATH_MAX)) {
-	    *error_code = ADIOI_Err_create_code("ADIOI_Shfp_fname",
-		    fd->filename, ENAMETOOLONG);
-	    return;
-	}
-	
-#ifdef ROMIO_NTFS
-	slash = strrchr(fd->filename, '\\');
-#else
-	slash = strrchr(fd->filename, '/');
-#endif
-	if (!slash) {
-	    if (ADIOI_Strncpy(fd->shared_fp_fname, ".", 2)) {
-		*error_code = ADIOI_Err_create_code("ADIOI_Shfp_fname",
-			fd->filename, ENAMETOOLONG);
-		return;
-	    }
-	    if (ADIOI_Strncpy(fd->shared_fp_fname + 1, fd->filename, PATH_MAX-1)) {
-		*error_code = ADIOI_Err_create_code("ADIOI_Shfp_fname",
-			fd->filename, ENAMETOOLONG);
-		return;
-	    }
-	}
-	else {
-	    ptr = slash;
-#ifdef ROMIO_NTFS
-		slash = strrchr(fd->shared_fp_fname, '\\');
-#else
-	    slash = strrchr(fd->shared_fp_fname, '/');
-#endif
-	    if (ADIOI_Strncpy(slash + 1, ".", 2))  {
-		*error_code = ADIOI_Err_create_code("ADIOI_Shfp_fname",
-			fd->filename, ENAMETOOLONG);
-		return;
-	    }
-	    /* ok to cast: file names bounded by PATH_MAX and NAME_MAX */
-	    len = (int) (PATH_MAX - (slash+2 - fd->shared_fp_fname));
-	    if (ADIOI_Strncpy(slash + 2, ptr + 1, len)) {
-		*error_code = ADIOI_Err_create_code("ADIOI_Shfp_fname",
-			ptr + 1, ENAMETOOLONG);
-		return;
-	    }
-	}
-	    
-	ADIOI_Snprintf(tmp, 128, ".shfp.%d.%d", pid, i);
-	/* ADIOI_Strnapp will return non-zero if truncated.  That's ok */
-	ADIOI_Strnapp(fd->shared_fp_fname, tmp, PATH_MAX);
-	
-	len = (int)strlen(fd->shared_fp_fname);
-	MPI_Bcast(&len, 1, MPI_INT, 0, fd->comm);
-	MPI_Bcast(fd->shared_fp_fname, len+1, MPI_CHAR, 0, fd->comm);
-    }
-    else {
-	MPI_Bcast(&len, 1, MPI_INT, 0, fd->comm);
-	MPI_Bcast(fd->shared_fp_fname, len+1, MPI_CHAR, 0, fd->comm);
-    }
-}
diff --git a/3rd-party/romio321/adio/common/status_setb.c b/3rd-party/romio321/adio/common/status_setb.c
deleted file mode 100644
index ec1e4ff7f69..00000000000
--- a/3rd-party/romio321/adio/common/status_setb.c
+++ /dev/null
@@ -1,37 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpi.h"
-
-#if defined(MPICH2) || (defined(MPICH) && (MPICH_NAME >= 3))
-/* Not quite correct, but much closer for MPI2 */
-/* TODO: still needs to handle partial datatypes and situations where the mpi
- * implementation fills status with something other than bytes (globus2 might
- * do this) */
-int MPIR_Status_set_bytes(MPI_Status *status, MPI_Datatype datatype, 
-			  MPI_Count nbytes)
-{
-    ADIOI_UNREFERENCED_ARG(datatype);
-    /* it's ok that ROMIO stores number-of-bytes in status, not 
-     * count-of-copies, as long as MPI_GET_COUNT knows what to do */
-    if (status != MPI_STATUS_IGNORE)
-        MPI_Status_set_elements_x(status, MPI_BYTE, nbytes);
-    return MPI_SUCCESS;
-}
-#elif defined(MPILAM) || defined(MPISGI)
-int MPIR_Status_set_bytes(MPI_Status *status, MPI_Datatype datatype,
-		MPI_Count nbytes)
-{
-  /* Bogusness to silence compiler warnings */
-  if (datatype == MPI_DATATYPE_NULL);
-
-  if (status != MPI_STATUS_IGNORE)
-	  status->st_length = nbytes;
-  return MPI_SUCCESS;
-}
-#endif
diff --git a/3rd-party/romio321/adio/common/strfns.c b/3rd-party/romio321/adio/common/strfns.c
deleted file mode 100644
index 65e5ac24e74..00000000000
--- a/3rd-party/romio321/adio/common/strfns.c
+++ /dev/null
@@ -1,315 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-
-/* style: allow:sprintf:3 sig:0 */
-
-/* 
- * Below are the "safe" versions of the various string and printf
- * operations. They are directly taken from MPICH, with MPIU replaced by ADIOI.
- */
-
-/*
- * ADIOI_Strncpy - Copy at most n character.  Stop once a null is reached.
- *
- * This is different from strncpy, which null pads so that exactly
- * n characters are copied.  The strncpy behavior is correct for many 
- * applications because it guarantees that the string has no uninitialized
- * data.
- *
- * If n characters are copied without reaching a null, return an error.
- * Otherwise, return 0.
- *
- */
-/*@ ADIOI_Strncpy - Copy a string with a maximum length
-  
-Input Parameters:
-+   instr - String to copy
--   maxlen - Maximum total length of 'outstr'
-
-Output Parameters:
-.   outstr - String to copy into
-
-    Notes:
-    This routine is the routine that you wish 'strncpy' was.  In copying 
-    'instr' to 'outstr', it stops when either the end of 'outstr' (the 
-    null character) is seen or the maximum length 'maxlen' is reached.
-    Unlike 'strncpy', it does not add enough nulls to 'outstr' after 
-    copying 'instr' in order to move precisely 'maxlen' characters.  
-    Thus, this routine may be used anywhere 'strcpy' is used, without any
-    performance cost related to large values of 'maxlen'.
-
-  Module:
-  Utility
-  @*/
-int ADIOI_Strncpy( char *dest, const char *src, size_t n )
-{
-    char * restrict d_ptr = dest;
-    const char * restrict s_ptr = src;
-    register int i;
-
-    i = (int)n;
-    while (*s_ptr && i-- > 0) {
-	*d_ptr++ = *s_ptr++;
-    }
-    
-    if (i > 0) { 
-	*d_ptr = 0;
-	return 0;
-    }
-    else
-	/* We may want to force an error message here, at least in the
-	   debugging version */
-	return 1;
-}
-
-/* Append src to dest, but only allow dest to contain n characters (including
-   any null, which is always added to the end of the line */
-/*@ ADIOI_Strnapp - Append to a string with a maximum length
-
-Input Parameters:
-+   instr - String to copy
--   maxlen - Maximum total length of 'outstr'
-
-Output Parameters:
-.   outstr - String to copy into
-
-    Notes:
-    This routine is similar to 'strncat' except that the 'maxlen' argument
-    is the maximum total length of 'outstr', rather than the maximum 
-    number of characters to move from 'instr'.  Thus, this routine is
-    easier to use when the declared size of 'instr' is known.
-
-  Module:
-  Utility
-  @*/
-int ADIOI_Strnapp( char *dest, const char *src, size_t n )
-{
-    char * restrict d_ptr = dest;
-    const char * restrict s_ptr = src;
-    register int i;
-
-    /* Get to the end of dest */
-    i = (int)n;
-    while (i-- > 0 && *d_ptr) d_ptr++;
-    if (i <= 0) return 1;
-
-    /* Append.  d_ptr points at first null and i is remaining space. */
-    while (*s_ptr && i-- > 0) {
-	*d_ptr++ = *s_ptr++;
-    }
-
-    /* We allow i >= (not just >) here because the first while decrements
-       i by one more than there are characters, leaving room for the null */
-    if (i >= 0) { 
-	*d_ptr = 0;
-	return 0;
-    }
-    else {
-	/* Force the null at the end */
-	*--d_ptr = 0;
-    
-	/* We may want to force an error message here, at least in the
-	   debugging version */
-	return 1;
-    }
-}
-
-/*@ 
-  ADIOI_Strdup - Duplicate a string
-
-  Synopsis:
-.vb
-    char *ADIOI_Strdup( const char *str )
-.ve
-
-Input Parameters:
-. str - null-terminated string to duplicate
-
-  Return value:
-  A pointer to a copy of the string, including the terminating null.  A
-  null pointer is returned on error, such as out-of-memory.
-
-  Notes:
-  Like 'ADIOI_Malloc' and 'ADIOI_Free', this will often be implemented as a 
-  macro but may use 'ADIOI_trstrdup' to provide a tracing version.
-
-  Module:
-  Utility
-  @*/
-char *ADIOI_Strdup( const char *str )
-{
-    char *p = ADIOI_Malloc( strlen(str) + 1 );
-    char *in_p = (char *)str;
-    char *save_p;
-
-    save_p = p;
-    if (p) {
-	while (*in_p) {
-	    *p++ = *in_p++;
-	}
-	*p = '\0';
-    }
-    return save_p;
-}
-
-
-/* 
- * We need an snprintf replacement for systems without one
- */
-#ifndef HAVE_SNPRINTF
-#include <ctype.h>
-/* FIXME: Really need a check for varargs.h vs stdarg.h */
-#include <stdarg.h>
-/* 
- * This is an approximate form which is suitable for most uses within
- * the MPICH code
- */
-int ADIOI_Snprintf( char *str, size_t size, const char *format, ... )
-{
-    int n;
-    const char *p;
-    char *out_str = str;
-    va_list list;
-
-    va_start(list, format);
-
-    p = format;
-    while (*p && size > 0) {
-	char *nf;
-
-	nf = strchr(p, '%');
-	if (!nf) {
-	    /* No more format characters */
-	    while (size-- > 0 && *p) {
-		*out_str++ = *p++;
-	    }
-	}
-	else {
-	    int nc;
-	    int width = -1;
-
-	    /* Copy until nf */
-	    while (p < nf && size-- > 0) {
-		*out_str++ = *p++;
-	    }
-	    /* p now points at nf */
-	    /* Handle the format character */
-	    nc = nf[1];
-	    if (isdigit(nc)) {
-		/* Get the field width */
-		/* FIXME : Assumes ASCII */
-		width = nc - '0';
-		p = nf + 2;
-		while (*p && isdigit(*p)) {
-		    width = 10 * width + (*p++ - '0');
-		}
-		/* When there is no longer a digit, get the format 
-		   character */
-		nc = *p++;
-	    }
-	    else {
-		/* Skip over the format string */
-		p += 2;
-	    }
-
-	    switch (nc) {
-	    case '%':
-		*out_str++ = '%';
-		size--;
-		break;
-
-	    case 'd':
-	    {
-		int val;
-		char tmp[20];
-		char *t = tmp;
-		/* Get the argument, of integer type */
-		val = va_arg( list, int );
-		sprintf( tmp, "%d", val );
-		if (width > 0) {
-		    int tmplen = strlen(tmp);
-		    /* If a width was specified, pad with spaces on the
-		       left (on the right if %-3d given; not implemented yet */
-		    while (size-- > 0 && width-- > tmplen) 
-			*out_str++ = ' ';
-		}
-		while (size-- > 0 && *t) {
-		    *out_str++ = *t++;
-		}
-	    }
-	    break;
-
-	    case 'x':
-	    {
-		int val;
-		char tmp[20];
-		char *t = tmp;
-		/* Get the argument, of integer type */
-		val = va_arg( list, int );
-		sprintf( tmp, "%x", val );
-		if (width > 0) {
-		    int tmplen = strlen(tmp);
-		    /* If a width was specified, pad with spaces on the
-		       left (on the right if %-3d given; not implemented yet */
-		    while (size-- > 0 && width-- > tmplen) 
-			*out_str++ = ' ';
-		}
-		while (size-- > 0 && *t) {
-		    *out_str++ = *t++;
-		}
-	    }
-	    break;
-
-	    case 'p':
-	    {
-		void *val;
-		char tmp[20];
-		char *t = tmp;
-		val = va_arg( list, void * );
-		sprintf( tmp, "%p", val );
-		if (width > 0) {
-		    int tmplen = strlen(tmp);
-		    /* If a width was specified, pad with spaces on the
-		       left (on the right if %-3d given; not implemented yet */
-		    while (size-- > 0 && width-- > tmplen) 
-			*out_str++ = ' ';
-		}
-		while (size-- > 0 && *t) {
-		    *out_str++ = *t++;
-		}
-	    }
-	    break;
-
-	    case 's':
-	    {
-		char *s_arg;
-		/* Get the argument, of pointer to char type */
-		s_arg = va_arg( list, char * );
-		while (size-- > 0 && s_arg && *s_arg) {
-		    *out_str++ = *s_arg++;
-		}
-	    }
-	    break;
-
-	    default:
-		/* Error, unknown case */
-		return -1;
-		break;
-	    }
-	}
-    }
-
-    va_end(list);
-
-    if (size-- > 0) *out_str++ = '\0';
-
-    n = (int)(out_str - str);
-    return n;
-}
-#endif
diff --git a/3rd-party/romio321/adio/common/system_hints.c b/3rd-party/romio321/adio/common/system_hints.c
deleted file mode 100644
index 1b049b1ff5b..00000000000
--- a/3rd-party/romio321/adio/common/system_hints.c
+++ /dev/null
@@ -1,202 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- 
- * vim: ts=8 sts=4 sw=4 noexpandtab
- *
- *   Copyright (C) 2007 UChicago/Argonne LLC. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include <adio.h>
-
-#include <stdio.h>
-
-#ifdef HAVE_FCNTL_H
-#include <fcntl.h>
-#endif
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-#ifdef HAVE_STDLIB_H
-#include <stdlib.h>
-#endif
-#ifdef HAVE_STRING_H
-#include <string.h>
-#endif
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_IO_H
-#include <io.h>
-#endif
-
-/*#define SYSHINT_DEBUG 1  */
-
-#define ROMIO_HINT_DEFAULT_CFG "/etc/romio-hints"
-#define ROMIO_HINT_ENV_VAR "ROMIO_HINTS"
-
-/* debug function: a routine I want in the library to make my life easier when
- * using a source debugger.  Now optionally used in ADIO_Open. */
-void ADIOI_Info_print_keyvals(MPI_Info info)
-{
-    int i, nkeys, flag;
-    char key[MPI_MAX_INFO_KEY+1];
-    char value[MPI_MAX_INFO_VAL+1];
-
-    if (info == MPI_INFO_NULL)
-	return;
-
-    MPI_Info_get_nkeys(info, &nkeys);
-
-    for (i=0; i<nkeys; i++) {
-	MPI_Info_get_nthkey(info, i, key);
-	ADIOI_Info_get(info, key, MPI_MAX_INFO_VAL, value, &flag);
-	printf("key = %-25s value = %-10s\n", key, value);
-    }
-    return;
-}
-
-/* if user set the environment variable, use its value to find the
- * file-of-hints.  Otherwise, we'll look for the default config file.  i.e. let
- * the user override systemwide hint processing */
-
-static int find_file(void)
-{
-    int fd=-1;
-    char * hintfile;
-    
-    hintfile = getenv(ROMIO_HINT_ENV_VAR);
-    if(hintfile)
-    fd = open(hintfile, O_RDONLY);
-    if (fd < 0 )
-	fd = open(ROMIO_HINT_DEFAULT_CFG, O_RDONLY);
-
-    return fd;
-}
-
-/* parse the file-of-hints.  Format is zero or more lines of "<key> <value>\n".
- * A # in collumn zero is a comment and the line will be ignored.  Do our best
- * to ignore badly formed lines too. 
- *
- * The caller provides an 'info' object.  Each key-value pair found by the
- * parser will get added to the info object.  any keys already set will be left
- * alone on the assumption that the caller knows best. 
- *
- * because MPI-IO hints are optional, we can get away with limited error
- * reporting.
- *
- * for better scalability, the config file will be read on one processor and
- * broadcast to all others */
-static int file_to_info_all(int fd, MPI_Info info, int rank, MPI_Comm comm)
-{
-    char *buffer, *token, *key, *val, *garbage;
-    char *pos1=NULL, *pos2=NULL;
-    int flag;
-    ssize_t ret;
-    int valuelen;
-
-    /* assumption: config files will be small */
-#define HINTFILE_MAX_SIZE 1024*4
-    buffer = (char *)ADIOI_Calloc(HINTFILE_MAX_SIZE, sizeof (char));
-
-    if (rank == 0) {
-	ret = read(fd, buffer, HINTFILE_MAX_SIZE);
-	/* any error: bad/nonexistent fd, no perms, anything: set up a null
-	 * buffer and the subsequent string parsing will quit immediately */
-	if (ret == -1)
-	    buffer[0] = '\0';
-    }
-    MPI_Bcast(buffer, HINTFILE_MAX_SIZE, MPI_BYTE, 0, comm);
-
-    token = strtok_r(buffer, "\n", &pos1);
-    if (token == NULL)
-	goto fn_exit;
-    do {
-	if ( (key = strtok_r(token, " \t", &pos2)) == NULL) 
-	    /* malformed line: found no items */
-	    continue;
-	if (token[0] == '#') 
-	    /* ignore '#'-delimited comments */
-	    continue;
-	if ( (val = strtok_r(NULL, " \t", &pos2))  == NULL) 
-	    /* malformed line: found key without value */
-	    continue;
-	if ( (garbage = strtok_r(NULL, " \t", &pos2)) != NULL) 
-	    /* malformed line: more than two items */
-	    continue;
-	    
-#ifdef SYSHINT_DEBUG
-	printf("found: key=%s val=%s\n", key, val);
-#endif
-	/* don't actually care what the value is. only want to know if key
-	 * exists: we leave it alone if so*/
-	ADIOI_Info_get_valuelen(info, key, &valuelen, &flag);
-	if (flag == 1) continue;
-	ADIOI_Info_set(info, key, val);
-    } while ((token = strtok_r(NULL, "\n", &pos1)) != NULL);
-
-fn_exit:
-    ADIOI_Free(buffer);
-    return 0;
-}
-
-void ADIOI_process_system_hints(ADIO_File fd, MPI_Info info)
-{
-    int hintfd=-1, rank;
-
-    MPI_Comm_rank(fd->comm, &rank);
-    if (rank == 0) {
-	hintfd = find_file();
-    }
-    /* hintfd only significant on rank 0.  -1 (on rank 0) means no hintfile found  */
-    file_to_info_all(hintfd, info, rank, fd->comm);
-
-    if (hintfd != -1)
-	close(hintfd);
-}
-
-/* given 'info', incorporate any hints in 'sysinfo' that are not already set
- * into 'new_info'.  Caller must free 'new_info' later. */
-void ADIOI_incorporate_system_hints(MPI_Info info, 
-	MPI_Info sysinfo, 
-	MPI_Info *new_info) 
-{
-    int i, nkeys_sysinfo, nkeys_info=0, flag=0; /* must initialize flag to 0 */
-    int valuelen;
-
-    char  val[MPI_MAX_INFO_VAL+1], key[MPI_MAX_INFO_KEY+1];
-
-    if (sysinfo == MPI_INFO_NULL)
-	nkeys_sysinfo = 0;
-    else
-	MPI_Info_get_nkeys(sysinfo, &nkeys_sysinfo);
-
-    /* short-circuit: return immediately if no hints to process */
-    if (info == MPI_INFO_NULL && nkeys_sysinfo == 0)  {
-	*new_info = MPI_INFO_NULL;
-	return;
-    }
-
-    if (info == MPI_INFO_NULL) 
-	MPI_Info_create(new_info);
-    else {
-	/* tiny optimization: if 'info' has no keys, we can skip the check if a
-	 * hint is set: no keys means nothing has been set, and there's nothing
-	 * we might step on */
-	MPI_Info_get_nkeys(info, &nkeys_info);
-	MPI_Info_dup(info, new_info);
-    }
-
-    for (i=0; i<nkeys_sysinfo; i++) {
-	MPI_Info_get_nthkey(sysinfo, i, key);
-	/* don't care about the value, just want to know if hint set already*/
-	if (info != MPI_INFO_NULL && nkeys_info)
-	    ADIOI_Info_get_valuelen(info, key, &valuelen, &flag);
-	if (flag == 1) continue;  /* skip any hints already set by user */
-	ADIOI_Info_get(sysinfo, key, MPI_MAX_INFO_VAL, val, &flag);
-	ADIOI_Info_set(*new_info, key, val);
-	flag = 0;
-    }
-
-    return;
-}
-
-
diff --git a/3rd-party/romio321/adio/common/utils.c b/3rd-party/romio321/adio/common/utils.c
deleted file mode 100644
index 16f7dc60e2c..00000000000
--- a/3rd-party/romio321/adio/common/utils.c
+++ /dev/null
@@ -1,159 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *
- *   Copyright (C) 2014 UChicgo/Argonne, LLC.
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include <adio.h>
-#include <limits.h>
-
-
-/* utility function for creating large contiguous types: algorithim from BigMPI
- * https://github.com/jeffhammond/BigMPI */
-
-static int type_create_contiguous_x(MPI_Count count,
-	MPI_Datatype oldtype, MPI_Datatype *newtype)
-{
-    /* to make 'count' fit MPI-3 type processing routines (which take integer
-     * counts), we construct a type consisting of N INT_MAX chunks followed by
-     * a remainder.  e.g for a count of 4000000000 bytes you would end up with
-     * one 2147483647-byte chunk followed immediately by a 1852516353-byte
-     * chunk */
-    MPI_Datatype chunks, remainder;
-    MPI_Aint lb, extent, disps[2];
-    int blocklens[2];
-    MPI_Datatype types[2];
-
-    /* truly stupendously large counts will overflow an integer with this math,
-     * but that is a problem for a few decades from now.  Sorry, few decades
-     * from now! */
-    ADIOI_Assert(count/INT_MAX == (int)(count/INT_MAX));
-    int c = (int)(count/INT_MAX); /* OK to cast until 'count' is 256 bits */
-    int r = count%INT_MAX;
-
-    MPI_Type_vector(c, INT_MAX, INT_MAX, oldtype, &chunks);
-    MPI_Type_contiguous(r, oldtype, &remainder);
-
-    MPI_Type_get_extent(oldtype, &lb, &extent);
-
-    blocklens[0] = 1;      blocklens[1] = 1;
-    disps[0]     = 0;      disps[1]     = c*extent*INT_MAX;
-    types[0]     = chunks; types[1]     = remainder;
-
-    MPI_Type_create_struct(2, blocklens, disps, types, newtype);
-
-    MPI_Type_free(&chunks);
-    MPI_Type_free(&remainder);
-
-    return MPI_SUCCESS;
-}
-/* like MPI_Type_create_hindexed, except array_of_lengths can be a larger datatype.
- *
- * Hindexed provides 'count' pairs of (displacement, length), but what if
- * length is longer than an integer?  We will create 'count' types, using
- * contig if length is small enough, or something more complex if not */
-
-int ADIOI_Type_create_hindexed_x(int count,
-		const MPI_Count array_of_blocklengths[],
-		const MPI_Aint array_of_displacements[],
-		MPI_Datatype oldtype,
-		MPI_Datatype *newtype)
-{
-    int i, ret;
-    MPI_Datatype *types;
-    int *blocklens;
-    int is_big=0;
-
-    types = ADIOI_Malloc(count*sizeof(MPI_Datatype));
-    blocklens = ADIOI_Malloc(count*sizeof(int));
-
-    /* squashing two loops into one.
-     * - Look in the array_of_blocklengths for any large values
-     * - convert MPI_Count items (if they are not too big) into int-sized items
-     * after this loop we will know if we can use MPI_type_hindexed or if we
-     * need a more complicated BigMPI-style struct-of-chunks.
-     *
-     * Why not use the struct-of-chunks in all cases?  HDF5 reported a bug,
-     * which I have not yet precicesly nailed down, but appears to have
-     * something to do with struct-of-chunks when the chunks are small */
-
-    for(i=0; i<count; i++) {
-	if (array_of_blocklengths[i] > INT_MAX) {
-	    blocklens[i] = 1;
-	    is_big=1;
-	    type_create_contiguous_x(array_of_blocklengths[i], oldtype,  &(types[i]));
-	} else {
-	    /* OK to cast: checked for "bigness" above */
-	    blocklens[i] = (int)array_of_blocklengths[i];
-	    MPI_Type_contiguous(blocklens[i], oldtype, &(types[i]));
-	}
-    }
-
-    if (is_big) {
-	ret = MPI_Type_create_struct(count, blocklens, array_of_displacements,
-		types, newtype);
-    } else {
-	ret = MPI_Type_create_hindexed(count, blocklens,
-		array_of_displacements, oldtype, newtype);
-    }
-    for (i=0; i< count; i++)
-	MPI_Type_free(&(types[i]));
-    ADIOI_Free(types);
-    ADIOI_Free(blocklens);
-
-    return ret;
-}
-
-/* some systems do not have pread/pwrite, or requrie XOPEN_SOURCE set higher
- * than we would like.  see #1973 */
-#if (HAVE_DECL_PWRITE == 0)
-
-#include <sys/types.h>
-#include <unistd.h>
-
-ssize_t pread(int fd, void *buf, size_t count, off_t offset);
-ssize_t pwrite(int fd, const void *buf, size_t count, off_t offset);
-
-ssize_t pread(int fd, void *buf, size_t count, off_t offset) {
-    off_t lseek_ret;
-    off_t old_offset;
-    ssize_t read_ret;
-
-    old_offset = lseek(fd, 0, SEEK_CUR);
-    lseek_ret = lseek(fd, offset, SEEK_SET);
-    if (lseek_ret == -1)
-	return lseek_ret;
-    read_ret = read(fd, buf, count);
-    if (read_ret < 0)
-	return read_ret;
-    /* man page says "file offset is not changed" */
-    if ( (lseek_ret = lseek(fd, old_offset, SEEK_SET)) < 0)
-	return lseek_ret;
-
-    return read_ret;
-}
-
-ssize_t pwrite(int fd, const void *buf, size_t count, off_t offset) {
-    off_t lseek_ret;
-    off_t old_offset;
-    ssize_t write_ret;
-
-    old_offset = lseek(fd, 0, SEEK_CUR);
-    lseek_ret = lseek(fd, offset, SEEK_SET);
-    if (lseek_ret == -1)
-	return lseek_ret;
-    write_ret = write(fd, buf, count);
-    if (write_ret < 0)
-	return write_ret;
-    /* man page says "file offset is not changed" */
-    if ( (lseek_ret = lseek(fd, old_offset, SEEK_SET)) < 0)
-	return lseek_ret;
-
-    return write_ret;
-}
-#endif
-
-/*
- * vim: ts=8 sts=4 sw=4 noexpandtab
- */
diff --git a/3rd-party/romio321/adio/include/BaseIOErrMsgs.msg b/3rd-party/romio321/adio/include/BaseIOErrMsgs.msg
deleted file mode 100644
index 0ca0526619a..00000000000
--- a/3rd-party/romio321/adio/include/BaseIOErrMsgs.msg
+++ /dev/null
@@ -1,51 +0,0 @@
-$set 12 MPI_ERR_ARG
-12	65	"Invalid offset argument"
-12	67	"Null datarep argument"
-12	69	"Invalid count argument"
-12	71	"Invalid size argument"
-12	73	"Invalid whence argument"
-12	75	"flag argument must be the same on all processes"
-12	77	"Invalid displacement argument"
-12	79	"Invalid etype argument"
-12	81	"Invalid filetype argument"
-12	83	"size argument must be the same on all processes"
-12	85	"offset points to a negative location in the file"
-12	87	"whence argument must be the same on all processes"
-12	89	"offset argument must be the same on all processes"
-$set 21 MPI_ERR_AMODE
-21	1	"Invalid amode"
-21	3	"Exactly one of MPI_MODE_RDONLY, MPI_MODE_WRONLY, or MPI_MODE_RDWR must be specified"
-21	5	"Cannot use MPI_MODE_CREATE or MPI_MODE_EXCL with MPI_MODE_RDONLY"
-21	7	"Cannot specify MPI_MODE_SEQUENTIAL with MPI_MODE_RDWR"
-$set 27 MPI_ERR_FILE
-27	1	"Invalid file handle"
-27	3	"Null file handle"
-27	5	"Corrupted file handle"
-$set 32 MPI_ERR_IO
-32	1	"I/O Error"
-32	2	"%s"	1
-32	3	"Only an integral number of etypes can be accessed"
-32	5	"Can't determine the file-system type. Check the filename/path you provided and try again. Otherwise, prefix the filename with a string to indicate the type of file system (piofs:, pfs:, nfs:, ufs:, hfs:, xfs:, sfs:, pvfs:, pvfs2:, ftp:, gsiftp:)"
-32	7	"ROMIO has not been configured to use the PFS file system"
-32	9	"ROMIO has not been configured to use the PIOFS file system"
-32	11	"ROMIO has not been configured to use the UFS file system"
-32	13	"ROMIO has not been configured to use the NFS file system"
-32	15	"ROMIO has not been configured to use the HFS file system"
-32	17	"ROMIO has not been configured to use the XFS file system"
-32	19	"ROMIO has not been configured to use the SFS file system"
-32	21	"ROMIO has not been configured to use the PVFS file system"
-32	23	"Only one active split collective I/O operation allowed per file handle"
-32	25	"No previous split collective begin"
-32	27	"There are outstanding nonblocking I/O operations on this file"
-32	29	"ROMIO tries to optimize this access by doing a read-modify-write, but is unable to read the file. Please give the file read permission and open it with MPI_MODE_RDWR."
-32	31	"To preallocate disk space, ROMIO needs to read the file and write it back, but is unable to read the file. Please give the file read permission and open it with MPI_MODE_RDWR."
-32	33	"Filetype must be constructed out of one or more etypes"
-$set 43 MPI_ERR_UNSUPPORTED_DATAREP
-43	1	"Unsupported data representation"
-43	3	"Only native data representation currently supported"
-$set 44 MPI_ERR_UNSUPPORTED_OPERATION
-44	1	"Unsupported operation"
-44	3	"Shared file pointer not supported on PIOFS and PVFS"
-44	5	"Cannot use this function when file is opened with amode MPI_MODE_SEQUENTIAL"
-44	7	"Cannot read from a file opened with amode MPI_MODE_WRONLY"
-44	9	"MPI_MODE_SEQUENTIAL not supported on PIOFS and PVFS"
diff --git a/3rd-party/romio321/adio/include/adio.h b/3rd-party/romio321/adio/include/adio.h
deleted file mode 100644
index fedd8f58f55..00000000000
--- a/3rd-party/romio321/adio/include/adio.h
+++ /dev/null
@@ -1,474 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-/* main include file for ADIO.
-   contains general definitions, declarations, and macros independent 
-   of the underlying file system */
-
-/* Functions and datataypes that are "internal" to the ADIO implementation 
-   prefixed ADIOI_. Functions and datatypes that are part of the
-   "externally visible" (documented) ADIO interface are prefixed ADIO_.
-
-   An implementation of MPI-IO, or any other high-level interface, should
-   not need to use any of the ADIOI_ functions/datatypes. 
-   Only someone implementing ADIO on a new file system, or modifying 
-   an existing ADIO implementation, would need to use the ADIOI_
-   functions/datatypes. */
-
-#ifndef ADIO_INCLUDE
-#define ADIO_INCLUDE
-
-#ifdef SPPUX
-#define _POSIX_SOURCE
-#endif
-
-#ifdef USE_FORT_STDCALL
-#define FORT_CALL __stdcall
-#elif defined (USE_FORT_CDECL)
-#define FORT_CALL __cdecl
-#else
-#define FORT_CALL
-#endif
-
-#ifdef USE_FORT_MIXED_STR_LEN
-#define FORT_MIXED_LEN_DECL   , int
-#define FORT_END_LEN_DECL
-#define FORT_MIXED_LEN(a)     , int a
-#define FORT_END_LEN(a)
-#else
-#define FORT_MIXED_LEN_DECL
-#define FORT_END_LEN_DECL     , int
-#define FORT_MIXED_LEN(a)
-#define FORT_END_LEN(a)       , int a
-#endif
-
-#ifdef HAVE_FORTRAN_API
-# ifdef FORTRAN_EXPORTS
-#  define FORTRAN_API __declspec(dllexport)
-# else
-#  define FORTRAN_API __declspec(dllimport)
-# endif
-#else
-# define FORTRAN_API
-#endif
-
-/* Use this macro for each parameter to a function that is not referenced in the body of the function */
-#ifdef HAVE_WINDOWS_H
-#define ADIOI_UNREFERENCED_ARG(a) a
-#else
-#define ADIOI_UNREFERENCED_ARG(a)
-#endif
-
-/* Include romioconf.h if we haven't already (some include files may
-   need to include romioconf before some system includes) */
-#ifndef ROMIOCONF_H_INCLUDED
-#include "romioconf.h"
-#define ROMIOCONF_H_INCLUDED
-#endif
-
-#include "mpi.h"
-#include "mpio.h"
-#ifdef HAVE_FCNTL_H
-#include <fcntl.h>
-#endif
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-#include <sys/stat.h>
-#include <string.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#ifdef SPPUX
-#include <sys/cnx_fcntl.h>
-#endif
-
-#ifdef ROMIO_NTFS
-#include <winsock2.h>
-#include <windows.h>
-#define FDTYPE HANDLE
-#else
-#define FDTYPE int
-#endif
-
-#ifdef MPI_OFFSET_IS_INT
-   typedef int ADIO_Offset;
-   typedef unsigned ADIO_Size;
-#  define ADIO_OFFSET MPI_INT
-#elif defined(HAVE_LONG_LONG_64)
-   typedef long long ADIO_Offset;
-   typedef unsigned long long ADIO_Size;
-#  ifdef HAVE_MPI_LONG_LONG_INT
-#     define ADIO_OFFSET MPI_LONG_LONG_INT
-#  else
-#     define ADIO_OFFSET MPI_DOUBLE
-#  endif
-#elif defined(HAVE_INT64)
-   typedef __int64 ADIO_Offset;
-   typedef unsigned __int64 ADIO_Size;
-#  define ADIO_OFFSET MPI_DOUBLE
-#else
-   typedef long ADIO_Offset;
-   typedef unsigned long ADIO_Size;
-#  define ADIO_OFFSET MPI_LONG
-#endif
-
-#define ADIO_Status MPI_Status   
-
-#ifndef MPIO_INCLUDE
-#  ifdef NEEDS_MPI_FINT
-      typedef int MPI_Fint; 
-#  endif
-#endif
-
-#if (!defined(HAVE_MPI_INFO) && !defined(MPIO_INCLUDE))
-   typedef struct MPIR_Info *MPI_Info;
-#  define MPI_INFO_NULL 0
-#  define MPI_MAX_INFO_VAL      1024
-
-int MPI_Info_create(MPI_Info *info);
-int MPI_Info_set(MPI_Info info, char *key, char *value);
-int MPI_Info_delete(MPI_Info info, char *key);
-int MPI_Info_get(MPI_Info info, char *key, int valuelen, 
-                         char *value, int *flag);
-int MPI_Info_get_valuelen(MPI_Info info, char *key, int *valuelen, int *flag);
-int MPI_Info_get_nkeys(MPI_Info info, int *nkeys);
-int MPI_Info_get_nthkey(MPI_Info info, int n, char *key);
-int MPI_Info_dup(MPI_Info info, MPI_Info *newinfo);
-int MPI_Info_free(MPI_Info *info);
-
-#ifdef MPI_Info_f2c
-#undef MPI_Info_f2c
-#endif
-#ifdef MPI_Info_c2f
-#undef MPI_Info_c2f
-#endif
-/* above needed for some versions of mpi.h in MPICH!! */
-MPI_Fint MPI_Info_c2f(MPI_Info info);
-MPI_Info MPI_Info_f2c(MPI_Fint info);
-
-int PMPI_Info_create(MPI_Info *info);
-int PMPI_Info_set(MPI_Info info, char *key, char *value);
-int PMPI_Info_delete(MPI_Info info, char *key);
-int PMPI_Info_get(MPI_Info info, char *key, int valuelen, 
-                         char *value, int *flag);
-int PMPI_Info_get_valuelen(MPI_Info info, char *key, int *valuelen, int *flag);
-int PMPI_Info_get_nkeys(MPI_Info info, int *nkeys);
-int PMPI_Info_get_nthkey(MPI_Info info, int n, char *key);
-int PMPI_Info_dup(MPI_Info info, MPI_Info *newinfo);
-int PMPI_Info_free(MPI_Info *info);
-
-MPI_Fint PMPI_Info_c2f(MPI_Info info);
-MPI_Info PMPI_Info_f2c(MPI_Fint info);
-
-#endif
-
-/* style: allow:strdup:1 sig:0 */
-
-#if defined(HAVE_STRDUP) && defined(NEEDS_STRDUP_DECL) && !defined(strdup)
-char *strdup(const char *s);
-# endif
-#if defined(HAVE_READLINK) && defined(NEEDS_READLINK_DECL) && !defined(readlink)
-ssize_t readlink(const char *path, char *buf, size_t bufsiz);
-# endif
-#if defined(HAVE_LSTAT) && defined(NEEDS_LSTAT_DECL) && !defined(lstat)
-int lstat(const char *file_name, struct stat *buf);
-# endif
-#if defined(HAVE_FSYNC) && defined(NEEDS_FSYNC_DECL) && !defined(fsync)
-int fsync(int fd);
-# endif
-#if defined(HAVE_FTRUNCATE) && defined(NEEDS_FTRUNCATE_DECL) && !defined(ftruncate)
-int ftruncate(int fd, off_t length);
-# endif
-
-
-typedef struct ADIOI_Fns_struct ADIOI_Fns;
-typedef struct ADIOI_Hints_struct ADIOI_Hints;
-
-typedef struct ADIOI_FileD {
-    int cookie;              /* for error checking */
-    FDTYPE fd_sys;              /* system file descriptor */
-    FDTYPE null_fd;          /* the null-device file descriptor: debug only (obviously)*/
-    int fd_direct;           /* On XFS, this is used for direct I/O; 
-                                fd_sys is used for buffered I/O */
-    int direct_read;         /* flag; 1 means use direct read */
-    int direct_write;        /* flag; 1 means use direct write  */
-    /* direct I/O attributes */
-    unsigned d_mem;          /* data buffer memory alignment */
-    unsigned d_miniosz;      /* min xfer size, xfer size multiple,
-                                and file seek offset alignment */
-    long blksize;            /* some optimizations benefit from knowing
-				underlying block size */
-    ADIO_Offset fp_ind;      /* individual file pointer in MPI-IO (in bytes)*/
-    ADIO_Offset fp_sys_posn; /* current location of the system file-pointer
-                                in bytes */
-    ADIOI_Fns *fns;          /* struct of I/O functions to use */
-    MPI_Comm comm;           /* communicator indicating who called open */
-    int is_open;	    /* deferred open: 0: not open yet 1: is open */
-    int is_agg;              /* bool: if I am an aggregator */
-    char *filename;          
-    int file_system;         /* type of file system */
-    int access_mode;         /* Access mode (sequential, append, etc.),
-				possibly modified to deal with
-				data sieving or deferred open*/
-    int orig_access_mode;    /* Access mode provided by user: unmodified */
-    ADIO_Offset disp;        /* reqd. for MPI-IO */
-    MPI_Datatype etype;      /* reqd. for MPI-IO */
-    MPI_Datatype filetype;   /* reqd. for MPI-IO */
-    MPI_Count etype_size;          /* in bytes */
-    ADIOI_Hints *hints;      /* structure containing fs-indep. info values */
-    MPI_Info info;
-
-    /* The following support the split collective operations */
-    int split_coll_count;    /* count of outstanding split coll. ops. */
-    MPI_Status split_status; /* status used for split collectives */
-    MPI_Datatype split_datatype; /* datatype used for split collectives */
-
-    /* The following support the shared file operations */
-    char *shared_fp_fname;   /* name of file containing shared file pointer */
-    struct ADIOI_FileD *shared_fp_fd;  /* file handle of file 
-                                         containing shared fp */
-    int async_count;         /* count of outstanding nonblocking operations */
-    int perm;
-    int atomicity;          /* true=atomic, false=nonatomic */
-    int fortran_handle;     /* handle for Fortran interface if needed */
-    MPI_Errhandler err_handler;
-    void *fs_ptr;            /* file-system specific information */
-
-    /* Two phase collective I/O support */
-    ADIO_Offset *file_realm_st_offs; /* file realm starting offsets */
-    MPI_Datatype *file_realm_types;  /* file realm datatypes */
-    int my_cb_nodes_index; /* my index into cb_config_list. -1 if N/A */
-    char *io_buf;          /* two-phase buffer allocated out of i/o path */
-    MPI_Win io_buf_window; /* Window over the io_buf to support one-sided aggregation */
-    int *io_buf_put_amounts; /* array tracking the amount of data mpi_put into the io_buf
-                                during the same round of one-sided write aggregation */
-    MPI_Win io_buf_put_amounts_window; /* Window over the io_buf_put_amounts */
-    /* External32 */
-    int is_external32;      /* bool:  0 means native view */
-
-} ADIOI_FileD;
-
-typedef struct ADIOI_FileD *ADIO_File;
-
-typedef MPI_Request ADIO_Request;
-
-/* fcntl structure */
-typedef struct {
-    ADIO_Offset disp;
-    MPI_Datatype etype;
-    MPI_Datatype filetype;
-    MPI_Info info;
-    int atomicity;
-    ADIO_Offset fsize;       /* for get_fsize only */
-    ADIO_Offset diskspace;   /* for file preallocation */
-} ADIO_Fcntl_t;              /* should contain more stuff */
-
-
-/* access modes */
-#define ADIO_CREATE              1
-#define ADIO_RDONLY              2
-#define ADIO_WRONLY              4
-#define ADIO_RDWR                8
-#define ADIO_DELETE_ON_CLOSE     16
-#define ADIO_UNIQUE_OPEN         32
-#define ADIO_EXCL                64
-#define ADIO_APPEND             128
-#define ADIO_SEQUENTIAL         256
-
-#define ADIO_AMODE_NOMATCH  ~(ADIO_CREATE|ADIO_RDONLY|ADIO_WRONLY|ADIO_RDWR|ADIO_DELETE_ON_CLOSE|ADIO_UNIQUE_OPEN|ADIO_EXCL|ADIO_APPEND|ADIO_SEQUENTIAL)
-
-/* file-pointer types */
-#define ADIO_EXPLICIT_OFFSET     100
-#define ADIO_INDIVIDUAL          101
-#define ADIO_SHARED              102
-
-#define ADIO_REQUEST_NULL        ((ADIO_Request) 0)
-#define ADIO_FILE_NULL           ((ADIO_File) 0)
-
-/* file systems */
-#define ADIO_NFS                 150
-#define ADIO_PIOFS               151   /* IBM */
-#define ADIO_UFS                 152   /* Unix file system */
-#define ADIO_PFS                 153   /* Intel */
-#define ADIO_XFS                 154   /* SGI */
-#define ADIO_HFS                 155   /* HP/Convex */
-#define ADIO_SFS                 156   /* NEC */
-#define ADIO_PVFS                157   /* PVFS for Linux Clusters from Clemson Univ. */
-#define ADIO_NTFS                158   /* NTFS for Windows NT */
-#define ADIO_TESTFS              159   /* fake file system for testing */
-#define ADIO_PVFS2               160   /* PVFS2: 2nd generation PVFS */
-#define ADIO_PANFS               161   /* Panasas FS */
-#define ADIO_GRIDFTP             162   /* Globus GridFTP */
-#define ADIO_LUSTRE              163   /* Lustre */
-/* #define ADIO_BGL              164 */  /* IBM BGL */
-/* #define ADIO_BGLOCKLESS       165 */  /* IBM BGL (lock-free) */
-#define ADIO_ZOIDFS              167   /* ZoidFS: the I/O forwarding fs */
-/* #define ADIO_BG               168 */
-#define ADIO_GPFS                  168
-
-#define ADIO_SEEK_SET            SEEK_SET
-#define ADIO_SEEK_CUR            SEEK_CUR
-#define ADIO_SEEK_END            SEEK_END
-
-#define ADIO_FCNTL_SET_ATOMICITY 180
-#define ADIO_FCNTL_SET_DISKSPACE 188
-#define ADIO_FCNTL_GET_FSIZE     200
-
-/* file system feature tests */
-#define ADIO_LOCKS               300 /* file system supports fcntl()-style locking */
-#define ADIO_SHARED_FP           301 /* file system supports shared file pointers */
-#define ADIO_ATOMIC_MODE         302 /* file system supports atomic mode */
-#define ADIO_DATA_SIEVING_WRITES 303 /* file system supports data sieving for writes */
-#define ADIO_SCALABLE_OPEN       304 /* one process can open the file and
-					broadcast result to all other
-					processors */
-#define ADIO_UNLINK_AFTER_CLOSE  305 /* supports posix semantic of keeping a
-					deleted file around until all
-					processors have closed it */
-#define ADIO_TWO_PHASE           306 /* file system implements some version of
-					two-phase collective buffering with
-					aggregation */
-#define ADIO_SCALABLE_RESIZE     307 /* file system supports resizing from one
-					processor (nfs, e.g. does not) */
-
-/* for default file permissions */
-#define ADIO_PERM_NULL           -1
-
-#define ADIOI_FILE_COOKIE 2487376
-#define ADIOI_REQ_COOKIE 3493740
-
-/* ADIO function prototypes */
-/* all these may not be necessary, as many of them are macro replaced to 
-   function pointers that point to the appropriate functions for each
-   different file system (in adioi.h), but anyway... */
-
-void ADIO_Init(int *argc, char ***argv, int *error_code);
-void ADIO_End(int *error_code);
-MPI_File ADIO_Open(MPI_Comm orig_comm, MPI_Comm comm, const char *filename,
-		   int file_system, ADIOI_Fns *ops,
-		   int access_mode, ADIO_Offset disp, MPI_Datatype etype,
-		   MPI_Datatype filetype,
-		   MPI_Info info, int perm, int *error_code);
-void ADIOI_OpenColl(ADIO_File fd, int rank, int acces_mode, int *error_code);
-void ADIO_ImmediateOpen(ADIO_File fd, int *error_code);
-void ADIO_Close(ADIO_File fd, int *error_code);
-void ADIO_ReadContig(ADIO_File fd, void *buf, int count, MPI_Datatype datatype,
-                    int file_ptr_type,  ADIO_Offset offset, 
-                    ADIO_Status *status, int *error_code);
-void ADIO_WriteContig(ADIO_File fd, void *buf, int count, 
-                     MPI_Datatype datatype, int file_ptr_type,
-                      ADIO_Offset offset, int *bytes_written, int
-		      *error_code);
-void ADIO_IwriteContig(ADIO_File fd, void *buf, int count, 
-                      MPI_Datatype datatype, int file_ptr_type,
-                      ADIO_Offset offset, ADIO_Request *request, int
-		      *error_code);   
-void ADIO_IreadContig(ADIO_File fd, void *buf, int count, 
-                      MPI_Datatype datatype, int file_ptr_type,
-                      ADIO_Offset offset, ADIO_Request *request, int
-		      *error_code);   
-int ADIO_ReadDone(ADIO_Request *request, ADIO_Status *status, 
-               int *error_code);
-int ADIO_WriteDone(ADIO_Request *request, ADIO_Status *status, 
-               int *error_code);
-int ADIO_ReadIcomplete(ADIO_Request *request, ADIO_Status *status, int
-		       *error_code); 
-int ADIO_WriteIcomplete(ADIO_Request *request, ADIO_Status *status,
-			int *error_code); 
-void ADIO_ReadComplete(ADIO_Request *request, ADIO_Status *status, int
-		       *error_code); 
-void ADIO_WriteComplete(ADIO_Request *request, ADIO_Status *status,
-			int *error_code); 
-void ADIO_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct, int
-		*error_code); 
-void ADIO_ReadStrided(ADIO_File fd, void *buf, int count,
-		       MPI_Datatype datatype, int file_ptr_type,
-		       ADIO_Offset offset, ADIO_Status *status, int
-		       *error_code);
-void ADIO_WriteStrided(ADIO_File fd, const void *buf, int count,
-		       MPI_Datatype datatype, int file_ptr_type,
-		       ADIO_Offset offset, ADIO_Status *status, int
-		       *error_code);
-void ADIO_ReadStridedColl(ADIO_File fd, void *buf, int count,
-		       MPI_Datatype datatype, int file_ptr_type,
-		       ADIO_Offset offset, ADIO_Status *status, int
-		       *error_code);
-void ADIO_WriteStridedColl(ADIO_File fd, void *buf, int count,
-		       MPI_Datatype datatype, int file_ptr_type,
-		       ADIO_Offset offset, ADIO_Status *status, int
-		       *error_code);
-void ADIO_IreadStrided(ADIO_File fd, void *buf, int count,
-		       MPI_Datatype datatype, int file_ptr_type,
-		       ADIO_Offset offset, ADIO_Request *request, int
-		       *error_code);
-void ADIO_IwriteStrided(ADIO_File fd, void *buf, int count,
-		       MPI_Datatype datatype, int file_ptr_type,
-		       ADIO_Offset offset, ADIO_Request *request, int
-		       *error_code);
-void ADIO_IreadStridedColl(ADIO_File fd, void *buf, int count,
-               MPI_Datatype datatype, int file_ptr_type,
-               ADIO_Offset offset, ADIO_Request *request,
-               int *error_code);
-void ADIO_IwriteStridedColl(ADIO_File fd, void *buf, int count,
-               MPI_Datatype datatype, int file_ptr_type,
-               ADIO_Offset offset, ADIO_Request *request,
-               int *error_code);
-ADIO_Offset ADIO_SeekIndividual(ADIO_File fd, ADIO_Offset offset, 
-                       int whence, int *error_code);
-void ADIO_Delete(char *filename, int *error_code);
-void ADIO_Flush(ADIO_File fd, int *error_code);
-void ADIO_Resize(ADIO_File fd, ADIO_Offset size, int *error_code);
-void ADIO_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code);
-void ADIO_ResolveFileType(MPI_Comm comm, const char *filename, int *fstype,
-          ADIOI_Fns **ops, int *error_code);
-void ADIO_Get_shared_fp(ADIO_File fd, ADIO_Offset size, ADIO_Offset *shared_fp,
-			 int *error_code);
-void ADIO_Set_shared_fp(ADIO_File fd, ADIO_Offset offset, int *error_code);
-void ADIO_Set_view(ADIO_File fd, ADIO_Offset disp, MPI_Datatype etype, 
-		MPI_Datatype filetype, MPI_Info info,  int *error_code);
-int  ADIO_Feature(ADIO_File fd, int flag);
-
-/* functions to help deal with the array datatypes */
-int ADIO_Type_create_subarray(int ndims,
-                              int *array_of_sizes,
-                              int *array_of_subsizes,
-                              int *array_of_starts,
-                              int order,
-                              MPI_Datatype oldtype,
-                              MPI_Datatype *newtype);
-int ADIO_Type_create_darray(int size, int rank, int ndims, 
-			    int *array_of_gsizes, int *array_of_distribs, 
-			    int *array_of_dargs, int *array_of_psizes, 
-			    int order, MPI_Datatype oldtype, 
-			    MPI_Datatype *newtype);
-
-/* MPI_File management functions (in mpio_file.c) */
-MPI_File MPIO_File_create(int size);
-ADIO_File MPIO_File_resolve(MPI_File mpi_fh);
-void MPIO_File_free(MPI_File *mpi_fh);
-MPI_File MPIO_File_f2c(MPI_Fint fh);
-MPI_Fint MPIO_File_c2f(MPI_File fh);
-int MPIO_Err_create_code(int lastcode, int fatal, const char fcname[],
-			 int line, int error_class, const char generic_msg[],
-			 const char specific_msg[], ... );
-int MPIO_Err_return_file(MPI_File mpi_fh, int error_code);
-int MPIO_Err_return_comm(MPI_Comm mpi_comm, int error_code);
-
-/* request managment helper functions */
-void MPIO_Completed_request_create(MPI_File *fh, MPI_Offset nbytes,
-		int * error_code, MPI_Request *request);
-
-#include "adioi.h"
-#include "adioi_fs_proto.h"
-#include "mpio_error.h"
-#include "mpipr.h"
-
-/* these two defines don't appear to be in any other header file */
-#define MPIR_ERR_FATAL 1
-#define MPIR_ERR_RECOVERABLE 0
-
-#endif
diff --git a/3rd-party/romio321/adio/include/adio_cb_config_list.h b/3rd-party/romio321/adio/include/adio_cb_config_list.h
deleted file mode 100644
index ab2c5b501f7..00000000000
--- a/3rd-party/romio321/adio/include/adio_cb_config_list.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2001 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-/* I have no idea what the "D" stands for; it's how things are done in adio.h 
- */
-struct ADIO_cb_name_arrayD {
-       int refct;
-       int namect;
-       char **names;
-};
-
-typedef struct ADIO_cb_name_arrayD *ADIO_cb_name_array;
-
-int ADIOI_cb_gather_name_array(MPI_Comm comm, MPI_Comm dupcomm, 
-			       ADIO_cb_name_array *arrayp);
-int ADIOI_cb_copy_name_array(MPI_Comm comm, int keyval, void *extra, 
-			     void *attr_in,
-			     void **attr_out, int *flag);
-int ADIOI_cb_delete_name_array(MPI_Comm comm, int keyval, void *attr_val, 
-			       void *extra);
-int ADIOI_cb_config_list_parse(char *config_list, ADIO_cb_name_array array, 
-			       int ranklist[], int cb_nodes);
-int ADIOI_cb_bcast_rank_map(ADIO_File fd);
diff --git a/3rd-party/romio321/adio/include/adio_extern.h b/3rd-party/romio321/adio/include/adio_extern.h
deleted file mode 100644
index 9a7f2e59b04..00000000000
--- a/3rd-party/romio321/adio/include/adio_extern.h
+++ /dev/null
@@ -1,31 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-extern ADIOI_Flatlist_node *ADIOI_Flatlist;
-
-extern ADIOI_Datarep *ADIOI_Datarep_head;
-
-/* for f2c and c2f conversion */
-extern ADIO_File *ADIOI_Ftable;
-extern int ADIOI_Ftable_ptr, ADIOI_Ftable_max;
-extern ADIO_Request *ADIOI_Reqtable;
-extern int ADIOI_Reqtable_ptr, ADIOI_Reqtable_max;
-#ifndef HAVE_MPI_INFO
-extern MPI_Info *MPIR_Infotable;
-extern int MPIR_Infotable_ptr, MPIR_Infotable_max;
-#endif
-#if defined(ROMIO_XFS) || defined(ROMIO_LUSTRE)
-extern int ADIOI_Direct_read, ADIOI_Direct_write;
-#endif
-
-extern MPI_Errhandler ADIOI_DFLT_ERR_HANDLER;
-
-extern MPI_Info ADIOI_syshints;
-
-extern MPI_Op ADIO_same_amode;
-
-extern int ADIOI_cb_config_list_keyval;
diff --git a/3rd-party/romio321/adio/include/adioi.h b/3rd-party/romio321/adio/include/adioi.h
deleted file mode 100644
index 5656f10bae7..00000000000
--- a/3rd-party/romio321/adio/include/adioi.h
+++ /dev/null
@@ -1,1104 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-
-/* contains general definitions, declarations, and macros internal to
-   the ADIO implementation, though independent of the underlying file
-   system. This file is included in adio.h */ 
-
-/* Functions and datataypes that are "internal" to the ADIO implementation 
-   prefixed ADIOI_. Functions and datatypes that are part of the
-   "externally visible" (documented) ADIO interface are prefixed ADIO_.
-
-   An implementation of MPI-IO, or any other high-level interface, should
-   not need to use any of the ADIOI_ functions/datatypes. 
-   Only someone implementing ADIO on a new file system, or modifying 
-   an existing ADIO implementation, would need to use the ADIOI_
-   functions/datatypes. */
-
-
-#ifndef ADIOI_INCLUDE
-#define ADIOI_INCLUDE
-/* used to keep track of hint/info values.
- * Note that there are a lot of int-sized values in here...they are
- * used as int-sized entities other places as well.  This would be a 
- * problem on 32-bit systems using > 2GB files in some cases...
- */
-struct ADIOI_Hints_struct {
-    int initialized;
-    int striping_factor;
-    int striping_unit;
-    int cb_read;
-    int cb_write;
-    int cb_nodes;
-    int cb_buffer_size;
-    int cb_pfr;
-    int cb_fr_type;
-    int cb_fr_alignment;
-    int cb_ds_threshold;
-    int cb_alltoall;
-    int ds_read;
-    int ds_write;
-    int no_indep_rw;
-    int ind_rd_buffer_size;
-    int ind_wr_buffer_size;
-    int deferred_open;
-    int min_fdomain_size;
-    char *cb_config_list;
-    int *ranklist;
-    union {
-	    struct {
-		    int listio_read;
-		    int listio_write;
-	    } pvfs;
-	    struct {
-		    int debugmask;
-		    int posix_read;
-		    int posix_write;
-		    int listio_read;
-		    int listio_write;
-		    int dtype_read;
-		    int dtype_write;
-	    } pvfs2;
-            struct {
-                    int start_iodevice;
-                    int co_ratio;
-                    int coll_threshold;
-                    int ds_in_coll;
-            } lustre;
-		struct {
-			unsigned read_chunk_sz; /* chunk size for direct reads */
-			unsigned write_chunk_sz; /* chunk size for direct writes */
-		} xfs;
-	struct {
-	    int *bridgelist; /* list of all bride ranks */
-	    int *bridgelistnum; /* each entry here is the number of aggregators
-				   associated with the bridge rank of the same
-				   index in bridgelist */
-	    int numbridges; /* total number of bridges */
-	} bg;
-    } fs_hints;
-
-};
-
-typedef struct ADIOI_Datarep {
-    char *name;
-    void *state;
-    MPI_Datarep_extent_function *extent_fn;
-    MPI_Datarep_conversion_function *read_conv_fn;
-    MPI_Datarep_conversion_function *write_conv_fn;
-    struct ADIOI_Datarep *next; /* pointer to next datarep */
-} ADIOI_Datarep;
-
-/* Values for use with cb_read, cb_write, ds_read, and ds_write 
- * and some fs-specific hints
-   (IBM xlc, Compaq Tru64 compilers object to a comma after the last item)
-   (that's just wrong)
- */
-enum {
-    ADIOI_HINT_AUTO    = 0,
-    ADIOI_HINT_ENABLE  = 1,
-    ADIOI_HINT_DISABLE = 2
-};
-
-/* flattened datatypes. Each datatype is stored as a node of a
-   globally accessible linked list. Once attribute caching on a
-   datatype is available (in MPI-2), that should be used instead. */  
-
-typedef struct ADIOI_Fl_node {  
-    MPI_Datatype type;
-    MPI_Count count;                   /* no. of contiguous blocks */
-    ADIO_Offset *blocklens;      /* array of contiguous block lengths (bytes)*/
-    ADIO_Offset *indices;        /* array of byte offsets of each block */
-    /* the type processing code in ROMIO loops through the flattened
-     * representation to tile file views.  so, we cannot simply indicate a
-     * lower bound and upper bound with entries here -- those are instead
-     * indicated by offset-length pairs with zero length.  In order to deal
-     * with repeatedly resetting the LB and UB though (as in resized of
-     * resized, or struct of resized, perhaps?), indicate where in the
-     * tuple-stream the lb and ub parts are kept
-     * (-1 indicates "not explicitly set") */
-    ADIO_Offset lb_idx;
-    ADIO_Offset ub_idx;
-    struct ADIOI_Fl_node *next;  /* pointer to next node */
-} ADIOI_Flatlist_node;
-
-#ifdef ROMIO_PVFS2
-#include <pvfs2.h>
-#endif
-typedef struct ADIOI_AIO_req_str {
-	/* very wierd: if this MPI_Request is a pointer, some C++ compilers
-	 * will clobber it when the MPICH C++ bindings are used */
-	MPI_Request req;
-	MPI_Offset nbytes;
-	/* should probably make this a union */
-#ifdef ROMIO_HAVE_WORKING_AIO
-	struct aiocb *aiocbp;
-#endif
-#ifdef ROMIO_PVFS2
-	PVFS_sys_op_id op_id;
-	PVFS_sysresp_io resp_io;
-	PVFS_Request file_req;
-	PVFS_Request mem_req;
-#endif
-#ifdef ROMIO_NTFS
-    /* Ptr to Overlapped struct */
-    LPOVERLAPPED    lpOvl;
-    /* Ptr to file handle */
-	HANDLE fd;
-#endif
-} ADIOI_AIO_Request;
-
-struct ADIOI_Fns_struct {
-    void (*ADIOI_xxx_Open) (ADIO_File fd, int *error_code);
-    void (*ADIOI_xxx_OpenColl) (ADIO_File fd, int rank, 
-		    int access_mode, int *error_code);
-    void (*ADIOI_xxx_ReadContig) (ADIO_File fd, void *buf, int count,
-                   MPI_Datatype datatype, int file_ptr_type,
-                   ADIO_Offset offset, ADIO_Status *status, int *error_code);
-    void (*ADIOI_xxx_WriteContig) (ADIO_File fd, const void *buf, int count,
-                   MPI_Datatype datatype, int file_ptr_type,
-                   ADIO_Offset offset, ADIO_Status *status, int *error_code);  
-    void (*ADIOI_xxx_ReadStridedColl) (ADIO_File fd, void *buf, int count,
-	       MPI_Datatype datatype, int file_ptr_type,
-	       ADIO_Offset offset, ADIO_Status *status, int *error_code);
-    void (*ADIOI_xxx_WriteStridedColl) (ADIO_File fd, const void *buf, int count,
-	       MPI_Datatype datatype, int file_ptr_type,
-	       ADIO_Offset offset, ADIO_Status *status, int *error_code);
-    ADIO_Offset (*ADIOI_xxx_SeekIndividual) (ADIO_File fd, ADIO_Offset offset, 
-                       int whence, int *error_code);
-    void (*ADIOI_xxx_Fcntl) (ADIO_File fd, int flag, 
-                             ADIO_Fcntl_t *fcntl_struct, int *error_code); 
-    void (*ADIOI_xxx_SetInfo) (ADIO_File fd, MPI_Info users_info, 
-                               int *error_code);
-    void (*ADIOI_xxx_ReadStrided) (ADIO_File fd, void *buf, int count,
-	       MPI_Datatype datatype, int file_ptr_type,
-	       ADIO_Offset offset, ADIO_Status *status, int *error_code);
-    void (*ADIOI_xxx_WriteStrided) (ADIO_File fd, const void *buf, int count,
-	       MPI_Datatype datatype, int file_ptr_type,
-	       ADIO_Offset offset, ADIO_Status *status, int *error_code);
-    void (*ADIOI_xxx_Close) (ADIO_File fd, int *error_code);
-    void (*ADIOI_xxx_IreadContig) (ADIO_File fd, void *buf, int count, 
-                   MPI_Datatype datatype, int file_ptr_type, 
-                   ADIO_Offset offset, ADIO_Request *request, int *error_code);
-    void (*ADIOI_xxx_IwriteContig) (ADIO_File fd, const void *buf, int count,
-                   MPI_Datatype datatype, int file_ptr_type,
-	           ADIO_Offset offset, ADIO_Request *request, int *error_code);
-    int (*ADIOI_xxx_ReadDone) (ADIO_Request *request, ADIO_Status *status, 
-               int *error_code); 
-    int (*ADIOI_xxx_WriteDone) (ADIO_Request *request, ADIO_Status *status, 
-               int *error_code);
-    void (*ADIOI_xxx_ReadComplete) (ADIO_Request *request, ADIO_Status *status, 
-               int *error_code); 
-    void (*ADIOI_xxx_WriteComplete) (ADIO_Request *request, ADIO_Status *status,
-	       int *error_code); 
-    void (*ADIOI_xxx_IreadStrided) (ADIO_File fd, void *buf, int count,
-	       MPI_Datatype datatype, int file_ptr_type,
-	       ADIO_Offset offset, ADIO_Request *request, int *error_code);
-    void (*ADIOI_xxx_IwriteStrided) (ADIO_File fd, const void *buf, int count,
-	       MPI_Datatype datatype, int file_ptr_type,
-	       ADIO_Offset offset, ADIO_Request *request, int *error_code);
-    void (*ADIOI_xxx_Flush) (ADIO_File fd, int *error_code); 
-    void (*ADIOI_xxx_Resize) (ADIO_File fd, ADIO_Offset size, int *error_code);
-    void (*ADIOI_xxx_Delete) (const char *filename, int *error_code);
-    int  (*ADIOI_xxx_Feature) (ADIO_File fd, int flag);
-    const char *fsname;
-    void (*ADIOI_xxx_IreadStridedColl) (ADIO_File fd, void *buf, int count,
-           MPI_Datatype datatype, int file_ptr_type,
-           ADIO_Offset offset, ADIO_Request *request, int *error_code);
-    void (*ADIOI_xxx_IwriteStridedColl) (ADIO_File fd, const void *buf,
-           int count, MPI_Datatype datatype, int file_ptr_type,
-           ADIO_Offset offset, ADIO_Request *request, int *error_code);
-};
-
-/* optypes for ADIO_RequestD */
-#define ADIOI_READ                26
-#define ADIOI_WRITE               27
-
-#define ADIOI_MIN(a, b) ((a) < (b) ? (a) : (b))
-#define ADIOI_MAX(a, b) ((a) > (b) ? (a) : (b))
-/* thanks stackoverflow:
- * http://stackoverflow.com/questions/3982348/implement-generic-swap-macro-in-c */
-#define ADIOI_SWAP(x, y, T) do { T temp##x##y = x; x = y; y = temp##x##y; } while (0);
-
-#define ADIOI_PREALLOC_BUFSZ      16777216    /* buffer size used to 
-                                                preallocate disk space */
-
-
-/* default values for some hints */
-    /* buffer size for collective I/O = 16 MB */
-#define ADIOI_CB_BUFFER_SIZE_DFLT         "16777216"
-    /* buffer size for data sieving in independent reads = 4MB */
-#define ADIOI_IND_RD_BUFFER_SIZE_DFLT     "4194304"
-    /* buffer size for data sieving in independent writes = 512KB. default is
-       smaller than for reads, because write requires read-modify-write
-       with file locking. If buffer size is large there is more contention 
-       for locks. */
-#define ADIOI_IND_WR_BUFFER_SIZE_DFLT     "524288"
-    /* use one process per processor name by default */
-#define ADIOI_CB_CONFIG_LIST_DFLT "*:1"
-
-
-/* some of the ADIO functions are macro-replaced */
-
-#define ADIOI_OpenColl(fd, rank, access_mode, error_code) \
-	(*(fd->fns->ADIOI_xxx_OpenColl))(fd, rank, access_mode, error_code)
-
-#define ADIO_ReadContig(fd,buf,count,datatype,file_ptr_type,offset,status,error_code) \
-        (*(fd->fns->ADIOI_xxx_ReadContig))(fd,buf,count,datatype,file_ptr_type,offset,status,error_code)
-
-#define ADIO_WriteContig(fd,buf,count,datatype,file_ptr_type,offset,status,error_code) \
-        (*(fd->fns->ADIOI_xxx_WriteContig))(fd,buf,count,datatype,file_ptr_type,offset,status,error_code)
-
-#define ADIO_SeekIndividual(fd,offset,whence,error_code) \
-        (*(fd->fns->ADIOI_xxx_SeekIndividual))(fd,offset,whence,error_code)
-
-#define ADIO_Fcntl(fd,flag,fcntl_struct,error_code) \
-        (*(fd->fns->ADIOI_xxx_Fcntl))(fd,flag,fcntl_struct,error_code)
-
-#define ADIO_IreadContig(fd,buf,count,datatype,file_ptr_type,offset,request,error_code) \
-        (*(fd->fns->ADIOI_xxx_IreadContig))(fd,buf,count,datatype,file_ptr_type,offset,request,error_code)
-
-#define ADIO_IwriteContig(fd,buf,count,datatype,file_ptr_type,offset,request,error_code) \
-        (*(fd->fns->ADIOI_xxx_IwriteContig))(fd,buf,count,datatype,file_ptr_type,offset,request,error_code)
-
-/* in these routines a pointer to request is passed */
-#define ADIO_ReadDone(request,status,error_code) \
-        (*((*(request))->fd->fns->ADIOI_xxx_ReadDone))(request,status,error_code)
-
-#define ADIO_WriteDone(request,status,error_code) \
-        (*((*(request))->fd->fns->ADIOI_xxx_WriteDone))(request,status,error_code)
-
-#define ADIO_ReadIcomplete(request,status,error_code) \
-        (*((*(request))->fd->fns->ADIOI_xxx_ReadIcomplete))(request,status,error_code)
-
-#define ADIO_WriteIcomplete(request,status,error_code) \
-        (*((*(request))->fd->fns->ADIOI_xxx_WriteIcomplete))(request,status,error_code)
-
-#define ADIO_ReadComplete(request,status,error_code) \
-        (*((*(request))->fd->fns->ADIOI_xxx_ReadComplete))(request,status,error_code)
-
-#define ADIO_WriteComplete(request,status,error_code) \
-        (*((*(request))->fd->fns->ADIOI_xxx_WriteComplete))(request,status,error_code)
-
-#define ADIO_ReadStrided(fd,buf,count,datatype,file_ptr_type,offset,status,error_code) \
-        (*(fd->fns->ADIOI_xxx_ReadStrided))(fd,buf,count,datatype,file_ptr_type,offset,status,error_code)
-
-#define ADIO_WriteStrided(fd,buf,count,datatype,file_ptr_type,offset,status,error_code) \
-        (*(fd->fns->ADIOI_xxx_WriteStrided))(fd,buf,count,datatype,file_ptr_type,offset,status,error_code)
-
-#define ADIO_ReadStridedColl(fd,buf,count,datatype,file_ptr_type,offset,status,error_code) \
-        (*(fd->fns->ADIOI_xxx_ReadStridedColl))(fd,buf,count,datatype,file_ptr_type,offset,status,error_code)
-
-#define ADIO_WriteStridedColl(fd,buf,count,datatype,file_ptr_type,offset,status,error_code) \
-        (*(fd->fns->ADIOI_xxx_WriteStridedColl))(fd,buf,count,datatype,file_ptr_type,offset,status,error_code)
-
-#define ADIO_IreadStrided(fd,buf,count,datatype,file_ptr_type,offset,request,error_code) \
-        (*(fd->fns->ADIOI_xxx_IreadStrided))(fd,buf,count,datatype,file_ptr_type,offset,request,error_code)
-
-#define ADIO_IwriteStrided(fd,buf,count,datatype,file_ptr_type,offset,request,error_code) \
-        (*(fd->fns->ADIOI_xxx_IwriteStrided))(fd,buf,count,datatype,file_ptr_type,offset,request,error_code)
-
-#define ADIO_IreadStridedColl(fd,buf,count,datatype,file_ptr_type,offset,request,error_code) \
-        (*(fd->fns->ADIOI_xxx_IreadStridedColl))(fd,buf,count,datatype,file_ptr_type,offset,request,error_code)
-
-#define ADIO_IwriteStridedColl(fd,buf,count,datatype,file_ptr_type,offset,request,error_code) \
-        (*(fd->fns->ADIOI_xxx_IwriteStridedColl))(fd,buf,count,datatype,file_ptr_type,offset,request,error_code)
-
-#define ADIO_Flush(fd,error_code) (*(fd->fns->ADIOI_xxx_Flush))(fd,error_code)
-
-#define ADIO_Resize(fd,size,error_code) \
-        (*(fd->fns->ADIOI_xxx_Resize))(fd,size,error_code)
-
-#define ADIO_Delete(filename,error_code) \
-        (*(fd->fns->ADIOI_xxx_Delete))(filename,error_code)
-
-#define ADIO_SetInfo(fd, users_info, error_code) \
-        (*(fd->fns->ADIOI_xxx_SetInfo))(fd, users_info, error_code)
-
-#define ADIO_Feature(fd, flag) \
-	(*(fd->fns->ADIOI_xxx_Feature))(fd, flag)
-
-
-/* structure for storing access info of this process's request 
-   from the file domain of other processes, and vice-versa. used 
-   as array of structures indexed by process number. */
-typedef struct {
-    ADIO_Offset *offsets;   /* array of offsets */
-    ADIO_Offset *lens;      /* array of lengths */
-    MPI_Aint *mem_ptrs;     /* array of pointers. used in the read/write
-			       phase to indicate where the data
-			       is stored in memory */
-    int count;             /* size of above arrays */
-} ADIOI_Access;
-
-/* structure for storing generic offset/length pairs.  used to describe
-   file realms among other things */
-typedef struct {
-    ADIO_Offset *offsets; /* array of offsets */
-    ADIO_Offset *lens;    /* array of lengths */
-    int count;            /* size of above arrays */
-} ADIOI_Offlen;
-
-/* prototypes for ADIO internal functions */
-
-
-int ADIOI_Type_get_envelope (MPI_Datatype datatype, int *num_integers,
-                             int *num_addresses, int *num_datatypes, int *combiner);
-int ADIOI_Type_get_contents (MPI_Datatype datatype, int max_integers,
-                             int max_addresses, int max_datatypes, int array_of_integers[],
-                             MPI_Aint array_of_addresses[], MPI_Datatype array_of_datatypes[]);
-void ADIOI_SetFunctions(ADIO_File fd);
-void ADIOI_Flatten_datatype(MPI_Datatype type);
-void ADIOI_Flatten(MPI_Datatype type, ADIOI_Flatlist_node *flat,
-		  ADIO_Offset st_offset, MPI_Count *curr_index);
-void ADIOI_Delete_flattened(MPI_Datatype datatype);
-ADIOI_Flatlist_node * ADIOI_Flatten_and_find(MPI_Datatype);
-MPI_Count ADIOI_Count_contiguous_blocks(MPI_Datatype type, MPI_Count *curr_index);
-void ADIOI_Complete_async(int *error_code);
-void *ADIOI_Malloc_fn(size_t size, int lineno, const char *fname);
-void *ADIOI_Calloc_fn(size_t nelem, size_t elsize, int lineno, const char *fname);
-void *ADIOI_Realloc_fn(void *ptr, size_t size, int lineno, const char *fname);
-void ADIOI_Free_fn(void *ptr, int lineno, const char *fname);
-void ADIOI_Datatype_iscontig(MPI_Datatype datatype, int *flag);
-void ADIOI_Get_position(ADIO_File fd, ADIO_Offset *offset);
-void ADIOI_Get_eof_offset(ADIO_File fd, ADIO_Offset *eof_offset);
-void ADIOI_Get_byte_offset(ADIO_File fd, ADIO_Offset offset,
-			   ADIO_Offset *disp);
-void ADIOI_process_system_hints(ADIO_File fd, MPI_Info info);
-void ADIOI_incorporate_system_hints(MPI_Info info, MPI_Info sysinfo, 
-		MPI_Info *new_info);
-void ADIOI_Info_print_keyvals(MPI_Info info);
-
-
-void ADIOI_GEN_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct,
-		     int *error_code);
-void ADIOI_GEN_Flush(ADIO_File fd, int *error_code);
-void ADIOI_GEN_OpenColl(ADIO_File fd, int rank, 
-		int access_mode, int *error_code);
-void ADIOI_SCALEABLE_OpenColl(ADIO_File fd, int rank, 
-		int access_mode, int *error_code);
-void ADIOI_FAILSAFE_OpenColl(ADIO_File fd, int rank, 
-		int access_mode, int *error_code);
-void ADIOI_GEN_Delete(const char *filename, int *error_code);
-void ADIOI_GEN_ReadContig(ADIO_File fd, void *buf, int count, 
-			  MPI_Datatype datatype, int file_ptr_type,
-			  ADIO_Offset offset, ADIO_Status *status,
-			  int *error_code);
-int ADIOI_GEN_aio(ADIO_File fd, void *buf, int len, ADIO_Offset offset,
-		  int wr, MPI_Request *request);
-void ADIOI_GEN_IreadContig(ADIO_File fd, void *buf, int count, 
-			   MPI_Datatype datatype, int file_ptr_type,
-			   ADIO_Offset offset, ADIO_Request *request,
-			   int *error_code);
-void ADIOI_GEN_WriteContig(ADIO_File fd, const void *buf, int count,
-			   MPI_Datatype datatype, int file_ptr_type,
-			   ADIO_Offset offset, ADIO_Status *status,
-			   int *error_code);
-void ADIOI_GEN_IwriteContig(ADIO_File fd, const void *buf, int count,
-			    MPI_Datatype datatype, int file_ptr_type,
-			    ADIO_Offset offset, ADIO_Request *request,
-			    int *error_code);
-void ADIOI_GEN_ReadStrided(ADIO_File fd, void *buf, int count,
-			   MPI_Datatype datatype, int file_ptr_type,
-			   ADIO_Offset offset, ADIO_Status *status, int
-			   *error_code);
-void ADIOI_GEN_IreadStrided(ADIO_File fd, void *buf, int count,
-			    MPI_Datatype datatype, int file_ptr_type,
-			    ADIO_Offset offset, ADIO_Request *request, int
-			    *error_code);
-void ADIOI_GEN_IwriteStrided(ADIO_File fd, const void *buf, int count,
-			     MPI_Datatype datatype, int file_ptr_type,
-			     ADIO_Offset offset, ADIO_Request *request, int
-			     *error_code);
-int ADIOI_GEN_IODone(ADIO_Request *request, ADIO_Status *status,
-		     int *error_code);
-void ADIOI_GEN_IOComplete(ADIO_Request *request, ADIO_Status *status,
-			  int *error_code);
-int ADIOI_GEN_aio_poll_fn(void *extra_state, ADIO_Status *status);
-int ADIOI_GEN_aio_wait_fn(int count, void **array_of_states, double timeout, 
-		ADIO_Status *status);
-int ADIOI_GEN_aio_query_fn(void *extra_state, ADIO_Status *status);
-int ADIOI_GEN_aio_free_fn(void *extra_state);
-int ADIOI_GEN_Feature(ADIO_File fd, int feature);
-
-void ADIOI_GEN_ReadStrided_naive(ADIO_File fd, void *buf, int count,
-                       MPI_Datatype buftype, int file_ptr_type,
-                       ADIO_Offset offset, ADIO_Status *status, int
-                       *error_code);
-void ADIOI_GEN_WriteStrided(ADIO_File fd, const void *buf, int count,
-                       MPI_Datatype datatype, int file_ptr_type,
-                       ADIO_Offset offset, ADIO_Status *status, int
-                       *error_code);
-void ADIOI_GEN_WriteStrided_naive(ADIO_File fd, const void *buf, int count,
-                       MPI_Datatype datatype, int file_ptr_type,
-                       ADIO_Offset offset, ADIO_Status *status, int
-                       *error_code);
-void ADIOI_NOLOCK_WriteStrided(ADIO_File fd, const void *buf, int count,
-                       MPI_Datatype datatype, int file_ptr_type,
-                       ADIO_Offset offset, ADIO_Status *status, int
-                       *error_code);
-void ADIOI_GEN_ReadStridedColl(ADIO_File fd, void *buf, int count,
-                       MPI_Datatype datatype, int file_ptr_type,
-                       ADIO_Offset offset, ADIO_Status *status, int
-                       *error_code);
-#ifdef HAVE_MPI_GREQUEST_EXTENSIONS
-void ADIOI_GEN_IreadStridedColl(ADIO_File fd, void *buf, int count,
-                       MPI_Datatype datatype, int file_ptr_type,
-                       ADIO_Offset offset, MPI_Request *request,
-                       int *error_code);
-#else
-#define ADIOI_GEN_IreadStridedColl NULL
-#endif
-void ADIOI_GEN_WriteStridedColl(ADIO_File fd, const void *buf, int count,
-                       MPI_Datatype datatype, int file_ptr_type,
-                       ADIO_Offset offset, ADIO_Status *status, int
-                       *error_code);
-#ifdef HAVE_MPI_GREQUEST_EXTENSIONS
-void ADIOI_GEN_IwriteStridedColl(ADIO_File fd, const void *buf, int count,
-                       MPI_Datatype datatype, int file_ptr_type,
-                       ADIO_Offset offset, MPI_Request *request,
-                       int *error_code);
-#else
-#define ADIOI_GEN_IwriteStridedColl NULL
-#endif
-void ADIOI_Calc_my_off_len(ADIO_File fd, int bufcount, MPI_Datatype
-			    datatype, int file_ptr_type, ADIO_Offset 
-			    offset, ADIO_Offset **offset_list_ptr, ADIO_Offset
-			    **len_list_ptr, ADIO_Offset *start_offset_ptr,
-			    ADIO_Offset *end_offset_ptr, int
-			   *contig_access_count_ptr);
-void ADIOI_Calc_file_domains(ADIO_Offset *st_offsets, ADIO_Offset
-			     *end_offsets, int nprocs, int nprocs_for_coll,
-			     ADIO_Offset *min_st_offset_ptr,
-			     ADIO_Offset **fd_start_ptr, ADIO_Offset 
-			     **fd_end_ptr, int min_fd_size, 
-			     ADIO_Offset *fd_size_ptr,
-			     int striping_unit);
-int ADIOI_Calc_aggregator(ADIO_File fd,
-                                 ADIO_Offset off,
-                                 ADIO_Offset min_off,
-                                 ADIO_Offset *len,
-                                 ADIO_Offset fd_size,
-                                 ADIO_Offset *fd_start,
-                                 ADIO_Offset *fd_end);
-void ADIOI_Calc_my_req(ADIO_File fd, ADIO_Offset *offset_list, 
-			    ADIO_Offset *len_list, int
-			    contig_access_count, ADIO_Offset 
-			    min_st_offset, ADIO_Offset *fd_start,
-			    ADIO_Offset *fd_end, ADIO_Offset fd_size,
-                            int nprocs,
-                            int *count_my_req_procs_ptr,
-			    int **count_my_req_per_proc_ptr,
-			    ADIOI_Access **my_req_ptr,
-			    int **buf_idx_ptr);
-void ADIOI_Calc_others_req(ADIO_File fd, int count_my_req_procs, 
-				int *count_my_req_per_proc,
-				ADIOI_Access *my_req, 
-				int nprocs, int myrank,
-				int *count_others_req_procs_ptr,
-				ADIOI_Access **others_req_ptr);  
-
-
-/* Nonblocking Collective I/O internals */
-typedef enum {
-    ADIOI_IRC_STATE_GEN_IREADSTRIDEDCOLL,
-    ADIOI_IRC_STATE_GEN_IREADSTRIDEDCOLL_INDIO,
-    ADIOI_IRC_STATE_ICALC_OTHERS_REQ,
-    ADIOI_IRC_STATE_ICALC_OTHERS_REQ_MAIN,
-    ADIOI_IRC_STATE_IREAD_AND_EXCH,
-    ADIOI_IRC_STATE_IREAD_AND_EXCH_L1_BEGIN,
-    ADIOI_IRC_STATE_R_IEXCHANGE_DATA,
-    ADIOI_IRC_STATE_R_IEXCHANGE_DATA_RECV,
-    ADIOI_IRC_STATE_R_IEXCHANGE_DATA_FILL,
-    ADIOI_IRC_STATE_COMPLETE
-} ADIOI_IRC_State;
-
-typedef enum {
-    ADIOI_IWC_STATE_GEN_IWRITESTRIDEDCOLL,
-    ADIOI_IWC_STATE_GEN_IWRITESTRIDEDCOLL_INDIO,
-    ADIOI_IWC_STATE_GEN_IWRITESTRIDEDCOLL_BCAST,
-    ADIOI_IWC_STATE_ICALC_OTHERS_REQ,
-    ADIOI_IWC_STATE_ICALC_OTHERS_REQ_MAIN,
-    ADIOI_IWC_STATE_IEXCH_AND_WRITE,
-    ADIOI_IWC_STATE_IEXCH_AND_WRITE_L1_BODY,
-    ADIOI_IWC_STATE_W_IEXCHANGE_DATA,
-    ADIOI_IWC_STATE_W_IEXCHANGE_DATA_HOLE,
-    ADIOI_IWC_STATE_W_IEXCHANGE_DATA_SEND,
-    ADIOI_IWC_STATE_W_IEXCHANGE_DATA_WAIT,
-    ADIOI_IWC_STATE_COMPLETE
-} ADIOI_IWC_State;
-
-typedef struct ADIOI_NBC_Request                ADIOI_NBC_Request;
-
-typedef struct ADIOI_GEN_IreadStridedColl_vars  ADIOI_GEN_IreadStridedColl_vars;
-typedef struct ADIOI_Iread_and_exch_vars        ADIOI_Iread_and_exch_vars;
-typedef struct ADIOI_R_Iexchange_data_vars      ADIOI_R_Iexchange_data_vars;
-
-typedef struct ADIOI_GEN_IwriteStridedColl_vars ADIOI_GEN_IwriteStridedColl_vars;
-typedef struct ADIOI_Iexch_and_write_vars       ADIOI_Iexch_and_write_vars;
-typedef struct ADIOI_W_Iexchange_data_vars      ADIOI_W_Iexchange_data_vars;
-
-typedef struct ADIOI_Icalc_others_req_vars {
-    /* requests */
-    MPI_Request req1;
-    MPI_Request *req2;
-    int num_req2;
-
-    /* parameters */
-    ADIO_File fd;
-    int count_my_req_procs;
-    int *count_my_req_per_proc;
-    ADIOI_Access *my_req;
-    int nprocs;
-    int myrank;
-    int *count_others_req_procs_ptr;
-    ADIOI_Access **others_req_ptr;
-
-    /* stack variables */
-    int *count_others_req_per_proc;
-    int count_others_req_procs;
-    ADIOI_Access *others_req;
-
-    /* next function to be called */
-    void (*next_fn)(ADIOI_NBC_Request *, int *);
-} ADIOI_Icalc_others_req_vars;
-
-struct ADIOI_NBC_Request {
-    int rdwr;           /* ADIOI_READ or ADIOI_WRITE */
-    MPI_Request req;    /* MPIX_Grequest */
-    MPI_Count nbytes;   /* data read or written */
-
-    union {
-        struct {
-            ADIOI_IRC_State state;      /* progress state */
-            ADIOI_GEN_IreadStridedColl_vars *rsc_vars;
-            ADIOI_Iread_and_exch_vars       *rae_vars;
-            ADIOI_R_Iexchange_data_vars     *red_vars;
-        } rd;
-        struct {
-            ADIOI_IWC_State state;      /* progress state */
-            ADIOI_GEN_IwriteStridedColl_vars *wsc_vars;
-            ADIOI_Iexch_and_write_vars       *eaw_vars;
-            ADIOI_W_Iexchange_data_vars      *wed_vars;
-        } wr;
-    } data;
-    ADIOI_Icalc_others_req_vars *cor_vars;
-};
-
-void ADIOI_Icalc_others_req(ADIOI_NBC_Request *nbc_req, int *error_code);
-void ADIOI_Icalc_others_req_main(ADIOI_NBC_Request *nbc_req, int *error_code);
-void ADIOI_Icalc_others_req_fini(ADIOI_NBC_Request *nbc_req, int *error_code);
-
-
-/* KC && AC - New Collective I/O internals*/
-
-#define TEMP_OFF 0
-#define REAL_OFF 1
-#define MAX_OFF_TYPE 2
-
-/* Communication Tags */
-#define DATA_TAG 30
-#define AMT_TAG 31
-
-/* cb_fr_type user size is non-zero */
-#define ADIOI_FR_AAR 0
-#define ADIOI_FR_FSZ -1
-#define ADIOI_FR_USR_REALMS -2
-
-typedef struct flatten_state
-{
-    ADIO_Offset abs_off;
-    ADIO_Offset cur_sz;
-    ADIO_Offset idx;
-    ADIO_Offset cur_reg_off;
-} flatten_state;
-
-typedef struct view_state
-{
-    ADIO_Offset fp_ind;    /* file view params*/
-    ADIO_Offset disp;      /* file view params*/
-    ADIO_Offset byte_off;
-    ADIO_Offset sz;
-    ADIO_Offset ext;       /* preserved extent from MPI_Type_extent */
-    ADIO_Offset type_sz;
-
-    /* Current state */
-    flatten_state cur_state;
-    /* Scratch state for counting up ol pairs */
-    flatten_state tmp_state;
-
-    /* Preprocessed data amount and ol pairs */
-    ADIO_Offset pre_sz;
-    int pre_ol_ct;
-    MPI_Aint *pre_disp_arr;
-    int *pre_blk_arr;
-    
-    ADIOI_Flatlist_node *flat_type_p;
-} view_state;
-
-void ADIOI_Calc_bounds (ADIO_File fd, int count, MPI_Datatype buftype,
-			int file_ptr_type, ADIO_Offset offset,
-			ADIO_Offset *st_offset, ADIO_Offset *end_offset);
-int ADIOI_Agg_idx (int rank, ADIO_File fd);
-void ADIOI_Calc_file_realms (ADIO_File fd, ADIO_Offset min_st_offset,
-			     ADIO_Offset max_end_offset);
-void ADIOI_IOFiletype(ADIO_File fd, void *buf, int count,
-		      MPI_Datatype datatype, int file_ptr_type,
-		      ADIO_Offset offset, MPI_Datatype custom_ftype,
-		      int rdwr, ADIO_Status *status, int
-		      *error_code);
-void ADIOI_IOStridedColl(ADIO_File fd, void *buf, int count, int rdwr,
-                       MPI_Datatype datatype, int file_ptr_type,
-                       ADIO_Offset offset, ADIO_Status *status, int
-                       *error_code);
-void ADIOI_Print_flatlist_node(ADIOI_Flatlist_node *flatlist_node_p);
-ADIOI_Flatlist_node * ADIOI_Add_contig_flattened(MPI_Datatype contig_type);
-void ADIOI_Exch_file_views(int myrank, int nprocs, int file_ptr_type,
-			   ADIO_File fd, int count,
-			   MPI_Datatype datatype, ADIO_Offset off,
-			   view_state *my_mem_view_state_arr,
-			   view_state *agg_file_view_state_arr,
-			   view_state *client_file_view_state_arr);
-int ADIOI_init_view_state(int file_ptr_type,
-		    int nprocs, 
-		    view_state *view_state_arr,
-		    int op_type);
-int ADIOI_Build_agg_reqs(ADIO_File fd, int rw_type, int nprocs,
-			 view_state *client_file_view_state_arr,
-			 MPI_Datatype *client_comm_dtype_arr,
-			 ADIO_Offset *client_comm_sz_arr,
-			 ADIO_Offset *agg_dtype_offset_p,
-			 MPI_Datatype *agg_dtype_p);
-int ADIOI_Build_client_reqs(ADIO_File fd, 
-			    int nprocs,
-			    view_state *my_mem_view_state_arr,
-			    view_state *agg_file_view_state_arr,
-			    ADIO_Offset *agg_comm_sz_arr,
-			    MPI_Datatype *agg_comm_dtype_arr);
-int ADIOI_Build_client_pre_req(ADIO_File fd,
-                               int agg_rank,
-			       int agg_idx,
-                               view_state *my_mem_view_state_p,
-                               view_state *agg_file_view_state_p,
-                               ADIO_Offset max_pre_req_sz,
-                               int max_ol_ct);
-int ADIOI_Build_client_req(ADIO_File fd,
-			   int agg_rank,
-			   int agg_idx,
-			   view_state *my_mem_view_state_p,
-			   view_state *agg_file_view_state_p,
-			   ADIO_Offset agg_comm_sz,
-			   MPI_Datatype *agg_comm_dtype_p);
-
-void ADIOI_P2PContigWriteAggregation(ADIO_File fd,
-	                             const void *buf,
-				     int *error_code,
-				     ADIO_Offset *st_offsets,
-				     ADIO_Offset *end_offset,
-				     ADIO_Offset *fd_start,
-				     ADIO_Offset *fd_end);
-
-void ADIOI_P2PContigReadAggregation(ADIO_File fd,
-	                             const void *buf,
-				     int *error_code,
-				     ADIO_Offset *st_offsets,
-				     ADIO_Offset *end_offset,
-				     ADIO_Offset *fd_start,
-				     ADIO_Offset *fd_end);
-
-int ADIOI_OneSidedCleanup(ADIO_File fd);
-void ADIOI_OneSidedWriteAggregation(ADIO_File fd,
-        ADIO_Offset *offset_list,
-        ADIO_Offset *len_list,
-        int contig_access_count,
-        const void *buf,
-        MPI_Datatype datatype,
-        int *error_code,
-        ADIO_Offset *st_offsets,
-        ADIO_Offset *end_offsets,
-        int numNonZeroDataOffsets,
-        ADIO_Offset *fd_start,
-        ADIO_Offset* fd_end,
-        int *hole_found);
-void ADIOI_OneSidedReadAggregation(ADIO_File fd,
-        ADIO_Offset *offset_list,
-        ADIO_Offset *len_list,
-        int contig_access_count,
-        const void *buf,
-        MPI_Datatype datatype,
-        int *error_code,
-        ADIO_Offset *st_offsets,
-        ADIO_Offset *end_offsets,
-        int numNonZeroDataOffsets,
-        ADIO_Offset *fd_start,
-        ADIO_Offset* fd_end);
-ADIO_Offset ADIOI_GEN_SeekIndividual(ADIO_File fd, ADIO_Offset offset, 
-				     int whence, int *error_code);
-void ADIOI_GEN_Resize(ADIO_File fd, ADIO_Offset size, int *error_code);
-void ADIOI_GEN_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code);
-void ADIOI_GEN_Close(ADIO_File fd, int *error_code);
-void ADIOI_Shfp_fname(ADIO_File fd, int rank, int *error_code);
-void ADIOI_GEN_Prealloc(ADIO_File fd, ADIO_Offset size, int *error_code);
-int ADIOI_Error(ADIO_File fd, int error_code, char *string);
-int MPIR_Err_setmsg( int, int, const char *, const char *, const char *, ... );
-int ADIOI_End_call(MPI_Comm comm, int keyval, void *attribute_val, void *extra_state);
-int MPIR_Status_set_bytes(MPI_Status *status, MPI_Datatype datatype, MPI_Count nbytes);
-int ADIOI_Uses_generic_read(ADIO_File fd);
-int ADIOI_Uses_generic_write(ADIO_File fd);
-int ADIOI_Err_create_code(const char *myname, const char *filename, int my_errno);
-int ADIOI_Type_create_hindexed_x(int count,
-		const MPI_Count array_of_blocklengths[],
-		const MPI_Aint array_of_displacements[],
-		MPI_Datatype oldtype,
-		MPI_Datatype *newtype);
-
-
-int ADIOI_FAKE_IODone(ADIO_Request *request, ADIO_Status *status,
-                      int *error_code);
-void ADIOI_FAKE_IreadContig(ADIO_File fd, void *buf, int count, 
-			   MPI_Datatype datatype, int file_ptr_type,
-			   ADIO_Offset offset, ADIO_Request *request,
-                            int *error_code);
-void ADIOI_FAKE_IreadStrided(ADIO_File fd, void *buf, int count, 
-			    MPI_Datatype datatype, int file_ptr_type,
-			    ADIO_Offset offset, ADIO_Request *request,
-                             int *error_code);
-void ADIOI_FAKE_IwriteContig(ADIO_File fd, const void *buf, int count,
-			    MPI_Datatype datatype, int file_ptr_type,
-			    ADIO_Offset offset, ADIO_Request *request,
-                             int *error_code);
-void ADIOI_FAKE_IwriteStrided(ADIO_File fd, const void *buf, int count,
-			     MPI_Datatype datatype, int file_ptr_type,
-			     ADIO_Offset offset, ADIO_Request *request,
-                              int *error_code);
-void ADIOI_FAKE_IOComplete(ADIO_Request *request, ADIO_Status *status,
-                           int *error_code);
-
-
-/* File I/O common functionality */
-int MPIOI_File_read(MPI_File fh,
-		    MPI_Offset offset,
-		    int file_ptr_type,
-		    void *buf,
-		    int count,
-		    MPI_Datatype datatype,
-		    char *myname,
-		    MPI_Status *status);
-int MPIOI_File_write(MPI_File fh,
-		     MPI_Offset offset,
-		     int file_ptr_type,
-		     const void *buf,
-		     int count,
-		     MPI_Datatype datatype,
-		     char *myname,
-		     MPI_Status *status);
-int MPIOI_File_read_all(MPI_File fh,
-			MPI_Offset offset,
-			int file_ptr_type,
-			void *buf,
-			int count,
-			MPI_Datatype datatype,
-			char *myname,
-			MPI_Status *status);
-int MPIOI_File_write_all(MPI_File fh,
-			 MPI_Offset offset,
-			 int file_ptr_type,
-			 const void *buf,
-			 int count,
-			 MPI_Datatype datatype,
-			 char *myname,
-			 MPI_Status *status);
-int MPIOI_File_read_all_begin(MPI_File fh,
-			      MPI_Offset offset,
-			      int file_ptr_type,
-			      void *buf,
-			      int count,
-			      MPI_Datatype datatype,
-			      char *myname);
-int MPIOI_File_write_all_begin(MPI_File fh,
-			       MPI_Offset offset,
-			       int file_ptr_type,
-			       const void *buf,
-			       int count,
-			       MPI_Datatype datatype,
-			       char *myname);
-int MPIOI_File_read_all_end(MPI_File fh,
-			    void *buf,
-			    char *myname,
-			    MPI_Status *status);
-int MPIOI_File_write_all_end(MPI_File fh,
-			     const void *buf,
-			     char *myname,
-			     MPI_Status *status);
-int MPIOI_File_iwrite(MPI_File fh,
-		      MPI_Offset offset,
-		      int file_ptr_type,
-		      const void *buf,
-		      int count,
-		      MPI_Datatype datatype,
-		      char *myname,
-		      MPI_Request *request);
-int MPIOI_File_iread(MPI_File fh,
-		     MPI_Offset offset,
-		     int file_ptr_type,
-		     void *buf,
-		     int count,
-		     MPI_Datatype datatype,
-		     char *myname,
-		     MPI_Request *request);
-int MPIOI_File_iwrite_all(MPI_File fh,
-            MPI_Offset offset,
-            int file_ptr_type,
-            const void *buf,
-            int count,
-            MPI_Datatype datatype,
-            char *myname,
-            MPI_Request *request);
-int MPIOI_File_iread_all(MPI_File fh,
-            MPI_Offset offset,
-            int file_ptr_type,
-            void *buf,
-            int count,
-            MPI_Datatype datatype,
-            char *myname,
-            MPI_Request *request);
-
-
-
-/* Unix-style file locking */
-
-#if (defined(ROMIO_HFS) || defined(ROMIO_XFS))
-
-# define ADIOI_WRITE_LOCK(fd, offset, whence, len) \
-   do {if (((fd)->file_system == ADIO_XFS) || ((fd)->file_system == ADIO_HFS)) \
-     ADIOI_Set_lock64((fd)->fd_sys, F_SETLKW64, F_WRLCK, offset, whence, len);\
-   else ADIOI_Set_lock((fd)->fd_sys, F_SETLKW, F_WRLCK, offset, whence, len); } while (0) 
-
-# define ADIOI_READ_LOCK(fd, offset, whence, len) \
-   do {if (((fd)->file_system == ADIO_XFS) || ((fd)->file_system == ADIO_HFS)) \
-     ADIOI_Set_lock64((fd)->fd_sys, F_SETLKW64, F_RDLCK, offset, whence, len);\
-   else ADIOI_Set_lock((fd)->fd_sys, F_SETLKW, F_RDLCK, offset, whence, len); }while (0)
-
-# define ADIOI_UNLOCK(fd, offset, whence, len) \
-   do {if (((fd)->file_system == ADIO_XFS) || ((fd)->file_system == ADIO_HFS)) \
-     ADIOI_Set_lock64((fd)->fd_sys, F_SETLK64, F_UNLCK, offset, whence, len); \
-   else ADIOI_Set_lock((fd)->fd_sys, F_SETLK, F_UNLCK, offset, whence, len); }while (0)
-
-#elif (defined(ROMIO_NTFS))
-
-#define ADIOI_LOCK_CMD		0
-#define ADIOI_UNLOCK_CMD	1
-
-#   define ADIOI_WRITE_LOCK(fd, offset, whence, len) \
-          ADIOI_Set_lock((fd)->fd_sys, ADIOI_LOCK_CMD, LOCKFILE_EXCLUSIVE_LOCK, offset, whence, len)
-#   define ADIOI_READ_LOCK(fd, offset, whence, len) \
-          ADIOI_Set_lock((fd)->fd_sys, ADIOI_LOCK_CMD, 0, offset, whence, len)
-#   define ADIOI_UNLOCK(fd, offset, whence, len) \
-          ADIOI_Set_lock((fd)->fd_sys, ADIOI_UNLOCK_CMD, LOCKFILE_FAIL_IMMEDIATELY, offset, whence, len)
-
-#else
-
-#ifdef ADIOI_MPE_LOGGING
-#   define ADIOI_WRITE_LOCK(fd, offset, whence, len) do { \
-        MPE_Log_event( ADIOI_MPE_writelock_a, 0, NULL ); \
-        ADIOI_Set_lock((fd)->fd_sys, F_SETLKW, F_WRLCK, offset, whence, len); \
-        MPE_Log_event( ADIOI_MPE_writelock_b, 0, NULL ); } while( 0 )
-#   define ADIOI_READ_LOCK(fd, offset, whence, len) \
-        MPE_Log_event( ADIOI_MPE_readlock_a, 0, NULL ); do { \
-        ADIOI_Set_lock((fd)->fd_sys, F_SETLKW, F_RDLCK, offset, whence, len); \
-        MPE_Log_event( ADIOI_MPE_readlock_b, 0, NULL ); } while( 0 )
-#   define ADIOI_UNLOCK(fd, offset, whence, len) do { \
-        MPE_Log_event( ADIOI_MPE_unlock_a, 0, NULL ); \
-        ADIOI_Set_lock((fd)->fd_sys, F_SETLK, F_UNLCK, offset, whence, len); \
-        MPE_Log_event( ADIOI_MPE_unlock_b, 0, NULL ); } while( 0 )
-#else
-#   define ADIOI_WRITE_LOCK(fd, offset, whence, len) \
-          ADIOI_Set_lock((fd)->fd_sys, F_SETLKW, F_WRLCK, offset, whence, len)
-#   define ADIOI_READ_LOCK(fd, offset, whence, len) \
-          ADIOI_Set_lock((fd)->fd_sys, F_SETLKW, F_RDLCK, offset, whence, len)
-#   define ADIOI_UNLOCK(fd, offset, whence, len) \
-          ADIOI_Set_lock((fd)->fd_sys, F_SETLK, F_UNLCK, offset, whence, len)
-#endif
-
-#endif
-
-int ADIOI_Set_lock(FDTYPE fd_sys, int cmd, int type, ADIO_Offset offset, int whence, ADIO_Offset len);
-int ADIOI_Set_lock64(FDTYPE fd_sys, int cmd, int type, ADIO_Offset offset, int whence, ADIO_Offset len);
-
-#define ADIOI_Malloc(a) ADIOI_Malloc_fn(a,__LINE__,__FILE__)
-#define ADIOI_Calloc(a,b) ADIOI_Calloc_fn(a,b,__LINE__,__FILE__)
-#define ADIOI_Realloc(a,b) ADIOI_Realloc_fn(a,b,__LINE__,__FILE__)
-#define ADIOI_Free(a) ADIOI_Free_fn(a,__LINE__,__FILE__)
-
-int ADIOI_Strncpy( char *outstr, const char *instr, size_t maxlen );
-int ADIOI_Strnapp( char *, const char *, size_t );
-char *ADIOI_Strdup( const char * );
-
-/* the current MPI standard is not const-correct, and modern compilers warn
- * about the following sort of code:
- *
- *   MPI_Info_set(info, "key", "val");
- *
- * reminding us that "key" and "val" are const.  We use the following macros to
- * cast away the const and suppress the warning. */
-#define ADIOI_Info_set(info_,key_str_,val_) \
-    MPI_Info_set((info_),((char*)key_str_),(char*)(val_))
-#define ADIOI_Info_get(info_,key_str_,val_len_,val_,flag_) \
-    MPI_Info_get((info_),((char*)key_str_),(val_len_),(val_),(flag_))
-#define ADIOI_Info_get_valuelen(info_,key_str_,val_len_,flag_) \
-    MPI_Info_get_valuelen((info_),((char*)key_str_),(val_len_),(flag_))
-#define ADIOI_Info_delete(info_,key_str_) \
-    MPI_Info_delete((info_),((char*)key_str_))
-
-
-/* Provide a fallback snprintf for systems that do not have one */
-/* Define attribute as empty if it has no definition */
-#ifndef ATTRIBUTE
-#ifdef HAVE_GCC_ATTRIBUTE
-#define ATTRIBUTE(a) __attribute__(a)
-#else
-#define ATTRIBUTE(a)
-#endif
-#endif
-
-/* style: allow:snprintf:1 sig:0 */
-
-#ifdef HAVE_SNPRINTF
-#define ADIOI_Snprintf snprintf
-/* Sometimes systems don't provide prototypes for snprintf */
-#ifdef NEEDS_SNPRINTF_DECL
-extern int snprintf( char *, size_t, const char *, ... ) ATTRIBUTE((format(printf,3,4)));
-#endif
-#else
-int ADIOI_Snprintf( char *str, size_t size, const char *format, ... ) 
-     ATTRIBUTE((format(printf,3,4)));
-#endif /* HAVE_SNPRINTF */
-
-#define FPRINTF fprintf
-
-#ifndef HAVE_STRERROR
-#  ifdef HAVE_SYSERRLIST
-      extern char *sys_errlist[];
-#     define strerror(n) sys_errlist[n]
-#  else 
-#     define PRINT_ERR_MSG
-#  endif
-#endif
-
-#include "adioi_error.h"
-
-/*  MPE logging variables  */
-
-#ifdef ADIOI_MPE_LOGGING
-#include "mpe.h"
-
-int  ADIOI_MPE_open_a;
-int  ADIOI_MPE_open_b;
-int  ADIOI_MPE_read_a;
-int  ADIOI_MPE_read_b;
-int  ADIOI_MPE_write_a;
-int  ADIOI_MPE_write_b;
-int  ADIOI_MPE_lseek_a;
-int  ADIOI_MPE_lseek_b;
-int  ADIOI_MPE_close_a;
-int  ADIOI_MPE_close_b;
-int  ADIOI_MPE_writelock_a;
-int  ADIOI_MPE_writelock_b;
-int  ADIOI_MPE_readlock_a;
-int  ADIOI_MPE_readlock_b;
-int  ADIOI_MPE_unlock_a;
-int  ADIOI_MPE_unlock_b;
-int  ADIOI_MPE_postwrite_a;
-int  ADIOI_MPE_postwrite_b;
-int  ADIOI_MPE_openinternal_a;
-int  ADIOI_MPE_openinternal_b;
-int  ADIOI_MPE_stat_a;
-int  ADIOI_MPE_stat_b;
-int  ADIOI_MPE_iread_a;
-int  ADIOI_MPE_iread_b;
-int  ADIOI_MPE_iwrite_a;
-int  ADIOI_MPE_iwrite_b;
-#endif
-
-#ifdef ROMIO_INSIDE_MPICH
-/* Assert that this MPI_Aint value can be cast to a ptr value without problem.*/
-/* Basic idea is the value should be unchanged after casting 
-   (no loss of (meaningful) high order bytes in 8 byte MPI_Aint 
-      to (possible) 4 byte ptr cast)                              */
-/* Should work even on 64bit or old 32bit configs                 */
-  /* Use MPIU_Ensure_Aint_fits_in_pointer from mpiutil.h and 
-         MPIU_AINT_CAST_TO_VOID_PTR from configure (mpi.h) */
-  #include "glue_romio.h"
-
-  #define ADIOI_AINT_CAST_TO_VOID_PTR (void*)(MPIU_Pint)
-  /* The next two casts are only used when you don't want sign extension
-     when casting a (possible 4 byte) aint to a (8 byte) long long or offset */
-  #define ADIOI_AINT_CAST_TO_LONG_LONG (long long)
-  #define ADIOI_AINT_CAST_TO_OFFSET ADIOI_AINT_CAST_TO_LONG_LONG
-
-  #define ADIOI_ENSURE_AINT_FITS_IN_PTR(aint_value) MPIR_Ext_ensure_Aint_fits_in_pointer(aint_value)
-  #define ADIOI_Assert MPIR_Ext_assert
-#else
-  #include <assert.h>
-  #define ADIOI_AINT_CAST_TO_VOID_PTR (void*)
-  #define ADIOI_AINT_CAST_TO_LONG_LONG (long long)
-  #define ADIOI_AINT_CAST_TO_OFFSET ADIOI_AINT_CAST_TO_LONG_LONG
-  #define ADIOI_ENSURE_AINT_FITS_IN_PTR(aint_value) 
-  #define ADIOI_Assert assert
-  #define MPIU_Upint unsigned long
-  #define MPID_THREADPRIV_DECL
-#endif
-
-#ifdef USE_DBG_LOGGING    /*todo fix dependency on mpich?*/
-/* DBGT_FPRINTF terse level printing */
-#define DBGT_FPRINTF if (MPIR_Ext_dbg_romio_verbose_enabled) fprintf(stderr,"%s:%d:",__FILE__,__LINE__); \
-if (MPIR_Ext_dbg_romio_terse_enabled) fprintf
-/* DBG_FPRINTF default (typical level) printing */
-#define DBG_FPRINTF if (MPIR_Ext_dbg_romio_verbose_enabled) fprintf(stderr,"%s:%d:",__FILE__,__LINE__); \
-if (MPIR_Ext_dbg_romio_typical_enabled) fprintf
-/* DBGV_FPRINTF verbose level printing */
-#define DBGV_FPRINTF if (MPIR_Ext_dbg_romio_verbose_enabled) fprintf(stderr,"%s:%d:",__FILE__,__LINE__); \
- if (MPIR_Ext_dbg_romio_verbose_enabled) fprintf
-#else /* compile it out */
-#define DBGT_FPRINTF if (0) fprintf
-#define DBG_FPRINTF if (0) fprintf
-#define DBGV_FPRINTF if (0) fprintf
-#endif
-
-/* declarations for threaded I/O */
-/* i/o thread data structure (bgmpio_pthreadwc) */
-typedef struct wcThreadFuncData {
-    ADIO_File fd;
-    int io_kind;
-    char *buf;
-    int size;
-    ADIO_Offset offset;
-    ADIO_Status *status;
-    int error_code;
-} ADIOI_IO_ThreadFuncData;
-
-void *ADIOI_IO_Thread_Func(void *vptr_args);
-
-
-#ifdef HAVE_LIMITS_H
-#include <limits.h>
-#endif
-
-#ifndef PATH_MAX
-#define PATH_MAX 65535
-#endif
-
-#if (HAVE_DECL_PWRITE == 0)
-#include <sys/types.h>
-#include <unistd.h>
-ssize_t pread(int fd, void *buf, size_t count, off_t offset);
-ssize_t pwrite(int fd, const void *buf, size_t count, off_t offset);
-
-#endif
-
-#endif  /* ADIOI_INCLUDE */
diff --git a/3rd-party/romio321/adio/include/adioi_errmsg.h b/3rd-party/romio321/adio/include/adioi_errmsg.h
deleted file mode 100644
index b1cdb4577a2..00000000000
--- a/3rd-party/romio321/adio/include/adioi_errmsg.h
+++ /dev/null
@@ -1,73 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *  (C) 2001 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-
-MPI_ERR_FILE
-    MPIR_ERR_FILE_NULL     "Null file handle"
-    MPIR_ERR_FILE_CORRUPT  "Corrupted file handle"
-
-MPI_ERR_AMODE
-    3  "Exactly one of MPI_MODE_RDONLY, MPI_MODE_WRONLY, or MPI_MODE_RDWR must be specified"
-    5  "Cannot use MPI_MODE_CREATE or MPI_MODE_EXCL with MPI_MODE_RDONLY"
-    7  "Cannot specify MPI_MODE_SEQUENTIAL with MPI_MODE_RDWR"
-
-MPI_ERR_ARG
-    MPIR_ERR_OFFSET_ARG "Invalid offset argument"
-    MPIR_ERR_DATAREP_ARG "Null datarep argument"
-    MPIR_ERR_COUNT_ARG "Invalid count argument"
-    MPIR_ERR_SIZE_ARG "Invalid size argument"
-    MPIR_ERR_WHENCE_ARG "Invalid whence argument"
-    MPIR_ERR_FLAG_ARG "flag argument must be the same on all processes"
-    MPIR_ERR_DISP_ARG "Invalid displacement argument"
-    MPIR_ERR_ETYPE_ARG "Invalid etype argument"
-    MPIR_ERR_FILETYPE_ARG "Invalid filetype argument"
-    MPIR_ERR_SIZE_ARG_NOT_SAME "size argument must be the same on all processes"
-    MPIR_ERR_OFFSET_ARG_NEG "offset points to a negative location in the file"
-    MPIR_ERR_WHENCE_ARG_NOT_SAME "whence argument must be the same on all processes"
-    MPIR_ERR_OFFSET_ARG_NOT_SAME "offset argument must be the same on all processes"
-
-MPI_ERR_TYPE
-    MPIR_ERR_TYPE_NULL (null datatype. from MPICH)
-
-MPI_ERR_UNSUPPORTED_OPERATION
-    MPIR_ERR_NO_SHARED_FP "Shared file pointer not supported on PIOFS and PVFS"
-    MPIR_ERR_AMODE_SEQ "Cannot use this function when file is opened with amode MPI_MODE_SEQUENTIAL"
-    MPIR_ERR_MODE_WRONLY "Cannot read from a file opened with amode MPI_MODE_WRONLY"
-    MPIR_ERR_NO_MODE_SEQ "MPI_MODE_SEQUENTIAL not supported on PIOFS and PVFS"
-
-MPI_ERR_REQUEST
-    MPIR_ERR_REQUEST_NULL (null request. from MPICH)
-
-MPI_ERR_IO
-    MPIR_ERR_ETYPE_FRACTIONAL "Only an integral number of etypes can be accessed"
-    MPIR_ERR_NO_FSTYPE "Can't determine the file-system type. Check the filename/path you provided and try again. Otherwise, prefix the filename with a string to indicate the type of file sytem (piofs:, pfs:, nfs:, ufs:, hfs:, xfs:, sfs:, pvfs:, panfs: ftp: gsiftp:)"
-    MPIR_ERR_NO_PFS "ROMIO has not been configured to use the PFS file system"
-    MPIR_ERR_NO_PIOFS "ROMIO has not been configured to use the PIOFS file system"
-    MPIR_ERR_NO_UFS "ROMIO has not been configured to use the UFS file system"
-    MPIR_ERR_NO_NFS "ROMIO has not been configured to use the NFS file system"
-    MPIR_ERR_NO_HFS "ROMIO has not been configured to use the HFS file system"
-    MPIR_ERR_NO_XFS "ROMIO has not been configured to use the XFS file system"
-    MPIR_ERR_NO_SFS "ROMIO has not been configured to use the SFS file system"
-    MPIR_ERR_NO_PVFS "ROMIO has not been configured to use the PVFS file system"
-    MPIR_ERR_NO_PANFS "ROMIO has not been configured to use the PANFS file system"
-    MPIR_ERR_MULTIPLE_SPLIT_COLL "Only one active split collective I/O operation allowed per file handle"
-    MPIR_ERR_NO_SPLIT_COLL "No previous split collective begin"
-    MPIR_ERR_ASYNC_OUTSTANDING "There are outstanding nonblocking I/O operations on this file"
-    MPIR_ADIO_ERROR "I/O Error"  strerror(errno)
-    MPIR_READ_PERM "ROMIO tries to optimize this access by doing a read-modify-write, but is unable to read the file. Please give the file read permission and open it with MPI_MODE_RDWR."
-    MPIR_PREALLOC_PERM "To preallocate disk space, ROMIO needs to read the file and write it back, but is unable to read the file. Please give the file read permission and open it with MPI_MODE_RDWR."
-    MPIR_ERR_FILETYPE  "Filetype must be constructed out of one or more etypes"
-    MPIR_ERR_NO_TESTFS "ROMIO has not been configured to use the TESTFS file system"
-    MPIR_ERR_DEFERRED "independent IO attempted even though no_indep_rw hint given"
-    MPIR_ERR_NO_BGL "ROMIO has not been configured to use the BGL file system"
-
-MPI_ERR_COMM
-    MPIR_ERR_COMM_NULL (null communicator. from MPICH)
-MPIR_ERR_COMM_INTER  (no intercommunicator. (from MPICH)
-
-MPI_ERR_UNSUPPORTED_DATAREP
-    MPIR_ERR_NOT_NATIVE_DATAREP "Only native data representation currently supported"
-
-
diff --git a/3rd-party/romio321/adio/include/adioi_error.h b/3rd-party/romio321/adio/include/adioi_error.h
deleted file mode 100644
index 300a0638615..00000000000
--- a/3rd-party/romio321/adio/include/adioi_error.h
+++ /dev/null
@@ -1,183 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *  (C) 2001 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-
-#include <string.h> /* for strerror() */
-
-/* MPIO_CHECK_XXX macros are used to clean up error checking and
- * handling in many of the romio/mpi-io/ source files.
- */
-#define MPIO_CHECK_FILE_HANDLE(fh, myname, error_code)          \
-if ((fh <= (ADIO_File) 0) ||					\
-    ((fh)->cookie != ADIOI_FILE_COOKIE)) {			\
-    error_code = MPIO_Err_create_code(MPI_SUCCESS,		\
-				      MPIR_ERR_RECOVERABLE,	\
-				      myname, __LINE__,		\
-				      MPI_ERR_FILE,		\
-				      "**iobadfh", 0);		\
-    error_code = MPIO_Err_return_file(MPI_FILE_NULL, error_code);\
-    goto fn_exit;                                               \
-}
-
-/* TODO could add more glue code to help check for handle validity, or perhaps
- * do some sort of always-safe attribute/info call to check for handle validity */
-#define MPIO_CHECK_COMM(comm_, myname_, error_code_)                          \
-    do {                                                                      \
-        if ((comm_) == MPI_COMM_NULL) {                                       \
-            error_code = MPIO_Err_create_code(MPI_SUCCESS,                    \
-                                              MPIR_ERR_RECOVERABLE,           \
-                                              (myname_), __LINE__,            \
-                                              MPI_ERR_COMM,                   \
-                                              "**commnull", 0);               \
-            error_code_ = MPIO_Err_return_file(MPI_FILE_NULL, (error_code_)); \
-            goto fn_exit;                                                     \
-        }                                                                     \
-    } while (0)
-
-#define MPIO_CHECK_COUNT(fh, count, myname, error_code)         \
-if (count < 0) {						\
-    error_code = MPIO_Err_create_code(MPI_SUCCESS,		\
-				      MPIR_ERR_RECOVERABLE,	\
-				      myname, __LINE__,		\
-				      MPI_ERR_COUNT, 		\
-				      "**iobadcount", 0);	\
-    error_code = MPIO_Err_return_file(fh, error_code);		\
-    goto fn_exit;                                               \
-}
-
-#define MPIO_CHECK_COUNT_SIZE(fh, count, datatype_size, myname, error_code)         \
-if (count*datatype_size != (ADIO_Offset)(unsigned)count*(ADIO_Offset)datatype_size) {	\
-    error_code = MPIO_Err_create_code(MPI_SUCCESS,		\
-				      MPIR_ERR_RECOVERABLE,	\
-				      myname, __LINE__,		\
-				      MPI_ERR_ARG, 		\
-				      "**iobadcount", 0);	\
-    error_code = MPIO_Err_return_file(fh, error_code);		\
-    goto fn_exit;                                               \
-}
-
-#define MPIO_CHECK_DATATYPE(fh, datatype, myname, error_code)       \
-    do {                                                            \
-        if (datatype == MPI_DATATYPE_NULL) {                        \
-            error_code = MPIO_Err_create_code(MPI_SUCCESS,          \
-                                              MPIR_ERR_RECOVERABLE, \
-                                              myname, __LINE__,     \
-                                              MPI_ERR_TYPE,         \
-                                              "**dtypenull", 0);    \
-        }                                                           \
-        else {                                                      \
-            MPIO_DATATYPE_ISCOMMITTED(datatype, error_code);        \
-        }                                                           \
-        if (error_code != MPI_SUCCESS) {                            \
-            error_code = MPIO_Err_return_file(fh, error_code);      \
-            goto fn_exit;                                           \
-        }                                                           \
-    } while (0)
-
-#define MPIO_CHECK_READABLE(fh, myname, error_code)		\
-if (fh->access_mode & ADIO_WRONLY) {			\
-    error_code = MPIO_Err_create_code(MPI_SUCCESS,		\
-				      MPIR_ERR_RECOVERABLE,	\
-				      myname, __LINE__,		\
-				      MPI_ERR_ACCESS, 		\
-				      "**iowronly", 0);		\
-    error_code = MPIO_Err_return_file(fh, error_code);          \
-    goto fn_exit;                                               \
-}
-
-#define MPIO_CHECK_WRITABLE(fh, myname, error_code)		\
-if (fh->access_mode & ADIO_RDONLY) {			\
-    error_code = MPIO_Err_create_code(MPI_SUCCESS,		\
-				      MPIR_ERR_RECOVERABLE,	\
-				      myname, __LINE__,		\
-				      MPI_ERR_READ_ONLY,	\
-				      "**iordonly",		\
-				      0);			\
-    error_code = MPIO_Err_return_file(fh, error_code);		\
-    goto fn_exit;                                               \
-}
-
-#define MPIO_CHECK_NOT_SEQUENTIAL_MODE(fh, myname, error_code)		\
-if (fh->access_mode & ADIO_SEQUENTIAL) {				\
-    error_code = MPIO_Err_create_code(MPI_SUCCESS,			\
-				      MPIR_ERR_RECOVERABLE,		\
-				      myname, __LINE__,			\
-				      MPI_ERR_UNSUPPORTED_OPERATION,	\
-				      "**ioamodeseq", 0);		\
-    error_code = MPIO_Err_return_file(fh, error_code);                  \
-    goto fn_exit;                                                       \
-}
-
-#define MPIO_CHECK_INTEGRAL_ETYPE(fh, count, dtype_size, myname, error_code) \
-if ((count*dtype_size) % fh->etype_size != 0) {				     \
-    error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,     \
-				      myname, __LINE__, MPI_ERR_IO, 	     \
-				      "**ioetype", 0);			     \
-    error_code = MPIO_Err_return_file(fh, error_code);			     \
-    goto fn_exit;                                                            \
-}
-
-#define MPIO_CHECK_FS_SUPPORTS_SHARED(fh, myname, error_code)		\
-if (!ADIO_Feature(fh, ADIO_SHARED_FP))                                  \
-{									\
-    error_code = MPIO_Err_create_code(MPI_SUCCESS,			\
-				      MPIR_ERR_RECOVERABLE,		\
-				      myname, __LINE__,			\
-				      MPI_ERR_UNSUPPORTED_OPERATION,	\
-				      "**iosharedunsupported", 0);	\
-    error_code = MPIO_Err_return_file(fh, error_code);			\
-    goto fn_exit;                                                       \
-}
-
-/* MPIO_ERR_CREATE_CODE_XXX macros are used to clean up creation of
- * error codes for common cases in romio/adio/
- */
-#define MPIO_ERR_CREATE_CODE_ERRNO(myname, myerrno, error_code_p) \
-*(error_code_p) = MPIO_Err_create_code(MPI_SUCCESS,		  \
-				       MPIR_ERR_RECOVERABLE,	  \
-				       myname, __LINE__,	  \
-				       MPI_ERR_IO,		  \
-				       "System call I/O error",	  \
-				       "Syscall error from %s: %s",	  \
-				       myname,                    \
-				       strerror(myerrno));
-
-#define MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname, key, error_code_p)	      \
-*(error_code_p) = MPIO_Err_create_code(MPI_SUCCESS,			      \
-				       MPIR_ERR_RECOVERABLE,		      \
-				       myname, __LINE__,		      \
-                                       MPI_ERR_NOT_SAME,                      \
-				       "Value for info key not same across processes",  \
-				       "Value for info key %s not same across processes",\
-				       key);
-
-
-/* TODO: handle the independent io case more gracefully  */
-#define ADIOI_TEST_DEFERRED(fh, myname, error_code)\
-    if(! (fh)->is_open ) {\
-	    ADIO_ImmediateOpen((fh), (error_code)); }
-
-/* Check MPI_Info object by calling MPI_Info_dup, if the info object is valid
-then the dup operation will succeed */
-/* a collective check for error makes this macro collective */
-#define MPIO_CHECK_INFO_ALL(info, error_code, comm) {     \
-    MPI_Info dupinfo;                           \
-    int tmp_err = MPI_SUCCESS;                  \
-    if (info == MPI_INFO_NULL) {                \
-	dupinfo = MPI_INFO_NULL;                    \
-	error_code = MPI_SUCCESS;                   \
-    } else {                                        \
-	error_code = MPI_Info_dup(info, &dupinfo);  \
-    }                                               \
-    MPI_Allreduce(&error_code, &tmp_err, 1, MPI_INT, MPI_MAX, comm); \
-    if(tmp_err != MPI_SUCCESS) {                                            \
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, \
-		myname, __LINE__, MPI_ERR_OTHER, "**info", 0); \
-	goto fn_fail; \
-    }                                           \
-    if (dupinfo != MPI_INFO_NULL) {             \
-        MPI_Info_free(&dupinfo);                \
-    }                                           \
-}
diff --git a/3rd-party/romio321/adio/include/adioi_fs_proto.h b/3rd-party/romio321/adio/include/adioi_fs_proto.h
deleted file mode 100644
index e3af9170821..00000000000
--- a/3rd-party/romio321/adio/include/adioi_fs_proto.h
+++ /dev/null
@@ -1,92 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-
-#ifndef ADIO_PROTO
-#define ADIO_PROTO
-
-#ifdef ROMIO_NFS
-extern struct ADIOI_Fns_struct ADIO_NFS_operations;
-/* prototypes are in adio/ad_nfs/ad_nfs.h */
-#endif
-
-#ifdef ROMIO_PANFS
-extern struct ADIOI_Fns_struct ADIO_PANFS_operations;
-/* prototypes are in adio/ad_panfs/ad_panfs.h */
-#endif
-
-#ifdef ROMIO_PFS
-extern struct ADIOI_Fns_struct ADIO_PFS_operations;
-/* prototypes are in adio/ad_pfs/ad_pfs.h */
-#endif
-
-#ifdef ROMIO_PIOFS
-extern struct ADIOI_Fns_struct ADIO_PIOFS_operations;
-/* prototypes are in adio/ad_piofs/ad_piofs.h */
-#endif
-
-#ifdef ROMIO_UFS
-extern struct ADIOI_Fns_struct ADIO_UFS_operations;
-/* prototypes are in adio/ad_ufs/ad_ufs.h */
-#endif
-
-#ifdef ROMIO_HFS
-extern struct ADIOI_Fns_struct ADIO_HFS_operations;
-/* prototypes are in adio/ad_hfs/ad_hfs.h */
-#endif
-
-#ifdef ROMIO_XFS
-extern struct ADIOI_Fns_struct ADIO_XFS_operations;
-/* prototypes are in adio/ad_xfs/ad_xfs.h */
-#endif
-
-#ifdef ROMIO_SFS
-extern struct ADIOI_Fns_struct ADIO_SFS_operations;
-/* prototypes are in adio/ad_sfs/ad_sfs.h */
-#endif
-
-#ifdef ROMIO_LUSTRE
-extern struct ADIOI_Fns_struct ADIO_LUSTRE_operations;
-/* prototypes are in adio/ad_lustre/ad_lustre.h */
-#endif
-
-#ifdef ROMIO_NTFS
-extern struct ADIOI_Fns_struct ADIO_NTFS_operations;
-/* prototypes are in adio/ad_ntfs/ad_ntfs.h */
-#endif
-
-#ifdef ROMIO_PVFS
-extern struct ADIOI_Fns_struct ADIO_PVFS_operations;
-/* prototypes are in adio/ad_pvfs/ad_pvfs.h */
-#endif
-
-#ifdef ROMIO_PVFS2
-extern struct ADIOI_Fns_struct ADIO_PVFS2_operations;
-/* prototypes are in adio/ad_pvfs2/ad_pvfs2.h */
-#endif
-
-#ifdef ROMIO_TESTFS
-extern struct ADIOI_Fns_struct ADIO_TESTFS_operations;
-/* prototypes are in adio/ad_testfs/ad_testfs.h */
-#endif
-
-#ifdef ROMIO_GPFS
-extern struct ADIOI_Fns_struct ADIO_GPFS_operations;
-/* prototypes are in adio/ad_gpfs/ad_gpfs.h */
-#endif
-
-#ifdef ROMIO_GRIDFTP
-/* prototypes are in adio/ad_gridftp/ad_gridftp.h */
-extern struct ADIOI_Fns_struct ADIO_GRIDFTP_operations;
-#endif
-
-#ifdef ROMIO_ZOIDFS
-/* prototypes are in adio/ad_zoidfs/ad_zoidfs.h */
-extern struct ADIOI_Fns_struct ADIO_ZOIDFS_operations;
-#endif
-
-#endif
diff --git a/3rd-party/romio321/adio/include/heap-sort.h b/3rd-party/romio321/adio/include/heap-sort.h
deleted file mode 100644
index 903a1e80606..00000000000
--- a/3rd-party/romio321/adio/include/heap-sort.h
+++ /dev/null
@@ -1,22 +0,0 @@
-#include "adio.h"
-
-typedef struct {
-    ADIO_Offset offset;
-    int proc;
-    ADIO_Offset reg_max_len;
-} heap_node_t;
-
-typedef struct {
-    heap_node_t *nodes;
-    int size;
-} heap_t;
-
-/*static inline int parent(heap_t *heap, int i);
-static inline int left(heap_t *heap, int i);
-static inline int right(heap_t *heap, int i); */
-void ADIOI_Heap_free(heap_t *heap);
-int ADIOI_Heap_create(heap_t *heap, int size);
-void ADIOI_Heap_insert(heap_t *heap, ADIO_Offset offset, int proc,
-		 ADIO_Offset reg_max_len);
-void ADIOI_Heap_extract_min(heap_t *heap, ADIO_Offset* key, int *proc,
-		      ADIO_Offset *reg_max_len);
diff --git a/3rd-party/romio321/adio/include/hint_fns.h b/3rd-party/romio321/adio/include/hint_fns.h
deleted file mode 100644
index 7c20177deb9..00000000000
--- a/3rd-party/romio321/adio/include/hint_fns.h
+++ /dev/null
@@ -1,23 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *
- *  (C) 2013 UChicago/Argonne LLC
- *      See COPYRIGHT in top-level directory.
- */
-
-#ifndef _HINT_FNS_H
-#define _HINT_FNS_H
-
-int ADIOI_Info_check_and_install_int(ADIO_File fd, MPI_Info info, const char *key,
-	        int *local_cache, char *funcname, int *error_code);
-
-int ADIOI_Info_check_and_install_enabled(ADIO_File fd, MPI_Info info, const char *key,
-	        int *local_cache, char *funcname, int *error_code);
-
-int ADIOI_Info_check_and_install_true(ADIO_File fd, MPI_Info info, const char *key,
-	        int *local_cache, char *funcname, int *error_code);
-
-int ADIOI_Info_check_and_install_str(ADIO_File fd, MPI_Info info, const char *key,
-	        char **local_cache, char *funcname, int *error_code);
-
-#endif
diff --git a/3rd-party/romio321/adio/include/mpio_error.h b/3rd-party/romio321/adio/include/mpio_error.h
deleted file mode 100644
index f9d22b17219..00000000000
--- a/3rd-party/romio321/adio/include/mpio_error.h
+++ /dev/null
@@ -1,76 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *  (C) 2001 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-/* MPI_ERR_FILE */
-#define MPIR_ERR_FILE_NULL 3
-#define MPIR_ERR_FILE_CORRUPT 5
-
-/* MPI_ERR_AMODE */
-/* kind values set directly to 3,5,7 in mpi-io/open.c */
-
-/* MPI_ERR_ARG */
-#define MPIR_ERR_OFFSET_ARG 65
-#define MPIR_ERR_DATAREP_ARG 67
-#define MPIR_ERR_COUNT_ARG 69
-#define MPIR_ERR_SIZE_ARG 71
-#define MPIR_ERR_WHENCE_ARG 73
-#define MPIR_ERR_FLAG_ARG 75
-#define MPIR_ERR_DISP_ARG 77
-#define MPIR_ERR_ETYPE_ARG 79
-#define MPIR_ERR_FILETYPE_ARG 81
-#define MPIR_ERR_SIZE_ARG_NOT_SAME 83
-#define MPIR_ERR_OFFSET_ARG_NEG 85
-#define MPIR_ERR_WHENCE_ARG_NOT_SAME 87
-#define MPIR_ERR_OFFSET_ARG_NOT_SAME 89
-
-/* MPI_ERR_TYPE */
-#ifndef MPIR_ERR_TYPE_NULL
-#define MPIR_ERR_TYPE_NULL 5
-#endif
-
-/* MPI_ERR_UNSUPPORTED_OPERATION */
-#define MPIR_ERR_NO_SHARED_FP 3
-#define MPIR_ERR_AMODE_SEQ 5
-#define MPIR_ERR_MODE_WRONLY 7
-#define MPIR_ERR_NO_MODE_SEQ 9
-
-/* MPI_ERR_REQUEST */
-#ifndef MPIR_ERR_REQUEST_NULL
-#define MPIR_ERR_REQUEST_NULL 3
-#endif
-
-/* MPI_ERR_IO */
-#define MPIR_ADIO_ERROR 1  /* used for strerror(errno) */
-#define MPIR_ERR_ETYPE_FRACTIONAL 3
-#define MPIR_ERR_NO_FSTYPE 5
-#define MPIR_ERR_NO_PFS 7
-#define MPIR_ERR_NO_PIOFS 9
-#define MPIR_ERR_NO_UFS 11
-#define MPIR_ERR_NO_NFS 13
-#define MPIR_ERR_NO_HFS 15
-#define MPIR_ERR_NO_XFS 17
-#define MPIR_ERR_NO_SFS 19
-#define MPIR_ERR_NO_PVFS 21
-#define MPIR_ERR_NO_PANFS 22
-#define MPIR_ERR_MULTIPLE_SPLIT_COLL 23
-#define MPIR_ERR_NO_SPLIT_COLL 25
-#define MPIR_ERR_ASYNC_OUTSTANDING 27
-#define MPIR_READ_PERM 29
-#define MPIR_PREALLOC_PERM 31
-#define MPIR_ERR_FILETYPE 33 
-#define MPIR_ERR_NO_NTFS 35
-#define MPIR_ERR_NO_TESTFS 36
-#define MPIR_ERR_NO_LUSTRE 37
-#define MPIR_ERR_NO_BGL 38
-
-/* MPI_ERR_COMM */
-#ifndef MPIR_ERR_COMM_NULL
-#define MPIR_ERR_COMM_NULL 3
-#define MPIR_ERR_COMM_INTER 5 
-#endif
-
-/* MPI_ERR_UNSUPPORTED_DATAREP */
-#define MPIR_ERR_NOT_NATIVE_DATAREP 3
-
diff --git a/3rd-party/romio321/adio/include/mpipr.h b/3rd-party/romio321/adio/include/mpipr.h
deleted file mode 100644
index 8b86c4cfe13..00000000000
--- a/3rd-party/romio321/adio/include/mpipr.h
+++ /dev/null
@@ -1,383 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *  (C) 2001 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-/* This file replaces all MPI function names with their PMPI equivalents.
-   PMPI versions are used by default so that the user can profile 
-   application code without interference from MPI functions used by 
-   MPI-IO. */
-
-#ifndef USE_MPI_VERSIONS
-
-#undef MPI_Abort
-#define MPI_Abort PMPI_Abort
-#undef MPI_Allgather
-#define MPI_Allgather PMPI_Allgather
-#undef MPI_Allgatherv
-#define MPI_Allgatherv PMPI_Allgatherv
-#undef MPI_Allreduce
-#define MPI_Allreduce PMPI_Allreduce
-#undef MPI_Alltoall
-#define MPI_Alltoall PMPI_Alltoall
-#undef MPI_Alltoallv
-#define MPI_Alltoallv PMPI_Alltoallv
-#undef MPI_Alltoallw
-#define MPI_Alltoallw PMPI_Alltoallw
-#undef MPI_Attr_delete
-#define MPI_Attr_delete PMPI_Attr_delete
-#undef MPI_Attr_get
-#define MPI_Attr_get PMPI_Attr_get
-#undef MPI_Barrier
-#define MPI_Barrier PMPI_Barrier
-#undef MPI_Bcast
-#define MPI_Bcast PMPI_Bcast
-#undef MPI_Bsend
-#define MPI_Bsend PMPI_Bsend
-#undef MPI_Bsend_init
-#define MPI_Bsend_init PMPI_Bsend_init
-#undef MPI_Buffer_attach
-#define MPI_Buffer_attach PMPI_Buffer_attach
-#undef MPI_Buffer_detach
-#define MPI_Buffer_detach PMPI_Buffer_detach
-#undef MPI_Cancel
-#define MPI_Cancel PMPI_Cancel
-#undef MPI_Cart_coords
-#define MPI_Cart_coords PMPI_Cart_coords
-#undef MPI_Cart_create
-#define MPI_Cart_create PMPI_Cart_create
-#undef MPI_Cart_get
-#define MPI_Cart_get PMPI_Cart_get
-#undef MPI_Cart_map
-#define MPI_Cart_map PMPI_Cart_map
-#undef MPI_Cart_rank
-#define MPI_Cart_rank PMPI_Cart_rank
-#undef MPI_Cart_shift
-#define MPI_Cart_shift PMPI_Cart_shift
-#undef MPI_Cart_sub
-#define MPI_Cart_sub PMPI_Cart_sub
-#undef MPI_Cartdim_get
-#define MPI_Cartdim_get PMPI_Cartdim_get
-#undef MPI_Comm_call_errhandler
-#define MPI_Comm_call_errhandler PMPI_Comm_call_errhandler
-#undef MPI_Comm_compare
-#define MPI_Comm_compare PMPI_Comm_compare
-#undef MPI_Comm_create
-#define MPI_Comm_create PMPI_Comm_create
-#undef MPI_Comm_create_keyval
-#define MPI_Comm_create_keyval PMPI_Comm_create_keyval
-#undef MPI_Comm_dup
-#define MPI_Comm_dup PMPI_Comm_dup
-#undef MPI_Comm_free
-#define MPI_Comm_free PMPI_Comm_free
-#undef MPI_Comm_free_keyval
-#define MPI_Comm_free_keyval PMPI_Comm_free_keyval
-#undef MPI_Comm_group
-#define MPI_Comm_group PMPI_Comm_group
-#undef MPI_Comm_rank
-#define MPI_Comm_rank PMPI_Comm_rank
-#undef MPI_Comm_remote_group
-#define MPI_Comm_remote_group PMPI_Comm_remote_group
-#undef MPI_Comm_remote_size
-#define MPI_Comm_remote_size PMPI_Comm_remote_size
-#undef MPI_Comm_set_attr
-#define MPI_Comm_set_attr PMPI_Comm_set_attr
-#undef MPI_Comm_size
-#define MPI_Comm_size PMPI_Comm_size
-#undef MPI_Comm_split
-#define MPI_Comm_split PMPI_Comm_split
-#undef MPI_Comm_test_inter
-#define MPI_Comm_test_inter PMPI_Comm_test_inter
-#undef MPI_Dims_create
-#define MPI_Dims_create PMPI_Dims_create
-#undef MPI_Errhandler_create
-#define MPI_Errhandler_create PMPI_Errhandler_create
-#undef MPI_Errhandler_free
-#define MPI_Errhandler_free PMPI_Errhandler_free
-#undef MPI_Errhandler_get
-#define MPI_Errhandler_get PMPI_Errhandler_get
-#undef MPI_Errhandler_set
-#define MPI_Errhandler_set PMPI_Errhandler_set
-#undef MPI_Error_class
-#define MPI_Error_class PMPI_Error_class
-#undef MPI_Error_string
-#define MPI_Error_string PMPI_Error_string
-#undef MPI_Finalize
-#define MPI_Finalize PMPI_Finalize
-#undef MPI_Gather
-#define MPI_Gather PMPI_Gather
-#undef MPI_Gatherv
-#define MPI_Gatherv PMPI_Gatherv
-#undef MPI_Get_address
-#define MPI_Get_address PMPI_Get_address
-#undef MPI_Get_count
-#define MPI_Get_count PMPI_Get_count
-#undef MPI_Get_elements
-#define MPI_Get_elements PMPI_Get_elements
-#undef MPI_Get_processor_name
-#define MPI_Get_processor_name PMPI_Get_processor_name
-#undef MPI_Get_version
-#define MPI_Get_version PMPI_Get_version
-#undef MPI_Graph_create
-#define MPI_Graph_create PMPI_Graph_create
-#undef MPI_Graph_get
-#define MPI_Graph_get PMPI_Graph_get
-#undef MPI_Graph_map
-#define MPI_Graph_map PMPI_Graph_map
-#undef MPI_Graph_neighbors
-#define MPI_Graph_neighbors PMPI_Graph_neighbors
-#undef MPI_Graph_neighbors_count
-#define MPI_Graph_neighbors_count PMPI_Graph_neighbors_count
-#undef MPI_Graphdims_get
-#define MPI_Graphdims_get PMPI_Graphdims_get
-#undef MPI_Group_compare
-#define MPI_Group_compare PMPI_Group_compare
-#undef MPI_Group_difference
-#define MPI_Group_difference PMPI_Group_difference
-#undef MPI_Group_excl
-#define MPI_Group_excl PMPI_Group_excl
-#undef MPI_Group_free
-#define MPI_Group_free PMPI_Group_free
-#undef MPI_Group_incl
-#define MPI_Group_incl PMPI_Group_incl
-#undef MPI_Group_intersection
-#define MPI_Group_intersection PMPI_Group_intersection
-#undef MPI_Group_range_excl
-#define MPI_Group_range_excl PMPI_Group_range_excl
-#undef MPI_Group_range_incl
-#define MPI_Group_range_incl PMPI_Group_range_incl
-#undef MPI_Group_rank
-#define MPI_Group_rank PMPI_Group_rank
-#undef MPI_Group_size
-#define MPI_Group_size PMPI_Group_size
-#undef MPI_Group_translate_ranks
-#define MPI_Group_translate_ranks PMPI_Group_translate_ranks
-#undef MPI_Group_union
-#define MPI_Group_union PMPI_Group_union
-#undef MPI_Ibsend
-#define MPI_Ibsend PMPI_Ibsend
-#undef MPI_Init
-#define MPI_Init PMPI_Init
-#undef MPI_Initialized
-#define MPI_Initialized PMPI_Initialized
-#undef MPI_Intercomm_create
-#define MPI_Intercomm_create PMPI_Intercomm_create
-#undef MPI_Intercomm_merge
-#define MPI_Intercomm_merge PMPI_Intercomm_merge
-#undef MPI_Iprobe
-#define MPI_Iprobe PMPI_Iprobe
-#undef MPI_Irecv
-#define MPI_Irecv PMPI_Irecv
-#undef MPI_Irsend
-#define MPI_Irsend PMPI_Irsend
-#undef MPI_Isend
-#define MPI_Isend PMPI_Isend
-#undef MPI_Issend
-#define MPI_Issend PMPI_Issend
-#undef MPI_Name_get
-#define MPI_Name_get PMPI_Name_get
-#undef MPI_Name_put
-#define MPI_Name_put PMPI_Name_put
-#undef MPI_Op_create
-#define MPI_Op_create PMPI_Op_create
-#undef MPI_Op_free
-#define MPI_Op_free PMPI_Op_free
-#undef MPI_Pack
-#define MPI_Pack PMPI_Pack
-#undef MPI_Pack_size
-#define MPI_Pack_size PMPI_Pack_size
-#undef MPI_Pcontrol
-#define MPI_Pcontrol PMPI_Pcontrol
-#undef MPI_Probe
-#define MPI_Probe PMPI_Probe
-#undef MPI_Recv
-#define MPI_Recv PMPI_Recv
-#undef MPI_Recv_init
-#define MPI_Recv_init PMPI_Recv_init
-#undef MPI_Reduce
-#define MPI_Reduce PMPI_Reduce
-#undef MPI_Reduce_scatter
-#define MPI_Reduce_scatter PMPI_Reduce_scatter
-#undef MPI_Request_c2f
-#define MPI_Request_c2f PMPI_Request_c2f
-#undef MPI_Request_free
-#define MPI_Request_free PMPI_Request_free
-#undef MPI_Rsend
-#define MPI_Rsend PMPI_Rsend
-#undef MPI_Rsend_init
-#define MPI_Rsend_init PMPI_Rsend_init
-#undef MPI_Scan
-#define MPI_Scan PMPI_Scan
-#undef MPI_Scatter
-#define MPI_Scatter PMPI_Scatter
-#undef MPI_Scatterv
-#define MPI_Scatterv PMPI_Scatterv
-#undef MPI_Send
-#define MPI_Send PMPI_Send
-#undef MPI_Send_init
-#define MPI_Send_init PMPI_Send_init
-#undef MPI_Sendrecv
-#define MPI_Sendrecv PMPI_Sendrecv
-#undef MPI_Sendrecv_replace
-#define MPI_Sendrecv_replace PMPI_Sendrecv_replace
-#undef MPI_Ssend
-#define MPI_Ssend PMPI_Ssend
-#undef MPI_Ssend_init
-#define MPI_Ssend_init PMPI_Ssend_init
-#undef MPI_Start
-#define MPI_Start PMPI_Start
-#undef MPI_Startall
-#define MPI_Startall PMPI_Startall
-#undef MPI_Test
-#define MPI_Test PMPI_Test
-#undef MPI_Test_cancelled
-#define MPI_Test_cancelled PMPI_Test_cancelled
-#undef MPI_Testall
-#define MPI_Testall PMPI_Testall
-#undef MPI_Testany
-#define MPI_Testany PMPI_Testany
-#undef MPI_Testsome
-#define MPI_Testsome PMPI_Testsome
-#undef MPI_Topo_status
-#define MPI_Topo_status PMPI_Topo_status
-#undef MPI_Topo_test
-#define MPI_Topo_test PMPI_Topo_test
-#undef MPI_Type_commit
-#define MPI_Type_commit PMPI_Type_commit
-#undef MPI_Type_contiguous
-#define MPI_Type_contiguous PMPI_Type_contiguous
-#undef MPI_Type_count
-#define MPI_Type_count PMPI_Type_count
-#undef MPI_Type_create_struct
-#define MPI_Type_create_struct PMPI_Type_create_struct
-#undef MPI_Type_create_resized
-#define MPI_Type_create_resized PMPI_Type_create_resized
-/* #define MPI_Type_create_darray PMPI_Type_create_darray */
-#undef MPI_Type_create_indexed_block
-#define MPI_Type_create_indexed_block PMPI_Type_create_indexed_block
-#undef MPI_Type_create_hindexed
-#define MPI_Type_create_hindexed PMPI_Type_create_hindexed
-#undef MPI_Type_create_hindexed_block
-#define MPI_Type_create_hindexed_block PMPI_Type_create_hindexed_block
-#undef MPI_Type_create_hvector
-#define MPI_Type_create_hvector PMPI_Type_create_hvector
-/* #define MPI_Type_create_subarray PMPI_Type_create_subarray */
-#undef MPI_Type_get_extent
-#define MPI_Type_get_extent PMPI_Type_get_extent
-#undef MPI_Type_free
-#define MPI_Type_free PMPI_Type_free
-#undef MPI_Type_get_contents
-#define MPI_Type_get_contents PMPI_Type_get_contents
-#undef MPI_Type_get_envelope
-#define MPI_Type_get_envelope PMPI_Type_get_envelope
-#undef MPI_Type_get_true_extent
-#define MPI_Type_get_true_extent PMPI_Type_get_true_extent
-#undef MPI_Type_indexed
-#define MPI_Type_indexed PMPI_Type_indexed
-#undef MPI_Type_size
-#define MPI_Type_size PMPI_Type_size
-#undef MPI_Type_vector
-#define MPI_Type_vector PMPI_Type_vector
-#undef MPI_Unpack
-#define MPI_Unpack PMPI_Unpack
-#undef MPI_Wait
-#define MPI_Wait PMPI_Wait
-#undef MPI_Waitall
-#define MPI_Waitall PMPI_Waitall
-#undef MPI_Waitany
-#define MPI_Waitany PMPI_Waitany
-#undef MPI_Waitsome
-#define MPI_Waitsome PMPI_Waitsome
-#undef MPI_Wtick
-#define MPI_Wtick PMPI_Wtick
-#undef MPI_Wtime
-#define MPI_Wtime PMPI_Wtime
-
-/* commented out because these could be macros themselves, as in MPICH 
-#undef MPI_Type_c2f
-#define MPI_Type_c2f PMPI_Type_c2f
-#undef MPI_Type_f2c
-#define MPI_Type_f2c PMPI_Type_f2c
-#undef MPI_Comm_c2f
-#define MPI_Comm_c2f PMPI_Comm_c2f
-#undef MPI_Comm_f2c
-#define MPI_Comm_f2c PMPI_Comm_f2c
-#undef MPI_Group_c2f
-#define MPI_Group_c2f PMPI_Group_c2f
-#undef MPI_Group_f2c
-#define MPI_Group_f2c PMPI_Group_f2c
-#undef MPI_Op_c2f
-#define MPI_Op_c2f PMPI_Op_c2f
-#undef MPI_Op_f2c
-#define MPI_Op_f2c PMPI_Op_f2c
-#undef MPI_Request_c2f
-#define MPI_Request_c2f PMPI_Request_c2f
-#undef MPI_Request_f2c
-#define MPI_Request_f2c PMPI_Request_f2c
-#undef MPI_Win_c2f
-#define MPI_Win_c2f PMPI_Win_c2f
-#undef MPI_Win_f2c
-#define MPI_Win_f2c PMPI_Win_f2c
-*/
-#undef MPI_Status_c2f
-#define MPI_Status_c2f PMPI_Status_c2f
-#undef MPI_Status_f2c
-#define MPI_Status_f2c PMPI_Status_f2c
-
-#undef MPI_Status_set_elements
-#define MPI_Status_set_elements PMPI_Status_set_elements
-
-#ifndef HAVE_MPI_INFO_SRC  /* everywhere except in info source directory */
-#undef MPI_Info_create
-#define MPI_Info_create PMPI_Info_create
-#undef MPI_Info_set
-#define MPI_Info_set PMPI_Info_set
-#undef MPI_Info_delete
-#define MPI_Info_delete PMPI_Info_delete
-#undef MPI_Info_get
-#define MPI_Info_get PMPI_Info_get
-#undef MPI_Info_get_valuelen
-#define MPI_Info_get_valuelen PMPI_Info_get_valuelen
-#undef MPI_Info_get_nkeys
-#define MPI_Info_get_nkeys PMPI_Info_get_nkeys
-#undef MPI_Info_get_nthkey
-#define MPI_Info_get_nthkey PMPI_Info_get_nthkey
-#undef MPI_Info_dup
-#define MPI_Info_dup PMPI_Info_dup
-#undef MPI_Info_free
-#define MPI_Info_free PMPI_Info_free
-/* #undef MPI_Info_c2f
-#define MPI_Info_c2f PMPI_Info_c2f
-#undef MPI_Info_f2c
-#define MPI_Info_f2c PMPI_Info_f2c
-*/
-#endif
-#undef MPI_Grequest_start
-#define MPI_Grequest_start PMPI_Grequest_start
-#undef MPI_Grequest_complete
-#define MPI_Grequest_complete PMPI_Grequest_complete
-#undef MPI_Status_set_cancelled
-#define MPI_Status_set_cancelled PMPI_Status_set_cancelled
-
-#undef MPIX_Grequest_class_create
-#define MPIX_Grequest_class_create PMPIX_Grequest_class_create
-#undef MPIX_Grequest_class_allocate
-#define MPIX_Grequest_class_allocate PMPIX_Grequest_class_allocate
-
-#ifdef MPIO_FORTRAN_SRC   /* only in MPI-IO Fortran source directory */
-#undef MPI_File_c2f
-#define MPI_File_c2f PMPI_File_c2f
-#undef MPI_File_f2c
-#define MPI_File_f2c PMPI_File_f2c
-#endif
-
-#undef MPI_Type_get_attr
-#define MPI_Type_get_attr PMPI_Type_get_attr
-#undef MPI_Type_set_attr
-#define MPI_Type_set_attr PMPI_Type_set_attr
-#undef MPI_Comm_set_attr
-#define MPI_Comm_set_attr PMPI_Comm_set_attr
-#undef MPI_Type_create_keyval
-#define MPI_Type_create_keyval PMPI_Type_create_keyval
-
-#endif
diff --git a/3rd-party/romio321/adio/include/mpiu_external32.h b/3rd-party/romio321/adio/include/mpiu_external32.h
deleted file mode 100644
index 43023737741..00000000000
--- a/3rd-party/romio321/adio/include/mpiu_external32.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *
- *  (C) 2012 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-
-#ifndef _MPIU_EXTERNAL32_H_
-#define _MPIU_EXTERNAL32_H_
-
-int MPIU_write_external32_conversion_fn (const void *userbuf, MPI_Datatype datatype,
-	        int count, void *filebuf);
-int MPIU_read_external32_conversion_fn(void *userbuf, MPI_Datatype datatype,
-	        int count, void *filebuf);
-int MPIU_datatype_full_size(MPI_Datatype datatype, MPI_Aint *size);
-
-/* given a buffer, count, and datatype, return an apropriately sized and
- *  * external32-formatted buffer, suitable for handing off to a subsequent write
- *   * routine */
-int MPIU_external32_buffer_setup(const void * buf, int count, MPI_Datatype type, void **newbuf);
-
-#endif
diff --git a/3rd-party/romio321/adio/include/mpiu_greq.h b/3rd-party/romio321/adio/include/mpiu_greq.h
deleted file mode 100644
index e3fce1c41d8..00000000000
--- a/3rd-party/romio321/adio/include/mpiu_greq.h
+++ /dev/null
@@ -1,15 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *
- *  (C) 2003 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-
-#ifndef _MPIU_GREQUEST_H
-#define _MPIU_GREQUEST_H
-
-int MPIU_Greq_query_fn(void *extra_state, MPI_Status *status);
-int MPIU_Greq_free_fn(void *extra_state);
-int MPIU_Greq_cancel_fn(void *extra_state, int complete);
-
-#endif
diff --git a/3rd-party/romio321/adio/include/nopackage.h b/3rd-party/romio321/adio/include/nopackage.h
deleted file mode 100644
index 184e4b4ac84..00000000000
--- a/3rd-party/romio321/adio/include/nopackage.h
+++ /dev/null
@@ -1,16 +0,0 @@
-/*
- *  (C) 2011 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-/* See comments in MPICH's configure.ac for an explanation of what this file is
- * and why it exists. */
-
-/* intentionally omitting any #ifndef guard, repeated inclusion is intentional */
-#undef PACKAGE
-#undef PACKAGE_BUGREPORT
-#undef PACKAGE_NAME
-#undef PACKAGE_STRING
-#undef PACKAGE_TARNAME
-#undef PACKAGE_URL
-#undef PACKAGE_VERSION
-
diff --git a/3rd-party/romio321/adio/include/romioconf-undefs.h b/3rd-party/romio321/adio/include/romioconf-undefs.h
deleted file mode 100644
index 5c21607a4ee..00000000000
--- a/3rd-party/romio321/adio/include/romioconf-undefs.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- *  Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                          University Research and Technology
- *                          Corporation.  All rights reserved.
- *  Copyright (c) 2004-2005 The University of Tennessee and The University
- *                          of Tennessee Research Foundation.  All rights
- *                          reserved.
- *  Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                          University of Stuttgart. All rights reserved.
- *  Copyright (c) 2004-2005 The Regents of the University of California.
- *                          All rights reserved.
- *  $COPYRIGHT$
- *
- *  Additional copyrights may follow
- *
- *  $HEADER$
- */
-
-#ifndef ROMIO_CONF_UNDEFS_H
-#define ROMIO_CONF_UNDEFS_H
-
-/* Need to add some undefs here so that we don't conflict with the
- * main ompi_config.h.  Arrgh.  Stupid autoconf not giving us the option
- * to not define these macros... grumble...
- */
-#if defined(PACKAGE_BUGREPORT)
-#undef PACKAGE_BUGREPORT
-#endif
-#if defined(PACKAGE_NAME)
-#undef PACKAGE_NAME
-#endif
-#if defined(PACKAGE_STRING)
-#undef PACKAGE_STRING
-#endif
-#if defined(PACKAGE_TARNAME)
-#undef PACKAGE_TARNAME
-#endif
-#if defined(PACKAGE_VERSION)
-#undef PACKAGE_VERSION
-#endif
-#if defined(PACKAGE_URL)
-#undef PACKAGE_URL
-#endif
-
-#endif /* ROMIOCONF_UNDEFS_H */
diff --git a/3rd-party/romio321/autogen.sh b/3rd-party/romio321/autogen.sh
deleted file mode 100755
index dcc84c738a7..00000000000
--- a/3rd-party/romio321/autogen.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-
-${AUTORECONF:-autoreconf} ${autoreconf_args:-"-vif"} -I confdb
diff --git a/3rd-party/romio321/confdb/aclocal_am.m4 b/3rd-party/romio321/confdb/aclocal_am.m4
deleted file mode 100644
index 4d3b0de2257..00000000000
--- a/3rd-party/romio321/confdb/aclocal_am.m4
+++ /dev/null
@@ -1,6 +0,0 @@
-dnl AM_IGNORE is an extension that tells (a patched) automake not to
-dnl include the specified AC_SUBST variable in the Makefile.in that
-dnl automake generates.  We don't use AC_DEFUN, since aclocal will 
-dnl then complain that AM_IGNORE is a duplicate (if you are using the
-dnl patched automake/aclocal).
-m4_ifdef([AM_IGNORE],[],[m4_define([AM_IGNORE],[])])
diff --git a/3rd-party/romio321/confdb/aclocal_atomic.m4 b/3rd-party/romio321/confdb/aclocal_atomic.m4
deleted file mode 100644
index 4e96cfbf366..00000000000
--- a/3rd-party/romio321/confdb/aclocal_atomic.m4
+++ /dev/null
@@ -1,227 +0,0 @@
-dnl /*D PAC_C_MEMATOMIC - Try and determine how to implement memory-atomic
-dnl   operations with the selected C compiler
-dnl
-dnl Synopsis:
-dnl PAC_C_MEMATOMIC
-dnl
-dnl Notes:
-dnl Defines names of the following form
-dnl + HAVE_GCC_ASM_AND_X86_{MFENCE,LFENCE,SFENCE} - gcc __asm__ will issue
-dnl    mfence, lfence, or sfence
-dnl . HAVE___ASM_AND_X86_{MFENCE,LFENCE,SFENCE} - __asm _emit will issue
-dnl    mfence, lfence, or sfence
-dnl . HAVE_ASM_AND_X86_{MFENCE,LFENCE,SFENCE} - asm("...") will issue
-dnl    mfence, lfence, or sfence
-dnl . HAVE__INTERLOCKEDEXCHANGE - _InterlockedExchange intrinsic is available
-dnl    (IA64)
-dnl . HAVE_GCC_ASM_SPARC_MEMBAR - gcc __asm__ will issue SPARC architecture
-dnl    memory barrier instruction
-dnl . HAVE_SOLARIS_ASM_SPARC_MEMBAR - Solaris asm() will issue SPARC 
-dnl    architecture memory barrier instruction
-dnl . HAVE_GCC_ASM_SPARC_STBAR - gcc __asm__ will issue stbar
-dnl - HAVE_SOLARIS_ASM_SPARC_STBAR - Solaris __asm() will issue stbar
-dnl 
-dnl D*/
-AC_DEFUN([PAC_C_MEMATOMIC],[
-AC_CACHE_CHECK([for x86 mfence instruction using __asm__],
-    pac_cv_have_gcc_asm_and_x86_mfence,[
-AC_TRY_RUN([
-int main(int argc, char **argv)
-{
-    __asm__ __volatile__  ( ".byte 0x0f, 0xae, 0xf0" ::: "memory" );
-    exit(0);
-}
-],
-pac_cv_have_gcc_asm_and_x86_mfence=yes,pac_cv_have_gcc_asm_and_x86_mfence=no)])
-
-if test "$pac_cv_have_gcc_asm_and_x86_mfence" = "yes" ; then
-    AC_DEFINE(HAVE_GCC_ASM_AND_X86_MFENCE, 1, [Define if using gcc on a x86 system with the mfence instruction])
-fi
-
-AC_CACHE_CHECK([for x86 sfence instruction using __asm__],
-    pac_cv_have_gcc_asm_and_x86_sfence,[
-AC_TRY_RUN([
-int main(int argc, char **argv)
-{
-    __asm__ __volatile__  ( ".byte 0x0f, 0xae, 0xf8" ::: "memory" );
-    exit(0);
-}
-],
-pac_cv_have_gcc_asm_and_x86_sfence=yes,pac_cv_have_gcc_asm_and_x86_sfence=no)])
-
-if test "$pac_cv_have_gcc_asm_and_x86_sfence" = "yes" ; then
-    AC_DEFINE(HAVE_GCC_ASM_AND_X86_SFENCE, 1, [Define if using gcc on a x86 system with the sfence instruction])
-fi
-
-AC_CACHE_CHECK([for x86 lfence instruction using __asm__],
-    pac_cv_have_gcc_asm_and_x86_lfence,[
-AC_TRY_RUN([
-int main(int argc, char **argv)
-{
-    __asm__ __volatile__  ( ".byte 0x0f, 0xae, 0xe8" ::: "memory" );
-    exit(0);
-}
-],
-pac_cv_have_gcc_asm_and_x86_lfence=yes,pac_cv_have_gcc_asm_and_x86_lfence=no)])
-
-if test "$pac_cv_have_gcc_asm_and_x86_lfence" = "yes" ; then
-    AC_DEFINE(HAVE_GCC_ASM_AND_X86_LFENCE, 1, [Define if using gcc on a x86 system with the lfence instruction])
-fi
-
-dnl Some compilers, like icc, may want __asm _emit
-AC_CACHE_CHECK([for x86 mfence instruction using __asm],
-     pac_cv_have___asm_and_x86_mfence,[
-AC_TRY_RUN([
-int main(int argc, char **argv)
-{
-    __asm _emit 0x0f __asm _emit 0xae __asm _emit 0xf0 ;
-    exit(0);
-}
-],
-pac_cv_have___asm_and_x86_mfence=yes,pac_cv_have___asm_and_x86_mfence=no)])
-
-if test "$pac_cv_have___asm_and_x86_mfence" = "yes" ; then
-    AC_DEFINE(HAVE___ASM_AND_X86_MFENCE, 1, [Define if using __asm on a x86 system with the mfence instruction])
-fi
-
-AC_CACHE_CHECK([for x86 sfence instruction using __asm],
-    pac_cv_have___asm_and_x86_sfence,[
-AC_TRY_RUN([
-int main(int argc, char **argv)
-{
-    __asm sfence ;
-    exit(0);
-}
-],
-pac_cv_have___asm_and_x86_sfence=yes,pac_cv_have___asm_and_x86_sfence=no)])
-
-if test "$pac_cv_have___asm_and_x86_sfence" = "yes" ; then
-    AC_DEFINE(HAVE___ASM_AND_X86_SFENCE, 1, [Define if using __asm on a x86 system with the sfence instruction])
-fi
-
-AC_CACHE_CHECK([for x86 lfence instruction using __asm],
-    pac_cv_have___asm_and_x86_lfence,[
-AC_TRY_RUN([
-int main(int argc, char **argv)
-{
-    __asm _emit 0x0f __asm _emit 0xae __asm _emit 0xe8 ;
-    exit(0);
-}
-],
-pac_cv_have___asm_and_x86_lfence=yes,pac_cv_have___asm_and_x86_lfence=no)])
-
-if test "$lac_cv_have___asm_and_x86_lfence" = "yes" ; then
-    AC_DEFINE(HAVE___ASM_AND_X86_LFENCE, 1, [Define if using __asm on a x86 system with the lfence instruction])
-fi
-
-dnl 
-dnl Some compilers, such as pgcc, may require additional arguments.
-dnl pgcc may need -Masmkeyword flag.  We may want to try this with and 
-dnl without adding -Masmkeyword to CFLAGS
-
-AC_CACHE_CHECK([for x86 mfence instruction using asm()],
-    pac_cv_have_asm_and_x86_mfence,[
-AC_TRY_RUN([
-int main(int argc, char **argv)
-{
-    asm("_emit 0x0f __asm _emit 0xae __asm _emit 0xf0");
-    exit(0);
-}
-],
-pac_cv_have_asm_and_x86_mfence=yes,pac_cv_have_asm_and_x86_mfence=no)])
-
-if test "$pac_cv_have_asm_and_x86_mfence" = "yes" ; then
-    AC_DEFINE(HAVE_ASM_AND_X86_MFENCE, 1, [Define if using asm() on a x86 system with the mfence instruction])
-fi
-
-AC_CACHE_CHECK([for x86 sfence instruction using asm()],
-    pac_cv_have_asm_and_x86_sfence,[
-AC_TRY_RUN([
-int main(int argc, char **argv)
-{
-    asm("sfence");
-    exit(0);
-}
-],
-pac_cv_have_asm_and_x86_sfence=yes,pac_cv_have_asm_and_x86_sfence=no)])
-
-if test "$pac_cv_have_asm_and_x86_sfence" = "yes" ; then
-    AC_DEFINE(HAVE_ASM_AND_X86_SFENCE, 1, [Define if using asm() on a x86 system with the sfence instruction])
-fi
-
-AC_CACHE_CHECK([for x86 lfence instruction using asm()],
-    pac_cv_have_asm_and_x86_lfence,[
-AC_TRY_RUN([
-int main(int argc, char **argv)
-{
-    asm("_emit 0x0f __asm _emit 0xae __asm _emit 0xe8");
-    exit(0);
-}
-],
-pac_cv_have_asm_and_x86_lfence=yes,pac_cv_have_asm_and_x86_lfence=no)])
-
-if test "$pac_cv_have_asm_and_x86_lfence" = "yes" ; then
-    AC_DEFINE(HAVE_ASM_AND_X86_LFENCE, 1, [Define if using asm() on a x86 system with the lfence instruction])
-fi
-
-AC_CACHE_CHECK([for _InterlockedExchange intrinsic],
-    pac_cv_have__InterlockedExchange,[
-AC_TRY_RUN([
-int main(int argc, char **argv)
-{
-    unsigned long lock, *lock_ptr;
-    lock_ptr = &lock;
-    _InterlockedExchange(lock_ptr, 1);
-    exit(0);
-}
-],
-pac_cv_have__InterlockedExchange=yes,pac_cv_have__InterlockedExchange=no)])
-
-if test "$pac_cv_have__InterlockedExchange" = "yes" ; then
-    AC_DEFINE(HAVE__INTERLOCKEDEXCHANGE, 1, [Define if _InterlockedExchange intrinsic is available])
-fi
-
-AC_CACHE_CHECK([for SPARC membar instruction with gcc],
-    pac_cv_gcc_sparc_membar,[
-AC_TRY_RUN([
-int main(int argc, char **argv){
-    __asm__ __volatile__ ( "membar #StoreLoad | #StoreStore" : : : "memory" );
-    exit(0);
-}],pac_cv_gcc_sparc_membar=yes,pac_cv_gcc_sparc_membar=no)])
-if test "$pac_cv_gcc_sparc_membar" = yes ; then
-    AC_DEFINE(HAVE_GCC_ASM_SPARC_MEMBAR,1,[Define if gcc asm membar supported])
-fi
-
-AC_CACHE_CHECK([for SPARC membar instruction with Solaris C],
-    pac_cv_solaris_sparc_membar,[
-AC_TRY_RUN([
-int main(int argc, char **argv){
-    __asm ( "membar #StoreLoad | #StoreStore");
-    exit(0);
-}],pac_cv_solaris_sparc_membar=yes,pac_cv_solaris_sparc_membar=no)])
-if test "$pac_cv_solaris_sparc_membar" = yes ; then
-    AC_DEFINE(HAVE_SOLARIS_ASM_SPARC_MEMBAR,1,[Define if solaris asm membar supported])
-fi
-
-AC_CACHE_CHECK([for SPARC stbar instruction with gcc],
-    pac_cv_gcc_sparc_stbar,[
-AC_TRY_RUN([
-int main(int argc, char **argv){
-    __asm__ __volatile__ ( "stbar" : : : "memory" );
-    exit(0);
-}],pac_cv_gcc_sparc_stbar=yes,pac_cv_gcc_sparc_stbar=no)])
-if test "$pac_cv_gcc_sparc_stbar" = yes ; then
-    AC_DEFINE(HAVE_GCC_ASM_SPARC_STBAR,1,[Define if gcc asm stbar supported])
-fi
-
-AC_CACHE_CHECK([for SPARC stbar instruction with Solaris C],
-    pac_cv_solaris_sparc_stbar,[
-AC_TRY_RUN([
-int main(int argc, char **argv){
-    __asm ( "stbar" );
-    exit(0);
-}],pac_cv_solaris_sparc_stbar=yes,pac_cv_solaris_sparc_stbar=no)])
-if test "$pac_cv_solaris_sparc_stbar" = yes ; then
-    AC_DEFINE(HAVE_SOLARIS_ASM_SPARC_STBAR,1,[Define if solaris asm stbar supported])
-fi
-])
\ No newline at end of file
diff --git a/3rd-party/romio321/confdb/aclocal_attr_alias.m4 b/3rd-party/romio321/confdb/aclocal_attr_alias.m4
deleted file mode 100644
index 32f016546dc..00000000000
--- a/3rd-party/romio321/confdb/aclocal_attr_alias.m4
+++ /dev/null
@@ -1,504 +0,0 @@
-dnl
-dnl Check for BSD or POSIZ style global symbol lister, nm.
-dnl If found, pac_path_NM_G contains the absolute athname of nm + options
-dnl pac_path_NM_G_type will be either POSIX or BSD.  NM_G will be
-dnl pac_path_NM_G without the absolute path.  Preference is BSD style.
-dnl
-dnl The test checks if nm accepts the known options and also if nm produces
-dnl the expected BSD or POSIX output format.
-dnl
-AC_DEFUN([PAC_PATH_NM_G],[
-AC_MSG_CHECKING([for BSD/POSIX style global symbol lister])
-AC_LANG_PUSH(C)
-AC_PATH_PROGS_FEATURE_CHECK(NM_G, nm, [
-    # Check if nm accepts -g and BSD or POSIX compatible flag.
-    # Use the `sed 1q' to avoid HP-UX's unknown option message:
-    #   nm: unknown option "B" ignored
-    # Tru64's nm complains that /dev/null is an invalid object file
-    #
-    # AIX's sed does not accept \+, 1) instead of doing 's|a\+||', do 's|aa*||'
-    # or 2) instead of 's|A \+B|AB|g', do 's|A  *B|AB|g' 
-
-    # Check if nm accepts -g
-    case `${ac_path_NM_G} -g /dev/null 2>&1 | sed '1q'` in
-    */dev/null* | *'Invalid file or object type'*)
-        ac_path_NM_G="${ac_path_NM_G} -g"
-        # Check if nm accepts -B
-        case `${ac_path_NM_G} -B /dev/null 2>&1 | sed '1q'` in
-        */dev/null* | *'Invalid file or object type'*)
-            AC_COMPILE_IFELSE([
-                AC_LANG_SOURCE([int iglobal;])
-            ],[
-                changequote(,)
-                case `${ac_path_NM_G} -B conftest.$OBJEXT | sed -e 's|[0-9][0-9]*  *[A-Z]  *iglobal|XXXX|g'` in
-                *XXXX*)
-                    pac_path_NM_G="${ac_path_NM_G} -B"
-                    pac_path_NM_G_type="BSD"
-                    ;;
-                esac
-                changequote([,])
-            ])
-            ;;
-        *)
-            # Check if nm accepts -P
-            case `${ac_path_NM_G} -P /dev/null 2>&1 | sed '1q'` in
-            */dev/null* | *'Invalid file or object type'*)
-                AC_COMPILE_IFELSE([
-                    AC_LANG_SOURCE([int iglobal;])
-                ],[
-                    changequote(,)
-                    case `${ac_path_NM_G} -P conftest.$OBJEXT | sed -e 's|iglobal  *[A-Z]  *[0-9][0-9]*|XXXX|g'` in
-                    *XXXX*)
-                        pac_path_NM_G="${ac_path_NM_G} -P"
-                        pac_path_NM_G_type="POSIX"
-                        ;;
-                    esac
-                    changequote([,])
-                ])
-                ;;
-            esac  # Endof case `${ac_path_NM_G} -P
-            ;;
-        esac   # Endof case `${ac_path_NM_G} -B
-        ;;
-    esac  # Endof case `${ac_path_NM_G} -g
-    if test "X$pac_path_NM_G" != "X" ; then
-        AC_MSG_RESULT([$pac_path_NM_G_type style,$pac_path_NM_G])
-        NM_G="`echo $pac_path_NM_G | sed -e 's|^.*nm |nm |g'`"
-    else
-        AC_MSG_RESULT(no)
-    fi
-    ac_cv_path_NM_G=${ac_path_NM_G}
-    ac_path_NM_G_found=:
-], [AC_MSG_RESULT(no)],
-[$PATH$PATH_SEPARATOR/usr/ccs/bin/elf$PATH_SEPARATOR/usr/ccs/bin$PATH_SEPARATOR/usr/ucb$PATH_SEPARATOR/bin])
-AC_LANG_POP(C)
-]) dnl Endof AC_DEFUN([PAC_PATH_NM_G]
-dnl
-dnl PAC_C_MULTI_ATTR_ALIAS()
-dnl
-dnl The checks if multiple __attribute__((alias)) is available
-dnl If the multiple __attribute((alias)) support is found,
-dnl pac_c_multi_attr_alias=yes is set.
-dnl
-dnl The default is to do a runtime test.  When cross_compiling=yes,
-dnl pac_path_NM_G will be used to determine the test result.
-dnl If CFLAGS(or CPPFLAGS) contains ATTR_ALIAS_DEBUG, the runtime will print
-dnl out addresses of struct(s) for debugging purpose.
-dnl
-dnl
-AC_DEFUN([PAC_C_MULTI_ATTR_ALIAS],[
-AC_REQUIRE([PAC_PATH_NM_G])
-AC_LANG_PUSH(C)
-AC_CHECK_HEADERS([stdio.h])
-AC_MSG_CHECKING([for multiple __attribute__((alias)) support])
-
-#Compile the "other" __attribute__ object file.
-AC_COMPILE_IFELSE([
-    AC_LANG_SOURCE([
-#if defined(HAVE_STDIO_H) || defined(STDC_HEADERS)
-#include <stdio.h>
-#endif
-
-struct mpif_cmblk_t_ { int imember; };
-typedef struct mpif_cmblk_t_ mpif_cmblk_t;
-
-/* NOT initialize these structure so it appears in BSS or as COMMON symbols */
-mpif_cmblk_t mpifcmb;
-mpif_cmblk_t MPIFCMB;
-
-/*
-   Do the test in this file instead in the file
-   where __attribute__((alias)) is used. 
-   This is needed for pgcc since pgcc seems to
-   define aliased symbols if they are in the same file.
-*/
-/*
-    We can't do the following comparision in one test:
-
-    ilogical = (( &mpifcmb == ptr && &MPIFCMB == ptr ) ? TRUE : FALSE) ;
-
-    because some compiler, like gcc 4.4.2's -O* optimizes the code
-    such that the ilogical expression is FALSE. The likely reason is that
-    mpifcmb and MPIFCMB are defined in the same scope in which C optimizer
-    may have treated them as different objects (with different addresses),
-    &mpifcmb != &MPIFCMB, before actually running the test and hence the
-    illogical expression is assumed to be always FALSE.  The solution taken
-    here is to prevent the optimizer the opportunity to equate &mpifcmb and
-    &MPIFCMB (in same scope), e.g. using 2 separate tests and combine the
-    test results in a different scope.
-*/
-int same_addrs1( void *ptr );
-int same_addrs1( void *ptr )
-{
-#if defined(ATTR_ALIAS_DEBUG)
-    printf( "others: addr(mpifcmb)=%p, addr(input ptr)=%p\n", &mpifcmb, ptr );
-#endif
-    return ( &mpifcmb == ptr ? 1 : 0 );
-}
-
-int same_addrs2( void *ptr );
-int same_addrs2( void *ptr )
-{
-#if defined(ATTR_ALIAS_DEBUG)
-    printf( "others: addr(MPIFCMB)=%p, addr(input ptr)=%p\n", &MPIFCMB, ptr );
-#endif
-    return ( &MPIFCMB == ptr ? 1 : 0 );
-}
-
-    ])
-],[
-    rm -f pac_conftest_other.$OBJEXT
-    PAC_RUNLOG([cp conftest.$OBJEXT pac_conftest_other.$OBJEXT])
-    test -s pac_conftest_other.$OBJEXT && pac_c_attr_alias_other=yes
-dnl     cp conftest.$ac_ext pac_conftest_other.$ac_ext
-dnl     echo
-dnl     echo "pac_conftest_other.$OBJEXT"
-dnl     nm -P -g pac_conftest_other.$OBJEXT | grep -i "mpifcmb"
-],[
-    pac_c_attr_alias_other=no
-])  dnl Endof AC_COMPILE_IFELSE
-
-pac_c_attr_alias_main=no
-if test "$pac_c_attr_alias_other" = "yes" ; then
-
-#   Push LIBS for later restoration.
-    PAC_PUSH_FLAG([LIBS])
-    LIBS="pac_conftest_other.$OBJEXT $LIBS"
-
-#   Link the "other" __attribute__ object file.
-    AC_LINK_IFELSE([
-        AC_LANG_PROGRAM([
-#if defined(HAVE_STDIO_H) || defined(STDC_HEADERS)
-#include <stdio.h>
-#endif
- 
-struct mpif_cmblk_t_ { int imember; };
-typedef struct mpif_cmblk_t_ mpif_cmblk_t;
-
-mpif_cmblk_t mpifcmbr = {0};
-extern mpif_cmblk_t MPIFCMB __attribute__ ((alias("mpifcmbr")));
-extern mpif_cmblk_t mpifcmb __attribute__ ((alias("mpifcmbr")));
-
-extern int same_addrs1( void *ptr );
-extern int same_addrs2( void *ptr );
-
-        ],[
-    int iaddr = 0;
-#if defined(ATTR_ALIAS_DEBUG)
-    printf( "main: addr(mpifcmbr) = %p\n", &mpifcmbr );
-    printf( "main: addr(mpifcmb) = %p\n", &mpifcmb );
-    printf( "main: addr(MPIFCMB) = %p\n", &MPIFCMB );
-#endif
-    iaddr = same_addrs1( &mpifcmbr ) && same_addrs2( &mpifcmbr );
-    FILE *file = fopen( "pac_conftestval", "w" );
-    if (!file) return 1;
-    fprintf( file, "%d\n", iaddr );
-        ])
-    ],[
-        rm -f pac_conftest_main$EXEEXT
-        PAC_RUNLOG([cp conftest$EXEEXT pac_conftest_main$EXEEXT])
-        test -x pac_conftest_main$EXEEXT && pac_c_attr_alias_main=yes
-dnl         cp conftest.$ac_ext pac_conftest_main.$ac_ext
-dnl         echo
-dnl         echo "pac_conftest_main$EXEEXT"
-dnl         nm -P -g pac_conftest_main$EXEEXT | grep -i "mpifcmb"
-    ],[
-        pac_c_attr_alias_main=no
-dnl         cp conftest.$ac_ext pac_conftest_main.$ac_ext
-    ])  dnl Endof AC_LINK_IFELSE
-
-# Restore the previously pushed LIBS
-    PAC_POP_FLAG([LIBS])
-    rm -f pac_conftest_other.$OBJEXT
-fi dnl Endof if test "$pac_c_attr_alias_other" = "yes"
-
-if test "$pac_c_attr_alias_main" = "yes" ; then
-    if test "$cross_compiling" = "yes" ; then
-        changequote(,)
-        # echo "PAC CROSS-COMPILING" dnl
-        # POSIX NM = nm -P format dnl
-        if test "$pac_path_NM_G_type" = "POSIX" ; then
-            addrs=`${pac_path_NM_G} ./pac_conftest_main$EXEEXT | grep -i mpifcmb | sed -e 's% *[a-zA-Z][a-zA-Z]*  *[a-zA-Z]  *\([0-9abcdef][0-9abcdef]*\).*%\1%g'`
-        fi
-
-        # BSD NM = nm -B format dnl
-        if test "$pac_path_NM_G_type" = "BSD" ; then
-            addrs=`${pac_path_NM_G} -g ./pac_conftest_main$EXEEXT | grep -i mpifcmb | sed -e "s% *\([0-9abcdef][0-9abcdef]*\)  *[a-zA-Z]  *[a-zA-Z][a-zA-A]*.*%\1%g"`
-        fi
-        changequote([,])
-
-        cmp_addr=""
-        diff_addrs=no
-        for addr in ${addrs} ; do
-            if test "X${cmp_addr}" != "X" ; then
-                if test "${cmp_addr}" != "${addr}" ; then
-                    diff_addrs=yes
-                    break
-                fi
-            else
-                cmp_addr=${addr}
-            fi
-        done
-        
-        if test "$diff_addrs" != "yes" ; then
-            dnl echo "Same addresses. Multiple aliases support"
-            AC_MSG_RESULT([${NM_G} says yes])
-            pac_c_multi_attr_alias=yes
-        else
-            dnl echo "Different addresses. No multiple aliases support."
-            AC_MSG_RESULT([${NM_G} says no])
-            pac_c_multi_attr_alias=no
-        fi
-
-    else # if test "$cross_compiling" != "yes"
-        rm -f pac_conftestval
-        ac_try="./pac_conftest_main$EXEEXT"
-        if AC_TRY_EVAL(ac_try) ; then
-            pac_c_attr_alias_val=0
-            if test -s pac_conftestval ; then
-                eval pac_c_attr_alias_val=`cat pac_conftestval`
-            fi
-            if test "$pac_c_attr_alias_val" -eq 1 ; then
-                AC_MSG_RESULT(yes)
-                pac_c_multi_attr_alias=yes
-            else
-                AC_MSG_RESULT(no)
-                pac_c_multi_attr_alias=no
-            fi
-            rm -f pac_conftestval
-        fi
-    fi
-    dnl Endof if test "$cross_compiling" = "yes"
-    rm -f pac_conftest_main$EXEEXT
-else
-    AC_MSG_RESULT(no! link failure)
-    pac_c_multi_attr_alias=no
-fi dnl Endof if test "$pac_c_attr_alias_main" = "yes"
-
-AC_LANG_POP(C)
-
-]) dnl  Endof AC_DEFUN([PAC_C_MULTI_ATTR_ALIAS]
-dnl
-dnl PAC_C_ATTR_ALIGNED()
-dnl
-dnl Check if __attribute__((aligned)) support is there.
-dnl If so, set pac_c_attr_aligned=yes.
-dnl
-dnl Do a link test instead of compile test to check if the linker
-dnl would emit an error.
-dnl
-AC_DEFUN([PAC_C_ATTR_ALIGNED],[
-AC_LANG_PUSH(C)
-AC_MSG_CHECKING([for __attribute__((aligned)) support])
-#Link the __attribute__ object file.
-AC_LINK_IFELSE([
-    AC_LANG_PROGRAM([
-struct mpif_cmblk_t_ { int imembers[5]; };
-typedef struct mpif_cmblk_t_ mpif_cmblk_t;
-mpif_cmblk_t mpifcmbr __attribute__((aligned)) = {0};
-    ],[])
-],[pac_c_attr_aligned=yes], [pac_c_attr_aligned=no])
-AC_MSG_RESULT([$pac_c_attr_aligned])
-AC_LANG_POP(C)
-])
-dnl
-dnl PAC_F2C_ATTR_ALIGNED_SIZE(ARRAY_SIZE, [OUTPUT_VAR], [MIN_ALIGNMENT])
-dnl
-dnl ARRAY_SIZE    : Size of the integer array within the fortran commmon block.
-dnl OUTPUT_VAR    : Optional variable to be set.
-dnl                 if test succeeds, set OUTPUT_VAR=$pac_f2c_attr_aligned_str.
-dnl                 if test fails, set OUTPUT_VAR="unknown".
-dnl MIN_ALIGNMENT : Optional value.
-dnl                 Minimum alignment size to be used in OUTPUT_VAR.
-dnl                 pac_f2c_attr_aligned_str won't be modified.
-dnl
-dnl "pac_f2c_attr_aligned_str" will be set with
-dnl 1) __attribute__((aligned(ALIGNMENT_SIZE))),
-dnl 2) __attribute__((aligned)).
-dnl 3) "", i.e. empty string.
-dnl
-dnl 2) means the test can't find a good alignment value, but both the Fortran
-dnl    and C compilers are OK with "aligned" which in principle means the C
-dnl    compiler will pick the maximum useful alignment supported by the
-dnl    architecture.
-dnl 3) means that the test has failed to find the alignment.
-dnl
-AC_DEFUN([PAC_F2C_ATTR_ALIGNED_SIZE],[
-cmblksize=$1
-AC_MSG_CHECKING([the minimum alignment of Fortran common block of $cmblksize integers])
-dnl To find the minmium alignment of Fortran common block (of integer array)
-dnl as seen by C object file, C object files of various (typical) alignments
-dnl are linked to the Fortran code using the common block of integer array.
-#
-dnl Since the incorrect alignment results only a warning messages from the
-dnl fortran compiler(or linker), so we use "diff" to compare the fortran
-dnl compiler/linker output.  We cannot use AC_LANG_WERROR,
-dnl i.e. ac_fc_werror_flag=yes, because compiler like pgf77 at version 10.x)
-dnl has non-zero stderr output if a fortran program is used in the linking.
-dnl The stderr contains the name of fortran program even if the linking is
-dnl successful.  We could avoid the non-zero stderr output in pgf77 by
-dnl compiling everthing into object files and linking all the object files
-dnl with pgf77.  Doing that would need us to use AC_TRY_EVAL instead of
-dnl AC_LINK_IFELSE, so "diff" approach is used instead.
-#
-dnl Using diff of compiler(linker) output requires a reference output file
-dnl as the base of diff.  The process of creating this reference output file
-dnl has to be exactly the same as the testing process, because pgf77 has
-dnl the following weird behavour
-dnl pgf77 -o ftest ftest.f         => when $?=0 with zero stderr output
-dnl pgf77 -o ftest ftest.f dummy.o => when $?=0 with non-zero stderr output.
-dnl                                   stderr has "ftest.f:".
-dnl 
-# First create a fortran CONFTEST which will be used repeatedly.
-AC_LANG_PUSH([Fortran]) dnl AC_LANG_PUSH([Fortran 77])
-AC_LANG_CONFTEST([
-    AC_LANG_SOURCE([
-        program fconftest
-        integer isize
-        parameter (isize=$cmblksize)
-        integer status_array(isize)
-        common /mpifcmb/ status_array
-        save /mpifcmb/
-        end
-    ])
-])
-AC_LANG_POP([Fortran]) dnl AC_LANG_POP([Fortran 77])
-dnl
-dnl
-dnl
-# Compile a C dummy.$OBJEXT and link with Fortran test program to create
-# a reference linker output file, pac_align0.log, as the base of "diff".
-AC_LANG_PUSH([C])
-AC_COMPILE_IFELSE([AC_LANG_SOURCE([])],[
-    cp conftest.$ac_ext pac_conftest.c
-    PAC_RUNLOG([mv conftest.$OBJEXT pac_conftest.$OBJEXT])
-    PAC_PUSH_FLAG([LIBS])
-    LIBS="pac_conftest.$OBJEXT $LIBS"
-    AC_LANG_PUSH([Fortran]) dnl AC_LANG_PUSH([Fortran 77])
-    PAC_PUSH_FLAG([ac_link])
-    ac_link="`echo $ac_link | sed -e 's|>.*$|> $pac_logfile 2>\&1|g'`"
-    pac_logfile="pac_align0.log"
-    rm -f $pac_logfile
-    AC_LINK_IFELSE([],[
-        pac_f2c_alignedn_diffbase=yes
-    ],[
-        pac_f2c_alignedn_diffbase=no
-    ])
-    # Be sure NOT to remove the conftest.f which is still needed for later use.
-    # rm -f conftest.$ac_ext 
-    # Restore everything in autoconf that has been overwritten
-    PAC_POP_FLAG([ac_link])
-    # restore previously pushed LIBS
-    PAC_POP_FLAG([LIBS])
-    AC_LANG_POP([Fortran]) dnl AC_LANG_POP([Fortran 77])
-],[
-    pac_f2c_alignedn_diffbase=no
-])
-AC_LANG_POP([C])
-dnl
-dnl
-if test "$pac_f2c_alignedn_diffbase" = "yes" ; then
-    # Initialize pac_result_str to empty string since part of the test
-    # depends on pac_result_str is empty or non-empty string.
-    pac_result_str=""
-    # Initialize pac_f2c_attr_aligned_str to empty string and
-    # it will remain as empty string if the following test fails.
-    pac_f2c_attr_aligned_str=""
-    for asize in 4 8 16 32 64 128 max ; do
-        if test "$asize" != "max" ; then
-            pac_attr_aligned_str="__attribute__((aligned($asize)))"
-        else
-            pac_attr_aligned_str="__attribute__((aligned))"
-        fi
-        AC_LANG_PUSH([C])
-        #Compile the __attribute__ object file.
-        AC_COMPILE_IFELSE([
-            AC_LANG_SOURCE([
-changequote(,)
-struct mpif_cmblk_t_ { $MPI_FINT imembers[$cmblksize]; };
-changequote([,])
-typedef struct mpif_cmblk_t_ mpif_cmblk_t;
-mpif_cmblk_t mpifcmbr $pac_attr_aligned_str = {0};
-
-extern mpif_cmblk_t _CMPIFCMB  __attribute__ ((alias("mpifcmbr")));
-extern mpif_cmblk_t   MPIFCMB  __attribute__ ((alias("mpifcmbr")));
-extern mpif_cmblk_t   MPIFCMB_ __attribute__ ((alias("mpifcmbr")));
-extern mpif_cmblk_t _Cmpifcmb  __attribute__ ((alias("mpifcmbr")));
-extern mpif_cmblk_t   mpifcmb  __attribute__ ((alias("mpifcmbr")));
-extern mpif_cmblk_t   mpifcmb_ __attribute__ ((alias("mpifcmbr")));
-            ])
-        ],[
-            cp conftest.$ac_ext pac_conftest.c
-            PAC_RUNLOG([mv conftest.$OBJEXT pac_conftest.$OBJEXT])
-	    PAC_PUSH_FLAG([LIBS])
-            LIBS="pac_conftest.$OBJEXT $LIBS"
-            AC_LANG_PUSH([Fortran]) dnl AC_LANG_PUSH([Fortran 77])
-	    PAC_PUSH_FLAG([ac_link])
-            ac_link="`echo $ac_link | sed -e 's|>.*$|> $pac_logfile 2>\&1|g'`"
-            pac_logfile="pac_align1.log"
-            rm -f $pac_logfile
-            # Use conftest.f created in CONFTEST.
-            AC_LINK_IFELSE([],[
-                PAC_RUNLOG_IFELSE([diff -b pac_align0.log pac_align1.log],[
-                    pac_attr_alignedn=yes
-                ],[
-                    pac_attr_alignedn=no
-                    cat $pac_logfile >&AS_MESSAGE_LOG_FD
-                    echo "failed C program was:" >&AS_MESSAGE_LOG_FD
-                    cat pac_conftest.c >&AS_MESSAGE_LOG_FD
-                    echo "failed Fortran program was:" >&AS_MESSAGE_LOG_FD
-                    cat conftest.$ac_ext >&AS_MESSAGE_LOG_FD
-                ])
-            ],[
-                pac_attr_alignedn=no
-            ])
-            # Restore everything in autoconf that has been overwritten
-	    PAC_POP_FLAG([ac_link])
-            # restore previously pushed LIBS
-	    PAC_POP_FLAG([LIBS])
-            AC_LANG_POP([Fortran]) dnl AC_LANG_POP([Fortran 77])
-            # remove previously generated object file and C file.
-            rm -f pac_conftest.$OBJEXT pac_conftest.c
-            rm -f $pac_logfile
-            if test "$pac_attr_alignedn" = yes ; then
-                ifelse([$3],[],[
-                    pac_result_str="$asize"
-                    pac_f2c_attr_aligned_str="$pac_attr_aligned_str"
-                    break
-                ],[
-                    if test "$asize" != "max" -a "$asize" -lt "$3" ; then
-                        if test "X$pac_result_str" = "X" ; then
-                            pac_result_str="$asize"
-                            pac_f2c_attr_aligned_str="$pac_attr_aligned_str"
-                        fi
-                        continue
-                    else
-                        pac_f2c_attr_aligned_str="$pac_attr_aligned_str"
-                        if test "X$pac_result_str" != "X" ; then
-                            pac_result_str="$pac_result_str, too small! reset to $asize"
-                        else
-                            pac_result_str="$asize"
-                        fi
-                        break
-                    fi
-                ])
-            fi
-        ], [
-            pac_attr_alignedn=no
-        ])
-        AC_LANG_POP([C])
-    done
-    ifelse([$2],[],[],[$2="$pac_f2c_attr_aligned_str"])
-else
-    pac_result_str=""
-    # Since the test fails, set pac_f2c_attr_aligned_str to empty string.
-    pac_f2c_attr_aligned_str=""
-fi
-if test "X$pac_result_str" != "X" ; then
-    AC_MSG_RESULT([$pac_result_str])
-else
-    AC_MSG_RESULT([unknown])
-fi
-rm -f pac_align0.log
-])
-dnl
diff --git a/3rd-party/romio321/confdb/aclocal_bugfix.m4 b/3rd-party/romio321/confdb/aclocal_bugfix.m4
deleted file mode 100644
index f580d3919ad..00000000000
--- a/3rd-party/romio321/confdb/aclocal_bugfix.m4
+++ /dev/null
@@ -1,75 +0,0 @@
-dnl This internal macro fails to work properly with OTHER internal macros.
-dnl Basically, if the prologue is [], then no message should be generated.
-dnl This macro is in autoconf 2.52
-m4_define([AC_LANG_PROGRAM(Fortran 77)],
-[m4_if([$1],[[[]]],,[m4_ifval([$1],
-       [m4_warn([syntax], [$0: ignoring PROLOGUE: $1])])])dnl
-      program main
-$2
-      end])
-
-
-dnl/*D
-dnl PAC_PROG_CHECK_INSTALL_WORKS - Check whether the install program in INSTALL
-dnl works.
-dnl
-dnl Synopsis:
-dnl PAC_PROG_CHECK_INSTALL_WORKS
-dnl
-dnl Output Effect:
-dnl   Sets the variable 'INSTALL' to the value of 'ac_sh_install' if 
-dnl   a file cannot be installed into a local directory with the 'INSTALL'
-dnl   program
-dnl
-dnl Notes:
-dnl   The 'AC_PROG_INSTALL' scripts tries to avoid broken versions of 
-dnl   install by avoiding directories such as '/usr/sbin' where some 
-dnl   systems are known to have bad versions of 'install'.  Unfortunately, 
-dnl   this is exactly the sort of test-on-name instead of test-on-capability
-dnl   that 'autoconf' is meant to eliminate.  The test in this script
-dnl   is very simple but has been adequate for working around problems 
-dnl   on Solaris, where the '/usr/sbin/install' program (known by 
-dnl   autoconf to be bad because it is in /usr/sbin) is also reached by a 
-dnl   soft link through /bin, which autoconf believes is good.
-dnl
-dnl   No variables are cached to ensure that we do not make a mistake in 
-dnl   our choice of install program.
-dnl
-dnl   The Solaris configure requires the directory name to immediately
-dnl   follow the '-c' argument, rather than the more common 
-dnl.vb
-dnl      args sourcefiles destination-dir
-dnl.ve
-dnl D*/
-AC_DEFUN([PAC_PROG_CHECK_INSTALL_WORKS],[
-if test -z "$INSTALL" ; then
-    AC_MSG_RESULT([No install program available])
-else
-    # first make any "confdb/install-sh -c" into an absolute path
-    # this is a hack, but it's still much cleaner than anything else I could
-    # come up with (see tt#1007) [goodell@]
-    AS_CASE(["$INSTALL"],
-            [/*],[:],
-            [*install-sh*],[INSTALL="$master_top_srcdir/$INSTALL"])
-
-    # Check that this install really works
-    rm -f conftest
-    echo "Test file" > conftest
-    if test ! -d .conftest ; then mkdir .conftest ; fi
-    AC_MSG_CHECKING([whether install works])
-    if $INSTALL conftest .conftest >/dev/null 2>&1 ; then
-        installOk=yes
-    else
-        installOk=no
-    fi
-    rm -rf .conftest conftest
-    AC_MSG_RESULT($installOk)
-    if test "$installOk" = no ; then
-        if test -n "$ac_install_sh" ; then
-            INSTALL=$ac_install_sh
-        else
-	    AC_MSG_ERROR([Unable to find working install])
-        fi
-    fi
-fi
-])
diff --git a/3rd-party/romio321/confdb/aclocal_cache.m4 b/3rd-party/romio321/confdb/aclocal_cache.m4
deleted file mode 100644
index 9d02c416387..00000000000
--- a/3rd-party/romio321/confdb/aclocal_cache.m4
+++ /dev/null
@@ -1,372 +0,0 @@
-dnl
-dnl/*D
-dnl AC_CACHE_LOAD - Replacement for autoconf cache load 
-dnl
-dnl Notes:
-dnl Caching in autoconf is broken (at least through version 2.13).  
-dnl The problem is that the cache is read
-dnl without any check for whether it makes any sense to read it.
-dnl A common problem is a build on a shared file system; connecting to 
-dnl a different computer and then building within the same directory will
-dnl lead to at best error messages from configure and at worse a build that
-dnl is wrong but fails only at run time (e.g., wrong datatype sizes used).
-dnl Later versions of autoconf do include some checks for changes in the
-dnl environment that impact the choices, but still misses problems with
-dnl multiple different systems.
-dnl 
-dnl This fixes that by requiring the user to explicitly enable caching 
-dnl before the cache file will be loaded.
-dnl
-dnl To use this version of 'AC_CACHE_LOAD', you need to include
-dnl 'aclocal_cache.m4' in your 'aclocal.m4' file.  The sowing 'aclocal.m4'
-dnl file includes this file.
-dnl
-dnl If no --enable-cache or --disable-cache option is selected, the
-dnl command causes configure to keep track of the system being configured
-dnl in a config.system file; if the current system matches the value stored
-dnl in that file (or there is neither a config.cache nor config.system file),
-dnl configure will enable caching.  In order to ensure that the configure
-dnl tests make sense, the values of CC, F77, F90, and CXX are also included 
-dnl in the config.system file.  In addition, the value of PATH is included
-dnl to ensure that changes in the PATH that might select a different version
-dnl of a program with the same name (such as a native make versus gnumake)
-dnl are detected.
-dnl
-dnl Bugs:
-dnl This does not work with the Cygnus configure because the enable arguments
-dnl are processed *after* AC_CACHE_LOAD (!).  To address this, we avoid 
-dnl changing the value of enable_cache, and use real_enable_cache, duplicating
-dnl the "notgiven" value.
-dnl
-dnl The environment variable CONFIGURE_DEBUG_CACHE, if set to yes,
-dnl will cause additional data to be written out during the configure process.
-dnl This can be helpful in debugging the cache file process.
-dnl
-dnl See Also:
-dnl PAC_ARG_CACHING
-dnl D*/
-define([AC_CACHE_LOAD],
-[if test "$CONFIGURE_DEBUG_CACHE" = yes ; then 
-    oldopts="$-"
-    clearMinusX=no
-    set -x 
-    if test "$oldopts" != "$-" ; then 
-        clearMinusX=yes
-    fi
-fi 
-if test "X$cache_system" = "X" ; then
-    # A default file name, just in case
-    cache_system="config.system"
-    if test "$cache_file" != "/dev/null" ; then
-        # Get the directory for the cache file, if any
-	changequote(,)
-        dnl Be careful to ensure that there is no doubled slash
-        cache_system=`echo $cache_file | sed -e 's%^\(.*/\)[^/]*%\1config.system%'`
-	changequote([,])
-        test "x$cache_system" = "x$cache_file" && cache_system="config.system"
-#    else
-#        We must *not* set enable_cache to no because we need to know if
-#        enable_cache was not set.  
-#        enable_cache=no
-    fi
-fi
-dnl
-dnl The "action-if-not-given" part of AC_ARG_ENABLE is not executed until
-dnl after the AC_CACHE_LOAD is executed (!).  Thus, the value of 
-dnl enable_cache if neither --enable-cache or --disable-cache is selected
-dnl is null.  Just in case autoconf ever fixes this, we test both cases.
-dnl
-dnl Include PATH in the cache.system file since changing the path can
-dnl change which versions of programs are found (such as vendor make
-dnl or GNU make).
-dnl
-#
-# Get a test value and flag whether we should remove/replace the 
-# cache_system file (do so unless cache_system_ok is yes)
-# FC and F77 should be synonyms.  Save both in case
-# We include the xxxFLAGS in case the user is using the flags to change
-# the language (either input or output) of the compiler.  E.g., 
-# using -xarch=v9 on Solaris to select 64 bit output or using -D_BSD_SOURCE 
-# with gcc to get different header files on input.
-cleanargs=`echo "$CC $F77 $FC $CXX $F90 $CFLAGS $FFLAGS $CXXFLAGS $F90FLAGS $PATH" | tr '"' ' '`
-if uname -srm >/dev/null 2>&1 ; then
-    cache_system_text="`uname -srm` $cleanargs"
-else
-    cache_system_text="-no-uname- $cleanargs"
-fi
-cache_system_ok=no
-#
-if test -z "$real_enable_cache" ; then
-    real_enable_cache=$enable_cache
-    if test -z "$real_enable_cache" ; then real_enable_cache="notgiven" ; fi
-fi
-if test "X$real_enable_cache" = "Xnotgiven" ; then
-    # check for valid cache file
-    if test -z "$cache_system" ; then cache_system="config.system" ; fi
-    if uname -srm >/dev/null 2>&1 ; then
-        if test -f "$cache_system" -a -n "$cache_system_text" ; then
-	    if test "$cache_system_text" = "`cat $cache_system`" ; then
-	        real_enable_cache="yes"
-                cache_system_ok=yes
-	    fi
-        elif test ! -f "$cache_system" -a -n "$cache_system_text" ; then
-	    # remove the cache file because it may not correspond to our
-	    # system
-	    if test "$cache_file" != "/dev/null" ; then 
-	        rm -f $cache_file
-	    fi
-	    real_enable_cache="yes"
-        fi
-    fi
-fi
-if test "X$real_enable_cache" = "Xyes" -a "$cache_file" = "/dev/null" ; then
-    real_enable_cache=no
-fi
-if test "X$real_enable_cache" = "Xyes" ; then
-  if test -r "$cache_file" ; then
-    echo "loading cache $cache_file"
-    if test -w "$cache_file" ; then
-        # Clean the cache file (ergh)
-	PAC_CACHE_CLEAN
-    fi
-    . $cache_file
-  else
-    echo "Configure in `pwd` creating cache $cache_file"
-    > $cache_file
-    rm -f $cache_system
-  fi
-else
-  cache_file="/dev/null"
-fi
-# Remember our location and the name of the cachefile
-pac_cv_my_conf_dir=`pwd`
-dnl do not include the cachefile name, since this may contain the process
-dnl number and cause comparisons looking for changes to the cache file
-dnl to detect a change that isn't real.
-dnl pac_cv_my_cachefile=$cachefile
-#
-# Update the cache_system file if necessary
-if test "$cache_system_ok" != yes ; then
-    if test -n "$cache_system" ; then
-        rm -f $cache_system
-        echo $cache_system_text > $cache_system
-    fi
-fi
-if test "$clearMinusX" = yes ; then
-    set +x
-fi
-])
-dnl
-dnl/*D 
-dnl PAC_ARG_CACHING - Enable caching of results from a configure execution
-dnl
-dnl Synopsis:
-dnl PAC_ARG_CACHING
-dnl
-dnl Output Effects:
-dnl Adds '--enable-cache' and '--disable-cache' to the command line arguments
-dnl accepted by 'configure'.  
-dnl
-dnl See Also:
-dnl AC_CACHE_LOAD
-dnl D*/
-dnl Add this call to the other ARG_ENABLE calls.  Note that the values
-dnl set here are redundant; the LOAD_CACHE call relies on the way autoconf
-dnl initially processes ARG_ENABLE commands.
-AC_DEFUN([PAC_ARG_CACHING],[
-AC_ARG_ENABLE(cache,
-	AC_HELP_STRING([--enable-cache], [Turn on configure caching]),,
-	[enable_cache="notgiven"])
-])
-
-dnl Clean the cache of extraneous quotes that AC_CACHE_SAVE may add
-dnl
-AC_DEFUN([PAC_CACHE_CLEAN],[
-    rm -f confcache
-    sed -e "s/'\\\\''//g" -e "s/'\\\\/'/" -e "s/\\\\'/'/" \
-		-e "s/'\\\\''//g" $cache_file > confcache
-    if cmp -s $cache_file confcache ; then
-        :
-    else
-        if test -w $cache_file ; then
-	    echo "updating cache $cache_file"
-            cat confcache > $cache_file
-        else
-            echo "not updating unwritable cache $cache_file"
-        fi
-    fi	
-    rm -f confcache
-    if test "$DEBUG_AUTOCONF_CACHE" = "yes" ; then
-        echo "Results of cleaned cache file:"
-	echo "--------------------------------------------------------"
-	cat $cache_file
-	echo "--------------------------------------------------------"
-    fi
-])
-
-dnl/*D
-dnl PAC_SUBDIR_CACHE - Create a cache file before ac_output for subdirectory
-dnl configures.
-dnl 
-dnl Synopsis:
-dnl PAC_SUBDIR_CACHE(when)
-dnl
-dnl Input Parameter:
-dnl . when - Indicates when the cache should be created (optional)
-dnl          If 'always', create a new cache file.  This option
-dnl          should be used if any of the cache parameters (such as 
-dnl          CFLAGS or LDFLAGS) may have changed.
-dnl
-dnl Output Effects:
-dnl 	
-dnl Create a cache file before ac_output so that subdir configures don't
-dnl make mistakes. 
-dnl We can't use OUTPUT_COMMANDS to remove the cache file, because those
-dnl commands are executed *before* the subdir configures.
-dnl
-dnl D*/
-AC_DEFUN([PAC_SUBDIR_CACHE],[])
-AC_DEFUN([PAC_SUBDIR_CACHE_OLD],[
-if test "x$1" = "xalways" -o \( "$cache_file" = "/dev/null" -a "X$real_enable_cache" = "Xnotgiven" \) ; then
-    # Use an absolute directory to help keep the subdir configures from getting
-    # lost
-    cache_file=`pwd`/$$conf.cache
-    touch $cache_file
-    dnl 
-    dnl For Autoconf 2.52+, we should ensure that the environment is set
-    dnl for the cache.  Make sure that we get the values and set the 
-    dnl xxx_set variables properly
-    ac_cv_env_CC_set=set
-    ac_cv_env_CC_value=$CC
-    ac_cv_env_CFLAGS_set=${CFLAGS+set}
-    ac_cv_env_CFLAGS_value=$CFLAGS
-    ac_cv_env_CPP_set=set
-    ac_cv_env_CPP_value=$CPP
-    ac_cv_env_CPPFLAGS_set=${CPPFLAGS+set}
-    ac_cv_env_CPPFLAGS_value=$CPPFLAGS
-    ac_cv_env_LDFLAGS_set=${LDFLAGS+set}
-    ac_cv_env_LDFLAGS_value=$LDFLAGS
-    ac_cv_env_LIBS_set=${LIBS+set}
-    ac_cv_env_LIBS_value=$LIBS
-    ac_cv_env_FC_set=${FC+set}
-    ac_cv_env_FC_value=$FC
-    ac_cv_env_F77_set=${F77+set}
-    ac_cv_env_F77_value=$F77
-    ac_cv_env_F90_set=${F90+set}
-    ac_cv_env_F90_value=$F90
-    ac_cv_env_FFLAGS_set=${FFLAGS+set}
-    ac_cv_env_FFLAGS_value=$FFLAGS
-    ac_cv_env_CXX_set=${CXX+set}
-    ac_cv_env_CXX_value=$CXX
-
-    ac_env_CC_set=set
-    ac_env_CC_value=$CC
-    ac_env_CFLAGS_set=${CFLAGS+set}
-    ac_env_CFLAGS_value=$CFLAGS
-    ac_env_CPP_set=set
-    ac_env_CPP_value=$CPP
-    ac_env_CPPFLAGS_set=${CPPFLAGS+set}
-    ac_env_CPPFLAGS_value=$CPPFLAGS
-    ac_env_LDFLAGS_set=${LDFLAGS+set}
-    ac_env_LDFLAGS_value=$LDFLAGS
-    ac_env_LIBS_set=${LIBS+set}
-    ac_env_LIBS_value=$LIBS
-    ac_env_FC_set=${FC+set}
-    ac_env_FC_value=$FC
-    ac_env_F77_set=${F77+set}
-    ac_env_F77_value=$F77
-    ac_env_F90_set=${F90+set}
-    ac_env_F90_value=$F90
-    ac_env_FFLAGS_set=${FFLAGS+set}
-    ac_env_FFLAGS_value=$FFLAGS
-    ac_env_CXX_set=${CXX+set}
-    ac_env_CXX_value=$CXX
-
-    dnl other parameters are
-    dnl build_alias, host_alias, target_alias
-
-    # It turns out that A C CACHE_SAVE can't be invoked more than once
-    # with data that contains blanks.  What happens is that the quotes
-    # that it adds get quoted and then added again.  To avoid this,
-    # we strip off the outer quotes for all cached variables
-    dnl We add pac_cv_my_conf_dir to give the source of this cachefile,
-    dnl and pac_cv_my_cachefile to indicate how it chose the cachefile.
-    pac_cv_my_conf_dir=`pwd`
-    pac_cv_my_cachefile=$cachefile
-    AC_CACHE_SAVE
-    PAC_CACHE_CLEAN
-    ac_configure_args="$ac_configure_args -enable-cache"
-fi
-dnl Unconditionally export these values.  Subdir configures break otherwise
-export CC
-export CFLAGS
-export LDFLAGS
-export LIBS
-export CPPFLAGS
-export CPP
-export FC
-export F77
-export F90
-export CXX
-export FFLAGS
-export CCFLAGS
-])
-AC_DEFUN([PAC_SUBDIR_CACHE_CLEANUP],[])
-AC_DEFUN([PAC_SUBDIR_CACHE_CLEANUP_OLD],[
-if test "$cache_file" != "/dev/null" -a "X$real_enable_cache" = "Xnotgiven" ; then
-   rm -f $cache_file
-   cache_file=/dev/null
-fi
-])
-dnl
-dnl The following three macros support the sharing of configure results
-dnl by configure scripts, including ones that are not run with 
-dnl AC_CONFIG_SUBDIRS (the cachefiles managed by --enable-cache can
-dnl only be used with AC_CONFIG_SUBDIRS; creating a autoconf-style
-dnl cachefile before the the end of the autoconf process will often
-dnl cause problems.
-dnl
-AC_DEFUN([PAC_CREATE_BASE_CACHE],[
-AC_ARG_ENABLE(base-cache,
-	AC_HELP_STRING([--enable-base-cache],
-		[Enable the use of a simple cache for the subsidiary
-                 configure scripts]),,enable_base_cache=default)
-# The default case is controlled by the environment variable CONF_USE_CACHEFILE
-if test "$enable_base_cache" = "default" ; then
-    if test "$CONF_USE_CACHEFILE" = yes ; then
-        enable_base_cache=yes
-    else 
-        enable_base_cache=no
-    fi
-fi
-if test "$enable_base_cache" != no ; then
-    if test "$enable_base_cache" = yes ; then
-        basecachefile=`pwd`/cache.base
-    else
-        basecachefile=`pwd`/$enable_base_cache
-    fi
-    set | grep ac_cv > $basecachefile
-    # Tell other configures to load this file
-    echo "Creating and exporting the base cache file $basecachefile"
-    CONF_BASE_CACHEFILE=$basecachefile
-    export CONF_BASE_CACHEFILE
-fi
-])
-AC_DEFUN([PAC_LOAD_BASE_CACHE],[
-if test -n "$CONF_BASE_CACHEFILE" -a -s "$CONF_BASE_CACHEFILE" ; then
-    echo "Loading base cachefile $CONF_BASE_CACHEFILE"
-    . $CONF_BASE_CACHEFILE
-    export CONF_BASE_CACHEFILE
-fi
-])
-AC_DEFUN([PAC_UPDATE_BASE_CACHE],[
-if test -n "$CONF_BASE_CACHEFILE" -a -s "$CONF_BASE_CACHEFILE" ; then
-    set | grep ac_cv > $CONF_BASE_CACHEFILE.new
-    if cmp -s $CONF_BASE_CACHEFILE.new $CONF_BASE_CACHEFILE ; then
-	:
-    else
-	echo "Replacing $CONF_BASE_CACHEFILE"
-	mv $CONF_BASE_CACHEFILE.new $CONF_BASE_CACHEFILE
-    fi
-fi
-])
diff --git a/3rd-party/romio321/confdb/aclocal_cc.m4 b/3rd-party/romio321/confdb/aclocal_cc.m4
deleted file mode 100644
index 666d0a4c262..00000000000
--- a/3rd-party/romio321/confdb/aclocal_cc.m4
+++ /dev/null
@@ -1,1695 +0,0 @@
-dnl AC_PROG_CC_GNU
-ifdef([AC_PROG_CC_GNU],,[AC_DEFUN([AC_PROG_CC_GNU],)])
-
-dnl PAC_PROG_CC - reprioritize the C compiler search order
-AC_DEFUN([PAC_PROG_CC],[
-        dnl Many standard autoconf/automake/libtool macros, such as LT_INIT,
-        dnl perform an AC_REQUIRE([AC_PROG_CC]).  If this macro (PAC_PROG_CC)
-        dnl comes after LT_INIT (or similar) then the default compiler search
-        dnl path will be used instead.  This AC_BEFORE macro ensures that a
-        dnl warning will be emitted at autoconf-time (autogen.sh-time) to help
-        dnl developers notice this case.
-        AC_BEFORE([$0],[AC_PROG_CC])
-	PAC_PUSH_FLAG([CFLAGS])
-	AC_PROG_CC([icc pgcc xlc xlC pathcc gcc clang cc])
-	PAC_POP_FLAG([CFLAGS])
-])
-dnl
-dnl/*D
-dnl PAC_C_CHECK_COMPILER_OPTION - Check that a compiler option is accepted
-dnl without warning messages
-dnl
-dnl Synopsis:
-dnl PAC_C_CHECK_COMPILER_OPTION(optionname,action-if-ok,action-if-fail)
-dnl
-dnl Output Effects:
-dnl
-dnl If no actions are specified, a working value is added to 'COPTIONS'
-dnl
-dnl Notes:
-dnl This is now careful to check that the output is different, since 
-dnl some compilers are noisy.
-dnl 
-dnl We are extra careful to prototype the functions in case compiler options
-dnl that complain about poor code are in effect.
-dnl
-dnl Because this is a long script, we have ensured that you can pass a 
-dnl variable containing the option name as the first argument.
-dnl
-dnl D*/
-AC_DEFUN([PAC_C_CHECK_COMPILER_OPTION],[
-AC_MSG_CHECKING([whether C compiler accepts option $1])
-pac_opt="$1"
-AC_LANG_PUSH([C])
-CFLAGS_orig="$CFLAGS"
-CFLAGS_opt="$pac_opt $CFLAGS"
-pac_result="unknown"
-
-AC_LANG_CONFTEST([
-	AC_LANG_PROGRAM([[#include <stdio.h>
-                          const char hw[] = "Hello, World\n";]],
-		[[fputs (hw, stdout);]])
-])
-CFLAGS="$CFLAGS_orig"
-rm -f pac_test1.log
-PAC_LINK_IFELSE_LOG([pac_test1.log], [], [
-    CFLAGS="$CFLAGS_opt"
-    rm -f pac_test2.log
-    PAC_LINK_IFELSE_LOG([pac_test2.log], [], [
-        PAC_RUNLOG_IFELSE([diff -b pac_test1.log pac_test2.log],
-                          [pac_result=yes],[pac_result=no])
-    ],[
-        pac_result=no
-    ])
-], [
-    pac_result=no
-])
-AC_MSG_RESULT([$pac_result])
-dnl Delete the conftest created by AC_LANG_CONFTEST.
-rm -f conftest.$ac_ext
-
-# gcc 4.2.4 on 32-bit does not complain about the -Wno-type-limits option 
-# even though it doesn't support it.  However, when another warning is 
-# triggered, it gives an error that the option is not recognized.  So we 
-# need to test with a conftest file that will generate warnings.
-# 
-# add an extra switch, pac_c_check_compiler_option_prototest, to
-# disable this test just in case some new compiler does not like it.
-#
-# Linking with a program with an invalid prototype to ensure a compiler warning.
-
-if test "$pac_result" = "yes" \
-     -a "$pac_c_check_compiler_option_prototest" != "no" ; then
-    AC_MSG_CHECKING([whether C compiler option $1 works with an invalid prototype program])
-    AC_LINK_IFELSE([
-        dnl We want a warning, but we don't want to inadvertently disable
-        dnl special warnings like -Werror-implicit-function-declaration (e.g.,
-        dnl in PAC_CC_STRICT) by compiling something that might actually be
-        dnl treated as an error by the compiler.  So we try to elicit an
-        dnl "unused variable" warning and/or an "uninitialized" warning with the
-        dnl test program below.
-        dnl
-        dnl The old sanity program was:
-        dnl   void main() {return 0;}
-        dnl which clang (but not GCC) would treat as an *error*, invalidating
-        dnl the test for any given parameter.
-        AC_LANG_SOURCE([int main(int argc, char **argv){ int foo, bar = 0; foo += 1; return foo; }])
-    ],[pac_result=yes],[pac_result=no])
-    AC_MSG_RESULT([$pac_result])
-fi
-#
-if test "$pac_result" = "yes" ; then
-    AC_MSG_CHECKING([whether routines compiled with $pac_opt can be linked with ones compiled without $pac_opt])
-    pac_result=unknown
-    CFLAGS="$CFLAGS_orig"
-    rm -f pac_test3.log
-    PAC_COMPILE_IFELSE_LOG([pac_test3.log], [
-        AC_LANG_SOURCE([
-            int foo(void);
-            int foo(void){return 0;}
-        ])
-    ],[
-        PAC_RUNLOG([mv conftest.$OBJEXT pac_conftest.$OBJEXT])
-        saved_LIBS="$LIBS"
-        LIBS="pac_conftest.$OBJEXT $LIBS"
-
-        rm -f pac_test4.log
-        PAC_LINK_IFELSE_LOG([pac_test4.log], [AC_LANG_PROGRAM()], [
-            CFLAGS="$CFLAGS_opt"
-            rm -f pac_test5.log
-            PAC_LINK_IFELSE_LOG([pac_test5.log], [AC_LANG_PROGRAM()], [
-                PAC_RUNLOG_IFELSE([diff -b pac_test4.log pac_test5.log],
-                                  [pac_result=yes], [pac_result=no])
-            ],[
-                pac_result=no
-            ])
-        ],[
-            pac_result=no
-        ])
-        LIBS="$saved_LIBS"
-        rm -f pac_conftest.$OBJEXT
-    ],[
-        pac_result=no
-    ])
-    AC_MSG_RESULT([$pac_result])
-    rm -f pac_test3.log pac_test4.log pac_test5.log
-fi
-rm -f pac_test1.log pac_test2.log
-
-dnl Restore CFLAGS before 2nd/3rd argument commands are executed,
-dnl as 2nd/3rd argument command could be modifying CFLAGS.
-CFLAGS="$CFLAGS_orig"
-if test "$pac_result" = "yes" ; then
-     ifelse([$2],[],[COPTIONS="$COPTIONS $1"],[$2])
-else
-     ifelse([$3],[],[:],[$3])
-fi
-AC_LANG_POP([C])
-])
-dnl
-dnl/*D
-dnl PAC_C_OPTIMIZATION - Determine C options for producing optimized code
-dnl
-dnl Synopsis
-dnl PAC_C_OPTIMIZATION([action if found])
-dnl
-dnl Output Effect:
-dnl Adds options to 'COPTIONS' if no other action is specified
-dnl 
-dnl Notes:
-dnl This is a temporary standin for compiler optimization.
-dnl It should try to match known systems to known compilers (checking, of
-dnl course), and then falling back to some common defaults.
-dnl Note that many compilers will complain about -g and aggressive
-dnl optimization.  
-dnl D*/
-AC_DEFUN([PAC_C_OPTIMIZATION],[
-    for copt in "-O4 -Ofast" "-Ofast" "-fast" "-O3" "-xO3" "-O" ; do
-        PAC_C_CHECK_COMPILER_OPTION($copt,found_opt=yes,found_opt=no)
-        if test "$found_opt" = "yes" ; then
-	    ifelse($1,,COPTIONS="$COPTIONS $copt",$1)
-	    break
-        fi
-    done
-    if test "$ac_cv_prog_gcc" = "yes" ; then
-	for copt in "-fomit-frame-pointer" "-finline-functions" \
-		 "-funroll-loops" ; do
-	    PAC_C_CHECK_COMPILER_OPTION($copt,found_opt=yes,found_opt=no)
-	    if test "$found_opt" = "yes" ; then
-	        ifelse($1,,COPTIONS="$COPTIONS $copt",$1)
-	        # no break because we're trying to add them all
-	    fi
-	done
-	# We could also look for architecture-specific gcc options
-    fi
-
-])
-
-dnl/*D
-dnl PAC_PROG_C_UNALIGNED_DOUBLES - Check that the C compiler allows unaligned
-dnl doubles
-dnl
-dnl Synopsis:
-dnl   PAC_PROG_C_UNALIGNED_DOUBLES(action-if-true,action-if-false,
-dnl       action-if-unknown)
-dnl
-dnl Notes:
-dnl 'action-if-unknown' is used in the case of cross-compilation.
-dnl D*/
-AC_DEFUN([PAC_PROG_C_UNALIGNED_DOUBLES],[
-AC_CACHE_CHECK([whether C compiler allows unaligned doubles],
-pac_cv_prog_c_unaligned_doubles,[
-AC_TRY_RUN([
-void fetch_double( v )
-double *v;
-{
-*v = 1.0;
-}
-int main( argc, argv )
-int argc;
-char **argv;
-{
-int p[4];
-double *p_val;
-fetch_double( (double *)&(p[0]) );
-p_val = (double *)&(p[0]);
-if (*p_val != 1.0) return 1;
-fetch_double( (double *)&(p[1]) );
-p_val = (double *)&(p[1]);
-if (*p_val != 1.0) return 1;
-return 0;
-}
-],pac_cv_prog_c_unaligned_doubles="yes",pac_cv_prog_c_unaligned_doubles="no",
-pac_cv_prog_c_unaligned_doubles="unknown")])
-ifelse($1,,,if test "X$pac_cv_prog_c_unaligned_doubles" = "yes" ; then 
-$1
-fi)
-ifelse($2,,,if test "X$pac_cv_prog_c_unaligned_doubles" = "no" ; then 
-$2
-fi)
-ifelse($3,,,if test "X$pac_cv_prog_c_unaligned_doubles" = "unknown" ; then 
-$3
-fi)
-])
-
-dnl/*D 
-dnl PAC_PROG_C_WEAK_SYMBOLS - Test whether C supports weak alias symbols.
-dnl
-dnl Synopsis
-dnl PAC_PROG_C_WEAK_SYMBOLS(action-if-true,action-if-false)
-dnl
-dnl Output Effect:
-dnl Defines one of the following if a weak symbol pragma is found:
-dnl.vb
-dnl    HAVE_PRAGMA_WEAK - #pragma weak
-dnl    HAVE_PRAGMA_HP_SEC_DEF - #pragma _HP_SECONDARY_DEF
-dnl    HAVE_PRAGMA_CRI_DUP  - #pragma _CRI duplicate x as y
-dnl.ve
-dnl May also define
-dnl.vb
-dnl    HAVE_WEAK_ATTRIBUTE
-dnl.ve
-dnl if functions can be declared as 'int foo(...) __attribute__ ((weak));'
-dnl sets the shell variable pac_cv_attr_weak to yes.
-dnl Also checks for __attribute__((weak_import)) which is supported by
-dnl Apple in Mac OSX (at least in Darwin).  Note that this provides only
-dnl weak symbols, not weak aliases
-dnl 
-dnl D*/
-AC_DEFUN([PAC_PROG_C_WEAK_SYMBOLS],[
-pragma_extra_message=""
-AC_CACHE_CHECK([for type of weak symbol alias support],
-pac_cv_prog_c_weak_symbols,[
-# Test for weak symbol support...
-# We can't put # in the message because it causes autoconf to generate
-# incorrect code
-AC_TRY_LINK([
-extern int PFoo(int);
-#pragma weak PFoo = Foo
-int Foo(int a) { return a; }
-],[return PFoo(1);],has_pragma_weak=yes)
-#
-# Some systems (Linux ia64 and ecc, for example), support weak symbols
-# only within a single object file!  This tests that case.
-# Note that there is an extern int PFoo declaration before the
-# pragma.  Some compilers require this in order to make the weak symbol
-# externally visible.  
-if test "$has_pragma_weak" = yes ; then
-    PAC_COMPLINK_IFELSE([
-        AC_LANG_SOURCE([
-extern int PFoo(int);
-#pragma weak PFoo = Foo
-int Foo(int);
-int Foo(int a) { return a; }
-        ])
-    ],[
-        AC_LANG_SOURCE([
-extern int PFoo(int);
-int main(int argc, char **argv) {
-return PFoo(0);}
-        ])
-    ],[
-        PAC_COMPLINK_IFELSE([
-            AC_LANG_SOURCE([
-extern int PFoo(int);
-#pragma weak PFoo = Foo
-int Foo(int);
-int Foo(int a) { return a; }
-            ])
-        ],[
-            AC_LANG_SOURCE([
-extern int Foo(int);
-int PFoo(int a) { return a+1;}
-int main(int argc, char **argv) {
-return Foo(0);}
-            ])
-        ],[
-            pac_cv_prog_c_weak_symbols="pragma weak"
-        ],[
-            has_pragma_weak=0
-            pragma_extra_message="pragma weak accepted but does not work (probably creates two non-weak entries)"
-        ])
-    ],[
-        has_pragma_weak=0
-        pragma_extra_message="pragma weak accepted but does not work (probably creates two non-weak entries)"
-    ])
-fi
-dnl
-if test -z "$pac_cv_prog_c_weak_symbols" ; then 
-    AC_TRY_LINK([
-extern int PFoo(int);
-#pragma _HP_SECONDARY_DEF Foo  PFoo
-int Foo(int a) { return a; }
-],[return PFoo(1);],pac_cv_prog_c_weak_symbols="pragma _HP_SECONDARY_DEF")
-fi
-dnl
-if test -z "$pac_cv_prog_c_weak_symbols" ; then
-    AC_TRY_LINK([
-extern int PFoo(int);
-#pragma _CRI duplicate PFoo as Foo
-int Foo(int a) { return a; }
-],[return PFoo(1);],pac_cv_prog_c_weak_symbols="pragma _CRI duplicate x as y")
-fi
-dnl
-if test -z "$pac_cv_prog_c_weak_symbols" ; then
-    pac_cv_prog_c_weak_symbols="no"
-fi
-dnl
-dnl If there is an extra explanatory message, echo it now so that it
-dnl doesn't interfere with the cache result value
-if test -n "$pragma_extra_message" ; then
-    echo $pragma_extra_message
-fi
-dnl
-])
-if test "$pac_cv_prog_c_weak_symbols" != "no" ; then
-    case "$pac_cv_prog_c_weak_symbols" in
-        "pragma weak") AC_DEFINE(HAVE_PRAGMA_WEAK,1,[Supports weak pragma])
-        ;;
-        "pragma _HP")  AC_DEFINE(HAVE_PRAGMA_HP_SEC_DEF,1,[HP style weak pragma])
-        ;;
-        "pragma _CRI") AC_DEFINE(HAVE_PRAGMA_CRI_DUP,1,[Cray style weak pragma])
-        ;;
-    esac
-fi
-AC_CACHE_CHECK([whether __attribute__ ((weak)) allowed],
-pac_cv_attr_weak,[
-AC_TRY_COMPILE([int foo(int) __attribute__ ((weak));],[int a;],
-pac_cv_attr_weak=yes,pac_cv_attr_weak=no)])
-# Note that being able to compile with weak_import doesn't mean that
-# it works.
-AC_CACHE_CHECK([whether __attribute__ ((weak_import)) allowed],
-pac_cv_attr_weak_import,[
-AC_TRY_COMPILE([int foo(int) __attribute__ ((weak_import));],[int a;],
-pac_cv_attr_weak_import=yes,pac_cv_attr_weak_import=no)])
-# Check if the alias option for weak attributes is allowed
-AC_CACHE_CHECK([whether __attribute__((weak,alias(...))) allowed],
-pac_cv_attr_weak_alias,[
-PAC_PUSH_FLAG([CFLAGS])
-# force an error exit if the weak attribute isn't understood
-CFLAGS=-Werror
-AC_TRY_COMPILE([int __foo(int a){return 0;} int foo(int) __attribute__((weak,alias("__foo")));],[int a;],
-pac_cv_attr_weak_alias=yes,pac_cv_attr_weak_alias=no)
-# Restore original CFLAGS
-PAC_POP_FLAG([CFLAGS])])
-if test "$pac_cv_attr_weak_alias" = "yes" ; then
-    AC_DEFINE(HAVE_WEAK_ATTRIBUTE,1,[Attribute style weak pragma])
-fi
-if test "$pac_cv_prog_c_weak_symbols" = "no" -a "$pac_cv_attr_weak_alias" = "no" ; then
-    ifelse([$2],,:,[$2])
-else
-    ifelse([$1],,:,[$1])
-fi
-])
-
-#
-# This is a replacement that checks that FAILURES are signaled as well
-# (later configure macros look for the .o file, not just success from the
-# compiler, but they should not HAVE to
-#
-dnl --- insert 2.52 compatibility here ---
-dnl 2.52 does not have AC_PROG_CC_WORKS
-ifdef([AC_PROG_CC_WORKS],,[AC_DEFUN([AC_PROG_CC_WORKS],)])
-dnl
-AC_DEFUN([PAC_PROG_CC_WORKS],
-[AC_PROG_CC_WORKS
-AC_MSG_CHECKING([whether the C compiler sets its return status correctly])
-AC_LANG_SAVE
-AC_LANG_C
-AC_TRY_COMPILE(,[int a = bzzzt;],notbroken=no,notbroken=yes)
-AC_MSG_RESULT($notbroken)
-if test "$notbroken" = "no" ; then
-    AC_MSG_ERROR([installation or configuration problem: C compiler does not
-correctly set error code when a fatal error occurs])
-fi
-])
-
-dnl/*D 
-dnl PAC_PROG_C_MULTIPLE_WEAK_SYMBOLS - Test whether C and the
-dnl linker allow multiple weak symbols.
-dnl
-dnl Synopsis
-dnl PAC_PROG_C_MULTIPLE_WEAK_SYMBOLS(action-if-true,action-if-false)
-dnl
-dnl 
-dnl D*/
-AC_DEFUN([PAC_PROG_C_MULTIPLE_WEAK_SYMBOLS],[
-AC_CACHE_CHECK([for multiple weak symbol support],
-pac_cv_prog_c_multiple_weak_symbols,[
-# Test for multiple weak symbol support...
-PAC_COMPLINK_IFELSE([
-    AC_LANG_SOURCE([
-extern int PFoo(int);
-extern int PFoo_(int);
-extern int pfoo_(int);
-#pragma weak PFoo = Foo
-#pragma weak PFoo_ = Foo
-#pragma weak pfoo_ = Foo
-int Foo(int);
-int Foo(a) { return a; }
-    ])
-],[
-    AC_LANG_SOURCE([
-extern int PFoo(int), PFoo_(int), pfoo_(int);
-int main() {
-return PFoo(0) + PFoo_(1) + pfoo_(2);}
-    ])
-],[
-    pac_cv_prog_c_multiple_weak_symbols="yes"
-])
-dnl
-])
-if test "$pac_cv_prog_c_multiple_weak_symbols" = "yes" ; then
-    ifelse([$1],,:,[$1])
-else
-    ifelse([$2],,:,[$2])
-fi
-])
-
-dnl Use the value of enable-strict to update CFLAGS
-dnl pac_cc_strict_flags contains the strict flags.
-dnl
-dnl -std=c89 is used to select the C89 version of the ANSI/ISO C standard.
-dnl As of this writing, many C compilers still accepted only this version,
-dnl not the later C99 version. When all compilers accept C99, this 
-dnl should be changed to the appropriate standard level.  Note that we've
-dnl had trouble with gcc 2.95.3 accepting -std=c89 but then trying to 
-dnl compile program with a invalid set of options 
-dnl (-D __STRICT_ANSI__-trigraphs)
-AC_DEFUN([PAC_CC_STRICT],[
-export enable_strict_done
-if test "$enable_strict_done" != "yes" ; then
-
-    # Some comments on strict warning options.
-    # These were added to improve portability
-    #   -Wstack-usage=262144 -- 32 bit FreeBSD did not like the mprobe test
-    #       allocating a big variable on the stack. (See tt#2160).  The "right"
-    #       value requires further investigation; 1 MiB would have at least
-    #       caught #2160 at compile-time, and only two other tests show a
-    #       warning at 256k.
-    #
-    # These were added to reduce warnings:
-    #   -Wno-missing-field-initializers  -- We want to allow a struct to be 
-    #       initialized to zero using "struct x y = {0};" and not require 
-    #       each field to be initialized individually.
-    #   -Wno-unused-parameter -- For portability, some parameters go unused
-    #	    when we have different implementations of functions for 
-    #	    different platforms
-    #   -Wno-unused-label -- We add fn_exit: and fn_fail: on all functions, 
-    #	    but fn_fail may not be used if the function doesn't return an 
-    #	    error.
-    #   -Wno-sign-compare -- read() and write() return bytes read/written
-    #       as a signed value, but we often compare this to size_t (or
-    #	    msg_sz_t) variables.
-    #   -Wno-format-zero-length -- this warning is irritating and useless, since
-    #                              a zero-length format string is very well defined
-    # These were removed to reduce warnings:
-    #   -Wcast-qual -- Sometimes we need to cast "volatile char*" to 
-    #	    "char*", e.g., for memcpy.
-    #   -Wpadded -- We catch struct padding with asserts when we need to
-    #   -Wredundant-decls -- Having redundant declarations is benign and the 
-    #	    code already has some.
-    #   -Waggregate-return -- This seems to be a performance-related warning
-    #       aggregate return values are legal in ANSI C, but they may be returned
-    #	    in memory rather than through a register.  We do use aggregate return
-    #	    values, but they are structs of a single basic type (used to enforce
-    #	    type checking for relative vs. absolute ptrs), and with optimization
-    #	    the aggregate value is converted to a scalar.
-    #   -Wdeclaration-after-statement -- This is a C89
-    #       requirement. When compiling with C99, this should be
-    #       disabled.
-    #   -Wfloat-equal -- There are places in hwloc that set a float var to 0, then 
-    #       compare it to 0 later to see if it was updated.  Also when using strtod()
-    #       one needs to compare the return value with 0 to see whether a conversion
-    #       was performed.
-    #   -Werror-implicit-function-declaration -- implicit function declarations
-    #       should never be tolerated.  This also ensures that we get quick
-    #       compilation failures rather than later link failures that usually
-    #       come from a function name typo.
-    #   -Wcast-align -- Casting alignment warnings.  This is an
-    #       important check, but is temporarily disabled, since it is
-    #       throwing too many (correct) warnings currently, causing us
-    #       to miss other warnings.
-    #   -Wshorten-64-to-32 -- Bad type-casting warnings.  This is an
-    #       important check, but is temporarily disabled, since it is
-    #       throwing too many (correct) warnings currently, causing us
-    #       to miss other warnings.
-    #
-    # This was removed because it masks important failures (see ticket #2094).
-    # However, since Intel compiler currently does not include -Wtype-limits
-    # in -Wextra, -Wtype-limits was added to handle warnings with the Intel
-    # compiler.
-    #   -Wno-type-limits -- There are places where we compare an unsigned to 
-    #	    a constant that happens to be zero e.g., if x is unsigned and 
-    #	    MIN_VAL is zero, we'd like to do "MPIU_Assert(x >= MIN_VAL);".
-    #       Note this option is not supported by gcc 4.2.  This needs to be added 
-    #	    after most other warning flags, so that we catch a gcc bug on 32-bit 
-    #	    that doesn't give a warning that this is unsupported, unless another
-    #	    warning is triggered, and then if gives an error.
-    # the embedded newlines in this string are safe because we evaluate each
-    # argument in the for-loop below and append them to the CFLAGS with a space
-    # as the separator instead
-    pac_common_strict_flags="
-        -Wall
-        -Wextra
-        -Wno-missing-field-initializers
-        -Wstrict-prototypes
-        -Wmissing-prototypes
-        -DGCC_WALL
-        -Wno-unused-parameter
-        -Wno-unused-label
-        -Wshadow
-        -Wmissing-declarations
-        -Wno-long-long
-        -Wundef
-        -Wno-endif-labels
-        -Wpointer-arith
-        -Wbad-function-cast
-        -Wwrite-strings
-        -Wno-sign-compare
-        -Wold-style-definition
-        -Wno-multichar
-        -Wno-deprecated-declarations
-        -Wpacked
-        -Wnested-externs
-        -Winvalid-pch
-        -Wno-pointer-sign
-        -Wvariadic-macros
-        -Wno-format-zero-length
-        -Wtype-limits
-        -Werror-implicit-function-declaration
-        -Wstack-usage=262144
-    "
-
-    enable_c89=no
-    enable_c99=yes
-    enable_posix=2001
-    enable_opt=yes
-    flags="`echo $1 | sed -e 's/:/ /g' -e 's/,/ /g'`"
-    for flag in ${flags}; do
-        case "$flag" in
-	     c89)
-		enable_strict_done="yes"
-		enable_c89=yes
-                enable_c99=no
-		;;
-	     c99)
-		enable_strict_done="yes"
-                enable_c89=no
-		enable_c99=yes
-		;;
-	     posix1995)
-		enable_strict_done="yes"
-		enable_posix=1995
-		;;
-	     posix|posix2001)
-		enable_strict_done="yes"
-		enable_posix=2001
-		;;
-	     posix2008)
-		enable_strict_done="yes"
-		enable_posix=2008
-		;;
-	     noposix)
-		enable_strict_done="yes"
-		enable_posix=no
-		;;
-	     opt)
-		enable_strict_done="yes"
-		enable_opt=yes
-		;;
-	     noopt)
-		enable_strict_done="yes"
-		enable_opt=no
-		;;
-	     all|yes)
-		enable_strict_done="yes"
-		enable_c99=yes
-		enable_posix=2001
-		enable_opt=yes
-	        ;;
-	     no)
-		# Accept and ignore this value
-		:
-		;;
-	     *)
-		if test -n "$flag" ; then
-		   AC_MSG_WARN([Unrecognized value for enable-strict:$flag])
-		fi
-		;;
-	esac
-    done
-
-    pac_cc_strict_flags=""
-    if test "${enable_strict_done}" = "yes" ; then
-       if test "${enable_opt}" = "yes" ; then
-       	  pac_cc_strict_flags="-O2"
-       fi
-       pac_cc_strict_flags="$pac_cc_strict_flags $pac_common_strict_flags"
-       # We only allow one of strict-C99 or strict-C89 to be
-       # enabled. If C99 is enabled, we automatically disable C89.
-       if test "${enable_c99}" = "yes" ; then
-       	  PAC_APPEND_FLAG([-std=c99],[pac_cc_strict_flags])
-          # Use -D_STDC_C99= for Solaris compilers. See
-          # http://lists.gnu.org/archive/html/autoconf/2010-12/msg00059.html
-          # for discussion on why not to use -xc99
-          PAC_APPEND_FLAG([-D_STDC_C99=],[pac_cc_strict_flags])
-       elif test "${enable_c89}" = "yes" ; then
-       	  PAC_APPEND_FLAG([-std=c89],[pac_cc_strict_flags])
-       	  PAC_APPEND_FLAG([-Wdeclaration-after-statement],[pac_cc_strict_flags])
-       fi
-       # POSIX 2001 should be used with C99. But the default standard for some
-       # compilers are not C99. We must test the support of POSIX 2001 after
-       # testing C99.
-       case "$enable_posix" in
-            no)   : ;;
-            1995) PAC_APPEND_FLAG([-D_POSIX_C_SOURCE=199506L],[pac_cc_strict_flags]) ;;
-            2001) PAC_APPEND_FLAG([-D_POSIX_C_SOURCE=200112L],[pac_cc_strict_flags]) ;;
-            2008) PAC_APPEND_FLAG([-D_POSIX_C_SOURCE=200809L],[pac_cc_strict_flags]) ;;
-            *)    AC_MSG_ERROR([internal error, unexpected POSIX version: '$enable_posix']) ;;
-       esac
-    fi
-
-    # See if the above options work with the compiler
-    accepted_flags=""
-    for flag in $pac_cc_strict_flags ; do
-        PAC_PUSH_FLAG([CFLAGS])
-	CFLAGS="$CFLAGS $accepted_flags"
-        PAC_C_CHECK_COMPILER_OPTION([$flag],[accepted_flags="$accepted_flags $flag"],)
-        PAC_POP_FLAG([CFLAGS])
-    done
-    pac_cc_strict_flags=$accepted_flags
-fi
-])
-
-dnl/*D
-dnl PAC_ARG_STRICT - Add --enable-strict to configure.  
-dnl
-dnl Synopsis:
-dnl PAC_ARG_STRICT
-dnl 
-dnl Output effects:
-dnl Adds '--enable-strict' to the command line.
-dnl
-dnl D*/
-AC_DEFUN([PAC_ARG_STRICT],[
-AC_ARG_ENABLE(strict,
-	AC_HELP_STRING([--enable-strict], [Turn on strict compilation testing]))
-PAC_CC_STRICT($enable_strict)
-CFLAGS="$CFLAGS $pac_cc_strict_flags"
-export CFLAGS
-])
-
-dnl Return the integer structure alignment in pac_cv_c_max_integer_align
-dnl Possible values include
-dnl	packed
-dnl	two
-dnl	four
-dnl	eight
-dnl
-dnl In addition, a "Could not determine alignment" and a "error!"
-dnl return is possible.  
-AC_DEFUN([PAC_C_MAX_INTEGER_ALIGN],[
-AC_CACHE_CHECK([for max C struct integer alignment],
-pac_cv_c_max_integer_align,[
-AC_TRY_RUN([
-#include <stdio.h>
-#define DBG(a,b,c)
-int main( int argc, char *argv[] )
-{
-    FILE *cf;
-    int is_packed  = 1;
-    int is_two     = 1;
-    int is_four    = 1;
-    int is_eight   = 1;
-    struct { char a; int b; } char_int;
-    struct { char a; short b; } char_short;
-    struct { char a; long b; } char_long;
-    struct { char a; int b; char c; } char_int_char;
-    struct { char a; short b; char c; } char_short_char;
-#ifdef HAVE_LONG_LONG_INT
-    struct { long long int a; char b; } lli_c;
-    struct { char a; long long int b; } c_lli;
-#endif
-    int size, extent, extent2;
-
-    /* assume max integer alignment isn't 8 if we don't have
-     * an eight-byte value :)
-     */
-#ifdef HAVE_LONG_LONG_INT
-    if (sizeof(int) < 8 && sizeof(long) < 8 && sizeof(long long int) < 8)
-	is_eight = 0;
-#else
-    if (sizeof(int) < 8 && sizeof(long) < 8) is_eight = 0;
-#endif
-
-    size = sizeof(char) + sizeof(int);
-    extent = sizeof(char_int);
-    if (size != extent) is_packed = 0;
-    if ( (extent % 2) != 0) is_two = 0;
-    if ( (extent % 4) != 0) is_four = 0;
-    if (sizeof(int) == 8 && (extent % 8) != 0) is_eight = 0;
-    DBG("char_int",size,extent);
-
-    size = sizeof(char) + sizeof(short);
-    extent = sizeof(char_short);
-    if (size != extent) is_packed = 0;
-    if ( (extent % 2) != 0) is_two = 0;
-    if (sizeof(short) == 4 && (extent % 4) != 0) is_four = 0;
-    if (sizeof(short) == 8 && (extent % 8) != 0) is_eight = 0;
-    DBG("char_short",size,extent);
-
-    size = sizeof(char) + sizeof(long);
-    extent = sizeof(char_long);
-    if (size != extent) is_packed = 0;
-    if ( (extent % 2) != 0) is_two = 0;
-    if ( (extent % 4) != 0) is_four = 0;
-    if (sizeof(long) == 8 && (extent % 8) != 0) is_eight = 0;
-    DBG("char_long",size,extent);
-
-#ifdef HAVE_LONG_LONG_INT
-    size = sizeof(char) + sizeof(long long int);
-    extent = sizeof(lli_c);
-    extent2 = sizeof(c_lli);
-    if (size != extent) is_packed = 0;
-    if ( (extent % 2) != 0 && (extent2 % 2) != 0) is_two = 0;
-    if ( (extent % 4) != 0 && (extent2 % 4) != 0) is_four = 0;
-    if (sizeof(long long int) >= 8 && (extent % 8) != 0 && (extent2 % 8) != 0)
-	is_eight = 0;
-#endif
-
-    size = sizeof(char) + sizeof(int) + sizeof(char);
-    extent = sizeof(char_int_char);
-    if (size != extent) is_packed = 0;
-    if ( (extent % 2) != 0) is_two = 0;
-    if ( (extent % 4) != 0) is_four = 0;
-    if (sizeof(int) == 8 && (extent % 8) != 0) is_eight = 0;
-    DBG("char_int_char",size,extent);
-
-    size = sizeof(char) + sizeof(short) + sizeof(char);
-    extent = sizeof(char_short_char);
-    if (size != extent) is_packed = 0;
-    if ( (extent % 2) != 0) is_two = 0;
-    if (sizeof(short) == 4 && (extent % 4) != 0) is_four = 0;
-    if (sizeof(short) == 8 && (extent % 8) != 0) is_eight = 0;
-    DBG("char_short_char",size,extent);
-
-    /* If aligned mod 8, it will be aligned mod 4 */
-    if (is_eight) { is_four = 0; is_two = 0; }
-
-    if (is_four) is_two = 0;
-
-    /* Tabulate the results */
-    cf = fopen( "ctest.out", "w" );
-    if (is_packed + is_two + is_four + is_eight == 0) {
-	fprintf( cf, "Could not determine alignment\n" );
-    }
-    else {
-	if (is_packed + is_two + is_four + is_eight != 1) {
-	    fprintf( cf, "error!\n" );
-	}
-	else {
-	    if (is_packed) fprintf( cf, "packed\n" );
-	    if (is_two) fprintf( cf, "two\n" );
-	    if (is_four) fprintf( cf, "four\n" );
-	    if (is_eight) fprintf( cf, "eight\n" );
-	}
-    }
-    fclose( cf );
-    return 0;
-}],
-pac_cv_c_max_integer_align=`cat ctest.out`,
-pac_cv_c_max_integer_align="unknown",
-pac_cv_c_max_integer_align="$CROSS_ALIGN_STRUCT_INT")
-rm -f ctest.out
-])
-if test -z "$pac_cv_c_max_integer_align" ; then
-    pac_cv_c_max_integer_align="unknown"
-fi
-])
-
-dnl Return the floating point structure alignment in
-dnl pac_cv_c_max_fp_align.
-dnl
-dnl Possible values include:
-dnl	packed
-dnl	two
-dnl	four
-dnl	eight
-dnl     sixteen
-dnl
-dnl In addition, a "Could not determine alignment" and a "error!"
-dnl return is possible.  
-AC_DEFUN([PAC_C_MAX_FP_ALIGN],[
-AC_CACHE_CHECK([for max C struct floating point alignment],
-pac_cv_c_max_fp_align,[
-AC_TRY_RUN([
-#include <stdio.h>
-#define DBG(a,b,c)
-int main( int argc, char *argv[] )
-{
-    FILE *cf;
-    int is_packed  = 1;
-    int is_two     = 1;
-    int is_four    = 1;
-    int is_eight   = 1;
-    int is_sixteen = 1;
-    struct { char a; float b; } char_float;
-    struct { float b; char a; } float_char;
-    struct { char a; double b; } char_double;
-    struct { double b; char a; } double_char;
-#ifdef HAVE_LONG_DOUBLE
-    struct { char a; long double b; } char_long_double;
-    struct { long double b; char a; } long_double_char;
-    struct { long double a; int b; char c; } long_double_int_char;
-#endif
-    int size, extent1, extent2;
-
-    size = sizeof(char) + sizeof(float);
-    extent1 = sizeof(char_float);
-    extent2 = sizeof(float_char);
-    if (size != extent1) is_packed = 0;
-    if ( (extent1 % 2) != 0 && (extent2 % 2) != 0) is_two = 0;
-    if ( (extent1 % 4) != 0 && (extent2 % 4) != 0) is_four = 0;
-    if (sizeof(float) == 8 && (extent1 % 8) != 0 && (extent2 % 8) != 0)
-	is_eight = 0;
-    DBG("char_float",size,extent1);
-
-    size = sizeof(char) + sizeof(double);
-    extent1 = sizeof(char_double);
-    extent2 = sizeof(double_char);
-    if (size != extent1) is_packed = 0;
-    if ( (extent1 % 2) != 0 && (extent2 % 2) != 0) is_two = 0;
-    if ( (extent1 % 4) != 0 && (extent2 % 4) != 0) is_four = 0;
-    if (sizeof(double) == 8 && (extent1 % 8) != 0 && (extent2 % 8) != 0)
-	is_eight = 0;
-    DBG("char_double",size,extent1);
-
-#ifdef HAVE_LONG_DOUBLE
-    size = sizeof(char) + sizeof(long double);
-    extent1 = sizeof(char_long_double);
-    extent2 = sizeof(long_double_char);
-    if (size != extent1) is_packed = 0;
-    if ( (extent1 % 2) != 0 && (extent2 % 2) != 0) is_two = 0;
-    if ( (extent1 % 4) != 0 && (extent2 % 4) != 0) is_four = 0;
-    if (sizeof(long double) >= 8 && (extent1 % 8) != 0 && (extent2 % 8) != 0)
-	is_eight = 0;
-    if (sizeof(long double) > 8 && (extent1 % 16) != 0
-	&& (extent2 % 16) != 0) is_sixteen = 0;
-    DBG("char_long-double",size,extent1);
-
-    extent1 = sizeof(long_double_int_char);
-    if ( (extent1 % 2) != 0) is_two = 0;
-    if ( (extent1 % 4) != 0) is_four = 0;
-    if (sizeof(long double) >= 8 && (extent1 % 8) != 0)	is_eight = 0;
-    if (sizeof(long double) > 8 && (extent1 % 16) != 0) is_sixteen = 0;
-#else
-    is_sixteen = 0;
-#endif
-
-    if (is_sixteen) { is_eight = 0; is_four = 0; is_two = 0; }
-
-    if (is_eight) { is_four = 0; is_two = 0; }
-
-    if (is_four) is_two = 0;
-
-    /* Tabulate the results */
-    cf = fopen( "ctest.out", "w" );
-    if (is_packed + is_two + is_four + is_eight + is_sixteen == 0) {
-	fprintf( cf, "Could not determine alignment\n" );
-    }
-    else {
-	if (is_packed + is_two + is_four + is_eight + is_sixteen != 1) {
-	    fprintf( cf, "error!\n" );
-	}
-	else {
-	    if (is_packed) fprintf( cf, "packed\n" );
-	    if (is_two) fprintf( cf, "two\n" );
-	    if (is_four) fprintf( cf, "four\n" );
-	    if (is_eight) fprintf( cf, "eight\n" );
-	    if (is_sixteen) fprintf( cf, "sixteen\n" );
-	}
-    }
-    fclose( cf );
-    return 0;
-}],
-pac_cv_c_max_fp_align=`cat ctest.out`,
-pac_cv_c_max_fp_align="unknown",
-pac_cv_c_max_fp_align="$CROSS_ALIGN_STRUCT_FP")
-rm -f ctest.out
-])
-if test -z "$pac_cv_c_max_fp_align" ; then
-    pac_cv_c_max_fp_align="unknown"
-fi
-])
-
-dnl Return the floating point structure alignment in
-dnl pac_cv_c_max_double_fp_align.
-dnl
-dnl Possible values include:
-dnl	packed
-dnl	two
-dnl	four
-dnl	eight
-dnl
-dnl In addition, a "Could not determine alignment" and a "error!"
-dnl return is possible.  
-AC_DEFUN([PAC_C_MAX_DOUBLE_FP_ALIGN],[
-AC_CACHE_CHECK([for max C struct alignment of structs with doubles],
-pac_cv_c_max_double_fp_align,[
-AC_TRY_RUN([
-#include <stdio.h>
-#define DBG(a,b,c)
-int main( int argc, char *argv[] )
-{
-    FILE *cf;
-    int is_packed  = 1;
-    int is_two     = 1;
-    int is_four    = 1;
-    int is_eight   = 1;
-    struct { char a; float b; } char_float;
-    struct { float b; char a; } float_char;
-    struct { char a; double b; } char_double;
-    struct { double b; char a; } double_char;
-    int size, extent1, extent2;
-
-    size = sizeof(char) + sizeof(float);
-    extent1 = sizeof(char_float);
-    extent2 = sizeof(float_char);
-    if (size != extent1) is_packed = 0;
-    if ( (extent1 % 2) != 0 && (extent2 % 2) != 0) is_two = 0;
-    if ( (extent1 % 4) != 0 && (extent2 % 4) != 0) is_four = 0;
-    if (sizeof(float) == 8 && (extent1 % 8) != 0 && (extent2 % 8) != 0)
-	is_eight = 0;
-    DBG("char_float",size,extent1);
-
-    size = sizeof(char) + sizeof(double);
-    extent1 = sizeof(char_double);
-    extent2 = sizeof(double_char);
-    if (size != extent1) is_packed = 0;
-    if ( (extent1 % 2) != 0 && (extent2 % 2) != 0) is_two = 0;
-    if ( (extent1 % 4) != 0 && (extent2 % 4) != 0) is_four = 0;
-    if (sizeof(double) == 8 && (extent1 % 8) != 0 && (extent2 % 8) != 0)
-	is_eight = 0;
-    DBG("char_double",size,extent1);
-
-    if (is_eight) { is_four = 0; is_two = 0; }
-
-    if (is_four) is_two = 0;
-
-    /* Tabulate the results */
-    cf = fopen( "ctest.out", "w" );
-    if (is_packed + is_two + is_four + is_eight == 0) {
-	fprintf( cf, "Could not determine alignment\n" );
-    }
-    else {
-	if (is_packed + is_two + is_four + is_eight != 1) {
-	    fprintf( cf, "error!\n" );
-	}
-	else {
-	    if (is_packed) fprintf( cf, "packed\n" );
-	    if (is_two) fprintf( cf, "two\n" );
-	    if (is_four) fprintf( cf, "four\n" );
-	    if (is_eight) fprintf( cf, "eight\n" );
-	}
-    }
-    fclose( cf );
-    return 0;
-}],
-pac_cv_c_max_double_fp_align=`cat ctest.out`,
-pac_cv_c_max_double_fp_align="unknown",
-pac_cv_c_max_double_fp_align="$CROSS_ALIGN_STRUCT_DOUBLE_FP")
-rm -f ctest.out
-])
-if test -z "$pac_cv_c_max_double_fp_align" ; then
-    pac_cv_c_max_double_fp_align="unknown"
-fi
-])
-AC_DEFUN([PAC_C_MAX_LONGDOUBLE_FP_ALIGN],[
-AC_CACHE_CHECK([for max C struct floating point alignment with long doubles],
-pac_cv_c_max_longdouble_fp_align,[
-AC_TRY_RUN([
-#include <stdio.h>
-#define DBG(a,b,c)
-int main( int argc, char *argv[] )
-{
-    FILE *cf;
-    int is_packed  = 1;
-    int is_two     = 1;
-    int is_four    = 1;
-    int is_eight   = 1;
-    int is_sixteen = 1;
-    struct { char a; long double b; } char_long_double;
-    struct { long double b; char a; } long_double_char;
-    struct { long double a; int b; char c; } long_double_int_char;
-    int size, extent1, extent2;
-
-    size = sizeof(char) + sizeof(long double);
-    extent1 = sizeof(char_long_double);
-    extent2 = sizeof(long_double_char);
-    if (size != extent1) is_packed = 0;
-    if ( (extent1 % 2) != 0 && (extent2 % 2) != 0) is_two = 0;
-    if ( (extent1 % 4) != 0 && (extent2 % 4) != 0) is_four = 0;
-    if (sizeof(long double) >= 8 && (extent1 % 8) != 0 && (extent2 % 8) != 0)
-	is_eight = 0;
-    if (sizeof(long double) > 8 && (extent1 % 16) != 0
-	&& (extent2 % 16) != 0) is_sixteen = 0;
-    DBG("char_long-double",size,extent1);
-
-    extent1 = sizeof(long_double_int_char);
-    if ( (extent1 % 2) != 0) is_two = 0;
-    if ( (extent1 % 4) != 0) is_four = 0;
-    if (sizeof(long double) >= 8 && (extent1 % 8) != 0)	is_eight = 0;
-    if (sizeof(long double) > 8 && (extent1 % 16) != 0) is_sixteen = 0;
-
-    if (is_sixteen) { is_eight = 0; is_four = 0; is_two = 0; }
-
-    if (is_eight) { is_four = 0; is_two = 0; }
-
-    if (is_four) is_two = 0;
-
-    /* Tabulate the results */
-    cf = fopen( "ctest.out", "w" );
-    if (is_packed + is_two + is_four + is_eight + is_sixteen == 0) {
-	fprintf( cf, "Could not determine alignment\n" );
-    }
-    else {
-	if (is_packed + is_two + is_four + is_eight + is_sixteen != 1) {
-	    fprintf( cf, "error!\n" );
-	}
-	else {
-	    if (is_packed) fprintf( cf, "packed\n" );
-	    if (is_two) fprintf( cf, "two\n" );
-	    if (is_four) fprintf( cf, "four\n" );
-	    if (is_eight) fprintf( cf, "eight\n" );
-	    if (is_sixteen) fprintf( cf, "sixteen\n" );
-	}
-    }
-    fclose( cf );
-    return 0;
-}],
-pac_cv_c_max_longdouble_fp_align=`cat ctest.out`,
-pac_cv_c_max_longdouble_fp_align="unknown",
-pac_cv_c_max_longdouble_fp_align="$CROSS_ALIGN_STRUCT_LONGDOUBLE_FP")
-rm -f ctest.out
-])
-if test -z "$pac_cv_c_max_longdouble_fp_align" ; then
-    pac_cv_c_max_longdouble_fp_align="unknown"
-fi
-])
-
-dnl Other tests assume that there is potentially a maximum alignment
-dnl and that if there is no maximum alignment, or a type is smaller than
-dnl that value, then we align on the size of the value, with the exception
-dnl of the "position-based alignment" rules we test for separately.
-dnl
-dnl It turns out that these assumptions have fallen short in at least one
-dnl case, on MacBook Pros, where doubles are aligned on 4-byte boundaries
-dnl even when long doubles are aligned on 16-byte boundaries. So this test
-dnl is here specifically to handle this case.
-dnl
-dnl Puts result in pac_cv_c_double_alignment_exception.
-dnl
-dnl Possible values currently include no and four.
-dnl
-AC_DEFUN([PAC_C_DOUBLE_ALIGNMENT_EXCEPTION],[
-AC_CACHE_CHECK([if double alignment breaks rules, find actual alignment],
-pac_cv_c_double_alignment_exception,[
-AC_TRY_RUN([
-#include <stdio.h>
-#define DBG(a,b,c)
-int main( int argc, char *argv[] )
-{
-    FILE *cf;
-    struct { char a; double b; } char_double;
-    struct { double b; char a; } double_char;
-    int extent1, extent2, align_4 = 0;
-
-    extent1 = sizeof(char_double);
-    extent2 = sizeof(double_char);
-
-    /* we're interested in the largest value, will let separate test
-     * deal with position-based issues.
-     */
-    if (extent1 < extent2) extent1 = extent2;
-    if ((sizeof(double) == 8) && (extent1 % 8) != 0) {
-       if (extent1 % 4 == 0) {
-#ifdef HAVE_MAX_FP_ALIGNMENT
-          if (HAVE_MAX_FP_ALIGNMENT >= 8) align_4 = 1;
-#else
-          align_4 = 1;
-#endif
-       }
-    }
-
-    cf = fopen( "ctest.out", "w" );
-
-    if (align_4) fprintf( cf, "four\n" );
-    else fprintf( cf, "no\n" );
-
-    fclose( cf );
-    return 0;
-}],
-pac_cv_c_double_alignment_exception=`cat ctest.out`,
-pac_cv_c_double_alignment_exception="unknown",
-pac_cv_c_double_alignment_exception="$CROSS_ALIGN_DOUBLE_EXCEPTION")
-rm -f ctest.out
-])
-if test -z "$pac_cv_c_double_alignment_exception" ; then
-    pac_cv_c_double_alignment_exception="unknown"
-fi
-])
-
-dnl Test for odd struct alignment rule that only applies max.
-dnl padding when double value is at front of type.
-dnl Puts result in pac_cv_c_double_pos_align.
-dnl
-dnl Search for "Power alignment mode" for more details.
-dnl
-dnl Possible values include yes, no, and unknown.
-dnl
-AC_DEFUN([PAC_C_DOUBLE_POS_ALIGN],[
-AC_CACHE_CHECK([if alignment of structs with doubles is based on position],
-pac_cv_c_double_pos_align,[
-AC_TRY_RUN([
-#include <stdio.h>
-#define DBG(a,b,c)
-int main( int argc, char *argv[] )
-{
-    FILE *cf;
-    int padding_varies_by_pos = 0;
-    struct { char a; double b; } char_double;
-    struct { double b; char a; } double_char;
-    int extent1, extent2;
-
-    extent1 = sizeof(char_double);
-    extent2 = sizeof(double_char);
-    if (extent1 != extent2) padding_varies_by_pos = 1;
-
-    cf = fopen( "ctest.out", "w" );
-    if (padding_varies_by_pos) fprintf( cf, "yes\n" );
-    else fprintf( cf, "no\n" );
-
-    fclose( cf );
-    return 0;
-}],
-pac_cv_c_double_pos_align=`cat ctest.out`,
-pac_cv_c_double_pos_align="unknown",
-pac_cv_c_double_pos_align="$CROSS_ALIGN_DOUBLE_POS")
-rm -f ctest.out
-])
-if test -z "$pac_cv_c_double_pos_align" ; then
-    pac_cv_c_double_pos_align="unknown"
-fi
-])
-
-dnl Test for odd struct alignment rule that only applies max.
-dnl padding when long long int value is at front of type.
-dnl Puts result in pac_cv_c_llint_pos_align.
-dnl
-dnl Search for "Power alignment mode" for more details.
-dnl
-dnl Possible values include yes, no, and unknown.
-dnl
-AC_DEFUN([PAC_C_LLINT_POS_ALIGN],[
-AC_CACHE_CHECK([if alignment of structs with long long ints is based on position],
-pac_cv_c_llint_pos_align,[
-AC_TRY_RUN([
-#include <stdio.h>
-#define DBG(a,b,c)
-int main( int argc, char *argv[] )
-{
-    FILE *cf;
-    int padding_varies_by_pos = 0;
-#ifdef HAVE_LONG_LONG_INT
-    struct { char a; long long int b; } char_llint;
-    struct { long long int b; char a; } llint_char;
-    int extent1, extent2;
-
-    extent1 = sizeof(char_llint);
-    extent2 = sizeof(llint_char);
-    if (extent1 != extent2) padding_varies_by_pos = 1;
-#endif
-
-    cf = fopen( "ctest.out", "w" );
-    if (padding_varies_by_pos) fprintf( cf, "yes\n" );
-    else fprintf( cf, "no\n" );
-
-    fclose( cf );
-    return 0;
-}],
-pac_cv_c_llint_pos_align=`cat ctest.out`,
-pac_cv_c_llint_pos_align="unknown",
-pac_cv_c_llint_pos_align="$CROSS_ALIGN_LLINT_POS")
-rm -f ctest.out
-])
-if test -z "$pac_cv_c_llint_pos_align" ; then
-    pac_cv_c_llint_pos_align="unknown"
-fi
-])
-
-dnl/*D
-dnl PAC_FUNC_NEEDS_DECL - Set NEEDS_<funcname>_DECL if a declaration is needed
-dnl
-dnl Synopsis:
-dnl PAC_FUNC_NEEDS_DECL(headerfiles,funcname)
-dnl
-dnl Output Effect:
-dnl Sets 'NEEDS_<funcname>_DECL' if 'funcname' is not declared by the 
-dnl headerfiles.
-dnl
-dnl Approach:
-dnl Attempt to assign library function to function pointer.  If the function
-dnl is not declared in a header, this will fail.  Use a non-static global so
-dnl the compiler does not warn about an unused variable.
-dnl
-dnl Simply calling the function is not enough because C89 compilers allow
-dnl calls to implicitly-defined functions.  Re-declaring a library function
-dnl with an incompatible prototype is also not sufficient because some
-dnl compilers (notably clang-3.2) only produce a warning in this case.
-dnl
-dnl D*/
-AC_DEFUN([PAC_FUNC_NEEDS_DECL],[
-AC_CACHE_CHECK([whether $2 needs a declaration],
-pac_cv_func_decl_$2,[
-AC_TRY_COMPILE([$1
-void (*fptr)(void) = (void(*)(void))$2;],[],
-pac_cv_func_decl_$2=no,pac_cv_func_decl_$2=yes)])
-if test "$pac_cv_func_decl_$2" = "yes" ; then
-changequote(<<,>>)dnl
-define(<<PAC_FUNC_NAME>>, translit(NEEDS_$2_DECL, [a-z *], [A-Z__]))dnl
-changequote([, ])dnl
-    AC_DEFINE_UNQUOTED(PAC_FUNC_NAME,1,[Define if $2 needs a declaration])
-undefine([PAC_FUNC_NAME])
-fi
-])
-
-dnl PAC_C_GNU_ATTRIBUTE - See if the GCC __attribute__ specifier is allow.
-dnl Use the following
-dnl #ifndef HAVE_GCC_ATTRIBUTE
-dnl #define __attribute__(a)
-dnl #endif
-dnl If *not*, define __attribute__(a) as null
-dnl
-dnl We start by requiring Gcc.  Some other compilers accept __attribute__
-dnl but generate warning messages, or have different interpretations 
-dnl (which seems to make __attribute__ just as bad as #pragma) 
-dnl For example, the Intel icc compiler accepts __attribute__ and
-dnl __attribute__((pure)) but generates warnings for __attribute__((format...))
-dnl
-AC_DEFUN([PAC_C_GNU_ATTRIBUTE],[
-AC_REQUIRE([AC_PROG_CC_GNU])
-if test "$ac_cv_prog_gcc" = "yes" ; then
-    AC_CACHE_CHECK([whether __attribute__ allowed],
-pac_cv_gnu_attr_pure,[
-AC_TRY_COMPILE([int foo(int) __attribute__ ((pure));],[int a;],
-pac_cv_gnu_attr_pure=yes,pac_cv_gnu_attr_pure=no)])
-AC_CACHE_CHECK([whether __attribute__((format)) allowed],
-pac_cv_gnu_attr_format,[
-AC_TRY_COMPILE([int foo(char *,...) __attribute__ ((format(printf,1,2)));],[int a;],
-pac_cv_gnu_attr_format=yes,pac_cv_gnu_attr_format=no)])
-    if test "$pac_cv_gnu_attr_pure" = "yes" -a "$pac_cv_gnu_attr_format" = "yes" ; then
-        AC_DEFINE(HAVE_GCC_ATTRIBUTE,1,[Define if GNU __attribute__ is supported])
-    fi
-fi
-])
-dnl
-dnl Check for a broken install (fails to preserve file modification times,
-dnl thus breaking libraries.
-dnl
-dnl Create a library, install it, and then try to link against it.
-AC_DEFUN([PAC_PROG_INSTALL_BREAKS_LIBS],[
-AC_CACHE_CHECK([whether install breaks libraries],
-ac_cv_prog_install_breaks_libs,[
-AC_REQUIRE([AC_PROG_RANLIB])
-AC_REQUIRE([AC_PROG_INSTALL])
-AC_REQUIRE([AC_PROG_CC])
-ac_cv_prog_install_breaks_libs=yes
-
-AC_COMPILE_IFELSE([
-    AC_LANG_SOURCE([ int foo(int); int foo(int a){return a;} ])
-],[
-    if ${AR-ar} ${AR_FLAGS-cr} libconftest.a conftest.$OBJEXT >/dev/null 2>&1 ; then
-        if ${RANLIB-:} libconftest.a >/dev/null 2>&1 ; then
-            # Anything less than sleep 10, and Mac OS/X (Darwin) 
-            # will claim that install works because ranlib won't complain
-            sleep 10
-            libinstall="$INSTALL_DATA"
-            eval "libinstall=\"$libinstall\""
-            if ${libinstall} libconftest.a libconftest1.a >/dev/null 2>&1 ; then
-                saved_LIBS="$LIBS"
-                LIBS="libconftest1.a"
-                AC_LINK_IFELSE([
-                    AC_LANG_SOURCE([
-extern int foo(int);
-int main(int argc, char **argv){ return foo(0); }
-                    ])
-                ],[
-                    # Success!  Install works
-                    ac_cv_prog_install_breaks_libs=no
-                ],[
-                    # Failure!  Does install -p work?        
-                    rm -f libconftest1.a
-                    if ${libinstall} -p libconftest.a libconftest1.a >/dev/null 2>&1 ; then
-                        AC_LINK_IFELSE([],[
-                            # Success!  Install works
-                            ac_cv_prog_install_breaks_libs="no, with -p"
-                        ])
-                    fi
-                ])
-                LIBS="$saved_LIBS"
-            fi
-        fi
-    fi
-])
-rm -f libconftest*.a
-]) dnl Endof ac_cache_check
-
-if test -z "$RANLIB_AFTER_INSTALL" ; then
-    RANLIB_AFTER_INSTALL=no
-fi
-case "$ac_cv_prog_install_breaks_libs" in
-    yes)
-        RANLIB_AFTER_INSTALL=yes
-    ;;
-    "no, with -p")
-        INSTALL_DATA="$INSTALL_DATA -p"
-    ;;
-    *)
-    # Do nothing
-    :
-    ;;
-esac
-AC_SUBST(RANLIB_AFTER_INSTALL)
-])
-
-#
-# determine if the compiler defines a symbol containing the function name
-#
-# These tests check not only that the compiler defines some symbol, such
-# as __FUNCTION__, but that the symbol correctly names the function.
-#
-# Defines 
-#   HAVE__FUNC__      (if __func__ defined)
-#   HAVE_CAP__FUNC__  (if __FUNC__ defined)
-#   HAVE__FUNCTION__  (if __FUNCTION__ defined)
-#
-AC_DEFUN([PAC_CC_FUNCTION_NAME_SYMBOL],[
-AC_CACHE_CHECK([whether the compiler defines __func__],
-pac_cv_have__func__,[
-tmp_am_cross=no
-AC_RUN_IFELSE([
-AC_LANG_SOURCE([
-#include <string.h>
-int foo(void);
-int foo(void)
-{
-    return (strcmp(__func__, "foo") == 0);
-}
-int main(int argc, char ** argv)
-{
-    return (foo() ? 0 : 1);
-}
-])
-], pac_cv_have__func__=yes, pac_cv_have__func__=no,tmp_am_cross=yes)
-if test "$tmp_am_cross" = yes ; then
-    AC_LINK_IFELSE([
-    AC_LANG_SOURCE([
-#include <string.h>
-int foo(void);
-int foo(void)
-{
-    return (strcmp(__func__, "foo") == 0);
-}
-int main(int argc, char ** argv)
-{
-    return (foo() ? 0 : 1);
-}
-    ])
-], pac_cv_have__func__=yes, pac_cv_have__func__=no)
-fi
-])
-
-if test "$pac_cv_have__func__" = "yes" ; then
-    AC_DEFINE(HAVE__FUNC__,,[define if the compiler defines __func__])
-fi
-
-AC_CACHE_CHECK([whether the compiler defines __FUNC__],
-pac_cv_have_cap__func__,[
-tmp_am_cross=no
-AC_RUN_IFELSE([
-AC_LANG_SOURCE([
-#include <string.h>
-int foo(void);
-int foo(void)
-{
-    return (strcmp(__FUNC__, "foo") == 0);
-}
-int main(int argc, char ** argv)
-{
-    return (foo() ? 0 : 1);
-}
-])
-], pac_cv_have_cap__func__=yes, pac_cv_have_cap__func__=no,tmp_am_cross=yes)
-if test "$tmp_am_cross" = yes ; then
-    AC_LINK_IFELSE([
-    AC_LANG_SOURCE([
-#include <string.h>
-int foo(void);
-int foo(void)
-{
-    return (strcmp(__FUNC__, "foo") == 0);
-}
-int main(int argc, char ** argv)
-{
-    return (foo() ? 0 : 1);
-}
-    ])
-], pac_cv_have__func__=yes, pac_cv_have__func__=no)
-fi
-])
-
-if test "$pac_cv_have_cap__func__" = "yes" ; then
-    AC_DEFINE(HAVE_CAP__FUNC__,,[define if the compiler defines __FUNC__])
-fi
-
-AC_CACHE_CHECK([whether the compiler sets __FUNCTION__],
-pac_cv_have__function__,[
-tmp_am_cross=no
-AC_RUN_IFELSE([
-AC_LANG_SOURCE([
-#include <string.h>
-int foo(void);
-int foo(void)
-{
-    return (strcmp(__FUNCTION__, "foo") == 0);
-}
-int main(int argc, char ** argv)
-{
-    return (foo() ? 0 : 1);
-}
-])
-], pac_cv_have__function__=yes, pac_cv_have__function__=no,tmp_am_cross=yes)
-if test "$tmp_am_cross" = yes ; then
-    AC_LINK_IFELSE([
-    AC_LANG_SOURCE([
-#include <string.h>
-int foo(void);
-int foo(void)
-{
-    return (strcmp(__FUNCTION__, "foo") == 0);
-}
-int main(int argc, char ** argv)
-{
-    return (foo() ? 0 : 1);
-}
-    ])
-], pac_cv_have__func__=yes, pac_cv_have__func__=no)
-fi
-])
-
-if test "$pac_cv_have__function__" = "yes" ; then
-    AC_DEFINE(HAVE__FUNCTION__,,[define if the compiler defines __FUNCTION__])
-fi
-
-])
-
-
-dnl Check structure alignment
-AC_DEFUN([PAC_STRUCT_ALIGNMENT],[
-	# Initialize alignment checks
-	is_packed=1
-	is_two=1
-	is_four=1
-	is_eight=1
-	is_largest=1
-
-	# See if long double exists
-	AC_TRY_COMPILE(,[long double a;],have_long_double=yes,have_long_double=no)
-
-	# Get sizes of regular types
-	AC_CHECK_SIZEOF(char)
-	AC_CHECK_SIZEOF(int)
-	AC_CHECK_SIZEOF(short)
-	AC_CHECK_SIZEOF(long)
-	AC_CHECK_SIZEOF(float)
-	AC_CHECK_SIZEOF(double)
-	AC_CHECK_SIZEOF(long double)
-
-	# char_int comparison
-	AC_CHECK_SIZEOF(char_int, 0, [typedef struct { char a; int b; } char_int; ])
-	size=`expr $ac_cv_sizeof_char + $ac_cv_sizeof_int`
-	extent=$ac_cv_sizeof_char_int
-	if test "$size" != "$extent" ; then is_packed=0 ; fi
-	if test "`expr $extent % $ac_cv_sizeof_int`" != "0" ; then is_largest=0 ; fi
-	if test "`expr $extent % 2`" != "0" ; then is_two=0 ; fi
-	if test "`expr $extent % 4`" != "0" ; then is_four=0 ; fi
-	if test "$ac_cv_sizeof_int" = "8" -a "`expr $extent % 8`" != "0" ; then
-	   is_eight=0
-	fi
-
-	# char_short comparison
-	AC_CHECK_SIZEOF(char_short, 0, [typedef struct { char a; short b; } char_short; ])
-	size=`expr $ac_cv_sizeof_char + $ac_cv_sizeof_short`
-	extent=$ac_cv_sizeof_char_short
-	if test "$size" != "$extent" ; then is_packed=0 ; fi
-	if test "`expr $extent % $ac_cv_sizeof_short`" != "0" ; then is_largest=0 ; fi
-	if test "`expr $extent % 2`" != "0" ; then is_two=0 ; fi
-	if test "$ac_cv_sizeof_short" = "4" -a "`expr $extent % 4`" != "0" ; then
-	   is_four=0
-	fi
-	if test "$ac_cv_sizeof_short" = "8" -a "`expr $extent % 8`" != "0" ; then
-	   is_eight=0
-	fi
-
-	# char_long comparison
-	AC_CHECK_SIZEOF(char_long, 0, [typedef struct { char a; long b; } char_long; ])
-	size=`expr $ac_cv_sizeof_char + $ac_cv_sizeof_long`
-	extent=$ac_cv_sizeof_char_long
-	if test "$size" != "$extent" ; then is_packed=0 ; fi
-	if test "`expr $extent % $ac_cv_sizeof_long`" != "0" ; then is_largest=0 ; fi
-	if test "`expr $extent % 2`" != "0" ; then is_two=0 ; fi
-	if test "`expr $extent % 4`" != "0" ; then is_four=0 ; fi
-	if test "$ac_cv_sizeof_long" = "8" -a "`expr $extent % 8`" != "0" ; then
-	   is_eight=0
-	fi
-
-	# char_float comparison
-	AC_CHECK_SIZEOF(char_float, 0, [typedef struct { char a; float b; } char_float; ])
-	size=`expr $ac_cv_sizeof_char + $ac_cv_sizeof_float`
-	extent=$ac_cv_sizeof_char_float
-	if test "$size" != "$extent" ; then is_packed=0 ; fi
-	if test "`expr $extent % $ac_cv_sizeof_float`" != "0" ; then is_largest=0 ; fi
-	if test "`expr $extent % 2`" != "0" ; then is_two=0 ; fi
-	if test "`expr $extent % 4`" != "0" ; then is_four=0 ; fi
-	if test "$ac_cv_sizeof_float" = "8" -a "`expr $extent % 8`" != "0" ; then
-	   is_eight=0
-	fi
-
-	# char_double comparison
-	AC_CHECK_SIZEOF(char_double, 0, [typedef struct { char a; double b; } char_double; ])
-	size=`expr $ac_cv_sizeof_char + $ac_cv_sizeof_double`
-	extent=$ac_cv_sizeof_char_double
-	if test "$size" != "$extent" ; then is_packed=0 ; fi
-	if test "`expr $extent % $ac_cv_sizeof_double`" != "0" ; then is_largest=0 ; fi
-	if test "`expr $extent % 2`" != "0" ; then is_two=0 ; fi
-	if test "`expr $extent % 4`" != "0" ; then is_four=0 ; fi
-	if test "$ac_cv_sizeof_double" = "8" -a "`expr $extent % 8`" != "0" ; then
-	   is_eight=0
-	fi
-
-	# char_long_double comparison
-	if test "$have_long_double" = "yes"; then
-	AC_CHECK_SIZEOF(char_long_double, 0, [
-				       typedef struct {
-				       	       char a;
-					       long double b;
-				       } char_long_double;
-				       ])
-	size=`expr $ac_cv_sizeof_char + $ac_cv_sizeof_long_double`
-	extent=$ac_cv_sizeof_char_long_double
-	if test "$size" != "$extent" ; then is_packed=0 ; fi
-	if test "`expr $extent % $ac_cv_sizeof_long_double`" != "0" ; then is_largest=0 ; fi
-	if test "`expr $extent % 2`" != "0" ; then is_two=0 ; fi
-	if test "`expr $extent % 4`" != "0" ; then is_four=0 ; fi
-	if test "$ac_cv_sizeof_long_double" = "8" -a "`expr $extent % 8`" != "0" ; then
-	   is_eight=0
-	fi
-	fi
-
-	# char_int_char comparison
-	AC_CHECK_SIZEOF(char_int_char, 0, [
-				       typedef struct {
-				       	       char a;
-					       int b;
-					       char c;
-				       } char_int_char;
-				       ])
-	size=`expr $ac_cv_sizeof_char + $ac_cv_sizeof_int + $ac_cv_sizeof_char`
-	extent=$ac_cv_sizeof_char_int_char
-	if test "$size" != "$extent" ; then is_packed=0 ; fi
-	if test "`expr $extent % $ac_cv_sizeof_int`" != "0" ; then is_largest=0 ; fi
-	if test "`expr $extent % 2`" != "0" ; then is_two=0 ; fi
-	if test "`expr $extent % 4`" != "0" ; then is_four=0 ; fi
-	if test "$ac_cv_sizeof_int" = "8" -a "`expr $extent % 8`" != "0" ; then
-	   is_eight=0
-	fi
-
-	# char_short_char comparison
-	AC_CHECK_SIZEOF(char_short_char, 0, [
-				       typedef struct {
-				       	       char a;
-					       short b;
-					       char c;
-				       } char_short_char;
-				       ])
-	size=`expr $ac_cv_sizeof_char + $ac_cv_sizeof_short + $ac_cv_sizeof_char`
-	extent=$ac_cv_sizeof_char_short_char
-	if test "$size" != "$extent" ; then is_packed=0 ; fi
-	if test "`expr $extent % $ac_cv_sizeof_short`" != "0" ; then is_largest=0 ; fi
-	if test "`expr $extent % 2`" != "0" ; then is_two=0 ; fi
-	if test "$ac_cv_sizeof_short" = "4" -a "`expr $extent % 4`" != "0" ; then
-	   is_four=0
-	fi
-	if test "$ac_cv_sizeof_short" = "8" -a "`expr $extent % 8`" != "0" ; then
-	   is_eight=0
-	fi
-
-	# If aligned mod 8, it will be aligned mod 4
-	if test $is_eight = 1 ; then is_four=0 ; is_two=0 ; fi
-	if test $is_four = 1 ; then is_two=0 ; fi
-
-	# Largest supersedes 8
-	if test $is_largest = 1 ; then is_eight=0 ; fi
-
-	# Find the alignment
-	if test "`expr $is_packed + $is_largest + $is_two + $is_four + $is_eight`" = "0" ; then
-	   pac_cv_struct_alignment="unknown"
-	elif test "`expr $is_packed + $is_largest + $is_two + $is_four + $is_eight`" != "1" ; then
-	   pac_cv_struct_alignment="unknown"
-	elif test $is_packed = 1 ; then
-	   pac_cv_struct_alignment="packed"
-	elif test $is_largest = 1 ; then
-	   pac_cv_struct_alignment="largest"
-	elif test $is_two = 1 ; then
-	   pac_cv_struct_alignment="two"
-	elif test $is_four = 1 ; then
-	   pac_cv_struct_alignment="four"
-	elif test $is_eight = 1 ; then
-	   pac_cv_struct_alignment="eight"
-	fi
-])
-dnl
-dnl PAC_C_MACRO_VA_ARGS
-dnl
-dnl will AC_DEFINE([HAVE_MACRO_VA_ARGS]) if the compiler supports C99 variable
-dnl length argument lists in macros (#define foo(...) bar(__VA_ARGS__))
-AC_DEFUN([PAC_C_MACRO_VA_ARGS],[
-    AC_MSG_CHECKING([for variable argument list macro functionality])
-    AC_LINK_IFELSE([AC_LANG_PROGRAM([
-        #include <stdio.h>
-        #define conftest_va_arg_macro(...) printf(__VA_ARGS__)
-    ],
-    [conftest_va_arg_macro("a test %d", 3);])],
-    [AC_DEFINE([HAVE_MACRO_VA_ARGS],[1],[Define if C99-style variable argument list macro functionality])
-     AC_MSG_RESULT([yes])],
-    [AC_MSG_RESULT([no])])
-])dnl
-
-# Will AC_DEFINE([HAVE_BUILTIN_EXPECT]) if the compiler supports __builtin_expect.
-AC_DEFUN([PAC_C_BUILTIN_EXPECT],[
-AC_MSG_CHECKING([if C compiler supports __builtin_expect])
-
-AC_TRY_LINK(, [
-    return __builtin_expect(1, 1) ? 1 : 0
-], [
-    have_builtin_expect=yes
-    AC_MSG_RESULT([yes])
-], [
-    have_builtin_expect=no
-    AC_MSG_RESULT([no])
-])
-if test x$have_builtin_expect = xyes ; then
-    AC_DEFINE([HAVE_BUILTIN_EXPECT], [1], [Define to 1 if the compiler supports __builtin_expect.])
-fi
-])
diff --git a/3rd-party/romio321/confdb/aclocal_coverage.m4 b/3rd-party/romio321/confdb/aclocal_coverage.m4
deleted file mode 100644
index ce949d40a65..00000000000
--- a/3rd-party/romio321/confdb/aclocal_coverage.m4
+++ /dev/null
@@ -1,90 +0,0 @@
-
-dnl Macro to add --enable-coverage option (disabled by default) and add
-dnl appropriate compiler flags to permit usage of gcov if that option is
-dnl enabled.  If WRAPPER_xFLAGS variables are set then the flags will also be
-dnl added to those variables.
-dnl
-dnl Sets "pac_cv_use_coverage=yes" and AC_DEFINEs USE_COVERAGE if coverage was
-dnl successfully enabled.  Also creates an AM_CONDITIONAL with the name
-dnl "BUILD_COVERAGE" that is true iff pac_cv_use_coverage=yes.
-dnl
-dnl Usage: PAC_CONFIG_SUBDIR_ARGS
-dnl
-dnl Assumes that all of the compiler macros have already been invoked
-dnl (AC_PROG_CC and friends).
-AC_DEFUN([PAC_ENABLE_COVERAGE],[
-
-AC_ARG_VAR([GCOV],[name/path for the gcov utility])
-AC_CHECK_PROGS([GCOV],[gcov])
-
-AC_ARG_ENABLE([coverage],
-              [AC_HELP_STRING([--enable-coverage],
-                              [Turn on coverage analysis using gcc and gcov])],
-              [],[enable_coverage=no])
-
-if test "$enable_coverage" = "yes" ; then
-    if test "$ac_cv_prog_gcc" = "yes" ; then
-        CFLAGS="$CFLAGS -fprofile-arcs -ftest-coverage"
-        LIBS="$LIBS -lgcov"
-        if test ${WRAPPER_CFLAGS+set} = set ; then
-            WRAPPER_CFLAGS="$WRAPPER_CFLAGS -fprofile-arcs -ftest-coverage"
-        fi
-    else
-        AC_MSG_WARN([--enable-coverage only supported for GCC])
-    fi
-    if test "$enable_cxx" = "yes" ; then
-        if test "$ac_cv_cxx_compiler_gnu" = "yes" ; then
-            CXXFLAGS="$CXXFLAGS -fprofile-arcs -ftest-coverage"
-            LIBS="$LIBS -lgcov"
-            if test ${WRAPPER_CXXFLAGS+set} = set ; then
-                WRAPPER_CXXFLAGS="$WRAPPER_CXXFLAGS -fprofile-arcs -ftest-coverage"
-            fi
-        else
-            AC_MSG_WARN([--enable-coverage only supported for GCC])
-        fi
-    fi
-    # Add similar options for g77 so that the Fortran tests will also
-    # 
-    if test "$enable_f77" = yes ; then
-        if test "$ac_cv_f77_compiler_gnu" = "yes" ; then
-             FFLAGS="$FFLAGS -fprofile-arcs -ftest-coverage"
-             LIBS="$LIBS -lgcov"
-             if test ${WRAPPER_FFLAGS+set} = set ; then
-                 WRAPPER_FFLAGS="$WRAPPER_FFLAGS -fprofile-arcs -ftest-coverage"
-             fi
-        else
-            AC_MSG_WARN([--enable-coverage only supported for G77/GFORTRAN])
-        fi
-    fi
-    if test "$enable_fc" = yes ; then
-        if test "$ac_cv_fc_compiler_gnu" = "yes" ; then
-             FCFLAGS="$FCFLAGS -fprofile-arcs -ftest-coverage"
-             LIBS="$LIBS -lgcov"
-             if test ${WRAPPER_FCFLAGS+set} = set ; then
-                 WRAPPER_FCFLAGS="$WRAPPER_FCFLAGS -fprofile-arcs -ftest-coverage"
-             fi
-        else
-            AC_MSG_WARN([--enable-coverage only supported for GFORTRAN])
-        fi
-    fi
-    # On some platforms (e.g., Mac Darwin), we must also *link* 
-    # with the -fprofile-args -ftest-coverage option.
-    AC_MSG_CHECKING([whether compilation with coverage analysis enabled works])
-    AC_LINK_IFELSE([AC_LANG_SOURCE([int main(int argc, char **argv){return 1;}])],
-                   [AC_MSG_RESULT([yes])],
-                   [AC_MSG_RESULT([no])
-                    AC_MSG_ERROR([Unable to link programs when coverage analysis enabled])])
-    
-    # Test for the routines that we need to use to ensure that the
-    # data files are (usually) written out
-    # FIXME: Some versions of Linux provide usleep, but it rounds times
-    # up to the next second (!)
-    AC_CHECK_FUNCS([usleep])
-
-    # NOTE: using a "pac_cv_" prefix but not caching because of xFLAGS "side effects"
-    pac_cv_use_coverage=yes
-    AC_DEFINE([USE_COVERAGE],[1],[Define if performing coverage tests])
-fi
-AM_CONDITIONAL([BUILD_COVERAGE],[test "X$pac_cv_use_coverage" = "Xyes"])
-])
-
diff --git a/3rd-party/romio321/confdb/aclocal_cxx.m4 b/3rd-party/romio321/confdb/aclocal_cxx.m4
deleted file mode 100644
index 72febd8ab9d..00000000000
--- a/3rd-party/romio321/confdb/aclocal_cxx.m4
+++ /dev/null
@@ -1,187 +0,0 @@
-dnl PAC_CXX_SEARCH_LIST - expands to a whitespace separated list of C++
-dnl compilers for use with AC_PROG_CXX that is more suitable for HPC software
-dnl packages
-AC_DEFUN([PAC_CXX_SEARCH_LIST],[$CCC icpc pgCC xlC pathCC g++ clang++ c++ cc++ cxx CC cl])
-dnl PAC_PROG_CXX - reprioritize the C++ compiler search order
-dnl NOTE: this macro suffers from a basically intractable "expanded before it
-dnl was required" problem when libtool is also used
-AC_DEFUN([PAC_PROG_CXX],[
-	PAC_PUSH_FLAG([CXXFLAGS])
-        # This test uses the list from a recent PROG_CXX, but with the
-        # addition of the Portland group, IBM, and Intel C++ compilers
-        # (While the Intel icc compiler will compile C++ programs, it will
-        # not *link* C++ object files unless there is at least one C++ source
-        # file present on the command that performs the linking.  icpc is the
-        # Intel C++ compiler that both compiles and links C++ programs)
-	AC_PROG_CXX([PAC_CXX_SEARCH_LIST])
-	PAC_POP_FLAG([CXXFLAGS])
-])
-
-dnl This is from crypt.to/autoconf-archive, slightly modified.
-dnl It defines bool as int if it is not availalbe
-dnl
-AC_DEFUN([AX_CXX_BOOL],
-[AC_CACHE_CHECK(whether the compiler recognizes bool as a built-in type,
-ac_cv_cxx_bool,
-[AC_LANG_SAVE
- AC_LANG_CPLUSPLUS
- AC_TRY_COMPILE([
-int f(int  x){return 1;}
-int f(char x){return 1;}
-int f(bool x){return 1;}
-],[bool b = true; return f(b);],
- ac_cv_cxx_bool=yes, ac_cv_cxx_bool=no)
- AC_LANG_RESTORE
-])
-if test "$ac_cv_cxx_bool" != yes; then
-  AC_DEFINE(bool,int,[define if bool is a built-in type])
-fi
-])
-
-dnl This is from crypt.to/autoconf-archive, slightly modified (name defined)
-dnl
-AC_DEFUN([AX_CXX_EXCEPTIONS],
-[AC_CACHE_CHECK(whether the compiler supports exceptions,
-ac_cv_cxx_exceptions,
-[AC_LANG_SAVE
- AC_LANG_CPLUSPLUS
- AC_TRY_COMPILE(,[try { throw  1; } catch (int i) { return i; }],
- ac_cv_cxx_exceptions=yes, ac_cv_cxx_exceptions=no)
- AC_LANG_RESTORE
-])
-if test "$ac_cv_cxx_exceptions" = yes; then
-  AC_DEFINE(HAVE_CXX_EXCEPTIONS,,[define if the compiler supports exceptions])
-fi
-])
-
-dnl This is from crypt.to/autoconf-archive
-dnl
-AC_DEFUN([AX_CXX_NAMESPACES],
-[AC_CACHE_CHECK(whether the compiler implements namespaces,
-ac_cv_cxx_namespaces,
-[AC_LANG_SAVE
- AC_LANG_CPLUSPLUS
- AC_TRY_COMPILE([namespace Outer { namespace Inner { int i = 0; }}],
-                [using namespace Outer::Inner; return i;],
- ac_cv_cxx_namespaces=yes, ac_cv_cxx_namespaces=no)
- AC_LANG_RESTORE
-])
-if test "$ac_cv_cxx_namespaces" = yes; then
-  AC_DEFINE(HAVE_NAMESPACES,,[define if the compiler implements namespaces])
-fi
-])
-
-dnl Some compilers support namespaces but don't know about std
-dnl
-AC_DEFUN([AX_CXX_NAMESPACE_STD],
-[AC_REQUIRE([AX_CXX_NAMESPACES])
-AC_CACHE_CHECK(whether the compiler implements the namespace std,
-ac_cv_cxx_namespace_std,
-[ac_cv_cxx_namespace_std=no
-if test "$ac_cv_cxx_namespaces" = yes ; then 
-   AC_LANG_SAVE
-   AC_LANG_CPLUSPLUS
-   AC_TRY_COMPILE([
-#include <iostream>
-using namespace std;],
-                [cout << "message\n";],
- ac_cv_cxx_namespace_std=yes, ac_cv_cxx_namespace_std=no)
-   AC_LANG_RESTORE
-fi
-])
-if test "$ac_cv_cxx_namespace_std" = yes; then
-  AC_DEFINE(HAVE_NAMESPACE_STD,,[define if the compiler implements namespace std])
-fi
-])
-
-dnl/*D
-dnl PAC_CXX_CHECK_COMPILER_OPTION - Check that a C++ compiler option is
-dnl accepted without warning messages
-dnl
-dnl Synopsis:
-dnl PAC_CXX_CHECK_COMPILER_OPTION(optionname,action-if-ok,action-if-fail)
-dnl
-dnl Output Effects:
-dnl
-dnl If no actions are specified, a working value is added to 'CXXOPTIONS'
-dnl
-dnl Notes:
-dnl This is now careful to check that the output is different, since 
-dnl some compilers are noisy.
-dnl 
-dnl We are extra careful to prototype the functions in case compiler options
-dnl that complain about poor code are in effect.
-dnl
-dnl Because this is a long script, we have ensured that you can pass a 
-dnl variable containing the option name as the first argument.
-dnl D*/
-AC_DEFUN([PAC_CXX_CHECK_COMPILER_OPTION],[
-AC_MSG_CHECKING([whether C++ compiler accepts option $1])
-pac_opt="$1"
-AC_LANG_PUSH([C++])
-CXXFLAGS_orig="$CXXFLAGS"
-CXXFLAGS_opt="$pac_opt $CXXFLAGS"
-pac_result="unknown"
-
-AC_LANG_CONFTEST([AC_LANG_PROGRAM()])
-CXXFLAGS="$CXXFLAGS_orig"
-rm -f pac_test1.log
-PAC_LINK_IFELSE_LOG([pac_test1.log], [], [
-    CXXFLAGS="$CXXFLAGS_opt"
-    rm -f pac_test2.log
-    PAC_LINK_IFELSE_LOG([pac_test2.log], [], [
-        PAC_RUNLOG_IFELSE([diff -b pac_test1.log pac_test2.log],
-                          [pac_result=yes],[pac_result=no])
-    ],[
-        pac_result=no
-    ])
-], [
-    pac_result=no
-])
-AC_MSG_RESULT([$pac_result])
-dnl Delete the conftest created by AC_LANG_CONFTEST.
-rm -f conftest.$ac_ext
-
-if test "$pac_result" = "yes" ; then
-    AC_MSG_CHECKING([whether routines compiled with $pac_opt can be linked with ones compiled without $pac_opt])
-    pac_result=unknown
-    CXXFLAGS="$CXXFLAGS_orig"
-    rm -f pac_test3.log
-    PAC_COMPILE_IFELSE_LOG([pac_test3.log], [
-        AC_LANG_SOURCE([
-            int foo(void);
-            int foo(void){return 0;}
-        ])
-    ],[
-        PAC_RUNLOG([mv conftest.$OBJEXT pac_conftest.$OBJEXT])
-        saved_LIBS="$LIBS"
-        LIBS="pac_conftest.$OBJEXT $LIBS"
-
-        CXXFLAGS="$CXXFLAGS_opt"
-        rm -f pac_test4.log
-        PAC_LINK_IFELSE_LOG([pac_test4.log], [AC_LANG_PROGRAM()], [
-            PAC_RUNLOG_IFELSE([diff -b pac_test2.log pac_test4.log],
-                              [pac_result=yes], [pac_result=no])
-        ],[
-            pac_result=no
-        ])
-        LIBS="$saved_LIBS"
-        rm -f pac_conftest.$OBJEXT
-    ],[
-        pac_result=no
-    ])
-    AC_MSG_RESULT([$pac_result])
-    rm -f pac_test3.log pac_test4.log
-fi
-rm -f pac_test1.log pac_test2.log
-
-dnl Restore CXXFLAGS before 2nd/3rd argument commands are executed,
-dnl as 2nd/3rd argument command could be modifying CXXFLAGS.
-CXXFLAGS="$CXXFLAGS_orig"
-if test "$pac_result" = "yes" ; then
-     ifelse([$2],[],[CXXOPTIONS="$CXXOPTIONS $1"],[$2])
-else
-     ifelse([$3],[],[:],[$3])
-fi
-AC_LANG_POP([C++])
-])
diff --git a/3rd-party/romio321/confdb/aclocal_f77.m4 b/3rd-party/romio321/confdb/aclocal_f77.m4
deleted file mode 100644
index 92d310b0f21..00000000000
--- a/3rd-party/romio321/confdb/aclocal_f77.m4
+++ /dev/null
@@ -1,1487 +0,0 @@
-dnl PAC_F77_SEARCH_LIST - expands to a whitespace separated list of fortran 77
-dnl compilers for use with AC_PROG_F77 that is more suitable for HPC software
-dnl packages
-AC_DEFUN([PAC_F77_SEARCH_LIST],[ifort pgf77 af77 xlf frt cf77 fort77 fl32 fort ifc efc ftn gfortran f77 g77])
-dnl PAC_PROG_F77 - reprioritize the F77 compiler search order
-dnl NOTE: this macro suffers from a basically intractable "expanded before it
-dnl was required" problem when libtool is also used
-AC_DEFUN([PAC_PROG_F77],[
-PAC_PUSH_FLAG([FFLAGS])
-AC_PROG_F77([PAC_F77_SEARCH_LIST])
-PAC_POP_FLAG([FFLAGS])
-])
-dnl
-dnl/*D
-dnl PAC_PROG_F77_NAME_MANGLE - Determine how the Fortran compiler mangles
-dnl names 
-dnl
-dnl Synopsis:
-dnl PAC_PROG_F77_NAME_MANGLE([action])
-dnl
-dnl Output Effect:
-dnl If no action is specified, one of the following names is defined:
-dnl.vb
-dnl If fortran names are mapped:
-dnl   lower -> lower                  F77_NAME_LOWER
-dnl   lower -> lower_                 F77_NAME_LOWER_USCORE
-dnl   lower -> UPPER                  F77_NAME_UPPER
-dnl   lower_lower -> lower__          F77_NAME_LOWER_2USCORE
-dnl   mixed -> mixed                  F77_NAME_MIXED
-dnl   mixed -> mixed_                 F77_NAME_MIXED_USCORE
-dnl   mixed -> UPPER@STACK_SIZE       F77_NAME_UPPER_STDCALL
-dnl.ve
-dnl If an action is specified, it is executed instead.
-dnl 
-dnl Notes:
-dnl We assume that if lower -> lower (any underscore), upper -> upper with the
-dnl same underscore behavior.  Previous versions did this by 
-dnl compiling a Fortran program and running strings -a over it.  Depending on 
-dnl strings is a bad idea, so instead we try compiling and linking with a 
-dnl C program, since that is why we are doing this anyway.  A similar approach
-dnl is used by FFTW, though without some of the cases we check (specifically, 
-dnl mixed name mangling).  STD_CALL not only specifies a particular name
-dnl mangling convention (adding the size of the calling stack into the function
-dnl name, but also the stack management convention (callee cleans the stack,
-dnl and arguments are pushed onto the stack from right to left)
-dnl
-dnl One additional problem is that some Fortran implementations include 
-dnl references to the runtime (like pgf90_compiled for the pgf90 compiler
-dnl used as the "Fortran 77" compiler).  This is not yet solved.
-dnl
-dnl D*/
-dnl
-AC_DEFUN([PAC_PROG_F77_NAME_MANGLE],[
-AC_REQUIRE([AC_F77_LIBRARY_LDFLAGS])
-AC_CACHE_CHECK([for Fortran 77 name mangling],
-pac_cv_prog_f77_name_mangle,[
-# Initialize pac_found to indicate if name mangling scheme has been found
-pac_found=no
-AC_LANG_PUSH([Fortran 77])
-AC_COMPILE_IFELSE([
-    AC_LANG_SOURCE([
-        subroutine MY_name( ii )
-        return
-        end
-    ])
-],[
-    PAC_RUNLOG([mv conftest.$OBJEXT f77conftest.$OBJEXT])
-    saved_LIBS="$LIBS"
-    dnl  FLIBS is set by AC_F77_LIBRARY_LDFLAGS
-    LIBS="f77conftest.$OBJEXT $FLIBS $LIBS"
-    AC_LANG_PUSH([C])
-    for call in "" __stdcall ; do
-        for sym in my_name_ my_name__ my_name MY_NAME MY_name MY_name_ NONE ; do
-            AC_LINK_IFELSE([
-                AC_LANG_PROGRAM([extern void ${call} ${sym}(int);],[${sym}(0);])
-            ],[
-                pac_found=yes
-                break
-            ])
-        done
-        test "$pac_found" = "yes" && break
-    done
-    AC_LANG_POP([C])
-    LIBS="$saved_LIBS"
-    rm -f f77conftest.$OBJEXT
-])
-AC_LANG_POP([Fortran 77])
-dnl
-# If we got to here and pac_cv_prog_f77_name_mangle is still NOT definable,
-# it may be that the programs have to be linked with the Fortran compiler,
-# not the C compiler.  Try reversing the language used for the test
-if test  "$pac_found" != "yes" ; then
-    AC_LANG_PUSH([C])
-    for call in "" __stdcall ; do
-        for sym in my_name_ my_name__ my_name MY_NAME MY_name MY_name_ NONE ; do
-            AC_COMPILE_IFELSE([
-                AC_LANG_SOURCE([void ${call} ${sym}(int a) {}])
-            ],[
-                PAC_RUNLOG([mv conftest.$OBJEXT cconftest.$OBJEXT])
-                saved_LIBS="$LIBS"
-                LIBS="cconftest.$OBJEXT $LIBS"
-                AC_LANG_PUSH([Fortran 77])
-                AC_LINK_IFELSE([
-                    AC_LANG_PROGRAM([],[      call my_name(0)])
-                ],[
-                    pac_found=yes
-                ]) 
-                AC_LANG_POP([Fortran 77])
-                LIBS="$saved_LIBS"
-                rm -f cconftest.$OBJEXT
-                test "$pac_found" = "yes" && break
-            ])
-        done
-        test "$pac_found" = "yes" && break
-    done
-    AC_LANG_POP([C])
-fi
-if test "$pac_found" = "yes" ; then
-    case ${sym} in
-        my_name_)
-            pac_cv_prog_f77_name_mangle="lower uscore" ;;
-        my_name__)
-            pac_cv_prog_f77_name_mangle="lower 2uscore" ;;
-        my_name)
-            pac_cv_prog_f77_name_mangle="lower" ;;
-        MY_NAME)
-            pac_cv_prog_f77_name_mangle="upper" ;;
-        MY_name)
-            pac_cv_prog_f77_name_mangle="mixed" ;;
-        MY_name_)
-            pac_cv_prog_f77_name_mangle="mixed uscore" ;;
-        *)
-            pac_cv_prog_f77_name_mangle=""
-            pac_found=no;
-            ;;
-    esac
-    if test "X$pac_cv_prog_f77_name_mangle" != "X" ; then
-        if test "$call" = "__stdcall" ; then
-            pac_cv_prog_f77_name_mangle="$pac_cv_prog_f77_name_mangle stdcall"
-        fi
-    fi
-fi
-])
-dnl Endof ac_cache_check
-case $pac_cv_prog_f77_name_mangle in
-    *stdcall)
-        F77_STDCALL="__stdcall" ;;
-    *)
-        F77_STDCALL="" ;;
-esac
-# Get the standard call definition
-# FIXME: This should use F77_STDCALL, not STDCALL (non-conforming name)
-F77_STDCALL="$call"
-AC_DEFINE_UNQUOTED(STDCALL,[$F77_STDCALL],[Define calling convention])
-
-# new_name="`echo $name | tr ' ' '_' | tr [a-z] [A-Z]`"
-# We could have done the character conversion with 'tr'
-# which may not be portable, e.g. solaris's /usr/ucb/bin/tr.
-# So use a conservative approach.
-
-# Replace blank with underscore
-name_scheme="`echo $pac_cv_prog_f77_name_mangle | sed 's% %_%g'`"
-# Turn lowercase into uppercase.
-name_scheme="`echo $name_scheme | sed -e 'y%abcdefghijklmnopqrstuvwxyz%ABCDEFGHIJKLMNOPQRSTUVWXYZ%'`"
-F77_NAME_MANGLE="F77_NAME_${name_scheme}"
-AC_DEFINE_UNQUOTED([$F77_NAME_MANGLE]) 
-AC_SUBST(F77_NAME_MANGLE)
-if test "X$pac_cv_prog_f77_name_mangle" = "X" ; then
-    AC_MSG_WARN([Unknown Fortran naming scheme])
-fi
-dnl
-dnl Define the macros that is needed by AC_DEFINE_UNQUOTED([$F77_NAME_MANGLE])
-AH_TEMPLATE([F77_NAME_LOWER],
-    [Fortran names are lowercase with no trailing underscore])
-AH_TEMPLATE([F77_NAME_LOWER_USCORE],
-    [Fortran names are lowercase with one trailing underscore])
-AH_TEMPLATE([F77_NAME_LOWER_2USCORE],
-    [Fortran names are lowercase with two trailing underscores])
-AH_TEMPLATE([F77_NAME_MIXED],
-    [Fortran names preserve the original case])
-AH_TEMPLATE([F77_NAME_MIXED_USCORE],
-    [Fortran names preserve the original case with one trailing underscore])
-AH_TEMPLATE([F77_NAME_UPPER],
-    [Fortran names are uppercase])
-AH_TEMPLATE([F77_NAME_LOWER_STDCALL],
-    [Fortran names are lowercase with no trailing underscore in stdcall])
-AH_TEMPLATE([F77_NAME_LOWER_USCORE_STDCALL],
-    [Fortran names are lowercase with one trailing underscore in stdcall])
-AH_TEMPLATE([F77_NAME_LOWER_2USCORE_STDCALL],
-    [Fortran names are lowercase with two trailing underscores in stdcall])
-AH_TEMPLATE([F77_NAME_MIXED_STDCALL],
-    [Fortran names preserve the original case in stdcall])
-AH_TEMPLATE([F77_NAME_MIXED_USCORE_STDCALL],
-    [Fortran names preserve the original case with one trailing underscore in stdcall])
-AH_TEMPLATE([F77_NAME_UPPER_STDCALL],
-    [Fortran names are uppercase in stdcall])
-])
-dnl
-dnl/*D
-dnl PAC_PROG_F77_CHECK_SIZEOF - Determine the size in bytes of a Fortran
-dnl type
-dnl
-dnl Synopsis:
-dnl PAC_PROG_F77_CHECK_SIZEOF(type,[cross-size])
-dnl
-dnl Output Effect:
-dnl Sets SIZEOF_F77_uctype to the size if bytes of type.
-dnl If type is unknown, the size is set to 0.
-dnl If cross-compiling, the value cross-size is used (it may be a variable)
-dnl For example 'PAC_PROG_F77_CHECK_SIZEOF(real)' defines
-dnl 'SIZEOF_F77_REAL' to 4 on most systems.  The variable 
-dnl 'pac_cv_sizeof_f77_<type>' (e.g., 'pac_cv_sizeof_f77_real') is also set to
-dnl the size of the type. 
-dnl If the corresponding variable is already set, that value is used.
-dnl If the name has an '*' in it (e.g., 'integer*4'), the defined name 
-dnl replaces that with an underscore (e.g., 'SIZEOF_F77_INTEGER_4').
-dnl
-dnl Notes:
-dnl If the 'cross-size' argument is not given, 'autoconf' will issue an error
-dnl message.  You can use '0' to specify undetermined.
-dnl
-dnl D*/
-AC_DEFUN([PAC_PROG_F77_CHECK_SIZEOF],[
-AC_REQUIRE([AC_HEADER_STDC])
-AC_REQUIRE([AC_F77_LIBRARY_LDFLAGS])
-changequote(<<, >>)dnl
-dnl The name to #define.
-dnl dnl If the arg value contains a variable, we need to update that
-define(<<PAC_TYPE_NAME>>, translit(sizeof_f77_$1, [a-z *], [A-Z__]))dnl
-dnl The cache variable name.
-define(<<PAC_CV_NAME>>, translit(pac_cv_f77_sizeof_$1, [ *], [__]))dnl
-changequote([, ])dnl
-AC_CACHE_CHECK([for size of Fortran type $1],PAC_CV_NAME,[
-AC_REQUIRE([PAC_PROG_F77_NAME_MANGLE])
-AC_LANG_PUSH([Fortran 77])
-AC_COMPILE_IFELSE([
-    AC_LANG_SOURCE([
-        subroutine isize()
-        $1 i(2)
-        call cisize( i(1), i(2) )
-        end
-    ])
-],[
-    # pac_f77compile_ok=yes
-    PAC_RUNLOG([mv conftest.$OBJEXT pac_f77conftest.$OBJEXT])
-    # Save original LIBS, prepend previously generated object file to LIBS
-    saved_LIBS="$LIBS"
-    LIBS="pac_f77conftest.$OBJEXT $FLIBS $LIBS"
-    AC_LANG_PUSH([C])
-    AC_RUN_IFELSE([
-        AC_LANG_PROGRAM([
-#if defined(HAVE_STDIO_H) || defined(STDC_HEADERS)
-#include <stdio.h>
-#endif
-#ifdef F77_NAME_UPPER
-#define cisize_ CISIZE
-#define isize_ ISIZE
-#elif defined(F77_NAME_LOWER) || defined(F77_NAME_MIXED)
-#define cisize_ cisize
-#define isize_ isize
-#endif
-static int isize_val=0;
-void cisize_(char *,char*);
-void isize_(void);
-void cisize_(char *i1p, char *i2p)
-{ 
-   isize_val = (int)(i2p - i1p);
-}
-        ],[
-    FILE *f = fopen("conftestval", "w");
-    if (!f) return 1;
-    isize_();
-    fprintf(f,"%d\n", isize_val);
-        ])
-        dnl Endof ac_lang_program
-    ],[
-        eval PAC_CV_NAME=`cat conftestval`
-    ],[
-        eval PAC_CV_NAME=0
-    ],[
-        # Use -9999 as value to emit a warning message after the cache_check.
-        ifelse([$2],[],[eval PAC_CV_NAME=-9999],[eval PAC_CV_NAME=$2])
-    ])
-    dnl Endof ac_run_ifelse
-    AC_LANG_POP([C])
-    LIBS="$saved_LIBS"
-    # remove previously generated object file.
-    rm -f pac_f77conftest.$OBJEXT
-],[
-    # pac_f77compile_ok=no
-    ifelse([$2],,eval PAC_CV_NAME=0,eval PAC_CV_NAME=$2)
-])  Endof ac_compile_ifelse
-AC_LANG_POP([Fortran 77])
-])
-dnl Endof ac_cache_check
-if test "$PAC_CV_NAME" = "-9999" ; then
-     AC_MSG_WARN([No value provided for size of $1 when cross-compiling])
-fi
-AC_DEFINE_UNQUOTED(PAC_TYPE_NAME,$PAC_CV_NAME,[Define size of PAC_TYPE_NAME])
-undefine([PAC_TYPE_NAME])
-undefine([PAC_CV_NAME])
-])
-dnl
-dnl This version uses a Fortran program to link programs.
-dnl This is necessary because some compilers provide shared libraries
-dnl that are not within the default linker paths (e.g., our installation
-dnl of the Portland Group compilers)
-dnl
-AC_DEFUN([PAC_PROG_F77_CHECK_SIZEOF_EXT],[
-changequote(<<,>>)dnl
-dnl The name to #define.
-dnl If the arg value contains a variable, we need to update that
-define(<<PAC_TYPE_NAME>>, translit(sizeof_f77_$1, [a-z *], [A-Z__]))dnl
-dnl The cache variable name.
-define(<<PAC_CV_NAME>>, translit(pac_cv_f77_sizeof_$1, [ *], [__]))dnl
-changequote([,])dnl
-AC_CACHE_CHECK([for size of Fortran type $1],PAC_CV_NAME,[
-AC_REQUIRE([AC_HEADER_STDC])
-AC_REQUIRE([PAC_PROG_F77_NAME_MANGLE])
-dnl if test "$cross_compiling" = yes ; then
-dnl     ifelse([$2],[],
-dnl         [AC_MSG_WARN([No value provided for size of $1 when cross-compiling])],
-dnl         [eval PAC_CV_NAME=$2])
-dnl fi
-AC_LANG_PUSH([C])
-AC_COMPILE_IFELSE([
-    AC_LANG_SOURCE([
-#if defined(HAVE_STDIO_H) || defined(STDC_HEADERS)
-#include <stdio.h>
-#endif
-#ifdef F77_NAME_UPPER
-#define cisize_ CISIZE
-#define isize_ ISIZE
-#elif defined(F77_NAME_LOWER) || defined(F77_NAME_MIXED)
-#define cisize_ cisize
-#define isize_ isize
-#endif
-int cisize_(char *,char*);
-int cisize_(char *i1p, char *i2p) {
-    int isize_val=0;
-    FILE *f = fopen("conftestval", "w");
-    if (!f) return 1;
-    isize_val = (int)(i2p - i1p);
-    fprintf(f,"%d\n", isize_val);
-    fclose(f);
-    return 0;
-}
-    ])
-    dnl Endof ac_lang_source
-],[
-    # pac_compile_ok=yes
-    PAC_RUNLOG([mv conftest.$OBJEXT pac_conftest.$OBJEXT])
-    # Save LIBS and prepend object file to LIBS
-    saved_LIBS="$LIBS"
-    LIBS="pac_conftest.$OBJEXT $LIBS"
-    AC_LANG_PUSH([Fortran 77])
-    AC_RUN_IFELSE([
-        AC_LANG_SOURCE([
-            program main
-            $1 a(2)
-            integer irc, cisize
-            irc = cisize(a(1),a(2))
-            end
-        ])
-    ],[
-        eval PAC_CV_NAME=`cat conftestval`
-    ],[
-        eval PAC_CV_NAME=0
-    ],[
-        # Use -9999 as value to emit a warning message after the cache_check.
-        ifelse([$2],[],[eval PAC_CV_NAME=-9999],[eval PAC_CV_NAME=$2])
-    ])
-    AC_LANG_POP([Fortran 77])
-    LIBS="$saved_LIBS"
-    # remove previously generated object file.
-    rm -f pac_conftest.$OBJEXT
-],[
-    AC_MSG_WARN([Unable to compile the C routine for finding the size of a $1])
-])
-AC_LANG_POP([C])
-])
-dnl Endof ac_cache_check
-if test "$PAC_CV_NAME" = "-9999" ; then
-     AC_MSG_WARN([No value provided for size of $1 when cross-compiling])
-fi
-AC_DEFINE_UNQUOTED(PAC_TYPE_NAME,$PAC_CV_NAME,[Define size of PAC_TYPE_NAME])
-undefine([PAC_TYPE_NAME])
-undefine([PAC_CV_NAME])
-])
-dnl
-dnl/*D
-dnl PAC_PROG_F77_EXCLAIM_COMMENTS
-dnl
-dnl Synopsis:
-dnl PAC_PROG_F77_EXCLAIM_COMMENTS([action-if-true],[action-if-false])
-dnl
-dnl Notes:
-dnl Check whether '!' may be used to begin comments in Fortran.
-dnl
-dnl This macro requires a version of autoconf `after` 2.13; the 'acgeneral.m4'
-dnl file contains an error in the handling of Fortran programs in 
-dnl 'AC_TRY_COMPILE' (fixed in our local version).
-dnl
-dnl D*/
-AC_DEFUN([PAC_PROG_F77_EXCLAIM_COMMENTS],[
-AC_CACHE_CHECK([whether Fortran 77 accepts ! for comments],
-pac_cv_prog_f77_exclaim_comments,[
-AC_LANG_PUSH([Fortran 77])
-AC_COMPILE_IFELSE([
-     AC_LANG_PROGRAM([],[!        This is a comment])
-],[
-    pac_cv_prog_f77_exclaim_comments="yes"
-],[
-    pac_cv_prog_f77_exclaim_comments="no"
-])
-AC_LANG_POP([Fortran 77])
-])
-if test "$pac_cv_prog_f77_exclaim_comments" = "yes" ; then
-    ifelse([$1],[],[:],[$1])
-else
-    ifelse([$2],[],[:],[$2])
-fi
-])dnl
-dnl
-dnl/*D
-dnl PAC_F77_CHECK_COMPILER_OPTION - Check that a F77 compiler option is
-dnl accepted without warning messages
-dnl
-dnl Synopsis:
-dnl PAC_F77_CHECK_COMPILER_OPTION(optionname,action-if-ok,action-if-fail)
-dnl
-dnl Output Effects:
-dnl
-dnl If no actions are specified, a working value is added to 'FOPTIONS'
-dnl
-dnl Notes:
-dnl This is now careful to check that the output is different, since 
-dnl some compilers are noisy.
-dnl 
-dnl We are extra careful to prototype the functions in case compiler options
-dnl that complain about poor code are in effect.
-dnl
-dnl Because this is a long script, we have ensured that you can pass a 
-dnl variable containing the option name as the first argument.
-dnl D*/
-AC_DEFUN([PAC_F77_CHECK_COMPILER_OPTION],[
-AC_MSG_CHECKING([whether Fortran 77 compiler accepts option $1])
-pac_opt="$1"
-AC_LANG_PUSH([Fortran 77])
-FFLAGS_orig="$FFLAGS"
-FFLAGS_opt="$pac_opt $FFLAGS"
-pac_result="unknown"
-
-AC_LANG_CONFTEST([AC_LANG_PROGRAM()])
-FFLAGS="$FFLAGS_orig"
-rm -f pac_test1.log
-PAC_LINK_IFELSE_LOG([pac_test1.log], [], [
-    FFLAGS="$FFLAGS_opt"
-    rm -f pac_test2.log
-    PAC_LINK_IFELSE_LOG([pac_test2.log], [], [
-        PAC_RUNLOG_IFELSE([diff -b pac_test1.log pac_test2.log],
-                          [pac_result=yes], [pac_result=no])
-    ],[
-        pac_result=no
-    ])
-], [
-    pac_result=no
-])
-AC_MSG_RESULT([$pac_result])
-dnl Delete the conftest created by AC_LANG_CONFTEST.
-rm -f conftest.$ac_ext
-#
-if test "$pac_result" = "yes" ; then
-    AC_MSG_CHECKING([whether routines compiled with $pac_opt can be linked with ones compiled without $pac_opt])
-    pac_result=unknown
-    FFLAGS="$FFLAGS_orig"
-    rm -f pac_test3.log
-    PAC_COMPILE_IFELSE_LOG([pac_test3.log], [
-        AC_LANG_SOURCE([
-            subroutine try()
-            end
-        ])
-    ],[
-        PAC_RUNLOG([mv conftest.$OBJEXT pac_conftest.$OBJEXT])
-        saved_LIBS="$LIBS"
-        LIBS="pac_conftest.$OBJEXT $LIBS"
-
-        FFLAGS="$FFLAGS_opt"
-        rm -f pac_test4.log
-        PAC_LINK_IFELSE_LOG([pac_test4.log], [AC_LANG_PROGRAM()], [
-            PAC_RUNLOG_IFELSE([diff -b pac_test2.log pac_test4.log],
-                              [pac_result=yes], [pac_result=no])
-        ],[
-            pac_result=no
-        ])
-        LIBS="$saved_LIBS"
-        rm -f pac_conftest.$OBJEXT
-    ],[
-        pac_result=no
-    ])
-    AC_MSG_RESULT([$pac_result])
-    rm -f pac_test3.log pac_test4.log
-fi
-rm -f pac_test1.log pac_test2.log
-
-dnl Restore FFLAGS before 2nd/3rd argument commands are executed,
-dnl as 2nd/3rd argument command could be modifying FFLAGS.
-FFLAGS="$FFLAGS_orig"
-if test "$pac_result" = "yes" ; then
-     ifelse([$2],[],[FOPTIONS="$FOPTIONS $1"],[$2])
-else
-     ifelse([$3],[],[:],[$3])
-fi
-AC_LANG_POP([Fortran 77])
-])
-dnl
-dnl/*D
-dnl PAC_PROG_F77_LIBRARY_DIR_FLAG - Determine the flag used to indicate
-dnl the directories to find libraries in
-dnl
-dnl Notes:
-dnl Many compilers accept '-Ldir' just like most C compilers.  
-dnl Unfortunately, some (such as some HPUX Fortran compilers) do not, 
-dnl and require instead either '-Wl,-L,dir' or something else.  This
-dnl command attempts to determine what is accepted.  The flag is 
-dnl placed into 'F77_LIBDIR_LEADER'.
-dnl
-dnl D*/
-dnl
-dnl An earlier version of this only tried the arguments without using
-dnl a library.  This failed when the HP compiler complained about the
-dnl arguments, but produced an executable anyway.
-AC_DEFUN([PAC_PROG_F77_LIBRARY_DIR_FLAG],[
-AC_CACHE_CHECK([for Fortran 77 flag for library directories],
-pac_cv_prog_f77_library_dir_flag,[
-AC_LANG_PUSH([Fortran 77])
-AC_COMPILE_IFELSE([
-    AC_LANG_SOURCE([
-        subroutine f1conf
-        end
-    ])
-],[
-    # pac_f77compile_ok=yes
-    PAC_RUNLOG([mv conftest.$OBJEXT pac_f77conftest.$OBJEXT])
-    PAC_RUNLOG([test -d conftestdir || mkdir conftestdir])
-    PAC_RUNLOG([${AR-ar} ${AR_FLAGS-cr} conftestdir/libf77conftest.a pac_f77conftest.$OBJEXT])
-    PAC_RUNLOG([${RANLIB-ranlib} conftestdir/libf77conftest.a])
-    # Save original LIBS, prepend previously generated object file to LIBS
-    saved_LIBS="$LIBS"
-    LIBS="-lf77conftest $LIBS"
-    saved_LDFLAGS="$LDFLAGS"
-    pac_cv_prog_f77_library_dir_flag="none"
-    for ldir in "-L" "-Wl,-L," ; do
-        LDFLAGS="${ldir}conftestdir $saved_LDFLAGS"
-        AC_LINK_IFELSE([
-            AC_LANG_SOURCE([
-                program main
-                call f1conf
-                end
-            ])
-        ],[pac_cv_prog_f77_library_dir_flag="$ldir";break])
-    done
-    LDFLAGS="$saved_LDFLAGS"
-    LIBS="$saved_LIBS"
-    rm -rf conftestdir
-    rm -f pac_f77conftest.$OBJEXT
-],[])
-AC_LANG_POP([Fortran 77])
-])
-dnl Endof ac_cache_check
-if test "X$pac_cv_prog_f77_library_dir_flag" != "Xnone" ; then
-    F77_LIBDIR_LEADER="$pac_cv_prog_f77_library_dir_flag"
-    AC_SUBST(F77_LIBDIR_LEADER)
-fi
-])
-dnl
-dnl/*D 
-dnl PAC_PROG_F77_HAS_INCDIR - Check whether Fortran accepts -Idir flag
-dnl
-dnl Syntax:
-dnl   PAC_PROG_F77_HAS_INCDIR(directory,action-if-true,action-if-false)
-dnl
-dnl Output Effect:
-dnl  Sets 'F77_INCDIR' to the flag used to choose the directory.  
-dnl
-dnl Notes:
-dnl This refers to the handling of the common Fortran include extension,
-dnl not to the use of '#include' with the C preprocessor.
-dnl If directory does not exist, it will be created.  In that case, the 
-dnl directory should be a direct descendant of the current directory.
-dnl
-dnl D*/
-AC_DEFUN([PAC_PROG_F77_HAS_INCDIR],[
-ifelse([$1],[],[checkdir=f77tmpdir],[checkdir=$1;checkdir_is_given=yes])
-AC_CACHE_CHECK([for include directory flag for Fortran],
-pac_cv_prog_f77_has_incdir,[
-test -d $checkdir || mkdir $checkdir
-dnl PAC_RUNLOG([echo '       call sub()' > $checkdir/conftestf.h])
-echo '       call sub()' > $checkdir/conftestf.h
-AC_LANG_PUSH([Fortran 77])
-saved_FFLAGS="$FFLAGS"
-pac_cv_prog_f77_has_incdir="none"
-# SGI wants -Wf,-I
-for idir in "-I" "-Wf,-I" ; do
-    FFLAGS="${idir} $checkdir $saved_FFLAGS"
-    AC_COMPILE_IFELSE([
-        AC_LANG_SOURCE([
-            program main
-            include 'conftestf.h'
-            end
-        ])
-    ],[pac_cv_prog_f77_has_incdir="$idir"; break])
-done
-FFLAGS="$saved_FFLAGS"
-AC_LANG_POP([Fortran 77])
-if test "$checkdir_is_given" = "yes" ; then
-    rm -f $checkdir/conftestf.h
-else
-    rm -rf $checkdir
-fi
-])
-dnl Endof ac_cache_check
-if test "X$pac_cv_prog_f77_has_incdir" != "Xnone" ; then
-    F77_INCDIR="$pac_cv_prog_f77_has_incdir"
-    AC_SUBST(F77_INCDIR)
-fi
-])
-dnl
-dnl/*D
-dnl PAC_PROG_F77_ALLOWS_UNUSED_EXTERNALS - Check whether the Fortran compiler
-dnl allows unused and undefined functions to be listed in an external 
-dnl statement
-dnl
-dnl Syntax:
-dnl   PAC_PROG_F77_ALLOWS_UNUSED_EXTERNALS(action-if-true,action-if-false)
-dnl
-dnl D*/
-AC_DEFUN([PAC_PROG_F77_ALLOWS_UNUSED_EXTERNALS],[
-AC_CACHE_CHECK([whether Fortran allows unused externals],
-pac_cv_prog_f77_allows_unused_externals,[
-AC_LANG_PUSH([Fortran 77])
-AC_LINK_IFELSE([
-    AC_LANG_SOURCE([
-        program main
-        external bar
-        end
-    ])
-],[
-    pac_cv_prog_f77_allows_unused_externals="yes"
-],[
-    pac_cv_prog_f77_allows_unused_externals="no"
-])
-AC_LANG_POP([Fortran 77])
-])
-dnl Endof ac_cache_check
-if test "X$pac_cv_prog_f77_allows_unused_externals" = "Xyes" ; then
-   ifelse([$1],[],[:],[$1])
-else
-   ifelse([$2],[],[:],[$2])
-fi
-])
-dnl PAC_PROG_F77_RUN_PROC_FROM_C( c main program, fortran routine, 
-dnl                               [action-if-works], [action-if-fails], 
-dnl                               [cross-action] )
-dnl Fortran routine MUST be named ftest unless you include code
-dnl to select the appropriate Fortran name.
-dnl 
-AC_DEFUN([PAC_PROG_F77_RUN_PROC_FROM_C],[
-AC_REQUIRE([AC_HEADER_STDC])
-AC_REQUIRE([AC_F77_LIBRARY_LDFLAGS])
-AC_LANG_PUSH([Fortran 77])
-AC_COMPILE_IFELSE([
-    AC_LANG_SOURCE([$2])
-],[
-    # pac_f77compile_ok=yes
-    PAC_RUNLOG([mv conftest.$OBJEXT pac_f77conftest.$OBJEXT])
-    # Save original LIBS, prepend previously generated object file to LIBS
-    saved_LIBS="$LIBS"
-    LIBS="pac_f77conftest.$OBJEXT $FLIBS $LIBS"
-    AC_LANG_PUSH([C])
-    AC_RUN_IFELSE([
-        AC_LANG_SOURCE([
-#if defined(HAVE_STDIO_H) || defined(STDC_HEADERS)
-#include <stdio.h>
-#endif
-#ifdef F77_NAME_UPPER
-#define ftest_ FTEST
-#elif defined(F77_NAME_LOWER) || defined(F77_NAME_MIXED)
-#define ftest_ ftest
-#endif
-$1
-        ])
-    ],[
-        ifelse([$3],[],[:],[$3])
-    ],[
-        ifelse([$4],[],[:],[$4])
-    ],[
-        ifelse([$5],[],[:],[$5])
-    ])
-    AC_LANG_POP([C])
-    LIBS="$saved_LIBS"
-    rm -f pac_f77conftest.$OBJEXT
-],[
-])
-AC_LANG_POP([Fortran 77])
-])
-dnl PAC_PROG_F77_IN_C_LIBS
-dnl
-dnl Find the essential libraries that are needed to use the C linker to 
-dnl create a program that includes a trival Fortran code.  
-dnl
-dnl For example, all pgf90 compiled objects include a reference to the
-dnl symbol pgf90_compiled, found in libpgf90 .
-dnl
-dnl There is an additional problem.  To *run* programs, we may need 
-dnl additional arguments; e.g., if shared libraries are used.  Even
-dnl with autoconf 2.52, the autoconf macro to find the library arguments
-dnl doesn't handle this, either by detecting the use of -rpath or
-dnl by trying to *run* a trivial program.  It only checks for *linking*.
-dnl 
-dnl
-AC_DEFUN([PAC_PROG_F77_IN_C_LIBS],[
-AC_REQUIRE([AC_HEADER_STDC])
-AC_REQUIRE([AC_F77_LIBRARY_LDFLAGS])
-AC_MSG_CHECKING([for which Fortran libraries are needed to link C with Fortran])
-F77_IN_C_LIBS="invalid"
-AC_LANG_PUSH([Fortran 77])
-AC_COMPILE_IFELSE([
-    AC_LANG_SOURCE([
-        subroutine ftest
-        end
-    ])
-],[
-    # pac_f77compile_ok=yes
-    PAC_RUNLOG([mv conftest.$OBJEXT pac_f77conftest.$OBJEXT])
-    # Save original LIBS, prepend previously generated object file to LIBS
-    saved_LIBS="$LIBS"
-    LIBS="pac_f77conftest.$OBJEXT $FLIBS $saved_LIBS"
-    AC_LANG_PUSH([C])
-
-    # Create conftest for all link tests.
-    AC_LANG_CONFTEST([
-        AC_LANG_PROGRAM([
-#if defined(HAVE_STDIO_H) || defined(STDC_HEADERS)
-#include <stdio.h>
-#endif
-        ],[
-#ifdef F77_NAME_UPPER
-#define ftest_ FTEST
-#elif defined(F77_NAME_LOWER) || defined(F77_NAME_MIXED)
-#define ftest_ ftest
-#endif
-extern void ftest_(void);
-ftest_();
-        ])
-    ])
-
-    F77_IN_C_LIBS=""
-    AC_LINK_IFELSE([],[:],[
-        flibdirs=`echo $FLIBS | tr ' ' '\012' | grep '\-L' | tr '\012' ' '`
-        fliblibs=`echo $FLIBS | tr ' ' '\012' | grep -v '\-L' | tr '\012' ' '`
-        for flibs in $fliblibs ; do
-            LIBS="pac_f77conftest.$OBJEXT $flibdirs $flibs $saved_LIBS"
-            AC_LINK_IFELSE([],[F77_IN_C_LIBS="$flibdirs $flibs"; break])
-        done
-        if test "X$F77_IN_C_LIBS" = "X" ; then
-            flibscat=""
-            for flibs in $fliblibs ; do
-                flibscat="$flibscat $flibs"
-                LIBS="pac_f77conftest.$OBJEXT $flibdirs $flibscat $saved_LIBS"
-                AC_LINK_IFELSE([],[F77_IN_C_LIBS="$flibdirs $flibscat";break])
-            done
-        fi
-    ])
-
-    # remove conftest created by ac_lang_conftest
-    rm -f conftest.$ac_ext
-    AC_LANG_POP([C])
-    LIBS="$saved_LIBS"
-    rm -f pac_f77conftest.$OBJEXT
-])
-AC_LANG_POP([Fortran 77])
-if test "X$F77_IN_C_LIBS" = "X" ; then
-    AC_MSG_RESULT(none)
-else
-    AC_MSG_RESULT($F77_IN_C_LIBS)
-fi
-])
-dnl
-dnl Test to see if we should use C or Fortran to link programs whose
-dnl main program is in Fortran.  We may find that neither work because 
-dnl we need special libraries in each case.
-dnl
-AC_DEFUN([PAC_PROG_F77_LINKER_WITH_C],[
-AC_REQUIRE([AC_F77_LIBRARY_LDFLAGS])
-AC_MSG_CHECKING([for linker for Fortran main program])
-dnl Create a C program that uses multiplication and division
-dnl in case that requires special libraries
-AC_LANG_PUSH([C])
-AC_COMPILE_IFELSE([
-    AC_LANG_PROGRAM([],[long long a;])
-],[
-    AC_DEFINE(HAVE_LONG_LONG,1,[Define if long long allowed])
-])
-AC_LANG_CONFTEST([
-    AC_LANG_SOURCE([
-#ifdef HAVE_LONG_LONG
-int f(int a, long long b) { int c; c = a * ( b / 3 ) / (b-1); return c ; }
-#else
-int f(int a, long b) { int c; c = a * b / (b-1); return c ; }
-#endif
-    ])
-])
-AC_LANG_POP([C])
-
-dnl Create a Fortran program for test
-AC_LANG_PUSH([Fortran 77])
-AC_LANG_CONFTEST([
-    AC_LANG_SOURCE([
-        program main
-        double precision d
-        print *, "hi"
-        end
-    ])
-])
-AC_LANG_POP([Fortran 77])
-
-dnl Initialize flags
-pac_linkwithf77=no
-pac_linkwithC=no
-
-dnl Use F77 as a linker to compile a Fortran main and C subprogram.
-if test "$pac_linkwithC" != "yes" ; then
-    AC_LANG_PUSH([C])
-    AC_COMPILE_IFELSE([],[
-        PAC_RUNLOG([mv conftest.$OBJEXT pac_conftest.$OBJEXT])
-        saved_LIBS="$LIBS"
-        LIBS="pac_conftest.$OBJEXT $saved_LIBS"
-        AC_LANG_PUSH([Fortran 77])
-        AC_LINK_IFELSE([],[
-            AC_MSG_RESULT([Use Fortran to link programs])
-            pac_linkwithf77=yes
-        ])
-        AC_LANG_POP([Fortran 77])
-        LIBS="$saved_LIBS"
-        rm -f pac_conftest.$OBJEXT
-    ])
-    AC_LANG_POP([C])
-fi
-
-dnl Use C as a linker and FLIBS to compile a Fortran main and C subprogram.
-if test "$pac_linkwithf77" != "yes" ; then
-    AC_LANG_PUSH([Fortran 77])
-    AC_COMPILE_IFELSE([],[
-        PAC_RUNLOG([mv conftest.$OBJEXT pac_f77conftest.$OBJEXT])
-        saved_LIBS="$LIBS"
-        LIBS="pac_f77conftest.$OBJEXT $FLIBS $saved_LIBS"
-        AC_LANG_PUSH([C])
-        AC_LINK_IFELSE([],[
-            pac_linkwithC=yes
-            AC_MSG_RESULT([Use C with FLIBS to link programs])
-            F77LINKER="$CC"
-            F77_LDFLAGS="$F77_LDFLAGS $FLIBS"
-        ])
-        AC_LANG_POP([C])
-        LIBS="$saved_LIBS"
-        rm -f pac_f77conftest.$OBJEXT
-    ])
-    AC_LANG_POP([Fortran 77])
-fi
-
-AC_LANG_PUSH([Fortran 77])
-rm -f conftest.$ac_ext
-AC_LANG_POP([Fortran 77])
-
-AC_LANG_PUSH([C])
-rm -f conftest.$ac_ext
-AC_LANG_POP([C])
-
-if test "$pac_linkwithf77" != "yes" -a "$pac_linkwithC" != "yes" ; then
-    AC_MSG_ERROR([Could not determine a way to link a Fortran test program!])
-fi
-])
-dnl
-dnl Check to see if a C program can be linked when using the libraries
-dnl needed by C programs
-dnl
-AC_DEFUN([PAC_PROG_F77_CHECK_FLIBS],[
-AC_REQUIRE([AC_F77_LIBRARY_LDFLAGS])
-AC_MSG_CHECKING([whether $CC links with FLIBS found by autoconf])
-AC_LANG_PUSH([C])
-# Create a simple C program for the tests.
-AC_LANG_CONFTEST([
-    AC_LANG_PROGRAM([],[int a;])
-])
-# Try to link a C program with all of these libraries
-saved_LIBS="$LIBS"
-LIBS="$FLIBS $saved_LIBS"
-AC_LINK_IFELSE([],[
-    AC_MSG_RESULT([yes])
-],[
-    AC_MSG_RESULT([no])
-    AC_MSG_CHECKING([for which libraries can be used])
-    pac_ldirs=""
-    pac_libs=""
-    pac_other=""
-    for name in $FLIBS ; do
-        case $name in 
-        -l*) pac_libs="$pac_libs $name"   ;;
-        -L*) pac_ldirs="$pac_ldirs $name" ;;
-          *) pac_other="$pac_other $name" ;;
-        esac
-    done
-    keep_libs=""
-    for name in $pac_libs ; do
-        LIBS="$saved_LIBS $pac_ldirs $pac_other $name"
-        AC_LINK_IFELSE([],[
-            keep_libs="$keep_libs $name"
-        ])
-    done
-    AC_MSG_RESULT($keep_libs)
-    FLIBS="$pac_ldirs $pac_other $keep_libs"
-])
-LIBS="$saved_LIBS"
-rm -f conftest.$ac_ext
-AC_LANG_PUSH([C])
-])
-dnl
-dnl Test for extra libraries needed when linking C routines that use
-dnl stdio with Fortran.  This test was created for OSX, which 
-dnl sometimes requires -lSystemStubs.  If another library is needed,
-dnl add it to F77_OTHER_LIBS
-dnl
-AC_DEFUN([PAC_PROG_F77_AND_C_STDIO_LIBS],[
-AC_REQUIRE([AC_HEADER_STDC])
-AC_REQUIRE([PAC_PROG_F77_NAME_MANGLE])
-# To simply the code in the cache_check macro, chose the routine name
-# first, in case we need it
-confname=conf1_
-case "$pac_cv_prog_f77_name_mangle" in
-    "lower underscore")       confname=conf1_ ;;
-    "upper stdcall")          confname=CONF1  ;;
-    "upper")                  confname=CONF1  ;;
-    "lower doubleunderscore") confname=conf1_ ;;
-    "lower")                  confname=conf1  ;;
-    "mixed underscore")       confname=conf1_ ;;
-    "mixed")                  confname=conf1  ;;
-esac
-
-AC_CACHE_CHECK([for libraries to link Fortran main with C stdio routines],
-pac_cv_prog_f77_and_c_stdio_libs,[
-pac_cv_prog_f77_and_c_stdio_libs=unknown
-AC_LANG_PUSH([C])
-AC_COMPILE_IFELSE([
-    AC_LANG_SOURCE([
-#if defined(HAVE_STDIO_H) || defined(STDC_HEADERS)
-#include <stdio.h>
-#endif
-int $confname(int a) {
-    printf( "The answer is %d\n", a ); fflush(stdout); return 0;
-}
-    ])
-],[
-    PAC_RUNLOG([mv conftest.$OBJEXT pac_conftest.$OBJEXT])
-    saved_LIBS="$LIBS"
-    AC_LANG_PUSH([Fortran 77])
-    AC_LANG_CONFTEST([
-        AC_LANG_SOURCE([
-            program main
-            call conf1(0)
-            end
-        ])
-    ])
-    for extralib in "" "-lSystemStubs" ; do
-        LIBS="pac_conftest.$OBJEXT $saved_LIBS $extralib"
-        AC_LINK_IFELSE([],[
-            pac_cv_prog_f77_and_c_stdio_libs="$extralib"; break
-        ])
-    done
-    if test "X$pac_cv_prog_f77_and_c_stdio_libs" = "X" ; then
-        pac_cv_prog_f77_and_c_stdio_libs=none
-    fi
-    rm -f conftest.$ac_ext
-    AC_LANG_POP([Fortran 77])
-    LIBS="$saved_LIBS"
-    rm -f pac_conftest.$OBJEXT
-])
-AC_LANG_POP([C])
-])
-dnl Endof ac_cache_check
-if test "$pac_cv_prog_f77_and_c_stdio_libs" != "none" \
-     -a "$pac_cv_prog_f77_and_c_stdio_libs" != "unknown" ; then
-    F77_OTHER_LIBS="$F77_OTHER_LIBS $pac_cv_prog_f77_and_c_stdio_libs"
-fi    
-])
-dnl
-dnl Check that the FLIBS determined by AC_F77_LIBRARY_LDFLAGS is valid.
-dnl That macro (at least as of autoconf 2.59) attempted to parse the output
-dnl of the compiler when asked to be verbose; in the case of the Fujitsu
-dnl frt Fortran compiler, it included files that frt looked for and then
-dnl discarded because they did not exist.
-dnl
-AC_DEFUN([PAC_PROG_F77_FLIBS_VALID],[
-AC_REQUIRE([AC_F77_LIBRARY_LDFLAGS])
-AC_MSG_CHECKING([whether $F77 accepts the FLIBS found by autoconf])
-pac_cv_f77_flibs_valid=unknown
-AC_LANG_PUSH([Fortran 77])
-AC_LANG_CONFTEST([
-    AC_LANG_SOURCE([
-        program main
-        end
-    ])
-])
-AC_LINK_IFELSE([],[
-    AC_MSG_RESULT([yes])
-],[
-    AC_MSG_RESULT([no])
-    AC_MSG_CHECKING([for valid entries in FLIBS])
-    goodFLIBS=""
-    saveFLIBS=$FLIBS
-    FLIBS=""
-    for arg in $saveFLIBS ; do
-        FLIBS="$goodFLIBS $arg"
-        AC_LINK_IFELSE([],[goodFLIBS=$FLIBS])
-    done
-    FLIBS=$goodFLIBS
-    AC_MSG_RESULT($FLIBS)
-])
-rm -f conftest.$ac_ext
-AC_LANG_POP([Fortran 77])
-])
-dnl
-dnl Check if the Fortran 77 and C objects are compatible in linking.
-dnl e.g. On some intel x86_64 Mac, Fortran compiler's default binary format
-dnl is different from C, so either -m64 or -m32 is needed in either CFLAGS
-dnl or FFLAGS.
-dnl
-AC_DEFUN([PAC_PROG_F77_OBJ_LINKS_WITH_C],[
-AC_REQUIRE([AC_F77_LIBRARY_LDFLAGS])
-AC_MSG_CHECKING([whether Fortran 77 and C objects are compatible])
-AC_LANG_PUSH([C])
-AC_LANG_CONFTEST([
-    AC_LANG_SOURCE([
-/* lower */
-void c_subpgm( int *rc );
-void c_subpgm( int *rc ) { *rc = 1; }
-
-/* lower underscore */
-void c_subpgm_( int *rc );
-void c_subpgm_( int *rc ) { *rc = 2; }
-
-/* upper */
-void C_SUBPGM( int *rc );
-void C_SUBPGM( int *rc ) { *rc = 3; }
-
-/* lower doubleunderscore */
-void c_subpgm__( int *rc );
-void c_subpgm__( int *rc ) { *rc = 4; }
-
-/* mixed */
-void C_subpgm( int *rc );
-void C_subpgm( int *rc ) { *rc = 5; }
-
-/* mixed underscore */
-void C_subpgm_( int *rc );
-void C_subpgm_( int *rc ) { *rc = 6; }
-    ])
-])
-AC_LANG_POP([C])
-
-AC_LANG_PUSH([Fortran 77])
-AC_LANG_CONFTEST([
-    AC_LANG_SOURCE([
-        program test
-        integer rc
-        rc = -1
-        call c_subpgm( rc )
-        write(6,*) "rc=", rc
-        end
-    ])
-])
-AC_LANG_POP([Fortran 77])
-
-dnl Initialize flags
-pac_linkwithf77=no
-pac_linkwithC=no
-
-dnl Use F77 as a linker to compile a Fortran main and C subprogram.
-if test "$pac_linkwithC" != "yes" ; then
-    AC_LANG_PUSH([C])
-    AC_COMPILE_IFELSE([],[
-        PAC_RUNLOG([mv conftest.$OBJEXT pac_conftest.$OBJEXT])
-        saved_LIBS="$LIBS"
-        LIBS="pac_conftest.$OBJEXT $saved_LIBS"
-        AC_LANG_PUSH([Fortran 77])
-        AC_LINK_IFELSE([],[
-            pac_linkwithf77=yes
-            AC_MSG_RESULT([yes])
-        ])
-        AC_LANG_POP([Fortran 77])
-        LIBS="$saved_LIBS"
-        if test "$pac_linkwithf77" = "yes" ; then
-            rm -f pac_conftest.$OBJEXT
-        fi
-    ])
-    AC_LANG_POP([C])
-fi
-
-dnl Use C as a linker and FLIBS to compile a Fortran main and C subprogram.
-if test "$pac_linkwithf77" != "yes" ; then
-    AC_LANG_PUSH([Fortran 77])
-    AC_COMPILE_IFELSE([],[
-        PAC_RUNLOG([mv conftest.$OBJEXT pac_f77conftest.$OBJEXT])
-        saved_LIBS="$LIBS"
-        LIBS="pac_f77conftest.$OBJEXT $FLIBS $saved_LIBS"
-        AC_LANG_PUSH([C])
-        AC_LINK_IFELSE([],[
-            pac_linkwithC=yes
-            AC_MSG_RESULT([yes])
-        ])
-        AC_LANG_POP([C])
-        LIBS="$saved_LIBS"
-        if test "$pac_linkwithC" = "yes" ; then
-            rm -f pac_f77conftest.$OBJEXT
-        fi
-    ])
-    AC_LANG_POP([Fortran 77])
-fi
-
-AC_LANG_PUSH([Fortran 77])
-rm -f conftest.$ac_ext
-AC_LANG_POP([Fortran 77])
-
-AC_LANG_PUSH([C])
-rm -f conftest.$ac_ext
-AC_LANG_POP([C])
-
-if test "$pac_linkwithf77" != "yes" -a "$pac_linkwithC" != "yes" ; then
-    AC_MSG_RESULT(no)
-    AC_CHECK_PROG(FILE, file, file, [])
-    if test "X$FILE" != "X" ; then
-        fobjtype="`${FILE} pac_f77conftest.$OBJEXT | sed -e \"s|pac_f77conftest\.$OBJEXT||g\"`"
-        cobjtype="`${FILE} pac_conftest.$OBJEXT | sed -e \"s|pac_conftest\.$OBJEXT||g\"`"
-        if test "$fobjtype" != "$cobjtype" ; then
-            AC_MSG_ERROR([****  Incompatible Fortran and C Object File Types!  ****
-F77 Object File Type produced by \"${F77} ${FFLAGS}\" is : ${fobjtype}.
- C  Object File Type produced by \"${CC} ${CFLAGS}\" is : ${cobjtype}.])
-        fi
-    fi
-fi
-])
-dnl
-dnl /*D
-dnl PAC_F77_WORKS_WITH_CPP
-dnl
-dnl Checks if Fortran 77 compiler works with C preprocessor
-dnl
-dnl Most systems allow the Fortran compiler to process .F and .F90 files
-dnl using the C preprocessor.  However, some systems either do not
-dnl allow this or have serious bugs (OSF Fortran compilers have a bug
-dnl that generates an error message from cpp).  The following test
-dnl checks to see if .F works, and if not, whether "cpp -P -C" can be used
-dnl D*/
-AC_DEFUN([PAC_F77_WORKS_WITH_CPP],[
-AC_REQUIRE([AC_PROG_CPP])
-AC_MSG_CHECKING([whether Fortran 77 compiler processes .F files with C preprocessor])
-AC_LANG_PUSH([Fortran 77])
-saved_f77_ext=${ac_ext}
-ac_ext="F"
-saved_FFLAGS="$FFLAGS"
-FFLAGS="$FFLAGS $CPPFLAGS"
-AC_LANG_CONFTEST([
-    AC_LANG_SOURCE([
-        program main
-#define ASIZE 10
-        integer a(ASIZE)
-        end
-    ])
-])
-AC_COMPILE_IFELSE([],[
-    pac_cv_f77_accepts_F=yes
-    ifelse([$1],[],[],[$1=""])
-],[
-    pac_cv_f77_accepts_F=no
-    ifelse([$1],[],[:],[$1="false"])
-])
-# Restore Fortran 77's ac_ext but not FFLAGS
-ac_ext="$saved_f77_ext"
-
-if test "$pac_cv_f77_accepts_F" != "yes" ; then
-    pac_cpp_f77="$ac_cpp -C -P conftest.F > conftest.$ac_ext"
-    PAC_RUNLOG_IFELSE([$pac_cpp_f77],[
-        if test -s conftest.${ac_ext} ; then
-            AC_COMPILE_IFELSE([],[
-                pac_cv_f77_accepts_F="no, use cpp"
-                ifelse([$1],[],[],[$1="$CPP -C -P"])
-            ],[])
-            rm -f conftest.${ac_ext}
-        fi
-    ],[])
-fi
-FFLAGS="$saved_FFLAGS"
-rm -f conftest.F
-AC_LANG_POP([Fortran 77])
-AC_MSG_RESULT([$pac_cv_f77_accepts_F])
-])
-dnl
-dnl /*D 
-dnl PAC_PROG_F77_CRAY_POINTER - Check if Fortran 77 supports Cray-style pointer.
-dnl                             If so, set pac_cv_prog_f77_has_pointer to yes
-dnl                             and find out if any extra compiler flag is
-dnl                             needed and set it as CRAYPTR_FFLAGS.
-dnl                             i.e. CRAYPTR_FFLAGS is meaningful only if
-dnl                             pac_cv_prog_f77_has_pointer = yes.
-dnl
-dnl Synopsis:
-dnl   PAC_PROG_F77_CRAY_POINTER([action-if-true],[action-if-false])
-dnl D*/
-AC_DEFUN([PAC_PROG_F77_CRAY_POINTER],[
-AC_CACHE_CHECK([whether Fortran 77 supports Cray-style pointer],
-pac_cv_prog_f77_has_pointer,[
-AC_LANG_PUSH([Fortran 77])
-AC_LANG_CONFTEST([
-    AC_LANG_PROGRAM([],[
-        integer M
-        pointer (MPTR,M)
-        data MPTR/0/
-    ])
-])
-saved_FFLAGS="$FFLAGS"
-pac_cv_prog_f77_has_pointer=no
-CRAYPTR_FFLAGS=""
-for ptrflag in '' '-fcray-pointer' ; do
-    FFLAGS="$saved_FFLAGS $ptrflag"
-    AC_COMPILE_IFELSE([], [
-        pac_cv_prog_f77_has_pointer=yes
-        CRAYPTR_FFLAGS="$ptrflag"
-        break
-    ])
-done
-dnl Restore FFLAGS first, since user may not want to modify FFLAGS
-FFLAGS="$saved_FFLAGS"
-dnl remove conftest after ac_lang_conftest
-rm -f conftest.$ac_ext
-AC_LANG_POP([Fortran 77])
-])
-if test "$pac_cv_prog_f77_has_pointer" = "yes" ; then
-    AC_MSG_CHECKING([for Fortran 77 compiler flag for Cray-style pointer])
-    if test "X$CRAYPTR_FFLAGS" != "X" ; then
-        AC_MSG_RESULT([$CRAYPTR_FFLAGS])
-    else
-        AC_MSG_RESULT([none])
-    fi
-    ifelse([$1],[],[:],[$1])
-else
-    ifelse([$2],[],[:],[$2])
-fi
-])
-dnl
-dnl
-dnl PAC_F77_INIT_WORKS_WITH_C
-dnl
-AC_DEFUN([PAC_F77_INIT_WORKS_WITH_C],[
-AC_REQUIRE([AC_HEADER_STDC])
-AC_MSG_CHECKING([whether Fortran init will work with C])
-pac_f_init_works_with_c=unknown
-AC_LANG_PUSH([Fortran 77])
-AC_COMPILE_IFELSE([
-    AC_LANG_SOURCE([
-        subroutine minit()
-        common /m1/ vc, vc2
-        character*1 vc(1,1), vc2(1)
-        common /m2/ vd
-        integer vd
-        save /m1/, /m2/
-        call minitc( vc, vc2, vd )
-        end
-    ])
-],[
-    PAC_RUNLOG([mv conftest.$OBJEXT pac_f77conftest.$OBJEXT])
-    saved_LIBS="$LIBS"
-    # This test checks if Fortran init can be done in pure C environment,
-    # i.e. no FLIBS in linking, so don't put FLIBS in LIBS below
-    dnl LIBS="pac_f77conftest.$OBJEXT $FLIBS $LIBS"
-    LIBS="pac_f77conftest.$OBJEXT $LIBS"
-    AC_LANG_PUSH([C])
-    AC_LINK_IFELSE([
-        AC_LANG_SOURCE([
-#if defined(HAVE_STDIO_H) || defined(STDC_HEADERS)
-#include <stdio.h>
-#endif
-#ifdef F77_NAME_UPPER
-#define minit_ MINIT
-#elif defined(F77_NAME_LOWER) || defined(F77_NAME_MIXED)
-#define minit_ minit
-#endif
-extern void minit_(void);
-int main( int argc, char **argv )
-{
-    minit_();
-    return 0;
-}
-char *v1 = 0;
-char *vc2 = 0;
-int  v2 = 0;
-void minitc_( char *dv1, int d, char *dv2, int d2, int dv3 );
-void minitc_( char *dv1, int d, char *dv2, int d2, int dv3 )
-{
-v1 = dv1;
-v2 = dv3;
-vc2 = dv2;
-*vc2 = ' ';
-}
-        ])
-    ],[pac_f_init_works_with_c=yes],[pac_f_init_works_with_c=no])
-    AC_LANG_POP([C])
-    LIBS="$saved_LIBS"
-    rm -f pac_f77conftest.$OBJEXT
-])
-AC_LANG_POP([Fortran 77])
-AC_MSG_RESULT([$pac_f_init_works_with_c])
-])
-dnl
-dnl PAC_F77_LOGICALS_IN_C(MPI_FINT)
-dnl
-dnl where MPI_FINT is the C type for Fortran integer.
-dnl
-dnl Use a Fortran main program.  This simplifies some steps, 
-dnl since getting all of the Fortran libraries (including shared 
-dnl libraries that are not in the default library search path) can 
-dnl be tricky.  Specifically, The PROG_F77_RUN_PROC_FROM_C failed with 
-dnl some installations of the Portland group compiler.
-dnl
-dnl We'd also like to check other values for .TRUE. and .FALSE. to see
-dnl if the compiler allows (or uses) more than one value (some DEC compilers,
-dnl for example, used the high (sign) bit to indicate true and false; the 
-dnl rest of the bits were ignored.  For now, we'll assume that there are 
-dnl unique true and false values.
-dnl
-AC_DEFUN([PAC_F77_LOGICALS_IN_C],[
-AC_REQUIRE([AC_HEADER_STDC])
-AC_REQUIRE([PAC_PROG_F77_NAME_MANGLE])
-pac_mpi_fint="$1"
-AC_MSG_CHECKING([for values of Fortran logicals])
-AC_CACHE_VAL(pac_cv_prog_f77_true_false_value,[
-pac_cv_prog_f77_true_false_value=""
-AC_LANG_PUSH([C])
-AC_COMPILE_IFELSE([
-    AC_LANG_SOURCE([
-#if defined(HAVE_STDIO_H) || defined(STDC_HEADERS)
-#include <stdio.h>
-#endif
-#if defined(HAVE_STDLIB_H) || defined(STDC_HEADERS)
-#include <stdlib.h>
-#endif
-#ifdef F77_NAME_UPPER
-#define ftest_ FTEST
-#elif defined(F77_NAME_LOWER) || defined(F77_NAME_MIXED)
-#define ftest_ ftest
-#endif
-void ftest_( $pac_mpi_fint *, $pac_mpi_fint *);
-void ftest_( $pac_mpi_fint *itrue, $pac_mpi_fint *ifalse )
-{
-  FILE *f = fopen("conftestval","w");
-  if (!f) exit(1);
-  fprintf( f, "%d %d\n", *itrue, *ifalse );
-  fclose(f);
-}
-    ])
-],[
-    PAC_RUNLOG([mv conftest.$OBJEXT pac_conftest.$OBJEXT])
-    saved_LIBS="$LIBS"
-    LIBS="pac_conftest.$OBJEXT $saved_LIBS"
-    AC_LANG_PUSH([Fortran 77])
-    AC_RUN_IFELSE([
-        AC_LANG_SOURCE([
-            program main
-            logical itrue, ifalse
-            itrue = .TRUE.
-            ifalse = .FALSE.
-            call ftest( itrue, ifalse )
-            end
-        ])
-    ],[
-        pac_cv_prog_f77_true_false_value="`cat conftestval`"
-    ],[
-        AC_MSG_WARN([Failed to build/run program to determine Fortran logical values.])
-    ],[
-        # Cross-Compiling.  Allow the user to set the values
-        if test -n "$CROSS_F77_TRUE_VALUE" -a -n "$CROSS_F77_FALSE_VALUE" ; then
-            pac_cv_prog_f77_true_false_value="$CROSS_F77_TRUE_VALUE $CROSS_F77_FALSE_VALUE"
-        else
-            AC_MSG_WARN([Either CROSS_F77_TRUE_VALUE="$CROSS_F77_TRUE_VALUE" or CROSS_F77_FALSE_VALUE="$CROSS_F77_FALSE_VALUE" is not set.])
-        fi
-    ])
-    AC_LANG_POP([Fortran 77])
-    LIBS="$saved_LIBS"
-    rm -f pac_conftest.$OBJEXT
-])
-AC_LANG_POP([C])
-])
-dnl Endof ac_cache_val
-if test "X$pac_cv_prog_f77_true_false_value" != "X" ; then
-    true_val="`echo $pac_cv_prog_f77_true_false_value | sed -e 's/ .*//g'`"
-    false_val="`echo $pac_cv_prog_f77_true_false_value | sed -e 's/.*  *//g'`"
-    if test -n "$true_val" -a -n "$false_val" ; then
-        AC_MSG_RESULT([True is $true_val and False is $false_val])
-    else
-        AC_MSG_RESULT([could not determine])
-    fi
-fi
-if test -n "$true_val" -a -n "$false_val" ; then
-    AC_DEFINE(F77_TRUE_VALUE_SET,1,[Define if we know the value of Fortran true and false])
-    AC_DEFINE_UNQUOTED(F77_TRUE_VALUE,$true_val,[The value of true in Fortran])
-    AC_DEFINE_UNQUOTED(F77_FALSE_VALUE,$false_val,[The value of false in Fortran])
-fi
-])
-dnl/*D
-dnl PAC_PROG_F77_MISMATCHED_ARGS([option],[AllOnly]) - Determine whether the 
-dnl Fortran compiler allows routines to be called with different 
-dnl argument types.  If not, attempts to determine a command-line argument 
-dnl that permits such use 
-dnl (The Fortran standard prohibits this usage)
-dnl
-dnl option is set to the compiler option to use.
-dnl if AllOnly is yes (literal, not variable with value), then only consider 
-dnl options that turn off checking
-dnl for all routines
-dnl
-dnl The NAG Fortran compiler, nagfor, is known to enforce this part of the
-dnl Fortran standard.
-dnl D*/
-AC_DEFUN([PAC_PROG_F77_MISMATCHED_ARGS],[
-AC_MSG_CHECKING([whether $F77 allows mismatched arguments])
-if test "X$pac_cv_prog_f77_mismatched_args" = X ; then
-    pac_cv_prog_f77_mismatched_args_parm=""
-    pac_cv_prog_f77_mismatched_args=no
-    AC_LANG_PUSH([Fortran 77])
-    AC_COMPILE_IFELSE([
-       AC_LANG_SOURCE([
-        program main
-        integer a
-        real b
-        character c
-        call foo1(a)
-        call foo1(b)
-        call foo1(c)
-        end
-])],[pac_cv_prog_f77_mismatched_args=yes])
-    if test "$pac_cv_prog_f77_mismatched_args" != "yes" ; then
-        # try again with -wmismatch=foo1
-        save_FFLAGS="$FFLAGS"
-	# The best solution is to turn off errors on particular routines
-	# if that isn't possible (e.g., too many of them), then
-	# just try arguments that turn off all checking
-	for flags in ifelse($2,yes,,"-wmismatch=foo1") "-mismatch" ; do
-            testok=no
-            FFLAGS="$FFLAGS $flags"
-            AC_COMPILE_IFELSE([
-            AC_LANG_SOURCE([
-        program main
-        integer a
-        real b
-        character c
-        call foo1(a)
-        call foo1(b)
-        call foo1(c)
-        end
-])],[testok=yes])
-            FFLAGS="$save_FFLAGS"
-            if test "$testok" = yes ; then break ; fi
-        done
-        if test "$testok" = yes ; then 
-	    pac_cv_prog_f77_mismatched_args_parm="$flags"
-            pac_cv_prog_f77_mismatched_args="yes, with $pac_cv_prog_f77_mismatched_args_parm"
-        fi
-    fi
-    AC_LANG_POP([Fortran 77])
-fi
-AC_MSG_RESULT($pac_cv_prog_f77_mismatched_args)
-if test "$pac_cv_prog_f77_mismatched_args" = no ; then
-    AC_MSG_ERROR([The Fortran compiler $F77 will not compile files that call 
-the same routine with arguments of different types.])
-fi
-
-ifelse($1,,,[$1=$pac_cv_prog_f77_mismatched_args_parm])
-])
diff --git a/3rd-party/romio321/confdb/aclocal_f77old.m4 b/3rd-party/romio321/confdb/aclocal_f77old.m4
deleted file mode 100644
index 5d6e2d9e70a..00000000000
--- a/3rd-party/romio321/confdb/aclocal_f77old.m4
+++ /dev/null
@@ -1,389 +0,0 @@
-dnl/*D
-dnl PAC_PROG_F77_CMDARGS - Determine how to access the command line from
-dnl Fortran 77
-dnl
-dnl Output Effects:
-dnl  The following variables are set:
-dnl.vb
-dnl    F77_GETARG         - Statement to get an argument i into string s
-dnl    F77_IARGC          - Routine to return the number of arguments
-dnl    FXX_MODULE         - Module command when using Fortran 90 compiler
-dnl    F77_GETARGDECL     - Declaration of routine used for F77_GETARG
-dnl    F77_GETARG_FFLAGS  - Flags needed when compiling/linking
-dnl    F77_GETARG_LDFLAGS - Flags needed when linking
-dnl.ve
-dnl If 'F77_GETARG' has a value, then that value and the values for these
-dnl other symbols will be used instead.  If no approach is found, all of these
-dnl variables will have empty values.
-dnl If no other approach works and a file 'f77argdef' is in the directory, 
-dnl that file will be sourced for the values of the above four variables.
-dnl
-dnl In most cases, you should add F77_GETARG_FFLAGS to the FFLAGS variable
-dnl and F77_GETARG_LDFLAGS to the LDFLAGS variable, to ensure that tests are
-dnl performed on the compiler version that will be used.
-dnl
-dnl 'AC_SUBST' is called for all six variables.
-dnl
-dnl One complication is that on systems with multiple Fortran compilers, 
-dnl some libraries used by one Fortran compiler may have been (mis)placed
-dnl in a common location.  We have had trouble with libg2c in particular.
-dnl To work around this, we test whether iargc etc. work first.  This
-dnl will catch most systems and will speed up the tests.
-dnl
-dnl Next, the libraries are only added if they are needed to complete a 
-dnl link; they aren''t added just because they exist.
-dnl
-dnl f77argdef
-dnl D*/
-dnl
-dnl Random notes
-dnl You can export the command line arguments from C to the g77 compiler
-dnl using
-dnl    extern char **__libc_argv;
-dnl    extern int  __libc_argc;
-dnl    f_setarg( __libc_argc, __libc_argv );
-dnl
-AC_DEFUN([PAC_PROG_F77_CMDARGS],[
-found_cached="yes"
-AC_MSG_CHECKING([for routines to access the command line from Fortran 77])
-AC_CACHE_VAL(pac_cv_prog_f77_cmdarg,
-[
-    AC_MSG_RESULT([searching...])
-    found_cached="no"
-    # First, we perform a quick check.  Does iargc and getarg work?
-    fxx_module="${FXX_MODULE:-}"
-    f77_getargdecl="${F77_GETARGDECL:-external getarg}"
-    f77_getarg="${F77_GETARG:-call GETARG(i,s)}"
-    f77_iargc="${F77_IARGC:-IARGC()}"
-    #    
-    # Grumble.  The Absoft Fortran compiler computes i - i as 0 and then
-    # 1.0 / 0 at compile time, even though the code may never be executed.
-    # What we need is a way to generate an error, so the second usage of i
-    # was replaced with f77_iargc.  
-    cat > conftest.f <<EOF
-        program main
-$fxx_module
-        integer i, j
-        character*20 s
-        $f77_getargdecl
-        i = 0
-        $f77_getarg
-        i=$f77_iargc
-        if (i .gt. 1) then
-            j = i - $f77_iargc
-            j = 1.0 / j
-        endif
-        end
-EOF
-    found_answer="no"
-    if test -z "$ac_fcompilelink" ; then
-        ac_fcompilelink="${F77-f77} -o conftest $FFLAGS $flags conftest.f $LDFLAGS $LIBS 1>&AC_FD_CC"
-    fi
-    AC_MSG_CHECKING([whether ${F77-f77} $flags $libs works with GETARG and IARGC])
-    if AC_TRY_EVAL(ac_fcompilelink) && test -x conftest ; then
-	# Check that cross != yes so that this works with autoconf 2.52
-	# Check that cross_compiling != yes so that this works with 
-	# autoconf 2.6x for some (but almost certainly not all)x
-	# Question: why do we test that this runs?  It looks like we
-	# needed this for some old Fortran compilers that produced
-	# executable code that then did not run.
-	if test "$ac_cv_prog_f77_cross" != "yes" -a \
-	        "$cross_compiling" != "yes" ; then
-	    if ./conftest >/dev/null 2>&1 ; then
-		found_answer="yes"
-	        FXX_MODULE="$fxx_module"
-		F77_GETARGDECL="$f77_getargdecl"
-		F77_GETARG="$f77_getarg"
-		F77_IARGC="$f77_iargc"
-		AC_MSG_RESULT(yes)
-     	    fi
-        fi
-    fi    
-    if test $found_answer = "no" ; then
-	AC_MSG_RESULT(no)
-    # Grumph.  Here are a bunch of different approaches
-    # We have several axes the check:
-    # Library to link with (none, -lU77 (HPUX), -lg2c (LINUX f77))
-    # PEPCF90 (Intel ifc)
-    # The first line is a dummy
-    # (we experimented with using a <space>, but this caused other 
-    # problems because we need <space> in the IFS)
-    trial_LIBS="0 -lU77 -lPEPCF90"
-    if test "$NOG2C" != "1" ; then
-        trial_LIBS="$trial_LIBS -lg2c"
-    fi
-    # Discard libs that are not availble:
-    save_IFS="$IFS"
-    # Make sure that IFS includes a space, or the tests that run programs
-    # may fail
-    IFS=" ""
-"
-    save_trial_LIBS="$trial_LIBS"
-    trial_LIBS=""
-    cat > conftest.f <<EOF
-        program main
-        end
-EOF
-    ac_fcompilelink_test='${F77-f77} -o conftest $FFLAGS conftest.f $LDFLAGS $libs $LIBS 1>&AC_FD_CC'
-    for libs in $save_trial_LIBS ; do
-	if test "$libs" = "0" ; then
-	    lib_ok="yes"
-        else
-	    AC_MSG_CHECKING([whether Fortran 77 links with $libs])
-	    if AC_TRY_EVAL(ac_fcompilelink_test) && test -x conftest ; then
-		AC_MSG_RESULT([yes])
-	        lib_ok="yes"
-	    else
-		AC_MSG_RESULT([no])
-	        lib_ok="no"
-	    fi
-	fi
-	if test "$lib_ok" = "yes" ; then
-	    trial_LIBS="$trial_LIBS
-$libs"
-        fi
-    done
-
-    # Options to use when compiling and linking
-    # +U77 is needed by HP Fortran to access getarg etc.
-    # The -N109 was used for getarg before we realized that GETARG
-    # was necessary with the (non standard conforming) Absoft compiler
-    # (Fortran is monocase; Absoft uses mixedcase by default)
-    # The -f is used by Absoft and is the compiler switch that folds 
-    # symbolic names to lower case.  Without this option, the compiler
-    # considers upper- and lower-case letters to be unique.
-    # The -YEXT_NAMES=LCS will cause external names to be output as lower
-    # case letter for Absoft F90 compilers (default is upper case)
-    # The first line is "<space><newline>, the space is important
-    # To make the Absoft f77 and f90 work together, we need to prefer the
-    # upper case versions of the arguments.  They also require libU77.
-    # -YCFRL=1 causes Absoft f90 to work with g77 and similar (f2c-based) 
-    # Fortran compilers
-    #
-    # Problem:  The Intel efc compiler hangs when presented with -N109 .
-    # The only real fix for this is to detect this compiler and exclude
-    # the test.  We may want to reorganize these tests so that if we
-    # can compile code without special options, we never look for them.
-    # 
-    using_intel_efc="no"
-    pac_test_msg=`$F77 -V 2>&1 | grep 'Intel(R) Fortran Itanium'`
-    if test "$pac_test_msg" != "" ; then
-	using_intel_efc="yes"
-    fi
-    if test "$using_intel_efc" = "yes" ; then
-        trial_FLAGS="000"
-    else
-        trial_FLAGS="000
--N109
--f
--YEXT_NAMES=UCS
--YEXT_NAMES=LCS
--YCFRL=1
-+U77"
-    fi
-    # Discard options that are not available:
-    # (IFS already saved above)
-    IFS=" ""
-"
-    save_trial_FLAGS="$trial_FLAGS"
-    trial_FLAGS=""
-    for flag in $save_trial_FLAGS ; do
-	if test "$flag" = " " -o "$flag" = "000" ; then
-	    opt_ok="yes"
-        else
-            PAC_F77_CHECK_COMPILER_OPTION($flag,opt_ok=yes,opt_ok=no)
-        fi
-	if test "$opt_ok" = "yes" ; then
-	    if test "$flag" = " " -o "$flag" = "000" ; then 
-		fflag="" 
-	    else 
-		fflag="$flag" 
-	    fi
-	    # discard options that don't allow mixed-case name matching
-	    cat > conftest.f <<EOF
-        program main
-        call aB()
-        end
-        subroutine Ab()
-        end
-EOF
-	    if test -n "$fflag" ; then flagval="with $fflag" ; else flagval="" ; fi
-	    AC_MSG_CHECKING([whether Fortran 77 routine names are case-insensitive $flagval])
-	    dnl we can use double quotes here because all is already
-            dnl evaluated
-            ac_fcompilelink_test="${F77-f77} -o conftest $fflag $FFLAGS conftest.f $LDFLAGS $LIBS 1>&AC_FD_CC"
-	    if AC_TRY_EVAL(ac_fcompilelink_test) && test -x conftest ; then
-	        AC_MSG_RESULT(yes)
-	    else
-	        AC_MSG_RESULT(no)
-	        opt_ok="no"
-            fi
-        fi
-        if test "$opt_ok" = "yes" ; then
-	    trial_FLAGS="$trial_FLAGS
-$flag"
-        fi
-    done
-    IFS="$save_IFS"
-    # Name of routines.  Since these are in groups, we use a case statement
-    # and loop until the end (accomplished by reaching the end of the
-    # case statement
-    # For one version of Nag F90, the names are 
-    # call f90_unix_MP_getarg(i,s) and f90_unix_MP_iargc().
-    trial=0
-    while test -z "$pac_cv_prog_f77_cmdarg" ; do
-        case $trial in 
-	0) # User-specified values, if any
-	   if test -z "$F77_GETARG" -o -z "$F77_IARGC" ; then 
-	       trial=`expr $trial + 1`
-	       continue 
-           fi
-           MSG="Using environment values of F77_GETARG etc."
-	   ;;
-	1) # Standard practice, uppercase (some compilers are case-sensitive)
-	   FXX_MODULE=""
-	   F77_GETARGDECL="external GETARG"
-	   F77_GETARG="call GETARG(i,s)"
-	   F77_IARGC="IARGC()"
-	   MSG="GETARG and IARGC"
-	   ;;
-	2) # Standard practice, lowercase
-	   FXX_MODULE=""
-           F77_GETARGDECL="external getarg"
-	   F77_GETARG="call getarg(i,s)"
-	   F77_IARGC="iargc()"
-	   MSG="getarg and iargc"
-	   ;;
-	3) # Posix alternative
-	   FXX_MODULE=""
-	   F77_GETARGDECL="external pxfgetarg"
-	   F77_GETARG="call pxfgetarg(i,s,l,ier)"
-	   F77_IARGC="ipxfargc()"
-	   MSG="pxfgetarg and ipxfargc"
-	   ;;
-	4) # Nag f90_unix_env module
-	   FXX_MODULE="        use f90_unix_env"
-	   F77_GETARGDECL=""
-	   F77_GETARG="call getarg(i,s)"
-	   F77_IARGC="iargc()"
-	   MSG="f90_unix_env module"
-	   ;;
-        5) # Nag f90_unix module
-	   FXX_MODULE="        use f90_unix"
-	   F77_GETARGDECL=""
-	   F77_GETARG="call getarg(i,s)"
-	   F77_IARGC="iargc()"
-	   MSG="f90_unix module"
-	   ;;
-	6) # user spec in a file
-	   if test -s f77argdef ; then
-		. ./f77argdef
-	       MSG="Using definitions in the file f77argdef"
-	   else
-	        trial=`expr $trial + 1`
-		continue
-	   fi
-	   ;;
-	7) # gfortran won't find getarg if it is marked as external 
-	   FXX_MODULE=""
-	   F77_GETARGDECL="intrinsic GETARG"
-	   F77_GETARG="call GETARG(i,s)"
-	   F77_IARGC="IARGC()"
-	   MSG="intrinsic GETARG and IARGC"
-	   ;;
-        *) # exit from while loop
-	   FXX_MODULE=""
-	   F77_GETARGDECL=""
-	   F77_GETARG=""
-	   F77_IARGC=""
-           break
-	   ;;
-	esac
-	# Create the program.  Make sure that we can run it.
-	# Force a divide-by-zero if there is a problem (but only at runtime!
-        # (the Absoft compiler does divide-by-zero at compile time)
-        cat > conftest.f <<EOF
-        program main
-$FXX_MODULE
-        integer i, j
-        character*20 s
-        $F77_GETARGDECL
-        i = 0
-        $F77_GETARG
-        i=$F77_IARGC
-        if (i .gt. 1) then
-            j = i - $F77_IARGC
-            j = 1.0 / j
-        endif
-        end
-EOF
-    #
-    # Now, try to find some way to compile and link that program, looping 
-    # over the possibilities of options and libraries
-        save_IFS="$IFS"
-        IFS=" ""
-"
-        for libs in $trial_LIBS ; do
-            if test -n "$pac_cv_prog_f77_cmdarg" ; then break ; fi
-	    if test "$libs" = " " -o "$libs" = "0" ; then libs="" ; fi
-            for flags in $trial_FLAGS ; do
-	        if test "$flags" = " " -o "$flags" = "000"; then flags="" ; fi
-                AC_MSG_CHECKING([whether ${F77-f77} $flags $libs works with $MSG])
-		IFS="$save_IFS"
-		dnl We need this here because we've fiddled with IFS
-	        ac_fcompilelink_test="${F77-f77} -o conftest $FFLAGS $flags conftest.f $LDFLAGS $libs $LIBS 1>&AC_FD_CC"
-		found_answer="no"
-                if AC_TRY_EVAL(ac_fcompilelink_test) && test -x conftest ; then
-		    if test "$ac_cv_prog_f77_cross" != "yes" -a \	 
-		            "$cross_compiling" != "yes" ; then
-			if ./conftest >/dev/null 2>&1 ; then
-			    found_answer="yes"
-			fi
-		    else 
-			found_answer="yes"
-		    fi
-                fi
-	        IFS=" ""
-"
-		if test "$found_answer" = "yes" ; then
-	            AC_MSG_RESULT([yes])
-		    pac_cv_prog_f77_cmdarg="$MSG"
-		    pac_cv_prog_f77_cmdarg_fflags="$flags"
-		    pac_cv_prog_f77_cmdarg_ldflags="$libs"
-		    break
-	        else
-                    AC_MSG_RESULT([no])
-		    echo "configure: failed program was:" >&AC_FD_CC
-                    cat conftest.f >&AC_FD_CC
-	        fi
-            done
-        done
-        IFS="$save_IFS"   
-	rm -f conftest.*
-        trial=`expr $trial + 1`   
-    done
-fi
-pac_cv_F77_GETARGDECL="$F77_GETARGDECL"
-pac_cv_F77_IARGC="$F77_IARGC"
-pac_cv_F77_GETARG="$F77_GETARG"
-pac_cv_FXX_MODULE="$FXX_MODULE"
-])
-if test "$found_cached" = "yes" ; then 
-    AC_MSG_RESULT([$pac_cv_prog_f77_cmdarg])
-elif test -z "$pac_cv_F77_IARGC" ; then
-    AC_MSG_WARN([Could not find a way to access the command line from Fortran 77])
-fi
-# Set the variable values based on pac_cv_prog_xxx
-F77_GETARGDECL="$pac_cv_F77_GETARGDECL"
-F77_IARGC="$pac_cv_F77_IARGC"
-F77_GETARG="$pac_cv_F77_GETARG"
-FXX_MODULE="$pac_cv_FXX_MODULE"
-F77_GETARG_FFLAGS="$pac_cv_prog_f77_cmdarg_fflags"
-F77_GETARG_LDFLAGS="$pac_cv_prog_f77_cmdarg_ldflags"
-AC_SUBST(F77_GETARGDECL)
-AC_SUBST(F77_IARGC)
-AC_SUBST(F77_GETARG)
-AC_SUBST(FXX_MODULE)
-AC_SUBST(F77_GETARG_FFLAGS)
-AC_SUBST(F77_GETARG_LDFLAGS)
-])
diff --git a/3rd-party/romio321/confdb/aclocal_fc.m4 b/3rd-party/romio321/confdb/aclocal_fc.m4
deleted file mode 100644
index c5e2d1fb0fb..00000000000
--- a/3rd-party/romio321/confdb/aclocal_fc.m4
+++ /dev/null
@@ -1,1235 +0,0 @@
-dnl PAC_FC_SEARCH_LIST - expands to a whitespace separated list of modern
-dnl fortran compilers for use with AC_PROG_FC that is more suitable for HPC
-dnl software packages
-AC_DEFUN([PAC_FC_SEARCH_LIST],
-         [ifort pgf90 pathf90 pathf95 xlf90 xlf95 xlf2003 gfortran f90 epcf90 f95 fort lf95 g95 ifc efc gfc])
-dnl /*D
-dnl PAC_PROG_FC([COMPILERS])
-dnl
-dnl COMPILERS is a space separated list of Fortran 77 compilers to search
-dnl for.  Fortran 95 isn't strictly backwards-compatible with Fortran 77,
-dnl but `f95' is worth trying.
-dnl
-dnl Compilers are ordered by
-dnl  1. F77, F90, F95
-dnl  2. Good/tested native compilers, bad/untested native compilers
-dnl  3. Wrappers around f2c go last.
-dnl
-dnl `fort77' is a wrapper around `f2c'.
-dnl It is believed that under HP-UX `fort77' is the name of the native
-dnl compiler.  On some Cray systems, fort77 is a native compiler.
-dnl frt is the Fujitsu F77 compiler.
-dnl pgf77 and pgf90 are the Portland Group F77 and F90 compilers.
-dnl xlf/xlf90/xlf95/xlf2003 are IBM (AIX) F77/F90/F95/F2003 compilers.
-dnl lf95 is the Lahey-Fujitsu compiler.
-dnl fl32 is the Microsoft Fortran "PowerStation" compiler.
-dnl af77 is the Apogee F77 compiler for Intergraph hardware running CLIX.
-dnl epcf90 is the "Edinburgh Portable Compiler" F90.
-dnl fort is the Compaq Fortran 90 (now 95) compiler for Tru64 and Linux/Alpha.
-dnl pathf90 is the Pathscale Fortran 90 compiler
-dnl ifort is another name for the Intel f90 compiler
-dnl efc - An older Intel compiler (?)
-dnl ifc - An older Intel compiler
-dnl fc  - A compiler on some unknown system.  This has been removed because
-dnl       it may also be the name of a command for something other than
-dnl       the Fortran compiler (e.g., fc=file system check!)
-dnl gfortran - The GNU Fortran compiler (not the same as g95) 
-dnl gfc - An alias for gfortran recommended in cygwin installations
-dnl NOTE: this macro suffers from a basically intractable "expanded before it
-dnl was required" problem when libtool is also used
-dnl D*/
-AC_DEFUN([PAC_PROG_FC],[
-PAC_PUSH_FLAG([FCFLAGS])
-AC_PROG_FC([m4_default([$1],[PAC_FC_SEARCH_LIST])])
-PAC_POP_FLAG([FCFLAGS])
-])
-dnl
-dnl PAC_FC_EXT checks for the default Fortran 90 program extension, f90 then f.
-dnl This could be replaced by AC_FC_SRCEXT but since AC_FC_SRCEXT
-dnl adds FCFLAGS_ext, which is used to modify FCFLAGS or Makefile.in.
-dnl So will do this later.
-dnl
-AC_DEFUN([PAC_FC_EXT],[
-AC_MSG_CHECKING([for extension for Fortran 90 programs])
-ac_fc_srcext="f90"
-AC_LANG_PUSH(Fortran)
-AC_COMPILE_IFELSE([
-    AC_LANG_PROGRAM()
-],[
-    AC_MSG_RESULT([f90])
-],[
-    ac_fc_srcext="f" 
-    AC_COMPILE_IFELSE([
-        AC_LANG_PROGRAM()
-    ],[
-        AC_MSG_RESULT([f])
-    ],[
-        AC_MSG_RESULT([unknown!])
-    ])
-])
-AC_LANG_POP(Fortran)
-])
-dnl
-dnl Internal routine for testing F90
-dnl PAC_PROG_FC_WORKS()
-dnl
-AC_DEFUN([PAC_PROG_FC_WORKS],[
-AC_REQUIRE([PAC_FC_EXT])
-AC_LANG_PUSH(Fortran)
-AC_MSG_CHECKING([whether the Fortran 90 compiler ($FC $FCFLAGS $LDFLAGS) works])
-AC_LINK_IFELSE([
-    AC_LANG_SOURCE([
-        program conftest
-        integer, dimension(10) :: n
-        end
-    ])
-],[
-    pac_cv_prog_fc_works="yes"
-    AC_MSG_RESULT([$pac_cv_prog_fc_works])
-    AC_MSG_CHECKING([whether the Fortran 90 compiler ($FC $FCFLAGS $LDFLAGS) is a cross-compiler])
-    AC_RUN_IFELSE([],
-    [pac_cv_prog_fc_cross="no"],
-    [pac_cv_prog_fc_cross="yes"],
-    [pac_cv_prog_fc_cross="$cross_compiling"]
-    )
-    AC_MSG_RESULT($pac_cv_prog_fc_cross)
-],[
-    pac_cv_prog_fc_works="no"
-    AC_MSG_WARN([installation or configuration problem: Fortran 90 compiler cannot create executables.])
-])
-# The intel compiler sometimes generates these work.pc and .pcl files
-rm -f work.pc work.pcl
-AC_LANG_POP(Fortran)
-dnl cross_compiling no longer maintained by autoconf as part of the
-dnl AC_LANG changes.  If we set it here, a later AC_LANG may not 
-dnl restore it (in the case where one compiler claims to be a cross compiler
-dnl and another does not)
-dnl cross_compiling=$pac_cv_prog_f90_cross
-])
-dnl/*D 
-dnl PAC_PROG_FC_INT_KIND - Determine kind parameter for an integer with
-dnl the specified number of bytes.
-dnl
-dnl Synopsis:
-dnl  PAC_PROG_FC_INT_KIND(variable-to-set,number-of-bytes,[cross-size])
-dnl
-dnl D*/
-AC_DEFUN([PAC_PROG_FC_INT_KIND],[
-# Set the default
-$1=-1
-if test "$pac_cv_prog_fc_cross" = "yes" ; then
-    AS_IF([test -z "$3"],[AC_MSG_ERROR(['$3' is empty])])
-    $1="$3"
-else
-    AC_LANG_PUSH(Fortran)
-    AC_MSG_CHECKING([for Fortran 90 integer kind for $2-byte integers])
-    # Convert bytes to digits
-    case $2 in 
-        1) sellen=2 ;;
-        2) sellen=4 ;;
-        4) sellen=8 ;;
-        8) sellen=16 ;;
-       16) sellen=30 ;;
-        *) sellen=8 ;;
-    esac
-    # Check for cached value
-    eval testval=\$"pac_cv_prog_fc_int_kind_$sellen"
-    if test -n "$testval" ; then 
-        AC_MSG_RESULT([$testval (cached)])
-        $1=$testval
-    else
-        KINDVAL="unavailable"
-        eval "pac_cv_prog_fc_int_kind_$sellen"=-1
-        AC_RUN_IFELSE([
-            AC_LANG_SOURCE([
-                program main
-                integer ii
-                ii = selected_int_kind($sellen)
-                open(8, file="conftest1.out", form="formatted")
-                write (8,*) ii
-                close(8)
-                stop
-                end
-            ])
-        ],[pac_run_ok=yes],[pac_run_ok=no])
-        if test "$pac_run_ok" = "yes" ; then
-            if test -s conftest1.out ; then
-                # Because of write, there may be a leading blank.
-                KINDVAL=`cat conftest1.out | sed 's/ //g'`
-                eval "pac_cv_prog_fc_int_kind_$sellen"=$KINDVAL
-                $1=$KINDVAL
-            fi
-        fi
-        AC_MSG_RESULT([$KINDVAL])
-    fi # not cached
-    AC_LANG_POP(Fortran)
-fi # is not cross compiling
-])dnl
-dnl
-dnl ------------------------------------------------------------------------
-dnl Special characteristics that have no autoconf counterpart but that
-dnl we need as part of the Fortran 90 support.  To distinquish these, they
-dnl have a [PAC] prefix.
-dnl 
-dnl At least one version of the Cray compiler needs the option -em to
-dnl generate a separate module file, rather than including the module
-dnl information in the object (.o) file.
-dnl
-dnl
-dnl PAC_FC_MODULE_EXT(action if found,action if not found)
-dnl
-AC_DEFUN([PAC_FC_MODULE_EXT],
-[AC_CACHE_CHECK([for Fortran 90 module extension],
-pac_cv_fc_module_ext,[
-pac_cv_fc_module_case="unknown"
-AC_LANG_PUSH(Fortran)
-AC_COMPILE_IFELSE([
-    AC_LANG_SOURCE([
-        module conftest
-        integer n
-        parameter (n=1)
-        end module conftest
-    ])
-],[
-    # Look for module name
-    # First, try to find known names.  This avoids confusion caused by
-    # additional files (like <name>.stb created by some versions of pgf90)
-    # Early versions of the Intel compiler used d as the module extension;
-    # we include that just to finish the test as early as possible.
-    for name in conftest CONFTEST ; do
-        for ext in mod MOD d ; do
-            if test -s $name.$ext ; then
-                if test $name = conftest ; then
-                    pac_cv_fc_module_case=lower
-                else
-                    pac_cv_fc_module_case=upper
-                fi
-                pac_cv_fc_module_ext=$ext
-                pac_MOD=$ext
-                break
-            fi
-        done
-        if test -n "$pac_cv_fc_module_ext" ; then break ; fi
-    done
-    if test -z "$pac_MOD" ; then
-        # The test on .err is needed for Cray Fortran.
-        pac_MOD=`ls conftest.* 2>&1 | grep -v conftest.${ac_fc_srcext} | grep -v conftest.o | grep -v conftest.err`
-        pac_MOD=`echo $pac_MOD | sed -e 's/conftest\.//g'`
-        pac_cv_fc_module_case="lower"
-        if test "X$pac_MOD" = "X" ; then
-            pac_MOD=`ls CONFTEST* 2>&1 | grep -v CONFTEST.${ac_fc_srcext} | grep -v CONFTEST.o | grep -v CONFTEST.err`
-            pac_MOD=`echo $pac_MOD | sed -e 's/CONFTEST\.//g'`
-            if test -n "$pac_MOD" -a -s "CONFTEST.$pac_MOD" ; then
-                pac_cv_fc_module_case="upper"
-            else
-                # Clear because we must have gotten an error message
-                pac_MOD=""
-            fi
-        fi
-        if test -z "$pac_MOD" ; then 
-            pac_cv_fc_module_ext="unknown"
-        else
-            pac_cv_fc_module_ext=$pac_MOD
-        fi
-    fi
-],[
-    pac_cv_fc_module_ext="unknown"
-])
-
-if test "$pac_cv_fc_module_ext" = "unknown" ; then
-    # Try again, but with an -em option.  Abbreviated, because we're
-    # just looking for the Cray option
-    saveFCFLAGS=$FCFLAGS
-    FCFLAGS="$FCFLAGS -em"
-    AC_COMPILE_IFELSE([
-    AC_LANG_SOURCE([
-        module conftest
-        integer n
-        parameter (n=1)
-        end module conftest
-    ])
-],[
-    if test -s conftest.mod ; then
-        pac_cv_fc_module_ext="mod"
-        pac_cv_fc_module_case="lower"
-    elif test -s CONFTEST.mod ; then
-        pac_cv_fc_module_ext="mod"
-        pac_cv_fc_module_case="upper"
-    fi
-],[
-    :
-    # do nothing - already have the unknown default value
-])
-    if test "$pac_cv_fc_module_ext" = "unknown" ; then
-        # The additional command line option did not help - restore
-        # the original flags.
-        FCFLAGS=$saveFCFLAGS
-    fi
-fi
-AC_LANG_POP(Fortran)
-])
-#
-AC_SUBST(FCMODEXT)
-if test "$pac_cv_fc_module_ext" = "unknown" ; then
-    ifelse($2,,:,[$2])
-else
-    ifelse($1,,FCMODEXT=$pac_MOD,[$1])
-fi
-])
-dnl
-dnl
-dnl PAC_FC_MODULE_INCFLAG
-AC_DEFUN([PAC_FC_MODULE_INCFLAG],[
-AC_REQUIRE([PAC_FC_MODULE_EXT])
-AC_CACHE_CHECK([for Fortran 90 module include flag],
-pac_cv_fc_module_incflag,[
-AC_LANG_PUSH(Fortran)
-AC_LANG_CONFTEST([
-    AC_LANG_SOURCE([
-        module conf
-        integer n
-        parameter (n=1)
-        end module conf
-    ])
-])
-pac_madedir="no"
-if test ! -d conf ; then mkdir conftestdir ; pac_madedir="yes"; fi
-if test "$pac_cv_fc_module_case" = "upper" ; then
-    pac_module="CONF.$pac_cv_fc_module_ext"
-else
-    pac_module="conf.$pac_cv_fc_module_ext"
-fi
-AC_COMPILE_IFELSE([],[
-    if test -s "$pac_module" ; then
-        mv $pac_module conftestdir
-        # Remove any temporary files, and hide the work.pc file
-        # (if the compiler generates them)
-        if test -f work.pc ; then 
-            mv -f work.pc conftest.pc
-        fi
-        rm -f work.pcl
-    else
-        AC_MSG_WARN([Unable to build a simple Fortran 90 module])
-        # echo "configure: failed program was:" >&AS_MESSAGE_LOG_FD
-        # cat conftest.$ac_ext >&AS_MESSAGE_LOG_FD
-        _AC_MSG_LOG_CONFTEST
-    fi
-],[])
-# Remove the conftest* after AC_LANG_CONFTEST
-rm -rf conftest.dSYM
-rm -f conftest.$ac_ext
-
-dnl Create the conftest here so the test isn't created everytime inside loop.
-AC_LANG_CONFTEST([AC_LANG_PROGRAM([],[use conf])])
-
-# Save the original FCFLAGS
-saved_FCFLAGS="$FCFLAGS"
-pac_cv_fc_module_incflag=""
-for inchdr in '-I' '-M' '-p' ; do
-    FCFLAGS="$saved_FCFLAGS ${inchdr}conftestdir"
-    AC_COMPILE_IFELSE([],[pac_cv_fc_module_incflag="$inchdr" ; break])
-done
-if test "X$pac_cv_fc_module_incflag" = "X" ; then
-    if test -s conftest.pc ; then
-        mv conftest.pc conftestdir/mpimod.pc
-        echo "mpimod.pc" > conftestdir/mpimod.pcl
-        echo "`pwd`/conftestdir/mpimod.pc" >> conftestdir/mpimod.pcl
-        inchdr='-cl,'
-        FCFLAGS="$save_FCFLAGS ${inchdr}conftestdir"
-        AC_COMPILE_IFELSE([], [pac_fcompile_ok=yes], [pac_fcompile_ok=no])
-        if test "$pac_fcompile_ok" = "yes" ; then
-            pac_cv_fc_module_incflag="$inchdr"
-            # Not quite right; see the comments that follow
-            AC_MSG_RESULT([-cl,filename where filename contains a list of files and directories])
-            FC_WORK_FILES_ARG="-cl,mpimod.pcl"
-            FCMODINCSPEC="-cl,<dir>/<file>mod.pcl"
-        else 
-            # The version of the Intel compiler that I have refuses to let
-            # you put the "work catalog" list anywhere but the current directory.
-            pac_cv_fc_module_incflag="Unavailable!"
-        fi
-    else
-        # Early versions of the Intel ifc compiler required a *file*
-        # containing the names of files that contained the names of the
-        # 
-        # -cl,filename.pcl
-        #   filename.pcl contains
-        #     fullpathname.pc
-        # The "fullpathname.pc" is generated, I believe, when a module is 
-        # compiled.  
-        # Intel compilers use a wierd system: -cl,filename.pcl .  If no file is
-        # specified, work.pcl and work.pc are created.  However, if you specify
-        # a file, it must contain the name of a file ending in .pc .  Ugh!
-        pac_cv_fc_module_incflag="unknown"
-    fi
-fi
-# Restore the original FCFLAGS
-FCFLAGS="$saved_FCFLAGS"
-if test "$pac_madedir" = "yes" ; then rm -rf conftestdir ; fi
-# Remove the conftest* after AC_LANG_CONFTEST
-# This is needed for Mac OSX 10.5
-rm -rf conftest.dSYM
-rm -f conftest*
-AC_LANG_POP(Fortran)
-])
-AC_SUBST(FC_WORK_FILES_ARG)
-AC_SUBST(FCMODINCFLAG)
-FCMODINCFLAG=$pac_cv_fc_module_incflag
-])
-dnl
-dnl
-dnl
-AC_DEFUN([PAC_FC_MODULE],[
-PAC_FC_MODULE_EXT
-PAC_FC_MODULE_INCFLAG
-PAC_FC_MODULE_OUTFLAG
-])
-dnl
-dnl PAC_FC_MODULE_OUTFLAG
-AC_DEFUN([PAC_FC_MODULE_OUTFLAG],[
-AC_REQUIRE([PAC_FC_MODULE_EXT])
-AC_CACHE_CHECK([for Fortran 90 module output directory flag],
-               [pac_cv_fc_module_outflag],
-[
-AC_LANG_PUSH([Fortran])
-AC_LANG_CONFTEST([
-    AC_LANG_SOURCE([
-        module conf
-        integer n
-        parameter (n=1)
-        end module conf
-    ])
-])
-pac_madedir="no"
-if test ! -d conf ; then mkdir conftestdir ; pac_madedir="yes"; fi
-if test "$pac_cv_fc_module_case" = "upper" ; then
-    pac_module="CONF.$pac_cv_fc_module_ext"
-else
-    pac_module="conf.$pac_cv_fc_module_ext"
-fi
-
-# check base case that the compiler can create modules and that they endup in
-# the current directory
-AC_COMPILE_IFELSE([],[
-    if test -s "$pac_module" ; then
-        rm -f "$pac_module"
-        # Remove any temporary files, and hide the work.pc file
-        # (if the compiler generates them)
-        if test -f work.pc ; then 
-            mv -f work.pc conftest.pc
-        fi
-        rm -f work.pcl
-    else
-        AC_MSG_WARN([Unable to build a simple Fortran 90 module])
-        # echo "configure: failed program was:" >&AS_MESSAGE_LOG_FD
-        # cat conftest.$ac_ext >&AS_MESSAGE_LOG_FD
-        _AC_MSG_LOG_CONFTEST
-    fi
-],[])
-
-# known flags for reasonably recent versions of various f90 compilers:
-#   gfortran -J${dir}
-#   xlf -qmoddir=${dir}
-#   pgf90 -module ${dir}
-#   ifort -module ${dir}
-#   nagfor -mdir ${dir}
-#   ftn -J ${dir}              ## the Cray fortran compiler
-#   ftn -em -J${dir}           ## the Cray fortran compiler (crayftn, in 2013)
-#      For this above case, we must have added -em to FCFLAGS, since other
-#      module tests do not always use the module output flag.  See
-#      FC_MODULE_EXT , where this is determined.
-#   f95 -YMOD_OUT_DIR=${dir}   ## the Absoft fortran compiler
-#   lf95 -M ${dir}             ## the Lahey/Fujitsu fortran compiler
-#   f90 -moddir=${dir}         ## the Sun f90 compiler
-#   g95 -fmod=${dir}
-#
-# If there are any compilers still out there that are totally brain-dead and
-# don't support an output directory flag, we can write a wrapper script to tell
-# users to use.  Alternatively they can use an older version of MPICH.
-
-pac_cv_fc_module_outflag=
-for mod_flag in '-J' '-J ' '-qmoddir=' '-module ' '-YMOD_OUT_DIR=' '-mdir ' '-moddir=' '-fmod=' '-M '; do
-    rm -f conftestdir/NONEXISTENT conftestdir/*
-    PAC_PUSH_FLAG([FCFLAGS])
-    FCFLAGS="$FCFLAGS ${mod_flag}conftestdir"
-    AC_COMPILE_IFELSE([],[pac_build_success=yes],[pac_build_success=no])
-    AS_IF([test "X$pac_build_success" = Xyes],
-          [AS_IF([test -s "conftestdir/${pac_module}"],
-                 [pac_cv_fc_module_outflag="$mod_flag"])])
-    PAC_POP_FLAG([FCFLAGS])
-    AS_IF([test "X$pac_cv_fc_module_outflag" = X],[:],[break])
-done
-
-# Remove the conftest* after AC_LANG_CONFTEST
-rm -rf conftest.dSYM
-rm -f conftest.$ac_ext
-
-if test "$pac_madedir" = "yes" ; then rm -rf conftestdir ; fi
-AS_UNSET([pac_madedir])
-# Remove the conftest* after AC_LANG_CONFTEST
-# This is needed for Mac OSX 10.5
-rm -rf conftest.dSYM
-rm -f conftest*
-AC_LANG_POP(Fortran)
-])dnl end AC_CACHE_CHECK
-
-AC_SUBST([FCMODOUTFLAG],[$pac_cv_fc_module_outflag])
-])dnl end AC_DEFUN([PAC_FC_MODULE_OUTFLAG])
-dnl
-dnl PAC_FC_AND_F77_COMPATIBLE([action-if-true],[action-if-false])
-dnl
-dnl Determine whether object files compiled with Fortran 77 can be 
-dnl linked to Fortran 90 main programs.
-dnl
-dnl The test uses a name that includes an underscore unless the 3rd
-dnl argument provides another routine name.
-dnl
-AC_DEFUN([PAC_FC_AND_F77_COMPATIBLE],[
-AC_REQUIRE([PAC_PROG_FC_WORKS])
-AC_CACHE_CHECK([whether Fortran 90 compiler works with Fortran 77 compiler],
-pac_cv_fc_and_f77,[
-pacTestRoutine=foo_abc
-ifelse([$3],,,[eval pacTestRoutine=$3])
-pac_cv_fc_and_f77="unknown"
-# compile the f77 program and link with the f90 program
-# The reverse may not work because the Fortran 90 environment may
-# expect to be in control (and to provide library files unknown to any other
-# environment, even Fortran 77!)
-AC_LANG_PUSH(Fortran 77)
-AC_COMPILE_IFELSE([
-    AC_LANG_SOURCE([
-        subroutine ${pacTestRoutine}(b)
-        integer b
-        b = b + 1
-        end
-    ])
-],[
-    # pac_f77compile_ok=yes
-    PAC_RUNLOG([mv conftest.$OBJEXT pac_f77conftest.$OBJEXT])
-    # Save original LIBS, prepend previously generated object file to LIBS
-    saved_LIBS="$LIBS"
-    LIBS="pac_f77conftest.$OBJEXT $LIBS"
-    AC_LANG_PUSH(Fortran)
-    AC_LINK_IFELSE([
-        AC_LANG_SOURCE([
-            program main
-            integer a
-            a = 1
-            call ${pacTestRoutine}(a)
-            end
-        ])
-    ],[pac_cv_fc_and_f77=yes],[pac_cv_fc_and_f77=no])
-    # Some versions of the Intel compiler produce these two files
-    rm -f work.pc work.pcl
-    # Restore LIBS
-    LIBS="$saved_LIBS"
-    AC_LANG_POP(Fortran)
-    # remove previously generated object file.
-    rm -f pac_f77conftest.$OBJEXT
-], [
-    # pac_f77compile_ok=no
-    pac_cv_fc_and_f77=no
-])
-AC_LANG_POP(Fortran 77)
-# Perform the requested action based on whether the test succeeded
-if test "$pac_cv_fc_and_f77" = yes ; then
-    ifelse($1,,:,[$1])
-else
-    ifelse($2,,:,[$2])
-    AC_MSG_WARN([See config.log for the failed test program and its output.])
-fi
-])
-dnl
-])
-dnl
-dnl
-dnl /*D 
-dnl PAC_PROG_FC_CRAY_POINTER - Check if Fortran supports Cray-style pointer.
-dnl                            If so, set pac_cv_prog_fc_has_pointer to yes
-dnl                            and find out if any extra compiler flag is
-dnl                            needed and set it as CRAYPTR_FCFLAGS.
-dnl                            i.e. CRAYPTR_FCFLAGS is meaningful only if
-dnl                            pac_cv_prog_fc_has_pointer = yes.
-dnl
-dnl Synopsis:
-dnl   PAC_PROG_FC_CRAY_POINTER([action-if-true],[action-if-false])
-dnl D*/
-AC_DEFUN([PAC_PROG_FC_CRAY_POINTER],[
-AC_CACHE_CHECK([whether Fortran 90 supports Cray-style pointer],
-pac_cv_prog_fc_has_pointer,[
-AC_LANG_PUSH([Fortran])
-AC_LANG_CONFTEST([
-    AC_LANG_PROGRAM([],[
-        integer M
-        pointer (MPTR,M)
-        data MPTR/0/
-    ])
-])
-saved_FCFLAGS="$FCFLAGS"
-pac_cv_prog_fc_has_pointer=no
-CRAYPTR_FCFLAGS=""
-for ptrflag in '' '-fcray-pointer' ; do
-    FCFLAGS="$saved_FCFLAGS $ptrflag"
-    AC_COMPILE_IFELSE([],[
-        pac_cv_prog_fc_has_pointer=yes
-        CRAYPTR_FCFLAGS="$ptrflag"
-        break
-    ])
-done
-dnl Restore FCFLAGS first, since user may not want to modify FCFLAGS
-FCFLAGS="$saved_FCFLAGS"
-dnl remove conftest after ac_lang_conftest
-rm -f conftest.$ac_ext
-AC_LANG_POP([Fortran])
-])
-if test "$pac_cv_prog_fc_has_pointer" = "yes" ; then
-    AC_MSG_CHECKING([for Fortran 90 compiler flag for Cray-style pointer])
-    if test "X$CRAYPTR_FCFLAGS" != "X" ; then
-        AC_MSG_RESULT([$CRAYPTR_FCFLAGS])
-    else
-        AC_MSG_RESULT([none])
-    fi
-    ifelse([$1],[],[:],[$1])
-else
-    ifelse([$2],[],[:],[$2])
-fi
-])
-dnl
-dnl
-dnl
-AC_DEFUN([PAC_PROG_FC_AND_C_STDIO_LIBS],[
-AC_REQUIRE([AC_HEADER_STDC])
-# To simply the code in the cache_check macro, chose the routine name
-# first, in case we need it
-confname=conf1_
-case "$pac_cv_prog_f77_name_mangle" in
-    "lower underscore")       confname=conf1_  ;;
-    "upper stdcall")          confname=CONF1   ;;
-    upper)                    confname=CONF1   ;;
-    "lower doubleunderscore") confname=conf1_  ;;
-    lower)                    confname=conf1   ;;
-    "mixed underscore")       confname=conf1_  ;;
-    mixed)                    confname=conf1   ;;
-esac
-
-AC_CACHE_CHECK([what libraries are needed to link Fortran90 programs with C routines that use stdio],pac_cv_prog_fc_and_c_stdio_libs,[
-pac_cv_prog_fc_and_c_stdio_libs=unknown
-
-AC_LANG_PUSH(C)
-AC_COMPILE_IFELSE([
-    AC_LANG_SOURCE([
-#if defined(HAVE_STDIO_H) || defined(STDC_HEADERS)
-#include <stdio.h>
-#endif
-int $confname( int a )
-{ printf( "The answer is %d\n", a ); fflush(stdout); return 0; }
-    ])
-],[
-    pac_compile_ok=yes
-    PAC_RUNLOG([mv conftest.$OBJEXT pac_conftest.$OBJEXT])
-    # Save LIBS and prepend object file to LIBS
-    saved_LIBS="$LIBS"
-    LIBS="pac_conftest.$OBJEXT $LIBS"
-    AC_LANG_PUSH(Fortran)
-    AC_LINK_IFELSE([
-        AC_LANG_PROGRAM([],[call conf1(0)])
-    ],[
-        pac_cv_prog_fc_and_c_stdio_libs=none
-    ],[
-        # Try again with -lSystemStubs
-        LIBS="$LIBS -lSystemStubs"
-        AC_LINK_IFELSE([],[
-            pac_cv_prog_fc_and_c_stdio_libs="-lSystemStubs"
-        ],[])
-    ])
-    LIBS="$saved_LIBS"
-    AC_LANG_POP(Fortran)
-    rm -f pac_conftest.$OBJEXT
-])
-AC_LANG_POP(C)
-dnl
-if test "$pac_cv_prog_fc_and_c_stdio_libs" != none -a \
-        "$pac_cv_prog_fc_and_c_stdio_libs" != unknown ; then
-    FC_OTHER_LIBS="$FC_OTHER_LIBS $pac_cv_prog_fc_and_c_stdio_libs"    
-fi
-])
-dnl
-])
-dnl
-dnl/*D
-dnl PAC_FC_CHECK_COMPILER_OPTION - Check that a FC compiler option is
-dnl accepted without warning messages
-dnl
-dnl Synopsis:
-dnl PAC_FC_CHECK_COMPILER_OPTION(optionname,action-if-ok,action-if-fail)
-dnl
-dnl Output Effects:
-dnl
-dnl If no actions are specified, a working value is added to 'FCOPTIONS'
-dnl
-dnl Notes:
-dnl This is now careful to check that the output is different, since 
-dnl some compilers are noisy.
-dnl 
-dnl We are extra careful to prototype the functions in case compiler options
-dnl that complain about poor code are in effect.
-dnl
-dnl Because this is a long script, we have ensured that you can pass a 
-dnl variable containing the option name as the first argument.
-dnl D*/
-AC_DEFUN([PAC_FC_CHECK_COMPILER_OPTION],[
-AC_MSG_CHECKING([whether Fortran 90 compiler accepts option $1])
-pac_opt="$1"
-AC_LANG_PUSH(Fortran)
-FCFLAGS_orig="$FCFLAGS"
-FCFLAGS_opt="$pac_opt $FCFLAGS"
-pac_result="unknown"
-
-AC_LANG_CONFTEST([AC_LANG_PROGRAM()])
-FCFLAGS="$FCFLAGS_orig"
-rm -f pac_test1.log
-PAC_LINK_IFELSE_LOG([pac_test1.log], [], [
-    FCFLAGS="$FCFLAGS_opt"
-    rm -f pac_test2.log
-    PAC_LINK_IFELSE_LOG([pac_test2.log], [], [
-        PAC_RUNLOG_IFELSE([diff -b pac_test1.log pac_test2.log],
-                          [pac_result=yes], [pac_result=no])
-    ],[
-        pac_result=no
-    ])
-], [
-    pac_result=no
-])
-AC_MSG_RESULT([$pac_result])
-dnl Delete the conftest created by AC_LANG_CONFTEST.
-rm -f conftest.$ac_ext
-#
-if test "$pac_result" = "yes" ; then
-    AC_MSG_CHECKING([whether routines compiled with $pac_opt can be linked with ones compiled without $pac_opt])
-    pac_result=unknown
-    FCFLAGS="$FCFLAGS_orig"
-    rm -f pac_test3.log
-    PAC_COMPILE_IFELSE_LOG([pac_test3.log], [
-        AC_LANG_SOURCE([
-            subroutine try()
-            end
-        ])
-    ],[
-        PAC_RUNLOG([mv conftest.$OBJEXT pac_conftest.$OBJEXT])
-        saved_LIBS="$LIBS"
-        LIBS="pac_conftest.$OBJEXT $LIBS"
-
-        FCFLAGS="$FCFLAGS_opt"
-        rm -f pac_test4.log
-        PAC_LINK_IFELSE_LOG([pac_test4.log], [AC_LANG_PROGRAM()], [
-            PAC_RUNLOG_IFELSE([diff -b pac_test2.log pac_test4.log],
-                              [pac_result=yes], [pac_result=no])
-        ],[
-            pac_result=no
-        ])
-        LIBS="$saved_LIBS"
-        rm -f pac_conftest.$OBJEXT
-    ],[
-        pac_result=no
-    ])
-    AC_MSG_RESULT([$pac_result])
-    rm -f pac_test3.log pac_test4.log
-fi
-rm -f pac_test1.log pac_test2.log
-
-dnl Restore FCFLAGS before 2nd/3rd argument commands are executed,
-dnl as 2nd/3rd argument command could be modifying FCFLAGS.
-FCFLAGS="$FCFLAGS_orig"
-if test "$pac_result" = "yes" ; then
-     ifelse([$2],[],[FCOPTIONS="$FCOPTIONS $1"],[$2])
-else
-     ifelse([$3],[],[:],[$3])
-fi
-AC_LANG_POP(Fortran)
-])
-dnl /*D
-dnl PAC_FC_WORKS_WITH_CPP
-dnl
-dnl Checks if Fortran 90 compiler works with C preprocessor
-dnl
-dnl Most systems allow the Fortran compiler to process .F and .F90 files
-dnl using the C preprocessor.  However, some systems either do not
-dnl allow this or have serious bugs (OSF Fortran compilers have a bug
-dnl that generates an error message from cpp).  The following test
-dnl checks to see if .F works, and if not, whether "cpp -P -C" can be used
-dnl D*/
-AC_DEFUN([PAC_FC_WORKS_WITH_CPP],[
-AC_REQUIRE([AC_PROG_CPP])
-AC_MSG_CHECKING([whether Fortran 90 compiler processes .F90 files with C preprocessor])
-AC_LANG_PUSH([Fortran])
-saved_fc_ext=${ac_ext}
-ac_ext="F90"
-saved_FCFLAGS="$FCFLAGS"
-FCFLAGS="$FCFLAGS $CPPFLAGS"
-AC_LANG_CONFTEST([
-    AC_LANG_SOURCE([
-        program main
-#define ASIZE 10
-        integer a(ASIZE)
-        end
-    ])
-])
-AC_COMPILE_IFELSE([],[
-    pac_cv_fc_accepts_F90=yes
-    ifelse([$1],[],[],[$1=""])
-],[
-    pac_cv_fc_accepts_F90=no
-    ifelse([$1],[],[:],[$1="false"])
-])
-# Restore Fortran's ac_ext but not FCFLAGS
-ac_ext="$saved_fc_ext"
-
-if test "$pac_cv_fc_accepts_F90" != "yes" ; then
-    pac_cpp_fc="$ac_cpp -C -P conftest.F90 > conftest.$ac_ext"
-    PAC_RUNLOG_IFELSE([$pac_cpp_fc],[
-        if test -s conftest.${ac_ext} ; then
-            AC_COMPILE_IFELSE([],[
-                pac_cv_fc_accepts_F90="no, use cpp"
-                ifelse([$1],[],[],[$1="$CPP -C -P"])
-            ],[])
-            rm -f conftest.${ac_ext}
-        fi
-    ],[])
-fi
-FCFLAGS="$saved_FCFLAGS"
-rm -f conftest.F90
-AC_LANG_POP([Fortran])
-AC_MSG_RESULT([$pac_cv_fc_accepts_F90])
-])
-dnl
-dnl PAC_FC_VENDOR:
-dnl Try to get a version string for the F90 compiler.  We may
-dnl need this to find likely command-line arguments for accessing
-dnl shared libraries
-dnl
-AC_DEFUN([PAC_FC_VENDOR],[
-AC_MSG_CHECKING([for Fortran 90 compiler vendor])
-# This is complicated by some compilers (such as the Intel 8.1 ifort)
-# that return a non-zero status even when they accept the -V option
-# (a zero status is returned only if there is a file).
-pac_cv_fc_vendor="unknown"
-for arg in --version -V -v ; do
-    rm -f conftest.txt
-    PAC_RUNLOG([$FC $arg </dev/null >conftest.txt 2>&1])
-    # Ignore the return code, because some compilers set the
-    # return code to zero on invalid arguments and some to 
-    # non-zero on success (with no files to compile)
-    if test -f conftest.txt ; then
-        if grep 'Portland Group' conftest.txt >/dev/null 2>&1 ; then
-            pac_cv_fc_vendor=pgi
-        elif grep 'Sun Workshop' conftest.txt >/dev/null 2>&1 ; then
-            pac_cv_fc_vendor=sun
-	elif grep 'Sun Fortran 9' conftest.txt >/dev/null 2>&1 ; then 
-	    pac_cv_fc_vendor=sun
-        elif grep 'Absoft' conftest.txt >/dev/null 2>&1 ; then
-            pac_cv_fc_vendor=absoft
-        elif grep 'G95' conftest.txt >/dev/null 2>&1 ; then
-            pac_cv_fc_vendor=gnu
-        elif grep 'GNU Fortran' conftest.txt >/dev/null 2>&1 ; then
-            # This is gfortran
-            pac_cv_fc_vendor=gnu
-        elif grep Intel conftest.txt >/dev/null 2>&1 ; then
-            pac_cv_fc_vendor=intel
-        fi
-    fi
-    if test "$pac_cv_fc_vendor" != "unknown" ; then break ; fi
-done
-if test "$pac_cv_fc_vendor" = "unknown" ; then
-    # Try to use the compiler name
-    if test "$FC" = "ifort" -o "$FC" = "ifc" ; then
-        pac_cv_fc_vendor=intel
-    elif test "$FC" = "pgf90" ; then
-        pac_cv_fc_vendor=pgi
-    elif test "$FC" = "xlf90" -o "$FC" = "xlf90_r" ; then
-        pac_cv_fc_vendor=ibm
-    elif test "$FC" = "xlf95" -o "$FC" = "xlf95_r" ; then
-        pac_cv_fc_vendor=ibm
-    fi
-fi
-AC_MSG_RESULT([$pac_cv_fc_vendor])
-rm -f conftest.txt
-# End of checking for F90 compiler vendor
-])
-dnl
-dnl PAC_F77_IS_FC([ACTION_IF_TRUE],[ACTION_IF_FALSE])
-dnl Check if F77 is a Fortran 90 compiler.
-dnl
-AC_DEFUN([PAC_F77_IS_FC],[
-AC_MSG_CHECKING([whether $F77 is a Fortran 90 compiler])
-AC_LANG_PUSH([Fortran 77])
-saved_ac_ext=$ac_ext
-ac_ext="f90"
-AC_LINK_IFELSE([
-    AC_LANG_SOURCE([
-        program main
-        integer, dimension(10) :: n
-        integer k
-        print *,  range(k)
-        end
-    ])
-],[
-    pac_cv_prog_f77_is_fc=yes
-    ifelse([$1],[],[],[$1])
-],[
-    pac_cv_prog_f77_is_fc=no
-    ifelse([$2],[],[],[$2])
-])
-AC_MSG_RESULT([$pac_cv_prog_f77_is_fc])
-AC_LANG_POP([Fortran 77])
-])
-dnl
-dnl PAC_FC_FLOAT_MODEL(float_type, [variable-set-if-successful-test])
-dnl variable-set-if-successful-test is optional variable.
-dnl
-dnl This is a runtime test.
-dnl
-AC_DEFUN([PAC_FC_FLOAT_MODEL],[
-type="$1"
-AC_MSG_CHECKING([for precision and range of $type])
-AC_LANG_PUSH([Fortran])
-rm -f pac_fconftest.out
-AC_RUN_IFELSE([
-    AC_LANG_SOURCE([
-        program main
-        $type aa
-        open(8, file="pac_fconftest.out", form="formatted")
-        write(8,*) precision(aa), ",", range(aa)
-        close(8)
-        end
-    ])
-],[
-    if test -s pac_fconftest.out ; then
-        pac_fc_num_model="`sed -e 's/  */ /g' pac_fconftest.out`"
-        AC_MSG_RESULT([$pac_fc_num_model])
-        ifelse([$2],[],[],[$2=$pac_fc_num_model])
-    else
-        AC_MSG_RESULT([Error])
-        AC_MSG_WARN([No output from test program!])
-    fi
-    rm -f pac_fconftest.out
-],[
-    AC_MSG_RESULT([Error])
-    AC_MSG_WARN([Failed to run program to determine the precision and range of $type])
-])
-AC_LANG_POP([Fortran])
-])
-dnl
-dnl PAC_FC_SIMPLE_NUMBER_MODEL(message, Fortran-type, Fortran-write,
-dnl                            [variable-set-if-successful-test],
-dnl                            [cross-value])
-dnl
-dnl message        : message of what test-fc-code is checking
-dnl Fortran-type   : Fortran90 type's data model to be examined.
-dnl Fortran-write  : Fortran90 type's write statement used with write(N,*).
-dnl variable-set-if-successful-test :
-dnl                  The optional variable to be set if the codelet:
-dnl                  "Fortran-type" + "write(N,*) Fortran-write"
-dnl                  is successful in returning the simple data model.
-dnl cross-value    : value to be used for above variable when
-dnl                  cross_compiling=yes
-dnl
-dnl This is a runtime test.
-dnl
-AC_DEFUN([PAC_FC_SIMPLE_NUMBER_MODEL],[
-pac_msg="$1"
-AC_MSG_CHECKING([for $pac_msg])
-AC_LANG_PUSH([Fortran])
-rm -f pac_fconftest.out
-AC_RUN_IFELSE([
-    AC_LANG_SOURCE([
-        program main
-        $2
-        open(8, file="pac_fconftest.out", form="formatted")
-        write(8,*) $3
-        close(8)
-        end
-    ])
-],[
-    if test -s pac_fconftest.out ; then
-        pac_fc_num_model="`sed -e 's/  */ /g' pac_fconftest.out`"
-        AC_MSG_RESULT([$pac_fc_num_model])
-        ifelse([$4],[],[],[$4=$pac_fc_num_model])
-    else
-        AC_MSG_RESULT([Error])
-        AC_MSG_WARN([No output from test program!])
-    fi
-    rm -f pac_fconftest.out
-],[
-    AC_MSG_RESULT([Error])
-    AC_MSG_WARN([Failed to run program to determine $pac_msg])
-],[
-    AC_MSG_RESULT([$5])
-    ifelse([$4],[],[],[$4=$5])
-])
-AC_LANG_POP([Fortran])
-])
-dnl
-dnl PAC_FC_AVAIL_INTEGER_MODELS([INTEGER-MODELS-FLAG],[CROSS-VARIABLE])
-dnl
-dnl INTEGER-MODELS-FLAG : an optional variable to be set if provided.
-dnl                       If it isn't provided, PAC_FC_ALL_INTEGER_MODELS
-dnl                       will be set.
-dnl CROSS-VALUE         : value will be used to set INTEGER-MODELS-FLAG
-dnl                       or PAC_FC_ALL_INTEGER_MODELS if cross_compiling=yes.
-dnl
-dnl This is a runtime test.
-dnl
-AC_DEFUN([PAC_FC_AVAIL_INTEGER_MODELS],[
-AC_MSG_CHECKING([for available integer kinds])
-AC_LANG_PUSH([Fortran])
-rm -f pac_fconftest.out
-AC_RUN_IFELSE([
-    AC_LANG_SOURCE([
-        program main
-        integer r, lastkind
-        lastkind=selected_int_kind(1)
-        open(8, file="pac_fconftest.out", form="formatted")
-        do r=2,30
-             k = selected_int_kind(r)
-             if (k .ne. lastkind) then
-                  write(8,*) r-1, ",", lastkind
-                  lastkind = k
-             endif
-             if (k .le. 0) then
-                 exit
-             endif
-        enddo
-        if (k.ne.lastkind) then
-            write(8,*) 31, ",", k
-        endif
-        close(8)
-        end
-    ])
-],[
-    if test -s pac_fconftest.out ; then
-        pac_flag="`sed -e 's/  */ /g' pac_fconftest.out | tr '\012' ','`"
-        AC_MSG_RESULT([$pac_flag])
-        pac_validKinds="`sed -e 's/  */ /g' pac_fconftest.out | tr '\012' ':'`"
-        ifelse([$1],[],[PAC_FC_ALL_INTEGER_MODELS=$pac_flag],[$1=$pac_flag])
-    else
-        AC_MSG_RESULT([Error])
-        AC_MSG_WARN([No output from test program!])
-    fi
-    rm -f pac_fconftest.out
-],[
-    AC_MSG_RESULT([Error])
-    AC_MSG_WARN([Failed to run program to determine available integer models])
-],[
-    dnl Even when cross_compiling=yes,
-    dnl pac_validKinds needs to be set for PAC_FC_INTEGER_MODEL_MAP()
-    pac_validKinds="`echo \"$2\" | tr ',' ':'`"
-    AC_MSG_RESULT([$2])
-    ifelse([$1],[],[PAC_FC_ALL_INTEGER_MODELS=$2],[$1=$2])
-])
-AC_LANG_POP([Fortran])
-])
-dnl
-dnl PAC_FC_INTEGER_MODEL_MAP([INTEGER-MODEL-MAP-FLAG],[CROSS-VALUE]))
-dnl
-dnl INTEGER-MODEL-MAP-FLAG : an optional variable to be set if provided.
-dnl                          If it isn't provided, PAC_FC_INTEGER_MODEL_MAP
-dnl                          will be set.
-dnl CROSS-VALUE            : value will be used to set INTEGER-MODEL-MAP-FLAG
-dnl                          or PAC_FC_INTEGER_MODEL_MAP if cross_compiling=yes.
-dnl
-dnl This test requires $pac_validKinds set by PAC_FC_ALL_INTEGER_MODELS().
-dnl
-dnl This is a runtime test.
-dnl
-dnl Compile the C subroutine as pac_conftest.o and Link it with a Fortran main.
-AC_DEFUN([PAC_FC_INTEGER_MODEL_MAP],[
-AC_REQUIRE([PAC_FC_AVAIL_INTEGER_MODELS])
-AC_MSG_CHECKING([for available integer ranges])
-AC_LANG_PUSH([C])
-AC_COMPILE_IFELSE([
-    AC_LANG_SOURCE([
-#ifdef F77_NAME_UPPER
-#define cisize_ CISIZE
-#define isize_ ISIZE
-#elif defined(F77_NAME_LOWER) || defined(F77_NAME_MIXED)
-#define cisize_ cisize
-#define isize_ isize
-#endif
-int cisize_(char *,char*);
-int cisize_(char *i1p, char *i2p)
-{
-    int isize_val=0;
-    isize_val = (int)(i2p - i1p);
-    return isize_val;
-}
-    ])
-],[
-    PAC_RUNLOG([mv conftest.$OBJEXT pac_conftest.$OBJEXT])
-    pac_ccompile_ok=yes
-],[
-    pac_ccompile_ok=no
-])
-AC_LANG_POP([C])
-dnl
-if test "$pac_ccompile_ok" = "yes" ; then
-    saved_LIBS="$LIBS"
-    LIBS="pac_conftest.$OBJEXT $LIBS"
-    saved_IFS=$IFS
-    IFS=:
-    AC_LANG_PUSH([Fortran])
-    pac_flag=""
-    for rangekind in $pac_validKinds ; do
-        kind="`echo $rangekind | sed -e 's/.*,//'`"
-        range="`echo $rangekind | sed -e 's/,.*//'`"
-        AC_LANG_CONFTEST([
-            AC_LANG_SOURCE([
-                program main
-                integer (kind=$kind) a(2)
-                integer cisize
-                open(8, file="pac_fconftest.out", form="formatted")
-                write(8,*) $range, ",", $kind, ",", cisize( a(1), a(2) )
-                close(8)
-                end
-            ])
-        ])
-        IFS=$saved_IFS
-        rm -f pac_fconftest.out
-        AC_RUN_IFELSE([],[
-            if test -s pac_fconftest.out ; then
-                sizes="`sed -e 's/  */ /g' pac_fconftest.out`"
-                pac_flag="$pac_flag { $sizes },"
-            else
-                AC_MSG_WARN([No output from test program!])
-            fi
-            rm -f pac_fconftest.out
-        ],[
-            AC_MSG_WARN([Fortran program fails to build or run!])
-        ],[
-            pac_flag="$2"
-        ])
-        IFS=:
-    done
-    IFS=$saved_IFS
-    AC_MSG_RESULT([$pac_flag])
-    ifelse([$1],[],[PAC_FC_INTEGER_MODEL_MAP=$pac_flag],[$1=$pac_flag])
-    AC_LANG_POP([Fortran])
-    LIBS="$saved_LIBS"
-    rm -f pac_conftest.$OBJEXT
-fi
-])
-
-
-AC_DEFUN([PAC_FC_2008_SUPPORT],[
-AC_MSG_CHECKING([for Fortran 2008 support])
-
-AC_LANG_PUSH([C])
-f08_works=yes
-AC_COMPILE_IFELSE([
-	AC_LANG_SOURCE(
-[[
-#include <ISO_Fortran_binding.h>
-
-int foo_c(CFI_cdesc_t * a_desc, CFI_cdesc_t * b_desc)
-{
-	char * a_row = (char*) a_desc->base_addr;
-	if (a_desc->type != CFI_type_int) { return 1; }
-	if (a_desc->rank != 2) { return 2; }
-	if (a_desc->dim[1].extent != b_desc->dim[0].extent) { return 3; }
-	return 0;
-}
-]])],[mv conftest.$OBJEXT conftest1.$OBJEXT],[f08_works=no])
-AC_LANG_POP([C])
-
-AC_LANG_PUSH([Fortran])
-PAC_PUSH_FLAG([LIBS])
-LIBS="conftest1.$OBJEXT $LIBS"
-AC_LINK_IFELSE([
-    AC_LANG_SOURCE([
-MODULE F08TS_MODULE
-IMPLICIT NONE
-
-! Test public, private, protected
-REAL, PUBLIC       :: x
-REAL, PRIVATE      :: y
-LOGICAL, PROTECTED :: z
-
-! Test abstract
-ABSTRACT INTERFACE
-    SUBROUTINE user_func(x, y)
-        INTEGER  :: x(*)
-        REAL     :: y
-    END SUBROUTINE
-END INTERFACE
-
-! Test TS 29113 assumed type , assumed rank and bind(C)
-INTERFACE
-    FUNCTION FOO(A, B, C) &
-        BIND(C,name="foo_c") RESULT(err)
-        USE, intrinsic :: iso_c_binding, ONLY : c_int
-        TYPE(*), DIMENSION(..) :: A, B, C
-        INTEGER(c_int) :: err
-    END FUNCTION FOO
-END INTERFACE
-
-! Test assumed-rank + asynchronous
-INTERFACE TEST_ASSUMED_RANK_ASYNC
-    SUBROUTINE TEST_ASSUMED_RANK_ASYNC_IMPL(BUF)
-        IMPLICIT NONE
-        TYPE(*), DIMENSION(..), ASYNCHRONOUS :: BUF
-    END SUBROUTINE TEST_ASSUMED_RANK_ASYNC_IMPL
-END INTERFACE TEST_ASSUMED_RANK_ASYNC
-
-CONTAINS
-
-! Test TS 29113 asychronous attribute and optional
-SUBROUTINE test1(buf, count, ierr)
-    INTEGER, ASYNCHRONOUS :: buf(*)
-    INTEGER               :: count
-    INTEGER, OPTIONAL     :: ierr
-END SUBROUTINE
-
-! Test procedure type and non-bind(c) x in C_FUNCLOC(x)
-SUBROUTINE test2(func)
-    USE, INTRINSIC :: ISO_C_BINDING, ONLY : C_FUNLOC, C_FUNPTR
-    PROCEDURE(user_func)  :: func
-    TYPE(C_FUNPTR) :: errhandler_fn
-    errhandler_fn = C_FUNLOC(func)
-END SUBROUTINE
-
-! Test intrinsic storage_size
-SUBROUTINE test3(x, size)
-    CHARACTER, DIMENSION(..) :: x
-    INTEGER, INTENT(OUT) :: size
-    size = storage_size(x)/8
-END SUBROUTINE test3
-
-END MODULE
-
-!==============================================
-PROGRAM MAIN
-USE :: F08TS_MODULE, ONLY : FOO, TEST_ASSUMED_RANK_ASYNC
-IMPLICIT NONE
-
-INTEGER, DIMENSION(4,4) :: A, B
-INTEGER, DIMENSION(2,2) :: C
-INTEGER                 :: ERRCODE
-INTEGER, DIMENSION(10), ASYNCHRONOUS :: IAR
-
-! Test contiguous and non-contiguous array section passing
-! and linkage with C code
-ERRCODE = FOO(A(1:4:2, :), B(:, 2:4:2), C)
-CALL TEST_ASSUMED_RANK_ASYNC(IAR(2:7))
-
-END PROGRAM
-    ])],[],[f08_works=no])
-PAC_POP_FLAG([LIBS])
-AC_LANG_POP([Fortran])
-
-if test "$f08_works" = "yes" ; then
-   $1
-else
-   $2
-fi
-rm -f conftest1.$OBJEXT F08TS_MODULE.* f08ts_module.*
-AC_MSG_RESULT([$f08_works])
-])
diff --git a/3rd-party/romio321/confdb/aclocal_libs.m4 b/3rd-party/romio321/confdb/aclocal_libs.m4
deleted file mode 100644
index 8868e994cba..00000000000
--- a/3rd-party/romio321/confdb/aclocal_libs.m4
+++ /dev/null
@@ -1,107 +0,0 @@
-
-dnl PAC_SET_HEADER_LIB_PATH(with_option,[default_path])
-dnl This macro looks for the --with-xxx=, --with-xxx-include and --with-xxx-lib=
-dnl options and sets the library and include paths.
-dnl
-dnl TODO as written, this macro cannot handle a "with_option" arg that has "-"
-dnl characters in it.  Use AS_TR_SH (and possibly AS_VAR_* macros) to handle
-dnl this case if it ever arises.
-AC_DEFUN([PAC_SET_HEADER_LIB_PATH],[
-    AC_ARG_WITH([$1],
-                [AC_HELP_STRING([--with-$1=PATH],
-                                [specify path where $1 include directory and lib directory can be found])],
-
-                [AS_CASE(["$withval"],
-                         [yes|no|''],
-                         [AC_MSG_WARN([--with[out]-$1=PATH expects a valid PATH])
-                          with_$1=""])],
-                [with_$1=$2])
-    AC_ARG_WITH([$1-include],
-                [AC_HELP_STRING([--with-$1-include=PATH],
-                                [specify path where $1 include directory can be found])],
-                [AS_CASE(["$withval"],
-                         [yes|no|''],
-                         [AC_MSG_WARN([--with[out]-$1-include=PATH expects a valid PATH])
-                          with_$1_include=""])],
-                [])
-    AC_ARG_WITH([$1-lib],
-                [AC_HELP_STRING([--with-$1-lib=PATH],
-                                [specify path where $1 lib directory can be found])],
-                [AS_CASE(["$withval"],
-                         [yes|no|''],
-                         [AC_MSG_WARN([--with[out]-$1-lib=PATH expects a valid PATH])
-                          with_$1_lib=""])],
-                [])
-
-    # The args have been sanitized into empty/non-empty values above.
-    # Now append -I/-L args to CPPFLAGS/LDFLAGS, with more specific options
-    # taking priority
-
-    AS_IF([test -n "${with_$1_include}"],
-          [PAC_APPEND_FLAG([-I${with_$1_include}],[CPPFLAGS])],
-          [AS_IF([test -n "${with_$1}"],
-                 [PAC_APPEND_FLAG([-I${with_$1}/include],[CPPFLAGS])])])
-
-    AS_IF([test -n "${with_$1_lib}"],
-          [PAC_APPEND_FLAG([-L${with_$1_lib}],[LDFLAGS])],
-          [AS_IF([test -n "${with_$1}"],
-                 dnl is adding lib64 by default really the right thing to do?  What if
-                 dnl we are on a 32-bit host that happens to have both lib dirs available?
-                 [PAC_APPEND_FLAG([-L${with_$1}/lib],[LDFLAGS])
-                  AS_IF([test -d "${with_$1}/lib64"],
-		        [PAC_APPEND_FLAG([-L${with_$1}/lib64],[LDFLAGS])])
-                 ])
-          ])
-])
-
-
-dnl PAC_CHECK_HEADER_LIB(header.h, libname, function, action-if-yes, action-if-no)
-dnl This macro checks for a header and lib.  It is assumed that the
-dnl user can specify a path to the includes and libs using --with-xxx=.
-dnl The xxx is specified in the "with_option" parameter.
-dnl
-dnl NOTE: This macro expects a corresponding PAC_SET_HEADER_LIB_PATH
-dnl macro (or equivalent logic) to be used before this macro is used.
-AC_DEFUN([PAC_CHECK_HEADER_LIB],[
-    failure=no
-    AC_CHECK_HEADER([$1],,failure=yes)
-    AC_CHECK_LIB($2,$3,,failure=yes)
-    if test "$failure" = "no" ; then
-       $4
-    else
-       $5
-    fi
-])
-
-dnl PAC_CHECK_HEADER_LIB_FATAL(with_option, header.h, libname, function)
-dnl Similar to PAC_CHECK_HEADER_LIB, but errors out on failure
-AC_DEFUN([PAC_CHECK_HEADER_LIB_FATAL],[
-	PAC_CHECK_HEADER_LIB($2,$3,$4,success=yes,success=no)
-	if test "$success" = "no" ; then
-	   AC_MSG_ERROR(['$2 or lib$3 library not found. Did you specify --with-$1= or --with-$1-include= or --with-$1-lib=?'])
-	fi
-])
-
-dnl PAC_CHECK_PREFIX(with_option,prefixvar)
-AC_DEFUN([PAC_CHECK_PREFIX],[
-	AC_ARG_WITH([$1-prefix],
-            [AS_HELP_STRING([[--with-$1-prefix[=DIR]]], [use the $1
-                            library installed in DIR, rather than the
-                            one included in the distribution.  Pass
-                            "embedded" to force usage of the included
-                            $1 source.])],
-            [if test "$withval" = "system" ; then
-                 :
-             elif test "$withval" = "embedded" ; then
-                 :
-             else
-                 PAC_APPEND_FLAG([-I${with_$1_prefix}/include],[CPPFLAGS])
-                 if test -d "${with_$1_prefix}/lib64" ; then
-                     PAC_APPEND_FLAG([-L${with_$1_prefix}/lib64],[LDFLAGS])
-                 fi
-                 PAC_APPEND_FLAG([-L${with_$1_prefix}/lib],[LDFLAGS])
-             fi
-             ],
-            [with_$1_prefix="embedded"])
-	]
-)
diff --git a/3rd-party/romio321/confdb/aclocal_make.m4 b/3rd-party/romio321/confdb/aclocal_make.m4
deleted file mode 100644
index 147e92e9c2e..00000000000
--- a/3rd-party/romio321/confdb/aclocal_make.m4
+++ /dev/null
@@ -1,315 +0,0 @@
-dnl
-dnl We need routines to check that make works.  Possible problems with
-dnl make include
-dnl
-dnl It is really gnumake, and contrary to the documentation on gnumake,
-dnl it insists on screaming everytime a directory is changed.  The fix
-dnl is to add the argument --no-print-directory to the make
-dnl
-dnl It is really BSD 4.4 make, and can't handle 'include'.  For some
-dnl systems, this can be fatal; there is no fix (other than removing this
-dnl alleged make).
-dnl
-dnl It is the OSF V3 make, and can't handle a comment in a block of target
-dnl code.  There is no acceptable fix.
-dnl
-dnl
-dnl
-dnl
-dnl Find a make program if none is defined.
-AC_DEFUN([PAC_PROG_MAKE_PROGRAM],[true
-if test "X$MAKE" = "X" ; then
-   AC_CHECK_PROGS(MAKE,make gnumake nmake pmake smake)
-fi
-])dnl
-
-dnl/*D
-dnl PAC_PROG_MAKE_INCLUDE - Check whether make supports include
-dnl
-dnl Synopsis:
-dnl PAC_PROG_MAKE_INCLUDE([action if true],[action if false])
-dnl
-dnl Output Effect:
-dnl   None
-dnl
-dnl Notes:
-dnl  This checks for makes that do not support 'include filename'.  Some
-dnl  versions of BSD 4.4 make required '#include' instead; some versions of
-dnl  'pmake' have the same syntax.
-dnl
-dnl See Also:
-dnl  PAC_PROG_MAKE
-dnl
-dnl D*/
-AC_DEFUN([PAC_PROG_MAKE_INCLUDE],[
-AC_CACHE_CHECK([whether make supports include],pac_cv_prog_make_include,[
-AC_REQUIRE([PAC_PROG_MAKE_PROGRAM])
-# This is needed for Mac OSX 10.5
-rm -rf conftest.dSYM
-rm -f conftest
-cat > conftest <<.
-ALL:
-	@echo "success"
-.
-cat > conftest1 <<.
-include conftest
-.
-pac_str=`$MAKE -f conftest1 2>&1`
-# This is needed for Mac OSX 10.5
-rm -rf conftest.dSYM
-rm -f conftest conftest1
-if test "$pac_str" != "success" ; then
-    pac_cv_prog_make_include="no"
-else
-    pac_cv_prog_make_include="yes"
-fi
-])
-if test "$pac_cv_prog_make_include" = "no" ; then
-    ifelse([$2],,:,[$2])
-else
-    ifelse([$1],,:,[$1])
-fi
-])dnl
-
-dnl/*D
-dnl PAC_PROG_MAKE_ALLOWS_COMMENTS - Check whether comments are allowed in 
-dnl   shell commands in a makefile
-dnl
-dnl Synopsis:
-dnl PAC_PROG_MAKE_ALLOWS_COMMENTS([false text])
-dnl
-dnl Output Effect:
-dnl Issues a warning message if comments are not allowed in a makefile.
-dnl Executes the argument if one is given.
-dnl
-dnl Notes:
-dnl Some versions of OSF V3 make do not all comments in action commands.
-dnl
-dnl See Also:
-dnl  PAC_PROG_MAKE
-dnl D*/
-dnl
-AC_DEFUN([PAC_PROG_MAKE_ALLOWS_COMMENTS],[
-AC_CACHE_CHECK([whether make allows comments in actions],
-pac_cv_prog_make_allows_comments,[
-AC_REQUIRE([PAC_PROG_MAKE_PROGRAM])
-# This is needed for Mac OSX 10.5
-rm -rf conftest.dSYM
-rm -f conftest
-cat > conftest <<.
-SHELL=/bin/sh
-ALL:
-	@# This is a valid comment!
-	@echo "success"
-.
-pac_str=`$MAKE -f conftest 2>&1`
-# This is needed for Mac OSX 10.5
-rm -rf conftest.dSYM
-rm -f conftest 
-if test "$pac_str" != "success" ; then
-    pac_cv_prog_make_allows_comments="no"
-else
-    pac_cv_prog_make_allows_comments="yes"
-fi
-])
-if test "$pac_cv_prog_make_allows_comments" = "no" ; then
-    AC_MSG_WARN([Your make does not allow comments in target code.
-Using this make may cause problems when building programs.
-You should consider using gnumake instead.])
-    ifelse([$1],,[$1])
-fi
-])dnl
-
-dnl/*D
-dnl PAC_PROG_MAKE_VPATH - Check whether make supports source-code paths.
-dnl
-dnl Synopsis:
-dnl PAC_PROG_MAKE_VPATH
-dnl
-dnl Output Effect:
-dnl Sets the variable 'VPATH' to either
-dnl.vb
-dnl VPATH = .:${srcdir}
-dnl.ve
-dnl or
-dnl.vb
-dnl .PATH: . ${srcdir}
-dnl.ve
-dnl 
-dnl Notes:
-dnl The test checks that the path works with implicit targets (some makes
-dnl support only explicit targets with 'VPATH' or 'PATH').
-dnl
-dnl NEED TO DO: Check that $< works on explicit targets.
-dnl
-dnl See Also:
-dnl PAC_PROG_MAKE
-dnl
-dnl D*/
-AC_DEFUN([PAC_PROG_MAKE_VPATH],[
-AC_SUBST(VPATH)
-dnl AM_IGNORE(VPATH)
-AC_CACHE_CHECK([for virtual path format],
-pac_cv_prog_make_vpath,[
-AC_REQUIRE([PAC_PROG_MAKE_PROGRAM])
-# This is needed for Mac OSX 10.5
-rm -rf conftest.dSYM
-rm -rf conftest*
-mkdir conftestdir
-cat >conftestdir/a.c <<EOF
-A sample file
-EOF
-cat > conftest <<EOF
-all: a.o
-VPATH=.:conftestdir
-.c.o:
-	@echo \$<
-EOF
-ac_out=`$MAKE -f conftest 2>&1 | grep 'conftestdir/a.c'`
-if test -n "$ac_out" ; then 
-    pac_cv_prog_make_vpath="VPATH"
-else
-    rm -f conftest
-    cat > conftest <<EOF
-all: a.o
-.PATH: . conftestdir
-.c.o:
-	@echo \$<
-EOF
-    ac_out=`$MAKE -f conftest 2>&1 | grep 'conftestdir/a.c'`
-    if test -n "$ac_out" ; then 
-        pac_cv_prog_make_vpath=".PATH"
-    else
-	pac_cv_prog_make_vpath="neither VPATH nor .PATH works"
-    fi
-fi
-# This is needed for Mac OSX 10.5
-rm -rf conftest.dSYM
-rm -rf conftest*
-])
-if test "$pac_cv_prog_make_vpath" = "VPATH" ; then
-    VPATH='VPATH=.:${srcdir}'
-elif test "$pac_cv_prog_make_vpath" = ".PATH" ; then
-    VPATH='.PATH: . ${srcdir}'
-fi
-])dnl
-
-dnl/*D
-dnl PAC_PROG_MAKE_SET_CFLAGS - Check whether make sets CFLAGS
-dnl
-dnl Synopsis:
-dnl PAC_PROG_MAKE_SET_CFLAGS([action if true],[action if false])
-dnl
-dnl Output Effects:
-dnl Executes the first argument if 'CFLAGS' is set by 'make'; executes
-dnl the second argument if 'CFLAGS' is not set by 'make'.
-dnl
-dnl Notes:
-dnl If 'CFLAGS' is set by make, you may wish to override that choice in your
-dnl makefile.
-dnl
-dnl See Also:
-dnl PAC_PROG_MAKE
-dnl D*/
-AC_DEFUN([PAC_PROG_MAKE_SET_CFLAGS],[
-AC_CACHE_CHECK([whether make sets CFLAGS],
-pac_cv_prog_make_set_cflags,[
-AC_REQUIRE([PAC_PROG_MAKE_PROGRAM])
-# This is needed for Mac OSX 10.5
-rm -rf conftest.dSYM
-rm -f conftest
-cat > conftest <<EOF
-SHELL=/bin/sh
-ALL:
-	@echo X[\$]{CFLAGS}X
-EOF
-pac_str=`$MAKE -f conftest 2>&1`
-# This is needed for Mac OSX 10.5
-rm -rf conftest.dSYM
-rm -f conftest 
-if test "$pac_str" = "XX" ; then
-    pac_cv_prog_make_set_cflags="no"
-else
-    pac_cv_prog_make_set_cflags="yes"
-fi
-])
-if test "$pac_cv_prog_make_set_cflags" = "no" ; then
-    ifelse([$2],,:,[$2])
-else
-    ifelse([$1],,:,[$1])
-fi
-])dnl
-
-dnl/*D
-dnl PAC_PROG_MAKE_CLOCK_SKEW - Check whether there is a problem with 
-dnl clock skew in suing make.
-dnl
-dnl Effect:
-dnl Sets the cache variable 'pac_cv_prog_make_found_clock_skew' to yes or no
-dnl D*/
-AC_DEFUN([PAC_PROG_MAKE_CLOCK_SKEW],[
-AC_CACHE_CHECK([whether clock skew breaks make],
-pac_cv_prog_make_found_clock_skew,[
-AC_REQUIRE([PAC_PROG_MAKE_PROGRAM])
-# This is needed for Mac OSX 10.5
-rm -rf conftest.dSYM
-rm -f conftest*
-cat > conftest <<EOF
-ALL:
-	@-echo "success"
-EOF
-$MAKE -f conftest > conftest.out 2>&1
-if grep -i skew conftest >/dev/null 2>&1 ; then
-    pac_cv_prog_make_found_clock_skew=yes
-else
-    pac_cv_prog_make_found_clock_skew=no
-fi
-# This is needed for Mac OSX 10.5
-rm -rf conftest.dSYM
-rm -f conftest*
-])
-dnl We should really do something if we detect clock skew.  The question is,
-dnl what?
-if test "$pac_cv_prog_make_found_clock_skew" = "yes" ; then
-    AC_MSG_WARN([Clock skew found by make.  The configure and build may fail.
-Consider building in a local instead of NFS filesystem.])
-fi
-])
-
-dnl/*D
-dnl PAC_PROG_MAKE - Checks for the varieties of MAKE, including support for 
-dnl VPATH
-dnl
-dnl Synopsis:
-dnl PAC_PROG_MAKE
-dnl
-dnl Output Effect:
-dnl Sets 'MAKE' to the make program to use if 'MAKE' is not already set.
-dnl Sets the variable 'SET_CFLAGS' to 'CFLAGS =' if make sets 'CFLAGS'.
-dnl
-dnl Notes:
-dnl This macro uses 'PAC_PROG_MAKE_INCLUDE',
-dnl 'PAC_PROG_MAKE_ALLOWS_COMMENTS', 'PAC_PROG_MAKE_VPATH', and
-dnl 'PAC_PROG_MAKE_SET_CFLAGS'.  See those commands for details about their
-dnl actions.
-dnl 
-dnl It may call 'AC_PROG_MAKE_SET', which sets 'SET_MAKE' to 'MAKE = @MAKE@'
-dnl if the make program does not set the value of make, otherwise 'SET_MAKE'
-dnl is set to empty; if the make program echos the directory name, then 
-dnl 'SET_MAKE' is set to 'MAKE = $MAKE'.
-dnl D*/
-AC_DEFUN([PAC_PROG_MAKE],[
-PAC_PROG_MAKE_PROGRAM
-PAC_PROG_MAKE_CLOCK_SKEW
-PAC_PROG_MAKE_INCLUDE
-PAC_PROG_MAKE_ALLOWS_COMMENTS
-PAC_PROG_MAKE_VPATH
-AC_SUBST(SET_CFLAGS)
-dnl AM_IGNORE(SET_CFLAGS)
-PAC_PROG_MAKE_SET_CFLAGS([SET_CFLAGS='CFLAGS='])
-if test "$pac_cv_prog_make_echos_dir" = "no" ; then
-    AC_PROG_MAKE_SET
-else
-    SET_MAKE="MAKE=${MAKE-make}"
-fi
-])
diff --git a/3rd-party/romio321/confdb/aclocal_mpi.m4 b/3rd-party/romio321/confdb/aclocal_mpi.m4
deleted file mode 100644
index f0013a9bd18..00000000000
--- a/3rd-party/romio321/confdb/aclocal_mpi.m4
+++ /dev/null
@@ -1,519 +0,0 @@
-dnl
-dnl/*D 
-dnl PAC_LIB_MPI - Check for MPI library
-dnl
-dnl Synopsis:
-dnl PAC_LIB_MPI([action if found],[action if not found])
-dnl
-dnl Output Effect:
-dnl
-dnl Notes:
-dnl Currently, only checks for lib mpi and mpi.h.  Later, we will add
-dnl MPI_Pcontrol prototype (const int or not?).  
-dnl
-dnl Prerequisites:
-dnl autoconf version 2.13 (for AC_SEARCH_LIBS)
-dnl D*/
-dnl Other tests to add:
-dnl Version of MPI
-dnl MPI-2 I/O?
-dnl MPI-2 Spawn?
-dnl MPI-2 RMA?
-dnl PAC_LIB_MPI([found text],[not found text])
-AC_DEFUN([PAC_LIB_MPI],[
-dnl Set the prereq to 2.50 to avoid having 
-AC_PREREQ(2.50)
-if test "X$pac_lib_mpi_is_building" != "Xyes" ; then
-  # Use CC if TESTCC is defined
-  if test "X$pac_save_level" != "X" ; then
-     pac_save_TESTCC="${TESTCC}"
-     pac_save_TESTCPP="${TESTCPP}"
-     CC="$pac_save_CC"
-     if test "X$pac_save_CPP" != "X" ; then
-         CPP="$pac_save_CPP"
-     fi
-  fi
-  # Look for MPILIB first if it is defined
-  AC_SEARCH_LIBS(MPI_Init,$MPILIB mpi mpich)
-  if test "$ac_cv_search_MPI_Init" = "no" ; then
-    ifelse($2,,
-    AC_MSG_ERROR([Could not find MPI library]),[$2])
-  fi
-  AC_CHECK_HEADER(mpi.h,pac_have_mpi_h="yes",pac_have_mpi_h="no")
-  if test $pac_have_mpi_h = "no" ; then
-    ifelse($2,,
-    AC_MSG_ERROR([Could not find mpi.h include file]),[$2])
-  fi
-  if test "X$pac_save_level" != "X" ; then
-     CC="$pac_save_TESTCC"
-     CPP="$pac_save_TESTCPP"
-  fi
-fi
-ifelse($1,,,[$1])
-])
-
-dnl This should also set MPIRUN.
-dnl
-dnl/*D
-dnl PAC_ARG_MPI_TYPES - Add command-line switches for different MPI 
-dnl environments
-dnl
-dnl Synopsis:
-dnl PAC_ARG_MPI_TYPES([default])
-dnl
-dnl Output Effects:
-dnl Adds the following command line options to configure
-dnl+ \-\-with\-mpich[=path] - MPICH.  'path' is the location of MPICH commands
-dnl. \-\-with\-ibmmpi - IBM MPI
-dnl. \-\-with\-lammpi[=path] - LAM/MPI
-dnl. \-\-with\-mpichnt - MPICH NT
-dnl- \-\-with\-sgimpi - SGI MPI
-dnl If no type is selected, and a default ("mpich", "ibmmpi", or "sgimpi")
-dnl is given, that type is used as if '--with-<default>' was given.
-dnl
-dnl Sets 'CC', 'F77', 'TESTCC', 'TESTF77', and 'MPILIBNAME'.  Does `not`
-dnl perform an AC_SUBST for these values.
-dnl Also sets 'MPIBOOT' and 'MPIUNBOOT'.  These are used to specify 
-dnl programs that may need to be run before and after running MPI programs.
-dnl For example, 'MPIBOOT' may start demons necessary to run MPI programs and
-dnl 'MPIUNBOOT' will stop those demons.
-dnl 
-dnl The two forms of the compilers are to allow for tests of the compiler
-dnl when the MPI version of the compiler creates executables that cannot
-dnl be run on the local system (for example, the IBM SP, where executables
-dnl created with mpcc will not run locally, but executables created
-dnl with xlc may be used to discover properties of the compiler, such as
-dnl the size of data types).
-dnl
-dnl Historical note:
-dnl Some common autoconf tests, such as AC_CHECK_SIZEOF, used to require
-dnl running a program.  But some MPI compilers (often really compilation 
-dnl scripts) produced programs that could only be run with special commands,
-dnl such as a batch submission system.  To allow these test programs to be
-dnl run, a separate set of compiler variables, TESTCC, TESTF77, etc., 
-dnl were defined.  However, in later versions of autoconf, it both became
-dnl unnecessary to run programs for tests such as AC_CHECK_SIZEOF and 
-dnl it became necessary to define CC etc. before invoking AC_PROG_CC (and
-dnl the othe language compilers), because those commands now do much, much
-dnl more than just determining the compiler.
-dnl
-dnl To address the change, we still define the TESTCC etc. compilers where
-dnl possible to allow the use of AC_TRY_RUN when required, but we define
-dnl the CC etc variables and do not define ac_cv_prog_CC etc., as these 
-dnl cause autoconf to skip all of the other initialization code that 
-dnl AC_PROG_CC etc. runs.  Note also that this command must occur before 
-dnl AC_PROG_CC (or anything that might cause AC_PROG_CC to be invoked).
-dnl
-dnl See also:
-dnl PAC_LANG_PUSH_COMPILERS, PAC_LIB_MPI
-dnl D*/
-AC_DEFUN([PAC_ARG_MPI_TYPES],[
-# known types
-PAC_ARG_MPI_KNOWN_TYPES
-# find compilers
-PAC_MPI_FIND_COMPILER_SCRIPTS
-PAC_MPI_FIND_COMPILERS
-# check for MPI library
-PAC_MPI_CHECK_MPI_LIB
-])
-dnl
-dnl To keep autoconf from prematurely invoking the compiler check scripts,
-dnl we need a command that first sets the compilers and a separate one
-dnl that makes any necessary checks for libraries
-dnl
-AC_DEFUN([PAC_ARG_MPI_KNOWN_TYPES],[
-AC_ARG_WITH(mpich,
-[--with-mpich=path  - Assume that we are building with MPICH],
-ac_mpi_type=mpich)
-# Allow MPICH as well as MPICH
-AC_ARG_WITH(mpich,
-[--with-mpich=path  - Assume that we are building with MPICH],
-ac_mpi_type=mpich)
-AC_ARG_WITH(lammpi,
-[--with-lammpi=path  - Assume that we are building with LAM/MPI],
-ac_mpi_type=lammpi)
-AC_ARG_WITH(ibmmpi,
-[--with-ibmmpi    - Use the IBM SP implementation of MPI],
-ac_mpi_type=ibmmpi)
-AC_ARG_WITH(sgimpi,
-[--with-sgimpi    - Use the SGI implementation of MPI],
-ac_mpi_type=sgimpi)
-AC_ARG_WITH(mpichnt,
-[--with-mpichnt - Use MPICH for Windows NT ],
-ac_mpi_type=mpichnt)
-AC_ARG_WITH(mpi,
-[--with-mpi=path    - Use an MPI implementation with compile scripts mpicc
-                     and mpif77 in path/bin],ac_mpi_type=generic)
-
-if test "X$ac_mpi_type" = "X" ; then
-    if test "X$1" != "X" ; then
-        ac_mpi_type=$1
-    else
-        ac_mpi_type=unknown
-    fi
-fi
-if test "$ac_mpi_type" = "unknown" -a "$pac_lib_mpi_is_building" = "yes" ; then
-    ac_mpi_type="mpich"
-fi
-])
-dnl
-dnl Because autoconf insists on moving code to the beginning of
-dnl certain definitions, it is *not possible* to define a single command
-dnl that selects compilation scripts and also check for other options.
-dnl Thus, this needs to be divided into 
-dnl   MPI_FIND_COMPILER_SCRIPTS
-dnl which can fail (i.e., not find a script), and
-dnl   MPI_FIND_COMPILERS
-dnl which runs the various PROC_xx for the compilers.
-dnl WARNING: this function ignores --program-suffix and --program-prefix.
-dnl However, this function is not currently used at all.
-AC_DEFUN([PAC_MPI_FIND_COMPILER_SCRIPTS],[
-# Set defaults
-MPIRUN_NP="-np "
-MPIEXEC_N="-n "
-AC_SUBST(MPIRUN_NP)
-AC_SUBST(MPIEXEC_N)
-dnl
-AC_ARG_VAR([MPIEXEC],[Name and path of mpiexec program])
-AC_ARG_VAR([MPIRUN],[Name and path of mpirun program])
-AC_ARG_VAR([MPIBOOT],[Name and path of program to run before mpirun])
-AC_ARG_VAR([MPIUNBOOT],[Name and path of program to run after all mpirun])
-AC_ARG_VAR([MPICC],[Name and absolute path of program used to compile MPI programs in C])
-AC_ARG_VAR([MPIF77],[Name and absolute path of program used to compile MPI programs in F77])
-AC_ARG_VAR([MPICXX],[Name and absolute path of program used to compile MPI programs in C++])
-AC_ARG_VAR([MPIF90],[Name and absolute path of program used to compile MPI programs in F90])
-#
-# Check for things that will cause trouble.  For example, 
-# if MPICC is defined but does not contain a / or \, then PATH_PROG will
-# ignore the value
-if test -n "$MPICC" ; then
-   case $MPICC in
-changequote(<<,>>)
-    [\\/]* | ?:[\\/]*)
-changequote([,])
-    # Ok, PATH_PROG will figure it out
-    ;;  
-  *)
-    AC_MSG_ERROR([MPICC must be set to an absolute path if it is set])
-  esac
-fi
-if test -n "$MPICXX" ; then
-   case $MPICXX in
-changequote(<<,>>)
-    [\\/]* | ?:[\\/]*)
-changequote([,])
-    # Ok, PATH_PROG will figure it out
-    ;;  
-  *)
-    AC_MSG_ERROR([MPICXX must be set to an absolute path if it is set])
-  esac
-fi
-if test -n "$MPIF77" ; then
-   case $MPIF77 in
-changequote(<<,>>)
-    [\\/]* | ?:[\\/]*)
-changequote([,])
-    # Ok, PATH_PROG will figure it out
-    ;;  
-  *)
-    AC_MSG_ERROR([MPIF77 must be set to an absolute path if it is set])
-  esac
-fi
-if test -n "$MPIF90" ; then
-   case $MPIF90 in
-changequote(<<,>>)
-    [\\/]* | ?:[\\/]*)
-changequote([,])
-    # Ok, PATH_PROG will figure it out
-    ;;  
-  *)
-    AC_MSG_ERROR([MPIF90 must be set to an absolute path if it is set])
-  esac
-fi
-
-case $ac_mpi_type in
-	mpich)
-        dnl 
-        dnl This isn't correct.  It should try to get the underlying compiler
-        dnl from the mpicc and mpif77 scripts or mpireconfig
-        if test "X$pac_lib_mpi_is_building" != "Xyes" ; then
-	    PAC_PUSH_FLAG([PATH])
-            if test "$with_mpich" != "yes" -a "$with_mpich" != "no" ; then 
-		# Look for commands; if not found, try adding bin to the
-		# path
-		if test ! -x $with_mpich/mpicc -a -x $with_mpich/bin/mpicc ; then
-			with_mpich="$with_mpich/bin"
-		fi
-                PATH=$with_mpich:${PATH}
-            fi
-            AC_PATH_PROG(MPICC,mpicc)
-	    if test -z "$TESTCC" ; then TESTCC=${CC-cc} ; fi
-            CC="$MPICC"
-	    # Note that autoconf may unconditionally change the value of 
-	    # CC (!) in some other command. Thus, we define CCMASTER
-	    CCMASTER=$CC
-	    # Force autoconf to respect this choice
-	    ac_ct_CC=$CC
-	    # to permit configure codes to recover the correct CC.  This
-	    # is an ugly not-quite-correct workaround for the fact that 
-	    # does not want you to change the C compiler once you have set it
-	    # (But since it does so unconditionally, it silently creates 
-	    # bogus output files.)
-            AC_PATH_PROG(MPIF77,mpif77)
-	    if test -z "$TESTF77" ; then TESTF77=${F77-f77} ; fi
-            F77="$MPIF77"
-            AC_PATH_PROG(MPIFC,mpif90)
-	    if test -z "$TESTFC" ; then TESTFC=${FC-f90} ; fi 
-            FC="$MPIFC"
-            AC_PATH_PROG(MPICXX,mpiCC)
-	    if test -z "$TESTCXX" ; then TESTCXX=${CXX-CC} ; fi
-            CXX="$MPICXX"
-	    # We may want to restrict this to the path containing mpirun
-	    AC_PATH_PROG(MPIEXEC,mpiexec)
-	    AC_PATH_PROG(MPIRUN,mpirun)
-	    AC_PATH_PROG(MPIBOOT,mpichboot)
-	    AC_PATH_PROG(MPIUNBOOT,mpichstop)
-	    PAC_POP_FLAG([PATH])
-  	    MPILIBNAME="mpich"
-        else 
-	    # All of the above should have been passed in the environment!
-	    :
-        fi
-	;;
-
-        mpichnt)
-        ;;
-
-	lammpi)
-	dnl
-        dnl This isn't correct.  It should try to get the underlying compiler
-        dnl from the mpicc and mpif77 scripts or mpireconfig
-	PAC_PUSH_FLAG([PATH])
-        if test "$with_mpich" != "yes" -a "$with_mpich" != "no" ; then 
-	    # Look for commands; if not found, try adding bin to the path
-		if test ! -x $with_lammpi/mpicc -a -x $with_lammpi/bin/mpicc ; then
-			with_lammpi="$with_lammpi/bin"
-		fi
-                PATH=$with_lammpi:${PATH}
-        fi
-        AC_PATH_PROG(MPICC,mpicc)
-        if test -z "$TESTCC" ; then TESTCC=${CC-cc} ; fi
-        CC="$MPICC"
-        AC_PATH_PROG(MPIF77,mpif77)
-	if test -z "$TESTCC" ; then TESTF77=${F77-f77} ; fi
-        F77="$MPIF77"
-        AC_PATH_PROG(MPIFC,mpif90)
-        TESTFC=${FC-f90}
-	if test -z "$TESTFC" ; then TESTFC=${FC-f90} ; fi
-        FC="$MPIFC"
-        AC_PATH_PROG(MPICXX,mpiCC)
-	if test -z "$TESTCXX" ; then TESTCXX=${CXX-CC} ; fi
-        CXX="$MPICXX"
-	PAC_POP_FLAG([PATH])
-  	MPILIBNAME="lammpi"
-	MPIBOOT="lamboot"
-	MPIUNBOOT="wipe"
-	MPIRUN="mpirun"
-	;;
-
-	ibmmpi)
-	AC_CHECK_PROGS(MPCC,mpcc)
-	AC_CHECK_PROGS(MPXLF,mpxlf mpfort)
-	if test -z "$MPCC" -o -z "$MPXLF" ; then
-	    AC_MSG_ERROR([Could not find IBM MPI compilation scripts.  Either mpcc or mpxlf/mpfort is missing])
-	fi
-	if test -z "$TESTCC" ; then TESTCC=${CC-xlC} ; fi
-	if test -z "$TESTF77" ; then TESTF77=${F77-xlf}; fi
-	CC=mpcc; F77=$MPXLF
-	# There is no mpxlf90, but the options langlvl and free can
-	# select the Fortran 90 version of xlf
-	if test "$enable_f90" != no ; then
-	    AC_CHECK_PROGS(MPIXLF90,mpxlf90 mpfort)
-	    if test -z "$TESTFC" ; then TESTFC=${FC-xlf90}; fi
-            if test "X$MPIXLF90" != "X" ; then 
-	        FC="$MPIXLF90"
-	    else
-	    	FC="$MPXLF -qlanglvl=90ext -qfree=f90"
-	    fi
-	fi
-	MPILIBNAME=""
-	cross_compiling=yes
-	# Turn off the autoconf version 3 warning message
-	ac_tool_warned=yes
-	;;
-
-	sgimpi)
-	if test -z "$TESTCC" ; then TESTCC=${CC:=cc} ; fi
-	if test -z "$TESTF77" ; then TESTF77=${F77:=f77} ; fi
-	if test -z "$TESTCXX" ; then TESTCXX=${CXX:=CC} ; fi
-	if test -z "$TESTFC" ; then TESTFC=${FC:=f90} ; fi
-	# Must check for the MPI library in a separate macro - adding
-	# a test here will cause autoconf to prematurely define the
-	# C compiler
-	MPIRUN=mpirun
-	MPIBOOT=""
-	MPIUNBOOT=""
-	;;
-
-	generic)
-	# in $with_mpi/bin or $with_mpi
-        if test "X$MPICC" = "X" ; then
-            if test -x "$with_mpi/bin/mpicc" ; then
-                MPICC=$with_mpi/bin/mpicc
-	    elif test -x "$with_mpi/mpicc" ; then
-	        MPICC=$with_mpi/mpicc
-            fi
-        fi
-        if test "X$MPICXX" = "X" ; then
-            if test -x "$with_mpi/bin/mpicxx" ; then
-                MPICXX=$with_mpi/bin/mpicxx
-	    elif test -x "$with_mpi/mpicxx" ; then
-	        MPICXX=$with_mpi/mpicxx
-            fi
-        fi
-        if test "X$MPIF77" = "X" ; then
-            if test -x "$with_mpi/bin/mpif77" ; then
-                MPIF77=$with_mpi/bin/mpif77
-	    elif test -x "$with_mpi/mpif77" ; then
-	        MPIF77=$with_mpi/mpif77
-            fi
-        fi
-        if test "X$MPIF90" = "X" ; then
-            if test -x "$with_mpi/bin/mpif90" ; then
-                MPIF90=$with_mpi/bin/mpif90
-	    elif test -x "$with_mpi/mpif90" ; then
-	        MPIF90=$with_mpi/mpif90
-            fi
-        fi
-        if test "X$MPIEXEC" = "X" ; then
-            if test -x "$with_mpi/bin/mpiexec" ; then
-                MPIEXEC=$with_mpi/bin/mpiexec
-	    elif test -x "$with_mpi/mpiexec" ; then
-	        MPIEXEC=$with_mpi/mpiexec
-            fi
-        fi
-        CC=$MPICC
-        F77=$MPIF77
-	if test "X$MPICXX" != "X" ; then CXX=$MPICXX ; fi
-	if test "X$MPIF90" != "X" ; then F90=$MPIF90 ; fi
-	;;
-
-	*)
-	# Use the default choices for the compilers
-	;;
-esac
-])
-
-AC_DEFUN([PAC_MPI_FIND_COMPILERS],[
-# Tell autoconf to determine properties of the compilers (these are the 
-# compilers for MPI programs)
-PAC_PROG_CC
-if test "$enable_f77" != no -a "$enable_fortran" != no ; then
-    AC_PROG_F77
-fi
-if test "$enable_cxx" != no ; then
-    AC_PROG_CXX
-fi
-if test "$enable_f90" != no ; then
-    PAC_PROG_FC
-fi
-])
-
-dnl
-dnl This uses the selected CC etc to check for include paths and libraries
-AC_DEFUN([PAC_MPI_CHECK_MPI_LIB],[
-AC_REQUIRE([AC_PROG_CC])
-case $ac_mpi_type in
-    mpich)
-	;;
-
-    mpichnt)
-        dnl
-        dnl This isn't adequate, but it helps with using MPICH-NT/SDK.gcc
-	PAC_PUSH_FLAG([CFLAGS])
-        CFLAGS="$CFLAGS -I$with_mpichnt/include"
-	PAC_PUSH_FLAG([CPPFLAGS])
-        CPPFLAGS="$CPPFLAGS -I$with_mpichnt/include"
-	PAC_PUSH_FLAG([LDFLAGS])
-        LDFLAGS="$LDFLAGS -L$with_mpichnt/lib"
-        AC_CHECK_LIB(mpich,MPI_Init,found="yes",found="no")
-        if test "$found" = "no" ; then
-          AC_CHECK_LIB(mpich,MPI_Init,found="yes",found="no")
-        fi
-	if test "$enable_cxx" != no ; then
-	    AC_PROG_CXX
-	fi
-	if test "$enable_f90" != no ; then
-	    PAC_PROG_FC
-	fi
-	# Set defaults for the TEST versions if not already set
-	if test -z "$TESTCC" ; then TESTCC=${CC:=cc} ; fi
-	if test -z "$TESTF77" ; then TESTF77=${F77:=f77} ; fi
-	if test -z "$TESTCXX" ; then TESTCXX=${CXX:=CC} ; fi
-	if test -z "$TESTFC" ; then TESTFC=${FC:=f90} ; fi
-        if test "$found" = "no" ; then
-	  PAC_POP_FLAG([CFLAGS])
-	  PAC_POP_FLAG([CPPFLAGS])
-	  PAC_POP_FLAG([LDFLAGS])
-        fi
-        ;;
-
-    lammpi)
-	;;
-
-    ibmmpi)
-	;;
-
-    sgimpi)
-	AC_CHECK_LIB(mpi,MPI_Init)
-	if test "$ac_cv_lib_mpi_MPI_Init" = "yes" ; then
-	    MPILIBNAME="mpi"
-	fi	
-	;;
-
-    generic)
-	AC_SEARCH_LIBS(MPI_Init,mpi mpich mpich)
-	if test "$ac_cv_lib_mpi_MPI_Init" = "yes" ; then
-	    MPILIBNAME="mpi"
-	fi	
-	;;
-
-    *)
-	;;
-esac
-])
-
-dnl
-dnl/*D
-dnl PAC_MPI_F2C - Determine if MPI has the MPI-2 functions MPI_xxx_f2c and
-dnl   MPI_xxx_c2f
-dnl
-dnl Output Effect:
-dnl Define 'HAVE_MPI_F2C' if the routines are found.
-dnl
-dnl Notes:
-dnl Looks only for 'MPI_Request_c2f'.
-dnl D*/
-AC_DEFUN([PAC_MPI_F2C],[
-AC_CACHE_CHECK([for MPI F2C and C2F routines],
-pac_cv_mpi_f2c,
-[
-AC_TRY_LINK([#include "mpi.h"],
-[MPI_Request request;MPI_Fint a;a = MPI_Request_c2f(request);],
-pac_cv_mpi_f2c="yes",pac_cv_mpi_f2c="no")
-])
-if test "$pac_cv_mpi_f2c" = "yes" ; then 
-    AC_DEFINE(HAVE_MPI_F2C,1,[Define if MPI has F2C]) 
-fi
-])
-dnl
-dnl/*D
-dnl PAC_HAVE_ROMIO - make mpi.h include mpio.h if romio enabled
-dnl
-dnl Output Effect:
-dnl expands @HAVE_ROMIO@ in mpi.h into #include "mpio.h"
-dnl D*/
-AC_DEFUN([PAC_HAVE_ROMIO],[
-if test "$enable_romio" = "yes" ; then HAVE_ROMIO='#include "mpio.h"'; fi
-AC_SUBST(HAVE_ROMIO)
-])
diff --git a/3rd-party/romio321/confdb/aclocal_romio.m4 b/3rd-party/romio321/confdb/aclocal_romio.m4
deleted file mode 100644
index af80d3c1024..00000000000
--- a/3rd-party/romio321/confdb/aclocal_romio.m4
+++ /dev/null
@@ -1,872 +0,0 @@
-dnl
-dnl This files contains additional macros for using autoconf to 
-dnl build configure scripts.
-dnl
-dnl Almost all of this file is taken from the aclocal.m4 of MPICH
-dnl
-dnl Get the format of Fortran names.  Uses F77, FFLAGS, and sets WDEF.
-dnl If the test fails, sets NOF77 to 1, HAVE_FORTRAN to 0.
-dnl
-dnl
-AC_DEFUN([PAC_GET_FORTNAMES],[
-   rm -f confftest.f confftest.$OBJEXT
-   cat > confftest.f <<EOF
-       subroutine mpir_init_fop( a )
-       integer a
-       a = 1
-       return
-       end
-EOF
-   $F77 $FFLAGS -c confftest.f > /dev/null 2>&1
-   if test ! -s confftest.$OBJEXT ; then
-	AC_MSG_WARN([Unable to test Fortran compiler.  Compiling a test 
-program failed to produce an object file])
-	NOF77=1
-   elif test -z "$FORTRANNAMES" ; then
-     # MAC OS X (and probably FreeBSD need strings - (not strings -a)
-     # Cray doesn't accept -a ...
-     allstrings="-a"
-     if test $arch_CRAY ; then 
-	allstrings="" 
-     elif strings - confftest.$OBJEXT < /dev/null >/dev/null 2>&1 ; then
-         allstrings="-"
-     elif strings -a confftest.$OBJEXT < /dev/null >/dev/null 2>&1 ; then
-         allstrings="-a"
-     fi
-    
-     nameform1=`strings $allstrings confftest.$OBJEXT | grep mpir_init_fop_  | head -1`
-     nameform2=`strings $allstrings confftest.$OBJEXT | grep MPIR_INIT_FOP   | head -1`
-     nameform3=`strings $allstrings confftest.$OBJEXT | grep mpir_init_fop   | head -1`
-     nameform4=`strings $allstrings confftest.$OBJEXT | grep mpir_init_fop__ | head -1`
-    rm -f confftest.f confftest.$OBJEXT
-    if test -n "$nameform4" ; then
-	echo "Fortran externals are lower case and have two trailing underscores"
-	FORTRANNAMES="FORTRANDOUBLEUNDERSCORE"
-    elif test -n "$nameform1" ; then
-        # We don't set this in CFLAGS; it is a default case
-        echo "Fortran externals have a trailing underscore and are lowercase"
-	FORTRANNAMES="FORTRANUNDERSCORE"
-    elif test -n "$nameform2" ; then
-	echo "Fortran externals are uppercase"     
-	FORTRANNAMES="FORTRANCAPS" 
-    elif test -n "$nameform3" ; then
-	echo "Fortran externals are lower case"
-	FORTRANNAMES="FORTRANNOUNDERSCORE"
-    else
-	AC_MSG_WARN([Unable to determine the form of Fortran external names.
-Make sure that the compiler $F77 can be run on this system.
-Turning off Fortran (-nof77 being assumed)])
-	NOF77=1
-    fi
-    fi
-    if test -n "$FORTRANNAMES" ; then
-        WDEF="-D$FORTRANNAMES"
-    fi
-    # Delete confftest files with any extension.  This catches the case
-    # where auxillary files, such as coverage files, are removed.
-    rm -f confftest.*
-    ])dnl
-dnl
-define(PAC_GET_SPECIAL_SYSTEM_INFO,[
-#
-if test -n "$arch_IRIX"; then
-   AC_MSG_CHECKING(for IRIX OS version)
-   dnl This block of code replaces a generic "IRIX" arch value with
-   dnl  IRIX_<version>_<chip>
-   dnl  For example
-   dnl  IRIX_5_4400 (IRIX 5.x, using MIPS 4400)
-   osversion=`uname -r | sed 's/\..*//'`
-   dnl Note that we need to allow brackets here, so we briefly turn off 
-   dnl the macro quotes
-   changequote(,)dnl
-   dnl Get the second field (looking for 6.1)
-   osvminor=`uname -r | sed 's/[0-9]\.\([0-9]*\)\..*/\1/'`
-   changequote([,])dnl
-   AC_MSG_RESULT($osversion)
-   dnl Get SGI processor count by quick hack
-   AC_MSG_CHECKING(for IRIX cpucount)
-   changequote(,)dnl
-   cpucount=`hinv | grep '[0-9]* [0-9]* MHZ IP[0-9]* Proc' | cut -f 1 -d' '`
-   if test "$cpucount" = "" ; then
-     cpucount=`hinv | grep 'Processor [0-9]*:' | wc -l | sed -e 's/ //g'`
-   fi
-   changequote([,])dnl
-   if test "$cpucount" = "" ; then
-     AC_MSG_RESULT([Could not determine cpucount.  Please send])
-     hinv
-     AC_MSG_ERROR([to romio-maint@mcs.anl.gov])
-   fi
-   AC_MSG_RESULT($cpucount)
-   dnl
-   AC_MSG_CHECKING(for IRIX cpumodel)
-   dnl The tail -1 is necessary for multiple processor SGI boxes
-   dnl We might use this to detect SGI multiprocessors and recommend
-   dnl -comm=shared
-   cputype=`hinv -t cpu | tail -1 | cut -f 3 -d' '`
-   if test -z "$cputype" ; then
-        AC_MSG_RESULT([Could not get cputype from hinv -t cpu command. Please send])
-        hinv -t cpu 2>&1
-        hinv -t cpu | cut -f 3 -d' ' 2>&1
-	AC_MSG_ERROR([to romio-maint@mcs.anl.gov])
-   fi
-   AC_MSG_RESULT($cputype)
-   dnl echo "checking for osversion and cputype"
-   dnl cputype may contain R4400, R2000A/R3000, or something else.  
-   dnl We may eventually need to look at it.
-   if test -z "$osversion" ; then
-        AC_MSG_RESULT([Could not determine OS version.  Please send])
-        uname -a
-        AC_MSG_ERROR([to romio-maint@mcs.anl.gov])
-   elif test $osversion = 4 ; then
-        true
-   elif test $osversion = 5 ; then
-        true
-   elif test $osversion = 6 ; then
-        true
-   else 
-       AC_MSG_RESULT([Could not recognize the version of IRIX (got $osversion).
-ROMIO knows about versions 4, 5 and 6; the version being returned from 
-uname -r is $osversion.  Please send])
-       uname -a 2>&1
-       hinv 2>&1
-       AC_MSG_ERROR([to romio-maint@mcs.anl.gov])
-   fi
-   AC_MSG_CHECKING(for cputype)
-   OLD_ARCH=IRIX
-   IRIXARCH="$ARCH_$osversion"
-   dnl Now, handle the chip set
-   changequote(,)dnl
-   cputype=`echo $cputype | sed -e 's%.*/%%' -e 's/R//' | tr -d "[A-Z]"`
-   changequote([,])dnl
-   case $cputype in 
-        3000) ;;
-        4000) ;;
-        4400) ;;
-        4600) ;;
-        5000) ;;
-        8000) ;;
-        10000);;
-	12000);;
-        *)
-	AC_MSG_WARN([Unexpected IRIX/MIPS chipset $cputype.  Please send the output])
-        uname -a 2>&1
-        hinv 2>&1 
-        AC_MSG_WARN([to romio-maint@mcs.anl.gov
-ROMIO will continue and assume that the cputype is
-compatible with a MIPS 4400 processor.])
-        cputype=4400
-        ;;
-   esac
-   AC_MSG_RESULT($cputype)
-   IRIXARCH="$IRIXARCH_$cputype"
-   echo "IRIX-specific architecture is $IRIXARCH"
-fi
-])dnl
-dnl
-dnl
-define(PAC_TEST_MPI,[
-  AC_MSG_CHECKING(if a simple MPI program compiles and links)
-  rm -f mpitest.c
-  cat > mpitest.c <<EOF
-#include "mpi.h"
-     main(int argc, char **argv)
-     {
-         MPI_Init(&argc,&argv);
-         MPI_Finalize(); 
-     }
-EOF
-  rm -f conftest$EXEEXT
-  cmd="$CC $USER_CFLAGS -I$MPI_INCLUDE_DIR -o conftest$EXEEXT mpitest.c $MPI_LIB"
-  echo "$as_me:$LINENO: $cmd" >&5
-  $cmd >&5 2>&5
-  if test ! -x conftest$EXEEXT ; then
-      echo "$as_me:$LINENO: failed program was:" >&5
-      sed 's/^/| /' mpitest.c >&5
-      rm -f conftest$EXEEXT mpitest.c
-      AC_MSG_ERROR([Unable to compile a simple MPI program.
-Use environment variables to provide the location of MPI libraries and
-include directories])
-  else
-      rm -f conftest$EXEEXT mpitest.c
-  fi
-AC_MSG_RESULT(yes)
-])dnl
-dnl
-dnl
-dnl
-define(PAC_NEEDS_FINT,[
-  AC_MSG_CHECKING(if MPI_Fint is defined in the MPI implementation)
-  cat > mpitest1.c <<EOF
-#include "mpi.h"
-     main()
-     {
-         MPI_Fint i;
-         i = 0;
-     }
-EOF
-  rm -f mpitest1.$OBJEXT
-  $CC $USER_CFLAGS -I$MPI_INCLUDE_DIR -c mpitest1.c > /dev/null 2>&1
-  if test ! -s mpitest1.$OBJEXT ; then
-      NEEDS_MPI_FINT="#define NEEDS_MPI_FINT"
-      CFLAGS="$CFLAGS -DNEEDS_MPI_FINT"
-      AC_MSG_RESULT(no)
-      rm -f mpitest1.$OBJEXT mpitest1.c
-  else
-      NEEDS_MPI_FINT=""
-      AC_MSG_RESULT(yes)
-      rm -f mpitest1.$OBJEXT mpitest1.c
-  fi
-])dnl
-dnl
-define(PAC_MPI_LONG_LONG_INT,[
-  AC_MSG_CHECKING(if MPI_LONG_LONG_INT is defined in mpi.h)
-  rm -f mpitest.c
-  cat > mpitest.c <<EOF
-#include "mpi.h"
-     main(int argc, char **argv)
-     {
-         long long i;   
-         MPI_Init(&argc,&argv);
-         MPI_Send(&i, 1, MPI_LONG_LONG_INT, 0, 0, MPI_COMM_WORLD);
-         MPI_Finalize(); 
-     }
-EOF
-  rm -f conftest$EXEEXT
-  $CC $USER_CFLAGS -I$MPI_INCLUDE_DIR -o conftest$EXEEXT mpitest.c $MPI_LIB > /dev/null 2>&1
-  if test -x conftest$EXEEXT ; then
-      AC_MSG_RESULT(yes)
-      AC_DEFINE(HAVE_MPI_LONG_LONG_INT,,[Define if mpi has long long it])
-  else
-      AC_MSG_RESULT(no)
-  fi
-  rm -f conftest$EXEEXT mpitest.c
-])dnl
-dnl
-dnl PAC_LONG_LONG_64: check if there is a 64-bit long long
-dnl
-define(PAC_LONG_LONG_64,[
-if test -n "$longlongsize" ; then
-    if test "$longlongsize" = 8 ; then
-       echo "defining MPI_Offset as long long in C and integer*8 in Fortran" 
-       AC_DEFINE(HAVE_LONG_LONG_64,,[Define if long long is 64 bits])
-       DEFINE_MPI_OFFSET="typedef long long MPI_Offset;"
-       FORTRAN_MPI_OFFSET="integer*8"
-       LL="\%lld"
-    elif test "$longlongsize" = "int" ; then  # a hack to set MPI_Offset as int
-       echo "defining MPI_Offset as int in C and integer in Fortran"
-       DEFINE_MPI_OFFSET="typedef int MPI_Offset;"
-       FORTRAN_MPI_OFFSET="integer"
-       AC_DEFINE(MPI_OFFSET_IS_INT,,[Define if MPI_Offset is int])
-       LL="\%d"
-       MPI_OFFSET_KIND1="!"
-       MPI_OFFSET_KIND2="!"
-    else 
-       echo "defining MPI_Offset as long in C and integer in Fortran" 
-       DEFINE_MPI_OFFSET="typedef long MPI_Offset;"
-       FORTRAN_MPI_OFFSET="integer"
-       LL="\%ld"
-       MPI_OFFSET_KIND1="!"
-       MPI_OFFSET_KIND2="!"
-    fi
-else
-   PAC_GET_TYPE_SIZE(long long, longlongsize)
-   if test -n "$longlongsize" ; then
-      if test "$longlongsize" = 8 ; then
-         PAC_TEST_LONG_LONG()
-      else
-         echo "defining MPI_Offset as long in C and integer in Fortran" 
-         DEFINE_MPI_OFFSET="typedef long MPI_Offset;"
-         FORTRAN_MPI_OFFSET="integer"
-         LL="\%ld"
-         MPI_OFFSET_KIND1="!"
-         MPI_OFFSET_KIND2="!"
-      fi
-   else 
-dnl   check if longlong is not supported or only its size cannot be determined
-dnl   because the program cannot be run.
-      rm -f ltest.c
-      cat > ltest.c <<EOF
-        main()
-        {
-           long long i=8;
-           return 0;
-        }
-EOF
-      rm -f conftest$EXEEXT
-      $CC $USER_CFLAGS -o conftest$EXEEXT ltest.c > /dev/null 2>&1
-      if test -x conftest$EXEEXT ; then
-         echo "assuming size of long long is 8bytes; use '-longlongsize' to indicate otherwise"
-         rm -f conftest$EXEEXT ltest.c
-         echo "defining MPI_Offset as long long in C and integer*8 in Fortran" 
-         AC_DEFINE(HAVE_LONG_LONG_64,,[Define if long long is 64 bits])
-         DEFINE_MPI_OFFSET="typedef long long MPI_Offset;"
-         FORTRAN_MPI_OFFSET="integer*8"
-         LL="\%lld"
-      else 
-         echo "assuming long long is not available; use '-longlongsize' to indicate otherwise"
-         echo "defining MPI_Offset as long in C and integer in Fortran" 
-         DEFINE_MPI_OFFSET="typedef long MPI_Offset;"
-         FORTRAN_MPI_OFFSET="integer"
-         LL="\%ld"
-         MPI_OFFSET_KIND1="!"
-         MPI_OFFSET_KIND2="!"
-      fi
-   fi
-fi
-])dnl
-dnl
-dnl
-define(PAC_MPI_INFO,[
-  AC_MSG_CHECKING(if MPI_Info functions are defined in the MPI implementation)
-  rm -f mpitest.c
-  cat > mpitest.c <<EOF
-#include "mpi.h"
-     main(int argc, char **argv)
-     {
-         MPI_Info info;
-         MPI_Init(&argc,&argv);
-         MPI_Info_create(&info);
-         MPI_Finalize(); 
-     }
-EOF
-  rm -f conftest$EXEEXT
-  $CC $USER_CFLAGS -I$MPI_INCLUDE_DIR -o conftest$EXEEXT mpitest.c $MPI_LIB > /dev/null 2>&1
-  if test -x conftest$EXEEXT ; then
-      AC_MSG_RESULT(yes)
-      AC_DEFINE(HAVE_MPI_INFO,1,[Define if MPI_Info available])
-      HAVE_MPI_INFO="#define HAVE_MPI_INFO"
-      MPI_FINFO1="!"
-      MPI_FINFO2="!"
-      MPI_FINFO3="!"
-      MPI_FINFO4="!"
-  else
-      AC_MSG_RESULT(no)
-      BUILD_MPI_INFO=1
-      MPI_FINFO1="      INTEGER MPI_MAX_INFO_KEY, MPI_MAX_INFO_VAL"
-      MPI_FINFO2="      PARAMETER (MPI_MAX_INFO_KEY=255, MPI_MAX_INFO_VAL=1024)"
-      MPI_FINFO3="      INTEGER MPI_INFO_NULL"
-      MPI_FINFO4="      PARAMETER (MPI_INFO_NULL=0)"
-  fi
-  rm -f conftest$EXEEXT mpitest.c
-])dnl
-dnl
-dnl
-define(PAC_MPI_DARRAY_SUBARRAY,[
-  AC_MSG_CHECKING(if darray and subarray constructors are defined in the MPI implementation)
-  rm -f mpitest.c
-  cat > mpitest.c <<EOF
-#include "mpi.h"
-     main(int argc, char **argv)
-     {
-         int i=MPI_DISTRIBUTE_CYCLIC;
-         MPI_Datatype t;
-         MPI_Init(&argc,&argv);
-         MPI_Type_create_darray(i, i, i, &i, &i, &i, &i, i, MPI_INT, &t);
-         MPI_Type_create_subarray(i, &i, &i, &i, i, MPI_INT, &t);
-         MPI_Finalize(); 
-     }
-EOF
-  rm -f conftest$EXEEXT
-  $CC $USER_CFLAGS -I$MPI_INCLUDE_DIR -o conftest$EXEEXT mpitest.c $MPI_LIB > /dev/null 2>&1
-  if test -x conftest$EXEEXT ; then
-      AC_MSG_RESULT(yes)
-      AC_DEFINE(HAVE_MPI_DARRAY_SUBARRAY,,[Define if MPI Darray available])
-      HAVE_MPI_DARRAY_SUBARRAY="#define HAVE_MPI_DARRAY_SUBARRAY"
-      MPI_FARRAY1="!"
-      MPI_FARRAY2="!"
-      MPI_FARRAY3="!"
-      MPI_FARRAY4="!"
-      MPI_FARRAY5="!"
-      MPI_FARRAY6="!"
-      MPI_FARRAY7="!"
-  else
-      AC_MSG_RESULT(no)
-      BUILD_MPI_ARRAY=1
-      MPI_FARRAY1="      INTEGER MPI_ORDER_C, MPI_ORDER_FORTRAN"
-      MPI_FARRAY2="      PARAMETER (MPI_ORDER_C=56, MPI_ORDER_FORTRAN=57)"
-      MPI_FARRAY3="      INTEGER MPI_DISTRIBUTE_BLOCK, MPI_DISTRIBUTE_CYCLIC"
-      MPI_FARRAY4="      INTEGER MPI_DISTRIBUTE_NONE, MPI_DISTRIBUTE_DFLT_DARG"
-      MPI_FARRAY5="      PARAMETER (MPI_DISTRIBUTE_BLOCK=121, MPI_DISTRIBUTE_CYCLIC=122)"
-      MPI_FARRAY6="      PARAMETER (MPI_DISTRIBUTE_NONE=123)"
-      MPI_FARRAY7="      PARAMETER (MPI_DISTRIBUTE_DFLT_DARG=-49767)"
-  fi
-  rm -f conftest$EXEEXT mpitest.c
-])dnl
-dnl
-dnl
-define(PAC_CHECK_MPI_SGI_INFO_NULL,[
-  AC_MSG_CHECKING([if MPI_INFO_NULL is defined in mpi.h])
-  rm -f mpitest.c
-  cat > mpitest.c <<EOF
-#include "mpi.h"
-     main(int argc, char **argv)
-     {
-	int i;
-	i = MPI_INFO_NULL;
-     }
-EOF
-  rm -f conftest$EXEEXT
-  $CC $USER_CFLAGS -I$MPI_INCLUDE_DIR -o conftest$EXEEXT mpitest.c $MPI_LIB > /dev/null 2>&1
-  if test -x conftest$EXEEXT ; then
-      AC_MSG_RESULT(yes)
-      cp adio/sgi/mpi3.1/*.h include
-  else
-      AC_MSG_RESULT(no)
-  fi
-  rm -f conftest$EXEEXT mpitest.c
-])dnl
-dnl
-dnl
-dnl
-define(PAC_CHECK_MPIOF_H,[
-  AC_MSG_CHECKING(if mpiof.h is included in mpif.h)
-  rm -f mpitest.f
-  cat > mpitest.f <<EOF
-      program main
-      implicit none
-      include 'mpif.h'
-      integer i
-      i = MPI_MODE_RDWR
-      stop
-      end
-EOF
-  rm -f conftest$EXEEXT
-  $F77 $FFLAGS -I$MPI_INCLUDE_DIR -o conftest$EXEEXT mpitest.f $MPI_LIB > /dev/null 2>&1
-  if test -x conftest$EXEEXT ; then
-      AC_MSG_RESULT(yes)
-      MPIOF_H_INCLUDED=1
-  else
-      AC_MSG_RESULT(no)
-  fi
-  rm -f conftest$EXEEXT mpitest.f
-])dnl
-dnl
-dnl
-dnl check if pread64 is defined in IRIX. needed on IRIX 6.5
-dnl
-define(PAC_HAVE_PREAD64,[
-  AC_MSG_CHECKING(if pread64 is defined)
-  rm -f conftest.c
-  cat > conftest.c <<EOF
-#include <unistd.h>
-     main()
-     {
-         int fd=0, buf=0, i=0;
-         off64_t off=0;
-         pread64(fd, &buf, i, off);
-     }
-EOF
-  rm -f conftest$EXEEXT
-  $CC $USER_CFLAGS -o conftest$EXEEXT conftest.c > /dev/null 2>&1
-  if test -x conftest$EXEEXT ; then
-      AC_MSG_RESULT(yes)
-      AC_DEFINE(HAVE_PREAD64,,[Define if pread64 available])
-  else
-      AC_MSG_RESULT(no)
-  fi
-rm -f conftest$EXEEXT conftest.c
-])dnl
-dnl
-dnl
-define(PAC_TEST_MPI_SGI_type_is_contig,[
-  AC_MSG_CHECKING(if MPI_SGI_type_is_contig is defined)
-  rm -f mpitest.c
-  cat > mpitest.c <<EOF
-#include "mpi.h"
-     main(int argc, char **argv)
-     {
-         MPI_Datatype type;
-         int i;
-
-         MPI_Init(&argc,&argv);
-         i = MPI_SGI_type_is_contig(type);
-         MPI_Finalize(); 
-     }
-EOF
-  rm -f conftest$EXEEXT
-  $CC $USER_CFLAGS -I$MPI_INCLUDE_DIR -o conftest$EXEEXT mpitest.c $MPI_LIB > /dev/null 2>&1
-  if test -x conftest$EXEEXT ; then
-     AC_MSG_RESULT(yes)
-  else
-     AC_MSG_RESULT(no)
-     AC_DEFINE(NO_MPI_SGI_type_is_contig,,[Define if no MPI type is contig])
-  fi
-  rm -f conftest$EXEEXT mpitest.c
-])dnl
-dnl
-dnl
-dnl
-define(PAC_TEST_MPI_COMBINERS,[
-  AC_MSG_CHECKING(if MPI-2 combiners are defined in mpi.h)
-  rm -f mpitest.c
-  cat > mpitest.c <<EOF
-#include "mpi.h"
-     main(int argc, char **argv)
-     {
-         int i;
-
-         MPI_Init(&argc,&argv);
-         i = MPI_COMBINER_STRUCT;
-         MPI_Finalize(); 
-     }
-EOF
-  rm -f conftest$EXEEXT
-  $CC $USER_CFLAGS -I$MPI_INCLUDE_DIR -o conftest$EXEEXT mpitest.c $MPI_LIB > /dev/null 2>&1
-  if test -x conftest$EXEEXT ; then
-     AC_MSG_RESULT(yes)
-     AC_DEFINE(HAVE_MPI_COMBINERS,,[Define if MPI combiners available])
-  else
-     AC_MSG_RESULT(no)
-  fi
-  rm -f conftest$EXEEXT mpitest.c
-])dnl
-dnl
-dnl
-dnl PAC_MPI_OFFSET_KIND()
-dnl
-dnl tries to determine the Fortran 90 kind parameter for 8-byte integers
-dnl
-define(PAC_MPI_OFFSET_KIND,
-[
-# This is needed for Mac OSX 10.5
-rm -rf conftest.dSYM
-rm -f conftest*
-# Determine the extension for Fortran 90 files (not all compilers accept
-# .f and not all accept .f90)
-if test -z "$ac_f90ext" ; then
-    if test -z "$FC" ; then
-       # This list should correspond to the list in aclocal_fc.m4
-       AC_CHECK_PROGS(FC,ifort pgf90 pathf90 pathf95 xlf90 xlf95 f90 epcf90 \
-       f95 fort lf95 gfortran g95 ifc efc)
-    fi
-    AC_MSG_CHECKING([for extension for Fortran 90 programs])
-    ac_f90ext="f90"
-    ac_f90compile='${FC-f90} -c $FCFLAGS conftest.$ac_f90ext 1>&AC_FD_CC'
-    cat > conftest.$ac_f90ext <<EOF
-      program conftest
-      end
-EOF
-    if AC_TRY_EVAL(ac_f90compile) ; then
-        AC_MSG_RESULT([f90])
-    else
-	# This is needed for Mac OSX 10.5
-	rm -rf conftest.dSYM
-        rm -f conftest*
-        ac_f90ext="f"
-        cat > conftest.$ac_f90ext <<EOF
-      program conftest
-      end
-EOF
-        if AC_TRY_EVAL(ac_f90compile) ; then
-            AC_MSG_RESULT([f])
-        else
-            AC_MSG_RESULT([unknown!])
-        fi
-    fi
-fi
-AC_MSG_CHECKING([for Fortran 90 KIND parameter for 8-byte integers])
-cat <<EOF > conftest.$ac_f90ext
-      program main
-      integer i
-      i = selected_int_kind(16)
-      open(8, file="conftest.out", form="formatted")
-      write (8,*) i
-      close(8)
-      stop
-      end
-EOF
-if test -z "$FC" ; then
-   FC=f90
-fi
-KINDVAL=""
-if $FC -o conftest$EXEEXT conftest.$ac_f90ext >/dev/null 2>&1 ; then
-    ./conftest$EXEEXT >/dev/null 2>&1
-    if test -s conftest.out ; then 
-        KINDVAL=`cat conftest.out`
-    fi
-fi
-# This is needed for Mac OSX 10.5
-rm -rf conftest.dSYM
-rm -f conftest*
-if test -n "$KINDVAL" -a "$KINDVAL" != "-1" ; then
-   AC_MSG_RESULT($KINDVAL)
-   MPI_OFFSET_KIND1="      INTEGER MPI_OFFSET_KIND"
-   MPI_OFFSET_KIND2="      PARAMETER (MPI_OFFSET_KIND=$KINDVAL)"
-else
-    AC_MSG_RESULT(unavailable)
-fi
-])dnl
-dnl
-dnl
-define(PAC_TEST_MPI_HAVE_OFFSET_KIND,[
-  AC_MSG_CHECKING(if MPI_OFFSET_KIND is defined in mpif.h)
-  rm -f mpitest.f
-  cat > mpitest.f <<EOF
-      program main
-      implicit none
-      include 'mpif.h'
-      integer i
-      i = MPI_OFFSET_KIND
-      stop
-      end
-EOF
-  rm -f conftest$EXEEXT
-  $F77 $FFLAGS -I$MPI_INCLUDE_DIR -o conftest$EXEEXT mpitest.f $MPI_LIB > /dev/null 2>&1
-  if test -x conftest$EXEEXT ; then
-     AC_MSG_RESULT(yes)
-     MPI_OFFSET_KIND1="!"
-     MPI_OFFSET_KIND2="!"
-  else
-     AC_MSG_RESULT(no)
-  fi
-  rm -f conftest$EXEEXT mpitest.f
-])dnl
-dnl
-dnl
-dnl PAC_GET_XFS_MEMALIGN
-dnl 
-dnl
-define(PAC_GET_XFS_MEMALIGN,
-[AC_MSG_CHECKING([for memory alignment needed for direct I/O])
-rm -f memalignval
-rm -f /tmp/romio_tmp.bin
-AC_TEST_PROGRAM([#include <stdio.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <stdio.h>
-main() { 
-  struct dioattr st;
-  int fd = open("/tmp/romio_tmp.bin", O_RDWR | O_CREAT, 0644);
-  FILE *f=fopen("memalignval","w");
-  if (fd == -1) exit(1);
-  if (!f) exit(1);
-  fcntl(fd, F_DIOINFO, &st);
-  fprintf( f, "%u\n", st.d_mem);
-  exit(0);
-}],Pac_CV_NAME=`cat memalignval`,Pac_CV_NAME="")
-rm -f memalignval
-rm -f /tmp/romio_tmp.bin
-if test -n "$Pac_CV_NAME" -a "$Pac_CV_NAME" != 0 ; then
-    AC_MSG_RESULT($Pac_CV_NAME)
-    CFLAGS="$CFLAGS -DXFS_MEMALIGN=$Pac_CV_NAME"
-else
-    AC_MSG_RESULT(unavailable, assuming 128)
-    CFLAGS="$CFLAGS -DXFS_MEMALIGN=128"
-fi
-])dnl
-dnl
-
-define(PAC_HAVE_MOUNT_NFS,[
-  AC_MSG_CHECKING([if MOUNT_NFS is defined in the include files])
-  rm -f conftest.c
-  cat > conftest.c <<EOF
-#include <sys/param.h>
-#include <sys/mount.h>
-     main()
-     {
-         int i=MOUNT_NFS;
-     }
-EOF
-  rm -f conftest$EXEEXT
-  $CC $USER_CFLAGS -o conftest$EXEEXT conftest.c > /dev/null 2>&1
-  if test -x conftest$EXEEXT ; then
-     AC_MSG_RESULT(yes)
-     ROMIO_HAVE_MOUNT_NFS=1
-     AC_DEFINE(HAVE_MOUNT_NFS,,[Define if MOUNT_NFS defined])
-  else
-     ROMIO_HAVE_MOUNT_NFS=0
-     AC_MSG_RESULT(no)
-  fi
-  rm -f conftest$EXEEXT conftest.c
-])dnl
-dnl
-dnl
-dnl PAC_MPI_OFFSET_KIND_4BYTE()
-dnl
-dnl tries to determine the Fortran 90 kind parameter for 4-byte integers
-dnl
-define(PAC_MPI_OFFSET_KIND_4BYTE,
-[AC_MSG_CHECKING([for Fortran 90 KIND parameter for 4-byte integers])
-rm -f kind.f kind.$OBJEXT kind$EXEEXT
-cat <<EOF > kind.f
-      program main
-      integer i
-      i = selected_int_kind(8)
-      open(8, file="k.out", form="formatted")
-      write (8,*) i
-      close(8)
-      stop
-      end
-EOF
-if test -z "$FC" ; then
-   FC=f90
-fi
-KINDVAL=""
-if $FC -o kind$EXEEXT kind.f >/dev/null 2>&1 ; then
-    ./kind >/dev/null 2>&1
-    if test -s k.out ; then 
-        KINDVAL=`cat k.out`
-    fi
-fi
-rm -f kind$EXEEXT k.out kind.f kind.$OBJEXT
-if test -n "$KINDVAL" -a "$KINDVAL" != "-1" ; then
-   AC_MSG_RESULT($KINDVAL)
-   MPI_OFFSET_KIND1="      INTEGER MPI_OFFSET_KIND"
-   MPI_OFFSET_KIND2="      PARAMETER (MPI_OFFSET_KIND=$KINDVAL)"
-else
-    AC_MSG_RESULT(unavailable)
-fi
-])dnl
-dnl
-dnl
-define(PAC_FUNC_STRERROR,[
-  AC_MSG_CHECKING([for strerror()])
-  rm -f conftest.c
-  cat > conftest.c <<EOF
-#include <string.h>
-     main()
-     {
-        char *s = strerror(5);
-     }
-EOF
-  rm -f conftest$EXEEXT
-  $CC $USER_CFLAGS -o conftest$EXEXT conftest.c >> config.log 2>&1
-  if test -x conftest$EXEEXT ; then
-     AC_MSG_RESULT(yes)
-     AC_DEFINE(HAVE_STRERROR,,[Define if strerror available])
-  else
-     AC_MSG_RESULT(no)
-     AC_MSG_CHECKING([for sys_errlist])
-     rm -f conftest.c
-changequote(,)
-     cat > conftest.c <<EOF
-#include <stdio.h>
-        main()
-        {
-           extern char *sys_errlist[];
-	   printf("%s\n", sys_errlist[34]);
-        }
-EOF
-changequote([,])
-     rm -f conftest$EXEEXT
-     $CC $USER_CFLAGS -o conftest$EXEEXT conftest.c > config.log 2>&1
-     if test -x conftest$EXEEXT ; then
-        AC_MSG_RESULT(yes)
-        AC_DEFINE(HAVE_SYSERRLIST,,[Define if syserrlist available])
-     else
-        AC_MSG_RESULT(no)
-     fi
-  fi
-  rm -f conftest$EXEEXT conftest.c
-])dnl
-dnl
-define(PAC_TEST_MPIR_STATUS_SET_BYTES,[
-  AC_MSG_CHECKING(if MPIR_Status_set_bytes is defined)
-  rm -f mpitest.c
-  cat > mpitest.c <<EOF
-#include "mpi.h"
-     main(int argc, char **argv)
-     {
-     	 MPI_Status status;
-         MPI_Datatype type;
-	 int err;
-
-         MPI_Init(&argc,&argv);
-         MPIR_Status_set_bytes(status,type,err);
-         MPI_Finalize(); 
-     }
-EOF
-  rm -f conftest$EXEEXT
-  $CC $USER_CFLAGS -I$MPI_INCLUDE_DIR -o conftest$EXEEXT mpitest.c $MPI_LIB > /dev/null 2>&1
-  if test -x conftest$EXEEXT ; then
-     AC_MSG_RESULT(yes)
-     AC_DEFINE(HAVE_STATUS_SET_BYTES,,[Define if status set bytes available])
-  else
-     AC_MSG_RESULT(no)
-  fi
-  rm -f conftest$EXEEXT mpitest.c
-])dnl
-define(PAC_TEST_MPIU_FUNCS,[
-  AC_MSG_CHECKING(support for MPICH memory macros)
-  rm -f mpitest.c
-  cat > mpitest.c <<EOF
-#include "mpi.h"
-#include "stdio.h"
-  main(Int argc, char **argv)
-  {
-      MPIU_Free(NULL);
-  }
-EOF
-  rm -f conftest$EXEEXT
-  $CC $USER_CFLAGS -I$MPI_INCLUDE_DIR -o conftest$EXEEXT mpitest.c $MPI_LIB > /dev/null 2>&1
-  if test -x conftest$EXEEXT ; then
-     AC_MSG_RESULT(yes)
-     AC_DEFINE(HAVE_MPIU_FUNCS,1,[Define if MPICH memory tracing macros defined])
-  else
-     AC_MSG_RESULT(no)
-  fi
-  rm -f conftest$EXEEXT mpitest.c
-])dnl
-dnl
-define(PAC_TEST_MPI_GREQUEST,[
-  AC_MSG_CHECKING(support for generalized requests)
-  rm -f mpitest.c
-  cat > mpitest.c <<EOF
-#include "mpi.h"
-#include "stdio.h"
-    main(int argc, char **argv)
-    {
-       MPI_Request request;
-       MPI_Init(&argc, &argv);
-       MPI_Grequest_start(NULL, NULL, NULL, NULL, &request);
-       MPI_Finalize();
-     }
-EOF
-  rm -f conftest$EXEEXT
-  $CC $USER_CFLAGS -I$MPI_INCLUDE_DIR -o conftest$EXEEXT mpitest.c $MPI_LIB > /dev/null 2>&1
-  if test -x conftest$EXEEXT ; then
-     AC_MSG_RESULT(yes)
-     AC_DEFINE(HAVE_MPI_GREQUEST,1,[Define if generalized requests avaliable])
-     DEFINE_HAVE_MPI_GREQUEST="#define HAVE_MPI_GREQUEST 1"
-  else
-     AC_MSG_RESULT(no)
-  fi
-  rm -f conftest$EXEEXT mpitest.c
-])dnl
-
-define(PAC_TEST_MPI_GREQUEST_EXTENSIONS,[
-  AC_MSG_CHECKING(support for non-standard extended generalized requests)
-  rm -f mpitest.c
-  cat > mpitest.c <<EOF
-#include "mpi.h"
-#include "stdio.h"
-    main(int argc, char **argv)
-    {
-       MPIX_Grequest_class classtest
-    }
-EOF
-  rm -f conftest$EXEEXT
-  $CC $USER_CFLAGS -I$MPI_INCLUDE_DIR -o conftest$EXEEXT mpitest.c $MPI_LIB > /dev/null 2>&1
-  if test -x conftest$EXEEXT ; then
-     AC_MSG_RESULT(yes)
-     AC_DEFINE(HAVE_MPI_GREQUEST_EXTENTIONS,1,[Define if non-standard generalized requests extensions avaliable])
-     DEFINE_HAVE_MPI_GREQUEST_EXTENSIONS="#define HAVE_MPI_GREQUEST_EXTENSIONS 1"
-  else
-     AC_MSG_RESULT(no)
-  fi
-  rm -f conftest$EXEEXT mpitest.c
-])dnl
-
-define(PAC_TEST_NEEDS_CONST,[
-   AC_MSG_CHECKING([const declarations needed in MPI routines])
-   AC_COMPILE_IFELSE([AC_LANG_SOURCE(
-   [ #include <mpi.h>
-     int MPI_File_delete(char *filename, MPI_Info info) { return (0); }
-   ] )],
-   [
-    AC_MSG_RESULT(no)
-   ],[
-    AC_MSG_RESULT(yes)
-    AC_DEFINE(HAVE_MPIIO_CONST, const, [Define if MPI-IO routines need a const qualifier])
-   ])
-   ])
diff --git a/3rd-party/romio321/confdb/aclocal_runlog.m4 b/3rd-party/romio321/confdb/aclocal_runlog.m4
deleted file mode 100644
index f22e6ddbc3a..00000000000
--- a/3rd-party/romio321/confdb/aclocal_runlog.m4
+++ /dev/null
@@ -1,218 +0,0 @@
-dnl
-dnl PAC_RUN_LOG mimics _AC_RUN_LOG which is autoconf internal routine.
-dnl We also make sure PAC_RUN_LOG can be used in AS_IF, so the last
-dnl test command should have terminating ]), i.e. without newline before ]).
-dnl
-AC_DEFUN([PAC_RUNLOG],[
-{ AS_ECHO(["$as_me:$LINENO: $1"]) >&AS_MESSAGE_LOG_FD
-  (eval $1) 2>&AS_MESSAGE_LOG_FD
-  ac_status=$?
-  AS_ECHO(["$as_me:$LINENO: \$? = $ac_status"]) >&AS_MESSAGE_LOG_FD
-  test $ac_status = 0; }])
-
-dnl
-dnl PAC_COMMAND_IFELSE is written to replace AC_TRY_EVAL with added logging
-dnl to config.log, i.e. AC_TRY_EVAL does not log anything to config.log.
-dnl If autoconf provides AC_COMMAND_IFELSE or AC_EVAL_IFELSE,
-dnl AC_COMMAND_IFELSE dnl should be replaced by the official autoconf macros.
-dnl
-dnl PAC_COMMAND_IFELSE(COMMMAND,[ACTION-IF-RUN-OK],[ACTION-IF-RUN-FAIL])
-dnl
-AC_DEFUN([PAC_COMMAND_IFELSE],[
-AS_IF([PAC_RUNLOG([$1])],[
-    $2
-],[
-    AS_ECHO(["$as_me: program exited with status $ac_status"]) >&AS_MESSAGE_LOG_FD
-    m4_ifvaln([$3],[
-        (exit $ac_status)
-        $3
-    ])
-])
-])
-
-AC_DEFUN([PAC_RUNLOG_IFELSE],[
-dnl pac_TESTLOG is the internal temporary logfile for this macro.
-pac_TESTLOG="pac_test.log"
-rm -f $pac_TESTLOG
-PAC_COMMAND_IFELSE([$1 > $pac_TESTLOG],[
-    ifelse([$2],[],[],[$2])
-],[
-    AS_ECHO(["*** $1 :"]) >&AS_MESSAGE_LOG_FD
-    cat $pac_TESTLOG >&AS_MESSAGE_LOG_FD
-    ifelse([$3],[],[],[$3])
-])
-rm -f $pac_TESTLOG
-])
-
-
-dnl PAC_VAR_PUSHVAL(VARNAME, [LastSavedValue]))
-dnl
-dnl Save the content of the shell variable, VARNAME, onto a stack.
-dnl The saved value of VARNAME is restorable with respect to the nesting
-dnl of the macro.
-dnl
-dnl The Last saved value of VARNAME on the stack is stored in shell variable
-dnl pac_LastSavedValueOf_$VARNAME if the 2nd argument is NOT supplied.
-dnl If the 2nd argument is present, the last saved value will be stored
-dnl in the 2nd argument instead.
-dnl
-dnl The First saved value of VARNAME on the stack is stored in shell variable
-dnl dnl pac_FirstSavedValueOf_$VARNAME.
-dnl
-AC_DEFUN([PAC_VAR_PUSHVAL],[
-# START of PUSHVAL
-dnl define local m4-name pac_stk_level.
-AS_VAR_PUSHDEF([pac_stk_level], [pac_stk_$1_level])
-AS_VAR_SET_IF([pac_stk_level],[
-    AS_VAR_ARITH([pac_stk_level], [$pac_stk_level + 1])
-],[
-    AS_VAR_SET([pac_stk_level], [0])
-])
-dnl AS_ECHO_N(["PUSHVAL: pac_stk_level = $pac_stk_level, "])
-dnl Save the content of VARNAME, i.e. $VARNAME, onto the stack.
-AS_VAR_SET([pac_stk_$1_$pac_stk_level],[$$1])
-AS_VAR_IF([pac_stk_level], [0], [
-    dnl Save the 1st pushed value of VARNAME as pac_FirstSavedValueOf_$VARNAME
-    AS_VAR_COPY([pac_FirstSavedValueOf_$1],[pac_stk_$1_$pac_stk_level])
-])
-ifelse([$2],[],[
-    dnl Save the last pushed value of VARNAME as pac_LastSavedValueOf_$VARNAME
-    AS_VAR_COPY([pac_LastSavedValueOf_$1],[pac_stk_$1_$pac_stk_level])
-    dnl AS_ECHO(["pac_LastSavedValueOf_$1 = $pac_LastSavedValueOf_$1"])
-],[
-    dnl Save the last pushed value of VARNAME as $2
-    AS_VAR_COPY([$2],[pac_stk_$1_$pac_stk_level])
-    dnl AS_ECHO(["$2 = $$2"])
-])
-AS_VAR_POPDEF([pac_stk_level])
-# END of PUSHVAL
-])
-dnl
-dnl
-dnl
-dnl PAC_VAR_POPVAL(VARNAME)
-dnl
-dnl Restore variable, VARNAME, from the stack.
-dnl This macro is safe with respect to the nesting.
-dnl Some minimal checking of nesting balance of PAC_VAR_PUSH[POP]VAL()
-dnl is done here.
-dnl
-AC_DEFUN([PAC_VAR_POPVAL],[
-# START of POPVAL
-dnl define local m4-name pac_stk_level.
-AS_VAR_PUSHDEF([pac_stk_level], [pac_stk_$1_level])
-AS_VAR_SET_IF([pac_stk_level],[
-    AS_VAR_IF([pac_stk_level],[-1],[
-        AC_MSG_WARN(["Imbalance of PUSHVAL/POPVAL of $1"])
-    ],[
-        dnl AS_ECHO_N(["POPVAL: pac_stk_level = $pac_stk_level, "])
-        AS_VAR_COPY([$1],[pac_stk_$1_$pac_stk_level])
-        dnl AS_ECHO(["popped_val = $$1"])
-        AS_VAR_ARITH([pac_stk_level], [ $pac_stk_level - 1 ])
-    ])
-],[
-    AC_MSG_WARN(["Uninitialized PUSHVAL/POPVAL of $1"])
-])
-AS_VAR_POPDEF([pac_stk_level])
-# END of POPVAL
-])
-dnl
-dnl
-dnl
-dnl PAC_COMPILE_IFELSE_LOG is a wrapper around AC_COMPILE_IFELSE with the
-dnl output of ac_compile to a specified logfile instead of AS_MESSAGE_LOG_FD
-dnl
-dnl PAC_COMPILE_IFELSE_LOG(logfilename, input,
-dnl                        [action-if-true], [action-if-false])
-dnl
-dnl where input, [action-if-true] and [action-if-false] are used
-dnl in AC_COMPILE_IFELSE(input, [action-if-true], [action-if-false]).
-dnl This macro is nesting safe.
-dnl
-AC_DEFUN([PAC_COMPILE_IFELSE_LOG],[
-dnl
-dnl Instead of defining our own ac_compile and do AC_TRY_EVAL
-dnl on these variables.  We modify ac_compile used by AC_*_IFELSE
-dnl by piping the output of the command to a logfile.  The reason is that
-dnl 1) AC_TRY_EVAL is discouraged by Autoconf. 2) defining our ac_compile
-dnl could mess up the usage and order of *CFLAGS, LDFLAGS and LIBS in
-dnl these commands, i.e. deviate from how GNU standard uses these variables.
-dnl
-dnl Replace ">&AS_MESSAGE_LOG_FD" by "> FILE 2>&1" in ac_compile.
-dnl Save a copy of ac_compile on a stack
-dnl which is safe through nested invocations of this macro.
-PAC_VAR_PUSHVAL([ac_compile])
-dnl Modify ac_compile based on the unmodified ac_compile.
-ac_compile="`echo $pac_FirstSavedValueOf_ac_compile | sed -e 's|>.*$|> $1 2>\&1|g'`"
-AC_COMPILE_IFELSE([$2],[
-    ifelse([$3],[],[:],[$3])
-],[
-    ifelse([$4],[],[:],[$4])
-])
-dnl Restore the original ac_compile from the stack.
-PAC_VAR_POPVAL([ac_compile])
-])
-dnl
-dnl
-dnl
-dnl PAC_LINK_IFELSE_LOG is a wrapper around AC_LINK_IFELSE with the
-dnl output of ac_link to a specified logfile instead of AS_MESSAGE_LOG_FD
-dnl
-dnl PAC_LINK_IFELSE_LOG(logfilename, input,
-dnl                     [action-if-true], [action-if-false])
-dnl
-dnl where input, [action-if-true] and [action-if-false] are used
-dnl in AC_LINK_IFELSE(input, [action-if-true], [action-if-false]).
-dnl This macro is nesting safe.
-dnl
-AC_DEFUN([PAC_LINK_IFELSE_LOG],[
-dnl
-dnl Instead of defining our own ac_link and do AC_TRY_EVAL
-dnl on these variables.  We modify ac_link used by AC_*_IFELSE
-dnl by piping the output of the command to a logfile.  The reason is that
-dnl 1) AC_TRY_EVAL is discouraged by Autoconf. 2) defining our ac_link
-dnl could mess up the usage and order of *CFLAGS, LDFLAGS and LIBS in
-dnl these commands, i.e. deviate from how GNU standard uses these variables.
-dnl
-dnl Replace ">&AS_MESSAGE_LOG_FD" by "> FILE 2>&1" in ac_link.
-dnl Save a copy of ac_link on a stack
-dnl which is safe through nested invocations of this macro.
-PAC_VAR_PUSHVAL([ac_link])
-dnl Modify ac_link based on the unmodified ac_link.
-ac_link="`echo $pac_FirstSavedValueOf_ac_link | sed -e 's|>.*$|> $1 2>\&1|g'`"
-dnl
-AC_LINK_IFELSE([$2],[
-    ifelse([$3],[],[:],[$3])
-],[
-    ifelse([$4],[],[:],[$4])
-])
-dnl Restore the original ac_link from the stack.
-PAC_VAR_POPVAL([ac_link])
-])
-dnl
-dnl
-dnl
-dnl PAC_COMPLINK_IFELSE (input1, input2, [action-if-true], [action-if-false])
-dnl
-dnl where input1 and input2 are either AC_LANG_SOURCE or AC_LANG_PROGRAM
-dnl enclosed input programs.
-dnl
-dnl The macro first compiles input1 and uses the object file created
-dnl as part of LIBS during linking.  This macro is nesting safe.
-dnl
-AC_DEFUN([PAC_COMPLINK_IFELSE],[
-AC_COMPILE_IFELSE([$1],[
-    PAC_RUNLOG([mv conftest.$OBJEXT pac_conftest.$OBJEXT])
-    PAC_VAR_PUSHVAL([LIBS])
-    LIBS="pac_conftest.$OBJEXT $pac_FirstSavedValueOf_LIBS"
-    AC_LINK_IFELSE([$2],[
-        ifelse([$3],[],[:],[$3])
-    ],[
-        ifelse([$4],[],[:],[$4])
-    ])
-    PAC_VAR_POPVAL([LIBS])
-    rm -f pac_conftest.$OBJEXT
-],[
-    ifelse([$4],[],[:],[$4])
-])
-])
diff --git a/3rd-party/romio321/confdb/aclocal_shl.m4 b/3rd-party/romio321/confdb/aclocal_shl.m4
deleted file mode 100644
index f974bfa2261..00000000000
--- a/3rd-party/romio321/confdb/aclocal_shl.m4
+++ /dev/null
@@ -1,500 +0,0 @@
-dnl
-dnl Definitions for creating shared libraries
-dnl
-dnl The purpose of these definitions is to provide common support for 
-dnl shared libraries, with *or without* the use of the GNU Libtool package.
-dnl For many of our important platforms, the Libtool approach is overkill,
-dnl and can be particularly painful for developers.
-dnl
-dnl To use libtool, you need macros that are defined by libtool for libtool
-dnl Don't even think about the consequences of this for updating and for
-dnl using user-versions of libtool :(
-dnl 
-dnl !!!!!!!!!!!!!!!!!!!!!
-dnl libtool requires ac 2.50 !!!!!!!!!!!!!!!!!
-dnl 
-dnl builtin(include,libtool.m4)
-dnl
-dnl/*D
-dnl PAC_ARG_SHAREDLIBS - Add --enable-sharedlibs=kind to configure.
-dnl 
-dnl Synopsis:
-dnl PAC_ARG_SHAREDLIBS
-dnl
-dnl Output effects:
-dnl Adds '--enable-sharedlibs=kind' to the command line.  If this is enabled,
-dnl then based on the value of 'kind', programs are selected for the 
-dnl names 'CC_SHL' and 'CC_LINK_SHL' that configure will substitute for in
-dnl 'Makefile.in's.  These symbols are generated by 'simplemake' when
-dnl shared library support is selected.
-dnl The variable 'C_LINKPATH_SHL' is set to the option to specify the 
-dnl path to search at runtime for libraries (-rpath in gcc/GNU ld).
-dnl This can be turned off with --disable-rpath , which is appropriate
-dnl for libraries and for executables that may be installed in different
-dnl locations.
-dnl The variable 'SHLIB_EXT' is set to the extension used by shared 
-dnl libraries; under most forms of Unix, this is 'so'; under Mac OS/X, this
-dnl is 'dylib', and under Windows (including cygwin), this is 'dll'.
-dnl
-dnl Supported values of 'kind' include \:
-dnl+    gcc - Use gcc to create both shared objects and libraries
-dnl.    osx-gcc - Use gcc on Mac OS/X to create both shared objects and
-dnl               libraries
-dnl.    solaris-cc - Use native Solaris cc to create shared objects and 
-dnl               libraries
-dnl.    cygwin-gcc - Use gcc on Cygwin to create shared objects and libraries
-dnl-    none - The same as '--disable-sharedlibs'
-dnl
-dnl Others will be added as experience dictates.  Likely names are
-dnl + libtool - For general GNU libtool
-dnl - linux-pgcc - For Portland group under Linux
-dnl
-dnl Notes:
-dnl Shared libraries are only partially implemented.  Additional symbols
-dnl will probably be defined, including symbols to specify how shared library
-dnl search paths are specified and how shared library names are set.
-dnl D*/
-AC_DEFUN([PAC_ARG_SHAREDLIBS],[
-
-AC_ARG_ENABLE(shared,
-	AC_HELP_STRING([--enable-shared], [Enable shared library builds]),,
-	enable_shared=no)
-
-AC_ARG_ENABLE(rpath,
-	AC_HELP_STRING([--enable-rpath],
-		[Determine whether the rpath is set when programs are
-		compiled and linked when shared libraries are built.
-		The default is yes; use --disable-rpath to turn this
-		feature off; in that case, shared libraries will be
-		found according to the rules for your system (e.g., in
-		LD_LIBRARY_PATH)]),,enable_rpath=yes)
-
-AC_ARG_ENABLE(sharedlibs,
-[  --enable-sharedlibs=kind - Enable shared libraries.  kind may be
-        gcc     - Standard gcc and GNU ld options for creating shared libraries
-        osx-gcc - Special options for gcc needed only on OS/X
-        solaris-cc - Solaris native (SPARC) compilers for 32 bit systems
-        cygwin-gcc - Special options for gcc needed only for cygwin
-        none    - same as --disable-sharedlibs
-      Only gcc, osx-gcc, and solaris-cc are currently supported
-],,enable_sharedlibs=default)
-
-if test "$enable_sharedlibs" = "default" ; then
-   if test "$enable_shared" = "yes" ; then
-      AS_CASE([$host],
-	      [*-*-darwin*], [enable_sharedlibs=gcc-osx],
-	      [*-*-cygwin*|*-*-mingw*|*-*-pw32*|*-*-cegcc*], [enable_sharedlibs=cygwin-gcc],
-	      [*-*-sunos*], [enable_sharedlibs=solaris-gcc],
-	      [enable_sharedlibs=gcc])
-   else
-      enable_sharedlibs=none
-   fi
-fi
-
-# If --enable-sharedlibs is given, but --enable-shared is not, throw
-# an error
-if test "$enable_sharedlibs" != "no" -a "$enable_sharedlibs" != "none" ; then
-   if test "$enable_shared" = "no" ; then
-      AC_MSG_ERROR([--enable-sharedlibs cannot be used without --enable-shared])
-   fi
-fi
-
-CC_SHL=true
-C_LINK_SHL=true
-C_LINKPATH_SHL=""
-SHLIB_EXT=unknown
-SHLIB_FROM_LO=no
-SHLIB_INSTALL='$(INSTALL_PROGRAM)'
-case "$enable_sharedlibs" in 
-    no|none)
-    ;;
-    gcc-osx|osx-gcc)
-    AC_MSG_RESULT([Creating shared libraries using GNU for Mac OSX])
-    C_LINK_SHL='${CC} -dynamiclib -undefined suppress -single_module -flat_namespace'
-    CC_SHL='${CC} -fPIC'
-    # No way in osx to specify the location of the shared libraries at link
-    # time (see the code in createshlib in mpich/src/util)
-    # As of 10.5, -Wl,-rpath,dirname should work .  The dirname 
-    # must be a single directory, not a colon-separated list (use multiple
-    # -Wl,-rpath,path for each of the paths in the list).  However, os x
-    # apparently records the library full path, so rpath isn't as useful
-    # as it is on other systems
-    C_LINKPATH_SHL=""
-    SHLIB_EXT="dylib"
-    enable_sharedlibs="osx-gcc"
-    ;;
-    gcc)
-    AC_MSG_RESULT([Creating shared libraries using GNU])
-    # Not quite right yet.  See mpich/util/makesharedlib
-    # Use syntax that works in both Make and the shell
-    #C_LINK_SHL='${CC} -shared -Wl,-r'
-    C_LINK_SHL='${CC} -shared'
-    # For example, include the libname as ${LIBNAME_SHL}
-    #C_LINK_SHL='${CC} -shared -Wl,-h,<finallibname>'
-    # May need -fPIC .  Test to see which one works.
-    for sh_arg in "-fPIC" "-fpic" "-KPIC" ; do
-        PAC_C_CHECK_COMPILER_OPTION($sh_arg,works=yes,works=no)
-        if test "$works" = "yes" ; then
-           CC_SHL="${CC} ${sh_arg}"
-	   break
-	fi
-    done
-    if test "$works" != "yes"; then
-       AC_MSG_ERROR([Cannot build shared libraries with this compiler])
-    fi
-    # This used to have -Wl,-rpath earlier, but that causes problems
-    # on many systems.
-    if test $enable_rpath = "yes" ; then
-        C_LINKPATH_SHL="-Wl,-rpath,"
-    fi
-    SHLIB_EXT=so
-    # We need to test that this isn't osx.  The following is a 
-    # simple hack
-    osname=`uname -s`
-    case $osname in 
-        *Darwin*|*darwin*)
-	AC_MSG_ERROR([You must specify --enable-sharedlibs=osx-gcc for Mac OS/X])
-        ;;	
-        *CYGWIN*|*cygwin*)
-	AC_MSG_ERROR([You must specify --enable-sharedlibs=cygwin-gcc for Cygwin])
-	;;
-	*SunOS*)
-	AC_MSG_ERROR([You must specify --enable-sharedlibs=solaris-gcc for Solaris with gcc])
-	;;
-    esac
-    ;;
-
-    cygwin|cygwin-gcc|gcc-cygwin)
-    AC_MSG_RESULT([Creating shared libraries using GNU under CYGWIN])
-    C_LINK_SHL='${CC} -shared'
-    CC_SHL='${CC}'
-    # DLL Libraries need to be in the user's path (!)
-    C_LINKPATH_SHL=""
-    SHLIB_EXT="dll"
-    enable_sharedlibs="cygwin-gcc"
-    ;;	
-
-    libtool)
-    # set TRY_LIBTOOL to yes to experiment with libtool.  You are on your
-    # own - only send fixes, not bug reports.
-    if test "$TRY_LIBTOOL" != yes ; then
-        AC_MSG_ERROR([Creating shared libraries using libtool not yet supported])
-    else
-    # Using libtool requires a heavy-weight process to test for 
-    # various stuff that libtool needs.  Without this, you'll get a
-    # bizarre error message about libtool being unable to find
-    # configure.in or configure.ac (!)
-        # Libtool expects to see at least enable-shared.
-        if test "X$enable_shared" = "X" ; then enable_shared=yes ; fi
-	# Initialize libtool
-	# This works, but libtool version 2 places the object files
-	# in a different place, making it harder to integrate with
-	# our base approach.  Disabling for now
-	dnl LT_PREREQ([2.2.6])
-        dnl LT_INIT([disable-shared])
-	AC_MSG_ERROR([To use this test verison, edit aclocal_shl.m4])
-        # Likely to be
-        # either CC or CC_SHL is libtool $cc
-        CC_SHL='${LIBTOOL} --mode=compile ${CC}'
-        # CC_LINK_SHL includes the final installation path
-        # For many systems, the link may need to include *all* libraries
-        # (since many systems don't allow any unsatisfied dependencies)
-        # We need to give libtool the .lo file, not the .o files
-        SHLIB_FROM_LO=yes
-        # We also need to add -no-undefined when the compiler is gcc and
-        # we are building under cygwin
-        sysname=`uname -s | tr abcdefghijklmnopqrstuvwxyz ABCDEFGHIJKLMNOPQRSTUVWXYZ`
-        isCygwin=no
-        case "$sysname" in 
- 	*CYGWIN*) isCygwin=yes ;;
-        esac
-        if test "$isCygwin" = yes ; then
-            C_LINK_SHL='${LIBTOOL} --mode=link ${CC} -no-undefined -rpath ${libdir}'
-        else
-            C_LINK_SHL='${LIBTOOL} --mode=link ${CC} -rpath ${libdir}'
-        fi
-	if test $enable_rpath = "yes" ; then
-            C_LINKPATH_SHL="-rpath "
-        fi
-        # We also need a special install process with libtool.  Note that this
-        # will also install the static libraries
-        SHLIB_INSTALL='$(LIBTOOL) --mode=install $(INSTALL_PROGRAM)'
-        # Note we may still need to add
-        #'$(LIBTOOL) --mode=finish $(libdir)'
-    fi
-    ;;
-dnl
-dnl Other, such as solaris-cc
-    solaris|solaris-cc)
-    AC_MSG_RESULT([Creating shared libraries using Solaris])
-    # pic32 is appropriate for both 32 and 64 bit Solaris
-    C_LINK_SHL='${CC} -G -xcode=pic32'
-    CC_SHL='${CC} -xcode=pic32'
-    if test $enable_rpath = "yes" ; then
-        C_LINKPATH_SHL="-R"
-    fi
-    SHLIB_EXT=so
-    enable_sharedlibs="solaris-cc"
-    ;;
-
-    solaris-gcc)
-    # This is the same as gcc, except for the C_LINKPATH_SHL
-    AC_MSG_RESULT([Creating shared libraries using Solaris with gcc])
-    C_LINK_SHL='${CC} -shared'
-    CC_SHL='${CC} -fPIC'
-    if test $enable_rpath = "yes" ; then
-        C_LINKPATH_SHL="-R"
-    fi
-    SHLIB_EXT=so
-    enable_sharedlibs="solaris-gcc"
-    ;;
-
-    linuxppc-xlc)
-    # This is only the beginning of xlc support, thanks to andy@vpac.org
-    CC_SHL='${CC} -qmkshrobj'
-    # More recent versions allow multiple args, separated by commas
-    if test $enable_rpath = "yes" ; then
-        C_LINKPATH_SHL="-Wl,-rpath,"
-    fi
-    #C_LINKPATH_SHL="-Wl,-rpath -Wl,"
-    C_LINK_SHL='${CC} -shared -qmkshrobj'
-    SHLIB_EXT=so
-    # Note that the full line should be more like
-    # $CLINKER -shared -qmkshrobj -Wl,-h,$libbase.$slsuffix -o ../shared/$libbase.$slsuffix *.o $OtherLibs
-    # for the appropriate values of $libbase and $slsuffix
-    # The -h name sets the name of the object; this is necessary to
-    # ensure that the dynamic linker can find the proper shared library.
-    ;;
-
-    *)
-    AC_MSG_ERROR([Unknown value $enable_sharedlibs for enable-sharedlibs.  Values should be gcc or osx-gcc])
-    enable_sharedlibs=no
-    ;;  
-esac
-# Check for the shared-library extension
-PAC_CC_SHLIB_EXT
-dnl
-AC_SUBST(CC_SHL)
-AC_SUBST(C_LINK_SHL)
-AC_SUBST(C_LINKPATH_SHL)
-AC_SUBST(SHLIB_EXT)
-AC_SUBST(SHLIB_FROM_LO)
-AC_SUBST(SHLIB_INSTALL)
-])
-
-dnl /*D
-dnl PAC_xx_SHAREDLIBS - Get compiler and linker for shared libraries
-dnl These routines may be used to determine the compiler and the
-dnl linker to be used in creating shared libraries
-dnl Rather than set predefined variable names, they set an argument 
-dnl (if provided)
-dnl
-dnl Synopsis
-dnl PAC_CC_SHAREDLIBS(type,CCvar,CLINKvar)
-dnl D*/
-AC_DEFUN([PAC_CC_SHAREDLIBS],
-[
-pac_kinds=$1
-ifelse($1,,[
-    pac_prog=""
-    AC_CHECK_PROG(pac_prog,gcc,yes,no)
-    # If we are gcc but OS X, set the special type
-    # We need a similar setting for cygwin
-    if test "$pac_prog" = yes ; then 
-        osname=`uname -s`
-    	case $osname in 
-             *Darwin*|*darwin*) pac_kinds=gcc-osx
-             ;;				
-	     *) pac_kinds=gcc
-	     ;;
-	esac
-    fi
-    pac_prog=""
-    AC_CHECK_PROG(pac_prog,libtool,yes,no)
-    if test "$pac_prog" = yes ; then pac_kinds="$pac_kinds libtool" ; fi
-])
-for pac_arg in $pac_kinds ; do
-    case $pac_arg in 
-    gcc)
-    # For example, include the libname as ${LIBNAME_SHL}
-    #C_LINK_SHL='${CC} -shared -Wl,-h,<finallibname>'
-    pac_cc_sharedlibs='gcc -shared'
-    # Make sure we select the correct fpic option
-    PAC_C_CHECK_COMPILER_OPTION(-fPIC,fPIC_OK=yes,fPIC_OK=no)
-    if test "$fPIC_OK" != yes ; then
-        PAC_C_CHECK_COMPILER_OPTION(-fpic,fpic_ok=yes,fpic_ok=no)
-        if test "$fpic_ok" != yes ; then
-	     AC_MSG_ERROR([Neither -fpic nor -fPIC accepted by $CC])
-        else
-	     pac_cc_sharedlibs="$pac_cc_sharedlibs -fpic"
-        fi
-    else
-        pac_cc_sharedlibs="$pac_cc_sharedlibs -fPIC"
-    fi
-    pac_clink_sharedlibs='gcc -shared'
-    pac_type_sharedlibs=gcc
-    ;;
-    gcc-osx|osx-gcc)
-    pac_clink_sharedlibs='${CC} -dynamiclib -undefined suppress -single_module -flat_namespace'
-    pac_cc_sharedlibs='${CC} -fPIC'
-    pac_type_sharedlibs=gcc-osx
-    ;;
-    libtool)
-    AC_CHECK_PROGS(LIBTOOL,libtool,false)
-    if test "$LIBTOOL" = "false" ; then
-	AC_MSG_WARN([Could not find libtool])
-    else
-        # Likely to be
-        # either CC or CC_SHL is libtool $cc
-        pac_cc_sharedlibs'${LIBTOOL} -mode=compile ${CC}'
-        pac_clink_sharedlibs='${LIBTOOL} -mode=link ${CC} -rpath ${libdir}'
-	pac_type_sharedlibs=libtool
-    fi
-    ;;
-    *)
-    ;;
-    esac
-    if test -n "$pac_cc_sharedlibs" ; then break ; fi
-done
-if test -z "$pac_cc_sharedlibs" ; then pac_cc_sharedlibs=true ; fi
-if test -z "$pac_clink_sharedlibs" ; then pac_clink_sharedlibs=true ; fi
-ifelse($2,,CC_SHL=$pac_cc_sharedlibs,$2=$pac_cc_sharedlibs)
-ifelse($3,,C_LINK_SHL=$pac_clink_sharedlibs,$3=$pac_clink_sharedlibs)
-ifelse($4,,SHAREDLIB_TYPE=$pac_type_sharedlibs,$4=$pac_type_sharedlibs)
-])
-
-dnl This macro ensures that all of the necessary substitutions are 
-dnl made by any subdirectory configure (which may simply SUBST the
-dnl necessary values rather than trying to determine them from scratch)
-dnl This is a more robust (and, in the case of libtool, only 
-dnl managable) method.
-AC_DEFUN([PAC_CC_SUBDIR_SHLIBS],[
-	AC_SUBST(CC_SHL)
-        AC_SUBST(C_LINK_SHL)
-        AC_SUBST(LIBTOOL)
-        AC_SUBST(ENABLE_SHLIB)
-        AC_SUBST(SHLIB_EXT)
-	if test "$ENABLE_SHLIB" = "libtool" ; then
-	    if test -z "$LIBTOOL" ; then
-		AC_MSG_WARN([libtool selected for shared library support but LIBTOOL is not defined])
-            fi
-	    # Libtool needs master_top_builddir
-	    if test "X$master_top_builddir" = "X" ; then
-	        AC_MSG_ERROR([Libtool requires master_top_builddir - check configure.ac sources])
-	    fi
-	    AC_SUBST(master_top_builddir)
-	fi
-])
-
-dnl PAC_CC_SHLIB_EXT - get the extension for shared libraries
-dnl Set the variable SHLIB_EXT if it is other than unknown.
-AC_DEFUN([PAC_CC_SHLIB_EXT],[
-# Not all systems use .so as the extension for shared libraries (cygwin
-# and OSX are two important examples).  If we did not set the SHLIB_EXT,
-# then try and determine it.  We need this to properly implement
-# clean steps that look for libfoo.$SHLIB_EXT .
-if test "$SHLIB_EXT" = "unknown" ; then
-    osname=`uname -s`
-    case $osname in 
-        *Darwin*|*darwin*) SHLIB_EXT=dylib
-        ;;	
-        *CYGWIN*|*cygwin*) SHLIB_EXT=dll
-        ;;
-	*Linux*|*LINUX*|*SunOS*) SHLIB_EXT=so
-	;;
-   esac
-fi
-])
-
-dnl PAC_COMPILER_SHLIB_FLAGS(compiler-var,output-file)
-dnl
-dnl Uses confdb/config.rpath to determine important linking flags and
-dnl information.  This is mainly intended to support the compiler wrapper
-dnl scripts in MPICH ("mpicc" and friends) which cannot directly use libtool to
-dnl handle linking.  MPICH's compiler wrappers attempt to link executables with
-dnl an rpath by default.  The resulting variable assignment statements will be
-dnl placed into "output-file", which is then suitable for AC_SUBST_FILE or
-dnl sourcing in a shell script (including configure itself).
-dnl
-dnl This macro assumes that the basic tests associated with "compiler-var" have
-dnl been run already, but does not AC_REQUIRE them in order to prevent problems
-dnl with "expanded before required" when requiring the AC_PROG_{CC,F77,FC,CXX}
-dnl macros.
-dnl
-dnl Example usage:
-dnl
-dnl ----8<----
-dnl # compute flags for linking executables against shared libraries when using
-dnl # the modern Fortran compiler ($FC).
-dnl PAC_COMPILER_SHLIB_FLAGS([FC],[src/env/fc_shlib.conf])
-dnl ----8<----
-AC_DEFUN([PAC_COMPILER_SHLIB_FLAGS],[
-AC_REQUIRE_AUX_FILE([config.rpath])
-AC_REQUIRE([AC_CANONICAL_HOST])
-
-# It appears that the libtool dynamic linking strategy on Darwin is this:
-# 1. Link all shared libs with "-install_name /full/path/to/libfoo.dylib"
-# 2. Don't do anything special when linking programs, since it seems that the
-# darwin dynamic linker will always use the "install_name" field from the lib
-# that was found at program link-time.  (CONFIRMED) This is in opposition to
-# the way that Linux does it, where specifying a "-rpath" argument at program
-# link-time is important.
-#
-# There is an alternative darwin strategy for versions
-# >= 10.5, see this: http://www.cmake.org/pipermail/cmake/2010-August/038970.html
-# (goodell@ 2011-06-17)
-
-AC_MSG_CHECKING([for shared library (esp. rpath) characteristics of $1])
-
-# unfortunately, config.rpath expects the compiler in question is always CC and
-# uses several other environment variables as input
-PAC_PUSH_FLAG([CC])
-PAC_PUSH_FLAG([GCC])
-PAC_PUSH_FLAG([LD])
-# these two don't currently get overridden, but we push/pop them for safety in
-# case they do in the future
-PAC_PUSH_FLAG([LDFLAGS])
-PAC_PUSH_FLAG([with_gnu_ld])
-
-# set the temporary override values (if any)
-m4_case([$1],
-[CC],
-    [:], dnl do nothing special for CC, values are already correct
-[F77],
-    [CC="$$1"
-     GCC="$G77"
-     LD="$LD_F77"],
-[FC],
-    [CC="$$1"
-     GCC="$GCC_FC"
-     LD="$LD_FC"],
-[CXX],
-    [CC="$$1"
-     GCC="$GXX"
-     LD="$LD_CXX"],
-[m4_fatal([unknown compiler argument "$1"])])
-
-# ensure the values are available to the script
-export CC
-export GCC
-export LDFLAGS
-export LD
-export with_gnu_ld
-
-AS_IF([$ac_aux_dir/config.rpath "$host" > $2],[:],[AC_MSG_ERROR([unable to execute $ac_aux_dir/config.rpath])])
-
-C_LINKPATH_SHL=""
-AC_SUBST([C_LINKPATH_SHL])
-
-rm -f conftest.out
-
-# restore the old values
-PAC_POP_FLAG([with_gnu_ld])
-PAC_POP_FLAG([LD])
-PAC_POP_FLAG([LDFLAGS])
-PAC_POP_FLAG([GCC])
-PAC_POP_FLAG([CC])
-
-AC_MSG_RESULT([done (results in $2)])
-])
-
diff --git a/3rd-party/romio321/confdb/aclocal_shm.m4 b/3rd-party/romio321/confdb/aclocal_shm.m4
deleted file mode 100644
index d2cbaade434..00000000000
--- a/3rd-party/romio321/confdb/aclocal_shm.m4
+++ /dev/null
@@ -1,49 +0,0 @@
-dnl
-dnl Definitions for using shared memory
-dnl
-
-dnl/*D
-dnl PAC_ARG_SHARED_MEMORY - add --with-shared-memory=kind to configure
-dnl
-dnl Synopsis:
-dnl PAC_ARG_SHARED_MEMORY
-dnl
-dnl Output effects:
-dnl Adds '--with-shared-memory' to the command line. Checks for available
-dnl shared memory functionality.
-dnl
-dnl Supported values of 'kind' include \:
-dnl+    auto - default
-dnl.    mmap - use mmap and munmap
-dnl-    sysv - use sysv shared memory functions
-dnl D*/
-AC_DEFUN([PAC_ARG_SHARED_MEMORY],[
-
-# check how to allocate shared memory
-AC_ARG_WITH(shared-memory,
-    AC_HELP_STRING([--with-shared-memory[=auto|sysv|mmap]], [create shared memory using sysv or mmap (default is auto)]),,
-    with_shared_memory=auto)
-
-if test "$with_shared_memory" = auto -o "$with_shared_memory" = mmap; then
-    found_mmap_funcs=yes
-    AC_CHECK_FUNCS(mmap munmap, , found_mmap_funcs=no)
-    if test "$found_mmap_funcs" = yes ; then
-        with_shared_memory=mmap
-        AC_DEFINE(USE_MMAP_SHM,1,[Define if we have sysv shared memory])
-        AC_MSG_NOTICE([Using a memory-mapped file for shared memory])
-    elif test "$with_shared_memory" = mmap ; then
-        AC_MSG_ERROR([cannot support shared memory:  mmap() or munmap() not found])
-    fi
-fi
-if test "$with_shared_memory" = auto -o "$with_shared_memory" = sysv; then
-    found_sysv_shm_funcs=yes
-    AC_CHECK_FUNCS(shmget shmat shmctl shmdt, , found_sysv_shm_funcs=no)
-    if test "$found_sysv_shm_funcs" = yes ; then
-        with_shared_memory=sysv
-        AC_DEFINE(USE_SYSV_SHM,1,[Define if we have sysv shared memory])
-        AC_MSG_NOTICE([Using SYSV shared memory])
-    elif test "$with_shared_memory" = sysv ; then
-        AC_MSG_ERROR([cannot support shared memory:  sysv shared memory functions functions not found])
-    fi
-fi
-])
diff --git a/3rd-party/romio321/confdb/aclocal_subcfg.m4 b/3rd-party/romio321/confdb/aclocal_subcfg.m4
deleted file mode 100644
index 53fb78e3d9e..00000000000
--- a/3rd-party/romio321/confdb/aclocal_subcfg.m4
+++ /dev/null
@@ -1,261 +0,0 @@
-dnl PAC_RESET_ALL_FLAGS - Reset precious flags to those set by the user
-AC_DEFUN([PAC_RESET_ALL_FLAGS],[
-	if test "$FROM_MPICH" = "yes" ; then
-	   CFLAGS="$USER_CFLAGS"
-	   CPPFLAGS="$USER_CPPFLAGS"
-	   CXXFLAGS="$USER_CXXFLAGS"
-	   FFLAGS="$USER_FFLAGS"
-	   FCFLAGS="$USER_FCFLAGS"
-	   LDFLAGS="$USER_LDFLAGS"
-	   LIBS="$USER_LIBS"
-	fi
-])
-
-dnl PAC_RESET_LINK_FLAGS - Reset precious link flags to those set by the user
-AC_DEFUN([PAC_RESET_LINK_FLAGS],[
-	if test "$FROM_MPICH" = "yes" ; then
-	   LDFLAGS="$USER_LDFLAGS"
-	   LIBS="$USER_LIBS"
-	fi
-])
-
-dnl Sandbox configure with additional arguments
-dnl Usage: PAC_CONFIG_SUBDIR_ARGS(subdir,configure-args,action-if-success,action-if-failure)
-dnl
-dnl The subconfigure argument list is created based on "ac_precious_vars"
-dnl instead of explicitly use of well-known Makefile variables, like
-dnl CC/CFLAGS/CPPFLAGS..., this generalization is effective as long as
-dnl calling configure.ac declares the needed variables to be passed down
-dnl to subconfigure as "precious" appropriately.  The precious variable
-dnl can be created in the following ways:
-dnl 1) implicit declaration through use of autoconf macros, like
-dnl    AC_PROG_CC (declares CC/CFLAGS/CPPFLAGS/LIBS/LDFLAGS), or
-dnl    AC_PROG_F77 (declares F77/FFLAGS/FLIBS) ... 
-dnl    which are in turns invoked by other subconfigure.
-dnl    When in doubt, check "ac_precious_var" in the calling configure.
-dnl 2) explicit "precious" declaration through AC_ARG_VAR.
-dnl Without correct "precious" declaration in the calling configure.ac,
-dnl there would be variables not being included in the subconfigure
-dnl argument list.
-dnl
-dnl Note: I suspect this DEFUN body is underquoted in places, but it does not
-dnl seem to cause problems in practice yet. [goodell@ 2010-05-18]
-AC_DEFUN([PAC_CONFIG_SUBDIR_ARGS],[
-        pac_dir="$1"
-        AC_MSG_NOTICE([===== configuring $1 =====])
-
-	pac_abs_srcdir=`(cd $srcdir && pwd)`
-
-	if test -f $pac_abs_srcdir/$1/setup ; then
-           AC_MSG_NOTICE([sourcing $pac_abs_srcdir/$1/setup])
-	   . $pac_abs_srcdir/$1/setup
-	fi
-
-        # Adapted for MPICH from the autoconf-2.67 implementation of
-        # AC_CONFIG_SUBDIRS.  Search for "MPICH note:" for relevant commentary and
-        # local modifications.
-      
-        # Remove --cache-file, --srcdir, and --disable-option-checking arguments
-        # so they do not pile up.  Otherwise relative paths (like --srcdir=.. from
-        # make distcheck) will be incorrect.
-        pac_sub_configure_args="$2"
-        pac_prev=
-        eval "set x $ac_configure_args"
-        shift
-        for pac_arg
-        do
-          if test -n "$pac_prev"; then
-            pac_prev=
-            continue
-          fi
-          case $pac_arg in
-          -cache-file | --cache-file | --cache-fil | --cache-fi \
-          | --cache-f | --cache- | --cache | --cach | --cac | --ca | --c)
-            pac_prev=cache_file ;;
-          -cache-file=* | --cache-file=* | --cache-fil=* | --cache-fi=* \
-          | --cache-f=* | --cache-=* | --cache=* | --cach=* | --cac=* | --ca=* \
-          | --c=*)
-            ;;
-          --config-cache | -C)
-            ;;
-          -srcdir | --srcdir | --srcdi | --srcd | --src | --sr)
-            pac_prev=srcdir ;;
-          -srcdir=* | --srcdir=* | --srcdi=* | --srcd=* | --src=* | --sr=*)
-            ;;
-          -prefix | --prefix | --prefi | --pref | --pre | --pr | --p)
-            pac_prev=prefix ;;
-          -prefix=* | --prefix=* | --prefi=* | --pref=* | --pre=* | --pr=* | --p=*)
-            ;;
-          --disable-option-checking)
-            ;;
-          *)
-            # MPICH note: this is a more robust version of the "precious
-            # variable" propagation code that was present in the previous
-            # incarnation of this macro
-            for pac_pvar in $ac_precious_vars ; do
-                # check if configure argument token contains the
-                # precious variable, i.e. "name_of_prec_var=".
-                if ( echo $pac_arg | grep "^$pac_pvar=" >/dev/null 2>&1 ) ; then
-                    # check if current precious variable is set in env
-                    eval pvar_set=\${$pac_pvar+set}
-                    if test "$pvar_set" = "set" ; then
-                        # Append 'name_of_prec_var=value_of_prec_var'
-                        # to the subconfigure arguments list, where
-                        # value_of_prec_var is fetched from the env.
-                        # this also overrides any value set on the command line
-                        eval pac_pval=\${$pac_pvar}
-                        pac_arg="$pac_pvar=$pac_pval"
-                        break
-                    fi
-                fi
-            done
-            case $pac_arg in
-            *\'*) pac_arg=`AS_ECHO(["$pac_arg"]) | sed "s/'/'\\\\\\\\''/g"` ;;
-            esac
-            AS_VAR_APPEND([pac_sub_configure_args], [" '$pac_arg'"]) ;;
-          esac
-        done
-      
-        # Always prepend --prefix to ensure using the same prefix
-        # in subdir configurations.
-        # MPICH note: see tt#983 for an example of why this is necessary
-        pac_arg="--prefix=$prefix"
-        case $pac_arg in
-        *\'*) pac_arg=`AS_ECHO(["$pac_arg"]) | sed "s/'/'\\\\\\\\''/g"` ;;
-        esac
-        pac_sub_configure_args="'$pac_arg' $pac_sub_configure_args"
-      
-        # Pass --silent
-        if test "$silent" = yes; then
-          pac_sub_configure_args="--silent $pac_sub_configure_args"
-        fi
-      
-        # Always prepend --disable-option-checking to silence warnings, since
-        # different subdirs can have different --enable and --with options.
-        pac_sub_configure_args="--disable-option-checking $pac_sub_configure_args"
-      
-        pac_popdir=`pwd`
-      
-        # Do not complain, so a configure script can configure whichever
-        # parts of a large source tree are present.
-        test -d "$srcdir/$pac_dir" || continue
-      
-        # MPICH note: modified to remove the internal "_AS_*" macro usage, also
-        # msg is already printed at top
-dnl        _AS_ECHO_LOG([$pac_msg])
-dnl        _AS_ECHO([$pac_msg])
-        AS_MKDIR_P(["$pac_dir"])
-        # MPICH note: we leave this internal macro reference for now.  We can clone
-        # the macro locally if this turns out to be non-portable across several autoconf
-        # versions.  It sets the following variables: ac_builddir,
-        # ac_top_builddir_sub, ac_top_build_prefix, ac_srcdir, ac_top_srcdir,
-        # ac_abs_top_builddir, ac_abs_builddir, ac_abs_top_srcdir, ac_abs_srcdir
-        _AC_SRCDIRS(["$pac_dir"])
-
-        cd "$pac_dir"
-
-        # Check for guested configure; otherwise get Cygnus style configure.
-        if test -f "$ac_srcdir/configure.gnu"; then
-          pac_sub_configure=$ac_srcdir/configure.gnu
-        elif test -f "$ac_srcdir/configure"; then
-          pac_sub_configure=$ac_srcdir/configure
-        elif test -f "$ac_srcdir/configure.ac"; then
-          # This should be Cygnus configure.
-          pac_sub_configure=$ac_aux_dir/configure
-        else
-          AC_MSG_WARN([no configuration information is in $pac_dir])
-          pac_sub_configure=
-        fi
-
-        # The recursion is here.
-        if test -n "$pac_sub_configure"; then
-          # MPICH note: overriding the cache file on purpose to prevent strange
-          # issues resulting from inter-dir caching
-dnl          # Make the cache file name correct relative to the subdirectory.
-dnl          case $cache_file in
-dnl          [[\\/]]* | ?:[[\\/]]* ) pac_sub_cache_file=$cache_file ;;
-dnl          *) # Relative name.
-dnl            pac_sub_cache_file=$ac_top_build_prefix$cache_file ;;
-dnl          esac
-          pac_sub_cache_file="/dev/null"
-
-          AC_MSG_NOTICE([running $SHELL $pac_sub_configure $pac_sub_configure_args --cache-file=$pac_sub_cache_file --srcdir=$ac_srcdir])
-          # The eval makes quoting arguments work.
-          # MPICH note: we want to execute the provided actions, not be silent
-          # or error out if the subconfigure succeeded/failed
-dnl          eval "\$SHELL \"\$pac_sub_configure\" $pac_sub_configure_args \
-dnl               --cache-file=\"\$pac_sub_cache_file\" --srcdir=\"\$ac_srcdir\"" ||
-dnl            AC_MSG_ERROR([$pac_sub_configure failed for $pac_dir])
-          if eval "\$SHELL \"\$pac_sub_configure\" $pac_sub_configure_args \
-               --cache-file=\"\$pac_sub_cache_file\" --srcdir=\"\$ac_srcdir\""
-          then
-            # restore the current dir for the provided actions
-            cd "$pac_popdir"
-            $3
-          else
-            # restore the current dir for the provided actions
-            cd "$pac_popdir"
-            $4
-          fi
-        fi
-
-        cd "$pac_popdir"
-
-        AC_MSG_NOTICE([===== done with $1 configure =====])
-
-	# Check for any localdefs files.  These may be created, so we
-	# look in the local directory first.
-	if test -f $1/localdefs ; then
-           AC_MSG_NOTICE([sourcing $1/localdefs])
-	   . $1/localdefs
-	elif test -f $pac_abs_srcdir/$1/localdefs ; then
-           AC_MSG_NOTICE([sourcing $pac_abs_srcdir/$1/localdefs])
-	   . $pac_abs_srcdir/$1/localdefs
-	fi
-])
-
-dnl Sandbox configure
-dnl Usage: PAC_CONFIG_SUBDIR(subdir,action-if-success,action-if-failure)
-AC_DEFUN([PAC_CONFIG_SUBDIR],[PAC_CONFIG_SUBDIR_ARGS([$1],[],[$2],[$3])])
-
-dnl PAC_SUBCFG_EXPAND_SUFFIX_MACRO(MACRO_PREFIX,MACRO_PATH_SUFFIX)
-dnl converts the path given by MACRO_PATH_SUFFIX (with '/' chars in it) to one
-dnl with '_' chars in it and then appends that to MACRO_PREFIX with '_' in
-dnl between.  The resulting macro name is then expanded, but with informative
-dnl "##" comments before and after the expansion.
-dnl
-dnl This is intended to be an internal helper macro for the PAC_SUBCFG
-dnl implementation.
-dnl
-dnl XXX DJG FIXME: need to be able to deal with PREREQ macros that potentially
-dnl aren't present while having safety for BODY macros when there are
-dnl misspellings
-AC_DEFUN([PAC_SUBCFG_EXPAND_PATH_SUFFIX_MACRO],[
-dnl convert path separators into '_', the m4_translit is intentionally unquoted
-m4_pushdef([subsys_uscore_name],[$1_]m4_translit([$2],[\/],[__]))dnl
-m4_ifdef(m4_defn([subsys_uscore_name]),[],[m4_fatal([macro ]m4_defn([subsys_uscore_name])[ is undefined])])dnl
-[##] begin expansion of m4_defn([subsys_uscore_name])
-dnl call the computed routine name with the remaining args
-m4_indir(m4_defn([subsys_uscore_name]),m4_shift($@))
-dnl there is intentionally no "dnl" on the previous line to reduce the chance of
-dnl a "fi## end expansion" bug when the BODY macro doesn't end in a newline
-[##] end expansion of m4_defn([subsys_uscore_name])
-])
-
-dnl invokes the PAC_SUBCFG_BODY_foo macro for the "foo" subsys, when "foo" is
-dnl passed as the only argument to this macro.  The first arg may be the '/'
-dnl path version instead of having underscores.
-AC_DEFUN([PAC_SUBCFG_CONFIGURE_SUBSYS],[PAC_SUBCFG_EXPAND_PATH_SUFFIX_MACRO([PAC_SUBCFG_BODY],[$1])])
-
-dnl invokes the PAC_SUBCFG_PREREQ_foo macro for the "foo" subsys, when "foo" is
-dnl passed as the only argument to this macro.  The first arg may be the '/'
-dnl path version instead of having underscores.
-AC_DEFUN([PAC_SUBCFG_DO_PREREQ],[PAC_SUBCFG_EXPAND_PATH_SUFFIX_MACRO([PAC_SUBCFG_PREREQ],[$1])])
-
-dnl takes no arguments, expands to "foo_bar_baz" when invoked in a file named
-dnl "foo/bar/baz/subconfigure.m4"
-dnl
-dnl This is useful for reducing copy-paste errors when defining PREREQ and BODY
-dnl macros.  If you tinker with this macro, watch the quoting carefully.
-AC_DEFUN([PAC_SUBCFG_AUTO_SUFFIX],[m4_translit(m4_bpatsubst(m4_dquote(__file__),[/[^/]+.m4],[]),[/],[_])])
-
diff --git a/3rd-party/romio321/confdb/aclocal_util.m4 b/3rd-party/romio321/confdb/aclocal_util.m4
deleted file mode 100644
index 2648be54303..00000000000
--- a/3rd-party/romio321/confdb/aclocal_util.m4
+++ /dev/null
@@ -1,216 +0,0 @@
-dnl Nesting safe macros for saving variables
-dnl Usage: PAC_PUSH_FLAG(CFLAGS)
-AC_DEFUN([PAC_PUSH_FLAG],[
-	if test -z "${pac_save_$1_nesting}" ; then
-	   pac_save_$1_nesting=0
-	fi
-	eval pac_save_$1_${pac_save_$1_nesting}='"$$1"'
-	pac_save_$1_nesting=`expr ${pac_save_$1_nesting} + 1`
-])
-
-dnl Usage: PAC_POP_FLAG(CFLAGS)
-AC_DEFUN([PAC_POP_FLAG],[
-	pac_save_$1_nesting=`expr ${pac_save_$1_nesting} - 1`
-	eval $1="\$pac_save_$1_${pac_save_$1_nesting}"
-	eval pac_save_$1_${pac_save_$1_nesting}=""
-])
-
-dnl Usage: PAC_PUSH_ALL_FLAGS
-AC_DEFUN([PAC_PUSH_ALL_FLAGS],[
-	PAC_PUSH_FLAG(CFLAGS)
-	PAC_PUSH_FLAG(CPPFLAGS)
-	PAC_PUSH_FLAG(CXXFLAGS)
-	PAC_PUSH_FLAG(FFLAGS)
-	PAC_PUSH_FLAG(FCFLAGS)
-	PAC_PUSH_FLAG(LDFLAGS)
-	PAC_PUSH_FLAG(LIBS)
-])
-
-dnl Usage: PAC_POP_ALL_FLAGS
-AC_DEFUN([PAC_POP_ALL_FLAGS],[
-	PAC_POP_FLAG(CFLAGS)
-	PAC_POP_FLAG(CPPFLAGS)
-	PAC_POP_FLAG(CXXFLAGS)
-	PAC_POP_FLAG(FFLAGS)
-	PAC_POP_FLAG(FCFLAGS)
-	PAC_POP_FLAG(LDFLAGS)
-	PAC_POP_FLAG(LIBS)
-])
-
-dnl PAC_PREFIX_FLAG - Save flag with a prefix
-dnl Usage: PAC_PREFIX_FLAG(PREFIX, FLAG)
-AC_DEFUN([PAC_PREFIX_FLAG],[
-	$1_$2=$$2
-	export $1_$2
-	AC_SUBST($1_$2)
-])
-
-dnl PAC_PREFIX_ALL_FLAGS - Save flags with a prefix
-dnl Usage: PAC_PREFIX_ALL_FLAGS(PREFIX)
-AC_DEFUN([PAC_PREFIX_ALL_FLAGS],[
-	PAC_PREFIX_FLAG($1, CFLAGS)
-	PAC_PREFIX_FLAG($1, CPPFLAGS)
-	PAC_PREFIX_FLAG($1, CXXFLAGS)
-	PAC_PREFIX_FLAG($1, FFLAGS)
-	PAC_PREFIX_FLAG($1, FCFLAGS)
-	PAC_PREFIX_FLAG($1, LDFLAGS)
-	PAC_PREFIX_FLAG($1, LIBS)
-	PAC_PREFIX_FLAG($1, EXTRA_LIBS)
-])
-
-dnl Usage: PAC_APPEND_FLAG([-02], [CFLAGS])
-dnl appends the given argument to the specified shell variable unless the
-dnl argument is already present in the variable
-AC_DEFUN([PAC_APPEND_FLAG],[
-	AC_REQUIRE([AC_PROG_FGREP])
-	AS_IF(
-		[echo "$$2" | $FGREP -e "\<$1\>" >/dev/null 2>&1],
-		[echo "$2(='$$2') contains '$1', not appending" >&AS_MESSAGE_LOG_FD],
-		[echo "$2(='$$2') does not contain '$1', appending" >&AS_MESSAGE_LOG_FD
-		$2="$$2 $1"]
-	)
-])
-
-dnl Usage: PAC_PREPEND_FLAG([-lpthread], [LIBS])
-dnl Prepends the given argument to the specified shell variable unless the
-dnl argument is already present in the variable.
-dnl
-dnl This is typically used for LIBS and similar variables because libraries
-dnl should be added in reverse order.
-AC_DEFUN([PAC_PREPEND_FLAG],[
-        AC_REQUIRE([AC_PROG_FGREP])
-        AS_IF(
-                [echo "$$2" | $FGREP -e "\<$1\>" >/dev/null 2>&1],
-                [echo "$2(='$$2') contains '$1', not prepending" >&AS_MESSAGE_LOG_FD],
-                [echo "$2(='$$2') does not contain '$1', prepending" >&AS_MESSAGE_LOG_FD
-                $2="$1 $$2"]
-        )
-])
-
-
-dnl PAC_MKDIRS(path)
-dnl Create any missing directories in the path
-AC_DEFUN([PAC_MKDIRS],[
-# Build any intermediate directories
-for dir in $1 ; do
-    PAC_PUSH_FLAG([IFS])
-    IFS="/"
-    tmp_curdir=""
-    for tmp_subdir in $dir ; do
-	tmp_curdir="${tmp_curdir}$tmp_subdir"
-	if test ! -d "$tmp_curdir" ; then mkdir "$tmp_curdir" ; fi
-        tmp_curdir="${tmp_curdir}/"
-    done
-    PAC_POP_FLAG([IFS])
-done
-])
-
-# Find something to use for mkdir -p.  Eventually, this will have a
-# script for backup. As of autoconf-2.63, AC_PROG_MKDIR_P was broken;
-# it was checking to see if it recognized the "version" of mkdir and
-# was deciding based on that. This should always be a feature test.
-AC_DEFUN([PAC_PROG_MKDIR_P],[
-AC_CACHE_CHECK([whether mkdir -p works],
-pac_cv_mkdir_p,[
-pac_cv_mkdir_p=no
-rm -rf .tmp
-if mkdir -p .tmp/.foo 1>/dev/null 2>&1 ; then
-    if test -d .tmp/.foo ; then
-        pac_cv_mkdir_p=yes
-    fi
-fi
-rm -rf .tmp
-])
-if test "$pac_cv_mkdir_p" = "yes" ; then
-   MKDIR_P="mkdir -p"
-   export MKDIR_P
-else
-   AC_MSG_WARN([mkdir -p does not work; the install step may fail])
-fi
-AC_SUBST(MKDIR_P)
-])
-
-dnl Test for a clean VPATH directory.  Provide this command with the names
-dnl of all of the generated files that might cause problems 
-dnl (Makefiles won't cause problems because there's no VPATH usage for them)
-dnl
-dnl Synopsis
-dnl PAC_VPATH_CHECK([file-names],[directory-names])
-dnl  file-names should be files other than config.status and any header (e.g.,
-dnl fooconf.h) file that should be removed.  It is optional
-AC_DEFUN([PAC_VPATH_CHECK],[
-# This is needed for Mac OSX 10.5
-rm -rf conftest.dSYM
-rm -f conftest*
-date >conftest$$
-# If creating a file in the current directory does not show up in the srcdir
-# then we're doing a VPATH build (or something is very wrong)
-if test ! -s $srcdir/conftest$$ ; then
-    pac_dirtyfiles=""
-    pac_dirtydirs=""
-    pac_header=""
-    ifdef([AC_LIST_HEADER],[pac_header=AC_LIST_HEADER])
-    for file in config.status $pac_header $1 ; do
-        if test -f $srcdir/$file ; then 
-	    pac_dirtyfiles="$pac_dirtyfiles $file"
-	fi
-    done
-    ifelse($2,,,[
- 	for dir in $2 ; do 
-            if test -d $srcdir/$dir ; then
-                pac_dirtydirs="$pac_dirtydirs $dir"
-	    fi
-	done
-    ])
-
-    if test -n "$pac_dirtyfiles" -o -n "$pac_dirtydirs" ; then
-	# Create a nice message about what to remove
-	rmmsg=""
-	if test -n "$pac_dirtyfiles" ; then
-	    rmmsg="files $pac_dirtyfiles"
-        fi
- 	if test -n "$pac_dirtydirs" ; then
-	    if test -n "$rmmsg" ; then
-	        rmmsg="$rmmsg and directories $pac_dirtydirs"
-            else
-                rmmsg="directories $pac_dirtydirs"
-            fi
-        fi
-        if test -f $srcdir/Makefile ; then
-            AC_MSG_ERROR([You cannot do a VPATH build if the source directory has been
-    configured.  Run "make distclean" in $srcdir first and make sure that the
-    $rmmsg have been removed.])
-        else
-            AC_MSG_ERROR([You cannot do a VPATH build if the source directory has been
-    configured.  Remove the $rmmsg in $srcdir.])
-        fi
-    fi
-fi
-# This is needed for Mac OSX 10.5
-rm -rf conftest.dSYM
-rm -f conftest*
-])
-
-dnl PAC_CONF_HEX_TO_DEC(value,out_var)
-dnl
-dnl Converts the given hexadecimal integer constant to an integer constant and
-dnl stores the result in the shell variable given by 'out_var'.
-dnl
-dnl I think that printf like this will be sufficiently portable, but I don't
-dnl have any guarantee of it.  If not, we can fall back to AS_VAR_ARITH
-dnl and/or AC_COMPUTE_INT (the latter will probably be slow)
-AC_DEFUN([PAC_CONV_HEX_TO_DEC],[AS_VAR_SET([$2],[`printf "%d" $1`])])
-
-dnl PAC_GET_EXENAME(exe_name, out_exe_name)
-dnl
-dnl Prepends and appends program prefix and suffix as supplied by --program_prefix
-dnl and --program-sufix
-AC_DEFUN([PAC_GET_EXENAME],[
-$2=$1
-if test "$program_prefix" != "NONE" ; then
-    $2="${program_prefix}$$2"
-fi
-if test "$program_suffix" != "NONE" ; then
-    $2="$$2$program_suffix"
-fi
-])
diff --git a/3rd-party/romio321/confdb/ax_prefix_config_h.m4 b/3rd-party/romio321/confdb/ax_prefix_config_h.m4
deleted file mode 100644
index 83f8df6f4b7..00000000000
--- a/3rd-party/romio321/confdb/ax_prefix_config_h.m4
+++ /dev/null
@@ -1,219 +0,0 @@
-# ===========================================================================
-#           http://autoconf-archive.cryp.to/ax_prefix_config_h.html
-# ===========================================================================
-#
-# SYNOPSIS
-#
-#   AX_PREFIX_CONFIG_H [(OUTPUT-HEADER [,PREFIX [,ORIG-HEADER]])]
-#
-# DESCRIPTION
-#
-#   This is a new variant from ac_prefix_config_ this one will use a
-#   lowercase-prefix if the config-define was starting with a
-#   lowercase-char, e.g. "#define const", "#define restrict", or "#define
-#   off_t", (and this one can live in another directory, e.g.
-#   testpkg/config.h therefore I decided to move the output-header to be the
-#   first arg)
-#
-#   takes the usual config.h generated header file; looks for each of the
-#   generated "#define SOMEDEF" lines, and prefixes the defined name (ie.
-#   makes it "#define PREFIX_SOMEDEF". The result is written to the output
-#   config.header file. The PREFIX is converted to uppercase for the
-#   conversions.
-#
-#   Defaults:
-#
-#     OUTPUT-HEADER = $PACKAGE-config.h
-#     PREFIX = $PACKAGE
-#     ORIG-HEADER, from AM_CONFIG_HEADER(config.h)
-#
-#   Your configure.ac script should contain both macros in this order, and
-#   unlike the earlier variations of this prefix-macro it is okay to place
-#   the AX_PREFIX_CONFIG_H call before the AC_OUTPUT invokation.
-#
-#   Example:
-#
-#     AC_INIT(config.h.in)        # config.h.in as created by "autoheader"
-#     AM_INIT_AUTOMAKE(testpkg, 0.1.1)    # makes #undef VERSION and PACKAGE
-#     AM_CONFIG_HEADER(config.h)          # prep config.h from config.h.in
-#     AX_PREFIX_CONFIG_H(mylib/_config.h) # prep mylib/_config.h from it..
-#     AC_MEMORY_H                         # makes "#undef NEED_MEMORY_H"
-#     AC_C_CONST_H                        # makes "#undef const"
-#     AC_OUTPUT(Makefile)                 # creates the "config.h" now
-#                                         # and also mylib/_config.h
-#
-#   if the argument to AX_PREFIX_CONFIG_H would have been omitted then the
-#   default outputfile would have been called simply "testpkg-config.h", but
-#   even under the name "mylib/_config.h" it contains prefix-defines like
-#
-#     #ifndef TESTPKG_VERSION
-#     #define TESTPKG_VERSION "0.1.1"
-#     #endif
-#     #ifndef TESTPKG_NEED_MEMORY_H
-#     #define TESTPKG_NEED_MEMORY_H 1
-#     #endif
-#     #ifndef _testpkg_const
-#     #define _testpkg_const _const
-#     #endif
-#
-#   and this "mylib/_config.h" can be installed along with other
-#   header-files, which is most convenient when creating a shared library
-#   (that has some headers) where some functionality is dependent on the
-#   OS-features detected at compile-time. No need to invent some
-#   "mylib-confdefs.h.in" manually. :-)
-#
-#   Note that some AC_DEFINEs that end up in the config.h file are actually
-#   self-referential - e.g. AC_C_INLINE, AC_C_CONST, and the AC_TYPE_OFF_T
-#   say that they "will define inline|const|off_t if the system does not do
-#   it by itself". You might want to clean up about these - consider an
-#   extra mylib/conf.h that reads something like:
-#
-#      #include <mylib/_config.h>
-#      #ifndef _testpkg_const
-#      #define _testpkg_const const
-#      #endif
-#
-#   and then start using _testpkg_const in the header files. That is also a
-#   good thing to differentiate whether some library-user has starting to
-#   take up with a different compiler, so perhaps it could read something
-#   like this:
-#
-#     #ifdef _MSC_VER
-#     #include <mylib/_msvc.h>
-#     #else
-#     #include <mylib/_config.h>
-#     #endif
-#     #ifndef _testpkg_const
-#     #define _testpkg_const const
-#     #endif
-#
-# LAST MODIFICATION
-#
-#   2008-04-12
-#
-# COPYLEFT
-#
-#   Copyright (c) 2008 Guido U. Draheim <guidod@gmx.de>
-#   Copyright (c) 2008 Marten Svantesson
-#   Copyright (c) 2008 Gerald Point <Gerald.Point@labri.fr>
-#
-#   This program is free software; you can redistribute it and/or modify it
-#   under the terms of the GNU General Public License as published by the
-#   Free Software Foundation; either version 2 of the License, or (at your
-#   option) any later version.
-#
-#   This program is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
-#   Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License along
-#   with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-#   As a special exception, the respective Autoconf Macro's copyright owner
-#   gives unlimited permission to copy, distribute and modify the configure
-#   scripts that are the output of Autoconf when processing the Macro. You
-#   need not follow the terms of the GNU General Public License when using
-#   or distributing such scripts, even though portions of the text of the
-#   Macro appear in them. The GNU General Public License (GPL) does govern
-#   all other use of the material that constitutes the Autoconf Macro.
-#
-#   This special exception to the GPL applies to versions of the Autoconf
-#   Macro released by the Autoconf Macro Archive. When you make and
-#   distribute a modified version of the Autoconf Macro, you may extend this
-#   special exception to the GPL to apply to your modified version as well.
-
-AC_DEFUN([AX_PREFIX_CONFIG_H],[dnl
-AC_BEFORE([AC_CONFIG_HEADERS],[$0])dnl
-AC_CONFIG_COMMANDS([ifelse($1,,$PACKAGE-config.h,$1)],[dnl
-AS_VAR_PUSHDEF([_OUT],[ac_prefix_conf_OUT])dnl
-AS_VAR_PUSHDEF([_DEF],[ac_prefix_conf_DEF])dnl
-AS_VAR_PUSHDEF([_PKG],[ac_prefix_conf_PKG])dnl
-AS_VAR_PUSHDEF([_LOW],[ac_prefix_conf_LOW])dnl
-AS_VAR_PUSHDEF([_UPP],[ac_prefix_conf_UPP])dnl
-AS_VAR_PUSHDEF([_INP],[ac_prefix_conf_INP])dnl
-m4_pushdef([_script],[conftest.prefix])dnl
-m4_pushdef([_symbol],[m4_cr_Letters[]m4_cr_digits[]_])dnl
-_OUT=`echo ifelse($1, , $PACKAGE-config.h, $1)`
-_DEF=`echo _$_OUT | sed -e "y:m4_cr_letters:m4_cr_LETTERS[]:" -e "s/@<:@^m4_cr_Letters@:>@/_/g"`
-_PKG=`echo ifelse($2, , $PACKAGE, $2)`
-_LOW=`echo _$_PKG | sed -e "y:m4_cr_LETTERS-:m4_cr_letters[]_:"`
-_UPP=`echo $_PKG | sed -e "y:m4_cr_letters-:m4_cr_LETTERS[]_:"  -e "/^@<:@m4_cr_digits@:>@/s/^/_/"`
-_INP=`echo "ifelse($3,,,$3)" | sed -e 's/ *//'`
-if test ".$_INP" = "."; then
-   for ac_file in : $CONFIG_HEADERS; do test "_$ac_file" = _: && continue
-     case "$ac_file" in
-        *.h) _INP=$ac_file ;;
-        *)
-     esac
-     test ".$_INP" != "." && break
-   done
-fi
-if test ".$_INP" = "."; then
-   case "$_OUT" in
-      */*) _INP=`basename "$_OUT"`
-      ;;
-      *-*) _INP=`echo "$_OUT" | sed -e "s/@<:@_symbol@:>@*-//"`
-      ;;
-      *) _INP=config.h
-      ;;
-   esac
-fi
-if test -z "$_PKG" ; then
-   AC_MSG_ERROR([no prefix for _PREFIX_PKG_CONFIG_H])
-else
-  if test ! -f "$_INP" ; then if test -f "$srcdir/$_INP" ; then
-     _INP="$srcdir/$_INP"
-  fi fi
-  AC_MSG_NOTICE(creating $_OUT - prefix $_UPP for $_INP defines)
-  if test -f $_INP ; then
-    echo "s/^@%:@undef  *\\(@<:@m4_cr_LETTERS[]_@:>@\\)/@%:@undef $_UPP""_\\1/" > _script
-    echo "s/^@%:@undef  *\\(@<:@m4_cr_letters@:>@\\)/@%:@undef $_LOW""_\\1/" >> _script
-    echo "s/^@%:@def[]ine  *\\(@<:@m4_cr_LETTERS[]_@:>@@<:@_symbol@:>@*\\)\\(.*\\)/@%:@ifndef $_UPP""_\\1 \\" >> _script
-    echo "@%:@def[]ine $_UPP""_\\1 \\2 \\" >> _script
-    echo "@%:@endif/" >>_script
-    echo "s/^@%:@def[]ine  *\\(@<:@m4_cr_letters@:>@@<:@_symbol@:>@*\\)\\(.*\\)/@%:@ifndef $_LOW""_\\1 \\" >> _script
-    echo "@%:@define $_LOW""_\\1 \\2 \\" >> _script
-    echo "@%:@endif/" >> _script
-    # now executing _script on _DEF input to create _OUT output file
-    echo "@%:@ifndef $_DEF"      >$tmp/pconfig.h
-    echo "@%:@def[]ine $_DEF 1" >>$tmp/pconfig.h
-    echo ' ' >>$tmp/pconfig.h
-    echo /'*' $_OUT. Generated automatically at end of configure. '*'/ >>$tmp/pconfig.h
-
-    sed -f _script $_INP >>$tmp/pconfig.h
-    echo ' ' >>$tmp/pconfig.h
-    echo '/* once:' $_DEF '*/' >>$tmp/pconfig.h
-    echo "@%:@endif" >>$tmp/pconfig.h
-    if cmp -s $_OUT $tmp/pconfig.h 2>/dev/null; then
-      AC_MSG_NOTICE([$_OUT is unchanged])
-    else
-      ac_dir=`AS_DIRNAME(["$_OUT"])`
-      AS_MKDIR_P(["$ac_dir"])
-      rm -f "$_OUT"
-      mv $tmp/pconfig.h "$_OUT"
-    fi
-    cp _script _configs.sed
-  else
-    AC_MSG_ERROR([input file $_INP does not exist - skip generating $_OUT])
-  fi
-  rm -f conftest.*
-fi
-m4_popdef([_symbol])dnl
-m4_popdef([_script])dnl
-AS_VAR_POPDEF([_INP])dnl
-AS_VAR_POPDEF([_UPP])dnl
-AS_VAR_POPDEF([_LOW])dnl
-AS_VAR_POPDEF([_PKG])dnl
-AS_VAR_POPDEF([_DEF])dnl
-AS_VAR_POPDEF([_OUT])dnl
-],[PACKAGE="$PACKAGE"])])
-
-dnl implementation note: a bug report (31.5.2005) from Marten Svantesson points
-dnl out a problem where `echo "\1"` results in a Control-A. The unix standard
-dnl    http://www.opengroup.org/onlinepubs/000095399/utilities/echo.html
-dnl defines all backslash-sequences to be inherently non-portable asking
-dnl for replacement mit printf. Some old systems had problems with that
-dnl one either. However, the latest libtool (!) release does export an $ECHO
-dnl (and $echo) that does the right thing - just one question is left: what
-dnl was the first version to have it? Is it greater 2.58 ?
diff --git a/3rd-party/romio321/confdb/ax_tls.m4 b/3rd-party/romio321/confdb/ax_tls.m4
deleted file mode 100644
index 3f6a0ac04f4..00000000000
--- a/3rd-party/romio321/confdb/ax_tls.m4
+++ /dev/null
@@ -1,90 +0,0 @@
-# ===========================================================================
-#             http://www.nongnu.org/autoconf-archive/ax_tls.html
-# ===========================================================================
-#
-# SYNOPSIS
-#
-#   AX_TLS
-#
-# DESCRIPTION
-#
-#   Provides a test for the compiler support of thread local storage (TLS)
-#   extensions. Defines TLS if it is found. Currently only knows about GCC
-#   and MSVC. I think SunPro uses the same as GCC, and Borland apparently
-#   supports either.
-#
-# LICENSE
-#
-#   Copyright (c) 2008 Alan Woodland <ajw05@aber.ac.uk>
-#
-#   This program is free software: you can redistribute it and/or modify it
-#   under the terms of the GNU General Public License as published by the
-#   Free Software Foundation, either version 3 of the License, or (at your
-#   option) any later version.
-#
-#   This program is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
-#   Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License along
-#   with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-#   As a special exception, the respective Autoconf Macro's copyright owner
-#   gives unlimited permission to copy, distribute and modify the configure
-#   scripts that are the output of Autoconf when processing the Macro. You
-#   need not follow the terms of the GNU General Public License when using
-#   or distributing such scripts, even though portions of the text of the
-#   Macro appear in them. The GNU General Public License (GPL) does govern
-#   all other use of the material that constitutes the Autoconf Macro.
-#
-#   This special exception to the GPL applies to versions of the Autoconf
-#   Macro released by the Autoconf Archive. When you make and distribute a
-#   modified version of the Autoconf Macro, you may extend this special
-#   exception to the GPL to apply to your modified version as well.
-
-AC_DEFUN([AX_TLS], [
-  AC_MSG_CHECKING(for thread local storage specifier)
-  AC_CACHE_VAL(ac_cv_tls, [
-    ax_tls_keywords="__thread __declspec(thread) none"
-    for ax_tls_keyword in $ax_tls_keywords; do
-       case $ax_tls_keyword in
-          none) ac_cv_tls=none ; break ;;
-	  *)
-             # MPICH modification: This was an AC_TRY_COMPILE before, but
-             # Darwin with non-standard compilers will accept __thread at
-             # compile time but fail to link due to an undefined
-             # "__emutls_get_address" symbol unless -lgcc_eh is added to the
-             # link line.
-             AC_LINK_IFELSE(
-                 [AC_LANG_PROGRAM([$ax_tls_keyword int bar = 5;],[++bar;])],
-                 [ac_cv_tls=$ax_tls_keyword],
-                 [ac_cv_tls=none])
-
-	     # MPICH modification: Also test with the extern keyword.
-	     # The intel compiler on Darwin (at least as of 15.0.1)
-	     # seems to break with the above error when the extern
-	     # keyword is specified in shared library builds.
-	     PAC_PUSH_FLAG([LIBS])
-	     PAC_APPEND_FLAG([-shared],[LIBS])
-	     if test "$ac_cv_tls" != "none" ; then
-                AC_LINK_IFELSE(
-			[AC_LANG_PROGRAM([extern $ax_tls_keyword int bar;],[++bar;])],
-                        [ac_cv_tls=$ax_tls_keyword],
-                        [ac_cv_tls=none])
-	     fi
-	     PAC_POP_FLAG([LIBS])
-
-	     if test "$ac_cv_tls" != "none" ; then break ; fi
-          esac
-    done
-])
-
-  if test "$ac_cv_tls" != "none"; then
-    # MPICH modification: this was "TLS" before instead of
-    # "MPICH_TLS_SPECIFIER", but TLS had a reasonably high chance of conflicting
-    # with a system library.
-    AC_DEFINE_UNQUOTED([MPICH_TLS_SPECIFIER], $ac_cv_tls, [If the compiler supports a TLS storage class define it to that here])
-  fi
-  AC_MSG_RESULT($ac_cv_tls)
-])
diff --git a/3rd-party/romio321/configure.ac b/3rd-party/romio321/configure.ac
deleted file mode 100644
index 20298739d8a..00000000000
--- a/3rd-party/romio321/configure.ac
+++ /dev/null
@@ -1,1835 +0,0 @@
-# -*- Mode: shell-script -*-
-# build with
-# autoconf --localdir=../confdb configure.ac
-# (or wherever the confdb is)
-#
-# irrelevant / unnecessary in an Open MPI environment, but are
-# harmless and are left here solely for the sake of ease of future
-# patching/importing.
-AC_PREREQ([2.63])
-
-# Open MPI: Modifications to this file were done on an "let's do the
-# minimum possible" basis, not so that we can skip on the work or
-# provide any less functionality, but more from a perspective that we
-# want to be able to import new versions of ROMIO in as easy a fashion
-# as possible.  Hence, there are some things in this file that are
-# irrelevant / unnecessary in an Open MPI environment, but are
-# harmless and are left here solely for the sake of ease of future
-# patching/importing.
-AC_INIT([ROMIO],
-        [Open MPI],
-        [discuss@mpich.org],
-        [romio],
-        [http://www.mpich.org/])
-
-dnl AC_CONFIG_AUX_DIR(../../../confdb)
-dnl Set the directory that contains the required install-sh, config.sub,
-dnl and config.guess .  Make sure that these are updated (in MPICH, use
-dnl the top-level confdb files).  This separate directory is used for
-dnl the moment to allow ROMIO to be separatedly distributed.
-dnl scripts.
-AC_CONFIG_AUX_DIR([confdb])
-AC_CONFIG_MACRO_DIR([confdb])
-
-AM_INIT_AUTOMAKE([-Wall -Werror -Wno-portability-recursive foreign 1.12 silent-rules subdir-objects])
-AM_MAINTAINER_MODE([enable])
-
-dnl must come before LT_INIT, which AC_REQUIREs AC_PROG_CC
-PAC_PROG_CC
-AM_PROG_CC_C_O
-
-AM_PROG_AR
-
-LT_INIT([])
-# Non-verbose make by default
-m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
-
-# VERSION=1.2.6
-# AC_MSG_RESULT([Configuring ROMIO Version $VERSION])
-CONFIGURE_ARGS="$*"
-if test -n "$CONFIGURE_ARGS" ; then
-    echo "Configuring with args $CONFIGURE_ARGS"
-fi
-
-AC_CONFIG_HEADER(adio/include/romioconf.h)
-# Open MPI: modified AH_TOP
-AH_TOP([/*
- *  (C) 2011 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#ifndef ROMIOCONF_H_INCLUDED
-#define ROMIOCONF_H_INCLUDED
-
-#include "romioconf-undefs.h"
-])
-AH_BOTTOM([
-/* quash PACKAGE and PACKAGE_* vars, see MPICH top-level configure.ac for
- * more info */
-#include "nopackage.h"
-
-#endif /* !defined(ROMIOCONF_H_INCLUDED) */
-])
-
-# Open MPI: this configure script doesn't seem to define these
-# anywhere, so just do them manually here because "we know better"
-# (i.e., Open MPI can be hard-wired to these values).
-AC_DEFINE([HAVE_MPI_OFFSET], [1], [Will always be 1 - OMPI has MPI_OFFSET])
-
-# Open MPI: look for top Open MPI directory
-AC_MSG_CHECKING([for Open MPI support files])
-if test -f "$srcdir/../../config/opal_mca.m4"; then
-
-    # This is needed for VPATH builds, so that it will -I the
-    # appropriate include directory (don't know why automake
-    # doesn't do this # automatically).
-
-    OMPI_TOP_SRCDIR='$(top_srcdir)/../..'
-    OMPI_TOP_BUILDDIR='$(top_builddir)/../..'
-    with_mpi="$OMPI_TOP_SRCDIR"
-    AC_MSG_RESULT([in Open MPI source tree -- good])
-    AC_SUBST(OMPI_TOP_SRCDIR)
-    AC_SUBST(OMPI_TOP_BUILDDIR)
-else
-    AC_MSG_RESULT([not found])
-    AC_MSG_WARN([*** Could not find Open MPI support files])
-    AC_MSG_WARN([*** Can only build this version of ROMIO in an Open MPI source tree])
-    AC_MSG_ERROR([*** Cannot continue])
-fi
-
-dnl
-# Open MPI: disable the f77 and f90 tests, as we provide our own
-# MPI interface and use only the C parts of ROMIO
-NOF77=1
-NOF90=1
-ARCH=""
-arch_IRIX=""
-MPI_IMPL=""
-MPI_INCLUDE_DIR=""
-ROMIO_INCLUDE=""
-
-# Used by the new build system, should contain zero or more "-Iblah" args for
-# inclusion in AM_CPPFLAGS.  Should not contain relative paths.  This probably
-# overlaps with ROMIO_INCLUDE some, but adding a new var is easier than teasing
-# apart all of the current usages of that variable and re-testing all of the
-# non-MPICH and exotic platform cases.
-MPI_H_INCLUDE=""
-AC_SUBST([MPI_H_INCLUDE])
-
-TEST_LIBNAME=""
-FILE_SYSTEM=""
-
-# Do not set variables to empty that may be communicated from the
-# outside environment (e.g., MPI_LIB, MPI_BIN_DIR, LIBNAME)
-DEBUG=no
-MIPS=0
-BITS=0
-
-AC_ARG_VAR([FROM_MPICH],[set to "yes" if building ROMIO inside of MPICH])
-FROM_MPICH=${FROM_MPICH:-no}
-
-AC_ARG_VAR([FROM_LAM],[set to "yes" if building ROMIO inside of LAM])
-FROM_LAM=${FROM_LAM:-no}
-if test "$FROM_LAM" = 1 ; then FROM_LAM=yes ; fi
-
-AC_ARG_VAR([FROM_OMPI],[set to "yes" if building ROMIO inside of Open MPI])
-FROM_OMPI=${FROM_OMPI:-no}
-if test "$FROM_OMPI" = 1 ; then FROM_OMPI=yes ; fi
-
-AC_ARG_VAR([FROM_OMPI],[set to "yes" if building ROMIO inside of Open MPI])
-FROM_OMPI=${FROM_OMPI:-no}
-if test "$FROM_OMPI" = 1 ; then FROM_OMPI=yes ; fi
-
-CFLAGS=${CFLAGS:-""}
-LL="lld"
-AR_LOCAL=""
-DEFINE_HAVE_MPI_GREQUEST="#undef HAVE_MPI_GREQUEST"
-HAVE_MPI_INFO=""
-BUILD_MPI_INFO=""
-MPI_FINFO1=""
-MPI_FINFO2=""
-MPI_FINFO3=""
-MPI_FINFO4=""
-MPI_FARRAY1=""
-MPI_FARRAY2=""
-MPI_FARRAY3=""
-MPI_FARRAY4=""
-MPI_FARRAY5=""
-MPI_FARRAY6=""
-MPI_FARRAY7=""
-DEFS=""
-ROMIO_LFLAGS=""
-ROMIO_TCFLAGS=""
-ROMIO_TCPPFLAGS=""
-ROMIO_TFFLAGS=""
-NOPROFILE=0
-MPIRUN=""
-FORTRAN_TEST=""
-# Open MPI: This (setting make) is a Bad Thing to do in Automake-based build systems
-# MAKE=${MAKE:-"make"}
-# foll. needed for f77 test programs
-F77GETARG="call getarg(i,str)"
-F77IARGC="iargc()"
-F77MPIOINC=""
-FTESTDEFINE=""
-FORTRAN_MPI_OFFSET=""
-MPIOF_H_INCLUDED=0
-MPI_OFFSET_KIND1="!"
-MPI_OFFSET_KIND2="!"
-TEST_CC=""
-TEST_F77=""
-#
-# Error handlers (not used with MPICH2, which provides its own routines)
-MPIO_EXTRA_OBJECTS="get_errh.o set_errh.o"
-MPIO_EXTRA_TMP_POBJECTS="get_errh.p set_errh.p"
-MPIO_EXTRA_REAL_POBJECTS="_get_errh.o _set_errh.o"
-#
-# Completion routines for MPIO_Requests.  MPI Implementations with
-# generalized requests do not need these
-# ioreq_c2f and ioreq_f2c are not MPIO_Requests; rather, they
-MPIO_REQOBJECTS="iotest.o iotestall.o iotestany.o iotestsome.o iowait.o iowaitall.o iowaitany.o iowaitsome.o ioreq_c2f.o ioreq_f2c.o"
-MPIO_REQ_TMP_POBJECTS="iotest.p iowait.p iowaitall.p iowaitany.p iotestall.p iotestany.p iowaitsome.p iotestsome.p"
-MPIO_REQ_REAL_POBJECTS="_iotest.o _iowait.o _iowaitall.o _iowaitany.o _iotestall.o _iotestany.o _iowaitsome.o _iotestsome.o"
-#
-known_mpi_impls="mpich_mpi mpich_mpi sgi_mpi hp_mpi cray_mpi lam_mpi open_mpi_mpi"
-
-dnl An m4 macro for use with m4_foreach_w and friends.  You should modify this
-dnl list if you want to add a known file system.  The list is just whitespace
-dnl separated, so you can use newlines and tabs as well.
-m4_define([known_filesystems_m4_w],
-          [nfs ufs pfs pvfs pvfs2 testfs xfs panfs gridftp lustre gpfs zoidfs hfs piofs sfs])dnl
-dnl
-dnl An m4 macro for use with m4_foreach and friends.  Expands to a quoted list of
-dnl quoted elements.  A bit easier to use without unintended expansion than the
-dnl whitespace version.
-m4_define([known_filesystems_m4], m4_dquote(m4_map_args_w(m4_defn([known_filesystems_m4_w]),[],[],[,])))dnl
-dnl
-# a shell var for checking arguments given via --with-file-system=...
-known_filesystems="m4_join([ ],known_filesystems_m4)"
-
-#####################################################################
-
-#
-# Defaults
-AC_ARG_ENABLE(aio,[
---enable-aio - Request use of asynchronous I/O routines (default)],
-[ 
-  if test "x$enableval" = "xno" ; then
-    disable_aio=yes
-  else
-    disable_aio=no
-  fi
-], disable_aio=no)
-AC_ARG_ENABLE(echo, 
-[--enable-echo  - Turn on strong echoing. The default is enable=no.] ,set -x)
-AC_ARG_ENABLE(f77,
-[--enable-f77 - Turn on support for Fortran 77 (default)],,enable_f77=yes)
-AC_ARG_ENABLE(f90,
-[--enable-f90 - Turn on support for Fortran 90 (default)],,enable_f90=yes)
-AC_ARG_ENABLE(weak-symbols,
-[--enable-weak-symbols - Turn on support for weak symbols],,enable_weak_symbols=no)
-AC_ARG_ENABLE(debug,
-[--enable-debug - Build a debugging version],,)
-AC_ARG_WITH(file-system,[
---with-file-system=name - Build with support for the named file systems],,)
-AC_ARG_WITH(pvfs2,[
---with-pvfs2=path - Path to installation of PVFS (version 2)],,)
-AC_ARG_WITH(mpi-impl,[
---with-mpi-impl=name - Specify MPI implementation to build ROMIO for],,)
-dnl
-AC_ARG_WITH(mpi, [
---with-mpi=path   - Path to instalation of MPI (headers, libs, etc)],,)
-dnl
-if test "$enable_f77" != "yes" ; then
-   NOF77=1
-fi
-if test "$enable_f90" != "yes" ; then
-   NOF90=1
-fi
-if test "$enable_debug" = "yes" ; then
-    DEBUG=yes
-fi
-MPI=$with_mpi
-# Open MPI: No!
-#if test -n "$with_mpi"; then
-#       CC=$MPI/bin/mpicc
-#fi
-
-# start with the set of file systems that the user asked for
-# FILE_SYSTEM=$with_file_system
-FILE_SYSTEM=`echo $with_file_system | sed -e 's/:.*$//'`
-
-# Check if Make is working
-PAC_PROG_MAKE
-#
-# Check that an arch was set
-# If it wasn't set, try to guess using "util/tarch"
-#
-# Sometimes tarch looses its execute bit (!)
-if test -s $srcdir/util/tarch -a ! -x $srcdir/util/tarch ; then
-    chmod a+x $srcdir/util/tarch
-fi
-if test -z "$ARCH" -a -x $srcdir/util/tarch ; then
-    AC_MSG_CHECKING([for architecture])
-    ARCH=`$srcdir/util/tarch | sed s/-/_/g`
-    if test -z "$ARCH" ; then
-       AC_MSG_RESULT([Unknown!])
-       AC_MSG_ERROR([Error: Could not guess target architecture, you must 
-set an architecture type with the environment variable ARCH]) 
-    fi 
-    eval "arch_$ARCH=1"
-    AC_MSG_RESULT($ARCH)
-fi
-#
-# check for valid architecture.  Use __ so that _ALPHA_ does not match
-# LINUX_ALPHA_
-#### WE SHOULD REMOVE THIS SOON
-grep __"$ARCH"_ $srcdir/.config_params > /dev/null 2>&1
-if test $? != 0 ; then
-   AC_MSG_WARN([Unknown architecture $ARCH... proceeding anyway])
-fi
-#
-#
-# Find the home directory if not specified
-if test "X$srcdir" != "X." -a -s $srcdir/mpi-io/Makefile.in ; then 
-    ROMIO_HOME_TRIAL=$srcdir
-else
-    # Take advantage of autoconf2 features
-    if test -n "$ac_confdir" ; then
-        ROMIO_HOME_TRIAL=$ac_confdir
-    else
-        if test -s configure ; then
-	    ROMIO_HOME_TRIAL=`pwd`
-	else
-	    ac_confdir=`dirname "$0" 2>/dev/null`
-	    if test -n "$ac_confdir" ; then
-	        ROMIO_HOME_TRIAL=$ac_confdir
-	    fi
-	fi
-    fi
-fi
-AC_MSG_RESULT([ROMIO home directory is $ROMIO_HOME_TRIAL])
-ROMIO_HOME=$ROMIO_HOME_TRIAL
-
-# get a fully qualified pathname for our build directory
-top_build_dir=`pwd`
-# used in romioinstall
-AC_SUBST(top_build_dir)
-
-# Open MPI: these shouldn't be needed with AM
-#
-# Create the "autoconf" style directory names...
-# Most of these are done for us; add the documentation directories
-#
-# mandir is the root for the man pages
-if test -z "$mandir" ; then mandir='${prefix}/man' ; fi
-AC_SUBST(mandir)
-if test -z "$docdir" ; then docdir='${prefix}/doc' ; fi
-AC_SUBST(docdir)
-if test -z "$htmldir" ; then htmldir='${prefix}/www' ; fi
-AC_SUBST(htmldir) 
-
-
-# If we are building within a known MPI implementation, we must avoid the
-# tests about an existing implementation
-if test "$FROM_MPICH" != no -o "$FROM_LAM" != no -o "$FROM_OMPI" != no ; then
-    WITHIN_KNOWN_MPI_IMPL=yes
-else
-    WITHIN_KNOWN_MPI_IMPL=no
-fi
-
-# Open MPI: Set the MPI implementation
-if test "$FROM_OMPI" = "yes" ; then
-    MPI_IMPL=open_mpi
-fi
-
-# check for valid MPI implementation
-if test -n "$MPI_IMPL" ; then
-   found=no
-   for mpi in $known_mpi_impls ; do
-      if test "${MPI_IMPL}_mpi" = "$mpi" ; then
-          found=yes
-	  break
-      fi
-   done
-   if test $found = no ; then
-      AC_MSG_WARN([Unknown MPI implementation $MPI... proceeding anyway])
-   fi
-fi
-#
-
-if test -n "${with_mpi}" ; then
-	MPI_INCLUDE_DIR="${with_mpi}"/include
-	MPI_LIB_DIR="${with_mpi}"/lib
-fi
-
-# check for valid MPI include directory if specified
-if test $WITHIN_KNOWN_MPI_IMPL = no ; then
-   if test -n "$MPI_INCLUDE_DIR"; then
-      if test ! -f "$MPI_INCLUDE_DIR/mpi.h" ; then
-         AC_MSG_ERROR([Include file $MPI_INCLUDE_DIR/mpi.h not found])
-      fi
-   else
-#     assume that mpi.h is in the default path
-#     set MPI_INCLUDE_DIR to ".", so that it translates to -I. in the
-#     compile command. Some compilers complain if it's only -I
-      MPI_INCLUDE_DIR=.
-   fi
-else 
-   MPI_INCLUDE_DIR=.
-fi
-#
-# check for valid MPI library if specified
-if test $WITHIN_KNOWN_MPI_IMPL = no ; then
-   if test -n "$MPI_LIB" ; then
-      if test ! -f "$MPI_LIB" ; then
-         AC_MSG_ERROR([MPI library $MPI_LIB not found])
-      fi
-   fi
-fi
-
-# USER_CFLAGS and USER_FFLAGS are used only in test/Makefile.in
-if test $DEBUG = "yes"; then
-    USER_CFLAGS="$CFLAGS -g"
-    USER_FFLAGS="$FFLAGS -g"
-else
-    USER_CFLAGS="$CFLAGS -O"
-    USER_FFLAGS="$FFLAGS -O"
-fi
-#
-# Here begin the architecture-specific tests.  
-# --------------------------------------------------------------------------
-# We must first select the C and Fortran compilers.  Because of the 
-# way that the PROG_CC autoconf macro works (and all of the macros that
-# require it, including CHECK_HEADERS), that macro must occur exactly
-# once in the configure.ac file, at least as of autoconf 2.57 .  
-# Unfortunately, this requirement is not enforced.  To handle this,
-# we first case on the architecture; then use PROG_CC, then case on the
-# architecture again for any arch-specific features.  We also set the
-# C_DEBUG_FLAG and F77_DEBUG_FLAG in case debugging is selected.
-# 
-# For the MPICH and MPICH configures, the compilers will already be
-# selected, so most of the compiler-selection code will be bypassed.
-# --------------------------------------------------------------------------
-# For historical reasons
-if test -z "$FC" ; then 
-    FC=$F77
-fi
-#
-C_DEBUG_FLAG="-g"
-F77_DEBUG_FLAG="-g"
-
-dnl AC_PROG_{CXX,F77,FC} must come early in configure.ac in order to control
-dnl compiler search order and avoid some esoteric autoconf macro expansion
-dnl errors
-if test "$enable_f77" = "yes" ; then
-    # suppress default "-g -O2" from AC_PROG_F77
-    : ${FFLAGS=""}
-    AC_PROG_F77([PAC_F77_SEARCH_LIST])
-fi
-if test "$enable_f90" = "yes" ; then
-    # suppress default "-g -O2" from AC_PROG_FC
-    : ${FCFLAGS=""}
-    AC_PROG_FC([PAC_FC_SEARCH_LIST])
-fi
-
-if test "$CC" = "gcc" -a -z "$C_DEBUG_FLAG" ; then
-     C_DEBUG_FLAG="-g -O -Wall -Wstrict-prototypes -Wmissing-prototypes"
-fi
-if test $DEBUG = "yes" ; then
-    CFLAGS="$CFLAGS $C_DEBUG_FLAG"
-# Open MPI: don't add optflags - they'll come from the top-level configure
-#else
-#    CFLAGS="$CFLAGS $C_OPT_FLAG"
-fi
-# ---------------------------------------------------------------------------
-# Here go the rest of the tests
-# ---------------------------------------------------------------------------
-
-AC_CHECK_TYPE(long long)
-AC_CHECK_SIZEOF(long long)
-if test -z "$MPI_IMPL" ; then
-	MPI_IMPL=mpich
-	mpi_mpich=1
-fi
-if test $MPI_IMPL = "mpich" ; then
-	TEST_CC=mpicc
-	TEST_F77=mpifort
-else
-	TEST_CC="$CC"
-	TEST_F77="$F77"
-fi
-# there used to be a ton of arch-specific stuff in here.  If some random
-# platform really truly needs it, restore it, but that defeats the whole
-
-PAC_GET_SPECIAL_SYSTEM_INFO
-
-AC_HAVE_FUNCS(memalign)
-
-#
-# Question: Should ROMIO under MPICH ignore the Fortran tests, since 
-# MPICH provides all of the Fortran interface routines?
-#
-if test $NOF77 = 0 ; then
-    echo "checking Fortran external names"
-    PAC_GET_FORTNAMES
-    if test -n "$WDEF" ; then
-        CFLAGS="$CFLAGS $WDEF"
-    fi
-    dnl PAC_PROG_F77_NAME_MANGLE
-    dnl (need to set the new name format)
-    rm -f test/mpif.h
-    if test "$MPI_INCLUDE_DIR" != "." && test $WITHIN_KNOWN_MPI_IMPL = no ; then
-        if test ! -d test ; then mkdir test ; fi
-        ln -s $MPI_INCLUDE_DIR/mpif.h test
-    fi
-else 
-    F77=":"
-fi
-#
-# Open MPI: We already do this test top-level
-dnl AC_C_INLINE
-
-AC_TYPE_SIZE_T
-AC_TYPE_SSIZE_T
-AC_TYPE_OFF_T
-
-# Header files
-# Find the CPP before the header check
-AC_PROG_CPP
-AC_CHECK_HEADERS([unistd.h fcntl.h malloc.h stddef.h sys/types.h limits.h time.h])
-AC_CHECK_HEADERS(mpix.h,,,[#include <mpi.h>])
-#
-
-# When compiling ROMIO on Darwin with _POSIX_C_SOURCE defined (such as when
-# using --enable-strict in MPICH), sys/types.h does not define u_short and
-# friends unless _DARWIN_C_SOURCE is also defined (see compat(5) on a Darwin
-# box).  This would normally be fine, except sys/stat.h defines struct stat to
-# use u_long, so strict compiles fail.  One option is to also compile with
-# _DARWIN_C_SOURCE, but this disables much of the strictness that is intended
-# by _POSIX_C_SOURCE.  Instead we just define our own types if they are not
-# provided by the system.  This isn't quite as safe as typedef'ing the
-# replacement types, but it will apply to later configure tests, which is
-# important.
-AC_CHECK_TYPE([u_char],[],[AC_DEFINE_UNQUOTED([u_char],[unsigned char],[Define to "unsigned char" if sys/types.h does not define.])])
-AC_CHECK_TYPE([u_short],[],[AC_DEFINE_UNQUOTED([u_short],[unsigned short],[Define to "unsigned short" if sys/types.h does not define.])])
-AC_CHECK_TYPE([u_int],[],[AC_DEFINE_UNQUOTED([u_int],[unsigned int],[Define to "unsigned int" if sys/types.h does not define.])])
-AC_CHECK_TYPE([u_long],[],[AC_DEFINE_UNQUOTED([u_long],[unsigned long],[Define to "unsigned long" if sys/types.h does not define.])])
-
-# must come _after_ the above checks for u_char/u_short/u_int/u_long
-AC_CHECK_HEADERS([sys/attr.h])
-
-AC_CHECK_SIZEOF(int)
-AC_CHECK_SIZEOF(void *)
-AC_CACHE_CHECK([for int large enough for pointers],
-pac_cv_int_hold_pointer,[
-if test "$ac_cv_sizeof_int" = "0" -o \
-	"$ac_cv_sizeof_void_p" = "0" ; then
-    pac_cv_int_hold_pointer=unknown
-elif test "$ac_cv_sizeof_int" -lt "$ac_cv_sizeof_void_p" ; then
-    pac_cv_int_hold_pointer=no
-else
-    pac_cv_int_hold_pointer=yes
-fi
-])
-if test "$pac_cv_int_hold_pointer" != yes ; then
-    AC_DEFINE(INT_LT_POINTER,1,[Define if int smaller than pointer])
-    dnl Switch to a conforming name (start with HAVE or USE)
-    AC_DEFINE(HAVE_INT_LT_POINTER,1,[Define if int smaller than pointer])
-fi
-
-# LL is the printf-style format name for output of a MPI_Offset.
-# We have to match this to the type that we use for MPI_Offset.
-AC_CHECK_SIZEOF(long long)
-if test "$ac_cv_sizeof_long_long" != 0 ; then
-    if test "$ac_cv_sizeof_long_long" = "8" ; then
-       AC_DEFINE(HAVE_LONG_LONG_64,1,[Define if long long is 64 bits])
-       MPI_OFFSET_TYPE="long long"
-       DEFINE_MPI_OFFSET="typedef long long MPI_Offset;"
-       FORTRAN_MPI_OFFSET="integer*8"
-       LL="lld"
-    elif test "$ac_cv_sizeof_long_long" = "$ac_cv_sizeof_int" ; then
-       MPI_OFFSET_TYPE="int"
-       DEFINE_MPI_OFFSET="typedef int MPI_Offset;"
-       FORTRAN_MPI_OFFSET="integer"
-       AC_DEFINE(MPI_OFFSET_IS_INT,1,[Define if MPI_Offset is int])
-       LL="d"
-       MPI_OFFSET_KIND1="!"
-       MPI_OFFSET_KIND2="!"
-    else
-       echo "defining MPI_Offset as long in C and integer in Fortran" 
-       MPI_OFFSET_TYPE="long"
-       DEFINE_MPI_OFFSET="typedef long MPI_Offset;"
-       FORTRAN_MPI_OFFSET="integer"
-       LL="ld"
-       MPI_OFFSET_KIND1="!"
-       MPI_OFFSET_KIND2="!"
-    fi
-else
-    echo "defining MPI_Offset as long in C and integer in Fortran" 
-    MPI_OFFSET_TYPE="long"
-    DEFINE_MPI_OFFSET="typedef long MPI_Offset;"
-    FORTRAN_MPI_OFFSET="integer"
-    LL="ld"
-    MPI_OFFSET_KIND1="!"
-    MPI_OFFSET_KIND2="!"
-fi
-
-
-#
-if test -n "$ac_cv_sizeof_long_long"; then
-   if test $WITHIN_KNOWN_MPI_IMPL = no ; then
-       PAC_MPI_LONG_LONG_INT
-   else
-       AC_DEFINE(HAVE_MPI_LONG_LONG_INT,1,[Define if supports long long int])
-   fi
-fi
-#
-if test -n "$OFFSET_KIND" -a "A$MPI_OFFSET_KIND1" = "A!" ; then 
-  MPI_OFFSET_KIND1="        INTEGER MPI_OFFSET_KIND"
-  MPI_OFFSET_KIND2="        PARAMETER (MPI_OFFSET_KIND=$OFFSET_KIND)"
-  MPI_OFFSET_KIND_VAL=$OFFSET_KIND
-else
- if test "$FORTRAN_MPI_OFFSET" = "integer*8" && test "A$MPI_OFFSET_KIND2" = "A!" && test $NOF77 = 0 && test $NOF90 = 0 ; then
-   PAC_MPI_OFFSET_KIND
- fi
- #
-  if test "$FORTRAN_MPI_OFFSET" = "integer" && test "A$MPI_OFFSET_KIND2" = "A!" && test $NOF77 = 0 && test $NOF90 = 0 ; then
-   PAC_MPI_OFFSET_KIND_4BYTE
-  fi
-fi
-#
-# Test that we can use the FORTRAN_MPI_OFFSET type.  If the environment
-# is a strict Fortran 90/95 or later compiler, the "integer*8" format
-# may not work.
-if test "$NOF77" = 0 ; then
-    rm -f conftest*
-    ac_cv_f77_offset_type_works=no
-    AC_MSG_CHECKING([that we can use $FORTRAN_MPI_OFFSET to declare MPI_DISPLACMENT_CURRENT])
-    cat >conftest.f <<EOF
-        program main
-        $FORTRAN_MPI_OFFSET j
-        end
-EOF
-    if $F77 -o conftest$EXEEXT conftest.f >>config.log 2>&1 && test -x conftest$EXEEXT ; then
-        ac_cv_f77_offset_type_works=yes
-    fi
-    rm -f conftest*
-    AC_MSG_RESULT($ac_cv_f77_offset_type_works)
-
-    if test "$ac_cv_f77_offset_type_works" != "yes" -a -n "$MPI_OFFSET_KIND_VAL"; then
-        AC_MSG_CHECKING([whether we can use KIND with the selected F77 compiler $F77])
-        ac_cv_f77_allows_offset_kind=no
-        rm -f conftest*
-        cat >conftest.f <<EOF
-        program main
-        integer (kind=$MPI_OFFSET_KIND_VAL) j
-        end
-EOF
-        if $F77 -o conftest$EXEEXT conftest.f >>config.log 2>&1 && test -x conftest$EXEEXT ; then
-            ac_cv_f77_allows_offset_kind=yes
-        fi
-        rm -f conftest*
-        AC_MSG_RESULT($ac_cv_f77_allows_offset_kind)
-        if test "$ac_cv_f77_allows_offset_kind" ; then
-             FORTRAN_MPI_OFFSET="integer (kind=$MPI_OFFSET_KIND_VAL)"
-        else
-             AC_MSG_WARN([Could not find a way to declare an integer type corresponding to MPI_Offset in Fortran.])
-        fi
-    fi
-fi
-
-#
-# check if MPI_Info functions are defined in the MPI implementation
-if test $WITHIN_KNOWN_MPI_IMPL = no ; then
-   PAC_MPI_INFO
-else
-   AC_DEFINE(HAVE_MPI_INFO,1,[Define if MPI Info is available])
-   HAVE_MPI_INFO="#define HAVE_MPI_INFO"
-   MPI_FINFO1="!"
-   MPI_FINFO2="!"
-   MPI_FINFO3="!"
-   MPI_FINFO4="!"
-fi   
-#
-if test -n "$mpi_sgi"; then
-dnl   if test -z "$HAVE_MPI_INFO" ; then
-dnl      PAC_CHECK_MPI_SGI_INFO_NULL  # is MPI_INFO_NULL defined in mpi.h?
-dnl   fi
-   PAC_TEST_MPI_SGI_type_is_contig
-   PAC_TEST_MPI_COMBINERS
-   PAC_TEST_MPI_HAVE_OFFSET_KIND
-fi
-#
-# check if darray and subarray constructors are defined in the MPI 
-# implementation 
-if test $WITHIN_KNOWN_MPI_IMPL = no ; then
-   PAC_MPI_DARRAY_SUBARRAY
-fi
-if test $FROM_MPICH = yes ; then
-   dnl Made this a message instead of a warning because the warning is
-   dnl likely to confuse users.
-   AC_MSG_RESULT([Overriding Array test for MPICH])
-   unset BUILD_MPI_ARRAY
-   AC_DEFINE(HAVE_MPI_DARRAY_SUBARRAY,1,[Define if Darray is available])
-   HAVE_MPI_DARRAY_SUBARRAY="#define HAVE_MPI_DARRAY_SUBARRAY"
-   MPI_FARRAY1="!"
-   MPI_FARRAY2="!"
-   MPI_FARRAY3="!"
-   MPI_FARRAY4="!"
-   MPI_FARRAY5="!"
-   MPI_FARRAY6="!"
-   MPI_FARRAY7="!"
-fi   
-
-# Check to see if weak symbols work correctly
-if test "$enable_weak_symbols" = "yes" ; then
-    # Turn off weak symbols if they aren't available
-    PAC_PROG_C_WEAK_SYMBOLS(,enable_weak_symbols=no)
-fi
-if test "$enable_weak_symbols" = "yes" ; then
-    AC_DEFINE(USE_WEAK_SYMBOLS,1,[Define if weak symbols should be used])
-    # Check for the ability to support multiple weak symbols
-    if test "$pac_cv_prog_c_weak_symbols" = "pragma weak" ; then
-       PAC_PROG_C_MULTIPLE_WEAK_SYMBOLS(AC_DEFINE(HAVE_MULTIPLE_PRAGMA_WEAK,1,[Define if multiple weak symbols may be defined]))
-    fi
-fi
-
-if test "$enable_weak_symbols" = "yes" ; then
-    AC_DEFINE(HAVE_WEAK_SYMBOLS,1,[Define if weak symbols available])
-    HAVE_WEAK_SYMBOLS=1
-else
-    HAVE_WEAK_SYMBOLS=0
-fi
-AC_SUBST(HAVE_WEAK_SYMBOLS)
-
-AM_CONDITIONAL([BUILD_ROMIO_EMBEDDED],[test "$WITHIN_KNOWN_MPI_IMPL" = "yes" ])
-# FIXME need to get this right for non-MPICH builds
-AM_CONDITIONAL([BUILD_MPIO_REQUEST],[false])
-# FIXME need to get this right for non-MPICH builds
-AM_CONDITIONAL([BUILD_MPIO_ERRHAN],[false])
-
-# if we don't have weak symbol support, we must build a separate convenience
-# library in order to provide the "PMPI_" symbols
-# Open MPI: Disable the profile library
-#AM_CONDITIONAL([BUILD_PROFILING_LIB],[test "x$HAVE_WEAK_SYMBOLS" = "x0"])
-AM_CONDITIONAL([BUILD_PROFILING_LIB],[false])
-
-# weird: we have conflated "buid ROMIO's versions of the fortran bindings" and
-# "build ROMIO"s fortran I/O tests". Of course the common situaiton is that we
-# are building as part of MPICH, which builds its own fortran bindings, but we
-# still want tests built
-AM_CONDITIONAL([BUILD_F77_BINDINGS],[test "x$NOF77" != "x1" && test "x$FROM_MPICH" != "xyes"])
-
-AM_CONDITIONAL([BUILD_F77_TESTS],[test "x$NOF77" != "x1"])
-
-#
-# Check whether the MPI Offset type is compatible with struct flock
-AC_MSG_CHECKING([whether struct flock compatible with MPI_Offset])
-AC_TRY_COMPILE([#include <fcntl.h>],
-[struct flock l;
- $MPI_OFFSET_TYPE a=1;
- l.l_start = a; 
- l.l_len   = a;
-],pac_cv_struct_flock_and_mpi_offset=yes,pac_cv_struct_flock_and_mpi_offset=no)
-AC_MSG_RESULT($pac_cv_struct_flock_and_mpi_offset)
-# FIXME: We should look for struct flock64 and the F_SETLK64/F_GETLK64
-# ADIOI_Set_lock could use these instead.
-if test "$pac_cv_struct_flock_and_mpi_offset" = no ; then
-    AC_MSG_CHECKING([whether struct flock compatible with int])
-    AC_TRY_COMPILE([#include <fcntl.h>],
-[struct flock l;
- int a=1;
- l.l_start = a; 
- l.l_len   = a;
-],pac_cv_struct_flock_and_int=yes,pac_cv_struct_flock_and_int=no)
-    AC_MSG_RESULT($pac_cv_struct_flock_and_int)
-    if test "$pac_cv_struct_flock_and_int" = yes ; then
-        AC_DEFINE(NEEDS_INT_CAST_WITH_FLOCK,1,[Define if l_start and l_len data should be cast as int])
-    fi
-    # FIXME.  Solaris header files define off_t as a UNION if 64bit file
-    # sizes are selected.  Gah!
-fi
-
-#
-# if FILE_SYSTEM is not set above, use ufs and nfs as default
-#
-if test -z "$FILE_SYSTEM" ; then
-    FILE_SYSTEM="ufs nfs"
-fi
-
-# no matter what, always build testfs
-FILE_SYSTEM="testfs $FILE_SYSTEM"
-
-# check for valid file system
-if test -n "$FILE_SYSTEM" ; then
-   # if multiple filesystems are passed in, they are '+'-delimited
-   # we could set the IFS to tokenize FILE_SYSTEM, but the FILE_SYSTEM env var
-   # is used in multiple places in the build system: get rid of the '+'s so we
-   # can use the 'for x in $FILE_SYSTEM ...' idiom 
-   FILE_SYSTEM=`echo $FILE_SYSTEM|sed -e 's/\+/ /g'`
-   for x in $FILE_SYSTEM
-   do
-      found=no
-      # We could also do test -d "ad_$y" to test for known file systems
-      # based on having access to the adio code.  Then adding a file 
-      # system would not require changing configure to change known_filesystems
-      for y in $known_filesystems ; do 
-          if test $x = $y ; then
-	      found=yes
-	      eval "file_system_`echo $x`=1"
-	      break
-	  fi
-      done
-      if test "$found" = "no" ; then
-         AC_MSG_WARN([Unknown file system $x... proceeding anyway])
-      fi
-   done
-fi
-
-#############################################
-# This PVFS2 logic is special because it's hard to get it right if it comes
-# before the known_filesystems check loop above.  So we handle it down here,
-# after the loop but before the AM_CONDITIONAL m4 loop.
-#############################################
-#
-# An attempt to "do the right thing" with as little help from the end-user as
-# possible:
-# - if 'with-pvfs2' given, use that to find pvfs2-config.  complain if we
-#   cannot find it, as this is probably what the user would expect
-# - if we can find 'pvfs2-config' in our path, we can use it to set CFLAGS,
-#   LIBS, and LDFLAGS accordingly
-# - as a fallback, use CFLAGS, LIBS, and LDFLAGS passed in by the user
-# - don't do any of this if --with-file-system was given and did not include
-#   'pvfs2': i.e. don't surprise the user with pvfs support.
-
-AC_PATH_PROG(PVFS2_CONFIG, pvfs2-config, notfound, [${with_pvfs2}/bin:$PATH])
-if test $PVFS2_CONFIG != "notfound" ; then
-	if test -n "${with_pvfs2}" -o -n "${file_system_pvfs2}" ; then
-		# the user either told us where pvfs is or asked for it in
-		# --with-file-system (or both)
-		CFLAGS="$CFLAGS $( $PVFS2_CONFIG --cflags)"
-		LIBS="$LIBS $( $PVFS2_CONFIG --libs)"
-		FILE_SYSTEM="pvfs2 $FILE_SYSTEM"
-		file_system_pvfs2=1
-	fi
-fi
-
-if test "$PVFS2_CONFIG" = "notfound" -a -n "$with_pvfs2" ; then
-	AC_MSG_ERROR([pvfs2-config not found in $with_pvfs2])
-fi
-#############################################
-
-
-# Setup an AM_CONDITIONAL named BUILD_AD_FOO for use in each adio's Makefile.mk.
-# This must come *after* the condition becomes valid, in this case after all
-# $file_system_foo variables have been set.
-#
-# If you fiddle with this, please watch the m4 quoting carefully.  m4_foreach
-# expands any macros in the "list" argument exactly once.  The defn bits ensure
-# that any macro names in "fs"'s value will not be expanded, such as if someone
-# is daft enough to "m4_define([ufs],[some crazy value])".
-m4_foreach([fs],[known_filesystems_m4],[
-AM_CONDITIONAL([BUILD_AD_]m4_toupper(defn([fs])),[test x$file_system_]defn([fs])[ = x1])
-])
-
-
-#
-# Print list of configured file systems
-#
-# TODO: REMOVE BAD ONES FROM THE LIST SOMEHOW?
-#
-AC_MSG_CHECKING([configured file systems])
-AC_MSG_RESULT([$FILE_SYSTEM])
-
-
-if test -n "$file_system_nfs" ; then
-    AC_DEFINE(ROMIO_NFS,1,[Define for ROMIO with NFS])
-    AC_MSG_WARN([File locks may not work with NFS.  See the Installation and
-users manual for instructions on testing and if necessary fixing this])
-fi
-
-if test -n "$file_system_panfs"; then
-    # TODO: are there ever any installations with the panfs SDK somewhere else?
-    CPPFLAGS="${CPPFLAGS} -I/opt/panfs/include"
-    AC_CHECK_HEADER(pan_fs_client_cw_mode.h,
-	AC_DEFINE(ROMIO_PANFS,1,[Define for ROMIO with PANFS]),
-	AC_MSG_ERROR([PANFS support requested but cannot find pan_fs_client_cw_mode.h header file])
-    )
-    AC_CHECK_TYPES([pan_fs_client_raidn_encoding_t], , ,
-		   [[#include <pan_fs_client_cw_mode.h>]])
-fi
-AM_CONDITIONAL([BUILD_PANFS_OPEN6], [test "X$ac_cv_type_pan_fs_client_raidn_encoding_t" = "Xyes"])
-
-if test -n "$file_system_ufs"; then
-    AC_DEFINE(ROMIO_UFS,1,[Define for ROMIO with UFS])
-fi
-
-changequote(<<,>>)
-file_system_args=`echo $with_file_system | sed -e 's/^[^:]*//' -e 's/^://'`
-changequote([,])
-
-if test -n "$file_system_gpfs"; then
-    AC_DEFINE(ROMIO_GPFS,1,[Define for ROMIO with GPFS])
-fi
-
-AM_CONDITIONAL([BUILD_AD_BG],[false])
-AM_CONDITIONAL([BUILD_AD_PE],[false])
-if test "$file_system_args" = "BGQ" -a -n "$file_system_gpfs"; then
-    AC_DEFINE(BGQPLATFORM,1,BGQ platform)
-    AM_CONDITIONAL([BUILD_AD_BG],[true])
-    dnl what if anything can make Blue Gene support aio?
-    disable_aio=yes
-fi
-if test "$file_system_args" = "PE" -a -n "$file_system_gpfs"; then
-    AC_DEFINE(PEPLATFORM,1,PE platform)
-    AM_CONDITIONAL([BUILD_AD_PE],[true])
-fi
-
-# echo "with_file_system is :"$with_file_system": file_system_args is :"$file_system_args": FILE_SYSTEM is :"$FILE_SYSTEM":"
-
-if test -n "$file_system_hfs"; then
-    AC_DEFINE(ROMIO_HFS,1,[Define for ROMIO with HFS])
-fi
-if test -n "$file_system_sfs"; then
-    AC_DEFINE(ROMIO_SFS,1,[Define for ROMIO with SFS])
-fi
-
-if test -n "$file_system_pfs"; then
-    AC_DEFINE(ROMIO_PFS,1,[Define for ROMIO with PFS])
-fi
-
-if test -n "$file_system_testfs"; then
-    AC_DEFINE(ROMIO_TESTFS,1,[Define for ROMIO with TESTFS])
-fi
-#
-# Verify presence of lustre/lustre_user.h
-#
-if test -n "$file_system_lustre"; then
-    AC_CHECK_HEADERS(lustre/lustre_user.h,
-        AC_DEFINE(ROMIO_LUSTRE,1,[Define for ROMIO with LUSTRE]),
-        AC_MSG_ERROR([LUSTRE support requested but cannot find lustre/lustre_user.h header file])                                        
-    )
-fi
-
-if test -n "$file_system_xfs"; then
-    AC_DEFINE(ROMIO_XFS,1,[Define for ROMIO with XFS])
-    # Check for memalign value
-    AC_CACHE_CHECK([for memory alignment needed for direct I/O],
-    pac_cv_memalignval,
-    [
-    rm -f confmemalignval
-    rm -f /tmp/romio_tmp.bin
-    AC_TRY_RUN([
-#include <stdio.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <stdio.h>
-int main(int argc, char **argv) { 
-	struct dioattr st;
-	int fd = open("/tmp/romio_tmp.bin", O_RDWR | O_CREAT, 0644);
-	FILE *f=fopen("confmemalignval","w");
-	if (fd == -1) exit(1);
-	if (!f) exit(1);
-	fcntl(fd, F_DIOINFO, &st);
-	fprintf( f, "%u\n", st.d_mem);
-	exit(0);
-	}
-	],
-	pac_cv_memalignval=`cat confmemalignval`,
-	pac_cv_memalignval="unknown",pac_cv_memalignval="unknown"
-    )
-    rm -f confmemalignval
-    rm -f /tmp/romio_tmp.bin
-    ])
-    if test -n "$pac_cv_memalignval" -a "$pac_cv_memalignval" != 0 -a \
-	"$pac_cv_memalignval" != "unknown" ; then
-        CFLAGS="$CFLAGS -DXFS_MEMALIGN=$pac_cv_memalignval"
-    else
-        AC_MSG_RESULT(assuming 128 for memory alignment)
-        CFLAGS="$CFLAGS -DXFS_MEMALIGN=128"
-    fi
-fi
-
-#
-# Verify presence of pvfs.h, issue with int64
-#
-if test -n "$file_system_pvfs"; then
-    CPPFLAGS="$CPPFLAGS $CFLAGS"
-    AC_CHECK_HEADERS(pvfs.h,
-    	AC_DEFINE(ROMIO_PVFS,1,[Define for ROMIO with PVFS]),
-	AC_MSG_ERROR([PVFS support requested but cannot find pvfs.h header file])
-	)
-    # Check for missing int64_t (intel version 8 compiler and -no-gcc
-    # selected)
-    AC_MSG_CHECKING([that pvfs.h can be compiled])
-    AC_TRY_COMPILE([
-#include <pvfs.h>
-        ],[
-        ],pvfs_header_ok=yes,pvfs_header_ok=no
-    )
-    AC_MSG_RESULT($pvfs_header_ok)
-    if test "$pvfs_header_ok" = no -a "$ac_cv_sizeof_long_long" = 8 ; then
-        AC_MSG_CHECKING([if pvfs.h can be compiled if we define int64_t])
-        # Try again, but with int64_t
-        AC_TRY_COMPILE([
-            typedef long long int int64_t;
-#include <pvfs.h>
-            ],[
-            ],
-            pvfs_header_ok="yes with int64 definition"
-        )
-        AC_MSG_RESULT($pvfs_header_ok)
-    fi
-    if test "$pvfs_header_ok" = "yes with int64 definition" ; then
-        AC_DEFINE(ROMIO_PVFS_NEEDS_INT64_DEFINITION,1,[Define if int64_t must be defined for PVFS])
-    fi
-    if test "$pvfs_header_ok" != "no" ; then
-        AC_DEFINE(ROMIO_PVFS,1,[Define for ROMIO with PVFS])
-	AC_DEFINE(HAVE_PVFS_SUPER_MAGIC, 1, [Define if PVFS_SUPER_MAGIC defined.])
-    else
-	AC_MSG_WARN(missing or broken pvfs.h header file; disabling ROMIO PVFS support)
-    fi
-fi
-
-
-
-if test -n "$file_system_zoidfs"; then
-	AC_CHECK_HEADERS(zoidfs.h,
-		AC_DEFINE(ROMIO_ZOIDFS,1,[Define for ROMIO with ZoidFD]),
-		AC_MSG_ERROR([ZoidFS support requested but cannot find zoidfs.h header file])
-	)
-fi
-
-#
-# Verify presence of pvfs2.h
-#
-if test -n "$file_system_pvfs2"; then
-    CPPFLAGS="$CPPFLAGS $CFLAGS"
-    AC_CHECK_HEADERS(pvfs2.h,
-	AC_DEFINE(ROMIO_PVFS2,1,[Define for ROMIO with PVFS2])
-	AC_DEFINE(HAVE_PVFS2_SUPER_MAGIC, 1, [Define if PVFS2_SUPER_MAGIC defined.]),
-        AC_MSG_ERROR([PVFS2 support requested but cannot find pvfs2.h header file])                                        
-    )
-fi
-
-# layout change after pvfs-2.6.3:
-if test -n "$file_system_pvfs2"; then
-    AC_COMPILE_IFELSE([
-        AC_LANG_SOURCE([
-#include <stdlib.h>
-#include "pvfs2.h"
-          int main(int argc, char **argv) { 
-	      PVFS_object_ref ref;
-	      PVFS_sys_attr attr;
-	      PVFS_sys_create(NULL, ref, attr, NULL, NULL, NULL, NULL); 
-	  return 0; }
-       ])],
-       , AC_DEFINE(HAVE_PVFS2_CREATE_WITHOUT_LAYOUT, 1, 
-       		[Define if PVFS_sys_create does not have layout parameter])
-       )
-fi
-
-
-if test -n "$file_system_gridftp"; then
-	AC_DEFINE(ROMIO_GRIDFTP, 1, [Define for ROMIO with gridftp])
-fi
-
-AS_IF([test -n "$file_system_gpfs"],
-    [SYSDEP_INC=-I${prefix}/include], [SYSDEP_INC=])
-
-AS_IF([test -n "$file_system_gpfs"],
-	AC_CHECK_HEADERS([gpfs.h gpfs_fcntl.h])
-	AS_IF([test "$ac_cv_header_gpfs_h" = "yes" -o "$ac_cv_header_gpfs_fcntl_h" = "yes"], [
-	  AC_SEARCH_LIBS([gpfs_fcntl], [gpfs], [],
-	    [AC_MSG_ERROR([Library containing gpfs_fcntl symbols not found])])
-	])
-	)
-# Check for presence and characteristics of async. I/O calls if
-# not disabled.
-
-# Some systems need pthreads to get AIO to work.  However, we don't want
-# to add pthreads just because it is there, as that can cause problems
-# with some implementations of pthreads and compilers (e.g., gcc version 3
-# would fail if there was an int a[100000] on the stack if the application
-# was *linked* with pthreads, but would succeed if the application was
-# *not linked* with pthreads.
-#
-if test "x$disable_aio" = "xno" ; then
-    AC_SEARCH_LIBS(aio_write,aio rt,aio_write_found=yes,aio_write_found=no)
-    if test "$aio_write_found" = no ; then
-        # If not found, try finding pthread_create first, and if
-        # found, try the test again.
-        AC_SEARCH_LIBS(pthread_create,pthread,foundPTHREAD=yes,foundPTHREAD=no)
-        if test "$foundPTHREAD" = yes ; then
-            AC_SEARCH_LIBS(aio_write,aio rt,aio_write_found=yes,aio_write_found=no)
-        fi
-    fi
-fi
-
-if test "x$disable_aio" = "xno" -a -n "$aio_write_found" ; then
-    AC_CHECK_HEADERS([signal.h aio.h sys/aio.h] )
-    if test "$ac_cv_header_aio_h" = "no" -a "$ac_cv_header_sys_aio_h" = "no" ; then
-    	disable_aio=yes
-    fi
-fi
-
-if test "$ac_cv_header_aio_h" = "yes" -o "$ac_cv_header_sys_aio_h" = "yes" -o "x$disable_aio" = "xno"; then
-
-    # Check that aio is available (many systems appear to have aio
-    # either installed improperly or turned off).
-    # The test is the following: if not cross compiling, try to run a 
-    # program that includes a *reference* to aio_write but does not call it
-    # If the libraries are not set up correctly, then this will fail.
-
-    AC_MSG_CHECKING([whether aio routines can be used])
-    # Include aio.h and the aiocb struct (since we'll need these to 
-    # actually use the aio_write interface).  Note that this will 
-    # fail for some pre-POSIX implementations of the aio interface 
-    # (an old IBM interface needs an fd argument as well)
-    AC_TRY_RUN([
-#include <sys/types.h>
-#ifdef HAVE_SIGNAL_H
-#include <signal.h>
-#endif
-#ifdef HAVE_AIO_H
-#include <aio.h>
-#endif
-#ifdef HAVE_SYS_AIO_H
-#include <sys/aio.h>
-#endif
-	int main(int argc, char **argv)
-	{
-            struct aiocb *aiocbp;
-	    if (argc > 10) aio_write(aiocbp);
-	    return 0;
-	}
-	],
-	aio_runs=yes
-	AC_MSG_RESULT(yes),
-	aio_runs=no
-	AC_MSG_RESULT(no),
-	aio_runs=no
-	AC_MSG_RESULT(no: aio routines disabled when cross compiling)
-    )
-    if test "$aio_runs" != "no" ; then
-	AC_DEFINE(ROMIO_HAVE_WORKING_AIO, 1, Define if AIO calls seem to work)
-    fi
-    
-    # now about that old IBM interface...
-    # modern AIO interfaces have the file descriptor in the aiocb structure,
-    # and will set ROMIO_HAVE_STRUCT_AIOCB_WITH_AIO_FILDES.  Old IBM
-    # implementations pass the file descriptor as an argument to aio_write and
-    # aio_read.  AIO still works on these platforms, but we have to test with
-    # two-argument aio_write to avoid a false negative.  no need to worry about
-    # the two-argument vs. one-argument aio_write and aio_read: ROMIO already
-    # uses ROMIO_HAVE_STRUCT_AIOCB_WITH_AIO_FILDES to call aio_write and
-    # aio_read correctly
-
-    AC_MSG_CHECKING([for obsolete two-argument aio_write])
-    AC_TRY_RUN([
-#include <sys/types.h>
-#ifdef HAVE_SIGNAL_H
-#include <signal.h>
-#endif
-#ifdef HAVE_AIO_H
-#include <aio.h>
-#endif
-#ifdef HAVE_SYS_AIO_H
-#include <sys/aio.h>
-#endif
-	int main(int argc, char **argv)
-	{
-		int fd;
-		struct aiocb *aiocbp;
-		if (argc > 10) aio_write(fd, aiocbp);
-		return 0;
-	}
-	],
-	aio_two_arg_write=yes
-	AC_MSG_RESULT(yes),
-	aio_two_arg_write=no 
-	AC_MSG_RESULT(no),
-	aio_two_arg_write=no
-	AC_MSG_RESULT(no: cannot test when cross-compiling)
-    )
-
-    if test "$aio_two_arg_write" != "no" -a "$aio_runs" != "yes" ; then
-    	AC_DEFINE(ROMIO_HAVE_WORKING_AIO, 1, Define if AIO calls seem to work)
-	AC_DEFINE(ROMIO_HAVE_AIO_CALLS_NEED_FILEDES, 1, Define if AIO calls need file descriptor)
-    fi
-
-    AC_MSG_CHECKING([for obsolete two-argument aio_suspend])
-    AC_TRY_RUN([
-#include <sys/types.h>
-#ifdef HAVE_SIGNAL_H
-#include <signal.h>
-#endif
-#ifdef HAVE_AIO_H
-#include <aio.h>
-#endif
-#ifdef HAVE_SYS_AIO_H
-#include <sys/aio.h>
-#endif
-	int main(int argc, char **argv)
-	{
-		struct aiocb *aiocbp;
-		if (argc > 10) aio_suspend(1, &aiocbp);
-		return 0;
-	}
-	],
-	aio_two_arg_suspend=yes
-	AC_MSG_RESULT(yes),
-	aio_two_arg_suspend=no 
-	AC_MSG_RESULT(no),
-	aio_two_arg_suspend=no
-	AC_MSG_RESULT(no: cannot test when cross compiling)
-    )
-
-    if test "$aio_two_arg_suspend" != "no" -a "$aio_runs" != "yes" ; then
-	AC_DEFINE(ROMIO_HAVE_AIO_SUSPEND_TWO_ARGS, 1, Define if aio_suspend needs two arguments)
-    fi
-
-    AC_CHECK_MEMBERS(
-		     [struct aiocb.aio_fildes,
-		      struct aiocb.aio_whence,
-		      struct aiocb.aio_handle,
-		      struct aiocb.aio_reqprio,
-		      struct aiocb.aio_sigevent],
-		      [],
-		      [],
-		      [[#ifdef HAVE_AIO_H
-		       #include <aio.h>
-		       #endif
-		       #ifdef HAVE_SYS_AIO_H
-		       #include <sys/aio.h>
-		       #endif] ]  )
-fi
-
-# the aio-lite package provides an aio facility in case system aio library is
-# buggy or does not perform well.  for example, one fortran test does not pass
-# its non-blocking collective I/O test
-# See http://trac.mpich.org/projects/mpich/ticket/2201 for one such failure.
-
-AC_ARG_WITH([aio-lite],
-	    [AS_HELP_STRING([--with-aio-lite],
-			    [use alternate external aio implementation])],
-	    [with_aiolite=yes],
-	    [])
-AS_IF([test "x$with_aiolite" == xyes],
-      AC_CHECK_LIB([aio-lite], [lio_listio],
-		   [], [], [-lpthread]
-		   )
-      AC_CHECK_HEADERS([aio-lite.h])
-     )
-# End of aio-related tests
-
-# Linux aio library seems to have problems when they are used in our NBC I/O
-# implementation. We let the code know when the host OS is Linux so that the
-# NBC I/O implementation uses blocking I/O operations.
-# See http://trac.mpich.org/projects/mpich/ticket/2201.
-
-# compute canonical system types
-AC_CANONICAL_HOST
-AS_CASE([$host_os],
-        [linux*], AC_DEFINE(ROMIO_RUN_ON_LINUX,1,[Define if run on Linux]))
-# End of OS check
-
-#
-# Check for statfs (many) and specifically f_fstypename field (BSD)
-#
-AC_CHECK_HEADERS(sys/vfs.h sys/param.h sys/mount.h sys/statvfs.h)
-AC_CHECK_FUNCS([statfs])
-AC_MSG_CHECKING([whether struct statfs properly defined])
-AC_TRY_COMPILE([
-#ifdef HAVE_SYS_VFS_H
-#include <sys/vfs.h>
-#endif
-#ifdef HAVE_SYS_STATVFS_H
-#include <sys/statvfs.h>
-#endif
-#ifdef HAVE_SYS_PARAM_H
-#include <sys/param.h>
-#endif
-#ifdef HAVE_SYS_MOUNT_H
-#include <sys/mount.h>
-#endif
-    ],[
-    struct statfs f;
-    ],
-    pac_cv_have_statfs=yes,pac_cv_have_statfs=no
-)
-AC_MSG_RESULT($pac_cv_have_statfs)
-# At this point, we could check for whether defining 
-# __SWORD_TYPE as sizet_t or int/long (size of pointer)
-# would help.  FIXME
-
-if test "$pac_cv_have_statfs" = yes ; then
-    AC_DEFINE(HAVE_STRUCT_STATFS,1,[Define if struct statfs can be compiled])
-fi
-	  
-AC_MSG_CHECKING([for f_fstypename member of statfs structure])
-AC_TRY_COMPILE([
-#include <sys/param.h>
-#include <sys/mount.h>
-#include <string.h>
-    ],[
-    struct statfs f;
-    memset(&f, 0, sizeof(f));
-    strncmp("nfs", f.f_fstypename, 3);
-    ],
-    pac_cv_have_statfs_f_fstypename=yes,
-    pac_cv_have_statfs_f_fstypename=no
-)
-AC_MSG_RESULT($pac_cv_have_statfs_f_fstypename)
-if test $pac_cv_have_statfs_f_fstypename = yes ; then
-    AC_DEFINE(ROMIO_HAVE_STRUCT_STATFS_WITH_F_FSTYPENAME, 1,[Define if statfs has f_fstypename])
-fi
-
-#
-# Check for stat and st_fstype field (NEC SX4)
-#
-AC_CHECK_HEADERS(sys/stat.h sys/types.h unistd.h)
-AC_CHECK_FUNCS(stat,
-    AC_DEFINE(HAVE_STAT, 1, Define if stat function is present)
-    AC_MSG_CHECKING([for st_fstype member of stat structure])
-    AC_TRY_COMPILE([
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-#ifdef HAVE_SYS_STAT_H
-#include <sys/stat.h>
-#endif
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-	],[
-	struct stat s;
-
-	s.st_fstype = NULL;
-	],
-	AC_MSG_RESULT(yes)
-	AC_DEFINE(ROMIO_HAVE_STRUCT_STAT_WITH_ST_FSTYPE, 1, Define if struct stat has a st_fstype member),
-	AC_MSG_RESULT(no)
-    )
-)
-
-#
-# Check for statvfs and f_basetype field (Solaris, Irix, AIX, etc.)
-#
-AC_CHECK_HEADERS(sys/types.h sys/statvfs.h sys/vfs.h)
-AC_CHECK_FUNCS(statvfs,
-    AC_DEFINE(HAVE_STATVFS, 1, Define if statvfs function is present)
-    AC_MSG_CHECKING([for f_basetype member of statvfs structure])
-    AC_TRY_COMPILE([
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-#ifdef HAVE_SYS_VFS_H
-#include <sys/vfs.h>
-#endif
-#ifdef HAVE_SYS_STATVFS_H
-#include <sys/statvfs.h>
-#endif
-	], [[
-	struct statvfs foo;
-
-	foo.f_basetype[0] = 'a';
-	]],
-	AC_MSG_RESULT(yes)
-	AC_DEFINE(ROMIO_HAVE_STRUCT_STATVFS_WITH_F_BASETYPE, 1, defined if struct statvfs has a f_basetype member),
-	AC_MSG_RESULT(no)
-    )
-)
-
-AC_CHECK_TYPE([blksize_t],[],[AC_DEFINE_UNQUOTED([blksize_t],[__blksize_t],[Provide blksize_t if not available]) ], [[
-	       #ifdef HAVE_SYS_TYPES_H
-	       #include <sys/types.h>
-	       #endif
-	       #ifdef HAVE_SYS_STAT_H
-	       #include <sys/stat.h>
-	       #endif
-	       #ifdef HAVE_UNISTD_H
-	       #include <unistd.h>
-	       #endif]] )
-
-#
-# in 2004, solairs defined off_t as a union.  Today (2016) that is not the case
-# and we can simplify this check
-#
-
-AC_SYS_LARGEFILE()
-
-# pread and pwrite are useful to ROMIO: if implemented well, they help us avoid
-# an extra system call.  But --enable-strict or CFLAGS="--std=c99" does not
-# expose this feature.  If we define XOPEN_SOURCE in ROMIO, we cause headaches
-# for some versions of lustre, quota.h, and caddr_t.  see if we need to provide
-# our own pread/pwrite
-
-AC_CHECK_DECLS([pwrite])
-
-
-####################################################################
-# We're about to mess with CC etc.  No more feature tests past here,
-# because we may set CC to something that does not yet exist!
-####################################################################
-
-if test -n "$mpi_mpich"; then
-   if test -z "$arch_SX4" ; then
-      MPIOF_H_INCLUDED=1
-   fi
-   if test "$FROM_MPICH" = no; then
-      AC_DEFINE(NEEDS_MPI_TEST,1,[Define if mpi_test needed])
-      AC_DEFINE(MPICH,1,[Define if using MPICH])
-   fi
-fi
-
-if test -n "$mpi_sgi"; then
-   AC_DEFINE(MPISGI,1,[Define if SGI MPI])
-fi
-
-if test -n "$mpi_lam"; then
-   AC_DEFINE(MPILAM,1,[Define if using LAM/MPI])
-fi
-
-if test -n "$mpi_hp"; then
-   AC_DEFINE(MPIHP,1,[Define if using HP MPI])
-   if test "$NOF77" = 0; then
-      PAC_CHECK_MPIOF_H
-   fi
-fi
-#
-AC_CHECK_FUNCS(strerror)
-if test -z "$srcdir" -o "$srcdir" = "." ; then srcdir="$ROMIO_HOME" ; fi
-AC_SUBST(srcdir)
-
-# preserve these values across a config.status --recheck
-AC_ARG_VAR([master_top_srcdir],[set by the MPICH configure to indicate the MPICH source root])
-AC_ARG_VAR([master_top_builddir],[set by the MPICH configure to indicate the MPICH build root])
-
-# The master_top_srcdir is the location of the source for the building
-# package.  This is used only as part of the MPICH build, including 
-# the documentation targets mandoc, htmldoc, and latexdoc
-if test -z "$master_top_srcdir" ; then
-    if test "$FROM_MPICH" = yes ; then
-        AC_MSG_WARN([Could not determine master_top_srcdir])
-    fi
-fi
-#
-# Get the master builddir (which may be imported from above)
-if test -z "$master_top_builddir" ; then
-    if test "$FROM_MPICH" = yes ; then
-        # this variable is essential to proper build operation
-        AC_MSG_ERROR([Could not determine master_top_srcdir])
-    fi
-    master_top_builddir=`pwd`
-fi
-# Make sure the alternate spelling is used until we clean up all of the code
-master_topbuild_dir=$master_top_builddir
-export master_topbuild_dir
-AC_SUBST(master_topbuild_dir)
-
-# The following definitions are needed within adio/common/status_setb.c
-if test "$FROM_MPICH" = yes ; then
-   AC_DEFINE(ROMIO_INSIDE_MPICH,1,[Define if compiling within MPICH])
-elif test "$FROM_MPILAM" = yes ; then
-   AC_DEFINE(MPILAM,1,[Define if compiling within LAM/MPI])
-fi
-
-if test "$FROM_MPICH" = no ; then
-    if test -z "$LIBNAME"; then
-        LIBNAME="$top_build_dir/lib/libmpio.a"
-    fi
-    #
-    if test ! -d $top_build_dir/lib ; then 
-        mkdir $top_build_dir/lib
-    fi
-else
-    MPILIBNAME=${MPILIBNAME:-mpich}
-    if test -z "$LIBNAME" ; then
-        if test -d "$master_top_builddir/lib" ; then
-            LIBNAME=$master_top_builddir/lib/lib${MPILIBNAME}.a
-        else
-            LIBNAME="$ROMIO_HOME/lib${MPILIBNAME}.a"
-	fi
-    fi
-fi
-if test "$FROM_MPICH" != no ; then
-    # use the error handlers from MPICH
-    MPIO_EXTRA_OBJECTS=
-    MPIO_EXTRA_TMP_POBJECTS=
-    MPIO_EXTRA_REAL_POBJECTS=
-    # Use generalized request to get the multiple-completion routines
-    MPIO_REQOBJECTS=
-fi
-AC_SUBST(MPIO_EXTRA_OBJECTS)
-AC_SUBST(MPIO_EXTRA_TMP_POBJECTS)
-AC_SUBST(MPIO_EXTRA_REAL_POBJECTS)
-#
-# Use DOCTEXT instead of doctext
-AC_CHECK_PROGS(DOCTEXT,doctext,true)
-AC_SUBST(DOCTEXT)
-#
-if test $NOF77 = 1 ; then
-   F77=":"
-else
-   FORTRAN_TEST="fperf fcoll_test fmisc pfcoll_test" 
-fi
-#
-if test $WITHIN_KNOWN_MPI_IMPL = no ; then
-   PAC_TEST_MPI
-   PAC_NEEDS_FINT
-else
-   NEEDS_MPI_FINT=""
-fi
-#
-if test "$MPI_INCLUDE_DIR" = "." ; then
-   ROMIO_INCLUDE="-I../include"
-else 
-   ROMIO_INCLUDE="-I../include -I$MPI_INCLUDE_DIR"
-fi
-#
-TEST_LIBNAME=$LIBNAME
-MPIRUN=mpirun
-if test $FROM_OMPI = yes ; then
-   # Open MPI does have the status set bytes functionality
-
-   AC_DEFINE(HAVE_STATUS_SET_BYTES,1,[Define if have MPIR_Status_set_bytes])
-
-   # Used in the tests/ subdirectory for after ROMIO is built
-
-   TEST_CC=mpicc
-   TEST_F77=mpifort
-   MPIRUN=mpirun
-   MPI_LIB=
-   NOPROFILE=1
-   ROMIO_INCLUDE=
-   USER_CFLAGS=
-   USER_FFLAGS=
-   TEST_LIBNAME=
-   AC_DEFINE(HAVE_MPI_DARRAY_SUBARRAY,1,[Define if Darray is available])
-   HAVE_MPI_DARRAY_SUBARRAY="#define HAVE_MPI_DARRAY_SUBARRAY"
-   # Open MPI: see comments in mpi-io/mpioprof.h
-   AC_DEFINE(MPIO_BUILD_PROFILING, 1, [hack to make ROMIO build without profiling])
-   DEFINE_HAVE_MPI_GREQUEST="#define HAVE_MPI_GREQUEST 1"
-   AC_ARG_ENABLE([grequest-extensions],
-       [AC_HELP_STRING([--enable-grequest-extensions],
-                       [Enable support for Grequest extensions (default: disabled)])])
-   AS_IF([test "x$enable_grequest_extensions" = "xyes"],
-         [DEFINE_HAVE_MPI_GREQUEST_EXTENSIONS="#include \"ompi_grequestx.h\""],
-         [DEFINE_HAVE_MPI_GREQUEST_EXTENSIONS="#undef HAVE_MPI_GREQUEST_EXTENSIONS"])
-
-   AC_DEFINE(HAVE_DECL_MPI_COMBINER_HINDEXED_BLOCK, 1, [Define if MPI library provides HINDEXED_BLOCK datatype])
-   AC_DEFINE(HAVE_MPIIO_CONST, 1, Set if MPI-IO prototypes use const qualifier)
-elif test $FROM_LAM = yes ; then
-   # LAM does have the status set bytes functionality
-   AC_DEFINE(HAVE_STATUS_SET_BYTES,1,[Define if have MPIR_Status_set_bytes])
-        
-   # Used in the tests/ subdirectory for after ROMIO is built
-   TEST_CC=mpicc
-   TEST_F77=mpifort
-   MPIRUN=mpirun 
-   MPI_LIB=
-   ROMIO_INCLUDE=
-   USER_CFLAGS=
-   USER_FFLAGS=
-   TEST_LIBNAME=
-   EXTRA_DIRS="mpi-io/fortran mpi2-other/info mpi2-other/info/fortran mpi2-other/array mpi2-other/array/fortran"
-elif test $FROM_MPICH = yes ; then
-   # For now, separate the mpich from mpich cases
-   MPICH_HOME=`dirname $ROMIO_HOME`
-   MPICH_HOME=`dirname $MPICH_HOME`
-   MPICH_HOME=`dirname $MPICH_HOME`
-   if test -z "$MPI_BIN_DIR" ; then MPI_BIN_DIR=$MPICH_HOME/bin ; fi
-   # No special compiler script.
-   # BUT we need the include paths
-   # CC="$CC -I${use_top_srcdir}/src/include -I${top_build_dir}/src/include"
-   # TEST_CC="$CC"
-   # MPI_LIB="$LIBNAME"
-   # To allow ROMIO to work with the LIBTOOL scripts, we want to 
-   # work directly with the CC, not the mpicc, compiler.
-   # Note that in the "FROM_MPICH" case, the CPPFLAGS and INCLUDES are already
-   # properly set
-   #CC=${top_build_dir}/bin/mpicc
-   #
-   # set the compilers to the ones in MPICH bin directory (master_top_builddir/bin)
-   TEST_CC='$(bindir)/mpicc'
-   TEST_F77='$(bindir)/mpifort'
-   MPI_H_INCLUDE="-I${master_top_builddir}/src/include"
-   ROMIO_INCLUDE=""
-   USER_CFLAGS=""
-   USER_FFLAGS=""
-   TEST_LIBNAME=""
-   MPIRUN='${bindir}/mpiexec'
-   #
-   # Turn off the building of the Fortran interface and the Info routines
-   EXTRA_DIRS=""
-   AC_DEFINE(HAVE_STATUS_SET_BYTES,1,[Define if status_set_bytes available])
-   DEFINE_HAVE_MPI_GREQUEST="#define HAVE_MPI_GREQUEST 1"
-   DEFINE_HAVE_MPI_GREQUEST_EXTENSIONS="#define HAVE_MPI_GREQUEST_EXTENSIONS 1"
-   AC_DEFINE(HAVE_MPIU_FUNCS,1,[Define if MPICH memory tracing macros defined])
-   AC_DEFINE(HAVE_MPIX_H, 1, [])
-   AC_DEFINE(HAVE_MPIIO_CONST, 1, Set if MPI-IO prototypes use const qualifier)
-   AC_DEFINE(HAVE_MPI_TYPE_SIZE_X, 1, [Define if MPI library provides MPI_TYPE_SIZE_X])
-   AC_DEFINE(HAVE_MPI_STATUS_SET_ELEMENTS_X, 1, [Define if MPI library provides MPI_STATUS_SET_ELEMENTS_X])
-   AC_DEFINE(HAVE_DECL_MPI_COMBINER_HINDEXED_BLOCK, 1, [Define if MPI library provides HINDEXED_BLOCK datatype])
-fi
-#
-#
-# feature tests:  we can't test features if building as part of MPICH because
-# we don't yet have an implementation against which we can test
-#
-if test $WITHIN_KNOWN_MPI_IMPL = no ; then
-   PAC_TEST_MPIR_STATUS_SET_BYTES
-   PAC_TEST_MPI_GREQUEST
-   PAC_TEST_MPIU_FUNCS
-   AC_DEFINE(PRINT_ERR_MSG,1,[Define for printing error messages])
-   AC_CHECK_TYPE([MPI_Count],[],[AC_DEFINE_UNQUOTED([MPI_Count],[MPI_Aint],[Define to "MPI_Aint" if MPI does not provide MPI_Count]) ], [[#include <mpi.h>]])
-   PAC_TEST_NEEDS_CONST
-   AC_CHECK_DECLS([MPI_COMBINER_HINDEXED_BLOCK], [], [], [[#include <mpi.h>]])
-   AC_CHECK_FUNCS(MPI_Type_size_x MPI_Status_set_elements_x)
-fi
-#
-if test -z "$TEST_CC" ; then
-   TEST_CC="$CC"
-fi
-if test -z "$TEST_F77" ; then
-   TEST_F77="$F77"
-fi
-#
-AC_CHECK_FUNCS(strdup)
-if test "$ac_cv_func_strdup" = "yes" ; then
-    # Do we need to declare strdup?
-    PAC_FUNC_NEEDS_DECL([#include <string.h>],strdup)
-fi
-AC_CHECK_FUNCS(snprintf)
-if test "$ac_cv_func_snprintf" = "yes" ; then
-    # Do we need to declare snprintf?
-    PAC_FUNC_NEEDS_DECL([#include <stdio.h>],snprintf)
-fi
-AC_CHECK_FUNCS(lstat)
-if test "$ac_cv_func_lstat" = "yes" ; then
-    # Do we need to declare lstat?
-    PAC_FUNC_NEEDS_DECL([#include <unistd.h>
-                         #include <sys/stat.h>],lstat)
-fi
-AC_CHECK_FUNCS(readlink)
-if test "$ac_cv_func_readlink" = "yes" ; then
-    # Do we need to declare readlink?
-    PAC_FUNC_NEEDS_DECL([#include <unistd.h>],readlink)
-fi
-AC_CHECK_FUNCS(fsync)
-if test "$ac_cv_func_fsync" = "yes" ; then
-    # Do we need to declare fsync?
-    PAC_FUNC_NEEDS_DECL([#include <unistd.h>],fsync)
-fi
-AC_CHECK_FUNCS(ftruncate)
-if test "$ac_cv_func_ftruncate" = "yes" ; then
-    # Do we need to declare ftruncate?
-    PAC_FUNC_NEEDS_DECL([#include <unistd.h>],ftruncate)
-fi
-
-AC_CHECK_FUNCS(lseek64)
-if test "$ac_cv_func_lseek64" = "yes" ; then
-    PAC_FUNC_NEEDS_DECL([#include <unistd.h>],lseek64)
-fi
-
-AC_CHECK_FUNCS(usleep)
-if test "$ac_cv_func_usleep" = "yes" ; then
-    PAC_FUNC_NEEDS_DECL([#include <unistd.h>],usleep)
-fi
-
-#
-# Create the directory lists for the Makefile
-FILE_SYS_DIRS=""
-for dir in $FILE_SYSTEM ; do
-    FILE_SYS_DIRS="$FILE_SYS_DIRS adio/ad_$dir"
-done
-# FIXME eliminate FILE_SYS_DIRS and EXTRA_SRC_DIRS
-EXTRA_SRC_DIRS=""
-
-mpio_glue=""
-if test "$FROM_MPICH" = yes -o "${MPI_IMPL}_mpi" = "mpich_mpi"; then
-    mpio_glue=mpich
-elif test "$FROM_OMPI" = yes -o "${MPI_IMPL}_mpi" = "open_mpi_mpi" ; then
-    mpio_glue=openmpi
-else
-    mpio_glue=default
-fi
-
-AM_CONDITIONAL([MPIO_GLUE_MPICH],[test "X$mpio_glue" = "Xmpich"])
-AM_CONDITIONAL([MPIO_GLUE_OPENMPI],[test "X$mpio_glue" = "Xopenmpi"])
-AM_CONDITIONAL([MPIO_GLUE_DEFAULT],[test "X$mpio_glue" = "Xdefault"])
-
-if test "$BUILD_MPI_INFO" = 1 ; then
-    EXTRA_SRC_DIRS="$EXTRA_SRC_DIRS mpi2-other/info"
-    if test "$NOF77" = 0 -a "$FROM_MPICH" != yes ; then
-        EXTRA_SRC_DIRS="$EXTRA_SRC_DIRS mpi2-other/info/fortran"
-    fi
-fi
-if test "$BUILD_MPI_ARRAY" = 1 ; then
-    EXTRA_SRC_DIRS="$EXTRA_SRC_DIRS mpi2-other/array"
-    if test "$NOF77" = 0 -a "$FROM_MPICH" != yes ; then
-        EXTRA_SRC_DIRS="$EXTRA_SRC_DIRS mpi2-other/array/fortran"
-    fi
-fi
-if test "$NOF77" = 0 -a "$FROM_MPICH" != yes ; then
-   EXTRA_SRC_DIRS="$EXTRA_SRC_DIRS mpi-io/fortran"
-fi
-AC_SUBST(EXTRA_SRC_DIRS)
-AC_SUBST(FILE_SYS_DIRS)
-
-#
-CFLAGS="$CFLAGS -DHAVE_ROMIOCONF_H"
-#
-
-if test -n "$MPIOF_H_INCLUDED"; then
-   F77MPIOINC=""
-else 
-   F77MPIOINC="include 'mpiof.h'"
-fi
-
-echo "setting SYSDEP_INC to $SYSDEP_INC"
-AC_SUBST(SYSDEP_INC)
-
-# Open MPI: use the exact same restrict test that we use in the
-# upper-level Open MPI configure script so that we always get the same
-# #define for "restrict" (there are a small number of files that will
-# end up including both ROMIO's romioconf.h and opal_config.h, so we
-# need to #defines to agree).
-AC_C_RESTRICT
-PAC_C_GNU_ATTRIBUTE
-
-# Open MPI: we need libtool
-AM_PROG_LIBTOOL
-
-# Open MPI: setup the AM_CONDITIONALs to build the different adio devices
- m4_foreach([my_fs],
-  [gpfs, gridftp, hfs, lustre, nfs, ntfs, panfs, pfs, pvfs, piofs, pvfs, pvfs2, sfs, testfs, ufs, xfs, zoidfs],
-  [AM_CONDITIONAL(BUILD_[]AS_TR_CPP(my_fs), [test -n "$file_system_]my_fs["])])
-
-# support gcov test coverage information
-PAC_ENABLE_COVERAGE
-
-echo "setting CC to $CC"
-echo "setting F77 to $F77"
-echo "setting TEST_CC to $TEST_CC"
-echo "setting TEST_F77 to $TEST_F77"
-echo "setting CFLAGS to $CFLAGS"
-echo "setting USER_CFLAGS to $USER_CFLAGS"
-echo "setting USER_FFLAGS to $USER_FFLAGS"
-
-# Open MPI: Add on CFLAGS that we figured out up top.  They have
-# makefile macros in them, so we couldn't substitute them until now.
-CFLAGS="$CFLAGS $OMPI_CFLAGS "'-I$(top_builddir)/include'
-#
-# Open MPI - AM doesn't want the following:
-#   VPATH, CC, CPPFLAGS, CFLAGS, AR, RANLIB, F77, MAKE
-AC_SUBST(ARCH)
-AC_SUBST(FILE_SYSTEM)
-#AC_SUBST(CC)
-#AC_SUBST(CPPFLAGS)
-#AC_SUBST(CFLAGS)
-AC_SUBST(USER_CFLAGS)
-AC_SUBST(USER_FFLAGS)
-AC_SUBST(MIPS)
-AC_SUBST(BITS)
-#AC_SUBST(AR)
-AC_SUBST(AR_FLAGS)
-AC_SUBST(MPI_INCLUDE_DIR)
-AC_SUBST(MPI_LIB)
-#AC_SUBST(F77)
-AC_SUBST(NOF77)
-AC_SUBST(NOPROFILE)
-#AC_SUBST(MAKE)
-AC_SUBST(arch_IRIX)
-AC_SUBST(ROMIO_HOME)
-AC_SUBST(LIBNAME)
-AC_SUBST(TEST_LIBNAME)
-AC_SUBST(LL)
-AC_SUBST(F77GETARG)
-AC_SUBST(F77IARGC)
-AC_SUBST(F77MPIOINC)
-AC_SUBST(FTESTDEFINE)
-AC_SUBST(FORTRAN_MPI_OFFSET)
-AC_SUBST(FROM_MPICH)
-AC_SUBST(FROM_LAM)
-AC_SUBST(WITHIN_KNOWN_MPI_IMPL)
-AC_SUBST(NEEDS_MPI_FINT)
-AC_SUBST(HAVE_MPI_INFO)
-AC_SUBST(BUILD_MPI_INFO)
-AC_SUBST(HAVE_MPI_DARRAY_SUBARRAY)
-AC_SUBST(BUILD_MPI_ARRAY)
-AC_SUBST(DEFINE_MPI_OFFSET)
-AC_SUBST(DEFINE_HAVE_MPI_GREQUEST)
-AC_SUBST(DEFINE_HAVE_MPI_GREQUEST_EXTENSIONS)
-AC_SUBST(MPI_OFFSET_TYPE)
-AC_SUBST(MPI_FINFO1)
-AC_SUBST(MPI_FINFO2)
-AC_SUBST(MPI_FINFO3)
-AC_SUBST(MPI_FINFO4)
-AC_SUBST(MPI_FARRAY1)
-AC_SUBST(MPI_FARRAY2)
-AC_SUBST(MPI_FARRAY3)
-AC_SUBST(MPI_FARRAY4)
-AC_SUBST(MPI_FARRAY5)
-AC_SUBST(MPI_FARRAY6)
-AC_SUBST(MPI_FARRAY7)
-AC_SUBST(MPI_OFFSET_KIND1)
-AC_SUBST(MPI_OFFSET_KIND2)
-AC_SUBST(TEST_CC)
-AC_SUBST(TEST_F77)
-AC_SUBST(ROMIO_INCLUDE)
-AC_SUBST(ROMIO_LFLAGS)
-AC_SUBST(ROMIO_TCFLAGS)
-AC_SUBST(ROMIO_TCPPFLAGS)
-AC_SUBST(ROMIO_TFFLAGS)
-AC_SUBST(MPIRUN)
-AC_SUBST(FORTRAN_TEST)
-#dnl
-#dnl Support shared libraries
-#if test -z "$ENABLE_SHLIB" ; then
-#   ENABLE_SHLIB=none
-#fi
-#AC_SUBST(ENABLE_SHLIB)
-#AC_SUBST(CC_SHL)
-#AC_SUBST(LIBTOOL)
-# Open MPI: This is no longer necessary with modern versions of autotools
-# Remove the .a from the library file name (so that we can use .so or
-# other appropriate suffix)
-#SHLIBNAME=`echo $LIBNAME | sed 's/\.a$//'`
-#AC_SUBST(SHLIBNAME)
-#dnl
-#if test ! -d adio ; then mkdir adio ; fi
-#if test ! -d adio/include ; then mkdir adio/include ; fi
-#if test ! -d mpi2-other ; then mkdir mpi2-other ; fi
-#if test ! -d mpi-io ; then mkdir mpi-io ; fi
-#if test ! -d mpi-io/glue ; then mkdir mpi-io/glue ; fi
-
-# Create makefiles for all of the adio devices.  Only the ones that 
-# are active will be called by the top level ROMIO make
-AC_OUTPUT_COMMANDS([chmod 755 util/romioinstall test/runtests])
-AC_CONFIG_FILES([
-    Makefile
-    localdefs
-    test/Makefile
-    test/misc.c
-    test/large_file.c
-    test/runtests
-    test-internal/Makefile
-    util/romioinstall
-    include/mpio.h
-    test/fmisc.f
-    test/fcoll_test.f
-    test/pfcoll_test.f
-    test/fperf.f
-])
-# Open MPI: intentionally skip the following:
-# mpi2-other/info/Makefile
-# mpi2-other/array/Makefile
-# mpi2-other/info/fortran/Makefile
-# mpi2-other/array/fortran/Makefile
-# include/mpiof.h
-AC_OUTPUT
-
-dnl PAC_SUBDIR_CACHE_CLEANUP
-exit 0
diff --git a/3rd-party/romio321/doc/README b/3rd-party/romio321/doc/README
deleted file mode 100644
index b61e73ee204..00000000000
--- a/3rd-party/romio321/doc/README
+++ /dev/null
@@ -1,35 +0,0 @@
-The ROMIO Users Guide is in the file users-guide.ps.gz.
-
-The book ``Using MPI-2: Advanced Features of the Message-Passing
-Interface,'' by William Gropp, Ewing Lusk, and Rajeev Thakur, 
-MIT Press, 1999, provides a tutorial introduction to all aspects of
-MPI-2, including I/O. It has lots of example programs.
-
-The following papers related to ROMIO are available online:
-
-* Rajeev Thakur, William Gropp, and Ewing Lusk, ``Optimizing
-Noncontiguous Accesses in MPI-IO,'' Parallel Computing, (28)1:83--105,
-January 2002. 
-http://www.mcs.anl.gov/~thakur/papers/mpi-io-noncontig.ps
-
-* R. Thakur, W. Gropp, and E. Lusk, ``On Implementing MPI-IO Portably
-and with High Performance,'' in Proc. of the Sixth Workshop on I/O in
-Parallel and Distributed Systems, May 1999. 
-http://www.mcs.anl.gov/~thakur/papers/mpio-impl.ps
-
-* R. Thakur, W. Gropp, and E. Lusk, ``Data Sieving and Collective I/O
-in ROMIO,'' in Proc. of the 7th Symposium on the Frontiers of
-Massively Parallel Computation, February 1999, pp. 182--189.
-http://www.mcs.anl.gov/~thakur/papers/romio-coll.ps 
-
-* R. Thakur, W. Gropp, and E. Lusk, ``A Case for Using MPI's Derived
-Datatypes to Improve I/O Performance,'' in Proc. of SC98: High
-Performance Networking and Computing, November 1998.
-http://www.mcs.anl.gov/~thakur/dtype
-
-* R. Thakur, W. Gropp, and E. Lusk, ``An Abstract-Device Interface for
-Implementing Portable Parallel-I/O Interfaces,'' in Proc. of the 6th
-Symposium on the Frontiers of Massively Parallel Computation, 
-October 1996, pp. 180-187.
-http://www.mcs.anl.gov/~thakur/papers/adio.ps
-
diff --git a/3rd-party/romio321/doc/makepubpage.sh b/3rd-party/romio321/doc/makepubpage.sh
deleted file mode 100755
index 390519e580c..00000000000
--- a/3rd-party/romio321/doc/makepubpage.sh
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/bin/sh
-
-# if you have bibtext2html installed (http://www.lri.fr/~filliatr/bibtex2html/
-# but I know there are other packages by that name), then you can re-generate
-# the "ROMIO publication page"
-# (http://www.mcs.anl.gov/research/projects/romio/pubs.html)
-
-# If you update the command below, please be sure to retain the link to the
-# older papers
-
-WEB_HOST=login3.mcs.anl.gov
-WEB_DIR=/mcs/web/research/projects/romio
-
-bibtex2html -t "Papers using ROMIO" \
-	--header "Please help us keep this list up to date. Contact mpich-discuss@mcs.anl.gov for any corrections or additions. <p>Last updated at $(date).<p><h2>Recent publications</h2><p>" \
-	--footer "<p><h2>Other publications</h2><p>
-	<ul>
-	<li><a href='http://cucis.ece.northwestern.edu/publications/subject.html#Scalable I/O'>Northwestern University CUCIS group</a>
-	<li><a href='http://www.eng.auburn.edu/~wkyu/'>Weikuan Yu</a>
-	<li><a href='http://www.umcs.maine.edu/~dickens/pubs.html'>Phillip Dickens</a>
-	<li><a href=apers-old.html>Older ROMIO papers</a>" \
-	-r -d -both pubs.bib
-
-if [ $? -eq 0 ] ; then
-	scp pubs* ${WEB_HOST}:${WEB_DIR}
-else
-	echo "error running bibtex2html.  website not updated"
-fi
diff --git a/3rd-party/romio321/doc/pubs.bib b/3rd-party/romio321/doc/pubs.bib
deleted file mode 100644
index 5799bc305cf..00000000000
--- a/3rd-party/romio321/doc/pubs.bib
+++ /dev/null
@@ -1,465 +0,0 @@
-@InProceedings{gropp:selfconsist-io,
-  author={William Gropp and Dries Kimpe and Robert B. Ross and Rajeev Thakur and Jesper Larsson Tr\"aff},
-  title={Self-Consistent {MPI-IO} Performance Requirements and Expectations},
-  booktitle={Proc. of the 15th European PVM/MPI Users' Group Meeting (Euro
-                  PVM/MPI 2008)},
-  month={September},
-  year={2008}
-}
-
-@InProceedings{kimpe:log-based,
-  author={D. Kimpe and R. Ross and S. Vandewalle and S. Poedts},
-  title={Transparent Log-Based Data Storage in {MPI-IO} Applications},
-  booktitle={Proc. of the 14th European PVM/MPI Users' Group Meeting (Euro PVM/MPI 2007)},
-  month={September},
-  year={2007}
-}
-
-@InProceedings{latham:gen-req,
-  author={Robert Latham and William Gropp and Robert B. Ross and Rajeev Thakur},
-  title={Extending the {MPI-2} Generalized Request Interface},
-  booktitle={Proc. of the 14th European PVM/MPI Users' Group Meeting (Euro PVM/MPI 2007)},
-  month={September},
-  year={2007},
-  pages={223--232}
-}
-
-@InProceedings{lee:mpiio-l,
-  author={Jonghyun Lee and Robert B. Ross and Scott Atchley and Micah Beck and
-                  and Rajeev Thakur},
-  title={{MPI-IO/L}: Efficient Remote {I/O} for {MPI-IO} via Logistical Networking},
-  booktitle={Proceedings of the 20th IEEE International Parallel and Distributed
-  Processing Symposium (IPDPS 2006)},
-  address={Rhodes Island, Greece},
-  month={April},
-  year={2006}
-}
-
-@InProceedings{ross:atomic,
-  author = {Robert Ross and Robert Latham and William Gropp and Rajeev Thakur
-                  and Brian Toonen},
-  title = {Implementing {MPI-IO} Atomic Mode Without File System Support},
-  booktitle = {Proceedings of CCGrid 2005},
-  month = {May},
-  year = {2005},
-}
-
-@InProceedings{latham:sharedfp,
-  author = {Robert Latham and Robert Ross and Rajeev Thakur},
-  title = {Implementing {MPI-IO} Shared File Pointers Without File
-	System Support},
-  booktitle = {Proceedings of EuroPVM/MPI 2005},
-  month={September},
-  year = {2005},
-}
-
-@article{latham:rma-ops,
-  author = {Latham, Robert and Ross, Robert and Thakur, Rajeev},
-  title = {{Implementing MPI-IO Atomic Mode and Shared File Pointers Using MPI
-  One-Sided Communication}},
-  journal = {International Journal of High Performance Computing Applications},
-  volume = {21},
-  number = {2},
-  pages = {132-143},
-  doi = {10.1177/1094342007077859},
-  year = {2007},
-  abstract = {The ROMIO implementation of the MPI-IO standard provides a
-  portable infrastructure for use on top of a variety of underlying storage
-  targets. These targets vary widely   in their capabilities, and in some cases
-  additional effort is needed within ROMIO to support all MPI-IO semantics. Two
-  aspects of the interface that can be problematic  to implement are MPI-IO
-  atomic mode and the shared file pointer access routines. Atomic mode requires
-  enforcing strict consistency semantics, and   shared file pointer routines
-  require communication and coordination in order to atomically update a shared
-  resource. For some file systems, native locks may be used to implement these
-  features, but not all file systems have lock support. In this work, we
-  describe algorithms for implementing efficient mutex locks using MPI-1 and
-  the one-sided capabilities from MPI-2. We then show how these algorithms may
-  be used   to implement both MPI-IO atomic mode and shared file pointer
-  methods for ROMIO without requiring any features from the underlying file
-  system. We show that these  algorithms can outperform traditional file system
-  lock approaches. Because of the portable nature of these algorithms, they are
-  likely useful in a variety of  situations where distributed locking or
-  coordination is needed in the MPI-2 environment.  },
-  URL = {http://hpc.sagepub.com/cgi/content/abstract/21/2/132},
-  pdf = {papers/latham_rmaops.pdf},
-  eprint = {http://hpc.sagepub.com/cgi/reprint/21/2/132.pdf}
-}
-
-@Article{latham:mpi-io-scalability,
-  author = {Rob Latham and Rob Ross and Rajeev Thakur},
-  title = {The impact of file systems on {MPI-IO} scalability},
-  journal = {Lecture Notes in Computer Science},
-  booktitle = {11th European Parallel Virtual Machine and Message Passing
-  Interface Users Group Meeting; September 19-22, 2004; Budapest, HUNGARY},
-  editor = {Kranzlmuller, D; Kacsuk, P; Dongarra, J},
-  year = {2004},
-  month = {September},
-  volume = {3241},
-  pages = {87--96},
-  institution = {Argonne Natl Lab, 9700 S Cass Ave, Argonne, IL 60439 USA;
-  Argonne Natl Lab, Argonne, IL 60439 USA},
-  publisher = {Springer-Verlag Heidelberg},
-  copyright = {(c)2004 Institute for Scientific Information, Inc.},
-  URL = {http://www.springerlink.com/link.asp?id=m31px2lt90296b62},
-  pdf = {papers/latham:scalable_ops.pdf},
-  keywords = {scalability analysis, MPI-IO, pario-bib},
-  abstract = {As the number of nodes in cluster systems continues to grow,
-  leveraging scalable algorithms in all aspects of such systems becomes key to
-  maintaining performance. While scalable algorithms have been applied
-  successfully in some areas of parallel I/O, many operations are still
-  performed in an uncoordinated manner. In this work we consider, in three file
-  system scenarios, the possibilities for applying scalable algorithms to the
-  many operations that make up the MPI-IO interface. From this evaluation we
-  extract a set of file system characteristics that aid in developing scalable
-  MPI-IO implementations.}
-}
-
-
-
-@InProceedings{thakur:byte-range,
-  author = {Rajeev Thakur and Robert Ross and Robert Latham},
-  title = {Implementing Byte-Range Locks Using {MPI} One-Sided Communication},
-  booktitle = {Proceedings of the 12th European PVM/MPI Users' Group Meeting (Euro PVM/MPI 2005), Recent Advances in Parallel Virtual Machine and Message Passing Interface, Lecture Notes in Computer Science, LNCS 3666, Springer},
-  month = {September},
-  year = {2005},
-  pages = {119-128},
-}
-
-@InProceedings{gropp:io-redundancy,
-  author={William Gropp and Robert B. Ross and Neill Miller},
-  title={Providing Efficient {I/O} Redundancy in {MPI} Environments},
-  booktitle={Proceedings of EuroPVM/MPI 2004},
-  month={September},
-  year={2004}
-}
-
-@InProceedings{lee:rfs,
-  author = {Jonghyun Lee and Xiaosong Ma and Robert B. Ross and Rajeev Thakur
-                  and Marianne Winslett},
-  title={{RFS}: Implementing Efficient and Flexible Remote File Access for
-                  {MPI-IO}},
-  booktitle={Proceedings of Cluster 2004},
-  month={September},
-  year={2004}
-}
-
-@InProceedings{yu:bgl-io,
-  author = {Hao Yu and R. K. Sahoo and C. Howson and George. Almasi and 
-  J. G. Castanos and M. Gupta and  Jose. E. Moreira and J. J. Parker and 
-  T. E. Engelsiepen and Robert Ross and Rajeev Thakur and Robert Latham 
-  and W. D. Gropp}, 
-  title = {High Performance File {I/O} for the {BlueGene/L} Supercomputer},
-  booktitle = {Proceedings of the 12th International Symposium on High-Performance Computer Architecture (HPCA-12)},
-  month = {February},
-  year = {2006},
-  url = {http://www.mcs.anl.gov/~thakur/papers/bgl-io.pdf},
-}
-
-@Article{Hastings:pio-shmem,
-  author = {Andrew B. Hastings and Alok Choudhary},
-  title = {Exploiting Shared Memory to Improve Parallel {I/O} Performance},
-  journal = {Lecture Notes in Computer Science},
-  booktitle = {13th European PVM/MPI User's Group Meeting; September 17-20, 2006; Bonn, Germany},
-  editor = {Mohr, B; Worringen, J; Dongarra, J},
-  year = {2006},
-  month = {September},
-  volume = {4192},
-  pages = {212-221},
-  publisher = {Springer-Verlag Heidelberg},
-}
-
-@Article{seidel:memfs,
-  author = {Jan Seidel and Rudolf Berrendorf and Marcel Birkner and Marc-Andre Hermanns},
-  title = {High-Bandwidth Remote Parallel {I/O} with the Distributed Memory Filesystem {MEMFS}},
-  journal = {Lecture Notes in Computer Science},
-  booktitle = {13th European PVM/MPI User's Group Meeting; September 17-20, 2006; Bonn, Germany},
-  editor = {Mohr, B; Worringen, J; Dongarra, J},
-  year = {2006},
-  month = {September},
-  volume = {4192},
-  pages = {222-229},
-  publisher = {Springer-Verlag Heidelberg},
-}
-
-@InProceedings{calderon:ft-mpiio,
-  author={A. Calderon and F. Garcia-Carballeira and Florin Isaila and Rainer Keller and Alexander Schulz},
-  title={Faul Tolerant File Models for {MPI-IO} Parallel File Systems},
-  booktitle={Proc. of the 14th European PVM/MPI Users' Group Meeting (Euro PVM/MPI 2007)},
-  month={September},
-  year={2007},
-  pages={153-160}
-}
-@InProceedings{Blas:romio-gpfs,
-  author={Francisco Javier Garc{\'i}a Blas and Florin Isaila and Jes{\'u}s Carretero
-  and Thomas Grossmann},
-  title={Implementation and Evaluation of an {MPI-IO} Interface for
-  {GPFS} in {ROMIO}},
-  booktitle={Proc. of the 15th European PVM/MPI Users' Group Meeting (Euro
-                  PVM/MPI 2008)},
-  month={September},
-  year={2008}
-}
-
-@InProceedings{dickens:y-lib,
- author = {Dickens, Phillip M. and Logan, Jeremy},
- title = {Y-lib: a user level library to increase the performance of {MPI-IO} in a lustre file system environment},
- booktitle = {HPDC '09: Proceedings of the 18th ACM international symposium on High performance distributed computing},
- year = {2009},
- isbn = {978-1-60558-587-1},
- pages = {31--38},
- location = {Garching, Germany},
- doi = {http://doi.acm.org/10.1145/1551609.1551617},
- publisher = {ACM},
- address = {New York, NY, USA},
- }
-
-
-% these ROMIO-related papers won't be published until EuroPVM/MPI 2009
-
-@InProceedings{buettner:nonblocking-io,
-    author = {David Buettner and Julian Kunkel and Thomas Ludwig},
-    title = {Using Non-Blocking {I/O} Operations in High Performance Computing to Reduce Execution Times},
-    booktitle = {To be Published in Proc. of the 16th European PVM/MPI User's Group Meeting (Euro PVM/MPI 2009)},
-    month={September},
-    year={2009}
-    }
-
-@InProceedings{sehrish:conflicts,
-    author = {Saba Sehrish and Jun Wang and Rajeev Thakur},
-    title = {Conflict Detection Algorithm to Minimize Locking for {MPI-IO} Atomicity},
-    booktitle = {To be Published in Proc. of the 16th European PVM/MPI User's Group Meeting (Euro PVM/MPI 2009)},
-    month={September},
-    year={2009}
-}
-
-@InProceedings{blas:bg-writeback,
-    author = {Javier Garc{\'i}a Blas and Florin Isaila and Jes{\'u}s Carretero and Robert Latham and Robert Ross},
-    title = {Multiple-level {MPI} file write-back and prefetching for {Blue Gene} systems},
-    booktitle = {To be Published in Proc. of the 16th European PVM/MPI User's Group Meeting (Euro PVM/MPI 2009)},
-    month={September},
-    year={2009}
-}
-
-@InProceedings{isaila:ahpios,
-    author = {Florin Isaila and Javier Garc{\'i}a Blas and Jes{\'u}s Carretero and Wei-keng Liao and Alok Choudhary},
-    title = {{AHPIOS}: An {MPI-based} ad-hoc parallel {I/O} system},
-    booktitle = {14th Intl Conference on Parallel and Distributed Systems. Melbourne, Victoria, Austraila},
-    year={2008}
-}
-
-@InProceedings{blas:viewio,
-    author = {Javier Garc{\'i}a Blas and Florin Isaila and David E. Singh and Jes{\'u}s Carretero},
-    title ={View-based collective {I/O} for {MPI-IO}},
-    booktitle={8th IEEE International Symposium on Cluster Computing and the Grid},
-    year={2008}
-}
-
-@InProceedings{yu:lustre-joining,
-    author = {Weikuan Yu and Jeffrey Vetter and R. Shane Canon and Song Jiang},
-    title = {Exploiting {Lustre} File Joining for Effective Collective {IO}},
-    booktitle = {Seventh IEEE International Symposium on Cluster Computing and the Grid (CCGrid 2007)},
-    month = {May},
-    year = {2007}, 
-}
-
-@InProceedings{yu:opal,
-    author = {Weikuan Yu and Jeffrey Vetter and Shane Canon},
-    title = {{OPAL}: An Open-Source {MPI-IO} Library over {Cray XT}},
-    booktitle = {International Workshop on Storage Network Architecture and Parallel I/O (SNAPI'07)},
-    month={September},
-    year = {2007}
-}
-
-@InProceedings{dickens:lustre-mpi,
-    title = {Towards a High Performance Implementation of {MPI-IO} on the {Lustre} File System},
-    author = { Phillip M. Dickens and Jeremy Logan},
-    booktitle = {Proceedings of GADA'08: Grid computing, high-performAnce and Distributed Applications. Monterrey, Mexico},
-    month = {November},
-    year = {2008}
-}
-
-@article{dickens:lustre-mpi-journal,
-    author = {Phillip Dickens and J. Logan},
-    title = {A High Performance Implementation of {MPI-IO} for a {Lustre} File System Environment},
-    journal = {To appear: Concurrency and Computation: Practice and Experience},
-    year = {2009}
-}
-
-@InProceedings{logan:objectio,
-    author = {J. Logan and Phillip Dickens},
-    title = {Using Object Based Files for High Performance Parallel {I/O}},
-    booktitle = {Proceedings of the IEEE 4th International Workshop on Intelligent Data acquisition and Advanced Computing Systems: Technology and Applications. Dortmund, Germany},
-    month = {September},
-    year = {2007}
-    }
-
-@InProceedings{logan:dynamic-object-io,
-    author = {J. Logan and Phillip Dickens},
-    title = {Dynamic Object Based {I/O}. (Invited Paper)},
-    booktitle = {To appear: IEEE 5th International Workshop on Intelligent Data
-    acquisition and Advanced Computing Systems: Technology and Applications},
-    year = {2009},
-    month = {August}
-}
-
-
-@InProceedings{liao:partitioning,
-    author = {Wei-keng Liao and Alok Choudhary},
-    title = {Dynamically Adapting File Domain Partitioning Methods for Collective {I/O} Based on Underlying Parallel File System Locking Protocols},
-    booktitle = {Proceedings of  International Conference for High Performance Computing, Networking, Storage and Analysis (SC08), Austin, Texas},
-    month={November},
-    year = {2008}
-}
-
-@InProceedings{nisar:io-delegation,
-    author = {Arifa Nisar and Wei-keng Liao and Alok Choudhary},
-    title = {Scaling Parallel {I/O} Performance through {I/O} Delegate and Caching System},
-    booktitle = {Proceedings of International Conference for High Performance Computing, Networking, Storage and Analysis (SC08), Austin, Texas},
-    month={November},
-    year ={2008}
-}
-
-@InProceedings{liao:caching-large-scale,
-    author = { Wei-keng Liao and Avery Ching and Kenin Coloma and Arifa Nisar and Alok Choudhary and Jackie Chen and Ramanan Sankaran and Scott Klasky},
-    title = {Using {MPI} File Caching to Improve Parallel Write Performance for Large-Scale Scientific Applications},
-    booktitle = {Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis (SC 2007), Reno, Nevada},
-    month = {November},
-    year = {2007}
-}
-
-@article{liao:cooperative_caching_mpi_journal,
-    author = {Wei-keng Liao and Kenin Coloma and Alok Choudhary and Lee Ward},
-    title = {Cooperative Client-side File Caching for {MPI} Applications},
-    journal = {International Journal of High Performance Computing Applications},
-    volume = {21},
-    number = {2}, 
-    pages = {144-154},
-    month = {May},
-    year = {2007}
-}
-
-@InProceedings{liao:client_cache_eval,
-    author = {Wei-keng Liao and Avery Ching and Kenin Coloma and Alok Choudhary and Lee Ward},
-    title = {An Implementation and Evaluation of Client-side File Caching for {MPI-IO}}, 
-    booktitle = {Proceedings of the 21st International Parallel and Distributed 
-Processing Symposium (IPDPS), Long Beach, California},
-    month = {March}, 
-    year = {2007}
-}
-
-@article{liao:overlapping_io,
-    author = {Wei-keng Liao and Kenin Coloma and Alok Choudhary and Lee Ward and Eric Russell and Neil Pundit},
-    title = {Scalable Design and Implementations for MPI Parallel Overlapping {I/O}},
-    journal = {{IEEE} Transactions on Parallel and Distributed System},
-    volume = {17},
-    number = {11},
-    pages = {1264-1276},
-    month = {November},
-    year = {2006}
-}
-
-
-@InProceedings{coloma:new_collective_io,
-    author = {Kenin Coloma and Avery Ching and Alok Choudhary and Wei-keng Liao and Robert Ross and Rajeev Thakur and Lee Ward},
-    title = {A New Flexible {MPI} Collective {I/O} Implementation},
-    booktitle = {Proceedings of the IEEE Conference on Cluster Computing (Cluster 2006), Barcelona, Spain},
-    month = {September},
-    year = {2006}
-}
-
-@article{memik:multicollective_io,
-    author = {Gokhan Memik and Mahmut T. Kandemir and Wei-Keng Liao and Alok Choudhary},
-    title = {Multicollective {I/O}: A Technique for Exploiting Inter-file Access  Patterns},
-    journal = {{ACM} Transactions on Storage (TOS)},
-    volume = {2},
-    number = {3},
-    pages = {349-360},
-    month = {August},
-    year = {2006}
-}
-
-@Inbook{liao:atomicity_overlap,
-    author = {Wei-keng Liao and Alok Choudhary and Kenin Coloma and Lee Ward and Eric Russell and Neil Pundit},
-    title = {{MPI} Atomicity and Concurrent Overlapping {I/O}},
-    booktitle = {High Performance Computing: Paradigm and Infrastructure},
-    pages = {203-218},
-    chapter = {10}, 
-    month = {November},
-    year = {2005},
-    publisher = {John Wiley & Sons Inc},
-    note = {ISBN: 978-0-471-65471-1}
-}
-
-@InProceedings{liao:cooperative_write-behind,
-    author = {Wei-keng Liao and Kenin Coloma and Alok Choudhary and Lee Ward},
-    title = {{Cooperative Write-Behind Data Buffering for MPI I/O}},
-    booktitle = {Proceedings of the 12th European Parallel Virtual Machine and Message Passing Interface Conference (EURO PVM/MPI), Sorrento (Naples), Italy},
-    month = {September},
-    year = {2005}  
-    }
-
-@InProceedings{liao:app_aware_caching,
-    author = {Wei-keng Liao and Kenin Coloma and Alok Choudhary and Lee Ward and Eric Russell and Sonja Tideman},
-    title = {{Collective Caching: Application-aware Client-side File Caching}},
-    booktitle = {Proceedings of the 14th IEEE International Symposium on High Performance Distributed Computing (HPDC-14), Research Triangle Park, NC},
-    month = {July},
-    year = {2005}
-    }
-
-@InProceedings{coloma:dache,
-    author = {Kenin Coloma and Alok Choudhary and Wei-keng Liao and Lee Ward and Sonja Tideman},
-    title = {{DAChe: Direct Access Cache System for Parallel I/O}},
-    booktitle = {Proceedings of the International Supercomputer Conference, Heidelberg, Germany},
-    month = {June},
-    year = {2005}
-    }
-
-@InProceedings{coloma:scalable_caching,
-    author = {Kenin Coloma and Alok Choudhary and Wei-keng Liao and Lee Ward and Eric Russell and Neil Pundit},
-    title = {{Scalable High-level Caching for Parallel I/O}},
-    booktitle = {Proceedings of the International Parallel and Distributed Parallel Processing Symposium (IPDPS), New Mexico},
-    month = {April},
-    year = {2004}
-}
-
-@InProceedings{Liao:scalable_atomicity,
-    author = {Wei-keng Liao and Alok Choudhary and Kenin Coloma and George K. Thiruvathukal and Lee Ward and Eric Russell and Neil Pundit},
-    title = {{Scalable Implementations of MPI Atomicity for Concurrent Overlapping I/O}},
-    booktitle = {Proceedings of the International Conference on Parallel Processing (ICPP), Kaohsiung, Taiwan},
-    month = {October},
-    year = {2003}
-    }
-
-@InProceedings{ching:noncontig,
-    author = {Avery Ching and Alok Choudhary and Kenin Coloma and Wei-keng Liao and Robert Ross and William Gropp},
-    title = {{Noncontiguous I/O Accesses Through MPI-IO}},
-    booktitle = {Proceedings of the International Symposium on Cluster Computing and the Grid (CCGrid), Tokyo, Japan},
-    month = {May},
-    year = {2003}
-}
-
-@Article{chaarawi:collective_writes,
-	author =  {Mohamad Chaarawi and Suneet Chandok and Edgar Gabriel},
-	title = {{Performance Evaluation of Collective Write Algorithms in MPI I/O}},
-	journal = {Lecture Notes in Computer Science},
-	booktitle = {International Conference on Computer Science (ICCS) 2009},
-	month = {May},
-	year = {2009},
-	volume = {5544},
-	pages = {185-194},
-	publisher = {Springer Berlin / Heidelberg},
-	doi = {10.1007/978-3-642-01970-8_19}
-}
-@Article{kulkarni::shared_file_pointers,
-	author = {Ketan Kulkarni and Edgar Gabriel},
-	title = {{Evaluating Algorithms for Shared File Pointer Operations in MPI I/O}},
-	journal = {Lecture Notes in Computer Science},
-	booktitle = {International Conference on Computer Science (ICCS) 2009},
-	month = {May},
-	year = {2009},
-	volume = {5544},
-	pages = {280-289},
-	publisher = {Springer Berlin / Heidelberg},
-	doi = {10.1007/978-3-642-01970-8_28}
-}
diff --git a/3rd-party/romio321/doc/romio.bib b/3rd-party/romio321/doc/romio.bib
deleted file mode 100644
index bb94a2d2ea1..00000000000
--- a/3rd-party/romio321/doc/romio.bib
+++ /dev/null
@@ -1,92 +0,0 @@
-@Book{grop99a,
-  author =       {William Gropp and Ewing Lusk and Rajeev Thakur},
-  title =        {{Using MPI-2: Advanced Features of the Message-Passing Interface}},
-  publisher =    {MIT Press},
-  year =         1999,
-  address =      {Cambridge, MA}
-}
-
-@Misc{mpi97a,
-  author       = "{Message Passing Interface Forum}",
-  title = 	 "{{MPI-2}: Extensions to the Message-Passing Interface}", 
-  note =	 "{\tt http://www.mpi-forum.org/docs/docs.html}",
-  year = {July 1997}
-}
-
-@InProceedings{thak96e,
-  author = {Rajeev Thakur and William Gropp and Ewing Lusk},
-  title = {An Abstract-Device Interface for Implementing Portable 
-             Parallel-{I/O} Interfaces},
-  booktitle = {Proceedings of the 6th Symposium on the Frontiers of
-               Massively Parallel Computation}, 
-  publisher = {IEEE Computer Society Press},
-  pages = {180--187},
-  month = {October},
-  year = {1996}
-}
-
-@InProceedings{thak99b,
-  author = {Rajeev Thakur and William Gropp and Ewing Lusk},
-  title = {On Implementing {MPI-IO} Portably and with High Performance},
-  booktitle = {Proceedings of the 6th Workshop on I/O in Parallel and 
-  Distributed Systems}, 
-  pages = {23--32},
-  year = {1999},
-  month = {May},
-  publisher = {ACM Press}
-}
-
-@InProceedings{bordawekar:primitives,
-  author = {Rajesh Bordawekar and Juan Miguel {del Rosario} and Alok
-  Choudhary},
-  title = {Design and Evaluation of Primitives for Parallel {I/O}},
-  booktitle = {Proceedings of Supercomputing '93},
-  year = {1993},
-  pages = {452--461},
-  publisher = {IEEE Computer Society Press},
-  address = {Portland, OR},
-  URL = {ftp://erc.cat.syr.edu/ece/choudhary/PASSION/sc93.ps.Z},
-  keywords = {parallel I/O, pario-bib},
-  abstract = {In this paper, we show that the performance of parallel file
-  systems can vary greatly as a function of the selected data distributions,
-  and that some data distributions can not be supported. Also, we describe how
-  the parallel language extensions, though simplifying the programming, do not
-  address the performance problems found in parallel file systems. \par We have
-  devised an alternative scheme for conducting parallel I/O - the Two-Phase
-  Access Strategy - which guarantees higher and more consistent performance
-  over a wider spectrum of data distributions. We have designed and implemented
-  runtime primitives that make use of the two-phase access strategy to conduct
-  parallel I/O, and facilitate the programming of parallel I/O operations. We
-  describe these primitives in detail and provide performance results which
-  show that I/O access rates are improved by up to several orders of magnitude.
-  Further, we show that the variation in performance over various data
-  distributions is restricted to within a factor of 2 of the best access
-  rate.},
-  comment = {Much of this is the same as delrosario:two-phase, except for
-  section~4 where they describe their actual run-time library of primitives,
-  with a little bit about how it works. It's not clear, for example, how their
-  meta-data structures are distributed across the machine. They also do not
-  describe their methods for the data redistribution.}
-}
-
-@TechReport{choudhary:passion,
-  author = {Alok Choudhary and Rajesh Bordawekar and Michael Harry and Rakesh
-  Krishnaiyer and Ravi Ponnusamy and Tarvinder Singh and Rajeev Thakur},
-  title = {{PASSION:} Parallel And Scalable Software for Input-Output},
-  year = {1994},
-  month = {September},
-  number = {SCCS-636},
-  institution = {ECE Dept., NPAC and CASE Center, Syracuse University},
-  later = {thakur:jpassion},
-  URL = {ftp://erc.cat.syr.edu/ece/choudhary/PASSION/passion_report.ps.Z},
-  keywords = {parallel I/O, out-of-core, pario-bib},
-  comment = {This TR overviews the PASSION project, and all its components:
-  two-phase access, out-of-core support for structured and unstructured
-  problems, data sieving, prefetching, caching, compiler and language support,
-  file system support, virtual parallel file system, and parallel pipes. They
-  reference many of their related papers in an extensive bibliography. See also
-  singh:adopt, jadav:ioschedule, thakur:passion, thakur:runtime,
-  bordawekar:efficient, thakur:out-of-core, delrosario:prospects,
-  delrosario:two-phase, bordawekar:primitives, bordawekar:delta-fs.}
-}
-
diff --git a/3rd-party/romio321/doc/source-guide.tex b/3rd-party/romio321/doc/source-guide.tex
deleted file mode 100644
index 96fa7ca58d9..00000000000
--- a/3rd-party/romio321/doc/source-guide.tex
+++ /dev/null
@@ -1,494 +0,0 @@
-% \documentstyle[11pt,psfig]{article}
-\documentclass[11pt]{article}
-\hoffset=-.7in
-\voffset=-.6in
-\textwidth=6.5in
-\textheight=8.5in
-
-\begin{document}
-\vspace*{-1in}
-\thispagestyle{empty}
-\begin{center}
-ARGONNE NATIONAL LABORATORY \\
-9700 South Cass Avenue \\
-Argonne, IL 60439
-\end{center}
-\vskip .5 in
-
-\begin{center}
-\rule{1.75in}{.01in} \\
-\vspace{.1in}
-
-ANL/MCS-TM-XXX \\
-
-\rule{1.75in}{.01in} \\
-
-\vskip 1.3 in
-{\Large\bf A Guide to the ROMIO MPI-IO Implementation } \\
-by \\ [2ex]
-{\large\it Robert Ross, Robert Latham, and Rajeev Thakur}
-\vspace{1in}
-
-Mathematics and Computer Science Division
-
-\bigskip
-
-Technical Memorandum No.\ XXX
-
-
-% \vspace{1.4in}
-% Revised May 2004
-
-\end{center}
-
-\vfill
-
-{\small
-\noindent
-This work was supported by the Mathematical, Information, and
-Computational Sciences Division subprogram of the Office of Advanced
-Scientific Computing Research, U.S. Department of Energy, under
-Contract W-31-109-Eng-38; and by the Scalable I/O Initiative, a
-multiagency project funded by the Defense Advanced Research Projects
-Agency (Contract DABT63-94-C-0049), the Department of Energy, the
-National Aeronautics and Space Administration, and the National
-Science Foundation.}
-
-\newpage
-
-
-%%  Line Spacing (e.g., \ls{1} for single, \ls{2} for double, even \ls{1.5})
-%%
-
-\newcommand{\ls}[1]
-   {\dimen0=\fontdimen6\the\font 
-    \lineskip=#1\dimen0
-    \advance\lineskip.5\fontdimen5\the\font
-    \advance\lineskip-\dimen0
-    \lineskiplimit=.9\lineskip
-    \baselineskip=\lineskip
-    \advance\baselineskip\dimen0
-    \normallineskip\lineskip
-    \normallineskiplimit\lineskiplimit
-    \normalbaselineskip\baselineskip
-    \ignorespaces
-   }
-\renewcommand{\baselinestretch}{1}
-\newcommand {\ix} {\hspace*{2em}}
-\newcommand {\mc} {\multicolumn}
-
-
-\tableofcontents
-\thispagestyle{empty}
-\newpage
-
-\pagenumbering{arabic}
-\setcounter{page}{1}
-\begin{center}
-{\bf Users Guide for ROMIO:  A High-Performance,\\[1ex]
-Portable MPI-IO Implementation} \\ [2ex]
-by \\ [2ex]
-{\it Rajeev Thakur, Robert Ross, Ewing Lusk, and William Gropp}
-
-\end{center}
-\addcontentsline{toc}{section}{Abstract}
-\begin{abstract}
-\noindent
-ROMIO is a high-performance, portable implementation of MPI-IO (the
-I/O chapter in the \mbox{MPI Standard}).
-This document describes the internals of the ROMIO implementation.
-\end{abstract}
-
-\section{Introduction}
-
-The ROMIO MPI-IO implementation, originally written by Rajeev Thakur, has been
-in existence since XXX.
-
-... Discussion of the evolution of ROMIO ...
-
-Architecturally, ROMIO is broken up into three layers: a layer implementing
-the MPI I/O routines in terms of an abstract device for I/O (ADIO), a layer of
-common code implementing a subset of the ADIO interface, and a set of storage
-system specific functions that complete the ADIO implementation in terms of
-that storage type.  These three layers work together to provide I/O support
-for MPI applications.
-
-In this document we will discuss the details of the ROMIO implementation,
-including the major components, how those components are implemented, and
-where those components are located in the ROMIO source tree.
-
-\section{The Directory Structure}
-
-The ROMIO directory structure consists of two main branches, the MPI-IO branch
-(mpi-io) and the ADIO branch (adio).  The MPI-IO branch contains code that
-implements the functions defined in the MPI specification for I/O, such as
-MPI\_File\_open.  These functions are then written in terms of other functions
-that provide an abstract interface to I/O resources, the ADIO functions.
-There is an additional glue subdirectory in the MPI-IO branch that defines
-functions related to the MPI implementation as a whole, such as how to
-allocate MPI\_File structures and how to report errors.
-
-Code for the ADIO functions is located under the ADIO branch.  This code is
-responsible for performing I/O operations on whatever underlying storage is
-available.  There are two categories of directories in this branch.  The first
-is the common directory.  This directory contains two distinct types of
-source: source that is used by all ADIO implementations and source that is
-common across many ADIO implementations.  This distinction will become more
-apparent when we discuss file system implementations.
-
-The second category of directory in the ADIO branch is the file system
-specific directory (e.g. ad\_ufs, ad\_pvfs2).  These directories provide code
-that is specific to a particular file system type and is only built if that
-file system type is selected at configure time.
-
-\section{The Configure Process}
-
-... What can be specified, AIO stuff, where romioconf exists, how to add
-another Makefile.in into the list.
-
-\section{File System Implementations}
-
-Each file system implementation exists in its own subdirectory under the adio
-directory in the source tree.  Each of these subdirectories must contain at
-least two files, a Makefile.in (describing how to build the code in the
-directory) and a C source file describing the mapping of ADIO operations to C
-functions.
-
-The common practice is to name this file based on the name of the ADIO
-implementation.  In the ad\_ufs implementation this file is called ad\_ufs.c,
-and contains the following:
-
-\begin{verbatim}
-struct ADIOI_Fns_struct ADIO_UFS_operations = {
-	ADIOI_UFS_Open, /* Open */
-	ADIOI_GEN_ReadContig, /* ReadContig */
-	ADIOI_GEN_WriteContig, /* WriteContig */
-	ADIOI_GEN_ReadStridedColl, /* ReadStridedColl */
-	ADIOI_GEN_WriteStridedColl, /* WriteStridedColl */
-	ADIOI_GEN_SeekIndividual, /* SeekIndividual */
-	ADIOI_GEN_Fcntl, /* Fcntl */
-	ADIOI_GEN_SetInfo, /* SetInfo */
-	ADIOI_GEN_ReadStrided, /* ReadStrided */
-	ADIOI_GEN_WriteStrided, /* WriteStrided */
-	ADIOI_GEN_Close, /* Close */
-	ADIOI_GEN_IreadContig, /* IreadContig */
-	ADIOI_GEN_IwriteContig, /* IwriteContig */
-	ADIOI_GEN_IODone, /* ReadDone */
-	ADIOI_GEN_IODone, /* WriteDone */
-	ADIOI_GEN_IOComplete, /* ReadComplete */
-	ADIOI_GEN_IOComplete, /* WriteComplete */
-	ADIOI_GEN_IreadStrided, /* IreadStrided */
-	ADIOI_GEN_IwriteStrided, /* IwriteStrided */
-	ADIOI_GEN_Flush, /* Flush */
-	ADIOI_GEN_Resize, /* Resize */
-	ADIOI_GEN_Delete, /* Delete */
-};
-\end{verbatim}
-
-The ADIOI\_Fns\_struct structure is defined in adio/include/adioi.h.  This
-structure holds pointers to appropriate functions for a given file system
-type.  "Generic" functions, defined in adio/common, are denoted by the
-"ADIOI\_GEN" prefix, while file system specific functions use a file system
-related prefix.  In this example, the only file system specific function is
-ADIOI\_UFS\_Open.  All other operations use the generic versions.
-
-Typically a third file, a header with file system specific defines and
-includes, is also provided and named based on the name of the ADIO
-implementation (e.g. ad\_ufs.h).
-
-Because the UFS implementation provides its own open function, that code must be provided in the ad\_ufs subdirectory.  That function is implemented in adio/ad\_ufs/ad\_ufs\_open.c.
-
-\section{Generic Functions}
-
-As we saw in the discussion above, generic ADIO function implementations are
-used to minimize the amount of code in the ROMIO tree by sharing common
-functionality between ADIO implementations.  As the ROMIO implementation has
-grown, a few categories of generic implementations have developed.  At this
-time, these are all lumped into the adio/common subdirectory together, which
-can be confusing.
-
-The easiest category of generic functions to understand is the ones that
-implement functionality in terms of some other ADIO function.
-ADIOI\_GEN\_ReadStridedColl is a good example of this type of function and is
-implemented in adio/common/ad\_read\_coll.c.  This function implements
-collective read operations (e.g. MPI\_File\_read\_at\_all).  We will discuss how
-it works later in this document, but for the time being it is sufficient to
-note that it is written in terms of ADIO ReadStrided or ReadContig calls.
-
-A second category of generic functions are ones that implement functionality
-in terms of POSIX I/O calls.  ADIOI\_GEN\_ReadContig (adio/common/ad\_read.c) is
-a good example of this type of function.  These "generic" functions are the
-result of a large number of ADIO implementations that are largely POSIX I/O
-based, such as the UFS, XFS, and PANFS implementations.  We have discussed
-moving these functions into a separate common/posix subdirectory and renaming
-them with ADIOI\_POSIX prefixes, but this has not been done as of the writing
-of this document.
-
-The next category of generic functions holds functions that do not actually
-require I/O at all.  ADIOI\_GEN\_SeekIndividual (adio/common/ad\_seek.c) is a
-good example of this.  Since we don't need to actually perform I/O at seek
-time, we can just update local variables at each process.  In fact, one could
-argue that we no longer need the ADIO SeekIndividual function at all - all the
-ADIO implementations simply use this generic version (with the exception of
-TESTFS, which prints the value as well).
-
-The next category of generic functions are the "FAKE" functions (e.g.
-ADIOI\_FAKE\_IODone implemented in adio/common/ad\_done\_fake.c).  These functions
-are all related to asynchronous I/O (AIO) operations.  These implement the AIO
-operations in terms of blocking operations - in other words, they follow the
-standard but do not allow for overlap of I/O and computation or communication.
-These are used in cases where AIO support is otherwise unavailable or
-unimplemented.
-
-The final category of generic functions are the "na��ve" functions (e.g.
-ADIOI\_GEN\_WriteStrided\_naive in adio/common/ad\_write\_str\_naive.c).  These
-functions avoid the use of certain optimizations, such as data sieving.
-
-Other Things in adio/common
-
-... what else is in there?
-
-\subsection{Calling ADIO Functions}
-
-Throughout the code you will see calls to functions such as ADIO\_ReadContig.
-There is no such function - this is actually a macro defined in
-adio/include/adioi.h that calls the particular function out of the correct
-ADIOI\_Fns\_struct for the file being accessed.  This is done for convenience.
-
-Exceptions!!!  ADIO\_Open, ADIO\_Close...
-
-\section{ROMIO Implementation Details}
-
-The ROMIO Implementation relies on some basic concepts in order to operate and
-to optimize I/O access.  In this section we will discuss these concepts and
-how they are implemented within ROMIO.  Before we do that though, we will
-discuss the core data structure of ROMIO, the ADIO\_File structure.
-
-\subsection{ADIO\_File}
-
-... discussion ...
-
-\subsection{I/O Aggregation and Aggregators}
-
-When performing collective I/O operations, it is often to our advantage to
-combine operations or eliminate redundant operations altogether.  We call this
-combining process "aggregation", and processes that perform these combined
-operations aggregators.
-
-Aggregators are defined at the time the file is opened.  A collection of MPI
-hints can be used to tune what processes become aggregators for a given file
-(see ROMIO User's Guide).  The aggregators will then interact with the file
-system during collective operations.
-
-Note that it is possible to implement a system where ALL I/O operations pass
-exclusively through aggregators, including independent I/O operations from
-non-aggregators.  However, this would require a guarantee of progress from the
-aggregators that for portability would mean adding a thread to manage I/O.  We
-have chosen not to pursue this path at this time, so independent operations
-continue to be serviced by the process making the call.
-
-... how implemented ...
-
-Rank 0 in the communicator opening a file \emph{always} processes the
-cb\_config\_list hint using ADIOI\_cb\_config\_list\_parse.  A previous call to
-ADIOI\_cb\_gather\_name\_array had collected the processor names from all hosts
-into an array that is cached on the communicator (so we don't have to gather
-it more than once).  This creates an ordered array of ranks (relative to the
-communicator used to open the file) that will be aggregators.  This array is
-distributed to all processes using ADIOI\_cb\_bcast\_rank\_map.  Aggregators are
-referenced by their rank in the communicator used to open the file.  These
-ranks are stored in fd->hints->ranklist[].
-
-Note that this could be a big list for very large runs.  If we were to
-restrict aggregators to a rank order subset, we could use a bitfield instead.
-
-If the user specified hints and met conditions for deferred open, then a
-separate communicator is also set up (fd->agg\_comm) that contains all the
-aggregators, in order of their original ranks (not their order in the rank
-list).  Otherwise this communicator is set to MPI\_COMM\_NULL, and in any case
-it is set to this for non-aggregators.  This communicator is currently only
-used at ADIO\_Close (adio/common/ad\_close.c), but could be useful in two-phase
-I/O as well (discussed later).
-
-
-\subsection{Deferred Open}
-
-We do not always want all processes to attempt to actually open a file when
-MPI\_File\_open is called.  We might want to avoid this open because in fact
-some processes (non-aggregators) cannot access the file at all and would get
-an error, or we might want to avoid this open to avoid a storm of system calls
-hitting the file system all at once.  In either case, ROMIO implements a
-"deferred open" mode that allows some processes to avoid opening the file
-until such time as they perform an independent I/O operation on the file (see
-ROMIO User's Guide).
-
-Deferred open has a broad impact on the ROMIO implementation, because with its
-addition there are now many places where we must first check to see if we have
-called the file system specific ADIO Open call before performing I/O.  This
-impact is limited to the MPI-IO layer by semantically guaranteeing the FS ADIO
-Open call has been made by the process prior to calling a read or write
-function.
-
-... how implemented ...
-
-\subsection{Two-Phase I/O}
-
-Two-Phase I/O is a technique for increasing the efficiency of I/O operations
-by reordering data between processes, either before writes, or after reads.
-
-ROMIO implements two-phase I/O as part of the generic implementations of
-ADIO\_WriteStridedColl and ADIO\_ReadStridedColl.  These implementations in turn
-rely heavily on the aggregation code to determine what processes will actually
-perform I/O on behalf of the application as a whole.
-
-
-
-\subsection{Data Sieving}
-
-Data sieving is a single-process technique for reducing the number of I/O
-operations used to service a MPI read or write operation by accessing a
-contiguous region of the file that contains more than one desired region at
-once.  Because often I/O operations require data movement across the network,
-this is usually a more efficient way to access data.
-
-Data sieving is implemented in the common strided I/O routines
-(adio/common/ad\_write\_str.c and adio/common/ad\_read\_str.c).  These functions
-use the contig read and write routines to perform actual I/O.  In the case of
-a write operation, a read/modify/write sequence is used.  In that case, as
-well as in the atomic mode case, locking is required on the region.  Some of
-the ADIO implementations do not currently support locking, and in those cases
-it would be erroneous to use the generic strided I/O routines.
-
-\subsection{Shared File Pointers}
-
-Because no file systems supported by ROMIO currently support a shared file
-pointer mode, ROMIO must implement shared file pointers under the covers on
-its own.
-
-Currently ROMIO implements shared file pointers by storing the file pointer
-value in a separate file...
-
-Note that the ROMIO team has devised a portable method for implementing shared
-file pointers using only MPI functions.  However, this method has
-not yet been implemented in ROMIO.
-
-file name is selected at end of mpi-io/open.c.
-
-\subsection{Error Handling}
-
-\subsection{MPI and MPIO Requests}
-
-\section*{Appendix A: ADIO Functions and Semantics}
-
-ADIOI\_Open(ADIO\_File fd, int *error\_code)
-
-Open is used in a strange way in ROMIO, as described previously.
-
-The Open function is used to perform whatever operations are necessary prior
-to actually accessing a file using read or write.  The file name for the file
-is stored in fd->filename prior to Open being called.
-
-Note that when deferred open is in effect, all processes may not immediately
-call Open at MPI\_File\_open time, but instead call open if they perform
-independent I/O.  This can result in somewhat unusual error returns to
-processes (e.g. learning that a file is not accessible at write time).
-
-ADIOI\_ReadContig(ADIO\_File fd, void *buf, int count, MPI\_Datatype datatype,
-int file\_ptr\_type, ADIO\_Offset offset, ADIO\_Status *status, int *error\_code)
-
-ReadContig is used to read a contiguous region from a file into a contiguous
-buffer.  The datatype (which refers to the buffer) can be assumed to be
-contiguous.  The offset is in bytes and is an absolute offset if
-ADIO\_EXPLICIT\_OFFSET was passed as the file\_ptr\_type or relative to the
-current individual file pointer if ADIO\_INDIVIDUAL was passed as
-file\_ptr\_type.  Open has been called by this process prior to the call to
-ReadContig.  There is no guarantee that any other processes will call this
-function at the same time.
-
-ADIOI\_WriteContig(ADIO\_File fd, void *buf, int count, MPI\_Datatype datatype,
-int file\_ptr\_type, ADIO\_Offset offset, ADIO\_Status *status, int *error\_code)
-
-WriteContig is used to write a contiguous region to a file from a contiguous
-buffer.  The datatype (which refers to the buffer) can be assumed to be
-contiguous.  The offset is in bytes and is an absolute offset if
-ADIO\_EXPLICIT\_OFFSET was passed as the file\_ptr\_type or relative to the
-current individual file pointer if ADIO\_INDIVIDUAL was passed as
-file\_ptr\_type.  Open has been called by this process prior to the call to
-WriteContig.  There is no guarantee that any other processes will call this
-function at the same time.
-
-ADIOI\_ReadStridedColl
-
-ADIOI\_WriteStridedColl
-
-ADIOI\_SeekIndividual
-
-ADIOI\_Fcntl
-
-ADIOI\_SetInfo
-
-ADIOI\_ReadStrided
-
-ADIOI\_WriteStrided
-
-ADIOI\_Close(ADIO\_File fd, int *error\_code)
-
-Close is responsible for releasing any resources associated with an open file.
-It is called on all processes that called the corresponding ADIOI Open, which
-might not be all the processes that opened the file (due to deferred open).
-Thus it is not safe to perform collective communication among all processes in
-the communicator during Close, although collective communication between
-aggregators would be safe (if desired).
-
-For performance reasons ROMIO does not guarantee that all file data is written
-to "storage" at MPI\_File\_close, instead only performing synchronization
-operations at MPI\_File\_sync time.  As a result, our Close implementations do
-not typically call a sync.  However, any locally cached data, if any, should
-be passed on to the underlying storage system at this time.
-
-Note that ADIOI\_GEN\_Close is implemented in adio/common/adi\_close.c;
-ad\_close.c implements ADIO\_Close, which is called by all processes that opened
-the file.
-
-ADIOI\_IreadContig
-
-ADIOI\_IwriteContig
-
-ADIOI\_ReadDone
-
-ADIOI\_WriteDone
-
-ADIOI\_ReadComplete
-
-ADIOI\_WriteComplete
-
-ADIOI\_IreadStrided
-
-ADIOI\_IwriteStrided
-
-ADIOI\_Flush
-
-ADIOI\_Resize(ADIO\_File fd, ADIO\_Offset size, int *error\_code)
-
-Resize is called collectively by all processes that opened the file referenced
-by fd.  It is not required that the Resize implementation block until all
-processes have completed resize operations, but each process should be able to
-see the correct size with a corresponding MPI\_File\_get\_size operation (an
-independent operation that results in an ADIO Fcntl to obtain the file size).
-
-ADIOI\_Delete(char *filename, int *error\_code)
-
-Delete is called independently, and because only a filename is passed, there
-is no opportunity to coordinate deletion if an application were to choose to
-have all processes call MPI\_File\_delete.  That's not likely to be an issue
-though.
-
-\section*{Appendix B: Status of ADIO Implementations}
-
-... who wrote what, status, etc.
-
-Appendix C: Adding a New ADIO Implementation
-
-References
-
-\end{document}
diff --git a/3rd-party/romio321/doc/users-guide.pdf b/3rd-party/romio321/doc/users-guide.pdf
deleted file mode 100644
index 4d060499a95..00000000000
Binary files a/3rd-party/romio321/doc/users-guide.pdf and /dev/null differ
diff --git a/3rd-party/romio321/doc/users-guide.tex b/3rd-party/romio321/doc/users-guide.tex
deleted file mode 100644
index 3715431b718..00000000000
--- a/3rd-party/romio321/doc/users-guide.tex
+++ /dev/null
@@ -1,1143 +0,0 @@
-% \documentstyle[11pt,psfig]{article}
-\documentstyle[11pt]{article}
-\hoffset=-.7in
-\voffset=-.6in
-\textwidth=6.5in
-\textheight=8.5in
-\begin{document}
-\vspace*{-1in}
-\thispagestyle{empty}
-\begin{center}
-ARGONNE NATIONAL LABORATORY \\
-9700 South Cass Avenue \\
-Argonne, IL 60439
-\end{center}
-\vskip .5 in
-\begin{center}
-\rule{1.75in}{.01in} \\
-\vspace{.1in}
-
-ANL/MCS-TM-234 \\
-
-\rule{1.75in}{.01in} \\
-
-\vskip 1.3in 
-{\Large\bf Users Guide for ROMIO: A High-Performance, \\ [1ex]
-Portable MPI-IO Implementation} \\ [4ex]
-by \\ [2ex]
-{\large\it Rajeev Thakur, Robert Ross, Ewing Lusk, William Gropp, Robert Latham}
-\vspace{1in}
-
-Mathematics and Computer Science Division
-
-\bigskip
-
-Technical Memorandum No.\ 234
-
-
-\vspace{1.4in}
-Revised May 2004, November 2007, April 2010
-
-\end{center}
-
-\vfill
-
-{\small
-\noindent
-This work was supported by the Mathematical, Information, and
-Computational Sciences Division subprogram of the Office of Advanced
-Scientific Computing Research, U.S. Department of Energy, under
-Contract W-31-109-Eng-38; and by the Scalable I/O Initiative, a
-multiagency project funded by the Defense Advanced Research Projects
-Agency (Contract DABT63-94-C-0049), the Department of Energy, the
-National Aeronautics and Space Administration, and the National
-Science Foundation.}
-
-\newpage
-
-
-%%  Line Spacing (e.g., \ls{1} for single, \ls{2} for double, even \ls{1.5})
-%%
-
-\newcommand{\ls}[1]
-   {\dimen0=\fontdimen6\the\font 
-    \lineskip=#1\dimen0
-    \advance\lineskip.5\fontdimen5\the\font
-    \advance\lineskip-\dimen0
-    \lineskiplimit=.9\lineskip
-    \baselineskip=\lineskip
-    \advance\baselineskip\dimen0
-    \normallineskip\lineskip
-    \normallineskiplimit\lineskiplimit
-    \normalbaselineskip\baselineskip
-    \ignorespaces
-   }
-\renewcommand{\baselinestretch}{1}
-\newcommand {\ix} {\hspace*{2em}}
-\newcommand {\mc} {\multicolumn}
-
-
-\tableofcontents
-\thispagestyle{empty}
-\newpage
-
-\pagenumbering{arabic}
-\setcounter{page}{1}
-\begin{center}
-{\bf Users Guide for ROMIO:  A High-Performance,\\[1ex]
-Portable MPI-IO Implementation} \\ [2ex]
-by \\ [2ex]
-{\it Rajeev Thakur, Robert Ross, Ewing Lusk, and William Gropp}
-
-\end{center}
-\addcontentsline{toc}{section}{Abstract}
-\begin{abstract}
-\noindent
-ROMIO is a high-performance, portable implementation of MPI-IO (the
-I/O chapter in the \mbox{MPI Standard}). This document describes how to install and use
-ROMIO version~1.2.4 on various machines.
-\end{abstract}
-
-\section{Introduction} 
-
-ROMIO\footnote{\tt http://www.mcs.anl.gov/romio} is a
-high-performance, portable implementation of MPI-IO (the I/O chapter in 
-MPI~\cite{mpi97a}). This document describes how to install and use
-ROMIO version~1.2.4 on various machines.
-
-
-%
-% MAJOR CHANGES IN THIS VERSION
-%
-\section{Major Changes in This Version}
-\begin{itemize}
-\item Added section describing ROMIO \texttt{MPI\_FILE\_SYNC} and
-      \texttt{MPI\_FILE\_CLOSE} behavior to User's Guide
-\item Bug removed from PVFS ADIO implementation regarding resize operations
-\item Added support for PVFS listio operations (see Section \ref{sec:hints})
-\item Added the following working hints:
-      \texttt{romio\_pvfs\_listio\_read}, \texttt{romio\_pvfs\_listio\_write}
-\end{itemize}
-
-%
-% GENERAL INFORMATION
-%
-\section{General Information}
-
-This version of ROMIO includes everything defined in the MPI I/O
-chapter except support for file interoperability and
-user-defined error handlers for files (\S~4.13.3).  The subarray and
-distributed array datatype constructor functions from Chapter 4
-(\S~4.14.4 \& \S~4.14.5) have been implemented. They are useful for
-accessing arrays stored in files. The functions {\tt MPI\_File\_f2c}
-and {\tt MPI\_File\_c2f} (\S~4.12.4) are also implemented.  C,
-Fortran, and profiling interfaces are provided for all functions that
-have been implemented.
-
-This version of ROMIO runs on at least the following machines: IBM SP; Intel
-Paragon; HP Exemplar; SGI Origin2000; Cray T3E; NEC SX-4; other
-symmetric multiprocessors from HP, SGI, DEC, Sun, and IBM; and networks of
-workstations (Sun, SGI, HP, IBM, DEC, Linux, and FreeBSD).
-Supported file systems are IBM PIOFS, Intel PFS, HP/Convex
-HFS, SGI XFS, NEC SFS, PVFS, NFS, NTFS, and any Unix file system (UFS).
-
-This version of ROMIO is included in MPICH 1.2.4; an earlier version
-is included in at least the following MPI implementations: LAM, HP
-MPI, SGI MPI, and NEC MPI. 
-
-Note that proper I/O error codes and classes are returned and the status
-variable is filled only when used with MPICH revision 1.2.1 or later.
-
-You can open files on multiple file systems in the same program. The
-only restriction is that the directory where the file is to be opened
-must be accessible from the process opening the file. For example, a
-process running on one workstation may not be able to access a
-directory on the local disk of another workstation, and therefore
-ROMIO will not be able to open a file in such a directory. NFS-mounted
-files can be accessed.
-
-An MPI-IO file created by ROMIO is no different from any other file
-created by the underlying file system. Therefore, you may use any of
-the commands provided by the file system to access the file, for example,
-{\tt ls}, {\tt mv}, {\tt cp}, {\tt rm}, {\tt ftp}.
-
-Please read the limitations of this version of ROMIO that are listed
-in Section~\ref{sec:limit} of this document (e.g., restriction to homogeneous
-environments). 
-
-\subsection{ROMIO Optimizations}
-\label{sec:opt}
-
-ROMIO implements two I/O optimization techniques that in general
-result in improved performance for applications.  The first of these
-is \emph{data sieving}~\cite{choudhary:passion}.  Data sieving is a
-technique for efficiently accessing noncontiguous regions of data in files
-when noncontiguous accesses are not provided as a file system primitive.
-The naive approach to accessing noncontiguous regions is to use a separate
-I/O call for each contiguous region in the file.  This results in a large
-number of I/O operations, each of which is often for a very small amount
-of data.  The added network cost of performing an I/O operation across the
-network, as in parallel I/O systems, is often high because of latency.
-Thus, this naive approach typically performs very poorly because of
-the overhead of multiple operations.  
-% 
-In the data sieving technique, a number of noncontiguous regions are
-accessed by reading a block of data containing all of the regions,
-including the unwanted data between them (called ``holes'').  The regions
-of interest are then extracted from this large block by the client.
-This technique has the advantage of a single I/O call, but additional
-data is read from the disk and passed across the network.
-
-There are four hints that can be used to control the application of
-data sieving in ROMIO: \texttt{ind\_rd\_buffer\_size},
-\texttt{ind\_wr\_buffer\_size}, \texttt{romio\_ds\_read},
-and \texttt{romio\_ds\_write}.  These are discussed in
-Section~\ref{sec:hints}.
-
-The second optimization is \emph{two-phase
-I/O}~\cite{bordawekar:primitives}.  Two-phase I/O, also called collective
-buffering, is an optimization that only applies to collective I/O
-operations.  In two-phase I/O, the collection of independent I/O operations
-that make up the collective operation are analyzed to determine what
-data regions must be transferred (read or written).  These regions are
-then split up amongst a set of aggregator processes that will actually
-interact with the file system.  In the case of a read, these aggregators
-first read their regions from disk and redistribute the data to the
-final locations, while in the case of a write, data is first collected
-from the processes before being written to disk by the aggregators.
-
-There are five hints that can be used to control the application
-of two-phase I/O: \texttt{cb\_config\_list}, \texttt{cb\_nodes},
-\texttt{cb\_buffer\_size}, \texttt{romio\_cb\_read},
-and \texttt{romio\_cb\_write}.  These are discussed in
-Subsection~\ref{sec:hints}.
-
-\subsection{Hints}
-\label{sec:hints}
-
-If ROMIO doesn't understand a hint, or if the value is invalid, the hint
-will be ignored. The values of hints being used by ROMIO for a file
-can be obtained at any time via {\tt MPI\_File\_get\_info}.
-
-The following hints control the data sieving optimization and are
-applicable to all file system types:
-
-\begin{itemize}
-\item \texttt{ind\_rd\_buffer\_size} -- Controls the size (in bytes) of the
-intermediate buffer used by ROMIO when performing data sieving during
-read operations.  Default is \texttt{4194304} (4~Mbytes).
-\item \texttt{ind\_wr\_buffer\_size} -- Controls the size (in bytes) of the
-intermediate buffer used by ROMIO when performing data sieving during
-write operations.  Default is \texttt{524288} (512~Kbytes).
-\item \texttt{romio\_ds\_read} -- 
-Determines when ROMIO will choose to perform data sieving.
-Valid values are \texttt{enable}, \texttt{disable}, or \texttt{automatic}.
-Default value is \texttt{automatic}.  In \texttt{automatic} mode ROMIO
-may choose to enable or disable data sieving based on heuristics.
-\item \texttt{romio\_ds\_write} -- Same as above, only for writes.
-\end{itemize}
-
-The following hints control the two-phase (collective buffering)
-optimization and are applicable to all file system types:
-\begin{itemize}
-\item \texttt{cb\_buffer\_size} -- Controls the size (in bytes) of the
-intermediate buffer used in two-phase collective I/O.  If the amount
-of data that an aggregator will transfer is larger than this value,
-then multiple operations are used.  The default is \texttt{4194304} (4~Mbytes).
-\item \texttt{cb\_nodes} -- Controls the maximum number of aggregators
-to be used.  By default this is set to the number of unique hosts in the
-communicator used when opening the file.
-\item \texttt{romio\_cb\_read} -- Controls when collective buffering is
-applied to collective read operations.  Valid values are
-\texttt{enable}, \texttt{disable}, and \texttt{automatic}.  Default is
-\texttt{automatic}.  When enabled, all collective reads will use
-collective buffering.  When disabled, all collective reads will be
-serviced with individual operations by each process.  When set to
-\texttt{automatic}, ROMIO will use heuristics to determine when to
-enable the optimization.
-\item \texttt{romio\_cb\_write} -- Controls when collective buffering is
-applied to collective write operations.  Valid values are 
-\texttt{enable}, \texttt{disable}, and \texttt{automatic}.  Default is 
-\texttt{automatic}.  See the description of \texttt{romio\_cb\_read} for
-an explanation of the values.
-\item \texttt{romio\_no\_indep\_rw} -- This hint controls when ``deferred
-open'' is used.  When set to \texttt{true}, ROMIO will make an effort to avoid
-performing any file operation on non-aggregator nodes.  The application is
-expected to use only collective operations.  This is discussed in further
-detail below.
-\item \texttt{cb\_config\_list} -- Provides explicit control over 
-aggregators.  This is discussed in further detail below.
-\end{itemize}
-
-For some systems configurations, more control is needed to specify which
-hardware resources (processors or nodes in an SMP) are preferred for
-collective I/O, either for performance reasons or because only certain
-resources have access to storage.  The additional MPI\_Info key name
-\texttt{cb\_config\_list} specifies a comma-separated list of strings,
-each string specifying a particular node and an optional limit on the
-number of processes to be used for collective buffering on this node.
-
-This refers to the same processes that \texttt{cb\_nodes} refers to,
-but specifies the available nodes more precisely.
-
-The format of the value of \texttt{cb\_config\_list} is given by the
-following BNF:
-\begin{verbatim}
-cb_config_list => hostspec [ ',' cb_config_list ]
-hostspec => hostname [ ':' maxprocesses ]
-hostname => <alphanumeric string>
-         |  '*' 
-maxprocesses => <digits>
-         |  '*'
-\end{verbatim}
-
-The value \texttt{hostname} identifies a processor. This name must match
-the name returned by \texttt{MPI\_Get\_processor\_name}~\footnote{The
-MPI standard requires that the output from this routine identify a
-particular piece of hardware; some MPI implementations may not conform
-to this requirement. MPICH does conform to the MPI standard.}
-%
-for the specified hardware. The value \texttt{*} as a hostname matches all
-processors. The value of maxprocesses may be any nonnegative integer
-(zero is allowed).
-
-The value \texttt{maxprocesses} specifies the maximum number of
-processes that may be used for collective buffering on the specified
-host. If no value is specified, the value one is assumed. If \texttt{*}
-is specified for the number of processes, then all MPI processes with
-this same hostname will be used..
-
-Leftmost components of the info value take precedence.
-
-Note: Matching of processor names to \texttt{cb\_config\_list} entries
-is performed with string matching functions and is independent of the
-listing of machines that the user provides to mpirun/mpiexec.  In other
-words, listing the same machine multiple times in the list of hosts to
-run on will not cause a \texttt{*:1} to assign the same host four
-aggregators, because the matching code will see that the processor name
-is the same for all four and will assign exactly one aggregator to the
-processor.
-
-The value of this info key must be the same for all processes (i.e., the
-call is collective and each process must receive the same hint value for
-these collective buffering hints).  Further, in the ROMIO implementation
-the hint is only recognized at \texttt{MPI\_File\_open} time.
-
-The set of hints used with a file is available through the routine
-\texttt{MPI\_File\_get\_info}, as documented in the MPI standard.
-As an additional feature in the ROMIO implementation, wildcards will
-be expanded to indicate the precise configuration used with the file,
-with the hostnames in the rank order used for the collective buffering
-algorithm (\emph{this is not implemented at this time}).
-
-Here are some examples of how this hint might be used:
-\begin{itemize}
-\item \texttt{*:1} One process per hostname (i.e., one process per node)
-\item \texttt{box12:30,*:0} Thirty processes on one machine, namely
-      \texttt{box12}, and none anywhere else.
-\item \texttt{n01,n11,n21,n31,n41} One process on each of these specific
-      nodes only.
-\end{itemize}
-
-When the values specified by \texttt{cb\_config\_list} conflict with
-other hints (e.g., the number of collective buffering nodes specified by
-\texttt{cb\_nodes}), the implementation is encouraged to take the minimum
-of the two values.  In other words, if \texttt{cb\_config\_list} specifies
-ten processors on which I/O should be performed, but \texttt{cb\_nodes}
-specifies a smaller number, then an implementation is encouraged to use
-only \texttt{cb\_nodes} total aggregators. If \texttt{cb\_config\_list}
-specifies fewer processes than \texttt{cb\_nodes}, no more than the
-number in \texttt{cb\_config\_list} should be used.
-
-The implementation is also encouraged to assign processes in the order
-that they are listed in \texttt{cb\_config\_list}.
-
-The following hint controls the deferred open feature of romio and are also
-applicable to all file system types: 
-\begin{itemize}
-\item \texttt{romio\_no\_indep\_rw} -- If the application plans on performing only 
-   collecitve operations and this hint is set to ``true'', then ROMIO can
-   have just the aggregators open a file.   The \texttt{cb\_config\_list} and
-   \texttt{cb\_nodes} hints can be given to further control which nodes are
-   aggregators.  
-\end{itemize}
-
-For PVFS, PIOFS, and PFS:
-\begin{itemize}
-\item \texttt{striping\_factor} -- Controls the number of I/O devices to
-stripe across.  The default is file system dependent, but for PVFS it is
-\texttt{-1}, indicating that the file should be striped across all I/O
-devices.
-\item \texttt{striping\_unit} --  Controls the striping unit (in bytes).
-For PVFS the default will be the PVFS file system default strip size.
-\item \texttt{start\_iodevice} -- Determines what I/O device data will
-first be written to.  This is a number in the range of 0 ...
-striping\_factor - 1.
-\end{itemize}
-
-\subsubsection{Hints for PFS}
-\label{sec:hints_pfs}
-\begin{itemize}
-\item \texttt{pfs\_svr\_buf} -- Turns on PFS server buffering.  Valid
-values are \texttt{true} and \texttt{false}.  Default is \texttt{false}.
-\end{itemize}
-
-\subsubsection{Hints for XFS}
-\label{sec:hints_xfs}
-For XFS control is provided for the direct I/O optimization:
-\begin{itemize}
-\item \texttt{direct\_read} -- Controls direct I/O for reads.  Valid
-values are \texttt{true} and \texttt{false}.  Default is \texttt{false}.
-\item \texttt{direct\_write} -- Controls direct I/O for writes.  Valid
-values are \texttt{true} and \texttt{false}.  Default is \texttt{false}.
-\end{itemize}
-
-\subsubsection{Hints for PVFS (v1)}
-\label{sec:hints_oldpvfs}
-
-For PVFS control is provided for the use of the listio interface.  This
-interface to PVFS allows for a collection of noncontiguous regions to be
-requested (for reading or writing) with a single operation.  This can result
-in substantially higher performance when accessing noncontiguous regions.
-Support for these operations in PVFS exists after version 1.5.4, but has not
-been heavily tested, so use of the interface is disabled in ROMIO by default
-at this time.  The hints to control listio use are:
-\begin{itemize}
-\item \texttt{romio\_pvfs\_listio\_read} -- Controls use of listio for reads.
-Valid values are \texttt{enable}, \texttt{disable}, and \texttt{automatic}.
-Default is \texttt{disable}.
-\item \texttt{romio\_pvfs\_listio\_write} -- Controls use of listio for writes.
-Valid values are \texttt{enable}, \texttt{disable}, and \texttt{automatic}.
-Default is \texttt{disable}.
-\end{itemize}
-
-\subsubsection{Hints for PVFS (v2)}
-\label{sec:hints_pvfs}
-
-The PVFS v2 file system has many tuning parameters. 
-\begin{itemize}
-\item dtype i/o
-\end{itemize}
-
-\subsubsection{Hints for Lustre}
-
-\begin{itemize}
-\item romio\_lustre\_co\_ratio
-
-In stripe-contiguous IO pattern, each OST will be accessed by a group of
-IO clients. CO means *C*lient/*O*ST ratio, or the max. number of IO clients
-for each OST.
-CO=1 by default.
-
-\item \texttt{romio\_lustre\_coll\_threshold}
-
-We won't do collective I/O if this hint is set and the IO request size is
-bigger than this value. That's because when the request size is big, the
-collective communication overhead increases and the benefits from collective
-I/O becomes limited.  A value of 0 means always perform collective I/O
-
-\item \texttt{romio\_lustre\_cb\_ds\_threshold}
-
-ROMIO can optimize collective I/O with a version of data sieving.  If the I/O
-request is smaller than this hint's value, though, ROMIO will not try to apply
-the data sieving optimization.
-
-\item \texttt{romio\_lustre\_ds\_in\_coll}
-
-Collective IO will apply read-modify-write to deal with non-contiguous
-data by default. However, it will introduce some overhead(IO operation and
-locking).  The Lustre developers have run tests where data sieving showed bad
-collective write performance for some kinds of workloads.  So, to avoid this,
-we define the \texttt{romio\_lustre\_ds\_in\_coll} hint to disable the read-modify-write
-step in collective I/O.  This optimization is distinct from the one in
-independent I/O (controlled by \texttt{romio\_ds\_read} and 
-\texttt{romio\_ds\_write}).
-
-\end{itemize}
-
-\subsubsection{Hints for PANFS (Panasas)}
-
-PanFS allows users to specify the layout of a file at file-creation time.
-Layout information includes the number of StorageBlades (SB) across which the
-data is stored, the number of SBs across which a parity stripe is written, and
-the number of consecutive stripes that are placed on the same set of SBs.   The
-\texttt{panfs\_layout\_*} hints are only used if supplied at file-creation
-time.
-\begin{itemize}
-
-\item \texttt{panfs\_layout\_type} Specifies the layout of a file: 2 = RAID0
-3 = RAID5 Parity Stripes 
-
-\item \texttt{panfs\_layout\_stripe\_unit} The size of the stripe unit
-in bytes
-
-\item \texttt{panfs\_layout\_total\_num\_comps} The total number of
-StorageBlades a file is striped across.
-
-\item \texttt{ panfs\_layout\_parity\_stripe\_width}  If the layout type is
-RAID5 Parity Stripes, this hint specifies the number of StorageBlades in a
-parity stripe.
-
-\item \texttt{panfs\_layout\_parity\_stripe\_depth} If the layout type is RAID5
-Parity Stripes, this hint specifies the number of contiguous parity stripes
-written across the same set of SBs.
-
-\item \texttt{panfs\_layout\_visit\_policy} If the layout type is RAID5 Parity
-Stripes, the policy used to determine the parity stripe a given file offset is
-written to: 1 = Round Robin 
-\end{itemize}
-
-PanFS supports the ``concurrent write'' (CW) mode, where groups of
-cooperating clients can disable the PanFS consistency mechanisms and use
-their own consistency protocol.  Clients participating in concurrent
-write mode use application specific information to improve performance
-while maintaining file consistency.  All clients accessing the file(s)
-must enable concurrent write mode.  If any client does not enable
-concurrent write mode, then the PanFS consistency protocol will be
-invoked.  Once a file is opened in CW mode on a machine, attempts to
-open a file in non-CW mode will fail with EACCES.  If a file is already
-opened in non-CW mode, attempts to open the file in CW mode will fail
-with EACCES.  The following hint is used to enable concurrent write
-mode.
-
-\begin{itemize}
-\item \texttt{panfs\_concurrent\_write} If set to 1 at file open time,
-the file is opened using the PanFS concurrent write mode flag.
-Concurrent write mode is not a persistent attribute of the file.
-\end{itemize}
-
-Below is an example PanFS layout using the following parameters:
-\begin{verbatim}
- 
-  - panfs_layout_type                = 3
-  - panfs_layout_total_num_comps     = 100
-  - panfs_layout_parity_stripe_width = 10
-  - panfs_layout_parity_stripe_depth = 8
-  - panfs_layout_visit_policy        = 1
-
-   Parity Stripe Group 1     Parity Stripe Group 2  . . . Parity Stripe Group 10
-  ----------------------    ----------------------        --------------------
-   SB1    SB2  ... SB10     SB11    SB12 ...  SB20  ...   SB91   SB92 ... SB100
-  -----------------------   -----------------------       ---------------------
-   D1     D2   ...  D10      D91    D92  ...  D100        D181   D182  ... D190
-   D11    D12       D20      D101   D102      D110        D191   D192      D193
-   D21    D22       D30      . . .                        . . .
-   D31    D32       D40
-   D41    D42       D50
-   D51    D52       D60
-   D61    D62       D70
-   D71    D72       D80
-   D81    D82       D90      D171   D172      D180        D261   D262   D270
-   D271   D272      D273     . . .                        . . .
-   ...
-\end{verbatim}
-
-\subsubsection{Systemwide Hints}
-\label{sec:system_hints}
-
-A site administrator with knowledge of the storage and networking capabilities
-of a machine might be able to come up with a set of hint values that work
-better for that machine than the ROMIO default values.  As an extention to the
-standard, ROMIO will consult a ``hints file''.  This file provides an
-additional mechanism for setting MPI-IO hints, albeit in a ROMIO-specific
-manner.  The hints file contains a list of hints and their values.  ROMIO will
-use these initial hint settings, though programs are free to override any of
-them.
-
-The format of the hints file is a list of hints and their values, one per line.
-A \# character in the first column indicates a comment, and ROMIO will ignore
-the entire line.  Here's an example:
-
-\begin{verbatim}
-# this is a comment describing the following setting
-cb_nodes 32
-# these nodes happen to have the best connection to storage
-cb_config_list n01,n11,n21,n31,n41
-\end{verbatim}
-
-ROMIO will look for these hints in the file \texttt{/etc/romio-hints}.  A user
-can set the environment variable \texttt{ROMIO\_HINTS} to the name of a file
-which ROMIO will use instead.   
-
-\subsection{Using ROMIO on NFS}
-
-It is worth first mentioning that in no way do we encourage the use
-of ROMIO on NFS volumes.  NFS is not a high-performance protocol, nor
-are NFS servers typically very good at handling the types of concurrent
-access seen from MPI-IO applications.  Nevertheless, NFS is a very popular
-mechanism for providing access to a shared space, and ROMIO does support
-MPI-IO to NFS volumes, provided that they are configured properly.
-
-To use ROMIO on NFS, file locking with {\tt fcntl} must work correctly
-on the NFS installation. On some installations, fcntl locks don't
-work.  To get them to work, you need to use Version~3 of NFS, ensure
-that the lockd daemon is running on all the machines, and have the system
-administrator mount the NFS file system with the ``{\tt noac}'' option
-(no attribute caching). Turning off attribute caching may reduce
-performance, but it is necessary for correct behavior.
-
-The following are some instructions we received from Ian Wells of HP
-for setting the {\tt noac} option on NFS. We have not tried them
-ourselves. We are including them here because you may find 
-them useful. Note that some of the steps may be specific to HP
-systems, and you may need root permission to execute some of the
-commands. 
-
-\begin{verbatim}   
-   >1. first confirm you are running nfs version 3
-   >
-   >rpcnfo -p `hostname` | grep nfs
-   >
-   >ie 
-   >    goedel >rpcinfo -p goedel | grep nfs
-   >    100003    2   udp   2049  nfs
-   >    100003    3   udp   2049  nfs
-   >
-   >
-   >2. then edit /etc/fstab for each nfs directory read/written by MPIO
-   >   on each  machine used for multihost MPIO.
-   >
-   >    Here is an example of a correct fstab entry for /epm1:
-   >
-   >   ie grep epm1 /etc/fstab
-   > 
-   >      ROOOOT 11>grep epm1 /etc/fstab
-   >      gershwin:/epm1 /rmt/gershwin/epm1 nfs bg,intr,noac 0 0
-   >
-   >   if the noac option is not present, add it 
-   >   and then remount this directory
-   >   on each of the machines that will be used to share MPIO files
-   >
-   >ie
-   >
-   >ROOOOT >umount /rmt/gershwin/epm1
-   >ROOOOT >mount  /rmt/gershwin/epm1
-   >
-   >3. Confirm that the directory is mounted noac:
-   >
-   >ROOOOT >grep gershwin /etc/mnttab 
-   >gershwin:/epm1 /rmt/gershwin/epm1 nfs
-   >noac,acregmin=0,acregmax=0,acdirmin=0,acdirmax=0 0 0 899911504
-\end{verbatim}
-
-\subsubsection{ROMIO, NFS, and Synchronization}
-
-NFS has a ``sync'' option that specifies that the server should put data on
-the disk before replying that an operation is complete.  This means that
-the actual I/O cost on the server side cannot be hidden with caching,
-etc. when this option is selected.
-                                                                                
-In the ``async'' mode the server can get the data into a buffer (and
-perhaps put it in the write queue; this depends on the implementation)
-and reply right away.  Obviously if the server were to go down after the
-reply was sent but before the data was written, the system would be in
-a strange state, which is why so many articles suggest the "sync" option.
-
-Some systems default to ``sync'', while others default to ``async'',
-and the default can change from version to version of the NFS software.  If
-you find that access to an NFS volume through MPI-IO is particularly slow,
-this is one thing to check out.
-
-
-\subsection{Using testfs}
-The testfs ADIO implementation provides a harness for testing components
-of ROMIO or discovering the underlying I/O access patterns of an
-application.  When testfs is specified as the file system type, no
-actual files will be opened.  Instead debugging information will be
-displayed on the processes opening the file.  Subsequent I/O operations
-on this testfs file will provide additional debugging information.
-
-The intention of the testfs implementation is that it serve as a
-starting point for further instrumentation when debugging new features
-or applications.  As such it is expected that users will want to modify
-the ADIO implementation in order to get the specific output they desire.
-
-\subsection{ROMIO and {\tt MPI\_FILE\_SYNC}}
-
-The MPI specification notes that a call to {\tt MPI\_FILE\_SYNC} ``causes
-all previous writes to {\tt fh} by the calling process to be transferred to
-the storage device.''  Likewise, calls to {\tt MPI\_FILE\_CLOSE} have this
-same semantic.  Further, ``if all processes have made updates to the storage
-device, then all such updates become visible to subsequent reads of {\tt fh}
-by the calling process.''
-
-The intended use of {\tt MPI\_FILE\_SYNC} is to allow all processes in the
-communicator used to open the file to see changes made to the file by each
-other (the second part of the specification).  The definition of ``storage
-device'' in the specification is vague, and it isn't necessarily the case that
-calling {\tt MPI\_FILE\_SYNC} will force data out to permanent storage.
-
-Since users often use {\tt MPI\_FILE\_SYNC} to attempt to force data out to
-permanent storage (i.e. disk), the ROMIO implementation of this call enforces
-stronger semantics for most underlying file systems by calling the appropriate
-file sync operation when {\tt MPI\_FILE\_SYNC} is called (e.g. {\tt fsync}).
-However, it is still unwise to assume that the data has all made it to disk
-because some file systems (e.g. NFS) may not force data to disk when a client
-system makes a sync call.
-
-For performance reasons we do \emph{not} make this same file system call at
-{\tt MPI\_FILE\_CLOSE} time.  At close time ROMIO ensures any data has been
-written out to the ``storage device'' (file system) as defined in the
-standard, but does not try to push the data beyond this and into physical
-storage. Users should call {\tt MPI\_FILE\_SYNC} before the close if they wish
-to encourage the underlying file system to push data to permanent storage.
-
-\subsection{ROMIO and {\tt MPI\_FILE\_SET\_SIZE}}
-
-{\tt MPI\_FILE\_SET\_SIZE} is a collective routine used to resize a file.  It
-is important to remember that a MPI-IO routine being collective does not imply
-that the routine synchronizes the calling processes in any way (unless this is
-specified explicitly).
-
-As of 1.2.4, ROMIO implements {\tt MPI\_FILE\_SET\_SIZE} by calling {\tt
-ftruncate} from all processes.  Since different processes may call the
-function at different times, it means that unless external synchronization is
-used, a resize operation mixed in with writes or reads could have unexpected
-results.
-
-In short, if synchronization after a set size is needed, the user should add a
-barrier or similar operation to ensure the set size has completed.
-
-
-%
-% INSTALLATION INSTRUCTIONS
-%
-\section{Installation Instructions}
-Since ROMIO is included in MPICH, LAM, HP MPI, SGI MPI, and NEC MPI, you don't
-need to install it separately if you are using any of these MPI
-implementations.  If you are using some other MPI, you
-can configure and build ROMIO as follows: 
-
-Untar the tar file as
-\begin{verbatim}
-    gunzip -c romio.tar.gz | tar xvf -
-\end{verbatim}
-{\noindent or}
-\begin{verbatim}
-    zcat romio.tar.Z | tar xvf -
-\end{verbatim}
-
-{\noindent then}
-
-\begin{verbatim}
-    cd romio
-    ./configure
-    make
-\end{verbatim}
-
-Some example programs and a Makefile are provided in the {\tt romio/test}
-directory.  Run the examples as you would run any MPI program. Each
-program takes the filename as a command-line argument ``{\tt -fname
-filename}''.
-
-The {\tt configure} script by default configures ROMIO for the file
-systems most likely 
-to be used on the given machine. If you wish, you can explicitly specify the file
-systems by using the ``{\tt -file\_system}'' option to configure. Multiple file
-systems can be specified by using `+' as a separator, e.g., \\
-\hspace*{.4in} {\tt ./configure -file\_system=xfs+nfs} \\
-For the entire list of options to configure, do\\ 
-\hspace*{.4in} {\tt ./configure -h | more} \\
-After building a specific version, you can install it in a
-particular directory with \\
-\hspace*{.4in} {\tt make install PREFIX=/usr/local/romio    (or whatever directory you like)} \\
-or just\\
-\hspace*{.4in} {\tt make install          (if you used -prefix at configure time)}
-
-If you intend to leave ROMIO where you built it, you should {\it not}
-install it; {\tt make install} is used only to move the necessary
-parts of a built ROMIO to another location. The installed copy will
-have the include files, libraries, man pages, and a few other odds and
-ends, but not the whole source tree.  It will have a {\tt test}
-directory for testing the installation and a location-independent
-Makefile built during installation, which users can copy and modify to
-compile and link against the installed copy.
-
-To rebuild ROMIO with a different set of configure options, do\\
-\hspace*{.4in} {\tt make distclean}\\
-to clean everything, including the Makefiles created by {\tt
-configure}.  Then run {\tt configure} again with the new options,
-followed by {\tt make}.
-
-\subsection{Configuring for Linux and Large Files }
-
-32-bit systems running linux kernel version 2.4.0 or newer and glibc
-version 2.2.0 or newer can support files greater than 2 GBytes in size.
-This support is currently automaticly detected and enabled.  We document the
-manual steps should the automatic detection not work for some reason.
-
-The two macros {\tt\_FILE\_OFFSET\_BITS=64} and
-{\tt\_LARGEFILE64\_SOURCE} tell gnu libc it's ok to support large files
-on 32 bit platforms.  The former changes the size of {\tt off\_t} (no
-need to change source.  might affect interoperability with libraries
-compiled with a different size of {\tt off\_t}).   The latter exposes
-the gnu libc functions open64(), write64(), read64(), etc.   ROMIO does
-not make use of the 64 bit system calls directly at this time, but we
-add this flag for good measure.  
-
-If your linux system is relatively new, there is an excellent chance it
-is running kernel 2.4.0 or newer and glibc-2.2.0 or newer.  Add the
-string
-\begin{verbatim}
-"-D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE"
-\end{verbatim}
-to your CFLAGS environment variable before runnint {\tt./configure}
-
-%
-% TESTING ROMIO
-%
-\section{Testing ROMIO}
-To test if the installation works, do\\
-\hspace*{.4in} {\tt make testing}\\
-in the {\tt romio/test} directory. This calls a script that runs the test
-programs and compares the results with what they should be. By
-default, {\tt make testing} causes the test programs to create files in
-the current directory and use whatever file system that corresponds
-to. To test with other file systems, you need to specify a filename in
-a directory corresponding to that file system as follows:\\
-\hspace*{.4in} {\tt make testing TESTARGS="-fname=/foo/piofs/test"}
-
-
-%
-% COMPILING AND RUNNING MPI-IO PROGRAMS
-%
-\section{Compiling and Running MPI-IO Programs}
-If ROMIO is not already included in the MPI implementation, you need
-to include the file {\tt mpio.h} for C or {\tt mpiof.h} for Fortran in
-your MPI-IO program.  
-
-Note that on HP machines running HPUX and on NEC SX-4, you need to
-compile Fortran programs with {\tt mpifort}, because {\tt mpif77} does
-not support 8-byte integers.
-
-With MPICH, HP MPI, or NEC MPI, you can compile MPI-IO programs as \\
-\hspace*{.4in} {\tt mpicc foo.c}\\
-or \\
-\hspace*{.4in} {\tt mpif77 foo.f }\\
-or\\
-\hspace*{.4in} {\tt mpifort foo.f}\\
-
-As mentioned above, mpifort is preferred over mpif77 on HPUX and NEC
-because the f77 compilers on those machines do not support 8-byte integers.
-
-With SGI MPI, you can compile MPI-IO programs as \\
-\hspace*{.4in} {\tt cc foo.c -lmpi}\\
-or \\
-\hspace*{.4in} {\tt f77 foo.f -lmpi}\\
-or \\
-\hspace*{.4in} {\tt f90 foo.f -lmpi}\\
-
-With LAM, you can compile MPI-IO programs as \\
-\hspace*{.4in} {\tt hcc foo.c -lmpi}\\
-or \\
-\hspace*{.4in} {\tt hf77 foo.f -lmpi}\\
-
-If you have built ROMIO with some other MPI implementation, you can
-compile MPI-IO programs by explicitly giving the path to the include
-file mpio.h or mpiof.h and explicitly specifying the path to the
-library libmpio.a, which is located in {\tt \$(ROMIO\_HOME)/lib/\$(ARCH)/libmpio.a}.
-
-Run the program as you would run any MPI program on the machine.
-If you use {\tt mpirun}, make sure you use the correct {\tt mpirun}
-for the MPI implementation you are using. For example, if you
-are using MPICH on an SGI machine, make sure that you use MPICH's
-{\tt mpirun} and not SGI's {\tt mpirun}.
-
-
-%
-% LIMITATIONS
-%
-\section{Limitations of This Version of ROMIO \label{sec:limit}}
-
-\begin{itemize}
-\item When used with any MPI implementation other than MPICH revision
-1.2.1 or later, the {\tt status} argument is not filled in any MPI-IO
-function. Consequently, {\tt MPI\_Get\_count} and\linebreak {\tt
-MPI\_Get\_elements} will not work when passed the {\tt status} object
-from an MPI-IO operation.
-
-\item Additionally, when used with any MPI implementation other than MPICH
-revision 1.2.1 or later, all MPI-IO functions return only two possible
-error codes---{\tt MPI\_SUCCESS} on success and {\tt MPI\_ERR\_UNKNOWN}
-on failure.
-
-\item This version works only on a homogeneous cluster of machines,
-and only the ``native'' file data representation is supported.
-
-\item Shared file pointers are not supported on PVFS and IBM PIOFS
-file systems because they don't support {\tt fcntl} file locks,
-and ROMIO uses that feature to implement shared file pointers.
-
-\item On HP machines running HPUX and on NEC SX-4, you need to compile
-Fortran programs with {\tt mpifort} instead of {\tt mpif77}, because
-the {\tt f77} compilers on these machines don't support 8-byte integers.
-
-\item The file-open mode {\tt MPI\_MODE\_EXCL} does not work on Intel
-PFS file system, due to a bug in PFS.
-
-\end{itemize}
-
-
-%
-% USAGE TIPS
-%
-\section{Usage Tips}
-\begin{itemize}
-\item When using ROMIO with SGI MPI, you may
-sometimes get an error message from SGI MPI: ``MPI has run out of
-internal datatype entries. Please set the environment variable
-{\tt MPI\_TYPE\_MAX} for additional space.'' If you get this error message,
-add the following line to your {\tt .cshrc} file:\\
-\hspace*{.4in} {\tt setenv MPI\_TYPE\_MAX 65536}\\
-Use a larger number if you still get the error message.
-\item If a Fortran program uses a file handle created using ROMIO's C
-interface, or vice versa, you must use the functions {\tt MPI\_File\_c2f} 
-or {\tt MPI\_File\_f2c} (see \S~4.12.4 in~\cite{mpi97a}). Such a
-situation occurs, for example, if a Fortran program uses an I/O
-library written in C 
-with MPI-IO calls. Similar functions {\tt MPIO\_Request\_f2c} and
-{\tt MPIO\_Request\_c2f} are also provided.
-\item For Fortran programs on the Intel Paragon, you may need
-to provide the complete path to {\tt mpif.h} in the {\tt include}
-statement, e.g., \\
-\hspace*{.4in} {\tt include '/usr/local/mpich/include/mpif.h'}\\
-instead of \\
-\hspace*{.4in} {\tt include 'mpif.h'}\\ 
-This is because the {\tt -I}
-option to the Paragon Fortran compiler {\tt if77} doesn't work
-correctly. It always looks in the default directories first and,
-therefore, picks up Intel's {\tt mpif.h}, which is actually the {\tt
-mpif.h} of an older version of MPICH.
-
-\end{itemize}
-
-%
-% MAILING LIST
-%
-% this mailing list has been dead for a while
-%
-% REPORTING BUGS
-%
-\section{Reporting Bugs}
-If you have trouble, first check the users guide. Then check if there
-is a list of known bugs and patches on the ROMIO web page at {\tt
-http://www.mcs.anl.gov/romio}.  Finally, if you still have problems, send a
-detailed message containing:\\
-\hspace*{.2in}$\bullet$ the type of system (often {\tt uname -a}),\\
-\hspace*{.2in}$\bullet$ the output of {\tt configure},\\
-\hspace*{.2in}$\bullet$ the output of {\tt make}, and \\
-\hspace*{.2in}$\bullet$ any programs or tests\\
-to {\tt romio-maint@mcs.anl.gov}.
-
-
-%
-% ROMIO INTERNALS
-%
-\section{ROMIO Internals}
-A key component of ROMIO that enables such a portable MPI-IO
-implementation is an internal abstract I/O device layer called
-ADIO~\cite{thak96e}. Most users of ROMIO will not need to deal with
-the ADIO layer at all. However, ADIO is useful to those who want to
-port ROMIO to some other file system. The ROMIO source code and the
-ADIO paper~\cite{thak96e} will help you get started.
-
-MPI-IO implementation issues are discussed in~\cite{thak99b}. All
-ROMIO-related papers are available online at {\tt
-http://www.mcs.anl.gov/romio}.
-
-
-\section{Learning MPI-IO}
-The book {\em Using MPI-2: Advanced Features of the Message-Passing
-Interface}~\cite{grop99a}, published by MIT Press, provides a tutorial
-introduction to all aspects of MPI-2, including parallel I/O. It has
-lots of example programs. See {\tt
-http://www.mcs.anl.gov/mpi/usingmpi2} for further information about
-the book.
-
-%
-% MAJOR CHANGES IN PREVIOUS RELEASES
-%
-\section{Major Changes in Previous Releases}
-
-\subsection{Major Changes in Version 1.2.3}
-\begin{itemize}
-\item Added explicit control over aggregators for collective operations
-      (see description of \texttt{cb\_config\_list}).
-\item Added the following working hints: \texttt{cb\_config\_list},
-      \texttt{romio\_cb\_read}, \texttt{romio\_cb\_write},\newline
-      \texttt{romio\_ds\_read}.  These additional hints have
-      been added but are currently ignored by the implementation:
-      \texttt{romio\_ds\_write}, \texttt{romio\_no\_indep\_rw}.
-\item Added NTFS ADIO implementation.
-\item Added testfs ADIO implementation for use in debugging.
-\item Added delete function to ADIO interface so that file systems that
-      need to use their own delete function may do so (e.g. PVFS).
-\item Changed version numbering to match version number of MPICH release.
-\end{itemize}
-
-\subsection{Major Changes in Version 1.0.3}
-\begin{itemize}
-\item When used with MPICH 1.2.1, the MPI-IO functions return proper
-error codes and classes, and the status object is filled in.
-
-\item On SGI's XFS file system, ROMIO can use direct I/O even if the
-user's request does not meet the various restrictions needed to use
-direct I/O. ROMIO does this by doing part of the request with buffered
-I/O (until all the restrictions are met) and doing the rest with
-direct I/O. (This feature hasn't been tested rigorously. Please check
-for errors.)
-
-By default, ROMIO will use only buffered I/O. Direct I/O can be
-enabled either by setting the environment variables {\tt
-MPIO\_DIRECT\_READ} and/or {\tt MPIO\_DIRECT\_WRITE} to {\tt TRUE}, or
-on a per-file basis by using the info keys {\tt direct\_read} and {\tt
-direct\_write}.
-
-Direct I/O will result in higher performance only if you are accessing
-a high-bandwidth disk system. Otherwise, buffered I/O is better and is
-therefore used as the default.
-
-\item Miscellaneous bug fixes.
-\end{itemize}
-
-\subsection{Major Changes in Version 1.0.2}
-\begin{itemize}
-\item Implemented the shared file pointer functions and
-  split collective I/O functions. Therefore, the main
-   components of the MPI I/O chapter not yet implemented are
-  file interoperability and error handling.
-
-\item Added support for using ``direct I/O'' on SGI's XFS file system. 
-  Direct I/O is an optional feature of XFS in which data is moved
-  directly between the user's buffer and the storage devices, bypassing 
-  the file-system cache. This can improve performance significantly on 
-  systems with high disk bandwidth. Without high disk bandwidth,
-  regular I/O (that uses the file-system cache) perfoms better.
-  ROMIO, therefore, does not use direct I/O by default. The user can
-  turn on direct I/O (separately for reading and writing) either by
-  using environment variables or by using MPI's hints mechanism (info). 
-  To use the environment-variables method, do
-\begin{verbatim}
-       setenv MPIO_DIRECT_READ TRUE
-       setenv MPIO_DIRECT_WRITE TRUE
-\end{verbatim}
-  To use the hints method, the two keys are {\tt direct\_read} and {\tt
-  direct\_write}.  By default their values are {\tt false}. To turn on
-  direct I/O, set the values to {\tt true}. The environment variables
-  have priority over the info keys.  In other words, if the environment
-  variables are set to {\tt TRUE}, direct I/O will be used even if the
-  info keys say {\tt false}, and vice versa.  Note that direct I/O must be
-  turned on separately for reading and writing.  The environment-variables
-  method assumes that the environment variables can be read by each
-  process in the MPI job. This is not guaranteed by the MPI Standard,
-  but it works with SGI's MPI and the {\tt ch\_shmem} device of MPICH.
-
-\item Added support (new ADIO device, {\tt ad\_pvfs}) for the PVFS parallel 
-  file system for Linux clusters, developed at Clemson University
-  (see {\tt http://www.parl.clemson.edu/pvfs}). To use it, you
-  must first install PVFS and then when configuring ROMIO, specify
-  {\tt -file\_system=pvfs} in addition to any other options to {\tt
-  configure}. (As usual, you can configure for multiple file systems by
-  using ``{\tt +}''; for example, {\tt -file\_system=pvfs+ufs+nfs}.) You
-  will need to specify the path to the PVFS include files via the {\tt
-  -cflags} option to {\tt configure}, for example, \newline {\tt configure
-  -cflags=-I/usr/pvfs/include}. You will also need to specify the full
-  path name of the PVFS library. The best way to do this is via the {\tt
-  -lib} option to MPICH's {\tt configure} script (assuming you are using
-  ROMIO from within MPICH).
-
-\item Uses weak symbols (where available) for building the profiling version,
-  i.e., the PMPI routines. As a result, the size of the library is reduced
-  considerably. 
-
-\item The Makefiles use {\em virtual paths} if supported by the make
-  utility. GNU {\tt make}
-  supports it, for example. This feature allows you to untar the
-  distribution in some directory, say a slow NFS directory,
-  and compile the library (create the .o files) in another 
-  directory, say on a faster local disk. For example, if the tar file
-  has been untarred in an NFS directory called {\tt /home/thakur/romio},
-  one can compile it in a different directory, say {\tt /tmp/thakur}, as
-  follows:
-\begin{verbatim}
-        cd /tmp/thakur
-        /home/thakur/romio/configure
-        make
-\end{verbatim}
-  The .o files will be created in {\tt /tmp/thakur}; the library will be created in\newline
-  {\tt /home/thakur/romio/lib/\$ARCH/libmpio.a}.
-  This method works only if the {\tt make} utility supports {\em
-  virtual paths}. 
-  If the default {\tt make} utility does not, you can install GNU {\tt
-  make} which does, and specify it to {\tt configure} as
-\begin{verbatim}
-       /home/thakur/romio/configure -make=/usr/gnu/bin/gmake (or whatever)
-\end{verbatim}
-
-\item Lots of miscellaneous bug fixes and other enhancements.
-
-\item This version is included in MPICH 1.2.0. If you are using MPICH, you
-  need not download ROMIO separately; it gets built as part of MPICH.
-  The previous version of ROMIO is included in LAM, HP MPI, SGI MPI, and 
-  NEC MPI. NEC has also implemented the MPI-IO functions missing 
-  in ROMIO, and therefore NEC MPI has a complete implementation of MPI-IO.
-\end{itemize}
-
-
-\subsection{Major Changes in Version 1.0.1}
-
-\begin{itemize}
-\item This version is included in MPICH 1.1.1 and HP MPI 1.4.
-
-\item Added support for NEC SX-4 and created a new device {\tt ad\_sfs} for
-NEC SFS file system.
-
-\item New devices {\tt ad\_hfs} for HP HFS file system and {\tt
-ad\_xfs} for SGI XFS file system.
-
-\item Users no longer need to prefix the filename with the type of 
-file system; ROMIO determines the file-system type on its own.
-
-\item Added support for 64-bit file sizes on IBM PIOFS, SGI XFS,
-HP HFS, and NEC SFS file systems.
-
-\item {\tt MPI\_Offset} is an 8-byte integer on machines that support
-8-byte integers. It is of type {\tt long long} in C and {\tt
-integer*8} in Fortran. With a Fortran 90 compiler, you can use either
-{\tt integer*8} or  {\tt integer(kind=MPI\_OFFSET\_KIND)}. 
-If you {\tt printf} an {\tt MPI\_Offset} in C, remember to use {\tt \%lld} 
-or {\tt \%ld} as required by your compiler. (See what is used in the test 
-program {\tt romio/test/misc.c}).
-On some machines, ROMIO detects at configure time that {\tt long long} is 
-either not supported by the C compiler or it doesn't work properly.
-In such cases, configure sets {\tt MPI\_Offset} to {\tt long} in C and {\tt
-integer} in Fortran. This happens on Intel Paragon, Sun4, and FreeBSD.
-
-\item Added support for passing hints to the implementation via the
-{\tt MPI\_Info} parameter. ROMIO understands the following hints (keys
-in {\tt MPI\_Info} object):
-\texttt{cb\_buffer\_size},
-\texttt{cb\_nodes},\newline
-\texttt{ind\_rd\_buffer\_size},
-\texttt{ind\_wr\_buffer\_size} (on all but IBM PIOFS),
-\texttt{striping\_factor} (on PFS and PIOFS),
-\texttt{striping\_unit} (on PFS and PIOFS),
-\texttt{start\_iodevice} (on PFS and PIOFS),
-and \texttt{pfs\_svr\_buf} (on PFS only).
-      
-\end{itemize}
-\newpage
-
-\addcontentsline{toc}{section}{References}
-\bibliographystyle{plain}
-%% these are the "full" bibliography databases
-%\bibliography{/homes/thakur/tex/bib/papers,/homes/robl/projects/papers/pario}
-% this is the pared-down one containing only those references used in
-% users-guide.tex
-% to regenerate, uncomment the full databases above, then run 
-%   ~gropp/bin/citetags users-guide.tex | sort | uniq | \
-%        ~gropp/bin/citefind - /homes/thakur/tex/bib/papers.bib \
-%                      /homes/robl/projects/papers/pario
-\bibliography{romio}
-
-\end{document}
diff --git a/3rd-party/romio321/include/io_romio_conv.h b/3rd-party/romio321/include/io_romio_conv.h
deleted file mode 100644
index ae3ea7deb48..00000000000
--- a/3rd-party/romio321/include/io_romio_conv.h
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef MCA_IO_ROMIO_CONV_H
-#define MCA_IO_ROMIO_CONV_H
-
-/* Prefix that we add to all ROMIO symbols */
-#ifdef ROMIO_PREFIX
-#undef ROMIO_PREFIX
-#endif
-#define ROMIO_PREFIX(foo) mca_io_romio_dist_##foo
-
-/* Section 9.2 */
-/* Begin Prototypes */
-#define MPI_File_open ROMIO_PREFIX(MPI_File_open)
-#define MPI_File_close ROMIO_PREFIX(MPI_File_close)
-#define MPI_File_delete ROMIO_PREFIX(MPI_File_delete)
-#define MPI_File_set_size ROMIO_PREFIX(MPI_File_set_size)
-#define MPI_File_preallocate ROMIO_PREFIX(MPI_File_preallocate)
-#define MPI_File_get_size ROMIO_PREFIX(MPI_File_get_size)
-#define MPI_File_get_group ROMIO_PREFIX(MPI_File_get_group)
-#define MPI_File_get_amode ROMIO_PREFIX(MPI_File_get_amode)
-#define MPI_File_set_info ROMIO_PREFIX(MPI_File_set_info)
-#define MPI_File_get_info ROMIO_PREFIX(MPI_File_get_info)
-
-/* Section 9.3 */
-#define MPI_File_set_view ROMIO_PREFIX(MPI_File_set_view)
-#define MPI_File_get_view ROMIO_PREFIX(MPI_File_get_view)
-
-/* Section 9.4.2 */
-#define MPI_File_read_at ROMIO_PREFIX(MPI_File_read_at)
-#define MPI_File_read_at_all ROMIO_PREFIX(MPI_File_read_at_all)
-#define MPI_File_write_at ROMIO_PREFIX(MPI_File_write_at)
-#define MPI_File_write_at_all ROMIO_PREFIX(MPI_File_write_at_all)
-#define MPI_File_iread_at ROMIO_PREFIX(MPI_File_iread_at)
-#define MPI_File_iwrite_at ROMIO_PREFIX(MPI_File_iwrite_at)
-
-/* Section 9.4.3 */
-#define MPI_File_read ROMIO_PREFIX(MPI_File_read)
-#define MPI_File_read_all ROMIO_PREFIX(MPI_File_read_all)
-#define MPI_File_write ROMIO_PREFIX(MPI_File_write)
-#define MPI_File_write_all ROMIO_PREFIX(MPI_File_write_all)
-
-#define MPI_File_iread ROMIO_PREFIX(MPI_File_iread)
-#define MPI_File_iread_all ROMIO_PREFIX(MPI_File_iread_all)
-#define MPI_File_iwrite ROMIO_PREFIX(MPI_File_iwrite)
-#define MPI_File_iwrite_all ROMIO_PREFIX(MPI_File_iwrite_all)
-#define MPI_File_iread_at_all ROMIO_PREFIX(MPI_File_iread_at_all)
-#define MPI_File_iwrite_at_all ROMIO_PREFIX(MPI_File_iwrite_at_all)
-
-
-#define MPI_File_seek ROMIO_PREFIX(MPI_File_seek)
-#define MPI_File_get_position ROMIO_PREFIX(MPI_File_get_position)
-#define MPI_File_get_byte_offset ROMIO_PREFIX(MPI_File_get_byte_offset)
-
-/* Section 9.4.4 */
-#define MPI_File_read_shared ROMIO_PREFIX(MPI_File_read_shared)
-#define MPI_File_write_shared ROMIO_PREFIX(MPI_File_write_shared)
-#define MPI_File_iread_shared ROMIO_PREFIX(MPI_File_iread_shared)
-#define MPI_File_iwrite_shared ROMIO_PREFIX(MPI_File_iwrite_shared)
-#define MPI_File_read_ordered ROMIO_PREFIX(MPI_File_read_ordered)
-#define MPI_File_write_ordered ROMIO_PREFIX(MPI_File_write_ordered)
-#define MPI_File_seek_shared ROMIO_PREFIX(MPI_File_seek_shared)
-#define MPI_File_get_position_shared ROMIO_PREFIX(MPI_File_get_position_shared)
-
-/* Section 9.4.5 */
-#define MPI_File_read_at_all_begin ROMIO_PREFIX(MPI_File_read_at_all_begin)
-#define MPI_File_read_at_all_end ROMIO_PREFIX(MPI_File_read_at_all_end)
-#define MPI_File_write_at_all_begin ROMIO_PREFIX(MPI_File_write_at_all_begin)
-#define MPI_File_write_at_all_end ROMIO_PREFIX(MPI_File_write_at_all_end)
-#define MPI_File_read_all_begin ROMIO_PREFIX(MPI_File_read_all_begin)
-#define MPI_File_read_all_end ROMIO_PREFIX(MPI_File_read_all_end)
-#define MPI_File_write_all_begin ROMIO_PREFIX(MPI_File_write_all_begin)
-#define MPI_File_write_all_end ROMIO_PREFIX(MPI_File_write_all_end)
-#define MPI_File_read_ordered_begin ROMIO_PREFIX(MPI_File_read_ordered_begin)
-#define MPI_File_read_ordered_end ROMIO_PREFIX(MPI_File_read_ordered_end)
-#define MPI_File_write_ordered_begin ROMIO_PREFIX(MPI_File_write_ordered_begin)
-#define MPI_File_write_ordered_end ROMIO_PREFIX(MPI_File_write_ordered_end)
-
-/* Section 9.5.1 */
-#define MPI_File_get_type_extent ROMIO_PREFIX(MPI_File_get_type_extent)
-
-/* Section 9.6.1 */
-#define MPI_File_set_atomicity ROMIO_PREFIX(MPI_File_set_atomicity)
-#define MPI_File_get_atomicity ROMIO_PREFIX(MPI_File_get_atomicity)
-#define MPI_File_sync ROMIO_PREFIX(MPI_File_sync)
-
-/* Section 4.13.3 */
-#define MPI_File_set_errhandler ROMIO_PREFIX(MPI_File_set_errhandler)
-#define MPI_File_get_errhandler ROMIO_PREFIX(MPI_File_get_errhandler)
-/* End Prototypes */
-
-#define MPI_Register_datarep ROMIO_PREFIX(MPI_Register_datarep)
-
-/* JMS these don't seem to work... */
-#define MPI_File_f2c ROMIO_PREFIX(MPI_File_f2c)
-#define MPI_File_c2f ROMIO_PREFIX(MPI_File_c2f)
-
-#define MPIO_Request_c2f ROMIO_PREFIX(MPIO_Request_c2f)
-#define MPIO_Request_f2c ROMIO_PREFIX(MPIO_Request_f2c)
-
-/* Conversion of MPI_File and MPIO_Request */
-#define MPI_File ROMIO_PREFIX(MPI_File)
-
-/* Open MPI's mpi.h #define's MPI_FILE_NULL, so we need to undef it
-   here and allow it to be re-assigned to whatever ROMIO wants */
-#undef MPI_FILE_NULL
-
-/* Let's not use MPIR_Status_set_bytes */
-#ifndef MPIR_Status_set_bytes
-#define MPIR_Status_set_bytes ROMIO_PREFIX(MPIR_Status_set_bytes)
-#endif
-
-#endif /* MCA_IO_ROMIO_CONV_H */
diff --git a/3rd-party/romio321/include/mpio.h.in b/3rd-party/romio321/include/mpio.h.in
deleted file mode 100644
index 9ef709dc651..00000000000
--- a/3rd-party/romio321/include/mpio.h.in
+++ /dev/null
@@ -1,558 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-/* user include file for MPI-IO programs */
-
-#ifndef MPIO_INCLUDE
-#define MPIO_INCLUDE
-
-#include "mpi.h"
-/* Open MPI: We need to rename almost all of these functions, as well
-   a the types to be names that conform to the prefix rule */
-#include "io_romio_conv.h"
-
-#define MPIIMPL_HAVE_MPI_COMBINER_DARRAY 1
-#define MPIIMPL_HAVE_MPI_TYPE_CREATE_DARRAY 1
-#define MPIIMPL_HAVE_MPI_COMBINER_SUBARRAY 1
-#define MPIIMPL_HAVE_MPI_TYPE_CREATE_DARRAY 1
-#define MPIIMPL_HAVE_MPI_COMBINER_DUP 1
-#define MPICH_ATTR_POINTER_WITH_TYPE_TAG(x,y)
-
-#if defined(__cplusplus)
-extern "C" {
-#endif
-
-#define ROMIO_VERSION 126 /* version 1.2.6 */
-
-/* define MPI-IO datatypes and constants */
-
-#ifndef MPI_FILE_DEFINED
-typedef struct ADIOI_FileD *MPI_File;
-#endif
-
-@DEFINE_HAVE_MPI_GREQUEST@
-#ifndef HAVE_MPI_GREQUEST
-typedef struct ADIOI_RequestD *MPIO_Request;  
-#else
-#define MPIO_Request MPI_Request
-#define MPIO_USES_MPI_REQUEST
-/* Also rename the MPIO routines to get the MPI versions */
-#define MPIO_Wait MPI_Wait
-#define MPIO_Test MPI_Test
-#define PMPIO_Wait PMPI_Wait
-#define PMPIO_Test PMPI_Test
-#endif
-#define MPIO_REQUEST_DEFINED
-
-@DEFINE_HAVE_MPI_GREQUEST_EXTENSIONS@
-
-#ifndef HAVE_MPI_OFFSET
-@DEFINE_MPI_OFFSET@
-/* If we needed to define MPI_Offset, then we also need to make
-   this definition. */
-#ifndef HAVE_MPI_DATAREP_FUNCTIONS
-#define HAVE_MPI_DATAREP_FUNCTIONS
-typedef int (MPI_Datarep_conversion_function)(void *, MPI_Datatype, int, 
-             void *, MPI_Offset, void *);
-typedef int (MPI_Datarep_extent_function)(MPI_Datatype datatype, MPI_Aint *,
-					  void *);
-#endif
-#endif
-
-#ifndef NEEDS_MPI_FINT
-@NEEDS_MPI_FINT@
-#endif
-#ifdef NEEDS_MPI_FINT
-typedef int MPI_Fint; 
-#endif
-
-#ifndef HAVE_MPI_INFO
-@HAVE_MPI_INFO@
-#endif
-#ifndef HAVE_MPI_INFO
-  typedef struct MPIR_Info *MPI_Info;
-# define MPI_INFO_NULL         ((MPI_Info) 0)
-# define MPI_MAX_INFO_KEY       255
-# define MPI_MAX_INFO_VAL      1024
-#endif
-
-#define MPI_MODE_RDONLY              2  /* ADIO_RDONLY */
-#define MPI_MODE_RDWR                8  /* ADIO_RDWR  */
-#define MPI_MODE_WRONLY              4  /* ADIO_WRONLY  */
-#define MPI_MODE_CREATE              1  /* ADIO_CREATE */ 
-#define MPI_MODE_EXCL               64  /* ADIO_EXCL */
-#define MPI_MODE_DELETE_ON_CLOSE    16  /* ADIO_DELETE_ON_CLOSE */
-#define MPI_MODE_UNIQUE_OPEN        32  /* ADIO_UNIQUE_OPEN */
-#define MPI_MODE_APPEND            128  /* ADIO_APPEND */
-#define MPI_MODE_SEQUENTIAL        256  /* ADIO_SEQUENTIAL */
-
-#define MPI_DISPLACEMENT_CURRENT   -54278278
-
-#ifndef MPICH
-/* FIXME: Make sure that we get a consistent definition of MPI_FILE_NULL
-	in MPICH */
-/* MPICH defines null object handles differently */
-#define MPI_FILE_NULL           ((MPI_File) 0)
-#endif
-#define MPIO_REQUEST_NULL       ((MPIO_Request) 0)
-
-#define MPI_SEEK_SET            600
-#define MPI_SEEK_CUR            602
-#define MPI_SEEK_END            604
-
-/* Open MPI: don't define MPI_MAX_DATAREP_STRING here; it's defined in
-   OMPI's mpi.h. */
-#ifndef OPEN_MPI
-#define MPI_MAX_DATAREP_STRING  128
-#endif
-
-#ifndef HAVE_MPI_DARRAY_SUBARRAY
-@HAVE_MPI_DARRAY_SUBARRAY@
-#endif
-#ifndef HAVE_MPI_DARRAY_SUBARRAY
-#  define MPI_ORDER_C             56
-#  define MPI_ORDER_FORTRAN       57
-#  define MPI_DISTRIBUTE_BLOCK    121
-#  define MPI_DISTRIBUTE_CYCLIC   122
-#  define MPI_DISTRIBUTE_NONE     123
-#  define MPI_DISTRIBUTE_DFLT_DARG -49767
-#endif
-
-
-/* MPI-IO function prototypes */
-
-/* The compiler must support ANSI C style prototypes, otherwise 
-   long long constants (e.g. 0) may get passed as ints. */
-
-#ifndef HAVE_PRAGMA_HP_SEC_DEF
-
-/* Section 9.2 */
-/* Begin Prototypes */
-int MPI_File_open(MPI_Comm comm, const char *filename, int amode, MPI_Info info, MPI_File *fh);
-int MPI_File_close(MPI_File *fh);
-int MPI_File_delete(const char *filename, MPI_Info info);
-int MPI_File_set_size(MPI_File fh, MPI_Offset size);
-int MPI_File_preallocate(MPI_File fh, MPI_Offset size);
-int MPI_File_get_size(MPI_File fh, MPI_Offset *size);
-int MPI_File_get_group(MPI_File fh, MPI_Group *group);
-int MPI_File_get_amode(MPI_File fh, int *amode);
-int MPI_File_set_info(MPI_File fh, MPI_Info info);
-int MPI_File_get_info(MPI_File fh, MPI_Info *info_used);
-
-/* Section 9.3 */
-int MPI_File_set_view(MPI_File fh, MPI_Offset disp, MPI_Datatype etype, MPI_Datatype filetype,
-                      const char *datarep, MPI_Info info);
-int MPI_File_get_view(MPI_File fh, MPI_Offset *disp, MPI_Datatype *etype, MPI_Datatype *filetype,
-                      char *datarep);
-
-/* Section 9.4.2 */
-int MPI_File_read_at(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype,
-                     MPI_Status *status) MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5);
-int MPI_File_read_at_all(MPI_File fh, MPI_Offset offset, void * buf, int count,
-                         MPI_Datatype datatype, MPI_Status *status)
-    MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5);
-int MPI_File_write_at(MPI_File fh, MPI_Offset offset, const void * buf, int count,
-                      MPI_Datatype datatype, MPI_Status *status)
-    MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5);
-int MPI_File_write_at_all(MPI_File fh, MPI_Offset offset, const void *buf, int count,
-                          MPI_Datatype datatype, MPI_Status *status)
-    MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5);
-
-/* nonblocking calls currently use MPIO_Request, because generalized
-   requests not yet implemented. For the same reason, MPIO_Test and 
-   MPIO_Wait are used to test and wait on nonblocking I/O requests */ 
-int MPI_File_iread_at(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype,
-                      MPIO_Request *request) MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5);
-int MPI_File_iwrite_at(MPI_File fh, MPI_Offset offset, const void *buf, int count,
-                       MPI_Datatype datatype, MPIO_Request *request)
-    MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5);
-
-/* Section 9.4.3 */
-int MPI_File_read(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status)
-    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-int MPI_File_read_all(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status)
-    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-int MPI_File_write(MPI_File fh, const void *buf, int count, MPI_Datatype datatype,
-                   MPI_Status *status) MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-int MPI_File_write_all(MPI_File fh, const void *buf, int count, MPI_Datatype datatype,
-                       MPI_Status *status) MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-
-/* nonblocking calls currently use MPIO_Request, because generalized
-   requests not yet implemented. For the same reason, MPIO_Test and 
-   MPIO_Wait are used to test and wait on nonblocking I/O requests */ 
-
-int MPI_File_iread(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPIO_Request *request)
-    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-int MPI_File_iwrite(MPI_File fh, const void *buf, int count, MPI_Datatype datatype,
-                    MPIO_Request *request) MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-
-int MPI_File_seek(MPI_File fh, MPI_Offset offset, int whence);
-int MPI_File_get_position(MPI_File fh, MPI_Offset *offset);
-int MPI_File_get_byte_offset(MPI_File fh, MPI_Offset offset, MPI_Offset *disp);
-
-/* Section 9.4.4 */
-int MPI_File_read_shared(MPI_File fh, void *buf, int count, MPI_Datatype datatype,
-                         MPI_Status *status) MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-int MPI_File_write_shared(MPI_File fh, const void *buf, int count, MPI_Datatype datatype,
-                          MPI_Status *status) MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-int MPI_File_iread_shared(MPI_File fh, void *buf, int count, MPI_Datatype datatype,
-                          MPIO_Request *request) MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-int MPI_File_iwrite_shared(MPI_File fh, const void *buf, int count, MPI_Datatype datatype,
-                           MPIO_Request *request) MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-int MPI_File_read_ordered(MPI_File fh, void *buf, int count, MPI_Datatype datatype,
-                          MPI_Status *status) MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-int MPI_File_write_ordered(MPI_File fh, const void *buf, int count, MPI_Datatype datatype,
-                           MPI_Status *status) MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-int MPI_File_seek_shared(MPI_File fh, MPI_Offset offset, int whence);
-int MPI_File_get_position_shared(MPI_File fh, MPI_Offset *offset);
-
-/* Section 9.4.5 */
-int MPI_File_read_at_all_begin(MPI_File fh, MPI_Offset offset, void *buf, int count,
-                               MPI_Datatype datatype) MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5);
-int MPI_File_read_at_all_end(MPI_File fh, void *buf, MPI_Status *status);
-int MPI_File_write_at_all_begin(MPI_File fh, MPI_Offset offset, const void *buf, int count,
-                                MPI_Datatype datatype) MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5);
-int MPI_File_write_at_all_end(MPI_File fh, const void *buf, MPI_Status *status);
-int MPI_File_read_all_begin(MPI_File fh, void *buf, int count, MPI_Datatype datatype)
-    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-int MPI_File_read_all_end(MPI_File fh, void *buf, MPI_Status *status);
-int MPI_File_write_all_begin(MPI_File fh, const void *buf, int count, MPI_Datatype datatype)
-    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-int MPI_File_write_all_end(MPI_File fh, const void *buf, MPI_Status *status);
-int MPI_File_read_ordered_begin(MPI_File fh, void *buf, int count, MPI_Datatype datatype)
-    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-int MPI_File_read_ordered_end(MPI_File fh, void *buf, MPI_Status *status);
-int MPI_File_write_ordered_begin(MPI_File fh, const void *buf, int count, MPI_Datatype datatype)
-    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-int MPI_File_write_ordered_end(MPI_File fh, const void *buf, MPI_Status *status);
-
-/* Section 9.5.1 */
-int MPI_File_get_type_extent(MPI_File fh, MPI_Datatype datatype, MPI_Aint *extent);
-
-/* Section 9.5.3 */
-int MPI_Register_datarep(const char *datarep, MPI_Datarep_conversion_function *read_conversion_fn,
-			 MPI_Datarep_conversion_function *write_conversion_fn,
-			 MPI_Datarep_extent_function *dtype_file_extent_fn, void *extra_state);
-
-/* Section 9.6.1 */
-int MPI_File_set_atomicity(MPI_File fh, int flag);
-int MPI_File_get_atomicity(MPI_File fh, int *flag);
-int MPI_File_sync(MPI_File fh);
-
-/* Section 4.13.3 */
-#ifndef MPICH
-/* MPICH provides these definitions */
-int MPI_File_set_errhandler(MPI_File file, MPI_Errhandler errhandler);
-int MPI_File_get_errhandler(MPI_File file, MPI_Errhandler *errhandler);
-#endif
-
-/* For MPI 3.1 */
-int MPI_File_iread_at_all(MPI_File fh, MPI_Offset offset, void *buf, int count,
-                           MPI_Datatype datatype, MPI_Request *request)
-    MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5);
-int MPI_File_iwrite_at_all(MPI_File fh, MPI_Offset offset, const void *buf, int count,
-                            MPI_Datatype datatype, MPI_Request *request)
-    MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5);
-int MPI_File_iread_all(MPI_File fh, void *buf, int count, MPI_Datatype datatype,
-                        MPI_Request *request)
-    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-int MPI_File_iwrite_all(MPI_File fh, const void *buf, int count, MPI_Datatype datatype,
-                         MPI_Request *request)
-    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-/* End Prototypes */
-
-#ifndef HAVE_MPI_DARRAY_SUBARRAY
-/* Section 4.14.4 */
-int MPI_Type_create_subarray(int ndims, const int array_of_sizes[], const int array_of_subsizes[],
-                             const int array_of_starts[], int order, MPI_Datatype oldtype,
-                             MPI_Datatype *newtype);
-
-/* Section 4.14.5 */
-int MPI_Type_create_darray(int size, int rank, int ndims, const int array_of_gsizes[],
-                           const int array_of_distribs[], const int array_of_dargs[],
-                           const int array_of_psizes, int order, MPI_Datatype oldtype,
-                           MPI_Datatype *newtype);
-#endif
-
-/* The globus2 device has to rename MPI_ symbols in order to use the vendor
-   MPI as one of its transport mechanisms.  Therefore, the following undefines
-   should only happen if MPICH_RENAMING_MPI_FUNCS is not defined. */
-/* Section 4.12.4 */
-#if !defined(MPICH_RENAMING_MPI_FUNCS)
-#ifdef MPI_File_f2c
-#undef MPI_File_f2c
-#endif
-#ifdef MPI_File_c2f
-#undef MPI_File_c2f
-#endif
-#endif
-/* above needed for some versions of mpi.h in MPICH!! */
-/* Open MPI: I can't seem to make these #define properly.  Oh well --
-   we don't need them anyway :-( */
-#if 0
-MPI_File MPI_File_f2c(MPI_Fint file);
-MPI_Fint MPI_File_c2f(MPI_File file);
-#endif
-
-#ifndef HAVE_MPI_GREQUEST
-/* The following functions are required if generalized requests are not
-   available, because in that case, an MPIO_Request object
-   is currently used for nonblocking I/O. */
-int MPIO_Test(MPIO_Request *request, int *flag, MPI_Status *status);
-int MPIO_Wait(MPIO_Request *request, MPI_Status *status);
-int MPIO_Testall(int count, MPIO_Request array_of_requests[], int *flag,
-                 MPI_Status array_of_statuses[]);
-int MPIO_Waitall(int count, MPIO_Request array_of_requests[], MPI_Status array_of_statuses[]);
-int MPIO_Testany(int count, MPIO_Request array_of_requests[], int *indx, int *flag,
-                 MPI_Status *status);
-int MPIO_Waitany(int count, MPIO_Request array_of_requests[], int *indx, MPI_Status *status);
-int MPIO_Waitsome(int incount, MPIO_Request array_of_requests[], int *outcount,
-                  int array_of_indices[], MPI_Status array_of_statuses[]);
-int MPIO_Testsome(int incount, MPIO_Request array_of_requests[], int *outcount,
-                  int array_of_indices[], MPI_Status array_of_statuses[]);
-
-MPI_Fint MPIO_Request_c2f(MPIO_Request request);
-MPIO_Request MPIO_Request_f2c(MPI_Fint request);
-#endif /* HAVE_MPI_GREQUEST */
-
-/* info functions if not defined in the MPI implementation */
-#ifndef HAVE_MPI_INFO
-
-int MPI_Info_create(MPI_Info *info);
-int MPI_Info_set(MPI_Info info, const char *key, const char *value);
-int MPI_Info_delete(MPI_Info info, const char *key);
-int MPI_Info_get(MPI_Info info, const char *key, int valuelen, char *value, int *flag);
-int MPI_Info_get_valuelen(MPI_Info info, const char *key, int *valuelen, int *flag);
-int MPI_Info_get_nkeys(MPI_Info info, int *nkeys);
-int MPI_Info_get_nthkey(MPI_Info info, int n, char *key);
-int MPI_Info_dup(MPI_Info info, MPI_Info *newinfo);
-int MPI_Info_free(MPI_Info *info);
-
-/* The globus2 device has to rename MPI_ symbols in order to use the vendor
-   MPI as one of its transport mechanisms.  Therefore, the following undefines
-   should only happen if MPICH_RENAMING_MPI_FUNCS is not defined. */
-#if !defined(MPICH_RENAMING_MPI_FUNCS)
-#ifdef MPI_Info_f2c
-#undef MPI_Info_f2c
-#endif
-#ifdef MPI_Info_c2f
-#undef MPI_Info_c2f
-#endif
-#endif
-/* above needed for some versions of mpi.h in MPICH!! */
-/* Open MPI: we don't need these in ROMIO */
-#if 0
-MPI_Fint MPI_Info_c2f(MPI_Info info);
-MPI_Info MPI_Info_f2c(MPI_Fint info);
-#endif
-#endif
-
-#endif   /* HAVE_PRAGMA_HP_SEC_DEF */
-
-
-/**************** BINDINGS FOR THE PROFILING INTERFACE ***************/
-
-/* Open MPI: We don't want any of the profiling layer */
-#if 0
-
-
-/* Section 9.2 */
-int PMPI_File_open(MPI_Comm, const char *, int, MPI_Info, MPI_File *);
-int PMPI_File_close(MPI_File *);
-int PMPI_File_delete(const char *, MPI_Info);
-int PMPI_File_set_size(MPI_File, MPI_Offset);
-int PMPI_File_preallocate(MPI_File, MPI_Offset);
-int PMPI_File_get_size(MPI_File, MPI_Offset *);
-int PMPI_File_get_group(MPI_File, MPI_Group *);
-int PMPI_File_get_amode(MPI_File, int *);
-int PMPI_File_set_info(MPI_File, MPI_Info);
-int PMPI_File_get_info(MPI_File, MPI_Info *);
-
-/* Section 9.3 */
-int PMPI_File_set_view(MPI_File, MPI_Offset, 
-    MPI_Datatype, MPI_Datatype, const char *, MPI_Info);
-int PMPI_File_get_view(MPI_File, MPI_Offset *, 
-      MPI_Datatype *, MPI_Datatype *, char *);
-
-/* Section 9.4.2 */
-int PMPI_File_read_at(MPI_File, MPI_Offset, void *,
-	      int, MPI_Datatype, MPI_Status *)
-              MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5);
-int PMPI_File_read_at_all(MPI_File, MPI_Offset, void *,
-	      int, MPI_Datatype, MPI_Status *)
-              MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5);
-int PMPI_File_write_at(MPI_File, MPI_Offset, const void *,
-	      int, MPI_Datatype, MPI_Status *)
-              MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5);
-int PMPI_File_write_at_all(MPI_File, MPI_Offset, const void *,
-	      int, MPI_Datatype, MPI_Status *)
-              MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5);
-
-/* nonblocking calls currently use MPIO_Request, because generalized
-   requests not yet implemented. For the same reason, MPIO_Test and 
-   MPIO_Wait are used to test and wait on nonblocking I/O requests */ 
-
-int PMPI_File_iread_at(MPI_File, MPI_Offset, void *,
-	      int, MPI_Datatype, MPIO_Request *)
-              MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5);
-int PMPI_File_iwrite_at(MPI_File, MPI_Offset, const void *,
-	      int, MPI_Datatype, MPIO_Request *)
-              MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5);
-
-/* Section 9.4.3 */
-int PMPI_File_read(MPI_File, void *, int, MPI_Datatype, MPI_Status *)
-                   MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-int PMPI_File_read_all(MPI_File, void *, int, MPI_Datatype, MPI_Status *)
-                       MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-int PMPI_File_write(MPI_File, const void *, int, MPI_Datatype, MPI_Status *)
-                    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-int PMPI_File_write_all(MPI_File, const void *, int, MPI_Datatype, MPI_Status *)
-                        MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-
-/* nonblocking calls currently use MPIO_Request, because generalized
-   requests not yet implemented. For the same reason, MPIO_Test and 
-   MPIO_Wait are used to test and wait on nonblocking I/O requests */ 
-
-int PMPI_File_iread(MPI_File, void *, int, MPI_Datatype, MPIO_Request *)
-                    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-int PMPI_File_iwrite(MPI_File, const void *, int, MPI_Datatype, MPIO_Request *)
-                     MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-
-int PMPI_File_seek(MPI_File, MPI_Offset, int);
-int PMPI_File_get_position(MPI_File, MPI_Offset *);
-int PMPI_File_get_byte_offset(MPI_File, MPI_Offset, MPI_Offset *);
-
-/* Section 9.4.4 */
-int PMPI_File_read_shared(MPI_File, void *, int, MPI_Datatype, MPI_Status *)
-                          MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-int PMPI_File_write_shared(MPI_File, const void *, int, MPI_Datatype, MPI_Status *)
-                           MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-int PMPI_File_iread_shared(MPI_File, void *, int, 
-			   MPI_Datatype, MPIO_Request *)
-                           MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-int PMPI_File_iwrite_shared(MPI_File, const void *, int,
-			    MPI_Datatype, MPIO_Request *)
-                            MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-int PMPI_File_read_ordered(MPI_File, void *, int, MPI_Datatype, MPI_Status *)
-                           MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-int PMPI_File_write_ordered(MPI_File, const void *, int, MPI_Datatype, MPI_Status *)
-                            MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-int PMPI_File_seek_shared(MPI_File, MPI_Offset, int);
-int PMPI_File_get_position_shared(MPI_File, MPI_Offset *);
-
-/* Section 9.4.5 */
-int PMPI_File_read_at_all_begin(MPI_File, MPI_Offset, void *,
-                               int, MPI_Datatype)
-                               MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5);
-int PMPI_File_read_at_all_end(MPI_File, void *, MPI_Status *);
-int PMPI_File_write_at_all_begin(MPI_File, MPI_Offset, const void *,
-                                 int, MPI_Datatype)
-                                 MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5);
-int PMPI_File_write_at_all_end(MPI_File, const void *, MPI_Status *);
-int PMPI_File_read_all_begin(MPI_File, void *, int, MPI_Datatype)
-                             MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-int PMPI_File_read_all_end(MPI_File, void *, MPI_Status *);
-int PMPI_File_write_all_begin(MPI_File, const void *, int, MPI_Datatype)
-                              MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-int PMPI_File_write_all_end(MPI_File, const void *, MPI_Status *);
-int PMPI_File_read_ordered_begin(MPI_File, void *, int, MPI_Datatype)
-                                 MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-int PMPI_File_read_ordered_end(MPI_File, void *, MPI_Status *);
-int PMPI_File_write_ordered_begin(MPI_File, const void *, int, MPI_Datatype)
-                                  MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-int PMPI_File_write_ordered_end(MPI_File, const void *, MPI_Status *);
-
-/* Section 9.5.1 */
-int PMPI_File_get_type_extent(MPI_File, MPI_Datatype, MPI_Aint *);
-
-/* Section 9.5.3 */
-int PMPI_Register_datarep(const char *,
-			 MPI_Datarep_conversion_function *,
-			 MPI_Datarep_conversion_function *,
-			 MPI_Datarep_extent_function *,
-			 void *);
-
-/* Section 9.6.1 */
-int PMPI_File_set_atomicity(MPI_File, int);
-int PMPI_File_get_atomicity(MPI_File, int *);
-int PMPI_File_sync(MPI_File);
-
-/* Section 4.13.3 */
-#ifndef MPICH
-/* MPICH provides these definitions */
-int PMPI_File_set_errhandler( MPI_File, MPI_Errhandler );
-int PMPI_File_get_errhandler( MPI_File, MPI_Errhandler * );
-#endif
-
-/* For MPI 3.1 */
-int PMPI_File_iread_at_all(MPI_File fh, MPI_Offset offset, void *buf, int count,
-                            MPI_Datatype datatype, MPI_Request *request)
-    MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5);
-int PMPI_File_iwrite_at_all(MPI_File fh, MPI_Offset offset, const void *buf, int count,
-                             MPI_Datatype datatype, MPI_Request *request)
-    MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5);
-int PMPI_File_iread_all(MPI_File fh, void *buf, int count, MPI_Datatype datatype,
-                         MPI_Request *request)
-    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-int PMPI_File_iwrite_all(MPI_File fh, const void *buf, int count, MPI_Datatype datatype,
-                          MPI_Request *request)
-    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4);
-
-#ifndef HAVE_MPI_DARRAY_SUBARRAY
-/* Section 4.14.4 */
-int PMPI_Type_create_subarray(int, int *, int *, int *, int, 
-                      MPI_Datatype, MPI_Datatype *);
-
-/* Section 4.14.5 */
-int PMPI_Type_create_darray(int, int, int, int *, int *, 
-                    int *, int *, int, MPI_Datatype, MPI_Datatype *);
-#endif
-
-/* Section 4.12.4 */
-MPI_File PMPI_File_f2c(MPI_Fint);
-MPI_Fint PMPI_File_c2f(MPI_File);
-
-#ifndef HAVE_MPI_GREQUEST
-/* The following functions are required if generalized requests are not
-   available, because in that case, an MPIO_Request object
-   is currently used for nonblocking I/O. */
-int PMPIO_Test(MPIO_Request *, int *, MPI_Status *);
-int PMPIO_Wait(MPIO_Request *, MPI_Status *);
-int PMPIO_Testall(int, MPIO_Request *, int *, MPI_Status *);
-int PMPIO_Waitall(int, MPIO_Request *, MPI_Status *);
-int PMPIO_Testany(int, MPIO_Request *, int *, int *, MPI_Status *);
-int PMPIO_Waitany(int, MPIO_Request *, int *, MPI_Status *);
-int PMPIO_Waitsome(int, MPIO_Request *, int *, int *, MPI_Status *);
-int PMPIO_Testsome(int, MPIO_Request *, int *, int *, MPI_Status *);
-MPI_Fint PMPIO_Request_c2f(MPIO_Request);
-MPIO_Request PMPIO_Request_f2c(MPI_Fint);
-#endif /* HAVE_MPI_GREQUEST */
-
-/* info functions if not defined in the MPI implementation */
-#ifndef HAVE_MPI_INFO
-
-int PMPI_Info_create(MPI_Info *);
-int PMPI_Info_set(MPI_Info, char *, char *);
-int PMPI_Info_delete(MPI_Info, char *);
-int PMPI_Info_get(MPI_Info, char *, int, char *, int *);
-int PMPI_Info_get_valuelen(MPI_Info, char *, int *, int *);
-int PMPI_Info_get_nkeys(MPI_Info, int *);
-int PMPI_Info_get_nthkey(MPI_Info, int, char *);
-int PMPI_Info_dup(MPI_Info, MPI_Info *);
-int PMPI_Info_free(MPI_Info *);
-
-MPI_Fint PMPI_Info_c2f(MPI_Info);
-MPI_Info PMPI_Info_f2c(MPI_Fint);
-#endif
-
-#endif /* Open MPI: We don't want any of the profiling layer */
-
-#if defined(__cplusplus)
-}
-#endif
-
-#endif
diff --git a/3rd-party/romio321/include/mpiof.h.in b/3rd-party/romio321/include/mpiof.h.in
deleted file mode 100644
index fa945807bdc..00000000000
--- a/3rd-party/romio321/include/mpiof.h.in
+++ /dev/null
@@ -1,46 +0,0 @@
-! 
-!     Copyright (C) 1997 University of Chicago. 
-!     See COPYRIGHT notice in top-level directory.
-!
-! 
-!    user include file for Fortran MPI-IO programs 
-!
-      INTEGER MPI_MODE_RDONLY, MPI_MODE_RDWR, MPI_MODE_WRONLY
-      INTEGER MPI_MODE_DELETE_ON_CLOSE, MPI_MODE_UNIQUE_OPEN
-      INTEGER MPI_MODE_CREATE, MPI_MODE_EXCL
-      INTEGER MPI_MODE_APPEND, MPI_MODE_SEQUENTIAL
-      PARAMETER (MPI_MODE_RDONLY=2, MPI_MODE_RDWR=8, MPI_MODE_WRONLY=4)
-      PARAMETER (MPI_MODE_CREATE=1, MPI_MODE_DELETE_ON_CLOSE=16)
-      PARAMETER (MPI_MODE_UNIQUE_OPEN=32, MPI_MODE_EXCL=64)
-      PARAMETER (MPI_MODE_APPEND=128, MPI_MODE_SEQUENTIAL=256)
-!
-      INTEGER MPI_FILE_NULL
-      PARAMETER (MPI_FILE_NULL=0)
-!
-      INTEGER MPI_MAX_DATAREP_STRING
-      PARAMETER (MPI_MAX_DATAREP_STRING=128)
-!
-      INTEGER MPI_SEEK_SET, MPI_SEEK_CUR, MPI_SEEK_END
-      PARAMETER (MPI_SEEK_SET=600, MPI_SEEK_CUR=602, MPI_SEEK_END=604)
-!
-      INTEGER MPIO_REQUEST_NULL
-      PARAMETER (MPIO_REQUEST_NULL=0)
-!
-      @FORTRAN_MPI_OFFSET@ MPI_DISPLACEMENT_CURRENT
-      PARAMETER (MPI_DISPLACEMENT_CURRENT=-54278278)
-!
-@MPI_OFFSET_KIND1@
-@MPI_OFFSET_KIND2@
-!
-@MPI_FARRAY1@
-@MPI_FARRAY2@
-@MPI_FARRAY3@
-@MPI_FARRAY4@
-@MPI_FARRAY5@
-@MPI_FARRAY6@
-@MPI_FARRAY7@
-!
-@MPI_FINFO1@
-@MPI_FINFO2@
-@MPI_FINFO3@
-@MPI_FINFO4@
diff --git a/3rd-party/romio321/include/ompi_grequestx.h b/3rd-party/romio321/include/ompi_grequestx.h
deleted file mode 100644
index 3aae40d8fb5..00000000000
--- a/3rd-party/romio321/include/ompi_grequestx.h
+++ /dev/null
@@ -1,34 +0,0 @@
-#define HAVE_MPI_GREQUEST_EXTENSIONS 1
-
-extern void opal_progress(void);
-
-typedef int (MPIX_Grequest_poll_function)(void *, MPI_Status *);
-typedef int (MPIX_Grequest_wait_function)(int, void **, double, MPI_Status *);
-typedef int MPIX_Grequest_class;
-
-extern int ompi_grequestx_start(
-    MPI_Grequest_query_function *gquery,
-    MPI_Grequest_free_function *gfree,
-    MPI_Grequest_cancel_function *gcancel,
-    MPIX_Grequest_poll_function *gpoll,
-    void* gstate,
-    MPI_Request* request);
-
-extern int ompi_grequestx_class_create(
-    MPI_Grequest_query_function *gquery,
-    MPI_Grequest_free_function *gfree,
-    MPI_Grequest_cancel_function *gcancel,
-    MPIX_Grequest_poll_function *gpoll,
-    MPIX_Grequest_wait_function *gwait,
-    MPIX_Grequest_class *greq_class);
-
-extern int ompi_grequestx_class_allocate(
-    MPIX_Grequest_class greq_class,
-    void *extra_state,
-    MPI_Request* request);
-
-#define MPIR_Ext_cs_yield opal_progress
-#define PMPIX_Grequest_class_allocate(greq_class,extra_state,request) ompi_grequestx_class_allocate(greq_class,extra_state,request)
-#define PMPIX_Grequest_class_create(query_fn,free_fn,cancel_fn,poll_fn,wait_fn,greq_class) ompi_grequestx_class_create(query_fn,free_fn,cancel_fn,poll_fn,wait_fn,greq_class)
-#define PMPIX_Grequest_start(query_fn,free_fn,cancel_fn,poll_fn,extra_state,request) ompi_grequestx_start(query_fn,free_fn,cancel_fn,poll_fn,extra_state,request)
-
diff --git a/3rd-party/romio321/localdefs.in b/3rd-party/romio321/localdefs.in
deleted file mode 100644
index 94fd9508dae..00000000000
--- a/3rd-party/romio321/localdefs.in
+++ /dev/null
@@ -1,7 +0,0 @@
-#! /bin/sh
-
-# Append ROMIO library dependencies to the global list
-WRAPPER_LIBS="$WRAPPER_LIBS @LIBS@"
-
-MPI_OFFSET_TYPE="@MPI_OFFSET_TYPE@"
-FORTRAN_MPI_OFFSET="@FORTRAN_MPI_OFFSET@"
diff --git a/3rd-party/romio321/mpi-io/Makefile.mk b/3rd-party/romio321/mpi-io/Makefile.mk
deleted file mode 100644
index 264976e06c1..00000000000
--- a/3rd-party/romio321/mpi-io/Makefile.mk
+++ /dev/null
@@ -1,106 +0,0 @@
-## -*- Mode: Makefile; -*-
-## vim: set ft=automake :
-##
-## (C) 2011 by Argonne National Laboratory.
-##     See COPYRIGHT in top-level directory.
-##
-
-include $(top_srcdir)/mpi-io/glue/Makefile.mk
-include $(top_srcdir)/mpi-io/fortran/Makefile.mk
-
-AM_CPPFLAGS += -I$(top_builddir)/mpi-io -I$(top_srcdir)/mpi-io
-noinst_HEADERS += mpi-io/mpioimpl.h mpi-io/mpioprof.h
-
-romio_mpi_sources +=          \
-    mpi-io/close.c            \
-    mpi-io/delete.c           \
-    mpi-io/fsync.c            \
-    mpi-io/get_amode.c        \
-    mpi-io/get_atom.c         \
-    mpi-io/get_bytoff.c       \
-    mpi-io/get_extent.c       \
-    mpi-io/get_group.c        \
-    mpi-io/get_info.c         \
-    mpi-io/get_posn.c         \
-    mpi-io/get_posn_sh.c      \
-    mpi-io/get_size.c         \
-    mpi-io/get_view.c         \
-    mpi-io/iread.c            \
-    mpi-io/iread_all.c        \
-    mpi-io/iread_at.c         \
-    mpi-io/iread_atall.c      \
-    mpi-io/iread_sh.c         \
-    mpi-io/iwrite.c           \
-    mpi-io/iwrite_all.c       \
-    mpi-io/iwrite_at.c        \
-    mpi-io/iwrite_atall.c     \
-    mpi-io/iwrite_sh.c        \
-    mpi-io/open.c             \
-    mpi-io/prealloc.c         \
-    mpi-io/rd_atallb.c        \
-    mpi-io/rd_atalle.c        \
-    mpi-io/read.c             \
-    mpi-io/read_all.c         \
-    mpi-io/read_allb.c        \
-    mpi-io/read_alle.c        \
-    mpi-io/read_at.c          \
-    mpi-io/read_atall.c       \
-    mpi-io/read_ord.c         \
-    mpi-io/read_ordb.c        \
-    mpi-io/read_orde.c        \
-    mpi-io/read_sh.c          \
-    mpi-io/register_datarep.c \
-    mpi-io/seek.c             \
-    mpi-io/seek_sh.c          \
-    mpi-io/set_atom.c         \
-    mpi-io/set_info.c         \
-    mpi-io/set_size.c         \
-    mpi-io/set_view.c         \
-    mpi-io/wr_atallb.c        \
-    mpi-io/wr_atalle.c        \
-    mpi-io/write.c            \
-    mpi-io/write_all.c        \
-    mpi-io/write_allb.c       \
-    mpi-io/write_alle.c       \
-    mpi-io/write_at.c         \
-    mpi-io/write_atall.c      \
-    mpi-io/write_ord.c        \
-    mpi-io/write_ordb.c       \
-    mpi-io/write_orde.c       \
-    mpi-io/write_sh.c
-
-
-# non-MPI/PMPI sources that will be included in libromio
-romio_other_sources +=       \
-    mpi-io/mpich_fileutil.c \
-    mpi-io/mpir-mpioinit.c   \
-    mpi-io/mpiu_greq.c \
-    mpi-io/mpiu_external32.c
-
-# helper variables for conditionally compiled sources
-mpio_request_sources=   \
-    mpi-io/ioreq_c2f.c  \
-    mpi-io/ioreq_f2c.c  \
-    mpi-io/iotest.c     \
-    mpi-io/iotestall.c  \
-    mpi-io/iotestany.c  \
-    mpi-io/iotestsome.c \
-    mpi-io/iowait.c     \
-    mpi-io/iowaitall.c  \
-    mpi-io/iowaitany.c  \
-    mpi-io/iowaitsome.c
-
-mpio_extra_sources =  \
-    mpi-io/get_errh.c \
-    mpi-io/set_errh.c
-
-# not used in MPICH, we use generalized requests instead
-if BUILD_MPIO_REQUEST
-romio_other_sources += $(mpio_request_sources)
-endif BUILD_MPIO_REQUEST
-
-# not used in MPICH
-if BUILD_MPIO_ERRHAN
-romio_other_sources += $(mpio_request_sources)
-endif BUILD_MPIO_ERRHAN
-
diff --git a/3rd-party/romio321/mpi-io/close.c b/3rd-party/romio321/mpi-io/close.c
deleted file mode 100644
index 12d5c4791f5..00000000000
--- a/3rd-party/romio321/mpi-io/close.c
+++ /dev/null
@@ -1,105 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_close = PMPI_File_close
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_close MPI_File_close
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_close as PMPI_File_close
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_close(MPI_File *fh) __attribute__((weak,alias("PMPI_File_close")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_close - Closes a file
-
-Input Parameters:
-. fh - file handle (handle)
-
-.N fortran
-@*/
-int MPI_File_close(MPI_File *fh)
-{
-    int error_code;
-    ADIO_File adio_fh;
-    static char myname[] = "MPI_FILE_CLOSE";
-#ifdef MPI_hpux
-    int fl_xmpi;
-
-    HPMP_IO_WSTART(fl_xmpi, BLKMPIFILECLOSE, TRDTBLOCK, *adio_fh);
-#endif /* MPI_hpux */
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(*fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    if (ADIO_Feature(adio_fh, ADIO_SHARED_FP))
-    {
-	ADIOI_Free((adio_fh)->shared_fp_fname);
-	/* POSIX semantics say a deleted file remains available until all
-	 * processes close the file.  But since when was NFS posix-compliant?
-	 */
-	/* this used to be gated by the lack of the UNLINK_AFTER_CLOSE feature,
-	 * but a race condition in GPFS necessated this.  See ticket #2214 */
-	MPI_Barrier((adio_fh)->comm);
-	if ((adio_fh)->shared_fp_fd != ADIO_FILE_NULL) {
-	    MPI_File *fh_shared = &(adio_fh->shared_fp_fd);
-	    ADIO_Close((adio_fh)->shared_fp_fd, &error_code);
-            MPIO_File_free(fh_shared);
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (error_code != MPI_SUCCESS) goto fn_fail;
-	    /* --END ERROR HANDLING-- */
-	}
-    }
-
-    /* Because ROMIO expects the MPI library to provide error handler management
-     * routines but it doesn't ever participate in MPI_File_close, we have to
-     * somehow inform the MPI library that we no longer hold a reference to any
-     * user defined error handler.  We do this by setting the errhandler at this
-     * point to MPI_ERRORS_RETURN. */
-/* Open MPI: The call to PMPI_File_set_errhandler has to be done in romio/src/io_romio_file_open.c
-   in routine mca_io_romio_file_close()
-*/
-#if 0
-    error_code = PMPI_File_set_errhandler(*fh, MPI_ERRORS_RETURN);
-    if (error_code != MPI_SUCCESS) goto fn_fail;
-#endif
-
-    ADIO_Close(adio_fh, &error_code);
-    MPIO_File_free(fh);
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS) goto fn_fail;
-    /* --END ERROR HANDLING-- */
-
-#ifdef MPI_hpux
-    HPMP_IO_WEND(fl_xmpi);
-#endif /* MPI_hpux */
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-    return error_code;
-fn_fail:
-    /* --BEGIN ERROR HANDLING-- */
-    error_code = MPIO_Err_return_file(adio_fh, error_code);
-    goto fn_exit;
-    /* --END ERROR HANDLING-- */
-}
diff --git a/3rd-party/romio321/mpi-io/delete.c b/3rd-party/romio321/mpi-io/delete.c
deleted file mode 100644
index 47b52a054c4..00000000000
--- a/3rd-party/romio321/mpi-io/delete.c
+++ /dev/null
@@ -1,95 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_delete = PMPI_File_delete
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_delete MPI_File_delete
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_delete as PMPI_File_delete
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_delete(const char *filename, MPI_Info info) __attribute__((weak,alias("PMPI_File_delete")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_delete - Deletes a file
-
-Input Parameters:
-. filename - name of file to delete (string)
-. info - info object (handle)
-
-.N fortran
-@*/
-int MPI_File_delete(ROMIO_CONST char *filename, MPI_Info info)
-{
-    int error_code, file_system;
-    char *tmp;
-    ADIOI_Fns *fsops;
-#ifdef MPI_hpux
-    int fl_xmpi;
-  
-    HPMP_IO_START(fl_xmpi, BLKMPIFILEDELETE, TRDTBLOCK,
-                MPI_FILE_NULL, MPI_DATATYPE_NULL, -1);
-#endif /* MPI_hpux */
-
-    MPIU_UNREFERENCED_ARG(info);
-
-    ROMIO_THREAD_CS_ENTER();
-
-    MPIR_MPIOInit(&error_code);
-    if (error_code != MPI_SUCCESS) goto fn_exit;
-
-    /* resolve file system type from file name; this is a collective call */
-    ADIO_ResolveFileType(MPI_COMM_SELF, filename, &file_system, &fsops, 
-			 &error_code);
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-    {
-	/* ADIO_ResolveFileType() will print as informative a message as it
-	 * possibly can or call MPIR_Err_setmsg.  We just need to propagate 
-	 * the error up.  In the PRINT_ERR_MSG case MPI_Abort has already
-	 * been called as well, so we probably didn't even make it this far.
-	 */
-	error_code = MPIO_Err_return_file(MPI_FILE_NULL, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    /* skip prefixes on file names if they have more than one character;
-     * single-character prefixes are assumed to be windows drive
-     * specifications (e.g. c:\foo) and are left alone.
-     */
-    tmp = strchr(filename, ':');
-    if (tmp > filename + 1)
-	filename = tmp + 1;
-
-    /* call the fs-specific delete function */
-    (fsops->ADIOI_xxx_Delete)(filename, &error_code);
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-	error_code = MPIO_Err_return_file(MPI_FILE_NULL, error_code);
-    /* --END ERROR HANDLING-- */
-	
-#ifdef MPI_hpux
-    HPMP_IO_END(fl_xmpi, MPI_FILE_NULL, MPI_DATATYPE_NULL, -1);
-#endif /* MPI_hpux */
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/file_c2f.c b/3rd-party/romio321/mpi-io/file_c2f.c
deleted file mode 100644
index 4eaae9fafa0..00000000000
--- a/3rd-party/romio321/mpi-io/file_c2f.c
+++ /dev/null
@@ -1,41 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_c2f = PMPI_File_c2f
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_c2f MPI_File_c2f
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_c2f as PMPI_File_c2f
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-MPI_Fint MPI_File_c2f(MPI_File fh) __attribute__((weak,alias("PMPI_File_c2f")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-#include "adio_extern.h"
-
-/*@
-    MPI_File_c2f - Translates a C file handle to a Fortran file handle
-
-Input Parameters:
-. fh - C file handle (handle)
-
-Return Value:
-  Fortran file handle (integer)
-@*/
-MPI_Fint MPI_File_c2f(MPI_File fh)
-{
-    return MPIO_File_c2f(fh);
-}
diff --git a/3rd-party/romio321/mpi-io/file_f2c.c b/3rd-party/romio321/mpi-io/file_f2c.c
deleted file mode 100644
index b6b4896b389..00000000000
--- a/3rd-party/romio321/mpi-io/file_f2c.c
+++ /dev/null
@@ -1,41 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_f2c = PMPI_File_f2c
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_f2c MPI_File_f2c
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_f2c as PMPI_File_f2c
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-MPI_File MPI_File_f2c(MPI_Fint fh) __attribute__((weak,alias("PMPI_File_f2c")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-#include "adio_extern.h"
-
-/*@
-    MPI_File_f2c - Translates a Fortran file handle to a C file handle
-
-Input Parameters:
-. fh - Fortran file handle (integer)
-
-Return Value:
-  C file handle (handle)
-@*/
-MPI_File MPI_File_f2c(MPI_Fint fh)
-{
-    return MPIO_File_f2c(fh);
-}
diff --git a/3rd-party/romio321/mpi-io/fortran/Makefile.mk b/3rd-party/romio321/mpi-io/fortran/Makefile.mk
deleted file mode 100644
index f7ed72b69b4..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/Makefile.mk
+++ /dev/null
@@ -1,68 +0,0 @@
-## -*- Mode: Makefile; -*-
-## vim: set ft=automake :
-##
-## (C) 2011 by Argonne National Laboratory.
-##     See COPYRIGHT in top-level directory.
-##
-
-if BUILD_F77_BINDINGS
-
-romio_mpi_sources +=              \
-    mpi-io/fortran/closef.c       \
-    mpi-io/fortran/deletef.c      \
-    mpi-io/fortran/fsyncf.c       \
-    mpi-io/fortran/get_amodef.c   \
-    mpi-io/fortran/get_atomf.c    \
-    mpi-io/fortran/get_bytofff.c  \
-    mpi-io/fortran/get_errhf.c    \
-    mpi-io/fortran/get_extentf.c  \
-    mpi-io/fortran/get_groupf.c   \
-    mpi-io/fortran/get_infof.c    \
-    mpi-io/fortran/get_posn_shf.c \
-    mpi-io/fortran/get_posnf.c    \
-    mpi-io/fortran/get_sizef.c    \
-    mpi-io/fortran/get_viewf.c    \
-    mpi-io/fortran/iotestf.c      \
-    mpi-io/fortran/iowaitf.c      \
-    mpi-io/fortran/iread_atf.c    \
-    mpi-io/fortran/iread_shf.c    \
-    mpi-io/fortran/ireadf.c       \
-    mpi-io/fortran/iwrite_atf.c   \
-    mpi-io/fortran/iwrite_shf.c   \
-    mpi-io/fortran/iwritef.c      \
-    mpi-io/fortran/openf.c        \
-    mpi-io/fortran/preallocf.c    \
-    mpi-io/fortran/rd_atallbf.c   \
-    mpi-io/fortran/rd_atallef.c   \
-    mpi-io/fortran/read_allbf.c   \
-    mpi-io/fortran/read_allef.c   \
-    mpi-io/fortran/read_allf.c    \
-    mpi-io/fortran/read_atallf.c  \
-    mpi-io/fortran/read_atf.c     \
-    mpi-io/fortran/read_ordbf.c   \
-    mpi-io/fortran/read_ordef.c   \
-    mpi-io/fortran/read_ordf.c    \
-    mpi-io/fortran/read_shf.c     \
-    mpi-io/fortran/readf.c        \
-    mpi-io/fortran/seek_shf.c     \
-    mpi-io/fortran/seekf.c        \
-    mpi-io/fortran/set_atomf.c    \
-    mpi-io/fortran/set_errhf.c    \
-    mpi-io/fortran/set_infof.c    \
-    mpi-io/fortran/set_sizef.c    \
-    mpi-io/fortran/set_viewf.c    \
-    mpi-io/fortran/wr_atallbf.c   \
-    mpi-io/fortran/wr_atallef.c   \
-    mpi-io/fortran/write_allbf.c  \
-    mpi-io/fortran/write_allef.c  \
-    mpi-io/fortran/write_allf.c   \
-    mpi-io/fortran/write_atallf.c \
-    mpi-io/fortran/write_atf.c    \
-    mpi-io/fortran/write_ordbf.c  \
-    mpi-io/fortran/write_ordef.c  \
-    mpi-io/fortran/write_ordf.c   \
-    mpi-io/fortran/write_shf.c    \
-    mpi-io/fortran/writef.c
-
-endif BUILD_F77_BINDINGS
-
diff --git a/3rd-party/romio321/mpi-io/fortran/closef.c b/3rd-party/romio321/mpi-io/fortran/closef.c
deleted file mode 100644
index aebdf56a611..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/closef.c
+++ /dev/null
@@ -1,102 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_CLOSE( MPI_File*, MPI_Fint * );
-#pragma weak MPI_FILE_CLOSE = PMPI_FILE_CLOSE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_close__( MPI_File*, MPI_Fint * );
-#pragma weak mpi_file_close__ = pmpi_file_close__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_close( MPI_File*, MPI_Fint * );
-#pragma weak mpi_file_close = pmpi_file_close
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_close_( MPI_File*, MPI_Fint * );
-#pragma weak mpi_file_close_ = pmpi_file_close_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_CLOSE MPI_FILE_CLOSE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_close__ mpi_file_close__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_close mpi_file_close
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_close_ mpi_file_close_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_CLOSE as PMPI_FILE_CLOSE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_close__ as pmpi_file_close__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_close as pmpi_file_close
-#else
-#pragma _CRI duplicate mpi_file_close_ as pmpi_file_close_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_close_ PMPI_FILE_CLOSE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_close_ pmpi_file_close__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_close pmpi_file_close_
-#endif
-#define mpi_file_close_ pmpi_file_close
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_close_ pmpi_file_close
-#endif
-#define mpi_file_close_ pmpi_file_close_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_close_ MPI_FILE_CLOSE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_close_ mpi_file_close__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_close mpi_file_close_
-#endif
-#define mpi_file_close_ mpi_file_close
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_close_ mpi_file_close
-#endif
-#endif
-#endif
-
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_close_(MPI_Fint *fh, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_close_(MPI_Fint *fh, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_close(&fh_c);
-    *fh = MPI_File_c2f(fh_c);
-}
diff --git a/3rd-party/romio321/mpi-io/fortran/deletef.c b/3rd-party/romio321/mpi-io/fortran/deletef.c
deleted file mode 100644
index 660129f7605..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/deletef.c
+++ /dev/null
@@ -1,144 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#ifdef _UNICOS
-#include <fortran.h>
-#endif
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_DELETE( char * FORT_MIXED_LEN_DECL, MPI_Fint *, MPI_Fint * FORT_END_LEN_DECL );
-#pragma weak MPI_FILE_DELETE = PMPI_FILE_DELETE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_delete__( char * FORT_MIXED_LEN_DECL, MPI_Fint *, MPI_Fint * FORT_END_LEN_DECL );
-#pragma weak mpi_file_delete__ = pmpi_file_delete__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_delete( char * FORT_MIXED_LEN_DECL, MPI_Fint *, MPI_Fint * FORT_END_LEN_DECL );
-#pragma weak mpi_file_delete = pmpi_file_delete
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_delete_( char * FORT_MIXED_LEN_DECL, MPI_Fint *, MPI_Fint * FORT_END_LEN_DECL );
-#pragma weak mpi_file_delete_ = pmpi_file_delete_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_DELETE MPI_FILE_DELETE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_delete__ mpi_file_delete__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_delete mpi_file_delete
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_delete_ mpi_file_delete_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_DELETE as PMPI_FILE_DELETE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_delete__ as pmpi_file_delete__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_delete as pmpi_file_delete
-#else
-#pragma _CRI duplicate mpi_file_delete_ as pmpi_file_delete_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_delete_ PMPI_FILE_DELETE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_delete_ pmpi_file_delete__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_delete pmpi_file_delete_
-#endif
-#define mpi_file_delete_ pmpi_file_delete
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_delete_ pmpi_file_delete
-#endif
-#define mpi_file_delete_ pmpi_file_delete_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_delete_ MPI_FILE_DELETE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_delete_ mpi_file_delete__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_delete mpi_file_delete_
-#endif
-#define mpi_file_delete_ mpi_file_delete
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_delete_ mpi_file_delete
-#endif
-#endif
-#endif
-
-/* Prototype to keep compiler happy */
-/*
-FORTRAN_API void FORT_CALL mpi_file_delete_(char *filename, MPI_Fint *info, MPI_Fint *ierr, int str_len);
-
-#ifdef _UNICOS
-void mpi_file_delete_(_fcd filename_fcd, MPI_Fint *info, MPI_Fint *ierr)
-{
-    char *filename = _fcdtocp(filename_fcd);
-    int str_len = _fcdlen(filename_fcd);
-#else
-FORTRAN_API void FORT_CALL mpi_file_delete_(char *filename, MPI_Fint *info, MPI_Fint *ierr, int str_len)
-*/
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_delete_(char *filename FORT_MIXED_LEN_DECL, MPI_Fint *info, MPI_Fint *ierr FORT_END_LEN_DECL);
-
-#ifdef _UNICOS
-void mpi_file_delete_(_fcd filename_fcd, MPI_Fint *info, MPI_Fint *ierr)
-{
-    char *filename = _fcdtocp(filename_fcd);
-    int str_len = _fcdlen(filename_fcd);
-#else
-FORTRAN_API void FORT_CALL mpi_file_delete_(char *filename FORT_MIXED_LEN(str_len), MPI_Fint *info, MPI_Fint *ierr FORT_END_LEN(str_len))
-{
-#endif
-    char *newfname;
-    int real_len, i;
-    MPI_Info info_c;
-
-    info_c = MPI_Info_f2c(*info);
-
-    /* strip trailing blanks */
-    if (filename <= (char *) 0) {
-        FPRINTF(stderr, "MPI_File_delete: filename is an invalid address\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-    for (i=str_len-1; i>=0; i--) if (filename[i] != ' ') break;
-    if (i < 0) {
-        FPRINTF(stderr, "MPI_File_delete: filename is a blank string\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-    real_len = i + 1;
-
-    newfname = (char *) ADIOI_Malloc((real_len+1)*sizeof(char));
-    ADIOI_Strncpy(newfname, filename, real_len);
-    newfname[real_len] = '\0';
-
-    *ierr = MPI_File_delete(newfname, info_c);
-
-    ADIOI_Free(newfname);
-}
diff --git a/3rd-party/romio321/mpi-io/fortran/fsyncf.c b/3rd-party/romio321/mpi-io/fortran/fsyncf.c
deleted file mode 100644
index b52245256b8..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/fsyncf.c
+++ /dev/null
@@ -1,101 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_SYNC( MPI_Fint *, MPI_Fint * );
-#pragma weak MPI_FILE_SYNC = PMPI_FILE_SYNC
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_sync__( MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_sync__ = pmpi_file_sync__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_sync( MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_sync = pmpi_file_sync
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_sync_( MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_sync_ = pmpi_file_sync_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_SYNC MPI_FILE_SYNC
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_sync__ mpi_file_sync__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_sync mpi_file_sync
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_sync_ mpi_file_sync_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_SYNC as PMPI_FILE_SYNC
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_sync__ as pmpi_file_sync__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_sync as pmpi_file_sync
-#else
-#pragma _CRI duplicate mpi_file_sync_ as pmpi_file_sync_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_sync_ PMPI_FILE_SYNC
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_sync_ pmpi_file_sync__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_sync pmpi_file_sync_
-#endif
-#define mpi_file_sync_ pmpi_file_sync
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_sync_ pmpi_file_sync
-#endif
-#define mpi_file_sync_ pmpi_file_sync_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_sync_ MPI_FILE_SYNC
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_sync_ mpi_file_sync__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_sync mpi_file_sync_
-#endif
-#define mpi_file_sync_ mpi_file_sync
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_sync_ mpi_file_sync
-#endif
-#endif
-#endif
-
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_sync_(MPI_Fint *fh, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_sync_(MPI_Fint *fh, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_sync(fh_c);
-}
diff --git a/3rd-party/romio321/mpi-io/fortran/get_amodef.c b/3rd-party/romio321/mpi-io/fortran/get_amodef.c
deleted file mode 100644
index 2427f068310..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/get_amodef.c
+++ /dev/null
@@ -1,101 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_GET_AMODE( MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak MPI_FILE_GET_AMODE = PMPI_FILE_GET_AMODE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_get_amode__( MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_get_amode__ = pmpi_file_get_amode__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_get_amode( MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_get_amode = pmpi_file_get_amode
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_get_amode_( MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_get_amode_ = pmpi_file_get_amode_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_GET_AMODE MPI_FILE_GET_AMODE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_amode__ mpi_file_get_amode__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_amode mpi_file_get_amode
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_get_amode_ mpi_file_get_amode_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_GET_AMODE as PMPI_FILE_GET_AMODE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_get_amode__ as pmpi_file_get_amode__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_get_amode as pmpi_file_get_amode
-#else
-#pragma _CRI duplicate mpi_file_get_amode_ as pmpi_file_get_amode_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_get_amode_ PMPI_FILE_GET_AMODE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_get_amode_ pmpi_file_get_amode__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_amode pmpi_file_get_amode_
-#endif
-#define mpi_file_get_amode_ pmpi_file_get_amode
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_amode_ pmpi_file_get_amode
-#endif
-#define mpi_file_get_amode_ pmpi_file_get_amode_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_get_amode_ MPI_FILE_GET_AMODE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_get_amode_ mpi_file_get_amode__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_get_amode mpi_file_get_amode_
-#endif
-#define mpi_file_get_amode_ mpi_file_get_amode
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_get_amode_ mpi_file_get_amode
-#endif
-#endif
-#endif
-
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_get_amode_(MPI_Fint *fh, MPI_Fint *amode, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_get_amode_(MPI_Fint *fh, MPI_Fint *amode, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_get_amode(fh_c, amode);
-}
diff --git a/3rd-party/romio321/mpi-io/fortran/get_atomf.c b/3rd-party/romio321/mpi-io/fortran/get_atomf.c
deleted file mode 100644
index 98bf557e34b..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/get_atomf.c
+++ /dev/null
@@ -1,102 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_GET_ATOMICITY( MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak MPI_FILE_GET_ATOMICITY = PMPI_FILE_GET_ATOMICITY
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_get_atomicity__( MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_get_atomicity__ = pmpi_file_get_atomicity__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_get_atomicity( MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_get_atomicity = pmpi_file_get_atomicity
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_get_atomicity_( MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_get_atomicity_ = pmpi_file_get_atomicity_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_GET_ATOMICITY MPI_FILE_GET_ATOMICITY
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_atomicity__ mpi_file_get_atomicity__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_atomicity mpi_file_get_atomicity
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_get_atomicity_ mpi_file_get_atomicity_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_GET_ATOMICITY as PMPI_FILE_GET_ATOMICITY
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_get_atomicity__ as pmpi_file_get_atomicity__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_get_atomicity as pmpi_file_get_atomicity
-#else
-#pragma _CRI duplicate mpi_file_get_atomicity_ as pmpi_file_get_atomicity_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_get_atomicity_ PMPI_FILE_GET_ATOMICITY
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_get_atomicity_ pmpi_file_get_atomicity__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_atomicity pmpi_file_get_atomicity_
-#endif
-#define mpi_file_get_atomicity_ pmpi_file_get_atomicity
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_atomicity_ pmpi_file_get_atomicity
-#endif
-#define mpi_file_get_atomicity_ pmpi_file_get_atomicity_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_get_atomicity_ MPI_FILE_GET_ATOMICITY
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_get_atomicity_ mpi_file_get_atomicity__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_get_atomicity mpi_file_get_atomicity_
-#endif
-#define mpi_file_get_atomicity_ mpi_file_get_atomicity
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_get_atomicity_ mpi_file_get_atomicity
-#endif
-#endif
-#endif
-
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_get_atomicity_(MPI_Fint *fh, MPI_Fint *flag, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_get_atomicity_(MPI_Fint *fh, MPI_Fint *flag, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_get_atomicity(fh_c, flag);
-}
-
diff --git a/3rd-party/romio321/mpi-io/fortran/get_bytofff.c b/3rd-party/romio321/mpi-io/fortran/get_bytofff.c
deleted file mode 100644
index fb38b7be4ca..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/get_bytofff.c
+++ /dev/null
@@ -1,101 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_GET_BYTE_OFFSET( MPI_Fint *, MPI_Offset *, MPI_Offset*, MPI_Fint * );
-#pragma weak MPI_FILE_GET_BYTE_OFFSET = PMPI_FILE_GET_BYTE_OFFSET
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_get_byte_offset__( MPI_Fint *, MPI_Offset *, MPI_Offset*, MPI_Fint * );
-#pragma weak mpi_file_get_byte_offset__ = pmpi_file_get_byte_offset__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_get_byte_offset( MPI_Fint *, MPI_Offset *, MPI_Offset*, MPI_Fint * );
-#pragma weak mpi_file_get_byte_offset = pmpi_file_get_byte_offset
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_get_byte_offset_( MPI_Fint *, MPI_Offset *, MPI_Offset*, MPI_Fint * );
-#pragma weak mpi_file_get_byte_offset_ = pmpi_file_get_byte_offset_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_GET_BYTE_OFFSET MPI_FILE_GET_BYTE_OFFSET
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_byte_offset__ mpi_file_get_byte_offset__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_byte_offset mpi_file_get_byte_offset
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_get_byte_offset_ mpi_file_get_byte_offset_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_GET_BYTE_OFFSET as PMPI_FILE_GET_BYTE_OFFSET
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_get_byte_offset__ as pmpi_file_get_byte_offset__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_get_byte_offset as pmpi_file_get_byte_offset
-#else
-#pragma _CRI duplicate mpi_file_get_byte_offset_ as pmpi_file_get_byte_offset_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_get_byte_offset_ PMPI_FILE_GET_BYTE_OFFSET
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_get_byte_offset_ pmpi_file_get_byte_offset__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_byte_offset pmpi_file_get_byte_offset_
-#endif
-#define mpi_file_get_byte_offset_ pmpi_file_get_byte_offset
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_byte_offset_ pmpi_file_get_byte_offset
-#endif
-#define mpi_file_get_byte_offset_ pmpi_file_get_byte_offset_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_get_byte_offset_ MPI_FILE_GET_BYTE_OFFSET
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_get_byte_offset_ mpi_file_get_byte_offset__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_get_byte_offset mpi_file_get_byte_offset_
-#endif
-#define mpi_file_get_byte_offset_ mpi_file_get_byte_offset
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_get_byte_offset_ mpi_file_get_byte_offset
-#endif
-#endif
-#endif
-
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_get_byte_offset_(MPI_Fint *fh, MPI_Offset *offset, MPI_Offset *disp, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_get_byte_offset_(MPI_Fint *fh,MPI_Offset *offset, MPI_Offset *disp, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_get_byte_offset(fh_c,*offset,disp);
-}
diff --git a/3rd-party/romio321/mpi-io/fortran/get_errhf.c b/3rd-party/romio321/mpi-io/fortran/get_errhf.c
deleted file mode 100644
index 96901ad7766..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/get_errhf.c
+++ /dev/null
@@ -1,103 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_GET_ERRHANDLER( MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak MPI_FILE_GET_ERRHANDLER = PMPI_FILE_GET_ERRHANDLER
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_get_errhandler__( MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_get_errhandler__ = pmpi_file_get_errhandler__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_get_errhandler( MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_get_errhandler = pmpi_file_get_errhandler
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_get_errhandler_( MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_get_errhandler_ = pmpi_file_get_errhandler_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_GET_ERRHANDLER MPI_FILE_GET_ERRHANDLER
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_errhandler__ mpi_file_get_errhandler__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_errhandler mpi_file_get_errhandler
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_get_errhandler_ mpi_file_get_errhandler_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_GET_ERRHANDLER as PMPI_FILE_GET_ERRHANDLER
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_get_errhandler__ as pmpi_file_get_errhandler__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_get_errhandler as pmpi_file_get_errhandler
-#else
-#pragma _CRI duplicate mpi_file_get_errhandler_ as pmpi_file_get_errhandler_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_get_errhandler_ PMPI_FILE_GET_ERRHANDLER
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_get_errhandler_ pmpi_file_get_errhandler__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_errhandler pmpi_file_get_errhandler_
-#endif
-#define mpi_file_get_errhandler_ pmpi_file_get_errhandler
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_errhandler_ pmpi_file_get_errhandler
-#endif
-#define mpi_file_get_errhandler_ pmpi_file_get_errhandler_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_get_errhandler_ MPI_FILE_GET_ERRHANDLER
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_get_errhandler_ mpi_file_get_errhandler__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_get_errhandler mpi_file_get_errhandler_
-#endif
-#define mpi_file_get_errhandler_ mpi_file_get_errhandler
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_get_errhandler_ mpi_file_get_errhandler
-#endif
-#endif
-#endif
-
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_get_errhandler_(MPI_Fint *fh, MPI_Fint *err_handler, MPI_Fint *ierr);
-
-FORTRAN_API void FORT_CALL mpi_file_get_errhandler_(MPI_Fint *fh, MPI_Fint *err_handler, MPI_Fint *ierr)
-{
-    MPI_File fh_c;
-    MPI_Errhandler err_handler_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_get_errhandler(fh_c, &err_handler_c);
-    *err_handler = MPI_Errhandler_c2f(err_handler_c);
-}
diff --git a/3rd-party/romio321/mpi-io/fortran/get_extentf.c b/3rd-party/romio321/mpi-io/fortran/get_extentf.c
deleted file mode 100644
index 9a11d3c946f..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/get_extentf.c
+++ /dev/null
@@ -1,126 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_GET_TYPE_EXTENT( MPI_Fint *, MPI_Fint *, MPI_Fint*, MPI_Fint * );
-#pragma weak MPI_FILE_GET_TYPE_EXTENT = PMPI_FILE_GET_TYPE_EXTENT
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_get_type_extent__( MPI_Fint *, MPI_Fint *, MPI_Fint*, MPI_Fint * );
-#pragma weak mpi_file_get_type_extent__ = pmpi_file_get_type_extent__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_get_type_extent( MPI_Fint *, MPI_Fint *, MPI_Fint*, MPI_Fint * );
-#pragma weak mpi_file_get_type_extent = pmpi_file_get_type_extent
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_get_type_extent_( MPI_Fint *, MPI_Fint *, MPI_Fint*, MPI_Fint * );
-#pragma weak mpi_file_get_type_extent_ = pmpi_file_get_type_extent_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_GET_TYPE_EXTENT MPI_FILE_GET_TYPE_EXTENT
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_type_extent__ mpi_file_get_type_extent__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_type_extent mpi_file_get_type_extent
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_get_type_extent_ mpi_file_get_type_extent_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_GET_TYPE_EXTENT as PMPI_FILE_GET_TYPE_EXTENT
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_get_type_extent__ as pmpi_file_get_type_extent__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_get_type_extent as pmpi_file_get_type_extent
-#else
-#pragma _CRI duplicate mpi_file_get_type_extent_ as pmpi_file_get_type_extent_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_get_type_extent_ PMPI_FILE_GET_TYPE_EXTENT
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_get_type_extent_ pmpi_file_get_type_extent__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_type_extent pmpi_file_get_type_extent_
-#endif
-#define mpi_file_get_type_extent_ pmpi_file_get_type_extent
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_type_extent_ pmpi_file_get_type_extent
-#endif
-#define mpi_file_get_type_extent_ pmpi_file_get_type_extent_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_get_type_extent_ MPI_FILE_GET_TYPE_EXTENT
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_get_type_extent_ mpi_file_get_type_extent__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_get_type_extent mpi_file_get_type_extent_
-#endif
-#define mpi_file_get_type_extent_ mpi_file_get_type_extent
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_get_type_extent_ mpi_file_get_type_extent
-#endif
-#endif
-#endif
-
-#if defined(MPIHP) || defined(MPILAM)
-/* Prototype to keep compiler happy */
-void mpi_file_get_type_extent_(MPI_Fint *fh,MPI_Fint *datatype,
-			       MPI_Fint *extent, MPI_Fint *ierr );
-
-void mpi_file_get_type_extent_(MPI_Fint *fh,MPI_Fint *datatype,
-                             MPI_Fint *extent, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPI_Datatype datatype_c;
-    MPI_Aint extent_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    datatype_c = MPI_Type_f2c(*datatype);
-
-    *ierr = MPI_File_get_type_extent(fh_c,datatype_c, &extent_c);
-    *(MPI_Aint*)extent = extent_c; /* Have to assume it's really an MPI_Aint?*/
-}
-
-#else
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_get_type_extent_(MPI_Fint *fh,MPI_Datatype *datatype,
-			       MPI_Fint *extent, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_get_type_extent_(MPI_Fint *fh,MPI_Datatype *datatype,
-                             MPI_Fint *extent, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPI_Aint extent_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_get_type_extent(fh_c,*datatype, &extent_c);
-    *(MPI_Aint*)extent = extent_c; /* Have to assume it's really an MPI_Aint?*/
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/fortran/get_groupf.c b/3rd-party/romio321/mpi-io/fortran/get_groupf.c
deleted file mode 100644
index bb8c9a9cba7..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/get_groupf.c
+++ /dev/null
@@ -1,116 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_GET_GROUP( MPI_Fint *, MPI_Group*, MPI_Fint * );
-#pragma weak MPI_FILE_GET_GROUP = PMPI_FILE_GET_GROUP
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_get_group__( MPI_Fint *, MPI_Group*, MPI_Fint * );
-#pragma weak mpi_file_get_group__ = pmpi_file_get_group__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_get_group( MPI_Fint *, MPI_Group*, MPI_Fint * );
-#pragma weak mpi_file_get_group = pmpi_file_get_group
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_get_group_( MPI_Fint *, MPI_Group*, MPI_Fint * );
-#pragma weak mpi_file_get_group_ = pmpi_file_get_group_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_GET_GROUP MPI_FILE_GET_GROUP
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_group__ mpi_file_get_group__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_group mpi_file_get_group
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_get_group_ mpi_file_get_group_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_GET_GROUP as PMPI_FILE_GET_GROUP
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_get_group__ as pmpi_file_get_group__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_get_group as pmpi_file_get_group
-#else
-#pragma _CRI duplicate mpi_file_get_group_ as pmpi_file_get_group_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_get_group_ PMPI_FILE_GET_GROUP
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_get_group_ pmpi_file_get_group__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_group pmpi_file_get_group_
-#endif
-#define mpi_file_get_group_ pmpi_file_get_group
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_group_ pmpi_file_get_group
-#endif
-#define mpi_file_get_group_ pmpi_file_get_group_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_get_group_ MPI_FILE_GET_GROUP
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_get_group_ mpi_file_get_group__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_get_group mpi_file_get_group_
-#endif
-#define mpi_file_get_group_ mpi_file_get_group
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_get_group_ mpi_file_get_group
-#endif
-#endif
-#endif
-
-#if defined(MPIHP) || defined(MPILAM)
-/* Prototype to keep compiler happy */
-void mpi_file_get_group_(MPI_Fint *fh, MPI_Fint *group, MPI_Fint *ierr );
-
-void mpi_file_get_group_(MPI_Fint *fh, MPI_Fint *group, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPI_Group group_c;
-
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_get_group(fh_c, &group_c);
-    *group = MPI_Group_c2f(group_c);
-}
-#else
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_get_group_(MPI_Fint *fh,MPI_Group *group, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_get_group_(MPI_Fint *fh,MPI_Group *group, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_get_group(fh_c, group);
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/fortran/get_infof.c b/3rd-party/romio321/mpi-io/fortran/get_infof.c
deleted file mode 100644
index eb2ba551a45..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/get_infof.c
+++ /dev/null
@@ -1,104 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_GET_INFO( MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak MPI_FILE_GET_INFO = PMPI_FILE_GET_INFO
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_get_info__( MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_get_info__ = pmpi_file_get_info__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_get_info( MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_get_info = pmpi_file_get_info
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_get_info_( MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_get_info_ = pmpi_file_get_info_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_GET_INFO MPI_FILE_GET_INFO
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_info__ mpi_file_get_info__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_info mpi_file_get_info
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_get_info_ mpi_file_get_info_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_GET_INFO as PMPI_FILE_GET_INFO
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_get_info__ as pmpi_file_get_info__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_get_info as pmpi_file_get_info
-#else
-#pragma _CRI duplicate mpi_file_get_info_ as pmpi_file_get_info_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_get_info_ PMPI_FILE_GET_INFO
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_get_info_ pmpi_file_get_info__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_info pmpi_file_get_info_
-#endif
-#define mpi_file_get_info_ pmpi_file_get_info
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_info_ pmpi_file_get_info
-#endif
-#define mpi_file_get_info_ pmpi_file_get_info_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_get_info_ MPI_FILE_GET_INFO
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_get_info_ mpi_file_get_info__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_get_info mpi_file_get_info_
-#endif
-#define mpi_file_get_info_ mpi_file_get_info
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_get_info_ mpi_file_get_info
-#endif
-#endif
-#endif
-
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_get_info_(MPI_Fint *fh, MPI_Fint *info_used, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_get_info_(MPI_Fint *fh, MPI_Fint *info_used, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPI_Info info_used_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-
-    *ierr = MPI_File_get_info(fh_c, &info_used_c);
-    *info_used = MPI_Info_c2f(info_used_c);
-}
diff --git a/3rd-party/romio321/mpi-io/fortran/get_posn_shf.c b/3rd-party/romio321/mpi-io/fortran/get_posn_shf.c
deleted file mode 100644
index 8f5bff5c06e..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/get_posn_shf.c
+++ /dev/null
@@ -1,102 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_GET_POSITION_SHARED( MPI_Fint *, MPI_Offset*, MPI_Fint * );
-#pragma weak MPI_FILE_GET_POSITION_SHARED = PMPI_FILE_GET_POSITION_SHARED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_get_position_shared__( MPI_Fint *, MPI_Offset*, MPI_Fint * );
-#pragma weak mpi_file_get_position_shared__ = pmpi_file_get_position_shared__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_get_position_shared( MPI_Fint *, MPI_Offset*, MPI_Fint * );
-#pragma weak mpi_file_get_position_shared = pmpi_file_get_position_shared
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_get_position_shared_( MPI_Fint *, MPI_Offset*, MPI_Fint * );
-#pragma weak mpi_file_get_position_shared_ = pmpi_file_get_position_shared_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_GET_POSITION_SHARED MPI_FILE_GET_POSITION_SHARED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_position_shared__ mpi_file_get_position_shared__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_position_shared mpi_file_get_position_shared
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_get_position_shared_ mpi_file_get_position_shared_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_GET_POSITION_SHARED as PMPI_FILE_GET_POSITION_SHARED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_get_position_shared__ as pmpi_file_get_position_shared__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_get_position_shared as pmpi_file_get_position_shared
-#else
-#pragma _CRI duplicate mpi_file_get_position_shared_ as pmpi_file_get_position_shared_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_get_position_shared_ PMPI_FILE_GET_POSITION_SHARED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_get_position_shared_ pmpi_file_get_position_shared__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_position_shared pmpi_file_get_position_shared_
-#endif
-#define mpi_file_get_position_shared_ pmpi_file_get_position_shared
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_position_shared_ pmpi_file_get_position_shared
-#endif
-#define mpi_file_get_position_shared_ pmpi_file_get_position_shared_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_get_position_shared_ MPI_FILE_GET_POSITION_SHARED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_get_position_shared_ mpi_file_get_position_shared__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_get_position_shared mpi_file_get_position_shared_
-#endif
-#define mpi_file_get_position_shared_ mpi_file_get_position_shared
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_get_position_shared_ mpi_file_get_position_shared
-#endif
-#endif
-#endif
-
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_get_position_shared_(MPI_Fint *fh, MPI_Offset *offset, 
-				   MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_get_position_shared_(MPI_Fint *fh, MPI_Offset *offset, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_get_position_shared(fh_c, offset);
-}
diff --git a/3rd-party/romio321/mpi-io/fortran/get_posnf.c b/3rd-party/romio321/mpi-io/fortran/get_posnf.c
deleted file mode 100644
index e21f37674a7..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/get_posnf.c
+++ /dev/null
@@ -1,101 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_GET_POSITION( MPI_Fint *, MPI_Offset*, MPI_Fint * );
-#pragma weak MPI_FILE_GET_POSITION = PMPI_FILE_GET_POSITION
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_get_position__( MPI_Fint *, MPI_Offset*, MPI_Fint * );
-#pragma weak mpi_file_get_position__ = pmpi_file_get_position__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_get_position( MPI_Fint *, MPI_Offset*, MPI_Fint * );
-#pragma weak mpi_file_get_position = pmpi_file_get_position
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_get_position_( MPI_Fint *, MPI_Offset*, MPI_Fint * );
-#pragma weak mpi_file_get_position_ = pmpi_file_get_position_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_GET_POSITION MPI_FILE_GET_POSITION
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_position__ mpi_file_get_position__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_position mpi_file_get_position
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_get_position_ mpi_file_get_position_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_GET_POSITION as PMPI_FILE_GET_POSITION
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_get_position__ as pmpi_file_get_position__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_get_position as pmpi_file_get_position
-#else
-#pragma _CRI duplicate mpi_file_get_position_ as pmpi_file_get_position_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_get_position_ PMPI_FILE_GET_POSITION
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_get_position_ pmpi_file_get_position__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_position pmpi_file_get_position_
-#endif
-#define mpi_file_get_position_ pmpi_file_get_position
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_position_ pmpi_file_get_position
-#endif
-#define mpi_file_get_position_ pmpi_file_get_position_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_get_position_ MPI_FILE_GET_POSITION
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_get_position_ mpi_file_get_position__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_get_position mpi_file_get_position_
-#endif
-#define mpi_file_get_position_ mpi_file_get_position
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_get_position_ mpi_file_get_position
-#endif
-#endif
-#endif
-
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_get_position_(MPI_Fint *fh, MPI_Offset *offset, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_get_position_(MPI_Fint *fh, MPI_Offset *offset, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_get_position(fh_c, offset);
-}
diff --git a/3rd-party/romio321/mpi-io/fortran/get_sizef.c b/3rd-party/romio321/mpi-io/fortran/get_sizef.c
deleted file mode 100644
index 71ff17bb562..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/get_sizef.c
+++ /dev/null
@@ -1,102 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_GET_SIZE( MPI_Fint *, MPI_Offset*, MPI_Fint * );
-#pragma weak MPI_FILE_GET_SIZE = PMPI_FILE_GET_SIZE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_get_size__( MPI_Fint *, MPI_Offset*, MPI_Fint * );
-#pragma weak mpi_file_get_size__ = pmpi_file_get_size__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_get_size( MPI_Fint *, MPI_Offset*, MPI_Fint * );
-#pragma weak mpi_file_get_size = pmpi_file_get_size
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_get_size_( MPI_Fint *, MPI_Offset*, MPI_Fint * );
-#pragma weak mpi_file_get_size_ = pmpi_file_get_size_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_GET_SIZE MPI_FILE_GET_SIZE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_size__ mpi_file_get_size__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_size mpi_file_get_size
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_get_size_ mpi_file_get_size_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_GET_SIZE as PMPI_FILE_GET_SIZE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_get_size__ as pmpi_file_get_size__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_get_size as pmpi_file_get_size
-#else
-#pragma _CRI duplicate mpi_file_get_size_ as pmpi_file_get_size_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_get_size_ PMPI_FILE_GET_SIZE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_get_size_ pmpi_file_get_size__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_size pmpi_file_get_size_
-#endif
-#define mpi_file_get_size_ pmpi_file_get_size
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_size_ pmpi_file_get_size
-#endif
-#define mpi_file_get_size_ pmpi_file_get_size_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_get_size_ MPI_FILE_GET_SIZE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_get_size_ mpi_file_get_size__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_get_size mpi_file_get_size_
-#endif
-#define mpi_file_get_size_ mpi_file_get_size
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_get_size_ mpi_file_get_size
-#endif
-#endif
-#endif
-
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_get_size_(MPI_Fint *fh, MPI_Offset *size, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_get_size_(MPI_Fint *fh, MPI_Offset *size, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_get_size(fh_c, size);
-}
-
diff --git a/3rd-party/romio321/mpi-io/fortran/get_viewf.c b/3rd-party/romio321/mpi-io/fortran/get_viewf.c
deleted file mode 100644
index 8a4bc3d7fb8..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/get_viewf.c
+++ /dev/null
@@ -1,187 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#ifdef _UNICOS
-#include <fortran.h>
-#endif
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_GET_VIEW( MPI_Fint *, MPI_Offset*, MPI_Fint*, MPI_Fint*, char * FORT_MIXED_LEN_DECL, MPI_Fint * FORT_END_LEN_DECL );
-#pragma weak MPI_FILE_GET_VIEW = PMPI_FILE_GET_VIEW
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_get_view__( MPI_Fint *, MPI_Offset*, MPI_Fint*, MPI_Fint*, char * FORT_MIXED_LEN_DECL, MPI_Fint * FORT_END_LEN_DECL );
-#pragma weak mpi_file_get_view__ = pmpi_file_get_view__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_get_view( MPI_Fint *, MPI_Offset*, MPI_Fint*, MPI_Fint*, char * FORT_MIXED_LEN_DECL, MPI_Fint * FORT_END_LEN_DECL );
-#pragma weak mpi_file_get_view = pmpi_file_get_view
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_get_view_( MPI_Fint *, MPI_Offset*, MPI_Fint*, MPI_Fint*, char * FORT_MIXED_LEN_DECL, MPI_Fint * FORT_END_LEN_DECL );
-#pragma weak mpi_file_get_view_ = pmpi_file_get_view_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_GET_VIEW MPI_FILE_GET_VIEW
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_view__ mpi_file_get_view__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_view mpi_file_get_view
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_get_view_ mpi_file_get_view_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_GET_VIEW as PMPI_FILE_GET_VIEW
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_get_view__ as pmpi_file_get_view__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_get_view as pmpi_file_get_view
-#else
-#pragma _CRI duplicate mpi_file_get_view_ as pmpi_file_get_view_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_get_view_ PMPI_FILE_GET_VIEW
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_get_view_ pmpi_file_get_view__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_view pmpi_file_get_view_
-#endif
-#define mpi_file_get_view_ pmpi_file_get_view
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_get_view_ pmpi_file_get_view
-#endif
-#define mpi_file_get_view_ pmpi_file_get_view_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_get_view_ MPI_FILE_GET_VIEW
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_get_view_ mpi_file_get_view__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_get_view mpi_file_get_view_
-#endif
-#define mpi_file_get_view_ mpi_file_get_view
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_get_view_ mpi_file_get_view
-#endif
-#endif
-#endif
-
-#if defined(MPIHP) || defined(MPILAM)
-/* Prototype to keep compiler happy */
-void mpi_file_get_view_(MPI_Fint *fh,MPI_Offset *disp,MPI_Fint *etype,
-		MPI_Fint *filetype,char *datarep, MPI_Fint *ierr, int str_len );
-
-void mpi_file_get_view_(MPI_Fint *fh,MPI_Offset *disp,MPI_Fint *etype,
-   MPI_Fint *filetype,char *datarep, MPI_Fint *ierr, int str_len )
-{
-    MPI_File fh_c;
-    MPI_Datatype etype_c, filetype_c;
-    int i, tmpreplen;
-    char *tmprep;
-
-    if (datarep <= (char *) 0) {
-        FPRINTF(stderr, "MPI_File_get_view: datarep is an invalid address\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    tmprep = (char *) ADIOI_Malloc((MPI_MAX_DATAREP_STRING+1) * sizeof(char));
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_get_view(fh_c, disp, &etype_c, &filetype_c, tmprep);
-
-    tmpreplen = strlen(tmprep);
-    if (tmpreplen <= str_len) {
-        ADIOI_Strncpy(datarep, tmprep, tmpreplen);
-
-        /* blank pad the remaining space */
-        for (i=tmpreplen; i<str_len; i++) datarep[i] = ' ';
-    }
-    else {
-        /* not enough space */
-        ADIOI_Strncpy(datarep, tmprep, str_len);
-        /* this should be flagged as an error. */
-        *ierr = MPI_ERR_UNKNOWN;
-    }
-    
-    *etype = MPI_Type_c2f(etype_c);
-    *filetype = MPI_Type_c2f(filetype_c);
-    ADIOI_Free(tmprep);
-}
-
-#else
-
-#ifdef _UNICOS
-void mpi_file_get_view_(MPI_Fint *fh,MPI_Offset *disp,MPI_Fint *etype,
-   MPI_Fint *filetype, _fcd datarep_fcd, MPI_Fint *ierr)
-{
-    char *datarep = _fcdtocp(datarep_fcd);
-    int str_len = _fcdlen(datarep_fcd);
-#else
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_get_view_( MPI_Fint *fh, MPI_Offset *disp, MPI_Fint *etype, MPI_Fint *filetype, char *datarep FORT_MIXED_LEN_DECL, MPI_Fint *ierr FORT_END_LEN_DECL );
-
-FORTRAN_API void FORT_CALL mpi_file_get_view_( MPI_Fint *fh, MPI_Offset *disp, MPI_Fint *etype, MPI_Fint *filetype, char *datarep FORT_MIXED_LEN(str_len), MPI_Fint *ierr FORT_END_LEN(str_len) )
-{
-#endif
-    MPI_File fh_c;
-    int i, tmpreplen;
-    MPI_Datatype etype_c, filetype_c;
-
-    char *tmprep;
-
-/* Initialize the string to all blanks */
-    if (datarep <= (char *) 0) {
-        FPRINTF(stderr, "MPI_File_get_view: datarep is an invalid address\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-    
-    tmprep = (char *) ADIOI_Malloc((MPI_MAX_DATAREP_STRING+1) * sizeof(char));
-    fh_c = MPI_File_f2c(*fh);
-    etype_c = MPI_Type_f2c(*etype);
-    filetype_c = MPI_Type_f2c(*filetype);
-    *ierr = MPI_File_get_view(fh_c, disp, &etype_c, &filetype_c, tmprep);
-
-    tmpreplen = strlen(tmprep);
-    if (tmpreplen <= str_len) {
-        ADIOI_Strncpy(datarep, tmprep, tmpreplen);
-
-        /* blank pad the remaining space */
-        for (i=tmpreplen; i<str_len; i++) datarep[i] = ' ';
-    }
-    else {
-        /* not enough space */
-        ADIOI_Strncpy(datarep, tmprep, str_len);
-        /* this should be flagged as an error. */
-        *ierr = MPI_ERR_UNKNOWN;
-    }
-
-    *etype = MPI_Type_c2f(etype_c);
-    *filetype = MPI_Type_c2f(filetype_c);
-    ADIOI_Free(tmprep);
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/fortran/iotestf.c b/3rd-party/romio321/mpi-io/fortran/iotestf.c
deleted file mode 100644
index 2a2c96c886b..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/iotestf.c
+++ /dev/null
@@ -1,101 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-FORTRAN_API void FORT_CALL MPIO_TEST(MPI_Fint *request,MPI_Fint *flag,MPI_Status *status, MPI_Fint *ierr );
-#pragma weak MPIO_TEST = PMPIO_TEST
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-FORTRAN_API void FORT_CALL mpio_test__(MPI_Fint *request,MPI_Fint *flag,MPI_Status *status, MPI_Fint *ierr );
-#pragma weak mpio_test__ = pmpio_test__
-#elif !defined(FORTRANUNDERSCORE)
-FORTRAN_API void FORT_CALL mpio_test(MPI_Fint *request,MPI_Fint *flag,MPI_Status *status, MPI_Fint *ierr );
-#pragma weak mpio_test = pmpio_test
-#else
-FORTRAN_API void FORT_CALL mpio_test_(MPI_Fint *request,MPI_Fint *flag,MPI_Status *status, MPI_Fint *ierr );
-#pragma weak mpio_test_ = pmpio_test_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPIO_TEST MPIO_TEST
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpio_test__ mpio_test__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpio_test mpio_test
-#else
-#pragma _HP_SECONDARY_DEF pmpio_test_ mpio_test_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPIO_TEST as PMPIO_TEST
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpio_test__ as pmpio_test__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpio_test as pmpio_test
-#else
-#pragma _CRI duplicate mpio_test_ as pmpio_test_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpio_test_ PMPIO_TEST
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpio_test_ pmpio_test__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpio_test pmpio_test_
-#endif
-#define mpio_test_ pmpio_test
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpio_test_ pmpio_test
-#endif
-#define mpio_test_ pmpio_test_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpio_test_ MPIO_TEST
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpio_test_ mpio_test__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpio_test mpio_test_
-#endif
-#define mpio_test_ mpio_test
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpio_test_ mpio_test
-#endif
-#endif
-#endif
-
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpio_test_(MPI_Fint *request,MPI_Fint *flag,MPI_Status *status, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpio_test_(MPI_Fint *request,MPI_Fint *flag,MPI_Status *status, MPI_Fint *ierr )
-{
-    MPIO_Request req_c;
-    
-    req_c = MPIO_Request_f2c(*request);
-    *ierr = MPIO_Test(&req_c,flag,status);
-    *request = MPIO_Request_c2f(req_c);
-}
diff --git a/3rd-party/romio321/mpi-io/fortran/iowaitf.c b/3rd-party/romio321/mpi-io/fortran/iowaitf.c
deleted file mode 100644
index a87ca6a6d91..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/iowaitf.c
+++ /dev/null
@@ -1,101 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-FORTRAN_API void FORT_CALL MPIO_WAIT(MPI_Fint *request,MPI_Status *status, MPI_Fint *ierr );
-#pragma weak MPIO_WAIT = PMPIO_WAIT
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-FORTRAN_API void FORT_CALL mpio_wait__(MPI_Fint *request,MPI_Status *status, MPI_Fint *ierr );
-#pragma weak mpio_wait__ = pmpio_wait__
-#elif !defined(FORTRANUNDERSCORE)
-FORTRAN_API void FORT_CALL mpio_wait(MPI_Fint *request,MPI_Status *status, MPI_Fint *ierr );
-#pragma weak mpio_wait = pmpio_wait
-#else
-FORTRAN_API void FORT_CALL mpio_wait_(MPI_Fint *request,MPI_Status *status, MPI_Fint *ierr );
-#pragma weak mpio_wait_ = pmpio_wait_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPIO_WAIT MPIO_WAIT
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpio_wait__ mpio_wait__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpio_wait mpio_wait
-#else
-#pragma _HP_SECONDARY_DEF pmpio_wait_ mpio_wait_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPIO_WAIT as PMPIO_WAIT
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpio_wait__ as pmpio_wait__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpio_wait as pmpio_wait
-#else
-#pragma _CRI duplicate mpio_wait_ as pmpio_wait_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpio_wait_ PMPIO_WAIT
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpio_wait_ pmpio_wait__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpio_wait pmpio_wait_
-#endif
-#define mpio_wait_ pmpio_wait
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpio_wait_ pmpio_wait
-#endif
-#define mpio_wait_ pmpio_wait_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpio_wait_ MPIO_WAIT
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpio_wait_ mpio_wait__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpio_wait mpio_wait_
-#endif
-#define mpio_wait_ mpio_wait
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpio_wait_ mpio_wait
-#endif
-#endif
-#endif
-
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpio_wait_(MPI_Fint *request,MPI_Status *status, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpio_wait_(MPI_Fint *request,MPI_Status *status, MPI_Fint *ierr )
-{
-    MPIO_Request req_c;
-    
-    req_c = MPIO_Request_f2c(*request);
-    *ierr = MPIO_Wait(&req_c, status);
-    *request = MPIO_Request_c2f(req_c);
-}
diff --git a/3rd-party/romio321/mpi-io/fortran/iread_atf.c b/3rd-party/romio321/mpi-io/fortran/iread_atf.c
deleted file mode 100644
index c6a687dcee0..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/iread_atf.c
+++ /dev/null
@@ -1,130 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_IREAD_AT( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint*, MPI_Fint * );
-#pragma weak MPI_FILE_IREAD_AT = PMPI_FILE_IREAD_AT
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_iread_at__( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint*, MPI_Fint * );
-#pragma weak mpi_file_iread_at__ = pmpi_file_iread_at__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_iread_at( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint*, MPI_Fint * );
-#pragma weak mpi_file_iread_at = pmpi_file_iread_at
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_iread_at_( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint*, MPI_Fint * );
-#pragma weak mpi_file_iread_at_ = pmpi_file_iread_at_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_IREAD_AT MPI_FILE_IREAD_AT
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_iread_at__ mpi_file_iread_at__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_iread_at mpi_file_iread_at
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_iread_at_ mpi_file_iread_at_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_IREAD_AT as PMPI_FILE_IREAD_AT
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_iread_at__ as pmpi_file_iread_at__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_iread_at as pmpi_file_iread_at
-#else
-#pragma _CRI duplicate mpi_file_iread_at_ as pmpi_file_iread_at_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_iread_at_ PMPI_FILE_IREAD_AT
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_iread_at_ pmpi_file_iread_at__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_iread_at pmpi_file_iread_at_
-#endif
-#define mpi_file_iread_at_ pmpi_file_iread_at
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_iread_at_ pmpi_file_iread_at
-#endif
-#define mpi_file_iread_at_ pmpi_file_iread_at_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_iread_at_ MPI_FILE_IREAD_AT
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_iread_at_ mpi_file_iread_at__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_iread_at mpi_file_iread_at_
-#endif
-#define mpi_file_iread_at_ mpi_file_iread_at
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_iread_at_ mpi_file_iread_at
-#endif
-#endif
-#endif
-
-#if defined(MPIHP) || defined(MPILAM)
-/* Prototype to keep compiler happy */
-void mpi_file_iread_at_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-			MPI_Fint *count,MPI_Fint *datatype,
-			MPI_Fint *request, MPI_Fint *ierr );
-
-void mpi_file_iread_at_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-                      MPI_Fint *count,MPI_Fint *datatype,
-                      MPI_Fint *request, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPIO_Request req_c;
-    MPI_Datatype datatype_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    datatype_c = MPI_Type_f2c(*datatype);
-
-    *ierr = MPI_File_iread_at(fh_c,*offset,buf,*count,datatype_c,&req_c);
-
-    *request = MPIO_Request_c2f(req_c);
-}
-#else
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_iread_at_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-			MPI_Fint *count,MPI_Datatype *datatype,
-			MPI_Fint *request, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_iread_at_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-                      MPI_Fint *count,MPI_Datatype *datatype,
-                      MPI_Fint *request, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPIO_Request req_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_iread_at(fh_c,*offset,buf,*count,*datatype,&req_c);
-    *request = MPIO_Request_c2f(req_c);
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/fortran/iread_shf.c b/3rd-party/romio321/mpi-io/fortran/iread_shf.c
deleted file mode 100644
index 84dc3753c4a..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/iread_shf.c
+++ /dev/null
@@ -1,124 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_IREAD_SHARED( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint*, MPI_Fint * );
-#pragma weak MPI_FILE_IREAD_SHARED = PMPI_FILE_IREAD_SHARED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_iread_shared__( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint*, MPI_Fint * );
-#pragma weak mpi_file_iread_shared__ = pmpi_file_iread_shared__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_iread_shared( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint*, MPI_Fint * );
-#pragma weak mpi_file_iread_shared = pmpi_file_iread_shared
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_iread_shared_( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint*, MPI_Fint * );
-#pragma weak mpi_file_iread_shared_ = pmpi_file_iread_shared_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_IREAD_SHARED MPI_FILE_IREAD_SHARED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_iread_shared__ mpi_file_iread_shared__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_iread_shared mpi_file_iread_shared
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_iread_shared_ mpi_file_iread_shared_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_IREAD_SHARED as PMPI_FILE_IREAD_SHARED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_iread_shared__ as pmpi_file_iread_shared__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_iread_shared as pmpi_file_iread_shared
-#else
-#pragma _CRI duplicate mpi_file_iread_shared_ as pmpi_file_iread_shared_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_iread_shared_ PMPI_FILE_IREAD_SHARED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_iread_shared_ pmpi_file_iread_shared__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_iread_shared pmpi_file_iread_shared_
-#endif
-#define mpi_file_iread_shared_ pmpi_file_iread_shared
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_iread_shared_ pmpi_file_iread_shared
-#endif
-#define mpi_file_iread_shared_ pmpi_file_iread_shared_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_iread_shared_ MPI_FILE_IREAD_SHARED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_iread_shared_ mpi_file_iread_shared__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_iread_shared mpi_file_iread_shared_
-#endif
-#define mpi_file_iread_shared_ mpi_file_iread_shared
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_iread_shared_ mpi_file_iread_shared
-#endif
-#endif
-#endif
-
-#if defined(MPIHP) || defined(MPILAM)
-/* Prototype to keep compiler happy */
-void mpi_file_iread_shared_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-		    MPI_Fint *datatype,MPI_Fint *request, MPI_Fint *ierr );
-
-void mpi_file_iread_shared_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                   MPI_Fint *datatype,MPI_Fint *request, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPIO_Request req_c;
-    MPI_Datatype datatype_c;
-    
-    datatype_c = MPI_Type_f2c(*datatype);
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_iread_shared(fh_c,buf,*count,datatype_c,&req_c);
-    *request = MPIO_Request_c2f(req_c);
-}
-#else
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_iread_shared_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-		    MPI_Datatype *datatype,MPI_Fint *request, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_iread_shared_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                   MPI_Datatype *datatype,MPI_Fint *request, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPIO_Request req_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_iread_shared(fh_c,buf,*count,*datatype,&req_c);
-    *request = MPIO_Request_c2f(req_c);
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/fortran/ireadf.c b/3rd-party/romio321/mpi-io/fortran/ireadf.c
deleted file mode 100644
index ae9fbaf3bfd..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/ireadf.c
+++ /dev/null
@@ -1,124 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_IREAD( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint*, MPI_Fint * );
-#pragma weak MPI_FILE_IREAD = PMPI_FILE_IREAD
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_iread__( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint*, MPI_Fint * );
-#pragma weak mpi_file_iread__ = pmpi_file_iread__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_iread( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint*, MPI_Fint * );
-#pragma weak mpi_file_iread = pmpi_file_iread
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_iread_( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint*, MPI_Fint * );
-#pragma weak mpi_file_iread_ = pmpi_file_iread_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_IREAD MPI_FILE_IREAD
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_iread__ mpi_file_iread__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_iread mpi_file_iread
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_iread_ mpi_file_iread_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_IREAD as PMPI_FILE_IREAD
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_iread__ as pmpi_file_iread__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_iread as pmpi_file_iread
-#else
-#pragma _CRI duplicate mpi_file_iread_ as pmpi_file_iread_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_iread_ PMPI_FILE_IREAD
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_iread_ pmpi_file_iread__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_iread pmpi_file_iread_
-#endif
-#define mpi_file_iread_ pmpi_file_iread
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_iread_ pmpi_file_iread
-#endif
-#define mpi_file_iread_ pmpi_file_iread_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_iread_ MPI_FILE_IREAD
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_iread_ mpi_file_iread__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_iread mpi_file_iread_
-#endif
-#define mpi_file_iread_ mpi_file_iread
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_iread_ mpi_file_iread
-#endif
-#endif
-#endif
-
-#if defined(MPIHP) || defined(MPILAM)
-/* Prototype to keep compiler happy */
-void mpi_file_iread_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-		     MPI_Fint *datatype,MPI_Fint *request, MPI_Fint *ierr );
-
-void mpi_file_iread_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                   MPI_Fint *datatype,MPI_Fint *request, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPIO_Request req_c;
-    MPI_Datatype datatype_c;
-    
-    datatype_c = MPI_Type_f2c(*datatype);
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_iread(fh_c,buf,*count,datatype_c,&req_c);
-    *request = MPIO_Request_c2f(req_c);
-}
-#else
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_iread_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-		     MPI_Datatype *datatype,MPI_Fint *request, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_iread_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                   MPI_Datatype *datatype,MPI_Fint *request, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPIO_Request req_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_iread(fh_c,buf,*count,*datatype,&req_c);
-    *request = MPIO_Request_c2f(req_c);
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/fortran/iwrite_atf.c b/3rd-party/romio321/mpi-io/fortran/iwrite_atf.c
deleted file mode 100644
index b326fc3d1a7..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/iwrite_atf.c
+++ /dev/null
@@ -1,129 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_IWRITE_AT( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint*, MPI_Fint * );
-#pragma weak MPI_FILE_IWRITE_AT = PMPI_FILE_IWRITE_AT
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_iwrite_at__( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint*, MPI_Fint * );
-#pragma weak mpi_file_iwrite_at__ = pmpi_file_iwrite_at__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_iwrite_at( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint*, MPI_Fint * );
-#pragma weak mpi_file_iwrite_at = pmpi_file_iwrite_at
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_iwrite_at_( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint*, MPI_Fint * );
-#pragma weak mpi_file_iwrite_at_ = pmpi_file_iwrite_at_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_IWRITE_AT MPI_FILE_IWRITE_AT
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_iwrite_at__ mpi_file_iwrite_at__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_iwrite_at mpi_file_iwrite_at
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_iwrite_at_ mpi_file_iwrite_at_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_IWRITE_AT as PMPI_FILE_IWRITE_AT
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_iwrite_at__ as pmpi_file_iwrite_at__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_iwrite_at as pmpi_file_iwrite_at
-#else
-#pragma _CRI duplicate mpi_file_iwrite_at_ as pmpi_file_iwrite_at_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_iwrite_at_ PMPI_FILE_IWRITE_AT
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_iwrite_at_ pmpi_file_iwrite_at__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_iwrite_at pmpi_file_iwrite_at_
-#endif
-#define mpi_file_iwrite_at_ pmpi_file_iwrite_at
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_iwrite_at_ pmpi_file_iwrite_at
-#endif
-#define mpi_file_iwrite_at_ pmpi_file_iwrite_at_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_iwrite_at_ MPI_FILE_IWRITE_AT
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_iwrite_at_ mpi_file_iwrite_at__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_iwrite_at mpi_file_iwrite_at_
-#endif
-#define mpi_file_iwrite_at_ mpi_file_iwrite_at
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_iwrite_at_ mpi_file_iwrite_at
-#endif
-#endif
-#endif
-
-#if defined(MPIHP) || defined(MPILAM)
-/* Prototype to keep compiler happy */
-void mpi_file_iwrite_at_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-                       MPI_Fint *count,MPI_Fint *datatype,
-			 MPI_Fint *request, MPI_Fint *ierr );
-
-void mpi_file_iwrite_at_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-                       MPI_Fint *count,MPI_Fint *datatype,
-                       MPI_Fint *request, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPIO_Request req_c;
-    MPI_Datatype datatype_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    datatype_c = MPI_Type_f2c(*datatype);
-
-    *ierr = MPI_File_iwrite_at(fh_c,*offset,buf,*count,datatype_c,&req_c);
-    *request = MPIO_Request_c2f(req_c);
-}
-#else
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_iwrite_at_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-                       MPI_Fint *count,MPI_Datatype *datatype,
-			 MPI_Fint *request, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_iwrite_at_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-                       MPI_Fint *count,MPI_Datatype *datatype,
-                       MPI_Fint *request, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPIO_Request req_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_iwrite_at(fh_c,*offset,buf,*count,*datatype,&req_c);
-    *request = MPIO_Request_c2f(req_c);
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/fortran/iwrite_shf.c b/3rd-party/romio321/mpi-io/fortran/iwrite_shf.c
deleted file mode 100644
index 90ea68777b6..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/iwrite_shf.c
+++ /dev/null
@@ -1,124 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_IWRITE_SHARED( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint*, MPI_Fint * );
-#pragma weak MPI_FILE_IWRITE_SHARED = PMPI_FILE_IWRITE_SHARED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_iwrite_shared__( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint*, MPI_Fint * );
-#pragma weak mpi_file_iwrite_shared__ = pmpi_file_iwrite_shared__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_iwrite_shared( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint*, MPI_Fint * );
-#pragma weak mpi_file_iwrite_shared = pmpi_file_iwrite_shared
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_iwrite_shared_( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint*, MPI_Fint * );
-#pragma weak mpi_file_iwrite_shared_ = pmpi_file_iwrite_shared_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_IWRITE_SHARED MPI_FILE_IWRITE_SHARED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_iwrite_shared__ mpi_file_iwrite_shared__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_iwrite_shared mpi_file_iwrite_shared
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_iwrite_shared_ mpi_file_iwrite_shared_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_IWRITE_SHARED as PMPI_FILE_IWRITE_SHARED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_iwrite_shared__ as pmpi_file_iwrite_shared__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_iwrite_shared as pmpi_file_iwrite_shared
-#else
-#pragma _CRI duplicate mpi_file_iwrite_shared_ as pmpi_file_iwrite_shared_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_iwrite_shared_ PMPI_FILE_IWRITE_SHARED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_iwrite_shared_ pmpi_file_iwrite_shared__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_iwrite_shared pmpi_file_iwrite_shared_
-#endif
-#define mpi_file_iwrite_shared_ pmpi_file_iwrite_shared
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_iwrite_shared_ pmpi_file_iwrite_shared
-#endif
-#define mpi_file_iwrite_shared_ pmpi_file_iwrite_shared_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_iwrite_shared_ MPI_FILE_IWRITE_SHARED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_iwrite_shared_ mpi_file_iwrite_shared__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_iwrite_shared mpi_file_iwrite_shared_
-#endif
-#define mpi_file_iwrite_shared_ mpi_file_iwrite_shared
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_iwrite_shared_ mpi_file_iwrite_shared
-#endif
-#endif
-#endif
-
-#if defined(MPIHP) || defined(MPILAM)
-/* Prototype to keep compiler happy */
-void mpi_file_iwrite_shared_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-			     MPI_Fint *datatype,MPI_Fint *request, MPI_Fint *ierr );
-void mpi_file_iwrite_shared_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                    MPI_Fint *datatype,MPI_Fint *request, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPIO_Request req_c;
-    MPI_Datatype datatype_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    datatype_c = MPI_Type_f2c(*datatype);
-
-    *ierr = MPI_File_iwrite_shared(fh_c,buf,*count,datatype_c,&req_c);
-    *request = MPIO_Request_c2f(req_c);
-}
-#else
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_iwrite_shared_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-			     MPI_Datatype *datatype,MPI_Fint *request, MPI_Fint *ierr ); 
-
-FORTRAN_API void FORT_CALL mpi_file_iwrite_shared_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                    MPI_Datatype *datatype,MPI_Fint *request, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPIO_Request req_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_iwrite_shared(fh_c,buf,*count,*datatype,&req_c);
-    *request = MPIO_Request_c2f(req_c);
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/fortran/iwritef.c b/3rd-party/romio321/mpi-io/fortran/iwritef.c
deleted file mode 100644
index 4410c011a14..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/iwritef.c
+++ /dev/null
@@ -1,125 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_IWRITE( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint*, MPI_Fint * );
-#pragma weak MPI_FILE_IWRITE = PMPI_FILE_IWRITE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_iwrite__( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint*, MPI_Fint * );
-#pragma weak mpi_file_iwrite__ = pmpi_file_iwrite__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_iwrite( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint*, MPI_Fint * );
-#pragma weak mpi_file_iwrite = pmpi_file_iwrite
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_iwrite_( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint*, MPI_Fint * );
-#pragma weak mpi_file_iwrite_ = pmpi_file_iwrite_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_IWRITE MPI_FILE_IWRITE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_iwrite__ mpi_file_iwrite__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_iwrite mpi_file_iwrite
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_iwrite_ mpi_file_iwrite_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_IWRITE as PMPI_FILE_IWRITE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_iwrite__ as pmpi_file_iwrite__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_iwrite as pmpi_file_iwrite
-#else
-#pragma _CRI duplicate mpi_file_iwrite_ as pmpi_file_iwrite_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_iwrite_ PMPI_FILE_IWRITE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_iwrite_ pmpi_file_iwrite__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_iwrite pmpi_file_iwrite_
-#endif
-#define mpi_file_iwrite_ pmpi_file_iwrite
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_iwrite_ pmpi_file_iwrite
-#endif
-#define mpi_file_iwrite_ pmpi_file_iwrite_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_iwrite_ MPI_FILE_IWRITE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_iwrite_ mpi_file_iwrite__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_iwrite mpi_file_iwrite_
-#endif
-#define mpi_file_iwrite_ mpi_file_iwrite
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_iwrite_ mpi_file_iwrite
-#endif
-#endif
-#endif
-
-#if defined(MPIHP) || defined(MPILAM)
-/* Prototype to keep compiler happy */
-void mpi_file_iwrite_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-		      MPI_Fint *datatype,MPI_Fint *request, MPI_Fint *ierr );
-
-void mpi_file_iwrite_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                    MPI_Fint *datatype,MPI_Fint *request, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPIO_Request req_c;
-    MPI_Datatype datatype_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    datatype_c = MPI_Type_f2c(*datatype);
-
-    *ierr = MPI_File_iwrite(fh_c,buf,*count,datatype_c,&req_c);
-    *request = MPIO_Request_c2f(req_c);
-}
-#else
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_iwrite_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-		      MPI_Datatype *datatype,MPI_Fint *request, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_iwrite_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                    MPI_Datatype *datatype,MPI_Fint *request, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPIO_Request req_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_iwrite(fh_c,buf,*count,*datatype,&req_c);
-    *request = MPIO_Request_c2f(req_c);
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/fortran/openf.c b/3rd-party/romio321/mpi-io/fortran/openf.c
deleted file mode 100644
index ea1a5d1c47e..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/openf.c
+++ /dev/null
@@ -1,187 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#ifdef _UNICOS
-#include <fortran.h>
-#endif
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_OPEN( MPI_Fint *, char * FORT_MIXED_LEN_DECL, MPI_Fint *, MPI_Fint *, MPI_File*, MPI_Fint * FORT_END_LEN_DECL );
-#pragma weak MPI_FILE_OPEN = PMPI_FILE_OPEN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_open__( MPI_Fint *, char * FORT_MIXED_LEN_DECL, MPI_Fint *, MPI_Fint *, MPI_File*, MPI_Fint * FORT_END_LEN_DECL );
-#pragma weak mpi_file_open__ = pmpi_file_open__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_open( MPI_Fint *, char * FORT_MIXED_LEN_DECL, MPI_Fint *, MPI_Fint *, MPI_File*, MPI_Fint * FORT_END_LEN_DECL );
-#pragma weak mpi_file_open = pmpi_file_open
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_open_( MPI_Fint *, char * FORT_MIXED_LEN_DECL, MPI_Fint *, MPI_Fint *, MPI_File*, MPI_Fint * FORT_END_LEN_DECL );
-#pragma weak mpi_file_open_ = pmpi_file_open_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_OPEN MPI_FILE_OPEN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_open__ mpi_file_open__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_open mpi_file_open
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_open_ mpi_file_open_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_OPEN as PMPI_FILE_OPEN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_open__ as pmpi_file_open__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_open as pmpi_file_open
-#else
-#pragma _CRI duplicate mpi_file_open_ as pmpi_file_open_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_open_ PMPI_FILE_OPEN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_open_ pmpi_file_open__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_open pmpi_file_open_
-#endif
-#define mpi_file_open_ pmpi_file_open
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_open_ pmpi_file_open
-#endif
-#define mpi_file_open_ pmpi_file_open_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_open_ MPI_FILE_OPEN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_open_ mpi_file_open__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_open mpi_file_open_
-#endif
-#define mpi_file_open_ mpi_file_open
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_open_ mpi_file_open
-#endif
-#endif
-#endif
-
-#if defined(MPIHP) || defined(MPILAM)
-/* Prototype to keep compiler happy */
-void mpi_file_open_(MPI_Fint *comm,char *filename,MPI_Fint *amode,
-		    MPI_Fint *info, MPI_Fint *fh, MPI_Fint *ierr, int str_len );
-
-void mpi_file_open_(MPI_Fint *comm,char *filename,MPI_Fint *amode,
-                  MPI_Fint *info, MPI_Fint *fh, MPI_Fint *ierr, int str_len )
-{
-    char *newfname;
-    MPI_File fh_c;
-    int real_len, i;
-    MPI_Comm comm_c;
-    MPI_Info info_c;
-
-    comm_c = MPI_Comm_f2c(*comm);
-    info_c = MPI_Info_f2c(*info);
-
-    /* strip trailing blanks */
-    if (filename <= (char *) 0) {
-        FPRINTF(stderr, "MPI_File_open: filename is an invalid address\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-    for (i=str_len-1; i>=0; i--) if (filename[i] != ' ') break;
-    if (i < 0) {
-	FPRINTF(stderr, "MPI_File_open: filename is a blank string\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-    real_len = i + 1;
-
-    newfname = (char *) ADIOI_Malloc((real_len+1)*sizeof(char));
-    ADIOI_Strncpy(newfname, filename, real_len);
-    newfname[real_len] = '\0';
-
-    *ierr = MPI_File_open(comm_c, newfname, *amode, info_c, &fh_c);
-
-    *fh = MPI_File_c2f(fh_c);
-    ADIOI_Free(newfname);
-}
-
-#else
-
-#ifdef _UNICOS
-void mpi_file_open_(MPI_Fint *comm,_fcd filename_fcd,MPI_Fint *amode,
-                  MPI_Fint *info, MPI_Fint *fh, MPI_Fint *ierr)
-{
-    char *filename = _fcdtocp(filename_fcd);
-    int str_len = _fcdlen(filename_fcd);
-#else
-/* Prototype to keep compiler happy */
-/*
-FORTRAN_API void FORT_CALL mpi_file_open_(MPI_Comm *comm,char *filename,MPI_Fint *amode,
-		    MPI_Fint *info, MPI_Fint *fh, MPI_Fint *ierr, int str_len );
-
-FORTRAN_API void FORT_CALL mpi_file_open_(MPI_Comm *comm,char *filename,MPI_Fint *amode,
-                  MPI_Fint *info, MPI_Fint *fh, MPI_Fint *ierr, int str_len )
-*/
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_open_(MPI_Fint *comm,char *filename FORT_MIXED_LEN_DECL,MPI_Fint *amode,
-		    MPI_Fint *info, MPI_Fint *fh, MPI_Fint *ierr FORT_END_LEN_DECL);
-
-FORTRAN_API void FORT_CALL mpi_file_open_(MPI_Fint *comm,char *filename FORT_MIXED_LEN(str_len),MPI_Fint *amode,
-                  MPI_Fint *info, MPI_Fint *fh, MPI_Fint *ierr FORT_END_LEN(str_len))
-{
-#endif
-    char *newfname;
-    MPI_File fh_c;
-    int real_len, i;
-    MPI_Info info_c;
-
-    info_c = MPI_Info_f2c(*info);
-
-    /* strip trailing blanks */
-    if (filename <= (char *) 0) {
-        FPRINTF(stderr, "MPI_File_open: filename is an invalid address\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-    for (i=str_len-1; i>=0; i--) if (filename[i] != ' ') break;
-    if (i < 0) {
-	FPRINTF(stderr, "MPI_File_open: filename is a blank string\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-    real_len = i + 1;
-
-    newfname = (char *) ADIOI_Malloc((real_len+1)*sizeof(char));
-    ADIOI_Strncpy(newfname, filename, real_len);
-    newfname[real_len] = '\0';
-
-    *ierr = MPI_File_open((MPI_Comm)(*comm), newfname, *amode, info_c, &fh_c);
-
-    *fh = MPI_File_c2f(fh_c);
-    ADIOI_Free(newfname);
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/fortran/preallocf.c b/3rd-party/romio321/mpi-io/fortran/preallocf.c
deleted file mode 100644
index 515aa4a9125..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/preallocf.c
+++ /dev/null
@@ -1,102 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_PREALLOCATE( MPI_Fint *, MPI_Offset *, MPI_Fint * );
-#pragma weak MPI_FILE_PREALLOCATE = PMPI_FILE_PREALLOCATE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_preallocate__( MPI_Fint *, MPI_Offset *, MPI_Fint * );
-#pragma weak mpi_file_preallocate__ = pmpi_file_preallocate__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_preallocate( MPI_Fint *, MPI_Offset *, MPI_Fint * );
-#pragma weak mpi_file_preallocate = pmpi_file_preallocate
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_preallocate_( MPI_Fint *, MPI_Offset *, MPI_Fint * );
-#pragma weak mpi_file_preallocate_ = pmpi_file_preallocate_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_PREALLOCATE MPI_FILE_PREALLOCATE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_preallocate__ mpi_file_preallocate__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_preallocate mpi_file_preallocate
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_preallocate_ mpi_file_preallocate_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_PREALLOCATE as PMPI_FILE_PREALLOCATE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_preallocate__ as pmpi_file_preallocate__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_preallocate as pmpi_file_preallocate
-#else
-#pragma _CRI duplicate mpi_file_preallocate_ as pmpi_file_preallocate_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_preallocate_ PMPI_FILE_PREALLOCATE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_preallocate_ pmpi_file_preallocate__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_preallocate pmpi_file_preallocate_
-#endif
-#define mpi_file_preallocate_ pmpi_file_preallocate
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_preallocate_ pmpi_file_preallocate
-#endif
-#define mpi_file_preallocate_ pmpi_file_preallocate_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_preallocate_ MPI_FILE_PREALLOCATE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_preallocate_ mpi_file_preallocate__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_preallocate mpi_file_preallocate_
-#endif
-#define mpi_file_preallocate_ mpi_file_preallocate
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_preallocate_ mpi_file_preallocate
-#endif
-#endif
-#endif
-
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_preallocate_(MPI_Fint *fh,MPI_Offset *size, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_preallocate_(MPI_Fint *fh,MPI_Offset *size, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_preallocate(fh_c,*size);
-}
-
diff --git a/3rd-party/romio321/mpi-io/fortran/rd_atallbf.c b/3rd-party/romio321/mpi-io/fortran/rd_atallbf.c
deleted file mode 100644
index 5fff4e9e7ec..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/rd_atallbf.c
+++ /dev/null
@@ -1,121 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_READ_AT_ALL_BEGIN( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak MPI_FILE_READ_AT_ALL_BEGIN = PMPI_FILE_READ_AT_ALL_BEGIN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_read_at_all_begin__( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_read_at_all_begin__ = pmpi_file_read_at_all_begin__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_read_at_all_begin( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_read_at_all_begin = pmpi_file_read_at_all_begin
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_read_at_all_begin_( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_read_at_all_begin_ = pmpi_file_read_at_all_begin_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_READ_AT_ALL_BEGIN MPI_FILE_READ_AT_ALL_BEGIN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_at_all_begin__ mpi_file_read_at_all_begin__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_at_all_begin mpi_file_read_at_all_begin
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_read_at_all_begin_ mpi_file_read_at_all_begin_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_READ_AT_ALL_BEGIN as PMPI_FILE_READ_AT_ALL_BEGIN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_read_at_all_begin__ as pmpi_file_read_at_all_begin__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_read_at_all_begin as pmpi_file_read_at_all_begin
-#else
-#pragma _CRI duplicate mpi_file_read_at_all_begin_ as pmpi_file_read_at_all_begin_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_read_at_all_begin_ PMPI_FILE_READ_AT_ALL_BEGIN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_read_at_all_begin_ pmpi_file_read_at_all_begin__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_at_all_begin pmpi_file_read_at_all_begin_
-#endif
-#define mpi_file_read_at_all_begin_ pmpi_file_read_at_all_begin
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_at_all_begin_ pmpi_file_read_at_all_begin
-#endif
-#define mpi_file_read_at_all_begin_ pmpi_file_read_at_all_begin_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_read_at_all_begin_ MPI_FILE_READ_AT_ALL_BEGIN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_read_at_all_begin_ mpi_file_read_at_all_begin__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_read_at_all_begin mpi_file_read_at_all_begin_
-#endif
-#define mpi_file_read_at_all_begin_ mpi_file_read_at_all_begin
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_read_at_all_begin_ mpi_file_read_at_all_begin
-#endif
-#endif
-#endif
-
-#if defined(MPIHP) || defined(MPILAM)
-/* Prototype to keep compiler happy */
-void mpi_file_read_at_all_begin_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-			 MPI_Fint *count,MPI_Fint *datatype, MPI_Fint *ierr );
-
-void mpi_file_read_at_all_begin_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-                         MPI_Fint *count,MPI_Fint *datatype, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPI_Datatype datatype_c;
-
-    fh_c = MPI_File_f2c(*fh);
-    datatype_c = MPI_Type_f2c(*datatype);
-
-    *ierr = MPI_File_read_at_all_begin(fh_c,*offset,buf,*count,datatype_c);
-}
-#else
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_read_at_all_begin_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-			 MPI_Fint *count,MPI_Fint *datatype, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_read_at_all_begin_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-                         MPI_Fint *count,MPI_Fint *datatype, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_read_at_all_begin(fh_c,*offset,buf,*count,(MPI_Datatype) *datatype);
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/fortran/rd_atallef.c b/3rd-party/romio321/mpi-io/fortran/rd_atallef.c
deleted file mode 100644
index 8a3441126dd..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/rd_atallef.c
+++ /dev/null
@@ -1,102 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_READ_AT_ALL_END( MPI_Fint *, void*, MPI_Status*, MPI_Fint * );
-#pragma weak MPI_FILE_READ_AT_ALL_END = PMPI_FILE_READ_AT_ALL_END
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_read_at_all_end__( MPI_Fint *, void*, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_read_at_all_end__ = pmpi_file_read_at_all_end__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_read_at_all_end( MPI_Fint *, void*, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_read_at_all_end = pmpi_file_read_at_all_end
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_read_at_all_end_( MPI_Fint *, void*, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_read_at_all_end_ = pmpi_file_read_at_all_end_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_READ_AT_ALL_END MPI_FILE_READ_AT_ALL_END
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_at_all_end__ mpi_file_read_at_all_end__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_at_all_end mpi_file_read_at_all_end
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_read_at_all_end_ mpi_file_read_at_all_end_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_READ_AT_ALL_END as PMPI_FILE_READ_AT_ALL_END
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_read_at_all_end__ as pmpi_file_read_at_all_end__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_read_at_all_end as pmpi_file_read_at_all_end
-#else
-#pragma _CRI duplicate mpi_file_read_at_all_end_ as pmpi_file_read_at_all_end_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_read_at_all_end_ PMPI_FILE_READ_AT_ALL_END
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_read_at_all_end_ pmpi_file_read_at_all_end__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_at_all_end pmpi_file_read_at_all_end_
-#endif
-#define mpi_file_read_at_all_end_ pmpi_file_read_at_all_end
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_at_all_end_ pmpi_file_read_at_all_end
-#endif
-#define mpi_file_read_at_all_end_ pmpi_file_read_at_all_end_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_read_at_all_end_ MPI_FILE_READ_AT_ALL_END
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_read_at_all_end_ mpi_file_read_at_all_end__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_read_at_all_end mpi_file_read_at_all_end_
-#endif
-#define mpi_file_read_at_all_end_ mpi_file_read_at_all_end
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_read_at_all_end_ mpi_file_read_at_all_end
-#endif
-#endif
-#endif
-
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_read_at_all_end_(MPI_Fint *fh,void *buf,MPI_Status *status, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_read_at_all_end_(MPI_Fint *fh,void *buf,MPI_Status *status, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_read_at_all_end(fh_c,buf,status);
-}
-
diff --git a/3rd-party/romio321/mpi-io/fortran/read_allbf.c b/3rd-party/romio321/mpi-io/fortran/read_allbf.c
deleted file mode 100644
index 5708a03c8dc..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/read_allbf.c
+++ /dev/null
@@ -1,120 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_READ_ALL_BEGIN( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak MPI_FILE_READ_ALL_BEGIN = PMPI_FILE_READ_ALL_BEGIN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_read_all_begin__( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_read_all_begin__ = pmpi_file_read_all_begin__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_read_all_begin( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_read_all_begin = pmpi_file_read_all_begin
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_read_all_begin_( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_read_all_begin_ = pmpi_file_read_all_begin_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_READ_ALL_BEGIN MPI_FILE_READ_ALL_BEGIN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_all_begin__ mpi_file_read_all_begin__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_all_begin mpi_file_read_all_begin
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_read_all_begin_ mpi_file_read_all_begin_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_READ_ALL_BEGIN as PMPI_FILE_READ_ALL_BEGIN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_read_all_begin__ as pmpi_file_read_all_begin__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_read_all_begin as pmpi_file_read_all_begin
-#else
-#pragma _CRI duplicate mpi_file_read_all_begin_ as pmpi_file_read_all_begin_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_read_all_begin_ PMPI_FILE_READ_ALL_BEGIN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_read_all_begin_ pmpi_file_read_all_begin__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_all_begin pmpi_file_read_all_begin_
-#endif
-#define mpi_file_read_all_begin_ pmpi_file_read_all_begin
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_all_begin_ pmpi_file_read_all_begin
-#endif
-#define mpi_file_read_all_begin_ pmpi_file_read_all_begin_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_read_all_begin_ MPI_FILE_READ_ALL_BEGIN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_read_all_begin_ mpi_file_read_all_begin__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_read_all_begin mpi_file_read_all_begin_
-#endif
-#define mpi_file_read_all_begin_ mpi_file_read_all_begin
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_read_all_begin_ mpi_file_read_all_begin
-#endif
-#endif
-#endif
-
-#if defined(MPIHP) || defined(MPILAM)
-/* Prototype to keep compiler happy */
-void mpi_file_read_all_begin_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-			      MPI_Fint *datatype, MPI_Fint *ierr );
-
-void mpi_file_read_all_begin_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                      MPI_Fint *datatype,MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPI_Datatype datatype_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    datatype_c = MPI_Type_f2c(*datatype);
-
-    *ierr = MPI_File_read_all_begin(fh_c,buf,*count,datatype_c);
-}
-#else
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_read_all_begin_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-			      MPI_Fint *datatype, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_read_all_begin_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                      MPI_Fint *datatype, MPI_Fint *ierr ){
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_read_all_begin(fh_c,buf,*count,(MPI_Datatype) *datatype);
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/fortran/read_allef.c b/3rd-party/romio321/mpi-io/fortran/read_allef.c
deleted file mode 100644
index 776b182d640..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/read_allef.c
+++ /dev/null
@@ -1,103 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_READ_ALL_END( MPI_Fint *, void*, MPI_Status*, MPI_Fint * );
-#pragma weak MPI_FILE_READ_ALL_END = PMPI_FILE_READ_ALL_END
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_read_all_end__( MPI_Fint *, void*, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_read_all_end__ = pmpi_file_read_all_end__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_read_all_end( MPI_Fint *, void*, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_read_all_end = pmpi_file_read_all_end
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_read_all_end_( MPI_Fint *, void*, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_read_all_end_ = pmpi_file_read_all_end_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_READ_ALL_END MPI_FILE_READ_ALL_END
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_all_end__ mpi_file_read_all_end__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_all_end mpi_file_read_all_end
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_read_all_end_ mpi_file_read_all_end_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_READ_ALL_END as PMPI_FILE_READ_ALL_END
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_read_all_end__ as pmpi_file_read_all_end__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_read_all_end as pmpi_file_read_all_end
-#else
-#pragma _CRI duplicate mpi_file_read_all_end_ as pmpi_file_read_all_end_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_read_all_end_ PMPI_FILE_READ_ALL_END
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_read_all_end_ pmpi_file_read_all_end__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_all_end pmpi_file_read_all_end_
-#endif
-#define mpi_file_read_all_end_ pmpi_file_read_all_end
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_all_end_ pmpi_file_read_all_end
-#endif
-#define mpi_file_read_all_end_ pmpi_file_read_all_end_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_read_all_end_ MPI_FILE_READ_ALL_END
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_read_all_end_ mpi_file_read_all_end__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_read_all_end mpi_file_read_all_end_
-#endif
-#define mpi_file_read_all_end_ mpi_file_read_all_end
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_read_all_end_ mpi_file_read_all_end
-#endif
-#endif
-#endif
-
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_read_all_end_(MPI_Fint *fh,void *buf,MPI_Status *status, 
-			    MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_read_all_end_(MPI_Fint *fh,void *buf,MPI_Status *status, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-
-    *ierr = MPI_File_read_all_end(fh_c,buf,status);
-}
diff --git a/3rd-party/romio321/mpi-io/fortran/read_allf.c b/3rd-party/romio321/mpi-io/fortran/read_allf.c
deleted file mode 100644
index 99e1229b7f1..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/read_allf.c
+++ /dev/null
@@ -1,120 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_READ_ALL( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak MPI_FILE_READ_ALL = PMPI_FILE_READ_ALL
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_read_all__( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_read_all__ = pmpi_file_read_all__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_read_all( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_read_all = pmpi_file_read_all
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_read_all_( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_read_all_ = pmpi_file_read_all_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_READ_ALL MPI_FILE_READ_ALL
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_all__ mpi_file_read_all__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_all mpi_file_read_all
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_read_all_ mpi_file_read_all_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_READ_ALL as PMPI_FILE_READ_ALL
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_read_all__ as pmpi_file_read_all__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_read_all as pmpi_file_read_all
-#else
-#pragma _CRI duplicate mpi_file_read_all_ as pmpi_file_read_all_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_read_all_ PMPI_FILE_READ_ALL
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_read_all_ pmpi_file_read_all__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_all pmpi_file_read_all_
-#endif
-#define mpi_file_read_all_ pmpi_file_read_all
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_all_ pmpi_file_read_all
-#endif
-#define mpi_file_read_all_ pmpi_file_read_all_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_read_all_ MPI_FILE_READ_ALL
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_read_all_ mpi_file_read_all__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_read_all mpi_file_read_all_
-#endif
-#define mpi_file_read_all_ mpi_file_read_all
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_read_all_ mpi_file_read_all
-#endif
-#endif
-#endif
-
-#if defined(MPIHP) || defined(MPILAM)
-/* Prototype to keep compiler happy */
-void mpi_file_read_all_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-			MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr );
-
-void mpi_file_read_all_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                      MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPI_Datatype datatype_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    datatype_c = MPI_Type_f2c(*datatype);
-
-    *ierr = MPI_File_read_all(fh_c,buf,*count,datatype_c,status);
-}
-#else
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_read_all_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-			MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_read_all_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                      MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr ){
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_read_all(fh_c,buf,*count,(MPI_Datatype)*datatype,status);
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/fortran/read_atallf.c b/3rd-party/romio321/mpi-io/fortran/read_atallf.c
deleted file mode 100644
index b9f30a00562..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/read_atallf.c
+++ /dev/null
@@ -1,125 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_READ_AT_ALL( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak MPI_FILE_READ_AT_ALL = PMPI_FILE_READ_AT_ALL
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_read_at_all__( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_read_at_all__ = pmpi_file_read_at_all__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_read_at_all( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_read_at_all = pmpi_file_read_at_all
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_read_at_all_( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_read_at_all_ = pmpi_file_read_at_all_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_READ_AT_ALL MPI_FILE_READ_AT_ALL
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_at_all__ mpi_file_read_at_all__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_at_all mpi_file_read_at_all
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_read_at_all_ mpi_file_read_at_all_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_READ_AT_ALL as PMPI_FILE_READ_AT_ALL
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_read_at_all__ as pmpi_file_read_at_all__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_read_at_all as pmpi_file_read_at_all
-#else
-#pragma _CRI duplicate mpi_file_read_at_all_ as pmpi_file_read_at_all_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_read_at_all_ PMPI_FILE_READ_AT_ALL
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_read_at_all_ pmpi_file_read_at_all__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_at_all pmpi_file_read_at_all_
-#endif
-#define mpi_file_read_at_all_ pmpi_file_read_at_all
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_at_all_ pmpi_file_read_at_all
-#endif
-#define mpi_file_read_at_all_ pmpi_file_read_at_all_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_read_at_all_ MPI_FILE_READ_AT_ALL
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_read_at_all_ mpi_file_read_at_all__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_read_at_all mpi_file_read_at_all_
-#endif
-#define mpi_file_read_at_all_ mpi_file_read_at_all
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_read_at_all_ mpi_file_read_at_all
-#endif
-#endif
-#endif
-
-#if defined(MPIHP) || defined(MPILAM)
-/* Prototype to keep compiler happy */
-void mpi_file_read_at_all_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-                           MPI_Fint *count,MPI_Fint *datatype,
-			   MPI_Status *status, MPI_Fint *ierr );
-
-void mpi_file_read_at_all_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-                         MPI_Fint *count,MPI_Fint *datatype,
-                         MPI_Status *status, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPI_Datatype datatype_c;
-
-    fh_c = MPI_File_f2c(*fh);
-    datatype_c = MPI_Type_f2c(*datatype);
-
-    *ierr = MPI_File_read_at_all(fh_c,*offset,buf,*count,datatype_c,status);
-}
-#else
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_read_at_all_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-                         MPI_Fint *count,MPI_Fint *datatype,
-			   MPI_Status *status, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_read_at_all_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-                         MPI_Fint *count,MPI_Fint *datatype,
-                         MPI_Status *status, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_read_at_all(fh_c,*offset,buf,*count,(MPI_Datatype)*datatype,status);
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/fortran/read_atf.c b/3rd-party/romio321/mpi-io/fortran/read_atf.c
deleted file mode 100644
index 2602e399bf6..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/read_atf.c
+++ /dev/null
@@ -1,121 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_READ_AT( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak MPI_FILE_READ_AT = PMPI_FILE_READ_AT
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_read_at__( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_read_at__ = pmpi_file_read_at__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_read_at( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_read_at = pmpi_file_read_at
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_read_at_( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_read_at_ = pmpi_file_read_at_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_READ_AT MPI_FILE_READ_AT
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_at__ mpi_file_read_at__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_at mpi_file_read_at
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_read_at_ mpi_file_read_at_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_READ_AT as PMPI_FILE_READ_AT
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_read_at__ as pmpi_file_read_at__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_read_at as pmpi_file_read_at
-#else
-#pragma _CRI duplicate mpi_file_read_at_ as pmpi_file_read_at_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_read_at_ PMPI_FILE_READ_AT
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_read_at_ pmpi_file_read_at__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_at pmpi_file_read_at_
-#endif
-#define mpi_file_read_at_ pmpi_file_read_at
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_at_ pmpi_file_read_at
-#endif
-#define mpi_file_read_at_ pmpi_file_read_at_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_read_at_ MPI_FILE_READ_AT
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_read_at_ mpi_file_read_at__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_read_at mpi_file_read_at_
-#endif
-#define mpi_file_read_at_ mpi_file_read_at
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_read_at_ mpi_file_read_at
-#endif
-#endif
-#endif
-
-#if defined(MPIHP) || defined(MPILAM)
-/* Prototype to keep compiler happy */
-void mpi_file_read_at_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-       MPI_Fint *count,MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr );
-
-void mpi_file_read_at_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-      MPI_Fint *count,MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPI_Datatype datatype_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    datatype_c = MPI_Type_f2c(*datatype);
-
-    *ierr = MPI_File_read_at(fh_c,*offset,buf,*count,datatype_c,status);
-}
-#else
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_read_at_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-       MPI_Fint *count,MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_read_at_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-      MPI_Fint *count,MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_read_at(fh_c,*offset,buf,*count,(MPI_Datatype)*datatype,status);
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/fortran/read_ordbf.c b/3rd-party/romio321/mpi-io/fortran/read_ordbf.c
deleted file mode 100644
index 74389174f4c..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/read_ordbf.c
+++ /dev/null
@@ -1,120 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_READ_ORDERED_BEGIN( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak MPI_FILE_READ_ORDERED_BEGIN = PMPI_FILE_READ_ORDERED_BEGIN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_read_ordered_begin__( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_read_ordered_begin__ = pmpi_file_read_ordered_begin__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_read_ordered_begin( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_read_ordered_begin = pmpi_file_read_ordered_begin
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_read_ordered_begin_( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_read_ordered_begin_ = pmpi_file_read_ordered_begin_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_READ_ORDERED_BEGIN MPI_FILE_READ_ORDERED_BEGIN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_ordered_begin__ mpi_file_read_ordered_begin__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_ordered_begin mpi_file_read_ordered_begin
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_read_ordered_begin_ mpi_file_read_ordered_begin_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_READ_ORDERED_BEGIN as PMPI_FILE_READ_ORDERED_BEGIN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_read_ordered_begin__ as pmpi_file_read_ordered_begin__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_read_ordered_begin as pmpi_file_read_ordered_begin
-#else
-#pragma _CRI duplicate mpi_file_read_ordered_begin_ as pmpi_file_read_ordered_begin_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_read_ordered_begin_ PMPI_FILE_READ_ORDERED_BEGIN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_read_ordered_begin_ pmpi_file_read_ordered_begin__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_ordered_begin pmpi_file_read_ordered_begin_
-#endif
-#define mpi_file_read_ordered_begin_ pmpi_file_read_ordered_begin
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_ordered_begin_ pmpi_file_read_ordered_begin
-#endif
-#define mpi_file_read_ordered_begin_ pmpi_file_read_ordered_begin_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_read_ordered_begin_ MPI_FILE_READ_ORDERED_BEGIN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_read_ordered_begin_ mpi_file_read_ordered_begin__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_read_ordered_begin mpi_file_read_ordered_begin_
-#endif
-#define mpi_file_read_ordered_begin_ mpi_file_read_ordered_begin
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_read_ordered_begin_ mpi_file_read_ordered_begin
-#endif
-#endif
-#endif
-
-#if defined(MPIHP) || defined(MPILAM)
-/* Prototype to keep compiler happy */
-void mpi_file_read_ordered_begin_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-				  MPI_Fint *datatype,MPI_Fint *ierr );
-
-void mpi_file_read_ordered_begin_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                      MPI_Fint *datatype,MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPI_Datatype datatype_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    datatype_c = MPI_Type_f2c(*datatype);
-
-    *ierr = MPI_File_read_ordered_begin(fh_c,buf,*count,datatype_c);
-}
-#else
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_read_ordered_begin_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-				  MPI_Fint *datatype,MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_read_ordered_begin_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                      MPI_Fint *datatype,MPI_Fint *ierr ){
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_read_ordered_begin(fh_c,buf,*count,(MPI_Datatype)*datatype);
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/fortran/read_ordef.c b/3rd-party/romio321/mpi-io/fortran/read_ordef.c
deleted file mode 100644
index 63425c8ad33..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/read_ordef.c
+++ /dev/null
@@ -1,103 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_READ_ORDERED_END( MPI_Fint *, void*, MPI_Status*, MPI_Fint * );
-#pragma weak MPI_FILE_READ_ORDERED_END = PMPI_FILE_READ_ORDERED_END
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_read_ordered_end__( MPI_Fint *, void*, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_read_ordered_end__ = pmpi_file_read_ordered_end__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_read_ordered_end( MPI_Fint *, void*, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_read_ordered_end = pmpi_file_read_ordered_end
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_read_ordered_end_( MPI_Fint *, void*, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_read_ordered_end_ = pmpi_file_read_ordered_end_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_READ_ORDERED_END MPI_FILE_READ_ORDERED_END
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_ordered_end__ mpi_file_read_ordered_end__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_ordered_end mpi_file_read_ordered_end
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_read_ordered_end_ mpi_file_read_ordered_end_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_READ_ORDERED_END as PMPI_FILE_READ_ORDERED_END
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_read_ordered_end__ as pmpi_file_read_ordered_end__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_read_ordered_end as pmpi_file_read_ordered_end
-#else
-#pragma _CRI duplicate mpi_file_read_ordered_end_ as pmpi_file_read_ordered_end_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_read_ordered_end_ PMPI_FILE_READ_ORDERED_END
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_read_ordered_end_ pmpi_file_read_ordered_end__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_ordered_end pmpi_file_read_ordered_end_
-#endif
-#define mpi_file_read_ordered_end_ pmpi_file_read_ordered_end
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_ordered_end_ pmpi_file_read_ordered_end
-#endif
-#define mpi_file_read_ordered_end_ pmpi_file_read_ordered_end_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_read_ordered_end_ MPI_FILE_READ_ORDERED_END
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_read_ordered_end_ mpi_file_read_ordered_end__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_read_ordered_end mpi_file_read_ordered_end_
-#endif
-#define mpi_file_read_ordered_end_ mpi_file_read_ordered_end
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_read_ordered_end_ mpi_file_read_ordered_end
-#endif
-#endif
-#endif
-
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_read_ordered_end_(MPI_Fint *fh,void *buf,MPI_Status *status, 
-				MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_read_ordered_end_(MPI_Fint *fh,void *buf,MPI_Status *status, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-
-    *ierr = MPI_File_read_ordered_end(fh_c,buf,status);
-}
diff --git a/3rd-party/romio321/mpi-io/fortran/read_ordf.c b/3rd-party/romio321/mpi-io/fortran/read_ordf.c
deleted file mode 100644
index a45ae1282ff..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/read_ordf.c
+++ /dev/null
@@ -1,120 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_READ_ORDERED( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak MPI_FILE_READ_ORDERED = PMPI_FILE_READ_ORDERED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_read_ordered__( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_read_ordered__ = pmpi_file_read_ordered__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_read_ordered( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_read_ordered = pmpi_file_read_ordered
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_read_ordered_( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_read_ordered_ = pmpi_file_read_ordered_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_READ_ORDERED MPI_FILE_READ_ORDERED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_ordered__ mpi_file_read_ordered__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_ordered mpi_file_read_ordered
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_read_ordered_ mpi_file_read_ordered_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_READ_ORDERED as PMPI_FILE_READ_ORDERED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_read_ordered__ as pmpi_file_read_ordered__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_read_ordered as pmpi_file_read_ordered
-#else
-#pragma _CRI duplicate mpi_file_read_ordered_ as pmpi_file_read_ordered_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_read_ordered_ PMPI_FILE_READ_ORDERED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_read_ordered_ pmpi_file_read_ordered__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_ordered pmpi_file_read_ordered_
-#endif
-#define mpi_file_read_ordered_ pmpi_file_read_ordered
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_ordered_ pmpi_file_read_ordered
-#endif
-#define mpi_file_read_ordered_ pmpi_file_read_ordered_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_read_ordered_ MPI_FILE_READ_ORDERED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_read_ordered_ mpi_file_read_ordered__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_read_ordered mpi_file_read_ordered_
-#endif
-#define mpi_file_read_ordered_ mpi_file_read_ordered
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_read_ordered_ mpi_file_read_ordered
-#endif
-#endif
-#endif
-
-#if defined(MPIHP) || defined(MPILAM)
-/* Prototype to keep compiler happy */
-void mpi_file_read_ordered_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-		    MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr );
-
-void mpi_file_read_ordered_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                      MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPI_Datatype datatype_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    datatype_c = MPI_Type_f2c(*datatype);
-
-    *ierr = MPI_File_read_ordered(fh_c,buf,*count,datatype_c,status);
-}
-#else
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_read_ordered_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-		    MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_read_ordered_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                      MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr ){
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_read_ordered(fh_c,buf,*count,(MPI_Datatype)*datatype,status);
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/fortran/read_shf.c b/3rd-party/romio321/mpi-io/fortran/read_shf.c
deleted file mode 100644
index 937e54fb95e..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/read_shf.c
+++ /dev/null
@@ -1,119 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_READ_SHARED( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak MPI_FILE_READ_SHARED = PMPI_FILE_READ_SHARED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_read_shared__( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_read_shared__ = pmpi_file_read_shared__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_read_shared( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_read_shared = pmpi_file_read_shared
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_read_shared_( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_read_shared_ = pmpi_file_read_shared_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_READ_SHARED MPI_FILE_READ_SHARED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_shared__ mpi_file_read_shared__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_shared mpi_file_read_shared
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_read_shared_ mpi_file_read_shared_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_READ_SHARED as PMPI_FILE_READ_SHARED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_read_shared__ as pmpi_file_read_shared__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_read_shared as pmpi_file_read_shared
-#else
-#pragma _CRI duplicate mpi_file_read_shared_ as pmpi_file_read_shared_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_read_shared_ PMPI_FILE_READ_SHARED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_read_shared_ pmpi_file_read_shared__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_shared pmpi_file_read_shared_
-#endif
-#define mpi_file_read_shared_ pmpi_file_read_shared
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_shared_ pmpi_file_read_shared
-#endif
-#define mpi_file_read_shared_ pmpi_file_read_shared_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_read_shared_ MPI_FILE_READ_SHARED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_read_shared_ mpi_file_read_shared__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_read_shared mpi_file_read_shared_
-#endif
-#define mpi_file_read_shared_ mpi_file_read_shared
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_read_shared_ mpi_file_read_shared
-#endif
-#endif
-#endif
-
-#if defined(MPIHP) || defined(MPILAM)
-/* Prototype to keep compiler happy */
-void mpi_file_read_shared_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-			   MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr );
-void mpi_file_read_shared_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                  MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPI_Datatype datatype_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    datatype_c = MPI_Type_f2c(*datatype);
-
-    *ierr = MPI_File_read_shared(fh_c,buf,*count,datatype_c,status);
-}
-#else
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_read_shared_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-			   MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr ); 
-FORTRAN_API void FORT_CALL mpi_file_read_shared_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                  MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_read_shared(fh_c,buf,*count,(MPI_Datatype)*datatype,status);
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/fortran/readf.c b/3rd-party/romio321/mpi-io/fortran/readf.c
deleted file mode 100644
index c802739139e..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/readf.c
+++ /dev/null
@@ -1,121 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_READ( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak MPI_FILE_READ = PMPI_FILE_READ
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_read__( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_read__ = pmpi_file_read__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_read( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_read = pmpi_file_read
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_read_( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_read_ = pmpi_file_read_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_READ MPI_FILE_READ
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_read__ mpi_file_read__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_read mpi_file_read
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_read_ mpi_file_read_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_READ as PMPI_FILE_READ
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_read__ as pmpi_file_read__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_read as pmpi_file_read
-#else
-#pragma _CRI duplicate mpi_file_read_ as pmpi_file_read_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_read_ PMPI_FILE_READ
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_read_ pmpi_file_read__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_read pmpi_file_read_
-#endif
-#define mpi_file_read_ pmpi_file_read
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_read_ pmpi_file_read
-#endif
-#define mpi_file_read_ pmpi_file_read_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_read_ MPI_FILE_READ
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_read_ mpi_file_read__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_read mpi_file_read_
-#endif
-#define mpi_file_read_ mpi_file_read
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_read_ mpi_file_read
-#endif
-#endif
-#endif
-
-#if defined(MPIHP) || defined(MPILAM)
-/* Prototype to keep compiler happy */
-void mpi_file_read_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-		    MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr );
-
-void mpi_file_read_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                  MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPI_Datatype datatype_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    datatype_c = MPI_Type_f2c(*datatype);
-
-    *ierr = MPI_File_read(fh_c,buf,*count,datatype_c,status);
-}
-#else
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_read_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-		    MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_read_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                  MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_read(fh_c,buf,*count,(MPI_Datatype)*datatype,status);
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/fortran/seek_shf.c b/3rd-party/romio321/mpi-io/fortran/seek_shf.c
deleted file mode 100644
index 158e626c0e7..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/seek_shf.c
+++ /dev/null
@@ -1,102 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_SEEK_SHARED( MPI_Fint *, MPI_Offset *, MPI_Fint *, MPI_Fint * );
-#pragma weak MPI_FILE_SEEK_SHARED = PMPI_FILE_SEEK_SHARED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_seek_shared__( MPI_Fint *, MPI_Offset *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_seek_shared__ = pmpi_file_seek_shared__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_seek_shared( MPI_Fint *, MPI_Offset *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_seek_shared = pmpi_file_seek_shared
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_seek_shared_( MPI_Fint *, MPI_Offset *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_seek_shared_ = pmpi_file_seek_shared_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_SEEK_SHARED MPI_FILE_SEEK_SHARED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_seek_shared__ mpi_file_seek_shared__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_seek_shared mpi_file_seek_shared
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_seek_shared_ mpi_file_seek_shared_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_SEEK_SHARED as PMPI_FILE_SEEK_SHARED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_seek_shared__ as pmpi_file_seek_shared__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_seek_shared as pmpi_file_seek_shared
-#else
-#pragma _CRI duplicate mpi_file_seek_shared_ as pmpi_file_seek_shared_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_seek_shared_ PMPI_FILE_SEEK_SHARED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_seek_shared_ pmpi_file_seek_shared__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_seek_shared pmpi_file_seek_shared_
-#endif
-#define mpi_file_seek_shared_ pmpi_file_seek_shared
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_seek_shared_ pmpi_file_seek_shared
-#endif
-#define mpi_file_seek_shared_ pmpi_file_seek_shared_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_seek_shared_ MPI_FILE_SEEK_SHARED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_seek_shared_ mpi_file_seek_shared__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_seek_shared mpi_file_seek_shared_
-#endif
-#define mpi_file_seek_shared_ mpi_file_seek_shared
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_seek_shared_ mpi_file_seek_shared
-#endif
-#endif
-#endif
-
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_seek_shared_(MPI_Fint *fh,MPI_Offset *offset,MPI_Fint *whence, 
-			   MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_seek_shared_(MPI_Fint *fh,MPI_Offset *offset,MPI_Fint *whence, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_seek_shared(fh_c,*offset,*whence);
-}
diff --git a/3rd-party/romio321/mpi-io/fortran/seekf.c b/3rd-party/romio321/mpi-io/fortran/seekf.c
deleted file mode 100644
index 8e125d1849c..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/seekf.c
+++ /dev/null
@@ -1,101 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_SEEK( MPI_Fint *, MPI_Offset *, MPI_Fint *, MPI_Fint * );
-#pragma weak MPI_FILE_SEEK = PMPI_FILE_SEEK
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_seek__( MPI_Fint *, MPI_Offset *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_seek__ = pmpi_file_seek__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_seek( MPI_Fint *, MPI_Offset *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_seek = pmpi_file_seek
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_seek_( MPI_Fint *, MPI_Offset *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_seek_ = pmpi_file_seek_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_SEEK MPI_FILE_SEEK
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_seek__ mpi_file_seek__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_seek mpi_file_seek
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_seek_ mpi_file_seek_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_SEEK as PMPI_FILE_SEEK
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_seek__ as pmpi_file_seek__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_seek as pmpi_file_seek
-#else
-#pragma _CRI duplicate mpi_file_seek_ as pmpi_file_seek_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_seek_ PMPI_FILE_SEEK
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_seek_ pmpi_file_seek__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_seek pmpi_file_seek_
-#endif
-#define mpi_file_seek_ pmpi_file_seek
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_seek_ pmpi_file_seek
-#endif
-#define mpi_file_seek_ pmpi_file_seek_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_seek_ MPI_FILE_SEEK
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_seek_ mpi_file_seek__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_seek mpi_file_seek_
-#endif
-#define mpi_file_seek_ mpi_file_seek
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_seek_ mpi_file_seek
-#endif
-#endif
-#endif
-
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_seek_(MPI_Fint *fh,MPI_Offset *offset,MPI_Fint *whence, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_seek_(MPI_Fint *fh,MPI_Offset *offset,MPI_Fint *whence, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_seek(fh_c,*offset,*whence);
-}
diff --git a/3rd-party/romio321/mpi-io/fortran/set_atomf.c b/3rd-party/romio321/mpi-io/fortran/set_atomf.c
deleted file mode 100644
index c4388950da5..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/set_atomf.c
+++ /dev/null
@@ -1,102 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_SET_ATOMICITY( MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak MPI_FILE_SET_ATOMICITY = PMPI_FILE_SET_ATOMICITY
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_set_atomicity__( MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_set_atomicity__ = pmpi_file_set_atomicity__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_set_atomicity( MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_set_atomicity = pmpi_file_set_atomicity
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_set_atomicity_( MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_set_atomicity_ = pmpi_file_set_atomicity_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_SET_ATOMICITY MPI_FILE_SET_ATOMICITY
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_set_atomicity__ mpi_file_set_atomicity__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_set_atomicity mpi_file_set_atomicity
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_set_atomicity_ mpi_file_set_atomicity_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_SET_ATOMICITY as PMPI_FILE_SET_ATOMICITY
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_set_atomicity__ as pmpi_file_set_atomicity__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_set_atomicity as pmpi_file_set_atomicity
-#else
-#pragma _CRI duplicate mpi_file_set_atomicity_ as pmpi_file_set_atomicity_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_set_atomicity_ PMPI_FILE_SET_ATOMICITY
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_set_atomicity_ pmpi_file_set_atomicity__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_set_atomicity pmpi_file_set_atomicity_
-#endif
-#define mpi_file_set_atomicity_ pmpi_file_set_atomicity
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_set_atomicity_ pmpi_file_set_atomicity
-#endif
-#define mpi_file_set_atomicity_ pmpi_file_set_atomicity_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_set_atomicity_ MPI_FILE_SET_ATOMICITY
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_set_atomicity_ mpi_file_set_atomicity__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_set_atomicity mpi_file_set_atomicity_
-#endif
-#define mpi_file_set_atomicity_ mpi_file_set_atomicity
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_set_atomicity_ mpi_file_set_atomicity
-#endif
-#endif
-#endif
-
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_set_atomicity_(MPI_Fint *fh,MPI_Fint *flag, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_set_atomicity_(MPI_Fint *fh,MPI_Fint *flag, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_set_atomicity(fh_c,*flag);
-}
-
diff --git a/3rd-party/romio321/mpi-io/fortran/set_errhf.c b/3rd-party/romio321/mpi-io/fortran/set_errhf.c
deleted file mode 100644
index c622660898c..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/set_errhf.c
+++ /dev/null
@@ -1,105 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_SET_ERRHANDLER( MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak MPI_FILE_SET_ERRHANDLER = PMPI_FILE_SET_ERRHANDLER
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_set_errhandler__( MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_set_errhandler__ = pmpi_file_set_errhandler__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_set_errhandler( MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_set_errhandler = pmpi_file_set_errhandler
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_set_errhandler_( MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_set_errhandler_ = pmpi_file_set_errhandler_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_SET_ERRHANDLER MPI_FILE_SET_ERRHANDLER
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_set_errhandler__ mpi_file_set_errhandler__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_set_errhandler mpi_file_set_errhandler
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_set_errhandler_ mpi_file_set_errhandler_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_SET_ERRHANDLER as PMPI_FILE_SET_ERRHANDLER
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_set_errhandler__ as pmpi_file_set_errhandler__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_set_errhandler as pmpi_file_set_errhandler
-#else
-#pragma _CRI duplicate mpi_file_set_errhandler_ as pmpi_file_set_errhandler_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_set_errhandler_ PMPI_FILE_SET_ERRHANDLER
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_set_errhandler_ pmpi_file_set_errhandler__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_set_errhandler pmpi_file_set_errhandler_
-#endif
-#define mpi_file_set_errhandler_ pmpi_file_set_errhandler
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_set_errhandler_ pmpi_file_set_errhandler
-#endif
-#define mpi_file_set_errhandler_ pmpi_file_set_errhandler_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_set_errhandler_ MPI_FILE_SET_ERRHANDLER
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_set_errhandler_ mpi_file_set_errhandler__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_set_errhandler mpi_file_set_errhandler_
-#endif
-#define mpi_file_set_errhandler_ mpi_file_set_errhandler
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_set_errhandler_ mpi_file_set_errhandler
-#endif
-#endif
-#endif
-
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_set_errhandler_(MPI_Fint *fh, MPI_Fint *err_handler, MPI_Fint *ierr);
-
-FORTRAN_API void FORT_CALL mpi_file_set_errhandler_(MPI_Fint *fh, MPI_Fint *err_handler, MPI_Fint *ierr)
-{
-    MPI_File fh_c;
-    MPI_Errhandler err_handler_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    err_handler_c = MPI_Errhandler_f2c(*err_handler);
-
-    *ierr = MPI_File_set_errhandler(fh_c,err_handler_c);
-}
-
diff --git a/3rd-party/romio321/mpi-io/fortran/set_infof.c b/3rd-party/romio321/mpi-io/fortran/set_infof.c
deleted file mode 100644
index 85c05e8f1da..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/set_infof.c
+++ /dev/null
@@ -1,104 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_SET_INFO( MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak MPI_FILE_SET_INFO = PMPI_FILE_SET_INFO
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_set_info__( MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_set_info__ = pmpi_file_set_info__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_set_info( MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_set_info = pmpi_file_set_info
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_set_info_( MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_set_info_ = pmpi_file_set_info_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_SET_INFO MPI_FILE_SET_INFO
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_set_info__ mpi_file_set_info__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_set_info mpi_file_set_info
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_set_info_ mpi_file_set_info_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_SET_INFO as PMPI_FILE_SET_INFO
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_set_info__ as pmpi_file_set_info__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_set_info as pmpi_file_set_info
-#else
-#pragma _CRI duplicate mpi_file_set_info_ as pmpi_file_set_info_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_set_info_ PMPI_FILE_SET_INFO
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_set_info_ pmpi_file_set_info__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_set_info pmpi_file_set_info_
-#endif
-#define mpi_file_set_info_ pmpi_file_set_info
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_set_info_ pmpi_file_set_info
-#endif
-#define mpi_file_set_info_ pmpi_file_set_info_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_set_info_ MPI_FILE_SET_INFO
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_set_info_ mpi_file_set_info__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_set_info mpi_file_set_info_
-#endif
-#define mpi_file_set_info_ mpi_file_set_info
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_set_info_ mpi_file_set_info
-#endif
-#endif
-#endif
-
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_set_info_(MPI_Fint *fh, MPI_Fint *info, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_set_info_(MPI_Fint *fh, MPI_Fint *info, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPI_Info info_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    info_c = MPI_Info_f2c(*info);
-
-    *ierr = MPI_File_set_info(fh_c, info_c);
-}
diff --git a/3rd-party/romio321/mpi-io/fortran/set_sizef.c b/3rd-party/romio321/mpi-io/fortran/set_sizef.c
deleted file mode 100644
index c595f2b8550..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/set_sizef.c
+++ /dev/null
@@ -1,102 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_SET_SIZE( MPI_Fint *, MPI_Offset *, MPI_Fint * );
-#pragma weak MPI_FILE_SET_SIZE = PMPI_FILE_SET_SIZE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_set_size__( MPI_Fint *, MPI_Offset *, MPI_Fint * );
-#pragma weak mpi_file_set_size__ = pmpi_file_set_size__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_set_size( MPI_Fint *, MPI_Offset *, MPI_Fint * );
-#pragma weak mpi_file_set_size = pmpi_file_set_size
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_set_size_( MPI_Fint *, MPI_Offset *, MPI_Fint * );
-#pragma weak mpi_file_set_size_ = pmpi_file_set_size_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_SET_SIZE MPI_FILE_SET_SIZE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_set_size__ mpi_file_set_size__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_set_size mpi_file_set_size
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_set_size_ mpi_file_set_size_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_SET_SIZE as PMPI_FILE_SET_SIZE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_set_size__ as pmpi_file_set_size__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_set_size as pmpi_file_set_size
-#else
-#pragma _CRI duplicate mpi_file_set_size_ as pmpi_file_set_size_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_set_size_ PMPI_FILE_SET_SIZE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_set_size_ pmpi_file_set_size__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_set_size pmpi_file_set_size_
-#endif
-#define mpi_file_set_size_ pmpi_file_set_size
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_set_size_ pmpi_file_set_size
-#endif
-#define mpi_file_set_size_ pmpi_file_set_size_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_set_size_ MPI_FILE_SET_SIZE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_set_size_ mpi_file_set_size__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_set_size mpi_file_set_size_
-#endif
-#define mpi_file_set_size_ mpi_file_set_size
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_set_size_ mpi_file_set_size
-#endif
-#endif
-#endif
-
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_set_size_(MPI_Fint *fh,MPI_Offset *size, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_set_size_(MPI_Fint *fh,MPI_Offset *size, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_set_size(fh_c,*size);
-}
-
diff --git a/3rd-party/romio321/mpi-io/fortran/set_viewf.c b/3rd-party/romio321/mpi-io/fortran/set_viewf.c
deleted file mode 100644
index 97151d978fb..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/set_viewf.c
+++ /dev/null
@@ -1,182 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#ifdef _UNICOS
-#include <fortran.h>
-#endif
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_SET_VIEW( MPI_Fint *, MPI_Offset *, MPI_Fint *, MPI_Fint *, char * FORT_MIXED_LEN_DECL, MPI_Fint *, MPI_Fint * FORT_END_LEN_DECL );
-#pragma weak MPI_FILE_SET_VIEW = PMPI_FILE_SET_VIEW
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_set_view__( MPI_Fint *, MPI_Offset *, MPI_Fint *, MPI_Fint *, char * FORT_MIXED_LEN_DECL, MPI_Fint *, MPI_Fint * FORT_END_LEN_DECL );
-#pragma weak mpi_file_set_view__ = pmpi_file_set_view__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_set_view( MPI_Fint *, MPI_Offset *, MPI_Fint *, MPI_Fint *, char * FORT_MIXED_LEN_DECL, MPI_Fint *, MPI_Fint * FORT_END_LEN_DECL );
-#pragma weak mpi_file_set_view = pmpi_file_set_view
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_set_view_( MPI_Fint *, MPI_Offset *, MPI_Fint *, MPI_Fint *, char * FORT_MIXED_LEN_DECL, MPI_Fint *, MPI_Fint * FORT_END_LEN_DECL );
-#pragma weak mpi_file_set_view_ = pmpi_file_set_view_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_SET_VIEW MPI_FILE_SET_VIEW
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_set_view__ mpi_file_set_view__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_set_view mpi_file_set_view
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_set_view_ mpi_file_set_view_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_SET_VIEW as PMPI_FILE_SET_VIEW
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_set_view__ as pmpi_file_set_view__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_set_view as pmpi_file_set_view
-#else
-#pragma _CRI duplicate mpi_file_set_view_ as pmpi_file_set_view_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_set_view_ PMPI_FILE_SET_VIEW
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_set_view_ pmpi_file_set_view__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_set_view pmpi_file_set_view_
-#endif
-#define mpi_file_set_view_ pmpi_file_set_view
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_set_view_ pmpi_file_set_view
-#endif
-#define mpi_file_set_view_ pmpi_file_set_view_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_set_view_ MPI_FILE_SET_VIEW
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_set_view_ mpi_file_set_view__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_set_view mpi_file_set_view_
-#endif
-#define mpi_file_set_view_ mpi_file_set_view
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_set_view_ mpi_file_set_view
-#endif
-#endif
-#endif
-
-#if defined(MPIHP) || defined(MPILAM)
-/* Prototype to keep compiler happy */
-void mpi_file_set_view_(MPI_Fint *fh,MPI_Offset *disp,MPI_Fint *etype,
-   MPI_Fint *filetype,char *datarep,MPI_Fint *info, MPI_Fint *ierr,
-			int str_len );
-
-void mpi_file_set_view_(MPI_Fint *fh,MPI_Offset *disp,MPI_Fint *etype,
-   MPI_Fint *filetype,char *datarep,MPI_Fint *info, MPI_Fint *ierr,
-   int str_len )
-{
-    char *newstr;
-    MPI_File fh_c;
-    int i, real_len; 
-    MPI_Datatype etype_c, filetype_c;
-    MPI_Info info_c;
-    
-    etype_c = MPI_Type_f2c(*etype);
-    filetype_c = MPI_Type_f2c(*filetype);
-    info_c = MPI_Info_f2c(*info);
-
-    /* strip trailing blanks in datarep */
-    if (datarep <= (char *) 0) {
-        FPRINTF(stderr, "MPI_File_set_view: datarep is an invalid address\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-    for (i=str_len-1; i>=0; i--) if (datarep[i] != ' ') break;
-    if (i < 0) {
-	FPRINTF(stderr, "MPI_File_set_view: datarep is a blank string\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-    real_len = i + 1;
-
-    newstr = (char *) ADIOI_Malloc((real_len+1)*sizeof(char));
-    ADIOI_Strncpy(newstr, datarep, real_len);
-    newstr[real_len] = '\0';
-    
-    fh_c = MPI_File_f2c(*fh);
- 
-    *ierr = MPI_File_set_view(fh_c,*disp,etype_c,filetype_c,newstr,info_c);
-
-    ADIOI_Free(newstr);
-}
-
-#else
-
-#ifdef _UNICOS
-void mpi_file_set_view_(MPI_Fint *fh,MPI_Offset *disp,MPI_Datatype *etype,
-   MPI_Datatype *filetype,_fcd datarep_fcd,MPI_Fint *info, MPI_Fint *ierr)
-{
-   char *datarep = _fcdtocp(datarep_fcd);
-   int str_len = _fcdlen(datarep_fcd);
-#else
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_set_view_( MPI_Fint *fh, MPI_Offset *disp, MPI_Fint *etype, MPI_Fint *filetype, char *datarep FORT_MIXED_LEN_DECL, MPI_Fint *info, MPI_Fint *ierr FORT_END_LEN_DECL );
-
-FORTRAN_API void FORT_CALL mpi_file_set_view_( MPI_Fint *fh, MPI_Offset *disp, MPI_Fint *etype, MPI_Fint *filetype, char *datarep FORT_MIXED_LEN(str_len), MPI_Fint *info, MPI_Fint *ierr FORT_END_LEN(str_len) )
-{
-#endif
-    char *newstr;
-    MPI_File fh_c;
-    int i, real_len; 
-    MPI_Info info_c;
-    
-    info_c = MPI_Info_f2c(*info);
-
-    /* strip trailing blanks in datarep */
-    if (datarep <= (char *) 0) {
-        FPRINTF(stderr, "MPI_File_set_view: datarep is an invalid address\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-    for (i=str_len-1; i>=0; i--) if (datarep[i] != ' ') break;
-    if (i < 0) {
-	FPRINTF(stderr, "MPI_File_set_view: datarep is a blank string\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-    real_len = i + 1;
-
-    newstr = (char *) ADIOI_Malloc((real_len+1)*sizeof(char));
-    ADIOI_Strncpy(newstr, datarep, real_len);
-    newstr[real_len] = '\0';
-    
-    fh_c = MPI_File_f2c(*fh);
- 
-    *ierr = MPI_File_set_view(fh_c,*disp,*etype,*filetype,newstr,info_c);
-
-    ADIOI_Free(newstr);
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/fortran/wr_atallbf.c b/3rd-party/romio321/mpi-io/fortran/wr_atallbf.c
deleted file mode 100644
index 836dd378aee..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/wr_atallbf.c
+++ /dev/null
@@ -1,122 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_WRITE_AT_ALL_BEGIN( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak MPI_FILE_WRITE_AT_ALL_BEGIN = PMPI_FILE_WRITE_AT_ALL_BEGIN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_write_at_all_begin__( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_write_at_all_begin__ = pmpi_file_write_at_all_begin__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_write_at_all_begin( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_write_at_all_begin = pmpi_file_write_at_all_begin
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_write_at_all_begin_( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_write_at_all_begin_ = pmpi_file_write_at_all_begin_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_WRITE_AT_ALL_BEGIN MPI_FILE_WRITE_AT_ALL_BEGIN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_at_all_begin__ mpi_file_write_at_all_begin__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_at_all_begin mpi_file_write_at_all_begin
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_write_at_all_begin_ mpi_file_write_at_all_begin_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_WRITE_AT_ALL_BEGIN as PMPI_FILE_WRITE_AT_ALL_BEGIN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_write_at_all_begin__ as pmpi_file_write_at_all_begin__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_write_at_all_begin as pmpi_file_write_at_all_begin
-#else
-#pragma _CRI duplicate mpi_file_write_at_all_begin_ as pmpi_file_write_at_all_begin_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_write_at_all_begin_ PMPI_FILE_WRITE_AT_ALL_BEGIN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_write_at_all_begin_ pmpi_file_write_at_all_begin__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_at_all_begin pmpi_file_write_at_all_begin_
-#endif
-#define mpi_file_write_at_all_begin_ pmpi_file_write_at_all_begin
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_at_all_begin_ pmpi_file_write_at_all_begin
-#endif
-#define mpi_file_write_at_all_begin_ pmpi_file_write_at_all_begin_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_write_at_all_begin_ MPI_FILE_WRITE_AT_ALL_BEGIN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_write_at_all_begin_ mpi_file_write_at_all_begin__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_write_at_all_begin mpi_file_write_at_all_begin_
-#endif
-#define mpi_file_write_at_all_begin_ mpi_file_write_at_all_begin
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_write_at_all_begin_ mpi_file_write_at_all_begin
-#endif
-#endif
-#endif
-
-
-#if defined(MPIHP) || defined(MPILAM)
-/* Prototype to keep compiler happy */
-void mpi_file_write_at_all_begin_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-				  MPI_Fint *count,MPI_Fint *datatype, MPI_Fint *ierr );
-
-void mpi_file_write_at_all_begin_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-                          MPI_Fint *count,MPI_Fint *datatype, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPI_Datatype datatype_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    datatype_c = MPI_Type_f2c(*datatype);
-
-    *ierr = MPI_File_write_at_all_begin(fh_c,*offset,buf,*count,datatype_c);
-}
-#else
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_write_at_all_begin_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-				  MPI_Fint *count,MPI_Fint *datatype, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_write_at_all_begin_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-                          MPI_Fint *count,MPI_Fint *datatype, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_write_at_all_begin(fh_c,*offset,buf,*count,(MPI_Datatype)*datatype);
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/fortran/wr_atallef.c b/3rd-party/romio321/mpi-io/fortran/wr_atallef.c
deleted file mode 100644
index 7426c48bf2f..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/wr_atallef.c
+++ /dev/null
@@ -1,103 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_WRITE_AT_ALL_END( MPI_Fint *, void*, MPI_Status*, MPI_Fint * );
-#pragma weak MPI_FILE_WRITE_AT_ALL_END = PMPI_FILE_WRITE_AT_ALL_END
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_write_at_all_end__( MPI_Fint *, void*, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_write_at_all_end__ = pmpi_file_write_at_all_end__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_write_at_all_end( MPI_Fint *, void*, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_write_at_all_end = pmpi_file_write_at_all_end
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_write_at_all_end_( MPI_Fint *, void*, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_write_at_all_end_ = pmpi_file_write_at_all_end_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_WRITE_AT_ALL_END MPI_FILE_WRITE_AT_ALL_END
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_at_all_end__ mpi_file_write_at_all_end__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_at_all_end mpi_file_write_at_all_end
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_write_at_all_end_ mpi_file_write_at_all_end_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_WRITE_AT_ALL_END as PMPI_FILE_WRITE_AT_ALL_END
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_write_at_all_end__ as pmpi_file_write_at_all_end__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_write_at_all_end as pmpi_file_write_at_all_end
-#else
-#pragma _CRI duplicate mpi_file_write_at_all_end_ as pmpi_file_write_at_all_end_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_write_at_all_end_ PMPI_FILE_WRITE_AT_ALL_END
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_write_at_all_end_ pmpi_file_write_at_all_end__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_at_all_end pmpi_file_write_at_all_end_
-#endif
-#define mpi_file_write_at_all_end_ pmpi_file_write_at_all_end
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_at_all_end_ pmpi_file_write_at_all_end
-#endif
-#define mpi_file_write_at_all_end_ pmpi_file_write_at_all_end_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_write_at_all_end_ MPI_FILE_WRITE_AT_ALL_END
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_write_at_all_end_ mpi_file_write_at_all_end__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_write_at_all_end mpi_file_write_at_all_end_
-#endif
-#define mpi_file_write_at_all_end_ mpi_file_write_at_all_end
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_write_at_all_end_ mpi_file_write_at_all_end
-#endif
-#endif
-#endif
-
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_write_at_all_end_(MPI_Fint *fh,void *buf,MPI_Status *status, 
-				MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_write_at_all_end_(MPI_Fint *fh,void *buf,MPI_Status *status, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-
-    *ierr = MPI_File_write_at_all_end(fh_c,buf,status);
-}
diff --git a/3rd-party/romio321/mpi-io/fortran/write_allbf.c b/3rd-party/romio321/mpi-io/fortran/write_allbf.c
deleted file mode 100644
index e47d3cd2008..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/write_allbf.c
+++ /dev/null
@@ -1,117 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_WRITE_ALL_BEGIN( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak MPI_FILE_WRITE_ALL_BEGIN = PMPI_FILE_WRITE_ALL_BEGIN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_write_all_begin__( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_write_all_begin__ = pmpi_file_write_all_begin__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_write_all_begin( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_write_all_begin = pmpi_file_write_all_begin
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_write_all_begin_( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_write_all_begin_ = pmpi_file_write_all_begin_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_WRITE_ALL_BEGIN MPI_FILE_WRITE_ALL_BEGIN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_all_begin__ mpi_file_write_all_begin__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_all_begin mpi_file_write_all_begin
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_write_all_begin_ mpi_file_write_all_begin_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_WRITE_ALL_BEGIN as PMPI_FILE_WRITE_ALL_BEGIN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_write_all_begin__ as pmpi_file_write_all_begin__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_write_all_begin as pmpi_file_write_all_begin
-#else
-#pragma _CRI duplicate mpi_file_write_all_begin_ as pmpi_file_write_all_begin_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_write_all_begin_ PMPI_FILE_WRITE_ALL_BEGIN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_write_all_begin_ pmpi_file_write_all_begin__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_all_begin pmpi_file_write_all_begin_
-#endif
-#define mpi_file_write_all_begin_ pmpi_file_write_all_begin
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_all_begin_ pmpi_file_write_all_begin
-#endif
-#define mpi_file_write_all_begin_ pmpi_file_write_all_begin_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_write_all_begin_ MPI_FILE_WRITE_ALL_BEGIN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_write_all_begin_ mpi_file_write_all_begin__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_write_all_begin mpi_file_write_all_begin_
-#endif
-#define mpi_file_write_all_begin_ mpi_file_write_all_begin
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_write_all_begin_ mpi_file_write_all_begin
-#endif
-#endif
-#endif
-
-#if defined(MPIHP) || defined(MPILAM)
-/* Prototype to keep compiler happy */
-void mpi_file_write_all_begin_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-			       MPI_Fint *datatype, MPI_Fint *ierr );
-void mpi_file_write_all_begin_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                       MPI_Fint *datatype, MPI_Fint *ierr ){
-    MPI_File fh_c;
-    MPI_Datatype datatype_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    datatype_c = MPI_Type_f2c(*datatype);
-
-    *ierr = MPI_File_write_all_begin(fh_c,buf,*count,datatype_c);
-}
-#else
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_write_all_begin_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-			       MPI_Fint *datatype, MPI_Fint *ierr );
-FORTRAN_API void FORT_CALL mpi_file_write_all_begin_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                       MPI_Fint *datatype, MPI_Fint *ierr ){
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_write_all_begin(fh_c,buf,*count,(MPI_Datatype)*datatype);
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/fortran/write_allef.c b/3rd-party/romio321/mpi-io/fortran/write_allef.c
deleted file mode 100644
index 9ccc1eb8db9..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/write_allef.c
+++ /dev/null
@@ -1,101 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_WRITE_ALL_END( MPI_Fint *, void*, MPI_Status*, MPI_Fint * );
-#pragma weak MPI_FILE_WRITE_ALL_END = PMPI_FILE_WRITE_ALL_END
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_write_all_end__( MPI_Fint *, void*, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_write_all_end__ = pmpi_file_write_all_end__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_write_all_end( MPI_Fint *, void*, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_write_all_end = pmpi_file_write_all_end
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_write_all_end_( MPI_Fint *, void*, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_write_all_end_ = pmpi_file_write_all_end_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_WRITE_ALL_END MPI_FILE_WRITE_ALL_END
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_all_end__ mpi_file_write_all_end__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_all_end mpi_file_write_all_end
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_write_all_end_ mpi_file_write_all_end_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_WRITE_ALL_END as PMPI_FILE_WRITE_ALL_END
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_write_all_end__ as pmpi_file_write_all_end__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_write_all_end as pmpi_file_write_all_end
-#else
-#pragma _CRI duplicate mpi_file_write_all_end_ as pmpi_file_write_all_end_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_write_all_end_ PMPI_FILE_WRITE_ALL_END
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_write_all_end_ pmpi_file_write_all_end__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_all_end pmpi_file_write_all_end_
-#endif
-#define mpi_file_write_all_end_ pmpi_file_write_all_end
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_all_end_ pmpi_file_write_all_end
-#endif
-#define mpi_file_write_all_end_ pmpi_file_write_all_end_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_write_all_end_ MPI_FILE_WRITE_ALL_END
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_write_all_end_ mpi_file_write_all_end__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_write_all_end mpi_file_write_all_end_
-#endif
-#define mpi_file_write_all_end_ mpi_file_write_all_end
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_write_all_end_ mpi_file_write_all_end
-#endif
-#endif
-#endif
-
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_write_all_end_(MPI_Fint *fh,void *buf,MPI_Status *status, MPI_Fint *ierr ); 
-
-FORTRAN_API void FORT_CALL mpi_file_write_all_end_(MPI_Fint *fh,void *buf,MPI_Status *status, MPI_Fint *ierr ){
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-
-    *ierr = MPI_File_write_all_end(fh_c,buf,status);
-}
diff --git a/3rd-party/romio321/mpi-io/fortran/write_allf.c b/3rd-party/romio321/mpi-io/fortran/write_allf.c
deleted file mode 100644
index bddcac15092..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/write_allf.c
+++ /dev/null
@@ -1,119 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_WRITE_ALL( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak MPI_FILE_WRITE_ALL = PMPI_FILE_WRITE_ALL
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_write_all__( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_write_all__ = pmpi_file_write_all__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_write_all( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_write_all = pmpi_file_write_all
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_write_all_( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_write_all_ = pmpi_file_write_all_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_WRITE_ALL MPI_FILE_WRITE_ALL
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_all__ mpi_file_write_all__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_all mpi_file_write_all
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_write_all_ mpi_file_write_all_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_WRITE_ALL as PMPI_FILE_WRITE_ALL
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_write_all__ as pmpi_file_write_all__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_write_all as pmpi_file_write_all
-#else
-#pragma _CRI duplicate mpi_file_write_all_ as pmpi_file_write_all_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_write_all_ PMPI_FILE_WRITE_ALL
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_write_all_ pmpi_file_write_all__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_all pmpi_file_write_all_
-#endif
-#define mpi_file_write_all_ pmpi_file_write_all
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_all_ pmpi_file_write_all
-#endif
-#define mpi_file_write_all_ pmpi_file_write_all_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_write_all_ MPI_FILE_WRITE_ALL
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_write_all_ mpi_file_write_all__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_write_all mpi_file_write_all_
-#endif
-#define mpi_file_write_all_ mpi_file_write_all
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_write_all_ mpi_file_write_all
-#endif
-#endif
-#endif
-
-#if defined(MPIHP) || defined(MPILAM)
-/* Prototype to keep compiler happy */
-void mpi_file_write_all_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-		 MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr );
-
-void mpi_file_write_all_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                       MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr ){
-    MPI_File fh_c;
-    MPI_Datatype datatype_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    datatype_c = MPI_Type_f2c(*datatype);
-
-    *ierr = MPI_File_write_all(fh_c,buf,*count,datatype_c,status);
-}
-#else
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_write_all_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-		 MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_write_all_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                  MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr ){
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_write_all(fh_c,buf,*count,*datatype,status);
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/fortran/write_atallf.c b/3rd-party/romio321/mpi-io/fortran/write_atallf.c
deleted file mode 100644
index 030e7ae359d..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/write_atallf.c
+++ /dev/null
@@ -1,125 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_WRITE_AT_ALL( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak MPI_FILE_WRITE_AT_ALL = PMPI_FILE_WRITE_AT_ALL
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_write_at_all__( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_write_at_all__ = pmpi_file_write_at_all__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_write_at_all( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_write_at_all = pmpi_file_write_at_all
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_write_at_all_( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_write_at_all_ = pmpi_file_write_at_all_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_WRITE_AT_ALL MPI_FILE_WRITE_AT_ALL
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_at_all__ mpi_file_write_at_all__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_at_all mpi_file_write_at_all
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_write_at_all_ mpi_file_write_at_all_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_WRITE_AT_ALL as PMPI_FILE_WRITE_AT_ALL
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_write_at_all__ as pmpi_file_write_at_all__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_write_at_all as pmpi_file_write_at_all
-#else
-#pragma _CRI duplicate mpi_file_write_at_all_ as pmpi_file_write_at_all_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_write_at_all_ PMPI_FILE_WRITE_AT_ALL
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_write_at_all_ pmpi_file_write_at_all__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_at_all pmpi_file_write_at_all_
-#endif
-#define mpi_file_write_at_all_ pmpi_file_write_at_all
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_at_all_ pmpi_file_write_at_all
-#endif
-#define mpi_file_write_at_all_ pmpi_file_write_at_all_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_write_at_all_ MPI_FILE_WRITE_AT_ALL
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_write_at_all_ mpi_file_write_at_all__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_write_at_all mpi_file_write_at_all_
-#endif
-#define mpi_file_write_at_all_ mpi_file_write_at_all
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_write_at_all_ mpi_file_write_at_all
-#endif
-#endif
-#endif
-
-#if defined(MPIHP) || defined(MPILAM)
-/* Prototype to keep compiler happy */
-void mpi_file_write_at_all_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-                          MPI_Fint *count,MPI_Fint *datatype,
-			    MPI_Status *status, MPI_Fint *ierr );
-
-void mpi_file_write_at_all_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-                          MPI_Fint *count,MPI_Fint *datatype,
-                          MPI_Status *status, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPI_Datatype datatype_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    datatype_c = MPI_Type_f2c(*datatype);
-
-    *ierr = MPI_File_write_at_all(fh_c,*offset,buf,*count,datatype_c,status);
-}
-#else
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_write_at_all_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-                          MPI_Fint *count,MPI_Fint *datatype,
-			    MPI_Status *status, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_write_at_all_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-                          MPI_Fint *count,MPI_Fint *datatype,
-                          MPI_Status *status, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_write_at_all(fh_c,*offset,buf,*count,*datatype,status);
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/fortran/write_atf.c b/3rd-party/romio321/mpi-io/fortran/write_atf.c
deleted file mode 100644
index 8ba429dcc6b..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/write_atf.c
+++ /dev/null
@@ -1,125 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_WRITE_AT( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak MPI_FILE_WRITE_AT = PMPI_FILE_WRITE_AT
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_write_at__( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_write_at__ = pmpi_file_write_at__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_write_at( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_write_at = pmpi_file_write_at
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_write_at_( MPI_Fint *, MPI_Offset *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_write_at_ = pmpi_file_write_at_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_WRITE_AT MPI_FILE_WRITE_AT
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_at__ mpi_file_write_at__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_at mpi_file_write_at
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_write_at_ mpi_file_write_at_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_WRITE_AT as PMPI_FILE_WRITE_AT
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_write_at__ as pmpi_file_write_at__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_write_at as pmpi_file_write_at
-#else
-#pragma _CRI duplicate mpi_file_write_at_ as pmpi_file_write_at_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_write_at_ PMPI_FILE_WRITE_AT
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_write_at_ pmpi_file_write_at__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_at pmpi_file_write_at_
-#endif
-#define mpi_file_write_at_ pmpi_file_write_at
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_at_ pmpi_file_write_at
-#endif
-#define mpi_file_write_at_ pmpi_file_write_at_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_write_at_ MPI_FILE_WRITE_AT
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_write_at_ mpi_file_write_at__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_write_at mpi_file_write_at_
-#endif
-#define mpi_file_write_at_ mpi_file_write_at
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_write_at_ mpi_file_write_at
-#endif
-#endif
-#endif
-
-#if defined(MPIHP) || defined(MPILAM)
-/* Prototype to keep compiler happy */
-void mpi_file_write_at_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-                      MPI_Fint *count,MPI_Fint *datatype,
-			MPI_Status *status, MPI_Fint *ierr );
-
-void mpi_file_write_at_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-                      MPI_Fint *count,MPI_Fint *datatype,
-                      MPI_Status *status, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPI_Datatype datatype_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    datatype_c = MPI_Type_f2c(*datatype);
-
-    *ierr = MPI_File_write_at(fh_c,*offset,buf,*count,datatype_c,status);
-}
-#else
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_write_at_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-                      MPI_Fint *count,MPI_Fint *datatype,
-			MPI_Status *status, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_write_at_(MPI_Fint *fh,MPI_Offset *offset,void *buf,
-                      MPI_Fint *count,MPI_Fint *datatype,
-                      MPI_Status *status, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_write_at(fh_c,*offset,buf,*count,*datatype,status);
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/fortran/write_ordbf.c b/3rd-party/romio321/mpi-io/fortran/write_ordbf.c
deleted file mode 100644
index 41b9accfbef..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/write_ordbf.c
+++ /dev/null
@@ -1,119 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_WRITE_ORDERED_BEGIN( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak MPI_FILE_WRITE_ORDERED_BEGIN = PMPI_FILE_WRITE_ORDERED_BEGIN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_write_ordered_begin__( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_write_ordered_begin__ = pmpi_file_write_ordered_begin__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_write_ordered_begin( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_write_ordered_begin = pmpi_file_write_ordered_begin
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_write_ordered_begin_( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Fint * );
-#pragma weak mpi_file_write_ordered_begin_ = pmpi_file_write_ordered_begin_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_WRITE_ORDERED_BEGIN MPI_FILE_WRITE_ORDERED_BEGIN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_ordered_begin__ mpi_file_write_ordered_begin__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_ordered_begin mpi_file_write_ordered_begin
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_write_ordered_begin_ mpi_file_write_ordered_begin_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_WRITE_ORDERED_BEGIN as PMPI_FILE_WRITE_ORDERED_BEGIN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_write_ordered_begin__ as pmpi_file_write_ordered_begin__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_write_ordered_begin as pmpi_file_write_ordered_begin
-#else
-#pragma _CRI duplicate mpi_file_write_ordered_begin_ as pmpi_file_write_ordered_begin_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_write_ordered_begin_ PMPI_FILE_WRITE_ORDERED_BEGIN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_write_ordered_begin_ pmpi_file_write_ordered_begin__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_ordered_begin pmpi_file_write_ordered_begin_
-#endif
-#define mpi_file_write_ordered_begin_ pmpi_file_write_ordered_begin
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_ordered_begin_ pmpi_file_write_ordered_begin
-#endif
-#define mpi_file_write_ordered_begin_ pmpi_file_write_ordered_begin_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_write_ordered_begin_ MPI_FILE_WRITE_ORDERED_BEGIN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_write_ordered_begin_ mpi_file_write_ordered_begin__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_write_ordered_begin mpi_file_write_ordered_begin_
-#endif
-#define mpi_file_write_ordered_begin_ mpi_file_write_ordered_begin
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_write_ordered_begin_ mpi_file_write_ordered_begin
-#endif
-#endif
-#endif
-
-#if defined(MPIHP) || defined(MPILAM)
-/* Prototype to keep compiler happy */
-void mpi_file_write_ordered_begin_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-				   MPI_Fint *datatype, MPI_Fint *ierr );
-
-void mpi_file_write_ordered_begin_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                       MPI_Fint *datatype, MPI_Fint *ierr ){
-    MPI_File fh_c;
-    MPI_Datatype datatype_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    datatype_c = MPI_Type_f2c(*datatype);
-
-    *ierr = MPI_File_write_ordered_begin(fh_c,buf,*count,datatype_c);
-}
-#else
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_write_ordered_begin_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-				   MPI_Fint *datatype, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_write_ordered_begin_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                       MPI_Fint *datatype, MPI_Fint *ierr ){
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_write_ordered_begin(fh_c,buf,*count,*datatype);
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/fortran/write_ordef.c b/3rd-party/romio321/mpi-io/fortran/write_ordef.c
deleted file mode 100644
index a979f70eee9..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/write_ordef.c
+++ /dev/null
@@ -1,102 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_WRITE_ORDERED_END( MPI_Fint *, void*, MPI_Status*, MPI_Fint * );
-#pragma weak MPI_FILE_WRITE_ORDERED_END = PMPI_FILE_WRITE_ORDERED_END
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_write_ordered_end__( MPI_Fint *, void*, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_write_ordered_end__ = pmpi_file_write_ordered_end__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_write_ordered_end( MPI_Fint *, void*, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_write_ordered_end = pmpi_file_write_ordered_end
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_write_ordered_end_( MPI_Fint *, void*, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_write_ordered_end_ = pmpi_file_write_ordered_end_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_WRITE_ORDERED_END MPI_FILE_WRITE_ORDERED_END
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_ordered_end__ mpi_file_write_ordered_end__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_ordered_end mpi_file_write_ordered_end
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_write_ordered_end_ mpi_file_write_ordered_end_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_WRITE_ORDERED_END as PMPI_FILE_WRITE_ORDERED_END
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_write_ordered_end__ as pmpi_file_write_ordered_end__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_write_ordered_end as pmpi_file_write_ordered_end
-#else
-#pragma _CRI duplicate mpi_file_write_ordered_end_ as pmpi_file_write_ordered_end_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_write_ordered_end_ PMPI_FILE_WRITE_ORDERED_END
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_write_ordered_end_ pmpi_file_write_ordered_end__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_ordered_end pmpi_file_write_ordered_end_
-#endif
-#define mpi_file_write_ordered_end_ pmpi_file_write_ordered_end
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_ordered_end_ pmpi_file_write_ordered_end
-#endif
-#define mpi_file_write_ordered_end_ pmpi_file_write_ordered_end_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_write_ordered_end_ MPI_FILE_WRITE_ORDERED_END
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_write_ordered_end_ mpi_file_write_ordered_end__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_write_ordered_end mpi_file_write_ordered_end_
-#endif
-#define mpi_file_write_ordered_end_ mpi_file_write_ordered_end
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_write_ordered_end_ mpi_file_write_ordered_end
-#endif
-#endif
-#endif
-
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_write_ordered_end_(MPI_Fint *fh,void *buf,MPI_Status *status, 
-				 MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_write_ordered_end_(MPI_Fint *fh,void *buf,MPI_Status *status, MPI_Fint *ierr ){
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-
-    *ierr = MPI_File_write_ordered_end(fh_c,buf,status);
-}
diff --git a/3rd-party/romio321/mpi-io/fortran/write_ordf.c b/3rd-party/romio321/mpi-io/fortran/write_ordf.c
deleted file mode 100644
index fe96406d9bb..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/write_ordf.c
+++ /dev/null
@@ -1,119 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_WRITE_ORDERED( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak MPI_FILE_WRITE_ORDERED = PMPI_FILE_WRITE_ORDERED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_write_ordered__( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_write_ordered__ = pmpi_file_write_ordered__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_write_ordered( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_write_ordered = pmpi_file_write_ordered
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_write_ordered_( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_write_ordered_ = pmpi_file_write_ordered_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_WRITE_ORDERED MPI_FILE_WRITE_ORDERED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_ordered__ mpi_file_write_ordered__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_ordered mpi_file_write_ordered
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_write_ordered_ mpi_file_write_ordered_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_WRITE_ORDERED as PMPI_FILE_WRITE_ORDERED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_write_ordered__ as pmpi_file_write_ordered__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_write_ordered as pmpi_file_write_ordered
-#else
-#pragma _CRI duplicate mpi_file_write_ordered_ as pmpi_file_write_ordered_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_write_ordered_ PMPI_FILE_WRITE_ORDERED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_write_ordered_ pmpi_file_write_ordered__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_ordered pmpi_file_write_ordered_
-#endif
-#define mpi_file_write_ordered_ pmpi_file_write_ordered
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_ordered_ pmpi_file_write_ordered
-#endif
-#define mpi_file_write_ordered_ pmpi_file_write_ordered_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_write_ordered_ MPI_FILE_WRITE_ORDERED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_write_ordered_ mpi_file_write_ordered__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_write_ordered mpi_file_write_ordered_
-#endif
-#define mpi_file_write_ordered_ mpi_file_write_ordered
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_write_ordered_ mpi_file_write_ordered
-#endif
-#endif
-#endif
-
-#if defined(MPIHP) || defined(MPILAM)
-/* Prototype to keep compiler happy */
-void mpi_file_write_ordered_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-		     MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr );
-
-void mpi_file_write_ordered_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                       MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr ){
-    MPI_File fh_c;
-    MPI_Datatype datatype_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    datatype_c = MPI_Type_f2c(*datatype);
-
-    *ierr = MPI_File_write_ordered(fh_c,buf,*count,datatype_c,status);
-}
-#else
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_write_ordered_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-		     MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_write_ordered_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                       MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr ){
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_write_ordered(fh_c,buf,*count,*datatype,status);
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/fortran/write_shf.c b/3rd-party/romio321/mpi-io/fortran/write_shf.c
deleted file mode 100644
index 30d80e0ee6d..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/write_shf.c
+++ /dev/null
@@ -1,121 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_WRITE_SHARED( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak MPI_FILE_WRITE_SHARED = PMPI_FILE_WRITE_SHARED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_write_shared__( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_write_shared__ = pmpi_file_write_shared__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_write_shared( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_write_shared = pmpi_file_write_shared
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_write_shared_( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_write_shared_ = pmpi_file_write_shared_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_WRITE_SHARED MPI_FILE_WRITE_SHARED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_shared__ mpi_file_write_shared__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_shared mpi_file_write_shared
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_write_shared_ mpi_file_write_shared_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_WRITE_SHARED as PMPI_FILE_WRITE_SHARED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_write_shared__ as pmpi_file_write_shared__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_write_shared as pmpi_file_write_shared
-#else
-#pragma _CRI duplicate mpi_file_write_shared_ as pmpi_file_write_shared_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_write_shared_ PMPI_FILE_WRITE_SHARED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_write_shared_ pmpi_file_write_shared__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_shared pmpi_file_write_shared_
-#endif
-#define mpi_file_write_shared_ pmpi_file_write_shared
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_shared_ pmpi_file_write_shared
-#endif
-#define mpi_file_write_shared_ pmpi_file_write_shared_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_write_shared_ MPI_FILE_WRITE_SHARED
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_write_shared_ mpi_file_write_shared__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_write_shared mpi_file_write_shared_
-#endif
-#define mpi_file_write_shared_ mpi_file_write_shared
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_write_shared_ mpi_file_write_shared
-#endif
-#endif
-#endif
-
-#if defined(MPIHP) || defined(MPILAM)
-/* Prototype to keep compiler happy */
-void mpi_file_write_shared_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-		    MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr );
-
-void mpi_file_write_shared_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                   MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPI_Datatype datatype_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    datatype_c = MPI_Type_f2c(*datatype);
-
-    *ierr = MPI_File_write_shared(fh_c, buf,*count,datatype_c,status);
-}
-#else
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_write_shared_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-		    MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_write_shared_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                   MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_write_shared(fh_c, buf,*count,*datatype,status);
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/fortran/writef.c b/3rd-party/romio321/mpi-io/fortran/writef.c
deleted file mode 100644
index 9b07ba095fa..00000000000
--- a/3rd-party/romio321/mpi-io/fortran/writef.c
+++ /dev/null
@@ -1,121 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_FILE_WRITE( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak MPI_FILE_WRITE = PMPI_FILE_WRITE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_write__( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_write__ = pmpi_file_write__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_file_write( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_write = pmpi_file_write
-#else
-extern FORTRAN_API void FORT_CALL mpi_file_write_( MPI_Fint *, void*, MPI_Fint *, MPI_Fint *, MPI_Status*, MPI_Fint * );
-#pragma weak mpi_file_write_ = pmpi_file_write_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_FILE_WRITE MPI_FILE_WRITE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_write__ mpi_file_write__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_file_write mpi_file_write
-#else
-#pragma _HP_SECONDARY_DEF pmpi_file_write_ mpi_file_write_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_FILE_WRITE as PMPI_FILE_WRITE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_file_write__ as pmpi_file_write__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_file_write as pmpi_file_write
-#else
-#pragma _CRI duplicate mpi_file_write_ as pmpi_file_write_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_file_write_ PMPI_FILE_WRITE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_write_ pmpi_file_write__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_write pmpi_file_write_
-#endif
-#define mpi_file_write_ pmpi_file_write
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_file_write_ pmpi_file_write
-#endif
-#define mpi_file_write_ pmpi_file_write_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_file_write_ MPI_FILE_WRITE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_file_write_ mpi_file_write__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_write mpi_file_write_
-#endif
-#define mpi_file_write_ mpi_file_write
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_file_write_ mpi_file_write
-#endif
-#endif
-#endif
-
-#if defined(MPIHP) || defined(MPILAM)
-/* Prototype to keep compiler happy */
-void mpi_file_write_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-		     MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr );
-
-void mpi_file_write_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                   MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    MPI_Datatype datatype_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    datatype_c = MPI_Type_f2c(*datatype);
-
-    *ierr = MPI_File_write(fh_c, buf,*count,datatype_c,status);
-}
-#else
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL mpi_file_write_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-		     MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL mpi_file_write_(MPI_Fint *fh,void *buf,MPI_Fint *count,
-                   MPI_Fint *datatype,MPI_Status *status, MPI_Fint *ierr )
-{
-    MPI_File fh_c;
-    
-    fh_c = MPI_File_f2c(*fh);
-    *ierr = MPI_File_write(fh_c, buf,*count,*datatype,status);
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/fsync.c b/3rd-party/romio321/mpi-io/fsync.c
deleted file mode 100644
index 00c94996294..00000000000
--- a/3rd-party/romio321/mpi-io/fsync.c
+++ /dev/null
@@ -1,76 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_sync = PMPI_File_sync
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_sync MPI_File_sync
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_sync as PMPI_File_sync
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_sync(MPI_File fh) __attribute__((weak,alias("PMPI_File_sync")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_sync - Causes all previous writes to be transferred
-                    to the storage device
-
-Input Parameters:
-. fh - file handle (handle)
-
-.N fortran
-@*/
-int MPI_File_sync(MPI_File fh)
-{
-    int error_code;
-    ADIO_File adio_fh;
-    static char myname[] = "MPI_FILE_SYNC";
-#ifdef MPI_hpux
-    int fl_xmpi;
-
-    HPMP_IO_START(fl_xmpi, BLKMPIFILESYNC, TRDTBLOCK, adio_fh,
-		  MPI_DATATYPE_NULL, -1);
-#endif /* MPI_hpux */
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-    /* --BEGIN ERROR HANDLING-- */
-    if ((adio_fh == NULL) || ((adio_fh)->cookie != ADIOI_FILE_COOKIE))
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "**iobadfh", 0);
-	error_code = MPIO_Err_return_file(MPI_FILE_NULL, error_code);
-	goto fn_exit;
-    }
-    MPIO_CHECK_WRITABLE(fh, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    ADIO_Flush(adio_fh, &error_code);
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-    /* --END ERROR HANDLING-- */
-
-#ifdef MPI_hpux
-    HPMP_IO_END(fl_xmpi, adio_fh, MPI_DATATYPE_NULL, -1);
-#endif /* MPI_hpux */
- 
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/get_amode.c b/3rd-party/romio321/mpi-io/get_amode.c
deleted file mode 100644
index 3e9e2c3120f..00000000000
--- a/3rd-party/romio321/mpi-io/get_amode.c
+++ /dev/null
@@ -1,55 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_get_amode = PMPI_File_get_amode
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_get_amode MPI_File_get_amode
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_get_amode as PMPI_File_get_amode
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_get_amode(MPI_File fh, int *amode) __attribute__((weak,alias("PMPI_File_get_amode")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_get_amode - Returns the file access mode
-
-Input Parameters:
-. fh - file handle (handle)
-
-Output Parameters:
-. amode - access mode (integer)
-
-.N fortran
-@*/
-int MPI_File_get_amode(MPI_File fh, int *amode)
-{
-    int error_code=MPI_SUCCESS;
-    static char myname[] = "MPI_FILE_GET_AMODE";
-    ADIO_File adio_fh;
-    
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    *amode = adio_fh->orig_access_mode;
-
-fn_exit:
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/get_atom.c b/3rd-party/romio321/mpi-io/get_atom.c
deleted file mode 100644
index 1a9eae1389e..00000000000
--- a/3rd-party/romio321/mpi-io/get_atom.c
+++ /dev/null
@@ -1,55 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_get_atomicity = PMPI_File_get_atomicity
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_get_atomicity MPI_File_get_atomicity
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_get_atomicity as PMPI_File_get_atomicity
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_get_atomicity(MPI_File fh, int *flag) __attribute__((weak,alias("PMPI_File_get_atomicity")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_get_atomicity - Returns the atomicity mode
-
-Input Parameters:
-. fh - file handle (handle)
-
-Output Parameters:
-. flag - true if atomic mode, false if nonatomic mode (logical)
-
-.N fortran
-@*/
-int MPI_File_get_atomicity(MPI_File fh, int *flag)
-{
-    int error_code;
-    ADIO_File adio_fh;
-    static char myname[] = "MPI_FILE_GET_ATOMICITY";
-    
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    *flag = adio_fh->atomicity;
-
-fn_exit:
-    return MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/mpi-io/get_bytoff.c b/3rd-party/romio321/mpi-io/get_bytoff.c
deleted file mode 100644
index f5cb452e5a8..00000000000
--- a/3rd-party/romio321/mpi-io/get_bytoff.c
+++ /dev/null
@@ -1,71 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-#include "adioi.h" /* ADIOI_Get_byte_offset() prototype */
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_get_byte_offset = PMPI_File_get_byte_offset
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_get_byte_offset MPI_File_get_byte_offset
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_get_byte_offset as PMPI_File_get_byte_offset
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_get_byte_offset(MPI_File fh, MPI_Offset offset, MPI_Offset *disp) __attribute__((weak,alias("PMPI_File_get_byte_offset")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_get_byte_offset - Returns the absolute byte position in 
-                the file corresponding to "offset" etypes relative to
-                the current view
-
-Input Parameters:
-. fh - file handle (handle)
-. offset - offset (nonnegative integer)
-
-Output Parameters:
-. disp - absolute byte position of offset (nonnegative integer)
-
-.N fortran
-@*/
-int MPI_File_get_byte_offset(MPI_File fh, MPI_Offset offset, MPI_Offset *disp)
-{
-    int error_code;
-    ADIO_File adio_fh;
-    static char myname[] = "MPI_FILE_GET_BYTE_OFFSET";
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-
-    if (offset < 0)
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "**iobadoffset", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-
-    MPIO_CHECK_NOT_SEQUENTIAL_MODE(adio_fh, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    ADIOI_Get_byte_offset(adio_fh, offset, disp);
-
-fn_exit:
-
-    return MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/mpi-io/get_errh.c b/3rd-party/romio321/mpi-io/get_errh.c
deleted file mode 100644
index 7962e33f921..00000000000
--- a/3rd-party/romio321/mpi-io/get_errh.c
+++ /dev/null
@@ -1,71 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-#include "adio_extern.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_get_errhandler = PMPI_File_get_errhandler
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_get_errhandler MPI_File_get_errhandler
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_get_errhandler as PMPI_File_get_errhandler
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_get_errhandler(MPI_File file, MPI_Errhandler *errhandler) __attribute__((weak,alias("PMPI_File_get_errhandler")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_get_errhandler - Returns the error handler for a file
-
-Input Parameters:
-. fh - file handle (handle)
-
-Output Parameters:
-. errhandler - error handler (handle)
-
-.N fortran
-@*/
-int MPI_File_get_errhandler(MPI_File mpi_fh, MPI_Errhandler *errhandler)
-{
-    int error_code = MPI_SUCCESS;
-    ADIO_File fh;
-    static char myname[] = "MPI_FILE_GET_ERRHANDLER";
-    MPID_THREADPRIV_DECL;
-
-    ROMIO_THREAD_CS_ENTER();
-
-    if (mpi_fh == MPI_FILE_NULL) {
-	*errhandler = ADIOI_DFLT_ERR_HANDLER;
-    }
-    else {
-	fh = MPIO_File_resolve(mpi_fh);
-	/* --BEGIN ERROR HANDLING-- */
-	if ((fh <= (MPI_File) 0) || ((fh)->cookie != ADIOI_FILE_COOKIE))
-	{
-	    error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					      myname, __LINE__, MPI_ERR_ARG,
-					      "**iobadfh", 0);
-	    error_code = MPIO_Err_return_file(MPI_FILE_NULL, error_code);
-	    goto fn_exit;
-	}
-	/* --END ERROR HANDLING-- */
-
-	*errhandler = fh->err_handler;
-    }
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-    return MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/mpi-io/get_extent.c b/3rd-party/romio321/mpi-io/get_extent.c
deleted file mode 100644
index 960afab5e55..00000000000
--- a/3rd-party/romio321/mpi-io/get_extent.c
+++ /dev/null
@@ -1,60 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_get_type_extent = PMPI_File_get_type_extent
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_get_type_extent MPI_File_get_type_extent
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_get_type_extent as PMPI_File_get_type_extent
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_get_type_extent(MPI_File fh, MPI_Datatype datatype, MPI_Aint *extent) __attribute__((weak,alias("PMPI_File_get_type_extent")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_get_type_extent - Returns the extent of datatype in the file
-
-Input Parameters:
-. fh - file handle (handle)
-. datatype - datatype (handle)
-
-Output Parameters:
-. extent - extent of the datatype (nonnegative integer)
-
-.N fortran
-@*/
-int MPI_File_get_type_extent(MPI_File fh, MPI_Datatype datatype, MPI_Aint *extent)
-{
-    int error_code;
-    ADIO_File adio_fh;
-    static char myname[] = "MPI_FILE_GET_TYPE_EXTENT";
-    MPI_Aint lb;
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    MPIO_CHECK_DATATYPE(adio_fh, datatype, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    /* FIXME: handle other file data representations */
-
-    error_code = MPI_Type_get_extent(datatype, &lb, extent);
-
-fn_exit:
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/get_group.c b/3rd-party/romio321/mpi-io/get_group.c
deleted file mode 100644
index 2b3077c7dc1..00000000000
--- a/3rd-party/romio321/mpi-io/get_group.c
+++ /dev/null
@@ -1,64 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_get_group = PMPI_File_get_group
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_get_group MPI_File_get_group
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_get_group as PMPI_File_get_group
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_get_group(MPI_File fh, MPI_Group *group) __attribute__((weak,alias("PMPI_File_get_group")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_get_group - Returns the group of processes that 
-                         opened the file
-
-Input Parameters:
-. fh - file handle (handle)
-
-Output Parameters:
-. group - group that opened the file (handle)
-
-.N fortran
-@*/
-int MPI_File_get_group(MPI_File fh, MPI_Group *group)
-{
-    int error_code;
-    ADIO_File adio_fh;
-    static char myname[] = "MPI_FILE_GET_GROUP";
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-
-    /* note: this will return the group of processes that called open, but
-     * with deferred open this might not be the group of processes that
-     * actually opened the file from the file system's perspective
-     */
-    error_code = MPI_Comm_group(adio_fh->comm, group);
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/get_info.c b/3rd-party/romio321/mpi-io/get_info.c
deleted file mode 100644
index e923e2b95ec..00000000000
--- a/3rd-party/romio321/mpi-io/get_info.c
+++ /dev/null
@@ -1,62 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_get_info = PMPI_File_get_info
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_get_info MPI_File_get_info
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_get_info as PMPI_File_get_info
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_get_info(MPI_File fh, MPI_Info *info_used) __attribute__((weak,alias("PMPI_File_get_info")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_get_info - Returns the hints for a file that are actually being used by MPI
-
-Input Parameters:
-. fh - file handle (handle)
-
-Output Parameters:
-. info_used - info object (handle)
-
-.N fortran
-@*/
-int MPI_File_get_info(MPI_File fh, MPI_Info *info_used)
-{
-    int error_code;
-    ADIO_File adio_fh;
-    static char myname[] = "MPI_FILE_GET_INFO";
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    error_code = MPI_Info_dup(adio_fh->info, info_used);
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-    /* --END ERROR HANDLING-- */
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-    return  error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/get_posn.c b/3rd-party/romio321/mpi-io/get_posn.c
deleted file mode 100644
index 49d8e4919bd..00000000000
--- a/3rd-party/romio321/mpi-io/get_posn.c
+++ /dev/null
@@ -1,59 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-#include "adioi.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_get_position = PMPI_File_get_position
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_get_position MPI_File_get_position
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_get_position as PMPI_File_get_position
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_get_position(MPI_File fh, MPI_Offset *offset) __attribute__((weak,alias("PMPI_File_get_position")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_get_position - Returns the current position of the 
-                individual file pointer in etype units relative to
-                the current view
-
-Input Parameters:
-. fh - file handle (handle)
-
-Output Parameters:
-. offset - offset of individual file pointer (nonnegative integer)
-
-.N fortran
-@*/
-int MPI_File_get_position(MPI_File fh, MPI_Offset *offset)
-{
-    int error_code;
-    ADIO_File adio_fh;
-    static char myname[] = "MPI_FILE_GET_POSITION";
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    MPIO_CHECK_NOT_SEQUENTIAL_MODE(adio_fh, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    ADIOI_Get_position(adio_fh, offset);
-
-fn_exit:
-    return MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/mpi-io/get_posn_sh.c b/3rd-party/romio321/mpi-io/get_posn_sh.c
deleted file mode 100644
index c6172c9d734..00000000000
--- a/3rd-party/romio321/mpi-io/get_posn_sh.c
+++ /dev/null
@@ -1,64 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_get_position_shared = PMPI_File_get_position_shared
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_get_position_shared MPI_File_get_position_shared
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_get_position_shared as PMPI_File_get_position_shared
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_get_position_shared(MPI_File fh, MPI_Offset *offset) __attribute__((weak,alias("PMPI_File_get_position_shared")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_get_position_shared - Returns the current position of the 
-               shared file pointer in etype units relative to the current view
-
-Input Parameters:
-. fh - file handle (handle)
-
-Output Parameters:
-. offset - offset of shared file pointer (nonnegative integer)
-
-.N fortran
-@*/
-int MPI_File_get_position_shared(MPI_File fh, MPI_Offset *offset)
-{
-    int error_code;
-    ADIO_File adio_fh;
-    static char myname[] = "MPI_FILE_GET_POSITION_SHARED";
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    MPIO_CHECK_NOT_SEQUENTIAL_MODE(adio_fh, myname, error_code);
-    MPIO_CHECK_FS_SUPPORTS_SHARED(adio_fh, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    ADIOI_TEST_DEFERRED(adio_fh, myname, &error_code);
-
-    ADIO_Get_shared_fp(adio_fh, 0, offset, &error_code);
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-    /* --END ERROR HANDLING-- */
-
-fn_exit:
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/get_size.c b/3rd-party/romio321/mpi-io/get_size.c
deleted file mode 100644
index d7836c2791e..00000000000
--- a/3rd-party/romio321/mpi-io/get_size.c
+++ /dev/null
@@ -1,86 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_get_size = PMPI_File_get_size
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_get_size MPI_File_get_size
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_get_size as PMPI_File_get_size
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_get_size(MPI_File fh, MPI_Offset *size) __attribute__((weak,alias("PMPI_File_get_size")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_get_size - Returns the file size
-
-Input Parameters:
-. fh - file handle (handle)
-
-Output Parameters:
-. size - size of the file in bytes (nonnegative integer)
-
-.N fortran
-@*/
-int MPI_File_get_size(MPI_File fh, MPI_Offset *size)
-{
-    int error_code;
-    ADIO_File adio_fh;
-    ADIO_Fcntl_t *fcntl_struct;
-    static char myname[] = "MPI_FILE_GET_SIZE";
-#ifdef MPI_hpux
-    int fl_xmpi;
-
-    HPMP_IO_START(fl_xmpi, BLKMPIFILEGETSIZE, TRDTBLOCK, adio_fh,
-		  MPI_DATATYPE_NULL, -1);
-#endif /* MPI_hpux */
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    if(size == NULL){
-        error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-                     myname, __LINE__, MPI_ERR_ARG,
-                     "**nullptr", "**nullptr %s", "size");
-        goto fn_fail;
-    }
-    /* --END ERROR HANDLING-- */
-
-    ADIOI_TEST_DEFERRED(adio_fh, myname, &error_code);
-
-    fcntl_struct = (ADIO_Fcntl_t *) ADIOI_Malloc(sizeof(ADIO_Fcntl_t));
-    ADIO_Fcntl(adio_fh, ADIO_FCNTL_GET_FSIZE, fcntl_struct, &error_code);
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-    /* --END ERROR HANDLING-- */
-    *size = fcntl_struct->fsize;
-    ADIOI_Free(fcntl_struct);
-
-#ifdef MPI_hpux
-    HPMP_IO_END(fl_xmpi, adio_fh, MPI_DATATYPE_NULL, -1);
-#endif /* MPI_hpux */
-
-fn_exit:
-    return error_code;
-fn_fail:
-    /* --BEGIN ERROR HANDLING-- */
-    error_code = MPIO_Err_return_file(fh, error_code);
-    goto fn_exit;
-    /* --END ERROR HANDLING-- */
-}
diff --git a/3rd-party/romio321/mpi-io/get_view.c b/3rd-party/romio321/mpi-io/get_view.c
deleted file mode 100644
index e07cd9fa951..00000000000
--- a/3rd-party/romio321/mpi-io/get_view.c
+++ /dev/null
@@ -1,102 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_get_view = PMPI_File_get_view
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_get_view MPI_File_get_view
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_get_view as PMPI_File_get_view
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_get_view(MPI_File fh, MPI_Offset *disp, MPI_Datatype *etype, MPI_Datatype *filetype,
-                      char *datarep) __attribute__((weak,alias("PMPI_File_get_view")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-#ifdef MPISGI
-#include "mpisgi2.h"
-#endif
-
-/*@
-    MPI_File_get_view - Returns the file view
-
-Input Parameters:
-. fh - file handle (handle)
-
-Output Parameters:
-. disp - displacement (nonnegative integer)
-. etype - elementary datatype (handle)
-. filetype - filetype (handle)
-. datarep - data representation (string)
-
-.N fortran
-@*/
-int MPI_File_get_view(MPI_File fh, MPI_Offset *disp, MPI_Datatype *etype,
-                      MPI_Datatype *filetype, char *datarep)
-{
-    int error_code;
-    ADIO_File adio_fh;
-    static char myname[] = "MPI_FILE_GET_VIEW";
-    int i, j, k, combiner;
-    MPI_Datatype copy_etype, copy_filetype;
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-
-    if (datarep == NULL)
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG, 
-					  "**iodatarepnomem", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    *disp = adio_fh->disp;
-    ADIOI_Strncpy(datarep, 
-	    (adio_fh->is_external32 ? "external32": "native"), MPI_MAX_DATAREP_STRING);
-
-    MPI_Type_get_envelope(adio_fh->etype, &i, &j, &k, &combiner);
-    if (combiner == MPI_COMBINER_NAMED) *etype = adio_fh->etype;
-    else {
-	/* FIXME: It is wrong to use MPI_Type_contiguous; the user could choose to
-	   re-implement MPI_Type_contiguous in an unexpected way.  Either use 
-	   MPIR_Barrier_impl as in MPICH or PMPI_Type_contiguous */
-        MPI_Type_contiguous(1, adio_fh->etype, &copy_etype);
-
-	/* FIXME: Ditto for MPI_Type_commit - use NMPI or PMPI */
-        MPI_Type_commit(&copy_etype);
-        *etype = copy_etype;
-    }
-    /* FIXME: Ditto for MPI_Type_xxx - use NMPI or PMPI */
-    MPI_Type_get_envelope(adio_fh->filetype, &i, &j, &k, &combiner);
-    if (combiner == MPI_COMBINER_NAMED) *filetype = adio_fh->filetype;
-    else {
-        MPI_Type_contiguous(1, adio_fh->filetype, &copy_filetype);
-
-        MPI_Type_commit(&copy_filetype);
-        *filetype = copy_filetype;
-    }
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-
-    return MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/mpi-io/glue/Makefile.mk b/3rd-party/romio321/mpi-io/glue/Makefile.mk
deleted file mode 100644
index 66f7f9efab9..00000000000
--- a/3rd-party/romio321/mpi-io/glue/Makefile.mk
+++ /dev/null
@@ -1,15 +0,0 @@
-## -*- Mode: Makefile; -*-
-## vim: set ft=automake :
-##
-## (C) 2011 by Argonne National Laboratory.
-##     See COPYRIGHT in top-level directory.
-##
-
-include $(top_srcdir)/mpi-io/glue/default/Makefile.mk
-include $(top_srcdir)/mpi-io/glue/mpich/Makefile.mk
-include $(top_srcdir)/mpi-io/glue/openmpi/Makefile.mk
-
-if !BUILD_ROMIO_EMBEDDED
-romio_other_sources += \
-	mpi-io/glue/large_count.c
-endif
diff --git a/3rd-party/romio321/mpi-io/glue/default/Makefile.mk b/3rd-party/romio321/mpi-io/glue/default/Makefile.mk
deleted file mode 100644
index 96995410b7a..00000000000
--- a/3rd-party/romio321/mpi-io/glue/default/Makefile.mk
+++ /dev/null
@@ -1,13 +0,0 @@
-## -*- Mode: Makefile; -*-
-## vim: set ft=automake :
-##
-## (C) 2011 by Argonne National Laboratory.
-##     See COPYRIGHT in top-level directory.
-##
-
-if MPIO_GLUE_DEFAULT
-romio_other_sources +=              \
-    mpi-io/glue/default/mpio_file.c \
-    mpi-io/glue/default/mpio_err.c
-endif MPIO_GLUE_DEFAULT
-
diff --git a/3rd-party/romio321/mpi-io/glue/default/mpio_err.c b/3rd-party/romio321/mpi-io/glue/default/mpio_err.c
deleted file mode 100644
index 880de9077a6..00000000000
--- a/3rd-party/romio321/mpi-io/glue/default/mpio_err.c
+++ /dev/null
@@ -1,88 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 2004 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-#include "adio_extern.h"
-
-#include <stdarg.h>
-#include <stdio.h>
-
-/* Default error handling implementation.
- *
- * Note that only MPI_ERRORS_ARE_FATAL and MPI_ERRORS_RETURN are
- * handled correctly; other handlers cause an abort.
- */
-
-int MPIO_Err_create_code(int lastcode, int fatal, const char fcname[],
-			 int line, int error_class, const char generic_msg[],
-			 const char specific_msg[], ... )
-{
-    va_list Argp;
-    int idx = 0;
-    char *buf;
-
-    buf = (char *) ADIOI_Malloc(1024);
-    if (buf != NULL) {
-	idx += ADIOI_Snprintf(buf, 1023, "%s (line %d): ", fcname, line);
-	if (specific_msg == NULL) {
-	    ADIOI_Snprintf(&buf[idx], 1023 - idx, "%s\n", generic_msg);
-	}
-	else {
-	    va_start(Argp, specific_msg);
-	    vsnprintf(&buf[idx], 1023 - idx, specific_msg, Argp);
-	    va_end(Argp);
-	}
-	FPRINTF(stderr, "%s", buf);
-	ADIOI_Free(buf);
-    }
-
-    return error_class;
-}
-
-int MPIO_Err_return_file(MPI_File mpi_fh, int error_code)
-{
-    ADIO_File adio_fh;
-
-    if (mpi_fh == MPI_FILE_NULL)
-    {
-	if (ADIOI_DFLT_ERR_HANDLER == MPI_ERRORS_ARE_FATAL ||
-	    ADIOI_DFLT_ERR_HANDLER != MPI_ERRORS_RETURN)
-	{
-	    MPI_Abort(MPI_COMM_WORLD, 1);
-	}
-	else
-	{
-	    return error_code;
-	}
-    }
-
-    adio_fh = MPIO_File_resolve(mpi_fh);
-
-    if (adio_fh->err_handler == MPI_ERRORS_ARE_FATAL ||
-	adio_fh->err_handler != MPI_ERRORS_RETURN)
-    {
-	MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-    else
-    {
-	return error_code;
-    }
-}
-
-int MPIO_Err_return_comm(MPI_Comm mpi_comm, int error_code)
-{
-    MPI_Errhandler errh;
-
-    MPI_Errhandler_get(mpi_comm, &errh);
-
-    if (errh == MPI_ERRORS_ARE_FATAL ||
-	errh != MPI_ERRORS_RETURN)
-    {
-	MPI_Abort(mpi_comm, 1);
-    }
-
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/glue/default/mpio_file.c b/3rd-party/romio321/mpi-io/glue/default/mpio_file.c
deleted file mode 100644
index 38222f77f28..00000000000
--- a/3rd-party/romio321/mpi-io/glue/default/mpio_file.c
+++ /dev/null
@@ -1,90 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2004 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-/* Hooks for allocation of MPI_File handles.
- *
- * Three functions are used in ROMIO for allocation/deallocation
- * of MPI_File structures:
- * - MPIO_File_create(size)
- * - MPIO_File_resolve(mpi_fh)
- * - MPIO_File_free(mpi_fh)
- *
- */
-
-MPI_File MPIO_File_create(int size)
-{
-    MPI_File mpi_fh;
-
-    mpi_fh = (MPI_File) ADIOI_Calloc(size,1);
-    return mpi_fh;
-}
-
-ADIO_File MPIO_File_resolve(MPI_File mpi_fh)
-{
-    return mpi_fh;
-}
-
-void MPIO_File_free(MPI_File *mpi_fh)
-{
-    ADIOI_Free(*mpi_fh);
-    *mpi_fh = MPI_FILE_NULL;
-}
-
-extern ADIO_File *ADIOI_Ftable;
-extern int ADIOI_Ftable_ptr;
-extern int ADIOI_Ftable_max;
-
-MPI_File MPIO_File_f2c(MPI_Fint fh)
-{
-#ifndef INT_LT_POINTER
-    return (MPI_File) ((void *) fh);  
-    /* the extra cast is to get rid of a compiler warning on Exemplar.
-       The warning is because MPI_File points to a structure containing
-       longlongs, which may be 8-byte aligned. But MPI_Fint itself
-       may not be 8-byte aligned.*/
-#else
-    if (!fh) return MPI_FILE_NULL;
-    if ((fh < 0) || (fh > ADIOI_Ftable_ptr)) {
-	FPRINTF(stderr, "MPI_File_f2c: Invalid file handle\n");
-	/* there is no way to return an error from MPI_File_f2c */
-	return MPI_FILE_NULL;
-    }
-    return ADIOI_Ftable[fh];
-#endif
-}
-
-MPI_Fint MPIO_File_c2f(MPI_File fh)
-{
-#ifndef INT_LT_POINTER
-    return (MPI_Fint) fh;
-#else
-    int i;
-
-    if ((fh == NULL) || (fh->cookie != ADIOI_FILE_COOKIE))
-	return (MPI_Fint) 0;
-    if (!ADIOI_Ftable) {
-	ADIOI_Ftable_max = 1024;
-	ADIOI_Ftable = (MPI_File *)
-	    ADIOI_Malloc(ADIOI_Ftable_max*sizeof(MPI_File)); 
-        ADIOI_Ftable_ptr = 0;  /* 0 can't be used though, because 
-                                  MPI_FILE_NULL=0 */
-	for (i=0; i<ADIOI_Ftable_max; i++) ADIOI_Ftable[i] = MPI_FILE_NULL;
-    }
-    if (ADIOI_Ftable_ptr == ADIOI_Ftable_max-1) {
-	ADIOI_Ftable = (MPI_File *) ADIOI_Realloc(ADIOI_Ftable, 
-                           (ADIOI_Ftable_max+1024)*sizeof(MPI_File));
-	for (i=ADIOI_Ftable_max; i<ADIOI_Ftable_max+1024; i++) 
-	    ADIOI_Ftable[i] = MPI_FILE_NULL;
-	ADIOI_Ftable_max += 1024;
-    }
-    ADIOI_Ftable_ptr++;
-    ADIOI_Ftable[ADIOI_Ftable_ptr] = fh;
-    return (MPI_Fint) ADIOI_Ftable_ptr;
-#endif
-}
diff --git a/3rd-party/romio321/mpi-io/glue/large_count.c b/3rd-party/romio321/mpi-io/glue/large_count.c
deleted file mode 100644
index ee31e2b746e..00000000000
--- a/3rd-party/romio321/mpi-io/glue/large_count.c
+++ /dev/null
@@ -1,26 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *   Copyright (C) 2013 University of Chicago.
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifndef HAVE_MPI_TYPE_SIZE_X
-int MPI_Type_size_x(MPI_Datatype datatype, MPI_Count *size)
-{
-    int size_int, ret;
-    ret = MPI_Type_size(datatype, &size_int);
-    *size = size_int;
-    return ret;
-}
-#endif
-
-#ifndef HAVE_MPI_STATUS_SET_ELEMENTS_X
-int MPI_Status_set_elements_x(MPI_Status *status, MPI_Datatype datatype,
-	MPI_Count count)
-{
-    int count_int = (int) count;
-    return MPI_Status_set_elements(status, datatype, count_int);
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/glue/mpich/Makefile.mk b/3rd-party/romio321/mpi-io/glue/mpich/Makefile.mk
deleted file mode 100644
index 4578670a319..00000000000
--- a/3rd-party/romio321/mpi-io/glue/mpich/Makefile.mk
+++ /dev/null
@@ -1,12 +0,0 @@
-## -*- Mode: Makefile; -*-
-## vim: set ft=automake :
-##
-## (C) 2011 by Argonne National Laboratory.
-##     See COPYRIGHT in top-level directory.
-##
-
-if MPIO_GLUE_MPICH
-romio_other_sources +=             \
-    mpi-io/glue/mpich/mpio_file.c \
-    mpi-io/glue/mpich/mpio_err.c
-endif MPIO_GLUE_MPICH
diff --git a/3rd-party/romio321/mpi-io/glue/mpich/mpio_err.c b/3rd-party/romio321/mpi-io/glue/mpich/mpio_err.c
deleted file mode 100644
index 520ba1ae805..00000000000
--- a/3rd-party/romio321/mpi-io/glue/mpich/mpio_err.c
+++ /dev/null
@@ -1,119 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 2004 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include <stdarg.h>
-#include <stdio.h>
-
-#include "mpioimpl.h"
-#include "adio_extern.h"
-
-/* MPICH error handling implementation */
-/* FIXME: These external prototypes should be included from 
-   mpich/src/include/mpiext.h */
-int MPIR_Err_create_code_valist(int, int, const char [], int, int, 
-				const char [], const char [], va_list );
-int MPIR_Err_is_fatal(int);
-
-void MPIR_Get_file_error_routine( MPI_Errhandler, 
-				  void (**)(MPI_File *, int *, ...), 
-				  int * );
-int MPIR_File_call_cxx_errhandler( MPI_File *, int *, 
-				   void (*)(MPI_File *, int *, ... ) );
-
-typedef int (* MPIR_Err_get_class_string_func_t)(int error, char *str, int length);
-void MPIR_Err_get_string( int, char *, int, MPIR_Err_get_class_string_func_t );
-
-struct MPID_Comm;
-int MPID_Abort(struct MPID_Comm *comm, int mpi_errno, int exit_code, const char *error_msg);
-
-int MPIO_Err_create_code(int lastcode, int fatal, const char fcname[],
-			 int line, int error_class, const char generic_msg[],
-			 const char specific_msg[], ... )
-{
-    va_list Argp;
-    int error_code;
-
-    va_start(Argp, specific_msg);
-
-    error_code = MPIR_Err_create_code_valist(lastcode, fatal, fcname, line,
-					     error_class, generic_msg,
-					     specific_msg, Argp);
-    
-    va_end(Argp);
-
-    return error_code;
-}
-
-int MPIO_Err_return_file(MPI_File mpi_fh, int error_code)
-{
-    MPI_Errhandler e;
-    void (*c_errhandler)(MPI_File *, int *, ... );
-    int  kind;   /* Error handler kind (see below) */
-    char error_msg[4096];
-    int len;
-
-    /* If the file pointer is not valid, we use the handler on
-       MPI_FILE_NULL (MPI-2, section 9.7).  For now, this code assumes that 
-       MPI_FILE_NULL has the default handler (return).  FIXME.  See
-       below - the set error handler uses ADIOI_DFLT_ERR_HANDLER; 
-    */
-
-    /* First, get the handler and the corresponding function */
-    if (mpi_fh == MPI_FILE_NULL) {
-	e = ADIOI_DFLT_ERR_HANDLER;
-    }
-    else {
-	ADIO_File fh;
-
-	fh = MPIO_File_resolve(mpi_fh);
-	e = fh->err_handler;
-    }
-
-    /* Actually, e is just the value provide by the MPICH routines
-       file_set_errhandler.  This is actually a *pointer* to the
-       errhandler structure.  We don't know that, so we ask
-       the MPICH code to translate this object into an error handler.
-       kind = 0: errors are fatal
-       kind = 1: errors return
-       kind = 2: errors call function
-    */
-    if (e == MPI_ERRORS_RETURN || e == MPIR_ERRORS_THROW_EXCEPTIONS || !e) {
-	/* FIXME: This is a hack in case no error handler was set */
-	kind = 1;
-	c_errhandler = 0;
-    }
-    else {
-	MPIR_Get_file_error_routine( e, &c_errhandler, &kind );
-    }
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (MPIR_Err_is_fatal(error_code) || kind == 0) 
-    {
-	ADIOI_Snprintf(error_msg, 4096, "I/O error: ");
-	len = (int)strlen(error_msg);
-	MPIR_Err_get_string(error_code, &error_msg[len], 4096-len, NULL);
-	MPID_Abort(NULL, MPI_SUCCESS, error_code, error_msg);
-    }
-    /* --END ERROR HANDLING-- */
-    else if (kind == 2) {
-	(*c_errhandler)( &mpi_fh, &error_code, 0 ); 
-    }
-    else if (kind == 3) {
-	MPIR_File_call_cxx_errhandler( &mpi_fh, &error_code, c_errhandler );
-    }
-
-    /* kind == 1 just returns */
-    return error_code;
-}
-
-int MPIO_Err_return_comm(MPI_Comm mpi_comm, int error_code)
-{
-    /* note: MPI calls inside the MPICH implementation are prefixed
-     * with an "N", indicating a nested call.
-     */
-    MPI_Comm_call_errhandler(mpi_comm, error_code);
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/glue/mpich/mpio_file.c b/3rd-party/romio321/mpi-io/glue/mpich/mpio_file.c
deleted file mode 100644
index a5e54a7b4bc..00000000000
--- a/3rd-party/romio321/mpi-io/glue/mpich/mpio_file.c
+++ /dev/null
@@ -1,92 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2004 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#include "adio_extern.h"
-
-/* Hooks for allocation of MPI_File handles.
- *
- * Three functions are used in ROMIO for allocation/deallocation
- * of MPI_File structures:
- * - MPIO_File_create(size)
- * - MPIO_File_resolve(mpi_fh)
- * - MPIO_File_free(mpi_fh)
- *
- */
-
-MPI_File MPIO_File_create(int size)
-{
-    MPI_File mpi_fh;
-
-    mpi_fh = (MPI_File) ADIOI_Calloc(size,1);
-    return mpi_fh;
-}
-
-ADIO_File MPIO_File_resolve(MPI_File mpi_fh)
-{
-    return mpi_fh;
-}
-
-void MPIO_File_free(MPI_File *mpi_fh)
-{
-    ADIOI_Free(*mpi_fh);
-    *mpi_fh = MPI_FILE_NULL;
-}
-
-MPI_File MPIO_File_f2c(MPI_Fint fh)
-{
-#ifndef INT_LT_POINTER
-    return (MPI_File) ((void *) fh);  
-    /* the extra cast is to get rid of a compiler warning on Exemplar.
-       The warning is because MPI_File points to a structure containing
-       longlongs, which may be 8-byte aligned. But MPI_Fint itself
-       may not be 8-byte aligned.*/
-#else
-    if (!fh) return MPI_FILE_NULL;
-    if ((fh < 0) || (fh > ADIOI_Ftable_ptr)) {
-	/* there is no way to return an error from MPI_File_f2c */
-	return MPI_FILE_NULL;
-    }
-    return ADIOI_Ftable[fh];
-#endif
-}
-
-MPI_Fint MPIO_File_c2f(MPI_File fh)
-{
-#ifndef INT_LT_POINTER
-    return (MPI_Fint) fh;
-#else
-    int i;
-
-    if ((fh == NULL) || (fh->cookie != ADIOI_FILE_COOKIE))
-	return (MPI_Fint) 0;
-
-    if (fh->fortran_handle != -1)
-	return fh->fortran_handle;
-
-    if (!ADIOI_Ftable) {
-	ADIOI_Ftable_max = 1024;
-	ADIOI_Ftable = (MPI_File *)
-	    ADIOI_Malloc(ADIOI_Ftable_max*sizeof(MPI_File)); 
-        ADIOI_Ftable_ptr = 0;  /* 0 can't be used though, because 
-                                  MPI_FILE_NULL=0 */
-	for (i=0; i<ADIOI_Ftable_max; i++) ADIOI_Ftable[i] = MPI_FILE_NULL;
-    }
-    if (ADIOI_Ftable_ptr == ADIOI_Ftable_max-1) {
-	ADIOI_Ftable = (MPI_File *) ADIOI_Realloc(ADIOI_Ftable, 
-                           (ADIOI_Ftable_max+1024)*sizeof(MPI_File));
-	for (i=ADIOI_Ftable_max; i<ADIOI_Ftable_max+1024; i++) 
-	    ADIOI_Ftable[i] = MPI_FILE_NULL;
-	ADIOI_Ftable_max += 1024;
-    }
-    ADIOI_Ftable_ptr++;
-    ADIOI_Ftable[ADIOI_Ftable_ptr] = fh;
-    fh->fortran_handle = ADIOI_Ftable_ptr;
-    return (MPI_Fint) ADIOI_Ftable_ptr;
-#endif
-}
diff --git a/3rd-party/romio321/mpi-io/glue/openmpi/Makefile.mk b/3rd-party/romio321/mpi-io/glue/openmpi/Makefile.mk
deleted file mode 100644
index ab7c15078e9..00000000000
--- a/3rd-party/romio321/mpi-io/glue/openmpi/Makefile.mk
+++ /dev/null
@@ -1,13 +0,0 @@
-## -*- Mode: Makefile; -*-
-## vim: set ft=automake :
-##
-## (C) 2011 by Argonne National Laboratory.
-##     See COPYRIGHT in top-level directory.
-##
-
-if MPIO_GLUE_OPENMPI
-romio_other_sources +=              \
-    mpi-io/glue/openmpi/mpio_file.c \
-    mpi-io/glue/openmpi/mpio_err.c
-endif MPIO_GLUE_OPENMPI
-
diff --git a/3rd-party/romio321/mpi-io/glue/openmpi/mpio_err.c b/3rd-party/romio321/mpi-io/glue/openmpi/mpio_err.c
deleted file mode 100644
index 2dddaee8edd..00000000000
--- a/3rd-party/romio321/mpi-io/glue/openmpi/mpio_err.c
+++ /dev/null
@@ -1,52 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/*
- *   Copyright (C) 2004 University of Chicago.
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include <stdarg.h>
-#include <stdio.h>
-
-#include "mpioimpl.h"
-#include "adio_extern.h"
-
-/* Default error handling implementation.
- *
- * Note that only MPI_ERRORS_ARE_FATAL and MPI_ERRORS_RETURN are
- * handled correctly; other handlers cause an abort.
- */
-
-int MPIO_Err_create_code(int lastcode, int fatal, const char fcname[],
-                         int line, int error_class, const char generic_msg[],
-                         const char specific_msg[], ...)
-{
-    va_list Argp;
-    int idx = 0;
-    char *buf;
-
-    buf = (char *) ADIOI_Malloc(1024);
-    if (buf != NULL) {
-        idx += ADIOI_Snprintf(buf, 1023, "%s (line %d): ", fcname, line);
-        if (specific_msg == NULL) {
-            ADIOI_Snprintf(&buf[idx], 1023 - idx, "%s\n", generic_msg);
-        }
-        else {
-            va_start(Argp, specific_msg);
-            vsnprintf(&buf[idx], 1023 - idx, specific_msg, Argp);
-            va_end(Argp);
-        }
-        ADIOI_Free(buf);
-    }
-
-    return error_class;
-}
-
-int MPIO_Err_return_file(MPI_File mpi_fh, int error_code)
-{
-    return error_code;
-}
-
-int MPIO_Err_return_comm(MPI_Comm mpi_comm, int error_code)
-{
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/glue/openmpi/mpio_file.c b/3rd-party/romio321/mpi-io/glue/openmpi/mpio_file.c
deleted file mode 100644
index f0b6d511889..00000000000
--- a/3rd-party/romio321/mpi-io/glue/openmpi/mpio_file.c
+++ /dev/null
@@ -1,95 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/*
- *
- *   Copyright (C) 2004 University of Chicago.
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#include "adio_extern.h"
-
-/* Hooks for allocation of MPI_File handles.
- *
- * Three functions are used in ROMIO for allocation/deallocation
- * of MPI_File structures:
- * - MPIO_File_create(size)
- * - MPIO_File_resolve(mpi_fh)
- * - MPIO_File_free(mpi_fh)
- *
- */
-
-MPI_File MPIO_File_create(int size)
-{
-    MPI_File mpi_fh;
-
-    mpi_fh = (MPI_File) ADIOI_Calloc(size, 1);
-    return mpi_fh;
-}
-
-ADIO_File MPIO_File_resolve(MPI_File mpi_fh)
-{
-    return mpi_fh;
-}
-
-void MPIO_File_free(MPI_File * mpi_fh)
-{
-    ADIOI_Free(*mpi_fh);
-    *mpi_fh = MPI_FILE_NULL;
-}
-
-/* these functions aren't needed with the way Open MPI uses ROMIO */
-#if 0
-
-
-MPI_File MPIO_File_f2c(MPI_Fint fh)
-{
-#ifndef INT_LT_POINTER
-    return (MPI_File) ((void *) fh);
-    /* the extra cast is to get rid of a compiler warning on Exemplar.
-     * The warning is because MPI_File points to a structure containing
-     * longlongs, which may be 8-byte aligned. But MPI_Fint itself
-     * may not be 8-byte aligned. */
-#else
-    if (!fh)
-        return MPI_FILE_NULL;
-    if ((fh < 0) || (fh > ADIOI_Ftable_ptr)) {
-        FPRINTF(stderr, "MPI_File_f2c: Invalid file handle\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-    return ADIOI_Ftable[fh];
-#endif
-}
-
-MPI_Fint MPIO_File_c2f(MPI_File fh)
-{
-#ifndef INT_LT_POINTER
-    return (MPI_Fint) fh;
-#else
-    int i;
-
-    if ((fh == NULL) || (fh->cookie != ADIOI_FILE_COOKIE))
-        return (MPI_Fint) 0;
-    if (!ADIOI_Ftable) {
-        ADIOI_Ftable_max = 1024;
-        ADIOI_Ftable = (MPI_File *)
-            ADIOI_Malloc(ADIOI_Ftable_max * sizeof(MPI_File));
-        ADIOI_Ftable_ptr = 0;   /* 0 can't be used though, because
-                                 * MPI_FILE_NULL=0 */
-        for (i = 0; i < ADIOI_Ftable_max; i++)
-            ADIOI_Ftable[i] = MPI_FILE_NULL;
-    }
-    if (ADIOI_Ftable_ptr == ADIOI_Ftable_max - 1) {
-        ADIOI_Ftable = (MPI_File *) ADIOI_Realloc(ADIOI_Ftable,
-                                                  (ADIOI_Ftable_max + 1024) * sizeof(MPI_File));
-        for (i = ADIOI_Ftable_max; i < ADIOI_Ftable_max + 1024; i++)
-            ADIOI_Ftable[i] = MPI_FILE_NULL;
-        ADIOI_Ftable_max += 1024;
-    }
-    ADIOI_Ftable_ptr++;
-    ADIOI_Ftable[ADIOI_Ftable_ptr] = fh;
-    return (MPI_Fint) ADIOI_Ftable_ptr;
-#endif
-}
-
-#endif /* #if 0 */
diff --git a/3rd-party/romio321/mpi-io/ioreq_c2f.c b/3rd-party/romio321/mpi-io/ioreq_c2f.c
deleted file mode 100644
index d49ebcd2b94..00000000000
--- a/3rd-party/romio321/mpi-io/ioreq_c2f.c
+++ /dev/null
@@ -1,82 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPIO_Request_c2f = PMPIO_Request_c2f
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPIO_Request_c2f MPIO_Request_c2f
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPIO_Request_c2f as PMPIO_Request_c2f
-/* end of weak pragmas */
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-#include "adio_extern.h"
-
-/*@
-    MPIO_Request_c2f - Translates a C I/O-request handle to a 
-                       Fortran I/O-request handle
-
-Input Parameters:
-. request - C I/O-request handle (handle)
-
-Return Value:
-  Fortran I/O-request handle (integer)
-@*/
-#ifdef HAVE_MPI_GREQUEST
-MPI_Fint MPIO_Request_c2f(MPIO_Request request)
-{
-    return ((MPI_Fint)request);
-}
-#else
-
-MPI_Fint MPIO_Request_c2f(MPIO_Request request)
-{
-#ifndef INT_LT_POINTER
-    return (MPI_Fint) request;
-#else
-    int i;
-    MPID_THREADPRIV_DECL;
-
-    /* We can make this test outside of the GLOBAL mutex because it does
-       not access any shared data */
-    if ((request <= (MPIO_Request) 0) || (request->cookie != ADIOI_REQ_COOKIE))
-    {
-	    return (MPI_Fint) 0;
-    }
-
-    ROMIO_THREAD_CS_ENTER();
-    if (!ADIOI_Reqtable) {
-	ADIOI_Reqtable_max = 1024;
-	ADIOI_Reqtable = (MPIO_Request *)
-	    ADIOI_Malloc(ADIOI_Reqtable_max*sizeof(MPIO_Request)); 
-        ADIOI_Reqtable_ptr = 0;  /* 0 can't be used though, because 
-                                  MPIO_REQUEST_NULL=0 */
-	for (i=0; i<ADIOI_Reqtable_max; i++) ADIOI_Reqtable[i] = MPIO_REQUEST_NULL;
-    }
-    if (ADIOI_Reqtable_ptr == ADIOI_Reqtable_max-1) {
-	ADIOI_Reqtable = (MPIO_Request *) ADIOI_Realloc(ADIOI_Reqtable, 
-                           (ADIOI_Reqtable_max+1024)*sizeof(MPIO_Request));
-	for (i=ADIOI_Reqtable_max; i<ADIOI_Reqtable_max+1024; i++) 
-	    ADIOI_Reqtable[i] = MPIO_REQUEST_NULL;
-	ADIOI_Reqtable_max += 1024;
-    }
-    ADIOI_Reqtable_ptr++;
-    ADIOI_Reqtable[ADIOI_Reqtable_ptr] = request;
-
-    ROMIO_THREAD_CS_EXIT();
-    return (MPI_Fint) ADIOI_Reqtable_ptr;
-#endif
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/ioreq_f2c.c b/3rd-party/romio321/mpi-io/ioreq_f2c.c
deleted file mode 100644
index a58e4a2e282..00000000000
--- a/3rd-party/romio321/mpi-io/ioreq_f2c.c
+++ /dev/null
@@ -1,72 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPIO_Request_f2c = PMPIO_Request_f2c
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPIO_Request_f2c MPIO_Request_f2c
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPIO_Request_f2c as PMPIO_Request_f2c
-/* end of weak pragmas */
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-#include "adio_extern.h"
-
-/*@
-    MPIO_Request_f2c - Translates a Fortran I/O-request handle to 
-                       a C I/O-request handle
-
-Input Parameters:
-. request - Fortran I/O-request handle (integer)
-
-Return Value:
-  C I/O-request handle (handle)
-@*/
-#ifdef HAVE_MPI_GREQUEST
-MPIO_Request MPIO_Request_f2c(MPI_Fint request) {
-    return((MPIO_Request) request);
-}
-#else
-MPIO_Request MPIO_Request_f2c(MPI_Fint request)
-{
-    int error_code;
-    static char myname[] = "MPIO_REQUEST_F2C";
-    MPID_THREADPRIV_DECL;
-
-#ifndef INT_LT_POINTER
-    return (MPIO_Request) request;
-#else
-
-    ROMIO_THREAD_CS_ENTER();
-    
-    if (!request) {
-	return MPIO_REQUEST_NULL;
-    }
-    /* --BEGIN ERROR HANDLING-- */
-    if ((request < 0) || (request > ADIOI_Reqtable_ptr)) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_REQUEST,
-					  "**request", 0);
-	error_code = MPIO_Err_return_file(MPI_FILE_NULL, error_code);
-	return MPIO_REQUEST_NULL;
-    }
-    /* --END ERROR HANDLING-- */
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-    return ADIOI_Reqtable[request];
-#endif
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/iotest.c b/3rd-party/romio321/mpi-io/iotest.c
deleted file mode 100644
index 42bc8a3ae5e..00000000000
--- a/3rd-party/romio321/mpi-io/iotest.c
+++ /dev/null
@@ -1,95 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPIO_Test = PMPIO_Test
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPIO_Test MPIO_Test
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPIO_Test as PMPIO_Test
-/* end of weak pragmas */
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/* status object not filled currently */
-
-/*@
-    MPIO_Test - Test the completion of a nonblocking read or write
-                
-Input Parameters:
-. request - request object (handle)
-
-Output Parameters:
-. flag - true if operation completed (logical)
-. status - status object (Status)
-
-.N fortran
-@*/
-#ifdef HAVE_MPI_GREQUEST
-int MPIO_Test(MPIO_Request *request, int *flag, MPI_Status *status)
-{
-	return (MPI_Test(request, flag, status));
-}
-#else
-int MPIO_Test(MPIO_Request *request, int *flag, MPI_Status *status)
-{
-    int error_code;
-    static char myname[] = "MPIO_TEST";
-    MPID_THREADPRIV_DECL;
-#ifdef MPI_hpux
-    int fl_xmpi;
-
-    if (*request != MPIO_REQUEST_NULL) {
-	HPMP_IO_WSTART(fl_xmpi, BLKMPIOTEST, TRDTSYSTEM, (*request)->fd);
-    }
-#endif /* MPI_hpux */
-
-    ROMIO_THREAD_CS_ENTER();
-
-    if (*request == MPIO_REQUEST_NULL) {
-	    error_code = MPI_SUCCESS;
-	    goto fn_exit;
-    }
-
-    /* --BEGIN ERROR HANDLING-- */
-    if ((*request < (MPIO_Request) 0) || 
-	((*request)->cookie != ADIOI_REQ_COOKIE))
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_REQUEST,
-					  "**request", 0);
-	error_code = MPIO_Err_return_file(MPI_FILE_NULL, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    switch ((*request)->optype) {
-    case ADIOI_READ:
-        *flag = ADIO_ReadDone(request, status, &error_code);
-        break;
-    case ADIOI_WRITE:
-        *flag = ADIO_WriteDone(request, status, &error_code);
-        break;
-    }
-
-#ifdef MPI_hpux
-    HPMP_IO_WEND(fl_xmpi);
-#endif /* MPI_hpux */
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-    return error_code;
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/iotestall.c b/3rd-party/romio321/mpi-io/iotestall.c
deleted file mode 100644
index 24dd8b2bc58..00000000000
--- a/3rd-party/romio321/mpi-io/iotestall.c
+++ /dev/null
@@ -1,77 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2003 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPIO_Testall = PMPIO_Testall
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPIO_Testall MPIO_Testall
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPIO_Testall as PMPIO_Testall
-/* end of weak pragmas */
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*
-  This is a temporary function until we switch to using MPI-2's generalized
-  requests.
-*/
-
-int MPIO_Testall(int count, MPIO_Request requests[], int *flag,
-		 MPI_Status statuses[])
-{
-    int done, i, err; 
-    MPID_THREADPRIV_DECL;
-
-    ROMIO_THREAD_CS_ENTER();
-    if (count == 1)  {
-	    err = MPIO_Test( requests, flag, statuses );
-	    goto fn_exit;
-    }
-
-    /* This is actually very difficult to do.  We can't use MPIO_Test, 
-       since we must change the requests only if *ALL* requests are complete
-    */
-    /* FIXME: THIS IS NOT CORRECT (see above).  But most applications won't 
-     care */
-    done = 1;
-    for (i=0; i<count; i++) {
-      if (requests[i] != MPIO_REQUEST_NULL) {
-	err = MPIO_Test( &requests[i], flag, &statuses[i] );
-	if (!*flag) done = 0;
-	if (err) goto fn_exit;
-      }
-      else {
-#ifdef MPICH
-	  /* need to set empty status */
-	  if (statuses != MPI_STATUSES_IGNORE) {
-	      statuses[i].MPI_SOURCE = MPI_ANY_SOURCE;
-	      statuses[i].MPI_TAG    = MPI_ANY_TAG;
-              MPIR_STATUS_SET_COUNT(statuses[i], 0);
-              MPIR_STATUS_SET_CANCEL_BIT(statuses[i], 0);
-	  }
-#else
-	  ;
-#endif
-      }
-    }
-    
-    *flag = done;
-
-    err = MPI_SUCCESS;
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-    return err;
-}
-
diff --git a/3rd-party/romio321/mpi-io/iotestany.c b/3rd-party/romio321/mpi-io/iotestany.c
deleted file mode 100644
index eac6197ae4e..00000000000
--- a/3rd-party/romio321/mpi-io/iotestany.c
+++ /dev/null
@@ -1,81 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2003 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPIO_Testany = PMPIO_Testany
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPIO_Testany MPIO_Testany
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPIO_Testany as PMPIO_Testany
-/* end of weak pragmas */
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*
-  This is a temporary function until we switch to using MPI-2's generalized
-  requests.
-*/
-
-int MPIO_Testany(int count, MPIO_Request requests[], int *index, 
-		 int *flag, MPI_Status *status)
-{
-    int i, err; 
-    MPID_THREADPRIV_DECL;
-
-    ROMIO_THREAD_CS_ENTER();
-
-    if (count == 1) {
-	err = MPIO_Test( requests, flag, status );
-	if (!err) *index = 0;
-	goto fn_exit;
-    }
-
-    /* Check for no active requests */
-    for (i=0; i<count; i++) {
-	if (requests[i] != MPIO_REQUEST_NULL) {
-	    break;
-	}
-    }
-    if (i == count) {
-	*index = MPI_UNDEFINED;
-#ifdef MPICH
-	/* need to set empty status */
-	if (status != MPI_STATUS_IGNORE) {
-	    status->MPI_SOURCE = MPI_ANY_SOURCE;
-	    status->MPI_TAG    = MPI_ANY_TAG;
-            MPIR_STATUS_SET_COUNT(*status, 0);
-            MPIR_STATUS_SET_CANCEL_BIT(*status, 0);
-	}
-#endif
-	err = MPI_SUCCESS;
-	goto fn_exit;
-    }
-
-    err = MPI_SUCCESS;
-    for (i=0; i<count; i++) {
-      if (requests[i] != MPIO_REQUEST_NULL) {
-	err = MPIO_Test( &requests[i], flag, status );
-	if (*flag) {
-	  if (!err) *index = i;
-	  break;
-	}
-      }
-    }
-
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-    return err;
-}
diff --git a/3rd-party/romio321/mpi-io/iotestsome.c b/3rd-party/romio321/mpi-io/iotestsome.c
deleted file mode 100644
index a7d7d4c949f..00000000000
--- a/3rd-party/romio321/mpi-io/iotestsome.c
+++ /dev/null
@@ -1,86 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2003 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPIO_Testsome = PMPIO_Testsome
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPIO_Testsome MPIO_Testsome
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPIO_Testsome as PMPIO_Testsome
-/* end of weak pragmas */
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*
-  This is a temporary function until we switch to using MPI-2's generalized
-  requests.
-*/
-
-int MPIO_Testsome(int count, MPIO_Request requests[], int *outcount,
-		  int indices[], MPI_Status *statuses)
-{
-    int i, err; 
-    int flag;
-    MPID_THREADPRIV_DECL;
-
-    ROMIO_THREAD_CS_ENTER();
-
-    if (count == 1) {
-	err = MPIO_Test( requests, &flag, statuses );
-	if (!err) {
-	    if (flag) {
-		indices[0] = 0;
-		*outcount = 1;
-	    }
-	    else {
-		*outcount = 0;
-	    }
-	}
-	goto fn_exit;
-    }
-
-    /* Check for no active requests */
-    for (i=0; i<count; i++) {
-	if (requests[i] != MPIO_REQUEST_NULL) {
-	    break;
-	}
-    }
-    if (i == count) {
-	*outcount = MPI_UNDEFINED;
-	err = MPI_SUCCESS;
-	goto fn_exit;
-    }
-
-    err = MPI_SUCCESS;
-    *outcount = 0;
-    for (i=0; i<count; i++) {
-      if (requests[i] != MPIO_REQUEST_NULL) {
-	err = MPIO_Test( &requests[i], &flag, statuses );
-	if (flag) {
-	  if (!err) {
-	      indices[0] = i;
-	      indices++;
-	      statuses++;
-	      *outcount = *outcount + 1;
-	  }
-	}
-      }
-    }
-
-fn_exit:
-
-    ROMIO_THREAD_CS_EXIT();
-    return err;
-}
diff --git a/3rd-party/romio321/mpi-io/iowait.c b/3rd-party/romio321/mpi-io/iowait.c
deleted file mode 100644
index 4d383c8cd81..00000000000
--- a/3rd-party/romio321/mpi-io/iowait.c
+++ /dev/null
@@ -1,96 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPIO_Wait = PMPIO_Wait
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPIO_Wait MPIO_Wait
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPIO_Wait as PMPIO_Wait
-/* end of weak pragmas */
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/* status object not filled currently */
-
-/*@
-    MPIO_Wait - Waits for the completion of a nonblocking read or write
-
-Input Parameters:
-. request - request object (handle)
-
-Output Parameters:
-. status - status object (Status)
-
-.N fortran
-@*/
-#ifdef HAVE_MPI_GREQUEST
-int MPIO_Wait(MPIO_Request *request, MPI_Status *status)
-{
-	return(MPI_Wait(request, status));
-}
-#else
-int MPIO_Wait(MPIO_Request *request, MPI_Status *status)
-{
-    int error_code;
-    static char myname[] = "MPIO_WAIT";
-    MPID_THREADPRIV_DECL;
-
-#ifdef MPI_hpux
-    int fl_xmpi;
-
-    if (*request != MPIO_REQUEST_NULL) {
-	HPMP_IO_WSTART(fl_xmpi, BLKMPIOWAIT, TRDTBLOCK, (*request)->fd);
-    }
-#endif /* MPI_hpux */
-
-    ROMIO_THREAD_CS_ENTER();
-
-    if (*request == MPIO_REQUEST_NULL) {
-	    error_code = MPI_SUCCESS;
-	    goto fn_exit;
-    }
-
-
-    /* --BEGIN ERROR HANDLING-- */
-    if ((*request < (MPIO_Request) 0) ||
-	((*request)->cookie != ADIOI_REQ_COOKIE))
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_REQUEST,
-					  "**request", 0);
-	error_code = MPIO_Err_return_file(MPI_FILE_NULL, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    switch ((*request)->optype) {
-    case ADIOI_READ:
-        ADIO_ReadComplete(request, status, &error_code);
-        break;
-    case ADIOI_WRITE:
-        ADIO_WriteComplete(request, status, &error_code);
-        break;
-    }
-
-#ifdef MPI_hpux
-    HPMP_IO_WEND(fl_xmpi);
-#endif /* MPI_hpux */
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-    return error_code;
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/iowaitall.c b/3rd-party/romio321/mpi-io/iowaitall.c
deleted file mode 100644
index 3e5e5b89da5..00000000000
--- a/3rd-party/romio321/mpi-io/iowaitall.c
+++ /dev/null
@@ -1,74 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2003 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPIO_Waitall = PMPIO_Waitall
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPIO_Waitall MPIO_Waitall
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPIO_Waitall as PMPIO_Waitall
-/* end of weak pragmas */
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*
-  This is a temporary function until we switch to using MPI-2's generalized
-  requests.
-*/
-
-int MPIO_Waitall( int count, MPIO_Request requests[], MPI_Status statuses[] )
-{
-    int notdone, i, flag, err; 
-    MPID_THREADPRIV_DECL;
-
-    ROMIO_THREAD_CS_ENTER();
-
-    if (count == 1)  {
-	    err = MPIO_Wait(requests, statuses);
-	    goto fn_exit;
-    }
-    
-    
-    do {
-	notdone = 0;
-	for (i=0; i<count; i++) {
-	    if (requests[i] != MPIO_REQUEST_NULL) {
-		err = MPIO_Test( &requests[i], &flag, &statuses[i] );
-		if (!flag) notdone = 1;
-		if (err) goto fn_exit;
-	    }
-	    else {
-#ifdef MPICH
-		/* need to set empty status */
-		if (statuses != MPI_STATUSES_IGNORE) {
-		    statuses[i].MPI_SOURCE = MPI_ANY_SOURCE;
-		    statuses[i].MPI_TAG    = MPI_ANY_TAG;
-                    MPIR_STATUS_SET_COUNT(statuses[i], 0);
-                    MPIR_STATUS_SET_CANCEL_BIT(statuses[i], 0);
-		}
-#else
-		;
-#endif
-	    }
-	}
-    } while (notdone);
-
-    err = MPI_SUCCESS;
-fn_exit:
-
-    ROMIO_THREAD_CS_EXIT();
-    return err;
-}
-
diff --git a/3rd-party/romio321/mpi-io/iowaitany.c b/3rd-party/romio321/mpi-io/iowaitany.c
deleted file mode 100644
index 747aa5bc48f..00000000000
--- a/3rd-party/romio321/mpi-io/iowaitany.c
+++ /dev/null
@@ -1,84 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2003 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPIO_Waitany = PMPIO_Waitany
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPIO_Waitany MPIO_Waitany
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPIO_Waitany as PMPIO_Waitany
-/* end of weak pragmas */
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*
-  This is a temporary function until we switch to using MPI-2's generalized
-  requests.
-*/
-
-int MPIO_Waitany(int count, MPIO_Request requests[], int *index, 
-		 MPI_Status *status)
-{
-    int i, flag, err; 
-    MPID_THREADPRIV_DECL;
-
-    ROMIO_THREAD_CS_ENTER();
-
-    if (count == 1) {
-	err = MPIO_Wait( requests, status );
-	if (!err) *index = 0;
-	goto fn_exit;
-    }
-
-    /* Check for no active requests */
-    for (i=0; i<count; i++) {
-	if (requests[i] != MPIO_REQUEST_NULL) {
-	    break;
-	}
-    }
-    if (i == count) {
-	*index = MPI_UNDEFINED;
-#ifdef MPICH
-	/* need to set empty status */
-	if (status != MPI_STATUS_IGNORE) {
-	    status->MPI_SOURCE = MPI_ANY_SOURCE;
-	    status->MPI_TAG    = MPI_ANY_TAG;
-            MPIR_STATUS_SET_COUNT(*status, 0);
-            MPIR_STATUS_SET_CANCEL_BIT(*status, 0);
-	}
-#endif
-	err = MPI_SUCCESS;
-	goto fn_exit;
-    }
-
-    err = MPI_SUCCESS;
-    do {
-	flag = 0;
-	for (i=0; i<count; i++) {
-	    if (requests[i] != MPIO_REQUEST_NULL) {
-		err = MPIO_Test( &requests[i], &flag, status );
-		if (flag) {
-		    if (!err) *index = i;
-		    break;
-		}
-	    }
-	}
-    } while (flag == 0);
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-
-    return err;
-}
diff --git a/3rd-party/romio321/mpi-io/iowaitsome.c b/3rd-party/romio321/mpi-io/iowaitsome.c
deleted file mode 100644
index 1bca9c661be..00000000000
--- a/3rd-party/romio321/mpi-io/iowaitsome.c
+++ /dev/null
@@ -1,81 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 2003 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPIO_Waitsome = PMPIO_Waitsome
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPIO_Waitsome MPIO_Waitsome
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPIO_Waitsome as PMPIO_Waitsome
-/* end of weak pragmas */
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*
-  This is a temporary function until we switch to using MPI-2's generalized
-  requests.
-*/
-
-int MPIO_Waitsome(int count, MPIO_Request requests[], int *outcount,
-		  int indices[], MPI_Status *statuses)
-{
-    int i, flag, err; 
-    MPID_THREADPRIV_DECL;
-
-    ROMIO_THREAD_CS_ENTER();
-
-    if (count == 1) {
-	err = MPIO_Wait( requests, statuses );
-	if (!err) {
-	    *outcount = 1;
-	    indices[0] = 0;
-	}
-	goto fn_exit;
-    }
-
-    /* Check for no active requests */
-    for (i=0; i<count; i++) {
-	if (requests[i] != MPIO_REQUEST_NULL) {
-	    break;
-	}
-    }
-    if (i == count) {
-	*outcount = MPI_UNDEFINED;
-	err = MPI_SUCCESS;
-	goto fn_exit;
-    }
-
-    err = MPI_SUCCESS;
-    *outcount = 0;
-    do {
-	for (i=0; i<count; i++) {
-	    if (requests[i] != MPIO_REQUEST_NULL) {
-		err = MPIO_Test( &requests[i], &flag, statuses );
-		if (flag) {
-		    if (!err) {
-			indices[0] = i;
-			indices++;
-			statuses++;
-			*outcount = *outcount + 1;
-		    }
-		}
-	    }
-	}
-    } while (*outcount == 0);
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-    return err;
-}
diff --git a/3rd-party/romio321/mpi-io/iread.c b/3rd-party/romio321/mpi-io/iread.c
deleted file mode 100644
index 4b23a11dee0..00000000000
--- a/3rd-party/romio321/mpi-io/iread.c
+++ /dev/null
@@ -1,161 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_iread = PMPI_File_iread
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_iread MPI_File_iread
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_iread as PMPI_File_iread
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_iread(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPIO_Request *request)
-    __attribute__((weak,alias("PMPI_File_iread")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-#ifdef HAVE_MPI_GREQUEST
-#include "mpiu_greq.h"
-#endif
-
-/*@
-    MPI_File_iread - Nonblocking read using individual file pointer
-
-Input Parameters:
-. fh - file handle (handle)
-. count - number of elements in buffer (nonnegative integer)
-. datatype - datatype of each buffer element (handle)
-
-Output Parameters:
-. buf - initial address of buffer (choice)
-. request - request object (handle)
-
-.N fortran
-@*/
-int MPI_File_iread(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Request *request)
-{
-    int error_code=MPI_SUCCESS;
-    static char myname[] = "MPI_FILE_IREAD";
-#ifdef MPI_hpux
-    int fl_xmpi;
-
-    HPMP_IO_START(fl_xmpi, BLKMPIFILEIREAD, TRDTSYSTEM, fh, datatype,
-		  count);
-#endif /* MPI_hpux */
-
-
-    error_code = MPIOI_File_iread(fh, (MPI_Offset) 0, ADIO_INDIVIDUAL,
-				  buf, count, datatype, myname, request);
-    
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-	error_code = MPIO_Err_return_file(fh, error_code);
-    /* --END ERROR HANDLING-- */
-
-#ifdef MPI_hpux
-    HPMP_IO_END(fl_xmpi, fh, datatype, count);
-#endif /* MPI_hpux */
-
-    return error_code;
-}
-
-/* prevent multiple definitions of this routine */
-#ifdef MPIO_BUILD_PROFILING
-int MPIOI_File_iread(MPI_File fh, MPI_Offset offset, int file_ptr_type, void *buf, int count,
-		     MPI_Datatype datatype, char *myname, MPI_Request *request)
-{
-    int error_code, buftype_is_contig, filetype_is_contig;
-    MPI_Count datatype_size;
-    ADIO_Status status;
-    ADIO_File adio_fh;
-    ADIO_Offset off, bufsize;
-    MPI_Offset nbytes=0;
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    MPIO_CHECK_COUNT(adio_fh, count, myname, error_code);
-    MPIO_CHECK_DATATYPE(adio_fh, datatype, myname, error_code);
-
-    if (file_ptr_type == ADIO_EXPLICIT_OFFSET && offset < 0) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "**iobadoffset", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    MPI_Type_size_x(datatype, &datatype_size);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_INTEGRAL_ETYPE(adio_fh, count, datatype_size, myname, error_code);
-    MPIO_CHECK_READABLE(adio_fh, myname, error_code);
-    MPIO_CHECK_NOT_SEQUENTIAL_MODE(adio_fh, myname, error_code);
-    MPIO_CHECK_COUNT_SIZE(adio_fh, count, datatype_size, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-    ADIOI_Datatype_iscontig(adio_fh->filetype, &filetype_is_contig);
-
-    ADIOI_TEST_DEFERRED(adio_fh, myname, &error_code);
-
-    if (buftype_is_contig && filetype_is_contig) {
-	/* convert count and offset to bytes */
-	bufsize = datatype_size * count;
-
-	if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-	    off = adio_fh->disp + adio_fh->etype_size * offset;
-	}
-	else {
-	    off = adio_fh->fp_ind;
-	}
-
-        if (!(adio_fh->atomicity))
-	    ADIO_IreadContig(adio_fh, buf, count, datatype, file_ptr_type,
-			off, request, &error_code); 
-        else {
-            /* to maintain strict atomicity semantics with other concurrent
-              operations, lock (exclusive) and call blocking routine */
-	    if (ADIO_Feature(adio_fh, ADIO_LOCKS))
-	    {
-                ADIOI_WRITE_LOCK(adio_fh, off, SEEK_SET, bufsize);
-	    }
-
-            ADIO_ReadContig(adio_fh, buf, count, datatype, file_ptr_type,
-			    off, &status, &error_code);
-
-	    if (ADIO_Feature(adio_fh, ADIO_LOCKS))
-	    {
-                ADIOI_UNLOCK(adio_fh, off, SEEK_SET, bufsize);
-	    }
-	    if (error_code == MPI_SUCCESS) {
-		nbytes = count*datatype_size;
-	    }
-	    MPIO_Completed_request_create(&adio_fh, nbytes, &error_code, request);
-        }
-    }
-    else ADIO_IreadStrided(adio_fh, buf, count, datatype, file_ptr_type,
-			   offset, request, &error_code); 
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-
-    return error_code;
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/iread_all.c b/3rd-party/romio321/mpi-io/iread_all.c
deleted file mode 100644
index 00a0640f5a4..00000000000
--- a/3rd-party/romio321/mpi-io/iread_all.c
+++ /dev/null
@@ -1,149 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *  (C) 2014 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_iread_all = PMPI_File_iread_all
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_iread_all MPI_File_iread_all
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_iread_all as PMPI_File_iread_all
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_iread_all(MPI_File fh, void *buf, int count, MPI_Datatype datatype,
-                       MPI_Request *request)
-    __attribute__((weak,alias("PMPI_File_iread_all")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-#if HAVE_MPI_GREQUEST
-#include "mpiu_greq.h"
-#endif
-
-/*@
-    MPI_File_iread_all - Nonblocking collective read using individual file pointer
-
-Input Parameters:
-. fh - file handle (handle)
-. count - number of elements in buffer (nonnegative integer)
-. datatype - datatype of each buffer element (handle)
-
-Output Parameters:
-. buf - initial address of buffer (choice)
-. request - request object (handle)
-
-.N fortran
-@*/
-int MPI_File_iread_all(MPI_File fh, void *buf, int count,
-                       MPI_Datatype datatype, MPI_Request *request)
-{
-    int error_code;
-    static char myname[] = "MPI_FILE_IREAD_ALL";
-#ifdef MPI_hpux
-    int fl_xmpi;
-
-    HPMP_IO_START(fl_xmpi, BLKMPIFILEREADALL, TRDTBLOCK, fh, datatype, count);
-#endif /* MPI_hpux */
-
-    error_code = MPIOI_File_iread_all(fh, (MPI_Offset)0,
-                     ADIO_INDIVIDUAL, buf,
-                     count, datatype, myname, request);
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS) {
-        error_code = MPIO_Err_return_file(fh, error_code);
-    }
-    /* --END ERROR HANDLING-- */
-
-#ifdef MPI_hpux
-    HPMP_IO_END(fl_xmpi, fh, datatype, count);
-#endif /* MPI_hpux */
-
-    return error_code;
-}
-
-/* Note: MPIOI_File_iread_all also used by MPI_File_iread_at_all */
-/* prevent multiple definitions of this routine */
-#ifdef MPIO_BUILD_PROFILING
-int MPIOI_File_iread_all(MPI_File fh,
-            MPI_Offset offset,
-            int file_ptr_type,
-            void *buf,
-            int count,
-            MPI_Datatype datatype,
-            char *myname,
-            MPI_Request *request)
-{
-    int error_code;
-    MPI_Count datatype_size;
-    ADIO_File adio_fh;
-    void *xbuf=NULL, *e32_buf=NULL;
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    MPIO_CHECK_COUNT(adio_fh, count, myname, error_code);
-    MPIO_CHECK_DATATYPE(adio_fh, datatype, myname, error_code);
-
-    if (file_ptr_type == ADIO_EXPLICIT_OFFSET && offset < 0) {
-        error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-                                          myname, __LINE__, MPI_ERR_ARG,
-                                          "**iobadoffset", 0);
-        error_code = MPIO_Err_return_file(adio_fh, error_code);
-        goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    MPI_Type_size_x(datatype, &datatype_size);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_INTEGRAL_ETYPE(adio_fh, count, datatype_size, myname, error_code);
-    MPIO_CHECK_READABLE(adio_fh, myname, error_code);
-    MPIO_CHECK_NOT_SEQUENTIAL_MODE(adio_fh, myname, error_code);
-    MPIO_CHECK_COUNT_SIZE(adio_fh, count, datatype_size, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    xbuf = buf;
-    if (adio_fh->is_external32) {
-        MPI_Aint e32_size = 0;
-        error_code = MPIU_datatype_full_size(datatype, &e32_size);
-        if (error_code != MPI_SUCCESS)
-            goto fn_exit;
-
-        e32_buf = ADIOI_Malloc(e32_size*count);
-        xbuf = e32_buf;
-    }
-
-    ADIO_IreadStridedColl(adio_fh, xbuf, count, datatype, file_ptr_type,
-                          offset, request, &error_code);
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-    error_code = MPIO_Err_return_file(adio_fh, error_code);
-    /* --END ERROR HANDLING-- */
-
-    if (e32_buf != NULL) {
-        error_code = MPIU_read_external32_conversion_fn(buf, datatype,
-                                                        count, e32_buf);
-        ADIOI_Free(e32_buf);
-    }
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-
-    return error_code;
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/iread_at.c b/3rd-party/romio321/mpi-io/iread_at.c
deleted file mode 100644
index 29bda8462f1..00000000000
--- a/3rd-party/romio321/mpi-io/iread_at.c
+++ /dev/null
@@ -1,75 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_iread_at = PMPI_File_iread_at
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_iread_at MPI_File_iread_at
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_iread_at as PMPI_File_iread_at
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_iread_at(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype,
-                      MPIO_Request *request) __attribute__((weak,alias("PMPI_File_iread_at")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-#ifdef HAVE_MPI_GREQUEST
-#include "mpiu_greq.h"
-#endif
-
-/*@
-    MPI_File_iread_at - Nonblocking read using explicit offset
-
-Input Parameters:
-. fh - file handle (handle)
-. offset - file offset (nonnegative integer)
-. count - number of elements in buffer (nonnegative integer)
-. datatype - datatype of each buffer element (handle)
-
-Output Parameters:
-. buf - initial address of buffer (choice)
-. request - request object (handle)
-
-.N fortran
-@*/
-int MPI_File_iread_at(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, 
-                      MPIO_Request *request)
-{
-    int error_code;
-    static char myname[] = "MPI_FILE_IREAD_AT";
-
-#ifdef MPI_hpux
-    int fl_xmpi;
-
-    HPMP_IO_START(fl_xmpi, BLKMPIFILEIREADAT, TRDTSYSTEM, fh, datatype,
-		  count);
-#endif /* MPI_hpux */
-
-
-    error_code = MPIOI_File_iread(fh, offset, ADIO_EXPLICIT_OFFSET, buf,
-				  count, datatype, myname, request);
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-	error_code = MPIO_Err_return_file(fh, error_code);
-    /* --END ERROR HANDLING-- */
-
-#ifdef MPI_hpux
-    HPMP_IO_END(fl_xmpi, fh, datatype, count);
-#endif /* MPI_hpux */
-
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/iread_atall.c b/3rd-party/romio321/mpi-io/iread_atall.c
deleted file mode 100644
index da9326a5b0d..00000000000
--- a/3rd-party/romio321/mpi-io/iread_atall.c
+++ /dev/null
@@ -1,74 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *  (C) 2014 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_iread_at_all = PMPI_File_iread_at_all
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_iread_at_all MPI_File_iread_at_all
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_iread_at_all as PMPI_File_iread_at_all
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_iread_at_all(MPI_File fh, MPI_Offset offset, void * buf, int count,
-                          MPI_Datatype datatype, MPI_Request *request)
-    __attribute__((weak,alias("PMPI_File_iread_at_all")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-#ifdef HAVE_MPI_GREQUEST
-#include "mpiu_greq.h"
-#endif
-
-/*@
-    MPI_File_iread_at_all - Nonblocking collective read using explicit offset
-
-Input Parameters:
-. fh - file handle (handle)
-. offset - file offset (nonnegative integer)
-. count - number of elements in buffer (nonnegative integer)
-. datatype - datatype of each buffer element (handle)
-
-Output Parameters:
-. buf - initial address of buffer (choice)
-. request - request object (handle)
-
-.N fortran
-@*/
-int MPI_File_iread_at_all(MPI_File fh, MPI_Offset offset, void *buf,
-                          int count, MPI_Datatype datatype,
-                          MPI_Request *request)
-{
-    int error_code;
-    static char myname[] = "MPI_FILE_IREAD_AT_ALL";
-#ifdef MPI_hpux
-    int fl_xmpi;
-
-    HPMP_IO_START(fl_xmpi, BLKMPIFILEIREADATALL, TRDTBLOCK, fh, datatype,
-		  count);
-#endif /* MPI_hpux */
-
-    error_code = MPIOI_File_iread_all(fh, offset, ADIO_EXPLICIT_OFFSET, buf,
-				     count, datatype, myname, request);
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-	error_code = MPIO_Err_return_file(fh, error_code);
-    /* --END ERROR HANDLING-- */
-
-#ifdef MPI_hpux
-    HPMP_IO_END(fl_xmpi, fh, datatype, count);
-#endif /* MPI_hpux */
-
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/iread_sh.c b/3rd-party/romio321/mpi-io/iread_sh.c
deleted file mode 100644
index bd643130590..00000000000
--- a/3rd-party/romio321/mpi-io/iread_sh.c
+++ /dev/null
@@ -1,140 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_iread_shared = PMPI_File_iread_shared
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_iread_shared MPI_File_iread_shared
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_iread_shared as PMPI_File_iread_shared
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_iread_shared(MPI_File fh, void *buf, int count, MPI_Datatype datatype,
-                          MPIO_Request *request) __attribute__((weak,alias("PMPI_File_iread_shared")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-#ifdef HAVE_MPI_GREQUEST
-#include "mpiu_greq.h"
-
-/*@
-    MPI_File_iread_shared - Nonblocking read using shared file pointer
-
-Input Parameters:
-. fh - file handle (handle)
-. count - number of elements in buffer (nonnegative integer)
-. datatype - datatype of each buffer element (handle)
-
-Output Parameters:
-. buf - initial address of buffer (choice)
-. request - request object (handle)
-
-.N fortran
-@*/
-int MPI_File_iread_shared(MPI_File fh, void *buf, int count,
-                          MPI_Datatype datatype, MPI_Request *request)
-{
-    int error_code, buftype_is_contig, filetype_is_contig;
-    ADIO_Offset bufsize;
-    ADIO_File adio_fh;
-    static char myname[] = "MPI_FILE_IREAD_SHARED";
-    MPI_Count datatype_size, incr;
-    MPI_Status status;
-    ADIO_Offset off, shared_fp;
-    MPI_Offset nbytes=0;
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    MPIO_CHECK_COUNT(adio_fh, count, myname, error_code);
-    MPIO_CHECK_DATATYPE(adio_fh, datatype, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    MPI_Type_size_x(datatype, &datatype_size);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_INTEGRAL_ETYPE(adio_fh, count, datatype_size, myname, error_code);
-    MPIO_CHECK_FS_SUPPORTS_SHARED(adio_fh, myname, error_code);
-    MPIO_CHECK_COUNT_SIZE(adio_fh, count, datatype_size, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-    ADIOI_Datatype_iscontig(adio_fh->filetype, &filetype_is_contig);
-
-    ADIOI_TEST_DEFERRED(adio_fh, myname, &error_code);
-
-    incr = (count*datatype_size)/adio_fh->etype_size;
-    ADIO_Get_shared_fp(adio_fh, incr, &shared_fp, &error_code);
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-    {
-	/* note: ADIO_Get_shared_fp should have set up error code already? */
-	MPIO_Err_return_file(adio_fh, error_code);
-    }
-    /* --END ERROR HANDLING-- */
-
-    if (buftype_is_contig && filetype_is_contig)
-    {
-    /* convert count and shared_fp to bytes */
-	bufsize = datatype_size * count;
-	off = adio_fh->disp + adio_fh->etype_size * shared_fp;
-        if (!(adio_fh->atomicity))
-	{
-	    ADIO_IreadContig(adio_fh, buf, count, datatype, ADIO_EXPLICIT_OFFSET,
-			off, request, &error_code);
-	}
-        else
-	{
-            /* to maintain strict atomicity semantics with other concurrent
-              operations, lock (exclusive) and call blocking routine */
-
-            if (adio_fh->file_system != ADIO_NFS)
-	    {
-                ADIOI_WRITE_LOCK(adio_fh, off, SEEK_SET, bufsize);
-	    }
-
-            ADIO_ReadContig(adio_fh, buf, count, datatype, ADIO_EXPLICIT_OFFSET,
-			    off, &status, &error_code);  
-
-            if (adio_fh->file_system != ADIO_NFS)
-	    {
-                ADIOI_UNLOCK(adio_fh, off, SEEK_SET, bufsize);
-	    }
-	    if (error_code == MPI_SUCCESS){
-		    nbytes = count * datatype_size;
-	    }
-	    MPIO_Completed_request_create(&adio_fh, nbytes, &error_code, request);
-        }
-    }
-    else
-    {
-	ADIO_IreadStrided(adio_fh, buf, count, datatype, ADIO_EXPLICIT_OFFSET,
-			   shared_fp, request, &error_code);
-    }
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-    /* --END ERROR HANDLING-- */
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-    return error_code;
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/iwrite.c b/3rd-party/romio321/mpi-io/iwrite.c
deleted file mode 100644
index ae6c6aec59c..00000000000
--- a/3rd-party/romio321/mpi-io/iwrite.c
+++ /dev/null
@@ -1,168 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_iwrite = PMPI_File_iwrite
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_iwrite MPI_File_iwrite
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_iwrite as PMPI_File_iwrite
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_iwrite(MPI_File fh, const void *buf, int count, MPI_Datatype datatype,
-                    MPIO_Request *request) __attribute__((weak,alias("PMPI_File_iwrite")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_iwrite - Nonblocking write using individual file pointer
-
-Input Parameters:
-. fh - file handle (handle)
-. buf - initial address of buffer (choice)
-. count - number of elements in buffer (nonnegative integer)
-. datatype - datatype of each buffer element (handle)
-
-Output Parameters:
-. request - request object (handle)
-
-.N fortran
-@*/
-#ifdef HAVE_MPI_GREQUEST
-#include "mpiu_greq.h"
-#endif
-
-int MPI_File_iwrite(MPI_File fh, ROMIO_CONST void *buf, int count,
-		    MPI_Datatype datatype, MPI_Request *request)
-{
-    int error_code=MPI_SUCCESS;
-    static char myname[] = "MPI_FILE_IWRITE";
-#ifdef MPI_hpux
-    int fl_xmpi;
-
-    HPMP_IO_START(fl_xmpi, BLKMPIFILEIWRITE, TRDTSYSTEM, fh, datatype,
-		  count);
-#endif /* MPI_hpux */
-
-
-    error_code = MPIOI_File_iwrite(fh, (MPI_Offset) 0, ADIO_INDIVIDUAL,
-				   buf, count, datatype, myname, request);
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-	error_code = MPIO_Err_return_file(fh, error_code);
-    /* --END ERROR HANDLING-- */
-
-#ifdef MPI_hpux
-    HPMP_IO_END(fl_xmpi, fh, datatype, count);
-#endif /* MPI_hpux */
-
-    return error_code;
-}
-
-/* prevent multiple definitions of this routine */
-#ifdef MPIO_BUILD_PROFILING
-int MPIOI_File_iwrite(MPI_File fh,
-		      MPI_Offset offset,
-		      int file_ptr_type,
-		      const void *buf,
-		      int count,
-		      MPI_Datatype datatype,
-		      char *myname,
-		      MPI_Request *request)
-{
-    int error_code, buftype_is_contig, filetype_is_contig;
-    MPI_Count datatype_size;
-    ADIO_Status status;
-    ADIO_Offset off, bufsize;
-    ADIO_File adio_fh;
-    MPI_Offset nbytes=0;
-
-    ROMIO_THREAD_CS_ENTER();
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    MPIO_CHECK_COUNT(adio_fh, count, myname, error_code);
-    MPIO_CHECK_DATATYPE(adio_fh, datatype, myname, error_code);
-
-    if (file_ptr_type == ADIO_EXPLICIT_OFFSET && offset < 0) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "**iobadoffset", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    MPI_Type_size_x(datatype, &datatype_size);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_INTEGRAL_ETYPE(adio_fh, count, datatype_size, myname, error_code);
-    MPIO_CHECK_WRITABLE(adio_fh, myname, error_code);
-    MPIO_CHECK_NOT_SEQUENTIAL_MODE(adio_fh, myname, error_code);
-    MPIO_CHECK_COUNT_SIZE(adio_fh, count, datatype_size, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-    ADIOI_Datatype_iscontig(adio_fh->filetype, &filetype_is_contig);
-    
-    ADIOI_TEST_DEFERRED(adio_fh, myname, &error_code);
-
-    if (buftype_is_contig && filetype_is_contig) {
-	/* convert sizes to bytes */
-	bufsize = datatype_size * count;
-	if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-	    off = adio_fh->disp + adio_fh->etype_size * offset;
-	}
-	else {
-	    off = adio_fh->fp_ind;
-	}
-
-        if (!(adio_fh->atomicity)) {
-	    ADIO_IwriteContig(adio_fh, buf, count, datatype, file_ptr_type,
-			      off, request, &error_code);
-	}
-	else {
-            /* to maintain strict atomicity semantics with other concurrent
-              operations, lock (exclusive) and call blocking routine */
-	    if (ADIO_Feature(adio_fh, ADIO_LOCKS) )
-	    {
-                ADIOI_WRITE_LOCK(adio_fh, off, SEEK_SET, bufsize);
-	    }
-
-            ADIO_WriteContig(adio_fh, buf, count, datatype, file_ptr_type, off,
-			     &status, &error_code);  
-
-	    if (ADIO_Feature(adio_fh, ADIO_LOCKS) )
-	    {
-                ADIOI_UNLOCK(adio_fh, off, SEEK_SET, bufsize);
-	    }
-	    if (error_code == MPI_SUCCESS) {
-		nbytes = count * datatype_size;
-	    }
-	    
-	    MPIO_Completed_request_create(&adio_fh, nbytes, &error_code, request);
-	}
-    }
-    else {
-	ADIO_IwriteStrided(adio_fh, buf, count, datatype, file_ptr_type,
-			   offset, request, &error_code);
-    }
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-    return error_code;
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/iwrite_all.c b/3rd-party/romio321/mpi-io/iwrite_all.c
deleted file mode 100644
index 9f486e700f5..00000000000
--- a/3rd-party/romio321/mpi-io/iwrite_all.c
+++ /dev/null
@@ -1,138 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *  (C) 2014 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_iwrite_all = PMPI_File_iwrite_all
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_iwrite_all MPI_File_iwrite_all
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_iwrite_all as PMPI_File_iwrite_all
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_iwrite_all(MPI_File fh, const void *buf, int count, MPI_Datatype datatype,
-                        MPI_Request *request)
-    __attribute__((weak,alias("PMPI_File_iwrite_all")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-#ifdef HAVE_MPI_GREQUEST
-#include "mpiu_greq.h"
-#endif
-
-/*@
-    MPI_File_iwrite_all - Nonblocking collective write using individual file pointer
-
-Input Parameters:
-. fh - file handle (handle)
-. buf - initial address of buffer (choice)
-. count - number of elements in buffer (nonnegative integer)
-. datatype - datatype of each buffer element (handle)
-
-Output Parameters:
-. request - request object (handle)
-
-.N fortran
-@*/
-int MPI_File_iwrite_all(MPI_File fh, ROMIO_CONST void *buf, int count,
-                        MPI_Datatype datatype, MPI_Request *request)
-{
-    int error_code;
-    static char myname[] = "MPI_FILE_IWRITE_ALL";
-#ifdef MPI_hpux
-    int fl_xmpi;
-
-    HPMP_IO_START(fl_xmpi, BLKMPIFILEIWRITEALL, TRDTBLOCK, fh, datatype, count);
-#endif /* MPI_hpux */
-
-    error_code = MPIOI_File_iwrite_all(fh, (MPI_Offset) 0,
-                      ADIO_INDIVIDUAL, buf,
-                      count, datatype, myname, request);
-
-#ifdef MPI_hpux
-    HPMP_IO_END(fl_xmpi, fh, datatype, count);
-#endif /* MPI_hpux */
-
-    return error_code;
-}
-
-/* Note: MPIOI_File_iwrite_all also used by MPI_File_iwrite_at_all */
-/* prevent multiple definitions of this routine */
-#ifdef MPIO_BUILD_PROFILING
-int MPIOI_File_iwrite_all(MPI_File fh,
-            MPI_Offset offset,
-            int file_ptr_type,
-            const void *buf,
-            int count,
-            MPI_Datatype datatype,
-            char *myname,
-            MPI_Request *request)
-{
-    int error_code;
-    MPI_Count datatype_size;
-    ADIO_File adio_fh;
-    void *e32buf=NULL;
-    const void *xbuf=NULL;
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    MPIO_CHECK_COUNT(adio_fh, count, myname, error_code);
-    MPIO_CHECK_DATATYPE(adio_fh, datatype, myname, error_code);
-
-    if (file_ptr_type == ADIO_EXPLICIT_OFFSET && offset < 0)
-    {
-        error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-                          myname, __LINE__, MPI_ERR_ARG,
-                          "**iobadoffset", 0);
-        error_code = MPIO_Err_return_file(adio_fh, error_code);
-        goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    MPI_Type_size_x(datatype, &datatype_size);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_INTEGRAL_ETYPE(adio_fh, count, datatype_size, myname, error_code);
-    MPIO_CHECK_WRITABLE(adio_fh, myname, error_code);
-    MPIO_CHECK_NOT_SEQUENTIAL_MODE(adio_fh, myname, error_code);
-    MPIO_CHECK_COUNT_SIZE(adio_fh, count, datatype_size, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    xbuf = buf;
-    if (adio_fh->is_external32) {
-        error_code = MPIU_external32_buffer_setup(buf, count, datatype, &e32buf);
-        if (error_code != MPI_SUCCESS)
-            goto fn_exit;
-
-        xbuf = e32buf;
-    }
-
-    ADIO_IwriteStridedColl(adio_fh, xbuf, count, datatype, file_ptr_type,
-                           offset, request, &error_code);
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-    error_code = MPIO_Err_return_file(adio_fh, error_code);
-    /* --END ERROR HANDLING-- */
-
-fn_exit:
-    if (e32buf != NULL) ADIOI_Free(e32buf);
-    ROMIO_THREAD_CS_EXIT();
-
-    return error_code;
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/iwrite_at.c b/3rd-party/romio321/mpi-io/iwrite_at.c
deleted file mode 100644
index 10862bd0334..00000000000
--- a/3rd-party/romio321/mpi-io/iwrite_at.c
+++ /dev/null
@@ -1,80 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_iwrite_at = PMPI_File_iwrite_at
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_iwrite_at MPI_File_iwrite_at
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_iwrite_at as PMPI_File_iwrite_at
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_iwrite_at(MPI_File fh, MPI_Offset offset, const void *buf, int count,
-                       MPI_Datatype datatype, MPIO_Request *request)
-    __attribute__((weak,alias("PMPI_File_iwrite_at")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_iwrite_at - Nonblocking write using explicit offset
-
-Input Parameters:
-. fh - file handle (handle)
-. offset - file offset (nonnegative integer)
-. buf - initial address of buffer (choice)
-. count - number of elements in buffer (nonnegative integer)
-. datatype - datatype of each buffer element (handle)
-
-Output Parameters:
-. request - request object (handle)
-
-.N fortran
-@*/
-#ifdef HAVE_MPI_GREQUEST
-#include "mpiu_greq.h"
-#endif
-
-int MPI_File_iwrite_at(MPI_File fh, MPI_Offset offset, ROMIO_CONST void *buf,
-                       int count, MPI_Datatype datatype, 
-                       MPIO_Request *request)
-{
-    int error_code;
-    ADIO_File adio_fh;
-    static char myname[] = "MPI_FILE_IWRITE_AT";
-
-#ifdef MPI_hpux
-    int fl_xmpi;
-
-    HPMP_IO_START(fl_xmpi, BLKMPIFILEIWRITEAT, TRDTSYSTEM,
-		  fh, datatype, count);
-#endif /* MPI_hpux */
-
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    error_code = MPIOI_File_iwrite(adio_fh, offset, ADIO_EXPLICIT_OFFSET, buf,
-				   count, datatype, myname, request);
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-    /* --END ERROR HANDLING-- */
-
-#ifdef MPI_hpux
-    HPMP_IO_END(fl_xmpi, fh, datatype, count)
-#endif /* MPI_hpux */
-
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/iwrite_atall.c b/3rd-party/romio321/mpi-io/iwrite_atall.c
deleted file mode 100644
index 9c0564efd8d..00000000000
--- a/3rd-party/romio321/mpi-io/iwrite_atall.c
+++ /dev/null
@@ -1,68 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *  (C) 2014 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_iwrite_at_all = PMPI_File_iwrite_at_all
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_iwrite_at_all MPI_File_iwrite_at_all
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_iwrite_at_all as PMPI_File_iwrite_at_all
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_iwrite_at_all(MPI_File fh, MPI_Offset offset, const void *buf, int count,
-                           MPI_Datatype datatype, MPI_Request *request)
-    __attribute__((weak,alias("PMPI_File_iwrite_at_all")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-#ifdef HAVE_MPI_GREQUEST
-#include "mpiu_greq.h"
-#endif
-
-/*@
-    MPI_File_iwrite_at_all - Nonblocking collective write using explicit offset
-
-Input Parameters:
-. fh - file handle (handle)
-. offset - file offset (nonnegative integer)
-. buf - initial address of buffer (choice)
-. count - number of elements in buffer (nonnegative integer)
-. datatype - datatype of each buffer element (handle)
-
-Output Parameters:
-. request - request object (handle)
-
-.N fortran
-@*/
-int MPI_File_iwrite_at_all(MPI_File fh, MPI_Offset offset, ROMIO_CONST void *buf,
-                           int count, MPI_Datatype datatype,
-                           MPI_Request *request)
-{
-    int error_code;
-    static char myname[] = "MPI_FILE_IWRITE_AT_ALL";
-#ifdef MPI_hpux
-    int fl_xmpi;
-
-    HPMP_IO_START(fl_xmpi, BLKMPIFILEIWRITEATALL, TRDTBLOCK, fh, datatype, count);
-#endif /* MPI_hpux */
-
-    error_code = MPIOI_File_iwrite_all(fh, offset, ADIO_EXPLICIT_OFFSET,
-				      buf, count, datatype, myname, request);
-
-#ifdef MPI_hpux
-    HPMP_IO_END(fl_xmpi, fh, datatype, count);
-#endif /* MPI_hpux */
-    return error_code;
-}
-
diff --git a/3rd-party/romio321/mpi-io/iwrite_sh.c b/3rd-party/romio321/mpi-io/iwrite_sh.c
deleted file mode 100644
index 3f9612a7895..00000000000
--- a/3rd-party/romio321/mpi-io/iwrite_sh.c
+++ /dev/null
@@ -1,120 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_iwrite_shared = PMPI_File_iwrite_shared
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_iwrite_shared MPI_File_iwrite_shared
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_iwrite_shared as PMPI_File_iwrite_shared
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_iwrite_shared(MPI_File fh, const void *buf, int count, MPI_Datatype datatype,
-                           MPIO_Request *request) __attribute__((weak,alias("PMPI_File_iwrite_shared")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_iwrite_shared - Nonblocking write using shared file pointer
-
-Input Parameters:
-. fh - file handle (handle)
-. buf - initial address of buffer (choice)
-. count - number of elements in buffer (nonnegative integer)
-. datatype - datatype of each buffer element (handle)
-
-Output Parameters:
-. request - request object (handle)
-
-.N fortran
-@*/
-#ifdef HAVE_MPI_GREQUEST
-#include "mpiu_greq.h"
-#endif
-
-int MPI_File_iwrite_shared(MPI_File fh, ROMIO_CONST void *buf, int count,
-			   MPI_Datatype datatype, MPIO_Request *request)
-{
-    int error_code, buftype_is_contig, filetype_is_contig;
-    ADIO_File adio_fh;
-    ADIO_Offset incr, bufsize;
-    MPI_Count datatype_size;
-    ADIO_Status status;
-    ADIO_Offset off, shared_fp;
-    static char myname[] = "MPI_FILE_IWRITE_SHARED";
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    MPIO_CHECK_COUNT(adio_fh, count, myname, error_code);
-    MPIO_CHECK_DATATYPE(adio_fh, datatype, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    MPI_Type_size_x(datatype, &datatype_size);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_INTEGRAL_ETYPE(adio_fh, count, datatype_size, myname, error_code);
-    MPIO_CHECK_FS_SUPPORTS_SHARED(adio_fh, myname, error_code);
-    MPIO_CHECK_COUNT_SIZE(adio_fh, count, datatype_size, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-    ADIOI_Datatype_iscontig(adio_fh->filetype, &filetype_is_contig);
-
-    ADIOI_TEST_DEFERRED(adio_fh, myname, &error_code);
-
-    incr = (count*datatype_size)/adio_fh->etype_size;
-    ADIO_Get_shared_fp(adio_fh, incr, &shared_fp, &error_code);
-    if (error_code != MPI_SUCCESS) {
-	/* note: ADIO_Get_shared_fp should have set up error code already? */
-	MPIO_Err_return_file(adio_fh, error_code);
-    }
-
-    /* contiguous or strided? */
-    if (buftype_is_contig && filetype_is_contig) {
-    /* convert sizes to bytes */
-	bufsize = datatype_size * count;
-	off = adio_fh->disp + adio_fh->etype_size * shared_fp;
-        if (!(adio_fh->atomicity))
-	    ADIO_IwriteContig(adio_fh, buf, count, datatype, ADIO_EXPLICIT_OFFSET,
-		            off, request, &error_code); 
-	else {
-            /* to maintain strict atomicity semantics with other concurrent
-              operations, lock (exclusive) and call blocking routine */
-
-            if (adio_fh->file_system != ADIO_NFS)
-                ADIOI_WRITE_LOCK(adio_fh, off, SEEK_SET, bufsize);
-
-            ADIO_WriteContig(adio_fh, buf, count, datatype, ADIO_EXPLICIT_OFFSET,
-			     off, &status, &error_code);  
-
-            if (adio_fh->file_system != ADIO_NFS)
-                ADIOI_UNLOCK(adio_fh, off, SEEK_SET, bufsize);
-
-	    MPIO_Completed_request_create(&adio_fh, bufsize, &error_code, request);
-	}
-    }
-    else
-	ADIO_IwriteStrided(adio_fh, buf, count, datatype, ADIO_EXPLICIT_OFFSET,
-			   shared_fp, request, &error_code); 
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/mpich_fileutil.c b/3rd-party/romio321/mpi-io/mpich_fileutil.c
deleted file mode 100644
index b853b588fa7..00000000000
--- a/3rd-party/romio321/mpi-io/mpich_fileutil.c
+++ /dev/null
@@ -1,60 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*  
- *  (C) 2001 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include "mpioimpl.h"
-#include "adio_extern.h"
-
-#ifdef MPICH
-
-/* Forward ref for the routine to extract and set the error handler
-   in a ROMIO File structure.  FIXME: These should be imported from a common
-   header file that is also used in errhan/file_set_errhandler.c
- */
-int MPIR_ROMIO_Get_file_errhand( MPI_File, MPI_Errhandler * );
-int MPIR_ROMIO_Set_file_errhand( MPI_File, MPI_Errhandler );
-void MPIR_Get_file_error_routine( MPI_Errhandler, 
-				  void (**)(MPI_File *, int *, ...), 
-				  int * );
-
-/* These next two routines are used to allow MPICH to access/set the
-   error handers in the MPI_File structure until MPICH knows about the
-   file structure, and to handle the errhandler structure, which 
-   includes a reference count.  Not currently used. */
-int MPIR_ROMIO_Set_file_errhand( MPI_File file_ptr, MPI_Errhandler e )
-{
-    if (file_ptr == MPI_FILE_NULL) ADIOI_DFLT_ERR_HANDLER = e;
-    /* --BEGIN ERROR HANDLING-- */
-    else if (file_ptr->cookie != ADIOI_FILE_COOKIE) {
-	return MPI_ERR_FILE;
-    }
-    /* --END ERROR HANDLING-- */
-    else 
-	file_ptr->err_handler = e;
-    return 0;
-}
-int MPIR_ROMIO_Get_file_errhand( MPI_File file_ptr, MPI_Errhandler *e )
-{
-    if (file_ptr == MPI_FILE_NULL) {
-	if (ADIOI_DFLT_ERR_HANDLER == MPI_ERRORS_RETURN)
-	    *e = 0;
-	else {
-	    *e = ADIOI_DFLT_ERR_HANDLER;
-	}
-    }
-    /* --BEGIN ERROR HANDLING-- */
-    else if (file_ptr->cookie != ADIOI_FILE_COOKIE) {
-	return MPI_ERR_FILE;
-    }
-    /* --END ERROR HANDLING-- */
-    else {
-	if (file_ptr->err_handler == MPI_ERRORS_RETURN) 
-	    *e = 0;
-	else
-	    *e = file_ptr->err_handler;
-    }
-    return 0;
-}
-
-#endif /* MPICH */
diff --git a/3rd-party/romio321/mpi-io/mpioimpl.h b/3rd-party/romio321/mpi-io/mpioimpl.h
deleted file mode 100644
index 4eb2843bcaf..00000000000
--- a/3rd-party/romio321/mpi-io/mpioimpl.h
+++ /dev/null
@@ -1,84 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   Copyright (c) 2018 IBM Corporation. All rights reserved.
- *   $COPYRIGHT$
- *   See COPYRIGHT notice in top-level directory.
- */
-
-
-/* header file for MPI-IO implementation. not intended to be
-   user-visible */ 
-
-#ifndef MPIOIMPL_INCLUDE
-#define MPIOIMPL_INCLUDE
-
-#include "adio.h"
-#include "mpio.h"
-
-#ifdef ROMIO_INSIDE_MPICH
-#include "glue_romio.h"
-
-#define ROMIO_THREAD_CS_ENTER() MPIR_Ext_cs_enter()
-#define ROMIO_THREAD_CS_EXIT() MPIR_Ext_cs_exit()
-#define ROMIO_THREAD_CS_YIELD() MPIR_Ext_cs_yield()
-
-/* committed datatype checking support in ROMIO */
-#define MPIO_DATATYPE_ISCOMMITTED(dtype_, err_)        \
-    do {                                               \
-        err_ =  MPIR_Ext_datatype_iscommitted(dtype_); \
-    } while (0)
-
-#else /* not ROMIO_INSIDE_MPICH */
-/* Any MPI implementation that wishes to follow the thread-safety and
-   error reporting features provided by MPICH must implement these 
-   four functions.  Defining these as empty should not change the behavior 
-   of correct programs */
-#define ROMIO_THREAD_CS_ENTER()
-#define ROMIO_THREAD_CS_EXIT()
-#define ROMIO_THREAD_CS_YIELD()
-/* The MPI_DATATYPE_ISCOMMITTED macro now always sets err_=0.
-   This is an optimistic approach for Open MPI, but it is likely other
-   upper layers already checked the datatype was committed.
-   Not setting err_ is incorrect since it can lead to use of
-   uninitialized variable.*/
-#define MPIO_DATATYPE_ISCOMMITTED(dtype_, err_) do { err_ = 0; } while (0)
-#ifdef HAVE_WINDOWS_H
-#define MPIU_UNREFERENCED_ARG(a) a
-#else
-#define MPIU_UNREFERENCED_ARG(a)
-#endif
-#endif /* ROMIO_INSIDE_MPICH */
-
-/* info is a linked list of these structures */
-struct MPIR_Info {
-    int cookie;
-    char *key, *value;
-    struct MPIR_Info *next;
-};
-
-#define MPIR_INFO_COOKIE 5835657
-
-MPI_Comm_delete_attr_function ADIOI_End_call;
-
-/* common initialization routine */
-void MPIR_MPIOInit(int * error_code);
-
-#ifdef HAVE_MPIIO_CONST
-#define ROMIO_CONST const
-#else
-#define ROMIO_CONST
-#endif
-
-#include "mpiu_external32.h"
-
-
-#include "mpioprof.h"
-
-#ifdef MPI_hpux
-#  include "mpioinst.h"
-#endif /* MPI_hpux */
-
-#endif
-
diff --git a/3rd-party/romio321/mpi-io/mpioprof.h b/3rd-party/romio321/mpi-io/mpioprof.h
deleted file mode 100644
index c74413d1e81..00000000000
--- a/3rd-party/romio321/mpi-io/mpioprof.h
+++ /dev/null
@@ -1,224 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-/* 
-   This header file converts all MPI_ names into PMPI_ names, for 
-   building the profiling interface
- */
-
-/*
- * Open MPI: Unfortunately, ROMIO doesn't seem to be able to build
- * without a profiling interface, but we don't want a profiling
- * interface, since we are just using ROMIO "behind the scenes".
- * So enable all the profiling defines, only compile once, and don't
- * do all the name mangling.  The effect is about the same, but without
- * modification to all the files in the mpi-io directory.
- */
-#if 0
-#ifdef MPIO_BUILD_PROFILING
-
-#undef MPI_File_open
-#define MPI_File_open PMPI_File_open
-#undef MPI_File_close
-#define MPI_File_close PMPI_File_close
-#undef MPI_File_delete
-#define MPI_File_delete PMPI_File_delete
-#undef MPI_File_set_size
-#define MPI_File_set_size PMPI_File_set_size
-#undef MPI_File_preallocate
-#define MPI_File_preallocate PMPI_File_preallocate
-#undef MPI_File_get_size
-#define MPI_File_get_size PMPI_File_get_size
-#undef MPI_File_get_group
-#define MPI_File_get_group PMPI_File_get_group
-#undef MPI_File_get_amode
-#define MPI_File_get_amode PMPI_File_get_amode
-#undef MPI_File_set_info
-#define MPI_File_set_info PMPI_File_set_info
-#undef MPI_File_get_info
-#define MPI_File_get_info PMPI_File_get_info
-
-#undef MPI_File_set_view
-#define MPI_File_set_view PMPI_File_set_view
-#undef MPI_File_get_view
-#define MPI_File_get_view PMPI_File_get_view
-
-#undef MPI_File_read_at
-#define MPI_File_read_at PMPI_File_read_at
-#undef MPI_File_read_at_all
-#define MPI_File_read_at_all PMPI_File_read_at_all
-#undef MPI_File_write_at
-#define MPI_File_write_at PMPI_File_write_at
-#undef MPI_File_write_at_all
-#define MPI_File_write_at_all PMPI_File_write_at_all
-#undef MPI_File_iread_at
-#define MPI_File_iread_at PMPI_File_iread_at
-#undef MPI_File_iwrite_at
-#define MPI_File_iwrite_at PMPI_File_iwrite_at
-
-#undef MPI_File_read
-#define MPI_File_read PMPI_File_read
-#undef MPI_File_read_all
-#define MPI_File_read_all  PMPI_File_read_all 
-#undef MPI_File_write
-#define MPI_File_write PMPI_File_write
-#undef MPI_File_write_all
-#define MPI_File_write_all PMPI_File_write_all
-#undef MPI_File_iread
-#define MPI_File_iread PMPI_File_iread
-#undef MPI_File_iwrite
-#define MPI_File_iwrite PMPI_File_iwrite
-#undef MPI_File_seek
-#define MPI_File_seek PMPI_File_seek
-#undef MPI_File_get_position
-#define MPI_File_get_position PMPI_File_get_position
-#undef MPI_File_get_byte_offset
-#define MPI_File_get_byte_offset PMPI_File_get_byte_offset
-
-#undef MPI_File_iread_at_all
-#define MPI_File_iread_at_all PMPI_File_iread_at_all
-#undef MPI_File_iwrite_at_all
-#define MPI_File_iwrite_at_all PMPI_File_iwrite_at_all
-#undef MPI_File_iread_all
-#define MPI_File_iread_all  PMPI_File_iread_all
-#undef MPI_File_iwrite_all
-#define MPI_File_iwrite_all PMPI_File_iwrite_all
-
-#undef MPI_File_read_shared
-#define MPI_File_read_shared PMPI_File_read_shared
-#undef MPI_File_write_shared
-#define MPI_File_write_shared PMPI_File_write_shared
-#undef MPI_File_iread_shared
-#define MPI_File_iread_shared PMPI_File_iread_shared
-#undef MPI_File_iwrite_shared
-#define MPI_File_iwrite_shared PMPI_File_iwrite_shared
-#undef MPI_File_read_ordered
-#define MPI_File_read_ordered PMPI_File_read_ordered
-#undef MPI_File_write_ordered
-#define MPI_File_write_ordered PMPI_File_write_ordered
-#undef MPI_File_seek_shared
-#define MPI_File_seek_shared PMPI_File_seek_shared
-#undef MPI_File_get_position_shared
-#define MPI_File_get_position_shared PMPI_File_get_position_shared
-
-#undef MPI_File_read_at_all_begin
-#define MPI_File_read_at_all_begin PMPI_File_read_at_all_begin
-#undef MPI_File_read_at_all_end
-#define MPI_File_read_at_all_end PMPI_File_read_at_all_end
-#undef MPI_File_write_at_all_begin
-#define MPI_File_write_at_all_begin PMPI_File_write_at_all_begin
-#undef MPI_File_write_at_all_end
-#define MPI_File_write_at_all_end PMPI_File_write_at_all_end
-#undef MPI_File_read_all_begin
-#define MPI_File_read_all_begin PMPI_File_read_all_begin
-#undef MPI_File_read_all_end
-#define MPI_File_read_all_end PMPI_File_read_all_end
-#undef MPI_File_write_all_begin
-#define MPI_File_write_all_begin PMPI_File_write_all_begin
-#undef MPI_File_write_all_end
-#define MPI_File_write_all_end PMPI_File_write_all_end
-#undef MPI_File_read_ordered_begin
-#define MPI_File_read_ordered_begin PMPI_File_read_ordered_begin
-#undef MPI_File_read_ordered_end
-#define MPI_File_read_ordered_end PMPI_File_read_ordered_end
-#undef MPI_File_write_ordered_begin
-#define MPI_File_write_ordered_begin PMPI_File_write_ordered_begin
-#undef MPI_File_write_ordered_end
-#define MPI_File_write_ordered_end PMPI_File_write_ordered_end
-
-#undef MPI_File_get_type_extent
-#define MPI_File_get_type_extent PMPI_File_get_type_extent
-#undef MPI_Register_datarep
-#define MPI_Register_datarep PMPI_Register_datarep
-#undef MPI_File_set_atomicity
-#define MPI_File_set_atomicity PMPI_File_set_atomicity
-#undef MPI_File_get_atomicity
-#define MPI_File_get_atomicity PMPI_File_get_atomicity
-#undef MPI_File_sync
-#define MPI_File_sync PMPI_File_sync
-
-#undef MPI_Type_create_subarray
-#define MPI_Type_create_subarray PMPI_Type_create_subarray
-#undef MPI_Type_create_darray
-#define MPI_Type_create_darray PMPI_Type_create_darray
-
-#undef MPI_File_set_errhandler
-#define MPI_File_set_errhandler PMPI_File_set_errhandler
-#undef MPI_File_get_errhandler
-#define MPI_File_get_errhandler PMPI_File_get_errhandler
-
-#if !defined(MPI_File_f2c) || defined(MPICH_RENAMING_MPI_FUNCS)
-#undef MPI_File_f2c
-#define MPI_File_f2c PMPI_File_f2c
-#undef MPI_File_c2f
-#define MPI_File_c2f PMPI_File_c2f
-#endif
-
-#undef MPIO_Test
-#undef PMPIO_Test
-#define MPIO_Test PMPIO_Test
-#undef MPIO_Wait
-#undef PMPIO_Wait
-#define MPIO_Wait PMPIO_Wait
-#undef MPIO_Testall
-#define MPIO_Testall PMPIO_Testall
-#undef MPIO_Waitall
-#define MPIO_Waitall PMPIO_Waitall
-#undef MPIO_Testany
-#define MPIO_Testany PMPIO_Testany
-#undef MPIO_Waitany
-#define MPIO_Waitany PMPIO_Waitany
-#undef MPIO_Testsome
-#define MPIO_Testsome PMPIO_Testsome
-#undef MPIO_Waitsome
-#define MPIO_Waitsome PMPIO_Waitsome
-#undef MPIO_Request_f2c
-#define MPIO_Request_f2c PMPIO_Request_f2c
-#undef MPIO_Request_c2f
-#define MPIO_Request_c2f PMPIO_Request_c2f
-
-#if defined(HAVE_MPI_INFO_SRC)  /* only in info source directory */
-
-#undef MPI_Info_create
-#define MPI_Info_create PMPI_Info_create
-#undef MPI_Info_set
-#define MPI_Info_set PMPI_Info_set
-#undef MPI_Info_delete
-#define MPI_Info_delete PMPI_Info_delete
-#undef MPI_Info_get
-#define MPI_Info_get PMPI_Info_get
-#undef MPI_Info_get_valuelen
-#define MPI_Info_get_valuelen PMPI_Info_get_valuelen
-#undef MPI_Info_get_nkeys
-#define MPI_Info_get_nkeys PMPI_Info_get_nkeys
-#undef MPI_Info_get_nthkey
-#define MPI_Info_get_nthkey PMPI_Info_get_nthkey
-#undef MPI_Info_dup
-#define MPI_Info_dup PMPI_Info_dup
-#undef MPI_Info_free
-#define MPI_Info_free PMPI_Info_free
-#undef MPI_Info_c2f
-#define MPI_Info_c2f PMPI_Info_c2f
-#undef MPI_Info_f2c
-#define MPI_Info_f2c PMPI_Info_f2c
-
-#endif
-
-#undef MPI_Grequest_start
-#define MPI_Grequest_start PMPI_Grequest_start
-#undef MPI_Grequest_complete
-#define MPI_Grequest_complete PMPI_Grequest_complete
-#undef MPI_Status_set_cancelled
-#define MPI_Status_set_cancelled PMPI_Status_set_cancelled
-
-#undef MPIX_Grequest_start
-#define MPIX_Grequest_start PMPIX_Grequest_start
-#undef MPIX_Grequest_class_create
-#define MPIX_Grequest_class_create PMPIX_Grequest_class_create
-
-#endif
-#endif
diff --git a/3rd-party/romio321/mpi-io/mpir-mpioinit.c b/3rd-party/romio321/mpi-io/mpir-mpioinit.c
deleted file mode 100644
index 90a2739a210..00000000000
--- a/3rd-party/romio321/mpi-io/mpir-mpioinit.c
+++ /dev/null
@@ -1,56 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *  (C) 2009 UChicago/Argonne LLC
- *      See COPYRIGHT in top-level directory.
- */
-#include "mpioimpl.h"
-#include <string.h>
-
-#ifdef HAVE_WEAK_SYMBOLS
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-extern int ADIO_Init_keyval;
-
-/* common code to stuff an attribute on a communicator for the purpose of
- * cleaning up in MPI_Finalize() */
-
-void MPIR_MPIOInit(int * error_code) {
-
-    int flag;
-    char myname[] = "MPIR_MPIOInit";
-
-    /* first check if ADIO has been initialized. If not, initialize it */
-    if (ADIO_Init_keyval == MPI_KEYVAL_INVALID) {
-        MPI_Initialized(&flag);
-
-	/* --BEGIN ERROR HANDLING-- */
-        if (!flag) {
-	    *error_code = MPIO_Err_create_code(MPI_SUCCESS, 
-		    MPIR_ERR_RECOVERABLE, myname, __LINE__, 
-		    MPI_ERR_OTHER, "**initialized", 0);
-	    *error_code = MPIO_Err_return_file(MPI_FILE_NULL, *error_code);
-	    return;
-	}
-	/* --END ERROR HANDLING-- */
-
-        MPI_Comm_create_keyval (MPI_COMM_NULL_COPY_FN, ADIOI_End_call, &ADIO_Init_keyval,
-                                (void *) 0);  
-
-	/* put a dummy attribute on MPI_COMM_SELF, because we want the delete
-	   function to be called when MPI_COMM_SELF is freed. Clarified
-	   in MPI-2 section 4.8, the standard mandates that attributes on
-	   MPI_COMM_SELF get cleaned up early in MPI_Finalize */
-
-        MPI_Comm_set_attr (MPI_COMM_SELF, ADIO_Init_keyval, (void *) 0);
-
-	/* initialize ADIO */
-        ADIO_Init( (int *)0, (char ***)0, error_code);
-    }
-    *error_code = MPI_SUCCESS;
-}
-/* 
- * vim: ts=8 sts=4 sw=4 noexpandtab 
- */
diff --git a/3rd-party/romio321/mpi-io/mpiu_external32.c b/3rd-party/romio321/mpi-io/mpiu_external32.c
deleted file mode 100644
index 8a8254cbb82..00000000000
--- a/3rd-party/romio321/mpi-io/mpiu_external32.c
+++ /dev/null
@@ -1,172 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*  (C) 2012 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-
-#include "mpioimpl.h"
-#include "mpiu_external32.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-int MPIU_write_external32_conversion_fn (const void *userbuf, MPI_Datatype datatype,
-        int count, void *filebuf)
-{
-    int position_i = 0;
-    MPI_Aint position = 0;
-    MPI_Aint bytes = 0;
-    int mpi_errno = MPI_SUCCESS;
-    int is_contig = 0;
-
-    ADIOI_Datatype_iscontig(datatype, &is_contig);
-    mpi_errno = MPI_Pack_external_size("external32", count, datatype, &bytes);
-    if (mpi_errno != MPI_SUCCESS)
-        goto fn_exit;
-
-    if (is_contig)
-    {
-#ifdef HAVE_MPIIO_CONST
-        mpi_errno = MPI_Pack_external("external32", userbuf, count,
-                datatype, filebuf, bytes, &position);
-#else
-        mpi_errno = MPI_Pack_external("external32", (void *)userbuf, count,
-                datatype, filebuf, bytes, &position);
-#endif
-        if (mpi_errno != MPI_SUCCESS)
-            goto fn_exit;
-    }
-    else
-    {
-        void *tmp_buf = NULL;
-        tmp_buf = ADIOI_Malloc(bytes); 
-        if (!tmp_buf)
-        {
-            mpi_errno = MPI_ERR_NO_MEM;
-            goto fn_exit;
-        }
-
-#ifdef HAVE_MPIIO_CONST
-        mpi_errno = MPI_Pack_external("external32", userbuf, count,
-                datatype, tmp_buf, bytes, &position);
-#else
-        mpi_errno = MPI_Pack_external("external32", (void *)userbuf, count,
-                datatype, tmp_buf, bytes, &position);
-#endif
-        if (mpi_errno != MPI_SUCCESS)
-        {
-            ADIOI_Free(tmp_buf);
-            goto fn_exit;
-        }
-
-        mpi_errno = MPI_Unpack(tmp_buf, bytes, &position_i, filebuf, count,
-                datatype, MPI_COMM_WORLD);
-        if (mpi_errno != MPI_SUCCESS)
-        {
-            ADIOI_Free(tmp_buf);
-            goto fn_exit;
-        }
-
-        ADIOI_Free(tmp_buf);
-    }
-fn_exit:
-    return mpi_errno;
-}
-
-int MPIU_read_external32_conversion_fn(void *userbuf, MPI_Datatype datatype,
-        int count, void *filebuf)
-{
-    int position_i = 0;
-    MPI_Aint position = 0;
-    MPI_Aint bytes = 0;
-    int mpi_errno = MPI_SUCCESS;
-    int is_contig = 0;
-
-    ADIOI_Datatype_iscontig(datatype, &is_contig);
-    mpi_errno = MPI_Pack_external_size("external32", count, datatype, &bytes);
-    if (mpi_errno != MPI_SUCCESS)
-        goto fn_exit;
-
-    if (is_contig)
-    {
-        mpi_errno = MPI_Unpack_external("external32", filebuf, bytes,
-                &position, userbuf, count,  datatype);
-        if (mpi_errno != MPI_SUCCESS)
-            goto fn_exit;
-    }
-    else
-    {
-        void *tmp_buf = NULL;
-        tmp_buf = ADIOI_Malloc(bytes); 
-        if (!tmp_buf)
-        {
-            mpi_errno = MPI_ERR_NO_MEM;
-            goto fn_exit;
-        }
-
-        mpi_errno = MPI_Pack(filebuf, count, datatype, tmp_buf, bytes,
-                &position_i, MPI_COMM_WORLD);
-        if (mpi_errno != MPI_SUCCESS)
-        {
-            ADIOI_Free(tmp_buf);
-            goto fn_exit;
-        }
-
-        mpi_errno = MPI_Unpack_external("external32", tmp_buf, bytes,
-                &position, userbuf, count, datatype);
-        if (mpi_errno != MPI_SUCCESS)
-        {
-            ADIOI_Free(tmp_buf);
-            goto fn_exit;
-        }
-
-        ADIOI_Free(tmp_buf);
-    }
-fn_exit:
-    return mpi_errno;
-}
-int MPIU_datatype_full_size(MPI_Datatype datatype, MPI_Aint *size)
-{
-    int error_code = MPI_SUCCESS;
-    MPI_Aint true_extent = 0;
-    MPI_Aint true_lb = 0;
-
-    error_code = MPI_Type_get_true_extent(datatype, &true_lb, &true_extent);
-    if (error_code != MPI_SUCCESS)
-        goto fn_exit;
-
-    *size = true_extent;
-fn_exit:
-    return error_code;
-}
-
-/* given a buffer, count, and datatype, return an apropriately allocated, sized
- * and external32-formatted buffer, suitable for handing off to a subsequent
- * write routine.  Caller is responsible for freeing 'newbuf' */
-int MPIU_external32_buffer_setup(const void * buf, int count, MPI_Datatype type, void **newbuf)
-{
-
-    MPI_Aint datatype_size=0, bytes=0;
-    int error_code;
-
-    error_code = MPIU_datatype_full_size(type, &datatype_size);
-    if (error_code != MPI_SUCCESS)
-	return error_code;
-
-    bytes = datatype_size * count;
-    *newbuf = ADIOI_Malloc(bytes);
-
-    error_code = MPIU_write_external32_conversion_fn(buf, type, count, *newbuf);
-    if (error_code != MPI_SUCCESS) {
-	ADIOI_Free(*newbuf);
-	return error_code;
-    }
-    return MPI_SUCCESS;
-}
-
-
-/* 
- * vim: ts=8 sts=4 sw=4 noexpandtab 
- */
diff --git a/3rd-party/romio321/mpi-io/mpiu_greq.c b/3rd-party/romio321/mpi-io/mpiu_greq.c
deleted file mode 100644
index ad1e4d25826..00000000000
--- a/3rd-party/romio321/mpi-io/mpiu_greq.c
+++ /dev/null
@@ -1,49 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *  (C) 2003 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include "mpioimpl.h"
-#include "mpiu_greq.h"
-
-#include <string.h>
-#ifdef HAVE_WEAK_SYMBOLS
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-int MPIU_Greq_query_fn(void *extra_state, MPI_Status *status)
-{
-    int foo;
-
-    /* can't touch user's MPI_ERROR, so hold it for a moment */
-    foo = status->MPI_ERROR;
-
-    /* get the status from the blocking operation */
-    memcpy(status, extra_state, sizeof(MPI_Status));
-
-    /* restore MPI_ERROR to whatever it had when we got it */
-    status->MPI_ERROR = foo;
-
-    /* and let Test|Wait know we weren't canceled */
-    MPI_Status_set_cancelled(status, 0);
-
-    return MPI_SUCCESS;
-}
-
-int MPIU_Greq_free_fn(void *extra_state)
-{
-    /* frees the memory allocated in MPIO_Completed_request_create */
-    ADIOI_Free(extra_state);
-	
-    return MPI_SUCCESS;
-}
-int MPIU_Greq_cancel_fn(void *extra_state, int complete)
-{
-    MPIU_UNREFERENCED_ARG(extra_state);
-    MPIU_UNREFERENCED_ARG(complete);
-
-    /* can't cancel */
-    return MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/mpi-io/open.c b/3rd-party/romio321/mpi-io/open.c
deleted file mode 100644
index 7cebb5abc79..00000000000
--- a/3rd-party/romio321/mpi-io/open.c
+++ /dev/null
@@ -1,209 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_open = PMPI_File_open
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_open MPI_File_open
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_open as PMPI_File_open
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_open(MPI_Comm comm, const char *filename, int amode, MPI_Info info, MPI_File *fh) __attribute__((weak,alias("PMPI_File_open")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/* for user-definde reduce operator */
-#include "adio_extern.h"
-
-
-extern int ADIO_Init_keyval;
-
-/*@
-    MPI_File_open - Opens a file
-
-Input Parameters:
-. comm - communicator (handle)
-. filename - name of file to open (string)
-. amode - file access mode (integer)
-. info - info object (handle)
-
-Output Parameters:
-. fh - file handle (handle)
-
-.N fortran
-@*/
-int MPI_File_open(MPI_Comm comm, ROMIO_CONST char *filename, int amode,
-                  MPI_Info info, MPI_File *fh)
-{
-    int error_code = MPI_SUCCESS, file_system, flag, tmp_amode=0, rank;
-    char *tmp;
-    MPI_Comm dupcomm = MPI_COMM_NULL;
-    ADIOI_Fns *fsops;
-    static char myname[] = "MPI_FILE_OPEN";
-#ifdef MPI_hpux
-    int fl_xmpi;
-
-    HPMP_IO_OPEN_START(fl_xmpi, comm);
-#endif /* MPI_hpux */
-
-    ROMIO_THREAD_CS_ENTER();
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_COMM(comm, myname, error_code);
-    MPIO_CHECK_INFO_ALL(info, error_code, comm);
-    /* --END ERROR HANDLING-- */
-
-    error_code = MPI_Comm_test_inter(comm, &flag);
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code || flag)
-    {
-	error_code = MPIO_Err_create_code(error_code, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_COMM, 
-					  "**commnotintra", 0);
-	goto fn_fail;
-    }
-
-    if ( ((amode&MPI_MODE_RDONLY)?1:0) + ((amode&MPI_MODE_RDWR)?1:0) +
-	 ((amode&MPI_MODE_WRONLY)?1:0) != 1 )
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_AMODE, 
-					  "**fileamodeone", 0);
-	goto fn_fail;
-    }
-
-    if ((amode & MPI_MODE_RDONLY) && 
-            ((amode & MPI_MODE_CREATE) || (amode & MPI_MODE_EXCL)))
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_AMODE, 
-					  "**fileamoderead", 0);
-	goto fn_fail;
-    }
-
-    if ((amode & MPI_MODE_RDWR) && (amode & MPI_MODE_SEQUENTIAL))
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_AMODE, 
-					  "**fileamodeseq", 0);
-	goto fn_fail;
-    }
-
-    MPI_Comm_dup(comm, &dupcomm);
-
-/* check if ADIO has been initialized. If not, initialize it */
-    MPIR_MPIOInit(&error_code);
-    if (error_code != MPI_SUCCESS) goto fn_fail;
-
-/* check if amode is the same on all processes: at first glance, one might try
- * to use a built-in operator like MPI_BAND, but we need every mpi process to
- * agree the amode was not the same.  Consider process A with
- * MPI_MODE_CREATE|MPI_MODE_RDWR, and B with MPI_MODE_RDWR:  MPI_BAND yields
- * MPI_MODE_RDWR.  A determines amodes are different, but B proceeds having not
- * detected an error */
-    MPI_Allreduce(&amode, &tmp_amode, 1, MPI_INT, ADIO_same_amode, dupcomm);
-
-    if (tmp_amode == ADIO_AMODE_NOMATCH) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-			myname, __LINE__, MPI_ERR_NOT_SAME,
-			"**fileamodediff", 0);
-	goto fn_fail;
-    }
-    /* --END ERROR HANDLING-- */
-
-    file_system = -1;
-
-    /* resolve file system type from file name; this is a collective call */
-    ADIO_ResolveFileType(dupcomm, filename, &file_system, &fsops, &error_code);
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-    {
-	/* ADIO_ResolveFileType() will print as informative a message as it
-	 * possibly can or call MPIO_Err_setmsg.  We just need to propagate 
-	 * the error up.
-	 */
-	goto fn_fail;
-    }
-
-    /* --END ERROR HANDLING-- */
-
-    /* strip off prefix if there is one, but only skip prefixes
-     * if they are greater than length one to allow for windows
-     * drive specifications (e.g. c:\...) */
-
-    tmp = strchr(filename, ':');
-    if (tmp > filename + 1) {
-	filename = tmp + 1;
-    }
-
-/* use default values for disp, etype, filetype */    
-
-    *fh = ADIO_Open(comm, dupcomm, filename, file_system, fsops, amode, 0,
-		    MPI_BYTE, MPI_BYTE, info, ADIO_PERM_NULL, &error_code);
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS) {
-	goto fn_fail;
-    }
-    /* --END ERROR HANDLING-- */
-
-    /* if MPI_MODE_SEQUENTIAL requested, file systems cannot do explicit offset
-     * or independent file pointer accesses, leaving not much else aside from
-     * shared file pointer accesses. */
-    if ( !ADIO_Feature((*fh), ADIO_SHARED_FP) && (amode & MPI_MODE_SEQUENTIAL)) 
-    {
-        error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, 
-			                  myname, __LINE__, 
-					  MPI_ERR_UNSUPPORTED_OPERATION,
-					  "**iosequnsupported", 0);
-	ADIO_Close(*fh, &error_code);
-	goto fn_fail;
-    }
-
-    /* determine name of file that will hold the shared file pointer */
-    /* can't support shared file pointers on a file system that doesn't
-       support file locking. */
-    if ((error_code == MPI_SUCCESS) && 
-		    ADIO_Feature((*fh), ADIO_SHARED_FP)) {
-	MPI_Comm_rank(dupcomm, &rank);
-	ADIOI_Shfp_fname(*fh, rank, &error_code);
-	if (error_code != MPI_SUCCESS)
-	    goto fn_fail;
-
-        /* if MPI_MODE_APPEND, set the shared file pointer to end of file.
-           indiv. file pointer already set to end of file in ADIO_Open. 
-           Here file view is just bytes. */
-	if ((*fh)->access_mode & MPI_MODE_APPEND) {
-	    if (rank == (*fh)->hints->ranklist[0])  /* only one person need set the sharedfp */
-		    ADIO_Set_shared_fp(*fh, (*fh)->fp_ind, &error_code);
-	    MPI_Barrier(dupcomm);
-	}
-    }
-
-#ifdef MPI_hpux
-    HPMP_IO_OPEN_END(fl_xmpi, *fh, comm);
-#endif /* MPI_hpux */
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-    return error_code;
-fn_fail:
-    /* --BEGIN ERROR HANDLING-- */
-    if (dupcomm != MPI_COMM_NULL) MPI_Comm_free(&dupcomm);
-    error_code = MPIO_Err_return_file(MPI_FILE_NULL, error_code);
-    goto fn_exit;
-    /* --END ERROR HANDLING-- */
-}
diff --git a/3rd-party/romio321/mpi-io/prealloc.c b/3rd-party/romio321/mpi-io/prealloc.c
deleted file mode 100644
index 1542e47a5d2..00000000000
--- a/3rd-party/romio321/mpi-io/prealloc.c
+++ /dev/null
@@ -1,107 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_preallocate = PMPI_File_preallocate
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_preallocate MPI_File_preallocate
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_preallocate as PMPI_File_preallocate
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_preallocate(MPI_File fh, MPI_Offset size) __attribute__((weak,alias("PMPI_File_preallocate")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_preallocate - Preallocates storage space for a file
-
-Input Parameters:
-. fh - file handle (handle)
-. size - size to preallocate (nonnegative integer)
-
-.N fortran
-@*/
-int MPI_File_preallocate(MPI_File fh, MPI_Offset size)
-{
-    ADIO_Fcntl_t *fcntl_struct;
-    int error_code=0, mynod=0;
-    ADIO_File adio_fh;
-    static char myname[] = "MPI_FILE_PREALLOCATE";
-    MPI_Offset tmp_sz, max_sz, min_sz;
-#ifdef MPI_hpux
-    int fl_xmpi;
-
-    HPMP_IO_START(fl_xmpi, BLKMPIFILEPREALLOCATE, TRDTBLOCK,
-		  adio_fh, MPI_DATATYPE_NULL, -1);
-#endif /* MPI_hpux */
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-
-    if (size < 0) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "**iobadsize", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-
-    tmp_sz = size;
-    MPI_Allreduce(&tmp_sz, &max_sz, 1, ADIO_OFFSET, MPI_MAX, adio_fh->comm);
-    MPI_Allreduce(&tmp_sz, &min_sz, 1, ADIO_OFFSET, MPI_MIN, adio_fh->comm);
-
-    if (max_sz != min_sz) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "**notsame", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    if (size == 0) goto fn_exit;
-
-    ADIOI_TEST_DEFERRED(adio_fh, myname, &error_code);
-
-    MPI_Comm_rank(adio_fh->comm, &mynod);
-    if (!mynod) {
-	fcntl_struct = (ADIO_Fcntl_t *) ADIOI_Malloc(sizeof(ADIO_Fcntl_t));
-	fcntl_struct->diskspace = size;
-	ADIO_Fcntl(adio_fh, ADIO_FCNTL_SET_DISKSPACE, fcntl_struct, &error_code);
-	ADIOI_Free(fcntl_struct);
-	/* --BEGIN ERROR HANDLING-- */
-	if (error_code != MPI_SUCCESS)
-	    error_code = MPIO_Err_return_file(adio_fh, error_code);
-	/* --END ERROR HANDLING-- */
-    }
-    MPI_Barrier(adio_fh->comm);
-    
-#ifdef MPI_hpux
-    HPMP_IO_END(fl_xmpi, adio_fh, MPI_DATATYPE_NULL, -1);
-#endif /* MPI_hpux */
-
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-
-    /* TODO: bcast result? */
-    if (!mynod) return error_code;
-    else return MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/mpi-io/rd_atallb.c b/3rd-party/romio321/mpi-io/rd_atallb.c
deleted file mode 100644
index a7120b3da21..00000000000
--- a/3rd-party/romio321/mpi-io/rd_atallb.c
+++ /dev/null
@@ -1,54 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_read_at_all_begin = PMPI_File_read_at_all_begin
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_read_at_all_begin MPI_File_read_at_all_begin
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_read_at_all_begin as PMPI_File_read_at_all_begin
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_read_at_all_begin(MPI_File fh, MPI_Offset offset, void *buf, int count,
-                               MPI_Datatype datatype) __attribute__((weak,alias("PMPI_File_read_at_all_begin")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_read_at_all_begin - Begin a split collective read using explicit offset
-
-Input Parameters:
-. fh - file handle (handle)
-. offset - file offset (nonnegative integer)
-. count - number of elements in buffer (nonnegative integer)
-. datatype - datatype of each buffer element (handle)
-
-Output Parameters:
-. buf - initial address of buffer (choice)
-
-.N fortran
-@*/
-int MPI_File_read_at_all_begin(MPI_File fh, MPI_Offset offset, void *buf,
-			       int count, MPI_Datatype datatype)
-{
-    int error_code;
-    static char myname[] = "MPI_FILE_READ_AT_ALL_BEGIN";
-
-    error_code = MPIOI_File_read_all_begin(fh, offset,
-					   ADIO_EXPLICIT_OFFSET,
-					   buf, count, datatype, myname);
-
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/rd_atalle.c b/3rd-party/romio321/mpi-io/rd_atalle.c
deleted file mode 100644
index 207d5ba3698..00000000000
--- a/3rd-party/romio321/mpi-io/rd_atalle.c
+++ /dev/null
@@ -1,50 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_read_at_all_end = PMPI_File_read_at_all_end
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_read_at_all_end MPI_File_read_at_all_end
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_read_at_all_end as PMPI_File_read_at_all_end
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_read_at_all_end(MPI_File fh, void *buf, MPI_Status *status) __attribute__((weak,alias("PMPI_File_read_at_all_end")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_read_at_all_end - Complete a split collective read using
-    explicit offset
-
-Input Parameters:
-. fh - file handle (handle)
-
-Output Parameters:
-. buf - initial address of buffer (choice)
-. status - status object (Status)
-
-.N fortran
-@*/
-int MPI_File_read_at_all_end(MPI_File fh, void *buf, MPI_Status *status)
-{
-    int error_code;
-    static char myname[] = "MPI_FILE_READ_AT_ALL_END";
-
-
-    error_code = MPIOI_File_read_all_end(fh, buf, myname, status);
-
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/read.c b/3rd-party/romio321/mpi-io/read.c
deleted file mode 100644
index 6f129148937..00000000000
--- a/3rd-party/romio321/mpi-io/read.c
+++ /dev/null
@@ -1,187 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_read = PMPI_File_read
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_read MPI_File_read
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_read as PMPI_File_read
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_read(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status)
-    __attribute__((weak,alias("PMPI_File_read")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/* status object not filled currently */
-
-/*@
-    MPI_File_read - Read using individual file pointer
-
-Input Parameters:
-. fh - file handle (handle)
-. count - number of elements in buffer (nonnegative integer)
-. datatype - datatype of each buffer element (handle)
-
-Output Parameters:
-. buf - initial address of buffer (choice)
-. status - status object (Status)
-
-.N fortran
-@*/
-int MPI_File_read(MPI_File fh, void *buf, int count,
-                  MPI_Datatype datatype, MPI_Status *status)
-{
-    int error_code;
-    static char myname[] = "MPI_FILE_READ";
-#ifdef MPI_hpux
-    int fl_xmpi;
-
-    HPMP_IO_START(fl_xmpi, BLKMPIFILEREAD, TRDTBLOCK, fh, datatype, count);
-#endif /* MPI_hpux */
-
-    error_code = MPIOI_File_read(fh, (MPI_Offset) 0, ADIO_INDIVIDUAL, buf,
-				 count, datatype, myname, status);
-
-#ifdef MPI_hpux
-    HPMP_IO_END(fl_xmpi, fh, datatype, count);
-#endif /* MPI_hpux */
-
-    return error_code;
-}
-
-/* prevent multiple definitions of this routine */
-#ifdef MPIO_BUILD_PROFILING
-int MPIOI_File_read(MPI_File fh,
-		    MPI_Offset offset,
-		    int file_ptr_type,
-		    void *buf,
-		    int count,
-		    MPI_Datatype datatype,
-		    char *myname,
-		    MPI_Status *status)
-{
-    int error_code, buftype_is_contig, filetype_is_contig;
-    MPI_Count datatype_size;
-    ADIO_File adio_fh;
-    ADIO_Offset off, bufsize;
-    void *xbuf=NULL, *e32_buf=NULL;
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    MPIO_CHECK_COUNT(adio_fh, count, myname, error_code);
-    MPIO_CHECK_DATATYPE(adio_fh, datatype, myname, error_code);
-
-    if (file_ptr_type == ADIO_EXPLICIT_OFFSET && offset < 0)
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "**iobadoffset", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    MPI_Type_size_x(datatype, &datatype_size);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_COUNT_SIZE(adio_fh, count, datatype_size, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    if (count*datatype_size == 0)
-    {
-#ifdef HAVE_STATUS_SET_BYTES
-       MPIR_Status_set_bytes(status, datatype, 0);
-#endif
-	error_code = MPI_SUCCESS;
-	goto fn_exit;
-    }
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_INTEGRAL_ETYPE(adio_fh, count, datatype_size, myname, error_code);
-    MPIO_CHECK_READABLE(adio_fh, myname, error_code);
-    MPIO_CHECK_NOT_SEQUENTIAL_MODE(adio_fh, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-    ADIOI_Datatype_iscontig(adio_fh->filetype, &filetype_is_contig);
-
-    ADIOI_TEST_DEFERRED(adio_fh, myname, &error_code);
-
-    xbuf = buf;
-    if (adio_fh->is_external32)
-    {
-        MPI_Aint e32_size = 0;
-        error_code = MPIU_datatype_full_size(datatype, &e32_size);
-        if (error_code != MPI_SUCCESS)
-            goto fn_exit;
-
-        e32_buf = ADIOI_Malloc(e32_size*count);
-	xbuf = e32_buf;
-    }
-
-    if (buftype_is_contig && filetype_is_contig)
-    {
-    /* convert count and offset to bytes */
-	bufsize = datatype_size * count;
-	if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-	    off = adio_fh->disp + adio_fh->etype_size * offset;
-	}
-	else /* ADIO_INDIVIDUAL */ {
-	    off = adio_fh->fp_ind;
-	}
-
-        /* if atomic mode requested, lock (exclusive) the region, because
-           there could be a concurrent noncontiguous request.
-	 */
-        if ((adio_fh->atomicity) && ADIO_Feature(adio_fh, ADIO_LOCKS)) {
-            ADIOI_WRITE_LOCK(adio_fh, off, SEEK_SET, bufsize);
-	}
-
-	ADIO_ReadContig(adio_fh, xbuf, count, datatype, file_ptr_type,
-			off, status, &error_code); 
-
-        if ((adio_fh->atomicity) && ADIO_Feature(adio_fh, ADIO_LOCKS)) {
-            ADIOI_UNLOCK(adio_fh, off, SEEK_SET, bufsize);
-	}
-    }
-    else
-    {
-	ADIO_ReadStrided(adio_fh, xbuf, count, datatype, file_ptr_type,
-			  offset, status, &error_code);
-	/* For strided and atomic mode, locking is done in ADIO_ReadStrided */
-    }
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-    /* --END ERROR HANDLING-- */
-
-    if (e32_buf != NULL) {
-        error_code = MPIU_read_external32_conversion_fn(buf, datatype,
-                count, e32_buf);
-	ADIOI_Free(e32_buf);
-    }
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-
-    return error_code;
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/read_all.c b/3rd-party/romio321/mpi-io/read_all.c
deleted file mode 100644
index efce7832b79..00000000000
--- a/3rd-party/romio321/mpi-io/read_all.c
+++ /dev/null
@@ -1,142 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_read_all = PMPI_File_read_all
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_read_all MPI_File_read_all
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_read_all as PMPI_File_read_all
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_read_all(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status)
-    __attribute__((weak,alias("PMPI_File_read_all")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/* status object not filled currently */
-
-/*@
-    MPI_File_read_all - Collective read using individual file pointer
-
-Input Parameters:
-. fh - file handle (handle)
-. count - number of elements in buffer (nonnegative integer)
-. datatype - datatype of each buffer element (handle)
-
-Output Parameters:
-. buf - initial address of buffer (choice)
-. status - status object (Status)
-
-.N fortran
-@*/
-int MPI_File_read_all(MPI_File fh, void *buf, int count,
-                      MPI_Datatype datatype, MPI_Status *status)
-{
-    int error_code;
-    static char myname[] = "MPI_FILE_READ_ALL";
-#ifdef MPI_hpux
-    int fl_xmpi;
-
-    HPMP_IO_START(fl_xmpi, BLKMPIFILEREADALL, TRDTBLOCK, fh, datatype, count);
-#endif /* MPI_hpux */
-
-    error_code = MPIOI_File_read_all(fh, (MPI_Offset) 0,
-				     ADIO_INDIVIDUAL, buf,
-				     count, datatype, myname, status);
-
-#ifdef MPI_hpux
-    HPMP_IO_END(fl_xmpi, fh, datatype, count);
-#endif /* MPI_hpux */
-
-    return error_code;
-}
-
-/* Note: MPIOI_File_read_all also used by MPI_File_read_at_all */
-/* prevent multiple definitions of this routine */
-#ifdef MPIO_BUILD_PROFILING
-int MPIOI_File_read_all(MPI_File fh,
-			MPI_Offset offset,
-			int file_ptr_type,
-			void *buf,
-			int count,
-			MPI_Datatype datatype,
-			char *myname,
-			MPI_Status *status)
-{
-    int error_code;
-    MPI_Count datatype_size;
-    ADIO_File adio_fh;
-    void *xbuf=NULL, *e32_buf=NULL;
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    MPIO_CHECK_COUNT(adio_fh, count, myname, error_code);
-    MPIO_CHECK_DATATYPE(adio_fh, datatype, myname, error_code);
-
-    if (file_ptr_type == ADIO_EXPLICIT_OFFSET && offset < 0)
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "**iobadoffset", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    MPI_Type_size_x(datatype, &datatype_size);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_INTEGRAL_ETYPE(adio_fh, count, datatype_size, myname, error_code);
-    MPIO_CHECK_READABLE(adio_fh, myname, error_code);
-    MPIO_CHECK_NOT_SEQUENTIAL_MODE(adio_fh, myname, error_code);
-    MPIO_CHECK_COUNT_SIZE(adio_fh, count, datatype_size, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    xbuf = buf;
-    if (adio_fh->is_external32)
-    {
-        MPI_Aint e32_size = 0;
-        error_code = MPIU_datatype_full_size(datatype, &e32_size);
-        if (error_code != MPI_SUCCESS)
-            goto fn_exit;
-
-        e32_buf = ADIOI_Malloc(e32_size*count);
-	xbuf = e32_buf;
-    }
-
-    ADIO_ReadStridedColl(adio_fh, xbuf, count, datatype, file_ptr_type,
-			 offset, status, &error_code);
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-    /* --END ERROR HANDLING-- */
-
-    if (e32_buf != NULL) {
-        error_code = MPIU_read_external32_conversion_fn(buf, datatype,
-                count, e32_buf);
-	ADIOI_Free(e32_buf);
-    }
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-
-    return error_code;
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/read_allb.c b/3rd-party/romio321/mpi-io/read_allb.c
deleted file mode 100644
index ae1053b1518..00000000000
--- a/3rd-party/romio321/mpi-io/read_allb.c
+++ /dev/null
@@ -1,139 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_read_all_begin = PMPI_File_read_all_begin
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_read_all_begin MPI_File_read_all_begin
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_read_all_begin as PMPI_File_read_all_begin
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_read_all_begin(MPI_File fh, void *buf, int count, MPI_Datatype datatype)
-    __attribute__((weak,alias("PMPI_File_read_all_begin")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_read_all_begin - Begin a split collective read using individual file pointer
-
-Input Parameters:
-. fh - file handle (handle)
-. count - number of elements in buffer (nonnegative integer)
-. datatype - datatype of each buffer element (handle)
-
-Output Parameters:
-. buf - initial address of buffer (choice)
-
-.N fortran
-@*/
-int MPI_File_read_all_begin(MPI_File fh, void *buf, int count,
-                            MPI_Datatype datatype)
-{
-    int error_code;
-    static char myname[] = "MPI_FILE_READ_ALL_BEGIN";
-
-    error_code = MPIOI_File_read_all_begin(fh, (MPI_Offset) 0,
-					   ADIO_INDIVIDUAL, buf, count,
-					   datatype, myname);
-
-    return error_code;
-}
-
-/* prevent multiple definitions of this routine */
-#ifdef MPIO_BUILD_PROFILING
-int MPIOI_File_read_all_begin(MPI_File fh,
-			      MPI_Offset offset,
-			      int file_ptr_type,
-			      void *buf,
-			      int count,
-			      MPI_Datatype datatype,
-			      char *myname)
-{
-    int error_code;
-    MPI_Count datatype_size;
-    ADIO_File adio_fh;
-    void *xbuf=NULL, *e32_buf=NULL;
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    MPIO_CHECK_COUNT(adio_fh, count, myname, error_code);
-    MPIO_CHECK_DATATYPE(adio_fh, datatype, myname, error_code);
-
-    if (file_ptr_type == ADIO_EXPLICIT_OFFSET && offset < 0)
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "**iobadoffset", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-    
-    MPI_Type_size_x(datatype, &datatype_size);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_INTEGRAL_ETYPE(adio_fh, count, datatype_size, myname, error_code);
-    MPIO_CHECK_READABLE(adio_fh, myname, error_code);
-    MPIO_CHECK_NOT_SEQUENTIAL_MODE(adio_fh, myname, error_code);
-
-    if (adio_fh->split_coll_count) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_IO, 
-					  "**iosplitcoll", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    MPIO_CHECK_COUNT_SIZE(adio_fh, count, datatype_size, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    adio_fh->split_coll_count = 1;
-
-    xbuf = buf;
-    if (adio_fh->is_external32)
-    {
-        MPI_Aint e32_size = 0;
-        error_code = MPIU_datatype_full_size(datatype, &e32_size);
-        if (error_code != MPI_SUCCESS)
-            goto fn_exit;
-
-        e32_buf = ADIOI_Malloc(e32_size*count);
-	xbuf = e32_buf;
-    }
-
-    ADIO_ReadStridedColl(adio_fh, xbuf, count, datatype, file_ptr_type,
-			 offset, &adio_fh->split_status, &error_code);
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-    /* --END ERROR HANDLING-- */
-
-    if (e32_buf != NULL) {
-        error_code = MPIU_read_external32_conversion_fn(buf, datatype,
-                count, e32_buf);
-	ADIOI_Free(e32_buf);
-    }
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-
-    return error_code;
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/read_alle.c b/3rd-party/romio321/mpi-io/read_alle.c
deleted file mode 100644
index e2853c13dd2..00000000000
--- a/3rd-party/romio321/mpi-io/read_alle.c
+++ /dev/null
@@ -1,90 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_read_all_end = PMPI_File_read_all_end
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_read_all_end MPI_File_read_all_end
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_read_all_end as PMPI_File_read_all_end
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_read_all_end(MPI_File fh, void *buf, MPI_Status *status) __attribute__((weak,alias("PMPI_File_read_all_end")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_read_all_end - Complete a split collective read using
-    individual file pointer
-
-Input Parameters:
-. fh - file handle (handle)
-
-Output Parameters:
-. buf - initial address of buffer (choice)
-. status - status object (Status)
-
-.N fortran
-@*/
-int MPI_File_read_all_end(MPI_File fh, void *buf, MPI_Status *status)
-{
-    int error_code;
-    static char myname[] = "MPI_FILE_IREAD";
-
-    error_code = MPIOI_File_read_all_end(fh, buf, myname, status);
-
-    return error_code;
-}
-
-/* prevent multiple definitions of this routine */
-#ifdef MPIO_BUILD_PROFILING
-int MPIOI_File_read_all_end(MPI_File fh,
-			    void *buf,
-			    char *myname,
-			    MPI_Status *status)
-{
-    int error_code = MPI_SUCCESS;
-    ADIO_File adio_fh;
-
-    MPIU_UNREFERENCED_ARG(buf);
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-
-    if (!(adio_fh->split_coll_count)) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_IO, 
-					  "**iosplitcollnone", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-#ifdef HAVE_STATUS_SET_BYTES
-    if (status != MPI_STATUS_IGNORE)
-       *status = adio_fh->split_status;
-#endif
-    adio_fh->split_coll_count = 0;
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-
-    return error_code;
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/read_at.c b/3rd-party/romio321/mpi-io/read_at.c
deleted file mode 100644
index 53dc8c42e3a..00000000000
--- a/3rd-party/romio321/mpi-io/read_at.c
+++ /dev/null
@@ -1,66 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_read_at = PMPI_File_read_at
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_read_at MPI_File_read_at
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_read_at as PMPI_File_read_at
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_read_at(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype,
-                     MPI_Status *status) __attribute__((weak,alias("PMPI_File_read_at")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/* status object not filled currently */
-
-/*@
-    MPI_File_read_at - Read using explicit offset
-
-Input Parameters:
-. fh - file handle (handle)
-. offset - file offset (nonnegative integer)
-. count - number of elements in buffer (nonnegative integer)
-. datatype - datatype of each buffer element (handle)
-
-Output Parameters:
-. buf - initial address of buffer (choice)
-. status - status object (Status)
-
-.N fortran
-@*/
-int MPI_File_read_at(MPI_File fh, MPI_Offset offset, void *buf,
-		     int count, MPI_Datatype datatype, MPI_Status *status)
-{
-    int error_code;
-    static char myname[] = "MPI_FILE_READ_AT";
-#ifdef MPI_hpux
-    int fl_xmpi;
-
-    HPMP_IO_START(fl_xmpi, BLKMPIFILEREADAT, TRDTBLOCK, fh, datatype,
-		  count);
-#endif /* MPI_hpux */
-
-    /* ADIOI_File_read() defined in mpi-io/read.c */
-    error_code = MPIOI_File_read(fh, offset, ADIO_EXPLICIT_OFFSET, buf,
-				 count, datatype, myname, status);
-
-#ifdef MPI_hpux
-    HPMP_IO_END(fl_xmpi, fh, datatype, count);
-#endif /* MPI_hpux */
-
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/read_atall.c b/3rd-party/romio321/mpi-io/read_atall.c
deleted file mode 100644
index 2bed66be4d6..00000000000
--- a/3rd-party/romio321/mpi-io/read_atall.c
+++ /dev/null
@@ -1,68 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_read_at_all = PMPI_File_read_at_all
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_read_at_all MPI_File_read_at_all
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_read_at_all as PMPI_File_read_at_all
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_read_at_all(MPI_File fh, MPI_Offset offset, void * buf, int count,
-                         MPI_Datatype datatype, MPI_Status *status)
-    __attribute__((weak,alias("PMPI_File_read_at_all")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/* status object not filled currently */
-
-/*@
-    MPI_File_read_at_all - Collective read using explicit offset
-
-Input Parameters:
-. fh - file handle (handle)
-. offset - file offset (nonnegative integer)
-. count - number of elements in buffer (nonnegative integer)
-. datatype - datatype of each buffer element (handle)
-
-Output Parameters:
-. buf - initial address of buffer (choice)
-. status - status object (Status)
-
-.N fortran
-@*/
-int MPI_File_read_at_all(MPI_File fh, MPI_Offset offset, void *buf,
-                         int count, MPI_Datatype datatype, 
-                         MPI_Status *status)
-{
-    int error_code;
-    static char myname[] = "MPI_FILE_IREAD_AT";
-#ifdef MPI_hpux
-    int fl_xmpi;
-
-    HPMP_IO_START(fl_xmpi, BLKMPIFILEREADATALL, TRDTBLOCK, fh, datatype,
-		  count);
-#endif /* MPI_hpux */
-
-    error_code = MPIOI_File_read_all(fh, offset, ADIO_EXPLICIT_OFFSET, buf,
-				     count, datatype, myname, status);
-
-#ifdef MPI_hpux
-    HPMP_IO_END(fl_xmpi, fh, datatype, count);
-#endif /* MPI_hpux */
-
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/read_ord.c b/3rd-party/romio321/mpi-io/read_ord.c
deleted file mode 100644
index be877633c8e..00000000000
--- a/3rd-party/romio321/mpi-io/read_ord.c
+++ /dev/null
@@ -1,111 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_read_ordered = PMPI_File_read_ordered
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_read_ordered MPI_File_read_ordered
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_read_ordered as PMPI_File_read_ordered
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_read_ordered(MPI_File fh, void *buf, int count, MPI_Datatype datatype,
-                          MPI_Status *status) __attribute__((weak,alias("PMPI_File_read_ordered")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/* status object not filled currently */
-
-/*@
-    MPI_File_read_ordered - Collective read using shared file pointer
-
-Input Parameters:
-. fh - file handle (handle)
-. count - number of elements in buffer (nonnegative integer)
-. datatype - datatype of each buffer element (handle)
-
-Output Parameters:
-. buf - initial address of buffer (choice)
-. status - status object (Status)
-
-.N fortran
-@*/
-int MPI_File_read_ordered(MPI_File fh, void *buf, int count,
-                          MPI_Datatype datatype, MPI_Status *status)
-{
-    int error_code, nprocs, myrank;
-    ADIO_Offset incr;
-    MPI_Count datatype_size;
-    int source, dest;
-    static char myname[] = "MPI_FILE_READ_ORDERED";
-    ADIO_Offset shared_fp=0;
-    ADIO_File adio_fh;
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    MPIO_CHECK_COUNT(adio_fh, count, myname, error_code);
-    MPIO_CHECK_DATATYPE(adio_fh, datatype, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    MPI_Type_size_x(datatype, &datatype_size);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_INTEGRAL_ETYPE(adio_fh, count, datatype_size, myname, error_code);
-    MPIO_CHECK_FS_SUPPORTS_SHARED(adio_fh, myname, error_code);
-    MPIO_CHECK_COUNT_SIZE(adio_fh, count, datatype_size, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    ADIOI_TEST_DEFERRED(adio_fh, "MPI_File_read_ordered", &error_code);
-
-    MPI_Comm_size(adio_fh->comm, &nprocs);
-    MPI_Comm_rank(adio_fh->comm, &myrank);
-
-    incr = (count*datatype_size)/adio_fh->etype_size;
-    
-    /* Use a message as a 'token' to order the operations */
-    source = myrank - 1;
-    dest   = myrank + 1;
-    if (source < 0) source = MPI_PROC_NULL;
-    if (dest >= nprocs) dest = MPI_PROC_NULL;
-    MPI_Recv(NULL, 0, MPI_BYTE, source, 0, adio_fh->comm, MPI_STATUS_IGNORE);
-
-    ADIO_Get_shared_fp(adio_fh, incr, &shared_fp, &error_code);
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS) {
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    MPI_Send(NULL, 0, MPI_BYTE, dest, 0, adio_fh->comm);
-
-    ADIO_ReadStridedColl(adio_fh, buf, count, datatype, ADIO_EXPLICIT_OFFSET,
-			 shared_fp, status, &error_code);
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-    /* --END ERROR HANDLING-- */
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-
-    /* FIXME: Check for error code from ReadStridedColl? */
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/read_ordb.c b/3rd-party/romio321/mpi-io/read_ordb.c
deleted file mode 100644
index 050934b731f..00000000000
--- a/3rd-party/romio321/mpi-io/read_ordb.c
+++ /dev/null
@@ -1,137 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_read_ordered_begin = PMPI_File_read_ordered_begin
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_read_ordered_begin MPI_File_read_ordered_begin
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_read_ordered_begin as PMPI_File_read_ordered_begin
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_read_ordered_begin(MPI_File fh, void *buf, int count, MPI_Datatype datatype)
-    __attribute__((weak,alias("PMPI_File_read_ordered_begin")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_read_ordered_begin - Begin a split collective read using shared file pointer
-
-Input Parameters:
-. fh - file handle (handle)
-. count - number of elements in buffer (nonnegative integer)
-. datatype - datatype of each buffer element (handle)
-
-Output Parameters:
-. buf - initial address of buffer (choice)
-
-.N fortran
-@*/
-int MPI_File_read_ordered_begin(MPI_File fh, void *buf, int count,
-				MPI_Datatype datatype)
-{
-    int error_code,  nprocs, myrank;
-    MPI_Count datatype_size;
-    int source, dest;
-    ADIO_Offset shared_fp, incr;
-    ADIO_File adio_fh;
-    static char myname[] = "MPI_FILE_READ_ORDERED_BEGIN";
-    void *xbuf=NULL, *e32_buf=NULL;
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    MPIO_CHECK_COUNT(adio_fh, count, myname, error_code);
-    MPIO_CHECK_DATATYPE(adio_fh, datatype, myname, error_code);
-
-    if (adio_fh->split_coll_count)
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_IO, 
-					  "**iosplitcoll", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    adio_fh->split_coll_count = 1;
-
-
-    MPI_Type_size_x(datatype, &datatype_size);
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_INTEGRAL_ETYPE(adio_fh, count, datatype_size, myname, error_code);
-    MPIO_CHECK_FS_SUPPORTS_SHARED(adio_fh, myname, error_code);
-    MPIO_CHECK_COUNT_SIZE(adio_fh, count, datatype_size, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    ADIOI_TEST_DEFERRED(adio_fh, myname, &error_code);
-
-    MPI_Comm_size(adio_fh->comm, &nprocs);
-    MPI_Comm_rank(adio_fh->comm, &myrank);
-
-    incr = (count*datatype_size)/adio_fh->etype_size;
-    /* Use a message as a 'token' to order the operations */
-    source = myrank - 1;
-    dest   = myrank + 1;
-    if (source < 0) source = MPI_PROC_NULL;
-    if (dest >= nprocs) dest = MPI_PROC_NULL;
-    MPI_Recv(NULL, 0, MPI_BYTE, source, 0, adio_fh->comm, MPI_STATUS_IGNORE);
-
-    ADIO_Get_shared_fp(adio_fh, incr, &shared_fp, &error_code);
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-    {
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    MPI_Send(NULL, 0, MPI_BYTE, dest, 0, adio_fh->comm);
-
-    xbuf = buf;
-    if (adio_fh->is_external32)
-    {
-        MPI_Aint e32_size = 0;
-        error_code = MPIU_datatype_full_size(datatype, &e32_size);
-        if (error_code != MPI_SUCCESS)
-            goto fn_exit;
-
-        e32_buf = ADIOI_Malloc(e32_size*count);
-	xbuf = e32_buf;
-    }
-
-
-    ADIO_ReadStridedColl(adio_fh, xbuf, count, datatype, ADIO_EXPLICIT_OFFSET,
-			 shared_fp, &adio_fh->split_status, &error_code);
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-    /* --END ERROR HANDLING-- */
-
-    if (e32_buf != NULL) {
-        error_code = MPIU_read_external32_conversion_fn(buf, datatype,
-                count, e32_buf);
-	ADIOI_Free(e32_buf);
-    }
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/read_orde.c b/3rd-party/romio321/mpi-io/read_orde.c
deleted file mode 100644
index 542f687bc20..00000000000
--- a/3rd-party/romio321/mpi-io/read_orde.c
+++ /dev/null
@@ -1,75 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_read_ordered_end = PMPI_File_read_ordered_end
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_read_ordered_end MPI_File_read_ordered_end
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_read_ordered_end as PMPI_File_read_ordered_end
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_read_ordered_end(MPI_File fh, void *buf, MPI_Status *status) __attribute__((weak,alias("PMPI_File_read_ordered_end")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_read_ordered_end - Complete a split collective read using shared file pointer
-
-Input Parameters:
-. fh - file handle (handle)
-
-Output Parameters:
-. buf - initial address of buffer (choice)
-. status - status object (Status)
-
-.N fortran
-@*/
-int MPI_File_read_ordered_end(MPI_File fh, void *buf, MPI_Status *status)
-{
-    int error_code=MPI_SUCCESS;
-    ADIO_File adio_fh;
-    static char myname[] = "MPI_FILE_READ_ORDERED_END";
-
-    MPIU_UNREFERENCED_ARG(buf);
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-
-    if (!(adio_fh->split_coll_count))
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_IO, 
-					  "**iosplitcollnone", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-#ifdef HAVE_STATUS_SET_BYTES
-    if (status != MPI_STATUS_IGNORE)
-       *status = adio_fh->split_status;
-#endif
-    adio_fh->split_coll_count = 0;
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/read_sh.c b/3rd-party/romio321/mpi-io/read_sh.c
deleted file mode 100644
index 83c29daa7e9..00000000000
--- a/3rd-party/romio321/mpi-io/read_sh.c
+++ /dev/null
@@ -1,155 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_read_shared = PMPI_File_read_shared
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_read_shared MPI_File_read_shared
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_read_shared as PMPI_File_read_shared
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_read_shared(MPI_File fh, void *buf, int count, MPI_Datatype datatype,
-                         MPI_Status *status) __attribute__((weak,alias("PMPI_File_read_shared")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/* status object not filled currently */
-
-/*@
-    MPI_File_read_shared - Read using shared file pointer
-
-Input Parameters:
-. fh - file handle (handle)
-. count - number of elements in buffer (nonnegative integer)
-. datatype - datatype of each buffer element (handle)
-
-Output Parameters:
-. buf - initial address of buffer (choice)
-. status - status object (Status)
-
-.N fortran
-@*/
-int MPI_File_read_shared(MPI_File fh, void *buf, int count,
-			 MPI_Datatype datatype, MPI_Status *status)
-{
-    int error_code, buftype_is_contig, filetype_is_contig;
-    static char myname[] = "MPI_FILE_READ_SHARED";
-    MPI_Count datatype_size;
-    ADIO_Offset off, shared_fp, incr, bufsize;
-    ADIO_File adio_fh;
-    void *xbuf=NULL, *e32_buf=NULL;
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    MPIO_CHECK_COUNT(adio_fh, count, myname, error_code);
-    MPIO_CHECK_DATATYPE(adio_fh, datatype, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    MPI_Type_size_x(datatype, &datatype_size);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_COUNT_SIZE(adio_fh, count, datatype_size, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    if (count*datatype_size == 0)
-    {
-#ifdef HAVE_STATUS_SET_BYTES
-	MPIR_Status_set_bytes(status, datatype, 0);
-#endif
-	error_code = MPI_SUCCESS;
-	goto fn_exit;
-    }
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_INTEGRAL_ETYPE(adio_fh, count, datatype_size, myname, error_code);
-    MPIO_CHECK_READABLE(adio_fh, myname, error_code);
-    MPIO_CHECK_FS_SUPPORTS_SHARED(adio_fh, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-    ADIOI_Datatype_iscontig(adio_fh->filetype, &filetype_is_contig);
-
-    ADIOI_TEST_DEFERRED(adio_fh, myname, &error_code);
-
-    incr = (count*datatype_size)/adio_fh->etype_size;
-
-    ADIO_Get_shared_fp(adio_fh, incr, &shared_fp, &error_code);
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-    {
-        error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    xbuf = buf;
-    if (adio_fh->is_external32)
-    {
-        MPI_Aint e32_size = 0;
-        error_code = MPIU_datatype_full_size(datatype, &e32_size);
-        if (error_code != MPI_SUCCESS)
-            goto fn_exit;
-
-        e32_buf = ADIOI_Malloc(e32_size*count);
-	xbuf = e32_buf;
-    }
-
-    /* contiguous or strided? */
-    if (buftype_is_contig && filetype_is_contig)
-    {
-	/* convert count and shared_fp to bytes */
-        bufsize = datatype_size * count;
-        off = adio_fh->disp + adio_fh->etype_size * shared_fp;
-
-        /* if atomic mode requested, lock (exclusive) the region, because there
-           could be a concurrent noncontiguous request. On NFS, locking 
-           is done in the ADIO_ReadContig.*/
-
-        if ((adio_fh->atomicity) && (adio_fh->file_system != ADIO_NFS))
-            ADIOI_WRITE_LOCK(adio_fh, off, SEEK_SET, bufsize);
-
-        ADIO_ReadContig(adio_fh, xbuf, count, datatype, ADIO_EXPLICIT_OFFSET,
-                        off, status, &error_code); 
-
-        if ((adio_fh->atomicity) && (adio_fh->file_system != ADIO_NFS))
-            ADIOI_UNLOCK(adio_fh, off, SEEK_SET, bufsize);
-    }
-    else
-    {
-	ADIO_ReadStrided(adio_fh, xbuf, count, datatype, ADIO_EXPLICIT_OFFSET,
-                          shared_fp, status, &error_code);
-	/* For strided and atomic mode, locking is done in ADIO_ReadStrided */
-    }
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-    /* --END ERROR HANDLING-- */
-
-    if (e32_buf != NULL) {
-        error_code = MPIU_read_external32_conversion_fn(buf, datatype,
-                count, e32_buf);
-	ADIOI_Free(e32_buf);
-    }
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/register_datarep.c b/3rd-party/romio321/mpi-io/register_datarep.c
deleted file mode 100644
index 38958723327..00000000000
--- a/3rd-party/romio321/mpi-io/register_datarep.c
+++ /dev/null
@@ -1,145 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-#include "adio_extern.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_Register_datarep = PMPI_Register_datarep
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_Register_datarep MPI_Register_datarep
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_Register_datarep as PMPI_Register_datarep
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_Register_datarep(const char *datarep, MPI_Datarep_conversion_function *read_conversion_fn,
-			 MPI_Datarep_conversion_function *write_conversion_fn,
-			 MPI_Datarep_extent_function *dtype_file_extent_fn, void *extra_state) __attribute__((weak,alias("PMPI_Register_datarep")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-  MPI_Register_datarep - Register functions for user-defined data 
-                         representations
-
-Input Parameters:
-+ datarep - data representation name (string)
-. read_conversion_fn - function invoked to convert from file representation to
-                 native representation (function)
-. write_conversion_fn - function invoked to convert from native representation to
-                  file representation (function)
-. dtype_file_extent_fn - function invoked to get the exted of a datatype as represented
-                  in the file (function)
-- extra_state - pointer to extra state that is passed to each of the
-                three functions
-
- Notes:
- This function allows the user to provide routines to convert data from
- an external representation, used within a file, and the native representation,
- used within the CPU.  There is one predefined data representation, 
- 'external32'.  Please consult the MPI-2 standard for details on this
- function.
-
-.N fortran
-  
-  @*/
-int MPI_Register_datarep(ROMIO_CONST char *datarep,
-			 MPI_Datarep_conversion_function *read_conversion_fn,
-			 MPI_Datarep_conversion_function *write_conversion_fn,
-			 MPI_Datarep_extent_function *dtype_file_extent_fn,
-			 void *extra_state)
-{
-    int error_code;
-    ADIOI_Datarep *adio_datarep;
-    static char myname[] = "MPI_REGISTER_DATAREP";
-
-    ROMIO_THREAD_CS_ENTER();
-
-    /* --BEGIN ERROR HANDLING-- */
-    /* check datarep name (use strlen instead of strnlen because
-       strnlen is not portable) */
-    if (datarep == NULL ||
-	strlen(datarep) < 1 ||
-	strlen(datarep) > MPI_MAX_DATAREP_STRING)
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					  MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__,
-					  MPI_ERR_ARG,
-					  "**datarepname", 0);
-	error_code = MPIO_Err_return_file(MPI_FILE_NULL, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    MPIR_MPIOInit(&error_code);
-    if (error_code != MPI_SUCCESS) goto fn_exit;
-
-    /* --BEGIN ERROR HANDLING-- */
-    /* check datarep isn't already registered */
-    for (adio_datarep = ADIOI_Datarep_head; adio_datarep; adio_datarep = adio_datarep->next) {
-	if (!strncmp(datarep, adio_datarep->name, MPI_MAX_DATAREP_STRING)) {
-	    error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					      MPIR_ERR_RECOVERABLE,
-					      myname, __LINE__,
-					      MPI_ERR_DUP_DATAREP,
-					      "**datarepused",
-					      "**datarepused %s",
-					      datarep);
-	    error_code = MPIO_Err_return_file(MPI_FILE_NULL, error_code);
-	    goto fn_exit;
-	}
-    }
-
-    /* Check Non-NULL Read and Write conversion function pointer */
-    /* Read and Write conversions are currently not supported.   */
-    if ( (read_conversion_fn != NULL) || (write_conversion_fn != NULL) )
-    {
-        error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-                                          myname, __LINE__,
-                                          MPI_ERR_CONVERSION,
-                                          "**drconvnotsupported", 0);
-
-	error_code = MPIO_Err_return_file(MPI_FILE_NULL, error_code);
-	goto fn_exit;
-    }
-
-    /* check extent function pointer */
-    if (dtype_file_extent_fn == NULL)
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					  MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__,
-					  MPI_ERR_ARG,
-					  "**datarepextent", 0);
-	error_code = MPIO_Err_return_file(MPI_FILE_NULL, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    adio_datarep = ADIOI_Malloc(sizeof(ADIOI_Datarep));
-    adio_datarep->name = ADIOI_Strdup(datarep);
-    adio_datarep->state         = extra_state;
-    adio_datarep->read_conv_fn  = read_conversion_fn;
-    adio_datarep->write_conv_fn = write_conversion_fn;
-    adio_datarep->extent_fn     = dtype_file_extent_fn;
-    adio_datarep->next          = ADIOI_Datarep_head;
-
-    ADIOI_Datarep_head = adio_datarep;
-
-    error_code = MPI_SUCCESS;
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/seek.c b/3rd-party/romio321/mpi-io/seek.c
deleted file mode 100644
index c68394a18a8..00000000000
--- a/3rd-party/romio321/mpi-io/seek.c
+++ /dev/null
@@ -1,139 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-#include "adioi.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_seek = PMPI_File_seek
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_seek MPI_File_seek
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_seek as PMPI_File_seek
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_seek(MPI_File fh, MPI_Offset offset, int whence) __attribute__((weak,alias("PMPI_File_seek")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_seek - Updates the individual file pointer
-
-Input Parameters:
-. fh - file handle (handle)
-. offset - file offset (integer)
-. whence - update mode (state)
-
-.N fortran
-@*/
-int MPI_File_seek(MPI_File fh, MPI_Offset offset, int whence)
-{
-    int error_code;
-    ADIO_File adio_fh;
-    static char myname[] = "MPI_FILE_SEEK";
-    MPI_Offset curr_offset, eof_offset;
-
-#ifdef MPI_hpux
-    int fl_xmpi;
-
-    HPMP_IO_START(fl_xmpi, BLKMPIFILESEEK, TRDTBLOCK, adio_fh, MPI_DATATYPE_NULL, -1);
-#endif /* MPI_hpux */
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    MPIO_CHECK_NOT_SEQUENTIAL_MODE(adio_fh, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    switch(whence) {
-    case MPI_SEEK_SET:
-	/* --BEGIN ERROR HANDLING-- */
-	if (offset < 0) {
-	    error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					      MPIR_ERR_RECOVERABLE, myname,
-					      __LINE__, MPI_ERR_ARG,
-					      "**iobadoffset", 0);
-	    error_code = MPIO_Err_return_file(adio_fh, error_code);
-	    goto fn_exit;
-	}
-	/* --END ERROR HANDLING-- */
-	break;
-    case MPI_SEEK_CUR:
-	/* find offset corr. to current location of file pointer */
-	ADIOI_Get_position(adio_fh, &curr_offset);
-	offset += curr_offset;
-
-	/* --BEGIN ERROR HANDLING-- */
-	if (offset < 0) {
-	    error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					      MPIR_ERR_RECOVERABLE, myname,
-					      __LINE__, MPI_ERR_ARG,
-					      "**ionegoffset", 0);
-	    error_code = MPIO_Err_return_file(adio_fh, error_code);
-	    goto fn_exit;
-	}
-	/* --END ERROR HANDLING-- */
-
-	break;
-    case MPI_SEEK_END:
-	/* we can in many cases do seeks w/o a file actually opened, but not in
-	 * the MPI_SEEK_END case */
-	ADIOI_TEST_DEFERRED(adio_fh, "MPI_File_seek", &error_code);
-
-	/* find offset corr. to end of file */
-	ADIOI_Get_eof_offset(adio_fh, &eof_offset);
-	offset += eof_offset;
-
-	/* --BEGIN ERROR HANDLING-- */
-	if (offset < 0) {
-	    error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					      MPIR_ERR_RECOVERABLE, myname,
-					      __LINE__, MPI_ERR_ARG,
-					      "**ionegoffset", 0);
-	    error_code = MPIO_Err_return_file(adio_fh, error_code);
-	    goto fn_exit;
-	}
-	/* --END ERROR HANDLING-- */
-
-	break;
-    default:
-	/* --BEGIN ERROR HANDLING-- */
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "**iobadwhence", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-	/* --END ERROR HANDLING-- */
-    }
-
-    ADIO_SeekIndividual(adio_fh, offset, ADIO_SEEK_SET, &error_code);
-    /* TODO: what do we do with this error? */
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-    /* --END ERROR HANDLING-- */
-
-#ifdef MPI_hpux
-    HPMP_IO_END(fl_xmpi, adio_fh, MPI_DATATYPE_NULL, -1);
-#endif /* MPI_hpux */
-
-    error_code = MPI_SUCCESS;
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/seek_sh.c b/3rd-party/romio321/mpi-io/seek_sh.c
deleted file mode 100644
index d3ce1ab8aa9..00000000000
--- a/3rd-party/romio321/mpi-io/seek_sh.c
+++ /dev/null
@@ -1,184 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_seek_shared = PMPI_File_seek_shared
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_seek_shared MPI_File_seek_shared
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_seek_shared as PMPI_File_seek_shared
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_seek_shared(MPI_File fh, MPI_Offset offset, int whence) __attribute__((weak,alias("PMPI_File_seek_shared")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_seek_shared - Updates the shared file pointer
-
-Input Parameters:
-. fh - file handle (handle)
-. offset - file offset (integer)
-. whence - update mode (state)
-
-.N fortran
-@*/
-int MPI_File_seek_shared(MPI_File fh, MPI_Offset offset, int whence)
-{
-    int error_code=MPI_SUCCESS, tmp_whence, myrank;
-    static char myname[] = "MPI_FILE_SEEK_SHARED";
-    MPI_Offset curr_offset, eof_offset, tmp_offset;
-    ADIO_File adio_fh;
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    MPIO_CHECK_NOT_SEQUENTIAL_MODE(adio_fh, myname, error_code);
-    MPIO_CHECK_FS_SUPPORTS_SHARED(adio_fh, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    tmp_offset = offset;
-    MPI_Bcast(&tmp_offset, 1, ADIO_OFFSET, 0, adio_fh->comm);
-    /* --BEGIN ERROR HANDLING-- */
-    if (tmp_offset != offset)
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "**notsame", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    tmp_whence = whence;
-    MPI_Bcast(&tmp_whence, 1, MPI_INT, 0, adio_fh->comm);
-    /* --BEGIN ERROR HANDLING-- */
-    if (tmp_whence != whence)
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "**iobadwhence", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    ADIOI_TEST_DEFERRED(adio_fh, "MPI_File_seek_shared", &error_code);
-
-    MPI_Comm_rank(adio_fh->comm, &myrank);
-
-    if (!myrank)
-    {
-	switch(whence)
-	{
-	case MPI_SEEK_SET:
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (offset < 0)
-	    {
-		error_code = MPIO_Err_create_code(MPI_SUCCESS,
-						  MPIR_ERR_RECOVERABLE,
-						  myname, __LINE__,
-						  MPI_ERR_ARG,
-						  "**iobadoffset", 0);
-		error_code = MPIO_Err_return_file(adio_fh, error_code);
-		goto fn_exit;
-	    }
-	    /* --END ERROR HANDLING-- */
-	    break;
-	case MPI_SEEK_CUR:
-	    /* get current location of shared file pointer */
-	    ADIO_Get_shared_fp(adio_fh, 0, &curr_offset, &error_code);
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (error_code != MPI_SUCCESS)
-	    {
-		error_code = MPIO_Err_create_code(MPI_SUCCESS,
-						  MPIR_ERR_FATAL,
-						  myname, __LINE__,
-						  MPI_ERR_INTERN, 
-						  "**iosharedfailed", 0);
-		error_code = MPIO_Err_return_file(adio_fh, error_code);
-		goto fn_exit;
-	    }
-	    /* --END ERROR HANDLING-- */
-	    offset += curr_offset;
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (offset < 0)
-	    {
-		error_code = MPIO_Err_create_code(MPI_SUCCESS,
-						  MPIR_ERR_RECOVERABLE,
-						  myname, __LINE__,
-						  MPI_ERR_ARG,
-						  "**ionegoffset", 0);
-		error_code = MPIO_Err_return_file(adio_fh, error_code);
-		goto fn_exit;
-	    }
-	    /* --END ERROR HANDLING-- */
-	    break;
-	case MPI_SEEK_END:
-	    /* find offset corr. to end of file */
-	    ADIOI_Get_eof_offset(adio_fh, &eof_offset);
-	    offset += eof_offset;
-	    /* --BEGIN ERROR HANDLING-- */
-	    if (offset < 0)
-	    {
-		error_code = MPIO_Err_create_code(MPI_SUCCESS,
-						  MPIR_ERR_RECOVERABLE,
-						  myname, __LINE__,
-						  MPI_ERR_ARG,
-						  "**ionegoffset", 0);
-		error_code = MPIO_Err_return_file(adio_fh, error_code);
-		goto fn_exit;
-	    }
-	    /* --END ERROR HANDLING-- */
-	    break;
-	default:
-	    /* --BEGIN ERROR HANDLING-- */
-	    error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					      MPIR_ERR_RECOVERABLE,
-					      myname, __LINE__, MPI_ERR_ARG,
-					      "**iobadwhence", 0);
-	    error_code = MPIO_Err_return_file(adio_fh, error_code);
-	    goto fn_exit;
-	    /* --END ERROR HANDLING-- */
-	}
-
-	ADIO_Set_shared_fp(adio_fh, offset, &error_code);
-	/* --BEGIN ERROR HANDLING-- */
-	if (error_code != MPI_SUCCESS)
-	{
-	    error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					      MPIR_ERR_FATAL,
-					      myname, __LINE__,
-					      MPI_ERR_INTERN, 
-					      "**iosharedfailed", 0);
-	    error_code = MPIO_Err_return_file(adio_fh, error_code);
-	    goto fn_exit;
-	}
-	/* --END ERROR HANDLING-- */
-
-    }
-
-    /* FIXME: explain why the barrier is necessary */
-    MPI_Barrier(adio_fh->comm);
-
-    error_code = MPI_SUCCESS;
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/set_atom.c b/3rd-party/romio321/mpi-io/set_atom.c
deleted file mode 100644
index 00be9fc35b3..00000000000
--- a/3rd-party/romio321/mpi-io/set_atom.c
+++ /dev/null
@@ -1,91 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_set_atomicity = PMPI_File_set_atomicity
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_set_atomicity MPI_File_set_atomicity
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_set_atomicity as PMPI_File_set_atomicity
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_set_atomicity(MPI_File fh, int flag) __attribute__((weak,alias("PMPI_File_set_atomicity")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_set_atomicity - Sets the atomicity mode
-
-Input Parameters:
-. fh - file handle (handle)
-. flag - true to set atomic mode, false to set nonatomic mode (logical)
-
-.N fortran
-@*/
-int MPI_File_set_atomicity(MPI_File fh, int flag)
-{
-    int error_code, tmp_flag;
-    static char myname[] = "MPI_FILE_SET_ATOMICITY";
-    ADIO_Fcntl_t *fcntl_struct;
-    ADIO_File adio_fh;
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    ADIOI_TEST_DEFERRED(adio_fh, myname, &error_code);
-
-    if (flag) flag = 1;  /* take care of non-one values! */
-
-/* check if flag is the same on all processes */
-    tmp_flag = flag;
-    MPI_Bcast(&tmp_flag, 1, MPI_INT, 0, adio_fh->comm);
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (tmp_flag != flag) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG, 
-					  "**notsame", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    if (adio_fh->atomicity == flag){
-	    error_code = MPI_SUCCESS;
-	    goto fn_exit;
-    }
-
-
-    fcntl_struct = (ADIO_Fcntl_t *) ADIOI_Malloc(sizeof(ADIO_Fcntl_t));
-    fcntl_struct->atomicity = flag;
-    ADIO_Fcntl(adio_fh, ADIO_FCNTL_SET_ATOMICITY, fcntl_struct, &error_code);
-    /* TODO: what do we do with this error code? */
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-    /* --END ERROR HANDLING-- */
-
-    ADIOI_Free(fcntl_struct);
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/set_errh.c b/3rd-party/romio321/mpi-io/set_errh.c
deleted file mode 100644
index d943787e358..00000000000
--- a/3rd-party/romio321/mpi-io/set_errh.c
+++ /dev/null
@@ -1,76 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-#include "adio_extern.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_set_errhandler = PMPI_File_set_errhandler
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_set_errhandler MPI_File_set_errhandler
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_set_errhandler as PMPI_File_set_errhandler
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_set_errhandler(MPI_File file, MPI_Errhandler errhandler) __attribute__((weak,alias("PMPI_File_set_errhandler")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_set_errhandler - Sets the error handler for a file
-
-Input Parameters:
-. fh - file handle (handle)
-. errhandler - error handler (handle)
-
-.N fortran
-@*/
-int MPI_File_set_errhandler(MPI_File mpi_fh, MPI_Errhandler errhandler)
-{
-    int error_code = MPI_SUCCESS;
-    static char myname[] = "MPI_FILE_SET_ERRHANDLER";
-    ADIO_File fh;
-    MPID_THREADPRIV_DECL;
-
-    ROMIO_THREAD_CS_ENTER();
-
-    if (mpi_fh == MPI_FILE_NULL) {
-	ADIOI_DFLT_ERR_HANDLER = errhandler;
-    }
-    else {
-	fh = MPIO_File_resolve(mpi_fh);
-
-	/* --BEGIN ERROR HANDLING-- */
-	MPIO_CHECK_FILE_HANDLE(fh, myname, error_code);
-	/* --END ERROR HANDLING-- */
-
-	if ((errhandler != MPI_ERRORS_RETURN) &&
-	    (errhandler != MPI_ERRORS_ARE_FATAL))
-	{
-	    error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					      MPIR_ERR_RECOVERABLE,
-					      myname, __LINE__,
-					      MPI_ERR_UNSUPPORTED_OPERATION,
-					      "**fileopunsupported",
-					      0);
-	    error_code = MPIO_Err_return_file(fh, error_code);
-	    goto fn_exit;
-	}
-
-	fh->err_handler = errhandler;
-    }
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/set_info.c b/3rd-party/romio321/mpi-io/set_info.c
deleted file mode 100644
index 2c898fe85ac..00000000000
--- a/3rd-party/romio321/mpi-io/set_info.c
+++ /dev/null
@@ -1,66 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_set_info = PMPI_File_set_info
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_set_info MPI_File_set_info
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_set_info as PMPI_File_set_info
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_set_info(MPI_File fh, MPI_Info info) __attribute__((weak,alias("PMPI_File_set_info")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_set_info - Sets new values for the hints associated with a file
-
-Input Parameters:
-. fh - file handle (handle)
-. info - info object (handle)
-
-.N fortran
-@*/
-int MPI_File_set_info(MPI_File fh, MPI_Info info)
-{
-    int error_code;
-    static char myname[] = "MPI_FILE_SET_INFO";
-    ADIO_File adio_fh;
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    MPIO_CHECK_INFO_ALL(info, error_code, fh->comm);
-    /* --END ERROR HANDLING-- */
-
-    /* set new info */
-    ADIO_SetInfo(adio_fh, info, &error_code);
-
-fn_exit:
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-    /* --END ERROR HANDLING-- */
-
-    ROMIO_THREAD_CS_EXIT();
-
-    return error_code;
-fn_fail:
-    goto fn_exit;
-}
diff --git a/3rd-party/romio321/mpi-io/set_size.c b/3rd-party/romio321/mpi-io/set_size.c
deleted file mode 100644
index b658929576d..00000000000
--- a/3rd-party/romio321/mpi-io/set_size.c
+++ /dev/null
@@ -1,105 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_set_size = PMPI_File_set_size
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_set_size MPI_File_set_size
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_set_size as PMPI_File_set_size
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_set_size(MPI_File fh, MPI_Offset size) __attribute__((weak,alias("PMPI_File_set_size")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_set_size - Sets the file size
-
-Input Parameters:
-. fh - file handle (handle)
-. size - size to truncate or expand file (nonnegative integer)
-
-.N fortran
-@*/
-int MPI_File_set_size(MPI_File fh, MPI_Offset size)
-{
-    int error_code;
-    ADIO_File adio_fh;
-    static char myname[] = "MPI_FILE_SET_SIZE";
-    MPI_Offset tmp_sz, max_sz, min_sz;
-
-#ifdef MPI_hpux
-    int fl_xmpi;
-
-    HPMP_IO_START(fl_xmpi, BLKMPIFILESETSIZE, TRDTBLOCK, adio_fh,
-		  MPI_DATATYPE_NULL, -1);
-#endif /* MPI_hpux */
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    MPIO_CHECK_NOT_SEQUENTIAL_MODE(adio_fh, myname, error_code);
-
-    if (size < 0) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "**iobadsize", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    MPIO_CHECK_WRITABLE(fh, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    tmp_sz = size;
-    MPI_Allreduce(&tmp_sz, &max_sz, 1, ADIO_OFFSET, MPI_MAX, adio_fh->comm);
-    MPI_Allreduce(&tmp_sz, &min_sz, 1, ADIO_OFFSET, MPI_MIN, adio_fh->comm);
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (max_sz != min_sz) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "**notsame", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    if (!ADIO_Feature(adio_fh, ADIO_SCALABLE_RESIZE)) {
-	/* rare stupid file systems (like NFS) need to carry out resize on all
-	 * processes */
-	ADIOI_TEST_DEFERRED(adio_fh, "MPI_File_set_size", &error_code);
-    }
-
-    ADIO_Resize(adio_fh, size, &error_code);
-    /* TODO: what to do with error code? */
-    
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-    /* --END ERROR HANDLING-- */
-
-#ifdef MPI_hpux
-    HPMP_IO_END(fl_xmpi, adio_fh, MPI_DATATYPE_NULL, -1);
-#endif /* MPI_hpux */
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/set_view.c b/3rd-party/romio321/mpi-io/set_view.c
deleted file mode 100644
index 4d98fdd1844..00000000000
--- a/3rd-party/romio321/mpi-io/set_view.c
+++ /dev/null
@@ -1,204 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_set_view = PMPI_File_set_view
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_set_view MPI_File_set_view
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_set_view as PMPI_File_set_view
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_set_view(MPI_File fh, MPI_Offset disp, MPI_Datatype etype, MPI_Datatype filetype,
-                      const char *datarep, MPI_Info info) __attribute__((weak,alias("PMPI_File_set_view")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_set_view - Sets the file view
-
-Input Parameters:
-. fh - file handle (handle)
-. disp - displacement (nonnegative integer)
-. etype - elementary datatype (handle)
-. filetype - filetype (handle)
-. datarep - data representation (string)
-. info - info object (handle)
-
-.N fortran
-@*/
-int MPI_File_set_view(MPI_File fh, MPI_Offset disp, MPI_Datatype etype,
-		      MPI_Datatype filetype, ROMIO_CONST char *datarep, MPI_Info info)
-{
-    int error_code;
-    MPI_Count filetype_size, etype_size;
-    static char myname[] = "MPI_FILE_SET_VIEW";
-    ADIO_Offset shared_fp, byte_off;
-    ADIO_File adio_fh;
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-
-    if ((disp < 0) && (disp != MPI_DISPLACEMENT_CURRENT))
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG, 
-					  "**iobaddisp", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-
-    /* rudimentary checks for incorrect etype/filetype.*/
-    if (etype == MPI_DATATYPE_NULL) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "**ioetype", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    MPIO_DATATYPE_ISCOMMITTED(etype, error_code);
-    if (error_code != MPI_SUCCESS) {
-        error_code = MPIO_Err_return_file(adio_fh, error_code);
-        goto fn_exit;
-    }
-
-    if (filetype == MPI_DATATYPE_NULL) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "**iofiletype", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    MPIO_DATATYPE_ISCOMMITTED(filetype, error_code);
-    if (error_code != MPI_SUCCESS) {
-        error_code = MPIO_Err_return_file(adio_fh, error_code);
-        goto fn_exit;
-    }
-
-    if ((adio_fh->access_mode & MPI_MODE_SEQUENTIAL) &&
-	(disp != MPI_DISPLACEMENT_CURRENT))
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG, 
-					  "**iodispifseq", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-
-    if ((disp == MPI_DISPLACEMENT_CURRENT) &&
-	!(adio_fh->access_mode & MPI_MODE_SEQUENTIAL))
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG, 
-					  "**iodispifseq", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    MPIO_CHECK_INFO_ALL(info, error_code, adio_fh->comm);
-    /* --END ERROR HANDLING-- */
-
-    MPI_Type_size_x(filetype, &filetype_size);
-    MPI_Type_size_x(etype, &etype_size);
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (etype_size != 0 && filetype_size % etype_size != 0)
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "**iofiletype", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-
-    if ((datarep == NULL) || (strcmp(datarep, "native") &&
-	    strcmp(datarep, "NATIVE") &&
-	    strcmp(datarep, "external32") &&
-	    strcmp(datarep, "EXTERNAL32") &&
-	    strcmp(datarep, "internal") &&
-	    strcmp(datarep, "INTERNAL")) )
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__,
-					  MPI_ERR_UNSUPPORTED_DATAREP, 
-					  "**unsupporteddatarep",0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    if (disp == MPI_DISPLACEMENT_CURRENT) {
-	MPI_Barrier(adio_fh->comm);
-	ADIO_Get_shared_fp(adio_fh, 0, &shared_fp, &error_code);
-	/* TODO: check error code */
-
-	MPI_Barrier(adio_fh->comm);
-	ADIOI_Get_byte_offset(adio_fh, shared_fp, &byte_off);
-	/* TODO: check error code */
-
-	disp = byte_off;
-    }
-
-    ADIO_Set_view(adio_fh, disp, etype, filetype, info, &error_code);
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS) {
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    /* reset shared file pointer to zero */
-    if (ADIO_Feature(adio_fh, ADIO_SHARED_FP) &&
-        (adio_fh->shared_fp_fd != ADIO_FILE_NULL))
-    {
-	/* only one process needs to set it to zero, but I don't want to 
-	   create the shared-file-pointer file if shared file pointers have 
-	   not been used so far. Therefore, every process that has already 
-	   opened the shared-file-pointer file sets the shared file pointer 
-	   to zero. If the file was not opened, the value is automatically 
-	   zero. Note that shared file pointer is stored as no. of etypes
-	   relative to the current view, whereas indiv. file pointer is
-	   stored in bytes. */
-
-	ADIO_Set_shared_fp(adio_fh, 0, &error_code);
-	/* --BEGIN ERROR HANDLING-- */
-	if (error_code != MPI_SUCCESS)
-	    error_code = MPIO_Err_return_file(adio_fh, error_code);
-	/* --END ERROR HANDLING-- */
-    }
-
-    if (ADIO_Feature(adio_fh, ADIO_SHARED_FP))
-    {
-	MPI_Barrier(adio_fh->comm); /* for above to work correctly */
-    }
-    if (strcmp(datarep, "external32") && strcmp(datarep, "EXTERNAL32"))
-	adio_fh->is_external32 = 0;
-    else
-	adio_fh->is_external32 = 1;
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-
-    return error_code;
-fn_fail:
-	/* --BEGIN ERROR HANDLING-- */
-	error_code = MPIO_Err_return_file(fh, error_code);
-	goto fn_exit;
-	/* --END ERROR HANDLING-- */
-}
diff --git a/3rd-party/romio321/mpi-io/wr_atallb.c b/3rd-party/romio321/mpi-io/wr_atallb.c
deleted file mode 100644
index 83eb28bc886..00000000000
--- a/3rd-party/romio321/mpi-io/wr_atallb.c
+++ /dev/null
@@ -1,53 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_write_at_all_begin = PMPI_File_write_at_all_begin
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_write_at_all_begin MPI_File_write_at_all_begin
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_write_at_all_begin as PMPI_File_write_at_all_begin
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_write_at_all_begin(MPI_File fh, MPI_Offset offset, const void *buf, int count,
-                                MPI_Datatype datatype) __attribute__((weak,alias("PMPI_File_write_at_all_begin")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_write_at_all_begin - Begin a split collective write using
-    explicit offset
-
-Input Parameters:
-. fh - file handle (handle)
-. offset - file offset (nonnegative integer)
-. buf - initial address of buffer (choice)
-. count - number of elements in buffer (nonnegative integer)
-. datatype - datatype of each buffer element (handle)
-
-.N fortran
-@*/
-int MPI_File_write_at_all_begin(MPI_File fh, MPI_Offset offset, ROMIO_CONST void *buf,
-				int count, MPI_Datatype datatype)
-{
-    int error_code;
-    static char myname[] = "MPI_FILE_WRITE_AT_ALL_BEGIN";
-
-    error_code = MPIOI_File_write_all_begin(fh, offset,
-					    ADIO_EXPLICIT_OFFSET,
-					    buf, count, datatype, myname);
-
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/wr_atalle.c b/3rd-party/romio321/mpi-io/wr_atalle.c
deleted file mode 100644
index fa2ab514231..00000000000
--- a/3rd-party/romio321/mpi-io/wr_atalle.c
+++ /dev/null
@@ -1,48 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_write_at_all_end = PMPI_File_write_at_all_end
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_write_at_all_end MPI_File_write_at_all_end
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_write_at_all_end as PMPI_File_write_at_all_end
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_write_at_all_end(MPI_File fh, const void *buf, MPI_Status *status) __attribute__((weak,alias("PMPI_File_write_at_all_end")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_write_at_all_end - Complete a split collective write using explicit offset
-
-Input Parameters:
-. fh - file handle (handle)
-. buf - initial address of buffer (choice)
-
-Output Parameters:
-. status - status object (Status)
-
-.N fortran
-@*/
-int MPI_File_write_at_all_end(MPI_File fh, ROMIO_CONST void *buf, MPI_Status *status)
-{
-    int error_code;
-    static char myname[] = "MPI_FILE_WRITE_AT_ALL_END";
-
-    error_code = MPIOI_File_write_all_end(fh, buf, myname, status);
-
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/write.c b/3rd-party/romio321/mpi-io/write.c
deleted file mode 100644
index eb9a77fb82d..00000000000
--- a/3rd-party/romio321/mpi-io/write.c
+++ /dev/null
@@ -1,186 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_write = PMPI_File_write
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_write MPI_File_write
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_write as PMPI_File_write
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_write(MPI_File fh, const void *buf, int count, MPI_Datatype datatype,
-                   MPI_Status *status) __attribute__((weak,alias("PMPI_File_write")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/* status object not filled currently */
-
-/*@
-    MPI_File_write - Write using individual file pointer
-
-Input Parameters:
-. fh - file handle (handle)
-. buf - initial address of buffer (choice)
-. count - number of elements in buffer (nonnegative integer)
-. datatype - datatype of each buffer element (handle)
-
-Output Parameters:
-. status - status object (Status)
-
-.N fortran
-@*/
-int MPI_File_write(MPI_File fh, ROMIO_CONST void *buf, int count,
-                   MPI_Datatype datatype, MPI_Status *status)
-{
-    int error_code;
-    static char myname[] = "MPI_FILE_WRITE";
-#ifdef MPI_hpux
-    int fl_xmpi;
-
-    HPMP_IO_START(fl_xmpi, BLKMPIFILEWRITE, TRDTBLOCK, fh, datatype, count);
-#endif /* MPI_hpux */
-
-    error_code = MPIOI_File_write(fh, (MPI_Offset) 0, ADIO_INDIVIDUAL, buf,
-				  count, datatype, myname, status);
-
-#ifdef MPI_hpux
-    HPMP_IO_END(fl_xmpi, fh, datatype, count);
-#endif /* MPI_hpux */
-
-    return error_code;
-}
-
-/* prevent multiple definitions of this routine */
-#ifdef MPIO_BUILD_PROFILING
-int MPIOI_File_write(MPI_File fh,
-		     MPI_Offset offset,
-		     int file_ptr_type,
-		     const void *buf,
-		     int count,
-		     MPI_Datatype datatype,
-		     char *myname,
-		     MPI_Status *status)
-{		      
-    int error_code, buftype_is_contig, filetype_is_contig;
-    MPI_Count datatype_size;
-    ADIO_Offset off, bufsize;
-    ADIO_File adio_fh;
-    void *e32buf=NULL;
-    const void *xbuf=NULL;
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    MPIO_CHECK_COUNT(adio_fh, count, myname, error_code);
-    MPIO_CHECK_DATATYPE(adio_fh, datatype, myname, error_code);
-
-    if (file_ptr_type == ADIO_EXPLICIT_OFFSET && offset < 0)
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "**iobadoffset", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    MPI_Type_size_x(datatype, &datatype_size);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_COUNT_SIZE(adio_fh, count, datatype_size, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    if (count*datatype_size == 0)
-    {
-#ifdef HAVE_STATUS_SET_BYTES
-       MPIR_Status_set_bytes(status, datatype, 0);
-#endif
-	error_code = MPI_SUCCESS;
-	goto fn_exit;
-    }
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_INTEGRAL_ETYPE(adio_fh, count, datatype_size, myname, error_code);
-    MPIO_CHECK_WRITABLE(adio_fh, myname, error_code);
-    MPIO_CHECK_NOT_SEQUENTIAL_MODE(adio_fh, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-    ADIOI_Datatype_iscontig(adio_fh->filetype, &filetype_is_contig);
-
-    ADIOI_TEST_DEFERRED(adio_fh, myname, &error_code);
-
-    xbuf = buf;
-    if (adio_fh->is_external32) {
-	error_code = MPIU_external32_buffer_setup(buf, count, datatype, &e32buf);
-	if (error_code != MPI_SUCCESS) 
-	    goto fn_exit;
-
-	xbuf = e32buf;
-    }
-
-    if (buftype_is_contig && filetype_is_contig)
-    {
-	/* convert bufcount and offset to bytes */
-	bufsize = datatype_size * count;
-	if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
-	    off = adio_fh->disp + adio_fh->etype_size * offset;
-	}
-	else /* ADIO_INDIVIDUAL */ {
-	    off = adio_fh->fp_ind;
-	}
-
-        /* if atomic mode requested, lock (exclusive) the region, because
-           there could be a concurrent noncontiguous request. Locking doesn't
-           work on PIOFS and PVFS, and on NFS it is done in the
-           ADIO_WriteContig.
-	 */
-
-        if ((adio_fh->atomicity) && ADIO_Feature(adio_fh, ADIO_LOCKS))
-	{
-            ADIOI_WRITE_LOCK(adio_fh, off, SEEK_SET, bufsize);
-	}
-
-	ADIO_WriteContig(adio_fh, xbuf, count, datatype, file_ptr_type,
-		     off, status, &error_code); 
-
-        if ((adio_fh->atomicity) && ADIO_Feature(adio_fh, ADIO_LOCKS))
-	{
-            ADIOI_UNLOCK(adio_fh, off, SEEK_SET, bufsize);
-	}
-    }
-    else
-    {
-	/* For strided and atomic mode, locking is done in ADIO_WriteStrided */
-	ADIO_WriteStrided(adio_fh, xbuf, count, datatype, file_ptr_type,
-			  offset, status, &error_code);
-    }
-
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-    /* --END ERROR HANDLING-- */
-
-fn_exit:
-    if (e32buf!= NULL) ADIOI_Free(e32buf);
-    ROMIO_THREAD_CS_EXIT();
-
-    return error_code;
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/write_all.c b/3rd-party/romio321/mpi-io/write_all.c
deleted file mode 100644
index 6f17cc30ad9..00000000000
--- a/3rd-party/romio321/mpi-io/write_all.c
+++ /dev/null
@@ -1,134 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_write_all = PMPI_File_write_all
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_write_all MPI_File_write_all
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_write_all as PMPI_File_write_all
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_write_all(MPI_File fh, const void *buf, int count, MPI_Datatype datatype,
-                       MPI_Status *status) __attribute__((weak,alias("PMPI_File_write_all")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/* status object not filled currently */
-
-/*@
-    MPI_File_write_all - Collective write using individual file pointer
-
-Input Parameters:
-. fh - file handle (handle)
-. buf - initial address of buffer (choice)
-. count - number of elements in buffer (nonnegative integer)
-. datatype - datatype of each buffer element (handle)
-
-Output Parameters:
-. status - status object (Status)
-
-.N fortran
-@*/
-int MPI_File_write_all(MPI_File fh, ROMIO_CONST void *buf, int count,
-                       MPI_Datatype datatype, MPI_Status *status)
-{
-    int error_code;
-    static char myname[] = "MPI_FILE_WRITE_ALL";
-#ifdef MPI_hpux
-    int fl_xmpi;
-
-    HPMP_IO_START(fl_xmpi, BLKMPIFILEWRITEALL, TRDTBLOCK, fh, datatype, count);
-#endif /* MPI_hpux */
-
-    error_code = MPIOI_File_write_all(fh, (MPI_Offset) 0,
-				      ADIO_INDIVIDUAL, buf,
-				      count, datatype, myname, status);
-
-#ifdef MPI_hpux
-    HPMP_IO_END(fl_xmpi, fh, datatype, count);
-#endif /* MPI_hpux */
-
-    return error_code;
-}
-
-/* prevent multiple definitions of this routine */
-#ifdef MPIO_BUILD_PROFILING
-int MPIOI_File_write_all(MPI_File fh,
-			 MPI_Offset offset,
-			 int file_ptr_type,
-			 const void *buf,
-			 int count,
-			 MPI_Datatype datatype,
-			 char *myname,
-			 MPI_Status *status)
-{
-    int error_code;
-    MPI_Count datatype_size;
-    ADIO_File adio_fh;
-    void *e32buf=NULL;
-    const void *xbuf=NULL;
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    MPIO_CHECK_COUNT(adio_fh, count, myname, error_code);
-    MPIO_CHECK_DATATYPE(adio_fh, datatype, myname, error_code);
-
-    if (file_ptr_type == ADIO_EXPLICIT_OFFSET && offset < 0)
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "**iobadoffset", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    MPI_Type_size_x(datatype, &datatype_size);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_INTEGRAL_ETYPE(adio_fh, count, datatype_size, myname, error_code);
-    MPIO_CHECK_WRITABLE(adio_fh, myname, error_code);
-    MPIO_CHECK_NOT_SEQUENTIAL_MODE(adio_fh, myname, error_code);
-    MPIO_CHECK_COUNT_SIZE(adio_fh, count, datatype_size, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    xbuf = buf;
-    if (adio_fh->is_external32) {
-	error_code = MPIU_external32_buffer_setup(buf, count, datatype, &e32buf);
-	if (error_code != MPI_SUCCESS) 
-	    goto fn_exit;
-
-	xbuf = e32buf;
-    }
-    ADIO_WriteStridedColl(adio_fh, xbuf, count, datatype, file_ptr_type,
-                          offset, status, &error_code);
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-    /* --END ERROR HANDLING-- */
-
-fn_exit:
-    if (e32buf != NULL) ADIOI_Free(e32buf);
-    ROMIO_THREAD_CS_EXIT();
-
-    return error_code;
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/write_allb.c b/3rd-party/romio321/mpi-io/write_allb.c
deleted file mode 100644
index e184723a876..00000000000
--- a/3rd-party/romio321/mpi-io/write_allb.c
+++ /dev/null
@@ -1,132 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_write_all_begin = PMPI_File_write_all_begin
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_write_all_begin MPI_File_write_all_begin
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_write_all_begin as PMPI_File_write_all_begin
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_write_all_begin(MPI_File fh, const void *buf, int count, MPI_Datatype datatype)
-    __attribute__((weak,alias("PMPI_File_write_all_begin")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_write_all_begin - Begin a split collective write using
-    individual file pointer
-
-Input Parameters:
-. fh - file handle (handle)
-. buf - initial address of buffer (choice)
-. count - number of elements in buffer (nonnegative integer)
-. datatype - datatype of each buffer element (handle)
-
-.N fortran
-@*/
-int MPI_File_write_all_begin(MPI_File fh, ROMIO_CONST void *buf, int count,
-			     MPI_Datatype datatype)
-{
-    int error_code;
-    static char myname[] = "MPI_FILE_WRITE_ALL_BEGIN";
-
-    error_code = MPIOI_File_write_all_begin(fh, (MPI_Offset) 0,
-					    ADIO_INDIVIDUAL, buf, count,
-					    datatype, myname);
-
-    return error_code;
-}
-
-/* prevent multiple definitions of this routine */
-#ifdef MPIO_BUILD_PROFILING
-int MPIOI_File_write_all_begin(MPI_File fh,
-			       MPI_Offset offset,
-			       int file_ptr_type,
-			       const void *buf,
-			       int count,
-			       MPI_Datatype datatype,
-			       char *myname)
-{
-    int error_code;
-    MPI_Count datatype_size;
-    ADIO_File adio_fh;
-    void *e32buf=NULL;
-    const void *xbuf=NULL;
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    MPIO_CHECK_COUNT(adio_fh, count, myname, error_code);
-    MPIO_CHECK_DATATYPE(adio_fh, datatype, myname, error_code);
-    MPIO_CHECK_NOT_SEQUENTIAL_MODE(adio_fh, myname, error_code);
-
-    if (file_ptr_type == ADIO_EXPLICIT_OFFSET && offset < 0)
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "**iobadoffset", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-
-    if (adio_fh->split_coll_count)
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_IO, 
-					  "**iosplitcoll", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    adio_fh->split_coll_count = 1;
-
-    MPI_Type_size_x(datatype, &datatype_size);
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_INTEGRAL_ETYPE(adio_fh, count, datatype_size, myname, error_code);
-    MPIO_CHECK_COUNT_SIZE(adio_fh, count, datatype_size, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-
-    xbuf = buf;
-    if (adio_fh->is_external32) {
-	error_code = MPIU_external32_buffer_setup(buf, count, datatype, &e32buf);
-	if (error_code != MPI_SUCCESS) 
-	    goto fn_exit;
-
-	xbuf = e32buf;
-    }
-
-    adio_fh->split_datatype = datatype;
-    ADIO_WriteStridedColl(adio_fh, xbuf, count, datatype, file_ptr_type,
-			  offset, &adio_fh->split_status, &error_code);
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-    /* --END ERROR HANDLING-- */
-
-fn_exit:
-    if ( e32buf != NULL) ADIOI_Free(e32buf);
-    ROMIO_THREAD_CS_EXIT();
-
-    return error_code;
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/write_alle.c b/3rd-party/romio321/mpi-io/write_alle.c
deleted file mode 100644
index 6dc7f6d1566..00000000000
--- a/3rd-party/romio321/mpi-io/write_alle.c
+++ /dev/null
@@ -1,95 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_write_all_end = PMPI_File_write_all_end
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_write_all_end MPI_File_write_all_end
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_write_all_end as PMPI_File_write_all_end
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_write_all_end(MPI_File fh, const void *buf, MPI_Status *status) __attribute__((weak,alias("PMPI_File_write_all_end")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_write_all_end - Complete a split collective write using individual file pointer
-
-Input Parameters:
-. fh - file handle (handle)
-
-Output Parameters:
-. buf - initial address of buffer (choice)
-. status - status object (Status)
-
-.N fortran
-@*/
-int MPI_File_write_all_end(MPI_File fh, ROMIO_CONST void *buf, MPI_Status *status)
-{
-    int error_code;
-    static char myname[] = "MPI_FILE_WRITE_ALL_END";
-
-    error_code = MPIOI_File_write_all_end(fh, buf, myname, status);
-
-    return error_code;
-}
-
-/* prevent multiple definitions of this routine */
-#ifdef MPIO_BUILD_PROFILING
-int MPIOI_File_write_all_end(MPI_File fh,
-			     const void *buf,
-			     char *myname,
-			     MPI_Status *status)
-{
-    int error_code;
-    ADIO_File adio_fh;
-
-    MPIU_UNREFERENCED_ARG(buf);
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-
-    if (!(adio_fh->split_coll_count))
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_IO, 
-					  "**iosplitcollnone", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-#ifdef HAVE_STATUS_SET_BYTES
-    /* FIXME - we should really ensure that the split_datatype remains
-       valid by incrementing the ref count in the write_allb.c routine
-       and decrement it here after setting the bytes */
-    if (status != MPI_STATUS_IGNORE)
-       *status = adio_fh->split_status;
-#endif
-    adio_fh->split_coll_count = 0;
-
-    error_code = MPI_SUCCESS;
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-
-    return error_code;
-}
-#endif
diff --git a/3rd-party/romio321/mpi-io/write_at.c b/3rd-party/romio321/mpi-io/write_at.c
deleted file mode 100644
index 4362322b67b..00000000000
--- a/3rd-party/romio321/mpi-io/write_at.c
+++ /dev/null
@@ -1,66 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_write_at = PMPI_File_write_at
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_write_at MPI_File_write_at
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_write_at as PMPI_File_write_at
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_write_at(MPI_File fh, MPI_Offset offset, const void * buf, int count,
-                      MPI_Datatype datatype, MPI_Status *status)
-    __attribute__((weak,alias("PMPI_File_write_at")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/* status object not filled currently */
-
-/*@
-    MPI_File_write_at - Write using explicit offset
-
-Input Parameters:
-. fh - file handle (handle)
-. offset - file offset (nonnegative integer)
-. buf - initial address of buffer (choice)
-. count - number of elements in buffer (nonnegative integer)
-. datatype - datatype of each buffer element (handle)
-
-Output Parameters:
-. status - status object (Status)
-
-.N fortran
-@*/
-int MPI_File_write_at(MPI_File fh, MPI_Offset offset, ROMIO_CONST void *buf,
-                      int count, MPI_Datatype datatype, MPI_Status *status)
-{
-    int error_code;
-    static char myname[] = "MPI_FILE_WRITE_AT";
-#ifdef MPI_hpux
-    int fl_xmpi;
-
-    HPMP_IO_START(fl_xmpi, BLKMPIFILEWRITEAT, TRDTBLOCK, fh, datatype, count);
-#endif /* MPI_hpux */
-
-    /* MPIOI_File_write() defined in mpi-io/write.c */
-    error_code = MPIOI_File_write(fh, offset, ADIO_EXPLICIT_OFFSET, buf,
-				  count, datatype, myname, status);
-
-#ifdef MPI_hpux
-    HPMP_IO_END(fl_xmpi, fh, datatype, count);
-#endif /* MPI_hpux */
-
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/write_atall.c b/3rd-party/romio321/mpi-io/write_atall.c
deleted file mode 100644
index 6f815cdd4cb..00000000000
--- a/3rd-party/romio321/mpi-io/write_atall.c
+++ /dev/null
@@ -1,67 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_write_at_all = PMPI_File_write_at_all
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_write_at_all MPI_File_write_at_all
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_write_at_all as PMPI_File_write_at_all
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_write_at_all(MPI_File fh, MPI_Offset offset, const void *buf, int count,
-                          MPI_Datatype datatype, MPI_Status *status)
-    __attribute__((weak,alias("PMPI_File_write_at_all")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/* status object not filled currently */
-
-/*@
-    MPI_File_write_at_all - Collective write using explicit offset
-
-Input Parameters:
-. fh - file handle (handle)
-. offset - file offset (nonnegative integer)
-. buf - initial address of buffer (choice)
-. count - number of elements in buffer (nonnegative integer)
-. datatype - datatype of each buffer element (handle)
-
-Output Parameters:
-. status - status object (Status)
-
-.N fortran
-@*/
-int MPI_File_write_at_all(MPI_File fh, MPI_Offset offset, ROMIO_CONST void *buf,
-                          int count, MPI_Datatype datatype, 
-                          MPI_Status *status)
-{
-    int error_code;
-    static char myname[] = "MPI_FILE_WRITE_AT_ALL";
-#ifdef MPI_hpux
-    int fl_xmpi;
-
-    HPMP_IO_START(fl_xmpi, BLKMPIFILEWRITEATALL, TRDTBLOCK, fh, datatype, count);
-#endif /* MPI_hpux */
-
-    error_code = MPIOI_File_write_all(fh, offset, ADIO_EXPLICIT_OFFSET,
-				      buf, count, datatype, myname, status);
-
-#ifdef MPI_hpux
-    HPMP_IO_END(fl_xmpi, fh, datatype, count);
-#endif /* MPI_hpux */
-    return error_code;
-}
-
diff --git a/3rd-party/romio321/mpi-io/write_ord.c b/3rd-party/romio321/mpi-io/write_ord.c
deleted file mode 100644
index eb72734e9c5..00000000000
--- a/3rd-party/romio321/mpi-io/write_ord.c
+++ /dev/null
@@ -1,128 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- *
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_write_ordered = PMPI_File_write_ordered
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_write_ordered MPI_File_write_ordered
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_write_ordered as PMPI_File_write_ordered
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_write_ordered(MPI_File fh, const void *buf, int count, MPI_Datatype datatype,
-                           MPI_Status *status) __attribute__((weak,alias("PMPI_File_write_ordered")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/* status object not filled currently */
-
-/*@
-    MPI_File_write_ordered - Collective write using shared file pointer
-
-Input Parameters:
-. fh - file handle (handle)
-. buf - initial address of buffer (choice)
-. count - number of elements in buffer (nonnegative integer)
-. datatype - datatype of each buffer element (handle)
-
-Output Parameters:
-. status - status object (Status)
-
-.N fortran
-@*/
-int MPI_File_write_ordered(MPI_File fh, ROMIO_CONST void *buf, int count,
-			   MPI_Datatype datatype, MPI_Status *status)
-{
-    int error_code, nprocs, myrank;
-    ADIO_Offset incr;
-    MPI_Count datatype_size;
-    int source, dest;
-    static char myname[] = "MPI_FILE_WRITE_ORDERED";
-    ADIO_Offset shared_fp;
-    ADIO_File adio_fh;
-    void *e32buf=NULL;
-    const void *xbuf;
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    MPIO_CHECK_COUNT(adio_fh, count, myname, error_code);
-    MPIO_CHECK_DATATYPE(adio_fh, datatype, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    MPI_Type_size_x(datatype, &datatype_size);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_INTEGRAL_ETYPE(adio_fh, count, datatype_size, myname, error_code);
-    MPIO_CHECK_FS_SUPPORTS_SHARED(adio_fh, myname, error_code);
-    MPIO_CHECK_COUNT_SIZE(adio_fh, count, datatype_size, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    ADIOI_TEST_DEFERRED(adio_fh, myname, &error_code);
-
-    MPI_Comm_size(adio_fh->comm, &nprocs);
-    MPI_Comm_rank(adio_fh->comm, &myrank);
-
-    incr = (count*datatype_size)/adio_fh->etype_size;
-    /* Use a message as a 'token' to order the operations */
-    source = myrank - 1;
-    dest   = myrank + 1;
-    if (source < 0) source = MPI_PROC_NULL;
-    if (dest >= nprocs) dest = MPI_PROC_NULL;
-    MPI_Recv(NULL, 0, MPI_BYTE, source, 0, adio_fh->comm, MPI_STATUS_IGNORE);
-
-    ADIO_Get_shared_fp(adio_fh, incr, &shared_fp, &error_code);
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL,
-					  myname, __LINE__, MPI_ERR_INTERN, 
-					  "**iosharedfailed", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    MPI_Send(NULL, 0, MPI_BYTE, dest, 0, adio_fh->comm);
-
-    xbuf = buf;
-    if (adio_fh->is_external32) {
-	error_code = MPIU_external32_buffer_setup(buf, count, datatype, &e32buf);
-	if (error_code != MPI_SUCCESS) 
-	    goto fn_exit;
-
-	xbuf = e32buf;
-    }
-
-    ADIO_WriteStridedColl(adio_fh, xbuf, count, datatype, ADIO_EXPLICIT_OFFSET,
-                          shared_fp, status, &error_code);
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-    /* --END ERROR HANDLING-- */
-
-fn_exit:
-    if (e32buf != NULL) ADIOI_Free(e32buf);
-    ROMIO_THREAD_CS_EXIT();
-
-    /* FIXME: Check for error code from WriteStridedColl? */
-    return error_code;
-}
-
diff --git a/3rd-party/romio321/mpi-io/write_ordb.c b/3rd-party/romio321/mpi-io/write_ordb.c
deleted file mode 100644
index 8b5a3023660..00000000000
--- a/3rd-party/romio321/mpi-io/write_ordb.c
+++ /dev/null
@@ -1,132 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_write_ordered_begin = PMPI_File_write_ordered_begin
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_write_ordered_begin MPI_File_write_ordered_begin
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_write_ordered_begin as PMPI_File_write_ordered_begin
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_write_ordered_begin(MPI_File fh, const void *buf, int count, MPI_Datatype datatype)
-    __attribute__((weak,alias("PMPI_File_write_ordered_begin")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_write_ordered_begin - Begin a split collective write using shared file pointer
-
-Input Parameters:
-. fh - file handle (handle)
-. count - number of elements in buffer (nonnegative integer)
-. datatype - datatype of each buffer element (handle)
-
-Output Parameters:
-. buf - initial address of buffer (choice)
-
-.N fortran
-@*/
-int MPI_File_write_ordered_begin(MPI_File fh, ROMIO_CONST void *buf, int count,
-				 MPI_Datatype datatype)
-{
-    int error_code, nprocs, myrank;
-    ADIO_Offset incr;
-    MPI_Count datatype_size;
-    int source, dest;
-    static char myname[] = "MPI_FILE_WRITE_ORDERED_BEGIN";
-    ADIO_Offset shared_fp;
-    ADIO_File adio_fh;
-    void *e32buf = NULL;
-    const void *xbuf=NULL;
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    MPIO_CHECK_COUNT(adio_fh, count, myname, error_code);
-    MPIO_CHECK_DATATYPE(adio_fh, datatype, myname, error_code);
-
-    if (adio_fh->split_coll_count)
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_IO, 
-					  "**iosplitcoll", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    adio_fh->split_coll_count = 1;
-
-    MPI_Type_size_x(datatype, &datatype_size);
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_INTEGRAL_ETYPE(adio_fh, count, datatype_size, myname, error_code);
-    MPIO_CHECK_FS_SUPPORTS_SHARED(adio_fh, myname, error_code);
-    MPIO_CHECK_COUNT_SIZE(adio_fh, count, datatype_size, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    ADIOI_TEST_DEFERRED(adio_fh, myname, &error_code);
-
-    MPI_Comm_size(adio_fh->comm, &nprocs);
-    MPI_Comm_rank(adio_fh->comm, &myrank);
-
-    incr = (count*datatype_size)/adio_fh->etype_size;
-    /* Use a message as a 'token' to order the operations */
-    source = myrank - 1;
-    dest   = myrank + 1;
-    if (source < 0) source = MPI_PROC_NULL;
-    if (dest >= nprocs) dest = MPI_PROC_NULL;
-    MPI_Recv(NULL, 0, MPI_BYTE, source, 0, adio_fh->comm, MPI_STATUS_IGNORE);
-
-    ADIO_Get_shared_fp(adio_fh, incr, &shared_fp, &error_code);
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL,
-					  myname, __LINE__, MPI_ERR_INTERN, 
-					  "**iosharedfailed", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    MPI_Send(NULL, 0, MPI_BYTE, dest, 0, adio_fh->comm);
-
-    xbuf = buf;
-    if (adio_fh->is_external32) {
-	error_code = MPIU_external32_buffer_setup(buf, count, datatype, &e32buf);
-	if (error_code != MPI_SUCCESS) 
-	    goto fn_exit;
-
-	xbuf = e32buf;
-    }
-
-    ADIO_WriteStridedColl(adio_fh, xbuf, count, datatype, ADIO_EXPLICIT_OFFSET,
-			  shared_fp, &adio_fh->split_status, &error_code);
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-    /* --END ERROR HANDLING-- */
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-
-    /* FIXME: Check for error code from WriteStridedColl? */
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi-io/write_orde.c b/3rd-party/romio321/mpi-io/write_orde.c
deleted file mode 100644
index 8cfdba2ecf6..00000000000
--- a/3rd-party/romio321/mpi-io/write_orde.c
+++ /dev/null
@@ -1,75 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_write_ordered_end = PMPI_File_write_ordered_end
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_write_ordered_end MPI_File_write_ordered_end
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_write_ordered_end as PMPI_File_write_ordered_end
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_write_ordered_end(MPI_File fh, const void *buf, MPI_Status *status) __attribute__((weak,alias("PMPI_File_write_ordered_end")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_File_write_ordered_end - Complete a split collective write using shared file pointer
-
-Input Parameters:
-. fh - file handle (handle)
-
-Output Parameters:
-. buf - initial address of buffer (choice)
-. status - status object (Status)
-
-.N fortran
-@*/
-int MPI_File_write_ordered_end(MPI_File fh, ROMIO_CONST void *buf, MPI_Status *status)
-{
-    int error_code;
-    static char myname[] = "MPI_FILE_WRITE_ORDERED_END";
-    ADIO_File adio_fh;
-
-    MPIU_UNREFERENCED_ARG(buf);
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-
-    if (!(adio_fh->split_coll_count))
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_IO, 
-					  "**iosplitcollnone", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-#ifdef HAVE_STATUS_SET_BYTES
-    if (status != MPI_STATUS_IGNORE)
-       *status = adio_fh->split_status;
-#endif
-    adio_fh->split_coll_count = 0;
-
-
-fn_exit:
-    ROMIO_THREAD_CS_EXIT();
-    return MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/mpi-io/write_sh.c b/3rd-party/romio321/mpi-io/write_sh.c
deleted file mode 100644
index ed8e1c198bb..00000000000
--- a/3rd-party/romio321/mpi-io/write_sh.c
+++ /dev/null
@@ -1,149 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_File_write_shared = PMPI_File_write_shared
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_File_write_shared MPI_File_write_shared
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_File_write_shared as PMPI_File_write_shared
-/* end of weak pragmas */
-#elif defined(HAVE_WEAK_ATTRIBUTE)
-int MPI_File_write_shared(MPI_File fh, const void *buf, int count, MPI_Datatype datatype,
-                          MPI_Status *status) __attribute__((weak,alias("PMPI_File_write_shared")));
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/* status object not filled currently */
-
-/*@
-    MPI_File_write_shared - Write using shared file pointer
-
-Input Parameters:
-. fh - file handle (handle)
-. buf - initial address of buffer (choice)
-. count - number of elements in buffer (nonnegative integer)
-. datatype - datatype of each buffer element (handle)
-
-Output Parameters:
-. status - status object (Status)
-
-.N fortran
-@*/
-int MPI_File_write_shared(MPI_File fh, ROMIO_CONST void *buf, int count,
-                          MPI_Datatype datatype, MPI_Status *status)
-{
-    int error_code, buftype_is_contig, filetype_is_contig;
-    ADIO_Offset bufsize;
-    static char myname[] = "MPI_FILE_READ_SHARED";
-    MPI_Count datatype_size, incr;
-    ADIO_Offset off, shared_fp;
-    ADIO_File adio_fh;
-    void *e32buf = NULL;
-    const void *xbuf = NULL;
-
-    ROMIO_THREAD_CS_ENTER();
-
-    adio_fh = MPIO_File_resolve(fh);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_FILE_HANDLE(adio_fh, myname, error_code);
-    MPIO_CHECK_COUNT(adio_fh, count, myname, error_code);
-    MPIO_CHECK_DATATYPE(adio_fh, datatype, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    MPI_Type_size_x(datatype, &datatype_size);
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_COUNT_SIZE(adio_fh, count, datatype_size, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    if (count*datatype_size == 0) {
-#ifdef HAVE_STATUS_SET_BYTES
-       MPIR_Status_set_bytes(status, datatype, 0);
-#endif
-       error_code = MPI_SUCCESS;
-       goto fn_exit;
-    }
-
-    /* --BEGIN ERROR HANDLING-- */
-    MPIO_CHECK_INTEGRAL_ETYPE(adio_fh, count, datatype_size, myname, error_code);
-    MPIO_CHECK_FS_SUPPORTS_SHARED(adio_fh, myname, error_code);
-    /* --END ERROR HANDLING-- */
-
-    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
-    ADIOI_Datatype_iscontig(adio_fh->filetype, &filetype_is_contig);
-
-    ADIOI_TEST_DEFERRED(adio_fh, myname, &error_code);
-
-    incr = (count*datatype_size)/adio_fh->etype_size;
-
-    ADIO_Get_shared_fp(adio_fh, incr, &shared_fp, &error_code);
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-    {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL,
-					  myname, __LINE__, MPI_ERR_INTERN, 
-					  "**iosharedfailed", 0);
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-	goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    xbuf = buf;
-    if (adio_fh->is_external32) {
-	error_code = MPIU_external32_buffer_setup(buf, count, datatype, &e32buf);
-	if (error_code != MPI_SUCCESS) 
-	    goto fn_exit;
-
-	xbuf = e32buf;
-    }
-
-    if (buftype_is_contig && filetype_is_contig)
-    {
-        /* convert bufocunt and shared_fp to bytes */
-	bufsize = datatype_size * count;
-	off = adio_fh->disp + adio_fh->etype_size * shared_fp;
-
-        /* if atomic mode requested, lock (exclusive) the region, because there
-           could be a concurrent noncontiguous request. On NFS, locking is 
-           done in the ADIO_WriteContig.*/
-
-        if ((adio_fh->atomicity) && (adio_fh->file_system != ADIO_NFS))
-            ADIOI_WRITE_LOCK(adio_fh, off, SEEK_SET, bufsize);
-
-	ADIO_WriteContig(adio_fh, xbuf, count, datatype, ADIO_EXPLICIT_OFFSET,
-		     off, status, &error_code); 
-
-        if ((adio_fh->atomicity) && (adio_fh->file_system != ADIO_NFS))
-            ADIOI_UNLOCK(adio_fh, off, SEEK_SET, bufsize);
-    }
-    else
-    {
-	ADIO_WriteStrided(adio_fh, xbuf, count, datatype, ADIO_EXPLICIT_OFFSET,
-			 shared_fp, status, &error_code);
-	/* For strided and atomic mode, locking is done in ADIO_WriteStrided */
-    }
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (error_code != MPI_SUCCESS)
-	error_code = MPIO_Err_return_file(adio_fh, error_code);
-    /* --END ERROR HANDLING-- */
-
-fn_exit:
-    if (e32buf != NULL) ADIOI_Free(e32buf);
-    ROMIO_THREAD_CS_EXIT();
-    return error_code;
-}
diff --git a/3rd-party/romio321/mpi2-other/array/Makefile.in b/3rd-party/romio321/mpi2-other/array/Makefile.in
deleted file mode 100644
index 41a366e398b..00000000000
--- a/3rd-party/romio321/mpi2-other/array/Makefile.in
+++ /dev/null
@@ -1,80 +0,0 @@
-CC          = @CC@
-AR          = @AR@
-LIBNAME     = @LIBNAME@
-RANLIB      = @RANLIB@
-srcdir      = @srcdir@
-CC_SHL      = @CC_SHL@
-SHLIBNAME   = @SHLIBNAME@
-
-INCLUDE_DIR = -I@MPI_INCLUDE_DIR@ -I../../include -I${srcdir}/../../mpi-io -I${srcdir}/../../adio/include -I../../adio/include
-CFLAGS      = @CPPFLAGS@ @CFLAGS@ $(MPIOPROFILE) $(INCLUDE_DIR)
-
-top_builddir  = @master_topbuild_dir@
-LIBTOOL       = @LIBTOOL@
-C_COMPILE_SHL = $(CC_SHL)
-
-@VPATH@
-
-MPIO_OBJECTS = subarray.o darray.o
-
-MPIO_TMP_POBJECTS = subarray.p darray.p
-
-MPIO_REAL_POBJECTS = _subarray.o _darray.o
-
-all: $(LIBNAME)
-	@if [ "@ENABLE_SHLIB@" != "none" ] ; then \
-	    $(MAKE) $(SHLIBNAME).la ;\
-	fi
-
-.SUFFIXES: $(SUFFIXES) .p .sp .lo
-
-.c.o:
-	$(CC) $(CFLAGS) -c $<
-.c.lo:
-	$(C_COMPILE_SHL) $(CFLAGS) -c $< -o _s$*.o
-	@mv -f _s$*.o $*.lo
-
-.c.p:
-	@cp $(srcdir)/$*.c _$*.c
-	$(CC) $(CFLAGS) -c _$*.c
-	@rm -f _$*.c
-
-.c.sp:
-	$(C_COMPILE_SHL) $(CFLAGS) -c $< -o _$*.lo
-
-profile: $(MPIO_REAL_POBJECTS)
-	$(AR) $(LIBNAME) $(MPIO_REAL_POBJECTS)
-	$(RANLIB) $(LIBNAME) 
-	@if [ "@ENABLE_SHLIB@" != "none" ] ; then \
-	    $(MAKE) P$(SHLIBNAME).la ;\
-	fi
-	@rm -f _*.o
-
-$(LIBNAME): $(MPIO_OBJECTS)
-	$(AR) $(LIBNAME) $(MPIO_OBJECTS)
-	$(RANLIB) $(LIBNAME) 
-
-MPIO_LOOBJECTS = $(MPIO_OBJECTS:.o=.lo)
-$(SHLIBNAME).la: $(MPIO_LOOBJECTS) 
-	$(AR) $(SHLIBNAME).la $(MPIO_LOOBJECTS)
-
-# --------------------------------------------------------------------------
-# We use P$(SHLBNAME) simply to distinguish the regular lib build from the
-# profiling library build
-MPIO_LOPOBJECTS = $(MPIO_REAL_POBJECTS:.o=.lo)
-MPIO_TMP_LOPOBJECTS = ${MPIO_TMP_POBJECTS:.p=.sp}
-P$(SHLIBNAME).la: $(MPIO_TMP_LOPOBJECTS)
-	$(AR) $(SHLIBNAME).la $(MPIO_LOPOBJECTS)
-# --------------------------------------------------------------------------
-
-clean:
-	@rm -f *.o *.lo *.gcno *.gcda *.bb *.bbg
-	@rm -f ${srcdir}/*.gcno ${srcdir}/*.gcda 
-	@rm -f ${srcdir}/*.bb ${srcdir}/*.bbg
-
-# Rules for the profiling objects
-_subarray.o: subarray.c
-	$(CC) $(CFLAGS) -c $(srcdir)/subarray.c -o _subarray.o
-_darray.o: darray.c
-	$(CC) $(CFLAGS) -c $(srcdir)/darray.c -o _darray.o
-
diff --git a/3rd-party/romio321/mpi2-other/array/darray.c b/3rd-party/romio321/mpi2-other/array/darray.c
deleted file mode 100644
index 17ee2a1fce6..00000000000
--- a/3rd-party/romio321/mpi2-other/array/darray.c
+++ /dev/null
@@ -1,209 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_Type_create_darray = PMPI_Type_create_darray
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_Type_create_darray MPI_Type_create_darray
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_Type_create_darray as PMPI_Type_create_darray
-/* end of weak pragmas */
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-MPI_Type_create_darray - Creates a datatype corresponding to a distributed, multidimensional array
-
-Input Parameters:
-. size - size of process group (positive integer)
-. rank - rank in process group (nonnegative integer)
-. ndims - number of array dimensions as well as process grid dimensions (positive integer)
-. array_of_gsizes - number of elements of type oldtype in each dimension of global array (array of positive integers)
-. array_of_distribs - distribution of array in each dimension (array of state)
-. array_of_dargs - distribution argument in each dimension (array of positive integers)
-. array_of_psizes - size of process grid in each dimension (array of positive integers)
-. order - array storage order flag (state)
-. oldtype - old datatype (handle)
-
-Output Parameters:
-. newtype - new datatype (handle)
-
-.N fortran
-@*/
-int MPI_Type_create_darray(int size, int rank, int ndims, 
-     	                   int *array_of_gsizes, int *array_of_distribs, 
-                           int *array_of_dargs, int *array_of_psizes, 
-                           int order, MPI_Datatype oldtype, 
-                           MPI_Datatype *newtype) 
-{
-    int err, error_code;
-    int i;
-    MPI_Aint orig_extent, orig_lb, size_with_aint;
-    MPI_Offset size_with_offset;
-    static char myname[] = "MPI_TYPE_CREATE_DARRAY";
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (size <= 0) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "Invalid size argument", 0);
-	return MPIO_Err_return_comm(MPI_COMM_SELF, error_code);
-    }
-    if (rank < 0) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "Invalid rank argument", 0);
-	return MPIO_Err_return_comm(MPI_COMM_SELF, error_code);
-    }
-    if (ndims <= 0) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "Invalid ndoms argument", 0);
-	return MPIO_Err_return_comm(MPI_COMM_SELF, error_code);
-    }
-    if (array_of_gsizes <= (int *) 0) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "Invalid array_of_gsizes argument", 0);
-	return MPIO_Err_return_comm(MPI_COMM_SELF, error_code);
-    }
-    if (array_of_distribs <= (int *) 0) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "Invalid array_of_distribs argument", 0);
-	return MPIO_Err_return_comm(MPI_COMM_SELF, error_code);
-    }
-    if (array_of_dargs <= (int *) 0) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "Invalid array_of_dargs argument", 0);
-	return MPIO_Err_return_comm(MPI_COMM_SELF, error_code);
-    }
-    if (array_of_psizes <= (int *) 0) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "Invalid array_of_psizes argument", 0);
-	return MPIO_Err_return_comm(MPI_COMM_SELF, error_code);
-    }
-
-    for (i=0; i<ndims; i++) {
-	if (array_of_gsizes[i] <= 0) {
-	    error_code = MPIO_Err_create_code(MPI_SUCCESS,MPIR_ERR_RECOVERABLE,
-					      myname, __LINE__, MPI_ERR_ARG,
-					      "Invalid gsize argument", 0);
-	    return MPIO_Err_return_comm(MPI_COMM_SELF, error_code);
-	}
-
-	/* array_of_distribs checked below */
-
-	if ((array_of_dargs[i] != MPI_DISTRIBUTE_DFLT_DARG) && 
-	    (array_of_dargs[i] <= 0))
-	{
-	    error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					      MPIR_ERR_RECOVERABLE,
-					      myname, __LINE__, MPI_ERR_ARG,
-					      "Invalid darg argument", 0);
-	    return MPIO_Err_return_comm(MPI_COMM_SELF, error_code);
-	}
-
-	if (array_of_psizes[i] <= 0) {
-	    error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					      MPIR_ERR_RECOVERABLE,
-					      myname, __LINE__, MPI_ERR_ARG,
-					      "Invalid psize argument", 0);
-	    return MPIO_Err_return_comm(MPI_COMM_SELF, error_code);
-	}
-	if (array_of_distribs[i] != MPI_DISTRIBUTE_BLOCK &&
-	    array_of_distribs[i] != MPI_DISTRIBUTE_CYCLIC &&
-	    array_of_distribs[i] != MPI_DISTRIBUTE_NONE)
-	{
-	    error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					      MPIR_ERR_RECOVERABLE,
-					      myname, __LINE__,
-					      MPI_ERR_ARG,
-					      "Invalid distrib argument",
-					      0);
-	    return MPIO_Err_return_comm(MPI_COMM_SELF, error_code);
-	}
-	if (array_of_distribs[i] == MPI_DISTRIBUTE_NONE &&
-	    array_of_psizes[i] != 1)
-	{
-		    error_code = MPIO_Err_create_code(MPI_SUCCESS,
-						      MPIR_ERR_RECOVERABLE,
-						      myname, __LINE__,
-						      MPI_ERR_ARG,
-						      "For MPI_DISTRIBUTE_NONE, the number of processes in that dimension of the grid must be 1",
-						      0);
-		    return MPIO_Err_return_comm(MPI_COMM_SELF, error_code);
-	}
-    }
-
-    /* order argument checked below */
-
-    if (oldtype == MPI_DATATYPE_NULL) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "Invalid type argument", 0);
-	return MPIO_Err_return_comm(MPI_COMM_SELF, error_code);
-    }
-
-    MPI_Type_get_extent(oldtype, &orig_lb, &orig_extent);
-
-/* check if MPI_Aint is large enough for size of global array. 
-   if not, complain. */
-
-    size_with_aint = orig_extent;
-    for (i=0; i<ndims; i++) size_with_aint *= array_of_gsizes[i];
-    size_with_offset = orig_extent;
-    for (i=0; i<ndims; i++) size_with_offset *= array_of_gsizes[i];
-
-    if (size_with_aint != size_with_offset) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "Invalid array size", 0);
-	return MPIO_Err_return_comm(MPI_COMM_SELF, error_code);
-    }
-
-    if (order != MPI_ORDER_FORTRAN && order != MPI_ORDER_C) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					  MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "Invalid order argument", 0);
-	return MPIO_Err_return_comm(MPI_COMM_SELF, error_code);
-    }
-    /* --END ERROR HANDLING-- */
-
-    err = ADIO_Type_create_darray(size,
-				  rank,
-				  ndims,
-				  array_of_gsizes,
-				  array_of_distribs,
-				  array_of_dargs,
-				  array_of_psizes,
-				  order,
-				  oldtype,
-				  newtype);
-    /* --BEGIN ERROR HANDLING-- */
-    if (err != MPI_SUCCESS) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					  MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, err,
-					  "Internal error", 0);
-    }
-    /* --END ERROR HANDLING-- */
-
-    return MPI_SUCCESS;
-}
-
diff --git a/3rd-party/romio321/mpi2-other/array/fortran/Makefile.in b/3rd-party/romio321/mpi2-other/array/fortran/Makefile.in
deleted file mode 100644
index 63c8fe5e02a..00000000000
--- a/3rd-party/romio321/mpi2-other/array/fortran/Makefile.in
+++ /dev/null
@@ -1,48 +0,0 @@
-CC          = @CC@
-AR          = @AR@
-RANLIB      = @RANLIB@
-LIBNAME     = @LIBNAME@
-srcdir      = @srcdir@
-
-INCLUDE_DIR = -I@MPI_INCLUDE_DIR@ -I../../../include -I${srcdir}/../../../mpi-io -I${srcdir}/../../../adio/include -I../../../adio/include
-CFLAGS      = @CFLAGS@ $(MPIOPROFILE) $(INCLUDE_DIR)
-
-@VPATH@
-
-MPIO_FOBJECTS = subarrayf.o darrayf.o
-
-MPIO_TMP_PFOBJECTS = subarrayf.p darrayf.p
-
-MPIO_REAL_PFOBJECTS = _subarrayf.o _darrayf.o
-
-all: $(LIBNAME)
-
-.SUFFIXES: $(SUFFIXES) .p
-
-.c.o:
-	$(CC) $(CFLAGS) -c $<
-
-.c.p:
-	@cp $(srcdir)/$*.c _$*.c
-	$(CC) $(CFLAGS) -c _$*.c
-	@rm -f _$*.c
-
-profile: $(MPIO_REAL_PFOBJECTS)
-	$(AR) $(LIBNAME) $(MPIO_REAL_PFOBJECTS)
-	$(RANLIB) $(LIBNAME) 
-	@rm -f _*.o
-
-$(LIBNAME): $(MPIO_FOBJECTS)
-	$(AR) $(LIBNAME) $(MPIO_FOBJECTS)
-	$(RANLIB) $(LIBNAME) 
-
-clean: 
-	@rm -f *.o *.lo *.gcno *.gcda *.bb *.bbg
-	@rm -f ${srcdir}/*.gcno ${srcdir}/*.gcda 
-	@rm -f ${srcdir}/*.bb ${srcdir}/*.bbg
-
-# Rules for the profiling objects
-_subarrayf.o: subarrayf.c
-	$(CC) $(CFLAGS) -c $(srcdir)/subarrayf.c -o _subarrayf.o
-_darrayf.o: darrayf.c
-	$(CC) $(CFLAGS) -c $(srcdir)/darrayf.c -o _darrayf.o
diff --git a/3rd-party/romio321/mpi2-other/array/fortran/darrayf.c b/3rd-party/romio321/mpi2-other/array/fortran/darrayf.c
deleted file mode 100644
index 7a16bbb3cc8..00000000000
--- a/3rd-party/romio321/mpi2-other/array/fortran/darrayf.c
+++ /dev/null
@@ -1,130 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_TYPE_CREATE_DARRAY(MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *);
-#pragma weak MPI_TYPE_CREATE_DARRAY = PMPI_TYPE_CREATE_DARRAY
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_type_create_darray__(MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *);
-#pragma weak mpi_type_create_darray__ = pmpi_type_create_darray__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_type_create_darray(MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *);
-#pragma weak mpi_type_create_darray = pmpi_type_create_darray
-#else
-extern FORTRAN_API void FORT_CALL mpi_type_create_darray_(MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *);
-#pragma weak mpi_type_create_darray_ = pmpi_type_create_darray_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_TYPE_CREATE_DARRAY MPI_TYPE_CREATE_DARRAY
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_type_create_darray__ mpi_type_create_darray__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_type_create_darray mpi_type_create_darray
-#else
-#pragma _HP_SECONDARY_DEF pmpi_type_create_darray_ mpi_type_create_darray_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_TYPE_CREATE_DARRAY as PMPI_TYPE_CREATE_DARRAY
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_type_create_darray__ as pmpi_type_create_darray__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_type_create_darray as pmpi_type_create_darray
-#else
-#pragma _CRI duplicate mpi_type_create_darray_ as pmpi_type_create_darray_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_type_create_darray_ PMPI_TYPE_CREATE_DARRAY
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_type_create_darray_ pmpi_type_create_darray__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_type_create_darray pmpi_type_create_darray_
-#endif
-#define mpi_type_create_darray_ pmpi_type_create_darray
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_type_create_darray_ pmpi_type_create_darray
-#endif
-#define mpi_type_create_darray_ pmpi_type_create_darray_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_type_create_darray_ MPI_TYPE_CREATE_DARRAY
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_type_create_darray_ mpi_type_create_darray__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_type_create_darray mpi_type_create_darray_
-#endif
-#define mpi_type_create_darray_ mpi_type_create_darray
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_type_create_darray_ mpi_type_create_darray
-#endif
-#endif
-#endif
-
-#ifdef MPIHP
-/* Prototype to keep compiler happy */
-void mpi_type_create_darray_(int *size,int *rank,int *ndims,
-                           int *array_of_gsizes,int *array_of_distribs,
-                           int *array_of_dargs,int *array_of_psizes,
-                           int *order, MPI_Fint *oldtype,
-			     MPI_Fint *newtype, int *ierr );
-
-void mpi_type_create_darray_(int *size,int *rank,int *ndims,
-                           int *array_of_gsizes,int *array_of_distribs,
-                           int *array_of_dargs,int *array_of_psizes,
-                           int *order, MPI_Fint *oldtype,
-                           MPI_Fint *newtype, int *ierr )
-{
-    MPI_Datatype oldtype_c, newtype_c;
-
-    oldtype_c = MPI_Type_f2c(*oldtype);
-
-    *ierr = MPI_Type_create_darray(*size,*rank,*ndims,array_of_gsizes,array_of_distribs,array_of_dargs,array_of_psizes,*order,oldtype_c,&newtype_c);
-
-    *newtype = MPI_Type_c2f(newtype_c);
-}
-
-#else
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL void mpi_type_create_darray_(MPI_Fint *size,MPI_Fint *rank,MPI_Fint *ndims,
-					       MPI_Fint *array_of_gsizes,MPI_Fint *array_of_distribs,
-					       MPI_Fint *array_of_dargs,MPI_Fint *array_of_psizes,
-                                               MPI_Fint *order,MPI_Fint *oldtype,
-				       		MPI_Fint *newtype, MPI_Fint *ierr);
-
-FORTRAN_API void FORT_CALL void mpi_type_create_darray_(MPI_Fint *size,MPI_Fint *rank,MPI_Fint *ndims,
-					       MPI_Fint *array_of_gsizes,MPI_Fint *array_of_distribs,
-					       MPI_Fint *array_of_dargs,MPI_Fint *array_of_psizes,
-                                               MPI_Fint *order,MPI_Fint *oldtype,
-                                               MPI_Fint *newtype, MPI_Fint *ierr )
-{
-    *ierr = MPI_Type_create_darray(*size,*rank,*ndims,array_of_gsizes,array_of_distribs,array_of_dargs,array_of_psizes,*order,*oldtype,newtype);
-}
-#endif
diff --git a/3rd-party/romio321/mpi2-other/array/fortran/subarrayf.c b/3rd-party/romio321/mpi2-other/array/fortran/subarrayf.c
deleted file mode 100644
index 7a2e54a530b..00000000000
--- a/3rd-party/romio321/mpi2-other/array/fortran/subarrayf.c
+++ /dev/null
@@ -1,127 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "adio.h"
-#include "mpio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-extern FORTRAN_API void FORT_CALL MPI_TYPE_CREATE_SUBARRAY(MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *);
-#pragma weak MPI_TYPE_CREATE_SUBARRAY = PMPI_TYPE_CREATE_SUBARRAY
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_type_create_subarray__(MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *);
-#pragma weak mpi_type_create_subarray__ = pmpi_type_create_subarray__
-#elif !defined(FORTRANUNDERSCORE)
-extern FORTRAN_API void FORT_CALL mpi_type_create_subarray(MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *);
-#pragma weak mpi_type_create_subarray = pmpi_type_create_subarray
-#else
-extern FORTRAN_API void FORT_CALL mpi_type_create_subarray_(MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *,  MPI_Fint *, MPI_Fint *);
-#pragma weak mpi_type_create_subarray_ = pmpi_type_create_subarray_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_TYPE_CREATE_SUBARRAY MPI_TYPE_CREATE_SUBARRAY
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_type_create_subarray__ mpi_type_create_subarray__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_type_create_subarray mpi_type_create_subarray
-#else
-#pragma _HP_SECONDARY_DEF pmpi_type_create_subarray_ mpi_type_create_subarray_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_TYPE_CREATE_SUBARRAY as PMPI_TYPE_CREATE_SUBARRAY
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_type_create_subarray__ as pmpi_type_create_subarray__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_type_create_subarray as pmpi_type_create_subarray
-#else
-#pragma _CRI duplicate mpi_type_create_subarray_ as pmpi_type_create_subarray_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#ifdef FORTRANCAPS
-#define mpi_type_create_subarray_ PMPI_TYPE_CREATE_SUBARRAY
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_type_create_subarray_ pmpi_type_create_subarray__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_type_create_subarray pmpi_type_create_subarray_
-#endif
-#define mpi_type_create_subarray_ pmpi_type_create_subarray
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_type_create_subarray_ pmpi_type_create_subarray
-#endif
-#define mpi_type_create_subarray_ pmpi_type_create_subarray_
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_type_create_subarray_ MPI_TYPE_CREATE_SUBARRAY
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_type_create_subarray_ mpi_type_create_subarray__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_type_create_subarray mpi_type_create_subarray_
-#endif
-#define mpi_type_create_subarray_ mpi_type_create_subarray
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_type_create_subarray_ mpi_type_create_subarray
-#endif
-#endif
-#endif
-
-#ifdef MPIHP
-/* Prototype to keep compiler happy */
-void mpi_type_create_subarray_(int *ndims,int *array_of_sizes,
-                             int *array_of_subsizes,int *array_of_starts,
-                             int *order,MPI_Fint *oldtype,
-			       MPI_Fint *newtype, int *ierr );
-
-void mpi_type_create_subarray_(int *ndims,int *array_of_sizes,
-                             int *array_of_subsizes,int *array_of_starts,
-                             int *order,MPI_Fint *oldtype,
-                             MPI_Fint *newtype, int *ierr )
-{
-    MPI_Datatype oldtype_c, newtype_c;
-
-    oldtype_c = MPI_Type_f2c(*oldtype);
-
-    *ierr = MPI_Type_create_subarray(*ndims,array_of_sizes,array_of_subsizes,array_of_starts,*order,oldtype_c,&newtype_c);
-    *newtype = MPI_Type_c2f(newtype_c);
-}
-
-#else
-
-/* Prototype to keep compiler happy */
-FORTRAN_API void FORT_CALL void mpi_type_create_subarray_(MPI_Fint *ndims,MPI_Fint *array_of_sizes,
-                               MPI_Fint *array_of_subsizes,MPI_Fint *array_of_starts,
-                               MPI_Fint *order,MPI_Fint *oldtype,
-                               MPI_Fint *newtype, MPI_Fint *ierr );
-
-FORTRAN_API void FORT_CALL void mpi_type_create_subarray_(MPI_Fint *ndims,MPI_Fint *array_of_sizes,
-                               MPI_Fint *array_of_subsizes,MPI_Fint *array_of_starts,
-                               MPI_Fint *order,MPI_Fint *oldtype,
-                               MPI_Fint *newtype, MPI_Fint *ierr )
-{
-*ierr = MPI_Type_create_subarray(*ndims,array_of_sizes,array_of_subsizes,array_of_starts,*order,*oldtype,newtype);
-}
-#endif
diff --git a/3rd-party/romio321/mpi2-other/array/subarray.c b/3rd-party/romio321/mpi2-other/array/subarray.c
deleted file mode 100644
index 006abce19c8..00000000000
--- a/3rd-party/romio321/mpi2-other/array/subarray.c
+++ /dev/null
@@ -1,174 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_Type_create_subarray = PMPI_Type_create_subarray
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_Type_create_subarray MPI_Type_create_subarray
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_Type_create_subarray as PMPI_Type_create_subarray
-/* end of weak pragmas */
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-MPI_Type_create_subarray - Creates a datatype describing a subarray of a multidimensional array
-
-Input Parameters:
-. ndims - number of array dimensions (positive integer)
-. array_of_sizes - number of elements of type oldtype in each dimension of the full array (array of positive integers)
-. array_of_subsizes - number of elements of type oldtype in each dimension of the subarray (array of positive integers)
-. array_of_starts - starting coordinates of the subarray in each dimension (array of nonnegative integers)
-. order - array storage order flag (state)
-. oldtype - old datatype (handle)
-
-Output Parameters:
-. newtype - new datatype (handle)
-
-.N fortran
-@*/
-int MPI_Type_create_subarray(int ndims, int *array_of_sizes, 
-                             int *array_of_subsizes, int *array_of_starts,
-                             int order, MPI_Datatype oldtype, 
-                             MPI_Datatype *newtype)
-{
-    MPI_Aint extent, lb, size_with_aint;
-    int i, err, error_code;
-    MPI_Offset size_with_offset;
-
-    /* --BEGIN ERROR HANDLING-- */
-    if (ndims <= 0) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					  MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "Invalid ndims argument", 0);
-	return MPIO_Err_return_comm(MPI_COMM_SELF, error_code);
-    }
-    if (array_of_sizes <= (int *) 0) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					  MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "Invalid array_of_sizes argument",
-					  0);
-	return MPIO_Err_return_comm(MPI_COMM_SELF, error_code);
-    }
-    if (array_of_subsizes <= (int *) 0) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					  MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "Invalid array_of_subsizes argument",
-					  0);
-	return MPIO_Err_return_comm(MPI_COMM_SELF, error_code);
-    }
-    if (array_of_starts <= (int *) 0) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					  MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "Invalid array_of_starts argument",
-					  0);
-	return MPIO_Err_return_comm(MPI_COMM_SELF, error_code);
-    }
-
-    for (i=0; i<ndims; i++) {
-        if (array_of_sizes[i] <= 0) {
-	    error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					      MPIR_ERR_RECOVERABLE,
-					      myname, __LINE__, MPI_ERR_ARG,
-					      "Invalid size argument", 0);
-	    return MPIO_Err_return_comm(MPI_COMM_SELF, error_code);
-        }
-        if (array_of_subsizes[i] <= 0) {
-	    error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					      MPIR_ERR_RECOVERABLE,
-					      myname, __LINE__, MPI_ERR_ARG,
-					      "Invalid subsize argument", 0);
-	    return MPIO_Err_return_comm(MPI_COMM_SELF, error_code);
-        }
-        if (array_of_starts[i] < 0) {
-	    error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					      MPIR_ERR_RECOVERABLE,
-					      myname, __LINE__, MPI_ERR_ARG,
-					      "Invalid start argument", 0);
-	    return MPIO_Err_return_comm(MPI_COMM_SELF, error_code);
-        }
-        if (array_of_subsizes[i] > array_of_sizes[i]) {
-	    error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					      MPIR_ERR_RECOVERABLE,
-					      myname, __LINE__, MPI_ERR_ARG,
-					      "Invalid subsize argument", 0);
-	    return MPIO_Err_return_comm(MPI_COMM_SELF, error_code);
-        }
-        if (array_of_starts[i] > (array_of_sizes[i] - array_of_subsizes[i])) {
-	    error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					      MPIR_ERR_RECOVERABLE,
-					      myname, __LINE__, MPI_ERR_ARG,
-					      "Invalid start argument", 0);
-	    return MPIO_Err_return_comm(MPI_COMM_SELF, error_code);
-        }
-    }
-
-    /* order argument checked below */
-
-    if (oldtype == MPI_DATATYPE_NULL) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					  MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "Invalid type argument", 0);
-	return MPIO_Err_return_comm(MPI_COMM_SELF, error_code);
-    }
-
-    MPI_Type_get_extent(oldtype, &lb, &extent);
-
-/* check if MPI_Aint is large enough for size of global array. 
-   if not, complain. */
-
-    size_with_aint = extent;
-    for (i=0; i<ndims; i++) size_with_aint *= array_of_sizes[i];
-    size_with_offset = extent;
-    for (i=0; i<ndims; i++) size_with_offset *= array_of_sizes[i];
-    if (size_with_aint != size_with_offset) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					  MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "Invalid size argument", 0);
-	return MPIO_Err_return_comm(MPI_COMM_SELF, error_code);
-    }
-
-    if (order != MPI_ORDER_FORTRAN && order != MPI_ORDER_C) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					  MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, MPI_ERR_ARG,
-					  "Invalid order argument", 0);
-	return MPIO_Err_return_comm(MPI_COMM_SELF, error_code);
-    }
-    /* --END ERROR HANDLING-- */
-
-    err = ADIO_Type_create_subarray(ndims,
-				    array_of_sizes,
-				    array_of_subsizes,
-				    array_of_starts,
-				    order,
-				    oldtype,
-				    newtype);
-    /* --BEGIN ERROR HANDLING-- */
-    if (err != MPI_SUCCESS) {
-	error_code = MPIO_Err_create_code(MPI_SUCCESS,
-					  MPIR_ERR_RECOVERABLE,
-					  myname, __LINE__, err,
-					  "Internal error", 0);
-    }
-    /* --END ERROR HANDLING-- */
-
-    return MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/mpi2-other/info/Makefile.in b/3rd-party/romio321/mpi2-other/info/Makefile.in
deleted file mode 100644
index b77f08e2b3a..00000000000
--- a/3rd-party/romio321/mpi2-other/info/Makefile.in
+++ /dev/null
@@ -1,105 +0,0 @@
-CC          = @CC@
-AR          = @AR@
-LIBNAME     = @LIBNAME@
-RANLIB      = @RANLIB@
-srcdir      = @srcdir@
-CC_SHL      = @CC_SHL@
-SHLIBNAME   = @SHLIBNAME@
-
-INCLUDE_DIR = -I@MPI_INCLUDE_DIR@ -I../../include -I${srcdir}/../../mpi-io -I${srcdir}/../../adio/include -I../../adio/include
-CFLAGS      = -DHAVE_MPI_INFO_SRC @CFLAGS@ $(MPIOPROFILE) $(INCLUDE_DIR)
-
-top_builddir  = @master_topbuild_dir@
-LIBTOOL       = @LIBTOOL@
-C_COMPILE_SHL = $(CC_SHL)
-
-@VPATH@
-
-MPIO_OBJECTS = info_create.o info_dup.o info_get.o info_getnth.o \
-      info_set.o info_delete.o info_free.o info_getnks.o info_getvln.o \
-      info_c2f.o info_f2c.o
-
-MPIO_TMP_POBJECTS = info_create.p info_dup.p info_get.p info_getnth.p \
-      info_set.p info_delete.p info_free.p info_getnks.p info_getvln.p \
-      info_c2f.p info_f2c.p
-
-MPIO_REAL_POBJECTS = _info_create.o _info_dup.o _info_get.o _info_getnth.o \
-      _info_set.o _info_delete.o _info_free.o _info_getnks.o _info_getvln.o \
-      _info_c2f.o _info_f2c.o
-
-all: $(LIBNAME)
-	@if [ "@ENABLE_SHLIB@" != "none" ] ; then \
-	    $(MAKE) $(SHLIBNAME).la ;\
-	fi
-
-.SUFFIXES: $(SUFFIXES) .p .sp .lo
-
-.c.o:
-	$(CC) $(CFLAGS) -c $<
-
-.c.lo:
-	$(C_COMPILE_SHL) $(CFLAGS) -c $< -o _s$*.o
-	@mv -f _s$*.o $*.lo
-
-.c.p:
-	@cp $(srcdir)/$*.c _$*.c
-	$(CC) $(CFLAGS) -c _$*.c
-	@rm -f _$*.c
-
-.c.sp:
-	$(C_COMPILE_SHL) $(CFLAGS) -c $< -o _$*.lo
-
-profile: $(MPIO_REAL_POBJECTS)
-	$(AR) $(LIBNAME) $(MPIO_REAL_POBJECTS)
-	$(RANLIB) $(LIBNAME) 
-	@if [ "@ENABLE_SHLIB@" != "none" ] ; then \
-	    $(MAKE) P$(SHLIBNAME).la ;\
-	fi
-	@rm -f _*.o
-
-$(LIBNAME): $(MPIO_OBJECTS)
-	$(AR) $(LIBNAME) $(MPIO_OBJECTS)
-	$(RANLIB) $(LIBNAME) 
-
-MPIO_LOOBJECTS = $(MPIO_OBJECTS:.o=.lo)
-$(SHLIBNAME).la: $(MPIO_LOOBJECTS) 
-	$(AR) $(SHLIBNAME).la $(MPIO_LOOBJECTS)
-
-# --------------------------------------------------------------------------
-# We use P$(SHLBNAME) simply to distinguish the regular lib build from the
-# profiling library build
-MPIO_LOPOBJECTS = $(MPIO_REAL_POBJECTS:.o=.lo)
-MPIO_TMP_LOPOBJECTS = ${MPIO_TMP_POBJECTS:.p=.sp}
-P$(SHLIBNAME).la: $(MPIO_TMP_LOPOBJECTS)
-	$(AR) $(SHLIBNAME).la $(MPIO_LOPOBJECTS)
-# --------------------------------------------------------------------------
-
-clean:
-	@rm -f *.o *.lo *.gcno *.gcda *.bb *.bbg
-	@rm -f ${srcdir}/*.gcno ${srcdir}/*.gcda 
-	@rm -f ${srcdir}/*.bb ${srcdir}/*.bbg
-
-
-# Rules for the profiling objects
-info_create.o: infocreate.c
-	$(CC) $(CFLAGS) -c $(srcdir)/infocreate.c -o _info_create.o
-info_dup.o: infodup.c
-	$(CC) $(CFLAGS) -c $(srcdir)/infodup.c -o _info_dup.o
-info_get.o: infoget.c
-	$(CC) $(CFLAGS) -c $(srcdir)/infoget.c -o _info_get.o
-info_getnth.o: infogetnth.c
-	$(CC) $(CFLAGS) -c $(srcdir)/infogetnth.c -o _info_getnth.o
-info_set.o: infoset.c
-	$(CC) $(CFLAGS) -c $(srcdir)/infoset.c -o _info_set.o
-info_delete.o: infodelete.c
-	$(CC) $(CFLAGS) -c $(srcdir)/infodelete.c -o _info_delete.o
-info_free.o: infofree.c
-	$(CC) $(CFLAGS) -c $(srcdir)/infofree.c -o _info_free.o
-info_getnks.o: infogetnks.c
-	$(CC) $(CFLAGS) -c $(srcdir)/infogetnks.c -o _info_getnks.o
-info_getvln.o: infogetvln.c
-	$(CC) $(CFLAGS) -c $(srcdir)/infogetvln.c -o _info_getvln.o
-info_c2f.o: infoc2f.c
-	$(CC) $(CFLAGS) -c $(srcdir)/infoc2f.c -o _info_c2f.o
-info_f2c.o: infof2c.c
-	$(CC) $(CFLAGS) -c $(srcdir)/infof2c.c -o _info_f2c.o
diff --git a/3rd-party/romio321/mpi2-other/info/fortran/Makefile.in b/3rd-party/romio321/mpi2-other/info/fortran/Makefile.in
deleted file mode 100644
index c84769c5467..00000000000
--- a/3rd-party/romio321/mpi2-other/info/fortran/Makefile.in
+++ /dev/null
@@ -1,65 +0,0 @@
-CC          = @CC@
-AR          = @AR@
-LIBNAME     = @LIBNAME@
-RANLIB      = @RANLIB@
-srcdir      = @srcdir@
-
-INCLUDE_DIR = -I@MPI_INCLUDE_DIR@ -I../../../include -I${srcdir}/../../../adio/include -I${srcdir}/../../../mpi-io -I../../../adio/include
-CFLAGS      = @CFLAGS@ $(MPIOPROFILE) $(INCLUDE_DIR)
-
-@VPATH@
-
-MPIO_OBJECTS = info_createf.o info_dupf.o info_getf.o info_getnthf.o \
-      info_setf.o info_deletef.o info_freef.o info_getnksf.o info_getvlnf.o
-
-MPIO_TMP_POBJECTS = info_createf.p info_dupf.p info_getf.p info_getnthf.p \
-      info_setf.p info_deletef.p info_freef.p info_getnksf.p info_getvlnf.p
-
-MPIO_REAL_POBJECTS = _info_createf.o _info_dupf.o _info_getf.o _info_getnthf.o\
-    _info_setf.o _info_deletef.o _info_freef.o _info_getnksf.o _info_getvlnf.o
-
-all: $(LIBNAME)
-
-.SUFFIXES: $(SUFFIXES) .p
-
-.c.o:
-	$(CC) $(CFLAGS) -c $<
-
-.c.p:
-	@cp $(srcdir)/$*.c _$*.c
-	$(CC) $(CFLAGS) -c _$*.c
-	@rm -f _$*.c
-
-profile: $(MPIO_REAL_POBJECTS)
-	$(AR) $(LIBNAME) $(MPIO_REAL_POBJECTS)
-	$(RANLIB) $(LIBNAME) 
-	@rm -f _*.o
-
-$(LIBNAME): $(MPIO_OBJECTS)
-	$(AR) $(LIBNAME) $(MPIO_OBJECTS)
-	$(RANLIB) $(LIBNAME) 
-
-clean:
-	@rm -f *.o *.lo *.gcno *.gcda *.bb *.bbg
-	@rm -f ${srcdir}/*.gcno ${srcdir}/*.gcda 
-	@rm -f ${srcdir}/*.bb ${srcdir}/*.bbg
-
-# Rules for the profiling objects
-_info_createf.o: info_createf.c
-	$(CC) $(CFLAGS) -c $(srcdir)/info_createf.c -o _info_createf.o
-_info_dupf.o: info_dupf.c
-	$(CC) $(CFLAGS) -c $(srcdir)/info_dupf.c -o _info_dupf.o
-_info_getf.o: info_getf.c
-	$(CC) $(CFLAGS) -c $(srcdir)/info_getf.c -o _info_getf.o
-_info_getnthf.o: info_getnthf.c
-	$(CC) $(CFLAGS) -c $(srcdir)/info_getnthf.c -o _info_getnthf.o
-_info_setf.o: info_setf.c
-	$(CC) $(CFLAGS) -c $(srcdir)/info_setf.c -o _info_setf.o
-_info_deletef.o: info_deletef.c
-	$(CC) $(CFLAGS) -c $(srcdir)/info_deletef.c -o _info_deletef.o
-_info_freef.o: info_freef.c
-	$(CC) $(CFLAGS) -c $(srcdir)/info_freef.c -o _info_freef.o
-_info_getnksf.o: info_getnksf.c
-	$(CC) $(CFLAGS) -c $(srcdir)/info_getnksf.c -o _info_getnksf.o
-_info_getvlnf.o: info_getvlnf.c
-	$(CC) $(CFLAGS) -c $(srcdir)/info_getvlnf.c -o _info_getvlnf.o
diff --git a/3rd-party/romio321/mpi2-other/info/fortran/info_createf.c b/3rd-party/romio321/mpi2-other/info/fortran/info_createf.c
deleted file mode 100644
index 5d01ce93d02..00000000000
--- a/3rd-party/romio321/mpi2-other/info/fortran/info_createf.c
+++ /dev/null
@@ -1,93 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpio.h"
-#include "adio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-#ifdef FORTRANCAPS
-#define mpi_info_create_ PMPI_INFO_CREATE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_info_create_ pmpi_info_create__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_info_create pmpi_info_create_
-#endif
-#define mpi_info_create_ pmpi_info_create
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_info_create_ pmpi_info_create
-#endif
-#define mpi_info_create_ pmpi_info_create_
-#endif
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-#pragma weak MPI_INFO_CREATE = PMPI_INFO_CREATE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma weak mpi_info_create__ = pmpi_info_create__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma weak mpi_info_create = pmpi_info_create
-#else
-#pragma weak mpi_info_create_ = pmpi_info_create_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_INFO_CREATE MPI_INFO_CREATE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_info_create__ mpi_info_create__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_info_create mpi_info_create
-#else
-#pragma _HP_SECONDARY_DEF pmpi_info_create_ mpi_info_create_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_INFO_CREATE as PMPI_INFO_CREATE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_info_create__ as pmpi_info_create__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_info_create as pmpi_info_create
-#else
-#pragma _CRI duplicate mpi_info_create_ as pmpi_info_create_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_info_create_ MPI_INFO_CREATE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_info_create_ mpi_info_create__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_info_create mpi_info_create_
-#endif
-#define mpi_info_create_ mpi_info_create
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_info_create_ mpi_info_create
-#endif
-#endif
-#endif
-
-void mpi_info_create_(MPI_Fint *info, int *ierr )
-{
-    MPI_Info info_c;
-
-    *ierr = MPI_Info_create(&info_c);
-    *info = MPI_Info_c2f(info_c);
-}
diff --git a/3rd-party/romio321/mpi2-other/info/fortran/info_deletef.c b/3rd-party/romio321/mpi2-other/info/fortran/info_deletef.c
deleted file mode 100644
index f7a7bb4d58e..00000000000
--- a/3rd-party/romio321/mpi2-other/info/fortran/info_deletef.c
+++ /dev/null
@@ -1,120 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpio.h"
-#include "adio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-#ifdef FORTRANCAPS
-#define mpi_info_delete_ PMPI_INFO_DELETE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_info_delete_ pmpi_info_delete__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_info_delete pmpi_info_delete_
-#endif
-#define mpi_info_delete_ pmpi_info_delete
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_info_delete_ pmpi_info_delete
-#endif
-#define mpi_info_delete_ pmpi_info_delete_
-#endif
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-#pragma weak MPI_INFO_DELETE = PMPI_INFO_DELETE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma weak mpi_info_delete__ = pmpi_info_delete__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma weak mpi_info_delete = pmpi_info_delete
-#else
-#pragma weak mpi_info_delete_ = pmpi_info_delete_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_INFO_DELETE MPI_INFO_DELETE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_info_delete__ mpi_info_delete__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_info_delete mpi_info_delete
-#else
-#pragma _HP_SECONDARY_DEF pmpi_info_delete_ mpi_info_delete_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_INFO_DELETE as PMPI_INFO_DELETE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_info_delete__ as pmpi_info_delete__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_info_delete as pmpi_info_delete
-#else
-#pragma _CRI duplicate mpi_info_delete_ as pmpi_info_delete_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_info_delete_ MPI_INFO_DELETE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_info_delete_ mpi_info_delete__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_info_delete mpi_info_delete_
-#endif
-#define mpi_info_delete_ mpi_info_delete
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_info_delete_ mpi_info_delete
-#endif
-#endif
-#endif
-
-void mpi_info_delete_(MPI_Fint *info, char *key, int *ierr, int keylen)
-{
-    MPI_Info info_c;
-    char *newkey;
-    int new_keylen, lead_blanks, i;
-
-    if (key <= (char *) 0) {
-        FPRINTF(stderr, "MPI_Info_delete: key is an invalid address\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    /* strip leading and trailing blanks in key */
-    lead_blanks = 0;
-    for (i=0; i<keylen; i++) 
-        if (key[i] == ' ') lead_blanks++;
-        else break;
-
-    for (i=keylen-1; i>=0; i--) if (key[i] != ' ') break;
-    if (i < 0) {
-        FPRINTF(stderr, "MPI_Info_delete: key is a blank string\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-    new_keylen = i + 1 - lead_blanks;
-    key += lead_blanks;
-
-    newkey = (char *) ADIOI_Malloc((new_keylen+1)*sizeof(char));
-    ADIOI_Strncpy(newkey, key, new_keylen);
-    newkey[new_keylen] = '\0';
-
-    info_c = MPI_Info_f2c(*info);
-    *ierr = MPI_Info_delete(info_c, newkey);
-    ADIOI_Free(newkey);
-}
-
diff --git a/3rd-party/romio321/mpi2-other/info/fortran/info_dupf.c b/3rd-party/romio321/mpi2-other/info/fortran/info_dupf.c
deleted file mode 100644
index 87162005ab6..00000000000
--- a/3rd-party/romio321/mpi2-other/info/fortran/info_dupf.c
+++ /dev/null
@@ -1,94 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpio.h"
-#include "adio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-#ifdef FORTRANCAPS
-#define mpi_info_dup_ PMPI_INFO_DUP
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_info_dup_ pmpi_info_dup__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_info_dup pmpi_info_dup_
-#endif
-#define mpi_info_dup_ pmpi_info_dup
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_info_dup_ pmpi_info_dup
-#endif
-#define mpi_info_dup_ pmpi_info_dup_
-#endif
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-#pragma weak MPI_INFO_DUP = PMPI_INFO_DUP
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma weak mpi_info_dup__ = pmpi_info_dup__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma weak mpi_info_dup = pmpi_info_dup
-#else
-#pragma weak mpi_info_dup_ = pmpi_info_dup_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_INFO_DUP MPI_INFO_DUP
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_info_dup__ mpi_info_dup__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_info_dup mpi_info_dup
-#else
-#pragma _HP_SECONDARY_DEF pmpi_info_dup_ mpi_info_dup_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_INFO_DUP as PMPI_INFO_DUP
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_info_dup__ as pmpi_info_dup__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_info_dup as pmpi_info_dup
-#else
-#pragma _CRI duplicate mpi_info_dup_ as pmpi_info_dup_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_info_dup_ MPI_INFO_DUP
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_info_dup_ mpi_info_dup__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_info_dup mpi_info_dup_
-#endif
-#define mpi_info_dup_ mpi_info_dup
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_info_dup_ mpi_info_dup
-#endif
-#endif
-#endif
-
-void mpi_info_dup_(MPI_Fint *info, MPI_Fint *newinfo, int *ierr )
-{
-    MPI_Info info_c, newinfo_c;
-
-    info_c = MPI_Info_f2c(*info);
-    *ierr = MPI_Info_dup(info_c, &newinfo_c);
-    *newinfo = MPI_Info_c2f(newinfo_c);
-}
diff --git a/3rd-party/romio321/mpi2-other/info/fortran/info_freef.c b/3rd-party/romio321/mpi2-other/info/fortran/info_freef.c
deleted file mode 100644
index 5f200c96227..00000000000
--- a/3rd-party/romio321/mpi2-other/info/fortran/info_freef.c
+++ /dev/null
@@ -1,95 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpio.h"
-#include "adio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-#ifdef FORTRANCAPS
-#define mpi_info_free_ PMPI_INFO_FREE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_info_free_ pmpi_info_free__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_info_free pmpi_info_free_
-#endif
-#define mpi_info_free_ pmpi_info_free
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_info_free_ pmpi_info_free
-#endif
-#define mpi_info_free_ pmpi_info_free_
-#endif
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-#pragma weak MPI_INFO_FREE = PMPI_INFO_FREE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma weak mpi_info_free__ = pmpi_info_free__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma weak mpi_info_free = pmpi_info_free
-#else
-#pragma weak mpi_info_free_ = pmpi_info_free_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_INFO_FREE MPI_INFO_FREE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_info_free__ mpi_info_free__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_info_free mpi_info_free
-#else
-#pragma _HP_SECONDARY_DEF pmpi_info_free_ mpi_info_free_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_INFO_FREE as PMPI_INFO_FREE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_info_free__ as pmpi_info_free__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_info_free as pmpi_info_free
-#else
-#pragma _CRI duplicate mpi_info_free_ as pmpi_info_free_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_info_free_ MPI_INFO_FREE
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_info_free_ mpi_info_free__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_info_free mpi_info_free_
-#endif
-#define mpi_info_free_ mpi_info_free
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_info_free_ mpi_info_free
-#endif
-#endif
-#endif
-
-void mpi_info_free_(MPI_Fint *info, int *ierr )
-{
-    MPI_Info info_c;
-
-    info_c = MPI_Info_f2c(*info);
-    *ierr = MPI_Info_free(&info_c);
-    *info = MPI_Info_c2f(info_c);
-}
-
diff --git a/3rd-party/romio321/mpi2-other/info/fortran/info_getf.c b/3rd-party/romio321/mpi2-other/info/fortran/info_getf.c
deleted file mode 100644
index ae60cc22b07..00000000000
--- a/3rd-party/romio321/mpi2-other/info/fortran/info_getf.c
+++ /dev/null
@@ -1,144 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpio.h"
-#include "adio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-#ifdef FORTRANCAPS
-#define mpi_info_get_ PMPI_INFO_GET
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_info_get_ pmpi_info_get__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_info_get pmpi_info_get_
-#endif
-#define mpi_info_get_ pmpi_info_get
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_info_get_ pmpi_info_get
-#endif
-#define mpi_info_get_ pmpi_info_get_
-#endif
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-#pragma weak MPI_INFO_GET = PMPI_INFO_GET
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma weak mpi_info_get__ = pmpi_info_get__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma weak mpi_info_get = pmpi_info_get
-#else
-#pragma weak mpi_info_get_ = pmpi_info_get_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_INFO_GET MPI_INFO_GET
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_info_get__ mpi_info_get__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_info_get mpi_info_get
-#else
-#pragma _HP_SECONDARY_DEF pmpi_info_get_ mpi_info_get_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_INFO_GET as PMPI_INFO_GET
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_info_get__ as pmpi_info_get__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_info_get as pmpi_info_get
-#else
-#pragma _CRI duplicate mpi_info_get_ as pmpi_info_get_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_info_get_ MPI_INFO_GET
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_info_get_ mpi_info_get__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_info_get mpi_info_get_
-#endif
-#define mpi_info_get_ mpi_info_get
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_info_get_ mpi_info_get
-#endif
-#endif
-#endif
-
-void mpi_info_get_(MPI_Fint *info, char *key, int *valuelen, char *value, 
-        int *flag, int *ierr, int keylen, int valspace)
-{
-    MPI_Info info_c;
-    char *newkey, *tmpvalue;
-    int new_keylen, lead_blanks, i, tmpvaluelen;
-
-    if (key <= (char *) 0) {
-        FPRINTF(stderr, "MPI_Info_get: key is an invalid address\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    /* strip leading and trailing blanks in key */
-    lead_blanks = 0;
-    for (i=0; i<keylen; i++) 
-        if (key[i] == ' ') lead_blanks++;
-        else break;
-
-    for (i=keylen-1; i>=0; i--) if (key[i] != ' ') break;
-    if (i < 0) {
-        FPRINTF(stderr, "MPI_Info_get: key is a blank string\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-    new_keylen = i + 1 - lead_blanks;
-    key += lead_blanks;
-
-    newkey = (char *) ADIOI_Malloc((new_keylen+1)*sizeof(char));
-    ADIOI_Strncpy(newkey, key, new_keylen);
-    newkey[new_keylen] = '\0';
-
-    if (value <= (char *) 0) {
-        FPRINTF(stderr, "MPI_Info_get: value is an invalid address\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-    if (*valuelen <= 0) {
-        FPRINTF(stderr, "MPI_Info_get: Invalid valuelen argument\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-    if (*valuelen > valspace) {
-        FPRINTF(stderr, "MPI_Info_get: valuelen is greater than the amount of memory available in value\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-    
-    tmpvalue = (char *) ADIOI_Malloc((*valuelen + 1)*sizeof(char));
-
-    info_c = MPI_Info_f2c(*info);
-    *ierr = MPI_Info_get(info_c, newkey, *valuelen, tmpvalue, flag);
-
-    if (*flag) {
-	tmpvaluelen = strlen(tmpvalue);
-	ADIOI_Strncpy(value, tmpvalue, tmpvaluelen);
-	/* blank pad the remaining space */
-	for (i=tmpvaluelen; i<valspace; i++) value[i] = ' ';
-    }
-	
-    ADIOI_Free(newkey);
-    ADIOI_Free(tmpvalue);
-}
diff --git a/3rd-party/romio321/mpi2-other/info/fortran/info_getnksf.c b/3rd-party/romio321/mpi2-other/info/fortran/info_getnksf.c
deleted file mode 100644
index ac1fa8bac1c..00000000000
--- a/3rd-party/romio321/mpi2-other/info/fortran/info_getnksf.c
+++ /dev/null
@@ -1,93 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpio.h"
-#include "adio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-#ifdef FORTRANCAPS
-#define mpi_info_get_nkeys_ PMPI_INFO_GET_NKEYS
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_info_get_nkeys_ pmpi_info_get_nkeys__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_info_get_nkeys pmpi_info_get_nkeys_
-#endif
-#define mpi_info_get_nkeys_ pmpi_info_get_nkeys
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_info_get_nkeys_ pmpi_info_get_nkeys
-#endif
-#define mpi_info_get_nkeys_ pmpi_info_get_nkeys_
-#endif
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-#pragma weak MPI_INFO_GET_NKEYS = PMPI_INFO_GET_NKEYS
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma weak mpi_info_get_nkeys__ = pmpi_info_get_nkeys__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma weak mpi_info_get_nkeys = pmpi_info_get_nkeys
-#else
-#pragma weak mpi_info_get_nkeys_ = pmpi_info_get_nkeys_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_INFO_GET_NKEYS MPI_INFO_GET_NKEYS
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_info_get_nkeys__ mpi_info_get_nkeys__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_info_get_nkeys mpi_info_get_nkeys
-#else
-#pragma _HP_SECONDARY_DEF pmpi_info_get_nkeys_ mpi_info_get_nkeys_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_INFO_GET_NKEYS as PMPI_INFO_GET_NKEYS
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_info_get_nkeys__ as pmpi_info_get_nkeys__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_info_get_nkeys as pmpi_info_get_nkeys
-#else
-#pragma _CRI duplicate mpi_info_get_nkeys_ as pmpi_info_get_nkeys_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_info_get_nkeys_ MPI_INFO_GET_NKEYS
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_info_get_nkeys_ mpi_info_get_nkeys__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_info_get_nkeys mpi_info_get_nkeys_
-#endif
-#define mpi_info_get_nkeys_ mpi_info_get_nkeys
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_info_get_nkeys_ mpi_info_get_nkeys
-#endif
-#endif
-#endif
-
-void mpi_info_get_nkeys_(MPI_Fint *info, int *nkeys, int *ierr )
-{
-    MPI_Info info_c;
-    
-    info_c = MPI_Info_f2c(*info);
-    *ierr = MPI_Info_get_nkeys(info_c, nkeys);
-}
diff --git a/3rd-party/romio321/mpi2-other/info/fortran/info_getnthf.c b/3rd-party/romio321/mpi2-other/info/fortran/info_getnthf.c
deleted file mode 100644
index 5434fe3dd3e..00000000000
--- a/3rd-party/romio321/mpi2-other/info/fortran/info_getnthf.c
+++ /dev/null
@@ -1,120 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpio.h"
-#include "adio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-#ifdef FORTRANCAPS
-#define mpi_info_get_nthkey_ PMPI_INFO_GET_NTHKEY
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_info_get_nthkey_ pmpi_info_get_nthkey__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_info_get_nthkey pmpi_info_get_nthkey_
-#endif
-#define mpi_info_get_nthkey_ pmpi_info_get_nthkey
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_info_get_nthkey_ pmpi_info_get_nthkey
-#endif
-#define mpi_info_get_nthkey_ pmpi_info_get_nthkey_
-#endif
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-#pragma weak MPI_INFO_GET_NTHKEY = PMPI_INFO_GET_NTHKEY
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma weak mpi_info_get_nthkey__ = pmpi_info_get_nthkey__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma weak mpi_info_get_nthkey = pmpi_info_get_nthkey
-#else
-#pragma weak mpi_info_get_nthkey_ = pmpi_info_get_nthkey_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_INFO_GET_NTHKEY MPI_INFO_GET_NTHKEY
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_info_get_nthkey__ mpi_info_get_nthkey__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_info_get_nthkey mpi_info_get_nthkey
-#else
-#pragma _HP_SECONDARY_DEF pmpi_info_get_nthkey_ mpi_info_get_nthkey_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_INFO_GET_NTHKEY as PMPI_INFO_GET_NTHKEY
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_info_get_nthkey__ as pmpi_info_get_nthkey__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_info_get_nthkey as pmpi_info_get_nthkey
-#else
-#pragma _CRI duplicate mpi_info_get_nthkey_ as pmpi_info_get_nthkey_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_info_get_nthkey_ MPI_INFO_GET_NTHKEY
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_info_get_nthkey_ mpi_info_get_nthkey__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_info_get_nthkey mpi_info_get_nthkey_
-#endif
-#define mpi_info_get_nthkey_ mpi_info_get_nthkey
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_info_get_nthkey_ mpi_info_get_nthkey
-#endif
-#endif
-#endif
-
-void mpi_info_get_nthkey_(MPI_Fint *info, int *n, char *key, int *ierr,
-                          int keylen)
-{
-    MPI_Info info_c;
-    int i, tmpkeylen;
-    char *tmpkey;
-
-    if (key <= (char *) 0) {
-        FPRINTF(stderr, "MPI_Info_get_nthkey: key is an invalid address\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    tmpkey = (char *) ADIOI_Malloc((MPI_MAX_INFO_KEY+1) * sizeof(char));
-    info_c = MPI_Info_f2c(*info);
-    *ierr = MPI_Info_get_nthkey(info_c, *n, tmpkey);
-
-    tmpkeylen = strlen(tmpkey);
-
-    if (tmpkeylen <= keylen) {
-	ADIOI_Strncpy(key, tmpkey, tmpkeylen);
-
-	/* blank pad the remaining space */
-	for (i=tmpkeylen; i<keylen; i++) key[i] = ' ';
-    }
-    else {
-	/* not enough space */
-	ADIOI_Strncpy(key, tmpkey, keylen);
-	/* this should be flagged as an error. */
-	*ierr = MPI_ERR_UNKNOWN;
-    }
-
-    ADIOI_Free(tmpkey);
-}
-
diff --git a/3rd-party/romio321/mpi2-other/info/fortran/info_getvlnf.c b/3rd-party/romio321/mpi2-other/info/fortran/info_getvlnf.c
deleted file mode 100644
index 95030277e63..00000000000
--- a/3rd-party/romio321/mpi2-other/info/fortran/info_getvlnf.c
+++ /dev/null
@@ -1,120 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpio.h"
-#include "adio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-#ifdef FORTRANCAPS
-#define mpi_info_get_valuelen_ PMPI_INFO_GET_VALUELEN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_info_get_valuelen_ pmpi_info_get_valuelen__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_info_get_valuelen pmpi_info_get_valuelen_
-#endif
-#define mpi_info_get_valuelen_ pmpi_info_get_valuelen
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_info_get_valuelen_ pmpi_info_get_valuelen
-#endif
-#define mpi_info_get_valuelen_ pmpi_info_get_valuelen_
-#endif
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-#pragma weak MPI_INFO_GET_VALUELEN = PMPI_INFO_GET_VALUELEN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma weak mpi_info_get_valuelen__ = pmpi_info_get_valuelen__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma weak mpi_info_get_valuelen = pmpi_info_get_valuelen
-#else
-#pragma weak mpi_info_get_valuelen_ = pmpi_info_get_valuelen_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_INFO_GET_VALUELEN MPI_INFO_GET_VALUELEN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_info_get_valuelen__ mpi_info_get_valuelen__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_info_get_valuelen mpi_info_get_valuelen
-#else
-#pragma _HP_SECONDARY_DEF pmpi_info_get_valuelen_ mpi_info_get_valuelen_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_INFO_GET_VALUELEN as PMPI_INFO_GET_VALUELEN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_info_get_valuelen__ as pmpi_info_get_valuelen__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_info_get_valuelen as pmpi_info_get_valuelen
-#else
-#pragma _CRI duplicate mpi_info_get_valuelen_ as pmpi_info_get_valuelen_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_info_get_valuelen_ MPI_INFO_GET_VALUELEN
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_info_get_valuelen_ mpi_info_get_valuelen__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_info_get_valuelen mpi_info_get_valuelen_
-#endif
-#define mpi_info_get_valuelen_ mpi_info_get_valuelen
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_info_get_valuelen_ mpi_info_get_valuelen
-#endif
-#endif
-#endif
-
-void mpi_info_get_valuelen_(MPI_Fint *info, char *key, int *valuelen,
-                 int *flag, int *ierr, int keylen )
-{
-    MPI_Info info_c;
-    char *newkey;
-    int new_keylen, lead_blanks, i;
-
-    if (key <= (char *) 0) {
-        FPRINTF(stderr, "MPI_Info_get_valuelen: key is an invalid address\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    /* strip leading and trailing blanks in key */
-    lead_blanks = 0;
-    for (i=0; i<keylen; i++) 
-        if (key[i] == ' ') lead_blanks++;
-        else break;
-
-    for (i=keylen-1; i>=0; i--) if (key[i] != ' ') break;
-    if (i < 0) {
-        FPRINTF(stderr, "MPI_Info_get_valuelen: key is a blank string\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-    new_keylen = i + 1 - lead_blanks;
-    key += lead_blanks;
-
-    newkey = (char *) ADIOI_Malloc((new_keylen+1)*sizeof(char));
-    ADIOI_Strncpy(newkey, key, new_keylen);
-    newkey[new_keylen] = '\0';
-
-    info_c = MPI_Info_f2c(*info);
-    *ierr = MPI_Info_get_valuelen(info_c, newkey, valuelen, flag);
-    ADIOI_Free(newkey);
-}
diff --git a/3rd-party/romio321/mpi2-other/info/fortran/info_setf.c b/3rd-party/romio321/mpi2-other/info/fortran/info_setf.c
deleted file mode 100644
index 6b6458d2611..00000000000
--- a/3rd-party/romio321/mpi2-other/info/fortran/info_setf.c
+++ /dev/null
@@ -1,146 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpio.h"
-#include "adio.h"
-
-
-#if defined(MPIO_BUILD_PROFILING) || defined(HAVE_WEAK_SYMBOLS)
-#ifdef FORTRANCAPS
-#define mpi_info_set_ PMPI_INFO_SET
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_info_set_ pmpi_info_set__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_info_set pmpi_info_set_
-#endif
-#define mpi_info_set_ pmpi_info_set
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF pmpi_info_set_ pmpi_info_set
-#endif
-#define mpi_info_set_ pmpi_info_set_
-#endif
-
-#if defined(HAVE_WEAK_SYMBOLS)
-#if defined(HAVE_PRAGMA_WEAK)
-#if defined(FORTRANCAPS)
-#pragma weak MPI_INFO_SET = PMPI_INFO_SET
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma weak mpi_info_set__ = pmpi_info_set__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma weak mpi_info_set = pmpi_info_set
-#else
-#pragma weak mpi_info_set_ = pmpi_info_set_
-#endif
-
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#if defined(FORTRANCAPS)
-#pragma _HP_SECONDARY_DEF PMPI_INFO_SET MPI_INFO_SET
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_info_set__ mpi_info_set__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _HP_SECONDARY_DEF pmpi_info_set mpi_info_set
-#else
-#pragma _HP_SECONDARY_DEF pmpi_info_set_ mpi_info_set_
-#endif
-
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#if defined(FORTRANCAPS)
-#pragma _CRI duplicate MPI_INFO_SET as PMPI_INFO_SET
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#pragma _CRI duplicate mpi_info_set__ as pmpi_info_set__
-#elif !defined(FORTRANUNDERSCORE)
-#pragma _CRI duplicate mpi_info_set as pmpi_info_set
-#else
-#pragma _CRI duplicate mpi_info_set_ as pmpi_info_set_
-#endif
-
-/* end of weak pragmas */
-#endif
-/* Include mapping from MPI->PMPI */
-#include "mpioprof.h"
-#endif
-
-#else
-
-#ifdef FORTRANCAPS
-#define mpi_info_set_ MPI_INFO_SET
-#elif defined(FORTRANDOUBLEUNDERSCORE)
-#define mpi_info_set_ mpi_info_set__
-#elif !defined(FORTRANUNDERSCORE)
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_info_set mpi_info_set_
-#endif
-#define mpi_info_set_ mpi_info_set
-#else
-#if defined(HPUX) || defined(SPPUX)
-#pragma _HP_SECONDARY_DEF mpi_info_set_ mpi_info_set
-#endif
-#endif
-#endif
-
-
-void mpi_info_set_(MPI_Fint *info, char *key, char *value, int *ierr, 
-                   int keylen, int vallen)
-{
-    MPI_Info info_c;
-    char *newkey, *newvalue;
-    int new_keylen, new_vallen, lead_blanks, i;
-
-    if (key <= (char *) 0) {
-        FPRINTF(stderr, "MPI_Info_set: key is an invalid address\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-    if (value <= (char *) 0) {
-        FPRINTF(stderr, "MPI_Info_set: value is an invalid address\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    /* strip leading and trailing blanks in key */
-    lead_blanks = 0;
-    for (i=0; i<keylen; i++) 
-	if (key[i] == ' ') lead_blanks++;
-	else break;
-
-    for (i=keylen-1; i>=0; i--) if (key[i] != ' ') break;
-    if (i < 0) {
-        FPRINTF(stderr, "MPI_Info_set: key is a blank string\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-    new_keylen = i + 1 - lead_blanks;
-    key += lead_blanks;
-
-    newkey = (char *) ADIOI_Malloc((new_keylen+1)*sizeof(char));
-    ADIOI_Strncpy(newkey, key, new_keylen);
-    newkey[new_keylen] = '\0';
-
-
-    /* strip leading and trailing blanks in value */
-    lead_blanks = 0;
-    for (i=0; i<vallen; i++) 
-	if (value[i] == ' ') lead_blanks++;
-	else break;
-
-    for (i=vallen-1; i>=0; i--) if (value[i] != ' ') break;
-    if (i < 0) {
-        FPRINTF(stderr, "MPI_Info_set: value is a blank string\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-    new_vallen = i + 1 - lead_blanks;
-    value += lead_blanks;
-
-    newvalue = (char *) ADIOI_Malloc((new_vallen+1)*sizeof(char));
-    ADIOI_Strncpy(newvalue, value, new_vallen);
-    newvalue[new_vallen] = '\0';
-
- 
-    info_c = MPI_Info_f2c(*info);
-    *ierr = MPI_Info_set(info_c, newkey, newvalue);
-    ADIOI_Free(newkey);
-    ADIOI_Free(newvalue);
-}
diff --git a/3rd-party/romio321/mpi2-other/info/info_c2f.c b/3rd-party/romio321/mpi2-other/info/info_c2f.c
deleted file mode 100644
index 4486887b545..00000000000
--- a/3rd-party/romio321/mpi2-other/info/info_c2f.c
+++ /dev/null
@@ -1,64 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_Info_c2f = PMPI_Info_c2f
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_Info_c2f MPI_Info_c2f
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_Info_c2f as PMPI_Info_c2f
-/* end of weak pragmas */
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-#include "adio_extern.h"
-
-/*@
-    MPI_Info_c2f - Translates a C info handle to a Fortran info handle
-
-Input Parameters:
-. info - C info handle (integer)
-
-Return Value:
-  Fortran info handle (handle)
-@*/
-MPI_Fint MPI_Info_c2f(MPI_Info info)
-{
-#ifndef INT_LT_POINTER
-    return (MPI_Fint) info;
-#else
-    int i;
-
-    if ((info <= (MPI_Info) 0) || (info->cookie != MPIR_INFO_COOKIE)) 
-	return (MPI_Fint) 0;
-    if (!MPIR_Infotable) {
-	MPIR_Infotable_max = 1024;
-	MPIR_Infotable = (MPI_Info *)
-	    ADIOI_Malloc(MPIR_Infotable_max*sizeof(MPI_Info)); 
-        MPIR_Infotable_ptr = 0;  /* 0 can't be used though, because 
-                                  MPI_INFO_NULL=0 */
-	for (i=0; i<MPIR_Infotable_max; i++) MPIR_Infotable[i] = MPI_INFO_NULL;
-    }
-    if (MPIR_Infotable_ptr == MPIR_Infotable_max-1) {
-	MPIR_Infotable = (MPI_Info *) ADIOI_Realloc(MPIR_Infotable, 
-                           (MPIR_Infotable_max+1024)*sizeof(MPI_Info));
-	for (i=MPIR_Infotable_max; i<MPIR_Infotable_max+1024; i++) 
-	    MPIR_Infotable[i] = MPI_INFO_NULL;
-	MPIR_Infotable_max += 1024;
-    }
-    MPIR_Infotable_ptr++;
-    MPIR_Infotable[MPIR_Infotable_ptr] = info;
-    return (MPI_Fint) MPIR_Infotable_ptr;
-#endif
-}
diff --git a/3rd-party/romio321/mpi2-other/info/info_create.c b/3rd-party/romio321/mpi2-other/info/info_create.c
deleted file mode 100644
index 9b223406edc..00000000000
--- a/3rd-party/romio321/mpi2-other/info/info_create.c
+++ /dev/null
@@ -1,51 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_Info_create = PMPI_Info_create
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_Info_create MPI_Info_create
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_Info_create as PMPI_Info_create
-/* end of weak pragmas */
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_Info_create - Creates a new info object
-
-Output Parameters:
-. info - info object (handle)
-
-.N fortran
-@*/
-int MPI_Info_create(MPI_Info *info)
-{
-    int error_code;
-
-    MPIR_MPIOInit(&error_code);
-    if (error_code != MPI_SUCCESS) goto fn_exit;
-
-    *info = (MPI_Info) ADIOI_Malloc(sizeof(struct MPIR_Info));
-    (*info)->cookie = MPIR_INFO_COOKIE;
-    (*info)->key = 0;
-    (*info)->value = 0;
-    (*info)->next = 0;
-    /* this is the first structure in this linked list. it is 
-       always kept empty. new (key,value) pairs are added after it. */
-
-fn_exit:
-    return MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/mpi2-other/info/info_delete.c b/3rd-party/romio321/mpi2-other/info/info_delete.c
deleted file mode 100644
index 06ec27ee8dd..00000000000
--- a/3rd-party/romio321/mpi2-other/info/info_delete.c
+++ /dev/null
@@ -1,83 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_Info_delete = PMPI_Info_delete
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_Info_delete MPI_Info_delete
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_Info_delete as PMPI_Info_delete
-/* end of weak pragmas */
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_Info_delete - Deletes a (key,value) pair from info
-
-Input Parameters:
-. info - info object (handle)
-. key - key (string)
-
-.N fortran
-@*/
-int MPI_Info_delete(MPI_Info info, char *key)
-{
-    MPI_Info prev, curr;
-    int done;
-
-    if ((info <= (MPI_Info) 0) || (info->cookie != MPIR_INFO_COOKIE)) {
-        FPRINTF(stderr, "MPI_Info_delete: Invalid info object\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    if (key <= (char *) 0) {
-	FPRINTF(stderr, "MPI_Info_delete: key is an invalid address\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    if (strlen(key) > MPI_MAX_INFO_KEY) {
-	FPRINTF(stderr, "MPI_Info_delete: key is longer than MPI_MAX_INFO_KEY\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    if (!strlen(key)) {
-	FPRINTF(stderr, "MPI_Info_delete: key is a null string\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    prev = info;
-    curr = info->next;
-    done = 0;
-
-    while (curr) {
-	if (!strcmp(curr->key, key)) {
-	    ADIOI_Free(curr->key); 
-	    ADIOI_Free(curr->value);
-	    prev->next = curr->next;
-	    ADIOI_Free(curr);
-	    done = 1;
-	    break;
-	}
-	prev = curr;
-	curr = curr->next;
-    }
-
-    if (!done) {
-	FPRINTF(stderr, "MPI_Info_delete: key not defined in info\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);	
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/mpi2-other/info/info_dup.c b/3rd-party/romio321/mpi2-other/info/info_dup.c
deleted file mode 100644
index a1c1a9969fc..00000000000
--- a/3rd-party/romio321/mpi2-other/info/info_dup.c
+++ /dev/null
@@ -1,66 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_Info_dup = PMPI_Info_dup
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_Info_dup MPI_Info_dup
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_Info_dup as PMPI_Info_dup
-/* end of weak pragmas */
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_Info_dup - Returns a duplicate of the info object
-
-Input Parameters:
-. info - info object (handle)
-
-Output Parameters:
-. newinfo - duplicate of info object (handle)
-
-.N fortran
-@*/
-int MPI_Info_dup(MPI_Info info, MPI_Info *newinfo)
-{
-    MPI_Info curr_old, curr_new;
-
-    if ((info <= (MPI_Info) 0) || (info->cookie != MPIR_INFO_COOKIE)) {
-        FPRINTF(stderr, "MPI_Info_dup: Invalid info object\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    *newinfo = (MPI_Info) ADIOI_Malloc(sizeof(struct MPIR_Info));
-    curr_new = *newinfo;
-    curr_new->cookie = MPIR_INFO_COOKIE;
-    curr_new->key = 0;
-    curr_new->value = 0;
-    curr_new->next = 0;
-
-    curr_old = info->next;
-    while (curr_old) {
-	curr_new->next = (MPI_Info) ADIOI_Malloc(sizeof(struct MPIR_Info));
-	curr_new = curr_new->next;
-	curr_new->cookie = 0;  /* cookie not set on purpose */
-	curr_new->key = ADIOI_Strdup(curr_old->key);
-	curr_new->value = ADIOI_Strdup(curr_old->value);
-	curr_new->next = 0;
-	
-	curr_old = curr_old->next;
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/mpi2-other/info/info_f2c.c b/3rd-party/romio321/mpi2-other/info/info_f2c.c
deleted file mode 100644
index 49d8ae5eec2..00000000000
--- a/3rd-party/romio321/mpi2-other/info/info_f2c.c
+++ /dev/null
@@ -1,49 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_Info_f2c = PMPI_Info_f2c
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_Info_f2c MPI_Info_f2c
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_Info_f2c as PMPI_Info_f2c
-/* end of weak pragmas */
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-#include "adio_extern.h"
-
-/*@
-    MPI_Info_f2c - Translates a Fortran info handle to a C info handle
-
-Input Parameters:
-. info - Fortran info handle (integer)
-
-Return Value:
-  C info handle (handle)
-@*/
-MPI_Info MPI_Info_f2c(MPI_Fint info)
-{
-
-#ifndef INT_LT_POINTER
-    return (MPI_Info) info;
-#else
-    if (!info) return MPI_INFO_NULL;
-    if ((info < 0) || (info > MPIR_Infotable_ptr)) {
-	FPRINTF(stderr, "MPI_Info_f2c: Invalid info handle\n");
-	MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-    return MPIR_Infotable[info];
-#endif
-}
diff --git a/3rd-party/romio321/mpi2-other/info/info_free.c b/3rd-party/romio321/mpi2-other/info/info_free.c
deleted file mode 100644
index e682ce0cd58..00000000000
--- a/3rd-party/romio321/mpi2-other/info/info_free.c
+++ /dev/null
@@ -1,56 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_Info_free = PMPI_Info_free
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_Info_free MPI_Info_free
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_Info_free as PMPI_Info_free
-/* end of weak pragmas */
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_Info_free - Frees an info object
-
-Input Parameters:
-. info - info object (handle)
-
-.N fortran
-@*/
-int MPI_Info_free(MPI_Info *info)
-{
-    MPI_Info curr, next;
-
-    if ((*info <= (MPI_Info) 0) || ((*info)->cookie != MPIR_INFO_COOKIE)) {
-        FPRINTF(stderr, "MPI_Info_free: Invalid info object\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    curr = (*info)->next;
-    ADIOI_Free(*info);
-    *info = MPI_INFO_NULL;
-
-    while (curr) {
-	next = curr->next;
-	ADIOI_Free(curr->key);
-	ADIOI_Free(curr->value);
-	ADIOI_Free(curr);
-	curr = next;
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/mpi2-other/info/info_get.c b/3rd-party/romio321/mpi2-other/info/info_get.c
deleted file mode 100644
index ca4274672b5..00000000000
--- a/3rd-party/romio321/mpi2-other/info/info_get.c
+++ /dev/null
@@ -1,88 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_Info_get = PMPI_Info_get
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_Info_get MPI_Info_get
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_Info_get as PMPI_Info_get
-/* end of weak pragmas */
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_Info_get - Retrieves the value associated with a key
-
-Input Parameters:
-. info - info object (handle)
-. key - key (string)
-. valuelen - length of value argument (integer)
-
-Output Parameters:
-. value - value (string)
-. flag - true if key defined, false if not (boolean)
-
-.N fortran
-@*/
-int MPI_Info_get(MPI_Info info, char *key, int valuelen, char *value, int *flag)
-{
-    MPI_Info curr;
-
-    if ((info <= (MPI_Info) 0) || (info->cookie != MPIR_INFO_COOKIE)) {
-        FPRINTF(stderr, "MPI_Info_get: Invalid info object\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    if (key <= (char *) 0) {
-	FPRINTF(stderr, "MPI_Info_get: key is an invalid address\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    if (strlen(key) > MPI_MAX_INFO_KEY) {
-	FPRINTF(stderr, "MPI_Info_get: key is longer than MPI_MAX_INFO_KEY\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    if (!strlen(key)) {
-	FPRINTF(stderr, "MPI_Info_get: key is a null string\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    if (valuelen <= 0) {
-	FPRINTF(stderr, "MPI_Info_get: Invalid valuelen argument\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    if (value <= (char *) 0) {
-	FPRINTF(stderr, "MPI_Info_get: value is an invalid address\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    curr = info->next;
-    *flag = 0;
-
-    while (curr) {
-	if (!strcmp(curr->key, key)) {
-	    ADIOI_Strncpy(value, curr->value, valuelen);
-	    value[valuelen] = '\0';
-	    *flag = 1;
-	    break;
-	}
-	curr = curr->next;
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/mpi2-other/info/info_getnks.c b/3rd-party/romio321/mpi2-other/info/info_getnks.c
deleted file mode 100644
index fd20826950f..00000000000
--- a/3rd-party/romio321/mpi2-other/info/info_getnks.c
+++ /dev/null
@@ -1,55 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_Info_get_nkeys = PMPI_Info_get_nkeys
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_Info_get_nkeys MPI_Info_get_nkeys
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_Info_get_nkeys as PMPI_Info_get_nkeys
-/* end of weak pragmas */
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_Info_get_nkeys - Returns the number of currently defined keys in info
-
-Input Parameters:
-. info - info object (handle)
-
-Output Parameters:
-. nkeys - number of defined keys (integer)
-
-.N fortran
-@*/
-int MPI_Info_get_nkeys(MPI_Info info, int *nkeys)
-{
-    MPI_Info curr;
-
-    if ((info <= (MPI_Info) 0) || (info->cookie != MPIR_INFO_COOKIE)) {
-        FPRINTF(stderr, "MPI_Info_get_nkeys: Invalid info object\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    curr = info->next;
-    *nkeys = 0;
-
-    while (curr) {
-	curr = curr->next;
-	(*nkeys)++;
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/mpi2-other/info/info_getnth.c b/3rd-party/romio321/mpi2-other/info/info_getnth.c
deleted file mode 100644
index fa325c41cc0..00000000000
--- a/3rd-party/romio321/mpi2-other/info/info_getnth.c
+++ /dev/null
@@ -1,74 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_Info_get_nthkey = PMPI_Info_get_nthkey
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_Info_get_nthkey MPI_Info_get_nthkey
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_Info_get_nthkey as PMPI_Info_get_nthkey
-/* end of weak pragmas */
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_Info_get_nthkey - Returns the nth defined key in info
-
-Input Parameters:
-. info - info object (handle)
-. n - key number (integer)
-
-Output Parameters:
-. keys - key (string)
-
-.N fortran
-@*/
-int MPI_Info_get_nthkey(MPI_Info info, int n, char *key)
-{
-    MPI_Info curr;
-    int nkeys, i;
-
-    if ((info <= (MPI_Info) 0) || (info->cookie != MPIR_INFO_COOKIE)) {
-        FPRINTF(stderr, "MPI_Info_get_nthkey: Invalid info object\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    if (key <= (char *) 0) {
-	FPRINTF(stderr, "MPI_Info_get: key is an invalid address\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    curr = info->next;
-    nkeys = 0;
-    while (curr) {
-	curr = curr->next;
-	nkeys++;
-    }
-
-    if ((n < 0) || (n >= nkeys)) {
-        FPRINTF(stderr, "MPI_Info_get_nthkey: n is an invalid number\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    curr = info->next;
-    i = 0;
-    while (i < n) {
-	curr = curr->next;
-	i++;
-    }
-    ADIOI_Strncpy(key, curr->key, MPI_MAX_INFO_KEY);
-
-    return MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/mpi2-other/info/info_getvln.c b/3rd-party/romio321/mpi2-other/info/info_getvln.c
deleted file mode 100644
index b484929b720..00000000000
--- a/3rd-party/romio321/mpi2-other/info/info_getvln.c
+++ /dev/null
@@ -1,76 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_Info_get_valuelen = PMPI_Info_get_valuelen
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_Info_get_valuelen MPI_Info_get_valuelen
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_Info_get_valuelen as PMPI_Info_get_valuelen
-/* end of weak pragmas */
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_Info_get_valuelen - Retrieves the length of the value associated with a key
-
-Input Parameters:
-. info - info object (handle)
-. key - key (string)
-
-Output Parameters:
-. valuelen - length of value argument (integer)
-. flag - true if key defined, false if not (boolean)
-
-.N fortran
-@*/
-int MPI_Info_get_valuelen(MPI_Info info, char *key, int *valuelen, int *flag)
-{
-    MPI_Info curr;
-
-    if ((info <= (MPI_Info) 0) || (info->cookie != MPIR_INFO_COOKIE)) {
-        FPRINTF(stderr, "MPI_Info_get_valuelen: Invalid info object\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    if (key <= (char *) 0) {
-	FPRINTF(stderr, "MPI_Info_get_valuelen: key is an invalid address\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    if (strlen(key) > MPI_MAX_INFO_KEY) {
-	FPRINTF(stderr, "MPI_Info_get_valuelen: key is longer than MPI_MAX_INFO_KEY\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    if (!strlen(key)) {
-	FPRINTF(stderr, "MPI_Info_get_valuelen: key is a null string\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    curr = info->next;
-    *flag = 0;
-
-    while (curr) {
-	if (!strcmp(curr->key, key)) {
-	    *valuelen = strlen(curr->value);
-	    *flag = 1;
-	    break;
-	}
-	curr = curr->next;
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/mpi2-other/info/info_set.c b/3rd-party/romio321/mpi2-other/info/info_set.c
deleted file mode 100644
index e60d89d79e5..00000000000
--- a/3rd-party/romio321/mpi2-other/info/info_set.c
+++ /dev/null
@@ -1,98 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "mpioimpl.h"
-
-#ifdef HAVE_WEAK_SYMBOLS
-
-#if defined(HAVE_PRAGMA_WEAK)
-#pragma weak MPI_Info_set = PMPI_Info_set
-#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
-#pragma _HP_SECONDARY_DEF PMPI_Info_set MPI_Info_set
-#elif defined(HAVE_PRAGMA_CRI_DUP)
-#pragma _CRI duplicate MPI_Info_set as PMPI_Info_set
-/* end of weak pragmas */
-#endif
-
-/* Include mapping from MPI->PMPI */
-#define MPIO_BUILD_PROFILING
-#include "mpioprof.h"
-#endif
-
-/*@
-    MPI_Info_set - Adds a (key,value) pair to info
-
-Input Parameters:
-. info - info object (handle)
-. key - key (string)
-. value - value (string)
-
-.N fortran
-@*/
-int MPI_Info_set(MPI_Info info, char *key, char *value)
-{
-    MPI_Info prev, curr;
-
-    if ((info <= (MPI_Info) 0) || (info->cookie != MPIR_INFO_COOKIE)) {
-        FPRINTF(stderr, "MPI_Info_set: Invalid info object\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    if (key <= (char *) 0) {
-	FPRINTF(stderr, "MPI_Info_set: key is an invalid address\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    if (value <= (char *) 0) {
-	FPRINTF(stderr, "MPI_Info_set: value is an invalid address\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    if (strlen(key) > MPI_MAX_INFO_KEY) {
-	FPRINTF(stderr, "MPI_Info_set: key is longer than MPI_MAX_INFO_KEY\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    if (strlen(value) > MPI_MAX_INFO_VAL) {
-	FPRINTF(stderr, "MPI_Info_set: value is longer than MPI_MAX_INFO_VAL\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    if (!strlen(key)) {
-	FPRINTF(stderr, "MPI_Info_set: key is a null string\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    if (!strlen(value)) {
-	FPRINTF(stderr, "MPI_Info_set: value is a null string\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    prev = info;
-    curr = info->next;
-
-    while (curr) {
-	if (!strcmp(curr->key, key)) {
-	    ADIOI_Free(curr->value); 
-	    curr->value = ADIOI_Strdup(value);
-	    break;
-	}
-	prev = curr;
-	curr = curr->next;
-    }
-
-    if (!curr) {
-	prev->next = (MPI_Info) ADIOI_Malloc(sizeof(struct MPIR_Info));
-	curr = prev->next;
-	curr->cookie = 0;  /* cookie not set on purpose */
-	curr->key = ADIOI_Strdup(key);
-	curr->value = ADIOI_Strdup(value);
-	curr->next = 0;
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/3rd-party/romio321/test-internal/Makefile.am b/3rd-party/romio321/test-internal/Makefile.am
deleted file mode 100644
index ec7c7eb0ef5..00000000000
--- a/3rd-party/romio321/test-internal/Makefile.am
+++ /dev/null
@@ -1,26 +0,0 @@
-# -*- Mode: Makefile; -*-
-#
-# (C) 2011 by Argonne National Laboratory.
-#     See COPYRIGHT in top-level directory.
-#
-
-# This directory is only listed in DIST_SUBDIRS, not SUBDIRS, so its contents
-# will not be built by default, but it will participate in "make distclean" and
-# friends.
-
-# override the normal compilers for the tests
-CC = $(TEST_CC)
-
-# because := is not universally avalible, we have to play games to use the
-# user-specified LDFLAGS and OUR_LIBS env. variables (if set)
-OUR_LIBS = $(TEST_LIBNAME) $(MPI_LIB) $(ROMIO_LIBLIST)
-
-LDADD = $(OUR_LIBS)
-
-AM_CPPFLAGS = $(ROMIO_INCLUDE)
-AM_CFLAGS = $(USER_CFLAGS)
-
-CTESTS = file_realms_test io_bounds_test heap_test
-
-noinst_PROGRAMS = $(CTESTS)
-
diff --git a/3rd-party/romio321/test-internal/file_realms_test.c b/3rd-party/romio321/test-internal/file_realms_test.c
deleted file mode 100644
index b848259e0ef..00000000000
--- a/3rd-party/romio321/test-internal/file_realms_test.c
+++ /dev/null
@@ -1,76 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 2008 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "../adio/include/adio.h"
-#include "../adio/include/adio_extern.h"
-#include "mpi.h"
-
-int main (int argc, char **argv)
-{
-    int i;
-    ADIO_File fd;
-    ADIO_Offset min_st_offset, max_end_offset;
-    int rank;
-    int nprocs_for_coll;
-    int lb;
-    MPI_Count size, extent;
-
-    MPI_Init (&argc, &argv);
-    MPI_Comm_rank (MPI_COMM_WORLD, &rank);
-
-    if (argc != 4) {
-	if (!rank)
-	    printf ("Usage: file_realms_test <number of aggregators> <lower bound> <upper bound>\n"
-		    "    simulates file_realm calculation\n");
-	MPI_Finalize();
-	return 1;
-    }
-
-    nprocs_for_coll = atoi (argv[1]);
-
-    min_st_offset = atoi (argv[2]);
-    max_end_offset = atoi (argv[3]);
-
-    if (max_end_offset < min_st_offset){
-	if (!rank)
-	    printf ("end offset %lld is less then start offset %lld\n",
-		    max_end_offset, min_st_offset);
-	MPI_Finalize();
-	return 1;
-    }
-
-    printf ("min_st_offset = %lld\nmax_end_offset = %lld\n",
-	    min_st_offset, max_end_offset);
-
-    fd = (ADIO_File) ADIOI_Malloc (sizeof (struct ADIOI_FileD));
-    fd->hints = (ADIOI_Hints *)
-	ADIOI_Malloc (sizeof(struct ADIOI_Hints_struct));
-    fd->hints->cb_nodes = nprocs_for_coll;
-    ADIOI_Calc_file_realms (fd, min_st_offset, max_end_offset);
-
-    for (i=0; i < nprocs_for_coll; i++) {
-	printf ("file_realm_st_offs[%d] = %lld\n", i, fd->file_realm_st_offs[i]);
-    }
-    for (i=0; i < nprocs_for_coll; i++) {
-	MPI_Type_size_x (fd->file_realm_types[i], &size);
-	printf ("file_realm [%d] size = %d\n", i, size);
-    }
-    for (i=0; i < nprocs_for_coll; i++) {
-	MPI_Type_get_extent (fd->file_realm_types[i], &lb, &extent);
-	printf ("file_realm [%d] extent = %d\n", i, extent);
-    }
-
-    for (i=0; i < nprocs_for_coll; i++)
-	MPI_Type_free (&fd->file_realm_types[i]);
-    ADIOI_Free (fd->file_realm_st_offs);
-    ADIOI_Free (fd->file_realm_types);
-    ADIOI_Free (fd->hints);
-    ADIOI_Free (fd);
-
-    MPI_Finalize();
-
-    return 0;
-}
diff --git a/3rd-party/romio321/test-internal/heap_test.c b/3rd-party/romio321/test-internal/heap_test.c
deleted file mode 100644
index 2f0041df368..00000000000
--- a/3rd-party/romio321/test-internal/heap_test.c
+++ /dev/null
@@ -1,459 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *
- *  (C) 2008 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include "../adio/include/heap-sort.h"
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <time.h>
-
-#define PREDEF_TESTS 2
-/* test types */
-#define ALL     0
-#define RANDOM -1
-#define CUSTOM -2
-
-/* ACTIONS */
-#define BUILD          0
-#define INSERT         1
-#define EXTRACT        2
-#define EXTRACT_INSERT 3
-
-typedef struct {
-    char name[64];
-    int heap_size;
-    int print;
-    int verify;
-    int action_arr_sz;
-    int *action_arr;
-    int *action_count_arr;
-    ADIO_Offset *offsets;
-    ADIO_Offset *correct_order;
-} test_params_t;
-
-void print_usage();
-void print_keys(ADIO_Offset* offsets, int size);
-void print_params(test_params_t *params);
-int run_test(test_params_t *test);
-void fill_random_test(test_params_t *params);
-void init_predefined_test(test_params_t *params, int index);
-void dumb_sort(test_params_t *params);
-
-int main(int argc, char **argv) {
-    int i, print = 1, verify = 1;
-    int adding_elements;
-    int curr_add_idx;
-    int test_type = RANDOM;
-    test_params_t predefined_tests[PREDEF_TESTS];
-    test_params_t test;
-    
-    /* parse args */
-    adding_elements = 0;
-    curr_add_idx = 0;
-    if (argc == 1) {
-	print_usage();
-	return 1;
-    }
-    i = 1;
-    while (i < argc) {
-	if (!strcmp("-A", argv[i])) {
-	    adding_elements = 0;
-	    test_type = ALL;
-	    i++;
-	}
-	else if (!strcmp("-T", argv[i])) {
-	    adding_elements = 0;
-	    test_type = atoi(argv[i+1]);
-	    i += 2;
-	}
-	else if (!strcmp("-r", argv[i])) {
-	    adding_elements = 0;
-	    test.heap_size = atoi(argv[i+1]);
-	    if (test.heap_size <= 0) {
-		printf("heap size should be a positive integer\n");
-		return 1;
-	    }
-	    test.offsets = (ADIO_Offset *) malloc(test.heap_size*sizeof(ADIO_Offset));
-	    test_type = RANDOM;
-	    i += 2;
-	}
-	else if (!strcmp("-e", argv[i])) {
-	    test.heap_size = argc - 2;
-	    if (test.heap_size <= 0) {
-		printf("need at least one key\n");
-		return 1;
-	    }
-	    test.offsets = (ADIO_Offset *) malloc(test.heap_size*sizeof(ADIO_Offset));
-	    adding_elements = 1;
-	    test_type = CUSTOM;
-	    i++;
-	}
-	else if (!strcmp("-v", argv[i])) {
-	    verify = 1;
-	    i++;
-	}
-	else if (!strcmp("-p", argv[i])) {
-	    print = 1;
-	    i++;
-	}
-	else if (!strcmp("-V", argv[i])) {
-	    verify = 0;
-	    i++;
-	}
-	else if (!strcmp("-P", argv[i])) {
-	    print = 0;
-	    i++;
-	}
-	else if (adding_elements) {
-	    test.offsets[curr_add_idx] = atoi(argv[i]);
-	    curr_add_idx++;
-	    i++;
-	}
-	else {
-	    printf("Illegal argument: %s", argv[i]);
-	    print_usage();
-	    return 1;
-	}
-    }
-
-    if (test_type == RANDOM) {
-	fill_random_test(&test);
-	strcpy(test.name, "RANDOMIZED TEST");
-    }
-    else if (test_type == CUSTOM)
-	strcpy(test.name, "CUSTOM TEST");
-    if ((test_type == CUSTOM) || (test_type == RANDOM)) {
-	test.print = print;
-	test.verify = verify;
-	test.action_arr_sz = 2;
-	test.action_arr = (int *) malloc(test.action_arr_sz*sizeof(int));
-	test.action_count_arr = (int *) malloc(test.action_arr_sz*sizeof(int));
-	/* build the entire heap */
-	/* test.action_arr[0]       = BUILD;
-	   test.action_count_arr[0] = 1; */
-	/* insert keys one at a time */
-	test.action_arr[0]       = INSERT;
-	test.action_count_arr[0] = test.heap_size;
-	/* extract all the keys */
-	test.action_arr[1]       = EXTRACT;
-	test.action_count_arr[1] = test.heap_size;
-
-	if (verify) {
-	    test.correct_order = (ADIO_Offset *)malloc(test.heap_size*sizeof(ADIO_Offset));
-	    dumb_sort(&test);
-	}
-	if (print)
-	    print_params(&test);
-	run_test(&test);
-    }
-    else {
-	if (test_type == ALL) {
-	    for (i=0; i<PREDEF_TESTS; i++) {
-		predefined_tests[i].print = print;
-		predefined_tests[i].verify = verify;
-		init_predefined_test(&predefined_tests[i], i);
-		if (print)
-		    print_params(&test);
-		run_test(&predefined_tests[i]);
-	    }
-	}
-	else {
-	    predefined_tests[test_type-1].print = print;
-	    predefined_tests[test_type-1].verify = verify;
-	    init_predefined_test(&predefined_tests[test_type-1], test_type-1);
-	    if (print)
-		print_params(&predefined_tests[test_type-1]);
-	    run_test(&predefined_tests[test_type-1]);
-	}
-    }
-
-    return 0;
-}
-
-void print_usage() {
-    printf(
-	"Usage: test <options>\n"
-	"   -r <size>  Create a random test and verify of size <size>\n"
-	"   -e <keys>  test with the space delimited list of keys\n"
-	"   -p         print parameters and keys (default)\n"
-	"   -P         do not print parameters and keys\n"
-	"   -v         verify keys (default)\n"
-	"   -V         do not verify keys\n"
-	);
-}
-
-void print_keys(ADIO_Offset *offsets, int size) {
-    int i;
-    for (i=0; i < size; i++)
-	printf("%lld ", offsets[i]);
-}
-
-void print_params(test_params_t *params) {
-    int i;
-    static char action_map[3][8] = {"BUILD", "INSERT", "EXTRACT"};
-
-    printf("----------------Test Parameters---------------\n");
-    printf("Actions:\n");
-    for (i=0; i<params->action_arr_sz; i++) {
-	printf("%sx%d\n", action_map[params->action_arr[i]],
-		params->action_count_arr[i]);
-    }
-
-    printf("Initial order :\n");
-    print_keys(params->offsets, params->heap_size);
-    printf("\n");
-
-    if (params->verify) {
-	printf("Expected order:\n");
-	print_keys(params->correct_order, params->heap_size);
-	printf("\n");
-    }
-    printf("----------------------------------------------\n");
-}
-
-void fill_random_test(test_params_t *params) {
-    int i;
-    int max_key;
-    time_t seed;
-    int order = 0;
-
-    time(&seed);
-    srand(seed);
-
-    order = 0;
-    max_key = 1;
-    while (order < 25) {
-	max_key *= 10;
-	if (!((int) (params->heap_size / max_key)))
-	    break;
-	order++;
-    }
-    for (i=0; i < params->heap_size; i++)
-	params->offsets[i] = (rand() % max_key);
-}
-
-void dumb_sort(test_params_t *params) {
-    ADIO_Offset *offsets, tmp_offset;
-    int i, j;
-
-    offsets = params->correct_order;
-    memcpy(offsets, params->offsets, params->heap_size*sizeof(ADIO_Offset));
-    for (i=0; i < params->heap_size; i++) {
-	for (j=i; j < params->heap_size; j++) {
-	    if (offsets[j] < offsets[i]) {
-		tmp_offset = offsets[i];
-		offsets[i] = offsets[j];
-		offsets[j] = tmp_offset;
-	    }
-	}
-    }
-}
-
-int run_test(test_params_t *test) {
-    heap_t myheap;
-    ADIO_Offset *extracted;
-    int stored_proc;
-    ADIO_Offset stored_reg_max_len;
-    int i, j, k, err_flag = 0;
-    int curr_insert_idx  = 0;
-    int curr_extract_idx = 0;
-    
-    create_heap(&myheap, test->heap_size);
-    myheap.size = 0;
-    
-    extracted = (ADIO_Offset *) malloc(test->heap_size * sizeof(ADIO_Offset));
-    for (i=0; i < test->action_arr_sz; i++) {
-	for (j=0; j<test->action_count_arr[i]; j++) {
-	    switch (test->action_arr[i])
-	    {
-		case BUILD:
-		    myheap.size = test->heap_size;
-		    for (k=0; k < test->heap_size; k++) {
-			myheap.nodes[k].offset   = test->offsets[k];
-			myheap.nodes[k].proc = k;
-		    }
-		    build_heap(&myheap);		    
-		    break;
-		case INSERT:
-		    ADIOI_Heap_insert(&myheap, test->offsets[curr_insert_idx],
-				curr_insert_idx, curr_insert_idx);
-		    curr_insert_idx++;
-		    break;
-		case EXTRACT:
-		    heap_extract_min(&myheap, &extracted[curr_extract_idx],
-				     &stored_proc, &stored_reg_max_len);
-		    if (test->verify && (extracted[curr_extract_idx] !=
-			test->correct_order[curr_extract_idx]))
-			err_flag++;
-		    curr_extract_idx++;
-		    break;
-		case EXTRACT_INSERT:
-		    heap_extract_min(&myheap, &extracted[curr_extract_idx],
-				     &stored_proc, &stored_reg_max_len);
-		    if (test->verify &&(extracted[curr_extract_idx] !=
-					test->correct_order[curr_extract_idx]))
-			err_flag++;
-		    curr_extract_idx++;
-
-		    ADIOI_Heap_insert(&myheap, test->offsets[curr_insert_idx],
-				curr_insert_idx, curr_insert_idx);
-		    curr_insert_idx++;
-		    break;
-		default:
-		    break;
-	    }
-	}
-    }
-
-    if (test->verify) {
-	if (err_flag) {
-	    printf("***%s FAILED***\n", test->name);
-	    if (test->print) {
-		printf("Min extraction:\n");
-		print_keys(extracted, test->heap_size);
-		printf("\n");
-	    }
-	}
-	else
-	    printf("***%s PASSED***\n", test->name);
-    }
-
-    free_heap(&myheap);
-    free(extracted);
-    /* clean up test params */
-    free(test->offsets);
-    if (test->verify)
-	free(test->correct_order);
-    free(test->action_arr);
-    free(test->action_count_arr);
-
-    return err_flag;
-}
-
-void init_predefined_test(test_params_t *params, int index) {
-
-    switch (index)
-    {
-	case 0:
-	    strcpy(params->name, "TEST 1");
-	    params->heap_size = 15;
-	    params->action_arr_sz = 3;
-	    
-	    /* allocate space */
-	    params->action_arr =
-		(int *) malloc (params->action_arr_sz*sizeof(int));
-	    params->action_count_arr =
-		(int *) malloc (params->action_arr_sz*sizeof(int));
-	    params->offsets = (ADIO_Offset *) malloc(params->heap_size*sizeof(ADIO_Offset));
-	    if (params->verify)
-		params->correct_order =
-		    (ADIO_Offset *) malloc(params->heap_size*sizeof(ADIO_Offset));
-
-	    /* Set procs */
-	    params->offsets[0]  = 65;
-	    params->offsets[1]  = 53;
-	    params->offsets[2]  = 51;
-	    params->offsets[3]  = 74;
-	    params->offsets[4]  = 1;
-	    params->offsets[5]  = 3;
-	    params->offsets[6]  = 86;
-	    params->offsets[7]  = 82;
-	    params->offsets[8]  = 42;
-	    params->offsets[9]  = 62;
-	    params->offsets[10] = 33;
-	    params->offsets[11] = 12;
-	    params->offsets[12] = 79;
-	    params->offsets[13] = 13;
-	    params->offsets[14] = 28;
-
-	    if (params->verify) {
-		params->correct_order[0]  = 1;
-		params->correct_order[1]  = 3;
-		params->correct_order[2]  = 12;
-		params->correct_order[3]  = 33;
-		params->correct_order[4]  = 13;
-		params->correct_order[5]  = 28;
-		params->correct_order[6]  = 42;
-		params->correct_order[7]  = 51;
-		params->correct_order[8]  = 53;
-		params->correct_order[9]  = 62;
-		params->correct_order[10] = 65;
-		params->correct_order[11] = 74;
-		params->correct_order[12] = 79;
-		params->correct_order[13] = 82;
-		params->correct_order[14] = 86;
-	    }
-
-	    params->action_arr[0]  = INSERT;
-	    params->action_arr[1]  = EXTRACT_INSERT;
-	    params->action_arr[11] = EXTRACT;
-
-	    params->action_count_arr[0]  = 10;
-	    params->action_count_arr[1]  = 5;
-	    params->action_count_arr[11] = 10;
-	    break;
-	case 1:
-	    strcpy(params->name, "TEST 1");
-	    params->heap_size = 15;
-	    params->action_arr_sz = 3;
-	    
-	    /* allocate space */
-	    params->action_arr =
-		(int *) malloc (params->action_arr_sz*sizeof(int));
-	    params->action_count_arr =
-		(int *) malloc (params->action_arr_sz*sizeof(int));
-	    params->offsets = (ADIO_Offset *) malloc(params->heap_size*sizeof(ADIO_Offset));
-	    if (params->verify)
-		params->correct_order =
-		    (ADIO_Offset *) malloc(params->heap_size*sizeof(ADIO_Offset));
-	    
-	    /* Set values */
-	    params->offsets[0]  = 65;
-	    params->offsets[1]  = 53;
-	    params->offsets[2]  = 51;
-	    params->offsets[3]  = 74;
-	    params->offsets[4]  = 1;
-	    params->offsets[5]  = 3;
-	    params->offsets[6]  = 86;
-	    params->offsets[7]  = 82;
-	    params->offsets[8]  = 42;
-	    params->offsets[9]  = 62;
-	    params->offsets[10] = 33;
-	    params->offsets[11] = 12;
-	    params->offsets[12] = 79;
-	    params->offsets[13] = 13;
-	    params->offsets[14] = 28;
-
-	    if (params->verify) {
-		params->correct_order[0]  = 1;
-		params->correct_order[1]  = 3;
-		params->correct_order[2]  = 12;
-		params->correct_order[3]  = 33;
-		params->correct_order[4]  = 13;
-		params->correct_order[5]  = 28;
-		params->correct_order[6]  = 42;
-		params->correct_order[7]  = 51;
-		params->correct_order[8]  = 53;
-		params->correct_order[9]  = 62;
-		params->correct_order[10] = 65;
-		params->correct_order[11] = 74;
-		params->correct_order[12] = 79;
-		params->correct_order[13] = 82;
-		params->correct_order[14] = 86;
-	    }
-
-	    params->action_arr[0]  = INSERT;
-	    params->action_arr[1]  = EXTRACT_INSERT;
-	    params->action_arr[11] = EXTRACT;
-
-	    params->action_count_arr[0]  = 10;
-	    params->action_count_arr[1]  = 5;
-	    params->action_count_arr[11] = 10;
-	    break;
-	default:
-	    break;
-    }
-}
diff --git a/3rd-party/romio321/test-internal/io_bounds_test.c b/3rd-party/romio321/test-internal/io_bounds_test.c
deleted file mode 100644
index f7514764e42..00000000000
--- a/3rd-party/romio321/test-internal/io_bounds_test.c
+++ /dev/null
@@ -1,302 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/* 
- *   Copyright (C) 2008 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "../adio/include/adio.h"
-#include "../adio/include/adioi.h"
-#include "../adio/include/adio_extern.h"
-#include "mpi.h"
-
-#define PREDEF_TESTS 5
-#define MAX_OFF_LENS 4
-
-typedef struct {
-    ADIO_Offset offset;
-    int count;
-    int type_blocklens[MAX_OFF_LENS];
-    int type_indices[MAX_OFF_LENS];
-    MPI_Datatype type_oldtypes[MAX_OFF_LENS];
-    int type_count;
-
-    ADIO_Offset correct_st_offset;
-    ADIO_Offset correct_end_offset;
-} test_param_t;
-
-int run_test (test_param_t *test);
-int setup_predefined (test_param_t *tests_arr, int count);
-int print_usage (void);
-int print_test_params (test_param_t *test);
-
-int main (int argc, char **argv) {
-    int rank;
-    int run_test_number = 0;
-    int failed;
-    int while_condition;
-    int i;
-
-    test_param_t predefined_tests[PREDEF_TESTS];
-
-    MPI_Init (&argc, &argv);
-    MPI_Comm_rank (MPI_COMM_WORLD, &rank);
-    
-    if (argc != 1) {
-	if (!rank) {
-	    printf ("Use only one process\n");
-	    print_usage ();
-	}
-	MPI_Finalize();
-	return 1;
-    }
-    i = 1;
-    while (i < argc) {
-        if (!strcmp (argv[i], "-A")) {
-            run_test_number = 0;
-            i++;
-        }
-        else if (!strcmp (argv[i], "-T")) {
-            run_test_number = atoi (argv[i+1]);
-	    if ((run_test_number > PREDEF_TESTS) || (run_test_number < 1)) {
-		if (!rank)
-		    printf ("Invalid test number, only %d tests\n",
-			    PREDEF_TESTS);
-		MPI_Finalize ();
-		return 1;
-	    }
-            i += 2;
-        }
-        else {
-	    if (!rank) {
-		printf ("Invalid Argument: %s\n", argv[i]);
-		print_usage ();
-	    }
-            i++;
-        }
-    }
-
-    setup_predefined (predefined_tests, PREDEF_TESTS);
-
-    if (!run_test_number) {
-	i = 0;
-	while_condition = PREDEF_TESTS;
-    }
-    else {
-	i = run_test_number - 1;
-	while_condition = run_test_number;
-    }
-    while (i < while_condition) {
-	printf ("***** Test %d *****\n", i+1);
-	failed = run_test (&predefined_tests[i]);
-	printf ("******************\n");
-	i++;
-    }
-
-    MPI_Finalize ();
-
-    return 0;
-}
-
-int run_test (test_param_t *test) {
-    ADIO_Offset st_offset, end_offset;
-    MPI_File fh;
-    int is_contig;
-    int ind_err = 0, exp_err = 0;
-
-    MPI_Datatype filetype;
-
-    MPI_Type_struct (test->type_count, test->type_blocklens,
-		     test->type_indices, test->type_oldtypes, &filetype);
-    MPI_Type_commit (&filetype);
-
-    MPI_File_open (MPI_COMM_WORLD, "test_file.txt" , MPI_MODE_RDWR,
-		   MPI_INFO_NULL, &fh);
-
-    MPI_File_set_view (fh, 0, MPI_BYTE, filetype, "native", MPI_INFO_NULL);
-
-    MPI_File_seek (fh, test->offset, MPI_SEEK_SET);
-    ADIOI_Calc_bounds ((ADIO_File) fh, test->count, MPI_BYTE, ADIO_INDIVIDUAL,
-		       test->offset, &st_offset, &end_offset);    
-
-    ind_err = 0;
-    if (st_offset != test->correct_st_offset) {
-	printf ("Individual st_offset = %lld end_offset = %lld\n",
-		st_offset, end_offset);
-	ind_err = 1;
-    }
-    if (end_offset != test->correct_end_offset) {
-	printf ("Individual st_offset = %lld end_offset = %lld\n",
-		st_offset, end_offset);
-	ind_err = 1;
-    }
-    MPI_File_close (&fh);
-    if (ind_err)
-	printf ("Individual Calc FAILED\n");
-
-    MPI_File_open (MPI_COMM_WORLD, "test_file.txt" , MPI_MODE_RDWR,
-		   MPI_INFO_NULL, &fh);
-
-    if (!is_contig)
-	MPI_File_set_view (fh, 0, MPI_BYTE, filetype, "native", MPI_INFO_NULL);
-
-    MPI_File_seek (fh, 0, MPI_SEEK_SET);
-    ADIOI_Calc_bounds ((ADIO_File) fh, test->count, MPI_BYTE,
-		       ADIO_EXPLICIT_OFFSET, test->offset, &st_offset,
-		       &end_offset);
-
-    exp_err = 0;
-    if (st_offset != test->correct_st_offset) {
-	printf ("Explicit   st_offset = %lld end_offset = %lld\n",
-		st_offset, end_offset);
-	exp_err = 1;
-    }
-    if (end_offset != test->correct_end_offset) {
-	printf ("Explicit   st_offset = %lld end_offset = %lld\n",
-		st_offset, end_offset);
-	exp_err = 1;
-    }
-    if (exp_err)
-	printf ("Explicit Calc FAILED\n");
-
-    MPI_File_close (&fh);
-
-    if (!is_contig)
-	MPI_Type_free (&filetype);
-
-    return (exp_err || ind_err);
-}
-
-int print_usage ()
-{
-    printf (
-	"Usage:\n"
-	"   io_bounds_test -A -T <test #>\n");
-}
-
-int print_test_params (test_param_t *test) 
-{
-    int i;
-    printf (
-	"I/O offset:     %lld\n"
-	"bytes:          %d\n"
-	"Filetype [n](disp, lens, type):\n",
-	test->offset, test->count);
-    
-    for (i=0; i<test->type_count; i++) {
-	printf (
-	    "    [%d](%lld, %d, ",
-	    i,
-	    test->type_blocklens[i],
-	    test->type_indices[i]);
-	if (test->type_oldtypes[i] == MPI_BYTE) {
-	    printf ( "%s)\n", "MPI_BYTE");
-	}
-	else if (test->type_oldtypes[i] == MPI_UB) {
-	    printf ( "%s)\n", "MPI_UB");
-	}
-	else if (test->type_oldtypes[i] == MPI_LB) {
-	    printf ( "%s)\n", "MPI_LB");
-	}
-    }
-    printf (
-	"Expected Start offset:  %lld\n"
-	"Expected End offset:    %lld\n",
-	test->correct_st_offset,
-	test->correct_end_offset);
-}
-
-int setup_predefined (test_param_t *tests_arr, int count)
-{
-    int i;
-    for (i=0; i < PREDEF_TESTS; i++) {
-	switch (i)
-	    {
-	    case 0:
-		tests_arr[i].offset            = 0;
-		tests_arr[i].count             = 0;
-		tests_arr[i].type_count        = 0;
-		tests_arr[i].type_indices[0]   = 0;
-		tests_arr[i].type_blocklens[0] = 0;
-		tests_arr[i].type_oldtypes[0]  = MPI_BYTE;
-		tests_arr[i].type_indices[1]   = 0;
-		tests_arr[i].type_blocklens[1] = 0;
-		tests_arr[i].type_oldtypes[1]  = MPI_BYTE;
-		tests_arr[i].type_indices[2]   = 0;
-		tests_arr[i].type_blocklens[2] = 0;
-		tests_arr[i].type_oldtypes[2]  = MPI_BYTE;
-		tests_arr[i].type_indices[3]   = 0;
-		tests_arr[i].type_blocklens[3] = 0;
-		tests_arr[i].type_oldtypes[3]  = MPI_BYTE;
-		break;
-	    case 1:
-		tests_arr[i].offset            = 0;
-		tests_arr[i].count             = 0;
-		tests_arr[i].type_count        = 0;
-		tests_arr[i].type_indices[0]   = 0;
-		tests_arr[i].type_blocklens[0] = 0;
-		tests_arr[i].type_oldtypes[0]  = MPI_BYTE;
-		tests_arr[i].type_indices[1]   = 0;
-		tests_arr[i].type_blocklens[1] = 0;
-		tests_arr[i].type_oldtypes[1]  = MPI_BYTE;
-		tests_arr[i].type_indices[2]   = 0;
-		tests_arr[i].type_blocklens[2] = 0;
-		tests_arr[i].type_oldtypes[2]  = MPI_BYTE;
-		tests_arr[i].type_indices[3]   = 0;
-		tests_arr[i].type_blocklens[3] = 0;
-		tests_arr[i].type_oldtypes[3]  = MPI_BYTE;
-		break;
-	    case 2:
-		tests_arr[i].offset            = 0;
-		tests_arr[i].count             = 0;
-		tests_arr[i].type_count        = 0;
-		tests_arr[i].type_indices[0]   = 0;
-		tests_arr[i].type_blocklens[0] = 0;
-		tests_arr[i].type_oldtypes[0]  = MPI_BYTE;
-		tests_arr[i].type_indices[1]   = 0;
-		tests_arr[i].type_blocklens[1] = 0;
-		tests_arr[i].type_oldtypes[1]  = MPI_BYTE;
-		tests_arr[i].type_indices[2]   = 0;
-		tests_arr[i].type_blocklens[2] = 0;
-		tests_arr[i].type_oldtypes[2]  = MPI_BYTE;
-		tests_arr[i].type_indices[3]   = 0;
-		tests_arr[i].type_blocklens[3] = 0;
-		tests_arr[i].type_oldtypes[3]  = MPI_BYTE;
-		break;
-	    case 3:
-		tests_arr[i].offset            = 0;
-		tests_arr[i].count             = 0;
-		tests_arr[i].type_count        = 0;
-		tests_arr[i].type_indices[0]   = 0;
-		tests_arr[i].type_blocklens[0] = 0;
-		tests_arr[i].type_oldtypes[0]  = MPI_BYTE;
-		tests_arr[i].type_indices[1]   = 0;
-		tests_arr[i].type_blocklens[1] = 0;
-		tests_arr[i].type_oldtypes[1]  = MPI_BYTE;
-		tests_arr[i].type_indices[2]   = 0;
-		tests_arr[i].type_blocklens[2] = 0;
-		tests_arr[i].type_oldtypes[2]  = MPI_BYTE;
-		tests_arr[i].type_indices[3]   = 0;
-		tests_arr[i].type_blocklens[3] = 0;
-		tests_arr[i].type_oldtypes[3]  = MPI_BYTE;
-		break;
-	    case 4:
-		tests_arr[i].offset            = 0;
-		tests_arr[i].count             = 0;
-		tests_arr[i].type_count        = 0;
-		tests_arr[i].type_indices[0]   = 0;
-		tests_arr[i].type_blocklens[0] = 0;
-		tests_arr[i].type_oldtypes[0]  = MPI_BYTE;
-		tests_arr[i].type_indices[1]   = 0;
-		tests_arr[i].type_blocklens[1] = 0;
-		tests_arr[i].type_oldtypes[1]  = MPI_BYTE;
-		tests_arr[i].type_indices[2]   = 0;
-		tests_arr[i].type_blocklens[2] = 0;
-		tests_arr[i].type_oldtypes[2]  = MPI_BYTE;
-		tests_arr[i].type_indices[3]   = 0;
-		tests_arr[i].type_blocklens[3] = 0;
-		tests_arr[i].type_oldtypes[3]  = MPI_BYTE;
-		break;
-	    }
-    }
-    return 0;
-}
diff --git a/3rd-party/romio321/test/.codingcheck b/3rd-party/romio321/test/.codingcheck
deleted file mode 100644
index 0d0a655fa81..00000000000
--- a/3rd-party/romio321/test/.codingcheck
+++ /dev/null
@@ -1,13 +0,0 @@
-#
-# We allow these routines for the romio test programs 
-%romioTestRoutines = ( 
-	'printf' => sys, 'fprintf' => sys , 'sprintf' => sys, 'vprintf' => sys,
-	'strcpy' => sys, 'strncpy' => sys, 'strcat' => sys, 
-	'malloc' => sys, 'free' => sys, 'calloc' => sys, 'strdup' => sys,
-	'assert' => sys, 'snprintf' => sys, 
-	);
-if (defined(&PushAllowFuncNames)) {
-    &PushAllowFuncNames( "romioTestRoutines", "tree", "add" );
-}
-
-1;
diff --git a/3rd-party/romio321/test/Makefile.am b/3rd-party/romio321/test/Makefile.am
deleted file mode 100644
index 5e2e0240856..00000000000
--- a/3rd-party/romio321/test/Makefile.am
+++ /dev/null
@@ -1,42 +0,0 @@
-# -*- Mode: Makefile; -*-
-#
-# (C) 2011 by Argonne National Laboratory.
-#     See COPYRIGHT in top-level directory.
-#
-
-# This directory is only listed in DIST_SUBDIRS, not SUBDIRS, so its contents
-# will not be built by default, but it will participate in "make distclean" and
-# friends.
-
-# override the normal compilers for the tests
-CC = $(TEST_CC)
-F77 = $(TEST_F77)
-
-# because := is not universally avalible, we have to play games to use the
-# user-specified LDFLAGS and OUR_LIBS env. variables (if set)
-OUR_LIBS = $(TEST_LIBNAME) $(MPI_LIB)
-
-LDADD = $(OUR_LIBS)
-
-AM_CPPFLAGS = $(ROMIO_INCLUDE)
-AM_CFLAGS = $(USER_CFLAGS)
-AM_FFLAGS = $(USER_FFLAGS)
-
-CTESTS = simple perf async coll_test coll_perf misc file_info excl large_array \
-     atomicity noncontig i_noncontig noncontig_coll split_coll shared_fp \
-     large_file psimple error status noncontig_coll2 aggregation1 aggregation2 \
-     async-multiple ordered_fp hindexed external32 types_with_zeros darray_read
-FTESTS = fcoll_test fperf fmisc pfcoll_test
-
-
-noinst_PROGRAMS = $(CTESTS)
-
-if BUILD_F77_TESTS
-
-noinst_PROGRAMS += $(FTESTS)
-fperf_SOURCES = fperf.f
-fcoll_test_SOURCES = fcoll_test.f
-fmisc_SOURCES = fmisc.f
-pfcoll_test_SOURCES = pfcoll_test.f
-
-endif BUILD_F77_TESTS
diff --git a/3rd-party/romio321/test/Mfile.in b/3rd-party/romio321/test/Mfile.in
deleted file mode 100644
index c2e1c298259..00000000000
--- a/3rd-party/romio321/test/Mfile.in
+++ /dev/null
@@ -1,84 +0,0 @@
-ALL: default
-
-# This is a special Makefile.in source for use by the test suite (see the
-# configure in examples/test)
-##### User configurable options #####
-
-MPIR_HOME    = @MPIR_HOME@
-CC           = @MPICC@
-CLINKER      = @MPICC@
-CCC          = @MPICPLUSPLUS@
-CCLINKER     = $(CCC)
-F77          = @MPIF77@
-F90BASE	     = @MPIF90BASE@
-F90          = @MPIF90@
-FLINKER      = @MPIF77@
-OPTFLAGS     = @OPTFLAGS@
-MPIFDEP      = @MPIFDEP@
-### End User configurable options ###
-
-SHELL = /bin/sh
-prefix     = @prefix@
-top_srcdir = @top_srcdir@
-srcdir     = @srcdir@
-@VPATH@
-
-PROFLIB =
-CFLAGS  = @CFLAGS@ @DEFS@ $(OPTFLAGS) 
-CCFLAGS = $(CFLAGS)
-FFLAGS = $(OPTFLAGS) @FFLAGS@
-# Use LIBS to add any special libraries for C programs
-LIBS = @LIB_PATH@ @LIB_LIST@
-# Use FLIBS to add any special libraries for Fortran programs
-FLIBS = @FLIB_PATH@ @LIB_LIST@ @F77EXTRALIBS@
-EXECS = $(CTESTS) $(FTESTS)
-OTHEREXECS = 
-CTESTS = simple perf async coll_test coll_perf misc file_info excl \
-       large_array \
-     atomicity noncontig i_noncontig noncontig_coll split_coll shared_fp \
-     large_file psimple error status noncontig_coll2
-FTESTS = fcoll_test fperf fmisc pfcoll_test 
-
-default: $(EXECS)
-
-#
-# Note that runtests builds the executables as required
-testing: 
-	  -./runtests $(TESTARGS)
-
-all: testing
-
-fortran_tests: $(FTESTS)
-#
-# The Fortran tests must be built with the Fortran linker
-fperf: fperf.f
-	$(F77) $(USER_FFLAGS) -o  $@ $< $(FLIBS)
-
-# Some of the Fortran tests must be derived, so their code will be in the
-# LOCAL directory
-fcoll_test: fcoll_test.f
-	$(F77) $(USER_FFLAGS) -o  fcoll_test fcoll_test.f $(FLIBS)
-
-fmisc: fmisc.f
-	$(F77) $(USER_FFLAGS) -o  fmisc fmisc.f $(FLIBS)
-
-pfcoll_test: pfcoll_test.f	
-	$(F77) $(USER_FFLAGS) -o  pfcoll_test pfcoll_test.f $(FLIBS)
-
-#
-# Make sure that we remove executables for specific architectures
-clean:
-	@-rm -f *.o *~ PI* $(EXECS) *.out core pt2pt.diff $(OTHEREXECS) \
-		*.trace rdb.* startup.* mpif.h ${srcdir}/*.o *.stdo
-	@-rm -f work.pc work.pcl
-	@-for file in $(EXECS) Makefile ; do \
-	   rm -f $$file.sun4 $$file.alpha $$file.IRIX $$file.freebsd ;\
-	done
-.c:
-	$(CC) $(CFLAGS) -o $* $< $(LIBS)
-.c.o:
-	$(CC) $(CFLAGS) -c $<
-.o:
-	${CLINKER} $(OPTFLAGS) -o $* $*.o $(LIBS)
-.f.o:
-	$(F77) $(FFLAGS) -c $<
diff --git a/3rd-party/romio321/test/README b/3rd-party/romio321/test/README
deleted file mode 100644
index dbab8e393d4..00000000000
--- a/3rd-party/romio321/test/README
+++ /dev/null
@@ -1,84 +0,0 @@
-This directory contains a few example programs. 
-
-Each program takes the filename as a command-line argument
-"-fname filename". 
-
-If you are using "mpirun" to run an MPI program, you can run the 
-program "simple" with two processes as follows:
-   mpirun -np 2 simple -fname test
-
-
-simple.c: Each process creates its own file, writes to it, reads it
-          back, and checks the data read.
-
-psimple.c: Same as simple.c but uses the PMPI versions of all MPI routines
-
-error.c: Tests if error messages are printed correctly
-
-status.c: Tests if the status object is filled correctly by I/O functions
-
-perf.c: A simple read and write performance test. Each process writes
-        4Mbytes to a file at a location determined by its rank and
-        reads it back. For a different access size, change the value
-        of SIZE in the code. The bandwidth is reported for two cases:
-        (1) without including MPI_File_sync and (2) including
-        MPI_File_sync. 
-     
-async.c: This program is the same as simple.c, except that it uses 
-        asynchronous I/O.
-
-coll_test.c: This program tests the use of collective I/O. It writes
-        a 3D block-distributed array to a file corresponding to the
-        global array in row-major (C) order, reads it back, and checks
-        that the data read is correct. The global array size has been
-        set to 32^3. If you are running it on NFS, which is very slow,
-        you may want to reduce that size to 16^3.
-
-coll_perf.c: Measures the I/O bandwidth for writing/reading a 3D
-      block-distributed array to a file corresponding to the global array
-      in row-major (C) order. The global array size has been
-      set to 128^3. If you are running it on NFS, which is very slow,
-      you may want to reduce that size to 16^3.
-
-misc.c: Tests various miscellaneous MPI-IO functions
-
-atomicity.c: Tests whether atomicity semantics are satisfied for 
-      overlapping accesses in atomic mode. The probability of detecting 
-      errors is higher if you run it on 8 or more processes.
-
-large_file.c: Tests access to large files. Writes a 4-Gbyte file and
-      reads it back. Run it only on one process and on a file system
-      on which ROMIO supports large files.
-
-large_array.c: Tests writing and reading a 4-Gbyte distributed array using
-      the distributed array datatype constructor. Works only on file
-      systems that support 64-bit file sizes and MPI implementations
-      that support 64-bit MPI_Aint. 
-
-file_info.c: Tests the setting and retrieval of hints via 
-      MPI_File_set_info and MPI_File_get_info
-
-excl.c: Tests MPI_File_open with MPI_MODE_EXCL
-
-noncontig.c: Tests noncontiguous accesses in memory and file using 
-             independent I/O. Run it on two processes only.
-
-noncontig_coll.c: Same as noncontig.c, but uses collective I/O
-
-noncontig_coll2.c: Same as noncontig_coll.c, but exercises the 
-             cb_config_list hint and aggregation handling more. 
-
-i_noncontig.c: Same as noncontig.c, but uses nonblocking I/O
-
-shared_fp.c: Tests the shared file pointer functions
-
-split_coll.c: Tests the split collective I/O functions
-
-fperf.f: Fortran version of perf.c
-
-fcoll_test.f: Fortran version of coll_test.c
-
-pfcoll_test.f: Same as fcoll_test.f but uses the PMPI versions of 
-               all MPI routines
-
-fmisc.f: Fortran version of misc.c
diff --git a/3rd-party/romio321/test/aggregation1.c b/3rd-party/romio321/test/aggregation1.c
deleted file mode 100644
index dee42d60a5c..00000000000
--- a/3rd-party/romio321/test/aggregation1.c
+++ /dev/null
@@ -1,266 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*  
- *  (C) 2007 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-
-/* Test case from John Bent (ROMIO req #835)
- * Aggregation code was not handling certain access patterns when collective
- * buffering forced */
-#define _XOPEN_SOURCE 500 /* strdup not in string.h otherwsie */
-#include <unistd.h>
-#include <stdlib.h>
-#include <mpi.h>
-#include <stdio.h>
-#include <string.h>
-
-#define NUM_OBJS 4
-#define OBJ_SIZE 1048576 
-
-extern char *optarg;
-extern int optind, opterr, optopt;
-
-
-char *prog = NULL;
-int  debug = 0;
-
-static void
-Usage( int line ) {
-    int rank;
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-    if ( rank == 0 ) {
-        fprintf( stderr, 
-                "Usage (line %d): %s [-d] [-h] -f filename\n"
-                "\t-d for debugging\n"
-                "\t-h to turn on the hints to force collective aggregation\n",
-                line, prog );
-    }
-    exit( 0 );
-}
-
-static void
-fatal_error( int mpi_ret, MPI_Status *mpi_stat, const char *msg ) {
-    fprintf( stderr, "Fatal error %s: %d\n", msg, mpi_ret );
-    MPI_Abort( MPI_COMM_WORLD, -1 );
-}
-
-static void
-print_hints( int rank, MPI_File *mfh ) {
-    MPI_Info info;
-    int nkeys;
-    int i, dummy_int;
-    char key[1024];
-    char value[1024];
-
-    MPI_Barrier( MPI_COMM_WORLD );
-    if ( rank == 0 ) {
-        MPI_File_get_info( *mfh, &info );
-        MPI_Info_get_nkeys( info, &nkeys );
-
-        printf( "HINTS:\n" );
-        for( i = 0; i < nkeys; i++ ) {
-            MPI_Info_get_nthkey( info, i, key );
-            printf( "%35s -> ", key );
-            MPI_Info_get( info, key, 1024, value, &dummy_int ); 
-            printf( "%s\n", value );
-        }
-	MPI_Info_free(&info);
-    }
-    MPI_Barrier( MPI_COMM_WORLD );
-}
-
-static void
-fill_buffer( char *buffer, int bufsize, int rank, MPI_Offset offset ) {
-    memset( (void*)buffer, 0, bufsize );
-    snprintf( buffer, bufsize, "Hello from %d at %lld\n", rank, offset );
-}
-
-static MPI_Offset
-get_offset( int rank, int num_objs, int obj_size, int which_obj ) {
-    MPI_Offset offset;
-    offset = (MPI_Offset)rank * num_objs * obj_size + which_obj * obj_size;
-    return offset;
-}
-
-static void
-write_file( char *target, int rank, MPI_Info *info ) {
-    MPI_File wfh;
-    MPI_Status mpi_stat;
-    int mpi_ret;
-    int i;
-    char *buffer;
-
-    buffer = malloc(OBJ_SIZE);
-
-    if ( debug ) printf( "%d writing file %s\n", rank, target );
-    
-    if( (mpi_ret = MPI_File_open(MPI_COMM_WORLD, target, 
-                    MPI_MODE_WRONLY | MPI_MODE_CREATE, *info, &wfh ) )
-                != MPI_SUCCESS ) 
-    {
-        fatal_error( mpi_ret, NULL, "open for write" );
-    }
-
-    for( i = 0; i < NUM_OBJS; i++ ) {
-        MPI_Offset offset = get_offset( rank, NUM_OBJS, OBJ_SIZE, i );
-        fill_buffer( buffer, OBJ_SIZE, rank, offset );
-        if ( debug ) printf( "%s", buffer );
-        if ( (mpi_ret = MPI_File_write_at_all( wfh, offset, buffer, OBJ_SIZE,
-                        MPI_CHAR, &mpi_stat ) ) != MPI_SUCCESS ) 
-        {
-            fatal_error( mpi_ret, &mpi_stat, "write" );
-        }
-    }
-
-    if ( debug ) print_hints( rank, &wfh );
-
-    if( (mpi_ret = MPI_File_close( &wfh ) ) != MPI_SUCCESS ) {
-        fatal_error( mpi_ret, NULL, "close for write" );
-    }
-    if ( debug ) printf( "%d wrote file %s\n", rank, target );
-    free(buffer);
-}
-
-static int
-reduce_corruptions( int corrupt_blocks ) {
-    int mpi_ret;
-    int sum;
-    if ( ( mpi_ret = MPI_Reduce( &corrupt_blocks, &sum, 1, 
-                    MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD ) ) != MPI_SUCCESS )
-    {
-        fatal_error( mpi_ret, NULL, "MPI_Reduce" );
-    }
-    return sum;
-}
-
-static void
-read_file( char *target, int rank, MPI_Info *info, int *corrupt_blocks ) {
-    MPI_File rfh;
-    MPI_Status mpi_stat;
-    int mpi_ret;
-    int i;
-    char *buffer;
-    char *verify_buf = NULL;
-    buffer = malloc(OBJ_SIZE);
-    verify_buf = (char *)malloc(OBJ_SIZE);
-
-    if ( debug ) printf( "%d reading file %s\n", rank, target );
-    
-    if( (mpi_ret = MPI_File_open(MPI_COMM_WORLD, target, 
-                    MPI_MODE_RDONLY, *info, &rfh ) ) != MPI_SUCCESS ) 
-    {
-        fatal_error( mpi_ret, NULL, "open for read" );
-    }
-
-    for( i = 0; i < NUM_OBJS; i++ ) {
-        MPI_Offset offset = get_offset( rank, NUM_OBJS, OBJ_SIZE, i );
-        fill_buffer( verify_buf, OBJ_SIZE, rank, offset );
-        if ( debug ) printf( "Expecting %s", buffer );
-        if ( (mpi_ret = MPI_File_read_at_all( rfh, offset, buffer, OBJ_SIZE,
-                        MPI_CHAR, &mpi_stat ) ) != MPI_SUCCESS ) 
-        {
-            fatal_error( mpi_ret, &mpi_stat, "read" );
-        }
-        if ( memcmp( verify_buf, buffer, OBJ_SIZE ) != 0 ) {
-            (*corrupt_blocks)++;
-            printf( "Corruption at %lld\n", offset );
-            if ( debug ) {
-                printf( "\tExpecting %s\n"
-                         "\tRecieved  %s\n",
-                         verify_buf, buffer );
-            }
-        }
-    }
-
-    if( (mpi_ret = MPI_File_close( &rfh ) ) != MPI_SUCCESS ) {
-        fatal_error( mpi_ret, NULL, "close for read" );
-    }
-    free (buffer);
-    free(verify_buf);
-
-}
-
-static void
-set_hints( MPI_Info *info ) {
-    MPI_Info_set( *info, "romio_cb_write", "enable" ); 
-    MPI_Info_set( *info, "romio_no_indep_rw", "1" ); 
-    MPI_Info_set( *info, "cb_nodes", "1" ); 
-    MPI_Info_set( *info, "cb_buffer_size", "4194304" );
-}
-
-/*
-void
-set_hints( MPI_Info *info, char *hints ) {
-    char *delimiter = " ";
-    char *hints_cp  = strdup( hints );
-    char *key = strtok( hints_cp, delimiter );
-    char *val;
-    while( key ) {
-        val = strtok( NULL, delimiter );
-        if ( debug ) printf( "HINT: %s = %s\n", key, val );
-        if ( ! val ) {
-            Usage( __LINE__ ); 
-        }
-        MPI_Info_set( *info, key, val );
-        key = strtok( NULL, delimiter );
-    }
-    free( hints_cp );
-}
-*/
-
-int 
-main( int argc, char *argv[] ) {
-	int nproc = 1, rank = 0;
-    char *target = NULL;
-    int c;
-    MPI_Info info;
-    int mpi_ret;
-    int corrupt_blocks = 0;
-
-    MPI_Init( &argc, &argv );
-    MPI_Comm_size(MPI_COMM_WORLD, &nproc);
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-
-    if( (mpi_ret = MPI_Info_create(&info)) != MPI_SUCCESS) {
-        if(rank == 0) fatal_error( mpi_ret, NULL, "MPI_info_create.\n");
-    }
-
-    prog = strdup( argv[0] );
-
-    while( ( c = getopt( argc, argv, "df:h" ) ) != EOF ) {
-        switch( c ) {
-            case 'd':
-                debug = 1;
-                break;
-            case 'f':
-                target = strdup( optarg );
-                break;
-            case 'h':
-                set_hints( &info );
-                break;
-            default:
-                Usage( __LINE__ );
-        }
-    }
-    if ( ! target ) {
-        Usage( __LINE__ );
-    }
-
-    write_file( target, rank, &info );
-    read_file(  target, rank, &info, &corrupt_blocks );
-
-    corrupt_blocks = reduce_corruptions( corrupt_blocks );
-    if ( rank == 0 ) {
-	if (corrupt_blocks == 0) {
-	    fprintf(stdout, " No Errors\n");
-	} else {
-            fprintf(stdout, "%d/%d blocks corrupt\n",
-                corrupt_blocks, nproc * NUM_OBJS );
-	}
-    }
-    MPI_Info_free(&info);
-
-    MPI_Finalize();
-    free(prog);
-    exit( 0 );
-}
diff --git a/3rd-party/romio321/test/aggregation2.c b/3rd-party/romio321/test/aggregation2.c
deleted file mode 100644
index a35ebe0caf5..00000000000
--- a/3rd-party/romio321/test/aggregation2.c
+++ /dev/null
@@ -1,89 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*  
- *  (C) 2007 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-
-/* Look for regressions in aggregator code.  A more simple access pattern than
- * aggregation1 */
-
-#include <mpi.h>
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <time.h>
-
-#include <sys/types.h>
-#include <unistd.h>
-
-#include <string.h>
-
-#define BUFSIZE 512
-
-static void handle_error(int errcode, const char *str)
-{
-        char msg[MPI_MAX_ERROR_STRING];
-        int resultlen;
-        MPI_Error_string(errcode, msg, &resultlen);
-        fprintf(stderr, "%s: %s\n", str, msg);
-        MPI_Abort(MPI_COMM_WORLD, 1);
-}
-
-int main(int argc, char ** argv) 
-{
-    MPI_Info info = MPI_INFO_NULL;
-    MPI_File fh;
-    MPI_Offset off=0;
-    MPI_Status status;
-    int errcode;
-    int i, rank, errs=0, toterrs, buffer[BUFSIZE], buf2[BUFSIZE];
-
-    MPI_Init(&argc, &argv);
-
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-
-    MPI_Info_create(&info);
-    MPI_Info_set(info, "romio_cb_write", "enable");
-    MPI_Info_set(info, "cb_nodes", "1");
-
-    for (i=0; i<BUFSIZE; i++) {
-        buffer[i] = 10000+rank;
-    }
-    off = rank*sizeof(buffer);
-
-    errcode = MPI_File_open(MPI_COMM_WORLD, argv[1], 
-		MPI_MODE_WRONLY|MPI_MODE_CREATE, info, &fh);
-    if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_open");
-    errcode = MPI_File_write_at_all(fh, off, buffer, BUFSIZE, 
-		MPI_INT,  &status);
-    if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_write_at_all");
-    errcode = MPI_File_close(&fh);
-    if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_close");
-
-    errcode = MPI_File_open(MPI_COMM_WORLD, argv[1], 
-		MPI_MODE_RDONLY, info, &fh);
-    if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_open");
-    errcode = MPI_File_read_at_all(fh, off, buf2, BUFSIZE, 
-		MPI_INT,  &status);
-    if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_read_at_all");
-    errcode = MPI_File_close(&fh);
-    if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_close");
-
-    for (i=0; i<BUFSIZE; i++) {
-        if (buf2[i] != 10000+rank)
-	    errs++;
-    }
-    MPI_Allreduce( &errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
-    if (rank == 0) {
-	if( toterrs > 0) {
-	    fprintf( stderr, "Found %d errors\n", toterrs );
-	}
-	else {
-	    fprintf( stdout, " No Errors\n" );
-	}
-    }
-    MPI_Info_free(&info);
-    MPI_Finalize();
-
-    return 0;
-}
diff --git a/3rd-party/romio321/test/async-multiple.c b/3rd-party/romio321/test/async-multiple.c
deleted file mode 100644
index ec9726ce55d..00000000000
--- a/3rd-party/romio321/test/async-multiple.c
+++ /dev/null
@@ -1,139 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- 
- * vim: ts=8 sts=4 sw=4 noexpandtab
- *  
- *  (C) 2001 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include "mpi.h"
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-
-#define SIZE (65536)
-#define NR_NBOPS (32)
-
-/* Uses asynchronous I/O. Each process writes to separate files and
-   reads them back. The file name is taken as a command-line argument,
-   and the process rank is appended to it.*/ 
-
-void handle_error(int errcode, const char *str);
-
-void handle_error(int errcode, const char *str) 
-{
-	char msg[MPI_MAX_ERROR_STRING];
-	int resultlen;
-	MPI_Error_string(errcode, msg, &resultlen);
-	fprintf(stderr, "%s: %s\n", str, msg);
-	MPI_Abort(MPI_COMM_WORLD, 1);
-}
-int main(int argc, char **argv)
-{
-    int *buf, i, rank, nints, len;
-    char *filename, *tmp;
-    int errs=0, toterrs;
-    MPI_File fh;
-    MPI_Status status[NR_NBOPS];
-    MPI_Request request[NR_NBOPS];
-    int errcode = 0;
-
-    MPI_Init(&argc,&argv);
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-
-/* process 0 takes the file name as a command-line argument and 
-   broadcasts it to other processes */
-    if (!rank) {
-	i = 1;
-	while ((i < argc) && strcmp("-fname", *argv)) {
-	    i++;
-	    argv++;
-	}
-	if (i >= argc) {
-	    fprintf(stderr, "\n*#  Usage: async -fname filename\n\n");
-	    MPI_Abort(MPI_COMM_WORLD, 1);
-	}
-	argv++;
-	len = strlen(*argv);
-	filename = (char *) malloc(len+10);
-	strcpy(filename, *argv);
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	MPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-    else {
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	filename = (char *) malloc(len+10);
-	MPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-
-
-    buf = (int *) malloc(SIZE);
-    nints = SIZE/sizeof(int);
-    for (i=0; i<nints; i++) buf[i] = rank*100000 + i;
-
-    /* each process opens a separate file called filename.'myrank' */
-    tmp = (char *) malloc(len+10);
-    strcpy(tmp, filename);
-    sprintf(filename, "%s.%d", tmp, rank);
-
-    errcode = MPI_File_open(MPI_COMM_SELF, filename, 
-		    MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_open");
-    }
-    MPI_File_set_view(fh, 0, MPI_INT, MPI_INT, "native", MPI_INFO_NULL);
-    for (i=0; i<NR_NBOPS; i++) { 
-	errcode = MPI_File_iwrite_at(fh, nints/NR_NBOPS*i, 
-		buf+(nints/NR_NBOPS*i), nints/NR_NBOPS, MPI_INT, &(request[i]));
-	if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_iwrite");
-	}
-    }
-    MPI_Waitall(NR_NBOPS, request, status);
-
-    MPI_File_close(&fh);
-
-    /* reopen the file and read the data back */
-
-    for (i=0; i<nints; i++) buf[i] = 0;
-    errcode = MPI_File_open(MPI_COMM_SELF, filename, 
-		    MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_open");
-    }
-
-    MPI_File_set_view(fh, 0, MPI_INT, MPI_INT, "native", MPI_INFO_NULL);
-    for (i=0; i<NR_NBOPS; i++) {
-	errcode = MPI_File_iread_at(fh, nints/NR_NBOPS*i, 
-		buf+(nints/NR_NBOPS*i), nints/NR_NBOPS, MPI_INT, &(request[i]));
-	if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_open");
-	}
-    }
-    MPI_Waitall(NR_NBOPS, request, status);
-
-    MPI_File_close(&fh);
-
-    /* check if the data read is correct */
-    for (i=0; i<nints; i++) {
-	if (buf[i] != (rank*100000 + i)) {
-	    errs++;
-	    fprintf(stderr, "Process %d: error, read %d, should be %d\n", rank, buf[i], rank*100000+i);
-	}
-    }
-
-    MPI_Allreduce( &errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
-    if (rank == 0) {
-	if( toterrs > 0) {
-	    fprintf( stderr, "Found %d errors\n", toterrs );
-	}
-	else {
-	    fprintf( stdout, " No Errors\n" );
-	}
-    }
-
-    free(buf);
-    free(filename);
-    free(tmp);
-
-    MPI_Finalize();
-    return 0; 
-}
diff --git a/3rd-party/romio321/test/async.c b/3rd-party/romio321/test/async.c
deleted file mode 100644
index 8882716dfed..00000000000
--- a/3rd-party/romio321/test/async.c
+++ /dev/null
@@ -1,138 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*  
- *  (C) 2001 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include "mpi.h"
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-
-#define SIZE (65536)
-
-/* Uses asynchronous I/O. Each process writes to separate files and
-   reads them back. The file name is taken as a command-line argument,
-   and the process rank is appended to it.*/ 
-
-void handle_error(int errcode, const char *str);
-
-void handle_error(int errcode, const char *str) 
-{
-	char msg[MPI_MAX_ERROR_STRING];
-	int resultlen;
-	MPI_Error_string(errcode, msg, &resultlen);
-	fprintf(stderr, "%s: %s\n", str, msg);
-	MPI_Abort(MPI_COMM_WORLD, 1);
-}
-int main(int argc, char **argv)
-{
-    int *buf, i, rank, nints, len;
-    char *filename, *tmp;
-    int errs=0, toterrs;
-    MPI_File fh;
-    MPI_Status status;
-    MPIO_Request request;
-    int errcode = 0;
-
-    MPI_Init(&argc,&argv);
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-
-/* process 0 takes the file name as a command-line argument and 
-   broadcasts it to other processes */
-    if (!rank) {
-	i = 1;
-	while ((i < argc) && strcmp("-fname", *argv)) {
-	    i++;
-	    argv++;
-	}
-	if (i >= argc) {
-	    fprintf(stderr, "\n*#  Usage: async -fname filename\n\n");
-	    MPI_Abort(MPI_COMM_WORLD, 1);
-	}
-	argv++;
-	len = strlen(*argv);
-	filename = (char *) malloc(len+10);
-	strcpy(filename, *argv);
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	MPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-    else {
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	filename = (char *) malloc(len+10);
-	MPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-
-
-    buf = (int *) malloc(SIZE);
-    nints = SIZE/sizeof(int);
-    for (i=0; i<nints; i++) buf[i] = rank*100000 + i;
-
-    /* each process opens a separate file called filename.'myrank' */
-    tmp = (char *) malloc(len+10);
-    strcpy(tmp, filename);
-    sprintf(filename, "%s.%d", tmp, rank);
-
-    errcode = MPI_File_open(MPI_COMM_SELF, filename, 
-		    MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_open");
-    }
-    MPI_File_set_view(fh, 0, MPI_INT, MPI_INT, "native", MPI_INFO_NULL);
-    errcode = MPI_File_iwrite(fh, buf, nints, MPI_INT, &request);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_iwrite");
-    }
-#ifdef MPIO_USES_MPI_REQUEST
-    MPI_Wait( &request, &status );
-#else    
-    MPIO_Wait(&request, &status);
-#endif
-    MPI_File_close(&fh);
-
-    /* reopen the file and read the data back */
-
-    for (i=0; i<nints; i++) buf[i] = 0;
-    errcode = MPI_File_open(MPI_COMM_SELF, filename, 
-		    MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_open");
-    }
-
-    MPI_File_set_view(fh, 0, MPI_INT, MPI_INT, "native", MPI_INFO_NULL);
-    errcode = MPI_File_iread(fh, buf, nints, MPI_INT, &request);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_open");
-    }
-#ifdef MPIO_USES_MPI_REQUEST
-    MPI_Wait( &request, &status );
-#else
-    MPIO_Wait(&request, &status);
-#endif
-
-    MPI_File_close(&fh);
-
-    /* check if the data read is correct */
-    for (i=0; i<nints; i++) {
-	if (buf[i] != (rank*100000 + i)) {
-	    errs++;
-	    fprintf(stderr, "Process %d: error, read %d, should be %d\n", rank, buf[i], rank*100000+i);
-	}
-    }
-
-    MPI_Allreduce( &errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
-    if (rank == 0) {
-	if( toterrs > 0) {
-	    fprintf( stderr, "Found %d errors\n", toterrs );
-	}
-	else {
-	    fprintf( stdout, " No Errors\n" );
-	}
-    }
-
-    free(buf);
-    free(filename);
-    free(tmp);
-
-    MPI_Finalize();
-    return 0; 
-}
diff --git a/3rd-party/romio321/test/atomicity.c b/3rd-party/romio321/test/atomicity.c
deleted file mode 100644
index 7797075ed95..00000000000
--- a/3rd-party/romio321/test/atomicity.c
+++ /dev/null
@@ -1,210 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*  
- *  (C) 2001 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include "mpi.h"
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-
-/* tests whether atomicity semantics are satisfied for overlapping accesses
-   in atomic mode. The probability of detecting errors is higher if you run 
-   it on 8 or more processes. */
-
-/* The file name is taken as a command-line argument. */
-
-#define BUFSIZE 10000    /* no. of integers */
-#define VERBOSE 0
-int main(int argc, char **argv)
-{
-    int *writebuf, *readbuf, i, mynod, nprocs, len, err;
-    char *filename;
-    int errs=0, toterrs;
-    MPI_Datatype newtype;
-    MPI_File fh;
-    MPI_Status status;
-    MPI_Info info;
-
-    MPI_Init(&argc,&argv);
-    MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
-    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
-
-/* process 0 takes the file name as a command-line argument and 
-   broadcasts it to other processes */
-    if (!mynod) {
-	i = 1;
-	while ((i < argc) && strcmp("-fname", *argv)) {
-	    i++;
-	    argv++;
-	}
-	if (i >= argc) {
-	    fprintf(stderr, "\n*#  Usage: coll_test -fname filename\n\n");
-	    MPI_Abort(MPI_COMM_WORLD, 1);
-	}
-	argv++;
-	len = strlen(*argv);
-	filename = (char *) malloc(len+1);
-	strcpy(filename, *argv);
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-    else {
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	filename = (char *) malloc(len+1);
-	MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-
-    writebuf = (int *) malloc(BUFSIZE*sizeof(int));
-    readbuf = (int *) malloc(BUFSIZE*sizeof(int));
-
-/* test atomicity of contiguous accesses */
-
-/* initialize file to all zeros */
-    if (!mynod) {
-	MPI_File_delete(filename, MPI_INFO_NULL);
-	MPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_CREATE | 
-             MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
-	for (i=0; i<BUFSIZE; i++) writebuf[i] = 0;
-	MPI_File_write(fh, writebuf, BUFSIZE, MPI_INT, &status);
-	MPI_File_close(&fh);
-#if VERBOSE
-	fprintf(stderr, "\ntesting contiguous accesses\n");
-#endif
-    }
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    for (i=0; i<BUFSIZE; i++) writebuf[i] = 10;
-    for (i=0; i<BUFSIZE; i++) readbuf[i] = 20;
-
-    MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | 
-             MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
-
-/* set atomicity to true */
-    err = MPI_File_set_atomicity(fh, 1);
-    if (err != MPI_SUCCESS) {
-	fprintf(stderr, "Atomic mode not supported on this file system.\n");fflush(stderr);
-	MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    MPI_Barrier(MPI_COMM_WORLD);
-    
-/* process 0 writes and others concurrently read. In atomic mode, 
-   the data read must be either all old values or all new values; nothing
-   in between. */ 
-
-    if (!mynod) MPI_File_write(fh, writebuf, BUFSIZE, MPI_INT, &status);
-    else {
-	err = MPI_File_read(fh, readbuf, BUFSIZE, MPI_INT, &status);
-	if (err == MPI_SUCCESS) {
-	    if (readbuf[0] == 0) { /* the rest must also be 0 */
-		for (i=1; i<BUFSIZE; i++) 
-		    if (readbuf[i] != 0) {
-			errs++;
-			fprintf(stderr, "Process %d: readbuf[%d] is %d, should be 0\n", mynod, i, readbuf[i]);
-			MPI_Abort(MPI_COMM_WORLD, 1);
-		    }
-	    }
-	    else if (readbuf[0] == 10) { /* the rest must also be 10 */
-		for (i=1; i<BUFSIZE; i++) 
-		    if (readbuf[i] != 10) {
-			errs++;
-			fprintf(stderr, "Process %d: readbuf[%d] is %d, should be 10\n", mynod, i, readbuf[i]);
-			MPI_Abort(MPI_COMM_WORLD, 1);
-		    }
-	    }
-	    else {
-		errs++;
-		fprintf(stderr, "Process %d: readbuf[0] is %d, should be either 0 or 10\n", mynod, readbuf[0]); 	
-	    }    
-	}
-    }
-
-    MPI_File_close(&fh);
-	
-    MPI_Barrier(MPI_COMM_WORLD);
-
-
-/* repeat the same test with a noncontiguous filetype */
-
-    MPI_Type_vector(BUFSIZE, 1, 2, MPI_INT, &newtype);
-    MPI_Type_commit(&newtype);
-
-    MPI_Info_create(&info);
-    /* I am setting these info values for testing purposes only. It is
-       better to use the default values in practice. */
-    MPI_Info_set(info, "ind_rd_buffer_size", "1209");
-    MPI_Info_set(info, "ind_wr_buffer_size", "1107");
-    
-    if (!mynod) {
-	MPI_File_delete(filename, MPI_INFO_NULL);
-	MPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_CREATE | 
-             MPI_MODE_RDWR, info, &fh);
-	for (i=0; i<BUFSIZE; i++) writebuf[i] = 0;
-	MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info);
-	MPI_File_write(fh, writebuf, BUFSIZE, MPI_INT, &status);
-	MPI_File_close(&fh);
-#if VERBOSE
-	fprintf(stderr, "\ntesting noncontiguous accesses\n");
-#endif
-    }
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    for (i=0; i<BUFSIZE; i++) writebuf[i] = 10;
-    for (i=0; i<BUFSIZE; i++) readbuf[i] = 20;
-
-    MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | 
-             MPI_MODE_RDWR, info, &fh);
-    MPI_File_set_atomicity(fh, 1);
-    MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info);
-    MPI_Barrier(MPI_COMM_WORLD);
-    
-    if (!mynod) MPI_File_write(fh, writebuf, BUFSIZE, MPI_INT, &status);
-    else {
-	err = MPI_File_read(fh, readbuf, BUFSIZE, MPI_INT, &status);
-	if (err == MPI_SUCCESS) {
-	    if (readbuf[0] == 0) {
-		for (i=1; i<BUFSIZE; i++) 
-		    if (readbuf[i] != 0) {
-			errs++;
-			fprintf(stderr, "Process %d: readbuf[%d] is %d, should be 0\n", mynod, i, readbuf[i]);
-			MPI_Abort(MPI_COMM_WORLD, 1);
-		    }
-	    }
-	    else if (readbuf[0] == 10) {
-		for (i=1; i<BUFSIZE; i++) 
-		    if (readbuf[i] != 10) {
-			errs++;
-			fprintf(stderr, "Process %d: readbuf[%d] is %d, should be 10\n", mynod, i, readbuf[i]);
-			MPI_Abort(MPI_COMM_WORLD, 1);
-		    }
-	    }
-	    else {
-		errs++;
-		fprintf(stderr, "Process %d: readbuf[0] is %d, should be either 0 or 10\n", mynod, readbuf[0]); 	    
-	    }
-	}
-    }
-
-    MPI_File_close(&fh);
-	
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    MPI_Allreduce( &errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
-    if (mynod == 0) {
-	if( toterrs > 0) {
-	    fprintf( stderr, "Found %d errors\n", toterrs );
-	}
-	else {
-	    fprintf( stdout, " No Errors\n" );
-	}
-    }
-    MPI_Type_free(&newtype);
-    MPI_Info_free(&info);
-    free(writebuf);
-    free(readbuf);
-    free(filename);
-
-    MPI_Finalize();
-    return 0;
-}
diff --git a/3rd-party/romio321/test/big_extents.c b/3rd-party/romio321/test/big_extents.c
deleted file mode 100644
index a1a9e5ff68b..00000000000
--- a/3rd-party/romio321/test/big_extents.c
+++ /dev/null
@@ -1,212 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*  
- *  (C) 2007 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-
-/* a test to exercise very large extents: on most platforms with 32 bit
- * integers, we'd expect these tests to give unexpected values.  On platforms
- * with 64 bit integers, these tests will be fine.  On BlueGene we're not sure
- * yet :> 
- */ 
-
-
-#include <mpi.h>
-#include <stdint.h>
-#include <math.h>
-#include <stdio.h>
-
-#define CHECK(fn) {int errcode; errcode = (fn); if (errcode != MPI_SUCCESS) handle_error(errcode, NULL); } 
-
-
-static void handle_error(int errcode, char *str) 
-{
-	char msg[MPI_MAX_ERROR_STRING];
-	int resultlen;
-	MPI_Error_string(errcode, msg, &resultlen);
-	fprintf(stderr, "%s: %s\n", str, msg);
-	MPI_Abort(MPI_COMM_WORLD, 1);
-}
-
-static void typestats(MPI_Datatype type) 
-{
-    MPI_Aint lb, extent;
-    MPI_Count size;
-
-    MPI_Type_get_extent(type, &lb, &extent);
-    MPI_Type_size_x(type, &size);
-
-    printf("dtype %d: lb = %ld extent = %ld size = %ld...",
-	    type, (long)lb, (long)extent, size);
-
-}
-
-static int verify_type(char *filename, MPI_Datatype type, 
-	int64_t expected_extent, int do_coll)
-{
-    int rank, canary;
-    MPI_Count tsize;
-    int compare=-1;
-    int errs=0, toterrs=0;
-    MPI_Status status;
-    MPI_File fh;
-
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-
-    CHECK( MPI_File_open(MPI_COMM_WORLD, filename, 
-		MPI_MODE_CREATE|MPI_MODE_RDWR, MPI_INFO_NULL, &fh));
-    CHECK( MPI_File_set_view(fh, rank*sizeof(int), 
-	    MPI_BYTE, type, "native", MPI_INFO_NULL)); 
-
-    MPI_Type_size_x(type, &tsize);
-
-    canary=rank+1000000;
-
-    /* skip over first instance of type */
-    if (do_coll) {
-	CHECK( MPI_File_write_at_all(fh, tsize, &canary, 1, MPI_INT, &status));
-    } else {
-	CHECK( MPI_File_write_at(fh, tsize, &canary, 1, MPI_INT, &status));
-    }
-
-    CHECK( MPI_File_set_view(fh, 0, MPI_INT, MPI_INT, "native", 
-		MPI_INFO_NULL)); 
-
-    if (do_coll) {
-	CHECK( MPI_File_read_at_all(fh, expected_extent/sizeof(int)+rank, 
-		&compare, 1, MPI_INT, &status));
-    } else {
-	CHECK( MPI_File_read_at(fh, expected_extent/sizeof(int)+rank, 
-		&compare, 1, MPI_INT, &status));
-    }
-
-    if (compare != canary)
-	errs=1;
-    MPI_Allreduce(&errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
-
-    MPI_File_close(&fh);
-
-    if (toterrs) {
-	printf("%d: got %d expected %d\n", rank, compare, canary);
-	/* keep file if there's an error */
-    } else {
-	if (rank == 0) MPI_File_delete(filename, MPI_INFO_NULL);
-    }
-	
-    return (toterrs);
-
-}
-
-static int testtype(char *filename, MPI_Datatype type, int64_t expected_extent)
-{
-    int rank, ret, errs=0;
-    int collective=1, nocollective=0;
-
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-    if (!rank) typestats(type);
-
-    ret = verify_type(filename, type, expected_extent, nocollective); 
-    if (ret) {
-	errs++;
-	fprintf(stderr, "type %d failed indep\n", type);
-    } else 
-	if (!rank) printf("indep: OK ");
-
-    ret = verify_type(filename, type, expected_extent, collective); 
-    if (ret) {
-	errs++;
-	fprintf(stderr, "type %d failed collective\n", type);
-    } else
-	if (!rank) printf("coll: OK\n");
-
-    return errs;
-}
-
-int main(int argc, char **argv)
-{
-    int count=2;
-    int blocks[2];
-    int disps[2];
-
-    int ndims=2;
-    int sizes[2];
-    int subs[2];
-    int starts[2];
-
-    MPI_Datatype baseindex, indexed1G, indexed3G, indexed6G; 
-    MPI_Datatype subarray1G, subarray3G, subarray6G;
-    int ret, rank;
-
-    MPI_Init(&argc, &argv);
-
-    if (argc != 2) {
-	fprintf(stderr, "usage: %s <filename>\n", argv[0]);
-	MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-
-    /* base type: 1MB indexed type of ints*/
-    count = 2;
-    blocks[0] = 1;
-    disps[0] = 0;
-    blocks[1] = 1;
-    disps[1] = 1024*256-1;
-
-    MPI_Type_indexed(count, blocks, disps, MPI_INT, &baseindex);
-    /* simple case: 1GB extent */
-    MPI_Type_contiguous(1024, baseindex, &indexed1G);
-    MPI_Type_commit(&indexed1G);
-
-    /* a little trickier: 3Gb extent */
-    MPI_Type_contiguous(3072, baseindex, &indexed3G);
-    MPI_Type_commit(&indexed3G);
-
-    /* and finally 6GB extent */
-    MPI_Type_contiguous(6144, baseindex, &indexed6G);
-    MPI_Type_commit(&indexed6G);
-
-    /* TODO: 
-     * - add a darray test
-     * - add a test with crazy extents */
-    sizes[0] = 1024*16;
-    sizes[1] = 1024*16;
-    subs[0] = subs[1] = 256;
-    starts[0] = starts[1] = 0;
-
-    MPI_Type_create_subarray(ndims, sizes, subs, starts, 
-	    MPI_ORDER_C, MPI_INT, &subarray1G);
-    MPI_Type_commit(&subarray1G);
-
-    sizes[1] = 1024*16*3;
-    MPI_Type_create_subarray(ndims, sizes, subs, starts, 
-	    MPI_ORDER_C, MPI_INT, &subarray3G);
-    MPI_Type_commit(&subarray3G);
-
-    sizes[1] = 1024*16*6;
-    MPI_Type_create_subarray(ndims, sizes, subs, starts, 
-	    MPI_ORDER_C, MPI_INT, &subarray6G);
-    MPI_Type_commit(&subarray6G);
-
-    /* assume command line arguments make it out to all processes */
-    ret = testtype(argv[1], indexed1G, (int64_t)1024*1024*1024);
-
-    ret = testtype(argv[1], indexed3G, (int64_t)1024*1024*1024*3);
-
-    ret = testtype(argv[1], indexed6G, (int64_t)1024*1024*1024*6);
-
-    ret = testtype(argv[1], subarray1G, (int64_t)1024*1024*1024);
-
-    ret = testtype(argv[1], subarray3G, (int64_t)1024*1024*1024*3);
-
-    ret = testtype(argv[1], subarray6G, (int64_t)1024*1024*1024*6);
-
-    if(!ret && !rank) fprintf(stderr, "  No Errors\n");
-    
-    MPI_Finalize();
-    return (-ret);
-
-}
-/* 
- * vim: ts=8 sts=4 sw=4 noexpandtab 
- */
diff --git a/3rd-party/romio321/test/coll_perf.c b/3rd-party/romio321/test/coll_perf.c
deleted file mode 100644
index 0f9042d5ff5..00000000000
--- a/3rd-party/romio321/test/coll_perf.c
+++ /dev/null
@@ -1,151 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*  
- *  (C) 2001 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include "mpi.h"
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-
-/* The file name is taken as a command-line argument. */
-
-/* Measures the I/O bandwidth for writing/reading a 3D
-   block-distributed array to a file corresponding to the global array
-   in row-major (C) order.
-   Note that the file access pattern is noncontiguous.
-  
-   Array size 128^3. For other array sizes, change array_of_gsizes below.*/
-
-
-int main(int argc, char **argv)
-{
-    MPI_Datatype newtype;
-    int i, ndims, array_of_gsizes[3], array_of_distribs[3];
-    int order, nprocs, len, *buf, mynod;
-    MPI_Count bufcount;
-    int array_of_dargs[3], array_of_psizes[3];
-    MPI_File fh;
-    MPI_Status status;
-    double stim, write_tim, new_write_tim, write_bw;
-    double read_tim, new_read_tim, read_bw;
-    char *filename;
-
-    MPI_Init(&argc,&argv);
-    MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
-    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
-
-/* process 0 takes the file name as a command-line argument and 
-   broadcasts it to other processes */
-    if (!mynod) {
-	i = 1;
-	while ((i < argc) && strcmp("-fname", *argv)) {
-	    i++;
-	    argv++;
-	}
-	if (i >= argc) {
-	    fprintf(stderr, "\n*#  Usage: coll_perf -fname filename\n\n");
-	    MPI_Abort(MPI_COMM_WORLD, 1);
-	}
-	argv++;
-	len = strlen(*argv);
-	filename = (char *) malloc(len+1);
-	strcpy(filename, *argv);
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-    else {
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	filename = (char *) malloc(len+1);
-	MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-
-
-    ndims = 3;
-    order = MPI_ORDER_C;
-
-    array_of_gsizes[0] = 128*17;
-    array_of_gsizes[1] = 128*9;
-    array_of_gsizes[2] = 128*11;
-
-    array_of_distribs[0] = MPI_DISTRIBUTE_BLOCK;
-    array_of_distribs[1] = MPI_DISTRIBUTE_BLOCK;
-    array_of_distribs[2] = MPI_DISTRIBUTE_BLOCK;
-
-    array_of_dargs[0] = MPI_DISTRIBUTE_DFLT_DARG;
-    array_of_dargs[1] = MPI_DISTRIBUTE_DFLT_DARG;
-    array_of_dargs[2] = MPI_DISTRIBUTE_DFLT_DARG;
-
-    for (i=0; i<ndims; i++) array_of_psizes[i] = 0;
-    MPI_Dims_create(nprocs, ndims, array_of_psizes);
-
-    MPI_Type_create_darray(nprocs, mynod, ndims, array_of_gsizes,
-			   array_of_distribs, array_of_dargs,
-			   array_of_psizes, order, MPI_INT, &newtype);
-    MPI_Type_commit(&newtype);
-
-    MPI_Type_size_x(newtype, &bufcount);
-    bufcount = bufcount/sizeof(int);
-    buf = (int *) malloc(bufcount * sizeof(int));
-
-/* to eliminate paging effects, do the operations once but don't time
-   them */
-
-    MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, 
-                  MPI_INFO_NULL, &fh);
-    MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", MPI_INFO_NULL);
-    MPI_File_write_all(fh, buf, bufcount, MPI_INT, &status);
-    MPI_File_seek(fh, 0, MPI_SEEK_SET);
-    MPI_File_read_all(fh, buf, bufcount, MPI_INT, &status);
-    MPI_File_close(&fh);
-
-    MPI_Barrier(MPI_COMM_WORLD);
-/* now time write_all */
-
-    MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, 
-                  MPI_INFO_NULL, &fh);
-    MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", MPI_INFO_NULL);
-
-    MPI_Barrier(MPI_COMM_WORLD);
-    stim = MPI_Wtime();
-    MPI_File_write_all(fh, buf, bufcount, MPI_INT, &status);
-    write_tim = MPI_Wtime() - stim;
-    MPI_File_close(&fh);
-
-    MPI_Allreduce(&write_tim, &new_write_tim, 1, MPI_DOUBLE, MPI_MAX,
-                    MPI_COMM_WORLD);
-
-    if (mynod == 0) {
-      write_bw = (array_of_gsizes[0]*array_of_gsizes[1]*array_of_gsizes[2]*sizeof(int))/(new_write_tim*1024.0*1024.0);
-      fprintf(stderr, "Global array size %d x %d x %d integers\n", array_of_gsizes[0], array_of_gsizes[1], array_of_gsizes[2]);
-      fprintf(stderr, "Collective write time = %f sec, Collective write bandwidth = %f Mbytes/sec\n", new_write_tim, write_bw);
-    }
-
-    MPI_Barrier(MPI_COMM_WORLD);
-/* now time read_all */
-
-    MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, 
-                  MPI_INFO_NULL, &fh); 
-    MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", MPI_INFO_NULL);
-
-    MPI_Barrier(MPI_COMM_WORLD);
-    stim = MPI_Wtime();
-    MPI_File_read_all(fh, buf, bufcount, MPI_INT, &status);
-    read_tim = MPI_Wtime() - stim;
-    MPI_File_close(&fh);
-
-    MPI_Allreduce(&read_tim, &new_read_tim, 1, MPI_DOUBLE, MPI_MAX,
-                    MPI_COMM_WORLD);
-
-    if (mynod == 0) {
-      read_bw = (array_of_gsizes[0]*array_of_gsizes[1]*array_of_gsizes[2]*sizeof(int))/(new_read_tim*1024.0*1024.0);
-      fprintf(stderr, "Collective read time = %f sec, Collective read bandwidth = %f Mbytes/sec\n", new_read_tim, read_bw);
-    }
-
-    MPI_Type_free(&newtype);
-    free(buf);
-    free(filename);
-
-    MPI_Finalize();
-    return 0;
-}
diff --git a/3rd-party/romio321/test/coll_test.c b/3rd-party/romio321/test/coll_test.c
deleted file mode 100644
index 4efc4d25b02..00000000000
--- a/3rd-party/romio321/test/coll_test.c
+++ /dev/null
@@ -1,206 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*  
- *  (C) 2001 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include "mpi.h"
-#include <stdlib.h>
-#include <string.h>
-#include <stdio.h>
-
-/* A 32^3 array. For other array sizes, change array_of_gsizes below. */
-
-/* Uses collective I/O. Writes a 3D block-distributed array to a file
-   corresponding to the global array in row-major (C) order, reads it
-   back, and checks that the data read is correct. */
-
-/* The file name is taken as a command-line argument. */
-
-/* Note that the file access pattern is noncontiguous. */
-   
-void handle_error(int errcode, const char *str);
-
-void handle_error(int errcode, const char *str)
-{
-	char msg[MPI_MAX_ERROR_STRING];
-	int resultlen;
-	MPI_Error_string(errcode, msg, &resultlen);
-	fprintf(stderr, "%s: %s\n", str, msg);
-	MPI_Abort(MPI_COMM_WORLD, 1);
-}
-
-int main(int argc, char **argv)
-{
-    MPI_Datatype newtype;
-    int i, ndims, array_of_gsizes[3], array_of_distribs[3];
-    int order, nprocs, j, len;
-    int array_of_dargs[3], array_of_psizes[3];
-    int *readbuf, *writebuf, mynod, *tmpbuf, array_size;
-    MPI_Count bufcount;
-    char *filename;
-    int errs=0, toterrs;
-    MPI_File fh;
-    MPI_Status status;
-    MPI_Request request;
-    MPI_Info info = MPI_INFO_NULL;
-    int errcode;
-
-    MPI_Init(&argc,&argv);
-    MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
-    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
-
-/* process 0 takes the file name as a command-line argument and 
-   broadcasts it to other processes */
-    if (!mynod) {
-	i = 1;
-	while ((i < argc) && strcmp("-fname", *argv)) {
-	    i++;
-	    argv++;
-	}
-	if (i >= argc) {
-	    fprintf(stderr, "\n*#  Usage: coll_test -fname filename\n\n");
-	    MPI_Abort(MPI_COMM_WORLD, 1);
-	}
-	argv++;
-	len = strlen(*argv);
-	filename = (char *) malloc(len+1);
-	strcpy(filename, *argv);
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-    else {
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	filename = (char *) malloc(len+1);
-	MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-
-
-/* create the distributed array filetype */
-    ndims = 3;
-    order = MPI_ORDER_C;
-
-    array_of_gsizes[0] = 32;
-    array_of_gsizes[1] = 32;
-    array_of_gsizes[2] = 32;
-
-    array_of_distribs[0] = MPI_DISTRIBUTE_BLOCK;
-    array_of_distribs[1] = MPI_DISTRIBUTE_BLOCK;
-    array_of_distribs[2] = MPI_DISTRIBUTE_BLOCK;
-
-    array_of_dargs[0] = MPI_DISTRIBUTE_DFLT_DARG;
-    array_of_dargs[1] = MPI_DISTRIBUTE_DFLT_DARG;
-    array_of_dargs[2] = MPI_DISTRIBUTE_DFLT_DARG;
-
-    for (i=0; i<ndims; i++) array_of_psizes[i] = 0;
-    MPI_Dims_create(nprocs, ndims, array_of_psizes);
-
-    MPI_Type_create_darray(nprocs, mynod, ndims, array_of_gsizes, 
-			   array_of_distribs, array_of_dargs,
-			   array_of_psizes, order, MPI_INT, &newtype);
-    MPI_Type_commit(&newtype);
-
-/* initialize writebuf */
-
-    MPI_Type_size_x(newtype, &bufcount);
-    bufcount = bufcount/sizeof(int);
-    writebuf = (int *) malloc(bufcount * sizeof(int));
-    for (i=0; i<bufcount; i++) writebuf[i] = 1;
-
-    array_size = array_of_gsizes[0]*array_of_gsizes[1]*array_of_gsizes[2];
-    tmpbuf = (int *) calloc(array_size, sizeof(int));
-    MPI_Irecv(tmpbuf, 1, newtype, mynod, 10, MPI_COMM_WORLD, &request);
-    MPI_Send(writebuf, bufcount, MPI_INT, mynod, 10, MPI_COMM_WORLD);
-    MPI_Wait(&request, &status);
-
-    j = 0;
-    for (i=0; i<array_size; i++)
-	if (tmpbuf[i]) {
-	    writebuf[j] = i;
-	    j++;
-	}
-    free(tmpbuf);
-
-    if (j != bufcount) {
-	fprintf(stderr, "Error in initializing writebuf on process %d\n", mynod);
-	MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-/* end of initialization */
-
-    /* write the array to the file */
-    errcode = MPI_File_open(MPI_COMM_WORLD, filename, 
-		    MPI_MODE_CREATE | MPI_MODE_RDWR, info, &fh);
-    if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_open");
-   
-    errcode = MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info);
-    if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_set_view");
-
-    errcode = MPI_File_write_all(fh, writebuf, bufcount, MPI_INT, &status);
-    if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_write_all");
-    errcode = MPI_File_close(&fh);
-    if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_close");
-
-    if (!mynod) {
-        /* wkl suggests potential for false " No Errors" if both read 
-	 * and write use the same file view */
-        /* solution: rank 0 reads entire file and checks write values */
-	errcode = MPI_File_open(MPI_COMM_SELF, filename, 
-			MPI_MODE_RDONLY, info, &fh);
-        if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_open");
-
-        readbuf = (int *) malloc(array_size * sizeof(int));
-        errcode = MPI_File_read(fh, readbuf, array_size, MPI_INT, &status);
-        if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_read");
-
-        errcode = MPI_File_close(&fh);
-        if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_close");
-
-        for (i=0; i<array_size; i++)
-            if (readbuf[i] != i) {
-                errs++;
-                fprintf(stderr, "Error: write integer %d but read %d\n", 
-				i,readbuf[i]);
-                break;
-            }
-        free(readbuf);
-    }
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    /* now read it back */
-    readbuf = (int *) malloc(bufcount * sizeof(int));
-    errcode = MPI_File_open(MPI_COMM_WORLD, filename, 
-		    MPI_MODE_CREATE | MPI_MODE_RDWR, info, &fh);
-    if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_open");
-
-    errcode = MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info);
-    if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_set_view");
-    errcode = MPI_File_read_all(fh, readbuf, bufcount, MPI_INT, &status);
-    if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_read_all");
-    errcode = MPI_File_close(&fh);
-    if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_close");
-
-    /* check the data read */
-    for (i=0; i<bufcount; i++) {
-	if (readbuf[i] != writebuf[i]) {
-	    errs++;
-	    fprintf(stderr, "Process %d, readbuf %d, writebuf %d, i %d\n", mynod, readbuf[i], writebuf[i], i);
-	}
-    }
-
-    MPI_Allreduce( &errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
-    if (mynod == 0) {
-	if( toterrs > 0) {
-	    fprintf( stderr, "Found %d errors\n", toterrs );
-	}
-	else {
-	    fprintf( stdout, " No Errors\n" );
-	}
-    }
-
-    MPI_Type_free(&newtype);
-    free(readbuf);
-    free(writebuf);
-    free(filename);
-
-    MPI_Finalize();
-    return 0;
-}
diff --git a/3rd-party/romio321/test/creat_excl.c b/3rd-party/romio321/test/creat_excl.c
deleted file mode 100644
index 858df00e8a1..00000000000
--- a/3rd-party/romio321/test/creat_excl.c
+++ /dev/null
@@ -1,134 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*  
- *  (C) 2001 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include "mpi.h"
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-/* 
- * the new defered open code made some changes to the way we manage CREAT|EXCL,
- * so test out that code path */
-
-struct options {
-	char *fname;
-	int verbose;
-	int do_aggregation;
-};
-typedef struct options options;
-
-
-void handle_error(int errcode, char *str) 
-{
-	char msg[MPI_MAX_ERROR_STRING];
-	int resultlen;
-	MPI_Error_string(errcode, msg, &resultlen);
-	fprintf(stderr, "%s: %s\n", str, msg);
-	MPI_Abort(MPI_COMM_WORLD, 1);
-}
-
-void parse_args(int argc, char ** argv, int rank, options *opts) 
-{
-	int i, len=0;
-	if (rank == 0) {
-		i = 1;
-		while (i < argc) {
-			if (strcmp("-fname", argv[i]) == 0) {
-				len = strlen(argv[i+1]);
-				opts->fname = (char *) malloc(len + 1);
-				strcpy(opts->fname, argv[i+1]);
-				i+=2;/* option and argument */
-				continue;
-			}
-			if (strcmp("-aggregate", argv[i]) == 0) {
-				opts->do_aggregation = 1;
-				i++;
-				continue;
-			}
-			if (strcmp("-verbose", argv[i]) == 0) {
-				opts->verbose = 1;
-				i++;
-				continue;
-			}
-		}
-		if (opts->fname == NULL) { /* didn't get a filename */
-			fprintf(stderr, "Usage: %s -fname filename [-aggregate] [-verbose]\n", argv[0]);
-			MPI_Abort(MPI_COMM_WORLD, 1);
-		}
-		MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-		MPI_Bcast(opts->fname, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
-		MPI_Bcast(&(opts->do_aggregation), 1, MPI_INT, 0, MPI_COMM_WORLD);
-		MPI_Bcast(&(opts->verbose), 1, MPI_INT, 0, MPI_COMM_WORLD);
-	} else {
-		MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-		opts->fname = (char *) malloc(len + 1);
-		MPI_Bcast(opts->fname, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
-		MPI_Bcast(&(opts->do_aggregation), 1, MPI_INT, 0, MPI_COMM_WORLD);
-		MPI_Bcast(&(opts->verbose), 1, MPI_INT, 0, MPI_COMM_WORLD);
-	}
-
-}
-
-int main(int argc, char ** argv) {
-	int nprocs, mynod, errcode;
-	options my_options = {NULL, 0, 0};
-	MPI_File fh;
-	MPI_Status status;
-	MPI_Info  info;
-
-	MPI_Init(&argc, &argv);
-	MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
-	MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
-
-	parse_args(argc, argv, mynod, &my_options);
-
-	if (my_options.do_aggregation) {
-		MPI_Info_create(&info);
-		MPI_Info_set(info, "romio_no_indep_rw", "true");
-		MPI_Info_set(info, "cb_config_list", "leela.mcs.anl.gov:1");
-	} else {
-		info = MPI_INFO_NULL;
-	}
-
-	/* create the file w/o EXCL: this must not fail */
-	errcode = MPI_File_open(MPI_COMM_WORLD, my_options.fname,
-			MPI_MODE_CREATE|MPI_MODE_RDWR, info, &fh);
-	if (errcode != MPI_SUCCESS) {
-		handle_error(errcode, "MPI_File_open");
-	}
-
-	errcode = MPI_File_close(&fh);
-	if (errcode != MPI_SUCCESS) {
-		handle_error(errcode, "MPI_File_close");
-	}
-
-	/* now try to open w/ CREAT|EXCL: this must fail */
-	errcode = MPI_File_open(MPI_COMM_WORLD, my_options.fname,
-			MPI_MODE_CREATE|MPI_MODE_EXCL|MPI_MODE_RDWR, info, &fh);
-	if (errcode == MPI_SUCCESS) {
-		handle_error(errcode, "MPI_File_open: expected an error: got");
-	}
-
-	/* ignore the error: File_delete is not aggregator-aware */
-	MPI_File_delete(my_options.fname, info);
-
-	/* this must succeed: the file no longer exists */
-	errcode = MPI_File_open(MPI_COMM_WORLD, my_options.fname,
-			MPI_MODE_CREATE|MPI_MODE_EXCL|MPI_MODE_RDWR, info, &fh);
-	if (errcode != MPI_SUCCESS) {
-		handle_error(errcode, "MPI_File_open");
-	}
-
-	errcode = MPI_File_close(&fh);
-	if (errcode != MPI_SUCCESS) {
-		handle_error(errcode, "MPI_File_close");
-	}
-
-	if (mynod == 0) {
-		printf(" No Errors\n");
-	}
-
-	MPI_Finalize();
-	return 0;
-}
diff --git a/3rd-party/romio321/test/darray_read.c b/3rd-party/romio321/test/darray_read.c
deleted file mode 100644
index 7b8e4c59e8c..00000000000
--- a/3rd-party/romio321/test/darray_read.c
+++ /dev/null
@@ -1,129 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *  (C) 2014 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <mpi.h>
-
-#define NSIDE 5
-#define NBLOCK 3
-#define NPROC 2
-
-#define CHECK(fn) {int errcode; errcode = (fn); if (errcode != MPI_SUCCESS) handle_error(errcode, #fn);}
-
-static void handle_error(int errcode, const char *str)
-{
-	char msg[MPI_MAX_ERROR_STRING];
-	int resultlen;
-	MPI_Error_string(errcode, msg, &resultlen);
-	fprintf(stderr, "%s: %s\n", str, msg);
-	MPI_Abort(MPI_COMM_WORLD, 1);
-}
-
-
-int main(int argc, char *argv[]) 
-{ 
-  int i, j, nerrors=0, total_errors=0; 
-
-  int rank, size;
-  int bpos;
-
-  MPI_Datatype darray;
-  MPI_Status status;
-  MPI_File mpi_fh;
-
-  /* Define array distribution
-      A 2x2 block size works with ROMIO, a 3x3 block size breaks it. */
-  int distrib[2] = { MPI_DISTRIBUTE_CYCLIC, MPI_DISTRIBUTE_CYCLIC };
-  int bsize[2] = { NBLOCK, NBLOCK };
-  int gsize[2] = { NSIDE, NSIDE };
-  int psize[2] = { NPROC, NPROC };
-
-  double data[NSIDE*NSIDE];
-  double *ldata, *pdata;
-
-  int tsize, nelem;
-
-  MPI_File dfile;
- 
-  MPI_Init(&argc, &argv);
-
-  MPI_Comm_size(MPI_COMM_WORLD, &size);
-  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-
-  /* Set up type */
-  CHECK(MPI_Type_create_darray(size, rank, 2, gsize, distrib,
-			 bsize, psize, MPI_ORDER_FORTRAN, MPI_DOUBLE, &darray));
-  CHECK(MPI_Type_commit(&darray));
-  CHECK(MPI_Type_size(darray, &tsize));
-  nelem = tsize / sizeof(double);
-
-  for(i = 0; i < (NSIDE*NSIDE); i++) data[i] = i;
-
-  if (rank == 0) {
-    CHECK(MPI_File_open(MPI_COMM_SELF, argv[1],
-		MPI_MODE_CREATE|MPI_MODE_WRONLY, MPI_INFO_NULL, &dfile));
-    CHECK(MPI_File_write(dfile, data, NSIDE*NSIDE, MPI_DOUBLE, &status));
-    CHECK(MPI_File_close(&dfile));
-  }
-  MPI_Barrier(MPI_COMM_WORLD);
-
-  /* Allocate buffer */
-  ldata = (double *)malloc(tsize);
-  pdata = (double *)malloc(tsize);
-
-  /* Use Pack to pull out array */
-  bpos = 0;
-  CHECK(MPI_Pack(data, 1, darray, pdata, tsize, &bpos, MPI_COMM_WORLD));
-
-  MPI_Barrier(MPI_COMM_WORLD);
-
-  /* Read in array from file.  */
-  CHECK(MPI_File_open(MPI_COMM_WORLD, argv[1], MPI_MODE_RDONLY, MPI_INFO_NULL, &mpi_fh));
-  CHECK(MPI_File_set_view(mpi_fh, 0, MPI_DOUBLE, darray, "native", MPI_INFO_NULL));
-  CHECK(MPI_File_read_all(mpi_fh, ldata, nelem, MPI_DOUBLE, &status));
-  CHECK(MPI_File_close(&mpi_fh));
-
-  for(i = 0; i < size; i++) {
-#ifdef VERBOSE
-    MPI_Barrier(MPI_COMM_WORLD);
-    if(rank == i) {
-      printf("=== Rank %i === (%i elements) \nPacked: ", rank, nelem);
-      for(j = 0; j < nelem; j++) {
-        printf("%4.1f ", pdata[j]);
-        fflush(stdout);
-      }
-      printf("\nRead:   ");
-      for(j = 0; j < nelem; j++) {
-        printf("%4.1f ", ldata[j]);
-        fflush(stdout);
-      }
-      printf("\n\n");
-      fflush(stdout);
-    }
-#endif
-    if(rank == i) {
-	for (j=0; j< nelem; j++) {
-	    if (pdata[j] != ldata[j]) {
-		fprintf(stderr, "rank %d at index %d: packbuf %4.1f filebuf %4.1f\n",
-			rank, j, pdata[j], ldata[j]);
-		nerrors++;
-	    }
-	}
-    }
-  }
-  MPI_Allreduce(&nerrors, &total_errors, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
-  if (rank == 0 && total_errors == 0)
-      printf(" No Errors\n");
-
-  free(ldata);
-  free(pdata);
-  MPI_Type_free(&darray);
-  MPI_Finalize();
-
-  exit(total_errors);
-
-} 
diff --git a/3rd-party/romio321/test/error.c b/3rd-party/romio321/test/error.c
deleted file mode 100644
index ee24e964596..00000000000
--- a/3rd-party/romio321/test/error.c
+++ /dev/null
@@ -1,106 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*  
- *  (C) 2001 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include "mpi.h"
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-
-#define VERBOSE 0
-/* tests if error message is printed correctly */
-
-int main(int argc, char **argv)
-{
-    int i, rank, len, err;
-    int errs = 0;
-    char *filename, *tmp;
-    MPI_File fh;
-    char string[MPI_MAX_ERROR_STRING];
-
-    MPI_Init(&argc,&argv);
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-
-#if VERBOSE
-    if (!rank) {
-	fprintf(stderr, "Tests if errors are reported correctly...\n");
-	fprintf(stderr, "Should say \"Invalid displacement argument\"\n\n");
-    }
-#endif
-
-/* process 0 takes the file name as a command-line argument and 
-   broadcasts it to other processes */
-    if (!rank) {
-	i = 1;
-	while ((i < argc) && strcmp("-fname", *argv)) {
-	    i++;
-	    argv++;
-	}
-	if (i >= argc) {
-	    fprintf(stderr, "\n*#  Usage: simple -fname filename\n\n");
-	    MPI_Abort(MPI_COMM_WORLD, 1);
-	}
-	argv++;
-	len = strlen(*argv);
-	filename = (char *) malloc(len+10);
-	strcpy(filename, *argv);
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	MPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-    else {
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	filename = (char *) malloc(len+10);
-	MPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-    
-    /* each process opens a separate file called filename.'myrank' */
-    tmp = (char *) malloc(len+10);
-    strcpy(tmp, filename);
-    sprintf(filename, "%s.%d", tmp, rank);
-
-    err = MPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_CREATE+MPI_MODE_RDWR,
-		        MPI_INFO_NULL, &fh);
-    err = MPI_File_set_view(fh, -1, MPI_BYTE, MPI_BYTE, "native", 
-                            MPI_INFO_NULL);
-    /* disp is deliberately passed as -1 */
-
-    /* This test is designed for ROMIO specifically and tests for a 
-       specific error message */
-    if (err != MPI_SUCCESS) {
-	MPI_Error_string(err, string, &len);
-	if (!rank) {
-#if VERBOSE
-	    fprintf(stderr, "%s\n", string);
-#else
-	    /* check for the word "displacement" in the message.
-	       This allows other formatting of the message */
-	    if (strstr( string, "displacement" ) == 0) {
-		fprintf( stderr, "Unexpected error message %s\n", string );
-		errs++;
-	    }
-#endif
-	}
-    }
-    else {
-	errs++;
-	fprintf( stderr, "File set view did not return an error\n" );
-    }
-
-    MPI_File_close(&fh);
-
-    free(filename);
-    free(tmp);
-
-    if (!rank) {
-	if (errs == 0) {
-	    printf( " No Errors\n" );
-	}
-	else {
-	    printf( " Found %d errors\n", errs );
-	}
-    }
-
-    MPI_Finalize();
-    return 0; 
-}
diff --git a/3rd-party/romio321/test/excl.c b/3rd-party/romio321/test/excl.c
deleted file mode 100644
index e48bb3deee5..00000000000
--- a/3rd-party/romio321/test/excl.c
+++ /dev/null
@@ -1,84 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*  
- *  (C) 2001 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include "mpi.h"
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-
-/* tests MPI_MODE_EXCL */
-
-int main(int argc, char **argv)
-{
-    MPI_File fh;
-    int rank, len, err, i;
-    int errs=0, toterrs;
-    char *filename;
-
-    MPI_Init(&argc,&argv);
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-
-/* process 0 takes the file name as a command-line argument and 
-   broadcasts it to other processes */
-    if (!rank) {
-        i = 1;
-        while ((i < argc) && strcmp("-fname", *argv)) {
-            i++;
-            argv++;
-        }
-        if (i >= argc) {
-            fprintf(stderr, "\n*#  Usage: excl -fname filename\n\n");
-            MPI_Abort(MPI_COMM_WORLD, 1);
-        }
-        argv++;
-        len = strlen(*argv);
-        filename = (char *) malloc(len+10);
-        strcpy(filename, *argv);
-        MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-        MPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-    else {
-        MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-        filename = (char *) malloc(len+10);
-        MPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-    
-
-    if (!rank) MPI_File_delete(filename, MPI_INFO_NULL);
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    /* this open should succeed */
-    err = MPI_File_open(MPI_COMM_WORLD, filename, 
-         MPI_MODE_CREATE | MPI_MODE_EXCL | MPI_MODE_RDWR, MPI_INFO_NULL , &fh);
-    if (err != MPI_SUCCESS) {
-	errs++;
-	fprintf(stderr, "Process %d: open failed when it should have succeeded\n", rank);
-    }
-    else MPI_File_close(&fh);
-
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    /* this open should fail */
-    err = MPI_File_open(MPI_COMM_WORLD, filename, 
-         MPI_MODE_CREATE | MPI_MODE_EXCL | MPI_MODE_RDWR, MPI_INFO_NULL , &fh);
-    if (err == MPI_SUCCESS) {
-	errs++;
-	fprintf(stderr, "Process %d: open succeeded when it should have failed\n", rank);
-    }
-
-    MPI_Allreduce( &errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
-    if (rank == 0) {
-	if( toterrs > 0) {
-	    fprintf( stderr, "Found %d errors\n", toterrs );
-	}
-	else {
-	    fprintf( stdout, " No Errors\n" );
-	}
-    }
-
-    free(filename);
-    MPI_Finalize();
-    return 0; 
-}
diff --git a/3rd-party/romio321/test/external32.c b/3rd-party/romio321/test/external32.c
deleted file mode 100644
index bca3555ae86..00000000000
--- a/3rd-party/romio321/test/external32.c
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- *  This code was written by Intel Corporation. Copyright (C) 2011-2012 Intel Corporation.
- *  Intel provides this material to Argonne National Laboratory subject to 
- *  Software Grant and Corporate Contributor License Agreement dated February 8, 2012.
- *
- *  See COPYRIGHT in top-level directory.
- */
-
-#include <stdlib.h>
-#include <stdio.h>
-#include "mpi.h"
-
-#define TEST_LE 0x1
-#define TEST_BE 0x2
-#define TEST_FILENAME "test.datarep"
-
-#define CHECK(fn) {int errcode; errcode = (fn); if (errcode != MPI_SUCCESS) handle_error(errcode, NULL); }
-
-static void handle_error(int errcode, char *str)
-{
-        char msg[MPI_MAX_ERROR_STRING];
-        int resultlen;
-        MPI_Error_string(errcode, msg, &resultlen);
-        fprintf(stderr, "%s: (%d) %s\n", str, errcode, msg);
-        MPI_Abort(MPI_COMM_WORLD, 1);
-}
-
-
-
-static void is_little_or_big_endian( const char* datarep, char* c, char* c_le, int len ) {
-    int i, is_le = 1, is_be = 1;
-    for( i = 0; i < len; i++ ) {
-        is_le = is_le && ( c[i] == c_le[i] );
-        is_be = is_be && ( c[i] == c_le[len-1-i] );
-    }
-    printf( "%s datarep is ", datarep );
-    switch ((is_le ? TEST_LE : 0x0) | (is_be ? TEST_BE : 0x0) ) {
-        case TEST_LE: printf( "LITTLE ENDIAN\n" ); break;
-        case TEST_BE: printf( "BIG ENDIAN\n" ); break;
-        case TEST_LE | TEST_BE: printf( "LITTLE or BIG ENDIAN\n" ); break;
-        default: printf( "unknown\n" ); break;
-    }
-}
-
-/* This test checks if datareps given are little- or big-endian */
-int main( int argc, char* argv[] ) {
-    int sample_i = 123456789, i, j;
-    char sample_i_le[4] = {0x15,0xcd,0x5b,0x07}, c[4];
-    const char* datarep[3] = { "native", "external32", "internal" };
-    MPI_File fileh;
-    int rank;
-    FILE* fileh_std;
-
-    if( sizeof(int) != 4 ) { printf( "non-supported sizeof(int)=%ld\n", sizeof(int) ); return (-1); }
-
-    MPI_Init( &argc, &argv );
-    MPI_Comm_rank( MPI_COMM_WORLD, &rank );
-
-    /* For each datarep */
-    for( i = 0; i < 3; i++ ) {
-
-        /* Open file */
-        CHECK(MPI_File_open( MPI_COMM_WORLD, TEST_FILENAME, 
-		    MPI_MODE_RDWR | MPI_MODE_CREATE, MPI_INFO_NULL, &fileh ) );
-
-        /* Set view */
-	CHECK(MPI_File_set_view( fileh, 0, MPI_INT, MPI_INT, datarep[i], MPI_INFO_NULL ));
-
-        /* Write into file */
-	CHECK(MPI_File_write_at( fileh, (MPI_Offset)rank, (void*)&sample_i, 1, 
-		    MPI_INT, MPI_STATUS_IGNORE ));
-
-        /* Close file */
-        CHECK(MPI_File_close( &fileh ));
-
-        /* Check if your datarep is little or big endian */
-        MPI_Barrier( MPI_COMM_WORLD );
-        if( rank == 0 ) {
-            fileh_std = fopen( TEST_FILENAME, "r" );
-            for( j = 0; j < 4; j++ ) {
-                if( feof( fileh_std ) ) { printf( "unexpected eof, aborted\n" ); return (-1); }
-                fscanf( fileh_std, "%c", &c[j] );
-            }
-            is_little_or_big_endian( datarep[i], c, sample_i_le, 4 );
-            fclose( fileh_std );
-        }
-
-        /* Delete file */
-        if( rank == 0 ) {
-            CHECK(MPI_File_delete( TEST_FILENAME, MPI_INFO_NULL ));
-        }
-    }
-
-    MPI_Finalize();
-
-    return 0;
-}
diff --git a/3rd-party/romio321/test/fcoll_test.f.in b/3rd-party/romio321/test/fcoll_test.f.in
deleted file mode 100644
index d6bae7446c5..00000000000
--- a/3rd-party/romio321/test/fcoll_test.f.in
+++ /dev/null
@@ -1,175 +0,0 @@
-! -*- Mode: Fortran; -*- 
-!  
-!  (C) 2001 by Argonne National Laboratory.
-!      See COPYRIGHT in top-level directory.
-!
-      program main
-      implicit none
-
-      include 'mpif.h'
-      @F77MPIOINC@
-
-!     Fortran equivalent of coll_test.c
-
-      integer FILESIZE 
-      parameter (FILESIZE=32*32*32*4)
-
-!     A 32^3 array. For other array sizes, change FILESIZE above and
-!     array_of_gsizes below.
-
-!     Uses collective I/O. Writes a 3D block-distributed array to a file
-!     corresponding to the global array in row-major (C) order, reads it
-!     back, and checks that the data read is correct.
-
-!     Note that the file access pattern is noncontiguous.
-   
-      integer newtype, i, ndims, array_of_gsizes(3)
-      integer order, intsize, nprocs, j, array_of_distribs(3)
-      integer array_of_dargs(3), array_of_psizes(3)
-      integer readbuf(FILESIZE), writebuf(FILESIZE), bufcount
-      integer mynod, tmpbuf(FILESIZE), array_size, argc, iargc
-      integer fh, status(MPI_STATUS_SIZE), request, ierr
-      character*1024 str    ! used to store the filename
-      integer errs, toterrs
-      @FORTRAN_MPI_OFFSET@ disp
-      @FTESTDEFINE@
-
-      errs = 0
-      call MPI_INIT(ierr)
-      call MPI_COMM_SIZE(MPI_COMM_WORLD, nprocs, ierr)
-      call MPI_COMM_RANK(MPI_COMM_WORLD, mynod, ierr)
-
-!     process 0 takes the file name as a command-line argument and 
-!     broadcasts it to other processes
-
-      if (mynod .eq. 0) then
-         argc = @F77IARGC@
-         i = 0
-         @F77GETARG@
-         do while ((i .lt. argc) .and. (str .ne. '-fname'))
-            i = i + 1
-            @F77GETARG@
-         end do
-         if (i .ge. argc) then
-            print *
-            print *, '*#  Usage: fcoll_test -fname filename'
-            print *
-            call MPI_ABORT(MPI_COMM_WORLD, 1, ierr)
-         end if
-
-         i = i + 1
-         @F77GETARG@
-         call MPI_BCAST(str, 1024, MPI_CHARACTER, 0,                    &
-     &        MPI_COMM_WORLD, ierr)
-      else 
-         call MPI_BCAST(str, 1024, MPI_CHARACTER, 0,                    &
-     &        MPI_COMM_WORLD, ierr)
-      end if
-
-
-!     create the distributed array filetype
-    
-      ndims = 3
-      order = MPI_ORDER_FORTRAN
-
-      array_of_gsizes(1) = 32
-      array_of_gsizes(2) = 32
-      array_of_gsizes(3) = 32
-
-      array_of_distribs(1) = MPI_DISTRIBUTE_BLOCK
-      array_of_distribs(2) = MPI_DISTRIBUTE_BLOCK
-      array_of_distribs(3) = MPI_DISTRIBUTE_BLOCK
-
-      array_of_dargs(1) = MPI_DISTRIBUTE_DFLT_DARG
-      array_of_dargs(2) = MPI_DISTRIBUTE_DFLT_DARG
-      array_of_dargs(3) = MPI_DISTRIBUTE_DFLT_DARG
-
-      do i=1, ndims
-         array_of_psizes(i) = 0
-      end do
-
-      call MPI_DIMS_CREATE(nprocs, ndims, array_of_psizes, ierr)
-
-      call MPI_TYPE_CREATE_DARRAY(nprocs, mynod, ndims,                 &
-     &     array_of_gsizes, array_of_distribs, array_of_dargs,          &
-     &     array_of_psizes, order, MPI_INTEGER, newtype, ierr)
-
-      call MPI_TYPE_COMMIT(newtype, ierr)
-
-!     initialize writebuf 
-
-      call MPI_TYPE_SIZE(newtype, bufcount, ierr)
-      call MPI_TYPE_SIZE(MPI_INTEGER, intsize, ierr)
-      bufcount = bufcount/intsize
-      do i=1, bufcount 
-         writebuf(i) = 1
-      end do
-
-      do i=1, FILESIZE
-         tmpbuf(i) = 0
-      end do
-
-      call MPI_IRECV(tmpbuf, 1, newtype, mynod, 10, MPI_COMM_WORLD,     &
-     &     request, ierr)
-      call MPI_SEND(writebuf, bufcount, MPI_INTEGER, mynod, 10,         &
-     &     MPI_COMM_WORLD, ierr)
-      call MPI_WAIT(request, status, ierr)
-
-      j = 1
-      array_size = array_of_gsizes(1) * array_of_gsizes(2) *            &
-     &     array_of_gsizes(3)
-      do i=1, array_size
-         if (tmpbuf(i) .ne. 0) then
-            writebuf(j) = i
-            j = j + 1
-         end if
-      end do
-
-!     end of initialization
-
-!     write the array to the file
-
-      call MPI_FILE_OPEN(MPI_COMM_WORLD, str,                           &
-     &     MPI_MODE_CREATE+MPI_MODE_RDWR, MPI_INFO_NULL, fh, ierr)
-
-      disp = 0 
-      call MPI_FILE_SET_VIEW(fh, disp, MPI_INTEGER, newtype, "native",  &
-     &     MPI_INFO_NULL, ierr)
-      call MPI_FILE_WRITE_ALL(fh, writebuf, bufcount, MPI_INTEGER,      &
-     &     status, ierr)
-      call MPI_FILE_CLOSE(fh, ierr)
-
-!    now read it back
-
-      call MPI_FILE_OPEN(MPI_COMM_WORLD, str,                           &
-     &     MPI_MODE_CREATE+MPI_MODE_RDWR, MPI_INFO_NULL, fh, ierr)
- 
-      call MPI_FILE_SET_VIEW(fh, disp, MPI_INTEGER, newtype, "native",  &
-     &     MPI_INFO_NULL, ierr)
-      call MPI_FILE_READ_ALL(fh, readbuf, bufcount, MPI_INTEGER,        &
-     &     status, ierr)
-      call MPI_FILE_CLOSE(fh, ierr)
-
-!     check the data read
-      do i=1, bufcount
-         if (readbuf(i) .ne. writebuf(i)) then
-            errs = errs + 1
-             print *, 'Node ', mynod, '  readbuf ', readbuf(i),         &
-     &          '  writebuf ', writebuf(i), '  i', i
-         end if
-      end do
-
-      call MPI_TYPE_FREE(newtype, ierr)
-      call MPI_Allreduce( errs, toterrs, 1, MPI_INTEGER, MPI_SUM,       &
-     $     MPI_COMM_WORLD, ierr )  
-      if (mynod .eq. 0) then
-        if( toterrs .gt. 0 ) then
-           print *, 'Found ', toterrs, ' errors'
-        else
-           print *, ' No Errors'
-        endif
-      endif
-
-      call MPI_FINALIZE(ierr)
-
-      end
diff --git a/3rd-party/romio321/test/file_info.c b/3rd-party/romio321/test/file_info.c
deleted file mode 100644
index fba618974fc..00000000000
--- a/3rd-party/romio321/test/file_info.c
+++ /dev/null
@@ -1,421 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*  
- *  (C) 2001 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-
-/* Change for BG/L made by Hao Yu, yuh@us.ibm.com
- */
-
-#include "mpi.h"
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-
-static void handle_error(int errcode, const char *str)
-{
-	char msg[MPI_MAX_ERROR_STRING];
-	int resultlen;
-	MPI_Error_string(errcode, msg, &resultlen);
-	fprintf(stderr, "%s: %s\n", str, msg);
-	MPI_Abort(MPI_COMM_WORLD, 1);
-}
-/* this test wants to compare the hints it gets from a file with a set of
- * default hints.  These hints are specific to the MPI-IO implementation, so if
- * you want to test something besides the default you'll have to use a command
- * line argument */
-
-typedef struct hint_defaults {
-    int cb_buffer_size;
-    int ind_rd_buffer_size;
-    int ind_wr_buffer_size;
-    const char *romio_cb_read;
-    const char *romio_cb_write;
-    const char *cb_config_list;
-} hint_defaults;
-
-hint_defaults UFS_DEFAULTS = {
-    .cb_buffer_size = 16777216,
-    .ind_rd_buffer_size = 4194304,
-    .ind_wr_buffer_size = 524288,
-    .romio_cb_read = "automatic",
-    .romio_cb_write = "automatic",
-    .cb_config_list = "*:1"
-};
-
-hint_defaults BLUEGENE_DEFAULTS = {
-    .cb_buffer_size = 16777216, 
-    .ind_rd_buffer_size = 4194304, 
-    .ind_wr_buffer_size = 4194304, 
-    .romio_cb_read = "enable", 
-    .romio_cb_write = "enable",
-    .cb_config_list = NULL};
-
-/* #undef INFO_DEBUG */
-
-/* Test will print out information about unexpected hint keys or values that
- * differ from the default.  Since this is often interesting but rarely an
- * error, default will be to increment errror cound for true error conditions
- * but not print out these "interesting" non-error cases. */
-
-static int verbose = 0;
-static int test_ufs = 0;
-static int test_bluegene = 0;
-
-int main(int argc, char **argv)
-{
-    int i, len, nkeys, flag, mynod, default_striping_factor=0, nprocs, errs = 0;
-    MPI_File fh;
-    MPI_Info info, info_used;
-    char *filename, key[MPI_MAX_INFO_KEY], value[MPI_MAX_INFO_VAL];
-    hint_defaults *defaults;
-    int ret;
-
-    MPI_Init(&argc,&argv);
-
-    MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
-    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
-
-/* process 0 takes the file name as a command-line argument and 
-   broadcasts it to other processes */
-    if (!mynod) {
-	i = 1;
-	while ((i < argc) && strcmp("-fname", *argv)) {
-	    if (!strcmp("-v", *argv)) verbose = 1;
-	    else if (!strcmp("-u", *argv)) test_ufs = 1;
-	    else if (!strcmp("-b", *argv)) test_bluegene = 1;
-	    i++;
-	    argv++;
-	}
-	if (i >= argc) {
-	    fprintf(stderr, "\n*#  Usage: file_info [-v] -fname filename\n\n");
-	    MPI_Abort(MPI_COMM_WORLD, 1);
-	}
-	argv++;
-	len = strlen(*argv);
-	filename = (char *) malloc(len+1);
-	strcpy(filename, *argv);
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
-	MPI_Bcast(&verbose, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	MPI_Bcast(&test_ufs, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	MPI_Bcast(&test_bluegene, 1, MPI_INT, 0, MPI_COMM_WORLD);
-    }
-    else {
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	filename = (char *) malloc(len+1);
-	MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
-	MPI_Bcast(&verbose, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	MPI_Bcast(&test_ufs, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	MPI_Bcast(&test_bluegene, 1, MPI_INT, 0, MPI_COMM_WORLD);
-    }
-    if (test_ufs) {
-	defaults = &UFS_DEFAULTS;
-    } else if (test_bluegene) {
-	defaults = &BLUEGENE_DEFAULTS;
-    } else {
-	defaults = NULL;
-    }
-
-
-/* open the file with MPI_INFO_NULL */
-    ret = MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, 
-                  MPI_INFO_NULL, &fh);
-    if (ret != MPI_SUCCESS) handle_error(ret, "MPI_File_open");
-
-/* check the default values set by ROMIO */
-    MPI_File_get_info(fh, &info_used);
-    MPI_Info_get_nkeys(info_used, &nkeys);
-
-    if (defaults != NULL) {
-	for (i=0; i<nkeys; i++) {
-	    MPI_Info_get_nthkey(info_used, i, key);
-	    MPI_Info_get(info_used, key, MPI_MAX_INFO_VAL-1, value, &flag);
-#ifdef INFO_DEBUG
-	    if (!mynod) 
-		fprintf(stderr, "Process %d, Default:  key = %s, value = %s\n", mynod, 
-			key, value);
-#endif
-	    if (!strcmp("striping_factor", key)) {
-		default_striping_factor = atoi(value);
-		/* no check */
-	    }
-	    else if (!strcmp("cb_buffer_size", key)) {
-		if (atoi(value) != defaults->cb_buffer_size) {
-		    errs++;
-		    if (verbose) fprintf(stderr, "cb_buffer_size is %d; should be %d\n",
-			    atoi(value), defaults->cb_buffer_size);
-		}
-	    }
-	    else if (!strcmp("romio_cb_read", key)) {
-		if (strcmp(defaults->romio_cb_read, value)) {
-		    errs++;
-		    if (verbose) fprintf(stderr, "romio_cb_read is set to %s; should be %s\n",
-			    value, defaults->romio_cb_read);
-		}
-	    }
-	    else if (!strcmp("romio_cb_write", key)) {
-		if (strcmp(defaults->romio_cb_write, value)) {
-		    errs++;
-		    if (verbose) fprintf(stderr, "romio_cb_write is set to %s; should be %s\n",
-			    value, defaults->romio_cb_write);
-		}
-	    }
-	    else if (!strcmp("cb_nodes", key)) {
-		/* unreliable test -- just ignore value */
-	    }
-	    else if (!strcmp("romio_no_indep_rw", key)) {
-		if (strcmp("false", value)) {
-		    errs++;
-		    if (verbose) fprintf(stderr, "romio_no_indep_rw is set to %s; should be %s\n",
-			    value, "false");
-		}
-	    }
-	    else if (!strcmp("ind_rd_buffer_size", key)) {
-		if (atoi(value) != defaults->ind_rd_buffer_size) {
-		    errs++;
-		    if (verbose) fprintf(stderr, "ind_rd_buffer_size is %d; should be %d\n",
-			    atoi(value), defaults->ind_rd_buffer_size);
-		}
-	    }
-	    else if (!strcmp("ind_wr_buffer_size", key)) {
-		if (atoi(value) != defaults->ind_wr_buffer_size) {
-		    errs++;
-		    if (verbose) fprintf(stderr, "ind_wr_buffer_size is %d; should be %d\n",
-			    atoi(value), defaults->ind_wr_buffer_size);
-		}
-	    }
-	    else if (!strcmp("romio_ds_read", key)) {
-		if (strcmp("automatic", value)) {
-		    errs++;
-		    if (verbose) fprintf(stderr, "romio_ds_read is set to %s; should be %s\n",
-			    value, "automatic");
-		}
-	    }
-	    else if (!strcmp("romio_ds_write", key)) {
-		/* Unreliable test -- value is file system dependent.  Ignore. */
-	    }
-	    else if (!strcmp("cb_config_list", key)) {
-#ifndef SKIP_CB_CONFIG_LIST_TEST
-		if (strcmp(defaults->cb_config_list, value)) {
-		    errs++;
-		    if (verbose) fprintf(stderr, "cb_config_list is set to %s; should be %s\n",
-			    value, defaults->cb_config_list);
-		}
-#endif
-	    }
-	    /* don't care about the defaults for these keys */
-	    else if (!strcmp("romio_cb_pfr", key)) {
-	    }
-	    else if (!strcmp("romio_cb_fr_types", key)) {
-	    }
-	    else if (!strcmp("romio_cb_fr_alignment", key)) {
-	    }
-	    else if (!strcmp("romio_cb_ds_threshold", key)) {
-	    }
-	    else if (!strcmp("romio_cb_alltoall", key)) {
-	    }
-	    else {
-		if (verbose) fprintf(stderr, "unexpected key %s (not counted as an error)\n", key);
-	    }
-	}
-    }
-    MPI_Info_free(&info_used);
-
-    MPI_File_close(&fh);
-
-    /* delete the file */
-    if (!mynod) MPI_File_delete(filename, MPI_INFO_NULL);
-    MPI_Barrier(MPI_COMM_WORLD);
-
-/* set new info values. */
-
-    MPI_Info_create(&info);
-
-/* The following four hints are accepted on all machines. They can
-   be specified at file-open time or later (any number of times). */
-
-    /* buffer size for collective I/O */
-    MPI_Info_set(info, "cb_buffer_size", "8388608");
-
-    /* number of processes that actually perform I/O in collective I/O */
-    sprintf(value, "%d", nprocs/2);
-    MPI_Info_set(info, "cb_nodes", value);
-
-    /* buffer size for data sieving in independent reads */
-    MPI_Info_set(info, "ind_rd_buffer_size", "2097152");
-
-    /* buffer size for data sieving in independent writes */
-    MPI_Info_set(info, "ind_wr_buffer_size", "1048576");
-
-
-/* The following three hints related to file striping are accepted only 
-   on Intel PFS and IBM PIOFS file systems and are ignored elsewhere. 
-   They can be specified only at file-creation time; if specified later 
-   they will be ignored. */
-
-    /* number of I/O devices across which the file will be striped.
-       accepted only if 0 < value < default_striping_factor; 
-       ignored otherwise */
-    if (default_striping_factor - 1 > 0) {
-        sprintf(value, "%d", default_striping_factor-1);
-        MPI_Info_set(info, "striping_factor", value);
-    }
-    else {
-        sprintf(value, "%d", default_striping_factor);
-        MPI_Info_set(info, "striping_factor", value);
-    }
-
-    /* the striping unit in bytes */
-    MPI_Info_set(info, "striping_unit", "131072");
-
-#ifndef SKIP_CB_CONFIG_LIST_TEST
-    /* set the cb_config_list so we'll get deterministic cb_nodes output */
-    MPI_Info_set(info, "cb_config_list", "*:*");
-#endif
-
-    /* the I/O device number from which to start striping the file.
-       accepted only if 0 <= value < default_striping_factor; 
-       ignored otherwise */
-    sprintf(value, "%d", default_striping_factor-2);
-    MPI_Info_set(info, "start_iodevice", value);
-
-
-/* The following hint about PFS server buffering is accepted only on 
-   Intel PFS. It can be specified anytime. */ 
-    MPI_Info_set(info, "pfs_svr_buf", "true");
-
-/* open the file and set new info */
-    ret = MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, 
-                  info, &fh);
-    if (ret != MPI_SUCCESS) handle_error(ret, "MPI_File_open");
-
-/* check the values set */
-    ret = MPI_File_get_info(fh, &info_used);
-    if (ret != MPI_SUCCESS) handle_error(ret, "MPI_File_get_info");
-    MPI_Info_get_nkeys(info_used, &nkeys);
-
-    for (i=0; i<nkeys; i++) {
-	MPI_Info_get_nthkey(info_used, i, key);
-	MPI_Info_get(info_used, key, MPI_MAX_INFO_VAL-1, value, &flag);
-#ifdef INFO_DEBUG	
-	if (!mynod) fprintf(stderr, "Process %d, key = %s, value = %s\n", mynod, 
-                key, value);
-#endif
-	if (!strcmp("striping_factor", key)) {
-	    if ((default_striping_factor - 1 > 0) && (atoi(value) != default_striping_factor-1)) {
-		errs++;
-		if (verbose) fprintf(stderr, "striping_factor is %d; should be %d\n",
-				     atoi(value), default_striping_factor-1);
-	    }
-	    else if (atoi(value) != default_striping_factor) {
-		errs++;
-		if (verbose) fprintf(stderr, "striping_factor is %d; should be %d\n",
-				     atoi(value), default_striping_factor);
-	    }
-	}
-	else if (!strcmp("cb_buffer_size", key)) {
-	    if (atoi(value) != 8388608) {
-		errs++;
-		if (verbose) fprintf(stderr, "cb_buffer_size is %d; should be %d\n",
-				     atoi(value), 8388608);
-	    }
-	}
-	/* only check the hints we set */
-	else if (!strcmp("cb_nodes", key)) {
-	    /* unreliable test: just skip */
-	}
-	else if (!strcmp("romio_no_indep_rw", key)) {
-	    if (strcmp("false", value)) {
-		errs++;
-		if (verbose) fprintf(stderr, "romio_no_indep_rw is set to %s; should be %s\n",
-				     value, "false");
-	    }
-	}
-	else if (!strcmp("ind_rd_buffer_size", key)) {
-	    if (atoi(value) != 2097152) {
-		errs++;
-		if (verbose) fprintf(stderr, "ind_rd_buffer_size is %d; should be %d\n",
-				     atoi(value), 2097152);
-	    }
-	}
-	else if (!strcmp("ind_wr_buffer_size", key)) {
-	    if (atoi(value) != 1048576) {
-		errs++;
-		if (verbose) fprintf(stderr, "ind_wr_buffer_size is %d; should be %d\n",
-				     atoi(value), 1048576);
-	    }
-	}
-	else if (!strcmp("romio_ds_read", key)) {
-	    if (strcmp("automatic", value)) {
-		errs++;
-		if (verbose) fprintf(stderr, "romio_ds_read is set to %s; should be %s\n",
-				     value, "automatic");
-	    }
-	}
-	else if (!strcmp("romio_ds_write", key)) {
-	    /* Unreliable test -- value is file system dependent.  Ignore. */
-	}
-	else if (!strcmp("cb_config_list", key)) {
-#ifndef SKIP_CB_CONFIG_LIST_TEST
-	    if (strcmp("*:*", value)) {
-		errs++;
-		if (verbose) fprintf(stderr, "cb_config_list is set to %s; should be %s\n",
-				     value, "*:*");
-	    }
-#endif
-	}
-	else if (!strcmp("romio_cb_pfr", key)) {
-   	    if(strcmp("disable", value)) {
-		errs++;
-		if (verbose) fprintf(stderr, "romio_cb_pfr is set to %s; should be %s\n",
-				     value, "automatic");
-	    }
-	}
-	else if (!strcmp("romio_cb_fr_types", key)) {
-   	    if(strcmp("aar", value)) {
-		errs++;
-		if (verbose) fprintf(stderr, "romio_cb_fr_types is set to %s; should be %s\n",
-				     value, "aar");
-	    }
-	}
-	else if (!strcmp("romio_cb_fr_alignment", key)) {
-   	    if(strcmp("1", value)) {
-		errs++;
-		if (verbose) fprintf(stderr, "romio_cb_fr_alignment is set to %s; should be %s\n",
-				     value, "1");
-	    }
-	}
-	else if (!strcmp("romio_cb_ds_threshold", key)) {
-   	    if(strcmp("0", value)) {
-		errs++;
-		if (verbose) fprintf(stderr, "romio_cb_ds_threshold is set to %s; should be %s\n",
-				     value, "0");
-	    }
-	}
-	else if (!strcmp("romio_cb_alltoall", key)) {
-   	    if(strcmp("automatic", value)) {
-		errs++;
-		if (verbose) fprintf(stderr, "romio_cb_alltoall is set to %s; should be %s\n",
-				     value, "automatic");
-	    }
-	}
-
-	else {
-	    if (verbose) fprintf(stderr, "unexpected key %s (not counted as an error)\n", key);
-	}
-    }
-	    
-    /* Q: SHOULD WE BOTHER LOOKING AT THE OTHER PROCESSES? */
-    if (!mynod) {
-	if (errs) fprintf(stderr, "Found %d errors.\n", errs);
-	else printf(" No Errors\n");
-    }
-    
-    MPI_File_close(&fh);
-    free(filename);
-    MPI_Info_free(&info_used);
-    MPI_Info_free(&info);
-    MPI_Finalize();
-    return 0;
-}
diff --git a/3rd-party/romio321/test/fmisc.f.in b/3rd-party/romio321/test/fmisc.f.in
deleted file mode 100644
index a75c4ee16ed..00000000000
--- a/3rd-party/romio321/test/fmisc.f.in
+++ /dev/null
@@ -1,204 +0,0 @@
-! -*- Mode: Fortran; -*- 
-!  
-!  (C) 2001 by Argonne National Laboratory.
-!      See COPYRIGHT in top-level directory.
-!
-      program main
-      implicit none
-
-      include 'mpif.h'
-      @F77MPIOINC@
-
-!     Fortran equivalent of misc.c
-!     tests various miscellaneous functions.
-
-      integer buf(1024), amode, fh, status(MPI_STATUS_SIZE)
-      logical flag
-      integer ierr, newtype, i, group
-      integer etype, filetype, mynod, argc, iargc
-      integer errs, toterrs
-      logical verbose
-      character*7 datarep
-      character*1024 str    ! used to store the filename
-      @FORTRAN_MPI_OFFSET@ disp, offset, filesize
-      @FTESTDEFINE@
-
-      errs = 0
-      verbose = .false.
-      call MPI_INIT(ierr)
-      call MPI_COMM_RANK(MPI_COMM_WORLD, mynod, ierr)
-
-!     process 0 takes the file name as a command-line argument and 
-!     broadcasts it to other processes
-
-      if (mynod .eq. 0) then
-         argc = @F77IARGC@
-         i = 0
-         @F77GETARG@
-         do while ((i .lt. argc) .and. (str .ne. '-fname'))
-            i = i + 1
-            @F77GETARG@
-         end do
-         if (i .ge. argc) then
-            print *
-            print *, '*#  Usage: fmisc -fname filename'
-            print *
-            call MPI_ABORT(MPI_COMM_WORLD, 1, ierr)
-         end if
-
-         i = i + 1
-         @F77GETARG@
-         call MPI_BCAST(str, 1024, MPI_CHARACTER, 0,                    &
-     &        MPI_COMM_WORLD, ierr)
-      else 
-         call MPI_BCAST(str, 1024, MPI_CHARACTER, 0,                    &
-     &        MPI_COMM_WORLD, ierr)
-      end if
-
-
-      call MPI_FILE_OPEN(MPI_COMM_WORLD, str,                           &
-     &     MPI_MODE_CREATE + MPI_MODE_RDWR, MPI_INFO_NULL, fh, ierr)
-
-      call MPI_FILE_WRITE(fh, buf, 1024, MPI_INTEGER, status, ierr)
-
-      call MPI_FILE_SYNC(fh, ierr)
-
-      call MPI_FILE_GET_AMODE(fh, amode, ierr)
-      if (mynod .eq. 0 .and. verbose) then
-         print *, ' testing MPI_FILE_GET_AMODE'
-      end if
-      if (amode .ne. (MPI_MODE_CREATE + MPI_MODE_RDWR)) then
-         errs = errs + 1
-         print *, 'amode is ', amode, ', should be ', MPI_MODE_CREATE   &
-     &           + MPI_MODE_RDWR
-      end if
-
-      call MPI_FILE_GET_ATOMICITY(fh, flag, ierr)
-      if (flag) then
-         errs = errs + 1
-         print *, 'atomicity is ', flag, ', should be .FALSE.'
-      end if
-      if (mynod .eq. 0 .and. verbose) then
-         print *, ' setting atomic mode'
-      end if
-      call MPI_FILE_SET_ATOMICITY(fh, .TRUE., ierr)
-      call MPI_FILE_GET_ATOMICITY(fh, flag, ierr)
-      if (.not. flag) then
-         errs = errs + 1
-         print *, 'atomicity is ', flag, ', should be .TRUE.'
-      end if
-      call MPI_FILE_SET_ATOMICITY(fh, .FALSE., ierr)
-      if (mynod .eq. 0 .and. verbose) then
-         print *, ' reverting back to nonatomic mode'
-      end if
-
-      call MPI_TYPE_VECTOR(10, 10, 20, MPI_INTEGER, newtype, ierr)
-      call MPI_TYPE_COMMIT(newtype, ierr)
-
-      disp = 1000
-      call MPI_FILE_SET_VIEW(fh, disp, MPI_INTEGER, newtype, 'native',  & 
-     &     MPI_INFO_NULL, ierr)
-      if (mynod .eq. 0 .and. verbose) then
-         print *, ' testing MPI_FILE_GET_VIEW'
-      end if
-
-      disp = 0
-      call MPI_FILE_GET_VIEW(fh, disp, etype, filetype, datarep, ierr)
-      if ((disp .ne. 1000) .or. (datarep .ne. 'native')) then
-         errs = errs + 1
-         print *, 'disp = ', disp, ', datarep = ', datarep,             &
-     &     ', should be 1000, native'
-      end if
-
-      if (mynod .eq. 0 .and. verbose) then
-         print *, ' testing MPI_FILE_GET_BYTE_OFFSET'
-      end if
-      offset = 10
-      call MPI_FILE_GET_BYTE_OFFSET(fh, offset, disp, ierr)
-      if (disp .ne. 1080) then 
-         errs = errs + 1
-         print *, 'byte offset = ', disp, ', should be 1080'
-      end if
-
-      call MPI_FILE_GET_GROUP(fh, group, ierr)
-
-      if (mynod .eq. 0 .and. verbose) then
-         print *, ' setting file size to 1060 bytes'
-      end if
-      filesize = 1060
-      call MPI_FILE_SET_SIZE(fh, filesize, ierr)
-      call MPI_BARRIER(MPI_COMM_WORLD, ierr)
-      call MPI_FILE_SYNC(fh, ierr)
-      filesize = 0
-      call MPI_FILE_GET_SIZE(fh, filesize, ierr)
-      if (filesize .ne. 1060) then
-         errs = errs + 1
-         print *, 'file size = ', filesize, ', should be 1060'
-      end if
- 
-      if (mynod .eq. 0 .and. verbose) then
-         print *, ' seeking to eof and testing MPI_FILE_GET_POSITION'
-      end if
-      offset = 0
-      call MPI_FILE_SEEK(fh, offset, MPI_SEEK_END, ierr)
-      call MPI_FILE_GET_POSITION(fh, offset, ierr)
-      if (offset .ne. 10) then 
-         errs = errs + 1
-         print *, 'file pointer posn = ', offset, ', should be 10'
-      end if
-
-      if (mynod .eq. 0 .and. verbose) then
-         print *, ' testing MPI_FILE_GET_BYTE_OFFSET'
-      end if
-      call MPI_FILE_GET_BYTE_OFFSET(fh, offset, disp, ierr)
-      if (disp .ne. 1080) then
-         errs = errs + 1
-         print *, 'byte offset = ', disp, ', should be 1080'
-      end if
-      call MPI_BARRIER(MPI_COMM_WORLD, ierr)
-
-      if (mynod .eq. 0 .and. verbose) then
-         print *, ' testing MPI_FILE_SEEK with MPI_SEEK_CUR'
-      end if
-      offset = -10
-      call MPI_FILE_SEEK(fh, offset, MPI_SEEK_CUR, ierr)
-      call MPI_FILE_GET_POSITION(fh, offset, ierr)
-      call MPI_FILE_GET_BYTE_OFFSET(fh, offset, disp, ierr)
-      if (disp .ne. 1000) then 
-         errs = errs + 1
-         print *, 'file pointer posn in bytes = ', disp,                &
-     &     ', should be 1000'
-      end if
-
-      if (mynod .eq. 0 .and. verbose) then
-         print *, ' preallocating disk space up to 8192 bytes'
-      end if
-      filesize = 8192
-      call MPI_FILE_PREALLOCATE(fh, filesize, ierr)
-
-      if (mynod .eq. 0 .and. verbose) then
-         print *, ' closing the file and deleting it'
-      end if
-      call MPI_FILE_CLOSE(fh, ierr)
-
-      call MPI_BARRIER(MPI_COMM_WORLD, ierr)
-      if (mynod .eq. 0) then
-         call MPI_FILE_DELETE(str, MPI_INFO_NULL, ierr)
-      end if
-
-      call MPI_Allreduce( errs, toterrs, 1, MPI_INTEGER, MPI_SUM,       &
-     $     MPI_COMM_WORLD, ierr )  
-      if (mynod .eq. 0) then
-        if( toterrs .gt. 0 ) then
-           print *, 'Found ', toterrs, ' errors'
-        else
-           print *, ' No Errors'
-        endif
-      endif
-
-      call MPI_TYPE_FREE(newtype, ierr)    
-      call MPI_TYPE_FREE(filetype, ierr)    
-      call MPI_GROUP_FREE(group, ierr)
-      call MPI_FINALIZE(ierr)
-
-      end
diff --git a/3rd-party/romio321/test/fperf.f.in b/3rd-party/romio321/test/fperf.f.in
deleted file mode 100644
index 627794955cf..00000000000
--- a/3rd-party/romio321/test/fperf.f.in
+++ /dev/null
@@ -1,179 +0,0 @@
-! -*- Mode: Fortran; -*- 
-!  
-!  (C) 2001 by Argonne National Laboratory.
-!      See COPYRIGHT in top-level directory.
-!
-      program main
-      implicit none
-
-      include 'mpif.h'
-      @F77MPIOINC@
-
-!     Fortran equivalent of perf.c
-
-      integer SIZE 
-      parameter (SIZE=1048576*4)
-!     read/write size per node in bytes
-
-      integer buf(SIZE/4), j, mynod, nprocs, ntimes, flag
-      double precision stim, read_tim, write_tim, new_read_tim
-      double precision new_write_tim, min_read_tim, min_write_tim
-      double precision read_bw, write_bw
-      integer fh, status(MPI_STATUS_SIZE), ierr, argc, iargc, i
-      character*1024 str    ! used to store the filename
-      @FORTRAN_MPI_OFFSET@ offset
-      @FTESTDEFINE@
-
-      ntimes = 5
-      min_read_tim = 10000000.0D0
-      min_write_tim = 10000000.0D0
-
-      call MPI_INIT(ierr)
-      call MPI_COMM_SIZE(MPI_COMM_WORLD, nprocs, ierr)
-      call MPI_COMM_RANK(MPI_COMM_WORLD, mynod, ierr)
-
-!     process 0 takes the file name as a command-line argument and 
-!     broadcasts it to other processes
-
-      if (mynod .eq. 0) then
-         argc = @F77IARGC@
-         i = 0
-         @F77GETARG@
-         do while ((i .lt. argc) .and. (str .ne. '-fname'))
-            i = i + 1
-            @F77GETARG@
-         end do
-         if (i .ge. argc) then
-            print *
-            print *, '*#  Usage: fperf -fname filename'
-            print *
-            call MPI_ABORT(MPI_COMM_WORLD, 1, ierr)
-         end if
-
-         i = i + 1
-         @F77GETARG@
-         call MPI_BCAST(str, 1024, MPI_CHARACTER, 0,                    &  
-     &        MPI_COMM_WORLD, ierr)
-         print *, 'Access size per process = ', SIZE, ' bytes',         &
-     &        ', ntimes = ', ntimes
-      else 
-         call MPI_BCAST(str, 1024, MPI_CHARACTER, 0,                    &
-     &        MPI_COMM_WORLD, ierr)
-      end if
-
-
-      offset = mynod*SIZE
-      do j=1, ntimes
-         call MPI_FILE_OPEN(MPI_COMM_WORLD, str,                        &
-     &        MPI_MODE_CREATE+MPI_MODE_RDWR, MPI_INFO_NULL, fh, ierr)
-
-         call MPI_FILE_SEEK(fh, offset, MPI_SEEK_SET, ierr)
-
-         call MPI_BARRIER(MPI_COMM_WORLD, ierr)
-         stim = MPI_WTIME()
-         call MPI_FILE_WRITE(fh, buf, SIZE, MPI_BYTE, status, ierr)
-         write_tim = MPI_WTIME() - stim
-  
-         call MPI_FILE_CLOSE(fh, ierr)
-
-         call MPI_BARRIER(MPI_COMM_WORLD, ierr)
-
-         call MPI_FILE_OPEN(MPI_COMM_WORLD, str,                        &
-     &        MPI_MODE_CREATE+MPI_MODE_RDWR, MPI_INFO_NULL, fh, ierr)
-
-         call MPI_FILE_SEEK(fh, offset, MPI_SEEK_SET, ierr)
-
-         call MPI_BARRIER(MPI_COMM_WORLD, ierr)
-         stim = MPI_WTIME()
-         call MPI_FILE_READ(fh, buf, SIZE, MPI_BYTE, status, ierr)
-         read_tim = MPI_WTIME() - stim
-  
-         call MPI_FILE_CLOSE(fh, ierr)
-
-         call MPI_ALLREDUCE(write_tim, new_write_tim, 1,                &
-     &        MPI_DOUBLE_PRECISION, MPI_MAX, MPI_COMM_WORLD, ierr)
-         call MPI_ALLREDUCE(read_tim, new_read_tim, 1,                  &
-     &        MPI_DOUBLE_PRECISION, MPI_MAX, MPI_COMM_WORLD, ierr)
-
-         if (new_read_tim .lt. min_read_tim) then
-            min_read_tim = new_read_tim
-         end if
-         if (new_write_tim .lt. min_write_tim) then
-            min_write_tim = new_write_tim
-         end if
-      end do
-    
-      if (mynod .eq. 0) then
-         read_bw = (SIZE*nprocs*1.0D0)/(min_read_tim*1000000.0D0)
-         write_bw = (SIZE*nprocs*1.0D0)/(min_write_tim*1000000.0D0)
-         print *, 'Write bandwidth without file sync = ',               &
-     &        write_bw, ' Mbytes/sec'
-         print *, 'Read bandwidth without prior file sync = ',          &
-     &        read_bw, ' Mbytes/sec'
-      end if 
-
-      min_read_tim = 10000000.0D0
-      min_write_tim = 10000000.0D0
-
-      flag = 0
-      do j=1, ntimes
-         call MPI_FILE_OPEN(MPI_COMM_WORLD, str,                        & 
-     &        MPI_MODE_CREATE+MPI_MODE_RDWR, MPI_INFO_NULL, fh, ierr)
-
-         call MPI_FILE_SEEK(fh, offset, MPI_SEEK_SET, ierr)
-
-         call MPI_BARRIER(MPI_COMM_WORLD, ierr)
-         stim = MPI_WTIME()
-         call MPI_FILE_WRITE(fh, buf, SIZE, MPI_BYTE, status, ierr)
-         call MPI_FILE_SYNC(fh, ierr)
-         write_tim = MPI_WTIME() - stim
-         if (ierr .eq. MPI_ERR_UNKNOWN) then 
-            flag = 1
-         end if
-  
-         call MPI_FILE_CLOSE(fh, ierr)
-
-         call MPI_BARRIER(MPI_COMM_WORLD, ierr)
-
-         call MPI_FILE_OPEN(MPI_COMM_WORLD, str,                        & 
-     &        MPI_MODE_CREATE+MPI_MODE_RDWR, MPI_INFO_NULL, fh, ierr)
-
-         call MPI_FILE_SEEK(fh, offset, MPI_SEEK_SET, ierr)
-
-         call MPI_BARRIER(MPI_COMM_WORLD, ierr)
-         stim = MPI_WTIME()
-         call MPI_FILE_READ(fh, buf, SIZE, MPI_BYTE, status, ierr)
-         read_tim = MPI_WTIME() - stim
-  
-         call MPI_FILE_CLOSE(fh, ierr)
-
-         call MPI_ALLREDUCE(write_tim, new_write_tim, 1,                &
-     &        MPI_DOUBLE_PRECISION, MPI_MAX, MPI_COMM_WORLD, ierr)
-         call MPI_ALLREDUCE(read_tim, new_read_tim, 1,                  &
-     &        MPI_DOUBLE_PRECISION, MPI_MAX, MPI_COMM_WORLD, ierr)
-
-         if (new_read_tim .lt. min_read_tim) then
-            min_read_tim = new_read_tim
-         end if
-         if (new_write_tim .lt. min_write_tim) then
-            min_write_tim = new_write_tim
-         end if
-
-      end do
-    
-      if (mynod .eq. 0) then
-         if (flag .eq. 1) then
-            print *, 'MPI_FILE_SYNC returns error.'
-         else
-            read_bw = (SIZE*nprocs*1.0D0)/(min_read_tim*1000000.0D0)
-            write_bw = (SIZE*nprocs*1.0D0)/(min_write_tim*1000000.0D0)
-            print *, 'Write bandwidth including file sync = ',          & 
-     &           write_bw, ' Mbytes/sec'
-            print *, 'Read bandwidth after file sync = ',               &
-     &           read_bw, ' Mbytes/sec'
-         end if 
-      end if
-
-      call MPI_FINALIZE(ierr)
-
-      end
diff --git a/3rd-party/romio321/test/hindexed.c b/3rd-party/romio321/test/hindexed.c
deleted file mode 100644
index ebc0ae81c65..00000000000
--- a/3rd-party/romio321/test/hindexed.c
+++ /dev/null
@@ -1,260 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *
- *  (C) 2008 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-/* Wei-keng Liao (wkliao@ece.northwestern.edu) September 8, 2008 */
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <mpi.h>
-
-#define YLEN 5
-#define XLEN 10
-#define SUB_XLEN 3
-
-/* rjl: I was just too lazy to compute this at run-time */
-char compare_buf[XLEN*4][YLEN*4] = {
-	{'0','1','2',0,0,'3','4','5',0,0,'D','E','F',0,0,'G','H','I'},
-	{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
-	{'6','7','8',0,0,'9',':',';',0,0,'J','K','L',0,0,'M','N','O'},
-	{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
-	{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
-	{'X','Y','Z',0,0,'[','\\',']',0,0,'l','m','n',0,0,'o','p','q'},
-	{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
-	{'^','_','`',0,0,'a','b','c',0,0,'r','s','t',0,0,'u','v','w'},
-	{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
-	{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
-	{'0','1','2',0,0,'3','4','5',0,0,'D','E','F',0,0,'G','H','I'},
-	{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
-	{'6','7','8',0,0,'9',':',';',0,0,'J','K','L',0,0,'M','N','O'},
-	{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
-	{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
-	{'X','Y','Z',0,0,'[','\\',']',0,0,'l','m','n',0,0,'o','p','q'},
-	{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
-	{'^','_','`',0,0,'a','b','c',0,0,'r','s','t',0,0,'u','v','w'},
-	{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
-	{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}
-};
-
-
-/* set this if you want a dump of the global array 
-#define VERBOSE 1
-*/
-
-/*----< main() >------------------------------------------------------------*/
-int main(int argc, char **argv) {
-    int          i, j, err, rank, np, num_io;
-    char        *buf, *filename;
-    int          rank_dim[2], array_of_sizes[2];
-    int          array_of_subsizes[2];
-    int          count, *blocklengths, global_array_size;
-    MPI_Count    ftype_size;
-    MPI_Aint    *displacements;
-    MPI_File     fh;
-    MPI_Datatype ftype;
-    MPI_Status   status;
-    MPI_Offset   offset=0;
-    int          nr_errors=0;
-#ifdef VERBOSE
-    int k;
-#endif
-
-    MPI_Init(&argc, &argv);
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-    MPI_Comm_size(MPI_COMM_WORLD, &np);
-
-    if (np != 4) {
-        if (!rank) printf("Please run with 4 processes. Exiting ...\n\n");
-        MPI_Finalize();
-        return 1;
-    }
-
-    filename = argv[1];
-
-    num_io = 2;
-
-    /*-----------------------------------------------------------------------*/
-    /* process rank in each dimension */
-    rank_dim[0] = rank /  2;
-    rank_dim[1] = rank %  2;
-
-    /* global 2D array size */
-    array_of_sizes[0] = YLEN * 2;
-    array_of_sizes[1] = XLEN * 2;
-
-    global_array_size = array_of_sizes[0] * array_of_sizes[1];
-
-    array_of_subsizes[0] = YLEN / 2;
-    array_of_subsizes[1] = XLEN * SUB_XLEN / 5;
-
-    offset = rank_dim[0] * YLEN * array_of_sizes[1] +
-             rank_dim[1] * XLEN;
-
-    /* define data type for file view */
-    count = array_of_subsizes[0] * 2;  /* 2 is the no. blocks along X */
-    blocklengths  = (int*)     malloc(count*sizeof(int));
-    displacements = (MPI_Aint*)malloc(count*sizeof(MPI_Aint));
-    for (i=0; i<count; i++)
-        blocklengths[i] = array_of_subsizes[1] / 2;
-    for (i=0; i<array_of_subsizes[0]; i++)
-        for (j=0; j<2; j++)
-            displacements[i*2+j] = offset + i*2*array_of_sizes[1] + j * XLEN/2;
-    MPI_Type_create_hindexed(count, blocklengths, displacements, MPI_CHAR, &ftype);
-    MPI_Type_commit(&ftype);
-    MPI_Type_size_x(ftype, &ftype_size);
-
-/* subarray's layout in the global array
-
-   P0's 's layout                               P1's layout
-   [ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9] | [ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9]
-[ 0] 0 1 2     3 4 5                          |                       D E F     G H I    
-[ 1]                                          |                                          
-[ 2] 6 7 8     9 : ;                          |                       J K L     M N O    
-[ 3]                                          |                                          
-[ 4]                                          |                                          
-[ 5]                                          |                                          
-[ 6]                                          |                                          
-[ 7]                                          |                                          
-[ 8]                                          |                                          
-[ 9]                                          |                                          
-
-   P2's 's layout                               P3's layout
-   [ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9] | [ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9]
-[ 0]                                          |                                          
-[ 1]                                          |                                          
-[ 2]                                          |                                          
-[ 3]                                          |                                          
-[ 4]                                          |                                          
-[ 5] X Y Z     [ \ ]                          |                       l m n     o p q    
-[ 6]                                          |                                          
-[ 7] ^ _ `     a b c                          |                       r s t     u v w    
-[ 8]                                          |                                          
-[ 9]                                          |                                          
-*/
-
-    /* initialize the write buffer */
-    buf = (char*) malloc(array_of_subsizes[0]*array_of_subsizes[1]);
-    for (i=0; i<array_of_subsizes[0]*array_of_subsizes[1]; i++)
-        buf[i] = '0' + rank*20  + i%79;
-
-    /* zero file contents ---------------------------------------------------*/
-    if (rank == 0) {
-        char *wr_buf = (char*) calloc(num_io*global_array_size,1);
-        MPI_File_open(MPI_COMM_SELF, filename,
-                      MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
-        MPI_File_write(fh, wr_buf, num_io*global_array_size, MPI_CHAR, &status);
-        MPI_File_close(&fh);
-        free(wr_buf);
-    }
-    /* open the file --------------------------------------------------------*/
-    err = MPI_File_open(MPI_COMM_WORLD, filename,
-                        MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
-    if (err != MPI_SUCCESS) {
-        printf("Error: MPI_File_open() filename %s\n",filename);
-	MPI_Abort(MPI_COMM_WORLD, -1);
-	exit(1);
-    }
-
-    /* MPI collective write */
-    for (i=0; i<num_io; i++) {
-        offset = i * global_array_size;
-        /* set the file view */
-        MPI_File_set_view(fh, offset, MPI_BYTE, ftype, "native", MPI_INFO_NULL);
-        MPI_File_write_all(fh, buf, ftype_size, MPI_CHAR, &status);
-    }
-    MPI_File_close(&fh);
-
-    /* read and print file contents -----------------------------------------*/
-    if (rank == 0) {
-        char *ptr;
-        char *rd_buf = (char*) calloc(num_io*global_array_size,1);
-        MPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_RDONLY, MPI_INFO_NULL, &fh);
-        MPI_File_read(fh, rd_buf, num_io*global_array_size, MPI_CHAR, &status);
-        MPI_File_close(&fh);
-
-#ifdef VERBOSE
-        printf("-------------------------------------------------------\n");
-        printf("   [");
-        for (i=0; i<2; i++) {
-            for (j=0; j<XLEN; j++) printf(" %d",j);
-            printf(" ");
-        }
-        printf("]\n\n");
-	
-
-        ptr = rd_buf;
-        for (k=0; k<num_io; k++) {
-            for (i=0; i<2*YLEN; i++) {
-                printf("[%2d]",k*2*YLEN+i);
-                for (j=0; j<2*XLEN; j++) {
-                    if (j>0 && j%XLEN==0) printf(" ");
-                    if (*ptr != 0)
-                        printf(" %c",*ptr);
-                    else
-                        printf("  ");
-                    ptr++;
-                }
-                printf("\n");
-            }
-            printf("\n");
-        }
-#endif
-	ptr = rd_buf;
-        for(i=0; i<2*YLEN*num_io; i++) {
-	    for(j=0; j<2*XLEN; j++) {
-		if( *ptr != compare_buf[i][j]) {
-			fprintf(stderr, "expected %d got %d at [%d][%d]\n", 
-					*ptr, compare_buf[i][j], i, j);
-			nr_errors++;
-		}
-		ptr++;
-	    }
-	}
-        free(rd_buf);
-
-        if (nr_errors == 0) 
-	    fprintf(stdout, " No Errors\n");
-        else
-	    fprintf(stderr, "Found %d errors\n", nr_errors);
-    }
-
-    free(blocklengths);
-    free(displacements);
-    free(buf);
-    MPI_Type_free(&ftype);
-    MPI_Finalize();
-    return 0;
-}
-
-/* command-line outputs are: (the global array is written twice)
-
-% mpiexec -n 4 wkl_subarray
--------------------------------------------------------
-   [ 0 1 2 3 4 5 6 7 8 9  0 1 2 3 4 5 6 7 8 9 ]
-
-[ 0] 0 1 2     3 4 5      D E F     G H I    
-[ 1]                                         
-[ 2] 6 7 8     9 : ;      J K L     M N O    
-[ 3]                                         
-[ 4]                                         
-[ 5] X Y Z     [ \ ]      l m n     o p q    
-[ 6]                                         
-[ 7] ^ _ `     a b c      r s t     u v w    
-[ 8]                                         
-[ 9]                                         
-
-[10] 0 1 2     3 4 5      D E F     G H I    
-[11]                                         
-[12] 6 7 8     9 : ;      J K L     M N O    
-[13]                                         
-[14]                                         
-[15] X Y Z     [ \ ]      l m n     o p q    
-[16]                                         
-[17] ^ _ `     a b c      r s t     u v w    
-[18]                                         
-[19]                                         
-
-*/
-
diff --git a/3rd-party/romio321/test/i_noncontig.c b/3rd-party/romio321/test/i_noncontig.c
deleted file mode 100644
index db5bd36f0f9..00000000000
--- a/3rd-party/romio321/test/i_noncontig.c
+++ /dev/null
@@ -1,267 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*  
- *  (C) 2001 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include "mpi.h"
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-
-/* tests noncontiguous reads/writes using nonblocking I/O */
-
-#define SIZE 5000
-
-#define VERBOSE 0
-
-int main(int argc, char **argv)
-{
-    int *buf, i, mynod, nprocs, len, b[3];
-    int errs=0, toterrs;
-    MPI_Aint d[3];
-    MPI_File fh;
-    MPI_Status status;
-    char *filename;
-    MPI_Datatype typevec, newtype, t[3];
-    MPIO_Request req;
-
-    MPI_Init(&argc,&argv);
-    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
-    MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
-
-    if (nprocs != 2) {
-        fprintf(stderr, "Run this program on two processes\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-/* process 0 takes the file name as a command-line argument and 
-   broadcasts it to other processes */
-    if (!mynod) {
-	i = 1;
-	while ((i < argc) && strcmp("-fname", *argv)) {
-	    i++;
-	    argv++;
-	}
-	if (i >= argc) {
-	    fprintf(stderr, "\n*#  Usage: i_noncontig -fname filename\n\n");
-	    MPI_Abort(MPI_COMM_WORLD, 1);
-	}
-	argv++;
-	len = strlen(*argv);
-	filename = (char *) malloc(len+1);
-	strcpy(filename, *argv);
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-    else {
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	filename = (char *) malloc(len+1);
-	MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-
-    buf = (int *) malloc(SIZE*sizeof(int));
-
-    MPI_Type_vector(SIZE/2, 1, 2, MPI_INT, &typevec);
-
-    b[0] = b[1] = b[2] = 1;
-    d[0] = 0;
-    d[1] = mynod*sizeof(int);
-    d[2] = SIZE*sizeof(int);
-    t[0] = MPI_LB;
-    t[1] = typevec;
-    t[2] = MPI_UB;
-
-    MPI_Type_struct(3, b, d, t, &newtype);
-    MPI_Type_commit(&newtype);
-    MPI_Type_free(&typevec);
-
-    if (!mynod) {
-#if VERBOSE
-	fprintf(stderr, "\ntesting noncontiguous in memory, noncontiguous in file using nonblocking I/O\n");
-#endif
-	MPI_File_delete(filename, MPI_INFO_NULL);
-    }
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | 
-             MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
-
-    MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", MPI_INFO_NULL);
-
-    for (i=0; i<SIZE; i++) buf[i] = i + mynod*SIZE;
-    MPI_File_iwrite(fh, buf, 1, newtype, &req);
-#ifdef MPIO_USES_MPI_REQUEST
-    MPI_Wait(&req, &status);
-#else
-    MPIO_Wait(&req, &status);
-#endif
-
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    for (i=0; i<SIZE; i++) buf[i] = -1;
-
-    MPI_File_iread_at(fh, 0, buf, 1, newtype, &req);
-#ifdef MPIO_USES_MPI_REQUEST
-    MPI_Wait(&req, &status);
-#else
-    MPIO_Wait(&req, &status);
-#endif
-
-    for (i=0; i<SIZE; i++) {
-	if (!mynod) {
-	    if ((i%2) && (buf[i] != -1)) {
-		errs++;
-		fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", 
-			mynod, i, buf[i]);
-	    }
-	    if (!(i%2) && (buf[i] != i)) {
-		errs++;
-		fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", 
-			mynod, i, buf[i], i);
-	    }
-	}
-	else {
-	    if ((i%2) && (buf[i] != i + mynod*SIZE)) {
-		errs++;
-		fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", 
-			mynod, i, buf[i], i + mynod*SIZE);
-	    }
-	    if (!(i%2) && (buf[i] != -1)) {
-		errs++;
-		fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", 
-			mynod, i, buf[i]);
-	    }
-	}
-    }
-
-    MPI_File_close(&fh);
-
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    if (!mynod) {
-#if VERBOSE
-	fprintf(stderr, "\ntesting noncontiguous in memory, contiguous in file using nonblocking I/O\n");
-#endif
-	MPI_File_delete(filename, MPI_INFO_NULL);
-    }
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | 
-             MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
-
-    for (i=0; i<SIZE; i++) buf[i] = i + mynod*SIZE;
-    MPI_File_iwrite_at(fh, mynod*(SIZE/2)*sizeof(int), buf, 1, newtype, &req);
-#ifdef MPIO_USES_MPI_REQUEST
-    MPI_Wait(&req, &status);
-#else
-    MPIO_Wait(&req, &status);
-#endif
-
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    for (i=0; i<SIZE; i++) buf[i] = -1;
-
-    MPI_File_iread_at(fh, mynod*(SIZE/2)*sizeof(int), buf, 1, newtype, &req);
-#ifdef MPIO_USES_MPI_REQUEST
-    MPI_Wait(&req, &status);
-#else
-    MPIO_Wait(&req, &status);
-#endif
-
-    for (i=0; i<SIZE; i++) {
-	if (!mynod) {
-	    if ((i%2) && (buf[i] != -1)) {
-		errs++;
-		fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", 
-			mynod, i, buf[i]);
-	    }
-	    if (!(i%2) && (buf[i] != i)) {
-		errs++;
-		fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
-			mynod, i, buf[i], i);
-	    }
-	}
-	else {
-	    if ((i%2) && (buf[i] != i + mynod*SIZE)) {
-		errs++;
-		fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", 
-			mynod, i, buf[i], i + mynod*SIZE);
-	    }
-	    if (!(i%2) && (buf[i] != -1)) {
-		errs++;
-		fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", 
-			mynod, i, buf[i]);
-	    }
-	}
-    }
-
-    MPI_File_close(&fh);
-
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    if (!mynod) {
-#if VERBOSE
-	fprintf(stderr, "\ntesting contiguous in memory, noncontiguous in file using nonblocking I/O\n");
-#endif
-	MPI_File_delete(filename, MPI_INFO_NULL);
-    }
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | 
-             MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
-
-    MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", MPI_INFO_NULL);
-
-    for (i=0; i<SIZE; i++) buf[i] = i + mynod*SIZE;
-    MPI_File_iwrite(fh, buf, SIZE, MPI_INT, &req);
-#ifdef MPIO_USES_MPI_REQUEST
-    MPI_Wait(&req, &status);
-#else
-    MPIO_Wait(&req, &status);
-#endif
-
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    for (i=0; i<SIZE; i++) buf[i] = -1;
-
-    MPI_File_iread_at(fh, 0, buf, SIZE, MPI_INT, &req);
-#ifdef MPIO_USES_MPI_REQUEST
-    MPI_Wait(&req, &status);
-#else
-    MPIO_Wait(&req, &status);
-#endif
-
-    for (i=0; i<SIZE; i++) {
-	if (!mynod) {
-	    if (buf[i] != i) {
-		errs++;
-		fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", 
-			mynod, i, buf[i], i);
-	    }
-	}
-	else {
-	    if (buf[i] != i + mynod*SIZE) {
-		errs++;
-		fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", 
-			mynod, i, buf[i], i + mynod*SIZE);
-	    }
-	}
-    }
-
-    MPI_File_close(&fh);
-
-    MPI_Allreduce( &errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
-    if (mynod == 0) {
-	if( toterrs > 0) {
-	    fprintf( stderr, "Found %d errors\n", toterrs );
-	}
-	else {
-	    fprintf( stdout, " No Errors\n" );
-	}
-    }
-    MPI_Type_free(&newtype);
-    free(buf);
-    free(filename);
-    MPI_Finalize();
-    return 0;
-}
diff --git a/3rd-party/romio321/test/large_array.c b/3rd-party/romio321/test/large_array.c
deleted file mode 100644
index 617c895dc9b..00000000000
--- a/3rd-party/romio321/test/large_array.c
+++ /dev/null
@@ -1,145 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*  
- *  (C) 2001 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include "mpi.h"
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-
-/* Writes a 4-Gbyte distributed array, reads it back, and then deletes the 
-   file. Uses collective I/O. */
-/* The file name is taken as a command-line argument. */
-/* Run it only on a machine with sufficient memory and a file system
-   on which ROMIO supports large files, i.e., PIOFS, XFS, SFS, and HFS */
-
-/* This program will work only if the MPI implementation defines MPI_Aint 
-   as a 64-bit integer. */
-   
-int main(int argc, char **argv)
-{
-    MPI_Datatype newtype;
-    int i, ndims, array_of_gsizes[3], array_of_distribs[3];
-    int order, nprocs, len, flag, err;
-    int array_of_dargs[3], array_of_psizes[3];
-    int *readbuf, *writebuf, mynod;
-    MPI_Count bufcount;
-    char filename[1024];
-    MPI_File fh;
-    MPI_Status status;
-    MPI_Aint size_with_aint;
-    MPI_Offset size_with_offset;
-
-    MPI_Init(&argc,&argv);
-    MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
-    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
-
-/* process 0 takes the file name as a command-line argument and 
-   broadcasts it to other processes */
-    if (!mynod) {
-	i = 1;
-	while ((i < argc) && strcmp("-fname", *argv)) {
-	    i++;
-	    argv++;
-	}
-	if (i >= argc) {
-	    fprintf(stderr, "\n*#  Usage: large_array -fname filename\n\n");
-	    MPI_Abort(MPI_COMM_WORLD, 1);
-	}
-	argv++;
-	len = strlen(*argv);
-	strcpy(filename, *argv);
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
-	fprintf(stderr, "This program creates a 4 Gbyte file. Don't run it if you don't have that much disk space!\n");
-    }
-    else {
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-
-/* create the distributed array filetype */
-    ndims = 3;
-    order = MPI_ORDER_C;
-
-    array_of_gsizes[0] = 1024;
-    array_of_gsizes[1] = 1024;
-    array_of_gsizes[2] = 4*1024/sizeof(int);
-
-    array_of_distribs[0] = MPI_DISTRIBUTE_BLOCK;
-    array_of_distribs[1] = MPI_DISTRIBUTE_BLOCK;
-    array_of_distribs[2] = MPI_DISTRIBUTE_BLOCK;
-
-    array_of_dargs[0] = MPI_DISTRIBUTE_DFLT_DARG;
-    array_of_dargs[1] = MPI_DISTRIBUTE_DFLT_DARG;
-    array_of_dargs[2] = MPI_DISTRIBUTE_DFLT_DARG;
-
-    for (i=0; i<ndims; i++) array_of_psizes[i] = 0;
-    MPI_Dims_create(nprocs, ndims, array_of_psizes);
-
-/* check if MPI_Aint is large enough for size of global array. 
-   if not, complain. */
-
-    size_with_aint = sizeof(int);
-    for (i=0; i<ndims; i++) size_with_aint *= array_of_gsizes[i];
-    size_with_offset = sizeof(int);
-    for (i=0; i<ndims; i++) size_with_offset *= array_of_gsizes[i];
-    if (size_with_aint != size_with_offset) {
-        fprintf(stderr, "Can't use an array of this size unless the MPI implementation defines a 64-bit MPI_Aint\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    MPI_Type_create_darray(nprocs, mynod, ndims, array_of_gsizes, 
-			   array_of_distribs, array_of_dargs,
-			   array_of_psizes, order, MPI_INT, &newtype);
-    MPI_Type_commit(&newtype);
-
-/* initialize writebuf */
-
-    MPI_Type_size_x(newtype, &bufcount);
-    bufcount = bufcount/sizeof(int);
-    writebuf = (int *) malloc(bufcount * sizeof(int));
-    if (!writebuf) fprintf(stderr, "Process %d, not enough memory for writebuf\n", mynod);
-    for (i=0; i<bufcount; i++) writebuf[i] = mynod*1024 + i;
-
-    /* write the array to the file */
-    MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, 
-                  MPI_INFO_NULL, &fh);
-    MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", MPI_INFO_NULL);
-    MPI_File_write_all(fh, writebuf, bufcount, MPI_INT, &status);
-    MPI_File_close(&fh);
-
-    free(writebuf);
-
-    /* now read it back */
-    readbuf = (int *) calloc(bufcount, sizeof(int));
-    if (!readbuf) fprintf(stderr, "Process %d, not enough memory for readbuf\n", mynod);
-
-    MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, 
-                  MPI_INFO_NULL, &fh);
-    MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", MPI_INFO_NULL);
-    MPI_File_read_all(fh, readbuf, bufcount, MPI_INT, &status);
-    MPI_File_close(&fh);
-
-    /* check the data read */
-    flag = 0;
-    for (i=0; i<bufcount; i++) 
-	if (readbuf[i] != mynod*1024 + i) {
-	    fprintf(stderr, "Process %d, readbuf=%d, writebuf=%d\n", mynod, readbuf[i], mynod*1024 + i);
-            flag = 1;
-	}
-    if (!flag) fprintf(stderr, "Process %d: data read back is correct\n", mynod);
-
-    MPI_Type_free(&newtype);
-    free(readbuf);
-
-    MPI_Barrier(MPI_COMM_WORLD);
-    if (!mynod) {
-	err = MPI_File_delete(filename, MPI_INFO_NULL);
-	if (err == MPI_SUCCESS) fprintf(stderr, "file deleted\n");
-    }
-
-    MPI_Finalize();
-    return 0;
-}
diff --git a/3rd-party/romio321/test/large_file.c.in b/3rd-party/romio321/test/large_file.c.in
deleted file mode 100644
index 3189e836fa2..00000000000
--- a/3rd-party/romio321/test/large_file.c.in
+++ /dev/null
@@ -1,107 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/*  
- *  (C) 2001 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include "mpi.h"
-#include <stdlib.h>
-#include <string.h>
-#include <stdio.h>
-
-/* writes a file of size 4 Gbytes and reads it back. 
-   should be run on one process only*/
-/* The file name is taken as a command-line argument. */
-/* Can be used only on file systems on which ROMIO supports large files, 
-   i.e., PIOFS, XFS, SFS, and HFS. */
-   
-#define SIZE 1048576*4     /* no. of long longs in each write/read */
-#define NTIMES 128         /* no. of writes/reads */
-
-int main(int argc, char **argv)
-{
-    MPI_File fh;
-    MPI_Status status;
-    MPI_Offset size;
-    long long *buf, i;
-    char *filename;
-    int j, mynod, nprocs, len, flag, err;
-
-    MPI_Init(&argc,&argv);
-
-    MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
-    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
-
-    if (nprocs != 1) {
-	fprintf(stderr, "Run this program on one process only\n");
-	MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    i = 1;
-    while ((i < argc) && strcmp("-fname", *argv)) {
-	i++;
-	argv++;
-    }
-    if (i >= argc) {
-	fprintf(stderr, "\n*#  Usage: large -fname filename\n\n");
-	MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-    argv++;
-    len = strlen(*argv);
-    filename = (char *) malloc(len+1);
-    strcpy(filename, *argv);
-    fprintf(stderr, "This program creates an 4 Gbyte file. Don't run it if you don't have that much disk space!\n");
-
-    buf = (long long *) malloc(SIZE * sizeof(long long));
-    if (!buf) {
-	fprintf(stderr, "not enough memory to allocate buffer\n");
-	MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    MPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
-                  MPI_INFO_NULL, &fh);
-
-    for (i=0; i<NTIMES; i++) {
-	for (j=0; j<SIZE; j++)
-	    buf[j] = i*SIZE + j;
-	
-	err = MPI_File_write(fh, buf, SIZE, MPI_DOUBLE, &status);
-        /* MPI_DOUBLE because not all MPI implementations define
-           MPI_LONG_LONG_INT, even though the C compiler supports long long. */
-        if (err != MPI_SUCCESS) {
-	    fprintf(stderr, "MPI_File_write returned error\n");
-	    MPI_Abort(MPI_COMM_WORLD, 1);
-	}
-    }
-
-    MPI_File_get_size(fh, &size);
-    fprintf(stderr, "file size = %@LL@ bytes\n", size);
-
-    MPI_File_seek(fh, 0, MPI_SEEK_SET);
-
-    for (j=0; j<SIZE; j++) buf[j] = -1;
-
-    flag = 0;
-    for (i=0; i<NTIMES; i++) {
-	err = MPI_File_read(fh, buf, SIZE, MPI_DOUBLE, &status);
-        /* MPI_DOUBLE because not all MPI implementations define
-           MPI_LONG_LONG_INT, even though the C compiler supports long long. */
-        if (err != MPI_SUCCESS) {
-	    fprintf(stderr, "MPI_File_write returned error\n");
-	    MPI_Abort(MPI_COMM_WORLD, 1);
-	}
-	for (j=0; j<SIZE; j++) 
-	    if (buf[j] != i*SIZE + j) {
-		fprintf(stderr, "error: buf %d is %@LL@, should be %@LL@ \n", j, buf[j], 
-                                 i*SIZE + j);
-		flag = 1;
-	    }
-    }
-
-    if (!flag) fprintf(stderr, "Data read back is correct\n");
-    MPI_File_close(&fh);
-
-    free(buf);
-    free(filename);
-    MPI_Finalize(); 
-    return 0;
-}
diff --git a/3rd-party/romio321/test/misc.c.in b/3rd-party/romio321/test/misc.c.in
deleted file mode 100644
index 5fad3968c07..00000000000
--- a/3rd-party/romio321/test/misc.c.in
+++ /dev/null
@@ -1,263 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/*  
- *  (C) 2001 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include "mpi.h"
-#include <stdlib.h>
-#include <string.h>
-#include <stdio.h>
-
-/* tests various miscellaneous functions. */
-#define VERBOSE 0
-
-static void handle_error(int errcode, const char *str)
-{
-	char msg[MPI_MAX_ERROR_STRING];
-	int resultlen;
-	MPI_Error_string(errcode, msg, &resultlen);
-	fprintf(stderr, "%s: %s\n", str, msg);
-}
-
-int main(int argc, char **argv)
-{
-    int buf[1024], amode, flag, mynod, len, i;
-    int errs = 0, toterrs;
-    MPI_File fh;
-    MPI_Status status;
-    MPI_Datatype newtype;
-    MPI_Offset disp, offset;
-    MPI_Group group;
-    MPI_Datatype etype, filetype;
-    char datarep[25], *filename;
-    int errcode=0;
-
-    MPI_Init(&argc,&argv);
-    MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
-
-/* process 0 takes the file name as a command-line argument and 
-   broadcasts it to other processes */
-    if (!mynod) {
-	i = 1;
-	while ((i < argc) && strcmp("-fname", *argv)) {
-	    i++;
-	    argv++;
-	}
-	if (i >= argc) {
-	    fprintf(stderr, "\n*#  Usage: misc -fname filename\n\n");
-	    MPI_Abort(MPI_COMM_WORLD, 1);
-	}
-	argv++;
-	len = strlen(*argv);
-	filename = (char *) malloc(len+1);
-	strcpy(filename, *argv);
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-    else {
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	filename = (char *) malloc(len+1);
-	MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-
-
-    errcode = MPI_File_open(MPI_COMM_WORLD, filename, 
-		    MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_open");
-    }
-
-    errcode = MPI_File_write(fh, buf, 1024, MPI_INT, &status);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_write");
-    }
-
-    errcode = MPI_File_sync(fh);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_sync");
-    }
-
-    errcode = MPI_File_get_amode(fh, &amode);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_get_amode");
-    }
-#if VERBOSE
-    if (!mynod) fprintf(stderr, "testing MPI_File_get_amode\n");
-#endif
-    if (amode != (MPI_MODE_CREATE | MPI_MODE_RDWR)) {
-	errs++;
-	fprintf(stderr, "amode is %d, should be %d\n\n", 
-		amode, MPI_MODE_CREATE | MPI_MODE_RDWR);
-    }
-
-    errcode = MPI_File_get_atomicity(fh, &flag);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_get_atomicity");
-    }
-    if (flag) {
-	errs++;
-	fprintf(stderr, "atomicity is %d, should be 0\n", flag);
-    }
-#if VERBOSE
-    if (!mynod) fprintf(stderr, "setting atomic mode\n");
-#endif
-    errcode = MPI_File_set_atomicity(fh, 1);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_set_atomicity");
-    }
-    errcode = MPI_File_get_atomicity(fh, &flag);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_get_atomicity");
-    }
-    if (!flag) {
-	errs++;
-	fprintf(stderr, "atomicity is %d, should be 1\n", flag);
-    }
-    errcode = MPI_File_set_atomicity(fh, 0);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_set_atomicity");
-    }
-#if VERBOSE
-    if (!mynod) fprintf(stderr, "reverting back to nonatomic mode\n");
-#endif
-
-    MPI_Type_vector(10, 10, 20, MPI_INT, &newtype);
-    MPI_Type_commit(&newtype);
-
-    errcode = MPI_File_set_view(fh, 1000, MPI_INT, newtype, "native", MPI_INFO_NULL);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_set_view");
-    }
-#if VERBOSE
-    if (!mynod) fprintf(stderr, "testing MPI_File_get_view\n");
-#endif
-    errcode = MPI_File_get_view(fh, &disp, &etype, &filetype, datarep);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_get_view");
-    }
-    if ((disp != 1000) || strcmp(datarep, "native")) {
-	errs++;
-	fprintf(stderr, "disp = %@LL@, datarep = %s, should be 1000, native\n\n", disp, datarep);
-    }
-#if VERBOSE
-    if (!mynod) fprintf(stderr, "testing MPI_File_get_byte_offset\n");
-#endif
-    errcode = MPI_File_get_byte_offset(fh, 10, &disp);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_get_byte_offset");
-    }
-    if (disp != (1000+20*sizeof(int))) {
-	errs++;
-	fprintf(stderr, "byte offset = %@LL@, should be %d\n\n", 
-		disp, (int) (1000+20*sizeof(int)));
-    }
-
-    errcode = MPI_File_get_group(fh, &group);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_get_group");
-    }
-
-#if VERBOSE
-    if (!mynod) fprintf(stderr, "testing MPI_File_set_size\n");
-#endif
-    errcode = MPI_File_set_size(fh, 1000+15*sizeof(int));
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_set_size");
-    }
-    MPI_Barrier(MPI_COMM_WORLD);
-    errcode = MPI_File_sync(fh);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_sync");
-    }
-    errcode = MPI_File_get_size(fh, &disp);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_get_size");
-    }
-    if (disp != 1000+15*sizeof(int)) {
-	errs++;
-	fprintf(stderr, "file size = %@LL@, should be %d\n\n", 
-		disp, (int) (1000+15*sizeof(int)));
-    }
-
-#if VERBOSE
-    if (!mynod) fprintf(stderr, "seeking to eof and testing MPI_File_get_position\n");
-#endif
-    errcode = MPI_File_seek(fh, 0, MPI_SEEK_END);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_seek");
-    }
-    errcode = MPI_File_get_position(fh, &disp);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_get_position");
-    }
-    if (disp != 10) {
-	errs++;
-	fprintf(stderr, "file pointer posn = %@LL@, should be 10\n\n", disp);
-    }
-
-#if VERBOSE
-    if (!mynod) fprintf(stderr, "testing MPI_File_get_byte_offset\n");
-#endif
-    errcode = MPI_File_get_byte_offset(fh, disp, &offset);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_get_byte_offset");
-    }
-    if (offset != (1000+20*sizeof(int))) {
-	errs++;
-	fprintf(stderr, "byte offset = %@LL@, should be %d\n\n", 
-		offset, (int) (1000+20*sizeof(int)));
-    }
-    MPI_Barrier(MPI_COMM_WORLD);
-
-#if VERBOSE
-    if (!mynod) fprintf(stderr, "testing MPI_File_seek with MPI_SEEK_CUR\n");
-#endif
-    errcode = MPI_File_seek(fh, -10, MPI_SEEK_CUR);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_seek");
-    }
-    errcode = MPI_File_get_position(fh, &disp);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_get_position");
-    }
-    errcode = MPI_File_get_byte_offset(fh, disp, &offset);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_get_byte_offset");
-    }
-    if (offset != 1000) {
-	errs++;
-	fprintf(stderr, "file pointer posn in bytes = %@LL@, should be 1000\n\n", offset);
-    }
-
-#if VERBOSE
-    if (!mynod) fprintf(stderr, "preallocating disk space up to 8192 bytes\n");
-#endif
-    errcode = MPI_File_preallocate(fh, 8192);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_Preallocate");
-    }
-
-#if VERBOSE
-    if (!mynod) fprintf(stderr, "closing the file and deleting it\n");
-#endif
-    MPI_File_close(&fh);
-    
-    MPI_Barrier(MPI_COMM_WORLD);
-    if (!mynod) MPI_File_delete(filename, MPI_INFO_NULL);
-
-    MPI_Allreduce( &errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
-    if (mynod == 0) {
-	if( toterrs > 0) {
-	    fprintf( stderr, "Found %d errors\n", toterrs );
-	}
-	else {
-	    fprintf( stdout, " No Errors\n" );
-	}
-    }
-    MPI_Type_free(&newtype);
-    MPI_Type_free(&filetype);
-    MPI_Group_free(&group);
-    free(filename);
-    MPI_Finalize(); 
-    return 0;
-}
diff --git a/3rd-party/romio321/test/noncontig.c b/3rd-party/romio321/test/noncontig.c
deleted file mode 100644
index 8be18dc6703..00000000000
--- a/3rd-party/romio321/test/noncontig.c
+++ /dev/null
@@ -1,267 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*  
- *  (C) 2001 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include "mpi.h"
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-
-/* tests noncontiguous reads/writes using independent I/O */
-
-#define SIZE 5000
-
-#define VERBOSE 0
-int main(int argc, char **argv)
-{
-    int *buf, i, mynod, nprocs, len, b[3];
-    int errs=0, toterrs;
-    MPI_Aint d[3];
-    MPI_File fh;
-    MPI_Status status;
-    char *filename;
-    MPI_Datatype typevec, newtype, t[3];
-    MPI_Info info;
-
-    MPI_Init(&argc,&argv);
-    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
-    MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
-
-    if (nprocs != 2) {
-        fprintf(stderr, "Run this program on two processes\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-/* process 0 takes the file name as a command-line argument and 
-   broadcasts it to other processes (length first, then string) */
-    if (!mynod) {
-	i = 1;
-	while ((i < argc) && strcmp("-fname", *argv)) {
-	    i++;
-	    argv++;
-	}
-	if (i >= argc) {
-	    fprintf(stderr, "\n*#  Usage: noncontig -fname filename\n\n");
-	    MPI_Abort(MPI_COMM_WORLD, 1);
-	}
-	argv++;
-	len = strlen(*argv);
-	filename = (char *) malloc(len+1);
-	strcpy(filename, *argv);
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-    else {
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	filename = (char *) malloc(len+1);
-	MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-
-    buf = (int *) malloc(SIZE*sizeof(int));
-
-    MPI_Type_vector(SIZE/2, 1, 2, MPI_INT, &typevec);
-
-    /* create a struct type with explicitly set LB and UB; displacements
-     * of typevec are such that the types for the two processes won't
-     * overlap.
-     */
-    b[0] = b[1] = b[2] = 1;
-    d[0] = 0;
-    d[1] = mynod*sizeof(int);
-    d[2] = SIZE*sizeof(int);
-    t[0] = MPI_LB;
-    t[1] = typevec;
-    t[2] = MPI_UB;
-
-    /* keep the struct, ditch the vector */
-    MPI_Type_struct(3, b, d, t, &newtype);
-    MPI_Type_commit(&newtype);
-    MPI_Type_free(&typevec);
-
-    MPI_Info_create(&info);
-    /* I am setting these info values for testing purposes only. It is
-       better to use the default values in practice. */
-    MPI_Info_set(info, "ind_rd_buffer_size", "1209");
-    MPI_Info_set(info, "ind_wr_buffer_size", "1107");
-
-    if (!mynod) {
-#if VERBOSE
-	fprintf(stderr, "\ntesting noncontiguous in memory, noncontiguous in file using independent I/O\n");
-#endif
-	MPI_File_delete(filename, MPI_INFO_NULL);
-    }
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
-                  info, &fh);
-
-    /* set the file view for each process -- now writes go into the non-
-     * overlapping but interleaved region defined by the struct type up above
-     */
-    MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info);
-
-    /* fill our buffer with a pattern and write, using our type again */
-    for (i=0; i<SIZE; i++) buf[i] = i + mynod*SIZE;
-    MPI_File_write(fh, buf, 1, newtype, &status);
-
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    /* fill the entire buffer with -1's.  read back with type.
-     * note that the result of this read should be that every other value
-     * in the buffer is still -1, as defined by our type.
-     */
-    for (i=0; i<SIZE; i++) buf[i] = -1;
-    MPI_File_read_at(fh, 0, buf, 1, newtype, &status);
-
-    /* check that all the values read are correct and also that we didn't
-     * overwrite any of the -1 values that we shouldn't have.
-     */
-    for (i=0; i<SIZE; i++) {
-	if (!mynod) {
-	    if ((i%2) && (buf[i] != -1)) {
-		errs++;
-		fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", 
-			mynod, i, buf[i]);
-	    }
-	    if (!(i%2) && (buf[i] != i)) {
-		errs++;
-		fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", 
-			mynod, i, buf[i], i);
-	    }
-	}
-	else {
-	    if ((i%2) && (buf[i] != i + mynod*SIZE)) {
-		errs++;
-		fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", 
-			mynod, i, buf[i], i + mynod*SIZE);
-	    }
-	    if (!(i%2) && (buf[i] != -1)) {
-		errs++;
-		fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", 
-			mynod, i, buf[i]);
-	    }
-	}
-    }
-
-    MPI_File_close(&fh);
-
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    if (!mynod) {
-#if VERBOSE
-	fprintf(stderr, "\ntesting noncontiguous in memory, contiguous in file using independent I/O\n");
-#endif
-	MPI_File_delete(filename, MPI_INFO_NULL);
-    }
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
-                  info, &fh);
-
-    /* in this case we write to either the first half or the second half
-     * of the file space, so the regions are not interleaved.  this is done
-     * by leaving the file view at its default.
-     */
-    for (i=0; i<SIZE; i++) buf[i] = i + mynod*SIZE;
-    MPI_File_write_at(fh, mynod*(SIZE/2)*sizeof(int), buf, 1, newtype, &status);
-
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    /* same as before; fill buffer with -1's and then read; every other
-     * value should still be -1 after the read
-     */
-    for (i=0; i<SIZE; i++) buf[i] = -1;
-    MPI_File_read_at(fh, mynod*(SIZE/2)*sizeof(int), buf, 1, newtype, &status);
-
-    /* verify that the buffer looks like it should */
-    for (i=0; i<SIZE; i++) {
-	if (!mynod) {
-	    if ((i%2) && (buf[i] != -1)) {
-		errs++;
-		fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", 
-			mynod, i, buf[i]);
-	    }
-	    if (!(i%2) && (buf[i] != i)) {
-		errs++;
-		fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", 
-			mynod, i, buf[i], i);
-	    }
-	}
-	else {
-	    if ((i%2) && (buf[i] != i + mynod*SIZE)) {
-		errs++;
-		fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", 
-			mynod, i, buf[i], i + mynod*SIZE);
-	    }
-	    if (!(i%2) && (buf[i] != -1)) {
-		errs++;
-		fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", 
-			mynod, i, buf[i]);
-	    }
-	}
-    }
-
-    MPI_File_close(&fh);
-
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    if (!mynod) {
-#if VERBOSE
-	fprintf(stderr, "\ntesting contiguous in memory, noncontiguous in file using independent I/O\n");
-#endif
-	MPI_File_delete(filename, MPI_INFO_NULL);
-    }
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, 
-                  info, &fh);
-
-    /* set the file view so that we have interleaved access again */
-    MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info);
-
-    /* this time write a contiguous buffer */
-    for (i=0; i<SIZE; i++) buf[i] = i + mynod*SIZE;
-    MPI_File_write(fh, buf, SIZE, MPI_INT, &status);
-
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    /* fill buffer with -1's; this time they will all be overwritten */
-    for (i=0; i<SIZE; i++) buf[i] = -1;
-    MPI_File_read_at(fh, 0, buf, SIZE, MPI_INT, &status);
-
-    for (i=0; i<SIZE; i++) {
-	if (!mynod) {
-	    if (buf[i] != i) {
-		errs++;
-		fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", 
-			mynod, i, buf[i], i);
-	    }
-	}
-	else {
-	    if (buf[i] != i + mynod*SIZE) {
-		errs++;
-		fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", 
-			mynod, i, buf[i], i + mynod*SIZE);
-	    }
-	}
-    }
-
-    MPI_File_close(&fh);
-
-    MPI_Allreduce( &errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
-    if (mynod == 0) {
-	if( toterrs > 0) {
-	    fprintf( stderr, "Found %d errors\n", toterrs );
-	}
-	else {
-	    fprintf( stdout, " No Errors\n" );
-	}
-    }
-    MPI_Type_free(&newtype);
-    MPI_Info_free(&info);
-    free(buf);
-    free(filename);
-    MPI_Finalize();
-    return 0;
-}
diff --git a/3rd-party/romio321/test/noncontig_coll.c b/3rd-party/romio321/test/noncontig_coll.c
deleted file mode 100644
index 2916776a63c..00000000000
--- a/3rd-party/romio321/test/noncontig_coll.c
+++ /dev/null
@@ -1,236 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*  
- *  (C) 2001 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include "mpi.h"
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-
-/* tests noncontiguous reads/writes using collective I/O */
-
-#define SIZE 5000
-
-#define VERBOSE 0
-int main(int argc, char **argv)
-{
-    int *buf, i, mynod, nprocs, len, b[3];
-    int errs=0, toterrs;
-    MPI_Aint d[3];
-    MPI_File fh;
-    MPI_Status status;
-    char *filename;
-    MPI_Datatype typevec, newtype, t[3];
-
-    MPI_Init(&argc,&argv);
-    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
-    MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
-
-    if (nprocs != 2) {
-        fprintf(stderr, "Run this program on two processes\n");
-        MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-/* process 0 takes the file name as a command-line argument and 
-   broadcasts it to other processes */
-    if (!mynod) {
-	i = 1;
-	while ((i < argc) && strcmp("-fname", *argv)) {
-	    i++;
-	    argv++;
-	}
-	if (i >= argc) {
-	    fprintf(stderr, "\n*#  Usage: noncontig_coll -fname filename\n\n");
-	    MPI_Abort(MPI_COMM_WORLD, 1);
-	}
-	argv++;
-	len = strlen(*argv);
-	filename = (char *) malloc(len+1);
-	strcpy(filename, *argv);
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-    else {
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	filename = (char *) malloc(len+1);
-	MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-
-    buf = (int *) malloc(SIZE*sizeof(int));
-
-    MPI_Type_vector(SIZE/2, 1, 2, MPI_INT, &typevec);
-
-    b[0] = b[1] = b[2] = 1;
-    d[0] = 0;
-    d[1] = mynod*sizeof(int);
-    d[2] = SIZE*sizeof(int);
-    t[0] = MPI_LB;
-    t[1] = typevec;
-    t[2] = MPI_UB;
-
-    MPI_Type_struct(3, b, d, t, &newtype);
-    MPI_Type_commit(&newtype);
-    MPI_Type_free(&typevec);
-
-    if (!mynod) {
-#if VERBOSE
-	fprintf(stderr, "\ntesting noncontiguous in memory, noncontiguous in file using collective I/O\n");
-#endif
-	MPI_File_delete(filename, MPI_INFO_NULL);
-    }
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
-                  MPI_INFO_NULL, &fh);
-
-    MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", MPI_INFO_NULL);
-
-    for (i=0; i<SIZE; i++) buf[i] = i + mynod*SIZE;
-    MPI_File_write_all(fh, buf, 1, newtype, &status);
-
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    for (i=0; i<SIZE; i++) buf[i] = -1;
-
-    MPI_File_read_at_all(fh, 0, buf, 1, newtype, &status);
-
-    for (i=0; i<SIZE; i++) {
-	if (!mynod) {
-	    if ((i%2) && (buf[i] != -1)) {
-		errs++;
-		fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", 
-			mynod, i, buf[i]);
-	    }
-	    if (!(i%2) && (buf[i] != i)) {
-		errs++;
-		fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", 
-			mynod, i, buf[i], i);
-	    }
-	}
-	else {
-	    if ((i%2) && (buf[i] != i + mynod*SIZE)) {
-		errs++;
-		fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", 
-			mynod, i, buf[i], i + mynod*SIZE);
-	    }
-	    if (!(i%2) && (buf[i] != -1)) {
-		errs++;
-		fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", 
-			mynod, i, buf[i]);
-	    }
-	}
-    }
-
-    MPI_File_close(&fh);
-
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    if (!mynod) {
-#if VERBOSE
-	fprintf(stderr, "\ntesting noncontiguous in memory, contiguous in file using collective I/O\n");
-#endif
-	MPI_File_delete(filename, MPI_INFO_NULL);
-    }
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
-                  MPI_INFO_NULL, &fh);
-
-    for (i=0; i<SIZE; i++) buf[i] = i + mynod*SIZE;
-    MPI_File_write_at_all(fh, mynod*(SIZE/2)*sizeof(int), buf, 1, newtype, &status);
-
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    for (i=0; i<SIZE; i++) buf[i] = -1;
-
-    MPI_File_read_at_all(fh, mynod*(SIZE/2)*sizeof(int), buf, 1, newtype, &status);
-
-    for (i=0; i<SIZE; i++) {
-	if (!mynod) {
-	    if ((i%2) && (buf[i] != -1)) {
-		errs++;
-		fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", 
-			mynod, i, buf[i]);
-	    }
-	    if (!(i%2) && (buf[i] != i)) {
-		errs++;
-		fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", 
-			mynod, i, buf[i], i);
-	    }
-	}
-	else {
-	    if ((i%2) && (buf[i] != i + mynod*SIZE)) {
-		errs++;
-		fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", 
-			mynod, i, buf[i], i + mynod*SIZE);
-	    }
-	    if (!(i%2) && (buf[i] != -1)) {
-		errs++;
-		fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", 
-			mynod, i, buf[i]);
-	    }
-	}
-    }
-
-    MPI_File_close(&fh);
-
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    if (!mynod) {
-#if VERBOSE
-	fprintf(stderr, "\ntesting contiguous in memory, noncontiguous in file using collective I/O\n");
-#endif
-	MPI_File_delete(filename, MPI_INFO_NULL);
-    }
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
-                  MPI_INFO_NULL, &fh);
-
-    MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", MPI_INFO_NULL);
-
-    for (i=0; i<SIZE; i++) buf[i] = i + mynod*SIZE;
-    MPI_File_write_all(fh, buf, SIZE, MPI_INT, &status);
-
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    for (i=0; i<SIZE; i++) buf[i] = -1;
-
-    MPI_File_read_at_all(fh, 0, buf, SIZE, MPI_INT, &status);
-
-    for (i=0; i<SIZE; i++) {
-	if (!mynod) {
-	    if (buf[i] != i) {
-		errs++;
-		fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", 
-			mynod, i, buf[i], i);
-	    }
-	}
-	else {
-	    if (buf[i] != i + mynod*SIZE) {
-		errs++;
-		fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", 
-			mynod, i, buf[i], i + mynod*SIZE);
-	    }
-	}
-    }
-
-    MPI_File_close(&fh);
-
-    MPI_Allreduce( &errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
-    if (mynod == 0) {
-	if( toterrs > 0) {
-	    fprintf( stderr, "Found %d errors\n", toterrs );
-	}
-	else {
-	    fprintf( stdout, " No Errors\n" );
-	}
-    }
-
-    MPI_Type_free(&newtype);
-    free(buf);
-    free(filename);
-    MPI_Finalize();
-    return 0;
-}
diff --git a/3rd-party/romio321/test/noncontig_coll2.c b/3rd-party/romio321/test/noncontig_coll2.c
deleted file mode 100644
index 4699da89aca..00000000000
--- a/3rd-party/romio321/test/noncontig_coll2.c
+++ /dev/null
@@ -1,539 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*  
- *  (C) 2001 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include "mpi.h"
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-
-/* tests noncontiguous reads/writes using collective I/O */
-
-/* this test is almost exactly like noncontig_coll.c with the following changes:
- *
- * . generalized file writing/reading to handle arbitrary number of processors
- * . provides the "cb_config_list" hint with several permutations of the
- *   avaliable processors.  
- *   [ makes use of code copied from ROMIO's ADIO code to collect the names of
- *   the processors ]
- */
-
-/* we are going to muck with this later to make it evenly divisible by however many compute nodes we have */
-#define STARTING_SIZE 5000
-
-int test_file(char *filename, int mynod, int nprocs, char * cb_hosts, 
-		const char *msg, int verbose); 
-
-#define ADIOI_Free free
-#define ADIOI_Malloc malloc
-#define FPRINTF fprintf
-/* I have no idea what the "D" stands for; it's how things are done in adio.h 
- */ 
-struct ADIO_cb_name_arrayD {   
-       int refct;              
-       int namect;
-       char **names;
-};  
-typedef struct ADIO_cb_name_arrayD *ADIO_cb_name_array;
-
-void handle_error(int errcode, const char *str);
-int cb_gather_name_array(MPI_Comm comm, ADIO_cb_name_array *arrayp);
-void default_str(int mynod, int len, ADIO_cb_name_array array, char *dest);
-void reverse_str(int mynod, int len, ADIO_cb_name_array array, char *dest);
-void reverse_alternating_str(int mynod, int len, ADIO_cb_name_array array, char *dest);
-void simple_shuffle_str(int mynod, int len, ADIO_cb_name_array array, char *dest);
-
-
-void handle_error(int errcode, const char *str) 
-{
-	char msg[MPI_MAX_ERROR_STRING];
-	int resultlen;
-	MPI_Error_string(errcode, msg, &resultlen);
-	fprintf(stderr, "%s: %s\n", str, msg);
-	MPI_Abort(MPI_COMM_WORLD, 1);
-}
-   
-
-/* cb_gather_name_array() - gather a list of processor names from all processes
- *                          in a communicator and store them on rank 0.
- *                   
- * This is a collective call on the communicator(s) passed in.
- *
- * Obtains a rank-ordered list of processor names from the processes in
- * "dupcomm".
- *
- * Returns 0 on success, -1 on failure.
- * 
- * NOTE: Needs some work to cleanly handle out of memory cases!  
- */
-int cb_gather_name_array(MPI_Comm comm, ADIO_cb_name_array *arrayp)
-{
-	/* this is copied from ROMIO, but since this test is for correctness,
-	 * not performance, note that we have removed the parts where ROMIO
-	 * uses a keyval to cache the name array.  We'll just rebuild it if we
-	 * need to */
-
-    char my_procname[MPI_MAX_PROCESSOR_NAME], **procname = 0;
-    int *procname_len = NULL, my_procname_len, *disp = NULL, i;
-    int commsize, commrank;
-    ADIO_cb_name_array array = NULL;
-
-    MPI_Comm_size(comm, &commsize);
-    MPI_Comm_rank(comm, &commrank);
-
-    MPI_Get_processor_name(my_procname, &my_procname_len);
-
-    /* allocate space for everything */
-    array = (ADIO_cb_name_array) malloc(sizeof(*array));
-    if (array == NULL) {
-	return -1;
-    }
-    array->refct = 1; 
-
-    if (commrank == 0) {
-	/* process 0 keeps the real list */
-	array->namect = commsize;
-
-	array->names = (char **) ADIOI_Malloc(sizeof(char *) * commsize);
-	if (array->names == NULL) {
-	    return -1;
-	}
-	procname = array->names; /* simpler to read */
-
-	procname_len = (int *) ADIOI_Malloc(commsize * sizeof(int));
-	if (procname_len == NULL) { 
-	    return -1;
-	}
-    }
-    else {
-	/* everyone else just keeps an empty list as a placeholder */
-	array->namect = 0;
-	array->names = NULL;
-    }
-    /* gather lengths first */
-    MPI_Gather(&my_procname_len, 1, MPI_INT, 
-	       procname_len, 1, MPI_INT, 0, comm);
-
-    if (commrank == 0) {
-#ifdef CB_CONFIG_LIST_DEBUG
-	for (i=0; i < commsize; i++) {
-	    FPRINTF(stderr, "len[%d] = %d\n", i, procname_len[i]);
-	}
-#endif
-
-	for (i=0; i < commsize; i++) {
-	    /* add one to the lengths because we need to count the
-	     * terminator, and we are going to use this list of lengths
-	     * again in the gatherv.  
-	     */
-	    procname_len[i]++;
-	    procname[i] = malloc(procname_len[i]);
-	    if (procname[i] == NULL) {
-		return -1;
-	    }
-	}
-	
-	/* create our list of displacements for the gatherv.  we're going
-	 * to do everything relative to the start of the region allocated
-	 * for procname[0]
-	 *
-	 * I suppose it is theoretically possible that the distance between 
-	 * malloc'd regions could be more than will fit in an int.  We don't
-	 * cover that case.
-	 */
-	disp = malloc(commsize * sizeof(int));
-	disp[0] = 0;
-	for (i=1; i < commsize; i++) {
-	    disp[i] = (int) (procname[i] - procname[0]);
-	}
-
-    }
-
-    /* now gather strings */
-    if (commrank == 0) {
-	MPI_Gatherv(my_procname, my_procname_len + 1, MPI_CHAR, 
-		    procname[0], procname_len, disp, MPI_CHAR,
-		    0, comm);
-    }
-    else {
-	/* if we didn't do this, we would need to allocate procname[]
-	 * on all processes...which seems a little silly.
-	 */
-	MPI_Gatherv(my_procname, my_procname_len + 1, MPI_CHAR, 
-		    NULL, NULL, NULL, MPI_CHAR, 0, comm);
-    }
-
-    if (commrank == 0) {
-	/* no longer need the displacements or lengths */
-	free(disp);
-	free(procname_len);
-
-#ifdef CB_CONFIG_LIST_DEBUG
-	for (i=0; i < commsize; i++) {
-	    fprintf(stderr, "name[%d] = %s\n", i, procname[i]);
-	}
-#endif
-    }
-
-    *arrayp = array;
-    return 0;
-}
-
-void default_str(int mynod, int len, ADIO_cb_name_array array, char *dest)
-{
-	char *ptr;
-	int i, p;
-	if (!mynod) {
-	    ptr = dest;
-	    for (i=0; i<array->namect; i++ ) {
-		    p = snprintf(ptr, len, "%s,", array->names[i]);
-		    ptr += p;
-	    }
-	    /* chop off that last comma */
-	    dest[strlen(dest) - 1] = '\0';
-	}
-	MPI_Bcast(dest, len, MPI_CHAR, 0, MPI_COMM_WORLD);
-}
-void reverse_str(int mynod, int len, ADIO_cb_name_array array, char *dest) 
-{
-	char *ptr;
-	int i, p;
-	if (!mynod) {
-	    ptr = dest;
-	    for (i=(array->namect - 1); i >= 0; i-- ) {
-		    p = snprintf(ptr, len, "%s,", array->names[i]);
-		    ptr += p;
-	    }
-	    dest[strlen(dest) - 1] = '\0';
-	}
-	MPI_Bcast(dest, len, MPI_CHAR, 0, MPI_COMM_WORLD);
-}
-
-void reverse_alternating_str(int mynod, int len, ADIO_cb_name_array array, char *dest)
-{
-	char *ptr;
-	int i, p;
-	if (!mynod) {
-	    ptr = dest;
-	    /* evens */
-	    for (i=(array->namect - 1); i>= 0; i-=2 ) {
-		    p = snprintf(ptr, len, "%s,", array->names[i]);
-		    ptr += p;
-	    }
-	    /* odds */
-	    for (i=(array->namect - 2); i > 0; i-=2 ) {
-		    p = snprintf(ptr, len, "%s,", array->names[i]);
-		    ptr += p;
-	    }
-	    dest[strlen(dest) - 1] = '\0';
-	}
-	MPI_Bcast(dest, len, MPI_CHAR, 0, MPI_COMM_WORLD);
-}
-
-void simple_shuffle_str(int mynod, int len, ADIO_cb_name_array array, char *dest)
-{
-	char *ptr;
-	int i, p;
-	if (!mynod) {
-	    ptr = dest;
-	    for (i=(array->namect / 2 ); i < array->namect; i++) {
-		    p = snprintf(ptr, len, "%s,", array->names[i]);
-		    ptr += p;
-	    }
-	    for (i=0; i < (array->namect / 2); i++ ) {
-		    p = snprintf(ptr, len, "%s,", array->names[i]);
-		    ptr += p;
-	    }
-	    dest[strlen(dest) - 1] = '\0';
-	}
-	MPI_Bcast(dest, len, MPI_CHAR, 0, MPI_COMM_WORLD);
-}
-
-int main(int argc, char **argv)
-{
-    int i, mynod, nprocs, len, errs=0, sum_errs=0, verbose=0;
-    char *filename;
-    char * cb_config_string;
-    int cb_config_len;
-    ADIO_cb_name_array array;
-
-
-    MPI_Init(&argc,&argv);
-    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
-    MPI_Comm_rank(MPI_COMM_WORLD, &mynod); 
-
-    
-    /* process 0 takes the file name as a command-line argument and 
-   broadcasts it to other processes */
-    if (!mynod) {
-	i = 1;
-	/* TODO: at some point, accept -v for verbose */
-	while ((i < argc) && strcmp("-fname", *argv)) {
-	    i++;
-	    argv++;
-	}
-	if (i >= argc) {
-	    fprintf(stderr, "\n*#  Usage: noncontig_coll -fname filename\n\n");
-	    MPI_Abort(MPI_COMM_WORLD, 1);
-	}
-	argv++;
-	len = strlen(*argv);
-	filename = (char *) malloc(len+1);
-	strcpy(filename, *argv);
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-    else {
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	filename = (char *) malloc(len+1);
-	MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-
-    /* want to hint the cb_config_list, but do so in a non-sequential way */
-    cb_gather_name_array(MPI_COMM_WORLD,  &array);
-
-    /* sanity check */
-    if (!mynod) {
-	    if (array->namect < 2 ) {
-		    fprintf(stderr, "Run this test on two or more hosts\n");
-		    MPI_Abort(MPI_COMM_WORLD, 1);
-	    }
-    }
-    /* get space for the permuted cb_config_string */
-    if (!mynod) {
-	    cb_config_len = 0;
-	    for (i=0; i < array->namect; i++) {
-		    /* +1: space for either a , or \0 if last */
-		    cb_config_len += strlen(array->names[i]) + 1;
-	    }
-	    ++cb_config_len;
-    }
-    MPI_Bcast(&cb_config_len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-    if ( (cb_config_string = malloc(cb_config_len)) == NULL ) {
-	    perror("malloc");
-	    MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-
-    /* first, no hinting */
-    errs += test_file(filename, mynod, nprocs, NULL, "collective w/o hinting", verbose);
-
-    /* hint, but no change in order */
-    default_str(mynod, cb_config_len, array, cb_config_string);
-    errs += test_file(filename, mynod, nprocs, cb_config_string, "collective w/ hinting: default order", verbose);
-
-    /*  reverse order */
-    reverse_str(mynod, cb_config_len, array, cb_config_string); 
-    errs += test_file(filename, mynod, nprocs, cb_config_string, "collective w/ hinting: reverse order", verbose);
-
-    /* reverse, every other */
-    reverse_alternating_str(mynod, cb_config_len, array, cb_config_string);
-    errs += test_file(filename, mynod, nprocs, cb_config_string,"collective w/ hinting: permutation1", verbose);
-
-    /* second half, first half */
-    simple_shuffle_str(mynod, cb_config_len, array, cb_config_string);
-    errs += test_file(filename, mynod, nprocs, cb_config_string, "collective w/ hinting: permutation2", verbose);
-
-    MPI_Allreduce(&errs, &sum_errs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
-	 
-    if (!mynod) {
-	    if (sum_errs) fprintf(stderr, "Found %d error cases\n", sum_errs);
-	    else printf(" No Errors\n");
-    }
-    free(filename);
-    free(cb_config_string);
-    MPI_Finalize();
-    return 0;
-}
-
-#define SEEDER(x,y,z) ((x)*1000000 + (y) + (x)*(z))
-
-int test_file(char *filename, int mynod, int nprocs, char * cb_hosts, const char *msg, int verbose) 
-{
-    MPI_Datatype typevec, newtype, t[3];
-    int *buf, i, b[3], errcode, errors=0;
-    MPI_File fh;
-    MPI_Aint d[3];
-    MPI_Status status;
-    int SIZE = (STARTING_SIZE/nprocs)*nprocs;
-    MPI_Info info;
-
-    if (mynod==0 && verbose) fprintf(stderr, "%s\n", msg);
-
-    buf = (int *) malloc(SIZE*sizeof(int));
-    if (buf == NULL) {
-	    perror("test_file");
-	    MPI_Abort(MPI_COMM_WORLD, -1);
-    }
-
-
-    if (cb_hosts != NULL ) {
-	    MPI_Info_create(&info);
-	    MPI_Info_set(info, "cb_config_list", cb_hosts);
-    } else {
-	    info = MPI_INFO_NULL;
-    }
-
-    MPI_Type_vector(SIZE/nprocs, 1, nprocs, MPI_INT, &typevec);
-
-    b[0] = b[1] = b[2] = 1;
-    d[0] = 0;
-    d[1] = mynod*sizeof(int);
-    d[2] = SIZE*sizeof(int);
-    t[0] = MPI_LB;
-    t[1] = typevec;
-    t[2] = MPI_UB;
-
-    MPI_Type_struct(3, b, d, t, &newtype);
-    MPI_Type_commit(&newtype);
-    MPI_Type_free(&typevec);
-
-    if (!mynod) {
-	if(verbose) fprintf(stderr, "\ntesting noncontiguous in memory, noncontiguous in file using collective I/O\n");
-	MPI_File_delete(filename, info);
-    }
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    errcode = MPI_File_open(MPI_COMM_WORLD, filename, 
-		    MPI_MODE_CREATE | MPI_MODE_RDWR, info, &fh);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_open");
-    }
-
-    MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info);
-
-    for (i=0; i<SIZE; i++) buf[i] = SEEDER(mynod,i,SIZE);
-    errcode = MPI_File_write_all(fh, buf, 1, newtype, &status);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "nc mem - nc file: MPI_File_write_all");
-    }
-
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    for (i=0; i<SIZE; i++) buf[i] = -1;
-
-    errcode = MPI_File_read_at_all(fh, 0, buf, 1, newtype, &status);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "nc mem - nc file: MPI_File_read_at_all");
-    }
-
-    /* the verification for N compute nodes is tricky. Say we have 3
-     * processors.  
-     * process 0 sees: 0 -1 -1 3 -1 -1 ...
-     * process 1 sees: -1 34 -1 -1 37 -1 ...
-     * process 2 sees: -1 -1 68 -1 -1 71 ... */
-
-    /* verify those leading -1s exist if they should */
-    for (i=0; i<mynod; i++ ) {
-	    if ( buf[i] != -1 ) {
-		    if(verbose) fprintf(stderr, "Process %d: buf is %d, should be -1\n", mynod, buf[i]);
-		    errors++;
-	    }
-    }
-    /* now the modulo games are hairy.  processor 0 sees real data in the 0th,
-     * 3rd, 6th... elements of the buffer (assuming nprocs==3 ).  proc 1 sees
-     * the data in 1st, 4th, 7th..., and proc 2 sees it in 2nd, 5th, 8th */
-
-    for(/* 'i' set in above loop */; i<SIZE; i++) {
-	    if ( ((i-mynod)%nprocs) && buf[i] != -1)  {
-		    if(verbose) fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", 
-				    mynod, i, buf[i]);
-		    errors++;
-	    }
-	    if ( !((i-mynod)%nprocs) && buf[i] != SEEDER(mynod,i,SIZE) ) {
-		    if(verbose) fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
-				    mynod, i, buf[i], SEEDER(mynod,i,SIZE));
-		    errors++;
-	    }
-    }
-    MPI_File_close(&fh);
-
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    if (!mynod) {
-	if(verbose) fprintf(stderr, "\ntesting noncontiguous in memory, contiguous in file using collective I/O\n");
-	MPI_File_delete(filename, info);
-    }
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
-                  info, &fh);
-
-    for (i=0; i<SIZE; i++) buf[i] = SEEDER(mynod,i,SIZE);
-    errcode = MPI_File_write_at_all(fh, mynod*(SIZE/nprocs)*sizeof(int), 
-		    buf, 1, newtype, &status);
-    if (errcode != MPI_SUCCESS)
-	    handle_error(errcode, "nc mem - c file: MPI_File_write_at_all");
-
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    for (i=0; i<SIZE; i++) buf[i] = -1;
-
-    errcode = MPI_File_read_at_all(fh, mynod*(SIZE/nprocs)*sizeof(int), 
-		    buf, 1, newtype, &status);
-    if (errcode != MPI_SUCCESS)
-	    handle_error(errcode, "nc mem - c file: MPI_File_read_at_all");
-
-    /* just like as above */
-    for (i=0; i<mynod; i++ ) {
-	    if ( buf[i] != -1 ) {
-		    if(verbose) fprintf(stderr, "Process %d: buf is %d, should be -1\n", mynod, buf[i]);
-		    errors++;
-	    }
-    }
-    for(/* i set in above loop */; i<SIZE; i++) {
-	    if ( ((i-mynod)%nprocs) && buf[i] != -1)  {
-		    if(verbose) fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", 
-				    mynod, i, buf[i]);
-		    errors++;
-	    }
-	    if ( !((i-mynod)%nprocs) && buf[i] != SEEDER(mynod,i,SIZE)) {
-		    if(verbose) fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
-				    mynod, i, buf[i], SEEDER(mynod,i,SIZE) );
-		    errors++;
-	    }
-    }
-
-    MPI_File_close(&fh);
-
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    if (!mynod) {
-	if(verbose) fprintf(stderr, "\ntesting contiguous in memory, noncontiguous in file using collective I/O\n");
-	MPI_File_delete(filename, info);
-    }
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
-                  info, &fh);
-
-    MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info);
-
-    for (i=0; i<SIZE; i++) buf[i] = SEEDER(mynod, i, SIZE);
-    errcode = MPI_File_write_all(fh, buf, SIZE, MPI_INT, &status);
-    if (errcode != MPI_SUCCESS)
-	    handle_error(errcode, "c mem - nc file: MPI_File_write_all");
-
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    for (i=0; i<SIZE; i++) buf[i] = -1;
-
-    errcode = MPI_File_read_at_all(fh, 0, buf, SIZE, MPI_INT, &status);
-    if (errcode != MPI_SUCCESS)
-	    handle_error(errcode, "c mem - nc file: MPI_File_read_at_all");
-
-    /* same crazy checking */
-    for (i=0; i<SIZE; i++) {
-	    if (buf[i] != SEEDER(mynod, i, SIZE)) {
-		if(verbose) fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", mynod, i, buf[i], SEEDER(mynod, i, SIZE));
-		errors++;
-	    }
-    }
-
-    MPI_File_close(&fh);
-
-    MPI_Type_free(&newtype);
-    free(buf);
-    if (info != MPI_INFO_NULL) MPI_Info_free(&info);
-    return errors;
-}
diff --git a/3rd-party/romio321/test/ordered_fp.c b/3rd-party/romio321/test/ordered_fp.c
deleted file mode 100644
index 8090e1f6c55..00000000000
--- a/3rd-party/romio321/test/ordered_fp.c
+++ /dev/null
@@ -1,144 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*  
- *  (C) 2001 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include "mpi.h"
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-
-#define COUNT (200)
-#undef TIMING
-
-void handle_error(int errcode, const char *str);
-
-void handle_error(int errcode, const char *str) 
-{
-	char msg[MPI_MAX_ERROR_STRING];
-	int resultlen;
-	MPI_Error_string(errcode, msg, &resultlen);
-	fprintf(stderr, "%s: %s\n", str, msg);
-	MPI_Abort(MPI_COMM_WORLD, 1);
-}
-
-/* tests shared file pointer functions */
-
-int main(int argc, char **argv)
-{
-    int *buf, i, rank, nprocs, len, sum;
-    int global_sum;
-    int errs=0, toterrs, errcode;
-    char *filename;
-    MPI_File fh;
-    MPI_Status status;
-
-    MPI_Init(&argc,&argv);
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-
-    double wr_stime, wr_etime, wr_time, wr_sumtime;
-    double rd_stime, rd_etime, rd_time, rd_sumtime;
-
-/* process 0 takes the file name as a command-line argument and 
-   broadcasts it to other processes */
-    if (!rank) {
-	i = 1;
-	while ((i < argc) && strcmp("-fname", *argv)) {
-	    i++;
-	    argv++;
-	}
-	if (i >= argc) {
-	    fprintf(stderr, "\n*#  Usage: shared_fp -fname filename\n\n");
-	    MPI_Abort(MPI_COMM_WORLD, 1);
-	}
-	argv++;
-	len = strlen(*argv);
-	filename = (char *) malloc(len+10);
-	strcpy(filename, *argv);
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	MPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-    else {
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	filename = (char *) malloc(len+10);
-	MPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-    
-    buf = (int *) malloc(COUNT * sizeof(int));
-
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
-
-    for (i=0; i<COUNT; i++) buf[i] = COUNT*rank + i;
-
-    errcode = MPI_File_open(MPI_COMM_WORLD, filename, 
-		    MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_open");
-    }
-
-    wr_stime = MPI_Wtime();
-
-    errcode = MPI_File_write_ordered(fh, buf, COUNT, MPI_INT, &status);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_write_shared");
-    }
-    wr_etime = MPI_Wtime();
-
-    for (i=0; i<COUNT; i++) buf[i] = 0;
-
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    rd_stime = MPI_Wtime();
-    errcode = MPI_File_seek_shared(fh, 0, MPI_SEEK_SET);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_seek_shared");
-    }
-
-    errcode = MPI_File_read_ordered(fh, buf, COUNT, MPI_INT, &status);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_read_shared");
-    }
-
-    rd_etime = MPI_Wtime();
-    MPI_File_close(&fh);
-
-    sum = 0;
-    for (i=0; i<COUNT; i++) sum += buf[i];
-
-    MPI_Allreduce(&sum, &global_sum, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
-
-    wr_time = wr_etime - wr_stime;
-    rd_time = rd_etime - rd_stime;
-
-    MPI_Allreduce(&wr_time, &wr_sumtime, 1, 
-        MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
-    MPI_Allreduce(&rd_time, &rd_sumtime, 1, 
-        MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
-
-    if (global_sum != (((COUNT*nprocs - 1)*(COUNT*nprocs))/2)) {
-	errs++;
-	fprintf(stderr, "Error: sum %d, global_sum %d, %d\n", 
-		sum, global_sum,(((COUNT*nprocs - 1)*(COUNT*nprocs))/2));
-    }
-    
-    free(buf);
-    free(filename);
-
-    MPI_Allreduce( &errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
-    if (rank == 0) {
-	if( toterrs > 0) {
-	    fprintf( stderr, "Found %d errors\n", toterrs );
-	}
-	else {
-	    fprintf( stdout, " No Errors\n" );
-#ifdef TIMING
-            fprintf( stderr, "nprocs: %d bytes: %d write: %f read %f\n", 
-                 nprocs, COUNT*sizeof(int), wr_sumtime, rd_sumtime);
-#endif
-	}
-    }
-
-    MPI_Finalize();
-    return 0; 
-}
diff --git a/3rd-party/romio321/test/perf.c b/3rd-party/romio321/test/perf.c
deleted file mode 100644
index b72cf145f8c..00000000000
--- a/3rd-party/romio321/test/perf.c
+++ /dev/null
@@ -1,160 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*  
- *  (C) 2001 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include "mpi.h"
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-
-/* A simple performance test. The file name is taken as a 
-   command-line argument. */
-
-#define SIZE (1048576*4)       /* read/write size per node in bytes */
-
-int main(int argc, char **argv)
-{
-    int *buf, i, j, mynod, nprocs, ntimes=5, len, err, flag;
-    double stim, read_tim, write_tim, new_read_tim, new_write_tim;
-    double min_read_tim=10000000.0, min_write_tim=10000000.0, read_bw, write_bw;
-    MPI_File fh;
-    MPI_Status status;
-    char *filename;
-
-    MPI_Init(&argc,&argv);
-    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
-    MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
-
-/* process 0 takes the file name as a command-line argument and 
-   broadcasts it to other processes */
-    if (!mynod) {
-	i = 1;
-	while ((i < argc) && strcmp("-fname", *argv)) {
-	    i++;
-	    argv++;
-	}
-	if (i >= argc) {
-	    fprintf(stderr, "\n*#  Usage: perf -fname filename\n\n");
-	    MPI_Abort(MPI_COMM_WORLD, 1);
-	}
-	argv++;
-	len = strlen(*argv);
-	filename = (char *) malloc(len+1);
-	strcpy(filename, *argv);
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
-	fprintf(stderr, "Access size per process = %d bytes, ntimes = %d\n", SIZE, ntimes);
-    }
-    else {
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	filename = (char *) malloc(len+1);
-	MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-
-
-    buf = (int *) malloc(SIZE);
-
-    for (j=0; j<ntimes; j++) {
-	MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | 
-             MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
-	MPI_File_seek(fh, mynod*SIZE, MPI_SEEK_SET);
-
-	MPI_Barrier(MPI_COMM_WORLD);
-	stim = MPI_Wtime();
-	MPI_File_write(fh, buf, SIZE, MPI_BYTE, &status);
-	write_tim = MPI_Wtime() - stim;
-  
-	MPI_File_close(&fh);
-
-	MPI_Barrier(MPI_COMM_WORLD);
-
-	MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | 
-                   MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
-	MPI_File_seek(fh, mynod*SIZE, MPI_SEEK_SET);
-      
-	MPI_Barrier(MPI_COMM_WORLD);
-	stim = MPI_Wtime();
-	MPI_File_read(fh, buf, SIZE, MPI_BYTE, &status);
-	read_tim = MPI_Wtime() - stim;
-  
-	MPI_File_close(&fh);
-  
-	MPI_Allreduce(&write_tim, &new_write_tim, 1, MPI_DOUBLE, MPI_MAX,
-		      MPI_COMM_WORLD);
-	MPI_Allreduce(&read_tim, &new_read_tim, 1, MPI_DOUBLE, MPI_MAX,
-		    MPI_COMM_WORLD);
-
-	min_read_tim = (new_read_tim < min_read_tim) ? 
-	    new_read_tim : min_read_tim;
-	min_write_tim = (new_write_tim < min_write_tim) ? 
-	    new_write_tim : min_write_tim;
-    }
-    
-    if (mynod == 0) {
-	read_bw = (SIZE*nprocs)/(min_read_tim*1024.0*1024.0);
-	write_bw = (SIZE*nprocs)/(min_write_tim*1024.0*1024.0);
-	fprintf(stderr, "Write bandwidth without file sync = %f Mbytes/sec\n", write_bw);
-	fprintf(stderr, "Read bandwidth without prior file sync = %f Mbytes/sec\n", read_bw);
-    }
-
-    min_write_tim=10000000.0;
-    min_read_tim=10000000.0;
-
-    flag = 0;
-    for (j=0; j<ntimes; j++) {
-	MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | 
-                 MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
-	MPI_File_seek(fh, mynod*SIZE, MPI_SEEK_SET);
-
-	MPI_Barrier(MPI_COMM_WORLD);
-	stim = MPI_Wtime();
-	MPI_File_write(fh, buf, SIZE, MPI_BYTE, &status);
-	err = MPI_File_sync(fh);
-	write_tim = MPI_Wtime() - stim;
-	if (err == MPI_ERR_UNKNOWN) {
-	    flag = 1;
-	    break;
-	}
-  
-	MPI_File_close(&fh);
-  
-	MPI_Barrier(MPI_COMM_WORLD);
-
-	MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | 
-                   MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
-	MPI_File_seek(fh, mynod*SIZE, MPI_SEEK_SET);
-      
-	MPI_Barrier(MPI_COMM_WORLD);
-	stim = MPI_Wtime();
-	MPI_File_read(fh, buf, SIZE, MPI_BYTE, &status);
-	read_tim = MPI_Wtime() - stim;
-  
-	MPI_File_close(&fh);
-  
-	MPI_Allreduce(&write_tim, &new_write_tim, 1, MPI_DOUBLE, MPI_MAX,
-		      MPI_COMM_WORLD);
-	MPI_Allreduce(&read_tim, &new_read_tim, 1, MPI_DOUBLE, MPI_MAX,
-		    MPI_COMM_WORLD);
-
-	min_read_tim = (new_read_tim < min_read_tim) ? 
-	    new_read_tim : min_read_tim;
-	min_write_tim = (new_write_tim < min_write_tim) ? 
-	    new_write_tim : min_write_tim;
-    }
-
-    if (mynod == 0) {
-	if (flag) fprintf(stderr, "MPI_File_sync returns error.\n");
-	else {
-	    read_bw = (SIZE*nprocs)/(min_read_tim*1024.0*1024.0);
-	    write_bw = (SIZE*nprocs)/(min_write_tim*1024.0*1024.0);
-	    fprintf(stderr, "Write bandwidth including file sync = %f Mbytes/sec\n", write_bw);
-	    fprintf(stderr, "Read bandwidth after file sync = %f Mbytes/sec\n", read_bw);
-	}
-    }
-
-    free(buf);
-    free(filename);
-    MPI_Finalize();
-    return 0;
-}
diff --git a/3rd-party/romio321/test/pfcoll_test.f.in b/3rd-party/romio321/test/pfcoll_test.f.in
deleted file mode 100644
index d05db1d9d0f..00000000000
--- a/3rd-party/romio321/test/pfcoll_test.f.in
+++ /dev/null
@@ -1,176 +0,0 @@
-! -*- Mode: Fortran; -*- 
-!  
-!  (C) 2001 by Argonne National Laboratory.
-!      See COPYRIGHT in top-level directory.
-!
-      program main
-      implicit none
-
-      include 'mpif.h'
-      @F77MPIOINC@
-
-!     This is the same as fcoll_test.f, but uses the PMPI versions
-!     of all functions in order to test the profiling interface.
-
-      integer FILESIZE 
-      parameter (FILESIZE=32*32*32*4)
-
-!     A 32^3 array. For other array sizes, change FILESIZE above and
-!     array_of_gsizes below.
-
-!     Uses collective I/O. Writes a 3D block-distributed array to a file
-!     corresponding to the global array in row-major (C) order, reads it
-!     back, and checks that the data read is correct.
-
-!     Note that the file access pattern is noncontiguous.
-   
-      integer newtype, i, ndims, array_of_gsizes(3)
-      integer order, intsize, nprocs, j, array_of_distribs(3)
-      integer array_of_dargs(3), array_of_psizes(3)
-      integer readbuf(FILESIZE), writebuf(FILESIZE), bufcount
-      integer mynod, tmpbuf(FILESIZE), array_size, argc, iargc
-      integer fh, status(MPI_STATUS_SIZE), request, ierr
-      character*1024 str    ! used to store the filename
-      integer errs, toterrs
-      @FORTRAN_MPI_OFFSET@ disp
-      @FTESTDEFINE@
-
-      errs = 0
-      call PMPI_INIT(ierr)
-      call PMPI_COMM_SIZE(MPI_COMM_WORLD, nprocs, ierr)
-      call PMPI_COMM_RANK(MPI_COMM_WORLD, mynod, ierr)
-
-!     process 0 takes the file name as a command-line argument and 
-!     broadcasts it to other processes
-
-      if (mynod .eq. 0) then
-         argc = @F77IARGC@
-         i = 0
-         @F77GETARG@
-         do while ((i .lt. argc) .and. (str .ne. '-fname'))
-            i = i + 1
-            @F77GETARG@
-         end do
-         if (i .ge. argc) then
-            print *
-            print *, '*#  Usage: fcoll_test -fname filename'
-            print *
-            call PMPI_ABORT(MPI_COMM_WORLD, 1, ierr)
-         end if
-
-         i = i + 1
-         @F77GETARG@
-         call PMPI_BCAST(str, 1024, MPI_CHARACTER, 0,                   &
-     &        MPI_COMM_WORLD, ierr)
-      else 
-         call PMPI_BCAST(str, 1024, MPI_CHARACTER, 0,                   &
-     &        MPI_COMM_WORLD, ierr)
-      end if
-
-
-!     create the distributed array filetype
-    
-      ndims = 3
-      order = MPI_ORDER_FORTRAN
-
-      array_of_gsizes(1) = 32
-      array_of_gsizes(2) = 32
-      array_of_gsizes(3) = 32
-
-      array_of_distribs(1) = MPI_DISTRIBUTE_BLOCK
-      array_of_distribs(2) = MPI_DISTRIBUTE_BLOCK
-      array_of_distribs(3) = MPI_DISTRIBUTE_BLOCK
-
-      array_of_dargs(1) = MPI_DISTRIBUTE_DFLT_DARG
-      array_of_dargs(2) = MPI_DISTRIBUTE_DFLT_DARG
-      array_of_dargs(3) = MPI_DISTRIBUTE_DFLT_DARG
-
-      do i=1, ndims
-         array_of_psizes(i) = 0
-      end do
-
-      call PMPI_DIMS_CREATE(nprocs, ndims, array_of_psizes, ierr)
-
-      call PMPI_TYPE_CREATE_DARRAY(nprocs, mynod, ndims,                &
-     &     array_of_gsizes, array_of_distribs, array_of_dargs,          &
-     &     array_of_psizes, order, MPI_INTEGER, newtype, ierr)
-
-      call PMPI_TYPE_COMMIT(newtype, ierr)
-
-!     initialize writebuf 
-
-      call PMPI_TYPE_SIZE(newtype, bufcount, ierr)
-      call PMPI_TYPE_SIZE(MPI_INTEGER, intsize, ierr)
-      bufcount = bufcount/intsize
-      do i=1, bufcount 
-         writebuf(i) = 1
-      end do
-
-      do i=1, FILESIZE
-         tmpbuf(i) = 0
-      end do
-
-      call PMPI_IRECV(tmpbuf, 1, newtype, mynod, 10, MPI_COMM_WORLD,    &
-     &     request, ierr)
-      call PMPI_SEND(writebuf, bufcount, MPI_INTEGER, mynod, 10,        &
-     &     MPI_COMM_WORLD, ierr)
-      call PMPI_WAIT(request, status, ierr)
-
-      j = 1
-      array_size = array_of_gsizes(1) * array_of_gsizes(2) *            &
-     &     array_of_gsizes(3)
-      do i=1, array_size
-         if (tmpbuf(i) .ne. 0) then
-            writebuf(j) = i
-            j = j + 1
-         end if
-      end do
-
-!     end of initialization
-
-!     write the array to the file
-
-      call PMPI_FILE_OPEN(MPI_COMM_WORLD, str,                          &
-     &     MPI_MODE_CREATE+MPI_MODE_RDWR, MPI_INFO_NULL, fh, ierr)
-
-      disp = 0 
-      call PMPI_FILE_SET_VIEW(fh, disp, MPI_INTEGER, newtype, "native", &
-     &     MPI_INFO_NULL, ierr)
-      call PMPI_FILE_WRITE_ALL(fh, writebuf, bufcount, MPI_INTEGER,     &
-     &     status, ierr)
-      call PMPI_FILE_CLOSE(fh, ierr)
-
-!    now read it back
-
-      call PMPI_FILE_OPEN(MPI_COMM_WORLD, str,                          &
-     &     MPI_MODE_CREATE+MPI_MODE_RDWR, MPI_INFO_NULL, fh, ierr)
- 
-      call PMPI_FILE_SET_VIEW(fh, disp, MPI_INTEGER, newtype, "native", &
-     &     MPI_INFO_NULL, ierr)
-      call PMPI_FILE_READ_ALL(fh, readbuf, bufcount, MPI_INTEGER,       &
-     &     status, ierr)
-      call PMPI_FILE_CLOSE(fh, ierr)
-
-!     check the data read
-      do i=1, bufcount
-         if (readbuf(i) .ne. writebuf(i)) then
-             errs = errs + 1
-             print *, 'Node ', mynod, '  readbuf ', readbuf(i),         &
-     &          '  writebuf ', writebuf(i), '  i', i
-         end if
-      end do
-
-      call PMPI_TYPE_FREE(newtype, ierr)
-      call MPI_Allreduce( errs, toterrs, 1, MPI_INTEGER, MPI_SUM,       &
-     $     MPI_COMM_WORLD, ierr )  
-      if (mynod .eq. 0) then
-        if( toterrs .gt. 0 ) then
-           print *, 'Found ', toterrs, ' errors'
-        else
-           print *, ' No Errors'
-        endif
-      endif
-
-      call PMPI_FINALIZE(ierr)
-
-      end
diff --git a/3rd-party/romio321/test/psimple.c b/3rd-party/romio321/test/psimple.c
deleted file mode 100644
index 1fc91f3e226..00000000000
--- a/3rd-party/romio321/test/psimple.c
+++ /dev/null
@@ -1,103 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*  
- *  (C) 2001 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include "mpi.h"
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-
-#define SIZE (65536)
-
-/* This is the same as simple.c, but uses the PMPI versions of all
-   MPI functions in order to test the profiling interface. */
-
-/* Each process writes to separate files and reads them back. 
-   The file name is taken as a command-line argument, and the process rank 
-   is appended to it. */ 
-
-int main(int argc, char **argv)
-{
-    int *buf, i, rank, nints, len;
-    char *filename, *tmp;
-    int errs=0, toterrs;
-    MPI_File fh;
-    MPI_Status status;
-
-    PMPI_Init(&argc,&argv);
-    PMPI_Comm_rank(MPI_COMM_WORLD, &rank);
-
-/* process 0 takes the file name as a command-line argument and 
-   broadcasts it to other processes */
-    if (!rank) {
-	i = 1;
-	while ((i < argc) && strcmp("-fname", *argv)) {
-	    i++;
-	    argv++;
-	}
-	if (i >= argc) {
-	    fprintf(stderr, "\n*#  Usage: simple -fname filename\n\n");
-	    PMPI_Abort(MPI_COMM_WORLD, 1);
-	}
-	argv++;
-	len = strlen(*argv);
-	filename = (char *) malloc(len+10);
-	strcpy(filename, *argv);
-	PMPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	PMPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-    else {
-	PMPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	filename = (char *) malloc(len+10);
-	PMPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-    
-
-    buf = (int *) malloc(SIZE);
-    nints = SIZE/sizeof(int);
-    for (i=0; i<nints; i++) buf[i] = rank*100000 + i;
-
-    /* each process opens a separate file called filename.'myrank' */
-    tmp = (char *) malloc(len+10);
-    strcpy(tmp, filename);
-    sprintf(filename, "%s.%d", tmp, rank);
-
-    PMPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
-		   MPI_INFO_NULL, &fh);
-    PMPI_File_write(fh, buf, nints, MPI_INT, &status);
-    PMPI_File_close(&fh);
-
-    /* reopen the file and read the data back */
-
-    for (i=0; i<nints; i++) buf[i] = 0;
-    PMPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, 
-                  MPI_INFO_NULL, &fh);
-    PMPI_File_read(fh, buf, nints, MPI_INT, &status);
-    PMPI_File_close(&fh);
-
-    /* check if the data read is correct */
-    for (i=0; i<nints; i++) {
-	if (buf[i] != (rank*100000 + i)) {
-	    errs++;
-	    fprintf(stderr, "Process %d: error, read %d, should be %d\n", rank, buf[i], rank*100000+i);
-	}
-    }
-
-    MPI_Allreduce( &errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
-    if (rank == 0) {
-	if( toterrs > 0) {
-	    fprintf( stderr, "Found %d errors\n", toterrs );
-	}
-	else {
-	    fprintf( stdout, " No Errors\n" );
-	}
-    }
-
-    free(buf);
-    free(filename);
-    free(tmp);
-
-    PMPI_Finalize();
-    return 0; 
-}
diff --git a/3rd-party/romio321/test/rtest.in b/3rd-party/romio321/test/rtest.in
deleted file mode 100644
index c6fd3058435..00000000000
--- a/3rd-party/romio321/test/rtest.in
+++ /dev/null
@@ -1,131 +0,0 @@
-#! /bin/sh
-#
-# Run some of the tests.  If any arguments are provided, pass them to the
-# test programs.
-#
-# -mvhome is needed for the ANL SP, and is ignored by others
-args=@MPIRUNARGS@
-device=@DEVICE@
-top_srcdir=@top_srcdir@
-srcdir=@srcdir@
-MPICH_VERSION=@MPICH_VERSION@
-STOPFILE=$HOME/.stopmpichtests
-mpirun=${MPIRUN:-"@MPIRUN@"}
-MAKE="@MAKE@"
-FILENAME=test
-#
-# Load basic procedures
-. ${top_srcdir}/runbase
-#
-# Set mpirun to the name/path of the mpirun program
-FindMPIRUN
-#
-#
-test_mpi2=@HAS_MPI2@
-runtests=1
-quiet=0
-makeeach=0
-writesummaryfile=no
-MAKE="@MAKE@"
-for arg in "$@" ; do
-    case $arg in 
-	-checkonly )
-	runtests=0
-	;;
-        -margs=*)
-	margs=`echo $arg | sed 's/-margs=//'`
-	args="$args $margs"
-	;;
-	-summaryfile=*)
-	writesummaryfile=yes
-	summaryfile=`echo A$arg | sed 's/A-summaryfile=//'`
-	;;
-	-small)
-	makeeach=1
-	;;
-	-fname=*)
-	FILENAME=`echo $arg|sed 's/-*fname=//'`
-	;;
-	-quiet)
-	shift
-	quiet=1
-	;;
-	-help|-u)
-	echo "runtests [-checkonly] [-margs='...']"
-	echo "run tests in this directory.  If -checkonly set, just run"
-	echo "the differences check (do NO rerun the test programs)."
-	echo "If -margs is used, these options are passed to mpirun."
-	echo "If -small is used, the examples are built, run, and deleted."
-	exit 1
-	;;
-	*)
-	if test -n "$arg" ; then
-   	    echo "runtests: Unknown argument ($arg)"
-	    exit 1
-        fi
-	;;
-    esac
-done
-
-# If the programs are not available, run make.
-if [ ! -x simple -a $makeeach = 0 -a $runtests = 1 ] ; then
-    $MAKE default
-fi
-
-testfiles=""
-if [ $runtests = 1 ] ; then
-
-
-RunTest simple 4 "**** Testing simple.c ****" "-fname $FILENAME"
-
-RunTest async 4 "**** Testing async.c ****" "-fname $FILENAME"
-
-RunTest atomicity 4 "**** Testing atomicity.out ****" "-fname $FILENAME"
-
-RunTest coll_test 4 "**** Testing coll_test.out ****" "-fname $FILENAME"
-
-RunTest excl 4 "**** Testing excl.c ****" "-fname $FILENAME"
-
-RunTest file_info 4 "**** Testing file_info.c ****" "-fname $FILENAME"
-
-RunTest i_noncontig 2 "**** Testing i_noncontig.c ****" "-fname $FILENAME"
-
-RunTest noncontig 2 "**** Testing noncontig.c ****" "-fname $FILENAME"
-
-RunTest noncontig_coll 2 "**** Testing noncontig_coll.c ****" "-fname $FILENAME"
-
-RunTest noncontig_coll2 4 "**** Testing noncontig_coll2.c ****" "-fname $FILENAME"
-
-RunTest misc 4 "**** Testing misc.c ****" "-fname $FILENAME"
-
-RunTest shared_fp 4 "**** Testing shared_fp.c ****" "-fname $FILENAME"
-
-RunTest split_coll 4 "**** Testing split_coll.c ****" "-fname $FILENAME"
-
-RunTest psimple 4 "**** Testing psimple.c ****" "-fname $FILENAME"
-
-RunTest error 1 "**** Testing error.c ****" "-fname $FILENAME"
-
-RunTest status 1 "**** Testing status.c ****" "-fname $FILENAME"
-
-#
-# Run Fortran tests ONLY if Fortran available
-if [ @HAS_FORTRAN@ = 1 ] ; then 
-    RunTest fmisc 4 "**** Testing fmisc.f ****" "-fname $FILENAME"
-
-    RunTest fcoll_test 4 "**** Testing fcoll_test.f ****" "-fname $FILENAME"
-
-    RunTest pfcoll_test 4 "**** Testing pfcoll_test.f ****" "-fname $FILENAME"
-fi
-
-else
-    # Just run checks
-    testfiles=`echo *.out`
-fi
-
-echo '*** Checking for differences from expected output ***'
-CheckAllOutput context.diff
-exit 0
-
-
-
diff --git a/3rd-party/romio321/test/runtests.in b/3rd-party/romio321/test/runtests.in
deleted file mode 100644
index 0e5e58fcd92..00000000000
--- a/3rd-party/romio321/test/runtests.in
+++ /dev/null
@@ -1,484 +0,0 @@
-#! /bin/sh
-prefix=@prefix@
-exec_prefix=@exec_prefix@
-bindir=@bindir@
-mpirun="@MPIRUN@"
-STOPFILE=${MPITEST_STOPTEST:-"$HOME/.stopmpichtests"}
-#
-# If the programs are not available, run make.
-runtests=1
-makeeach=0
-check_at_once=1
-quiet=0
-MAKE="@MAKE@"
-srcdir=@srcdir@
-check_canrun=0
-subset_only=0
-FILENAME=test
-# Using shifts should remove args from the list.
-for arg in "$@" ; do
-    case $arg in 
-	-checkonly)
-	shift
-	runtests=0
-	;;
-        -margs=*)
-	shift
-	margs=`echo $arg | sed 's/-margs=//'`
-	args="$args $margs"
-	;;
-	-echo)
-	shift
-	set -x
-	;;
-	-check)
-	check_canrun=1
-	;;
-	-quiet)
-	shift
-	quiet=1
-	;;
-	-small)
-	shift
-	makeeach=1
-	;;
-	-subset)
-	shift
-	subset_only=1
-	;;
-	-fname=*)
-	FILENAME=`echo $arg|sed 's/-*fname=//'`
-	;;
-	-atend)
-	shift
-	check_at_once=0
-	;;
-	-help|-u)
-	shift
-	echo "runtests [-fname=FILENAME] [-checkonly] [-atend] [-check]"
-	echo "run tests in this directory.  If -checkonly set, just run"
-	echo "the differences check (do NOT rerun the test programs)."
-	echo "If -small is used, the examples are built, run, and deleted."
-	echo "If -atend is used, the success of the tests is checked only"
-	echo "at the end of the test rather than also after each test."
-	echo "If -check is used, only a single simple test is run; this"
-	echo "is used to check that mpirun can run an MPI program."
-	echo "If -subset is used, we skip tests for atomicity and shared"
-	echo "file pointers which can fail on some distributed file systems" 
-	exit 1
-	;;
-	*)
-	if test -n "$arg" ; then
-	    echo "Passing remaining arguments to programs ($*)"
-	    break
-        fi
-	;;
-    esac
-done
-
-# MakeExe program-name
-MakeExe() {
-    if [ -s $STOPFILE ] ; then 
-        echo "Found stopfile; exiting"
-        exit 0
-    fi
-    if [ ! -x $1 ] ; then
-	$MAKE $1
-        if [ ! -x $1 ] ; then 
-	    echo "Could not build executable $1; aborting tests"
-	    exit 1
-        fi
-    fi
-}
-# CleanExe program-name
-CleanExe() {
-    if [ $makeeach = 1 ] ; then
-	rm -f $1 $1.o
-    fi
-}
-# Output marker
-OutTime() {
-    if [ $quiet = 0 ] ; then
-	if [ -z "$hostname" ] ; then
-	    hostname=`hostname`
-	fi
-	d=`date`
-	echo "$hostname : $d"
-    fi
-}
-
-# Do an "on the fly" check for problems.
-CheckOutput() {
-  bfile=$1
-  if [ $check_at_once = 1 ] ; then
-    stdfile=${srcdir}/std/$bfile.std
-    if [ ! -s $bfile.out ] ; then
-        echo "No output file $bfile.out!"
-    elif [ -s $stdfile ] ; then
-        stdfile2=${stdfile}2
-        # Handle Fortran systems that generate stop statements
-        rm -f ${bfile}.tout
-	grep -v 'FORTRAN STOP' ${bfile}.out > ${bfile}.tout
-        if diff -b $bfile.tout $stdfile > /dev/null ; then
-	    true
-	elif [ -s $stdfile2 ] ; then 
-	    # check for alternate in case configuration has fewer datatypes
-	    if diff -b $bfile.tout $stdfile2 > /dev/null ; then
-	        true
-            else
-	        echo "Differences in $bfile.out"
-	        diff -b $bfile.tout $stdfile
-	        nodiff=0
-	    fi
-        else
-	    echo "Differences in $bfile.out"
-	    diff -b $bfile.tout $stdfile
-	fi
-        rm -f ${bfile}.tout
-    else
-        echo "Can not find file $stdfile to compare against for test $bfile"
-    fi
-  fi
-}
-
-# nothing uses this function yet... just toying with the concept
-RunTest() {
-      OutTime
-      testfiles="$testfiles ${1}.out"
-      rm -rf ${1}.out
-      MakeExe simple
-      rm -rf $FILENAME*
-      echo "**** Testing ${1}.c ****"
-      mpirun -np 4 ./${1} -fname $FILENAME "$@" >>${1}.out 2>1
-      CheckOutput ${1} 
-      CleanExe ${1}
-}
-
-# If the programs are not available, run make.
-if [ ! -x simple -a $makeeach = 0 ] ; then
-    $MAKE default
-fi
-#
-testfiles=""
-if [ $runtests = 1 ] ; then
-echo '**** Testing I/O functions ****'
-#
-OutTime
-testfiles="$testfiles simple.out"
-\rm -f simple.out
-MakeExe simple
-\rm -f $FILENAME*
-echo '**** Testing simple.c ****'
-$mpirun -np 4 ./simple -fname $FILENAME
-# CheckOutput simple
-CleanExe simple
-#
-OutTime
-testfiles="$testfiles async.out"
-\rm -f async.out
-MakeExe async
-\rm -f $FILENAME*
-echo '**** Testing async.c ****'
-$mpirun -np 4 ./async -fname $FILENAME
-# CheckOutput async
-CleanExe async
-#
-OutTime
-testfiles="$testfiles async-multi.out"
-\rm -f async-multi.out
-MakeExe async-multiple
-\rm -f $FILENAME*
-echo '**** Testing async-multiple.c ****'
-$mpirun -np 4 ./async-multiple -fname $FILENAME
-# CheckOutput async-multiple
-CleanExe async-multiple
-#
-OutTime
-if [ $subset_only -eq 0 ] ; then 
-	testfiles="$testfiles atomicity.out"
-	\rm -f atomicity.out
-	MakeExe atomicity
-	\rm -f $FILENAME*
-	echo '**** Testing atomicity.c ****'
-	# Atomicity test recommends at least 8 processes (separate processors 
-	# even better)
-	$mpirun -np 4 ./atomicity -fname $FILENAME
-	# CheckOutput atomicity
-	CleanExe atomicity
-	#
-	OutTime
-fi
-testfiles="$testfiles coll_test.out"
-\rm -f coll_test.out
-MakeExe coll_test
-\rm -f $FILENAME*
-echo '**** Testing coll_test.c ****'
-$mpirun -np 4 ./coll_test -fname $FILENAME
-# CheckOutput coll_test
-CleanExe coll_test
-#
-OutTime
-testfiles="$testfiles excl.out"
-\rm -f excl.out
-MakeExe excl
-\rm -f $FILENAME*
-echo '**** Testing excl.c ****'
-$mpirun -np 4 ./excl -fname $FILENAME
-# CheckOutput excl
-CleanExe excl
-#
-OutTime
-testfiles="$testfiles file_info.out"
-\rm -f file_info.out
-MakeExe file_info
-\rm -f $FILENAME*
-echo '**** Testing file_info.c ****'
-$mpirun -np 4 ./file_info -fname $FILENAME
-# CheckOutput file_info
-CleanExe file_info
-#
-OutTime
-testfiles="$testfiles i_noncontig.out"
-\rm -f i_noncontig.out
-MakeExe i_noncontig
-\rm -f $FILENAME*
-echo '**** Testing i_noncontig.c ****'
-$mpirun -np 2 ./i_noncontig -fname $FILENAME
-# CheckOutput i_noncontig
-CleanExe i_noncontig
-#
-OutTime
-testfiles="$testfiles noncontig.out"
-\rm -f noncontig.out
-MakeExe noncontig
-\rm -f $FILENAME*
-echo '**** Testing noncontig.c ****'
-$mpirun -np 2 ./noncontig -fname $FILENAME
-# CheckOutput noncontig
-CleanExe noncontig
-#
-OutTime
-testfiles="$testfiles noncontig_coll.out"
-\rm -f noncontig_coll.out
-MakeExe noncontig_coll
-\rm -f $FILENAME*
-echo '**** Testing noncontig_coll.c ****'
-$mpirun -np 2 ./noncontig_coll -fname $FILENAME
-# CheckOutput noncontig_coll
-CleanExe noncontig_coll
-#
-OutTime
-testfiles="$testfiles noncontig_coll2.out"
-\rm -f noncontig_coll2.out
-MakeExe noncontig_coll2
-\rm -f $FILENAME*
-echo '**** Testing noncontig_coll2.c ****'
-$mpirun -np 4 ./noncontig_coll2 -fname $FILENAME
-# CheckOutput noncontig_coll2
-CleanExe noncontig_coll2
-MakeExe aggregation1
-echo '**** Testing aggregation1 ****'
-$mpirun -np 4 ./aggregation1 -h -f $FILENAME
-# CheckOutput aggregation1
-CleanExe aggregation1
-MakeExe aggregation2
-echo '**** Testing aggregation2 ****'
-$mpirun -np 4 ./aggregation2 $FILENAME
-# CheckOutput aggregation2
-CleanExe aggregation2
-MakeExe hindexed
-echo '**** Testing hindexed ****'
-$mpirun -np 4 ./hindexed $FILENAME
-# CheckOutput hindexed
-CleanExe hindexed
-#
-#echo '**** Testing write_all_test (run 1)****'
-#$mpirun -np 4 ./write_all_test -nzp 2 -zplace 2 -nzw 2 -naw 2 -size 100 \
-#	-offm 8 -hints romio_cb_write enable -fname $FILENAME
-#echo '**** Testing write_all_test (run 2)****'
-#$mpirun -np 4 ./write_all_test -nzp 2 -zplace 2 -nzw 2 -naw 2 -size 100 \
-#	-offm 4 -hints romio_cb_write enable -fname $FILENAME
-#CleanExe write_all_test
-OutTime
-if [ $subset_only -eq 0 ] ; then 
-	testfiles="$testfiles misc.out"
-	\rm -f misc.out
-	MakeExe misc
-	\rm -f $FILENAME*
-	echo '**** Testing misc.c ****'
-	$mpirun -np 4 ./misc -fname $FILENAME
-	# CheckOutput misc
-	CleanExe misc
-	#
-	OutTime
-fi
-if [ $subset_only -eq 0 ] ; then
-	testfiles="$testfiles shared_fp.out"
-	\rm -f shared_fp.out
-	MakeExe shared_fp
-	\rm -f $FILENAME*
-	echo '**** Testing shared_fp.c ****'
-	$mpirun -np 4 ./shared_fp -fname $FILENAME
-	# CheckOutput shared_fp
-	CleanExe shared_fp
-	#
-	OutTime
-	testfiles="$testfiles ordered_fp.out"
-	\rm -f ordered_fp.out
-	MakeExe ordered_fp
-	\rm -f $FILENAME*
-	echo '**** Testing ordered_fp.c ****'
-	$mpirun -np 4 ./ordered_fp -fname $FILENAME
-	CleanExe ordered_fp
-	#
-	OutTime
-fi
-testfiles="$testfiles split_coll.out"
-\rm -f split_coll.out
-MakeExe split_coll
-\rm -f $FILENAME*
-echo '**** Testing split_coll.c ****'
-$mpirun -np 4 ./split_coll -fname $FILENAME
-# CheckOutput split_coll
-CleanExe split_coll
-#
-OutTime
-testfiles="$testfiles psimple.out"
-\rm -f psimple.out
-MakeExe psimple
-\rm -f $FILENAME*
-echo '**** Testing psimple.c ****'
-$mpirun -np 4 ./psimple -fname $FILENAME
-# CheckOutput psimple
-CleanExe psimple
-#
-OutTime
-testfiles="$testfiles error.out"
-\rm -f error.out
-MakeExe error
-\rm -f $FILENAME*
-echo '**** Testing error.c ****'
-$mpirun -np 1 ./error -fname $FILENAME
-# CheckOutput error
-CleanExe error
-#
-OutTime
-testfiles="$testfiles status.out"
-\rm -f status.out
-MakeExe status
-\rm -f $FILENAME*
-echo '**** Testing status.c ****'
-# Some systems have a status program.
-$mpirun -np 1 ./status -fname $FILENAME
-# CheckOutput status
-CleanExe status
-testfiles="$testfiles types_with_zeros.out"
-\rm -f types_with_zeros.out
-MakeExe types_with_zeros
-\rm -f $FILENAME*
-echo '**** Testing types_with_zeros ****'
-$mpirun -np 2 ./types_with_zeros $FILENAME
-CleanExe types_with_zeros
-testfiles="$testfiles darray_read.out"
-\rm -f darray_read.out
-MakeExe darray_read
-\rm -f $FILENAME*
-echo '**** Testing darray_read ****'
-$mpirun -np 4 ./darray_read $FILENAME
-CleanExe darray_read
-MakeExe syshints
-echo '**** Testing syshints ****'
-$mpirun -np 1 ./syshints $srcdir/test_hintfile
-CleanExe syshints
-
-#
-if [ @NOF77@ = 0 ] ; then 
-    echo ""
-    echo "FORTRAN TESTS"
-    OutTime
-    if [ $subset_only -eq 0 ] ; then
-	    testfiles="$testfiles fmisc.out"
-	    \rm -f fmisc.out
-	    MakeExe fmisc
-	    \rm -f $FILENAME*
-	    echo '**** Testing fmisc.f ****'
-	    $mpirun -np 4 ./fmisc -fname $FILENAME
-	    # CheckOutput fmisc
-	    CleanExe fmisc
-	    #
-	    OutTime
-    fi
-    testfiles="$testfiles fcoll_test.out"
-    \rm -f fcoll_test.out
-    MakeExe fcoll_test
-    \rm -f $FILENAME*
-    echo '**** Testing fcoll_test.f ****'
-    $mpirun -np 4 ./fcoll_test -fname $FILENAME
-    # CheckOutput fcoll_test
-    CleanExe fcoll_test
-#
-    OutTime
-    testfiles="$testfiles pfcoll_test.out"
-    \rm -f pfcoll_test.out
-    MakeExe pfcoll_test
-    \rm -f $FILENAME*
-    echo '**** Testing pfcoll_test.f ****'
-    $mpirun -np 4 ./pfcoll_test -fname $FILENAME
-    # CheckOutput pfcoll_test
-    CleanExe pfcoll_test
-#
-echo ""
-fi
-#
-else
-    # Just run checks
-    testfiles=`echo *.out`
-    if test "$testfiles" = "*.out" ; then
-	echo "No output files remain from previous test!"
-	exit 1
-    fi
-fi
-#
-# echo '*** Checking for differences from expected output ***'
-# rm -f iotests.diff
-# nodiff=1
-# for file in $testfiles ; do
-#     stdfile="${srcdir}/std/`basename $file .out`.std"
-#     # if basename is sendrecv or isndrcv, then we may want to test 
-#     # with .std2 as well.  We should really separate out the long double
-#     # tests ...
-#     if [ -s $stdfile ] ; then
-# 	stdfile2=${stdfile}2
-#         # Handle Fortran systems that generate stop statements
-# 	bfile="`basename $file .out`.tout"
-#         rm -f ${bfile}
-# 	grep -v 'FORTRAN STOP' ${file} > ${bfile}
-#         if diff -b $bfile $stdfile > /dev/null ; then
-# 	    true
-# 	elif [ -s $stdfile2 ] ; then 
-# 	    # check for alternate in case configuration has fewer datatypes
-# 	    if diff -b $bfile $stdfile2 > /dev/null ; then
-# 	        true
-#             else
-# 	        echo "Differences in `basename $file .out`" >> iotests.diff
-# 	        diff -b $bfile $stdfile >> iotests.diff
-# 	        nodiff=0
-# 	    fi
-#         else
-# 	    echo "Differences in `basename $file .out`" >> iotests.diff
-# 	    diff -b $bfile $stdfile >> iotests.diff
-# 	    nodiff=0
-# 	fi
-#         rm -f ${bfile}
-#     else
-#         echo "Can not find file $stdfile to compare against for test `basename $file .out`"
-# 	nodiff=0
-#     fi
-# done
-# if [ -s iotests.diff ] ; then
-#    cat iotests.diff
-# elif [ $nodiff = 1 ] ; then
-#    echo "-- No differences found; test successful"
-# fi
- exit 0
- 
diff --git a/3rd-party/romio321/test/shared_fp.c b/3rd-party/romio321/test/shared_fp.c
deleted file mode 100644
index df41bdd818d..00000000000
--- a/3rd-party/romio321/test/shared_fp.c
+++ /dev/null
@@ -1,122 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*  
- *  (C) 2001 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include "mpi.h"
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-
-#define COUNT 1024
-
-void handle_error(int errcode, const char *str);
-
-void handle_error(int errcode, const char *str) 
-{
-	char msg[MPI_MAX_ERROR_STRING];
-	int resultlen;
-	MPI_Error_string(errcode, msg, &resultlen);
-	fprintf(stderr, "%s: %s\n", str, msg);
-	MPI_Abort(MPI_COMM_WORLD, 1);
-}
-
-/* tests shared file pointer functions */
-
-int main(int argc, char **argv)
-{
-    int *buf, i, rank, nprocs, len, sum, global_sum;
-    int errs=0, toterrs, errcode;
-    char *filename;
-    MPI_File fh;
-    MPI_Status status;
-
-    MPI_Init(&argc,&argv);
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-
-/* process 0 takes the file name as a command-line argument and 
-   broadcasts it to other processes */
-    if (!rank) {
-	i = 1;
-	while ((i < argc) && strcmp("-fname", *argv)) {
-	    i++;
-	    argv++;
-	}
-	if (i >= argc) {
-	    fprintf(stderr, "\n*#  Usage: shared_fp -fname filename\n\n");
-	    MPI_Abort(MPI_COMM_WORLD, 1);
-	}
-	argv++;
-	len = strlen(*argv);
-	filename = (char *) malloc(len+10);
-	strcpy(filename, *argv);
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	MPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-    else {
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	filename = (char *) malloc(len+10);
-	MPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-    
-    buf = (int *) malloc(COUNT * sizeof(int));
-
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
-
-    for (i=0; i<COUNT; i++) buf[i] = COUNT*rank + i;
-
-    errcode = MPI_File_open(MPI_COMM_WORLD, filename, 
-		    MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_open");
-    }
-
-    errcode = MPI_File_write_shared(fh, buf, COUNT, MPI_INT, &status);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_write_shared");
-    }
-
-    for (i=0; i<COUNT; i++) buf[i] = 0;
-
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    errcode = MPI_File_seek_shared(fh, 0, MPI_SEEK_SET);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_seek_shared");
-    }
-
-    errcode = MPI_File_read_shared(fh, buf, COUNT, MPI_INT, &status);
-    if (errcode != MPI_SUCCESS) {
-	    handle_error(errcode, "MPI_File_read_shared");
-    }
-
-    MPI_File_close(&fh);
-
-    sum = 0;
-    for (i=0; i<COUNT; i++) sum += buf[i];
-
-    MPI_Allreduce(&sum, &global_sum, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
-
-    if (global_sum != (((COUNT*nprocs - 1)*(COUNT*nprocs))/2)) {
-	errs++;
-	fprintf(stderr, "Error: sum %d, global_sum %d, %d\n", 
-		sum, global_sum,(((COUNT*nprocs - 1)*(COUNT*nprocs))/2));
-    }
-    
-    free(buf);
-    free(filename);
-
-    MPI_Allreduce( &errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
-    if (rank == 0) {
-	if( toterrs > 0) {
-	    fprintf( stderr, "Found %d errors\n", toterrs );
-	}
-	else {
-	    fprintf( stdout, " No Errors\n" );
-	}
-    }
-
-    MPI_Finalize();
-    return 0; 
-}
diff --git a/3rd-party/romio321/test/simple.c b/3rd-party/romio321/test/simple.c
deleted file mode 100644
index 2e90c9dca99..00000000000
--- a/3rd-party/romio321/test/simple.c
+++ /dev/null
@@ -1,119 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*  
- *  (C) 2001 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include "mpi.h"
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-
-#define SIZE (65536)
-
-static void handle_error(int errcode, const char *str)
-{
-	char msg[MPI_MAX_ERROR_STRING];
-	int resultlen;
-	MPI_Error_string(errcode, msg, &resultlen);
-	fprintf(stderr, "%s: %s\n", str, msg);
-	MPI_Abort(MPI_COMM_WORLD, 1);
-}
-/* Each process writes to separate files and reads them back. 
-   The file name is taken as a command-line argument, and the process rank 
-   is appended to it. */ 
-
-int main(int argc, char **argv)
-{
-    int *buf, i, rank, nints, len;
-    char *filename, *tmp;
-    int  errs = 0, toterrs, errcode;
-    MPI_File fh;
-    MPI_Status status;
-
-    MPI_Init(&argc,&argv);
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-
-/* process 0 takes the file name as a command-line argument and 
-   broadcasts it to other processes */
-    if (!rank) {
-	i = 1;
-	while ((i < argc) && strcmp("-fname", *argv)) {
-	    i++;
-	    argv++;
-	}
-	if (i >= argc) {
-	    fprintf(stderr, "\n*#  Usage: simple -fname filename\n\n");
-	    MPI_Abort(MPI_COMM_WORLD, 1);
-	}
-	argv++;
-	len = strlen(*argv);
-	filename = (char *) malloc(len+10);
-	strcpy(filename, *argv);
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	MPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-    else {
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	filename = (char *) malloc(len+10);
-	MPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-    
-
-    buf = (int *) malloc(SIZE);
-    nints = SIZE/sizeof(int);
-    for (i=0; i<nints; i++) buf[i] = rank*100000 + i;
-
-    /* each process opens a separate file called filename.'myrank' */
-    tmp = (char *) malloc(len+10);
-    strcpy(tmp, filename);
-    sprintf(filename, "%s.%d", tmp, rank);
-
-    errcode = MPI_File_open(MPI_COMM_SELF, filename, 
-		    MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
-    if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_open(1)");
-
-    errcode = MPI_File_write(fh, buf, nints, MPI_INT, &status);
-    if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_write");
-
-    errcode = MPI_File_close(&fh);
-    if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_clode (1)");
-
-    /* reopen the file and read the data back */
-
-    for (i=0; i<nints; i++) buf[i] = 0;
-    errcode = MPI_File_open(MPI_COMM_SELF, filename, 
-		    MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
-    if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_open(2)");
-
-    errcode = MPI_File_read(fh, buf, nints, MPI_INT, &status);
-    if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_read");
-
-    errcode = MPI_File_close(&fh);
-    if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_close(2)");
-
-    /* check if the data read is correct */
-    for (i=0; i<nints; i++) {
-	if (buf[i] != (rank*100000 + i)) {
-	    errs++;
-	    fprintf(stderr, "Process %d: error, read %d, should be %d\n", 
-		    rank, buf[i], rank*100000+i);
-	}
-    }
-
-    MPI_Allreduce( &errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
-    if (rank == 0) {
-	if( toterrs > 0) {
-	    fprintf( stderr, "Found %d errors\n", toterrs );
-	}
-	else {
-	    fprintf( stdout, " No Errors\n" );
-	}
-    }
-
-    free(buf);
-    free(filename);
-    free(tmp);
-
-    MPI_Finalize();
-    return 0; 
-}
diff --git a/3rd-party/romio321/test/split_coll.c b/3rd-party/romio321/test/split_coll.c
deleted file mode 100644
index cc3d039b569..00000000000
--- a/3rd-party/romio321/test/split_coll.c
+++ /dev/null
@@ -1,160 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*  
- *  (C) 2001 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include "mpi.h"
-#include <stdlib.h>
-#include <string.h>
-#include <stdio.h>
-
-/* A 32^3 array. For other array sizes, change array_of_gsizes below. */
-
-/* Uses split collective I/O. Writes a 3D block-distributed array to a file
-   corresponding to the global array in row-major (C) order, reads it
-   back, and checks that the data read is correct. */
-
-/* The file name is taken as a command-line argument. */
-
-/* Note that the file access pattern is noncontiguous. */
-   
-int main(int argc, char **argv)
-{
-    MPI_Datatype newtype;
-    int i, ndims, array_of_gsizes[3], array_of_distribs[3];
-    int order, nprocs, j, len;
-    int array_of_dargs[3], array_of_psizes[3];
-    int *readbuf, *writebuf, mynod, *tmpbuf, array_size;
-    MPI_Count bufcount;
-    int errs=0, toterrs;
-    char *filename;
-    MPI_File fh;
-    MPI_Status status;
-    MPI_Request request;
-
-    MPI_Init(&argc,&argv);
-    MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
-    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
-
-/* process 0 takes the file name as a command-line argument and 
-   broadcasts it to other processes */
-    if (!mynod) {
-	i = 1;
-	while ((i < argc) && strcmp("-fname", *argv)) {
-	    i++;
-	    argv++;
-	}
-	if (i >= argc) {
-	    fprintf(stderr, "\n*#  Usage: coll_test -fname filename\n\n");
-	    MPI_Abort(MPI_COMM_WORLD, 1);
-	}
-	argv++;
-	len = strlen(*argv);
-	filename = (char *) malloc(len+1);
-	strcpy(filename, *argv);
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-    else {
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	filename = (char *) malloc(len+1);
-	MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-
-
-/* create the distributed array filetype */
-    ndims = 3;
-    order = MPI_ORDER_C;
-
-    array_of_gsizes[0] = 32;
-    array_of_gsizes[1] = 32;
-    array_of_gsizes[2] = 32;
-
-    array_of_distribs[0] = MPI_DISTRIBUTE_BLOCK;
-    array_of_distribs[1] = MPI_DISTRIBUTE_BLOCK;
-    array_of_distribs[2] = MPI_DISTRIBUTE_BLOCK;
-
-    array_of_dargs[0] = MPI_DISTRIBUTE_DFLT_DARG;
-    array_of_dargs[1] = MPI_DISTRIBUTE_DFLT_DARG;
-    array_of_dargs[2] = MPI_DISTRIBUTE_DFLT_DARG;
-
-    for (i=0; i<ndims; i++) array_of_psizes[i] = 0;
-    MPI_Dims_create(nprocs, ndims, array_of_psizes);
-
-    MPI_Type_create_darray(nprocs, mynod, ndims, array_of_gsizes, 
-			   array_of_distribs, array_of_dargs,
-			   array_of_psizes, order, MPI_INT, &newtype);
-    MPI_Type_commit(&newtype);
-
-/* initialize writebuf */
-
-    MPI_Type_size_x(newtype, &bufcount);
-    bufcount = bufcount/sizeof(int);
-    writebuf = (int *) malloc(bufcount * sizeof(int));
-    for (i=0; i<bufcount; i++) writebuf[i] = 1;
-
-    array_size = array_of_gsizes[0]*array_of_gsizes[1]*array_of_gsizes[2];
-    tmpbuf = (int *) calloc(array_size, sizeof(int));
-    MPI_Irecv(tmpbuf, 1, newtype, mynod, 10, MPI_COMM_WORLD, &request);
-    MPI_Send(writebuf, bufcount, MPI_INT, mynod, 10, MPI_COMM_WORLD);
-    MPI_Wait(&request, &status);
-
-    j = 0;
-    for (i=0; i<array_size; i++)
-	if (tmpbuf[i]) {
-	    writebuf[j] = i;
-	    j++;
-	}
-    free(tmpbuf);
-
-    if (j != bufcount) {
-	fprintf(stderr, "Error in initializing writebuf on process %d\n", mynod);
-	MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-/* end of initialization */
-
-    /* write the array to the file */
-    MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, 
-                  MPI_INFO_NULL, &fh);
-    MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", MPI_INFO_NULL);
-    MPI_File_write_all_begin(fh, writebuf, bufcount, MPI_INT);
-    MPI_File_write_all_end(fh, writebuf, &status);
-    MPI_File_close(&fh);
-
-
-    /* now read it back */
-    readbuf = (int *) malloc(bufcount * sizeof(int));
-    MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, 
-                  MPI_INFO_NULL, &fh);
-    MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", MPI_INFO_NULL);
-    MPI_File_read_all_begin(fh, readbuf, bufcount, MPI_INT);
-    MPI_File_read_all_end(fh, readbuf, &status);
-    MPI_File_close(&fh);
-
-    /* check the data read */
-    for (i=0; i<bufcount; i++) {
-	if (readbuf[i] != writebuf[i]) {
-	    errs++;
-	    fprintf(stderr, "Process %d, readbuf %d, writebuf %d, i %d\n", 
-		    mynod, readbuf[i], writebuf[i], i);
-	}
-    }
-
-    MPI_Allreduce( &errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
-    if (mynod == 0) {
-	if( toterrs > 0) {
-	    fprintf( stderr, "Found %d errors\n", toterrs );
-	}
-	else {
-	    fprintf( stdout, " No Errors\n" );
-	}
-    }
-
-    MPI_Type_free(&newtype);
-    free(readbuf);
-    free(writebuf);
-    free(filename);
-
-    MPI_Finalize();
-    return 0;
-}
diff --git a/3rd-party/romio321/test/status.c b/3rd-party/romio321/test/status.c
deleted file mode 100644
index d64d5b2b0e0..00000000000
--- a/3rd-party/romio321/test/status.c
+++ /dev/null
@@ -1,93 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*  
- *  (C) 2001 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include "mpi.h"
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-
-#define SIZE (65536)
-
-/* Checks if the status objects is filled correctly by I/O functions */ 
-
-int main(int argc, char **argv)
-{
-    int *buf, i, rank, nints, len, count, elements;
-    char *filename, *tmp;
-    int errs=0, toterrs;
-    MPI_File fh;
-    MPI_Status status;
-
-    MPI_Init(&argc,&argv);
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-
-/* process 0 takes the file name as a command-line argument and 
-   broadcasts it to other processes */
-    if (!rank) {
-	i = 1;
-	while ((i < argc) && strcmp("-fname", *argv)) {
-	    i++;
-	    argv++;
-	}
-	if (i >= argc) {
-	    fprintf(stderr, "\n*#  Usage: simple -fname filename\n\n");
-	    MPI_Abort(MPI_COMM_WORLD, 1);
-	}
-	argv++;
-	len = strlen(*argv);
-	filename = (char *) malloc(len+10);
-	strcpy(filename, *argv);
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	MPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-    else {
-	MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
-	filename = (char *) malloc(len+10);
-	MPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
-    }
-    
-    buf = (int *) malloc(SIZE);
-    nints = SIZE/sizeof(int);
-
-    /* each process opens a separate file called filename.'myrank' */
-    tmp = (char *) malloc(len+10);
-    strcpy(tmp, filename);
-    sprintf(filename, "%s.%d", tmp, rank);
-
-    MPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
-		   MPI_INFO_NULL, &fh);
-    MPI_File_write(fh, buf, nints, MPI_INT, &status);
-
-    MPI_Get_count(&status, MPI_INT, &count);
-    MPI_Get_elements(&status, MPI_INT, &elements);
-    if (!rank) {
-	if (count != nints) {
-	    errs++;
-	    printf("count = %d, should be %d\n", count, nints);
-	}
-	if (elements != nints) {
-	    errs++;
-	    printf("elements = %d, should be %d\n", elements, nints);
-	}
-    }
-
-    MPI_File_close(&fh);
-
-    MPI_Allreduce( &errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
-    if (rank == 0) {
-	if( toterrs > 0) {
-	    fprintf( stderr, "Found %d errors\n", toterrs );
-	}
-	else {
-	    fprintf( stdout, " No Errors\n" );
-	}
-    }
-    free(buf);
-    free(filename);
-    free(tmp);
-
-    MPI_Finalize();
-    return 0; 
-}
diff --git a/3rd-party/romio321/test/test_hintfile b/3rd-party/romio321/test/test_hintfile
deleted file mode 100644
index edb72b0d6cb..00000000000
--- a/3rd-party/romio321/test/test_hintfile
+++ /dev/null
@@ -1,6 +0,0 @@
-romio_cb_read enable
-# multiple info keys are strage but not an error.  Note only the first one will
-# be processed
-ind_rd_buffer_size 49
-ind_rd_buffer_size 60
-romio_no_indep_rw true
diff --git a/3rd-party/romio321/test/types_with_zeros.c b/3rd-party/romio321/test/types_with_zeros.c
deleted file mode 100644
index e2f7799605b..00000000000
--- a/3rd-party/romio321/test/types_with_zeros.c
+++ /dev/null
@@ -1,145 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *  (C) 2014 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <sys/types.h>
-#include <sys/uio.h>
-
-#include <mpi.h>
-
-#define MAXLEN 9
-
-static void handle_error(int errcode, const char *str)
-{
-	char msg[MPI_MAX_ERROR_STRING];
-	int resultlen;
-	MPI_Error_string(errcode, msg, &resultlen);
-	fprintf(stderr, "%s: %s\n", str, msg);
-	MPI_Abort(MPI_COMM_WORLD, 1);
-}
-
-enum {
-    INDEXED,
-    HINDEXED,
-    STRUCT
-} testcases;
-
-static int test_indexed_with_zeros(char *filename, int testcase)
-{
-    int i, rank, np, buflen, num, err, nr_errors=0;
-    int  nelms[MAXLEN], buf[MAXLEN], indices[MAXLEN], blocklen[MAXLEN];
-    MPI_File fh;
-    MPI_Status status;
-    MPI_Datatype filetype;
-    MPI_Datatype types[MAXLEN];
-    MPI_Aint addrs[MAXLEN];
-
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-    MPI_Comm_size(MPI_COMM_WORLD, &np);
-
-    /* set up the number of integers to write in each iteration */
-    for (i=0; i<MAXLEN; i++) nelms[i] = 0;
-    if (rank == 0) nelms[4]=nelms[5]=nelms[7]=1;
-    if (rank == 1) nelms[0]=nelms[1]=nelms[2]=nelms[3]=nelms[6]=nelms[8]=1;
-
-    /* pre-fill the file with integers -999 */
-    if (rank == 0) {
-        for (i=0; i<MAXLEN; i++) buf[i] = -999;
-	err =MPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_CREATE|MPI_MODE_WRONLY,
-		MPI_INFO_NULL, &fh);
-	if (err != MPI_SUCCESS) handle_error(err, "MPI_File_open");
-        err = MPI_File_write(fh, buf, MAXLEN, MPI_INT, &status);
-	if (err != MPI_SUCCESS) handle_error(err, "MPI_File_write");
-        err = MPI_File_close(&fh);
-	if (err != MPI_SUCCESS) handle_error(err, "MPI_File_close");
-    }
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    /* define a filetype with spurious leading zeros */
-    buflen = num = 0;
-    for (i=0; i<MAXLEN; i++) {
-        buflen       += nelms[i];
-        indices[num]  = i;
-        addrs[num] = i*sizeof(int);
-        blocklen[num] = nelms[i];
-        types[num] = MPI_INT;
-        num++;
-    }
-    switch (testcase) {
-	case INDEXED:
-	    MPI_Type_indexed(num, blocklen, indices, MPI_INT, &filetype);
-	    break;
-	case HINDEXED:
-	    MPI_Type_hindexed(num, blocklen, addrs, MPI_INT, &filetype);
-	    break;
-	case STRUCT:
-	    MPI_Type_create_struct(num, blocklen, addrs, types, &filetype);
-	    break;
-	default:
-	    fprintf(stderr, "unknown testcase!\n");
-	    return(-100);
-
-    }
-
-    MPI_Type_commit(&filetype);
-
-    /* initialize write buffer and write to file*/
-    for (i=0; i<MAXLEN; i++) buf[i] = 1;
-    err =MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
-    if (err != MPI_SUCCESS) handle_error(err, "MPI_File_open");
-    err = MPI_File_set_view(fh, 0, MPI_INT, filetype, "native", MPI_INFO_NULL);
-    if (err != MPI_SUCCESS) handle_error(err, "MPI_File_set_view");
-    err = MPI_File_write_all(fh, buf, buflen, MPI_INT, &status);
-    if (err != MPI_SUCCESS) handle_error(err, "MPI_File_write_all");
-    MPI_Type_free(&filetype);
-    err = MPI_File_close(&fh);
-    if (err != MPI_SUCCESS) handle_error(err, "MPI_File_close");
-
-    /* read back and check */
-    if (rank == 0) {
-        err = MPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_RDONLY, MPI_INFO_NULL, &fh);
-	if (err != MPI_SUCCESS) handle_error(err, "MPI_File_open");
-        err = MPI_File_read(fh,buf, MAXLEN, MPI_INT, &status);
-	if (err != MPI_SUCCESS) handle_error(err, "MPI_File_read");
-        err = MPI_File_close(&fh);
-	if (err != MPI_SUCCESS) handle_error(err, "MPI_File_close");
-        for (i=0; i<MAXLEN; i++) {
-            if (buf[i] < 0) {
-		nr_errors++;
-                printf("Error: unexpected value for case %d at buf[%d] == %d\n",
-			testcase,i,buf[i]);
-	    }
-	}
-    }
-    return nr_errors;
-}
-
-int main(int argc, char **argv)
-{
-
-    int nr_errors, rank, np;
-
-    MPI_Init(&argc, &argv);
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-    MPI_Comm_size(MPI_COMM_WORLD, &np);
-
-    if (np != 2) {
-        if (rank == 0) fprintf(stderr,"Must run on 2 MPI processes\n");
-        MPI_Finalize(); return 1;
-    }
-    nr_errors = test_indexed_with_zeros(argv[1], INDEXED);
-    nr_errors += test_indexed_with_zeros(argv[1], HINDEXED);
-    nr_errors += test_indexed_with_zeros(argv[1], STRUCT);
-
-    if (rank == 0 && nr_errors == 0) printf(" No Errors\n");
-
-    MPI_Finalize();
-    return 0;
-}
-
diff --git a/3rd-party/romio321/util/nfslock.c b/3rd-party/romio321/util/nfslock.c
deleted file mode 100644
index 69d5e7995c5..00000000000
--- a/3rd-party/romio321/util/nfslock.c
+++ /dev/null
@@ -1,72 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*  
- *  (C) 2004 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include <stdio.h>
-#include <fcntl.h>
-#include <errno.h>
-#include <unistd.h>
-
-/*
- * This program tests to see if fcntl returns success when asked to 
- * establish a file lock.  This test is intended for use on file systems
- * such as NFS that may not implement file locks.  ROMIO makes use
- * of file locks to implement certain operations, and may not work
- * properly if file locks are not available.  
- *
- * This is a simple test and has at least two limitations:
- * 
- * 1. Some implementations of NFS are known to return success for 
- * setting a file lock when in fact no lock has been set.  This
- * test will not detect such erroneous implementations of NFS
- *
- * 2. Some implementations will hang (enter and wait indefinitately)
- * within the fcntl call.  This program will also hang in that case.
- * Under normal conditions, this program should only take a few seconds to 
- * run.
- *
- * The program prints a message showing the success or failure of
- * setting the file lock and sets the return status to 0 on success and
- * non-zero on failure.  If there is a failure, the system routine
- * perror is also called to explain the reason.
- */
-
-/* style: allow:printf:2 sig:0 */
-
-int main( int argc, char *argv[] )
-{
-    struct flock lock;
-    int fd, err;
-    char *filename;
-
-    /* Set the filename.  Either arg[1] or conftest.dat */
-    if (argc > 1 && argv[1]) {
-      filename = argv[1];
-    }
-    else {
-      filename = "conftest.dat";
-    }
-
-       
-    lock.l_type   = F_WRLCK;
-    lock.l_start  = 0;
-    lock.l_whence = SEEK_SET;
-    lock.l_len    = 100;
-
-    fd = open(filename, O_RDWR | O_CREAT, 0644);
-
-    err = fcntl(fd, F_SETLKW, &lock);
-
-    if (err) {
-      printf( "Failed to set a file lock on %s\n", filename );
-      perror( "Reason " );
-    }
-    else {
-      printf( "fcntl claims success in setting a file lock on %s\n", filename );
-    }
-   /* printf("err = %d, errno = %d\n", err, errno); */
-    close(fd);
-    unlink( filename );
-    return err;
-}
diff --git a/3rd-party/romio321/util/romioinstall.in b/3rd-party/romio321/util/romioinstall.in
deleted file mode 100644
index 95a63092315..00000000000
--- a/3rd-party/romio321/util/romioinstall.in
+++ /dev/null
@@ -1,220 +0,0 @@
-#!/bin/sh
-#
-# This script needs to be built by configure.
-#
-# This is a script to install ROMIO. It can be invoked with 
-#    make install
-#
-#    (if you used -prefix at configure time) or,
-#
-#    make install prefix=/usr/local/romio  (or whatever directory you like)
-#
-# in the top-level romio directory
-#
-PREFIX=@prefix@
-ROMIO_HOME=@ROMIO_HOME@
-ARCH=@ARCH@
-RANLIB=@RANLIB@
-TOP_BUILD_DIR=@top_build_dir@
-# LAM Do we want to install or uninstall?
-WANT_INSTALL=1
-#
-# Default paths (set at configure time)
-prefix=@prefix@
-exec_prefix=@exec_prefix@
-bindir=@bindir@
-sbindir=@sbindir@
-includedir=@includedir@
-#sysconfdir=@sysconfdir@
-libdir=@libdir@
-#sharedlib_dir=@sharedlib_dir@
-mandir=@mandir@
-htmldir=@htmldir@
-# datarootdir required as of autoconf 2.60
-datarootdir=@datatrootdir@
-#datadir=@datadir@
-docdir=@docdir@
-## Location of sources
-#top_srcdir=@top_srcdir@
-#
-# File access mode
-MODE=0644
-XMODE=0755
-replace=1
-verbose=0
-
-failmode=soft
-
-# Shell procedures to copy files and create directories
-#
-# We could use install, but install is too different and too hard to 
-# test.  So here are the routines to copy file, make directories, and 
-# replace #...# and @...@ in files
-CP=cp
-#
-MkDir() {
-    if [ ! -e $1 ]; then mkdir $1 ; fi
-}
-CopyFile() {
-if [ -z "$3" ] ; then
-mode=$MODE
-else
-mode=$3
-fi
-if [ -d $2 ] ; then
-dest=$2/`basename $1`
-else
-    dest=$2
-fi
-if [ $replace = 0 -a -f $dest ] ; then
-    if [ $verbose = 1 ] ; then echo "$dest exists; not changed" ; fi
-elif [ -d $1 ] ; then
-    echo ">>> $1 is a directory; not copied <<<"
-    errs=`expr $errs + 1`
-    if [ $failmode = "hard" ] ; then exit 1 ; fi
-elif [ ! -f $1 ] ; then 
-    echo "**File $1 does not exist (or is not a regular file)!"
-    errs=`expr $errs + 1`
-    if [ $failmode = "hard" ] ; then exit 1 ; fi
-else
-    if [ $verbose = 1 ] ; then echo "Copying $1 to $dest" ; fi
-    # We don't delete the file in the event that we are copying the
-    # file over itself (we SHOULD check for that separately, by checking 
-    # that directories are distinct)
-    #if [ -f $dest ] ; then $Show rm -f $dest ; fi
-    $Show $CP $1 $dest
-    rc=$?
-    if [ $rc != 0 ] ; then
-	echo "**Error copying file $1 to $dest **"
-        errs=`expr $errs + 1`
-        if [ $failmode = "hard" ] ; then exit $rc ; fi
-    else
-	echo "rm -f $dest" >> $UNINSTALLFILE
-    fi
-    $Show chmod $mode $dest
-    rc=$?
-    if [ $rc != 0 ] ; then
-	echo "**Error setting mode on file $dest**"
-        errs=`expr $errs + 1`
-        if [ $chmodefailmode = "hard" ] ; then exit $rc ; fi
-    fi
-fi
-}
-
-#
-# A version of copy file that preserves file dates
-CopyFileP() {
-    CP="cp -p"
-    CopyFile $1 $2 $3
-    CP=cp
-}
-
-#
-for arg in "$@" ; do
-    case "$arg" in 
-        -prefix=*)
-            PREFIX=`echo $arg | sed -e 's/-prefix=//'`
-	    prefix=$PREFIX
-            ;;
-# LAM Added -uninstall option
-        -uninstall)
-            WANT_INSTALL=0
-            ;;
-        *)
-            echo "romioinstall: Unrecognized argument $arg ."
-            exit 1
-            ;;
-    esac
-done
-if test -z "$prefix" ; then
-    echo "Usage: make install prefix=/usr/local/romio  (or whatever directory you like)"
-    echo "in the top-level romio directory"
-fi
-
-# Uninstall filename
-if [ -z "$UNINSTALLFILE" ] ; then 
-    UNINSTALLFILE="$sbindir/romiouninstall"
-    MkDir `dirname $UNINSTALLFILE`
-fi
-
-# LAM top-level switch to install or uninstall
-if test "$WANT_INSTALL" = "1"; then
-
-# Directories
-
-# Files
-    MkDir $prefix
-
-    echo "copying directory $ROMIO_HOME/doc to $docdir"
-    MkDir $docdir
-    for file in COPYRIGHT README users-guide.ps.gz ; do
-	CopyFile $ROMIO_HOME/$file $docdir
-    done
-
-    echo "copying directory include to $includedir"
-    MkDir $includedir
-    for file in include/*.h ; do
-        CopyFile $file $includedir
-    done
-
-    MkDir $libdir
-    echo "copying directory $TOP_BUILD_DIR/lib to $libdir"
-    #cp -r $ROMIO_HOME/lib $PREFIX
-    #chmod 755 $PREFIX/lib
-    for file in $TOP_BUILD_DIR/lib/* ; do
-	if [ -f $file ] ; then 
-            CopyFileP $file $libdir 
-        fi
-    done
-    # Romio also copies directories in the lib directory.  
-
-    if [ -z "$mandir" ] ; then 
-	mandir=$PREFIX/man
-    fi
-    echo "copying directory $ROMIO_HOME/man to $mandir"
-    MkDir $mandir
-    MkDir $mandir/man3
-    for file in $ROMIO_HOME/man/man3/* ; do
-        CopyFile $file $mandir/man3
-    done
-
-    if [ -z "$exampledir" ] ; then exampledir=$PREFIX/examples ; fi
-    echo "copying directory $ROMIO_HOME/test to $exampledir"
-    MkDir $exampledir
-    MkDir $exampledir/std
-    for file in $ROMIO_HOME/test/std/* ; do 
-	CopyFile $file $exampledir/std
-    done
-    CopyFile test/Makefile $exampledir
-    CopyFile test/README $exampledir
-    for file in test/*.c test/*.f test/runtests ; do
-	CopyFile $file $exampledir
-    done
-    # Ranlib should not be necessary if cp -p is used
-    #$RANLIB $PREFIX/lib/$ARCH/libmpio.a
-    # change the include and lib paths in Makefile in the test directory
-
-    echo "updating paths in $exampledir/Makefile"
-    TMPNEWINC=-I$includedir
-    TMPNEWLIB=$libdir/$ARCH/libmpio.a
-    NEWINC=`echo $TMPNEWINC | sed 's/\//\\\\\//g'`
-    NEWLIB=`echo $TMPNEWLIB | sed 's/\//\\\\\//g'`
-    sed -e 5s/INCLUDE_DIR[\ ]*=\ [/a-z0-9.A-Z_-]*/INCLUDE_DIR\ =\ $NEWINC/ \
-        -e 6s/LIBS[\ ]*=\ [/a-z0-9.A-Z_-]*/LIBS\ =\ $NEWLIB/ $exampledir/Makefile\
-           > $exampledir/.romiotmp 
-    mv $exampledir/.romiotmp $exampledir/Makefile
-    
-    #
-    MkDir $sbindir
-    echo "rm -f $UNINSTALLFILE" >> $UNINSTALLFILE
-    # Add the directory removes to the UNINSTALL file
-    for dir in $dirlist ; do
-        echo "if [ -d $dir ] ; then rmdir $dir ; fi" >> $UNINSTALLFILE
-    done
-else
-
-# LAM do all the things for uninstall -- do the opposite of above.
-# This whole section is new.
-    $sbindir/romiouninstall
-    echo "ROMIO uninstalled"
-fi
diff --git a/3rd-party/romio321/util/tarch b/3rd-party/romio321/util/tarch
deleted file mode 100755
index ccccf4d514a..00000000000
--- a/3rd-party/romio321/util/tarch
+++ /dev/null
@@ -1,120 +0,0 @@
-#! /bin/sh
-# set -x
-#
-#  Returns the arch of the machine. This file is from MPICH.
-#
-# First, try some special cases:
-if [ -d "/dev/elan" ] ; then
-  FARCH="meiko"
-elif [ -f /usr/bin/uxpm ] && /usr/bin/uxpm ; then
-  FARCH="UXPM"
-elif [ -f /usr/bin/uxpv ] && /usr/bin/uxpv ; then
-  FARCH="uxpv"
-fi
-if [ -n "$FARCH" ] ; then
-    echo $FARCH
-    exit 0
-fi
-#
-# Try to find uname
-for dir in /bin /usr/bin /usr/local/bin ; do
-    if [ -x $dir/uname ] ; then
-	UNAME="$dir/uname"
-        break
-    fi
-done
-#
-# Get uname -s, uname -m, and arch values
-#
-if [ -n "$UNAME" ] ; then
-    ARCHLIST="`uname -s`"
-    ARCHLIST="$ARCHLIST `uname -m`"
-fi
-#
-# Get ARCH variable name
-if [ -n "$ARCH" ] ; then
-    ARCHLIST="$ARCHLIST $ARCH"
-fi
-#
-# Get arch command
-if [ -x /bin/arch ] ; then
-    ARCHLIST="$ARCHLIST `/bin/arch`"
-elif [ -x /usr/local/bin/arch ] ; then
-    ARCHLIST="$ARCHLIST `/usr/local/bin/arch`"
-fi
-#
-# GARCH is a guess if we don't find something better
-GARCH=
-# Now, weed through all of these values until we find something useful.
-for LARCH in $ARCHLIST ; do
-   # Remove blanks
-   LARCH=`echo $LARCH | sed 's/ //g'`
-   # Get the first 4 characters (you'd be surprised)
-   #   LARCH4=`expr "$LARCH" : "\(....\)"`
-   #   LARCH6=`expr "$LARCH" : "\(......\)"`
-   case $LARCH in
-       SUPER-UX) FARCH=SX4; break ;;
-       AIX|RIOS) FARCH=rs6000; break ;;
-       HP-UX) 
- 	if [ -a /dev/kmem ] ; then
- 	    FARCH=hpux ;
- 	else
- 	    FARCH=sppux ;
- 	fi
-        break ;;
-       IRIX64|IRIX) FARCH=$LARCH ; break ;;
-       Linux)  FARCH=LINUX ; break ;;
-       i586|i486|i86pc)
-	    GARCH=$LARCH ;;
-       sun4*)
-	Version=`$UNAME -r`
-        # In "improving" SunOS, the useful feature of "substr" was withdrawn 
-        # from expr.  Can't let the users have life too easy, can we?  This 
-        # means that we can't just use 
-        #   set MajorVersion = `expr substr $Version 1 1`
-        # because it won't work on Solaris systems.  The following should work
-        # on both:
-        MajorVersion=`expr "$Version" : "\(.\)"`
-        if [ "$MajorVersion" -ge 5 ] ; then
-            FARCH=solaris
-	else
-	    FARCH=sun4
-	fi
-	break ;;
-       hp9000*|hp7000*)
- 	if [ -a /dev/kmem ] ; then
- 	    FARCH=hpux ;
- 	else
- 	    FARCH=sppux ;
- 	fi
- 	break ;;
-       Darwin*)
-	   FARCH=Darwin
-	   break ;;
-
-       mips|dec-5000)   FARCH=dec5000 ; break ;;
-        next)            FARCH=NeXT ; break ;;	
-	KSR1|KSR2)       FARCH=ksr ; break ;;	
-        FreeBSD)         FARCH=freebsd ; break ;;
-        OpenBSD)         FARCH=openbsd ; break ;;
-        NetBSD)          FARCH=netbsd ; break ;;
-
-       # The following are guesses
-        IP12|iris-4d)    GARCH=IRIX ;;
-        cray|CRAY*)      GARCH=CRAY ;;
-	i386)            GARCH=ipsc2 ;;
-	ULTRIX|RISC)     GARCH=dec5000 ;;
-   esac
-   LLARCH=$LARCH
-done
-if [ -z "$FARCH" ] ; then
-    FARCH=$GARCH
-    if [ -z "$FARCH" ] ; then
-	FARCH=$LLARCH
-    fi
-    if [ -z "$FARCH" ] ; then
-	FARCH=unknown
-    fi
-fi
-echo $FARCH
-exit 0
diff --git a/3rd-party/treematch/IntConstantInitializedVector.c b/3rd-party/treematch/IntConstantInitializedVector.c
index 25a6708b2c9..8b5961df7df 100644
--- a/3rd-party/treematch/IntConstantInitializedVector.c
+++ b/3rd-party/treematch/IntConstantInitializedVector.c
@@ -2,7 +2,7 @@
 #include <stdio.h>
 #include "IntConstantInitializedVector.h"
 
-int intCIV_isInitialized(int_CIVector * v, int i)
+int tm_intCIV_isInitialized(int_CIVector * v, int i)
 {
   if(v->top == 0)
     return 0;
@@ -14,7 +14,7 @@ int intCIV_isInitialized(int_CIVector * v, int i)
 
 
 
-void intCIV_init(int_CIVector * v, int size, int init_value)
+void tm_intCIV_init(int_CIVector * v, int size, int init_value)
 {
   v->init_value = init_value;
   v->size = size;
@@ -24,20 +24,20 @@ void intCIV_init(int_CIVector * v, int size, int init_value)
   v->vec = malloc(sizeof(int)*size);
 }
 
-void intCIV_exit(int_CIVector * v)
+static inline void intCIV_exit(int_CIVector * v)
 {
   free(v->to);
   free(v->from);
   free(v->vec);
  }
 
-int intCIV_set(int_CIVector * v, int i, int val)
+int tm_intCIV_set(int_CIVector * v, int i, int val)
 {
   if(v == NULL)
     return -1;
   if(i < 0 || i >= v->size)
     return -1;
-  if(!intCIV_isInitialized(v,i))
+  if(!tm_intCIV_isInitialized(v,i))
     {
       v->from[i] = v->top;
       v->to[v->top] = i;
@@ -47,13 +47,13 @@ int intCIV_set(int_CIVector * v, int i, int val)
   return 0;  
 }
 
-int intCIV_get(int_CIVector * v, int i)
+int tm_intCIV_get(int_CIVector * v, int i)
 {
   if(v == NULL)
     return -1;
   if(i < 0 || i >= v->size)
     return -1;
-  if(intCIV_isInitialized(v,i))
+  if(tm_intCIV_isInitialized(v,i))
     return v->vec[i];
   return v->init_value;
 }
diff --git a/3rd-party/treematch/IntConstantInitializedVector.h b/3rd-party/treematch/IntConstantInitializedVector.h
index 25e5a1d759f..0906f8cf05a 100644
--- a/3rd-party/treematch/IntConstantInitializedVector.h
+++ b/3rd-party/treematch/IntConstantInitializedVector.h
@@ -1,15 +1,16 @@
 #ifndef INTEGER_CONSTANT_INITIALIZED_VECTOR
 #define INTEGER_CONSTANT_INITIALIZED_VECTOR
 
+#include "ompi_config.h"
+
 typedef struct int_CIVector_
 {
   int init_value, size, top, *to, *from, *vec;
 } int_CIVector;
 
-int intCIV_isInitialized(int_CIVector * v, int i);
-void intCIV_init(int_CIVector * v, int size, int init_value);
-void intCIV_exit(int_CIVector * v);
-int intCIV_set(int_CIVector * v, int i, int val);
-int intCIV_get(int_CIVector * v, int i);
+OMPI_HIDDEN int tm_intCIV_isInitialized(int_CIVector * v, int i);
+OMPI_HIDDEN void tm_intCIV_init(int_CIVector * v, int size, int init_value);
+OMPI_HIDDEN int tm_intCIV_set(int_CIVector * v, int i, int val);
+OMPI_HIDDEN int tm_intCIV_get(int_CIVector * v, int i);
 
 #endif /*INTEGER_CONSTANT_INITIALIZED_VECTOR*/
diff --git a/3rd-party/treematch/Makefile.am b/3rd-party/treematch/Makefile.am
index ef84755ebf9..42baa2f63d9 100644
--- a/3rd-party/treematch/Makefile.am
+++ b/3rd-party/treematch/Makefile.am
@@ -1,3 +1,12 @@
+#
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
 sources = tm_bucket.h \
     tm_mapping.h \
     tm_timings.h tm_tree.h \
@@ -21,7 +30,7 @@ EXTRA_DIST = COPYING LICENSE
 
 noinst_LTLIBRARIES = libtreematch.la
 libtreematch_la_SOURCES = $(sources)
-libtreematch_la_LDFLAGS = -static
+libtreematch_la_LDFLAGS =
 
 distclean-local:
 	rm config.h
diff --git a/3rd-party/treematch/PriorityQueue.c b/3rd-party/treematch/PriorityQueue.c
index 83e08ad2f2d..2b8fc387ec4 100644
--- a/3rd-party/treematch/PriorityQueue.c
+++ b/3rd-party/treematch/PriorityQueue.c
@@ -21,7 +21,7 @@ int PQ_init(PriorityQueue * const q, int size)
   q->elements = malloc(sizeof(QueueElement *) * size);
   for(i=0; i < size; i++)
     q->elements[i]=NULL;
-  return fiboTreeInit((FiboTree *)q, compFunc);
+  return tm_fiboTreeInit((FiboTree *)q, compFunc);
 }
 
 void PQ_exit(PriorityQueue * const q)
@@ -35,7 +35,7 @@ void PQ_exit(PriorityQueue * const q)
     }
   if(q->elements != NULL)
     free(q->elements);
-  fiboTreeExit((FiboTree *)q);
+  tm_fiboTreeExit((FiboTree *)q);
 }
 void PQ_free(PriorityQueue * const q)
 {
@@ -45,7 +45,7 @@ void PQ_free(PriorityQueue * const q)
       if(q->elements[i] != NULL)
 	free(q->elements[i]);
     }
-  fiboTreeFree((FiboTree *)q);
+  tm_fiboTreeFree((FiboTree *)q);
 }
 
 int PQ_isEmpty(PriorityQueue * const q)
@@ -68,7 +68,7 @@ void PQ_insertElement(PriorityQueue * const q, QueueElement * const e)
 }
 void PQ_deleteElement(PriorityQueue * const q, QueueElement * const e)
 {
-  fiboTreeDel((FiboTree *)q, (FiboNode *)(e));
+  tm_fiboTreeDel((FiboTree *)q, (FiboNode *)(e));
   q->elements[e->value] = NULL;
   e->isInQueue = 0;
 }
@@ -93,12 +93,12 @@ void PQ_delete(PriorityQueue * const q, int val)
 
 QueueElement * PQ_findMaxElement(PriorityQueue * const q)
 {
-  QueueElement * e = (QueueElement *)(fiboTreeMin((FiboTree *)q));
+  QueueElement * e = (QueueElement *)(tm_fiboTreeMin((FiboTree *)q));
   return e;
 }
 QueueElement * PQ_deleteMaxElement(PriorityQueue * const q)
 {
-  QueueElement * e = (QueueElement *)(fiboTreeMin((FiboTree *)q));
+  QueueElement * e = (QueueElement *)(tm_fiboTreeMin((FiboTree *)q));
   if(e != NULL)
     {
       PQ_deleteElement(q, e);
diff --git a/3rd-party/treematch/fibo.c b/3rd-party/treematch/fibo.c
index 97070e7273a..81ddac82a75 100644
--- a/3rd-party/treematch/fibo.c
+++ b/3rd-party/treematch/fibo.c
@@ -28,6 +28,7 @@
 ** 
 ** The fact that you are presently reading this means that you have had
 ** knowledge of the CeCILL-B license and that you accept its terms.
+**
 */
 /************************************************************/
 /**                                                        **/
@@ -69,6 +70,8 @@
 #define memFree                     free
 #endif /* memAlloc */
 
+static FiboNode *fiboTreeConsolidate (FiboTree * const);
+
 /*********************************************/
 /*                                           */
 /* These routines deal with Fibonacci trees. */
@@ -83,7 +86,7 @@
 */
 
 int
-fiboTreeInit (
+tm_fiboTreeInit (
 FiboTree * const            treeptr,
 int                      (* cmpfptr) (const FiboNode * const, const FiboNode * const))
 {
@@ -106,7 +109,7 @@ int                      (* cmpfptr) (const FiboNode * const, const FiboNode * c
 */
 
 void
-fiboTreeExit (
+tm_fiboTreeExit (
 FiboTree * const            treeptr)
 {
   if (treeptr->degrtab != NULL)
@@ -122,7 +125,7 @@ FiboTree * const            treeptr)
 */
 
 void
-fiboTreeFree (
+tm_fiboTreeFree (
 FiboTree * const            treeptr)
 {
   treeptr->rootdat.linkdat.prevptr =              /* Link root node to itself */
@@ -138,7 +141,7 @@ FiboTree * const            treeptr)
 ** - NULL   : Fibonacci tree is empty.
 */
 
-FiboNode *
+static FiboNode *
 fiboTreeConsolidate (
 FiboTree * const            treeptr)
 {
@@ -229,7 +232,7 @@ FiboTree * const            treeptr)
 #ifndef fiboTreeMin
 
 FiboNode *
-fiboTreeMin (
+tm_fiboTreeMin (
 FiboTree * const            treeptr)
 {
   FiboNode *            bestptr;
@@ -279,7 +282,7 @@ FiboNode * const            nodeptr)
 #ifndef fiboTreeDel
 
 void
-fiboTreeDel (
+tm_fiboTreeDel (
 FiboTree * const            treeptr,
 FiboNode * const            nodeptr)
 {
diff --git a/3rd-party/treematch/fibo.h b/3rd-party/treematch/fibo.h
index 32e0a7c0824..52d587a4f99 100644
--- a/3rd-party/treematch/fibo.h
+++ b/3rd-party/treematch/fibo.h
@@ -28,6 +28,7 @@
 ** 
 ** The fact that you are presently reading this means that you have had
 ** knowledge of the CeCILL-B license and that you accept its terms.
+** 
 */
 /************************************************************/
 /**                                                        **/
@@ -56,6 +57,8 @@
 /**                                                        **/
 /************************************************************/
 
+#include "ompi_config.h"
+
 /*
 **  The type and structure definitions.
 */
@@ -101,13 +104,13 @@ typedef struct FiboTree_ {
    the algorithms have been de-recursived
    and written as macros.                 */
 
-#define fiboTreeLinkAfter(o,n)      do {                              \
-                                      FiboNode *        nextptr;      \
-                                      nextptr = (o)->linkdat.nextptr; \
-                                      (n)->linkdat.nextptr = nextptr; \
-                                      (n)->linkdat.prevptr = (o);     \
-                                      nextptr->linkdat.prevptr = (n); \
-                                      (o)->linkdat.nextptr = (n);     \
+#define fiboTreeLinkAfter(o,n)      do {                                  \
+                                      FiboNode *        nextptr_loc;      \
+                                      nextptr_loc = (o)->linkdat.nextptr; \
+                                      (n)->linkdat.nextptr = nextptr_loc; \
+                                      (n)->linkdat.prevptr = (o);         \
+                                      nextptr_loc->linkdat.prevptr = (n); \
+                                      (o)->linkdat.nextptr = (n);         \
                                     } while (0)
 
 #define fiboTreeUnlink(n)           do {                                                            \
@@ -184,19 +187,18 @@ typedef struct FiboTree_ {
 #define static
 #endif
 
-int                         fiboTreeInit        (FiboTree * const, int (*) (const FiboNode * const, const FiboNode * const));
-void                        fiboTreeExit        (FiboTree * const);
-void                        fiboTreeFree        (FiboTree * const);
-FiboNode *                  fiboTreeConsolidate (FiboTree * const);
+OMPI_HIDDEN int             tm_fiboTreeInit        (FiboTree * const, int (*) (const FiboNode * const, const FiboNode * const));
+OMPI_HIDDEN void            tm_fiboTreeExit        (FiboTree * const);
+OMPI_HIDDEN void            tm_fiboTreeFree        (FiboTree * const);
 #ifndef fiboTreeAdd
 void                        fiboTreeAdd         (FiboTree * const, FiboNode * const);
 #endif /* fiboTreeAdd */
 #ifndef fiboTreeDel
-void                        fiboTreeDel         (FiboTree * const, FiboNode * const);
+OMPI_HIDDEN void            tm_fiboTreeDel (FiboTree * const, FiboNode * const);
 #endif /* fiboTreeDel */
-#ifndef fiboTreeMin
-FiboNode *                  fiboTreeMin         (FiboTree * const);
-#endif /* fiboTreeMin */
+#ifndef treematch_fiboTreeMin
+OMPI_HIDDEN FiboNode *      tm_fiboTreeMin         (FiboTree * const);
+#endif /* treematch_fiboTreeMin */
 #ifdef FIBO_DEBUG
 int                         fiboTreeCheck       (const FiboTree * const);
 static int                  fiboTreeCheck2      (const FiboNode * const);
diff --git a/3rd-party/treematch/k-partitioning.c b/3rd-party/treematch/k-partitioning.c
index f035ffa24a1..567a69ae2e5 100644
--- a/3rd-party/treematch/k-partitioning.c
+++ b/3rd-party/treematch/k-partitioning.c
@@ -4,19 +4,19 @@
 #include "tm_mt.h"
 #include "tm_verbose.h"
 
-void memory_allocation(PriorityQueue ** Q, PriorityQueue ** Qinst, double *** D, int n, int k);
-void initialization(int * const part, double ** const matrice, PriorityQueue * const Qpart, PriorityQueue * const Q, PriorityQueue * const Qinst, double ** const D, int n, int k, int * const deficit, int * const surplus);
-void algo(int * const part, double ** const matrice, PriorityQueue * const Qpart, PriorityQueue * const Q, PriorityQueue * const Qinst, double ** const D, int n,  int * const deficit, int * const surplus);
-double nextGain(PriorityQueue * const Qpart, PriorityQueue * const Q, int * const deficit, int * const surplus);
-void balancing(int n, int deficit, int surplus, double ** const D, int * const part);
-void destruction(PriorityQueue * Qpart, PriorityQueue * Q, PriorityQueue * Qinst, double ** D, int n, int k);
-
-void allocate_vertex2(int u, int *res, double **comm, int n, int *size, int max_size);
-double eval_cost2(int *,int,double **);
-int  *kpartition_greedy2(int k, double **comm, int n, int nb_try_max, int *constraints, int nb_constraints);
-int*  build_p_vector(double **comm, int n, int k, int greedy_trials, int * constraints, int nb_constraints);
-
-int* kPartitioning(double ** comm, int n, int k, int * constraints, int nb_constraints, int greedy_trials)
+static void memory_allocation(PriorityQueue ** Q, PriorityQueue ** Qinst, double *** D, int n, int k);
+static void initialization(int * const part, double ** const matrice, PriorityQueue * const Qpart, PriorityQueue * const Q, PriorityQueue * const Qinst, double ** const D, int n, int k, int * const deficit, int * const surplus);
+static void algo(int * const part, double ** const matrice, PriorityQueue * const Qpart, PriorityQueue * const Q, PriorityQueue * const Qinst, double ** const D, int n,  int * const deficit, int * const surplus);
+static double nextGain(PriorityQueue * const Qpart, PriorityQueue * const Q, int * const deficit, int * const surplus);
+static void balancing(int n, int deficit, int surplus, double ** const D, int * const part);
+static void destruction(PriorityQueue * Qpart, PriorityQueue * Q, PriorityQueue * Qinst, double ** D, int n, int k);
+
+static void allocate_vertex2(int u, int *res, double **comm, int n, int *size, int max_size);
+static double eval_cost2(int *,int,double **);
+static int  *kpartition_greedy2(int k, double **comm, int n, int nb_try_max, int *constraints, int nb_constraints);
+static int*  build_p_vector(double **comm, int n, int k, int greedy_trials, int * constraints, int nb_constraints);
+
+int* tm_kPartitioning(double ** comm, int n, int k, int * constraints, int nb_constraints, int greedy_trials)
 {
   /* ##### declarations & allocations ##### */
 
@@ -48,7 +48,7 @@ int* kPartitioning(double ** comm, int n, int k, int * constraints, int nb_const
   return part;
 }
 
-void memory_allocation(PriorityQueue ** Q, PriorityQueue ** Qinst, double *** D, int n, int k)
+static void memory_allocation(PriorityQueue ** Q, PriorityQueue ** Qinst, double *** D, int n, int k)
 {
   int i;
   *Q = calloc(k, sizeof(PriorityQueue)); /*one Q for each partition*/
@@ -58,7 +58,7 @@ void memory_allocation(PriorityQueue ** Q, PriorityQueue ** Qinst, double *** D,
     (*D)[i] = calloc(k, sizeof(double));
 }
 
-void initialization(int * const part, double ** const matrice, PriorityQueue * const Qpart, PriorityQueue * const Q, PriorityQueue * const Qinst, double ** const D, int n, int k, int * const deficit, int * const surplus)
+static void initialization(int * const part, double ** const matrice, PriorityQueue * const Qpart, PriorityQueue * const Q, PriorityQueue * const Qinst, double ** const D, int n, int k, int * const deficit, int * const surplus)
 {
   int i,j;
 
@@ -103,7 +103,7 @@ void initialization(int * const part, double ** const matrice, PriorityQueue * c
   *surplus = *deficit = 0;
 }
 
-void algo(int * const part, double ** const matrice, PriorityQueue * const Qpart, PriorityQueue * const Q, PriorityQueue * const Qinst, double ** const D, int n, int * const deficit, int * const surplus)
+static void algo(int * const part, double ** const matrice, PriorityQueue * const Qpart, PriorityQueue * const Q, PriorityQueue * const Qinst, double ** const D, int n, int * const deficit, int * const surplus)
 {
   int p,u,v,j;
   double d;
@@ -149,7 +149,7 @@ void algo(int * const part, double ** const matrice, PriorityQueue * const Qpart
   PQ_adjustKey(Qpart, part[u], d); /*we update the new highest possible gain in u's subset*/
 }
 
-double nextGain(PriorityQueue * const Qpart, PriorityQueue * const Q, int * const deficit, int * const surplus)
+static double nextGain(PriorityQueue * const Qpart, PriorityQueue * const Q, int * const deficit, int * const surplus)
 {
   double res;
   if(*deficit == *surplus) /*if the current partition is balanced*/
@@ -159,7 +159,7 @@ double nextGain(PriorityQueue * const Qpart, PriorityQueue * const Q, int * cons
   return res;
 }
 
-void balancing(int n, int deficit, int surplus, double ** const D, int * const part)
+static void balancing(int n, int deficit, int surplus, double ** const D, int * const part)
 {
   if(surplus != deficit) /*if the current partition is not balanced*/
     {
@@ -176,7 +176,7 @@ void balancing(int n, int deficit, int surplus, double ** const D, int * const p
     }
 }
 
-void destruction(PriorityQueue * Qpart, PriorityQueue * Q, PriorityQueue * Qinst, double ** D, int n, int k)
+static void destruction(PriorityQueue * Qpart, PriorityQueue * Q, PriorityQueue * Qinst, double ** D, int n, int k)
 {
   int i;
   PQ_exit(Qpart);
@@ -195,7 +195,7 @@ void destruction(PriorityQueue * Qpart, PriorityQueue * Q, PriorityQueue * Qinst
 }
 
 
-int  *kpartition_greedy2(int k, double **comm, int n, int nb_try_max, int *constraints, int nb_constraints)
+static int  *kpartition_greedy2(int k, double **comm, int n, int nb_try_max, int *constraints, int nb_constraints)
 {
   int *res = NULL, *best_res=NULL, *size = NULL;
   int i,j,nb_trials;
@@ -229,7 +229,7 @@ int  *kpartition_greedy2(int k, double **comm, int n, int nb_try_max, int *const
       /* find a vertex not already partitionned*/
       do{
 	/* call the mersenne twister PRNG of tm_mt.c*/
-	j =  genrand_int32() % n;
+	j =  tm_genrand_int32() % n;
       } while ( res[j] != -1 );
       /* allocate and update size of partition*/
       res[j] = i;
@@ -261,7 +261,7 @@ int  *kpartition_greedy2(int k, double **comm, int n, int nb_try_max, int *const
   return best_res;
 }
 
-void allocate_vertex2(int u, int *res, double **comm, int n, int *size, int max_size)
+static void allocate_vertex2(int u, int *res, double **comm, int n, int *size, int max_size)
 {
   int i,best_part = -1;
   double cost, best_cost = -1;
@@ -285,7 +285,7 @@ void allocate_vertex2(int u, int *res, double **comm, int n, int *size, int max_
   size[best_part]++;
 }
 
-double eval_cost2(int *partition, int n, double **comm)
+static double eval_cost2(int *partition, int n, double **comm)
 {
   double cost = 0;
   int i,j;
@@ -298,7 +298,7 @@ double eval_cost2(int *partition, int n, double **comm)
   return cost;
 }
 
-int* build_p_vector(double **comm, int n, int k, int greedy_trials, int * constraints, int nb_constraints)
+static int* build_p_vector(double **comm, int n, int k, int greedy_trials, int * constraints, int nb_constraints)
 {
   int * part = NULL;
   if(greedy_trials>0) /*if greedy_trials > 0 then we use kpartition_greedy with greedy_trials trials*/
diff --git a/3rd-party/treematch/k-partitioning.h b/3rd-party/treematch/k-partitioning.h
index 8db2392e1a5..6af695eb5c3 100644
--- a/3rd-party/treematch/k-partitioning.h
+++ b/3rd-party/treematch/k-partitioning.h
@@ -1,6 +1,8 @@
 #ifndef K_PARTITIONING
 #define K_PARTITIONING
 
+#include "ompi_config.h"
+
 #include "PriorityQueue.h"
 
 /*
@@ -14,7 +16,7 @@
           - 0 : cyclic distribution of vertices
 	  - > 0 : use of kpartition_greedy with greedy_trials number of trials 
  */
-             
-int* kPartitioning(double ** comm, int n, int k, int * const constraints, int nb_constraints, int greedy_trials);
+
+OMPI_HIDDEN int* tm_kPartitioning(double ** comm, int n, int k, int * const constraints, int nb_constraints, int greedy_trials);
 
 #endif /*K_PARTITIONING*/
diff --git a/3rd-party/treematch/tm_bucket.c b/3rd-party/treematch/tm_bucket.c
index 2389aa4fc4e..73cc8017341 100644
--- a/3rd-party/treematch/tm_bucket.c
+++ b/3rd-party/treematch/tm_bucket.c
@@ -33,27 +33,28 @@ static int verbose_level = ERROR;
 
 static bucket_list_t global_bl = {0};
 
-int tab_cmp(const void*,const void*);
-int old_bucket_id(int,int,bucket_list_t);
-int bucket_id(int,int,bucket_list_t);
-void display_bucket(bucket_t *);
-void check_bucket(bucket_t *,double **,double, double);
-void display_pivots(bucket_list_t);
-void display_bucket_list(bucket_list_t);
-void add_to_bucket(int,int,int,bucket_list_t);
-void dfs(int,int,int,double *,double *,int,int);
-void built_pivot_tree(bucket_list_t);
-void fill_buckets(bucket_list_t);
-int is_power_of_2(int);
-void partial_sort(bucket_list_t *,double **,int);
-void next_bucket_elem(bucket_list_t,int *,int *);
-int add_edge_3(tm_tree_t *,tm_tree_t *,int,int,int *);
-void free_bucket(bucket_t *);
-void free_tab_bucket(bucket_t **,int);
-void free_bucket_list(bucket_list_t);
-void partial_update_val (int nb_args, void **args, int thread_id);
-double bucket_grouping(tm_affinity_mat_t *,tm_tree_t *, tm_tree_t *, int ,int);
-int tab_cmp(const void* x1,const void* x2)
+static int tab_cmp(const void*,const void*);
+static int old_bucket_id(int,int,bucket_list_t);
+static int bucket_id(int,int,bucket_list_t);
+static void display_bucket(bucket_t *);
+static void check_bucket(bucket_t *,double **,double, double);
+static void display_pivots(bucket_list_t);
+static void display_bucket_list(bucket_list_t);
+static void add_to_bucket(int,int,int,bucket_list_t);
+static void dfs(int,int,int,double *,double *,int,int);
+static void built_pivot_tree(bucket_list_t);
+static void fill_buckets(bucket_list_t);
+static int is_power_of_2(int);
+static void partial_sort(bucket_list_t *,double **,int);
+static void next_bucket_elem(bucket_list_t,int *,int *);
+static int add_edge_3(tm_tree_t *,tm_tree_t *,int,int,int *);
+static void free_bucket(bucket_t *);
+static void free_tab_bucket(bucket_t **,int);
+static void free_bucket_list(bucket_list_t);
+static void partial_update_val (int nb_args, void **args, int thread_id);
+double tm_bucket_grouping(tm_affinity_mat_t *,tm_tree_t *, tm_tree_t *, int ,int);
+
+static int tab_cmp(const void* x1,const void* x2)
 {
   int *e1 = NULL,*e2 = NULL,i1,i2,j1,j2;
   double **tab = NULL;
@@ -82,7 +83,7 @@ int tab_cmp(const void* x1,const void* x2)
 }
 
 
-int old_bucket_id(int i,int j,bucket_list_t bucket_list)
+static inline int old_bucket_id(int i,int j,bucket_list_t bucket_list)
 {
   double *pivot = NULL,val;
   int n,sup,inf,p;
@@ -111,7 +112,7 @@ int old_bucket_id(int i,int j,bucket_list_t bucket_list)
   return sup;
 }
 
-int bucket_id(int i,int j,bucket_list_t bucket_list)
+static int bucket_id(int i,int j,bucket_list_t bucket_list)
 {
   double *pivot_tree = NULL,val;
   int p,k;
@@ -131,14 +132,14 @@ int bucket_id(int i,int j,bucket_list_t bucket_list)
   return (int)pivot_tree[p];
 }
 
-void  display_bucket(bucket_t *b)
+static void  display_bucket(bucket_t *b)
 {
   printf("\tb.bucket=%p\n",(void *)b->bucket);
   printf("\tb.bucket_len=%d\n",(int)b->bucket_len);
   printf("\tb.nb_elem=%d\n",(int)b->nb_elem);
 }
 
-void check_bucket(bucket_t *b,double **tab,double inf, double sup)
+static void check_bucket(bucket_t *b,double **tab,double inf, double sup)
 {
   int i,j,k;
   for( k = 0 ; k < b->nb_elem ; k++ ){
@@ -152,7 +153,7 @@ void check_bucket(bucket_t *b,double **tab,double inf, double sup)
   }
 }
 
-void display_pivots(bucket_list_t bucket_list)
+static void display_pivots(bucket_list_t bucket_list)
 {
   int i;
   for( i = 0 ; i < bucket_list->nb_buckets-1 ; i++)
@@ -160,7 +161,7 @@ void display_pivots(bucket_list_t bucket_list)
   printf("\n");
 }
 
-void display_bucket_list(bucket_list_t bucket_list)
+static inline void display_bucket_list(bucket_list_t bucket_list)
 {
   int i;
   double inf,sup;
@@ -184,7 +185,7 @@ void display_bucket_list(bucket_list_t bucket_list)
 
 }
 
-void add_to_bucket(int id,int i,int j,bucket_list_t bucket_list)
+static void add_to_bucket(int id,int i,int j,bucket_list_t bucket_list)
 {
   bucket_t *bucket = NULL;
   int N,n,size;
@@ -222,7 +223,7 @@ void add_to_bucket(int id,int i,int j,bucket_list_t bucket_list)
  /* exit(-1); */
 }
 
-void dfs(int i,int inf,int sup,double *pivot,double *pivot_tree,int depth,int max_depth)
+static void dfs(int i,int inf,int sup,double *pivot,double *pivot_tree,int depth,int max_depth)
 {
   int p;
   if( depth == max_depth )
@@ -235,7 +236,7 @@ void dfs(int i,int inf,int sup,double *pivot,double *pivot_tree,int depth,int ma
   dfs(2*i+1,p+1,sup,pivot,pivot_tree,depth+1,max_depth);
 }
 
-void  built_pivot_tree(bucket_list_t bucket_list)
+static void  built_pivot_tree(bucket_list_t bucket_list)
 {
   double *pivot_tree = NULL,*pivot = NULL;
   int n,i,k;
@@ -261,7 +262,7 @@ void  built_pivot_tree(bucket_list_t bucket_list)
   }
 }
 
-void fill_buckets(bucket_list_t bucket_list)
+static void fill_buckets(bucket_list_t bucket_list)
 {
   int N,i,j,id;
 
@@ -274,7 +275,7 @@ void fill_buckets(bucket_list_t bucket_list)
     }
 }
 
-int is_power_of_2(int val)
+static int is_power_of_2(int val)
 {
   int n = 1;
   do{
@@ -286,7 +287,7 @@ int is_power_of_2(int val)
 }
 
 
-void partial_sort(bucket_list_t *bl,double **tab,int N)
+static void partial_sort(bucket_list_t *bl,double **tab,int N)
 {
   double *pivot = NULL;
   int *sample = NULL;
@@ -325,11 +326,11 @@ void partial_sort(bucket_list_t *bl,double **tab,int N)
   sample = (int*)MALLOC(2*sizeof(int)*n);
 
   for( k =  0 ; k < n ; k++ ){
-    i = genrand_int32()%(N-2)+1;
+    i = tm_genrand_int32()%(N-2)+1;
     if( i == N-2 )
       j = N-1;
     else
-      j = genrand_int32()%(N-i-2)+i+1;
+      j = tm_genrand_int32()%(N-i-2)+i+1;
     if(verbose_level >= DEBUG)
       printf("i=%d, j=%d\n",i,j);
     assert( i != j );
@@ -386,7 +387,7 @@ void partial_sort(bucket_list_t *bl,double **tab,int N)
   *bl = bucket_list;
 }
 
-void next_bucket_elem(bucket_list_t bucket_list,int *i,int *j)
+static void next_bucket_elem(bucket_list_t bucket_list,int *i,int *j)
 {
   bucket_t *bucket = bucket_list->bucket_tab[bucket_list->cur_bucket];
 
@@ -415,7 +416,7 @@ void next_bucket_elem(bucket_list_t bucket_list,int *i,int *j)
 }
 
 
-int add_edge_3(tm_tree_t *tab_node, tm_tree_t *parent,int i,int j,int *nb_groups)
+static int add_edge_3(tm_tree_t *tab_node, tm_tree_t *parent,int i,int j,int *nb_groups)
 {
   /* printf("%d <-> %d ?\n",tab_node[i].id,tab_node[j].id); */
   if((!tab_node[i].parent) && (!tab_node[j].parent)){
@@ -464,7 +465,7 @@ int add_edge_3(tm_tree_t *tab_node, tm_tree_t *parent,int i,int j,int *nb_groups
   return 0;
 }
 
-int try_add_edge(tm_tree_t *tab_node, tm_tree_t *parent,int arity,int i,int j,int *nb_groups)
+int tm_try_add_edge(tm_tree_t *tab_node, tm_tree_t *parent,int arity,int i,int j,int *nb_groups)
 {
   assert( i != j );
 
@@ -492,13 +493,13 @@ int try_add_edge(tm_tree_t *tab_node, tm_tree_t *parent,int arity,int i,int j,in
   }
 }
 
-void free_bucket(bucket_t *bucket)
+static void free_bucket(bucket_t *bucket)
 {
   FREE(bucket->bucket);
   FREE(bucket);
 }
 
-void free_tab_bucket(bucket_t **bucket_tab,int N)
+static void free_tab_bucket(bucket_t **bucket_tab,int N)
 {
   int i;
   for( i = 0 ; i < N ; i++ )
@@ -506,7 +507,7 @@ void free_tab_bucket(bucket_t **bucket_tab,int N)
   FREE(bucket_tab);
 }
 
-void free_bucket_list(bucket_list_t bucket_list)
+static void free_bucket_list(bucket_list_t bucket_list)
 {
   /* Do not free the tab field it is used elsewhere */
   free_tab_bucket(bucket_list->bucket_tab,bucket_list->nb_buckets);
@@ -515,7 +516,7 @@ void free_bucket_list(bucket_list_t bucket_list)
   FREE(bucket_list);
 }
 
-void partial_update_val (int nb_args, void **args, int thread_id){
+static void partial_update_val (int nb_args, void **args, int thread_id){
   int inf = *(int*)args[0];
   int sup = *(int*)args[1];
   tm_affinity_mat_t *aff_mat = (tm_affinity_mat_t*)args[2];
@@ -530,12 +531,12 @@ void partial_update_val (int nb_args, void **args, int thread_id){
   }
 
   for( l = inf ; l < sup ; l++ ){
-      update_val(aff_mat,&new_tab_node[l]);
+      tm_update_val(aff_mat,&new_tab_node[l]);
       *res += new_tab_node[l].val;
     }
 }
 
-double bucket_grouping(tm_affinity_mat_t *aff_mat,tm_tree_t *tab_node, tm_tree_t *new_tab_node,
+double tm_bucket_grouping(tm_affinity_mat_t *aff_mat,tm_tree_t *tab_node, tm_tree_t *new_tab_node,
 		     int arity,int M)
 {
   bucket_list_t bucket_list;
@@ -576,7 +577,7 @@ double bucket_grouping(tm_affinity_mat_t *aff_mat,tm_tree_t *tab_node, tm_tree_t
 	printf("elem[%d][%d]=%f ",i,j,mat[i][j]);
       gr1_1 += TOC;
       TIC;
-      if(try_add_edge(tab_node,&new_tab_node[l],arity,i,j,&nb_groups)){
+      if(tm_try_add_edge(tab_node,&new_tab_node[l],arity,i,j,&nb_groups)){
 	l++;
       }
       gr1_2 += TOC;
@@ -584,7 +585,7 @@ double bucket_grouping(tm_affinity_mat_t *aff_mat,tm_tree_t *tab_node, tm_tree_t
   }else{
     while( l < M ){
       next_bucket_elem(bucket_list,&i,&j);
-      if(try_add_edge(tab_node,&new_tab_node[l],arity,i,j,&nb_groups)){
+      if(tm_try_add_edge(tab_node,&new_tab_node[l],arity,i,j,&nb_groups)){
 	l++;
       }
     }
@@ -600,7 +601,7 @@ double bucket_grouping(tm_affinity_mat_t *aff_mat,tm_tree_t *tab_node, tm_tree_t
   TIC;
   while( nb_groups < M ){
     next_bucket_elem(bucket_list,&i,&j);
-    try_add_edge(tab_node,NULL,arity,i,j,&nb_groups);
+    tm_try_add_edge(tab_node,NULL,arity,i,j,&nb_groups);
   }
 
   gr2=TOC;
@@ -621,7 +622,7 @@ double bucket_grouping(tm_affinity_mat_t *aff_mat,tm_tree_t *tab_node, tm_tree_t
     int *sup;
     double *tab_val;
 
-    nb_threads = get_nb_threads();
+    nb_threads = tm_get_nb_threads();
     works = (work_t**)MALLOC(sizeof(work_t*)*nb_threads);
     inf = (int*)MALLOC(sizeof(int)*nb_threads);
     sup = (int*)MALLOC(sizeof(int)*nb_threads);
@@ -637,18 +638,18 @@ double bucket_grouping(tm_affinity_mat_t *aff_mat,tm_tree_t *tab_node, tm_tree_t
       args[3]=(void*)new_tab_node;
       args[4]=(void*)(tab_val+id);
 
-      works[id]= create_work(5,args,partial_update_val);
+      works[id]= tm_create_work(5,args,partial_update_val);
       if(verbose_level >= DEBUG)
 	printf("Executing %p\n",(void *)works[id]);
 
-      submit_work( works[id], id);
+      tm_submit_work( works[id], id);
     }
 
     for(id=0;id<nb_threads;id++){
-      wait_work_completion(works[id]);
+      tm_wait_work_completion(works[id]);
       val+=tab_val[id];
       FREE(works[id]->args);
-      destroy_work(works[id]);
+      tm_destroy_work(works[id]);
     }
 
 
@@ -659,7 +660,7 @@ double bucket_grouping(tm_affinity_mat_t *aff_mat,tm_tree_t *tab_node, tm_tree_t
   }else{
     for( l = 0 ; l < M ; l++ ){
 
-      update_val(aff_mat,&new_tab_node[l]);
+      tm_update_val(aff_mat,&new_tab_node[l]);
       val += new_tab_node[l].val;
     }
   }
diff --git a/3rd-party/treematch/tm_bucket.h b/3rd-party/treematch/tm_bucket.h
index 433d4816466..b9ff10f0c39 100644
--- a/3rd-party/treematch/tm_bucket.h
+++ b/3rd-party/treematch/tm_bucket.h
@@ -1,6 +1,8 @@
 #ifndef __BUCKET_H__
 #define __BUCKET_H__
 
+#include "ompi_config.h"
+
 typedef struct{
   int i;
   int j;
@@ -28,8 +30,8 @@ typedef struct{
 
 typedef _bucket_list_t *bucket_list_t;
 
-double bucket_grouping(tm_affinity_mat_t *aff_mat,tm_tree_t *tab_node, tm_tree_t *new_tab_node, 
+OMPI_HIDDEN double tm_bucket_grouping(tm_affinity_mat_t *aff_mat,tm_tree_t *tab_node, tm_tree_t *new_tab_node, 
 		       int arity,int M);
-int try_add_edge(tm_tree_t *tab_node, tm_tree_t *parent,int arity,int i,int j,int *nb_groups);
+OMPI_HIDDEN int tm_try_add_edge(tm_tree_t *tab_node, tm_tree_t *parent,int arity,int i,int j,int *nb_groups);
 #endif
 
diff --git a/3rd-party/treematch/tm_kpartitioning.c b/3rd-party/treematch/tm_kpartitioning.c
index 09cb35add50..278c76880a0 100644
--- a/3rd-party/treematch/tm_kpartitioning.c
+++ b/3rd-party/treematch/tm_kpartitioning.c
@@ -5,6 +5,7 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include "config.h"
+#include "ompi_config.h"
 
 #if defined(HAVE_LIBSCOTCH)
 #include <scotch.h>
@@ -20,22 +21,22 @@ static int verbose_level = ERROR;
 #define USE_KL_STRATEGY 1
 
 
-#define MIN(a,b) ((a)<(b)?(a):(b))
+#define TM_MIN(a,b) ((a)<(b)?(a):(b))
 
 
-int  fill_tab(int **,int *,int,int,int,int);
-void complete_obj_weight(double **,int,int);
+OMPI_HIDDEN int  tm_fill_tab(int **,int *,int,int,int,int);
+OMPI_HIDDEN void tm_complete_obj_weight(double **,int,int);
 
-void allocate_vertex(int,int *,com_mat_t *,int,int *,int);
-double eval_cost(int *, com_mat_t *);
-int *kpartition_greedy(int, com_mat_t *,int,int *,int);
-constraint_t *split_constraints (int *,int,int,tm_topology_t *,int, int);
-com_mat_t **split_com_mat(com_mat_t *,int,int,int *);
-int **split_vertices(int *,int,int,int *);
-void free_tab_com_mat(com_mat_t **,int);
-void free_tab_local_vertices(int **,int);
-void free_const_tab(constraint_t *,int);
-void kpartition_build_level_topology(tm_tree_t *,com_mat_t *,int,int,tm_topology_t *,
+static void allocate_vertex(int,int *,com_mat_t *,int,int *,int);
+static double eval_cost(int *, com_mat_t *);
+static int *kpartition_greedy(int, com_mat_t *,int,int *,int);
+static constraint_t *split_constraints (int *,int,int,tm_topology_t *,int, int);
+static com_mat_t **split_com_mat(com_mat_t *,int,int,int *);
+static int **split_vertices(int *,int,int,int *);
+static void free_tab_com_mat(com_mat_t **,int);
+static void free_tab_local_vertices(int **,int);
+static void free_const_tab(constraint_t *,int);
+static void kpartition_build_level_topology(tm_tree_t *,com_mat_t *,int,int,tm_topology_t *,
 				     int *,int *,int,double *,double *);
 
 static int greedy_flag = 0;
@@ -287,7 +288,7 @@ int  *kpartition_scotch(int k, com_mat_t *com_mat, int n, int *constraints, int
 #endif /* defined(HAVE_LIBSCOTCH) */
 
 
-void allocate_vertex(int u, int *res, com_mat_t *com_mat, int n, int *size, int max_size)
+static void allocate_vertex(int u, int *res, com_mat_t *com_mat, int n, int *size, int max_size)
 {
   int i,best_part=0;
   double cost, best_cost = -1;
@@ -324,7 +325,7 @@ void allocate_vertex(int u, int *res, com_mat_t *com_mat, int n, int *size, int
   size[best_part]++;
 }
 
-double eval_cost(int *partition, com_mat_t *com_mat)
+static double eval_cost(int *partition, com_mat_t *com_mat)
 {
   double cost = 0;
   int i,j;
@@ -337,7 +338,7 @@ double eval_cost(int *partition, com_mat_t *com_mat)
   return cost;
 }
 
-int  *kpartition_greedy(int k, com_mat_t *com_mat, int n, int *constraints, int nb_constraints)
+static int  *kpartition_greedy(int k, com_mat_t *com_mat, int n, int *constraints, int nb_constraints)
 {
   int *partition = NULL, *best_partition=NULL, *size = NULL;
   int i,j,nb_trials;
@@ -362,7 +363,8 @@ int  *kpartition_greedy(int k, com_mat_t *com_mat, int n, int *constraints, int
     printf("nb_constraints = %d\n",nb_constraints);
 
     if(n<=16){
-      printf("Constraints: ");print_1D_tab(constraints,nb_constraints);
+      printf("Constraints: ");
+      tm_print_1D_tab(constraints,nb_constraints);
     }
   }
   /* if(com_mat->n){ */
@@ -424,7 +426,7 @@ int  *kpartition_greedy(int k, com_mat_t *com_mat, int n, int *constraints, int
       /* find a vertex not allready partitionned*/
       do{
 	/* call the mersenne twister PRNG of tm_mt.c*/
-	j =  genrand_int32() % n;
+	j =  tm_genrand_int32() % n;
       } while ( partition[j] != -1 );
       /* allocate and update size of partition*/
       partition[j] = i;
@@ -460,7 +462,7 @@ int  *kpartition_greedy(int k, com_mat_t *com_mat, int n, int *constraints, int
   return best_partition;
 }
 
-int *kpartition(int k, com_mat_t *com_mat, int n, int *constraints, int nb_constraints)
+int *tm_kpartition(int k, com_mat_t *com_mat, int n, int *constraints, int nb_constraints)
 {
   int *res= NULL;
 
@@ -493,7 +495,7 @@ int *kpartition(int k, com_mat_t *com_mat, int n, int *constraints, int nb_const
   return res;
 }
 
-constraint_t *split_constraints (int *constraints, int nb_constraints, int k, tm_topology_t *topology, int depth, int N)
+static constraint_t *split_constraints (int *constraints, int nb_constraints, int k, tm_topology_t *topology, int depth, int N)
 {
   constraint_t *const_tab = NULL;
   int nb_leaves, start, end;
@@ -505,7 +507,7 @@ constraint_t *split_constraints (int *constraints, int nb_constraints, int k, tm
   /* nb_leaves is the number of leaves of the current subtree
      this will help to determine where to split constraints and how to shift values
   */
-  nb_leaves = compute_nb_leaves_from_level( depth + 1, topology );
+  nb_leaves = tm_compute_nb_leaves_from_level( depth + 1, topology );
 
 /* split the constraints into k sub-constraints
      each sub-contraints 'i' contains constraints of value in [i*nb_leaves,(i+1)*nb_leaves[
@@ -515,12 +517,12 @@ constraint_t *split_constraints (int *constraints, int nb_constraints, int k, tm
   for( i = 0; i < k; i++ ){
     /*returns the indice in constraints that contains the smallest value not copied
       end is used to compute the number of copied elements (end-size) and is used as the next staring indices*/
-    end = fill_tab(&(const_tab[i].constraints), constraints, nb_constraints,start, (i+1) * nb_leaves, i * nb_leaves);
+    end = tm_fill_tab(&(const_tab[i].constraints), constraints, nb_constraints,start, (i+1) * nb_leaves, i * nb_leaves);
     const_tab[i].length = end-start;
     if(vl>=DEBUG){
       printf("Step %d\n",i);
-      printf("\tConstraint: "); print_1D_tab(constraints, nb_constraints);
-      printf("\tSub constraint: "); print_1D_tab(const_tab[i].constraints, end-start);
+      printf("\tConstraint: "); tm_print_1D_tab(constraints, nb_constraints);
+      printf("\tSub constraint: "); tm_print_1D_tab(const_tab[i].constraints, end-start);
     }
 
     if(end-start > N/k){
@@ -539,7 +541,7 @@ constraint_t *split_constraints (int *constraints, int nb_constraints, int k, tm
 
 
 /* split the com_mat of order n in k partiton according to parmutition table*/
-com_mat_t **split_com_mat(com_mat_t *com_mat, int n, int k, int *partition)
+static com_mat_t **split_com_mat(com_mat_t *com_mat, int n, int k, int *partition)
 {
   com_mat_t **res = NULL, *sub_com_mat;
   double **sub_mat = NULL;
@@ -550,8 +552,8 @@ com_mat_t **split_com_mat(com_mat_t *com_mat, int n, int k, int *partition)
 
 
   if(verbose_level >= DEBUG){
-    printf("Partition: "); print_1D_tab(partition,n);
-    display_tab(com_mat->comm,com_mat->n);
+    printf("Partition: "); tm_print_1D_tab(partition,n);
+    tm_display_tab(com_mat->comm,com_mat->n);
     printf("m=%d,n=%d,k=%d\n",m,n,k);
     printf("perm=%p\n", (void *)perm);
   }
@@ -571,8 +573,8 @@ com_mat_t **split_com_mat(com_mat_t *com_mat, int n, int k, int *partition)
 
     if(s>m){
       if(verbose_level >= CRITICAL){
-	fprintf(stderr,"Partition: "); print_1D_tab(partition,n);
-	display_tab(com_mat->comm,com_mat->n);
+	fprintf(stderr,"Partition: "); tm_print_1D_tab(partition,n);
+	tm_display_tab(com_mat->comm,com_mat->n);
 	fprintf(stderr,"too many elements of the partition for the permuation (s=%d>%d=m). n=%d, k=%d, cur_part= %d\n",s,m,n,k, cur_part);
       }
       exit(-1);
@@ -609,7 +611,7 @@ com_mat_t **split_com_mat(com_mat_t *com_mat, int n, int k, int *partition)
   return res;
 }
 
-int **split_vertices( int *vertices, int n, int k, int *partition)
+static int **split_vertices( int *vertices, int n, int k, int *partition)
 {
   int **res = NULL, *sub_vertices = NULL;
   int m = n/k;
@@ -620,8 +622,8 @@ int **split_vertices( int *vertices, int n, int k, int *partition)
 
 
   if(verbose_level >= DEBUG){
-    printf("Partition: ");print_1D_tab(partition,n);
-    printf("Vertices id: ");print_1D_tab(vertices,n);
+    printf("Partition: ");tm_print_1D_tab(partition,n);
+    printf("Vertices id: ");tm_print_1D_tab(vertices,n);
   }
 
   /*split the vertices tab of the partition cur_part  to the sub_vertices tab*/
@@ -633,14 +635,14 @@ int **split_vertices( int *vertices, int n, int k, int *partition)
 	sub_vertices[i++] = vertices[j];
     res[cur_part] = sub_vertices;
     if(verbose_level >= DEBUG){
-      printf("partition %d: ",cur_part);print_1D_tab(sub_vertices,m);
+      printf("partition %d: ",cur_part);tm_print_1D_tab(sub_vertices,m);
     }
   }
   /*exit(-1);*/
   return res;
 }
 
-void free_tab_com_mat(com_mat_t **mat,int k)
+static void free_tab_com_mat(com_mat_t **mat,int k)
 {
   int i,j;
   if( !mat )
@@ -656,7 +658,7 @@ void free_tab_com_mat(com_mat_t **mat,int k)
   FREE(mat);
 }
 
-void free_tab_local_vertices(int **mat, int k)
+static void free_tab_local_vertices(int **mat, int k)
 {
   int i; /* m=n/k; */
   if( !mat )
@@ -669,7 +671,7 @@ void free_tab_local_vertices(int **mat, int k)
 }
 
 
-void free_const_tab(constraint_t *const_tab, int k)
+static void free_const_tab(constraint_t *const_tab, int k)
 {
   int i;
 
@@ -717,7 +719,7 @@ static void display_partition(int *partition, int *local_vertices, int n, int de
   }	
 }
 
-void kpartition_build_level_topology(tm_tree_t *cur_node, com_mat_t *com_mat, int N, int depth,
+static void kpartition_build_level_topology(tm_tree_t *cur_node, com_mat_t *com_mat, int N, int depth,
 				     tm_topology_t *topology, int *local_vertices,
 				     int *constraints, int nb_constraints,
 				     double *obj_weight, double *comm_speed)
@@ -736,7 +738,7 @@ void kpartition_build_level_topology(tm_tree_t *cur_node, com_mat_t *com_mat, in
   if ( depth == topology->nb_levels - 1 ){
     if(verbose_level>=DEBUG)
       printf("id : %d, com_mat= %p\n",local_vertices[0], (void *)com_mat->comm);
-    set_node(cur_node,NULL, 0, NULL, local_vertices[0], 0, NULL, depth);
+    tm_set_node(cur_node,NULL, 0, NULL, local_vertices[0], 0, NULL, depth);
     return;
   }
 
@@ -748,7 +750,7 @@ void kpartition_build_level_topology(tm_tree_t *cur_node, com_mat_t *com_mat, in
   /* check_com_mat(com_mat); */
 
   /* partition the com_matrix in k partitions*/
-  partition = kpartition(k, com_mat, N, constraints, nb_constraints);
+  partition = tm_kpartition(k, com_mat, N, constraints, nb_constraints);
 
   if(verbose_level>=INFO)
     display_partition(partition, local_vertices, N, depth, k);
@@ -780,7 +782,7 @@ void kpartition_build_level_topology(tm_tree_t *cur_node, com_mat_t *com_mat, in
   }
 
   /* link the node with its child */
-  set_node( cur_node, tab_child, k, NULL, cur_node->id, 0, NULL, depth);
+  tm_set_node( cur_node, tab_child, k, NULL, cur_node->id, 0, NULL, depth);
 
   /* free local data*/
   FREE(partition);
@@ -790,7 +792,7 @@ void kpartition_build_level_topology(tm_tree_t *cur_node, com_mat_t *com_mat, in
 }
 
 
-tm_tree_t *kpartition_build_tree_from_topology(tm_topology_t *topology,double **comm,int N, int *constraints, int nb_constraints, double *obj_weight, double *com_speed)
+tm_tree_t *tm_kpartition_build_tree_from_topology(tm_topology_t *topology,double **comm,int N, int *constraints, int nb_constraints, double *obj_weight, double *com_speed)
 {
   int depth,i, K;
   tm_tree_t *root = NULL;
@@ -801,7 +803,7 @@ tm_tree_t *kpartition_build_tree_from_topology(tm_topology_t *topology,double **
   verbose_level = tm_get_verbose_level();
 
 
-  nb_cores=nb_processing_units(topology)*topology->oversub_fact;
+  nb_cores=tm_nb_processing_units(topology)*topology->oversub_fact;
 
 
   if(verbose_level>=INFO)
@@ -824,7 +826,7 @@ tm_tree_t *kpartition_build_tree_from_topology(tm_topology_t *topology,double **
   /* if we have more cores than processes add new dumb process to the com matrix*/
   if((K=nb_cores - N)>0){
     /* add K element to the object weight*/
-    complete_obj_weight(&obj_weight,N,K);
+    tm_complete_obj_weight(&obj_weight,N,K);
   } else if( K < 0){
     if(verbose_level>=ERROR)
       fprintf(stderr,"Not enough cores!\n");
@@ -847,9 +849,9 @@ tm_tree_t *kpartition_build_tree_from_topology(tm_topology_t *topology,double **
 
   local_vertices = (int*) MALLOC (sizeof(int) * (K+N));
 
-  for( i = 0 ; i < MIN(N,nb_constraints) ; i++)
+  for( i = 0 ; i < TM_MIN(N,nb_constraints) ; i++)
     local_vertices[i] = i;
-  for( i = MIN(N,nb_constraints) ;i < N + K ; i++)
+  for( i = TM_MIN(N,nb_constraints) ;i < N + K ; i++)
     local_vertices[i] = -1;
 
   /* we assume all objects have the same arity*/
diff --git a/3rd-party/treematch/tm_kpartitioning.h b/3rd-party/treematch/tm_kpartitioning.h
index 09c2227c066..766a3c9cfc3 100644
--- a/3rd-party/treematch/tm_kpartitioning.h
+++ b/3rd-party/treematch/tm_kpartitioning.h
@@ -1,3 +1,5 @@
+#include "ompi_config.h"
+
 typedef struct _com_mat_t{
   double **comm; 
   int n;  /*comm is of size n by n the other element are zeroes*/
@@ -5,5 +7,5 @@ typedef struct _com_mat_t{
 } com_mat_t;
 
 
-int  *kpartition(int, com_mat_t*, int, int *, int);
-tm_tree_t * kpartition_build_tree_from_topology(tm_topology_t *topology,double **com_mat,int N, int *constraints, int nb_constraints, double *obj_weight, double *com_speed);
+OMPI_HIDDEN int  *tm_kpartition(int, com_mat_t*, int, int *, int);
+OMPI_HIDDEN tm_tree_t * tm_kpartition_build_tree_from_topology(tm_topology_t *topology,double **com_mat,int N, int *constraints, int nb_constraints, double *obj_weight, double *com_speed);
diff --git a/3rd-party/treematch/tm_malloc.c b/3rd-party/treematch/tm_malloc.c
index b0eaa9e7030..422c94c28bf 100644
--- a/3rd-party/treematch/tm_malloc.c
+++ b/3rd-party/treematch/tm_malloc.c
@@ -10,7 +10,7 @@
 #include "tm_mt.h"
 
 
-#define MIN(a,b) ((a)<(b)?(a):(b))
+#define TM_MIN(a,b) ((a)<(b)?(a):(b))
 
 #define EXTRA_BYTE 100
 
@@ -102,10 +102,10 @@ void init_extra_data(void){
   if(done)
     return;
 
-  init_genrand(0);
+  tm_init_genrand(0);
 
   for( i = 0 ; i < EXTRA_BYTE; i++)
-    extra_data[i] = (char) genrand_int32() % 256;
+    extra_data[i] = (char) tm_genrand_int32() % 256;
 
   done = 1;
 }
@@ -182,7 +182,7 @@ void *tm_realloc(void *old_ptr, size_t size, char *file, int line){
     byte *original_ptr = ((byte *)old_ptr) - EXTRA_BYTE;
     size_t old_ptr_size = retreive_size(original_ptr);
 
-    memcpy(ptr + EXTRA_BYTE, old_ptr, MIN(old_ptr_size - 2 * EXTRA_BYTE, size));
+    memcpy(ptr + EXTRA_BYTE, old_ptr, TM_MIN(old_ptr_size - 2 * EXTRA_BYTE, size));
 
     if((bcmp(original_ptr ,extra_data, EXTRA_BYTE)) && ((tm_get_verbose_level()>=ERROR))){
       fprintf(stderr,"Realloc: cannot find special string ***before*** %p!\n", (void *)original_ptr);
diff --git a/3rd-party/treematch/tm_malloc.h b/3rd-party/treematch/tm_malloc.h
index f74cd3db6af..79754c6c70d 100644
--- a/3rd-party/treematch/tm_malloc.h
+++ b/3rd-party/treematch/tm_malloc.h
@@ -2,11 +2,13 @@
 #define _TM_MALLOC_H_
 
 #include <stdlib.h>
-void *tm_malloc(size_t size, char *, int);
-void *tm_calloc(size_t count, size_t size, char *, int);
-void *tm_realloc(void *ptr, size_t size, char *, int);
-void tm_free(void *ptr);
-void tm_mem_check(void);
+#include "ompi_config.h"
+
+OMPI_HIDDEN void *tm_malloc(size_t size, char *, int);
+OMPI_HIDDEN void *tm_calloc(size_t count, size_t size, char *, int);
+OMPI_HIDDEN void *tm_realloc(void *ptr, size_t size, char *, int);
+OMPI_HIDDEN void tm_free(void *ptr);
+OMPI_HIDDEN void tm_mem_check(void);
 
 /* for debugging malloc */
 /* #define __DEBUG_TM_MALLOC__ */
diff --git a/3rd-party/treematch/tm_mapping.c b/3rd-party/treematch/tm_mapping.c
index b30846c0ccd..6de0d6bcc7d 100644
--- a/3rd-party/treematch/tm_mapping.c
+++ b/3rd-party/treematch/tm_mapping.c
@@ -28,9 +28,6 @@
 
 #include <sys/mman.h>
 
-
-#define MIN(a,b) (a)<(b)?(a):(b)
-
 #define TEST_ERROR(n) do{ \
     if( (n) != 0 ){       \
        fprintf(stderr,"Error %d Line %d\n",n,__LINE__); \
@@ -47,25 +44,25 @@ typedef struct {
 } hash2_t;
 
 
-tm_affinity_mat_t * new_affinity_mat(double **mat, double *sum_row, int order, long int nnz);
-int compute_nb_leaves_from_level(int depth,tm_topology_t *topology);
-void depth_first(tm_tree_t *comm_tree, int *proc_list,int *i);
-int  fill_tab(int **new_tab,int *tab, int n, int start, int max_val, int shift);
-long int init_mat(char *filename,int N, double **mat, double *sum_row);
-void map_topology(tm_topology_t *topology,tm_tree_t *comm_tree, int level,
+OMPI_HIDDEN tm_affinity_mat_t * tm_new_affinity_mat(double **mat, double *sum_row, int order, long int nnz);
+OMPI_HIDDEN int tm_compute_nb_leaves_from_level(int depth,tm_topology_t *topology);
+static void depth_first(tm_tree_t *comm_tree, int *proc_list,int *i);
+OMPI_HIDDEN int  tm_fill_tab(int **new_tab,int *tab, int n, int start, int max_val, int shift);
+static long int init_mat(char *filename,int N, double **mat, double *sum_row);
+OMPI_HIDDEN void tm_map_topology(tm_topology_t *topology,tm_tree_t *comm_tree, int level,
 		  int *sigma, int nb_processes, int **k, int nb_compute_units);
-int nb_leaves(tm_tree_t *comm_tree);
-int nb_lines(char *filename);
-void print_1D_tab(int *tab,int N);
+static int nb_leaves(tm_tree_t *comm_tree);
+static int nb_lines(char *filename);
+OMPI_HIDDEN void tm_print_1D_tab(int *tab,int N);
 tm_solution_t * tm_compute_mapping(tm_topology_t *topology,tm_tree_t *comm_tree);
 void tm_free_affinity_mat(tm_affinity_mat_t *aff_mat);
-tm_affinity_mat_t *tm_load_aff_mat(char *filename);
-void update_comm_speed(double **comm_speed,int old_size,int new_size);
+OMPI_HIDDEN tm_affinity_mat_t *tm_load_aff_mat(char *filename);
+static void update_comm_speed(double **comm_speed,int old_size,int new_size);
 tm_affinity_mat_t * tm_build_affinity_mat(double **mat, int order);
 
 
 /* compute the number of leaves of any subtree starting froma node of depth depth*/
-int compute_nb_leaves_from_level(int depth,tm_topology_t *topology)
+int tm_compute_nb_leaves_from_level(int depth,tm_topology_t *topology)
 {
   int res = 1;
 
@@ -76,13 +73,13 @@ int compute_nb_leaves_from_level(int depth,tm_topology_t *topology)
 }
 
 void tm_finalize(){
-  terminate_thread_pool();
+  tm_terminate_thread_pool();
   tm_mem_check();
 }
 
 
 
-void print_1D_tab(int *tab,int N)
+void tm_print_1D_tab(int *tab,int N)
 {
   int i;
   for (i = 0; i < N; i++) {
@@ -93,7 +90,7 @@ void print_1D_tab(int *tab,int N)
   printf("\n");
 }
 
-int nb_lines(char *filename)
+static int nb_lines(char *filename)
 {
   FILE *pf = NULL;
   char line[LINE_SIZE];
@@ -117,7 +114,7 @@ int nb_lines(char *filename)
 
 
 
-long int  init_mat(char *filename,int N, double **mat, double *sum_row){
+static inline long int  init_mat(char *filename,int N, double **mat, double *sum_row){
   FILE *pf = NULL;
   char *ptr= NULL;
   char line[LINE_SIZE];
@@ -275,7 +272,7 @@ static long int init_mat_long(char *filename,int N, double **mat, double *sum_ro
 }
 #endif
 
-tm_affinity_mat_t * new_affinity_mat(double **mat, double *sum_row, int order, long int nnz){
+tm_affinity_mat_t * tm_new_affinity_mat(double **mat, double *sum_row, int order, long int nnz){
   tm_affinity_mat_t * aff_mat;
 
   aff_mat = (tm_affinity_mat_t *) MALLOC(sizeof(tm_affinity_mat_t));
@@ -304,7 +301,7 @@ tm_affinity_mat_t * tm_build_affinity_mat(double **mat, int order){
     }
   }
 
-  return new_affinity_mat(mat, sum_row, order, nnz);
+  return tm_new_affinity_mat(mat, sum_row, order, nnz);
 }
 
 
@@ -368,7 +365,7 @@ tm_affinity_mat_t *tm_load_aff_mat(char *filename)
     if(tm_get_verbose_level() >= INFO)
     printf("Affinity matrix built from %s!\n",filename);
 
-    return new_affinity_mat(mat, sum_row, order, nnz);
+    return tm_new_affinity_mat(mat, sum_row, order, nnz);
 
 
 }
@@ -396,7 +393,7 @@ tm_affinity_mat_t *tm_load_aff_mat(char *filename)
     }*/
 /* } */
 
-void depth_first(tm_tree_t *comm_tree, int *proc_list,int *i)
+static void depth_first(tm_tree_t *comm_tree, int *proc_list,int *i)
 {
   int j;
   if(!comm_tree->child){
@@ -408,7 +405,7 @@ void depth_first(tm_tree_t *comm_tree, int *proc_list,int *i)
     depth_first(comm_tree->child[j],proc_list,i);
 }
 
-int nb_leaves(tm_tree_t *comm_tree)
+static int nb_leaves(tm_tree_t *comm_tree)
 {
   int j,n=0;
 
@@ -452,7 +449,7 @@ static void set_val(int *tab, int val, int n){
  k_i =-1 if no process is mapped on core i
 */
 
-void map_topology(tm_topology_t *topology,tm_tree_t *comm_tree, int level,
+void tm_map_topology(tm_topology_t *topology,tm_tree_t *comm_tree, int level,
 		  int *sigma, int nb_processes, int **k, int nb_compute_units)
 {
   int *nodes_id = NULL;
@@ -482,7 +479,7 @@ void map_topology(tm_topology_t *topology,tm_tree_t *comm_tree, int level,
   if(k){/*if we need the k vector*/
     if(vl >= INFO)
       printf("M=%d, N=%d, BS=%d\n",M,N,block_size);
-    for( i = 0 ; i < nb_processing_units(topology) ; i++ )
+    for( i = 0 ; i < tm_nb_processing_units(topology) ; i++ )
       for(j = 0 ; j < topology->oversub_fact ; j++){
 	k[i][j] = -1;
     }
@@ -511,7 +508,7 @@ void map_topology(tm_topology_t *topology,tm_tree_t *comm_tree, int level,
 
   if((vl >= DEBUG) && (k)){
     printf("k: ");
-    for( i = 0 ; i < nb_processing_units(topology) ; i++ ){
+    for( i = 0 ; i < tm_nb_processing_units(topology) ; i++ ){
       printf("Procesing unit %d: ",i);
       for (j = 0 ; j<topology->oversub_fact; j++){
 	if( k[i][j] == -1)
@@ -531,7 +528,7 @@ tm_solution_t * tm_compute_mapping(tm_topology_t *topology,tm_tree_t *comm_tree)
   tm_solution_t *solution;
   int *sigma, **k;
   size_t sigma_length  = comm_tree->nb_processes;
-  size_t k_length      = nb_processing_units(topology);
+  size_t k_length      = tm_nb_processing_units(topology);
 
   solution =  (tm_solution_t *)MALLOC(sizeof(tm_solution_t));
   sigma    =  (int*)  MALLOC(sizeof(int) * sigma_length);
@@ -540,7 +537,7 @@ tm_solution_t * tm_compute_mapping(tm_topology_t *topology,tm_tree_t *comm_tree)
     k[i] =  (int*) MALLOC(sizeof(int) * topology->oversub_fact);
   }
 
-  map_topology(topology, comm_tree, topology->nb_levels-1, sigma, sigma_length ,k, k_length);
+  tm_map_topology(topology, comm_tree, topology->nb_levels-1, sigma, sigma_length ,k, k_length);
 
   solution->sigma         = sigma;
   solution->sigma_length  = sigma_length;
@@ -553,7 +550,7 @@ tm_solution_t * tm_compute_mapping(tm_topology_t *topology,tm_tree_t *comm_tree)
 
 
 
-void update_comm_speed(double **comm_speed,int old_size,int new_size)
+static inline void update_comm_speed(double **comm_speed,int old_size,int new_size)
 {
   double *old_tab = NULL,*new_tab= NULL;
   int i;
@@ -588,7 +585,7 @@ void update_comm_speed(double **comm_speed,int old_size,int new_size)
    copy element of tab in *new_tab from start to end and shift negativeley them
    allocates *new_tab
 */
-int  fill_tab(int **new_tab,int *tab, int n, int start, int max_val, int shift)
+int  tm_fill_tab(int **new_tab,int *tab, int n, int start, int max_val, int shift)
 {
   int *res = NULL,i,j,end;
 
diff --git a/3rd-party/treematch/tm_mapping.h b/3rd-party/treematch/tm_mapping.h
index 8e8c1985535..c19681eda7a 100644
--- a/3rd-party/treematch/tm_mapping.h
+++ b/3rd-party/treematch/tm_mapping.h
@@ -4,24 +4,19 @@
 #include "tm_topology.h"
 #include "tm_timings.h"
 #include "tm_verbose.h"
+#include "ompi_config.h"
 
-tm_affinity_mat_t * new_affinity_mat(double **mat, double *sum_row, int order, long int nnz);
-void   build_synthetic_proc_id(tm_topology_t *topology);
-tm_topology_t  *build_synthetic_topology(int *arity, int nb_levels, int *core_numbering, int nb_core_per_nodes);
-int compute_nb_leaves_from_level(int depth,tm_topology_t *topology);
-void depth_first(tm_tree_t *comm_tree, int *proc_list,int *i);
-int  fill_tab(int **new_tab,int *tab, int n, int start, int max_val, int shift);
-long int  init_mat(char *filename,int N, double **mat, double *sum_row);
-void map_topology(tm_topology_t *topology,tm_tree_t *comm_tree, int level,
+OMPI_HIDDEN tm_affinity_mat_t * tm_new_affinity_mat(double **mat, double *sum_row, int order, long int nnz);
+OMPI_HIDDEN tm_topology_t  *build_synthetic_topology(int *arity, int nb_levels, int *core_numbering, int nb_core_per_nodes);
+OMPI_HIDDEN int tm_compute_nb_leaves_from_level(int depth,tm_topology_t *topology);
+OMPI_HIDDEN int  tm_fill_tab(int **new_tab,int *tab, int n, int start, int max_val, int shift);
+OMPI_HIDDEN void tm_map_topology(tm_topology_t *topology,tm_tree_t *comm_tree, int level,
 		  int *sigma, int nb_processes, int **k, int nb_compute_units);
-int nb_leaves(tm_tree_t *comm_tree);
-int nb_lines(char *filename);
-int nb_processing_units(tm_topology_t *topology);
-void print_1D_tab(int *tab,int N);
-tm_solution_t * tm_compute_mapping(tm_topology_t *topology,tm_tree_t *comm_tree);
-void tm_free_affinity_mat(tm_affinity_mat_t *aff_mat);
-tm_affinity_mat_t *tm_load_aff_mat(char *filename);
-void update_comm_speed(double **comm_speed,int old_size,int new_size);
+OMPI_HIDDEN int tm_nb_processing_units(tm_topology_t *topology);
+OMPI_HIDDEN void tm_print_1D_tab(int *tab,int N);
+OMPI_HIDDEN tm_solution_t * tm_compute_mapping(tm_topology_t *topology,tm_tree_t *comm_tree);
+OMPI_HIDDEN void tm_free_affinity_mat(tm_affinity_mat_t *aff_mat);
+OMPI_HIDDEN tm_affinity_mat_t *tm_load_aff_mat(char *filename);
 
 /* use to split a constaint into subconstraint according the tree*/
 typedef struct{
diff --git a/3rd-party/treematch/tm_mt.c b/3rd-party/treematch/tm_mt.c
index 55c8f7bd905..791f5325c28 100644
--- a/3rd-party/treematch/tm_mt.c
+++ b/3rd-party/treematch/tm_mt.c
@@ -50,6 +50,8 @@
 
 #include "tm_mt.h"
 
+#include "ompi_config.h"
+
 /* Period parameters */
 #define N 624
 #define M 397
@@ -60,6 +62,13 @@
 static unsigned long x[N];      /* the array for the state vector  */
 static unsigned long *p0, *p1, *pm;
 
+OMPI_HIDDEN long tm_genrand_int31(void);
+OMPI_HIDDEN double tm_genrand_real1(void);
+OMPI_HIDDEN double tm_genrand_real2(void);
+OMPI_HIDDEN double tm_genrand_real3(void);
+OMPI_HIDDEN double tm_genrand_res53(void);
+OMPI_HIDDEN void init_by_array(unsigned long init_key[], int key_length);
+
 /*
    initialize with a seed
 
@@ -71,7 +80,7 @@ static unsigned long *p0, *p1, *pm;
    2002-01-09 modified by Makoto Matsumoto
 */
 void
-init_genrand(unsigned long s)
+tm_init_genrand(unsigned long s)
 {
   int i;
 
@@ -99,7 +108,7 @@ init_by_array(unsigned long init_key[], int key_length)
 {
   int i, j, k;
 
-  init_genrand(19650218UL);
+  tm_init_genrand(19650218UL);
   i = 1;
   j = 0;
   for (k = (N > key_length ? N : key_length); k; --k) {
@@ -128,13 +137,13 @@ init_by_array(unsigned long init_key[], int key_length)
 
 /* generates a random number on the interval [0,0xffffffff] */
 unsigned long
-genrand_int32(void)
+tm_genrand_int32(void)
 {
   unsigned long y;
 
   if (!p0) {
     /* Default seed */
-    init_genrand(5489UL);
+    tm_init_genrand(5489UL);
   }
   /* Twisted feedback */
   y = *p0 = *pm++ ^ (((*p0 & UPPER_MASK) | (*p1 & LOWER_MASK)) >> 1)
@@ -156,40 +165,40 @@ genrand_int32(void)
 
 /* generates a random number on the interval [0,0x7fffffff] */
 long
-genrand_int31(void)
+tm_genrand_int31(void)
 {
-  return (long) (genrand_int32() >> 1);
+  return (long) (tm_genrand_int32() >> 1);
 }
 
 /* generates a random number on the real interval [0,1] */
 double
-genrand_real1(void)
+tm_genrand_real1(void)
 {
-  return genrand_int32() * (1.0 / 4294967295.0);
+  return tm_genrand_int32() * (1.0 / 4294967295.0);
   /* divided by 2^32-1 */
 }
 
 /* generates a random number on the real interval [0,1) */
 double
-genrand_real2(void)
+tm_genrand_real2(void)
 {
-  return genrand_int32() * (1.0 / 4294967296.0);
+  return tm_genrand_int32() * (1.0 / 4294967296.0);
   /* divided by 2^32 */
 }
 
 /* generates a random number on the real interval (0,1) */
 double
-genrand_real3(void)
+tm_genrand_real3(void)
 {
-  return (((double) genrand_int32()) + 0.5) * (1.0 / 4294967296.0);
+  return (((double) tm_genrand_int32()) + 0.5) * (1.0 / 4294967296.0);
   /* divided by 2^32 */
 }
 
 /* generates a 53-bit random number on the real interval [0,1) */
 double
-genrand_res53(void)
+tm_genrand_res53(void)
 {
-  unsigned long a = genrand_int32() >> 5, b = genrand_int32() >> 6;
+  unsigned long a = tm_genrand_int32() >> 5, b = tm_genrand_int32() >> 6;
 
   return (a * 67108864.0 + b) * (1.0 / 9007199254740992.0);
 }
diff --git a/3rd-party/treematch/tm_mt.h b/3rd-party/treematch/tm_mt.h
index 58f50d8f509..a0044b2c7ce 100644
--- a/3rd-party/treematch/tm_mt.h
+++ b/3rd-party/treematch/tm_mt.h
@@ -1,10 +1,6 @@
-void init_genrand(unsigned long s);
-void init_by_array(unsigned long init_key[], int key_length);
+#include "ompi_config.h"
+
+OMPI_HIDDEN void tm_init_genrand(unsigned long s);
 
 /* generates a random number on the interval [0,0x7fffffff] */
-unsigned long genrand_int32(void); 
-long genrand_int31(void);
-double genrand_real1(void);
-double genrand_real2(void);
-double genrand_real3(void);
-double genrand_res53(void);
+OMPI_HIDDEN unsigned long tm_genrand_int32(void);
diff --git a/3rd-party/treematch/tm_solution.c b/3rd-party/treematch/tm_solution.c
index 4205c0d9d9a..a6cfc36a815 100644
--- a/3rd-party/treematch/tm_solution.c
+++ b/3rd-party/treematch/tm_solution.c
@@ -11,24 +11,25 @@ typedef struct {
 
 
 
-void tm_free_solution(tm_solution_t *sol);
-int distance(tm_topology_t *topology,int i, int j);
-double display_sol_sum_com(tm_topology_t *topology, tm_affinity_mat_t *aff_mat, int *sigma);
-  double display_sol(tm_topology_t *topology, tm_affinity_mat_t *aff_mat, int *sigma, tm_metric_t metric);
-double tm_display_solution(tm_topology_t *topology, tm_affinity_mat_t *aff_mat, tm_solution_t *sol,
+OMPI_HIDDEN void tm_free_solution(tm_solution_t *sol);
+static int distance(tm_topology_t *topology,int i, int j);
+static double display_sol_sum_com(tm_topology_t *topology, tm_affinity_mat_t *aff_mat, int *sigma);
+static double display_sol(tm_topology_t *topology, tm_affinity_mat_t *aff_mat, int *sigma, tm_metric_t metric);
+OMPI_HIDDEN double tm_display_solution(tm_topology_t *topology, tm_affinity_mat_t *aff_mat, tm_solution_t *sol,
 			   tm_metric_t metric);
-void tm_display_other_heuristics(tm_topology_t *topology, tm_affinity_mat_t *aff_mat, tm_metric_t metric);
-int in_tab(int *tab, int n, int val);
-void map_Packed(tm_topology_t *topology, int N, int *sigma);
-void map_RR(tm_topology_t * topology, int N, int *sigma);
-int hash_asc(const void* x1,const void* x2);
-int *generate_random_sol(tm_topology_t *topology,int N, int seed);
-double eval_sol(int *sol,int N,double **comm, double **arch);
-void exchange(int *sol,int i,int j);
-double gain_exchange(int *sol,int l,int m,double eval1,int N,double **comm, double **arch);
-void select_max(int *l,int *m,double **gain,int N,int *state);
-void compute_gain(int *sol,int N,double **gain,double **comm, double **arch);
-void map_MPIPP(tm_topology_t *topology,int nb_seed,int N,int *sigma,double **comm, double **arch);
+OMPI_HIDDEN void tm_display_other_heuristics(tm_topology_t *topology, tm_affinity_mat_t *aff_mat, tm_metric_t metric);
+OMPI_HIDDEN int tm_in_tab(int *tab, int n, int val);
+OMPI_HIDDEN void tm_map_Packed(tm_topology_t *topology, int N, int *sigma);
+OMPI_HIDDEN void tm_map_RR(tm_topology_t * topology, int N, int *sigma);
+static int hash_asc(const void* x1,const void* x2);
+static int *generate_random_sol(tm_topology_t *topology,int N, int seed);
+static double eval_sol(int *sol,int N,double **comm, double **arch);
+static void exchange(int *sol,int i,int j);
+static double gain_exchange(int *sol,int l,int m,double eval1,int N,double **comm, double **arch);
+static void select_max(int *l,int *m,double **gain,int N,int *state);
+static void compute_gain(int *sol,int N,double **gain,double **comm, double **arch);
+OMPI_HIDDEN void tm_map_MPIPP(tm_topology_t *topology,int nb_seed,int N,int *sigma,double **comm, double **arch);
+static void compute_gain(int *sol,int N,double **gain,double **comm, double **arch);
 
 
 void tm_free_solution(tm_solution_t *sol){
@@ -54,7 +55,7 @@ void tm_free_solution(tm_solution_t *sol){
    for which node i and j are still in the same subtree. This is done
    by iteratively dividing their numbering by the arity of the levels
 */
-int distance(tm_topology_t *topology,int i, int j)
+static int distance(tm_topology_t *topology,int i, int j)
 {
   int level = 0;
   int arity;
@@ -84,7 +85,7 @@ int distance(tm_topology_t *topology,int i, int j)
   return level;
 }
 
-double display_sol_sum_com(tm_topology_t *topology, tm_affinity_mat_t *aff_mat, int *sigma)
+static double display_sol_sum_com(tm_topology_t *topology, tm_affinity_mat_t *aff_mat, int *sigma)
 {
   double a,c,sol;
   int i,j;
@@ -186,7 +187,7 @@ static double display_sol_hop_byte(tm_topology_t *topology, tm_affinity_mat_t *a
 
 
 
-double display_sol(tm_topology_t *topology, tm_affinity_mat_t *aff_mat, int *sigma, tm_metric_t metric){
+static double display_sol(tm_topology_t *topology, tm_affinity_mat_t *aff_mat, int *sigma, tm_metric_t metric){
   switch (metric){
   case TM_METRIC_SUM_COM:
     return display_sol_sum_com(topology, aff_mat, sigma);
@@ -212,7 +213,7 @@ double tm_display_solution(tm_topology_t *topology, tm_affinity_mat_t *aff_mat,
 
   if(tm_get_verbose_level() >= DEBUG){
     printf("k: \n");
-    for( i = 0 ; i < nb_processing_units(topology) ; i++ ){
+    for( i = 0 ; i < tm_nb_processing_units(topology) ; i++ ){
       if(k[i][0] != -1){
 	printf("\tProcessing unit %d: ",i);
 	for (j = 0 ; j<topology->oversub_fact; j++){
@@ -236,11 +237,11 @@ void tm_display_other_heuristics(tm_topology_t *topology, tm_affinity_mat_t *aff
 
   sigma = (int*)MALLOC(sizeof(int)*N);
 
-  map_Packed(topology, N, sigma);
+  tm_map_Packed(topology, N, sigma);
   printf("Packed: ");
   display_sol(topology, aff_mat, sigma, metric);
 
-  map_RR(topology, N, sigma);
+  tm_map_RR(topology, N, sigma);
   printf("RR: ");
   display_sol(topology, aff_mat, sigma, metric);
 
@@ -272,7 +273,7 @@ void tm_display_other_heuristics(tm_topology_t *topology, tm_affinity_mat_t *aff
 }
 
 
-int in_tab(int *tab, int n, int val){
+int tm_in_tab(int *tab, int n, int val){
   int i;
   for( i = 0; i < n ; i++)
     if(tab[i] == val)
@@ -281,7 +282,7 @@ int in_tab(int *tab, int n, int val){
   return 0;
 }
 
-void map_Packed(tm_topology_t *topology, int N, int *sigma)
+void tm_map_Packed(tm_topology_t *topology, int N, int *sigma)
 {
   size_t i;
   int j = 0,depth;
@@ -291,7 +292,7 @@ void map_Packed(tm_topology_t *topology, int N, int *sigma)
 
   for( i = 0 ; i < topology->nb_nodes[depth] ; i++){
     /* printf ("%d -> %d\n",objs[i]->os_index,i); */
-    if((!topology->constraints) || (in_tab(topology->constraints, topology->nb_constraints, topology->node_id[i]))){
+    if((!topology->constraints) || (tm_in_tab(topology->constraints, topology->nb_constraints, topology->node_id[i]))){
       if(vl >= DEBUG)
 	printf ("%lu: %d -> %d\n", i, j, topology->node_id[i]);
       sigma[j++]=topology->node_id[i];
@@ -301,7 +302,7 @@ void map_Packed(tm_topology_t *topology, int N, int *sigma)
   }
 }
 
-void map_RR(tm_topology_t *topology, int N,int *sigma)
+void tm_map_RR(tm_topology_t *topology, int N,int *sigma)
 {
   int i;
   int vl = tm_get_verbose_level();
@@ -316,7 +317,7 @@ void map_RR(tm_topology_t *topology, int N,int *sigma)
   }
 }
 
-int hash_asc(const void* x1,const void* x2)
+static int hash_asc(const void* x1,const void* x2)
 {
   hash_t *e1 = NULL,*e2 = NULL;
 
@@ -327,7 +328,7 @@ int hash_asc(const void* x1,const void* x2)
 }
 
 
-int *generate_random_sol(tm_topology_t *topology,int N, int seed)
+static int *generate_random_sol(tm_topology_t *topology,int N, int seed)
 {
   hash_t *hash_tab = NULL;
   int *sol = NULL;
@@ -339,11 +340,11 @@ int *generate_random_sol(tm_topology_t *topology,int N, int seed)
   hash_tab = (hash_t*)MALLOC(sizeof(hash_t)*N);
   sol = (int*)MALLOC(sizeof(int)*N);
 
-  init_genrand(seed);
+  tm_init_genrand(seed);
 
   for( i = 0 ; i < N ; i++ ){
     hash_tab[i].val = nodes_id[i];
-    hash_tab[i].key = genrand_int32();
+    hash_tab[i].key = tm_genrand_int32();
   }
 
   qsort(hash_tab,N,sizeof(hash_t),hash_asc);
@@ -355,7 +356,7 @@ int *generate_random_sol(tm_topology_t *topology,int N, int seed)
 }
 
 
-double eval_sol(int *sol,int N,double **comm, double **arch)
+static double eval_sol(int *sol,int N,double **comm, double **arch)
 {
   double a,c,res;
   int i,j;
@@ -371,7 +372,7 @@ double eval_sol(int *sol,int N,double **comm, double **arch)
   return res;
 }
 
-void exchange(int *sol,int i,int j)
+static void exchange(int *sol,int i,int j)
 {
   int tmp;
   tmp = sol[i];
@@ -379,7 +380,7 @@ void exchange(int *sol,int i,int j)
   sol[j] = tmp;
 }
 
-double gain_exchange(int *sol,int l,int m,double eval1,int N,double **comm, double **arch)
+static double gain_exchange(int *sol,int l,int m,double eval1,int N,double **comm, double **arch)
 {
   double eval2;
   if( l == m )
@@ -391,7 +392,7 @@ double gain_exchange(int *sol,int l,int m,double eval1,int N,double **comm, doub
   return eval1-eval2;
 }
 
-void select_max(int *l,int *m,double **gain,int N,int *state)
+static void select_max(int *l,int *m,double **gain,int N,int *state)
 {
   double max;
   int i,j;
@@ -411,7 +412,7 @@ void select_max(int *l,int *m,double **gain,int N,int *state)
 }
 
 
-void compute_gain(int *sol,int N,double **gain,double **comm, double **arch)
+static void compute_gain(int *sol,int N,double **gain,double **comm, double **arch)
 {
   double eval1;
   int i,j;
@@ -429,7 +430,7 @@ parallel process placement toolset for smp clusters and multiclusters. In
 Gregory K. Egan and Yoichi Muraoka, editors, ICS, pages 353-360. ACM, 2006.
  */
 
-void map_MPIPP(tm_topology_t *topology,int nb_seed,int N,int *sigma,double **comm, double **arch)
+void tm_map_MPIPP(tm_topology_t *topology,int nb_seed,int N,int *sigma,double **comm, double **arch)
 {
   int *sol = NULL;
   int *state = NULL;
diff --git a/3rd-party/treematch/tm_solution.h b/3rd-party/treematch/tm_solution.h
index 8cc38a87558..ab5a061d37b 100644
--- a/3rd-party/treematch/tm_solution.h
+++ b/3rd-party/treematch/tm_solution.h
@@ -1,26 +1,16 @@
 #ifndef TM_SOLUION_H
 #define TM_SOLUION_H
 
+#include "ompi_config.h"
 #include "treematch.h"
 
-void tm_free_solution(tm_solution_t *sol);
-int distance(tm_topology_t *topology,int i, int j);
-double display_sol_sum_com(tm_topology_t *topology, tm_affinity_mat_t *aff_mat, int *sigma);
-  double display_sol(tm_topology_t *topology, tm_affinity_mat_t *aff_mat, int *sigma, tm_metric_t metric);
-double tm_display_solution(tm_topology_t *topology, tm_affinity_mat_t *aff_mat, tm_solution_t *sol,
-			   tm_metric_t metric);
-void tm_display_other_heuristics(tm_topology_t *topology, tm_affinity_mat_t *aff_mat, tm_metric_t metric);
-int in_tab(int *tab, int n, int val);
-void map_Packed(tm_topology_t *topology, int N, int *sigma);
-void map_RR(tm_topology_t *topology, int N, int *sigma);
-int hash_asc(const void* x1,const void* x2);
-int *generate_random_sol(tm_topology_t *topology,int N, int seed);
-double eval_sol(int *sol,int N,double **comm, double **arch);
-void exchange(int *sol,int i,int j);
-double gain_exchange(int *sol,int l,int m,double eval1,int N,double **comm, double **arch);
-void select_max(int *l,int *m,double **gain,int N,int *state);
-void compute_gain(int *sol,int N,double **gain,double **comm, double **arch);
-void map_MPIPP(tm_topology_t *topology,int nb_seed,int N,int *sigma,double **comm, double **arch);
+OMPI_HIDDEN void tm_free_solution(tm_solution_t *sol);
+OMPI_HIDDEN double tm_display_solution(tm_topology_t *topology, tm_affinity_mat_t *aff_mat, tm_solution_t *sol, tm_metric_t metric);
+OMPI_HIDDEN void tm_display_other_heuristics(tm_topology_t *topology, tm_affinity_mat_t *aff_mat, tm_metric_t metric);
+int tm_in_tab(int *tab, int n, int val);
+OMPI_HIDDEN void tm_map_Packed(tm_topology_t *topology, int N, int *sigma);
+OMPI_HIDDEN void tm_map_RR(tm_topology_t *topology, int N, int *sigma);
+OMPI_HIDDEN void tm_map_MPIPP(tm_topology_t *topology,int nb_seed,int N,int *sigma,double **comm, double **arch);
 
 
 #endif
diff --git a/3rd-party/treematch/tm_thread_pool.c b/3rd-party/treematch/tm_thread_pool.c
index 6617d2c6d55..52a737da55c 100644
--- a/3rd-party/treematch/tm_thread_pool.c
+++ b/3rd-party/treematch/tm_thread_pool.c
@@ -24,8 +24,8 @@ static thread_pool_t *create_threads(void);
 static void f1 (int nb_args, void **args, int thread_id);
 static void f2 (int nb_args, void **args, int thread_id);
 
-#define MIN(a, b) ((a)<(b)?(a):(b))
-#define MAX(a, b) ((a)>(b)?(a):(b))
+#define TM_MIN(a, b) ((a)<(b)?(a):(b))
+#define TM_MAX(a, b) ((a)>(b)?(a):(b))
 
 
 
@@ -45,7 +45,7 @@ int bind_myself_to_core(hwloc_topology_t topology, int id){
   int depth = hwloc_topology_get_depth(topology);
   int nb_cores = hwloc_get_nbobjs_by_depth(topology, depth-1);
   int my_core;
-  int nb_threads = get_nb_threads();
+  int nb_threads = tm_get_nb_threads();
   /* printf("depth=%d\n",depth); */
 
   switch (mapping_policy){
@@ -156,7 +156,7 @@ void add_work(pthread_mutex_t *list_lock, pthread_cond_t *cond_var, work_t *work
 }
 
 
-void wait_work_completion(work_t *work){
+void tm_wait_work_completion(work_t *work){
   pthread_mutex_lock(&work->mutex);
   while(!work->done)
     pthread_cond_wait(&work->work_done, &work->mutex);
@@ -164,7 +164,7 @@ void wait_work_completion(work_t *work){
 }
 
 
-int submit_work(work_t *work, int thread_id){
+int tm_submit_work(work_t *work, int thread_id){
   if( (thread_id>=0) && (thread_id< pool->nb_threads)){
     work->thread_id = thread_id;
     add_work(&pool->list_lock[thread_id], &pool->cond_var[thread_id], &pool->working_list[thread_id], work);
@@ -201,7 +201,7 @@ thread_pool_t *create_threads(){
 
   /* at depth 'depth' it is necessary a PU/core where we can execute things*/
   nb_cores = hwloc_get_nbobjs_by_depth(topology, depth-1);
-  nb_threads = MIN(nb_cores,  max_nb_threads);
+  nb_threads = TM_MIN(nb_cores,  max_nb_threads);
 
   if(verbose_level>=INFO)
     printf("nb_threads = %d\n",nb_threads);
@@ -241,7 +241,7 @@ thread_pool_t *get_thread_pool(){;
   return pool;
 }
 
-void terminate_thread_pool(){
+void tm_terminate_thread_pool(){
   int id;
   int *ret=NULL;
   work_t work;
@@ -249,7 +249,7 @@ void terminate_thread_pool(){
   if(pool){
     work.task=NULL;
     for (id=0;id<pool->nb_threads;id++){
-      submit_work(&work,id);
+      tm_submit_work(&work,id);
     }
 
 
@@ -277,13 +277,13 @@ void terminate_thread_pool(){
 
 
 
-int get_nb_threads(){
+int tm_get_nb_threads(){
   pool = get_thread_pool();
   return pool -> nb_threads;
 }
 
 
-work_t *create_work(int nb_args, void **args, void (*task) (int, void **, int)){
+work_t *tm_create_work(int nb_args, void **args, void (*task) (int, void **, int)){
   work_t *work;
   work = MALLOC(sizeof(work_t));
   work -> nb_args = nb_args;
@@ -298,7 +298,7 @@ work_t *create_work(int nb_args, void **args, void (*task) (int, void **, int)){
 }
 
 
-void destroy_work(work_t *work){
+void tm_destroy_work(work_t *work){
   pthread_cond_destroy(&work->work_done);
   pthread_mutex_destroy(&work->mutex);
   FREE(work);
@@ -333,7 +333,7 @@ void f2 (int nb_args, void **args, int thread_id){
 
 
 
-int test_main(void){
+int tm_test_main(void){
 
   int a=3, c;
   int b=-5;
@@ -342,7 +342,7 @@ int test_main(void){
   int tab[100];
   int i,res;
   work_t *work1,*work2,*work3,*work4;
-  int nb_threads = get_nb_threads();
+  int nb_threads = tm_get_nb_threads();
 
 
   printf("nb_threads= %d\n", nb_threads);
@@ -350,7 +350,7 @@ int test_main(void){
 
   args1[0] = &a;
   args1[1] = &b;
-  work1 = create_work(2,args1,f1);
+  work1 = tm_create_work(2,args1,f1);
 
 
   for (i=0;i<100;i++)
@@ -361,28 +361,28 @@ int test_main(void){
   args2[1] = tab;
   args2[2] = &res;
 
-  work2 = create_work(3, args2, f2);
-  work3 = create_work(4, args2, f2);
-  work4 = create_work(5, args2, f2);
+  work2 = tm_create_work(3, args2, f2);
+  work3 = tm_create_work(4, args2, f2);
+  work4 = tm_create_work(5, args2, f2);
 
-  submit_work(work1,0);
-  submit_work(work2,1);
-  submit_work(work3,1);
-  submit_work(work4,1);
+  tm_submit_work(work1,0);
+  tm_submit_work(work2,1);
+  tm_submit_work(work3,1);
+  tm_submit_work(work4,1);
 
 
 
-  terminate_thread_pool();
-  wait_work_completion(work1);
-  wait_work_completion(work2);
-  wait_work_completion(work3);
-  wait_work_completion(work4);
+  tm_terminate_thread_pool();
+  tm_wait_work_completion(work1);
+  tm_wait_work_completion(work2);
+  tm_wait_work_completion(work3);
+  tm_wait_work_completion(work4);
 
   printf("res=%d\n",res);
 
-  destroy_work(work1);
-  destroy_work(work2);
-  destroy_work(work3);
-  destroy_work(work4);
+  tm_destroy_work(work1);
+  tm_destroy_work(work2);
+  tm_destroy_work(work3);
+  tm_destroy_work(work4);
   return 0;
 }
diff --git a/3rd-party/treematch/tm_thread_pool.h b/3rd-party/treematch/tm_thread_pool.h
index 3499d261c0b..811b173d57b 100644
--- a/3rd-party/treematch/tm_thread_pool.h
+++ b/3rd-party/treematch/tm_thread_pool.h
@@ -3,6 +3,7 @@
 
 #include <pthread.h>
 #include <hwloc.h>
+#include "ompi_config.h"
 
 
 typedef struct _work_t{
@@ -35,13 +36,13 @@ typedef struct _thread_pool_t{
   hwloc_topology_t topology;
 }thread_pool_t;
 
-int get_nb_threads(void);
-int submit_work(work_t *work, int thread_id);
-void wait_work_completion(work_t *work);
-void terminate_thread_pool(void);
-work_t *create_work(int nb_args, void **args, void (int, void **, int));
-int test_main(void);
-void destroy_work(work_t *work);
+OMPI_HIDDEN int tm_get_nb_threads(void);
+OMPI_HIDDEN int tm_submit_work(work_t *work, int thread_id);
+OMPI_HIDDEN void tm_wait_work_completion(work_t *work);
+OMPI_HIDDEN void tm_terminate_thread_pool(void);
+OMPI_HIDDEN work_t *tm_create_work(int nb_args, void **args, void (int, void **, int));
+OMPI_HIDDEN int tm_test_main(void);
+OMPI_HIDDEN void tm_destroy_work(work_t *work);
 
 
 
diff --git a/3rd-party/treematch/tm_timings.c b/3rd-party/treematch/tm_timings.c
index b20747370e5..abebabbe656 100644
--- a/3rd-party/treematch/tm_timings.c
+++ b/3rd-party/treematch/tm_timings.c
@@ -3,7 +3,7 @@
 static CLOCK_T time_tab[MAX_CLOCK];
 static int clock_num = -1;
 
-void get_time(void)
+void tm_get_time(void)
 {
   clock_num++;
 
@@ -13,7 +13,7 @@ void get_time(void)
   CLOCK(time_tab[clock_num]);
 }
 
-double time_diff(void)
+double tm_time_diff(void)
 {
   CLOCK_T t2,t1;
 
diff --git a/3rd-party/treematch/tm_timings.h b/3rd-party/treematch/tm_timings.h
index 377a1cd46ef..8bc3760b7e0 100644
--- a/3rd-party/treematch/tm_timings.h
+++ b/3rd-party/treematch/tm_timings.h
@@ -2,6 +2,8 @@
 #define TIMINGS_H
 #include <stdio.h>
 
+#include "ompi_config.h"
+
 #ifndef _WIN32
 #include <sys/time.h>
 #else
@@ -36,11 +38,11 @@ typedef struct _timeb CLOCK_T;
 
 #endif
 
-double time_diff(void);
-void get_time(void);
+OMPI_HIDDEN double tm_time_diff(void);
+OMPI_HIDDEN void tm_get_time(void);
 
-#define TIC get_time()
-#define TOC time_diff()
+#define TIC tm_get_time()
+#define TOC tm_time_diff()
 
 #endif /*TIMINGS_H*/
 
diff --git a/3rd-party/treematch/tm_topology.c b/3rd-party/treematch/tm_topology.c
index 2c05c91dccb..8a3c8ab04fa 100644
--- a/3rd-party/treematch/tm_topology.c
+++ b/3rd-party/treematch/tm_topology.c
@@ -8,26 +8,27 @@
 
 
 tm_topology_t* tm_get_local_topo_with_hwloc(void);
-tm_topology_t* hwloc_to_tm(char *filename);
-int int_cmp_inc(const void* x1,const void* x2);
-void optimize_arity(int **arity, double **cost, int *nb_levels,int n);
-int symetric(hwloc_topology_t topology);
-tm_topology_t * tgt_to_tm(char *filename);
-void tm_display_arity(tm_topology_t *topology);
-void tm_display_topology(tm_topology_t *topology);
-void tm_free_topology(tm_topology_t *topology);
-tm_topology_t *tm_load_topology(char *arch_filename, tm_file_type_t arch_file_type);
-void tm_optimize_topology(tm_topology_t **topology);
-int  tm_topology_add_binding_constraints(char *constraints_filename, tm_topology_t *topology);
-int topo_nb_proc(hwloc_topology_t topology,int N);
-void topology_arity_cpy(tm_topology_t *topology,int **arity,int *nb_levels);
-void topology_constraints_cpy(tm_topology_t *topology,int **constraints,int *nb_constraints);
-void topology_cost_cpy(tm_topology_t *topology,double **cost);
-void topology_numbering_cpy(tm_topology_t *topology,int **numbering,int *nb_nodes);
-double ** topology_to_arch(hwloc_topology_t topology);
-void   build_synthetic_proc_id(tm_topology_t *topology);
+static tm_topology_t* hwloc_to_tm(char *filename);
+OMPI_HIDDEN int tm_int_cmp_inc(const void* x1,const void* x2);
+static void optimize_arity(int **arity, double **cost, int *nb_levels,int n);
+static int symetric(hwloc_topology_t topology);
+static tm_topology_t * tgt_to_tm(char *filename);
+OMPI_HIDDEN void tm_display_arity(tm_topology_t *topology);
+OMPI_HIDDEN void tm_display_topology(tm_topology_t *topology);
+OMPI_HIDDEN void tm_free_topology(tm_topology_t *topology);
+OMPI_HIDDEN tm_topology_t *tm_load_topology(char *arch_filename, tm_file_type_t arch_file_type);
+OMPI_HIDDEN void tm_optimize_topology(tm_topology_t **topology);
+OMPI_HIDDEN int  tm_topology_add_binding_constraints(char *constraints_filename, tm_topology_t *topology);
+static int topo_nb_proc(hwloc_topology_t topology,int N);
+static void topology_arity_cpy(tm_topology_t *topology,int **arity,int *nb_levels);
+static void topology_constraints_cpy(tm_topology_t *topology,int **constraints,int *nb_constraints);
+static void topology_cost_cpy(tm_topology_t *topology,double **cost);
+static void topology_numbering_cpy(tm_topology_t *topology,int **numbering,int *nb_nodes);
+static double ** topology_to_arch(hwloc_topology_t topology);
+static void   build_synthetic_proc_id(tm_topology_t *topology);
 tm_topology_t  *tm_build_synthetic_topology(int *arity, double *cost, int nb_levels, int *core_numbering, int nb_core_per_nodes);
-void            tm_set_numbering(tm_numbering_t new_val); /* TM_NUMBERING_LOGICAL or TM_NUMBERING_PHYSICAL */
+OMPI_HIDDEN void tm_set_numbering(tm_numbering_t new_val); /* TM_NUMBERING_LOGICAL or TM_NUMBERING_PHYSICAL */
+void   build_synthetic_proc_id(tm_topology_t *topology);
 
 
 #define LINE_SIZE (1000000)
@@ -46,7 +47,7 @@ tm_numbering_t  tm_get_numbering(){
 
 
 /* transform a tgt scotch file into a topology file*/
-tm_topology_t * tgt_to_tm(char *filename)
+static tm_topology_t * tgt_to_tm(char *filename)
 {
   tm_topology_t *topology = NULL;
   FILE *pf = NULL;
@@ -68,7 +69,13 @@ tm_topology_t * tgt_to_tm(char *filename)
     printf("Reading TGT file: %s\n",filename);
 
 
-  fgets(line,1024,pf);
+  if (NULL == fgets(line,1024,pf)) {
+      /* either an error has occurred (and is in an unknown state) or
+         we hit EOF and line is empty.  Either way, make line the
+         empty string to avoid errors later */
+      line[0] = '\0';
+  }
+
   fclose(pf);
 
   s = strstr(line,"tleaf");
@@ -116,12 +123,12 @@ tm_topology_t * tgt_to_tm(char *filename)
 
 
 
-int nb_processing_units(tm_topology_t *topology)
+int tm_nb_processing_units(tm_topology_t *topology)
 {
   return topology->nb_proc_units;
 }
 
-int topo_nb_proc(hwloc_topology_t topology,int N)
+static inline int topo_nb_proc(hwloc_topology_t topology,int N)
 {
   hwloc_obj_t *objs = NULL;
   int nb_proc;
@@ -152,14 +159,20 @@ static double link_cost(int depth)
   */
 }
 
-double ** topology_to_arch(hwloc_topology_t topology)
+static inline double ** topology_to_arch(hwloc_topology_t topology)
 {
   int nb_proc,i,j;
   hwloc_obj_t obj_proc1,obj_proc2,obj_res;
   double **arch = NULL;
 
   nb_proc = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_PU);
-  arch = (double**)MALLOC(sizeof(double*)*nb_proc);
+  if (nb_proc < 0) {
+    return NULL;
+  }
+  arch = (double**)malloc(sizeof(double*)*nb_proc);
+  if (NULL == arch) {
+    return NULL;
+  }
   for( i = 0 ; i < nb_proc ; i++ ){
     obj_proc1 = hwloc_get_obj_by_type(topology,HWLOC_OBJ_PU,i);
     arch[obj_proc1->os_index] = (double*)MALLOC(sizeof(double)*nb_proc);
@@ -173,7 +186,7 @@ double ** topology_to_arch(hwloc_topology_t topology)
   return arch;
 }
 
-int symetric(hwloc_topology_t topology)
+static int symetric(hwloc_topology_t topology)
 {
    int depth,i,topodepth = hwloc_topology_get_depth(topology);
    unsigned int arity;
@@ -235,7 +248,7 @@ static void build_process_tab_id(tm_topology_t *topology,  hwloc_obj_t *objs, ch
 }
 
 
-tm_topology_t* hwloc_to_tm(char *filename)
+static tm_topology_t* hwloc_to_tm(char *filename)
 {
   hwloc_topology_t topology;
   tm_topology_t *res = NULL;
@@ -473,7 +486,7 @@ void tm_display_arity(tm_topology_t *topology){
   printf("\n");
 }
 
-int int_cmp_inc(const void* x1,const void* x2)
+int tm_int_cmp_inc(const void* x1,const void* x2)
 {
   return *((int *)x1) < *((int *)x2) ? -1 : 1;
 }
@@ -484,7 +497,7 @@ static int topo_check_constraints(tm_topology_t *topology){
   int i;
   int depth = topology->nb_levels-1;
   for (i=0;i<n;i++){
-    if(!in_tab(topology->node_id, topology->nb_nodes[depth], topology->constraints[i])){
+    if(!tm_in_tab(topology->node_id, topology->nb_nodes[depth], topology->constraints[i])){
       if(tm_get_verbose_level() >= CRITICAL){
 	fprintf(stderr,"Error! Incompatible constraint with the topology: rank %d in the constraints is not a valid id of any nodes of the topology.\n",topology->constraints[i]);
       }
@@ -534,7 +547,9 @@ int  tm_topology_add_binding_constraints(char *constraints_filename, tm_topology
 
   /* compute the size of the array to store the constraints*/
   n = 0;
-  fgets(line, LINE_SIZE, pf);
+  if (NULL == fgets(line, LINE_SIZE, pf)) {
+    line[0] = '\0';
+  }
   l = line;
   while((ptr=strtok(l," \t"))){
     l = NULL;
@@ -545,7 +560,9 @@ int  tm_topology_add_binding_constraints(char *constraints_filename, tm_topology
   tab = (int*)MALLOC(n*sizeof(int));
 
   rewind(pf);
-  fgets(line, LINE_SIZE, pf);
+  if (NULL == fgets(line, LINE_SIZE, pf)) {
+    line[0] = '\0';
+  }
   fclose(pf);
   l = line;
   i = 0;
@@ -569,13 +586,13 @@ int  tm_topology_add_binding_constraints(char *constraints_filename, tm_topology
     exit(-1);
   }
 
-  qsort(tab,n,sizeof(int),int_cmp_inc);
+  qsort(tab,n,sizeof(int),tm_int_cmp_inc);
 
   return tm_topology_set_binding_constraints_cpy(tab, n, topology, 0);
 }
 
 
-void topology_numbering_cpy(tm_topology_t *topology,int **numbering,int *nb_nodes)
+static void topology_numbering_cpy(tm_topology_t *topology,int **numbering_loc,int *nb_nodes)
 {
   int nb_levels;
   unsigned int vl = tm_get_verbose_level();
@@ -584,18 +601,18 @@ void topology_numbering_cpy(tm_topology_t *topology,int **numbering,int *nb_node
   *nb_nodes = topology->nb_nodes[nb_levels-1];
   if(vl >= INFO)
     printf("nb_nodes=%d\n",*nb_nodes);
-  *numbering = (int*)MALLOC(sizeof(int)*(*nb_nodes));
-  memcpy(*numbering,topology->node_id,sizeof(int)*(*nb_nodes));
+  *numbering_loc = (int*)MALLOC(sizeof(int)*(*nb_nodes));
+  memcpy(*numbering_loc,topology->node_id,sizeof(int)*(*nb_nodes));
 }
 
-void topology_arity_cpy(tm_topology_t *topology,int **arity,int *nb_levels)
+static void topology_arity_cpy(tm_topology_t *topology,int **arity,int *nb_levels)
 {
   *nb_levels = topology->nb_levels;
   *arity = (int*)MALLOC(sizeof(int)*(*nb_levels));
   memcpy(*arity,topology->arity,sizeof(int)*(*nb_levels));
 }
 
-void topology_constraints_cpy(tm_topology_t *topology,int **constraints,int *nb_constraints)
+static void topology_constraints_cpy(tm_topology_t *topology,int **constraints,int *nb_constraints)
 {
   *nb_constraints = topology->nb_constraints;
   if(topology->constraints){
@@ -606,13 +623,13 @@ void topology_constraints_cpy(tm_topology_t *topology,int **constraints,int *nb_
   }
 }
 
-void topology_cost_cpy(tm_topology_t *topology,double **cost)
+static void topology_cost_cpy(tm_topology_t *topology,double **cost)
 {
   *cost = (double*)MALLOC(sizeof(double)*(topology->nb_levels));
   memcpy(*cost,topology->cost,sizeof(double)*(topology->nb_levels));
 }
 
-void optimize_arity(int **arity, double **cost, int *nb_levels,int n)
+static void optimize_arity(int **arity, double **cost, int *nb_levels,int n)
 {
   int a,i;
   int *new_arity = NULL;
@@ -699,7 +716,7 @@ void optimize_arity(int **arity, double **cost, int *nb_levels,int n)
 
 void tm_optimize_topology(tm_topology_t **topology){
   int *arity = NULL,nb_levels;
-  int *numbering = NULL,nb_nodes;
+  int *numbering_loc = NULL,nb_nodes;
   tm_topology_t *new_topo;
   double *cost;
   unsigned int vl = tm_get_verbose_level();
@@ -710,13 +727,13 @@ void tm_optimize_topology(tm_topology_t **topology){
     tm_display_arity(*topology);
 
   topology_arity_cpy(*topology,&arity,&nb_levels);
-  topology_numbering_cpy(*topology,&numbering,&nb_nodes);
+  topology_numbering_cpy(*topology,&numbering_loc,&nb_nodes);
   topology_constraints_cpy(*topology,&constraints,&nb_constraints);
   topology_cost_cpy(*topology,&cost);
 
 
   optimize_arity(&arity,&cost,&nb_levels,nb_levels-2);
-  new_topo = tm_build_synthetic_topology(arity, NULL, nb_levels,numbering,nb_nodes);
+  new_topo = tm_build_synthetic_topology(arity, NULL, nb_levels,numbering_loc,nb_nodes);
   new_topo->cost = cost;
   new_topo->constraints    = constraints;
   new_topo->nb_constraints = nb_constraints;
@@ -736,7 +753,7 @@ void tm_optimize_topology(tm_topology_t **topology){
     tm_display_arity(new_topo);
   }
   FREE(arity);
-  FREE(numbering);
+  FREE(numbering_loc);
   tm_free_topology(*topology);
 
   *topology = new_topo;
@@ -811,7 +828,7 @@ tm_topology_t  *tm_build_synthetic_topology(int *arity, double *cost, int nb_lev
 }
 
 
-void   build_synthetic_proc_id(tm_topology_t *topology)
+static void   build_synthetic_proc_id(tm_topology_t *topology)
 {
   int i;
   size_t j,n = 1;
diff --git a/3rd-party/treematch/tm_topology.h b/3rd-party/treematch/tm_topology.h
index a7b04dee745..2eb742954f4 100644
--- a/3rd-party/treematch/tm_topology.h
+++ b/3rd-party/treematch/tm_topology.h
@@ -2,22 +2,16 @@
 #include "tm_tree.h"
 
 tm_topology_t* get_local_topo_with_hwloc(void);
-tm_topology_t* hwloc_to_tm(char *filename);
-int int_cmp_inc(const void* x1,const void* x2);
-void optimize_arity(int **arity, double **cost, int *nb_levels,int n);
-int symetric(hwloc_topology_t topology);
-tm_topology_t * tgt_to_tm(char *filename);
-void tm_display_arity(tm_topology_t *topology);
-void tm_display_topology(tm_topology_t *topology);
-void tm_free_topology(tm_topology_t *topology);
-tm_topology_t *tm_load_topology(char *arch_filename, tm_file_type_t arch_file_type);
-void tm_optimize_topology(tm_topology_t **topology);
-int  tm_topology_add_binding_constraints(char *constraints_filename, tm_topology_t *topology);
-int topo_nb_proc(hwloc_topology_t topology,int N);
+OMPI_HIDDEN int tm_int_cmp_inc(const void* x1,const void* x2);
+OMPI_HIDDEN void tm_display_arity(tm_topology_t *topology);
+OMPI_HIDDEN void tm_display_topology(tm_topology_t *topology);
+OMPI_HIDDEN void tm_free_topology(tm_topology_t *topology);
+OMPI_HIDDEN tm_topology_t *tm_load_topology(char *arch_filename, tm_file_type_t arch_file_type);
+OMPI_HIDDEN void tm_optimize_topology(tm_topology_t **topology);
+OMPI_HIDDEN int  tm_topology_add_binding_constraints(char *constraints_filename, tm_topology_t *topology);
 void topology_arity(tm_topology_t *topology,int **arity,int *nb_levels);
 void topology_constraints(tm_topology_t *topology,int **constraints,int *nb_constraints);
 void topology_cost(tm_topology_t *topology,double **cost);
 void topology_numbering(tm_topology_t *topology,int **numbering,int *nb_nodes);
-double ** topology_to_arch(hwloc_topology_t topology);
-int nb_processing_units(tm_topology_t *topology);
+OMPI_HIDDEN int tm_nb_processing_units(tm_topology_t *topology);
 
diff --git a/3rd-party/treematch/tm_tree.c b/3rd-party/treematch/tm_tree.c
index 11d3e46a2db..06a369e1d8d 100644
--- a/3rd-party/treematch/tm_tree.c
+++ b/3rd-party/treematch/tm_tree.c
@@ -5,6 +5,7 @@
 #include <assert.h>
 #include <pthread.h>
 
+#include "ompi_config.h"
 #include "tm_tree.h"
 #include "tm_mapping.h"
 #include "tm_timings.h"
@@ -14,8 +15,8 @@
 #include "tm_thread_pool.h"
 
 
-#define MIN(a, b) ((a)<(b)?(a):(b))
-#define MAX(a, b) ((a)>(b)?(a):(b))
+#define TM_MIN(a, b) ((a)<(b)?(a):(b))
+#define TM_MAX(a, b) ((a)>(b)?(a):(b))
 
 #ifndef __CHARMC__
 #define __CHARMC__ 0
@@ -32,62 +33,63 @@
 static int verbose_level = ERROR;
 static int exhaustive_search_flag = 0;
 
-void free_list_child(tm_tree_t *);void free_tab_child(tm_tree_t *);
-double choose (long, long);void display_node(tm_tree_t *);
-void clone_tree(tm_tree_t *, tm_tree_t *);
-double *aggregate_obj_weight(tm_tree_t *, double *, int);
+static void free_list_child(tm_tree_t *);
+static void free_tab_child(tm_tree_t *);
+static double choose (long, long);void display_node(tm_tree_t *);
+static void clone_tree(tm_tree_t *, tm_tree_t *);
+static double *aggregate_obj_weight(tm_tree_t *, double *, int);
 tm_affinity_mat_t *aggregate_com_mat(tm_tree_t *, tm_affinity_mat_t *, int);
-double eval_grouping(tm_affinity_mat_t *, tm_tree_t **, int);
-group_list_t *new_group_list(tm_tree_t **, double, group_list_t *);
-void add_to_list(group_list_t *, tm_tree_t **, int, double);
-void  list_all_possible_groups(tm_affinity_mat_t *, tm_tree_t *, int, int, int, tm_tree_t **, group_list_t *);
-int independent_groups(group_list_t **, int, group_list_t *, int);
-void display_selection (group_list_t**, int, int, double);
-void display_grouping (tm_tree_t *, int, int, double);
-int recurs_select_independent_groups(group_list_t **, int, int, int, int,
-				     int, double, double *, group_list_t **, group_list_t **);
-int test_independent_groups(group_list_t **, int, int, int, int, int, double, double *,
-			    group_list_t **, group_list_t **);
-void delete_group_list(group_list_t *);
-int group_list_id(const void*, const void*);
-int group_list_asc(const void*, const void*);
-int group_list_dsc(const void*, const void*);
-int weighted_degree_asc(const void*, const void*);
-int weighted_degree_dsc(const void*, const void*);
-int  select_independent_groups(group_list_t **, int, int, int, double *, group_list_t **, int, double);
-int  select_independent_groups_by_largest_index(group_list_t **, int, int, int, double *,
-						group_list_t **, int, double);
-void list_to_tab(group_list_t *, group_list_t **, int);
-void display_tab_group(group_list_t **, int, int);
-int independent_tab(tm_tree_t **, tm_tree_t **, int);
-void compute_weighted_degree(group_list_t **, int, int);
+static double eval_grouping(tm_affinity_mat_t *, tm_tree_t **, int);
+static group_list_t *new_group_list(tm_tree_t **, double, group_list_t *);
+static void add_to_list(group_list_t *, tm_tree_t **, int, double);
+static void  list_all_possible_groups(tm_affinity_mat_t *, tm_tree_t *, int, int, int, tm_tree_t **, group_list_t *);
+static int independent_groups(group_list_t **, int, group_list_t *, int);
+static void display_selection (group_list_t**, int, int, double);
+static void display_grouping (tm_tree_t *, int, int, double);
+static int recurs_select_independent_groups(group_list_t **, int, int, int, int,
+                                            int, double, double *, group_list_t **, group_list_t **);
+static int test_independent_groups(group_list_t **, int, int, int, int, int, double, double *,
+                                   group_list_t **, group_list_t **);
+static void delete_group_list(group_list_t *);
+static int group_list_id(const void*, const void*);
+static int group_list_asc(const void*, const void*);
+static int group_list_dsc(const void*, const void*);
+static int weighted_degree_asc(const void*, const void*);
+static int weighted_degree_dsc(const void*, const void*);
+static int  select_independent_groups(group_list_t **, int, int, int, double *, group_list_t **, int, double);
+static int  select_independent_groups_by_largest_index(group_list_t **, int, int, int, double *,
+                                                       group_list_t **, int, double);
+static void list_to_tab(group_list_t *, group_list_t **, int);
+static void display_tab_group(group_list_t **, int, int);
+static int independent_tab(tm_tree_t **, tm_tree_t **, int);
+static void compute_weighted_degree(group_list_t **, int, int);
 void  group(tm_affinity_mat_t *, tm_tree_t *, tm_tree_t *, int, int, int, double *, tm_tree_t **);
-void  fast_group(tm_affinity_mat_t *, tm_tree_t *, tm_tree_t *, int, int, int, double *, tm_tree_t **, int *, int);
-int adjacency_asc(const void*, const void*);
-int adjacency_dsc(const void*, const void*);
-		 void super_fast_grouping(tm_affinity_mat_t *, tm_tree_t *, tm_tree_t *, int, int);
-tm_affinity_mat_t *build_cost_matrix(tm_affinity_mat_t *, double *, double);
-void group_nodes(tm_affinity_mat_t *, tm_tree_t *, tm_tree_t *, int , int, double*, double);
-double fast_grouping(tm_affinity_mat_t *, tm_tree_t *, tm_tree_t *, int, int, double);
-void complete_aff_mat(tm_affinity_mat_t **, int, int);
-void complete_obj_weight(double **, int, int);
-void create_dumb_tree(tm_tree_t *, int, tm_topology_t *);
-void complete_tab_node(tm_tree_t **, int, int, int, tm_topology_t *);
-void set_deb_tab_child(tm_tree_t *, tm_tree_t *, int);
-tm_tree_t *build_level_topology(tm_tree_t *, tm_affinity_mat_t *, int, int, tm_topology_t *, double *, double *);
-int check_constraints(tm_topology_t  *, int **);
-tm_tree_t *bottom_up_build_tree_from_topology(tm_topology_t *, tm_affinity_mat_t *, double *, double *);
-void free_non_constraint_tree(tm_tree_t *);
-void free_constraint_tree(tm_tree_t *);
-void free_tab_double(double**, int);
-void free_tab_int(int**, int );
+static void  fast_group(tm_affinity_mat_t *, tm_tree_t *, tm_tree_t *, int, int, int, double *, tm_tree_t **, int *, int);
+static int adjacency_asc(const void*, const void*);
+static int adjacency_dsc(const void*, const void*);
+static void super_fast_grouping(tm_affinity_mat_t *, tm_tree_t *, tm_tree_t *, int, int);
+static tm_affinity_mat_t *build_cost_matrix(tm_affinity_mat_t *, double *, double);
+static void group_nodes(tm_affinity_mat_t *, tm_tree_t *, tm_tree_t *, int , int, double*, double);
+static double fast_grouping(tm_affinity_mat_t *, tm_tree_t *, tm_tree_t *, int, int, double);
+static void complete_aff_mat(tm_affinity_mat_t **, int, int);
+OMPI_HIDDEN void tm_complete_obj_weight(double **, int, int);
+static void create_dumb_tree(tm_tree_t *, int, tm_topology_t *);
+static void complete_tab_node(tm_tree_t **, int, int, int, tm_topology_t *);
+static void set_deb_tab_child(tm_tree_t *, tm_tree_t *, int);
+static tm_tree_t *build_level_topology(tm_tree_t *, tm_affinity_mat_t *, int, int, tm_topology_t *, double *, double *);
+static int check_constraints(tm_topology_t  *, int **);
+static tm_tree_t *bottom_up_build_tree_from_topology(tm_topology_t *, tm_affinity_mat_t *, double *, double *);
+static void free_non_constraint_tree(tm_tree_t *);
+static void free_constraint_tree(tm_tree_t *);
+static void free_tab_double(double**, int);
+static void free_tab_int(int**, int );
 static void partial_aggregate_aff_mat (int, void **, int);
-void free_affinity_mat(tm_affinity_mat_t *aff_mat);
-int int_cmp_inc(const void* x1, const void* x2);
+static void free_affinity_mat(tm_affinity_mat_t *aff_mat);
+OMPI_HIDDEN int tm_int_cmp_inc(const void* x1, const void* x2);
 
 
 
-double choose (long n, long k)
+static double choose (long n, long k)
 {
   /* compute C_n_k */
   double res = 1;
@@ -109,13 +111,13 @@ int tm_get_exhaustive_search_flag(){
 }
 
 
-void free_affinity_mat(tm_affinity_mat_t *aff_mat){
+static void free_affinity_mat(tm_affinity_mat_t *aff_mat){
   free_tab_double(aff_mat->mat, aff_mat->order);
   FREE(aff_mat->sum_row);
   FREE(aff_mat);
 }
 
-void free_list_child(tm_tree_t *tree)
+static void free_list_child(tm_tree_t *tree)
 {
   int i;
 
@@ -128,7 +130,7 @@ void free_list_child(tm_tree_t *tree)
       FREE(tree);
   }
 }
-void free_tab_child(tm_tree_t *tree)
+static void free_tab_child(tm_tree_t *tree)
 {
   if(tree){
     /*in a non constaint tree internal node are allocated in an array an stored ib tab_child : they are freed globaly here */
@@ -137,7 +139,7 @@ void free_tab_child(tm_tree_t *tree)
   }
 }
 
-void free_non_constraint_tree(tm_tree_t *tree)
+static void free_non_constraint_tree(tm_tree_t *tree)
 {
   if(tree->dumb){
     if(tm_get_verbose_level() <= CRITICAL){
@@ -151,7 +153,7 @@ void free_non_constraint_tree(tm_tree_t *tree)
   FREE(tree);
 }
 
-void free_constraint_tree(tm_tree_t *tree)
+static void free_constraint_tree(tm_tree_t *tree)
 {
   int i;
 
@@ -174,7 +176,7 @@ void tm_free_tree(tm_tree_t *tree)
 }
 
 
-void set_node(tm_tree_t *node, tm_tree_t ** child, int arity, tm_tree_t *parent,
+void tm_set_node(tm_tree_t *node, tm_tree_t ** child, int arity, tm_tree_t *parent,
 	      int id, double val, tm_tree_t *tab_child, int depth)
 {
   static int uniq = 0;
@@ -189,14 +191,7 @@ void set_node(tm_tree_t *node, tm_tree_t ** child, int arity, tm_tree_t *parent,
   node->dumb = 0;
 }
 
-void display_node(tm_tree_t *node)
-{
-  if (verbose_level >= DEBUG)
-    printf("child : %p\narity : %d\nparent : %p\nid : %d\nval : %f\nuniq : %d\n\n",
-	   (void *)(node->child), node->arity, (void *)(node->parent), node->id, node->val, node->uniq);
-}
-
-void clone_tree(tm_tree_t *new, tm_tree_t *old)
+static void clone_tree(tm_tree_t *new, tm_tree_t *old)
 {
   int i;
   new->child = old->child;
@@ -213,7 +208,7 @@ void clone_tree(tm_tree_t *new, tm_tree_t *old)
 }
 
 
-double *aggregate_obj_weight(tm_tree_t *new_tab_node, double *tab, int M)
+static double *aggregate_obj_weight(tm_tree_t *new_tab_node, double *tab, int M)
 {
   int i, i1, id1;
   double *res = NULL;
@@ -298,7 +293,7 @@ static tm_affinity_mat_t *aggregate_aff_mat(tm_tree_t *tab_node, tm_affinity_mat
     int *sup;
     long int *nnz_tab;
 
-    nb_threads = MIN(M/512, get_nb_threads());
+    nb_threads = TM_MIN(M/512, tm_get_nb_threads());
     works = (work_t**)MALLOC(sizeof(work_t*)*nb_threads);
     inf = (int*)MALLOC(sizeof(int)*nb_threads);
     sup = (int*)MALLOC(sizeof(int)*nb_threads);
@@ -318,18 +313,18 @@ static tm_affinity_mat_t *aggregate_aff_mat(tm_tree_t *tab_node, tm_affinity_mat
       args[6]=(void*)sum_row;
       args[7]=(void*)(nnz_tab+id);
 
-      works[id]= create_work(8, args, partial_aggregate_aff_mat);
+      works[id]= tm_create_work(8, args, partial_aggregate_aff_mat);
       if(verbose_level >= DEBUG)
 	printf("Executing %p\n", (void *)works[id]);
 
-      submit_work( works[id], id);
+      tm_submit_work( works[id], id);
     }
 
     for(id=0;id<nb_threads;id++){
-      wait_work_completion(works[id]);
+      tm_wait_work_completion(works[id]);
       FREE(works[id]->args);
       nnz += nnz_tab[id];
-      destroy_work(works[id]);
+      tm_destroy_work(works[id]);
     }
 
     FREE(inf);
@@ -358,10 +353,10 @@ static tm_affinity_mat_t *aggregate_aff_mat(tm_tree_t *tab_node, tm_affinity_mat
     }
   }
  
-  return new_affinity_mat(new_mat, sum_row, M, nnz);
+  return tm_new_affinity_mat(new_mat, sum_row, M, nnz);
 }
 
-void free_tab_double(double**tab, int mat_order)
+static void free_tab_double(double**tab, int mat_order)
 {
   int i;
   for( i = 0 ; i < mat_order ; i++ )
@@ -369,7 +364,7 @@ void free_tab_double(double**tab, int mat_order)
   FREE(tab);
 }
 
-void free_tab_int(int**tab, int mat_order)
+static inline void free_tab_int(int**tab, int mat_order)
 {
   int i;
   for( i = 0 ; i < mat_order ; i++ )
@@ -377,7 +372,7 @@ void free_tab_int(int**tab, int mat_order)
   FREE(tab);
 }
 
-void display_tab(double **tab, int mat_order)
+void tm_display_tab(double **tab, int mat_order)
 {
   int i, j;
   double line, total = 0;
@@ -402,8 +397,7 @@ void display_tab(double **tab, int mat_order)
   /* printf("Total: %.2f\n", total);*/
 }
 
-
-double eval_grouping(tm_affinity_mat_t *aff_mat, tm_tree_t **cur_group, int arity)
+static double eval_grouping(tm_affinity_mat_t *aff_mat, tm_tree_t **cur_group, int arity)
 {
   double res = 0;
   int i, j, id, id1, id2;
@@ -430,7 +424,7 @@ double eval_grouping(tm_affinity_mat_t *aff_mat, tm_tree_t **cur_group, int arit
 }
 
 
-group_list_t *new_group_list(tm_tree_t **tab, double val, group_list_t *next)
+static group_list_t *new_group_list(tm_tree_t **tab, double val, group_list_t *next)
 {
   group_list_t *res = NULL;
 
@@ -443,7 +437,7 @@ group_list_t *new_group_list(tm_tree_t **tab, double val, group_list_t *next)
 }
 
 
-void add_to_list(group_list_t *list, tm_tree_t **cur_group, int arity, double val)
+static void add_to_list(group_list_t *list, tm_tree_t **cur_group, int arity, double val)
 {
   group_list_t *elem = NULL;
   tm_tree_t **tab = NULL;
@@ -466,7 +460,7 @@ void add_to_list(group_list_t *list, tm_tree_t **cur_group, int arity, double va
 }
 
 
-void  list_all_possible_groups(tm_affinity_mat_t *aff_mat, tm_tree_t *tab_node, int id, int arity, int depth,
+static void  list_all_possible_groups(tm_affinity_mat_t *aff_mat, tm_tree_t *tab_node, int id, int arity, int depth,
 			       tm_tree_t **cur_group, group_list_t *list)
 {
   double val;
@@ -490,7 +484,7 @@ void  list_all_possible_groups(tm_affinity_mat_t *aff_mat, tm_tree_t *tab_node,
   }
 }
 
-void update_val(tm_affinity_mat_t *aff_mat, tm_tree_t *parent)
+void tm_update_val(tm_affinity_mat_t *aff_mat, tm_tree_t *parent)
 {
   /* int i; */
 
@@ -508,7 +502,7 @@ void update_val(tm_affinity_mat_t *aff_mat, tm_tree_t *parent)
   /*printf(": %f\n", parent->val);*/
 }
 
-int independent_groups(group_list_t **selection, int d, group_list_t *elem, int arity)
+static int independent_groups(group_list_t **selection, int d, group_list_t *elem, int arity)
 {
   int i, j, k;
 
@@ -525,7 +519,7 @@ int independent_groups(group_list_t **selection, int d, group_list_t *elem, int
 
 
 
-void display_selection (group_list_t** selection, int M, int arity, double val)
+static void display_selection (group_list_t** selection, int M, int arity, double val)
 {
   int i, j;
   double local_val = 0;
@@ -544,7 +538,7 @@ void display_selection (group_list_t** selection, int M, int arity, double val)
 }
 
 
-void display_grouping (tm_tree_t *father, int M, int arity, double val)
+static void display_grouping (tm_tree_t *father, int M, int arity, double val)
 {
   int i, j;
 
@@ -561,7 +555,7 @@ void display_grouping (tm_tree_t *father, int M, int arity, double val)
 }
 
 
-int recurs_select_independent_groups(group_list_t **tab, int i, int n, int arity, int d, int M, double val, double *best_val, group_list_t **selection, group_list_t **best_selection)
+static int recurs_select_independent_groups(group_list_t **tab, int i, int n, int arity, int d, int M, double val, double *best_val, group_list_t **selection, group_list_t **best_selection)
 {
   group_list_t *elem = NULL;
   /*
@@ -597,7 +591,7 @@ int recurs_select_independent_groups(group_list_t **tab, int i, int n, int arity
 
 
 
-int test_independent_groups(group_list_t **tab, int i, int n, int arity, int d, int M, double val, double *best_val, group_list_t **selection, group_list_t **best_selection)
+static int test_independent_groups(group_list_t **tab, int i, int n, int arity, int d, int M, double val, double *best_val, group_list_t **selection, group_list_t **best_selection)
 {
   group_list_t *elem = NULL;
 
@@ -619,7 +613,7 @@ int test_independent_groups(group_list_t **tab, int i, int n, int arity, int d,
   return 0;
 }
 
-void  delete_group_list(group_list_t *list)
+static void  delete_group_list(group_list_t *list)
 {
 
   if(list){
@@ -629,7 +623,7 @@ void  delete_group_list(group_list_t *list)
   }
 }
 
-int group_list_id(const void* x1, const void* x2)
+static int group_list_id(const void* x1, const void* x2)
 {
   group_list_t *e1 = NULL, *e2= NULL;
 
@@ -639,7 +633,7 @@ int group_list_id(const void* x1, const void* x2)
   return (e1->tab[0]->id < e2->tab[0]->id) ? - 1 : 1;
 }
 
-int group_list_asc(const void* x1, const void* x2)
+static int group_list_asc(const void* x1, const void* x2)
 {
   group_list_t *e1 = NULL, *e2 = NULL;
 
@@ -649,7 +643,7 @@ int group_list_asc(const void* x1, const void* x2)
   return (e1->val < e2->val) ? - 1 : 1;
 }
 
-int group_list_dsc(const void* x1, const void* x2)
+static int group_list_dsc(const void* x1, const void* x2)
 {
   group_list_t *e1 = NULL, *e2 = NULL;
 
@@ -659,7 +653,7 @@ int group_list_dsc(const void* x1, const void* x2)
   return (e1->val > e2->val) ? -1 : 1;
 }
 
-int weighted_degree_asc(const void* x1, const void* x2)
+static inline int weighted_degree_asc(const void* x1, const void* x2)
 {
   group_list_t *e1= NULL, *e2 = NULL;
 
@@ -669,7 +663,7 @@ int weighted_degree_asc(const void* x1, const void* x2)
   return (e1->wg > e2->wg) ? 1 : -1;
 }
 
-int weighted_degree_dsc(const void* x1, const void* x2)
+static int weighted_degree_dsc(const void* x1, const void* x2)
 {
   group_list_t *e1 = NULL, *e2 = NULL;
 
@@ -679,8 +673,8 @@ int weighted_degree_dsc(const void* x1, const void* x2)
   return (e1->wg > e2->wg) ? - 1 : 1;
 }
 
-int  select_independent_groups(group_list_t **tab_group, int n, int arity, int M, double *best_val,
-			       group_list_t **best_selection, int bound, double max_duration)
+static int select_independent_groups(group_list_t **tab_group, int n, int arity, int M, double *best_val,
+                                     group_list_t **best_selection, int bound, double max_duration)
 {
   int i, j;
   group_list_t **selection = NULL;
@@ -700,8 +694,8 @@ int  select_independent_groups(group_list_t **tab_group, int n, int arity, int M
 
   selection = (group_list_t **)MALLOC(sizeof(group_list_t*)*M);
   CLOCK(time0);
-  for( i = 0 ; i < MIN(bound, n) ; i++ ){
-    /* if(!(i%100)) {printf("%d/%d ", i, MIN(bound, n)); fflush(stdout);} */
+  for( i = 0 ; i < TM_MIN(bound, n) ; i++ ){
+    /* if(!(i%100)) {printf("%d/%d ", i, TM_MIN(bound, n)); fflush(stdout);} */
     selection[0] = tab_group[i];
     val = tab_group[i]->val;
     recurs_select_independent_groups(tab_group, i+1, n, arity, 1, M, val, best_val, selection, best_selection);
@@ -1138,7 +1132,7 @@ static int thread_exhaustive_search(group_list_t **tab_group, int nb_groups, int
   TIC;
 
   pthread_mutex_init(&lock, NULL);
-  nb_threads   = get_nb_threads();
+  nb_threads   = tm_get_nb_threads();
   nb_threads   = 4;
   works        = (work_t**)MALLOC(sizeof(work_t*)*nb_threads);
 
@@ -1186,17 +1180,17 @@ static int thread_exhaustive_search(group_list_t **tab_group, int nb_groups, int
     args[6]=(void*)indep_mat;
     args[7]=(void*)work_list;
     args[8]=(void*)&lock;
-    works[id]= create_work(9, args, partial_exhaustive_search);
+    works[id]= tm_create_work(9, args, partial_exhaustive_search);
     if(verbose_level >= DEBUG)
       printf("Executing %p\n", (void *)works[id]);
 
-    submit_work( works[id], id);
+    tm_submit_work( works[id], id);
   }
 
   for(id=0;id<nb_threads;id++){
-    wait_work_completion(works[id]);
+    tm_wait_work_completion(works[id]);
     FREE(works[id]->args);
-    destroy_work(works[id]);
+    tm_destroy_work(works[id]);
   }
 
   exit(-1);
@@ -1385,7 +1379,7 @@ static int  exhaustive_search(group_list_t **tab_group, int n, int arity, int so
 #endif
 
 
-int  select_independent_groups_by_largest_index(group_list_t **tab_group, int n, int arity, int solution_size, double *best_val, group_list_t **best_selection, int bound, double max_duration)
+static int select_independent_groups_by_largest_index(group_list_t **tab_group, int n, int arity, int solution_size, double *best_val, group_list_t **best_selection, int bound, double max_duration)
 {
   int i, dec, nb_groups=0;
   group_list_t **selection = NULL;
@@ -1395,7 +1389,7 @@ int  select_independent_groups_by_largest_index(group_list_t **tab_group, int n,
   selection = (group_list_t **)MALLOC(sizeof(group_list_t*)*solution_size);
   CLOCK(time0);
 
-  dec = MAX(n/10000, 2);
+  dec = TM_MAX(n/10000, 2);
   for( i = n-1 ; i >= 0 ; i -= dec*dec){
     selection[0] = tab_group[i];
     val = tab_group[i]->val;
@@ -1425,7 +1419,7 @@ int  select_independent_groups_by_largest_index(group_list_t **tab_group, int n,
   return 0;
 }
 
-void list_to_tab(group_list_t *list, group_list_t **tab, int n)
+static void list_to_tab(group_list_t *list, group_list_t **tab, int n)
 {
   int i;
   for( i = 0 ; i < n ; i++ ){
@@ -1445,7 +1439,7 @@ void list_to_tab(group_list_t *list, group_list_t **tab, int n)
   }
 }
 
-void display_tab_group(group_list_t **tab, int n, int arity)
+static inline void display_tab_group(group_list_t **tab, int n, int arity)
 {
   int i, j;
   if(verbose_level<DEBUG)
@@ -1457,7 +1451,7 @@ void display_tab_group(group_list_t **tab, int n, int arity)
   }
 }
 
-int independent_tab(tm_tree_t **tab1, tm_tree_t **tab2, int arity)
+static int independent_tab(tm_tree_t **tab1, tm_tree_t **tab2, int arity)
 {
   int ii, jj;
   for( ii = 0 ; ii < arity ; ii++ ){
@@ -1470,7 +1464,7 @@ int independent_tab(tm_tree_t **tab1, tm_tree_t **tab2, int arity)
   return 1;
 }
 
-void compute_weighted_degree(group_list_t **tab, int n, int arity)
+static void compute_weighted_degree(group_list_t **tab, int n, int arity)
 {
   int i, j;
   for( i = 0 ; i < n ; i++)
@@ -1501,7 +1495,7 @@ void compute_weighted_degree(group_list_t **tab, int n, int arity)
   cur_group: current grouping
   mat_order: size of tab and tab_node. i.e. number of nodes at the considered level
  */
-void  fast_group(tm_affinity_mat_t *aff_mat, tm_tree_t *tab_node, tm_tree_t *parent, int id, int arity, int n,
+static void  fast_group(tm_affinity_mat_t *aff_mat, tm_tree_t *tab_node, tm_tree_t *parent, int id, int arity, int n,
 		 double *best_val, tm_tree_t **cur_group, int *nb_groups, int max_groups)
 {
   double val;
@@ -1553,7 +1547,7 @@ void  fast_group(tm_affinity_mat_t *aff_mat, tm_tree_t *tab_node, tm_tree_t *par
 
   
 
-double fast_grouping(tm_affinity_mat_t *aff_mat, tm_tree_t *tab_node, tm_tree_t *new_tab_node, int arity, int solution_size, double nb_groups)
+static double fast_grouping(tm_affinity_mat_t *aff_mat, tm_tree_t *tab_node, tm_tree_t *new_tab_node, int arity, int solution_size, double nb_groups)
 {
   tm_tree_t **cur_group = NULL;
   int l, i, nb_done;
@@ -1565,12 +1559,12 @@ double fast_grouping(tm_affinity_mat_t *aff_mat, tm_tree_t *tab_node, tm_tree_t
     nb_done = 0;
     /*printf("nb_groups%d/%d, nb_groups=%ld\n", l, M, nb_groups);*/
     /* select the best greedy grouping among the 10 first one*/
-    /*fast_group(tab, tab_node, &new_tab_node[l], -1, arity, 0, &best_val, cur_group, mat_order, &nb_done, MAX(2, (int)(50-log2(nb_groups))-M/10));*/
-    fast_group(aff_mat, tab_node, &new_tab_node[l], -1, arity, 0, &best_val, cur_group, &nb_done, MAX(10, (int)(50-CmiLog2(nb_groups))-solution_size/10));
+    /*fast_group(tab, tab_node, &new_tab_node[l], -1, arity, 0, &best_val, cur_group, mat_order, &nb_done, TM_MAX(2, (int)(50-log2(nb_groups))-M/10));*/
+    fast_group(aff_mat, tab_node, &new_tab_node[l], -1, arity, 0, &best_val, cur_group, &nb_done, TM_MAX(10, (int)(50-CmiLog2(nb_groups))-solution_size/10));
     val += best_val;
     for( i = 0 ; i < new_tab_node[l].arity ; i++ )
       new_tab_node[l].child[i]->parent=&new_tab_node[l];
-    update_val(aff_mat, &new_tab_node[l]);
+    tm_update_val(aff_mat, &new_tab_node[l]);
     if(new_tab_node[l].val != best_val){
           if(verbose_level>=CRITICAL)
 	    printf("Error: best_val = %f, new_tab_node[%d].val = %f\n", best_val, l, new_tab_node[l].val);
@@ -1596,7 +1590,7 @@ static double k_partition_grouping(tm_affinity_mat_t *aff_mat, tm_tree_t *tab_no
   if(verbose_level>=DEBUG)
     printf("K-Partitionning: n=%d, solution_size=%d, arity=%d\n",n, solution_size,arity);
 
-  partition = kpartition(solution_size, &com_mat, n, NULL, 0);
+  partition = tm_kpartition(solution_size, &com_mat, n, NULL, 0);
 
   /* new_tab_node[i]->child[j] = &tab_node[k] where 0<=i< solution size, 0<=j<arity and partition[k]=i and the jth occurence of i in the partition*/
 
@@ -1612,7 +1606,7 @@ static double k_partition_grouping(tm_affinity_mat_t *aff_mat, tm_tree_t *tab_no
   
   for( i = 0 ; i < solution_size ; i++ ){
     new_tab_node[i].arity = arity;
-    update_val(aff_mat, &new_tab_node[i]);
+    tm_update_val(aff_mat, &new_tab_node[i]);
     val += new_tab_node[i].val;
   }
 
@@ -1623,7 +1617,7 @@ static double k_partition_grouping(tm_affinity_mat_t *aff_mat, tm_tree_t *tab_no
 
 }
 
-int adjacency_asc(const void* x1, const void* x2)
+static inline int adjacency_asc(const void* x1, const void* x2)
 {
   adjacency_t *e1 = NULL, *e2 = NULL;
 
@@ -1633,7 +1627,7 @@ int adjacency_asc(const void* x1, const void* x2)
   return (e1->val < e2->val) ? - 1 : 1;
 }
 
-int adjacency_dsc(const void* x1, const void* x2)
+static int adjacency_dsc(const void* x1, const void* x2)
 {
   adjacency_t *e1 = NULL, *e2 = NULL;
 
@@ -1644,7 +1638,7 @@ int adjacency_dsc(const void* x1, const void* x2)
   return (e1->val > e2->val) ? -1 : 1;
 }
 
-void super_fast_grouping(tm_affinity_mat_t *aff_mat, tm_tree_t *tab_node, tm_tree_t *new_tab_node, int arity, int solution_size)
+static inline void super_fast_grouping(tm_affinity_mat_t *aff_mat, tm_tree_t *tab_node, tm_tree_t *new_tab_node, int arity, int solution_size)
 {
   double val = 0, duration;
   adjacency_t *graph;
@@ -1683,11 +1677,11 @@ TIC;
   l = 0;
   nb_groups = 0;
   for( i = 0 ; (i < e) && (l < solution_size) ; i++ )
-    if(try_add_edge(tab_node, &new_tab_node[l], arity, graph[i].i, graph[i].j, &nb_groups))
+    if(tm_try_add_edge(tab_node, &new_tab_node[l], arity, graph[i].i, graph[i].j, &nb_groups))
       l++;
 
   for( l = 0 ; l < solution_size ; l++ ){
-    update_val(aff_mat, &new_tab_node[l]);
+    tm_update_val(aff_mat, &new_tab_node[l]);
     val += new_tab_node[l].val;
   }
 
@@ -1706,7 +1700,7 @@ TIC;
 }
 
 
-tm_affinity_mat_t *build_cost_matrix(tm_affinity_mat_t *aff_mat, double* obj_weight, double comm_speed)
+static tm_affinity_mat_t *build_cost_matrix(tm_affinity_mat_t *aff_mat, double* obj_weight, double comm_speed)
 {
   double **mat = NULL, *sum_row;
   double **old_mat;
@@ -1747,7 +1741,7 @@ tm_affinity_mat_t *build_cost_matrix(tm_affinity_mat_t *aff_mat, double* obj_wei
       }
       if(mat[i][j]) nnz++;
     }
-  return new_affinity_mat(mat, sum_row, mat_order,nnz);
+  return tm_new_affinity_mat(mat, sum_row, mat_order,nnz);
 
 }
 
@@ -1759,7 +1753,7 @@ tm_affinity_mat_t *build_cost_matrix(tm_affinity_mat_t *aff_mat, double* obj_wei
   arity: number of children of parent (i.e.) size of the group to compute
   solution_size: size of new_tab_node (i.e) the number of parents
 */
-void group_nodes(tm_affinity_mat_t *aff_mat, tm_tree_t *tab_node, tm_tree_t *new_tab_node,
+static void group_nodes(tm_affinity_mat_t *aff_mat, tm_tree_t *tab_node, tm_tree_t *new_tab_node,
 		 int arity, int solution_size, double* obj_weigth, double comm_speed){
 
  /*
@@ -1793,14 +1787,14 @@ void group_nodes(tm_affinity_mat_t *aff_mat, tm_tree_t *tab_node, tm_tree_t *new
   /*  if(nb_groups>30000||depth>5){*/
   if( nbg > 30000 ){
 
-    double duration;
+    double duration_loc;
 
     TIC;
     if( arity <= 2 ){
       /*super_fast_grouping(tab, tab_node, new_tab_node, arity, mat_order, solution_size, k);*/
       if(verbose_level >= INFO )
 	printf("Bucket Grouping...\n");
-      val = bucket_grouping(cost_mat, tab_node, new_tab_node, arity, solution_size);
+      val = tm_bucket_grouping(cost_mat, tab_node, new_tab_node, arity, solution_size);
     }else if( arity <= 5){
       if(verbose_level >= INFO)
 	printf("Fast Grouping...\n");
@@ -1811,9 +1805,9 @@ void group_nodes(tm_affinity_mat_t *aff_mat, tm_tree_t *tab_node, tm_tree_t *new
       val = k_partition_grouping(cost_mat, tab_node, new_tab_node, arity, solution_size);
     }
 
-    duration = TOC;
+    duration_loc = TOC;
     if(verbose_level >= INFO)
-      printf("Fast grouping duration=%f\n", duration);
+      printf("Fast grouping duration=%f\n", duration_loc);
 
     if(verbose_level >= INFO)
       display_grouping(new_tab_node, solution_size, arity, val);
@@ -1932,7 +1926,7 @@ void group_nodes(tm_affinity_mat_t *aff_mat, tm_tree_t *tab_node, tm_tree_t *new
       new_tab_node[l].arity = arity;
 
       /* printf("arity=%d\n", new_tab_node[l].arity); */
-      update_val(cost_mat, &new_tab_node[l]);
+      tm_update_val(cost_mat, &new_tab_node[l]);
     }
 
     delete_group_list((&list)->next);
@@ -1952,7 +1946,7 @@ void group_nodes(tm_affinity_mat_t *aff_mat, tm_tree_t *tab_node, tm_tree_t *new
     printf("Grouping done in %.4fs!\n", duration);
 }
 
-void complete_aff_mat(tm_affinity_mat_t **aff_mat , int mat_order, int K)
+static void complete_aff_mat(tm_affinity_mat_t **aff_mat , int mat_order, int K)
 {
   double **old_mat = NULL, **new_mat = NULL; double *sum_row;
   int M, i;
@@ -1971,10 +1965,10 @@ void complete_aff_mat(tm_affinity_mat_t **aff_mat , int mat_order, int K)
     sum_row[i] = (*aff_mat)->sum_row[i];
   }
 
-  *aff_mat = new_affinity_mat(new_mat, sum_row, M, (*aff_mat)->nnz);
+  *aff_mat = tm_new_affinity_mat(new_mat, sum_row, M, (*aff_mat)->nnz);
 }
 
-void complete_obj_weight(double **tab, int mat_order, int K)
+void tm_complete_obj_weight(double **tab, int mat_order, int K)
 {
   double *old_tab = NULL, *new_tab = NULL, avg;
   int M, i;
@@ -2000,13 +1994,13 @@ void complete_obj_weight(double **tab, int mat_order, int K)
       new_tab[i] = avg;
 }
 
-void create_dumb_tree(tm_tree_t *node, int depth, tm_topology_t *topology)
+static void create_dumb_tree(tm_tree_t *node, int depth, tm_topology_t *topology)
 {
   tm_tree_t **list_child = NULL;
   int arity, i;
 
   if( depth == topology->nb_levels-1) {
-    set_node(node, NULL, 0, NULL, -1, 0, NULL, depth);
+    tm_set_node(node, NULL, 0, NULL, -1, 0, NULL, depth);
     return;
   }
 
@@ -2022,9 +2016,9 @@ void create_dumb_tree(tm_tree_t *node, int depth, tm_topology_t *topology)
 
   /* list_child => node->child ; list_child[0] => node->tab_child */
   /* printf("list_child[0] =  %p\n",list_child[0]); */
-  set_node(node, list_child, arity, NULL, -1, 0, NULL, depth);
+  tm_set_node(node, list_child, arity, NULL, -1, 0, NULL, depth);
 }
-void complete_tab_node(tm_tree_t **tab, int mat_order, int K, int depth, tm_topology_t *topology)
+static void complete_tab_node(tm_tree_t **tab, int mat_order, int K, int depth, tm_topology_t *topology)
 {
   tm_tree_t *old_tab = NULL, *new_tab = NULL;
   int M, i;
@@ -2050,7 +2044,7 @@ void complete_tab_node(tm_tree_t **tab, int mat_order, int K, int depth, tm_topo
   FREE(old_tab);
 }
 
-void set_deb_tab_child(tm_tree_t *tree, tm_tree_t *child, int depth)
+static void set_deb_tab_child(tm_tree_t *tree, tm_tree_t *child, int depth)
 {
   /* printf("depth=%d\t%p\t%p\n", depth, child, tree);*/
   if( depth > 0 )
@@ -2071,7 +2065,7 @@ depth: current depth of the algorithm
 toplogy: description of the hardware topology.
 constraints:  set of constraints: core ids where to bind the processes
 */
-tm_tree_t *build_level_topology(tm_tree_t *tab_node, tm_affinity_mat_t *aff_mat, int arity, int depth, tm_topology_t *topology,
+static tm_tree_t *build_level_topology(tm_tree_t *tab_node, tm_affinity_mat_t *aff_mat, int arity, int depth, tm_topology_t *topology,
 			     double *obj_weight, double *comm_speed)
 {
 
@@ -2107,7 +2101,7 @@ tm_tree_t *build_level_topology(tm_tree_t *tab_node, tm_affinity_mat_t *aff_mat,
     /* add K rows and columns to comm_matrix*/
     complete_aff_mat(&aff_mat, mat_order, K);
     /* add K element to the object weight*/
-    complete_obj_weight(&obj_weight, mat_order, K);
+    tm_complete_obj_weight(&obj_weight, mat_order, K);
     /*display_tab(tab, mat_order+K);*/
     /* add a dumb tree to the K new "virtual nodes"*/
     complete_tab_node(&tab_node, mat_order, K, depth, topology);
@@ -2129,7 +2123,7 @@ tm_tree_t *build_level_topology(tm_tree_t *tab_node, tm_affinity_mat_t *aff_mat,
   for( i = 0 ; i < M ; i++ ){
     tm_tree_t **list_child = NULL;
     list_child = (tm_tree_t**)CALLOC(arity, sizeof(tm_tree_t*));
-    set_node(&new_tab_node[i], list_child, arity, NULL, i, 0, tab_node, depth); 
+    tm_set_node(&new_tab_node[i], list_child, arity, NULL, i, 0, tab_node, depth); 
  }
   duration = TOC;
   if(verbose_level >= INFO)
@@ -2192,7 +2186,7 @@ tm_tree_t *build_level_topology(tm_tree_t *tab_node, tm_affinity_mat_t *aff_mat,
 
 
 
-tm_tree_t *bottom_up_build_tree_from_topology(tm_topology_t *topology, tm_affinity_mat_t *aff_mat,
+static tm_tree_t *bottom_up_build_tree_from_topology(tm_topology_t *topology, tm_affinity_mat_t *aff_mat,
 					      double *obj_weight, double *comm_speed){
   int depth, i;
   tm_tree_t *res = NULL, *tab_node = NULL;
@@ -2201,7 +2195,7 @@ tm_tree_t *bottom_up_build_tree_from_topology(tm_topology_t *topology, tm_affini
   tab_node = (tm_tree_t*)MALLOC(sizeof(tm_tree_t)*mat_order);
   depth = topology->nb_levels;
   for( i = 0 ; i < mat_order ; i++ )
-    set_node(&tab_node[i], NULL, 0, NULL, i, 0, NULL, depth);
+    tm_set_node(&tab_node[i], NULL, 0, NULL, i, 0, NULL, depth);
 
 
   if(verbose_level >= INFO)
@@ -2229,7 +2223,7 @@ tm_tree_t *bottom_up_build_tree_from_topology(tm_topology_t *topology, tm_affini
 
 */
 
-int check_constraints(tm_topology_t  *topology, int **constraints)
+static int check_constraints(tm_topology_t  *topology, int **constraints)
 {
 
   int sorted = 1;
@@ -2254,7 +2248,7 @@ int check_constraints(tm_topology_t  *topology, int **constraints)
     }
 
     if(!sorted){
-      qsort(*constraints, nb_constraints , sizeof(int), int_cmp_inc);
+      qsort(*constraints, nb_constraints , sizeof(int), tm_int_cmp_inc);
     }
 
   }else{
@@ -2280,14 +2274,14 @@ tm_tree_t * tm_build_tree_from_topology(tm_topology_t *topology, tm_affinity_mat
   /* Here constraints expended to take into account the oversuscribing factor */
   nb_constraints = check_constraints (topology, &constraints);
   nb_processes   = aff_mat->order;
-  npu            = nb_processing_units(topology);
+  npu            = tm_nb_processing_units(topology);
   nb_slots       = npu * oversub_fact;
 
   if(verbose_level >= INFO){
     printf("Com matrix size      : %d\n", nb_processes);
     printf("nb_constraints       : %d\n", nb_constraints);
     if(constraints)
-      print_1D_tab(constraints, nb_constraints);
+      tm_print_1D_tab(constraints, nb_constraints);
     printf("nb_processing units  : %d\n", npu);
     printf("Oversubscrbing factor: %d\n", oversub_fact);
     printf("Nb of slots          : %d\n", nb_slots);
@@ -2315,7 +2309,7 @@ tm_tree_t * tm_build_tree_from_topology(tm_topology_t *topology, tm_affinity_mat
     if(verbose_level >= INFO){
       printf("Partitionning with constraints\n");
     }
-    result = kpartition_build_tree_from_topology(topology, aff_mat->mat, nb_processes, constraints, nb_constraints,
+    result = tm_kpartition_build_tree_from_topology(topology, aff_mat->mat, nb_processes, constraints, nb_constraints,
 						 obj_weight, com_speed);
     result->nb_processes = aff_mat->order;
     FREE(constraints);
diff --git a/3rd-party/treematch/tm_tree.h b/3rd-party/treematch/tm_tree.h
index 6168f501618..4a472abdd2e 100644
--- a/3rd-party/treematch/tm_tree.h
+++ b/3rd-party/treematch/tm_tree.h
@@ -3,9 +3,9 @@
 #include <stdlib.h>
 #include "treematch.h"
 
-void update_val(tm_affinity_mat_t *aff_mat,tm_tree_t *parent);
-void display_tab(double **tab,int N);
-void set_node(tm_tree_t *node,tm_tree_t ** child, int arity,tm_tree_t *parent,
+OMPI_HIDDEN void tm_update_val(tm_affinity_mat_t *aff_mat,tm_tree_t *parent);
+OMPI_HIDDEN void tm_display_tab(double **tab,int N);
+OMPI_HIDDEN void tm_set_node(tm_tree_t *node,tm_tree_t ** child, int arity,tm_tree_t *parent,
 	      int id,double val,tm_tree_t *tab_child,int depth);
 
 
diff --git a/3rd-party/treematch/tm_verbose.h b/3rd-party/treematch/tm_verbose.h
index e16cbbc6c00..0a533d32e03 100644
--- a/3rd-party/treematch/tm_verbose.h
+++ b/3rd-party/treematch/tm_verbose.h
@@ -1,4 +1,5 @@
 #include <stdio.h>
+#include "ompi_config.h"
 
 #define NONE     0
 /* output in stderr*/
@@ -12,11 +13,11 @@
 
 
 /* return 0 on errror and  1 on success */
-int          tm_open_verbose_file(char *filename);
-int          tm_close_verbose_file(void);
-void         tm_set_verbose_level(unsigned int level);
-unsigned int tm_get_verbose_level(void);
-FILE *       tm_get_verbose_output(void);
+OMPI_HIDDEN int          tm_open_verbose_file(char *filename);
+OMPI_HIDDEN int          tm_close_verbose_file(void);
+OMPI_HIDDEN void         tm_set_verbose_level(unsigned int level);
+OMPI_HIDDEN unsigned int tm_get_verbose_level(void);
+OMPI_HIDDEN FILE *       tm_get_verbose_output(void);
 
 #define tm_verbose_printf(level, ...) level <= tm_get_verbose_level()?fprintf(tm_get_verbose_output(),__VA_ARGS__):0
 
diff --git a/3rd-party/treematch/treematch.h b/3rd-party/treematch/treematch.h
index e15e9ab5a4a..9d4c943b1ad 100644
--- a/3rd-party/treematch/treematch.h
+++ b/3rd-party/treematch/treematch.h
@@ -4,6 +4,7 @@
 /* size_t definition */
 #include <stddef.h>
 #include "tm_verbose.h"
+#include "ompi_config.h"
 
 /********* TreeMatch Public Enum **********/
 
@@ -105,10 +106,10 @@ typedef struct {
 
 /************ TreeMatch Public API ************/
 /* construct topology from local one using hwloc */
-tm_topology_t* tm_get_local_topology_with_hwloc(void);
+OMPI_HIDDEN tm_topology_t* tm_get_local_topology_with_hwloc(void);
 
 /* Aletrnatively, load XML or TGT topology */
-tm_topology_t *tm_load_topology(char *arch_filename, tm_file_type_t arch_file_type);
+OMPI_HIDDEN tm_topology_t *tm_load_topology(char *arch_filename, tm_file_type_t arch_file_type);
 /*
    Alternatively, build a synthetic balanced topology.
 
@@ -136,74 +137,74 @@ tm_topology_t *tm_load_topology(char *arch_filename, tm_file_type_t arch_file_ty
    topology = tm_build_synthetic_topology(arity,cost,5,cn,2);
 
  */
-tm_topology_t  *tm_build_synthetic_topology(int *arity, double *cost, int nb_levels, int *core_numbering, int nb_core_per_nodes);
+OMPI_HIDDEN tm_topology_t  *tm_build_synthetic_topology(int *arity, double *cost, int nb_levels, int *core_numbering, int nb_core_per_nodes);
 /* load affinity matrix */
-tm_affinity_mat_t *tm_load_aff_mat(char *com_filename);
+OMPI_HIDDEN tm_affinity_mat_t *tm_load_aff_mat(char *com_filename);
 /*
    Alternativelly, build the affinity matrix from a array of array of matrix of size order by order
    For performance reason mat is not copied.
 */
-tm_affinity_mat_t * tm_build_affinity_mat(double **mat, int order);
+OMPI_HIDDEN tm_affinity_mat_t * tm_build_affinity_mat(double **mat, int order);
 /* Add constraints to toplogy
    Return 1 on success and 0  if the constari,ts id are not compatible withe nodes id */
-int tm_topology_add_binding_constraints(char *bind_filename, tm_topology_t *topology);
+OMPI_HIDDEN int tm_topology_add_binding_constraints(char *bind_filename, tm_topology_t *topology);
 /* Alternatively, set the constraints from an array.
    Return 1 on success and 0  if the constari,ts id are not compatible withe nodes id
 
    The array constraints is copied inside tm_topology_set_binding_constraints
 
 */
-int tm_topology_set_binding_constraints(int *constraints, int nb_constraints, tm_topology_t *topology);
+OMPI_HIDDEN int tm_topology_set_binding_constraints(int *constraints, int nb_constraints, tm_topology_t *topology);
 /* display arity of the topology */
-void  tm_display_arity(tm_topology_t *topology);
+OMPI_HIDDEN void  tm_display_arity(tm_topology_t *topology);
 /* display the full topology */
-void  tm_display_topology(tm_topology_t *topology);
+OMPI_HIDDEN void  tm_display_topology(tm_topology_t *topology);
 /* Optimize the topology by decomposing arities */
-void tm_optimize_topology(tm_topology_t **topology);
+OMPI_HIDDEN void tm_optimize_topology(tm_topology_t **topology);
 /* Manage oversubscribing */
-void tm_enable_oversubscribing(tm_topology_t *topology, unsigned int oversub_fact);
+OMPI_HIDDEN void tm_enable_oversubscribing(tm_topology_t *topology, unsigned int oversub_fact);
 /* core of the treematch: compute the solution tree */
-tm_tree_t *tm_build_tree_from_topology(tm_topology_t *topology, tm_affinity_mat_t *aff_mat, double *obj_weight, double *com_speed);
+OMPI_HIDDEN tm_tree_t *tm_build_tree_from_topology(tm_topology_t *topology, tm_affinity_mat_t *aff_mat, double *obj_weight, double *com_speed);
 /* compute the mapping according to the tree and the core numbering*/
-tm_solution_t *tm_compute_mapping(tm_topology_t *topology, tm_tree_t *comm_tree);
+OMPI_HIDDEN tm_solution_t *tm_compute_mapping(tm_topology_t *topology, tm_tree_t *comm_tree);
 /* display the solution*/
-double tm_display_solution(tm_topology_t *topology, tm_affinity_mat_t *aff_mat, tm_solution_t *sol, tm_metric_t metric);
+OMPI_HIDDEN double tm_display_solution(tm_topology_t *topology, tm_affinity_mat_t *aff_mat, tm_solution_t *sol, tm_metric_t metric);
 /* display RR, packed, MPIPP*/
-void tm_display_other_heuristics(tm_topology_t *topology, tm_affinity_mat_t *aff_mat, tm_metric_t metric);
+OMPI_HIDDEN void tm_display_other_heuristics(tm_topology_t *topology, tm_affinity_mat_t *aff_mat, tm_metric_t metric);
 /* free TM strutures*/
-void tm_free_topology(tm_topology_t *topology);
-void tm_free_tree(tm_tree_t *comm_tree);
-void tm_free_solution(tm_solution_t *sol);
-void tm_free_affinity_mat(tm_affinity_mat_t *aff_mat);
+OMPI_HIDDEN void tm_free_topology(tm_topology_t *topology);
+OMPI_HIDDEN void tm_free_tree(tm_tree_t *comm_tree);
+OMPI_HIDDEN void tm_free_solution(tm_solution_t *sol);
+OMPI_HIDDEN void tm_free_affinity_mat(tm_affinity_mat_t *aff_mat);
 /* manage verbosity of TM*/
 void tm_set_verbose_level(unsigned int level);
 unsigned int  tm_get_verbose_level(void);
 /* finalize treematch :check memory if necessary, and free internal variables (thread pool)*/
-void tm_finalize(void);
+OMPI_HIDDEN void tm_finalize(void);
 
 /*
 Ask for exhaustive search: may be very long
    new_val == 0 : no exhuative search
    new_val != 0 : exhuative search
 */
-void tm_set_exhaustive_search_flag(int new_val);
-int tm_get_exhaustive_search_flag(void);
+OMPI_HIDDEN void tm_set_exhaustive_search_flag(int new_val);
+OMPI_HIDDEN int tm_get_exhaustive_search_flag(void);
 
 /*
 Ask for greedy k-partitionning even if scotch is available
    new_val == 0 : no greedy k-partitionning
    new_val != 0 : greedy k-partitionning
 */
-void tm_set_greedy_flag(int new_val);
-int tm_get_greedy_flag(void);
+OMPI_HIDDEN void tm_set_greedy_flag(int new_val);
+OMPI_HIDDEN int tm_get_greedy_flag(void);
 
 
 /* Setting the maximum number of threads you want to use in parallel parts of TreeMatch */
-void tm_set_max_nb_threads(unsigned int val);
+OMPI_HIDDEN void tm_set_max_nb_threads(unsigned int val);
 
 /* managing the usage of physical vs. logical core numbering when using hwloc/xml files */
-void            tm_set_numbering(tm_numbering_t new_val); /* TM_NUMBERING_LOGICAL or TM_NUMBERING_PHYSICAL */
-tm_numbering_t  tm_get_numbering(void); /* TM_NUMBERING_LOGICAL or TM_NUMBERING_PHYSICAL */
+OMPI_HIDDEN void tm_set_numbering(tm_numbering_t new_val); /* TM_NUMBERING_LOGICAL or TM_NUMBERING_PHYSICAL */
+OMPI_HIDDEN tm_numbering_t  tm_get_numbering(void); /* TM_NUMBERING_LOGICAL or TM_NUMBERING_PHYSICAL */
 
 #include "tm_malloc.h"
 
diff --git a/Doxyfile b/Doxyfile
index 1f19f0ad769..e8c88d33574 100644
--- a/Doxyfile
+++ b/Doxyfile
@@ -401,7 +401,7 @@ RECURSIVE              = YES
 # excluded from the INPUT source files. This way you can easily exclude a
 # subdirectory from a directory tree whose root is specified with the INPUT tag.
 
-EXCLUDE                = opal/event ompi/mca/io/romio
+EXCLUDE                = opal/event
 
 # The EXCLUDE_SYMLINKS tag can be used select whether or not files or
 # directories that are symbolic links (a Unix filesystem feature) are
diff --git a/HACKING.md b/HACKING.md
index fe045ecf0e4..1c55040a98e 100644
--- a/HACKING.md
+++ b/HACKING.md
@@ -6,253 +6,11 @@ This file is here for those who are building/exploring OMPI in its
 source code form, most likely through a developer's tree (i.e., a
 Git clone).
 
+## This file has moved
 
-## Developer Builds: Compiler Pickyness by Default
+Please see the content of this file in its new location:
+https://docs.open-mpi.org/en/main/developers/
 
-If you are building Open MPI from a Git clone (i.e., there is a `.git`
-directory in your build tree), the default build includes extra
-compiler pickyness, which will result in more compiler warnings than
-in non-developer builds.  Getting these extra compiler warnings is
-helpful to Open MPI developers in making the code base as clean as
-possible.
-
-Developers can disable this picky-by-default behavior by using the
-`--disable-picky` configure option.  Also note that extra-picky compiles
-do *not* happen automatically when you do a VPATH build (e.g., if
-`.git` is in your source tree, but not in your build tree).
-
-Prior versions of Open MPI would automatically activate a lot of
-(performance-reducing) debugging code by default if `.git` was found
-in your build tree.  This is no longer true.  You can manually enable
-these (performance-reducing) debugging features in the Open MPI code
-base with these configure options:
-
-* `--enable-debug`
-* `--enable-mem-debug`
-* `--enable-mem-profile`
-
-***NOTE:*** These options are really only relevant to those who are
-developing Open MPI itself.  They are not generally helpful for
-debugging general MPI applications.
-
-
-## Use of GNU Autoconf, Automake, and Libtool (and m4)
-
-You need to read/care about this section *ONLY* if you are building
-from a developer's tree (i.e., a Git clone of the Open MPI source
-tree).  If you have an Open MPI distribution tarball, the contents of
-this section are optional -- you can (and probably should) skip
-reading this section.
-
-If you are building Open MPI from a developer's tree, you must first
-install fairly recent versions of the GNU tools Autoconf, Automake,
-and Libtool (and possibly GNU m4, because recent versions of Autoconf
-have specific GNU m4 version requirements).  The specific versions
-required depend on if you are using the Git master branch or a release
-branch (and which release branch you are using).  [The specific
-versions can be found
-here](https://www.open-mpi.org/source/building.php).
-
-You can check what versions of the autotools you have installed with
-the following:
-
-```
-shell$ m4 --version
-shell$ autoconf --version
-shell$ automake --version
-shell$ libtoolize --version
-```
-
-[Required version levels for all the OMPI releases can be found
-here](https://www.open-mpi.org/source/building.php).
-
-To strengthen the above point: the core Open MPI developers typically
-use very, very recent versions of the GNU tools.  There are known bugs
-in older versions of the GNU tools that Open MPI no longer compensates
-for (it seemed senseless to indefinitely support patches for ancient
-versions of Autoconf, for example).  You *WILL* have problems if you
-do not use recent versions of the GNU tools.
-
-***NOTE:*** On MacOS/X, the default `libtool` program is different
-than the GNU libtool.  You must download and install the GNU version
-(e.g., via MacPorts, Homebrew, or some other mechanism).
-
-If you need newer versions, you are *strongly* encouraged to heed the
-following advice:
-
-1. Unless your OS distribution has easy-to-use binary installations,
-   the sources can be can be downloaded from:
-   * https://ftp.gnu.org/gnu/autoconf/
-   * https://ftp.gnu.org/gnu/automake/
-   * https://ftp.gnu.org/gnu/libtool/
-   * And if you need it: https://ftp.gnu.org/gnu/m4/
-
-   ***NOTE:*** It is certainly easiest to download/build/install all
-   four of these tools together.  But note that Open MPI has no
-   specific m4 requirements; it is only listed here because Autoconf
-   requires minimum versions of GNU m4.  Hence, you may or may not
-   *need* to actually install a new version of GNU m4.  That being
-   said, if you are confused or don't know, just install the latest
-   GNU m4 with the rest of the GNU Autotools and everything will work
-   out fine.
-
-1. Build and install the tools in the following order:
-   1. m4
-   1. Autoconf
-   1. Automake
-   1. Libtool
-
-1. You MUST install the last three tools (Autoconf, Automake, Libtool)
-   into the same prefix directory.  These three tools are somewhat
-   inter-related, and if they're going to be used together, they MUST
-   share a common installation prefix.
-
-   You can install m4 anywhere as long as it can be found in the path;
-   it may be convenient to install it in the same prefix as the other
-   three.  Or you can use any recent-enough m4 that is in your path.
-
-   1. It is *strongly* encouraged that you do not install your new
-      versions over the OS-installed versions.  This could cause
-      other things on your system to break.  Instead, install into
-      `$HOME/local`, or `/usr/local`, or wherever else you tend to
-      install "local" kinds of software.
-   1. In doing so, be sure to prefix your $path with the directory
-      where they are installed.  For example, if you install into
-      `$HOME/local`, you may want to edit your shell startup file
-      (`.bashrc`, `.cshrc`, `.tcshrc`, etc.) to have something like:
-
-      ```sh
-      # For bash/sh:
-      export PATH=$HOME/local/bin:$PATH
-      # For csh/tcsh:
-      set path = ($HOME/local/bin $path)
-      ```
-
-   1. Ensure to set your `$PATH` *BEFORE* you configure/build/install
-      the four packages.
-
-1. All four packages require two simple commands to build and
-   install (where PREFIX is the prefix discussed in 3, above).
-
-   ```
-   shell$ cd <m4 directory>
-   shell$ ./configure --prefix=PREFIX
-   shell$ make; make install
-   ```
-
-   ***NOTE:*** If you are using the `csh` or `tcsh` shells, be sure to
-   run the `rehash` command after you install each package.
-
-   ```
-   shell$ cd <autoconf directory>
-   shell$ ./configure --prefix=PREFIX
-   shell$ make; make install
-   ```
-
-   ***NOTE:*** If you are using the `csh` or `tcsh` shells, be sure to
-   run the `rehash` command after you install each package.
-
-   ```
-   shell$ cd <automake directory>
-   shell$ ./configure --prefix=PREFIX
-   shell$ make; make install
-   ```
-
-   ***NOTE:*** If you are using the `csh` or `tcsh` shells, be sure to
-   run the `rehash` command after you install each package.
-
-   ```
-   shell$ cd <libtool directory>
-   shell$ ./configure --prefix=PREFIX
-   shell$ make; make install
-   ```
-
-   ***NOTE:*** If you are using the `csh` or `tcsh` shells, be sure to
-   run the `rehash` command after you install each package.
-
-   m4, Autoconf and Automake build and install very quickly; Libtool
-   will take a minute or two.
-
-1. You can now run OMPI's top-level `autogen.pl` script.  This script
-   will invoke the GNU Autoconf, Automake, and Libtool commands in the
-   proper order and setup to run OMPI's top-level `configure` script.
-
-   Running `autogen.pl` may take a few minutes, depending on your
-   system.  It's not very exciting to watch.  :smile:
-
-   If you have a multi-processor system, enabling the multi-threaded
-   behavior in Automake 1.11 (or newer) can result in `autogen.pl`
-   running faster.  Do this by setting the `AUTOMAKE_JOBS` environment
-   variable to the number of processors (threads) that you want it to
-   use before invoking `autogen`.pl.  For example (you can again put
-   this in your shell startup files):
-
-   ```sh
-    # For bash/sh:
-    export AUTOMAKE_JOBS=4
-    # For csh/tcsh:
-    set AUTOMAKE_JOBS 4
-    ```
-
-   1. You generally need to run autogen.pl whenever the top-level file
-      `configure.ac` changes, or any files in the `config/` or
-      `<project>/config/` directories change (these directories are
-      where a lot of "include" files for Open MPI's `configure` script
-      live).
-
-   1. You do *NOT* need to re-run `autogen.pl` if you modify a
-      `Makefile.am`.
-
-## Use of Flex
-
-Flex is used during the compilation of a developer's checkout (it is
-not used to build official distribution tarballs).  Other flavors of
-lex are *not* supported: given the choice of making parsing code
-portable between all flavors of lex and doing more interesting work on
-Open MPI, we greatly prefer the latter.
-
-Note that no testing has been performed to see what the minimum
-version of Flex is required by Open MPI.  We suggest that you use
-v2.5.35 at the earliest.
-
-***NOTE:*** Windows developer builds of Open MPI *require* Flex version
-2.5.35.  Specifically, we know that v2.5.35 works and 2.5.4a does not.
-We have not tested to figure out exactly what the minimum required
-flex version is on Windows; we suggest that you use 2.5.35 at the
-earliest.  It is for this reason that the
-`contrib/dist/make_dist_tarball` script checks for a Windows-friendly
-version of Flex before continuing.
-
-For now, Open MPI will allow developer builds with Flex 2.5.4.  This
-is primarily motivated by the fact that RedHat/Centos 5 ships with
-Flex 2.5.4.  It is likely that someday Open MPI developer builds will
-require Flex version >=2.5.35.
-
-Note that the `flex`-generated code generates some compiler warnings
-on some platforms, but the warnings do not seem to be consistent or
-uniform on all platforms, compilers, and flex versions.  As such, we
-have done little to try to remove those warnings.
-
-If you do not have Flex installed, see [the Flex Github
-repository](https://github.com/westes/flex).
-
-## Use of Pandoc
-
-Similar to prior sections, you need to read/care about this section
-*ONLY* if you are building from a developer's tree (i.e., a Git clone
-of the Open MPI source tree).  If you have an Open MPI distribution
-tarball, the contents of this section are optional -- you can (and
-probably should) skip reading this section.
-
-The Pandoc tool is used to generate Open MPI's man pages.
-Specifically: Open MPI's man pages are written in Markdown; Pandoc is
-the tool that converts that Markdown to nroff (i.e., the format of man
-pages).
-
-You must have Pandoc >=v1.12 when building Open MPI from a developer's
-tree.  If configure cannot find Pandoc >=v1.12, it will abort.
-
-If you need to install Pandoc, check your operating system-provided
-packages (to include MacOS Homebrew and MacPorts).  [The Pandoc
-project web site](https://pandoc.org/) itself also offers binaries for
-their releases.
+Additionally, see
+https://docs.open-mpi.org/en/main/developers/prerequisites.html#sphinx-and-therefore-python
+if you want to edit and build the documentation locally.
diff --git a/INSTALL b/INSTALL
deleted file mode 100644
index 33a6bb554b5..00000000000
--- a/INSTALL
+++ /dev/null
@@ -1,101 +0,0 @@
-Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-                        University Research and Technology
-                        Corporation.  All rights reserved.
-Copyright (c) 2004-2005 The University of Tennessee and The University
-                        of Tennessee Research Foundation.  All rights
-                        reserved.
-Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-                        University of Stuttgart.  All rights reserved.
-Copyright (c) 2004-2005 The Regents of the University of California.
-                        All rights reserved.
-Copyright (c) 2008-2020 Cisco Systems, Inc.  All rights reserved.
-$COPYRIGHT$
-
-Additional copyrights may follow
-
-$HEADER$
-
-
-For More Information
-====================
-
-This file is a *very* short overview of building and installing Open
-MPI, and building MPI applications.  Much more information is
-available on the Open MPI web site (e.g., see the FAQ section):
-
-    https://www.open-mpi.org/
-
-
-Developer Builds
-================
-
-If you have checked out a DEVELOPER'S COPY of Open MPI (i.e., you
-cloned from Git), you should read the HACKING file before attempting
-to build Open MPI.  You must then run:
-
-shell$ ./autogen.pl
-
-You will need very recent versions of GNU Autoconf, Automake, and
-Libtool.  If autogen.pl fails, read the HACKING file.  If anything
-else fails, read the HACKING file.  Finally, we suggest reading the
-HACKING file.
-
-*** NOTE: Developer's copies of Open MPI typically include a large
-performance penalty at run-time because of extra debugging overhead.
-
-
-User Builds
-===========
-
-Building Open MPI is typically a combination of running "configure"
-and "make".  Execute the following commands to install the Open MPI
-system from within the directory at the top of the tree:
-
-shell$ ./configure --prefix=/where/to/install
-[...lots of output...]
-shell$ make all install
-
-If you need special access to install, then you can execute "make
-all" as a user with write permissions in the build tree, and a
-separate "make install" as a user with write permissions to the
-install tree.
-
-Compiling support for various networks or other specific hardware may
-require additional command ling flags when running configure.  See the
-README file for more details.  Note that VPATH builds are fully
-supported.  For example:
-
-shell$ tar xf openmpi-X.Y.Z.tar.gz
-shell$ cd openmpi-X.Y.Z
-shell$ mkdir build
-shell$ cd build
-shell$ ../configure ...your options...
-[...lots of output...]
-shell$ make all install
-
-Parallel builds are also supported (although some versions of "make",
-such as GNU make, will only use the first target listed on the command
-line when executable parallel builds).  For example (assume GNU make):
-
-shell$ make -j 4 all
-[...lots of output...]
-shell$ make install
-
-Parallel make is generally only helpful in the build phase (i.e.,
-"make all"); the installation process (i.e., "make install") is mostly
-serial and does not benefit much from parallelization.
-
-
-Compiling MPI Applications
-==========================
-
-MPI applications should be compiled using the Open MPI "wrapper"
-compilers:
-
-C programs:       mpicc your-code.c
-C++ programs:     mpiCC your-code.cc    or
-                  mpic++ your-code.cc   (for case-insensitive filesystems)
-Fortran programs: mpifort your-code.f90
-
-These compilers simply add various command line flags (such as -lmpi)
-and invoke a back-end compiler; they are not compilers in themselves.
diff --git a/LICENSE b/LICENSE
index 4ed90b0f879..679847e168b 100644
--- a/LICENSE
+++ b/LICENSE
@@ -5,37 +5,37 @@ team who have contributed code to this release.  The copyrights for
 code used under license from other parties are included in the
 corresponding files.
 
-Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+Copyright (c) 2004-2012 The Trustees of Indiana University and Indiana
                         University Research and Technology
                         Corporation.  All rights reserved.
-Copyright (c) 2004-2017 The University of Tennessee and The University
+Copyright (c) 2004-2021 The University of Tennessee and The University
                         of Tennessee Research Foundation.  All rights
                         reserved.
-Copyright (c) 2004-2010 High Performance Computing Center Stuttgart,
+Copyright (c) 2004-2018 High Performance Computing Center Stuttgart,
                         University of Stuttgart.  All rights reserved.
 Copyright (c) 2004-2008 The Regents of the University of California.
                         All rights reserved.
 Copyright (c) 2006-2018 Los Alamos National Security, LLC.  All rights
                         reserved.
-Copyright (c) 2006-2020 Cisco Systems, Inc.  All rights reserved.
+Copyright (c) 2006-2021 Cisco Systems, Inc.  All rights reserved.
 Copyright (c) 2006-2010 Voltaire, Inc. All rights reserved.
-Copyright (c) 2006-2017 Sandia National Laboratories. All rights reserved.
+Copyright (c) 2006-2021 Sandia National Laboratories. All rights reserved.
 Copyright (c) 2006-2010 Sun Microsystems, Inc.  All rights reserved.
                         Use is subject to license terms.
-Copyright (c) 2006-2017 The University of Houston. All rights reserved.
+Copyright (c) 2006-2021 The University of Houston. All rights reserved.
 Copyright (c) 2006-2009 Myricom, Inc.  All rights reserved.
 Copyright (c) 2007-2017 UT-Battelle, LLC. All rights reserved.
-Copyright (c) 2007-2020 IBM Corporation.  All rights reserved.
+Copyright (c) 2007-2021 IBM Corporation.  All rights reserved.
 Copyright (c) 1998-2005 Forschungszentrum Juelich, Juelich Supercomputing
                         Centre, Federal Republic of Germany
 Copyright (c) 2005-2008 ZIH, TU Dresden, Federal Republic of Germany
 Copyright (c) 2007      Evergrid, Inc. All rights reserved.
-Copyright (c) 2008      Chelsio, Inc.  All rights reserved.
+Copyright (c) 2008-2016 Chelsio, Inc.  All rights reserved.
 Copyright (c) 2008-2009 Institut National de Recherche en
                         Informatique.  All rights reserved.
 Copyright (c) 2007      Lawrence Livermore National Security, LLC.
                         All rights reserved.
-Copyright (c) 2007-2017 Mellanox Technologies.  All rights reserved.
+Copyright (c) 2007-2019 Mellanox Technologies.  All rights reserved.
 Copyright (c) 2006-2010 QLogic Corporation.  All rights reserved.
 Copyright (c) 2008-2017 Oak Ridge National Labs.  All rights reserved.
 Copyright (c) 2006-2012 Oracle and/or its affiliates.  All rights reserved.
@@ -43,23 +43,28 @@ Copyright (c) 2009-2015 Bull SAS.  All rights reserved.
 Copyright (c) 2010      ARM ltd.  All rights reserved.
 Copyright (c) 2016      ARM, Inc.  All rights reserved.
 Copyright (c) 2010-2011 Alex Brick <bricka@ccs.neu.edu>.  All rights reserved.
-Copyright (c) 2012      The University of Wisconsin-La Crosse. All rights
-                        reserved.
+Copyright (c) 2012      The University of Wisconsin-La Crosse. All rights reserved.
 Copyright (c) 2013-2020 Intel, Inc. All rights reserved.
-Copyright (c) 2011-2017 NVIDIA Corporation.  All rights reserved.
-Copyright (c) 2016      Broadcom Limited.  All rights reserved.
-Copyright (c) 2011-2017 Fujitsu Limited.  All rights reserved.
+Copyright (c) 2011-2021 NVIDIA Corporation.  All rights reserved.
+Copyright (c) 2016-2018 Broadcom Limited.  All rights reserved.
+Copyright (c) 2011-2021 Fujitsu Limited.  All rights reserved.
 Copyright (c) 2014-2015 Hewlett-Packard Development Company, LP.  All
                         rights reserved.
-Copyright (c) 2013-2017 Research Organization for Information Science (RIST).
+Copyright (c) 2013-2021 Research Organization for Information Science (RIST).
                         All rights reserved.
-Copyright (c) 2017-2020 Amazon.com, Inc. or its affiliates.  All Rights
+Copyright (c) 2017-2021 Amazon.com, Inc. or its affiliates.  All Rights
                         reserved.
 Copyright (c) 2018      DataDirect Networks. All rights reserved.
-Copyright (c) 2018-2020 Triad National Security, LLC. All rights reserved.
-Copyright (c) 2020      Google, LLC. All rights reserved.
+Copyright (c) 2018-2021 Triad National Security, LLC. All rights reserved.
+Copyright (c) 2019-2021 Hewlett Packard Enterprise Development, LP.
+Copyright (c) 2020-2021 Google, LLC. All rights reserved.
 Copyright (c) 2002      University of Chicago
 Copyright (c) 2001      Argonne National Laboratory
+Copyright (c) 2020-2021 Cornelis Networks, Inc. All rights reserved.
+Copyright (c) 2021      Nanook Consulting
+Copyright (c) 2017-2019 Iowa State University Research Foundation, Inc.
+                        All rights reserved.
+Copyright (c) 2024      Advanced Micro Devices, Inc. All rights reserved.
 
 $COPYRIGHT$
 
diff --git a/Makefile.am b/Makefile.am
index 63613685ffb..bff3c79c64f 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -9,10 +9,10 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2006-2016 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2006-2022 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2012-2015 Los Alamos National Security, Inc.  All rights reserved.
 # Copyright (c) 2014-2019 Intel, Inc.  All rights reserved.
-# Copyright (c) 2017-2020 Amazon.com, Inc. or its affiliates.
+# Copyright (c) 2017-2022 Amazon.com, Inc. or its affiliates.  All Rights reserved.
 #                         All Rights reserved.
 # Copyright (c) 2020      IBM Corporation.  All rights reserved.
 # $COPYRIGHT$
@@ -22,9 +22,12 @@
 # $HEADER$
 #
 
-SUBDIRS = config contrib 3rd-party $(MCA_PROJECT_SUBDIRS) test
-DIST_SUBDIRS = config contrib 3rd-party $(MCA_PROJECT_DIST_SUBDIRS) test
-EXTRA_DIST = README.md INSTALL VERSION Doxyfile LICENSE autogen.pl README.JAVA.md AUTHORS
+# There is an assumption in config/Makefile.am that config is the first
+# subdirectory in the DIST_SUBDIRS list.  If that changes, work may
+# be required.
+SUBDIRS = config contrib 3rd-party $(MCA_PROJECT_SUBDIRS) test docs
+DIST_SUBDIRS = config contrib 3rd-party $(MCA_PROJECT_DIST_SUBDIRS) test docs
+EXTRA_DIST = README.md VERSION Doxyfile LICENSE autogen.pl AUTHORS
 
 include examples/Makefile.include
 
diff --git a/Makefile.ompi-rules b/Makefile.ompi-rules
index d8aefcb8f3d..d18d49c4978 100644
--- a/Makefile.ompi-rules
+++ b/Makefile.ompi-rules
@@ -1,7 +1,8 @@
 # -*- makefile -*-
-# Copyright (c) 2008-2018 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2008-2022 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
 # Copyright (c) 2020      Intel, Inc.  All rights reserved.
+# Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -9,72 +10,6 @@
 # $HEADER$
 #
 
-MD2NROFF = $(OMPI_TOP_SRCDIR)/config/md2nroff.pl
-
-TRIM_OPTIONS=
-if ! MAN_PAGE_BUILD_MPIFH_BINDINGS
-    TRIM_OPTIONS += --nofortran
-endif
-if ! MAN_PAGE_BUILD_USEMPIF08_BINDINGS
-    TRIM_OPTIONS += --nof08
-endif
-
-# JMS This rule can be deleted once all man pages have been converted
-# to markdown.
-.1in.1:
-	$(OMPI_V_GEN) $(top_srcdir)/config/make_manpage.pl \
-	  --package-name='@PACKAGE_NAME@' \
-	  --package-version='@PACKAGE_VERSION@' \
-	  --ompi-date='@OMPI_RELEASE_DATE@' \
-	  --opal-date='@OPAL_RELEASE_DATE@' \
-	  --input=$< \
-	  --output=$@
-
-# JMS This rule can be deleted once all man pages have been converted
-# to markdown.
-.3in.3:
-	$(OMPI_V_GEN) $(top_srcdir)/config/make_manpage.pl \
-	  --package-name='@PACKAGE_NAME@' \
-	  --package-version='@PACKAGE_VERSION@' \
-	  --ompi-date='@OMPI_RELEASE_DATE@' \
-	  --opal-date='@OPAL_RELEASE_DATE@' \
-	  $(TRIM_OPTIONS) \
-	  --input=$< \
-	  --output=$@
-
-# JMS This rule can be deleted once all man pages have been converted
-# to markdown.
-.7in.7:
-	$(OMPI_V_GEN) $(top_srcdir)/config/make_manpage.pl \
-	  --package-name='@PACKAGE_NAME@' \
-	  --package-version='@PACKAGE_VERSION@' \
-	  --ompi-date='@OMPI_RELEASE_DATE@' \
-	  --opal-date='@OPAL_RELEASE_DATE@' \
-	  --input=$< \
-	  --output=$@
-
-%.1: %.1.md
-	$(OMPI_V_GEN) $(MD2NROFF) --source=$< --dest=$@ --pandoc=$(PANDOC)
-
-%.3: %.3.md
-	$(OMPI_V_GEN) $(MD2NROFF) --source=$< --dest=$@ --pandoc=$(PANDOC)
-
-%.5: %.5.md
-	$(OMPI_V_GEN) $(MD2NROFF) --source=$< --dest=$@ --pandoc=$(PANDOC)
-
-%.7: %.7.md
-	$(OMPI_V_GEN) $(MD2NROFF) --source=$< --dest=$@ --pandoc=$(PANDOC)
-
-# It is an error to "configure --disable-man-pages" and then try to
-# "make dist".
-if !OPAL_ENABLE_MAN_PAGES
-dist-hook:
-	@echo "************************************************************************************"
-	@echo "ERROR: 'make dist' inoperable when Open MPI is configured with --disable-man-pages"
-	@echo "************************************************************************************"
-	@/bin/false
-endif
-
 # A little verbosity magic; "make" will show the terse output.  "make
 # V=1" will show the actual commands used (just like the other
 # Automake-generated compilation/linker rules).
@@ -91,3 +26,19 @@ ompi__v_MKDIR_0 = @echo "  MKDIR   " $@;
 OMPI_V_GEN = $(ompi__v_GEN_$V)
 ompi__v_GEN_ = $(ompi__v_GEN_$AM_DEFAULT_VERBOSITY)
 ompi__v_GEN_0 = @echo "  GENERATE" $@;
+
+OMPI_V_COPYALL = $(ompi__v_COPYALL_$V)
+ompi__v_COPYALL_ = $(ompi__v_COPYALL_$AM_DEFAULT_VERBOSITY)
+ompi__v_COPYALL_0 = @echo "  COPY tree $@";
+
+OMPI_V_SPHINX_COPYRST = $(ompi__v_SPHINX_COPYRST_$V)
+ompi__v_SPHINX_COPYRST_ = $(ompi__v_SPHINX_COPYRST_$AM_DEFAULT_VERBOSITY)
+ompi__v_SPHINX_COPYRST_0 = @echo "  COPY RST source files";
+
+OMPI_V_SPHINX_HTML = $(ompi__v_SPHINX_HTML_$V)
+ompi__v_SPHINX_HTML_ = $(ompi__v_SPHINX_HTML_$AM_DEFAULT_VERBOSITY)
+ompi__v_SPHINX_HTML_0 = @echo "  GENERATE HTML docs";
+
+OMPI_V_SPHINX_MAN = $(ompi__v_SPHINX_MAN_$V)
+ompi__v_SPHINX_MAN_ = $(ompi__v_SPHINX_MAN_$AM_DEFAULT_VERBOSITY)
+ompi__v_SPHINX_MAN_0 = @echo "  GENERATE man pages";
diff --git a/NEWS b/NEWS
deleted file mode 100644
index fe089b84f87..00000000000
--- a/NEWS
+++ /dev/null
@@ -1,4638 +0,0 @@
-Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
-                        University Research and Technology
-                        Corporation.  All rights reserved.
-Copyright (c) 2004-2006 The University of Tennessee and The University
-                        of Tennessee Research Foundation.  All rights
-                        reserved.
-Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
-                        University of Stuttgart.  All rights reserved.
-Copyright (c) 2004-2006 The Regents of the University of California.
-                        All rights reserved.
-Copyright (c) 2006-2020 Cisco Systems, Inc.  All rights reserved.
-Copyright (c) 2006      Voltaire, Inc. All rights reserved.
-Copyright (c) 2006      Sun Microsystems, Inc.  All rights reserved.
-                        Use is subject to license terms.
-Copyright (c) 2006-2017 Los Alamos National Security, LLC.  All rights
-                        reserved.
-Copyright (c) 2010-2017 IBM Corporation.  All rights reserved.
-Copyright (c) 2012      Oak Ridge National Labs.  All rights reserved.
-Copyright (c) 2012      Sandia National Laboratories.  All rights reserved.
-Copyright (c) 2012      University of Houston. All rights reserved.
-Copyright (c) 2013      NVIDIA Corporation.  All rights reserved.
-Copyright (c) 2013-2018 Intel, Inc. All rights reserved.
-Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights
-                        reserved.
-$COPYRIGHT$
-
-Additional copyrights may follow
-
-$HEADER$
-
-===========================================================================
-
-This file contains the main features as well as overviews of specific
-bug fixes (and other actions) for each version of Open MPI since
-version 1.0.
-
-As more fully described in the "Software Version Number" section in
-the README file, Open MPI typically releases two separate version
-series simultaneously.  Since these series have different goals and
-are semi-independent of each other, a single NEWS-worthy item may be
-introduced into different series at different times.  For example,
-feature F was introduced in the vA.B series at version vA.B.C, and was
-later introduced into the vX.Y series at vX.Y.Z.
-
-The first time feature F is released, the item will be listed in the
-vA.B.C section, denoted as:
-
-   (** also to appear: X.Y.Z) -- indicating that this item is also
-                                 likely to be included in future release
-                                 version vX.Y.Z.
-
-When vX.Y.Z is later released, the same NEWS-worthy item will also be
-included in the vX.Y.Z section and be denoted as:
-
-   (** also appeared: A.B.C)  -- indicating that this item was previously
-                                 included in release version vA.B.C.
-
-Master (not on release branches yet)
-------------------------------------
-
-**********************************************************************
-*  PRE-DEPRECATION WARNING: MPIR Support
-*
-* As was announced in summer 2017, Open MPI is deprecating support for
-* MPIR-based tools beginning with the future release of OMPI v5.0, with
-* full removal of that support tentatively planned for OMPI v6.0.
-*
-* This serves as a pre-deprecation warning to provide tools developers
-* with sufficient time to migrate to PMIx. Support for PMIx-based
-* tools will be rolled out during the OMPI v4.x series. No runtime
-* deprecation warnings will be output during this time.
-*
-* Runtime deprecation warnings will be output beginning with the OMPI v5.0
-* release whenever MPIR-based tools connect to Open MPI's mpirun/mpiexec
-* launcher.
-**********************************************************************
-
-- Fix rank-by algorithms to properly rank by object and span
-- Do not build Open SHMEM layer when there are no SPMLs available.
-  Currently, this means the Open SHMEM layer will only build if
-  a MXM or UCX library is found.
-- Remove all vestiges of the C/R support
-
-4.0.5 -- August, 2020
----------------------
-
-- Fix a problem with MPI RMA compare and swap operations.  Thanks
-  to Wojciech Chlapek for reporting.
-- Disable binding of MPI processes to system resources by Open MPI
-  if an application is launched using SLURM's srun command.
-- Disable building of the Fortran mpi_f08 module when configuring
-  Open MPI with default 8 byte Fortran integer size.  Thanks to
-  @ahcien for reporting.
-- Fix a problem with mpirun when the --map-by option is used.
-  Thanks to Wenbin Lyu for reporting.
-- Fix some issues with MPI one-sided operations uncovered using Global
-  Arrays regression test-suite.  Thanks to @bjpalmer for reporting.
-- Fix a problem with make check when using the PGI compiler.  Thanks to
-  Carl Ponder for reporting.
-- Fix a problem with MPI_FILE_READ_AT_ALL that could lead to application
-  hangs under certain circumstances.  Thanks to Scot Breitenfeld for
-  reporting.
-- Fix a problem building C++ applications with newer versions of GCC.
-  Thanks to Constantine Khrulev for reporting.
-
-4.0.4 -- June, 2020
--------------------
-
-- Fix a memory patcher issue intercepting shmat and shmdt.  This was
-  observed on RHEL 8.x ppc64le (see README for more info).
-- Fix an illegal access issue caught using gcc's address sanitizer.
-  Thanks to  Georg Geiser for reporting.
-- Add checks to avoid conflicts with a libevent library shipped with LSF.
-- Switch to linking against libevent_core rather than libevent, if present.
-- Add improved support for UCX 1.9 and later.
-- Fix an ABI compatibility issue with the Fortran 2008 bindings.
-  Thanks to Alastair McKinstry for reporting.
-- Fix an issue with rpath of /usr/lib64 when building OMPI on
-  systems with Lustre.  Thanks to David Shrader for reporting.
-- Fix a memory leak occurring with certain MPI RMA operations.
-- Fix an issue with ORTE's mapping of MPI processes to resources.
-  Thanks to Alex Margolin for reporting and providing a fix.
-- Correct a problem with incorrect error codes being returned
-  by OMPI MPI_T functions.
-- Fix an issue with debugger tools not being able to attach
-  to mpirun more than once.  Thanks to Gregory Lee for reporting.
-- Fix an issue with the Fortran compiler wrappers when using
-  NAG compilers.  Thanks to Peter Brady for reporting.
-- Fix an issue with the ORTE ssh based process launcher at scale.
-  Thanks to Benjamín Hernández for reporting.
-- Address an issue when using shared MPI I/O operations.  OMPIO will
-  now successfully return from the file open statement but will
-  raise an error if the file system does not supported shared I/O
-  operations.  Thanks to Romain Hild for reporting.
-- Fix an issue with MPI_WIN_DETACH.  Thanks to Thomas Naughton for reporting.
-
-4.0.3 -- March, 2020
------------------------
-
-- Update embedded PMIx to 3.1.5
-- Add support for Mellanox ConnectX-6.
-- Fix an issue in OpenMPI IO when using shared file pointers.
-  Thanks to Romain Hild for reporting.
-- Fix a problem with Open MPI using a previously installed
-  Fortran mpi module during compilation.  Thanks to Marcin
-  Mielniczuk for reporting
-- Fix a problem with Fortran compiler wrappers ignoring use of
-  disable-wrapper-runpath configure option.  Thanks to David
-  Shrader for reporting.
-- Fixed an issue with trying to use mpirun on systems where neither
-  ssh nor rsh is installed.
-- Address some problems found when using XPMEM for intra-node message
-  transport.
-- Improve dimensions returned by MPI_Dims_create for certain
-  cases.  Thanks to @aw32 for reporting.
-- Fix an issue when sending messages larger than 4GB. Thanks to
-  Philip Salzmann for reporting this issue.
-- Add ability to specify alternative module file path using
-  Open MPI's RPM spec file.  Thanks to @jschwartz-cray for reporting.
-- Clarify use of --with-hwloc configuration option in the README.
-  Thanks to Marcin Mielniczuk for raising this documentation issue.
-- Fix an issue with shmem_atomic_set.  Thanks to Sameh Sharkawi for reporting.
-- Fix a problem with MPI_Neighbor_alltoall(v,w) for cartesian communicators
-  with cyclic boundary conditions.  Thanks to Ralph Rabenseifner and
-  Tony Skjellum for reporting.
-- Fix an issue using Open MPIO on 32 bit systems.  Thanks to
-  Orion Poplawski for reporting.
-- Fix an issue with NetCDF test deadlocking when using the vulcan
-  Open MPIO component.  Thanks to Orion Poplawski for reporting.
-- Fix an issue with the mpi_yield_when_idle parameter being ignored
-  when set in the Open MPI MCA parameter configuration file.
-  Thanks to @iassiour for reporting.
-- Address an issue with Open MPIO when writing/reading more than 2GB
-  in an operation.  Thanks to Richard Warren for reporting.
-
-4.0.2 -- September, 2019
-------------------------
-
-- Update embedded PMIx to 3.1.4
-- Enhance Open MPI to detect when processes are running in
-  different name spaces on the same node, in which case the
-  vader CMA single copy mechanism is disabled.  Thanks
-  to Adrian Reber for reporting and providing a fix.
-- Fix an issue with ORTE job tree launch mechanism.  Thanks
-  to @lanyangyang for reporting.
-- Fix an issue with env processing when running as root.
-  Thanks to Simon Byrne for reporting and providing a fix.
-- Fix Fortran MPI_FILE_GET_POSITION return code bug.
-  Thanks to Wei-Keng Liao for reporting.
-- Fix user defined datatypes/ops leak in nonblocking base collective
-  component.  Thanks to Andrey Maslennikov for verifying fix.
-- Fixed shared memory not working with spawned processes.
-  Thanks to @rodarima for reporting.
-- Fix data corruption of overlapping datatypes on sends.
-  Thanks to DKRZ for reporting.
-- Fix segfault in oob_tcp component on close with active listeners.
-  Thanks to Orivej Desh for reporting and providing a fix.
-- Fix divide by zero segfault in ompio.
-  Thanks to @haraldkl for reporting and providing a fix.
-- Fix finalize of flux compnents.
-  Thanks to Stephen Herbein and Jim Garlick for providing a fix.
-- Fix osc_rdma_acc_single_intrinsic regression.
-  Thanks to Joseph Schuchart for reporting and providing a fix.
-- Fix hostnames with large integers.
-  Thanks to @perrynzhou for reporting and providing a fix.
-- Fix Deadlock in MPI_Fetch_and_op when using UCX
-  Thanks to Joseph Schuchart for reporting.
-- Fix the SLURM plm for mpirun-based launching.
-  Thanks to Jordon Hayes for reporting and providing a fix.
-- Prevent grep failure in rpmbuild from aborting.
-  Thanks to Daniel Letai for reporting.
-- Fix btl/vader finalize sequence.
-  Thanks to Daniel Vollmer for reporting.
-- Fix pml/ob1 local handle sent during PUT control message.
-  Thanks to @EmmanuelBRELLE for reporting and providing a fix.
-- Fix Memory leak with persistent MPI sends and the ob1 "get" protocol.
-  Thanks to @s-kuberski for reporting.
-- v4.0.x: mpi: mark MPI_COMBINER_{HVECTOR,HINDEXED,STRUCT}_INTEGER
-  removed unless configured with --enable-mpi1-compatibility
-- Fix make-authors.pl when run in a git submodule.
-  Thanks to Michael Heinz for reporting and providing a fix.
-- Fix deadlock with mpi_assert_allow_overtaking in MPI_Issend.
-  Thanks to Joseph Schuchart and George Bosilca for reporting.
-- Add compilation flag to allow unwinding through files that are
-  present in the stack when attaching with MPIR.
-  Thanks to James A Clark for reporting and providing a fix.
-
-Known issues:
-
-- There is a known issue with the OFI libfabric and PSM2 MTLs when trying to send
-  very long (> 4 GBytes) messages.  In this release, these MTLs will catch
-  this case and abort the transfer.  A future release will provide a
-  better solution to this issue.
-
-4.0.1 -- March, 2019
---------------------
-
-- Update embedded PMIx to 3.1.2.
-- Fix an issue with Vader (shared-memory) transport on OS-X. Thanks
-  to Daniel Vollmer for reporting.
-- Fix a problem with the usNIC BTL Makefile.  Thanks to George Marselis
-  for reporting.
-- Fix an issue when using --enable-visibility configure option
-  and older versions of hwloc.  Thanks to Ben Menadue for reporting
-  and providing a fix.
-- Fix an issue with MPI_WIN_CREATE_DYNAMIC and MPI_GET from self.
-  Thanks to Bart Janssens for reporting.
-- Fix an issue of excessive compiler warning messages from mpi.h
-  when using newer C++ compilers.  Thanks to @Shadow-fax for
-  reporting.
-- Fix a problem when building Open MPI using clang 5.0.
-- Fix a problem with MPI_WIN_CREATE when using UCX.  Thanks
-  to Adam Simpson for reporting.
-- Fix a memory leak encountered for certain MPI datatype
-  destructor operations.  Thanks to Axel Huebl for reporting.
-- Fix several problems with MPI RMA accumulate operations.
-  Thanks to Jeff Hammond for reporting.
-- Fix possible race condition in closing some file descriptors
-  during job launch using mpirun.  Thanks to Jason Williams
-  for reporting and providing a fix.
-- Fix a problem in OMPIO for large individual write operations.
-  Thanks to Axel Huebl for reporting.
-- Fix a problem with parsing of map-by ppr options to mpirun.
-  Thanks to David Rich for reporting.
-- Fix a problem observed when using the mpool hugepage component.  Thanks
-  to Hunter Easterday for reporting and fixing.
-- Fix valgrind warning generated when invoking certain MPI Fortran
-  data type creation functions.  Thanks to @rtoijala for reporting.
-- Fix a problem when trying to build with a PMIX 3.1 or newer
-  release.  Thanks to Alastair McKinstry for reporting.
-- Fix a problem encountered with building MPI F08 module files.
-  Thanks to Igor Andriyash and Axel Huebl for reporting.
-- Fix two memory leaks encountered for certain MPI-RMA usage patterns.
-  Thanks to Joseph Schuchart for reporting and fixing.
-- Fix a problem with the ORTE rmaps_base_oversubscribe MCA paramater.
-  Thanks to @iassiour for reporting.
-- Fix a problem with UCX PML default error handler for MPI communicators.
-  Thanks to Marcin Krotkiewski for reporting.
-- Fix various issues with OMPIO uncovered by the testmpio test suite.
-
-4.0.0 -- September, 2018
-------------------------
-
-- OSHMEM updated to the OpenSHMEM 1.4 API.
-- Do not build OpenSHMEM layer when there are no SPMLs available.
-  Currently, this means the OpenSHMEM layer will only build if
-  a MXM or UCX library is found.
-- A UCX BTL was added for enhanced MPI RMA support using UCX
-- With this release,  OpenIB BTL now only supports iWarp and RoCE by default.
-- Updated internal HWLOC to 2.0.2
-- Updated internal PMIx to 3.0.2
-- Change the priority for selecting external verses internal HWLOC
-  and PMIx packages to build.  Starting with this release, configure
-  by default selects available external HWLOC and PMIx packages over
-  the internal ones.
-- Updated internal ROMIO to 3.2.1.
-- Removed support for the MXM MTL.
-- Removed support for SCIF.
-- Improved CUDA support when using UCX.
-- Enable use of CUDA allocated buffers for OMPIO.
-- Improved support for two phase MPI I/O operations when using OMPIO.
-- Added support for Software-based Performance Counters, see
-  https://github.com/davideberius/ompi/wiki/How-to-Use-Software-Based-Performance-Counters-(SPCs)-in-Open-MPI
-- Change MTL OFI from opting-IN on "psm,psm2,gni" to opting-OUT on
-  "shm,sockets,tcp,udp,rstream"
-- Various improvements to MPI RMA performance when using RDMA
-  capable interconnects.
-- Update memkind component to use the memkind 1.6 public API.
-- Fix a problem with javadoc builds using OpenJDK 11.  Thanks to
-  Siegmar Gross for reporting.
-- Fix a memory leak using UCX.  Thanks to Charles Taylor for reporting.
-- Fix hangs in MPI_FINALIZE when using UCX.
-- Fix a problem with building Open MPI using an external PMIx 2.1.2
-  library.  Thanks to Marcin Krotkiewski for reporting.
-- Fix race conditions in Vader (shared memory) transport.
-- Fix problems with use of newer map-by mpirun options.  Thanks to
-  Tony Reina for reporting.
-- Fix rank-by algorithms to properly rank by object and span
-- Allow for running as root of two environment variables are set.
-  Requested by Axel Huebl.
-- Fix a problem with building the Java bindings when using Java 10.
-  Thanks to Bryce Glover for reporting.
-- Fix a problem with ORTE not reporting error messages if an application
-  terminated normally but exited with non-zero error code.  Thanks to
-  Emre Brookes for reporting.
-
-3.1.6 -- March, 2020
---------------------
-
-- Fix one-sided shared memory window configuration bug.
-- Fix support for PGI'18 compiler.
-- Fix issue with zero-length blockLength in MPI_TYPE_INDEXED.
-- Fix run-time linker issues with OMPIO on newer Linux distros.
-- Fix PMIX dstore locking compilation issue.  Thanks to Marco Atzeri
-  for reporting the issue.
-- Allow the user to override modulefile_path in the Open MPI SRPM,
-  even if install_in_opt is set to 1.
-- Properly detect ConnectX-6 HCAs in the openib BTL.
-- Fix segfault in the MTL/OFI initialization for large jobs.
-- Fix issue to guarantee to properly release MPI one-sided lock when
-  using UCX transports to avoid a deadlock.
-- Fix potential deadlock when processing outstanding transfers with
-  uGNI transports.
-- Fix various portals4 control flow bugs.
-- Fix communications ordering for alltoall and Cartesian neighborhood
-  collectives.
-- Fix an infinite recursion crash in the memory patcher on systems
-  with glibc v2.26 or later (e.g., Ubuntu 18.04) when using certain
-  OS-bypass interconnects.
-
-3.1.5 -- November, 2019
------------------------
-
-- Fix OMPIO issue limiting file reads/writes to 2GB.  Thanks to
-  Richard Warren for reporting the issue.
-- At run time, automatically disable Linux cross-memory attach (CMA)
-  for vader BTL (shared memory) copies when running in user namespaces
-  (i.e., containers).  Many thanks to Adrian Reber for raising the
-  issue and providing the fix.
-- Sending very large MPI messages using the ofi MTL will fail with
-  some of the underlying Libfabric transports (e.g., PSM2 with
-  messages >=4GB, verbs with messages >=2GB).  Prior version of Open
-  MPI failed silently; this version of Open MPI invokes the
-  appropriate MPI error handler upon failure.  See
-  https://github.com/open-mpi/ompi/issues/7058 for more details.
-  Thanks to Emmanuel Thomé for raising the issue.
-- Fix case where 0-extent datatypes might be eliminated during
-  optimization.  Thanks to Github user @tjahns for raising the issue.
-- Ensure that the MPIR_Breakpoint symbol is not optimized out on
-  problematic platforms.
-- Fix MPI one-sided 32 bit atomic support.
-- Fix OMPIO offset calculations with SEEK_END and SEEK_CUR in
-  MPI_FILE_GET_POSITION.  Thanks to Wei-keng Liao for raising the
-  issue.
-- Add "naive" regx component that will never fail, no matter how
-  esoteric the hostnames are.
-- Fix corner case for datatype extent computations.  Thanks to David
-  Dickenson for raising the issue.
-- Allow individual jobs to set their map/rank/bind policies when
-  running LSF.  Thanks to Nick R. Papior for assistance in solving the
-  issue.
-- Fix MPI buffered sends with the "cm" PML.
-- Properly propagate errors to avoid deadlocks in MPI one-sided operations.
-- Update to PMIx v2.2.3.
-- Fix data corruption in non-contiguous MPI accumulates over UCX.
-- Fix ssh-based tree-based spawning at scale.  Many thanks to Github
-  user @zrss for the report and diagnosis.
-- Fix the Open MPI RPM spec file to not abort when grep fails.  Thanks
-  to Daniel Letai for bringing this to our attention.
-- Handle new SLURM CLI options (SLURM 19 deprecated some options that
-  Open MPI was using).  Thanks to Jordan Hayes for the report and the
-  initial fix.
-- OMPI: fix division by zero with an empty file view.
-- Also handle shmat()/shmdt() memory patching with OS-bypass networks.
-- Add support for unwinding info to all files that are present in the
-  stack starting from MPI_Init, which is helpful with parallel
-  debuggers.  Thanks to James Clark for the report and initial fix.
-- Fixed inadvertant use of bitwise operators in the MPI C++ bindings
-  header files.  Thanks to Bert Wesarg for the report and the fix.
-
-3.1.4 -- April, 2019
---------------------
-
-- Fix compile error when configured with --enable-mpi-java and
-  --with-devel-headers.  Thanks to @g-raffy for reporting the issue
-  (** also appeared: v3.0.4).
-- Only use hugepages with appropriate permissions.  Thanks to Hunter
-  Easterday for the fix.
-- Fix possible floating point rounding and division issues in OMPIO
-  which led to crashes and/or data corruption with very large data.
-  Thanks to Axel Huebl and René Widera for identifing the issue,
-  supplying and testing the fix (** also appeared: v3.0.4).
-- Use static_cast<> in mpi.h where appropriate.  Thanks to @shadow-fx
-  for identifying the issue (** also appeared: v3.0.4).
-- Fix RMA accumulate of non-predefined datatypes with predefined
-  operators.  Thanks to Jeff Hammond for raising the issue (** also
-  appeared: v3.0.4).
-- Fix race condition when closing open file descriptors when launching
-  MPI processes.  Thanks to Jason Williams for identifying the issue and
-  supplying the fix (** also appeared: v3.0.4).
-- Fix support for external PMIx v3.1.x.
-- Fix Valgrind warnings for some MPI_TYPE_CREATE_* functions.  Thanks
-  to Risto Toijala for identifying the issue and supplying the fix (**
-  also appeared: v3.0.4).
-- Fix MPI_TYPE_CREATE_F90_{REAL,COMPLEX} for r=38 and r=308 (** also
-  appeared: v3.0.4).
-- Fix assembly issues with old versions of gcc (<6.0.0) that affected
-  the stability of shared memory communications (e.g., with the vader
-  BTL) (** also appeared: v3.0.4).
-- Fix MPI_Allreduce crashes with some cases in the coll/spacc module.
-- Fix the OFI MTL handling of MPI_ANY_SOURCE (** also appeared:
-  v3.0.4).
-- Fix noisy errors in the openib BTL with regards to
-  ibv_exp_query_device().  Thanks to Angel Beltre and others who
-  reported the issue (** also appeared: v3.0.4).
-- Fix zero-size MPI one-sided windows with UCX.
-
-3.1.3 -- October, 2018
-----------------------
-
-- Fix race condition in MPI_THREAD_MULTIPLE support of non-blocking
-  send/receive path.
-- Fix error handling SIGCHLD forwarding.
-- Add support for CHARACTER and LOGICAL Fortran datatypes for MPI_SIZEOF.
-- Fix compile error when using OpenJDK 11 to compile the Java bindings.
-- Fix crash when using a hostfile with a 'user@host' line.
-- Numerous Fortran '08 interface fixes.
-- TCP BTL error message fixes.
-- OFI MTL now will use any provider other than shm, sockets, tcp, udp, or
-  rstream, rather than only supporting gni, psm, and psm2.
-- Disable async receive of CUDA buffers by default, fixing a hang
-  on large transfers.
-- Support the BCM57XXX and BCM58XXX Broadcomm adapters.
-- Fix minmax datatype support in ROMIO.
-- Bug fixes in vader shared memory transport.
-- Support very large buffers with MPI_TYPE_VECTOR.
-- Fix hang when launching with mpirun on Cray systems.
-
-3.1.2 -- August, 2018
-------------------------
-
-- A subtle race condition bug was discovered in the "vader" BTL
-  (shared memory communications) that, in rare instances, can cause
-  MPI processes to crash or incorrectly classify (or effectively drop)
-  an MPI message sent via shared memory.  If you are using the "ob1"
-  PML with "vader" for shared memory communication (note that vader is
-  the default for shared memory communication with ob1), you need to
-  upgrade to v3.1.2 or later to fix this issue.  You may also upgrade
-  to the following versions to fix this issue:
-  - Open MPI v2.1.5 (expected end of August, 2018) or later in the
-    v2.1.x series
-  - Open MPI v3.0.1 (released March, 2018) or later in the v3.0.x
-    series
-- Assorted Portals 4.0 bug fixes.
-- Fix for possible data corruption in MPI_BSEND.
-- Move shared memory file for vader btl into /dev/shm on Linux.
-- Fix for MPI_ISCATTER/MPI_ISCATTERV Fortran interfaces with MPI_IN_PLACE.
-- Upgrade PMIx to v2.1.3.
-- Numerous One-sided bug fixes.
-- Fix for race condition in uGNI BTL.
-- Improve handling of large number of interfaces with TCP BTL.
-- Numerous UCX bug fixes.
-
-3.1.1 -- June, 2018
--------------------
-
-- Fix potential hang in UCX PML during MPI_FINALIZE
-- Update internal PMIx to v2.1.2rc2 to fix forward version compatibility.
-- Add new MCA parameter osc_sm_backing_store to allow users to specify
-  where in the filesystem the backing file for the shared memory
-  one-sided component should live.  Defaults to /dev/shm on Linux.
-- Fix potential hang on non-x86 platforms when using builds with
-  optimization flags turned off.
-- Disable osc/pt2pt when using MPI_THREAD_MULTIPLE due to numerous
-  race conditions in the component.
-- Fix dummy variable names for the mpi and mpi_f08 Fortran bindings to
-  match the MPI standard.  This may break applications which use
-  name-based parameters in Fortran which used our internal names
-  rather than those documented in the MPI standard.
-- Revamp Java detection to properly handle new Java versions which do
-  not provide a javah wrapper.
-- Fix RMA function signatures for use-mpi-f08 bindings to have the
-  asynchonous property on all buffers.
-- Improved configure logic for finding the UCX library.
-
-3.1.0 -- May, 2018
-------------------
-
-- Various OpenSHMEM bug fixes.
-- Properly handle array_of_commands argument to Fortran version of
-  MPI_COMM_SPAWN_MULTIPLE.
-- Fix bug with MODE_SEQUENTIAL and the sharedfp MPI-IO component.
-- Use "javac -h" instead of "javah" when building the Java bindings
-  with a recent version of Java.
-- Fix mis-handling of jostepid under SLURM that could cause problems
-  with PathScale/OmniPath NICs.
-- Disable the POWER 7/BE block in configure.  Note that POWER 7/BE is
-  still not a supported platform, but it is no longer automatically
-  disabled.  See
-  https://github.com/open-mpi/ompi/issues/4349#issuecomment-374970982
-  for more information.
-- The output-filename option for mpirun is now converted to an
-  absolute path before being passed to other nodes.
-- Add monitoring component for PML, OSC, and COLL to track data
-  movement of MPI applications.  See
-  ompi/mca/commmon/monitoring/HowTo_pml_monitoring.tex for more
-  information about the monitoring framework.
-- Add support for communicator assertions: mpi_assert_no_any_tag,
-  mpi_assert_no_any_source, mpi_assert_exact_length, and
-  mpi_assert_allow_overtaking.
-- Update PMIx to version 2.1.1.
-- Update hwloc to 1.11.7.
-- Many one-sided behavior fixes.
-- Improved performance for Reduce and Allreduce using Rabenseifner's algorithm.
-- Revamped mpirun --help output to make it a bit more manageable.
-- Portals4 MTL improvements: Fix race condition in rendezvous protocol and
-  retry logic.
-- UCX OSC: initial implementation.
-- UCX PML improvements: add multi-threading support.
-- Yalla PML improvements: Fix error with irregular contiguous datatypes.
-- Openib BTL: disable XRC support by default.
-- TCP BTL: Add check to detect and ignore connections from processes
-  that aren't MPI (such as IDS probes) and verify that source and
-  destination are using the same version of Open MPI, fix issue with very
-  large message transfer.
-- ompi_info parsable output now escapes double quotes in values, and
-  also quotes values can contains colons.  Thanks to Lev Givon for the
-  suggestion.
-- CUDA-aware support can now handle GPUs within a node that do not
-  support CUDA IPC.  Earlier versions would get error and abort.
-- Add a mca parameter ras_base_launch_orted_on_hn to allow for launching
-  MPI processes on the same node where mpirun is executing using a separate
-  orte daemon, rather than the mpirun process.   This may be useful to set to
-  true when using SLURM, as it improves interoperability with SLURM's signal
-  propagation tools.  By default it is set to false, except for Cray XC systems.
-- Remove LoadLeveler RAS support.
-- Remove IB XRC support from the OpenIB BTL due to lack of support.
-- Add functionality for IBM s390 platforms.  Note that regular
-  regression testing does not occur on the s390 and it is not
-  considered a supported platform.
-- Remove support for big endian PowerPC.
-- Remove support for XL compilers older than v13.1.
-- Remove support for atomic operations using MacOS atomics library.
-
-3.0.6 -- March, 2020
---------------------
-
-- Fix one-sided shared memory window configuration bug.
-- Fix support for PGI'18 compiler.
-- Fix run-time linker issues with OMPIO on newer Linux distros.
-- Allow the user to override modulefile_path in the Open MPI SRPM,
-  even if install_in_opt is set to 1.
-- Properly detect ConnectX-6 HCAs in the openib BTL.
-- Fix segfault in the MTL/OFI initialization for large jobs.
-- Fix various portals4 control flow bugs.
-- Fix communications ordering for alltoall and Cartesian neighborhood
-  collectives.
-- Fix an infinite recursion crash in the memory patcher on systems
-  with glibc v2.26 or later (e.g., Ubuntu 18.04) when using certain
-  OS-bypass interconnects.
-
-3.0.5 -- November, 2019
------------------------
-
-- Fix OMPIO issue limiting file reads/writes to 2GB.  Thanks to
-  Richard Warren for reporting the issue.
-- At run time, automatically disable Linux cross-memory attach (CMA)
-  for vader BTL (shared memory) copies when running in user namespaces
-  (i.e., containers).  Many thanks to Adrian Reber for raising the
-  issue and providing the fix.
-- Sending very large MPI messages using the ofi MTL will fail with
-  some of the underlying Libfabric transports (e.g., PSM2 with
-  messages >=4GB, verbs with messages >=2GB).  Prior version of Open
-  MPI failed silently; this version of Open MPI invokes the
-  appropriate MPI error handler upon failure.  See
-  https://github.com/open-mpi/ompi/issues/7058 for more details.
-  Thanks to Emmanuel Thomé for raising the issue.
-- Fix case where 0-extent datatypes might be eliminated during
-  optimization.  Thanks to Github user @tjahns for raising the issue.
-- Ensure that the MPIR_Breakpoint symbol is not optimized out on
-  problematic platforms.
-- Fix OMPIO offset calculations with SEEK_END and SEEK_CUR in
-  MPI_FILE_GET_POSITION.  Thanks to Wei-keng Liao for raising the
-  issue.
-- Fix corner case for datatype extent computations.  Thanks to David
-  Dickenson for raising the issue.
-- Fix MPI buffered sends with the "cm" PML.
-- Update to PMIx v2.2.3.
-- Fix ssh-based tree-based spawning at scale.  Many thanks to Github
-  user @zrss for the report and diagnosis.
-- Fix the Open MPI RPM spec file to not abort when grep fails.  Thanks
-  to Daniel Letai for bringing this to our attention.
-- Handle new SLURM CLI options (SLURM 19 deprecated some options that
-  Open MPI was using).  Thanks to Jordan Hayes for the report and the
-  initial fix.
-- OMPI: fix division by zero with an empty file view.
-- Also handle shmat()/shmdt() memory patching with OS-bypass networks.
-- Add support for unwinding info to all files that are present in the
-  stack starting from MPI_Init, which is helpful with parallel
-  debuggers.  Thanks to James Clark for the report and initial fix.
-- Fixed inadvertant use of bitwise operators in the MPI C++ bindings
-  header files.  Thanks to Bert Wesarg for the report and the fix.
-- Added configure option --disable-wrappers-runpath (alongside the
-  already-existing --disable-wrappers-rpath option) to prevent Open
-  MPI's configure script from automatically adding runpath CLI options
-  to the wrapper compilers.
-
-3.0.4 -- April, 2019
---------------------
-
-- Fix compile error when configured with --enable-mpi-java and
-  --with-devel-headers.  Thanks to @g-raffy for reporting the issue.
-- Fix possible floating point rounding and division issues in OMPIO
-  which led to crashes and/or data corruption with very large data.
-  Thanks to Axel Huebl and René Widera for identifing the issue,
-  supplying and testing the fix (** also appeared: v3.0.4).
-- Use static_cast<> in mpi.h where appropriate.  Thanks to @shadow-fx
-  for identifying the issue.
-- Fix datatype issue with RMA accumulate.  Thanks to Jeff Hammond for
-  raising the issue.
-- Fix RMA accumulate of non-predefined datatypes with predefined
-  operators.  Thanks to Jeff Hammond for raising the issue.
-- Fix race condition when closing open file descriptors when launching
-  MPI processes.  Thanks to Jason Williams for identifying the issue and
-  supplying the fix.
-- Fix Valgrind warnings for some MPI_TYPE_CREATE_* functions.  Thanks
-  to Risto Toijala for identifying the issue and supplying the fix.
-- Fix MPI_TYPE_CREATE_F90_{REAL,COMPLEX} for r=38 and r=308.
-- Fix assembly issues with old versions of gcc (<6.0.0) that affected
-  the stability of shared memory communications (e.g., with the vader
-  BTL).
-- Fix the OFI MTL handling of MPI_ANY_SOURCE.
-- Fix noisy errors in the openib BTL with regards to
-  ibv_exp_query_device().  Thanks to Angel Beltre and others who
-  reported the issue.
-
-3.0.3 -- October, 2018
-----------------------
-
-- Fix race condition in MPI_THREAD_MULTIPLE support of non-blocking
-  send/receive path.
-- Fix error handling SIGCHLD forwarding.
-- Add support for CHARACTER and LOGICAL Fortran datatypes for MPI_SIZEOF.
-- Fix compile error when using OpenJDK 11 to compile the Java bindings.
-- Fix crash when using a hostfile with a 'user@host' line.
-- Numerous Fortran '08 interface fixes.
-- TCP BTL error message fixes.
-- OFI MTL now will use any provider other than shm, sockets, tcp, udp, or
-  rstream, rather than only supporting gni, psm, and psm2.
-- Disable async receive of CUDA buffers by default, fixing a hang
-  on large transfers.
-- Support the BCM57XXX and BCM58XXX Broadcomm adapters.
-- Fix minmax datatype support in ROMIO.
-- Bug fixes in vader shared memory transport.
-- Support very large buffers with MPI_TYPE_VECTOR.
-- Fix hang when launching with mpirun on Cray systems.
-- Bug fixes in OFI MTL.
-- Assorted Portals 4.0 bug fixes.
-- Fix for possible data corruption in MPI_BSEND.
-- Move shared memory file for vader btl into /dev/shm on Linux.
-- Fix for MPI_ISCATTER/MPI_ISCATTERV Fortran interfaces with MPI_IN_PLACE.
-- Upgrade PMIx to v2.1.4.
-- Fix for Power9 built-in atomics.
-- Numerous One-sided bug fixes.
-- Fix for race condition in uGNI BTL.
-- Improve handling of large number of interfaces with TCP BTL.
-- Numerous UCX bug fixes.
-- Add support for QLogic and Broadcom Cumulus RoCE HCAs to Open IB BTL.
-- Add patcher support for aarch64.
-- Fix hang on Power and ARM when Open MPI was built with low compiler
-  optimization settings.
-
-3.0.2 -- June, 2018
--------------------
-
-- Disable osc/pt2pt when using MPI_THREAD_MULTIPLE due to numerous
-  race conditions in the component.
-- Fix dummy variable names for the mpi and mpi_f08 Fortran bindings to
-  match the MPI standard.  This may break applications which use
-  name-based parameters in Fortran which used our internal names
-  rather than those documented in the MPI standard.
-- Fixed MPI_SIZEOF in the "mpi" Fortran module for the NAG compiler.
-- Fix RMA function signatures for use-mpi-f08 bindings to have the
-  asynchonous property on all buffers.
-- Fix Fortran MPI_COMM_SPAWN_MULTIPLE to properly follow the count
-  length argument when parsing the array_of_commands variable.
-- Revamp Java detection to properly handle new Java versions which do
-  not provide a javah wrapper.
-- Improved configure logic for finding the UCX library.
-- Add support for HDR InfiniBand link speeds.
-- Disable the POWER 7/BE block in configure.  Note that POWER 7/BE is
-  still not a supported platform, but it is no longer automatically
-  disabled.  See
-  https://github.com/open-mpi/ompi/issues/4349#issuecomment-374970982
-  for more information.
-
-3.0.1 -- March, 2018
-----------------------
-
-- Fix ability to attach parallel debuggers to MPI processes.
-- Fix a number of issues in MPI I/O found by the HDF5 test suite.
-- Fix (extremely) large message transfers with shared memory.
-- Fix out of sequence bug in multi-NIC configurations.
-- Fix stdin redirection bug that could result in lost input.
-- Disable the LSF launcher if CSM is detected.
-- Plug a memory leak in MPI_Mem_free().  Thanks to Philip Blakely for reporting.
-- Fix the tree spawn operation when the number of nodes is larger than the radix.
-  Thanks to Carlos Eduardo de Andrade for reporting.
-- Fix Fortran 2008 macro in MPI extensions.  Thanks to Nathan T. Weeks for
-  reporting.
-- Add UCX to list of interfaces that OpenSHMEM will use by default.
-- Add --{enable|disable}-show-load-errors-by-default to control
-  default behavior of the load errors option.
-- OFI MTL improvements: handle empty completion queues properly, fix
-  incorrect error message around fi_getinfo(), use default progress
-  option for provider by default, Add support for reading multiple
-  CQ events in ofi_progress.
-- PSM2 MTL improvements: Allow use of GPU buffers, thread fixes.
-- Numerous corrections to memchecker behavior.
-- Add a mca parameter ras_base_launch_orted_on_hn to allow for launching
-  MPI processes on the same node where mpirun is executing using a separate
-  orte daemon, rather than the mpirun process.   This may be useful to set to
-  true when using SLURM, as it improves interoperability with SLURM's signal
-  propagation tools.  By default it is set to false, except for Cray XC systems.
-- Fix a problem reported on the mailing separately by Kevin McGrattan and Stephen
-  Guzik about consistency issues on NFS file systems when using OMPIO. This fix
-  also introduces a new mca parameter fs_ufs_lock_algorithm which allows to
-  control the locking algorithm used by ompio for read/write operations. By
-  default, ompio does not perfom locking on local UNIX file systems, locks the
-  entire file per operation on NFS file systems, and selective byte-range
-  locking on other distributed file systems.
-- Add an mca parameter pmix_server_usock_connections to allow mpirun to
-  support applications statically built against the Open MPI v2.x release,
-  or installed in a container along with the Open MPI v2.x libraries. It is
-  set to false by default.
-
-3.0.0 -- September, 2017
-------------------------
-
-Major new features:
-
-- Use UCX allocator for OSHMEM symmetric heap allocations to optimize intra-node
-  data transfers.  UCX SPML only.
-- Use UCX multi-threaded API in the UCX PML.  Requires UCX 1.0 or later.
-- Added support for Flux PMI
-- Update embedded PMIx to version 2.1.0
-- Update embedded hwloc to version 1.11.7
-
-Changes in behavior compared to prior versions:
-
-- Per Open MPI's versioning scheme (see the README), increasing the
-  major version number to 3 indicates that this version is not
-  ABI-compatible with prior versions of Open MPI. In addition, there may
-  be differences in MCA parameter names and defaults from previous releases.
-  Command line options for mpirun and other commands may also differ from
-  previous versions. You will need to recompile MPI and OpenSHMEM applications
-  to work with this version of Open MPI.
-- With this release, Open MPI supports MPI_THREAD_MULTIPLE by default.
-- New configure options have been added to specify the locations of libnl
-  and zlib.
-- A new configure option has been added to request Flux PMI support.
-- The help menu for mpirun and related commands is now context based.
-  "mpirun --help compatibility" generates the help menu in the same format
-  as previous releases.
-
-Removed legacy support:
-- AIX is no longer supported.
-- Loadlever is no longer supported.
-- OpenSHMEM currently supports the UCX and MXM transports via the ucx and ikrit
-  SPMLs respectively.
-- Remove IB XRC support from the OpenIB BTL due to lack of support.
-- Remove support for big endian PowerPC.
-- Remove support for XL compilers older than v13.1
-
-Known issues:
-
-- MPI_Connect/accept between applications started by different mpirun
-  commands will fail, even if ompi-server is running.
-
-2.1.5 -- August 2018
---------------------
-
-- A subtle race condition bug was discovered in the "vader" BTL
-  (shared memory communications) that, in rare instances, can cause
-  MPI processes to crash or incorrectly classify (or effectively drop)
-  an MPI message sent via shared memory.  If you are using the "ob1"
-  PML with "vader" for shared memory communication (note that vader is
-  the default for shared memory communication with ob1), you need to
-  upgrade to v2.1.5 to fix this issue.  You may also upgrade to the
-  following versions to fix this issue:
-  - Open MPI v3.0.1 (released March, 2018) or later in the v3.0.x
-    series
-  - Open MPI v3.1.2 (expected end of August, 2018) or later
-- A link issue was fixed when the UCX library was not located in the
-  linker-default search paths.
-
-2.1.4 -- August, 2018
----------------------
-
-Bug fixes/minor improvements:
-- Disable the POWER 7/BE block in configure.  Note that POWER 7/BE is
-  still not a supported platform, but it is no longer automatically
-  disabled.  See
-  https://github.com/open-mpi/ompi/issues/4349#issuecomment-374970982
-  for more information.
-- Fix bug with request-based one-sided MPI operations when using the
-  "rdma" component.
-- Fix issue with large data structure in the TCP BTL causing problems
-  in some environments.  Thanks to @lgarithm for reporting the issue.
-- Minor Cygwin build fixes.
-- Minor fixes for the openib BTL:
-  - Support for the QLogic RoCE HCA
-  - Support for the Boradcom Cumulus RoCE HCA
-  - Enable support for HDR link speeds
-- Fix MPI_FINALIZED hang if invoked from an attribute destructor
-  during the MPI_COMM_SELF destruction in MPI_FINALIZE.  Thanks to
-  @AndrewGaspar for reporting the issue.
-- Java fixes:
-  - Modernize Java framework detection, especially on OS X/MacOS.
-    Thanks to Bryce Glover for reporting and submitting the fixes.
-  - Prefer "javac -h" to "javah" to support newer Java frameworks.
-- Fortran fixes:
-  - Use conformant dummy parameter names for Fortran bindings.  Thanks
-    to Themos Tsikas for reporting and submitting the fixes.
-  - Build the MPI_SIZEOF() interfaces in the "TKR"-style "mpi" module
-    whenever possible.  Thanks to Themos Tsikas for reporting the
-    issue.
-  - Fix array of argv handling for the Fortran bindings of
-    MPI_COMM_SPAWN_MULTIPLE (and its associated man page).
-  - Make NAG Fortran compiler support more robust in configure.
-- Disable the "pt2pt" one-sided MPI component when MPI_THREAD_MULTIPLE
-  is used.  This component is simply not safe in MPI_THREAD_MULTIPLE
-  scenarios, and will not be fixed in the v2.1.x series.
-- Make the "external" hwloc component fail gracefully if it is tries
-  to use an hwloc v2.x.y installation.  hwloc v2.x.y will not be
-  supported in the Open MPI v2.1.x series.
-- Fix "vader" shared memory support for messages larger than 2GB.
-  Thanks to Heiko Bauke for the bug report.
-- Configure fixes for external PMI directory detection.  Thanks to
-  Davide Vanzo for the report.
-
-2.1.3 -- March, 2018
---------------------
-
-Bug fixes/minor improvements:
-- Update internal PMIx version to 1.2.5.
-- Fix a problem with ompi_info reporting using param option.
-  Thanks to Alexander Pozdneev for reporting.
-- Correct PMPI_Aint_{add|diff} to be functions (not subroutines)
-  in the Fortran mpi_f08 module.
-- Fix a problem when doing MPI I/O using data types with large
-  extents in conjunction with MPI_TYPE_CREATE_SUBARRAY.  Thanks to
-  Christopher Brady for reporting.
-- Fix a problem when opening many files using MPI_FILE_OPEN.
-  Thanks to William Dawson for reporting.
-- Fix a problem with debuggers failing to attach to a running job.
-  Thanks to Dirk Schubert for reporting.
-- Fix a problem when using madvise and the OpenIB BTL.  Thanks to
-  Timo Bingmann for reporting.
-- Fix a problem in the Vader BTL that resulted in failures of
-  IMB under certain circumstances.  Thanks to Nicolas Morey-
-  Chaisemartin for reporting.
-- Fix a problem preventing Open MPI from working under Cygwin.
-  Thanks to Marco Atzeri for reporting.
-- Reduce some verbosity being emitted by the USNIC BTL under certain
-  circumstances.  Thanks to Peter Forai for reporting.
-- Fix a problem with misdirection of SIGKILL.  Thanks to Michael Fern
-  for reporting.
-- Replace use of posix_memalign with malloc for small allocations.  Thanks
-  to Ben Menaude for reporting.
-- Fix a problem with Open MPI's out of band TCP network for file descriptors
-  greater than 32767.  Thanks to Wojtek Wasko for reporting and fixing.
-- Plug a memory leak in MPI_Mem_free().  Thanks to Philip Blakely for reporting.
-
-2.1.2 -- September, 2017
-------------------------
-
-Bug fixes/minor improvements:
-- Update internal PMIx version to 1.2.3.
-- Fix some problems when using the NAG Fortran compiler to build Open MPI
-  and when using the compiler wrappers.  Thanks to Neil Carlson for reporting.
-- Fix a compilation problem with the SM BTL.  Thanks to Paul Hargrove for
-  reporting.
-- Fix a problem with MPI_IALLTOALLW when using zero-length messages.
-  Thanks to Dahai Guo for reporting.
-- Fix a problem with C11 generic type interface for SHMEM_G.  Thanks
-  to Nick Park for reporting.
-- Switch to using the lustreapi.h include file when building Open MPI
-  with Lustre support.
-- Fix a problem in the OB1 PML that led to hangs with OSU collective tests.
-- Fix a progression issue with MPI_WIN_FLUSH_LOCAL.  Thanks to
-  Joseph Schuchart for reporting.
-- Fix an issue with recent versions of PBSPro requiring libcrypto.
-  Thanks to Petr Hanousek for reporting.
-- Fix a problem when using MPI_ANY_SOURCE with MPI_SENDRECV.
-- Fix an issue that prevented signals from being propagated to ORTE
-  daemons.
-- Ensure that signals are forwarded from ORTE daemons to all processes
-  in the process group created by the daemons.  Thanks to Ted Sussman
-  for reporting.
-- Fix a problem with launching a job under a debugger. Thanks to
-  Greg Lee for reporting.
-- Fix a problem with Open MPI native I/O MPI_FILE_OPEN when using
-  a communicator having an associated topology.  Thanks to
-  Wei-keng Liao for reporting.
-- Fix an issue when using MPI_ACCUMULATE with derived datatypes.
-- Fix a problem with Fortran bindings that led to compilation errors
-  for user defined reduction operations.  Thanks to Nathan Weeks for
-  reporting.
-- Fix ROMIO issues with large writes/reads when using NFS file systems.
-- Fix definition of Fortran MPI_ARGV_NULL and MPI_ARGVS_NULL.
-- Enable use of the head node of a SLURM allocation on Cray XC systems.
-- Fix a problem with synchronous sends when using the UCX PML.
-- Use default socket buffer size to improve TCP BTL performance.
-- Add a mca parameter ras_base_launch_orted_on_hn to allow for launching
-  MPI processes on the same node where mpirun is executing using a separate
-  orte daemon, rather than the mpirun process.   This may be useful to set to
-  true when using SLURM, as it improves interoperability with SLURM's signal
-  propagation tools.  By default it is set to false, except for Cray XC systems.
-- Fix --without-lsf when lsf is installed in the default search path.
-- Remove support for big endian PowerPC.
-- Remove support for XL compilers older than v13.1
-- Remove IB XRC support from the OpenIB BTL due to loss of maintainer.
-
-2.1.1 -- April, 2017
---------------------
-
-Bug fixes/minor improvements:
-
-- Fix a problem with one of Open MPI's fifo data structures which led to
-  hangs in a make check test.  Thanks to Nicolas Morey-Chaisemartin for
-  reporting.
-- Add missing MPI_AINT_ADD/MPI_AINT_DIFF function definitions to mpif.h.
-  Thanks to Aboorva Devarajan for reporting.
-- Fix the error return from MPI_WIN_LOCK when rank argument is invalid.
-  Thanks to Jeff Hammond for reporting and fixing this issue.
-- Fix a problem with mpirun/orterun when started under a debugger. Thanks
-  to Gregory Leff for reporting.
-- Add configury option to disable use of CMA by the vader BTL.  Thanks
-  to Sascha Hunold for reporting.
-- Add configury check for MPI_DOUBLE_COMPLEX datatype support.
-  Thanks to Alexander Klein for reporting.
-- Fix memory allocated by MPI_WIN_ALLOCATE_SHARED to
-  be 64 bit aligned.  Thanks to Joseph Schuchart for
-  reporting.
-- Update MPI_WTICK man page to reflect possibly higher
-  resolution than 10e-6.  Thanks to Mark Dixon for
-  reporting
-- Add missing MPI_T_PVAR_SESSION_NULL definition to mpi.h
-  include file.  Thanks to Omri Mor for this contribution.
-- Enhance the Open MPI spec file to install modulefile in /opt
-  if installed in a non-default location.  Thanks to Kevin
-  Buckley for reporting and supplying a fix.
-- Fix a problem with conflicting PMI symbols when linking statically.
-  Thanks to Kilian Cavalotti for reporting.
-
-Known issues (to be addressed in v2.1.2):
-
-- See the list of fixes slated for v2.1.2 here:
-  https://github.com/open-mpi/ompi/milestone/28
-
-2.1.0 -- March, 2017
---------------------
-
-Major new features:
-
-- The main focus of the Open MPI v2.1.0 release was to update to PMIx
-  v1.2.1.  When using PMIx (e.g., via mpirun-based launches, or via
-  direct launches with recent versions of popular resource managers),
-  launch time scalability is improved, and the run time memory
-  footprint is greatly decreased when launching large numbers of MPI /
-  OpenSHMEM processes.
-- Update OpenSHMEM API conformance to v1.3.
-- The usnic BTL now supports MPI_THREAD_MULTIPLE.
-- General/overall performance improvements to MPI_THREAD_MULTIPLE.
-- Add a summary message at the bottom of configure that tells you many
-  of the configuration options specified and/or discovered by Open
-  MPI.
-
-Changes in behavior compared to prior versions:
-
-- None.
-
-Removed legacy support:
-
-- The ptmalloc2 hooks have been removed from the Open MPI code base.
-  This is not really a user-noticable change; it is only mentioned
-  here because there was much rejoycing in the Open MPI developer
-  community.
-
-Bug fixes/minor improvements:
-
-- New MCA parameters:
-  - iof_base_redirect_app_stderr_to_stdout: as its name implies, it
-    combines MPI / OpenSHMEM applications' stderr into its stdout
-    stream.
-  - opal_event_include: allow the user to specify which FD selection
-    mechanism is used by the underlying event engine.
-  - opal_stacktrace_output: indicate where stacktraces should be sent
-    upon MPI / OpenSHMEM process crashes ("none", "stdout", "stderr",
-    "file:filename").
-  - orte_timeout_for_stack_trace: number of seconds to wait for stack
-    traces to be reported (or <=0 to wait forever).
-  - mtl_ofi_control_prog_type/mtl_ofi_data_prog_type: specify libfabric
-    progress model to be used for control and data.
-- Fix MPI_WTICK regression where the time reported may be inaccurate
-  on systems with processor frequency scalaing enabled.
-- Fix regression that lowered the memory maximum message bandwidth for
-  large messages on some BTL network transports, such as openib, sm,
-  and vader.
-- Fix a name collision in the shared file pointer MPI IO file locking
-  scheme.  Thanks to Nicolas Joly for reporting the issue.
-- Fix datatype extent/offset errors in MPI_PUT and MPI_RACCUMULATE
-  when using the Portals 4 one-sided component.
-- Add support for non-contiguous datatypes to the Portals 4 one-sided
-  component.
-- Various updates for the UCX PML.
-- Updates to the following man pages:
-  - mpirun(1)
-  - MPI_COMM_CONNECT(3)
-  - MPI_WIN_GET_NAME(3). Thanks to Nicolas Joly for reporting the
-    typo.
-  - MPI_INFO_GET_[NKEYS|NTHKEY](3). Thanks to Nicolas Joly for
-    reporting the typo.
-- Fixed a problem in the TCP BTL when using MPI_THREAD_MULTIPLE.
-  Thanks to Evgueni Petrov for reporting.
-- Fixed external32 representation in the romio314 module.  Note that
-  for now, external32 representation is not correctly supported by the
-  ompio module.  Thanks to Thomas Gastine for bringing this to our
-  attention.
-- Add note how to disable a warning message about when a high-speed
-  MPI transport is not found.  Thanks to Susan Schwarz for reporting
-  the issue.
-- Ensure that sending SIGINT when using the rsh/ssh launcher does not
-  orphan children nodes in the launch tree.
-- Fix the help message when showing deprecated MCA param names to show
-  the correct (i.e., deprecated) name.
-- Enable support for the openib BTL to use multiple different
-  InfiniBand subnets.
-- Fix a minor error in MPI_AINT_DIFF.
-- Fix bugs with MPI_IN_PLACE handling in:
-  - MPI_ALLGATHER[V]
-  - MPI_[I][GATHER|SCATTER][V]
-  - MPI_IREDUCE[_SCATTER]
-  - Thanks to all the users who helped diagnose these issues.
-- Allow qrsh to tree spawn (if the back-end system supports it).
-- Fix MPI_T_PVAR_GET_INDEX to return the correct index.
-- Correctly position the shared file pointer in append mode in the
-  OMPIO component.
-- Add some deprecated names into shmem.h for backwards compatibility
-  with legacy codes.
-- Fix MPI_MODE_NOCHECK support.
-- Fix a regression in PowerPC atomics support.  Thanks to Orion
-  Poplawski for reporting the issue.
-- Fixes for assembly code with aggressively-optimized compilers on
-  x86_64/AMD64 platforms.
-- Fix one more place where configure was mangling custom CFLAGS.
-  Thanks to Phil Tooley (@Telemin) for reporting the issue.
-- Better handle builds with external installations of hwloc.
-- Fixed a hang with MPI_PUT and MPI_WIN_LOCK_ALL.
-- Fixed a bug when using MPI_GET on non-contiguous datatypes and
-  MPI_LOCK/MPI_UNLOCK.
-- Fixed a bug when using POST/START/COMPLETE/WAIT after a fence.
-- Fix configure portability by cleaning up a few uses of "==" with
-  "test".  Thanks to Kevin Buckley for pointing out the issue.
-- Fix bug when using darrays with lib and extent of darray datatypes.
-- Updates to make Open MPI binary builds more bit-for-bit
-  reproducable.  Thanks to Alastair McKinstry for the suggestion.
-- Fix issues regarding persistent request handling.
-- Ensure that shmemx.h is a standalone OpenSHMEM header file.  Thanks
-  to Nick Park (@nspark) for the report.
-- Ensure that we always send SIGTERM prior to SIGKILL.  Thanks to Noel
-  Rycroft for the report.
-- Added ConnectX-5 and Chelsio T6 device defaults for the openib BTL.
-- OpenSHMEM no longer supports MXM less than v2.0.
-- Plug a memory leak in ompi_osc_sm_free.  Thanks to Joseph Schuchart
-  for the report.
-- The "self" BTL now uses less memory.
-- The vader BTL is now more efficient in terms of memory usage when
-  using XPMEM.
-- Removed the --enable-openib-failover configure option.  This is not
-  considered backwards-incompatible because this option was stale and
-  had long-since stopped working, anyway.
-- Allow jobs launched under Cray aprun to use hyperthreads if
-  opal_hwloc_base_hwthreads_as_cpus MCA parameter is set.
-- Add support for 32-bit and floating point Cray Aries atomic
-  operations.
-- Add support for network AMOs for MPI_ACCUMULATE, MPI_FETCH_AND_OP,
-  and MPI_COMPARE_AND_SWAP if the "ompi_single_intrinsic" info key is
-  set on the window or the "acc_single_intrinsic" MCA param is set.
-- Automatically disqualify RDMA CM support in the openib BTL if
-  MPI_THREAD_MULTIPLE is used.
-- Make configure smarter/better about auto-detecting Linux CMA
-  support.
-- Improve the scalability of MPI_COMM_SPLIT_TYPE.
-- Fix the mixing of C99 and C++ header files with the MPI C++
-  bindings.  Thanks to Alastair McKinstry for the bug report.
-- Add support for ARM v8.
-- Several MCA parameters now directly support MPI_T enumerator
-  semantics (i.e., they accept a limited set of values -- e.g., MCA
-  parameters that accept boolean values).
-- Added --with-libmpi-name=STRING configure option for vendor releases
-  of Open MPI.  See the README for more detail.
-- Fix a problem with Open MPI's internal memory checker.  Thanks to Yvan
-  Fournier for reporting.
-- Fix a multi-threaded issue with MPI_WAIT.  Thanks to Pascal Deveze for
-  reporting.
-
-Known issues (to be addressed in v2.1.1):
-
-- See the list of fixes slated for v2.1.1 here:
-  https://github.com/open-mpi/ompi/milestone/26
-
-2.0.4 -- November, 2017
------------------------
-
-Bug fixes/minor improvements:
-- Fix an issue with visibility of functions defined in the built-in PMIx.
-  Thanks to Siegmar Gross for reporting this issue.
-- Add configure check to prevent trying to build this release of
-  Open MPI with an external hwloc 2.0 or newer release.
-- Add ability to specify layered providers for OFI MTL.
-- Fix a correctness issue with Open MPI's memory manager code
-  that could result in corrupted message data.  Thanks to
-  Valentin Petrov for reporting.
-- Fix issues encountered when using newer versions of PBS Pro.
-  Thanks to Petr Hanousek for reporting.
-- Fix a problem with MPI_GET when using the vader BTL.  Thanks
-  to Dahai Guo for reporting.
-- Fix a problem when using MPI_ANY_SOURCE with MPI_SENDRECV_REPLACE.
-  Thanks to Dahai Guo for reporting.
-- Fix a problem using MPI_FILE_OPEN with a communicator with an
-  attached cartesian topology.  Thanks to Wei-keng Liao for reporting.
-- Remove IB XRC support from the OpenIB BTL due to lack of support.
-- Remove support for big endian PowerPC.
-- Remove support for XL compilers older than v13.1
-
-2.0.3 -- June 2017
-------------------
-
-Bug fixes/minor improvements:
-
- - Fix a problem with MPI_IALLTOALLW when zero size messages are present.
-   Thanks to @mathbird for reporting.
- - Add missing MPI_USER_FUNCTION definition to the mpi_f08 module.
-   Thanks to Nathan Weeks for reporting this issue.
- - Fix a problem with MPI_WIN_LOCK not returning an error code when
-   a negative rank is supplied.  Thanks to Jeff Hammond for reporting and
-   providing a fix.
- - Fix a problem with make check that could lead to hangs.  Thanks to
-   Nicolas Morey-Chaisemartin for reporting.
- - Resolve a symbol conflict problem with PMI-1 and PMI-2 PMIx components.
-   Thanks to Kilian Cavalotti for reporting this issue.
- - Insure that memory allocations returned from MPI_WIN_ALLOCATE_SHARED are
-   64 byte aligned.  Thanks to Joseph Schuchart for reporting this issue.
- - Make use of DOUBLE_COMPLEX, if available, for Fortran bindings.  Thanks
-   to Alexander Klein for reporting this issue.
- - Add missing MPI_T_PVAR_SESSION_NULL definition to Open MPI mpi.h include
-   file.  Thanks to Omri Mor for reporting and fixing.
- - Fix a problem with use of MPI shared file pointers when accessing
-   a file from independent jobs.  Thanks to Nicolas Joly for reporting
-   this issue.
- - Optimize zero size MPI_IALLTOALL{V,W} with MPI_IN_PLACE.  Thanks to
-   Lisandro Dalcín for the report.
- - Fix a ROMIO buffer overflow problem for large transfers when using NFS
-   filesystems.
- - Fix type of MPI_ARGV[S]_NULL which prevented it from being used
-   properly with MPI_COMM_SPAWN[_MULTIPLE] in the mpi_f08 module.
- - Ensure to add proper linker flags to the wrapper compilers for
-   dynamic libraries on platforms that need it (e.g., RHEL 7.3 and
-   later).
- - Get better performance on TCP-based networks 10Gbps and higher by
-   using OS defaults for buffer sizing.
- - Fix a bug with MPI_[R][GET_]ACCUMULATE when using DARRAY datatypes.
- - Fix handling of --with-lustre configure command line argument.
-   Thanks to Prentice Bisbal and Tim Mattox for reporting the issue.
- - Added MPI_AINT_ADD and MPI_AINT_DIFF declarations to mpif.h.  Thanks
-   to Aboorva Devarajan (@AboorvaDevarajan) for the bug report.
- - Fix a problem in the TCP BTL when Open MPI is initialized with
-   MPI_THREAD_MULTIPLE support.  Thanks to Evgueni Petro for analyzing and
-   reporting this issue.
- - Fix yalla PML to properly handle underflow errors, and fixed a
-   memory leak with blocking non-contiguous sends.
- - Restored ability to run autogen.pl on official distribution tarballs
-   (although this is still not recommended for most users!).
- - Fix accuracy problems with MPI_WTIME on some systems by always using
-   either clock_gettime(3) or gettimeofday(3).
- - Fix a problem where MPI_WTICK was not returning a higher time resolution
-   when available.  Thanks to Mark Dixon for reporting this issue.
- - Restore SGE functionality.  Thanks to Kevin Buckley for the initial
-   report.
- - Fix external hwloc compilation issues, and extend support to allow
-   using external hwloc installations as far back as v1.5.0.  Thanks to
-   Orion Poplawski for raising the issue.
- - Added latest Mellanox Connect-X and Chelsio T-6 adapter part IDs to
-   the openib list of default values.
- - Do a better job of cleaning up session directories (e.g., in /tmp).
- - Update a help message to indicate how to suppress a warning about
-   no high performance networks being detected by Open MPI.  Thanks to
-   Susan Schwarz for reporting this issue.
- - Fix a problem with mangling of custom CFLAGS when configuring Open MPI.
-   Thanks to Phil Tooley for reporting.
- - Fix some minor memory leaks and remove some unused variables.
-   Thanks to Joshua Gerrard for reporting.
- - Fix MPI_ALLGATHERV bug with MPI_IN_PLACE.
-
-Known issues (to be addressed in v2.0.4):
-
-- See the list of fixes slated for v2.0.4 here:
-  https://github.com/open-mpi/ompi/milestone/29
-
-2.0.2 -- 26 January 2017
--------------------------
-
-Bug fixes/minor improvements:
-
-- Fix a problem with MPI_FILE_WRITE_SHARED when using MPI_MODE_APPEND and
-  Open MPI's native MPI-IO implementation.  Thanks to Nicolas Joly for
-  reporting.
-- Fix a typo in the MPI_WIN_GET_NAME man page.  Thanks to Nicolas Joly
-  for reporting.
-- Fix a race condition with ORTE's session directory setup.  Thanks to
-  @tbj900 for reporting this issue.
-- Fix a deadlock issue arising from Open MPI's approach to catching calls to
-  munmap. Thanks to Paul Hargrove for reporting and helping to analyze this
-  problem.
-- Fix a problem with PPC atomics which caused make check to fail unless builtin
-  atomics configure option was enabled.  Thanks to Orion Poplawski for reporting.
-- Fix a problem with use of x86_64 cpuid instruction which led to segmentation
-  faults when Open MPI was configured with -O3 optimization.  Thanks to Mark
-  Santcroos for reporting this problem.
-- Fix a problem when using built in atomics configure options on PPC platforms
-  when building 32 bit applications.  Thanks to Paul Hargrove for reporting.
-- Fix a problem with building Open MPI against an external hwloc installation.
-  Thanks to Orion Poplawski for reporting this issue.
-- Remove use of DATE in the message queue version string reported to debuggers to
-  insure bit-wise reproducibility of binaries.  Thanks to Alastair McKinstry
-  for help in fixing this problem.
-- Fix a problem with early exit of a MPI process without calling MPI_FINALIZE
-  or MPI_ABORT that could lead to job hangs.  Thanks to Christof Koehler for
-  reporting.
-- Fix a problem with forwarding of SIGTERM signal from mpirun to MPI processes
-  in a job.  Thanks to Noel Rycroft for reporting this problem
-- Plug some memory leaks in MPI_WIN_FREE discovered using Valgrind.  Thanks
-  to Joseph Schuchart for reporting.
-- Fix a problems  MPI_NEIGHOR_ALLTOALL when using a communicator with an empty topology
-  graph.  Thanks to Daniel Ibanez for reporting.
-- Fix a typo in a PMIx component help file.  Thanks to @njoly for reporting this.
-- Fix a problem with Valgrind false positives when using Open MPI's internal memchecker.
-  Thanks to Yvan Fournier for reporting.
-- Fix a problem with MPI_FILE_DELETE returning MPI_SUCCESS when
-  deleting a non-existent file. Thanks to Wei-keng Liao for reporting.
-- Fix a problem with MPI_IMPROBE that could lead to hangs in subsequent MPI
-  point to point or collective calls.  Thanks to Chris Pattison for reporting.
-- Fix a problem when configure Open MPI for powerpc with --enable-mpi-cxx
-  enabled.  Thanks to Alastair McKinstry for reporting.
-- Fix a problem using MPI_IALLTOALL with MPI_IN_PLACE argument.  Thanks to
-  Chris Ward for reporting.
-- Fix a problem using MPI_RACCUMULATE with the Portals4 transport.  Thanks to
-  @PDeveze for reporting.
-- Fix an issue with static linking and duplicate symbols arising from PMIx
-  Slurm components.  Thanks to Limin Gu for reporting.
-- Fix a problem when using MPI dynamic memory windows.  Thanks to
-  Christoph Niethammer for reporting.
-- Fix a problem with Open MPI's pkgconfig files.  Thanks to Alastair McKinstry
-  for reporting.
-- Fix a problem with MPI_IREDUCE when the same buffer is supplied for the
-  send and recv buffer arguments.  Thanks to Valentin Petrov for reporting.
-- Fix a problem with atomic operations on PowerPC.  Thanks to Paul
-  Hargrove for reporting.
-
-Known issues (to be addressed in v2.0.3):
-
-- See the list of fixes slated for v2.0.3 here:
-  https://github.com/open-mpi/ompi/milestone/23
-
-2.0.1 -- 2 September 2016
------------------------
-
-Bug fixes/minor improvements:
-
-- Short message latency and message rate performance improvements for
-  all transports.
-- Fix shared memory performance when using RDMA-capable networks.
-  Thanks to Tetsuya Mishima and Christoph Niethammer for reporting.
-- Fix bandwith performance degredation in the yalla (MXM) PML.  Thanks
-  to Andreas Kempf for reporting the issue.
-- Fix OpenSHMEM crash when running on non-Mellanox MXM-based networks.
-  Thanks to Debendra Das for reporting the issue.
-- Fix a crash occuring after repeated calls to MPI_FILE_SET_VIEW with
-  predefined datatypes.  Thanks to Eric Chamberland and Matthew
-  Knepley for reporting and helping chase down this issue.
-- Fix stdin propagation to MPI processes.  Thanks to Jingchao Zhang
-  for reporting the issue.
-- Fix various runtime and portability issues by updating the PMIx
-  internal component to v1.1.5.
-- Fix process startup failures on Intel MIC platforms due to very
-  large entries in /proc/mounts.
-- Fix a problem with use of relative path for specifing executables to
-  mpirun/oshrun.  Thanks to David Schneider for reporting.
-- Various improvements when running over portals-based networks.
-- Fix thread-based race conditions with GNI-based networks.
-- Fix a problem with MPI_FILE_CLOSE and MPI_FILE_SET_SIZE.  Thanks
-  to Cihan Altinay for reporting.
-- Remove all use of rand(3) from within Open MPI so as not to perturb
-  applications use of it.  Thanks to Matias Cabral and Noel Rycroft
-  for reporting.
-- Fix crash in MPI_COMM_SPAWN.
-- Fix types for MPI_UNWEIGHTED and MPI_WEIGHTS_EMPTY.  Thanks to
-  Lisandro Dalcín for reporting.
-- Correctly report the name of MPI_INTEGER16.
-- Add some missing MPI constants to the Fortran bindings.
-- Fixed compile error when configuring Open MPI with --enable-timing.
-- Correctly set the shared library version of libompitrace.so.  Thanks
-  to Alastair McKinstry for reporting.
-- Fix errors in the MPI_RPUT, MPI_RGET, MPI_RACCUMULATE, and
-  MPI_RGET_ACCUMULATE Fortran bindings.  Thanks to Alfio Lazzaro and
-  Joost VandeVondele for tracking this down.
-- Fix problems with use of derived datatypes in non-blocking
-  collectives.  Thanks to Yuki Matsumoto for reporting.
-- Fix problems with OpenSHMEM header files when using CMake.  Thanks to
-  Paul Kapinos for reporting the issue.
-- Fix problem with use use of non-zero lower bound datatypes in
-  collectives.  Thanks to Hristo Iliev for reporting.
-- Fix a problem with memory allocation within MPI_GROUP_INTERSECTION.
-  Thanks to Lisandro Dalcín for reporting.
-- Fix an issue with MPI_ALLGATHER for communicators that don't consist
-  of two ranks.  Thanks to David Love for reporting.
-- Various fixes for collectives when used with esoteric MPI datatypes.
-- Fixed corner cases of handling DARRAY and HINDEXED_BLOCK datatypes.
-- Fix a problem with filesystem type check for OpenBSD.
-  Thanks to Paul Hargrove for reporting.
-- Fix some debug input within Open MPI internal functions.  Thanks to
-  Durga Choudhury for reporting.
-- Fix a typo in a configury help message.  Thanks to Paul Hargrove for
-  reporting.
-- Correctly support MPI_IN_PLACE in MPI_[I]ALLTOALL[V|W] and
-  MPI_[I]EXSCAN.
-- Fix alignment issues on SPARC platforms.
-
-Known issues (to be addressed in v2.0.2):
-
-- See the list of fixes slated for v2.0.2 here:
-  https://github.com/open-mpi/ompi/milestone/20, and
-  https://github.com/open-mpi/ompi-release/milestone/19
-  (note that the "ompi-release" Github repo will be folded/absorbed
-  into the "ompi" Github repo at some point in the future)
-
-
-2.0.0 -- 12 July 2016
----------------------
-
- **********************************************************************
- *  Open MPI is now fully MPI-3.1 compliant
- **********************************************************************
-
-Major new features:
-
-- Many enhancements to MPI RMA.  Open MPI now maps MPI RMA operations
-  on to native RMA operations for those networks which support this
-  capability.
-- Greatly improved support for MPI_THREAD_MULTIPLE (when configured
-  with --enable-mpi-thread-multiple).
-- Enhancements to reduce the memory footprint for jobs at scale.  A
-  new MCA parameter, "mpi_add_procs_cutoff", is available to set the
-  threshold for using this feature.
-- Completely revamped support for memory registration hooks when using
-  OS-bypass network transports.
-- Significant OMPIO performance improvements and many bug fixes.
-- Add support for PMIx - Process Management Interface for Exascale.
-  Version 1.1.2 of PMIx is included internally in this release.
-- Add support for PLFS file systems in Open MPI I/O.
-- Add support for UCX transport.
-- Simplify build process for Cray XC systems.  Add support for
-  using native SLURM.
-- Add a --tune mpirun command line option to simplify setting many
-  environment variables and MCA parameters.
-- Add a new MCA parameter "orte_default_dash_host" to offer an analogue
-  to the existing "orte_default_hostfile" MCA parameter.
-- Add the ability to specify the number of desired slots in the mpirun
-  --host option.
-
-Changes in behavior compared to prior versions:
-
-- In environments where mpirun cannot automatically determine the
-  number of slots available (e.g., when using a hostfile that does not
-  specify "slots", or when using --host without specifying a ":N"
-  suffix to hostnames), mpirun now requires the use of "-np N" to
-  specify how many MPI processes to launch.
-- The MPI C++ bindings -- which were removed from the MPI standard in
-  v3.0 -- are no longer built by default and will be removed in some
-  future version of Open MPI.  Use the --enable-mpi-cxx-bindings
-  configure option to build the deprecated/removed MPI C++ bindings.
-- ompi_info now shows all components, even if they do not have MCA
-  parameters.  The prettyprint output now separates groups with a
-  dashed line.
-- OMPIO is now the default implementation of parallel I/O, with the
-  exception for Lustre parallel filesystems (where ROMIO is still the
-  default).  The default selection of OMPI vs. ROMIO can be controlled
-  via the "--mca io ompi|romio" command line switch to mpirun.
-- Per Open MPI's versioning scheme (see the README), increasing the
-  major version number to 2 indicates that this version is not
-  ABI-compatible with prior versions of Open MPI.  You will need to
-  recompile MPI and OpenSHMEM applications to work with this version
-  of Open MPI.
-- Removed checkpoint/restart code due to loss of maintainer. :-(
-- Change the behavior for handling certain signals when using PSM and
-  PSM2 libraries.  Previously, the PSM and PSM2 libraries would trap
-  certain signals in order to generate tracebacks.  The mechanism was
-  found to cause issues with Open MPI's own error reporting mechanism.
-  If not already set, Open MPI now sets the IPATH_NO_BACKTRACE and
-  HFI_NO_BACKTRACE environment variables to disable PSM/PSM2's
-  handling these signals.
-
-Removed legacy support:
-
-- Removed support for OS X Leopard.
-- Removed support for Cray XT systems.
-- Removed VampirTrace.
-- Removed support for Myrinet/MX.
-- Removed legacy collective module:ML.
-- Removed support for Alpha processors.
-- Removed --enable-mpi-profiling configure option.
-
-Known issues (to be addressed in v2.0.1):
-
-- See the list of fixes slated for v2.0.1 here:
-  https://github.com/open-mpi/ompi/milestone/16, and
-  https://github.com/open-mpi/ompi-release/milestone/16
-  (note that the "ompi-release" Github repo will be folded/absorbed
-  into the "ompi" Github repo at some point in the future)
-
-- ompi-release#986: Fix data size counter for large ops with fcoll/static
-- ompi-release#987: Fix OMPIO performance on Lustre
-- ompi-release#1013: Fix potential inconsistency in btl/openib default settings
-- ompi-release#1014: Do not return MPI_ERR_PENDING from collectives
-- ompi-release#1056: Remove dead profile code from oshmem
-- ompi-release#1081: Fix MPI_IN_PLACE checking for IALLTOALL{V|W}
-- ompi-release#1081: Fix memchecker in MPI_IALLTOALLW
-- ompi-release#1081: Support MPI_IN_PLACE in MPI_(I)ALLTOALLW and MPI_(I)EXSCAN
-- ompi-release#1107: Allow future PMIx support for RM spawn limits
-- ompi-release#1108: Fix sparse group process reference counting
-- ompi-release#1109: If specified to be oversubcribed, disable binding
-- ompi-release#1122: Allow NULL arrays for empty datatypes
-- ompi-release#1123: Fix signed vs. unsigned compiler warnings
-- ompi-release#1123: Make max hostname length uniform across code base
-- ompi-release#1127: Fix MPI_Compare_and_swap
-- ompi-release#1127: Fix MPI_Win_lock when used with MPI_Win_fence
-- ompi-release#1132: Fix typo in help message for --enable-mca-no-build
-- ompi-release#1154: Ensure pairwise coll algorithms disqualify themselves properly
-- ompi-release#1165: Fix typos in debugging/verbose message output
-- ompi-release#1178: Fix ROMIO filesystem check on OpenBSD 5.7
-- ompi-release#1197: Fix Fortran pthread configure check
-- ompi-release#1205: Allow using external PMIx 1.1.4 and 2.0
-- ompi-release#1215: Fix configure to support the NAG Fortran compiler
-- ompi-release#1220: Fix combiner args for MPI_HINDEXED_BLOCK
-- ompi-release#1225: Fix combiner args for MPI_DARRAY
-- ompi-release#1226: Disable old memory hooks with recent gcc versions
-- ompi-release#1231: Fix new "patcher" support for some XLC platforms
-- ompi-release#1244: Fix Java error handling
-- ompi-release#1250: Ensure TCP is not selected for RDMA operations
-- ompi-release#1252: Fix verbose output in coll selection
-- ompi-release#1253: Set a default name for user-defined MPI_Op
-- ompi-release#1254: Add count==0 checks in some non-blocking colls
-- ompi-release#1258: Fix "make distclean" when using external pmix/hwloc/libevent
-- ompi-release#1260: Clean up/uniform mca/coll/base memory management
-- ompi-release#1261: Remove "patcher" warning message for static builds
-- ompi-release#1263: Fix IO MPI_Request for 0-size read/write
-- ompi-release#1264: Add blocking fence for SLURM operations
-
-Bug fixes / minor enhancements:
-
-- Updated internal/embedded copies of third-party software:
-  - Update the internal copy of ROMIO to that which shipped in MPICH
-    3.1.4.
-  - Update internal copy of libevent to v2.0.22.
-  - Update internal copy of hwloc to v1.11.2.
-- Notable new MCA parameters:
-  - opal_progress_lp_call_ration: Control how often low-priority
-    callbacks are made during Open MPI's main progress loop.
-  - opal_common_verbs_want_fork_support: This replaces the
-    btl_openib_want_fork_support parameter.
-- Add --with-platform-patches-dir configure option.
-- Add --with-pmi-libdir configure option for environments that install
-  PMI libs in a non-default location.
-- Various configure-related compatibility updates for newer versions
-  of libibverbs and OFED.
-- Numerous fixes/improvements to orte-dvm.  Special thanks to Mark
-  Santcroos for his help.
-- Fix a problem with timer code on ia32 platforms.  Thanks to
-  Paul Hargrove for reporting this and providing a patch.
-- Fix a problem with use of a 64 bit atomic counter.  Thanks to
-  Paul Hargrove for reporting.
-- Fix a problem with singleton job launching.  Thanks to Lisandro
-  Dalcín for reporting.
-- Fix a problem with use of MPI_UNDEFINED with MPI_COMM_SPLIT_TYPE.
-  Thanks to Lisandro Dalcín for reporting.
-- Silence a compiler warning in PSM MTL.  Thanks to Adrian Reber for
-  reporting this.
-- Properly detect Intel TrueScale and OmniPath devices in the ACTIVE
-  state.  Thanks to Durga Choudhury for reporting the issue.
-- Fix detection and use of Solaris Studio 12.5 (beta) compilers.
-  Thanks to Paul Hargrove for reporting and debugging.
-- Fix various small memory leaks.
-- Allow NULL arrays when creating empty MPI datatypes.
-- Replace use of alloca with malloc for certain datatype creation
-  functions.  Thanks to Bogdan Sataric for reporting this.
-- Fix use of MPI_LB and MPI_UB in creation of of certain MPI datatypes.
-  Thanks to Gus Correa for helping to fix this.
-- Implement a workaround for a GNU Libtool problem.  Thanks to Eric
-  Schnetter for reporting and fixing.
-- Improve hcoll library detection in configure.  Thanks to David
-  Shrader and Åke Sandgren for reporting this.
-- Miscellaneous minor bug fixes in the hcoll component.
-- Miscellaneous minor bug fixes in the ugni component.
-- Fix problems with XRC detection in OFED 3.12 and older releases.
-  Thanks to Paul Hargrove for his analysis of this problem.
-- Update (non-standard/experimental) Java MPI interfaces to support
-  MPI-3.1 functionality.
-- Fix an issue with MCA parameters for Java bindings.  Thanks to
-  Takahiro Kawashima and Siegmar Gross for reporting this issue.
-- Fix a problem when using persistent requests in the Java bindings.
-  Thanks to Nate Chambers for reporting.
-- Fix problem with Java bindings on OX X 10.11.  Thanks to Alexander
-  Daryin for reporting this issue.
-- Fix a performance problem for large messages for Cray XC systems.
-  Thanks to Jerome Vienne for reporting this.
-- Fix an issue with MPI_WIN_LOCK_ALL.  Thanks to Thomas Jahns for
-  reporting.
-- Fix an issue with passing a parameter to configure multiple times.
-  Thanks to QuesarVII for reporting and supplying a fix.
-- Add support for ALPS resource allocation system on Cray CLE 5.2 and
-  later.  Thanks to Mark Santcroos.
-- Corrections to the HACKING file.  Thanks to Maximilien Levesque.
-- Fix an issue with user supplied reduction operator functions.
-  Thanks to Rupert Nash for reporting this.
-- Fix an issue with an internal list management function.  Thanks to
-  Adrian Reber for reporting this.
-- Fix a problem with MPI-RMA PSCW epochs.  Thanks to Berk Hess for
-  reporting this.
-- Fix a problem in neighborhood collectives.  Thanks to Lisandro
-  Dalcín for reporting.
-- Fix MPI_IREDUCE_SCATTER_BLOCK for a one-process communicator. Thanks
-  to Lisandro Dalcín for reporting.
-- Add (Open MPI-specific) additional flavors to MPI_COMM_SPLIT_TYPE.
-  See MPI_Comm_split_type(3) for details.  Thanks to Nick Andersen for
-  supplying this enhancement.
-- Improve closing of file descriptors during the job launch phase.
-  Thanks to Piotr Lesnicki for reporting and providing this
-  enhancement.
-- Fix a problem in MPI_GET_ACCUMULATE and MPI_RGET_ACCUMULATE when
-  using Portals4.  Thanks to Nicolas Chevalier for reporting.
-- Use correct include file for lstat prototype in ROMIO. Thanks to
-  William Throwe for finding and providing a fix.
-- Add missing Fortran bindings for MPI_WIN_ALLOCATE.  Thanks to Christoph
-  Niethammer for reporting and fixing.
-- Fortran related fixes to handle Intel 2016 compiler.  Thanks to
-  Fabrice Roy for reporting this.
-- Fix a Fortran linkage issue.  Thanks to Macro Atzeri for finding and
-  suggesting a fix.
-- Fix problem with using BIND(C) for Fortran bindings with logical
-  parameters.  Thanks to Paul Romano for reporting.
-- Fix an issue with use of DL-related macros in opal library.  Thanks to
-  Scott Atchley for finding this.
-- Fix an issue with parsing mpirun command line options which contain
-  colons.  Thanks to Lev Given for reporting.
-- Fix a problem with Open MPI's package configury files.  Thanks to
-  Christoph Junghans for reporting.
-- Fix a typo in the MPI_INTERCOMM_MERGE man page.  Thanks To Harald
-  Servat for reporting and correcting.
-- Update man pages for non-blocking sends per MPI 3.1 standard.
-  Thanks to Alexander Pozdneev for reporting.
-- Fix problem when compiling against PVFS2.  Thanks to Dave Love for
-  reporting.
-- Fix problems with MPI_NEIGHBOR_ALLTOALL{V,W}.  Thanks to Willem
-  Vermin for reporting this issue.
-- Fix various compilation problems on Cygwin.  Thanks to Marco Atzeri
-  for supplying these fixes.
-- Fix problem with resizing of subarray and darray data types.  Thanks
-  to Keith Bennett and Dan Garmann for reporting.
-- Fix a problem with MPI_COMBINER_RESIZED.  Thanks to James Ramsey for
-  the report.
-- Fix an hwloc binding issue.  Thanks to Ben Menadue for reporting.
-- Fix a problem with the shared memory (sm) BTL.  Thanks to Peter Wind
-  for the report.
-- Fixes for heterogeneous support. Thanks to Siegmar Gross for reporting.
-- Fix a problem with memchecker.  Thanks to Clinton Simpson for reporting.
-- Fix a problem with MPI_UNWEIGHTED in topology functions.  Thanks to
-  Jun Kudo for reporting.
-- Fix problem with a MCA parameter base filesystem types.  Thanks to
-  Siegmar Gross for reporting.
-- Fix a problem with some windows info argument types.  Thanks to
-  Alastair McKinstry for reporting.
-
-
-1.10.7 - 16 May 2017
-------
-- Fix bug in TCP BTL that impacted performance on 10GbE (and faster)
-  networks by not adjusting the TCP send/recv buffer sizes and using
-  system default values
-- Add missing MPI_AINT_ADD and MPI_AINT_DIFF function delcarations in
-  mpif.h
-- Fixed time reported by MPI_WTIME; it was previously reported as
-  dependent upon the CPU frequency.
-- Fix platform detection on FreeBSD
-- Fix a bug in the handling of MPI_TYPE_CREATE_DARRAY in
-  MPI_(R)(GET_)ACCUMULATE
-- Fix openib memory registration limit calculation
-- Add missing MPI_T_PVAR_SESSION_NULL in mpi.h
-- Fix "make distcheck" when using external hwloc and/or libevent packages
-- Add latest ConnectX-5 vendor part id to OpenIB device params
-- Fix race condition in the UCX PML
-- Fix signal handling for rsh launcher
-- Fix Fortran compilation errors by removing MPI_SIZEOF in the Fortran
-  interfaces when the compiler does not support it
-- Fixes for the pre-ignore-TKR "mpi" Fortran module implementation
-  (i.e., for older Fortran compilers -- these problems did not exist
-  in the "mpi" module implementation for modern Fortran compilers):
-  - Add PMPI_* interfaces
-  - Fix typo in MPI_FILE_WRITE_AT_ALL_BEGIN interface name
-  - Fix typo in MPI_FILE_READ_ORDERED_BEGIN interface name
-- Fixed the type of MPI_DISPLACEMENT_CURRENT in all Fortran interfaces
-  to be an INTEGER(KIND=MPI_OFFSET_KIND).
-- Fixed typos in MPI_INFO_GET_* man pages.  Thanks to Nicolas Joly for
-  the patch
-- Fix typo bugs in wrapper compiler script
-
-
-1.10.6 - 17 Feb 2017
-------
-- Fix bug in timer code that caused problems at optimization settings
-  greater than 2
-- OSHMEM: make mmap allocator the default instead of sysv or verbs
-- Support MPI_Dims_create with dimension zero
-- Update USNIC support
-- Prevent 64-bit overflow on timer counter
-- Add support for forwarding signals
-- Fix bug that caused truncated messages on large sends over TCP BTL
-- Fix potential infinite loop when printing a stacktrace
-
-
-1.10.5 - 19 Dec 2016
-------
-- Update UCX APIs
-- Fix bug in darray that caused MPI/IO failures
-- Use a MPI_Get_library_version() like string to tag the debugger DLL.
-  Thanks to Alastair McKinstry for the report
-- Fix multi-threaded race condition in coll/libnbc
-- Several fixes to OSHMEM
-- Fix bug in UCX support due to uninitialized field
-- Fix MPI_Ialltoallv with MPI_IN_PLACE and without MPI param check
-- Correctly reset receive request type before init. Thanks Chris Pattison
-  for the report and test case.
-- Fix bug in iallgather[v]
-- Fix concurrency issue with MPI_Comm_accept. Thanks to Pieter Noordhuis
-  for the patch
-- Fix ompi_coll_base_{gather,scatter}_intra_binomial
-- Fixed an issue with MPI_Type_get_extent returning the wrong extent
-  for distributed array datatypes.
-- Re-enable use of rtdtsc instruction as a monotonic clock source if
-  the processor has a core-invariant tsc. This is a partial fix for a
-  performance regression introduced in Open MPI v1.10.3.
-
-
-1.10.4 - 01 Sept 2016
-------
-
-- Fix assembler support for MIPS
-- Improve memory handling for temp buffers in collectives
-- Fix [all]reduce with non-zero lower bound datatypes
-  Thanks Hristo Iliev for the report
-- Fix non-standard ddt handling. Thanks Yuki Matsumoto for the report
-- Various libnbc fixes. Thanks Yuki Matsumoto for the report
-- Fix typos in request RMA bindings for Fortran. Thanks to @alazzaro
-  and @vondele for the assist
-- Various bug fixes and enhancements to collective support
-- Fix predefined types mapping in hcoll
-- Revive the coll/sync component to resolve unexpected message issues
-  during tight loops across collectives
-- Fix typo in wrapper compiler for Fortran static builds
-
-
-1.10.3 - 15 June 2016
-------
-
-- Fix zero-length datatypes.  Thanks to Wei-keng Liao for reporting
-  the issue.
-- Minor manpage cleanups
-- Implement atomic support in OSHMEM/UCX
-- Fix support of MPI_COMBINER_RESIZED. Thanks to James Ramsey
-  for the report
-- Fix computation of #cpus when --use-hwthread-cpus is used
-- Add entry points for Allgatherv, iAllgatherv, Reduce, and iReduce
-  for the HCOLL library
-- Fix an HCOLL integration bug that could signal completion of request
-  while still being worked
-- Fix computation of cores when SMT is enabled. Thanks to Ben Menadue
-  for the report
-- Various USNIC fixes
-- Create a datafile in the per-proc directory in order to make it
-  unique per communicator. Thanks to Peter Wind for the report
-- Fix zero-size malloc in one-sided pt-to-pt code. Thanks to Lisandro
-  Dalcín for the report
-- Fix MPI_Get_address when passed MPI_BOTTOM to not return an error.
-  Thanks to Lisandro Dalcín for the report
-- Fix MPI_TYPE_SET_ATTR with NULL value. Thanks to Lisandro Dalcín for
-  the report
-- Fix various Fortran08 binding issues
-- Fix memchecker no-data case. Thanks to Clinton Stimpson for the report
-- Fix CUDA support under OS-X
-- Fix various OFI/MTL integration issues
-- Add MPI_T man pages
-- Fix one-sided pt-to-pt issue by preventing communication from happening
-  before a target enters a fence, even in the no-precede case
-- Fix a bug that disabled Totalview for MPMD use-case
-- Correctly support MPI_UNWEIGHTED in topo-graph-neighbors. Thanks to
-  Jun Kudo for the report
-- Fix singleton operations under SLURM when PMI2 is enabled
-- Do not use MPI_IN_PLACE in neighborhood collectives for non-blocking
-  collectives (libnbc). Thanks to Jun Kudo for the report
-- Silence autogen deprecation warnings for newer versions of Perl
-- Do not return MPI_ERR_PENDING from collectives
-- Use type int* for MPI_WIN_DISP_UNIT, MPI_WIN_CREATE_FLAVOR, and MPI_WIN_MODEL.
-  Thanks to Alastair McKinstry for the report
-- Fix register_datarep stub function in IO/OMPIO. Thanks to Eric
-  Chamberland for the report
-- Fix a bus error on MPI_WIN_[POST,START] in the shared memory one-sided component
-- Add several missing MPI_WIN_FLAVOR constants to the Fortran support
-- Enable connecting processes from different subnets using the openib BTL
-- Fix bug in basic/barrier algorithm in OSHMEM
-- Correct process binding for the --map-by node case
-- Include support for subnet-to-subnet routing over InfiniBand networks
-- Fix usnic resource check
-- AUTHORS: Fix an errant reference to Subversion IDs
-- Fix affinity for MPMD jobs running under LSF
-- Fix many Fortran binding bugs
-- Fix `MPI_IN_PLACE`-related bugs
-- Fix PSM/PSM2 support for singleton operations
-- Ensure MPI transports continue to progress during RTE barriers
-- Update HWLOC to 1.9.1 end-of-series
-- Fix a bug in the Java command line parser when the
-  -Djava.library.path options was given by the user
-- Update the MTL/OFI provider selection behavior
-- Add support for clock_gettime on Linux.
-- Correctly detect and configure for Solaris Studio 12.5
-  beta compilers
-- Correctly compute #slots when -host is used for MPMD case
-- Fix a bug in the hcoll collectives due to an uninitialized field
-- Do not set a binding policy when oversubscribing a node
-- Fix hang in intercommunicator operations when oversubscribed
-- Speed up process termination during MPI_Abort
-- Disable backtrace support by default in the PSM/PSM2 libraries to
-  prevent unintentional conflicting behavior.
-
-
-
-1.10.2: 26 Jan 2016
--------------------
-
- **********************************************************************
- *  OSHMEM is now 1.2 compliant
- **********************************************************************
-
-- Fix NBC_Copy for legitimate zero-size messages
-- Fix multiple bugs in OSHMEM
-- Correctly handle mpirun --host <user>@<ip-address>
-- Centralize two MCA params to avoid duplication between OMPI and
-  OSHMEM layers: opal_abort_delay and opal_abort_print_stack
-- Add support for Fujitsu compilers
-- Add UCX support for OMPI and OSHMEM
-- Correctly handle oversubscription when not given directives
-  to permit it. Thanks to @ammore1 for reporting it
-- Fix rpm spec file to not include the /usr directory
-- Add Intel HFI1 default parameters for the openib BTL
-- Resolve symbol conflicts in the PSM2 library
-- Add ability to empty the rgpusm cache when full if requested
-- Fix another libtool bug when -L requires a space between it
-  and the path. Thanks to Eric Schnetter for the patch.
-- Add support for OSHMEM v1.2 APIs
-- Improve efficiency of oshmem_preconnect_all algorithm
-- Fix bug in buffered sends support
-- Fix double free in edge case of mpirun. Thanks to @jsharpe for
-  the patch
-- Multiple one-sided support fixes
-- Fix integer overflow in the tuned "reduce" collective when
-  using buffers larger than INT_MAX in size
-- Fix parse of user environment variables in mpirun. Thanks to
-  Stefano Garzarella for the patch
-- Performance improvements in PSM2 support
-- Fix NBS iBarrier for inter-communicators
-- Fix bug in vader BTL during finalize
-- Improved configure support for Fortran compilers
-- Fix rank_file mapper to support default --slot-set. Thanks
-  to Matt Thompson for reporting it
-- Update MPI_Testsome man page. Thanks to Eric Schnetter for
-  the suggestion
-- Fix missing resize of the returned type for subarray and
-  darray types. Thanks to Keith Bennett and Dan Garmann for
-  reporting it
-- Fix Java support on OSX 10.11. Thanks to Alexander Daryin
-  for reporting the problem
-- Fix some compilation issues on Solaris 11.2. Thanks to
-  Paul Hargrove for his continued help in such areas
-
-
-1.10.1: 4 Nov 2015
-------------------
-
-- Workaround an optimization problem with gcc compilers >= 4.9.2 that
-  causes problems with memory registration, and forced
-  mpi_leave_pinned to default to 0 (i.e., off).  Thanks to @oere for
-  the fix.
-- Fix use of MPI_LB and MPI_UB in subarray and darray datatypes.
-  Thanks to Gus Correa and Dimitar Pashov for pointing out the issue.
-- Minor updates to mpi_show_mpi_alloc_mem_leaks and
-  ompi_debug_show_handle_leaks functionality.
-- Fix segv when invoking non-blocking reductions with a user-defined
-  operation.  Thanks to Rupert Nash and Georg Geiser for identifying
-  the issue.
-- No longer probe for PCI topology on Solaris (unless running as root).
-- Fix for Intel Parallel Studio 2016 ifort partial support of the
-  !GCC$ pragma.  Thanks to Fabrice Roy for reporting the problem.
-- Bunches of Coverity / static analysis fixes.
-- Fixed ROMIO to look for lstat in <sys/stat.h>.  Thanks to William
-  Throwe for submitting the patch both upstream and to Open MPI.
-- Fixed minor memory leak when attempting to open plugins.
-- Fixed type in MPI_IBARRIER C prototype.  Thanks to Harald Servat for
-  reporting the issue.
-- Add missing man pages for MPI_WIN_CREATE_DYNAMIC, MPI_WIN_ATTACH,
-  MPI_WIN_DETACH, MPI_WIN_ALLOCATE, MPI_WIN_ALLOCATE_SHARED.
-- When mpirun-launching new applications, only close file descriptors
-  that are actually open (resulting in a faster launch in some
-  environments).
-- Fix "test ==" issues in Open MPI's configure script.  Thank to Kevin
-  Buckley for pointing out the issue.
-- Fix performance issue in usnic BTL: ensure progress thread is
-  throttled back to not aggressively steal CPU cycles.
-- Fix cache line size detection on POWER architectures.
-- Add missing #include in a few places.  Thanks to Orion Poplawski for
-  supplying the patch.
-- When OpenSHMEM building is disabled, no longer install its header
-  files, help files, or man pages.  Add man pages for oshrun, oshcc,
-  and oshfort.
-- Fix mpi_f08 implementations of MPI_COMM_SET_INFO, and profiling
-  versions of MPI_BUFFER_DETACH, MPI_WIN_ALLOCATE,
-  MPI_WIN_ALLOCATE_SHARED, MPI_WTICK, and MPI_WTIME.
-- Add orte_rmaps_dist_device MCA param, allowing users to map near a
-  specific device.
-- Various updates/fixes to the openib BTL.
-- Add missing defaults for the Mellanox ConnectX 3 card to the openib BTL.
-- Minor bug fixes in the OFI MTL.
-- Various updates to Mellanox's MXM, hcoll, and FCA components.
-- Add OpenSHMEM man pages.  Thanks to Tony Curtis for sharing the man
-  pages files from openshmem.org.
-- Add missing "const" attributes to MPI_COMPARE_AND_SWAP,
-  MPI_FETCH_AND_OP, MPI_RACCUMULATE, and MPI_WIN_DETACH prototypes.
-  Thanks to Michael Knobloch and Takahiro Kawashima for bringing this
-  to our attention.
-- Fix linking issues on some platforms (e.g., SLES 12).
-- Fix hang on some corner cases when MPI applications abort.
-- Add missing options to mpirun man page. Thanks to Daniel Letai
-  for bringing this to our attention.
-- Add new --with-platform-patches-dir configure option
-- Adjust relative selection priorities to ensure that MTL
-  support is favored over BTL support when both are available
-- Use CUDA IPC for all sized messages for performance
-
-
-1.10.0: 25 Aug 2015
--------------------
-
-** NOTE: The v1.10.0 release marks the transition to Open MPI's new
-** version numbering scheme.  The v1.10.x release series is based on
-** the v1.8.x series, but with a few new features.  v2.x will be the
-** next series after the v1.10.x series, and complete the transition
-** to the new version numbering scheme.  See README for more details
-** on the new versioning scheme.
-**
-** NOTE: In accordance with OMPI version numbering, the v1.10 is *not*
-** API compatible with the v1.8 release series.
-
-- Added libfabric support (see README for more details):
-  - usNIC BTL updated to use libfabric.
-  - Added OFI MTL (usable with PSM in libfabric v1.1.0).
-- Added Intel Omni-Path support via new PSM2 MTL.
-- Added "yalla" PML for faster MXM support.
-- Removed support for MX
-- Added persistent distributed virtual machine (pDVM) support for fast
-  workflow executions.
-- Fixed typo in GCC inline assembly introduced in Open MPI v1.8.8.
-  Thanks to Paul Hargrove for pointing out the issue.
-- Add missing man pages for MPI_Win_get|set_info(3).
-- Ensure that session directories are cleaned up at the end of a run.
-- Fixed linking issues on some OSs where symbols of dependent
-  libraries are not automatically publicly available.
-- Improve hcoll and fca configury library detection.  Thanks to David
-  Shrader for helping track down the issue.
-- Removed the LAMA mapper (for use in setting affinity).  Its
-  functionality has been largely superseded by other mpirun CLI
-  options.
-- CUDA: Made the asynchronous copy mode be the default.
-- Fix a malloc(0) warning in MPI_IREDUCE_SCATTER_BLOCK.  Thanks to
-  Lisandro Dalcín for reporting the issue.
-- Fix typo in MPI_Scatter(3) man page.  Thanks to Akshay Venkatesh for
-  noticing the mistake.
-- Add rudimentary protection from TCP port scanners.
-- Fix typo in Open MPI error handling.  Thanks to Åke Sandgren for
-  pointing out the error.
-- Increased the performance of the CM PML (i.e., the Portals, PSM,
-  PSM2, MXM, and OFI transports).
-- Restored visibility of blocking send requests in message queue
-  debuggers (e.g., TotalView, DDT).
-- Fixed obscure IPv6-related bug in the TCP BTL.
-- Add support for the "no_locks" MPI_Info key for one-sided
-  functionality.
-- Fixed ibv_fork support for verbs-based networks.
-- Fixed a variety of small bugs in OpenSHMEM.
-- Fixed MXM configure with additional CPPFLAGS and LDFLAGS.  Thanks to
-  David Shrader for the patch.
-- Fixed incorrect memalign threshhold in the openib BTL.  Thanks to
-  Xavier Besseron for pointing out the issue.
-
-
-1.8.8: 5 Aug 2015
------------------
-
-- Fix a segfault in MPI_FINALIZE with the PSM MTL.
-- Fix mpi_f08 sentinels (e.g., MPI_STATUS_IGNORE) handling.
-- Set some additional MXM default values for OSHMEM.
-- Fix an invalid memory access in MPI_MRECV and MPI_IMRECV.
-- Include two fixes that were mistakenly left out of the official
-  v1.8.7 tarball:
-  - Fixed MPI_WIN_POST and MPI_WIN_START for zero-size messages
-  - Protect the OOB TCP ports from segfaulting when accessed by port
-    scanners
-
-
-1.8.7: 15 Jul 2015
-------------------
-
-** NOTE: v1.8.7 technically breaks ABI with prior versions
-** in the 1.8 series because it repairs two incorrect API
-** signatures. However, users will only need to recompile
-** if they were using those functions - which they couldn't
-** have been, because the signatures were wrong :-)
-
-- Plugged a memory leak that impacted blocking sends
-- Fixed incorrect declaration for MPI_T_pvar_get_index and added
-  missing return code MPI_T_INVALID_NAME.
-- Fixed an uninitialized variable in PMI2 support
-- Added new vendor part id for Mellanox ConnectX4-LX
-- Fixed NBC_Copy for legitimate zero-size messages
-- Fixed MPI_Win_post and MPI_Win_start for zero-size messages
-- Protect the OOB ports from segfaulting when accessed by port scanners
-- Fixed several Fortran typos
-- Fixed configure detection of XRC support
-- Fixed support for highly heterogeneous systems to avoid
-  memory corruption when printing out the bindings
-
-1.8.6: 17 Jun 2015
-------------------
-
-- Fixed memory leak on Mac OS-X exposed by TCP keepalive
-- Fixed keepalive support to ensure that daemon/node failure
-  results in complete job cleanup
-- Update Java binding support
-- Fixed MPI_THREAD_MULTIPLE bug in vader shared memory BTL
-- Fixed issue during shutdown when CUDA initialization wasn't complete
-- Fixed orted environment when no prefix given
-- Fixed trivial typo in MPI_Neighbor_allgather manpage
-- Fixed tree-spawn support for sh and ksh shells
-- Several data type fixes
-- Fixed IPv6 support bug
-- Cleaned up an unlikely build issue
-- Fixed PMI2 process map parsing for cyclic mappings
-- Fixed memalign threshold in openib BTL
-- Fixed debugger access to message queues for blocking send/recv
-
-
-1.8.5: 5 May 2015
------------------
-
-- Fixed configure problems in some cases when using an external hwloc
-  installation.  Thanks to Erick Schnetter for reporting the error and
-  helping track down the source of the problem.
-- Fixed linker error on OS X when using the clang compiler.  Thanks to
-  Erick Schnetter for reporting the error and helping track down the
-  source of the problem.
-- Fixed MPI_THREAD_MULTIPLE deadlock error in the vader BTL.  Thanks
-  to Thomas Klimpel for reporting the issue.
-- Fixed several Valgrind warnings.  Thanks for Lisandro Dalcín for
-  contributing a patch fixing some one-sided code paths.
-- Fixed version compatibility test in OOB that broke ABI within the
-  1.8 series. NOTE: this will not resolve the problem between pre-1.8.5
-  versions, but will fix it going forward.
-- Fix some issues related to running on Intel Xeon Phi coprocessors.
-- Opportunistically switch away from using GNU Libtool's libltdl
-  library when possible (by default).
-- Fix some VampirTrace errors.  Thanks to Paul Hargrove for reporting
-  the issues.
-- Correct default binding patterns when --use-hwthread-cpus was
-  specified and nprocs <= 2.
-- Fix warnings about -finline-functions when compiling with clang.
-- Updated the embedded hwloc with several bug fixes, including the
-  "duplicate Lhwloc1 symbol" that multiple users reported on some
-  platforms.
-- Do not error when mpirun is invoked with with default bindings
-  (i.e., no binding was specified), and one or more nodes do not
-  support bindings.  Thanks to Annu Desari for pointing out the
-  problem.
-- Let root invoke "mpirun --version" to check the version without
-  printing the "Don't run as root!" warnings.  Thanks to Robert McLay
-  for the suggestion.
-- Fixed several bugs in OpenSHMEM support.
-- Extended vader shared memory support to 32-bit architectures.
-- Fix handling of very large datatypes.  Thanks to Bogdan Sataric for
-  the bug report.
-- Fixed a bug in handling subarray MPI datatypes, and a bug when using
-  MPI_LB and MPI_UB.  Thanks to Gus Correa for pointing out the issue.
-- Restore user-settable bandwidth and latency PML MCA variables.
-- Multiple bug fixes for cleanup during MPI_FINALIZE in unusual
-  situations.
-- Added support for TCP keepalive signals to ensure timely termination
-  when sockets between daemons cannot be created (e.g., due to a
-  firewall).
-- Added MCA parameter to allow full use of a SLURM allocation when
-  started from a tool (supports LLNL debugger).
-- Fixed several bugs in the configure logic for PMI and hwloc.
-- Fixed incorrect interface index in TCP communications setup.  Thanks
-  to Mark Kettenis for spotting the problem and providing a patch.
-- Fixed MPI_IREDUCE_SCATTER with single-process communicators when
-  MPI_IN_PLACE was not used.
-- Added XRC support for OFED v3.12 and higher.
-- Various updates and bug fixes to the Mellanox hcoll collective
-  support.
-- Fix problems with Fortran compilers that did not support
-  REAL*16/COMPLEX*32 types.  Thanks to Orion Poplawski for identifying
-  the issue.
-- Fixed problem with rpath/runpath support in pkg-config files.
-  Thanks to Christoph Junghans for notifying us of the issue.
-- Man page fixes:
-  - Removed erroneous "color" discussion from MPI_COMM_SPLIT_TYPE.
-    Thanks to Erick Schnetter for spotting the outdated text.
-  - Fixed prototypes for MPI_IBARRIER.  Thanks to Maximilian for
-    finding the issue.
-  - Updated docs about buffer usage in non-blocking communications.
-    Thanks to Alexander Pozdneev for citing the outdated text.
-  - Added documentation about the 'ompi_unique' MPI_Info key with
-    MPI_PUBLISH_NAME.
-  - Fixed typo in MPI_INTERCOMM_MERGE.  Thanks to Harald Servat for
-    noticing and sending a patch.
-  - Updated configure paths in HACKING.  Thanks to Maximilien Levesque
-    for the fix.
-  - Fixed Fortran typo in MPI_WIN_LOCK_ALL.  Thanks to Thomas Jahns
-    for pointing out the issue.
-- Fixed a number of MPI one-sided bugs.
-- Fixed MPI_COMM_SPAWN when invoked from a singleton job.
-- Fixed a number of minor issues with CUDA support, including
-  registering of shared memory and supporting reduction support for
-  GPU buffers.
-- Improved support for building OMPI on Cray platforms.
-- Fixed performance regression introduced by the inadvertent default
-  enabling of MPI_THREAD_MULTIPLE support.
-
-
-1.8.4: 19 Dec 2014
-------------------
-
-- Fix MPI_SIZEOF; now available in mpif.h for modern Fortran compilers
-  (see README for more details).  Also fixed various compiler/linker
-  errors.
-- Fixed inadvertant Fortran ABI break between v1.8.1 and v1.8.2 in the
-  mpi interface module when compiled with gfortran >= v4.9.
-- Fix various MPI_THREAD_MULTIPLE issues in the TCP BTL.
-- mpirun no longer requires the --hetero-nodes switch; it will
-  automatically detect when running in heterogeneous scenarios.
-- Update LSF support, to include revamped affinity functionality.
-- Update embedded hwloc to v1.9.1.
-- Fixed max registerable memory computation in the openib BTL.
-- Updated error message when debuggers are unable to find various
-  symbols/types to be more clear.  Thanks to Dave Love for raising the
-  issue.
-- Added proper support for LSF and PBS/Torque libraries in static builds.
-- Rankfiles now support physical processor IDs.
-- Fixed potential hang in MPI_ABORT.
-- Fixed problems with the PSM MTL and "re-connect" scenarios, such as
-  MPI_INTERCOMM_CREATE.
-- Fix MPI_IREDUCE_SCATTER with a single process.
-- Fix (rare) race condition in stdout/stderr funneling to mpirun where
-  some trailing output could get lost when a process terminated.
-- Removed inadvertent change that set --enable-mpi-thread-multiple "on"
-  by default, thus impacting performance for non-threaded apps.
-- Significantly reduced startup time by optimizing internal hash table
-  implementation.
-- Fixed OS X linking with the Fortran mpi module when used with
-  gfortran >= 4.9.  Thanks to Github user yafshar for raising the
-  issue.
-- Fixed memory leak on Cygwin platforms.  Thanks for Marco Atzeri for
-  reporting the issue.
-- Fixed seg fault in neighborhood collectives when the degree of the
-  topology is higher than the communicator size.  Thanks to Lisandro
-  Dalcín for reporting the issue.
-- Fixed segfault in neighborhood collectives under certain use-cases.
-- Fixed various issues regarding Solaris support.  Thanks to Siegmar
-  Gross for patiently identifying all the issues.
-- Fixed PMI configure tests for certain Slurm installation patterns.
-- Fixed param registration issue in Java bindings.  Thanks to Takahiro
-  Kawashima and Siegmar Gross for identifying the issue.
-- Several man page fixes.
-- Silence several warnings and close some memory leaks (more remain,
-  but it's better than it was).
-- Re-enabled the use of CMA and knem in the shared memory BTL.
-- Updated mpirun manpage to correctly explain new map/rank/binding options.
-- Fixed MPI_IALLGATHER problem with intercommunicators.  Thanks for
-  Takahiro Kawashima for the patch.
-- Numerous updates and performance improvements to OpenSHMEM.
-- Turned off message coalescing in the openib BTL until a proper fix
-  for that capability can be provided (tentatively expected for 1.8.5)
-- Fix a bug in iof output that dates back to the dinosaurs which would
-  output extra bytes if the system was very heavily loaded
-- Fix a bug where specifying mca_component_show_load_errors=0 could
-  cause ompi_info to segfault
-- Updated valgrind suppression file
-
-
-1.8.3: 26 Sep 2014
-------------------
-
-- Fixed application abort bug to ensure that MPI_Abort exits appropriately
-  and returns the provided exit status
-- Fixed some alignment (not all) issues identified by Clang
-- Allow CUDA-aware to work with nonblocking collectives. Forces packing to
-  happen when using GPU buffers.
-- Fixed configure test issue with Intel 2015 Fortran compiler
-- Fixed some PGI-related errors
-- Provide better help message when encountering a firewall
-- Fixed MCA parameter quoting to protect multi-word params and params
-  that contain special characters
-- Improved the bind-to help message to clarify the defaults
-- Add new MPI-3.1 tools interface
-- Several performance optimizations and memory leak cleanups
-- Turn off the coll/ml plugin unless specifically requested as it
-  remains in an experimental state
-- Fix LSF support by adding required libraries for the latest LSF
-  releases.  Thanks to Joshua Randal for supplying the initial
-  patches.
-
-
-1.8.2: 25 Aug 2014
-------------------
-
-- Fix auto-wireup of OOB, allowing ORTE to automatically
-  test all available NICs
-- "Un-deprecate" pernode, npernode, and npersocket options
-  by popular demand
-- Add missing Fortran bindings for MPI_WIN_LOCK_ALL,
-  MPI_WIN_UNLOCK_ALL, and MPI_WIN_SYNC.
-- Fix cascading/over-quoting in some cases with the rsh/ssh-based
-  launcher.  Thanks to multiple users for raising the issue.
-- Properly add support for gfortran 4.9 ignore TKR pragma (it was
-  erroneously only partially added in v1.7.5).  Thanks to Marcus
-  Daniels for raising the issue.
-- Update/improve help messages in the usnic BTL.
-- Resolve a race condition in MPI_Abort.
-- Fix obscure cases where static linking from wrapper compilers would
-  fail.
-- Clarify the configure --help message about when OpenSHMEM is
-  enabled/disabled by default.  Thanks to Paul Hargrove for the
-  suggestion.
-- Align pages properly where relevant.  Thanks to Paul Hargrove for
-  identifying the issue.
-- Various compiler warning and minor fixes for OpenBSD, FreeBSD, and
-  Solaris/SPARC.  Thanks to Paul Hargrove for the patches.
-- Properly pass function pointers from Fortran to C in the mpi_f08
-  module, thereby now supporting gfortran 4.9.  Thanks to Tobias
-  Burnus for assistance and testing with this issue.
-- Improve support for Cray CLE 5.
-- Fix mpirun regression: ensure exit status is non-zero if mpirun is
-  terminated due to signal.
-- Improved CUDA efficiency of asynchronous copies.
-- Fix to parameter type in MPI_Type_indexed.3.  Thanks to Bastian
-  Beischer for reporting the mistake.
-- Fix NUMA distance calculations in the openib BTL.
-- Decrease time required to shut down mpirun at the end of a job.
-- More RMA fixes.
-- More hostfile fixes from Tetsuya Mishima.
-- Fix darray issue where UB was not computed correctly.
-- Fix mpi_f08 parameter name for MPI_GET_LIBRARY_VERSION.  Thanks to
-  Junchao Zhang for pointing out the issue.
-- Ensure mpirun aborts properly when unable to map processes in
-  scheduled environments.
-- Ensure that MPI RMA error codes show up properly.  Thanks to
-  Lisandro Dalcín for reporting the issue.
-- Minor bug fixes and improvements to the bash and zsh mpirun
-  autocompletion scripts.
-- Fix sequential mpirun process mapper.  Thanks to Bill Chen for
-  reporting the issue.
-- Correct SLURM stdout/stderr redirection.
-- Added missing portals 4 files.
-- Performance improvements for blocking sends and receives.
-- Lots of cleanup to the ml collective component
-- Added new Java methods to provide full MPI coverage
-- Many OSHMEM cleanups
-- Prevent comm_spawn from automatically launching a VM across
-  all available nodes
-- Close many memory leaks to achieve valgrind-clean operation
-- Better handling of TCP connection discovery for mismatched networks
-  where we don't have a direct 1:1 subnet match between nodes
-- Prevent segfault when OMPI info tools are used in pipes and user
-  exits one step of that pipe before completing output
-
-
-1.8.1: 23 Apr 2014
-------------------
-
-- Fix for critical bug: mpirun removed files (but not directories)
-  from / when run as root.  Thanks to Jay Fenlason and Orion Poplawski
-  for bringing the issue to our attention and helping identify the
-  fix.
-
-
-1.8: 31 Mar 2014
-----------------
-
-- Commit upstream ROMIO fix for mixed NFS+local filesystem environments.
-- Several fixes for MPI-3 one-sided support.  For example,
-  arbitrary-length datatypes are now supported.
-- Add config support for the Mellanox ConnectX 4 card.
-- Add missing MPI_COMM_GET|SET_INFO functions, and missing
-  MPI_WEIGHTS_EMPTY and MPI_ERR_RMA_SHARED constants.  Thanks to
-  Lisandro Dalcín for pointing out the issue.
-- Update some help messages in OSHMEM, the usnic BTL, the TCP BTL, and
-  ORTE, and update documentation about ompi_info's --level option.
-- Fix some compiler warnings.
-- Ensure that ORTE daemons are not bound to a single processor
-  if TaskAffinity is set on by default in Slurm. Thanks to Artem Polyakov
-  for identifying the problem and providing a patch
-
-
-1.7.5 20 Mar 2014
------------------
-
- **********************************************************************
- *  Open MPI is now fully MPI-3.0 compliant
- **********************************************************************
-
-- Add Linux OpenSHMEM support built on top of Open MPI's MPI
-  layer. Thanks to Mellanox for contributing this new feature.
-- Allow restricting ORTE daemons to specific cores using the
-  orte_daemon_cores MCA param.
-- Ensure to properly set "locality" flags for processes launched via
-  MPI dynamic functions such as MPI_COMM_SPAWN.
-- Fix MPI_GRAPH_CREATE when nnodes is smaller than the size of the old
-  communicator.
-- usnic BTL now supports underlying UDP transport.
-- usnic BTL now checks for common connectivty errors at first send to
-  a remote server.
-- Minor scalability improvements in the usnic BTL.
-- ompi_info now lists whether the Java MPI bindings are available or not.
-- MPI-3: mpi.h and the Fortran interfaces now report MPI_VERSION==3
-  and MPI_SUBVERSION==0.
-- MPI-3: Added support for new RMA functions and functionality.
-- Fix MPI_Info "const buglet.  Thanks to Orion Poplawski for
-  identifying the issue.
-- Multiple fixes to mapping/binding options. Thanks to Tetsuya Mishima
-  for his assistance.
-- Multiple fixes for normal and abnormal process termination,
-  including singleton MPI_Abort and ensuring to kill entire process
-  groups when abnormally terminating a job.
-- Fix DESTDIR install for javadocs.  Thanks to Orion Poplawski for
-  pointing out the issue.
-- Various performance improvements for the MPI Java bindings.
-- OMPI now uses its own internal random number generator and will not
-  perturb srand() and friends.
-- Some cleanups for Cygwin builds.  Thanks to Marco Atzeri for the
-  patches.
-- Add a new collective component (coll/ml) that provides substantially
-  improved performance.  It is still experimental, and requires
-  setting coll_ml_priority > 0 to become active.
-- Add version check during startup to ensure you are using the same
-  version of Open MPI on all nodes in a job.
-- Significantly improved the performance of MPI_DIMS_CREATE for large
-  values.  Thanks to Andreas Schäfer for the contribution.
-- Removed ASYNCHRONOUS keyword from the "ignore TKR" mpi_f08 module.
-- Deprecated the following mpirun options:
-  --bynode, --bycore, --byslot: replaced with --map-by node|core|slot.
-  --npernode, --npersocket: replaced with --map-by ppr:N:node and
-       --map-by ppr:N:socket, respectively
-- Pick NFS "infinitely stale" fix from ROMIO upstream.
-- Various PMI2 fixes and extension to support broader range of mappings.
-- Improve launch performance at large scale.
-- Add support for PBS/Torque environments that set environment
-  variables to indicate the number of slots available on each nodes.
-  Set the ras_tm_smp MCA parameter to "1" to enable this mode.
-- Add new, more scalable endpoint exchange (commonly called "modex")
-  method that only exchanges endpoint data on a per-peer basis
-  on first message. Not all transports have been updated to use
-  this feature. Set the rte_orte_direct_modex parameter to "1"
-  to enable this mode.
-
-
-1.7.4: 5 Feb 2014
------------------
-
- **********************************************************************
- *      CRITICAL CHANGE
- *
- * As of release 1.7.4, OpenMPI's default mapping, ranking, and binding
- * settings have changed:
- *
- * Mapping:
- *   if #procs <= 2, default to map-by core
- *   if #procs > 2, default to map-by socket
- * Ranking:
- *   if default mapping is used, then default to rank-by slot
- *   if map-by <obj> is given, then default to rank-by <obj>,
- *      where <obj> is whatever object we mapped against
- * Binding:
- *   default to bind-to core
- *
- * Users can override any of these settings individually using the
- * corresponding MCA parameter. Note that multi-threaded applications
- * in particular may want to override at least the binding default
- * to allow threads to use multiple cores.
- **********************************************************************
-
-- Restore version number output in "ompi_info --all".
-- Various bug fixes for the mpi_f08 Fortran bindings.
-- Fix ROMIO compile error with Lustre 2.4.  Thanks to Adam Moody for
-  reporting the issue.
-- Various fixes for 32 bit platforms.
-- Add ability to selectively disable building the mpi or mpi_f08
-  module.  See the README file for details.
-- Fix MX MTL finalization issue.
-- Fix ROMIO issue when opening a file with MPI_MODE_EXCL.
-- Fix PowerPC and MIPS assembly issues.
-- Various fixes to the hcoll and FCA collective offload modules.
-- Prevent integer overflow when creating datatypes.  Thanks to
-  original patch from Gilles Gouaillardet.
-- Port some upstream hwloc fixes to Open MPI's embedded copy for
-  working around buggy NUMA node cpusets and including mising header
-  files.  Thanks to Jeff Becker and Paul Hargrove for reporting the
-  issues.
-- Fix recursive invocation issues in the MXM MTL.
-- Various bug fixes to the new MCA parameter back-end system.
-- Have the posix fbtl module link against -laio on NetBSD platforms.
-  Thanks to Paul Hargrove for noticing the issue.
-- Various updates and fixes to network filesystem detection to support
-  more operating systems.
-- Add gfortran v4.9 "ignore TKR" syntax to the mpi Fortran module.
-- Various compiler fixes for several BSD-based platforms.  Thanks to
-  Paul Hargrove for reporting the issues.
-- Fix when MPI_COMM_SPAWN[_MULTIPLE] is used on oversubscribed
-  systems.
-- Change the output from --report bindings to simply state that a
-  process is not bound, instead of reporting that it is bound to all
-  processors.
-- Per MPI-3.0 guidance, remove support for all MPI subroutines with
-  choice buffers from the TKR-based mpi Fortran module.  Thanks to Jed
-  Brown for raising the issue.
-- Only allow the usnic BTL to build on 64 bit platforms.
-- Various bug fixes to SLURM support, to include ensuring proper
-  exiting on abnormal termination.
-- Ensure that MPI_COMM_SPAWN[_MULTIPLE] jobs get the same mapping
-  directives that were used with mpirun.
-- Fixed the application of TCP_NODELAY.
-- Change the TCP BTL to not warn if a non-existent interface is
-  ignored.
-- Restored the "--bycore" mpirun option for backwards compatibility.
-- Fixed debugger attach functionality.  Thanks to Ashley Pittman for
-  reporting the issue and suggesting the fix.
-- Fixed faulty MPI_IBCAST when invoked on a communicator with only
-  one process.
-- Add new Mellanox device IDs to the openib BTL.
-- Progress towards cleaning up various internal memory leaks as
-  reported by Valgrind.
-- Fixed some annoying flex-generated warnings that have been there for
-  years.  Thanks to Tom Fogal for the initial patch.
-- Support user-provided environment variables via the "env" info key
-  to MPI_COMM_SPAWN[_MULTIPLE].  Thanks to Tom Fogal for the feature
-  request.
-- Fix uninitialized variable in MPI_DIST_GRAPH_CREATE.
-- Fix a variety of memory errors on SPARC platforms.  Thanks to
-  Siegmar Gross for reporting and testing all the issues.
-- Remove Solaris threads support.  When building on Solaris, pthreads
-  will be used.
-- Correctly handle the convertor internal stack for persistent
-  receives.  Thanks to Guillaume Gouaillardet for identifying the
-  problem.
-- Add support for using an external libevent via --with-libevent.  See
-  the README for more details.
-- Various OMPIO updates and fixes.
-- Add support for the MPIEXEC_TIMEOUT environment variable.  If set,
-  mpirun will terminate the job after this many seconds.
-- Update the internal copy of ROMIO to that which shipped in MPICH
-  3.0.4.
-- Various performance tweaks and improvements in the usnic BTL,
-  including now reporting MPI_T performance variables for each usnic
-  device.
-- Fix to not access send datatypes for non-root processes with
-  MPI_ISCATTER[V] and MPI_IGATHER[V].  Thanks to Pierre Jolivet for
-  supplying the initial patch.
-- Update VampirTrace to 5.14.4.9.
-- Fix ptmalloc2 hook disable when used with ummunotify.
-- Change the default connection manager for the openib BTL to be based
-  on UD verbs data exchanges instead of ORTE OOB data exchanges.
-- Fix Fortran compile error when compiling with 8-byte INTEGERs and
-  4-byte ints.
-- Fix C++11 issue identified by Jeremiah Willcock.
-- Many changes, updates, and bug fixes to the ORTE run-time layer.
-- Correctly handle MPI_REDUCE_SCATTER with recvcounts of 0.
-- Update man pages for MPI-3, and add some missing man pages for
-  MPI-2.x functions.
-- Updated mpi_f08 module in accordance with post-MPI-3.0 errata which
-  basically removed BIND(C) from all interfaces.
-- Fixed MPI_IN_PLACE detection for MPI_SCATTER[V] in Fortran
-  routines.  Thanks to Charles Gerlach for identifying the issue.
-- Added support for routable RoCE to the openib BTL.
-- Update embedded hwloc to v1.7.2.
-- ErrMgr framework redesigned to better support fault tolerance development
-  activities. See the following RFC for details:
-  https://www.open-mpi.org/community/lists/devel/2010/03/7589.php
-- Added database framework to OPAL and changed all modex operations
-  to flow thru it, also included additional system info in the
-  available data
-- Added staged state machine to support sequential work flows
-- Added distributed file system support for accessing files across
-  nodes that do not have networked file systems
-- Extended filem framework to support scalable pre-positioning of
-  files for use by applications, adding new "raw" component that
-  transmits files across the daemon network
-- Native Windows support has been removed. A cygwin package is
-  available from that group for Windows-based use.
-- Added new MPI Java bindings.  See the Javadocs for more details on
-  the API.
-- Wrapper compilers now add rpath support by default to generated
-  executables on systems that support it.  This behavior can be
-  disabled via --disable-wrapper-rpath.  See note in README about ABI
-  issues when using rpath in MPI applications.
-- Added a new parallel I/O component and multiple new frameworks to
-  support parallel I/O operations.
-- Fixed MPI_STATUS_SIZE Fortran issue when used with 8-byte Fortran
-  INTEGERs and 4-byte C ints.  Since this issue affects ABI, it is
-  only enabled if Open MPI is configured with
-  --enable-abi-breaking-fortran-status-i8-fix.  Thanks to Jim Parker
-  for supplying the initial patch.
-- Add support for Intel Phi SCIF transport.
-- For CUDA-aware MPI configured with CUDA 6.0, use new pointer
-  attribute to avoid extra synchronization in stream 0 when using
-  CUDA IPC between GPUs on the same node.
-- For CUDA-aware MPI configured with CUDA 6.0, compile in support
-  of GPU Direct RDMA in openib BTL to improve small message latency.
-- Updated ROMIO from MPICH v3.0.4.
-- MPI-3: Added support for remaining non-blocking collectives.
-- MPI-3: Added support for neighborhood collectives.
-- MPI-3: Updated C bindings with consistent use of [].
-- MPI-3: Added the const keyword to read-only buffers.
-- MPI-3: Added support for non-blocking communicator duplication.
-- MPI-3: Added support for non-collective communicator creation.
-
-
-1.7.3: 17 Oct 2013
-------------------
-
-- Make CUDA-aware support dynamically load libcuda.so so CUDA-aware
-  MPI library can run on systems without CUDA software.
-- Fix various issues with dynamic processes and intercommunicator
-  operations under Torque.  Thanks to Suraj Prabhakaran for reporting
-  the problem.
-- Enable support for the Mellanox MXM2 library by default.
-- Improve support for Portals 4.
-- Various Solaris fixes.  Many thanks to Siegmar Gross for his
-  incredible patience in reporting all the issues.
-- MPI-2.2: Add reduction support for MPI_C_*COMPLEX and MPI::*COMPLEX.
-- Fixed internal accounting when openpty() fails.  Thanks to Michal
-  Peclo for reporting the issue and providing a patch.
-- Fixed too-large memory consumption in XRC mode of the openib BTL.
-  Thanks to Alexey Ryzhikh for the patch.
-- Add bozo check for negative np values to mpirun to prevent a
-  deadlock.  Thanks to Upinder Malhi for identifying the issue.
-- Fixed MPI_IS_THREAD_MAIN behavior.  Thanks to Lisandro Dalcín for
-  pointing out the problem.
-- Various rankfile fixes.
-- Fix functionality over iWARP devices.
-- Various memory and performance optimizations and tweaks.
-- Fix MPI_Cancel issue identified by Fujitsu.
-- Add missing support for MPI_Get_address in the "use mpi" TKR
-  implementation.  Thanks to Hugo Gagnon for identifying the issue.
-- MPI-3: Add support for MPI_Count.
-- MPI-2.2: Add missing MPI_IN_PLACE support for MPI_ALLTOALL.
-- Added new usnic BTL to support the Cisco usNIC device.
-- Minor VampirTrace update to 5.14.4.4.
-- Removed support for ancient OS X systems (i.e., prior to 10.5).
-- Fixed obscure packing/unpacking datatype bug.  Thanks to Takahiro
-  Kawashima for identifying the issue.
-- Add run-time support for PMI2 environments.
-- Update openib BTL default parameters to include support for Mellanox
-  ConnectX3-Pro devices.
-- Update libevent to v2.0.21.
-- "ompi_info --param TYPE PLUGIN" now only shows a small number of MCA
-  parameters by default.  Add "--level 9" or "--all" to see *all* MCA
-  parameters.  See README for more details.
-- Add support for asynchronous CUDA-aware copies.
-- Add support for Mellanox MPI collective operation offload via the
-  "hcoll" library.
-- MPI-3: Add support for the MPI_T interface.  Open MPI's MCA
-  parameters are now accessible via the MPI_T control variable
-  interface.  Support has been added for a small number of MPI_T
-  performance variables.
-- Add Gentoo memory hooks override.  Thanks to Justin Bronder for the
-  patch.
-- Added new "mindist" process mapper, allowing placement of processes
-  via PCI locality information reported by the BIOS.
-- MPI-2.2: Add support for MPI_Dist_graph functionality.
-- Enable generic, client-side support for PMI2 implementations. Can
-  be leveraged by any resource manager that implements PMI2; e.g. SLURM,
-  versions 2.6 and higher.
-
-1.7.2: 26 Jun 2013
-------------------
-
-- Major VampirTrace update to 5.14.4.2.
-  (** also appeared: 1.6.5)
-- Fix to set flag==1 when MPI_IPROBE is called with MPI_PROC_NULL.
-  (** also appeared: 1.6.5)
-- Set the Intel Phi device to be ignored by default by the openib BTL.
-  (** also appeared: 1.6.5)
-- Decrease the internal memory storage used by intrinsic MPI datatypes
-  for Fortran types.  Thanks to Takahiro Kawashima for the initial
-  patch.
-  (** also appeared: 1.6.5)
-- Fix total registered memory calculation for Mellanox ConnectIB and
-  OFED 2.0.
-  (** also appeared: 1.6.5)
-- Fix possible data corruption in the MXM MTL component.
-  (** also appeared: 1.6.5)
-- Remove extraneous -L from hwloc's embedding.  Thanks to Stefan
-  Friedel for reporting the issue.
-  (** also appeared: 1.6.5)
-- Fix contiguous datatype memory check.  Thanks to Eric Chamberland
-  for reporting the issue.
-  (** also appeared: 1.6.5)
-- Make the openib BTL more friendly to ignoring verbs devices that are
-  not RC-capable.
-  (** also appeared: 1.6.5)
-- Fix some MPI datatype engine issues.  Thanks to Thomas Jahns for
-  reporting the issue.
-  (** also appeared: 1.6.5)
-- Add INI information for Chelsio T5 device.
-  (** also appeared: 1.6.5)
-- Integrate MXM STREAM support for MPI_ISEND and MPI_IRECV, and other
-  minor MXM fixes.
-  (** also appeared: 1.6.5)
-- Fix to not show amorphous "MPI was already finalized" error when
-  failing to MPI_File_close an open file.  Thanks to Brian Smith for
-  reporting the issue.
-  (** also appeared: 1.6.5)
-- Add a distance-based mapping component to find the socket "closest"
-  to the PCI bus.
-- Fix an error that caused epoll to automatically be disabled
-  in libevent.
-- Upgrade hwloc to 1.5.2.
-- *Really* fixed XRC compile issue in Open Fabrics support.
-- Fix MXM connection establishment flow.
-- Fixed parallel debugger ability to attach to MPI jobs.
-- Fixed some minor memory leaks.
-- Fixed datatype corruption issue when combining datatypes of specific
-  formats.
-- Added Location Aware Mapping Algorithm (LAMA) mapping component.
-- Fixes for MPI_STATUS handling in corner cases.
-- Add a distance-based mapping component to find the socket "closest"
-  to the PCI bus.
-
-
-1.7.1: 16 Apr 2013
-------------------
-
-- Fixed compile error when --without-memory-manager was specified
-  on Linux
-- Fixed XRC compile issue in Open Fabrics support.
-
-
-1.7: 1 Apr 2013
----------------
-
-- Added MPI-3 functionality:
-    - MPI_GET_LIBRARY_VERSION
-    - Matched probe
-    - MPI_TYPE_CREATE_HINDEXED_BLOCK
-    - Non-blocking collectives
-    - MPI_INFO_ENV support
-    - Fortran '08 bindings (see below)
-- Dropped support for checkpoint/restart due to loss of maintainer :-(
-- Enabled compile-time warning of deprecated MPI functions by default
-  (in supported compilers).
-- Revamped Fortran MPI bindings (see the README for details):
-  - "mpifort" is now the preferred wrapper compiler for Fortran
-  - Added "use mpi_f08" bindings (for compilers that support it)
-  - Added better "use mpi" support (for compilers that support it)
-  - Removed incorrect MPI_SCATTERV interface from "mpi" module that
-    was added in the 1.5.x series for ABI reasons.
-- Lots of VampirTrace upgrades and fixes; upgrade to v5.14.3.
-- Modified process affinity system to provide warning when bindings
-  result in being "bound to all", which is equivalent to not being
-  bound.
-- Removed maffinity, paffinity, and carto frameworks (and associated
-  MCA params).
-- Upgraded to hwloc v1.5.1.
-- Added performance improvements to the OpenIB (OpenFabrics) BTL.
-- Made malloc hooks more friendly to IO interprosers.  Thanks to the
-  bug report and suggested fix from Darshan maintainer Phil Carns.
-- Added support for the DMTCP checkpoint/restart system.
-- Added support for the Cray uGNI interconnect.
-- Fixed header file problems on OpenBSD.
-- Fixed issue with MPI_TYPE_CREATE_F90_REAL.
-- Wrapper compilers now explicitly list/link all Open MPI libraries if
-  they detect static linking CLI arguments.
-- Open MPI now requires a C99 compiler to build.  Please upgrade your
-  C compiler if you do not have a C99-compliant compiler.
-- Fix MPI_GET_PROCESSOR_NAME Fortran binding to set ierr properly.
-  Thanks to LANL for spotting the error.
-- Many MXM and FCA updates.
-- Fixed erroneous free of putenv'ed string that showed up in Valgrind
-  reports.
-- Fixed MPI_IN_PLACE case for MPI_ALLGATHER.
-- Fixed a bug that prevented MCA params from being forwarded to
-  daemons upon launch.
-- Fixed issues with VT and CUDA --with-cuda[-libdir] configuration CLI
-  parameters.
-- Entirely new implementation of many MPI collective routines focused
-  on better performance.
-- Revamped autogen / build system.
-- Add new sensor framework to ORTE that includes modules for detecting
-  stalled applications and processes that consume too much memory.
-- Added new state machine framework to ORTE that converts ORTE into an
-  event-driven state machine using the event library.
-- Added a new MCA parameter (ess_base_stream_buffering) that allows the user
-  to override the system default for buffering of stdout/stderr streams
-  (via setvbuf). Parameter is not visible via ompi_info.
-- Revamped the launch system to allow consideration of node hardware
-  in assigning process locations and bindings.
-- Added the -novm option to preserve the prior launch behavior.
-- Revamped the process mapping system to utilize node hardware by adding
-  new map-by, rank-by, and bind-to cmd line options.
-- Added new MCA parameter to provide protection against IO forwarding
-  backlog.
-- Dropped support for native Windows due to loss of maintainers. :-(
-- Added a new parallel I/O component and multiple new frameworks to
-  support parallel I/O operations.
-- Fix typo in orte_setup_hadoop.m4. Thanks to Aleksej Saushev for
-  reporting it
-- Fix a very old error in opal_path_access(). Thanks to Marco Atzeri
-  for chasing it down.
-
-
-1.6.6: Not released
--------------------
-
-- Prevent integer overflow in datatype creation.  Thanks to Gilles
-  Gouaillardet for identifying the problem and providing a preliminary
-  version of the patch.
-- Ensure help-opal-hwloc-base.txt is included in distribution
-  tarballs.  Thanks to Gilles Gouaillardet for supplying the patch.
-- Correctly handle the invalid status for NULL and inactive requests.
-  Thanks to KAWASHIMA Takahiro for submitting the initial patch.
-- Fixed MPI_STATUS_SIZE Fortran issue when used with 8-byte Fortran
-  INTEGERs and 4-byte C ints.  Since this issue affects ABI, it is
-  only enabled if Open MPI is configured with
-  --enable-abi-breaking-fortran-status-i8-fix.  Thanks to Jim Parker
-  for supplying the initial patch.
-- Fix datatype issue for sending from the middle of non-contiguous
-  data.
-- Fixed failure error with pty support.  Thanks to Michal Pecio for
-  the patch.
-- Fixed debugger support for direct-launched jobs.
-- Fix MPI_IS_THREAD_MAIN to return the correct value.  Thanks to
-  Lisandro Dalcín for pointing out the issue.
-- Update VT to 5.14.4.4:
-  - Fix C++-11 issue.
-  - Fix support for building RPMs on Fedora with CUDA libraries.
-- Add openib part number for ConnectX3-Pro HCA.
-- Ensure to check that all resolved IP addresses are local.
-- Fix MPI_COMM_SPAWN via rsh when mpirun is on a different server.
-- Add Gentoo "sandbox" memory hooks override.
-
-
-1.6.5: 26 Jun 2013
-------------------
-
-- Updated default SRQ parameters for the openib BTL.
-  (** also to appear: 1.7.2)
-- Major VampirTrace update to 5.14.4.2.
-  (** also to appear: 1.7.2)
-- Fix to set flag==1 when MPI_IPROBE is called with MPI_PROC_NULL.
-  (** also to appear: 1.7.2)
-- Set the Intel Phi device to be ignored by default by the openib BTL.
-  (** also to appear: 1.7.2)
-- Decrease the internal memory storage used by intrinsic MPI datatypes
-  for Fortran types.  Thanks to Takahiro Kawashima for the initial
-  patch.
-  (** also to appear: 1.7.2)
-- Fix total registered memory calculation for Mellanox ConnectIB and
-  OFED 2.0.
-  (** also to appear: 1.7.2)
-- Fix possible data corruption in the MXM MTL component.
-  (** also to appear: 1.7.2)
-- Remove extraneous -L from hwloc's embedding.  Thanks to Stefan
-  Friedel for reporting the issue.
-  (** also to appear: 1.7.2)
-- Fix contiguous datatype memory check.  Thanks to Eric Chamberland
-  for reporting the issue.
-  (** also to appear: 1.7.2)
-- Make the openib BTL more friendly to ignoring verbs devices that are
-  not RC-capable.
-  (** also to appear: 1.7.2)
-- Fix some MPI datatype engine issues.  Thanks to Thomas Jahns for
-  reporting the issue.
-  (** also to appear: 1.7.2)
-- Add INI information for Chelsio T5 device.
-  (** also to appear: 1.7.2)
-- Integrate MXM STREAM support for MPI_ISEND and MPI_IRECV, and other
-  minor MXM fixes.
-  (** also to appear: 1.7.2)
-- Improved alignment for OpenFabrics buffers.
-- Fix to not show amorphous "MPI was already finalized" error when
-  failing to MPI_File_close an open file.  Thanks to Brian Smith for
-  reporting the issue.
-  (** also to appear: 1.7.2)
-
-
-1.6.4: 21 Feb 2013
-------------------
-
-- Fix Cygwin shared memory and debugger plugin support.  Thanks to
-  Marco Atzeri for reporting the issue and providing initial patches.
-- Fix to obtaining the correct available nodes when a rankfile is
-  providing the allocation.  Thanks to Siegmar Gross for reporting the
-  problem.
-- Fix process binding issue on Solaris.  Thanks to Siegmar Gross for
-  reporting the problem.
-- Updates for MXM 2.0.
-- Major VT update to 5.14.2.3.
-- Fixed F77 constants for Cygwin/Cmake build.
-- Fix a linker error when configuring --without-hwloc.
-- Automatically provide compiler flags that compile properly on some
-  types of ARM systems.
-- Fix slot_list behavior when multiple sockets are specified.  Thanks
-  to Siegmar Gross for reporting the problem.
-- Fixed memory leak in one-sided operations.  Thanks to Victor
-  Vysotskiy for letting us know about this one.
-- Added performance improvements to the OpenIB (OpenFabrics) BTL.
-- Improved error message when process affinity fails.
-- Fixed MPI_MINLOC on man pages for MPI_REDUCE(_LOCAL).  Thanks to Jed
-  Brown for noticing the problem and supplying a fix.
-- Made malloc hooks more friendly to IO interprosers.  Thanks to the
-  bug report and suggested fix from Darshan maintainer Phil Carns.
-- Restored ability to direct launch under SLURM without PMI support.
-- Fixed MPI datatype issues on OpenBSD.
-- Major VT update to 5.14.2.3.
-- Support FCA v3.0+.
-- Fixed header file problems on OpenBSD.
-- Fixed issue with MPI_TYPE_CREATE_F90_REAL.
-- Fix an issue with using external libltdl installations.  Thanks to
-  opolawski for identifying the problem.
-- Fixed MPI_IN_PLACE case for MPI_ALLGATHER for FCA.
-- Allow SLURM PMI support to look in lib64 directories.  Thanks to
-  Guillaume Papaure for the patch.
-- Restore "use mpi" ABI compatibility with the rest of the 1.5/1.6
-  series (except for v1.6.3, where it was accidentally broken).
-- Fix a very old error in opal_path_access(). Thanks to Marco Atzeri
-  for chasing it down.
-
-
-1.6.3: 30 Oct 2012
-------------------
-
-- Fix mpirun --launch-agent behavior when a prefix is specified.
-  Thanks to Reuti for identifying the issue.
-- Fixed memchecker configury.
-- Brought over some compiler warning squashes from the development trunk.
-- Fix spawning from a singleton to multiple hosts when the "add-host"
-  MPI_Info key is used.  Thanks to Brian Budge for pointing out the
-  problem.
-- Add Mellanox ConnextIB IDs and max inline value.
-- Fix rankfile when no -np is given.
-- FreeBSD detection improvement.  Thanks to Brooks Davis for the
-  patch.
-- Removed TCP warnings on Windows.
-- Improved collective algorithm selection for very large messages.
-- Fix PSM MTL affinity settings.
-- Fix issue with MPI_OP_COMMUTATIVE in the mpif.h bindings.  Thanks to
-  Åke Sandgren for providing a patch to fix the issue.
-- Fix issue with MPI_SIZEOF when using CHARACTER and LOGICAL types in
-  the mpi module.  Thanks to Åke Sandgren for providing a patch to fix
-  the issue.
-
-
-1.6.2: 25 Sep 2012
-------------------
-
-- Fix issue with MX MTL.  Thanks to Doug Eadline for raising the issue.
-- Fix singleton MPI_COMM_SPAWN when the result job spans multiple nodes.
-- Fix MXM hang, and update for latest version of MXM.
-- Update to support Mellanox FCA 2.5.
-- Fix startup hang for large jobs.
-- Ensure MPI_TESTANY / MPI_WAITANY properly set the empty status when
-  count==0.
-- Fix MPI_CART_SUB behavior of not copying periods to the new
-  communicator properly.  Thanks to John Craske for the bug report.
-- Add btl_openib_abort_not_enough_reg_mem MCA parameter to cause Open
-  MPI to abort MPI jobs if there is not enough registered memory
-  available on the system (vs. just printing a warning).  Thanks to
-  Brock Palen for raising the issue.
-- Minor fix to Fortran MPI_INFO_GET: only copy a value back to the
-  user's buffer if the flag is .TRUE.
-- Fix VampirTrace compilation issue with the PGI compiler suite.
-
-
-1.6.1: 22 Aug 2012
-------------------
-
-- A bunch of changes to eliminate hangs on OpenFabrics-based networks.
-  Users with Mellanox hardware are ***STRONGLY ENCOURAGED*** to check
-  their registered memory kernel module settings to ensure that the OS
-  will allow registering more than 8GB of memory.  See this FAQ item
-  for details:
-
-  https://www.open-mpi.org/faq/?category=openfabrics#ib-low-reg-mem
-
-  - Fall back to send/receive semantics if registered memory is
-    unavilable for RDMA.
-  - Fix two fragment leaks when registered memory is exhausted.
-  - Hueristically determine how much registered memory is available
-    and warn if it's significantly less than all of RAM.
-  - Artifically limit the amount of registered memory each MPI process
-    can use to about 1/Nth to total registered memory available.
-  - Improve error messages when events occur that are likely due to
-    unexpected registered memory exhaustion.
-
-- Fix double semicolon error in the C++ in <mpi.h>.  Thanks to John
-  Foster for pointing out the issue.
-- Allow -Xclang to be specified multiple times in CFLAGS.  Thanks to
-  P. Martin for raising the issue.
-- Break up a giant "print *" statement in the ABI-preserving incorrect
-  MPI_SCATTER interface in the "large" Fortran "mpi" module.  Thanks
-  to Juan Escobar for the initial patch.
-- Switch the MPI_ALLTOALLV default algorithm to a pairwise exchange.
-- Increase the openib BTL default CQ length to handle more types of
-  OpenFabrics devices.
-- Lots of VampirTrace fixes; upgrade to v5.13.0.4.
-- Map MPI_2INTEGER to underlying MPI_INTEGERs, not MPI_INTs.
-- Ensure that the OMPI version number is toleant of handling spaces.
-  Thanks to dragonboy for identifying the issue.
-- Fixed IN parameter marking on Fortran "mpi" module
-  MPI_COMM_TEST_INTER interface.
-- Various MXM improvements.
-- Make the output of "mpirun --report-bindings" much more friendly /
-  human-readable.
-- Properly handle MPI_COMPLEX8|16|32.
-- More fixes for mpirun's processor affinity options (--bind-to-core
-  and friends).
-- Use aligned memory for OpenFabrics registered memory.
-- Multiple fixes for parameter checking in MPI_ALLGATHERV,
-  MPI_REDUCE_SCATTER, MPI_SCATTERV, and MPI_GATHERV.  Thanks to the
-  mpi4py community (Bennet Fauber, Lisandro Dalcín, Jonathan Dursi).
-- Fixed file positioning overflows in MPI_FILE_GET_POSITION,
-  MPI_FILE_GET_POSITION_SHARED, FILE_GET_SIZE, FILE_GET_VIEW.
-- Removed the broken --cpu-set mpirun option.
-- Fix cleanup of MPI errorcodes.  Thanks to Alexey Bayduraev for the
-  patch.
-- Fix default hostfile location.  Thanks to Götz Waschk for noticing
-  the issue.
-- Improve several error messages.
-
-
-1.6: 14 May 2012
-----------------
-
-- Fix some process affinity issues.  When binding a process, Open MPI
-  will now bind to all available hyperthreads in a core (or socket,
-  depending on the binding options specified).
-  --> Note that "mpirun --bind-to-socket ..." does not work on POWER6-
-      and POWER7-based systems with some Linux kernel versions.  See
-      the FAQ on the Open MPI web site for more information.
-- Add support for ARM5 and ARM6 (in addition to the existing ARM7
-  support).  Thanks to Evan Clinton for the patch.
-- Minor Mellanox MXM fixes.
-- Properly detect FDR10, FDR, and EDR OpenFabrics devices.
-- Minor fixes to the mpirun(1) and MPI_Comm_create(3) man pages.
-- Prevent segv if COMM_SPAWN_MULTIPLE fails.  Thanks to Fujitsu for
-  the patch.
-- Disable interposed memory management in fakeroot environments.  This
-  fixes a problem in some build environments.
-- Minor hwloc updates.
-- Array versions of MPI_TEST and MPI_WAIT with a count==0 will now
-  return immediately with MPI_SUCCESS.  Thanks to Jeremiah Willcock
-  for the suggestion.
-- Update VampirTrace to v5.12.2.
-- Properly handle forwarding stdin to all processes when "mpirun
-  --stdin all" is used.
-- Workaround XLC assembly bug.
-- OS X Tiger (10.4) has not been supported for a while, so forcibly
-  abort configure if we detect it.
-- Fix segv in the openib BTL when running on SPARC 64 systems.
-- Fix some include file ordering issues on some BSD-based platforms.
-  Thanks to Paul Hargove for this (and many, many other) fixes.
-- Properly handle .FALSE. return parameter value to attribute copy
-  callback functions.
-- Fix a bunch of minor C++ API issues; thanks to Fujitsu for the patch.
-- Fixed the default hostfile MCA parameter behavior.
-- Per the MPI spec, ensure not to touch the port_name parameter to
-  MPI_CLOSE_PORT (it's an IN parameter).
-
-
-1.5.5: 27 Mar 2012
-------------------
-
-- Many, many portability configure/build fixes courtesy of Paul
-  Hargrove.  Thanks, Paul!
-- Fixed shared memory fault tolerance support compiler errors.
-- Removed not-production-quality rshd and tmd PLM launchers.
-- Minor updates to the Open MPI SRPM spec file.
-- Fixed mpirun's --bind-to-socket option.
-- A few MPI_THREAD_MULTIPLE fixes in the shared memory BTL.
-- Upgrade the GNU Autotools used to bootstrap the 1.5/1.6 series to
-  all the latest versions at the time of this release.
-- Categorically state in the README that if you're having a problem
-  with Open MPI with the Linux Intel 12.1 compilers, *upgrade your
-  Intel Compiler Suite to the latest patch version*, and the problems
-  will go away. :-)
-- Fix the --without-memory-manager configure option.
-- Fixes for Totalview/DDT MPI-capable debuggers.
-- Update rsh/ssh support to properly handle the Mac OS X library path
-  (i.e., DYLD_LIBRARY_PATH).
-- Make warning about shared memory backing files on a networked file
-  system be optional (i.e., can be disabled via MCA parameter).
-- Several fixes to processor and memory affinity.
-- Various shared memory infrastructure improvements.
-- Various checkpoint/restart fixes.
-- Fix MPI_IN_PLACE (and other MPI sentinel values) on OS X.  Thanks to
-  Dave Goodell for providing the magic OS X gcc linker flags necessary.
-- Various man page corrections and typo fixes.  Thanks to Fujitsu for
-  the patch.
-- Updated wrapper compiler man pages to list the various --showme
-  options that are available.
-- Add PMI direct-launch support (e.g., "srun mpi_application" under
-  SLURM).
-- Correctly compute the aligned address when packing the
-  datatype description. Thanks to Fujitsu for the patch.
-- Fix MPI obscure corner case handling in packing MPI datatypes.
-  Thanks to Fujitsu for providing the patch.
-- Workaround an Intel compiler v12.1.0 2011.6.233 vector optimization
-  bug.
-- Output the MPI API in ompi_info output.
-- Major VT update to 5.12.1.4.
-- Upgrade embedded Hardware Locality (hwloc) v1.3.2, plus some
-  post-1.3.2-release bug fixes.  All processor and memory binding is
-  now done through hwloc.  Woo hoo!  Note that this fixes core binding
-  on AMD Opteron 6200 and 4200 series-based systems (sometimes known
-  as Interlagos, Valencia, or other Bulldozer-based chips).
-- New MCA parameters to control process-wide memory binding policy:
-  hwloc_base_mem_alloc_policy, hwloc_base_mem_bind_failure_action (see
-  ompi_info --param hwloc base).
-- Removed direct support for libnuma.  Libnuma support may now be
-  picked up through hwloc.
-- Added MPI_IN_PLACE support to MPI_EXSCAN.
-- Various fixes for building on Windows, including MinGW support.
-- Removed support for the OpenFabrics IBCM connection manager.
-- Updated Chelsio T4 and Intel NE OpenFabrics default buffer settings.
-- Increased the default RDMA CM timeout to 30 seconds.
-- Issue a warning if both btl_tcp_if_include and btl_tcp_if_exclude
-  are specified.
-- Many fixes to the Mellanox MXM transport.
-
-
-1.5.4: 18 Aug 2011
-------------------
-
-- Add support for the (as yet unreleased) Mellanox MXM transport.
-- Add support for dynamic service levels (SLs) in the openib BTL.
-- Fixed C++ bindings cosmetic/warnings issue with
-  MPI::Comm::NULL_COPY_FN and MPI::Comm::NULL_DELETE_FN.  Thanks to
-  Júlio Hoffimann for identifying the issues.
-- Also allow the word "slots" in rankfiles (i.e., not just "slot").
-  (** also to appear in 1.4.4)
-- Add Mellanox ConnectX 3 device IDs to the openib BTL defaults.
-  (** also to appear in 1.4.4)
-- Various FCA updates.
-- Fix 32 bit SIGBUS errors on Solaris SPARC platforms.
-- Add missing ARM assembly code files.
-- Update to allow more than 128 entries in an appfile.
-  (** also to appear in 1.4.4)
-- Various VT updates and bug fixes.
-- Update description of btl_openib_cq_size to be more accurate.
-  (** also to appear in 1.4.4)
-- Various assembly "clobber" fixes.
-- Fix a hang in carto selection in obscure situations.
-- Guard the inclusion of execinfo.h since not all platforms have it.  Thanks
-  to Aleksej Saushev for identifying this issue.
-  (** also to appear in 1.4.4)
-- Support Solaris legacy munmap prototype changes.
-  (** also to appear in 1.4.4)
-- Updated to Automake 1.11.1 per
-  https://www.open-mpi.org/community/lists/devel/2011/07/9492.php.
-- Fix compilation of LSF support.
-- Update MPI_Comm_spawn_multiple.3 man page to reflect what it
-  actually does.
-- Fix for possible corruption of the environment.  Thanks to Peter
-  Thompson for the suggestion.  (** also to appear in 1.4.4)
-- Enable use of PSM on direct-launch SLURM jobs.
-- Update paffinity hwloc to v1.2, and to fix minor bugs affinity
-  assignment bugs on PPC64/Linux platforms.
-- Let the openib BTL auto-detect its bandwidth.
-- Support new MPI-2.2 datatypes.
-- Updates to support more datatypes in MPI one-sided communication.
-- Fix recursive locking bug when MPI-IO was used with
-  MPI_THREAD_MULTIPLE.  (** also to appear in 1.4.4)
-- Fix mpirun handling of prefix conflicts.
-- Ensure mpirun's --xterm options leaves sessions attached.
-  (** also to appear in 1.4.4)
-- Fixed type of sendcounts and displs in the "use mpi" F90 module.
-  ABI is preserved, but applications may well be broken.  See the
-  README for more details.  Thanks to Stanislav Sazykin for
-  identifying the issue.  (** also to appear in 1.4.4)
-- Fix indexed datatype leaks.  Thanks to Pascal Deveze for supplying
-  the initial patch.  (** also to appear in 1.4.4)
-- Fix debugger mapping when mpirun's -npernode option is used.
-- Fixed support for configure's --disable-dlopen option when used with
-  "make distclean".
-- Fix segv associated with MPI_Comm_create with MPI_GROUP_EMPTY.
-  Thanks to Dominik Goeddeke for finding this.
-  (** also to appear in 1.4.4)
-- Improved LoadLeveler ORTE support.
-- Add new WinVerbs BTL plugin, supporting native OpenFabrics verbs on
-  Windows (the "wv" BTL).
-- Add new btl_openib_gid_index MCA parameter to allow selecting which
-  GID to use on an OpenFabrics device's GID table.
-- Add support for PCI relaxed ordering in the OpenFabrics BTL (when
-  available).
-- Update rsh logic to allow correct SGE operation.
-- Ensure that the mca_paffinity_alone MCA parameter only appears once
-  in the ompi_info output.  Thanks to Gus Correa for identifying the
-  issue.
-- Fixed return codes from MPI_PROBE and MPI_IPROBE.
-  (** also to appear in 1.4.4)
-- Remove --enable-progress-thread configure option; it doesn't work on
-  the v1.5 branch.  Rename --enable-mpi-threads to
-  --enable-mpi-thread-multiple.  Add new --enable-opal-multi-threads
-  option.
-- Updates for Intel Fortran compiler version 12.
-- Remove bproc support.  Farewell bproc!
-- If something goes wrong during MPI_INIT, fix the error
-  message to say that it's illegal to invoke MPI_INIT before
-  MPI_INIT.
-
-
-1.5.3: 16 Mar 2011
-------------------
-
-- Add missing "affinity" MPI extension (i.e., the OMPI_Affinity_str()
-  API) that was accidentally left out of the 1.5.2 release.
-
-
-1.5.2: 9 Mar 2011
------------------
-
-- Replaced all custom topology / affinity code with initial support
-  for hwloc v1.1.1 (PLPA has been removed -- long live hwloc!).  Note
-  that hwloc is bundled with Open MPI, but an external hwloc can be
-  used, if desired.  See README for more details.
-- Many CMake updates for Windows builds.
-- Updated opal_cr_thread_sleep_wait MCA param default value to make it
-  less aggressive.
-- Updated debugger support to allow Totalview attaching from jobs
-  launched directly via srun (not mpirun).  Thanks to Nikolay Piskun
-  for the patch.
-- Added more FTB/CIFTS support.
-- Fixed compile error with the PGI compiler.
-- Portability fixes to allow the openib BTL to run on the Solaris
-  verbs stack.
-- Fixed multi-token command-line issues when using the mpirun
-  --debug switch.  For example:
-      mpirun --debug -np 2 a.out "foo bar"
-  Thanks to Gabriele Fatigati for reporting the issue.
-- Added ARM support.
-- Added the MPI_ROOT environment variable in the Open MPI Linux SRPM
-  for customers who use the BPS and LSF batch managers.
-- Updated ROMIO from MPICH v1.3.1 (plus one additional patch).
-- Fixed some deprecated MPI API function notification messages.
-- Added new "bfo" PML that provides failover on OpenFabrics networks.
-- Fixed some buffer memcheck issues in MPI_*_init.
-- Added Solaris-specific chip detection and performance improvements.
-- Fix some compile errors on Solaris.
-- Updated the "rmcast" framework with bug fixes, new functionality.
-- Updated the Voltaire FCA component with bug fixes, new
-  functionality.  Support for FCA version 2.1.
-- Fix gcc 4.4.x and 4.5.x over-aggressive warning notifications on
-  possibly freeing stack variables.  Thanks to the Gentoo packagers
-  for reporting the issue.
-- Make the openib component be verbose when it disqualifies itself due
-  to MPI_THREAD_MULTIPLE.
-- Minor man page fixes.
-- Various checkpoint / restart fixes.
-- Fix race condition in the one-sided unlock code.  Thanks to
-  Guillaume Thouvenin for finding the issue.
-- Improve help message aggregation.
-- Add OMPI_Affinity_str() optional user-level API function (i.e., the
-  "affinity" MPI extension).  See README for more details.
-- Added btl_tcp_if_seq MCA parameter to select a different ethernet
-  interface for each MPI process on a node.  This parameter is only
-  useful when used with virtual ethernet interfaces on a single
-  network card (e.g., when using virtual interfaces give dedicated
-  hardware resources on the NIC to each process).
-- Changed behavior of mpirun to terminate if it receives 10 (or more)
-  SIGPIPEs.
-- Fixed oversubscription detection.
-- Added new mtl_mx_board and mtl_mx_endpoint MCA parameters.
-- Added ummunotify support for OpenFabrics-based transports.  See the
-  README for more details.
-
-
-1.5.1: 15 Dec 2010
-------------------
-
-- Fixes for the Oracle Studio 12.2 Fortran compiler.
-- Fix SPARC and SPARCv9 atomics.  Thanks to Nicola Stange for the
-  initial patch.
-- Fix Libtool issues with the IBM XL compiler in 64-bit mode.
-- Restore the reset of the libevent progress counter to avoid
-  over-sampling the event library.
-- Update memory barrier support.
-- Use memmove (instead of memcpy) when necessary (e.g., source and
-  destination overlap).
-- Fixed ompi-top crash.
-- Fix to handle Autoconf --program-transforms properly and other
-  m4/configury updates.  Thanks to the GASNet project for the
-  --program transforms fix.
-- Allow hostfiles to specify usernames on a per-host basis.
-- Update wrapper compiler scripts to search for perl during configure,
-  per request from the BSD maintainers.
-- Minor man page fixes.
-- Added --with-libltdl option to allow building Open MPI with an
-  external installation of libltdl.
-- Fixed various issues with -D_FORTIFY_SOURCE=2.
-- Various VT fixes and updates.
-
-
-1.5: 10 Oct 2010
-----------------
-
-- Added "knem" support: direct process-to-process copying for shared
-  memory message passing.  See https://runtime.bordeaux.inria.fr/knem/
-  and the README file for more details.
-- Updated shared library versioning scheme and linking style of MPI
-  applications.  The MPI application ABI has been broken from the
-  v1.3/v1.4 series.  MPI applications compiled against any prior
-  version of Open MPI will need to, at a minimum, re-link.  See the
-  README file for more details.
-- Added "fca" collective component, enabling MPI collective offload
-  support for Voltaire switches.
-- Fixed MPI one-sided operations with large target displacements.
-  Thanks to Brian Price and Jed Brown for reporting the issue.
-- Fixed MPI_GET_COUNT when used with large counts.  Thanks to Jed
-  Brown for reporting the issue.
-- Made the openib BTL safer if extremely low SRQ settings are used.
-- Fixed handling of the array_of_argv parameter in the Fortran
-  binding of MPI_COMM_SPAWN_MULTIPLE (** also to appear: 1.4.3).
-- Fixed malloc(0) warnings in some collectives.
-- Fixed a problem with the Fortran binding for
-  MPI_FILE_CREATE_ERRHANDLER.  Thanks to Secretan Yves for identifying
-  the issue (** also to appear: 1.4.3).
-- Updates to the LSF PLM to ensure that the path is correctly passed.
-  Thanks to Teng Lin for the patch (** also to appear: 1.4.3).
-- Fixes for the F90 MPI_COMM_SET_ERRHANDLER and MPI_WIN_SET_ERRHANDLER
-  bindings.  Thanks to Paul Kapinos for pointing out the issue
-  (** also to appear: 1.4.3).
-- Fixed extra_state parameter types in F90 prototypes for
-  MPI_COMM_CREATE_KEYVAL, MPI_GREQUEST_START, MPI_REGISTER_DATAREP,
-  MPI_TYPE_CREATE_KEYVAL, and MPI_WIN_CREATE_KEYVAL.
-- Fixes for Solaris oversubscription detection.
-- If the PML determines it can't reach a peer process, print a
-  slightly more helpful message.  Thanks to Nick Edmonds for the
-  suggestion.
-- Make btl_openib_if_include/exclude function the same way
-  btl_tcp_if_include/exclude works (i.e., supplying an _include list
-  overrides supplying an _exclude list).
-- Apply more scalable reachability algorithm on platforms with more
-  than 8 TCP interfaces.
-- Various assembly code updates for more modern platforms / compilers.
-- Relax restrictions on using certain kinds of MPI datatypes with
-  one-sided operations.  Users beware; not all MPI datatypes are valid
-  for use with one-sided operations!
-- Improve behavior of MPI_COMM_SPAWN with regards to --bynode.
-- Various threading fixes in the openib BTL and other core pieces of
-  Open MPI.
-- Various help file and man pages updates.
-- Various FreeBSD and NetBSD updates and fixes.  Thanks to Kevin
-  Buckley and Aleksej Saushev for their work.
-- Fix case where freeing communicators in MPI_FINALIZE could cause
-  process failures.
-- Print warnings if shared memory state files are opened on what look
-  like networked filesystems.
-- Update libevent to v1.4.13.
-- Allow propagating signals to processes that call fork().
-- Fix bug where MPI_GATHER was sometimes incorrectly examining the
-  datatype on non-root processes.  Thanks to Michael Hofmann for
-  investigating the issue.
-- Various Microsoft Windows fixes.
-- Various Catamount fixes.
-- Various checkpoint / restart fixes.
-- Xgrid support has been removed until it can be fixed (patches
-  would be welcome).
-- Added simplistic "libompitrace" contrib package.  Using the MPI
-  profiling interface, it essentially prints out to stderr when select
-  MPI functions are invoked.
-- Update bundled VampirTrace to v5.8.2.
-- Add pkg-config(1) configuration files for ompi, ompi-c, ompi-cxx,
-  ompi-f77, ompi-f90.  See the README for more details.
-- Removed the libopenmpi_malloc library (added in the v1.3 series)
-  since it is no longer necessary
-- Add several notifier plugins (generally used when Open MPI detects
-  system/network administrator-worthy problems); each have their own
-  MCA parameters to govern their usage.  See "ompi_info --param
-  notifier <name>" for more details.
-  - command to execute arbitrary commands (e.g., run a script).
-  - file to send output to a file.
-  - ftb to send output to the Fault Tolerant Backplane (see
-    https://wiki.mcs.anl.gov/cifts/index.php/CIFTS)
-  - hnp to send the output to mpirun.
-  - smtp (requires libesmtp) to send an email.
-
-1.4.5: 12 Feb 2012
-------------------
-
-- Fixed the --disable-memory-manager configure switch.
-  (** also to appear in 1.5.5)
-- Fix typos in code and man pages.  Thanks to Fujitsu for these fixes.
-  (** also to appear in 1.5.5)
-- Improve management of the registration cache; when full, try freeing
-  old entries and attempt to re-register.
-- Fixed a data packing pointer alignment issue.  Thanks to Fujitsu
-  for the patch.
-  (** also to appear in 1.5.5)
-- Add ability to turn off warning about having the shared memory backing
-  store over a networked filesystem.  Thanks to Chris Samuel for this
-  suggestion.
-  (** also to appear in 1.5.5)
-- Removed an unnecessary memmove() and plugged a couple of small memory leaks
-  in the openib OOB connection setup code.
-- Fixed some QLogic bugs. Thanks to Mark Debbage from QLogic for the patches.
-- Fixed problem with MPI_IN_PLACE and other sentinel Fortran constants
-  on OS X.
-  (** also to appear in 1.5.5)
-- Fix SLURM cpus-per-task allocation.
-  (** also to appear in 1.5.5)
-- Fix the datatype engine for when data left over from the previous
-  pack was larger than the allowed space in the pack buffer. Thanks to
-  Yuki Matsumoto and Takahiro Kawashima for the bug report and the
-  patch.
-- Fix Fortran value for MPI_MAX_PORT_NAME.  Thanks to Enzo Dari for
-  raising the issue.
-- Workaround an Intel compiler v12.1.0 2011.6.233 vector optimization
-  bug.
-- Fix issues on Solaris with the openib BTL.
-- Fixes for the Oracle Studio 12.2 Fortran compiler.
-- Update iWARP parameters for the Intel NICs.
-  (** also to appear in 1.5.5)
-- Fix obscure cases where MPI_ALLGATHER could crash.  Thanks to Andrew
-  Senin for reporting the problem.
-  (** also to appear in 1.5.5)
-
-
-1.4.4: 11 Oct 2011
-------------------
-
-- Modified a memcpy() call in the openib btl connection setup to use
-  memmove() instead because of the possibility of an overlapping
-  copy (as identified by valgrind).
-- Changed use of sys_timer_get_cycles() to the more appropriate
-  wrapper: opal_timer_base_get_cycles().  Thanks to Jani Monoses
-  for this fix.
-- Corrected the reported default value of btl_openib_ib_timeout
-  in the "IB retries exceeded" error message.  Thanks to Kevin Buckley
-  for this correction.
-- Increased rdmacm address resolution timeout from 1s to 30s &
-  updated Chelsio T4 openib BTL defaults.  Thanks to Steve Wise
-  for these updates.
-  (** also to appear in 1.5.5)
-- Ensure that MPI_Accumulate error return in 1.4 is consistent with
-  1.5.x and trunk.
-- Allow the word "slots" in rankfiles (i.e., not just "slot").
-  (** also appeared in 1.5.4)
-- Add Mellanox ConnectX 3 device IDs to the openib BTL defaults.
-  (** also appeared in 1.5.4)
-- Update description of btl_openib_cq_size to be more accurate.
-- Ensure mpirun's --xterm options leaves sessions attached.
-  (** also appeared in 1.5.4)
-- Update to allow more than 128 entries in an appfile.
-  (** also appeared in 1.5.4)
-- Update description of btl_openib_cq_size to be more accurate.
-  (** also appeared in 1.5.4)
-- Fix for deadlock when handling recursive attribute keyval deletions
-  (e.g., when using ROMIO with MPI_THREAD_MULTIPLE).
-- Fix indexed datatype leaks.  Thanks to Pascal Deveze for supplying
-  the initial patch.  (** also appeared in 1.5.4)
-- Fixed the F90 types of the sendcounts and displs parameters to
-  MPI_SCATTERV.  Thanks to Stanislav Sazykin for identifying the issue.
-  (** also appeared in 1.5.4)
-- Exclude opal/libltdl from "make distclean" when --disable-dlopen is
-  used.  Thanks to David Gunter for reporting the issue.
-- Fixed a segv in MPI_Comm_create when called with GROUP_EMPTY.
-  Thanks to Dominik Goeddeke for finding this.
-  (** also appeared in 1.5.4)
-- Fixed return codes from MPI_PROBE and MPI_IPROBE.
-  (** also appeared in 1.5.4)
-- Fixed undefined symbol error when using the vtf90 profiling tool.
-- Fix for referencing an uninitialized variable in DPM ORTE.  Thanks
-  to Avinash Malik for reporting the issue.
-- Fix for correctly handling multi-token args when using debuggers.
-- Eliminated the unneeded u_int*_t datatype definitions.
-- Change in ORTE DPM to get around gcc 4.[45].x compiler wanrings
-  about possibly calling free() on a non-heap variable, even though it
-  will never happen because the refcount will never go to zero.
-- Fixed incorrect text in MPI_File_set_view man page.
-- Fix in MPI_Init_thread for checkpoint/restart.
-- Fix for libtool issue when using pgcc to compile ompi in conjunction
-  with the -tp option.
-- Fixed a race condition in osc_rdma_sync.  Thanks to Guillaume
-  Thouvenin for finding this issue.
-- Clarification of MPI_Init_thread man page.
-- Fixed an indexing problem in precondition_transports.
-- Fixed a problem in which duplicated libs were being specified for
-  linking.  Thanks to Hicham Mouline for noticing it.
-- Various autogen.sh fixes.
-- Fix for memchecking buffers during MPI_*INIT.
-- Man page cleanups.  Thanks to Jeremiah Willcock and Jed Brown.
-- Fix for VT rpmbuild on RHEL5.
-- Support Solaris legacy munmap prototype changes.
-  (** also appeared in 1.5.4)
-- Expands app_idx to int32_t to allow more than 127 app_contexts.
-- Guard the inclusion of execinfo.h since not all platforms have it.  Thanks
-  to Aleksej Saushev for identifying this issue.
-  (** also appeared in 1.5.4)
-- Fix to avoid possible environment corruption.  Thanks to Peter Thompson
-  for identifying the issue and supplying a patch.
-  (** also appeared in 1.5.4)
-- Fixed paffinity base MCA duplicate registrations.  Thanks to Gus
-  Correa for bringing this to our attention.
-- Fix recursive locking bug when MPI-IO was used with
-  MPI_THREAD_MULTIPLE.  (** also appeared in 1.5.4)
-- F90 MPI API fixes.
-- Fixed a misleading MPI_Bcast error message.  Thanks to Jeremiah
-  Willcock for reporting this.
-- Added <sys/stat.h> to ptmalloc's hooks.c (it's not always included
-  by default on some systems).
-- Libtool patch to get around a build problem when using the IBM XL
-  compilers.
-- Fix to detect and avoid overlapping memcpy().  Thanks to
-  Francis Pellegrini for identifying the issue.
-- Fix to allow ompi to work on top of RoCE vLANs.
-- Restored a missing debugger flag to support TotalView.  Thanks to
-  David Turner and the TV folks for supplying the fix.
-- Updated SLURM support to 1.5.1.
-- Removed an extraneous #include from the TCP BTL.
-- When specifying OOB ports, fix to convert the ports into network
-  byte order before binding.
-- Fixed use of memory barriers in the SM BTL.  This fixed segv's when
-  compiling with Intel 10.0.025 or PGI 9.0-3.
-- Fix to prevent the SM BTL from creating its mmap'd file in
-  directories that are remotely mounted.
-
-
-1.4.3: 6 Sep 2010
------------------
-
-- Fixed handling of the array_of_argv parameter in the Fortran
-  binding of MPI_COMM_SPAWN_MULTIPLE (** also to appear: 1.5).
-- Fixed a problem with the Fortran binding for
-  MPI_FILE_CREATE_ERRHANDLER.  Thanks to Secretan Yves for identifying
-  the issue (** also to appear: 1.5).
-- Updates to the LSF PLM to ensure that the path is correctly passed.
-  Thanks to Teng Lin for the patch (** also to appear: 1.5).
-- Fixes for the F90 MPI_COMM_SET_ERRHANDLER and MPI_WIN_SET_ERRHANDLER
-  bindings.  Thanks to Paul Kapinos for pointing out the issue.
-  (** also to appear: 1.5).
-- Fixed various MPI_THREAD_MULTIPLE race conditions.
-- Fixed an issue with an undeclared variable from ptmalloc2 munmap on
-  BSD systems.
-- Fixes for BSD interface detection.
-- Various other BSD fixes.  Thanks to Kevin Buckley helping to track.
-  all of this down.
-- Fixed issues with the use of the -nper* mpirun command line arguments.
-- Fixed an issue with coll tuned dynamic rules.
-- Fixed an issue with the use of OPAL_DESTDIR being applied too aggressively.
-- Fixed an issue with one-sided xfers when the displacement exceeds 2GBytes.
-- Change to ensure TotalView works properly on Darwin.
-- Added support for Visual Studio 2010.
-- Fix to ensure proper placement of VampirTrace header files.
-- Needed to add volatile keyword to a varialbe used in debugging
-  (MPIR_being_debugged).
-- Fixed a bug in inter-allgather.
-- Fixed malloc(0) warnings.
-- Corrected a typo the MPI_Comm_size man page (intra -> inter).  Thanks
-  to Simon number.cruncher for pointing this out.
-- Fixed a SegV in orted when given more than 127 app_contexts.
-- Removed xgrid source code from the 1.4 branch since it is no longer
-  supported in the 1.4 series.
-- Removed the --enable-opal-progress-threads config option since
-  opal progress thread support does not work in 1.4.x.
-- Fixed a defect in VampirTrace's vtfilter.
-- Fixed wrong Windows path in hnp_contact.
-- Removed the requirement for a paffinity component.
-- Removed a hardcoded limit of 64 interconnected jobs.
-- Fix to allow singletons to use ompi-server for rendezvous.
-- Fixed bug in output-filename option.
-- Fix to correctly handle failures in mx_init().
-- Fixed a potential Fortran memory leak.
-- Fixed an incorrect branch in some ppc32 assembly code.  Thanks
-  to Matthew Clark for this fix.
-- Remove use of undocumented AS_VAR_GET macro during configuration.
-- Fixed an issue with VampirTrace's wrapper for MPI_init_thread.
-- Updated mca-btl-openib-device-params.ini file with various new vendor id's.
-- Configuration fixes to ensure CPPFLAGS in handled properly if a non-standard
-  valgrind location was specified.
-- Various man page updates
-
-
-1.4.2: 4 May 2010
------------------
-
-- Fixed problem when running in heterogeneous environments.  Thanks to
-  Timur Magomedov for helping to track down this issue.
-- Update LSF support to ensure that the path is passed correctly.
-  Thanks to Teng Lin for submitting a patch.
-- Fixed some miscellaneous oversubscription detection bugs.
-- IBM re-licensed its LoadLeveler code to be BSD-compliant.
-- Various OpenBSD and NetBSD build and run-time fixes.  Many thanks to
-  the OpenBSD community for their time, expertise, and patience
-  getting these fixes incorporated into Open MPI's main line.
-- Various fixes for multithreading deadlocks, race conditions, and
-  other nefarious things.
-- Fixed ROMIO's handling of "nearly" contiguous issues (e.g., with
-  non-zero true_lb).  Thanks for Pascal Deveze for the patch.
-- Bunches of Windows build fixes.  Many thanks to several Windows
-  users for their help in improving our support on Windows.
-- Now allow the graceful failover from MTLs to BTLs if no MTLs can
-  initialize successfully.
-- Added "clobber" information to various atomic operations, fixing
-  erroneous behavior in some newer versions of the GNU compiler suite.
-- Update various iWARP and InfiniBand device specifications in the
-  OpenFabrics .ini support file.
-- Fix the use of hostfiles when a username is supplied.
-- Various fixes for rankfile support.
-- Updated the internal version of VampirTrace to 5.4.12.
-- Fixed OS X TCP wireup issues having to do with IPv4/IPv6 confusion
-  (see https://svn.open-mpi.org/trac/ompi/changeset/22788 for more
-  details).
-- Fixed some problems in processor affinity support, including when
-  there are "holes" in the processor namespace (e.g., offline
-  processors).
-- Ensure that Open MPI's "session directory" (usually located in /tmp)
-  is cleaned up after process termination.
-- Fixed some problems with the collective "hierarch" implementation
-  that could occur in some obscure conditions.
-- Various MPI_REQUEST_NULL, API parameter checking, and attribute
-  error handling fixes.  Thanks to Lisandro Dalcín for reporting the
-  issues.
-- Fix case where MPI_GATHER erroneously used datatypes on non-root
-  nodes.  Thanks to Michael Hofmann for investigating the issue.
-- Patched ROMIO support for PVFS2 > v2.7 (patch taken from MPICH2
-  version of ROMIO).
-- Fixed "mpirun --report-bindings" behavior when used with
-  mpi_paffinity_alone=1.  Also fixed mpi_paffinity_alone=1 behavior
-  with non-MPI applications.  Thanks to Brice Goglin for noticing the
-  problem.
-- Ensure that all OpenFabrics devices have compatible receive_queues
-  specifications before allowing them to communicate.  See the lengthy
-  comment in https://svn.open-mpi.org/trac/ompi/changeset/22592 for
-  more details.
-- Fix some issues with checkpoint/restart.
-- Improve the pre-MPI_INIT/post-MPI_FINALIZE error messages.
-- Ensure that loopback addresses are never advertised to peer
-  processes for RDMA/OpenFabrics support.
-- Fixed a CSUM PML false positive.
-- Various fixes for Catamount support.
-- Minor update to wrapper compilers in how user-specific argv is
-  ordered on the final command line.  Thanks to Jed Brown for the
-  suggestions.
-- Removed flex.exe binary from Open MPI tarballs; now generate flex
-  code from a newer (Windows-friendly) flex when we make official
-  tarballs.
-
-
-1.4.1: 15 Jan 2010
-------------------
-
-- Update to PLPA v1.3.2, addressing a licensing issue identified by
-  the Fedora project.  See
-  https://svn.open-mpi.org/trac/plpa/changeset/262 for details.
-- Add check for malformed checkpoint metadata files (Ticket #2141).
-- Fix error path in ompi-checkpoint when not able to checkpoint
-  (Ticket #2138).
-- Cleanup component release logic when selecting checkpoint/restart
-  enabled components (Ticket #2135).
-- Fixed VT node name detection for Cray XT platforms, and fixed some
-  broken VT documentation files.
-- Fix a possible race condition in tearing down RDMA CM-based
-  connections.
-- Relax error checking on MPI_GRAPH_CREATE.  Thanks to David Singleton
-  for pointing out the issue.
-- Fix a shared memory "hang" problem that occurred on x86/x86_64
-  platforms when used with the GNU >=4.4.x compiler series.
-- Add fix for Libtool 2.2.6b's problems with the PGI 10.x compiler
-  suite.  Inspired directly from the upstream Libtool patches that fix
-  the issue (but we need something working before the next Libtool
-  release).
-
-
-1.4: 8 Dec 2009
----------------
-
-The *only* change in the Open MPI v1.4 release (as compared to v1.3.4)
-was to update the embedded version of Libtool's libltdl to address a
-potential security vulnerability.  Specifically: Open MPI v1.3.4 was
-created with GNU Libtool 2.2.6a; Open MPI v1.4 was created with GNU
-Libtool 2.2.6b.  There are no other changes between Open MPI v1.3.4
-and v1.4.
-
-
-1.3.4: 13 Feb 2010
-------------------
-
-- Fix some issues in OMPI's SRPM with regard to shell_scripts_basename
-  and its use with mpi-selector.  Thanks to Bill Johnstone for
-  pointing out the problem.
-- Added many new MPI job process affinity options to mpirun.  See the
-  newly-updated mpirun(1) man page for details.
-- Several updates to mpirun's XML output.
-- Update to fix a few Valgrind warnings with regards to the ptmalloc2
-  allocator and Open MPI's use of PLPA.
-- Many updates and fixes to the (non-default) "sm" collective
-  component (i.e., native shared memory MPI collective operations).
-- Updates and fixes to some MPI_COMM_SPAWN_MULTIPLE corner cases.
-- Fix some internal copying functions in Open MPI's use of PLPA.
-- Correct some SLURM nodelist parsing logic that may have interfered
-  with large jobs.  Additionally, per advice from the SLURM team,
-  change the environment variable that we use for obtaining the job's
-  allocation.
-- Revert to an older, safer (but slower) communicator ID allocation
-  algorithm.
-- Fixed minimum distance finding for OpenFabrics devices in the openib
-  BTL.
-- Relax the parameter checking MPI_CART_CREATE a bit.
-- Fix MPI_COMM_SPAWN[_MULTIPLE] to only error-check the info arguments
-  on the root process.  Thanks to Federico Golfre Andreasi for
-  reporting the problem.
-- Fixed some BLCR configure issues.
-- Fixed a potential deadlock when the openib BTL was used with
-  MPI_THREAD_MULTIPLE.
-- Fixed dynamic rules selection for the "tuned" coll component.
-- Added a launch progress meter to mpirun (useful for large jobs; set
-  the orte_report_launch_progress MCA parameter to 1 to see it).
-- Reduced the number of file descriptors consumed by each MPI process.
-- Add new device IDs for Chelsio T3 RNICs to the openib BTL config file.
-- Fix some CRS self component issues.
-- Added some MCA parameters to the PSM MTL to tune its run-time
-  behavior.
-- Fix some VT issues with MPI_BOTTOM/MPI_IN_PLACE.
-- Man page updates from the Debain Open MPI package maintainers.
-- Add cycle counter support for the Alpha and Sparc platforms.
-- Pass visibility flags to libltdl's configure script, resulting in
-  those symbols being hidden.  This appears to mainly solve the
-  problem of applications attempting to use different versions of
-  libltdl from that used to build Open MPI.
-
-
-1.3.3: 14 Jul 2009
-------------------
-
-- Fix a number of issues with the openib BTL (OpenFabrics) RDMA CM,
-  including a memory corruption bug, a shutdown deadlock, and a route
-  timeout.  Thanks to David McMillen and Hal Rosenstock for help in
-  tracking down the issues.
-- Change the behavior of the EXTRA_STATE parameter that is passed to
-  Fortran attribute callback functions: this value is now stored
-  internally in MPI -- it no longer references the original value
-  passed by MPI_*_CREATE_KEYVAL.
-- Allow the overriding RFC1918 and RFC3330 for the specification of
-  "private" networks, thereby influencing Open MPI's TCP
-  "reachability" computations.
-- Improve flow control issues in the sm btl, by both tweaking the
-  shared memory progression rules and by enabling the "sync" collective
-  to barrier every 1,000th collective.
-- Various fixes for the IBM XL C/C++ v10.1 compiler.
-- Allow explicit disabling of ptmalloc2 hooks at runtime (e.g., enable
-  support for Debian's builtroot system).  Thanks to Manuel Prinz and
-  the rest of the Debian crew for helping identify and fix this issue.
-- Various minor fixes for the I/O forwarding subsystem.
-- Big endian iWARP fixes in the Open Fabrics RDMA CM support.
-- Update support for various OpenFabrics devices in the openib BTL's
-  .ini file.
-- Fixed undefined symbol issue with Open MPI's parallel debugger
-  message queue support so it can be compiled by Sun Studio compilers.
-- Update MPI_SUBVERSION to 1 in the Fortran bindings.
-- Fix MPI_GRAPH_CREATE Fortran 90 binding.
-- Fix MPI_GROUP_COMPARE behavior with regards to MPI_IDENT.  Thanks to
-  Geoffrey Irving for identifying the problem and supplying the fix.
-- Silence gcc 4.1 compiler warnings about type punning.  Thanks to
-  Number Cruncher for the fix.
-- Added more Valgrind and other memory-cleanup fixes.  Thanks to
-  various Open MPI users for help with these issues.
-- Miscellaneous VampirTrace fixes.
-- More fixes for openib credits in heavy-congestion scenarios.
-- Slightly decrease the latency in the openib BTL in some conditions
-  (add "send immediate" support to the openib BTL).
-- Ensure to allow MPI_REQUEST_GET_STATUS to accept an
-  MPI_STATUS_IGNORE parameter.  Thanks to Shaun Jackman for the bug
-  report.
-- Added Microsoft Windows support.  See README.WINDOWS file for
-  details.
-
-
-1.3.2: 22 Apr 2009
-------------------
-
-- Fixed a potential infinite loop in the openib BTL that could occur
-  in senders in some frequent-communication scenarios.  Thanks to Don
-  Wood for reporting the problem.
-- Add a new checksum PML variation on ob1 (main MPI point-to-point
-  communication engine) to detect memory corruption in node-to-node
-  messages
-- Add a new configuration option to add padding to the openib
-  header so the data is aligned
-- Add a new configuration option to use an alternative checksum algo
-  when using the checksum PML
-- Fixed a problem reported by multiple users on the mailing list that
-  the LSF support would fail to find the appropriate libraries at
-  run-time.
-- Allow empty shell designations from getpwuid().  Thanks to Sergey
-  Koposov for the bug report.
-- Ensure that mpirun exits with non-zero status when applications die
-  due to user signal.  Thanks to Geoffroy Pignot for suggesting the
-  fix.
-- Ensure that MPI_VERSION / MPI_SUBVERSION match what is returned by
-  MPI_GET_VERSION.  Thanks to Rob Egan for reporting the error.
-- Updated MPI_*KEYVAL_CREATE functions to properly handle Fortran
-  extra state.
-- A variety of ob1 (main MPI point-to-point communication engine) bug
-  fixes that could have caused hangs or seg faults.
-- Do not install Open MPI's signal handlers in MPI_INIT if there are
-  already signal handlers installed.  Thanks to Kees Verstoep for
-  bringing the issue to our attention.
-- Fix GM support to not seg fault in MPI_INIT.
-- Various VampirTrace fixes.
-- Various PLPA fixes.
-- No longer create BTLs for invalid (TCP) devices.
-- Various man page style and lint cleanups.
-- Fix critical OpenFabrics-related bug noted here:
-  https://www.open-mpi.org/community/lists/announce/2009/03/0029.php.
-  Open MPI now uses a much more robust memory intercept scheme that is
-  quite similar to what is used by MX.  The use of "-lopenmpi-malloc"
-  is no longer necessary, is deprecated, and is expected to disappear
-  in a future release.  -lopenmpi-malloc will continue to work for the
-  duration of the Open MPI v1.3 and v1.4 series.
-- Fix some OpenFabrics shutdown errors, both regarding iWARP and SRQ.
-- Allow the udapl BTL to work on Solaris platforms that support
-  relaxed PCI ordering.
-- Fix problem where the mpirun would sometimes use rsh/ssh to launch on
-  the localhost (instead of simply forking).
-- Minor SLURM stdin fixes.
-- Fix to run properly under SGE jobs.
-- Scalability and latency improvements for shared memory jobs: convert
-  to using one message queue instead of N queues.
-- Automatically size the shared-memory area (mmap file) to match
-  better what is needed;  specifically, so that large-np jobs will start.
-- Use fixed-length MPI predefined handles in order to provide ABI
-  compatibility between Open MPI releases.
-- Fix building of the posix paffinity component to properly get the
-  number of processors in loosely tested environments (e.g.,
-  FreeBSD).  Thanks to Steve Kargl for reporting the issue.
-- Fix --with-libnuma handling in configure.  Thanks to Gus Correa for
-  reporting the problem.
-
-
-1.3.1: 19 Mar 2009
-------------------
-
-- Added "sync" coll component to allow users to synchronize every N
-  collective operations on a given communicator.
-- Increased the default values of the IB and RNR timeout MCA parameters.
-- Fix a compiler error noted by Mostyn Lewis with the PGI 8.0 compiler.
-- Fix an error that prevented stdin from being forwarded if the
-  rsh launcher was in use.  Thanks to Branden Moore for pointing out
-  the problem.
-- Correct a case where the added datatype is considered as contiguous but
-  has gaps in the beginning.
-- Fix an error that limited the number of comm_spawns that could
-  simultaneously be running in some environments
-- Correct a corner case in OB1's GET protocol for long messages; the
-  error could sometimes cause MPI jobs using the openib BTL to hang.
-- Fix a bunch of bugs in the IO forwarding (IOF) subsystem and add some
-  new options to output to files and redirect output to xterm.  Thanks to
-  Jody Weissmann for helping test out many of the new fixes and
-  features.
-- Fix SLURM race condition.
-- Fix MPI_File_c2f(MPI_FILE_NULL) to return 0, not -1.  Thanks to
-  Lisandro Dalcín for the bug report.
-- Fix the DSO build of tm PLM.
-- Various fixes for size disparity between C int's and Fortran
-  INTEGER's.  Thanks to Christoph van Wullen for the bug report.
-- Ensure that mpirun exits with a non-zero exit status when daemons or
-  processes abort or fail to launch.
-- Various fixes to work around Intel (NetEffect) RNIC behavior.
-- Various fixes for mpirun's --preload-files and --preload-binary
-  options.
-- Fix the string name in MPI::ERRORS_THROW_EXCEPTIONS.
-- Add ability to forward SIFTSTP and SIGCONT to MPI processes if you
-  set the MCA parameter orte_forward_job_control to 1.
-- Allow the sm BTL to allocate larger amounts of shared memory if
-  desired (helpful for very large multi-core boxen).
-- Fix a few places where we used PATH_MAX instead of OPAL_PATH_MAX,
-  leading to compile problems on some platforms.  Thanks to Andrea Iob
-  for the bug report.
-- Fix mca_btl_openib_warn_no_device_params_found MCA parameter; it
-  was accidentally being ignored.
-- Fix some run-time issues with the sctp BTL.
-- Ensure that RTLD_NEXT exists before trying to use it (e.g., it
-  doesn't exist on Cygwin).  Thanks to Gustavo Seabra for reporting
-  the issue.
-- Various fixes to VampirTrace, including fixing compile errors on
-  some platforms.
-- Fixed missing MPI_Comm_accept.3 man page; fixed minor issue in
-  orterun.1 man page.  Thanks to Dirk Eddelbuettel for identifying the
-  problem and submitting a patch.
-- Implement the XML formatted output of stdout/stderr/stddiag.
-- Fixed mpirun's -wdir switch to ensure that working directories for
-  multiple app contexts are properly handled.  Thanks to Geoffroy
-  Pignot for reporting the problem.
-- Improvements to the MPI C++ integer constants:
-  - Allow MPI::SEEK_* constants to be used as constants
-  - Allow other MPI C++ constants to be used as array sizes
-- Fix minor problem with orte-restart's command line options.  See
-  ticket #1761 for details.  Thanks to Gregor Dschung for reporting
-  the problem.
-
-
-1.3: 19 Jan 2009
-----------------
-
-- Extended the OS X 10.5.x (Leopard) workaround for a problem when
-  assembly code is compiled with -g[0-9].  Thanks to Barry Smith for
-  reporting the problem.  See ticket #1701.
-- Disabled MPI_REAL16 and MPI_COMPLEX32 support on platforms where the
-  bit representation of REAL*16 is different than that of the C type
-  of the same size (usually long double).  Thanks to Julien Devriendt
-  for reporting the issue.  See ticket #1603.
-- Increased the size of MPI_MAX_PORT_NAME to 1024 from 36. See ticket #1533.
-- Added "notify debugger on abort" feature. See tickets #1509 and #1510.
-  Thanks to Seppo Sahrakropi for the bug report.
-- Upgraded Open MPI tarballs to use Autoconf 2.63, Automake 1.10.1,
-  Libtool 2.2.6a.
-- Added missing MPI::Comm::Call_errhandler() function.  Thanks to Dave
-  Goodell for bringing this to our attention.
-- Increased MPI_SUBVERSION value in mpi.h to 1 (i.e., MPI 2.1).
-- Changed behavior of MPI_GRAPH_CREATE, MPI_TOPO_CREATE, and several
-  other topology functions per MPI-2.1.
-- Fix the type of the C++ constant MPI::IN_PLACE.
-- Various enhancements to the openib BTL:
-  - Added btl_openib_if_[in|ex]clude MCA parameters for
-    including/excluding comma-delimited lists of HCAs and ports.
-  - Added RDMA CM support, includng btl_openib_cpc_[in|ex]clude MCA
-    parameters
-  - Added NUMA support to only use "near" network adapters
-  - Added "Bucket SRQ" (BSRQ) support to better utilize registered
-    memory, including btl_openib_receive_queues MCA parameter
-  - Added ConnectX XRC support (and integrated with BSRQ)
-  - Added btl_openib_ib_max_inline_data MCA parameter
-  - Added iWARP support
-  - Revamped flow control mechansisms to be more efficient
-  - "mpi_leave_pinned=1" is now the default when possible,
-    automatically improving performance for large messages when
-    application buffers are re-used
-- Elimiated duplicated error messages when multiple MPI processes fail
-  with the same error.
-- Added NUMA support to the shared memory BTL.
-- Add Valgrind-based memory checking for MPI-semantic checks.
-- Add support for some optional Fortran datatypes (MPI_LOGICAL1,
-  MPI_LOGICAL2, MPI_LOGICAL4 and MPI_LOGICAL8).
-- Remove the use of the STL from the C++ bindings.
-- Added support for Platform/LSF job launchers.  Must be Platform LSF
-  v7.0.2 or later.
-- Updated ROMIO with the version from MPICH2 1.0.7.
-- Added RDMA capable one-sided component (called rdma), which
-  can be used with BTL components that expose a full one-sided
-  interface.
-- Added the optional datatype MPI_REAL2. As this is added to the "end of"
-  predefined datatypes in the fortran header files, there will not be
-  any compatibility issues.
-- Added Portable Linux Processor Affinity (PLPA) for Linux.
-- Addition of a finer symbols export control via the visibiliy feature
-  offered by some compilers.
-- Added checkpoint/restart process fault tolerance support. Initially
-  support a LAM/MPI-like protocol.
-- Removed "mvapi" BTL; all InfiniBand support now uses the OpenFabrics
-  driver stacks ("openib" BTL).
-- Added more stringent MPI API parameter checking to help user-level
-  debugging.
-- The ptmalloc2 memory manager component is now by default built as
-  a standalone library named libopenmpi-malloc.  Users wanting to
-  use leave_pinned with ptmalloc2 will now need to link the library
-  into their application explicitly.  All other users will use the
-  libc-provided allocator instead of Open MPI's ptmalloc2.  This change
-  may be overriden with the configure option enable-ptmalloc2-internal
-- The leave_pinned options will now default to using mallopt on
-  Linux in the cases where ptmalloc2 was not linked in.  mallopt
-  will also only be available if munmap can be intercepted (the
-  default whenever Open MPI is not compiled with --without-memory-
-  manager.
-- Open MPI will now complain and refuse to use leave_pinned if
-  no memory intercept / mallopt option is available.
-- Add option of using Perl-based wrapper compilers instead of the
-  C-based wrapper compilers.  The Perl-based version does not
-  have the features of the C-based version, but does work better
-  in cross-compile environments.
-
-
-1.2.9: 14 Feb 2009
-------------------
-
-- Fix a segfault when using one-sided communications on some forms of derived
-  datatypes.  Thanks to Dorian Krause for reporting the bug. See #1715.
-- Fix an alignment problem affecting one-sided communications on
-  some architectures (e.g., SPARC64). See #1738.
-- Fix compilation on Solaris when thread support is enabled in Open MPI
-  (e.g., when using --with-threads). See #1736.
-- Correctly take into account the MTU that an OpenFabrics device port
-  is using. See #1722 and
-  https://bugs.openfabrics.org/show_bug.cgi?id=1369.
-- Fix two datatype engine bugs. See #1677.
-  Thanks to Peter Kjellstrom for the bugreport.
-- Fix the bml r2 help filename so the help message can be found. See #1623.
-- Fix a compilation problem on RHEL4U3 with the PGI 32 bit compiler
-  caused by <infiniband/driver.h>.  See ticket #1613.
-- Fix the --enable-cxx-exceptions configure option. See ticket #1607.
-- Properly handle when the MX BTL cannot open an endpoint. See ticket #1621.
-- Fix a double free of events on the tcp_events list. See ticket #1631.
-- Fix a buffer overun in opal_free_list_grow (called by MPI_Init).
-  Thanks to Patrick Farrell for the bugreport and Stephan Kramer for
-  the bugfix.  See ticket #1583.
-- Fix a problem setting OPAL_PREFIX for remote sh-based shells.
-  See ticket #1580.
-
-
-1.2.8: 14 Oct 2008
-------------------
-
-- Tweaked one memory barrier in the openib component to be more conservative.
-  May fix a problem observed on PPC machines.  See ticket #1532.
-- Fix OpenFabrics IB partition support. See ticket #1557.
-- Restore v1.1 feature that sourced .profile on remote nodes if the default
-  shell will not do so (e.g. /bin/sh and /bin/ksh).  See ticket #1560.
-- Fix segfault in MPI_Init_thread() if ompi_mpi_init() fails. See ticket #1562.
-- Adjust SLURM support to first look for $SLURM_JOB_CPUS_PER_NODE instead of
-  the deprecated $SLURM_TASKS_PER_NODE environment variable.  This change
-  may be *required* when using SLURM v1.2 and above.  See ticket #1536.
-- Fix the MPIR_Proctable to be in process rank order. See ticket #1529.
-- Fix a regression introduced in 1.2.6 for the IBM eHCA. See ticket #1526.
-
-
-1.2.7: 28 Aug 2008
-------------------
-
-- Add some Sun HCA vendor IDs.  See ticket #1461.
-- Fixed a memory leak in MPI_Alltoallw when called from Fortran.
-  Thanks to Dave Grote for the bugreport.  See ticket #1457.
-- Only link in libutil when it is needed/desired.  Thanks to
-  Brian Barret for diagnosing and fixing the problem.  See ticket #1455.
-- Update some QLogic HCA vendor IDs.  See ticket #1453.
-- Fix F90 binding for MPI_CART_GET.  Thanks to Scott Beardsley for
-  bringing it to our attention. See ticket #1429.
-- Remove a spurious warning message generated in/by ROMIO. See ticket #1421.
-- Fix a bug where command-line MCA parameters were not overriding
-  MCA parameters set from environment variables.  See ticket #1380.
-- Fix a bug in the AMD64 atomics assembly.  Thanks to Gabriele Fatigati
-  for the bug report and bugfix.  See ticket #1351.
-- Fix a gather and scatter bug on intercommunicators when the datatype
-  being moved is 0 bytes. See ticket #1331.
-- Some more man page fixes from the Debian maintainers.
-  See tickets #1324 and #1329.
-- Have openib BTL (OpenFabrics support) check for the presence of
-  /sys/class/infiniband before allowing itself to be used.  This check
-  prevents spurious "OMPI did not find RDMA hardware!" notices on
-  systems that have the software drivers installed, but no
-  corresponding hardware.  See tickets #1321 and #1305.
-- Added vendor IDs for some ConnectX openib HCAs. See ticket #1311.
-- Fix some RPM specfile inconsistencies.  See ticket #1308.
-  Thanks to Jim Kusznir for noticing the problem.
-- Removed an unused function prototype that caused warnings on
-  some systems (e.g., OS X).  See ticket #1274.
-- Fix a deadlock in inter-communicator scatter/gather operations.
-  Thanks to Martin Audet for the bug report.  See ticket #1268.
-
-
-1.2.6: 7 Apr 2008
------------------
-
-- Fix a bug in the inter-allgather for asymmetric inter-communicators.
-  Thanks to Martin Audet for the bug report. See ticket #1247.
-- Fix a bug in the openib BTL when setting the CQ depth.  Thanks
-  to Jon Mason for the bug report and fix.  See ticket #1245.
-- On Mac OS X Leopard, the execinfo component will be used for
-  backtraces, making for a more durable solution.  See ticket #1246.
-- Added vendor IDs for some QLogic DDR openib HCAs. See ticket #1227.
-- Updated the URL to get the latest config.guess and config.sub files.
-  Thanks to Ralf Wildenhues for the bug report. See ticket #1226.
-- Added shared contexts support to PSM MTL.  See ticket #1225.
-- Added pml_ob1_use_early_completion MCA parameter to allow users
-  to turn off the OB1 early completion semantic and avoid "stall"
-  problems seen on InfiniBand in some cases.  See ticket #1224.
-- Sanitized some #define macros used in mpi.h to avoid compiler warnings
-  caused by MPI programs built with different autoconf versions.
-  Thanks to Ben Allan for reporting the problem, and thanks to
-  Brian Barrett for the fix. See ticket #1220.
-- Some man page fixes from the Debian maintainers. See ticket #1219.
-- Made the openib BTL a bit more resilient in the face of driver
-  errors.  See ticket #1217.
-- Fixed F90 interface for MPI_CART_CREATE.  See ticket #1208.
-  Thanks to Michal Charemza for reporting the problem.
-- Fixed some C++ compiler warnings. See ticket #1203.
-- Fixed formatting of the orterun man page.  See ticket #1202.
-  Thanks to Peter Breitenlohner for the patch.
-
-
-1.2.5: 8 Jan 2008
------------------
-
-- Fixed compile issue with open() on Fedora 8 (and newer) platforms.
-  Thanks to Sebastian Schmitzdorff for noticing the problem.
-- Added run-time warnings during MPI_INIT when MPI_THREAD_MULTIPLE
-  and/or progression threads are used (the OMPI v1.2 series does not
-  support these well at all).
-- Better handling of ECONNABORTED from connect on Linux.  Thanks to
-  Bob Soliday for noticing the problem; thanks to Brian Barrett for
-  submitting a patch.
-- Reduce extraneous output from OOB when TCP connections must
-  be retried.  Thanks to Brian Barrett for submitting a patch.
-- Fix for ConnectX devices and OFED 1.3.  See ticket #1190.
-- Fixed a configure problem for Fortran 90 on Cray systems.  Ticket #1189.
-- Fix an uninitialized variable in the error case in opal_init.c.
-  Thanks to Åke Sandgren for pointing out the mistake.
-- Fixed a hang in configure if $USER was not defined.  Thanks to
-  Darrell Kresge for noticing the problem.  See ticket #900.
-- Added support for parallel debuggers even when we have an optimized build.
-  See ticket #1178.
-- Worked around a bus error in the Mac OS X 10.5.X (Leopard) linker when
-  compiling Open MPI with -g.  See ticket #1179.
-- Removed some warnings about 'rm' from Mac OS X 10.5 (Leopard) builds.
-- Fix the handling of mx_finalize().  See ticket #1177.
-  Thanks to Åke Sandgren for bringing this issue to our attention.
-- Fixed minor file descriptor leak in the Altix timer code.  Thanks to
-  Paul Hargrove for noticing the problem and supplying the fix.
-- Fix a problem when using a different compiler for C and Objective C.
-  See ticket #1153.
-- Fix segfault in MPI_COMM_SPAWN when the user specified a working
-  directory.  Thanks to Murat Knecht for reporting this and suggesting
-  a fix.
-- A few manpage fixes from the Debian Open MPI maintainers.  Thanks to
-  Tilman Koschnick, Sylvestre Ledru, and Dirk Eddelbuettel.
-- Fixed issue with pthread detection when compilers are not all
-  from the same vendor.  Thanks to Åke Sandgren for the bug
-  report.  See ticket #1150.
-- Fixed vector collectives in the self module.  See ticket #1166.
-- Fixed some data-type engine bugs: an indexing bug, and an alignment bug.
-  See ticket #1165.
-- Only set the MPI_APPNUM attribute if it is defined.  See ticket
-  #1164.
-
-
-1.2.4: 26 Sep 2007
-------------------
-
-- Really added support for TotalView/DDT parallel debugger message queue
-  debugging (it was mistakenly listed as "added" in the 1.2 release).
-- Fixed a build issue with GNU/kFreeBSD. Thanks to Petr Salinger for
-  the patch.
-- Added missing MPI_FILE_NULL constant in Fortran.  Thanks to
-  Bernd Schubert for bringing this to our attention.
-- Change such that the UDAPL BTL is now only built in Linux when
-  explicitly specified via the --with-udapl configure command line
-  switch.
-- Fixed an issue with umask not being propagated when using the TM
-  launcher.
-- Fixed behavior if number of slots is not the same on all bproc nodes.
-- Fixed a hang on systems without GPR support (ex. Cray XT3/4).
-- Prevent users of 32-bit MPI apps from requesting >= 2GB of shared
-  memory.
-- Added a Portals MTL.
-- Fix 0 sized MPI_ALLOC_MEM requests.  Thanks to Lisandro Dalcín for
-  pointing out the problem.
-- Fixed a segfault crash on large SMPs when doing collectives.
-- A variety of fixes for Cray XT3/4 class of machines.
-- Fixed which error handler is used when MPI_COMM_SELF is passed
-  to MPI_COMM_FREE.  Thanks to Lisandro Dalcín for the bug report.
-- Fixed compilation on platforms that don't have hton/ntoh.
-- Fixed a logic problem in the fortran binding for MPI_TYPE_MATCH_SIZE.
-  Thanks to Jeff Dusenberry for pointing out the problem and supplying
-  the fix.
-- Fixed a problem with MPI_BOTTOM in various places of the f77-interface.
-  Thanks to Daniel Spangberg for bringing this up.
-- Fixed problem where MPI-optional Fortran datatypes were not
-  correctly initialized.
-- Fixed several problems with stdin/stdout forwarding.
-- Fixed overflow problems with the sm mpool MCA parameters on large SMPs.
-- Added support for the DDT parallel debugger via orterun's --debug
-  command line option.
-- Added some sanity/error checks to the openib MCA parameter parsing
-  code.
-- Updated the udapl BTL to use RDMA capabilities.
-- Allow use of the BProc head node if it was allocated to the user.
-  Thanks to Sean Kelly for reporting the problem and helping debug it.
-- Fixed a ROMIO problem where non-blocking I/O errors were not properly
-  reported to the user.
-- Made remote process launch check the $SHELL environment variable if
-  a valid shell was not otherwise found for the user.
-  Thanks to Alf Wachsmann for the bugreport and suggested fix.
-- Added/updated some vendor IDs for a few openib HCAs.
-- Fixed a couple of failures that could occur when specifying devices
-  for use by the OOB.
-- Removed dependency on sysfsutils from the openib BTL for
-  libibverbs >=v1.1 (i.e., OFED 1.2 and beyond).
-
-
-1.2.3: 20 Jun 2007
-------------------
-
-- Fix a regression in comm_spawn functionality that inadvertently
-  caused the mapping of child processes to always start at the same
-  place.  Thanks to Prakash Velayutham for helping discover the
-  problem.
-- Fix segfault when a user's home directory is unavailable on a remote
-  node.  Thanks to Guillaume Thomas-Collignon for bringing the issue
-  to our attention.
-- Fix MPI_IPROBE to properly handle MPI_STATUS_IGNORE on mx and psm
-  MTLs. Thanks to Sophia Corwell for finding this and supplying a
-  reproducer.
-- Fix some error messages in the tcp BTL.
-- Use _NSGetEnviron instead of environ on Mac OS X so that there
-  are no undefined symbols in the shared libraries.
-- On OS X, when MACOSX_DEPLOYMENT_TARGET is 10.3 or higher, support
-  building the Fortran 90 bindings as a shared library.  Thanks to
-  Jack Howarth for his advice on making this work.
-- No longer require extra include flag for the C++ bindings.
-- Fix detection of weak symbols support with Intel compilers.
-- Fix issue found by Josh England: ompi_info would not show framework
-  MCA parameters set in the environment properly.
-- Rename the oob_tcp_include/exclude MCA params to oob_tcp_if_include/exclude
-  so that they match the naming convention of the btl_tcp_if_include/exclude
-  params.  The old names are depreciated, but will still work.
-- Add -wd as a synonym for the -wdir orterun/mpirun option.
-- Fix the mvapi BTL to compile properly with compilers that do not support
-  anonymous unions.  Thanks to Luis Kornblueh for reporting the bug.
-
-
-1.2.2: 16 May 2007
-------------------
-
-- Fix regression in 1.2.1 regarding the handling of $CC with both
-  absolute and relative path names.
-- Fix F90 array of status dimensions.  Thanks to Randy Bramley for
-  noticing the problem.
-- Add btl_openib_ib_pkey_value MCA parameter for controlling IB port selection.
-- Fixed a variety of threading/locking bugs.
-- Fixed some compiler warnings associated with ROMIO, OS X, and gridengine.
-- If pbs-config can be found, use it to look for TM support.  Thanks
-  to Bas van der Vlies for the inspiration and preliminary work.
-- Fixed a deadlock in orterun when the rsh PLS encounters some errors.
-
-
-1.2.1: 25 Apr 2007
-------------------
-
-- Fixed a number of connection establishment errors in the TCP out-
-  of-band messaging system.
-- Fixed a memory leak when using mpi_comm calls.
-  Thanks to Bas van der Vlies for reporting the problem.
-- Fixed various memory leaks in OPAL and ORTE.
-- Improved launch times when using TM (PBS Pro, Torque, Open PBS).
-- Fixed mpi_leave_pinned to work for all datatypes.
-- Fix functionality allowing users to disable sbrk() (the
-  mpool_base_disable_sbrk MCA parameter) on platforms that support it.
-- Fixed a pair of problems with the TCP "listen_thread" mode for the
-  oob_tcp_listen_mode MCA parameter that would cause failures when
-  attempting to launch applications.
-- Fixed a segfault if there was a failure opening a BTL MX endpoint.
-- Fixed a problem with mpirun's --nolocal option introduced in 1.2.
-- Re-enabled MPI_COMM_SPAWN_MULTIPLE from singletons.
-- LoadLeveler and TM configure fixes, Thanks to Martin Audet for the
-  bug report.
-- Various C++ MPI attributes fixes.
-- Fixed issues with backtrace code on 64 bit Intel & PPC OS X builds.
-- Fixed issues with multi-word CC variables and libtool.
-  Thanks to Bert Wesarg for the bug reports.
-- Fix issue with non-uniform node naming schemes in SLURM.
-- Fix file descriptor leak in the Grid Engine/N1GE support.
-- Fix compile error on OS X 10.3.x introduced with Open MPI 1.1.5.
-- Implement MPI_TYPE_CREATE_DARRAY function (was in 1.1.5 but not 1.2).
-- Recognize zsh shell when using rsh/ssh for launching MPI jobs.
-- Ability to set the OPAL_DESTDIR or OPAL_PREFIX environment
-  variables to "re-root" an existing Open MPI installation.
-- Always include -I for Fortran compiles, even if the prefix is
-  /usr/local.
-- Support for "fork()" in MPI applications that use the
-  OpenFabrics stack (OFED v1.2 or later).
-- Support for setting specific limits on registered memory.
-
-
-1.2: 15 Mar 2007
-----------------
-
-- Fixed race condition in the shared memory fifo's, which led to
-  orphaned messages.
-- Corrected the size of the shared memory file - subtracted out the
-  space the header was occupying.
-- Add support for MPI_2COMPLEX and MPI_2DOUBLE_COMPLEX.
-- Always ensure to create $(includedir)/openmpi, even if the C++
-  bindings are disabled so that the wrapper compilers don't point to
-  a directory that doesn't exist.  Thanks to Martin Audet for
-  identifying the problem.
-- Fixes for endian handling in MPI process startup.
-- Openib BTL initialization fixes for cases where MPI processes in the
-  same job has different numbers of active ports on the same physical
-  fabric.
-- Print more descriptive information when displaying backtraces on
-  OS's that support this functionality, such as the hostname and PID
-  of the process in question.
-- Fixes to properly handle MPI exceptions in C++ on communicators,
-  windows, and files.
-- Much more reliable runtime support, particularly with regards to MPI
-  job startup scalability, BProc support, and cleanup in failure
-  scenarios (e.g., MPI_ABORT, MPI processes abnormally terminating,
-  etc.).
-- Significant performance improvements for MPI collectives,
-  particularly on high-speed networks.
-- Various fixes in the MX BTL component.
-- Fix C++ typecast problems with MPI_ERRCODES_IGNORE.  Thanks to
-  Satish Balay for bringing this to our attention.
-- Allow run-time specification of the maximum amount of registered
-  memory for OpenFabrics and GM.
-- Users who utilize the wrapper compilers (e.g., mpicc and mpif77)
-  will not notice, but the underlying library names for ORTE and OPAL
-  have changed to libopen-rte and libopen-pal, respectively (listed
-  here because there are undoubtedly some users who are not using the
-  wrapper compilers).
-- Many bug fixes to MPI-2 one-sided support.
-- Added support for TotalView message queue debugging.
-- Fixes for MPI_STATUS_SET_ELEMENTS.
-- Print better error messages when mpirun's "-nolocal" is used when
-  there is only one node available.
-- Added man pages for several Open MPI executables and the MPI API
-  functions.
-- A number of fixes for Alpha platforms.
-- A variety of Fortran API fixes.
-- Build the Fortran MPI API as a separate library to allow these
-  functions to be profiled properly.
-- Add new --enable-mpirun-prefix-by-default configure option to always
-  imply the --prefix option to mpirun, preventing many rsh/ssh-based
-  users from needing to modify their shell startup files.
-- Add a number of missing constants in the C++ bindings.
-- Added tight integration with Sun N1 Grid Engine (N1GE) 6 and the
-  open source Grid Engine.
-- Allow building the F90 MPI bindings as shared libraries for most
-  compilers / platforms.  Explicitly disallow building the F90
-  bindings as shared libraries on OS X because of complicated
-  situations with Fortran common blocks and lack of support for
-  unresolved common symbols in shared libraries.
-- Added stacktrace support for Solaris and Mac OS X.
-- Update event library to libevent-1.1b.
-- Fixed standards conformance issues with MPI_ERR_TRUNCATED and
-  setting MPI_ERROR during MPI_TEST/MPI_WAIT.
-- Addition of "cm" PML to better support library-level matching
-  interconnects, with support for Myrinet/MX, and QLogic PSM-based
-  networks.
-- Addition of "udapl" BTL for transport across uDAPL interconnects.
-- Really check that the $CXX given to configure is a C++ compiler
-  (not a C compiler that "sorta works" as a C++ compiler).
-- Properly check for local host only addresses properly, looking
-  for 127.0.0.0/8, rather than just 127.0.0.1.
-
-
-1.1.5: 19 Mar 2007
-------------------
-
-- Implement MPI_TYPE_CREATE_DARRAY function.
-- Fix race condition in shared memory BTL startup that could cause MPI
-  applications to hang in MPI_INIT.
-- Fix syntax error in a corner case of the event library.  Thanks to
-  Bert Wesarg for pointing this out.
-- Add new MCA parameter (mpi_preconnect_oob) for pre-connecting the
-  "out of band" channels between all MPI processes.  Most helpful for
-  MPI applications over InfiniBand where process A sends an initial
-  message to process B, but process B does not enter the MPI library
-  for a long time.
-- Fix for a race condition in shared memory locking semantics.
-- Add major, minor, and release version number of Open MPI to mpi.h.
-  Thanks to Martin Audet for the suggestion.
-- Fix the "restrict" compiler check in configure.
-- Fix a problem with argument checking in MPI_TYPE_CREATE_SUBARRAY.
-- Fix a problem with compiling the XGrid components with non-gcc
-  compilers.
-
-
-1.1.4: 30 Jan 2007
-------------------
-
-- Fixed 64-bit alignment issues with TCP interface detection on
-  intel-based OS X machines.
-- Adjusted TCP interface selection to automatically ignore Linux
-  channel-bonded slave interfaces.
-- Fixed the type of the first parameter to the MPI F90 binding for
-  MPI_INITIALIZED.  Thanks to Tim Campbell for pointing out the
-  problem.
-- Fix a bunch of places in the Fortran MPI bindings where (MPI_Fint*)
-  was mistakenly being used instead of (MPI_Aint*).
-- Fixes for fortran MPI_STARTALL, which could sometimes return
-  incorrect request values.  Thanks to Tim Campbell for pointing out
-  the problem.
-- Include both pre- and post-MPI-2 errata bindings for
-  MPI::Win::Get_attr.
-- Fix math error on Intel OS X platforms that would greatly increase
-  shared memory latency.
-- Fix type casting issue with MPI_ERRCODES_IGNORE that would cause
-  errors when using a C++ compiler.  Thanks to Barry Smith for
-  bringing this to our attention.
-- Fix possible segmentation fault during shutdown when using the
-  MX BTL.
-
-
-1.1.3: 26 Jan 2007
-------------------
-
-- Remove the "hierarch" coll component; it was not intended to be
-  included in stable releases yet.
-- Fix a race condition with stdout/stderr not appearing properly from
-  all processes upon termination of an MPI job.
-- Fix internal accounting errors with the self BTL.
-- Fix typos in the code path for when sizeof(int) != sizeof(INTEGER)
-  in the MPI F77 bindings functions.  Thanks to Pierre-Matthieu
-  Anglade for bringing this problem to our attention.
-- Fix for a memory leak in the derived datatype function
-  ompi_ddt_duplicate().  Thanks to Andreas Schäfer for reporting,
-  diagnosing, and patching the leak.
-- Used better performing basic algorithm for MPI_ALLGATHERV.
-- Added a workaround for a bug in the Intel 9.1 C++ compiler (all
-  versions up to and including 20060925) in the MPI C++ bindings that
-  caused run-time failures.  Thanks to Scott Weitzenkamp for reporting
-  this problem.
-- Fix MPI_SIZEOF implementation in the F90 bindings for COMPLEX
-  variable types.
-- Fixes for persistent requests involving MPI_PROC_NULL.  Thanks to
-  Lisandro Dalcín for reporting the problem.
-- Fixes to MPI_TEST* and MPI_WAIT* for proper MPI exception reporting.
-  Thanks to Lisandro Dalcín for finding the issue.
-- Various fixes for MPI generalized request handling; addition of
-  missing MPI::Grequest functionality to the C++ bindings.
-- Add "mpi_preconnect_all" MCA parameter to force wireup of all MPI
-  connections during MPI_INIT (vs. making connections lazily whenever
-  the first MPI communication occurs between a pair of peers).
-- Fix a problem for when $FC and/or $F77 were specified as multiple
-  tokens.  Thanks to Orion Poplawski for identifying the problem and
-  to Ralf Wildenhues for suggesting the fix.
-- Fix several MPI_*ERRHANDLER* functions and MPI_GROUP_TRANSLATE_RANKS
-  with respect to what arguments they allowed and the behavior that
-  they effected.  Thanks to Lisandro Dalcín for reporting the
-  problems.
-
-
-1.1.2: 18 Oct 2006
-------------------
-
-- Really fix Fortran status handling in MPI_WAITSOME and MPI_TESTSOME.
-- Various datatype fixes, reported by several users as causing
-  failures in the BLACS testing suite.  Thanks to Harald Forbert, Åke
-  Sandgren and, Michael Kluskens for reporting the problem.
-- Correctness and performance fixes for heterogeneous environments.
-- Fixed a error in command line parsing on some platforms (causing
-  mpirun to crash without doing anything).
-- Fix for initialization hangs on 64 bit Mac OS X PowerPC systems.
-- Fixed some memory allocation problems in mpirun that could cause
-  random problems if "-np" was not specified on the command line.
-- Add Kerberos authentication support for XGrid.
-- Added LoadLeveler support for jobs larger than 128 tasks.
-- Fix for large-sized Fortran LOGICAL datatypes.
-- Fix various error checking in MPI_INFO_GET_NTHKEY and
-  MPI_GROUP_TRANSLATE_RANKS, and some collective operations
-  (particularly with regards to MPI_IN_PLACE).  Thanks to Lisandro
-  Dalcín for reporting the problems.
-- Fix receiving messages to buffers allocated by MPI_ALLOC_MEM.
-- Fix a number of race conditions with the MPI-2 Onesided
-  interface.
-- Fix the "tuned" collective componenete where some cases where
-  MPI_BCAST could hang.
-- Update TCP support to support non-uniform TCP environments.
-- Allow the "poe" RAS component to be built on AIX or Linux.
-- Only install mpif.h if the rest of the Fortran bindings are
-  installed.
-- Fixes for BProc node selection.
-- Add some missing Fortran MPI-2 IO constants.
-
-
-1.1.1: 28 Aug 2006
-------------------
-
-- Fix for Fortran string handling in various MPI API functions.
-- Fix for Fortran status handling in MPI_WAITSOME and MPI_TESTSOME.
-- Various fixes for the XL compilers.
-- Automatically disable using mallot() on AIX.
-- Memory fixes for 64 bit platforms with registering MCA parameters in
-  the self and MX BTL components.
-- Fixes for BProc to support oversubscription and changes to the
-  mapping algorithm so that mapping processes "by slot" works as
-  expected.
-- Fixes for various abort cases to not hang and clean up nicely.
-- If using the Intel 9.0 v20051201 compiler on an IA64 platform, the
-  ptmalloc2 memory manager component will automatically disable
-  itself.  Other versions of the Intel compiler on this platform seem
-  to work fine (e.g., 9.1).
-- Added "host" MPI_Info key to MPI_COMM_SPAWN and
-  MPI_COMM_SPAWN_MULTIPLE.
-- Add missing C++ methods: MPI::Datatype::Create_indexed_block,
-  MPI::Datatype::Create_resized, MPI::Datatype::Get_true_extent.
-- Fix OSX linker issue with Fortran bindings.
-- Fixed MPI_COMM_SPAWN to start spawning new processes in slots that
-  (according to Open MPI) are not already in use.
-- Added capability to "mpirun a.out" (without specifying -np) that
-  will run on all currently-allocated resources (e.g., within a batch
-  job such as SLURM, Torque, etc.).
-- Fix a bug with one particular case of MPI_BCAST.  Thanks to Doug
-  Gregor for identifying the problem.
-- Ensure that the shared memory mapped file is only created when there
-  is more than one process on a node.
-- Fixed problems with BProc stdin forwarding.
-- Fixed problem with MPI_TYPE_INDEXED datatypes.  Thanks to Yven
-  Fournier for identifying this problem.
-- Fix some thread safety issues in MPI attributes and the openib BTL.
-- Fix the BProc allocator to not potentially use the same resources
-  across multiple ORTE universes.
-- Fix gm resource leak.
-- More latency reduction throughout the code base.
-- Make the TM PLS (PBS Pro, Torque, Open PBS) more scalable, and fix
-  some latent bugs that crept in v1.1.  Thanks to the Thunderbird crew
-  at Sandia National Laboratories and Martin Schaffoner for access to
-  testing facilities to make this happen.
-- Added new command line options to mpirun:
-  --nolocal: Do not run any MPI processes on the same node as mpirun
-    (compatibility with the OSC mpiexec launcher)
-  --nooversubscribe: Abort if the number of processes requested would
-    cause oversubscription
-  --quiet / -q: do not show spurious status messages
-  --version / -V: show the version of Open MPI
-- Fix bus error in XGrid process starter.  Thanks to Frank from the
-  Open MPI user's list for identifying the problem.
-- Fix data size mismatches that caused memory errors on PPC64
-  platforms during the startup of the openib BTL.
-- Allow propagation of SIGUSR1 and SIGUSR2 signals from mpirun to
-  back-end MPI processes.
-- Add missing MPI::Is_finalized() function.
-
-
-1.1: 23 Jun 2006
-----------------
-
-- Various MPI datatype fixes, optimizations.
-- Fixed various problems on the SPARC architecture (e.g., not
-  correctly aligning addresses within structs).
-- Improvements in various run-time error messages to be more clear
-  about what they mean and where the errors are occurring.
-- Various fixes to mpirun's handling of --prefix.
-- Updates and fixes for Cray/Red Storm support.
-- Major improvements to the Fortran 90 MPI bindings:
-  - General improvements in compile/linking time and portability
-    between different F90 compilers.
-  - Addition of "trivial", "small" (the default), and "medium"
-    Fortran 90 MPI module sizes (v1.0.x's F90 module was
-    equivalent to "medium").  See the README file for more
-    explanation.
-  - Fix various MPI F90 interface functions and constant types to
-    match.  Thanks to Michael Kluskens for pointing out the problems
-    to us.
-- Allow short messagees to use RDMA (vs. send/receive semantics) to a
-  limited number peers in both the mvapi and openib BTL components.
-  This reduces communication latency over IB channels.
-- Numerous performance improvements throughout the entire code base.
-- Many minor threading fixes.
-- Add a define OMPI_SKIP_CXX to allow the user to skip the mpicxx.h from
-  being included in mpi.h. It allows the user to compile C code with a CXX
-  compiler without including the CXX bindings.
-- PERUSE support has been added. In order to activate it add
-  --enable-peruse to the configure options. All events described in
-  the PERUSE 2.0 draft are supported, plus one Open MPI
-  extension. PERUSE_COMM_REQ_XFER_CONTINUE allow to see how the data
-  is segmented internally, using multiple interfaces or the pipeline
-  engine. However, this version only support one event of each type
-  simultaneously attached to a communicator.
-- Add support for running jobs in heterogeneous environments.
-  Currently supports environments with different endianness and
-  different representations of C++ bool and Fortran LOGICAL.
-  Mismatched sizes for other datatypes is not supported.
-- Open MPI now includes an implementation of the MPI-2 One-Sided
-  Communications specification.
-- Open MPI is now configurable in cross-compilation environments.
-  Several Fortran 77 and Fortran 90 tests need to be pre-seeded with
-  results from a config.cache-like file.
-- Add --debug option to mpirun to generically invoke a parallel debugger.
-
-
-1.0.3: Not released (all fixes included in 1.1)
------------------------------------------------
-
-- Fix a problem noted by Chris Hennes where MPI_INFO_SET incorrectly
-  disallowed long values.
-- Fix a problem in the launch system that could cause inconsistent
-  launch behavior, particularly when launching large jobs.
-- Require that the openib BTL find <sysfs/libsysfs.h>.  Thanks to Josh
-  Aune for the suggestion.
-- Include updates to support the upcoming Autoconf 2.60 and Libtool
-  2.0.  Thanks to Ralf Wildenhues for all the work!
-- Fix bug with infinite loop in the "round robin" process mapper.
-  Thanks to Paul Donohue for reporting the problem.
-- Enusre that memory hooks are removed properly during MPI_FINALIZE.
-  Thanks to Neil Ludban for reporting the problem.
-- Various fixes to the included support for ROMIO.
-- Fix to ensure that MPI_LONG_LONG and MPI_LONG_LONG_INT are actually
-  synonyms, as defined by the MPI standard.  Thanks to Martin Audet
-  for reporting this.
-- Fix Fortran 90 configure tests to properly utilize LDFLAGS and LIBS.
-  Thanks to Terry Reeves for reporting the problem.
-- Fix shared memory progression in asynchronous progress scenarios.
-  Thanks to Mykael Bouquey for reporting the problem.
-- Fixed back-end operations for predefined MPI_PROD for some
-  datatypes.  Thanks to Bert Wesarg for reporting this.
-- Adapted configure to be able to handle Torque 2.1.0p0's (and above)
-  new library name.  Thanks to Brock Palen for pointing this out and
-  providing access to a Torque 2.1.0p0 cluster to test with.
-- Fixed situation where mpirun could set a shell pipeline's stdout
-  to non-blocking, causing the shell pipeline to prematurely fail.
-  Thanks to Darrell Kresge for figuring out what was happening.
-- Fixed problems with leave_pinned that could cause Badness with the
-  mvapi BTL.
-- Fixed problems with MPI_FILE_OPEN and non-blocking MPI-2 IO access.
-- Fixed various InfiniBand port matching issues during startup.
-  Thanks to Scott Weitzenkamp for identifying these problems.
-- Fixed various configure, build and run-time issues with ROMIO.
-  Thanks to Dries Kimpe for bringing them to our attention.
-- Fixed error in MPI_COMM_SPLIT when dealing with intercommunicators.
-  Thanks to Bert Wesarg for identifying the problem.
-- Fixed backwards handling of "high" parameter in MPI_INTERCOMM_MERGE.
-  Thanks to Michael Kluskens for pointing this out to us.
-- Fixed improper handling of string arguments in Fortran bindings
-  for MPI-IO functionality
-- Fixed segmentation fault with 64 bit applications on Solaris when
-  using the shared memory transports.
-- Fixed MPI_COMM_SELF attributes to free properly at the beginning of
-  MPI_FINALIZE.  Thanks to Martin Audet for bringing this to our
-  attention.
-- Fixed alignment tests for cross-compiling to not cause errors with
-  recent versions of GCC.
-
-
-1.0.2: 7 Apr 2006
------------------
-
-- Fixed assembly race condition on AMD64 platforms.
-- Fixed residual .TRUE. issue with copying MPI attributes set from
-  Fortran.
-- Remove unnecessary logic from Solaris pty I/O forwarding.  Thanks to
-  Francoise Roch for bringing this to our attention.
-- Fixed error when count = 0 was given for multiple completion MPI
-  functions (MPI_TESTSOME, MPI_TESTANY, MPI_TESTALL, MPI_WAITSOME,
-  MPI_WAITANY, MPI_WAITALL).
-- Better handling in MPI_ABORT for when peer processes have already
-  died, especially under some resource managers.
-- Random updates to README file, to include notes about the Portland
-  compilers.
-- Random, small threading fixes to prevent deadlock.
-- Fixed a problem with handling long mpirun app files.  Thanks to Ravi
-  Manumachu for identifying the problem.
-- Fix handling of strings in several of the Fortran 77 bindings.
-- Fix LinuxPPC assembly issues.  Thanks to Julian Seward for reporting
-  the problem.
-- Enable pty support for standard I/O forwarding on platforms that
-  have ptys but do not have openpty().  Thanks to Pierre Valiron for
-  bringing this to our attention.
-- Disable inline assembly for PGI compilers to avoid compiler errors.
-  Thanks to Troy Telford for bringing this to our attention.
-- Added MPI_UNSIGNED_CHAR and MPI_SIGNED_CHAR to the allowed reduction
-  types.
-- Fix a segv in variable-length message displays on Opterons running
-  Solaris.  Thanks to Pierre Valiron for reporting the issue.
-- Added MPI_BOOL to the intrinsic reduction operations MPI_LAND,
-  MPI_LOR, MPI_LXOR.  Thanks to Andy Selle for pointing this out to us.
-- Fixed TCP BTL network matching logic during MPI_INIT; in some cases
-  on multi-NIC nodes, a NIC could get paired with a NIC on another
-  network (typically resulting in deadlock).  Thanks to Ken Mighell
-  for pointing this out to us.
-- Change the behavior of orterun (mpirun, mpirexec) to search for
-  argv[0] and the cwd on the target node (i.e., the node where the
-  executable will be running in all systems except BProc, where the
-  searches are run on the node where orterun is invoked).
-- Fix race condition in shared memory transport that could cause
-  crashes on machines with weak memory consistency models (including
-  POWER/PowerPC machines).
-- Fix warnings about setting read-only MCA parameters on bproc systems.
-- Change the exit status set by mpirun when an application process is
-  killed by a signal.  The exit status is now set to signo + 128, which
-  conforms with the behavior of (almost) all shells.
-- Correct a datatype problem with the convertor when partially
-  unpacking data. Now we can position the convertor to any position
-  not only on the predefined types boundaries. Thanks to Yvan Fournier
-  for reporting this to us.
-- Fix a number of standard I/O forwarding issues, including the
-  ability to background mpirun and a loss of data issue when
-  redirecting mpirun's standard input from a file.
-- Fixed bug in ompi_info where rcache and bml MCA parameters would not
-  be displayed.
-- Fixed umask issues in the session directory.  Thanks to Glenn Morris
-  for reporting this to us.
-- Fixed tcsh-based LD_LIBRARY_PATH issues with --prefix.  Thanks to
-  Glen Morris for identifying the problem and suggesting the fix.
-- Removed extraneous \n's when setting PATH and LD_LIBRARY_PATH in the
-  rsh startup.  Thanks to Glen Morris for finding these typos.
-- Fixed missing constants in MPI C++ bindings.
-- Fixed some errors caused by threading issues.
-- Fixed openib BTL flow control logic to not overrun the number of
-  send wqes available.
-- Update to match newest OpenIB user-level library API.  Thanks to
-  Roland Dreier for submitting this patch.
-- Report errors properly when failing to register memory in the openib
-  BTL.
-- Reduce memory footprint of openib BTL.
-- Fix parsing problem with mpirun's "-tv" switch.  Thanks to Chris
-  Gottbrath for supplying the fix.
-- Fix Darwin net/if.h configure warning.
-- The GNU assembler unbelievably defaults to making stacks executable.
-  So when using gas, add flags to explicitly tell it to not make
-  stacks executable (lame but necessary).
-- Add missing MPI::Request::Get_status() methods.  Thanks to Bill
-  Saphir for pointing this out to us.
-- Improved error messages on memory registration errors (e.g., when
-  using high-speed networks).
-- Open IB support now checks firmware for how many outstanding RDMA
-  requests are supported.  Thanks to Mellanox for pointing this out to
-  us.
-- Enable printing of stack traces in MPI processes upon SIGBUS,
-  SIGSEGV, and SIGFPE if the platform supports it.
-- Fixed F90 compilation support for the Lahey compiler.
-- Fixed issues with ROMIO shared library support.
-- Fixed internal accounting problems with rsh support.
-- Update to GNU Libtool 1.5.22.
-- Fix error in configure script when setting CCAS to ias (the Intel
-  assembler).
-- Added missing MPI::Intercomm collectives.
-- Fixed MPI_IN_PLACE handling for Fortran collectives.
-- Fixed some more C++ const_cast<> issues.  Thanks for Martin Audet
-  (again) for bringing this to our attention.
-- Updated ROMIO with the version from MPICH 1.2.7p1, marked as version
-  2005-06-09.
-- Fixes for some cases where the use of MPI_BOTTOM could cause
-  problems.
-- Properly handle the case where an mVAPI does not have shared receive
-  queue support (such as the one shipped by SilverStorm / Infinicon
-  for OS X).
-
-
-1.0.1: 12 Dec 2005
-------------------
-
-- Fixed assembly on Solaris AMD platforms.  Thanks to Pierre Valiron
-  for bringing this to our attention.
-- Fixed long messages in the send-to-self case.
-- Ensure that when the "leave_pinned" option is used, the memory hooks
-  are also enabled.  Thanks to Gleb Natapov for pointing this out.
-- Fixed compile errors for IRIX.
-- Allow hostfiles to have integer host names (for BProc clusters).
-- Fixed a problem with message matching of out-of-order fragments in
-  multiple network device scenarios.
-- Converted all the C++ MPI bindings to use proper const_cast<>'s
-  instead of old C-style casts to get rid of const-ness.  Thanks to
-  Martin Audet for raising the issue with us.
-- Converted MPI_Offset to be a typedef instead of a #define because it
-  causes problems for some C++ parsers.  Thanks to Martin Audet for
-  bringing this to our attention.
-- Improved latency of TCP BTL.
-- Fixed index value in MPI_TESTANY to be MPI_UNDEFINED if some
-  requests were not MPI_REQUEST_NULL, but no requests finished.
-- Fixed several Fortran MPI API implementations that incorrectly used
-  integers instead of logicals or address-sized integers.
-- Fix so that Open MPI correctly handles the Fortran value for .TRUE.,
-  regardless of what the Fortran compiler's value for .TRUE. is.
-- Improved scalability of MX startup.
-- Fix datatype offset handling in the coll basic component's
-  MPI_SCATTERV implementation.
-- Fix EOF handling on stdin.
-- Fix missing MPI_F_STATUS_IGNORE and MPI_F_STATUSES_IGNORE
-  instanatiations.  Thanks to Anthony Chan for pointing this out.
-- Add a missing value for MPI_WIN_NULL in mpif.h.
-- Bring over some fixes for the sm btl that somehow didn't make it
-  over from the trunk before v1.0.  Thanks to Beth Tibbitts and Bill
-  Chung for helping identify this issue.
-- Bring over some fixes for the iof that somehow didn't make it over
-  from the trunk before v1.0.
-- Fix for --with-wrapper-ldflags handling.  Thanks to Dries Kimpe for
-  pointing this out to us.
-
-
-1.0: 17 Nov 2005
-----------------
-
-Initial public release.
diff --git a/README.FT.ULFM.md b/README.FT.ULFM.md
deleted file mode 100644
index ae727a7f6a9..00000000000
--- a/README.FT.ULFM.md
+++ /dev/null
@@ -1,483 +0,0 @@
-ULFM Open MPI
-
-This README.md documents the features and options specific to the
-**User Level Failure Mitigation (ULFM)** Open MPI implementation.
-The upstream (i.e. non-resilient) Open MPI directions also apply to
-this release, except when specified here, and can be found in its
-README file.
-
-[TOC]
-
-Features
-========
-This implementation conforms to the User Level Failure Mitigation (ULFM)
-MPI Standard draft proposal. The ULFM proposal is developed by the MPI
-Forum's Fault Tolerance Working Group to support the continued operation of
-MPI programs after crash (node failures) have impacted the execution. The key
-principle is that no MPI call (point-to-point, collective, RMA, IO, ...) can
-block indefinitely after a failure, but must either succeed or raise an MPI
-error.
-
-This implementation produces the three supplementary error codes and five
-supplementary interfaces defined in the communicator section of the
-[http://fault-tolerance.org/wp-content/uploads/2012/10/20170221-ft.pdf]
-(ULFM chapter) standard draft document.
-
-+ `MPIX_ERR_PROC_FAILED` when a process failure prevents the completion of
-  an MPI operation.
-+ `MPIX_ERR_PROC_FAILED_PENDING` when a potential sender matching a non-blocking
-  wildcard source receive has failed.
-+ `MPIX_ERR_REVOKED` when one of the ranks in the application has invoked the
-  `MPI_Comm_revoke` operation on the communicator.
-+ `MPIX_Comm_revoke(MPI_Comm comm)` Interrupts any communication pending on
-  the communicator at all ranks.
-+ `MPIX_Comm_shrink(MPI_Comm comm, MPI_Comm* newcomm)` creates a new
-  communicator where dead processes in comm were removed.
-+ `MPIX_Comm_agree(MPI_Comm comm, int *flag)` performs a consensus (i.e. fault
-  tolerant allreduce operation) on flag (with the operation bitwise AND).
-+ `MPIX_Comm_failure_get_acked(MPI_Comm, MPI_Group*)` obtains the group of
-  currently acknowledged failed processes.
-+ `MPIX_Comm_failure_ack(MPI_Comm)` acknowledges that the application intends
-  to ignore the effect of currently known failures on wildcard receive
-  completions and agreement return values.
-
-## Supported Systems
-There are several MPI engines available in Open MPI,
-notably, PML "ob1", "cm", "ucx", and MTL "ofi", "portals4", "psm2".
-At this point, only "ob1" is adapted to support fault tolerance.
-
-"ob1" uses BTL ("Byte Transfer Layer") components for each supported
-network. "ob1" supports a variety of networks that can be used in
-combination with each other. Collective operations (blocking and
-non-blocking) use an optimized implementation on top of "ob1".
-
-- Loopback (send-to-self)
-- TCP
-- UCT (InfiniBand)
-- uGNI (Cray Gemini, Aries)
-- Shared Memory (FT supported w/CMA and XPmem; KNEM is untested)
-- Tuned and non-blocking collective communications
-
-A full list of supported, untested and disabled components is provided
-later in this document.
-
-## More Information
-More information (tutorials, examples, build instructions for leading
-top500 systems) is also available in the Fault Tolerance Research
-Hub website:
-  <https://fault-tolerance.org>
-
-## Bibliographic References
-If you are looking for, or want to cite a general reference for ULFM,
-please use
-
-_Wesley Bland, Aurelien Bouteiller, Thomas Herault, George Bosilca, Jack
-J. Dongarra: Post-failure recovery of MPI communication capability: Design
-and rationale. IJHPCA 27(3): 244-254 (2013)._
-
-Available from: http://journals.sagepub.com/doi/10.1177/1094342013488238.
-___________________________________________________________________________
-
-Building ULFM Open MPI
-======================
-```bash
-./configure --with-ft [...options...]
-#    use --with-ft to enable building with ULFM (default),
-#        --without-ft to disable it
-make [-j N] all install
-#    use an integer value of N for parallel builds
-```
-There are many available configure options (see `./configure --help`
-for a full list); a summary of the more commonly used ones is included
-in the upstream Open MPI README file. The following paragraph gives a
-summary of ULFM Open MPI specific options behavior.
-
-## Configure options
-+ `--with-ft=TYPE`
-  Specify the type of fault tolerance to enable.  Options: mpi (ULFM MPI
-  draft standard).  Fault tolerance build support is **enabled by default**.
-
-+ `--enable-mca-no-build=LIST`
-  Comma-separated list of _<type>-<component>_ pairs that will not be built.
-  For example, `--enable-mca-no-build=btl-portals,oob-ud` will disable
-  building the _portals BTL_ and the _ud OOB_ component. You can use this
-  option to disable components that are known to prevent supporting failure
-  management __when built-in__. By default, this list is empty, as ulfm has
-  a separate mechanism to warn about, or disable loading a component that
-  are poorly tested or known to cause breakage when fault-tolerance is
-  selected at runtime.
-
-+ `--with-pmi`
-  `--with-slurm`
-  Force the building of SLURM scheduler support.
-  Slurm with fault tolerance is tested. **Do not use `srun`**, otherwise your
-  application gets killed by the scheduler upon the first failure. Instead,
-  **Use `mpirun` in an `salloc/sbatch`**.
-
-+ `-with-lsf`
-  This is untested with fault tolerance.
-
-+ `--with-alps`
-  `--with-tm`
-  Force the building of PBS/Torque scheduler support.
-  PBS is tested with fault tolerance. **Use `mpirun` in a `qsub`
-  allocation.**
-
-+ `--disable-oshmem`
-  Disable building the OpenSHMEM implementation (by default, it is
-  enabled).
-  ULFM Fault Tolerance does not apply to OpenSHMEM.
-___________________________________________________________________________
-
-## Modified, Untested and Disabled Components
-Frameworks and components which are not listed in the following list are
-unmodified and support fault tolerance. Listed frameworks may be **modified**
-(and work after a failure), **untested** (and work before a failure, but may
-malfunction after a failure), or **disabled** (they cause unspecified behavior
-all around when FT is enabled).
-
-All runtime disabled components are listed in the `ft-mpi` aggregate MCA param file
-`$installdir/share/openmpi/amca-param-sets/ft-mpi`. You can tune the runtime behavior
-with ULFM by either setting or unsetting variables in this file (or by overiding
-the variable on the command line (e.g., ``--omca btl ofi,self``). Note that if FT is
-runtime disabled, these components will load normally (this may change observed
-performance when comparing with and without fault tolerance).
-
-- **pml** MPI point-to-point management layer
-    - "ob1" modified to **handle errors**
-    - "monitoring", "v" unmodified, **untested**
-    - "cm", "crcpw", "ucx" **disabled**
-
-- **btl** Point-to-point Byte Transfer Layer
-    - "ugni", "uct", "tcp", "sm(+cma,+xpmem)" modified to **handle errors** (removed
-      unconditional abort on error, expect performance similar to upstream)
-    - "ofi", "portals4", "smcuda", "usnic", "sm(+knem)" unmodified,
-      **untested** (may work properly, please report)
-
-- **mtl** Matching transport layer Used for MPI point-to-point messages on
-  some types of networks
-    - All "mtl" components are **disabled**
-
-- **coll** MPI collective algorithms
-    - "base", "basic", "tuned", "nbc" modified to **handle errors**
-    - "cuda", "inter", "sync", "sm" unmodified, **untested** (expect correct post-failure behavior)
-    - "hcoll", "portals4" unmodified, **disabled** (expect
-      unspecified post-failure behavior)
-
-- **osc** MPI one-sided communications
-    - Unmodified, **untested** (expect unspecified post-failure behavior)
-
-- **io** MPI I/O and dependent components
-    - _fs_ File system functions for MPI I/O
-    - _fbtl_ File byte transfer layer: abstraction for individual read/write
-      operations for OMPIO
-    - _fcoll_ Collective read and write operations for MPI I/O
-    - _sharedfp_ Shared file pointer operations for MPI I/O
-    - All components in these frameworks are unmodified, **untested**
-      (expect clean post-failure abort)
-
-- **vprotocol** Checkpoint/Restart components
-    - unmodified **untested**
-
-- **threads** `wait-sync` Multithreaded wait-synchronization object
-    - "pthreads" modified to **handle errors** (added a global interrupt to 
-       trigger all wait_sync objects)
-    - "argotbots", "qthreads" unmodified, **disabled** (expect post-failure
-      deadlock)
-
-___________________________________________________________________________
-
-Running ULFM Open MPI
-=====================
-
-## Building your application
-
-As ULFM is still an extension to the MPI standard, you will need to
-`#include <mpi-ext.h>` in C, or `use mpi_ext` in Fortran to access the
-supplementary error codes and functions.
-
-Compile your application as usual, using the provided `mpicc`, `mpif90`, or
-`mpicxx` wrappers.
-
-## Running your application
-
-You can launch your application with fault tolerance by simply using the
-provided `mpiexec`. Beware that your distribution may already provide a
-version of MPI, make sure to set your `PATH` and `LD_LIBRARY_PATH` properly.
-Note that fault tolerance is disabled by default in ULFM Open MPI; you can
-enable the fault tolerance components by launching your application with
-`mpiexec --enable-recovery`.
-
-## Running under a batch scheduler
-
-ULFM can operate under a job/batch scheduler, and is tested routinely with
-both ALPS, PBS and Slurm. One difficulty comes from the fact that many job
-schedulers will "cleanup" the application as soon as a process fails. In
-order to avoid this problem, it is preferred that you use `mpiexec`
-within an allocation (e.g. `salloc`, `sbatch`, `qsub`) rather than
-a direct launch (e.g. `srun`).
-
-## Run-time tuning knobs
-
-ULFM comes with a variety of knobs for controlling how it runs. The default
-parameters are sane and should result in very good performance in most
-cases. You can change the default settings with `--omca mpi_ft_foo <value>`
-for OMPI options, and with `--prtemca errmgr_detector_bar <value>` for
-PRTE options.
-
-### PRTE level options
-
-- `prrte_enable_recovery <true|false> (default: false)` controls automatic
-  cleanup of apps with failed processes within mpirun. Enabling this option 
-  also enable `mpi_ft_enable`.
-- `errmgr_detector_priority <int> (default 1005`) selects the prte-based
-  failure detector. Only available when `prte_enable_recovery` is `true`. You
-  can set this to `0` when using the (experimental) OMPI detector instead.
-- err`mgr_detector_heartbeat_period <float> (default: 5e0)` controls the
-  heartbeat period. Recommended value is 1/2 of the timeout.
-- `errmgr_detector_heartbeat_timeout <float> (default: 1e1 seconds)` heartbeat
-  timeout (i.e. failure detection speed). Recommended value is 2 times
-  the heartbeat period. The default setup is tuned for failure-free
-  performance at the expense of fault detection reactivity. In environments
-  where faults are expected to be common, less conservative values can be
-  used (e.g., 100ms); Values lower than the TCP poll rate (typically 10ms)
-  can cause false positive.
-
-### OMPI level options
-
-- `mpi_ft_enable <true|false> (default: same as prrte_enable_recovery)`
-  permits turning on/off fault tolerance at runtime. When false, failure
-  detection is disabled; Interfaces defined by the fault tolerance extensions
-  are substituted with dummy non-fault tolerant implementations (e.g.,
-  `MPIX_Comm_agree` is implemented with `MPI_Allreduce`); All other controls
-  below become irrelevant.
-- `mpi_ft_verbose <int> (default: 0)` increases the output of the fault
-  tolerance activities. A value of 1 will report detected failures.
-- `mpi_ft_detector <true|false> (default: false)`, EXPERIMENTAL, controls
-  the activation of the OMPI level failure detector. When this detector
-  is turned off, all failure detection is delegated to PRTE (see above).
-  The OMPI level fault detector is experimental. There is a tradeoff between
-  failure detection accuracy and performance with this detector. Users that
-  experience accuracy issues may enable a more precise mode.
-  See the tuning knobs below to adjust to taste;
-  The OMPI failure detector operates on MPI_COMM_WORLD exclusively.
-  Processes connected from MPI_COMM_CONNECT/ACCEPT and MPI_COMM_SPAWN may
-  occasionally not be detected when they fail.
-- `mpi_ft_detector_thread <true|false> (default: false)` controls the use
-  of a thread to emit and receive failure detector's heartbeats. _Setting
-  this value to "true" will also set `MPI_THREAD_MULTIPLE` support, which
-  has a noticeable effect on latency (typically 1us increase)._ You may
-  want to **enable this option if you experience false positive**
-  processes incorrectly reported as failed with the OMPI failure detector.
-- `mpi_ft_detector_period <float> (default: 3e0 seconds)` heartbeat
-  period. Recommended value is 1/3 of the timeout. _Values lower than
-  100us may impart a noticeable effect on latency (typically a 3us
-  increase)._
-- `mpi_ft_detector_timeout <float> (default: 1e1 seconds)` heartbeat
-  timeout (i.e. failure detection speed). Recommended value is 3 times
-  the heartbeat period.
-
-## Known Limitations in ULFM
-
-- Infiniband support is provided through the OpenIB or UCT BTL, fault
-  tolerant operation over the UCX PML is not yet supported.
-- TOPO, FILE, RMA are not fault tolerant. They are expected to work properly
-  before the occurence of the first failure.
-
-___________________________________________________________________________
-
-
-Changelog
-=========
-
-## ULFM Integrated in Open MPI
-ULFM is now integrated in Open MPI. This text will be updated when a new 
-Open MPI release is made.
-
-## ULFM Standalone Release 4.0.2u1
-This is a stability and upstream parity upgrade. It is based on the most
-current Open MPI Release (v4.0.2, October 2019).
-
-- This release is based on Open MPI release v4.0.2 (ompi #cb5f4e737a).
-- This release is based on ULFM master (ulfm #0e249ca1).
-- New features
-    - Support for the UCT BTL enters beta stage.
-- Bugfixes
-    - High sensitivity to noise in the failure detector.
-    - Deadlocks when revoking while BTL progress threads are updating messages.
-    - A case where the failure detector would keep observing a dead process forever.
-    - Disable the use of external pmix/libevent by default (the internals are modified
-      to handle error cases).
-    - Clean error paths leaving some rdma registration dangling.
-    - Do not remove the orte job/proc session dir prematurely upon error.
-
-## ULFM Standalone Release 4.0.1u1
-This is a stability and upstream parity upgrade. It improves stability, 
-performance and is based on the most current Open MPI Release (v4.0.1,
-May 2019).
-
-- This release is based on Open MPI release v4.0.1 (ompi #b780667).
-- This release is based on ULFM master (ulfm #cf8dc43f).
-- New features
-    - Addition of the `MPI_Comm_is_revoked` function
-    - Renamed `ftbasic` collective component to `ftagree`
-    - Restored the `pcollreq` extension
-- Bugfixes
-    - Failures of node-local siblings were not always detected
-    - Failure propagation and detection was slowed down by trying to 
-      notify known dead processes
-    - There were deadlocks in multithreaded programs
-    - There were issues with PMPI when compiling Fortran Interfaces
-    - There were deadlocks on OS-X
-
-## ULFM Standalone Release 2.1
-This release is a bugfix and upstream parity upgrade. It improves stability,
-performance and is based on the most current Open MPI master (November 2018).
-
-- ULFM is now based upon Open MPI master branch (#37954b5f).
-- ULFM tuning MCA parameters are exposed by `ompi_info`.
-- Fortran 90 bindings have been updated
-- Bugfixes:
-    - Correct the behavior of process placement during an MPI_COMM_SPAWN when 
-      some slots were occcupied by failed processes.
-    - MPI_COMM_SPAWN accepts process placement directives in the Info object.
-    - Fixed deadlocks in some NBC collective operations.
-    - Crashes and deadlocks in MPI_FINALIZE have been resolved.
-    - Any-source requests that returned with an error status of 
-      MPIX_PROC_FAILED_PENDING can now correctly complete during 
-      later MPI_WAIT/TEST.
-
-## ULFM Standalone Release 2.0
-Focus has been toward integration with current Open MPI master (November 2017),
-performance, and stability.
-
-- ULFM is now based upon Open MPI master branch (#689f1be9). It will be
-  regularly updated until it will eventually be merged.
-- Fault Tolerance is enabled by default and is controlled with MCA variables.
-- Added support for multithreaded modes (MPI_THREAD_MULTIPLE, etc.)
-- Added support for non-blocking collective operations (NBC).
-- Added support for CMA shared memory transport (Vader).
-- Added support for advanced failure detection at the MPI level.
-  Implements the algorithm described in "Failure detection and
-  propagation in HPC systems." <https://doi.org/10.1109/SC.2016.26>.
-- Removed the need for special handling of CID allocation.
-- Non-usable components are automatically removed from the build during configure
-- RMA, FILES, and TOPO components are enabled by default, and usage in a fault
-  tolerant execution warns that they may cause undefined behavior after a failure.
-- Bugfixes:
-    - Code cleanup and performance cleanup in non-FT builds; --without-ft at
-      configure time gives an almost stock Open MPI.
-    - Code cleanup and performance cleanup in FT builds with FT runtime disabled;
-      --mca ft_enable_mpi false thoroughly disables FT runtime activities.
-    - Some error cases would return ERR_PENDING instead of ERR_PROC_FAILED in
-      collective operations.
-    - Some test could set ERR_PENDING or ERR_PROC_FAILED instead of
-      ERR_PROC_FAILED_PENDING for ANY_SOURCE receptions.
-___________________________________________________________________________
-
-## ULFM Standalone Release 1.1
-Focus has been toward improving stability, feature coverage for intercomms,
-and following the updated specification for MPI_ERR_PROC_FAILED_PENDING.
-
-- Forked from Open MPI 1.5.5 devel branch
-- Addition of the MPI_ERR_PROC_FAILED_PENDING error code, as per newer specification
-  revision. Properly returned from point-to-point, non-blocking ANY_SOURCE operations.
-- Alias MPI_ERR_PROC_FAILED, MPI_ERR_PROC_FAILED_PENDING and MPI_ERR_REVOKED to the
-  corresponding standard blessed -extension- names MPIX_ERR_xxx.
-- Support for Intercommunicators:
-    - Support for the blocking version of the agreement, MPI_COMM_AGREE on Intercommunicators.
-    - MPI_COMM_REVOKE tested on intercommunicators.
-- Disabled completely (.ompi_ignore) many untested components.
-- Changed the default ORTE failure notification propagation aggregation delay from 1s to 25ms.
-- Added an OMPI internal failure propagator; failure propagation between SM domains is now
-  immediate.
-- Bugfixes:
-    - SendRecv would not always report MPI_ERR_PROC_FAILED correctly.
-    - SendRecv could incorrectly update the status with errors pertaining to the Send portion
-      of the Sendrecv.
-    - Revoked send operations are now always completed or remote cancelled and may not
-      deadlock anymore.
-    - Cancelled send operations to a dead peer will not trigger an assert when the BTL reports
-      that same failure.
-    - Repeat calls to operations returning MPI_ERR_PROC_FAILED will eventually return
-      MPI_ERR_REVOKED when another process revokes the communicator.
-___________________________________________________________________________
-
-## ULFM Standalone Release 1.0
-Focus has been toward improving performance, both before and after the occurence of failures.
-The list of new features includes:
-
-- Support for the non-blocking version of the agreement, MPI_COMM_IAGREE.
-- Compliance with the latest ULFM specification draft. In particular, the
-  MPI_COMM_(I)AGREE semantic has changed.
-- New algorithm to perform agreements, with a truly logarithmic complexity in number of
-  ranks, which translates into huge performance boosts in MPI_COMM_(I)AGREE and
-  MPI_COMM_SHRINK.
-- New algorithm to perform communicator revocation. MPI_COMM_REVOKE performs a reliable
-  broadcast with a fixed maximum output degree, which scales logarithmically with the
-  number of ranks.
-- Improved support for our traditional network layer:
-    - TCP: fully tested
-    - SM: fully tested (with the exception of XPMEM, which remains unsupported)
-- Added support for High Performance networks
-    - Open IB: reasonably tested
-    - uGNI: reasonably tested
-- The tuned collective module is now enabled by default (reasonably tested), expect a
-  huge performance boost compared to the former basic default setting
-    - Back-ported PBS/ALPS fixes from Open MPI
-    - Back-ported OpenIB bug/performance fixes from Open MPI
-    - Improve Context ID allocation algorithm to reduce overheads of Shrink
-    - Miscellaneous bug fixes
-___________________________________________________________________________
-
-## Binary Compatibility
-ULFM Open MPI is binary compatible with any version of Open MPI compatible
-with the underlying Open MPI master branch or release (see
-the binary compatibility and version number section in the upstream Open MPI
-README). That is, applications compiled with a compatible Open MPI can run
-with the ULFM Open MPI `mpirun` and MPI libraries. Conversely, _as long as
-the application does not employ one of the MPIX functions_, which are
-exclusively defined in ULFM Open MPI, an application compiled with
-ULFM Open MPI can be launched with a compatible Open MPI `mpirun` and run
-with the non-fault tolerant MPI library.
-___________________________________________________________________________
-
-Contacting the Authors
-======================
-Found a bug?  Got a question?  Want to make a suggestion?  Want to
-contribute to ULFM Open MPI?  Working on a cool use-case?
-Please let us know!
-
-The best way to report bugs, send comments, or ask questions is to
-sign up on the user's mailing list:
-  <ulfm+subscribe@googlegroups.com>
-
-Because of spam, only subscribers are allowed to post to these lists
-(ensure that you subscribe with and post from exactly the same e-mail
-address -- joe@example.com is considered different than
-joe@mycomputer.example.com!).  Visit these pages to subscribe to the
-lists:
-  <https://groups.google.com/forum/#!forum/ulfm>
-
-When submitting questions and problems, be sure to include as much
-extra information as possible.  This web page details all the
-information that we request in order to provide assistance:
-  <http://www.open-mpi.org/community/help/>
-
-Thanks for your time.
-___________________________________________________________________________
-
-Copyright
-=========
-
-```
-Copyright (c) 2012-2020 The University of Tennessee and The University
-                        of Tennessee Research Foundation.  All rights
-                        reserved.
-
-$COPYRIGHT$
-
-Additional copyrights may follow
-
-$HEADER$
-```
diff --git a/README.JAVA.md b/README.JAVA.md
deleted file mode 100644
index 234c7a6a1c6..00000000000
--- a/README.JAVA.md
+++ /dev/null
@@ -1,281 +0,0 @@
-# Open MPI Java Bindings
-
-## Important node
-
-JAVA BINDINGS ARE PROVIDED ON A "PROVISIONAL" BASIS - I.E., THEY ARE
-NOT PART OF THE CURRENT OR PROPOSED MPI STANDARDS. THUS, INCLUSION OF
-JAVA SUPPORT IS NOT REQUIRED BY THE STANDARD. CONTINUED INCLUSION OF
-THE JAVA BINDINGS IS CONTINGENT UPON ACTIVE USER INTEREST AND
-CONTINUED DEVELOPER SUPPORT.
-
-## Overview
-
-This version of Open MPI provides support for Java-based
-MPI applications.
-
-The rest of this document provides step-by-step instructions on
-building OMPI with Java bindings, and compiling and running Java-based
-MPI applications. Also, part of the functionality is explained with
-examples. Further details about the design, implementation and usage
-of Java bindings in Open MPI can be found in [1]. The bindings follow
-a JNI approach, that is, we do not provide a pure Java implementation
-of MPI primitives, but a thin layer on top of the C
-implementation. This is the same approach as in mpiJava [2]; in fact,
-mpiJava was taken as a starting point for Open MPI Java bindings, but
-they were later totally rewritten.
-
-1. O. Vega-Gisbert, J. E. Roman, and J. M. Squyres. "Design and
-   implementation of Java bindings in Open MPI". Parallel Comput.
-   59: 1-20 (2016).
-2. M. Baker et al. "mpiJava: An object-oriented Java interface to
-   MPI". In Parallel and Distributed Processing, LNCS vol. 1586,
-   pp. 748-762, Springer (1999).
-
-## Building Java Bindings
-
-If this software was obtained as a developer-level checkout as opposed
-to a tarball, you will need to start your build by running
-`./autogen.pl`. This will also require that you have a fairly recent
-version of GNU Autotools on your system - see the HACKING.md file for
-details.
-
-Java support requires that Open MPI be built at least with shared libraries
-(i.e., `--enable-shared`) - any additional options are fine and will not
-conflict. Note that this is the default for Open MPI, so you don't
-have to explicitly add the option. The Java bindings will build only
-if `--enable-mpi-java` is specified, and a JDK is found in a typical
-system default location.
-
-If the JDK is not in a place where we automatically find it, you can
-specify the location. For example, this is required on the Mac
-platform as the JDK headers are located in a non-typical location. Two
-options are available for this purpose:
-
-1. `--with-jdk-bindir=<foo>`: the location of `javac` and `javah`
-1. `--with-jdk-headers=<bar>`: the directory containing `jni.h`
-
-For simplicity, typical configurations are provided in platform files
-under `contrib/platform/hadoop`. These will meet the needs of most
-users, or at least provide a starting point for your own custom
-configuration.
-
-In summary, therefore, you can configure the system using the
-following Java-related options:
-
-```
-$ ./configure --with-platform=contrib/platform/hadoop/<your-platform> ...
-
-````
-
-or
-
-```
-$ ./configure --enable-mpi-java --with-jdk-bindir=<foo> --with-jdk-headers=<bar> ...
-```
-
-or simply
-
-```
-$ ./configure --enable-mpi-java ...
-```
-
-if JDK is in a "standard" place that we automatically find.
-
-## Running Java Applications
-
-For convenience, the `mpijavac` wrapper compiler has been provided for
-compiling Java-based MPI applications. It ensures that all required MPI
-libraries and class paths are defined. You can see the actual command
-line using the `--showme` option, if you are interested.
-
-Once your application has been compiled, you can run it with the
-standard `mpirun` command line:
-
-```
-$ mpirun <options> java <your-java-options> <my-app>
-```
-
-For convenience, `mpirun` has been updated to detect the `java` command
-and ensure that the required MPI libraries and class paths are defined
-to support execution. You therefore do _NOT_ need to specify the Java
-library path to the MPI installation, nor the MPI classpath. Any class
-path definitions required for your application should be specified
-either on the command line or via the `CLASSPATH` environment
-variable. Note that the local directory will be added to the class
-path if nothing is specified.
-
-As always, the `java` executable, all required libraries, and your
-application classes must be available on all nodes.
-
-## Basic usage of Java bindings
-
-There is an MPI package that contains all classes of the MPI Java
-bindings: `Comm`, `Datatype`, `Request`, etc. These classes have a
-direct correspondence with classes defined by the MPI standard. MPI
-primitives are just methods included in these classes. The convention
-used for naming Java methods and classes is the usual camel-case
-convention, e.g., the equivalent of `MPI_File_set_info(fh,info)` is
-`fh.setInfo(info)`, where `fh` is an object of the class `File`.
-
-Apart from classes, the MPI package contains predefined public
-attributes under a convenience class `MPI`. Examples are the
-predefined communicator `MPI.COMM_WORLD` or predefined datatypes such
-as `MPI.DOUBLE`. Also, MPI initialization and finalization are methods
-of the `MPI` class and must be invoked by all MPI Java
-applications. The following example illustrates these concepts:
-
-```java
-import mpi.*;
-
-class ComputePi {
-
-    public static void main(String args[]) throws MPIException {
-
-        MPI.Init(args);
-
-        int rank = MPI.COMM_WORLD.getRank(),
-            size = MPI.COMM_WORLD.getSize(),
-            nint = 100; // Intervals.
-        double h = 1.0/(double)nint, sum = 0.0;
-
-        for(int i=rank+1; i<=nint; i+=size) {
-            double x = h * ((double)i - 0.5);
-            sum += (4.0 / (1.0 + x * x));
-        }
-
-        double sBuf[] = { h * sum },
-               rBuf[] = new double[1];
-
-        MPI.COMM_WORLD.reduce(sBuf, rBuf, 1, MPI.DOUBLE, MPI.SUM, 0);
-
-        if(rank == 0) System.out.println("PI: " + rBuf[0]);
-        MPI.Finalize();
-    }
-}
-```
-
-## Exception handling
-
-Java bindings in Open MPI support exception handling. By default, errors
-are fatal, but this behavior can be changed. The Java API will throw
-exceptions if the MPI.ERRORS_RETURN error handler is set:
-
-```java
-MPI.COMM_WORLD.setErrhandler(MPI.ERRORS_RETURN);
-```
-
-If you add this statement to your program, it will show the line
-where it breaks, instead of just crashing in case of an error.
-Error-handling code can be separated from main application code by
-means of try-catch blocks, for instance:
-
-```java
-try
-{
-    File file = new File(MPI.COMM_SELF, "filename", MPI.MODE_RDONLY);
-}
-catch(MPIException ex)
-{
-    System.err.println("Error Message: "+ ex.getMessage());
-    System.err.println("  Error Class: "+ ex.getErrorClass());
-    ex.printStackTrace();
-    System.exit(-1);
-}
-```
-
-## How to specify buffers
-
-In MPI primitives that require a buffer (either send or receive) the
-Java API admits a Java array. Since Java arrays can be relocated by
-the Java runtime environment, the MPI Java bindings need to make a
-copy of the contents of the array to a temporary buffer, then pass the
-pointer to this buffer to the underlying C implementation. From the
-practical point of view, this implies an overhead associated to all
-buffers that are represented by Java arrays. The overhead is small
-for small buffers but increases for large arrays.
-
-There is a pool of temporary buffers with a default capacity of 64K.
-If a temporary buffer of 64K or less is needed, then the buffer will
-be obtained from the pool. But if the buffer is larger, then it will
-be necessary to allocate the buffer and free it later.
-
-The default capacity of pool buffers can be modified with an Open MPI
-MCA parameter:
-
-```
-shell$ mpirun --mca mpi_java_eager size ...
-```
-
-Where `size` is the number of bytes, or kilobytes if it ends with 'k',
-or megabytes if it ends with 'm'.
-
-An alternative is to use "direct buffers" provided by standard classes
-available in the Java SDK such as `ByteBuffer`. For convenience we
-provide a few static methods `new[Type]Buffer` in the `MPI` class to
-create direct buffers for a number of basic datatypes. Elements of the
-direct buffer can be accessed with methods `put()` and `get()`, and
-the number of elements in the buffer can be obtained with the method
-`capacity()`. This example illustrates its use:
-
-```java
-int myself = MPI.COMM_WORLD.getRank();
-int tasks  = MPI.COMM_WORLD.getSize();
-
-IntBuffer in  = MPI.newIntBuffer(MAXLEN * tasks),
-          out = MPI.newIntBuffer(MAXLEN);
-
-for(int i = 0; i < MAXLEN; i++)
-    out.put(i, myself);      // fill the buffer with the rank
-
-Request request = MPI.COMM_WORLD.iAllGather(
-                  out, MAXLEN, MPI.INT, in, MAXLEN, MPI.INT);
-request.waitFor();
-request.free();
-
-for(int i = 0; i < tasks; i++)
-{
-    for(int k = 0; k < MAXLEN; k++)
-    {
-        if(in.get(k + i * MAXLEN) != i)
-            throw new AssertionError("Unexpected value");
-    }
-}
-```
-
-Direct buffers are available for: `BYTE`, `CHAR`, `SHORT`, `INT`,
-`LONG`, `FLOAT`, and `DOUBLE`. There is no direct buffer for booleans.
-
-Direct buffers are not a replacement for arrays, because they have
-higher allocation and deallocation costs than arrays. In some
-cases arrays will be a better choice. You can easily convert a
-buffer into an array and vice versa.
-
-All non-blocking methods must use direct buffers and only
-blocking methods can choose between arrays and direct buffers.
-
-The above example also illustrates that it is necessary to call
-the `free()` method on objects whose class implements the `Freeable`
-interface. Otherwise a memory leak is produced.
-
-## Specifying offsets in buffers
-
-In a C program, it is common to specify an offset in a array with
-`&array[i]` or `array+i`, for instance to send data starting from
-a given position in the array. The equivalent form in the Java bindings
-is to `slice()` the buffer to start at an offset. Making a `slice()`
-on a buffer is only necessary, when the offset is not zero. Slices
-work for both arrays and direct buffers.
-
-```java
-import static mpi.MPI.slice;
-// ...
-int numbers[] = new int[SIZE];
-// ...
-MPI.COMM_WORLD.send(slice(numbers, offset), count, MPI.INT, 1, 0);
-```
-
-## Questions?  Problems?
-
-If you have any problems, or find any bugs, please feel free to report
-them to [Open MPI user's mailing
-list](https://www.open-mpi.org/community/lists/ompi.php).
diff --git a/README.md b/README.md
index 572ee880d14..99f0124ebfb 100644
--- a/README.md
+++ b/README.md
@@ -1,2191 +1,31 @@
 # Open MPI
 
-The Open MPI Project is an open source Message Passing Interface (MPI)
-implementation that is developed and maintained by a consortium of
-academic, research, and industry partners.  Open MPI is therefore able
-to combine the expertise, technologies, and resources from all across
-the High Performance Computing community in order to build the best
-MPI library available.  Open MPI offers advantages for system and
-software vendors, application developers and computer science
-researchers.
-
-See [the MPI Forum web site](https://mpi-forum.org/) for information
-about the MPI API specification.
-
-## Quick start
-
-In many cases, Open MPI can be built and installed by simply
-indicating the installation directory on the command line:
-
-```
-$ tar xf openmpi-<version>.tar.bz2
-$ cd openmpi-<version>
-$ ./configure --prefix=<path> |& tee config.out
-...lots of output...
-$ make -j 8 |& tee make.out
-...lots of output...
-$ make install |& tee install.out
-...lots of output...
-```
-
-Note that there are many, many configuration options to the
-`./configure` step.  Some of them may be needed for your particular
-environmnet; see below for desciptions of the options available.
-
-If your installation prefix path is not writable by a regular user,
-you may need to use sudo or su to run the `make install` step.  For
-example:
-
-```
-$ sudo make install |& tee install.out
-[sudo] password for jsquyres: <enter your password here>
-...lots of output...
-```
-
-Finally, note that VPATH builds are fully supported.  For example:
-
-```
-$ tar xf openmpi-<version>.tar.bz2
-$ cd openmpi-<version>
-$ mkdir build
-$ cd build
-$ ../configure --prefix=<path> |& tee config.out
-...etc.
-```
-
-## Table of contents
-
-The rest of this file contains:
-
-* [General release notes about Open MPI](#general-notes)
-  * [Platform-specific notes](#platform-notes)
-  * [Compiler-specific notes](#compiler-notes)
-  * [Run-time support notes](#general-run-time-support-notes)
-  * [MPI functionality and features](#mpi-functionality-and-features)
-  * [OpenSHMEM functionality and
-    features](#openshmem-functionality-and-features)
-  * [MPI collectives](#mpi-collectives)
-  * [OpenSHMEM collectives](#openshmem-collectives)
-  * [Network support](#network-support)
-  * [Open MPI extensions](#open-mpi-extensions)
-* [Detailed information on building Open MPI](#building-open-mpi)
-  * [Installation options](#installation-options)
-  * [Networking support and options](#networking-support--options)
-  * [Run-time system support and options](#run-time-system-support)
-  * [Miscellaneous support
-    libraries](#miscellaneous-support-libraries)
-  * [MPI functionality options](#mpi-functionality)
-  * [OpenSHMEM functionality options](#openshmem-functionality)
-  * [Miscellaneous functionality
-    options](#miscellaneous-functionality)
-* [Open MPI version and library numbering
-  policies](#open-mpi-version-numbers-and-binary-compatibility)
-  * [Backwards compatibility polices](#backwards-compatibility)
-  * [Software version numbering](#software-version-number)
-  * [Shared library version numbering](#shared-library-version-number)
-* [Information on how to both query and validate your Open MPI
-  installation](#checking-your-open-mpi-installation)
-* [Description of Open MPI extensions](#open-mpi-api-extensions)
-  * [Compiling the extensions](#compiling-the-extensions)
-  * [Using the extensions](#using-the-extensions)
-* [Examples showing how to compile Open MPI applications](#compiling-open-mpi-applications)
-* [Examples showing how to run Open MPI applications](#running-open-mpi-applications)
-* [Summary information on the various plugin
- frameworks](#the-modular-component-architecture-mca)
-  * [MPI layer frameworks](#mpi-layer-frameworks)
-  * [OpenSHMEM component frameworks](#openshmem-component-frameworks)
-  * [Run-time environment
-    frameworks](#back-end-run-time-environment-rte-component-frameworks)
-  * [Miscellaneous frameworks](#miscellaneous-frameworks)
-  * [Other notes about frameworks](#framework-notes)
-* [How to get more help](#questions--problems)
-
-Also, note that much, much more information is also available [in the
-Open MPI FAQ](https://www.open-mpi.org/faq/).
-
-
-## General notes
-
-The following abbreviated list of release notes applies to this code
-base as of this writing (April 2020):
-
-* Open MPI now includes two public software layers: MPI and OpenSHMEM.
-  Throughout this document, references to Open MPI implicitly include
-  both of these layers. When distinction between these two layers is
-  necessary, we will reference them as the "MPI" and "OpenSHMEM"
-  layers respectively.
-
-* OpenSHMEM is a collaborative effort between academia, industry, and
-  the U.S. Government to create a specification for a standardized API
-  for parallel programming in the Partitioned Global Address Space
-  (PGAS).  For more information about the OpenSHMEM project, including
-  access to the current OpenSHMEM specification, please visit
-  http://openshmem.org/.
-
-  This OpenSHMEM implementation will only work in Linux environments
-  with a restricted set of supported networks.
-
-* Open MPI includes support for a wide variety of supplemental
-  hardware and software package.  When configuring Open MPI, you may
-  need to supply additional flags to the `configure` script in order
-  to tell Open MPI where the header files, libraries, and any other
-  required files are located.  As such, running `configure` by itself
-  may not include support for all the devices (etc.) that you expect,
-  especially if their support headers / libraries are installed in
-  non-standard locations.  Network interconnects are an easy example
-  to discuss -- Libfabric and OpenFabrics networks, for example, both
-  have supplemental headers and libraries that must be found before
-  Open MPI can build support for them.  You must specify where these
-  files are with the appropriate options to configure.  See the
-  listing of configure command-line switches, below, for more details.
-
-* The majority of Open MPI's documentation is here in this file, the
-  included man pages, and on [the web site
-  FAQ](https://www.open-mpi.org/).
-
-* Note that Open MPI documentation uses the word "component"
-  frequently; the word "plugin" is probably more familiar to most
-  users.  As such, end users can probably completely substitute the
-  word "plugin" wherever you see "component" in our documentation.
-  For what it's worth, we use the word "component" for historical
-  reasons, mainly because it is part of our acronyms and internal API
-  function calls.
-
-* The run-time systems that are currently supported are:
-  * rsh / ssh
-  * PBS Pro, Torque
-  * Platform LSF (tested with v9.1.1 and later)
-  * SLURM
-  * Cray XE, XC, and XK
-  * Oracle Grid Engine (OGE) 6.1, 6.2 and open source Grid Engine
-
-* Systems that have been tested are:
-  * Linux (various flavors/distros), 64 bit (x86, ppc, aarch64),
-    with gcc (>=4.8.x+), clang (>=3.6.0), Absoft (fortran), Intel,
-    and Portland (*)
-  * macOS (10.14-10.15), 64 bit (x86_64) with XCode compilers
-
-  (*) Be sure to read the Compiler Notes, below.
-
-* Other systems have been lightly (but not fully) tested:
-  * Linux (various flavors/distros), 32 bit, with gcc
-  * Cygwin 32 & 64 bit with gcc
-  * ARMv6, ARMv7
-  * Other 64 bit platforms.
-  * OpenBSD.  Requires configure options `--enable-mca-no-build=patcher`
-    and `--disable-dlopen` with this release.
-  * Problems have been reported when building Open MPI on FreeBSD 11.1
-    using the clang-4.0 system compiler. A workaround is to build
-    Open MPI using the GNU compiler.
-
-* Open MPI has taken some steps towards [Reproducible
-  Builds](https://reproducible-builds.org/).  Specifically, Open MPI's
-  `configure` and `make` process, by default, records the build date
-  and some system-specific information such as the hostname where Open
-  MPI was built and the username who built it.  If you desire a
-  Reproducible Build, set the `$SOURCE_DATE_EPOCH`, `$USER` and
-  `$HOSTNAME` environment variables before invoking `configure` and
-  `make`, and Open MPI will use those values instead of invoking
-  `whoami` and/or `hostname`, respectively.  See
-  https://reproducible-builds.org/docs/source-date-epoch/ for
-  information on the expected format and content of the
-  `$SOURCE_DATE_EPOCH` variable.
-
-
-### Platform Notes
-
-- N/A
-
-
-### Compiler Notes
-
-* Open MPI requires a C99-capable compiler to build.
-
-* On platforms other than x86-64, ARM, and PPC, Open MPI requires a
-  compiler that either supports C11 atomics or the GCC `__atomic`
-  atomics (e.g., GCC >= v4.7.2).
-
-* Mixing compilers from different vendors when building Open MPI
-  (e.g., using the C/C++ compiler from one vendor and the Fortran
-  compiler from a different vendor) has been successfully employed by
-  some Open MPI users (discussed on the Open MPI user's mailing list),
-  but such configurations are not tested and not documented.  For
-  example, such configurations may require additional compiler /
-  linker flags to make Open MPI build properly.
-
-  A not-uncommon case for this is when building on MacOS with the
-  system-default GCC compiler (i.e., `/usr/bin/gcc`), but a 3rd party
-  gfortran (e.g., provided by Homebrew, in `/usr/local/bin/gfortran`).
-  Since these compilers are provided by different organizations, they
-  have different default search paths.  For example, if Homebrew has
-  also installed a local copy of Libevent (a 3rd party package that
-  Open MPI requires), the MacOS-default `gcc` linker will find it
-  without any additional command line flags, but the Homebrew-provided
-  gfortran linker will not.  In this case, it may be necessary to
-  provide the following on the configure command line:
-
-  ```
-  $ ./configure FCFLAGS=-L/usr/local/lib ...
-  ```
-
-  This `-L` flag will then be passed to the Fortran linker when
-  creating Open MPI's Fortran libraries, and it will therefore be able
-  to find the installed Libevent.
-
-* In general, the latest versions of compilers of a given vendor's
-  series have the least bugs.  We have seen cases where Vendor XYZ's
-  compiler version A.B fails to compile Open MPI, but version A.C
-  (where C>B) works just fine.  If you run into a compile failure, you
-  might want to double check that you have the latest bug fixes and
-  patches for your compiler.
-
-* Users have reported issues with older versions of the Fortran PGI
-  compiler suite when using Open MPI's (non-default) `--enable-debug`
-  configure option.  Per the above advice of using the most recent
-  version of a compiler series, the Open MPI team recommends using the
-  latest version of the PGI suite, and/or not using the `--enable-debug`
-  configure option.  If it helps, here's what we have found with some
-  (not comprehensive) testing of various versions of the PGI compiler
-  suite:
-
-  * pgi-8 : NO known good version with `--enable-debug`
-  * pgi-9 : 9.0-4 known GOOD
-  * pgi-10: 10.0-0 known GOOD
-  * pgi-11: NO known good version with `--enable-debug`
-  * pgi-12: 12.10 known BAD with `-m32`, but known GOOD without `-m32`
-            (and 12.8 and 12.9 both known BAD with `--enable-debug`)
-  * pgi-13: 13.9 known BAD with `-m32`, 13.10 known GOOD without `-m32`
-  * pgi-15: 15.10 known BAD with `-m32`
-
-* Similarly, there is a known Fortran PGI compiler issue with long
-  source directory path names that was resolved in 9.0-4 (9.0-3 is
-  known to be broken in this regard).
-
-* Open MPI does not support the PGI compiler suite on OS X or MacOS.
-  See issues below for more details:
-  * https://github.com/open-mpi/ompi/issues/2604
-  * https://github.com/open-mpi/ompi/issues/2605
-
-* OpenSHMEM Fortran bindings do not support the "no underscore"
-  Fortran symbol convention. IBM's `xlf` compilers build in that mode
-  by default.  As such, IBM's `xlf` compilers cannot build/link the
-  OpenSHMEM Fortran bindings by default. A workaround is to pass
-  `FC="xlf -qextname"` at configure time to force a trailing
-  underscore. See [this
-  issue](https://github.com/open-mpi/ompi/issues/3612) for more
-  details.
-
-* MPI applications that use the mpi_f08 module on PowerPC platforms
-  (tested ppc64le) will likely experience runtime failures if:
-   * they are using a GNU linker (ld) version after v2.25.1 and before v2.28,
-     *and*
-   * they compiled with PGI (tested 17.5) or XL (tested v15.1.5) compilers.
-  This was noticed on Ubuntu 16.04 which uses the 2.26.1 version of
-  `ld` by default. However, this issue impacts any OS using a version
-  of `ld` noted above. This GNU linker regression will be fixed in
-  version 2.28.  [Here is a link to the GNU bug on this
-  issue](https://sourceware.org/bugzilla/show_bug.cgi?id=21306).  The
-  XL compiler will include a fix for this issue in a future release.
-
-* On NetBSD-6 (at least AMD64 and i386), and possibly on OpenBSD,
-  Libtool misidentifies properties of f95/g95, leading to obscure
-  compile-time failures if used to build Open MPI.  You can work
-  around this issue by ensuring that libtool will not use f95/g95
-  (e.g., by specifying `FC=<some_other_compiler>`, or otherwise ensuring
-  a different Fortran compiler will be found earlier in the path than
-  `f95`/`g95`), or by disabling the Fortran MPI bindings with
-  `--disable-mpi-fortran`.
-
-* On OpenBSD/i386, if you configure with
-  `--enable-mca-no-build=patcher`, you will also need to add
-  `--disable-dlopen`.  Otherwise, odd crashes can occur
-  nondeterministically.
-
-* Absoft 11.5.2 plus a service pack from September 2012 (which Absoft
-  says is available upon request), or a version later than 11.5.2
-  (e.g., 11.5.3), is required to compile the Fortran `mpi_f08`
-  module.
-
-* Open MPI does not support the Sparc v8 CPU target.  However,
-  as of Solaris Studio 12.1, and later compilers, one should not
-  specify `-xarch=v8plus` or `-xarch=v9`.  The use of the options
-  `-m32` and `-m64` for producing 32 and 64 bit targets, respectively,
-  are now preferred by the Solaris Studio compilers.  GCC may
-  require either `-m32` or `-mcpu=v9 -m32`, depending on GCC version.
-
-* If one tries to build OMPI on Ubuntu with Solaris Studio using the C++
-  compiler and the `-m32` option, you might see a warning:
-
-  ```
-  CC: Warning: failed to detect system linker version, falling back to custom linker usage
-  ```
-
-  And the build will fail.  One can overcome this error by either
-  setting `LD_LIBRARY_PATH` to the location of the 32 bit libraries
-  (most likely /lib32), or giving `LDFLAGS="-L/lib32 -R/lib32"` to the
-  `configure` command.  Officially, Solaris Studio is not supported on
-  Ubuntu Linux distributions, so additional problems might be
-  incurred.
-
-* Open MPI does not support the `gccfss` compiler (GCC For SPARC
-  Systems; a now-defunct compiler project from Sun).
-
-* At least some versions of the Intel 8.1 compiler seg fault while
-  compiling certain Open MPI source code files.  As such, it is not
-  supported.
-
-* It has been reported that the Intel 9.1 and 10.0 compilers fail to
-  compile Open MPI on IA64 platforms.  As of 12 Sep 2012, there is
-  very little (if any) testing performed on IA64 platforms (with any
-  compiler).  Support is "best effort" for these platforms, but it is
-  doubtful that any effort will be expended to fix the Intel 9.1 /
-  10.0 compiler issuers on this platform.
-
-* Early versions of the Intel 12.1 Linux compiler suite on x86_64 seem
-  to have a bug that prevents Open MPI from working.  Symptoms
-  including immediate segv of the wrapper compilers (e.g., `mpicc`) and
-  MPI applications.  As of 1 Feb 2012, if you upgrade to the latest
-  version of the Intel 12.1 Linux compiler suite, the problem will go
-  away.
-
-* The Portland Group compilers prior to version 7.0 require the
-  `-Msignextend` compiler flag to extend the sign bit when converting
-  from a shorter to longer integer.  This is is different than other
-  compilers (such as GNU).  When compiling Open MPI with the Portland
-  compiler suite, the following flags should be passed to Open MPI's
-  `configure` script:
-
-  ```
-  shell$ ./configure CFLAGS=-Msignextend CXXFLAGS=-Msignextend \
-         --with-wrapper-cflags=-Msignextend \
-         --with-wrapper-cxxflags=-Msignextend ...
-  ```
-
-  This will both compile Open MPI with the proper compile flags and
-  also automatically add "-Msignextend" when the C and C++ MPI wrapper
-  compilers are used to compile user MPI applications.
-
-* It has been reported that Pathscale 5.0.5 and 6.0.527 compilers
-  give an internal compiler error when trying to build Open MPI.
-
-* As of July 2017, the Pathscale compiler suite apparently has no
-  further commercial support, and it does not look like there will be
-  further releases.  Any issues discovered regarding building /
-  running Open MPI with the Pathscale compiler suite therefore may not
-  be able to be resolved.
-
-* Using the Absoft compiler to build the MPI Fortran bindings on Suse
-  9.3 is known to fail due to a Libtool compatibility issue.
-
-* MPI Fortran API support has been completely overhauled since the
-  Open MPI v1.5/v1.6 series.
-
-  There is now only a single Fortran MPI wrapper compiler and a
-  single Fortran OpenSHMEM wrapper compiler: `mpifort` and `oshfort`,
-  respectively.  `mpif77` and `mpif90` still exist, but they are
-  symbolic links to `mpifort`.
-
-  Similarly, Open MPI's `configure` script only recognizes the `FC`
-  and `FCFLAGS` environment variables (to specify the Fortran
-  compiler and compiler flags, respectively).  The `F77` and `FFLAGS`
-  environment variables are ***IGNORED***.
-
-  As a direct result, it is ***STRONGLY*** recommended that you
-  specify a Fortran compiler that uses file suffixes to determine
-  Fortran code layout (e.g., free form vs. fixed).  For example, with
-  some versions of the IBM XLF compiler, it is preferable to use
-  `FC=xlf` instead of `FC=xlf90`, because `xlf` will automatically
-  determine the difference between free form and fixed Fortran source
-  code.
-
-  However, many Fortran compilers allow specifying additional
-  command-line arguments to indicate which Fortran dialect to use.
-  For example, if `FC=xlf90`, you may need to use `mpifort --qfixed ...`
-  to compile fixed format Fortran source files.
-
-  You can use either `ompi_info` or `oshmem_info` to see with which
-  Fortran compiler Open MPI was configured and compiled.
-
-  There are up to three sets of Fortran MPI bindings that may be
-  provided (depending on your Fortran compiler):
-
-  1. `mpif.h`: This is the first MPI Fortran interface that was
-     defined in MPI-1.  It is a file that is included in Fortran
-     source code.  Open MPI's `mpif.h` does not declare any MPI
-     subroutines; they are all implicit.
-
-  1. `mpi` module: The `mpi` module file was added in MPI-2.  It
-     provides strong compile-time parameter type checking for MPI
-     subroutines.
-
-  1. `mpi_f08` module: The `mpi_f08` module was added in MPI-3.  It
-     provides many advantages over the `mpif.h` file and `mpi` module.
-     For example, MPI handles have distinct types (vs. all being
-     integers).  See the MPI-3 document for more details.
-
-  ***NOTE:*** The `mpi_f08` module is ***STRONGLY*** recommended for
-  all new MPI Fortran subroutines and applications.  Note that the
-  `mpi_f08` module can be used in conjunction with the other two
-  Fortran MPI bindings in the same application (only one binding can
-  be used per subroutine/function, however).  Full interoperability
-  between `mpif.h`/`mpi` module and `mpi_f08` module MPI handle types
-  is provided, allowing `mpi_f08` to be used in new subroutines in
-  legacy MPI applications.
-
-  Per the OpenSHMEM specification, there is only one Fortran OpenSHMEM
-  binding provided:
-
-  * `shmem.fh`: All Fortran OpenSHMEM programs should include
-    `shmem.f`, and Fortran OpenSHMEM programs that use constants
-    defined by OpenSHMEM ***MUST*** include `shmem.fh`.
-
-  The following notes apply to the above-listed Fortran bindings:
-
-  * All Fortran compilers support the `mpif.h`/`shmem.fh`-based
-    bindings, with one exception: the `MPI_SIZEOF` interfaces will
-    only be present when Open MPI is built with a Fortran compiler
-    that supports the `INTERFACE` keyword and `ISO_FORTRAN_ENV`.  Most
-    notably, this excludes the GNU Fortran compiler suite before
-    version 4.9.
-
-  * The level of support provided by the `mpi` module is based on your
-    Fortran compiler.
-
-    If Open MPI is built with a non-GNU Fortran compiler, or if Open
-    MPI is built with the GNU Fortran compiler >= v4.9, all MPI
-    subroutines will be prototyped in the `mpi` module.  All calls to
-    MPI subroutines will therefore have their parameter types checked
-    at compile time.
-
-    If Open MPI is built with an old `gfortran` (i.e., < v4.9), a
-    limited `mpi` module will be built.  Due to the limitations of
-    these compilers, and per guidance from the MPI-3 specification,
-    all MPI subroutines with "choice" buffers are specifically *not*
-    included in the `mpi` module, and their parameters will not be
-    checked at compile time.  Specifically, all MPI subroutines with
-    no "choice" buffers are prototyped and will receive strong
-    parameter type checking at run-time (e.g., `MPI_INIT`,
-    `MPI_COMM_RANK`, etc.).
-
-    Similar to the `mpif.h` interface, `MPI_SIZEOF` is only supported
-    on Fortran compilers that support `INTERFACE` and
-    `ISO_FORTRAN_ENV`.
-
-  * The `mpi_f08` module has been tested with the Intel Fortran
-    compiler and gfortran >= 4.9.  Other modern Fortran compilers
-    likely also work.
-
-    Many older Fortran compilers do not provide enough modern Fortran
-    features to support the `mpi_f08` module.  For example, `gfortran`
-    < v4.9 does provide enough support for the `mpi_f08` module.
-
-  You can examine the output of the following command to see all
-  the Fortran features that are/are not enabled in your Open MPI
-  installation:
-
-  ```
-  shell$ ompi_info | grep -i fort
-  ```
-
-
-### General Run-Time Support Notes
-
-* The Open MPI installation must be in your `PATH` on all nodes (and
-  potentially `LD_LIBRARY_PATH` or `DYLD_LIBRARY_PATH`, if
-  `libmpi`/`libshmem` is a shared library), unless using the
-  `--prefix` or `--enable-mpirun-prefix-by-default` functionality (see
-  below).
-
-* Open MPI's run-time behavior can be customized via Modular Component
-  Architecture (MCA) parameters (see below for more information on how
-  to get/set MCA parameter values).  Some MCA parameters can be set in
-  a way that renders Open MPI inoperable (see notes about MCA
-  parameters later in this file).  In particular, some parameters have
-  required options that must be included.
-
-  * If specified, the `btl` parameter must include the `self`
-    component, or Open MPI will not be able to deliver messages to the
-    same rank as the sender.  For example: `mpirun --mca btl tcp,self
-    ...`
-  * If specified, the `btl_tcp_if_exclude` parameter must include the
-    loopback device (`lo` on many Linux platforms), or Open MPI will
-    not be able to route MPI messages using the TCP BTL.  For example:
-    `mpirun --mca btl_tcp_if_exclude lo,eth1 ...`
-
-* Running on nodes with different endian and/or different datatype
-  sizes within a single parallel job is supported in this release.
-  However, Open MPI does not resize data when datatypes differ in size
-  (for example, sending a 4 byte `MPI_DOUBLE` and receiving an 8 byte
-  `MPI_DOUBLE` will fail).
-
-
-### MPI Functionality and Features
-
-* All MPI-3.1 functionality is supported.
-
-* Note that starting with Open MPI v4.0.0, prototypes for several
-  legacy MPI-1 symbols that were deleted in the MPI-3.0 specification
-  (which was published in 2012) are no longer available by default in
-  `mpi.h`.  Specifically, several MPI-1 symbols were deprecated in the
-  1996 publishing of the MPI-2.0 specification.  These deprecated
-  symbols were eventually removed from the MPI-3.0 specification in
-  2012.
-
-  The symbols that now no longer appear by default in Open MPI's
-  `mpi.h` are:
-
-  * `MPI_Address` (replaced by `MPI_Get_address`)
-  * `MPI_Errhandler_create` (replaced by `MPI_Comm_create_errhandler`)
-  * `MPI_Errhandler_get` (replaced by `MPI_Comm_get_errhandler`)
-  * `MPI_Errhandler_set` (replaced by `MPI_Comm_set_errhandler`)
-  * `MPI_Type_extent` (replaced by `MPI_Type_get_extent`)
-  * `MPI_Type_hindexed` (replaced by `MPI_Type_create_hindexed`)
-  * `MPI_Type_hvector` (replaced by `MPI_Type_create_hvector`)
-  * `MPI_Type_lb` (replaced by `MPI_Type_get_extent`)
-  * `MPI_Type_struct` (replaced by `MPI_Type_create_struct`)
-  * `MPI_Type_ub` (replaced by `MPI_Type_get_extent`)
-  * `MPI_LB` (replaced by `MPI_Type_create_resized`)
-  * `MPI_UB` (replaced by `MPI_Type_create_resized`)
-  * `MPI_COMBINER_HINDEXED_INTEGER`
-  * `MPI_COMBINER_HVECTOR_INTEGER`
-  * `MPI_COMBINER_STRUCT_INTEGER`
-  * `MPI_Handler_function` (replaced by `MPI_Comm_errhandler_function`)
-
-  Although these symbols are no longer prototyped in `mpi.h`, they
-  are still present in the MPI library in Open MPI v4.0.x. This
-  enables legacy MPI applications to link and run successfully with
-  Open MPI v4.0.x, even though they will fail to compile.
-
-  ***WARNING:*** Future releases of Open MPI beyond the v4.0.x series
-  may remove these symbols altogether.
-
-  ***WARNING:*** The Open MPI team ***STRONGLY*** encourages all MPI
-  application developers to stop using these constructs that were
-  first deprecated over 20 years ago, and finally removed from the MPI
-  specification in MPI-3.0 (in 2012).
-
-  ***WARNING:*** [The Open MPI
-  FAQ](https://www.open-mpi.org/faq/?category=mpi-removed) contains
-  examples of how to update legacy MPI applications using these
-  deleted symbols to use the "new" symbols.
-
-  All that being said, if you are unable to immediately update your
-  application to stop using these legacy MPI-1 symbols, you can
-  re-enable them in `mpi.h` by configuring Open MPI with the
-  `--enable-mpi1-compatibility` flag.
-
-* Rank reordering support is available using the TreeMatch library. It
-  is activated for the graph and `dist_graph` communicator topologies.
-
-* When using MPI deprecated functions, some compilers will emit
-  warnings.  For example:
-
-  ```
-  shell$ cat deprecated_example.c
-  #include <mpi.h>
-  void foo(void) {
-      MPI_Datatype type;
-      MPI_Type_struct(1, NULL, NULL, NULL, &type);
-  }
-  shell$ mpicc -c deprecated_example.c
-  deprecated_example.c: In function 'foo':
-  deprecated_example.c:4: warning: 'MPI_Type_struct' is deprecated (declared at /opt/openmpi/include/mpi.h:1522)
-  shell$
-  ```
-
-* `MPI_THREAD_MULTIPLE` is supported with some exceptions.
-
-  The following PMLs support `MPI_THREAD_MULTIPLE`:
-  1. `cm` (see list (1) of supported MTLs, below)
-  1. `ob1` (see list (2) of supported BTLs, below)
-  1. `ucx`
-
-  (1) The `cm` PML and the following MTLs support `MPI_THREAD_MULTIPLE`:
-     1. `ofi` (Libfabric)
-     1. `portals4`
-
-  (2) The `ob1` PML and the following BTLs support `MPI_THREAD_MULTIPLE`:
-     1. `self`
-     1. `sm`
-     1. `smcuda`
-     1. `tcp`
-     1. `ugni`
-     1. `usnic`
-
-  Currently, MPI File operations are not thread safe even if MPI is
-  initialized for `MPI_THREAD_MULTIPLE` support.
-
-* `MPI_REAL16` and `MPI_COMPLEX32` are only supported on platforms
-  where a portable C datatype can be found that matches the Fortran
-  type `REAL*16`, both in size and bit representation.
-
-* The "libompitrace" library is bundled in Open MPI and is installed
-  by default (it can be disabled via the `--disable-libompitrace`
-  flag).  This library provides a simplistic tracing of select MPI
-  function calls via the MPI profiling interface.  Linking it in to
-  your application via (e.g., via `-lompitrace`) will automatically
-  output to stderr when some MPI functions are invoked:
-
-  ```
-  shell$ cd examples/
-  shell$ mpicc hello_c.c -o hello_c -lompitrace
-  shell$ mpirun -np 1 hello_c
-  MPI_INIT: argc 1
-  Hello, world, I am 0 of 1
-  MPI_BARRIER[0]: comm MPI_COMM_WORLD
-  MPI_FINALIZE[0]
-  shell$
-  ```
-
-  Keep in mind that the output from the trace library is going to
-  `stderr`, so it may output in a slightly different order than the
-  `stdout` from your application.
-
-  This library is being offered as a "proof of concept" / convenience
-  from Open MPI.  If there is interest, it is trivially easy to extend
-  it to printf for other MPI functions.  Pull requests on github.com
-  would be greatly appreciated.
-
-
-### OpenSHMEM Functionality and Features
-
-All OpenSHMEM-1.3 functionality is supported.
-
-
-### MPI Collectives
-
-* The `cuda` coll component provides CUDA-aware support for the
-  reduction type collectives with GPU buffers. This component is only
-  compiled into the library when the library has been configured with
-  CUDA-aware support.  It intercepts calls to the reduction
-  collectives, copies the data to staging buffers if GPU buffers, then
-  calls underlying collectives to do the work.
-
-
-### OpenSHMEM Collectives
-
-* The `fca` scoll component: the Mellanox Fabric Collective
-  Accelerator (FCA) is a solution for offloading collective operations
-  from the MPI process onto Mellanox QDR InfiniBand switch CPUs and
-  HCAs.
-
-* The `basic` scoll component: Reference implementation of all
-  OpenSHMEM collective operations.
-
-
-### Network Support
-
-* There are several main MPI network models available: `ob1`, `cm`,
-  and `ucx`.  `ob1` uses BTL ("Byte Transfer Layer")
-  components for each supported network.  `cm` uses MTL ("Matching
-  Transport Layer") components for each supported network.  `ucx` uses
-  the OpenUCX transport.
-
-  * `ob1` supports a variety of networks that can be used in
-    combination with each other:
-    * OpenFabrics: InfiniBand, iWARP, and RoCE
-    * Loopback (send-to-self)
-    * Shared memory
-    * TCP
-    * SMCUDA
-    * Cisco usNIC
-    * uGNI (Cray Gemini, Aries)
-    * shared memory (XPMEM, Linux CMA, Linux KNEM, and
-      copy-in/copy-out shared memory)
-
-  * `cm` supports a smaller number of networks (and they cannot be
-    used together), but may provide better overall MPI performance:
-    * Intel Omni-Path PSM2 (version 11.2.173 or later)
-    * Intel True Scale PSM (QLogic InfiniPath)
-    * OpenFabrics Interfaces ("libfabric" tag matching)
-    * Portals 4
-
-  * UCX is the [Unified Communication X (UCX) communication
-    library](https://www.openucx.org/).  This is an open-source
-    project developed in collaboration between industry, laboratories,
-    and academia to create an open-source production grade
-    communication framework for data centric and high-performance
-    applications.  The UCX library can be downloaded from repositories
-    (e.g., Fedora/RedHat yum repositories).  The UCX library is also
-    part of Mellanox OFED and Mellanox HPC-X binary distributions.
-
-    UCX currently supports:
-
-    * OpenFabrics Verbs (including InfiniBand and RoCE)
-    * Cray's uGNI
-    * TCP
-    * Shared memory
-    * NVIDIA CUDA drivers
-
-  While users can manually select any of the above transports at run
-  time, Open MPI will select a default transport as follows:
-
-  1. If InfiniBand devices are available, use the UCX PML.
-  1. If PSM, PSM2, or other tag-matching-supporting Libfabric
-     transport devices are available (e.g., Cray uGNI), use the `cm`
-     PML and a single appropriate corresponding `mtl` module.
-  1. Otherwise, use the `ob1` PML and one or more appropriate `btl`
-     modules.
-
-  Users can override Open MPI's default selection algorithms and force
-  the use of a specific transport if desired by setting the `pml` MCA
-  parameter (and potentially the `btl` and/or `mtl` MCA parameters) at
-  run-time:
-
-  ```
-  shell$ mpirun --mca pml ob1 --mca btl [comma-delimted-BTLs] ...
-  or
-  shell$ mpirun --mca pml cm --mca mtl [MTL] ...
-  or
-  shell$ mpirun --mca pml ucx ...
-  ```
-
-  There is a known issue when using UCX with very old Mellanox
-  Infiniband HCAs, in particular HCAs preceding the introduction of
-  the ConnectX product line, which can result in Open MPI crashing in
-  MPI_Finalize.  This issue is addressed by UCX release 1.9.0 and
-  newer.
-
-* The main OpenSHMEM network model is `ucx`; it interfaces directly
-  with UCX.
-
-* In prior versions of Open MPI, InfiniBand and RoCE support was
-  provided through the `openib` BTL and `ob1` PML plugins.  Starting
-  with Open MPI 4.0.0, InfiniBand support through the `openib` plugin
-  is both deprecated and superseded by the `ucx` PML component.  The
-  `openib` BTL was removed in Open MPI v5.0.0.
-
-  While the `openib` BTL depended on `libibverbs`, the UCX PML depends
-  on the UCX library.
-
-  Once installed, Open MPI can be built with UCX support by adding
-  `--with-ucx` to the Open MPI configure command. Once Open MPI is
-  configured to use UCX, the runtime will automatically select the
-  `ucx` PML if one of the supported networks is detected (e.g.,
-  InfiniBand).  It's possible to force using UCX in the `mpirun` or
-  `oshrun` command lines by specifying any or all of the following mca
-  parameters: `--mca pml ucx` for MPI point-to-point operations,
-  `--mca spml ucx` for OpenSHMEM support, and `--mca osc ucx` for MPI
-  RMA (one-sided) operations.
-
-* The `usnic` BTL is support for Cisco's usNIC device ("userspace NIC")
-  on Cisco UCS servers with the Virtualized Interface Card (VIC).
-  Although the usNIC is accessed via the OpenFabrics Libfabric API
-  stack, this BTL is specific to Cisco usNIC devices.
-
-* uGNI is a Cray library for communicating over the Gemini and Aries
-  interconnects.
-
-* The OpenFabrics Enterprise Distribution (OFED) software package v1.0
-  will not work properly with Open MPI v1.2 (and later) due to how its
-  Mellanox InfiniBand plugin driver is created.  The problem is fixed
-  with OFED v1.1 (and later).
-
-* The use of `fork()` with Libiverbs-based networks (i.e., the UCX
-  PML) is only partially supported, and only on Linux kernels >=
-  v2.6.15 with `libibverbs` v1.1 or later (first released as part of
-  OFED v1.2), per restrictions imposed by the OFED network stack.
-
-* Linux `knem` support is used when the `sm` (shared memory) BTL is
-  compiled with knem support (see the `--with-knem` configure option)
-  and the `knem` Linux module is loaded in the running kernel.  If the
-  `knem` Linux kernel module is not loaded, the `knem` support is (by
-  default) silently deactivated during Open MPI jobs.
-
-  See https://knem.gforge.inria.fr/ for details on Knem.
-
-* Linux Cross-Memory Attach (CMA) or XPMEM is used by the `sm` shared
-  memory BTL when the CMA/XPMEM libraries are installed,
-  respectively.  Linux CMA and XPMEM are similar (but different)
-  mechanisms for Open MPI to utilize single-copy semantics for shared
-  memory.
-
-
-### Open MPI Extensions
-
-An MPI "extensions" framework is included in Open MPI, but is not
-enabled by default.  See the "Open MPI API Extensions" section below
-for more information on compiling and using MPI extensions.
-
-The following extensions are included in this version of Open MPI:
-
-1. `pcollreq`: Provides routines for persistent collective
-   communication operations and persistent neighborhood collective
-   communication operations, which are planned to be included in
-   MPI-4.0.  The function names are prefixed with `MPIX_` instead of
-   `MPI_`, like `MPIX_Barrier_init`, because they are not
-   standardized yet.  Future versions of Open MPI will switch to the
-   `MPI_` prefix once the MPI Standard which includes this feature is
-   published.  See their man page for more details.
-1. `shortfloat`: Provides MPI datatypes `MPIX_C_FLOAT16`,
-   `MPIX_SHORT_FLOAT`, `MPIX_SHORT_FLOAT`, and
-   `MPIX_CXX_SHORT_FLOAT_COMPLEX` if corresponding language types are
-   available. See `ompi/mpiext/shortfloat/README.txt` for details.
-1. `affinity`: Provides the `OMPI_Affinity_str()` API, which returns
-   a string indicating the resources which a process is bound. For
-   more details, see its man page.
-1. `cuda`: When the library is compiled with CUDA-aware support, it
-   provides two things.  First, a macro
-   `MPIX_CUDA_AWARE_SUPPORT`. Secondly, the function
-   `MPIX_Query_cuda_support()` that can be used to query for support.
-1. `example`: A non-functional extension; its only purpose is to
-   provide an example for how to create other extensions.
-
-
-## Building Open MPI
-
-If you have checked out a ***developer's copy*** of Open MPI (i.e.,
-you cloned from Git), you really need to read the `HACKING` file
-before attempting to build Open MPI. Really.
-
-If you have downloaded a tarball, then things are much simpler.
-Open MPI uses a traditional `configure` script paired with `make` to
-build.  Typical installs can be of the pattern:
-
-```
-shell$ ./configure [...options...]
-shell$ make [-j N] all install
-      (use an integer value of N for parallel builds)
-```
-
-There are many available `configure` options (see `./configure --help`
-for a full list); a summary of the more commonly used ones is included
-below.
-
-***NOTE:*** if you are building Open MPI on a network filesystem, the
-machine you on which you are building *must* be time-synchronized with
-the file server.  Specifically: Open MPI's build system *requires*
-accurate filesystem timestamps.  If your `make` output includes
-warning about timestamps in the future or runs GNU Automake, Autoconf,
-and/or Libtool, this is *not normal*, and you may have an invalid
-build.  Ensure that the time on your build machine is synchronized
-with the time on your file server, or build on a local filesystem.
-Then remove the Open MPI source directory and start over (e.g., by
-re-extracting the Open MPI tarball).
-
-Note that for many of Open MPI's `--with-FOO` options, Open MPI will,
-by default, search for header files and/or libraries for `FOO`.  If
-the relevant files are found, Open MPI will built support for `FOO`;
-if they are not found, Open MPI will skip building support for `FOO`.
-However, if you specify `--with-FOO` on the configure command line and
-Open MPI is unable to find relevant support for `FOO`, configure will
-assume that it was unable to provide a feature that was specifically
-requested and will abort so that a human can resolve out the issue.
-
-Additionally, if a search directory is specified in the form
-`--with-FOO=DIR`, Open MPI will:
-
-1. Search for `FOO`'s header files in `DIR/include`.
-2. Search for `FOO`'s library files:
-   1. If `--with-FOO-libdir=<libdir>` was specified, search in
-      `<libdir>`.
-   1. Otherwise, search in `DIR/lib`, and if they are not found
-      there, search again in `DIR/lib64`.
-3. If both the relevant header files and libraries are found:
-   1. Open MPI will build support for `FOO`.
-   1. If the root path where the FOO libraries are found is neither
-      `/usr` nor `/usr/local`, Open MPI will compile itself with
-      RPATH flags pointing to the directory where FOO's libraries
-      are located.  Open MPI does not RPATH `/usr/lib[64]` and
-      `/usr/local/lib[64]` because many systems already search these
-      directories for run-time libraries by default; adding RPATH for
-      them could have unintended consequences for the search path
-      ordering.
-
-
-### Installation Options
-
-* `--prefix=DIR`:
-  Install Open MPI into the base directory named `DIR`.  Hence, Open
-  MPI will place its executables in `DIR/bin`, its header files in
-  `DIR/include`, its libraries in `DIR/lib`, etc.
-
-* `--disable-shared`:
-  By default, Open MPI and OpenSHMEM build shared libraries, and all
-  components are built as dynamic shared objects (DSOs). This switch
-  disables this default; it is really only useful when used with
-  `--enable-static`.  Specifically, this option does *not* imply
-  `--enable-static`; enabling static libraries and disabling shared
-  libraries are two independent options.
-
-* `--enable-static`:
-  Build MPI and OpenSHMEM as static libraries, and statically link in
-  all components.  Note that this option does *not* imply
-  `--disable-shared`; enabling static libraries and disabling shared
-  libraries are two independent options.
-
-  Be sure to read the description of `--without-memory-manager`,
-  below; it may have some effect on `--enable-static`.
-
-* `--disable-wrapper-rpath`:
-  By default, the wrapper compilers (e.g., `mpicc`) will enable
-  "rpath" support in generated executables on systems that support it.
-  That is, they will include a file reference to the location of Open
-  MPI's libraries in the application executable itself.  This means
-  that the user does not have to set `LD_LIBRARY_PATH` to find Open
-  MPI's libraries (e.g., if they are installed in a location that the
-  run-time linker does not search by default).
-
-  On systems that utilize the GNU `ld` linker, recent enough versions
-  will actually utilize "runpath" functionality, not "rpath".  There
-  is an important difference between the two:
-
-  1. "rpath": the location of the Open MPI libraries is hard-coded into
-     the MPI/OpenSHMEM application and cannot be overridden at
-     run-time.
-  1. "runpath": the location of the Open MPI libraries is hard-coded into
-     the MPI/OpenSHMEM application, but can be overridden at run-time
-     by setting the `LD_LIBRARY_PATH` environment variable.
-
-  For example, consider that you install Open MPI vA.B.0 and
-  compile/link your MPI/OpenSHMEM application against it.  Later, you
-  install Open MPI vA.B.1 to a different installation prefix (e.g.,
-  `/opt/openmpi/A.B.1` vs. `/opt/openmpi/A.B.0`), and you leave the old
-  installation intact.
-
-  In the rpath case, your MPI application will always use the
-  libraries from your A.B.0 installation.  In the runpath case, you
-  can set the `LD_LIBRARY_PATH` environment variable to point to the
-  A.B.1 installation, and then your MPI application will use those
-  libraries.
-
-  Note that in both cases, however, if you remove the original A.B.0
-  installation and set `LD_LIBRARY_PATH` to point to the A.B.1
-  installation, your application will use the A.B.1 libraries.
-
-  This rpath/runpath behavior can be disabled via
-  `--disable-wrapper-rpath`.
-
-  If you would like to keep the rpath option, but not enable runpath
-  a different configure option is avalabile
-  `--disable-wrapper-runpath`.
-
-* `--enable-dlopen`:
-  Build all of Open MPI's components as standalone Dynamic Shared
-  Objects (DSO's) that are loaded at run-time (this is the default).
-  The opposite of this option, `--disable-dlopen`, causes two things:
-
-  1. All of Open MPI's components will be built as part of Open MPI's
-     normal libraries (e.g., `libmpi`).
-  1. Open MPI will not attempt to open any DSO's at run-time.
-
-  Note that this option does *not* imply that OMPI's libraries will be
-  built as static objects (e.g., `libmpi.a`).  It only specifies the
-  location of OMPI's components: standalone DSOs or folded into the
-  Open MPI libraries.  You can control whether Open MPI's libraries
-  are build as static or dynamic via `--enable|disable-static` and
-  `--enable|disable-shared`.
-
-* `--disable-show-load-errors-by-default`:
-  Set the default value of the `mca_base_component_show_load_errors`
-  MCA variable: the `--enable` form of this option sets the MCA
-  variable to true, the `--disable` form sets the MCA variable to
-  false.  The MCA `mca_base_component_show_load_errors` variable can
-  still be overridden at run time via the usual MCA-variable-setting
-  mechanisms; this configure option simply sets the default value.
-
-  The `--disable` form of this option is intended for Open MPI
-  packagers who tend to enable support for many different types of
-  networks and systems in their packages.  For example, consider a
-  packager who includes support for both the FOO and BAR networks in
-  their Open MPI package, both of which require support libraries
-  (`libFOO.so` and `libBAR.so`).  If an end user only has BAR
-  hardware, they likely only have `libBAR.so` available on their
-  systems -- not `libFOO.so`.  Disabling load errors by default will
-  prevent the user from seeing potentially confusing warnings about
-  the FOO components failing to load because `libFOO.so` is not
-  available on their systems.
-
-  Conversely, system administrators tend to build an Open MPI that is
-  targeted at their specific environment, and contains few (if any)
-  components that are not needed.  In such cases, they might want
-  their users to be warned that the FOO network components failed to
-  load (e.g., if `libFOO.so` was mistakenly unavailable), because Open
-  MPI may otherwise silently failover to a slower network path for MPI
-  traffic.
-
-* `--with-platform=FILE`:
-  Load configure options for the build from `FILE`.  Options on the
-  command line that are not in `FILE` are also used.  Options on the
-  command line and in `FILE` are replaced by what is in `FILE`.
-
-* `--with-libmpi-name=STRING`:
-  Replace `libmpi.*` and `libmpi_FOO.*` (where `FOO` is one of the
-  fortran supporting libraries installed in lib) with `libSTRING.*`
-  and `libSTRING_FOO.*`. This is provided as a convenience mechanism
-  for third-party packagers of Open MPI that might want to rename
-  these libraries for their own purposes. This option is *not*
-  intended for typical users of Open MPI.
-
-* `--enable-mca-no-build=LIST`:
-  Comma-separated list of `<type>-<component>` pairs that will not be
-  built. For example, `--enable-mca-no-build=btl-portals,oob-ud` will
-  disable building the portals BTL and the ud OOB component.
-
-
-### Networking support / options
-
-* `--with-fca=DIR`:
-  Specify the directory where the Mellanox FCA library and
-  header files are located.
-
-  FCA is the support library for Mellanox switches and HCAs.
-
-* `--with-hcoll=DIR`:
-  Specify the directory where the Mellanox hcoll library and header
-  files are located.  This option is generally only necessary if the
-  hcoll headers and libraries are not in default compiler/linker
-  search paths.
-
-  hcoll is the support library for MPI collective operation offload on
-  Mellanox ConnectX-3 HCAs (and later).
-
-* `--with-knem=DIR`:
-  Specify the directory where the knem libraries and header files are
-  located.  This option is generally only necessary if the knem headers
-  and libraries are not in default compiler/linker search paths.
-
-  knem is a Linux kernel module that allows direct process-to-process
-  memory copies (optionally using hardware offload), potentially
-  increasing bandwidth for large messages sent between messages on the
-  same server.  See [the Knem web site](https://knem.gforge.inria.fr/)
-  for details.
-
-* `--with-libfabric=DIR`:
-  Specify the directory where the OpenFabrics Interfaces `libfabric`
-  library and header files are located.  This option is generally only
-  necessary if the libfabric headers and libraries are not in default
-  compiler/linker search paths.
-
-  Libfabric is the support library for OpenFabrics Interfaces-based
-  network adapters, such as Cisco usNIC, Intel True Scale PSM, Cray
-  uGNI, etc.
-
-* `--with-libfabric-libdir=DIR`:
-  Look in directory for the libfabric libraries.  By default, Open MPI
-  will look in `DIR/lib` and `DIR/lib64`, which covers most cases.
-  This option is only needed for special configurations.
-
-* `--with-portals4=DIR`:
-  Specify the directory where the Portals4 libraries and header files
-  are located.  This option is generally only necessary if the Portals4
-  headers and libraries are not in default compiler/linker search
-  paths.
-
-  Portals is a low-level network API for high-performance networking
-  on high-performance computing systems developed by Sandia National
-  Laboratories, Intel Corporation, and the University of New Mexico.
-  The Portals 4 Reference Implementation is a complete implementation
-  of Portals 4, with transport over InfiniBand verbs and UDP.
-
-* `--with-portals4-libdir=DIR`:
-  Location of libraries to link with for Portals4 support.
-
-* `--with-portals4-max-md-size=SIZE` and
-  `--with-portals4-max-va-size=SIZE`:
-  Set configuration values for Portals 4
-
-* `--with-psm=<directory>`:
-  Specify the directory where the QLogic InfiniPath / Intel True Scale
-  PSM library and header files are located.  This option is generally
-  only necessary if the PSM headers and libraries are not in default
-  compiler/linker search paths.
-
-  PSM is the support library for QLogic InfiniPath and Intel TrueScale
-  network adapters.
-
-* `--with-psm-libdir=DIR`:
-  Look in directory for the PSM libraries.  By default, Open MPI will
-  look in `DIR/lib` and `DIR/lib64`, which covers most cases.  This
-  option is only needed for special configurations.
-
-* `--with-psm2=DIR`:
-  Specify the directory where the Intel Omni-Path PSM2 library and
-  header files are located.  This option is generally only necessary
-  if the PSM2 headers and libraries are not in default compiler/linker
-  search paths.
-
-  PSM is the support library for Intel Omni-Path network adapters.
-
-* `--with-psm2-libdir=DIR`:
-  Look in directory for the PSM2 libraries.  By default, Open MPI will
-  look in `DIR/lib` and `DIR/lib64`, which covers most cases.  This
-  option is only needed for special configurations.
-
-* `--with-ucx=DIR`:
-  Specify the directory where the UCX libraries and header files are
-  located.  This option is generally only necessary if the UCX headers
-  and libraries are not in default compiler/linker search paths.
-
-* `--with-ucx-libdir=DIR`:
-  Look in directory for the UCX libraries.  By default, Open MPI will
-  look in `DIR/lib` and `DIR/lib64`, which covers most cases.  This
-  option is only needed for special configurations.
-
-* `--with-usnic`:
-  Abort configure if Cisco usNIC support cannot be built.
-
-
-### Run-time system support
-
-* `--enable-mpirun-prefix-by-default`:
-  This option forces the `mpirun` command to always behave as if
-  `--prefix $prefix` was present on the command line (where `$prefix`
-  is the value given to the `--prefix` option to configure).  This
-  prevents most `rsh`/`ssh`-based users from needing to modify their
-  shell startup files to set the `PATH` and/or `LD_LIBRARY_PATH` for
-  Open MPI on remote nodes.  Note, however, that such users may still
-  desire to set `PATH` -- perhaps even in their shell startup files --
-  so that executables such as `mpicc` and `mpirun` can be found
-  without needing to type long path names.
-
-* `--enable-orte-static-ports`:
-   Enable ORTE static ports for TCP OOB (default: enabled).
-
-* `--with-alps`:
-  Force the building of for the Cray Alps run-time environment.  If
-  Alps support cannot be found, configure will abort.
-
-* `--with-lsf=DIR`:
-  Specify the directory where the LSF libraries and header files are
-  located.  This option is generally only necessary if the LSF headers
-  and libraries are not in default compiler/linker search paths.
-
-  LSF is a resource manager system, frequently used as a batch
-  scheduler in HPC systems.
-
-* `--with-lsf-libdir=DIR`:
-  Look in directory for the LSF libraries.  By default, Open MPI will
-  look in `DIR/lib` and `DIR/lib64`, which covers most cases.  This
-  option is only needed for special configurations.
-
-* `--with-slurm`:
-  Force the building of SLURM scheduler support.
-
-* `--with-sge`:
-  Specify to build support for the Oracle Grid Engine (OGE) resource
-  manager and/or the Open Grid Engine.  OGE support is disabled by
-  default; this option must be specified to build OMPI's OGE support.
-
-  The Oracle Grid Engine (OGE) and open Grid Engine packages are
-  resource manager systems, frequently used as a batch scheduler in
-  HPC systems.  It used to be called the "Sun Grid Engine", which is
-  why the option is still named `--with-sge`.
-
-* `--with-tm=DIR`:
-  Specify the directory where the TM libraries and header files are
-  located.  This option is generally only necessary if the TM headers
-  and libraries are not in default compiler/linker search paths.
-
-  TM is the support library for the Torque and PBS Pro resource
-  manager systems, both of which are frequently used as a batch
-  scheduler in HPC systems.
-
-
-### Miscellaneous support libraries
-
-* `--with-libevent(=VALUE)`
-  This option specifies where to find the libevent support headers and
-  library.  The following `VALUE`s are permitted:
-
-  * `internal`: Use Open MPI's internal copy of libevent.
-  * `external`: Use an external Libevent installation (rely on default
-    compiler and linker paths to find it)
-  * `<no value>`:  Same as `internal`.
-  * `DIR`: Specify the location of a specific libevent
-    installation to use
-
-  By default (or if `--with-libevent` is specified with no `VALUE`),
-  Open MPI will build and use the copy of libevent that it has in its
-  source tree.  However, if the `VALUE` is `external`, Open MPI will
-  look for the relevant libevent header file and library in default
-  compiler / linker locations.  Or, `VALUE` can be a directory tree
-  where the libevent header file and library can be found.  This
-  option allows operating systems to include Open MPI and use their
-  default libevent installation instead of Open MPI's bundled
-  libevent.
-
-  libevent is a support library that provides event-based processing,
-  timers, and signal handlers.  Open MPI requires libevent to build;
-  passing --without-libevent will cause configure to abort.
-
-* `--with-libevent-libdir=DIR`:
-  Look in directory for the libevent libraries.  This option is only
-  usable when building Open MPI against an external libevent
-  installation.  Just like other `--with-FOO-libdir` configure
-  options, this option is only needed for special configurations.
-
-* `--with-hwloc(=VALUE)`:
-  hwloc is a support library that provides processor and memory
-  affinity information for NUMA platforms.  It is required by Open
-  MPI.  Therefore, specifying `--with-hwloc=no` (or `--without-hwloc`)
-  is disallowed.
-
-  By default (i.e., if `--with-hwloc` is not specified, or if
-  `--with-hwloc` is specified without a value), Open MPI will first try
-  to find/use an hwloc installation on the current system.  If Open
-  MPI cannot find one, it will fall back to build and use the internal
-  copy of hwloc included in the Open MPI source tree.
-
-  Alternatively, the `--with-hwloc` option can be used to specify
-  where to find the hwloc support headers and library.  The following
-  `VALUE`s are permitted:
-
-  * `internal`: Only use Open MPI's internal copy of hwloc.
-  * `external`: Only use an external hwloc installation (rely on
-    default compiler and linker paths to find it).
-  * `DIR`: Only use the specific hwloc installation found in
-    the specified directory.
-
-* `--with-hwloc-libdir=DIR`:
-  Look in directory for the hwloc libraries.  This option is only
-  usable when building Open MPI against an external hwloc
-  installation.  Just like other `--with-FOO-libdir` configure options,
-  this option is only needed for special configurations.
-
-* `--disable-hwloc-pci`:
-  Disable building hwloc's PCI device-sensing capabilities.  On some
-  platforms (e.g., SusE 10 SP1, x86-64), the libpci support library is
-  broken.  Open MPI's configure script should usually detect when
-  libpci is not usable due to such brokenness and turn off PCI
-  support, but there may be cases when configure mistakenly enables
-  PCI support in the presence of a broken libpci.  These cases may
-  result in `make` failing with warnings about relocation symbols in
-  libpci.  The `--disable-hwloc-pci` switch can be used to force Open
-  MPI to not build hwloc's PCI device-sensing capabilities in these
-  cases.
-
-  Similarly, if Open MPI incorrectly decides that libpci is broken,
-  you can force Open MPI to build hwloc's PCI device-sensing
-  capabilities by using `--enable-hwloc-pci`.
-
-  hwloc can discover PCI devices and locality, which can be useful for
-  Open MPI in assigning message passing resources to MPI processes.
-
-* `--with-libltdl=DIR`:
-  Specify the directory where the GNU Libtool libltdl libraries and
-  header files are located.  This option is generally only necessary
-  if the libltdl headers and libraries are not in default
-  compiler/linker search paths.
-
-  Note that this option is ignored if `--disable-dlopen` is specified.
-
-* `--disable-libompitrace`:
-  Disable building the simple `libompitrace` library (see note above
-  about libompitrace)
-
-* `--with-valgrind(=DIR)`:
-  Directory where the valgrind software is installed.  If Open MPI
-  finds Valgrind's header files, it will include additional support
-  for Valgrind's memory-checking debugger.
-
-  Specifically, it will eliminate a lot of false positives from
-  running Valgrind on MPI applications.  There is a minor performance
-  penalty for enabling this option.
-
-
-### MPI Functionality
-
-* `--with-mpi-param-check(=VALUE)`:
-  Whether or not to check MPI function parameters for errors at
-  runtime.  The following `VALUE`s are permitted:
-
-  * `always`: MPI function parameters are always checked for errors
-  * `never`: MPI function parameters are never checked for errors
-  * `runtime`: Whether MPI function parameters are checked depends on
-    the value of the MCA parameter `mpi_param_check` (default: yes).
-  * `yes`: Synonym for "always" (same as `--with-mpi-param-check`).
-  * `no`: Synonym for "never" (same as `--without-mpi-param-check`).
-
-  If `--with-mpi-param` is not specified, `runtime` is the default.
-
-* `--disable-mpi-thread-multiple`:
-  Disable the MPI thread level `MPI_THREAD_MULTIPLE` (it is enabled by
-  default).
-
-* `--enable-mpi-java`:
-  Enable building of an ***EXPERIMENTAL*** Java MPI interface
-  (disabled by default).  You may also need to specify
-  `--with-jdk-dir`, `--with-jdk-bindir`, and/or `--with-jdk-headers`.
-  See [README.JAVA.md](README.JAVA.md) for details.
-
-  Note that this Java interface is ***INCOMPLETE*** (meaning: it does
-  not support all MPI functionality) and ***LIKELY TO CHANGE***.  The
-  Open MPI developers would very much like to hear your feedback about
-  this interface.  See [README.JAVA.md](README.JAVA.md) for more
-  details.
-
-* `--enable-mpi-fortran(=VALUE)`:
-  By default, Open MPI will attempt to build all 3 Fortran bindings:
-  `mpif.h`, the `mpi` module, and the `mpi_f08` module.  The following
-  `VALUE`s are permitted:
-
-  * `all`: Synonym for `yes`.
-  * `yes`: Attempt to build all 3 Fortran bindings; skip
-    any binding that cannot be built (same as
-    `--enable-mpi-fortran`).
-  * `mpifh`: Only build `mpif.h` support.
-  * `usempi`: Only build `mpif.h` and `mpi` module support.
-  * `usempif08`:  Build `mpif.h`, `mpi` module, and `mpi_f08`
-     module support.
-  * `none`: Synonym for `no`.
-  * `no`: Do not build any MPI Fortran support (same as
-    `--disable-mpi-fortran`).  This is mutually exclusive
-     with building the OpenSHMEM Fortran interface.
-
-* `--enable-mpi-ext(=LIST)`:
-  Enable Open MPI's non-portable API extensions.  `LIST` is a
-  comma-delmited list of extensions.  If no `LIST` is specified, all
-  of the extensions are enabled.
-
-  See the "Open MPI API Extensions" section for more details.
-
-* `--disable-mpi-io`:
-  Disable built-in support for MPI-2 I/O, likely because an
-  externally-provided MPI I/O package will be used. Default is to use
-  the internal framework system that uses the ompio component and a
-  specially modified version of ROMIO that fits inside the romio
-  component
-
-* `--disable-io-romio`:
-  Disable the ROMIO MPI-IO component
-
-* `--with-io-romio-flags=FLAGS`:
-  Pass `FLAGS` to the ROMIO distribution configuration script.  This
-  option is usually only necessary to pass
-  parallel-filesystem-specific preprocessor/compiler/linker flags back
-  to the ROMIO system.
-
-* `--disable-io-ompio`:
-  Disable the ompio MPI-IO component
-
-* `--enable-sparse-groups`:
-  Enable the usage of sparse groups. This would save memory
-  significantly especially if you are creating large
-  communicators. (Disabled by default)
-
-
-### OpenSHMEM Functionality
-
-* `--disable-oshmem`:
-  Disable building the OpenSHMEM implementation (by default, it is
-  enabled).
-
-* `--disable-oshmem-fortran`:
-  Disable building only the Fortran OpenSHMEM bindings. Please see
-  the "Compiler Notes" section herein which contains further
-  details on known issues with various Fortran compilers.
-
-
-### Miscellaneous Functionality
-
-* `--without-memory-manager`:
-  Disable building Open MPI's memory manager.  Open MPI's memory
-  manager is usually built on Linux based platforms, and is generally
-  only used for optimizations with some OpenFabrics-based networks (it
-  is not *necessary* for OpenFabrics networks, but some performance
-  loss may be observed without it).
-
-  However, it may be necessary to disable the memory manager in order
-  to build Open MPI statically.
-
-* `--with-ft=TYPE`:
-  Specify the type of fault tolerance to enable.  Options: LAM
-  (LAM/MPI-like), cr (Checkpoint/Restart).  Fault tolerance support is
-  disabled unless this option is specified.
-
-* `--enable-peruse`:
-  Enable the PERUSE MPI data analysis interface.
-
-* `--enable-heterogeneous`:
-  Enable support for running on heterogeneous clusters (e.g., machines
-  with different endian representations).  Heterogeneous support is
-  disabled by default because it imposes a minor performance penalty.
-
-  ***THIS FUNCTIONALITY IS CURRENTLY BROKEN - DO NOT USE***
-
-* `--with-wrapper-cflags=CFLAGS`
-* `--with-wrapper-cxxflags=CXXFLAGS`
-* `--with-wrapper-fflags=FFLAGS`
-* `--with-wrapper-fcflags=FCFLAGS`
-* `--with-wrapper-ldflags=LDFLAGS`
-* `--with-wrapper-libs=LIBS`:
-  Add the specified flags to the default flags that are used in Open
-  MPI's "wrapper" compilers (e.g., `mpicc` -- see below for more
-  information about Open MPI's wrapper compilers).  By default, Open
-  MPI's wrapper compilers use the same compilers used to build Open
-  MPI and specify a minimum set of additional flags that are necessary
-  to compile/link MPI applications.  These configure options give
-  system administrators the ability to embed additional flags in
-  OMPI's wrapper compilers (which is a local policy decision).  The
-  meanings of the different flags are:
-
-  `CFLAGS`: Flags passed by the `mpicc` wrapper to the C compiler
-  `CXXFLAGS`: Flags passed by the `mpic++` wrapper to the C++ compiler
-  `FCFLAGS`: Flags passed by the `mpifort` wrapper to the Fortran compiler
-  `LDFLAGS`: Flags passed by all the wrappers to the linker
-  `LIBS`: Flags passed by all the wrappers to the linker
-
-  There are other ways to configure Open MPI's wrapper compiler
-  behavior; see [the Open MPI FAQ](https://www.open-mpi.org/faq/) for
-  more information.
-
-There are many other options available -- see `./configure --help`.
-
-Changing the compilers that Open MPI uses to build itself uses the
-standard Autoconf mechanism of setting special environment variables
-either before invoking configure or on the configure command line.
-The following environment variables are recognized by configure:
-
-* `CC`: C compiler to use
-* `CFLAGS`: Compile flags to pass to the C compiler
-* `CPPFLAGS`: Preprocessor flags to pass to the C compiler
-* `CXX`: C++ compiler to use
-* `CXXFLAGS`: Compile flags to pass to the C++ compiler
-* `CXXCPPFLAGS`: Preprocessor flags to pass to the C++ compiler
-* `FC`: Fortran compiler to use
-* `FCFLAGS`: Compile flags to pass to the Fortran compiler
-* `LDFLAGS`: Linker flags to pass to all compilers
-* `LIBS`: Libraries to pass to all compilers (it is rarely
-   necessary for users to need to specify additional `LIBS`)
-* `PKG_CONFIG`: Path to the `pkg-config` utility
-
-For example:
-
-```
-shell$ ./configure CC=mycc CXX=myc++ FC=myfortran ...
-```
-
-***NOTE:*** We generally suggest using the above command line form for
-setting different compilers (vs. setting environment variables and
-then invoking `./configure`).  The above form will save all variables
-and values in the `config.log` file, which makes post-mortem analysis
-easier if problems occur.
-
-Note that if you intend to compile Open MPI with a `make` other than
-the default one in your `PATH`, then you must either set the `$MAKE`
-environment variable before invoking Open MPI's `configure` script, or
-pass `MAKE=your_make_prog` to configure.  For example:
-
-```
-shell$ ./configure MAKE=/path/to/my/make ...
-```
-
-This could be the case, for instance, if you have a shell alias for
-`make`, or you always type `gmake` out of habit.  Failure to tell
-`configure` which non-default `make` you will use to compile Open MPI
-can result in undefined behavior (meaning: don't do that).
-
-Note that you may also want to ensure that the value of
-`LD_LIBRARY_PATH` is set appropriately (or not at all) for your build
-(or whatever environment variable is relevant for your operating
-system).  For example, some users have been tripped up by setting to
-use a non-default Fortran compiler via the `FC` environment variable,
-but then failing to set `LD_LIBRARY_PATH` to include the directory
-containing that non-default Fortran compiler's support libraries.
-This causes Open MPI's `configure` script to fail when it tries to
-compile / link / run simple Fortran programs.
-
-It is required that the compilers specified be compile and link
-compatible, meaning that object files created by one compiler must be
-able to be linked with object files from the other compilers and
-produce correctly functioning executables.
-
-Open MPI supports all the `make` targets that are provided by GNU
-Automake, such as:
-
-* `all`: build the entire Open MPI package
-* `install`: install Open MPI
-* `uninstall`: remove all traces of Open MPI from the $prefix
-* `clean`: clean out the build tree
-
-Once Open MPI has been built and installed, it is safe to run `make
-clean` and/or remove the entire build tree.
-
-VPATH and parallel builds are fully supported.
-
-Generally speaking, the only thing that users need to do to use Open
-MPI is ensure that `PREFIX/bin` is in their `PATH` and `PREFIX/lib` is
-in their `LD_LIBRARY_PATH`.  Users may need to ensure to set the
-`PATH` and `LD_LIBRARY_PATH` in their shell setup files (e.g.,
-`.bashrc`, `.cshrc`) so that non-interactive `rsh`/`ssh`-based logins
-will be able to find the Open MPI executables.
-
-
-## Open MPI Version Numbers and Binary Compatibility
-
-Open MPI has two sets of version numbers that are likely of interest
-to end users / system administrator:
-
-1. Software version number
-1. Shared library version numbers
-
-Both are predicated on Open MPI's definition of "backwards
-compatibility."
-
-***NOTE:*** The version numbering conventions were changed with the
-release of v1.10.0.  Most notably, Open MPI no longer uses an
-"odd/even" release schedule to indicate feature development vs. stable
-releases.  See the README in releases prior to v1.10.0 for more
-information (e.g.,
-https://github.com/open-mpi/ompi/blob/v1.8/README#L1392-L1475).
-
-
-### Backwards Compatibility
-
-Open MPI version Y is backwards compatible with Open MPI version X
-(where Y>X) if users can:
-
-* Compile an MPI/OpenSHMEM application with version X,
-  `mpirun`/`oshrun` it with version Y, and get the same
-  user-observable behavior.
-* Invoke `ompi_info` with the same CLI options in versions X and Y and
-  get the same user-observable behavior.
-
-Note that this definition encompasses several things:
-
-* Application Binary Interface (ABI)
-* MPI / OpenSHMEM run time system
-* `mpirun` / `oshrun` command line options
-* MCA parameter names / values / meanings
-
-However, this definition only applies when the same version of Open
-MPI is used with all instances of the runtime and MPI / OpenSHMEM
-processes in a single MPI job.  If the versions are not exactly the
-same everywhere, Open MPI is not guaranteed to work properly in any
-scenario.
-
-Backwards compatibility tends to work best when user applications are
-dynamically linked to one version of the Open MPI / OSHMEM libraries,
-and can be updated at run time to link to a new version of the Open
-MPI / OSHMEM libraries.
-
-For example, if an MPI / OSHMEM application links statically against
-the libraries from Open MPI vX, then attempting to launch that
-application with `mpirun` / `oshrun` from Open MPI vY is not guaranteed to
-work (because it is mixing vX and vY of Open MPI in a single job).
-
-Similarly, if using a container technology that internally bundles all
-the libraries from Open MPI vX, attempting to launch that container
-with `mpirun` / `oshrun` from Open MPI vY is not guaranteed to work.
-
-### Software Version Number
-
-Official Open MPI releases use the common "A.B.C" version identifier
-format.  Each of the three numbers has a specific meaning:
-
-* Major: The major number is the first integer in the version string
-  Changes in the major number typically indicate a significant
-  change in the code base and/or end-user functionality, and also
-  indicate a break from backwards compatibility.  Specifically: Open
-  MPI releases with different major version numbers are not
-  backwards compatibale with each other.
-
-  ***CAVEAT:*** This rule does not extend to versions prior to v1.10.0.
-  Specifically: v1.10.x is not guaranteed to be backwards
-  compatible with other v1.x releases.
-
-* Minor: The minor number is the second integer in the version string.
-  Changes in the minor number indicate a user-observable change in the
-  code base and/or end-user functionality.  Backwards compatibility
-  will still be preserved with prior releases that have the same major
-  version number (e.g., v2.5.3 is backwards compatible with v2.3.1).
-
-* Release: The release number is the third integer in the version
-  string.  Changes in the release number typically indicate a bug fix
-  in the code base and/or end-user functionality.  For example, if
-  there is a release that only contains bug fixes and no other
-  user-observable changes or new features, only the third integer will
-  be increased (e.g., from v4.3.0 to v4.3.1).
-
-The "A.B.C" version number may optionally be followed by a Quantifier:
-
-* Quantifier: Open MPI version numbers sometimes have an arbitrary
-  string affixed to the end of the version number. Common strings
-  include:
-  * aX: Indicates an alpha release. X is an integer indicating the
-    number of the alpha release (e.g., v1.10.3a5 indicates the 5th
-    alpha release of version 1.10.3).
-  * bX: Indicates a beta release. X is an integer indicating the
-    number of the beta release (e.g., v1.10.3b3 indicates the 3rd beta
-    release of version 1.10.3).
-  * rcX: Indicates a release candidate. X is an integer indicating the
-    number of the release candidate (e.g., v1.10.3rc4 indicates the
-    4th release candidate of version 1.10.3).
-
-Nightly development snapshot tarballs use a different version number
-scheme; they contain three distinct values:
-
-* The git branch name from which the tarball was created.
-* The date/timestamp, in `YYYYMMDDHHMM` format.
-* The hash of the git commit from which the tarball was created.
-
-For example, a snapshot tarball filename of
-`openmpi-v2.x-201703070235-e4798fb.tar.gz` indicates that this tarball
-was created from the v2.x branch, on March 7, 2017, at 2:35am GMT,
-from git hash e4798fb.
-
-### Shared Library Version Number
-
-The GNU Libtool official documentation details how the versioning
-scheme works.  The quick version is that the shared library versions
-are a triple of integers: (current,revision,age), or `c:r:a`.  This
-triple is not related to the Open MPI software version number.  There
-are six simple rules for updating the values (taken almost verbatim
-from the Libtool docs):
-
-1. Start with version information of `0:0:0` for each shared library.
-1. Update the version information only immediately before a public
-  release of your software. More frequent updates are unnecessary,
-  and only guarantee that the current interface number gets larger
-  faster.
-1. If the library source code has changed at all since the last
-   update, then increment revision (`c:r:a` becomes `c:r+1:a`).
-1. If any interfaces have been added, removed, or changed since the
-   last update, increment current, and set revision to 0.
-1. If any interfaces have been added since the last public release,
-   then increment age.
-1. If any interfaces have been removed since the last public release,
-   then set age to 0.
-
-Here's how we apply those rules specifically to Open MPI:
-
-1. The above rules do not apply to MCA components (a.k.a. "plugins");
-   MCA component `.so` versions stay unspecified.
-1. The above rules apply exactly as written to the following libraries
-   starting with Open MPI version v1.5 (prior to v1.5, `libopen-pal`
-   and `libopen-rte` were still at `0:0:0` for reasons discussed in bug
-   ticket #2092 https://svn.open-mpi.org/trac/ompi/ticket/2092):
-    * `libopen-rte`
-    * `libopen-pal`
-    * `libmca_common_*`
-1. The following libraries use a slightly modified version of the
-   above rules: rules 4, 5, and 6 only apply to the official MPI and
-   OpenSHMEM interfaces (functions, global variables).  The rationale
-   for this decision is that the vast majority of our users only care
-   about the official/public MPI/OpenSHMEM interfaces; we therefore
-   want the `.so` version number to reflect only changes to the
-   official MPI/OpenSHMEM APIs.  Put simply: non-MPI/OpenSHMEM API /
-   internal changes to the MPI-application-facing libraries are
-   irrelevant to pure MPI/OpenSHMEM applications.
-   * `libmpi`
-   * `libmpi_mpifh`
-   * `libmpi_usempi_tkr`
-   * `libmpi_usempi_ignore_tkr`
-   * `libmpi_usempif08`
-   * `libmpi_cxx`
-   * `libmpi_java`
-   * `liboshmem`
-
-
-## Checking Your Open MPI Installation
-
-The `ompi_info` command can be used to check the status of your Open
-MPI installation (located in `PREFIX/bin/ompi_info`).  Running it with
-no arguments provides a summary of information about your Open MPI
-installation.
-
-Note that the `ompi_info` command is extremely helpful in determining
-which components are installed as well as listing all the run-time
-settable parameters that are available in each component (as well as
-their default values).
-
-The following options may be helpful:
-
-* `--all`: Show a *lot* of information about your Open MPI
-  installation.
-* `--parsable`:  Display all the information in an easily
-  `grep`/`cut`/`awk`/`sed`-able format.
-* `--param FRAMEWORK COMPONENT`:
-  A `FRAMEWORK` value of `all` and a `COMPONENT` value of `all` will
-  show all parameters to all components.  Otherwise, the parameters of
-  all the components in a specific framework, or just the parameters
-  of a specific component can be displayed by using an appropriate
-  FRAMEWORK and/or COMPONENT name.
-* `--level LEVEL`:
-  By default, `ompi_info` only shows "Level 1" MCA parameters --
-  parameters that can affect whether MPI processes can run
-  successfully or not (e.g., determining which network interfaces to
-  use).  The `--level` option will display all MCA parameters from
-  level 1 to `LEVEL` (the max `LEVEL` value is 9).  Use `ompi_info
-  --param FRAMEWORK COMPONENT --level 9` to see *all* MCA parameters
-  for a given component.  See "The Modular Component Architecture
-  (MCA)" section, below, for a fuller explanation.
-
-Changing the values of these parameters is explained in the "The
-Modular Component Architecture (MCA)" section, below.
-
-When verifying a new Open MPI installation, we recommend running six
-tests:
-
-1. Use `mpirun` to launch a non-MPI program (e.g., `hostname` or
-   `uptime`) across multiple nodes.
-1. Use `mpirun` to launch a trivial MPI program that does no MPI
-   communication (e.g., the `hello_c` program in the `examples/`
-   directory in the Open MPI distribution).
-1. Use `mpirun` to launch a trivial MPI program that sends and
-   receives a few MPI messages (e.g., the `ring_c` program in the
-   `examples/` directory in the Open MPI distribution).
-1. Use `oshrun` to launch a non-OpenSHMEM program across multiple
-   nodes.
-1. Use `oshrun` to launch a trivial MPI program that does no OpenSHMEM
-   communication (e.g., `hello_shmem.c` program in the `examples/`
-   directory in the Open MPI distribution.)
-1. Use `oshrun` to launch a trivial OpenSHMEM program that puts and
-   gets a few messages (e.g., the `ring_shmem.c` in the `examples/`
-   directory in the Open MPI distribution.)
-
-If you can run all six of these tests successfully, that is a good
-indication that Open MPI built and installed properly.
-
-
-## Open MPI API Extensions
-
-Open MPI contains a framework for extending the MPI API that is
-available to applications.  Each extension is usually a standalone set
-of functionality that is distinct from other extensions (similar to
-how Open MPI's plugins are usually unrelated to each other).  These
-extensions provide new functions and/or constants that are available
-to MPI applications.
-
-WARNING: These extensions are neither standard nor portable to other
-MPI implementations!
-
-### Compiling the extensions
-
-Open MPI extensions are all enabled by default; they can be disabled
-via the `--disable-mpi-ext` command line switch.
-
-Since extensions are meant to be used by advanced users only, this
-file does not document which extensions are available or what they
-do.  Look in the ompi/mpiext/ directory to see the extensions; each
-subdirectory of that directory contains an extension.  Each has a
-README file that describes what it does.
-
-### Using the extensions
-
-To reinforce the fact that these extensions are non-standard, you must
-include a separate header file after `<mpi.h>` to obtain the function
-prototypes, constant declarations, etc.  For example:
-
-```c
-#include <mpi.h>
-#if defined(OPEN_MPI) && OPEN_MPI
-#include <mpi-ext.h>
-#endif
-
-int main() {
-    MPI_Init(NULL, NULL);
-
-#if defined(OPEN_MPI) && OPEN_MPI
-    {
-        char ompi_bound[OMPI_AFFINITY_STRING_MAX];
-        char current_binding[OMPI_AFFINITY_STRING_MAX];
-        char exists[OMPI_AFFINITY_STRING_MAX];
-        OMPI_Affinity_str(OMPI_AFFINITY_LAYOUT_FMT, ompi_bound,
-                          current_bindings, exists);
-    }
-#endif
-    MPI_Finalize();
-    return 0;
-}
-```
-
-Notice that the Open MPI-specific code is surrounded by the `#if`
-statement to ensure that it is only ever compiled by Open MPI.
-
-The Open MPI wrapper compilers (`mpicc` and friends) should
-automatically insert all relevant compiler and linker flags necessary
-to use the extensions.  No special flags or steps should be necessary
-compared to "normal" MPI applications.
-
-
-## Compiling Open MPI Applications
-
-Open MPI provides "wrapper" compilers that should be used for
-compiling MPI and OpenSHMEM applications:
-
-* C: `mpicc`, `oshcc`
-* C++: `mpiCC`, `oshCC` (or `mpic++` if your filesystem is case-insensitive)
-* Fortran: `mpifort`, `oshfort`
-
-For example:
-
-```
-shell$ mpicc hello_world_mpi.c -o hello_world_mpi -g
-shell$
-```
-
-For OpenSHMEM applications:
-
-```
-shell$ oshcc hello_shmem.c -o hello_shmem -g
-shell$
-```
-
-All the wrapper compilers do is add a variety of compiler and linker
-flags to the command line and then invoke a back-end compiler.  To be
-specific: the wrapper compilers do not parse source code at all; they
-are solely command-line manipulators, and have nothing to do with the
-actual compilation or linking of programs.  The end result is an MPI
-executable that is properly linked to all the relevant libraries.
-
-Customizing the behavior of the wrapper compilers is possible (e.g.,
-changing the compiler [not recommended] or specifying additional
-compiler/linker flags); see the Open MPI FAQ for more information.
-
-Alternatively, Open MPI also installs `pkg-config(1)` configuration
-files under `$libdir/pkgconfig`.  If `pkg-config` is configured to find
-these files, then compiling / linking Open MPI programs can be
-performed like this:
-
-```
-shell$ gcc hello_world_mpi.c -o hello_world_mpi -g \
-            `pkg-config ompi-c --cflags --libs`
-shell$
-```
-
-Open MPI supplies multiple `pkg-config(1)` configuration files; one
-for each different wrapper compiler (language):
-
-* `ompi`: Synonym for `ompi-c`; Open MPI applications using the C
-   MPI bindings
-* `ompi-c`: Open MPI applications using the C MPI bindings
-* `ompi-cxx`: Open MPI applications using the C MPI bindings
-* `ompi-fort`: Open MPI applications using the Fortran MPI bindings
-
-The following `pkg-config(1)` configuration files *may* be installed,
-depending on which command line options were specified to Open MPI's
-configure script.  They are not necessary for MPI applications, but
-may be used by applications that use Open MPI's lower layer support
-libraries.
-
-* `opal`: Open Portable Access Layer applications
-
-
-## Running Open MPI Applications
-
-Open MPI supports both `mpirun` and `mpiexec` (they are exactly
-equivalent) to launch MPI applications.  For example:
-
-```
-shell$ mpirun -np 2 hello_world_mpi
-or
-shell$ mpiexec -np 1 hello_world_mpi : -np 1 hello_world_mpi
-```
-
-are equivalent.
-
-The `rsh` launcher (which defaults to using `ssh`) accepts a
-`--hostfile` parameter (the option `--machinefile` is equivalent); you
-can specify a `--hostfile` parameter indicating a standard
-`mpirun`-style hostfile (one hostname per line):
-
-```
-shell$ mpirun --hostfile my_hostfile -np 2 hello_world_mpi
-```
-
-If you intend to run more than one process on a node, the hostfile can
-use the "slots" attribute.  If "slots" is not specified, a count of 1
-is assumed.  For example, using the following hostfile:
-
-```
-shell$ cat my_hostfile
-node1.example.com
-node2.example.com
-node3.example.com slots=2
-node4.example.com slots=4
-```
-
-```
-shell$ mpirun --hostfile my_hostfile -np 8 hello_world_mpi
-```
-
-will launch `MPI_COMM_WORLD` rank 0 on node1, rank 1 on node2, ranks 2
-and 3 on node3, and ranks 4 through 7 on node4.
-
-Other starters, such as the resource manager / batch scheduling
-environments, do not require hostfiles (and will ignore the hostfile
-if it is supplied).  They will also launch as many processes as slots
-have been allocated by the scheduler if no "-np" argument has been
-provided.  For example, running a SLURM job with 8 processors:
-
-```
-shell$ salloc -n 8 mpirun a.out
-```
-
-The above command will reserve 8 processors and run 1 copy of mpirun,
-which will, in turn, launch 8 copies of a.out in a single
-`MPI_COMM_WORLD` on the processors that were allocated by SLURM.
-
-Note that the values of component parameters can be changed on the
-`mpirun` / `mpiexec` command line.  This is explained in the section
-below, "The Modular Component Architecture (MCA)".
-
-Open MPI supports `oshrun` to launch OpenSHMEM applications. For
-example:
-
-```
-shell$ oshrun -np 2 hello_world_oshmem
-```
-
-OpenSHMEM applications may also be launched directly by resource
-managers such as SLURM. For example, when OMPI is configured
-`--with-pmix` and `--with-slurm`, one may launch OpenSHMEM applications
-via `srun`:
-
-```
-shell$ srun -N 2 hello_world_oshmem
-```
-
-## The Modular Component Architecture (MCA)
-
-The MCA is the backbone of Open MPI -- most services and functionality
-are implemented through MCA components.
-
-### MPI layer frameworks
-
-Here is a list of all the component frameworks in the MPI layer of
-Open MPI:
-
-* `bml`: BTL management layer
-* `coll`: MPI collective algorithms
-* `fbtl`: file byte transfer layer: abstraction for individual
-   read: collective read and write operations for MPI I/O
-* `fs`: file system functions for MPI I/O
-* `io`: MPI I/O
-* `mtl`: Matching transport layer, used for MPI point-to-point
-   messages on some types of networks
-* `op`: Back end computations for intrinsic MPI_Op operators
-* `osc`: MPI one-sided communications
-* `pml`: MPI point-to-point management layer
-* `rte`: Run-time environment operations
-* `sharedfp`: shared file pointer operations for MPI I/O
-* `topo`: MPI topology routines
-* `vprotocol`: Protocols for the "v" PML
-
-### OpenSHMEM component frameworks
-
-* `atomic`: OpenSHMEM atomic operations
-* `memheap`: OpenSHMEM memory allocators that support the
-  PGAS memory model
-* `scoll`: OpenSHMEM collective operations
-* `spml`: OpenSHMEM "pml-like" layer: supports one-sided,
-  point-to-point operations
-* `sshmem`: OpenSHMEM shared memory backing facility
-
-### Back-end run-time environment (RTE) component frameworks:
-
-* `dfs`: Distributed file system
-* `errmgr`: RTE error manager
-* `ess`: RTE environment-specific services
-* `filem`: Remote file management
-* `grpcomm`: RTE group communications
-* `iof`: I/O forwarding
-* `notifier`: System-level notification support
-* `odls`: OpenRTE daemon local launch subsystem
-* `oob`: Out of band messaging
-* `plm`: Process lifecycle management
-* `ras`: Resource allocation system
-* `rmaps`: Resource mapping system
-* `rml`: RTE message layer
-* `routed`: Routing table for the RML
-* `rtc`: Run-time control framework
-* `schizo`: OpenRTE personality framework
-* `state`: RTE state machine
-
-### Miscellaneous frameworks:
-
-* `allocator`: Memory allocator
-* `backtrace`: Debugging call stack backtrace support
-* `btl`: Point-to-point Byte Transfer Layer
-* `dl`: Dynamic loading library interface
-* `event`: Event library (libevent) versioning support
-* `hwloc`: Hardware locality (hwloc) versioning support
-* `if`: OS IP interface support
-* `installdirs`: Installation directory relocation services
-* `memchecker`: Run-time memory checking
-* `memcpy`: Memory copy support
-* `memory`: Memory management hooks
-* `mpool`: Memory pooling
-* `patcher`: Symbol patcher hooks
-* `pmix`: Process management interface (exascale)
-* `pstat`: Process status
-* `rcache`: Memory registration cache
-* `sec`: Security framework
-* `shmem`: Shared memory support (NOT related to OpenSHMEM)
-* `timer`: High-resolution timers
-
-### Framework notes
-
-Each framework typically has one or more components that are used at
-run-time.  For example, the `btl` framework is used by the MPI layer
-to send bytes across different types underlying networks.  The `tcp`
-`btl`, for example, sends messages across TCP-based networks; the
-`ucx` `pml` sends messages across InfiniBand-based networks.
-
-Each component typically has some tunable parameters that can be
-changed at run-time.  Use the `ompi_info` command to check a component
-to see what its tunable parameters are.  For example:
-
-```
-shell$ ompi_info --param btl tcp
-```
-
-shows some of the parameters (and default values) for the `tcp` `btl`
-component (use `--level` to show *all* the parameters; see below).
-
-Note that `ompi_info` only shows a small number a component's MCA
-parameters by default.  Each MCA parameter has a "level" value from 1
-to 9, corresponding to the MPI-3 MPI_T tool interface levels.  In Open
-MPI, we have interpreted these nine levels as three groups of three:
-
-1. End user / basic
-1. End user / detailed
-1. End user / all
-1. Application tuner / basic
-1. Application tuner / detailed
-1. Application tuner / all
-1. MPI/OpenSHMEM developer / basic
-1. MPI/OpenSHMEM developer / detailed
-1. MPI/OpenSHMEM developer / all
-
-Here's how the three sub-groups are defined:
-
-1. End user: Generally, these are parameters that are required for
-   correctness, meaning that someone may need to set these just to
-   get their MPI/OpenSHMEM application to run correctly.
-1. Application tuner: Generally, these are parameters that can be
-   used to tweak MPI application performance.
-1. MPI/OpenSHMEM developer: Parameters that either don't fit in the
-   other two, or are specifically intended for debugging /
-   development of Open MPI itself.
-
-Each sub-group is broken down into three classifications:
-
-1. Basic: For parameters that everyone in this category will want to
-   see.
-1. Detailed: Parameters that are useful, but you probably won't need
-   to change them often.
-1. All: All other parameters -- probably including some fairly
-   esoteric parameters.
-
-To see *all* available parameters for a given component, specify that
-ompi_info should use level 9:
-
-```
-shell$ ompi_info --param btl tcp --level 9
-```
-
-These values can be overridden at run-time in several ways.  At
-run-time, the following locations are examined (in order) for new
-values of parameters:
-
-1. `PREFIX/etc/openmpi-mca-params.conf`:
-   This file is intended to set any system-wide default MCA parameter
-   values -- it will apply, by default, to all users who use this Open
-   MPI installation.  The default file that is installed contains many
-   comments explaining its format.
-
-1. `$HOME/.openmpi/mca-params.conf`:
-   If this file exists, it should be in the same format as
-   `PREFIX/etc/openmpi-mca-params.conf`.  It is intended to provide
-   per-user default parameter values.
-
-1. environment variables of the form `OMPI_MCA_<name>` set equal to a
-   `VALUE`:
-
-   Where `<name>` is the name of the parameter.  For example, set the
-   variable named `OMPI_MCA_btl_tcp_frag_size` to the value 65536
-   (Bourne-style shells):
-
-   ```
-   shell$ OMPI_MCA_btl_tcp_frag_size=65536
-   shell$ export OMPI_MCA_btl_tcp_frag_size
-   ```
-
-4. the `mpirun`/`oshrun` command line: `--mca NAME VALUE`
-
-   Where <name> is the name of the parameter.  For example:
-
-   ```
-   shell$ mpirun --mca btl_tcp_frag_size 65536 -np 2 hello_world_mpi
-   ```
-
-These locations are checked in order.  For example, a parameter value
-passed on the `mpirun` command line will override an environment
-variable; an environment variable will override the system-wide
-defaults.
-
-Each component typically activates itself when relevant.  For example,
-the usNIC component will detect that usNIC devices are present and
-will automatically be used for MPI communications.  The SLURM
-component will automatically detect when running inside a SLURM job
-and activate itself.  And so on.
-
-Components can be manually activated or deactivated if necessary, of
-course.  The most common components that are manually activated,
-deactivated, or tuned are the `btl` components -- components that are
-used for MPI point-to-point communications on many types common
-networks.
-
-For example, to *only* activate the `tcp` and `self` (process loopback)
-components are used for MPI communications, specify them in a
-comma-delimited list to the `btl` MCA parameter:
-
-```
-shell$ mpirun --mca btl tcp,self hello_world_mpi
-```
-
-To add shared memory support, add `sm` into the command-delimited list
-(list order does not matter):
-
-```
-shell$ mpirun --mca btl tcp,sm,self hello_world_mpi
-```
-
-(there used to be a `vader` BTL for shared memory support; it was
-renamed to `sm` in Open MPI v5.0.0, but the alias `vader` still works
-as well)
-
-To specifically deactivate a specific component, the comma-delimited
-list can be prepended with a `^` to negate it:
-
-```
-shell$ mpirun --mca btl ^tcp hello_mpi_world
-```
-
-The above command will use any other `btl` component other than the
-`tcp` component.
-
-
-## Questions?  Problems?
-
-Found a bug?  Got a question?  Want to make a suggestion?  Want to
-contribute to Open MPI?  Please let us know!
-
-When submitting questions and problems, be sure to include as much
-extra information as possible.  [See the community help web
-page](https://www.open-mpi.org/community/help/) for details on all the
-information that we request in order to provide assistance:
-
-The best way to report bugs, send comments, or ask questions is to
-sign up on the user's and/or developer's mailing list (for user-level
-and developer-level questions; when in doubt, send to the user's
-list):
-
-* users@lists.open-mpi.org
-* devel@lists.open-mpi.org
-
-Because of spam, only subscribers are allowed to post to these lists
-(ensure that you subscribe with and post from exactly the same e-mail
-address -- joe@example.com is considered different than
-joe@mycomputer.example.com!).  Visit these pages to subscribe to the
-lists:
-
-* [Subscribe to the users mailing
-  list](https://lists.open-mpi.org/mailman/listinfo/users)
-* [Subscribe to the developers mailing
-  list](https://lists.open-mpi.org/mailman/listinfo/devel)
-
-Make today an Open MPI day!
+[The Open MPI Project](https://www.open-mpi.org/) is an open source
+implementation of the [Message Passing Interface (MPI)
+specification](https://www.mpi-forum.org/docs/) that is developed and
+maintained by a consortium of academic, research, and industry
+partners.  Open MPI is therefore able to combine the expertise,
+technologies, and resources from all across the High Performance
+Computing community in order to build the best MPI library available.
+Open MPI offers advantages for system and software vendors,
+application developers and computer science researchers.
+
+## Official documentation
+
+The Open MPI documentation can be viewed in the following ways:
+
+1. Online at https://docs.open-mpi.org/
+1. In self-contained (i.e., suitable for local viewing, without an
+   internet connection) in official distribution tarballs under
+   `docs/_build/html/index.html`.
+
+## Building the documentation locally
+
+The source code for Open MPI's docs can be found in the Open MPI Git
+repository under the `docs` folder.
+
+Developers who clone the Open MPI Git repository will not have the
+HTML documentation and man pages by default; it must be built.
+Instructions for how to build the Open MPI documentation can be found
+here:
+https://docs.open-mpi.org/en/main/developers/prerequisites.html#sphinx-and-therefore-python.
diff --git a/VERSION b/VERSION
index d616e1c6c77..f1c8928faf6 100644
--- a/VERSION
+++ b/VERSION
@@ -3,7 +3,7 @@
 # Copyright (c) 2011      NVIDIA Corporation.  All rights reserved.
 # Copyright (c) 2013      Mellanox Technologies, Inc.
 #                         All rights reserved.
-# Copyright (c) 2016      IBM Corporation.  All rights reserved.
+# Copyright (c) 2016-2022 IBM Corporation.  All rights reserved.
 # Copyright (c) 2017      Los Alamos National Security, LLC.  All rights
 #                         reserved.
 
@@ -16,9 +16,24 @@
 # <major>.<minor>.<release>.
 
 major=5
-minor=0
+minor=1
 release=0
 
+# MPI Standard Compliance Level
+mpi_standard_version=3
+mpi_standard_subversion=1
+
+# OMPI required dependency versions.
+# List in x.y.z format.
+pmix_min_version=4.2.0
+prte_min_version=3.0.0
+hwloc_min_version=2.1.0
+event_min_version=2.0.21
+automake_min_version=1.13.4
+autoconf_min_version=2.69.0
+libtool_min_version=2.4.2
+flex_min_version=2.5.4
+
 # greek is generally used for alpha or beta release tags.  If it is
 # non-empty, it will be appended to the version number.  It does not
 # have to be numeric.  Common examples include: a1 (alpha release 1),
@@ -103,4 +118,3 @@ libmca_opal_common_cuda_so_version=0:0:0
 libmca_opal_common_ofi_so_version=0:0:0
 libmca_opal_common_sm_so_version=0:0:0
 libmca_opal_common_ucx_so_version=0:0:0
-libmca_opal_common_ugni_so_version=0:0:0
diff --git a/autogen.pl b/autogen.pl
index 67361f0ec81..ecc726b54eb 100755
--- a/autogen.pl
+++ b/autogen.pl
@@ -1,16 +1,17 @@
 #!/usr/bin/env perl
 #
-# Copyright (c) 2009-2021 Cisco Systems, Inc.  All rights reserved
+# Copyright (c) 2009-2022 Cisco Systems, Inc.  All rights reserved
 # Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved.
 # Copyright (c) 2013      Mellanox Technologies, Inc.
 #                         All rights reserved.
 # Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
-# Copyright (c) 2015-2020 Research Organization for Information Science
+# Copyright (c) 2015-2024 Research Organization for Information Science
 #                         and Technology (RIST).  All rights reserved.
-# Copyright (c) 2015-2021 IBM Corporation.  All rights reserved.
+# Copyright (c) 2015-2022 IBM Corporation.  All rights reserved.
 # Copyright (c) 2020      Amazon.com, Inc. or its affiliates.
 #                         All Rights reserved.
 #
+# Copyright (c) 2025      Nanook Consulting  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -43,14 +44,9 @@
 # Data structures to fill up with all the stuff we find
 my $mca_found;
 my $mpiext_found;
-my $mpicontrib_found;
 my @subdirs;
 
 # Command line parameters
-my $no_ompi_arg = 0;
-my $no_orte_arg = 0;
-my $no_prrte_arg = 0;
-my $no_oshmem_arg = 0;
 my $no_3rdparty_arg = "";
 my $quiet_arg = 0;
 my $debug_arg = 0;
@@ -66,9 +62,9 @@
 my $exclude_list;
 
 # Minimum versions
-my $ompi_automake_version = "1.13.4";
-my $ompi_autoconf_version = "2.69";
-my $ompi_libtool_version = "2.4.2";
+my $ompi_automake_version;
+my $ompi_autoconf_version;
+my $ompi_libtool_version;
 
 # Search paths
 my $ompi_autoconf_search = "autoconf";
@@ -76,8 +72,8 @@
 my $ompi_libtoolize_search = "libtoolize;glibtoolize";
 
 # version of packages we ship as tarballs
-my $libevent_version="2.1.12-stable";
-my $hwloc_version="2.4.0";
+my $libevent_version="2.1.12-stable-ompi";
+my $hwloc_version="2.7.1";
 
 # One-time setup
 my $username;
@@ -86,12 +82,6 @@
 
 # Patch program
 my $patch_prog = "patch";
-# Solaris "patch" doesn't understand unified diffs, and will cause
-# autogen.pl to hang with a "File to patch:" prompt. Default to Linux
-# "patch", but use "gpatch" on Solaris.
-if ($^O eq "solaris") {
-    $patch_prog = "gpatch";
-}
 
 $username = $ENV{USER} || getpwuid($>);
 $full_hostname = $ENV{HOSTNAME} || `hostname`;
@@ -134,6 +124,18 @@ sub debug_dump {
 
 ##############################################################################
 
+sub list_contains {
+    my $searched_string = shift;
+    foreach my $str (@_) {
+	if ($searched_string eq $str) {
+	    return 1;
+	}
+    }
+    return 0;
+}
+
+##############################################################################
+
 sub read_config_params {
     my ($filename, $dir_prefix) = @_;
 
@@ -231,7 +233,7 @@ sub process_autogen_subdirs {
                 # Note: there's no real technical reason to defer
                 # processing the subdirs.  It's more of an aesthetic
                 # reason -- don't interrupt the current flow of
-                # finding mca / ext / contribs (which is a nice, fast
+                # finding mca / ext (which is a nice, fast
                 # process).  Then process the subdirs (which is a slow
                 # process) all at once.
                 push(@subdirs, "$dir/$_");
@@ -348,10 +350,18 @@ sub mca_process_framework {
                 verbose "--- Found $pname / $framework / $d component\n";
 
                 # Skip if specifically excluded
-                if (exists($exclude_list->{$framework}) &&
-                    $exclude_list->{$framework}[0] eq $d) {
-                    verbose "    => Excluded\n";
-                    next;
+                if (exists($exclude_list->{$framework})) {
+                    my $tst = 0;
+                    foreach my $ck (@{$exclude_list->{$framework}}) {
+                        if ($ck eq $d) {
+                            verbose "    => Excluded\n";
+                            $tst = 1;
+                            last;
+                        }
+                    }
+                    if ($tst) {
+                        next;
+                    }
                 }
 
                 # Skip if the framework is on the include list, but
@@ -695,93 +705,6 @@ sub mpiext_run_global {
     }
 }
 
-##############################################################################
-
-sub mpicontrib_process {
-    my ($topdir, $contrib_prefix, $contribdir) = @_;
-
-    my $cdir = "$topdir/$contrib_prefix/$contribdir";
-    return
-        if (! -d $cdir);
-
-    # Process this directory (pretty much the same treatment as for
-    # MCA components, so it's in a sub).
-    my $found_contrib;
-
-    $found_contrib->{"name"} = $contribdir;
-
-    # Push the results onto the hash array
-    push(@{$mpicontrib_found}, $found_contrib);
-
-    # Is there an autogen.subdirs in here?
-    process_autogen_subdirs($cdir);
-}
-
-##############################################################################
-
-sub mpicontrib_run_global {
-    my ($contrib_prefix) = @_;
-
-    my $topdir = Cwd::cwd();
-
-    my $dir = "$topdir/$contrib_prefix";
-    opendir(DIR, $dir) ||
-        my_die "Can't open $dir directory";
-    foreach my $d (sort(readdir(DIR))) {
-        # Skip any non-directory, "base", or any dir that begins with "."
-        next
-            if (! -d "$dir/$d" || $d eq "base" || substr($d, 0, 1) eq ".");
-
-        # If this directory has a configure.m4, then it's an
-        # contrib.
-        if (-f "$dir/$d/configure.m4") {
-            verbose "=== Found $d MPI contrib";
-
-            # Check ignore status
-            if (ignored("$dir/$d")) {
-                verbose " (ignored)\n";
-            } else {
-                verbose "\n";
-                mpicontrib_process($topdir, $contrib_prefix, $d);
-            }
-        }
-    }
-    closedir(DIR);
-    debug_dump($mpicontrib_found);
-
-    #-----------------------------------------------------------------------
-
-    $m4 .= "\n$dnl_line
-$dnl_line
-$dnl_line
-
-dnl Open MPI contrib information
-$dnl_line\n\n";
-
-    # Array for all the m4_includes that we'll need to pick up the
-    # configure.m4's.
-    my @includes;
-    my $m4_config_contrib_list;
-
-    # Troll through each of the found contribs
-    foreach my $contrib (@{$mpicontrib_found}) {
-        my $c = $contrib->{name};
-        push(@includes, "$contrib_prefix/$c/configure.m4");
-        $m4_config_contrib_list .= ", $c";
-    }
-
-    $m4_config_contrib_list =~ s/^, //;
-
-    # List the M4 and no configure contribs
-    $m4 .= "dnl List of all MPI contribs
-m4_define([ompi_mpicontrib_list], [$m4_config_contrib_list])\n";
-    # List out all the m4_include
-    $m4 .= "\ndnl List of configure.m4 files to include\n";
-    foreach my $i (@includes) {
-        $m4 .= "m4_include([$i])\n";
-    }
-}
-
 ##############################################################################
 # Find and remove stale files
 
@@ -939,6 +862,10 @@ sub patch_autotools_output {
         verbose "$indent_str"."Patching \"-pthread\" option for NAG compiler in ltmain.sh\n";
         system("$patch_prog -N -p0 < $topdir/config/ltmain_nag_pthread.diff >/dev/null 2>&1");
         unlink("config/ltmain.sh.rej");
+
+        verbose "$indent_str"."Patching \"-framework\" option for flang compiler in ltmain.sh\n";
+        system("$patch_prog -N -p0 < $topdir/config/ltmain_flang_darwin.diff >/dev/null 2>&1");
+        unlink("config/ltmain.sh.rej");
     }
 
     # If there's no configure script, there's nothing else to do.
@@ -963,9 +890,9 @@ sub patch_autotools_output {
     # source tree); we can't fix it.  So all we can do is patch the
     # resulting configure script.  :-(
     push(@verbose_out, $indent_str . "Patching configure for Libtool PGI 10 fortran compiler name\n");
-    $c =~ s/gfortran g95 xlf95 f95 fort ifort ifc efc pgf95 lf95 ftn/gfortran g95 xlf95 f95 fort ifort ifc efc pgfortran pgf95 lf95 ftn/g;
-    $c =~ s/pgcc\* \| pgf77\* \| pgf90\* \| pgf95\*\)/pgcc* | pgf77* | pgf90* | pgf95* | pgfortran*)/g;
-    $c =~ s/pgf77\* \| pgf90\* \| pgf95\*\)/pgf77* | pgf90* | pgf95* | pgfortran*)/g;
+    $c =~ s/gfortran g95 xlf95 f95 fort ifort ifc efc pgf95 lf95 ftn/gfortran g95 xlf95 f95 fort ifort ifc efc pgfortran nvfortran pgf95 lf95 ftn/g;
+    $c =~ s/pgcc\* \| pgf77\* \| pgf90\* \| pgf95\*\)/pgcc* | pgf77* | pgf90* | pgf95* | pgfortran* | nvfortran*)/g;
+    $c =~ s/pgf77\* \| pgf90\* \| pgf95\*\)/pgf77* | pgf90* | pgf95* | pgfortran* | nvfortran*)/g;
 
     # Similar issue as above -- the PGI 10 version number broke <=LT
     # 2.2.6b's version number checking regexps.  Again, we can't fix the
@@ -995,7 +922,7 @@ sub patch_autotools_output {
                             '# ICC 10 doesn\047t accept -KPIC any more.\n.*\n\s+' .
 	                    "lt_prog_compiler_wl${tag}=";
         my $replace_string = "# Flang compiler
-      *flang)
+      *flang*)
 	lt_prog_compiler_wl${tag}='-Wl,'
 	lt_prog_compiler_pic${tag}='-fPIC -DPIC'
 	lt_prog_compiler_static${tag}='-static'
@@ -1045,10 +972,28 @@ sub patch_autotools_output {
 	  whole_archive_flag_spec${tag}=
 	  tmp_sharedflag='--shared' ;;
 	nagfor*)			# NAGFOR 5.3
-	  tmp_sharedflag='-Wl,-shared';;
+	  tmp_sharedflag='-Wl,-shared' ;;
 	xl";
 
-        push(@verbose_out, $indent_str . "Patching configure for NAG compiler ($tag)\n");
+        push(@verbose_out, $indent_str . "Patching configure for NAG compiler #1 ($tag)\n");
+        $c =~ s/$search_string/$replace_string/;
+
+        # Newer versions of Libtool have the previous patch already. Therefore,
+        # we add the support for convenience libraries separately
+        my $search_string = "whole_archive_flag_spec${tag}=" . '\n\s+' .
+            "tmp_sharedflag='--shared' ;;" . '\n\s+' .
+            'nagfor\052.*# NAGFOR 5.3\n\s+' .
+            "tmp_sharedflag='-Wl,-shared' ;;" . '\n\s+' .
+            'xl';
+        my $replace_string = "whole_archive_flag_spec${tag}=
+	  tmp_sharedflag='--shared' ;;
+        nagfor*)                        # NAGFOR 5.3
+          whole_archive_flag_spec${tag}='\$wl--whole-archive`for conv in \$convenience\\\"\\\"; do test  -n \\\"\$conv\\\" && new_convenience=\\\"\$new_convenience,\$conv\\\"; done; func_echo_all \\\"\$new_convenience\\\"` \$wl--no-whole-archive'
+          compiler_needs_object=yes
+          tmp_sharedflag='-Wl,-shared' ;;
+	xl";
+
+        push(@verbose_out, $indent_str . "Patching configure for NAG compiler #2 ($tag)\n");
         $c =~ s/$search_string/$replace_string/;
     }
 
@@ -1115,6 +1060,77 @@ sub patch_autotools_output {
       *)';
     $c =~ s/$search_string/$replace_string/g;
 
+    # Fix ifort support on OSX
+    # see https://ntq1982.github.io/files/20200621.html
+    $search_string = 'case \$cc_basename in
+      nagfor\*\)
+        # NAG Fortran compiler
+        lt_prog_compiler_wl_FC=\'-Wl,-Wl,,\'
+        lt_prog_compiler_pic_FC=\'-PIC\'
+        lt_prog_compiler_static_FC=\'-Bstatic\'
+        ;;';
+    $replace_string = "case \$cc_basename in
+      flang*)
+        # flang compiler
+        lt_prog_compiler_wl_FC='-Wl,'
+        lt_prog_compiler_pic_FC='-fPIC'
+        lt_prog_compiler_static_FC='-static'
+        ;;
+      icc* | ifort*)
+        # Intel Fortran compiler
+        lt_prog_compiler_wl_FC='-Wl,'
+        lt_prog_compiler_pic_FC='-fno-common -PIC'
+        lt_prog_compiler_static_FC=''
+        ;;
+      nagfor*)
+        # NAG Fortran compiler
+        lt_prog_compiler_wl_FC='-Wl,-Wl,,'
+        lt_prog_compiler_pic_FC='-PIC'
+        lt_prog_compiler_static_FC='-Bstatic'
+        ;;";
+    $c =~ s/$search_string/$replace_string/g;
+
+    $c =~ s/for ac_prog in gfortran f95 fort xlf95 ifort ifc efc pgfortran pgf95 lf95 f90 xlf90 pgf90 epcf90 nagfor/for ac_prog in gfortran f95 fort xlf95 ifort ifc efc pgfortran pgf95 lf95 f90 xlf90 pgf90 epcf90 nagfor nvfortran/g;
+    foreach my $tag (("", "_FC")) {
+        $search_string = 'tcc\*\)
+	# Fabrice Bellard et al\'s Tiny C Compiler
+	lt_prog_compiler_wl'."${tag}".'=\'-Wl,\'
+	lt_prog_compiler_pic'."${tag}".'=\'-fPIC\'
+	lt_prog_compiler_static'."${tag}".'=\'-static\'
+	;;';
+        $replace_string = "tcc*)
+        # Fabrice Bellard et al's Tiny C Compiler
+        lt_prog_compiler_wl${tag}='-Wl,'
+        lt_prog_compiler_pic${tag}='-fPIC'
+        lt_prog_compiler_static${tag}='-static'
+        ;;
+    nvc* | nvcc* | nvfortran*)
+	# NVIDIA Fortran compiler
+        lt_prog_compiler_wl${tag}='-Wl,'
+        lt_prog_compiler_pic${tag}='-fPIC'
+        lt_prog_compiler_static${tag}='-Bstatic'
+        ;;";
+        push(@verbose_out, $indent_str . "Patching configure for NVIDIA Fortran compiler (${tag})\n");
+        $c =~ s/$search_string/$replace_string/g;
+    }
+
+    $search_string = 'case \$cc_basename in
+     ifort\*\|nagfor\*\) _lt_dar_can_shared=yes ;;
+     \*\) _lt_dar_can_shared=\$GCC ;;
+  esac';
+    $replace_string = "case \$cc_basename in
+     flang*|ifort*|nagfor*) _lt_dar_can_shared=yes ;;
+     *) _lt_dar_can_shared=\$GCC ;;
+  esac";
+    push(@verbose_out, $indent_str . "Patching configure for flang compiler on Darwin (FC)\n");
+    $c =~ s/$search_string/$replace_string/g;
+
+    $search_string = 'archive_cmds_FC="\\\\\\$CC -dynamiclib';
+    $replace_string = 'archive_cmds_FC="\$CC --shared';
+    push(@verbose_out, $indent_str . "Patching configure for flang compiler on Darwin (FC)\n");
+    $c =~ s/$search_string/$replace_string/g;
+    $c =~ s/(archive_cmds_FC.*)-install_name \\\$rpath/$1-Wl,-install_name,\\\`echo \\\$rpath | sed \'s%^[ ]*%%\'\\\`/g;
+
     # Only write out verbose statements and a new configure if the
     # configure content actually changed
     return
@@ -1131,6 +1147,76 @@ sub patch_autotools_output {
     unlink("configure.patched");
 }
 
+sub export_version {
+    my ($name,$version) = @_;
+    $version =~ s/[^a-zA-Z0-9,.]//g;
+    my @version_splits = split(/\./,$version);
+    my $hex = sprintf("0x%04x%02x%02x", $version_splits[0], $version_splits[1], $version_splits[2]);
+    $m4 .= "m4_define([OMPI_${name}_MIN_VERSION], [$version])\n";
+    $m4 .= "m4_define([OMPI_${name}_NUMERIC_MIN_VERSION], [$hex])\n";
+}
+
+sub export_mpi_version {
+    my ($name,$version) = @_;
+    $version =~ s/[^a-zA-Z0-9,.]//g;
+    $m4 .= "m4_define([$name], [$version])\n";
+}
+
+sub get_and_define_min_versions() {
+
+    open(IN, "VERSION") || my_die "Can't open VERSION";
+    while (<IN>) {
+          my $line = $_;
+          my @fields = split(/=/,$line);
+          if ($fields[0] eq "automake_min_version") {
+              if ($fields[1] ne "\n") {
+                  $ompi_automake_version = $fields[1];
+              }
+          }
+          elsif($fields[0] eq "autoconf_min_version") {
+              if ($fields[1] ne "\n") {
+                  $ompi_autoconf_version = $fields[1];
+              }
+          }
+          elsif($fields[0] eq "libtool_min_version") {
+              if ($fields[1] ne "\n") {
+                  $ompi_libtool_version = $fields[1];
+              }
+          }
+          elsif($fields[0] eq "pmix_min_version") {
+              if ($fields[1] ne "\n") {
+                  export_version("PMIX", $fields[1]);
+              }
+          }
+          elsif($fields[0] eq "prte_min_version") {
+              if ($fields[1] ne "\n") {
+                  export_version("PRTE", $fields[1]);
+              }
+          }
+          elsif($fields[0] eq "hwloc_min_version") {
+              if ($fields[1] ne "\n") {
+                  export_version("HWLOC", $fields[1]);
+              }
+          }
+          elsif($fields[0] eq "event_min_version") {
+              if ($fields[1] ne "\n") {
+                  export_version("EVENT", $fields[1]);
+              }
+          }
+          elsif($fields[0] eq "mpi_standard_version") {
+              if ($fields[1] ne "\n") {
+                  export_mpi_version("MPI_VERSION_NUM", $fields[1]);
+              }
+          }
+          elsif($fields[0] eq "mpi_standard_subversion") {
+              if ($fields[1] ne "\n") {
+                  export_mpi_version("MPI_SUBVERSION_NUM", $fields[1]);
+              }
+          }
+    }
+    close(IN);
+}
+
 sub in_tarball {
     my $tarball = 0;
     open(IN, "VERSION") || my_die "Can't open VERSION";
@@ -1253,11 +1339,7 @@ sub replace_config_sub_guess {
 
 # Command line parameters
 
-my $ok = Getopt::Long::GetOptions("no-ompi" => \$no_ompi_arg,
-                                  "no-orte" => \$no_orte_arg,
-                                  "no-prrte" => \$no_prrte_arg,
-                                  "no-oshmem" => \$no_oshmem_arg,
-                                  "no-3rdparty=s" => \$no_3rdparty_arg,
+my $ok = Getopt::Long::GetOptions("no-3rdparty=s" => \$no_3rdparty_arg,
                                   "quiet|q" => \$quiet_arg,
                                   "debug|d" => \$debug_arg,
                                   "help|h" => \$help_arg,
@@ -1272,10 +1354,6 @@ sub replace_config_sub_guess {
     print "Invalid command line argument.\n\n"
         if (!$ok);
     print "Options:
-  --no-ompi | -no-ompi          Do not build the Open MPI layer
-  --no-orte | -no-orte          Do not build Open MPI's runtime support (alias for --no-prrte)
-  --no-prrte | -no-prrte        Do not build Open MPI's runtime support
-  --no-oshmem | -no-oshmem      Do not build the OSHMEM layer
   --no-3rdparty <package>       Do not build the listed 3rd-party package (comma separtated list)
   --quiet | -q                  Do not display normal verbose output
   --debug | -d                  Output lots of debug information
@@ -1300,24 +1378,6 @@ sub replace_config_sub_guess {
 my $project_name_long = "Open MPI";
 my $project_name_short = "openmpi";
 
-if (! -e "ompi") {
-    $no_ompi_arg = 1;
-    debug "No ompi subdirectory found - will not build MPI layer\n";
-}
-if (! -e "oshmem") {
-    $no_oshmem_arg = 1;
-    debug "No oshmem subdirectory found - will not build OSHMEM\n";
-}
-# alias --no-orte to --no-prrte
-if ($no_orte_arg == 1) {
-    $no_prrte_arg = 1;
-}
-
-if ($no_ompi_arg == 1) {
-    $project_name_long = "Open Portability Access Layer";
-    $project_name_short = "open-pal";
-}
-
 my_die "Invalid value for --jobs $automake_jobs. Must be greater than 0."
     if (defined $automake_jobs && $automake_jobs <= 0);
 
@@ -1340,10 +1400,10 @@ sub replace_config_sub_guess {
 
 #---------------------------------------------------------------------------
 
-# Verify that we're in the OMPI root directorty by checking for a token file.
+# Verify that we're in the OMPI root directory by checking for a token file.
 
 my_die "Not at the root directory of an OMPI source tree"
-    if (! -f "config/opal_try_assemble.m4");
+    if (! -f "config/opal_mca.m4");
 
 my_die "autogen.pl has been invoked in the source tree of an Open MPI distribution tarball; aborting...
 You likely do not need to invoke \"autogen.pl\" -- you can probably run \"configure\" directly.
@@ -1361,6 +1421,8 @@ sub replace_config_sub_guess {
 
 $step. Checking tool versions\n\n";
 
+get_and_define_min_versions();
+
 # Check the autotools revision levels
 &find_and_check("autoconf", $ompi_autoconf_search, $ompi_autoconf_version);
 &find_and_check("libtool", $ompi_libtoolize_search, $ompi_libtool_version);
@@ -1371,24 +1433,49 @@ sub replace_config_sub_guess {
 ++$step;
 verbose "\n$step. Checking for git submodules\n\n";
 
+my @enabled_3rdparty_packages = ();
+my @disabled_3rdparty_packages = split(/,/, $no_3rdparty_arg);
+# Alias: 'openpmix' -> 'pmix'
+if (list_contains("openpmix", @disabled_3rdparty_packages)) {
+    push(@disabled_3rdparty_packages, "pmix");
+}
+
 # Make sure we got a submodule-full clone.  If not, abort and let a
 # human figure it out.
 if (-f ".gitmodules") {
-    open(IN, "git submodule status|")
-        || die "Can't run \"git submodule status\"";
+    # Do a quick sanity check to ensure that non-3rd-party
+    # submodules are at least present (e.g., they won't be present
+    # if you downloaded a GitHub.com-created tarball).
+    open(IN, ".gitmodules") ||
+        die "Can't open .gitmodules";
     while (<IN>) {
-        chomp;
-        $_ =~ m/^(.)(.{40}) ([^ ]+) *\(*([^\(\)]*)\)*$/;
-        my $status     = $1;
-        my $local_hash = $2;
-        my $path       = $3;
-        my $extra      = $4;
+        # Find "path = " lines
+        if (!($_ =~ m/^\s+path = (.+)$/)) {
+            next;
+        }
+        my $path = $1;
+
+        # Only care about paths that do not include "3rd-party"
+        if (index($path, "3rd-party") != -1) {
+            next;
+        }
+
+        # Check that the path exists and is non-empty.
+        my $happy = 1;
+        if (! -d $path) {
+            $happy = 0;
+        } else {
+            opendir(DIR, $path) ||
+                my_die "Can't open $path directory";
+            my @files = readdir(DIR);
+            closedir(DIR);
 
-        print("=== Submodule: $path\n");
+            $happy = 0
+                if ($#files < 2);
+        }
 
-        # Make sure the submodule is there
-        if ($status eq "-") {
-            print("    ==> ERROR: Missing
+        if (!$happy) {
+                print("    ==> ERROR: Missing
 
 The submodule \"$path\" is missing.
 
@@ -1396,19 +1483,45 @@ sub replace_config_sub_guess {
 \"git submodule update --init --recursive\"...?\n\n");
             exit(1);
         }
+    }
+
+    if (-d ".git") {
+        open(IN, "git submodule status|")
+            || die "Can't run \"git submodule status\"";
+        while (<IN>) {
+            $_ =~ m/^(.)[0-9a-f]{40}\s+(\S+)/;
+            my $status = $1;
+            my $path   = $2;
+
+            print("=== Submodule: $path\n");
+            if (index($path, "pmix") != -1 and list_contains("pmix", @disabled_3rdparty_packages)) {
+                print("Disabled - skipping openpmix");
+                next;
+            }
+            if (index($path, "prrte") != -1 and list_contains("prrte", @disabled_3rdparty_packages)) {
+                print("Disabled - skipping prrte");
+                next;
+            }
+
+            # Make sure the submodule is there
+            if ($status eq "-") {
+                print("    ==> ERROR: Missing
+
+The submodule \"$path\" is missing.
 
-        # See if the submodule is at the expected git hash
-        # (it may be ok if it's not -- just warn the user)
-        $extra =~ m/-g(.+)/;
-        my $remote_hash = $1;
-        if ($remote_hash) {
-            my $abbrev_local_hash = substr($local_hash, 0, length($remote_hash));
-            if ($remote_hash ne $abbrev_local_hash) {
+Perhaps you forgot to \"git clone --recursive ...\", or you need to
+\"git submodule update --init --recursive\"...?\n\n");
+                exit(1);
+            }
+
+            # See if the commit in the submodule is not the same as the
+            # commit that the git submodule thinks it should be.
+            elsif ($status eq "+") {
                 print("    ==> WARNING: Submodule hash is different than upstream.
          If this is not intentional, you may want to run:
          \"git submodule update --init --recursive\"\n");
             } else {
-                print("    Local hash == remote hash (good!)\n");
+                print("    Local hash is what is expected by the submodule (good!)\n");
             }
         }
     }
@@ -1527,10 +1640,8 @@ sub replace_config_sub_guess {
 # Top-level projects to examine
 my $projects;
 push(@{$projects}, { name => "opal", dir => "opal", need_base => 1 });
-push(@{$projects}, { name => "ompi", dir => "ompi", need_base => 1 })
-    if (!$no_ompi_arg);
-push(@{$projects}, { name => "oshmem", dir => "oshmem", need_base => 1 })
-    if (!$no_ompi_arg && !$no_oshmem_arg);
+push(@{$projects}, { name => "ompi", dir => "ompi", need_base => 1 });
+push(@{$projects}, { name => "oshmem", dir => "oshmem", need_base => 1 });
 
 $m4 .= "dnl Separate m4 define for each project\n";
 foreach my $p (@$projects) {
@@ -1550,12 +1661,6 @@ sub replace_config_sub_guess {
 ++$step;
 verbose "\n$step. Setup for 3rd-party packages\n";
 
-my @enabled_3rdparty_packages = ();
-my @disabled_3rdparty_packages = split(/,/, $no_3rdparty_arg);
-if ($no_prrte_arg) {
-    push(@disabled_3rdparty_packages, "prrte");
-}
-
 $m4 .= "\n$dnl_line
 $dnl_line
 $dnl_line
@@ -1563,7 +1668,8 @@ sub replace_config_sub_guess {
 dnl 3rd-party package information\n";
 
 # Extract the OMPI options to exclude them when processing PMIx and PRRTE
-if ( ! ("pmix" ~~ @disabled_3rdparty_packages && "prrte" ~~ @disabled_3rdparty_packages) ) {
+if ( ! (list_contains("pmix", @disabled_3rdparty_packages) &&
+	list_contains("prrte", @disabled_3rdparty_packages))) {
     safe_system("./config/extract-3rd-party-configure.pl -p . -n \"OMPI\" -l > config/auto-generated-ompi-exclude.ini");
 }
 
@@ -1571,7 +1677,7 @@ sub replace_config_sub_guess {
 # generic. Sorry :).
 
 verbose "=== Libevent\n";
-if ("libevent" ~~ @disabled_3rdparty_packages) {
+if (list_contains("libevent", @disabled_3rdparty_packages)) {
     verbose "--- Libevent disabled\n";
 } else {
     my $libevent_directory = "libevent-" . $libevent_version;
@@ -1586,7 +1692,7 @@ sub replace_config_sub_guess {
 }
 
 verbose "=== hwloc\n";
-if ("hwloc" ~~ @disabled_3rdparty_packages) {
+if (list_contains("hwloc", @disabled_3rdparty_packages)) {
     verbose "--- hwloc disabled\n";
 } else {
     my $hwloc_directory = "hwloc-" . $hwloc_version;
@@ -1597,11 +1703,11 @@ sub replace_config_sub_guess {
     $m4 .= "m4_define([package_hwloc], [1])\n";
     $m4 .= "m4_define([hwloc_tarball], [" . $hwloc_tarball . "])\n";
     $m4 .= "m4_define([hwloc_directory], [" . $hwloc_directory . "])\n";
-    verbose "--- hwloc enabled\n";
+    verbose "--- hwloc enabled (" . $hwloc_version . ")\n";
 }
 
 verbose "=== PMIx\n";
-if ("pmix" ~~ @disabled_3rdparty_packages) {
+if (list_contains("pmix", @disabled_3rdparty_packages)) {
     verbose "--- PMIx disabled\n";
 } else {
     # sanity check pmix files exist
@@ -1620,7 +1726,7 @@ sub replace_config_sub_guess {
 }
 
 verbose "=== PRRTE\n";
-if ("prrte" ~~ @disabled_3rdparty_packages) {
+if (list_contains("prrte", @disabled_3rdparty_packages)) {
     verbose "--- PRRTE disabled\n";
 } else {
     # sanity check prrte files exist
@@ -1642,16 +1748,10 @@ sub replace_config_sub_guess {
 
 #---------------------------------------------------------------------------
 
-# Find MPI extensions and contribs
-if (!$no_ompi_arg) {
-    ++$step;
-    verbose "\n$step. Searching for Open MPI extensions\n\n";
-    mpiext_run_global("ompi/mpiext");
-
-    ++$step;
-    verbose "\n$step. Searching for Open MPI contribs\n\n";
-    mpicontrib_run_global("ompi/contrib");
-}
+# Find MPI extensions
+++$step;
+verbose "\n$step. Searching for Open MPI extensions\n\n";
+mpiext_run_global("ompi/mpiext");
 
 #---------------------------------------------------------------------------
 
@@ -1694,7 +1794,7 @@ sub replace_config_sub_guess {
 # Run autoreconf
 verbose "==> Running autoreconf\n";
 chdir("..");
-my $cmd = "autoreconf -ivf --warnings=all,no-obsolete,no-override -I config";
+my $cmd = "autoreconf -ivf --warnings=all,no-obsolete,no-override -I config -I config/oac";
 foreach my $project (@{$projects}) {
     $cmd .= " -I $project->{dir}/config"
         if (-d "$project->{dir}/config");
diff --git a/config/Makefile.am b/config/Makefile.am
index 9d539357a75..4af3a33f87c 100644
--- a/config/Makefile.am
+++ b/config/Makefile.am
@@ -9,12 +9,13 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2006-2021 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2006-2022 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2010      Oracle and/or its affiliates.  All rights
 #                         reserved.
 # Copyright (c) 2014-2015 Intel, Inc. All rights reserved.
 # Copyright (c) 2016      Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -22,6 +23,19 @@
 # $HEADER$
 #
 
+# This is a weird place for this, but config/ is the first directory that
+# is entered, and the dist-hook runs at the end of the current directory,
+# so this is the earliest we can run this test.
+dist-hook:
+	@if test -n "$(OPAL_MAKEDIST_DISABLE)" ; then \
+		echo "#########################################################################"; \
+		echo "#"; \
+		echo "# make dist is disabled due to the following packages: $(OPAL_MAKEDIST_DISABLE)"; \
+		echo "#"; \
+		echo "#########################################################################"; \
+		exit 1; \
+	fi
+
 EXTRA_DIST = \
         distscript.sh \
         opal_get_version.m4sh \
@@ -30,10 +44,9 @@ EXTRA_DIST = \
         opal_mca_priority_sort.pl \
         find_common_syms \
         getdate.sh \
-        make_manpage.pl \
-        md2nroff.pl \
         from-savannah/upstream-config.guess \
-        from-savannah/upstream-config.sub
+        from-savannah/upstream-config.sub \
+        extract-3rd-party-configure.pl
 
 maintainer-clean-local:
 	rm -f opal_get_version.sh
diff --git a/config/c_get_alignment.m4 b/config/c_get_alignment.m4
index a300cc3d1a3..339018712d1 100644
--- a/config/c_get_alignment.m4
+++ b/config/c_get_alignment.m4
@@ -14,6 +14,7 @@ dnl Copyright (c) 2009      Sun Microsystems, Inc.  All rights reserved.
 dnl Copyright (c) 2014-2015 Intel, Inc. All rights reserved.
 dnl Copyright (c) 2015      Research Organization for Information Science
 dnl                         and Technology (RIST). All rights reserved.
+dnl Copyright (c) 2021      Cisco Systems, Inc.  All rights reserved.
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
@@ -21,10 +22,10 @@ dnl
 dnl $HEADER$
 dnl
 
-# OPAL_C_GET_ALIGN(type, config_var)
+# OPAL_C_GET_ALIGNMENT(c_type, c_macro_name)
 # ----------------------------------
 # Determine datatype alignment.
-# First arg is type, 2nd arg is config var to define.
+# First arg is type, 2nd arg is macro name to define.
 # Now that we require C99 compilers, we include stdbool.h
 # in the alignment test so that we can find the definition
 # of "bool" when we test for its alignment. We might be able
@@ -33,41 +34,26 @@ dnl
 # what we use. Yes, they should be the same - but "should" and
 # "are" frequently differ
 AC_DEFUN([OPAL_C_GET_ALIGNMENT],[
-    AC_CACHE_CHECK([alignment of $1],
-                   [AS_TR_SH([opal_cv_c_align_$1])],
-		   [AC_RUN_IFELSE([AC_LANG_PROGRAM([AC_INCLUDES_DEFAULT
-                                                    #include <stdbool.h> ],
-[[
-    struct foo { char c; $1 x; };
-    struct foo *p = (struct foo *) malloc(sizeof(struct foo));
-    int diff;
-    FILE *f=fopen("conftestval", "w");
-    if (!f) exit(1);
-    diff = ((char *)&p->x) - ((char *)&p->c);
-    free(p);
-    fprintf(f, "%d\n", (diff >= 0) ? diff : -diff);
-]])],                         [AS_TR_SH([opal_cv_c_align_$1])=`cat conftestval`],
-                               [AC_MSG_WARN([*** Problem running configure test!])
-                                AC_MSG_WARN([*** See config.log for details.])
-                                AC_MSG_ERROR([*** Cannot continue.])],
-                               [ # cross compile - do a non-executable test.  Trick
-                                 # taken from the Autoconf 2.59c.  Switch to using
-                                 # AC_CHECK_ALIGNOF when we can require Autoconf 2.60.
-                                 _AC_COMPUTE_INT([(long int) offsetof (opal__type_alignof_, y)],
-                                                 [AS_TR_SH([opal_cv_c_align_$1])],
-                                                 [AC_INCLUDES_DEFAULT
-#include <stdbool.h>
+    OPAL_VAR_SCOPE_PUSH([opal_align_value])
+    AC_LANG_PUSH([C])
 
-#ifndef offsetof
-# define offsetof(type, member) ((char *) &((type *) 0)->member - (char *) 0)
-#endif
-typedef struct { char x; $1 y; } opal__type_alignof_;
-],
-                                                 [AC_MSG_WARN([*** Problem running configure test!])
-                                                  AC_MSG_WARN([*** See config.log for details.])
-                                                  AC_MSG_ERROR([*** Cannot continue.])])])])
+    AC_CHECK_ALIGNOF([$1], [AC_INCLUDES_DEFAULT
+                            #include <stdbool.h>
+                           ])
 
-AC_DEFINE_UNQUOTED([$2], [$AS_TR_SH([opal_cv_c_align_$1])], [Alignment of type $1])
-eval "$2=$AS_TR_SH([opal_cv_c_align_$1])"
+    # Put the value determined from AC CHECK_ALIGNOF into an
+    # easy-to-access shell variable.
+    AS_VAR_COPY([opal_align_value],
+                [ac_cv_alignof_]AS_TR_SH([$1]))
 
-rm -rf conftest* ]) dnl
+    # This $opal_cv_c_align_* shell variable is used elsewhere in
+    # configure.ac
+    AS_VAR_COPY([opal_cv_c_align_]AS_TR_SH([$1]),
+                [opal_align_value])
+
+    # This #define is used in C code.
+    AC_DEFINE_UNQUOTED([$2], [$opal_align_value], [Alignment of $1])
+
+    AC_LANG_POP([C])
+    OPAL_VAR_SCOPE_POP
+])
diff --git a/config/extract-3rd-party-configure.pl b/config/extract-3rd-party-configure.pl
index 15ca89323d0..7afa31d9b4e 100755
--- a/config/extract-3rd-party-configure.pl
+++ b/config/extract-3rd-party-configure.pl
@@ -1,6 +1,7 @@
 #!/usr/bin/env perl
 #
 # Copyright (c) 2021      IBM Corporation.  All rights reserved.
+# Copyright (c) 2021 Cisco Systems.  All rights reserved.
 #
 # $COPYRIGHT$
 #
@@ -119,7 +120,7 @@ ($)
     #  AC_ARG_WITH (package, help-string, [action-if-given], [action-if-not-given])
     my @conf_keywords = ("AC_ARG_ENABLE", "AC_ARG_WITH");
 
-    # stack with funciton parameters: '(', ')'
+    # stack with function parameters: '(', ')'
     my @the_stack = ();
     # push(@the_stack) / pop(@the_stack) / scalar(@the_stack)
     my $arg_num = 0;
@@ -150,7 +151,7 @@ ($)
                         last;
                     }
 
-                    # Skip any option that contains a variable substition
+                    # Skip any option that contains a variable substitution
                     if ( $option_name =~ /\$/ ) {
                         if( !$list_only && !$quiet_arg) {
                             print "# Warning: Skipped (embedded variable): $option_name\n";
@@ -216,7 +217,7 @@ ($)
                 # Argument separator
                 # We only care about counting arguments for the outermost
                 # function (e.g., AC_ARG_ENABLE), not the innermost
-                # function (e.g., AC_HELP_STRING) - which we take all of.
+                # function (e.g., AS_HELP_STRING) - which we take all of.
                 # We know we are in the outermost because the '(' will be
                 # only thing on the stack.
                 elsif( scalar(@the_stack) == 1 && $char eq "," ) {
diff --git a/config/find_common_syms b/config/find_common_syms
index 26e603cb49b..84f4293ea4c 100755
--- a/config/find_common_syms
+++ b/config/find_common_syms
@@ -109,7 +109,7 @@ sub object_find {
     # that turns out to be the case, we can try switching to "nm -P", which is
     # supposed to activate the "portable" (yet ugly) format.  It's also unclear
     # at this point how common support for "nm -P" is.
-    open(NM, '-|', "nm '${obj}' 2>/dev/null | egrep '\\s[cC]\\s'");
+    open(NM, '-|', "nm '${obj}' 2>/dev/null | grep -E '\\s[cC]\\s'");
     SYMBOL: while (my $sym_line = <NM>) {
         if (!$all and is_allowlisted($sym_line)) {
             next SYMBOL;
diff --git a/config/ltmain_flang_darwin.diff b/config/ltmain_flang_darwin.diff
new file mode 100644
index 00000000000..02dc81d988a
--- /dev/null
+++ b/config/ltmain_flang_darwin.diff
@@ -0,0 +1,39 @@
+--- config/ltmain.sh
++++ config/ltmain.sh
+@@ -9024,7 +9024,14 @@
+	  compile_deplibs="$new_inherited_linker_flags $compile_deplibs"
+	  finalize_deplibs="$new_inherited_linker_flags $finalize_deplibs"
+	else
+-	  compiler_flags="$compiler_flags "`$ECHO " $new_inherited_linker_flags" | $SED 's% \([^ $]*\).ltframework% -framework \1%g'`
++	  case $host in
++	    *-*-darwin*)
++	      case $CC in
++		flang*) compiler_flags="$compiler_flags "`$ECHO " $new_inherited_linker_flags" | $SED 's% \([^ $]*\).ltframework% -Wl,-framework,\1%g'`;;
++		*) compiler_flags="$compiler_flags "`$ECHO " $new_inherited_linker_flags" | $SED 's% \([^ $]*\).ltframework% -framework \1%g'`;;
++	      esac;;
++	  *) compiler_flags="$compiler_flags "`$ECHO " $new_inherited_linker_flags" | $SED 's% \([^ $]*\).ltframework% -framework \1%g'`;;
++	  esac
+	fi
+       fi
+       dependency_libs=$newdependency_libs
+@@ -9369,7 +9376,7 @@
+           # On Darwin other compilers
+           func_cc_basename $CC
+           case $func_cc_basename_result in
+-              nagfor*)
++              flang*|nagfor*)
+                   verstring="$wl-compatibility_version $wl$minor_current $wl-current_version $wl$minor_current.$revision"
+                   ;;
+               *)
+@@ -9869,7 +9876,10 @@
+       # Time to change all our "foo.ltframework" stuff back to "-framework foo"
+       case $host in
+	  *-*-darwin*)
+-	  newdeplibs=`$ECHO " $newdeplibs" | $SED 's% \([^ $]*\).ltframework% -framework \1%g'`
++	  case $CC in
++	    flang*) newdeplibs=`$ECHO " $newdeplibs" | $SED 's% \([^ $]*\).ltframework% -Wl,-framework,\1%g'`;;
++	    *) newdeplibs=`$ECHO " $newdeplibs" | $SED 's% \([^ $]*\).ltframework% -framework \1%g'`;;
++	  esac
+	  new_inherited_linker_flags=`$ECHO " $new_inherited_linker_flags" | $SED 's% \([^ $]*\).ltframework% -framework \1%g'`
+	  deplibs=`$ECHO " $deplibs" | $SED 's% \([^ $]*\).ltframework% -framework \1%g'`
+	  ;;
diff --git a/config/ltmain_nag_pthread.diff b/config/ltmain_nag_pthread.diff
index 927b671f9ae..c1fbd0582df 100644
--- a/config/ltmain_nag_pthread.diff
+++ b/config/ltmain_nag_pthread.diff
@@ -1,18 +1,40 @@
 --- config/ltmain.sh
 +++ config/ltmain.sh
-@@ -6417,8 +6417,14 @@
- 	func_source "$lib"
-
+@@ -7862,6 +7862,13 @@ func_mode_link ()
  	# Convert "-framework foo" to "foo.ltframework"
-+        # and "-pthread" to "-Wl,-pthread" if NAG compiler
  	if test -n "$inherited_linker_flags"; then
--	  tmp_inherited_linker_flags=`$ECHO "$inherited_linker_flags" | $SED 's/-framework \([^ $]*\)/\1.ltframework/g'`
-+          case "$CC" in
-+            *nagfor*)
-+	      tmp_inherited_linker_flags=`$ECHO "$inherited_linker_flags" | $SED 's/-framework \([^ $]*\)/\1.ltframework/g' | $SED 's/-pthread/-Wl,-pthread/g'`;;
-+            *)
-+	      tmp_inherited_linker_flags=`$ECHO "$inherited_linker_flags" | $SED 's/-framework \([^ $]*\)/\1.ltframework/g'`;;
-+          esac
+ 	  tmp_inherited_linker_flags=`$ECHO "$inherited_linker_flags" | $SED 's/-framework \([^ $]*\)/\1.ltframework/g'`
++
++	  # Additionally convert " -pthread" to " -Wl,-pthread" for nagfor
++	  func_cc_basename $CC
++	  case $func_cc_basename_result in
++	    nagfor*) tmp_inherited_linker_flags=`$ECHO "$tmp_inherited_linker_flags" | $SED 's/ -pthread/ -Wl,-pthread/g'` ;;
++	  esac
++
  	  for tmp_inherited_linker_flag in $tmp_inherited_linker_flags; do
  	    case " $new_inherited_linker_flags " in
  	      *" $tmp_inherited_linker_flag "*) ;;
+@@ -8881,7 +8888,8 @@ func_mode_link ()
+ 	  xlcverstring="$wl-compatibility_version $wl$minor_current $wl-current_version $wl$minor_current.$revision"
+ 	  verstring="-compatibility_version $minor_current -current_version $minor_current.$revision"
+           # On Darwin other compilers
+-          case $CC in
++          func_cc_basename $CC
++          case $func_cc_basename_result in
+               nagfor*)
+                   verstring="$wl-compatibility_version $wl$minor_current $wl-current_version $wl$minor_current.$revision"
+                   ;;
+@@ -9493,6 +9501,13 @@ EOF
+ 	  ;;
+       esac
+ 
++      # Time to revert the changes made for nagfor.
++      func_cc_basename $CC
++      case $func_cc_basename_result in
++        nagfor*)
++          new_inherited_linker_flags=`$ECHO " $new_inherited_linker_flags" | $SED 's% -Wl,-pthread% -pthread%g'` ;;
++      esac
++
+       # move library search paths that coincide with paths to not yet
+       # installed libraries to the beginning of the library search list
+       new_libs=
diff --git a/config/make_manpage.pl b/config/make_manpage.pl
deleted file mode 100755
index 28c01887e80..00000000000
--- a/config/make_manpage.pl
+++ /dev/null
@@ -1,73 +0,0 @@
-#!/usr/bin/env perl
-#
-# Copyright (c) 2015      Research Organization for Information Science
-#                         and Technology (RIST). All rights reserved.
-# Copyright (c) 2015      Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2020      Intel, Inc.  All rights reserved.
-# $COPYRIGHT$
-#
-# Subroutine to generate a bunch of Fortran declarations and symbols
-#
-
-use strict;
-
-use Getopt::Long;
-
-my $package_name;
-my $package_version;
-my $ompi_date;
-my $opal_date;
-my $fortran = '1';
-my $f08 = '1';
-my $input;
-my $output;
-my $help_arg = 0;
-
-&Getopt::Long::Configure("bundling");
-my $ok = Getopt::Long::GetOptions("package-name=s" => \$package_name,
-                                  "package-version=s" => \$package_version,
-                                  "ompi-date=s" => \$ompi_date,
-                                  "opal-date=s" => \$opal_date,
-                                  "fortran!" => \$fortran,
-                                  "f08!" => \$f08,
-                                  "input=s" => \$input,
-                                  "output=s" => \$output);
-
-if ($help_arg || !$ok ||
-    !defined($input) ||
-    !defined($output) ||
-    !defined($package_name) ||
-    !defined($package_version) ||
-    !defined($ompi_date) ||
-    !defined($opal_date)) {
-    print "Usage: $0 --package-name=<package name> --package-version=<package version> --ompi-date=<ompi date> --opal-date=<opal date> --input=<input file> --output=<output file> [--nocxx] [ --nofortran] [--nof08]\n";
-    exit(1 - $ok);
-}
-
-open(FILE, $input) ||
-    die "Can't open $input";
-my $file;
-$file .= $_
-    while(<FILE>);
-close(FILE);
-
-$file =~ s/#PACKAGE_NAME#/$package_name/g;
-$file =~ s/#PACKAGE_VERSION#/$package_version/g;
-$file =~ s/#OMPI_DATE#/$ompi_date/g;
-$file =~ s/#OPAL_DATE#/$opal_date/g;
-
-if ($fortran == 0) {
-    $file =~ s/\n\.SH Fortran Syntax.+?\n\.SH/\n\.SH/s;
-}
-
-if ($f08 == 0) {
-    $file =~ s/\n\.SH Fortran 2008 Syntax.+?\n\.SH/\n\.SH/s;
-}
-
-open(FILE, ">$output") ||
-    die "Can't open $output";
-print FILE $file;
-close(FILE);
-
-exit(0);
-
diff --git a/config/md2nroff.pl b/config/md2nroff.pl
deleted file mode 100755
index a8ee21d449a..00000000000
--- a/config/md2nroff.pl
+++ /dev/null
@@ -1,130 +0,0 @@
-#!/usr/bin/env perl
-#
-# Copyright (c) 2020 Cisco Systems, Inc.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-use strict;
-
-use IPC::Open3;
-use File::Basename;
-use Getopt::Long;
-
-#--------------------------------------------------------------------------
-
-my $source_arg;
-my $dest_arg;
-my $pandoc_arg = "pandoc";
-my $help_arg;
-my $verbose_arg;
-
-my $ok = Getopt::Long::GetOptions("source=s" => \$source_arg,
-                                  "dest=s" => \$dest_arg,
-                                  "pandoc=s" => \$pandoc_arg,
-                                  "help" => \$help_arg,
-                                  "verbose" => \$verbose_arg);
-
-if (!$source_arg || !$dest_arg) {
-    print("Must specify --source and --dest\n");
-    $ok = 0;
-}
-
-if (!$ok || $help_arg) {
-    print "Invalid command line argument.\n\n"
-        if (!$ok);
-    print "Options:
-  --source FILE  Source Markdown filename
-  --dest FILE    Destination nroff file
-  --pandoc FILE  Location of pandoc executable
-  --help         This help list
-  --verbose      Be verbose when running\n";
-    exit($ok ? 0 : 1);
-}
-
-#--------------------------------------------------------------------------
-
-# Read in the source
-die "Error: $source_arg does not exist"
-    if (! -f $source_arg);
-
-my $source_content;
-open(FILE, $source_arg) ||
-    die "Can't open $source_arg";
-$source_content .= $_
-    while(<FILE>);
-close(FILE);
-
-#--------------------------------------------------------------------------
-
-# Figure out the section of man page
-die "Cannot figure out man page section from source filename"
-    if (!($source_arg =~ m/(\d+).md$/));
-my $man_section = $1;
-
-my $shortfile = basename($source_arg);
-$shortfile =~ s/\.$man_section\.md$//;
-
-#--------------------------------------------------------------------------
-
-my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime();
-my $today = sprintf("%04d-%02d-%02d", ($year+1900), $mon, $mday);
-
-# Run opal_get_version.sh to get the OMPI version.
-my $config_dir   = dirname($0);
-my $get_version  = "$config_dir/opal_get_version.sh";
-my $VERSION_file = "$config_dir/../VERSION";
-my $out          = `$get_version $VERSION_file --full`;
-chomp($out);
-
-# Pandoc does not handle markdown links in output nroff properly, so
-# just remove all links.  Specifically: some versions of Pandoc ignore
-# the links, but others handle it badly.
-$source_content =~ s/\[(.+)\]\((.+)\)/\1/g;
-
-# Add the pandoc header
-$source_content = "---
-section: $man_section
-title: $shortfile
-header: Open MPI
-footer: $today
----
-
-$source_content";
-
-#--------------------------------------------------------------------------
-
-print("*** Processing: $source_arg --> $dest_arg\n")
-    if ($verbose_arg);
-
-my $pid = open3(my $child_stdin, my $child_stdout, my $child_stderr,
-                "$pandoc_arg -s --from=markdown --to=man");
-print $child_stdin $source_content;
-close($child_stdin);
-my $pandoc_rendered;
-$pandoc_rendered .= $_
-    while(<$child_stdout>);
-close($child_stdout);
-close($child_stderr)
-    if ($child_stderr);
-waitpid($pid, 0);
-
-print("Writing new file $dest_arg\n")
-    if ($verbose_arg);
-
-# Make the target directory if it does not exist (needed for VPATH
-# builds)
-my $dest_dir = dirname($dest_arg);
-mkdir($dest_dir)
-    if (! -d $dest_dir);
-
-# Write the output file
-open(FILE, ">$dest_arg") ||
-    die "Can't open $dest_arg for writing";
-print FILE $pandoc_rendered;
-close(FILE);
-
-exit(0);
diff --git a/config/oac b/config/oac
new file mode 160000
index 00000000000..dfff67569fb
--- /dev/null
+++ b/config/oac
@@ -0,0 +1 @@
+Subproject commit dfff67569fb72dbf8d73a1dcf74d091dad93f71b
diff --git a/config/ompi_check_gpfs.m4 b/config/ompi_check_gpfs.m4
index 5f3b10e0e06..3ab33b36f16 100644
--- a/config/ompi_check_gpfs.m4
+++ b/config/ompi_check_gpfs.m4
@@ -24,46 +24,29 @@ dnl
 # LDFLAGS, LIBS} as needed and runs action-if-found if there is
 # support, otherwise executes action-if-not-found
 AC_DEFUN([OMPI_CHECK_GPFS],[
-
-    check_gpfs_CPPFLAGS=
-    check_gpfs_LDFLAGS=
-    check_gpfs_LIBS=
-
-    check_gpfs_save_LIBS="$LIBS"
-    check_gpfs_save_LDFLAGS="$LDFLAGS"
-    check_gpfs_save_CPPFLAGS="$CPPFLAGS"
-
-    check_gpfs_configuration="none"
-    ompi_check_gpfs_happy="yes"
+    OPAL_VAR_SCOPE_PUSH([ompi_check_gpfs_happy])
 
     # Get some configuration information
     AC_ARG_WITH([gpfs],
         [AS_HELP_STRING([--with-gpfs(=DIR)],
              [Build Gpfs support, optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries])])
-    OPAL_CHECK_WITHDIR([gpfs], [$with_gpfs], [include/gpfs.h])
 
-    AS_IF([test "$with_gpfs" = "no"],
-        [ompi_check_gpfs_happy="no"],
-        [AS_IF([test -z "$with_gpfs" || test "$with_gpfs" = "yes"],
-                [ompi_check_gpfs_dir="/usr"],
-                [ompi_check_gpfs_dir=$with_gpfs])
-            
-            if test -e "$ompi_check_gpfs_dir/lib64" ; then
-                ompi_check_gpfs_libdir="$ompi_check_gpfs_dir/lib64"
-            else
-                ompi_check_gpfs_libdir="$ompi_check_gpfs_dir/lib"
-            fi
-            
-            # Add correct -I and -L flags
-            OPAL_CHECK_PACKAGE([$1], [gpfs.h], [gpfs], [gpfs_lib_init], 
-                [], [$ompi_check_gpfs_dir], [$ompi_check_gpfs_libdir], 
-                [ompi_check_gpfs_happy="yes"],
-                [ompi_check_gpfs_happy="no"])
-  ])          
+    # Add correct -I and -L flags
+    OAC_CHECK_PACKAGE([gpfs],
+                      [$1],
+                      [gpfs.h],
+                      [gpfs],
+                      [gpfs_lib_init],
+                      [ompi_check_gpfs_happy="yes"],
+                      [ompi_check_gpfs_happy="no"])
+
+    OPAL_SUMMARY_ADD([OMPIO File Systems], [IBM Spectrum Scale/GPFS], [], [${$1_SUMMARY}])
 
     AS_IF([test "$ompi_check_gpfs_happy" = "yes"],
           [$2],
           [AS_IF([test ! -z "$with_gpfs" && test "$with_gpfs" != "no"],
                  [AC_MSG_ERROR([GPFS support requested but not found.  Aborting])])
            $3])
+
+    OPAL_VAR_SCOPE_POP
 ])
diff --git a/config/ompi_check_ime.m4 b/config/ompi_check_ime.m4
index 870bba8a143..8af8899c411 100644
--- a/config/ompi_check_ime.m4
+++ b/config/ompi_check_ime.m4
@@ -1,6 +1,8 @@
 dnl -*- shell-script -*-
 dnl
 dnl Copyright (c) 2018      DataDirect Networks. All rights reserved.
+dnl Copyright (c) 2021      Research Organization for Information Science
+dnl                         and Technology (RIST).  All rights reserved.
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
@@ -14,49 +16,27 @@ dnl
 # LDFLAGS, LIBS} as needed and runs action-if-found if there is
 # support, otherwise executes action-if-not-found
 AC_DEFUN([OMPI_CHECK_IME],[
-
-    check_ime_CPPFLAGS=
-    check_ime_LDFLAGS=
-    check_ime_LIBS=
-
-    check_ime_configuration="none"
-    ompi_check_ime_happy="yes"
-
+    OPAL_VAR_SCOPE_PUSH([ompi_check_ime_happy])
 
     # Get some configuration information
     AC_ARG_WITH([ime],
         [AS_HELP_STRING([--with-ime(=DIR)],
              [Build IME support, optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries])])
-    OPAL_CHECK_WITHDIR([ime], [$with_ime], [include/ime_native.h])
 
-    AS_IF([test "$with_ime" = "no"],
-        [ompi_check_ime_happy="no"],
-        [AS_IF([test -z "$with_ime"],
-                [ompi_check_ime_dir="/usr/local"],
-                [ompi_check_ime_dir=$with_ime])
-
-            if test -e "$ompi_check_ime_dir/lib64" ; then
-                ompi_check_ime_libdir="$ompi_check_ime_dir/lib64"
-            else
-                ompi_check_ime_libdir="$ompi_check_ime_dir/lib"
-            fi
-
-            # Add correct -I and -L flags
-            OPAL_CHECK_PACKAGE([$1], [ime_native.h], [im_client], [ime_client_native2_init], [],
-                [$ompi_check_ime_dir], [$ompi_check_ime_libdir], 
-                [ompi_check_ime_happy="yes"],
-                [OPAL_CHECK_PACKAGE([$1], [ime_native.h], [im_client], [ime_native_init], [],
-                    [$ompi_check_ime_dir], [$ompi_check_ime_libdir], 
-                    [ompi_check_ime_happy="yes"],
-                    [ompi_check_ime_happy="no"])
-                ])
-        ])
+    OAC_CHECK_PACKAGE([ime],
+                      [$1],
+                      [ime_native.h],
+                      [im_client],
+                      [ime_client_native2_init],
+                      [ompi_check_ime_happy="yes"],
+                      [ompi_check_ime_happy="no"])
 
     AS_IF([test "$ompi_check_ime_happy" = "yes"],
-        [$2],
-        [AS_IF([test ! -z "$with_ime" && test "$with_ime" != "no"],
-                [echo IME support not found])
-            $3])
-    
-    ])
+          [$2],
+          [AS_IF([test ! -z "$with_ime" && test "$with_ime" != "no"],
+                 [AC_MSG_ERROR([IME support requested but not found.  Aborting])])
+           $3])
+
+    OPAL_VAR_SCOPE_POP
+])
 
diff --git a/config/ompi_check_libhcoll.m4 b/config/ompi_check_libhcoll.m4
index 44774ab68cf..2151148ab14 100644
--- a/config/ompi_check_libhcoll.m4
+++ b/config/ompi_check_libhcoll.m4
@@ -17,49 +17,35 @@ dnl
 # LDFLAGS, LIBS} as needed and runs action-if-found if there is
 # support, otherwise executes action-if-not-found
 AC_DEFUN([OMPI_CHECK_HCOLL],[
-    OPAL_VAR_SCOPE_PUSH([ompi_check_hcoll_dir ompi_check_hcoll_libs ompi_check_hcoll_happy CPPFLAGS_save LDFLAGS_save LIBS_save])
+    OPAL_VAR_SCOPE_PUSH([ompi_check_hcoll_happy CPPFLAGS_save LDFLAGS_save LIBS_save])
 
     AC_ARG_WITH([hcoll],
         [AS_HELP_STRING([--with-hcoll(=DIR)],
              [Build hcoll (Mellanox Hierarchical Collectives) support, optionally adding
               DIR/include and DIR/lib or DIR/lib64 to the search path for headers and libraries])])
 
-    AS_IF([test "$with_hcoll" != "no"],
-          [ompi_check_hcoll_libs=hcoll
-           AS_IF([test ! -z "$with_hcoll" && test "$with_hcoll" != "yes"],
-                 [ompi_check_hcoll_dir=$with_hcoll])
+    OAC_CHECK_PACKAGE([hcoll],
+                      [$1],
+                      [hcoll/api/hcoll_api.h],
+                      [hcoll],
+                      [hcoll_get_version],
+                      [ompi_check_hcoll_happy="yes"],
+                      [ompi_check_hcoll_happy="no"])
 
-           CPPFLAGS_save=$CPPFLAGS
+    AS_IF([test "$ompi_check_hcoll_happy" = "yes"],
+          [CPPFLAGS_save=$CPPFLAGS
            LDFLAGS_save=$LDFLAGS
            LIBS_save=$LIBS
 
-           OPAL_LOG_MSG([$1_CPPFLAGS : $$1_CPPFLAGS], 1)
-           OPAL_LOG_MSG([$1_LDFLAGS  : $$1_LDFLAGS], 1)
-           OPAL_LOG_MSG([$1_LIBS     : $$1_LIBS], 1)
-
-           OPAL_CHECK_PACKAGE([$1],
-                              [hcoll/api/hcoll_api.h],
-                              [$ompi_check_hcoll_libs],
-                              [hcoll_get_version],
-                              [],
-                              [$ompi_check_hcoll_dir],
-                              [],
-                              [ompi_check_hcoll_happy="yes"],
-                              [ompi_check_hcoll_happy="no"])
+           CPPFLAGS="${$1_CPPFLAGS} ${CPPFLAGS}"
+           LDFLAGS="${$1_LDFLAGS} ${LDFLAGS}"
+           LIBS="${$1_LIBS} ${LIBS}"
 
-           AS_IF([test "$ompi_check_hcoll_happy" = "yes"],
-                 [
-                     CPPFLAGS=$coll_hcoll_CPPFLAGS
-                     LDFLAGS=$coll_hcoll_LDFLAGS
-                     LIBS=$coll_hcoll_LIBS
-                     AC_CHECK_FUNCS(hcoll_context_free, [], [])
-                 ],
-                 [])
+           AC_CHECK_FUNCS(hcoll_context_free, [], [])
 
            CPPFLAGS=$CPPFLAGS_save
            LDFLAGS=$LDFLAGS_save
-           LIBS=$LIBS_save],
-          [ompi_check_hcoll_happy=no])
+           LIBS=$LIBS_save])
 
     AS_IF([test "$ompi_check_hcoll_happy" = "yes" && test "$enable_progress_threads" = "yes"],
           [AC_MSG_WARN([hcoll driver does not currently support progress threads.  Disabling HCOLL.])
diff --git a/config/ompi_check_lustre.m4 b/config/ompi_check_lustre.m4
index 7421ba01c67..4401ad050cb 100644
--- a/config/ompi_check_lustre.m4
+++ b/config/ompi_check_lustre.m4
@@ -12,7 +12,7 @@ dnl Copyright (c) 2004-2006 The Regents of the University of California.
 dnl                         All rights reserved.
 dnl Copyright (c) 2009-2017 Cisco Systems, Inc.  All rights reserved
 dnl Copyright (c) 2008-2018 University of Houston. All rights reserved.
-dnl Copyright (c) 2015-2018 Research Organization for Information Science
+dnl Copyright (c) 2015-2021 Research Organization for Information Science
 dnl                         and Technology (RIST).  All rights reserved.
 dnl Copyright (c) 2020      Triad National Security, LLC. All rights
 dnl                         reserved.
@@ -29,46 +29,26 @@ dnl
 # LDFLAGS, LIBS} as needed and runs action-if-found if there is
 # support, otherwise executes action-if-not-found
 AC_DEFUN([OMPI_CHECK_LUSTRE],[
-
-    check_lustre_CPPFLAGS=
-    check_lustre_LDFLAGS=
-    check_lustre_LIBS=
-
-    check_lustre_save_LIBS="$LIBS"
-    check_lustre_save_LDFLAGS="$LDFLAGS"
-    check_lustre_save_CPPFLAGS="$CPPFLAGS"
-
-    ompi_check_lustre_happy="yes"
+    OPAL_VAR_SCOPE_PUSH([ompi_check_lustre_happy])
 
     # Get some configuration information
     AC_ARG_WITH([lustre],
         [AS_HELP_STRING([--with-lustre(=DIR)],
              [Build Lustre support, optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries])])
-    OPAL_CHECK_WITHDIR([lustre], [$with_lustre], [include/lustre/lustreapi.h])
 
-    AS_IF([test "$with_lustre" = "no"],
-          [ompi_check_lustre_happy=no])
+     OAC_CHECK_PACKAGE([lustre],
+                       [$1],
+                       [lustre/lustreapi.h],
+                       [lustreapi],
+                       [llapi_file_create],
+                       [ompi_check_lustre_happy="yes"],
+                       [ompi_check_lustre_happy="no"])
 
-    AS_IF([test "$ompi_check_lustre_happy" != "no" ],
-          [AC_MSG_CHECKING([looking for lustre libraries and header files in])
-           AS_IF([test "$with_lustre" != "yes"],
-                 [ompi_check_lustre_dir=$with_lustre
-                  AC_MSG_RESULT([($ompi_check_lustre_dir)])],
-                 [AC_MSG_RESULT([(default search paths)])])
-           AS_IF([test -n "$with_lustre_libdir" && \
-                         test "$with_lustre_libdir" != "yes"],
-                 [ompi_check_lustre_libdir=$with_lustre_libdir])
-          ])
-
-    AS_IF([test "$ompi_check_lustre_happy" != "no" ],
-          [OPAL_CHECK_PACKAGE([$1], [lustre/lustreapi.h], [lustreapi], [llapi_file_create], 
-                [], [$ompi_check_lustre_dir], [$ompi_check_lustre_libdir], 
-                [ompi_check_lustre_happy="yes"],
-                [ompi_check_lustre_happy="no"])])
-            
     AS_IF([test "$ompi_check_lustre_happy" = "yes"],
-          [AC_MSG_CHECKING([for required lustre data structures])
-           cat > conftest.c <<EOF
+          [AC_CACHE_CHECK([for required lustre data structures],
+              [ompi_check_lustre_cv_ost_data_v1],
+              [ompi_check_lustre_cv_ost_data_v1=yes
+               cat > conftest.c <<EOF
 #include "lustre/lustreapi.h"
 void alloc_lum()
 {
@@ -79,20 +59,21 @@ void alloc_lum()
     LOV_MAX_STRIPE_COUNT * sizeof(struct lov_user_ost_data_v1);
 }
 EOF
+               # Try the compile
+               OPAL_LOG_COMMAND([$CC $CFLAGS ${$1_CPPFLAGS} -c conftest.c],
+                                [], [ompi_check_lustre_cv_ost_data_v1=no])
+               rm -f conftest.c conftest.o])
+           AS_IF([test "${ompi_check_lustre_cv_ost_data_v1}" = "no"],
+                 [ompi_check_lustre_happy=no
+                  fs_lustre_SUMMARY="no"])])
 
-           # Try the compile
-           OPAL_LOG_COMMAND(
-               [$CC $CFLAGS -I$ompi_check_lustre_dir/include -c conftest.c],
-               [ompi_check_lustre_struct_happy="yes"],
-               [ompi_check_lustre_struct_happy="no"
-                ompi_check_lustre_happy="no"]
-            )
-            rm -f conftest.c conftest.o
-            AC_MSG_RESULT([$ompi_check_lustre_struct_happy])])
+    OPAL_SUMMARY_ADD([OMPIO File Systems], [Lustre], [], [${$1_SUMMARY}])
 
     AS_IF([test "$ompi_check_lustre_happy" = "yes"],
           [$2],
           [AS_IF([test -n "$with_lustre" && test "$with_lustre" != "no"],
                  [AC_MSG_ERROR([Lustre support requested but not found.  Aborting])])
-                  $3])
+           $3])
+
+    OPAL_VAR_SCOPE_POP
 ])
diff --git a/config/ompi_check_plfs.m4 b/config/ompi_check_plfs.m4
deleted file mode 100644
index 9601a2c220b..00000000000
--- a/config/ompi_check_plfs.m4
+++ /dev/null
@@ -1,114 +0,0 @@
-dnl -*- shell-script -*-
-dnl
-dnl Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-dnl                         University Research and Technology
-dnl                         Corporation.  All rights reserved.
-dnl Copyright (c) 2004-2005 The University of Tennessee and The University
-dnl                         of Tennessee Research Foundation.  All rights
-dnl                         reserved.
-dnl Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-dnl                         University of Stuttgart.  All rights reserved.
-dnl Copyright (c) 2004-2006 The Regents of the University of California.
-dnl                         All rights reserved.
-dnl Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
-dnl Copyright (c) 2008-2012 University of Houston. All rights reserved.
-dnl Copyright (c) 2015      Research Organization for Information Science
-dnl                         and Technology (RIST). All rights reserved.
-dnl $COPYRIGHT$
-dnl
-dnl Additional copyrights may follow
-dnl
-dnl $HEADER$
-dnl
-
-# OMPI_CHECK_PLFS(prefix, [action-if-found], [action-if-not-found])
-# --------------------------------------------------------
-# check if PLFS support can be found.  sets prefix_{CPPFLAGS,
-# LDFLAGS, LIBS} as needed and runs action-if-found if there is
-# support, otherwise executes action-if-not-found
-AC_DEFUN([OMPI_CHECK_PLFS],[
-
-    check_plfs_CPPFLAGS=
-    check_plfs_LDFLAGS=
-    check_plfs_LIBS=
-
-    check_plfs_save_LIBS="$LIBS"
-    check_plfs_save_LDFLAGS="$LDFLAGS"
-    check_plfs_save_CPPFLAGS="$CPPFLAGS"
-
-    check_plfs_configuration="none"
-    ompi_check_plfs_happy="yes"
-
-
-    # Get some configuration information
-    AC_ARG_WITH([plfs],
-        [AS_HELP_STRING([--with-plfs(=DIR)],
-             [Build Plfs support, optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries])])
-    OPAL_CHECK_WITHDIR([plfs], [$with_plfs], [include/plfs.h])
-
-    AC_ARG_WITH([plfs-libs],
-        [AS_HELP_STRING([--with-plfs-libs=LIBS],
-                       [Libraries to link with for plfs])])
-
-    temp_with_plfs="$with_plfs"
-    AS_IF([test -z "$with_plfs"],
-          [with_plfs="/usr/local"])
-
-    temp_with_plfs_libs="$with_plfs_libs"
-    AS_IF([test -z "$with_plfs_libs"],
-	[with_plfs_libs="plfs pthread"])
-
-    # Add correct -I and -L flags
-    AS_IF([test -d "$with_plfs/include"],
-        [check_plfs_CPPFLAGS="-I$with_plfs/include"
-            $1_CPPFLAGS="$check_plfs_CPPFLAGS"
-            CPPFLAGS="$CPPFLAGS $check_plfs_CPPFLAGS"],
-	[ompi_check_plfs_happy="no"])
-
-    AS_IF([test "$ompi_check_plfs_happy" = "yes"],
-	[AS_IF([test -d "$with_plfs/lib"],
-		[check_plfs_LDFLAGS="-L$with_plfs/lib"
-		    $1_LDFLAGS="$check_plfs_LDFLAGS"
-		    LDFLAGS="$LDFLAGS $check_plfs_LDFLAGS"],
-		[ompi_check_plfs_happy="no"])
-    ],[])
-
-    # Try to find all the plfs libraries
-    AS_IF([test "$ompi_check_plfs_happy" = "yes"],
-	[ AS_IF([test -n "$with_plfs_libs"]
-		[for lib in $with_plfs_libs ; do
-		    check_plfs_LIBS="$check_plfs_LIBS -l$lib"
-		    done])
-
-	    $1_LIBS="$check_plfs_LIBS"
-	    LIBS="$LIBS $check_plfs_LIBS"
-
-            # check for plfs
-	    AC_CHECK_HEADERS([plfs.h],
-		[AC_MSG_CHECKING([if possible to link PLFS])
-		    AC_LINK_IFELSE(
-			[AC_LANG_PROGRAM(
-				[[#include <plfs.h>]],
-				[[is_plfs_path(NULL);]])],
-			[AC_MSG_RESULT([yes])
-			    ompi_check_plfs_happy="yes"],
-			[AC_MSG_RESULT([no])
-			    ompi_check_plfs_happy="no"])],
-		[ompi_check_plfs_happy="no"])
-    ])
-
-
-    LDFLAGS="$check_plfs_save_LDFLAGS"
-    CPPFLAGS="$check_plfs_save_CPPFLAGS"
-    LIBS="$check_plfs_save_LIBS"
-    AS_IF([test "$ompi_check_plfs_happy" = "yes"],
-          [$2],
-          [AS_IF([test ! -z "$with_plfs" && test "$with_plfs" != "no"],
-                  [echo PLFS support not found])
-              $3])
-
-    with_plfs="$temp_with_plfs"
-    with_plfs_libs="$temp_with_plfs_libs"
-
-])
-
diff --git a/config/ompi_check_psm2.m4 b/config/ompi_check_psm2.m4
index 00ccffd98fa..2f5322d2b9e 100644
--- a/config/ompi_check_psm2.m4
+++ b/config/ompi_check_psm2.m4
@@ -11,12 +11,15 @@
 # Copyright (c) 2004-2006 The Regents of the University of California.
 #                         All rights reserved.
 # Copyright (c) 2006      QLogic Corp. All rights reserved.
-# Copyright (c) 2009-2016 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2009-2021 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2016      Intel Corporation. All rights reserved.
 # Copyright (c) 2015      Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
 # Copyright (c) 2016      Los Alamos National Security, LLC. All rights
 #                         reserved.
+#  Copyright (c) 2021     Triad National Security, LLC. All rights
+#                         reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -30,68 +33,58 @@
 # LDFLAGS, LIBS} as needed and runs action-if-found if there is
 # support, otherwise executes action-if-not-found
 AC_DEFUN([OMPI_CHECK_PSM2],[
-    if test -z "$ompi_check_psm2_happy" ; then
-	AC_ARG_WITH([psm2],
-		    [AS_HELP_STRING([--with-psm2(=DIR)],
-				    [Build PSM2 (Intel PSM2) support, optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries])])
-	OPAL_CHECK_WITHDIR([psm2], [$with_psm2], [include/psm2.h])
-	AC_ARG_WITH([psm2-libdir],
-		    [AS_HELP_STRING([--with-psm2-libdir=DIR],
-				    [Search for PSM (Intel PSM2) libraries in DIR])])
-	OPAL_CHECK_WITHDIR([psm2-libdir], [$with_psm2_libdir], [libpsm2.*])
+    OPAL_VAR_SCOPE_PUSH([opal_psm2_CPPFLAGS_save opal_psm2_LDFLAGS_save opal_psm2_LIBS_save])
 
-	ompi_check_psm2_$1_save_CPPFLAGS="$CPPFLAGS"
-	ompi_check_psm2_$1_save_LDFLAGS="$LDFLAGS"
-	ompi_check_psm2_$1_save_LIBS="$LIBS"
+    AC_ARG_WITH([psm2],
+                [AS_HELP_STRING([--with-psm2(=DIR)],
+                    [Build PSM2 (Intel PSM2) support, optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries])])
+    AC_ARG_WITH([psm2-libdir],
+                [AS_HELP_STRING([--with-psm2-libdir=DIR],
+                    [Search for PSM (Intel PSM2) libraries in DIR])])
 
-	AS_IF([test "$with_psm2" != "no"],
-              [AS_IF([test ! -z "$with_psm2" && test "$with_psm2" != "yes"],
-                     [ompi_check_psm2_dir="$with_psm2"])
-               AS_IF([test ! -z "$with_psm2_libdir" && test "$with_psm2_libdir" != "yes"],
-                     [ompi_check_psm2_libdir="$with_psm2_libdir"])
+    AC_ARG_ENABLE([psm2-version-check],
+                  [AS_HELP_STRING([--disable-psm2-version-check],
+                      [Disable PSM2 version checking.  Not recommended to disable. (default: enabled)])])
 
-               OPAL_CHECK_PACKAGE([ompi_check_psm2],
-				  [psm2.h],
-				  [psm2],
-				  [psm2_mq_irecv2],
-				  [],
-				  [$ompi_check_psm2_dir],
-				  [$ompi_check_psm2_libdir],
-				  [ompi_check_psm2_happy="yes"],
-				  [ompi_check_psm2_happy="no"])],
-              [ompi_check_psm2_happy="no"])
+    OAC_CHECK_PACKAGE([psm2],
+                      [$1],
+                      [psm2.h],
+                      [psm2],
+                      [psm2_mq_irecv2],
+                      [ompi_check_psm2_happy="yes"],
+                      [ompi_check_psm2_happy="no"])
 
-	CPPFLAGS="$ompi_check_psm2_$1_save_CPPFLAGS"
-	LDFLAGS="$ompi_check_psm2_$1_save_LDFLAGS"
-	LIBS="$ompi_check_psm2_$1_save_LIBS"
+    AS_IF([test "$ompi_check_psm2_happy" = "yes" && test "$enable_progress_threads" = "yes"],
+          [AC_MSG_WARN([PSM2 driver does not currently support progress threads.  Disabling MTL.])
+           $1_SUMMARY="no (progress threads not supported)"
+           ompi_check_psm2_happy="no"])
 
-	AS_IF([test "$ompi_check_psm2_happy" = "yes" && test "$enable_progress_threads" = "yes"],
-              [AC_MSG_WARN([PSM2 driver does not currently support progress threads.  Disabling MTL.])
-               ompi_check_psm2_happy="no"])
+    opal_psm2_CPPFLAGS_save="$CPPFLAGS"
+    CPPFLAGS="${$1_CPPFLAGS} ${CPPFLAGS}"
 
-    	AS_IF([test "$ompi_check_psm2_happy" = "yes"],
-              [AC_CHECK_HEADERS(
-               glob.h,
-               [],
+    AS_IF([test "$ompi_check_psm2_happy" = "yes"],
+          [AC_CHECK_HEADERS([glob.h], [],
                [AC_MSG_WARN([glob.h not found.  Can not build component.])
-               ompi_check_psm2_happy="no"])])
+                $1_SUMMARY="no (glob.h not found))"
+                ompi_check_psm2_happy="no"])])
+
+    AS_IF([test "$ompi_check_psm2_happy" = "yes"],
+          [AC_CHECK_DECL([PSM2_LIB_REFCOUNT_CAP],
+                         [AC_DEFINE([HAVE_PSM2_LIB_REFCOUNT_CAP], [1],
+                                    [have PSM2_LIB_REFCOUNT_CAP in psm2.h])],
+                         [AS_IF([test "x$enable_psm2_version_check" != "xno"],
+                                [ompi_check_psm2_happy="no"
+                                 $1_SUMMARY="no (version too old)"
+                                 AC_MSG_WARN([PSM2 needs to be version 11.2.173 or later. Disabling MTL.])]
+                         )],
+                          [#include <psm2.h>])])
 
-        AS_IF([test "$ompi_check_psm2_happy" = "yes"],
-              [AC_CHECK_DECL([PSM2_LIB_REFCOUNT_CAP],
-                        [],
-                        [AC_MSG_WARN([PSM2 needs to be version 11.2.173 or later. Disabling MTL.])
-                         ompi_check_psm2_happy="no"],
-                        [#include <psm2.h>])
-              ])
+    CPPFLAGS="${opal_psm2_CPPFLAGS_save}"
 
-        OPAL_SUMMARY_ADD([[Transports]],[[Intel Omnipath (PSM2)]],[$1],[$ompi_check_psm2_happy])
-    fi
+    OPAL_SUMMARY_ADD([Transports], [Intel Omnipath (PSM2)], [], [${$1_SUMMARY}])
 
     AS_IF([test "$ompi_check_psm2_happy" = "yes"],
-          [$1_LDFLAGS="[$]$1_LDFLAGS $ompi_check_psm2_LDFLAGS"
-	   $1_CPPFLAGS="[$]$1_CPPFLAGS $ompi_check_psm2_CPPFLAGS"
-	   $1_LIBS="[$]$1_LIBS $ompi_check_psm2_LIBS"
-	   $2],
+          [$2],
           [AS_IF([test ! -z "$with_psm2" && test "$with_psm2" != "no"],
                  [AC_MSG_ERROR([PSM2 support requested but not found.  Aborting])])
            $3])
diff --git a/config/ompi_check_pvfs2.m4 b/config/ompi_check_pvfs2.m4
deleted file mode 100644
index 149d973f2be..00000000000
--- a/config/ompi_check_pvfs2.m4
+++ /dev/null
@@ -1,71 +0,0 @@
-dnl -*- shell-script -*-
-dnl
-dnl Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-dnl                         University Research and Technology
-dnl                         Corporation.  All rights reserved.
-dnl Copyright (c) 2004-2005 The University of Tennessee and The University
-dnl                         of Tennessee Research Foundation.  All rights
-dnl                         reserved.
-dnl Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-dnl                         University of Stuttgart.  All rights reserved.
-dnl Copyright (c) 2004-2006 The Regents of the University of California.
-dnl                         All rights reserved.
-dnl Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
-dnl Copyright (c) 2008-2018 University of Houston. All rights reserved.
-dnl Copyright (c) 2015      Research Organization for Information Science
-dnl                         and Technology (RIST). All rights reserved.
-dnl $COPYRIGHT$
-dnl
-dnl Additional copyrights may follow
-dnl
-dnl $HEADER$
-dnl
-
-# OMPI_CHECK_PVFS2(prefix, [action-if-found], [action-if-not-found])
-# --------------------------------------------------------
-# check if PVFS2 support can be found.  sets prefix_{CPPFLAGS,
-# LDFLAGS, LIBS} as needed and runs action-if-found if there is
-# support, otherwise executes action-if-not-found
-AC_DEFUN([OMPI_CHECK_PVFS2],[
-
-    check_pvfs2_CPPFLAGS=
-    check_pvfs2_LDFLAGS=
-    check_pvfs2_LIBS=
-
-    check_pvfs2_configuration="none"
-    ompi_check_pvfs2_happy="yes"
-
-
-    # Get some configuration information
-    AC_ARG_WITH([pvfs2],
-        [AS_HELP_STRING([--with-pvfs2(=DIR)],
-             [Build Pvfs2 support, optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries])])
-    OPAL_CHECK_WITHDIR([pvfs2], [$with_pvfs2], [include/pvfs2.h])
-
-    AS_IF([test "$with_pvfs2" = "no"],
-        [ompi_check_pvfs2_happy="no"],
-        [AS_IF([test -z "$with_pvfs2"],
-                [ompi_check_pvfs2_dir="/usr/local"],
-                [ompi_check_pvfs2_dir=$with_pvfs2])
-
-            if test -e "$ompi_check_pvfs2_dir/lib64" ; then
-                ompi_check_pvfs2_libdir="$ompi_check_pvfs2_dir/lib64"
-            else
-                ompi_check_pvfs2_libdir="$ompi_check_pvfs2_dir/lib"
-            fi
-
-            # Add correct -I and -L flags
-            OPAL_CHECK_PACKAGE([$1], [pvfs2.h], [pvfs2], [PVFS_util_resolve], [],
-                [$ompi_check_pvfs2_dir], [$ompi_check_pvfs2_libdir], 
-                [ompi_check_pvfs2_happy="yes"],
-                [ompi_check_pvfs2_happy="no"])
-            ])
-
-    AS_IF([test "$ompi_check_pvfs2_happy" = "yes"],
-        [$2],
-        [AS_IF([test ! -z "$with_pvfs2" && test "$with_pvfs2" != "no"],
-                [echo PVFS2 support not found])
-            $3])
-    
-    ])
-
diff --git a/config/ompi_check_ucc.m4 b/config/ompi_check_ucc.m4
new file mode 100644
index 00000000000..4fb05f658d2
--- /dev/null
+++ b/config/ompi_check_ucc.m4
@@ -0,0 +1,68 @@
+dnl -*- shell-script -*-
+dnl
+dnl Copyright (c) 2021      Mellanox Technologies. All rights reserved.
+dnl Copyright (c) 2013-2021 Cisco Systems, Inc.  All rights reserved.
+dnl Copyright (c) 2015      Research Organization for Information Science
+dnl                         and Technology (RIST). All rights reserved.
+dnl $COPYRIGHT$
+dnl
+dnl Additional copyrights may follow
+dnl
+dnl $HEADER$
+dnl
+
+# OMPI_CHECK_UCC(prefix, [action-if-found], [action-if-not-found])
+# --------------------------------------------------------
+# check if ucc support can be found.  sets prefix_{CPPFLAGS,
+# LDFLAGS, LIBS} as needed and runs action-if-found if there is
+# support, otherwise executes action-if-not-found
+AC_DEFUN([OMPI_CHECK_UCC],[
+    OPAL_VAR_SCOPE_PUSH([ompi_check_ucc_happy CPPFLAGS_save LDFLAGS_save LIBS_save])
+
+    AC_ARG_WITH([ucc],
+                [AS_HELP_STRING([--with-ucc(=DIR)],
+                                [Build UCC (Unified Collective Communication)])])
+
+    OAC_CHECK_PACKAGE([ucc],
+                      [$1],
+                      [ucc/api/ucc.h],
+                      [ucc],
+                      [ucc_init_version],
+                      [ompi_check_ucc_happy="yes"],
+                      [ompi_check_ucc_happy="no"])
+
+    AS_IF([test "$ompi_check_ucc_happy" = "yes"],
+          [CPPFLAGS_save=$CPPFLAGS
+           LDFLAGS_save=$LDFLAGS
+           LIBS_save=$LIBS
+
+           CPPFLAGS="${$1_CPPFLAGS} ${CPPFLAGS}"
+           LDFLAGS="${$1_LDFLAGS} ${LDFLAGS}"
+           LIBS="${$1_LIBS} ${LIBS}"
+           AC_CHECK_FUNCS(ucc_comm_free, [], [])
+
+           AC_MSG_CHECKING([if UCC supports float128 and float32(64,128)_complex datatypes])
+           AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <ucc/api/ucc.h>]],
+                                         [[ucc_datatype_t dt = UCC_DT_FLOAT32_COMPLEX;]])],
+                       [flag=1
+                       AC_MSG_RESULT([yes])],
+                       [flag=0
+                       AC_MSG_RESULT([no])])
+           AC_DEFINE_UNQUOTED(UCC_HAVE_COMPLEX_AND_FLOAT128_DT, $flag, [Check if float128 and float32(64,128)_complex dt are available in ucc.])
+
+           CPPFLAGS=$CPPFLAGS_save
+           LDFLAGS=$LDFLAGS_save
+           LIBS=$LIBS_save])
+
+    AS_IF([test "$ompi_check_ucc_happy" = "yes" && test "$enable_progress_threads" = "yes"],
+          [AC_MSG_WARN([ucc driver does not currently support progress threads.  Disabling UCC.])
+           ompi_check_ucc_happy="no"])
+
+    AS_IF([test "$ompi_check_ucc_happy" = "yes"],
+          [$2],
+          [AS_IF([test ! -z "$with_ucc" && test "$with_ucc" != "no"],
+                 [AC_MSG_ERROR([UCC support requested but not found.  Aborting])])
+           $3])
+
+    OPAL_VAR_SCOPE_POP
+])
diff --git a/config/ompi_check_ucx.m4 b/config/ompi_check_ucx.m4
index 151bf8cf58c..d473454a8ce 100644
--- a/config/ompi_check_ucx.m4
+++ b/config/ompi_check_ucx.m4
@@ -7,6 +7,7 @@
 # Copyright (c) 2016      Los Alamos National Security, LLC. All rights
 #                         reserved.
 # Copyright (c) 2016 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -20,151 +21,117 @@
 # LDFLAGS, LIBS} as needed and runs action-if-found if there is
 # support, otherwise executes action-if-not-found
 AC_DEFUN([OMPI_CHECK_UCX],[
-    OPAL_VAR_SCOPE_PUSH([ompi_check_ucx_dir])
+    OPAL_VAR_SCOPE_PUSH([ompi_check_ucx_happy ompi_check_ucx_CPPFLAGS_save ompi_check_ucx_LDFLAGS_save ompi_check_ucx_LIBS_save])
 
-    AS_IF([test -z "$ompi_check_ucx_happy"],
-          [AC_ARG_WITH([ucx],
-		       [AS_HELP_STRING([--with-ucx(=DIR)],
-				       [Build with Unified Communication X library support])])
-	   OPAL_CHECK_WITHDIR([ucx], [$with_ucx], [include/ucp/api/ucp.h])
-	   AC_ARG_WITH([ucx-libdir],
-		       [AS_HELP_STRING([--with-ucx-libdir=DIR],
-				       [Search for Unified Communication X libraries in DIR])])
-	   OPAL_CHECK_WITHDIR([ucx-libdir], [$with_ucx_libdir], [libucp.*])
+    m4_ifblank([$1], [m4_fatal([First argument to OMPI_CHECK_UCX cannot be blank])])
 
-	   AS_IF([test "$with_ucx" != "no"],
-                 [AS_IF([test -n "$with_ucx" && test "$with_ucx" != "yes"],
-                        [ompi_check_ucx_dir="$with_ucx"],
-                        [PKG_CHECK_MODULES_STATIC([ucx],[ucx],
-                                             [ompi_check_ucx_dir=`$PKG_CONFIG --variable=prefix ucx`
-                                              AS_IF([test "$ompi_check_ucx_dir" = "/usr"],
-                                                    [ompi_check_ucx_dir=])],
-                                             [true])])
-                  ompi_check_ucx_happy="no"
-                  AS_IF([test -z "$ompi_check_ucx_dir"],
-                        [OPAL_CHECK_PACKAGE([ompi_check_ucx],
-                                   [ucp/api/ucp.h],
-                                   [ucp],
-                                   [ucp_cleanup],
-                                   [-luct -lucm -lucs],
-                                   [],
-                                   [],
-                                   [ompi_check_ucx_happy="yes"],
-                                   [ompi_check_ucx_happy="no"])
-                         AS_IF([test "$ompi_check_ucx_happy" = yes],
-                               [AC_MSG_CHECKING(for UCX version compatibility)
-                                AC_REQUIRE_CPP
-                                AC_COMPILE_IFELSE(
-                                    [AC_LANG_PROGRAM([[#include <uct/api/version.h>]],[[]])],
-                                    [ompi_check_ucx_happy="yes"],
-                                    [ompi_check_ucx_happy="no"])
+    AC_ARG_WITH([ucx],
+                [AS_HELP_STRING([--with-ucx(=DIR)],
+                                [Build with Unified Communication X library support])])
+    AC_ARG_WITH([ucx-libdir],
+                [AS_HELP_STRING([--with-ucx-libdir=DIR],
+                                [Search for Unified Communication X libraries in DIR])])
 
-                                AC_MSG_RESULT([$ompi_check_ucx_happy])])
-                         AS_IF([test "$ompi_check_ucx_happy" = "no"],
-                               [ompi_check_ucx_dir=/opt/ucx])])
-                  AS_IF([test "$ompi_check_ucx_happy" != yes],
-                        [AS_IF([test -n "$with_ucx_libdir"],
-                               [ompi_check_ucx_libdir="$with_ucx_libdir"],
-                               [files=`ls $ompi_check_ucx_dir/lib64/libucp.* 2> /dev/null | wc -l`
-                                AS_IF([test "$files" -gt 0],
-                                      [ompi_check_ucx_libdir=$ompi_check_ucx_dir/lib64],
-                                      [ompi_check_ucx_libdir=$ompi_check_ucx_dir/lib])])
+    OAC_CHECK_PACKAGE([ucx],
+                      [$1],
+                      [ucp/api/ucp.h],
+                      [ucp -luct -lucm -lucs],
+                      [ucp_cleanup],
+                      [ompi_check_ucx_happy="yes"],
+                      [ompi_check_ucx_happy="no"])
 
-                         ompi_check_ucx_$1_save_CPPFLAGS="$CPPFLAGS"
-                         ompi_check_ucx_$1_save_LDFLAGS="$LDFLAGS"
-                         ompi_check_ucx_$1_save_LIBS="$LIBS"
+    ompi_check_ucx_CPPFLAGS_save=${CPPFLAGS}
+    ompi_check_ucx_LDFLAGS_save=${LDFLAGS}
+    ompi_check_ucx_LIBS_save=${LIBS}
 
-                         OPAL_CHECK_PACKAGE([ompi_check_ucx],
-                                            [ucp/api/ucp.h],
-                                            [ucp],
-                                            [ucp_cleanup],
-                                            [-luct -lucm -lucs],
-                                            [$ompi_check_ucx_dir],
-                                            [$ompi_check_ucx_libdir],
-                                            [ompi_check_ucx_happy="yes"],
-                                            [ompi_check_ucx_happy="no"])
+    OPAL_FLAGS_APPEND_UNIQ([CPPFLAGS], [${$1_CPPFLAGS}])
+    OPAL_FLAGS_APPEND_UNIQ([LDFLAGS], [${$1_LDFLAGS}])
+    OPAL_FLAGS_APPEND_MOVE([LIBS], [${$1_LIBS}])
 
-                         CPPFLAGS="$ompi_check_ucx_$1_save_CPPFLAGS"
-                         LDFLAGS="$ompi_check_ucx_$1_save_LDFLAGS"
-                         LIBS="$ompi_check_ucx_$1_save_LIBS"
+    AS_IF([test "$ompi_check_ucx_happy" = yes],
+          [AC_CACHE_CHECK([for UCX version header],
+               [ompi_check_ucx_cv_have_version_header],
+               [AC_REQUIRE_CPP
+                AC_COMPILE_IFELSE(
+                   [AC_LANG_PROGRAM([[#include <uct/api/version.h>]],[[]])],
+                   [ompi_check_ucx_cv_have_version_header="yes"],
+                   [ompi_check_ucx_cv_have_version_header="no"])])
+           AS_IF([test "${ompi_check_ucx_cv_have_version_header}" != "yes"],
+                 [ompi_check_ucx_happy=no])])
 
-                         AS_IF([test "$ompi_check_ucx_happy" = yes],
-                               [AC_MSG_CHECKING(for UCX version compatibility)
-                                AC_REQUIRE_CPP
-                                old_CPPFLAGS="$CPPFLAGS"
-                                CPPFLAGS="$CPPFLAGS -I$ompi_check_ucx_dir/include"
-                                AC_COMPILE_IFELSE(
-                                    [AC_LANG_PROGRAM([[#include <uct/api/version.h>]],[[]])],
-                                    [ompi_check_ucx_happy="yes"],
-                                    [ompi_check_ucx_happy="no"])
+    AS_IF([test "$ompi_check_ucx_happy" = yes],
+          [# Turn off UCX version v1.8 due to issue #8321
+           AC_CACHE_CHECK([UCX version > 1.9.x],
+               [ompi_check_ucx_cv_have_version_gt_1_9],
+               [AC_PREPROC_IFELSE([AC_LANG_PROGRAM([[
+#include <ucp/api/ucp_version.h>
+                                 ]], [[
+#if (UCP_API_MAJOR < 1) || ((UCP_API_MAJOR == 1) && (UCP_API_MINOR < 9))
+#error "Version too low"
+#endif
+                                 ]])],
+                             [ompi_check_ucx_cv_have_version_gt_1_9=yes],
+                             [ompi_check_ucx_cv_have_version_gt_1_9=no
+                              ompi_check_ucx_happy=no])])])
 
-                                AC_MSG_RESULT([$ompi_check_ucx_happy])
-                                CPPFLAGS=$old_CPPFLAGS])])
+    AS_IF([test "$ompi_check_ucx_happy" = yes],
+          [AC_CHECK_DECLS([ucp_tag_send_nbr],
+                          [AC_DEFINE([HAVE_UCP_TAG_SEND_NBR],[1],
+                                     [have ucp_tag_send_nbr()])], [],
+                          [#include <ucp/api/ucp.h>])
+           AC_CHECK_DECLS([ucp_ep_flush_nb, ucp_worker_flush_nb,
+                           ucp_request_check_status, ucp_put_nb, ucp_get_nb,
+                           ucp_put_nbx, ucp_get_nbx, ucp_atomic_op_nbx,
+                           ucp_ep_flush_nbx],
+                          [], [],
+                          [#include <ucp/api/ucp.h>])
+           AC_CHECK_DECLS([ucm_test_events,
+                           ucm_test_external_events],
+                          [], [],
+                          [#include <ucm/api/ucm.h>])
+           AC_CHECK_DECLS([UCP_ATOMIC_POST_OP_AND,
+                           UCP_ATOMIC_POST_OP_OR,
+                           UCP_ATOMIC_POST_OP_XOR,
+                           UCP_ATOMIC_FETCH_OP_FAND,
+                           UCP_ATOMIC_FETCH_OP_FOR,
+                           UCP_ATOMIC_FETCH_OP_FXOR,
+                           UCP_PARAM_FIELD_ESTIMATED_NUM_PPN,
+                           UCP_WORKER_FLAG_IGNORE_REQUEST_LEAK,
+                           UCP_OP_ATTR_FLAG_MULTI_SEND,
+                           UCS_MEMORY_TYPE_RDMA,
+                           UCP_MEM_MAP_SYMMETRIC_RKEY],
+                          [], [],
+                          [#include <ucp/api/ucp.h>])
+           AC_CHECK_DECLS([UCP_WORKER_ATTR_FIELD_ADDRESS_FLAGS],
+                          [AC_DEFINE([HAVE_UCP_WORKER_ADDRESS_FLAGS], [1],
+                                     [have worker address attribute])], [],
+                          [#include <ucp/api/ucp.h>])
+           AC_CHECK_DECLS([UCP_ATTR_FIELD_MEMORY_TYPES],
+                          [AC_DEFINE([HAVE_UCP_ATTR_MEMORY_TYPES], [1],
+                                     [have memory types attribute])], [],
+                          [#include <ucp/api/ucp.h>])
+           AC_CHECK_DECLS([UCP_EP_ATTR_FIELD_TRANSPORTS],
+                          [], [],
+                          [#include <ucp/api/ucp.h>])
+           AC_CHECK_DECLS([ucp_tag_send_nbx,
+                           ucp_tag_send_sync_nbx,
+                           ucp_tag_recv_nbx,
+                           ucp_rkey_compare],
+                          [], [],
+                          [#include <ucp/api/ucp.h>])
+           AC_CHECK_TYPES([ucp_request_param_t],
+                          [], [],
+                          [[#include <ucp/api/ucp.h>]])
+           ])
 
-                  old_CPPFLAGS="$CPPFLAGS"
-                  AS_IF([test -n "$ompi_check_ucx_dir"],
-                        [CPPFLAGS="$CPPFLAGS -I$ompi_check_ucx_dir/include"])
-                  # Turn off UCX version v1.8 due to issue #8321
-                  AC_MSG_CHECKING([UCX version])
-                  AC_PREPROC_IFELSE([AC_LANG_PROGRAM([#include <ucp/api/ucp_version.h>
-                                                      #if (UCP_API_MAJOR == 1) && (UCP_API_MINOR == 8)
-                                                      #error "Invalid version"
-                                                      #endif], [])],
-                                    [AC_MSG_RESULT([ok (not 1.8.x)])],
-                                    [AC_MSG_RESULT([bad (1.8.x)])
-                                     AC_MSG_WARN([UCX support skipped because version 1.8.x was found, which has a known catastrophic issue.])
-                                     AC_MSG_WARN([Please upgrade to UCX version 1.9 or higher.])
-                                     ompi_check_ucx_happy=no])
-                  AS_IF([test "$ompi_check_ucx_happy" = yes],
-                        [
-                         AC_CHECK_DECLS([ucp_tag_send_nbr],
-                                        [AC_DEFINE([HAVE_UCP_TAG_SEND_NBR],[1],
-                                                   [have ucp_tag_send_nbr()])], [],
-                                        [#include <ucp/api/ucp.h>])
-                         AC_CHECK_DECLS([ucp_ep_flush_nb, ucp_worker_flush_nb,
-                                         ucp_request_check_status, ucp_put_nb, ucp_get_nb,
-                                         ucp_put_nbx, ucp_get_nbx, ucp_atomic_op_nbx],
-                                        [], [],
-                                        [#include <ucp/api/ucp.h>])
-                         AC_CHECK_DECLS([ucm_test_events,
-                                         ucm_test_external_events],
-                                        [], [],
-                                        [#include <ucm/api/ucm.h>])
-                         AC_CHECK_DECLS([UCP_ATOMIC_POST_OP_AND,
-                                         UCP_ATOMIC_POST_OP_OR,
-                                         UCP_ATOMIC_POST_OP_XOR,
-                                         UCP_ATOMIC_FETCH_OP_FAND,
-                                         UCP_ATOMIC_FETCH_OP_FOR,
-                                         UCP_ATOMIC_FETCH_OP_FXOR,
-                                         UCP_PARAM_FIELD_ESTIMATED_NUM_PPN],
-                                        [], [],
-                                        [#include <ucp/api/ucp.h>])
-                         AC_CHECK_DECLS([UCP_WORKER_ATTR_FIELD_ADDRESS_FLAGS],
-                                        [AC_DEFINE([HAVE_UCP_WORKER_ADDRESS_FLAGS], [1],
-                                                   [have worker address attribute])], [],
-                                        [#include <ucp/api/ucp.h>])
-                         AC_CHECK_DECLS([UCP_ATTR_FIELD_MEMORY_TYPES],
-                                        [AC_DEFINE([HAVE_UCP_ATTR_MEMORY_TYPES], [1],
-                                                   [have memory types attribute])], [],
-                                        [#include <ucp/api/ucp.h>])
-                         AC_CHECK_DECLS([ucp_tag_send_nbx,
-                                         ucp_tag_send_sync_nbx,
-                                         ucp_tag_recv_nbx],
-                                        [], [],
-                                        [#include <ucp/api/ucp.h>])
-                         AC_CHECK_TYPES([ucp_request_param_t],
-                                        [], [],
-                                        [[#include <ucp/api/ucp.h>]])
-                        ])
-                  CPPFLAGS=$old_CPPFLAGS
+    CPPFLAGS=${ompi_check_ucx_CPPFLAGS_save}
+    LDFLAGS=${ompi_check_ucx_LDFLAGS_save}
+    LIBS=${ompi_check_ucx_LIBS_save}
 
-                  OPAL_SUMMARY_ADD([[Transports]],[[Open UCX]],[$1],[$ompi_check_ucx_happy])])])
+    OPAL_SUMMARY_ADD([Transports], [Open UCX], [], [$ompi_check_ucx_happy])
 
     AS_IF([test "$ompi_check_ucx_happy" = "yes"],
-          [$1_CPPFLAGS="[$]$1_CPPFLAGS $ompi_check_ucx_CPPFLAGS"
-           $1_LDFLAGS="[$]$1_LDFLAGS $ompi_check_ucx_LDFLAGS"
-           $1_LIBS="[$]$1_LIBS $ompi_check_ucx_LIBS"
-           $2],
+          [$2],
           [AS_IF([test ! -z "$with_ucx" && test "$with_ucx" != "no"],
                  [AC_MSG_ERROR([UCX support requested but not found.  Aborting])])
            $3])
diff --git a/config/ompi_check_udapl.m4 b/config/ompi_check_udapl.m4
deleted file mode 100644
index 114ec124c54..00000000000
--- a/config/ompi_check_udapl.m4
+++ /dev/null
@@ -1,120 +0,0 @@
-dnl -*- shell-script -*-
-dnl
-dnl Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-dnl                         University Research and Technology
-dnl                         Corporation.  All rights reserved.
-dnl Copyright (c) 2004-2005 The University of Tennessee and The University
-dnl                         of Tennessee Research Foundation.  All rights
-dnl                         reserved.
-dnl Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-dnl                         University of Stuttgart.  All rights reserved.
-dnl Copyright (c) 2004-2005 The Regents of the University of California.
-dnl                         All rights reserved.
-dnl Copyright (c) 2007-2009 Cisco Systems, Inc.  All rights reserved.
-dnl Copyright (c) 2015      Research Organization for Information Science
-dnl                         and Technology (RIST). All rights reserved.
-dnl $COPYRIGHT$
-dnl
-dnl Additional copyrights may follow
-dnl
-dnl $HEADER$
-dnl
-
-# OMPI_CHECK_UDAPL(prefix, [action-if-found], [action-if-not-found])
-# --------------------------------------------------------
-# check if uDAPL support can be found.  sets prefix_{CPPFLAGS,
-# LDFLAGS, LIBS} as needed and runs action-if-found if there is
-# support, otherwise executes action-if-not-found
-AC_DEFUN([OMPI_CHECK_UDAPL],[
-    AC_ARG_WITH([udapl],
-        [AS_HELP_STRING([--with-udapl(=DIR)],
-             [Build uDAPL support, optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries])])
-    OPAL_CHECK_WITHDIR([udapl], [$with_udapl], [include/dat/udat.h])
-    AC_ARG_WITH([udapl-libdir],
-       [AS_HELP_STRING([--with-udapl-libdir=DIR],
-             [Search for uDAPL libraries in DIR])])
-    OPAL_CHECK_WITHDIR([udapl-libdir], [$with_udapl_libdir], [libdat.*])
-
-    # Special case for OFED/Linux: the default /etc/dat.conf that
-    # ships with OFED is broken in that it includes DAT providers that
-    # are not guarnateed to work (e.g., it includes providers for ib0,
-    # ib1, ib2, ib3, and bond0).  Usually, a sysadmin will need to
-    # edit this file to configure it for the specific environment in
-    # which it will be used.  Hence, if you run the udapl BTL on
-    # Linux/OFED, you'll get a bunch of warning messages about the
-    # providers that don't work.  However, on Linux/OFED, you don't
-    # really want to use udapl anyway; you likely really want to use
-    # the openib BTL (i.e., native verbs, not udapl).
-
-    # So after exploring many different scenarios, the least evil
-    # solution seemed to be to disable building the udapl BTL on
-    # Linux/OFED *unless the user specifically asks for it.* To be
-    # specific: on Linux/OFED, if you do not specify
-    # --with-udapl(=DIR), the udapl BTL will not be built.
-    AS_IF([test -z "$with_udapl"],
-          [case $host in
-              *linux*)
-                  AC_MSG_WARN([On Linux and --with-udapl was not specified])
-                  AC_MSG_WARN([Not building the udapl BTL])
-                  with_udapl=no
-                  ;;
-           esac])
-
-    AS_IF([test ! -z "$with_udapl" && test "$with_udapl" != "yes"],
-          [ompi_check_udapl_dir="$with_udapl"])
-    AS_IF([test ! -z "$with_udapl_libdir" && test "$with_udapl_libdir" != "yes"],
-          [ompi_check_udapl_libdir="$with_udapl_libdir"])
-    AS_IF([test "$with_udapl" = "no"],
-          [ompi_check_udapl_happy="no"],
-          [ompi_check_udapl_happy="yes"])
-
-dnl Do not use ompi_check_package directly, because then we have
-dnl to test for the header file twice, and caching is disabled
-dnl for all ompi_check_package checks.  Instead, do what
-dnl ompi_check_package does, but only do the header check once.
-dnl Still do the lib check twice, the second time if it turns
-dnl out we need -ldapl to link (looks like udapl over GM).
-
-    ompi_check_package_$1_save_CPPFLAGS="$CPPFLAGS"
-    ompi_check_package_$1_save_LDFLAGS="$LDFLAGS"
-    ompi_check_package_$1_save_LIBS="$LIBS"
-
-    ompi_check_package_$1_orig_CPPFLAGS="$$1_CPPFLAGS"
-    ompi_check_package_$1_orig_LDFLAGS="$$1_LDFLAGS"
-    ompi_check_package_$1_orig_LIBS="$$1_LIBS"
-
-    AS_IF([test "$ompi_check_udapl_happy" = "yes"],
-          [_OPAL_CHECK_PACKAGE_HEADER([$1],
-                [dat/udat.h],
-                [$ompi_check_udapl_dir],
-                [ompi_check_udapl_happy="yes"],
-                [ompi_check_udapl_happy="no"])])
-
-    AS_IF([test "$ompi_check_udapl_happy" = "yes"],
-          [_OPAL_CHECK_PACKAGE_LIB([$1],
-                [dat],
-                [dat_registry_list_providers],
-                [],
-                [$ompi_check_udapl_dir],
-                [$ompi_check_udapl_libdir],
-                [ompi_check_udapl_happy="yes"],
-                [_OPAL_CHECK_PACKAGE_LIB([$1],
-                      [dat],
-                      [dat_registry_list_providers],
-                      [-ldapl],
-                      [$ompi_check_udapl_dir],
-                      [$ompi_check_udapl_libdir],
-                      [ompi_check_udapl_happy="yes"],
-                      [ompi_check_udapl_happy="no"])])])
-
-    CPPFLAGS="$ompi_check_package_$1_save_CPPFLAGS"
-    LDFLAGS="$ompi_check_package_$1_save_LDFLAGS"
-    LIBS="$ompi_check_package_$1_save_LIBS"
-
-    AS_IF([test "$ompi_check_udapl_happy" = "yes"],
-          [$2],
-          [AS_IF([test ! -z "$with_udapl" && test "$with_udapl" != "no"],
-                 [AC_MSG_ERROR([uDAPL support requested but not found.  Aborting])])
-           $3])
-])
-
diff --git a/config/ompi_config_files.m4 b/config/ompi_config_files.m4
index 14e3077b03d..21d1e3eb791 100644
--- a/config/ompi_config_files.m4
+++ b/config/ompi_config_files.m4
@@ -1,11 +1,13 @@
 # -*- shell-script -*-
 #
-# Copyright (c) 2009-2020 Cisco Systems, Inc.  All rights reserved
+# Copyright (c) 2009-2022 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2017-2020 Research Organization for Information Science
 #                         and Technology (RIST).  All rights reserved.
 # Copyright (c) 2018      Los Alamos National Security, LLC. All rights
 #                         reserved.
 # Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+# Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+#                         reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -25,32 +27,23 @@ AC_DEFUN([OMPI_CONFIG_FILES],[
         ompi/debuggers/Makefile
 
         ompi/mpi/c/Makefile
-        ompi/mpi/c/profile/Makefile
         ompi/mpi/fortran/base/Makefile
         ompi/mpi/fortran/mpif-h/Makefile
         ompi/mpi/fortran/mpif-h/profile/Makefile
         ompi/mpi/fortran/use-mpi/Makefile
         ompi/mpi/fortran/use-mpi/mpi-types.F90
-        ompi/mpi/fortran/use-mpi-tkr/Makefile
-        ompi/mpi/fortran/use-mpi-tkr/fortran_sizes.h
-        ompi/mpi/fortran/use-mpi-tkr/fortran_kinds.sh
         ompi/mpi/fortran/use-mpi-ignore-tkr/Makefile
         ompi/mpi/fortran/use-mpi-ignore-tkr/mpi-ignore-tkr-interfaces.h
         ompi/mpi/fortran/use-mpi-ignore-tkr/mpi-ignore-tkr-file-interfaces.h
         ompi/mpi/fortran/use-mpi-ignore-tkr/mpi-ignore-tkr-removed-interfaces.h
         ompi/mpi/fortran/use-mpi-f08/Makefile
         ompi/mpi/fortran/use-mpi-f08/base/Makefile
-        ompi/mpi/fortran/use-mpi-f08/profile/Makefile
         ompi/mpi/fortran/use-mpi-f08/bindings/Makefile
         ompi/mpi/fortran/use-mpi-f08/mod/Makefile
         ompi/mpi/fortran/use-mpi-f08/mod/mpi-f08-interfaces.h
         ompi/mpi/fortran/mpiext-use-mpi/Makefile
         ompi/mpi/fortran/mpiext-use-mpi-f08/Makefile
         ompi/mpi/tool/Makefile
-        ompi/mpi/tool/profile/Makefile
-
-        ompi/mpi/man/man3/Makefile
-        ompi/mpi/man/man5/Makefile
 
         ompi/tools/ompi_info/Makefile
         ompi/tools/wrappers/Makefile
diff --git a/config/ompi_configure_options.m4 b/config/ompi_configure_options.m4
index c7780baf020..0593a317024 100644
--- a/config/ompi_configure_options.m4
+++ b/config/ompi_configure_options.m4
@@ -28,7 +28,7 @@ dnl $HEADER$
 dnl
 
 AC_DEFUN([OMPI_CONFIGURE_OPTIONS],[
-opal_show_subtitle "OMPI Configuration options"
+opal_show_subtitle "MPI layer configuration options"
 
 #
 # Disable MPI layer?
@@ -166,6 +166,15 @@ case "x$enable_mpi_fortran" in
         ;;
 esac
 
+AC_MSG_CHECKING([where to install Fortran MPI modules])
+AC_ARG_WITH([mpi-moduledir],
+    [AS_HELP_STRING([--with-mpi-moduledir],
+                    [specify where to install Fortran MPI modules (default: $libdir)])],
+    [OMPI_FORTRAN_MODULEDIR=$withval],
+    [OMPI_FORTRAN_MODULEDIR=$libdir])
+AC_SUBST(OMPI_FORTRAN_MODULEDIR)
+AC_MSG_RESULT([$OMPI_FORTRAN_MODULEDIR])
+
 # Remove these when we finally kill them once and for all
 AC_ARG_ENABLE([mpi1-compatibility],
     [AS_HELP_STRING([--enable-mpi1-compatibility],
@@ -231,9 +240,25 @@ AC_DEFINE_UNQUOTED(MPI_PARAM_CHECK, $mpi_param_check,
 AC_DEFINE_UNQUOTED(OMPI_PARAM_CHECK, $ompi_param_check,
     [Whether we want to check MPI parameters never or possible (an integer constant)])
 
+AC_MSG_CHECKING([if want OMPIO support])
 AC_ARG_ENABLE([io-ompio],
     [AS_HELP_STRING([--disable-io-ompio],
         [Disable the ompio MPI-IO component])])
+if test "$enable_io_ompio" = "no" ; then
+   AC_MSG_RESULT([no])
+   ompi_want_ompio=0
+else
+   AC_MSG_RESULT([yes])
+   ompi_want_ompio=1
+fi
+AM_CONDITIONAL(OMPI_OMPIO_SUPPORT, test "$ompi_want_ompio" = "1")
+
+# If the binding source files don't exist, then we need Python to generate them
+AM_PATH_PYTHON([3.6],,[:])
+binding_file="${srcdir}/ompi/mpi/c/ompi_send.c"
+AS_IF([! test -e "$binding_file" && test "$PYTHON" = ":"],
+      [AC_MSG_ERROR([Open MPI requires Python >=3.6 for generating the bindings. Aborting])])
+AM_CONDITIONAL(OMPI_GENERATE_BINDINGS,[test "$PYTHON" != ":"])
 
 ])dnl
 
diff --git a/config/ompi_contrib.m4 b/config/ompi_contrib.m4
deleted file mode 100644
index 27f6a37ccc6..00000000000
--- a/config/ompi_contrib.m4
+++ /dev/null
@@ -1,140 +0,0 @@
-dnl -*- shell-script -*-
-dnl
-dnl Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-dnl                         University Research and Technology
-dnl                         Corporation.  All rights reserved.
-dnl Copyright (c) 2004-2005 The University of Tennessee and The University
-dnl                         of Tennessee Research Foundation.  All rights
-dnl                         reserved.
-dnl Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-dnl                         University of Stuttgart.  All rights reserved.
-dnl Copyright (c) 2004-2005 The Regents of the University of California.
-dnl                         All rights reserved.
-dnl Copyright (c) 2007-2010 Cisco Systems, Inc.  All rights reserved.
-dnl Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
-dnl Copyright (c) 2015      Research Organization for Information Science
-dnl                         and Technology (RIST). All rights reserved.
-dnl $COPYRIGHT$
-dnl
-dnl Additional copyrights may follow
-dnl
-dnl $HEADER$
-dnl
-
-######################################################################
-#
-# OMPI_CONTRIB
-#
-# configure the contributed software components.  Currently fairly
-# hard-wired, but someday should be much more like OMPI_MCA.  See
-# https://svn.open-mpi.org/trac/ompi/ticket/1162.
-#
-# USAGE:
-#   OMPI_CONTRIB()
-#
-######################################################################
-AC_DEFUN([OMPI_CONTRIB],[
-    dnl for OPAL_CONFIGURE_USER env variable
-    AC_REQUIRE([OPAL_CONFIGURE_SETUP])
-
-    # Option to not build some of the contributed software packages
-    AC_ARG_ENABLE([contrib-no-build],
-        [AS_HELP_STRING([--enable-contrib-no-build=LIST],
-                        [Comma-separated list of contributed package names that will not be built.  Possible values: ompi_mpicontrib_list.  Example: "--enable-contrib-no-build=foo,bar" will disable building both the "foo" and "bar" contributed software packages (default: none -- i.e., build all possible contrib packages)])])
-
-    # Parse the list to see what we should not build
-    opal_show_subtitle "Configuring contributed software packages"
-    AC_MSG_CHECKING([which contributed software packages should be disabled])
-    if test "$enable_contrib_no_build" = "yes"; then
-        AC_MSG_RESULT([yes])
-        AC_MSG_ERROR([*** The enable-contrib-no-build flag requires an explicit list
-*** of packages to not build.  For example, --enable-contrib-no-build=libompitrace])
-    else
-        ifs_save="$IFS"
-        IFS="${IFS}$PATH_SEPARATOR,"
-        msg=
-        for item in $enable_contrib_no_build; do
-            str="`echo DISABLE_contrib_${item}=1 | sed s/-/_/g`"
-            eval $str
-            msg="$item $msg"
-        done
-        IFS="$ifs_save"
-    fi
-    AC_MSG_RESULT([$msg])
-    unset msg
-
-    # List of contrib subdirs to traverse into
-    OMPI_CONTRIB_SUBDIRS=
-    OMPI_CONTRIB_DIST_SUBDIRS=
-    OMPI_MPI_CONTRIBS=
-
-    # Cycle through each of the software packages and
-    # configure them if not disabled.
-    m4_foreach(software, [ompi_mpicontrib_list],
-              [_OMPI_CONTRIB_CONFIGURE(software)])
-
-    # Setup the top-level glue
-    AC_DEFINE_UNQUOTED([OMPI_MPI_CONTRIBS], ["$OMPI_MPI_CONTRIBS"],
-                       [Contributed software packages built with Open MPI])
-    AC_SUBST(OMPI_CONTRIB_SUBDIRS)
-    AC_SUBST(OMPI_CONTRIB_DIST_SUBDIRS)
-])dnl
-
-
-######################################################################
-#
-# _OMPI_CONTRIB_SOFTWARE
-#
-# Setup a specific contributed software package.  This is a subroutine
-# because the work to setup each package is essentially the same.
-# Currently assumes that there is a configure.m4 file in the
-# contributed software directory.  May someday be expanded to handle
-# other things.
-#
-# USAGE:
-#   _OMPI_CONTRIB_SOFTARE([package_name])
-#
-######################################################################
-AC_DEFUN([_OMPI_CONTRIB_CONFIGURE],[
-
-    opal_show_subsubsubtitle "$1 (m4 configuration macro)"
-
-    # Put in a convenient enable/disable switch (it's a little more
-    # user friendly than
-    # --enable-contrib-no-build=<comma_delimited_list>, although each
-    # works just as well as the other).
-    AC_ARG_ENABLE([$1],
-            [AS_HELP_STRING([--disable-$1],
-                            [disable support for contributed package $1 (default: enabled)])])
-    AS_IF([test "x$enable_$1" = xno], [DISABLE_contrib_$1=yes])
-
-    OMPI_CONTRIB_HAPPY=0
-    if test "$DISABLE_contrib_$1" = "" && test "$DISABLE_contrib_all" = ""; then
-        OMPI_contrib_$1_CONFIG([OMPI_CONTRIB_HAPPY=1], [])
-        AC_MSG_CHECKING([if contributed component $1 can compile])
-        if test "$OMPI_CONTRIB_HAPPY" = "1"; then
-            OMPI_CONTRIB_SUBDIRS="$OMPI_CONTRIB_SUBDIRS contrib/$1"
-            OMPI_CONTRIB_DIST_SUBDIRS="$OMPI_CONTRIB_DIST_SUBDIRS contrib/$1"
-            if test "$OMPI_MPI_CONTRIBS" = ""; then
-                OMPI_MPI_CONTRIBS=$1
-            else
-                OMPI_MPI_CONTRIBS="$1, $OMPI_MPI_CONTRIBS"
-            fi
-            AC_MSG_RESULT([yes])
-        else
-            AC_MSG_RESULT([no])
-
-            # If this component was requested via command line switch, then abort.
-            if test "x$enable_$1" = xyes ; then
-                AC_MSG_WARN([Contributed component "$1" failed to configure properly])
-                AC_MSG_WARN([This component was requested via command line switch])
-                AC_MSG_ERROR([Cannot continue])
-            fi
-        fi
-    else
-        AC_MSG_NOTICE([disabled via command line switch])
-    fi
-    AC_DEFINE_UNQUOTED(OMPI_ENABLE_CONTRIB_$1, [$OMPI_CONTRIB_HAPPY],
-                       [Enable contributed software package $1])
-    unset OMPI_CONTRIB_HAPPY
-])dnl
diff --git a/config/ompi_deleted_options.m4 b/config/ompi_deleted_options.m4
index d068c7c0e57..b3e53bb42d6 100644
--- a/config/ompi_deleted_options.m4
+++ b/config/ompi_deleted_options.m4
@@ -1,7 +1,7 @@
 # -*- shell-script -*-
 #
 # Copyright (c) 2020      Intel, Inc.  All rights reserved.
-# Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved
+# Copyright (c) 2020-2021 Cisco Systems, Inc.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -10,7 +10,7 @@
 #
 
 AC_DEFUN([OMPI_CHECK_DELETED_OPTIONS],[
-    OPAL_VAR_SCOPE_PUSH([with_pmi_given with_pmi_libdir_given cxx])
+    OPAL_VAR_SCOPE_PUSH([with_pmi_given with_pmi_libdir_given ompi_cxx_warn ompi_cxx_error])
 
     # --with-pmi options were removed in v5.0
     AC_ARG_WITH([pmi],
@@ -41,38 +41,55 @@ AC_DEFUN([OMPI_CHECK_DELETED_OPTIONS],[
         AC_MSG_ERROR([Build cannot continue.])
     fi
 
-    # Open MPI C++ bindings were removed in v5.0
-    cxx=0
-    cxxseek=0
-    cxxex=0
+    # Note that we always *warn* if someone used a CLI option for a
+    # feature that has been deleted.  If, however, they are disabling
+    # the deleted feature (e.g., --disable-mpi-cxx), then emitting a
+    # warning is good enough -- allow configure to continue.  If,
+    # however, the user asked to enable a deleted feature, then
+    # configure needs to error out.
+    ompi_cxx_warn=0
+    ompi_cxx_error=0
     AC_ARG_ENABLE([mpi-cxx],
                   [AS_HELP_STRING([--enable-mpi-cxx],
                                   [*DELETED* Build the MPI C++ bindings])],
-                  [cxx=1])
+                  [ompi_cxx_warn=1
+                   AS_IF([test "$enable_mpi_cxx" != "no"],
+                         [ompi_cxx_error=1])
+                  ])
     AC_ARG_ENABLE([mpi-cxx-seek],
                   [AS_HELP_STRING([--enable-mpi-cxx-seek],
                                   [*DELETED* Build support for MPI::SEEK])],
-                  [cxxseek=1])
+                  [ompi_cxx_warn=1
+                   AS_IF([test "$enable_mpi_cxx_seek" != "no"],
+                         [ompi_cxx_error=1])
+                  ])
     AC_ARG_ENABLE([cxx-exceptions],
                   [AS_HELP_STRING([--enable-cxx-exceptions],
                                   [*DELETED* Build support for C++ exceptions in the MPI C++ bindings])],
-                  [cxxex=1])
+                  [ompi_cxx_warn=1
+                   AS_IF([test "$enable_cxx_exceptions" != "no"],
+                         [ompi_cxx_error=1])
+                  ])
 
-    AS_IF([test "$enable_mpi_cxx" = "no" ],
-          [cxx=0])
-
-    AS_IF([test "$enable_mpi_cxx_seek" = "no" ],
-          [cxxseek=0])
-
-    AS_IF([test "$enable_cxx_exceptions" = "no" ],
-          [cxxex=0])
-
-    AS_IF([test $cxx -eq 1 || test $cxxseek -eq 1 || test $cxxex -eq 1],
-          [AC_MSG_WARN([The MPI C++ bindings have been removed from Open MPI.])
+    AS_IF([test $ompi_cxx_warn -eq 1],
+          [AC_MSG_WARN([An MPI C++ bindings-related command line option])
+           AC_MSG_WARN([was given to "configure".])
+           AC_MSG_WARN([ ])
+           AC_MSG_WARN([This command line option will be removed in a future])
+           AC_MSG_WARN([version of Open MPI; you should discontinue using it.])
+           AC_MSG_WARN([You have been warned!])
+           AC_MSG_WARN([ ])
+           AC_MSG_WARN([The MPI C++ bindings were deprecated in the MPI-2.2])
+           AC_MSG_WARN([standard in 2009, and removed from the MPI-3.0])
+           AC_MSG_WARN([standard in 2012.  The MPI C++ bindings were then])
+           AC_MSG_WARN([removed from Open MPI v5.0.0 in 2022.])
+           AC_MSG_WARN([ ])
            AC_MSG_WARN([If you need support for the MPI C++ bindings, you])
            AC_MSG_WARN([will need to use an older version of Open MPI.])
-           AC_MSG_ERROR([Build cannot continue.])
           ])
 
+    AS_IF([test $ompi_cxx_error -eq 1],
+          [AC_MSG_ERROR([Build cannot continue.])])
+
     OPAL_VAR_SCOPE_POP
 ])
diff --git a/config/ompi_ext.m4 b/config/ompi_ext.m4
index 6f96ef2fdbe..5120b3603e7 100644
--- a/config/ompi_ext.m4
+++ b/config/ompi_ext.m4
@@ -34,6 +34,9 @@ AC_DEFUN([OMPI_EXT],[
     dnl for OPAL_CONFIGURE_USER env variable
     AC_REQUIRE([OPAL_CONFIGURE_SETUP])
 
+    m4_ifdef([ompi_mpiext_list], [],
+             [m4_fatal([Could not find MPI Extensions list.  Aborting.])])
+
     # Note that we do not build DSO's here -- we *only* build convenience
     # libraries that get slurped into higher-level libraries
     #
@@ -42,7 +45,7 @@ AC_DEFUN([OMPI_EXT],[
     #
     AC_ARG_ENABLE([mpi-ext],
         [AS_HELP_STRING([--enable-mpi-ext[=LIST]],
-                       [Comma-separated list of extensions that should be built.  Possible values: ompi_mpiext_list.  Example: "--enable-mpi-ext=foo,bar" will enable building the MPI extensions "foo" and "bar".  If LIST is empty or the special value "all", then all available MPI extensions will be built (default: all).])])
+                       [Comma-separated list of extensions that should be built.  Possible values: ]m4_quote(ompi_mpiext_list)[.  Example: "--enable-mpi-ext=foo,bar" will enable building the MPI extensions "foo" and "bar".  If LIST is empty or the special value "all", then all available MPI extensions will be built (default: all).])])
 
     # print some nice messages about what we're about to do...
     AC_MSG_CHECKING([for available MPI Extensions])
@@ -74,9 +77,6 @@ AC_DEFUN([OMPI_EXT],[
     AC_MSG_RESULT([$msg])
     unset msg
 
-    m4_ifdef([ompi_mpiext_list], [],
-             [m4_fatal([Could not find MPI Extensions list.  Aborting.])])
-
     EXT_CONFIGURE
 ])
 
@@ -390,6 +390,9 @@ AC_DEFUN([EXT_CONFIGURE_M4_CONFIG_COMPONENT],[
     AS_IF([test $should_build -eq 1],
           [EXT_PROCESS_COMPONENT([$1], [$2], [$3], [$4], [$5], [$6], [$7])],
           [EXT_PROCESS_DEAD_COMPONENT([$1], [$2])])
+
+    m4_ifdef([OMPI_MPIEXT_$1_POST_CONFIG],
+             [OMPI_MPIEXT_$1_POST_CONFIG($should_build)])
 ])
 
 ######################################################################
diff --git a/config/ompi_find_mpi_aint_count_offset.m4 b/config/ompi_find_mpi_aint_count_offset.m4
index 5aec6d3ed9f..6684dd10e2f 100644
--- a/config/ompi_find_mpi_aint_count_offset.m4
+++ b/config/ompi_find_mpi_aint_count_offset.m4
@@ -95,7 +95,7 @@ AC_DEFUN([_OMPI_FIND_MPI_COUNT_TYPE], [
     # memory space, the scheme we use below may not work.  ...but such
     # systems are pretty rare today.
 
-    MPI_COUNT_TYPE=unknonwn
+    MPI_COUNT_TYPE=unknown
     AC_MSG_CHECKING([for type of MPI_Count])
     if test $ac_cv_sizeof_long_long -le $ac_cv_sizeof_size_t && \
        test $ac_cv_sizeof_long_long -ge $MPI_AINT_SIZE; then
diff --git a/config/ompi_fortran_check.m4 b/config/ompi_fortran_check.m4
index 86265473a2f..777da4779a4 100644
--- a/config/ompi_fortran_check.m4
+++ b/config/ompi_fortran_check.m4
@@ -32,7 +32,7 @@ dnl
 # - alignment
 # - associated C type
 #
-# types to search is a comma-seperated list of values
+# types to search is a comma-separated list of values
 AC_DEFUN([OMPI_FORTRAN_CHECK], [
     OPAL_VAR_SCOPE_PUSH([ofc_have_type ofc_type_size ofc_type_alignment ofc_c_type ofc_expected_size])
 
@@ -117,7 +117,7 @@ AC_DEFUN([OMPI_FORTRAN_CHECK], [
 
             AC_MSG_CHECKING([for corresponding KIND value of $1])
             case "$ofc_c_type" in
-            char)     	          ofc_type_kind=C_SIGNED_CHAR         ;;
+            char)                 ofc_type_kind=C_SIGNED_CHAR         ;;
             double)               ofc_type_kind=C_DOUBLE              ;;
             float)                ofc_type_kind=C_FLOAT               ;;
             int)                  ofc_type_kind=C_INT                 ;;
diff --git a/config/ompi_fortran_check_asynchronous.m4 b/config/ompi_fortran_check_asynchronous.m4
index 0cc3c84bfe5..62c53159c50 100644
--- a/config/ompi_fortran_check_asynchronous.m4
+++ b/config/ompi_fortran_check_asynchronous.m4
@@ -11,6 +11,8 @@ dnl                         University of Stuttgart.  All rights reserved.
 dnl Copyright (c) 2004-2005 The Regents of the University of California.
 dnl                         All rights reserved.
 dnl Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
+dnl Copyright (c) 2024      Research Organization for Information Science
+dnl                         and Technology (RIST).  All rights reserved.
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
@@ -35,6 +37,10 @@ SUBROUTINE binky(buf)
  REAL, DIMENSION(*), ASYNCHRONOUS :: buf
 END SUBROUTINE
 END INTERFACE
+CONTAINS
+SUBROUTINE wookie(buf)
+ REAL, DIMENSION(*), ASYNCHRONOUS :: buf
+END SUBROUTINE
 END MODULE asynch_mod]])],
              [AS_VAR_SET(asynchronous_var, yes)],
              [AS_VAR_SET(asynchronous_var, no)])
diff --git a/config/ompi_fortran_check_elemental.m4 b/config/ompi_fortran_check_elemental.m4
new file mode 100644
index 00000000000..ca3658f1dc8
--- /dev/null
+++ b/config/ompi_fortran_check_elemental.m4
@@ -0,0 +1,47 @@
+dnl -*- shell-script -*-
+dnl
+dnl Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+dnl                         University Research and Technology
+dnl                         Corporation.  All rights reserved.
+dnl Copyright (c) 2004-2005 The University of Tennessee and The University
+dnl                         of Tennessee Research Foundation.  All rights
+dnl                         reserved.
+dnl Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+dnl                         University of Stuttgart.  All rights reserved.
+dnl Copyright (c) 2004-2005 The Regents of the University of California.
+dnl                         All rights reserved.
+dnl Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
+dnl Copyright (c) 2022      Triad National Security, LLC. All rights
+dnl                         reserved.
+dnl $COPYRIGHT$
+dnl
+dnl Additional copyrights may follow
+dnl
+dnl $HEADER$
+dnl
+
+# Check whether or not the Fortran compiler supports the "elemental"
+# keyword in derived types or not.
+
+# OMPI_FORTRAN_CHECK_ELEMENTAL([action if found],
+#                            [action if not found])
+# ----------------------------------------------------
+AC_DEFUN([OMPI_FORTRAN_CHECK_ELEMENTAL],[
+    AS_VAR_PUSHDEF([elemental_var], [ompi_cv_fortran_elemental])
+
+    AC_CACHE_CHECK([if Fortran compiler supports ELEMENTAL], elemental_var,
+       [AC_LANG_PUSH([Fortran])
+        AC_COMPILE_IFELSE([AC_LANG_SOURCE([[ELEMENTAL SUBROUTINE binky(buf)
+ REAL,INTENT(INOUT) :: buf
+ BUF = BUF + 1
+END SUBROUTINE binky]])],
+             [AS_VAR_SET(elemental_var, yes)],
+             [AS_VAR_SET(elemental_var, no)])
+        touch conftest_foo.mod
+        rm -rf *.mod 2>/dev/null
+        AC_LANG_POP([Fortran])
+       ])
+
+    AS_VAR_IF(elemental_var, [yes], [$1], [$2])
+    AS_VAR_POPDEF([elemental_var])dnl
+])
diff --git a/config/ompi_fortran_check_f08_assumed_rank.m4 b/config/ompi_fortran_check_f08_assumed_rank.m4
index 513bbdb2435..82ed6aafe7c 100644
--- a/config/ompi_fortran_check_f08_assumed_rank.m4
+++ b/config/ompi_fortran_check_f08_assumed_rank.m4
@@ -39,7 +39,7 @@ AC_DEFUN([OMPI_FORTRAN_CHECK_F08_ASSUMED_RANK], [
 ###################################
 
 AC_DEFUN([_OMPI_FORTRAN_CHECK_F08_ASSUMED_RANK], [
-    OPAL_VAR_SCOPE_PUSH([happy])
+    OPAL_VAR_SCOPE_PUSH([f08_assumed_rank_happy])
 
     # If we were called here, it means that the value was not cached,
     # so we need to check several different things.  Since CACHE_CHECK
@@ -50,9 +50,9 @@ AC_DEFUN([_OMPI_FORTRAN_CHECK_F08_ASSUMED_RANK], [
     # Check for the F08 type(*),dimension(..) syntax
     OMPI_FORTRAN_CHECK_IGNORE_TKR_SUB([!], [TYPE(*), DIMENSION(..)],
                                       [TYPE(*), DIMENSION(..)],
-                                      [happy=yes], [happy=no])
+                                      [f08_assumed_rank_happy=yes], [f08_assumed_rank_happy=no])
 
-    AS_VAR_SET(fortran_f08_assumed_rank, [$happy]);
+    AS_VAR_SET(fortran_f08_assumed_rank, [$f08_assumed_rank_happy]);
 
     # Now put the original CACHE_CHECK MSG_CHECKING back so that it can
     # output the MSG_RESULT.
diff --git a/config/ompi_fortran_check_ignore_tkr.m4 b/config/ompi_fortran_check_ignore_tkr.m4
index bb64eb50565..ac23f42b913 100644
--- a/config/ompi_fortran_check_ignore_tkr.m4
+++ b/config/ompi_fortran_check_ignore_tkr.m4
@@ -14,6 +14,8 @@ dnl Copyright (c) 2007      Los Alamos National Security, LLC.  All rights
 dnl                         reserved.
 dnl Copyright (c) 2007      Sun Microsystems, Inc.  All rights reserved.
 dnl Copyright (c) 2009-2015 Cisco Systems, Inc.  All rights reserved.
+dnl Copyright (c) 2024      Research Organization for Information Science
+dnl                         and Technology (RIST).  All rights reserved.
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
@@ -23,7 +25,7 @@ dnl $HEADER$
 # Does this compiler support (void*)-like functionality for MPI choice
 # buffers?  If so, which flavor?
 AC_DEFUN([OMPI_FORTRAN_CHECK_IGNORE_TKR], [
-    OPAL_VAR_SCOPE_PUSH([result happy type predecl])
+    OPAL_VAR_SCOPE_PUSH([result ignore_tkr_happy type predecl])
 
     OMPI_FORTRAN_IGNORE_TKR_PREDECL=
     OMPI_FORTRAN_IGNORE_TKR_TYPE=
@@ -41,11 +43,11 @@ AC_DEFUN([OMPI_FORTRAN_CHECK_IGNORE_TKR], [
     AS_VAR_COPY([result], [fortran_ignore_tkr_data])
 
     # Parse the result
-    happy=`echo $result | cut -d: -f1`
+    ignore_tkr_happy=`echo $result | cut -d: -f1`
     type=`echo $result | cut -d: -f2`
     predecl=`echo $result | cut -d: -f3-`
 
-    AS_IF([test $happy -eq 1],
+    AS_IF([test $ignore_tkr_happy -eq 1],
           [OMPI_FORTRAN_IGNORE_TKR_PREDECL=$predecl
            OMPI_FORTRAN_IGNORE_TKR_TYPE=$type
            $1],
@@ -58,7 +60,7 @@ AC_DEFUN([OMPI_FORTRAN_CHECK_IGNORE_TKR], [
 ################
 
 AC_DEFUN([_OMPI_FORTRAN_CHECK_IGNORE_TKR], [
-    OPAL_VAR_SCOPE_PUSH([happy ompi_fortran_ignore_tkr_predecl ompi_fortran_ignore_tkr_type])
+    OPAL_VAR_SCOPE_PUSH([internal_ignore_tkr_happy ompi_fortran_ignore_tkr_predecl ompi_fortran_ignore_tkr_type])
 
     # If we were called here, it means that the value was not cached,
     # so we need to check several different things.  Since CACHE_CHECK
@@ -74,43 +76,49 @@ AC_DEFUN([_OMPI_FORTRAN_CHECK_IGNORE_TKR], [
     OMPI_FORTRAN_CHECK_IGNORE_TKR_SUB(
          [!], [type(*)],
          [TYPE(*), DIMENSION(*)],
-         [happy=1], [happy=0])
+         [internal_ignore_tkr_happy=1], [internal_ignore_tkr_happy=0])
 
     # GCC compilers
-    AS_IF([test $happy -eq 0],
+    AS_IF([test $internal_ignore_tkr_happy -eq 0],
           [OMPI_FORTRAN_CHECK_IGNORE_TKR_SUB(
               [!GCC\$ ATTRIBUTES NO_ARG_CHECK ::], [type(*), dimension(*)],
               [!GCC\$ ATTRIBUTES NO_ARG_CHECK],
-              [happy=1], [happy=0])])
+              [internal_ignore_tkr_happy=1], [internal_ignore_tkr_happy=0])])
+    # LLVM compilers
+    AS_IF([test $internal_ignore_tkr_happy -eq 0],
+          [OMPI_FORTRAN_CHECK_IGNORE_TKR_SUB(
+              [!DIR\$ IGNORE_TKR], [type(*)],
+              [!DIR\$ IGNORE_TKR],
+              [internal_ignore_tkr_happy=1], [internal_ignore_tkr_happy=0])])
     # Intel compilers
-    AS_IF([test $happy -eq 0],
+    AS_IF([test $internal_ignore_tkr_happy -eq 0],
           [OMPI_FORTRAN_CHECK_IGNORE_TKR_SUB(
               [!DEC\$ ATTRIBUTES NO_ARG_CHECK ::], [real, dimension(*)],
               [!DEC\$ ATTRIBUTES NO_ARG_CHECK],
-              [happy=1], [happy=0])])
+              [internal_ignore_tkr_happy=1], [internal_ignore_tkr_happy=0])])
     # Solaris Studio compilers
     # Note that due to a compiler bug, we have been advised by Oracle to
     # use the "character(*)" type
-    AS_IF([test $happy -eq 0],
+    AS_IF([test $internal_ignore_tkr_happy -eq 0],
           [OMPI_FORTRAN_CHECK_IGNORE_TKR_SUB(
               [!\$PRAGMA IGNORE_TKR], [character(*)],
               [!\$PRAGMA IGNORE_TKR],
-              [happy=1], [happy=0])])
+              [internal_ignore_tkr_happy=1], [internal_ignore_tkr_happy=0])])
     # Cray compilers
-    AS_IF([test $happy -eq 0],
+    AS_IF([test $internal_ignore_tkr_happy -eq 0],
           [OMPI_FORTRAN_CHECK_IGNORE_TKR_SUB(
               [!DIR\$ IGNORE_TKR], [real, dimension(*)],
               [!DIR\$ IGNORE_TKR],
-              [happy=1], [happy=0])])
+              [internal_ignore_tkr_happy=1], [internal_ignore_tkr_happy=0])])
     # IBM compilers
-    AS_IF([test $happy -eq 0],
+    AS_IF([test $internal_ignore_tkr_happy -eq 0],
           [OMPI_FORTRAN_CHECK_IGNORE_TKR_SUB(
               [!IBM* IGNORE_TKR], [real, dimension(*)],
               [!IBM* IGNORE_TKR],
-              [happy=1], [happy=0])])
+              [internal_ignore_tkr_happy=1], [internal_ignore_tkr_happy=0])])
 
     AS_VAR_SET(fortran_ignore_tkr_data,
-               [${happy}:${ompi_fortran_ignore_tkr_type}:${ompi_fortran_ignore_tkr_predecl}])
+               [${internal_ignore_tkr_happy}:${ompi_fortran_ignore_tkr_type}:${ompi_fortran_ignore_tkr_predecl}])
 
     # Now put the original CACHE_CHECK MSG_CHECKING back so that it can
     # output the MSG_RESULT.
@@ -133,6 +141,7 @@ AC_DEFUN([OMPI_FORTRAN_CHECK_IGNORE_TKR_SUB], [
     AC_MSG_CHECKING([for Fortran compiler support of $3])
     AC_COMPILE_IFELSE(AC_LANG_PROGRAM([],[[!
 ! Autoconf puts "program main" at the top
+ implicit none
 
   interface
      subroutine force_assumed_shape(a, count)
@@ -157,6 +166,7 @@ AC_DEFUN([OMPI_FORTRAN_CHECK_IGNORE_TKR_SUB], [
   complex, pointer, dimension(:,:) :: ptr
   target :: buffer3
   integer :: buffer4
+  integer :: a
   ptr => buffer3
 
 ! Set some known values (somewhat irrelevant for this test, but just be
@@ -189,8 +199,23 @@ AC_DEFUN([OMPI_FORTRAN_CHECK_IGNORE_TKR_SUB], [
     call foo(a, count)
   end subroutine force_assumed_shape
 
+  module check_ignore_tkr
+  interface foobar
+     subroutine foobar_x(buffer, count)
+       $1 buffer
+       $2, intent(in) :: buffer
+       integer, intent(in) :: count
+     end subroutine foobar_x
+  end interface
+  end module
+
+  subroutine bar(var)
+    use check_ignore_tkr
+    implicit none
+    real, intent(inout) :: var(:, :, :)
+
+    call foobar(var(1,1,1), 1)
 ! Autoconf puts "end" after the last line
-  subroutine bogus
 ]]),
                     [msg=yes
                      ompi_fortran_ignore_tkr_predecl="$1"
diff --git a/config/ompi_fortran_check_iso_fortran_env_real16.m4 b/config/ompi_fortran_check_iso_fortran_env_real16.m4
new file mode 100644
index 00000000000..9089098a1f6
--- /dev/null
+++ b/config/ompi_fortran_check_iso_fortran_env_real16.m4
@@ -0,0 +1,34 @@
+dnl -*- shell-script -*-
+dnl
+dnl Copyright (c) 2021      Research Organization for Information Science
+dnl                         and Technology (RIST).  All rights reserved.
+dnl $COPYRIGHT$
+dnl
+dnl Additional copyrights may follow
+dnl
+dnl $HEADER$
+dnl
+
+# Check whether or not the Fortran compiler supports the non standard
+# iso_fortran_env:real16 or not
+#
+# OMPI_FORTRAN_CHECK_ISO_FORTRAN_ENV_REAL16([action if found], [action if not found])
+# ----------------------------------------------------
+AC_DEFUN([OMPI_FORTRAN_CHECK_ISO_FORTRAN_ENV_REAL16],[
+    AS_VAR_PUSHDEF([iso_fortran_env_var_real16], [ompi_cv_fortran_have_iso_fortran_env_real16])
+
+    AC_CACHE_CHECK([if Fortran compiler supports ISO_FORTRAN_ENV:REAL16], iso_fortran_env_var_real16,
+       [AC_LANG_PUSH([Fortran])
+        AC_COMPILE_IFELSE([AC_LANG_SOURCE([[program check_for_iso_fortran_env_real16
+   use, intrinsic :: iso_fortran_env
+   real(REAL16) :: var
+   var = 12.34
+end program]])],
+             [AS_VAR_SET(iso_fortran_env_var_real16, yes)],
+             [AS_VAR_SET(iso_fortran_env_var_real16, no)])
+        AC_LANG_POP([Fortran])
+       ])
+
+    AS_VAR_IF(iso_fortran_env_var_real16, [yes], [$1], [$2])
+    AS_VAR_POPDEF([iso_fortran_env_var_real16])
+])
diff --git a/config/ompi_fortran_check_real16_c_equiv.m4 b/config/ompi_fortran_check_real16_c_equiv.m4
index 12e427d754b..3e39798d1c7 100644
--- a/config/ompi_fortran_check_real16_c_equiv.m4
+++ b/config/ompi_fortran_check_real16_c_equiv.m4
@@ -24,8 +24,7 @@ dnl
 # OMPI_FORTRAN_CHECK_REAL16_C_EQUIV
 # ----------------------------------------------------
 AC_DEFUN([OMPI_FORTRAN_CHECK_REAL16_C_EQUIV],[
-    unset happy
-    OPAL_VAR_SCOPE_PUSH([happy define_value msg CFLAGS_save])
+    OPAL_VAR_SCOPE_PUSH([fortran_real16_happy define_value msg CFLAGS_save])
     AS_VAR_PUSHDEF([real16_matches_c_var], [ompi_cv_real16_c_equiv])
 
     # We have to do this as a cache check for cross-compilation platforms
@@ -42,16 +41,15 @@ AC_DEFUN([OMPI_FORTRAN_CHECK_REAL16_C_EQUIV],[
                 OMPI_FORTRAN_CHECK_REAL16_EQUIV_TYPE([$OMPI_FORTRAN_REAL16_C_TYPE], [L])
                 # If that didn't work, see if we have a compiler-specific
                 # type that might work
-                AS_IF([test "$happy" = "no"],
-                      [AC_MSG_RESULT([$happy])
+                AS_IF([test "$fortran_real16_happy" = "no"],
+                      [AC_MSG_RESULT([$fortran_real16_happy])
                        # Intel compiler has a special type that should work
                        AS_IF([test "$opal_cv_c_compiler_vendor" = "intel"],
                              [AC_MSG_CHECKING([if intel compiler _Quad == REAL*16])
                               CFLAGS_save="$CFLAGS"
-                              CFLAGS="$CFLAGS -Qoption,cpp,--extended_float_types"
-                              OPAL_FLAGS_UNIQ([CFLAGS])
+                              OPAL_FLAGS_APPEND_UNIQ([CFLAGS], ["-Qoption,cpp,--extended_float_types"])
                               OMPI_FORTRAN_CHECK_REAL16_EQUIV_TYPE([_Quad], [q])
-                              AS_IF([test "$happy" = "yes"],
+                              AS_IF([test "$fortran_real16_happy" = "yes"],
                                     [OMPI_FORTRAN_REAL16_C_TYPE="_Quad"
                                      AC_MSG_RESULT([works!])],
                                     [CFLAGS="$CFLAGS_save"
@@ -59,9 +57,8 @@ AC_DEFUN([OMPI_FORTRAN_CHECK_REAL16_C_EQUIV],[
                              ])
                        AS_IF([test "$opal_cv_c_compiler_vendor" = "gnu" && test "$ac_cv_type___float128" = "yes"],
                              [AC_MSG_CHECKING([if gnu compiler __float128 == REAL*16])
-                              OPAL_FLAGS_UNIQ([CFLAGS])
                               OMPI_FORTRAN_CHECK_REAL16_EQUIV_TYPE([__float128], [q])
-                              AS_IF([test "$happy" = "yes"],
+                              AS_IF([test "$fortran_real16_happy" = "yes"],
                                     [OMPI_FORTRAN_REAL16_C_TYPE="__float128"
                                      AC_MSG_RESULT([works!])],
                                     [AC_MSG_RESULT([does not work])])
@@ -70,7 +67,7 @@ AC_DEFUN([OMPI_FORTRAN_CHECK_REAL16_C_EQUIV],[
                        # AC_CACHE_CHECK will automatically AC_MSG_RESULT
                        AC_MSG_CHECKING([for C type matching bit representation of REAL*16])
                       ])
-                AS_VAR_SET(real16_matches_c_var, [$happy])
+                AS_VAR_SET(real16_matches_c_var, [$fortran_real16_happy])
                ],[
                 # No fortran bindings or no REAL*16
                 AS_IF([test "$OMPI_TRY_FORTRAN_BINDINGS" = "$OMPI_FORTRAN_NO_BINDINGS"],
@@ -136,15 +133,15 @@ EOF
     # Compile and link
     OPAL_LOG_COMMAND([$CC $CFLAGS -I. -c conftest_c.c],
         [OPAL_LOG_COMMAND([$FC $FCFLAGS conftest_f.f conftest_c.o -o conftest $LDFLAGS $LIBS],
-            [happy="yes"], [happy="no"])], [happy="no"])
-    AS_IF([test "$happy" = "no"],
+            [fortran_real16_happy="yes"], [fortran_real16_happy="no"])], [fortran_real16_happy="no"])
+    AS_IF([test "$fortran_real16_happy" = "no"],
         [AC_MSG_RESULT([Could not determine if REAL*16 bit-matches C type])],
         # If it worked so far, try running to see what we get
-        [AS_IF([test "$happy" = "yes" && test "$cross_compiling" = "yes"],
+        [AS_IF([test "$fortran_real16_happy" = "yes" && test "$cross_compiling" = "yes"],
              [AC_MSG_RESULT([Error!])
               AC_MSG_ERROR([Can not determine if REAL*16 bit-matches C if cross compiling])],
              [OPAL_LOG_COMMAND([./conftest],
-                 [happy=`cat conftestval`],
+                 [fortran_real16_happy=`cat conftestval`],
                  [AC_MSG_RESULT([Error!])
                   AC_MSG_ERROR([Could not determine if REAL*16 bit-matches C type])
                  ])
diff --git a/config/ompi_fortran_check_ts.m4 b/config/ompi_fortran_check_ts.m4
new file mode 100644
index 00000000000..1cc9d07a827
--- /dev/null
+++ b/config/ompi_fortran_check_ts.m4
@@ -0,0 +1,69 @@
+dnl -*- shell-script -*-
+dnl
+dnl Copyright (c) 2019      Research Organization for Information Science
+dnl                         and Technology (RIST).  All rights reserved.
+dnl $COPYRIGHT$
+dnl
+dnl Additional copyrights may follow
+dnl
+dnl $HEADER$
+dnl
+
+# Check whether or not the C compiler supports ISO_Fortran_binding.h
+# Also check whether C and Fortran compiler interoperate.
+#
+# OMPI_FORTRAN_CHECK_TS([action if found], [action if not found])
+# ----------------------------------------------------
+AC_DEFUN([OMPI_FORTRAN_CHECK_TS],[
+    AS_VAR_PUSHDEF([fortran_ts], [ompi_cv_fortran_have_ts])
+
+    AC_CHECK_HEADERS([ISO_Fortran_binding.h],
+      [AC_CACHE_CHECK([if Fortran and C compilers support ISO_Fortran_binding.h], fortran_ts,
+         [mkdir conftest.$$
+          cd conftest.$$
+
+          # Try to compile the C bindings
+          cat > conftest_c.c << EOF
+#include <ISO_Fortran_binding.h>
+
+int is_contiguous_c(CFI_cdesc_t* x) {
+    return CFI_is_contiguous(x);
+}
+EOF
+          OPAL_LOG_COMMAND([$CC $CCFLAGS -c conftest_c.c],
+              [cat > conftest.f90 << EOF
+module MOD_IS_CONTIGUOUS
+
+interface
+
+function is_contiguous(buf) BIND(C, name="is_contiguous_c")
+   implicit none
+   type(*), dimension(..) :: buf
+   integer :: is_contiguous
+end function is_contiguous
+
+end interface
+
+end module
+
+program test_is_contiguous
+   use MOD_IS_CONTIGUOUS
+   implicit none
+   integer :: a0, a1(2), a2(2,2), a3(2,2,2)
+   write (*,*) is_contiguous(a0)
+   write (*,*) is_contiguous(a1)
+   write (*,*) is_contiguous(a2)
+   write (*,*) is_contiguous(a3)
+end program
+EOF
+               OPAL_LOG_COMMAND([$FC $FCFLAGS $FCFLAGS_f90 -o conftest conftest.f90 conftest_c.o $LDFLAGS $LIBS],
+                   [AS_VAR_SET(fortran_ts, yes)],
+                   [AS_VAR_SET(fortran_ts, no)])],
+              [AS_VAR_SET(fortran_ts, no)])
+          cd ..
+          rm -rf conftest.$$])],
+         [AS_VAR_SET(fortran_ts, no)])
+
+    AS_VAR_IF(fortran_ts, [yes], [$1], [$2])
+    AS_VAR_POPDEF([fortran_ts])dnl
+])
diff --git a/config/ompi_fortran_find_module_include_flag.m4 b/config/ompi_fortran_find_module_include_flag.m4
index e6d513fe751..4a44bf12174 100644
--- a/config/ompi_fortran_find_module_include_flag.m4
+++ b/config/ompi_fortran_find_module_include_flag.m4
@@ -54,7 +54,7 @@ EOF
         cd ..
 
         #
-        # Now try to compile a simple program usinng that module, iterating
+        # Now try to compile a simple program using that module, iterating
         # through the possible flags that the compiler might use
         #
 
diff --git a/config/ompi_fortran_get_alignment.m4 b/config/ompi_fortran_get_alignment.m4
index 141e062d5d7..9ccca2a22fb 100644
--- a/config/ompi_fortran_get_alignment.m4
+++ b/config/ompi_fortran_get_alignment.m4
@@ -10,6 +10,8 @@ dnl Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
 dnl                         University of Stuttgart.  All rights reserved.
 dnl Copyright (c) 2004-2005 The Regents of the University of California.
 dnl                         All rights reserved.
+dnl Copyright (c) 2025      Research Organization for Information Science
+dnl                         and Technology (RIST).  All rights reserved.
 dnl Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
 dnl $COPYRIGHT$
 dnl
@@ -55,8 +57,7 @@ dnl
 # OMPI_FORTRAN_GET_ALIGNMENT(type, shell variable to set)
 # ----------------------------------------------------
 AC_DEFUN([OMPI_FORTRAN_GET_ALIGNMENT],[
-    unset happy
-    OPAL_VAR_SCOPE_PUSH([happy ompi_conftest_h])
+    OPAL_VAR_SCOPE_PUSH([fortran_get_alignment_happy ompi_conftest_h])
     # Use of m4_translit suggested by Eric Blake:
     # http://lists.gnu.org/archive/html/bug-autoconf/2010-10/msg00016.html
     AS_VAR_PUSHDEF([type_var],
@@ -112,9 +113,11 @@ EOF
 
         OPAL_LOG_COMMAND([$CC $CFLAGS -I. -c conftest.c],
             [OPAL_LOG_COMMAND([$FC $FCFLAGS conftestf.f conftest.o -o conftest $LDFLAGS $LIBS],
-                [happy="yes"], [happy="no"])], [happy="no"])
+                [fortran_get_alignment_happy="yes"],
+                [fortran_get_alignment_happy="no"])],
+            [fortran_get_alignment_happy="no"])
 
-        if test "$happy" = "no" ; then
+        if test "$fortran_get_alignment_happy" = "no" ; then
             AC_MSG_RESULT([Error!])
             AC_MSG_ERROR([Could not determine alignment of $1])
         fi
@@ -177,3 +180,41 @@ end program]])],
     AS_VAR_COPY([$2], [type_var])
     AS_VAR_POPDEF([type_var])dnl
 ])dnl
+
+# OMPI_FORTRAN_GET_COMMON_ALIGNMENT(variable to set)
+# ------------------------------------------
+AC_DEFUN([OMPI_FORTRAN_GET_COMMON_ALIGNMENT],[
+    AS_IF([test $OMPI_TRY_FORTRAN_BINDINGS -gt $OMPI_FORTRAN_NO_BINDINGS],
+          [AC_CACHE_CHECK([alignment of Fortran common], ompi_cv_fortran_common_alignment,
+              [AC_LANG_PUSH([Fortran])
+               AC_LINK_IFELSE([AC_LANG_SOURCE([[ program falignment
+   CHARACTER A,B
+   COMMON /AA/A
+   COMMON /BB/B
+   OPEN(UNIT=10, FILE="conftestval")
+   if (LOC(A) > LOC(B)) then
+      write (10,'(I5)') LOC(A)-LOC(B)
+   else
+      write (10,'(I5)') LOC(B)-LOC(A)
+   endif
+   CLOSE(10)
+
+end program]])],
+                          [AS_IF([test "$cross_compiling" = "yes"],
+                                 [AC_MSG_ERROR([Can not determine common alignment when cross-compiling])],
+                                 [OPAL_LOG_COMMAND([./conftest],
+                                                   [AS_VAR_SET(ompi_cv_fortran_common_alignment, [`cat conftestval`])],
+                                                   [AC_MSG_ERROR([Could not determine common alignment])])])],
+
+                          [AC_MSG_WARN([Could not determine common alignment])
+                           AC_MSG_WARN([See config.log for details])
+                           AC_MSG_ERROR([Cannot continue])])
+               rm -rf conftest* *.mod 2> /dev/null
+               AC_LANG_POP([Fortran])])
+
+           AS_VAR_COPY([$1], [ompi_cv_fortran_common_alignment])],
+          [AC_MSG_CHECKING([Fortran common alignment])
+           $1=0
+           AC_MSG_RESULT([skipped])])
+
+])dnl
diff --git a/config/ompi_fortran_get_value_true.m4 b/config/ompi_fortran_get_value_true.m4
index 2ca2cb78213..8e68a86bf6b 100644
--- a/config/ompi_fortran_get_value_true.m4
+++ b/config/ompi_fortran_get_value_true.m4
@@ -67,7 +67,7 @@ void ompi_print_f(ompi_fortran_logical_t * logical)
     if( SIZEOF_INT >= sizeof(ompi_fortran_logical_t) ) {
         fprintf(f, "%d\n", (int)*logical);
     } else if (SIZEOF_LONG >= sizeof(ompi_fortran_logical_t) ) {
-	fprintf(f, "%ld\n", (long) *logical);
+        fprintf(f, "%ld\n", (long) *logical);
     } else if (SIZEOF_LONG_LONG >= sizeof(ompi_fortran_logical_t) ) {
         fprintf(f, "%lld\n", (long long) *logical);
     } else {
diff --git a/config/ompi_interix.m4 b/config/ompi_interix.m4
index 7e4339a497b..2348128e769 100644
--- a/config/ompi_interix.m4
+++ b/config/ompi_interix.m4
@@ -3,6 +3,7 @@ dnl
 dnl Copyright (c)      2008 The University of Tennessee and The University
 dnl                         of Tennessee Research Foundation.  All rights
 dnl                         reserved.
+dnl Copyright (c) 2021 Cisco Systems, Inc.  All rights reserved.
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
@@ -24,10 +25,10 @@ dnl
 AC_DEFUN([OMPI_INTERIX],[
 
     AC_MSG_CHECKING(for Interix environment)
-    AC_TRY_COMPILE([],
+    AC_COMPILE_IFELSE([AC_LANG_PROGRAM([],
                    [#if !defined(__INTERIX)
                     #error Normal Unix environment
-                    #endif],
+                    #endif])],
                    is_interix=yes,
                    is_interix=no)
     AC_MSG_RESULT([$is_interix])
diff --git a/config/ompi_microsoft.m4 b/config/ompi_microsoft.m4
deleted file mode 100644
index b50db392c89..00000000000
--- a/config/ompi_microsoft.m4
+++ /dev/null
@@ -1,118 +0,0 @@
-dnl -*- shell-script -*-
-dnl
-dnl Copyright (c) 2004-2007 The University of Tennessee and The University
-dnl                         of Tennessee Research Foundation.  All rights
-dnl                         reserved.
-dnl $COPYRIGHT$
-dnl
-dnl Additional copyrights may follow
-dnl
-dnl $HEADER$
-dnl
-
-######################################################################
-#
-# OMPI_MICROSOFT_COMPILER
-#
-# Keep all the Windows checks in one place.
-#
-# USAGE:
-#   OMPI_MICROSOFT_COMPILER()
-#
-######################################################################
-AC_DEFUN([OMPI_MICROSOFT_COMPILER],[
-
-    # If we are using  one of the Microsoft compilers check that we are
-    # able to include windows.h. Most of the types that follow are defined
-    # in this file. If we check for it here it will get included in the
-    # default list of header files.
-    if test "x$opal_cv_c_compiler_vendor" = "xmicrosoft" ; then
-
-        opal_show_subtitle "Microsoft specific detection"
-
-        #
-        # These 2 libraries are a minimum ...
-        SAVE_LDFLAGS="$LDFLAGS"
-        LDFLAGS="$LDFLAGS Ws2_32.lib Advapi32.lib"
-
-        AC_CHECK_HEADERS([windows.h winsock2.h wdm.h])
-
-        # The atomic functions are defined in a very unuasual manner.
-        # Some of them are intrinsic defined in windows.h others are
-        # exported by kernel32.dll. If we force the usage of AC_TRY_RUN
-        # here we will check for both in same time: compilation and run.
-
-        AC_MSG_CHECKING(for working InterlockedCompareExchange)
-        AC_TRY_RUN( [#include <windows.h>
-                     int main() {
-                     LONG dest = 1, exchange = 0, comperand = 1;
-                     SetErrorMode(SEM_FAILCRITICALERRORS);
-                     InterlockedCompareExchange( &dest, exchange, comperand );
-                     return (int)dest;
-                    }],
-                    [AC_MSG_RESULT(yes)
-                     ompi_windows_have_support_for_32_bits_atomic=1],
-                    [AC_MSG_RESULT(no)
-                     ompi_windows_have_support_for_32_bits_atomic=0])
-        AC_DEFINE_UNQUOTED(HAVE_INTERLOCKEDCOMPAREEXCHANGE,
-                       $ompi_windows_have_support_for_32_bits_atomic,
-                       [Whether we support 32 bits atomic operations on Windows])
-
-        AC_MSG_CHECKING(for working InterlockedCompareExchangeAcquire)
-        AC_TRY_RUN( [#include <windows.h>
-                 int main() {
-                     LONG dest = 1, exchange = 0, comperand = 1;
-                     SetErrorMode(SEM_FAILCRITICALERRORS);
-                     InterlockedCompareExchangeAcquire( &dest, exchange, comperand );
-                     return (int)dest;
-                 }],
-                    [AC_MSG_RESULT(yes)
-                     ompi_windows_have_support_for_32_bits_atomic=1],
-                    [AC_MSG_RESULT(no)
-                     ompi_windows_have_support_for_32_bits_atomic=0])
-        AC_DEFINE_UNQUOTED(HAVE_INTERLOCKEDCOMPAREEXCHANGEACQUIRE,
-                       $ompi_windows_have_support_for_32_bits_atomic,
-                       [Whether we support 32 bits atomic operations on Windows])
-
-        AC_MSG_CHECKING(for working InterlockedCompareExchangeRelease)
-        AC_TRY_RUN( [#include <windows.h>
-                 int main() {
-                     LONG dest = 1, exchange = 0, comperand = 1;
-                     SetErrorMode(SEM_FAILCRITICALERRORS);
-                     InterlockedCompareExchangeRelease( &dest, exchange, comperand );
-                     return (int)dest;
-                 }],
-                    [AC_MSG_RESULT(yes)
-                     ompi_windows_have_support_for_32_bits_atomic=1],
-                    [AC_MSG_RESULT(no)
-                     ompi_windows_have_support_for_32_bits_atomic=0])
-        AC_DEFINE_UNQUOTED(HAVE_INTERLOCKEDCOMPAREEXCHANGERELEASE,
-                       $ompi_windows_have_support_for_32_bits_atomic,
-                       [Whether we support 32 bits atomic operations on Windows])
-
-        AC_MSG_CHECKING(for working InterlockedCompareExchange64)
-        AC_TRY_RUN( [#include <windows.h>
-                 int main() {
-                     LONGLONG dest = 1, exchange = 0, comperand = 1;
-                     SetErrorMode(SEM_FAILCRITICALERRORS);
-                     InterlockedCompareExchange64( &dest, exchange, comperand );
-                     return (int)dest;
-                 }],
-                    [AC_MSG_RESULT(yes)
-                     ompi_windows_have_support_for_64_bits_atomic=1],
-                    [AC_MSG_RESULT(no)
-                     ompi_windows_have_support_for_64_bits_atomic=0])
-        AC_DEFINE_UNQUOTED(HAVE_INTERLOCKEDCOMPAREEXCHANGE64,
-                       $ompi_windows_have_support_for_64_bits_atomic,
-                       [Whether we support 64 bits atomic operations on Windows])
-
-        opal_show_title "Windows Type tests"
-
-        AC_DEFINE([pid_t], [intptr_t], [Windows pid_t type is a pointer])
-        AC_DEFINE_UNQUOTED([SIZEOF_PID_T], $ac_cv_sizeof_int,
-                           [and here is it's size])
-
-        LDFLAGS="$SAVE_LDFLAGS"
-fi
-
-])
diff --git a/config/ompi_setup_contrib.m4 b/config/ompi_setup_contrib.m4
deleted file mode 100644
index 58c470ef131..00000000000
--- a/config/ompi_setup_contrib.m4
+++ /dev/null
@@ -1,29 +0,0 @@
-# -*- shell-script -*-
-#
-# Copyright (c) 2004-2009 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2007 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2006-2009 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2006-2008 Sun Microsystems, Inc.  All rights reserved.
-# Copyright (c) 2006-2007 Los Alamos National Security, LLC.  All rights
-#                         reserved.
-# Copyright (c) 2009      Oak Ridge National Labs.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-AC_DEFUN([OMPI_SETUP_CONTRIB],[
-    opal_show_title "Contributed software setup"
-
-    OMPI_CONTRIB
-])
diff --git a/config/ompi_setup_cxx.m4 b/config/ompi_setup_cxx.m4
index d1c05f03601..c51d21e8dcc 100644
--- a/config/ompi_setup_cxx.m4
+++ b/config/ompi_setup_cxx.m4
@@ -16,6 +16,7 @@ dnl Copyright (c) 2007-2009 Sun Microsystems, Inc.  All rights reserved.
 dnl Copyright (c) 2008-2020 Cisco Systems, Inc.  All rights reserved.
 dnl Copyright (c) 2015-2016 Research Organization for Information Science
 dnl                         and Technology (RIST). All rights reserved.
+dnl Copyright (c) 2021      Nanook Consulting.  All rights reserved.
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
@@ -29,11 +30,33 @@ AC_DEFUN([OMPI_SETUP_CXX_BANNER],[
 ])
 
 AC_DEFUN([OMPI_PROG_CXX],[
-    OPAL_VAR_SCOPE_PUSH([ompi_cxxflags_save])
+    OPAL_VAR_SCOPE_PUSH([ompi_cxxflags_save ompi_cxx_argv0])
+
     ompi_cxxflags_save="$CXXFLAGS"
     AC_PROG_CXX
-    AC_PROG_CXXCPP
     CXXFLAGS="$ompi_cxxflags_save"
+
+    # Note: according to the Autoconf docs, if no C++ compiler is
+    # found, $CXX is still set to "g++" (!!).  So make sure that we
+    # actually found a C++ compiler; if not, set CXX to "no", per
+    # thread at
+    # https://www.open-mpi.org/community/lists/users/2013/02/21356.php,
+    # which advises us to set Libtool precious variables to "no" if we
+    # don't want Libtool to setup that language at all.
+    set dummy $CXX
+    ompi_cxx_argv0=[$]2
+    OPAL_WHICH([$ompi_cxx_argv0], [OMPI_CXX_ABSOLUTE])
+    AS_IF([test "x$OMPI_CXX_ABSOLUTE" = "x"],
+          [CXX=no
+           OMPI_CXX_ABSOLUTE=no],
+          [ # If we did actually find a C++ compiler, find the C++ CPP
+           AC_PROG_CXXCPP])
+
+    AC_DEFINE_UNQUOTED(OMPI_CXX, "$CXX", [OMPI underlying C++ compiler])
+    AC_SUBST(OMPI_CXX_ABSOLUTE)
+
+    AM_CONDITIONAL([OMPI_HAVE_CXX_COMPILER], [test "$CXX" != "no"])
+
     OPAL_VAR_SCOPE_POP
 ])
 
@@ -44,8 +67,6 @@ dnl wrapper compiler (there is no C++ code in Open MPI, so we do not
 dnl need to setup for internal C++ compilations).  Safe to AC_REQUIRE
 dnl this macro.
 AC_DEFUN([OMPI_SETUP_CXX],[
-    OPAL_VAR_SCOPE_PUSH([ompi_cxx_argv0])
-
     # Do a little tomfoolery to get the subsection title printed first
     AC_REQUIRE([OMPI_SETUP_CXX_BANNER])
 
@@ -54,20 +75,11 @@ AC_DEFUN([OMPI_SETUP_CXX],[
     # see the docs for AC PROG_CC for details.
     AC_REQUIRE([OMPI_PROG_CXX])
 
-    BASECXX="`basename $CXX`"
-
-    AS_IF([test "x$CXX" = "x"], [CXX=none])
-    set dummy $CXX
-    ompi_cxx_argv0=[$]2
-    OPAL_WHICH([$ompi_cxx_argv0], [OMPI_CXX_ABSOLUTE])
-    AS_IF([test "x$OMPI_CXX_ABSOLUTE" = "x"], [OMPI_CXX_ABSOLUTE=none])
-
-    AC_DEFINE_UNQUOTED(OMPI_CXX, "$CXX", [OMPI underlying C++ compiler])
-    AC_SUBST(OMPI_CXX_ABSOLUTE)
-
-    # Make sure we can link with the C compiler
-    OPAL_LANG_LINK_WITH_C([C++], [],
-        [cat <<EOF >&2
+    # If we have a C++ compiler, do some additional tests
+    AS_IF([test "$CXX" != "no"],
+        [ # Make sure we can link with the C compiler
+         OPAL_LANG_LINK_WITH_C([C++], [],
+            [cat <<EOF >&2
 **********************************************************************
 * It appears that your C++ compiler is unable to link against object
 * files created by your C compiler.  This generally indicates either
@@ -78,13 +90,6 @@ AC_DEFUN([OMPI_SETUP_CXX],[
 * available in the config.log file in this directory.
 **********************************************************************
 EOF
-         AC_MSG_ERROR([C and C++ compilers are not link compatible.  Can not continue.])])
-
-    # bool type size and alignment
-    AC_LANG_PUSH(C++)
-    AC_CHECK_SIZEOF(bool)
-    OPAL_C_GET_ALIGNMENT(bool, OPAL_ALIGNMENT_CXX_BOOL)
-    AC_LANG_POP(C++)
-
-    OPAL_VAR_SCOPE_POP
+             AC_MSG_ERROR([C and C++ compilers are not link compatible.  Can not continue.])])
+        ])
 ])
diff --git a/config/ompi_setup_fc.m4 b/config/ompi_setup_fc.m4
index ab60092f674..157f5a6301d 100644
--- a/config/ompi_setup_fc.m4
+++ b/config/ompi_setup_fc.m4
@@ -43,16 +43,190 @@ AC_DEFUN_ONCE([_OMPI_SETUP_FC_COMPILER],[
     # Fortran compilers (excluding the f77 compiler names) from AC's
     # default list of compilers and use it here.  This is the main
     # reason we have an OMPI-ized version of the PROG_FC macro.
-    AC_PROG_FC([gfortran f95 fort xlf95 ifort ifc efc pgfortran pgf95 lf95 f90 xlf90 pgf90 epcf90 nagfor])
+    AC_PROG_FC([gfortran f95 fort xlf95 ifort ifc efc pgfortran pgf95 lf95 f90 xlf90 pgf90 epcf90 nagfor nvfortran])
     FCFLAGS="$ompi_fcflags_save"
     OPAL_VAR_SCOPE_POP
 ])
 
 #############################################################################
 
+dnl On macOS with Xcode, test whether -Wl,-commons,use_dylibs works
+dnl by itself or whether it also needs -Wl,-ld_classic.
+dnl
+dnl Backstory
+dnl
+dnl The history is that for a long time (decades),
+dnl -Wl,-commons,use_dylibs worked by itself.
+dnl
+dnl XCode 15 introduced a a new linker (either "the new linker" or
+dnl "ld_prime", according to
+dnl https://developer.apple.com/forums/thread/715385). The new linker
+dnl originally did not support "-commons use_dylibs", but Apple recently
+dnl added support for that feature to the new linker in the XCode 16
+dnl beta. "-ld_classic" forces using the old linker (which doesn't support
+dnl some other features that customers might like, but Open MPI doesn't
+dnl use for its Fortran bindings, like mergable libraries).
+dnl
+dnl Sidenode: Open MPI needs this "-commons use_dylibs" functionality
+dnl because Fortran sentinel values (e.g., MPI_BOTTOM) are implemented
+dnl with Fortran common blocks.
+dnl
+dnl So there's three cases:
+dnl
+dnl 1. Customer's default linker is the classic linker, which always
+dnl    supported "-commons use_dylibs".
+dnl 2. Customer's default linker is the new linker, but not new enough
+dnl    to support "-commons use_dylibs", so we need to force using the old
+dnl    linker via "-ld_classic".
+dnl 3. Customer's default linker is the new linker, new enough to support
+dnl    "-commons use_dylibs", so we do not want to force using the old
+dnl    linker.
+dnl
+dnl We have to use a slightly complex test code that will actually
+dnl fail if the version of Xcode being used requires "-ld_classic"
+dnl with "-commons,use_dylibs".
+dnl
+dnl 1. Build a shared library (with C source code) with a public
+dnl    symbol that can be used as a Fortran common block symbol.
+dnl 2. Compile a Fortran program that calls a function in the shared
+dnl    library, and link it against the shared library.
+dnl
+dnl Note: This is a linker test; we are checking to see if this all
+dnl compiles and links properly.  The logic in the C / Fortran code
+dnl below specifically does not test for correctness because we do not
+dnl actually run the code.
+AC_DEFUN([_OMPI_SETUP_FC_XCODE_COMMONS_LDFLAGS],[
+    OPAL_VAR_SCOPE_PUSH([xcode_flags])
+
+    # This variable is used by the invoking macro to display the
+    # results via AC RESULT (just to keep the symmetry of
+    # MSG_CHECKING / RESULT in the same upper-level macro).
+    OMPI_FORTRAN_WRAPPER_FLAGS=
+
+    xcode_flags="-Wl,-commons,use_dylibs"
+    _OMPI_SETUP_FC_XCODE_COMMONS_LDFLAGS_BACKEND(
+        [$xcode_flags],
+        [OMPI_FORTRAN_WRAPPER_FLAGS=$xcode_flags], [])
+    AS_IF([test -z "$OMPI_FORTRAN_WRAPPER_FLAGS"],
+          [xcode_flags="-Wl,-commons,use_dylibs -Wl,-ld_classic"
+           _OMPI_SETUP_FC_XCODE_COMMONS_LDFLAGS_BACKEND(
+               [$xcode_flags],
+               [OMPI_FORTRAN_WRAPPER_FLAGS=$xcode_flags], [])])
+    AS_IF([test -z "$OMPI_FORTRAN_WRAPPER_FLAGS"],
+          [OMPI_FORTRAN_WRAPPER_FLAGS="none"])
+
+    OPAL_VAR_SCOPE_POP
+])
+
+dnl Companion to _OMPI SETUP_FC_XCODE_COMMONS_LDFLAGS;
+dnl see that macro for an explanation of this macro.
+dnl
+dnl $1: LDFLAGS to test
+dnl $2: action to perform upon success
+dnl $3: action to perform upon failure
+AC_DEFUN([_OMPI_SETUP_FC_XCODE_COMMONS_LDFLAGS_BACKEND],[
+    OPAL_VAR_SCOPE_PUSH([xcode_happy xcode_dir LDFLAGS_save_xcode LIBS_save_xcode])
+
+    xcode_dir=conftest.$$
+    rm -rf $xcode_dir
+    mkdir -p $xcode_dir
+    cd $xcode_dir
+
+    LIBS_save_xcode=$LIBS
+    LDFLAGS_save_xcode=$LDFLAGS
+    LDFLAGS="$LDFLAGS -L. $1"
+
+    # Note: we use COMPILE_IFELSE and LANG_SOURCE below, which assume
+    # that confdefs.h exists.  This is being invoked extremely early
+    # in the configure sequence, so we haven't AC DEFINE'ed anything
+    # yet, and therefore confdefs.h won't be automatically created
+    # yet.  So we'll make an empty confdefs.h to avoid some error
+    # messages (it'll be removed with the whole tempdir, later).
+    touch confdefs.h
+
+    # Step 1: make a C library with some public symbols
+    xcode_happy=0
+    AC_LANG_PUSH([C])
+    AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
+/* Must end the symbol in _ (remember: we are specifically targeting
+   the MacOS compilation environment, so it is ok to target a specific
+   Fortran symbol convention), otherwise the Fortran linker will not
+   find it, and will just create a new Fortran symbol for it */
+int ompi_mpi_bottom_ = 42;
+
+void ompi_init_f(int *bogus);
+
+/* Empty / useless function that still ensures that this compilation
+   unit will not be optimized out */
+void ompi_init_f(int *bogus)
+{
+    *bogus = ompi_mpi_bottom_;
+}
+]])],
+        [ # If the above compiled successfully, Then use
+          # conftest.OBJEXT to make the library.  Note that
+          # conftest.OBJEXT will automatically be deleted upon exit of
+          # COMPILE_IFELSE.
+          #
+          # NOTE: this is pretty gross -- we're manually making a
+          # shared library.  But the libtool binary doesn't exist yet,
+          # so this is the best that we can do.
+         OPAL_LOG_COMMAND([$CC -dynamiclib -Wl,-undefined -Wl,dynamic_lookup $LDFLAGS conftest.$OBJEXT -o libconftest.dylib],
+                          [xcode_happy=1])])
+    AC_LANG_POP
+
+    # Now compile and link a Fortran program against this shared
+    # library.
+    AC_LANG_PUSH([Fortran])
+    AS_IF([test $xcode_happy -eq 1],
+          [LIBS="$LIBS -lconftest"
+           AC_LINK_IFELSE([AC_LANG_SOURCE([
+program test
+  integer :: mpi_bottom
+  common/ompi_mpi_bottom/mpi_bottom
+
+  interface
+     subroutine ompi_init(bogus) BIND(C, name="ompi_init_f")
+       implicit none
+       integer bogus
+     end subroutine ompi_init
+  end interface
+
+  integer bogus
+  call ompi_init(bogus)
+end program
+])],
+
+                          [],
+                          [xcode_happy=0])])
+    AC_LANG_POP
+
+    # Exit the temp dir
+    cd ..
+    rm -rf $xcode_dir
+
+    # LIBS was set specifically for the artificial conditions of this
+    # test, so reset it
+    LIBS=$LIBS_save_xcode
+
+    AS_IF([test $xcode_happy -eq 1],
+          [ # Restore LDFLAGS + the new flags (i.e., get rid of the
+            # "-L." we added for this test)
+           LDFLAGS="$LDFLAGS_save_xcode $1"
+           $2],
+          [ # If we failed the test, reset LDFLAGS back to its
+            # original value.
+           LDFLAGS=$LDFLAGS_save_xcode
+           $3])
+
+    OPAL_VAR_SCOPE_POP
+])
+
+#############################################################################
+
 # General Fortran compiler setup
 AC_DEFUN([OMPI_SETUP_FC],[
-    OPAL_VAR_SCOPE_PUSH([ompi_fc_happy LDFLAGS_save fc_version])
+    OPAL_VAR_SCOPE_PUSH([ompi_fc_happy LDFLAGS_save fc_version OMPI_FORTRAN_WRAPPER_FLAGS])
 
     # Force the intro banner to be displayed first
     AC_REQUIRE([_OMPI_SETUP_FC_BANNER])
@@ -61,17 +235,17 @@ AC_DEFUN([OMPI_SETUP_FC],[
     # Autoconf (version >= 2.64) docs.
     AC_REQUIRE([_OMPI_SETUP_FC_COMPILER])
 
+    # If $FC is "no", that's another way of the user telling us "I
+    # don't want any Fortran compiler".  That being said, there are
+    # already existing code paths that expect an empty $FC to mean "no
+    # Fortran compiler", so turn "no" into "" here.
+    AS_IF([test "$FC" = "no"], [FC=])
+
     AS_IF([test -z "$FC"],
           [AC_MSG_WARN([*** All Fortran MPI bindings disabled (could not find compiler)])
            ompi_fc_happy=0],
           [ompi_fc_happy=1])
 
-    AS_IF([test $ompi_fc_happy -eq 1 && test "$WANT_DEBUG" = "1" && test "$enable_debug_symbols" != "no"],
-          [FCFLAGS="$FCFLAGS -g"
-           OPAL_FLAGS_UNIQ(FCFLAGS)
-           AC_MSG_WARN([-g has been added to FCFLAGS (--enable-debug)])
-          ])
-
     # Make sure the compiler actually works, if not cross-compiling.
     # Don't just use the AC macro so that we can have a pretty
     # message.
@@ -81,7 +255,7 @@ AC_DEFUN([OMPI_SETUP_FC],[
 
     # OS X before 10.3 (deployment target) does not allow undefined common
     # symbols in shared libraries.  Because we can't figure out how to
-    # implement MPI_STATUSES_IGNORE and friends wihtout common symbols, on
+    # implement MPI_STATUSES_IGNORE and friends without common symbols, on
     # OS X we can't build the F90 bindings as a shared library.
     OMPI_FORTRAN_EXTRA_SHARED_LIBRARY_FLAGS=
     AS_IF([test $ompi_fc_happy -eq 1],
@@ -226,23 +400,12 @@ I = 3]])],
                  ])
           ])
 
-    # Per #1982, on OS X, we may need some esoteric linker flags in the
+    # Per Trac #1982, on OS X, we may need some esoteric linker flags in the
     # Fortran wrapper compiler.
     AC_MSG_CHECKING([to see if mpifort compiler needs additional linker flags])
     case "$host" in
     *apple-darwin*)
-        # Test whether -Wl,-commons,use_dylibs works; if it does, use it.
-        LDFLAGS_save=$LDFLAGS
-        LDFLAGS="$LDFLAGS -Wl,-commons,use_dylibs"
-        AC_LANG_PUSH([Fortran])
-        AC_LINK_IFELSE([AC_LANG_SOURCE([[program test
-    integer :: i
-end program]])],
-                       [OMPI_FORTRAN_WRAPPER_FLAGS="-Wl,-commons,use_dylibs"
-                        OPAL_WRAPPER_FLAGS_ADD([FCFLAGS], [$OMPI_FORTRAN_WRAPPER_FLAGS])],
-                       [OMPI_FORTRAN_WRAPPER_FLAGS=none])
-        AC_LANG_POP([Fortran])
-        LDFLAGS=$LDFLAGS_save
+        _OMPI_SETUP_FC_XCODE_COMMONS_LDFLAGS
         AC_MSG_RESULT([$OMPI_FORTRAN_WRAPPER_FLAGS])
         ;;
     *)
diff --git a/config/ompi_setup_java.m4 b/config/ompi_setup_java.m4
index b0212b230e7..8339fe3cccb 100644
--- a/config/ompi_setup_java.m4
+++ b/config/ompi_setup_java.m4
@@ -18,6 +18,7 @@ dnl Copyright (c) 2013      Intel, Inc.  All rights reserved.
 dnl Copyright (c) 2015-2018 Research Organization for Information Science
 dnl                         and Technology (RIST). All rights reserved.
 dnl Copyright (c) 2017      FUJITSU LIMITED.  All rights reserved.
+dnl Copyright (c) 2025      Nanook Consulting  All rights reserved.
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
@@ -101,7 +102,7 @@ AC_DEFUN([_OMPI_SETUP_JAVA],[
            if test "$ompi_java_found" = "0"; then
                # Various Linux
                if test -z "$JAVA_HOME"; then
-                   ompi_java_dir='/usr/lib/jvm/java-*-openjdk-*/include/'
+                   ompi_java_dir='/usr/lib/jvm/java-*-openjdk*/include/'
                else
                    ompi_java_dir=$JAVA_HOME/include
                fi
@@ -128,17 +129,6 @@ AC_DEFUN([_OMPI_SETUP_JAVA],[
                              [AC_MSG_RESULT([not found])])])
            fi
 
-           if test "$ompi_java_found" = "0"; then
-               # Solaris
-               ompi_java_dir=/usr/java
-               AC_MSG_CHECKING([for Java in Solaris locations])
-               AS_IF([test -d $ompi_java_dir && test -r "$ompi_java_dir/include/jni.h"],
-                     [AC_MSG_RESULT([found ($ompi_java_dir)])
-                      with_jdk_headers=$ompi_java_dir/include
-                      with_jdk_bindir=$ompi_java_dir/bin
-                      ompi_java_found=1],
-                     [AC_MSG_RESULT([not found])])
-           fi
           ],
           [ompi_java_found=1])
 
@@ -188,11 +178,6 @@ EOF
                       # too.  Ugh.
                       AS_IF([test -d "$with_jdk_headers/linux"],
                             [OMPI_JDK_CPPFLAGS="$OMPI_JDK_CPPFLAGS -I$with_jdk_headers/linux"])
-                      # Solaris JDK also require -I<blah>/solaris.
-                      # See if that's there, and if so, add a -I for that,
-                      # too.  Ugh.
-                      AS_IF([test -d "$with_jdk_headers/solaris"],
-                            [OMPI_JDK_CPPFLAGS="$OMPI_JDK_CPPFLAGS -I$with_jdk_headers/solaris"])
                       # Darwin JDK also require -I<blah>/darwin.
                       # See if that's there, and if so, add a -I for that,
                       # too.  Ugh.
diff --git a/config/ompi_setup_mpi_fortran.m4 b/config/ompi_setup_mpi_fortran.m4
index fbfd9c5fcf1..45a8db19317 100644
--- a/config/ompi_setup_mpi_fortran.m4
+++ b/config/ompi_setup_mpi_fortran.m4
@@ -15,10 +15,12 @@ dnl Copyright (c) 2006-2008 Sun Microsystems, Inc.  All rights reserved.
 dnl Copyright (c) 2006-2007 Los Alamos National Security, LLC.  All rights
 dnl                         reserved.
 dnl Copyright (c) 2009      Oak Ridge National Labs.  All rights reserved.
-dnl Copyright (c) 2014-2020 Research Organization for Information Science
+dnl Copyright (c) 2014-2025 Research Organization for Information Science
 dnl                         and Technology (RIST).  All rights reserved.
-dnl Copyright (c) 2016      IBM Corporation.  All rights reserved.
+dnl Copyright (c) 2016-2022 IBM Corporation.  All rights reserved.
 dnl Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+dnl Copyright (c) 2022      Triad National Security, LLC. All rights
+dnl                         reserved.
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
@@ -97,6 +99,8 @@ AC_DEFUN([OMPI_SETUP_MPI_FORTRAN],[
     AS_IF([test $OMPI_TRY_FORTRAN_BINDINGS -gt $OMPI_FORTRAN_NO_BINDINGS],
           [OMPI_SETUP_FC([ompi_fortran_happy=1])])
 
+    AM_CONDITIONAL([OMPI_HAVE_FORTRAN_COMPILER], [test -n "$FC"])
+
     # These values will be determined by SETUP_FC.  We must always
     # AC_DEFINE these results, even in the --disable-mpi-fortran case,
     # for ompi_info.
@@ -134,10 +138,8 @@ AC_DEFUN([OMPI_SETUP_MPI_FORTRAN],[
     AC_DEFINE([ompi_fortran_bogus_type_t], [int],
               [A bogus type that allows us to have sentinel type values that are still valid])
 
-    # These get filled in as we check for each type
-    OMPI_FORTRAN_IKINDS=
-    OMPI_FORTRAN_RKINDS=
-    OMPI_FORTRAN_CKINDS=
+    OMPI_FORTRAN_GET_COMMON_ALIGNMENT([OMPI_FORTRAN_COMMON_ALIGNMENT])
+    AC_SUBST([OMPI_FORTRAN_COMMON_ALIGNMENT])
 
     # We want to set the #define's for all of these, so invoke the macros
     # regardless of whether we have F77 support or not.
@@ -280,14 +282,6 @@ AC_DEFUN([OMPI_SETUP_MPI_FORTRAN],[
                        [$OMPI_FORTRAN_STATUS_SIZE],
                        [The number or Fortran INTEGER in MPI Status])
 
-    # Setup for the compilers that don't support ignore TKR functionality
-    OPAL_UNIQ(OMPI_FORTRAN_IKINDS)
-    AC_SUBST(OMPI_FORTRAN_IKINDS)
-    OPAL_UNIQ(OMPI_FORTRAN_RKINDS)
-    AC_SUBST(OMPI_FORTRAN_RKINDS)
-    OPAL_UNIQ(OMPI_FORTRAN_CKINDS)
-    AC_SUBST(OMPI_FORTRAN_CKINDS)
-
     # We can't use C_INTxx_T KIND values in mpif.h because many
     # existing MPI Fortran applications are of the form:
     #
@@ -297,12 +291,44 @@ AC_DEFUN([OMPI_SETUP_MPI_FORTRAN],[
     #
     # ...and you can't have a "use..." statement before that (to get
     # the Fortran/C interop C_INTxx_T KIND values).  So figure out
-    # those KIND values here and just substitue them in via
+    # those KIND values here and just substitute them in via
     # AC_DEFINE's.  Kinda gross, but there you are.  :-\
     OMPI_FORTRAN_GET_KIND_VALUE([C_INT16_T], 4, [OMPI_FORTRAN_C_INT16_T_KIND])
     OMPI_FORTRAN_GET_KIND_VALUE([C_INT32_T], 9, [OMPI_FORTRAN_C_INT32_T_KIND])
     OMPI_FORTRAN_GET_KIND_VALUE([C_INT64_T], 18, [OMPI_FORTRAN_C_INT64_T_KIND])
 
+    #
+    # See if "attributes deprecated" is supported
+    # - False positives were seen with compilers like XL which will ignore
+    #   this option by default, but if the user specifies some options to
+    #   the wrapper compiler later it will fail to recognize the option.
+    # - For now just limit this check to compilers that we know will work
+    # This directive is only recognized, and works, for gfortran 11.0 and
+    # later. As a result, this directive is generated only if the
+    # Fortran compiler building Open MPI is gfortran 11.0 or later.
+    OMPI_FORTRAN_HAVE_ATTR_DEPRECATED=0
+    AS_IF([test $ompi_fortran_happy -eq 1],
+          [AC_MSG_CHECKING([if Fortran compiler supports the deprecated attribute])
+           AS_IF([test "$opal_cv_c_compiler_vendor" = "gnu"],
+                 [AC_LANG_PUSH([Fortran])
+                  AC_COMPILE_IFELSE([AC_LANG_SOURCE([[program check_for_attr_deprecated
+!GCC$ ATTRIBUTES DEPRECATED :: x
+  real*4 x
+  x = 123.4
+  print *,x
+end program]])],
+                                    [OMPI_FORTRAN_HAVE_ATTR_DEPRECATED=1
+                                     AC_MSG_RESULT([Yes])],
+                                    [AC_MSG_RESULT([No])])
+                  AC_LANG_POP([Fortran])],
+                 [AC_MSG_RESULT([Unknown])])
+          ])
+    AC_SUBST(OMPI_FORTRAN_HAVE_ATTR_DEPRECATED)
+    AC_DEFINE_UNQUOTED(OMPI_FORTRAN_HAVE_ATTR_DEPRECATED,
+                       $OMPI_FORTRAN_HAVE_ATTR_DEPRECATED,
+                       [Whether the compiler supports Fortran ATTRIBUTES DEPRECATED or not])
+
+
     #--------------------------------------------------------
     # Fortran mpif.h MPI bindings
     #--------------------------------------------------------
@@ -328,6 +354,13 @@ AC_DEFUN([OMPI_SETUP_MPI_FORTRAN],[
                [OMPI_FORTRAN_HAVE_ISO_FORTRAN_ENV=0])])
     AC_SUBST(OMPI_FORTRAN_HAVE_ISO_FORTRAN_ENV)
 
+    # The non standard iso_fortran_env:real16 can be used for MPI_SIZEOF
+    OMPI_FORTRAN_HAVE_ISO_FORTRAN_ENV_REAL16=0
+    AS_IF([test $ompi_fortran_happy -eq 1],
+          [OMPI_FORTRAN_CHECK_ISO_FORTRAN_ENV_REAL16(
+               [OMPI_FORTRAN_HAVE_ISO_FORTRAN_ENV_REAL16=1])])
+    AC_SUBST(OMPI_FORTRAN_HAVE_ISO_FORTRAN_ENV_REAL16)
+
     # Ensure that the fortran compiler supports STORAGE_SIZE for
     # enough relevant types.
     AS_IF([test $ompi_fortran_happy -eq 1],
@@ -362,16 +395,15 @@ AC_DEFUN([OMPI_SETUP_MPI_FORTRAN],[
           ])
 
     # If we got here, we can build the mpi module if it was requested.
-    # Decide whether to build the ignore TKR version or the
-    # non-ignore-TKR/legacy version.
+    # We only support compilers which have an ignore TKR feature.
     AS_IF([test $OMPI_TRY_FORTRAN_BINDINGS -ge $OMPI_FORTRAN_USEMPI_BINDINGS && \
            test $ompi_fortran_happy -eq 1],
           [OMPI_BUILD_FORTRAN_BINDINGS=$OMPI_FORTRAN_USEMPI_BINDINGS
            AS_IF([test $OMPI_FORTRAN_HAVE_IGNORE_TKR -eq 1],
                  [OMPI_FORTRAN_USEMPI_DIR=mpi/fortran/use-mpi-ignore-tkr
                   OMPI_FORTRAN_USEMPI_LIB=-l${with_libmpi_name}_usempi_ignore_tkr],
-                 [OMPI_FORTRAN_USEMPI_DIR=mpi/fortran/use-mpi-tkr
-                  OMPI_FORTRAN_USEMPI_LIB=-l${with_libmpi_name}_usempi])
+                 [AC_MSG_WARN([** Fortran compiler does not support ignoring TKR.  Please use a newer fortran compiler.])
+                  AC_MSG_ERROR([*** Cannot continue])])
           ])
 
     OMPI_FORTRAN_HAVE_ISO_C_BINDING=0
@@ -406,14 +438,27 @@ AC_DEFUN([OMPI_SETUP_MPI_FORTRAN],[
     # If we got all the stuff from above, then also look for the new
     # F08 syntax that we can use for the use_mpif08 module.
 
-    # We need to have ignore TKR functionality to build the mpi_f08
+    OMPI_FORTRAN_HAVE_TS=0
+    OMPI_MPI_SUBARRAYS_SUPPORTED=.false.
+    OMPI_MPI_ASYNC_PROTECTS_NONBLOCKING=.false.
+    AS_IF([test $OMPI_TRY_FORTRAN_BINDINGS -ge $OMPI_FORTRAN_USEMPIF08_BINDINGS],
+          [OMPI_FORTRAN_CHECK_TS([OMPI_FORTRAN_HAVE_TS=1])])
+
+    # We need to have ignore TKR or the ISO Fortran bindings functionality to build the mpi_f08
     # module
-    AS_IF([test $OMPI_TRY_FORTRAN_BINDINGS -ge $OMPI_FORTRAN_USEMPIF08_BINDINGS && \
-           test $OMPI_FORTRAN_HAVE_IGNORE_TKR -eq 1],
-          [OMPI_BUILD_FORTRAN_BINDINGS=$OMPI_FORTRAN_USEMPIF08_BINDINGS
-           OMPI_FORTRAN_F08_PREDECL=$OMPI_FORTRAN_IGNORE_TKR_PREDECL
-           OMPI_FORTRAN_F08_TYPE=$OMPI_FORTRAN_IGNORE_TKR_TYPE
-          ])
+    AS_IF([test $OMPI_TRY_FORTRAN_BINDINGS -ge $OMPI_FORTRAN_USEMPIF08_BINDINGS],
+          [AS_IF([test $OMPI_FORTRAN_HAVE_IGNORE_TKR -eq 1],
+                 [OMPI_BUILD_FORTRAN_BINDINGS=$OMPI_FORTRAN_USEMPIF08_BINDINGS
+                  OMPI_FORTRAN_F08_PREDECL=$OMPI_FORTRAN_IGNORE_TKR_PREDECL
+                  OMPI_FORTRAN_F08_TYPE=$OMPI_FORTRAN_IGNORE_TKR_TYPE
+                 ])
+           AS_IF([test $OMPI_FORTRAN_HAVE_TS -eq 1],
+                 [OMPI_BUILD_FORTRAN_BINDINGS=$OMPI_FORTRAN_USEMPIF08_BINDINGS
+                  OMPI_MPI_SUBARRAYS_SUPPORTED=.true.
+                  OMPI_MPI_ASYNC_PROTECTS_NONBLOCKING=.true.])])
+
+    AC_SUBST(OMPI_MPI_SUBARRAYS_SUPPORTED)
+    AC_SUBST(OMPI_MPI_ASYNC_PROTECTS_NONBLOCKING)
 
     # The overall "_BIND_C" variable will be set to 1 if we have all
     # the necessary forms of BIND(C)
@@ -547,8 +592,6 @@ end type test_mpi_handle],
           ])
 
     OMPI_FORTRAN_NEED_WRAPPER_ROUTINES=1
-    OMPI_FORTRAN_F08_PREDECL='!'
-    OMPI_FORTRAN_F08_TYPE=real
     OMPI_FORTRAN_HAVE_F08_ASSUMED_RANK=0
     AS_IF([test $OMPI_TRY_FORTRAN_BINDINGS -ge $OMPI_FORTRAN_USEMPIF08_BINDINGS && \
            test $OMPI_BUILD_FORTRAN_BINDINGS -ge $OMPI_FORTRAN_USEMPIF08_BINDINGS],
@@ -556,8 +599,6 @@ end type test_mpi_handle],
            OMPI_FORTRAN_CHECK_F08_ASSUMED_RANK(
                [ # If we have assumed rank, we can build the use
                  # mpi_f08 module "better"
-                OMPI_FORTRAN_F08_PREDECL='!'
-                OMPI_FORTRAN_F08_TYPE='type(*), dimension(..)'
                 OMPI_FORTRAN_HAVE_F08_ASSUMED_RANK=1])
 
                # Which mpi_f08 implementation are we using?
@@ -578,6 +619,21 @@ end type test_mpi_handle],
                      ])
           ])
 
+    OMPI_FORTRAN_HAVE_ELEMENTAL=0
+    AS_IF([test $OMPI_TRY_FORTRAN_BINDINGS -ge $OMPI_FORTRAN_USEMPIF08_BINDINGS && \
+           test $OMPI_BUILD_FORTRAN_BINDINGS -ge $OMPI_FORTRAN_USEMPIF08_BINDINGS],
+          [ # Does the compiler support "elemental"
+           OMPI_FORTRAN_CHECK_ELEMENTAL(
+               [OMPI_FORTRAN_ELEMENTAL_TYPE="ELEMENTAL"],
+               [OMPI_FORTRAN_ELEMENTAL_TYPE=])])
+    AC_SUBST(OMPI_FORTRAN_ELEMENTAL_TYPE)
+
+    OMPI_FORTRAN_HAVE_C_ISO_FORTRAN=0
+    AS_IF([test $OMPI_TRY_FORTRAN_BINDINGS -ge $OMPI_FORTRAN_USEMPIF08_BINDINGS && \
+           test $OMPI_BUILD_FORTRAN_BINDINGS -ge $OMPI_FORTRAN_USEMPIF08_BINDINGS],
+          [OMPI_FORTRAN_CHECK_TS([OMPI_FORTRAN_HAVE_TS=1],
+                                 [OMPI_FORTRAN_HAVE_TS=0])])
+
     # Note: the current implementation *only* has wrappers;
     # there is no optimized implementation for a "good"
     # compiler.  I'm leaving the above logic in place for
@@ -624,7 +680,7 @@ end type test_mpi_handle],
     AC_SUBST(OMPI_FC_ABSOLUTE)
     AC_DEFINE_UNQUOTED(OMPI_FC, ["$OMPI_FC"], [Underlying Fortran compiler])
     AC_DEFINE_UNQUOTED(OMPI_FC_ABSOLUTE, ["$OMPI_FC_ABSOLUTE"],
-                       [Absolutey path to the underlying Fortran compiler found by configure])
+                       [Absolute path to the underlying Fortran compiler found by configure])
 
     # These go into ompi/info/param.c
     AC_DEFINE_UNQUOTED([OMPI_FORTRAN_BUILD_SIZEOF],
@@ -640,6 +696,18 @@ end type test_mpi_handle],
                        [$OMPI_FORTRAN_HAVE_STORAGE_SIZE],
                        [Whether the compiler supports STORAGE_SIZE on relevant types])
 
+    # This token is used in the mpifort wrapper compiler data file.
+    # If we are building the Fortran bindings, then include
+    # -lompi_mpifh in the link line.  If we're not building the
+    # Fortran bindings, then do not include that token in the link
+    # line (because we'll still install mpifort to link Fortran
+    # applications with the C bindings, even if the Fortran MPI
+    # bindings are not being built).
+    AS_IF([test $OMPI_BUILD_FORTRAN_BINDINGS -gt $OMPI_FORTRAN_NO_BINDINGS],
+          [OMPI_FORTRAN_MPIFH_LINK=-l${OMPI_LIBMPI_NAME}_mpifh],
+          [OMPI_FORTRAN_MPIFH_LINK=])
+    AC_SUBST(OMPI_FORTRAN_MPIFH_LINK)
+
     # This conditional is used to determine whether we compile the
     # various .f90 files that contain MPI_SIZEOF implementations.
     AM_CONDITIONAL([BUILD_FORTRAN_SIZEOF],
@@ -728,10 +796,9 @@ end type test_mpi_handle],
     # This goes into mpifort-wrapper-data.txt
     AC_SUBST(OMPI_FORTRAN_USEMPIF08_LIB)
 
-    # These go into interfaces/mpi-f08-interfaces-[no]bind.h (and
-    # mpi-f*-interfaces*.h files)
-    AC_SUBST(OMPI_FORTRAN_F08_PREDECL)
-    AC_SUBST(OMPI_FORTRAN_F08_TYPE)
+    # These go into mod/mpi-f08-interfaces.h
+    AC_SUBST(OMPI_F08_IGNORE_TKR_PREDECL)
+    AC_SUBST(OMPI_F08_IGNORE_TKR_TYPE)
 
     AC_SUBST(OMPI_MPI_PREFIX)
     AC_SUBST(OMPI_MPI_BIND_PREFIX)
@@ -813,6 +880,25 @@ end type test_mpi_handle],
     # For configure-fortran-output.h
     AC_SUBST(OMPI_FORTRAN_HAVE_BIND_C)
 
+    AM_CONDITIONAL(OMPI_FORTRAN_HAVE_TS,
+                   [test $OMPI_FORTRAN_HAVE_TS -eq 1])
+    AC_SUBST(OMPI_FORTRAN_HAVE_TS)
+    AC_DEFINE_UNQUOTED([OMPI_FORTRAN_HAVE_TS],
+                       [$OMPI_FORTRAN_HAVE_TS],
+                       [For ompi/mpi/fortran/use-mpi-f08/base/ts.*: whether the compiler supports TS 29113 or not])
+
+    AS_IF([test $OMPI_FORTRAN_HAVE_TS -eq 1],
+          [OMPI_F08_IGNORE_TKR_TYPE="type(*), dimension(..)"
+           OMPI_F08_IGNORE_TKR_PREDECL="no attribute required for"
+           OMPI_F08_BINDINGS_EXTENSION="ts"
+           OMPI_F08_BINDINGS_TS_SUFFIX="ts"],
+          [OMPI_F08_IGNORE_TKR_TYPE=$OMPI_FORTRAN_IGNORE_TKR_TYPE
+           OMPI_F08_IGNORE_TKR_PREDECL=${OMPI_FORTRAN_IGNORE_TKR_PREDECL:1}
+           OMPI_F08_BINDINGS_EXTENSION="f"
+           OMPI_F08_BINDINGS_TS_SUFFIX=""])
+    AC_SUBST(OMPI_F08_BINDINGS_EXTENSION)
+    AC_SUBST(OMPI_F08_BINDINGS_TS_SUFFIX)
+
     # Somewhat redundant because ompi/Makefile.am won't traverse into
     # ompi/mpi/fortran/use-mpi-f08 if it's not to be built, but we
     # might as well have ompi/mpi/fortran/use-mpi-f08/Makefile.am be
@@ -823,4 +909,9 @@ end type test_mpi_handle],
     AC_DEFINE_UNQUOTED(OMPI_BUILD_FORTRAN_BINDINGS,
                        $OMPI_BUILD_FORTRAN_BINDINGS,
                        [The level of fortran bindings to be built])
+
+    # This is an overall "are we building any Fortran MPI bindings"
+    # conditional
+    AM_CONDITIONAL([OMPI_BUILD_ANY_FORTRAN_BINDINGS],
+                   [test $OMPI_BUILD_FORTRAN_BINDINGS -gt 0])
 ])
diff --git a/config/ompi_setup_prrte.m4 b/config/ompi_setup_prrte.m4
index fe6eb17308a..28140bad066 100644
--- a/config/ompi_setup_prrte.m4
+++ b/config/ompi_setup_prrte.m4
@@ -16,10 +16,11 @@ dnl Copyright (c) 2006-2007 Los Alamos National Security, LLC.  All rights
 dnl                         reserved.
 dnl Copyright (c) 2009      Oak Ridge National Labs.  All rights reserved.
 dnl Copyright (c) 2019-2020 Intel, Inc.  All rights reserved.
-dnl Copyright (c) 2020      Amazon.com, Inc. or its affiliates.
-dnl                         All Rights reserved.
+dnl Copyright (c) 2020-2022 Amazon.com, Inc. or its affiliates.  All Rights reserved.
 dnl Copyright (c) 2021      Nanook Consulting.  All rights reserved.
-dnl Copyright (c) 2021      IBM Corporation.  All rights reserved.
+dnl Copyright (c) 2021-2022 IBM Corporation.  All rights reserved.
+dnl Copyright (c) 2023-2024 Jeffrey M. Squyres.  All rights reserved.
+dnl Copyright (c) 2025      Advanced Micro Devices, Inc. All rights reserved.
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
@@ -31,63 +32,61 @@ dnl Check for / configure PRRTE package.  Prefer finding an external
 dnl PRRTE, build our internal one if required.  If we can not find an
 dnl external PRRTE and the internal one fails to configure, abort.
 dnl
-dnl This macro wil change the environment in the following way:
-dnl
-dnl * PRTE_PATH will be AC_SUBST'ed to the full path (minus the EXE
-dnl   extension) of the prte binary.
+dnl This macro will change the environment in the following way:
 dnl
 dnl A Makefile conditional OMPI_WANT_PRRTE will be defined based on the
 dnl results of the build.
 AC_DEFUN([OMPI_SETUP_PRRTE],[
-    OPAL_VAR_SCOPE_PUSH([prrte_setup_internal_happy prrte_setup_external_happy prrte_setup_success_var])
+    AC_REQUIRE([AC_PROG_LN_S])
+
+OPAL_VAR_SCOPE_PUSH([prrte_setup_internal_happy prrte_setup_external_happy target_rst_dir])
 
     opal_show_subtitle "Configuring PRRTE"
 
-    # Don't use OPAL_3RDPARTY_WITH because it will not allow packages
-    # to be disabled
-    m4_ifdef([package_prrte],
-        [AC_ARG_WITH([prrte],
-            [AS_HELP_STRING([--with-prrte(=DIR)],
-                           [Build PRTE support.  DIR can take one of four values: "internal", "external", "no", or a valid directory name.  "internal" forces Open MPI to use its internal copy of PRRTE.  "external" forces Open MPI to use an external installation of PRRTE.  Supplying a valid directory name also forces Open MPI to use an external installation of PRRTE, and adds DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries. If no argument is specified, Open MPI will search default locations for PRRTE and fall back to an internal version if one is not found.])])],
-        [AC_ARG_WITH([prrte],
-            [AS_HELP_STRING([--with-prrte(=DIR)],
-                           [Build PRRTE support.  DIR can take one of three values:  "external", "no", or a valid directory name.  "external" forces Open MPI to use an external installation of PRRTE.  Supplying a valid directory name also forces Open MPI to use an external installation of PRRTE, and adds DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries. If no argument is specified, Open MPI will search default locations for PRRTE and disable creating mpirun symlinks if one is not found.])])])
+    # We *must* have setup Sphinx before invoking this macro (i.e., it
+    # is a programming error -- not a run-time error -- if Sphinx was
+    # not previously setup).
+    OAC_ASSERT_BEFORE([OAC_SETUP_SPHINX], [OMPI_SETUP_PRRTE])
 
-    m4_ifdef([package_prrte],
-         [OMPI_PRRTE_ADD_ARGS])
-
-    # clean up $with_prrte so that it contains only a path or empty
-    # string.  To determine internal or external preferences, use
-    # $opal_prrte_mode.
-    AS_IF([test "$with_prrte" = "yes"], [with_prrte=])
-    AS_CASE([$with_prrte],
-            ["internal"], [with_prrte=""
-                           opal_prrte_mode="internal"],
-            ["external"], [with_prrte=""
-                           opal_prrte_mode="external"],
-            [""], [opal_prrte_mode="unspecified"],
-            ["no"], [opal_prrte_mode="disabled"],
-            [opal_prrte_mode="external"])
-
-    echo "with_prrte: $with_prrte"
-    echo "opal_prrte_mode: $opal_prrte_mode"
-
-    m4_ifdef([package_prrte], [],
-             [AS_IF([test "$opal_prrte_mode" = "internal"],
-                    [AC_MSG_WARN([Invalid argument to --with-prrte: internal.])
-                     AC_MSG_ERROR([Cannot continue])])])
+    # These are sym links to folders with PRRTE's RST files that we'll
+    # slurp into mpirun.1.rst.  We'll remove these links (or even
+    # accidental full copies) now and replace them with new links to
+    # the PRRTE that we find, below.
+    target_rst_dir="$OMPI_TOP_BUILDDIR/docs"
+    rm -rf "$target_rst_dir/prrte-rst-content"
+    rm -rf "$target_rst_dir/schizo-ompi-rst-content"
+
+    OPAL_3RDPARTY_WITH([prrte], [prrte], [package_prrte], [1])
+
+    AC_ARG_WITH([prrte-bindir],
+       [AS_HELP_STRING([--with-prrte-bindir=DIR],
+           [Search for PRRTE binaries in DIR.  Defaults to PRRTE_DIR/bin if not specified])])
 
     prrte_setup_internal_happy=0
-    m4_ifdef([package_prrte], [
-         # always configure the internal prrte, so that
-         # make dist always works.
-         AS_IF([test "$opal_prrte_mode" = "disabled"], [prrte_setup_success_var=0], [prrte_setup_success_var=1])
-         _OMPI_SETUP_PRRTE_INTERNAL([prrte_setup_internal_happy=$prrte_setup_success_var])])
+    m4_ifdef([package_prrte],
+        [OMPI_PRRTE_ADD_ARGS
+         AS_IF([test "$opal_prrte_mode" = "unspecified" -o "$opal_prrte_mode" = "internal"],
+               [# Run PRRTE's configure script unless the user
+                # explicitly asked us to use an external PMIX, so that
+                # "make dist" includes PRRTE in the dist tarball.  This
+                # does mean that "make dist" will not work if Open MPI
+                # was configured to use an external PRRTE library, but
+                # we decided this was a reasonable tradeoff for not
+                # having to deal with PRRTE (or PMIx) potentially
+                # failing to configure in a situation where it isn't
+                # desired.
+                _OMPI_SETUP_PRRTE_INTERNAL([prrte_setup_internal_happy=1],
+                                           [prrte_setup_internal_happy=0])])
+
+         # if we have a pmix package and configure did not complete
+         # successfully (or wasn't started), then disable make dist.
+         AS_IF([test $prrte_setup_internal_happy != 1],
+               [OPAL_MAKEDIST_DISABLE="$OPAL_MAKEDIST_DISABLE PRRTE"])])
 
     # unless internal specifically requested by the user, try to find
     # an external that works.
     prrte_setup_external_happy=0
-    AS_IF([test "$opal_prrte_mode" != "internal" -o "$opal_prrte_mode" != "disabled"],
+    AS_IF([test "$opal_prrte_mode" != "internal" -a "$opal_prrte_mode" != "disabled"],
           [_OMPI_SETUP_PRRTE_EXTERNAL(
               [prrte_setup_external_happy=1
                opal_prrte_mode="external"],
@@ -98,7 +97,9 @@ AC_DEFUN([OMPI_SETUP_PRRTE],[
     # so try the internal version.
     AS_IF([test "$prrte_setup_external_happy" = "0" -a "$prrte_setup_internal_happy" = "1"],
           [opal_prrte_mode="internal"
-           _OMPI_SETUP_PRRTE_INTERNAL_POST()])
+           OMPI_USING_INTERNAL_PRRTE=1
+           _OMPI_SETUP_PRRTE_INTERNAL_POST()],
+          [OMPI_USING_INTERNAL_PRRTE=0])
 
     AS_IF([test "$opal_prrte_mode" != "disabled"],
           [AS_IF([test "$prrte_setup_external_happy" = "0" -a "$prrte_setup_internal_happy" = "0"],
@@ -106,8 +107,6 @@ AC_DEFUN([OMPI_SETUP_PRRTE],[
            OMPI_HAVE_PRRTE=1],
           [OMPI_HAVE_PRRTE=0])
 
-    AC_SUBST([PRTE_PATH])
-
     AM_CONDITIONAL([OMPI_WANT_PRRTE],
            [test "$prrte_setup_internal_happy" = "1" -o "$prrte_setup_external_happy" = "1"])
 
@@ -115,17 +114,24 @@ AC_DEFUN([OMPI_SETUP_PRRTE],[
                        [$OMPI_HAVE_PRRTE],
                        [Whether or not PRRTE is available])
 
-    OPAL_SUMMARY_ADD([[Miscellaneous]], [[prrte]], [prrte], [$opal_prrte_mode])
+    AC_DEFINE_UNQUOTED([OMPI_USING_INTERNAL_PRRTE],
+                       [$OMPI_USING_INTERNAL_PRRTE],
+                       [Whether or not we are using the internal PRRTE])
+
+    AC_SUBST(OMPI_PRRTE_RST_CONTENT_DIR)
+    AC_SUBST(OMPI_SCHIZO_OMPI_RST_CONTENT_DIR)
+    AM_CONDITIONAL(OMPI_HAVE_PRRTE_RST, [test $OMPI_HAVE_PRRTE_RST -eq 1])
+
+    OPAL_SUMMARY_ADD([Miscellaneous], [PRRTE], [], [$opal_prrte_mode])
 
     OPAL_VAR_SCOPE_POP
 ])
 
-
 dnl _OMPI_SETUP_PRRTE_INTERNAL([action-if-success], [action-if-not-success])
 dnl
 dnl Attempt to configure the built-in PRRTE.
 AC_DEFUN([_OMPI_SETUP_PRRTE_INTERNAL], [
-    OPAL_VAR_SCOPE_PUSH([internal_prrte_args internal_prrte_extra_libs internal_prrte_happy deprecated_prefix_by_default print_prrte_warning internal_prrte_CPPFLAGS])
+    OPAL_VAR_SCOPE_PUSH([internal_prrte_args internal_prrte_happy deprecated_prefix_by_default print_prrte_warning internal_prrte_CPPFLAGS opal_prrte_CPPFLAGS_save])
 
     # This is really a PRTE option that should not be in Open MPI, but
     # there is not a great way to support the orterun/mpirun checks
@@ -159,7 +165,8 @@ AC_DEFUN([_OMPI_SETUP_PRRTE_INTERNAL], [
 
     internal_prrte_CPPFLAGS=
     internal_prrte_args="--with-proxy-version-string=$OPAL_VERSION --with-proxy-package-name=\"Open MPI\" --with-proxy-bugreport=\"https://www.open-mpi.org/community/help/\""
-    internal_prrte_libs=
+    # PRRTE sets -Werror on devel builds so avoid buid breaks caused by 3rd-party codes
+    internal_prrte_args="$internal_prrte_args --disable-devel-check"
 
     # Set --enable-prte-prefix-by-default to the deprecated options,
     # if they were specified.  Otherwise, set it to enabled if the
@@ -171,19 +178,22 @@ AC_DEFUN([_OMPI_SETUP_PRRTE_INTERNAL], [
               [internal_prrte_args="$internal_prrte_args --enable-prte-prefix-by-default"])
 
     AS_IF([test "$opal_libevent_mode" = "internal"],
-          [internal_prrte_args="$internal_prrte_args --with-libevent-header=$opal_libevent_header"
-           internal_prrte_CPPFLAGS="$internal_prrte_CPPFLAGS $opal_libevent_CPPFLAGS"
-           internal_prrte_libs="$internal_prrte_libs $opal_libevent_LIBS"])
+          [internal_prrte_args="$internal_prrte_args --with-libevent --disable-libevent-lib-checks"
+           internal_prrte_args="$internal_prrte_args --with-libevent-extra-libs=\"$opal_libevent_BUILD_LIBS\""
+           internal_prrte_CPPFLAGS="$internal_prrte_CPPFLAGS $opal_libevent_BUILD_CPPFLAGS"])
 
     AS_IF([test "$opal_hwloc_mode" = "internal"],
-          [internal_prrte_args="$internal_prrte_args --with-hwloc-header=$opal_hwloc_header"
-           internal_prrte_CPPFLAGS="$internal_prrte_CPPFLAGS $opal_hwloc_CPPFLAGS"
-           internal_prrte_libs="$internal_prrte_libs $opal_hwloc_LIBS"])
+          [internal_prrte_args="$internal_prrte_args --disable-hwloc-lib-checks"
+           internal_prrte_args="$internal_prrte_args --with-hwloc-extra-libs=\"$opal_hwloc_BUILD_LIBS\""
+           internal_prrte_CPPFLAGS="$internal_prrte_CPPFLAGS $opal_hwloc_BUILD_CPPFLAGS"])
 
     AS_IF([test "$opal_pmix_mode" = "internal"],
-          [internal_prrte_args="$internal_prrte_args --with-pmix-header=$opal_pmix_header"
-           internal_prrte_CPPFLAGS="$internal_prrte_CPPFLAGS $opal_pmix_CPPFLAGS"
-           internal_prrte_libs="$internal_prrte_libs $opal_pmix_LIBS"])
+          [internal_prrte_args="$internal_prrte_args --disable-pmix-lib-checks"
+           internal_prrte_args="$internal_prrte_args --with-pmix-extra-libs=\"$opal_pmix_BUILD_LIBS\""
+           internal_prrte_CPPFLAGS="$internal_prrte_CPPFLAGS $opal_pmix_BUILD_CPPFLAGS"])
+
+    opal_prrte_CPPFLAGS_save="${CPPFLAGS}"
+    OPAL_FLAGS_APPEND_UNIQ([CPPFLAGS], [${opal_pmix_CPPFLAGS}])
 
     AC_MSG_CHECKING([if PMIx version is 4.0.0 or greater])
     AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <pmix_version.h>]],
@@ -202,8 +212,8 @@ AC_DEFUN([_OMPI_SETUP_PRRTE_INTERNAL], [
              AC_MSG_WARN([--without-prrte option.])
              AC_MSG_ERROR([Cannot continue])])
 
-    # add the extra libs
-    internal_prrte_args="$internal_prrte_args --with-prte-extra-lib=\"$internal_prrte_libs\" --with-prte-extra-ltlib=\"$internal_prrte_libs\""
+    CPPFLAGS="${opal_prrte_CPPFLAGS_save}"
+
     AS_IF([test "$with_ft" != "no"],
           [internal_prrte_args="--enable-prte-ft $internal_prrte_args"],
           [])
@@ -231,8 +241,17 @@ AC_DEFUN([_OMPI_SETUP_PRRTE_INTERNAL], [
     AS_IF([test "$internal_prrte_happy" = "no" -a "$enable_internal_rte" != "no"],
           [AC_MSG_ERROR([PRRTE configuration failed.  Cannot continue.])])
 
+    OMPI_HAVE_PRRTE_RST=0
     AS_IF([test "$internal_prrte_happy" = "yes"],
-          [$1], [$2])
+          [AC_MSG_CHECKING([for internal PRRTE RST files])
+           AS_IF([test -n "$SPHINX_BUILD"],
+                 [OMPI_HAVE_PRRTE_RST=1
+                  OMPI_PRRTE_RST_CONTENT_DIR="$OMPI_TOP_SRCDIR/3rd-party/prrte/src/docs/prrte-rst-content"
+                  OMPI_SCHIZO_OMPI_RST_CONTENT_DIR="$OMPI_TOP_SRCDIR/3rd-party/prrte/src/mca/schizo/ompi"
+                  AC_MSG_RESULT([found])],
+		 [AC_MSG_RESULT([not found])])
+           $1],
+	  [$2])
 
     OPAL_VAR_SCOPE_POP
 ])
@@ -245,55 +264,78 @@ dnl caller configured libprrte configure, and the configure script
 dnl succeeded.
 AC_DEFUN([_OMPI_SETUP_PRRTE_INTERNAL_POST], [
     OPAL_3RDPARTY_SUBDIRS="$OPAL_3RDPARTY_SUBDIRS prrte"
-
-    PRTE_PATH="prte"
 ])
 
 
 dnl _OMPI_SETUP_PRRTE_EXTERNAL([action if success], [action if not success])
 dnl
-dnl Try to find an external prrte with sufficient version.  Since we
-dnl don't link against prrte, only output environment variable is
-dnl PRTE_PATH.
+dnl Try to find an external prrte with sufficient version.
 AC_DEFUN([_OMPI_SETUP_PRRTE_EXTERNAL], [
-    OPAL_VAR_SCOPE_PUSH([setup_prrte_external_happy opal_prrte_CPPFLAGS_save])
+    OPAL_VAR_SCOPE_PUSH([ompi_prte_min_version ompi_prte_min_num_version setup_prrte_external_happy opal_prrte_CPPFLAGS_save])
 
     opal_prrte_CPPFLAGS_save=$CPPFLAGS
 
-    _OPAL_CHECK_PACKAGE_HEADER([opal_prrte], [prte.h], [$with_prrte],
-			       [setup_prrte_external_happy=yes],
-			       [setup_prrte_external_happy=no])
-
-    CPPFLAGS="$opal_prrte_CPPFLAGS_save $opal_prrte_CPPFLAGS"
-
-    AC_MSG_CHECKING([if external PRRTE version is 2.0.0 or greater])
-    AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <prte_version.h>]],
-                 [[
-#if PRTE_NUMERIC_VERSION < 0x00020000
-#error "prrte API version is less than 2.0.0"
+    AS_IF([test -n "${with_prrte}" -a "${with_prrte}" != "yes" -a "${with_prrte}" != "no"],
+          [OPAL_FLAGS_APPEND_UNIQ([CPPFLAGS], ["-I${with_prrte}/include"])])
+
+    AC_CHECK_HEADER([prte.h], [setup_prrte_external_happy=yes],
+                    [setup_prrte_external_happy=no])
+
+    ompi_prte_min_version=OMPI_PRTE_MIN_VERSION
+    ompi_prte_min_num_version=OMPI_PRTE_NUMERIC_MIN_VERSION
+    AS_IF([test "${setup_prrte_external_happy}" = "yes"],
+          [AC_CACHE_CHECK([if external PRRTE version is OMPI_PRTE_MIN_VERSION or greater],
+              [ompi_setup_prrte_cv_version_happy],
+              [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <prte_version.h>
+                 ]], [[
+#if PRTE_NUMERIC_VERSION < $ompi_prte_min_num_version
+#error "prrte API version is less than $ompi_prte_min_version"
 #endif
                  ]])],
-                 [AC_MSG_RESULT([yes])
-                  setup_prrte_external_happy=yes],
-                 [AC_MSG_RESULT([no])
-                  setup_prrte_external_happy=no])
+                 [ompi_setup_prrte_cv_version_happy="yes"],
+                 [ompi_setup_prrte_cv_version_happy="no"])])
+           AS_IF([test "${ompi_setup_prrte_cv_version_happy}" = "no"],
+                 [setup_prrte_external_happy="no"])])
 
     CPPFLAGS="$opal_prrte_CPPFLAGS_save"
 
+    # If an external build and the user told us where to find PRRTE,
+    # find prterun and save that path.
+    prterun_path=
     AS_IF([test "$setup_prrte_external_happy" = "yes"],
-          [AS_IF([test -n "$with_prrte"],
-                 [PRTE_PATH="${with_prrte}/bin/prte"
-                  AS_IF([test ! -r ${PRTE_PATH}], [AC_MSG_ERROR([Could not find prte binary at $PRTE_PATH])])],
-		 [PRTE_PATH=""
-                  OPAL_WHICH([prte], [PRTE_PATH])
-                  AS_IF([tets -z "$PRTE_PATH"],
-                        [AC_MSG_WARN([Could not find prte in PATH])
-                         setup_prrte_external_happy=no])])])
-
+          [AS_IF([test "${with_prrte_bindir}" = "yes" -o "${with_prrte_bindir}" = "no"],
+                 [AC_MSG_ERROR(["yes" and "no" are not valid arguments for --with-prrte-bindir])])
+           AS_IF([test -z "${with_prrte_bindir}" -a -n "${with_prrte}"],
+                 [with_prrte_bindir="${with_prrte}/bin"])
+           AS_IF([test -n "${with_prrte_bindir}"],
+                 [AS_IF([test -x ${with_prrte_bindir}/prterun],
+                        [prterun_path="${with_prrte_bindir}/prterun"],
+                        [AC_MSG_ERROR([Could not find executable prterun: ${with_prrte_bindir}/prterun])])])])
+    AS_IF([test -n "${prterun_path}"],
+          [AC_DEFINE_UNQUOTED([OMPI_PRTERUN_PATH], ["${prterun_path}"], [Path to prterun])])
+
+    OMPI_HAVE_PRRTE_RST=0
     AS_IF([test "$setup_prrte_external_happy" = "yes"],
-          [$1], [$2])
+          [ # Determine if this external PRRTE has installed the RST
+            # directories that we care about
+
+           AC_MSG_CHECKING([for external PRRTE RST files])
+           prrte_install_dir=${with_prrte}/share/prte/rst
+           AS_IF([test -n "$SPHINX_BUILD"],
+                 [AS_IF([test -d "$prrte_install_dir/prrte-rst-content" && \
+                         test -d "$prrte_install_dir/schizo-ompi-rst-content"],
+                        [OMPI_HAVE_PRRTE_RST=1
+                         OMPI_PRRTE_RST_CONTENT_DIR="$prrte_install_dir/prrte-rst-content"
+                         OMPI_SCHIZO_OMPI_RST_CONTENT_DIR="$prrte_install_dir/schizo-ompi-rst-content"
+                         AC_MSG_RESULT([found])
+                         ],
+                        [ # This version of PRRTE doesn't have installed RST
+                          # files.
+                          AC_MSG_RESULT([not found])
+                        ])
+                 ])
+           $1],
+	  [$2])
 
     OPAL_VAR_SCOPE_POP
 ])
-
-
diff --git a/config/opal_case_sensitive_fs_setup.m4 b/config/opal_case_sensitive_fs_setup.m4
index c901e3ddb9d..7267c8616db 100644
--- a/config/opal_case_sensitive_fs_setup.m4
+++ b/config/opal_case_sensitive_fs_setup.m4
@@ -10,6 +10,7 @@ dnl Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
 dnl                         University of Stuttgart.  All rights reserved.
 dnl Copyright (c) 2004-2005 The Regents of the University of California.
 dnl                         All rights reserved.
+dnl Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved.
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
@@ -63,6 +64,8 @@ AC_ARG_WITH([cs_fs],
             [AS_HELP_STRING([--with-cs-fs],
                            [Destination FS is case sensitive (default: set to value of the build FS's case sensitivity)])])
 
+dnl Stupid emacs syntax highlighting: '
+
 if test "$with_cs_fs" = "yes"; then
     OPAL_WANT_CS_FS=1
 elif test -z "$with_cs_fs"; then
@@ -78,26 +81,12 @@ else
 fi
 
 AM_CONDITIONAL(CASE_SENSITIVE_FS, test "$OPAL_WANT_CS_FS" = "1")
-
-if test "$OPAL_WANT_CS_FS" = "0"; then
-	cat <<EOF
-
-*******************************************************************************
-NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE
-*******************************************************************************
-
-Because OPAL is being installed on a non-case sensitive file
-system, the C++ wrapper compiler will be named opalc++ instead of the
-traditional opalCC.
-
-Please update any makefiles appropriately.
-
-*******************************************************************************
-NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE
-*******************************************************************************
-
-EOF
-fi
+# There is a case in the ompi/tools/wrappers/Makefile.am where we need
+# to know if there is a case sensitive filesystem *and* if we have a
+# C++ compiler.  Since we can't use operators like "&&" or "and" to
+# join together AM CONDITIONALs in a Makefile.am, effectively make a
+# combo CONDITIONAL here.
+AM_CONDITIONAL([CASE_SENSITIVE_FS_AND_HAVE_CXX_COMPILER], [test "$OPAL_WANT_CS_FS" = "1" && test "$CXX" != "no"])
 
 # Clean up
 unset have_cs_fs])dnl
diff --git a/config/opal_check_alps.m4 b/config/opal_check_alps.m4
deleted file mode 100644
index 30932108f85..00000000000
--- a/config/opal_check_alps.m4
+++ /dev/null
@@ -1,172 +0,0 @@
-dnl -*- shell-script -*-
-dnl
-dnl Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-dnl                         University Research and Technology
-dnl                         Corporation.  All rights reserved.
-dnl Copyright (c) 2004-2005 The University of Tennessee and The University
-dnl                         of Tennessee Research Foundation.  All rights
-dnl                         reserved.
-dnl Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-dnl                         University of Stuttgart.  All rights reserved.
-dnl Copyright (c) 2004-2005 The Regents of the University of California.
-dnl                         All rights reserved.
-dnl Copyright (c) 2009-2016 Cisco Systems, Inc.  All rights reserved.
-dnl Copyright (c) 2015      Research Organization for Information Science
-dnl                         and Technology (RIST). All rights reserved.
-dnl Copyright (c) 2016      Los Alamos National Security, LLC. All rights
-dnl                         reserved.
-dnl $COPYRIGHT$
-dnl
-dnl Additional copyrights may follow
-dnl
-dnl $HEADER$
-dnl
-
-# OPAL_CHECK_ALPS_CLE4([action-if-found], [action-if-not-found])
-# --------------------------------------------------------
-AC_DEFUN([OPAL_CHECK_ALPS_CLE4],[
-
-#
-#   if we've gotten here, its because we are building on a CLE 4 system
-#
-    opal_check_alps_cle4_libdir_happy="no"
-    opal_check_alps_cle4_dir_happy="no"
-
-    AC_MSG_CHECKING([Checking for ALPS components on a CLE 4 system with alps  $with_alps])
-
-    AC_ARG_WITH([alps-libdir],
-                [AS_HELP_STRING([--with-alps-libdir=DIR],
-                [Location of alps libraries (alpslli, alpsutil) (default: /usr/lib/alps (/opt/cray/xe-sysroot/default/user on eslogin nodes))])])
-
-#
-#   check to see if Open MPI is being built on a CLE 4 eslogin node
-#
-    AS_IF([test -f /etc/opt/cray/release/ESLrelease],
-          [default_alps_dir="/opt/cray/xe-sysroot/default/usr"],
-          [default_alps_dir="/usr"])
-
-     AS_IF([test -z "$with_alps_libdir"],
-           [AS_IF([test "$with_alps" != "yes" && test "$with_alps" != "auto"],
-                  [AS_IF([test -d "$with_alps_libdir/lib64"],
-                         [opal_check_alps_libdir="$with_alps_libdir/lib64"],
-                         [opal_check_alps_libdir="$with_alps_libdir/lib"])],
-                  [ opal_check_alps_libdir="$default_alps_dir/lib/alps"])
-           ],[])
-
-     AS_IF([test "$with_alps" = "yes" || test "$with_alps" = "auto"],
-           [opal_check_alps_dir=$default_alps_dir],
-           [opal_check_alps_dir=$with_alps])
-
-    AC_MSG_CHECKING([if $opal_check_alps_libdir/libalps.a is present])
-    AS_IF([test -f "$opal_check_alps_libdir/libalps.a"],
-          [opal_check_alps_libdir_cle4_happy="yes"],
-          [opal_check_alps_libdir_cle4_happy="no",
-           AC_MSG_RESULT([no])])
-
-    AC_MSG_CHECKING([if $opal_check_alps_dir/include/alps/apInfo.h is present])
-    AS_IF([test -f "$opal_check_alps_dir/include/alps/apInfo.h"],
-          [opal_check_alps_dir_cle4_happy="yes"],
-          [opal_check_alps_dir_cle4_happy="no"
-           AC_MSG_RESULT([no])])
-
-    AS_IF([test "$opal_check_alps_libdir_cle4_happy" = "yes" && test "$opal_check_alps_dir_cle4_happy" = "yes"],
-          [CRAY_ALPSLLI_CFLAGS="-I$opal_check_alps_dir/include"
-           CRAY_ALPSLLI_LIBS="-L$opal_check_alps_libdir -lalpslli -lalpsutil"
-           CRAY_ALPSLLI_STATIC_LIBS="-L$opal_check_alps_libdir -lalpslli -lalpsutil"
-           $1],
-          [$2])
-])
-
-
-
-
-# OPAL_CHECK_ALPS(prefix, [action-if-found], [action-if-not-found])
-# --------------------------------------------------------
-AC_DEFUN([OPAL_CHECK_ALPS],[
-    if test -z "$opal_check_cray_alps_happy"; then
-
-        AC_ARG_WITH([alps],
-                    [AS_HELP_STRING([--with-alps(=DIR|yes|no)],
-                    [Build with ALPS scheduler component, optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries (default: auto)])],[],with_alps=auto)
-
-        if test -f /etc/opt/cray/release/clerelease; then
-            cle_level=`awk -F. '{print [$]1}' /etc/opt/cray/release/clerelease`
-        else
-            cle_level="unknown"
-        fi
-
-        AC_MSG_CHECKING([for ALPS support cle level $cle_level])
-        AS_IF([test "$cle_level" = "4" && test "$with_alps" != "no"],
-              [OPAL_CHECK_ALPS_CLE4([opal_check_cray_alps_happy="yes"],
-                                    [opal_check_cray_alps_happy="no"])],
-              [AS_IF([test "$with_alps" = "no"],
-                     [AC_MSG_RESULT([no])
-                      opal_check_cray_alps_happy=no
-                      $3],
-                     [AS_IF([test "$with_alps" = "auto" || test "$with_alps" = "yes"],
-                            [PKG_CHECK_MODULES_STATIC([CRAY_ALPSLLI], [cray-alpslli],
-                                                      [opal_check_cray_alps_happy="yes"],
-                                                      [opal_check_cray_alps_happy="no"]
-                                                      [AS_IF([test "$with_alps" = "yes"],
-                                                             [AC_MSG_WARN([ALPS support requested but pkg-config failed.])
-                                                              AC_MSG_WARN([Need to explicitly indicate ALPS directory])
-                                                              AC_MSG_WARN([on the configure line using --with-alps option.])
-                                                              AC_MSG_ERROR([Aborting])],[])]
-                                                      )
-                              PKG_CHECK_MODULES_STATIC([CRAY_ALPSUTIL], [cray-alpsutil],
-                                                [opal_check_cray_alps_happy="yes"],
-                                                [opal_check_cray_alps_happy="no"]
-                                                [AS_IF([test "$with_alps" = "yes"],
-                                                       [AC_MSG_WARN([ALPS support requested but pkg-config failed.])
-                                                        AC_MSG_WARN([Need to explicitly indicate ALPS directory])
-                                                        AC_MSG_WARN([on the configure line using --with-alps option.])
-                                                        AC_MSG_ERROR([Aborting])],[])]
-                                                       )
-
-                               PKG_CHECK_MODULES_STATIC([CRAY_ALPS], [cray-alps],
-                                               [opal_check_cray_alps_happy="yes"],
-                                               [opal_check_cray_alps_happy="no"]
-                                               [AS_IF([test "$with_alps" = "yes"],
-                                                      [AC_MSG_WARN([ALPS support requested but pkg-config failed.])
-                                                       AC_MSG_WARN([Need to explicitly indicate ALPS directory])
-                                                       AC_MSG_WARN([on the configure line using --with-alps option.])
-                                                       AC_MSG_ERROR([Aborting])],[])]
-                                                       )
-                               PKG_CHECK_MODULES_STATIC([CRAY_WLM_DETECT], [cray-wlm_detect],
-                                               [opal_check_cray_alps_happy="yes"
-                                                AC_DEFINE_UNQUOTED([CRAY_WLM_DETECT],[1],
-                                                                   [defined to 1 if cray wlm available, 0 otherwise])
-                                               ],
-                                               [opal_check_cray_alps_happy="no"]
-                                               [AS_IF([test "$with_alps" = "yes"],
-                                                      [AC_MSG_WARN([ALPS support requested but pkg-config failed.])
-                                                       AC_MSG_WARN([Need to explicitly indicate ALPS directory])
-                                                       AC_MSG_WARN([on the configure line using --with-alps option.])
-                                                       AC_MSG_ERROR([Aborting])],[])]
-                                                       )
-                            ],
-                            [AC_MSG_WARN([See ./configure --help for how to control Open MPI])
-                             AC_MSG_WARN([configuration for ALPS on CLE 5 and higher systems])
-                             AC_MSG_ERROR([Aborting])])
-                    ])
-               ])
-
-        AC_MSG_RESULT([opal_check_cray_alps_happy = $opal_check_cray_alps_happy])
-
-        AS_IF([test "$opal_check_cray_alps_happy" = "yes" && test "$enable_static" = "yes"],
-              [CRAY_ALPSLLI_LIBS = $CRAY_ALPSLLI_STATIC_LIBS
-               CRAY_ALPSUTIL_LIBS = $CRAY_ALPSUTIL_STATIC_LIBS],
-              [])
-
-	OPAL_SUMMARY_ADD([[Resource Managers]],[[Cray Alps]],[$1],[$opal_check_cray_alps_happy])
-    fi
-
-    AS_IF([test "$opal_check_cray_alps_happy" = "yes"],
-          [$1_LDFLAGS="[$]$1_LDFLAGS $CRAY_ALPSLLI_LIBS $CRAY_ALPSUTIL_LIBS"
-           $1_CPPFLAGS="[$]$1_CPPFLAGS $CRAY_ALPSLLI_CFLAGS $CRAY_ALPSUTIL_CFLAGS $CRAY_ALPS_CFLAGS $CRAY_WLM_DETECT_CFLAGS"
-           $1_LIBS="[$]$1_LIBS $CRAY_ALPSLLI_LIBS $CRAY_ALPSUTIL_LIBS $CRAY_WLM_DETECT_LIBS"
-           $1_WRAPPER_EXTRA_LDFLAGS="$CRAY_ALPSLLI_LIBS $CRAY_ALPSUTIL_LIBS $CRAY_WLM_DETECT_LIBS"
-           $1_WRAPPER_EXTRA_LIBS="$CRAY_ALPSLLI_LIBS $CRAY_ALPSUTIL_LIBS $CRAY_WLM_DETECT_LIBS"
-	   $2],
-	  [$3])
-])
diff --git a/config/opal_check_alt_short_float.m4 b/config/opal_check_alt_short_float.m4
index bbdfcad2825..e7ff008ba6b 100644
--- a/config/opal_check_alt_short_float.m4
+++ b/config/opal_check_alt_short_float.m4
@@ -1,6 +1,6 @@
 dnl -*- shell-script -*-
 dnl
-dnl Copyright (c) 2018-2020 FUJITSU LIMITED.  All rights reserved.
+dnl Copyright (c) 2018-2021 FUJITSU LIMITED.  All rights reserved.
 dnl Copyright (c) 2020 Cisco Systems, Inc.  All rights reserved.
 dnl $COPYRIGHT$
 dnl
@@ -47,9 +47,9 @@ AC_DEFUN([OPAL_CHECK_ALT_SHORT_FLOAT], [
         # automagically add that flag -- we'll just emit a warning and
         # point the user to a README where more information is
         # available.
-        AC_MSG_CHECKING([if compiler supports arithmetic operations on $opal_short_float_type])
         AS_IF([test $opal_alt_short_float_exists -eq 1],
-              [AC_LINK_IFELSE([AC_LANG_PROGRAM([], [[
+              [AC_MSG_CHECKING([if compiler supports arithmetic operations on $opal_short_float_type])
+               AC_LINK_IFELSE([AC_LANG_PROGRAM([], [[
 static $opal_short_float_type a = 2.5, b = 3.8;
 a += b;]])],
                                  [AC_MSG_RESULT([yes])
@@ -74,6 +74,29 @@ a += b;]])],
             AC_CHECK_SIZEOF(opal_short_float_t)
             AC_CHECK_SIZEOF(opal_short_float_complex_t)
             OPAL_C_GET_ALIGNMENT(opal_short_float_t, OPAL_ALIGNMENT_OPAL_SHORT_FLOAT_T)
+
+            # Some versions of GCC (around 9.1.0?) emit a warning for _Float16
+            # when compiling with -pedantic. Using __extension__ can suppress
+            # the warning. The warning can be detected by -Werror in configure.
+            # See https://github.com/open-mpi/ompi/issues/8840
+            AC_MSG_CHECKING([if $opal_short_float_type needs __extension__ keyword])
+            opal_alt_short_float_needs_extension=0
+            OPAL_VAR_SCOPE_PUSH([CFLAGS_save])
+            CFLAGS_save=$CFLAGS
+            CFLAGS="-Werror $CFLAGS"
+            AC_COMPILE_IFELSE([AC_LANG_SOURCE([$opal_short_float_type a;])],
+                              [AC_MSG_RESULT([no])],
+                              [AC_COMPILE_IFELSE([AC_LANG_SOURCE([__extension__ $opal_short_float_type a;])],
+                                                 [opal_alt_short_float_needs_extension=1
+                                                  AC_MSG_RESULT([yes])],
+                                                 [AC_MSG_RESULT([no])])])
+            CFLAGS=$CFLAGS_save
+            OPAL_VAR_SCOPE_POP
+            AC_DEFINE_UNQUOTED(OPAL_SHORT_FLOAT_TYPE, [[$opal_short_float_type]],
+                               [User-selected alternate C type of short float (used to redefine opal_short_float_t in opal_bottom.h)])
+            AC_DEFINE_UNQUOTED(OPAL_SHORT_FLOAT_NEEDS_EXTENSION,
+                               [$opal_alt_short_float_needs_extension],
+                               [Whether $opal_short_float_type needs __extension__ keyword])
         elif test "$enable_alt_short_float" != ""; then
             AC_MSG_ERROR([Alternate C type of short float $opal_short_float_type requested but not available.  Aborting])
         fi
diff --git a/config/opal_check_attributes.m4 b/config/opal_check_attributes.m4
index b4da2ca54cd..2d2e1ba3d7f 100644
--- a/config/opal_check_attributes.m4
+++ b/config/opal_check_attributes.m4
@@ -11,7 +11,7 @@ dnl                         University of Stuttgart.  All rights reserved.
 dnl Copyright (c) 2004-2005 The Regents of the University of California.
 dnl                         All rights reserved.
 dnl Copyright (c) 2009      Oak Ridge National Labs.  All rights reserved.
-dnl Copyright (c) 2010-2018 Cisco Systems, Inc.  All rights reserved
+dnl Copyright (c) 2010-2021 Cisco Systems, Inc.  All rights reserved
 dnl Copyright (c) 2013      Mellanox Technologies, Inc.
 dnl                         All rights reserved.
 dnl Copyright (c) 2015      Research Organization for Information Science
@@ -64,7 +64,7 @@ AC_DEFUN([_OPAL_CHECK_SPECIFIC_ATTRIBUTE], [
         #
         # Try to compile using the C compiler, then C++
         #
-        AC_TRY_COMPILE([$2],[],
+        AC_COMPILE_IFELSE([AC_LANG_PROGRAM([$2],[])],
                        [
                         #
                         # In case we did succeed: Fine, but was this due to the
@@ -82,10 +82,10 @@ AC_DEFUN([_OPAL_CHECK_SPECIFIC_ATTRIBUTE], [
         m4_ifdef([project_ompi],
                  [if test "$opal_cv___attribute__[$1]" = "1" ; then
                       AC_LANG_PUSH(C++)
-                      AC_TRY_COMPILE([
+                      AC_COMPILE_IFELSE([AC_LANG_PROGRAM([
                            extern "C" {
                            $2
-                           }],[],
+                           }],[])],
                            [
                             opal_cv___attribute__[$1]=1
                             _OPAL_ATTRIBUTE_FAIL_SEARCH([$1])
@@ -103,11 +103,11 @@ AC_DEFUN([_OPAL_CHECK_SPECIFIC_ATTRIBUTE], [
             CFLAGS_safe=$CFLAGS
             CFLAGS="$CFLAGS [$4]"
 
-            AC_TRY_COMPILE([$3],
+            AC_COMPILE_IFELSE([AC_LANG_PROGRAM([$3],
                 [
                  int i=4711;
                  i=usage(&i);
-                ],
+                ])],
                 [opal_cv___attribute__[$1]=0],
                 [
                  #
@@ -161,7 +161,7 @@ AC_DEFUN([OPAL_CHECK_ATTRIBUTES], [
   AC_MSG_CHECKING(for __attribute__)
 
   AC_CACHE_VAL(opal_cv___attribute__, [
-    AC_TRY_COMPILE(
+    AC_COMPILE_IFELSE([AC_LANG_PROGRAM(
       [#include <stdlib.h>
        /* Check for the longest available __attribute__ (since gcc-2.3) */
        struct foo {
@@ -169,13 +169,13 @@ AC_DEFUN([OPAL_CHECK_ATTRIBUTES], [
            int x[2] __attribute__ ((__packed__));
         };
       ],
-      [],
+      [])],
       [opal_cv___attribute__=1],
       [opal_cv___attribute__=0],
     )
 
     if test "$opal_cv___attribute__" = "1" ; then
-        AC_TRY_COMPILE(
+        AC_COMPILE_IFELSE([AC_LANG_PROGRAM(
           [#include <stdlib.h>
            /* Check for the longest available __attribute__ (since gcc-2.3) */
            struct foo {
@@ -183,7 +183,7 @@ AC_DEFUN([OPAL_CHECK_ATTRIBUTES], [
                int x[2] __attribute__ ((__packed__));
             };
           ],
-          [],
+          [])],
           [opal_cv___attribute__=1],
           [opal_cv___attribute__=0],
         )
@@ -559,6 +559,14 @@ AC_DEFUN([OPAL_CHECK_ATTRIBUTES], [
         [],
         [])
 
+    _OPAL_CHECK_SPECIFIC_ATTRIBUTE([constructor],
+        [
+        void foo(void) __attribute__ ((__constructor__));
+        void foo(void) { return ; }
+        ],
+        [],
+        [])
+
     _OPAL_CHECK_SPECIFIC_ATTRIBUTE([destructor],
         [
         void foo(void) __attribute__ ((__destructor__));
@@ -631,6 +639,8 @@ AC_DEFUN([OPAL_CHECK_ATTRIBUTES], [
                      [Whether your compiler has __attribute__ warn unused result or not])
   AC_DEFINE_UNQUOTED(OPAL_HAVE_ATTRIBUTE_WEAK_ALIAS, [$opal_cv___attribute__weak_alias],
                      [Whether your compiler has __attribute__ weak alias or not])
+  AC_DEFINE_UNQUOTED(OPAL_HAVE_ATTRIBUTE_CONSTRUCTOR, [$opal_cv___attribute__constructor],
+                     [Whether your compiler has __attribute__ constructor or not])
   AC_DEFINE_UNQUOTED(OPAL_HAVE_ATTRIBUTE_DESTRUCTOR, [$opal_cv___attribute__destructor],
                      [Whether your compiler has __attribute__ destructor or not])
   AC_DEFINE_UNQUOTED(OPAL_HAVE_ATTRIBUTE_OPTNONE, [$opal_cv___attribute__optnone],
diff --git a/config/opal_check_cflags.m4 b/config/opal_check_cflags.m4
index 59fc0ce1ace..fd04f12ab45 100644
--- a/config/opal_check_cflags.m4
+++ b/config/opal_check_cflags.m4
@@ -1,6 +1,7 @@
 dnl -*- shell-script -*-
 dnl
 dnl Copyright (c) 2021 IBM Corporation.  All rights reserved.
+dnl Copyright (c) 2021 Cisco Systems, Inc.  All rights reserved.
 dnl
 dnl $COPYRIGHT$
 dnl
@@ -22,16 +23,17 @@ AC_DEFUN([_OPAL_CFLAGS_FAIL_SEARCH],[
 AC_DEFUN([_OPAL_CHECK_SPECIFIC_CFLAGS], [
 AC_MSG_CHECKING(if $CC supports ([$1]))
             CFLAGS_orig=$CFLAGS
-            CFLAGS="$CFLAGS $1"
+            OPAL_FLAGS_APPEND_UNIQ([CFLAGS], ["$1"])
             AC_CACHE_VAL(opal_cv_cc_[$2], [
-                   AC_TRY_COMPILE([], [$3],
+                   AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [$3])],
                                    [
                                     opal_cv_cc_[$2]=1
-                                    _OPAL_CFLAGS_FAIL_SEARCH("ignored\|not recognized\|not supported\|not compatible\|unrecognized\|unknown", [$2])
+                                    _OPAL_CFLAGS_FAIL_SEARCH(["ignored\|not recognized\|not supported\|not compatible\|unrecognized\|unknown"], [$2])
                                    ],
+                                  [
                                     opal_cv_cc_[$2]=1
-                                    _OPAL_CFLAGS_FAIL_SEARCH("ignored\|not recognized\|not supported\|not compatible\|unrecognized\|unknown\|error", [$2])
-                                 )])
+                                    _OPAL_CFLAGS_FAIL_SEARCH(["ignored\|not recognized\|not supported\|not compatible\|unrecognized\|unknown\|error"], [$2])
+                                 ])])
             if test "$opal_cv_cc_[$2]" = "0" ; then
                 CFLAGS="$CFLAGS_orig"
                 AC_MSG_RESULT([no])
@@ -53,16 +55,17 @@ AC_DEFUN([_OPAL_CXXFLAGS_FAIL_SEARCH],[
 AC_DEFUN([_OPAL_CHECK_SPECIFIC_CXXFLAGS], [
 AC_MSG_CHECKING(if $CXX supports ([$1]))
             CXXFLAGS_orig=$CXXFLAGS
-            CXXFLAGS="$CXXFLAGS $1"
+            OPAL_FLAGS_APPEND_UNIQ([CXXFLAGS], ["$1"])
             AC_CACHE_VAL(opal_cv_cxx_[$2], [
-                   AC_TRY_COMPILE([], [$3],
+                   AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [$3])],
                                    [
                                     opal_cv_cxx_[$2]=1
-                                    _OPAL_CXXFLAGS_FAIL_SEARCH("ignored\|not recognized\|not supported\|not compatible\|unrecognized\|unknown", [$2])
+                                    _OPAL_CXXFLAGS_FAIL_SEARCH(["ignored\|not recognized\|not supported\|not compatible\|unrecognized\|unknown"], [$2])
                                    ],
+                                  [
                                     opal_cv_cxx_[$2]=1
-                                    _OPAL_CXXFLAGS_FAIL_SEARCH("ignored\|not recognized\|not supported\|not compatible\|unrecognized\|unknown\|error", [$2])
-                                 )])
+                                    _OPAL_CXXFLAGS_FAIL_SEARCH(["ignored\|not recognized\|not supported\|not compatible\|unrecognized\|unknown\|error"], [$2])
+                                 ])])
             if test "$opal_cv_cxx_[$2]" = "0" ; then
                 CXXFLAGS="$CXXFLAGS_orig"
                 AC_MSG_RESULT([no])
diff --git a/config/opal_check_cma.m4 b/config/opal_check_cma.m4
index cd43d232ab4..ceaf6d615fd 100644
--- a/config/opal_check_cma.m4
+++ b/config/opal_check_cma.m4
@@ -4,9 +4,10 @@
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
 # Copyright (c) 2009-2016 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2010-2012 IBM Corporation.  All rights reserved.
+# Copyright (c) 2010-2022 IBM Corporation.  All rights reserved.
 # Copyright (c) 2013-2016 Los Alamos National Security, LLC. All rights
 #                         reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -44,8 +45,8 @@ AC_DEFUN([OPAL_CHECK_CMA_BACKEND],
 [
     OPAL_VAR_SCOPE_PUSH([opal_check_cma_need_defs opal_check_cma_kernel_version opal_check_cma_CFLAGS opal_check_cma_msg])
 
-    # Some systems have process_cm_readv() in libc, which means CMA is
-    # supported.  Other systems do not have process_cm_readv() in
+    # Some systems have process_vm_readv() in libc, which means CMA is
+    # supported.  Other systems do not have process_vm_readv() in
     # libc, but have support for it in the kernel if we invoke it
     # directly.  Check for both.
     AC_CHECK_HEADERS([sys/prctl.h])
@@ -114,23 +115,42 @@ static void do_check (pid_t pid, int *in, int *out)
     /* all good */
     return 0;
 ]])],
-		      [AC_MSG_RESULT([yes])
-		       opal_check_cma_happy=1],
-		      [AC_MSG_RESULT([no])
-		       opal_check_cma_happy=0],
-		      [AC_MSG_RESULT([no (cross-compiling)])
-		       opal_check_cma_happy=0])
-	CFLAGS=$opal_check_cma_CFLAGS
+                      [AC_MSG_RESULT([yes])
+                       opal_check_cma_happy=1],
+                      [AC_MSG_RESULT([no])
+                       opal_check_cma_happy=0],
+                      [AC_MSG_RESULT([no (cross-compiling)])
+                       opal_check_cma_happy=0])
+        CFLAGS=$opal_check_cma_CFLAGS
     else
         # If we didn't need the defs, then we have process_vm_readv(),
         # and CMA is happy.
-	opal_check_cma_happy=1
+        opal_check_cma_happy=1
     fi
 
     OPAL_VAR_SCOPE_POP
 
+    # Testing CAP_SYS_PTRACE between two processes with kcmp
+    AC_CHECK_HEADERS([linux/kcmp.h])
+    AC_CHECK_HEADERS([sys/syscall.h])
+    AC_MSG_CHECKING([if kcmp works])
+    AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
+#include <linux/kcmp.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+                                       ]], [[
+syscall(SYS_kcmp, 123, 456, KCMP_VM, 0, 0);
+                                           ]])],
+                      [AC_MSG_RESULT([yes])
+                       opal_check_cma_kcmp_happy=1],
+                      [AC_MSG_RESULT([no])
+                       opal_check_cma_kcmp_happy=0])
+    AC_DEFINE_UNQUOTED([OPAL_CMA_KCMP_AVAIL],
+                       [$opal_check_cma_kcmp_happy],
+                       [If kcmp is available])
+
     AS_IF([test $opal_check_cma_happy -eq 1],
           [opal_check_cma_msg=yes],
           [opal_check_cma_msg=no])
-    OPAL_SUMMARY_ADD([[Transports]],[[Shared memory/Linux CMA]],[$1],[$opal_check_cma_msg])
+    OPAL_SUMMARY_ADD([Transports], [Shared memory/Linux CMA], [], [$opal_check_cma_msg])
 ])
diff --git a/config/opal_check_compiler_version.m4 b/config/opal_check_compiler_version.m4
index da8321cdf1e..75ee688f2ad 100644
--- a/config/opal_check_compiler_version.m4
+++ b/config/opal_check_compiler_version.m4
@@ -1,6 +1,9 @@
 dnl -*- shell-script -*-
 dnl
 dnl Copyright (c) 2009      Oak Ridge National Labs.  All rights reserved.
+dnl Copyright (c) 2021 Cisco Systems, Inc.  All rights reserved.
+dnl Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+dnl                         reserved.
 dnl
 dnl $COPYRIGHT$
 dnl
@@ -19,105 +22,35 @@ dnl
 AC_DEFUN([OPAL_CHECK_COMPILER_VERSION_ID],
 [
     OPAL_CHECK_COMPILER(FAMILYID)
-    OPAL_CHECK_COMPILER_STRINGIFY(FAMILYNAME)
     OPAL_CHECK_COMPILER(VERSION)
-    OPAL_CHECK_COMPILER_STRING(VERSION_STR)
 ])dnl
 
 
 AC_DEFUN([OPAL_CHECK_COMPILER], [
-    lower=m4_tolower($1)
-    AC_CACHE_CHECK([for compiler $lower], opal_cv_compiler_[$1],
+    OAC_ASSERT_LITERAL([$1], [1])dnl
+    lower=m4_tolower([$1])
+    AC_CACHE_CHECK([for compiler $lower], [opal_cv_compiler_$1],
     [
             CPPFLAGS_orig=$CPPFLAGS
-            CPPFLAGS="-I${OPAL_TOP_SRCDIR}/opal/include/opal $CPPFLAGS"
-            AC_TRY_RUN([
+            CPPFLAGS="-I${OPAL_TOP_SRCDIR}/opal/include $CPPFLAGS"
+            AC_RUN_IFELSE([AC_LANG_PROGRAM([[
 #include <stdio.h>
 #include <stdlib.h>
-#include "opal_portable_platform.h"
-
-int main (int argc, char * argv[])
-{
+#include "opal/opal_portable_platform.h"
+]],[[
     FILE * f;
     f=fopen("conftestval", "w");
     if (!f) exit(1);
     fprintf (f, "%d", PLATFORM_COMPILER_$1);
-    return 0;
-}
-            ], [
-                eval opal_cv_compiler_$1=`cat conftestval`;
-            ], [
-                eval opal_cv_compiler_$1=0
-            ], [
-                eval opal_cv_compiler_$1=0
-            ])
-            CPPFLAGS=$CPPFLAGS_orig
-    ])
-    AC_DEFINE_UNQUOTED([OPAL_BUILD_PLATFORM_COMPILER_$1], $opal_cv_compiler_[$1],
-                       [The compiler $lower which OMPI was built with])
-])dnl
-
-AC_DEFUN([OPAL_CHECK_COMPILER_STRING], [
-    lower=m4_tolower($1)
-    AC_CACHE_CHECK([for compiler $lower], opal_cv_compiler_[$1],
-    [
-            CPPFLAGS_orig=$CPPFLAGS
-            CPPFLAGS="-I${OPAL_TOP_SRCDIR}/opal/include/opal $CPPFLAGS"
-            AC_TRY_RUN([
-#include <stdio.h>
-#include <stdlib.h>
-#include "opal_portable_platform.h"
-
-int main (int argc, char * argv[])
-{
-    FILE * f;
-    f=fopen("conftestval", "w");
-    if (!f) exit(1);
-    fprintf (f, "%s", PLATFORM_COMPILER_$1);
-    return 0;
-}
-            ], [
-                eval opal_cv_compiler_$1=`cat conftestval`;
-            ], [
-                eval opal_cv_compiler_$1=UNKNOWN
-            ], [
-                eval opal_cv_compiler_$1=UNKNOWN
-            ])
-            CPPFLAGS=$CPPFLAGS_orig
-    ])
-    AC_DEFINE_UNQUOTED([OPAL_BUILD_PLATFORM_COMPILER_$1], $opal_cv_compiler_[$1],
-                       [The compiler $lower which OMPI was built with])
-])dnl
-
-
-AC_DEFUN([OPAL_CHECK_COMPILER_STRINGIFY], [
-    lower=m4_tolower($1)
-    AC_CACHE_CHECK([for compiler $lower], opal_cv_compiler_[$1],
-    [
-            CPPFLAGS_orig=$CPPFLAGS
-            CPPFLAGS="-I${OPAL_TOP_SRCDIR}/opal/include/opal $CPPFLAGS"
-            AC_TRY_RUN([
-#include <stdio.h>
-#include <stdlib.h>
-#include "opal_portable_platform.h"
-
-int main (int argc, char * argv[])
-{
-    FILE * f;
-    f=fopen("conftestval", "w");
-    if (!f) exit(1);
-    fprintf (f, "%s", _STRINGIFY(PLATFORM_COMPILER_$1));
-    return 0;
-}
-            ], [
-                eval opal_cv_compiler_$1=`cat conftestval`;
+            ]])], [
+                opal_cv_compiler_$1=`cat conftestval`
             ], [
-                eval opal_cv_compiler_$1=UNKNOWN
+                opal_cv_compiler_$1=0
             ], [
-                eval opal_cv_compiler_$1=UNKNOWN
+                opal_cv_compiler_$1=0
             ])
             CPPFLAGS=$CPPFLAGS_orig
     ])
-    AC_DEFINE_UNQUOTED([OPAL_BUILD_PLATFORM_COMPILER_$1], $opal_cv_compiler_[$1],
+    AC_DEFINE_UNQUOTED([OPAL_BUILD_PLATFORM_COMPILER_$1], [$opal_cv_compiler_$1],
                        [The compiler $lower which OMPI was built with])
 ])dnl
diff --git a/config/opal_check_cray_pmi.m4 b/config/opal_check_cray_pmi.m4
deleted file mode 100644
index 4a0f1e0116f..00000000000
--- a/config/opal_check_cray_pmi.m4
+++ /dev/null
@@ -1,107 +0,0 @@
-dnl -*- shell-script ; indent-tabs-mode:nil -*-
-dnl
-dnl Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-dnl                         University Research and Technology
-dnl                         Corporation.  All rights reserved.
-dnl Copyright (c) 2004-2005 The University of Tennessee and The University
-dnl                         of Tennessee Research Foundation.  All rights
-dnl                         reserved.
-dnl Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-dnl                         University of Stuttgart.  All rights reserved.
-dnl Copyright (c) 2004-2005 The Regents of the University of California.
-dnl                         All rights reserved.
-dnl Copyright (c) 2009-2011 Cisco Systems, Inc.  All rights reserved.
-dnl Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights
-dnl                         reserved.
-dnl Copyright (c) 2014-2015 Intel, Inc. All rights reserved.
-dnl Copyright (c) 2014-2015 Research Organization for Information Science
-dnl                         and Technology (RIST). All rights reserved.
-dnl $COPYRIGHT$
-dnl
-dnl Additional copyrights may follow
-dnl
-dnl $HEADER$
-dnl
-
-AC_DEFUN([OPAL_CHECK_CRAY_PMI_EXPLICIT],[
-
-    opal_cray_pmi_lib_good=0
-    opal_cray_pmi_include_good=0
-
-    AC_MSG_CHECKING([Checking for Cray PMI with explicit path $with_cray_pmi])
-    AS_IF([test -d "$with_cray_pmi/lib64"],
-          [opal_cray_pmi_lib_good=1],
-          [opal_cray_pmi_lib_good=0
-          AC_MSG_WARN([No libpmi in path specified by $with_cray_pmi])])
-
-    AS_IF([test -f "$with_cray_pmi/include/pmi.h"],
-          [opal_cray_pmi_include_good=1],
-          [opal_cray_pmi_include_good=0
-          AC_MSG_WARN([No pmi.h in path specified by $with_cray_pmi])])
-
-    AS_IF([test "$opal_cray_pmi_lib_good" -eq 1 && test "$opal_cray_pmi_include_good" -eq 1],
-           [CRAY_PMI_LDFLAGS="-L$with_cray_pmi/lib64 -L/usr/lib/alps"
-            CRAY_PMI_LIBS="-L$with_cray_pmi/lib64 -lpmi"
-            CRAY_PMI_CFLAGS="-I $with_cray_pmi/include"
-            $1],
-            [$2])
-
-#
-#   this logic assumes knowledge about all the dependencies of the Cray PMI library,
-#   something that Cray doesn't generally document
-#
-    AS_IF([test "$enable_static" = "yes"],
-          [AS_IF([test -d /usr/lib/alps],
-                 [AC_MSG_RESULT([Detected presence of /usr/lib/alps])
-                  CRAY_PMI_LDFLAGS="$CRAY_PMI_LDFLAGS -L/usr/lib/alps -lalpslli -lalpsutil"
-                  CRAY_PMI_LIBS="$CRAY_PMI_LIBS -L/usr/lib/alps -lalpslli -lalpsutil"],
-                 [AS_IF([test -d /opt/cray/xe-sysroot/default/usr/lib/alps],
-                        [AC_MSG_RESULT([Detected presence of /opt/cray/xe-sysroot/default/usr/lib/alps])
-                         CRAY_PMI_LDFLAGS="$CRAY_PMI_LDFLAGS -L/opt/cray/xe-sysroot/default/usr/lib/alps -lalpslli -lalpsutil"
-                         CRAY_PMI_LIBS="$CRAY_PMI_LIBS -L/opt/cray/xe-sysroot/default/usr/lib/alps -lalpslli -lalpsutil"],
-                        [AC_MSG_ERROR([Requested enabling static linking but unable to local libalpslli and libalpsutil])])
-                  ])
-           ])
-])
-
-#
-# special check for cray pmi, uses macro(s) from pkg.m4
-#
-# OPAL_CHECK_CRAY_PMI(prefix, [action-if-found], [action-if-not-found])
-# --------------------------------------------------------
-AC_DEFUN([OPAL_CHECK_CRAY_PMI],[
-    AC_ARG_WITH([cray_pmi],
-                [AS_HELP_STRING([--with-cray-pmi(=DIR)],
-                [Build Cray PMI support, optionally adding DIR to the search path (default: auto)])],
-                                                                                [], with_cray_pmi=auto)
-   AC_MSG_CHECKING([for Cray PMI support])
-   AS_IF([test "$with_cray_pmi" = "no"],
-         [AC_MSG_RESULT([no])
-          $3],
-         [AS_IF([test "$with_cray_pmi" = "auto" || test "$with_cray_pmi" = "yes"],
-                 [PKG_CHECK_MODULES_STATIC([CRAY_PMI], [cray-pmi],
-                                    [opal_check_cray_pmi_happy="yes"],
-                                    [opal_check_cray_pmi_happy="no"]
-                                    [AS_IF([test "$with_cray_pmi" = "yes"],
-                                           [AC_MSG_WARN([Cray PMI support requested but pkg-config failed.])
-                                            AC_MSG_WARN([Need to explicitly indicate cray pmi directory])
-                                            AC_MSG_WARN([on the configure line using --with-cray-pmi option.])
-                                            AC_MSG_ERROR([Aborting])],[])]
-                                     )],
-                 [OPAL_CHECK_CRAY_PMI_EXPLICIT([opal_check_cray_pmi_happy="yes"],
-                                               [opal_check_cray_pmi_happy="no"])
-                  AC_MSG_WARN([opal_chack_cray_pmi_happy = $opal_check_cray_pmi_happy])])
-         ])
-
-    AS_IF([test "$opal_check_cray_pmi_happy" = "yes" && test "$enable_static" = "yes"],
-          [CRAY_PMI_LIBS = $CRAY_PMI_STATIC_LIBS],[])
-
-    AS_IF([test "$opal_check_cray_pmi_happy" = "yes"],
-          [$1_LDFLAGS="$CRAY_PMI_LIBS"
-           $1_CPPFLAGS="$CRAY_PMI_CFLAGS"
-           $1_LIBS="$CRAY_PMI_LIBS"
-           $2], [$3])
-])
-
-
-
diff --git a/config/opal_check_cuda.m4 b/config/opal_check_cuda.m4
index dcfe9d117c7..ebd70d59e32 100644
--- a/config/opal_check_cuda.m4
+++ b/config/opal_check_cuda.m4
@@ -1,5 +1,6 @@
-dnl -*- shell-script -*-
+dnl -*- autoconf -*-
 dnl
+dnl Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
 dnl Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
 dnl                         University Research and Technology
 dnl                         Corporation.  All rights reserved.
@@ -19,7 +20,7 @@ dnl Copyright (c) 2009-2011 Oak Ridge National Labs.  All rights reserved.
 dnl Copyright (c) 2011-2015 NVIDIA Corporation.  All rights reserved.
 dnl Copyright (c) 2015      Research Organization for Information Science
 dnl                         and Technology (RIST). All rights reserved.
-dnl
+dnl Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
@@ -27,7 +28,22 @@ dnl
 dnl $HEADER$
 dnl
 
+
+# OPAL_CHECK_CUDA(prefix, [action-if-found], [action-if-not-found])
+# --------------------------------------------------------
+# check if CUDA support can be found.  sets prefix_{CPPFLAGS,
+# LDFLAGS, LIBS} as needed and runs action-if-found if there is
+# support, otherwise executes action-if-not-found
+
+#
+# Check for CUDA support
+#
 AC_DEFUN([OPAL_CHECK_CUDA],[
+OPAL_VAR_SCOPE_PUSH([cuda_save_CPPFLAGS cuda_save_LDFLAGS cuda_save_LIBS])
+
+cuda_save_CPPFLAGS="$CPPFLAGS"
+cuda_save_LDFLAGS="$LDFLAGS"
+cuda_save_LIBS="$LIBS"
 #
 # Check to see if user wants CUDA support
 #
@@ -36,6 +52,18 @@ AC_ARG_WITH([cuda],
             [Build cuda support, optionally adding DIR/include])])
 AC_MSG_CHECKING([if --with-cuda is set])
 
+# Search for libcuda.so in $with_cuda if the user didn't pass --with-cuda-libdir
+# Otherwise check for cuda in the default path, /usr/local/cuda. If the default
+# path doesn't exist, set with_cuda_libdir to empty.
+AC_ARG_WITH([cuda-libdir],
+            [AS_HELP_STRING([--with-cuda-libdir=DIR],
+                            [Search for CUDA libraries in DIR])],
+            [],
+            [AS_IF([test -d "$with_cuda"],
+             [with_cuda_libdir=$(dirname $(find -H $with_cuda -name libcuda.so 2> /dev/null) 2> /dev/null)],
+             [with_cuda_libdir=$(dirname $(find -H /usr/local/cuda -name libcuda.so 2> /dev/null) 2> /dev/null)])
+            ])
+
 # Note that CUDA support is off by default.  To turn it on, the user has to
 # request it.  The user can just ask for --with-cuda and it that case we
 # look for the cuda.h file in /usr/local/cuda.  Otherwise, they can give
@@ -72,12 +100,15 @@ AS_IF([test "$with_cuda" = "no" || test "x$with_cuda" = "x"],
                             opal_cuda_incdir="$with_cuda/include"
                             AC_MSG_RESULT([found ($opal_cuda_incdir/cuda.h)])])])])])
 
-dnl We cannot have CUDA support without dlopen support.  HOWEVER, at
-dnl this point in configure, we can't know whether the DL framework
-dnl has been configured or not yet (it likely hasn't, since CUDA is a
-dnl common framework, and likely configured first).  So we have to
-dnl defer this check until later (see the OPAL_CHECK_CUDA_AFTER_OPAL_DL m4
-dnl macro, below).  :-(
+AS_IF([test "$opal_check_cuda_happy" = "yes"],
+    [OAC_CHECK_PACKAGE([cuda],
+                       [$1],
+                       [cuda.h],
+                       [cuda],
+                       [cuMemFree],
+                       [opal_check_cuda_happy="yes"],
+                       [opal_check_cuda_happy="no"])],
+    [])
 
 # We require CUDA IPC support which started in CUDA 4.1. Error
 # out if the support is not there.
@@ -88,9 +119,15 @@ AS_IF([test "$opal_check_cuda_happy" = "yes"],
         [#include <$opal_cuda_incdir/cuda.h>])],
     [])
 
+# If we have CUDA support, check to see if we have support for cuMemCreate memory on host NUMA.
+AS_IF([test "$opal_check_cuda_happy"="yes"],
+    [AC_CHECK_DECL([CU_MEM_LOCATION_TYPE_HOST_NUMA], [CUDA_VMM_SUPPORT=1], [CUDA_VMM_SUPPORT=0],
+        [#include <$opal_cuda_incdir/cuda.h>])],
+    [])
+
 # If we have CUDA support, check to see if we have support for SYNC_MEMOPS
 # which was first introduced in CUDA 6.0.
-AS_IF([test "$opal_check_cuda_happy"="yes"],
+AS_IF([test "$opal_check_cuda_happy" = "yes"],
     [AC_CHECK_DECL([CU_POINTER_ATTRIBUTE_SYNC_MEMOPS], [CUDA_SYNC_MEMOPS=1], [CUDA_SYNC_MEMOPS=0],
         [#include <$opal_cuda_incdir/cuda.h>])],
     [])
@@ -108,9 +145,9 @@ AC_COMPILE_IFELSE(
 
 # If we have CUDA support, check to see if we have support for cuPointerGetAttributes
 # which was first introduced in CUDA 7.0.
-AS_IF([test "$opal_check_cuda_happy"="yes"],
-    AC_CHECK_DECL([cuPointerGetAttributes], [CUDA_GET_ATTRIBUTES=1], [CUDA_GET_ATTRIBUTES=0],
-        [#include <$opal_cuda_incdir/cuda.h>]),
+AS_IF([test "$opal_check_cuda_happy" = "yes"],
+    [AC_CHECK_DECL([cuPointerGetAttributes], [CUDA_GET_ATTRIBUTES=1], [CUDA_GET_ATTRIBUTES=0],
+        [#include <$opal_cuda_incdir/cuda.h>])],
     [])
 
 AC_MSG_CHECKING([if have cuda support])
@@ -124,12 +161,16 @@ else
     CUDA_SUPPORT=0
 fi
 
-OPAL_SUMMARY_ADD([[Miscellaneous]],[[CUDA support]],[opal_cuda], [$opal_check_cuda_happy])
+OPAL_SUMMARY_ADD([Accelerators], [CUDA support], [], [$opal_check_cuda_happy])
 
 AM_CONDITIONAL([OPAL_cuda_support], [test "x$CUDA_SUPPORT" = "x1"])
 AC_DEFINE_UNQUOTED([OPAL_CUDA_SUPPORT],$CUDA_SUPPORT,
                    [Whether we want cuda device pointer support])
 
+AM_CONDITIONAL([OPAL_cuda_vmm_support], [test "x$CUDA_VMM_SUPPORT" = "x1"])
+AC_DEFINE_UNQUOTED([OPAL_CUDA_VMM_SUPPORT],$CUDA_VMM_SUPPORT,
+                   [Whether we have CU_MEM_LOCATION_TYPE_HOST_NUMA support available])
+
 AM_CONDITIONAL([OPAL_cuda_sync_memops], [test "x$CUDA_SYNC_MEMOPS" = "x1"])
 AC_DEFINE_UNQUOTED([OPAL_CUDA_SYNC_MEMOPS],$CUDA_SYNC_MEMOPS,
                    [Whether we have CUDA CU_POINTER_ATTRIBUTE_SYNC_MEMOPS support available])
@@ -144,22 +185,9 @@ AM_CONDITIONAL([OPAL_cuda_gdr_support], [test "x$CUDA_VERSION_60_OR_GREATER" = "
 AC_DEFINE_UNQUOTED([OPAL_CUDA_GDR_SUPPORT],$CUDA_VERSION_60_OR_GREATER,
                    [Whether we have CUDA GDR support available])
 
+CPPFLAGS=${cuda_save_CPPFLAGS}
+LDFLAGS=${cuda_save_LDFLAGS}
+LIBS=${cuda_save_LIBS}
+OPAL_VAR_SCOPE_POP
 ])
 
-dnl
-dnl CUDA support requires DL support (it dynamically opens the CUDA
-dnl library at run time).  But we do not check for OPAL DL support
-dnl until lafter the initial OPAL_CHECK_CUDA is called.  So put the
-dnl CUDA+DL check in a separate macro that can be called after the DL MCA
-dnl framework checks in the top-level configure.ac.
-dnl
-AC_DEFUN([OPAL_CHECK_CUDA_AFTER_OPAL_DL],[
-
-    # We cannot have CUDA support without OPAL DL support.  Error out
-    # if the user wants CUDA but we do not have OPAL DL support.
-    AS_IF([test $OPAL_HAVE_DL_SUPPORT -eq 0 && \
-           test "$opal_check_cuda_happy" = "yes"],
-          [AC_MSG_WARN([--with-cuda was specified, but dlopen support is disabled.])
-           AC_MSG_WARN([You must reconfigure Open MPI with dlopen ("dl") support.])
-           AC_MSG_ERROR([Cannot continue.])])
-])
diff --git a/config/opal_check_icc.m4 b/config/opal_check_icc.m4
deleted file mode 100644
index 9c60fa29701..00000000000
--- a/config/opal_check_icc.m4
+++ /dev/null
@@ -1,60 +0,0 @@
-dnl -*- shell-script -*-
-dnl
-dnl Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-dnl                         University Research and Technology
-dnl                         Corporation.  All rights reserved.
-dnl Copyright (c) 2004-2005 The University of Tennessee and The University
-dnl                         of Tennessee Research Foundation.  All rights
-dnl                         reserved.
-dnl Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-dnl                         University of Stuttgart.  All rights reserved.
-dnl Copyright (c) 2004-2005 The Regents of the University of California.
-dnl                         All rights reserved.
-dnl Copyright (c) 2014      Intel, Inc. All rights reserved.
-dnl $COPYRIGHT$
-dnl
-dnl Additional copyrights may follow
-dnl
-dnl $HEADER$
-dnl
-
-AC_DEFUN([OPAL_CHECK_ICC_VARARGS],[
-dnl
-dnl On EM64T, icc-8.1 before version 8.1.027 segfaulted, since
-dnl va_start was miscompiled...
-dnl
-AC_MSG_CHECKING([whether icc-8.1 for EM64T works with variable arguments])
-AC_TRY_RUN([
-#include <stdarg.h>
-#include <stdio.h>
-#include <stdlib.h>
-
-void func (int c, char * f, ...)
-{
-  va_list arglist;
-  va_start (arglist, f);
-  /* vprintf (f, arglist); */
-  va_end (arglist);
-}
-
-int main ()
-{
-  FILE *f;
-  func (4711, "Help %d [%s]\n", 10, "ten");
-  f=fopen ("conftestval", "w");
-  if (!f) exit (1);
-  return 0;
-}
-
-],[opal_ac_icc_varargs=`test -f conftestval`],[opal_ac_icc_varargs=1],[opal_ac_icc_varargs=1])
-
-if test "$opal_ac_icc_varargs" = "1"; then
-    AC_MSG_WARN([*** Problem running configure test!])
-    AC_MSG_WARN([*** Your icc-8.1 compiler seems to miscompile va_start!])
-    AC_MSG_WARN([*** Please upgrade compiler to at least version 8.1.027])
-    AC_MSG_ERROR([*** Cannot continue.])
-fi
-
-AC_MSG_RESULT([yes])
-
-rm -rf conftest*])dnl
diff --git a/config/opal_check_ident.m4 b/config/opal_check_ident.m4
deleted file mode 100644
index 7f79b68648e..00000000000
--- a/config/opal_check_ident.m4
+++ /dev/null
@@ -1,101 +0,0 @@
-dnl -*- shell-script -*-
-dnl
-dnl Copyright (c) 2007      Sun Microsystems, Inc.  All rights reserved.
-dnl Copyright (c) 2014      Cisco Systems, Inc.  All rights reserved.
-dnl Copyright (c) 2015      Research Organization for Information Science
-dnl                         and Technology (RIST). All rights reserved.
-dnl $COPYRIGHT$
-dnl
-dnl Additional copyrights may follow
-dnl
-dnl $HEADER$
-dnl
-dnl defines:
-dnl   OPAL_$1_USE_PRAGMA_IDENT
-dnl   OPAL_$1_USE_IDENT
-dnl   OPAL_$1_USE_CONST_CHAR_IDENT
-dnl
-
-# OPAL_CHECK_IDENT(compiler-env, compiler-flags,
-# file-suffix, lang) Try to compile a source file containing
-# a #pragma ident, and determine whether the ident was
-# inserted into the resulting object file
-# -----------------------------------------------------------
-AC_DEFUN([OPAL_CHECK_IDENT], [
-    AC_MSG_CHECKING([for $4 ident string support])
-
-    opal_pragma_ident_happy=0
-    opal_ident_happy=0
-    opal_static_const_char_happy=0
-    _OPAL_CHECK_IDENT(
-        [$1], [$2], [$3],
-        [[#]pragma ident], [],
-        [opal_pragma_ident_happy=1
-         opal_message="[#]pragma ident"],
-        _OPAL_CHECK_IDENT(
-            [$1], [$2], [$3],
-            [[#]ident], [],
-            [opal_ident_happy=1
-             opal_message="[#]ident"],
-            _OPAL_CHECK_IDENT(
-                [$1], [$2], [$3],
-                [[#]pragma comment(exestr, ], [)],
-                [opal_pragma_comment_happy=1
-                 opal_message="[#]pragma comment"],
-                [opal_static_const_char_happy=1
-                 opal_message="static const char[[]]"])))
-
-    AC_DEFINE_UNQUOTED([OPAL_$1_USE_PRAGMA_IDENT],
-        [$opal_pragma_ident_happy], [Use #pragma ident strings for $4 files])
-    AC_DEFINE_UNQUOTED([OPAL_$1_USE_IDENT],
-        [$opal_ident_happy], [Use #ident strings for $4 files])
-    AC_DEFINE_UNQUOTED([OPAL_$1_USE_PRAGMA_COMMENT],
-        [$opal_pragma_comment_happy], [Use #pragma comment for $4 files])
-    AC_DEFINE_UNQUOTED([OPAL_$1_USE_CONST_CHAR_IDENT],
-        [$opal_static_const_char_happy], [Use static const char[] strings for $4 files])
-
-    AC_MSG_RESULT([$opal_message])
-
-    unset opal_pragma_ident_happy opal_ident_happy opal_static_const_char_happy opal_message
-])
-
-# _OPAL_CHECK_IDENT(compiler-env, compiler-flags,
-# file-suffix, header_prefix, header_suffix, action-if-success, action-if-fail)
-# Try to compile a source file containing a #-style ident,
-# and determine whether the ident was inserted into the
-# resulting object file
-# -----------------------------------------------------------
-AC_DEFUN([_OPAL_CHECK_IDENT], [
-    eval opal_compiler="\$$1"
-    eval opal_flags="\$$2"
-
-    opal_ident="string_not_coincidentally_inserted_by_the_compiler"
-    cat > conftest.$3 <<EOF
-#define IDENT_MSG "$opal_ident"
-$4 IDENT_MSG $5
-int main(int argc, char** argv);
-int main(int argc, char** argv) { return 0; }
-EOF
-
-    # "strings" won't always return the ident string.  objdump isn't
-    # universal (e.g., OS X doesn't have it), and ...other
-    # complications.  So just try to "grep" for the string in the
-    # resulting object file.  If the ident is found in "strings" or
-    # the grep succeeds, rule that we have this flavor of ident.
-
-    OPAL_LOG_COMMAND([$opal_compiler $opal_flags -c conftest.$3 -o conftest.${OBJEXT}],
-                     [AS_IF([test -f conftest.${OBJEXT}],
-                            [opal_output="`strings -a conftest.${OBJEXT} | grep $opal_ident`"
-                             grep $opal_ident conftest.${OBJEXT} 2>&1 1>/dev/null
-                             opal_status=$?
-                             AS_IF([test "$opal_output" != "" || test "$opal_status" = "0"],
-                                   [$6],
-                                   [$7])],
-                            [OPAL_LOG_MSG([the failed program was:])
-                             OPAL_LOG_FILE([conftest.$3])
-                             $7]
-                            [$7])])
-
-    unset opal_compiler opal_flags opal_output opal_status
-    rm -rf conftest.* conftest${EXEEXT}
-])dnl
diff --git a/config/opal_check_knem.m4 b/config/opal_check_knem.m4
index e3f1b90448a..a45547a47e6 100644
--- a/config/opal_check_knem.m4
+++ b/config/opal_check_knem.m4
@@ -9,6 +9,7 @@ dnl Copyright (c) 2014-2016 Los Alamos National Security, LLC. All rights
 dnl                         reserved.
 dnl Copyright (c) 2015      Research Organization for Information Science
 dnl                         and Technology (RIST). All rights reserved.
+dnl Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
@@ -22,53 +23,45 @@ dnl
 # LDFLAGS, LIBS} as needed and runs action-if-found if there is
 # support, otherwise executes action-if-not-found
 AC_DEFUN([OPAL_CHECK_KNEM],[
-    if test -z "$opal_check_knem_happy" ; then
-	OPAL_VAR_SCOPE_PUSH([opal_check_knem_$1_save_CPPFLAGS opal_check_knem_dir])
-	AC_ARG_WITH([knem],
-		    [AS_HELP_STRING([--with-knem(=DIR)],
-				    [Build knem Linux kernel module support, searching for headers in DIR/include])])
+    OPAL_VAR_SCOPE_PUSH([opal_check_knem_happy opal_check_knem_CPPFLAGS_save])
 
-	OPAL_CHECK_WITHDIR([knem], [$with_knem], [include/knem_io.h])
-	opal_check_knem_$1_save_CPPFLAGS="$CPPFLAGS"
+    AC_ARG_WITH([knem],
+                [AS_HELP_STRING([--with-knem(=DIR)],
+                                [Build knem Linux kernel module support, searching for headers in DIR/include])])
 
-	opal_check_knem_happy=no
+    opal_check_knem_CPPFLAGS_save="${CPPFLAGS}"
 
-	AS_IF([test "$with_knem" != "no"],
-	      [AS_IF([test ! -z "$with_knem" && test "$with_knem" != "yes"],
-		     [opal_check_knem_dir="$with_knem"])
+    AS_IF([test -n "${with_knem}" -a "${with_knem}" != "yes" -a "${with_knem}" != "no"],
+          [$1_CPPFLAGS="-I${with_knem}/include"
+           CPPFLAGS="$CPPFLAGS ${$1_CPPFLAGS}"])
 
-	       _OPAL_CHECK_PACKAGE_HEADER([ompi_check_knem],
-					  [knem_io.h],
-					  [$opal_check_knem_dir],
-					  [opal_check_knem_happy="yes"],
-					  [])],
-	      [])
+    AC_CHECK_HEADER([knem_io.h], [opal_check_knem_happy="yes"], [opal_check_knem_happy="no"])
 
-	CPPFLAGS="$CPPFLAGS $ompi_check_knem_CPPFLAGS"
-
-	# need at least version 0x0000000b
-	if test "$opal_check_knem_happy" = "yes" ; then
-	    AC_PREPROC_IFELSE([AC_LANG_PROGRAM([
+    # need at least version 0x0000000b
+    AS_IF([test "$opal_check_knem_happy" = "yes"],
+          [AC_CACHE_CHECK([for recent version of knem ABI],
+              [opal_check_knem_cv_abi_version],
+              [AC_PREPROC_IFELSE([AC_LANG_PROGRAM([[
 #include <knem_io.h>
-					      ],[
+                                              ]], [[
 #if KNEM_ABI_VERSION < 0xc
 #error "Version less than 0xc"
 #endif
-					      ])],
-			      [opal_check_knem_happy=yes],
-			      [opal_check_knem_happy=no])
-	fi
+                                              ]])],
+                              [opal_check_knem_cv_abi_version=yes],
+                              [opal_check_knem_cv_abi_version=no])])
+               AS_IF([test "${opal_check_knem_cv_abi_version}" = "no"],
+                     [opal_check_knem_happy=no])])
 
-	CPPFLAGS="$opal_check_knem_$1_save_CPPFLAGS"
+    CPPFLAGS="${opal_check_knem_CPPFLAGS_save}"
 
-        OPAL_SUMMARY_ADD([[Transports]],[[Shared memory/Linux KNEM]],[$1],[$opal_check_knem_happy])
-	OPAL_VAR_SCOPE_POP
-    fi
+    OPAL_SUMMARY_ADD([Transports], [Shared memory/Linux KNEM], [], [$opal_check_knem_happy])
 
     AS_IF([test "$opal_check_knem_happy" = "yes"],
-          [$1_CPPFLAGS="[$]$1_CPPFLAGS $ompi_check_knem_CPPFLAGS"
-	   $2],
+          [$2],
           [AS_IF([test ! -z "$with_knem" && test "$with_knem" != "no"],
                  [AC_MSG_ERROR([KNEM support requested but not found.  Aborting])])
            $3])
+
+    OPAL_VAR_SCOPE_POP
 ])dnl
diff --git a/config/opal_check_libnl.m4 b/config/opal_check_libnl.m4
index e246d39ddb2..5f2c711fa03 100644
--- a/config/opal_check_libnl.m4
+++ b/config/opal_check_libnl.m4
@@ -3,6 +3,7 @@ dnl
 dnl Copyright (c) 2015-2017 Research Organization for Information Science
 dnl                         and Technology (RIST). All rights reserved.
 dnl Copyright (c) 2017      Cisco Systems, Inc.  All rights reserved.
+dnl Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
@@ -23,7 +24,7 @@ dnl cannot assume is always present) or we need to look in a
 dnl particular directory for the right libnl3 include files.  For
 dnl now, just hard code the special path into this logic.
 dnl
-dnl _OPAL_CHECK_PACKAGE_LIB() invokes OPAL_LIBNL_SANITY_CHECK() in order
+dnl OAC_CHEC_PACKAGE() invokes OPAL_LIBNL_CHECK_PACKAGE_CALLBACK() in order
 dnl to keep track of which libraries depend on libnl and which libraries
 dnl depend on libnl3.
 dnl Open MPI will not be able to build a component vs a given version of libnl
@@ -37,40 +38,30 @@ AC_DEFUN([OPAL_LIBNL_SANITY_INIT], [
     opal_libnl_version=0
     opal_libnlv1_libs=
     opal_libnlv3_libs=
+
     AC_ARG_WITH([libnl],
                 [AS_HELP_STRING([--with-libnl(=DIR)],
-                                [Directory prefix for libnl (typically only necessary if libnl is installed in a location that the compiler/linker will not search by default)])])
-
-    # The --with options carry two pieces of information: 1) do
-    # you want a specific version of libnl, and 2) where that
-    # version of libnl lives.  For simplicity, let's separate
-    # those two pieces of information.
-    case "$with_libnl" in
-        no)
-            # Nope, don't want it
-            opal_want_libnl=no
-            ;;
-        "")
-            # Just try to build with libnl
-            opal_want_libnl=try
-            opal_libnl_location=
-            ;;
-        yes)
-            # Yes, definitely want it
-            opal_want_libnl=yes
-            opal_libnl_location=
-            ;;
-        *)
-            # Yes, definitely want it -- at a specific location
-            opal_want_libnl=yes
-            opal_libnl_location=$with_libnl
-            ;;
-    esac
+                                [Directory prefix for libnlv3 (typically only necessary if libnl is installed in a location that the compiler/linker will not search by default)])])
+    AC_ARG_WITH([libnl-libdir],
+                [AS_HELP_STRING([--with-libnl-libdir=DIR],
+                                [Directory prefix for libnlv3 libs (typically only necessary if libnl is installed in a location that the compiler/linker will not search by default)])])
+
+    # ugly hack to deal with potentially alternate locations for
+    # libnl3 headers.  Note that if the pkg-config file is found,
+    # this ugly hack won't be used.
+    AS_IF([test -n "$with_libnl_incdir"],
+          [# skip check if someone above set incdir],
+          [test -d "/usr/include/libnl3"],
+          [with_libnl_incdir="/usr/include/libnl3"],
+          [test -d "/usr/local/include/libnl3"],
+          [with_libnl_incdir="/usr/local/include/libnl3"])
+
+    OAC_CHECK_PACKAGE_VERIFY_COMMANDS([[OPAL_LIBNL_CHECK_PACKAGE_CALLBACK]])
 ])
 
 dnl OPAL_LIBNL_SANITY_FAIL_MSG(lib)
 dnl
-dnl Helper to pring a big warning message when we detect a libnl conflict.
+dnl Helper to print a big warning message when we detect a libnl conflict.
 dnl
 dnl --------------------------------------------------------------------
 AC_DEFUN([OPAL_LIBNL_SANITY_FAIL_MSG], [
@@ -79,39 +70,31 @@ AC_DEFUN([OPAL_LIBNL_SANITY_FAIL_MSG], [
     AC_MSG_WARN([Open MPI will therefore skip using lib$1.])
 ])
 
-dnl OPAL_LIBNL_SANITY_CHECK(lib, function, LIBS, libnl_check_ok)
-dnl
-dnl This macro is invoked from OPAL_CHECK_PACKAGE to make sure that
-dnl new libraries that are added to LIBS do not pull in conflicting
-dnl versions of libnl.  E.g., if we already have a library in LIBS
-dnl that pulls in libnl v3, if OPAL_CHECK_PACKAGE is later called that
-dnl pulls in a library that pulls in libnl v1, this macro will detect
-dnl the conflict and will abort configure.
+dnl OPAL_LIBNL_CHECK_PACKAGE_CALLBACK(package name, prefix,
+dnl                   headers, function name,
+dnl                   action if happy, action if not happy)
 dnl
-dnl We abort rather than silently ignore this library simply because
-dnl we are now multiple levels deep in the M4 "call stack", and this
-dnl layer does not know the intent of the user.  Hence, all we can do
-dnl is abort with a hopefully helpful error message (that we aborted
-dnl because Open MPI would have been built in a configuration that is
-dnl known to crash).
-dnl
-dnl --------------------------------------------------------------------
-AC_DEFUN([OPAL_LIBNL_SANITY_CHECK], [
+dnl Callback from OAC_CHECK_PACKAGE to verify that there is
+dnl not a conflict.  Note that CPPFLAGS, LDFLAGS, and LIBS
+dnl are setup to compile/link package.
+AC_DEFUN([OPAL_LIBNL_CHECK_PACKAGE_CALLBACK], [
     OPAL_VAR_SCOPE_PUSH([opal_libnl_sane])
     opal_libnl_sane=1
     case $host in
         *linux*)
-            OPAL_LIBNL_SANITY_CHECK_LINUX($1, $2, $3, opal_libnl_sane)
+            OPAL_LIBNL_SANITY_CHECK_LINUX([$1], [$4], [], [opal_libnl_sane])
             ;;
     esac
 
-    $4=$opal_libnl_sane
+    AS_IF([test ${opal_libnl_sane} -eq 1],
+          [$5], [$6])
+
     OPAL_VAR_SCOPE_POP([opal_libnl_sane])
 ])
 
 dnl
 dnl Simple helper for OPAL_LIBNL_SANITY_CHECK
-dnl $1: library name
+dnl $1: package name
 dnl $2: function
 dnl $3: LIBS
 dnl $4: output variable (1=ok, 0=not ok)
@@ -119,10 +102,12 @@ dnl
 AC_DEFUN([OPAL_LIBNL_SANITY_CHECK_LINUX], [
     OPAL_VAR_SCOPE_PUSH([this_requires_v1 libnl_sane this_requires_v3 ldd_output result_msg])
 
-    AC_LANG_PUSH(C)
+    AS_VAR_PUSHDEF([libnl_check_lib], [opal_libnl_sanity_check_cv_$1])
 
-    AC_MSG_CHECKING([if lib$1 requires libnl v1 or v3])
-    cat > conftest_c.$ac_ext << EOF
+    AC_CACHE_CHECK([if $1 requires libnl v1 or v3],
+        [libnl_check_lib],
+        [AC_LANG_PUSH([C])
+         cat > conftest_c.$ac_ext << EOF
 extern void $2 (void);
 int main(int argc, char *argv[[]]) {
     $2 ();
@@ -130,21 +115,25 @@ int main(int argc, char *argv[[]]) {
 }
 EOF
 
+         result_msg=
+         OPAL_LOG_COMMAND([$CC -o conftest $CFLAGS $CPPFLAGS conftest_c.$ac_ext $LDFLAGS $LIBS $3],
+             [ldd_output=`ldd conftest`
+              AS_IF([echo $ldd_output | grep -q libnl-3.so],
+                    [result_msg="v3"])
+              AS_IF([echo $ldd_output | grep -q libnl.so],
+                    [OPAL_APPEND([result_msg], ["v1"])])
+              AS_IF([test -z "${result_msg}"], [result_msg="none"])],
+             [AC_MSG_WARN([Could not link a simple program with lib $1])])
+         AC_LANG_POP([C])
+         AS_VAR_SET([libnl_check_lib], [${result_msg}])
+         rm -f conftest conftest_c.$ac_ext])
+    AS_VAR_COPY([result_msg], [libnl_check_lib])
     this_requires_v1=0
     this_requires_v3=0
-    result_msg=
-    OPAL_LOG_COMMAND([$CC -o conftest $CFLAGS $CPPFLAGS conftest_c.$ac_ext $LDFLAGS -l$1 $LIBS $3],
-        [ldd_output=`ldd conftest`
-         AS_IF([echo $ldd_output | grep -q libnl-3.so],
-               [this_requires_v3=1
-                result_msg="v3"])
-         AS_IF([echo $ldd_output | grep -q libnl.so],
-               [this_requires_v1=1
-                result_msg="v1 $result_msg"])
-         AC_MSG_RESULT([$result_msg])
-         ],
-        [AC_MSG_WARN([Could not link a simple program with lib $1])
-        ])
+    AS_IF([echo "${result_msg}" | grep -q v1], [this_requires_v1=1])
+    AS_IF([echo "${result_msg}" | grep -q v3], [this_requires_v3=1])
+
+    AS_VAR_POPDEF([libnl_check_lib])
 
     # Assume that our configuration is sane; this may get reset below
     libnl_sane=1
@@ -156,7 +145,7 @@ EOF
 
     # Does this library require both v1 and v3?  If so, fail.
     AS_IF([test $this_requires_v1 -eq 1 && test $this_requires_v3 -eq 1],
-          [AC_MSG_WARN([Unfortunately, lib$1 links to both libnl and libnl-3.])
+          [AC_MSG_WARN([Unfortunately, $1 links to both libnl and libnl-3.])
            OPAL_LIBNL_SANITY_FAIL_MSG($1)
            libnl_sane=0])
 
@@ -176,7 +165,7 @@ EOF
     # v1?  If so, fail.
     AS_IF([test $libnl_sane -eq 1 && test $this_requires_v3 -eq 1],
           [AS_IF([test $opal_libnl_version -eq 1],
-                 [AC_MSG_WARN([libnl version conflict: $opal_libnlv1_libs requires libnl whereas lib$1 requires libnl-3])
+                 [AC_MSG_WARN([libnl version conflict: $opal_libnlv1_libs requires libnl whereas $1 requires libnl-3])
                   OPAL_LIBNL_SANITY_FAIL_MSG($1)
                   libnl_sane=0],
                  [opal_libnlv3_libs="$opal_libnlv3_libs $1"
@@ -184,167 +173,11 @@ EOF
                   opal_libnl_version=3])
           ])
 
-    AC_LANG_POP(C)
-    rm -f conftest conftest_c.$ac_ext
-
     $4=$libnl_sane
 
     OPAL_VAR_SCOPE_POP([ldd_output libnl_sane this_requires_v1 this_requires_v3 result_msg])
 ])
 
-dnl
-dnl Check for libnl-3.
-dnl
-dnl Inputs:
-dnl
-dnl $1: prefix where to look for libnl-3
-dnl $2: var name prefix of _CPPFLAGS and _LDFLAGS and _LIBS
-dnl
-dnl Outputs:
-dnl
-dnl - Set $2_CPPFLAGS necessary to compile with libnl-3
-dnl - Set $2_LDFLAGS necessary to link with libnl-3
-dnl - Set $2_LIBS necessary to link with libnl-3
-dnl - Set OPAL_HAVE_LIBNL3 1 if libnl-3 will be used
-dnl
-AC_DEFUN([OPAL_CHECK_LIBNL_V3],[
-    OPAL_VAR_SCOPE_PUSH([CPPFLAGS_save opal_tmp_CPPFLAGS LIBS_save LDFLAGS_save])
-    AC_MSG_NOTICE([checking for libnl v3])
-
-    AS_IF([test "$opal_want_libnl" != "no"],
-          [AS_IF([test -z "$opal_libnl_location"],
-                 [AC_MSG_CHECKING([for /usr/include/libnl3])
-                  AS_IF([test -d "/usr/include/libnl3"],
-                        [opal_tmp_CPPFLAGS=-I/usr/include/libnl3
-                         opal_libnlv3_happy=1
-                         AC_MSG_RESULT([found])],
-                        [AC_MSG_RESULT([not found])
-                         AC_MSG_CHECKING([for /usr/local/include/libnl3])
-                         AS_IF([test -d "/usr/local/include/libnl3"],
-                               [opal_tmp_CPPFLAGS=-I/usr/local/include/netlink3
-                                opal_libnlv3_happy=1
-                                AC_MSG_RESULT([found])],
-                               [opal_libnlv3_happy=0
-                                AC_MSG_RESULT([not found])])])],
-                 [AC_MSG_CHECKING([for $1/include/libnl3])
-                  AS_IF([test -d "$1/include/libnl3"],
-                        [opal_tmp_CPPFLAGS="-I$1/include/libnl3"
-                         opal_libnlv3_happy=1
-                         AC_MSG_RESULT([found])],
-                        [opal_libnlv3_happy=0
-                         AC_MSG_RESULT([not found])])])
-           CPPFLAGS_save=$CPPFLAGS
-           CPPFLAGS="$opal_tmp_CPPFLAGS $CPPFLAGS"
-
-           # Random note: netlink/version.h is only in libnl v3 - it is not in libnl v1.
-           # Also, nl_recvmsgs_report is only in libnl v3.
-           AS_IF([test $opal_libnlv3_happy -eq 1],
-                 [OPAL_CHECK_PACKAGE([$2],
-                                     [netlink/version.h],
-                                     [nl-3],
-                                     [nl_recvmsgs_report],
-                                     [],
-                                     [$1],
-                                     [],
-                                     [],
-                                     [opal_libnlv3_happy=0])
-
-                  # Note that OPAL_CHECK_PACKAGE is going to add
-                  # -I$dir/include into $2_CPPFLAGS.  But because libnl v3
-                  # puts the headers in $dir/include/libnl3, we need to
-                  # overwrite $2_CPPFLAGS with -I$dir/include/libnl3.  We can do
-                  # this unconditionally; we don't have to check for
-                  # success (checking for success occurs below).
-                  $2_CPPFLAGS=$opal_tmp_CPPFLAGS])
-
-           # If we found libnl-3, we *also* need libnl-route-3
-           LIBS_save=$LIBS
-           LDFLAGS_save=$LDFLAGS
-           AS_IF([test -n "$$2_LDFLAGS"],
-                 [LDFLAGS="$$2_LDFLAGS $LDFLAGS"])
-           AS_IF([test $opal_libnlv3_happy -eq 1],
-                 [AC_SEARCH_LIBS([nl_rtgen_request],
-                                 [nl-route-3],
-                                 [],
-                                 [opal_libnlv3_happy=0])])
-           LIBS=$LIBS_save
-           LDFLAGS=$LDFLAGS_save
-
-           # Just because libnl* is evil, double check that the
-           # netlink/version.h we found was for libnl v3.  As far as we
-           # know, netlink/version.h only first appeared in version
-           # 3... but let's really be sure.
-           AS_IF([test $opal_libnlv3_happy -eq 1],
-                 [AC_MSG_CHECKING([to ensure these really are libnl v3 headers])
-                  AS_IF([test -n "$$2_CPPFLAGS"],
-                        [CPPFLAGS="$$2_CPPFLAGS $CPPFLAGS"])
-                  AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
-#include <netlink/netlink.h>
-#include <netlink/version.h>
-#ifndef LIBNL_VER_MAJ
-#error "LIBNL_VER_MAJ not defined!"
-#endif
-/* to the best of our knowledge, version.h only exists in libnl v3 */
-#if LIBNL_VER_MAJ != 3
-#error "LIBNL_VER_MAJ != 3, I am sad"
-#endif
-                                     ]])],
-                                    [AC_MSG_RESULT([yes])],
-                                    [AC_MSG_RESULT([no])
-                                     opal_libnlv3_happy=0])])
-
-           CPPFLAGS=$CPPFLAGS_save],
-
-          [opal_libnlv3_happy=0])
-
-    # If we found everything
-    AS_IF([test $opal_libnlv3_happy -eq 1],
-          [$2_LIBS="-lnl-3 -lnl-route-3"
-           OPAL_HAVE_LIBNL3=1],
-          [# OPAL_CHECK_PACKAGE(...,nl_recvmsgs_report,...) might have set the variables below
-           # so reset them if libnl v3 cannot be used
-           $2_CPPFLAGS=""
-           $2_LDFLAGS=""
-           $2_LIBS=""])
-
-   OPAL_VAR_SCOPE_POP
-])
-
-dnl
-dnl Check for libnl.
-dnl
-dnl Inputs:
-dnl
-dnl $1: prefix where to look for libnl
-dnl $2: var name prefix of _CPPFLAGS and _LDFLAGS and _LIBS
-dnl
-dnl Outputs:
-dnl
-dnl - Set $2_CPPFLAGS necessary to compile with libnl
-dnl - Set $2_LDFLAGS necessary to link with libnl
-dnl - Set $2_LIBS necessary to link with libnl
-dnl - Set OPAL_HAVE_LIBNL3 0 if libnl will be used
-dnl
-AC_DEFUN([OPAL_CHECK_LIBNL_V1],[
-    AC_MSG_NOTICE([checking for libnl v1])
-
-    AS_IF([test "$opal_want_libnl" != "no"],
-          [OPAL_CHECK_PACKAGE([$2],
-                              [netlink/netlink.h],
-                              [nl],
-                              [nl_connect],
-                              [-lm],
-                              [$1],
-                              [],
-                              [opal_libnlv1_happy=1],
-                              [opal_libnlv1_happy=0])],
-          [opal_libnlv1_happy=0])
-
-    AS_IF([test $opal_libnlv1_happy -eq 1],
-          [$2_LIBS="-lnl -lm"
-           OPAL_HAVE_LIBNL3=0])
-])
-
 dnl
 dnl Summarize libnl and libnl3 usage,
 dnl and abort if conflict is found
diff --git a/config/opal_check_ofi.m4 b/config/opal_check_ofi.m4
index 072d6ab1e7c..ce575e0554e 100644
--- a/config/opal_check_ofi.m4
+++ b/config/opal_check_ofi.m4
@@ -1,8 +1,11 @@
-dnl -*- shell-script -*-
+dnl -*- autoconf -*-
 dnl
 dnl Copyright (c) 2015-2020 Cisco Systems, Inc.  All rights reserved.
 dnl Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights
 dnl                         reserved.
+dnl Copyright (c) 2021-2022 Amazon.com, Inc. or its affiliates.  All Rights reserved.
+dnl Copyright (c) 2023      Triad National Security, LLC. All rights
+dnl                         reserved.
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
@@ -23,42 +26,55 @@ dnl $3: action if OFI API version is < $1
 AC_DEFUN([OPAL_CHECK_OFI_VERSION_GE],[
     OPAL_VAR_SCOPE_PUSH([opal_ofi_ver_ge_save_CPPFLAGS opal_ofi_ver_ge_happy])
 
-    AC_MSG_CHECKING([if OFI API version number is >= $1])
-    opal_ofi_ver_ge_save_CPPFLAGS=$CPPFLAGS
-    CPPFLAGS=$opal_ofi_CPPFLAGS
+    AS_LITERAL_WORD_IF([$1], [], [m4_fatal([OPAL_CHECK_OFI_VERSION_GE called with non-literal first argument])])dnl
+    AS_VAR_PUSHDEF([version_cache_var], [opal_ofi_ver_ge_cv_$1])dnl
+    m4_pushdef([version_pretty_print], [m4_translit([$1], [,], [.])])dnl
+
+    AC_CACHE_CHECK([if OFI API version number is >= version_pretty_print],
+        [version_cache_var],
+        [opal_ofi_ver_ge_save_CPPFLAGS=$CPPFLAGS
+         CPPFLAGS=$opal_ofi_internal_CPPFLAGS
 
-    AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <rdma/fabric.h>]],
-[[
+         AC_COMPILE_IFELSE([AC_LANG_PROGRAM(
+                [[#include <rdma/fabric.h>
+]], [[
 #if !defined(FI_MAJOR_VERSION)
 #error "we cannot check the version -- sad panda"
 #elif FI_VERSION_LT(FI_VERSION(FI_MAJOR_VERSION, FI_MINOR_VERSION), FI_VERSION($1))
 #error "version is too low -- nopes"
 #endif
 ]])],
-                      [opal_ofi_ver_ge_happy=1],
-                      [opal_ofi_ver_ge_happy=0])
+                      [version_cache_var=yes],
+                      [version_cache_var=no])])
 
-    AS_IF([test $opal_ofi_ver_ge_happy -eq 1],
-          [AC_MSG_RESULT([yes])
-           $2],
-          [AC_MSG_RESULT([no])
-           $3])
+    AS_IF([test "${version_cache_var}" = "yes"],
+          [$2],
+          [$3])
 
     CPPFLAGS=$opal_ofi_ver_ge_save_CPPFLAGS
 
+    m4_popdef([version_pretty_print])
+    AS_VAR_POPDEF([version_cache_var])
     OPAL_VAR_SCOPE_POP
 ])dnl
 
-dnl
-dnl _OPAL_CHECK_OFI
+
+dnl OPAL_CHECK_OFI(prefix, [action if found], [action if not found])
 dnl --------------------------------------------------------
 dnl Do the real work of checking for OFI libfabric.
 dnl Upon return:
 dnl
-dnl - opal_ofi_happy: will be "yes" or "no"
-dnl - opal_ofi_{CPPFLAGS|LDFLAGS|LIBS} will be loaded (if relevant)
+dnl opal_ofi_{CPPFLAGS, LDFLAGS, LIBS} and prefix_{CPPFLAGS, LDFLAGS,
+dnl LIBS} will be set as needed.
 dnl
-AC_DEFUN([_OPAL_CHECK_OFI],[
+dnl This macro intentionally leaks opal_ofi_happy = yes/no as well as
+dnl evaluating the action if found / action if not found
+dnl
+AC_DEFUN([OPAL_CHECK_OFI],[
+    OPAL_VAR_SCOPE_PUSH([opal_check_ofi_save_CPPFLAGS opal_check_ofi_save_LDFLAGS opal_check_ofi_save_LIBS opal_check_fi_info_pci])
+
+    m4_ifblank([$1], [m4_fatal([First argument must be set for call to OPAL_CHECK_OFI])])
+
     # Add --with options
     AC_ARG_WITH([libfabric],
                 [AS_HELP_STRING([--with-libfabric=DIR],
@@ -70,113 +86,103 @@ AC_DEFUN([_OPAL_CHECK_OFI],[
     AC_ARG_WITH([ofi],
                 [AS_HELP_STRING([--with-ofi=DIR],
                                 [Specify location of OFI libfabric installation, adding DIR/include to the default search location for libfabric headers, and DIR/lib or DIR/lib64 to the default search location for libfabric libraries.  Error if libfabric support cannot be found.])])
-
     AC_ARG_WITH([ofi-libdir],
                 [AS_HELP_STRING([--with-ofi-libdir=DIR],
                                 [Search for OFI libfabric libraries in DIR])])
 
-    if test "$with_ofi" = ""; then
-         with_ofi=$with_libfabric
-    fi
+    AS_IF([test -z "${with_ofi}"], [with_ofi=${with_libfabric}])
+    AS_IF([test -z "${with_ofi_libdir}"], [with_ofi_libdir=${with_libfabric_libdir}])
 
-    if test "$with_ofi_libdir" = ""; then
-         with_ofi_libdir=$with_libfabric_libdir
-    fi
+    opal_check_ofi_save_CPPFLAGS=${CPPFLAGS}
+    opal_check_ofi_save_LDFLAGS=${LDFLAGS}
+    opal_check_ofi_save_LIBS=${LIBS}
 
-    # Sanity check the --with values
-    OPAL_CHECK_WITHDIR([ofi], [$with_ofi],
-                       [include/rdma/fabric.h])
-    OPAL_CHECK_WITHDIR([ofi-libdir], [$with_ofi_libdir],
-                       [libfabric.*])
-
-    OPAL_VAR_SCOPE_PUSH([opal_check_ofi_save_CPPFLAGS opal_check_ofi_save_LDFLAGS opal_check_ofi_save_LIBS opal_check_fi_info_pci])
-    opal_check_ofi_save_CPPFLAGS=$CPPFLAGS
-    opal_check_ofi_save_LDFLAGS=$LDFLAGS
-    opal_check_ofi_save_LIBS=$LIBS
     opal_check_fi_info_pci=0
 
-    opal_ofi_happy=yes
-    AS_IF([test "$with_ofi" = "no"],
-          [opal_ofi_happy=no])
+    dnl OMPI has used ofi everywhere for some time, but the pkg-config
+    dnl module name is libfabric.  Easier to set the pkg-config module
+    dnl name explicitly than change everything in OMPI.
+    m4_define([ofi_pkgconfig_module], [libfabric])
+    OAC_CHECK_PACKAGE([ofi],
+                      [$1],
+                      [rdma/fabric.h],
+                      [fabric],
+                      [fi_getinfo],
+                      [opal_ofi_happy=yes],
+                      [opal_ofi_happy=no])
 
-    AS_IF([test $opal_ofi_happy = yes],
-          [AC_MSG_CHECKING([looking for OFI libfabric in])
-           AS_IF([test "$with_ofi" != "yes"],
-                 [opal_ofi_dir=$with_ofi
-                  AC_MSG_RESULT([($opal_ofi_dir)])],
-                 [AC_MSG_RESULT([(default search paths)])])
-           AS_IF([test ! -z "$with_ofi_libdir" && \
-                         test "$with_ofi_libdir" != "yes"],
-                 [opal_ofi_libdir=$with_ofi_libdir])
-          ])
+    OPAL_FLAGS_APPEND_UNIQ([CPPFLAGS], [${$1_CPPFLAGS}])
 
     AS_IF([test $opal_ofi_happy = yes],
-          [OPAL_CHECK_PACKAGE([opal_ofi],
-                              [rdma/fabric.h],
-                              [fabric],
-                              [fi_getinfo],
-                              [],
-                              [$opal_ofi_dir],
-                              [$opal_ofi_libdir],
-                              [],
-                              [opal_ofi_happy=no])])
-
-    CPPFLAGS="$CPPFLAGS $opal_ofi_CPPFLAGS"
+          [AC_CHECK_HEADERS([rdma/fi_ext.h])
 
-    AS_IF([test $opal_ofi_happy = yes],
-          [AC_CHECK_MEMBER([struct fi_info.nic],
+           AC_CHECK_MEMBER([struct fi_info.nic],
                            [opal_check_fi_info_pci=1],
                            [opal_check_fi_info_pci=0],
-                           [[#include <rdma/fabric.h>]])])
+                           [[#include <rdma/fabric.h>]])
 
-    AC_DEFINE_UNQUOTED([OPAL_OFI_PCI_DATA_AVAILABLE],
-                       [$opal_check_fi_info_pci],
-                       [check if pci data is available in ofi])
+           AC_DEFINE_UNQUOTED([OPAL_OFI_PCI_DATA_AVAILABLE],
+                              [${opal_check_fi_info_pci}],
+                              [check if pci data is available in ofi])
 
-    AC_CHECK_DECLS([PMIX_PACKAGE_RANK],
-                   [],
-                   [],
-                   [#include <pmix.h>])
+           AC_CHECK_DECLS([FI_OPT_FI_HMEM_P2P],
+                          [], [],
+                          [#include <rdma/fi_endpoint.h>])
 
-    CPPFLAGS=$opal_check_ofi_save_CPPFLAGS
-    LDFLAGS=$opal_check_ofi_save_LDFLAGS
-    LIBS=$opal_check_ofi_save_LIBS
+           AC_CHECK_TYPES([struct fi_ops_mem_monitor], [], [],
+                          [#ifdef HAVE_RDMA_FI_EXT_H
+#include <rdma/fi_ext.h>
+#endif
+                           ])
 
-    AC_SUBST([opal_ofi_CPPFLAGS])
-    AC_SUBST([opal_ofi_LDFLAGS])
-    AC_SUBST([opal_ofi_LIBS])
+           OPAL_FLAGS_APPEND_UNIQ([CPPFLAGS], [${opal_pmix_CPPFLAGS}])
+           AC_CHECK_DECLS([PMIX_PACKAGE_RANK],
+                          [],
+                          [],
+                          [#include <pmix.h>])
 
-    OPAL_SUMMARY_ADD([[Transports]],[[OpenFabrics OFI Libfabric]],[],[$opal_ofi_happy])
+           AC_CHECK_MEMBER([struct fi_mr_attr.iface],
+                           [opal_check_fi_mr_attr_iface=1],
+                           [opal_check_fi_mr_attr_iface=0],
+                           [[#include <rdma/fi_domain.h>]])
 
-    OPAL_VAR_SCOPE_POP
+           AC_DEFINE_UNQUOTED([OPAL_OFI_HAVE_FI_MR_IFACE],
+                              [${opal_check_fi_mr_attr_iface}],
+                              [check if iface avaiable in fi_mr_attr])
 
-    AS_IF([test $opal_ofi_happy = no],
-          [AS_IF([test -n "$with_ofi" && test "$with_ofi" != "no"],
+           AC_CHECK_DECL([FI_HMEM_ROCR],
+                         [opal_check_fi_hmem_rocr=1],
+                         [opal_check_fi_hmem_rocr=0],
+                         [#include <rdma/fi_domain.h>])
+
+           AC_DEFINE_UNQUOTED([OPAL_OFI_HAVE_FI_HMEM_ROCR],
+                              [${opal_check_fi_hmem_rocr}],
+                              [check if FI_HMEM_ROCR avaiable in fi_hmem_iface])
+
+           AC_CHECK_DECL([FI_HMEM_ZE],
+                         [opal_check_fi_hmem_ze=1],
+                         [opal_check_fi_hmem_ze=0],
+                         [#include <rdma/fi_domain.h>])
+
+           AC_DEFINE_UNQUOTED([OPAL_OFI_HAVE_FI_HMEM_ZE],
+                              [${opal_check_fi_hmem_ze}],
+                              [check if FI_HMEM_ZE avaiable in fi_hmem_iface])])
+
+    CPPFLAGS=${opal_check_ofi_save_CPPFLAGS}
+    LDFLAGS=${opal_check_ofi_save_LDFLAGS}
+    LIBS=${opal_check_ofi_save_LIBS}
+
+    dnl for version compare tests
+    opal_ofi_internal_CPPFLAGS="${$1_CPPFLAGS}"
+
+    OPAL_SUMMARY_ADD([Transports], [OpenFabrics OFI Libfabric], [], [${$1_SUMMARY}])
+
+    AS_IF([test "${opal_ofi_happy}" = "yes"],
+          [$2],
+          [AS_IF([test -n "${with_ofi}" && test "${with_ofi}" != "no"],
                  [AC_MSG_WARN([OFI libfabric support requested (via --with-ofi or --with-libfabric), but not found.])
                   AC_MSG_ERROR([Cannot continue.])])
-           ])
-])dnl
-
+           $3])
 
-dnl
-dnl OPAL_CHECK_OFI
-dnl --------------------------------------------------------
-dnl Check to see if OFI libfabric is available.
-dnl
-dnl This is a simple wrapper around _OPAL_CHECK_OFI that just
-dnl ensures to only run the checks once.  We do not use AC_REQUIRE
-dnl because that re-orders the texts and makes ordering in stdout
-dnl quite confusing / difficult to grok.
-dnl
-AC_DEFUN([OPAL_CHECK_OFI],[
-    # Check for OFI libfabric.  Note that $opal_ofi_happy is used in
-    # other configure.m4's to know if OFI/libfabric configured
-    # successfully.  We only need to run the back-end checks once, but
-    # at least emit a "checking..." statement each subsequent time
-    # this macro is invoked so that configure's stdout has
-    # sensible/logical output.
-    AS_IF([test -z "$opal_ofi_happy"],
-        [_OPAL_CHECK_OFI],
-        [AC_MSG_CHECKING([if OFI libfabric is available])
-         AC_MSG_RESULT([$opal_ofi_happy])])
-])
+    OPAL_VAR_SCOPE_POP
+])dnl
diff --git a/config/opal_check_os_flavors.m4 b/config/opal_check_os_flavors.m4
deleted file mode 100644
index f055d949b06..00000000000
--- a/config/opal_check_os_flavors.m4
+++ /dev/null
@@ -1,69 +0,0 @@
-dnl -*- shell-script -*-
-dnl
-dnl Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
-dnl Copyright (c) 2014      Intel, Inc. All rights reserved.
-dnl Copyright (c) 2014      Research Organization for Information Science
-dnl                         and Technology (RIST). All rights reserved.
-dnl
-dnl $COPYRIGHT$
-dnl
-dnl Additional copyrights may follow
-dnl
-dnl $HEADER$
-dnl
-
-# OPAL_CHECK_OS_FLAVOR_SPECIFIC()
-# ----------------------------------------------------
-# Helper macro from OPAL-CHECK-OS-FLAVORS(), below.
-# $1 = macro to look for
-# $2 = suffix of env variable to set with results
-AC_DEFUN([OPAL_CHECK_OS_FLAVOR_SPECIFIC],
-[
-    AC_MSG_CHECKING([$1])
-    AC_COMPILE_IFELSE([AC_LANG_PROGRAM(
-     [[#ifndef $1
-      error: this is not $1
-      #endif
-     ]])],
-                      [opal_found_$2=yes],
-                      [opal_found_$2=no])
-    AC_MSG_RESULT([$opal_found_$2])
-])dnl
-
-# OPAL_CHECK_OS_FLAVORS()
-# ----------------------------------------------------
-# Try to figure out the various OS flavors out there.
-#
-AC_DEFUN([OPAL_CHECK_OS_FLAVORS],
-[
-    OPAL_CHECK_OS_FLAVOR_SPECIFIC([__NetBSD__], [netbsd])
-    OPAL_CHECK_OS_FLAVOR_SPECIFIC([__FreeBSD__], [freebsd])
-    OPAL_CHECK_OS_FLAVOR_SPECIFIC([__OpenBSD__], [openbsd])
-    OPAL_CHECK_OS_FLAVOR_SPECIFIC([__DragonFly__], [dragonfly])
-    OPAL_CHECK_OS_FLAVOR_SPECIFIC([__386BSD__], [386bsd])
-    OPAL_CHECK_OS_FLAVOR_SPECIFIC([__bsdi__], [bsdi])
-    OPAL_CHECK_OS_FLAVOR_SPECIFIC([__APPLE__], [apple])
-    OPAL_CHECK_OS_FLAVOR_SPECIFIC([__linux__], [linux])
-    OPAL_CHECK_OS_FLAVOR_SPECIFIC([__sun__], [sun])
-    AS_IF([test "$opal_found_sun" = "no"],
-          OPAL_CHECK_OS_FLAVOR_SPECIFIC([__sun], [sun]))
-
-    AS_IF([test "$opal_found_sun" = "yes"],
-          [opal_have_solaris=1
-           CFLAGS="$CFLAGS -D_REENTRANT"
-           CPPFLAGS="$CPPFLAGS -D_REENTRANT"],
-          [opal_have_solaris=0])
-    AC_DEFINE_UNQUOTED([OPAL_HAVE_SOLARIS],
-                       [$opal_have_solaris],
-                       [Whether or not we have solaris])
-
-    # check for sockaddr_in (a good sign we have TCP)
-    AC_CHECK_HEADERS([netdb.h netinet/in.h netinet/tcp.h])
-    AC_CHECK_TYPES([struct sockaddr_in],
-                   [opal_found_sockaddr=yes],
-                   [opal_found_sockaddr=no],
-                   [AC_INCLUDES_DEFAULT
-#ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
-#endif])
-])dnl
diff --git a/config/opal_check_package.m4 b/config/opal_check_package.m4
deleted file mode 100644
index dcf4c45bffc..00000000000
--- a/config/opal_check_package.m4
+++ /dev/null
@@ -1,229 +0,0 @@
-dnl -*- shell-script -*-
-dnl
-dnl Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-dnl                         University Research and Technology
-dnl                         Corporation.  All rights reserved.
-dnl Copyright (c) 2004-2005 The University of Tennessee and The University
-dnl                         of Tennessee Research Foundation.  All rights
-dnl                         reserved.
-dnl Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-dnl                         University of Stuttgart.  All rights reserved.
-dnl Copyright (c) 2004-2005 The Regents of the University of California.
-dnl                         All rights reserved.
-dnl Copyright (c) 2012-2017 Cisco Systems, Inc.  All rights reserved.
-dnl Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
-dnl Copyright (c) 2014      Intel, Inc. All rights reserved.
-dnl Copyright (c) 2015-2016 Research Organization for Information Science
-dnl                         and Technology (RIST). All rights reserved.
-dnl $COPYRIGHT$
-dnl
-dnl Additional copyrights may follow
-dnl
-dnl $HEADER$
-dnl
-
-dnl _OPAL_CHECK_PACKAGE_HEADER(prefix, header, dir-prefix,
-dnl                            [action-if-found], [action-if-not-found],
-dnl                            includes)
-dnl --------------------------------------------------------------------
-AC_DEFUN([_OPAL_CHECK_PACKAGE_HEADER], [
-    # This is stolen from autoconf to peek under the covers to get the
-    # cache variable for the library check.  one should not copy this
-    # code into other places unless you want much pain and suffering
-    AS_VAR_PUSHDEF([opal_Header], [ac_cv_header_$2])
-    OPAL_VAR_SCOPE_PUSH([dir_prefix])
-
-    # so this sucks, but there's no way to get through the progression
-    # of header includes without killing off the cache variable and trying
-    # again...
-    unset opal_Header
-
-    # get rid of the trailing slash(es)
-    dir_prefix=$(echo $3 | sed -e 'sX/*$XXg')
-    opal_check_package_header_happy="no"
-    AS_IF([test "$dir_prefix" = "/usr" || \
-           test "$dir_prefix" = "/usr/local"],
-           [ # try as is...
-            AC_MSG_NOTICE([looking for header without includes])
-            AC_CHECK_HEADERS([$2], [opal_check_package_header_happy="yes"], [])
-            AS_IF([test "$opal_check_package_header_happy" = "no"],
-                  [# no go on the as is - reset the cache and try again
-                   unset opal_Header])])
-
-    AS_IF([test "$opal_check_package_header_happy" = "no"],
-          [AS_IF([test "$dir_prefix" != ""],
-                 [$1_CPPFLAGS="$$1_CPPFLAGS -I$dir_prefix/include"
-                  CPPFLAGS="$CPPFLAGS -I$dir_prefix/include"])
-          AC_CHECK_HEADERS([$2], [opal_check_package_header_happy="yes"], [], [$6])
-	  AS_IF([test "$opal_check_package_header_happy" = "yes"], [$4], [$5])],
-          [$4])
-    unset opal_check_package_header_happy
-
-    OPAL_VAR_SCOPE_POP([dir_prefix])
-    AS_VAR_POPDEF([opal_Header])dnl
-])
-
-
-dnl _OPAL_CHECK_PACKAGE_LIB(prefix, library, function, extra-libraries,
-dnl                         dir-prefix, libdir,
-dnl                         [action-if-found], [action-if-not-found]])
-dnl --------------------------------------------------------------------
-AC_DEFUN([_OPAL_CHECK_PACKAGE_LIB], [
-    # This is stolen from autoconf to peek under the covers to get the
-    # cache variable for the library check.  one should not copy this
-    # code into other places unless you want much pain and suffering
-    AS_VAR_PUSHDEF([opal_Lib], [ac_cv_search_$3])
-
-    # see comment above
-    unset opal_Lib
-    opal_check_package_lib_happy="no"
-    AS_IF([test "$6" != ""],
-          [ # libdir was specified - search only there
-           $1_LDFLAGS="$$1_LDFLAGS -L$6"
-           LDFLAGS="$LDFLAGS -L$6"
-           AC_SEARCH_LIBS([$3], [$2],
-                        [opal_check_package_lib_happy="yes"],
-                        [opal_check_package_lib_happy="no"], [$4])
-           AS_IF([test "$opal_check_package_lib_happy" = "no"],
-                 [LDFLAGS="$opal_check_package_$1_save_LDFLAGS"
-                  $1_LDFLAGS="$opal_check_package_$1_orig_LDFLAGS"
-                  unset opal_Lib])],
-          [ # libdir was not specified - go through search path
-           opal_check_package_libdir="$5"
-           AS_IF([test "$opal_check_package_libdir" = "" || \
-                  test "$opal_check_package_libdir" = "/usr" || \
-                  test "$opal_check_package_libdir" = "/usr/local"],
-               [ # try as is...
-                AC_MSG_NOTICE([looking for library without search path])
-                AC_SEARCH_LIBS([$3], [$2],
-                        [opal_check_package_lib_happy="yes"],
-                        [opal_check_package_lib_happy="no"], [$4])
-                AS_IF([test "$opal_check_package_lib_happy" = "no"],
-                    [ # no go on the as is..  see what happens later...
-                     LDFLAGS="$opal_check_package_$1_save_LDFLAGS"
-                     $1_LDFLAGS="$opal_check_package_$1_orig_LDFLAGS"
-                     unset opal_Lib])])
-
-           AS_IF([test "$opal_check_package_lib_happy" = "no"],
-               [AS_IF([test "$opal_check_package_libdir" != ""],
-                    [$1_LDFLAGS="$$1_LDFLAGS -L$opal_check_package_libdir/lib"
-                     LDFLAGS="$LDFLAGS -L$opal_check_package_libdir/lib"
-                     AC_MSG_NOTICE([looking for library in lib])
-                     AC_SEARCH_LIBS([$3], [$2],
-                               [opal_check_package_lib_happy="yes"],
-                               [opal_check_package_lib_happy="no"], [$4])
-                     AS_IF([test "$opal_check_package_lib_happy" = "no"],
-                         [ # no go on the as is..  see what happens later...
-                          LDFLAGS="$opal_check_package_$1_save_LDFLAGS"
-                          $1_LDFLAGS="$opal_check_package_$1_orig_LDFLAGS"
-                          unset opal_Lib])])])
-
-           AS_IF([test "$opal_check_package_lib_happy" = "no"],
-               [AS_IF([test "$opal_check_package_libdir" != ""],
-                    [$1_LDFLAGS="$$1_LDFLAGS -L$opal_check_package_libdir/lib64"
-                     LDFLAGS="$LDFLAGS -L$opal_check_package_libdir/lib64"
-                     AC_MSG_NOTICE([looking for library in lib64])
-                     AC_SEARCH_LIBS([$3], [$2],
-                               [opal_check_package_lib_happy="yes"],
-                               [opal_check_package_lib_happy="no"], [$4])
-                     AS_IF([test "$opal_check_package_lib_happy" = "no"],
-                         [ # no go on the as is..  see what happens later...
-                          LDFLAGS="$opal_check_package_$1_save_LDFLAGS"
-                          $1_LDFLAGS="$opal_check_package_$1_orig_LDFLAGS"
-                          unset opal_Lib])])])])
-
-    AS_IF([test "$opal_check_package_lib_happy" = "yes"],
-          [ # libnl v1 and libnl3 are known to *not* coexist
-            # harmoniously in the same process.  Check to see if this
-            # new package will introduce such a conflict.
-           OPAL_LIBNL_SANITY_CHECK([$2], [$3], [$$1_LIBS],
-                                   [opal_check_package_libnl_check_ok])
-           AS_IF([test $opal_check_package_libnl_check_ok -eq 0],
-                 [opal_check_package_lib_happy=no])
-           ])
-
-    AS_IF([test "$opal_check_package_lib_happy" = "yes"],
-          [ # The result of AC SEARCH_LIBS is cached in $ac_cv_search_[function]
-           AS_IF([test "$ac_cv_search_$3" != "no" &&
-                  test "$ac_cv_search_$3" != "none required"],
-                 [$1_LIBS="$ac_cv_search_$3 $4"],
-                 [$1_LIBS="$4"])
-           $7],
-          [$8])
-
-    AS_VAR_POPDEF([opal_Lib])dnl
-])
-
-
-dnl OPAL_CHECK_PACKAGE(prefix,
-dnl                    header,
-dnl                    library,
-dnl                    function,
-dnl                    extra-libraries,
-dnl                    dir-prefix,
-dnl                    libdir-prefix,
-dnl                    [action-if-found], [action-if-not-found],
-dnl                    includes)
-dnl -----------------------------------------------------------
-dnl Check for package defined by header and libs, and probably
-dnl located in dir-prefix, possibly with libs in libdir-prefix.
-dnl Both dir-prefix and libdir-prefix can be empty.  Will set
-dnl prefix_{CPPFLAGS, LDFLAGS, LIBS} as needed.
-dnl
-dnl The general intent of this macro is to provide finer-grained scoping
-dnl of C preprocessor flags, linker flags, and libraries (as opposed to
-dnl unconditionally adding to the top-level CPFLAGS, LDFLAGS, and LIBS,
-dnl which get used to compile/link *everything*).
-dnl
-dnl Here is a breakdown of the parameters:
-dnl
-dnl * prefix: the macro sets $prefix_CPPFLAGS, $prefix_LDFLAGS, and
-dnl   $prefix_LIBS (and AC_SUBSTs all of them).  For example, if a
-dnl   provider uses this macro to check for a header/library that it
-dnl   needs, it might well set prefix to be its provider name.
-dnl * header_filename: the foo.h file to check for
-dnl * library_name / function_name: check for function function_name in
-dnl   -llibrary_name.  Specifically, for library_name, use the "foo" form,
-dnl   as opposed to "libfoo".
-dnl * extra_libraries: if the library_name you are checking for requires
-dnl   additional -l arguments to link successfully, list them here.
-dnl * dir_prefix: if the header/library is located in a non-standard
-dnl   location (e.g., /opt/foo as opposed to /usr), list it here
-dnl * libdir_prefix: if the library is not under $dir_prefix/lib or
-dnl   $dir_prefix/lib64, list it here.
-dnl * action_if_found: if both the header and library are found and
-dnl   usable, execute action_if_found
-dnl * action_if_not_found: otherwise, execute action_if_not_found
-dnl * extra_includes: if including header_filename requires additional
-dnl   headers to be included first, list them here
-dnl
-dnl The output _CPPFLAGS, _LDFLAGS, and _LIBS can be used to limit the
-dnl scope various flags in Makefiles.
-dnl
-AC_DEFUN([OPAL_CHECK_PACKAGE],[
-    opal_check_package_$1_save_CPPFLAGS="$CPPFLAGS"
-    opal_check_package_$1_save_LDFLAGS="$LDFLAGS"
-    opal_check_package_$1_save_LIBS="$LIBS"
-
-    opal_check_package_$1_orig_CPPFLAGS="$$1_CPPFLAGS"
-    opal_check_package_$1_orig_LDFLAGS="$$1_LDFLAGS"
-    opal_check_package_$1_orig_LIBS="$$1_LIBS"
-
-    _OPAL_CHECK_PACKAGE_HEADER([$1], [$2], [$6],
-          [_OPAL_CHECK_PACKAGE_LIB([$1], [$3], [$4], [$5], [$6], [$7],
-                [opal_check_package_happy="yes"],
-                [opal_check_package_happy="no"])],
-          [opal_check_package_happy="no"],
-          [$10])
-
-    AS_IF([test "$opal_check_package_happy" = "yes"],
-          [$8],
-          [$1_CPPFLAGS="$opal_check_package_$1_orig_CPPFLAGS"
-           $1_LDFLAGS="$opal_check_package_$1_orig_LDFLAGS"
-           $1_LIBS="$opal_check_package_$1_orig_LIBS"
-           $9])
-
-    CPPFLAGS="$opal_check_package_$1_save_CPPFLAGS"
-    LDFLAGS="$opal_check_package_$1_save_LDFLAGS"
-    LIBS="$opal_check_package_$1_save_LIBS"
-])
diff --git a/config/opal_check_portals4.m4 b/config/opal_check_portals4.m4
index 06a69c38a01..a1811832ebd 100644
--- a/config/opal_check_portals4.m4
+++ b/config/opal_check_portals4.m4
@@ -16,6 +16,7 @@ dnl Copyright (c) 2015      Research Organization for Information Science
 dnl                         and Technology (RIST). All rights reserved.
 dnl Copyright (c) 2016      Los Alamos National Security, LLC. All rights
 dnl                         reserved.
+dnl Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
@@ -29,81 +30,61 @@ dnl
 # LDFLAGS, LIBS} as needed and runs action-if-found if there is
 # support, otherwise executes action-if-not-found
 AC_DEFUN([OPAL_CHECK_PORTALS4],[
-    if test -z "$ompi_check_portals4_happy" ; then
-	AC_ARG_WITH([portals4],
-		    [AS_HELP_STRING([--with-portals4(=DIR)],
-				    [Build Portals4 support, optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries])])
-	OPAL_CHECK_WITHDIR([portals4], [$with_portals4], [include/portals4.h])
-	AC_ARG_WITH([portals4-libdir],
-		    [AS_HELP_STRING([--with-portals4-libdir=DIR],
-				    [Search for Portals4 libraries in DIR])])
-	OPAL_CHECK_WITHDIR([portals4-libdir], [$with_portals4_libdir], [libportals.*])
+    OPAL_VAR_SCOPE_PUSH([ompi_check_portals4_happy max_md_size max_va_size])
 
-	ompi_check_portals4_$1_save_CPPFLAGS="$CPPFLAGS"
-	ompi_check_portals4_$1_save_LDFLAGS="$LDFLAGS"
-	ompi_check_portals4_$1_save_LIBS="$LIBS"
+    AC_ARG_WITH([portals4],
+                [AS_HELP_STRING([--with-portals4(=DIR)],
+                                [Build Portals4 support, optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries])])
+    AC_ARG_WITH([portals4-libdir],
+                [AS_HELP_STRING([--with-portals4-libdir=DIR],
+                                [Search for Portals4 libraries in DIR])])
 
-	AS_IF([test "$with_portals4" != "no"],
-              [AS_IF([test ! -z "$with_portals4" && test "$with_portals4" != "yes"],
-                     [ompi_check_portals4_dir="$with_portals4"])
-               AS_IF([test ! -z "$with_portals4_libdir" && test "$with_portals4_libdir" != "yes"],
-                     [ompi_check_portals4_libdir="$with_portals4_libdir"])
+    OAC_CHECK_PACKAGE([portals4],
+                      [$1],
+                      [portals4.h],
+                      [portals],
+                      [PtlLEAppend],
+                      [ompi_check_portals4_happy="yes"],
+                      [ompi_check_portals4_happy="no"])
 
-               OPAL_CHECK_PACKAGE([opal_check_portals4],
-				  [portals4.h],
-				  [portals],
-				  [PtlLEAppend],
-				  [],
-				  [$ompi_check_portals4_dir],
-				  [$ompi_check_portals4_libdir],
-				  [ompi_check_portals4_happy="yes"],
-				  [ompi_check_portals4_happy="no"])],
-              [ompi_check_portals4_happy="no"])
+    max_md_size=0
+    AC_ARG_WITH([portals4-max-md-size],
+                [AS_HELP_STRING([--with-portals4-max-md-size=SIZE],
+                                [Log base 2 of the maximum size in bytes of a memory descriptor.  Should only be set for implementations which do not support binding all of virtual address space.])])
+    AS_IF([test "$with_portals4_max_md_size" = "yes" || test "$with_portals4_max_md_size" = "no"],
+          [AC_MSG_ERROR([--with-portals4-max-md-size requires an integer argument])],
+          [AS_IF([test -n "$with_portals4_max_md_size"],
+                 [max_md_size="$with_portals4_max_md_size"])])
+    AC_DEFINE_UNQUOTED([OPAL_PORTALS4_MAX_MD_SIZE], [$max_md_size],
+                       [Log base 2 of the maximum size in bytes of a memory descriptor.  Set to 0 if MD can bind all of memory.])
 
-	CPPFLAGS="$ompi_check_portals4_$1_save_CPPFLAGS"
-	LDFLAGS="$ompi_check_portals4_$1_save_LDFLAGS"
-	LIBS="$ompi_check_portals4_$1_save_LIBS"
+    max_va_size=0
+    AC_ARG_WITH([portals4-max-va-size],
+                [AS_HELP_STRING([--with-portals4-max-va-size=SIZE],
+                                [Log base 2 of the maximum size in bytes of the user virtual address space.  Should only be set for implementations which do not support binding all of virtual address space.])])
+    AS_IF([test "$with_portals4_max_va_size" = "yes" || test "$with_portals4_max_va_size" = "no"],
+          [AC_MSG_ERROR([--with-portals4-max-va-size requires an integer argument])],
+          [AS_IF([test -n "$with_portals4_max_va_size"],
+                 [max_va_size="$with_portals4_max_va_size"])])
+    AC_DEFINE_UNQUOTED([OPAL_PORTALS4_MAX_VA_SIZE], [$max_va_size],
+                       [Log base 2 of the maximum size in bytes of the user virtual address space.  Set to 0 if MD can bind all of memory.])
 
-	max_md_size=0
-	AC_ARG_WITH([portals4-max-md-size],
-		    [AS_HELP_STRING([--with-portals4-max-md-size=SIZE],
-				    [Log base 2 of the maximum size in bytes of a memory descriptor.  Should only be set for implementations which do not support binding all of virtual address space.])])
-	AS_IF([test "$with_portals4_max_md_size" = "yes" || test "$with_portals4_max_md_size" = "no"],
-              [AC_MSG_ERROR([--with-portals4-max-md-size requires an integer argument])],
-              [AS_IF([test -n "$with_portals4_max_md_size"],
-                     [max_md_size="$with_portals4_max_md_size"])])
-	AC_DEFINE_UNQUOTED([OPAL_PORTALS4_MAX_MD_SIZE], [$max_md_size],
-			   [Log base 2 of the maximum size in bytes of a memory descriptor.  Set to 0 if MD can bind all of memory.])
-
-	max_va_size=0
-	AC_ARG_WITH([portals4-max-va-size],
-		    [AS_HELP_STRING([--with-portals4-max-va-size=SIZE],
-				    [Log base 2 of the maximum size in bytes of the user virtual address space.  Should only be set for implementations which do not support binding all of virtual address space.])])
-	AS_IF([test "$with_portals4_max_va_size" = "yes" || test "$with_portals4_max_va_size" = "no"],
-              [AC_MSG_ERROR([--with-portals4-max-va-size requires an integer argument])],
-              [AS_IF([test -n "$with_portals4_max_va_size"],
-                     [max_va_size="$with_portals4_max_va_size"])])
-	AC_DEFINE_UNQUOTED([OPAL_PORTALS4_MAX_VA_SIZE], [$max_va_size],
-			   [Log base 2 of the maximum size in bytes of the user virtual address space.  Set to 0 if MD can bind all of memory.])
-
-	AS_IF([(test $max_md_size -eq 0 && test $max_va_size -ne 0 ) || (test $max_md_size -ne 0 && test $max_va_size -eq 0 )],
+    AS_IF([(test $max_md_size -eq 0 && test $max_va_size -ne 0 ) || (test $max_md_size -ne 0 && test $max_va_size -eq 0 )],
           [AC_MSG_ERROR([If either --with-portals4-max-md-size or --with-portals4-max-va-size is set, both must be set.])])
-	AS_IF([test $max_md_size -ge $max_va_size],
-              [max_md_size=0
-               max_va_size=0])
-	AS_IF([test $max_md_size -ne 0 && test $max_va_size -ne 0],
-              [AC_MSG_NOTICE([Portals 4 address space size: $max_md_size, $max_va_size])])
+    AS_IF([test $max_md_size -ge $max_va_size],
+          [max_md_size=0
+           max_va_size=0])
+    AS_IF([test $max_md_size -ne 0 && test $max_va_size -ne 0],
+          [AC_MSG_NOTICE([Portals 4 address space size: $max_md_size, $max_va_size])])
 
-	OPAL_SUMMARY_ADD([[Transports]],[[Portals4]],[$1],[$ompi_check_portals4_happy])
-    fi
+    OPAL_SUMMARY_ADD([Transports], [Portals4], [], [${$1_SUMMARY}])
 
     AS_IF([test "$ompi_check_portals4_happy" = "yes"],
-          [$1_LDFLAGS="[$]$1_LDFLAGS $opal_check_portals4_LDFLAGS"
-	   $1_CPPFLAGS="[$]$1_CPPFLAGS $opal_check_portals4_CPPFLAGS"
-	   $1_LIBS="[$]$1_LIBS $opal_check_portals4_LIBS"
-	   $2],
+          [$2],
           [AS_IF([test ! -z "$with_portals4" && test "$with_portals4" != "no"],
                  [AC_MSG_ERROR([Portals4 support requested but not found.  Aborting])])
            $3])
-])
+
+    OPAL_VAR_SCOPE_POP
+])dnl
 
diff --git a/config/opal_check_ps.m4 b/config/opal_check_ps.m4
deleted file mode 100644
index 8a47b1c8343..00000000000
--- a/config/opal_check_ps.m4
+++ /dev/null
@@ -1,44 +0,0 @@
-dnl -*- shell-script -*-
-dnl
-dnl Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-dnl                         University Research and Technology
-dnl                         Corporation.  All rights reserved.
-dnl Copyright (c) 2004-2005 The University of Tennessee and The University
-dnl                         of Tennessee Research Foundation.  All rights
-dnl                         reserved.
-dnl Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-dnl                         University of Stuttgart.  All rights reserved.
-dnl Copyright (c) 2004-2005 The Regents of the University of California.
-dnl                         All rights reserved.
-dnl Copyright (c) 2007      Sun Microsystems, Inc.  All rights reserved.
-dnl Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
-dnl Copyright (c) 2014      Intel, Inc. All rights reserved.
-dnl Copyright (c) 2017      UT-Battelle, LLC. All rights reserved.
-dnl $COPYRIGHT$
-dnl
-dnl Additional copyrights may follow
-dnl
-dnl $HEADER$
-dnl
-
-# See if there is a ps command that will produce the output we are
-# interested in.  If so, then save it away.  Otherwise, the string is
-# set to unknown.
-
-AC_DEFUN([OPAL_PS_FLAVOR_CHECK],[
-AC_MSG_CHECKING([for flavor of ps to use])
-PS_FLAVOR="unknown"
-ps -A -o fname > /dev/null 2>&1
-
-if test "$?" = "0"; then
-     PS_FLAVOR="ps -A -o fname,pid,uid"
-else
-     ps -A -o command > /dev/null 2>&1
-     if test "$?" = "0"; then
-         PS_FLAVOR="ps -A -o command,pid,uid"
-     fi
-fi
-AC_MSG_RESULT([$PS_FLAVOR])
-AC_DEFINE_UNQUOTED([ORTE_CLEAN_PS_CMD], ["$PS_FLAVOR"], [Specific ps command to use in orte-clean])
-])
-
diff --git a/config/opal_check_rocm.m4 b/config/opal_check_rocm.m4
new file mode 100644
index 00000000000..25ac54e438e
--- /dev/null
+++ b/config/opal_check_rocm.m4
@@ -0,0 +1,72 @@
+dnl
+dnl Copyright (C) 2022-2023 Advanced Micro Devices, Inc. All rights reserved.
+dnl $COPYRIGHT$
+dnl
+dnl Additional copyrights may follow
+dnl
+dnl $HEADER$
+dnl
+
+
+# OMPI_CHECK_ROCM(prefix, [action-if-found], [action-if-not-found])
+# --------------------------------------------------------
+# check if ROCM support can be found.  sets prefix_{CPPFLAGS,
+# LDFLAGS, LIBS} as needed and runs action-if-found if there is
+# support, otherwise executes action-if-not-found
+
+
+#
+# Check for ROCm  support
+#
+AC_DEFUN([OPAL_CHECK_ROCM],[
+
+     OPAL_VAR_SCOPE_PUSH([opal_check_rocm_happy rocm_save_CPPFLAGS rocm_save_LDFLAGS rocm_save_LIBS rocm_CPPFLAGS rocm_LDFLAGS rocm_LIBS])
+
+     rocm_save_CPPFLAGS="$CPPFLAGS"
+     rocm_save_LDFLAGS="$LDFLAGS"
+     rocm_save_LIBS="$LIBS"
+
+     # Get some configuration information
+     AC_ARG_WITH([rocm],
+        [AS_HELP_STRING([--with-rocm(=DIR)],
+        [Build ROCm support, optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries])])
+
+
+     AS_IF([ test -n "$with_rocm" && test "$with_rocm" = "yes" ],
+           [ with_rocm="/opt/rocm"] )
+
+     rocm_CPPFLAGS="-D__HIP_PLATFORM_AMD__"
+     rocm_LDFLAGS="-L${with_rocm}/lib/"
+
+     AS_IF([ test -n "$with_rocm" && test "$with_rocm" != "no" ],
+           [ OPAL_APPEND([CPPFLAGS], [$rocm_CPPFLAGS])
+	     OPAL_APPEND([LDFLAGS], [$rocm_LDFLAGS]) ])
+
+     OAC_CHECK_PACKAGE([rocm],
+                       [$1],
+                       [hip/hip_runtime.h],
+                       [amdhip64],
+		       [hipFree],
+                       [opal_check_rocm_happy="yes"],
+                       [opal_check_rocm_happy="no"])
+
+     LDFLAGS="$rocm_save_LDFLAGS"
+     LIBS="$rocm_save_LIBS"
+     CPPFLAGS="$rocm_save_CPPFLAGS"
+
+     AS_IF([ test "$opal_check_rocm_happy" = "yes" ],
+           [ OPAL_APPEND([$1_CPPFLAGS], [$rocm_CPPFLAGS])
+	     AC_DEFINE_UNQUOTED([OPAL_ROCM_SUPPORT], [1], [Enable ROCm support])
+             ROCM_SUPPORT=1 ],
+           [ AC_DEFINE_UNQUOTED([OPAL_ROCM_SUPPORT], [0], [Disable ROCm support])
+             ROCM_SUPPORT=0 ])
+
+     AS_IF([ test "$opal_check_rocm_happy" = "yes" ],
+            [$2],
+            [AS_IF([test -n "$with_rocm" && test "$with_rocm" != "no"],
+                   [AC_MSG_ERROR([ROCm support requested but not found.  Aborting])])
+            $3])
+
+     AM_CONDITIONAL([OPAL_rocm_support], [test "$opal_check_rocm_happy" = "yes"])
+     OPAL_VAR_SCOPE_POP
+])
diff --git a/config/opal_check_ugni.m4 b/config/opal_check_ugni.m4
deleted file mode 100644
index 29a1f543acc..00000000000
--- a/config/opal_check_ugni.m4
+++ /dev/null
@@ -1,82 +0,0 @@
-dnl -*- Mode: Shell-script ; indent-tabs-mode:nil -*-
-dnl
-dnl Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-dnl                         University Research and Technology
-dnl                         Corporation.  All rights reserved.
-dnl Copyright (c) 2004-2005 The University of Tennessee and The University
-dnl                         of Tennessee Research Foundation.  All rights
-dnl                         reserved.
-dnl Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-dnl                         University of Stuttgart.  All rights reserved.
-dnl Copyright (c) 2004-2006 The Regents of the University of California.
-dnl                         All rights reserved.
-dnl Copyright (c) 2006      QLogic Corp. All rights reserved.
-dnl Copyright (c) 2009-2016 Cisco Systems, Inc.  All rights reserved.
-dnl Copyright (c) 2011-2016 Los Alamos National Security, LLC. All rights
-dnl                         reserved.
-dnl Copyright (c) 2014      Intel, Inc. All rights reserved
-dnl Copyright (c) 2015      Research Organization for Information Science
-dnl                         and Technology (RIST). All rights reserved.
-dnl $COPYRIGHT$
-dnl
-dnl Additional copyrights may follow
-dnl
-dnl $HEADER$
-dnl
-
-# OPAL_CHECK_UGNI(prefix, [action-if-found], [action-if-not-found])
-# --------------------------------------------------------
-# check if GNI support can be found.  sets prefix_{CPPFLAGS,
-# LDFLAGS, LIBS} as needed and runs action-if-found if there is
-# support, otherwise executes action-if-not-found
-#
-
-AC_DEFUN([OPAL_CHECK_UGNI], [
-    if test -z "$opal_check_ugni_happy" ; then
-        AC_ARG_WITH([ugni], [AS_HELP_STRING([--with-ugni],
-            [Build support for Cray GNI. Set PKG_CONFIG_PATH env. variable to specify alternate path.])])
-
-        opal_check_ugni_happy="no"
-
-        AS_IF([test "$with_ugni" = "no"],
-              [opal_check_ugni_happy="no"],
-              [PKG_CHECK_MODULES([CRAY_UGNI], [cray-ugni],
-			  [opal_check_ugni_happy="yes"],
-                          [opal_check_ugni_happy="no"])])
-
-        opal_check_ugni_$1_save_CPPFLAGS="$CPPFLAGS"
-        opal_check_ugni_$1_save_LIBS="$LIBS"
-
-        if test "$opal_check_ugni_happy" = "yes" ; then
-            CPPFLAGS="$CPPFLAGS $CRAY_UGNI_CFLAGS"
-            LIBS="$LIBS $CRAY_UGNI_LIBS"
-#    echo "+++++++++++++++++++++++CPPFLAGS",$CPPFLAGS
-#    echo "+++++++++++++++++++++++LDFLAGSS",$LDFLAGS
-#    echo "+++++++++++++++++++++++1_CPPFLAGS",$$1_CPPFLAGS
-#    echo "+++++++++++++++++++++++1_LDFLAGSS",$$1_LDFLAGS
-
-#   sanity checks
-
-            AC_CHECK_HEADER([gni_pub.h],[],AC_MSG_ERROR(['gni_pub.h not found.']))
-            AC_CHECK_FUNCS([GNI_GetJobResInfo])
-
-            CPPFLAGS="$opal_check_ugni_$1_save_CPPFLAGS"
-            LIBS="$opal_check_ugni_$1_save_LIBS"
-        fi
-
-        AS_IF([test "$opal_check_ugni_happy" = "yes" && test "$enable_progress_threads" = "yes"],
-              [AC_MSG_WARN([GNI driver does not currently support progress threads.  Disabling.])
-               opal_check_ugni_happy="no"])
-
-	OPAL_SUMMARY_ADD([[Transports]],[[Cray uGNI (Gemini/Aries)]],[$1],[$opal_check_ugni_happy])
-    fi
-
-    AS_IF([test "$opal_check_ugni_happy" = "yes"],
-          [$1_CPPFLAGS="[$]$1_CPPFLAGS $CRAY_UGNI_CFLAGS"
-	   $1_LIBS="[$]$1_LIBS $CRAY_UGNI_LIBS"
-	   $2],
-          [AS_IF([test ! -z "$with_ugni" && test "$with_ugni" != "no"],
-                 [AC_MSG_ERROR([GNI support requested but not found.  Cannot continue.])])
-           $3])
-
-])
diff --git a/config/opal_check_vendor.m4 b/config/opal_check_vendor.m4
index 578f542d0b1..06fedf89996 100644
--- a/config/opal_check_vendor.m4
+++ b/config/opal_check_vendor.m4
@@ -62,7 +62,7 @@ AC_DEFUN([OPAL_CXX_COMPILER_VENDOR], [
 
 # workaround to avoid syntax error with Autoconf < 2.68:
 m4_ifndef([AC_LANG_DEFINES_PROVIDED],
-	  [m4_define([AC_LANG_DEFINES_PROVIDED])])
+          [m4_define([AC_LANG_DEFINES_PROVIDED])])
 
 # OPAL_IFDEF_IFELSE(symbol, [action-if-defined],
 #                   [action-if-not-defined])
diff --git a/config/opal_check_version.m4 b/config/opal_check_version.m4
deleted file mode 100644
index bcc411733e6..00000000000
--- a/config/opal_check_version.m4
+++ /dev/null
@@ -1,79 +0,0 @@
-dnl -*- shell-script -*-
-dnl
-dnl Copyright (c) 2016      IBM Corporation.  All rights reserved.
-dnl $COPYRIGHT$
-dnl
-dnl Additional copyrights may follow
-dnl
-dnl $HEADER$
-dnl
-
-# Parameters: (use a version "1.1.4rc2" as the example)
-# * prefix
-#   Will export a variable $prefix_version_cmp
-#   - action_if_less         => "less"
-#   - action_if_equal        => "equal"
-#   - action_if_equal_series => "series"
-#   - action_if_greater      => "greater"
-# * version_actual
-#   Actual version string
-# * version_desired
-#   Desired version string to check against
-# * action_if_less
-#   Action to take if the version is strictly less than
-#   "1.1.3" < "1.1.4rc2"
-# * action_if_equal
-#   Action to take if the version matches exactly
-#   "1.1.4rc2" = "1.1.4rc2"
-# * action_if_equal_series
-#   Action to take if the version matches to this series
-#   "1.1.4rc1" ~=~ "1.1.4rc2"
-#   "1.1.4" ~=~ "1.1.4rc2"
-# * action_if_greater
-#   Action to take if the version is strictly greater than
-#   "1.1.5" > "1.1.4rc2"
-#   "2.0" > "1.1.4rc2"
-#
-# See documentation on m4_version_compare and AS_VERSION_COMPARE for more
-# precise definitions
-# OPAL_CHECK_VERSION(prefix, version_actual, version_desired,
-#                    action_if_less, action_if_equal, action_if_equal_series,
-#                    action_if_greater)
-# ----------------------------------------------------
-AC_DEFUN([OPAL_CHECK_VERSION],[
-    version_actual=$2
-    version_desired=$3
-
-    AC_MSG_CHECKING([Checking library version is $version_desired])
-    #
-    # Example: If version_desired=1.1.4 and
-    # version_actual=1.1.3    -> -1
-    # version_actual=1.1.4    ->  0
-    # version_actual=1.1.4rc1 ->  1
-    # version_actual=1.1.5    ->  1 (need further check)
-    #
-    AS_VERSION_COMPARE(["$version_actual"], [$version_desired],
-            [AC_MSG_RESULT([Earlier than expected ($version_actual < $$version_desired)])
-             $1_version_cmp="less"
-             $4],
-            [AC_MSG_RESULT([Equal])
-             $1_version_cmp="equal"
-             $5],
-            [
-             # Need further check to make sure we are < 1.1.5
-             # version_actual=1.1.4rc1 ->  -1
-             # version_actual=1.1.4    ->   0 (caught above)
-             # version_actual=1.1.5    ->   1
-             AS_VERSION_COMPARE(["$version_actual"], [$version_desired"zzzz"],
-                 [AC_MSG_RESULT([Within release series ($version_actual)])
-                  $1_version_cmp="series"
-                  $6],
-                 [AC_MSG_RESULT([Within release series ($version_actual)])
-                  $1_version_cmp="series"
-                  $6],
-                 [AC_MSG_RESULT([Later than expected ($version_actual > $$version_desired)])
-                  $1_version_cmp="greater"
-                  $7]
-             )]
-         )
-])dnl
diff --git a/config/opal_check_xpmem.m4 b/config/opal_check_xpmem.m4
index 0a30263bc6f..f1dc8ef8fc0 100644
--- a/config/opal_check_xpmem.m4
+++ b/config/opal_check_xpmem.m4
@@ -16,6 +16,7 @@
 # Copyright (c) 2014      Intel, Inc. All rights reserved.
 # Copyright (c) 2014-2015 Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -58,7 +59,7 @@ AC_DEFUN([OPAL_CHECK_CRAY_XPMEM],[
           [$1_LDFLAGS="[$]$1_LDFLAGS $CRAY_XPMEM_LIBS"
            $1_CPPFLAGS="[$]$1_CPPFLAGS $CRAY_XPMEM_CFLAGS"
            $1_LIBS="[$]$1_LIBS $CRAY_XPMEM_LIBS"
-           AC_DEFINE_UNQUOTED([HAVE_XPMEM_H], [1],[is xpmem.h available])
+           AC_DEFINE_UNQUOTED([HAVE_XPMEM_H], [1], [is xpmem.h available])
            $2], [$3])
 ])
 
@@ -68,45 +69,45 @@ AC_DEFUN([OPAL_CHECK_CRAY_XPMEM],[
 # LDFLAGS, LIBS} as needed and runs action-if-found if there is
 # support, otherwise executes action-if-not-found
 AC_DEFUN([OPAL_CHECK_XPMEM], [
-    if test -z "$opal_check_xpmem_happy" ; then
-	# check for a cray installed xpmem first
-	OPAL_CHECK_CRAY_XPMEM([opal_check_xpmem],[opal_check_xpmem_happy=yes],[opal_check_xpmem_happy=no])
+    OPAL_VAR_SCOPE_PUSH([opal_check_xpmem_happy])
 
-	if test "$opal_check_xpmem_happy" = no ; then
-	    AC_ARG_WITH([xpmem],
-			[AS_HELP_STRING([--with-xpmem(=DIR)],
-					[Build with XPMEM kernel module support, searching for headers in DIR])])
-	    OPAL_CHECK_WITHDIR([xpmem], [$with_xpmem], [include/xpmem.h])
+    # check for a cray installed xpmem first
+    OPAL_CHECK_CRAY_XPMEM([$1], [opal_check_xpmem_happy=yes], [opal_check_xpmem_happy=no])
+    AS_IF([test "${opal_check_xpmem_happy}" = "no"],
+          [OPAL_CHECK_BASE_XPMEM([$1], [opal_check_xpmem_happy=yes], [opal_check_xpmem_happy=no])])
 
-	    AC_ARG_WITH([xpmem-libdir],
-			[AS_HELP_STRING([--with-xpmem-libdir=DIR],
-					[Search for XPMEM library in DIR])])
-	    OPAL_CHECK_WITHDIR([xpmem-libdir], [$with_xpmem_libdir], [libxpmem.*])
+    OPAL_SUMMARY_ADD([Transports], [Shared memory/XPMEM], [], [$opal_check_xpmem_happy])
 
-	    if test ! "$with_xpmem" = "no" ; then
-		if test ! -z "$with_xpmem" && test "$with_xpmem" != "yes" ; then
-		    opal_check_xpmem_dir="$with_xpmem"
-		fi
+    AS_IF([test "${opal_check_xpmem_happy}" = "yes"], [$2], [$3])
 
-		if test ! -z "$with_xpmem_libdir" && test "$with_xpmem_libdir" != "yes" ; then
-		    opal_check_xpmem_libdir="$with_xpmem_libdir"
-		fi
+    OPAL_VAR_SCOPE_POP
+])dnl
 
-		OPAL_CHECK_PACKAGE([opal_check_xpmem],[xpmem.h],[xpmem],[xpmem_make],[],
-				   [$opal_check_xpmem_dir],[$opal_check_xpmem_libdir], [opal_check_xpmem_happy="yes"], [])
 
-		if test "$opal_check_xpmem_happy" = "no" && test -n "$with_xpmem" && test "$with_xpmem" != "yes" ; then
-		    AC_MSG_ERROR([XPMEM support requested but not found.  Aborting])
-		fi
-	    fi
-	fi
+AC_DEFUN([OPAL_CHECK_BASE_XPMEM], [
+    OPAL_VAR_SCOPE_PUSH([opal_check_xpmem_base_happy])
 
-	OPAL_SUMMARY_ADD([[Transports]],[[Shared memory/XPMEM]],[$1],[$opal_check_xpmem_happy])
-    fi
+    AC_ARG_WITH([xpmem],
+                [AS_HELP_STRING([--with-xpmem(=DIR)],
+                                [Build with XPMEM kernel module support, searching for headers in DIR])])
+    AC_ARG_WITH([xpmem-libdir],
+                [AS_HELP_STRING([--with-xpmem-libdir=DIR],
+                                [Search for XPMEM library in DIR])])
+
+    OAC_CHECK_PACKAGE([xpmem],
+                      [$1],
+                      [xpmem.h],
+                      [xpmem],
+                      [xpmem_make],
+                      [opal_check_xpmem_base_happy="yes"],
+                      [opal_check_xpmem_base_happy="no"])
+
+     AS_IF([test "${opal_check_xpmem_base_happy}" = "yes"],
+           [AC_DEFINE_UNQUOTED([HAVE_XPMEM_H], [1], [is xpmem.h available])
+            $2],
+           [AS_IF([test -n "${with_xpmem}" -a "${with_xpmem}" != "no"],
+                  [AC_MSG_ERROR([XPMEM support requested but not found.  Aborting])])
+            $3])
 
-    AS_IF([test "$opal_check_xpmem_happy" = "yes"], [
-	      $1_CPPFLAGS="[$]$1_CPPFLAGS $opal_check_xpmem_CPPFLAGS"
-	      $1_LDFLAGS="[$]$1_LDFLAGS $opal_check_xpmem_LDFLAGS"
-	      $1_LIBS="[$]$1_LIBS $opal_check_xpmem_LIBS"
-	      $2], [$3])
+    OPAL_VAR_SCOPE_POP
 ])dnl
diff --git a/config/opal_check_ze.m4 b/config/opal_check_ze.m4
new file mode 100644
index 00000000000..d1d47bb67c1
--- /dev/null
+++ b/config/opal_check_ze.m4
@@ -0,0 +1,71 @@
+dnl
+dnl Copyright (C) 2022-2023 Advanced Micro Devices, Inc. All rights reserved.
+dnl Copyright (c) 2023     Triad National Security, LLC. All rights reserved.
+dnl $COPYRIGHT$
+dnl
+dnl Additional copyrights may follow
+dnl
+dnl $HEADER$
+dnl
+
+
+# OPAL_CHECK_ZE(prefix, [action-if-found], [action-if-not-found])
+# --------------------------------------------------------
+# check if Intel ZE support can be found.  sets prefix_{CPPFLAGS,
+# LDFLAGS, LIBS} as needed and runs action-if-found if there is
+# support, otherwise executes action-if-not-found
+
+
+#
+# Check for ZE  support
+#
+AC_DEFUN([OPAL_CHECK_ZE],[
+
+     OPAL_VAR_SCOPE_PUSH([opal_check_ze_happy ze_save_CPPFLAGS ze_save_LDFLAGS ze_save_LIBS ze_CPPFLAGS ze_LDFLAGS ze_LIBS])
+
+     ze_save_CPPFLAGS="$CPPFLAGS"
+     ze_save_LDFLAGS="$LDFLAGS"
+     ze_save_LIBS="$LIBS"
+
+     # Get some configuration information
+     AC_ARG_WITH([ze],
+        [AS_HELP_STRING([--with-ze(=DIR)],
+        [Build Intel ZE support, optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries])])
+
+
+     AS_IF([ test -n "$with_ze" && test "$with_ze" = "yes" ],
+           [ with_ze="/opt/ze"] )
+
+     m4_define([ze_pkgconfig_module], [level-zero])
+     OAC_CHECK_PACKAGE([ze],
+                       [$1],
+                       [level_zero/ze_api.h],
+                       [ze_loader],
+		       [zeInit],
+                       [opal_check_ze_happy="yes"],
+                       [opal_check_ze_happy="no"])
+
+     LDFLAGS="$ze_save_LDFLAGS"
+     LIBS="$ze_save_LIBS"
+     OPAL_APPEND([CPPFLAGS], [${$1_CPPFLAGS}] )
+     OPAL_APPEND([LDFLAGS], [${$1_LDFLAGS}] )
+     OPAL_APPEND([LIBS], [${$1_LIBS}] )
+
+     AS_IF([ test "$opal_check_ze_happy" = "no" ],
+           [ CPPFLAGS="$ze_save_CPPFLAGS"])
+
+     AS_IF([ test "$opal_check_ze_happy" = "yes" ],
+           [ AC_DEFINE_UNQUOTED([OPAL_ZE_SUPPORT], [1], [Enable Intel ZE support])
+             ZE_SUPPORT=1 ],
+           [ AC_DEFINE_UNQUOTED([OPAL_ZE_SUPPORT], [0], [Disable Intel ZE support])
+             ZE_SUPPORT=0 ])
+
+     AS_IF([ test "$opal_check_ze_happy" = "yes" ],
+            [$2],
+            [AS_IF([test -n "$with_ze" && test "$with_ze" != "no"],
+                   [AC_MSG_ERROR([Intel ZE support requested but not found.  Aborting])])
+            $3])
+
+     AM_CONDITIONAL([OPAL_ze_support], [test "$opal_check_ze_happy" = "yes"])
+     OPAL_VAR_SCOPE_POP
+])
diff --git a/config/opal_config_3rdparty.m4 b/config/opal_config_3rdparty.m4
index c80f20e12d7..e931850a9cf 100644
--- a/config/opal_config_3rdparty.m4
+++ b/config/opal_config_3rdparty.m4
@@ -4,7 +4,7 @@ dnl Copyright (c) 2009-2018 Cisco Systems, Inc.  All rights reserved
 dnl Copyright (c) 2013      Los Alamos National Security, LLC.  All rights reserved.
 dnl Copyright (c) 2015-2018 Research Organization for Information Science
 dnl                         and Technology (RIST). All rights reserved.
-dnl Copyright (c) 2020      Amazon.com, Inc. or its affiliates.  All Rights
+dnl Copyright (c) 2020-2021 Amazon.com, Inc. or its affiliates.  All Rights
 dnl                         reserved.
 dnl $COPYRIGHT$
 dnl
@@ -13,44 +13,58 @@ dnl
 dnl $HEADER$
 dnl
 
-dnl OPAL_3RDPARTY_WITH(short package name, long package name, internal supported)
+dnl OPAL_3RDPARTY_WITH(short package name, long package name,
+dnl                    internal supported, disabled ok)
 dnl
 dnl Basic --with-pkg/--with-pkg-libdir handling for 3rd party
 dnl packages, with the big long description of internal/external/path
 dnl handling.
 dnl
 dnl At the end of this macro, with_pkg will contain an empty string or
-dnl a path (implying external).  Further, the shell variable opal_pkg_mode
-dnl will be set to "internal", "external", or "unspecified".  If a path is
-dnl given to --with-pkg, then opal_pkg_mode will be set to external.
+dnl a path (the later implying external).  Further, the shell variable
+dnl opal_pkg_mode will be set to "internal", "external",
+dnl "unspecified", or "disabled".  If a path is given to --with-pkg, then
+dnl opal_pkg_mode will be set to external.  If "internal supported" is
+dnl not defined, then opal_pkg_mode will not be internal.  If
+dnl "disabled ok" is not defined, then opal_pkg_mode will not be
+dnl "disabled".
 dnl
 dnl If m4_ifdef(internal support) does not evaluate to true (ie, at
 dnl autogen time), the references to internal in the help strings will
 dnl be removed and internal will not be a supported option.
 dnl
+dnl If m4_ifval(ddisbaled ok) does not evaluate to true (ie, at autogen
+dnl time), then --without-pkg will not be a valid configure option and
+dnl will raise an error.
+dnl
 dnl $1: short package name
 dnl $2: long pacakage name
 AC_DEFUN([OPAL_3RDPARTY_WITH], [
-    m4_ifdef([$3],
-        [AC_ARG_WITH([$1],
-            [AS_HELP_STRING([--with-$1(=DIR)],
-                           [Build $2 support.  DIR can take one of three values: "internal", "external", or a valid directory name.  "internal" forces Open MPI to use its internal copy of $2.  "external" forces Open MPI to use an external installation of $2.  Supplying a valid directory name also forces Open MPI to use an external installation of $2, and adds DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries. Note that Open MPI no longer supports --without-$1.  If no argument is specified, Open MPI will search default locations for $2 and fall back to an internal version if one is not found.])])
-
-         AC_ARG_WITH([$1-libdir],
-            [AS_HELP_STRING([--with-$1-libdir=DIR],
-             [Search for $2 libraries in DIR.  Should only be used if an external copy of $2 is being used.])])],
-        [AC_ARG_WITH([$1],
-            [AS_HELP_STRING([--with-$1(=DIR)],
-                           [Build $2 support.  DIR can take one of two values:  "external" or a valid directory name.  "external" forces Open MPI to use an external installation of $2.  Supplying a valid directory name also forces Open MPI to use an external installation of $2, and adds DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries. Note that Open MPI no longer supports --without-$1.  If no argument is specified, Open MPI will search default locations for $2 and error if one is not found.])])
+    m4_ifval([$4],
+        [m4_ifdef([$3],
+            [AC_ARG_WITH([$1],
+                [AS_HELP_STRING([--with-$1(=DIR)],
+                                [Build $2 support.  DIR can take one of four values: "internal", "external", "no", or a valid directory name.  "internal" forces Open MPI to use its internal copy of $2.  "external" forces Open MPI to use an external installation of $2.  Supplying a valid directory name also forces Open MPI to use an external installation of $2, and adds DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries. "no" means that Open MPI will not build components that require this package.  If no argument is specified, Open MPI will search default locations for $2 and fall back to an internal version if one is not found.])])],
+            [AC_ARG_WITH([$1],
+                [AS_HELP_STRING([--with-$1(=DIR)],
+                                [Build $2 support.  DIR can take one of three values: "external", "no", or a valid directory name.  "external" forces Open MPI to use an external installation of $2.  Supplying a valid directory name also forces Open MPI to use an external installation of $2, and adds DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries. "no" means that Open MPI will not build components that require this package.  If no argument is specified, Open MPI will search default locations for $2 and error if one is not found.])])])],
+        [m4_ifdef([$3],
+            [AC_ARG_WITH([$1],
+                [AS_HELP_STRING([--with-$1(=DIR)],
+                                [Build $2 support.  DIR can take one of three values: "internal", "external", or a valid directory name.  "internal" forces Open MPI to use its internal copy of $2.  "external" forces Open MPI to use an external installation of $2.  Supplying a valid directory name also forces Open MPI to use an external installation of $2, and adds DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries. Note that Open MPI no longer supports --without-$1.  If no argument is specified, Open MPI will search default locations for $2 and fall back to an internal version if one is not found.])])],
+            [AC_ARG_WITH([$1],
+                [AS_HELP_STRING([--with-$1(=DIR)],
+                                [Build $2 support.  DIR can take one of two values: "external" or a valid directory name.  "external" forces Open MPI to use an external installation of $2.  Supplying a valid directory name also forces Open MPI to use an external installation of $2, and adds DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries. Note that Open MPI no longer supports --without-$1.  If no argument is specified, Open MPI will search default locations for $2 and error if one is not found.])])])])
 
-         AC_ARG_WITH([$1-libdir],
-            [AS_HELP_STRING([--with-$1-libdir=DIR],
-             [Search for $2 libraries in DIR.  Should only be used if an external copy of $2 is being used.])])])
+    AC_ARG_WITH([$1-libdir],
+       [AS_HELP_STRING([--with-$1-libdir=DIR],
+           [Search for $2 libraries in DIR.  Should only be used if an external copy of $2 is being used.])])
 
     # Bozo check
-    AS_IF([test "$with_$1" = "no"],
-          [AC_MSG_WARN([It is not possible to configure Open MPI --without-$1])
-           AC_MSG_ERROR([Cannot continue])])
+    m4_ifval([$4], [],
+        [AS_IF([test "$with_$1" = "no"],
+               [AC_MSG_WARN([It is not possible to configure Open MPI --without-$1])
+                AC_MSG_ERROR([Cannot continue])])])
 
     AS_IF([test "$with_$1_libdir" = "no" -o "$with_$1_libdir" = "yes"],
           [AC_MSG_WARN([yes/no are invalid responses for --with-$1-libdir.  Please specify a path.])
@@ -73,6 +87,8 @@ AC_DEFUN([OPAL_3RDPARTY_WITH], [
                            opal_$1_mode="internal"],
             ["external"], [with_$1=""
                            opal_$1_mode="external"],
+            ["no"],       [with_$1=""
+                           opal_$1_mode="disabled"],
             [""], [opal_$1_mode="unspecified"],
             [opal_$1_mode="external"])
 
diff --git a/config/opal_config_asm.m4 b/config/opal_config_asm.m4
index aed2139a88b..1a458e748f1 100644
--- a/config/opal_config_asm.m4
+++ b/config/opal_config_asm.m4
@@ -9,14 +9,13 @@ dnl Copyright (c) 2004-2006 High Performance Computing Center Stuttgart,
 dnl                         University of Stuttgart.  All rights reserved.
 dnl Copyright (c) 2004-2005 The Regents of the University of California.
 dnl                         All rights reserved.
-dnl Copyright (c) 2008-2018 Cisco Systems, Inc.  All rights reserved.
+dnl Copyright (c) 2008-2021 Cisco Systems, Inc.  All rights reserved.
 dnl Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved.
 dnl Copyright (c) 2015-2018 Research Organization for Information Science
 dnl                         and Technology (RIST).  All rights reserved.
 dnl Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights
 dnl                         reserved.
-dnl Copyright (c) 2017      Amazon.com, Inc. or its affiliates.  All Rights
-dnl                         reserved.
+dnl Copyright (c) 2017-2022 Amazon.com, Inc. or its affiliates.  All Rights reserved.
 dnl Copyright (c) 2020      Google, LLC. All rights reserved.
 dnl Copyright (c) 2020      Intel, Inc.  All rights reserved.
 dnl Copyright (c) 2021      IBM Corporation.  All rights reserved.
@@ -334,7 +333,7 @@ AC_DEFUN([OPAL_CHECK_GCC_ATOMIC_BUILTINS], [
   if test -z "$opal_cv_have___atomic" ; then
     AC_MSG_CHECKING([for 32-bit GCC built-in atomics])
 
-    AC_TRY_LINK([
+    AC_LINK_IFELSE([AC_LANG_PROGRAM([
 #include <stdint.h>
 uint32_t tmp, old = 0;
 uint64_t tmp64, old64 = 0;], [
@@ -342,35 +341,35 @@ __atomic_thread_fence(__ATOMIC_SEQ_CST);
 __atomic_compare_exchange_n(&tmp, &old, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
 __atomic_add_fetch(&tmp, 1, __ATOMIC_RELAXED);
 __atomic_compare_exchange_n(&tmp64, &old64, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
-__atomic_add_fetch(&tmp64, 1, __ATOMIC_RELAXED);],
-		[opal_cv_have___atomic=yes],
-		[opal_cv_have___atomic=no])
+__atomic_add_fetch(&tmp64, 1, __ATOMIC_RELAXED);])],
+                [opal_cv_have___atomic=yes],
+                [opal_cv_have___atomic=no])
 
     AC_MSG_RESULT([$opal_cv_have___atomic])
 
     if test $opal_cv_have___atomic = "yes" ; then
-	AC_MSG_CHECKING([for 64-bit GCC built-in atomics])
+        AC_MSG_CHECKING([for 64-bit GCC built-in atomics])
 
-	AC_TRY_LINK([
+        AC_LINK_IFELSE([AC_LANG_PROGRAM([
 #include <stdint.h>
 uint64_t tmp64, old64 = 0;], [
 __atomic_compare_exchange_n(&tmp64, &old64, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
-__atomic_add_fetch(&tmp64, 1, __ATOMIC_RELAXED);],
-		    [opal_cv_have___atomic_64=yes],
-		    [opal_cv_have___atomic_64=no])
-
-	AC_MSG_RESULT([$opal_cv_have___atomic_64])
-
-	if test $opal_cv_have___atomic_64 = "yes" ; then
-	    AC_MSG_CHECKING([if 64-bit GCC built-in atomics are lock-free])
-	    AC_RUN_IFELSE([AC_LANG_PROGRAM([], [if (!__atomic_is_lock_free (8, 0)) { return 1; }])],
-			  [AC_MSG_RESULT([yes])],
-			  [AC_MSG_RESULT([no])
-			   opal_cv_have___atomic_64=no],
-			  [AC_MSG_RESULT([cannot test -- assume yes (cross compiling)])])
-	fi
+__atomic_add_fetch(&tmp64, 1, __ATOMIC_RELAXED);])],
+                    [opal_cv_have___atomic_64=yes],
+                    [opal_cv_have___atomic_64=no])
+
+        AC_MSG_RESULT([$opal_cv_have___atomic_64])
+
+        if test $opal_cv_have___atomic_64 = "yes" ; then
+            AC_MSG_CHECKING([if 64-bit GCC built-in atomics are lock-free])
+            AC_RUN_IFELSE([AC_LANG_PROGRAM([], [if (!__atomic_is_lock_free (8, 0)) { return 1; }])],
+                          [AC_MSG_RESULT([yes])],
+                          [AC_MSG_RESULT([no])
+                           opal_cv_have___atomic_64=no],
+                          [AC_MSG_RESULT([cannot test -- assume yes (cross compiling)])])
+        fi
     else
-	opal_cv_have___atomic_64=no
+        opal_cv_have___atomic_64=no
     fi
 
     # Check for 128-bit support
@@ -378,6 +377,7 @@ __atomic_add_fetch(&tmp64, 1, __ATOMIC_RELAXED);],
   fi
 ])
 
+
 AC_DEFUN([OPAL_CHECK_C11_CSWAP_INT128], [
   OPAL_VAR_SCOPE_PUSH([atomic_compare_exchange_result atomic_compare_exchange_CFLAGS_save atomic_compare_exchange_LIBS_save])
 
@@ -422,510 +422,6 @@ AC_DEFUN([OPAL_CHECK_C11_CSWAP_INT128], [
   OPAL_VAR_SCOPE_POP
 ])
 
-dnl #################################################################
-dnl
-dnl OPAL_CHECK_ASM_TEXT
-dnl
-dnl Determine how to set current mode as text.
-dnl
-dnl #################################################################
-AC_DEFUN([OPAL_CHECK_ASM_TEXT],[
-    AC_MSG_CHECKING([directive for setting text section])
-    opal_cv_asm_text=""
-    if test "$opal_cv_c_compiler_vendor" = "microsoft" ; then
-        # text section will be brought in with the rest of
-        # header for MS - leave blank for now
-        opal_cv_asm_text=""
-    else
-        case $host in
-            *-aix*)
-                opal_cv_asm_text=[".csect .text[PR]"]
-            ;;
-            *)
-                opal_cv_asm_text=".text"
-            ;;
-        esac
-    fi
-    AC_MSG_RESULT([$opal_cv_asm_text])
-    AC_DEFINE_UNQUOTED([OPAL_ASM_TEXT], ["$opal_cv_asm_text"],
-                       [Assembly directive for setting text section])
-    OPAL_ASM_TEXT="$opal_cv_asm_text"
-    AC_SUBST(OPAL_ASM_TEXT)
-])dnl
-
-
-dnl #################################################################
-dnl
-dnl OPAL_CHECK_ASM_GLOBAL
-dnl
-dnl Sets OPAL_ASM_GLOBAL to the value to prefix global values
-dnl
-dnl I'm sure if I don't have a test for this, there will be some
-dnl dumb platform that uses something else
-dnl
-dnl #################################################################
-AC_DEFUN([OPAL_CHECK_ASM_GLOBAL],[
-    AC_MSG_CHECKING([directive for exporting symbols])
-    opal_cv_asm_global=""
-    if test "$opal_cv_c_compiler_vendor" = "microsoft" ; then
-        opal_cv_asm_global="PUBLIC"
-    else
-        case $host in
-            *)
-                opal_cv_asm_global=".globl"
-            ;;
-        esac
-    fi
-    AC_MSG_RESULT([$opal_cv_asm_global])
-    AC_DEFINE_UNQUOTED([OPAL_ASM_GLOBAL], ["$opal_cv_asm_global"],
-                       [Assembly directive for exporting symbols])
-    OPAL_ASM_GLOBAL="$opal_cv_asm_global"
-    AC_SUBST(OPAL_AS_GLOBAL)
-])dnl
-
-
-dnl #################################################################
-dnl
-dnl OPAL_CHECK_ASM_LSYM
-dnl
-dnl Sets OPAL_ASM_LSYM to the prefix value on a symbol to make it
-dnl an internal label (jump target and whatnot)
-dnl
-dnl We look for L .L $ L$ (in that order) for something that both
-dnl assembles and does not leave a label in the output of nm.  Fall
-dnl back to L if nothing else seems to work :/
-dnl
-dnl #################################################################
-
-# _OPAL_CHECK_ASM_LSYM([variable-to-set])
-# ---------------------------------------
-AC_DEFUN([_OPAL_CHECK_ASM_LSYM],[
-    AC_REQUIRE([AC_PROG_GREP])
-
-    $1="L"
-
-    for sym in L .L $ L$ ; do
-        asm_result=0
-        echo "configure: trying $sym" >&AS_MESSAGE_LOG_FD
-        OPAL_TRY_ASSEMBLE([foobar$opal_cv_asm_label_suffix
-${sym}mytestlabel$opal_cv_asm_label_suffix],
-            [# ok, we succeeded at assembling.  see if we can nm,
-             # throwing the results in a file
-            if $NM conftest.$OBJEXT > conftest.out 2>&AS_MESSAGE_LOG_FD ; then
-                if test "`$GREP mytestlabel conftest.out`" = "" ; then
-                    # there was no symbol...  looks promising to me
-                    $1="$sym"
-                    asm_result=1
-                elif test ["`$GREP ' [Nt] .*mytestlabel' conftest.out`"] = "" ; then
-                    # see if we have a non-global-ish symbol
-                    # but we should see if we can do better.
-                    $1="$sym"
-                fi
-            else
-                # not so much on the NM goodness :/
-                echo "$NM failed.  Output from NM was:" >&AS_MESSAGE_LOG_FD
-                cat conftest.out >&AS_MESSAGE_LOG_FD
-                AC_MSG_WARN([$NM could not read object file])
-            fi
-            ])
-        if test "$asm_result" = "1" ; then
-            break
-        fi
-    done
-    rm -f conftest.out
-    unset asm_result sym
-])
-
-# OPAL_CHECK_ASM_LSYM()
-# ---------------------
-AC_DEFUN([OPAL_CHECK_ASM_LSYM],[
-    AC_REQUIRE([LT_PATH_NM])
-
-    AC_CACHE_CHECK([prefix for lsym labels],
-                   [opal_cv_asm_lsym],
-                   [_OPAL_CHECK_ASM_LSYM([opal_cv_asm_lsym])])
-    AC_DEFINE_UNQUOTED([OPAL_ASM_LSYM], ["$opal_cv_asm_lsym"],
-                       [Assembly prefix for lsym labels])
-    OPAL_ASM_LSYM="$opal_cv_asm_lsym"
-    AC_SUBST(OPAL_ASM_LSYM)
-])dnl
-
-dnl #################################################################
-dnl
-dnl OPAL_CHECK_ASM_PROC
-dnl
-dnl Sets a cv-flag, if the compiler needs a proc/endp-definition to
-dnl link with C.
-dnl
-dnl #################################################################
-AC_DEFUN([OPAL_CHECK_ASM_PROC],[
-    AC_CACHE_CHECK([if .proc/endp is needed],
-                   [opal_cv_asm_need_proc],
-                   [opal_cv_asm_need_proc="no"
-                    OPAL_TRY_ASSEMBLE([
-     .proc mysym
-mysym:
-     .endp mysym],
-                          [opal_cv_asm_need_proc="yes"])
-                    rm -f conftest.out])
-
-    if test "$opal_cv_asm_need_proc" = "yes" ; then
-       opal_cv_asm_proc=".proc"
-       opal_cv_asm_endproc=".endp"
-    else
-       opal_cv_asm_proc="#"
-       opal_cv_asm_endproc="#"
-    fi
-])dnl
-
-
-dnl #################################################################
-dnl
-dnl OPAL_CHECK_ASM_GSYM
-dnl
-dnl Sets OPAL_ASM_GSYM to the prefix value on a symbol to make it
-dnl a global linkable from C.  Basically, an _ or not.
-dnl
-dnl #################################################################
-AC_DEFUN([OPAL_CHECK_ASM_GSYM],[
-    AC_CACHE_CHECK([prefix for global symbol labels],
-                   [opal_cv_asm_gsym],
-                   [_OPAL_CHECK_ASM_GSYM])
-
-    if test "$opal_cv_asm_gsym" = "none" ; then
-       AC_MSG_ERROR([Could not determine global symbol label prefix])
-    fi
-
-    AC_DEFINE_UNQUOTED([OPAL_ASM_GSYM], ["$opal_cv_asm_gsym"],
-                       [Assembly prefix for gsym labels])
-    OPAL_ASM_GSYM="$opal_cv_asm_gsym"
-    AC_SUBST(OPAL_ASM_GSYM)
-
-])
-
-AC_DEFUN([_OPAL_CHECK_ASM_GSYM],[
-    opal_cv_asm_gsym="none"
-
-    for sym in "_" "" "." ; do
-        asm_result=0
-        echo "configure: trying $sym" >&AS_MESSAGE_LOG_FD
-cat > conftest_c.c <<EOF
-#ifdef __cplusplus
-extern "C" {
-#endif
-void gsym_test_func(void);
-#ifdef __cplusplus
-}
-#endif
-int
-main()
-{
-    gsym_test_func();
-    return 0;
-}
-EOF
-        OPAL_TRY_ASSEMBLE([
-$opal_cv_asm_text
-$opal_cv_asm_proc ${sym}gsym_test_func
-$opal_cv_asm_global ${sym}gsym_test_func
-${sym}gsym_test_func${opal_cv_asm_label_suffix}
-$opal_cv_asm_endproc ${sym}gsym_test_func
-            ],
-            [opal_compile="$CC $CFLAGS -I. conftest_c.c -c > conftest.cmpl 2>&1"
-             if AC_TRY_EVAL(opal_compile) ; then
-                # save the warnings
-                 cat conftest.cmpl >&AS_MESSAGE_LOG_FD
-                 opal_link="$CC $CFLAGS conftest_c.$OBJEXT conftest.$OBJEXT -o conftest  $LDFLAGS $LIBS > conftest.link 2>&1"
-                 if AC_TRY_EVAL(opal_link) ; then
-                     # save the warnings
-                     cat conftest.link >&AS_MESSAGE_LOG_FD
-                     asm_result=1
-                 else
-                     cat conftest.link >&AS_MESSAGE_LOG_FD
-                     echo "configure: failed C program was: " >&AS_MESSAGE_LOG_FD
-                     cat conftest_c.c >&AS_MESSAGE_LOG_FD
-                     echo "configure: failed ASM program was: " >&AS_MESSAGE_LOG_FD
-                     cat conftest.s >&AS_MESSAGE_LOG_FD
-                     asm_result=0
-                 fi
-             else
-                # save output and failed program
-                 cat conftest.cmpl >&AS_MESSAGE_LOG_FD
-                 echo "configure: failed C program was: " >&AS_MESSAGE_LOG_FD
-                 cat conftest.c >&AS_MESSAGE_LOG_FD
-                 asm_result=0
-             fi],
-            [asm_result=0])
-        if test "$asm_result" = "1" ; then
-            opal_cv_asm_gsym="$sym"
-            break
-        fi
-    done
-    rm -rf conftest.*
-])dnl
-
-
-dnl #################################################################
-dnl
-dnl OPAL_CHECK_ASM_LABEL_SUFFIX
-dnl
-dnl Sets OPAL_ASM_LABEL_SUFFIX to the value to suffix for labels
-dnl
-dnl I'm sure if I don't have a test for this, there will be some
-dnl dumb platform that uses something else
-dnl
-dnl #################################################################
-AC_DEFUN([OPAL_CHECK_ASM_LABEL_SUFFIX],[
-    AC_MSG_CHECKING([suffix for labels])
-    opal_cv_asm_label_suffix=""
-    case $host in
-        *)
-                opal_cv_asm_label_suffix=":"
-        ;;
-    esac
-    AC_MSG_RESULT([$opal_cv_asm_label_suffix])
-    AC_DEFINE_UNQUOTED([OPAL_ASM_LABEL_SUFFIX], ["$opal_cv_asm_label_suffix"],
-                       [Assembly suffix for labels])
-    OPAL_ASM_LABEL_SUFFIX="$opal_cv_asm_label_suffix"
-    AC_SUBST(OPAL_AS_LABEL_SUFFIX)
-])dnl
-
-
-dnl #################################################################
-dnl
-dnl OPAL_CHECK_ASM_ALIGN_LOG
-dnl
-dnl Sets OPAL_ASM_ALIGN_LOG to 1 if align is specified
-dnl logarithmically, 0 otherwise
-dnl
-dnl #################################################################
-AC_DEFUN([OPAL_CHECK_ASM_ALIGN_LOG],[
-    AC_REQUIRE([LT_PATH_NM])
-    AC_REQUIRE([AC_PROG_GREP])
-
-    AC_CACHE_CHECK([if .align directive takes logarithmic value],
-                   [opal_cv_asm_align_log],
-                   [ OPAL_TRY_ASSEMBLE([        $opal_cv_asm_text
-        .align 4
-        $opal_cv_asm_global foo
-        .byte 1
-        .align 4
-foo$opal_cv_asm_label_suffix
-        .byte 2],
-        [opal_asm_addr=[`$NM conftest.$OBJEXT | $GREP foo | sed -e 's/.*\([0-9a-fA-F][0-9a-fA-F]\).*foo.*/\1/'`]],
-        [opal_asm_addr=""])
-    # test for both 16 and 10 (decimal and hex notations)
-    echo "configure: .align test address offset is $opal_asm_addr" >&AS_MESSAGE_LOG_FD
-    if test "$opal_asm_addr" = "16" || test "$opal_asm_addr" = "10" ; then
-       opal_cv_asm_align_log="yes"
-    else
-        opal_cv_asm_align_log="no"
-    fi])
-
-    if test "$opal_cv_asm_align_log" = "yes" || test "$opal_cv_asm_align_log" = "1" ; then
-        opal_asm_align_log_result=1
-    else
-        opal_asm_align_log_result=0
-    fi
-
-    AC_DEFINE_UNQUOTED([OPAL_ASM_ALIGN_LOG],
-                       [$asm_align_log_result],
-                       [Assembly align directive expects logarithmic value])
-
-    unset omp_asm_addr asm_result
-])dnl
-
-
-dnl #################################################################
-dnl
-dnl OPAL_CHECK_ASM_TYPE
-dnl
-dnl Sets OPAL_ASM_TYPE to the prefix for the function type to
-dnl set a symbol's type as function (needed on ELF for shared
-dnl libraries).  If no .type directive is needed, sets OPAL_ASM_TYPE
-dnl to an empty string
-dnl
-dnl We look for @ \# %
-dnl
-dnl #################################################################
-AC_DEFUN([OPAL_CHECK_ASM_TYPE],[
-        AC_CACHE_CHECK([prefix for function in .type],
-                       [opal_cv_asm_type],
-                       [_OPAL_CHECK_ASM_TYPE])
-
-    AC_DEFINE_UNQUOTED([OPAL_ASM_TYPE], ["$opal_cv_asm_type"],
-                       [How to set function type in .type directive])
-    OPAL_ASM_TYPE="$opal_cv_asm_type"
-    AC_SUBST(OPAL_ASM_TYPE)
-])
-
-AC_DEFUN([_OPAL_CHECK_ASM_TYPE],[
-    opal_cv_asm_type=""
-
-    case "${host}" in
-    *-sun-solaris*)
-        # GCC on solaris seems to accept just about anything, not
-        # that what it defines actually works...  So just hardwire
-        # to the right answer
-        opal_cv_asm_type="#"
-    ;;
-    *)
-        for type  in @ \# % ; do
-            asm_result=0
-            echo "configure: trying $type" >&AS_MESSAGE_LOG_FD
-            OPAL_TRY_ASSEMBLE([     .type mysym, ${type}function
-mysym:],
-                 [opal_cv_asm_type="${type}"
-                    asm_result=1])
-            if test "$asm_result" = "1" ; then
-                break
-            fi
-        done
-    ;;
-    esac
-    rm -f conftest.out
-
-    unset asm_result type
-])dnl
-
-
-dnl #################################################################
-dnl
-dnl OPAL_CHECK_ASM_SIZE
-dnl
-dnl Sets OPAL_ASM_SIZE to 1 if we should set .size directives for
-dnl each function, 0 otherwise.
-dnl
-dnl #################################################################
-AC_DEFUN([OPAL_CHECK_ASM_SIZE],[
-    AC_CACHE_CHECK([if .size is needed],
-                   [opal_cv_asm_need_size],
-                   [opal_cv_asm_need_size="no"
-                    OPAL_TRY_ASSEMBLE([     .size mysym, 1],
-                          [opal_cv_asm_need_size="yes"])
-                    rm -f conftest.out])
-
-    if test "$opal_cv_asm_need_size" = "yes" ; then
-       opal_asm_size=1
-    else
-       opal_asm_size=0
-    fi
-
-    AC_DEFINE_UNQUOTED([OPAL_ASM_SIZE], ["$opal_asm_size"],
-                       [Do we need to give a .size directive])
-    OPAL_ASM_SIZE="$opal_asm_size"
-    AC_SUBST(OPAL_ASM_TYPE)
-    unset asm_result
-])dnl
-
-
-# OPAL_CHECK_ASM_GNU_STACKEXEC(var)
-# ----------------------------------
-# sets shell variable var to the things necessary to
-# disable execable stacks with GAS
-AC_DEFUN([OPAL_CHECK_ASM_GNU_STACKEXEC], [
-    AC_REQUIRE([AC_PROG_GREP])
-
-    AC_CHECK_PROG([OBJDUMP], [objdump], [objdump])
-    AC_CACHE_CHECK([if .note.GNU-stack is needed],
-        [opal_cv_asm_gnu_stack_result],
-        [AS_IF([test "$OBJDUMP" != ""],
-            [ # first, see if a simple C program has it set
-             cat >conftest.c <<EOF
-int testfunc() {return 0; }
-EOF
-             OPAL_LOG_COMMAND([$CC $CFLAGS -c conftest.c -o conftest.$OBJEXT],
-                 [$OBJDUMP -x conftest.$OBJEXT 2>&1 | $GREP '\.note\.GNU-stack' &> /dev/null && opal_cv_asm_gnu_stack_result=yes],
-                 [OPAL_LOG_MSG([the failed program was:], 1)
-                  OPAL_LOG_FILE([conftest.c])
-                  opal_cv_asm_gnu_stack_result=no])
-             if test "$opal_cv_asm_gnu_stack_result" != "yes" ; then
-                 opal_cv_asm_gnu_stack_result="no"
-             fi
-             rm -rf conftest.*],
-            [opal_cv_asm_gnu_stack_result="no"])])
-    if test "$opal_cv_asm_gnu_stack_result" = "yes" ; then
-        opal_cv_asm_gnu_stack=1
-    else
-        opal_cv_asm_gnu_stack=0
-    fi
-])dnl
-
-
-dnl #################################################################
-dnl
-dnl OPAL_CHECK_POWERPC_REG
-dnl
-dnl See if the notation for specifying registers is X (most everyone)
-dnl or rX (OS X)
-dnl
-dnl #################################################################
-AC_DEFUN([OPAL_CHECK_POWERPC_REG],[
-    AC_MSG_CHECKING([if PowerPC registers have r prefix])
-    OPAL_TRY_ASSEMBLE([$opal_cv_asm_text
-        addi 1,1,0],
-        [opal_cv_asm_powerpc_r_reg=0],
-        [OPAL_TRY_ASSEMBLE([$opal_cv_asm_text
-        addi r1,r1,0],
-            [opal_cv_asm_powerpc_r_reg=1],
-            [AC_MSG_ERROR([Can not determine how to use PPC registers])])])
-    if test "$opal_cv_asm_powerpc_r_reg" = "1" ; then
-        AC_MSG_RESULT([yes])
-    else
-        AC_MSG_RESULT([no])
-    fi
-
-    AC_DEFINE_UNQUOTED([OPAL_POWERPC_R_REGISTERS],
-                       [$opal_cv_asm_powerpc_r_reg],
-                       [Whether r notation is used for ppc registers])
-])dnl
-
-
-dnl #################################################################
-dnl
-dnl OPAL_CHECK_POWERPC_64BIT
-dnl
-dnl On some powerpc chips (the PPC970 or G5), the OS usually runs in
-dnl 32 bit mode, even though the hardware can do 64bit things.  If
-dnl the compiler will let us, emit code for 64bit test and set type
-dnl operations (on a long long).
-dnl
-dnl #################################################################
-AC_DEFUN([OPAL_CHECK_POWERPC_64BIT],[
-    if test "$ac_cv_sizeof_long" != "4" ; then
-        # this function should only be called in the 32 bit case
-        AC_MSG_ERROR([CHECK_POWERPC_64BIT called on 64 bit platform.  Internal error.])
-    fi
-    AC_MSG_CHECKING([for 64-bit PowerPC assembly support])
-        case $host in
-            *-darwin*)
-                ppc64_result=0
-                if test "$opal_cv_asm_powerpc_r_reg" = "1" ; then
-                   ldarx_asm="        ldarx r1,r1,r1";
-                else
-                   ldarx_asm="        ldarx 1,1,1";
-                fi
-                OPAL_TRY_ASSEMBLE([$opal_cv_asm_text
-        $ldarx_asm],
-                    [ppc64_result=1],
-                    [ppc64_result=0])
-            ;;
-            *)
-                ppc64_result=0
-            ;;
-        esac
-
-    if test "$ppc64_result" = "1" ; then
-        AC_MSG_RESULT([yes])
-        ifelse([$1],,:,[$1])
-    else
-        AC_MSG_RESULT([no])
-        ifelse([$2],,:,[$2])
-    fi
-
-    unset ppc64_result ldarx_asm
-])dnl
-
 
 dnl #################################################################
 dnl
@@ -1002,20 +498,16 @@ AC_DEFUN([OPAL_CHECK_CMPXCHG16B],[
     OPAL_VAR_SCOPE_POP
 ])dnl
 
+
 dnl #################################################################
 dnl
-dnl OPAL_CHECK_INLINE_GCC
+dnl OPAL_CHECK_INLINE_GCC([action-if-found], [action-if-not-found])
 dnl
 dnl Check if the compiler is capable of doing GCC-style inline
 dnl assembly.  Some compilers emit a warning and ignore the inline
 dnl assembly (xlc on OS X) and compile without error.  Therefore,
 dnl the test attempts to run the emitted code to check that the
-dnl assembly is actually run.  To run this test, one argument to
-dnl the macro must be an assembly instruction in gcc format to move
-dnl the value 0 into the register containing the variable ret.
-dnl For PowerPC, this would be:
-dnl
-dnl   "li %0,0" : "=&r"(ret)
+dnl assembly is actually run.
 dnl
 dnl For testing ia32 assembly, the assembly instruction xaddl is
 dnl tested.  The xaddl instruction is used by some of the atomic
@@ -1024,289 +516,195 @@ dnl some compilers (i.e. earlier versions of Sun Studio 12) do not
 dnl necessarily handle xaddl properly, so that needs to be detected
 dnl during configure time.
 dnl
-dnl DEFINE OPAL_GCC_INLINE_ASSEMBLY to 0 or 1 depending on GCC
+dnl DEFINE OPAL_C_GCC_INLINE_ASSEMBLY to 0 or 1 depending on GCC
 dnl                support
 dnl
 dnl #################################################################
 AC_DEFUN([OPAL_CHECK_INLINE_C_GCC],[
-    assembly="$1"
-    asm_result="unknown"
+    AC_CACHE_CHECK([if $CC supports GCC inline assembly],
+       [opal_cv_asm_gcc_inline_assembly],
+       [OPAL_VAR_SCOPE_PUSH([asm_result opal_gcc_inline_assign OPAL_C_GCC_INLINE_ASSEMBLY])
+
+        asm_result="unknown"
 
-    AC_MSG_CHECKING([if $CC supports GCC inline assembly])
+        opal_gcc_inline_assign=""
+        case "${host}" in
+            x86_64-*x32|i?86-*|x86_64*|amd64*)
+                opal_gcc_inline_assign='"xaddl %1,%0" : "=m"(ret), "+r"(negone) : "m"(ret)'
+                ;;
+            aarch64*)
+                opal_gcc_inline_assign='"mov %0, #0" : "=&r"(ret)'
+                ;;
+            powerpc-*|powerpc64-*|powerpcle-*|powerpc64le-*|rs6000-*|ppc-*)
+                opal_gcc_inline_assign='"1: li %0,0" : "=&r"(ret)'
+                ;;
+        esac
 
-    if test ! "$assembly" = "" ; then
-        AC_RUN_IFELSE([AC_LANG_PROGRAM([AC_INCLUDES_DEFAULT],[[
+        AS_IF([test  "$opal_gcc_inline_assign" != ""],
+            [AC_RUN_IFELSE([AC_LANG_PROGRAM([AC_INCLUDES_DEFAULT],[[
 int ret = 1;
 int negone = -1;
-__asm__ __volatile__ ($assembly);
+__asm__ __volatile__ ($opal_gcc_inline_assign);
 return ret;
-                                                             ]])],
-                      [asm_result="yes"], [asm_result="no"],
-                      [asm_result="unknown"])
-    else
-        assembly="test skipped - assuming no"
-    fi
-
-    # if we're cross compiling, just try to compile and figure good enough
-    if test "$asm_result" = "unknown" ; then
-        AC_LINK_IFELSE([AC_LANG_PROGRAM([AC_INCLUDES_DEFAULT],[[
+                ]])],
+               [asm_result="yes"], [asm_result="no"],
+               [asm_result="unknown"])],
+            [asm_result="no - architecture not supported"])
+
+        # if we're cross compiling, just try to compile and figure good enough
+        AS_IF([test "$asm_result" = "unknown"],
+            [AC_LINK_IFELSE([AC_LANG_PROGRAM([AC_INCLUDES_DEFAULT],[[
 int ret = 1;
 int negone = -1;
-__asm__ __volatile__ ($assembly);
+__asm__ __volatile__ ($opal_gcc_inline_assign);
 return ret;
-                                                              ]])],
-                       [asm_result="yes"], [asm_result="no"])
-    fi
-
-    AC_MSG_RESULT([$asm_result])
+                ]])],
+            [asm_result="yes"], [asm_result="no"])])
+    opal_cv_asm_gcc_inline_assembly="$asm_result"
+    OPAL_VAR_SCOPE_POP])
 
-    if test "$asm_result" = "yes" ; then
-        OPAL_C_GCC_INLINE_ASSEMBLY=1
-        opal_cv_asm_inline_supported="yes"
-    else
-        OPAL_C_GCC_INLINE_ASSEMBLY=0
-    fi
+    AS_IF([test "$opal_cv_asm_gcc_inline_assembly" = "yes"],
+          [OPAL_C_GCC_INLINE_ASSEMBLY=1
+           $1],
+          [OPAL_C_GCC_INLINE_ASSEMBLY=0
+           $2])
 
     AC_DEFINE_UNQUOTED([OPAL_C_GCC_INLINE_ASSEMBLY],
                        [$OPAL_C_GCC_INLINE_ASSEMBLY],
                        [Whether C compiler supports GCC style inline assembly])
-
-    unset OPAL_C_GCC_INLINE_ASSEMBLY assembly asm_result
 ])dnl
 
+
 dnl #################################################################
 dnl
 dnl OPAL_CONFIG_ASM
 dnl
-dnl DEFINE OPAL_ASSEMBLY_ARCH to something in sys/architecture.h
-dnl DEFINE OPAL_ASSEMBLY_FORMAT to string containing correct
-dnl                             format for assembly (not user friendly)
-dnl SUBST OPAL_ASSEMBLY_FORMAT to string containing correct
-dnl                             format for assembly (not user friendly)
+dnl Configure assembly support.  AC_DEFINES the following:
+dnl   - OPAL_C_GCC_INLINE_ASSEMBLY - 1 if C compiler supports
+dnl         GCC-style inline assembly
+dnl   - OPAL_USE_C11_ATOMICS - 1 if atomics implementation should
+dnl         use C11-style atomics
+dnl   - OPAL_USE_GCC_BUILTIN_ATOMICS - 1 if atomics implementation
+dnl         should use GCC built-in style atomics
+dnl   - OPAL_USE_ASM_ATOMICS - 1 if atomics implementation should
+dnl         use inline assembly (using GCC-style inline assembly)
+dnl        for atomics implementation
 dnl
 dnl #################################################################
 AC_DEFUN([OPAL_CONFIG_ASM],[
     AC_REQUIRE([OPAL_SETUP_CC])
-    AC_REQUIRE([OPAL_SETUP_CXX])
-    AC_REQUIRE([AM_PROG_AS])
 
-    AC_ARG_ENABLE([c11-atomics],[AS_HELP_STRING([--enable-c11-atomics],
-                  [Enable use of C11 atomics if available (default: enabled)])])
+    OPAL_VAR_SCOPE_PUSH([atomics_found want_c11_atomics want_gcc_builtin_atomics want_asm_atomics])
+
+    # only assembly style we support today is gcc-style inline
+    # assembly, find out if it works.  We need this even for C11/GCC
+    # builtin atomics cases, because we use inline assembly for
+    # timers, LLSC, and 16 byte compare and swap routines.
+    OPAL_CHECK_INLINE_C_GCC([gcc_inline=1], [gcc_inline=0])
+
+    atomics_found=no
+    want_c11_atomics=0
+    want_gcc_builtin_atomics=0
+    want_asm_atomics=0
+
+    AC_ARG_ENABLE([c11-atomics],
+        [AS_HELP_STRING([--enable-c11-atomics],
+             [Enable use of C11 atomics if available. Note: GCC builtin atomics are currently preferred over C11 atomics. (default: use if available, disabled by default on 64-bit PowerPC)])])
 
     AC_ARG_ENABLE([builtin-atomics],
-      [AS_HELP_STRING([--enable-builtin-atomics],
-         [Enable use of GCC built-in atomics (default: autodetect)])])
-
-    OPAL_CHECK_C11_CSWAP_INT128
-    opal_cv_asm_builtin="BUILTIN_NO"
-    OPAL_CHECK_GCC_ATOMIC_BUILTINS
-
-    if test "x$enable_c11_atomics" != "xno" && test "$opal_cv_c11_supported" = "yes" ; then
-        opal_cv_asm_builtin="BUILTIN_C11"
-        OPAL_CHECK_C11_CSWAP_INT128
-    elif test "x$enable_c11_atomics" = "xyes"; then
-        AC_MSG_WARN([C11 atomics were requested but are not supported])
-        AC_MSG_ERROR([Cannot continue])
-    elif test "$enable_builtin_atomics" = "yes" ; then
-	if test $opal_cv_have___atomic = "yes" ; then
-	   opal_cv_asm_builtin="BUILTIN_GCC"
-	else
-	    AC_MSG_WARN([GCC built-in atomics requested but not found.])
-	    AC_MSG_ERROR([Cannot continue])
-	fi
-    fi
+        [AS_HELP_STRING([--enable-builtin-atomics],
+             [Enable use of GCC built-in atomics. Currently preferred over C11 atomics. (default: use if available, disabled by default on 64-bit PowerPC)])])
 
-        OPAL_CHECK_ASM_PROC
-        OPAL_CHECK_ASM_TEXT
-        OPAL_CHECK_ASM_GLOBAL
-        OPAL_CHECK_ASM_GNU_STACKEXEC
-        OPAL_CHECK_ASM_LABEL_SUFFIX
-        OPAL_CHECK_ASM_GSYM
-        OPAL_CHECK_ASM_LSYM
-        OPAL_CHECK_ASM_TYPE
-        OPAL_CHECK_ASM_SIZE
-        OPAL_CHECK_ASM_ALIGN_LOG
-
-        # find our architecture for purposes of assembly stuff
-        opal_cv_asm_arch="UNSUPPORTED"
-        OPAL_GCC_INLINE_ASSIGN=""
-	if test "$opal_cv_have___atomic_64" ; then
-            OPAL_ASM_SUPPORT_64BIT=1
-	else
-	    OPAL_ASM_SUPPORT_64BIT=0
-	fi
+    AC_ARG_ENABLE([builtin-atomics-for-ppc],
+        [AS_HELP_STRING([--enable-builtin-atomics-for-ppc],
+             [For performance reasons, 64-bit POWER architectures will not use C11 or GCC built-in atomics, even if --enable-c11-atomics is passed to configure.  Enabling this option will re-enable support for both C11 and GCC built-in atomics.])])
 
-        case "${host}" in
-        x86_64-*x32)
-            opal_cv_asm_arch="X86_64"
-            OPAL_ASM_SUPPORT_64BIT=1
-            OPAL_GCC_INLINE_ASSIGN='"xaddl %1,%0" : "=m"(ret), "+r"(negone) : "m"(ret)'
-            ;;
-        i?86-*|x86_64*|amd64*)
-            if test "$ac_cv_sizeof_long" = "4" ; then
-                opal_cv_asm_arch="IA32"
-            else
-                opal_cv_asm_arch="X86_64"
-            fi
-            OPAL_ASM_SUPPORT_64BIT=1
-            OPAL_GCC_INLINE_ASSIGN='"xaddl %1,%0" : "=m"(ret), "+r"(negone) : "m"(ret)'
-            OPAL_CHECK_CMPXCHG16B
-            ;;
+    # See the following github PR and some performance numbers/discussion:
+    # https://github.com/open-mpi/ompi/pull/8649
+    #
+    # This logic is a bit convoluted, but matches existing logic in v4.x.
+    case "${host}" in
+    powerpc-*|powerpc64-*|powerpcle-*|powerpc64le-*|rs6000-*|ppc-*)
+          AS_IF([test "$ac_cv_sizeof_long" = "8" -a "$enable_builtin_atomics_for_ppc" != "yes"],
+                [AS_IF([test "$enable_c11_atomics" != "no" -a "$enable_builtin_atomics" != "no"],
+                       [AC_MSG_NOTICE([Disabling built-in and C11 atomics due to known performance issues on Powerpc])])
+                 AS_IF([test "$enable_c11_atomics" = "yes" -o "$enable_builtin_atomics" = "yes"],
+                       [AC_MSG_WARN([Ignoring --enable-c11-atomics and --enable-builtin-atomics options on POWER.  Set
+--enable-builtin-atomics-for-ppc to re-enable.])])
+                 enable_c11_atomics="no"
+                 enable_builtin_atomics="no"])
+    ;;
+    esac
 
-        aarch64*)
-            opal_cv_asm_arch="ARM64"
-            OPAL_ASM_SUPPORT_64BIT=1
-            OPAL_ASM_ARM_VERSION=8
-            OPAL_GCC_INLINE_ASSIGN='"mov %0, #0" : "=&r"(ret)'
+    # Option 1 for atomics: GCC-style Builtin
+    #
+    # We prefer builtin atomics over C11 atomics because our use of C11 atomics
+    # at this point is broken as it either incurs undue overheads or
+    # requires casts to _Atomic where there should be no casts.
+    AS_IF([test "$atomics_found" = "no" -a "$enable_builtin_atomics" != "no"],
+          [OPAL_CHECK_GCC_ATOMIC_BUILTINS
+           AS_IF([test $opal_cv_have___atomic = "yes"],
+                 [AC_MSG_NOTICE([Using GCC built-in style atomics])
+                  atomics_found="GCC built-in style atomics"
+                  want_gcc_builtin_atomics=1],
+                 [test "$enable_builtin_atomics" = "yes"],
+                 [AC_MSG_WARN([GCC built-in atomics requested but not found.])
+                  AC_MSG_ERROR([Cannot continue])])])
+
+    # Option 2 for atomics: C11
+    #
+    # We currently always disable C11 atomics with the Intel compilers.
+    # We know builds older than 20200310 are broken with respect to
+    # C11 atomics, but have not apparently found a build we are happy
+    # with.  In the future, this should be changed to a check for a
+    # particular Intel version.
+    AS_IF([test "$atomics_found" = "no" -a "$enable_c11_atomics" != "no" -a "$opal_cv_c11_supported" = "yes" -a "$opal_cv_c_compiler_vendor" != "intel"],
+          [AC_MSG_NOTICE([Using C11 atomics])
+           OPAL_CHECK_C11_CSWAP_INT128
+           want_c11_atomics=1
+           atomics_found="C11 atomics"],
+          [test "$enable_c11_atomics" = "yes"],
+          [AC_MSG_WARN([C11 atomics were requested but are not supported])
+           AC_MSG_ERROR([Cannot continue])])
+
+    # Option 3 for atomics: inline assembly
+    AS_IF([test "$atomics_found" = "no" -a "$gcc_inline" = "1"],
+          [case "${host}" in
+           x86_64-*x32|i?86-*|x86_64*|amd64*)
+               AS_IF([test "$ac_cv_sizeof_long" = "8"],
+                     [OPAL_CHECK_CMPXCHG16B
+                      atomics_found="x86_64 assembly"])
             ;;
 
-        armv7*|arm-*-linux-gnueabihf)
-            opal_cv_asm_arch="ARM"
-            OPAL_ASM_SUPPORT_64BIT=1
-            OPAL_ASM_ARM_VERSION=7
-            OPAL_GCC_INLINE_ASSIGN='"mov %0, #0" : "=&r"(ret)'
+            aarch64*)
+                atomics_found="aarch64 assembly"
             ;;
 
-        armv6*)
-            opal_cv_asm_arch="ARM"
-            OPAL_ASM_SUPPORT_64BIT=0
-            OPAL_ASM_ARM_VERSION=6
-            CCASFLAGS="$CCASFLAGS -march=armv7-a"
-            OPAL_GCC_INLINE_ASSIGN='"mov %0, #0" : "=&r"(ret)'
+            powerpc-*|powerpc64-*|powerpcle-*|powerpc64le-*|rs6000-*|ppc-*)
+                AS_IF([test "$ac_cv_sizeof_long" = "8"],
+                      [atomics_found="PowerPC assembly"])
             ;;
+            esac
 
-        powerpc-*|powerpc64-*|powerpcle-*|powerpc64le-*|rs6000-*|ppc-*)
-            OPAL_CHECK_POWERPC_REG
-            if test "$ac_cv_sizeof_long" = "4" ; then
-                opal_cv_asm_arch="POWERPC32"
-
-                # Note that on some platforms (Apple G5), even if we are
-                # compiling in 32 bit mode (and therefore should assume
-                # sizeof(long) == 4), we can use the 64 bit test and set
-                # operations.
-                OPAL_CHECK_POWERPC_64BIT(OPAL_ASM_SUPPORT_64BIT=1)
-            elif test "$ac_cv_sizeof_long" = "8" ; then
-                OPAL_ASM_SUPPORT_64BIT=1
-                opal_cv_asm_arch="POWERPC64"
-            else
-                AC_MSG_ERROR([Could not determine PowerPC word size: $ac_cv_sizeof_long])
-            fi
-            OPAL_GCC_INLINE_ASSIGN='"1: li %0,0" : "=&r"(ret)'
-            ;;
-        *)
-	    if test $opal_cv_have___atomic = "yes" ; then
-		opal_cv_asm_builtin="BUILTIN_GCC"
-	    else
-		AC_MSG_ERROR([No atomic primitives available for $host])
-	    fi
-	    ;;
-        esac
+            AS_IF([test "$atomics_found" != "no"],
+                  [want_asm_atomics=1])
+            AC_MSG_CHECKING([for inline assembly atomics])
+            AC_MSG_RESULT([$atomics_found])])
 
-	if ! test -z "$OPAL_ASM_ARM_VERSION" ; then
-	    AC_DEFINE_UNQUOTED([OPAL_ASM_ARM_VERSION], [$OPAL_ASM_ARM_VERSION],
-                               [What ARM assembly version to use])
-	fi
-
-	if test "$opal_cv_asm_builtin" = "BUILTIN_GCC" ; then
-         AC_DEFINE([OPAL_C_GCC_INLINE_ASSEMBLY], [1],
-		   [Whether C compiler supports GCC style inline assembly])
-	else
-         AC_DEFINE_UNQUOTED([OPAL_ASM_SUPPORT_64BIT],
-			    [$OPAL_ASM_SUPPORT_64BIT],
-			    [Whether we can do 64bit assembly operations or not.  Should not be used outside of the assembly header files])
-         AC_SUBST([OPAL_ASM_SUPPORT_64BIT])
-
-         opal_cv_asm_inline_supported="no"
-         # now that we know our architecture, try to inline assemble
-         OPAL_CHECK_INLINE_C_GCC([$OPAL_GCC_INLINE_ASSIGN])
-
-         # format:
-         #   config_file-text-global-label_suffix-gsym-lsym-type-size-align_log-ppc_r_reg-64_bit-gnu_stack
-         asm_format="default"
-         asm_format="${asm_format}-${opal_cv_asm_text}-${opal_cv_asm_global}"
-         asm_format="${asm_format}-${opal_cv_asm_label_suffix}-${opal_cv_asm_gsym}"
-         asm_format="${asm_format}-${opal_cv_asm_lsym}"
-         asm_format="${asm_format}-${opal_cv_asm_type}-${opal_asm_size}"
-         asm_format="${asm_format}-${opal_asm_align_log_result}"
-         if test "$opal_cv_asm_arch" = "POWERPC32" || test "$opal_cv_asm_arch" = "POWERPC64" ; then
-             asm_format="${asm_format}-${opal_cv_asm_powerpc_r_reg}"
-         else
-             asm_format="${asm_format}-1"
-         fi
-         asm_format="${asm_format}-${OPAL_ASM_SUPPORT_64BIT}"
-         opal_cv_asm_format="${asm_format}-${opal_cv_asm_gnu_stack}"
-         # For the Makefile, need to escape the $ as $$.  Don't display
-         # this version, but make sure the Makefile gives the right thing
-         # when regenerating the files because the base has been touched.
-         OPAL_ASSEMBLY_FORMAT=`echo "$opal_cv_asm_format" | sed -e 's/\\\$/\\\$\\\$/'`
-
-        AC_MSG_CHECKING([for assembly format])
-        AC_MSG_RESULT([$opal_cv_asm_format])
-        AC_DEFINE_UNQUOTED([OPAL_ASSEMBLY_FORMAT], ["$OPAL_ASSEMBLY_FORMAT"],
-                           [Format of assembly file])
-        AC_SUBST([OPAL_ASSEMBLY_FORMAT])
-      fi # if opal_cv_asm_builtin = BUILTIN_GCC
-
-    result="OPAL_$opal_cv_asm_arch"
-    OPAL_ASSEMBLY_ARCH="$opal_cv_asm_arch"
-    AC_MSG_CHECKING([for assembly architecture])
-    AC_MSG_RESULT([$opal_cv_asm_arch])
-    AC_DEFINE_UNQUOTED([OPAL_ASSEMBLY_ARCH], [$result],
-        [Architecture type of assembly to use for atomic operations and CMA])
-    AC_SUBST([OPAL_ASSEMBLY_ARCH])
-
-    # Check for RDTSCP support
-    result=0
-    AS_IF([test "$opal_cv_asm_arch" = "X86_64" || test "$opal_cv_asm_arch" = "IA32"],
-          [AC_MSG_CHECKING([for RDTSCP assembly support])
-           AC_LANG_PUSH([C])
-           AC_TRY_RUN([[
-int main(int argc, char* argv[])
-{
-  unsigned int rax, rdx;
-  __asm__ __volatile__ ("rdtscp\n": "=a" (rax), "=d" (rdx):: "%rax", "%rdx");
-  return 0;
-}
-           ]],
-           [result=1
-            AC_MSG_RESULT([yes])],
-           [AC_MSG_RESULT([no])],
-           [#cross compile not supported
-            AC_MSG_RESULT(["no (cross compiling)"])])
-           AC_LANG_POP([C])])
-    AC_DEFINE_UNQUOTED([OPAL_ASSEMBLY_SUPPORTS_RDTSCP], [$result],
-                       [Whether we have support for RDTSCP instruction])
-
-    result="OPAL_$opal_cv_asm_builtin"
-    OPAL_ASSEMBLY_BUILTIN="$opal_cv_asm_builtin"
-    AC_MSG_CHECKING([for builtin atomics])
-    AC_MSG_RESULT([$opal_cv_asm_builtin])
-    AC_DEFINE_UNQUOTED([OPAL_ASSEMBLY_BUILTIN], [$result],
-        [Whether to use builtin atomics])
-    AC_SUBST([OPAL_ASSEMBLY_BUILTIN])
-
-    OPAL_ASM_FIND_FILE
-
-    unset result asm_format
-])dnl
+    AS_IF([test "$atomics_found" = "no"],
+          [AC_MSG_ERROR([No usable atomics implementation found.  Cannot continue.])])
 
+    AC_DEFINE_UNQUOTED([OPAL_USE_C11_ATOMICS],
+        [$want_c11_atomics],
+        [Whether to use C11 atomics for atomics implementation])
+    AC_DEFINE_UNQUOTED([OPAL_USE_GCC_BUILTIN_ATOMICS],
+        [$want_gcc_builtin_atomics],
+        [Whether to use GCC-style built-in atomics for atomics implementation])
+    AC_DEFINE_UNQUOTED([OPAL_USE_ASM_ATOMICS],
+        [$want_asm_atomics],
+        [Whether to use assembly-coded atomics for atomics implementation])
 
-dnl #################################################################
-dnl
-dnl OPAL_ASM_FIND_FILE
-dnl
-dnl
-dnl do all the evil mojo to provide a working assembly file
-dnl
-dnl #################################################################
-AC_DEFUN([OPAL_ASM_FIND_FILE], [
-    AC_REQUIRE([AC_PROG_GREP])
-    AC_REQUIRE([AC_PROG_FGREP])
-
-if test "$opal_cv_asm_arch" != "WINDOWS" && test "$opal_cv_asm_builtin" != "BUILTIN_GCC" && test "$opal_cv_asm_builtin" != "BUILTIN_OSX"  && test "$opal_cv_asm_inline_arch" = "no" ; then
-    AC_MSG_ERROR([no atomic support available. exiting])
-else
-    # On windows with VC++, atomics are done with compiler primitives
-    opal_cv_asm_file=""
-fi
+    OPAL_SUMMARY_ADD([Miscellaneous], [Atomics], [], [$atomics_found])
+
+    OPAL_VAR_SCOPE_POP
 ])dnl
diff --git a/config/opal_config_files.m4 b/config/opal_config_files.m4
index 090b476d35a..cf4e658ef9f 100644
--- a/config/opal_config_files.m4
+++ b/config/opal_config_files.m4
@@ -4,6 +4,8 @@
 # Copyright (c) 2020      The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
+# Copyright (c) 2022-2024 Amazon.com, Inc. or its affiliates.
+#                         All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -20,11 +22,11 @@ AC_DEFUN([OPAL_CONFIG_FILES],[
         opal/include/Makefile
         opal/datatype/Makefile
         opal/util/Makefile
+        opal/util/json/Makefile
         opal/util/keyval/Makefile
         opal/mca/base/Makefile
         opal/tools/wrappers/Makefile
         opal/tools/wrappers/opalcc-wrapper-data.txt
-        opal/tools/wrappers/opalc++-wrapper-data.txt
         opal/tools/wrappers/opal.pc
     ])
 ])
diff --git a/config/opal_config_hwloc.m4 b/config/opal_config_hwloc.m4
index 6f006f4044b..779e3daa585 100644
--- a/config/opal_config_hwloc.m4
+++ b/config/opal_config_hwloc.m4
@@ -3,9 +3,9 @@ dnl
 dnl Copyright (c) 2009-2017 Cisco Systems, Inc.  All rights reserved
 dnl Copyright (c) 2014-2018 Research Organization for Information Science
 dnl                         and Technology (RIST). All rights reserved.
-dnl Copyright (c) 2020      Amazon.com, Inc. or its affiliates.  All Rights
-dnl                         reserved.
+dnl Copyright (c) 2020-2022 Amazon.com, Inc. or its affiliates.  All Rights reserved.
 dnl Copyright (c) 2020      Intel, Inc.  All rights reserved.
+dnl Copyright (c) 2022      IBM Corporation.  All rights reserved.
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
@@ -20,36 +20,37 @@ dnl configure, abort.
 dnl
 dnl This macro will change the environment in the following way:
 dnl
-dnl   * opal_hwloc_header [legacy] - will be set if building
-dnl         internally, to the header file that should be included for
-dnl         embedded builds.  This is used by PRRTE, but should not
-dnl         be used by new code.
 dnl   * opal_hwloc_mode - either external or internal.  If internal,
 dnl         --with-hwloc should be ignored by other packages
-dnl   * opal_hwloc_CPPFLAGS - the C Preprocessor flags necessary to
-dnl         run the preprocessor on a file which relies on Hwloc
-dnl         headers.  This will be folded into the global CPPFLAGS,
-dnl         so most people should never need this.
-dnl   * opal_hwloc_LDFLAGS - the linker flags necessary to run the
-dnl         linker on a file which relies on Hwloc libraries.  This
-dnl         will be folded into the global CPPFLAGS, so most people
-dnl         should never need this.
-dnl   * opal_hwloc_LIBS - the libraries necessary to link source which
-dnl         uses Hwloc.  Cannot be added to LIBS yet, because then
-dnl         other execution tests later in configure (there are sadly
-dnl         some) would fail if the path in LDFLAGS was not added to
-dnl         LD_LIBRARY_PATH.
-dnl   * CPPFLAGS, LDFLAGS - Updated opal_hwloc_CPPFLAGS and
-dnl         opal_hwloc_LDFLAGS.
+dnl   * CPPFLAGS, LDFLAGS, LIBS - Updated to build against hwloc.
+dnl         Note that the values may be updated right before
+dnl         config.status.
+dnl
+dnl OPAL_WRAPPER_FLAGS_ADD will be called to add the correct LDFLAGS,
+dnl STATIC_LDFLAGS, LIBS, and STATIC_LIBS for hwloc.
+dnl
+dnl The following environment variables will only be set if
+dnl opal_hwloc_mode is "internal":
+dnl
+dnl   * opal_hwloc_BUILD_CPPFLAGS - the C Preprocessor flags
+dnl         necessary to run the preprocessor on a file which relies
+dnl         on hwloc headers.  This will be folded into the global
+dnl         CPPFLAGS (see note above).
+dnl   * opal_hwloc_BUILD_LIBS - the libraries necessary to link
+dnl         source which uses hwloc.  Cannot be added to LIBS yet,
+dnl         because then other execution tests later in configure
+dnl         (there are sadly some) would fail if the path in LDFLAGS
+dnl         was not added to LD_LIBRARY_PATH.
+dnl   * opal_hwloc_WRAPPER_LIBS - the linker flags necessary to
+dnl         add to the wrapper compilers in order to link an opal
+dnl         application when opal is built as a static library.
 AC_DEFUN([OPAL_CONFIG_HWLOC], [
-    OPAL_VAR_SCOPE_PUSH([external_hwloc_happy internal_hwloc_happy])
+    OPAL_VAR_SCOPE_PUSH([external_hwloc_happy internal_hwloc_happy  opal_hwloc_STATIC_LDFLAGS opal_hwloc_LIBS opal_hwloc_STATIC_LIBS])
 
     opal_show_subtitle "Configuring hwloc"
 
     OPAL_3RDPARTY_WITH([hwloc], [hwloc], [package_hwloc])
 
-    opal_hwloc_header=""
-
     # unless internal specifically requested by the user, try to find
     # an external that works.
     external_hwloc_happy=0
@@ -70,73 +71,106 @@ AC_DEFUN([OPAL_CONFIG_HWLOC], [
     AS_IF([test "$external_hwloc_happy" = "0" -a "$internal_hwloc_happy" = "0"],
           [AC_MSG_ERROR([Could not find viable hwloc build.])])
 
-    # this will work even if there is no hwloc package included,
-    # because hwloc_tarball and hwloc_directory will evaluate to an
-    # empty string.  These are relative to the 3rd-party/ directory.
+    dnl this will work even if there is no hwloc package included,
+    dnl because hwloc_tarball and hwloc_directory will evaluate to an
+    dnl empty string.  These are relative to the 3rd-party/ directory.
     OPAL_3RDPARTY_EXTRA_DIST="$OPAL_3RDPARTY_EXTRA_DIST hwloc_tarball"
     OPAL_3RDPARTY_DISTCLEAN_DIRS="$OPAL_3RDPARTY_DISTCLEAN_DIRS hwloc_directory"
 
-    AC_SUBST(opal_hwloc_LIBS)
+    OPAL_WRAPPER_FLAGS_ADD([LDFLAGS], [${opal_hwloc_LDFLAGS}])
+    OPAL_WRAPPER_FLAGS_ADD([STATIC_LDFLAGS], [${opal_hwloc_STATIC_LDFLAGS}])
+    OPAL_WRAPPER_FLAGS_ADD([LIBS], [${opal_hwloc_LIBS}])
+    OPAL_WRAPPER_FLAGS_ADD([STATIC_LIBS], [${opal_hwloc_STATIC_LIBS}])
+    OPAL_WRAPPER_FLAGS_ADD([PC_MODULES], [${opal_hwloc_PC_MODULES}])
 
-    OPAL_SUMMARY_ADD([[Miscellaneous]], [[hwloc]], [hwloc], [$opal_hwloc_mode])
+    AC_CONFIG_COMMANDS_PRE([OPAL_CONFIG_HWLOC_INTERNAL_LIBS_HANDLER])
+
+    OPAL_SUMMARY_ADD([Miscellaneous], [hwloc], [], [$opal_hwloc_mode])
 
     OPAL_VAR_SCOPE_POP
 ])
 
+
 dnl _OPAL_CONFIG_HWLOC_EXTERNAL(action-if-happy, action-if-not-happy)
 dnl
 dnl only safe to call from OPAL_CONFIG_HWLOC, assumes variables from
 dnl there are set.
 AC_DEFUN([_OPAL_CONFIG_HWLOC_EXTERNAL], [
-    OPAL_VAR_SCOPE_PUSH([opal_hwloc_CPPFLAGS_save opal_hwloc_LDFLAGS_save opal_hwloc_LIBS_save opal_hwloc_external_support])
+    OPAL_VAR_SCOPE_PUSH([opal_hwloc_min_num_version opal_hwloc_min_version opal_hwlox_max_num_version opal_hwloc_CPPFLAGS_save opal_hwloc_LDFLAGS_save opal_hwloc_LIBS_save opal_hwloc_external_support])
+
+    OAC_CHECK_PACKAGE([hwloc],
+                      [opal_hwloc],
+                      [hwloc.h],
+                      [hwloc],
+                      [hwloc_topology_init],
+                      [opal_hwloc_external_support=yes],
+                      [opal_hwloc_external_support=no])
 
+    # need these set for the tests below.
     opal_hwloc_CPPFLAGS_save=$CPPFLAGS
     opal_hwloc_LDFLAGS_save=$LDFLAGS
     opal_hwloc_LIBS_save=$LIBS
 
-    AS_IF([test ! -z "$with_hwloc_libdir"],
-          [OPAL_CHECK_WITHDIR([hwloc-libdir], [$with_hwloc_libdir],
-                              [libhwloc.*])])
-
-    OPAL_CHECK_PACKAGE([opal_hwloc],
-                       [hwloc.h],
-                       [hwloc],
-                       [hwloc_topology_init],
-                       [],
-                       [$with_hwloc],
-                       [$with_hwloc_libdir],
-                       [opal_hwloc_external_support=yes],
-                       [opal_hwloc_external_support=no])
-
-    # need these set for the tests below.  If things fail, will undo at the end.
-    CPPFLAGS="$opal_hwloc_CPPFLAGS_save $opal_hwloc_CPPFLAGS"
-    LDFLAGS="$opal_hwloc_LDFLAGS_save $opal_hwloc_LDFLAGS"
-    LIBS="$opal_hwloc_LIBS_save $opal_hwloc_LIBS"
+    OPAL_FLAGS_APPEND_UNIQ([CPPFLAGS], [$opal_hwloc_CPPFLAGS])
+    OPAL_FLAGS_APPEND_UNIQ([LDFLAGS], [$opal_hwloc_LDFLAGS])
+    OPAL_FLAGS_APPEND_UNIQ([LIBS], [$opal_hwloc_LIBS])
 
+    opal_hwloc_min_num_version=OMPI_HWLOC_NUMERIC_MIN_VERSION
+    opal_hwloc_min_version=OMPI_HWLOC_NUMERIC_MIN_VERSION
     AS_IF([test "$opal_hwloc_external_support" = "yes"],
-          [AC_MSG_CHECKING([if external hwloc version is 1.6 or greater])
-           AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <hwloc.h>]],
-                               [[
-#if HWLOC_API_VERSION < 0x00010500
-#error "hwloc API version is less than 0x00010500"
+          [AC_MSG_CHECKING([if external hwloc version is version OMPI_HWLOC_MIN_VERSION or greater])
+           AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <hwloc.h>
+                              ]], [[
+#if HWLOC_API_VERSION < $opal_hwloc_min_num_version
+#error "hwloc API version is less than $opal_hwloc_min_version"
 #endif
                                ]])],
                    [AC_MSG_RESULT([yes])],
                    [AC_MSG_RESULT([no])
-                    AC_MSG_WARN([external hwloc version is too old (1.5 or later required)])
+                    AC_MSG_WARN([external hwloc version is too old (OMPI_HWLOC_MIN_VERSION or later required)])
                     opal_hwloc_external_support="no"])])
 
+    # Ensure that we are not using Hwloc >= v3.x.  Open MPI does not
+    # (yet) support Hwloc >= v3.x (which will potentially have ABI and
+    # API breakage compared to <= v2.x), and using it would lead to
+    # complicated failure cases.  Hence, we just abort outright if we
+    # find an external Hwloc >= v3.x.
     AS_IF([test "$opal_hwloc_external_support" = "yes"],
-          [AC_CHECK_DECLS([HWLOC_OBJ_OSDEV_COPROC], [], [], [#include <hwloc.h>])
+          [AC_MSG_CHECKING([if external hwloc version is less than version 3.0.0])
+	   opal_hwloc_max_num_version=0x00030000
+           AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <hwloc.h>
+                              ]], [[
+#if HWLOC_API_VERSION >= $opal_hwloc_max_num_version
+#error "hwloc API version is >= $opal_hwloc_max_num_version"
+#endif
+                               ]])],
+                   [AC_MSG_RESULT([yes])],
+                   [AC_MSG_RESULT([no])
+                    AC_MSG_WARN([External hwloc version is too new (less than v3.0.0 is required)])
+		    dnl Yes, the URL below will be wrong for master
+		    dnl builds.  But this is "good enough" -- we're
+		    dnl more concerned about getting the URL correct
+		    dnl for end-user builds of official release Open
+		    dnl MPI distribution tarballs.
+		    AC_MSG_WARN([See https://docs.open-mpi.org/en/v$OMPI_MAJOR_VERSION.$OMPI_MINOR_VERSION.x/installing-open-mpi/required-support-libraries.html for more details])
+		    AC_MSG_ERROR([Cannot continue])])])
+
+    AS_IF([test "$opal_hwloc_external_support" = "yes"],
+          [AC_CHECK_DECLS([HWLOC_OBJ_OSDEV_COPROC], [], [], [#include <hwloc.h>
+])
            AC_CHECK_FUNCS([hwloc_topology_dup])])
 
+    CPPFLAGS="$opal_hwloc_CPPFLAGS_save"
     LDFLAGS="$opal_hwloc_LDFLAGS_save"
     LIBS="$opal_hwloc_LIBS_save"
 
     AS_IF([test "$opal_hwloc_external_support" = "yes"],
-          [$1],
-          [CPPFLAGS="$opal_hwloc_CPPFLAGS_save"
-           $2])
+          [dnl Do not add hwloc libs to LIBS until late, because
+           dnl it will screw up other tests (like the pthread tests)
+           opal_hwloc_BUILD_LIBS="${opal_hwloc_LIBS}"
+
+           $1],
+          [$2])
 
     OPAL_VAR_SCOPE_POP
 ])
@@ -148,31 +182,78 @@ dnl external hwloc is not going to be used.  Assumes that if
 dnl this function is called, that success means the internal package
 dnl will be used.
 AC_DEFUN([_OPAL_CONFIG_HWLOC_INTERNAL], [
-    OPAL_VAR_SCOPE_PUSH(subconfig_happy subconfig_prefix internal_hwloc_location)
+    OPAL_VAR_SCOPE_PUSH([subconfig_happy internal_hwloc_location extra_configure_args found_enable_plugins hwloc_config_arg pkg_config_file pkg_config_happy])
+
+    extra_configure_args=
 
-    AS_IF([test ! -z $prefix], [subconfig_prefix="--prefix=$prefix"])
+    # look for a --{enable/disable}-plugins option in the top level
+    # configure arguments, so that we can add --enable-plugins if
+    # appropriate.
+    found_enable_plugins=0
+    eval "set x $ac_configure_args"
+    shift
+    for hwloc_config_arg
+    do
+        case $hwloc_config_arg in
+        --enable-plugins|--enable-plugins=*|--disable-plugins)
+            found_enable_plugins=1
+            ;;
+        esac
+    done
+
+    # while the plugins in hwloc are not explicitly using Open MPI's dlopen
+    # interface, it seems rude to enable plugins in hwloc if the builder asked
+    # us not to use plugins in Open MPI.  So only enable plugins in hwloc if there's
+    # a chance we're going to do so.  We enable plugins by default so that libhwloc
+    # does not end up with a dependency on libcuda, which would mean everything else
+    # would end up with a dependency on libcuda (and similar).
+    AS_IF([test $found_enable_plugins -eq 0 -a "$enable_dlopen" != "no"],
+          [extra_configure_args="--enable-plugins"])
 
     # Note: To update the version of hwloc shipped, update the
     # constant in autogen.pl.
     OPAL_EXPAND_TARBALL([3rd-party/hwloc_tarball], [3rd-party/hwloc_directory], [configure])
     OPAL_SUBDIR_ENV_CLEAN([opal_hwloc_configure])
-    PAC_CONFIG_SUBDIR_ARGS([3rd-party/hwloc_directory], [], [[--enable-debug]],
+    PAC_CONFIG_SUBDIR_ARGS([3rd-party/hwloc_directory], [$extra_configure_args], [[--enable-debug]],
         [subconfig_happy=1], [subconfig_happy=0])
     OPAL_SUBDIR_ENV_RESTORE([opal_hwloc_configure])
 
-    AS_IF([test "$subconfig_happy" = "1"],
+    AS_IF([test ${subconfig_happy} -eq 1],
         [internal_hwloc_location="3rd-party/hwloc_directory"
-         # note: because we only ship/commit a tarball (and not the source
-         # directory), the source is always expanded in the builddir, so we
-         # only need to add a -I to the builddir.
+
+         dnl We do not consider it an error if pkg-config doesn't work / exist / etc.
+         pkg_config_file="${OMPI_TOP_BUILDDIR}/3rd-party/hwloc_directory/hwloc.pc"
+         pkg_config_happy=0
+
+         OAC_CHECK_PACKAGE_PARSE_PKGCONFIG([hwloc_internal], [opal_hwloc], [${pkg_config_file}], [pkg_config_happy=1])
+
+         dnl Don't pull LDFLAGS, because we don't have a good way to avoid
+         dnl a -L to our install directory, which can cause some weirdness
+         dnl if there's an old OMPI install there.  And it makes filtering
+         dnl redundant flags easier.
+         opal_hwloc_LDFLAGS=
+
+         dnl with no pkg-config data, guess.  assume that -L${libdir} is already added to LDFLAGS
+         AS_IF([test $pkg_config_happy -eq 0],
+               [opal_hwloc_STATIC_LDFLAGS=
+                opal_hwloc_LIBS="-lhwloc"
+                opal_hwloc_STATIC_LIBS=
+                opal_hwloc_PC_MODULES=])
+
+         # note: because we only ship/commit a tarball (and not the
+         # source directory), the source is always expanded in the
+         # builddir, so we only need to add a -I to the builddir.
+         # Overwrite the OAC_CHECK_PACKAGE_PARSE PKGCONFIG results,
+         # because it's the install dir location, not the build
+         # location.
          opal_hwloc_CPPFLAGS="-I$OMPI_TOP_BUILDDIR/$internal_hwloc_location/include -I$OMPI_TOP_SRCDIR/$internal_hwloc_location/include"
-         CPPFLAGS="$CPPFLAGS $opal_hwloc_CPPFLAGS"
+         opal_hwloc_BUILD_CPPFLAGS="${opal_hwloc_CPPFLAGS}"
+
          # No need to update LDFLAGS, because they will install into
          # our tree and in the mean time are referenced by their .la
          # files.
-         opal_hwloc_LIBS="$OMPI_TOP_BUILDDIR/$internal_hwloc_location/hwloc/libhwloc.la"
-
-         opal_hwloc_header="$OMPI_TOP_BUILDDIR/$internal_hwloc_location/include/hwloc.h"
+         opal_hwloc_BUILD_LIBS="$OMPI_TOP_BUILDDIR/$internal_hwloc_location/hwloc/libhwloc.la"
+         opal_hwloc_WRAPPER_LIBS="${opal_hwloc_LIBS}"
 
          # no need to add to DIST_SUBDIRS, because we only ship the
          # tarball.  This is relative to the 3rd-party/ directory.
@@ -182,3 +263,12 @@ AC_DEFUN([_OPAL_CONFIG_HWLOC_INTERNAL], [
 
     OPAL_VAR_SCOPE_POP
 ])
+
+
+dnl We need to delay adding .la files to LIBS until the very end of
+dnl configure, to avoid pulling it into other configure tests.
+AC_DEFUN([OPAL_CONFIG_HWLOC_INTERNAL_LIBS_HANDLER], [
+    OPAL_FLAGS_APPEND_UNIQ([CPPFLAGS], [${opal_hwloc_CPPFLAGS}])
+    OPAL_FLAGS_APPEND_UNIQ([LDFLAGS], [$opal_hwloc_LDFLAGS])
+    OPAL_FLAGS_APPEND_MOVE([LIBS], [${opal_hwloc_BUILD_LIBS}])
+])
diff --git a/config/opal_config_libevent.m4 b/config/opal_config_libevent.m4
index 7b78e0dede9..4e16d899696 100644
--- a/config/opal_config_libevent.m4
+++ b/config/opal_config_libevent.m4
@@ -4,8 +4,8 @@ dnl Copyright (c) 2009-2018 Cisco Systems, Inc.  All rights reserved
 dnl Copyright (c) 2013      Los Alamos National Security, LLC.  All rights reserved.
 dnl Copyright (c) 2015-2018 Research Organization for Information Science
 dnl                         and Technology (RIST). All rights reserved.
-dnl Copyright (c) 2020      Amazon.com, Inc. or its affiliates.  All Rights
-dnl                         reserved.
+dnl Copyright (c) 2020-2022 Amazon.com, Inc. or its affiliates.  All Rights reserved.
+dnl Copyright (c) 2022      IBM Corporation.  All rights reserved.
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
@@ -20,36 +20,39 @@ dnl configure, abort.
 dnl
 dnl This macro will change the environment in the following way:
 dnl
-dnl   * opal_libevent_header [legacy] - will be set if building
-dnl         internally, to the header file that should be included for
-dnl         embedded builds.  This is used by PRRTE, but should not
-dnl         be used by new code.
 dnl   * opal_libevent_mode - either external or internal.  If internal,
 dnl         --with-libevent should be ignored by other packages
-dnl   * opal_libevent_CPPFLAGS - the C Preprocessor flags necessary to
-dnl         run the preprocessor on a file which relies on Libevent
-dnl         headers.  This will be folded into the global CPPFLAGS,
-dnl         so most people should never need this.
-dnl   * opal_libevent_LDFLAGS - the linker flags necessary to run the
-dnl         linker on a file which relies on Libevent libraries.  This
-dnl         will be folded into the global CPPFLAGS, so most people
-dnl         should never need this.
-dnl   * opal_libevent_LIBS - the libraries necessary to link source which
-dnl         uses Libevent.  Cannot be added to LIBS yet, because then
-dnl         other execution tests later in configure (there are sadly
-dnl         some) would fail if the path in LDFLAGS was not added to
-dnl         LD_LIBRARY_PATH.
-dnl   * CPPFLAGS, LDFLAGS - Updated opal_libevent_CPPFLAGS and
-dnl         opal_libevent_LDFLAGS.
+dnl   * CPPFLAGS, LDFLAGS, LIBS - Updated to build against libevent.
+dnl         Note that the value may be updated right before
+dnl         config.status.
+dnl
+dnl OPAL_WRAPPER_FLAGS_ADD will be called to add the correct LDFLAGS,
+dnl STATIC_LDFLAGS, LIBS, and STATIC_LIBS for libevent.  Note that we
+dnl intentionally do not use pkg-config modules for Libevent because
+dnl we want to avoid libevent in preference for libevent_core.
+dnl
+dnl The following environment variables will only be set if
+dnl opal_libevent_mode is "internal":
+dnl
+dnl   * opal_libevent_BUILD_CPPFLAGS - the C Preprocessor flags
+dnl         necessary to run the preprocessor on a file which relies
+dnl         on Libevent headers.  This will be folded into the global
+dnl         CPPFLAGS (see note above).
+dnl   * opal_libevent_BUILD_LIBS - the libraries necessary to link
+dnl         source which uses Libevent.  Cannot be added to LIBS yet,
+dnl         because then other execution tests later in configure
+dnl         (there are sadly some) would fail if the path in LDFLAGS
+dnl         was not added to LD_LIBRARY_PATH.
+dnl   * opal_libevent_WRAPPER_LIBS - the linker flags necessary to
+dnl         add to the wrapper compilers in order to link an opal
+dnl         application when opal is built as a static library.
 AC_DEFUN([OPAL_CONFIG_LIBEVENT], [
-    OPAL_VAR_SCOPE_PUSH([internal_libevent_happy external_libevent_happy])
+    OPAL_VAR_SCOPE_PUSH([internal_libevent_happy external_libevent_happy opal_libevent_STATIC_LDFLAGS opal_libevent_LIBS opal_libevent_STATIC_LIBS])
 
     opal_show_subtitle "Configuring Libevent"
 
     OPAL_3RDPARTY_WITH([libevent], [libevent], [package_libevent])
 
-    opal_libevent_header=""
-
     # unless internal specifically requested by the user, try to find
     # an external that works.
     external_libevent_happy=0
@@ -70,101 +73,128 @@ AC_DEFUN([OPAL_CONFIG_LIBEVENT], [
     AS_IF([test "$external_libevent_happy" = "0" -a "$internal_libevent_happy" = "0"],
           [AC_MSG_ERROR([Could not find viable libevent build.])])
 
-    # this will work even if there is no libevent package included,
-    # because libevent_tarball and libevent_directory will evaluate to
-    # an empty string.  These are relative to the 3rd-party/
-    # directory.
-    OPAL_3RDPARTY_EXTRA_DIST="$OPAL_3RDPARTY_EXTRA_DIST libevent_tarball"
-    OPAL_3RDPARTY_DISTCLEAN_DIRS="$OPAL_3RDPARTY_DISTCLEAN_DIRS libevent_directory"
+    dnl this will work even if there is no libevent package included,
+    dnl because libevent_tarball and libevent_directory will evaluate to
+    dnl an empty string.  These are relative to the 3rd-party/
+    dnl directory.
+    OPAL_APPEND([OPAL_3RDPARTY_EXTRA_DIST], [libevent_tarball])
+    OPAL_APPEND([OPAL_3RDPARTY_DISTCLEAN_DIRS], [libevent_directory])
+
+    OPAL_WRAPPER_FLAGS_ADD([LDFLAGS], [${opal_libevent_LDFLAGS}])
+    OPAL_WRAPPER_FLAGS_ADD([STATIC_LDFLAGS], [${opal_libevent_STATIC_LDFLAGS}])
+    OPAL_WRAPPER_FLAGS_ADD([LIBS], [${opal_libevent_LIBS}])
+    OPAL_WRAPPER_FLAGS_ADD([STATIC_LIBS], [${opal_libevent_STATIC_LIBS}])
+    OPAL_WRAPPER_FLAGS_ADD([PC_MODULES], [${opal_libevent_PC_MODULES}])
 
-    AC_SUBST(opal_libevent_LIBS)
+    AC_CONFIG_COMMANDS_PRE([OPAL_CONFIG_LIBEVENT_INTERNAL_LIBS_HANDLER])
 
-    OPAL_SUMMARY_ADD([[Miscellaneous]],[[libevent]],[libevent], [$opal_libevent_mode])
+    OPAL_SUMMARY_ADD([Miscellaneous], [libevent], [], [$opal_libevent_mode])
 
     OPAL_VAR_SCOPE_POP
 ])
 
+
 dnl _OPAL_CONFIG_LIBEVENT_EXTERNAL(action-if-happy, action-if-not-happy)
 dnl
 dnl only safe to call from OPAL_CONFIG_LIBEVENT, assumes variables
 dnl from there are set.
 AC_DEFUN([_OPAL_CONFIG_LIBEVENT_EXTERNAL], [
-    OPAL_VAR_SCOPE_PUSH([opal_libevent_CPPFLAGS_save opal_libevent_LDFLAGS_save opal_libevent_LIBS_save opal_libevent_external_support])
+    OPAL_VAR_SCOPE_PUSH([opal_event_min_version opal_event_min_num_version opal_libevent_CPPFLAGS_save opal_libevent_LDFLAGS_save opal_libevent_LIBS_save opal_libevent_external_support])
+
+    dnl Look at libevent_core, not libevent_pthread, because
+    dnl trying to avoid picking up libevent.so.  The wrappers and
+    dnl ompi.pc will include the -levent_pthreads, so we're
+    dnl still good from a linking perspective.
+    m4_define([libevent_pkgconfig_module], [libevent_core])
+    OAC_CHECK_PACKAGE([libevent],
+                      [opal_libevent],
+                      [event2/event.h],
+                      [event_core],
+                      [event_config_new],
+                      [opal_libevent_external_support=yes],
+                      [opal_libevent_external_support=no])
+    dnl Manually add libevent_pthreads.
+    OPAL_APPEND([opal_libevent_LIBS], [-levent_pthreads])
 
+    # need these set for the tests below.
     opal_libevent_CPPFLAGS_save=$CPPFLAGS
-    opal_libevent_LDFLAGS_save=$LDFLAGS
-    opal_libevent_LIBS_save=$LIBS
-
-    AS_IF([test ! -z "$with_libevent_libdir"],
-          [OPAL_CHECK_WITHDIR([libevent-libdir], [$with_libevent_libdir],
-                              [libevent.*])])
-
-    OPAL_CHECK_PACKAGE([opal_libevent],
-                       [event2/event.h],
-                       [event],
-                       [event_config_new],
-                       [-levent_pthreads],
-                       [$with_libevent],
-                       [$with_libevent_libdir],
-                       [opal_libevent_external_support=yes],
-                       [opal_libevent_external_support=no])
-
-    # need these set for the tests below.  If things fail, will undo at the end.
-    CPPFLAGS="$opal_libevent_CPPFLAGS_save $opal_libevent_CPPFLAGS"
-    LDFLAGS="$opal_libevent_LDFLAGS_save $opal_libevent_LDFLAGS"
-    LIBS="$opal_libevent_LIBS_save $opal_libevent_LIBS"
-
-    # Ensure that this libevent has the symbol
-    # "evthread_set_lock_callbacks", which will only exist if
-    # libevent was configured with thread support.
+    OPAL_FLAGS_APPEND_UNIQ([CPPFLAGS], [$opal_libevent_CPPFLAGS])
+
+    # verify libevent is configured with thread support
     AS_IF([test "$opal_libevent_external_support" = "yes"],
-          [AC_CHECK_LIB([event], [evthread_set_lock_callbacks],
-                        [],
-                        [AC_MSG_WARN([External libevent does not have thread support])
-                         AC_MSG_WARN([Open MPI requires libevent to be compiled with])
-                         AC_MSG_WARN([thread support enabled])
-                         opal_libevent_external_support=no])])
+          [AC_CACHE_CHECK([if libevent threads enabled],
+              [opal_libevent_cv_threads_enabled],
+              [# Check for general threading support
+               AC_COMPILE_IFELSE([AC_LANG_PROGRAM([
+#include <event.h>
+#include <event2/thread.h>
+               ], [[
+#if !(EVTHREAD_LOCK_API_VERSION >= 1)
+#  error "No threads!"
+#endif
+               ]])],
+              [opal_libevent_cv_threads_enabled="yes"],
+              [opal_libevent_cv_threads_enabled="no"])])
+           AS_IF([test "${opal_libevent_cv_threads_enabled}" = "no"],
+                 [AC_MSG_WARN([Open MPI requires libevent to be compiled with thread support enabled])
+                  opal_libevent_external_support="no"])])
 
     AS_IF([test "$opal_libevent_external_support" = "yes"],
-          [AC_CHECK_LIB([event_pthreads], [evthread_use_pthreads],
-                        [],
-                        [AC_MSG_WARN([External libevent does not have thread support])
-                         AC_MSG_WARN([Open MPI requires libevent to be compiled with])
-                         AC_MSG_WARN([thread support enabled])
-                         opal_libevent_external_support=no])])
+          [AC_CACHE_CHECK([for libevent pthreads support],
+              [opal_libevent_cv_pthread_support],
+              [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([
+#include <event.h>
+#include <event2/thread.h>
+               ], [[
+#if !defined(EVTHREAD_USE_PTHREADS_IMPLEMENTED) || !EVTHREAD_USE_PTHREADS_IMPLEMENTED
+#  error "No pthreads!"
+#endif
+               ]])],
+              [opal_libevent_cv_pthread_support="yes"],
+              [opal_libevent_cv_pthread_support="no"])])
+           AS_IF([test "${opal_libevent_cv_pthread_support}" = "no"],
+                 [AC_MSG_WARN([PMIX requires libevent to be compiled with pthread support enabled])
+                  opal_libevent_external_support="no"])])
 
     # Open MPI used to fall back to the internal libevent if the
     # installed version was older than the internal version.  This
     # isn't what we want, because we really want to prefer external
     # versions.  Pin the "oldest supported" external version to
     # 2.0.21, which we know works from testing on RHEL7.
+    opal_event_min_num_version=OMPI_EVENT_NUMERIC_MIN_VERSION
+    opal_event_min_version=OMPI_EVENT_MIN_VERSION
     AS_IF([test "$opal_libevent_external_support" = "yes"],
-          [AC_MSG_CHECKING([if external libevent version is 2.0.21 or greater])
-              AC_COMPILE_IFELSE(
+          [AC_CACHE_CHECK([if external libevent version is OMPI_EVENT_MIN_VERSION or greater],
+              [opal_libevent_cv_version_check],
+              [AC_COMPILE_IFELSE(
                   [AC_LANG_PROGRAM([[#include <event2/event.h>]],
-                                 [[
-#if defined(_EVENT_NUMERIC_VERSION) && _EVENT_NUMERIC_VERSION < 0x02001500
-#error "libevent API version is less than 0x02001500"
-#elif defined(EVENT__NUMERIC_VERSION) && EVENT__NUMERIC_VERSION < 0x02001500
-#error "libevent API version is less than 0x02001500"
+                                   [[
+#if defined(_EVENT_NUMERIC_VERSION) && _EVENT_NUMERIC_VERSION < $opal_event_min_num_version
+#error "libevent API version is less than $opal_event_min_version"
+#elif defined(EVENT__NUMERIC_VERSION) && EVENT__NUMERIC_VERSION < $opal_event_min_num_version
+#error "libevent API version is less than $opal_event_min_version"
 #endif
-                                 ]])],
-                  [AC_MSG_RESULT([yes])],
-                  [AC_MSG_RESULT([no])
-                   AC_MSG_WARN([external libevent version is too old (2.0.21 or later required)])
-                   opal_libevent_external_support=no])])
+                                    ]])],
+                  [opal_libevent_cv_version_check="yes"],
+                  [opal_libevent_cv_version_check="no"])])
+          AS_IF([test "${opal_libevent_cv_version_check}" = "no"],
+                [AC_MSG_WARN([external libevent version is too old (OMPI_EVENT_MIN_VERSION or later required)])
+                 opal_libevent_external_support=no])])
 
-    LDFLAGS="$opal_libevent_LDFLAGS_save"
-    LIBS="$opal_libevent_LIBS_save"
+    CPPFLAGS="$opal_libevent_CPPFLAGS_save"
 
     AS_IF([test "$opal_libevent_external_support" = "yes"],
-          [$1],
-          [CPPFLAGS="$opal_libevent_CPPFLAGS_save"
-           $2])
+          [dnl Do not add libevent flags until late, because
+           dnl it will screw up other tests (like the pthread tests)
+           opal_libevent_BUILD_LIBS="${opal_libevent_LIBS}"
+
+           $1],
+          [$2])
 
     OPAL_VAR_SCOPE_POP
 ])
 
+
 dnl _OPAL_CONFIG_LIBEVENT_INTERNAL(action-if-happy, action-if-not-happy)
 dnl
 dnl Configure the packaged libevent.  Only needs to be run if the
@@ -189,34 +219,55 @@ AC_DEFUN([_OPAL_CONFIG_LIBEVENT_INTERNAL], [
        [], [subconfig_happy=1], [subconfig_happy=0])
     OPAL_SUBDIR_ENV_RESTORE([opal_libevent_configure])
 
-    AS_IF([test "$subconfig_happy" = "1"],
+    AS_IF([test ${subconfig_happy} -eq 1],
         [internal_libevent_location="3rd-party/libevent_directory"
+
+         # The tarball as configured can not be used for compile
+         # tests, because libevent uses a Makefile rule rather than
+         # Autoconf to generate their config file (sigh).  Force
+         # generation of that file now, so that other 3rd party
+         # packages can run compile tests.
+         AC_MSG_NOTICE([Generating Libevent's event-config.h])
+         (cd $OMPI_TOP_BUILDDIR/$internal_libevent_location/ ; ${MAKE-make} include/event2/event-config.h)
+         AS_IF([test $? -ne 0], [AC_MSG_ERROR([Could not generate event-config.h.])])
+
+         # because we don't use pkg-config with libevent, have to
+         # guess at many of these fields (and likely get static
+         # versions wrong).
+         opal_libevent_LDFLAGS=
+         opal_libevent_STATIC_LDFLAGS=
+         opal_libevent_LIBS="-levent_core -levent_pthreads"
+         opal_libevent_STATIC_LIBS=
+
+         AC_MSG_CHECKING([for internal libevent LIBS])
+         AC_MSG_RESULT([${opal_libevent_LIBS}])
+
          # note: because we only ship/commit a tarball (and not the source
          # directory), the source is always expanded in the builddir, so we
          # only need to add a -I to the builddir.
          opal_libevent_CPPFLAGS="-I$OMPI_TOP_BUILDDIR/$internal_libevent_location -I$OMPI_TOP_BUILDDIR/$internal_libevent_location/include"
-         CPPFLAGS="$CPPFLAGS $opal_libevent_CPPFLAGS"
+         opal_libevent_BUILD_CPPFLAGS="${opal_libevent_CPPFLAGS}"
+
          # No need to update LDFLAGS, because they will install into
          # our tree and in the mean time are referenced by their .la
          # files.
-         opal_libevent_LIBS="$OMPI_TOP_BUILDDIR/$internal_libevent_location/libevent.la $OMPI_TOP_BUILDDIR/$internal_libevent_location/libevent_pthreads.la"
-
-         opal_libevent_header="$OMPI_TOP_BUILDDIR/$internal_libevent_location/event.h"
+         opal_libevent_BUILD_LIBS="$OMPI_TOP_BUILDDIR/$internal_libevent_location/libevent_core.la $OMPI_TOP_BUILDDIR/$internal_libevent_location/libevent_pthreads.la"
+         opal_libevent_WRAPPER_LIBS="${opal_libevent_LIBS}"
 
          # no need to add to DIST_SUBDIRS, because we only ship the
          # tarball.  This is relative to the 3rd-party/ directory.
          OPAL_3RDPARTY_SUBDIRS="$OPAL_3RDPARTY_SUBDIRS libevent_directory"
 
-         # The tarball as configured can not be used for compile
-         # tests, because libevent uses a Makefile rule rather than
-         # Autoconf to generate their config file (sigh).  Force
-         # generation of that file now, so that other 3rd party
-         # packages can run compile tests.
-         AC_MSG_NOTICE([Generating Libevent's event-config.h])
-         (cd $OMPI_TOP_BUILDDIR/$internal_libevent_location/ ; ${MAKE-make} include/event2/event-config.h)
-         AS_IF([test $? -ne 0], [AC_MSG_ERROR([Could not generate event-config.h.])])
-
          $1], [$2])
 
     OPAL_VAR_SCOPE_POP
 ])
+
+
+dnl We need to delay adding .la files to LIBS until the very end of
+dnl configure, to avoid pulling it into other configure tests.
+AC_DEFUN([OPAL_CONFIG_LIBEVENT_INTERNAL_LIBS_HANDLER], [
+    OPAL_FLAGS_APPEND_UNIQ([CPPFLAGS], [${opal_libevent_CPPFLAGS}])
+    OPAL_FLAGS_APPEND_UNIQ([LDFLAGS], [${opal_libevent_LDFLAGS}])
+    OPAL_FLAGS_APPEND_MOVE([LIBS], [${opal_libevent_BUILD_LIBS}])
+])
diff --git a/config/opal_config_pmix.m4 b/config/opal_config_pmix.m4
index 5b13e6bd082..bea801c335e 100644
--- a/config/opal_config_pmix.m4
+++ b/config/opal_config_pmix.m4
@@ -16,11 +16,10 @@ dnl                         reserved.
 dnl Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
 dnl Copyright (c) 2014-2021 Research Organization for Information Science
 dnl                         and Technology (RIST).  All rights reserved.
-dnl Copyright (c) 2016-2021 IBM Corporation.  All rights reserved.
+dnl Copyright (c) 2016-2022 IBM Corporation.  All rights reserved.
 dnl Copyright (c) 2020      Triad National Security, LLC. All rights
 dnl                         reserved.
-dnl Copyright (c) 2020      Amazon.com, Inc. or its affiliates.  All Rights
-dnl                         reserved.
+dnl Copyright (c) 2020-2022 Amazon.com, Inc. or its affiliates.  All Rights reserved.
 dnl Copyright (c) 2021      Nanook Consulting.  All rights reserved.
 dnl $COPYRIGHT$
 dnl
@@ -36,77 +35,92 @@ dnl configure, abort.
 dnl
 dnl This macro will change the environment in the following way:
 dnl
-dnl   * opal_pmix_header [legacy] - will be set if building
-dnl         internally, to the header file that should be included for
-dnl         embedded builds.  This is used by PRRTE, but should not
-dnl         be used by new code.
 dnl   * opal_pmix_mode - either external or internal.  If internal,
 dnl         --with-pmix should be ignored by other packages
-dnl   * opal_pmix_CPPFLAGS - the C Preprocessor flags necessary to
+dnl   * CPPFLAGS, LDFLAGS, LIBS - Updated to build against PMIx.
+dnl         Note that the values may be updated right before
+dnl         config.status, to avoid interfering with other tests.
+dnl
+dnl OPAL_WRAPPER_FLAGS_ADD will be called to add the correct LDFLAGS,
+dnl STATIC_LDFLAGS, LIBS, and STATIC_LIBS for PMIx.
+dnl
+dnl The following environment variables will only be set if
+dnl opal_pmix_mode is "internal":
+dnl
+dnl   * opal_pmix_BUILD_CPPFLAGS - the C Preprocessor flags necessary to
 dnl         run the preprocessor on a file which relies on PMIx
-dnl         headers.  This will be folded into the global CPPFLAGS,
-dnl         so most people should never need this.
-dnl   * opal_pmix_LDFLAGS - the linker flags necessary to run the
-dnl         linker on a file which relies on PMIx libraries.  This
-dnl         will be folded into the global CPPFLAGS, so most people
-dnl         should never need this.
-dnl   * opal_pmix_LIBS - the libraries necessary to link source which
+dnl         headers.  This will be folded into the global
+dnl         CPPFLAGS (see note above).
+dnl   * opal_pmix_BUILD_LIBS - the libraries necessary to link source which
 dnl         uses PMIx.  Cannot be added to LIBS yet, because then
 dnl         other execution tests later in configure (there are sadly
 dnl         some) would fail if the path in LDFLAGS was not added to
 dnl         LD_LIBRARY_PATH.
-dnl   * CPPFLAGS, LDFLAGS - Updated opal_pmix_CPPFLAGS and
-dnl         opal_pmix_LDFLAGS.
 AC_DEFUN([OPAL_CONFIG_PMIX], [
-    OPAL_VAR_SCOPE_PUSH([external_pmix_happy internal_pmix_happy internal_pmix_args internal_pmix_libs internal_pmix_CPPFLAGS])
+    OPAL_VAR_SCOPE_PUSH([external_pmix_happy internal_pmix_happy internal_pmix_args internal_pmix_wrapper_libs internal_pmix_CPPFLAGS opal_pmix_STATIC_LDFLAGS opal_pmix_LIBS opal_pmix_STATIC_LIBS])
 
     opal_show_subtitle "Configuring PMIx"
 
     OPAL_3RDPARTY_WITH([pmix], [pmix], [package_pmix])
 
-    opal_pmix_header=""
-
     internal_pmix_happy=0
     m4_ifdef([package_pmix],
-        [# always configure the internal pmix, so that
-         # make dist always works.
-	 internal_pmix_args="--without-tests-examples --disable-pmix-binaries --disable-pmix-backward-compatibility --disable-visibility"
-         internal_pmix_libs=
-         internal_pmix_CPPFLAGS=
-
-         OMPI_PMIX_ADD_ARGS
-
-         AS_IF([test "$opal_libevent_mode" = "internal"],
-               [internal_pmix_args="$internal_pmix_args --with-libevent=cobuild"
-                internal_pmix_CPPFLAGS="$internal_pmix_CPPFLAGS $opal_libevent_CPPFLAGS"
-                internal_pmix_libs="$internal_pmix_libs $opal_libevent_LIBS"])
-
-         AS_IF([test "$opal_hwloc_mode" = "internal"],
-               [internal_pmix_args="$internal_pmix_args --with-hwloc=cobuild"
-                internal_pmix_CPPFLAGS="$internal_pmix_CPPFLAGS $opal_hwloc_CPPFLAGS"
-                internal_pmix_libs="$internal_pmix_libs $opal_hwloc_LIBS"])
-
-         AS_IF([test ! -z "$internal_pmix_libs"],
-               [internal_pmix_args="$internal_pmix_args --with-prte-extra-lib=\"$internal_pmix_libs\""])
-
-         if test "$WANT_DEBUG" = "1"; then
-             internal_pmix_args="$internal_pmix_args --enable-debug"
-         fi
-
-         # Pass all our compiler/linker flags to PMIx, so that it
-         # picks up how to build an internal HWLOC and libevent, plus
-         # picks up any user-specified compiler flags from the master
-         # configure run.
-         OPAL_SUBDIR_ENV_CLEAN([opal_pmix_configure])
-         AS_IF([test -n "$internal_pmix_CPPFLAGS"],
-               [OPAL_SUBDIR_ENV_APPEND([CPPFLAGS], [$internal_pmix_CPPFLAGS])])
-         PAC_CONFIG_SUBDIR_ARGS([3rd-party/openpmix], [$internal_pmix_args],
-                           [[--with-libevent=internal], [--with-hwloc=internal],
-                            [--with-libevent=external], [--with-hwloc=external],
-                            [--with-pmix=[[^ 	]]*], [--with-platform=[[^ 	]]*]],
-                           [internal_pmix_happy=1])
-         OPAL_SUBDIR_ENV_RESTORE([opal_pmix_configure])
-         OPAL_3RDPARTY_DIST_SUBDIRS="$OPAL_3RDPARTY_DIST_SUBDIRS openpmix"])
+        [OMPI_PMIX_ADD_ARGS
+         AS_IF([test "$opal_pmix_mode" = "unspecified" -o "$opal_pmix_mode" = "internal"],
+               [# Run PMIx's configure script unless the user
+                # explicitly asked us to use an external PMIX, so that
+                # "make dist" includes PMIx in the dist tarball.  This
+                # does mean that "make dist" will not work if Open MPI
+                # was configured to use an external PMIx library, but
+                # we decided this was a reasonable tradeoff for not
+                # having to deal with PMIx (or PRRTE) potentially
+                # failing to configure in a situation where it isn't
+                # desired.
+
+                internal_pmix_args="--without-tests-examples --enable-pmix-binaries --disable-pmix-backward-compatibility --disable-visibility"
+                # Open PMIx sets -Werror on devel builds so avoid buid breaks caused by 3rd-party codes
+                internal_pmix_args="$internal_pmix_args --disable-devel-check"
+                internal_pmix_wrapper_libs=
+                internal_pmix_CPPFLAGS=
+
+                AS_IF([test "$opal_libevent_mode" = "internal"],
+                      [OPAL_APPEND([internal_pmix_args], [--with-libevent --disable-libevent-lib-checks])
+                       OPAL_APPEND([internal_pmix_args], [--with-libevent-extra-libs=\"$opal_libevent_BUILD_LIBS\"])
+                       OPAL_APPEND([internal_pmix_wrapper_libs], [$opal_libevent_WRAPPER_LIBS])
+                       OPAL_APPEND([internal_pmix_CPPFLAGS], [$opal_libevent_BUILD_CPPFLAGS])])
+
+                AS_IF([test "$opal_hwloc_mode" = "internal"],
+                      [OPAL_APPEND([internal_pmix_args], [--disable-hwloc-lib-checks])
+                       OPAL_APPEND([internal_pmix_args], [--with-hwloc-extra-libs=\"$opal_hwloc_BUILD_LIBS\"])
+                       OPAL_APPEND([internal_pmix_wrapper_libs], [$opal_hwloc_WRAPPER_BUILD_LIBS])
+                       OPAL_APPEND([internal_pmix_CPPFLAGS], [$opal_hwloc_BUILD_CPPFLAGS])])
+
+                if test "$WANT_DEBUG" = "1"; then
+                     OPAL_APPEND([internal_pmix_args], [--enable-debug])
+                fi
+
+                # Pass all our compiler/linker flags to PMIx, so that it
+                # picks up how to build an internal HWLOC and libevent, plus
+                # picks up any user-specified compiler flags from the master
+                # configure run.
+                OPAL_SUBDIR_ENV_CLEAN([opal_pmix_configure])
+                AS_IF([test -n "$internal_pmix_CPPFLAGS"],
+                      [OPAL_SUBDIR_ENV_APPEND([CPPFLAGS], [$internal_pmix_CPPFLAGS])])
+                AS_IF([test -n "$internal_pmix_wrapper_libs"],
+                      [OPAL_APPEND([internal_pmix_args], [--with-wrapper-libs=\"$internal_pmix_wrapper_libs\"])])
+
+                PAC_CONFIG_SUBDIR_ARGS([3rd-party/openpmix], [$internal_pmix_args],
+                                       [[--with-libevent=internal], [--with-hwloc=internal],
+                                        [--with-libevent=external], [--with-hwloc=external],
+                                        [--with-pmix=[[^ 	]]*], [--with-platform=[[^ 	]]*]],
+                                       [internal_pmix_happy=1])
+                OPAL_SUBDIR_ENV_RESTORE([opal_pmix_configure])
+                OPAL_3RDPARTY_DIST_SUBDIRS="$OPAL_3RDPARTY_DIST_SUBDIRS openpmix"])
+
+         # if we have a pmix package and configure did not complete
+         # successfully (or wasn't started), then disable make dist.
+         AS_IF([test $internal_pmix_happy != 1],
+               [OPAL_MAKEDIST_DISABLE="$OPAL_MAKEDIST_DISABLE PMIX"])])
 
     # unless internal specifically requested by the user, try to find
     # an external that works.
@@ -122,16 +136,26 @@ AC_DEFUN([OPAL_CONFIG_PMIX], [
     # so try the internal version.
     AS_IF([test "$external_pmix_happy" = "0" -a "$internal_pmix_happy" = "1"],
           [opal_pmix_mode="internal"
-           _OPAL_CONFIG_PMIX_INTERNAL_POST()])
+           OPAL_USING_INTERNAL_PMIX=1
+           _OPAL_CONFIG_PMIX_INTERNAL_POST()],
+          [OPAL_USING_INTERNAL_PMIX=0])
 
     AS_IF([test "$external_pmix_happy" = "0" -a "$internal_pmix_happy" = "0"],
           [AC_MSG_ERROR([Could not find viable pmix build.])])
 
-    AC_SUBST(opal_pmix_CPPFLAGS)
-    AC_SUBST(opal_pmix_LDFLAGS)
-    AC_SUBST(opal_pmix_LIBS)
+    OPAL_WRAPPER_FLAGS_ADD([LDFLAGS], [${opal_pmix_LDFLAGS}])
+    OPAL_WRAPPER_FLAGS_ADD([STATIC_LDFLAGS], [${opal_pmix_STATIC_LDFLAGS}])
+    OPAL_WRAPPER_FLAGS_ADD([LIBS], [${opal_pmix_LIBS}])
+    OPAL_WRAPPER_FLAGS_ADD([STATIC_LIBS], [${opal_pmix_STATIC_LIBS}])
+    OPAL_WRAPPER_FLAGS_ADD([PC_MODULES], [${opal_pmix_PC_MODULES}])
 
-    OPAL_SUMMARY_ADD([[Miscellaneous]], [[pmix]], [pmix], [$opal_pmix_mode])
+    AC_CONFIG_COMMANDS_PRE([OPAL_CONFIG_PMIX_INTERNAL_LIBS_HANDLER])
+
+    AC_DEFINE_UNQUOTED([OPAL_USING_INTERNAL_PMIX],
+                       [$OPAL_USING_INTERNAL_PMIX],
+                       [Whether or not we are using the internal PMIx])
+
+    OPAL_SUMMARY_ADD([Miscellaneous], [pmix], [], [$opal_pmix_mode])
 
     OPAL_VAR_SCOPE_POP
 ])
@@ -141,11 +165,7 @@ dnl
 dnl only safe to call from OPAL_CONFIG_PMIX, assumes variables from
 dnl there are set.
 AC_DEFUN([_OPAL_CONFIG_PMIX_EXTERNAL], [
-    OPAL_VAR_SCOPE_PUSH([opal_pmix_CPPFLAGS_save opal_pmix_LDFLAGS_save opal_pmix_LIBS_save opal_pmix_external_support])
-
-    opal_pmix_CPPFLAGS_save=$CPPFLAGS
-    opal_pmix_LDFLAGS_save=$LDFLAGS
-    opal_pmix_LIBS_save=$LIBS
+    OPAL_VAR_SCOPE_PUSH([opal_pmix_min_version opal_pmix_min_num_version opal_pmix_CPPFLAGS_save opal_pmix_LDFLAGS_save opal_pmix_LIBS_save opal_pmix_external_support])
 
     opal_pmix_external_support="yes"
 
@@ -153,44 +173,51 @@ AC_DEFUN([_OPAL_CONFIG_PMIX_EXTERNAL], [
           [opal_pmix_external_support="no"
            AS_IF([test "$opal_pmix_mode" = "external"],
                  [AC_MSG_ERROR([Building against an external PMIx with an internal Libevent or HWLOC is unsupported.  Cannot continue.])])],
-          [AS_IF([test ! -z "$with_pmix_libdir"],
-                 [OPAL_CHECK_WITHDIR([pmix-libdir], [$with_pmix_libdir],
-                                     [libpmix.*])])
-
-           OPAL_CHECK_PACKAGE([opal_pmix],
-                              [pmix.h],
-                              [pmix],
-                              [PMIx_Init],
-                              [],
-                              [$with_pmix],
-                              [$with_pmix_libdir],
-                              [],
-                              [opal_pmix_external_support=no])
-
-           # need these set for the tests below.  If things fail, will undo at the end.
+
+          [dnl Need to explicitly enable wrapper compiler to get the dependent libraries
+           dnl when pkg-config is not available.
+           pmix_USE_WRAPPER_COMPILER=1
+           OAC_CHECK_PACKAGE([pmix],
+                             [opal_pmix],
+                             [pmix.h],
+                             [pmix],
+                             [PMIx_Init],
+                             [],
+                             [opal_pmix_external_support=no])
+
+           opal_pmix_CPPFLAGS_save=$CPPFLAGS
+           opal_pmix_LDFLAGS_save=$LDFLAGS
+           opal_pmix_LIBS_save=$LIBS
+
            CPPFLAGS="$opal_pmix_CPPFLAGS_save $opal_pmix_CPPFLAGS"
            LDFLAGS="$opal_pmix_LDFLAGS_save $opal_pmix_LDFLAGS"
            LIBS="$opal_pmix_LIBS_save $opal_pmix_LIBS"
 
+           opal_pmix_min_num_version=OMPI_PMIX_NUMERIC_MIN_VERSION
+           opal_pmix_min_version=OMPI_PMIX_MIN_VERSION
            AS_IF([test "$opal_pmix_external_support" = "yes"],
-                 [AC_MSG_CHECKING([if external PMIx version is 3.1.5 or greater])
-                    AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <pmix_version.h>]],
-                          [[
-#if PMIX_NUMERIC_VERSION < 0x00030105
-#error "pmix API version is less than 3.1.5"
+                 [AC_MSG_CHECKING([if external PMIx version is OMPI_PMIX_MIN_VERSION or greater])
+                    AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <pmix_version.h>
+                          ]], [[
+#if PMIX_NUMERIC_VERSION < $opal_pmix_min_num_version
+#error "pmix API version is less than $opal_pmix_min_version"
 #endif
                           ]])],
                           [AC_MSG_RESULT([yes])],
                           [AC_MSG_RESULT([no])
                            opal_pmix_external_support=no])])
 
+           CPPFLAGS="$opal_pmix_CPPFLAGS_save"
            LDFLAGS="$opal_pmix_LDFLAGS_save"
            LIBS="$opal_pmix_LIBS_save"
 
            AS_IF([test "$opal_pmix_external_support" = "yes"],
-                 [$1],
-                 [CPPFLAGS="$opal_pmix_CPPFLAGS_save"
-                  $2])])
+                 [dnl Do not add libevent libs to LIBS until late, because
+                  dnl it will screw up other tests (like the pthread tests)
+                  opal_pmix_BUILD_LIBS="${opal_pmix_LIBS}"
+
+                  $1],
+                 [$2])])
 
     OPAL_VAR_SCOPE_POP
 ])
@@ -201,13 +228,74 @@ dnl Expectation is that this is called only if external fails, the
 dnl caller configured libpmix configure, and the configure script
 dnl succeeded.
 AC_DEFUN([_OPAL_CONFIG_PMIX_INTERNAL_POST], [
-    opal_pmix_CPPFLAGS="-I$OMPI_TOP_BUILDDIR/3rd-party/openpmix/include -I$OMPI_TOP_SRCDIR/3rd-party/openpmix/include"
-    opal_pmix_LDFLAGS=""
-    opal_pmix_LIBS="$OMPI_TOP_BUILDDIR/3rd-party/openpmix/src/libpmix.la"
+    dnl pkg-config might not always be available, and we always need
+    dnl to pull in the relevant flags.  We parse the compiler wrapper
+    dnl config file all the time rather than use pkg-config when
+    dnl available, so that we have a smaller testing surface.
+    datafile="${OMPI_TOP_BUILDDIR}/3rd-party/openpmix/src/tools/wrapper/pmixcc-wrapper-data.txt"
+    AS_IF([test ! -f "${datafile}"],
+          [AC_MSG_ERROR([Cannot find PMIx wrapper compiler data.  Aborting])])
+
+    pmix_internal_happy=1
+
+    dnl Don't pull LDFLAGS, because we don't have a good way to avoid
+    dnl a -L to our install directory, which can cause some weirdness
+    dnl if there's an old OMPI install there.  And it makes filtering
+    dnl redundant flags easier.
+    opal_pmix_LDFLAGS=
+
+    AC_CACHE_CHECK([for internal pmix static LDFLAGS],
+                   [opal_internal_pmix_cv_STATIC_LDFLAGS],
+                   [_OPAL_CONFIG_PMIX_GET_WRAPPER_FIELD([${datafile}], [linker_flags_static],
+                                                        [opal_internal_pmix_cv_STATIC_LDFLAGS],
+                                                        [], [pmix_internal_happy=0])])
+    opal_pmix_STATIC_LDFLAGS="${opal_internal_pmix_cv_STATIC_LDFLAGS}"
+
+    AC_CACHE_CHECK([for internal pmix LIBS],
+                   [opal_internal_pmix_cv_LIBS],
+                   [_OPAL_CONFIG_PMIX_GET_WRAPPER_FIELD([${datafile}], [libs],
+                                                        [opal_internal_pmix_cv_LIBS],
+                                                        [], [pmix_internal_happy=0])])
+    opal_pmix_LIBS="${opal_internal_pmix_cv_LIBS}"
+
+    AC_CACHE_CHECK([for internal pmix static LIBS],
+                   [opal_internal_pmix_cv_STATIC_LIBS],
+                   [_OPAL_CONFIG_PMIX_GET_WRAPPER_FIELD([${datafile}], [libs_static],
+                                                        [opal_internal_pmix_cv_STATIC_LIBS],
+                                                        [], [pmix_internal_happy=0])])
+    opal_pmix_STATIC_LIBS="${opal_internal_pmix_cv_STATIC_LIBS}"
 
-    CPPFLAGS="$CPPFLAGS $opal_pmix_CPPFLAGS"
+    AC_MSG_CHECKING([for internal pmix pkg-config module])
+    opal_pmix_PC_MODULES="pmix"
+    AC_MSG_RESULT([${opal_pmix_PC_MODULES}])
 
-    opal_pmix_header="$OMPI_TOP_SRCDIR/opal/mca/pmix/pmix-3rdparty.h"
+    AS_IF([test ${pmix_internal_happy} -eq 0],
+          [AC_MSG_ERROR([Failed to retrieve PMIx wrapper data.  Aborting.])])
+
+    # Overwrite the wrapper data results for CPPFLAGS, because it's
+    # the install dir location, not the build location.
+    opal_pmix_CPPFLAGS="-I$OMPI_TOP_BUILDDIR/3rd-party/openpmix/include -I$OMPI_TOP_SRCDIR/3rd-party/openpmix/include -I$OMPI_TOP_BUILDDIR/3rd-party/openpmix/ -I$OMPI_TOP_SRCDIR/3rd-party/openpmix/"
+    opal_pmix_BUILD_CPPFLAGS="${opal_pmix_CPPFLAGS}"
+
+    opal_pmix_BUILD_LIBS="$OMPI_TOP_BUILDDIR/3rd-party/openpmix/src/libpmix.la"
 
     OPAL_3RDPARTY_SUBDIRS="$OPAL_3RDPARTY_SUBDIRS openpmix"
 ])
+
+
+AC_DEFUN([_OPAL_CONFIG_PMIX_GET_WRAPPER_FIELD], [
+    OAC_LOG_COMMAND([wrapper_field_results=`sed -ne 's/$2=\(.*\)/\1/p' $1 2>&1`],
+                    [AS_VAR_COPY([$3], [wrapper_field_results])
+                     $4],
+                    [$5])
+    OAC_LOG_MSG([wrapper field output: $2: ${wrapper_field_results}], [1])
+])
+
+
+dnl We need to delay adding .la files to LIBS until the very end of
+dnl configure, to avoid pulling it into other configure tests.
+AC_DEFUN([OPAL_CONFIG_PMIX_INTERNAL_LIBS_HANDLER], [
+    OPAL_FLAGS_APPEND_UNIQ([CPPFLAGS], [${opal_pmix_CPPFLAGS}])
+    OPAL_FLAGS_APPEND_UNIQ([LDFLAGS], [${opal_pmix_LDFLAGS}])
+    OPAL_FLAGS_APPEND_MOVE([LIBS], [${opal_pmix_BUILD_LIBS}])
+])
diff --git a/config/opal_config_subdir.m4 b/config/opal_config_subdir.m4
index b6e20c802b3..0b920bf7e3a 100644
--- a/config/opal_config_subdir.m4
+++ b/config/opal_config_subdir.m4
@@ -52,30 +52,30 @@ if test "$subdir_dir" != ":" && test -d $srcdir/$subdir_dir; then
 
     case $srcdir in
     .)
-	;;
+        ;;
     *)
-	{ case $subdir_dir in
-	[[\\/]]* | ?:[[\\/]]* ) total_dir=;;
-	*)                      total_dir=.;;
-	esac
-	temp=$subdir_dir
-	for dir_part in `IFS='/\\'; set X $temp; shift; echo "$[@]"`; do
-	    case $dir_part in
-	    # Skip DOS drivespec
-	    ?:) total_dir=$dir_part ;;
-	    *)  total_dir=$total_dir/$dir_part
-		test -d "$total_dir" ||
-		mkdir "$total_dir" ||
-		AC_MSG_ERROR([cannot create $subdir_dir])
-		;;
-	    esac
-	done; }
-
-	if test -d ./$subdir_dir; then :;
-	else
-	    AC_MSG_ERROR([cannot create `pwd`/$subdir_dir])
-	fi
-	;;
+        { case $subdir_dir in
+        [[\\/]]* | ?:[[\\/]]* ) total_dir=;;
+        *)                      total_dir=.;;
+        esac
+        temp=$subdir_dir
+        for dir_part in `IFS='/\\'; set X $temp; shift; echo "$[@]"`; do
+            case $dir_part in
+            # Skip DOS drivespec
+            ?:) total_dir=$dir_part ;;
+            *)  total_dir=$total_dir/$dir_part
+                test -d "$total_dir" ||
+                mkdir "$total_dir" ||
+                AC_MSG_ERROR([cannot create $subdir_dir])
+                ;;
+            esac
+        done; }
+
+        if test -d ./$subdir_dir; then :;
+        else
+            AC_MSG_ERROR([cannot create `pwd`/$subdir_dir])
+        fi
+        ;;
     esac
 
     #
@@ -96,17 +96,17 @@ if test "$subdir_dir" != ":" && test -d $srcdir/$subdir_dir; then
 
     case $srcdir in
     .)
-	# In place
-	subdir_srcdir="$srcdir"
-	;;
+        # In place
+        subdir_srcdir="$srcdir"
+        ;;
     [[\\/]* | ?:[\\/]*] )
-	# Absolute path
-	subdir_srcdir="$srcdir/$subdir_dir"
-	;;
+        # Absolute path
+        subdir_srcdir="$srcdir/$subdir_dir"
+        ;;
     *)
-	# Relative path
-	subdir_srcdir="$subdir_dots$srcdir/$subdir_dir"
-	;;
+        # Relative path
+        subdir_srcdir="$subdir_dots$srcdir/$subdir_dir"
+        ;;
     esac
 
     #
@@ -124,13 +124,13 @@ if test "$subdir_dir" != ":" && test -d $srcdir/$subdir_dir; then
     sub_configure="$SHELL '$subdir_srcdir/configure'"
     AC_MSG_NOTICE([running $sub_configure $subdir_args --cache-file=$subdir_cache_file --srcdir=$subdir_srcdir --disable-option-checking])
     eval "$sub_configure $subdir_args \
-	--cache-file=\"\$subdir_cache_file\" --srcdir=\"$subdir_srcdir\" --disable-option-checking"
+        --cache-file=\"\$subdir_cache_file\" --srcdir=\"$subdir_srcdir\" --disable-option-checking"
     if test "$?" = "0"; then
-	eval $subdir_success
-	AC_MSG_NOTICE([$sub_configure succeeded for $subdir_dir])
+        eval $subdir_success
+        AC_MSG_NOTICE([$sub_configure succeeded for $subdir_dir])
     else
-	eval $subdir_failure
-	AC_MSG_NOTICE([$sub_configure *failed* for $subdir_dir])
+        eval $subdir_failure
+        AC_MSG_NOTICE([$sub_configure *failed* for $subdir_dir])
     fi
 
     #
diff --git a/config/opal_config_subdir_args.m4 b/config/opal_config_subdir_args.m4
index 0d5f8febabc..4afc51275e5 100644
--- a/config/opal_config_subdir_args.m4
+++ b/config/opal_config_subdir_args.m4
@@ -43,34 +43,34 @@ shift
 for subdirs_arg
 do
     if test "$subdirs_skip" = "yes"; then
-	subdirs_skip=no
+        subdirs_skip=no
     else
-	case $subdirs_arg in
-	-cache-file | --cache-file | -cache | --cache)
-	    subdirs_skip=yes
-	    ;;
-	--config-cache | -C)
-	    ;;
-	-cache-file=* | --cache-file=*)
-	    ;;
-	-srcdir | --srcdir)
-	    subdirs_skip=yes
-	    ;;
-	-srcdir=* | --srcdir=*)
-	    ;;
-	-with-platform=* | --with-platform=*)
-	    ;;
+        case $subdirs_arg in
+        -cache-file | --cache-file | -cache | --cache)
+            subdirs_skip=yes
+            ;;
+        --config-cache | -C)
+            ;;
+        -cache-file=* | --cache-file=*)
+            ;;
+        -srcdir | --srcdir)
+            subdirs_skip=yes
+            ;;
+        -srcdir=* | --srcdir=*)
+            ;;
+        -with-platform=* | --with-platform=*)
+            ;;
         --with*=internal)
             ;;
         --with*=external)
             ;;
-	*)
-	    case $subdir_arg in
-	    *\'*) subdir_arg=`echo "$subdir_arg" | sed "s/'/'\\\\\\\\''/g"` ;;
-	    esac
-	    subdirs_args="$subdirs_args '$subdirs_arg'"
-	    ;;
-	esac
+        *)
+            case $subdir_arg in
+            *\'*) subdir_arg=`echo "$subdir_arg" | sed "s/'/'\\\\\\\\''/g"` ;;
+            esac
+            subdirs_args="$subdirs_args '$subdirs_arg'"
+            ;;
+        esac
     fi
 done
 
diff --git a/config/opal_config_treematch.m4 b/config/opal_config_treematch.m4
index cfcfba1f119..967946b53ef 100644
--- a/config/opal_config_treematch.m4
+++ b/config/opal_config_treematch.m4
@@ -9,8 +9,7 @@
 # Copyright (c) 2015      Intel, Inc. All rights reserved.
 # Copyright (c) 2019      Research Organization for Information Science
 #                         and Technology (RIST).  All rights reserved.
-# Copyright (c) 2020      Amazon.com, Inc. or its affiliates.
-#                         All Rights reserved.
+# Copyright (c) 2020-2022 Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # Copyright (c) 2021      Google, LLC. All rights reserved.
 # $COPYRIGHT$
 #
@@ -20,73 +19,23 @@
 #
 
 AC_DEFUN([OPAL_CONFIG_TREEMATCH], [
-    OPAL_VAR_SCOPE_PUSH(treematch_files_local)
+    OPAL_VAR_SCOPE_PUSH(treematch_builddir)
 
-    OPAL_3RDPARTY_WITH([treematch], [treematch], [package_treematch])
-
-    if test -z "$opal_config_treematch_happy" ; then
-        AC_ARG_WITH([treematch],
-                    [AS_HELP_STRING([--with-treematch(=DIR)],
-                                    [Build TreeMatch topology support, optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries])],
+    AC_ARG_WITH([treematch],
+                [AS_HELP_STRING([--with-treematch],
+                                [Build TreeMatch topology support.  Building against an external TreeMatch is no longer supported, so valid values are "yes" or "no".])],
 		    [], [with_treematch=yes])
-        AC_ARG_WITH([treematch-include],
-                    [AS_HELP_STRING([--with-treematch-include(=DIR)],
-                                    ["Search for TreeMatch headers in DIR"])])
-        AC_ARG_WITH([treematch-libdir],
-                    [AS_HELP_STRING([--with-treematch-libdir(=DIR)],
-                                    ["Search for TreeMatch libraries in DIR"])])
-
-        treematch_files_local=no
-        opal_config_treematch_happy=0
 
-	if test "$with_treematch" != no ; then
-	    # Argument sanity checks.
-	    if (test -z "$with_treematch_include" && test -n "$with_treematch_libdir") ||
-		   (test -n "$with_treematch_include" && test -z "$with_treematch_libdir") ; then
-		AC_MSG_ERROR([specified only one of --with-treematch-include or --with-treematch-libdir. Both (or neither) should be specified])
-	    fi
-	    if test -n "$with_treematch_include" && test "$with_treematch" != yes ; then
-		AC_MSG_ERROR([specified both --with-treematch-include and either --with-treematch=path or --without-treematch])
-	    fi
+    AS_IF([test "$with_treematch" != "yes" -a "$with_treematch" != "no"],
+          [AC_MSG_ERROR([Valid values for --with-treematch are yes or no.])])
 
-	    AC_MSG_CHECKING([TreeMatch source])
-	    if test -n "$with_treematch_include" || test "$with_treematch" != yes ; then
-		if test -z "$with_treematch_include" ; then
-		    with_treematch_include="$with_treematch/include"
-		    with_treematch_libdir="$with_treematch/lib"
-		fi
-		AC_MSG_RESULT([user provided])
-	    else
-		treematch_files_local=yes
-		with_treematch_include="$OMPI_TOP_SRCDIR/3rd-party/treematch"
-		with_treematch_libdir="$OMPI_TOP_BUILDDIR/3rd-party/treematch"
-		AC_MSG_RESULT([in tree (3rd-party)])
-	    fi
+    treematch_builddir="${OMPI_TOP_BUILDDIR}/3rd-party/treematch"
 
-	    OPAL_CHECK_WITHDIR([treematch],
-			       [$with_treematch_include],
-			       [tm_tree.h])
-	    if test $treematch_files_local == no ; then
-		OPAL_CHECK_WITHDIR([treematch-libdir], [$with_treematch_libdir], [libtreematch.*])
-		OPAL_CHECK_PACKAGE([topo_treematch],
-				   [treematch.h],
-				   [treematch],
-				   [tm_build_synthetic_topology],
-				   [],
-				   [${with_treematch_include%include}],
-				   [$with_treematch_libdir],
-				   [opal_config_treematch_happy=1],
-				   [AC_MSG_ERROR([TreeMatch support requested but not found.  Aborting])])
-	    else
-		opal_config_treematch_happy=1
-	    fi
-	fi
+    AS_IF([test "${opal_config_treematch_run_once}" != "yes"],
+          [AS_IF([! test -d ${treematch_builddir}],
+                 [mkdir -p ${treematch_builddir}])
 
-	if test $opal_config_treematch_happy -eq 1 && test $treematch_files_local = yes ; then
-	    AS_IF([! test -d $OMPI_TOP_BUILDDIR/3rd-party/treematch],
-                  [mkdir -p $OMPI_TOP_BUILDDIR/3rd-party/treematch])
-	    AC_CONFIG_FILES([3rd-party/treematch/Makefile])
-	    cat > $OMPI_TOP_BUILDDIR/3rd-party/treematch/config.h << EOF
+           cat > ${treematch_builddir}/config.h << EOF
 /*
  * This file is automatically generated by configure.  Edits will be lost
  *
@@ -96,25 +45,22 @@ AC_DEFUN([OPAL_CONFIG_TREEMATCH], [
  * see https://github.com/open-mpi/ompi/pull/6185#issuecomment-458807930
  */
 EOF
-	    opal_config_treematch_LIBS=$OMPI_TOP_BUILDDIR/3rd-party/treematch/libtreematch.la
-	    opal_config_treematch_LDFLAGS=
-	else
-	    opal_config_treematch_LIBS="-ltreematch"
-	    opal_config_treematch_LDFLAGS="-L$with_treematch_libdir"
-	fi
-    fi
 
-    if test $opal_config_treematch_happy -eq 1 ; then
-	$1_CPPFLAGS="[$]$1_CPPFLAGS -I$with_treematch_include"
-	$1_LDFLAGS="[$]$1_LDFLAGS $opal_config_treematch_LDFLAGS"
-	$1_LIBS="[$]$1_LIBS $opal_config_treematch_LIBS"
-    fi
+           AC_CONFIG_FILES([3rd-party/treematch/Makefile])
+           OPAL_3RDPARTY_DIST_SUBDIRS="$OPAL_3RDPARTY_DIST_SUBDIRS treematch"
+
+           AS_IF([test "${with_treematch}" = "yes"],
+                 [OPAL_3RDPARTY_SUBDIRS="$OPAL_3RDPARTY_SUBDIRS treematch"])
+
+           opal_config_treematch_run_once=yes])
+
+    $1_CPPFLAGS="-I$OMPI_TOP_SRCDIR/3rd-party/treematch"
+    $1_LIBS="${OMPI_TOP_BUILDDIR}/3rd-party/treematch/libtreematch.la"
+
+    AS_IF([test "${with_treematch}" = "yes"],
+          [$2],
+          [AC_MSG_NOTICE([Treematch explicitly disabled via --with-treematch=no])
+           $3])
 
-    OPAL_3RDPARTY_SUBDIRS="$OPAL_3RDPARTY_SUBDIRS treematch"
-    OPAL_3RDPARTY_DIST_SUBDIRS="$OPAL_3RDPARTY_DIST_SUBDIRS treematch"
-    
-    AS_IF([test $opal_config_treematch_happy -eq 1], [$2], [$3])
     OPAL_VAR_SCOPE_POP
 ])
-
-    
diff --git a/config/opal_configure_options.m4 b/config/opal_configure_options.m4
index 78dc6ba413c..66a5b3e5a39 100644
--- a/config/opal_configure_options.m4
+++ b/config/opal_configure_options.m4
@@ -3,14 +3,14 @@ dnl
 dnl Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
 dnl                         University Research and Technology
 dnl                         Corporation.  All rights reserved.
-dnl Copyright (c) 2004-2005 The University of Tennessee and The University
+dnl Copyright (c) 2004-2024 The University of Tennessee and The University
 dnl                         of Tennessee Research Foundation.  All rights
 dnl                         reserved.
 dnl Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
 dnl                         University of Stuttgart.  All rights reserved.
 dnl Copyright (c) 2004-2005 The Regents of the University of California.
 dnl                         All rights reserved.
-dnl Copyright (c) 2006-2020 Cisco Systems, Inc.  All rights reserved
+dnl Copyright (c) 2006-2022 Cisco Systems, Inc.  All rights reserved
 dnl Copyright (c) 2007      Sun Microsystems, Inc.  All rights reserved.
 dnl Copyright (c) 2009      IBM Corporation.  All rights reserved.
 dnl Copyright (c) 2009      Los Alamos National Security, LLC.  All rights
@@ -20,6 +20,9 @@ dnl Copyright (c) 2011-2013 NVIDIA Corporation.  All rights reserved.
 dnl Copyright (c) 2013-2017 Intel, Inc.  All rights reserved.
 dnl Copyright (c) 2015      Research Organization for Information Science
 dnl                         and Technology (RIST). All rights reserved.
+dnl Copyright (c) 2020      Amazon.com, Inc. or its affiliates.  All Rights
+dnl Copyright (c) 2019-2021 Triad National Security, LLC. All rights
+dnl                         reserved.
 dnl
 dnl $COPYRIGHT$
 dnl
@@ -29,14 +32,14 @@ dnl $HEADER$
 dnl
 
 AC_DEFUN([OPAL_CONFIGURE_OPTIONS],[
-opal_show_subtitle "OPAL Configuration options"
+opal_show_subtitle "General configuration options"
 
 
 #
 # Is this a developer copy?
 #
 
-if test -d .git; then
+if test -d ${OPAL_TOP_SRCDIR}/.git; then
     OPAL_DEVEL=1
 else
     OPAL_DEVEL=0
@@ -178,10 +181,47 @@ AC_DEFINE_UNQUOTED(OPAL_ENABLE_TIMING, $WANT_TIMING,
 AM_CONDITIONAL([OPAL_COMPILE_TIMING], [test "$WANT_TIMING" = "1"])
 AM_CONDITIONAL([OPAL_INSTALL_TIMING_BINARIES], [test "$WANT_TIMING" = "1" && test "$enable_binaries" != "no"])
 
+# Later calls to AC_PROG_CC/CXX/FC can
+# inject things like -O2 into compile flags if they are
+# not defined, which we don't want. Make sure these flags
+# are at least set to an empty string now.
+#
+# Complicating matters is that autogen can re-order
+# these calls toward the top of configure. This block should
+# be at/near the top, so do it now.
+#
 if test "$WANT_DEBUG" = "0"; then
     CFLAGS="-DNDEBUG $CFLAGS"
     CXXFLAGS="-DNDEBUG $CXXFLAGS"
+
+    # NDEBUG doesn't exist in fortran, so just make sure it's defined.
+    if [ test -z "$FCFLAGS" ]; then
+      FCFLAGS=""
+    fi
+else
+    # Do we want debugging symbols?
+    if test "$enable_debug_symbols" != "no" ; then
+        CFLAGS="$CFLAGS -g"
+        CXXFLAGS="$CXXFLAGS -g"
+        FCFLAGS="$FCFLAGS -g"
+        AC_MSG_WARN([-g has been added to compiler (--enable-debug)])
+    else
+        # If not set, define compile flags to an empty string
+        # to prevent AC_PROG_CC/FC/CXX from modifying compiler flags.
+        # See: https://www.gnu.org/software/autoconf/manual/autoconf-2.69/html_node/C-Compiler.html
+        # for more info.
+        if [ test -z "$CFLAGS" ]; then
+            CFLAGS=""
+        fi
+        if [ test -z "$CXXFLAGS" ]; then
+            CXXFLAGS=""
+        fi
+        if [ test -z "$FCFLAGS" ]; then
+            FCFLAGS=""
+        fi
+    fi
 fi
+
 AC_DEFINE_UNQUOTED(OPAL_ENABLE_DEBUG, $WANT_DEBUG,
     [Whether we want developer-level debugging code or not])
 
@@ -206,7 +246,6 @@ else
 fi
 AM_CONDITIONAL(WANT_INSTALL_HEADERS, test "$WANT_INSTALL_HEADERS" = 1)
 
-
 #
 # Do we want the pretty-print stack trace feature?
 #
@@ -275,16 +314,12 @@ AC_ARG_ENABLE([dlopen],
                      Disabling dlopen implies --disable-mca-dso.
                      (default: enabled)])])
 if test "$enable_dlopen" = "no" ; then
-    enable_mca_dso=no
-    enable_mca_static=yes
     OPAL_ENABLE_DLOPEN_SUPPORT=0
     AC_MSG_RESULT([no])
 else
     OPAL_ENABLE_DLOPEN_SUPPORT=1
     AC_MSG_RESULT([yes])
 fi
-AC_DEFINE_UNQUOTED(OPAL_ENABLE_DLOPEN_SUPPORT, $OPAL_ENABLE_DLOPEN_SUPPORT,
-    [Whether we want to enable dlopen support])
 
 
 #
@@ -292,25 +327,24 @@ AC_DEFINE_UNQUOTED(OPAL_ENABLE_DLOPEN_SUPPORT, $OPAL_ENABLE_DLOPEN_SUPPORT,
 #
 
 AC_MSG_CHECKING([for default value of mca_base_component_show_load_errors])
-AC_ARG_ENABLE([show-load-errors-by-default],
-    [AS_HELP_STRING([--enable-show-load-errors-by-default],
-                    [Set the default value for the MCA parameter
-                     mca_base_component_show_load_errors (but can be
-                     overridden at run time by the usual
-                     MCA-variable-setting mechansism).  This MCA variable
-                     controls whether warnings are displayed when an MCA
-                     component fails to load at run time due to an error.
-                     (default: enabled, meaning that
-                      mca_base_component_show_load_errors is enabled
-                      by default])])
-if test "$enable_show_load_errors_by_default" = "no" ; then
-    OPAL_SHOW_LOAD_ERRORS_DEFAULT=0
-    AC_MSG_RESULT([disabled by default])
-else
-    OPAL_SHOW_LOAD_ERRORS_DEFAULT=1
-    AC_MSG_RESULT([enabled by default])
-fi
-AC_DEFINE_UNQUOTED(OPAL_SHOW_LOAD_ERRORS_DEFAULT, $OPAL_SHOW_LOAD_ERRORS_DEFAULT,
+AC_ARG_WITH([show-load-errors],
+            [AS_HELP_STRING([--with-show-load-errors],
+                            [Set the default value for the MCA
+                            parameter
+                            mca_base_component_show_load_errors (but
+                            can be overridden at run time by the usual
+                            MCA-variable-setting mechansism).
+                            (default: "all")])])
+
+AS_IF([test -z "$with_show_load_errors" -o "$with_show_load_errors" = "yes"],
+      [with_show_load_errors=all
+       AC_MSG_RESULT([enabled for all])],
+      [AS_IF([test "$with_show_load_errors" = "no"],
+             [with_show_load_errors=none
+              AC_MSG_RESULT([disabled for all])],
+             [AC_MSG_RESULT([$with_show_load_errors])])])
+
+AC_DEFINE_UNQUOTED(OPAL_SHOW_LOAD_ERRORS_DEFAULT, ["$with_show_load_errors"],
                    [Default value for mca_base_component_show_load_errors MCA variable])
 
 
@@ -335,29 +369,6 @@ AC_DEFINE_UNQUOTED([OPAL_ENABLE_HETEROGENEOUS_SUPPORT],
                    [Enable features required for heterogeneous support])
 
 
-if test "$opal_want_heterogeneous" = 1; then
-    ompi_cv_c_word_size_align=yes
-else
-    AC_CACHE_CHECK([if word-sized integers must be word-size aligned],
-        [ompi_cv_c_word_size_align],
-        [AC_LANG_PUSH(C)
-         AC_RUN_IFELSE([AC_LANG_PROGRAM([dnl
-#include <stdlib.h>], [[    long data[2] = {0, 0};
-    long *lp;
-    int *ip;
-    ip = (int*) data;
-    ip++;
-    lp = (long*) ip;
-    return lp[0]; ]])],
-            [ompi_cv_c_word_size_align=no],
-            [ompi_cv_c_word_size_align=yes],
-            [ompi_cv_c_word_size_align=yes])])
-fi
-AS_IF([test $ompi_cv_c_word_size_align = yes], [results=1], [results=0])
-AC_DEFINE_UNQUOTED([OPAL_ALIGN_WORD_SIZE_INTEGERS], [$results],
-    [set to 1 if word-size integers must be aligned to word-size padding to prevent bus errors])
-
-
 #
 # Cross-compile data
 #
@@ -400,6 +411,7 @@ AM_CONDITIONAL([OPAL_WANT_SCRIPT_WRAPPER_COMPILERS], [test "$enable_script_wrapp
 #
 # Support per-user config files?
 #
+OPAL_VAR_SCOPE_PUSH([result])
 AC_ARG_ENABLE([per-user-config-files],
    [AS_HELP_STRING([--enable-per-user-config-files],
       [Disable per-user configuration files, to save disk accesses during job start-up.  This is likely desirable for large jobs.  Note that this can also be achieved by environment variables at run-time.  (default: enabled)])])
@@ -410,6 +422,7 @@ else
 fi
 AC_DEFINE_UNQUOTED([OPAL_WANT_HOME_CONFIG_FILES], [$result],
      [Enable per-user config files])
+OPAL_VAR_SCOPE_POP
 
 #
 # Do we want to enable IPv6 support?
@@ -500,7 +513,7 @@ OPAL_WITH_OPTION_MIN_MAX_VALUE(error_string,    256,  64, 1024)
 # Min length according to MPI-2.1, p. 236 (information passed in ompi/communicator/comm.c: min only 48)
 OPAL_WITH_OPTION_MIN_MAX_VALUE(object_name,      64,  64,  256)
 
-# Min and Max length according to MPI-2.1, p. 287 is 32; longest key in ROMIO however 33
+# Min and Max length according to MPI-2.1, p. 287 is 32
 OPAL_WITH_OPTION_MIN_MAX_VALUE(info_key,         36,  34,  255)
 
 # No lower and upper bound required or enforced!
@@ -509,9 +522,13 @@ OPAL_WITH_OPTION_MIN_MAX_VALUE(info_val,        256,  32, 1024)
 # Min length according to _POSIX_HOST_NAME_MAX=255 (4*HOST_NAME_MAX)
 OPAL_WITH_OPTION_MIN_MAX_VALUE(port_name,      1024, 255, 2048)
 
-# Min length accroding to MPI-2.1, p. 418
+# Min length according to MPI-2.1, p. 418
 OPAL_WITH_OPTION_MIN_MAX_VALUE(datarep_string,  128,  64,  256)
 
+OPAL_WITH_OPTION_MIN_MAX_VALUE(pset_name_len,  512,  512, 4096)
+
+OPAL_WITH_OPTION_MIN_MAX_VALUE(stringtag_len,     1024, 256, 2048)
+
 # some systems don't want/like getpwuid
 AC_MSG_CHECKING([if want getpwuid support])
 AC_ARG_ENABLE([getpwuid],
diff --git a/config/opal_functions.m4 b/config/opal_functions.m4
index 7ae5ff994cb..aed34160edc 100644
--- a/config/opal_functions.m4
+++ b/config/opal_functions.m4
@@ -16,6 +16,8 @@ dnl Copyright (c) 2009-2020 Cisco Systems, Inc.  All rights reserved.
 dnl Copyright (c) 2014      Intel, Inc. All rights reserved.
 dnl Copyright (c) 2015-2017 Research Organization for Information Science
 dnl                         and Technology (RIST). All rights reserved.
+dnl Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+dnl                         reserved.
 dnl
 dnl $COPYRIGHT$
 dnl
@@ -200,274 +202,52 @@ dnl #######################################################################
 dnl #######################################################################
 dnl #######################################################################
 
-AC_DEFUN([OPAL_LOG_MSG],[
 # 1 is the message
 # 2 is whether to put a prefix or not
-if test -n "$2"; then
-    echo "configure:__oline__: $1" >&5
-else
-    echo $1 >&5
-fi])dnl
+AC_DEFUN([OPAL_LOG_MSG],
+[AS_IF([test -n "$2"], [OAC_LOG_MSG([$1])], [OAC_LOG_MSG_NOPREFIX([$1])])])
 
 dnl #######################################################################
 dnl #######################################################################
 dnl #######################################################################
 
-AC_DEFUN([OPAL_LOG_FILE],[
-# 1 is the filename
-if test -n "$1" && test -f "$1"; then
-    cat $1 >&5
-fi])dnl
+m4_copy([OAC_LOG_FILE], [OPAL_LOG_FILE])
 
 dnl #######################################################################
 dnl #######################################################################
 dnl #######################################################################
 
-AC_DEFUN([OPAL_LOG_COMMAND],[
-# 1 is the command
-# 2 is actions to do if success
-# 3 is actions to do if fail
-echo "configure:__oline__: $1" >&5
-$1 1>&5 2>&1
-opal_status=$?
-OPAL_LOG_MSG([\$? = $opal_status], 1)
-if test "$opal_status" = "0"; then
-    unset opal_status
-    $2
-else
-    unset opal_status
-    $3
-fi])dnl
+m4_copy([OAC_LOG_COMMAND], [OPAL_LOG_COMMAND])
 
 dnl #######################################################################
 dnl #######################################################################
 dnl #######################################################################
 
-AC_DEFUN([OPAL_UNIQ],[
-# 1 is the variable name to be uniq-ized
-opal_name=$1
-
-# Go through each item in the variable and only keep the unique ones
-
-opal_count=0
-for val in ${$1}; do
-    opal_done=0
-    opal_i=1
-    opal_found=0
-
-    # Loop over every token we've seen so far
-
-    opal_done="`expr $opal_i \> $opal_count`"
-    while test "$opal_found" = "0" && test "$opal_done" = "0"; do
-
-	# Have we seen this token already?  Prefix the comparison with
-	# "x" so that "-Lfoo" values won't be cause an error.
-
-	opal_eval="expr x$val = x\$opal_array_$opal_i"
-	opal_found=`eval $opal_eval`
-
-	# Check the ending condition
-
-	opal_done="`expr $opal_i \>= $opal_count`"
-
-	# Increment the counter
-
-	opal_i="`expr $opal_i + 1`"
-    done
-
-    # If we didn't find the token, add it to the "array"
-
-    if test "$opal_found" = "0"; then
-	opal_eval="opal_array_$opal_i=$val"
-	eval $opal_eval
-	opal_count="`expr $opal_count + 1`"
-    else
-	opal_i="`expr $opal_i - 1`"
-    fi
-done
-
-# Take all the items in the "array" and assemble them back into a
-# single variable
-
-opal_i=1
-opal_done="`expr $opal_i \> $opal_count`"
-opal_newval=
-while test "$opal_done" = "0"; do
-    opal_eval="opal_newval=\"$opal_newval \$opal_array_$opal_i\""
-    eval $opal_eval
-
-    opal_eval="unset opal_array_$opal_i"
-    eval $opal_eval
-
-    opal_done="`expr $opal_i \>= $opal_count`"
-    opal_i="`expr $opal_i + 1`"
-done
-
-# Done; do the assignment
-
-opal_newval="`echo $opal_newval`"
-opal_eval="$opal_name=\"$opal_newval\""
-eval $opal_eval
-
-# Clean up
-
-unset opal_name opal_i opal_done opal_newval opal_eval opal_count])dnl
+m4_copy([OAC_UNIQ], [OPAL_UNIQ])
 
 dnl #######################################################################
 dnl #######################################################################
 dnl #######################################################################
 
-# OPAL_APPEND_UNIQ(variable, new_argument)
-# ----------------------------------------
-# Append new_argument to variable if not already in variable.  This assumes a
-# space separated list.
-#
-# This could probably be made more efficient :(.
-AC_DEFUN([OPAL_APPEND_UNIQ], [
-for arg in $2; do
-    opal_found=0;
-    for val in ${$1}; do
-        if test "x$val" = "x$arg" ; then
-            opal_found=1
-            break
-        fi
-    done
-    if test "$opal_found" = "0" ; then
-        if test -z "$$1"; then
-            $1="$arg"
-        else
-            $1="$$1 $arg"
-        fi
-    fi
-done
-unset opal_found
-])
+m4_copy([OAC_APPEND], [OPAL_APPEND])
 
 dnl #######################################################################
 dnl #######################################################################
 dnl #######################################################################
 
-# Remove all duplicate -I, -L, and -l flags from the variable named $1
-AC_DEFUN([OPAL_FLAGS_UNIQ],[
-    # 1 is the variable name to be uniq-ized
-    opal_name=$1
-
-    # Go through each item in the variable and only keep the unique ones
-
-    opal_count=0
-    for val in ${$1}; do
-        opal_done=0
-        opal_i=1
-        opal_found=0
-
-        # Loop over every token we've seen so far
-
-        opal_done="`expr $opal_i \> $opal_count`"
-        while test "$opal_found" = "0" && test "$opal_done" = "0"; do
+m4_copy([OAC_APPEND_UNIQ], [OPAL_APPEND_UNIQ])
 
-            # Have we seen this token already?  Prefix the comparison
-            # with "x" so that "-Lfoo" values won't be cause an error.
-
-	    opal_eval="expr x$val = x\$opal_array_$opal_i"
-	    opal_found=`eval $opal_eval`
-
-            # Check the ending condition
-
-	    opal_done="`expr $opal_i \>= $opal_count`"
-
-            # Increment the counter
-
-	    opal_i="`expr $opal_i + 1`"
-        done
-
-        # Check for special cases where we do want to allow repeated
-        # arguments (per
-        # https://www.open-mpi.org/community/lists/devel/2012/08/11362.php
-        # and
-        # https://github.com/open-mpi/ompi/issues/324).
-
-        case $val in
-        -Xclang)
-                opal_found=0
-                opal_i=`expr $opal_count + 1`
-                ;;
-        -framework)
-                opal_found=0
-                opal_i=`expr $opal_count + 1`
-                ;;
-        --param)
-                opal_found=0
-                opal_i=`expr $opal_count + 1`
-                ;;
-        esac
-
-        # If we didn't find the token, add it to the "array"
-
-        if test "$opal_found" = "0"; then
-	    opal_eval="opal_array_$opal_i=$val"
-	    eval $opal_eval
-	    opal_count="`expr $opal_count + 1`"
-        else
-	    opal_i="`expr $opal_i - 1`"
-        fi
-    done
-
-    # Take all the items in the "array" and assemble them back into a
-    # single variable
-
-    opal_i=1
-    opal_done="`expr $opal_i \> $opal_count`"
-    opal_newval=
-    while test "$opal_done" = "0"; do
-        opal_eval="opal_newval=\"$opal_newval \$opal_array_$opal_i\""
-        eval $opal_eval
-
-        opal_eval="unset opal_array_$opal_i"
-        eval $opal_eval
-
-        opal_done="`expr $opal_i \>= $opal_count`"
-        opal_i="`expr $opal_i + 1`"
-    done
-
-    # Done; do the assignment
-
-    opal_newval="`echo $opal_newval`"
-    opal_eval="$opal_name=\"$opal_newval\""
-    eval $opal_eval
-
-    # Clean up
+dnl #######################################################################
+dnl #######################################################################
+dnl #######################################################################
 
-    unset opal_name opal_i opal_done opal_newval opal_eval opal_count
-])dnl
+m4_copy([OAC_FLAGS_APPEND_UNIQ], [OPAL_FLAGS_APPEND_UNIQ])
 
 dnl #######################################################################
 dnl #######################################################################
 dnl #######################################################################
 
-# OPAL_FLAGS_APPEND_UNIQ(variable, new_argument)
-# ----------------------------------------------
-# Append new_argument to variable if:
-#
-# - the argument does not begin with -I, -L, or -l, or
-# - the argument begins with -I, -L, or -l, and it's not already in variable
-#
-# This macro assumes a space separated list.
-AC_DEFUN([OPAL_FLAGS_APPEND_UNIQ], [
-    OPAL_VAR_SCOPE_PUSH([opal_tmp opal_append])
-
-    for arg in $2; do
-        opal_tmp=`echo $arg | cut -c1-2`
-        opal_append=1
-        AS_IF([test "$opal_tmp" = "-I" || test "$opal_tmp" = "-L" || test "$opal_tmp" = "-l"],
-              [for val in ${$1}; do
-                   AS_IF([test "x$val" = "x$arg"], [opal_append=0])
-               done])
-        AS_IF([test "$opal_append" = "1"],
-              [AS_IF([test -z "$$1"], [$1=$arg], [$1="$$1 $arg"])])
-    done
-
-    OPAL_VAR_SCOPE_POP
-])
+m4_copy([OAC_FLAGS_APPEND_MOVE], [OPAL_FLAGS_APPEND_MOVE])
 
 dnl #######################################################################
 dnl #######################################################################
@@ -479,88 +259,68 @@ dnl #######################################################################
 # ~/.whatever file (and we do not want the error messages to be part
 # of the assignment in foo=`which <prog>`). This macro ensures that we
 # get a sane executable value.
-AC_DEFUN([OPAL_WHICH],[
+#
 # 1 is the variable name to do "which" on
 # 2 is the variable name to assign the return value to
+AC_DEFUN([OPAL_WHICH],[
+    OPAL_VAR_SCOPE_PUSH([opal_prog opal_file opal_dir opal_sentinel])
+
+    opal_prog=$1
+
+    # There are 3 cases:
+
+    # 1. opal_prog is an absolute filename.  If that absolute filename
+    # exists and is executable, return $2 with that name.  Otherwise,
+    # $2 is unchanged.
+
+    # 2. opal_prog is a relative filename (i.e., it contains one or
+    # more /, but does not begin with a /).  If that file exists
+    # relative to where we are right now in the filesystem and is
+    # executable, return the absolute path of that value in $2.
+    # Otherwise, $2 is unchanged.
+
+    # 3. opal_prog contains no /.  Search the PATH for an executable
+    # with the appropriate name.  If found, return the absolute path
+    # in $2.  Otherwise, $2 is unchanged.
+
+    # Note that these three cases are exactly what which(1) does.
+
+    # Note the double square brackets around the case expressions for
+    # m4 escaping.
+    case $opal_prog in
+        [[\\/]]* | ?:[[\\/]]* )
+            # Case 1: absolute
+            AS_IF([test -x "$opal_prog"],
+                  [$2=$opal_prog])
+            ;;
+
+        *[[\\/]]*)
+            # Case 2: relative with 1 or more /
+            AS_IF([test -x "$opal_prog"],
+                  [$2="$cwd/$opal_prog"])
+            ;;
+
+        *)
+            # Case 3: no / at all
+            IFS_SAVE=$IFS
+            IFS=$PATH_SEPARATOR
+            for opal_dir in $PATH; do
+               AS_IF([test -x "$opal_dir/$opal_prog"],
+                     [$2="$opal_dir/$opal_prog"])
+            done
+            IFS=$IFS_SAVE
+            ;;
+    esac
 
-OPAL_VAR_SCOPE_PUSH([opal_prog opal_file opal_dir opal_sentinel])
-
-opal_prog=$1
-
-IFS_SAVE=$IFS
-IFS="$PATH_SEPARATOR"
-for opal_dir in $PATH; do
-    if test -x "$opal_dir/$opal_prog"; then
-        $2="$opal_dir/$opal_prog"
-        break
-    fi
-done
-IFS=$IFS_SAVE
-
-OPAL_VAR_SCOPE_POP
+    OPAL_VAR_SCOPE_POP
 ])dnl
 
 dnl #######################################################################
 dnl #######################################################################
 dnl #######################################################################
 
-# Declare some variables; use OPAL_VAR_SCOPE_POP to ensure that they
-# are cleaned up / undefined.
-AC_DEFUN([OPAL_VAR_SCOPE_PUSH],[
-
-    # Is the private index set?  If not, set it.
-    if test "x$opal_scope_index" = "x"; then
-        opal_scope_index=1
-    fi
-
-    # First, check to see if any of these variables are already set.
-    # This is a simple sanity check to ensure we're not already
-    # overwriting pre-existing variables (that have a non-empty
-    # value).  It's not a perfect check, but at least it's something.
-    for opal_var in $1; do
-        opal_str="opal_str=\"\$$opal_var\""
-        eval $opal_str
-
-        if test "x$opal_str" != "x"; then
-            AC_MSG_WARN([Found configure shell variable clash at line $LINENO!])
-            AC_MSG_WARN([[OPAL_VAR_SCOPE_PUSH] called on "$opal_var",])
-            AC_MSG_WARN([but it is already defined with value "$opal_str"])
-            AC_MSG_WARN([This usually indicates an error in configure.])
-            AC_MSG_ERROR([Cannot continue])
-        fi
-    done
-
-    # Ok, we passed the simple sanity check.  Save all these names so
-    # that we can unset them at the end of the scope.
-    opal_str="opal_scope_$opal_scope_index=\"$1\""
-    eval $opal_str
-    unset opal_str
-
-    env | grep opal_scope
-    opal_scope_index=`expr $opal_scope_index + 1`
-])dnl
-
-# Unset a bunch of variables that were previously set
-AC_DEFUN([OPAL_VAR_SCOPE_POP],[
-    # Unwind the index
-    opal_scope_index=`expr $opal_scope_index - 1`
-    opal_scope_test=`expr $opal_scope_index \> 0`
-    if test "$opal_scope_test" = "0"; then
-        AC_MSG_WARN([[OPAL_VAR_SCOPE_POP] popped too many OPAL configure scopes.])
-        AC_MSG_WARN([This usually indicates an error in configure.])
-        AC_MSG_ERROR([Cannot continue])
-    fi
-
-    # Get the variable names from that index
-    opal_str="opal_str=\"\$opal_scope_$opal_scope_index\""
-    eval $opal_str
-
-    # Iterate over all the variables and unset them all
-    for opal_var in $opal_str; do
-        unset $opal_var
-    done
-])dnl
-
+m4_copy([OAC_VAR_SCOPE_PUSH], [OPAL_VAR_SCOPE_PUSH])
+m4_copy([OAC_VAR_SCOPE_POP], [OPAL_VAR_SCOPE_POP])
 
 dnl #######################################################################
 dnl #######################################################################
diff --git a/config/opal_get_version.m4 b/config/opal_get_version.m4
index 5c200e2161d..3a8f78565e5 100644
--- a/config/opal_get_version.m4
+++ b/config/opal_get_version.m4
@@ -14,6 +14,7 @@ dnl Copyright (c) 2008-2020 Cisco Systems, Inc.  All rights reserved.
 dnl Copyright (c) 2014      Intel, Inc. All rights reserved.
 dnl Copyright (c) 2014-2020 Research Organization for Information Science
 dnl                         and Technology (RIST).  All rights reserved.
+dnl Copyright (c) 2022      IBM Corporation.  All rights reserved.
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
@@ -44,20 +45,20 @@ m4_define([OPAL_GET_VERSION],[
     if test -f "$1"; then
         srcdir=`dirname $1`
         opal_vers=`sed -n "
-	t clear
-	: clear
-	s/^major/$2_MAJOR_VERSION/
-	s/^minor/$2_MINOR_VERSION/
-	s/^release/$2_RELEASE_VERSION/
-	s/^greek/$2_GREEK_VERSION/
-	s/^repo_rev/$2_REPO_REV/
-	s/^tarball_version/$2_TARBALL_VERSION/
-	s/^date/$2_RELEASE_DATE/
-	t print
-	b
-	: print
-	p" < "$1"`
-	[eval] "$opal_vers"
+        t clear
+        : clear
+        s/^major/$2_MAJOR_VERSION/
+        s/^minor/$2_MINOR_VERSION/
+        s/^release/$2_RELEASE_VERSION/
+        s/^greek/$2_GREEK_VERSION/
+        s/^repo_rev/$2_REPO_REV/
+        s/^tarball_version/$2_TARBALL_VERSION/
+        s/^date/$2_RELEASE_DATE/
+        t print
+        b
+        : print
+        p" < "$1"`
+        [eval] "$opal_vers"
 
         $2_VERSION="$$2_MAJOR_VERSION.$$2_MINOR_VERSION.$$2_RELEASE_VERSION"
         $2_VERSION="${$2_VERSION}${$2_GREEK_VERSION}"
diff --git a/config/opal_get_version.m4sh b/config/opal_get_version.m4sh
index d9d45d972a2..4ef056ca1e3 100644
--- a/config/opal_get_version.m4sh
+++ b/config/opal_get_version.m4sh
@@ -33,29 +33,29 @@ else
     OPAL_GET_VERSION([$srcfile],[OPAL])
 
     if test "$option" = ""; then
-	option="--full"
+        option="--full"
     fi
 fi
 
 case "$option" in
     --full|-v|--version)
-	echo $OPAL_VERSION
-	;;
+        echo $OPAL_VERSION
+        ;;
     --major)
-	echo $OPAL_MAJOR_VERSION
-	;;
+        echo $OPAL_MAJOR_VERSION
+        ;;
     --minor)
-	echo $OPAL_MINOR_VERSION
-	;;
+        echo $OPAL_MINOR_VERSION
+        ;;
     --release)
-	echo $OPAL_RELEASE_VERSION
-	;;
+        echo $OPAL_RELEASE_VERSION
+        ;;
     --greek)
-	echo $OPAL_GREEK_VERSION
-	;;
+        echo $OPAL_GREEK_VERSION
+        ;;
     --repo-rev)
-	echo $OPAL_REPO_REV
-	;;
+        echo $OPAL_REPO_REV
+        ;;
     --tarball)
         echo $OPAL_TARBALL_VERSION
         ;;
@@ -66,7 +66,7 @@ case "$option" in
         echo ${OPAL_VERSION} : ${OPAL_MAJOR_VERSION} : ${OPAL_MINOR_VERSION} : ${OPAL_RELEASE_VERSION} : ${OPAL_GREEK_VERSION} : ${OPAL_REPO_REV} : ${OPAL_TARBALL_VERSION}
         ;;
     -h|--help)
-	cat <<EOF
+        cat <<EOF
 $0 <srcfile> [<option>]
 
 <srcfile> - Text version file
diff --git a/config/opal_mca.m4 b/config/opal_mca.m4
index 6db2e742103..b425fe63bf2 100644
--- a/config/opal_mca.m4
+++ b/config/opal_mca.m4
@@ -1,4 +1,4 @@
-dnl -*- shell-script -*-
+dnl -*- autoconf -*-
 dnl
 dnl Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
 dnl                         University Research and Technology
@@ -10,10 +10,11 @@ dnl Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
 dnl                         University of Stuttgart.  All rights reserved.
 dnl Copyright (c) 2004-2005 The Regents of the University of California.
 dnl                         All rights reserved.
-dnl Copyright (c) 2010-2016 Cisco Systems, Inc.  All rights reserved.
+dnl Copyright (c) 2010-2021 Cisco Systems, Inc.  All rights reserved
 dnl Copyright (c) 2013-2017 Intel, Inc. All rights reserved.
-dnl Copyright (c) 2018      Amazon.com, Inc. or its affiliates.
-dnl                         All Rights reserved.
+dnl Copyright (c) 2018-2022 Amazon.com, Inc. or its affiliates.  All Rights reserved.
+dnl Copyright (c) 2021-2023 Triad National Security, LLC. All rights
+dnl                         reserved.
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
@@ -41,6 +42,8 @@ AC_DEFUN([OPAL_EVAL_ARG], [$1])
 AC_DEFUN([OPAL_MCA],[
     dnl for OPAL_CONFIGURE_USER env variable
     AC_REQUIRE([OPAL_CONFIGURE_SETUP])
+    dnl For all the path management
+    AC_REQUIRE([OPAL_SETUP_WRAPPER_INIT])
 
     # Set a special flag so that we can detect if the user calls
     # OPAL_WRAPPER_FLAGS_ADD and error.
@@ -59,17 +62,16 @@ AC_DEFUN([OPAL_MCA],[
         [AS_HELP_STRING([--enable-mca-no-build=LIST],
                         [Comma-separated list of <type>-<component> pairs
                          that will not be built.  Example:
-                         "--enable-mca-no-build=btl-portals,oob-ud" will
-                         disable building the "portals" btl and the "ud"
-                         oob components.])])
+                         "--enable-mca-no-build=btl-portals4,topo-treematch" will
+                         disable building the "portals4" btl and the "treematch"
+                         topo components.])])
     AC_ARG_ENABLE([mca-dso],
         [AS_HELP_STRING([--enable-mca-dso=LIST],
                        [Comma-separated list of types and/or
                         type-component pairs that will be built as
                         run-time loadable components (as opposed to
                         statically linked in), if supported on this
-                        platform.  The default is to build all components
-                        as DSOs.])])
+                        platform.])])
     AC_ARG_ENABLE([mca-static],
         [AS_HELP_STRING([--enable-mca-static=LIST],
                        [Comma-separated list of types and/or
@@ -77,7 +79,8 @@ AC_DEFUN([OPAL_MCA],[
                         linked into the library.  The default (if DSOs are
                         supported) is to build all components as DSOs.
                         Enabling a component as static disables it
-                        building as a DSO.])])
+                        building as a DSO.  The default is to build all
+                        components statically.])])
     AC_ARG_ENABLE([mca-direct],
         [AS_HELP_STRING([--enable-mca-direct=LIST],
                        [Comma-separated list of type-component pairs that
@@ -91,7 +94,7 @@ AC_DEFUN([OPAL_MCA],[
     if test "$enable_mca_no_build" = "yes"; then
         AC_MSG_RESULT([yes])
         AC_MSG_ERROR([*** The enable-mca-no-build flag requires an explicit list
-*** of type-component pairs.  For example, --enable-mca-no-build=pml-ob1])
+of type-component pairs.  For example, --enable-mca-no-build=pml-ob1])
     else
         ifs_save="$IFS"
         IFS="${IFS}$PATH_SEPARATOR,"
@@ -103,12 +106,10 @@ AC_DEFUN([OPAL_MCA],[
                 type=$item
             fi
             if test -z $comp ; then
-                str="`echo DISABLE_${type}=1 | sed s/-/_/g`"
-                eval $str
+                AS_VAR_SET([AS_TR_SH([DISABLE_${type}])], [1])
                 msg="$item $msg"
             else
-                str="`echo DISABLE_${type}_${comp}=1 | sed s/-/_/g`"
-                eval $str
+                AS_VAR_SET([AS_TR_SH([DISABLE_${type}_${comp}])], [1])
                 msg="$item $msg"
             fi
         done
@@ -123,11 +124,7 @@ AC_DEFUN([OPAL_MCA],[
     # in the form DIRECT_[type]=[component]
     #
     AC_MSG_CHECKING([which components should be direct-linked into the library])
-    if test "$enable_mca_direct" = "yes" ; then
-        AC_MSG_RESULT([yes])
-        AC_MSG_ERROR([*** The enable-mca-direct flag requires an explicit list of
-*** type-component pairs.  For example, --enable-mca-direct=pml-ob1,coll-basic])
-    elif test ! -z "$enable_mca_direct" && test "$enable_mca_direct" != "" ; then
+    if test -n "$enable_mca_direct" ; then
         #
         # we need to add this into the static list, unless the static list
         # is everything
@@ -145,16 +142,21 @@ AC_DEFUN([OPAL_MCA],[
         IFS="${IFS}$PATH_SEPARATOR,"
         msg=
         for item in $enable_mca_direct; do
-            type="`echo $item | cut -f1 -d-`"
-            comp="`echo $item | cut -f2- -d-`"
+            type="`echo $item | cut -s -f1 -d-`"
+            comp="`echo $item | cut -s -f2- -d-`"
             if test -z $type || test -z $comp ; then
-                AC_MSG_ERROR([*** The enable-mca-direct flag requires a
-*** list of type-component pairs.  Invalid input detected.])
-            else
-                str="`echo DIRECT_$type=$comp | sed s/-/_/g`"
-                eval $str
-                msg="$item $msg"
+                AC_MSG_ERROR([enable-mca-direct requires a list of type-component pairs (ex. --enable-mca-direct=pml-ob1,smsc-xpmem)])
             fi
+
+            var_name=AS_TR_SH([DIRECT_${type}])
+            AS_VAR_COPY([var_value], [$var_name])
+
+            if test -n "$var_value" ; then
+                AC_MSG_ERROR([enable-mca-direct can only enable a single component per framework: specified both ${type}-${var_value} and ${type}-${comp}.])
+            fi
+
+            AS_VAR_SET([$var_name], AS_TR_SH([${comp}]))
+            msg="$item $msg"
         done
         IFS="$ifs_save"
     fi
@@ -165,28 +167,36 @@ AC_DEFUN([OPAL_MCA],[
     # Second, set the DSO_all and STATIC_all variables.  conflict
     # resolution (prefer static) is done in the big loop below
     #
+    # Exception here is the components of the accelerator framework,
+    # which by default are built to be dynamic, except for null.
+    #
     AC_MSG_CHECKING([which components should be run-time loadable])
     if test "$enable_static" != "no"; then
         DSO_all=0
-        msg=none
-    elif test -z "$enable_mca_dso" || test "$enable_mca_dso" = "yes"; then
-        DSO_all=1
-        msg=all
+        msg="none (static libraries built)"
+    elif test "$OPAL_ENABLE_DLOPEN_SUPPORT" = 0; then
+        DSO_all=0
+        msg="none (dlopen disabled)"
     elif test "$enable_mca_dso" = "no"; then
         DSO_all=0
         msg=none
-        enable_dlopen=no
+    elif test "$enable_mca_dso" = "yes"; then
+        DSO_all=1
+        msg=all
     else
-        DSO_all=0
-        ifs_save="$IFS"
-        IFS="${IFS}$PATH_SEPARATOR,"
-        msg=
-        for item in $enable_mca_dso; do
-            str="`echo DSO_$item=1 | sed s/-/_/g`"
-            eval $str
-            msg="$item $msg"
-        done
-        IFS="$ifs_save"
+       msg=
+       if test -z "$enable_mca_dso"; then
+           enable_mca_dso="accelerator-cuda,accelerator-rocm,accelerator-ze,btl-smcuda,rcache-gpusm,rcache-rgpusm"
+           msg="(default)"
+       fi
+       DSO_all=0
+       ifs_save="$IFS"
+       IFS="${IFS}$PATH_SEPARATOR,"
+       for item in $enable_mca_dso; do
+           AS_VAR_SET([AS_TR_SH([DSO_$item])], [1])
+           msg="$item $msg"
+       done
+       IFS="$ifs_save"
     fi
     AC_MSG_RESULT([$msg])
     unset msg
@@ -196,20 +206,22 @@ AC_DEFUN([OPAL_MCA],[
     fi
 
     AC_MSG_CHECKING([which components should be static])
-    if test "$enable_mca_static" = "yes"; then
-        STATIC_all=1
-        msg=all
-    elif test -z "$enable_mca_static" || test "$enable_mca_static" = "no"; then
+    if test -z "$enable_mca_static" ; then
+        STATIC_all=0
+        msg=default
+    elif test "$enable_mca_static" = "no"; then
         STATIC_all=0
         msg=none
+    elif test "$enable_mca_static" = "yes"; then
+        STATIC_all=1
+        msg=all
     else
         STATIC_all=0
         ifs_save="$IFS"
         IFS="${IFS}$PATH_SEPARATOR,"
         msg=
         for item in $enable_mca_static; do
-            str="`echo STATIC_$item=1 | sed s/-/_/g`"
-            eval $str
+            AS_VAR_SET([AS_TR_SH([STATIC_$item])], [1])
             msg="$item $msg"
         done
         IFS="$ifs_save"
@@ -229,13 +241,8 @@ AC_DEFUN([OPAL_MCA],[
     MCA_PROJECT_SUBDIRS=
     MCA_PROJECT_DIST_SUBDIRS=
     m4_foreach(mca_project, [mca_project_list],
-               [# BWB: Until projects have separate configure scripts
-                # and can skip running all of ORTE, just avoid recursing
-                # into orte sub directory if orte disabled
-                if (test "mca_project" = "ompi" && test "$enable_mpi" != "no") || test "mca_project" = "opal" || test "mca_project" = "orte" || test "mca_project" = "oshmem"; then
-                   MCA_PROJECT_SUBDIRS="$MCA_PROJECT_SUBDIRS mca_project"
-                   MCA_PROJECT_DIST_SUBDIRS="$MCA_PROJECT_DIST_SUBDIRS mca_project"
-                fi
+               [MCA_PROJECT_SUBDIRS="$MCA_PROJECT_SUBDIRS mca_project"
+                MCA_PROJECT_DIST_SUBDIRS="$MCA_PROJECT_DIST_SUBDIRS mca_project"
                 MCA_CONFIGURE_PROJECT(mca_project)])
 
     AC_SUBST(MCA_PROJECT_SUBDIRS)
@@ -262,6 +269,9 @@ AC_DEFUN([MCA_CONFIGURE_PROJECT],[
     # can't use a variable rename here because these need to be evaled
     # at auto* time.
 
+    AS_LITERAL_IF([$1], [],
+                  [m4_fatal([MCA_CONFIGURE_PROJECT argument must be a literal])])
+
     opal_show_subtitle "Configuring MCA for $1"
 
     AC_MSG_CHECKING([for frameworks for $1])
@@ -288,6 +298,7 @@ AC_DEFUN([MCA_CONFIGURE_PROJECT],[
     MCA_$1_FRAMEWORK_COMPONENT_DSO_SUBDIRS=
     MCA_$1_FRAMEWORK_COMPONENT_STATIC_SUBDIRS=
     MCA_$1_FRAMEWORK_LIBS=
+    MCA_$1_FRAMEWORK_CORE_LIBS=
 
     m4_foreach(mca_framework, [mca_$1_framework_list],
                [m4_ifval(mca_framework,
@@ -298,14 +309,17 @@ AC_DEFUN([MCA_CONFIGURE_PROJECT],[
                                  MCA_$1_FRAMEWORK_COMPONENT_ALL_SUBDIRS="[\$(MCA_]$1[_]mca_framework[_ALL_SUBDIRS)] $MCA_$1_FRAMEWORK_COMPONENT_ALL_SUBDIRS"
                                  MCA_$1_FRAMEWORK_COMPONENT_DSO_SUBDIRS="[\$(MCA_]$1[_]mca_framework[_DSO_SUBDIRS)] $MCA_$1_FRAMEWORK_COMPONENT_DSO_SUBDIRS"
                                  MCA_$1_FRAMEWORK_COMPONENT_STATIC_SUBDIRS="[\$(MCA_]$1[_]mca_framework[_STATIC_SUBDIRS)] $MCA_$1_FRAMEWORK_COMPONENT_STATIC_SUBDIRS"
+                                 mca_$1_framework_base_lib=
                                 ], [
                                  MCA_$1_FRAMEWORKS="$MCA_$1_FRAMEWORKS mca_framework"
                                  MCA_$1_FRAMEWORKS_SUBDIRS="$MCA_$1_FRAMEWORKS_SUBDIRS [mca/]mca_framework"
                                  MCA_$1_FRAMEWORK_COMPONENT_ALL_SUBDIRS="$MCA_$1_FRAMEWORK_COMPONENT_ALL_SUBDIRS [\$(MCA_]$1[_]mca_framework[_ALL_SUBDIRS)]"
                                  MCA_$1_FRAMEWORK_COMPONENT_DSO_SUBDIRS="$MCA_$1_FRAMEWORK_COMPONENT_DSO_SUBDIRS [\$(MCA_]$1[_]mca_framework[_DSO_SUBDIRS)]"
                                  MCA_$1_FRAMEWORK_COMPONENT_STATIC_SUBDIRS="$MCA_$1_FRAMEWORK_COMPONENT_STATIC_SUBDIRS [\$(MCA_]$1[_]mca_framework[_STATIC_SUBDIRS)]"
-                                 MCA_$1_FRAMEWORK_LIBS="$MCA_$1_FRAMEWORK_LIBS [mca/]mca_framework[/libmca_]mca_framework[.la]"])
-                          MCA_$1_FRAMEWORK_LIBS="$MCA_$1_FRAMEWORK_LIBS [\$(MCA_]$1[_]mca_framework[_STATIC_LTLIBS)]"
+                                 mca_$1_framework_base_lib="[mca/]mca_framework[/libmca_]mca_framework[.la]"])
+                          m4_ifdef([MCA_]$1[_]mca_framework[_CORE_LIB],
+                                   [MCA_$1_FRAMEWORK_CORE_LIBS="$MCA_$1_FRAMEWORK_CORE_LIBS ${mca_$1_framework_base_lib} [\$(MCA_]$1[_]mca_framework[_STATIC_LTLIBS)]"],
+                                   [MCA_$1_FRAMEWORK_LIBS="$MCA_$1_FRAMEWORK_LIBS ${mca_$1_framework_base_lib} [\$(MCA_]$1[_]mca_framework[_STATIC_LTLIBS)]"])
                           m4_ifdef([MCA_]$1[_]mca_framework[_CONFIG],
                                    [MCA_]$1[_]mca_framework[_CONFIG]($1, mca_framework),
                                    [MCA_CONFIGURE_FRAMEWORK($1, mca_framework, 1)])])])
@@ -321,6 +335,7 @@ AC_DEFUN([MCA_CONFIGURE_PROJECT],[
     AC_SUBST(MCA_$1_FRAMEWORK_COMPONENT_DSO_SUBDIRS)
     AC_SUBST(MCA_$1_FRAMEWORK_COMPONENT_STATIC_SUBDIRS)
     AC_SUBST(MCA_$1_FRAMEWORK_LIBS)
+    AC_SUBST(MCA_$1_FRAMEWORK_CORE_LIBS)
 ])
 
 # MCA_ORDER_COMPONENT_LIST(project_name, framework_name)
@@ -444,17 +459,17 @@ AC_DEFUN([MCA_CONFIGURE_FRAMEWORK],[
                          [AS_IF([test $components_last_result -eq 1], [best_mca_component_priority=]OPAL_EVAL_ARG([MCA_$1_$2_]mca_component[_PRIORITY]))])
                    ])])
 
-    # configure components that provide their own configure script.
-    # It would be really hard to run these for "find first that
-    # works", so we don't :)
-    m4_if(OPAL_EVAL_ARG([MCA_$1_]$2[_CONFIGURE_MODE]), [STOP_AT_FIRST], [],
-        [m4_if(OPAL_EVAL_ARG([MCA_$1_]$2[_CONFIGURE_MODE]), [STOP_AT_FIRST_PRIORITY], [],
-             [m4_if(OPAL_EVAL_ARG([MCA_$1_]$2[_CONFIGURE_MODE]), [PRIORITY], [],
-                 [MCA_CHECK_IGNORED_PRIORITY($1, $2)
-                  AS_IF([test "$3" != "0"],
-                        [MCA_CONFIGURE_ALL_CONFIG_COMPONENTS($1, $2, [all_components],
-                                               [static_components], [dso_components],
-                                               [static_ltlibs])])])])])
+    AS_VAR_SET_IF([OPAL_EVAL_ARG([DIRECT_$2])], [
+                      AC_MSG_CHECKING([if direct-selection component exists for $2 framework])
+                      direct_component_happy=no
+                      for component in $all_components ; do
+                          AS_IF([test $component = "$DIRECT_$2"], [direct_component_happy=yes])
+                      done
+                      if test $direct_component_happy = no ; then
+                          AC_MSG_ERROR([direct component $DIRECT_$2 requested but not found in $all_components])
+                      fi
+                      AC_MSG_RESULT([$DIRECT_$2])
+                  ])
 
     MCA_$1_$2_ALL_COMPONENTS="$all_components"
     MCA_$1_$2_STATIC_COMPONENTS="$static_components"
@@ -588,9 +603,7 @@ AC_DEFUN([MCA_CONFIGURE_M4_CONFIG_COMPONENT],[
     MCA_COMPONENT_BUILD_CHECK($1, $2, $3, [should_build=$8], [should_build=0])
     # Allow the component to override the build mode if it really wants to.
     # It is, of course, free to end up calling MCA_COMPONENT_COMPILE_MODE
-    m4_ifdef([MCA_$1_$2_$3_COMPILE_MODE],
-             [MCA_$1_$2_$3_COMPILE_MODE($1, $2, $3, compile_mode)],
-             [MCA_COMPONENT_COMPILE_MODE($1, $2, $3, compile_mode)])
+    MCA_COMPONENT_COMPILE_MODE([$1], [$2], [$3], [compile_mode])
 
     # try to configure the component
     m4_ifdef([MCA_$1_$2_$3_CONFIG],
@@ -619,117 +632,106 @@ AC_DEFUN([MCA_CONFIGURE_M4_CONFIG_COMPONENT],[
     unset compile_mode
 ])
 
-
-######################################################################
-#
-# MCA_CONFIGURE_ALL_CONFIG_COMPONENTS
-#
-# configure all components in the given framework that have configure
-# scripts and should be configured according to the usual rules...
-#
-# USAGE:
-#   MCA_CONFIGURE_ALL_CONFIG_COMPONENTS(project_name,
-#                         framework_name,
-#                         all_components_variable,
-#                         static_components_variable,
-#                         dso_components_variable,
-#                         static_ltlibs_variable)
-#
-######################################################################
-AC_DEFUN([MCA_CONFIGURE_ALL_CONFIG_COMPONENTS],[
-    for component_path in $srcdir/$1/mca/$2/* ; do
-        component="`basename $component_path`"
-        if test -d $component_path && test -x $component_path/configure ; then
-            opal_show_subsubsubtitle "MCA component $2:$component (need to configure)"
-
-            opal_show_verbose "OPAL_MCA_ALL_CONFIG_COMPONENTS: before, should_build=$8"
-            MCA_COMPONENT_BUILD_CHECK($1, $2, $component,
-                                      [should_build=1], [should_build=0])
-            MCA_COMPONENT_COMPILE_MODE($1, $2, $component, compile_mode)
-            opal_show_verbose "OPAL_MCA_ALL_CONFIG_COMPONENTS: after, should_build=$should_build"
-
-            if test "$should_build" = "1" ; then
-                OPAL_CONFIG_SUBDIR([$1/mca/$2/$component],
-                                   [$opal_subdir_args],
-                                   [should_build=1], [should_build=0])
-                opal_show_verbose "OPAL_MCA_ALL_CONFIG_COMPONENTS: after subdir, should_build=$should_build"
-            fi
-
-            if test "$should_build" = "1" ; then
-                # do some extra work to pass flags back from the
-                # top-level configure, the way a configure.m4
-                # component would.
-                infile="$srcdir/$1/mca/$2/$3/post_configure.sh"
-                if test -f $infile; then
-
-                    # First check for the ABORT tag
-                    line="`$GREP ABORT= $infile | cut -d= -f2-`"
-                    if test -n "$line" && test "$line" != "no"; then
-                        AC_MSG_WARN([MCA component configure script told me to abort])
-                        AC_MSG_ERROR([cannot continue])
-                    fi
-
-                    m4_foreach(flags, [LDFLAGS, LIBS],
-                        [[line="`$GREP WRAPPER_EXTRA_]flags[= $infile | cut -d= -f2-`"]
-                            eval "line=$line"
-                            if test -n "$line"; then
-                                $2[_]$3[_WRAPPER_EXTRA_]flags[="$line"]
-                            fi
-                        ])dnl
-                fi
-
-                MCA_PROCESS_COMPONENT($1, $2, $component, $3, $4, $5, $6, $compile_mode)
-            else
-                MCA_PROCESS_DEAD_COMPONENT($1, $2, $component)
-            fi
-        fi
-    done
+# MCA_COMPONENT_COMPILE_MODE(project_name (1), framework_name (2),
+#                            component_name (3), compile_mode_variable (4))
+# -------------------------------------------------------------------------
+# set compile_mode_variable to the compile mode for the given component.
+# this macro will only evaluate the build mode once per configure, returning
+# the cached value for subsequent tests.  The string is not stored in a cache
+# variable (ie .*_cv_.*) because cache variables would not be invalidated
+# based on changes to --enable-mca-dso or --enable-mca-static.
+AC_DEFUN([MCA_COMPONENT_COMPILE_MODE],[
+    OAC_ASSERT_LITERAL([$1], [1])dnl
+    OAC_ASSERT_LITERAL([$2], [2])dnl
+    OAC_ASSERT_LITERAL([$3], [3])dnl
+
+    AS_VAR_PUSHDEF([compile_mode_cv], [$1_$2_$3_compile_mode])dnl
+    AS_VAR_SET_IF([compile_mode_cv],
+        [],
+	[m4_ifdef([MCA_$1_$2_$3_COMPILE_MODE],
+                  [dnl We introduced caching of this check after setting the compile
+                   dnl mode by the substitute macro was common, and there was not a
+                   dnl polymorphic variable assumption in all those macros, so we use
+                   dnl a temporary with a fixed name.
+                   OPAL_VAR_SCOPE_PUSH([component_compile_mode_tmp])
+                   MCA_$1_$2_$3_COMPILE_MODE([$1], [$2], [$3], [component_compile_mode_tmp])
+                   AS_VAR_COPY([compile_mode_cv], [$component_compile_mode_tmp])
+                   OPAL_VAR_SCOPE_POP],
+                  [MCA_COMPONENT_COMPILE_MODE_INTERNAL([$1], [$2], [$3], [compile_mode_cv])])])
+    AS_VAR_COPY([$4], [compile_mode_cv])
+    AS_VAR_POPDEF([compile_mode_cv])dnl
 ])
 
-
-# MCA_COMPONENT_COMPILE_MODE(project_name (1), framework_name (2),
+# MCA_COMPONENT_COMPILE_MODE_INTERNAL(project_name (1), framework_name (2),
 #                            component_name (3), compile_mode_variable (4))
 # -------------------------------------------------------------------------
-# set compile_mode_variable to the compile mode for the given component
+# Determine compile mode of the given component.  Prefer a static
+# build by default.  Users can customize build mode by influencing the
+# component, framework, or all-up build flags.  This function starts
+# at the most specific (component) and works its way out, looking for
+# a set option.
+#
+# Components can avoid calling this function by defining a macro
+# MCA_<project>_<framework>_<component>_COMPILE_MODE.
 #
 #   NOTE: component_name may not be determined until runtime....
-AC_DEFUN([MCA_COMPONENT_COMPILE_MODE],[
+AC_DEFUN([MCA_COMPONENT_COMPILE_MODE_INTERNAL], [
+    OPAL_VAR_SCOPE_PUSH([compile_mode_internal_tmp SHARED_FRAMEWORK SHARED_COMPONENT STATIC_FRAMEWORK STATIC_COMPONENT])
+
     SHARED_FRAMEWORK="$DSO_$2"
-    AS_LITERAL_IF([$3],
-        [SHARED_COMPONENT="$DSO_$2_$3"],
-        [str="SHARED_COMPONENT=\$DSO_$2_$3"
-         eval $str])
+    SHARED_COMPONENT="$DSO_$2_$3"
 
     STATIC_FRAMEWORK="$STATIC_$2"
-    AS_LITERAL_IF([$3],
-        [STATIC_COMPONENT="$STATIC_$2_$3"],
-        [str="STATIC_COMPONENT=\$STATIC_$2_$3"
-         eval $str])
-
-    shared_mode_override=static
-
-    # Setup for either shared or static
-    if test "$STATIC_FRAMEWORK" = "1" || \
-       test "$STATIC_COMPONENT" = "1" || \
-       test "$STATIC_all" = "1" ; then
-        $4="static"
-    elif test "$shared_mode_override" = "dso" || \
-         test "$SHARED_FRAMEWORK" = "1" || \
-         test "$SHARED_COMPONENT" = "1" || \
-         test "$DSO_all" = "1"; then
-        $4="dso"
+    STATIC_COMPONENT="$STATIC_$2_$3"
+
+    # Look for the most specific specifier between static/dso.  If
+    # there is a tie (either neither or both specified), prefer
+    # static.
+    if test "$STATIC_COMPONENT" = "1"; then
+        compile_mode_internal_tmp=static
+    elif test "$SHARED_COMPONENT" = "1"; then
+        compile_mode_internal_tmp=dso
+    elif test "$STATIC_FRAMEWORK" = "1"; then
+        compile_mode_internal_tmp=static
+    elif test "$SHARED_FRAMEWORK" = "1"; then
+        compile_mode_internal_tmp=dso
+    elif test "$STATIC_all" = "1"; then
+        compile_mode_internal_tmp=static
+    elif test "$DSO_all" = "1"; then
+        compile_mode_internal_tmp=dso
     else
-        $4="static"
+        compile_mode_internal_tmp=static
     fi
 
+    AS_VAR_SET([$4], [$compile_mode_internal_tmp])
+
     AC_MSG_CHECKING([for MCA component $2:$3 compile mode])
     if test "$DIRECT_$2" = "$3" ; then
-        AC_MSG_RESULT([$$4 - direct])
+        AC_MSG_RESULT([$compile_mode_internal_tmp - direct])
     else
-        AC_MSG_RESULT([$$4])
+        AC_MSG_RESULT([$compile_mode_internal_tmp])
     fi
+
+    OPAL_VAR_SCOPE_POP
 ])
 
+# OPAL_MCA_STRIP_LAFILES(output_variable(1),
+#                        input_list(2)
+#--------------------------------------------
+# Helper function to MCA_PROCESS_COMPONENT which will strip
+# any .la file entries in the LIBS list.  Used for when copying
+# a component's LIBS into WRAPPER_LIBS.
+AC_DEFUN([OPAL_MCA_STRIP_LAFILES], [
+    OPAL_VAR_SCOPE_PUSH([opal_tmp])
+
+    for arg in $2; do
+        opal_tmp=`echo $arg | awk '{print substr([$][1], length([$][1])-2) }'`
+        AS_IF([test "$opal_tmp" != ".la"],
+              [AS_IF([test -z "$$1"], [$1=$arg], [$1="$$1 $arg"])])
+    done
+
+    OPAL_VAR_SCOPE_POP
+])
 
 # MCA_PROCESS_COMPONENT(project_name(1), framework_name (2), component_name (3),
 #                        all_components_variable (4), static_components_variable (5)
@@ -738,8 +740,6 @@ AC_DEFUN([MCA_COMPONENT_COMPILE_MODE],[
 #---------------------------------------------------------------------
 # Final setup work for a given component.  It should be known before
 # calling that this component can build properly (and exists)
-#
-#   NOTE: component_name may not be determined until runtime....
 AC_DEFUN([MCA_PROCESS_COMPONENT],[
     AC_REQUIRE([AC_PROG_GREP])
 
@@ -755,12 +755,12 @@ AC_DEFUN([MCA_PROCESS_COMPONENT],[
     else
         if test "$2" = "common"; then
             # Static libraries in "common" frameworks are installed, and
-            # therefore must obey the $FRAMEWORK_LIB_PREFIX that was
+            # therefore must obey the $FRAMEWORK_LIB_NAME that was
             # set.
-            $7="mca/$2/$3/lib${m4_translit([$1], [a-z], [A-Z])_LIB_PREFIX}mca_$2_$3.la $$7"
+            $7="mca/$2/$3/lib${m4_translit([$1], [a-z], [A-Z])_LIB_NAME}mca_$2_$3.la $$7"
         else
             # Other frameworks do not have to obey the
-            # $FRAMEWORK_LIB_PREFIX prefix.
+            # $FRAMEWORK_LIB_NAME prefix.
             $7="mca/$2/$3/libmca_$2_$3.la $$7"
         fi
         echo "extern const mca_base_component_t mca_$2_$3_component;" >> $outfile.extern
@@ -801,48 +801,49 @@ AC_MSG_ERROR([*** $2 component $3 was supposed to be direct-called, but
         fi
     fi
 
-    # if the component is building, add it's WRAPPER_EXTRA_LDFLAGS and
-    # WRAPPER_EXTRA_LIBS.  If the component doesn't specify it's
-    # WRAPPER_EXTRA_LIBS and WRAPPER_EXTRA_LDFLAGS, try using LDFLAGS and LIBS if
-    # component didn't have it's own configure script (in which case,
-    # we know it didn't set LDFLAGS and LIBS because it can't) Don't
-    # have to do this if the component is building dynamically,
-    # because it will link against these (without a dependency from
-    # libmpi.so to these flags)
-    if test "$8" = "static"; then
-        AS_LITERAL_IF([$3],
-            [m4_foreach(flags, [LDFLAGS, LIBS],
-                    [AS_IF([test "$$2_$3_WRAPPER_EXTRA_]flags[" = ""],
-                           [OPAL_FLAGS_APPEND_UNIQ([mca_wrapper_extra_]m4_tolower(flags), [$$2_$3_]flags)],
-                           [OPAL_FLAGS_APPEND_UNIQ([mca_wrapper_extra_]m4_tolower(flags), [$$2_$3_WRAPPER_EXTRA_]flags)])
-                        ])],
-            [m4_foreach(flags, [LDFLAGS, LIBS],
-                    [[str="line=\$$2_$3_WRAPPER_EXTRA_]flags["]
-                      eval "$str"
-                      OPAL_FLAGS_APPEND_UNIQ([mca_wrapper_extra_]m4_tolower(flags), [$line])])])
-    fi
-
-    # if needed, copy over WRAPPER_EXTRA_CPPFLAGS.  Since a configure script
-    # component can never be used in a STOP_AT_FIRST framework, we
-    # don't have to implement the else clause in the literal check...
-    AS_LITERAL_IF([$3],
-        [AS_IF([test "$$2_$3_WRAPPER_EXTRA_CPPFLAGS" != ""],
-           [m4_if(OPAL_EVAL_ARG([MCA_$1_$2_CONFIGURE_MODE]), [STOP_AT_FIRST], [stop_at_first=1], [stop_at_first=0])
-            AS_IF([test "$8" = "static" && test "$stop_at_first" = "1"],
-              [AS_IF([test "$with_devel_headers" = "yes"],
-                     [OPAL_FLAGS_APPEND_UNIQ([mca_wrapper_extra_cppflags], [$$2_$3_WRAPPER_EXTRA_CPPFLAGS])])],
-              [AC_MSG_WARN([ignoring $2_$3_WRAPPER_EXTRA_CPPFLAGS ($$2_$3_WRAPPER_EXTRA_CPPFLAGS): component conditions not met])])])])
+    # If a component is building static, we need to provide LDFLAGS
+    # and LIBS configuration to the wrapper compiler, so that it can
+    # provide them for the final link of the application.  Components
+    # can explicitly set <framework>_<component>_WRAPPER_EXTRA_<flag>
+    # for either LDFLAGS or LIBS, for cases where the component wants
+    # to explicitly manage which flags are passed to the wrapper
+    # compiler.  If the <framework>_<component>_WRAPPER_EXTRA_<flag>
+    # variable is not set, then it is assumed that the component
+    # wishes all LDFLAGS and LIBS to be provided as wrapper flags.
+    AS_IF([test "$8" = "static"],
+          [AS_VAR_SET_IF([$2_$3_WRAPPER_EXTRA_LDFLAGS],
+              [tmp_flags=${$2_$3_WRAPPER_EXTRA_LDFLAGS}],
+              [tmp_flags=${$2_$3_LDFLAGS}])
+           OPAL_FLAGS_APPEND_UNIQ([mca_wrapper_extra_ldflags], [$tmp_flags])
+
+           AS_VAR_SET_IF([$2_$3_WRAPPER_EXTRA_LIBS],
+              [tmp_flags=${$2_$3_WRAPPER_EXTRA_LIBS}],
+              [tmp_all_flags=${$2_$3_LIBS}
+               OPAL_MCA_STRIP_LAFILES([tmp_flags], [$tmp_all_flags])])
+           OPAL_FLAGS_APPEND_MOVE([mca_wrapper_extra_libs], [$tmp_flags])])
+
+    # WRAPPER_EXTRA_CPPFLAGS are only needed for STOP_AT_FIRST
+    # components, as all other components are not allowed to leak
+    # headers or compile-time flags into the top-level library or
+    # wrapper compilers.  If needed, copy over WRAPPER_EXTRA_CPPFLAGS.
+    # Since a configure script component can never be used in a
+    # STOP_AT_FIRST framework, we don't have to implement the else
+    # clause in the literal check.
+    AS_IF([test "$$2_$3_WRAPPER_EXTRA_CPPFLAGS" != ""],
+          [m4_if(OPAL_EVAL_ARG([MCA_$1_$2_CONFIGURE_MODE]), [STOP_AT_FIRST], [stop_at_first=1], [stop_at_first=0])
+           AS_IF([test "$8" = "static" && test "$stop_at_first" = "1"],
+                 [AS_IF([test "$with_devel_headers" = "yes"],
+                        [OPAL_FLAGS_APPEND_UNIQ([mca_wrapper_extra_cppflags], [$$2_$3_WRAPPER_EXTRA_CPPFLAGS])])],
+                 [AC_MSG_WARN([ignoring $2_$3_WRAPPER_EXTRA_CPPFLAGS ($$2_$3_WRAPPER_EXTRA_CPPFLAGS): component conditions not met])])])
 ])
 
 
 # MCA_PROCESS_DEAD_COMPONENT(project_name (1), framework_name (2),
 #                            component_name (3))
 # ----------------------------------------------------------------
-# Finall setup work for a component that can not be built.  Do the
+# Final setup work for a component that can not be built.  Do the
 # last minute checks to make sure the user isn't doing something
 # stupid.
-#
-#   NOTE: component_name may not be determined until runtime....
 AC_DEFUN([MCA_PROCESS_DEAD_COMPONENT],[
     AC_MSG_CHECKING([if MCA component $2:$3 can compile])
     AC_MSG_RESULT([no])
@@ -919,13 +920,7 @@ AC_DEFUN([MCA_COMPONENT_BUILD_CHECK],[
 
     # if we were explicitly disabled, don't build :)
     AS_IF([test "$DISABLE_$2" = "1"], [want_component=0])
-    AS_LITERAL_IF([$3],
-        [AS_IF([test "$DISABLE_$2_$3" = "1"], [want_component=0])],
-        [str="DISABLED_COMPONENT_CHECK=\$DISABLE_$2_$3"
-         eval $str
-         if test "$DISABLED_COMPONENT_CHECK" = "1" ; then
-             want_component=0
-         fi])
+    AS_IF([test "${DISABLE_$2_$3}" = "1"], [want_component=0])
 
     AS_IF([test "$want_component" = "1"], [$4], [$5])
 ])
@@ -962,3 +957,29 @@ AC_DEFUN([OPAL_MCA_MAKE_DIR_LIST],[
     done
     AC_SUBST($1)
 ])
+
+
+# OPAL_MCA_CHECK_DEPENDENCY(check project, check framework, check component,
+#                           dependent project, dependent framework, dependent component)
+# --------------------------------------------------------------------------
+# Enforce that the check component does not introduce a dependency from its
+# project library (ie libmpi.so or libopen-pal.so) to another component
+# (ie, a common_*.so).  This can happen if the check component is set to
+# build static and the common library is set to build dso.  If this situation
+# is detected, print an error and abort configure.
+AC_DEFUN([OPAL_MCA_CHECK_DEPENDENCY],
+[
+    OPAL_VAR_SCOPE_PUSH([component_compile_mode dependency_compile_mode])
+
+    MCA_COMPONENT_COMPILE_MODE([$1], [$2], [$3], [component_compile_mode])
+    MCA_COMPONENT_COMPILE_MODE([$4], [$5], [$6], [dependency_compile_mode])
+
+    AS_IF([test "${component_compile_mode}" = "static" -a "${dependency_compile_mode}" = "dso"],
+         [AC_MSG_ERROR([Component $2:$3 is set to build as a
+static component, but its dependency $5:$6 is set to build as
+a dynamic component.  This configuration is not supported.  Please
+either build $5:$6 as a static component or build $2:$3 as a dynamic
+component.])])
+
+    OPAL_VAR_SCOPE_POP
+])
diff --git a/config/opal_save_version.m4 b/config/opal_save_version.m4
index 961aba9f33e..11f346a5662 100644
--- a/config/opal_save_version.m4
+++ b/config/opal_save_version.m4
@@ -11,6 +11,7 @@ dnl                         University of Stuttgart.  All rights reserved.
 dnl Copyright (c) 2004-2005 The Regents of the University of California.
 dnl                         All rights reserved.
 dnl Copyright (c) 2008-2014 Cisco Systems, Inc.  All rights reserved.
+dnl Copyright (c) 2022-2023 IBM Corporation.  All rights reserved.
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
@@ -55,10 +56,27 @@ AC_DEFUN([OPAL_SAVE_VERSION], [
         [The repository version ]$2)
     AC_DEFINE_UNQUOTED($1[_TARBALL_VERSION], ["$]$1[_TARBALL_VERSION"],
         [Tarball filename version string of ]$2)
-    AC_DEFINE_UNQUOTED($1[_VERSION], ["$]$1[_RELEASE_VERSION"],
+    AC_DEFINE_UNQUOTED($1[_VERSION], ["$]$1[_VERSION"],
         [Complete release number of ]$2)
     AC_DEFINE_UNQUOTED($1[_RELEASE_DATE], ["$]$1[_RELEASE_DATE"],
         [Release date of ]$2)
 
     AC_CONFIG_FILES([$4])
 ])dnl
+
+# OPAL_SAVE_MPI_STANDARD_VERSION(version_file)
+# ----------------------------------------------
+# Export MPI versions.
+AC_DEFUN([OPAL_SAVE_MPI_STANDARD_VERSION], [
+
+    AC_DEFINE_UNQUOTED([MPI_VERSION], [MPI_VERSION_NUM],
+                       [MPI Standard Major version number])
+    AC_DEFINE_UNQUOTED([MPI_SUBVERSION], [MPI_SUBVERSION_NUM],
+                       [MPI Standard Minor version number])
+
+    AC_SUBST([MPI_VERSION], [MPI_VERSION_NUM])
+    AC_SUBST([MPI_SUBVERSION], [MPI_SUBVERSION_NUM])
+
+    AC_MSG_CHECKING([MPI Standard version])
+    AC_MSG_RESULT([MPI_VERSION_NUM.MPI_SUBVERSION_NUM])
+])dnl
diff --git a/config/opal_set_lib_name.m4 b/config/opal_set_lib_name.m4
new file mode 100644
index 00000000000..e0f29cd0372
--- /dev/null
+++ b/config/opal_set_lib_name.m4
@@ -0,0 +1,53 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2014      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2016      IBM Corporation.  All rights reserved.
+# Copyright (c) 2020      Intel, Inc.  All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+#                         All Rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# OPAL_SET_LIB_PREFIX([library_prefix]
+#
+# This macro sets a name for the libopen-pal library.  Specifically,
+# libopen-pal.la becomes libopen-pal.la by default.
+#
+# --------------------------------------------------------
+AC_DEFUN([OPAL_SET_LIB_NAME],[
+    AS_IF([test "$opal_lib_prefix_set" = "yes"],
+          [AC_MSG_WARN([OPAL lib name was already set!])
+           AC_MSG_WARN([This is a configury programming error])
+           AC_MSG_ERROR([Cannot continue])])
+
+    OPAL_LIB_NAME=$1
+    opal_lib_prefix_set=yes
+    AC_SUBST(OPAL_LIB_NAME)
+])dnl
+
+#
+# Rename 'libmpi' and 'libmpi_FOO' with a configure time option.
+#
+AC_DEFUN([OMPI_SET_LIB_NAME],[
+    AC_MSG_CHECKING([if want custom libmpi(_FOO) name])
+    AC_ARG_WITH([libmpi-name],
+        [AS_HELP_STRING([--with-libmpi-name=STRING],
+                ["Replace \"libmpi(_FOO)\" with \"libSTRING(_FOO)\" (default=mpi)"])])
+
+    AS_IF([test "$with_libmpi_name" = "no"],
+        [AC_MSG_RESULT([Error])
+         AC_MSG_WARN([Invalid to specify --without-libmpi-name])
+         AC_MSG_ERROR([Cannot continue])])
+
+    AS_IF([test "$with_libmpi_name" = "" || test "$with_libmpi_name" = "yes"],
+        [with_libmpi_name="mpi"])
+
+    OMPI_LIBMPI_NAME=${with_libmpi_name}
+
+    AC_MSG_RESULT([$OMPI_LIBMPI_NAME])
+    AC_SUBST(OMPI_LIBMPI_NAME)
+])dnl
diff --git a/config/opal_set_lib_prefix.m4 b/config/opal_set_lib_prefix.m4
deleted file mode 100644
index cea813a5151..00000000000
--- a/config/opal_set_lib_prefix.m4
+++ /dev/null
@@ -1,49 +0,0 @@
-# -*- shell-script -*-
-#
-# Copyright (c) 2014      Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2016      IBM Corporation.  All rights reserved.
-# Copyright (c) 2020      Intel, Inc.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-# OPAL_SET_LIB_PREFIX([library_prefix]
-#
-# This macro sets a prefix for the libopen-pal library.  Specifically,
-# libopen-pal.la becomes libPREFIXopen-pal.la.
-#
-# --------------------------------------------------------
-AC_DEFUN([OPAL_SET_LIB_PREFIX],[
-    AS_IF([test "$opal_lib_prefix_set" = "yes"],
-          [AC_MSG_WARN([OPAL lib prefix was already set!])
-           AC_MSG_WARN([This is a configury programming error])
-           AC_MSG_ERROR([Cannot continue])])
-
-    OPAL_LIB_PREFIX=$1
-    opal_lib_prefix_set=yes
-    AC_SUBST(OPAL_LIB_PREFIX)
-])dnl
-
-#
-# Rename 'libmpi' and 'libmpi_FOO' with a configure time option.
-#
-AC_DEFUN([OMPI_SET_LIB_NAME],[
-    AC_MSG_CHECKING([if want custom libmpi(_FOO) name])
-    AC_ARG_WITH([libmpi-name],
-        [AS_HELP_STRING([--with-libmpi-name=STRING],
-                ["Replace \"libmpi(_FOO)\" with \"libSTRING(_FOO)\" (default=mpi)"])])
-
-    AS_IF([test "$with_libmpi_name" = "no"],
-        [AC_MSG_RESULT([Error])
-         AC_MSG_WARN([Invalid to specify --without-libmpi-name])
-         AC_MSG_ERROR([Cannot continue])])
-
-    AS_IF([test "$with_libmpi_name" = "" || test "$with_libmpi_name" = "yes"],
-        [with_libmpi_name="mpi"])
-
-    AC_MSG_RESULT([$with_libmpi_name])
-    AC_SUBST(OMPI_LIBMPI_NAME, $with_libmpi_name)
-])dnl
diff --git a/config/opal_setup_cc.m4 b/config/opal_setup_cc.m4
index a4eb3582734..394826223fb 100644
--- a/config/opal_setup_cc.m4
+++ b/config/opal_setup_cc.m4
@@ -11,7 +11,7 @@ dnl                         University of Stuttgart.  All rights reserved.
 dnl Copyright (c) 2004-2006 The Regents of the University of California.
 dnl                         All rights reserved.
 dnl Copyright (c) 2007-2009 Sun Microsystems, Inc.  All rights reserved.
-dnl Copyright (c) 2008-2015 Cisco Systems, Inc.  All rights reserved.
+dnl Copyright (c) 2008-2021 Cisco Systems, Inc.  All rights reserved.
 dnl Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights
 dnl                         reserved.
 dnl Copyright (c) 2015-2019 Research Organization for Information Science
@@ -106,7 +106,7 @@ AC_DEFUN([OPAL_PROG_CC_C11],[
             for flag in $(echo $opal_prog_cc_c11_flags | tr ' ' '\n') ; do
                 OPAL_PROG_CC_C11_HELPER([$flag],[opal_cv_c11_flag=$flag],[])
                 if test "x$opal_cv_c11_flag" != "x" ; then
-                    CFLAGS="$CFLAGS $opal_cv_c11_flag"
+                    OPAL_FLAGS_APPEND_UNIQ([CFLAGS], ["$opal_cv_c11_flag"])
                     AC_MSG_NOTICE([using $flag to enable C11 support])
                     opal_cv_c11_supported=yes
                     break
@@ -135,13 +135,28 @@ AC_DEFUN([OPAL_CHECK_CC_IQUOTE],[
     CFLAGS="${CFLAGS} -iquote ."
     AC_MSG_CHECKING([for $CC option to add a directory only to the search path for the quote form of include])
     AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]],[])],
-		      [opal_cc_iquote="-iquote"],
-		      [opal_cc_iquote="-I"])
+                      [opal_cc_iquote="-iquote"],
+                      [opal_cc_iquote="-I"])
     CFLAGS=${opal_check_cc_iquote_CFLAGS_save}
     OPAL_VAR_SCOPE_POP
     AC_MSG_RESULT([$opal_cc_iquote])
 ])
 
+
+# OPAL_SETUP_CC_WERROR_HANDLER()
+# ------------------------------
+# Runs right before config.status, to possibly
+# add -Werror to CFLAGS.  Can't do this earlier
+# in configure, as many built-in tests generate
+# harmless warnings which would cause the wrong
+# answer.
+AC_DEFUN([OPAL_SETUP_CC_WERROR_HANDLER], [
+    AS_IF([test "$enable_werror" = "yes"],
+          [AC_MSG_NOTICE([Adding -Werror to CFLAGS])
+           CFLAGS="$CFLAGS -Werror"])
+])
+
+
 # OPAL_SETUP_CC()
 # ---------------
 # Do everything required to setup the C compiler.  Safe to AC_REQUIRE
@@ -155,7 +170,7 @@ AC_DEFUN([OPAL_SETUP_CC],[
     AC_REQUIRE([_OPAL_PROG_CC])
     AC_REQUIRE([AM_PROG_CC_C_O])
 
-    OPAL_VAR_SCOPE_PUSH([opal_prog_cc_c11_helper__Thread_local_available opal_prog_cc_c11_helper_atomic_var_available opal_prog_cc_c11_helper__Atomic_available opal_prog_cc_c11_helper__static_assert_available opal_prog_cc_c11_helper__Generic_available opal_prog_cc__thread_available opal_prog_cc_c11_helper_atomic_fetch_xor_explicit_available opal_prog_cc_c11_helper_proper__Atomic_support_in_atomics])
+    OPAL_VAR_SCOPE_PUSH([opal_prog_cc_c11_helper__Thread_local_available opal_prog_cc_c11_helper_atomic_var_available opal_prog_cc_c11_helper__Atomic_available opal_prog_cc_c11_helper__static_assert_available opal_prog_cc_c11_helper__Generic_available opal_prog_cc_c11_helper_atomic_fetch_xor_explicit_available opal_prog_cc_c11_helper_proper__Atomic_support_in_atomics])
 
     OPAL_PROG_CC_C11
 
@@ -164,33 +179,11 @@ AC_DEFUN([OPAL_SETUP_CC],[
     OPAL_C_COMPILER_VENDOR([opal_c_vendor])
 
     if test $opal_cv_c11_supported = no ; then
-        # It is not currently an error if C11 support is not available. Uncomment the
-        # following lines and update the warning when we require a C11 compiler.
-        # AC_MSG_WARNING([Open MPI requires a C11 (or newer) compiler])
-        # AC_MSG_ERROR([Aborting.])
-        # From Open MPI 1.7 on we require a C99 compiant compiler
-        dnl with autoconf 2.70 AC_PROG_CC makes AC_PROG_CC_C99 obsolete
-        m4_version_prereq([2.70],
-            [],
-            [AC_PROG_CC_C99])
-        # The result of AC_PROG_CC_C99 is stored in ac_cv_prog_cc_c99
-        if test "x$ac_cv_prog_cc_c99" = xno ; then
-            AC_MSG_WARN([Open MPI requires a C99 (or newer) compiler. C11 is recommended.])
-            AC_MSG_ERROR([Aborting.])
-        fi
-
-        # Get the correct result for C11 support flags now that the compiler flags have
-        # changed
-        OPAL_PROG_CC_C11_HELPER([], [], [])
+        # C11 is required
+        AC_MSG_WARN([Open MPI requires a C11 (or newer) compiler. C11 is required.])
+        AC_MSG_ERROR([Aborting.])
     fi
 
-    # Check if compiler support __thread
-    OPAL_CC_HELPER([if $CC $1 supports __thread], [opal_prog_cc__thread_available],
-               [],[[static __thread int  foo = 1;++foo;]])
-
-    OPAL_CC_HELPER([if $CC $1 supports C11 _Thread_local], [opal_prog_cc_c11_helper__Thread_local_available],
-                   [],[[static _Thread_local int  foo = 1;++foo;]])
-
     dnl At this time Open MPI only needs thread local and the atomic convenience types for C11 support. These
     dnl will likely be required in the future.
     AC_DEFINE_UNQUOTED([OPAL_C_HAVE__THREAD_LOCAL], [$opal_prog_cc_c11_helper__Thread_local_available],
@@ -208,13 +201,11 @@ AC_DEFUN([OPAL_SETUP_CC],[
     AC_DEFINE_UNQUOTED([OPAL_C_HAVE__STATIC_ASSERT], [$opal_prog_cc_c11_helper__static_assert_available],
                        [Whether C compiler support _Static_assert keyword])
 
-    AC_DEFINE_UNQUOTED([OPAL_C_HAVE___THREAD], [$opal_prog_cc__thread_available],
-                       [Whether C compiler supports __thread])
-
-
     # Check for standard headers, needed here because needed before
-    # the types checks.
-    AC_HEADER_STDC
+    # the types checks.  This is only necessary for Autoconf < v2.70.
+    m4_version_prereq([2.70],
+                      [],
+                      [AC_HEADER_STDC])
 
     # GNU C and autotools are inconsistent about whether this is
     # defined so let's make it true everywhere for now...  However, IBM
@@ -252,7 +243,7 @@ AC_DEFUN([OPAL_SETUP_CC],[
         # compiling and linking to circumvent trouble with
         # libgcov.
         LDFLAGS_orig="$LDFLAGS"
-        LDFLAGS="$LDFLAGS_orig --coverage"
+        OPAL_FLAGS_APPEND_UNIQ([LDFLAGS], ["--coverage"])
         OPAL_COVERAGE_FLAGS=
 
         _OPAL_CHECK_SPECIFIC_CFLAGS(--coverage, coverage)
@@ -271,19 +262,8 @@ AC_DEFUN([OPAL_SETUP_CC],[
             CLEANFILES="*.bb *.bbg ${CLEANFILES}"
             OPAL_COVERAGE_FLAGS="-ftest-coverage -fprofile-arcs"
         fi
-        OPAL_FLAGS_UNIQ(CFLAGS)
-        OPAL_FLAGS_UNIQ(LDFLAGS)
         WANT_DEBUG=1
    fi
-    
-
-    # Do we want debugging?
-    if test "$WANT_DEBUG" = "1" && test "$enable_debug_symbols" != "no" ; then
-        CFLAGS="$CFLAGS -g"
-
-        OPAL_FLAGS_UNIQ(CFLAGS)
-        AC_MSG_WARN([-g has been added to CFLAGS (--enable-debug)])
-    fi
 
     # These flags are generally gcc-specific; even the
     # gcc-impersonating compilers won't accept them.
@@ -296,11 +276,17 @@ AC_DEFUN([OPAL_SETUP_CC],[
         _OPAL_CHECK_SPECIFIC_CFLAGS(-Wmissing-prototypes, Wmissing_prototypes)
         _OPAL_CHECK_SPECIFIC_CFLAGS(-Wstrict-prototypes, Wstrict_prototypes)
         _OPAL_CHECK_SPECIFIC_CFLAGS(-Wcomment, Wcomment)
+        _OPAL_CHECK_SPECIFIC_CFLAGS(-Wshadow, Wshadow)
         _OPAL_CHECK_SPECIFIC_CFLAGS(-Werror-implicit-function-declaration, Werror_implicit_function_declaration)
         _OPAL_CHECK_SPECIFIC_CFLAGS(-Wno-long-double, Wno_long_double, int main() { long double x; })
         _OPAL_CHECK_SPECIFIC_CFLAGS(-fno-strict-aliasing, fno_strict_aliasing, int main() { long double x; })
         _OPAL_CHECK_SPECIFIC_CFLAGS(-pedantic, pedantic)
         _OPAL_CHECK_SPECIFIC_CFLAGS(-Wall, Wall)
+
+        # There are some warnings that we specifically do not care
+        # about / do not agree that gcc emits warnings about them.  So
+        # we turn them off.
+        _OPAL_CHECK_SPECIFIC_CFLAGS(-Wformat-truncation=0, format_truncation)
     fi
 
     # Note: Some versions of clang (at least >= 3.5 -- perhaps
@@ -314,6 +300,11 @@ AC_DEFUN([OPAL_SETUP_CC],[
     # warning: "-qinline" is not compatible with "-g". "-qnoinline" is being set.
     _OPAL_CHECK_SPECIFIC_CFLAGS(-finline-functions, finline_functions)
 
+    AC_ARG_ENABLE([werror],
+         [AS_HELP_STRING([--enable-werror],
+                         [Enable -Werror to convert warnings to errors.  Note that 3rd-party packages will not be built with -Werror (unless they also support the --enable-werror configure option)])])
+    AC_CONFIG_COMMANDS_PRE([OPAL_SETUP_CC_WERROR_HANDLER])
+
     # Try to enable restrict keyword
     RESTRICT_CFLAGS=
     case "$opal_c_vendor" in
@@ -331,9 +322,9 @@ AC_DEFUN([OPAL_SETUP_CC],[
     # see if the C compiler supports __builtin_expect
     AC_CACHE_CHECK([if $CC supports __builtin_expect],
         [opal_cv_cc_supports___builtin_expect],
-        [AC_TRY_LINK([],
+        [AC_LINK_IFELSE([AC_LANG_PROGRAM([],
           [void *ptr = (void*) 0;
-           if (__builtin_expect (ptr != (void*) 0, 1)) return 0;],
+           if (__builtin_expect (ptr != (void*) 0, 1)) return 0;])],
           [opal_cv_cc_supports___builtin_expect="yes"],
           [opal_cv_cc_supports___builtin_expect="no"])])
     if test "$opal_cv_cc_supports___builtin_expect" = "yes" ; then
@@ -347,9 +338,9 @@ AC_DEFUN([OPAL_SETUP_CC],[
     # see if the C compiler supports __builtin_prefetch
     AC_CACHE_CHECK([if $CC supports __builtin_prefetch],
         [opal_cv_cc_supports___builtin_prefetch],
-        [AC_TRY_LINK([],
+        [AC_LINK_IFELSE([AC_LANG_PROGRAM([],
           [int ptr;
-           __builtin_prefetch(&ptr,0,0);],
+           __builtin_prefetch(&ptr,0,0);])],
           [opal_cv_cc_supports___builtin_prefetch="yes"],
           [opal_cv_cc_supports___builtin_prefetch="no"])])
     if test "$opal_cv_cc_supports___builtin_prefetch" = "yes" ; then
@@ -363,9 +354,9 @@ AC_DEFUN([OPAL_SETUP_CC],[
     # see if the C compiler supports __builtin_clz
     AC_CACHE_CHECK([if $CC supports __builtin_clz],
         [opal_cv_cc_supports___builtin_clz],
-        [AC_TRY_LINK([],
+        [AC_LINK_IFELSE([AC_LANG_PROGRAM([],
             [int value = 0xffff; /* we know we have 16 bits set */
-             if ((8*sizeof(int)-16) != __builtin_clz(value)) return 0;],
+             if ((8*sizeof(int)-16) != __builtin_clz(value)) return 0;])],
             [opal_cv_cc_supports___builtin_clz="yes"],
             [opal_cv_cc_supports___builtin_clz="no"])])
     if test "$opal_cv_cc_supports___builtin_clz" = "yes" ; then
@@ -401,7 +392,6 @@ AC_DEFUN([OPAL_SETUP_CC],[
     OPAL_ENSURE_CONTAINS_OPTFLAGS(["$CFLAGS"])
     AC_MSG_RESULT([$co_result])
     CFLAGS="$co_result"
-    OPAL_FLAGS_UNIQ([CFLAGS])
     AC_MSG_RESULT(CFLAGS result: $CFLAGS)
     OPAL_VAR_SCOPE_POP
 ])
@@ -409,26 +399,30 @@ AC_DEFUN([OPAL_SETUP_CC],[
 
 AC_DEFUN([_OPAL_START_SETUP_CC],[
     opal_show_subtitle "C compiler and preprocessor"
-
-	# $%@#!@#% AIX!!  This has to be called before anything invokes the C
-    # compiler.
-    dnl AC_AIX
 ])
 
 
 AC_DEFUN([_OPAL_PROG_CC],[
+    dnl It is really easy to accidentally call AC_PROG_CC implicitly through
+    dnl some other test run before OPAL_SETUP_CC.  Try to make that harder.
+    m4_provide_if([AC_PROG_CC],
+                  [m4_fatal([AC_PROG_CC called before OPAL_SETUP_CC])])
+
     #
     # Check for the compiler
     #
-    OPAL_VAR_SCOPE_PUSH([opal_cflags_save dummy opal_cc_arvgv0])
-    opal_cflags_save="$CFLAGS"
+    OPAL_VAR_SCOPE_PUSH([dummy opal_cc_arvgv0])
+
+    AC_USE_SYSTEM_EXTENSIONS
+
     AC_PROG_CC
     BASECC="`basename $CC`"
-    CFLAGS="$opal_cflags_save"
-    AC_DEFINE_UNQUOTED(OPAL_CC, "$CC", [OMPI underlying C compiler])
+    OPAL_CC="$CC"
+    AC_DEFINE_UNQUOTED(OPAL_CC, "$OPAL_CC", [OMPI underlying C compiler])
     set dummy $CC
     opal_cc_argv0=[$]2
     OPAL_WHICH([$opal_cc_argv0], [OPAL_CC_ABSOLUTE])
     AC_SUBST(OPAL_CC_ABSOLUTE)
+
     OPAL_VAR_SCOPE_POP
 ])
diff --git a/config/opal_setup_cli.m4 b/config/opal_setup_cli.m4
index 401da9bfaed..10df8237e1d 100644
--- a/config/opal_setup_cli.m4
+++ b/config/opal_setup_cli.m4
@@ -26,7 +26,7 @@ AC_DEFUN([OPAL_CAPTURE_CONFIGURE_CLI],[
         sed_quote_subst='s/\(@<:@`"\\@:>@\)/\\\1/g'
         case "$arg" in
           *@<:@\\\`\"\$[]@:>@*)
-	    quoted_arg=\'`echo "$arg" | sed $sed_quote_subst`\' ;;
+            quoted_arg=\'`echo "$arg" | sed $sed_quote_subst`\' ;;
           *)
             quoted_arg="\'$arg\'" ;;
         esac
diff --git a/config/opal_setup_component_package.m4 b/config/opal_setup_component_package.m4
deleted file mode 100644
index a4206c09d0a..00000000000
--- a/config/opal_setup_component_package.m4
+++ /dev/null
@@ -1,105 +0,0 @@
-dnl -*- shell-script -*-
-dnl
-dnl Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-dnl                         University Research and Technology
-dnl                         Corporation.  All rights reserved.
-dnl Copyright (c) 2004-2005 The University of Tennessee and The University
-dnl                         of Tennessee Research Foundation.  All rights
-dnl                         reserved.
-dnl Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-dnl                         University of Stuttgart.  All rights reserved.
-dnl Copyright (c) 2004-2005 The Regents of the University of California.
-dnl                         All rights reserved.
-dnl Copyright (c) 2008-2009 Cisco Systems, Inc.  All rights reserved.
-dnl Copyright (c) 2014      Intel, Inc. All rights reserved.
-dnl Copyright (c) 2015      Research Organization for Information Science
-dnl                         and Technology (RIST). All rights reserved.
-dnl $COPYRIGHT$
-dnl
-dnl Additional copyrights may follow
-dnl
-dnl $HEADER$
-dnl
-
-# OPAL_SETUP_COMPONENT_PACKAGE(1: framework_name,
-#                              2: component_name,
-#                              3: option_name,
-#                              4: withdir_dir_check_file,
-#                              5: withdir_libdir_check_file,
-#                              6: header,
-#                              7: library,
-#                              8: function,
-#                              9: extra-libraries,
-#                              10: [action-if-found],
-#                              11: [action-if-not-found])
-# ------------------------------------------------
-# Many components need to just check for one package, and if it's all
-# good, set themselves up with appropriate CPPFLAGS, LDFLAGS, and
-# LIBS.  This macro templates all of that for the common case.
-#
-# This macro does the following:
-#
-# - Assumes that this component should be built by default if all
-#   headers and libraries can be found
-# - Adds --with-<name> and --with-<name>-libdir options to configure
-# - Sanity checks directory names given to the above options (i.e.,
-#   look for a token file in each, but only if the directory argument
-#   is given)
-# - Assumes that if --with-<name> is supplied and we can't build the
-#   component, it's a fatal error.
-# - Assumes that if --with-<name> is NOT supplied and we can't build
-#   the component, it's NOT a fatal error.
-# - Run OPAL_CHECK_PACKAGE (check for the specific presence of header
-#   files and/or libraries) to determine if the package is available
-# - Set and AC_SUBST <framework>_<component>_CPPFLAGS
-# - Set and AC_SUBST <framework>_<component>_CFLAGS
-# - Set and AC_SUBST <framework>_<component>_LDFLAGS
-# - Set and AC_SUBST <framework>_<component>_LIBS
-#
-# Enjoy.
-#
-AC_DEFUN([OPAL_SETUP_COMPONENT_PACKAGE],[
-    AC_ARG_WITH([$3],
-        [AS_HELP_STRING([--with-$3(=DIR)],
-                        [Build $3 support, optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries])])
-    OPAL_CHECK_WITHDIR([$3], [$with_$3], [$4])
-    AC_ARG_WITH([$3-libdir],
-        [AS_HELP_STRING([--with-$3-libdir=DIR],
-                        [Search for the $3 libraries in DIR])])
-    OPAL_CHECK_WITHDIR([$3-libdir], [$with_$3_libdir], [$5])
-
-    AS_IF([test ! -z "$with_$3" && test "$with_$3" != "yes"],
-          [$1_$2_dir="$with_$3"])
-    AS_IF([test ! -z "$with_$3_libdir" && test "$with_$3_libdir" != "yes"],
-          [$1_$2_libdir="$with_$3_libdir"])
-
-    AS_IF([test "$with_$3" = "no"],
-          [$1_$2_happy="no"],
-          [$1_$2_happy="yes"])
-
-    AS_IF([test "$$1_$2_happy" = "yes"],
-          [OPAL_CHECK_PACKAGE([$1_$2],
-                              [$6],
-                              [$7],
-                              [$8],
-                              [$9],
-                              [$$1_$2_dir],
-                              [$$1_$2_libdir],
-                              [$1_$2_happy="yes"],
-                              [$1_$2_happy="no"])])
-
-    AS_IF([test "$$1_$2_happy" = "yes"],
-          [$10],
-          [$11])
-
-    # sanity check
-    AS_IF([test "$$1_$2_happy" = "no"],
-          [AS_IF([test "$with_$3" != "no" && test ! -z "$with_$3"],
-                 [AC_MSG_WARN([$1:$2 requested but not found])
-                  AC_MSG_ERROR([Cannot continue])])])
-
-    AC_SUBST([$1_$2_CFLAGS])
-    AC_SUBST([$1_$2_CPPFLAGS])
-    AC_SUBST([$1_$2_LDFLAGS])
-    AC_SUBST([$1_$2_LIBS])
-])
diff --git a/config/opal_setup_cxx.m4 b/config/opal_setup_cxx.m4
deleted file mode 100644
index ceb9d6264a9..00000000000
--- a/config/opal_setup_cxx.m4
+++ /dev/null
@@ -1,218 +0,0 @@
-dnl -*- shell-script -*-
-dnl
-dnl Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-dnl                         University Research and Technology
-dnl                         Corporation.  All rights reserved.
-dnl Copyright (c) 2004-2006 The University of Tennessee and The University
-dnl                         of Tennessee Research Foundation.  All rights
-dnl                         reserved.
-dnl Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
-dnl                         University of Stuttgart.  All rights reserved.
-dnl Copyright (c) 2004-2006 The Regents of the University of California.
-dnl                         All rights reserved.
-dnl Copyright (c) 2006      Los Alamos National Security, LLC.  All rights
-dnl                         reserved.
-dnl Copyright (c) 2007-2009 Sun Microsystems, Inc.  All rights reserved.
-dnl Copyright (c) 2008-2013 Cisco Systems, Inc.  All rights reserved.
-dnl Copyright (c) 2015-2016 Research Organization for Information Science
-dnl                         and Technology (RIST). All rights reserved.
-dnl Copyright (c) 2020      Triad National Security, LLC. All rights
-dnl                         reserved.
-dnl Copyright (c) 2021      IBM Corporation.  All rights reserved.
-dnl
-dnl $COPYRIGHT$
-dnl
-dnl Additional copyrights may follow
-dnl
-dnl $HEADER$
-dnl
-
-# This macro is necessary to get the title to be displayed first.  :-)
-AC_DEFUN([OPAL_SETUP_CXX_BANNER],[
-    opal_show_subtitle "C++ compiler and preprocessor"
-])
-
-# This macro is necessary because PROG_CXX* is REQUIREd by multiple
-# places in SETUP_CXX.
-AC_DEFUN([OPAL_PROG_CXX],[
-    OPAL_VAR_SCOPE_PUSH([opal_cxxflags_save])
-    opal_cxxflags_save="$CXXFLAGS"
-    AC_PROG_CXX
-    AC_PROG_CXXCPP
-    CXXFLAGS="$opal_cxxflags_save"
-    OPAL_VAR_SCOPE_POP
-])
-
-# OPAL_SETUP_CXX()
-# ----------------
-# Do everything required to setup the C++ compiler.  Safe to AC_REQUIRE
-# this macro.
-AC_DEFUN([OPAL_SETUP_CXX],[
-    AC_REQUIRE([OPAL_SETUP_CXX_BANNER])
-
-    _OPAL_SETUP_CXX_COMPILER
-
-    OPAL_CXX_COMPILER_VENDOR([opal_cxx_vendor])
-
-    _OPAL_SETUP_CXX_COMPILER_BACKEND
-])
-
-# _OPAL_SETUP_CXX_COMPILER()
-# --------------------------
-# Setup the CXX compiler
-AC_DEFUN([_OPAL_SETUP_CXX_COMPILER],[
-    OPAL_VAR_SCOPE_PUSH(opal_cxx_compiler_works)
-
-    # Must REQUIRE the PROG_CXX macro and not call it directly here for
-    # reasons well-described in the AC2.64 (and beyond) docs.
-    AC_REQUIRE([OPAL_PROG_CXX])
-    BASECXX="`basename $CXX`"
-
-    AS_IF([test "x$CXX" = "x"], [CXX=none])
-    set dummy $CXX
-    opal_cxx_argv0=[$]2
-    OPAL_WHICH([$opal_cxx_argv0], [OPAL_CXX_ABSOLUTE])
-    AS_IF([test "x$OPAL_CXX_ABSOLUTE" = "x"], [OPAL_CXX_ABSOLUTE=none])
-
-    AC_DEFINE_UNQUOTED(OPAL_CXX, "$CXX", [OPAL underlying C++ compiler])
-    AC_SUBST(OPAL_CXX_ABSOLUTE)
-
-    OPAL_VAR_SCOPE_POP
-])
-
-# OPAL_CHECK_CXX_IQUOTE()
-# ----------------------
-# Check if the compiler supports the -iquote option. This options
-# removes the specified directory from the search path when using
-# #include <>. This check works around an issue caused by C++20
-# which added a <version> header. This conflicts with the
-# VERSION file at the base of our source directory on case-
-# insensitive filesystems.
-AC_DEFUN([OPAL_CHECK_CXX_IQUOTE],[
-    OPAL_VAR_SCOPE_PUSH([opal_check_cxx_iquote_CXXFLAGS_save])
-    opal_check_cxx_iquote_CXXFLAGS_save=${CXXFLAGS}
-    CXXFLAGS="${CXXFLAGS} -iquote ."
-    AC_MSG_CHECKING([for $CXX option to add a directory only to the search path for the quote form of include])
-    AC_LANG_PUSH(C++)
-    AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]],[])],
-		      [opal_cxx_iquote="-iquote"],
-		      [opal_cxx_iquote="-I"])
-    CXXFLAGS=${opal_check_cxx_iquote_CXXFLAGS_save}
-    AC_LANG_POP(C++)
-    OPAL_VAR_SCOPE_POP
-    AC_MSG_RESULT([$opal_cxx_iquote])
-])
-
-# _OPAL_SETUP_CXX_COMPILER_BACKEND()
-# ----------------------------------
-# Back end of _OPAL_SETUP_CXX_COMPILER_BACKEND()
-AC_DEFUN([_OPAL_SETUP_CXX_COMPILER_BACKEND],[
-    AC_LANG_PUSH(C++)
-    OPAL_CHECK_CXX_IQUOTE
-
-    # Do we want code coverage
-    if test "$WANT_COVERAGE" = "1"; then
-        # For compilers > gcc-4.x, use --coverage for
-        # compiling and linking to circumvent trouble with
-        # libgcov.
-        OPAL_COVERAGE_FLAGS=
-
-        _OPAL_CHECK_SPECIFIC_CXXFLAGS(--coverage, coverage)
-        if test "$opal_cv_cxx_coverage" = "1" ; then
-            OPAL_COVERAGE_FLAGS="--coverage"
-            CLEANFILES="*.gcno ${CLEANFILES}"
-            CONFIG_CLEAN_FILES="*.gcda *.gcov ${CONFIG_CLEAN_FILES}"
-            AC_MSG_WARN([$OPAL_COVERAGE_FLAGS has been added to CXXFLAGS (--enable-coverage)])
-        else
-            _OPAL_CHECK_SPECIFIC_CXXFLAGS(-ftest-coverage, ftest_coverage)
-            _OPAL_CHECK_SPECIFIC_CXXFLAGS(-fprofile-arcs, fprofile_arcs)
-            if test "$opal_cv_cxx_ftest_coverage" = "0" || test "opal_cv_cxx_fprofile_arcs" = "0" ; then
-                AC_MSG_WARN([Code coverage functionality is not currently available with $CXX])
-                AC_MSG_ERROR([Configure: Cannot continue])
-            fi
-            CLEANFILES="*.bb *.bbg ${CLEANFILES}"
-            OPAL_COVERAGE_FLAGS="-ftest-coverage -fprofile-arcs"
-        fi
-        OPAL_FLAGS_UNIQ(CXXFLAGS)
-        WANT_DEBUG=1
-   fi
-
-    # Do we want debugging?
-    if test "$WANT_DEBUG" = "1" && test "$enable_debug_symbols" != "no" ; then
-        CXXFLAGS="$CXXFLAGS -g"
-        AC_MSG_WARN([-g has been added to CXXFLAGS (--enable-debug)])
-    fi
-
-    if test "$WANT_DEBUG"  = "0" ; then
-        OPAL_ENSURE_CONTAINS_OPTFLAGS(["$CXXFLAGS"])
-    fi
-
-    # These flags are generally g++-specific; even the g++-impersonating
-    # compilers won't accept them.
-    OPAL_CXXFLAGS_BEFORE_PICKY="$CXXFLAGS"
-    if test "$WANT_PICKY_COMPILER" = 1; then
-        _OPAL_CHECK_SPECIFIC_CXXFLAGS(-Wundef, Wundef)
-        _OPAL_CHECK_SPECIFIC_CXXFLAGS(-Wno-long-long, Wno_long_long, int main() { long long x; } )
-        _OPAL_CHECK_SPECIFIC_CXXFLAGS(-Wno-long-double, Wno_long_double, int main () { long double x; })
-        _OPAL_CHECK_SPECIFIC_CXXFLAGS(-fstrict-prototype, fstrict_prototype)
-        _OPAL_CHECK_SPECIFIC_CXXFLAGS(-Wall, Wall)
-    fi
-
-    _OPAL_CHECK_SPECIFIC_CXXFLAGS(-finline-functions, finline_functions)
-
-    # Make sure we can link with the C compiler
-    if test "$opal_cv_cxx_compiler_vendor" != "microsoft"; then
-      OPAL_LANG_LINK_WITH_C([C++], [],
-        [cat <<EOF >&2
-**********************************************************************
-* It appears that your C++ compiler is unable to link against object
-* files created by your C compiler.  This generally indicates either
-* a conflict between the options specified in CFLAGS and CXXFLAGS
-* or a problem with the local compiler installation.  More
-* information (including exactly what command was given to the
-* compilers and what error resulted when the commands were executed) is
-* available in the config.log file in this directory.
-**********************************************************************
-EOF
-         AC_MSG_ERROR([C and C++ compilers are not link compatible.  Can not continue.])])
-    fi
-
-    # If we are on HP-UX, ensure that we're using aCC
-    case "$host" in
-    *hpux*)
-        if test "$BASECXX" = "CC"; then
-            AC_MSG_WARN([*** You will probably have problems compiling the MPI 2])
-            AC_MSG_WARN([*** C++ bindings with the HP-UX CC compiler.  You should])
-            AC_MSG_WARN([*** probably be using the aCC compiler.  Re-run configure])
-            AC_MSG_WARN([*** with the environment variable "CXX=aCC".])
-        fi
-        ;;
-    esac
-
-    # Note: gcc-imperonating compilers accept -O3
-    if test "$WANT_DEBUG" = "1"; then
-        OPTFLAGS=
-    else
-        if test "$GXX" = yes; then
-            OPTFLAGS="-O3"
-        else
-            OPTFLAGS="-O"
-        fi
-    fi
-
-
-    # bool type size and alignment
-    AC_CHECK_SIZEOF(bool)
-    OPAL_C_GET_ALIGNMENT(bool, OPAL_ALIGNMENT_CXX_BOOL)
-
-    OPAL_ENSURE_CONTAINS_OPTFLAGS("$OPAL_CXXFLAGS_BEFORE_PICKY")
-    OPAL_CXXFLAGS_BEFORE_PICKY="$co_result"
-
-    AC_MSG_CHECKING([for CXX optimization flags])
-    OPAL_ENSURE_CONTAINS_OPTFLAGS(["$CXXFLAGS"])
-    AC_MSG_RESULT([$co_result])
-    CXXFLAGS="$co_result"
-    OPAL_FLAGS_UNIQ([CXXFLAGS])
-    AC_MSG_RESULT(CXXFLAGS result: $CXXFLAGS)
-    AC_LANG_POP(C++)
-])
diff --git a/config/opal_setup_ft.m4 b/config/opal_setup_ft.m4
index 7d3666a3d19..c80c9858fe2 100644
--- a/config/opal_setup_ft.m4
+++ b/config/opal_setup_ft.m4
@@ -6,6 +6,7 @@ dnl Copyright (c) 2009-2012 Oak Ridge National Labs.  All rights reserved.
 dnl Copyright (c) 2013-2014 Intel, Inc.  All rights reserved.
 dnl Copyright (c) 2015      Research Organization for Information Science
 dnl                         and Technology (RIST). All rights reserved.
+dnl Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
@@ -20,8 +21,8 @@ dnl
 #    - LAM (synonym for 'cr' currently)
 #    - cr
 #  /* General FT sections */
-#  #if OPAL_ENABLE_FT == 0 /* FT Disabled globaly */
-#  #if OPAL_ENABLE_FT == 1 /* FT Enabled globaly */
+#  #if OPAL_ENABLE_FT == 0 /* FT Disabled globally */
+#  #if OPAL_ENABLE_FT == 1 /* FT Enabled globally */
 #  /* ULFM Specific sections */
 #  #if OPAL_ENABLE_FT_MPI == 0 /* FT ULFM Disabled */
 #  #if OPAL_ENABLE_FT_MPI == 1 /* FT ULFM Enabled */
@@ -138,5 +139,5 @@ AC_DEFUN([OPAL_SETUP_FT],[
         fi
     fi
 
-    OPAL_SUMMARY_ADD([[Miscellaneous]],[[Fault Tolerance support]],[unnecessary], [$opal_want_ft_type])
+    OPAL_SUMMARY_ADD([Miscellaneous], [Fault Tolerance support], [], [$opal_want_ft_type])
 ])
diff --git a/config/opal_setup_man_pages.m4 b/config/opal_setup_man_pages.m4
deleted file mode 100644
index b9b0617ddbe..00000000000
--- a/config/opal_setup_man_pages.m4
+++ /dev/null
@@ -1,79 +0,0 @@
-dnl -*- shell-script -*-
-dnl
-dnl Copyright (c) 2020-2021 Cisco Systems, Inc.  All rights reserved.
-dnl
-dnl $COPYRIGHT$
-dnl
-dnl Additional copyrights may follow
-dnl
-dnl $HEADER$
-dnl
-
-dnl
-dnl Just in case someone looks for it here someday, here is a
-dnl conveninent reference for what Markdown pandoc supports:
-dnl
-dnl https://rmarkdown.rstudio.com/authoring_pandoc_markdown.html
-dnl
-
-AC_DEFUN([OPAL_SETUP_MAN_PAGES],[
-    AC_ARG_ENABLE([man-pages],
-                  [AS_HELP_STRING([--disable-man-pages],
-                                  [Do not generate/install man pages (default: enable)])])
-
-    PANDOC=
-    OPAL_ENABLE_MAN_PAGES=0
-    AC_MSG_CHECKING([if want man pages])
-    AS_IF([test -z "$enable_man_pages" || test "$enable_man_pages" = "yes"],
-          [AC_MSG_RESULT([yes])
-	   OPAL_ENABLE_MAN_PAGES=1
-	   _OPAL_SETUP_PANDOC],
-	  [AC_MSG_RESULT([no])])
-
-    AC_SUBST(PANDOC)
-    AM_CONDITIONAL([OPAL_ENABLE_MAN_PAGES], [test $OPAL_ENABLE_MAN_PAGES -eq 1])
-])
-
-dnl Back-end pandoc setup
-AC_DEFUN([_OPAL_SETUP_PANDOC],[
-    OPAL_VAR_SCOPE_PUSH([min_major_version min_minor_version pandoc_version pandoc_major pandoc_minor])
-
-    # If we need to generate man pages, we need pandoc >v1.12.
-    AC_PATH_PROG([PANDOC], [pandoc])
-
-    # If we found Pandoc, check its version.  We need >=v1.12.
-    # To be clear: I know that v1.12 works, and I know that v1.9 does not
-    # work.  I did not test the versions in between to know exactly what
-    # the lowest version is that works.  Someone is free to update this
-    # check someday to be more accurate if they wish.
-    min_major_version=1
-    min_minor_version=12
-    AS_IF([test -n "$PANDOC"],
-          [pandoc_version=`pandoc --version | head -n 1 | awk '{ print $[2] }'`
-           pandoc_major=`echo $pandoc_version | cut -d\. -f1`
-           pandoc_minor=`echo $pandoc_version | cut -d\. -f2`
-           AC_MSG_CHECKING([pandoc version])
-           AC_MSG_RESULT([major: $pandoc_major, minor: $pandoc_minor])
-
-           AC_MSG_CHECKING([if pandoc version is >=v$min_major_version.$min_minor_version])
-           AS_IF([test $pandoc_major -lt $min_major_version], [PANDOC=])
-           AS_IF([test $pandoc_major -eq $min_major_version && test $pandoc_minor -lt $min_minor_version],
-                 [PANDOC=])
-           AS_IF([test -n "$PANDOC"],
-                 [AC_MSG_RESULT([yes])],
-                 [AC_MSG_RESULT([no])])
-          ])
-
-    AS_IF([test -z "$PANDOC" || test -n "`echo $PANDOC | $GREP missing`"],
-          [AS_IF([test ! -f "$srcdir/ompi/mpi/man/man5/MPI_T.5"],
-                 [AC_MSG_WARN([*** Could not find a suitable pandoc on your system.])
-                  AC_MSG_WARN([*** You need pandoc >=$min_major_version.$min_minor_version to build Open MPI with man pages.])
-                  AC_MSG_WARN([*** If no man pages are needed, rerun configure with option '--disable-man-pages'.])
-                  AC_MSG_WARN([*** NOTE: If you are building from a tarball downloaded from www.open-mpi.org, you do not need Pandoc.])
-                  AC_MSG_WARN([*** See pandoc.org.])
-                  AC_MSG_ERROR([Cannot continue])
-                 ])
-           ])
-
-    OPAL_VAR_SCOPE_POP
-])
diff --git a/config/opal_setup_wrappers.m4 b/config/opal_setup_wrappers.m4
index 9875e668947..697f4652248 100644
--- a/config/opal_setup_wrappers.m4
+++ b/config/opal_setup_wrappers.m4
@@ -1,4 +1,4 @@
-dnl -*- shell-script -*-
+dnl -*- autoconf -*-
 dnl
 dnl Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
 dnl                         University Research and Technology
@@ -11,12 +11,13 @@ dnl                         University of Stuttgart.  All rights reserved.
 dnl Copyright (c) 2004-2005 The Regents of the University of California.
 dnl                         All rights reserved.
 dnl Copyright (c) 2006-2010 Oracle and/or its affiliates.  All rights reserved.
-dnl Copyright (c) 2009-2016 Cisco Systems, Inc.  All rights reserved.
+dnl Copyright (c) 2009-2021 Cisco Systems, Inc.  All rights reserved.
 dnl Copyright (c) 2015-2017 Research Organization for Information Science
 dnl                         and Technology (RIST). All rights reserved.
 dnl Copyright (c) 2016      IBM Corporation.  All rights reserved.
 dnl Copyright (c) 2020      Triad National Security, LLC. All rights
 dnl                         reserved.
+dnl Copyright (c) 2021-2022 Amazon.com, Inc. or its affiliates.  All Rights reserved.
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
@@ -41,8 +42,12 @@ AC_DEFUN([OPAL_WRAPPER_FLAGS_ADD], [
           [$1], [CXXFLAGS], [OPAL_FLAGS_APPEND_UNIQ([wrapper_extra_cxxflags], [$2])],
           [$1], [FCFLAGS], [OPAL_FLAGS_APPEND_UNIQ([wrapper_extra_fcflags], [$2])],
           [$1], [LDFLAGS], [OPAL_FLAGS_APPEND_UNIQ([wrapper_extra_ldflags], [$2])],
-          [$1], [LIBS], [OPAL_FLAGS_APPEND_UNIQ([wrapper_extra_libs], [$2])],
+          [$1], [STATIC_LDFLAGS], [OPAL_FLAGS_APPEND_UNIQ([wrapper_extra_static_ldflags], [$2])],
+          [$1], [LIBS], [OPAL_FLAGS_APPEND_MOVE([wrapper_extra_libs], [$2])],
+          [$1], [STATIC_LIBS], [OPAL_FLAGS_APPEND_UNIQ([wrapper_extra_static_libs], [$2])],
+          [$1], [PC_MODULES], [OPAL_APPEND_UNIQ([wrapper_extra_pkgconfig_modules], [$2])],
           [m4_fatal([Unknown wrapper flag type $1])])
+    opal_show_verbose "Adding \"$2\" to \"$1\""
 ])
 
 
@@ -77,27 +82,30 @@ AC_DEFUN([OPAL_WRAPPER_FLAGS_ADD], [
 #     <flag>_prefix, configure is not.  There's no known use case for
 #     doing so, and we'd like to force the issue.
 AC_DEFUN([OPAL_SETUP_WRAPPER_INIT],[
+    dnl for OPAL_CC
+    AC_REQUIRE([OPAL_SETUP_CC])
+
+    opal_show_subtitle "Wrapper compiler setup"
+
     OPAL_VAR_SCOPE_PUSH([wrapper_cc_tmp])
-    # AC_PROG_CC_C99 changes CC (instead of CFLAGS) so this method
-    # must be called before OPAL_SETUP_CC.
     AC_ARG_WITH([wrapper_cc],
-		[AS_HELP_STRING([--with-wrapper-cc=path],
-				[Set a different wrapper C compiler than the one used to build Open MPI])],
-		[], [with_wrapper_cc="$CC"])
+                [AS_HELP_STRING([--with-wrapper-cc=path],
+                                [Set a different wrapper C compiler than the one used to build Open MPI])],
+                [], [with_wrapper_cc="$OPAL_CC"])
 
     AC_MSG_CHECKING([for wrapper C compiler])
 
     if test "$with_wrapper_cc" = "yes" || test "$with_wrapper_cc" = "no" ; then
-	AC_MSG_ERROR([--with-wrapper-cc must have an argument.])
+        AC_MSG_ERROR([--with-wrapper-cc must have an argument.])
     fi
 
     # Get the full path to the wrapper compiler. If it doesn't exist
     # assume that the path is not currently valid.
     wrapper_tmp="$(type -p "$with_wrapper_cc")"
-    WRAPPER_CC="${wrapper_tmp:-$with_wrapper_cc}"
     if test -z "$wrapper_tmp" ; then
-	AC_MSG_WARN([could not find \"$with_wrapper_cc\" in path])
+        AC_MSG_WARN([could not find "$with_wrapper_cc" in path])
     fi
+    WRAPPER_CC=$with_wrapper_cc
 
     AC_MSG_RESULT([$WRAPPER_CC])
 
@@ -115,19 +123,19 @@ AC_DEFUN([OPAL_SETUP_WRAPPER_INIT],[
     AS_IF([test "$with_wrapper_cflags_prefix" = "yes" || test "$with_wrapper_cflags_prefix" = "no"],
           [AC_MSG_ERROR([--with-wrapper-cflags-prefix must have an argument.])])
 
-    AC_ARG_WITH([wrapper-cxxflags],
-        [AS_HELP_STRING([--with-wrapper-cxxflags],
-                        [Extra flags to add to CXXFLAGS when using mpiCC/mpic++])])
-    AS_IF([test "$with_wrapper_cxxflags" = "yes" || test "$with_wrapper_cxxflags" = "no"],
-          [AC_MSG_ERROR([--with-wrapper-cxxflags must have an argument.])])
-
-    AC_ARG_WITH([wrapper-cxxflags-prefix],
-        [AS_HELP_STRING([--with-wrapper-cxxflags-prefix],
-                        [Extra flags to add to CXXFLAGS when using mpiCC/mpic++])])
-    AS_IF([test "$with_wrapper_cxxflags_prefix" = "yes" || test "$with_wrapper_cxxflags_prefix" = "no"],
-          [AC_MSG_ERROR([--with-wrapper-cxxflags-prefix must have an argument.])])
-
     m4_ifdef([project_ompi], [
+            AC_ARG_WITH([wrapper-cxxflags],
+                [AS_HELP_STRING([--with-wrapper-cxxflags],
+                                [Extra flags to add to CXXFLAGS when using mpiCC/mpic++])])
+            AS_IF([test "$with_wrapper_cxxflags" = "yes" || test "$with_wrapper_cxxflags" = "no"],
+                  [AC_MSG_ERROR([--with-wrapper-cxxflags must have an argument.])])
+
+            AC_ARG_WITH([wrapper-cxxflags-prefix],
+                [AS_HELP_STRING([--with-wrapper-cxxflags-prefix],
+                                [Extra flags to add to CXXFLAGS when using mpiCC/mpic++])])
+            AS_IF([test "$with_wrapper_cxxflags_prefix" = "yes" || test "$with_wrapper_cxxflags_prefix" = "no"],
+                  [AC_MSG_ERROR([--with-wrapper-cxxflags-prefix must have an argument.])])
+
             AC_ARG_WITH([wrapper-fcflags],
                 [AS_HELP_STRING([--with-wrapper-fcflags],
                         [Extra flags to add to FCFLAGS when using mpifort])])
@@ -261,6 +269,7 @@ AC_DEFUN([OPAL_SETUP_RUNPATH],[
                      [OPAL_LIBTOOL_CONFIG([wl],[wl_fc],[--tag=FC],[])
                       LDFLAGS="$LDFLAGS_save ${wl_fc}--enable-new-dtags"
                       AC_LANG_PUSH([Fortran])
+                      AC_MSG_CHECKING([if Fortran linker supports RUNPATH])
                       AC_LINK_IFELSE([AC_LANG_SOURCE([[program test 
 end program]])],
                                      [runpath_fc_args="${wl_fc}--enable-new-dtags"
@@ -291,31 +300,91 @@ AC_DEFUN([RPATHIFY_LDFLAGS_INTERNAL],[
                esac
            done
 
-           # Now add in the RPATH args for @{libdir}, and the RUNPATH args
-           rpath_tmp=`echo ${$2} | sed -e s/LIBDIR/@{libdir}/`
-           $1="${$1} $rpath_out $rpath_tmp ${$3}"
+           $1="${$1} $rpath_out"
           ])
     OPAL_VAR_SCOPE_POP
 ])
 
-AC_DEFUN([RPATHIFY_LDFLAGS],[RPATHIFY_LDFLAGS_INTERNAL([$1], [rpath_args], [runpath_args])])
+AC_DEFUN([RPATHIFY_LDFLAGS],[RPATHIFY_LDFLAGS_INTERNAL([$1], [rpath_args])])
 
-AC_DEFUN([RPATHIFY_FC_LDFLAGS],[RPATHIFY_LDFLAGS_INTERNAL([$1], [rpath_fc_args], [runpath_fc_args])])
-
-dnl
-dnl Avoid some repetitive code below
-dnl
-AC_DEFUN([_OPAL_SETUP_WRAPPER_FINAL_PKGCONFIG],[
-    AC_MSG_CHECKING([for $1 pkg-config LDFLAGS])
-    $1_PKG_CONFIG_LDFLAGS=`echo "$$1_WRAPPER_EXTRA_LDFLAGS" | sed -e 's/@{libdir}/\${libdir}/g'`
-    AC_SUBST([$1_PKG_CONFIG_LDFLAGS])
-    AC_MSG_RESULT([$$1_PKG_CONFIG_LDFLAGS])
-])
+AC_DEFUN([RPATHIFY_FC_LDFLAGS],[RPATHIFY_LDFLAGS_INTERNAL([$1], [rpath_fc_args])])
 
 
 # OPAL_SETUP_WRAPPER_FINAL()
 # ---------------------------
+#
+# Here are the situations that we need to cover with wrapper compilers
+# and pkg-config files:
+#
+# 1) --enable-shared --disable-static (today's default): Any
+#    application linking against libmpi will be a dynamically linked
+#    application
+# 2) --enable-shared --enable-static: An application linking against
+#    libmpi will dynamically link against libmpi unless -static (or
+#    similar) is passed, in which case it will static link against
+#    libmpi (and the static versions of all of libmpi's dependencies).
+# 3) --disable-shared --enable-static: Any application linking against
+#    libmpi will link against libmpi.a.  That application will link
+#    against the dynamic versions of libmpi's dependencies, unless
+#    -static is passed.
+#
+# There is one situation we should explicitly handle in terms of
+# wrapper compilers (someone could parse out all the right pkg-config
+# or wrapper compiler options to get the right dependent libraries, of
+# course):
+#
+# 1) --enable-shared --enable-static: An application links via
+#    /usr/lib/libmpi.a instead of -lmpi.  We'll make no attempts to
+#    recognize this case with the wrapper compiler
+#
+# So, we essentially have 5 cases above to cover with the wrapper
+# compiler and pkg-config.  For the wrapper compiler, this means:
+#
+# 1) --enable-shared --disable-static: Regardless of the -static flag,
+#    we only add the -L${libdir} -lmpi
+# 2) --enable-shared --enable-static / no -static flag: we add
+#    -L${libdir} -lmpi
+# 3) --enable-shared --enable-static / -static flag: we add
+#    -L${libdir} -lmpi plus the LDFLAGS and LIBS for our dependencies
+#    AND their reported dependencies (ie the results of pkg-config --libs
+#    --static for all our dependencies).
+# 4) --disable-shared --enable-static / no -static flag: We add
+#    -L${libdir} -lmpi plus the LDFLAGS and LIBS for our dependencies,
+#    but not their dependencies (ie, the results of pkg-config --libs for
+#    all our dependencies)
+# 5) --disable-shared --enable-static / -static flag: We add
+#    -L${libdir} -lmpi plus the LDFLAGS and LIBs for our dependencies
+#    AND their reported dependencies (ie, the results of pkg-config
+#    --libs --static for all our dependencies)
+#
+# For the pkg-config modules, this means:
+#
+# 1) --enable-shared --disable-static: We add -L${libdir} -lmpi to
+#    Libs and Libs.private, Modules, and Modules.private are empty
+# 2/3) --enable-shared --enable-static: We add -L${libdir} -lmpi to
+#    Libs, Libs.private contains all the -L/-ls from our dependencies
+#    that don't have pkg-config modules, Modules is empty, and
+#    Modules.private contains all the modules for our dependencies.
+# 4/5) --disable-shared --enable-static: We add -L${libdir} -lmpi to
+#    Libs, Libs.private contains all the -L/-ls from our dependencies that
+#    don't have pkg-config modules, Modules contains all the modules for
+#    our dependencies, and Modules.private is empty.
+#
+# 2/3 means that `pkg-config --libs mpi` would return -L${libdir}
+#  -lmpi and `pkg-config --libs --static mpi` would return
+# -L${libdir} -lmpi -Lnon-pkg-config-dependency
+# -lnon-pkg-config-dependency ...., plus all the `pkg-config --libs
+# --static` results for all our pkg-config dependencies.
+#
+# 4/5 means that `pkg-config --libs mpi` would return -L${libdir}
+# -lmpi -Lnon-pkg-config-dependency -lnon-pkg-config-dependency ....,
+# plus all the `pkg-config --libs` results for all our pkg-config
+# dependencies AND that `pkg-config --libs --static mpi` would return
+# -L${libdir} -lmpi -Lnon-pkg-config-dependency
+# -lnon-pkg-config-dependency ...., plus all the `pkg-config --libs
+# --static` results for all our pkg-config dependencies.
 AC_DEFUN([OPAL_SETUP_WRAPPER_FINAL],[
+    OPAL_VAR_SCOPE_PUSH([wrapper_tmp_arg wrapper_finalize_opal_libs wrapper_finalize_ompi_libs])
 
     # Setup RPATH support, if desired
     WRAPPER_RPATH_SUPPORT=disabled
@@ -325,189 +394,262 @@ AC_DEFUN([OPAL_SETUP_WRAPPER_FINAL],[
           [AC_MSG_WARN([RPATH support requested but not available])
            AC_MSG_ERROR([Cannot continue])])
 
-    # Note that we have to setup <package>_PKG_CONFIG_LDFLAGS for the
-    # pkg-config files to parallel the
-    # <package>_WRAPPER_EXTRA_LDFLAGS.  This is because pkg-config
-    # will not understand the @{libdir} notation in
-    # *_WRAPPER_EXTRA_LDFLAGS; we have to translate it to ${libdir}.
+    AC_DEFINE_UNQUOTED(WRAPPER_RPATH_SUPPORT, "$WRAPPER_RPATH_SUPPORT",
+        [Whether the wrapper compilers add rpath flags by default])
 
     # We now have all relevant flags.  Substitute them in everywhere.
-    m4_ifdef([project_opal], [
-       AC_MSG_CHECKING([for OPAL CPPFLAGS])
-       if test "$WANT_INSTALL_HEADERS" = "1" ; then
-           OPAL_WRAPPER_EXTRA_CPPFLAGS='-I${includedir}/openmpi'
-       fi
-       OPAL_WRAPPER_EXTRA_CPPFLAGS="$OPAL_WRAPPER_EXTRA_CPPFLAGS $opal_mca_wrapper_extra_cppflags $wrapper_extra_cppflags $with_wrapper_cppflags"
-       AC_SUBST([OPAL_WRAPPER_EXTRA_CPPFLAGS])
-       AC_MSG_RESULT([$OPAL_WRAPPER_EXTRA_CPPFLAGS])
-
-       AC_MSG_CHECKING([for OPAL CFLAGS])
-       OPAL_WRAPPER_EXTRA_CFLAGS="$wrapper_extra_cflags $with_wrapper_cflags"
-       AC_SUBST([OPAL_WRAPPER_EXTRA_CFLAGS])
-       AC_MSG_RESULT([$OPAL_WRAPPER_EXTRA_CFLAGS])
-
-       AC_MSG_CHECKING([for OPAL CFLAGS_PREFIX])
-       OPAL_WRAPPER_EXTRA_CFLAGS_PREFIX="$with_wrapper_cflags_prefix"
-       AC_SUBST([OPAL_WRAPPER_EXTRA_CFLAGS_PREFIX])
-       AC_MSG_RESULT([$OPAL_WRAPPER_EXTRA_CFLAGS_PREFIX])
-
-       AC_MSG_CHECKING([for OPAL CXXFLAGS])
-       OPAL_WRAPPER_EXTRA_CXXFLAGS="$wrapper_extra_cxxflags $with_wrapper_cxxflags"
-       AC_SUBST([OPAL_WRAPPER_EXTRA_CXXFLAGS])
-       AC_MSG_RESULT([$OPAL_WRAPPER_EXTRA_CXXFLAGS])
-
-       AC_MSG_CHECKING([for OPAL CXXFLAGS_PREFIX])
-       OPAL_WRAPPER_EXTRA_CXXFLAGS_PREFIX="$with_wrapper_cxxflags_prefix"
-       AC_SUBST([OPAL_WRAPPER_EXTRA_CXXFLAGS_PREFIX])
-       AC_MSG_RESULT([$OPAL_WRAPPER_EXTRA_CXXFLAGS_PREFIX])
-
-       AC_MSG_CHECKING([for OPAL LDFLAGS])
-       OPAL_WRAPPER_EXTRA_LDFLAGS="$opal_mca_wrapper_extra_ldflags $wrapper_extra_ldflags $with_wrapper_ldflags"
-       RPATHIFY_LDFLAGS([OPAL_WRAPPER_EXTRA_LDFLAGS])
-       AC_SUBST([OPAL_WRAPPER_EXTRA_LDFLAGS])
-       AC_MSG_RESULT([$OPAL_WRAPPER_EXTRA_LDFLAGS])
-
-       # Convert @{libdir} to ${libdir} for pkg-config
-       _OPAL_SETUP_WRAPPER_FINAL_PKGCONFIG([OPAL])
-
-       # wrapper_extra_libs doesn't really get populated until after the mca system runs
-       # since most of the libs come from libtool.  So this is the first time we can
-       # uniq them.  ROMIO in particular adds lots of things already in wrapper_extra_libs,
-       # and this cleans the duplication up a bunch.  Always add everything the user
-       # asked for, as they know better than us.
-       AC_MSG_CHECKING([for OPAL LIBS])
-       OPAL_WRAPPER_EXTRA_LIBS="$opal_mca_wrapper_extra_libs"
-       OPAL_FLAGS_APPEND_UNIQ([OPAL_WRAPPER_EXTRA_LIBS], [$wrapper_extra_libs])
-       OPAL_WRAPPER_EXTRA_LIBS="$OPAL_WRAPPER_EXTRA_LIBS $with_wrapper_libs"
-       AC_SUBST([OPAL_WRAPPER_EXTRA_LIBS])
-       AC_MSG_RESULT([$OPAL_WRAPPER_EXTRA_LIBS])
-    ])
 
-    m4_ifdef([project_orte], [
-       AC_MSG_CHECKING([for ORTE CPPFLAGS])
-       if test "$WANT_INSTALL_HEADERS" = "1" ; then
-           ORTE_WRAPPER_EXTRA_CPPFLAGS='-I${includedir}/openmpi'
-       fi
-       ORTE_WRAPPER_EXTRA_CPPFLAGS="$ORTE_WRAPPER_EXTRA_CPPFLAGS $orte_mca_wrapper_extra_cppflags $wrapper_extra_cppflags $with_wrapper_cppflags"
-       AC_SUBST([ORTE_WRAPPER_EXTRA_CPPFLAGS])
-       AC_MSG_RESULT([$ORTE_WRAPPER_EXTRA_CPPFLAGS])
-
-       AC_MSG_CHECKING([for ORTE CFLAGS])
-       ORTE_WRAPPER_EXTRA_CFLAGS="$wrapper_extra_cflags $with_wrapper_cflags"
-       AC_SUBST([ORTE_WRAPPER_EXTRA_CFLAGS])
-       AC_MSG_RESULT([$ORTE_WRAPPER_EXTRA_CFLAGS])
-
-       AC_MSG_CHECKING([for ORTE CFLAGS_PREFIX])
-       ORTE_WRAPPER_EXTRA_CFLAGS_PREFIX="$with_wrapper_cflags_prefix"
-       AC_SUBST([ORTE_WRAPPER_EXTRA_CFLAGS_PREFIX])
-       AC_MSG_RESULT([$ORTE_WRAPPER_EXTRA_CFLAGS_PREFIX])
-
-       AC_MSG_CHECKING([for ORTE LDFLAGS])
-       ORTE_WRAPPER_EXTRA_LDFLAGS="$orte_mca_wrapper_extra_ldflags $wrapper_extra_ldflags $with_wrapper_ldflags"
-       RPATHIFY_LDFLAGS([ORTE_WRAPPER_EXTRA_LDFLAGS])
-       AC_SUBST([ORTE_WRAPPER_EXTRA_LDFLAGS])
-       AC_MSG_RESULT([$ORTE_WRAPPER_EXTRA_LDFLAGS])
-
-       # Convert @{libdir} to ${libdir} for pkg-config
-       _OPAL_SETUP_WRAPPER_FINAL_PKGCONFIG([ORTE])
-
-       AC_MSG_CHECKING([for ORTE LIBS])
-       ORTE_WRAPPER_EXTRA_LIBS="$orte_mca_wrapper_extra_libs"
-       OPAL_FLAGS_APPEND_UNIQ([ORTE_WRAPPER_EXTRA_LIBS], [$wrapper_extra_libs])
-       ORTE_WRAPPER_EXTRA_LIBS="$ORTE_WRAPPER_EXTRA_LIBS $with_wrapper_libs"
-       AC_SUBST([ORTE_WRAPPER_EXTRA_LIBS])
-       AC_MSG_RESULT([$ORTE_WRAPPER_EXTRA_LIBS])
-
-       m4_ifdef([project_ompi], [], [
-          # these are used by orte_info/ompi_info (yes, they are named poorly)
-          AC_DEFINE_UNQUOTED(WRAPPER_EXTRA_CFLAGS, "$ORTE_WRAPPER_EXTRA_CFLAGS",
-              [Additional CFLAGS to pass through the wrapper compilers])
-          AC_DEFINE_UNQUOTED(WRAPPER_EXTRA_CFLAGS_PREFIX, "$ORTE_WRAPPER_EXTRA_CFLAGS_PREFIX",
-              [Additional CFLAGS_PREFIX to pass through the wrapper compilers])
-          AC_DEFINE_UNQUOTED(WRAPPER_EXTRA_LDFLAGS, "$ORTE_WRAPPER_EXTRA_LDFLAGS",
-              [Additional LDFLAGS to pass through the wrapper compilers])
-          AC_DEFINE_UNQUOTED(WRAPPER_EXTRA_LIBS, "$ORTE_WRAPPER_EXTRA_LIBS",
-              [Additional LIBS to pass through the wrapper compilers])
-       ])
+    dnl Add LIBS into the extra wrapper libs, since this is as last
+    dnl minute as we can get.  We do the temp variable bit because of
+    dnl libevent and hwloc dependencies.  LIBS is going to contain
+    dnl libevent/libev/hwloc libraries, but their dependencies are
+    dnl already in wrapper_extra_libs.  We do not want to move -lhwloc
+    dnl (for example) to the far right, right of its dependencies.  So
+    dnl we start with our base libs, and add all the wrapper extra
+    dnl bits to that.
+    wapper_tmp_arg="${LIBS}"
+    OPAL_FLAGS_APPEND_MOVE([wrapper_tmp_arg], [${wrapper_extra_libs}])
+    wrapper_extra_libs="${wrapper_tmp_arg}"
+
+    dnl We do not want ${includedir} to be expanded, as we want that
+    dnl expansion to happen in the wrapper or pkg-config.
+    m4_ifdef([project_opal], [
+       AC_MSG_CHECKING([for OPAL wrapper CPPFLAGS])
+       AS_IF([test "$WANT_INSTALL_HEADERS" = "1"],
+             [OPAL_WRAPPER_CPPFLAGS='-I${includedir} -I${includedir}/openmpi'],
+             [OPAL_WRAPPER_CPPFLAGS='-I${includedir}'])
+       OPAL_FLAGS_APPEND_UNIQ([OPAL_WRAPPER_CPPFLAGS], [$opal_mca_wrapper_extra_cppflags $wrapper_extra_cppflags $with_wrapper_cppflags])
+       AC_SUBST([OPAL_WRAPPER_CPPFLAGS])
+       AC_MSG_RESULT([$OPAL_WRAPPER_CPPFLAGS])
+
+       AC_MSG_CHECKING([for OPAL wrapper CFLAGS])
+       OPAL_WRAPPER_CFLAGS="$wrapper_extra_cflags $with_wrapper_cflags"
+       AC_SUBST([OPAL_WRAPPER_CFLAGS])
+       AC_MSG_RESULT([$OPAL_WRAPPER_CFLAGS])
+
+       AC_MSG_CHECKING([for OPAL wrapper CFLAGS_PREFIX])
+       OPAL_WRAPPER_CFLAGS_PREFIX="$with_wrapper_cflags_prefix"
+       AC_SUBST([OPAL_WRAPPER_CFLAGS_PREFIX])
+       AC_MSG_RESULT([$OPAL_WRAPPER_CFLAGS_PREFIX])
+
+       AC_MSG_CHECKING([for OPAL wrapper CXXFLAGS])
+       OPAL_WRAPPER_CXXFLAGS="$wrapper_extra_cxxflags $with_wrapper_cxxflags"
+       AC_SUBST([OPAL_WRAPPER_CXXFLAGS])
+       AC_MSG_RESULT([$OPAL_WRAPPER_CXXFLAGS])
+
+       AC_MSG_CHECKING([for OPAL wrapper CXXFLAGS_PREFIX])
+       OPAL_WRAPPER_CXXFLAGS_PREFIX="$with_wrapper_cxxflags_prefix"
+       AC_SUBST([OPAL_WRAPPER_CXXFLAGS_PREFIX])
+       AC_MSG_RESULT([$OPAL_WRAPPER_CXXFLAGS_PREFIX])
+
+       wrapper_finalize_opal_libs="-l${OPAL_LIB_NAME}"
+
+       dnl No matter the configuration (see the 5 cases above), the base
+       dnl flags should contain a -L${libdir} and -lopen-pal, so that those
+       dnl are found.
+       OPAL_WRAPPER_LDFLAGS='-L${libdir}'
+       OPAL_WRAPPER_LIBS="${wrapper_finalize_opal_libs}"
+       OPAL_WRAPPER_LIBS_STATIC=
+       OPAL_WRAPPER_LDFLAGS_STATIC=
+
+       AS_IF(dnl shared only case.  We add no flags beyond the base -L/-l
+             [test "${enable_shared}" != "no" -a "${enable_static}" != "yes"],
+             [],
+             dnl building both shared and static libraries.  The base
+             dnl case remains the same as the shared-only case (because
+             dnl the app will link against the shared library, but the
+             dnl static case is the full dependency tree.  Our full
+             dnl dependency tree is both the wrapper_extra_libs and
+             dnl wrapper_extra_static_libs, because wrapper_extra_libs
+             dnl was not added to the normal case.
+             [test "${enable_shared}" != "no" -a "${enable_static}" = "yes"],
+             [OPAL_FLAGS_APPEND_UNIQ([OPAL_WRAPPER_LDFLAGS_STATIC], [${opal_mca_wrapper_extra_ldflags} ${wrapper_extra_ldflags}])
+              OPAL_FLAGS_APPEND_MOVE([OPAL_WRAPPER_LIBS_STATIC], [${opal_mca_wrapper_extra_libs} ${wrapper_extra_libs}])
+              OPAL_FLAGS_APPEND_UNIQ([OPAL_WRAPPER_LDFLAGS_STATIC], [${opal_mca_wrapper_extra_static_ldflags} ${wrapper_extra_static_ldflags}])
+              OPAL_FLAGS_APPEND_MOVE([OPAL_WRAPPER_LIBS_STATIC], [${opal_mca_wrapper_extra_static_libs} ${wrapper_extra_static_libs}])],
+             dnl building static only.  The base case is that we need to
+             dnl list our dependencies, but not the full treee, because
+             dnl we assume that our dependencies will be shared libraries
+             dnl (unless they too were built static only, in which case
+             dnl their dependencies will be our direct dependencies if
+             dnl their modules are setup correctly).  The static case is
+             dnl our full dependency tree, but we only need to list the
+             dnl second leve explicitly, because the wrapper compiler
+             dnl and/or pkg-config merge use the normal case data in the
+             dnl static case.
+             [OPAL_FLAGS_APPEND_UNIQ([OPAL_WRAPPER_LDFLAGS], [${opal_mca_wrapper_extra_ldflags} ${wrapper_extra_ldflags}])
+              OPAL_FLAGS_APPEND_MOVE([OPAL_WRAPPER_LIBS], [${opal_mca_wrapper_extra_libs} ${wrapper_extra_libs}])
+              OPAL_FLAGS_APPEND_UNIQ([OPAL_WRAPPER_LDFLAGS_STATIC], [${opal_mca_wrapper_extra_static_ldflags} ${wrapper_extra_static_ldflags}])
+              OPAL_FLAGS_APPEND_MOVE([OPAL_WRAPPER_LIBS_STATIC], [${opal_mca_wrapper_extra_static_libs} ${wrapper_extra_static_libs}])])
+
+       dnl Add the user-provided flags
+       OPAL_FLAGS_APPEND_UNIQ([OPAL_WRAPPER_LDFLAGS], [${with_wrapper_ldflags}])
+       OPAL_FLAGS_APPEND_MOVE([OPAL_WRAPPER_LIBS], [${with_wrapper_libs}])
+
+       RPATHIFY_LDFLAGS([OPAL_WRAPPER_LDFLAGS])
+       RPATHIFY_LDFLAGS([OPAL_WRAPPER_LDFLAGS_STATIC])
+
+       OPAL_FLAGS_APPEND_UNIQ([OPAL_WRAPPER_LDFLAGS], [${runpath_args}])
+
+       AC_MSG_CHECKING([for OPAL wrapper LDFLAGS])
+       AC_SUBST([OPAL_WRAPPER_LDFLAGS])
+       AC_MSG_RESULT([$OPAL_WRAPPER_LDFLAGS])
+
+       AC_MSG_CHECKING([for OPAL wrapper static LDFLAGS])
+       AC_SUBST([OPAL_WRAPPER_LDFLAGS_STATIC])
+       AC_MSG_RESULT([$OPAL_WRAPPER_LDFLAGS_STATIC])
+
+       AC_MSG_CHECKING([for OPAL wrapper LIBS])
+       AC_SUBST([OPAL_WRAPPER_LIBS])
+       AC_MSG_RESULT([$OPAL_WRAPPER_LIBS])
+
+       AC_MSG_CHECKING([for OPAL wrapper static LIBS])
+       AC_SUBST([OPAL_WRAPPER_LIBS_STATIC])
+       AC_MSG_RESULT([$OPAL_WRAPPER_LIBS_STATIC])
     ])
 
     m4_ifdef([project_ompi], [
-       AC_MSG_CHECKING([for OMPI CPPFLAGS])
-       if test "$WANT_INSTALL_HEADERS" = "1" ; then
-           OMPI_WRAPPER_EXTRA_CPPFLAGS='-I${includedir}/openmpi'
-       fi
-       OMPI_WRAPPER_EXTRA_CPPFLAGS="$OMPI_WRAPPER_EXTRA_CPPFLAGS $ompi_mca_wrapper_extra_cppflags $wrapper_extra_cppflags $with_wrapper_cppflags"
-       AC_SUBST([OMPI_WRAPPER_EXTRA_CPPFLAGS])
-       AC_MSG_RESULT([$OMPI_WRAPPER_EXTRA_CPPFLAGS])
-
-       AC_MSG_CHECKING([for OMPI CFLAGS])
-       OMPI_WRAPPER_EXTRA_CFLAGS="$wrapper_extra_cflags $with_wrapper_cflags"
-       AC_SUBST([OMPI_WRAPPER_EXTRA_CFLAGS])
-       AC_MSG_RESULT([$OMPI_WRAPPER_EXTRA_CFLAGS])
-
-       AC_MSG_CHECKING([for OMPI CFLAGS_PREFIX])
-       OMPI_WRAPPER_EXTRA_CFLAGS_PREFIX="$with_wrapper_cflags_prefix"
-       AC_SUBST([OMPI_WRAPPER_EXTRA_CFLAGS_PREFIX])
-       AC_MSG_RESULT([$OMPI_WRAPPER_EXTRA_CFLAGS_PREFIX])
-
-       AC_MSG_CHECKING([for OMPI CXXFLAGS])
-       OMPI_WRAPPER_EXTRA_CXXFLAGS="$wrapper_extra_cxxflags $with_wrapper_cxxflags"
-       AC_SUBST([OMPI_WRAPPER_EXTRA_CXXFLAGS])
-       AC_MSG_RESULT([$OMPI_WRAPPER_EXTRA_CXXFLAGS])
-
-       AC_MSG_CHECKING([for OMPI CXXFLAGS_PREFIX])
-       OMPI_WRAPPER_EXTRA_CXXFLAGS_PREFIX="$with_wrapper_cxxflags_prefix"
-       AC_SUBST([OMPI_WRAPPER_EXTRA_CXXFLAGS_PREFIX])
-       AC_MSG_RESULT([$OMPI_WRAPPER_EXTRA_CXXFLAGS_PREFIX])
-
-       AC_MSG_CHECKING([for OMPI FCFLAGS])
-       OMPI_WRAPPER_EXTRA_FCFLAGS="$wrapper_extra_fcflags $with_wrapper_fcflags"
-       if test "$OMPI_FC_MODULE_FLAG" != "" ; then
-           OMPI_WRAPPER_EXTRA_FCFLAGS="$OMPI_WRAPPER_EXTRA_FCFLAGS $OMPI_FC_MODULE_FLAG"'${libdir}'
-       fi
-       AC_SUBST([OMPI_WRAPPER_EXTRA_FCFLAGS])
+       AC_MSG_CHECKING([for OMPI wrapper CPPFLAGS])
+       AS_IF([test "$WANT_INSTALL_HEADERS" = "1"],
+             [OMPI_WRAPPER_CPPFLAGS='-I${includedir} -I${includedir}/openmpi'],
+             [OMPI_WRAPPER_CPPFLAGS='-I${includedir}'])
+       OPAL_FLAGS_APPEND_UNIQ([OMPI_WRAPPER_CPPFLAGS], [$ompi_mca_wrapper_extra_cppflags $wrapper_extra_cppflags $with_wrapper_cppflags])
+       AC_SUBST([OMPI_WRAPPER_CPPFLAGS])
+       AC_MSG_RESULT([$OMPI_WRAPPER_CPPFLAGS])
+
+       AC_MSG_CHECKING([for OMPI wrapper CFLAGS])
+       OMPI_WRAPPER_CFLAGS="$wrapper_extra_cflags $with_wrapper_cflags"
+       AC_SUBST([OMPI_WRAPPER_CFLAGS])
+       AC_MSG_RESULT([$OMPI_WRAPPER_CFLAGS])
+
+       AC_MSG_CHECKING([for OMPI wrapper CFLAGS_PREFIX])
+       OMPI_WRAPPER_CFLAGS_PREFIX="$with_wrapper_cflags_prefix"
+       AC_SUBST([OMPI_WRAPPER_CFLAGS_PREFIX])
+       AC_MSG_RESULT([$OMPI_WRAPPER_CFLAGS_PREFIX])
+
+       AC_MSG_CHECKING([for OMPI wrapper CXXFLAGS])
+       OMPI_WRAPPER_CXXFLAGS="$wrapper_extra_cxxflags $with_wrapper_cxxflags"
+       AC_SUBST([OMPI_WRAPPER_CXXFLAGS])
+       AC_MSG_RESULT([$OMPI_WRAPPER_CXXFLAGS])
+
+       AC_MSG_CHECKING([for OMPI wrapper CXXFLAGS_PREFIX])
+       OMPI_WRAPPER_CXXFLAGS_PREFIX="$with_wrapper_cxxflags_prefix"
+       AC_SUBST([OMPI_WRAPPER_CXXFLAGS_PREFIX])
+       AC_MSG_RESULT([$OMPI_WRAPPER_CXXFLAGS_PREFIX])
+
+       AC_MSG_CHECKING([for OMPI wrapper FCFLAGS])
+       OMPI_WRAPPER_FCFLAGS='-I${includedir}'" ${wrapper_extra_fcflags} ${with_wrapper_fcflags}"
+       AS_IF([test -n "${OMPI_FC_MODULE_FLAG}"],
+             [dnl deal with some interesting expansion behavior in OPAL_APPEND
+              wrapper_tmp_arg="${OMPI_FC_MODULE_FLAG}${OMPI_FORTRAN_MODULEDIR}"
+              OPAL_APPEND([OMPI_WRAPPER_FCFLAGS], [${wrapper_tmp_arg}])])
+       AC_SUBST([OMPI_WRAPPER_FCFLAGS])
        AC_MSG_RESULT([$OMPI_WRAPPER_EXTRA_FCFLAGS])
 
-       AC_MSG_CHECKING([for OMPI FCFLAGS_PREFIX])
-       OMPI_WRAPPER_EXTRA_FCFLAGS_PREFIX="$with_wrapper_fcflags_prefix"
-       AC_SUBST([OMPI_WRAPPER_EXTRA_FCFLAGS_PREFIX])
-       AC_MSG_RESULT([$OMPI_WRAPPER_EXTRA_FCFLAGS_PREFIX])
-
-       AC_MSG_CHECKING([for OMPI LDFLAGS])
-       OMPI_WRAPPER_EXTRA_LDFLAGS="$ompi_mca_wrapper_extra_ldflags $wrapper_extra_ldflags $with_wrapper_ldflags"
-       OMPI_WRAPPER_EXTRA_FC_LDFLAGS=$OMPI_WRAPPER_EXTRA_LDFLAGS
-       RPATHIFY_LDFLAGS([OMPI_WRAPPER_EXTRA_LDFLAGS])
-       AC_SUBST([OMPI_WRAPPER_EXTRA_LDFLAGS])
-       AC_MSG_RESULT([$OMPI_WRAPPER_EXTRA_LDFLAGS])
-       RPATHIFY_FC_LDFLAGS([OMPI_WRAPPER_EXTRA_FC_LDFLAGS])
-       AC_SUBST([OMPI_WRAPPER_EXTRA_FC_LDFLAGS])
-
-       # Convert @{libdir} to ${libdir} for pkg-config
-       _OPAL_SETUP_WRAPPER_FINAL_PKGCONFIG([OMPI])
-
-       AC_MSG_CHECKING([for OMPI LIBS])
-       OMPI_WRAPPER_EXTRA_LIBS="$ompi_mca_wrapper_extra_libs"
-       OPAL_FLAGS_APPEND_UNIQ([OMPI_WRAPPER_EXTRA_LIBS], [$wrapper_extra_libs])
-       OMPI_WRAPPER_EXTRA_LIBS="$OMPI_WRAPPER_EXTRA_LIBS $with_wrapper_libs"
-       OPAL_FLAGS_APPEND_UNIQ([OMPI_WRAPPER_EXTRA_LIBS], [$LIBS])
-       AC_SUBST([OMPI_WRAPPER_EXTRA_LIBS])
-       AC_MSG_RESULT([$OMPI_WRAPPER_EXTRA_LIBS])
-
-      # language binding support.  C++ is a bit different, as the
-      # compiler should work even if there is no MPI C++ bindings
-      # support.  However, we do want it to fail if there is no C++
-      # compiler.
-       if test "$WANT_MPI_CXX_SUPPORT" = "1" ; then
-          OMPI_WRAPPER_CXX_LIB="-l${with_libmpi_name}_cxx"
-          OMPI_WRAPPER_CXX_REQUIRED_FILE=""
-       elif test "$CXX" = "none"; then
-          OMPI_WRAPPER_CXX_LIB=""
+       AC_MSG_CHECKING([for OMPI wrapper FCFLAGS_PREFIX])
+       OMPI_WRAPPER_FCFLAGS_PREFIX="$with_wrapper_fcflags_prefix"
+       AC_SUBST([OMPI_WRAPPER_FCFLAGS_PREFIX])
+       AC_MSG_RESULT([$OMPI_WRAPPER_FCFLAGS_PREFIX])
+
+       wrapper_finalize_ompi_libs="-l${OMPI_LIBMPI_NAME}"
+
+       dnl No matter the configuration (see the 5 cases above), the base
+       dnl flags should contain a -L${libdir} and -lmpi, so that those
+       dnl are found.
+       OMPI_WRAPPER_LDFLAGS='-L${libdir}'
+       OMPI_WRAPPER_LIBS="${wrapper_finalize_ompi_libs}"
+       OMPI_WRAPPER_LIBS_STATIC=
+       OMPI_WRAPPER_LDFLAGS_STATIC=
+
+       AS_IF(dnl shared only case.  We add no flags beyond the base -L/-l
+             [test "${enable_shared}" != "no" -a "${enable_static}" != "yes"],
+             [],
+             dnl building both shared and static libraries.  The base
+             dnl case remains the same as the shared-only case (because
+             dnl the app will link against the shared library, but the
+             dnl static case is the full dependency tree.  Our full
+             dnl dependency tree is both the wrapper_extra_libs and
+             dnl wrapper_extra_static_libs, because wrapper_extra_libs
+             dnl was not added to the normal case.
+             [test "${enable_shared}" != "no" -a "${enable_static}" = "yes"],
+             [OPAL_FLAGS_APPEND_UNIQ([OMPI_WRAPPER_LDFLAGS_STATIC], [${ompi_mca_wrapper_extra_ldflags} ${wrapper_extra_ldflags}])
+              OPAL_FLAGS_APPEND_MOVE([OMPI_WRAPPER_LIBS_STATIC], [${wrapper_finalize_opal_libs} ${ompi_mca_wrapper_extra_libs} ${wrapper_extra_libs}])
+              OPAL_FLAGS_APPEND_UNIQ([OMPI_WRAPPER_LDFLAGS_STATIC], [${ompi_mca_wrapper_extra_static_ldflags} ${wrapper_extra_static_ldflags}])
+              OPAL_FLAGS_APPEND_MOVE([OMPI_WRAPPER_LIBS_STATIC], [${ompi_mca_wrapper_extra_static_libs} ${wrapper_extra_static_libs}])],
+             dnl building static only.  The base case is that we need to
+             dnl list our dependencies, but not the full treee, because
+             dnl we assume that our dependencies will be shared libraries
+             dnl (unless they too were built static only, in which case
+             dnl their dependencies will be our direct dependencies if
+             dnl their modules are setup correctly).  The static case is
+             dnl our full dependency tree, but we only need to list the
+             dnl second leve explicitly, because the wrapper compiler
+             dnl and/or pkg-config merge use the normal case data in the
+             dnl static case.
+             [OPAL_FLAGS_APPEND_UNIQ([OMPI_WRAPPER_LDFLAGS], [${ompi_mca_wrapper_extra_ldflags} ${wrapper_extra_ldflags}])
+              OPAL_FLAGS_APPEND_MOVE([OMPI_WRAPPER_LIBS], [${wrapper_finalize_opal_libs} ${ompi_mca_wrapper_extra_libs} ${wrapper_extra_libs}])
+              OPAL_FLAGS_APPEND_UNIQ([OMPI_WRAPPER_LDFLAGS_STATIC], [${ompi_mca_wrapper_extra_static_ldflags} ${wrapper_extra_static_ldflags}])
+              OPAL_FLAGS_APPEND_MOVE([OMPI_WRAPPER_LIBS_STATIC], [${ompi_mca_wrapper_extra_static_libs} ${wrapper_extra_static_libs}])])
+
+       dnl Add the user-provided flags
+       OPAL_FLAGS_APPEND_UNIQ([OMPI_WRAPPER_LDFLAGS], [${with_wrapper_ldflags}])
+       OPAL_FLAGS_APPEND_MOVE([OMPI_WRAPPER_LIBS], [${with_wrapper_libs}])
+
+       dnl fortran FTW!
+       OMPI_WRAPPER_FC_LIBS="${OMPI_FORTRAN_USEMPIF08_LIB} ${OMPI_FORTRAN_USEMPI_LIB} ${OMPI_FORTRAN_MPIFH_LINK} ${OMPI_WRAPPER_LIBS}"
+       OMPI_WRAPPER_FC_LIBS_STATIC=${OMPI_WRAPPER_LIBS_STATIC}
+       OMPI_WRAPPER_FC_LDFLAGS=$OMPI_WRAPPER_LDFLAGS
+       OMPI_WRAPPER_FC_LDFLAGS_STATIC=$OMPI_WRAPPER_LDFLAGS
+
+       RPATHIFY_LDFLAGS([OMPI_WRAPPER_LDFLAGS])
+       RPATHIFY_LDFLAGS([OMPI_WRAPPER_LDFLAGS_STATIC])
+       RPATHIFY_FC_LDFLAGS([OMPI_WRAPPER_FC_LDFLAGS])
+       RPATHIFY_FC_LDFLAGS([OMPI_WRAPPER_FC_LDFLAGS_STATIC])
+
+       OPAL_FLAGS_APPEND_UNIQ([OMPI_WRAPPER_LDFLAGS], [${runpath_args}])
+       OPAL_FLAGS_APPEND_UNIQ([OMPI_WRAPPER_FC_LDFLAGS], [${runpath_fc_args}])
+
+       AC_MSG_CHECKING([for OMPI wrapper LDFLAGS])
+       AC_SUBST([OMPI_WRAPPER_LDFLAGS])
+       AC_MSG_RESULT([$OMPI_WRAPPER_LDFLAGS])
+
+       AC_MSG_CHECKING([for OMPI wrapper static LDFLAGS])
+       AC_SUBST([OMPI_WRAPPER_LDFLAGS_STATIC])
+       AC_MSG_RESULT([$OMPI_WRAPPER_LDFLAGS_STATIC])
+
+       AC_MSG_CHECKING([for OMPI wrapper LIBS])
+       AC_SUBST([OMPI_WRAPPER_LIBS])
+       AC_MSG_RESULT([$OMPI_WRAPPER_LIBS])
+
+       AC_MSG_CHECKING([for OMPI wrapper static LIBS])
+       AC_SUBST([OMPI_WRAPPER_LIBS_STATIC])
+       AC_MSG_RESULT([$OMPI_WRAPPER_LIBS_STATIC])
+
+       AC_MSG_CHECKING([for OMPI wrapper Fortran LDFLAGS])
+       AC_SUBST([OMPI_WRAPPER_FC_LDFLAGS])
+       AC_MSG_RESULT([$OMPI_WRAPPER_FC_LDFLAGS])
+
+       AC_MSG_CHECKING([for OMPI wrapper Fortran static LDFLAGS])
+       AC_SUBST([OMPI_WRAPPER_FC_LDFLAGS_STATIC])
+       AC_MSG_RESULT([$OMPI_WRAPPER_LDFLAGS_STATIC])
+
+       AC_MSG_CHECKING([for OMPI wrapper Fortran LIBS])
+       AC_SUBST([OMPI_WRAPPER_FC_LIBS])
+       AC_MSG_RESULT([$OMPI_WRAPPER_FC_LIBS])
+
+       AC_MSG_CHECKING([for OMPI wrapper Fortran static LIBS])
+       AC_SUBST([OMPI_WRAPPER_FC_LIBS_STATIC])
+       AC_MSG_RESULT([$OMPI_WRAPPER_FC_LIBS_STATIC])
+
+       # language binding support.  C++ is a bit different, as the
+       # compiler should work even if there is no MPI C++ bindings
+       # support.  However, we do want it to fail if there is no C++
+       # compiler.
+       if test "$CXX" = "none"; then
           OMPI_WRAPPER_CXX_REQUIRED_FILE="not supported"
        else
-          OMPI_WRAPPER_CXX_LIB=""
           OMPI_WRAPPER_CXX_REQUIRED_FILE=""
        fi
-       AC_SUBST([OMPI_WRAPPER_CXX_LIB])
        AC_SUBST([OMPI_WRAPPER_CXX_REQUIRED_FILE])
 
        if test "$OMPI_TRY_FORTRAN_BINDINGS" -gt "$OMPI_FORTRAN_NO_BINDINGS" ; then
@@ -533,24 +675,299 @@ AC_DEFUN([OPAL_SETUP_WRAPPER_FINAL],[
                          [chmod +x ompi/tools/wrappers/ompi_wrapper_script])
        fi
 
-       AC_DEFINE_UNQUOTED(WRAPPER_EXTRA_CFLAGS, "$OMPI_WRAPPER_EXTRA_CFLAGS",
-           [Additional CFLAGS to pass through the wrapper compilers])
-       AC_DEFINE_UNQUOTED(WRAPPER_EXTRA_CFLAGS_PREFIX, "$OMPI_WRAPPER_EXTRA_CFLAGS_PREFIX",
-           [Additional CFLAGS_PREFIX to pass through the wrapper compilers])
-       AC_DEFINE_UNQUOTED(WRAPPER_EXTRA_CXXFLAGS, "$OMPI_WRAPPER_EXTRA_CXXFLAGS",
-           [Additional CXXFLAGS to pass through the wrapper compilers])
-       AC_DEFINE_UNQUOTED(WRAPPER_EXTRA_CXXFLAGS_PREFIX, "$OMPI_WRAPPER_EXTRA_CXXFLAGS_PREFIX",
-           [Additional CXXFLAGS_PREFIX to pass through the wrapper compilers])
-       AC_DEFINE_UNQUOTED(WRAPPER_EXTRA_FCFLAGS, "$OMPI_WRAPPER_EXTRA_FCFLAGS",
-           [Additional FCFLAGS to pass through the wrapper compilers])
-       AC_DEFINE_UNQUOTED(WRAPPER_EXTRA_FCFLAGS_PREFIX, "$OMPI_WRAPPER_EXTRA_FCFLAGS_PREFIX",
-           [Additional FCFLAGS to pass through the wrapper compilers])
-       AC_DEFINE_UNQUOTED(WRAPPER_EXTRA_LDFLAGS, "$OMPI_WRAPPER_EXTRA_LDFLAGS",
-           [Additional LDFLAGS to pass through the wrapper compilers])
-       AC_DEFINE_UNQUOTED(WRAPPER_EXTRA_LIBS, "$OMPI_WRAPPER_EXTRA_LIBS",
-           [Additional LIBS to pass through the wrapper compilers])
+       AC_DEFINE_UNQUOTED(OMPI_WRAPPER_CFLAGS, "$OMPI_WRAPPER_CFLAGS",
+           [CFLAGS to pass through the wrapper compilers])
+       AC_DEFINE_UNQUOTED(OMPI_WRAPPER_CXXFLAGS, "$OMPI_WRAPPER_CXXFLAGS",
+           [CXXFLAGS to pass through the wrapper compilers])
+       AC_DEFINE_UNQUOTED(OMPI_WRAPPER_FCFLAGS, "$OMPI_WRAPPER__FCFLAGS",
+           [FCFLAGS to pass through the wrapper compilers])
+       AC_DEFINE_UNQUOTED(OMPI_WRAPPER_LDFLAGS, "$OMPI_WRAPPER_LDFLAGS",
+           [LDFLAGS to pass through the wrapper compilers])
+       AC_DEFINE_UNQUOTED(OMPI_WRAPPER_LIBS, "$OMPI_WRAPPER_LIBS",
+           [LIBS to pass through the wrapper compilers])
+
+       dnl ####################################################################
+       dnl  Setup variables for pkg-config files
+       dnl
+       dnl Add all our dependent libraries to libs.Private for users that want
+       dnl to static build, unless we're only building static libraries, in
+       dnl which case, add the dependent libraries to libs itself, since any
+       dnl linking will require the full set of libraries.
+       dnl ####################################################################
+       AC_MSG_CHECKING([for OMPI pkg-config Cflags])
+       OMPI_PC_CFLAGS="${OMPI_WRAPPER_CPPFLAGS} ${OMPI_WRAPPER_CFLAGS} ${OMPI_WRAPPER_CFLAGS_PREFIX}"
+       OMPI_PC_CFLAGS=`echo ${OMPI_PC_CFLAGS} | sed -e 's/@{/\${/g'`
+       AC_SUBST([OMPI_PC_CFLAGS])
+       AC_MSG_RESULT([${OMPI_PC_CFLAGS}])
+
+       AC_MSG_CHECKING([for OMPI pkg-config Libs])
+       OMPI_PC_LIBS="${OMPI_WRAPPER_LDFLAGS} ${OMPI_WRAPPER_LIBS}"
+       OMPI_PC_LIBS=`echo ${OMPI_PC_LIBS} | sed -e 's/@{/\${/g'`
+       AC_SUBST([OMPI_PC_LIBS])
+       AC_MSG_RESULT([${OMPI_PC_LIBS}])
+
+       AC_MSG_CHECKING([for OMPI pkg-config Libs.private])
+       OMPI_PC_LIBS_PRIVATE="${OMPI_WRAPPER_LDFLAGS_STATIC} ${OMPI_WRAPPER_LIBS_STATIC}"
+       OMPI_PC_LIBS_PRIVATE=`echo ${OMPI_PC_LIBS_PRIVATE} | sed -e 's/@{/\${/g'`
+       AC_SUBST([OMPI_PC_LIBS_PRIVATE])
+       AC_MSG_RESULT([${OMPI_PC_LIBS_PRIVATE}])
+
+       AC_MSG_CHECKING([for OMPI pkg-config Fortran Cflags])
+       OMPI_PC_FC_CFLAGS="${OMPI_WRAPPER_FCFLAGS} ${OMPI_WRAPPER_FCFLAGS_PREFIX}"
+       OMPI_PC_FC_CFLAGS=`echo ${OMPI_PC_FC_CFLAGS} | sed -e 's/@{/\${/g'`
+       AC_SUBST([OMPI_PC_FC_CFLAGS])
+       AC_MSG_RESULT([${OMPI_PC_FC_CFLAGS}])
+
+       AC_MSG_CHECKING([for OMPI pkg-config Fortran Libs])
+       OMPI_PC_FC_LIBS="${OMPI_WRAPPER_FC_LDFLAGS} ${OMPI_WRAPPER_FC_LIBS}"
+       OMPI_PC_FC_LIBS=`echo ${OMPI_PC_FC_LIBS} | sed -e 's/@{/\${/g'`
+       AC_SUBST([OMPI_PC_FC_LIBS])
+       AC_MSG_RESULT([${OMPI_PC_FC_LIBS}])
+
+       AC_MSG_CHECKING([for OMPI pkg-config Fortran Libs.private])
+       OMPI_PC_FC_LIBS_PRIVATE="${OMPI_WRAPPER_FC_LDFLAGS_STATIC} ${OMPI_WRAPPER_FC_LIBS_STATIC}"
+       OMPI_PC_FC_LIBS_PRIVATE=`echo ${OMPI_PC_FC_LIBS_PRIVATE} | sed -e 's/@{/\${/g'`
+       AC_SUBST([OMPI_PC_FC_LIBS_PRIVATE])
+       AC_MSG_RESULT([${OMPI_PC_FC_LIBS_PRIVATE}])
+
+       OMPI_PC_MODULES=
+       OMPI_PC_MODULES_PRIVATE=
+       AS_IF([test "${enable_shared}" != "no" -a "${enable_static}" != "yes"],
+             [],
+             [test "${enable_shared}" != "no" -a "${enable_static}" = "yes"],
+             [OMPI_PC_MODULES_PRIVATE="${wrapper_extra_pkgconfig_modules} ${ompi_mca_wrapper_extra_pc_modules}"],
+             [OMPI_PC_MODULES="${wrapper_extra_pkgconfig_modules} ${ompi_mca_wrapper_extra_pc_modules}"])
+
+       AC_MSG_CHECKING([for OMPI pkg-config Modules])
+       AC_SUBST([OMPI_PC_MODULES])
+       AC_MSG_RESULT([${OMPI_PC_MODULES}])
+
+       AC_MSG_CHECKING([for OMPI pkg-config Modules.private])
+       AC_SUBST([OMPI_PC_MODULES_PRIVATE])
+       AC_MSG_RESULT([${OMPI_PC_MODULES_PRIVATE}])
     ])
 
-    AC_DEFINE_UNQUOTED(WRAPPER_RPATH_SUPPORT, "$WRAPPER_RPATH_SUPPORT",
-        [Whether the wrapper compilers add rpath flags by default])
+    m4_ifdef([project_oshmem], [
+       AC_MSG_CHECKING([for OSHMEM wrapper CPPFLAGS])
+       AS_IF([test "$WANT_INSTALL_HEADERS" = "1"],
+             [OSHMEM_WRAPPER_CPPFLAGS='-I${includedir} -I${includedir}/openmpi'],
+             [OSHMEM_WRAPPER_CPPFLAGS='-I${includedir}'])
+       OPAL_FLAGS_APPEND_UNIQ([OSHMEM_WRAPPER_CPPFLAGS], [$oshmem_mca_wrapper_extra_cppflags $wrapper_extra_cppflags $with_wrapper_cppflags])
+       AC_SUBST([OSHMEM_WRAPPER_CPPFLAGS])
+       AC_MSG_RESULT([$OSHMEM_WRAPPER_CPPFLAGS])
+
+       AC_MSG_CHECKING([for OSHMEM wrapper CFLAGS])
+       OSHMEM_WRAPPER_CFLAGS="$wrapper_extra_cflags $with_wrapper_cflags"
+       AC_SUBST([OSHMEM_WRAPPER_CFLAGS])
+       AC_MSG_RESULT([$OSHMEM_WRAPPER_CFLAGS])
+
+       AC_MSG_CHECKING([for OSHMEM wrapper CFLAGS_PREFIX])
+       OSHMEM_WRAPPER_CFLAGS_PREFIX="$with_wrapper_cflags_prefix"
+       AC_SUBST([OSHMEM_WRAPPER_CFLAGS_PREFIX])
+       AC_MSG_RESULT([$OSHMEM_WRAPPER_CFLAGS_PREFIX])
+
+       AC_MSG_CHECKING([for OSHMEM wrapper CXXFLAGS])
+       OSHMEM_WRAPPER_CXXFLAGS="$wrapper_extra_cxxflags $with_wrapper_cxxflags"
+       AC_SUBST([OSHMEM_WRAPPER_CXXFLAGS])
+       AC_MSG_RESULT([$OSHMEM_WRAPPER_CXXFLAGS])
+
+       AC_MSG_CHECKING([for OSHMEM wrapper CXXFLAGS_PREFIX])
+       OSHMEM_WRAPPER_CXXFLAGS_PREFIX="$with_wrapper_cxxflags_prefix"
+       AC_SUBST([OSHMEM_WRAPPER_CXXFLAGS_PREFIX])
+       AC_MSG_RESULT([$OSHMEM_WRAPPER_CXXFLAGS_PREFIX])
+
+       AC_MSG_CHECKING([for OSHMEM wrapper FCFLAGS])
+       OSHMEM_WRAPPER_FCFLAGS='-I${includedir}'" ${wrapper_extra_fcflags} ${with_wrapper_fcflags}"
+       AS_IF([test -n "${OMPI_FC_MODULE_FLAG}"],
+             [dnl deal with some interesting expansion behavior in OPAL_APPEND
+              wrapper_tmp_arg="${OMPI_FC_MODULE_FLAG}${OMPI_FORTRAN_MODULEDIR}"
+              OPAL_APPEND([OSHMEM_WRAPPER_FCFLAGS], [${wrapper_tmp_arg}])])
+       AC_SUBST([OSHMEM_WRAPPER_FCFLAGS])
+       AC_MSG_RESULT([$OSHMEM_WRAPPER_EXTRA_FCFLAGS])
+
+       AC_MSG_CHECKING([for OSHMEM wrapper FCFLAGS_PREFIX])
+       OSHMEM_WRAPPER_FCFLAGS_PREFIX="$with_wrapper_fcflags_prefix"
+       AC_SUBST([OSHMEM_WRAPPER_FCFLAGS_PREFIX])
+       AC_MSG_RESULT([$OSHMEM_WRAPPER_FCFLAGS_PREFIX])
+
+       dnl No matter the configuration (see the 5 cases above), the
+       dnl base flags should contain a -L${libdir} and -loshmem -lmpi,
+       dnl so that those are found.
+       OSHMEM_WRAPPER_LDFLAGS='-L${libdir}'
+       OSHMEM_WRAPPER_LIBS="-loshmem ${wrapper_finalize_ompi_libs}"
+       OSHMEM_WRAPPER_LIBS_STATIC=
+       OSHMEM_WRAPPER_LDFLAGS_STATIC=
+
+       AS_IF(dnl shared only case.  We add no flags beyond the base -L/-l
+             [test "${enable_shared}" != "no" -a "${enable_static}" != "yes"],
+             [],
+             dnl building both shared and static libraries.  The base
+             dnl case remains the same as the shared-only case (because
+             dnl the app will link against the shared library, but the
+             dnl static case is the full dependency tree.  Our full
+             dnl dependency tree is both the wrapper_extra_libs and
+             dnl wrapper_extra_static_libs, because wrapper_extra_libs
+             dnl was not added to the normal case.
+             [test "${enable_shared}" != "no" -a "${enable_static}" = "yes"],
+             [OPAL_FLAGS_APPEND_UNIQ([OSHMEM_WRAPPER_LDFLAGS_STATIC], [${oshmem_mca_wrapper_extra_ldflags} ${wrapper_extra_ldflags}])
+              OPAL_FLAGS_APPEND_MOVE([OSHMEM_WRAPPER_LIBS_STATIC], [${wrapper_finalize_opal_libs} ${oshmem_mca_wrapper_extra_libs} ${wrapper_extra_libs}])
+              OPAL_FLAGS_APPEND_UNIQ([OSHMEM_WRAPPER_LDFLAGS_STATIC], [${oshmem_mca_wrapper_extra_static_ldflags} ${wrapper_extra_static_ldflags}])
+              OPAL_FLAGS_APPEND_MOVE([OSHMEM_WRAPPER_LIBS_STATIC], [${oshmem_mca_wrapper_extra_static_libs} ${wrapper_extra_static_libs}])],
+             dnl building static only.  The base case is that we need to
+             dnl list our dependencies, but not the full treee, because
+             dnl we assume that our dependencies will be shared libraries
+             dnl (unless they too were built static only, in which case
+             dnl their dependencies will be our direct dependencies if
+             dnl their modules are setup correctly).  The static case is
+             dnl our full dependency tree, but we only need to list the
+             dnl second leve explicitly, because the wrapper coshmemler
+             dnl and/or pkg-config merge use the normal case data in the
+             dnl static case.
+             [OPAL_FLAGS_APPEND_UNIQ([OSHMEM_WRAPPER_LDFLAGS], [${oshmem_mca_wrapper_extra_ldflags} ${wrapper_extra_ldflags}])
+              OPAL_FLAGS_APPEND_MOVE([OSHMEM_WRAPPER_LIBS], [${wrapper_finalize_opal_libs} ${oshmem_mca_wrapper_extra_libs} ${wrapper_extra_libs}])
+              OPAL_FLAGS_APPEND_UNIQ([OSHMEM_WRAPPER_LDFLAGS_STATIC], [${oshmem_mca_wrapper_extra_static_ldflags} ${wrapper_extra_static_ldflags}])
+              OPAL_FLAGS_APPEND_MOVE([OSHMEM_WRAPPER_LIBS_STATIC], [${oshmem_mca_wrapper_extra_static_libs} ${wrapper_extra_static_libs}])])
+
+       dnl Add the user-provided flags
+       OPAL_FLAGS_APPEND_UNIQ([OSHMEM_WRAPPER_LDFLAGS], [${with_wrapper_ldflags}])
+       OPAL_FLAGS_APPEND_MOVE([OSHMEM_WRAPPER_LIBS], [${with_wrapper_libs}])
+
+       dnl fortran FTW!
+       OSHMEM_WRAPPER_FC_LIBS="-loshmem ${OMPI_FORTRAN_MPIFH_LINK}"
+       OPAL_FLAGS_APPEND_UNIQ([OSHMEM_WRAPPER_FC_LIBS], [${OSHMEM_WRAPPER_LIBS}])
+       OSHMEM_WRAPPER_FC_LIBS_STATIC=${OSHMEM_WRAPPER_LIBS_STATIC}
+       OSHMEM_WRAPPER_FC_LDFLAGS=$OSHMEM_WRAPPER_LDFLAGS
+       OSHMEM_WRAPPER_FC_LDFLAGS_STATIC=$OSHMEM_WRAPPER_LDFLAGS
+
+       RPATHIFY_LDFLAGS([OSHMEM_WRAPPER_LDFLAGS])
+       RPATHIFY_LDFLAGS([OSHMEM_WRAPPER_LDFLAGS_STATIC])
+       RPATHIFY_FC_LDFLAGS([OSHMEM_WRAPPER_FC_LDFLAGS])
+       RPATHIFY_FC_LDFLAGS([OSHMEM_WRAPPER_FC_LDFLAGS_STATIC])
+
+       OPAL_FLAGS_APPEND_UNIQ([OSHMEM_WRAPPER_LDFLAGS], [${runpath_args}])
+       OPAL_FLAGS_APPEND_UNIQ([OSHMEM_WRAPPER_FC_LDFLAGS], [${runpath_fc_args}])
+
+       AC_MSG_CHECKING([for OSHMEM wrapper LDFLAGS])
+       AC_SUBST([OSHMEM_WRAPPER_LDFLAGS])
+       AC_MSG_RESULT([$OSHMEM_WRAPPER_LDFLAGS])
+
+       AC_MSG_CHECKING([for OSHMEM wrapper static LDFLAGS])
+       AC_SUBST([OSHMEM_WRAPPER_LDFLAGS_STATIC])
+       AC_MSG_RESULT([$OSHMEM_WRAPPER_LDFLAGS_STATIC])
+
+       AC_MSG_CHECKING([for OSHMEM wrapper LIBS])
+       AC_SUBST([OSHMEM_WRAPPER_LIBS])
+       AC_MSG_RESULT([$OSHMEM_WRAPPER_LIBS])
+
+       AC_MSG_CHECKING([for OSHMEM wrapper static LIBS])
+       AC_SUBST([OSHMEM_WRAPPER_LIBS_STATIC])
+       AC_MSG_RESULT([$OSHMEM_WRAPPER_LIBS_STATIC])
+
+       AC_MSG_CHECKING([for OSHMEM wrapper Fortran LDFLAGS])
+       AC_SUBST([OSHMEM_WRAPPER_FC_LDFLAGS])
+       AC_MSG_RESULT([$OSHMEM_WRAPPER_FC_LDFLAGS])
+
+       AC_MSG_CHECKING([for OSHMEM wrapper Fortran static LDFLAGS])
+       AC_SUBST([OSHMEM_WRAPPER_FC_LDFLAGS_STATIC])
+       AC_MSG_RESULT([$OSHMEM_WRAPPER_FC_LDFLAGS_STATIC])
+
+       AC_MSG_CHECKING([for OSHMEM wrapper Fortran LIBS])
+       AC_SUBST([OSHMEM_WRAPPER_FC_LIBS])
+       AC_MSG_RESULT([$OSHMEM_WRAPPER_FC_LIBS])
+
+       AC_MSG_CHECKING([for OSHMEM wrapper Fortran static LIBS])
+       AC_SUBST([OSHMEM_WRAPPER_FC_LIBS_STATIC])
+       AC_MSG_RESULT([$OSHMEM_WRAPPER_FC_LIBS_STATIC])
+
+       # language binding support.  C++ is a bit different, as the
+       # coshmemler should work even if there is no MPI C++ bindings
+       # support.  However, we do want it to fail if there is no C++
+       # coshmemler.
+       if test "$CXX" = "none"; then
+          OSHMEM_WRAPPER_CXX_REQUIRED_FILE="not supported"
+       else
+          OSHMEM_WRAPPER_CXX_REQUIRED_FILE=""
+       fi
+       AC_SUBST([OSHMEM_WRAPPER_CXX_REQUIRED_FILE])
+
+       if test "$OMPI_TRY_FORTRAN_BINDINGS" -gt "$OMPI_FORTRAN_NO_BINDINGS" ; then
+          OSHMEM_WRAPPER_FORTRAN_REQUIRED_FILE=""
+       else
+          OSHMEM_WRAPPER_FORTRAN_REQUIRED_FILE="not supported"
+       fi
+       AC_SUBST([OSHMEM_WRAPPER_FORTRAN_REQUIRED_FILE])
+
+       AC_DEFINE_UNQUOTED(OSHMEM_WRAPPER_CFLAGS, "$OSHMEM_WRAPPER_CFLAGS",
+           [CFLAGS to pass through the wrapper compilers])
+       AC_DEFINE_UNQUOTED(OSHMEM_WRAPPER_CXXFLAGS, "$OSHMEM_WRAPPER_CXXFLAGS",
+           [CXXFLAGS to pass through the wrapper compilers])
+       AC_DEFINE_UNQUOTED(OSHMEM_WRAPPER_FCFLAGS, "$OSHMEM_WRAPPER__FCFLAGS",
+           [FCFLAGS to pass through the wrapper compilers])
+       AC_DEFINE_UNQUOTED(OSHMEM_WRAPPER_LDFLAGS, "$OSHMEM_WRAPPER_LDFLAGS",
+           [LDFLAGS to pass through the wrapper compilers])
+       AC_DEFINE_UNQUOTED(OSHMEM_WRAPPER_LIBS, "$OSHMEM_WRAPPER_LIBS",
+           [LIBS to pass through the wrapper compilers])
+
+       dnl ####################################################################
+       dnl  Setup variables for pkg-config files
+       dnl
+       dnl Add all our dependent libraries to libs.Private for users that want
+       dnl to static build, unless we're only building static libraries, in
+       dnl which case, add the dependent libraries to libs itself, since any
+       dnl linking will require the full set of libraries.
+       dnl ####################################################################
+       AC_MSG_CHECKING([for OSHMEM pkg-config Cflags])
+       OSHMEM_PC_CFLAGS="${OSHMEM_WRAPPER_CPPFLAGS} ${OSHMEM_WRAPPER_CFLAGS} ${OSHMEM_WRAPPER_CFLAGS_PREFIX}"
+       OSHMEM_PC_CFLAGS=`echo ${OSHMEM_PC_CFLAGS} | sed -e 's/@{/\${/g'`
+       AC_SUBST([OSHMEM_PC_CFLAGS])
+       AC_MSG_RESULT([${OSHMEM_PC_CFLAGS}])
+
+       AC_MSG_CHECKING([for OSHMEM pkg-config Libs])
+       OSHMEM_PC_LIBS="${OSHMEM_WRAPPER_LDFLAGS} ${OSHMEM_WRAPPER_LIBS}"
+       OSHMEM_PC_LIBS=`echo ${OSHMEM_PC_LIBS} | sed -e 's/@{/\${/g'`
+       AC_SUBST([OSHMEM_PC_LIBS])
+       AC_MSG_RESULT([${OSHMEM_PC_LIBS}])
+
+       AC_MSG_CHECKING([for OSHMEM pkg-config Libs.private])
+       OSHMEM_PC_LIBS_PRIVATE="${OSHMEM_WRAPPER_LDFLAGS_STATIC} ${OSHMEM_WRAPPER_LIBS_STATIC}"
+       OSHMEM_PC_LIBS_PRIVATE=`echo ${OSHMEM_PC_LIBS_PRIVATE} | sed -e 's/@{/\${/g'`
+       AC_SUBST([OSHMEM_PC_LIBS_PRIVATE])
+       AC_MSG_RESULT([${OSHMEM_PC_LIBS_PRIVATE}])
+
+       AC_MSG_CHECKING([for OSHMEM pkg-config Fortran Cflags])
+       OSHMEM_PC_CFLAGS="${OSHMEM_WRAPPER_FCFLAGS} ${OSHMEM_WRAPPER_FCFLAGS_PREFIX}"
+       OSHMEM_PC_CFLAGS=`echo ${OSHMEM_PC_CFLAGS} | sed -e 's/@{/\${/g'`
+       AC_SUBST([OSHMEM_PC_FC_CFLAGS])
+       AC_MSG_RESULT([${OSHMEM_PC_FC_CFLAGS}])
+
+       AC_MSG_CHECKING([for OSHMEM pkg-config Fortran Libs])
+       OSHMEM_PC_FC_LIBS="${OSHMEM_WRAPPER_FC_LDFLAGS} ${OSHMEM_WRAPPER_FC_LIBS}"
+       OSHMEM_PC_FC_LIBS=`echo ${OSHMEM_PC_FC_LIBS} | sed -e 's/@{/\${/g'`
+       AC_SUBST([OSHMEM_PC_FC_LIBS])
+       AC_MSG_RESULT([${OSHMEM_PC_FC_LIBS}])
+
+       AC_MSG_CHECKING([for OSHMEM pkg-config Fortran Libs.private])
+       OSHMEM_PC_FC_LIBS_PRIVATE="${OSHMEM_WRAPPER_FC_LDFLAGS_STATIC} ${OSHMEM_WRAPPER_FC_LIBS_STATIC}"
+       OSHMEM_PC_FC_LIBS_PRIVATE=`echo ${OSHMEM_PC_FC_LIBS_PRIVATE} | sed -e 's/@{/\${/g'`
+       AC_SUBST([OSHMEM_PC_FC_LIBS_PRIVATE])
+       AC_MSG_RESULT([${OSHMEM_PC_FC_LIBS_PRIVATE}])
+
+       OSHMEM_PC_MODULES=
+       OSHMEM_PC_MODULES_PRIVATE=
+       AS_IF([test "${enable_shared}" != "no" -a "${enable_static}" != "yes"],
+             [],
+             [test "${enable_shared}" != "no" -a "${enable_static}" = "yes"],
+             [OSHMEM_PC_MODULES_PRIVATE="${wrapper_extra_pkgconfig_modules} ${oshmem_mca_wrapper_extra_pc_modules}"],
+             [OSHMEM_PC_MODULES="${wrapper_extra_pkgconfig_modules} ${oshmem_mca_wrapper_extra_pc_modules}"])
+
+       AC_MSG_CHECKING([for OSHMEM pkg-config Modules])
+       AC_SUBST([OSHMEM_PC_MODULES])
+       AC_MSG_RESULT([${OSHMEM_PC_MODULES}])
+
+       AC_MSG_CHECKING([for OSHMEM pkg-config Modules.private])
+       AC_SUBST([OSHMEM_PC_MODULES_PRIVATE])
+       AC_MSG_RESULT([${OSHMEM_PC_MODULES_PRIVATE}])
+    ])
+
+    OPAL_VAR_SCOPE_POP
 ])
diff --git a/config/opal_subdir_env.m4 b/config/opal_subdir_env.m4
index 805aa5f3d76..c0ad9a4ac15 100644
--- a/config/opal_subdir_env.m4
+++ b/config/opal_subdir_env.m4
@@ -17,7 +17,7 @@ dnl That is, if a precious variable (say CPPFLAGS) was set on the
 dnl configure command line or in the environment at the start of
 dnl configure, it will be reset to that stored value.  Otherwise, it
 dnl will be unset.  The current state is recorded so that it can be
-dnl restored with OPAL_SUBDIR_ENV_RESTORE.  Useful for wrapping calles
+dnl restored with OPAL_SUBDIR_ENV_RESTORE.  Useful for wrapping calls
 dnl to subconfigure scripts.
 AC_DEFUN([OPAL_SUBDIR_ENV_CLEAN], [
     for temp_var in $ac_precious_vars; do
@@ -36,8 +36,8 @@ AC_DEFUN([OPAL_SUBDIR_ENV_CLEAN], [
         fi
         unset tmp_var_set
 
-	# restore the variables that were set at the start of
-	# configure and unset the ones that were not.
+        # restore the variables that were set at the start of
+        # configure and unset the ones that were not.
         eval temp_var_orig_set=\$ac_env_${temp_var}_set
         if test "$temp_var_set" = "set" ; then
              eval ${temp_var}=\$ac_env_${temp_var}_value
diff --git a/config/opal_summary.m4 b/config/opal_summary.m4
index a59ea0818d9..009e2c28e31 100644
--- a/config/opal_summary.m4
+++ b/config/opal_summary.m4
@@ -1,111 +1,96 @@
-dnl -*- shell-script -*-
+dnl -*- autoconf -*-
 dnl
 dnl Copyright (c) 2016      Los Alamos National Security, LLC. All rights
 dnl                         reserved.
 dnl Copyright (c) 2016-2018 Cisco Systems, Inc.  All rights reserved
 dnl Copyright (c) 2016      Research Organization for Information Science
 dnl                         and Technology (RIST). All rights reserved.
+dnl Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+dnl Copyright (c) 2022      IBM Corporation.  All rights reserved.
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
 dnl
 dnl $HEADER$
 dnl
-AC_DEFUN([OPAL_SUMMARY_ADD],[
-    OPAL_VAR_SCOPE_PUSH([ompi_summary_section ompi_summary_line ompi_summary_section_current])
-
-    dnl need to replace spaces in the section name with somethis else. _ seems like a reasonable
-    dnl choice. if this changes remember to change OMPI_PRINT_SUMMARY as well.
-    ompi_summary_section=$(echo $1 | tr ' ' '_')
-    ompi_summary_line="$2: $4"
-    ompi_summary_section_current=$(eval echo \$ompi_summary_values_$ompi_summary_section)
-
-    if test -z "$ompi_summary_section_current" ; then
-        if test -z "$ompi_summary_sections" ; then
-            ompi_summary_sections=$ompi_summary_section
-        else
-            ompi_summary_sections="$ompi_summary_sections $ompi_summary_section"
-        fi
-        eval ompi_summary_values_$ompi_summary_section=\"$ompi_summary_line\"
-    else
-        eval ompi_summary_values_$ompi_summary_section=\"$ompi_summary_section_current,$ompi_summary_line\"
-    fi
 
-    OPAL_VAR_SCOPE_POP
+dnl OPAL_SUMMARY_ADD(section, topic, unused, result)
+dnl
+dnl Turn around and call OAC_SUMMARY_ADD
+dnl -----------------------------------------------------------
+AC_DEFUN([OPAL_SUMMARY_ADD],[
+    OAC_SUMMARY_ADD([$1], [$2], [$4])
 ])
 
+dnl OPAL_SUMMARY_PRINT
+dnl
+dnl Print a bunch of Open MPI summary configuration information and
+dnl then turn around and call OAC_SUMMARY_PRINT.
+dnl -----------------------------------------------------------
 AC_DEFUN([OPAL_SUMMARY_PRINT],[
-    OPAL_VAR_SCOPE_PUSH([ompi_summary_section ompi_summary_section_name])
     cat <<EOF
 
 Open MPI configuration:
 -----------------------
 Version: $OMPI_MAJOR_VERSION.$OMPI_MINOR_VERSION.$OMPI_RELEASE_VERSION$OMPI_GREEK_VERSION
+MPI Standard Version: $MPI_VERSION.$MPI_SUBVERSION
 EOF
 
     if test "$project_ompi_amc" = "true" ; then
-        echo "Build MPI C bindings: yes"
+        echo "Build MPI C bindings: yes" >&AS_MESSAGE_FD
     else
-        echo "Build MPI C bindings: no"
+        echo "Build MPI C bindings: no" >&AS_MESSAGE_FD
     fi
 
     dnl Print out the bindings if we are building OMPI
     if test "$project_ompi_amc" = "true" ; then
-	if test $OMPI_BUILD_FORTRAN_BINDINGS = $OMPI_FORTRAN_MPIFH_BINDINGS ; then
-            echo "Build MPI Fortran bindings: mpif.h"
-	elif test $OMPI_BUILD_FORTRAN_BINDINGS = $OMPI_FORTRAN_USEMPI_BINDINGS ; then
-            echo "Build MPI Fortran bindings: mpif.h, use mpi"
-	elif test $OMPI_BUILD_FORTRAN_BINDINGS = $OMPI_FORTRAN_USEMPIF08_BINDINGS ; then
-	    echo "Build MPI Fortran bindings: mpif.h, use mpi, use mpi_f08"
-	else
-	    echo "Build MPI Fortran bindings: no"
-	fi
+        if test $OMPI_BUILD_FORTRAN_BINDINGS = $OMPI_FORTRAN_MPIFH_BINDINGS ; then
+            echo "Build MPI Fortran bindings: mpif.h" >&AS_MESSAGE_FD
+        elif test $OMPI_BUILD_FORTRAN_BINDINGS = $OMPI_FORTRAN_USEMPI_BINDINGS ; then
+            echo "Build MPI Fortran bindings: mpif.h, use mpi" >&AS_MESSAGE_FD
+        elif test $OMPI_BUILD_FORTRAN_BINDINGS = $OMPI_FORTRAN_USEMPIF08_BINDINGS ; then
+            echo "Build MPI Fortran bindings: mpif.h, use mpi, use mpi_f08" >&AS_MESSAGE_FD
+        else
+            echo "Build MPI Fortran bindings: no" >&AS_MESSAGE_FD
+        fi
 
         if test $WANT_MPI_JAVA_BINDINGS -eq 1 ; then
-            echo "Build MPI Java bindings (experimental): yes"
+            echo "Build MPI Java bindings (experimental): yes" >&AS_MESSAGE_FD
         else
-            echo "Build MPI Java bindings (experimental): no"
+            echo "Build MPI Java bindings (experimental): no" >&AS_MESSAGE_FD
         fi
     fi
 
     if test "$project_oshmem_amc" = "true" ; then
-        echo "Build Open SHMEM support: yes"
+        echo "Build Open SHMEM support: yes" >&AS_MESSAGE_FD
     elif test -z "$project_oshmem_amc" ; then
-        echo "Build Open SHMEM support: no"
+        echo "Build Open SHMEM support: no" >&AS_MESSAGE_FD
     else
-	echo "Build Open SHMEM support: $project_oshmem_amc"
+        echo "Build Open SHMEM support: $project_oshmem_amc" >&AS_MESSAGE_FD
     fi
 
     if test $WANT_DEBUG = 0 ; then
-        echo "Debug build: no"
+        echo "Debug build: no" >&AS_MESSAGE_FD
     else
-        echo "Debug build: yes"
+        echo "Debug build: yes" >&AS_MESSAGE_FD
     fi
 
     if test ! -z $with_platform ; then
-        echo "Platform file: $with_platform"
+        echo "Platform file: $with_platform" >&AS_MESSAGE_FD
     else
-        echo "Platform file: (none)"
+        echo "Platform file: (none)" >&AS_MESSAGE_FD
     fi
 
-    echo
+    echo " " >&AS_MESSAGE_FD
 
-    for ompi_summary_section in $(echo $ompi_summary_sections) ; do
-        ompi_summary_section_name=$(echo $ompi_summary_section | tr '_' ' ')
-        echo "$ompi_summary_section_name"
-        echo "-----------------------"
-        echo "$(eval echo \$ompi_summary_values_$ompi_summary_section)" | tr ',' $'\n' | sort -f
-        echo " "
-    done
+    OAC_SUMMARY_PRINT
 
     if test $WANT_DEBUG = 1 ; then
-        cat <<EOF
+        cat >&AS_MESSAGE_FD <<EOF
 *****************************************************************************
  THIS IS A DEBUG BUILD!  DO NOT USE THIS BUILD FOR PERFORMANCE MEASUREMENTS!
 *****************************************************************************
 
 EOF
     fi
-
-    OPAL_VAR_SCOPE_POP
 ])
diff --git a/config/opal_try_assemble.m4 b/config/opal_try_assemble.m4
deleted file mode 100644
index 9c538d8459d..00000000000
--- a/config/opal_try_assemble.m4
+++ /dev/null
@@ -1,52 +0,0 @@
-dnl
-dnl Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-dnl                         University Research and Technology
-dnl                         Corporation.  All rights reserved.
-dnl Copyright (c) 2004-2005 The University of Tennessee and The University
-dnl                         of Tennessee Research Foundation.  All rights
-dnl                         reserved.
-dnl Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-dnl                         University of Stuttgart.  All rights reserved.
-dnl Copyright (c) 2004-2005 The Regents of the University of California.
-dnl                         All rights reserved.
-dnl Copyright (c) 2014      Intel, Inc. All rights reserved.
-dnl $COPYRIGHT$
-dnl
-dnl Additional copyrights may follow
-dnl
-dnl $HEADER$
-dnl
-
-dnl OPAL_TRY_ASSEMBLE(asm-code, [action-if-success], [action-if-fail])
-dnl
-dnl Attempt to assemble asm-code.  If success, run action-if-success.
-dnl Otherwise, run action-if-fail.  Neither action-if-success nor
-dnl action-if-fail are required.
-dnl
-dnl No preprocessing is guaranteed to be done on asm-code.  Some
-dnl compilers do not run the preprocessor on assembly files.
-dnl
-dnl On failure, asm-test.s will be included in config.out
-AC_DEFUN([OPAL_TRY_ASSEMBLE],
-[cat >conftest.s <<EOF
-[$1]
-EOF
-if test "$CC" = "$CCAS" ; then
-    opal_assemble="$CCAS $CCASFLAGS -c conftest.s >conftest.out 2>&1"
-else
-    opal_assemble="$CCAS $CCASFLAGS -o conftest.o conftest.s >conftest.out 2>&1"
-fi
-if AC_TRY_EVAL(opal_assemble); then
-  # save the warnings
-  cat conftest.out >&AS_MESSAGE_LOG_FD
-  ifelse([$2],,:,[$2])
-else
-  # save compiler output and failed program
-  cat conftest.out >&AS_MESSAGE_LOG_FD
-  echo "configure: failed program was:" >&AS_MESSAGE_LOG_FD
-  cat conftest.s >&AS_MESSAGE_LOG_FD
-  ifelse([$3],,:,[$3])
-fi
-rm -rf conftest*
-unset opal_assemble
-])dnl
diff --git a/config/oshmem_config_files.m4 b/config/oshmem_config_files.m4
index 779757dfdde..00bdaafed2d 100644
--- a/config/oshmem_config_files.m4
+++ b/config/oshmem_config_files.m4
@@ -3,8 +3,10 @@
 # Copyright (c) 2013      Mellanox Technologies, Inc.
 #                         All rights reserved.
 # Copyright (c) 2013      Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2017      Research Organization for Information Science
-#                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2017-2018 Research Organization for Information Science
+#                         and Technology (RIST).  All rights reserved.
+# Copyright (c) 2021      Amazon.com, Inc. or its affiliates.
+#                         All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -17,15 +19,17 @@ AC_DEFUN([OSHMEM_CONFIG_FILES],[
     oshmem/Makefile
     oshmem/include/Makefile
     oshmem/shmem/c/Makefile
-    oshmem/shmem/c/profile/Makefile
 
     oshmem/shmem/fortran/Makefile
-    oshmem/shmem/fortran/profile/Makefile
 
     oshmem/tools/oshmem_info/Makefile
     oshmem/tools/wrappers/Makefile
     oshmem/tools/wrappers/shmemcc-wrapper-data.txt
     oshmem/tools/wrappers/shmemc++-wrapper-data.txt
     oshmem/tools/wrappers/shmemfort-wrapper-data.txt
+    oshmem/tools/wrappers/oshmem.pc
+    oshmem/tools/wrappers/oshmem-c.pc
+    oshmem/tools/wrappers/oshmem-cxx.pc
+    oshmem/tools/wrappers/oshmem-fort.pc
     ])
 ])
diff --git a/config/oshmem_configure_options.m4 b/config/oshmem_configure_options.m4
index 0ceac244b40..0e4b0ea55bc 100644
--- a/config/oshmem_configure_options.m4
+++ b/config/oshmem_configure_options.m4
@@ -16,7 +16,7 @@ dnl $HEADER$
 dnl
 
 AC_DEFUN([OSHMEM_CONFIGURE_OPTIONS],[
-opal_show_subtitle "OSHMEM Configuration options"
+opal_show_subtitle "OpenSHMEM Configuration options"
 
 AC_SUBST(OSHMEM_LIBSHMEM_EXTRA_LIBS)
 AC_SUBST(OSHMEM_LIBSHMEM_EXTRA_LDFLAGS)
@@ -24,26 +24,34 @@ AC_SUBST(OSHMEM_LIBSHMEM_EXTRA_LDFLAGS)
 #
 # Disable Open SHMEM?
 #
-AC_MSG_CHECKING([if want oshmem])
+AC_MSG_CHECKING([if want OpenSHMEM])
 AC_ARG_ENABLE([oshmem],
               [AS_HELP_STRING([--enable-oshmem],
                               [Enable building the OpenSHMEM interface (available on Linux only, where it is enabled by default)])])
+
 if test "$enable_oshmem" = "no"; then
     AC_MSG_RESULT([no])
 elif test "$enable_oshmem" = ""; then
-    if test "$opal_found_linux" = "yes"; then
+    case $host_os in
+    linux*)
         AC_MSG_RESULT([yes])
-    else
+        ;;
+    *)
         enable_oshmem=no
         AC_MSG_RESULT([not supported on this platform])
-    fi
+        ;;
+    esac
 else
     AC_MSG_RESULT([yes])
-    if test "$opal_found_linux" != "yes"; then
+    case $host_os in
+    linux*)
+        ;;
+    *)
         AC_MSG_WARN([OpenSHMEM support was requested, but currently])
         AC_MSG_WARN([only supports Linux.])
         AC_MSG_ERROR([Cannot continue])
-    fi
+        ;;
+    esac
 fi
 
 #
@@ -68,12 +76,12 @@ AC_DEFINE_UNQUOTED([OSHMEM_SPEC_COMPAT], [$OSHMEM_SPEC_COMPAT],
 #
 # Do we want to disable OSHMEM parameter checking at run-time?
 #
-AC_MSG_CHECKING([if want OSHMEM API parameter checking])
+AC_MSG_CHECKING([if want OpenSHMEM API parameter checking])
 AC_ARG_WITH([oshmem-param-check],
     [AS_HELP_STRING([--with-oshmem-param-check(=VALUE)],
                    [behavior of OSHMEM API function parameter checking.  Valid values are: always, never.  If --with-oshmem-param-check is specified with no VALUE argument, it is equivalent to a VALUE of "always"; --without-oshmem-param-check is equivalent to "never" (default: always).])])
 if test "$with_oshmem_param_check" = "no" || \
-	test "$with_oshmem_param_check" = "never"; then
+        test "$with_oshmem_param_check" = "never"; then
     shmem_param_check=0
     AC_MSG_RESULT([never])
 elif test "$with_oshmem_param_check" = "yes" || \
@@ -91,17 +99,12 @@ fi
 AC_DEFINE_UNQUOTED(OSHMEM_PARAM_CHECK, $shmem_param_check,
     [Whether we want to check OSHMEM parameters always or never])
 
-#
-# check for on_exit
-#
-AC_CHECK_FUNCS([on_exit])
-
 #
 #  OSHMEM profiling support
 #
 AC_ARG_ENABLE([oshmem-profile],
     [AS_HELP_STRING([--enable-oshmem-profile],
-                   [enable OSHMEM profiling (default: enabled)])])
+                   [enable OpenSHMEM profiling (default: enabled)])])
 AC_MSG_CHECKING([if want pshmem])
 AS_IF([test "$enable_oshmem_profile" != "no"],
       [AC_MSG_RESULT([yes])],
@@ -120,17 +123,17 @@ AS_IF([test "$enable_oshmem" = "no" && \
 #
 # Fortran bindings
 #
-AC_MSG_CHECKING([if want to build OSHMEM fortran bindings])
+AC_MSG_CHECKING([if want to build OpenSHMEM fortran bindings])
 AC_ARG_ENABLE([oshmem-fortran],
 AS_HELP_STRING([--enable-oshmem-fortran],
-               [enable OSHMEM Fortran bindings (default: enabled if Fortran compiler found)]))
+               [enable OpenSHMEM Fortran bindings (default: enabled if Fortran compiler found)]))
 if test "$enable_oshmem_fortran" != "no"; then
 # If no OMPI FORTRAN, bail
    AS_IF([test $OMPI_TRY_FORTRAN_BINDINGS -eq $OMPI_FORTRAN_NO_BINDINGS && \
           test "$enable_oshmem_fortran" = "yes"],
                [AC_MSG_RESULT([bad value OMPI_TRY_FORTRAN_BINDINGS: ($OMPI_TRY_FORTRAN_BINDINGS)])
                 AC_MSG_WARN([Your request to --enable-oshmem-fortran can only be satisfied if fortran support is enabled in OMPI.
-You see this message because OMPI fortran support has been explicitly disabled via --disable-mpi-fortran and OSHMEM fortran support was explicitly enabled with --enable-oshmem-fortran.
+You see this message because MPI fortran support has been explicitly disabled via --disable-mpi-fortran and OpenSHMEM fortran support was explicitly enabled with --enable-oshmem-fortran.
 Configure will abort because you, a human, have asked for something that cannot be provided.])
                 AC_MSG_ERROR([Cannot continue])])
     if test $OMPI_TRY_FORTRAN_BINDINGS -gt $OMPI_FORTRAN_NO_BINDINGS; then
diff --git a/configure.ac b/configure.ac
index bc44f05df63..8dea10cb32a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -10,7 +10,7 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2006-2020 Cisco Systems, Inc.  All rights reserved
+# Copyright (c) 2006-2023 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2006-2008 Sun Microsystems, Inc.  All rights reserved.
 # Copyright (c) 2006-2017 Los Alamos National Security, LLC.  All rights
 #                         reserved.
@@ -20,16 +20,16 @@
 # Copyright (c) 2013      Mellanox Technologies, Inc.
 #                         All rights reserved.
 # Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
-# Copyright (c) 2014-2017 Research Organization for Information Science
-#                         and Technology (RIST). All rights reserved.
-# Copyright (c) 2016-2017 IBM Corporation.  All rights reserved.
-# Copyright (c) 2018      Amazon.com, Inc. or its affiliates.
+# Copyright (c) 2014-2022 Research Organization for Information Science
+#                         and Technology (RIST).  All rights reserved.
+# Copyright (c) 2016-2022 IBM Corporation.  All rights reserved.
+# Copyright (c) 2018-2021 Amazon.com, Inc. or its affiliates.
 #                         All Rights reserved.
 # Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
 # Copyright (c) 2019      Triad National Security, LLC. All rights
 #                         reserved.
-# Copyright (c) 2020      Amazon.com, Inc. or its affiliates.
-#                         All Rights reserved.
+# Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
+# Copyright (c) 2025      Nanook Consulting  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -62,12 +62,17 @@ AC_PREREQ(2.60)
 AC_CONFIG_AUX_DIR(config)
 AC_CONFIG_MACRO_DIR(config)
 
+OAC_PUSH_PREFIX([OPAL])
+
 OPAL_CAPTURE_CONFIGURE_CLI([OPAL_CONFIGURE_CLI])
 
 # Get our platform support file.  This has to be done very, very early
 # because it twiddles random bits of autoconf
 OPAL_LOAD_PLATFORM
 
+# Start a list of packages / modules / etc. that want to disable "make dist".
+OPAL_MAKEDIST_DISABLE=""
+
 #
 # Start it up
 #
@@ -76,6 +81,8 @@ OPAL_CONFIGURE_SETUP
 opal_show_title "Configuring project_name_long"
 opal_show_subtitle "Prerequisites"
 
+AC_PROG_SED
+
 AC_CHECK_PROG([PERL],[perl],[perl],[no])
 AS_IF([test "X$PERL" = "Xno"],
       [AC_MSG_ERROR(["Open MPI requires perl. Aborting"])])
@@ -93,20 +100,11 @@ AS_IF([test "$host" != "$target"],
        AC_MSG_WARN([Cross-compiling is only partially supported])
        AC_MSG_WARN([Proceed at your own risk!])])
 
-# AC_USE_SYSTEM_EXTENSIONS alters CFLAGS (e.g., adds -g -O2)
-OPAL_VAR_SCOPE_PUSH([CFLAGS_save])
-CFLAGS_save=$CFLAGS
-AC_USE_SYSTEM_EXTENSIONS
-# AC_USE_SYSTEM_EXTENSIONS will modify CFLAGS if nothing was in there
-# beforehand.  We don't want that.  So if there was nothing in
-# CFLAGS, put nothing back in there.
-AS_IF([test -z "$CFLAGS_save"], [CFLAGS=])
-OPAL_VAR_SCOPE_POP
 
 #
 # Init automake
 #
-AM_INIT_AUTOMAKE([foreign dist-bzip2 subdir-objects no-define 1.13.4 tar-ustar])
+AM_INIT_AUTOMAKE([foreign dist-bzip2 subdir-objects no-define 1.13.4 tar-pax])
 
 # SILENT_RULES is new in AM 1.11, but we require 1.11 or higher via
 # autogen.  Limited testing shows that calling SILENT_RULES directly
@@ -131,29 +129,10 @@ AC_DEFUN([OMPI_CHECK_DIR_FOR_SPACES],[
            AC_MSG_ERROR([Cannot continue.])])
 ])
 
-AC_DEFUN([OMPI_CANONICALIZE_PATH],[
-    case $host_os in
-    darwin*)
-        # MacOS does not have "readlink -f" or realpath (at least as
-        # of MacOS Cataline / 10.15).  Instead, use Python, because we
-        # know MacOS comes with a /usr/bin/python that has
-        # os.path.realpath.
-        $2=`/usr/bin/python -c 'import os; print os.path.realpath("'$1'")'`
-        ;;
-    *)
-        $2=`readlink -f $1`
-        ;;
-    esac
-])
-
-OMPI_CHECK_DIR_FOR_SPACES([$srcdir], [a], [source tree])
-OMPI_CANONICALIZE_PATH([$srcdir], [ompi_dir])
-OMPI_CHECK_DIR_FOR_SPACES([$ompi_dir], [an], [absolute source tree])
-OMPI_CANONICALIZE_PATH([.], [ompi_dir])
+ompi_dir=`pwd`
 OMPI_CHECK_DIR_FOR_SPACES([$ompi_dir], [a], [build tree])
+OMPI_CHECK_DIR_FOR_SPACES([$srcdir], [a], [source tree])
 OMPI_CHECK_DIR_FOR_SPACES([$prefix], [a], [prefix])
-OMPI_CANONICALIZE_PATH([$prefix], [ompi_dir])
-OMPI_CHECK_DIR_FOR_SPACES([$ompi_dir], [an], [absolute prefix])
 
 opal_show_subtitle "Checking versions"
 
@@ -161,7 +140,8 @@ opal_show_subtitle "Checking versions"
 
 m4_ifdef([project_ompi],
          [OPAL_SAVE_VERSION([OMPI], [Open MPI], [$srcdir/VERSION],
-                            [ompi/include/ompi/version.h])])
+                            [ompi/include/ompi/version.h])
+          OPAL_SAVE_MPI_STANDARD_VERSION([$srcdir/VERSION])])
 
 m4_ifdef([project_oshmem],
          [OPAL_SAVE_VERSION([OSHMEM], [Open SHMEM],
@@ -193,7 +173,6 @@ AC_SUBST(libopen_pal_so_version)
 AC_SUBST(libmca_opal_common_ofi_so_version)
 AC_SUBST(libmca_opal_common_cuda_so_version)
 AC_SUBST(libmca_opal_common_sm_so_version)
-AC_SUBST(libmca_opal_common_ugni_so_version)
 AC_SUBST(libmca_ompi_common_ompio_so_version)
 AC_SUBST(libmca_ompi_common_monitoring_so_version)
 AC_SUBST(libmca_opal_common_ucx_so_version)
@@ -207,10 +186,16 @@ acversion=`grep "Generated by GNU Autoconf" $0 | head -n 1 | awk '{ print $6 }'`
 AC_MSG_RESULT([$acversion])
 AC_MSG_CHECKING([for bootstrap Automake version])
 AC_MSG_RESULT([$am__api_version])
-AC_MSG_CHECKING([for boostrap Libtool version])
+AC_MSG_CHECKING([for bootstrap Libtool version])
 ltversion=`grep VERSION= $srcdir/config/ltmain.sh | head -n 1 | cut -d= -f2`
 AC_MSG_RESULT([$ltversion])
 
+# Ensure mpi.h.in and mca_base_event.h have consistent callback safety values
+AC_DEFINE_UNQUOTED([OPAL_MCA_BASE_CB_REQUIRE_NONE], [0], [Keeping OPAL and OMPI values in sync])
+AC_DEFINE_UNQUOTED([OPAL_MCA_BASE_CB_REQUIRE_MPI_RESTRICTED], [1], [Keeping OPAL and OMPI values in sync])
+AC_DEFINE_UNQUOTED([OPAL_MCA_BASE_CB_REQUIRE_THREAD_SAFE], [2], [Keeping OPAL and OMPI values in sync])
+AC_DEFINE_UNQUOTED([OPAL_MCA_BASE_CB_REQUIRE_ASYNC_SIGNAL_SAFE], [3], [Keeping OPAL and OMPI values in sync])
+
 # List header files to generate
 
 AC_CONFIG_HEADERS([opal/include/opal_config.h])
@@ -290,20 +275,13 @@ m4_ifdef([project_oshmem],
 ############################################################################
 
 OPAL_CONFIGURE_OPTIONS
-OPAL_CHECK_OS_FLAVORS
-OPAL_CHECK_CUDA
 
+###########################
+# Open MPI options
 m4_ifdef([project_ompi], [OMPI_CONFIGURE_OPTIONS])
-m4_ifdef([project_oshmem], [OSHMEM_CONFIGURE_OPTIONS])
-
-# Set up project specific AM_CONDITIONALs
-AS_IF([test "$enable_ompi" != "no"], [project_ompi_amc=true], [project_ompi_amc=false])
-m4_ifndef([project_ompi], [project_ompi_amc=false])
-
-AS_IF([test "$enable_oshmem" != "no"], [project_oshmem_amc=true], [project_oshmem_amc="no (disabled)"])
-m4_ifndef([project_oshmem], [project_oshmem_amc="no (not available)"])
 
 # Enable/Disable Software-Based Performance Counters Capability
+AC_MSG_CHECKING([if want software-based performance counters (SPC)])
 AC_ARG_ENABLE([spc],
     [AS_HELP_STRING([--enable-spc],
                    [Enable software-based performance counters capability (default: disabled)])])
@@ -328,9 +306,22 @@ fi
 
 # The library prefixes must be set before we call OPAL MCA.  Here is
 # as good a place as any.
-OPAL_SET_LIB_PREFIX([])
+m4_ifdef([project_opal],
+         [OPAL_SET_LIB_NAME([open-pal])])
+
 m4_ifdef([project_ompi],
-          [OMPI_SET_LIB_NAME([])])
+         [OMPI_SET_LIB_NAME([])])
+
+###########################
+# OSHMEM options
+m4_ifdef([project_oshmem], [OSHMEM_CONFIGURE_OPTIONS])
+
+# Set up project specific AM_CONDITIONALs
+AS_IF([test "$enable_ompi" != "no"], [project_ompi_amc=true], [project_ompi_amc=false])
+m4_ifndef([project_ompi], [project_ompi_amc=false])
+
+AS_IF([test "$enable_oshmem" != "no"], [project_oshmem_amc=true], [project_oshmem_amc="no (disabled)"])
+m4_ifndef([project_oshmem], [project_oshmem_amc="no (not available)"])
 
 ############################################################################
 # Libtool: part one
@@ -344,10 +335,6 @@ m4_ifdef([project_ompi],
 AC_ENABLE_SHARED
 AC_DISABLE_STATIC
 
-# Must be called before OPAL_SETUP_CC to get the value of CC
-# before it is modified by the C99/C11 checks.
-OPAL_SETUP_WRAPPER_INIT
-
 ##################################
 # Check for known incompatibility
 ##################################
@@ -384,18 +371,9 @@ opal_show_title "Compiler and preprocessor tests"
 
 OPAL_SETUP_CC
 
-# If we build on a windows environment with the windows compiler and linker
-# then we need some translation functions from the opal/win32 directory.
-AM_CONDITIONAL(OMPI_NEED_WINDOWS_REPLACEMENTS,
-               test "$opal_cv_c_compiler_vendor" = "microsoft" )
-
 # Do all Interix detections if necessary
 OMPI_INTERIX
 
-# Does the compiler support "ident"-like constructs?
-
-OPAL_CHECK_IDENT([CC], [CFLAGS], [c], [C])
-
 #
 # Check for some types
 #
@@ -482,6 +460,10 @@ AC_CHECK_SIZEOF(double _Complex)
 AC_CHECK_SIZEOF(long double _Complex)
 
 AC_CHECK_SIZEOF(void *)
+AS_IF([test "$ac_cv_sizeof_void_p" -eq 4],
+      [AC_MSG_WARN([Open MPI no longer supports 32 bit builds.])
+       AC_MSG_WARN([Please use Open MPI v4.x or earlier if you need 32 bit support.])
+       AC_MSG_ERROR([Cannot continue])])
 AC_CHECK_SIZEOF(size_t)
 if test "$ac_cv_type_ssize_t" = yes ; then
     AC_CHECK_SIZEOF(ssize_t)
@@ -548,6 +530,29 @@ OPAL_C_GET_ALIGNMENT(size_t, OPAL_ALIGNMENT_SIZE_T)
 
 OPAL_CHECK_ALT_SHORT_FLOAT
 
+# Check system alignment requirements
+if test "$opal_want_heterogeneous" = 1; then
+    ompi_cv_c_word_size_align=yes
+else
+    AC_CACHE_CHECK([if word-sized integers must be word-size aligned],
+        [ompi_cv_c_word_size_align],
+        [AC_LANG_PUSH(C)
+         AC_RUN_IFELSE([AC_LANG_PROGRAM([dnl
+#include <stdlib.h>], [[    long data[2] = {0, 0};
+    long *lp;
+    int *ip;
+    ip = (int*) data;
+    ip++;
+    lp = (long*) ip;
+    return lp[0]; ]])],
+            [ompi_cv_c_word_size_align=no],
+            [ompi_cv_c_word_size_align=yes],
+            [ompi_cv_c_word_size_align=yes])])
+fi
+AS_IF([test $ompi_cv_c_word_size_align = yes], [results=1], [results=0])
+AC_DEFINE_UNQUOTED([OPAL_ALIGN_WORD_SIZE_INTEGERS], [$results],
+    [set to 1 if word-size integers must be aligned to word-size padding to prevent bus errors])
+
 #
 # Check for other compiler characteristics
 #
@@ -555,10 +560,6 @@ OPAL_CHECK_ALT_SHORT_FLOAT
 OPAL_C_WEAK_SYMBOLS
 OPAL_C_MACRO_WEAK_SYMBOLS
 
-if test "x$CC" = "xicc"; then
-    OPAL_CHECK_ICC_VARARGS
-fi
-
 # If we want the profiling layer:
 # - If the C compiler has weak symbols, use those.
 # - If not, then set to compile the code again with #define's in a
@@ -598,6 +599,22 @@ opal_show_subtitle "Compiler characteristics"
 OPAL_CHECK_ATTRIBUTES
 OPAL_CHECK_COMPILER_VERSION_ID
 
+# Open MPI only supports GCC >=v4.8.1.  Notes:
+#
+# 1. The default compiler that comes with RHEL 7 is v4.8.5 (version ID
+#    264197).
+# 2. We regularly test with GCC v4.8.1 (version ID 264193).
+# 3. GCC 4.8.0 probably also works; we just haven't tested it.
+#
+# Since we regularly test with 4.8.1, that's what we check for.
+AS_IF([test "$opal_cv_compiler_FAMILYNAME" = "GNU" && \
+       test "$opal_cv_compiler_VERSION" -lt 264193],
+      [AC_MSG_WARN([Open MPI no longer supports versions of the GNU compiler suite])
+       AC_MSG_WARN([less than v4.8.1.])
+       AC_MSG_WARN([Please upgrade your GNU compiler suite, or use])
+       AC_MSG_WARN([a different compiler to build Open MPI.])
+       AC_MSG_ERROR([Cannot continue])
+      ])
 
 ##################################
 # Java MPI Binding request
@@ -649,6 +666,14 @@ AM_CONDITIONAL(OSHMEM_BUILD_FORTRAN_BINDINGS,
 AC_CACHE_SAVE
 
 
+##################################
+# Wrapper compilers.
+#
+# Must be called before MCA system
+##################################
+OPAL_SETUP_WRAPPER_INIT
+
+
 ##################################
 # Header files
 ##################################
@@ -663,11 +688,11 @@ AC_CHECK_HEADERS([alloca.h aio.h arpa/inet.h dirent.h \
     sys/fcntl.h sys/ipc.h sys/shm.h \
     sys/ioctl.h sys/mman.h sys/param.h sys/queue.h \
     sys/resource.h sys/select.h sys/socket.h sys/sockio.h \
-    sys/stat.h sys/statfs.h sys/statvfs.h sys/time.h sys/tree.h \
+    sys/stat.h sys/statfs.h sys/statvfs.h time.h sys/time.h sys/tree.h \
     sys/types.h sys/uio.h sys/un.h net/uio.h sys/utsname.h sys/vfs.h sys/wait.h syslog.h \
     termios.h ulimit.h unistd.h util.h utmp.h malloc.h \
     ifaddrs.h crt_externs.h regex.h mntent.h paths.h \
-    ioLib.h sockLib.h hostLib.h shlwapi.h sys/synch.h db.h ndbm.h ieee754.h])
+    ioLib.h sockLib.h hostLib.h shlwapi.h sys/synch.h db.h ndbm.h ieee754.h syslimits.h])
 
 AC_CHECK_HEADERS([sys/mount.h], [], [],
 [AC_INCLUDES_DEFAULT
@@ -852,6 +877,15 @@ AC_INCLUDES_DEFAULT
 #endif
 ])
 
+AC_CHECK_MEMBERS([struct timespec.tv_nsec],
+               [], [], [AC_INCLUDES_DEFAULT
+#ifdef HAVE_TIME_H
+#include <time.h>
+#endif
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif])
+
 #
 # Find corresponding types for MPI_Aint, MPI_Count, and MPI_Offset.
 # And if relevant, find the corresponding MPI_ADDRESS_KIND,
@@ -870,7 +904,8 @@ AC_MSG_CHECKING([the linker for support for the -fini option])
 OPAL_VAR_SCOPE_PUSH([LDFLAGS_save])
 LDFLAGS_save=$LDFLAGS
 LDFLAGS="$LDFLAGS_save -Wl,-fini -Wl,finalize"
-AC_TRY_LINK([void finalize (void) {}], [], [AC_MSG_RESULT([yes])
+AC_LINK_IFELSE([AC_LANG_PROGRAM([void finalize (void) {}], [])],
+               [AC_MSG_RESULT([yes])
         opal_ld_have_fini=1], [AC_MSG_RESULT([no])
         opal_ld_have_fini=0])
 LDFLAGS=$LDFLAGS_save
@@ -901,7 +936,7 @@ OPAL_SEARCH_LIBS_CORE([ceil], [m])
 # -lrt might be needed for clock_gettime
 OPAL_SEARCH_LIBS_CORE([clock_gettime], [rt])
 
-AC_CHECK_FUNCS([asprintf snprintf vasprintf vsnprintf openpty isatty getpwuid fork waitpid execve pipe ptsname setsid mmap tcgetpgrp posix_memalign strsignal sysconf syslog vsyslog regcmp regexec regfree _NSGetEnviron socketpair usleep mkfifo dbopen dbm_open statfs statvfs setpgid setenv __malloc_initialize_hook __clear_cache])
+AC_CHECK_FUNCS([asprintf snprintf vasprintf vsnprintf openpty isatty getpwuid fork waitpid execve pipe ptsname setsid mmap tcgetpgrp posix_memalign strsignal sysconf syslog vsyslog regcmp regexec regfree _NSGetEnviron socketpair usleep mkfifo dbopen dbm_open statfs statvfs setpgid setenv __malloc_initialize_hook __clear_cache on_exit])
 
 # Sanity check: ensure that we got at least one of statfs or statvfs.
 if test $ac_cv_func_statfs = no && test $ac_cv_func_statvfs = no; then
@@ -960,6 +995,11 @@ AC_CACHE_SAVE
 
 opal_show_title "System-specific tests"
 
+##################################
+OAC_CHECK_OS_FLAVORS
+
+
+
 # Do we have _SC_NPROCESSORS_ONLN? (only going to pass if we also have
 # <unistd.h> and sysconf(), which is ok) OS X 10.4 has <unistd.h> and
 # sysconf(), but does not have _SC_NPROCESSORS_ONLN.  Doh!
@@ -1001,10 +1041,14 @@ AC_PROG_GREP
 AC_PROG_EGREP
 
 #
-# We need as and lex
+# We need as and flex
 #
 AM_PROG_AS
-AM_PROG_LEX
+
+dnl Note that prior to AC v2.70, PROG_LEX did not take any arguments.
+dnl But it is harmless to pass an argument to it ($1 will just be
+dnl ignored).
+AC_PROG_LEX([noyywrap])
 
 # If we don't have Flex and we don't have a generated .c file
 # (distribution tarballs will have the .c file included, but git
@@ -1012,24 +1056,27 @@ AM_PROG_LEX
 # Lex are not workable (all things being equal, since this is *only*
 # required for developers, we decided that it really was not worth it
 # to be portable between different versions of lex ;-).
-
-if test -z "$LEX" || \
+AS_IF([test -z "$LEX" || \
    test -n "`echo $LEX | $GREP missing`" || \
-   test "`basename $LEX`" != "flex"; then
-    if test ! -f "$srcdir/opal/util/show_help_lex.c"; then
-        AC_MSG_WARN([*** Could not find Flex on your system.])
+   test "`basename $LEX`" != "flex"],
+    [AS_IF([test ! -f "$srcdir/opal/util/show_help_lex.c"],
+       [AC_MSG_WARN([*** Could not find Flex on your system.])
         AC_MSG_WARN([*** Flex is required for developer builds of Open MPI.])
         AC_MSG_WARN([*** Other versions of Lex are not supported.])
         AC_MSG_WARN([*** NOTE: If you are building a tarball downloaded from www.open-mpi.org, you do not need Flex])
         AC_MSG_ERROR([Cannot continue])
-    fi
-fi
+       ])
+    ])
 
 #
-# Setup man page processing
+# Setup HTML and man page processing
 #
 
-OPAL_SETUP_MAN_PAGES
+dnl Note that we have to double escape the URL below
+dnl so that the # it contains doesn't confuse the Autotools
+OAC_SETUP_SPHINX([$srcdir/docs/man/MPI_T.3],
+                 [[https://docs.open-mpi.org/en/main/developers/prerequisites.html#sphinx-and-therefore-python]],
+                 [$srcdir/docs/requirements.txt])
 
 #
 # File system case sensitivity
@@ -1187,18 +1234,6 @@ fi
 # checkpoint results
 AC_CACHE_SAVE
 
-##################################
-# CUDA: part two
-##################################
-
-# This is somewhat gross to have a configure check for a common MCA
-# component outside of the normal MCA checks, but this check must come
-# after the opal DL MCA checks have done.  Someday this could perhaps
-# be done better by having some kind of "run this check at the end of
-# all other MCA checks" hook...?
-
-OPAL_CHECK_CUDA_AFTER_OPAL_DL
-
 ##################################
 # MPI Extended Interfaces
 ##################################
@@ -1208,15 +1243,6 @@ m4_ifdef([project_ompi], [OMPI_SETUP_MPI_EXT])
 # checkpoint results
 AC_CACHE_SAVE
 
-##################################
-# Contributed software
-##################################
-
-m4_ifdef([project_ompi], [OMPI_SETUP_CONTRIB])
-
-# checkpoint results
-AC_CACHE_SAVE
-
 ##################################
 # Visibility
 ##################################
@@ -1242,12 +1268,6 @@ opal_show_title "Final top-level OMPI configuration"
 
 opal_show_subtitle "Libtool configuration"
 
-# Use the undocumented solaris_use_stlport4 libtool variable to turn off any
-# Cstd/stlport4 linkage. This allows Open MPI to be C++ STL agnostic.
-if test "x$opal_cv_c_compiler_vendor" = "xsun"; then
-    solaris_use_stlport4="yes"
-fi
-
 # Due to this thread:
 # https://www.open-mpi.org/community/lists/users/2013/02/21356.php and
 # a helpful tip from Dave Goodell, set the precious variables for
@@ -1366,10 +1386,13 @@ OPAL_SETUP_WRAPPER_FINAL
 # autoconf macro defines in mpi.h.  Since AC sometimes changes whether
 # things are defined as null tokens or an integer result, two projects
 # with different versions of AC can cause problems.
-if test $ac_cv_header_stdc = yes; then
-    AC_DEFINE(OPAL_STDC_HEADERS, 1,
-              [Do not use outside of mpi.h.  Define to 1 if you have the ANSI C header files.])
-fi
+
+# According to the autoconf 2.67 documentation the AC_HEADER_STDC macro,
+# and therefore the ac_cv_header_stdc cache variable, is obsolescent, as
+# current systems have conforming header files. Instead of removing the
+# protection completely, let's just make sure it is always on.
+AC_DEFINE(OPAL_STDC_HEADERS, 1,
+          [Do not use outside of mpi.h.  Define to 1 if you have the ANSI C header files.])
 if test $ac_cv_header_sys_time_h = yes ; then
     AC_DEFINE(OPAL_HAVE_SYS_TIME_H, 1,
               [Do not use outside of mpi.h.  Define to 1 if you have the <sys/time.h> header file.])
@@ -1383,6 +1406,7 @@ if test $ac_cv_header_sys_synch_h = yes ; then
               [Do not use outside of mpi.h.  Define to 1 if you have the <sys/synch.h> header file.])
 fi
 
+
 # If there is a local hook for each project, call it.  This allows 3rd
 # parties to add configuration steps to OPAL and/or OMPI simply
 # by placing a file in [opal|ompi]/config/whatever.m4 that
@@ -1472,6 +1496,8 @@ AC_MSG_RESULT([$LIBS])
 AC_CONFIG_FILES([
     Makefile
 
+    docs/Makefile
+
     config/Makefile
 
     contrib/Makefile
@@ -1500,6 +1526,10 @@ AC_CONFIG_FILES([contrib/dist/mofed/debian/rules],
 AC_CONFIG_FILES([contrib/dist/mofed/compile_debian_mlnx_example],
                 [chmod +x contrib/dist/mofed/compile_debian_mlnx_example])
 
+AS_IF([test -n "$OPAL_MAKEDIST_DISABLE"],
+      [AC_MSG_WARN(["make dist" will be disabled due to: $OPAL_MAKEDIST_DISABLE])])
+AC_SUBST([OPAL_MAKEDIST_DISABLE])
+
 OPAL_CONFIG_FILES
 m4_ifdef([project_ompi], [OMPI_CONFIG_FILES])
 m4_ifdef([project_oshmem], [OSHMEM_CONFIG_FILES])
diff --git a/contrib/amca-param-sets/ft-mpi b/contrib/amca-param-sets/ft-mpi
index 82382529f5e..45eac4c39ee 100644
--- a/contrib/amca-param-sets/ft-mpi
+++ b/contrib/amca-param-sets/ft-mpi
@@ -39,7 +39,7 @@ btl_base_warn_peer_error=false
 
 #
 # The following frameworks/components are TESTED
-# They handle faults amd should be prefered when running with FT.
+# They handle faults amd should be preferred when running with FT.
 #   pml     ob1
 #   btl     tcp, self, sm(+xpmem,+cma), ugni, uct
 #   coll    base/basic, tuned, ftagree, libnbc
@@ -49,7 +49,7 @@ threads=pthreads
 #
 # The following frameworks/components are UNTESTED, but **may** work.
 # They should run without faults, and **may** work with faults.
-# You may try and report if successfull.
+# You may try and report if successful.
 #   btl     ofi, portals4, smcuda, usnic, sm(+knem)
 #   coll    inter, sm, sync, cuda, monitoring
 #   pml     monitoring, v/vprotocol
@@ -79,9 +79,9 @@ coll=^hcoll,portals4
 #   mtl     (all)
 #   pml     cm, crcpw, ucx
 mtl=^ofi,portals4,psm2
-# allready enforced by pml=ob1 above
+# already enforced by pml=ob1 above
 #pml=^cm,crcpw,ucx
-# allready enforced by threads=pthreads above
+# already enforced by threads=pthreads above
 #threads=^argobots,qthreads
 # There is a bug in libevent with the "select" backend that causes an infinite loop
 # when an unplanned disconnect happens. Use something else, or bail.
diff --git a/contrib/annual-maintenance/convert-to-git.txt b/contrib/annual-maintenance/convert-to-git.txt
index 5f5afd259b2..fde32ad9205 100644
--- a/contrib/annual-maintenance/convert-to-git.txt
+++ b/contrib/annual-maintenance/convert-to-git.txt
@@ -3,7 +3,7 @@ From Dave:
 For fun, here's the Git version of the "AUTHORS with commits in the past year" part of your script:
 
 ----8<----
-savbu-usnic-a ~/g/ompi-svn-mirror git:master ❮❮❮ git log --all --since='1 year ago' --pretty=tformat:'%ae' | sort | uniq -c
+savbu-usnic-a ~/g/ompi-svn-mirror git:main ❮❮❮ git log --all --since='1 year ago' --pretty=tformat:'%ae' | sort | uniq -c
     39 adrian@open-mpi-git-mirror.example.com
      3 alex@open-mpi-git-mirror.example.com
      5 alinas@open-mpi-git-mirror.example.com
@@ -40,7 +40,7 @@ savbu-usnic-a ~/g/ompi-svn-mirror git:master ❮❮❮ git log --all --since='1
 
 And the "NO commits":
 ----8<----
-savbu-usnic-a ~/g/ompi-svn-mirror git:master ❯❯❯ git log --all --since='1 year ago' --pretty=tformat:'%ae' | sort | uniq > /tmp/active ; git log --pretty=tformat:'%ae' --all | sort | uniq > /tmp/all ; diff -u
+savbu-usnic-a ~/g/ompi-svn-mirror git:main ❯❯❯ git log --all --since='1 year ago' --pretty=tformat:'%ae' | sort | uniq > /tmp/active ; git log --pretty=tformat:'%ae' --all | sort | uniq > /tmp/all ; diff -u
 /tmp/all /tmp/active | grep '^-[^-]'
 -abbyz@open-mpi-git-mirror.example.com
 -adi@open-mpi-git-mirror.example.com
diff --git a/contrib/build-mca-comps-outside-of-tree/btl_tcp2.c b/contrib/build-mca-comps-outside-of-tree/btl_tcp2.c
index fe612bd8fe4..ddd7ba1a9e9 100644
--- a/contrib/build-mca-comps-outside-of-tree/btl_tcp2.c
+++ b/contrib/build-mca-comps-outside-of-tree/btl_tcp2.c
@@ -309,7 +309,7 @@ mca_btl_base_descriptor_t* mca_btl_tcp2_prepare_src(
 
 /**
  * Prepare a descriptor for send/rdma using the supplied
- * convertor. If the convertor references data that is contigous,
+ * convertor. If the convertor references data that is contiguous,
  * the descriptor may simply point to the user buffer. Otherwise,
  * this routine is responsible for allocating buffer space and
  * packing if required.
@@ -359,7 +359,7 @@ mca_btl_base_descriptor_t* mca_btl_tcp2_prepare_dst(
  *
  * @param btl (IN)         BTL module
  * @param endpoint (IN)    BTL addressing information
- * @param descriptor (IN)  Description of the data to be transfered
+ * @param descriptor (IN)  Description of the data to be transferred
  * @param tag (IN)         The tag value used to notify the peer.
  */
 
diff --git a/contrib/build-mca-comps-outside-of-tree/btl_tcp2.h b/contrib/build-mca-comps-outside-of-tree/btl_tcp2.h
index 9ff97dc0ea3..d87b9c0b2a7 100644
--- a/contrib/build-mca-comps-outside-of-tree/btl_tcp2.h
+++ b/contrib/build-mca-comps-outside-of-tree/btl_tcp2.h
@@ -98,7 +98,7 @@ struct mca_btl_tcp2_component_t {
 };
 typedef struct mca_btl_tcp2_component_t mca_btl_tcp2_component_t;
 
-OMPI_MODULE_DECLSPEC extern mca_btl_tcp2_component_t mca_btl_tcp2_component;
+OMPI_DECLSPEC extern mca_btl_tcp2_component_t mca_btl_tcp2_component;
 
 /**
  * BTL Module Interface
diff --git a/contrib/build-mca-comps-outside-of-tree/btl_tcp2_component.c b/contrib/build-mca-comps-outside-of-tree/btl_tcp2_component.c
index 72582a8e6de..b4bc0d00357 100644
--- a/contrib/build-mca-comps-outside-of-tree/btl_tcp2_component.c
+++ b/contrib/build-mca-comps-outside-of-tree/btl_tcp2_component.c
@@ -740,7 +740,7 @@ static int mca_btl_tcp2_component_create_listen(uint16_t af_family)
         return OMPI_ERROR;
     }
   socket_binded:
-    /* resolve system assignend port */
+    /* resolve system assigned port */
     if(getsockname(sd, (struct sockaddr*)&inaddr, &addrlen) < 0) {
         BTL_ERROR(("getsockname() failed: %s (%d)",
                    strerror(opal_socket_errno), opal_socket_errno));
diff --git a/contrib/build-mca-comps-outside-of-tree/btl_tcp2_endpoint.c b/contrib/build-mca-comps-outside-of-tree/btl_tcp2_endpoint.c
index a1998de71af..385f645d062 100644
--- a/contrib/build-mca-comps-outside-of-tree/btl_tcp2_endpoint.c
+++ b/contrib/build-mca-comps-outside-of-tree/btl_tcp2_endpoint.c
@@ -748,7 +748,7 @@ static void mca_btl_tcp2_endpoint_recv_handler(int sd, short flags, void* user)
                     frag = NULL;
 #else
                     /* If the cache still contain some data we can reuse the same fragment
-                     * until we flush it completly.
+                     * until we flush it completely.
                      */
                     MCA_BTL_TCP_FRAG_INIT_DST(frag, btl_endpoint);
 #endif  /* MCA_BTL_TCP_USES_PROGRESS_THREAD */
diff --git a/contrib/build-mca-comps-outside-of-tree/btl_tcp2_proc.c b/contrib/build-mca-comps-outside-of-tree/btl_tcp2_proc.c
index 30563f65966..daabece3570 100644
--- a/contrib/build-mca-comps-outside-of-tree/btl_tcp2_proc.c
+++ b/contrib/build-mca-comps-outside-of-tree/btl_tcp2_proc.c
@@ -190,7 +190,7 @@ static void evaluate_assignment(int *a) {
     }
 
     /*
-     * check wether current solution beats all previous solutions
+     * check whether current solution beats all previous solutions
      */
     if(assignment_cardinality > max_assignment_cardinality
             || (assignment_cardinality == max_assignment_cardinality
@@ -692,7 +692,7 @@ int mca_btl_tcp2_proc_remove(mca_btl_tcp2_proc_t* btl_proc, mca_btl_base_endpoin
                 OBJ_RELEASE(btl_proc);
                 return OMPI_SUCCESS;
             }
-            /* The endpoint_addr may still be NULL if this enpoint is
+            /* The endpoint_addr may still be NULL if this endpoint is
                being removed early in the wireup sequence (e.g., if it
                is unreachable by all other procs) */
             if (NULL != btl_endpoint->endpoint_addr) {
diff --git a/contrib/check-help-strings.pl b/contrib/check-help-strings.pl
index a1fcaeeaa36..3c766909cfe 100755
--- a/contrib/check-help-strings.pl
+++ b/contrib/check-help-strings.pl
@@ -1,6 +1,7 @@
 #!/usr/bin/env perl
 #
-# Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2014-2022 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Simple script to check all the opal_show_help (and orte_show_help)
@@ -94,6 +95,7 @@ sub match_files {
     # Don't recurse down "special" directories
     if (-d $_ &&
         ((/^\.deps$/) || (/^\.libs$/) ||
+         (/3rd-party/) ||
          (/^\.svn$/) || (/^\.hg$/) || (/^\.git$/))) {
         $File::Find::prune = 1;
         return;
@@ -242,13 +244,13 @@ sub check_name {
     my $sep = shift;
     my $src = shift;
 
-    while ($src =~ m/$name\s*$sep\s*"(.+?)"\s*,\s*"(.+?)"/) {
+    while ($src =~ m/$name\s*$sep\s*"(.+?)"\s*,.*?"(.+?)"/) {
         my $file = $1;
         my $topic = $2;
         check_file_topic($info, $file, $topic);
 
         # Don't find this one again
-        $src =~ s/$name\s*$sep\s*"(.+?)"\s*,\s*"(.+?)"/SHOW_HELP_REPLACED/;
+        $src =~ s/$name\s*$sep\s*"(.+?)"\s*,.*?"(.+?)"/SHOW_HELP_REPLACED/;
     }
 
     return $src;
diff --git a/contrib/check-owner.pl b/contrib/check-owner.pl
index fd4375faab1..590cee49fe8 100755
--- a/contrib/check-owner.pl
+++ b/contrib/check-owner.pl
@@ -1,6 +1,6 @@
 #!/usr/bin/env perl
 #
-# Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2015-2022 Cisco Systems, Inc.  All rights reserved
 # Copyright (c) 2015 Los Alamos National Security, LLC.  All rights reserved.
 # $COPYRIGHT$
 #
@@ -55,7 +55,7 @@ sub DebugDump {
 sub isTopDir {
     my ($d) = @_;
 
-    # master
+    # main
     if (-f "$d/Makefile.ompi-rules") {
         return 1;
     }
diff --git a/contrib/check_unnecessary_headers.sh b/contrib/check_unnecessary_headers.sh
old mode 100644
new mode 100755
index d3444700a6c..d11e5bf467f
--- a/contrib/check_unnecessary_headers.sh
+++ b/contrib/check_unnecessary_headers.sh
@@ -11,6 +11,8 @@
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
 # Copyright (c) 2009      Oak Ridge National Labs.  All rights reserved.
+# Copyright (c)           Amazon.com, Inc. or its affiliates.
+#                         All Rights reserved.
 #
 #
 # Some grep/sed mojo may be of interest to others...
@@ -35,7 +37,7 @@ function del_header()
 
 #
 # In Subdirectory DIR, search for headers provided in array SEARCH_HEADER
-# SEARCH_HEADER must initilized with a[0]="header_file.h  typdefs  macros  functions"
+# SEARCH_HEADER must be initialized with a[0]="header_file.h  typdefs  macros  functions"
 #
 # If any of the typedefs, macros or functions show up, then
 # including the header is valid required, otherwise del_header
@@ -87,7 +89,7 @@ SEARCH_HEADER[2]="opal/class/opal_bitmap.h opal_bitmap_t opal_bitmap_set_max_siz
 SEARCH_HEADER[3]="opal/class/opal_free_list.h opal_free_list_t opal_free_list_item_t opal_free_list_init opal_free_list_grow OPAL_FREE_LIST_GET OPAL_FREE_LIST_WAIT OPAL_FREE_LIST_RETURN"
 SEARCH_HEADER[4]="opal/class/opal_graph.h opal_graph_vertex_t opal_graph_edge_t opal_adjacency_list_t opal_graph_t opal_graph_copy_vertex_data opal_graph_free_vertex_data opal_graph_alloc_vertex_data vertex_distance_from_t opal_graph_add_vertex opal_graph_remove_vertex opal_graph_add_edge opal_graph_remove_edge opal_graph_adjacent opal_graph_get_order opal_graph_get_size opal_graph_find_vertex opal_graph_get_graph_vertices opal_graph_get_adjacent_vertices opal_graph_duplicate opal_graph_spf opal_graph_dijkstra opal_graph_print"
 SEARCH_HEADER[5]="opal/class/opal_hash_table.h opal_hash_table_t opal_hash_table_init opal_hash_table_get_size opal_hash_table_remove_all opal_hash_table_get_value_uint32 opal_hash_table_set_value_uint32 opal_hash_table_remove_value_uint32 opal_hash_table_get_value_uint64 opal_hash_table_set_value_uint64 opal_hash_table_remove_value_uint64 opal_hash_table_get_value_ptr opal_hash_table_set_value_ptr opal_hash_table_remove_value_ptr opal_hash_table_get_first_key_uint32 opal_hash_table_get_next_key_uint32 opal_hash_table_get_first_key_uint64 opal_hash_table_get_next_key_uint64"
-SEARCH_HEADER[6]="opal/class/opal_list.h opal_list_t opal_list_item_t opal_list_get_next opal_list_get_prev opal_list_is_empty opal_list_get_first opal_list_get_last opal_list_get_begin opal_list_get_end opal_list_get_size opal_list_remove_item opal_list_append opal_list_prepend opal_list_remove_first opal_list_remove_last opal_list_insert_pos opal_list_insert opal_list_join opal_list_splice opal_list_sort opal_list_item_compare_fn_t"
+SEARCH_HEADER[6]="opal/class/opal_list.h opal_list_t opal_list_item_t opal_list_get_next opal_list_get_prev opal_list_is_empty opal_list_get_first opal_list_get_last opal_list_get_begin opal_list_get_end opal_list_get_size opal_list_remove_item opal_list_append opal_list_prepend opal_list_remove_first opal_list_remove_last opal_list_insert_pos opal_list_join opal_list_splice opal_list_sort opal_list_item_compare_fn_t"
 SEARCH_HEADER[7]="opal/class/opal_object.h opal_object_t opal_class_t opal_construct_t opal_destruct_t OPAL_OBJ_STATIC_INIT OBJ_CLASS OBJ_CLASS_INSTANCE OBJ_CLASS_DECLARATION OBJ_NEW OBJ_RETAIN OBJ_RELEASE OBJ_CONSTRUCT OBJ_DESTRUCT opal_class_initialize opal_class_finalize opal_obj_run_constructors opal_obj_run_destructors opal_obj_new opal_obj_update"
 SEARCH_HEADER[8]="opal/class/opal_pointer_array.h opal_pointer_array_t opal_pointer_array_init opal_pointer_array_add opal_pointer_array_set_item opal_pointer_array_get_item opal_pointer_array_get_size opal_pointer_array_set_size opal_pointer_array_test_and_set_item opal_pointer_array_remove_all"
 SEARCH_HEADER[9]="opal/class/opal_value_array.h opal_value_array_t opal_value_array_init opal_value_array_reserve opal_value_array_get_size opal_value_array_set_size OPAL_VALUE_ARRAY_GET_ITEM opal_value_array_get_item OPAL_VALUE_ARRAY_SET_ITEM opal_value_array_set_item opal_value_array_append_item opal_value_array_remove_item OPAL_VALUE_ARRAY_GET_BASE"
@@ -114,17 +116,18 @@ SEARCH_HEADER[30]="opal/util/convert.h opal_size2int"
 SEARCH_HEADER[31]="opal/util/daemon_init.h opal_daemon_init"
 SEARCH_HEADER[32]="opal/util/error.h opal_perror opal_strerror opal_strerror_r opal_err2str_fn_t opal_error_register"
 SEARCH_HEADER[33]="opal/util/if.h IF_NAMESIZE opal_ifnametoaddr opal_ifaddrtoname opal_ifnametoindex opal_ifnametokindex opal_ifindextokindex opal_ifcount opal_ifbegin opal_ifnext opal_ifindextoname opal_ifkindextoname opal_ifindextoaddr opal_ifindextomask opal_ifindextoflags opal_ifislocal opal_iffinalize"
-SEARCH_HEADER[34]="opal/util/net.h opal_net_init opal_net_finalize opal_net_prefix2netmask opal_net_islocalhost opal_net_samenetwork opal_net_addr_isipv4public opal_net_get_hostname opal_net_get_port"
-SEARCH_HEADER[35]="opal/util/opal_environ.h opal_environ_merge opal_setenv opal_unsetenv opal_home_directory opal_tmp_directory environ"
-SEARCH_HEADER[36]="opal/util/opal_getcwd.h opal_getcwd"
-SEARCH_HEADER[37]="opal/util/os_dirpath.h opal_os_dirpath_create opal_os_dirpath_is_empty opal_os_dirpath_access opal_os_dirpath_destroy"
-SEARCH_HEADER[38]="opal/util/os_path.h opal_os_path opal_make_filename_os_friendly"
-SEARCH_HEADER[39]="opal/util/output.h opal_output_stream_t opal_output_init opal_output_finalize opal_output_open opal_output_reopen opal_output_switch opal_output_reopen_all opal_output_close opal_output opal_output_verbose opal_output_vverbose opal_output_string opal_output_vstring opal_output_set_verbosity opal_output_get_verbosity opal_output_set_output_file_info OPAL_OUTPUT OPAL_OUTPUT_VERBOSE"
-SEARCH_HEADER[40]="opal/util/path.h opal_path_find opal_path_findv opal_path_is_absolute opal_find_absolute_path opal_path_access"
-SEARCH_HEADER[41]="opal/util/printf.h snprintf vsnprintf"
-SEARCH_HEADER[42]="opal/util/show_help.h opal_show_help_init opal_show_help_finalize opal_show_help opal_show_vhelp opal_show_help_string opal_show_help_finish_parsing"
-SEARCH_HEADER[43]="opal/util/strncpy.h opal_strncpy"
-SEARCH_HEADER[44]=""
+SEARCH_HEADER[34]="opal/util/misc.h container_of"
+SEARCH_HEADER[35]="opal/util/net.h opal_net_init opal_net_finalize opal_net_prefix2netmask opal_net_islocalhost opal_net_samenetwork opal_net_addr_isipv4public opal_net_get_hostname opal_net_get_port"
+SEARCH_HEADER[36]="opal/util/opal_environ.h opal_environ_merge opal_setenv opal_unsetenv opal_home_directory opal_tmp_directory environ"
+SEARCH_HEADER[37]="opal/util/opal_getcwd.h opal_getcwd"
+SEARCH_HEADER[38]="opal/util/os_dirpath.h opal_os_dirpath_create opal_os_dirpath_is_empty opal_os_dirpath_access opal_os_dirpath_destroy"
+SEARCH_HEADER[39]="opal/util/os_path.h opal_os_path opal_make_filename_os_friendly"
+SEARCH_HEADER[40]="opal/util/output.h opal_output_stream_t opal_output_init opal_output_finalize opal_output_open opal_output_reopen opal_output_switch opal_output_reopen_all opal_output_close opal_output opal_output_verbose opal_output_vverbose opal_output_string opal_output_vstring opal_output_set_verbosity opal_output_get_verbosity opal_output_set_output_file_info OPAL_OUTPUT OPAL_OUTPUT_VERBOSE"
+SEARCH_HEADER[41]="opal/util/path.h opal_path_find opal_path_findv opal_path_is_absolute opal_find_absolute_path opal_path_access"
+SEARCH_HEADER[42]="opal/util/printf.h snprintf vsnprintf"
+SEARCH_HEADER[43]="opal/util/show_help.h opal_show_help_init opal_show_help_finalize opal_show_help opal_show_vhelp opal_show_help_string opal_show_help_finish_parsing"
+SEARCH_HEADER[44]="opal/util/strncpy.h opal_strncpy"
+SEARCH_HEADER[45]=""
 
 delete_unnessary_header .
 
@@ -175,12 +178,12 @@ SEARCH_HEADER[29]=""
 delete_unnessary_header .
 
 ####################################
-SEARCH_HEADER[0]="ompi/attribute/attribute.h ATTR_HASH_SIZE OMPI_KEYVAL_PREDEFINED OMPI_KEYVAL_F77 ompi_attribute_type_t ompi_mpi1_fortran_copy_attr_function ompi_mpi1_fortran_delete_attr_function ompi_mpi2_fortran_copy_attr_function ompi_mpi2_fortran_delete_attr_function MPI_Comm_internal_copy_attr_function MPI_Type_internal_copy_attr_function MPI_Win_internal_copy_attr_function ompi_attribute_keyval_destructor_fn_t ompi_attribute_fn_ptr_union_t ompi_attribute_fortran_ptr_t ompi_attribute_keyval_t ompi_attr_hash_init ompi_attr_init ompi_attr_finalize ompi_attr_create_keyval ompi_attr_free_keyval ompi_attr_set_c ompi_attr_set_fortran_mpi1 ompi_attr_set_fortran_mpi2 ompi_attr_get_c ompi_attr_get_fortran_mpi1 ompi_attr_get_fortran_mpi2 ompi_attr_delete ompi_attr_copy_all ompi_attr_delete_all ompi_attr_create_predefined ompi_attr_free_predefined"
+SEARCH_HEADER[0]="ompi/attribute/attribute.h ATTR_HASH_SIZE OMPI_KEYVAL_PREDEFINED OMPI_KEYVAL_F77 ompi_attribute_type_t ompi_mpi1_fortran_copy_attr_function ompi_mpi1_fortran_delete_attr_function ompi_mpi2_fortran_copy_attr_function ompi_mpi2_fortran_delete_attr_function  ompi_attribute_keyval_destructor_fn_t ompi_attribute_fn_ptr_union_t ompi_attribute_fortran_ptr_t ompi_attribute_keyval_t ompi_attr_hash_init ompi_attr_init ompi_attr_finalize ompi_attr_create_keyval ompi_attr_free_keyval ompi_attr_set_c ompi_attr_set_fortran_mpi1 ompi_attr_set_fortran_mpi2 ompi_attr_get_c ompi_attr_get_fortran_mpi1 ompi_attr_get_fortran_mpi2 ompi_attr_delete ompi_attr_copy_all ompi_attr_delete_all ompi_attr_create_predefined ompi_attr_free_predefined"
 SEARCH_HEADER[1]="ompi/class/ompi_free_list.h ompi_free_list_item_init_fn_t ompi_free_list_t ompi_free_list_item_t ompi_free_list_init_ex ompi_free_list_init ompi_free_list_init_ex_new ompi_free_list_init_new ompi_free_list_grow ompi_free_list_resize ompi_free_list_pos_t OMPI_FREE_LIST_POS_BEGINNING ompi_free_list_parse OMPI_FREE_LIST_GET OMPI_FREE_LIST_WAIT __ompi_free_list_wait OMPI_FREE_LIST_RETURN"
 SEARCH_HEADER[2]="ompi/class/ompi_rb_tree.h ompi_rb_tree_nodecolor_t ompi_rb_tree_node_t ompi_rb_tree_comp_fn_t ompi_rb_tree_t ompi_rb_tree_condition_fn_t ompi_rb_tree_action_fn_t ompi_rb_tree_construct ompi_rb_tree_destruct ompi_rb_tree_init ompi_rb_tree_insert ompi_rb_tree_find_with ompi_rb_tree_find ompi_rb_tree_delete ompi_rb_tree_destroy ompi_rb_tree_traverse ompi_rb_tree_size"
 SEARCH_HEADER[3]="ompi/class/ompi_seq_tracker.h ompi_seq_tracker_range_t ompi_seq_tracker_t ompi_seq_tracker_check_duplicate ompi_seq_tracker_insert ompi_seq_tracker_copy"
-SEARCH_HEADER[4]="ompi/communicator/communicator.h MPI_Comm MPI_COMM_WORLD ompi_communicator_t OMPI_COMM_INTER OMPI_COMM_CART OMPI_COMM_GRAPH OMPI_COMM_NAMEISSET OMPI_COMM_ISFREED OMPI_COMM_INTRINSIC OMPI_COMM_DYNAMIC OMPI_COMM_INVALID OMPI_COMM_PML_ADDED OMPI_COMM_IS_ OMPI_COMM_SET_ OMPI_COMM_ALLGATHER_TAG OMPI_COMM_BARRIER_TAG OMPI_COMM_ALLREDUCE_TAG OMPI_COMM_CID_ OMPI_COMM_BLOCK_ ompi_predefined_communicator_t ompi_mpi_comm_parent ompi_mpi_comm_world ompi_mpi_comm_self ompi_mpi_comm_null ompi_comm_invalid ompi_comm_rank ompi_comm_size ompi_comm_remote_size ompi_comm_get_cid ompi_comm_lookup ompi_comm_peer_lookup ompi_comm_peer_invalid ompi_comm_init ompi_comm_link_function ompi_comm_group ompi_comm_create ompi_topo_create ompi_comm_split ompi_comm_dup ompi_comm_compare ompi_comm_free ompi_comm_allocate ompi_comm_nextcid ompi_comm_finalize ompi_comm_set ompi_comm_get_rprocs ompi_comm_overlapping_groups ompi_comm_determine_first ompi_comm_activate ompi_comm_dump ompi_comm_set_name ompi_comm_reg_init ompi_comm_reg_finalize ompi_comm_num_dyncomm ompi_mpi_cxx_comm_errhandler_invoke"
-SEARCH_HEADER[5]="ompi/datatype/convertor.h OMPI_COMM_INTER OMPI_COMM_CART OMPI_COMM_GRAPH OMPI_COMM_NAMEISSET OMPI_COMM_ISFREED OMPI_COMM_INTRINSIC OMPI_COMM_DYNAMIC OMPI_COMM_INVALID OMPI_COMM_PML_ADDED OMPI_COMM_IS_ OMPI_COMM_SET_ OMPI_COMM_ALLGATHER_TAG OMPI_COMM_BARRIER_TAG OMPI_COMM_ALLREDUCE_TAG OMPI_COMM_CID_ OMPI_COMM_BLOCK_ ompi_predefined_communicator_t ompi_mpi_comm_parent ompi_mpi_comm_null ompi_comm_invalid ompi_comm_rank ompi_comm_size ompi_comm_remote_size ompi_comm_get_cid ompi_comm_lookup ompi_comm_peer_lookup ompi_comm_peer_invalid ompi_comm_init ompi_comm_link_function ompi_comm_group ompi_comm_create ompi_topo_create ompi_comm_split ompi_comm_dup ompi_comm_compare ompi_comm_free ompi_comm_allocate ompi_comm_nextcid ompi_comm_finalize ompi_comm_set ompi_comm_get_rprocs ompi_comm_overlapping_groups ompi_comm_determine_first ompi_comm_activate ompi_comm_dump ompi_comm_set_name ompi_comm_reg_init ompi_comm_reg_finalize ompi_comm_num_dync CONVERTOR_DATATYPE_MASK CONVERTOR_SEND_CONVERSION CONVERTOR_RECV CONVERTOR_SEND CONVERTOR_HOMOGENEOUS CONVERTOR_NO_OP CONVERTOR_WITH_CHECKSUM CONVERTOR_TYPE_MASK CONVERTOR_STATE_START CONVERTOR_STATE_COMPLETE CONVERTOR_STATE_ALLOC CONVERTOR_COMPLETED ompi_convertor_t ompi_convertor_master_t dt_stack_t DT_STATIC_STACK_SIZE ompi_convertor_get_checksum ompi_convertor_pack ompi_convertor_unpack ompi_convertor_create ompi_convertor_cleanup ompi_convertor_need_buffers ompi_convertor_get_packed_size ompi_convertor_get_unpacked_size ompi_convertor_get_current_pointer ompi_convertor_prepare_for_send ompi_convertor_copy_and_prepare_for_send ompi_convertor_prepare_for_recv ompi_convertor_copy_and_prepare_for_recv ompi_convertor_raw ompi_convertor_set_position_nocheck ompi_convertor_set_position ompi_convertor_personalize ompi_convertor_clone ompi_convertor_clone_with_position ompi_convertor_dump ompi_ddt_dump_stack ompi_convertor_generic_simple_position MPI_Datatype"
+SEARCH_HEADER[4]="ompi/communicator/communicator.h MPI_Comm MPI_COMM_WORLD ompi_communicator_t OMPI_COMM_INTER OMPI_COMM_CART OMPI_COMM_GRAPH OMPI_COMM_NAMEISSET OMPI_COMM_ISFREED OMPI_COMM_INTRINSIC OMPI_COMM_DYNAMIC OMPI_COMM_INVALID OMPI_COMM_DISJOINT_SET OMPI_COMM_DISJOINT OMPI_COMM_PML_ADDED OMPI_COMM_IS_ OMPI_COMM_SET_ OMPI_COMM_ALLGATHER_TAG OMPI_COMM_BARRIER_TAG OMPI_COMM_ALLREDUCE_TAG OMPI_COMM_CID_ OMPI_COMM_BLOCK_ ompi_predefined_communicator_t ompi_mpi_comm_parent ompi_mpi_comm_world ompi_mpi_comm_self ompi_mpi_comm_null ompi_comm_invalid ompi_comm_rank ompi_comm_size ompi_comm_remote_size ompi_comm_get_cid ompi_comm_lookup ompi_comm_peer_lookup ompi_comm_peer_invalid ompi_comm_init ompi_comm_link_function ompi_comm_group ompi_comm_create ompi_topo_create ompi_comm_split ompi_comm_dup ompi_comm_compare ompi_comm_free ompi_comm_allocate ompi_comm_nextcid ompi_comm_finalize ompi_comm_set ompi_comm_get_rprocs ompi_comm_overlapping_groups ompi_comm_determine_first ompi_comm_activate ompi_comm_dump ompi_comm_set_name ompi_comm_reg_init ompi_comm_reg_finalize ompi_comm_num_dyncomm ompi_mpi_cxx_comm_errhandler_invoke"
+SEARCH_HEADER[5]="ompi/datatype/convertor.h OMPI_COMM_INTER OMPI_COMM_CART OMPI_COMM_GRAPH OMPI_COMM_NAMEISSET OMPI_COMM_ISFREED OMPI_COMM_INTRINSIC OMPI_COMM_DYNAMIC OMPI_COMM_INVALID OMPI_COMM_DISJOINT_SET OMPI_COMM_DISJOINT OMPI_COMM_PML_ADDED OMPI_COMM_IS_ OMPI_COMM_SET_ OMPI_COMM_ALLGATHER_TAG OMPI_COMM_BARRIER_TAG OMPI_COMM_ALLREDUCE_TAG OMPI_COMM_CID_ OMPI_COMM_BLOCK_ ompi_predefined_communicator_t ompi_mpi_comm_parent ompi_mpi_comm_null ompi_comm_invalid ompi_comm_rank ompi_comm_size ompi_comm_remote_size ompi_comm_get_cid ompi_comm_lookup ompi_comm_peer_lookup ompi_comm_peer_invalid ompi_comm_init ompi_comm_link_function ompi_comm_group ompi_comm_create ompi_topo_create ompi_comm_split ompi_comm_dup ompi_comm_compare ompi_comm_free ompi_comm_allocate ompi_comm_nextcid ompi_comm_finalize ompi_comm_set ompi_comm_get_rprocs ompi_comm_overlapping_groups ompi_comm_determine_first ompi_comm_activate ompi_comm_dump ompi_comm_set_name ompi_comm_reg_init ompi_comm_reg_finalize ompi_comm_num_dync CONVERTOR_DATATYPE_MASK CONVERTOR_SEND_CONVERSION CONVERTOR_RECV CONVERTOR_SEND CONVERTOR_HOMOGENEOUS CONVERTOR_NO_OP CONVERTOR_WITH_CHECKSUM CONVERTOR_TYPE_MASK CONVERTOR_STATE_START CONVERTOR_STATE_COMPLETE CONVERTOR_STATE_ALLOC CONVERTOR_COMPLETED ompi_convertor_t ompi_convertor_master_t dt_stack_t DT_STATIC_STACK_SIZE ompi_convertor_get_checksum ompi_convertor_pack ompi_convertor_unpack ompi_convertor_create ompi_convertor_cleanup ompi_convertor_need_buffers ompi_convertor_get_packed_size ompi_convertor_get_unpacked_size ompi_convertor_get_current_pointer ompi_convertor_prepare_for_send ompi_convertor_copy_and_prepare_for_send ompi_convertor_prepare_for_recv ompi_convertor_copy_and_prepare_for_recv ompi_convertor_raw ompi_convertor_set_position_nocheck ompi_convertor_set_position ompi_convertor_personalize ompi_convertor_clone ompi_convertor_clone_with_position ompi_convertor_dump ompi_ddt_dump_stack ompi_convertor_generic_simple_position MPI_Datatype"
 SEARCH_HEADER[6]="ompi/datatype/datatype.h MPI_Datatype DT_MAX_PREDEFINED DT_FLAG_ MAX_DT_COMPONENT_COUNT opal_ddt_count_t dt_type_desc_t ompi_datatype_t ompi_predefined_datatype_t ompi_ddt_init ompi_ddt_finalize ompi_ddt_create_ ompi_ddt_duplicate ompi_ddt_is_predefined ompi_ddt_create_from_packed_description"
 SEARCH_HEADER[7]="ompi/datatype/datatype_internal.h DDT_DUMP_STACK DT_ ddt_elem_id_description ddt_elem_desc ddt_elem_desc_t ddt_loop_desc ddt_loop_desc_t ddt_endloop_desc ddt_endloop_desc_t dt_elem_desc CREATE_LOOP_START CREATE_LOOP_END CREATE_ELEM ompi_complex_float_t ompi_complex_double_t ompi_complex_long_double_t ompi_ddt_basicDatatypes BASIC_DDT_FROM_ELEM ompi_ddt_default_convertors_init ompi_ddt_default_convertors_fini SAVE_STACK PUSH_STACK ompi_ddt_safeguard_pointer_debug_breakpoint OMPI_DDT_SAFEGUARD_POINTER GET_FIRST_NON_LOOP UPDATE_INTERNAL_COUNTERS ompi_ddt_print_args"
 SEARCH_HEADER[8]="ompi/errhandler/errhandler.h OMPI_ERRHANDLER_LANG_ ompi_errhandler_lang_t OMPI_ERRHANDLER_TYPE_ ompi_errhandler_type_t ompi_errhandler_t ompi_predefined_errhandler_t ompi_mpi_errhandler_null OMPI_ERRHANDLER_CHECK OMPI_ERRHANDLER_RETURN ompi_errhandler_init ompi_errhandler_finalize OMPI_ERRHANDLER_INVOKE ompi_errhandler_invoke ompi_errhandler_request_invoke ompi_errhandler_create ompi_errhandler_is_intrinsic ompi_errhandler_fortran_handler_fn_t OMPI_ERR_INIT_FINALIZE MPI_Errhandler"
@@ -230,7 +233,7 @@ echo "# for i in *.c ; do grep -q '#include \"ompi/runtime/params.h\"' $$i || ad
 
 
 # Finally erase a header that has been introduced for the STCI replacement
-# and is not necessary, if no occurences of #include "orte..."  reside...
+# and is not necessary, if no occurrences of #include "orte..."  reside...
 SEARCH_HEADER[0]="rte.h orte"
 SEARCH_HEADER[1]=""
 
diff --git a/contrib/clang-format-ompi.sh b/contrib/clang-format-ompi.sh
new file mode 100755
index 00000000000..4acf48ae6cd
--- /dev/null
+++ b/contrib/clang-format-ompi.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+echo "Running clang-format on code base..."
+
+files=($(git ls-tree -r main --name-only | grep -v -E '3rd-party/|contrib/' | grep -e '.*\.[ch]$' | xargs grep -E -L -- "-*- fortran -*-|-*- f90 -*-"))
+
+for file in "${files[@]}" ; do
+    if test "$1" = "-d" ; then
+	echo "Would have formatted: ${file}"
+    else
+	clang-format --style=file --verbose -i "${file}"
+    fi
+done
+
+echo "Done"
diff --git a/contrib/coll_tuned_rulefile_converter.py b/contrib/coll_tuned_rulefile_converter.py
new file mode 100755
index 00000000000..f9709653d70
--- /dev/null
+++ b/contrib/coll_tuned_rulefile_converter.py
@@ -0,0 +1,185 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2024-2025 Amazon.com, Inc. or its affiliates.
+#                         All Rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+
+import re
+import json
+from collections import OrderedDict
+
+coll_dict = {
+    'allgather'             : 0,
+    'allgatherv'            : 1,
+    'allreduce'             : 2,
+    'alltoall'              : 3,
+    'alltoallv'             : 4,
+    'alltoallw'             : 5,
+    'barrier'               : 6,
+    'bcast'                 : 7,
+    'exscan'                : 8,
+    'gather'                : 9,
+    'gatherv'               : 10,
+    'reduce'                : 11,
+    'reducescatter'         : 12,
+    'reducescatterblock'    : 13,
+    'scan'                  : 14,
+    'scatter'               : 15,
+    'scatterv'              : 16,
+    'neighbor_allgather'    : 17,
+    'neighbor_allgatherv'   : 18,
+    'neighbor_alltoall'     : 19,
+    'neighbor_alltoallv'    : 20,
+    'neighbor_alltoallw'    : 21 }
+coll_dict_rev = { v:k for k,v in coll_dict.items() }
+
+han_component_dict = {
+    "self"                  : 0,
+    "basic"                 : 1,
+    "libnbc"                : 2,
+    "tuned"                 : 3,
+    "sm"                    : 4,
+    "adapt"                 : 5,
+    "han"                   : 6,
+}
+
+han_topo_level_dict = {
+    'intra_node'            : 0,
+    'inter_node'            : 1,
+    'global_communicator'   : 2,
+}
+
+
+def strip_comments(line):
+    return re.sub(r"#.*","",line).strip()
+
+class GenericOpenMPIRuleReader():
+    def __init__(self, fp, fname_for_prints=""):
+        self.fp = fp
+        # The 1-indexed line number which corresponds to the next byte of fp read.
+        self.jline = 1
+        self.line_start = 0
+    def get_next_line(self):
+        while True:
+            self.line_start = self.fp.tell()
+            line = self.fp.readline()
+            if not line: return None
+            self.jline += 1
+            if strip_comments(line):
+                return line
+
+    def isnext_digit(self):
+        # ompi_coll_base_file_peek_next_char_isdigit
+        tell = self.fp.tell()
+        while True:
+            next = self.fp.read(1)
+            if next in ' \t':
+                tell += 1
+                continue
+            self.fp.seek(tell)
+            return next in '0123456789'
+
+    def get_next(self):
+        # (ompi_coll_base_file_getnext_long)
+        while True:
+            line = self.get_next_line()
+            if not line: return None
+            UNK = -1
+            jnum_start = UNK
+            jnum_end = UNK
+            for jc in range(len(line)):
+                if line[jc] in "#":
+                    break
+                if line[jc] in '0123456789':
+                    if jnum_start == UNK:
+                        jnum_start = jc
+                    jnum_end = jc
+                else:
+                    if jnum_end != UNK:
+                        break
+            if jnum_end != UNK:
+                self.fp.seek(self.line_start+jnum_end+1)
+                # decrement the line number, the next read will continue on this line.
+                self.jline -= 1
+                return int(line[jnum_start:jnum_end+1])
+
+    def read_header(self):
+        line = self.get_next_line()
+        match = re.match("rule-file-version-([0-9])", line)
+        if match:
+            return int(match.group(1))
+        else:
+            self.jline -= 1
+            self.fp.seek(self.line_start)
+            return 1
+
+class TunedRuleReader(GenericOpenMPIRuleReader):
+    def load_rulefile(self):
+        json_root = OrderedDict()
+        file_ver = self.read_header()
+        json_root['rule_file_version'] = 3
+        json_root['module'] = 'tuned'
+        json_root['collectives'] = OrderedDict()
+
+        ncollectives = self.get_next()
+        for jcol in range(ncollectives):
+            coll_id = self.get_next()
+            coll_name = coll_dict_rev[coll_id]
+            comm_rules = []
+            ncomm_sizes = self.get_next()
+            for jcomm_size in range(ncomm_sizes):
+                comm_size = self.get_next()
+                nmsg_sizes = self.get_next()
+                comm_rule = OrderedDict()
+                comm_rule['comm_size_min'] = 0
+                if jcomm_size+1 < ncomm_sizes:
+                    comm_rule['comm_size_max'] = max(comm_size-1, 0)
+                if jcomm_size > 0:
+                    comm_rule['comm_size_min'] = comm_rules[jcomm_size-1]['comm_size_max'] + 1
+                msg_rules = []
+                for jmsg in range(nmsg_sizes):
+                    msg_size = self.get_next()
+                    result_alg = self.get_next()
+                    result_topo_faninout = self.get_next()
+                    result_segsize = self.get_next()
+                    rule = OrderedDict()
+                    rule['msg_size_min'] = msg_size
+                    if jmsg < nmsg_sizes - 1:
+                        rule['msg_size_max'] = 'Inf'
+                    if jmsg > 0:
+                        msg_rules[jmsg-1]['msg_size_max'] = msg_size - 1
+                    rule['alg'] = result_alg
+                    if result_topo_faninout != 0:
+                        rule['faninout'] = result_topo_faninout
+                    if result_segsize != 0:
+                        rule['segsize'] = result_segsize
+                    result_maxreq = 0
+                    if file_ver > 1 and self.isnext_digit():
+                        result_maxreq = self.get_next()
+                    if result_maxreq != 0:
+                        rule['reqs'] = result_maxreq
+                    msg_rules.append(rule)
+                comm_rule['rules'] = msg_rules
+                comm_rules.append(comm_rule)
+            json_root['collectives'][coll_name] = comm_rules
+        return json_root
+
+class TunedRuleWriter():
+    def __init__(self):
+        pass
+    def to_file(json_rules):
+        for coll in coll_dict.keys():
+            if coll in json_rules['collectives']:
+                pass
+
+if __name__ == '__main__':
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input","-i", type=argparse.FileType('r'), required=True)
+    # parser.add_argument("--output","-o",type=argparse.FileType('w'), required=True)
+
+    args = parser.parse_args()
+    reader = TunedRuleReader(args.input)
+    print(json.dumps(reader.load_rulefile(), indent=4))
diff --git a/contrib/completion/mpirun.sh b/contrib/completion/mpirun.sh
old mode 100644
new mode 100755
diff --git a/contrib/dist/find-copyrights.pl b/contrib/dist/find-copyrights.pl
index 112d75b4a4e..2cdff82769c 100755
--- a/contrib/dist/find-copyrights.pl
+++ b/contrib/dist/find-copyrights.pl
@@ -75,8 +75,7 @@ sub save {
     $core = 1;
     $core = 0
         if ($f =~ /ompi\/contrib\/[a-zA-Z0-9]+\// ||
-            $f =~ /opal\/event/ ||
-            $f =~ /ompi\/mca\/io\/romio\/romio/);
+            $f =~ /opal\/event/);
 
     # Scan the file for copyrights
     open FILE, $f || die "Can't open file: $f";
diff --git a/contrib/dist/linux/README.md b/contrib/dist/linux/README.md
index 65aae3c3c20..5ab9db102fe 100644
--- a/contrib/dist/linux/README.md
+++ b/contrib/dist/linux/README.md
@@ -19,7 +19,7 @@ filename pointing to an Open MPI tarball (may be either `.gz` or
    * `openmpi-devel`
    * `openmpi-docs`
 
-The folowing arguments could be used to affect script behaviour.
+The following arguments could be used to affect script behaviour.
 Please, do NOT set the same settings with parameters and config vars.
 
 * `-b`:
@@ -32,7 +32,7 @@ Please, do NOT set the same settings with parameters and config vars.
    This option will change the name of the produced RPM to the "name".
    It is useful to use with "-o" and "-m" options if you want to have
    multiple Open MPI versions installed simultaneously in the same
-   enviroment. Requires use of option `-b`.
+   environment. Requires use of option `-b`.
 
 * `-o`:
    With this option the install path of the binary RPM will be changed
@@ -61,6 +61,13 @@ Please, do NOT set the same settings with parameters and config vars.
 * `-c parameter`:
    Add custom configure parameter.
 
+   **NOTE:** As of Open MPI v5.0.x, there are default configure
+   options for `--with-FOO=external` for all the 3rd party packages
+   (libevent, hwloc, pmix, prrte).  This makes a pure Open MPI RPM,
+   not an RPM that includes those 3rd party packages.  If you specify
+   `-c`, if you want to preserve the default options, you will need to
+   include those in the parameter value.
+
 * `-r parameter`:
    Add custom RPM build parameter.
 
@@ -70,7 +77,7 @@ Please, do NOT set the same settings with parameters and config vars.
    the script will look for the specfile in the current directory.
 
 * `-R directory`:
-   Specifies the top level RPM build direcotry.
+   Specifies the top level RPM build directory.
 
 * `-h`:
    Prints script usage information.
diff --git a/contrib/dist/linux/buildrpm.sh b/contrib/dist/linux/buildrpm.sh
index 8e749ec621c..3fcb5b147bf 100755
--- a/contrib/dist/linux/buildrpm.sh
+++ b/contrib/dist/linux/buildrpm.sh
@@ -8,7 +8,7 @@
 
 #
 # External script parameters
-# The folowing parameters could be used to affect script behaviour.
+# The following parameters could be used to affect script behaviour.
 # Please, do NOT set the same settings with parameters and config vars.
 #
 # -b
@@ -21,7 +21,7 @@
 #    This option will change the name of the produced RPM to the "name".
 #    It is useful to use with "-o" and "-m" options if you want to have
 #    multiple Open MPI versions installed simultaneously in the same
-#    enviroment. Requires use of option "-b".
+#    environment. Requires use of option "-b".
 #
 # -o
 #    With this option the install path of the binary RPM will be changed
@@ -59,7 +59,7 @@
 #    the script will look for the specfile in the current directory.
 #
 # -R directory
-#    Specifies the top level RPM build direcotry.
+#    Specifies the top level RPM build directory.
 #
 # -h
 #    Prints script usage information.
@@ -85,7 +85,6 @@ unpack_spec=0
 # "normal" names.
 #export CC=gcc
 #export CXX=g++
-#export F77=f77
 #export FC=
 
 # Note that this script can build one or all of the following RPMs:
@@ -150,7 +149,7 @@ usage="Usage: $0 [-b][-o][-m][-d][-u][-s][-h] [-n name][-f lf_location][-t tm_lo
              add custom RPM build parameter
 
   -R directory
-             Specifies the top level RPM build direcotry.
+             Specifies the top level RPM build directory.
 
   -h         print this message and exit
 
@@ -254,7 +253,7 @@ echo "--> Found specfile: $specfile"
 #
 # try to find Libfabric lib subir
 #
-if test -n $libfabric_path; then
+if test -n "$libfabric_path"; then
     # does lib64 exist?
     if test -d $libfabric_path/lib64; then
         # yes, so I will use lib64 as include dir
@@ -353,7 +352,6 @@ cat <<EOF
 --> Hard-wired for compilers:
     CC = $CC
     CXX = $CXX
-    F77 = $F77
     FC = $FC
 EOF
 
@@ -391,9 +389,6 @@ fi
 if test "$CXX" != ""; then
     configure_options="$configure_options CXX=$CXX"
 fi
-if test "$F77" != ""; then
-    configure_options="$configure_options F77=$F77"
-fi
 if test "$FC" != ""; then
     configure_options="$configure_options FC=$FC"
 fi
@@ -471,7 +466,7 @@ cat <<EOF
 ====                FINISHED BUILDING Open MPI RPM                        ====
 ------------------------------------------------------------------------------
 A copy of the tarball is located in: $rpmtopdir/SOURCES/
-The completed rpms are located in:   $rpmtopdir/RPMS/i<something>86/
+The completed rpms are located in:   $rpmtopdir/RPMS/
 The sources rpms are located in:     $rpmtopdir/SRPMS/
 The spec files are located in:       $rpmtopdir/SPECS/
 ------------------------------------------------------------------------------
diff --git a/contrib/dist/linux/ompi-spec-generator.py b/contrib/dist/linux/ompi-spec-generator.py
old mode 100644
new mode 100755
index 95168accd16..902d7ed7451
--- a/contrib/dist/linux/ompi-spec-generator.py
+++ b/contrib/dist/linux/ompi-spec-generator.py
@@ -1,10 +1,10 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # generator for Open MPI
 import sys
 import os
 import optparse
-import ConfigParser
+import configparser
 
 
 ######################################################################
@@ -209,7 +209,7 @@
    puts stderr \"This module adds Open MPI v%(ompi_version)s to various paths\"
 }
 
-module-whatis   \"Sets up Open MPI v%(ompi_version)s in your enviornment\"
+module-whatis   \"Sets up Open MPI v%(ompi_version)s in your environment\"
 
 append-path PATH \"%%{_prefix}/bin/\"
 append-path LD_LIBRARY_PATH %%{_libdir}
@@ -230,15 +230,15 @@
 
 CHANGED=0
 if test -z \"`echo $PATH | grep %%{_prefix}/bin`\"; then
-    PATH=\${PATH}:%%{_prefix}/bin/
+    PATH=\\${PATH}:%%{_prefix}/bin/
     CHANGED=1
 fi
 if test -z \"`echo $LD_LIBRARY_PATH | grep %%{_libdir}`\"; then
-    LD_LIBRARY_PATH=\${LD_LIBRARY_PATH}:%%{_libdir}
+    LD_LIBRARY_PATH=\\${LD_LIBRARY_PATH}:%%{_libdir}
     CHANGED=1
 fi
 if test -z \"`echo $MANPATH | grep %%{_mandir}`\"; then
-    MANPATH=\${MANPATH}:%%{_mandir}
+    MANPATH=\\${MANPATH}:%%{_mandir}
     CHANGED=1
 fi
 if test \"$CHANGED\" = \"1\"; then
@@ -252,16 +252,16 @@
 # uninstalled, or b) if the RPM is upgraded or uninstalled.
 
 if (\"`echo $PATH | grep %%{_prefix}/bin`\") then
-    setenv PATH \${PATH}:%%{_prefix}/bin/
+    setenv PATH \\${PATH}:%%{_prefix}/bin/
 endif
 if (\"$?LD_LIBRARY_PATH\") then
     if (\"`echo $LD_LIBRARY_PATH | grep %%{_libdir}`\") then
-        setenv LD_LIBRARY_PATH \${LD_LIBRARY_PATH}:%%{_libdir}
+        setenv LD_LIBRARY_PATH \\${LD_LIBRARY_PATH}:%%{_libdir}
     endif
 endif
 if (\"$?MANPATH\") then
     if (\"`echo $MANPATH | grep %%{_mandir}`\") then
-        setenv MANPATH \${MANPATH}:%%{_mandir}
+        setenv MANPATH \\${MANPATH}:%%{_mandir}
     endif
 endif
 EOF
@@ -483,7 +483,7 @@ def Validate(self):
     def Dump(self, prefix=""):
         global options
         for option in options:
-            print "%(prefix)s %(name)-15s : %(value)s" % {"prefix":prefix, "name":option, "value":self.options[option]}
+            print("%(prefix)s %(name)-15s : %(value)s" % {"prefix":prefix, "name":option, "value":self.options[option]})
 
 
 ######################################################################
@@ -506,7 +506,7 @@ def get_package(name):
 ######################################################################
 def verbose(msg):
     if (params.verbose or params.debug):
-        print msg
+        print(msg)
 
 
 ######################################################################
@@ -516,7 +516,7 @@ def verbose(msg):
 ######################################################################
 def debug(msg):
     if (params.debug):
-        print msg
+        print(msg)
 
 
 ######################################################################
@@ -525,7 +525,7 @@ def debug(msg):
 #
 ######################################################################
 def error(msg):
-    print "+++ ERROR : " + msg
+    print("+++ ERROR : " + msg)
 
 
 ######################################################################
@@ -549,7 +549,7 @@ def  get_compiler(name):
 
 def shell_help(cmd):
     for item in shell_cmds.values():
-        print "%(cmd)-10s - %(help)s" % {"cmd":item["name"], "help":item["help"]}
+        print("%(cmd)-10s - %(help)s" % {"cmd":item["name"], "help":item["help"]})
     return True
 
 def shell_quit(cmd):
@@ -560,28 +560,28 @@ def shell_list(cmd):
 
 def shell_list(cmd):
     for package in packages.keys():
-        print package
+        print(package)
     return True
 
 def shell_show(cmd):
     if len(cmd) < 2:
-        print "Usage : show PACKAGE\n"
+        print("Usage : show PACKAGE\n")
     else:
         if cmd[1] in packages.keys():
             package = packages[cmd[1]]
             package.Dump()
         else:
-            print "Invalid package : " + cmd[1]
+            print("Invalid package : " + cmd[1])
     return True
 
 def shell_drop(cmd):
     if len(cmd) < 2:
-        print "Usage : drop PACKAGE"
+        print("Usage : drop PACKAGE")
     else:
         if cmd[1] in packages.keys():
             packages.pop(cmd[1])
         else:
-            print "Package not found : " + cmd[1]
+            print("Package not found : " + cmd[1])
     return True
 
 def shell_write(cmd):
@@ -599,8 +599,8 @@ def shell():
     register_shell_cmd("show", "Display one specific package", shell_show)
     register_shell_cmd("drop", "Remove the specified package from the list", shell_drop)
     register_shell_cmd("write", "Write the specfile", shell_write)
-    print "ompi-spec-generator interactive shell"
-    print "Type 'help' to get more information about available commands."
+    print("ompi-spec-generator interactive shell")
+    print("Type 'help' to get more information about available commands.")
     while loop:
         try:
             cmd = raw_input("ompi-spec-generator> ")
@@ -612,9 +612,9 @@ def shell():
                 shell_cmd_func = shell_cmd["function"]
                 loop = shell_cmd_func(cmd)
             else:
-                print "Invalid command"
+                print("Invalid command")
         except Exception:
-            print "\n"
+            print("\n")
             continue
 
 
@@ -627,8 +627,8 @@ def shell():
 def write_specfile(build_packages):
     global params
     # create output file
-    print "--> Create output file"
-    verbose("     Open outout file : %(filename)s" % {"filename":params.output})
+    print("--> Create output file")
+    verbose("     Open output file : %(filename)s" % {"filename":params.output})
     specfile = open(params.output, 'w')
     verbose("     Write copyright header")
     specfile.write(copyright_template)
@@ -705,7 +705,7 @@ def write_specfile(build_packages):
 #
 ######################################################################
 def main():
-    # parse comand line parameters
+    # parse command line parameters
     global params
     param_parser = optparse.OptionParser()
     param_parser.add_option("-r", "--root", action="store", dest="root", default="../../../",help="Specify the top root directory of the Open MPI Sources.")
@@ -735,12 +735,12 @@ def main():
     if ( params.ompi_prefix == "/opt/openmpi" ):
         params.ompi_prefix = params.ompi_prefix + "/" + params.ompi_version
 
-    print "--> Using root " + params.root
+    print("--> Using root " + params.root)
     if params.output == None:
         params.output = params.ompi_name_prefix + params.ompi_name + "-" + params.ompi_version + ".spec"
 
     # find config files
-    print "--> Search for configuration files"
+    print("--> Search for configuration files")
     for root, dirs, files in os.walk(params.root):
         for f in files:
             if configext != os.path.splitext(f)[1]:
@@ -750,7 +750,7 @@ def main():
             verbose("     Found : " + cf)
 
     # parse config files
-    print "--> Parse config files"
+    print("--> Parse config files")
     for file in configfiles:
         verbose("     Parse " + file)
         # parse configfile
@@ -779,7 +779,7 @@ def main():
         return
 
     # filter packages
-    print "--> Select packages"
+    print("--> Select packages")
     build_packages = []
     # filter packages
     if params.packages != None:
@@ -795,7 +795,7 @@ def main():
         build_packages = packages.values()
 
     # do sanity check on the components
-    print "--> Sanity check packages"
+    print("--> Sanity check packages")
     for package in build_packages:
         verbose("     Check package " + package.getOption("name") )
         if params.debug:
@@ -806,7 +806,7 @@ def main():
     write_specfile(build_packages)
 
     # done
-    print "--> Finished."
+    print("--> Finished.")
 
 
 if ("__main__" == __name__):
diff --git a/contrib/dist/linux/openmpi.spec b/contrib/dist/linux/openmpi.spec
index e854b7b5192..cfaae7f1fe4 100644
--- a/contrib/dist/linux/openmpi.spec
+++ b/contrib/dist/linux/openmpi.spec
@@ -9,7 +9,7 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2006-2020 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2006-2022 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2013      Mellanox Technologies, Inc.
 #                         All rights reserved.
 # Copyright (c) 2015      Research Organization for Information Science
@@ -43,11 +43,6 @@
 #
 # shell$ rpmbuild ... --define 'install_in_opt 1' ...
 #
-# Or (a multi-token example):
-#
-# shell$ rpmbuild ... \
-#    --define 'configure_options CFLAGS=-g --with-openib=/usr/local/ofed' ...
-#
 #############################################################################
 
 # Define this if you want to make this SRPM build in
@@ -58,6 +53,19 @@
 # type: bool (0/1)
 %{!?install_in_opt: %define install_in_opt 0}
 
+# This specfile expects to find all required 3rd party packages
+# (Libevent, Hwloc, PMIx, PRRTE) externally, and will not use the
+# internal/embedded copies of these packages.  This behavior is
+# strongly recommended for packagers.  However, if you want to override
+# this behavior, change the definition below to 0.
+#
+# NOTE: This option will cause "--with-libevent=external
+# --with-hwloc=external --with-pmix=external --with-prrte=external" to
+# be added to the arguments to configure. If you wish to use different
+# CLI options, set this value to 0 and set configure_options to the
+# CLI options you want.
+%{!?all_external_3rd_party: %define all_external_3rd_party 1}
+
 # Define this if you want this RPM to install environment setup
 # shell scripts.
 # type: bool (0/1)
@@ -97,7 +105,7 @@
 %{!?mpi_selector_rpm_name: %define mpi_selector_rpm_name mpi-selector}
 # The location of the mpi-selector executable (can be a relative path
 # name if "mpi-selector" can be found in the path)
-# type: string (path to mpi-selector exectuable)
+# type: string (path to mpi-selector executable)
 %{!?mpi_selector: %define mpi_selector mpi-selector}
 
 # Should we build a debuginfo RPM or not?
@@ -138,7 +146,7 @@
 
 # On some platforms, Open MPI/SHMEM just flat-out doesn't work with
 # -D_FORTIFY_SOURCE (e.g., some users have reported that there are
-# problems on ioa64 platforms).  In this case, just turn it off
+# problems on ia64 platforms).  In this case, just turn it off
 # (meaning: this specfile will strip out that flag from the
 # OS-provided compiler flags).  We already strip out _FORTIFY_SOURCE
 # for non-GCC compilers; setting this option to 0 will *always* strip
@@ -146,14 +154,16 @@
 # type: bool (0/1)
 %{!?allow_fortify_source: %define allow_fortify_source 1}
 
-# Select md5 packing algorithm, that src.rpm created on one distro can be read on another.
+# Select md5 packing algorithm, that src.rpm created on one distro can
+# be read on another.
 %global _binary_filedigest_algorithm 1
 %global _source_filedigest_algorithm 1
 
-# Define this to 1 if you want to keep libtool achive files
+# Define this to 1 if you want to keep libtool archive files
 # Default is 0 (remove *.la files)
 # type: bool (0/1)
 %{!?install_libtool_archive: %define install_libtool_archive 0}
+
 #############################################################################
 #
 # Configuration Logic
@@ -177,8 +187,9 @@
 # Per advice from Doug Ledford at Red Hat, docdir is supposed to be in
 # a fixed location.  But if you're installing a package in /opt, all
 # bets are off.  So feel free to install it anywhere in your tree.  He
-# suggests $prefix/doc.
-%define _defaultdocdir /opt/%{name}/%{version}/doc
+# suggests $prefix/doc, but the GNU Autotools these days default to
+# $prefix/share/doc.
+%define _defaultdocdir /opt/%{name}/%{version}/share/doc
 
 # Also put the modulefile in /opt (unless the user already specified
 # where they want it to go -- the modulefile is a bit different in
@@ -186,6 +197,12 @@
 %{!?modulefile_path: %define modulefile_path /opt/%{name}/%{version}/share/openmpi/modulefiles}
 %endif
 
+%if %{all_external_3rd_party}
+%define _configure_3rd_party --with-libevent=external --with-hwloc=external --with-pmix=external --with-prrte=external
+%else
+%define _configure_3rd_party %{nil}
+%endif
+
 # Now that we have processed install_in_opt, we can see if
 # modulefile_path was not set.  If it was not, then set it to a
 # default value.
@@ -212,6 +229,7 @@
 %define __check_files %{nil}
 %endif
 
+# Set this to any options you want to pass in to configure.
 %{!?configure_options: %define configure_options %{nil}}
 
 %if !%{use_default_rpm_opt_flags}
@@ -245,6 +263,12 @@ BuildRoot: /var/tmp/%{name}-%{version}-%{release}-root
 %if %{disable_auto_requires}
 AutoReq: no
 %endif
+%if %{all_external_3rd_party}
+# If we require all external 3rd party packages, then assume the use
+# of the OS Libevent and Hwloc packages.
+BuildRequires: libevent-devel hwloc-devel
+Requires: libevent hwloc
+%endif
 %if %{install_modulefile}
 Requires: %{modules_rpm_name}
 %endif
@@ -284,6 +308,12 @@ Provides: openmpi-runtime = %{version}
 %if %{disable_auto_requires}
 AutoReq: no
 %endif
+%if %{all_external_3rd_party}
+# If we require all external 3rd party packages, then assume the use
+# of the OS Libevent and Hwloc packages.
+BuildRequires: libevent-devel hwloc-devel
+Requires: libevent hwloc
+%endif
 %if %{install_modulefile}
 Requires: %{modules_rpm_name}
 %endif
@@ -368,7 +398,7 @@ This subpackage provides the documentation for Open MPI/OpenSHMEM.
 
 #############################################################################
 #
-# Prepatory Section
+# Preparatory Section
 #
 #############################################################################
 %prep
@@ -465,7 +495,7 @@ CXXFLAGS="%{?cxxflags:%{cxxflags}}%{!?cxxflags:$RPM_OPT_FLAGS}"
 FCFLAGS="%{?fcflags:%{fcflags}}%{!?fcflags:$RPM_OPT_FLAGS}"
 export CFLAGS CXXFLAGS FCFLAGS
 
-%configure %{configure_options}
+%configure %{configure_options} %{_configure_3rd_party}
 %{__make} %{?mflags}
 
 
@@ -509,7 +539,7 @@ proc ModulesHelp { } {
    puts stderr "This module adds Open MPI/SHMEM v%{version} to various paths"
 }
 
-module-whatis   "Sets up Open MPI/SHMEM v%{version} in your enviornment"
+module-whatis   "Sets up Open MPI/SHMEM v%{version} in your environment"
 
 prepend-path PATH "%{_prefix}/bin/"
 prepend-path LD_LIBRARY_PATH %{_libdir}
@@ -595,7 +625,7 @@ EOF
 # always gives a 0 exit status.
 
 # First, find all the files
-rm -f all.files runtime.files remaining.files devel.files docs.files
+rm -f all.files runtime.files remaining.files devel.files
 find $RPM_BUILD_ROOT -type f -o -type l | \
    sed -e "s@$RPM_BUILD_ROOT@@" \
    > all.files | /bin/true
@@ -603,55 +633,26 @@ find $RPM_BUILD_ROOT -type f -o -type l | \
 # Runtime files.  This should generally be library files and some
 # executables (no man pages, no doc files, no header files).  Do *not*
 # include wrapper compilers.
-cat all.files | egrep '/lib/|/lib64/|/lib32/|/bin/|/etc/|/help-' > tmp.files | /bin/true
+cat all.files | \
+    egrep '/lib/|/lib64/|/lib32/|/bin/|/etc/|/help-' \
+    > tmp.files | /bin/true
 # Snip out a bunch of executables (e.g., wrapper compilers, pkgconfig
-# files, .la and .a files)
-egrep -vi 'mpic|mpif|ortec|f77|f90|pkgconfig|\.la$|\.a$' tmp.files > runtime.files | /bin/true
+# files, .la and .a files) and docs
+egrep -vi 'bin/mpic|bin/mpif|bin/mpif77|bin/mpif90|pkgconfig|wrapper|\.mod$|\.la$|\.a$' tmp.files > runtime.files | /bin/true
+
 rm -f tmp.files
 
 # Now take the runtime files out of all.files so that we don't get
 # duplicates.
 grep -v -f runtime.files all.files > remaining.files
 
-# Devel files, potentially including VT files.  Basically -- just
-# exclude the man pages and doc files.
+# Devel files.  Basically -- just exclude the man pages and doc files.
 cat remaining.files | \
    egrep -v '/man/|/doc/' \
    > devel.files | /bin/true
 
-# Now take those files out of reaming.files so that we don't get
-# duplicates.
-grep -v -f devel.files remaining.files > docs.files
-
-#################################################
-
-# Now that we have a final list of files for each of the runtime,
-# devel, and docs RPMs, snip even a few more files out of those lists
-# because for directories that are wholly in only one RPM, we just
-# list that directory in the file lists below, and RPM will pick up
-# all files in that tree.  We therefore don't want to list any files
-# in those trees in our *.files file lists.  Additionally, the man
-# pages may get compressed by rpmbuild after this "install" step, so we
-# might not even have their final filenames, anyway.
-
-# runtime sub package
-%if !%{sysconfdir_in_prefix}
-grep -v %{_sysconfdir} runtime.files > tmp.files | /bin/true
-mv tmp.files runtime.files
 %endif
-grep -v %{_pkgdatadir} runtime.files > tmp.files | /bin/true
-mv tmp.files runtime.files
-
-# devel sub package
-grep -v %{_includedir} devel.files > tmp.files | /bin/true
-mv tmp.files devel.files
-
-# docs sub package
-grep -v %{_mandir} docs.files > tmp.files | /bin/true
-mv tmp.files docs.files
-
-%endif
-# End of build_all_in_one_rpm
+# End of !build_all_in_one_rpm
 
 #############################################################################
 #
@@ -713,7 +714,7 @@ test "x$RPM_BUILD_ROOT" != "x" && rm -rf $RPM_BUILD_ROOT
 %{_bindir}/*
 %{_includedir}/*
 %{_libdir}/*
-%{_datadir}
+%{_datadir}/*
 %else
 %{_prefix}
 %endif
@@ -721,7 +722,7 @@ test "x$RPM_BUILD_ROOT" != "x" && rm -rf $RPM_BUILD_ROOT
 %if !%{sysconfdir_in_prefix}
 %{_sysconfdir}
 %endif
-# If %{install_in_opt}, then we're instaling OMPI to
+# If install_in_opt, then we're installing OMPI to
 # /opt/openmpi/<version>.  But be sure to also explicitly mention
 # /opt/openmpi so that it can be removed by RPM when everything under
 # there is also removed.
@@ -737,33 +738,30 @@ test "x$RPM_BUILD_ROOT" != "x" && rm -rf $RPM_BUILD_ROOT
 %{shell_scripts_path}/%{shell_scripts_basename}.sh
 %{shell_scripts_path}/%{shell_scripts_basename}.csh
 %endif
-%doc README.md INSTALL LICENSE
+%doc README.md LICENSE
 
 %else
 
 #
 # Sub-package RPMs
 #
-# Harder than all-in-one.  We list the directories specifically so
-# that if the RPM creates directories when it is installed, we will
-# remove them when the RPM is uninstalled.  We also have to use
-# specific file lists.
-#
+# Harder than all-in-one.  Explicitly list all the files we want by
+# the various *.files we generated above.  Only list directories if
+# they are not /usr (or assumedly any other location that does not
+# already exist).
 
 %files runtime -f runtime.files
 %defattr(-, root, root, -)
-%if %(test "%{_prefix}" = "/usr" && echo 1 || echo 0)
-%{_bindir}/*
-%{_libdir}/*
-%{_datadir}
-%else
-%{_prefix}
-%endif
+%if %(test "%{_prefix}" != "/usr" && echo 1 || echo 0)
+%dir %{_prefix}
+%dir %{_pkgdatadir}
 # If the sysconfdir is not under the prefix, then list it explicitly.
 %if !%{sysconfdir_in_prefix}
-%{_sysconfdir}
+%dir %{_sysconfdir}
+%endif
 %endif
-# If %{install_in_opt}, then we're instaling OMPI to
+
+# If install_in_opt, then we're installing OMPI to
 # /opt/openmpi/<version>.  But be sure to also explicitly mention
 # /opt/openmpi so that it can be removed by RPM when everything under
 # there is also removed.  Also list /opt/openmpi/<version>/share so
@@ -781,21 +779,23 @@ test "x$RPM_BUILD_ROOT" != "x" && rm -rf $RPM_BUILD_ROOT
 %{shell_scripts_path}/%{shell_scripts_basename}.sh
 %{shell_scripts_path}/%{shell_scripts_basename}.csh
 %endif
-%doc README.md INSTALL LICENSE
-%{_pkgdatadir}
+%doc README.md LICENSE
 
 %files devel -f devel.files
 %defattr(-, root, root, -)
-%{_includedir}
 
+%files docs
+%defattr(-, root, root, -)
 # Note that we list the mandir specifically here, because we want all
 # files found in that tree, because rpmbuild may have compressed them
-# (e.g., foo.1.gz or foo.1.bz2) -- and we therefore don't know the
-# exact filenames.
-%files docs -f docs.files
-%defattr(-, root, root, -)
+# after we installed them (e.g., foo.1.gz or foo.1.bz2), and we
+# therefore don't know the exact filenames.
 %{_mandir}
 
+# Explicitly list the pkgdocdir so that it will be removed when the
+# RPM is removed.
+%{_pkgdocdir}
+
 %endif
 
 
@@ -805,6 +805,15 @@ test "x$RPM_BUILD_ROOT" != "x" && rm -rf $RPM_BUILD_ROOT
 #
 #############################################################################
 %changelog
+* Mon Apr 04 2022 Jeff Squyres <jsquyres@cisco.com>
+- Updates for v5.0.0:
+  - Default all 3rd-party packages to be "external".
+  - Remove no-longer-existing INSTALL file.
+  - Tighten up files listings for the individual RPMs.
+  - Convert -docs sub-package to exclusively use mandir and pkgdocdir
+    (instead of docs.files).
+  - Remove some stale F77 and VT references.
+
 * Tue Mar 28 2017 Jeff Squyres <jsquyres@cisco.com>
 - Reverting a decision from a prior changelog entry: if
   install_in_opt==1, then even put the modulefile under /opt.
@@ -862,11 +871,11 @@ test "x$RPM_BUILD_ROOT" != "x" && rm -rf $RPM_BUILD_ROOT
 
 * Tue Sep  8 2009 Jeff Squyres <jsquyres@cisco.com>
 - Change shell_scripts_basename to not include version number to
-  accomodate what mpi-selector expects.
+  accommodate what mpi-selector expects.
 
 * Mon Feb  4 2008 Jeff Squyres <jsquyres@cisco.com>
 - OFED 1.3 has a much better installer; remove all the
-  leave_build_root kludge nastyness.  W00t!
+  leave_build_root kludge nastiness.  W00t!
 
 * Fri Jan 18 2008 Jeff Squyres <jsquyres@cisco.com>
 - Remove the hard-coded "openmpi" name from two Requires statements
@@ -909,7 +918,7 @@ test "x$RPM_BUILD_ROOT" != "x" && rm -rf $RPM_BUILD_ROOT
   shell script files be %{_bindir}/mpivars.{sh|csh}
 - Add %{munge_build_into_install} option for OFED 1.2 installer on SLES
 - Change shell script files and modulefile to *pre*pend all the OMPI paths
-- Make shell script and modulefile installation indepdendent of
+- Make shell script and modulefile installation independent of
   %{install_in_opt} (they're really separate issues)
 - Add more "ofed" shortcut qualifiers
 - Slightly better test for basename CC in the fortify source section
@@ -959,4 +968,3 @@ test "x$RPM_BUILD_ROOT" != "x" && rm -rf $RPM_BUILD_ROOT
 
 * Wed Mar 23 2005 Mezzanine <mezzanine@kainx.org>
 - Specfile auto-generated by Mezzanine
-
diff --git a/contrib/dist/make-html-man-pages.pl b/contrib/dist/make-html-man-pages.pl
deleted file mode 100755
index 58f7679638c..00000000000
--- a/contrib/dist/make-html-man-pages.pl
+++ /dev/null
@@ -1,278 +0,0 @@
-#!/usr/bin/env perl
-#
-# Copyright (c) 2010 Cisco Systems, Inc.
-#
-# Script to generate PHP-ized files of Open MPI tarball-installed man
-# pages.  Run it from the top-level directory of an Open MPI tarball
-# or source checkout.  It will:
-#
-# - run autogen if necessary
-# - run configure
-# - run make install
-# - frob the generated man pages a bit
-# - generate PHP versions of the man pages
-#
-# The PHP can then be copied to the OMPI web site.
-#
-
-use strict;
-use File::Find;
-use File::Basename;
-use Cwd;
-
-sub absoluteize {
-    my ($dir) = shift;
-
-    mkdir_p($dir)
-        if (! -d $dir);
-
-    my $start = cwd();
-    chdir($dir);
-    $dir = cwd();
-    chdir($start);
-
-    return $dir;
-}
-
-sub mkdir_p {
-    my ($dir) = @_;
-    if (!mkdir($dir)) {
-        mkdir_p(dirname($dir));
-        mkdir($dir) || die "Can't make directory $dir";
-    }
-}
-
-# Check that we have what we need
-if (!(-f "VERSION" && -f "ompi/include/mpi.h.in")) {
-    print "Run this script from the top-level Open MPI directory\n";
-    exit(1);
-}
-
-# Setup
-my @files;
-my $pwd = Cwd::cwd();
-print "PWD: $pwd\n";
-my $basedir = "$pwd/man-page-generator";
-my $prefix = "$basedir/install";
-my $mandir = absoluteize("$prefix/share/man");
-my $outdir_base = absoluteize("$basedir/php");
-
-# Remove old results
-system("rm -rf $basedir");
-
-# Configure, build, and install so that we get a full set of man pages
-
-sub doit {
-    my @cmd = @_;
-    print "Running: @cmd\n";
-    my $ret = system(@cmd);
-    die "Failed to run (@cmd)"
-        if (-1 == $ret);
-    $ret = $ret >> 8;
-    die "Command failed (@cmd) with status $ret"
-        if ($ret != 0);
-}
-
-# Autogen if we don't have a configure script
-doit("./autogen.pl")
-    if (! -x "configure");
-doit("./configure --prefix=$prefix --enable-mpi-ext=all --without-cs-fs");
-
-# Find this OMPI's version
-my $version = `fgrep PACKAGE_VERSION opal/include/opal_config.h | cut -d\\\" -f2`;
-chomp($version);
-print "Open MPI version: $version\n";
-
-# Build so that we get fresh man pages
-doit("make clean");
-doit("make -j 4 install");
-
-# Find all *.[0-9] files in the installed mandir tree.
-&File::Find::find(
-    sub {
-        push(@files, $File::Find::name) if (-f $_ && $_ =~ /\.[1-9]$/);
-    }, $mandir);
-
-# Must cd into the $mandir directory because some of the man pages
-# refer to other man pages by "man/<filename>" relative path names.
-chdir($mandir);
-
-my %dirs;
-my $outfiles;
-
-# Generate a PHP file for each man page.
-foreach my $file (@files) {
-    my $b = basename($file);
-    $b =~ m/(.*)\.([0-9])$/;
-
-    my $name = $1;
-    my $section = $2;
-
-    my $outdir = "$outdir_base/man$section";
-    my $outfile = "$outdir/$b.php";
-    $dirs{$outdir} = "";
-    push(@{$outfiles->{$section}}, {
-        name => $name,
-        file => "man$section/$b.php",
-         });
-
-    mkdir_p($outdir)
-        if (! -d $outdir);
-
-    print "*** Generating: $name ($section)\n";
-
-    # man2html clips many of our man pages -- it just stops halfway
-    # through the file.  Weird.
-    #print "man $file | man2html -bare -botm 4 -topm 4\n";
-    open(CMD, "rman -f HTML -r off -p $file|") || die("Can't open command");
-    my $text;
-    $text .= $_
-        while (<CMD>);
-    close(CMD);
-
-    # Post-process the text:
-    # Remove <head> ... </head>
-    # Remove <!doctype ...>
-    # Remove <meta ...>
-    # Remove <style ...> ... </style>
-    # Remove <title> ... </title>
-    # Remove <html> and </html>
-    # Remove <body> and </body>
-
-    $text =~ s/<head>.*<\/head>//is;
-    $text =~ s/<!doctype.*?>//is;
-    $text =~ s/<html>//i;
-    $text =~ s/<\/html>//i;
-    $text =~ s/<body>//i;
-    $text =~ s/<\/body>//i;
-
-    # Remove carriage returns, extra white space, and double blank
-    # lines
-    $text =~ s/\r//g;
-    $text =~ s/[ \t]+\n/\n/g;
-    $text =~ s/\n{3,}/\n\n/g;
-
-    # Cross-link to any other man pages that we might have.  Search
-    # through the string for MPI_<foo> and look for any corresponding
-    # man pages in @files.  Sequentially replace MPI_<foo> with
-    # $replaced<foo> so that we can find all the MPI_<foo>'s (we'll
-    # put the "MPI_" back when we're ).
-    my $replace = "ZAZZZERZAZ_";
-
-    # This is a doozy of a regexp (see perlre(1)).  Look for MPI_<foo>
-    # cases that aren't followed by .[0-9].php (i.e., not the href
-    # clause of an A HTML tag).
-    while ($text =~ m/^(.*\W)MPI_(\w+(?!\.[0-9]\.php))(\W.*)$/s) {
-        my $comp = lc("mpi_$2");
-#        print "Found: $2 -- looking for $comp: ";
-
-        my $prefix = $1;
-        my $meat = $2;
-        my $suffix = $3;
-
-        my $replaced = 0;
-        foreach my $f2 (@files) {
-            # If we have another file that matches the regexp that we
-            # just pulled out from the text, *and* that file is not
-            # the same file that we're already processing (i.e., don't
-            # link to myself!), then link to it.
-            if (basename(lc($f2)) =~ /^$comp\.[0-9]/ && $f2 ne $file) {
-                # Hard-coded to link only to MPI API functions in
-                # section 3 (i.e., ../man3/<foo>).
-                my $link_file = "../man3/" . basename($f2) . ".php";
-#                print "Linked to $link_file!\n";
-                $text = "$prefix<a href=\"$link_file\">$replace$meat</a>$suffix";
-                $replaced = 1;
-                last;
-            }
-        }
-        if (!$replaced) {
-#            print "Not linked\n";
-            $text = "$prefix$replace$meat$suffix";
-        }
-    }
-    # Now replace the $replaced back with MPI_.
-    $text =~ s/$replace/MPI_/g;
-
-    # Obscure any email addresses in there; don't want to give the
-    # spammers any new fodder!
-    $text =~ s/(\W)[\w\.\-]+@[\w.\-]+(\W)/$1email-address-removed$2/g;
-
-    # Setup meta name: make the MPI name be all caps if we're in
-    # section 3 and it has an MPI_ prefix.
-    my $meta_name = $name;
-    if (3 == $section && $name =~ /^MPI_/) {
-        $meta_name = uc($name);
-    }
-
-    # Now we're left with what we want.  Output the PHP page.
-    # Write the output PHP file with our own header and footer,
-    # suitable for www.open-mpi.org.
-    unlink($outfile);
-    open(FILE, ">$outfile") || die "Can't open $outfile";
-    print FILE '<?php
-$topdir = "../../..";
-$title = "' . "$name($section) man page (version $version)" . '";
-$meta_desc = "Open MPI v' . "$version man page: $meta_name($section)" . '";
-
-include_once("$topdir/doc/nav.inc");
-include_once("$topdir/includes/header.inc");
-?>
-<p> <a href="../">&laquo; Return to documentation listing</a></p>
-' . $text . '
-<p> <a href="../">&laquo; Return to documentation listing</a></p>
-<?php
-include_once("$topdir/includes/footer.inc");
-';
-    close(FILE);
-}
-
-# Now write out an index.php file in each subdir
-foreach my $dir (keys(%dirs)) {
-    print "Writing index file in $dir...\n";
-    open(FILE, ">$dir/index.php") || die "Can't open $dir/index.php";
-    print FILE '<?php header("Location: ..");
-';
-    close(FILE);
-}
-
-# Now write out a top-level data-<version>.inc file
-my $file = "$outdir_base/data-$version.inc";
-print "Writing $file...\n";
-open(FILE, ">$file") || die "Can't open $file";
-print FILE '<?php
-$version = "' . $version . '";
-
-';
-
-foreach my $section (sort(keys(%{$outfiles}))) {
-    print FILE '$files[] = "' . $section . '";
-$files_' . $section . ' = array(';
-    my $first = 1;
-    my @f;
-    # The hash isn't sorted, so build up an array of just the
-    # filenames
-    foreach my $file (@{$outfiles->{$section}}) {
-        push(@f, $file->{name});
-    }
-    # Now output the sorted filenames
-    foreach my $file (sort(@f)) {
-        print FILE ", "
-            if (!$first);
-        $first = 0;
-        print FILE '"' . $file . '"';
-    }
-    print FILE ");\n\n";
-}
-close(FILE);
-
-# Print the top-level engine file for this version (it will use the
-# data-<version>.inc file).
-open(FILE, ">$outdir_base/index.php") || die "Can't open $outdir_base/index.php";
-print FILE '<?php
-$topdir = "../..";
-include_once("data-' . $version . '.inc");
-include_once("../engine.inc");
-';
-close(FILE);
-
diff --git a/contrib/dist/make_dist_tarball b/contrib/dist/make_dist_tarball
index f009e3f7e9d..94186714818 100755
--- a/contrib/dist/make_dist_tarball
+++ b/contrib/dist/make_dist_tarball
@@ -10,7 +10,7 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2008-2015 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2008-2023 Cisco Systems, Inc.  All rights reserved
 # Copyright (c) 2016      Intel, Inc. All rights reserved
 # $COPYRIGHT$
 #
@@ -23,11 +23,11 @@
 # Version of auto tools that we want
 #
 
-M4_TARGET_VERSION=1.4.17
-AM_TARGET_VERSION=1.15
-AC_TARGET_VERSION=2.69
+M4_TARGET_VERSION=1.4.19
+AM_TARGET_VERSION=1.16.5
+AC_TARGET_VERSION=2.71
 LT_TARGET_VERSION=2.4.6
-FLEX_TARGET_VERSION=2.5.35
+FLEX_TARGET_VERSION=2.6.4
 
 #
 # When running "make distcheck", use these parallelization flags.  Can
diff --git a/contrib/doc-ft-components/DEVEL.FT-REQUIREMENTS.md b/contrib/doc-ft-components/DEVEL.FT-REQUIREMENTS.md
new file mode 100644
index 00000000000..863f3c1ad20
--- /dev/null
+++ b/contrib/doc-ft-components/DEVEL.FT-REQUIREMENTS.md
@@ -0,0 +1,777 @@
+ULFM OPEN MPI COMPONENTS DESIGN AND REQUIREMENTS FOR FAULT TOLERANT OPERATION
+=============================================================================
+
+This documents the requirements and best practices for Open MPI
+components to support operating across failures with the
+**User Level Failure Mitigation (ULFM)** Open MPI implementation.
+___________________________________________________________________________
+[TOC]
+___________________________________________________________________________
+
+Introduction
+============
+
+This document concerns itself with the support of the User Level Failure
+Mitigation (ULFM) MPI Standard draft proposal in Open MPI. The ULFM proposal
+is developed by the MPI Forum's Fault Tolerance Working Group to support
+the continued operation of MPI programs after crash (node failures) have
+impacted the execution. The key principle is that no MPI call (point-to-point,
+collective, RMA, IO, ...) can block indefinitely after a failure, but must
+either succeed or raise an MPI error.
+
+Open MPI contains the general infrastructure to support this mode of operation.
+MPI operations may produce three supplementary error codes and the recovery
+of the MPI capability for communicating with five supplementary interfaces
+(see [http://fault-tolerance.org/wp-content/uploads/2012/10/20170221-ft.png]
+(ULFM chapter) standard draft document).
+
++ `MPIX_ERR_PROC_FAILED` when a process failure prevents the completion of
+  an MPI operation.
++ `MPIX_ERR_PROC_FAILED_PENDING` when a potential sender matching a non-blocking
+  wildcard source receive has failed.
++ `MPIX_ERR_REVOKED` when one of the ranks in the application has invoked the
+  `MPI_Comm_revoke` operation on the communicator.
++ `MPIX_Comm_revoke(MPI_Comm comm)` Interrupts any communication pending on
+  the communicator at all ranks.
++ `MPIX_Comm_shrink(MPI_Comm comm, MPI_Comm* newcomm)` creates a new
+  communicator where dead processes in comm were removed.
++ `MPIX_Comm_agree(MPI_Comm comm, int *flag)` performs a consensus (i.e. fault
+  tolerant allreduce operation) on flag (with the operation bitwise or).
++ `MPIX_Comm_failure_get_acked(MPI_Comm, MPI_Group*)` obtains the group of
+  currently acknowledged failed processes.
++ `MPIX_Comm_failure_ack(MPI_Comm)` acknowledges that the application intends
+  to ignore the effect of currently known failures on wildcard receive
+  completions and agreement return values.
+
+While the general infrastructure for fault tolerance support is available in
+Open MPI, communication components (i.e., BTL, PML, MTL components) need to
+refrain from aborting the whole application, and return
+errors for fault event detected during communication operations to the upper
+layers of the Open MPI infrastructure to handle the error condition.
+The following sections describe specific requirements for the named area.
+
+Pathways for reporting failures
+===============================
+
+Process failures can be detected through two separate mechanisms as illustrated
+in the following figure:
+
+![Fault detection pathways](./general-ulfm-fault-handling.png)
+
+1. The network transport is incapable of completing some communication operation
+   with the peer (send, recv, atomic operation, etc.). The network transport
+   component (BTL, MTL) should report the failure to the PML, so that the PML
+   has a chance of reacting, performing necessary cleanup, and initiate recovery
+   actions as needed by triggering the `errhandler_proc_failed_internal` function.
+
+2. The runtime environment (PRTE) has identified a peer failure. A PMIx event
+   notification containing the identification of the impacted processes will be
+   received from the PMIx thread, and transitioned as an event in the main
+   OMPI progress loop. This event will cause the OMPI errhandler function
+   `errhandler_proc_failed_internal` to be called.
+
+Error Completion and Requests
+-----------------------------
+
+Once a failure has been reported, MPI will have to complete requests that are
+representing communication operations with the failed process(es). The operation
+may involve the failed process directly (a receive from a failed process), or
+indirectly (a collective operation in which a failed process should participate;
+an ANY_SOURCE receive).
+
+The `errhandler_proc_failed_internal` function considers all communicators in turn
+and if a failed process appears in the group of the communicator, 1) pending
+any-source receive on comm return an error; 2) collective operations
+on comm will be **revoked**, that is, complete at all ranks (in error at some of the
+ranks, but maybe not all); 3) point-to-point communication with a failed peer are
+completed in error.
+
+Request fields used for Fault Tolerance
+---------------------------------------
+
+In order to track the error status of a request, some fields of a request must always
+be initialized and updated correctly.
+
+* `req->req_status.MPI_ERROR`: The error status field of the request needs to be
+  initialized to `OMPI_SUCCESS` when the request is allocated. This field will be
+  updated by `ompi_request_wait` and similar functions to mark the request as in
+  need of attention. Of note: this is the internal status of the request, and
+  it can be initialized without undesirable side effect on the `MPI_STATUS` object
+  returned from `MPI_WAIT` (and similar).
+
+* `req->req_type`: Fault tolerance operations support the following types of requests:
+  `OMPI_REQUEST_PML` (a point-to-point request), `OMPI_REQUEST_COLL` (a non-blocking
+  collective communication request), and `OMPI_REQUEST_COMM` (a communicator creation
+  request, e.g., `ompi_comm_dup`, `ompi_comm_idup`). Other types of requests are not yet
+  supported and if your component requires such support, cases handling new types of
+  requests must be added in `ompi/request/req_ft.c::ompi_request_is_failed_fn`.
+
+* `req->req_mpi_object`: This field must contain the `comm` on which the request operates.
+  This is true for PML, NBC, as well as COMM types of requests (see above).
+
+* `pml_req->req_peer`: for a PML type request, this field must contain the `peer` that the
+  request is trying to communicate with. This field is used to identify requests that
+  directly involve failed processes.
+
+* `pml_req->req_tag`: for a PML type request, this field is used to identify requests that
+  represent collective operations (refer to `ompi_request_tag_is_collective()` for the
+  tag space), or resilient operations, i.e., for implementing internal resilient operations,
+  like `MPI_COMM_AGREE`, `MPI_COMM_SHRINK`, etc. (refer to `ompi_request_tag_is_ft()`).
+
+Wait-sync
+---------
+
+The **wait_sync** Multithreaded wait-synchronization object is an optimization that reduces
+concurrent active polling in `ompi_request_wait` and other similar functions. In order for
+requests to be rechecked for fault error condition (and hence completed in error), the
+wait_sync must be interrupted. Note that all sync objects will be interrupted upon an
+error. Request whose state is not failed will return into a new sync object; request
+whose state is failed will complete in error and return control to the user. The
+`sync->status` field will contain the reason for interrupting the sync object.
+
+Modifications to the wait-sync logic must pay attention to not damage the error path.
+That is, a meaningful error must be present in `sync->status` when an error condition is
+found, and the `wait_sync_global_wakeup()` function does require tracking of all currently
+active `sync` objects. In the `pthreads` sync implementation, the management of multiple
+threads each using a different `sync` object also uses a list of active `sync`, which is
+parsed by the global wakeup function. The only needed adaptation for supported
+fault tolerance in the `pthreads sync `is to also populate the `sync` list in the
+single-threaded code path (so that it can be found by global wakeup using the same
+mechanism as for the multi-threaded case).
+
+BTL components
+==============
+
+In order to support fault tolerance, BTL components must provide the following functionality:
+
+Driver error capture
+--------------------
+
+Capture and convert driver error codes returned from the NIC driver operations (e.g.,
+`fi_write`). Most driver functions provide an error code that indicate error conditions on a
+per-operation basis by default. Some network drivers may require setting the endpoint in a
+special mode to enable error reporting. For example, the `UCX` driver aborts within the driver
+function itself if the endpoint is not created with the `UCT_IFACE_PARAM_FIELD_ERR_HANDLER` mode.
+
+Driver error verbosity
+----------------------
+
+The BTL that encounters a process failure or unexpected peer disconnect can output an error
+message to the console using the `BTL_PEER_ERROR` macro. Non process fault error should use
+the `BTL_ERROR` macro instead. When fault tolerance is inactive (default), both macros are
+equally verbose and will inform the end-user of the details of the communication issue.
+However, when fault tolerance is active, process failure events are correctible, and
+verbose error reporting is generally undesired at the BTL level. Thus, unlike the
+`BTL_ERROR` macro, the `BTL_PEER_ERROR` macro is silent when fault tolerance is active.
+For debugging purposes, the messages produced by `BTL_PEER_ERROR` can be enabled with
+fault tolerance by either setting the `btl_base_verbose` MCA to a value greater than 1,
+or by setting the `"btl_base_warn_peer_error"` to `true`.
+
+Do not abort within the component
+---------------------------------
+
+Refrain from aborting within the component itself when an error is reported by a driver
+function. Driver errors should be converted to an appropriate `OPAL_ERR_XYZ` and transmitted
+to the caller component. For example, in the BTL `ofi` component, every error would be
+treated as fatal by calling the `MCA_BTL_OFI_ABORT()`. As a replacement, the `ofi` BTL has
+been adapted to replace these aborting calls with `BTL_PEER_ERROR`, followed by
+`mca_btl_ofi_context_finalize()` to clean the endpoint associated with the error peer,
+before returning the error code `OPAL_ERR_UNREACH` to the caller.
+
+The following code sample shows one place where the OFI BTL is adapted to avoid aborting
+when a CQ read error occurs, output an error message with the appropriate verbosity,
+perform endpoint cleanup, and return an appropriate error code to the upper layer. This
+change (and similar changes at other places) are sufficient to enable resumed operation
+of OB1 over the OFI BTL, and thus fault recovery.
+
+    diff --git a/opal/mca/btl/ofi/btl_ofi_context.c b/opal/mca/btl/ofi/btl_ofi_context.c
+    index bd399c62..ffafb96c 100644
+    --- a/opal/mca/btl/ofi/btl_ofi_context.c
+    +++ b/opal/mca/btl/ofi/btl_ofi_context.c
+    @@ -437,13 +437,14 @@ int mca_btl_ofi_context_progress(mca_btl_ofi_context_t *context) {
+
+         /* cq readerr failed!? */
+         if (0 > ret) {
+    -            BTL_ERROR(("%s:%d: Error returned from fi_cq_readerr: %s(%d)",
+    +            BTL_PEER_ERROR(NULL, ("%s:%d: Error returned from fi_cq_readerr: %s(%d)",
+                        __FILE__, __LINE__, fi_strerror(-ret), ret));
+             } else {
+    -            BTL_ERROR(("fi_cq_readerr: (provider err_code = %d)\n",
+    +            BTL_PEER_ERROR(NULL, ("fi_cq_readerr: (provider err_code = %d)\n",
+                            cqerr.prov_errno));
+             }
+    -        MCA_BTL_OFI_ABORT();
+    +        mca_btl_ofi_context_finalize(context, context->rx_ctx != context->tx_ctx);
+    +        return OPAL_ERR_UNREACH;
+         }
+     #ifdef FI_EINTR
+         /* sometimes, sockets provider complain about interrupt. We do nothing. */
+
+Complete affected fragments
+---------------------------
+
+Closing the affected endpoint should have the effect of completing the associated
+fragments pending on the endpoint. If fragments remain pending in the network interface,
+RDMA operations may still be updating the user buffers and prevent the PML level from
+releasing the associated MPI requests. Thus, the BTL should complete and cleanup
+pending fragments and update the associated requests so that request completion
+(potentially in error) can be safely reported by the PML layer to the end-user.
+
+Use the error callback mechanism
+--------------------------------
+
+The BTL should invoke the error callback registered by the PML when a process error is
+produced within the component. The BTL module contains the API function `btl_register_error`.
+The PML will attach a callback for managing the error at the PML level. A good example of
+using the error callback is found in the TCP BTL. When an error is produced within the
+TCP BTL, the endpoint is marked with a special internal flag, and is then closed. Upon
+closing an endpoint, the error status is checked, and if an error condition caused the
+endpoint closing, the registered error callback is triggered.
+
+MTL Components
+==============
+
+In general terms, the same functionalities must be provided by MTL components to support
+fault tolerance. The MTL component should not abort internally. The peer failure should be
+visible to the `cm` PML through error codes passed in the return values of the MTL module calls.
+Most of the actual management of the error is expected to happen at the level of the `cm` PML.
+
+PML Components
+==============
+
+Capturing errors with the PML
+-----------------------------
+
+The PML should not abort internally, but instead set error codes in the internal status
+of requests that have been impacted by a failure. The PML can capture errors from the
+BTL components through the registered error callback that can be attached using the
+`bml_register` interface. This callback is a prime position in the software stack to
+transfer the handling of process failure errors to the cleanup and recovery part of the
+MPI library, i.e., to invoke the errhandler `errhandler_proc_failed_internal` function.
+
+This function will then be in charge of updating requests, proc arrays, communicators, etc,
+and will also call-back into the PML to trigger the cleanup of PML level structures and
+matching queues in a generic and organized way.
+
+Cancellation of failed requests
+-------------------------------
+
+When a request is identified as being in error due to a process failure, it needs to be
+completed. This is achieved by having MPI completing calls (e.g., MPI_WAIT) to reconsider
+the status of the requests. If a request is found to be in an error state, the request is
+first cancelled, and then completed in error. The reason for cancelling the request instead
+of directly completing it is to leave an opportunity for the BTL to purge pending fragments
+and RDMA orders on the buffer associated with the request, hence ensuring that the request
+is `pml_complete` before it is marked as `mpi_complete` (to enforce that the user-buffer is not
+updated past the completion of the request).
+
+A consequence is that it is expected that the PML supports the cancellation of requests
+that communicate with failed processes. The following code-snippet illustrates how the
+cancellation of requests in error is managed in the OB1 PML. Note how general support for
+cancellation of request to/from non-failed processes is not required.
+
+    if( true != request->req_match_received ) { /* the match has not been already done */
+        assert( OMPI_ANY_TAG == ompi_request->req_status.MPI_TAG ); /* not matched isn't it */
+        if( request->req_recv.req_base.req_peer == OMPI_ANY_SOURCE ) {
+            opal_list_remove_item( &ob1_comm->wild_receives, (opal_list_item_t*)request );
+        } else {
+            mca_pml_ob1_comm_proc_t* proc = mca_pml_ob1_peer_lookup (comm, request->req_recv.req_base.req_peer);
+            opal_list_remove_item(&proc->specific_receives, (opal_list_item_t*)request);
+        }
+    }
+    else { /* it has matched */
+        if( ompi_comm_is_proc_active( comm, request->req_recv.req_base.req_peer,
+                                              OMPI_COMM_IS_INTER(comm) ) ) {
+            opal_output_verbose(10, ompi_ftmpi_output_handle,
+                                "Recv_request_cancel: cancel denied for request %p because it has matched peer %d\n",
+                                (void*)request, request->req_recv.req_base.req_peer);
+            return OMPI_SUCCESS;
+        }
+        else {
+            /* This process is dead, therefore this request is complete */
+            opal_output_verbose(10, ompi_ftmpi_output_handle,
+                                "Recv_request_cancel: cancel granted for request %p because peer %d is dead\n",
+                                (void*)request, request->req_recv.req_base.req_peer);
+        }
+    }
+    /**
+     * As now the PML is done with this request we have to force the pml_complete
+     * to true. Otherwise, the request will never be freed.
+     */
+
+    ompi_request->req_status._cancelled = true;
+    recv_request_pml_complete(request);
+
+Supporting the Revoke operation
+-------------------------------
+
+The `MPIX_COMM_REVOKE` operation lets the user interrupt all ongoing operation on `comm`.
+The principal use case is to stop the ongoing communication pattern when a process
+failure error is reported at a given rank. Once a rank has experienced an error,
+it is likely that it needs to stop communicating according to the pre-fault
+communication pattern, and must instead enter a distinct communication pattern
+to recover MPI object or application dataset. However, another process that has
+a pending operation that peer with that rank is at risk of deadlocking in that
+case: its pending operation does not necessarily peer with a failed process, thus may
+not be interrupted with an `MPI_ERR_PROC_FAILED` error. Yet, that operation will not be
+matched, hence it may never complete. The solution is for the rank that initially
+experienced the process fault error to call the `MPIX_COMM_REVOKE` operation when it
+is intent in abandoning an ongoing communication pattern.
+
+The `MPIX_COMM_REVOKE` operation will cause all operations on `comm` (pending or future)
+to be interrupted with the specific error class `MPI_ERR_REVOKED`. This in turn will
+ensure that no process deadlock on unmatched operations.
+
+It is unlikely that a developer will need to concern himself with the reliable
+broadcast algorithm that propagates the revoke order between the processes of
+a communicator. On the other hand, when a communicator is revoked, the PML needs to
+identify which requests need to be interrupted, and do so in an orderly fashion.
+
+When a communicator is revoked, the new PML module function `pml_revoke_comm()` is
+called. This function considers the unexpected fragments and matching queues, and
+makes a determination of how to best interrupt the corresponding requests. A
+difficult part of this operation is that the communication must be interrupted
+in a state that is consistent between the two peers. For example, if a send
+request is revoked before completion, it is mandatory that the corresponding
+recv request is also revoked (otherwise a deadlock would arise). Multiple
+implementation choices are possible, but lets describe how the PML OB1 cleans
+ongoing requests. The OB1 implementation makes a determination of when to
+cancel an ongoing communication by considering the state of the request at
+the receiver. There are 3 cases:
+
+1. Send requests are never directly revoked. The decision to revoke a particular
+   communication is taken at the receiver end, and the communication protocol
+   will inform the sender of what actions to take if the receiver decided to revoke
+   the communication.
+2. The receive request is not present in the matching queue anymore (it has matched,
+   and the communication protocol has advanced to the next stage (possibly RDMA
+   operations at the NIC level). Cancelling this case can be very difficult and is not
+   always supported by all network types. For this reason, in this case, the operation
+   is allowed to proceed to completion. Since the request has already matched, as long
+   as the sender does not fail, this request will complete.
+3. If the receive request is still present in the matching queue (that is, the
+   request has not matched yet), or is just posted. In this case, the request is marked
+   with the appropriate error code, and is removed from the matching queue. The communicator
+   is marked as revoked already.
+4. A new fragment is received, or a fragment is available in one of the unexpected
+   queues. In this case, the fragment has not yet matched a receive request. The
+   receiver decides that the communication is now revoked. Instead of sending the
+   normal control message that initiates the next stage of the communication protocol
+   (e.g., rendez-vous, rdma pipeline, etc.), the receiver answers to the frag
+   with a NACK control message that indicates to the sender that
+   the communication is now completed in error and no further steps should
+   be taken at the sender to continue that send-request.
+
+Again, different PML/MTL may have different implementation for stopping ongoing
+communication when a revoke occurs. The described implementation has the major
+advantage that it does not require the BTL/network driver to be able to
+cancel or interrupt ongoing RDMA operations. Another implementation may be able
+to close an endpoint, thus causing all ongoing operations to implicitly terminate.
+
+Note that legacy components may use static initializers to fill-in the component
+module structure. This practice should be avoided in general, because it will
+cause silently compiling incorrect code when the module function ordering changes.
+The addition of `pml_revoke_comm` is one such change. It is recommended
+that all `pml` components move to named fields static initializers to ensure the
+appropriate functions are set to the appropriate module fields regardless of
+field ordering.
+
+Collective operations
+=====================
+
+The agree operations
+--------------------
+
+The `coll` framework module now includes two new operations `coll_agree` and `coll_iagree`.
+It is expected that, at least at an initial stage, most `coll` framework will not
+provide their own implementation of the agreement operations. When fault tolerance
+is runtime disabled, a base implementation of these operations is provided
+(`coll_base_agree_noft`) by the `coll_basic` module and will be installed as the
+default implementation. That is, a collective module does not need to implement
+an agreement.
+
+The `coll_ftagree` component provides an optimized fault tolerant
+implementation of these operations, and is loaded when fault tolerance is
+enabled at runtime. If a collective component wants to substitute its own
+implementation of the agreement, the normal priority rules for component selection
+apply (i.e., users and develloppers can tweak the values of MCA parameters
+`coll_ftagree_priority` and `coll_xyz_priority` to prefer one over the other.
+
+Note that legacy components may use static initializers to fill-in the component
+module structure. This practice should be avoided in general, because it will
+cause silently compiling incorrect code when the module function ordering changes.
+The addition of `coll_agree` and `coll_iagree` is one such change. It is recommended
+that all `coll` components move to named fields static initializers to ensure the
+appropriate functions are set to the appropriate module fields regardless of
+field ordering.
+
+Interrupting collectives in error
+---------------------------------
+
+The same issue that commands the availability of the `MPIX_COMM_REVOKE` can also
+happen within the implementation of the collective operation itself. Many `coll`
+components use a set of point-to-point operations internally, and when a failure
+is reported at a given step in the collective operation at a particular rank, it
+does not entail that other ranks have also observed that same failure. Thus, if
+the rank that has experienced a point-to-point error jumps out of the collective
+operation without completing the full communication schedule for the collective
+communication, it may leave other rank pending on unmatched collective internal
+point-to-point communication operation. In that case, the collective operation
+would deadlock at the ranks that have not observed the failure through
+point-to-point operations with the failed process.
+
+Two approaches are possible to eliminate this issue.
+
+1. The first approach requires heavy modification of the `coll` component to support
+fault tolerance operation. It is based on unconditionally completing the communication
+schedule of the collective operation, but substituting the actual content of
+post-error messages with special values indicating that the collective operation
+should cause an MPI error at the peer rank. This in essence requires the implementation
+of the collective operations to itself become fault-aware and, to some extent resilient.
+As a consequence, this approach is expected to be used only in special cases (e.g., it
+is currently used in the implementation of MPI_COMM_SPAWN), but in the general case,
+collective frameworks require less intrusive modifications when using the second mechanism.
+When a collective operation wants to use this methodology, it needs to add its collective
+tags to the range of reserved collective tags that are non-interruptible (see the
+function `ompi_request_tag_is_ft()` for details).
+
+2. The second approach, which is provided by the shared Open MPI infrastructure,
+is for the occurrence of a fault at any rank to cause the interruption of all
+collective operations at all ranks on the communicator. This feature is
+implemented by the `errhandler_proc_failed_internal()` calling the `ompi_comm_revoke()`
+operation internally, with the special mode `coll_only`. This call will cause the
+interruption of all requests on `comm` with a collective tag (i.e., one of the tags
+in the range defined by function `ompi_request_tag_is_collective()`) with the error
+`MPI_ERR_PROC_FAILED`. Thanks to this mechanism, the collective implementation
+can do per-request error handling, and bail-out from the collective communication
+pattern as soon as one error is reported. The implicit revoke of collective
+operations will ensure that other ranks will either complete the collective
+successfully (if all contributions to their output buffers were available
+despite the failure), or be appropriately interrupted during one of the
+internal communication operations, even when it does not directly peers with
+a failed process.
+
+Internal request cleanup
+------------------------
+
+Collective operations often require the use of internal requests to follow
+the advancement of internal point-to-point operation comprising the
+communication pattern. Care must be taken to not simply `ompi_request_free`
+these requests in the error case, as doing so does not enforce completion.
+That is, the buffer associated with such a request may still be receiving
+updates from the network receiving message fragments, or RDMA updates.
+
+Instead, the cleanup of such requests must wait that they complete in error
+(as they will, due to the mechanism discussed in the above paragrah). Note
+that only process failure errors are guaranteed to complete in error. Other
+types of errors may deadlock if the request is waited. An example of the
+appropriate cleanup of fault-interrupted requests is found in `coll_base_util.c`
+
+    @@ -76,6 +76,26 @@ int ompi_coll_base_sendrecv_actual( const void* sendbuf, size_t scount,
+         if (MPI_STATUS_IGNORE != status) {
+             status->MPI_ERROR = err;
+         }
+    +    if( MPI_REQUEST_NULL != req ) {
+    +#if OPAL_ENABLE_FT_MPI
+    +        if( MPI_ERR_PROC_FAILED == req->req_status.MPI_ERROR
+    +         || MPI_ERR_PROC_FAILED_PENDING == req->req_status.MPI_ERROR
+    +         || MPI_ERR_REVOKED == req->req_status.MPI_ERROR ) {
+    +            /* We cannot just 'free' and forget, as the PML/BTLS would still
+    +             * be updating the request buffer after we return from the MPI
+    +             * call!
+    +             * For other errors that do not have a well defined post-error
+    +             * behavior, calling the cancel/wait could deadlock, so we just
+    +             * free, as this is the best that can be done in this case. */
+    +            ompi_request_cancel(req);
+    +            ompi_request_wait(&req, MPI_STATUS_IGNORE);
+    +            if( MPI_ERR_PROC_FAILED_PENDING == err ) {
+    +                err = MPI_ERR_PROC_FAILED;
+    +            }
+    +        } else /* this 'else' intentionally spills outside the ifdef */
+    +#endif /* OPAL_ENABLE_FT_MPI */
+    +        ompi_request_free(&req);
+    +    }
+         return (err);
+
+Non-blocking collectives and Generalized requests
+-------------------------------------------------
+
+Non-blocking collective and generalized requests operations can be comprised of
+sub-requests. The final error that is presented to the end-user is presented
+in the completion status of the top-level request. It is however common for the
+top-level request to simply orchestrate the submission and completion of
+a number of sub-requests, rather than engaging directly in communication
+operations. The general design for fault tolerance in this case is that the
+lowest level of requests (that is, request that represent actual communication
+that go down to the BTL/MTL layers) produce errors. These errors are then captured
+during the execution of the sub-request schedule. If a sub-request is found in error,
+the schedule is interrupted, the higher level request is also marked in error, and
+control is returned to the upper level of the schedule, up until the top-level request
+is also marked in error. Note that like normal collective operations, the implicit
+revocation of collective operations on the communicator impacted by a failure
+ensures that it is safe to bail-out from the schedule in a localized manner.
+Other ranks will also experience an error in operations that are blocking thus
+ensuring that no deadlock arises.
+
+A good example of that mode of operation can be found in the
+`libnbc` collective communication component progress in `mca/coll/libnb/nbc.c`:
+
+    @@ -334,8 +336,20 @@ int NBC_Progress(NBC_Handle *handle) {
+         /* don't call ompi_request_test_all as it causes a recursive call into opal_progress */
+         while (handle->req_count) {
+            ompi_request_t *subreq = handle->req_array[handle->req_count - 1];
+    +#if OPAL_ENABLE_FT_MPI
+    +        if (REQUEST_COMPLETE(subreq)
+    +         || OPAL_UNLIKELY( ompi_request_is_failed(subreq) )) {
+    +#else
+             if (REQUEST_COMPLETE(subreq)) {
+    +#endif /* OPAL_ENABLE_FT_MPI */
+                 if(OPAL_UNLIKELY( OMPI_SUCCESS != subreq->req_status.MPI_ERROR )) {
+    +#if OPAL_ENABLE_FT_MPI
+    +                if( MPI_ERR_PROC_FAILED == subreq->req_status.MPI_ERROR ||
+    +                    MPI_ERR_PROC_FAILED_PENDING == subreq->req_status.MPI_ERROR ||
+    +                    MPI_ER_REVOKED == subreq->req_status.MPI_ERROR ) {
+    +                    NBC_DEBUG (1, "MPI Error in NBC subrequest %p : %d)", subreq, subreq->req_status.MPI_ERROR);
+    +                } else // this 'else' intentionally spills outside the ifdef
+    +#endif /* OPAL_ENABLE_FT_MPI */
+                     NBC_Error ("MPI Error in NBC subrequest %p : %d", subreq, subreq->req_status.MPI_ERROR);
+                     /* copy the error code from the underlying request and let the
+                      * round finish */
+
+Changing CID algorithms
+-----------------------
+
+This section describes some specificities of the allocation of CID under fault
+tolerant operations, and is addressed to developers intending to modify the CID
+algorithm.
+
+The `MPI_COMM_REVOKE` operation interrupts all calls on the communicator. Because it is
+a resilient operation, the revoke notification is sent using a reliable broadcast.
+Unlike a normal broadcast, a given process will receive multiple reliable broadcast
+messages for each revoke operation. As is the nature of asynchronous systems, timing
+of reception may cause a revoke notification to reach a particular process much later
+than anticipated. In that case, the process may have already produced the
+`MPI_ERR_REVOKED` to the end-user, and the impacted communicator may have then been
+freed using `MPI_COMM_FREE`. In this case, a new communicator may have re-used the
+context identifier (CID), since it is now free from reuse. Thus, late reliable
+broadcast that pertain to a prior communicator using that CID could erroneously
+revoke that new communicator, and must be ignored.
+
+In order to perform that discrimination between communicators using the same CID at
+different time periods, revoke notice messages themselves are decorated with the CID
+of the communicator to revoke, as well as the epoch at which they have been produced.
+This epoch number is also attached to every communicator when it is created.
+This uniquely identify the communicator that must be revoked independently of the time
+period at which the revoke notification is received.
+
+In order to associate an unique couple (CID,epoch) to communicators, the CID allocation
+algorithm has been expanded to integrate the computation of the epoch as a byproduct
+of the normal operation of the CID algorithm. The way the normal CID algorithm operates
+is based on a multi-step non-blocking allreduce operation to select the best global
+CID. At each round, every rank proposes one (or many) CID that are locally available.
+A non-blocking allreduce then identifies the best among all propositions (if any). If
+a good CID is found, a second allreduce confirms the choice. If no good CID is found,
+that second allreduce will trigger another round of propositions and the algorithm
+repeats. In order to not increase the complexity of that algorithm, the selection of the
+epoch is integrated into the second allreduce that flags the successful selection of a
+proposed CID. A developer that wants to modify the CID algorithm must pay attention to
+the  computation of the epoch.  Ideally the two numbers can be computed in a combined fashion.
+
+Testing Fault Tolerant Operation
+================================
+
+Testing fault-tolerant operation extensively can be hard. Unlike normal communication for
+which the matching and ordering semantic simplify the space of possible ordering of
+messages, faults may interrupt communication at any time, including in the middle
+of a transfer, during RDMA updates, etc. Over the years we have come up with a number of
+tests that stress particular aspects of the communication infrastructure, or represent
+common resilience patterns that users are likely to exercise.
+
+The following repository contains a set of tests that can be used to exercise fault
+tolerance in MPI: [https://github.com/ICLDisco/ulfm-testing]
+
+Testing general error handling
+------------------------------
+
+Test `api/err_handler.c` gives an estimate of the time it takes to report an error during
+an `MPI_BARRIER`. The test times a barrier without fault, injects a process failure,
+times the barrier again (time for new fault to be reported), and then times a barrier again
+(time for an old fault to be reported). The test also verifies that communication between
+non-faulty processes remain possible.
+
+Test `api/err_any.c` verifies that ANY_SOURCE receptions are correctly interrupted when a
+potential sender fails. The test also verifies that ANY_SOURCE operations are again
+operational after the use of `MPIX_Comm_failure_ack()` and counts that the correct number
+of errors gets reported to the application level.
+
+Test `api/getack.c` verifies that errors that have been reported through `MPIX_Comm_get_failed()`
+are indeed resulting in communication with the reported process returning an error of class
+`MPIX_ERR_PROC_FAILED` as mandated.
+
+Stress test for the Revoke operation
+------------------------------------
+
+Test `api/revoke.c` verifies that non-matched pending operations get correctly interrupted by
+the use of API call` MPIX_Comm_revoke()`. ISend and IRecv operations are posted before a
+barrier ensures that the corresponding completion cannot happen before the operation
+gets revoked. This is a good stress test when debugging the cleanup of revoked
+operations from the PML matching queues.
+
+Stress test for BTL timeout
+---------------------------
+
+Test `stress/pingpairs.c` is designed to exercise the case of in-band detection of failure,
+and thus the cleanup code for fragments of requests whose peer with a failed process.
+This test executes a ring-like neighbor pattern that is repeated for a long-enough
+period of time that the network driver will reach the maximum retry-count timeout.
+This will verify that errors are reported quickly from the failure detector, and that
+the occurrence of retry timeout at the BTL level do not cause issues (like unexpected
+aborts in the BTL, or crash due to orphaned fragments cleanup).
+
+Stress test for MPI_COMM_SPAWN
+------------------------------
+
+`MPI_COMM_SPAWN` is often employed to replace failed processes in application usage patterns.
+The test `stress/spawn.c` is designed to force the detection of a process failure to happen
+during the `MPI_COMM_SPAWN` operation itself, thus exercising the error reporting in that case.
+
+Application patterns
+--------------------
+
+The `tutorial/sc19` directory contains a set of application patterns that are demonstrated for
+instruction purposes. These application pattern also represent common use-cases that are to
+be found in the wild, and are good representative of what capabilities will get exercised
+in production.
+
+Test `03.undisturbed.c` verifies that communication in communicators that do not have failed
+processes can continue, meanwhile communication with a failed process report an error. It
+also exercise `MPI_Sendrecv`.
+
+Test `05.err_coll.c` verifies that a blocking collective communication completes (possibly in
+error) at all ranks. This test is designed for a process failure to be detected during the
+collective operation, and can be used as a skeletton to track issues with a particular
+collective operation by just substituting the collective call of interest.
+
+Test `07.err_comm_grid2d.c` is designed for a process failure to be detected during a communicator
+creation function. The test issues two `MPI_COMM_SPLIT`, and then proceed to agree if all
+ranks have succeeded the operation. This is a good test to track issues with the CID
+algorithm. This test will also exercise the non-blocking allreduce, as this operation
+is employed in the CID algorithm.
+
+Test `09.insulation` verifies that errors are reported in collective operations only if the
+underlying communicator contains a failed process. The world is divided in two halves,
+only one of which suffers a failure.
+
+Test `12.buddycr` is representative of the very common usage pattern where replacement processes
+are spawned to take the place of failed processes. The test will perform some mockup computation
+with an iterative-like compute loop with collective communication in the loop. Periodically, the
+dataset is checkpointed on a *buddy* rank. When a fault is injected, it is likely detected during
+the checkpoint operation, thus stress testing `MPIX_COMM_AGREE`, `MPIX_COMM_SHRINK` and `MPI_COMM_SPAWN`.
+
+
+Examples of issues in components that are not yet fault tolerant
+================================================================
+
+In this section, we discuss the features that were missing in some of the major Open MPI components.
+The goal of this section is to present an example of the process and methodology used to add
+fault tolerance to not yet enabled components.
+
+OFI fault tolerance with the OFI MTL
+------------------------------------
+
+While OFI fault tolerance is supported using the OB1 PML, with the OFI BTL, this is a workaround
+for the fact that the OFI MTL does not currently support fault tolerance. Here is a list of
+issues that have been identified to prevent fault-tolerant operations with this component.
+
+
+1. The PML CM does not have a `pml_revoke_comm` function
+2. The PML CM uses requests of type `OMPI_REQUEST_PML` that are not compatible with a `
+   ompi_pml_base_request_t`. This prevents checking the important flags that should be
+   present in such a request type, notably as used in `ompi_request_is_ft()`, `
+   ompi_request_is_collective()`, and `ompi_request_is_failed_fn()`.
+3. The PML CM does not support cancelling requests with a failed peer.
+4. The MTL OFI has instances of calling `abort()/exit()` when communication errors are
+   reported. In `ompi_mtl_ofi_context_progress()`, an error may be returned from
+   `fi_cq_read()` as a consequence of a process failure; this will abort the process
+   instead of returning the error.
+
+
+The UCX PML
+-----------
+
+Similarly to OFI, UCX fault tolerance is supported using the OB1 PML, with the UCT BTL. Again, this
+is a workaround for the fact that the UCX PML does not currently support fault tolerance.
+
+1. The UCX PML does not pass flags to set UCP endpoints in fault-tolerant mode
+2. The UCX PML uses requests of type `OMPI_REQUEST_PML` that are not compatible with
+   a `ompi_pml_base_request_t`. Fields needed by `ompi_request_is_failed_fn()` are missing
+3. The UCX PML does not have a `pml_revoke_comm` function. The unexpected queues, and
+   matching logic is within the UCP component of UCX.
+4. When using the UCX PML, no BTL is initialized and the `ftagree` component and the `ompi_comm_revoke()`
+   function do direct calls to the BTL to send control messages.
+
+RMA Components
+==============
+
+The **OSC** framework is in charge of managing RMA communication.
+At the moment, fault tolerance with RMA communication is not supported.
+However, the ULFM standard draft document ***does*** define semantics for
+RMA fault tolerance, and contributions to enable this feature in Open MPI
+are welcome.
+
+I/O Components
+==============
+
+The following components manage MPI I/O in Open MPI
+
+- _fs_ File system functions for MPI I/O
+- _fbtl_ File byte transfer layer: abstraction for individual read/write
+  operations for OMPIO
+- _fcoll_ Collective read and write operations for MPI I/O
+- _sharedfp_ Shared file pointer operations for MPI I/O
+
+All components in these frameworks are unmodified, **untested** (expect clean
+post-failure abort). Again, the ULFM standard draft document **does** define
+semantics for File operation fault tolerance, and contributions to enable
+this feature in Open MPI are welcome.
+
+___________________________________________________________________________
+
+More Information
+================
+
+Author: Aurelien Bouteiller
+
+The [README.FT.ULFM.md] gives a more complete description of current
+component support in Open MPI ULFM.
+
+More information (tutorials, examples, build instructions for leading
+top500 systems) is also available in the Fault Tolerance Research
+Hub website:
+  <https://fault-tolerance.org>
+
+## Bibliographic References
+If you are looking for, or want to cite a general reference for ULFM,
+please use
+
+_Wesley Bland, Aurelien Bouteiller, Thomas Herault, George Bosilca, Jack
+J. Dongarra: Post-failure recovery of MPI communication capability: Design
+and rationale. IJHPCA 27(3): 244-254 (2013)._
+
+Available from: http://journals.sagepub.com/doi/10.1177/1094342013488238.
+___________________________________________________________________________
+
+Copyright
+=========
+
+```
+Copyright (c) 2019-2020 The University of Tennessee and The University
+                        of Tennessee Research Foundation.  All rights
+                        reserved.
+
+$COPYRIGHT$
+
+Additional copyrights may follow
+
+$HEADER$
+```
diff --git a/contrib/doc-ft-components/general-ulfm-fault-handling.png b/contrib/doc-ft-components/general-ulfm-fault-handling.png
new file mode 100644
index 00000000000..c2510e3aa29
Binary files /dev/null and b/contrib/doc-ft-components/general-ulfm-fault-handling.png differ
diff --git a/contrib/find_occurence.pl b/contrib/find_occurence.pl
index 0e9e1ce1bd0..f4de3256330 100755
--- a/contrib/find_occurence.pl
+++ b/contrib/find_occurence.pl
@@ -18,7 +18,7 @@
 #
 
 if (scalar(@ARGV) != 2) {
-    print "Usage: #find_occurence <string> <source-path>\n";
+    print "Usage: #find_occurrence <string> <source-path>\n";
     exit(3);
 }
 
@@ -28,7 +28,7 @@
 open (SOURCE_FILES, "find $source_path -name *.c |") || print "could not open the pipe\n";
 while (<SOURCE_FILES>) {
 
-    #open the file and delete the occurence
+    #open the file and delete the occurrence
 
     $file_name = $_;
 
diff --git a/contrib/fix_headers.pl b/contrib/fix_headers.pl
index c2527678f5d..04bbc09becb 100755
--- a/contrib/fix_headers.pl
+++ b/contrib/fix_headers.pl
@@ -78,7 +78,7 @@
                 while (<C_FILE>) {
                     if (/$header_string/) {
                         print TEMP $string_to_replace;
-                        print "Replacing defintion ---- $c_file";
+                        print "Replacing definition ---- $c_file";
                         if ($written_to_file == 0) {
                             print MOD_FILES $c_file;
                             $written_to_file = 1;
@@ -121,7 +121,7 @@
                 while (<H_FILE>) {
                     if (/$header_string/) {
                         print TEMP $string_to_replace;
-                        print "Replacing defintion ---- $h_file";
+                        print "Replacing definition ---- $h_file";
                         if ($written_to_file == 0) {
                             print MOD_FILES $h_file;
                             $written_to_file = 1;
diff --git a/contrib/gen_stats.pl b/contrib/gen_stats.pl
index 82252196633..8d652c493b0 100755
--- a/contrib/gen_stats.pl
+++ b/contrib/gen_stats.pl
@@ -213,7 +213,7 @@ sub generate_stats {
 
     while (<INPUT>) {
         #generate the gcov file for this particular file
-        #1. Get the directory name and filename seperately
+        #1. Get the directory name and filename separately
         #2. Invoke gcov on the file
         #3. Print the statistic onto a file
 
diff --git a/contrib/header_replacement.sh b/contrib/header_replacement.sh
index 1bf873143fa..918caec80be 100755
--- a/contrib/header_replacement.sh
+++ b/contrib/header_replacement.sh
@@ -12,7 +12,7 @@
 #                         All rights reserved.
 # Copyright (c) 2009      Oak Ridge National Labs.  All rights reserved.
 #
-# orte_show_help_replacement.sh Script to detect occurences of
+# orte_show_help_replacement.sh Script to detect occurrences of
 # #include "orte/util/show_help.h", where actually either
 #  1. #include "opal/util/output.h"
 #  2. #include "orte/mca/rml/rml_types.h"
@@ -50,7 +50,7 @@ function add_header()
 {
     file=$1                               # File to add header to
     header=$2                             # E.g. opal/util/output.h for #include "opal/util/output.h"
-    after_header_pattern=$3               # Add after occurences of pattern, e.g. opal/util
+    after_header_pattern=$3               # Add after occurrences of pattern, e.g. opal/util
     line=`grep -n "#include \"$after_header_pattern" $file | cut -f1 -d':' | head -n1`
     if [ $# -gt 3 -a "x$line" = "x" ] ; then
         after_header_pattern=$4           # If above pattern is not found, try more generic, e.g. opal/
diff --git a/contrib/libadd_mca_comp_update.py b/contrib/libadd_mca_comp_update.py
index 5d069c2a4ae..26870945e63 100755
--- a/contrib/libadd_mca_comp_update.py
+++ b/contrib/libadd_mca_comp_update.py
@@ -7,7 +7,7 @@
 
 import glob, os, re, shutil
 
-projects= {'opal' : ["$(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la"],
+projects= {'opal' : ["$(top_builddir)/opal/lib@OPAL_LIB_NAME@.la"],
            'ompi' : ["$(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la"],
            'oshmem' : ["$(top_builddir)/oshmem/liboshmem.la"],
           }
diff --git a/contrib/nightly/reports/check_devel_headers.pl b/contrib/nightly/reports/check_devel_headers.pl
index 3e647de8d18..b18e45177af 100755
--- a/contrib/nightly/reports/check_devel_headers.pl
+++ b/contrib/nightly/reports/check_devel_headers.pl
@@ -52,7 +52,7 @@
 }
 
 sub wanted {
-    # don't process directories or links, and dont' recurse down
+    # don't process directories or links, and don't recurse down
     # "special" directories
     if ( -l $_ ) { return; }
     if ( -d $_ ) {
diff --git a/contrib/nightly/reports/illegal_symbols_report.pl b/contrib/nightly/reports/illegal_symbols_report.pl
index e8c44930bdd..9f97a1ccb50 100755
--- a/contrib/nightly/reports/illegal_symbols_report.pl
+++ b/contrib/nightly/reports/illegal_symbols_report.pl
@@ -300,7 +300,7 @@ sub mail_symbols {
 if ($$bad_compsymbols || $$bad_libsymbols) {
 
     open MAIL, "|$mail -s \"$subject\" \"$email_arg\"" ||
-        die "Could ot open pipe to output e-mail\n";
+        die "Could not open pipe to output e-mail\n";
     print MAIL "Found global symbols with missing or illegal prefixes\n\n";
 
     if ($$bad_compsymbols) {
diff --git a/contrib/ompi-time.sh b/contrib/ompi-time.sh
index 51badedb2e4..7ba4b4233c1 100755
--- a/contrib/ompi-time.sh
+++ b/contrib/ompi-time.sh
@@ -2,6 +2,7 @@
 #
 # Copyright (c) 2015      Mellanox Technologies, Inc.
 #                         All rights reserved.
+# Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -201,11 +202,11 @@ function do_checksync_mpisync() {
   if [ ! -e ${tooldir} ]; then
     mkdir -p ${tooldir}
     cd ${tooldir}
-    wget --no-check-certificate https://github.com/open-mpi/ompi/raw/master/ompi/tools/mpisync/mpigclock.c >> $logfile 2>&1
-    wget --no-check-certificate https://github.com/open-mpi/ompi/raw/master/ompi/tools/mpisync/mpigclock.h >> $logfile 2>&1
-    wget --no-check-certificate https://github.com/open-mpi/ompi/raw/master/ompi/tools/mpisync/hpctimer.c >> $logfile 2>&1
-    wget --no-check-certificate https://github.com/open-mpi/ompi/raw/master/ompi/tools/mpisync/hpctimer.h >> $logfile 2>&1
-    wget --no-check-certificate https://github.com/open-mpi/ompi/raw/master/ompi/tools/mpisync/sync.c >> $logfile 2>&1
+    wget --no-check-certificate https://github.com/open-mpi/ompi/raw/main/ompi/tools/mpisync/mpigclock.c >> $logfile 2>&1
+    wget --no-check-certificate https://github.com/open-mpi/ompi/raw/main/ompi/tools/mpisync/mpigclock.h >> $logfile 2>&1
+    wget --no-check-certificate https://github.com/open-mpi/ompi/raw/main/ompi/tools/mpisync/hpctimer.c >> $logfile 2>&1
+    wget --no-check-certificate https://github.com/open-mpi/ompi/raw/main/ompi/tools/mpisync/hpctimer.h >> $logfile 2>&1
+    wget --no-check-certificate https://github.com/open-mpi/ompi/raw/main/ompi/tools/mpisync/sync.c >> $logfile 2>&1
     mpicc hpctimer.c mpigclock.c sync.c -o mpisync >> $logfile 2>&1
   fi
   if [ ! -e "$tooldir" ] || [ ! -f "$tooldir/mpisync" ]; then
@@ -263,7 +264,7 @@ function do_checksync_mpiperf() {
 }
 
 # $1 - sync filename
-# $2 - verbose mode: on - exit in case synchronization values exceed a treshold and off - silent mode (default: off)
+# $2 - verbose mode: on - exit in case synchronization values exceed a threshold and off - silent mode (default: off)
 # $3+ - application additional options
 function do_checksync() {
   if [ -z "$1" ]; then
diff --git a/contrib/ompi_cplusplus.sed b/contrib/ompi_cplusplus.sed
index df2c8ca637c..4af63fd37c8 100644
--- a/contrib/ompi_cplusplus.sed
+++ b/contrib/ompi_cplusplus.sed
@@ -1,6 +1,6 @@
 #
 # Best attempt at a sed script to remove (almost) all combinations
-# of C++ namespace occurences: While std. 3-line occurences with any
+# of C++ namespace occurrences: While std. 3-line occurrences with any
 # combination of spaces and ordering of [c_plusplus,__cplusplus] can
 # be detected, sed does not allow multi-line (here >3 lines) parsing.
 # This all depends on the number of "N" the macro swallows.
diff --git a/contrib/ompi_cplusplus.sh b/contrib/ompi_cplusplus.sh
index 693867a4336..637fe3f7549 100755
--- a/contrib/ompi_cplusplus.sh
+++ b/contrib/ompi_cplusplus.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# Change all occurences of #if defined(c_plusplus) and variations
+# Change all occurrences of #if defined(c_plusplus) and variations
 # to BEGIN_C_DECLS and END_C_DECLS using sed.
 #
 # List of files grepped are in XXX, sed script is in XXX
diff --git a/contrib/ompi_cplusplus.txt b/contrib/ompi_cplusplus.txt
index a61994b0e69..35f2c95e36a 100644
--- a/contrib/ompi_cplusplus.txt
+++ b/contrib/ompi_cplusplus.txt
@@ -132,7 +132,6 @@
 ./opal/util/few.h: defined(c_plusplus)  defined(__cplusplus)
 ./opal/util/keyval_parse.h: defined(c_plusplus)  defined(__cplusplus)
 ./opal/util/malloc.h: defined(c_plusplus)  defined(__cplusplus)
-./opal/util/opal_pty.h: defined(c_plusplus)  defined(__cplusplus)
 ./opal/util/os_path.h: defined(c_plusplus)  defined(__cplusplus)
 ./opal/util/qsort.h: defined(c_plusplus)  defined(__cplusplus)
 ./opal/util/show_help_lex.h: defined(c_plusplus)  defined(__cplusplus)
diff --git a/contrib/platform/google/debug b/contrib/platform/google/debug
new file mode 100644
index 00000000000..36f68f21857
--- /dev/null
+++ b/contrib/platform/google/debug
@@ -0,0 +1,13 @@
+# (c) 2023 Google, LLC. All rights reserved.
+
+enable_mem_debug=no
+enable_mem_profile=no
+enable_debug_symbols=yes
+enable_picky=no
+enable_debug=yes
+with_valgrind=no
+enable_memchecker=no
+
+# For now ensure that ob1 is used. It gives better performance than UCX at this
+# time.
+enable_pml_direct=ob1
diff --git a/contrib/platform/google/debug.conf b/contrib/platform/google/debug.conf
new file mode 100644
index 00000000000..3165f941a0b
--- /dev/null
+++ b/contrib/platform/google/debug.conf
@@ -0,0 +1,87 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights
+#                         reserved.
+# Copyright (c) 2023      Google, LLC. All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# This is the default system-wide MCA parameters defaults file.
+# Specifically, the MCA parameter "mca_param_files" defaults to a
+# value of
+# "$HOME/.openmpi/mca-params.conf:$sysconf/openmpi-mca-params.conf"
+# (this file is the latter of the two).  So if the default value of
+# mca_param_files is not changed, this file is used to set system-wide
+# MCA parameters.  This file can therefore be used to set system-wide
+# default MCA parameters for all users.  Of course, users can override
+# these values if they want, but this file is an excellent location
+# for setting system-specific MCA parameters for those users who don't
+# know / care enough to investigate the proper values for them.
+
+# Note that this file is only applicable where it is visible (in a
+# filesystem sense).  Specifically, MPI processes each read this file
+# during their startup to determine what default values for MCA
+# parameters should be used.  mpirun does not bundle up the values in
+# this file from the node where it was run and send them to all nodes;
+# the default value decisions are effectively distributed.  Hence,
+# these values are only applicable on nodes that "see" this file.  If
+# $sysconf is a directory on a local disk, it is likely that changes
+# to this file will need to be propagated to other nodes.  If $sysconf
+# is a directory that is shared via a networked filesystem, changes to
+# this file will be visible to all nodes that share this $sysconf.
+
+# The format is straightforward: one per line, mca_param_name =
+# rvalue.  Quoting is ignored (so if you use quotes or escape
+# characters, they'll be included as part of the value).  For example:
+
+# Disable run-time MPI parameter checking
+#   mpi_param_check = 0
+
+# Note that the value "~/" will be expanded to the current user's home
+# directory.  For example:
+
+# Change component loading path
+#   component_path = /usr/local/lib/openmpi:~/my_openmpi_components
+
+# See "ompi_info --param all all" for a full listing of Open MPI MCA
+# parameters available and their default values.
+#
+
+# Basic behavior to smooth startup
+mca_base_component_show_load_errors = 0
+
+btl = self,sm,tcp
+
+# Enabling the progress thread should give better performance in
+# most cases.
+btl_tcp_progress_thread = 1
+
+# Enable hierarchal collectives.
+coll_han_priority = 100
+
+# Enable coll/sm. This is commented out for now since coll/sm needs
+# some work before it is ready to be used in most cases.
+# coll_sm_priority = 100
+
+# If the system has busy polling enable (net.core.busy_read > 0)
+# the performance suffers greatly when using poll. Use epoll
+# instead. This should not hurt performance in other cases.
+opal_event_include = epoll
+
+## Setup MPI options
+mpi_show_handle_leaks = 0
+mpi_abort_print_stack = 0
diff --git a/contrib/platform/google/optimized b/contrib/platform/google/optimized
new file mode 100644
index 00000000000..2c69e38513a
--- /dev/null
+++ b/contrib/platform/google/optimized
@@ -0,0 +1,13 @@
+# (c) 2023 Google, LLC. All rights reserved.
+
+enable_mem_debug=no
+enable_mem_profile=no
+enable_debug_symbols=no
+enable_picky=no
+enable_debug=no
+with_valgrind=no
+enable_memchecker=no
+
+# For now ensure that ob1 is used. It gives better performance than UCX at this
+# time.
+enable_pml_direct=ob1
diff --git a/contrib/platform/google/optimized.conf b/contrib/platform/google/optimized.conf
new file mode 100644
index 00000000000..3165f941a0b
--- /dev/null
+++ b/contrib/platform/google/optimized.conf
@@ -0,0 +1,87 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights
+#                         reserved.
+# Copyright (c) 2023      Google, LLC. All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# This is the default system-wide MCA parameters defaults file.
+# Specifically, the MCA parameter "mca_param_files" defaults to a
+# value of
+# "$HOME/.openmpi/mca-params.conf:$sysconf/openmpi-mca-params.conf"
+# (this file is the latter of the two).  So if the default value of
+# mca_param_files is not changed, this file is used to set system-wide
+# MCA parameters.  This file can therefore be used to set system-wide
+# default MCA parameters for all users.  Of course, users can override
+# these values if they want, but this file is an excellent location
+# for setting system-specific MCA parameters for those users who don't
+# know / care enough to investigate the proper values for them.
+
+# Note that this file is only applicable where it is visible (in a
+# filesystem sense).  Specifically, MPI processes each read this file
+# during their startup to determine what default values for MCA
+# parameters should be used.  mpirun does not bundle up the values in
+# this file from the node where it was run and send them to all nodes;
+# the default value decisions are effectively distributed.  Hence,
+# these values are only applicable on nodes that "see" this file.  If
+# $sysconf is a directory on a local disk, it is likely that changes
+# to this file will need to be propagated to other nodes.  If $sysconf
+# is a directory that is shared via a networked filesystem, changes to
+# this file will be visible to all nodes that share this $sysconf.
+
+# The format is straightforward: one per line, mca_param_name =
+# rvalue.  Quoting is ignored (so if you use quotes or escape
+# characters, they'll be included as part of the value).  For example:
+
+# Disable run-time MPI parameter checking
+#   mpi_param_check = 0
+
+# Note that the value "~/" will be expanded to the current user's home
+# directory.  For example:
+
+# Change component loading path
+#   component_path = /usr/local/lib/openmpi:~/my_openmpi_components
+
+# See "ompi_info --param all all" for a full listing of Open MPI MCA
+# parameters available and their default values.
+#
+
+# Basic behavior to smooth startup
+mca_base_component_show_load_errors = 0
+
+btl = self,sm,tcp
+
+# Enabling the progress thread should give better performance in
+# most cases.
+btl_tcp_progress_thread = 1
+
+# Enable hierarchal collectives.
+coll_han_priority = 100
+
+# Enable coll/sm. This is commented out for now since coll/sm needs
+# some work before it is ready to be used in most cases.
+# coll_sm_priority = 100
+
+# If the system has busy polling enable (net.core.busy_read > 0)
+# the performance suffers greatly when using poll. Use epoll
+# instead. This should not hurt performance in other cases.
+opal_event_include = epoll
+
+## Setup MPI options
+mpi_show_handle_leaks = 0
+mpi_abort_print_stack = 0
diff --git a/contrib/platform/intel/bend/linux b/contrib/platform/intel/bend/linux
index 98df8b25777..10580121a58 100644
--- a/contrib/platform/intel/bend/linux
+++ b/contrib/platform/intel/bend/linux
@@ -1,35 +1,21 @@
 enable_orterun_prefix_by_default=yes
-enable_mpi_thread_multiple=no
-enable_mem_debug=no
+enable_oshmem=no
+enable_sensors=no
 enable_mem_profile=no
 enable_debug_symbols=yes
 enable_binaries=yes
-enable_heterogeneous=no
 enable_picky=yes
 enable_debug=yes
 enable_shared=yes
 enable_static=no
-enable_memchecker=no
+enable_io_romio=no
 enable_ipv6=no
+enable_man_pages=no
 enable_mpi_fortran=no
-enable_mpi_cxx=no
-enable_mpi_cxx_seek=no
-enable_cxx_exceptions=no
-enable_mpi_java=no
-enable_io_romio=no
-enable_contrib_no_build=libnbc
-enable_install_libpmix=yes
+enable_memchecker=no
+enable_mca_no_build=memchecker,coll-adapt,coll-cuda,coll-demo,coll-ftagree,coll-han,coll-hcoll,coll-inter,coll-libnbc,coll-monitoring,coll-portals4,coll-tuned,common-monitoring,common-ompio,fbtl,fcoll,fs,io,mtl,osc,pml-cm,pml-monitoring,pml-ucx,pml-v,sharedfp,topo,vprotocol,btl-ofi,btl-portals4,btl-smcuda,btl-uct,btl-ugni,btl-usnic,common-cuda,common-ofi,common-ucx
+enable_contrib_no_build=libompitrace
 with_memory_manager=no
-with_tm=no
-with_psm=no
-with_psm2=no
 with_devel_headers=yes
-with_libfabric=no
-with_portals=no
-with_valgrind=no
-if [ -n "$SLURMHOME" ] ; then
-    with_slurm=$SLURMHOME
-    with_pmi=$SLURMHOME
-else
-    with_slurm=no
-fi
+with_xgrid=no
+with_slurm=no
diff --git a/contrib/platform/intel/bend/mac b/contrib/platform/intel/bend/mac
index 7c1dcaddf53..a16f5d29b58 100644
--- a/contrib/platform/intel/bend/mac
+++ b/contrib/platform/intel/bend/mac
@@ -1,24 +1,19 @@
-enable_orcm=no
 enable_oshmem=no
 enable_sensors=no
-enable_mpi_thread_multiple=yes
-enable_mem_debug=yes
 enable_mem_profile=no
 enable_debug_symbols=yes
 enable_binaries=yes
-enable_heterogeneous=yes
 enable_picky=yes
 enable_debug=yes
 enable_shared=yes
 enable_static=no
 enable_io_romio=no
 enable_ipv6=no
+enable_man_pages=no
 enable_mpi_fortran=no
-enable_mpi_cxx=no
-enable_mpi_cxx_seek=no
 enable_memchecker=no
-enable_mca_no_build=memchecker
-enable_contrib_no_build=libnbc
+enable_mca_no_build=memchecker,common-monitoring,common-ompio,fbtl,fcoll,fs,io,mtl,osc,pml-cm,pml-monitoring,pml-ucx,pml-v,sharedfp,topo,vprotocol,btl-ofi,btl-portals4,btl-smcuda,btl-uct,btl-ugni,btl-usnic,common-cuda,common-ofi,common-ucx,osc-monitoring,coll-monitoring
+enable_contrib_no_build=libompitrace
 with_memory_manager=no
 with_devel_headers=yes
 with_xgrid=no
diff --git a/contrib/platform/mellanox/optimized.conf b/contrib/platform/mellanox/optimized.conf
index 543fd8d1e22..6a7be025a66 100644
--- a/contrib/platform/mellanox/optimized.conf
+++ b/contrib/platform/mellanox/optimized.conf
@@ -63,6 +63,7 @@ hwloc_base_binding_policy = core
 btl = self
 pml_ucx_tls = any
 pml_ucx_devices = any
+opal_common_ucx_opal_mem_hooks = 0
 # Basic behavior to smooth startup
 mca_base_component_show_load_errors = 0
 orte_abort_timeout = 10
diff --git a/contrib/purge-tab-indents.pl b/contrib/purge-tab-indents.pl
index f5d115309c5..015e32e9cb4 100755
--- a/contrib/purge-tab-indents.pl
+++ b/contrib/purge-tab-indents.pl
@@ -10,7 +10,7 @@
 # Will exit with status 111 if there are out of date copyrights which this
 # script can correct.
 my $CHECK_ONLY = 0;
-# used by $CHECK_ONLY logic for bookeeping
+# used by $CHECK_ONLY logic for bookkeeping
 my $would_replace = 0;
 
 # Set to true to suppress most informational messages.  Only out of date files
diff --git a/contrib/purge-trailing-blank-lines.pl b/contrib/purge-trailing-blank-lines.pl
index 2880c5afb97..451a476e843 100755
--- a/contrib/purge-trailing-blank-lines.pl
+++ b/contrib/purge-trailing-blank-lines.pl
@@ -8,7 +8,7 @@
 # Will exit with status 111 if there are out of date copyrights which this
 # script can correct.
 my $CHECK_ONLY = 0;
-# used by $CHECK_ONLY logic for bookeeping
+# used by $CHECK_ONLY logic for bookkeeping
 my $would_replace = 0;
 
 # Set to true to suppress most informational messages.  Only out of date files
diff --git a/contrib/scaling/Makefile b/contrib/scaling/Makefile
index 585f427c953..5054a1a653b 100644
--- a/contrib/scaling/Makefile
+++ b/contrib/scaling/Makefile
@@ -11,7 +11,7 @@ mpi_no_op: mpi_no_op.c
 	mpicc -o mpi_no_op mpi_no_op.c
 
 mpi_memprobe: mpi_memprobe.c
-	mpicc -o mpi_memprobe mpi_memprobe.c -lopen-pal -lopen-rte
+	mpicc -o mpi_memprobe mpi_memprobe.c -lopen-pal
 
 clean:
 	rm -f $(PROGS) *~
diff --git a/contrib/search_replace.pl b/contrib/search_replace.pl
index 3d23c63848c..f2131c5ae22 100755
--- a/contrib/search_replace.pl
+++ b/contrib/search_replace.pl
@@ -31,7 +31,7 @@
 print "replace: $replace_string\n";
 
 sub replace {
-    # don't process directories or links, and dont' recurse down
+    # don't process directories or links, and don't recurse down
     # "special" directories
     if ( -l $_ ) { return; }
     if ( -d $_ ) {
diff --git a/contrib/spread/spread-init.txt b/contrib/spread/spread-init.txt
deleted file mode 100644
index af6290b02b1..00000000000
--- a/contrib/spread/spread-init.txt
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/bash
-
-case "$1" in
-'start')
-	mkdir -p /var/run/spread
-	cd /var/run/spread
-
-	spread&
-	;;
-
-'stop')
-	killall spread 2>/dev/null
-	;;
-
-*)
-	echo "Usage: $0 { start | stop }"
-	exit 1
-	;;
-esac
-exit 0
diff --git a/contrib/spread/spread.conf b/contrib/spread/spread.conf
deleted file mode 100644
index 25c1381a3d8..00000000000
--- a/contrib/spread/spread.conf
+++ /dev/null
@@ -1,242 +0,0 @@
-# Blank lines are permitted in this file.
-# spread.conf sample file
-#
-# questions to spread@spread.org
-#
-
-#MINIMAL REQUIRED FILE
-#
-# Spread should work fine on one machine with just the uncommented
-# lines below. The rest of the file documents all the options and
-# more complex network setups.
-#
-# This configures one spread daemon running on port 4803 on localhost.
-
-Spread_Segment  192.168.203.255:4803 {
-
-	sjc-rcastain-8713	192.168.203.1
-	ubuntu	192.168.203.192
-}
-
-
-
-
-# Spread options
-#---------------------------------------------------------------------------
-#---------------------------------------------------------------------------
-#Set what internal Spread events are logged to the screen or file
-# (see EventLogFile).
-# Default setting is to enable PRINT and EXIT events only.
-#The PRINT and EXIT types should always be enabled. The names of others are:
-#    	EXIT PRINT DEBUG DATA_LINK NETWORK PROTOCOL SESSION
-#	CONFIGURATION MEMBERSHIP FLOW_CONTROL STATUS EVENTS
-#	GROUPS MEMORY SKIPLIST ALL NONE
-#    ALL and NONE are special and represent either enabling every type
-#                                           or enabling none of them.
-#    You can also use a "!" sign to negate a type,
-#        so { ALL !DATA_LINK } means log all events except data_link ones.
-
-#DebugFlags = { PRINT EXIT }
-DebugFlags = { ALL }
-
-# Set priority level of events to output to log file or screen
-# The possible levels are:
-#	pDEBUG INFO WARNING ERROR CRITICAL FATAL
-# Once selected all events tagged with that priority or higher will
-# be output. FATAL events are always output and cause the daemon to
-# shut down. Some Events are tagged with a priority of PRINT which
-# causes them to print out no matter what priority level is set.
-#
-# The default level used if nothing is set is INFO.
-
-#EventPriority =  INFO
-
-#Set whether to log to a file as opposed to stdout/stderr and what
-# file to log to.
-# Default is to log to stdout.
-#
-#If option is not set then logging is to stdout.
-#If option is set then logging is to the filename specified.
-# The filename can include a %h or %H escape that will be replaced at runtime
-# by the hostname of the machine upon which the daemon is running.
-# For example "EventLogFile = spreadlog_%h.log" with 2 machines
-# running Spread (machine1.mydomain.com and machine2.mydomain.com) will
-# cause the daemons to log to "spreadlog_machine1.mydomain.com.log" and
-# "spreadlog_machine2.mydomain.com.log" respectively.
-
-#EventLogFile = testlog.out
-EventLogFile = spread_%h.log
-
-#Set whether to add a timestamp in front of all logged events or not.
-# Default is no timestamps. Default format is "[%a %d %b %Y %H:%M:%S]".
-#If option is commented out then no timestamp is added.
-#If option is enabled then a timestamp is added with the default format
-#If option is enabled and set equal to a string, then that string is used
-#   as the format string for the timestamp. The string must be a valid time
-#   format string as used by the strftime() function.
-
-#EventTimeStamp
-# or
-#EventTimeStamp = "[%a %d %b %Y %H:%M:%S]"
-EventTimeStamp = "[%a %d %b %Y %H:%M:%S]"
-
-#Set whether to add a precise (microsecond) resolution timestamp to all logged
-# events or not. This option requires that EventTimeStamp is also enabled.
-# If the option is commented out then the microsecond timestamp is not added
-# If the option is uncommented then a microsecond time will print in addition
-#  to the H:M:S resolution timestamp provided by EventTimeStamp.
-
-#EventPreciseTimeStamp
-
-# Set to initialize daemon sequence numbers to a 'large' number for testing
-# this is purely a debugging capability and should never be enabled on
-# production systems (note one side effect of enabling this is that
-# your system will experience an extra daemon membership every few messages
-# so you REALLY do not want this turned on)
-# If you want to change the initial value the sequence number is set to
-# you need to edit the #define INITIAL_SEQUENCE_NEAR_WRAP at the top
-# of configuration.h
-
-#DebugInitialSequence
-
-#Set whether to allow dangerous monitor commands
-# like "partition, flow_control, or kill"
-# Default setting is FALSE.
-#If option is set to false then only "safe" monitor commands are allowed
-#    (such as requesting a status update).
-#If option is set to true then all monitor commands are enabled.
-#   THIS IS A SECURTIY RISK IF YOUR NETWORK IS NOT PROTECTED!
-
-#DangerousMonitor = false
-DangerousMonitor = true
-
-#Set handling of SO_REUSEADDR socket option for the daemon's TCP
-# listener.  This is useful for facilitating quick daemon restarts (OSes
-# often hold onto the interface/port combination for a short period of time
-# after daemon shut down).
-#
-# AUTO - Active when bound to specific interfaces (default).
-# ON   - Always active, regardless of interface.
-#        SECURITY RISK FOR ANY OS WHICH ALLOW DOUBLE BINDS BY DIFFERENT USERS
-# OFF  - Always off.
-
-#SocketPortReuse = AUTO
-
-#Set what the maximum per-session queue should be for messages before disconnecting
-# a session. Spread will buffer upto that number of messages that are destined to the
-# session, but that can not be delivered currently because the session is not reading fast enough.
-# The compiled in default is usually 1000 if you havn't changed it in the spread_params.h file.
-
-#MaxSessionMessages = 5000
-MaxSessionMessages = 5000
-
-#Sets the runtime directory used when the Spread daemon is run as root
-# as the directory to chroot to.  Defaults to the value of the
-# compile-time preprocessor define SP_RUNTIME_DIR, which is generally
-# "/var/run/spread".
-
-#RuntimeDir = /var/run/spread
-
-#Sets the unix user that the Spread daemon runs as (when launched as
-# the "root" user).  Not effective on a Windows system.  Defaults to
-# the user and group "spread".
-
-#DaemonUser = spread
-#DaemonGroup = spread
-
-
-#Set the list of authentication methods that the daemon will allow
-# and those which are required in all cases.
-# All of the methods listed in "RequiredAuthMethods" will be checked,
-# irregardless of what methods the client chooses.
-# Of the methods listed is "AllowedAuthMethods" the client is
-# permitted to choose one or more, and all the ones the client chooses
-# will also be checked.
-#
-# To support older clients, if NULL is enabled, then older clients can
-# connect without any authentication. Any methods which do not require
-# any interaction with the client (such as IP) can also be enabled
-# for older clients. If you enable methods that require interaction,
-# then essentially all older clients will be locked out.
-#
-#The current choices are:
-#	NULL for default, allow anyone authentication
-#	IP for IP based checks using the spread.access_ip file
-
-#RequiredAuthMethods = "   "
-#AllowedAuthMethods = "NULL"
-
-#Set the current access control policy.
-# This is only needed if you want to establish a customized policy.
-# The default policy is to allow any actions by authenticated clients.
-#AccessControlPolicy = "PERMIT"
-
-
-# network description line.
-# Spread_Segment <multicast address for subnet> <port> {
-# port is optional, if not specified the default 4803 port is used.
-
-#Spread_Segment  127.0.0.255:4803 {
-
-# either a name or IP address.  If both are given, than the name is taken
-# as-is, and the IP address is used for that name.
-
-#	localhost		127.0.0.1
-#}
-# repeat for next sub-network
-
-#Spread_Segment x.2.2.255 {
-
-#	other1			128.2.2.10
-#				128.2.2.11
-#	other3.my.com
-#}
-# Spread will feel free to use broadcast messages within a sub-network.
-# if you do not want this to happen, you should specify your machines on
-# different logical sub-networks.
-
-# IP-Multicast addresses can also be used as the multicast address for
-# the logical sub-network as in this example. If IP-multicast is supported
-# by the operating system, then the messages will only be received
-# by those machines who are in the group and not by all others in the same
-# sub-network as happens with broadcast addresses
-
-#Spread_Segment 225.0.1.1:3333 {
-#	mcast1			1.2.3.4
-#	mcast2			1.2.3.6
-#}
-
-# Multi-homed host setup
-#
-# If you run Spread on hosts with multiple interfaces you may want to
-# control which interfaces Spread uses for client connections and for
-# the daemon-to-daemon (and monitor control) messages. This can be done
-# by adding an extra stanza to each configured machine.
-#
-#Sample:
-#
-#Spread_Segment 225.0.1.1 {
-# 	multihomed1		1.2.3.4 {
-#		D 192.168.0.4
-#		C 1.2.3.4 }
-#	multihomed2		1.2.3.5 {
-#		D 192.168.0.5
-#		C 1.2.3.5
-#		C 127.0.0.1 }
-#	multihomed3		1.2.3.6 {
-#		192.168.0.6
-#		1.2.3.6 }
-#}
-# This configuration sets up three multihomed machines into a Spread segment.
-# The first host has a 'main' IP address of 1.2.3.4 and listens for client
-# connections only on that interface. All daemon-to-daemon UDP multicasts and
-# the tokens and any monitor messages must use the 192.168.0.4 interface.
-# The second host multihomed2 has a similar setup, except it also listens for
-# client connections on the localhost interface as well as the 1.2.3.5 interface.
-# If you make any use of the extra interface stanza ( a { } block ) then you must
-# explicitly configure ALL interfaces you want as Spread removes all defaults when
-# you use the explicit notation.
-# The third multihomed3 host uses a shorthand form of omitting the D or C option and
-# just listening for all types of traffic and events on both the 192.168.0 and 1.2.3
-# networks. If no letter is listed before the interface address then ALL types of
-# events are handled on that interface.
diff --git a/contrib/submit_test.pl b/contrib/submit_test.pl
old mode 100644
new mode 100755
diff --git a/contrib/test_headers_in_ompi.pl b/contrib/test_headers_in_ompi.pl
index 16595e1f01a..d780535dc4f 100755
--- a/contrib/test_headers_in_ompi.pl
+++ b/contrib/test_headers_in_ompi.pl
@@ -122,7 +122,7 @@ sub parse_header_files {
     close(HEADER);
     close(TEMP);
 
-    #remove duplicate occurences of the file
+    #remove duplicate occurrences of the file
     system("sort temp.txt | uniq > $test_file");
 
     return 0;
diff --git a/contrib/uncrustify_open_mpi.cfg b/contrib/uncrustify_open_mpi.cfg
index ca03062e7e0..2bd87b7b0fa 100644
--- a/contrib/uncrustify_open_mpi.cfg
+++ b/contrib/uncrustify_open_mpi.cfg
@@ -297,10 +297,10 @@ sp_before_semi                           = ignore   # ignore/add/remove/force
 # Add or remove space before ';' in non-empty 'for' statements
 sp_before_semi_for                       = ignore   # ignore/add/remove/force
 
-# Add or remove space before a semicolon of an empty part of a for statment.
+# Add or remove space before a semicolon of an empty part of a for statement.
 sp_before_semi_for_empty                 = ignore   # ignore/add/remove/force
 
-# Add or remove space after the final semicolon of an empty part of a for statment: for ( ; ; <here> ).
+# Add or remove space after the final semicolon of an empty part of a for statement: for ( ; ; <here> ).
 sp_after_semi_for_empty                  = ignore   # ignore/add/remove/force
 
 # Add or remove space before '[' (except '[]')
@@ -517,7 +517,7 @@ sp_cmt_cpp_start                         = ignore   # ignore/add/remove/force
 # Whether to keep non-indenting tabs
 align_keep_tabs                          = false    # false/true
 
-# Whether to use tabs for alinging
+# Whether to use tabs for aligning
 align_with_tabs                          = false    # false/true
 
 # Whether to bump out to the next tab when aligning
@@ -1014,7 +1014,7 @@ mod_full_brace_do                        = add      # ignore/add/remove/force
 # Add or remove braces on single-line 'for' statement
 mod_full_brace_for                       = add      # ignore/add/remove/force
 
-# Add or remove braces on single-line function defintions. (Pawn)
+# Add or remove braces on single-line function definitions. (Pawn)
 mod_full_brace_function                  = ignore   # ignore/add/remove/force
 
 # Add or remove braces on single-line 'if' statement. Will not remove the braces if they contain an 'else'.
@@ -1117,20 +1117,20 @@ cmt_sp_after_star_cont                   = 0        # number
 cmt_multi_check_last                     = true     # false/true
 
 # The filename that contains text to insert at the head of a file if the file doesn't start with a C/C++ comment.
-# Will substitue $(filename) with the current file's name.
+# Will substitute $(filename) with the current file's name.
 cmt_insert_file_header                   = ""         # string
 
 # The filename that contains text to insert at the end of a file if the file doesn't end with a C/C++ comment.
-# Will substitue $(filename) with the current file's name.
+# Will substitute $(filename) with the current file's name.
 cmt_insert_file_footer                   = ""         # string
 
-# The filename that contains text to insert before a function implementation if the function isn't preceeded with a C/C++ comment.
-# Will substitue $(function) with the function name and $(javaparam) with the javadoc @param and @return stuff.
+# The filename that contains text to insert before a function implementation if the function isn't preceded with a C/C++ comment.
+# Will substitute $(function) with the function name and $(javaparam) with the javadoc @param and @return stuff.
 # Will also substitute $(fclass) with the class name: void CFoo::Bar() { ... }
 cmt_insert_func_header                   = ""         # string
 
-# The filename that contains text to insert before a class if the class isn't preceeded with a C/C++ comment.
-# Will substitue $(class) with the class name.
+# The filename that contains text to insert before a class if the class isn't preceded with a C/C++ comment.
+# Will substitute $(class) with the class name.
 cmt_insert_class_header                  = ""         # string
 
 # If a preprocessor is encountered when stepping backwards from a function name, then
diff --git a/contrib/update-my-copyright.pl b/contrib/update-my-copyright.pl
index 9900ff0654c..2bfe9f36a49 100755
--- a/contrib/update-my-copyright.pl
+++ b/contrib/update-my-copyright.pl
@@ -10,7 +10,7 @@
 #
 # This script automates the tedious task of updating copyright notices
 # in the tops of OMPI/ORTE/OPAL source files before committing back to
-# the respository.  Set the environment variable
+# the repository.  Set the environment variable
 # OMPI_COPYRIGHT_SEARCH_NAME to a short (case-insensitive) name that
 # indicates your copyright line (e.g., "cisco"), and set the env
 # variable OMPI_COPYRIGHT_FORMAL_NAME with your organization's formal
@@ -54,7 +54,7 @@
 # Will exit with status 111 if there are out of date copyrights which this
 # script can correct.
 my $CHECK_ONLY = 0;
-# used by $CHECK_ONLY logic for bookeeping
+# used by $CHECK_ONLY logic for bookkeeping
 my $would_replace = 0;
 
 # Set to true to suppress most informational messages.  Only out of date files
diff --git a/docs/Makefile.am b/docs/Makefile.am
new file mode 100644
index 00000000000..f6993850939
--- /dev/null
+++ b/docs/Makefile.am
@@ -0,0 +1,1196 @@
+#
+# Copyright (c) 2022 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2023-2025 Jeffrey M. Squyres.  All rights reserved.
+#
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# We need this Makefile to be executed serially.  Below, we list all
+# the man pages as the targets of the rule that invokes Sphinx for
+# dependency/generation reasons.  But a *single* execution of the make
+# target will generate *all* of the man pages and HTML files.  Hence,
+# when "make" determines that none of the man page files exist, it
+# should execute the Sphinx-invocation rule once, and then it will
+# realize that all the man pages files exist.  More specifically: if
+# someone invokes "make -j N", we need make to not execute the
+# Sphinx-invocation rule multiple times simultaneously.  Both GNU Make
+# and BSD Make will honor the .NOTPARALLEL target to disable all
+# parallel invocation in this Makefile[.am].
+#
+# Note that even though we explicitly disable make's parallelism,
+# we'll use Sphinx's internal parallelism via "-j auto" -- see
+# SPHINX_OPTS.
+.NOTPARALLEL:
+
+OUTDIR             = _build
+SPHINX_CONFIG      = $(srcdir)/conf.py
+SPHINX_OPTS       ?= -W --keep-going -j auto
+
+# Note: it is significantly more convenient to list all the source
+# files here using wildcards (vs. listing every single .rst file).
+# However, it is necessary to list $(srcdir) when using wildcards.
+TEXT_SOURCE_FILES  = \
+        $(srcdir)/license/*.txt
+IMAGE_SOURCE_FILES = \
+        $(srcdir)/openmpi_logo.png \
+        $(srcdir)/installing-open-mpi/required-support-libraries-dependency-graph.png
+RST_SOURCE_FILES   = \
+        $(srcdir)/*.rst \
+        $(srcdir)/release-notes/*.rst \
+        $(srcdir)/release-notes/changelog/*.rst \
+        $(srcdir)/installing-open-mpi/*.rst \
+        $(srcdir)/installing-open-mpi/*/*.rst \
+        $(srcdir)/features/*.rst \
+        $(srcdir)/building-apps/*.rst \
+        $(srcdir)/launching-apps/*.rst \
+        $(srcdir)/tuning-apps/*.rst \
+        $(srcdir)/tuning-apps/*/*.rst \
+        $(srcdir)/app-debug/*.rst \
+        $(srcdir)/developers/*.rst \
+        $(srcdir)/license/*.rst \
+        $(srcdir)/man-openmpi/*.rst \
+        $(srcdir)/man-openmpi/man*/*.rst \
+        $(srcdir)/man-openshmem/*.rst \
+        $(srcdir)/man-openshmem/man*/*.rst
+
+EXTRA_DIST         = \
+        requirements.txt \
+        no-prrte-content.rst.txt \
+        generate-mpi-man3-bindings.py \
+        mpi-standard-apis.json \
+        html \
+        man \
+        $(SPHINX_CONFIG) \
+        $(TEXT_SOURCE_FILES) \
+        $(IMAGE_SOURCE_FILES) \
+        $(RST_SOURCE_FILES)
+
+###########################################################################
+
+# Note: we list all the man pages explicitly (as opposed to using
+# wildcards, like we do to list all the RST source files in
+# EXTRA_DIST) because these files get installed by Automake.  There
+# are less complications and portability issues (between GNU and BSD
+# make(1), for example) if all the files to be installed are
+# explicitly listed.
+
+OMPI_MAN1 = \
+        ompi-wrapper-compiler.1 \
+        mpirun.1 \
+        mpisync.1 \
+        ompi_info.1 \
+        opal_wrapper.1
+
+OMPI_MAN3 = \
+        MPI_Abort.3 \
+        MPI_Accumulate.3 \
+        MPI_Add_error_class.3 \
+        MPI_Add_error_code.3 \
+        MPI_Add_error_string.3 \
+        MPI_Address.3 \
+        MPI_Aint_add.3 \
+        MPI_Aint_diff.3 \
+        MPI_Allgather.3 \
+        MPI_Allgather_init.3 \
+        MPI_Allgatherv.3 \
+        MPI_Allgatherv_init.3 \
+        MPI_Alloc_mem.3 \
+        MPI_Allreduce.3 \
+        MPI_Allreduce_init.3 \
+        MPI_Alltoall.3 \
+        MPI_Alltoall_init.3 \
+        MPI_Alltoallv.3 \
+        MPI_Alltoallv_init.3 \
+        MPI_Alltoallw.3 \
+        MPI_Alltoallw_init.3 \
+        MPI_Attr_delete.3 \
+        MPI_Attr_get.3 \
+        MPI_Attr_put.3 \
+        MPI_Barrier.3 \
+        MPI_Barrier_init.3 \
+        MPI_Bcast.3 \
+        MPI_Bcast_init.3 \
+        MPI_Bsend.3 \
+        MPI_Bsend_init.3 \
+        MPI_Buffer_attach.3 \
+        MPI_Buffer_detach.3 \
+        MPI_Cancel.3 \
+        MPI_Cart_coords.3 \
+        MPI_Cart_create.3 \
+        MPI_Cartdim_get.3 \
+        MPI_Cart_get.3 \
+        MPI_Cart_map.3 \
+        MPI_Cart_rank.3 \
+        MPI_Cart_shift.3 \
+        MPI_Cart_sub.3 \
+        MPI_Close_port.3 \
+        MPI_Comm_accept.3 \
+        MPI_Comm_c2f.3 \
+        MPI_Comm_call_errhandler.3 \
+        MPI_Comm_compare.3 \
+        MPI_Comm_connect.3 \
+        MPI_Comm_create.3 \
+        MPI_Comm_create_errhandler.3 \
+        MPI_Comm_create_from_group.3 \
+        MPI_Comm_create_group.3 \
+        MPI_Comm_create_keyval.3 \
+        MPI_Comm_delete_attr.3 \
+        MPI_Comm_disconnect.3 \
+        MPI_Comm_dup.3 \
+        MPI_Comm_dup_with_info.3 \
+        MPI_Comm_f2c.3 \
+        MPI_Comm_free.3 \
+        MPI_Comm_free_keyval.3 \
+        MPI_Comm_get_attr.3 \
+        MPI_Comm_get_errhandler.3 \
+        MPI_Comm_get_info.3 \
+        MPI_Comm_get_name.3 \
+        MPI_Comm_get_parent.3 \
+        MPI_Comm_group.3 \
+        MPI_Comm_idup.3 \
+        MPI_Comm_idup_with_info.3 \
+        MPI_Comm_join.3 \
+        MPI_Comm_rank.3 \
+        MPI_Comm_remote_group.3 \
+        MPI_Comm_remote_size.3 \
+        MPI_Comm_set_attr.3 \
+        MPI_Comm_set_errhandler.3 \
+        MPI_Comm_set_info.3 \
+        MPI_Comm_set_name.3 \
+        MPI_Comm_size.3 \
+        MPI_Comm_spawn.3 \
+        MPI_Comm_spawn_multiple.3 \
+        MPI_Comm_split.3 \
+        MPI_Comm_split_type.3 \
+        MPI_Comm_test_inter.3 \
+        MPI_Compare_and_swap.3 \
+        MPI_Dims_create.3 \
+        MPI_Dist_graph_create.3 \
+        MPI_Dist_graph_create_adjacent.3 \
+        MPI_Dist_graph_neighbors.3 \
+        MPI_Dist_graph_neighbors_count.3 \
+        MPI_Errhandler_create.3 \
+        MPI_Errhandler_free.3 \
+        MPI_Errhandler_get.3 \
+        MPI_Errhandler_set.3 \
+        MPI_Error_class.3 \
+        MPI_Error_string.3 \
+        MPI_Exscan.3 \
+        MPI_Exscan_init.3 \
+        MPI_Fetch_and_op.3 \
+        MPI_File_c2f.3 \
+        MPI_File_call_errhandler.3 \
+        MPI_File_close.3 \
+        MPI_File_create_errhandler.3 \
+        MPI_File_delete.3 \
+        MPI_File_f2c.3 \
+        MPI_File_get_amode.3 \
+        MPI_File_get_atomicity.3 \
+        MPI_File_get_byte_offset.3 \
+        MPI_File_get_errhandler.3 \
+        MPI_File_get_group.3 \
+        MPI_File_get_info.3 \
+        MPI_File_get_position.3 \
+        MPI_File_get_position_shared.3 \
+        MPI_File_get_size.3 \
+        MPI_File_get_type_extent.3 \
+        MPI_File_get_view.3 \
+        MPI_File_iread.3 \
+        MPI_File_iread_all.3 \
+        MPI_File_iread_at.3 \
+        MPI_File_iread_at_all.3 \
+        MPI_File_iread_shared.3 \
+        MPI_File_iwrite.3 \
+        MPI_File_iwrite_all.3 \
+        MPI_File_iwrite_at.3 \
+        MPI_File_iwrite_at_all.3 \
+        MPI_File_iwrite_shared.3 \
+        MPI_File_open.3 \
+        MPI_File_preallocate.3 \
+        MPI_File_read.3 \
+        MPI_File_read_all.3 \
+        MPI_File_read_all_begin.3 \
+        MPI_File_read_all_end.3 \
+        MPI_File_read_at.3 \
+        MPI_File_read_at_all.3 \
+        MPI_File_read_at_all_begin.3 \
+        MPI_File_read_at_all_end.3 \
+        MPI_File_read_ordered.3 \
+        MPI_File_read_ordered_begin.3 \
+        MPI_File_read_ordered_end.3 \
+        MPI_File_read_shared.3 \
+        MPI_File_seek.3 \
+        MPI_File_seek_shared.3 \
+        MPI_File_set_atomicity.3 \
+        MPI_File_set_errhandler.3 \
+        MPI_File_set_info.3 \
+        MPI_File_set_size.3 \
+        MPI_File_set_view.3 \
+        MPI_File_sync.3 \
+        MPI_File_write.3 \
+        MPI_File_write_all.3 \
+        MPI_File_write_all_begin.3 \
+        MPI_File_write_all_end.3 \
+        MPI_File_write_at.3 \
+        MPI_File_write_at_all.3 \
+        MPI_File_write_at_all_begin.3 \
+        MPI_File_write_at_all_end.3 \
+        MPI_File_write_ordered.3 \
+        MPI_File_write_ordered_begin.3 \
+        MPI_File_write_ordered_end.3 \
+        MPI_File_write_shared.3 \
+        MPI_Finalize.3 \
+        MPI_Finalized.3 \
+        MPI_Free_mem.3 \
+        MPI_Gather.3 \
+        MPI_Gather_init.3 \
+        MPI_Gatherv.3 \
+        MPI_Gatherv_init.3 \
+        MPI_Get.3 \
+        MPI_Get_accumulate.3 \
+        MPI_Get_address.3 \
+        MPI_Get_count.3 \
+        MPI_Get_elements.3 \
+        MPI_Get_elements_x.3 \
+        MPI_Get_library_version.3 \
+        MPI_Get_processor_name.3 \
+        MPI_Get_version.3 \
+        MPI_Graph_create.3 \
+        MPI_Graphdims_get.3 \
+        MPI_Graph_get.3 \
+        MPI_Graph_map.3 \
+        MPI_Graph_neighbors.3 \
+        MPI_Graph_neighbors_count.3 \
+        MPI_Grequest_complete.3 \
+        MPI_Grequest_start.3 \
+        MPI_Group_c2f.3 \
+        MPI_Group_compare.3 \
+        MPI_Group_difference.3 \
+        MPI_Group_excl.3 \
+        MPI_Group_f2c.3 \
+        MPI_Group_free.3 \
+        MPI_Group_from_session_pset.3 \
+        MPI_Group_incl.3 \
+        MPI_Group_intersection.3 \
+        MPI_Group_range_excl.3 \
+        MPI_Group_range_incl.3 \
+        MPI_Group_rank.3 \
+        MPI_Group_size.3 \
+        MPI_Group_translate_ranks.3 \
+        MPI_Group_union.3 \
+        MPI_Iallgather.3 \
+        MPI_Iallgatherv.3 \
+        MPI_Iallreduce.3 \
+        MPI_Ialltoall.3 \
+        MPI_Ialltoallv.3 \
+        MPI_Ialltoallw.3 \
+        MPI_Ibarrier.3 \
+        MPI_Ibcast.3 \
+        MPI_Ibsend.3 \
+        MPI_Iexscan.3 \
+        MPI_Igather.3 \
+        MPI_Igatherv.3 \
+        MPI_Improbe.3 \
+        MPI_Imrecv.3 \
+        MPI_Ineighbor_allgather.3 \
+        MPI_Ineighbor_allgatherv.3 \
+        MPI_Ineighbor_alltoall.3 \
+        MPI_Ineighbor_alltoallv.3 \
+        MPI_Ineighbor_alltoallw.3 \
+        MPI_Info_c2f.3 \
+        MPI_Info_create.3 \
+        MPI_Info_delete.3 \
+        MPI_Info_dup.3 \
+        MPI_Info_env.3 \
+        MPI_Info_f2c.3 \
+        MPI_Info_free.3 \
+        MPI_Info_get.3 \
+        MPI_Info_get_nkeys.3 \
+        MPI_Info_get_nthkey.3 \
+        MPI_Info_get_string.3 \
+        MPI_Info_get_valuelen.3 \
+        MPI_Info_set.3 \
+        MPI_Init.3 \
+        MPI_Initialized.3 \
+        MPI_Init_thread.3 \
+        MPI_Intercomm_create.3 \
+        MPI_Intercomm_create_from_groups.3 \
+        MPI_Intercomm_merge.3 \
+        MPI_Iprobe.3 \
+        MPI_Irecv.3 \
+        MPI_Ireduce.3 \
+        MPI_Ireduce_scatter.3 \
+        MPI_Ireduce_scatter_block.3 \
+        MPI_Irsend.3 \
+        MPI_Iscan.3 \
+        MPI_Iscatter.3 \
+        MPI_Iscatterv.3 \
+        MPI_Isend.3 \
+        MPI_Isendrecv.3 \
+        MPI_Isendrecv_replace.3 \
+        MPI_Issend.3 \
+        MPI_Is_thread_main.3 \
+        MPI_Keyval_create.3 \
+        MPI_Keyval_free.3 \
+        MPI_Lookup_name.3 \
+        MPI_Message_c2f.3 \
+        MPI_Message_f2c.3 \
+        MPI_Mprobe.3 \
+        MPI_Mrecv.3 \
+        MPI_Neighbor_allgather.3 \
+        MPI_Neighbor_allgather_init.3 \
+        MPI_Neighbor_allgatherv.3 \
+        MPI_Neighbor_allgatherv_init.3 \
+        MPI_Neighbor_alltoall.3 \
+        MPI_Neighbor_alltoall_init.3 \
+        MPI_Neighbor_alltoallv.3 \
+        MPI_Neighbor_alltoallv_init.3 \
+        MPI_Neighbor_alltoallw.3 \
+        MPI_Neighbor_alltoallw_init.3 \
+        MPI_Op_c2f.3 \
+        MPI_Op_commutative.3 \
+        MPI_Op_create.3 \
+        MPI_Open_port.3 \
+        MPI_Op_f2c.3 \
+        MPI_Op_free.3 \
+        MPI_Pack.3 \
+        MPI_Pack_external.3 \
+        MPI_Pack_external_size.3 \
+        MPI_Pack_size.3 \
+        MPI_Parrived.3 \
+        MPI_Pcontrol.3 \
+        MPI_Pready.3 \
+        MPI_Pready_list.3 \
+        MPI_Pready_range.3 \
+        MPI_Precv_init.3 \
+        MPI_Probe.3 \
+        MPI_Psend_init.3 \
+        MPI_Publish_name.3 \
+        MPI_Put.3 \
+        MPI_Query_thread.3 \
+        MPI_Raccumulate.3 \
+        MPI_Recv.3 \
+        MPI_Recv_init.3 \
+        MPI_Reduce.3 \
+        MPI_Reduce_init.3 \
+        MPI_Reduce_local.3 \
+        MPI_Reduce_scatter.3 \
+        MPI_Reduce_scatter_block.3 \
+        MPI_Reduce_scatter_block_init.3 \
+        MPI_Reduce_scatter_init.3 \
+        MPI_Register_datarep.3 \
+        MPI_Request_c2f.3 \
+        MPI_Request_f2c.3 \
+        MPI_Request_free.3 \
+        MPI_Request_get_status.3 \
+        MPI_Rget.3 \
+        MPI_Rget_accumulate.3 \
+        MPI_Rput.3 \
+        MPI_Rsend.3 \
+        MPI_Rsend_init.3 \
+        MPI_Scan.3 \
+        MPI_Scan_init.3 \
+        MPI_Scatter.3 \
+        MPI_Scatter_init.3 \
+        MPI_Scatterv.3 \
+        MPI_Scatterv_init.3 \
+        MPI_Send.3 \
+        MPI_Send_init.3 \
+        MPI_Sendrecv.3 \
+        MPI_Sendrecv_replace.3 \
+        MPI_Session_create_errhandler.3 \
+        MPI_Session_f2c.3 \
+        MPI_Session_finalize.3 \
+        MPI_Session_get_info.3 \
+        MPI_Session_get_nth_pset.3 \
+        MPI_Session_get_num_psets.3 \
+        MPI_Session_get_pset_info.3 \
+        MPI_Session_init.3 \
+        MPI_Sizeof.3 \
+        MPI_Ssend.3 \
+        MPI_Ssend_init.3 \
+        MPI_Start.3 \
+        MPI_Startall.3 \
+        MPI_Status_c2f08.3 \
+        MPI_Status_c2f.3 \
+        MPI_Status_f082c.3 \
+        MPI_Status_f082f.3 \
+        MPI_Status_f2c.3 \
+        MPI_Status_f2f08.3 \
+        MPI_Status_get_error.3 \
+        MPI_Status_get_source.3 \
+        MPI_Status_get_tag.3 \
+        MPI_Status_set_cancelled.3 \
+        MPI_Status_set_elements.3 \
+        MPI_Status_set_elements_x.3 \
+        MPI_Status_set_error.3 \
+        MPI_Status_set_source.3 \
+        MPI_Status_set_tag.3 \
+        MPI_T.3 \
+        MPI_T_category_changed.3 \
+        MPI_T_category_get_categories.3 \
+        MPI_T_category_get_cvars.3 \
+        MPI_T_category_get_events.3 \
+        MPI_T_category_get_info.3 \
+        MPI_T_category_get_num.3 \
+        MPI_T_category_get_num_events.3 \
+        MPI_T_category_get_pvars.3 \
+        MPI_T_cvar_get_info.3 \
+        MPI_T_cvar_get_num.3 \
+        MPI_T_cvar_handle_alloc.3 \
+        MPI_T_cvar_handle_free.3 \
+        MPI_T_cvar_read.3 \
+        MPI_T_cvar_write.3 \
+        MPI_T_enum_get_info.3 \
+        MPI_T_enum_get_item.3 \
+        MPI_T_event_callback_get_info.3 \
+        MPI_T_event_callback_set_info.3 \
+        MPI_T_event_copy.3 \
+        MPI_T_event_get_index.3 \
+        MPI_T_event_get_info.3 \
+        MPI_T_event_get_num.3 \
+        MPI_T_event_get_source.3 \
+        MPI_T_event_get_timestamp.3 \
+        MPI_T_event_handle_alloc.3 \
+        MPI_T_event_handle_free.3 \
+        MPI_T_event_handle_get_info.3 \
+        MPI_T_event_handle_set_info.3 \
+        MPI_T_event_read.3 \
+        MPI_T_event_register_callback.3 \
+        MPI_T_event_set_dropped_handler.3 \
+        MPI_Test.3 \
+        MPI_Testall.3 \
+        MPI_Testany.3 \
+        MPI_Test_cancelled.3 \
+        MPI_Testsome.3 \
+        MPI_T_finalize.3 \
+        MPI_T_init_thread.3 \
+        MPI_Topo_test.3 \
+        MPI_T_pvar_get_info.3 \
+        MPI_T_pvar_get_num.3 \
+        MPI_T_pvar_handle_alloc.3 \
+        MPI_T_pvar_handle_free.3 \
+        MPI_T_pvar_read.3 \
+        MPI_T_pvar_readreset.3 \
+        MPI_T_pvar_reset.3 \
+        MPI_T_pvar_session_create.3 \
+        MPI_T_pvar_session_free.3 \
+        MPI_T_pvar_start.3 \
+        MPI_T_pvar_stop.3 \
+        MPI_T_pvar_write.3 \
+        MPI_T_source_get_info.3 \
+        MPI_T_source_get_num.3 \
+        MPI_T_source_get_timestamp.3 \
+        MPI_Type_c2f.3 \
+        MPI_Type_commit.3 \
+        MPI_Type_contiguous.3 \
+        MPI_Type_create_darray.3 \
+        MPI_Type_create_f90_complex.3 \
+        MPI_Type_create_f90_integer.3 \
+        MPI_Type_create_f90_real.3 \
+        MPI_Type_create_hindexed.3 \
+        MPI_Type_create_hindexed_block.3 \
+        MPI_Type_create_hvector.3 \
+        MPI_Type_create_indexed_block.3 \
+        MPI_Type_create_keyval.3 \
+        MPI_Type_create_resized.3 \
+        MPI_Type_create_struct.3 \
+        MPI_Type_create_subarray.3 \
+        MPI_Type_delete_attr.3 \
+        MPI_Type_dup.3 \
+        MPI_Type_extent.3 \
+        MPI_Type_f2c.3 \
+        MPI_Type_free.3 \
+        MPI_Type_free_keyval.3 \
+        MPI_Type_get_attr.3 \
+        MPI_Type_get_contents.3 \
+        MPI_Type_get_envelope.3 \
+        MPI_Type_get_extent.3 \
+        MPI_Type_get_extent_x.3 \
+        MPI_Type_get_name.3 \
+        MPI_Type_get_true_extent.3 \
+        MPI_Type_get_true_extent_x.3 \
+        MPI_Type_hindexed.3 \
+        MPI_Type_hvector.3 \
+        MPI_Type_indexed.3 \
+        MPI_Type_lb.3 \
+        MPI_Type_match_size.3 \
+        MPI_Type_set_attr.3 \
+        MPI_Type_set_name.3 \
+        MPI_Type_size.3 \
+        MPI_Type_size_x.3 \
+        MPI_Type_struct.3 \
+        MPI_Type_ub.3 \
+        MPI_Type_vector.3 \
+        MPI_Unpack.3 \
+        MPI_Unpack_external.3 \
+        MPI_Unpublish_name.3 \
+        MPI_Wait.3 \
+        MPI_Waitall.3 \
+        MPI_Waitany.3 \
+        MPI_Waitsome.3 \
+        MPI_Win_allocate.3 \
+        MPI_Win_allocate_shared.3 \
+        MPI_Win_attach.3 \
+        MPI_Win_c2f.3 \
+        MPI_Win_call_errhandler.3 \
+        MPI_Win_complete.3 \
+        MPI_Win_create.3 \
+        MPI_Win_create_dynamic.3 \
+        MPI_Win_create_errhandler.3 \
+        MPI_Win_create_keyval.3 \
+        MPI_Win_delete_attr.3 \
+        MPI_Win_detach.3 \
+        MPI_Win_f2c.3 \
+        MPI_Win_fence.3 \
+        MPI_Win_flush.3 \
+        MPI_Win_flush_all.3 \
+        MPI_Win_flush_local.3 \
+        MPI_Win_flush_local_all.3 \
+        MPI_Win_free.3 \
+        MPI_Win_free_keyval.3 \
+        MPI_Win_get_attr.3 \
+        MPI_Win_get_errhandler.3 \
+        MPI_Win_get_group.3 \
+        MPI_Win_get_info.3 \
+        MPI_Win_get_name.3 \
+        MPI_Win_lock.3 \
+        MPI_Win_lock_all.3 \
+        MPI_Win_post.3 \
+        MPI_Win_set_attr.3 \
+        MPI_Win_set_errhandler.3 \
+        MPI_Win_set_info.3 \
+        MPI_Win_set_name.3 \
+        MPI_Win_shared_query.3 \
+        MPI_Win_start.3 \
+        MPI_Win_sync.3 \
+        MPI_Win_test.3 \
+        MPI_Win_unlock.3 \
+        MPI_Win_unlock_all.3 \
+        MPI_Win_wait.3 \
+        MPI_Wtick.3 \
+        MPI_Wtime.3 \
+        MPIX_Comm_ack_failed.3 \
+        MPIX_Comm_agree.3 \
+        MPIX_Comm_get_failed.3 \
+        MPIX_Comm_iagree.3 \
+        MPIX_Comm_ishrink.3 \
+        MPIX_Comm_is_revoked.3 \
+        MPIX_Comm_revoke.3 \
+        MPIX_Comm_shrink.3 \
+        MPIX_Query_cuda_support.3 \
+        MPIX_Query_rocm_support.3 \
+        OMPI_Affinity_str.3
+
+OMPI_MAN7 = \
+        Open-MPI.7
+
+OSHMEM_MAN1 = \
+        oshmem-wrapper-compiler.1 \
+        oshmem_info.1
+
+OSHMEM_MAN3 = \
+        intro_shmem.3 \
+        _my_pe.3 \
+        _num_pes.3 \
+        OpenSHMEM.3 \
+        shfree.3 \
+        shmalloc.3 \
+        shmem_addr_accessible.3 \
+        shmem_align.3 \
+        shmemalign.3 \
+        shmem_alltoall32.3 \
+        shmem_alltoall64.3 \
+        shmem_alltoalls32.3 \
+        shmem_alltoalls64.3 \
+        shmem_barrier.3 \
+        shmem_barrier_all.3 \
+        shmem_broadcast32.3 \
+        shmem_broadcast64.3 \
+        shmem_char_g.3 \
+        shmem_char_get.3 \
+        shmem_char_get_nbi.3 \
+        shmem_char_p.3 \
+        shmem_char_put.3 \
+        shmem_char_put_nbi.3 \
+        shmem_clear_cache_inv.3 \
+        shmem_clear_cache_line_inv.3 \
+        shmem_clear_lock.3 \
+        shmem_collect32.3 \
+        shmem_collect64.3 \
+        shmem_complexd_prod_to_all.3 \
+        shmem_complexd_sum_to_all.3 \
+        shmem_complexf_prod_to_all.3 \
+        shmem_complexf_sum_to_all.3 \
+        shmem_double_fetch.3 \
+        shmem_double_g.3 \
+        shmem_double_get.3 \
+        shmem_double_get_nbi.3 \
+        shmem_double_iget.3 \
+        shmem_double_iput.3 \
+        shmem_double_max_to_all.3 \
+        shmem_double_min_to_all.3 \
+        shmem_double_p.3 \
+        shmem_double_prod_to_all.3 \
+        shmem_double_put.3 \
+        shmem_double_put_nbi.3 \
+        shmem_double_set.3 \
+        shmem_double_sum_to_all.3 \
+        shmem_double_swap.3 \
+        shmem_fcollect32.3 \
+        shmem_fcollect64.3 \
+        shmem_fence.3 \
+        shmem_finalize.3 \
+        shmem_float_fetch.3 \
+        shmem_float_g.3 \
+        shmem_float_get.3 \
+        shmem_float_get_nbi.3 \
+        shmem_float_iget.3 \
+        shmem_float_iput.3 \
+        shmem_float_max_to_all.3 \
+        shmem_float_min_to_all.3 \
+        shmem_float_p.3 \
+        shmem_float_prod_to_all.3 \
+        shmem_float_put.3 \
+        shmem_float_put_nbi.3 \
+        shmem_float_set.3 \
+        shmem_float_sum_to_all.3 \
+        shmem_float_swap.3 \
+        shmem_free.3 \
+        shmem_get128.3 \
+        shmem_get128_nbi.3 \
+        shmem_get16_nbi.3 \
+        shmem_get32.3 \
+        shmem_get32_nbi.3 \
+        shmem_get64.3 \
+        shmem_get64_nbi.3 \
+        shmem_get8_nbi.3 \
+        shmem_getmem.3 \
+        shmem_getmem_nbi.3 \
+        shmem_global_exit.3 \
+        shmem_iget128.3 \
+        shmem_iget32.3 \
+        shmem_iget64.3 \
+        shmem_info_get_name.3 \
+        shmem_info_get_version.3 \
+        shmem_init.3 \
+        shmem_int_add.3 \
+        shmem_int_and_to_all.3 \
+        shmem_int_cswap.3 \
+        shmem_int_fadd.3 \
+        shmem_int_fetch.3 \
+        shmem_int_finc.3 \
+        shmem_int_g.3 \
+        shmem_int_get.3 \
+        shmem_int_get_nbi.3 \
+        shmem_int_iget.3 \
+        shmem_int_inc.3 \
+        shmem_int_iput.3 \
+        shmem_int_max_to_all.3 \
+        shmem_int_min_to_all.3 \
+        shmem_int_or_to_all.3 \
+        shmem_int_p.3 \
+        shmem_int_prod_to_all.3 \
+        shmem_int_put.3 \
+        shmem_int_put_nbi.3 \
+        shmem_int_set.3 \
+        shmem_int_sum_to_all.3 \
+        shmem_int_swap.3 \
+        shmem_int_wait.3 \
+        shmem_int_wait_until.3 \
+        shmem_int_xor_to_all.3 \
+        shmem_iput128.3 \
+        shmem_iput32.3 \
+        shmem_iput64.3 \
+        shmem_long_add.3 \
+        shmem_long_and_to_all.3 \
+        shmem_long_cswap.3 \
+        shmem_longdouble_g.3 \
+        shmem_longdouble_get.3 \
+        shmem_longdouble_get_nbi.3 \
+        shmem_longdouble_iget.3 \
+        shmem_longdouble_iput.3 \
+        shmem_longdouble_max_to_all.3 \
+        shmem_longdouble_min_to_all.3 \
+        shmem_longdouble_p.3 \
+        shmem_longdouble_prod_to_all.3 \
+        shmem_longdouble_put.3 \
+        shmem_longdouble_put_nbi.3 \
+        shmem_longdouble_sum_to_all.3 \
+        shmem_long_fadd.3 \
+        shmem_long_fetch.3 \
+        shmem_long_finc.3 \
+        shmem_long_g.3 \
+        shmem_long_get.3 \
+        shmem_long_get_nbi.3 \
+        shmem_long_iget.3 \
+        shmem_long_inc.3 \
+        shmem_long_iput.3 \
+        shmem_longlong_add.3 \
+        shmem_longlong_and_to_all.3 \
+        shmem_longlong_cswap.3 \
+        shmem_longlong_fadd.3 \
+        shmem_longlong_fetch.3 \
+        shmem_longlong_finc.3 \
+        shmem_longlong_g.3 \
+        shmem_longlong_get.3 \
+        shmem_longlong_get_nbi.3 \
+        shmem_longlong_iget.3 \
+        shmem_longlong_inc.3 \
+        shmem_longlong_iput.3 \
+        shmem_longlong_max_to_all.3 \
+        shmem_longlong_min_to_all.3 \
+        shmem_longlong_or_to_all.3 \
+        shmem_longlong_p.3 \
+        shmem_longlong_prod_to_all.3 \
+        shmem_longlong_put.3 \
+        shmem_longlong_put_nbi.3 \
+        shmem_longlong_set.3 \
+        shmem_longlong_sum_to_all.3 \
+        shmem_longlong_swap.3 \
+        shmem_longlong_wait.3 \
+        shmem_longlong_wait_until.3 \
+        shmem_longlong_xor_to_all.3 \
+        shmem_long_max_to_all.3 \
+        shmem_long_min_to_all.3 \
+        shmem_long_or_to_all.3 \
+        shmem_long_p.3 \
+        shmem_long_prod_to_all.3 \
+        shmem_long_put.3 \
+        shmem_long_put_nbi.3 \
+        shmem_long_set.3 \
+        shmem_long_sum_to_all.3 \
+        shmem_long_swap.3 \
+        shmem_long_wait.3 \
+        shmem_long_wait_until.3 \
+        shmem_long_xor_to_all.3 \
+        shmem_malloc.3 \
+        shmem_my_pe.3 \
+        shmem_n_pes.3 \
+        shmem_pe_accessible.3 \
+        shmem_ptr.3 \
+        shmem_put128.3 \
+        shmem_put128_nbi.3 \
+        shmem_put16_nbi.3 \
+        shmem_put32.3 \
+        shmem_put32_nbi.3 \
+        shmem_put64.3 \
+        shmem_put64_nbi.3 \
+        shmem_put8_nbi.3 \
+        shmem_putmem.3 \
+        shmem_putmem_nbi.3 \
+        shmem_quiet.3 \
+        shmem_realloc.3 \
+        shmem_set_cache_inv.3 \
+        shmem_set_cache_line_inv.3 \
+        shmem_set_lock.3 \
+        shmem_short_and_to_all.3 \
+        shmem_short_g.3 \
+        shmem_short_get.3 \
+        shmem_short_get_nbi.3 \
+        shmem_short_iget.3 \
+        shmem_short_iput.3 \
+        shmem_short_max_to_all.3 \
+        shmem_short_min_to_all.3 \
+        shmem_short_or_to_all.3 \
+        shmem_short_p.3 \
+        shmem_short_prod_to_all.3 \
+        shmem_short_put.3 \
+        shmem_short_put_nbi.3 \
+        shmem_short_sum_to_all.3 \
+        shmem_short_wait.3 \
+        shmem_short_wait_until.3 \
+        shmem_short_xor_to_all.3 \
+        shmem_swap.3 \
+        shmem_test_lock.3 \
+        shmem_udcflush.3 \
+        shmem_udcflush_line.3 \
+        shmem_wait.3 \
+        shmem_wait_until.3 \
+        shrealloc.3 \
+        start_pes.3
+
+MAN_OUTDIR = $(OUTDIR)/man
+
+# If we're building the docs, then we install from the just-built
+# docs.  Otherwise, we install from the pre-built docs (i.e., the docs
+# included in the tarball).
+#
+# NOTE: If we're in a git clone with a) no pre-built docs and b)
+# Sphinx is not found, then both OPAL_BUILD_DOCS and OPAL_INSTALL_DOCS
+# will be false, and the value of MAN_INSTALL_FROM will not not used.
+if OPAL_BUILD_DOCS
+MAN_INSTALL_FROM = $(MAN_OUTDIR)
+HTML_INSTALL_FROM = $(OUTDIR)/html
+else
+MAN_INSTALL_FROM = man
+HTML_INSTALL_FROM = html
+endif
+
+# For each of the man page macros below:
+#
+# *_RST: the .rst source files
+# *_BUILT: the files in the _build/man directory
+# *_INSTALL_FROM: the files in either the _build/man/ directory (if we
+#   are building the Sphinx docs) or the man/ directory (if we are not
+#   building the Sphinx docs, and are using the pre-built docs that
+#   are included in the tarballl).
+OMPI_MAN1_RST = $(OMPI_MAN1:%.1=man-openmpi/man1/%.1.rst)
+OMPI_MAN1_BUILT = $(OMPI_MAN1:%.1=$(MAN_OUTDIR)/%.1)
+OMPI_MAN1_INSTALL_FROM = $(OMPI_MAN1:%.1=$(MAN_INSTALL_FROM)/%.1)
+
+OMPI_MAN3_RST = $(OMPI_MAN3:%.3=man-openmpi/man3/%.3.rst)
+OMPI_MAN3_BUILT = $(OMPI_MAN3:%.3=$(MAN_OUTDIR)/%.3)
+OMPI_MAN3_INSTALL_FROM = $(OMPI_MAN3:%.3=$(MAN_INSTALL_FROM)/%.3)
+
+# Use this one file as a sentinel for building all the Open MPI man
+# page API bindings files
+SENTINEL_OMPI_MAN3_BINDING = $(builddir)/man-openmpi/man3/bindings/mpi_init.rst
+
+OMPI_MAN7_RST = $(OMPI_MAN7:%.7=man-openmpi/man7/%.7.rst)
+OMPI_MAN7_BUILT = $(OMPI_MAN7:%.7=$(MAN_OUTDIR)/%.7)
+OMPI_MAN7_INSTALL_FROM = $(OMPI_MAN7:%.7=$(MAN_INSTALL_FROM)/%.7)
+
+OSHMEM_MAN1_RST = $(OSHMEM_MAN1:%.1=man-oshmem/man1/%.1.rst)
+OSHMEM_MAN1_BUILT = $(OSHMEM_MAN1:%.1=$(MAN_OUTDIR)/%.1)
+OSHMEM_MAN1_INSTALL_FROM = $(OSHMEM_MAN1:%.1=$(MAN_INSTALL_FROM)/%.1)
+
+OSHMEM_MAN3_RST = $(OSHMEM_MAN3:%.3=man-oshmem/man3/%.3.rst)
+OSHMEM_MAN3_BUILT = $(OSHMEM_MAN3:%.3=$(MAN_OUTDIR)/%.3)
+OSHMEM_MAN3_INSTALL_FROM = $(OSHMEM_MAN3:%.3=$(MAN_INSTALL_FROM)/%.3)
+
+###########################################################################
+
+# These files are not generated by Sphinx (they are redirects to
+# Sphinx-generated man pages) -- they exist directly in Git.  We
+# *always* want these files to be in EXTRA_DIST (so that they're in
+# distribution tarballs).  We'll decide later whether we install each
+# of these (e.g., if configure disabled the Java bindings, we won't
+# install mpijavac.1).
+OMPI_MAN1_C_REDIRECTS = man-openmpi/man1/mpicc.1
+OMPI_MAN1_CXX_REDIRECTS = \
+        man-openmpi/man1/mpicxx.1 \
+        man-openmpi/man1/mpic++.1
+OMPI_MAN1_FORTRAN_REDIRECTS = \
+        man-openmpi/man1/mpifort.1 \
+        man-openmpi/man1/mpif77.1 \
+        man-openmpi/man1/mpif90.1
+OMPI_MAN1_JAVA_REDIRECTS = man-openmpi/man1/mpijavac.1
+
+OSHMEM_MAN1_C_REDIRECTS = \
+        man-openshmem/man1/oshcc.1 \
+        man-openshmem/man1/shmemcc.1
+OSHMEM_MAN1_CXX_REDIRECTS = \
+        man-openshmem/man1/oshcxx.1 \
+        man-openshmem/man1/oshc++.1 \
+        man-openshmem/man1/shmemcxx.1 \
+        man-openshmem/man1/shmemc++.1
+OSHMEM_MAN1_FORTRAN_REDIRECTS = \
+        man-openshmem/man1/oshfort.1 \
+        man-openshmem/man1/shmemfort.1
+
+EXTRA_DIST += \
+        $(OMPI_MAN1_C_REDIRECTS) \
+        $(OMPI_MAN1_CXX_REDIRECTS) \
+        $(OMPI_MAN1_FORTRAN_REDIRECTS) \
+        $(OMPI_MAN1_JAVA_REDIRECTS) \
+        $(OSHMEM_MAN1_C_REDIRECTS) \
+        $(OSHMEM_MAN1_CXX_REDIRECTS) \
+        $(OSHMEM_MAN1_FORTRAN_REDIRECTS)
+
+
+###########################################################################
+
+ALL_MAN_BUILT = \
+        $(OMPI_MAN1_BUILT) $(OMPI_MAN3_BUILT) $(OMPI_MAN7_BUILT) \
+        $(OSHMEM_MAN1_BUILT) $(OSHMEM_MAN_3_BUILT)
+
+# These 2 targets are used in EXTRA_DIST: we make a full copy of the
+# built HTML and man docs into a separate location that is included in
+# the tarball.  This gives users a fully copy of the docs included in
+# distribution tarballs.
+#
+# Note: we name this "html-local" because otherwise Automake issues a
+# warning about us overriding its default "html" target.  In our
+# particular use case, the rules generated if we name this target
+# "html-local" instead of "html" are effectively equivalent to when we
+# name this target "html", so we might as well avoid the Automake
+# warning.
+html-local: $(ALL_MAN_BUILT)
+	$(OMPI_V_COPYALL) rm -rf html; cp -rp $(OUTDIR)/html .
+
+man: $(ALL_MAN_BUILT)
+	$(OMPI_V_COPYALL) rm -rf man; cp -rp $(OUTDIR)/man .
+
+# Remove the copies of the built HTML and man pages to get back to a
+# clean git clone.
+maintainer-clean-local:
+	rm -rf html man man-openmpi/man3/bindings
+
+# If we're doing a VPATH build, we may have "html" and "man"
+# directories in the build tree (e.g., if we did "make dist").  Remove
+# these copies so that we can pass distcheck (of course: we never
+# remove these directories from the source tree).
+distclean-local:
+	if test "$(srcdir)" != "$(builddir)"; then \
+	    rm -rf html man; \
+	fi
+
+###########################################################################
+if OPAL_BUILD_DOCS
+
+include $(top_srcdir)/Makefile.ompi-rules
+
+# Copy over the PRRTE RST files to this build tree.
+#
+# 1. If we're building with PRRTE support:
+#
+#    1a. If we're building the internal/bundled PRRTE, then we'll copy
+#        the internal/bundled PRRTE's RST files to the build tree.
+#    1b. If we're building against an external PRRTE installation that
+#        has RST files in its install tree, then we'll copy that
+#        external PRRTE's RST files to the build tree.
+#    1c. If we're building against an external PRRTE installation that
+#        does NOT have RST files in its install tree, then we'll
+#        create some dummy RST files instead.
+#
+# 2. If we're building without PRRTE support, we'll create some dummy
+# RST files instead.
+#
+# NOTE: We specifically list $(builddir) in the target name, just to
+# ensure that "make" doesn't accidentally find this directory in the
+# VPATH srcdir, and therefore not execute this rule (because Sphinx
+# does not understand VPATH, and will ignore this directory in the
+# VPATH srcdir).  We can have this directory in the srcdir by doing a
+# VPATH build of an official distribution tarball.
+
+# Make the 2 directories that we need: schizo-ompi-rst-content and
+# prrte-rst-content.
+$(builddir)/schizo-ompi-rst-content:
+	$(OMPI_V_MKDIR) if test ! -d "$@"; then mkdir "$@"; fi
+$(builddir)/prrte-rst-content:
+	$(OMPI_V_MKDIR) if test ! -d "$@"; then mkdir "$@"; fi
+
+# Get the schizo-ompi-cli.rst file that we need.  CAVEAT: we name it
+# ".rstxt" so that Sphinx doesn't slurp it in via two different
+# locations in the RST docroot (i.e., via
+# /schizo-ompi-rst-content/schizo-ompi-cli.rstxt and via
+# /man-openmpi/man1/mpirun.1.rst).  Sphinx *shouldn't* do this -- it
+# should see the ".. include...." directive in mpirun.1.rst and *only*
+# include the file once.  But somehow it's also seeing it a 2nd time.
+# So -- fine.  We'll name it something other than .rst so that Sphinx
+# doesn't do that.
+#
+# Regardless, either copy this file from the PRRTE install tree or
+# make a bogus one (if we don't have one in the PRRTE install tree).
+#
+# Also, note: the rule to make the $(builddir)/schizo-ompi-rst-content
+# directory must be in the AM_CONDITIONAL here, otherwise Automake
+# complains.  Meaning: we have to have same dependency listed in both
+# the "if" and the "else" blocks.  Grumble.
+if OMPI_HAVE_PRRTE_RST
+$(builddir)/schizo-ompi-rst-content/schizo-ompi-cli.rstxt: $(builddir)/schizo-ompi-rst-content
+$(builddir)/schizo-ompi-rst-content/schizo-ompi-cli.rstxt: $(OMPI_SCHIZO_OMPI_RST_CONTENT_DIR)/*
+	$(OMPI_V_SPHINX_COPYRST) \
+	dir=`dirname $@`; \
+	cp -rpf $(OMPI_SCHIZO_OMPI_RST_CONTENT_DIR)/* "$$dir"
+else
+$(builddir)/schizo-ompi-rst-content/schizo-ompi-cli.rstxt: $(builddir)/schizo-ompi-rst-content
+$(builddir)/schizo-ompi-rst-content/schizo-ompi-cli.rstxt: $(srcdir)/no-prrte-content.rst.txt
+	$(OMPI_V_SPHINX_COPYRST) \
+	dir=`dirname $@`; if test ! -d "$$dir"; then mkdir "$$dir"; fi; \
+	cp -pf $(srcdir)/no-prrte-content.rst.txt "$@"
+endif
+
+$(builddir)/man-openmpi/man3/bindings:
+	$(OMPI_V_MKDIR) if test ! -d "$@"; then mkdir -p "$@"; fi
+
+$(SENTINEL_OMPI_MAN3_BINDING): $(builddir)/man-openmpi/man3/bindings
+$(SENTINEL_OMPI_MAN3_BINDING): generate-mpi-man3-bindings.py
+$(SENTINEL_OMPI_MAN3_BINDING): mpi-standard-apis.json
+	$(OMPI_V_GEN) $(PYTHON3) $(srcdir)/generate-mpi-man3-bindings.py \
+		--srcdir $(srcdir) --builddir $(builddir)
+
+$(ALL_MAN_BUILT): $(builddir)/prrte-rst-content
+$(ALL_MAN_BUILT): $(builddir)/schizo-ompi-rst-content/schizo-ompi-cli.rstxt
+$(ALL_MAN_BUILT): $(RST_SOURCE_FILES) $(IMAGE_SOURCE_FILES)
+$(ALL_MAN_BUILT): $(TEXT_SOURCE_FILES) $(SPHINX_CONFIG)
+$(ALL_MAN_BUILT): $(SENTINEL_OMPI_MAN3_BINDING)
+
+# Render the RST source into both 1) full HTML docs and 2) nroff man
+# pages.
+#
+# List both commands (HTML and man) in a single rule because they
+# really need to be run in serial.  Specifically, if they were two
+# different rules and someone ran "make -j", then both of them could
+# be writing to $(OUTDIR)/doctrees simultaneously, which would be Bad.
+# Use one of the man pages as a sentinel file to indicate whether all
+# the HTML docs and man pages have been built.
+#
+# It's therefore a little bit of a lie to have the target named
+# $(ALL_MAN_BUILT) *also* generate all the HTML content, but... so be
+# it.
+#
+# Also note that Open MPI's RST includes some conditional RST (from
+# PRRTE -- i.e., whether we get the source RST from the internal
+# PRRTE, an external PRRTE, or whether we create RST files from
+# scratch).  These conditionals mean that we have to make some changes
+# to the input Sphinx RST tree before building it.  But -- by Automake
+# convention -- we can't modify the source tree.  Hence, we have to
+# copy over all the source RST files -- including its internal
+# directory structure -- to the build tree, and then make our desired
+# changes here in the build tree.  This is a bit ugly, but we could
+# not think of anything better to do.
+#
+# NOTE: This is a little gross in that for a VPATH build, we *always*
+# copy from the source tree to the build tree (if the target does not
+# exist or any of the sources in the source tree -- thanks to
+# make/VPATH handling -- have changed compared to the target).
+# However, we're using "cp -p", so even though we're copying *all the
+# sources* from the source tree to the build tree, the timestamp will
+# reflect what is in the source tree.  Hence, if the source file has
+# not changed, then it won't look like the file in the build tree has
+# changed.  We're going to overwrite any local changes in the build
+# tree, but you shouldn't be editing the build tree, anyway.  So --
+# good enough.
+
+$(ALL_MAN_BUILT):
+	$(OMPI_V_SPHINX_COPYRST) if test "$(srcdir)" != "$(builddir)"; then \
+	    len=`echo "$(srcdir)/" | wc -c`; \
+	    for file in $(RST_SOURCE_FILES) $(IMAGE_SOURCE_FILES) $(TEXT_SOURCE_FILES) $(SPHINX_CONFIG); do \
+	        dir=`dirname $$file | cut -c$$len-`; \
+	        if test -z "$$dir"; then \
+	            dir=.; \
+	        fi; \
+	        if test ! -d "$$dir"; then \
+	            mkdir -p "$$dir"; \
+	        fi; \
+	        cp -p "$$file" "$$dir"; \
+	    done; \
+	fi; \
+	cp -rpf "$(OMPI_PRRTE_RST_CONTENT_DIR)" "$(builddir)"; \
+	copied_dir=`basename $(OMPI_PRRTE_RST_CONTENT_DIR)`; \
+	chmod -R u+w "$(builddir)/$$copied_dir"
+	$(OMPI_V_SPHINX_HTML) OMPI_TOP_SRCDIR=$(top_srcdir) $(SPHINX_BUILD) -M html "$(builddir)" "$(OUTDIR)" $(SPHINX_OPTS)
+	$(OMPI_V_SPHINX_MAN) OMPI_TOP_SRCDIR=$(top_srcdir) $(SPHINX_BUILD) -M man "$(builddir)" "$(OUTDIR)" $(SPHINX_OPTS)
+
+# A useful rule to invoke manually to ensure that all of the external
+# HTML links we have are valid.  Running this rule requires
+# connectivity to the general internet.
+linkcheck:
+	$(SPHINX_BUILD) -M linkcheck "$(builddir)" "$(OUTDIR)" $(SPHINX_OPTS)
+
+.PHONY: linkcheck
+
+# Since we are building the docs, we built $(OUTDIR).  Hence, we need
+# to delete it during "make clean".  Note that we can't add
+# directories to CLEANFILES, because Automake only (effectively) does
+# "rm -f $(CLEANFILES)" (not "rm -rf ...").  So we have to delete
+# directories ourselves.
+#
+# Also, if this is a VPATH build, then we made a copy of a bunch of
+# RST source files to the build tree.  So delete all of those, too.
+clean-local:
+	rm -rf $(OUTDIR)
+	rm -rf prrte-rst-content schizo-ompi-rst-content
+	if test "$(srcdir)" != "$(builddir)"; then \
+	    len=`echo "$(srcdir)/" | wc -c`; \
+	    for file in $(RST_SOURCE_FILES) $(IMAGE_SOURCE_FILES) $(TEXT_SOURCE_FILES) $(SPHINX_CONFIG); do \
+	        dir=`dirname $$file | cut -c$$len-`; \
+	        if test -z "$$dir"; then \
+	            rm -rf `basename $$file`; \
+	        fi; \
+	        if test -n "$$dir" && test -d "$$dir"; then \
+	            rm -rf "$$dir"; \
+	        fi; \
+	    done; \
+	fi
+
+# List all the built man pages here in the Automake BUILT_SOURCES
+# macro.  This hooks into the normal Automake build mechanisms, and
+# will ultimately cause the invocation of the above rule that runs
+# Sphinx to build the HTML and man pages.
+BUILT_SOURCES = $(ALL_MAN_BUILT)
+
+endif OPAL_BUILD_DOCS
+
+###########################################################################
+if OPAL_INSTALL_DOCS
+
+man1_MANS = \
+        $(OMPI_MAN1_INSTALL_FROM) \
+        $(OMPI_MAN1_C_REDIRECTS)
+if OMPI_HAVE_CXX_COMPILER
+man1_MANS += $(OMPI_MAN1_CXX_REDIRECTS)
+endif
+if OMPI_BUILD_ANY_FORTRAN_BINDINGS
+man1_MANS += $(OMPI_MAN1_FORTRAN_REDIRECTS)
+endif
+if OMPI_WANT_JAVA_BINDINGS
+man1_MANS += $(OMPI_MAN1_JAVA_REDIRECTS)
+endif
+
+man3_MANS = $(OMPI_MAN3_INSTALL_FROM)
+man7_MANS = $(OMPI_MAN7_INSTALL_FROM)
+
+if PROJECT_OSHMEM
+man1_MANS += \
+        $(OSHMEM_MAN1_INSTALL_FROM) \
+        $(OSHMEM_MAN1_C_REDIRECTS)
+# There is no OSHMEM equivalent of this conditional; just use the OMPI
+# conditional.
+if OMPI_HAVE_CXX_COMPILER
+man1_MANS += $(OSHMEM_MAN1_CXX_REDIRECTS)
+endif
+if OSHMEM_BUILD_FORTRAN_BINDINGS
+man1_MANS += $(OSHMEM_MAN1_FORTRAN_REDIRECTS)
+endif
+
+man3_MANS += $(OSHMEM_MAN3_INSTALL_FROM)
+endif
+
+# We do not know the names of all the generated HTML files: we only
+# know that the generated tree will of files be in _build/html/.
+# Unfortunately, Automake's installation process only handles
+# individual files -- not entire trees of files.  Hence, we use a
+# "find ..." to copy the entire generated tree to the target
+# directory.
+#
+# Note: we do not use "cp -r ..." because if the source directory is
+# write-protected (e.g., during "make distcheck"), then the cp -r will
+# also write-protect the target directory.  Instead, use the
+# Automake-provided install macros to set desirable permissions on the
+# target directories and files.
+#
+# Check to see if we actually built the docs.  If we did, copy from
+# the _build/html tree in the builddir.  In all other cases, see if
+# there's a _build/html in the source tree (e.g., if this is a build
+# from a tarball that included a _build/html); if that exists, copy
+# from that.
+#
+# NOTE: We can't use the AM_CONDITIONAL OPAL_BUILD_DOCS in the middle
+# of a block that uses the shell continuation character at the end of
+# each line.  Instead, we check if $(SPHINX_BUILD) is non-empty, which
+# is the test used to construct OPAL_BUILD_DOCS.
+install-data-hook:
+	$(MKDIR_P) $(DESTDIR)$(docdir)
+	topdir= ; \
+	if test -n "$(SPHINX_BUILD)" && test -d $(builddir)/$(HTML_INSTALL_FROM); then \
+	    topdir="$(builddir)/$(HTML_INSTALL_FROM)"; \
+	elif test -d $(srcdir)/$(HTML_INSTALL_FROM); then \
+	    topdir="$(srcdir)/$(HTML_INSTALL_FROM)"; \
+	fi; \
+	if test -n "$$topdir"; then \
+	    cd $$topdir/..; \
+	    find html -type d -exec $(mkinstalldirs) $(DESTDIR)$(docdir)/{} \; ; \
+	    find html -type f -exec $(INSTALL_DATA) {} $(DESTDIR)$(docdir)/{} \; ; \
+	fi
+
+uninstall-hook:
+	rm -rf $(DESTDIR)$(docdir)
+
+endif OPAL_INSTALL_DOCS
diff --git a/docs/app-debug/debug-options.rst b/docs/app-debug/debug-options.rst
new file mode 100644
index 00000000000..890ce2f07cd
--- /dev/null
+++ b/docs/app-debug/debug-options.rst
@@ -0,0 +1,61 @@
+Open MPI Runtime Debugging Options
+==================================
+
+Open MPI has a series of MCA parameters for the MPI layer
+itself that are designed to help with debugging.
+These parameters :ref:`can be set <label-running-setting-mca-param-values>`
+in the usual ways.  MPI-level MCA parameters can be
+displayed by invoking the following command:
+
+.. code-block:: sh
+
+   # Use "--level 9" to see all the MCA parameters
+   # (the default is "--level 1"):
+   shell$ ompi_info --param mpi all --level 9
+
+Here is a summary of the debugging parameters for the MPI layer:
+
+* ``mpi_param_check``: If set to true (any positive value), and when
+  Open MPI is compiled with parameter checking enabled (the default),
+  the parameters to each MPI function can be passed through a series
+  of correctness checks.  Problems such as passing illegal values
+  (e.g., NULL or ``MPI_DATATYPE_NULL`` or other "bad" values) will be
+  discovered at run time and an MPI exception will be invoked (the
+  default of which is to print a short message and abort the entire
+  MPI job).  If set to false, these checks are disabled, slightly
+  increasing performance.
+
+* ``mpi_show_handle_leaks``: If set to true (any positive value),
+  Open MPI will display lists of any MPI handles that were not freed before
+  :ref:`MPI_Finalize(3) <mpi_finalize>`  (e.g., communicators,
+  datatypes, requests, etc.)
+
+* ``mpi_no_free_handles``: If set to true (any positive value), do not
+  actually free MPI objects when their corresponding MPI "free"
+  function is invoked (e.g., do not free communicators when
+  :ref:`MPI_Comm_free(3) <mpi_comm_free>`.  This can be helpful in tracking down
+  applications that accidentally continue to use MPI handles after
+  they have been freed.
+
+* ``mpi_show_mca_params``: If set to true (any positive value), show a
+  list of all MCA parameters and their values when MPI is initialized.
+  This can be quite helpful for reproducibility of MPI applications.
+
+* ``mpi_show_mca_params_file``: If set to a non-empty value, and if
+  the value of ``mpi_show_mca_params`` is true, then output the list
+  of MCA parameters to the filename value.  If this parameter is an
+  empty value, the list is sent to ``stderr``.
+
+* ``mpi_abort_delay``: If nonzero, print out an identifying message
+  when :ref:`MPI_Abort(3) <mpi_abort>` is invoked showing the hostname and PID of the
+  process that invoked :ref:`MPI_Abort(3) <mpi_abort>`, and then delay that many seconds
+  before exiting.  A negative value means to delay indefinitely.  This
+  allows a user to manually come in and attach a debugger when an
+  error occurs.  Remember that the default MPI error handler |mdash|
+  ``MPI_ERRORS_ABORT`` |mdash| invokes :ref:`MPI_Abort(3) <mpi_abort>`, so this
+  parameter can be useful to discover problems identified by
+  ``mpi_param_check``.
+
+* ``mpi_abort_print_stack``: If nonzero, print out a stack trace (on
+  supported systems) when :ref:`MPI_Abort(3) <mpi_abort>` is invoked.
+
diff --git a/docs/app-debug/debug-tools.rst b/docs/app-debug/debug-tools.rst
new file mode 100644
index 00000000000..ead997a150b
--- /dev/null
+++ b/docs/app-debug/debug-tools.rst
@@ -0,0 +1,19 @@
+Parallel Debugging Tools
+========================
+
+There are two main categories of tools that can aid in
+parallel debugging:
+
+* **Debuggers:** Both serial and parallel debuggers are useful.  Serial
+  debuggers are what most programmers are used to (e.g.,
+  the GNU debugger, ``gdb``), while
+  parallel debuggers can attach to all the individual processes in an
+  MPI job simultaneously, treating the MPI application as a single
+  entity.  This can be an extremely powerful abstraction, allowing the
+  user to control every aspect of the MPI job, manually replicate race
+  conditions, etc.
+
+* **Profilers:** Tools that analyze your usage of MPI and display
+  statistics and meta information about your application's run.  Some
+  tools present the information "live" (as it occurs), while others
+  collect the information and display it in a post mortem analysis.
diff --git a/docs/app-debug/index.rst b/docs/app-debug/index.rst
new file mode 100644
index 00000000000..91bf934b6c4
--- /dev/null
+++ b/docs/app-debug/index.rst
@@ -0,0 +1,30 @@
+.. Open MPI Application Debugging
+
+Debugging Open MPI Parallel Applications
+========================================
+
+Debugging a serial applications includes solving problems like
+logic errors, uninitialized variables, storage overlays and timing
+problems.
+
+Debugging a parallel application can be further complicated
+by problems that can include additional race conditions and aysynchronous
+events, as well as understanding execution of multiple application
+processes running simultaneously.
+
+This section of the documentation describes some techniques that can
+be useful for parallel debugging. This section also describes some
+tools that can be useful as well as some Open MPI runtime options
+that can aid debugging.
+
+.. toctree::
+   :maxdepth: 1
+
+   debug-tools
+   debug-options
+   serial-debug
+   parallel-debug
+   lost-output
+   memchecker
+   valgrind
+   mpir-tools
diff --git a/docs/app-debug/lost-output.rst b/docs/app-debug/lost-output.rst
new file mode 100644
index 00000000000..57af5cd5f25
--- /dev/null
+++ b/docs/app-debug/lost-output.rst
@@ -0,0 +1,34 @@
+Application Output Lost with Abnormal Termination
+=================================================
+
+There many be many reasons for application output to be lost when
+an application abnormally terminates. The Open MPI Team strongly
+encourages the use of tools (such as debuggers) whenever possible.
+
+One of the reasons, however, may come from inside Open MPI itself.  If
+your application fails due to memory corruption, Open MPI may
+subsequently fail to output an error message before terminating.
+Open MPI attempts to aggregate error
+messages from multiple processes in an attempt to show unique error
+messages only once (vs. once for each MPI process |mdash| which can be
+unwieldy, especially when running large MPI jobs).
+
+However, this aggregation process requires allocating memory in the
+MPI process when it displays the error message.  If the process's
+memory is already corrupted, Open MPI's attempt to allocate memory may
+fail and the process will simply terminate, possibly silently.  When Open
+MPI does not attempt to aggregate error messages, most of its setup
+work is done when the MPI library is initiaized  and no memory is allocated
+during the "print the error" routine.  It therefore almost always successfully
+outputs error messages in real time |mdash| but at the expense that you'll
+potentially see the same error message for *each* MPI process that
+encountered the error.
+
+Hence, the error message aggregation is *usually* a good thing, but
+sometimes it can mask a real error.  You can disable Open MPI's error
+message aggregation with the ``opal_base_help_aggregate`` MCA
+parameter.  For example:
+
+.. code-block:: sh
+
+   shell$ mpirun --mca opal_base_help_aggregate 0 ...
diff --git a/docs/app-debug/memchecker.rst b/docs/app-debug/memchecker.rst
new file mode 100644
index 00000000000..6e6b536722c
--- /dev/null
+++ b/docs/app-debug/memchecker.rst
@@ -0,0 +1,193 @@
+Using Memchecker
+================
+
+The Memchecker functionality in Open MPI provides MPI semantic
+checking for your application (as well as internals of Open MPI), with
+the help of memory checking tools such as the ``memcheck`` component of
+`the Valgrind suite <https://www.valgrind.org/>`_.
+
+/////////////////////////////////////////////////////////////////////////
+
+Types of Errors Detected by Memchecker
+--------------------------------------
+
+Open MPI's Memchecker is based on the ``memcheck`` tool included with
+Valgrind, so it takes all the advantages from it. Firstly, it checks
+all reads and writes of memory, and intercepts calls to
+``malloc(3)``/``free(3)`` and C++'s ``new``/``delete`` operators.
+Most importantly, Memchecker is able to detect
+the user buffer errors in both non-blocking and one-sided
+communications, e.g. reading or writing to buffers of active
+non-blocking receive operations and writing to buffers of active
+non-blocking send operations.
+
+Here are some example problems that Memchecker can detect:
+
+Accessing buffer under control of non-blocking communication:
+
+.. code-block:: c
+
+   int buf;
+   MPI_Irecv(&buf, 1, MPI_INT, 1, 0, MPI_COMM_WORLD, &req);
+   // The following line will produce a memchecker warning
+   buf = 4711;
+   MPI_Wait (&req, &status);
+
+Wrong input parameters, e.g., wrong-sized send buffers:
+
+.. code-block:: c
+
+   char *send_buffer;
+   send_buffer = malloc(5);
+   memset(send_buffer, 0, 5);
+   // The following line will produce a memchecker warning
+   MPI_Send(send_buffer, 10, MPI_CHAR, 1, 0, MPI_COMM_WORLD);
+
+Accessing a window in a one-sided communication:
+
+.. code-block:: c
+
+   MPI_Get(A, 10, MPI_INT, 1, 0, 1, MPI_INT, win);
+   A[0] = 4711;
+   MPI_Win_fence(0, win);
+
+Uninitialized input buffers:
+
+.. code-block:: c
+
+   char *buffer;
+   buffer = malloc(10);
+   // The following line will produce a memchecker warning
+   MPI_Send(buffer, 10, MPI_INT, 1, 0, MPI_COMM_WORLD);
+
+Usage of the uninitialized ``MPI_Status`` field in ``MPI_ERROR``
+structure: (the MPI-1 standard defines the ``MPI ERROR`` field to be
+undefined for single-completion calls such as :ref:`MPI_Wait(3) <mpi_wait>` or
+:ref:`MPI_Test(3) <mpi_test>`, see MPI-1 p. 22):
+
+.. code-block:: c
+
+   MPI_Wait(&request, &status);
+   // The following line will produce a memchecker warning
+   if (status.MPI_ERROR != MPI_SUCCESS)
+       return ERROR;
+
+/////////////////////////////////////////////////////////////////////////
+
+Building Open MPI with Memchecker Support
+-----------------------------------------
+
+To use Memchecker, you need Valgrind 3.2.0 or later, and have an Open
+MPI that was configured with the ``--enable-memchecker`` and
+``--enable-debug`` flags.
+
+.. note:: The Memchecker functionality is off by default, because it
+          incurs a performance penalty.
+
+When ``--enable-memchecker`` is specified, ``configure`` will check
+for a recent-enable valgrind distribution.  If found, Open MPI will
+build Memchecker support.
+
+For example:
+
+.. code-block:: sh
+
+   shell$ ./configure --prefix=/path/to/openmpi --enable-debug \
+       --enable-memchecker --with-valgrind=/path/to/valgrind
+
+You can check that Open MPI was built with Memchecker support by using
+the :ref:`ompi_info(1) <man1-ompi_info>` command.
+
+.. code-block:: sh
+
+   # The exact version numbers shown may be different for your Open
+   # MPI installation
+   shell$ ompi_info | grep memchecker
+   MCA memchecker: valgrind (MCA v1.0, API v1.0, Component v1.3)
+
+If you do not see the "MCA memchecker: valgrind" line, you probably
+did not configure and install Open MPI correctly.
+
+/////////////////////////////////////////////////////////////////////////
+
+Running an Open MPI Application with Memchecker
+-----------------------------------------------
+
+After Open MPI was built and installed with Memchecker support, 
+simply run your application with Valgrind, e.g.:
+
+.. code-block:: sh
+
+   shell$ mpirun -n 2 valgrind ./my_app
+
+If you enabled Memchecker, but you don't want to check the
+application at this time, then just run your application as
+usual. E.g.:
+
+.. code-block:: sh
+
+   shell$ mpirun -n 2 ./my_app
+
+/////////////////////////////////////////////////////////////////////////
+
+Application Performance Impacts Using Memchecker
+------------------------------------------------
+
+The configure option ``--enable-memchecker`` (together with
+``--enable-debug``) *does* cause performance degradation, even if not
+running under Valgrind.  The following explains the mechanism and may
+help in making the decision whether to provide a cluster-wide
+installation with ``--enable-memchecker``.
+
+There are two cases:
+
+#. If run without Valgrind, the Valgrind ClientRequests (assembler
+   instructions added to the normal execution path for checking) do
+   not affect overall MPI performance. Valgrind ClientRequests are
+   explained in detail `in Valgrind's documentation
+   <https://valgrind.org/docs/manual/manual-core-adv.html>`_.
+   In the case of x86-64, ClientRequests boil down to the following
+   four rotate-left (ROL) and one xchange (XCHG) assembler instructions
+   from ``valgrind.h``:
+
+   .. code-block:: c
+
+      #define __SPECIAL_INSTRUCTION_PREAMBLE                      \
+                     "rolq \$3,  %%rdi; rolq \$13, %%rdi\\n\\t"   \
+                     "rolq \$61, %%rdi; rolq \$51, %%rdi\\n\\t"
+
+   and
+
+   .. We do not make the code block below as "c" because the Sphinx C
+      syntax highlighter fails to parse it as C and emits a warning.
+      So we might as well just leave it as a plan verbatim block
+      (i.e., not syntax highlighted).
+
+   .. code-block::
+
+      __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
+                     /* %RDX = client_request ( %RAX ) */           \
+                     "xchgq %%rbx,%%rbx"                            \
+                     : "=d" (_zzq_result)                           \
+                     : "a" (& _zzq_args``0``), "0" (_zzq_default)   \
+                     : "cc", "memory"                               \
+                    );
+
+   for every single ClientRequest.  In the case of not running
+   Valgrind, these ClientRequest instructions do not change the
+   arithmetic outcome (rotating a 64-bit register left by 128-Bits,
+   exchanging a register with itself), except for the carry flag.
+
+   The first request is checking whether we're running under Valgrind.
+   In case we're not running under Valgrind subsequent checks (a.k.a.
+   ClientRequests) are not done.
+
+#. If the application is run under Valgrind, performance is naturally reduced due
+   to the Valgrind JIT and the checking tool employed.
+   For costs and overheads of Valgrind's Memcheck tool on the SPEC 2000 Benchmark,
+   please see the excellent paper
+   `Valgrind: A Framework for Heavyweight Dynamic Binary Instrumentation
+   <https://valgrind.org/docs/valgrind2007.pdf>`_.
+   For an evaluation of various internal implementation alternatives of Shadow Memory, please see
+   `Building Workload Characterization Tools with Valgrind
+   <https://valgrind.org/docs/iiswc2006.pdf>`_.
diff --git a/docs/app-debug/mpir-tools.rst b/docs/app-debug/mpir-tools.rst
new file mode 100644
index 00000000000..953e6bf5ec7
--- /dev/null
+++ b/docs/app-debug/mpir-tools.rst
@@ -0,0 +1,127 @@
+.. _using-mpir-based-tools-label:
+
+Using MPIR-based tools with Open MPI
+====================================
+
+The `legacy MPIR specification
+<https://www.mpi-forum.org/docs/mpir-specification-03-01-2018.pdf>`_
+is the specification of an interface that debuggers and others tools
+can use to obtain the PID, hostname, and executable name for each MPI
+application process.
+
+This specification has been replaced by the PMIx tools API that is part of the
+`OpenPMIx project <https://openpmix.github.io/>`_.
+
+Debuggers and tools which use the legacy MPIR specification can
+continue functioning correctly with the `MPIR shim module
+<https://github.com/hpc/mpir-to-pmix-guide>`_ described in this
+section.
+
+Obtaining Application Process Mapping Information
+-------------------------------------------------
+
+The `MPIR shim module <https://github.com/hpc/mpir-to-pmix-guide>`_ is
+a stand-alone module which can be used by debuggers and tools that
+still utilize the legacy MPIR interface to attach to parallel MPI
+applications.  The debugger or tool launches the shim module as an
+intermediate process between it and the application's :ref:`mpirun(1)
+<man1-mpirun>` command.
+
+The MPIR shim module implements the ``MPIR_Breakpoint`` function, as a
+hook where debuggers and tools can set a breakpoint to read the MPIR
+processing mapping when it is available.
+
+Instructions for use of the shim module are available at
+https://github.com/hpc/mpir-to-pmix-guide/blob/master/README.md.
+
+You may need to build this shim module before you can use it. The
+source code for this module can be downloaded from
+`MPIR to PMIx Shim repository <https://github.com/hpc/mpir-to-pmix-guide>`_.
+
+Once you have the MPIR shim module built and installed, follow the
+instructions included with the debugger or tool to tell it where to
+find the MPIR shim module.
+
+/////////////////////////////////////////////////////////////////////////
+
+Building the MPIR Shim Module
+-----------------------------
+
+Here is an example shell script that builds the MPIR shim module
+|mdash| you may need to customize it for your specific environment:
+
+.. code-block:: bash
+
+    #!/usr/bin/env bash
+
+    set -euxo pipefail
+
+    cd $HOME
+    mkdir -p git
+    cd git
+
+    git clone https://github.com/hpc/mpir-to-pmix-guide.git
+    cd mpir-to-pmix-guide
+
+    # PMIX_ROOT must be set to your PMIx install root directory
+    # (which may be the same as your Open MPI installation directory)
+    if test -z "$PMIX_ROOT" -o ! -d "$PMIX_ROOT"; then
+        echo "Set the env variable PMIX_ROOT to the location of your PMIx installation"
+        exit 1
+    fi
+    
+    ./autogen.sh
+    ./configure --prefix=$HOME/MPIR --with-pmix=$PMIX_ROOT
+    make
+    make install
+
+
+/////////////////////////////////////////////////////////////////////////
+
+Verification of MPIR shim functionality using Open MPI
+------------------------------------------------------
+
+Once installed, you may want to verify that the MPIR shim is
+functioning correctly.  Below is a shell script that builds and runs a
+trivial MPI application, and tests the MPIR shim with it.  Just like
+the prior shell script example, this script may need to be adapted for
+your environment.
+
+.. code-block:: bash
+
+    #!/usr/bin/env bash
+
+    set -euxo pipefail
+
+    # This script assumes the same directories and locations as the
+    # prior script.
+    BUILD_DIR=$HOME/git/mpir-to-pmix-guide
+
+    # Add the previously-installed MPIR shim into PATH and
+    # LD_LIBRARY_PATH (we assume that all relevant Open MPI / PMIx /
+    # etc. values already exist in PATH and LD_LIBRARY_PATH)
+    export PATH=$HOME/MPIR/bin:$PATH
+    export LD_LIBRARY_PATH=$HOME/MPIR/lib:$LD_LIBRARY_PATH
+
+    cat > testprog.c <<EOF
+    #include <mpi.h>
+    #include <unistd.h>
+    int main(int argc, char **argv) {
+        MPI_Init(&argc, &argv);
+        sleep(60);
+        MPI_Finalize();
+        return 0;
+    }
+    EOF
+    mpicc -o testprog testprog.c
+
+    # Test the shim in proxy mode.
+    # Manually verify that displayed process mapping is correct.
+    $BUILD_DIR/test/mpirshim_test mpirun -n 2 ./testprog
+    
+    # Launch mpirun for attach test and get its PID
+    mpirun -n 2 ./testprog &
+    PID=$!
+    # Test shim attach mode.
+    # Manually verify that displayed process mapping is correct.
+    $BUILD_DIR/test/mpirshim_test -c $PID
diff --git a/docs/app-debug/parallel-debug.rst b/docs/app-debug/parallel-debug.rst
new file mode 100644
index 00000000000..c4a48b165c6
--- /dev/null
+++ b/docs/app-debug/parallel-debug.rst
@@ -0,0 +1,41 @@
+Using Parallel Debuggers to Debug Open MPI Applications
+=======================================================
+
+Debugging with TotalView
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Debugging MPI applications has changed with different releases of
+TotalView and Open MPI; it is best to consult TotalView's
+documentation for how you should debug Open MPI applications with
+TotalView.
+
+Starting with Open MPI v5.0.0, debugging with Totalview requires the
+use of the MPIR shim module to attach to MPI processes.  See the
+:ref:`using MPIR-based tools with Open MPI section
+<using-mpir-based-tools-label>` for instructions on building and
+installing the MPIR shim module.
+
+Once TotalView is attached to Open MPI processes, it can also view
+Open MPI's message queues (for some transports), which can aid in
+debugging.
+
+.. note:: The integration of Open MPI message queue support is
+   problematic with 64-bit versions of TotalView prior to v8.3:
+
+   * The message queues view will be truncated.
+   * Both the communicators and requests lists will be incomplete.
+   * Both the communicators and requests lists may be filled with wrong
+     values (such as an ``MPI_Send`` to the destination
+     ``MPI_ANY_SOURCE``).
+
+   There are two workarounds:
+
+   * Use a 32-bit version of TotalView
+   * Upgrade to TotalView v8.3 or later
+
+Debugging with DDT
+~~~~~~~~~~~~~~~~~~
+
+Debugging MPI application with DDT has changed with different releases
+of DDT and Open MPI; it is best to consult DDT's documentation for how
+you should debug Open MPI applications with DDT.
diff --git a/docs/app-debug/serial-debug.rst b/docs/app-debug/serial-debug.rst
new file mode 100644
index 00000000000..d2228590f16
--- /dev/null
+++ b/docs/app-debug/serial-debug.rst
@@ -0,0 +1,175 @@
+Using Serial Debuggers to Debug Open MPI Applications
+=====================================================
+
+Since the GNU debugger (``gdb``) is fairly ubiquitiously
+available, it is common to use a serial debugger for debugging Open
+MPI applications.  Parallel debuggers are generally *better*, but
+``gdb`` is free, and therefore quite common.
+
+There are two common ways to use serial debuggers.
+
+Attach to Individual Running MPI processes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can use multiple invocations of a serial debugger to individually
+attach to every application process, or to a subset of application process. 
+
+If you are using ``gdb``, then you would log onto the nodes where you want
+to debug application processes and invoke gdb specifying the
+gdb ``--pid`` option and the PID for each application process you want to
+attach to.
+
+You can use a similar technique to attach to application processes in
+an application that was just invoked by :ref:`mpirun(1) <man1-mpirun>`.
+
+An inelegant-but-functional technique commonly used with this method
+is to insert the following code in your application where you want to
+suspend the application process until the debugger is attached, such as
+just after MPI is initialized.
+
+.. code-block:: c
+
+   {
+       volatile int i = 0;
+       char hostname[256];
+       gethostname(hostname, sizeof(hostname));
+       printf("PID %d on %s ready for attach\n", getpid(), hostname);
+       fflush(stdout);
+       while (0 == i)
+           sleep(5);
+   }
+
+This code will output a line to stdout outputting the name of the host
+where the process is running and the PID to attach to.  It will then
+spin on the ``sleep(3)`` function forever waiting for you to attach
+with a debugger.  Using ``sleep(3)`` as the inside of the loop means
+that the processor won't be pegged at 100% while waiting for you to
+attach.
+
+Once you attach with a debugger, go up the function stack until you
+are in this block of code (you'll likely attach during the
+``sleep(3)``) then set the variable ``i`` to a nonzero value.  With
+GDB, the syntax is:
+
+.. code-block:: sh
+
+   (gdb) set var i = 7
+
+Then set a breakpoint after your block of code and continue execution
+until the breakpoint is hit.  Now you have control of your live MPI
+application and use of the full functionality of the debugger.
+
+You  can even  add  conditionals to  only allow  this  "pause" in  the
+application for specific MPI  processes (e.g., ``MPI_COMM_WORLD`` rank
+0, or whatever process is misbehaving).
+
+Use ``mpirun`` to Launch Separate Instances of Serial Debuggers
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This technique launches a separate window for each MPI process in
+``MPI_COMM_WORLD``, each one running a serial debugger (such as
+``gdb``) that will launch and run your MPI application.  Having a
+separate window for each MPI process can be quite handy for low
+process-count MPI jobs, but requires a bit of setup and configuration
+that is outside of Open MPI to work properly.  A naive approach would
+be to assume that the following would immediately work:
+
+.. code-block:: sh
+
+    shell$ mpirun -n 4 xterm -e gdb my_mpi_application
+
+If running on a personal computer, this will probably work.  You can
+also use `tmpi <https://github.com/Azrael3000/tmpi>`_ to launch the
+debuggers in separate ``tmux`` panes instead of separate ``xterm``
+windows, which has the advantage of synchronizing keyboard input
+between all debugger instances.
+
+Unfortunately, the ``tmpi`` or ``xterm`` approaches likely *won't*
+work on an computing cluster. Several factors must be considered:
+
+#.  What launcher is Open MPI using?  In an ``ssh``-based environment,
+    Open MPI will use ``ssh`` by default.
+    But note that Open MPI closes the ``ssh``
+    sessions when the MPI job starts for scalability reasons.  This
+    means that the built-in SSH X forwarding tunnels will be shut down
+    before the ``xterms`` can be launched.  Although it is possible to
+    force Open MPI to keep its SSH connections active (to keep the X
+    tunneling available), we recommend using non-SSH-tunneled X
+    connections, if possible (see below).
+
+#. In non-``ssh`` environments (such as when using resource managers),
+   the environment of the process invoking ``mpirun`` may be copied to
+   all nodes.  In this case, the ``DISPLAY`` environment variable may
+   not be suitable.
+
+#. Some operating systems default to disabling the X11 server from
+   listening for remote/network traffic.  For example, see `this post
+   on the Open MPI user's mailing list
+   <https://www.open-mpi.org/community/lists/users/2008/02/4995.php>`_
+   describing how to enable network access to the X11 server on Fedora
+   Linux.
+
+#. There may be intermediate firewalls or other network blocks that
+   prevent X traffic from flowing between the hosts where the MPI
+   processes (and ``xterm``) are running and the host connected to
+   the output display.
+
+The easiest way to get remote X applications (such as ``xterm``) to
+display on your local screen is to forego the security of SSH-tunneled
+X forwarding.  In a closed environment such as an HPC cluster, this
+may be an acceptable practice (indeed, you may not even have the
+option of using SSH X forwarding if SSH logins to cluster nodes are
+disabled), but check with your security administrator to be sure.
+
+If using non-encrypted X11 forwarding is permissible, we recommend the
+following:
+
+#. For each non-local host where you will be running an MPI process,
+   add it to your X server's permission list with the ``xhost``
+   command.  For example:
+
+   .. code-block:: sh
+
+      shell$ cat my_hostfile
+      inky
+      blinky
+      stinky
+      clyde
+      shell$ for host in `cat my_hostfile` ; do xhost +host ; done
+
+#. Use the ``-x`` option to ``mpirun`` to export an appropriate
+   DISPLAY variable so that the launched X applications know where to
+   send their output.  An appropriate value is *usually* (but not
+   always) the hostname containing the display where you want the
+   output and the ``:0`` (or ``:0.0``) suffix.  For example:
+
+   .. code-block:: sh
+
+      shell$ hostname
+      arcade.example.come
+      shell$ mpirun -n 4 --hostfile my_hostfile \
+          -x DISPLAY=arcade.example.com:0 xterm -e gdb my_mpi_application
+
+   .. warning:: X traffic is fairly "heavy" |mdash| if you are
+                operating over a slow network connection, it may take
+                some time before the ``xterm`` windows appear on your
+                screen.
+
+#. If your ``xterm`` supports it, the ``-hold`` option may be useful.
+   ``-hold`` tells ``xterm`` to stay open even when the application
+   has completed.  This means that if something goes wrong (e.g.,
+   ``gdb`` fails to execute, or unexpectedly dies, or ...), the
+   ``xterm`` window will stay open, allowing you to see what happened,
+   instead of closing immediately and losing whatever error message
+   may have been output.
+
+#. When you have finished, you may wish to disable X11 network
+   permissions from the hosts that you were using.  Use ``xhost``
+   again to disable these permissions:
+
+   .. code-block:: sh
+
+      shell$ for host in `cat my_hostfile` ; do xhost -host ; done
+
+.. note:: ``mpirun`` will not complete until all the ``xterm``
+          instances are complete.
diff --git a/docs/app-debug/valgrind.rst b/docs/app-debug/valgrind.rst
new file mode 100644
index 00000000000..93b7d57ba21
--- /dev/null
+++ b/docs/app-debug/valgrind.rst
@@ -0,0 +1,24 @@
+Using Valgrind to Find Open MPI Application Errors
+==================================================
+
+The `Valgrind <https://valgrind.org/>`_ utilities may be useful in identifying application
+logic errors. However, there are many situations where Open MPI purposefully does
+not initialize and subsequently communicate memory, e.g., by calling ``writev(2)``.
+Furthermore, several cases are known where memory is not properly freed when
+MPI is finalized.
+
+This certainly does not help distinguishing real errors from false positives.
+Valgrind provides functionality to suppress errors and warnings from certain
+function contexts.
+
+In an attempt to ease debugging using Valgrind, Open MPI provides a
+Valgrind suppression file, that can be passed on the command
+line:
+
+.. code-block:: sh
+
+   shell$ mpirun -n 2 valgrind --suppressions=$PREFIX/share/openmpi/openmpi-valgrind.supp
+
+More information on suppression files and how to generate them can be
+found in `Valgrind's documentation
+<https://valgrind.org/docs/manual/manual-core.html#manual-core.suppress>`_.
diff --git a/docs/building-apps/abi-compatibility.rst b/docs/building-apps/abi-compatibility.rst
new file mode 100644
index 00000000000..ab9272241bd
--- /dev/null
+++ b/docs/building-apps/abi-compatibility.rst
@@ -0,0 +1,59 @@
+.. _label-binary-compatibility:
+
+ABI compatibility to previous versions of Open MPI
+==================================================
+
+The Open MPI |ompi_series| series maintains Application Binary Interface (ABI)
+compatibility for the C MPI bindings to the last major Open MPI release. Specifically, an
+application compiled with Open MPI v4.x can be executed with Open MPI
+|ompi_series| without having to recompile the application.
+
+.. important:: ABI is maintained for *most* of the Fortran MPI bindings, too |mdash| see below for additional information.
+
+There are however a few scenarios where an application compiled with
+Open MPI v4.x might not execute correctly with Open MPI 5.0.
+
+- Fortran compilers provide varying degrees of ABI guarantees between
+  their releases.  As such, Open MPI can only provide ABI guarantees
+  with MPI applications that use the Fortran MPI bindings when all
+  three entities |mdash| Open MPI v4.x, Open MPI |ompi_series|, and
+  the MPI application |mdash| were built with exactly the same Fortran
+  compiler and environment.
+
+  If these conditions are met, Open MPI's ABI guarantees between the
+  Open MPI v4.x and |ompi_series| series are in effect, with the
+  exception of the following cases:
+
+  * When using the Fortran ``mpi`` module bindings with GCC compiler
+    versions prior to v4.8, non-compliant Fortran interfaces for
+    ``MPI_Comm_spawn_multiple``, ``MPI_Testall``, ``MPI_Testsome``,
+    ``MPI_Waitall``, and ``MPI_Waitsome`` were removed starting with
+    Open MPI v5.0.0.
+
+  * When using the Fortran ``mpi`` module with modern Fortran
+    compilers (e.g., GNU Fortran >= v4.9), Open MPI v5.0.0 removed the
+    names from the MPI interfaces when there is only a single
+    subroutine in the interface, and that subroutine name exactly
+    matches the iterface name.  This change is likely to make Open MPI
+    |ompi_series|'s ``mpi`` module bindings *less* restrictive than
+    Open MPI v4.x, but it *may* also have ABI implications, depending
+    on your Fortran compiler.
+
+    `See this git commit message
+    <https://github.com/open-mpi/ompi/commit/f34782fe0c493963ec44bcf7dde9d94b88a7ea91>`_
+    for more details.
+
+    .. important:: This is likely a compiler-specific issue, and may
+                   need to be tested in your environment.
+
+  * When using the Fortran ``mpi_f08`` module bindings in an
+    environment where a Fortran ``INTEGER`` is 8 bytes but a C ``int``
+    is 4 bytes, the size of a ``Type(MPI_Status)`` was increased
+    starting with Open MPI v5.0.0.
+
+- Open MPI v5.0.0 removed support for the MPI C++ bindings. If an application
+  was using the deprecated and now removed C++ bindings, it will not
+  be able to compile or execute with Open MPI v5.0.0.  For details on deprecated and
+  removed functions see :ref:`Removed MPI constructs
+  <label-removed-mpi-constructs>` and :ref:`Deprecation warnings
+  <label-deprecated-functions>`
diff --git a/docs/building-apps/building-static-apps.rst b/docs/building-apps/building-static-apps.rst
new file mode 100644
index 00000000000..2a73a2c84e3
--- /dev/null
+++ b/docs/building-apps/building-static-apps.rst
@@ -0,0 +1,182 @@
+.. _label-building-fully-static-apps:
+
+Building fully-static MPI applications
+======================================
+
+Is fully-static really what you need?
+-------------------------------------
+
+No one ends up in this documentation section by accident.
+
+If you are reading this text, it is likely because you are looking to
+solve a problem, and fully-static MPI applications sound like the
+right answer.  There are two common problems that people think that
+fully-static MPI applications will solve:
+
+#. MPI applications fail to launch because dependent shared libraries
+   are not found at run-time.
+
+#. Filesystem performance when launching at scale is terrible.
+
+If either of these are your problems, the Open MPI community *strongly
+encourages you to use other mechanisms to fix the problem*:
+fully-static MPI applications are *possible*, but are sub-optimal for
+other reasons.
+
+The following sections discuss the above problems.
+
+Finding dependent shared libraries at run-time
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you are running in to problems finding shared libraries at run time
+|mdash| particularly on nodes that are remote from where you have invoked
+``mpirun`` |mdash| your best bet is to set ``LD_LIBRARY_PATH`` (or
+equivalent mechanism) properly on *all* nodes (including remote
+nodes).
+
+This is an involved topic, but even so, it is generally simpler to
+solve this problem than creating and maintaining static builds.  See
+:ref:`the section on Running MPI applications
+<label-running-mpi-applications>` for more details.
+
+Improving filesystem performance at scale
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Prior to v5.0.0, Open MPI compiled a large number of plugins as
+individual dynamic shared objects (DSOs) |mdash| i.e., individual
+files in the filesystem.  Many of these DSOs would be opened by each
+MPI process at run time.
+
+This could cause filesystem congestion, particularly when Open MPI is
+installed on a network filesystem and a large job is launched: many
+nodes will simultaneously communicate with the file server(s), and
+potentially need to transfer a large number of small(ish) files.
+
+Starting with v5.0.0, by default, Open MPI's plugins are no longer
+built as DSOs.  As such, Open MPI typically only opens a small number
+of shared libraries at launch time.  Even if Open MPI is installed on
+a network filesystem, these libraries are likely to be cached on nodes
+over time, and therefore generate a fairly small amount network
+filesystem traffic when MPI jobs are launched.
+
+In short: Open MPI |ompi_ver|'s impact on network filesystems is
+greatly diminished compared to prior versions.  Compiling fully-static
+applications to eliminate the open-every-DSO-file-at-launch-time
+behavior is no longer necessary.
+
+Other reasons fully-static applications are bad
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Here are a few other reasons that fully-static MPI applications are
+sub-optimal:
+
+#. When applications link all of their dependencies statically, the
+   operating system cannot share code between multiple copies of the
+   process.
+
+   For example, if you launch N copies of your fully-statically-linked
+   MPI application on a node, it will consume (N *
+   size_of_the_application) bytes of RAM.  Alternately, launching N
+   copies of a dynamically-linked MPI application |mdash| where each
+   of the copies have the same dependent libraries |mdash| will only
+   load each shared dependent library into RAM once.
+
+   In other words: *using dynamic linking saves memory.*
+
+#. *Fully*-static applications are not linked to the dynamic linking library,
+   e.g. ``libdl`` on Linux, which provides ``dlopen(3)``, ``dlsym(3)``, etc.
+   This will break Open MPI functionalities that depend on such interfaces.
+
+   .. warning:: Open MPI's memory management functionality, which provides
+                important performance optimizations on OS-bypass networks
+                such as InfiniBand, requires the ``dlsym(3)`` interface,
+                and therefore does not work with fully-static applications.
+
+Are you convinced yet?  *Please try to avoid building fully-static MPI
+applications if at all possible.*
+
+
+Building fully-static MPI applications
+--------------------------------------
+
+.. caution:: If, after reading all of the above, you are still of the
+             mind that you want to build fully-static MPI
+             applications, be aware that fully static linking is not
+             for the meek, and it is not recommended.  But it is
+             possible, with some caveats.
+
+#. You must have static libraries available for *everything* to which
+   your program links.  This includes Open MPI; you must have used the
+   ``--enable-static`` option to Open MPI's ``configure`` or otherwise
+   have available the static versions of the Open MPI libraries.
+
+   .. note:: Some Linux distributions may not have static versions of
+             popular Linux libraries by default (e.g., libnuma), or
+             require additional RPMs to be installed to get the
+             equivalent static libraries.
+
+#. Open MPI must have been built without a memory manager.  This means
+   that Open MPI must have been configured with the
+   ``--without-memory-manager`` flag.  This is irrelevant on some
+   platforms for which Open MPI does not have a memory manager, but on
+   some platforms it is necessary (Linux when using many OS-bypass
+   networks).  It is harmless to use this flag on platforms where Open
+   MPI does not have a memory manager.
+
+   .. important:: Not including memory manager support can lead to
+                  lower performance when Open MPI is used with
+                  OS-bypass networks.
+
+This is how to configure Open MPI to build fully-static libraries on
+Linux:
+
+.. code-block:: sh
+
+   shell$ ./configure --without-memory-manager --disable-dlopen \
+       --enable-static --disable-shared ...
+
+The ``--disable-shared`` flag is optional; it will prevent Open MPI
+from *also* building shared libraries.
+
+Alternatively, you could build Open MPI with as many static libraries
+as possible, but still preserve ``dlopen`` functionality by omitting
+the ``--disable-dlopen`` flag:
+
+.. code-block:: sh
+
+   shell$ ./configure --without-memory-manager \
+       --enable-static --disable-shared ...
+
+This gives you a *mostly* static build of Open MPI, but has the
+advantage of preserving at least some dynamic libraries.
+
+Including whole archives
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+Some systems may have additional constraints about their support
+libraries that require additional steps to produce working
+fully-static MPI applications.  For example, any library that has its
+own run-time plugin system (i.e., that opens dynamically shared
+objects ("DSOs") at run time) will have additional complications in
+producing fully-static builds.
+
+In such cases, you generally want to run ``mpicc ... --showme`` to see
+the compiler / linker commands that Open MPI's wrapper commands will
+use, and then augment those commands with linker arguments for the
+static versions of the DSO plugins that you will need at run time.
+
+For example, if you have ``libfoo.a`` that dynamically loads
+``plugin.so`` at run time, you'll need to have a ``plugin.a`` and
+|mdash| assuming the GNU linker |mdash| add arguments similar to the
+following:
+
+* ``-static``: Tell the linker to generate a static executable.
+* ``-Wl,--whole-archive -lfoo /path/to/plugin.a -Wl,--no-whole-archive``:
+  Tell the linker to include the entire ``foo`` library and the entire
+  ``plugin.a`` archive in the executable.
+
+You can either add these arguments on the command line manually, or
+you can :ref:`modify the default behavior of the wrapper compilers
+<label-customizing-wrapper-compiler>` to hide this complexity from end
+users (but be aware that if you modify the wrapper compilers' default
+behavior, *all* users will be creating static applications!).
diff --git a/docs/building-apps/customizing-wrappers.rst b/docs/building-apps/customizing-wrappers.rst
new file mode 100644
index 00000000000..65a750c064d
--- /dev/null
+++ b/docs/building-apps/customizing-wrappers.rst
@@ -0,0 +1,122 @@
+.. _label-customizing-wrapper-compiler:
+
+Customizing wrapper compiler behavior
+=====================================
+
+The Open MPI wrapper compilers are driven by text files that contain,
+among other things, the flags that are passed to the underlying
+compiler.  These text files are generated automatically for Open MPI
+and are customized for the compiler set that was selected when Open
+MPI was configured; it is *not* recommended that users edit these
+files.
+
+However, there are cases where it may be necessary or desirable to
+edit these files and add to or subtract from the flags that Open MPI
+selected.  These files are installed in ``$pkgdatadir``, which
+defaults to ``$prefix/share/openmpi/WRAPPER_NAME-wrapper-data.txt``.
+Several environment variables are also available for run-time
+replacement of the wrapper's default values (from the text files):
+
+.. note:: You may need to scroll right in the following table.
+
+.. list-table::
+   :header-rows: 1
+
+   * - Wrapper compiler
+     - Compiler
+     - Preprocessor flags
+     - Compiler flags
+     - Linker flags
+     - Linker library flags
+     - Data file
+
+   * - ``mpicc``
+     - ``OMPI_CC``
+     - ``OMPI_CPPFLAGS``
+     - ``OMPI_CFLAGS``
+     - ``OMPI_LDFLAGS``
+     - ``OMPI_LIBS``
+     - ``mpicc-wrapper-data.txt``
+
+   * - ``mpic++`` and ``mpiCC``
+     - ``OMPI_CXX``
+     - ``OMPI_CPPFLAGS``
+     - ``OMPI_CXXFLAGS``
+     - ``OMPI_LDFLAGS``
+     - ``OMPI_LIBS``
+     - ``mpic++-wrapper-data.txt`` and ``mpiCC-wrapper-data.txt``,
+       respectively
+
+   * - ``mpifort``
+     - ``OMPI_FC``
+     - ``OMPI_CPPFLAGS``
+     - ``OMPI_FCFLAGS``
+     - ``OMPI_LDFLAGS``
+     - ``OMPI_LIBS``
+     - ``mpifort-wrapper-data.txt``
+
+.. caution:: Note that changing the underlying compiler may not work
+   at all.
+
+   For example, C++ and Fortran compilers are notoriously binary
+   incompatible with each other (sometimes even within multiple
+   releases of the same compiler).  If you compile/install Open MPI
+   with C++ compiler vX.Y.Z and then use the ``OMPI_CXX`` environment
+   variable to change the ``mpicxx`` wrapper compiler to use the
+   vA.B.C C++ compiler, your application code may not compile and/or
+   link.  The traditional method of using multiple different compilers
+   with Open MPI is to install Open MPI multiple times; each
+   installation should be built/installed with a different compiler.
+   This is annoying, but it is beyond the scope of Open MPI to be able
+   to fix.
+
+Note that the values of these fields can be directly influenced by
+passing flags to Open MPI's ``configure`` script.  :ref:`See this
+section in the Installation guide <install-wrapper-flags-label>` for
+more details.
+
+The files cited in the above table use fairly simplistic "key=value"
+data formats.  The following are several fields that are likely to be
+of interest to end-users:
+
+* ``project_short``: Prefix for all environment variables.  See
+  below.
+
+* ``compiler_env``: Specifies the base name of the environment
+  variable that can be used to override the wrapper's underlying
+  compiler at run-time.  The full name of the environment variable is
+  of the form ``<project_short>_<compiler_env>``; see table above.
+
+* ``compiler_flags_env``: Specifies the base name of the environment
+  variable that can be used to override the wrapper's compiler flags
+  at run-time.  The full name of the environment variable is of the
+  form ``<project_short>_<compiler_flags_env>``; see table above.
+
+* ``compiler``: The executable name of the underlying compiler.
+
+* ``extra_includes``: Relative to ``$installdir``, a list of directories
+  to also list in the preprocessor flags to find header files.
+
+* ``preprocessor_flags``: A list of flags passed to the preprocessor.
+
+* ``compiler_flags``: A list of flags passed to the compiler.
+
+* ``linker_flags``: A list of flags passed to the linker.
+
+* ``libs``: A list of libraries passed to the linker.
+
+* ``required_file``: If non-empty, check for the presence of this file
+  before continuing.  If the file is not there, the wrapper will abort
+  saying that the language is not supported.
+
+* ``includedir``: Directory containing Open MPI's header files.  The
+  proper compiler "include" flag is prepended to this directory and
+  added into the preprocessor flags.
+
+* ``libdir``: Directory containing Open MPI's library files.  The
+  proper compiler "include" flag is prepended to this directory and
+  added into the linker flags.
+
+* ``module_option``: This field only appears in ``mpifort``.  It is
+  the flag that the Fortran compiler requires to declare where module
+  files are located.
diff --git a/docs/building-apps/deprecation-warnings.rst b/docs/building-apps/deprecation-warnings.rst
new file mode 100644
index 00000000000..dabf456978d
--- /dev/null
+++ b/docs/building-apps/deprecation-warnings.rst
@@ -0,0 +1,266 @@
+.. _label-deprecated-functions:
+
+Deprecation warnings while compiling MPI applications
+=====================================================
+
+If you see deprecation warnings when compiling MPI applications, it is
+because your application is symbols / functions that are deprecated in
+MPI. For example:
+
+.. code-block:: sh
+
+    shell$ mpicc deprecated-example.c -c
+    deprecated-example.c: In function 'foo':
+    deprecated-example.c:6:5: warning: 'MPI_Attr_delete' is deprecated: MPI_Attr_delete was deprecated in MPI-2.0; use MPI_Comm_delete_attr instead [-Wdeprecated-declarations]
+         MPI_Attr_delete(MPI_COMM_WORLD, 2);
+         ^~~~~~~~~~~~~~~
+    In file included from deprecated-example.c:2:
+    /usr/local/openmpi/include/mpi.h:2601:20: note: declared here
+     OMPI_DECLSPEC  int MPI_Attr_delete(MPI_Comm comm, int keyval)
+                        ^~~~~~~~~~~~~~~
+
+Note that the deprecation compiler warnings tells you how to upgrade
+your code to avoid the deprecation warnings.  In this example, it
+advises you to use ``MPI_Comm_delete_attr()`` instead of
+``MPI_Attr_delete()``.
+
+Also, note that even if Open MPI was configured with
+``--enable-mpi1-compatibility`` to re-enable removed MPI-1 symbols,
+you will still get compiler warnings when you use the removed symbols.
+
+The following is a list of functions that have been deprecated in MPI,
+and the function that is replacing them. Some functions have been
+deprecated and removed from the MPI specification, these functions are
+listed :ref:`here <label-removed-mpi-constructs>`.
+
+.. list-table::
+    :header-rows: 1
+
+    * - Deprecated symbol
+
+        (click for more details, below)
+      - Replaced with
+
+        (click to go to the corresponding man page)
+      - MPI version deprecating the symbol
+
+    * - :ref:`MPI_KEYVAL_CREATE <label-mpi-keyval-create>`
+      - :ref:`MPI_COMM_CREATE_KEYVAL <mpi_comm_create_keyval>`
+      - MPI-2.0 (1996)
+
+    * - :ref:`MPI_KEYVAL_FREE <label-mpi-keyval-free>`
+      - :ref:`MPI_COMM_FREE_KEYVAL <mpi_comm_free_keyval>`
+      - MPI-2.0 (1996)
+
+    * - :ref:`MPI_COPY_FUNCTION <label-mpi-copy-delete-function>`
+      - :ref:`MPI_COMM_COPY_ATTR_FUNCTION <mpi_comm_create_keyval>`
+      - MPI-2.0 (1996)
+
+    * - :ref:`MPI_DELETE_FUNCTION <label-mpi-copy-delete-function>`
+      - :ref:`MPI_COMM_DELETE_ATTR_FUNCTION <mpi_comm_create_keyval>`
+      - MPI-2.0 (1996)
+
+    * - :ref:`MPI_ATTR_PUT <label-mpi-attr-put>`
+      - :ref:`MPI_COMM_SET_ATTR <mpi_comm_set_attr>`
+      - MPI-2.0 (1996)
+
+    * - :ref:`MPI_ATTR_GET <label-mpi-attr-get>`
+      - :ref:`MPI_COMM_GET_ATTR <mpi_comm_get_attr>`
+      - MPI-2.0 (1996)
+
+    * - :ref:`MPI_ATTR_DELETE <label-mpi-attr-delete>`
+      - :ref:`MPI_COMM_DELETE_ATTR <mpi_comm_delete_attr>`
+      - MPI-2.0 (1996)
+
+    * - :ref:`MPI_Comm_errhandler_fn <label-errhandler-fn>`
+      - :ref:`MPI_Comm_errhandler_function <mpi_comm_create_errhandler>`
+      - MPI-2.2 (2009)
+
+    * - :ref:`MPI_File_errhandler_fn <label-errhandler-fn>`
+      - :ref:`MPI_File_errhandler_function <mpi_file_create_errhandler>`
+      - MPI-2.2 (2009)
+
+    * - :ref:`MPI_Win_errhandler_fn <label-errhandler-fn>`
+      - :ref:`MPI_Win_errhandler_function <mpi_win_create_errhandler>`
+      - MPI-2.2 (2009)
+
+    * - :ref:`MPI_INFO_GET <label-mpi-info-get>`
+      - :ref:`MPI_INFO_GET_STRING <mpi_info_get_string>`
+      - MPI-4.0 (2021)
+
+    * - :ref:`MPI_INFO_GET_VALUELEN <label-mpi-info-get-valuelen>`
+      - :ref:`MPI_INFO_GET_STRING <mpi_info_get_string>`
+      - MPI-4.0 (2021)
+
+    * - :ref:`MPI_SIZEOF <label-mpi-sizeof>`
+      - Fortran intrinsics``c_sizeof`` or ``storage_size``
+      - MPI-4.0 (2021)
+
+.. _label-mpi-keyval-create:
+
+MPI_Keyval_create
+-----------------
+
+``MPI_Keyval_create`` has been deprecated and replaced by
+``MPI_Comm_create_keyval``. The C binding of the new function is
+identical to the deprecated version. Hence, applications can simply
+replace the function that is being invoked.
+
+The Fortran binding differs in that the ``extra_state`` argument is an
+address-sized integer in the new interfaces (vs. a regular integer in
+the old interfaces). Also, the copy and delete callback functions have
+Fortran bindings that are consistent with address-sized attributes.
+
+.. code-block:: Fortran
+
+    USE mpi
+    EXTERNAL my_copy_attr_function
+    EXTERNAL my_copy_delete_function
+    INTEGER ierror
+    INTEGER comm_keyval
+    INTEGER old_extra_state
+    INTEGER(KIND=MPI_ADDRESS_KIND) new_extra_state
+
+    ! Old way
+    CALL MPI_KEYVAL_CREATE(my_copy_attr_function, my_copy_delete_function,
+                           comm_keyval, old_extra_state, ierror)
+
+    ! New way
+    CALL MPI_COMM_CREATE_KEYVAL(my_copy_attr_function, my_delete_attr_function,
+                                comm_keyval, new_extra_state, ierror)
+
+
+.. _label-mpi-keyval-free:
+
+MPI_Keyval_free
+----------------
+
+The binding of ``MPI_Keyval_free`` and ``MPI_Comm_free_keyval`` are identical
+for both C and Fortran. Users can directly replace the deprecated function with its
+new version.
+
+.. _label-mpi-copy-delete-function:
+
+MPI_Copy_function and MPI_Delete_function
+------------------------------------------
+
+The ``MPI_Copy_function`` and ``MPI_Delete_function`` are only used in the
+deprecated function ``MPI_Keyval_create()``, as described in the
+:ref:`MPI_COMM_CREATE_KEYVAL <label-mpi-keyval-create>`.
+
+For C codes, developers can simply use the new, exactly-equivalent
+type name (i.e., the return type, number, and type of parameters
+didn't change) ``MPI_Comm_copy_attr_function``, and
+``MPI_Comm_delete_attr_function`` respectively.
+
+For Fortran applications, the only difference lies in required integer type for the
+``extra_state`` argument, which now has to be an address-sized integer.
+
+.. _label-mpi-attr-put:
+
+MPI_Attr_put
+------------
+
+The C binding for the deprecated ``MPI_Attr_put`` is identical to its
+replacement, ``MPI_Comm_set_attr``.  The Fortran binding differ in the
+usage of an addressed size integer for the attribute value in the new
+``MPI_Comm_set_attr`` vs. a regular integer in ``MPI_Attr_put``.
+
+.. code-block:: Fortran
+
+    USE mpi
+    INTEGER ierror
+    INTEGER comm_keyval
+    INTEGER old_attr_val
+    INTEGER(KIND=MPI_ADDRESS_KIND) new_attr_val
+
+    ! Old way
+    CALL MPI_ATTR_PUT(MPI_COMM_WORLD, comm_keyval,
+                       old_attr_val, ierror)
+
+    ! New way
+    CALL MPI_COMM_SET_ATTR(MPI_COMM_WORLD, comm_keyval,
+	                   new_attr_val, ierror)
+
+.. _label-mpi-attr-get:
+
+MPI_Attr_get
+------------
+
+The C bindings of the old and the new interfaces are identical.
+Fortran binding differ in the usage of an addressed size integer for
+the attribute value in the new ``MPI_Comm_get_attr`` vs. a regular
+integer in ``MPI_Attr_get``.
+
+.. _label-mpi-attr-delete:
+
+MPI_Attr_delete
+---------------
+
+C and Fortran bindings are identical for ``MPI_Attr_delete`` and
+``MPI_Comm_delete_attr``, hence developers should be able to just
+directly substitute one function call by the other.
+
+
+.. _label-mpi-info-get:
+
+MPI_Info_get
+------------
+
+Applications should replace the use of ``MPI_Info_get`` with ``MPI_Info_get_string``,
+but the usage differs slightly. See the example below.
+
+.. code-block:: c++
+
+        MPI_Info info;
+
+	// Create an info object using MPI_Info_create()
+	...
+
+	// Retrieve the the value of a provided key later in the code
+	char key[] = "my_key";
+	char value[64];
+        int valuelen=64;
+	int flag;
+
+	// Old way
+        MPI_Info_get(info, key, valuelen, &value, &flag);
+
+        // New way
+	// Note that we pass the address of valuelen with
+	// the new interfaces, since the variable will
+	// contain the length of the value string after
+	// the function call.
+        MPI_Info_get_string(info, key, &valuelen, &value, &flag);
+    }
+
+.. _label-mpi-info-get-valuelen:
+
+MPI_Info_get_valuelen
+---------------------
+
+``MPI_Info_get_valuelen`` has been deprecated since the new function
+``MPI_Info_get_string`` also returns the length of the value string.
+Please refer to the example shown in :ref:`MPI_INFO_GET <label-mpi-info-get>`.
+
+.. _label-mpi-sizeof:
+
+MPI_Sizeof
+----------
+
+The ``MPI_SIZEOF`` construct in Fortran has been deprected since there
+are standard Fortran language constructs such as ``c_sizeof`` and
+``storage_size`` that can be used instead.
+
+.. _label-errhandler-fn:
+
+MPI_Comm_errhandler_fn, MPI_File_errhandler_fn, MPI_Win_errhandler_fn
+---------------------------------------------------------------------
+
+The following function typedefs have been deprecated and are superseded by new
+names. Other than the typedef names, the function signatures are exactly the same; the
+names were updated to match conventions of other function typedef names.
+
+* ``MPI_Comm_errhandler_fn`` |rarrow| ``MPI_Comm_errhandler_function``
+* ``MPI_File_errhandler_fn`` |rarrow| ``MPI_File_errhandler_function``
+* ``MPI_Win_errhandler_fn`` |rarrow| ``MPI_Win_errhandler_function``
diff --git a/docs/building-apps/extracting-wrapper-flags.rst b/docs/building-apps/extracting-wrapper-flags.rst
new file mode 100644
index 00000000000..538e2a8a7b3
--- /dev/null
+++ b/docs/building-apps/extracting-wrapper-flags.rst
@@ -0,0 +1,127 @@
+Extracting flags from the wrapper compilers
+===========================================
+
+If you cannot use the wrapper compilers for some reason, there are
+multiple supported ways to extract the compiler/linker flags that you
+will need.
+
+Using the ``--showme`` option
+-----------------------------
+
+The wrapper compilers all support a ``--showme`` command line option
+that will show what commands would have been invoked.
+
+.. code-block:: sh
+
+   # Show the flags necessary to compile MPI C applications
+   shell$ mpicc --showme:compile
+
+   # Show the flags necessary to link MPI C applications
+   shell$ mpicc --showme:link
+
+   # Show all the flags necessary to build MPI C applications
+   shell$ mpicc --showme
+
+.. note:: If you pass ``--showme`` *and additional command line
+   parameters* to the wrapper compiler, be sure to *also* pass in a
+   filename.  Otherwise, the ``--showme`` functionality will not
+   display output as expected.
+
+.. warning:: It is almost never a good idea to hard-code these results
+   in a ``Makefile`` (or other build system).  It is almost always
+   best to run (for example) ``mpicc --showme:compile`` in a dynamic
+   fashion to find out what you need.  For example, GNU Make allows
+   running commands and assigning their results to variables:
+
+   .. code-block:: make
+
+      MPI_COMPILE_FLAGS = $(shell mpicc --showme:compile)
+      MPI_LINK_FLAGS = $(shell mpicc --showme:link)
+
+      my_app: my_app.c
+              $(CC) $(MPI_COMPILE_FLAGS) my_app.c $(MPI_LINK_FLAGS) -o my_app
+
+Using ``pkg-config``
+--------------------
+
+Alternatively, Open MPI also installs ``pkg-config(1)`` configuration
+files under ``$libdir/pkgconfig``.  If ``pkg-config`` is configured to
+find these files (e.g., if you add ``$libdir/pkgconfig`` |mdash| which
+is usually ``$prefix/lib/pkgconfig`` |mdash| to the
+``PKG_CONFIG_PATH`` environment variable), then compiling / linking
+Open MPI programs can be performed like this:
+
+.. code-block:: sh
+
+   shell$ export PKG_CONFIG_PATH=/opt/openmpi/lib/pkgconfig
+   shell$ gcc hello_world_mpi.c -o hello_world_mpi -g \
+               `pkg-config ompi-c --cflags --libs`
+   shell$
+
+Open MPI supplies multiple ``pkg-config`` configuration files; one for
+each different wrapper compiler (language):
+
+* ``ompi``: Synonym for ``ompi-c``; Open MPI applications using the C
+  MPI bindings
+* ``ompi-c``: Open MPI applications using the C MPI bindings
+* ``ompi-cxx``: Open MPI applications using the C MPI bindings
+* ``ompi-fort``: Open MPI applications using the Fortran MPI bindings
+
+.. note:: Open MPI's ``pkg-config`` files *work properly*, but they
+          probably aren't *technically correct*.
+
+          Specifically: Open MPI will list all of its dependent
+          libraries that are necessary to link an MPI application,
+          even if a given dependency has a ``.pc`` file and should
+          therefore be listed as a ``Requires`` and/or
+          ``Requires.private`` in Open MPI's ``.pc`` files.
+
+          For example, Open MPI lists ``-lpmix`` in both ``Libs`` and
+          ``Libs.private``.  But since PMIx provides its own
+          ``pmix.pc`` file, it would be more correct for Open MPI to
+          *not* list ``-lpmix`` in ``Libs`` / ``Libs.private``, and
+          instead include:
+
+          .. code-block::
+
+             Requires: pmix
+             Requires.private: pmix
+
+          The end result is likely immaterial, but we document this
+          just in case it ever becomes an issue someday.
+
+
+Using ``ompi_info``
+-------------------
+
+This method is not directly suitable for getting all the compiler /
+linker flags needed to compile MPI applications because it does not
+include the relevant flags to find Open MPI's headers and libraries.
+But it does show a breakdown of all other flags.
+
+.. code-block::
+
+   shell$ ompi_info --all | grep -i wrapper
+      Wrapper extra CFLAGS:
+    Wrapper extra CXXFLAGS:
+      Wrapper extra FFLAGS:
+     Wrapper extra FCFLAGS:
+     Wrapper extra LDFLAGS:
+        Wrapper extra LIBS: -lutil -lnsl -ldl -Wl,--export-dynamic -lm
+
+This installation is *only* adding options in the ``xLIBS`` areas of the
+wrapper compilers; all other values are blank (remember: the ``-I``'s
+and ``-L``'s are implicit).
+
+Note that the ``--parsable`` option can be used to obtain
+machine-parsable versions of this output.  For example:
+
+.. code-block::
+
+   shell$ ompi_info --all --parsable | grep wrapper:extra
+   option:wrapper:extra_cflags:
+   option:wrapper:extra_cxxflags:
+   option:wrapper:extra_fflags:
+   option:wrapper:extra_fcflags:
+   option:wrapper:extra_ldflags:
+   option:wrapper:extra_libs:-lutil -lnsl  -ldl  -Wl,--export-dynamic -lm
diff --git a/docs/building-apps/index.rst b/docs/building-apps/index.rst
new file mode 100644
index 00000000000..5c140be17e0
--- /dev/null
+++ b/docs/building-apps/index.rst
@@ -0,0 +1,26 @@
+Building MPI applications
+=========================
+
+The simplest way to compile and link MPI applications is to use the
+Open MPI "wrapper" compilers.
+
+.. toctree::
+   :maxdepth: 1
+
+   quickstart
+   customizing-wrappers
+   extracting-wrapper-flags
+   abi-compatibility
+   removed-mpi-constructs
+   deprecation-warnings
+   building-static-apps
+
+.. warning:: Open MPI does not attempt to interoperate with other MPI
+             implementations, nor executables that were compiled for
+             them.  MPI applications need to be compiled and linked
+             with Open MPI in order to run under Open MPI.
+
+             See the :doc:`section on Open MPI's version numbering
+             scheme </version-numbering>` for information about binary
+             compatibility between different versions of Open MPI for
+             applications compiled with another version.
diff --git a/docs/building-apps/quickstart.rst b/docs/building-apps/quickstart.rst
new file mode 100644
index 00000000000..9d1637e2499
--- /dev/null
+++ b/docs/building-apps/quickstart.rst
@@ -0,0 +1,67 @@
+.. _label-quickstart-building-apps:
+
+Quick start: Building MPI applications
+======================================
+
+Although this section skips many details, it offers examples that will
+probably work in many environments.
+
+.. caution:: Note that this section is a "Quick start" |mdash| it does
+   not attempt to be comprehensive or describe how to build Open MPI
+   in all supported environments.  The examples below may therefore
+   not work exactly as shown in your environment.
+
+   Please consult the other sections in this chapter for more details,
+   if necessary.
+
+Open MPI provides "wrapper" compilers that should be used for
+compiling MPI and OpenSHMEM applications:
+
++---------+-----------------------------------+
+| C       | :ref:`mpicc(1) <man1-mpicc>`,     |
+|         | :ref:`oshcc(1) <man1-oshcc>`      |
++---------+-----------------------------------+
+| C++     | :ref:`mpic++(1) <man1-mpic++>`,   |
+|         | :ref:`oshc++(1) <man1-oshc++>`    |
+|         | (and :ref:`mpiCC(1) <man1-mpiCC>` |
+|         | and :ref:`oshCC(1) <man1-oshCC>`  |
+|         | if you filesystem is              |
+|         | case-sensitive)                   |
++---------+-----------------------------------+
+| Fortran | :ref:`mpifort(1) <man1-mpifort>`, |
+|         | :ref:`oshfort(1) <man1-oshfort>`  |
++---------+-----------------------------------+
+
+.. caution:: The legacy names ``mpif77`` and ``mpif90`` still exist,
+             and are simply symbolic links to the ``mpifort`` wrapper
+             compiler.  Users are strongly encouraged to update all
+             build scripts to use ``mpifort`` instead of ``mpif77``
+             and ``mpif90``.
+
+The intent is that users can simply invoke the Open MPI wrapper
+compiler instead of their usual language compiler.  For example,
+instead of invoking your usual C compiler to build your MPI C
+appliance, use ``mpicc``:
+
+.. code-block:: sh
+
+   shell$ mpicc hello_world_mpi.c -o hello_world_mpi -g
+   shell$
+
+For OpenSHMEM applications:
+
+.. code-block:: sh
+
+   shell$ oshcc hello_shmem.c -o hello_shmem -g
+   shell$
+
+All the wrapper compilers do is add a variety of compiler and linker
+flags to the command line and then invoke a back-end compiler.  To be
+specific: the wrapper compilers do not parse source code at all; they
+are solely command-line manipulators, and have nothing to do with the
+actual compilation or linking of programs.  The end result is an MPI
+executable that is properly linked to all the relevant libraries.
+
+.. caution:: It is *absolutely not sufficient* to simply add ``-lmpi``
+             to your link line and assume that you will obtain a valid
+             Open MPI executable.
diff --git a/docs/building-apps/removed-mpi-constructs.rst b/docs/building-apps/removed-mpi-constructs.rst
new file mode 100644
index 00000000000..4dfa5b77293
--- /dev/null
+++ b/docs/building-apps/removed-mpi-constructs.rst
@@ -0,0 +1,531 @@
+.. _label-removed-mpi-constructs:
+
+Removed MPI constructs
+======================
+
+Starting with v4.0.0, Open MPI |mdash| by default |mdash| removes the
+prototypes from ``mpi.h`` for MPI symbols that were deprecated in 1996
+in the MPI-2.0 standard, and finally removed from the MPI-3.0 standard
+(2012).
+
+Specifically, the following symbols (specified in the MPI
+language-neutral names) are no longer prototyped in ``mpi.h`` by
+default:
+
+.. note:: You may need to scroll right in the following table.
+
+.. list-table::
+    :header-rows: 1
+
+    * - Removed symbol
+
+        (click for more details, below)
+      - Replaced with
+
+        (click to go to the corresponding man page)
+      - Deprecated
+      - Removed
+
+    * - :ref:`MPI_ADDRESS <label-mpi-address>`
+      - :ref:`MPI_GET_ADDRESS <mpi_get_address>`
+      - MPI-2.0 (1996)
+      - MPI-3.0 (2012)
+
+    * - :ref:`MPI_ERRHANDLER_CREATE <label-mpi-errhandler-create>`
+      - :ref:`MPI_COMM_CREATE_ERRHANDLER <mpi_comm_create_errhandler>`
+      - MPI-2.0 (1996)
+      - MPI-3.0 (2012)
+
+    * - :ref:`MPI_ERRHANDLER_GET <label-mpi-errhandler-get>`
+      - :ref:`MPI_COMM_GET_ERRHANDLER <mpi_comm_get_errhandler>`
+      - MPI-2.0 (1996)
+      - MPI-3.0 (2012)
+
+    * - :ref:`MPI_ERRHANDLER_SET <label-mpi-errhandler-set>`
+      - :ref:`MPI_COMM_SET_ERRHANDLER <mpi_comm_set_errhandler>`
+      - MPI-2.0 (1996)
+      - MPI-3.0 (2012)
+
+    * - :ref:`MPI_TYPE_EXTENT <label-mpi-type-extent>`
+      - :ref:`MPI_TYPE_GET_EXTENT <mpi_type_get_extent>`
+      - MPI-2.0 (1996)
+      - MPI-3.0 (2012)
+
+    * - :ref:`MPI_TYPE_HINDEXED <label-mpi-type-hindexed>`
+      - :ref:`MPI_TYPE_CREATE_HINDEXED <mpi_type_create_hindexed>`
+      - MPI-2.0 (1996)
+      - MPI-3.0 (2012)
+
+    * - :ref:`MPI_TYPE_HVECTOR <label-mpi-type-hvector>`
+      - :ref:`MPI_TYPE_CREATE_HVECTOR <mpi_type_create_hvector>`
+      - MPI-2.0 (1996)
+      - MPI-3.0 (2012)
+
+    * - :ref:`MPI_TYPE_LB <label-mpi-type-lb>`
+      - :ref:`MPI_TYPE_GET_EXTENT <mpi_type_get_extent>`
+      - MPI-2.0 (1996)
+      - MPI-3.0 (2012)
+
+    * - :ref:`MPI_TYPE_STRUCT <label-mpi-type-struct>`
+      - :ref:`MPI_TYPE_CREATE_STRUCT <mpi_type_create_struct>`
+      - MPI-2.0 (1996)
+      - MPI-3.0 (2012)
+
+    * - :ref:`MPI_TYPE_UB <label-mpi-type-ub>`
+      - :ref:`MPI_TYPE_GET_EXTENT <mpi_type_get_extent>`
+      - MPI-2.0 (1996)
+      - MPI-3.0 (2012)
+
+    * - :ref:`MPI_LB <label-mpi-lb-ub>`
+      - :ref:`MPI_TYPE_CREATE_RESIZED <mpi_type_create_resized>`
+      - MPI-2.0 (1996)
+      - MPI-3.0 (2012)
+
+    * - :ref:`MPI_UB <label-mpi-lb-ub>`
+      - :ref:`MPI_TYPE_CREATE_RESIZED <mpi_type_create_resized>`
+      - MPI-2.0 (1996)
+      - MPI-3.0 (2012)
+
+    * - :ref:`MPI_COMBINED_HINDEXED_INTEGER <label-mpi-combiner-fortran-integers>`
+      - :ref:`MPI_COMBINER_HINDEXED <mpi_type_get_envelope>`
+      - MPI-2.0 (1996)
+      - MPI-3.0 (2012)
+
+    * - :ref:`MPI_COMBINED_HVECTOR_INTEGER <label-mpi-combiner-fortran-integers>`
+      - :ref:`MPI_COMBINER_HVECTOR <mpi_type_get_envelope>`
+      - MPI-2.0 (1996)
+      - MPI-3.0 (2012)
+
+    * - :ref:`MPI_COMBINED_STRUCT_INTEGER <label-mpi-combiner-fortran-integers>`
+      - :ref:`MPI_COMBINER_STRUCT <mpi_type_get_envelope>`
+      - MPI-2.0 (1996)
+      - MPI-3.0 (2012)
+
+    * - :ref:`MPI_HANDLER_FUNCTION <label-mpi-handler-function>`
+      - :ref:`MPI_COMM_ERRHANDLER_FUNCTION <mpi_comm_create_errhandler>`
+      - MPI-2.0 (1996)
+      - MPI-3.0 (2012)
+
+Although these symbols are no longer prototyped in ``mpi.h``, *they are
+still present in the MPI library in Open MPI* |ompi_series|. This enables
+legacy MPI applications to *link and run* successfully with Open MPI
+|ompi_series|, even though they will fail to *compile*.
+
+Furthermore, the MPI C++ interfaces were deprecated in version
+2.2, and then were removed in MPI-3.0.  Starting
+from v5.0.0, Open MPI does not support the C++ interfaces
+any more. Users who would like to continue using the C++ interfaces of
+MPI will need to use an older release of Open MPI.
+
+.. warning:: The Open MPI team **strongly** encourages all
+   MPI application developers to stop using these constructs that were
+   first deprecated over 20 years ago, and finally removed from the MPI
+   specification in MPI-3.0 (in 2012).
+
+The FAQ items in this category show how to update your application to
+stop using these removed symbols.
+
+All that being said, if you are unable to immediately update your
+application to stop using these removed MPI-1 symbols, you can
+re-enable them in ``mpi.h`` by configuring Open MPI with the
+``--enable-mpi1-compatibility`` flag.
+
+.. note:: Future releases of Open MPI may remove these symbols
+   altogether.
+
+Why is Open MPI breaking the compilation of MPI applications?
+------------------------------------------------------------------
+
+The Open MPI developer community decided to take a first step of
+removing prototypes of deprecated functions from ``mpi.h`` starting
+with the Open MPI v4.0.x series for the following reasons:
+
+#. The first set of symbols have been deprecated since *1996.* It's
+   time to start raising awareness for developers who are
+   inadvertently still using these removed symbols.
+#. The MPI Forum removed a substantial set of symbols from the MPI-3.0
+   specification in 2012. This is a sign that the Forum itself
+   recognizes that these removed symbols are no longer needed.
+#. More functions have been deprecated in MPI 2.2 and MPI 4.0, and
+   additional functions are expected to be deprecated and removed in
+   future MPI versions. It is in the interest of both, developers and
+   end-users, to minimize confusion as much as possible, and stick
+   closely to the MPI specification.
+#. Note that Open MPI *did not fully remove* these removed symbols: we
+   just made it slightly more painful to get to them.  This is an
+   attempt to raise awareness so that MPI application developers can
+   update their applications (it's easy!).
+
+In short: the only way to finally be able to remove these removed
+symbols from Open MPI someday is to have a "grace period" where the
+MPI application developers are a) made aware that they are using
+removed symbols, and b) educated how to update their applications.
+
+We, the Open MPI developers, recognize that your MPI application
+failing to compile with Open MPI may be a nasty surprise.  We
+apologize for that.
+
+Our intent is simply to use this minor shock to raise awareness and
+use it as an educational opportunity to show you how to update your
+application (or direct your friendly neighborhood MPI application
+developer to this FAQ) to stop using these removed MPI symbols.
+
+Thank you!
+
+.. _label-mpi-address:
+
+Stop using MPI_ADDRESS
+----------------------
+
+In C, the only thing that changed was the function name:
+``MPI_Address()`` |rarrow| ``MPI_Get_address()``.  Nothing else needs
+to change:
+
+.. code-block:: c++
+
+    char buffer[30];
+    MPI_Aint address;
+
+    // Old way
+    MPI_Address(buffer, &address);
+
+    // New way
+    MPI_Get_address(buffer, &address);
+
+In Fortran, the type of the parameter changed from ``INTEGER``
+|rarrow| ``INTEGER(KIND=MPI_ADDRESS_KIND)`` so that it can hold
+larger values (e.g., 64 bit pointers):
+
+.. code-block:: Fortran
+
+    USE mpi
+    REAL buffer
+    INTEGER ierror
+    INTEGER old_address
+    INTEGER(KIND = MPI_ADDRESS_KIND) new_address
+
+    ! Old way
+    CALL MPI_ADDRESS(buffer, old_address, ierror)
+
+    ! New way
+    CALL MPI_GET_ADDRESS(buffer, new_address, ierror)
+
+.. _label-mpi-errhandler-create:
+
+Stop using MPI_ERRHANDLER_CREATE
+--------------------------------
+
+In C, *effectively* the only thing that changed was the name of the
+function: ``MPI_Errhandler_create()`` |rarrow|
+``MPI_Comm_create_errhandler()``.
+
+*Technically*, the type of the first parameter also changed
+(``MPI_Handler_function`` |rarrow| ``MPI_Comm_errhandler_function``),
+but most applications do not use this type directly and may not even
+notice the change.
+
+.. code-block:: c++
+
+    void my_errhandler_function(MPI_Comm *comm, int *code, ...)
+    {
+        // Do something useful to handle the error
+    }
+
+    void some_function(void)
+    {
+        MPI_Errhandler my_handler;
+
+        // Old way
+        MPI_Errhandler_create(my_errhandler_function, &my_handler);
+
+        // New way
+        MPI_Comm_create_errhandler(my_errhandler_function, &my_handler);
+    }
+
+In Fortran, only the subroutine name changed:
+``MPI_ERRHANDLER_CREATE`` |rarrow| ``MPI_COMM_CREATE_ERRHANDLER``.
+
+.. code-block:: Fortran
+
+    USE mpi
+    EXTERNAL my_errhandler_function
+    INTEGER ierror
+    INTEGER my_handler
+
+    ! Old way
+    CALL MPI_ERRHANDLER_CREATE(my_errhandler_function, my_handler, ierror)
+
+    ! New way
+    CALL MPI_COMM_CREATE_ERRHANDLER(my_errhandler_function, my_handler, ierror)
+
+.. _label-mpi-errhandler-get:
+
+Stop using MPI_ERRHANDLER_GET
+-----------------------------
+
+In both C and Fortran, the only thing that changed with regards to
+``MPI_ERRHANDLER_GET`` is the name: ``MPI_ERRHANDLER_GET`` |rarrow|
+``MPI_COMM_GET_ERRHANDLER``.
+
+All parameter types stayed the same.
+
+.. _label-mpi-errhandler-set:
+
+Stop using MPI_ERRHANDLER_SET
+-----------------------------
+
+In both C and Fortran, the only thing that changed with regards to
+``MPI_ERRHANDLER_SET`` is the name: ``MPI_ERRHANDLER_SET`` |rarrow|
+``MPI_COMM_SET_ERRHANDLER``.
+
+All parameter types stayed the same.
+
+.. _label-mpi-type-hindexed:
+
+Stop using MPI_TYPE_HINDEXED
+----------------------------
+
+In both C and Fortran, *effectively* the only change is the name of
+the function: ``MPI_TYPE_HINDEXED`` |rarrow|
+``MPI_TYPE_CREATE_HINDEXED``.
+
+In C, the new function also has a ``const`` attribute on the two array
+parameters, but most applications won't notice the difference.
+
+All other parameter types stayed the same.
+
+.. code-block:: c++
+
+    int count = 2;
+    int block_lengths[] = { 1, 2 };
+    MPI_Aint displacements[] = { 0, sizeof(int) };
+    MPI_Datatype newtype;
+
+    // Old way
+    MPI_Type_hindexed(count, block_lengths, displacements, MPI_INT, &newtype);
+
+    // New way
+    MPI_Type_create_hindexed(count, block_lengths, displacements, MPI_INT, &newtype);
+
+.. _label-mpi-type-hvector:
+
+Stop using MPI_TYPE_HVECTOR
+---------------------------
+
+In both C and Fortran, the only change is the name of the function:
+``MPI_TYPE_HVECTOR`` |rarrow| ``MPI_TYPE_CREATE_HVECTOR``.
+
+All parameter types stayed the same.
+
+.. _label-mpi-type-struct:
+
+Stop using MPI_TYPE_STRUCT
+--------------------------
+
+In both C and Fortran, *effectively* the only change is the name of
+the function: ``MPI_TYPE_STRUCT`` |rarrow| ``MPI_TYPE_CREATE_STRUCT``.
+
+In C, the new function also has a ``const`` attribute on the three
+array parameters, but most applications won't notice the difference.
+
+All other parameter types stayed the same.
+
+.. code-block:: c++
+
+    int count = 2;
+    int block_lengths[] = { 1, 2 };
+    MPI_Aint displacements[] = { 0, sizeof(int) };
+    MPI_Datatype datatypes[] = { MPI_INT, MPI_DOUBLE };
+    MPI_Datatype newtype;
+
+    // Old way
+    MPI_Type_struct(count, block_lengths, displacements, datatypes, &newtype);
+
+    // New way
+    MPI_Type_create_struct(count, block_lengths, displacements, datatypes, &newtype);
+
+.. _label-mpi-type-extent:
+
+Stop using MPI_TYPE_EXTENT
+--------------------------
+
+In both C and Fortran, the ``MPI_TYPE_EXTENT`` function is superseded
+by the slightly-different ``MPI_TYPE_GET_EXTENT`` function: the new
+function also returns the lower bound.
+
+.. code-block:: c++
+
+    MPI_Aint lb;
+    MPI_Aint extent;
+
+    // Old way
+    MPI_Type_extent(MPI_INT, &extent);
+
+    // New way
+    MPI_Type_get_extent(MPI_INT, &lb, &extent);
+
+.. _label-mpi-type-lb:
+
+Stop using MPI_TYPE_LB
+----------------------
+
+In both C and Fortran, the ``MPI_TYPE_LB`` function is superseded by
+the slightly-different ``MPI_TYPE_GET_EXTENT`` function: the new
+function also returns the extent.
+
+.. code-block:: c++
+
+    MPI_Aint lb;
+    MPI_Aint extent;
+
+    // Old way
+    MPI_Type_lb(MPI_INT, &lb);
+
+    // New way
+    MPI_Type_get_extent(MPI_INT, &lb, &extent);
+
+.. _label-mpi-type-ub:
+
+Stop using MPI_TYPE_UB
+----------------------
+
+In both C and Fortran, the ``MPI_TYPE_UB`` function is superseded by
+the slightly-different ``MPI_TYPE_GET_EXTENT`` function: the new
+function returns the lower bound and the extent, which can be used to
+compute the upper bound.
+
+.. code-block:: c++
+
+    MPI_Aint lb, ub;
+    MPI_Aint extent;
+
+    // Old way
+    MPI_Type_ub(MPI_INT, &ub);
+
+    // New way
+    MPI_Type_get_extent(MPI_INT, &lb, &extent);
+    ub = lb + extent
+
+Note the ``ub`` calculation after calling ``MPI_Type_get_extent()``.
+
+.. _label-mpi-lb-ub:
+
+Stop using MPI_LB / MPI_UB
+--------------------------
+
+The ``MPI_LB`` and ``MPI_UB`` positional markers were fully replaced
+with ``MPI_TYPE_CREATE_RESIZED`` in MPI-2.0.
+
+Prior to MPI-2.0, ``MPI_UB`` and ``MPI_LB`` were intended to be used
+as input to ``MPI_TYPE_STRUCT`` (which, itself, has been deprecated
+and renamed to ``MPI_TYPE_CREATE_STRUCT``).  The same end effect can
+now be achieved with ``MPI_TYPE_CREATE_RESIZED``.  For example, using
+the old method:
+
+.. code-block:: c++
+
+    int count = 3;
+    int block_lengths[] = { 1, 1, 1 };
+    MPI_Aint displacements[] = { -2, 0, 10 };
+    MPI_Datatype datatypes[] = { MPI_LB, MPI_INT, MPI_UB };
+    MPI_Datatype newtype;
+
+    MPI_Type_struct(count, block_lengths, displacements, datatypes, &newtype);
+    MPI_Type_commit(&newtype);
+
+    MPI_Aint ub, lb, extent;
+    MPI_Type_lb(newtype, &lb);
+    MPI_Type_ub(newtype, &ub);
+    MPI_Type_extent(newtype, &extent);
+    printf("OLD: LB=%d, UB=%d, extent=%d\n",
+           lb, ub, extent);
+
+If we run the above, we get an output of:
+
+.. code-block::
+
+    OLD: LB=-2, UB=10, extent=12
+
+The ``MPI_TYPE_RESIZED`` function allows us to take any arbitrary
+datatype and set the lower bound and extent directly (which indirectly
+sets the upper bound), without needing to setup the arrays and
+computing the displacements necessary to invoke
+``MPI_TYPE_CREATE_STRUCT``.
+
+Aside from the ``printf`` statement, the following example is exactly
+equivalent to the prior example (:ref:`see the MPI_TYPE_UB section
+<label-mpi-type-ub>` for a mapping of ``MPI_TYPE_UB`` to
+``MPI_TYPE_GET_EXTENT``):
+
+.. code-block:: c++
+
+    MPI_Datatype newtype;
+
+    MPI_Type_create_resized(MPI_INT, -2, 12, &newtype);
+    MPI_Type_commit(&newtype);
+
+    MPI_Aint ub, lb, extent;
+    MPI_Type_get_extent(newtype, &lb, &extent);
+    ub = lb + extent;
+    printf("NEW: LB=%d, UB=%d, extent=%d\n",
+           lb, ub, extent);
+
+If we run the above, we get an output of:
+
+.. code-block::
+
+    NEW: LB=-2, UB=10, extent=12
+
+.. _label-mpi-combiner-fortran-integers:
+
+Stop using MPI_COMBINER_HINDEXED_INTEGER, MPI_COMBINER_HVECTOR_INTEGER, and MPI_COMBINER_STRUCT_INTEGER
+-------------------------------------------------------------------------------------------------------
+
+The ``MPI_COMBINER_HINDEXED_INTEGER``,
+``MPI_COMBINER_HVECTOR_INTEGER``, and ``MPI_COMBINER_STRUCT_INTEGER``
+constants could previously be returned from ``MPI_TYPE_GET_ENVELOPE``.
+
+Starting with MPI-3.0, these values will never be returned.  Instead,
+they will just return the same names, but without the ``_INTEGER``
+suffix.  Specifically:
+
+* ``MPI_COMBINER_HINDEXED_INTEGER`` |rarrow| ``MPI_COMBINER_HINDEXED``
+* ``MPI_COMBINER_HVECTOR_INTEGER`` |rarrow| ``MPI_COMBINER_HVECTOR``
+* ``MPI_COMBINER_STRUCT_INTEGER`` |rarrow| ``MPI_COMBINER_STRUCT``
+
+If your Fortran code is using any of the ``_INTEGER``-suffixed names,
+you can just delete the ``_INTEGER`` suffix.
+
+.. _label-mpi-handler-function:
+
+Stop using MPI_Handler_function
+-------------------------------
+
+The ``MPI_Handler_function`` C type is only used in the
+deprecated/removed function ``MPI_Errhandler_create()``, as described
+:ref:`in the MPI_ERRHANDLER_CREATE section
+<label-mpi-errhandler-create>`.
+
+Most MPI applications likely won't use this type at all.  But if they
+do, they can simply use the new, exactly-equivalent type name (i.e.,
+the return type, number, and type of parameters didn't change):
+``MPI_Comm_errhandler_function``.
+
+.. code-block:: c++
+
+    void my_errhandler_function(MPI_Comm *comm, int *code, ...)
+    {
+        // Do something useful to handle the error
+    }
+
+    void some_function(void)
+    {
+        // Old way
+        MPI_Handler_function *old_ptr = my_errhandler_function;
+
+        // New way
+        MPI_Comm_errhandler_function *new_ptr = my_errhandler_function;
+    }
+
+The ``MPI_Handler_function`` type isn't used at all in the Fortran
+bindings.
diff --git a/docs/conf.py b/docs/conf.py
new file mode 100644
index 00000000000..87c2d696ada
--- /dev/null
+++ b/docs/conf.py
@@ -0,0 +1,327 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+
+# -- Project information -----------------------------------------------------
+
+import os
+import re
+import datetime
+
+year = datetime.datetime.now().year
+
+project = 'Open MPI'
+copyright = f'2003-{year}, The Open MPI Community'
+author = 'The Open MPI Community'
+
+# ---------------------------
+
+# The docs/Makefile.am will set the env var OMPI_TOP_SRCDIR, because
+# we might be doing a VPATH build.
+ompi_top_srcdir = '..'
+if 'OMPI_TOP_SRCDIR' in os.environ:
+    ompi_top_srcdir = os.environ['OMPI_TOP_SRCDIR']
+
+# Read an Open MPI-style VERSION file
+def read_version_file(path):
+    if not os.path.exists(path):
+        print(f"ERROR: Unable to find file {path}")
+        exit(1)
+
+    with open(path) as fp:
+        version_lines = fp.readlines()
+
+    data = dict()
+    for line in version_lines:
+        if '#' in line:
+            parts = line.split("#")
+            line = parts[0]
+        line = line.strip()
+
+        if '=' not in line:
+            continue
+
+        key, val = line.split("=")
+        data[key.strip()] = val.strip()
+
+    return data
+
+# Look for a version string via a regular expresion of a filename in a
+# given directory
+def get_tarball_version(path, expr):
+    if not os.path.exists(path):
+        print(f"ERROR: Unable to find path {path}")
+        exit(1)
+
+    for file in os.listdir(path):
+        m = re.match(expr, file)
+        if not m:
+            continue
+        return m.group(1)
+
+    return ""
+
+# Read all the various versions from the source tree
+
+ompi_data = read_version_file(f"{ompi_top_srcdir}/VERSION")
+pmix_data = read_version_file(f"{ompi_top_srcdir}/3rd-party/openpmix/VERSION")
+prte_data = read_version_file(f"{ompi_top_srcdir}/3rd-party/prrte/VERSION")
+
+hwloc_embedded_version = get_tarball_version(f"{ompi_top_srcdir}/3rd-party/",
+                                             r"hwloc-([^-]+).*\.tar")
+event_embedded_version = get_tarball_version(f"{ompi_top_srcdir}/3rd-party/",
+                                             r"libevent-([^-]+).*\.tar")
+
+# ---------------------------
+
+# Assemble several different combinations of version strings
+
+ompi_series = f"v{ompi_data['major']}.{ompi_data['minor']}.x"
+ompi_ver = f"v{ompi_data['major']}.{ompi_data['minor']}.{ompi_data['release']}{ompi_data['greek']}"
+
+pmix_embedded_version = f"v{pmix_data['major']}.{pmix_data['minor']}.{pmix_data['release']}{pmix_data['greek']}"
+prte_embedded_version = f"v{prte_data['major']}.{prte_data['minor']}.{prte_data['release']}{prte_data['greek']}"
+prte_embedded_series = f"v{prte_data['major']}.{prte_data['minor']}"
+
+pmix_min_version = f"{ompi_data['pmix_min_version']}"
+prte_min_version = f"{ompi_data['prte_min_version']}"
+hwloc_min_version = f"{ompi_data['hwloc_min_version']}"
+event_min_version = f"{ompi_data['event_min_version']}"
+automake_min_version = f"{ompi_data['automake_min_version']}"
+autoconf_min_version = f"{ompi_data['autoconf_min_version']}"
+libtool_min_version = f"{ompi_data['libtool_min_version']}"
+flex_min_version = f"{ompi_data['flex_min_version']}"
+mpi_standard_major_version = f"{ompi_data['mpi_standard_version']}"
+mpi_standard_minor_version = f"{ompi_data['mpi_standard_subversion']}"
+
+# "release" is a sphinx config variable: assign it to the computed
+# Open MPI version number.  The ompi_ver string begins with a "v"; the
+# Sphinx release variable should not include this prefix "v".
+release = ompi_ver[1:]
+
+# If we are building in a ReadTheDocs.io environment, there will be a
+# READTHEDOCS environment variables that tell us what version to use.
+# Relevant RTD env variables (documented
+# https://docs.readthedocs.io/en/stable/builds.html#build-environment):
+#
+# - READTHEDOCS: will be "True"
+# - READTHEDOCS_VERSION: The RTD slug of the version which is being
+#   built (e.g., "latest")
+# - READTHEDOCS_VERSION_NAME: Corresponding version name as displayed
+#   in RTD's version switch menu (e.g., "stable")
+# - READTHEDOCS_VERSION_TYPE: Type of the event triggering the build
+#   (e.g., "branch", "tag", "external" (for PRs), or "unknown").
+#
+# If we're building in an RTD environment for a tag or external (i.e.,
+# PR), use the RTD version -- not what we just read from the VERSIONS
+# file.
+key = 'READTHEDOCS'
+if key in os.environ and os.environ[key] == 'True':
+    print("OMPI: found ReadTheDocs build environment")
+
+    # Tell Jinja2 templates the build is running on Read the Docs
+    if "html_context" not in globals():
+        html_context = {}
+    html_context["READTHEDOCS"] = True
+
+    # Define the canonical URL if you are using a custom domain on
+    # Read the Docs
+    html_baseurl = os.environ.get("READTHEDOCS_CANONICAL_URL", "")
+
+    rtd_v = os.environ['READTHEDOCS_VERSION']
+    if os.environ['READTHEDOCS_VERSION_TYPE'] == 'external':
+        # Make "release" be shorter than the full "ompi_ver" value.
+        release = f'PR #{rtd_v}'
+        ompi_ver += f' (Github PR #{rtd_v})'
+    else:
+        ompi_ver = rtd_v
+
+        # The "release" Sphinx variable is supposed to be expressed as
+        # a simple value, such as "A.B.C[rcX]" (without a leading
+        # "v").  The ompi_ver value will be one of two things:
+        #
+        # - a git branch name (of the form "vA.B.x")
+        # - a git tag (of the form "A.B.C[rcX]")
+        #
+        # If there's a leading "v", we need to strip it.
+        release = ompi_ver
+        if ompi_ver[0] == 'v':
+            release = ompi_ver[1:]
+
+    # Override the branch names "master" and "main" (that would have
+    # come from the ReadTheDocs version slug) to be "head of
+    # development".
+    if release == 'main' or release == 'master':
+        ompi_ver = 'head of development'
+
+    print(f"OMPI: release = {release}")
+    print(f"OMPI: ompi_ver = {ompi_ver}")
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+import sphinx_rtd_theme
+extensions = [
+    'recommonmark',
+    "sphinx_rtd_theme",
+    "sphinx.ext.extlinks",
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+#
+# Note that we exclude the entire prrte-rst-content/ directory.
+# Here's why:
+#
+# * By default, Sphinx will warn about any .rst file that it finds in
+#   the doc tree that is not referenced via either the "include"
+#   directive or via a TOC.
+# * The prrte-rst-content/ directory contains files that we *do* use
+#   here in the OMPI docs, but it also contains files that we do
+#   *not* use here in the OMPI docs.
+# * Consequently, we explicitly ".. include:: <filename>" the specific
+#   files that we want from the prrte-rst-content/ directory.  And we
+#   specifically do *not* include at least some files in the
+#   prrte-rst-content directory.
+# * Listing files/patterns in exclude_patterns:
+#   * Will prevent Sphinx from searching for/finding new .rst files
+#     that match those patterns.
+#   * Will not prevent explicitly ".. include:"'ing a file with a name
+#     that matches a pattern in exclude_patterns.
+#
+# Hence, listing prrte-rst-content in exclude_patterns means that
+# Sphinx won't complain about the .rst files in that tree that we are
+# not referencing from here in the OMPI docs.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'venv', 'py*/**',
+                    'prrte-rst-content' ]
+
+
+# Clarify the language for verbatim blocks (::)
+# Never change this to anything else. Lots of verbatim code blocks
+# exists across the documentation pages.
+# The default value is actually "default" which is something like Python3
+# and on fail it reverts to "none". We just go directly to none for clarity.
+highlight_language = "none"
+
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_rtd_theme'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+#html_static_path = ['_static']
+
+
+# Short hand external links
+# Allows smoother transitioning for commonly used articles and sites
+extlinks = {
+    'doi': ('https://doi.org/%s', '%s'),
+}
+
+
+# -- Options for MAN output -------------------------------------------------
+
+# Dynamically find all the man pages and build the appropriate list of
+# tuples so that we don't have to manually maintain it.
+
+def find_man_pages_top():
+    def _doit(topdir):
+        for root, dirs, files in os.walk(topdir):
+            for filename in files:
+                # Parse filenames of the format a "foo.X.rst"
+                parts = re.search(r"^([^/]+?)\.([0-9]+)\.rst$", filename)
+
+                # Skip files that do not match that format (e.g.,
+                # "index.rst")
+                if parts is None:
+                    continue
+
+                base_name = parts.group(1)
+                section   = int(parts.group(2))
+
+                full_filename_without_rst = f'{root}/{base_name}.{section}'
+
+                # Append a tuple: (filename, name, description, authors, section)
+                # We leave description blank.
+                # We also leave authors blank, because "The Open MPI community"
+                # already shows up in the copyright section.
+                tuple = (full_filename_without_rst, base_name, '', '', section)
+                tuples.append(tuple)
+
+    tuples = list()
+    _doit("man-openmpi")
+    _doit("man-openshmem")
+
+    return tuples
+
+man_pages = find_man_pages_top()
+
+# -- Open MPI-specific options -----------------------------------------------
+
+# This prolog is included in every file.  Put common stuff here.
+
+rst_prolog = f"""
+.. |mdash|  unicode:: U+02014 .. Em dash
+.. |rarrow| unicode:: U+02192 .. Right arrow
+
+.. |year| replace:: {year}
+.. |ompi_ver| replace:: {ompi_ver}
+.. |ompi_series| replace:: {ompi_series}
+.. |pmix_min_version| replace:: {pmix_min_version}
+.. |pmix_embedded_version| replace:: {pmix_embedded_version}
+.. |prte_min_version| replace:: {prte_min_version}
+.. |prte_embedded_version| replace:: {prte_embedded_version}
+.. |prte_embedded_series| replace:: {prte_embedded_series}
+.. |hwloc_min_version| replace:: {hwloc_min_version}
+.. |hwloc_embedded_version| replace:: {hwloc_embedded_version}
+.. |event_min_version| replace:: {event_min_version}
+.. |event_embedded_version| replace:: {event_embedded_version}
+.. |automake_min_version| replace:: {automake_min_version}
+.. |autoconf_min_version| replace:: {autoconf_min_version}
+.. |libtool_min_version| replace:: {libtool_min_version}
+.. |flex_min_version| replace:: {flex_min_version}
+.. |mpi_standard_version| replace:: {mpi_standard_major_version}.{mpi_standard_minor_version}
+.. |mpi_standard_major_version| replace:: {mpi_standard_major_version}
+.. |mpi_standard_minor_version| replace:: {mpi_standard_minor_version}
+.. |deprecated_favor| replace:: this routine is deprecated in favor of
+
+.. |br| raw:: html
+
+   <br />
+
+"""
+
+# The sphinx_rtd_theme does not properly handle wrapping long lines in
+# table cells when rendering to HTML due to a CSS issue (see
+# https://github.com/readthedocs/sphinx_rtd_theme/issues/1505).  Until
+# the issue is fixed upstream in sphinx_rtd_theme, we can simply
+# override the CSS here.
+rst_prolog += """
+.. raw:: html
+
+   <style>
+   .wy-table-responsive table td,.wy-table-responsive table th{white-space:normal}
+   </style>
+"""
diff --git a/docs/contributing.rst b/docs/contributing.rst
new file mode 100644
index 00000000000..adc3f39524b
--- /dev/null
+++ b/docs/contributing.rst
@@ -0,0 +1,208 @@
+Contributing to Open MPI
+========================
+
+There are many ways to contribute.  Here are a few:
+
+#. Subscribe to `the mailing lists
+   <https://www.open-mpi.org/community/lists/ompi.php>`_ and become
+   active in the discussions.
+
+#. Obtain `a Git clone <https://github.com/open-mpi/ompi/>`_ of Open
+   MPI's code base and start looking through the code.
+
+   .. note:: Be sure to see the :doc:`Developers guide
+             </developers/index>` for technical details about the
+             code base and how to build it).
+
+#. Report bug fixes to the main code base.
+
+   * First, ensure the bug was not already reported by searching
+     `existing      GitHub Issues
+     <https://github.com/open-mpi/ompi/issues>`_.
+   * If you're unable to find an open issue addressing the problem,
+     `open a new one <https://github.com/open-mpi/ompi/issues/new>`_.
+
+#. Submit bug fixes to the main code base.
+
+   * Awesome!  Open a new GitHub pull request with the patch.
+
+     * Please submit bug fixes / new features on the ``main`` branch
+       first, and then port them to the relevant release branch(es)
+       after they have been accepted on ``main``.  :ref:`See this
+       section for more information <git-github-branch-scheme-label>`.
+
+   * Ensure the PR description clearly describes the problem and
+     solution. If there is an existing GitHub issue open describing
+     this bug, please include it in the description so we can track
+     them together.
+   * Be sure to see the :ref:`Open source contributions
+     <contributing-open-source-label>` section, below.
+
+#. Submit feature enhancements and/or new components to the main code
+   base.
+
+   * Awesome!  We love new ideas!
+   * You might want to suggest your change on the `devel mail list
+     <https://www.open-mpi.org/community/lists/ompi.php>`_ before you
+     start writing code.  The `developer level technical information
+     on the internals of Open MPI
+     <https://www.open-mpi.org/faq/?category=developers>`_ may also be
+     useful for large scale features.
+   * If you're contributing a large new piece of functionality, that
+     will be best viewed if you |mdash| or someone, anyone |mdash| is
+     also stepping up to help maintain that functionality over time.
+     We love new ideas and new features, but we do need to be
+     realistic in what we can reliably test and deliver to our users.
+   * Be sure to see the :ref:`Open source contributions
+     <contributing-open-source-label>` section, below.
+
+#. Submit fixes and/or entirely new content to this documentation.
+
+   * Docs are great!  We always need help with documentation.
+   * These docs are authored using ReStructured Text markup with the
+     Sphinx rendering tool under the ``docs/`` directory in the
+     repository.  This means you can submit a pull request with your
+     docs updates, just like you would for any Open MPI code contribution.
+   * :ref:`See this section <developers-installing-sphinx-label>` and
+     :ref:`also this section <developers-rst-for-markdown-expats>` for
+     information about how to install Sphinx locally and some
+     high-level logistical instructions on how to write these docs.
+
+#. Provide testing resources:
+
+   #. For Github Pull Request Continuous Integration (CI)
+   #. For nightly snapshot builds and testing
+
+.. _contributing-open-source-label:
+
+Open source contributions
+-------------------------
+
+All code contributions are submitted as pull requests on the `Open
+MPI GitHub repository <https://github.com/open-mpi/ompi/>`_.
+
+.. important:: All commits must include a ``Signed-off-by:`` line,
+               indicating the submitter's agreement to the :ref:`Open
+               MPI Contributor's Declaration
+               <contributing-contributors-declaration-label>`.
+
+.. _contributing-contributors-declaration-label:
+
+Contributor's Declaration
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In order to ensure that we can keep distributing Open MPI under our
+:doc:`open source license </license/index>`, we need to ensure that
+all contributions are compatible with that license.  Put differently:
+we need to have an established intellectual property pedigree of the
+code in Open MPI.  This means being able to ensure that all code
+included in Open MPI is free, open source, and able to be distributed
+under :doc:`the BSD license </license/index>`.
+
+Open MPI has therefore adopted requirements based on the signed-off-by
+process as described in Section 11 of the Linux kernel document on
+`Submitting Patches
+<https://www.kernel.org/doc/html/latest/process/submitting-patches.html#sign-your-work-the-developer-s-certificate-of-origin>`_.
+Each proposed contribution to the Open MPI code base must include the
+text ``Signed-off-by:`` followed by the contributor's name and email
+address.
+
+.. admonition:: Pro tip
+   :class: tip
+
+   You can use the ``-s`` flag to the ``git commit`` command (i.e.,
+   ``git commit -s ...``) to automatically add the appropriate
+   ``Signed-off-by:`` line to your commit message.
+
+The ``Signed-off-by:`` line is a developer's certification that he or
+she has the right to submit the patch for inclusion into the project,
+and indicates agreement to the Developer's Certificate of Origin:
+
+    By making a contribution to this project, I certify that:
+
+    #. The contribution was created in whole or in part by me and I
+       have the right to submit it under the :doc:`Open MPI open
+       source license </license/index>`; or
+
+    #. The contribution is based upon previous work that, to the best
+       of my knowledge, is covered under an appropriate open source
+       license and I have the right under that license to submit that
+       work with modifications, whether created in whole or in part by
+       me, under the :doc:`Open MPI open source license
+       </license/index/>` (unless I am permitted to submit under a
+       different license); or
+
+    #. The contribution was provided directly to me by some other
+       person who certified (1) or (2) and I have not modified it.
+
+    #. I understand and agree that this project and the contribution
+       are public and that a record of the contribution (including all
+       personal information I submit with it, including my sign-off)
+       is maintained indefinitely and may be redistributed consistent
+       with this project and the open source license(s) involved.
+
+Proposed contributions failing to include the ``Signed-off-by:``
+certification will not be accepted into any Open MPI code
+repository. The community reserves the right to revert any commit
+inadvertently made without the required certification.
+
+.. note:: This policy prevents a situation where intellectual property
+          gets into the Open MPI code base and then someone later
+          claims that we owe them money for it.  Open MPI is a free,
+          open source code base.  We intend it to remain that way.
+
+If you have not already done so, please ensure that *every* commit in
+your pull request contains the ``Signed-off-by:`` line.
+
+Git commit messages
+^^^^^^^^^^^^^^^^^^^
+
+Please write a good Git commit message, with a short first line
+describing *what* was done, and then a description of *why* it was
+done.
+
+.. admonition:: Pro tip
+   :class: tip
+
+   `See this blog entry <https://cbea.ms/git-commit/>`_ for an
+   excellent description of how to write a good Git commit message.
+
+Code style
+^^^^^^^^^^
+
+There are a small number of style rules for Open MPI:
+
+#. For all code:
+
+   * 4 space tabs.  No more, no less.
+   * No tab characters **at all**.  2 indentations are 8 spaces
+     |mdash| not a tab.
+   * m4 code is a bit weird in terms of indentation: we don't have a
+     good, consistent indentation style in our existing code.  But
+     still: no tab characters at all.
+
+#. For C code:
+
+   * We prefer if all blocks are enclosed in ``{}`` (even 1-line
+     blocks).
+   * We prefer that if you are testing equality with a constant, put
+     the constant on the **left** of the ``==``.  E.g., ``if (NULL ==
+     ptr)``.
+   * If there are no parameters to a C function, declare it with
+     ``(void)`` (vs. ``()``).
+
+Closed source contributions
+---------------------------
+
+While we are creating free / open-source software, and we would prefer
+if everyone's contributions to Open MPI were also free / open-source,
+we certainly recognize that other organizations have different goals
+from us.  Such is the reality of software development in today's
+global economy.
+
+As such, it is perfectly acceptable to make non-free / non-open-source
+contributions to Open MPI.  We obviously cannot accept such
+contributions into the main code base, but you are free to distribute
+plugins, enhancements, etc. as you see fit.  Indeed, the :doc:`the BSD
+license </license/index>` is extremely liberal in its redistribution
+provisions.
diff --git a/docs/developers/autogen.rst b/docs/developers/autogen.rst
new file mode 100644
index 00000000000..8239719623e
--- /dev/null
+++ b/docs/developers/autogen.rst
@@ -0,0 +1,32 @@
+Running ``autogen.pl``
+======================
+
+You can now run OMPI's top-level ``autogen.pl`` script.  This script
+will invoke the GNU Autoconf, Automake, and Libtool commands in the
+proper order and do a bunch of component discovery and housekeeping to
+setup to run OMPI's top-level ``configure`` script.
+
+Running ``autogen.pl`` may take a few minutes, depending on your
+system.  It's not very exciting to watch.
+
+If you have a multi-processor system, enabling the multi-threaded
+behavior in Automake 1.11 (or newer) can result in ``autogen.pl``
+running faster.  Do this by setting the ``AUTOMAKE_JOBS`` environment
+variable to the number of processors (threads) that you want it to use
+before invoking ``autogen.pl``.  For example (you can put this in your
+shell startup files)::
+
+   # For bash/sh/zsh:
+   export AUTOMAKE_JOBS=4
+
+   # For csh/tcsh:
+   setenv AUTOMAKE_JOBS 4
+
+.. important:: You generally need to run ``autogen.pl`` whenever the
+   top-level file ``configure.ac`` changes, or any files in the
+   ``config/`` or ``<project>/config/`` directories change (these
+   directories are where a lot of "include" files for Open MPI's
+   ``configure`` script live).
+
+.. note:: You do *NOT* need to re-run ``autogen.pl`` if you modify a
+   ``Makefile.am``.
diff --git a/docs/developers/bindings.rst b/docs/developers/bindings.rst
new file mode 100644
index 00000000000..5363f559494
--- /dev/null
+++ b/docs/developers/bindings.rst
@@ -0,0 +1,112 @@
+C and Fortran Bindings
+======================
+
+The C and Fortran (mpi_f08) bindings are generated from Python code in
+``ompi/mpi/bindings``. Both the language bindings are generated from
+template files for each function. In the C case, each template file corresponds
+to a single generated C file, while in the Fortran case there are three major
+files generated for all functions.
+
+The Python code depends on special prototype lines used with both the C and
+Fortran bindings. These "prototypes" are designed to be easy to parse and use
+specific type constants that can be mapped directly to the expanded
+language-specific code, error-handling, and conversion code.
+
+C Bindings
+----------
+
+This will walk through adding (or converting) a plain-C binding into a
+templated version controlled by the script.
+
+As an example, for ``MPI_Send`` you might have a C file that looks something
+like this:
+
+.. code-block:: c
+
+    #include "ompi_config.h"
+    ...other includes...
+
+    int MPI_Send(const void *buf, int count, MPI_Datatype datatype, int dest,
+                 int tag, MPI_Comm comm)
+    {
+        ...internal checks...
+        return internal_mpi_send(buf, count, datatype, dest, tag, comm);
+    }
+
+To convert this to a template, you will have to first ensure that only a single
+function is defined in the file, removing or abstracting out static functions,
+and separating multiple definitions, such as ``MPI_Send`` and ``MPI_Isend``,
+into different files. The template should also not include any macro-processing
+that attempts to change the name of the function or parameter types; this code
+should be generated by the script, or abstracted into header files that can
+work easily with multiple functions.
+
+At this point, the template should look like the example above, with a "header"
+section, with simple includes or macros, maybe a static global, and the
+function defintion and nothing else.
+
+The next step is to convert the signature line into the prototype format that
+the script expects. For ``MPI_Send``, this should look something like this:
+
+.. code-block:: c
+
+    PROTOTYPE ERROR_CLASS send(BUFFER buf, COUNT count, DATATYPE type, RANK dest,
+                               TAG tag, COMM comm)
+
+Notice how the function name is changed, the ``MPI_`` prefix removed and the
+rest converted to lowercase, and also how each parameter is simplified into a
+``TYPE name`` format, where the ``TYPE`` conforms to an allowed list in
+``ompi/mpi/bindings/ompi_bindings/c_type.py``. For newer functions and types,
+you may have to extend the ``c_type.py`` file with a new class showing how to
+handle the type.
+
+The final step is to update ``Makefile.am``, adding the template name, in this
+case ``send.c.in``, to the ``prototype_sources`` variable, and the generated
+file name, ``generated_send.c``, to ``interface_profile_sources``. The
+generated file name must be of the form ``generated_${basename}.c``, where
+``${basename}`` is the name of the template file stripped of all extensions.
+
+Fortran Bindings
+----------------
+
+Adding new Fortran bindings follows a similar process to the C version above.
+All new interfaces are actually based on a single C-template file following the
+same format as the C interface templates. However, the C file generated will
+use Fortran-specific arguments, including ``CFI_*`` arguments, when TS 29113 is
+enabled, ``MPI_Fint *`` arguments in other cases, and others specific to how
+the Fortran MPI types are defined. Most of these files perform Fortran-specific
+error handling, Fortran-to-C type conversion, and other necessary steps before
+calling the actually C bindings with the proper arguments.
+
+These templates are used not only to generate a C backing file for the Fortran
+code, but also the Fortran interface definitions and the Fortran subroutines
+corresponding to the generated C file. These are output in three separate files:
+
+* ``ompi/mpi/fortran/use-mpi-f08/api_f08_generated.F90``
+* ``ompi/mpi/fortran/use-mpi-f08/base/api_f08_generated.c``
+* ``ompi/mpi/fortran/use-mpi-f08/mod/mpi-f08-interfaces-generated.h``
+
+The Fortran file ``api_f08_generated.F90`` contains all the internal subroutine
+definitions, each of which makes a call into corresponding C functions. The
+internal subroutine names are mapped to the external interface, including
+multiple interfaces for the bigcount version of functions, in
+``mpi-f08-interfaces-generated.h``. The C file ``api_f08_generated.c``
+basically contains a concatenation of all fully expanded C templates.
+These files contain preprocessing directives to ensure they can support
+compilers with and without TS 29113 support, allowing use of
+``CFI_cdesc_t`` types when available (see `Fortran 2018`_ for more details).
+
+.. _Fortran 2018: https://fortranwiki.org/fortran/show/Fortran+2018
+
+If a new type needs to be added, then one will need to extend
+``fortran_type.py`` in ``ompi/mpi/bindings/ompi_bindings`` with an additional
+type class specifying how to handle the type in the above generated files,
+including any required key-value attributes for more complicated types. New
+types use a ``Type`` base class with functions that can be implemented by
+derived classes, each returning expanded Fortran or C code.
+
+Other Considerations
+--------------------
+
+Keep in mind that the generated files will not be deleted with a ``make clean``
+or ``make distclean``; instead use ``make maintainer-clean`` to delete those.
diff --git a/docs/developers/building-open-mpi.rst b/docs/developers/building-open-mpi.rst
new file mode 100644
index 00000000000..ee09d294e20
--- /dev/null
+++ b/docs/developers/building-open-mpi.rst
@@ -0,0 +1,9 @@
+Building Open MPI
+=================
+
+Once you have run ``autogen.pl`` successfully, you can configure and
+build Open MPI just like end users do with official distribution Open
+MPI tarballs.
+
+See the :doc:`general "Install Open MPI" documentation for more
+details. </installing-open-mpi/index>`
diff --git a/docs/developers/compiler-pickyness.rst b/docs/developers/compiler-pickyness.rst
new file mode 100644
index 00000000000..bab9db0900d
--- /dev/null
+++ b/docs/developers/compiler-pickyness.rst
@@ -0,0 +1,29 @@
+Compiler Pickyness by Default
+=============================
+
+If you are building Open MPI from a Git clone (i.e., there is a
+``.git`` directory in your build tree), the default build includes
+extra compiler pickyness, which will result in more compiler warnings
+than in non-developer builds.  Getting these extra compiler warnings
+is helpful to Open MPI developers in making the code base as clean as
+possible.
+
+Developers can disable this picky-by-default behavior by using the
+``--disable-picky`` configure option.  Also note that extra-picky
+compiles do *not* happen automatically when you do a VPATH build
+(e.g., if ``.git`` is in your source tree, but not in your build
+tree).
+
+Prior versions of Open MPI would automatically activate a lot of
+(performance-reducing) debugging code by default if ``.git`` was found
+in your build tree.  This is no longer true.  You can manually enable
+these (performance-reducing) debugging features in the Open MPI code
+base with these configure options:
+
+* ``--enable-debug``
+* ``--enable-mem-debug``
+* ``--enable-mem-profile``
+
+.. note:: These options are really only relevant to those who are
+   developing Open MPI itself.  They are not generally helpful for
+   debugging general MPI applications.
diff --git a/docs/developers/frameworks.rst b/docs/developers/frameworks.rst
new file mode 100644
index 00000000000..ddbbc05acbc
--- /dev/null
+++ b/docs/developers/frameworks.rst
@@ -0,0 +1,142 @@
+.. _label-frameworks:
+
+Internal frameworks
+===================
+
+The Modular Component Architecture (MCA) is the backbone of Open MPI
+-- most services and functionality are implemented through MCA
+components.
+
+MPI layer frameworks
+--------------------
+
+Here is a list of all the component frameworks in the MPI layer of
+Open MPI:
+
+* ``bml``: BTL management layer
+* ``coll``: MPI collective algorithms
+* ``fbtl``: point to point file byte transfer layer: abstraction for
+  individual read: collective read and write operations for MPI I/O
+* ``fcoll``: collective file system functions for MPI I/O
+* ``fs``: file system functions for MPI I/O
+* ``hook``: Generic hooks into Open MPI
+* ``io``: MPI I/O
+* ``mtl``: Matching transport layer, used for MPI point-to-point
+  messages on some types of networks
+* ``op``: Back end computations for intrinsic MPI_Op operators
+* ``osc``: MPI one-sided communications
+* ``pml``: MPI point-to-point management layer
+* ``sharedfp``: shared file pointer operations for MPI I/O
+* ``topo``: MPI topology routines
+* ``vprotocol``: Protocols for the "v" PML
+
+OpenSHMEM component frameworks
+------------------------------
+
+* ``atomic``: OpenSHMEM atomic operations
+* ``memheap``: OpenSHMEM memory allocators that support the
+  PGAS memory model
+* ``scoll``: OpenSHMEM collective operations
+* ``spml``: OpenSHMEM "pml-like" layer: supports one-sided,
+  point-to-point operations
+* ``sshmem``: OpenSHMEM shared memory backing facility
+
+Miscellaneous frameworks
+------------------------
+
+* ``allocator``: Memory allocator
+* ``backtrace``: Debugging call stack backtrace support
+* ``btl``: Point-to-point Byte Transfer Layer
+* ``dl``: Dynamic loading library interface
+* ``hwloc``: Hardware locality (hwloc) versioning support
+* ``if``: OS IP interface support
+* ``installdirs``: Installation directory relocation services
+* ``memchecker``: Run-time memory checking
+* ``memcpy``: Memory copy support
+* ``memory``: Memory management hooks
+* ``mpool``: Memory pooling
+* ``patcher``: Symbol patcher hooks
+* ``pmix``: Process management interface (exascale)
+* ``rcache``: Memory registration cache
+* ``reachable``: Network reachability determination
+* ``shmem``: Shared memory support (NOT related to OpenSHMEM)
+* ``smsc``: Shared memory single-copy support
+* ``threads``: OS and userspace thread support
+* ``timer``: High-resolution timers
+
+Framework notes
+---------------
+
+Each framework typically has one or more components that are used at
+run-time.  For example, the ``btl`` framework is used by the MPI layer
+to send bytes across different types underlying networks.  The ``tcp``
+``btl``, for example, sends messages across TCP-based networks; the
+``ucx`` ``pml`` sends messages across InfiniBand-based networks.
+
+MCA parameter notes
+-------------------
+
+Each component typically has some tunable parameters that can be
+changed at run-time.  Use the :ref:`ompi_info(1) <man1-ompi_info>`
+command to check a component to see what its tunable parameters are.
+For example:
+
+.. code-block:: sh
+
+   shell$ ompi_info --param btl tcp
+
+shows some of the parameters (and default values) for the ``tcp`` ``btl``
+component (use ``--all`` or ``--level 9`` to show *all* the parameters).
+
+Note that ``ompi_info`` (without ``--all`` or a specified level) only
+shows a small number a component's MCA parameters by default.  Each
+MCA parameter has a "level" value from 1 to 9, corresponding to the
+MPI-3 MPI_T tool interface levels.  :ref:`See the LEVELS section in
+the ompi_info(1) man page <man1-ompi_info-levels>` for an explanation
+of the levels and how they correspond to Open MPI's code.
+
+Here's rules of thumb to keep in mind when using Open MPI's levels:
+
+* Levels 1-3:
+
+  * These levels should contain only a few MCA parameters.
+  * Generally, only put MCA parameters in these levels that matter to
+    users who just need to *run* Open MPI applications (and don't
+    know/care anything about MPI).  Examples (these are not
+    comprehensive):
+
+    * Selection of which network interfaces to use.
+    * Selection of which MCA components to use.
+    * Selective disabling of warning messages (e.g., show warning
+      message XYZ unless a specific MCA parameter is set, which
+      disables showing that warning message).
+    * Enabling additional stderr logging verbosity.  This allows a
+      user to run with this logging enabled, and then use that output
+      to get technical assistance.
+
+* Levels 4-6:
+
+  * These levels should contain any other MCA parameters that are
+    useful to expose to end users.
+  * There is an expectation that "power users" will utilize these MCA
+    parameters |mdash| e.g., those who are trying to tune the system
+    and extract more performance.
+  * Here's some examples of MCA parameters suitable for these levels
+    (these are not comprehensive):
+
+    * When you could have hard-coded a constant size of a resource
+      (e.g., a resource pool size or buffer length), make it an MCA
+      parameter instead.
+    * When there are multiple different algorithms available for a
+      particular operation, code them all up and provide an MCA
+      parameter to let the user select between them.
+
+* Levels 7-9:
+
+  * Put any other MCA parameters here.
+  * It's ok for these MCA parameters to be esoteric and only relevant
+    to deep magic / the internals of Open MPI.
+  * There is little expectation of users using these MCA parameters.
+
+See :ref:`this section <label-running-setting-mca-param-values>` for
+details on how to set MCA parameters at run time.
diff --git a/docs/developers/git-github.rst b/docs/developers/git-github.rst
new file mode 100644
index 00000000000..f9d5c75671a
--- /dev/null
+++ b/docs/developers/git-github.rst
@@ -0,0 +1,196 @@
+GitHub, Git, and related topics
+===============================
+
+GitHub
+------
+
+Open MPI's Git repositories are `hosted at GitHub
+<https://github.com/open-mpi/ompi>`_.
+
+#. First, you will need a Git client. We recommend getting the latest
+   version available. If you do not have the command ``git`` in your
+   path, you will likely need to download and install Git.
+#. `ompi <https://github.com/open-mpi/ompi/>`_ is the main Open MPI
+   repository where most active development is done.  Git clone this
+   repository.  Note that the use of the ``--recursive`` CLI option is
+   necessary because Open MPI uses Git submodules::
+
+      shell$ git clone --recursive https://github.com/open-mpi/ompi.git
+
+Note that Git is natively capable of using many forms of web
+proxies. If your network setup requires the user of a web proxy,
+`consult the Git documentation for more details
+<https://git-scm.com/>`_.
+
+Git commits: open source / contributor's declaration
+----------------------------------------------------
+
+In order to remain open source, all new commits to the Open MPI
+repository must include a ``Signed-off-by:`` line, indicating the
+submitter's agreement to the :ref:`Open MPI Contributor's Declaration
+<contributing-contributors-declaration-label>`.
+
+.. tip:: You can use the ``-s`` option to ``git commit`` to
+         automatically add the ``Signed-off-by:`` line to your commit
+         message.
+
+.. _git-github-branch-scheme-label:
+
+Git branch scheme
+-----------------
+
+Generally, Open MPI has two types of branches in its Git repository:
+
+#. ``main``:
+
+   * All active development occurs on the ``main`` branch (new features,
+     bug fixes, etc.).
+
+#. Release branches of the form ``vMAJOR.MINOR.x`` (e.g., ``v4.0.x``,
+   ``v4.1.x``, ``v5.0.x``).
+
+   * The ``.x`` suffix indicates that this branch is used to create
+     all releases in the Open MPI vMAJOR.MINOR series.
+   * Periodically, the Open MPI community will make a new release
+     branch, typically from ``main``.
+   * A Git tag of the form ``vMAJOR.MINOR.RELEASE`` is used to
+     indicate the specific commit on a release branch from where
+     official Open MPI release tarball was created (e.g., ``v4.1.0``,
+     ``v4.1.1``, ``v4.1.2``, etc.).
+
+Once a bug is fixed or a new feature is implemented on ``main``, it is
+cherry-picked over to the relevant release branch(es).
+
+.. attention:: It may seem odd to some, but the Open MPI community
+               development model does *not* PR bug fixes or new
+               features directly to release branches.  Instead,
+               initial bug-fix / feature PRs are generally first made
+               to ``main``.
+
+               This helps us ensure that future releases (with
+               ``main`` as a Git ancestor) will contain the bug fix /
+               feature.
+
+For example:
+
+.. code:: sh
+
+   shell$ git checkout main
+   shell$ git pull --rebase
+   shell$ git checkout pr/bug-fix
+
+   # ... make changes / fix a bug / etc. ...
+
+   shell$ git add ...
+   shell$ git commit -s ...
+   shell$ git push myfork
+
+At this point, you go create a PR from your fork's ``pr/bug-fix``
+branch to the Open MPI community GitHub repo ``main`` branch.  Work
+with the community to get the PR completed and merged.  Then you can
+open a new PR to cherry pick the Git commits from that bug fix to each
+of the relevant release branches.
+
+Depending on how far the release branch has diverged from ``main``,
+there may be some porting effort involved in the cherry-pick.
+
+For example, if your bug fix on ``main`` is comprised of a single Git
+commit hash ``123abc``:
+
+.. code:: sh
+
+   # Fetch all upstream git activity, including the merge of the "main" PR.
+   shell$ get fetch --all
+
+   # Check out the target release branch, and advance to the most recent commit.
+   shell$ git checkout v5.0.x
+   shell$ git pull --rebase
+
+   # Make a branch for your bug fix
+   shell$ git checkout -b pr/v5.0.x/bug-fix
+   # Cherry pick the commit from the "main" branch
+   shell$ git cherry-pick -x 123abc
+   # Push to your fork
+   shell$ git push myfork
+
+The Open MPI development community *requires* adding the following
+line to the commit message of cherry-picked commits on release
+branches:
+
+.. code:: text
+
+   (cherry picked from commit [git_hash_of_original_commit])
+
+.. note:: Note the use of the ``-x`` option to ``git cherry-pick``.
+          This option automatically adds the ``(cherry picked from
+          ...)`` line to your commit message.
+
+.. admonition:: Rationale
+   :class: tip
+
+   Git does not actually store any meta data about Git cherry-picks in
+   the commit.  Having a standardized text line containing the source
+   Git commit hash in the commit messages helps the Open MPI
+   development community track where commits came from on release
+   branches, and therefore allows us to check whether all relevant
+   commits have been ported to a given release branch.
+
+Once your commits are ready and pushed up to your fork, make a PR to
+the target release branch.
+
+.. warning:: A GitHub PR CI job checks all commits on release branches
+             for the ``(cherry picked from...)`` line. It will also
+             ensure that the Git hash cited in that line actually
+             exists on the ``main`` branch.
+
+             This check ensures that commits are not made to release
+             branches before their corresponding ``main`` PR was
+             merged.
+
+All this being said, sometimes there is a need for a non-cherry-picked
+commit on a release branch. E.g., sometimes a release branch has
+diverged so much that the bug no longer exists on ``main``.  It would
+therefore not make sense |mdash| or even be impossible |mdash| to
+commit the bug fix in question to ``main``.
+
+In such cases, make a regular PR to the target branch (with commits
+that do *not* include ``(cherry picked from ...)`` lines).  In the PR
+description, add a line with the following token:
+
+.. code:: text
+
+   bot:notacherrypick
+
+This tells the GitHub CI job that this PR contains commits that are
+not cherry-picked from ``main``.
+
+.. warning:: ``bot:notacherrypick`` should only be used when
+             absolutely necessary.  It is not a license to avoid
+             the process of PR'ing to ``main`` first.
+
+CI (testing)
+------------
+
+The Open MPI community generally runs two flavors of testing:
+
+#. A bunch of tests on each PR (Continuous Integration / CI).  These
+   tests are a mixture of GitHub Actions and other CI systems (e.g.,
+   Jenkins).  Examples include (but are not limited to):
+
+   * Check each Git commit for bozo email addresses
+   * Check that each Git commit contains a ``Signed-off-by`` line
+   * Check that commits on release branches contain a cherry-pick
+     notice
+   * Build and publish the docs
+   * Build Open MPI in a variety of environments and run sanity tests
+     with that installation
+
+#. Daily testing via the MPI Testing Tool (MTT).
+
+   * These are generally tests that take much longer to run than on a
+     per-PR basis.  `A "nightly snapshot" tarball
+     <https://www.open-mpi.org/nightly/>`_ is created for ``main`` and
+     each relevant release branch.
+   * MTT tests are run with this snapshot tarball so that all
+     organizations are testing with the same snapshots.
+   * `Results are available here <https://mtt.open-mpi.org/>`_.
diff --git a/docs/developers/gnu-autotools.rst b/docs/developers/gnu-autotools.rst
new file mode 100644
index 00000000000..c6a9c96ca56
--- /dev/null
+++ b/docs/developers/gnu-autotools.rst
@@ -0,0 +1,312 @@
+.. _developers-installing-autotools-label:
+
+Manually installing the GNU Autootools
+======================================
+
+There is enough detail in building the GNU Autotools that it warrants
+its own section.
+
+.. note:: As noted above, you only need to read/care about this
+          section if you are building Open MPI from a Git clone.  End
+          users installing an Open MPI distribution tarball do *not*
+          need to have the GNU Autotools installed.
+
+Use a package manager
+---------------------
+
+It is recommended that you use your Linux distribution's package
+manager, or Homebrew or MacPorts on macOS to install recent versions
+of GNU Autoconf, Automake, and Libtool.
+
+If you cannot, keep reading in this section to see how to build and
+install these applications manually (i.e., download the source from
+the internet and build/install it yourself).
+
+Autotools versions
+------------------
+
+The following tools are required for developers to compile Open MPI
+from its repository sources (users who download Open MPI tarballs do
+not need these tools - they are only required for developers working
+on the internals of Open MPI itself):
+
+.. list-table::
+   :header-rows: 1
+
+   * - Software package
+     - Notes
+     - URL
+
+   * - GNU m4
+     - See version chart below
+     - https://ftp.gnu.org/gnu/m4/
+   * - GNU Autoconf
+     - See version chart below
+     - https://ftp.gnu.org/gnu/autoconf/
+   * - GNU Automake
+     - See version chart below
+     - https://ftp.gnu.org/gnu/automake/
+   * - GNU Libtool
+     - See version chart below
+     - https://ftp.gnu.org/gnu/libtool/
+
+The table below lists the versions that are used to make nightly
+snapshot and official release Open MPI tarballs. Other versions of the
+tools *may* work for some (but almost certainly not all) platforms;
+the ones listed below are the versions that we know work across an
+extremely wide variety of platforms and environments.
+
+To strengthen the above point: the core Open MPI developers typically
+use very, very recent versions of the GNU tools.  There are known bugs
+in older versions of the GNU tools that Open MPI no longer compensates
+for (it seemed senseless to indefinitely support patches for ancient
+versions of Autoconf, for example).
+
+.. warning:: You **will** have problems if you do not use recent
+             versions of the GNU Autotools.
+
+That being said, ``autogen.pl`` and ``configure.ac`` scripts tend to
+be a bit lenient and enforce slightly older minimum versions than the
+ones listed below. This is because such older versions still make
+usable Open MPI builds on many platforms - especially Linux on x86_64
+with GNU compilers - and are convenient for developers whose Linux
+distribution may not have as recent as the versions listed below (but are
+recent enough to produce a working version for their platform).
+
+To be clear: the versions listed below are required to support a wide
+variety of platforms and environments, and are used to make nightly
+and official release tarballs. When building Open MPI, YMMV when using
+versions older than those listed below |mdash| especially if you are
+not building on Linux x86_64 with the GNU compilers.
+
+Using older versions is unsupported. If you run into problems, upgrade
+to at least the versions listed below.
+
+.. note:: You may need to scroll right in the following table.
+
+.. list-table::
+   :header-rows: 1
+
+   * - Open MPI
+     - M4
+     - Autoconf
+     - Automake
+     - Libtool
+     - Flex
+     - Sphinx
+
+   * - v1.0.x
+     - NA
+     - 2.58 - 2.59
+     - 1.7 - 1.9.6
+     - 1.5.16 - 1.5.22
+     - 2.5.4
+     - NA
+   * - v1.1.x
+     - NA
+     - 2.59
+     - 1.9.6
+     - 1.5.16 - 1.5.22
+     - 2.5.4
+     - NA
+   * - v1.2.x
+     - NA
+     - 2.59
+     - 1.9.6
+     - 1.5.22 - 2.1a
+     - 2.5.4
+     - NA
+   * - v1.3.x
+     - 1.4.11
+     - 2.63
+     - 1.10.1
+     - 2.2.6b
+     - 2.5.4
+     - NA
+   * - v1.4.x
+     - 1.4.11
+     - 2.63
+     - 1.10.3
+     - 2.2.6b
+     - 2.5.4
+     - NA
+   * - v1.5.x for x=0-4
+     - 1.4.13
+     - 2.65
+     - 1.11.1
+     - 2.2.6b
+     - 2.5.4
+     - NA
+   * - v1.5.x for x>=5
+     - 1.4.16
+     - 2.68
+     - 1.11.3
+     - 2.4.2
+     - 2.5.35
+     - NA
+   * - v1.6.x
+     - 1.4.16
+     - 2.68
+     - 1.11.3
+     - 2.4.2
+     - 2.5.35
+     - NA
+   * - v1.7.x
+     - 1.4.16
+     - 2.69
+     - 1.12.2
+     - 2.4.2
+     - 2.5.35
+     - NA
+   * - v1.8.x
+     - 1.4.16
+     - 2.69
+     - 1.12.2
+     - 2.4.2
+     - 2.5.35
+     - NA
+   * - v1.10.x
+     - 1.4.16
+     - 2.69
+     - 1.12.2
+     - 2.4.2
+     - 2.5.35
+     - NA
+   * - v2.0.x through v4.y
+     - 1.4.17
+     - 2.69
+     - 1.15
+     - 2.4.6
+     - 2.5.35
+     - NA
+   * - v5.0.x
+     - 1.4.17
+     - 2.69
+     - 1.15
+     - 2.4.6
+     - 2.5.35
+     - 4.2.0
+   * - Git main
+     - 1.4.17
+     - 2.69
+     - 1.15
+     - 2.4.6
+     - 2.5.35
+     - 4.2.0
+
+Checking your versions
+----------------------
+
+You can check what versions of the Autotools you have installed with
+the following:
+
+.. code-block:: sh
+
+   shell$ m4 --version
+   shell$ autoconf --version
+   shell$ automake --version
+   shell$ libtoolize --version
+
+Installing the GNU Autotools from source
+----------------------------------------
+
+.. note:: Most operating system packaging systems (to include Homebrew
+          and MacPorts on MacOS) install recent-enough versions of the
+          GNU Autotools.  You should generally only install the GNU
+          Autotools manually if you can't use your operating system
+          packaging system to install them for you.
+
+The GNU Autotools sources can be can be downloaded from:
+
+* https://ftp.gnu.org/gnu/autoconf/
+* https://ftp.gnu.org/gnu/automake/
+* https://ftp.gnu.org/gnu/libtool/
+* And if you need it: https://ftp.gnu.org/gnu/m4/
+
+It is certainly easiest to download/build/install all four of these
+tools together.  But note that Open MPI has no specific m4
+requirements; it is only listed here because Autoconf requires minimum
+versions of GNU m4.  Hence, you may or may not *need* to actually
+install a new version of GNU m4.  That being said, if you are confused
+or don't know, just install the latest GNU m4 with the rest of the GNU
+Autotools and everything will work out fine.
+
+
+Build and install ordering
+--------------------------
+
+You must build and install the GNU Autotools in the following order:
+
+#. m4
+#. Autoconf
+#. Automake
+#. Libtool
+
+.. important:: You *must* install the last three tools (Autoconf,
+               Automake, Libtool) into the same prefix directory.
+               These three tools are somewhat inter-related, and if
+               they're going to be used together, they *must* share a
+               common installation prefix.
+
+You can install m4 anywhere as long as it can be found in the path;
+it may be convenient to install it in the same prefix as the other
+three.  Or you can use any recent-enough m4 that is in your path.
+
+.. warning:: It is *strongly* encouraged that you do **not** install
+   your new versions over the OS-installed versions.  This could cause
+   other things on your system to break.  Instead, install into
+   ``$HOME/local``, or ``/usr/local``, or wherever else you tend to
+   install "local" kinds of software.
+
+   In doing so, be sure to prefix your ``$PATH`` with the directory
+   where they are installed.  For example, if you install into
+   ``$HOME/local``, you may want to edit your shell startup file
+   (``.bashrc``, ``.cshrc``, ``.tcshrc``, etc.) to have something
+   like
+
+   .. code-block:: sh
+
+      # For bash/sh:
+      export PATH=$HOME/local/bin:$PATH
+      # For csh/tcsh:
+      set path = ($HOME/local/bin $path)
+
+   Ensure to set your ``$PATH`` *before* you configure/build/install
+   the four packages.
+
+All four packages require two simple commands to build and
+install:
+
+.. code-block:: sh
+
+   shell$ cd M4_DIRECTORY
+   shell$ ./configure --prefix=PREFIX
+   shell$ make all install
+
+.. important:: If you are using a shell that does not automatically
+               re-index the ``$PATH`` (e.g., the ``csh`` or ``tcsh``
+               shells), be sure to run the ``rehash`` command before
+               you install the next package so that the executables
+               that were just installed can be found by the next
+               package.
+
+.. code-block:: sh
+
+   # Make $PATH be re-indexed if necessary, e.g., via "rehash"
+   shell$ cd AUTOCONF_DIRECTORY
+   shell$ ./configure --prefix=PREFIX
+   shell$ make all install
+
+.. code-block:: sh
+
+   # Make $PATH be re-indexed if necessary, e.g., via "rehash"
+   shell$ cd AUTOMAKE_DIRECTORY
+   shell$ ./configure --prefix=PREFIX
+   shell$ make all install
+
+.. code-block:: sh
+
+   # Make $PATH be re-indexed if necessary, e.g., via "rehash"
+   shell$ cd LIBTOOL_DIRECTORY
+   shell$ ./configure --prefix=PREFIX
+   shell$ make all install
diff --git a/docs/developers/index.rst b/docs/developers/index.rst
new file mode 100644
index 00000000000..6e82d253d37
--- /dev/null
+++ b/docs/developers/index.rst
@@ -0,0 +1,25 @@
+Developer's guide
+=================
+
+This section is here for those who are building/exploring the
+internals of Open MPI in its source code form, most likely through a
+developer's tree (i.e., a Git clone).
+
+If you are writing MPI applications that simply *use* Open MPI, you
+probably don't need to read this section.
+
+.. toctree::
+   :maxdepth: 1
+
+   prerequisites
+   git-github
+   compiler-pickyness
+   autogen
+   building-open-mpi
+   terminology
+   source-code
+   frameworks
+   gnu-autotools
+   sphinx
+   rst-for-markdown-expats.rst
+   bindings
diff --git a/docs/developers/prerequisites.rst b/docs/developers/prerequisites.rst
new file mode 100644
index 00000000000..e2aa34d4c9c
--- /dev/null
+++ b/docs/developers/prerequisites.rst
@@ -0,0 +1,147 @@
+Prerequisites
+=============
+
+Compilers
+---------
+
+Although it should probably be assumed, you'll need a C compiler that
+supports C11.
+
+You'll also need a Fortran compiler if you want to build the Fortran
+MPI bindings (the more recent the Fortran compiler, the better), and a
+Java compiler if you want to build the (unofficial) Java MPI bindings.
+
+GNU Autotools
+-------------
+
+When building Open MPI from its repository sources, the GNU Autotools
+must be installed (i.e., `GNU Autoconf
+<https://www.gnu.org/software/autoconf/>`_, `GNU Automake
+<https://www.gnu.org/software/automake/>`_, and `GNU Libtool
+<https://www.gnu.org/software/libtool/>`_).
+
+.. list-table::
+   :header-rows: 1
+   :widths: 10 10
+
+   * - Tool
+     - Minimum version
+   * - Autoconf
+     - |autoconf_min_version|
+   * - Automake
+     - |automake_min_version|
+   * - Libtool
+     - |libtool_min_version|
+
+.. note:: The GNU Autotools are *not* required when building Open MPI
+          from distribution tarballs.  Open MPI distribution tarballs
+          are bootstrapped such that end-users do not need to have the
+          GNU Autotools installed.
+
+You can generally install GNU Autoconf, Automake, and Libtool via your
+Linux distribution native package system, or via Homebrew or MacPorts
+on MacOS.  This usually "just works."
+
+If you run into problems with the GNU Autotools, or need to download /
+build them manually, see the :ref:`how to build and install GNU
+Autotools section <developers-installing-autotools-label>` for much
+more detail.
+
+Python
+------
+
+Python >= v3.6 is required for generating the Fortran bindings, which
+is necessary if you build Open MPI from a Git clone.
+
+Python is also required for running Sphinx to generate the docs, too
+(:ref:`see below <developers-requirements-sphinx-label>`).
+
+Perl
+----
+
+Open MPI still uses Perl for a few of its build scripts (most notably,
+``autogen.pl``).
+
+Generally speaking, any recent-ish release of Perl 5 should be
+sufficient to correctly execute Open MPI's Perl scripts.
+
+Flex
+----
+
+Minimum supported version: |flex_min_version|.
+
+`Flex <https://github.com/westes/flex>`_ is used during the
+compilation of a developer's checkout (it is not used to build
+official distribution tarballs).  Other flavors of lex are *not*
+supported: given the choice of making parsing code portable between
+all flavors of lex and doing more interesting work on Open MPI, we
+greatly prefer the latter.
+
+Note that no testing has been performed to see what the minimum
+version of Flex is required by Open MPI.  We suggest that you use
+v2.5.35 at the earliest.
+
+For now, Open MPI will allow developer builds with Flex |flex_min_version|.  This
+is primarily motivated by the fact that RedHat/CentOS 5 ships with
+Flex 2.5.4.  It is likely that someday Open MPI developer builds will
+require Flex version >=2.5.35.
+
+Note that the ``flex``-generated code generates some compiler warnings
+on some platforms, but the warnings do not seem to be consistent or
+uniform on all platforms, compilers, and flex versions.  As such, we
+have done little to try to remove those warnings.
+
+If you do not have Flex installed and cannot easily install it via
+your operating system's packaging system (to include Homebrew or
+MacPorts on MacOS), see `the Flex Github repository
+<https://github.com/westes/flex>`_.
+
+
+.. _developers-requirements-sphinx-label:
+
+Sphinx (and therefore Python)
+-----------------------------
+
+`Sphinx <https://www.sphinx-doc.org/>`_ is a Python-based tool used to
+generate both the HTML version of the documentation (that you are
+reading right now) and the nroff man pages.
+
+Official Open MPI distribution tarballs contain pre-built HTML
+documentation and man pages.  This means that |mdash| similar to the GNU
+Autotools |mdash| end users do not need to have Sphinx installed, but will
+still have both the HTML documentation and man pages installed as part
+of the normal configure / build / install process.
+
+However, the HTML documentation and man pages are *not* stored in Open
+MPI's Git repository; only the ReStructred Text source code of the
+documentation is in the Git repository.  Hence, if you are building
+Open MPI from a Git clone, you will need Sphinx (and some Python
+modules) in order to build the HTML documentation and man pages.
+
+.. important:: Most systems do not have Sphinx and/or the required
+               Python modules installed by default.  :ref:`See the
+               Installing Sphinx section
+               <developers-installing-sphinx-label>` for details on
+               how to install Sphinx and the required Python modules.
+
+If ``configure`` is able to find Sphinx and the required Python
+modules, it will automatically generate the HTML documentation and man
+pages during the normal build procedure (i.e., during ``make all``).
+If ``configure`` is *not* able to find Sphinx and/or the required
+Python modules, it will simply skip building the documentation.
+
+.. note:: If you have built/installed Open MPI from a Git clone and
+          unexpectedly did not have the man pages installed, it is
+          likely that you do not have Sphinx and/or the required
+          Python modules available.
+
+          :ref:`See the Installing Sphinx section
+          <developers-installing-sphinx-label>` for details on how
+          to install Sphinx and the required Python modules.
+
+.. important:: ``make dist`` will fail if ``configure`` did not find
+               Sphinx and/or the required Python modules.
+               Specifically: if ``make dist`` is not able to generate
+               the most up-to-date HTML documentation and man pages,
+               you cannot build a distribution tarball.  **This is an
+               intentional design decision.**
diff --git a/docs/developers/rst-for-markdown-expats.rst b/docs/developers/rst-for-markdown-expats.rst
new file mode 100644
index 00000000000..af09f7dee33
--- /dev/null
+++ b/docs/developers/rst-for-markdown-expats.rst
@@ -0,0 +1,404 @@
+.. _developers-rst-for-markdown-expats:
+
+ReStructured Text for those who know Markdown
+=============================================
+
+You can think of RST as "Markdown, but much better".  Meaning:
+
+#. RST is basically just as simple as Markdown
+#. But RST is both more precise, and has more available formatting
+   constructs (without getting crazy complicated)
+
+The full Sphinx / RST documentation is available here:
+https://www.sphinx-doc.org/en/master/index.html
+
+If you're familiar with Markdown, the following sections contain some
+tips to get you started in RST.
+
+Whitespace and indenting
+------------------------
+
+* MD: Whitespace and indenting generally doesn't matter in most
+  cases.  It does matter with bullets and sub bullets, but the rules
+  get pretty weird, and vary between different Markdown renderers.
+
+* RST: **Indenting matters**.  A lot.  Just like Python.  In
+  general, you indent all RST text to keep it within the same level.
+  For example, all this text would be a single paragraph
+
+  **Blank lines also matter**.  A lot.  You use blank lines to
+  delimit sections within an indenting level.  For example, the
+  blank line before this paragraph denotes a paragraph break.
+
+  .. note:: RST was created by the Python community.  Hence,
+            whitespace is quite important.
+
+            * Indenting matters
+            * Blank lines between content matter
+
+Using a blank line and outdenting indicates the end of the previous
+item.  For example, this paragraph is not part of the MD/RST
+bulleted list.
+
+Fixed width font
+----------------
+
+* MD: Use single quotes:
+
+  .. code-block:: md
+
+     `hello world`
+
+* RST: Use a pair of single quotes:
+
+  .. code-block:: rst
+
+     ``hello world``
+
+Italics
+-------
+
+* MD: ``*hello world*`` or ``_hello world_``
+* RST: ``*hello world*``
+
+Boldface
+--------
+
+* MD: ``**hello world**``
+* RST: Same as MD
+
+Chapter and section delimiters
+------------------------------
+
+* MD: Either use one or more pound signs (#, ##, ###) to the left of
+  the line of text, or underline the line of text with pound signs
+
+* RST: Have a single line of text, underlined by non-ASCII
+  characters.
+
+  * The length of the underlying *must* be at least as long as the
+    line of text
+  * Which non-ASCII character is used for the underlying does not
+    matter, but the order in which they are used denotes chapters
+    / sections / subsections / etc.
+
+    In these OMPI docs, the sequence of underline characters we use
+    are:
+
+    .. code-block:: rst
+
+       Chapter 1: hello world
+       ======================
+
+    .. code-block:: rst
+
+       Section 1: hello world
+       ----------------------
+
+    .. code-block:: rst
+
+       Subsection 1: hello world
+       ^^^^^^^^^^^^^^^^^^^^^^^^^
+
+    .. code-block:: rst
+
+       Subsubsection 1: hello world
+       ++++++++++++++++++++++++++++
+
+    Meaning: underlines made of ``=`` denotes chapters, underlines
+    made of ``-`` denotes sections, underlines made of ``^`` denotes
+    subsections, and underlines made of ``+`` denote subsubsections.
+
+Multi-line code/fixed-width font
+--------------------------------
+
+* MD: Use three single quotes to delimit blocks of text.  Optionally
+  include a token keyword to denote the syntax highlighting to use
+  inside that block.
+
+  .. code-block:: md
+
+     ```c
+     int main() { printf("Hello world\n"); return 0 }
+     ```
+
+* RST: Use ``.. code-block:: KEYWORD`` to start a section of code.
+
+  .. code-block:: rst
+
+     .. code-block:: c
+
+        int main() { printf("Hello world\n"); return 0 }
+
+  * KEYWORD indicates which syntax highlighting to use (e.g., ``c``,
+    ``c++`` ``make``, ``sh``, ``ini``, ``Fortran``, ``diff``,
+    ``python``, ``java``, ``rst``, ... etc.).
+  * KEYWORD can be omitted if no specific highlighting is to be
+    used.
+  * There *MUST* be a blank line after the ``code-block`` line.
+  * The lines in the block must be indented to the same column as the
+    first ``c`` in ``code-block``.  For example:
+
+    .. code-block:: rst
+
+       .. code-block:: sh
+
+          shell$ tar xf openmpi-<version>.tar.bz2
+          shell$ cd openmpi-<version>
+          shell$ ./configure --prefix=<path> |& tee config.out
+
+    Note that the code block will be rendered at the same level as
+    where the first ``.`` of ``.. code-block::`` starts.  In this
+    case, the example code block will be rendered in the bulleted
+    item.
+
+Whereas this parargraph and code block will be outside of the
+above bulleted list:
+
+.. code-block:: sh
+
+   shell$ tar xf openmpi-<version>.tar.bz2
+   shell$ cd openmpi-<version>
+   shell$ ./configure --prefix=<path> |& tee config.out
+
+   # Fun note: the code-block can contain blank lines.
+
+The code-block is terminated by a blank line and then outdent back
+to the same level as the first ``.`` in ``.. code-block::``.
+
+Un-numbered bullets
+-------------------
+
+* MD: Start lines with ``*`` or ``-``
+* RST: Start lines with ``*``.  You can wrap lines at the same
+  indenting level to make paragraphs in the same bullet.
+
+  Having a blank line and then more text at the same indenting level
+  makes another paragraph in the same bullet.  You even put other
+  directives in this same indenting level.
+
+  * For example, you can start a sub bullet.
+
+    This text is the next paragraph in the same sub bullet.
+
+    .. code-block:: none
+
+       This is a verbatim code block within this same sub bullet.
+       More about code-blocks below.
+
+    This is the next paragraph (after the code block) in the same
+    sub bullet.
+
+  * If you start a new bullet, that terminates the previous bullet.
+
+  * You **MUST** put blank lines between bullets!
+
+Numbered bullets:
+-----------------
+
+* MD: Start lines with ``#``
+* RST: Start lines with ``#.``
+
+  .. important:: Yes, the trailing ``.`` is important
+
+  For example:
+
+  .. code-block:: rst
+
+     #. Item number 1
+     #. The second item
+     #. A third item
+
+  All the same rules for indentation apply as described above.
+
+Comments
+--------
+
+* MD: Enclose content in ``<!--`` and ``-->`` (i.e., HTML comments,
+  but they are included in the output)
+* RST: Start a line with two periods and a space.
+
+  For example, the following block is a comment, and will not be
+  included in the output:
+
+  .. code-block:: rst
+
+     .. Hello world.  This is a comment.  This whole block is a
+        comment.  You can leave it here in the final document, and it
+        will not be included in the rendered output.
+
+        Your comment can even include blank lines.  You terminate a
+        comment |mdash| just like most other things in RST |mdash| by a blank
+        line and then outdenting back out to the same column as the
+        first ".".
+
+     This line is no longer part of the comment.
+
+Including files
+---------------
+
+* MD: You cannot include files in Markdown.
+* RST: Use the ``.. include:: FILENAME`` directive.  For example:
+
+  .. code-block:: rst
+
+     .. include:: features-extensions.rst
+     .. include:: features-java.rst
+
+  Those directives include those 2 files right here in this RST file.
+
+  .. important:: Chapter/section/subsection delimiters will be
+                 continued in those files as part of rendering this
+                 file.
+
+Hyperlinks to URLs
+------------------
+
+* MD:
+
+  .. code-block:: md
+
+     [this is the link text](https://example.com/)
+
+* RST:
+
+  .. code-block:: rst
+
+     `this is the link text <https://example.com/>`_
+
+  .. important:: Yes, the trailing underscore in RST is important.
+                 It's a little weird, but you'll cope.
+
+Hyperlinks to anchors
+---------------------
+
+* MD: I forget offhand how to make anchors and links to them in MD.
+* RST: Use the ``:ref:`` directive.
+
+  Make an anchor like this:
+
+  .. code-block:: rst
+
+     .. _ANCHOR_NAME:
+
+  It *must* start with and underscore and end with a colon.
+
+  I've typically used anchor names that either begin with ``label-``
+  or end in ``-label`` to make it blatantly obvious that it's a
+  label. For example:
+
+  .. code-block:: rst
+
+     .. _building-and-installing-section-label:
+
+  Then you can use the ``:ref:`` directive:
+
+  .. code-block:: rst
+
+     be sure to see :ref:`the VPATH build section
+     <building-and-installing-section-label>`.
+
+Hyperlinks to other (RST) pages
+-------------------------------
+
+* MD:
+
+  .. code-block:: md
+
+     [link text](page_name)
+
+* RST: Use the ``:doc:`` directive.
+
+  General format:
+
+  .. code-block:: rst
+
+     :doc:`link text <PAGE_PATH>`
+
+  For example:
+
+  .. code-block:: rst
+
+     You should read :doc:`the Developer's Guide </developers>`.
+
+  The page path is relative to the ``docs`` dir in the OMPI git tree.
+
+Macros
+------
+
+* MD: There are no macros in Markdown.
+* RST: We have defined a few OMPI-specific macros in RST.  You can
+  insert these macros anywhere in RST content text.
+
+  * ``|ompi_ver|`` is the full Open MPI version number, including
+    alpha/beta/rc/greek denotation.  For example ``5.0.0rc1``.
+
+  * ``|ompi_series|`` is the major/minor Open MPI version, e.g.,
+    ``5.0.x``.
+
+    .. important:: Think twice about hard-coding the Open MPI version
+                   number or series when referring to the current
+                   version or series.  It can be appropriate to
+                   hard-code an "x.y.0" version to denote a
+                   generational epoch, but in most other cases, you
+                   probably want to use one of the macros.
+
+  * ``|mdash|`` is a unicode long dash, an "em" dash.  Use it instead
+    of ``--``.
+
+  * ``|rarrow|`` is a unicode right arrow.  Use it instead of ``->``
+    or ``-->``.
+
+Brightly-colored boxes
+----------------------
+
+* MD: There are no brightly-colored boxes in MD.
+
+* RST: You can use various directives to make brightly-colored
+  "note" boxes (Called admonitions) in RST.  For example:
+
+  .. important:: a green box with a "!" icon
+
+     Standard indenting rules apply for the content in the box.  You
+     can have multiple lines and multiple paragraphs, for example.
+
+     Yippee.
+
+     * You can even have bullets.
+
+       .. code-block:: none
+
+          You can even have code blocks inside the bullet inside the
+          caution box.
+
+     * All the standard indenting rules apply.
+
+  .. hint:: a green box with a "!" icon
+
+  .. note:: a blue box with a "!" icon
+
+  .. caution:: an orange box with a "!" icon
+
+  .. attention:: an orange box with a "!" icon
+
+  .. warning:: an orange box with a "!" icon
+
+  .. error:: a red box with a "!" icon
+
+  .. danger:: a red box with a "!" icon
+
+  .. admonition:: Custom title
+     :class: tip
+
+     You can name this box whatever you want:
+
+     .. code-block:: rst
+
+        .. admonition:: Custom title
+           :class: tip
+
+           Content of your box here.
+
+     Custom text for this custom admonition.  Note that the ``:class: <type>``
+     will change the coloring to the color for the basic admonition of that
+     type.  E.g., ``:class: tip`` makes the box be green.
diff --git a/docs/developers/source-code.rst b/docs/developers/source-code.rst
new file mode 100644
index 00000000000..a2108e62446
--- /dev/null
+++ b/docs/developers/source-code.rst
@@ -0,0 +1,295 @@
+Source code
+===========
+
+Code style
+----------
+
+We intentionally do not have too many code conventions in the Open MPI
+code base.
+
+All languages
+^^^^^^^^^^^^^
+
+* 4 space tabs.  No more, no less.
+* **NEVER** use actual tab characters; always use spaces.  Both emacs
+  and vim have secret mojo that can automatically use spaces when you
+  hit the ``<TAB>`` key.  This makes the code look the same in every
+  browser, regardless of individual tab display settings.
+
+C / C++
+^^^^^^^
+
+* When comparing constants for equality or inequality, always put the
+  constant on the left.  This is defensive programming: if you have a
+  typo in the test and miss a ``!`` or ``=``, you'll get a compiler error.
+  For example:
+
+  .. code-block:: c
+
+     /* Do this */
+     if (NULL == foo) { ... }
+
+     /* Because if you have a typo (i.e., = instead of ==), this will
+        be a compile error rather than a subtle bug */
+     if (NULL = foo) { ... }
+
+* More defensive programming: *always* include blocks in curly braces
+  ``{ }``, even if they are only one line long.  For example:
+
+  .. code-block:: c
+
+     /* Do this */
+     if (whatever) {
+         return OMPI_SUCCESS;
+     }
+
+     /* Not this */
+     if (whatever)
+        return OMPI_SUCCESS;
+
+* Starting with Open MPI 6.0, Open MPI requires a C11-compliant
+  compiler.
+
+  * C++-style comments are now allowed (and preferred).
+  * C99-style mixing declarations are allow allowable (and preferred).
+
+* **ALWAYS** include ``<level>_config.h`` as your first #include file,
+  where ``<level>`` is one of ``ompi``, ``oshmem``, or ``opal`` |mdash| the
+  level that you're writing in.  There are very, very few cases where
+  this is not true (E.g., some bizarre Windows scenarios).  But in
+  99.9999% of cases, this file should be included **first** so that it
+  can affect system-level #include files if necessary.
+* Filenames and symbols must follow the **prefix rule** (see [e-mail
+  thread](http://www.open-mpi.org/community/lists/devel/2009/07/6389.php)):
+
+  * Filenames must be prefixed with ``<framework>_<component>``.
+  * Public symbols must be prefixed in components with
+    ``<project>_<framework>_<component>``, where ``<project>`` is one
+    of ``mca``, ``ompi``, ``oshmem``, or ``opal``.  Note that `mca`
+    used to be the most common, but it has fallen out of favor
+    compared to the other ``<project>`` prefixes.  When in doubt about
+    whether a symbol is public, be safe and add the prefix.
+  * Non-public symbols must be declared ``static`` or otherwise made to
+    not appear in the global scope.
+
+* **ALWAYS** #define macros, even for logical values.
+
+  * The GNU Way is to ``#define`` a macro when it is "true" and to
+    ``#undef`` it when it is "false".
+  * In Open MPI, we **always** ``#define`` a logical macro to be
+    either 0 or 1 |mdash| we never ``#undef`` it.
+  * The reason for this is defensive programming: if you are only
+    checking if a preprocessor macro is defined (via ``#ifdef FOO`` or
+    ``#if defined(FOO)``), you will get no warning when compiling if
+    you accidentally misspell the macro name.  However, if you use the
+    logic test ``#if FOO`` with an undefined macro (e.g., because you
+    misspelled it), you'll get a compiler warning or error.
+
+    .. admonition:: Rationale
+       :class: tip
+
+       Misspelled macro names can be tremendously difficult to find
+       when they are buried in thousands of lines of code; we will
+       take all the help from the preprocessor/compiler that we can
+       get!
+
+  .. code-block:: c
+
+     /* GNU Way - you will get no warning from the compiler if you
+        misspell "FOO"; the test will simply be false */
+     #ifdef FOO
+     ...
+     #else
+     ...
+     #endif
+
+     /* Open MPI Way - you will get a warning from the compiler if you
+        misspell "FOO"; the result of the test is a different value
+        than whether you spelled the macro name right or not */
+     #if FOO
+     ...
+     #else
+     ...
+     #endif
+
+Fortran
+^^^^^^^
+
+We do not have specific coding style guidelines for Fortran.  Please
+read some of the existing Fortran code in the source code tree and try
+to use a similar style.
+
+Shell scripting
+^^^^^^^^^^^^^^^
+
+Please read some of the existing shell code in the source code tree
+and try to use a similar style.
+
+* Always enclose evaluated shell variables in quotes to ensure that
+  multi-token values are handled properly.
+
+  .. code-block:: sh
+
+     # This is bad
+     if test $foo = bar; then
+
+     # This is good
+     if test "$foo" = "bar"; then
+
+  * The one exception to this is that when doing an assignment to a
+    shell variable from another shell variable, it is not necessary to
+    use quotes on the right hand side:
+
+    .. code-block:: sh
+
+       # This is harmless, but unnecessary
+       foo="$bar"
+
+       # This is actually sufficient, even for multi-token values of $bar
+       foo=$bar
+
+* Do not use the ``==`` operator for ``test`` |mdash| this is a GNU
+  extension and can cause portability problems on BSD systems.
+  Instead, use the single ``=`` operator.
+
+  .. code-block:: sh
+
+     # This is bad
+     if test "$foo" == "bar"; then
+
+     # This is good
+     if test "$foo" = "bar"; then
+
+m4
+^^^
+
+We do not have specific coding style guidelines for m4 (the language
+used to create the ``configure`` script).  Please read some of the
+existing m4 code in the source code tree and try to use a similar
+style.
+
+Tree layout
+-----------
+
+There are a few notable top-level directories in the source
+tree:
+
+* The main sub-projects:
+
+    * ``oshmem``: Top-level OpenSHMEM code base
+    * ``ompi``: The Open MPI code base
+    * ``opal``: The OPAL code base
+
+* ``config``: M4 scripts supporting the top-level ``configure`` script
+  ``mpi.h``
+* ``etc``: Some miscellaneous text files
+* ``docs``: Source code for Open MPI documentation
+* ``examples``: Trivial MPI / OpenSHMEM example programs
+* ``3rd-party``: Included copies of required core libraries (either
+  via Git submodules in Git clones or via binary tarballs).
+
+  .. note:: While it may be considered unusual, we include binary
+     tarballs (instead of Git submodules) for 3rd party projects that
+     are:
+
+     #. Needed by Open MPI for correct operation, and
+     #. Not universally included in OS distributions, and
+     #. Rarely updated.
+
+Each of the three main source directories (``oshmem``, ``ompi``, and
+``opal``) generate at least a top-level library named ``liboshmem``,
+``libmpi``, and ``libopen-pal``, respectively.  They can be built as
+either static or shared libraries.  Executables are also produced in
+subdirectories of some of the trees.
+
+The ``libopen-pal`` top-level library is built internally in two parts:
+
+* ``libopen-pal_core`` Internal "core" portion of OPAL containing the essential source and MCA needed for tools like mpicc/mpirun to link against. The "core" library is not installed.
+
+   * Includes the following MCA frameworks: ``backtrace``, ``dl``, ``installdirs``,  ``threads``,  ``timer``
+   * Includes all of the source under ``opal/class`` and most of ``opal/util``
+   * Includes the files suffixed with ``_core`` in ``opal/runtime``
+
+* ``libopen-pal``  Includes "core" plus all of the other OPAL project sources. This is installed.
+
+Each of the sub-project source directories have similar (but not
+identical) directory structures under them:
+
+* ``class``: C++-like "classes" (using the OPAL class system)
+  specific to this project
+* ``include``: Top-level include files specific to this project
+* ``mca``: MCA frameworks and components specific to this project
+* ``runtime``: Startup and shutdown of this project at runtime
+* ``tools``: Executables specific to this project
+* ``util``: Random utility code
+
+There are other top-level directories in each of the sub-projects,
+each having to do with specific logic and code for that project.  For
+example, the MPI API implementations can be found under
+``ompi/mpi/LANGUAGE``, where ``LANGUAGE`` is ``c``, ``fortran``, or
+``java``.
+
+The layout of the ``mca`` trees are strictly defined.  They are of the
+form:
+
+.. code-block:: text
+
+    PROJECT/mca/FRAMEWORK/COMPONENT
+
+To be explicit: it is forbidden to have a directory under the ``mca``
+trees that does not meet this template (with the exception of ``base``
+directories, explained below).  Hence, only framework and component
+code can be in the ``mca`` trees.
+
+That is, framework and component names must be valid directory names
+(and C variables; more on that later).  For example, the TCP BTL
+component is located in ``opal/mca/btl/tcp/``.
+
+The name ``base`` is reserved; there cannot be a framework or component
+named ``base``. Directories named ``base`` are reserved for the
+implementation of the MCA and frameworks.  Here are a few examples (as
+of the |ompi_series| source tree):
+
+.. code-block:: sh
+
+    # Main implementation of the MCA
+    opal/mca/base
+
+    # Implementation of the btl framework
+    opal/mca/btl/base
+
+    # Implementation of the sysv framework
+    oshmem/mcs/sshmem/sysv
+
+    # Implementation of the pml framework
+    ompi/mca/pml/base
+
+Under these mandated directories, frameworks and/or components may have
+arbitrary directory structures, however.
+
+Symbol Visibility
+-----------------
+
+The ``*_DECLSPEC`` macros provide a method to annotate symbols to indicate
+their intended visibility when compiling dynamically shared object files
+(e.g., ``libmpi.so``).  The macros are defined on a per project basis:
+
+* Open MPI: ``OMPI_DECLSPEC``
+* Open PAL: ``OPAL_DECLSPEC``
+* OpenSHMEM: ``OSHMEM_DECLSPEC``
+
+The macros expand to the appropriate compiler and platform flags for marking
+whether a symbol should be explicitly made public in the target project's
+library namespace.
+The ``*_DECLSPEC`` attributes are used to declare that a symbol is to be
+visible outside of that library/DSO's scope.  For example, ``OMPI_DECLSPEC``
+is used to control what symbols are visible in the ``libmpi.so`` scope.
+
+.. note:: This is entirely related to dynamic library compilation and does not
+   apply to static compilation.
+
+.. note:: The macros were originally introduced when Open MPI supported
+   Windows (circa Open MPI v1.0.0) and are motivated by the Windows
+   `__declspec <https://docs.microsoft.com/en-us/cpp/cpp/declspec?view=msvc-170>`_.
+   While support for Windows has been dropped from Open MPI, the symbol
+   visibility macros remain.
diff --git a/docs/developers/sphinx.rst b/docs/developers/sphinx.rst
new file mode 100644
index 00000000000..8587d46e6f9
--- /dev/null
+++ b/docs/developers/sphinx.rst
@@ -0,0 +1,242 @@
+.. _developers-installing-sphinx-label:
+
+Installing and running Sphinx (building the Open MPI docs)
+==========================================================
+
+As with all content in the Developer's Guide, this section is only
+relevant for developers who work in the Open MPI code base itself.
+End users who install a binary Open MPI package or build from an
+official Open MPI distribution tarball do not need to have Sphinx
+installed.
+
+Installing Python
+-----------------
+
+The `Sphinx tool <https://www.sphinx-doc.org/>`_ is written in Python,
+and therefore needs to have Python available.  As of late 2022, Sphinx
+requires Python >= v3.7.
+
+This documentation does not contain detailed instructions for
+installing a Python version sufficient for using Sphinx.  Consult your
+local OS documentation for how to obtain Python >= v3.7, or search the
+internet for further information.
+
+Installing Sphinx
+-----------------
+
+`The Sphinx documentation
+<https://www.sphinx-doc.org/en/master/usage/installation.html>`_
+recommends installing Sphinx (and its required Python dependencies)
+via ``pip``, which typically requires connectivity to the general
+internet.
+
+.. note:: If you are running on MacOS, you may be tempted to use
+   Homebrew or MacPorts to install Sphinx.  The Sphinx documentation
+   recommends **against** this.  Instead, you should use ``pip`` to
+   install Sphinx.
+
+There are three general ways to install Sphinx; you only need one of
+them.
+
+Install Sphinx in a Python virtual environment
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The preferred method of installing Sphinx for Open MPI documentation
+development is to install Sphinx in a Python virtual environment.
+This places Sphinx in a sandbox that will not conflict with other
+``pip``-installed Python modules.  This example installs Sphinx and
+other Python modules in the ``ompi-docs-venv`` tree under your Open
+MPI Git clone directory:
+
+.. code-block:: sh
+
+   # Create the Python virtual environment
+   shell$ cd TOP_OF_OPEN_MPI_GIT_CLONE
+   shell$ python3 -m venv ompi-docs-venv
+   # Or: python3 -m virtualenv ompi-docs-venv
+   # Or: virtualenv --python=python3 ompi-docs-venv
+
+   # Activate the virtual environment
+   shell$ . ./ompi-docs-venv/bin/activate
+
+   # Notice that the shell prompt changes
+   # Now install the required Python modules
+   (ompi-docs-venv) shell$ pip3 install -r docs/requirements.txt
+   # Or: python3 -m pip install install -r docs/requirements.txt
+
+Note that sourcing the ``activate`` script will change your prompt to
+put the name of your virtual environment directory at the front, just
+as a visual reminder that you are operating in a Python virtual
+environment.  You can run ``deactivate`` to leave the virtual
+environment.
+
+.. important:: You will need to source the ``activate`` script to put
+               Sphinx in your ``PATH`` *before* running ``configure``.
+
+Install Sphinx globally
+^^^^^^^^^^^^^^^^^^^^^^^
+
+If Python virtual environments are not desirable on your system, you
+can install Sphinx globally on your system (you may need to run with
+root privileges):
+
+.. code-block:: sh
+
+   shell$ cd TOP_OF_OPEN_MPI_GIT_CLONE
+   shell$ pip3 install -r docs/requirements.txt
+   # Or: python3 -m pip install install -r docs/requirements.txt
+
+This will install Sphinx and some Python modules required for building
+the Open MPI documentation in a system-wide location.
+
+This will likely install the ``sphinx-build`` executable in a location
+that is already in your ``PATH``.  If the location is not already in
+your ``PATH``, then you need to add it to your ``PATH``.
+
+.. important:: You will need to ensure that Sphinx is in your ``PATH``
+               *before* running ``configure``.
+
+
+Install Sphinx locally
+^^^^^^^^^^^^^^^^^^^^^^
+
+If you cannot or do not want to install Sphinx globally on your
+system, the following will install Sphinx somewhere under your
+``$HOME``.  It is the same ``pip`` command as shown above, but with
+the addition of the ``--user`` flag (you should not need ``root``
+permissions to run this command):
+
+.. code-block:: sh
+
+   shell$ cd TOP_OF_OPEN_MPI_GIT_CLONE
+   shell$ pip3 install --user -r docs/requirements.txt
+   # Or: python3 -m pip install install -r docs/requirements.txt
+
+This will install Sphinx and some Python modules required for building
+the Open MPI documentation in a user-specific location, likely
+somewhere under ``$HOME``
+
+You will likely need to find the location where ``sphinx-build`` was
+installed and add it to your ``PATH`` (e.g., on macOS, it might appear
+under ``$HOME/Library/Python/PYTHON_VERSION/bin/sphinx-build``).
+
+.. note:: On MacOS, look for ``sphinx-build`` under
+          ``$HOME/Library/Python/VERSION/bin`` (where ``VERSION`` is
+          the version number of Python).  Or it may have installed to
+          ``/usr/local/bin/sphinx-build``.  YMMV.
+
+.. important:: You will need to ensure that Sphinx is in your ``PATH``
+               *before* running ``configure``.
+
+Running Sphinx
+--------------
+
+Open MPI's build environment is setup to invoke Sphinx automatically;
+you should not need to invoke Sphinx manually.
+
+.. important:: You will need to ensure that Sphinx is in your ``PATH``
+               *before* running ``configure``.
+
+As long as ``configure`` found Sphinx, ``make`` will invoke Sphinx to
+build the documentation.  You can also run ``make`` directly in the
+``docs/`` directory to build *just* the docs and skip building the
+rest of the Open MPI software.  This can be a huge time-saver when
+iteratively writing, rendering, and viewing/proofing documentation.
+
+.. note:: The fully-built HTML and man page docs are included in
+          official Open MPI distribution tarballs.  Meaning: if you
+          download an Open MPI tarball from
+          https://www.open-mpi.org/software/ompi (version v5.0.0 or
+          later), the pre-built HTML and man page files are included
+          in the tarball.
+
+          Sphinx is a requirement for *developers* who want to build
+          the Open MPI docs.  End users do *not* need to have Sphinx
+          available to build Open MPI or have its docs installed from
+          an official distribution tarball.
+
+Sphinx execution time
+^^^^^^^^^^^^^^^^^^^^^
+
+The first time you invoke Sphinx on a clean tree, it takes a little
+time to render all the docs.
+
+However, Sphinx is stateful: subsequent runs can be significantly
+faster because Sphinx will only re-render HTML files that have
+changes.  This is an enormous time saver for Open MPI (e.g., if you
+are iterating over writing the docs and running ``make`` to see how
+they rendered in HTML).
+
+.. caution:: Sphinx is only *somewhat* smart in its partial
+             re-rendering.  If you change a title in an RST file, for
+             example, Sphinx will (by default) only re-render *that*
+             page.  The Tables of Contents / left hand navigation on
+             other pages may not be updated.
+
+             You can always force a full re-render via:
+
+             .. code:: sh
+
+                shell$ cd docs
+                shell$ rm -rf _build
+                shell$ make
+
+Checking Sphinx HTML links
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+``make linkcheck`` will invoke Sphinx's functionality to check all the
+external links in the documentation:
+
+.. code:: sh
+
+   shell$ cd docs
+   shell$ make linkcheck
+
+.. important:: You will need to be on a computer that has good access
+               to the internet when running this command.
+
+Viewing docs locally
+^^^^^^^^^^^^^^^^^^^^
+
+Once you have built the docs in your local Git clone, you can view
+them locally in the build tree:
+
+#. Open ``docs/_build/html/index.html`` in a browser to view the HTML
+   docs.  For example, on MacOS, the following command opens the build
+   tree docs in the default web browser:
+
+   .. code:: sh
+
+      shell$ open docs/_build/html/index.html
+
+#. Use the ``man`` command to view the Nroff files in
+   ``docs/_build/man`` (you may need to use an absolute or relative
+   filename to prevent ``man`` from using its search paths).  For
+   example:
+
+   .. code:: sh
+
+      shell$ cd docs/_build/man
+      shell$ man ./MPI_Send.3
+
+Alternatively, you can view these files in their installed locations
+after running ``make install``:
+
+#. The HTML docs are installed (by default) to
+   ``$prefix/share/doc/openmpi/html``.  You can use a web browser to
+   open the ``index.html`` in that directory to view the docs locally.
+   For example, on MacOS, the following command opens the installed
+   docs in the default web browser:
+
+   .. code:: sh
+
+      shell$ open $prefix/share/doc/openmpi/html/index.html
+
+#. The man pages are installed (by default) to ``$preix/share/man``.
+   If your man page search path includes this location, you can invoke
+   commands similar to the following to see the same content that you
+   see in these HTML pages:
+
+   .. code:: sh
+
+      shell$ man MPI_Send
diff --git a/docs/developers/terminology.rst b/docs/developers/terminology.rst
new file mode 100644
index 00000000000..2e31c784f62
--- /dev/null
+++ b/docs/developers/terminology.rst
@@ -0,0 +1,81 @@
+Open MPI terminology
+====================
+
+Open MPI is a large project containing many different sub-systems and
+a relatively large code base.  Let's first cover some fundamental
+terminology in order to make the rest of the discussion easier.
+
+Modular Component Architecture (MCA)
+------------------------------------
+
+:ref:`See this section <label-mca>` for a discussion of the Modular
+Component Architecture (MCA).  Seriously.  Go read it now.  From
+reading that section, you should understand the following terms before
+continuing reading these docs:
+
+* Project
+* Framework
+* Component
+* Module
+* Parameters (variables)
+
+Notes on projects
+-----------------
+
+Projects are strict abstraction barriers in the code.  That is, they
+are compiled into separate libraries: ``liboshmem``, ``libmpi``,
+``libopen-pal`` with a strict dependency order: OSHMEM depends on
+OMPI, OMPI depends on OPAL.  For example, MPI executables are linked
+with:
+
+.. code-block:: sh
+
+   shell$ mpicc myapp.c -o myapp
+   # This actually turns into:
+   shell$ cc myapp.c -o myapp -lmpi ...
+
+More system-level libraries may listed after ``-lmpi``, but you get
+the idea.  ``libmpi`` will implicitly pull ``libopen-pal`` into the
+overall link step.
+
+Strictly speaking, these are not "layers" in the classic software
+engineering sense (even though it is convenient to refer to them as
+such).  They are listed above in dependency order, but that does not
+mean that, for example, the OMPI code must go through the OPAL code in
+order to reach the operating system or a network interface.
+
+As such, this code organization more reflects abstractions and
+software engineering, not a strict hierarchy of functions that must be
+traversed in order to reach a lower layer.  For example, OMPI can
+directly call the operating system as necessary (and not go through
+OPAL).  Indeed, many top-level MPI API functions are quite performance
+sensitive; it would not make sense to force them to traverse an
+arbitrarily deep call stack just to move some bytes across a network.
+
+Frameworks, components, and modules can be dynamic or static. That is,
+they can be available as plugins or they may be compiled statically
+into libraries (e.g., ``libmpi``).
+
+In Open MPI |ompi_ver|, ``configure`` defaults to:
+
+* Building projects as dynamic libraries
+* Linking all components into their parent project libraries
+  (vs. compiling them as independent DSOs)
+
+Although these defaults can be modified by :doc:`command line
+arguments to configure
+</installing-open-mpi/configure-cli-options/index>`.
+
+Required 3rd party libraries
+----------------------------
+
+Note that Open MPI also uses some third-party libraries for core
+functionality:
+
+* PMIx
+* PRRTE
+* Libevent
+* Hardware Locality ("hwloc")
+
+These are discussed in detail in the :ref:`required support libraries
+section <label-install-required-support-libraries>`.
diff --git a/docs/features/extensions.rst b/docs/features/extensions.rst
new file mode 100644
index 00000000000..592d56e7687
--- /dev/null
+++ b/docs/features/extensions.rst
@@ -0,0 +1,91 @@
+.. _ompi-features-extentions-label:
+
+Open MPI extensions
+===================
+
+Open MPI contains a framework for extending the MPI API that is
+available to applications.  Each extension is usually a standalone set
+of functionality that is distinct from other extensions (similar to
+how Open MPI's plugins are usually unrelated to each other).  These
+extensions provide new functions and/or constants that are available
+to MPI applications.
+
+.. warning:: These extensions are neither standard nor portable to
+   other MPI implementations!
+
+   They are a mechanism for the Open MPI developer community to
+   provide new functionality to users, typically before it becomes
+   standardized by the MPI Forum.
+
+Available extensions
+--------------------
+
+The following extensions are included in this version of Open MPI:
+
+#. ``shortfloat``: Provides MPI datatypes ``MPIX_C_FLOAT16``,
+   ``MPIX_SHORT_FLOAT``, ``MPIX_SHORT_FLOAT``, and
+   ``MPIX_CXX_SHORT_FLOAT_COMPLEX`` if corresponding language types are
+   available. See ``ompi/mpiext/shortfloat/README.txt`` for details.
+#. ``affinity``: Provides the ``OMPI_Affinity_str()`` API, which returns
+   a string indicating the resources which a process is bound. For
+   more details, see its man page.
+#. ``cuda``: When the library is compiled with CUDA-aware support, it
+   provides two things.  First, a macro
+   ``MPIX_CUDA_AWARE_SUPPORT``. Secondly, the function
+   ``MPIX_Query_cuda_support()`` that can be used to query for support.
+#. ``example``: A non-functional extension; its only purpose is to
+   provide an example for how to create other extensions.
+#. ``ftmpi``: An implementation of the User Level Fault Mitigation
+   (ULFM) proposal.  :ref:`See its documentation section <ulfm-label>`
+   for more details.
+
+Compiling the extensions
+------------------------
+
+Open MPI extensions are all enabled by default; they can be disabled
+via the ``--disable-mpi-ext`` command line switch.
+
+Since extensions are meant to be used by advanced users only, this
+file does not document which extensions are available or what they do.
+Look in the ``ompi/mpiext`` directory in a distribution Open MPI
+tarball to see the extensions; each subdirectory of that directory
+contains an extension.  Each has a ``README`` file that describes what
+it does.
+
+Using the extensions
+--------------------
+
+To reinforce the fact that these extensions are non-standard, you must
+include a separate header file after ``<mpi.h>`` to obtain the function
+prototypes, constant declarations, etc.  For example:
+
+.. code-block:: c
+
+   #include <mpi.h>
+   #if defined(OPEN_MPI) && OPEN_MPI
+   #include <mpi-ext.h>
+   #endif
+
+   int main() {
+       MPI_Init(NULL, NULL);
+
+   #if defined(OPEN_MPI) && OPEN_MPI
+       char ompi_bound[OMPI_AFFINITY_STRING_MAX];
+       char current_binding[OMPI_AFFINITY_STRING_MAX];
+       char exists[OMPI_AFFINITY_STRING_MAX];
+
+       OMPI_Affinity_str(OMPI_AFFINITY_LAYOUT_FMT, ompi_bound,
+                         current_bindings, exists);
+   #endif
+
+       MPI_Finalize();
+       return 0;
+   }
+
+Notice that the Open MPI-specific code is surrounded by the ``#if``
+statement to ensure that it is only ever compiled by Open MPI.
+
+The Open MPI wrapper compilers (``mpicc`` and friends) should
+automatically insert all relevant compiler and linker flags necessary
+to use the extensions.  No special flags or steps should be necessary
+compared to "normal" MPI applications.
diff --git a/docs/features/index.rst b/docs/features/index.rst
new file mode 100644
index 00000000000..3afb117dabb
--- /dev/null
+++ b/docs/features/index.rst
@@ -0,0 +1,17 @@
+Open MPI-specific features
+==========================
+
+Open MPI has multiple features that are above and beyond what is
+specified in the `MPI Standard <https://www.mpi-forum.org/docs/>`_.
+
+This section does not list *all* Open MPI features that are not
+specified in the official MPI Standard, but does list two major
+categories of Open MPI-specific features.
+
+.. toctree::
+   :maxdepth: 1
+
+   profiling
+   extensions
+   ulfm
+   java
diff --git a/docs/features/java.rst b/docs/features/java.rst
new file mode 100644
index 00000000000..81fedd72bb7
--- /dev/null
+++ b/docs/features/java.rst
@@ -0,0 +1,350 @@
+.. _open-mpi-java-label:
+
+Open MPI Java bindings
+======================
+
+Open MPI |ompi_ver| provides support for Java-based MPI applications.
+
+.. warning:: The Open MPI Java bindings are provided on a
+   "provisional" basis |mdash| i.e., they are not part of the current or
+   proposed MPI standards.  Thus, inclusion of Java support is not
+   required by the standard.  Continued inclusion of the Java bindings
+   is contingent upon active user interest and continued developer
+   support.
+
+The rest of this document provides step-by-step instructions on
+building OMPI with Java bindings, and compiling and running Java-based
+MPI applications. Also, part of the functionality is explained with
+examples. Further details about the design, implementation and usage
+of Java bindings in Open MPI can be found in its canonical reference
+paper [#ompijava]_. The bindings follow a JNI approach, that is, we do
+not provide a pure Java implementation of MPI primitives, but a thin
+layer on top of the C implementation. This is the same approach as in
+mpiJava [#mpijava]_; in fact, mpiJava was taken as a starting point
+for Open MPI Java bindings, but they were later totally rewritten.
+
+Building the Java bindings
+--------------------------
+
+Java support requires that Open MPI be built at least with shared
+libraries (i.e., ``--enable-shared``).  Note that this is the default
+for Open MPI, so you don't have to explicitly add the option. The Java
+bindings will build only if ``--enable-mpi-java`` is specified, and a
+JDK is found in a typical system default location.
+
+If the JDK is not in a place where we automatically find it, you can
+specify the location. For example, this is required on the Mac
+platform as the JDK headers are located in a non-typical location. Two
+options are available for this purpose:
+
+#. ``--with-jdk-bindir=<foo>``: the location of ``javac`` and ``javah``
+#. ``--with-jdk-headers=<bar>``: the directory containing ``jni.h``
+
+Some example configurations are provided in Open MPI configuration
+platform files under ``contrib/platform/hadoop``. These examples can
+provide a starting point for your own custom configuration.
+
+In summary, therefore, you can configure the system using the
+following Java-related options::
+
+  $ ./configure --with-platform=contrib/platform/hadoop/<your-platform> ...
+
+or::
+
+  $ ./configure --enable-mpi-java --with-jdk-bindir=<foo> --with-jdk-headers=<bar> ...
+
+or simply::
+
+  $ ./configure --enable-mpi-java ...
+
+if JDK is in a "standard" place that ``configure`` can automatically
+find.
+
+Building Java MPI applications
+------------------------------
+
+The ``mpijavac`` wrapper compiler is available for compiling
+Java-based MPI applications. It ensures that all required Open MPI
+libraries and classpaths are defined.  For example:
+
+.. code-block::
+
+   $ mpijavac Hello.java
+
+You can use the ``--showme`` option to see the full command line of
+the Java compiler that is invoked:
+
+.. code-block::
+
+   $ mpijavac Hello.java --showme
+   /usr/bin/javac -cp /opt/openmpi/lib/mpi.jar Hello.java
+
+Note that if you are specifying a ``-cp`` argument on the command line
+to pass your application-specific classpaths, Open MPI will *extend*
+that argument to include the ``mpi.jar``:
+
+.. code-block::
+
+   $ mpijavac -cp /path/to/my/app.jar Hello.java --showme
+   /usr/bin/javac -cp /path/to/my/app.jar:/opt/openmpi/lib/mpi.jar Hello.java
+
+Similarly, if you have a ``CLASSPATH`` environment variable defined,
+``mpijavac`` will convert that into a ``-cp`` argument and extend it
+to include the ``mpi.jar``:
+
+.. code-block::
+
+   $ export CLASSPATH=/path/to/my/app.jar
+   $ mpijavac Hello.java --showme
+   /usr/bin/javac -cp /path/to/my/app.jar:/opt/openmpi/lib/mpi.jar Hello.java
+
+
+Running Java MPI applications
+-----------------------------
+
+Once your application has been compiled, you can run it with the
+standard ``mpirun`` command line::
+
+  $ mpirun <options> java <your-java-options> <my-app>
+
+``mpirun`` will detect the ``java`` token and ensure that the required
+MPI libraries and class paths are defined to support execution. You
+therefore do **not** need to specify the Java library path to the MPI
+installation, nor the MPI classpath. Any classpath definitions
+required for your application should be specified either on the
+command line or via the ``CLASSPATH`` environment variable. Note that
+the local directory will be added to the classpath if nothing is
+specified.
+
+.. note:: The ``java`` executable, all required libraries, and your
+          application classes must be available on all nodes.
+
+Basic usage of the Java bindings
+--------------------------------
+
+There is an MPI package that contains all classes of the MPI Java
+bindings: ``Comm``, ``Datatype``, ``Request``, etc. These classes have a
+direct correspondence with handle types defined by the MPI standard. MPI
+primitives are just methods included in these classes. The convention
+used for naming Java methods and classes is the usual camel-case
+convention, e.g., the equivalent of ``MPI_File_set_info(fh,info)`` is
+``fh.setInfo(info)``, where ``fh`` is an object of the class ``File``.
+
+Apart from classes, the MPI package contains predefined public
+attributes under a convenience class ``MPI``. Examples are the
+predefined communicator ``MPI.COMM_WORLD`` and predefined datatypes such
+as ``MPI.DOUBLE``. Also, MPI initialization and finalization are methods
+of the ``MPI`` class and must be invoked by all MPI Java
+applications. The following example illustrates these concepts:
+
+.. code-block:: java
+
+   import mpi.*;
+
+   class ComputePi {
+
+      public static void main(String args[]) throws MPIException {
+
+          MPI.Init(args);
+
+          int rank = MPI.COMM_WORLD.getRank(),
+              size = MPI.COMM_WORLD.getSize(),
+              nint = 100; // Intervals.
+          double h = 1.0/(double)nint, sum = 0.0;
+
+          for (int i=rank+1; i<=nint; i+=size) {
+              double x = h * ((double)i - 0.5);
+              sum += (4.0 / (1.0 + x * x));
+          }
+
+          double sBuf[] = { h * sum },
+                 rBuf[] = new double[1];
+
+          MPI.COMM_WORLD.reduce(sBuf, rBuf, 1, MPI.DOUBLE, MPI.SUM, 0);
+
+          if (rank == 0) {
+              System.out.println("PI: " + rBuf[0]);
+          }
+          MPI.Finalize();
+      }
+   }
+
+Exception handling
+------------------
+
+The Java bindings in Open MPI support exception handling. By default,
+errors are fatal, but this behavior can be changed. The Java API will
+throw exceptions if the ``MPI.ERRORS_RETURN`` error handler is set:
+
+.. code-block:: java
+
+   MPI.COMM_WORLD.setErrhandler(MPI.ERRORS_RETURN);
+
+If you add this statement to your program, it will show the line
+where it breaks, instead of just crashing in case of an error.
+Error-handling code can be separated from main application code by
+means of try-catch blocks, for instance:
+
+.. code-block:: java
+
+   try
+   {
+       File file = new File(MPI.COMM_SELF, "filename", MPI.MODE_RDONLY);
+   }
+   catch(MPIException ex)
+   {
+       System.err.println("Error Message: "+ ex.getMessage());
+       System.err.println("  Error Class: "+ ex.getErrorClass());
+       ex.printStackTrace();
+       System.exit(-1);
+   }
+
+How to specify buffers
+----------------------
+
+In MPI primitives that require a buffer (either send or receive), the
+Java API admits a Java array. Since Java arrays can be relocated by
+the Java runtime environment, the MPI Java bindings need to make a
+copy of the contents of the array to a temporary buffer, then pass the
+pointer to this buffer to the underlying C implementation. From the
+practical point of view, this implies an overhead associated to all
+buffers that are represented by Java arrays. The overhead is small for
+small buffers but increases for large arrays.
+
+There is a pool of temporary buffers with a default capacity of 64K.
+If a temporary buffer of 64K or less is needed, then the buffer will
+be obtained from the pool. But if the buffer is larger, then it will
+be necessary to allocate the buffer and free it later.
+
+The default capacity of pool buffers can be modified with an Open MPI
+MCA parameter::
+
+  $ mpirun --mca ompi_mpi_java_eager SIZE ...
+
+The value of ``SIZE`` can be:
+
+* ``N``: An integer number of bytes
+* ``Nk``: An integer number (suffixed with ``k``) of kilobytes
+* ``Nm``: An integer number (suffixed with ``m``) of megabytes
+
+An alternative is to use "direct buffers" provided by standard classes
+available in the Java SDK such as ``ByteBuffer``. For convenience,
+Open MPI provides a few static methods ``new[Type]Buffer`` in the
+``MPI`` class to create direct buffers for a number of basic
+datatypes. Elements of the direct buffer can be accessed with methods
+``put()`` and ``get()``, and the number of elements in the buffer can
+be obtained with the method ``capacity()``. This example illustrates
+its use:
+
+.. code-block:: java
+
+   int myself = MPI.COMM_WORLD.getRank();
+   int tasks  = MPI.COMM_WORLD.getSize();
+
+   IntBuffer in  = MPI.newIntBuffer(MAXLEN * tasks),
+             out = MPI.newIntBuffer(MAXLEN);
+
+   for (int i = 0; i < MAXLEN; i++)
+       out.put(i, myself);      // fill the buffer with the rank
+
+   Request request = MPI.COMM_WORLD.iAllGather(
+                     out, MAXLEN, MPI.INT, in, MAXLEN, MPI.INT);
+   request.waitFor();
+   request.free();
+
+   for (int i = 0; i < tasks; i++) {
+       for (int k = 0; k < MAXLEN; k++) {
+           if (in.get(k + i * MAXLEN) != i)
+               throw new AssertionError("Unexpected value");
+       }
+   }
+
+Direct buffers are available for: ``BYTE``, ``CHAR``, ``SHORT``,
+``INT``, ``LONG``, ``FLOAT``, and ``DOUBLE``.
+
+.. note:: There is no direct buffer for booleans.
+
+Direct buffers are not a replacement for arrays, because they have
+higher allocation and deallocation costs than arrays. In some cases
+arrays will be a better choice. You can easily convert a buffer into
+an array and vice versa.
+
+.. important:: All non-blocking methods *must* use direct buffers.
+               Only blocking methods can choose between arrays and
+               direct buffers.
+
+The above example also illustrates that it is necessary to call the
+``free()`` method on objects whose class implements the ``Freeable``
+interface. Otherwise, a memory leak will occur.
+
+Specifying offsets in buffers
+-----------------------------
+
+In a C program, it is common to specify an offset in a array with
+``&array[i]`` or ``array+i`` to send data starting from a given
+position in the array. The equivalent form in the Java bindings is to
+``slice()`` the buffer to start at an offset. Making a ``slice()`` on
+a buffer is only necessary, when the offset is not zero. Slices work
+for both arrays and direct buffers.
+
+.. code-block:: java
+
+   import static mpi.MPI.slice;
+   // ...
+   int numbers[] = new int[SIZE];
+   // ...
+   MPI.COMM_WORLD.send(slice(numbers, offset), count, MPI.INT, 1, 0);
+
+
+Supported APIs
+--------------
+
+Complete MPI-3.1 coverage is provided in the Open MPI Java bindings,
+with a few exceptions:
+
+* The bindings for the ``MPI_Neighbor_alltoallw`` and
+  ``MPI_Ineighbor_alltoallw`` functions are not implemented.
+
+* Also excluded are functions that incorporate the concepts of
+  explicit virtual memory addressing, such as
+  ``MPI_Win_shared_query``.
+
+
+Known issues
+------------
+
+There exist issues with the Omnipath (PSM2) interconnect involving
+Java. The problems definitely exist in PSM2 v10.2; we have not tested
+previous versions.
+
+As of November 2016, there is not yet a PSM2 release that completely
+fixes the issue.
+
+The following ``mpirun`` command options will disable PSM2::
+
+   shell$ mpirun ... --mca mtl ^psm2 java ...your-java-options... your-app-class
+
+
+Questions?  Problems?
+---------------------
+
+The Java API documentation is generated at build time in
+``$prefix/share/doc/openmpi/javadoc``.
+
+Additionally, `this Cisco blog post
+<https://blogs.cisco.com/performance/java-bindings-for-open-mpi>`_ has
+quite a bit of information about the Open MPI Java bindings.
+
+If you have any problems, or find any bugs, please feel free to report
+them to `Open MPI user's mailing list
+<https://www.open-mpi.org/community/lists/ompi.php>`_.
+
+.. rubric:: Footnotes
+
+.. [#ompijava] O. Vega-Gisbert, J. E. Roman, and J. M. Squyres. "Design
+   and implementation of Java bindings in Open MPI". Parallel Comput.
+   59: 1-20 (2016).
+
+.. [#mpijava] M. Baker et al. "mpiJava: An object-oriented Java
+   interface to MPI". In Parallel and Distributed Processing, LNCS
+   vol. 1586, pp. 748-762, Springer (1999).
diff --git a/docs/features/profiling.rst b/docs/features/profiling.rst
new file mode 100644
index 00000000000..e0de0597cad
--- /dev/null
+++ b/docs/features/profiling.rst
@@ -0,0 +1,81 @@
+.. _open-mpi-profileing-label:
+
+Open MPI profiling interface
+============================
+
+Open MPI |ompi_ver| supportings the "PMPI" profiling interface as
+perscribed by the MPI standard for the C and Fortran bindings (*not*
+the :ref:`Open MPI Java binding extensions <open-mpi-java-label>`).
+
+Per MPI-4.0 section 15.2.1, MPI implementations must document which
+bindings layer on top of each other, so that profile developers know
+whether to implement the profiling interface for each binding, or
+whether they can economize by implementing it only for the lowest
+level routines.
+
+In general, Open MPI's Fortran bindings are implemented on top of the
+C bindings.  Hence, a profile developer who implements ``MPI_Init()``
+in C will also intecept all Fortran calls to ``MPI_INIT`` regardless
+of whether the user is utilizing the ``mpif.h``, ``use mpi``, or ``use
+mpi_f08`` Fortran interfaces.
+
+However, there are a handful of routines where Open MPI's Fortran
+bindings are *not* a simple wrapper around the back-end C MPI binding.
+Profile developers must therefore intercept the APIs listed below in
+their source language interface bindings in order to receive full
+profiling coverage.  Note, however, that it is only necessary for
+profile developers to intercept the ``mpif.h`` binding of each of the
+routines listed below; the ``use mpi`` and ``use mpi_f08`` bindings
+will ultimately invoke the ``mpif.h`` binding.
+
+* APIs that contain function pointer parameters:
+
+  .. admonition:: Rationale
+     :class: Hint
+
+     In order for Open MPI to invoke callbacks through the function
+     pointer with the proper language-specific calling conventions, it
+     must know the source language from which the function pointer was
+     passed.
+
+  * :ref:`MPI_COMM_CREATE_KEYVAL`
+  * :ref:`MPI_COMM_CREATE_ERRHANDLER`
+  * :ref:`MPI_ERRHANDLER_CREATE`
+
+    .. warning:: This function was deprecated by MPI-3.0.
+
+  * :ref:`MPI_FILE_CREATE_ERRHANDLER`
+  * :ref:`MPI_GREQUEST_START`
+  * :ref:`MPI_KEYVAL_CREATE`
+
+    .. warning:: This function was deprecated by MPI-3.0.
+
+  * :ref:`MPI_OP_CREATE`
+  * :ref:`MPI_REGISTER_DATAREP`
+  * :ref:`MPI_SESSION_CREATE_ERRHANDLER`
+  * :ref:`MPI_TYPE_CREATE_KEYVAL`
+  * :ref:`MPI_WIN_CREATE_KEYVAL`
+  * :ref:`MPI_WIN_CREATE_ERRHANDLER`
+
+* APIs dealing with MPI attributes:
+
+  .. admonition:: Rationale
+     :class: Hint
+
+     The MPI standard's treatment of attributes differs depending on
+     which language / interface was used to create the attribute
+     keyval, get the attribute value, or set the attribute value.
+
+  * :ref:`MPI_COMM_GET_ATTR`
+  * :ref:`MPI_COMM_SET_ATTR`
+  * :ref:`MPI_TYPE_GET_ATTR`
+  * :ref:`MPI_TYPE_SET_ATTR`
+  * :ref:`MPI_WIN_GET_ATTR`
+  * :ref:`MPI_WIN_SET_ATTR`
+
+Note that there is no harm in intercepting *all* routines in *all*
+interfaces.  Indeed, that is the most portable way to implement a
+profiling interface.  Since Open MPI's Fortran bindings are |mdash|
+for the most part |mdash| implemented on top of its C bindings,
+profile developers can ignore all Fortran interfaces except for the
+ones enumated above.
diff --git a/docs/features/ulfm.rst b/docs/features/ulfm.rst
new file mode 100644
index 00000000000..86815b7e435
--- /dev/null
+++ b/docs/features/ulfm.rst
@@ -0,0 +1,591 @@
+.. _ulfm-label:
+
+User-Level Fault Mitigation (ULFM)
+==================================
+
+This chapter documents the features and options specific to the **User
+Level Failure Mitigation (ULFM)** Open MPI implementation.
+
+Quick Start
+-----------
+
+This is an extremely terse summary of how to use ULFM:
+
+.. code-block::
+
+  shell$ ./configure --with-ft=ulfm [...options...]
+  shell$ make [-j N] all install
+  shell$ mpicc my-ft-program.c -o my-ft-program
+  shell$ mpirun -n 4 --with-ft ulfm my-ft-program
+
+Features
+--------
+
+This implementation conforms to the User Level Failure Mitigation
+(ULFM) MPI Standard draft proposal. The ULFM proposal is developed by
+the MPI Forum's Fault Tolerance Working Group to support the continued
+operation of MPI programs after any type of failures, hard or soft,
+have impacted the execution. The key principle is that no MPI call
+(point-to-point, collective, RMA, IO, ...) can block indefinitely
+after a failure, but must either succeed or raise an MPI
+error. Accordingly, the errors are not all fatal, the MPI
+implementations will do a best-effort approach to maintain the
+execution environment up and running.
+
+This implementation produces the three supplementary error codes and
+five supplementary interfaces defined in the communicator section of
+the `ULFM chapter
+<https://fault-tolerance.org/wp-content/uploads/2012/10/20170221-ft.pdf>`_
+standard draft document.
+
+* ``MPIX_ERR_PROC_FAILED`` when a process failure prevents the
+  completion of an MPI operation (error code).
+* ``MPIX_ERR_PROC_FAILED_PENDING`` when a potential sender matching a
+  non-blocking wildcard source receive has failed (error code).
+* ``MPIX_ERR_REVOKED`` when the communicator is revoked (error
+  code).
+* ``MPIX_Comm_revoke(MPI_Comm comm)`` Interrupts any communication
+  pending on the communicator at all ranks (API). See :ref:`MPIX_Comm_revoke`.
+* ``MPIX_Comm_is_revoked(MPI_Comm comm, int *flag)`` Test if a Communicator
+  is currently revoked (API). See :ref:`MPIX_Comm_is_revoked`.
+* ``MPIX_Comm_shrink(MPI_Comm comm, MPI_Comm* newcomm)`` creates a new
+  communicator where dead processes in comm were removed, and the
+  remaining processes are renamed to cover all the gaps in the naming
+  from the original communicator (API). See :ref:`MPIX_Comm_shrink`,
+  :ref:`MPIX_Comm_ishrink`.
+* ``MPIX_Comm_agree(MPI_Comm comm, int *flag)`` performs a consensus
+  (i.e. fault tolerant allreduce operation) on flag (with the
+  operation bitwise AND) (API).  Absorbs all new failures, and
+  propagate the knowledge about failures among the participants. see
+  :ref:`MPIX_Comm_agree`, :ref:`MPIX_Comm_iagree`.
+* ``MPIX_Comm_get_failed(MPI_Comm comm, MPI_Group* failedgrp)`` obtains the
+  group of currently failed processes (API). See :ref:`MPIX_Comm_get_failed`.
+* ``MPIX_Comm_ack_failed(MPI_Comm comm, int num_to_ack, int* num_acked)``
+  acknowledges that the application intends to ignore the effect of currently
+  known failures on wildcard receive completions and agreement return values
+  (API). See :ref:`MPIX_Comm_ack_failed`.
+
+Supported Systems
+-----------------
+
+There are several MPI communication engines available in Open MPI,
+notably:
+
+* PML: ``ob1``, ``cm``, ``ucx``
+* MTL: ``ofi``, ``portals4``, ``psm2``
+
+However, in Open MPI |ompi_ver|, only ``ob1`` is fully adapted to support
+fault tolerance. The UCX PML has been successfully tested in some setups,
+but at this point we cannot confirm that all UCT devices are fully capable
+to provide the necessary features.
+
+``ob1`` uses BTL ("Byte Transfer Layer") components for each supported
+network. ``ob1`` supports a variety of networks that can be used in
+combination with each other. Collective operations (blocking and
+non-blocking) use an optimized implementation on top of  ``ob1``.
+
+- Loopback (send-to-self)
+- TCP
+- UCT (InfiniBand)
+- uGNI (Cray Gemini, Aries)
+- Shared Memory (FT supported with CMA and XPMEM; KNEM is untested)
+- Tuned and non-blocking collective communications
+
+A full list of supported, untested and disabled components is provided
+later in this document.
+
+ULFM web site
+-------------
+
+More information (tutorials, examples, build instructions for leading
+top500 systems) is also available in the Fault Tolerance Research
+Hub website: https://fault-tolerance.org
+
+Bibliographic References
+------------------------
+
+If you are looking for, or want to cite a general reference for ULFM,
+please use:
+
+    *Wesley Bland, Aurelien Bouteiller, Thomas Herault, George Bosilca, Jack
+    J. Dongarra: Post-failure recovery of MPI communication
+    capability: Design and rationale. IJHPCA 27(3): 244-254 (2013).*
+
+Available from DOI :doi:`10.1177/1094342013488238`.
+
+Building ULFM support in Open MPI
+---------------------------------
+
+In Open MPI |ompi_ver|, ULFM support is **built-in by default** |mdash|
+that is, when you build Open MPI, unless you specify ``--without-ft``, ULFM
+support is automatically available (but is inactive unless enabled at
+runtime).
+
+Optionally, you can specify ``--with-ft ulfm`` to ensure that ULFM support
+is definitely built.
+
+.. note:: ULFM Fault Tolerance does not apply to OpenSHMEM.  It is recommended
+   that if you are going to use ULFM, you should disable building OpenSHMEM
+   with ``--disable-oshmem``.
+
+Running ULFM Open MPI
+---------------------
+
+Building your application
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+As ULFM is still an extension to the MPI standard, you will need to
+``#include <mpi-ext.h>`` in C, or ``use mpi_ext`` in Fortran to access
+the supplementary error codes and functions.
+
+Compile your application as usual, using the provided ``mpicc`` or
+``mpifort`` wrappers.
+
+Running your application
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+You can launch your application with fault tolerance by simply using
+the normal Open MPI ``mpirun`` launcher, with the
+``--with-ft ulfm`` CLI option (or its synonym ``--with-ft mpi``):
+
+.. code-block::
+
+   shell$ mpirun --with-ft ulfm ...
+
+.. important:: By default, fault tolerance is not active at run time.
+   It must be enabled via ``--with-ft ulfm``.
+
+Running under a batch scheduler
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+ULFM can operate under a job/batch scheduler, and is tested routinely
+with ALPS, PBS, and Slurm. One difficulty comes from the fact that
+many job schedulers handle failures by triggering an immediate "cleanup"
+of the application as soon as any process fails. In addition, failure
+detection subsystems integrated into PRTE are not active in direct launch
+scenarios and may not have a launcher specific alternative. This may cause
+the application to not detect failures and lock. In order to avoid these
+problems, it is preferred that you use ``mpirun`` within an allocation
+(e.g., ``salloc``, ``sbatch``, ``qsub``) rather than a direct launch.
+
+* SLURM is tested and supported with fault tolerance.
+
+  .. important:: Use ``mpirun`` in an ``salloc`` or ``sbatch`` allocation.
+     Direct launch with ``srun`` is not supported.
+
+* PBS/Torque is tested and supported with fault tolerance.
+
+  .. important:: Use ``mpirun`` in a ``qsub`` allocation. Direct launch
+     with ``aprun`` is not supported.
+
+* LSF is untested with fault tolerance.
+
+Run-time tuning knobs
+^^^^^^^^^^^^^^^^^^^^^
+
+ULFM comes with a variety of knobs for controlling how it runs. The
+default parameters are sane and should result in good performance in
+most cases. You can change the default settings with ``--mca
+mpi_ft_foo <value>`` for Open MPI options, and with ``--prtemca
+errmgr_detector_bar <value>`` for PRTE options.
+
+.. important:: The main control for enabling/disabling fault tolerance
+   at runtime is the ``--with-ft ulfm`` (or its synomym ``--with-ft mpi``)
+   ``mpirun`` CLI option. This option sets up multiple subsystems in
+   Open MPI to enable fault tolerance. The options described below are
+   best used to override the default behavior after the ``--with-ft ulfm``
+   opion is used.
+
+PRTE level options
+~~~~~~~~~~~~~~~~~~
+
+* ``prrte_enable_ft <true|false> (default: false)`` controls
+  automatic cleanup of apps with failed processes within
+  ``mpirun``. This option is automatically set to ``true`` when using
+  ``--with-ft ulfm``.
+* ``errmgr_detector_priority <int> (default 1005``) selects the
+  PRRTE-based failure detector. Only available when
+  ``prte_enable_recovery`` is ``true``. You can set this to ``0`` when
+  using the (experimental) Open MPI detector instead.
+* ``errmgr_detector_heartbeat_period <float> (default: 5e0)`` controls
+  the heartbeat period. Recommended value is 1/2 of the timeout.
+* ``errmgr_detector_heartbeat_timeout <float> (default: 1e1 seconds)``
+  heartbeat timeout (i.e. failure detection speed). Recommended value
+  is 2 times the heartbeat period. The default setup is tuned for
+  failure-free performance at the expense of fault detection
+  reactivity. In environments where faults are expected to be common,
+  less conservative values can be used (e.g., 100ms); Values lower
+  than the TCP poll rate (typically 10ms) can cause false positive.
+
+Open MPI level options
+~~~~~~~~~~~~~~~~~~~~~~
+
+Default values are applied to some Open MPI parameters when using
+``mpirun --with-ft ulfm``. These defaults are obtained from the ``ft-mpi``
+aggregate MCA param file
+``$installdir/share/openmpi/amca-param-sets/ft-mpi``. You can tune the
+runtime behavior of ULFM by either setting or unsetting variables in
+this file, or by overriding the variable on the command line (e.g.,
+``--mca btl ofi,self``).
+
+.. important:: Note that if fault tolerance is disabled at runtime,
+   (that is, when not using ``--with-ft ulfm``), the ``ft-mpi`` AMCA
+   param file is not loaded, thus components that are unsafe for fault
+   tolerance will load normally (this may change observed performance
+   when comparing with and without fault tolerance).
+
+* ``mpi_ft_enable <true|false> (default: false)``
+  permits turning on/off fault tolerance at runtime. This option is
+  automatically set to ``true`` from the aggregate MCA param file
+  ``ft-mpi`` loaded when using ``--with-ft ulfm``. When false, failure
+  detection is disabled; Interfaces defined by the fault tolerance extensions
+  are substituted with dummy non-fault tolerant implementations (e.g.,
+  ``MPIX_Comm_agree`` is implemented with ``MPI_Allreduce``); All other
+  controls below become irrelevant.
+* ``mpi_ft_verbose <int> (default: 0)`` increases the output of the
+  fault tolerance activities. A value of 1 will report detected
+  failures.
+* ``mpi_ft_detector <true|false> (default: false)``, **DEPRECATED**
+  controls the activation of the Open MPI level failure detector. When
+  this detector is turned off, all failure detection is delegated to
+  PRTE (see above).  The Open MPI level fault detector is
+  experimental. There is a tradeoff between failure detection accuracy
+  and performance with this detector. Users that experience accuracy
+  issues may enable a more precise mode.  See the tuning knobs below
+  to adjust to taste; The Open MPI failure detector operates on
+  ``MPI_COMM_WORLD`` exclusively.  Processes connected from
+  ``MPI_COMM_CONNECT``/``ACCEPT`` and ``MPI_COMM_SPAWN`` may
+  occasionally not be detected when they fail.
+
+  .. caution:: This component is deprecated. Failure detection is now
+     performed at the PRTE level. See the section above on controlling
+     PRTE behavior for information about how to tune the failure detector.
+
+* ``mpi_ft_detector_thread <true|false> (default: false)`` controls
+  the use of a thread to emit and receive failure detector's
+  heartbeats. *Setting this value to "true" will also set
+  MPI_THREAD_MULTIPLE support, which has a noticeable effect on
+  latency (typically 1us increase).* You may want to **enable this
+  option if you experience false positive** processes incorrectly
+  reported as failed with the Open MPI failure detector.
+
+  .. important:: This option is only relevant when ``mpi_ft_detector`` is ``true``.
+
+* ``mpi_ft_detector_period <float> (default: 3e0 seconds)`` heartbeat
+  period. Recommended value is 1/3 of the timeout. _Values lower than
+  100us may impart a noticeable effect on latency (typically a 3us
+  increase)._
+
+  .. important:: This option is only relevant when ``mpi_ft_detector`` is ``true``.
+
+* ``mpi_ft_detector_timeout <float> (default: 1e1 seconds)`` heartbeat
+  timeout (i.e. failure detection speed). Recommended value is 3 times
+  the heartbeat period.
+
+  .. important:: This option is only relevant when ``mpi_ft_detector`` is ``true``.
+
+Known Limitations in ULFM
+-------------------------
+
+* InfiniBand support is provided through the UCT BTL; fault tolerant
+  operation over the UCX PML is not yet supported for production runs.
+* TOPO, FILE, RMA are not fault tolerant. They are expected to work
+  properly before the occurrence of the first failure.
+
+Modified, Untested and Disabled Components
+------------------------------------------
+
+Frameworks and components are listed below and categorized into one of
+three classifications:
+
+1. **Modified:** This framework/component has been specifically modified
+   such that it will continue to work after a failure.
+2. **Untested:** This framework/component has not been modified and/or
+   tested with fault tolerance scenarios, and _may_ malfunction
+   after a failure.
+3. **Disabled:** This framework/component will cause unspecified behavior when
+   fault tolerance is enabled. As a consequence, it will be disabled when the
+   ``--with-ft ulfm`` option is used (see above for defails about implicit
+   parameters loaded from the ``ft-mpi`` aggregate param file).
+
+Any framework or component not listed below are categorized as **Unmodified**,
+meaning that it is unmodified for fault tolerance, but will continue to work
+correctly after a failure.
+
+* ``pml``: MPI point-to-point management layer
+
+  * ``monitoring``, ``v``: **untested** (they have not been modified to handle
+    faults)
+  * ``cm``, ``crcpw``, ``ucx``: **disabled**
+
+* ``btl``: Point-to-point Byte Transfer Layer
+
+  * ``ofi``, ``portals4``, ``smcuda``, ``usnic``, ``sm(+knem)``: **untested**
+    (they may work properly, please report)
+
+* ``mtl``: Matching transport layer Used for MPI point-to-point messages on
+  some types of networks
+
+  * All ``mtl`` components are **disabled**
+
+* ``coll``: MPI collective algorithms
+
+  * ``cuda``, ``inter``, ``sync``, ``sm``: **untested** (they have not
+    been modified to handle faults, but we expect correct post-fault
+    behavior)
+  * ``hcoll``, ``portals4`` **disabled** (they have not been modified
+    to handle faults, and we expect unspecified post-fault behavior)
+
+* ``osc``: MPI one-sided communications
+
+  * All ``osc`` components are **untested** (they have not been
+    modified to handle faults, and we expect unspecified post-fault
+    behavior)
+
+* ``io``: MPI I/O and dependent components
+
+  * ``fs``: File system functions for MPI I/O
+  * ``fbtl``: File byte transfer layer: abstraction for individual
+    read/write operations for OMPIO
+  * ``fcoll``: Collective read and write operations for MPI I/O
+  * ``sharedfp``: Shared file pointer operations for MPI I/O
+  * All components in these frameworks are unmodified, **untested**
+    (we expect clean post-failure abort)
+
+* ``vprotocol``: Checkpoint/Restart components
+
+  * All ``vprotocol`` components are **untested**
+
+* ``threads``, ``wait-sync``: Multithreaded wait-synchronization
+  object
+
+  * ``argotbots``, ``qthreads``: **disabled** (these components have
+    not been modified to handle faults; we expect post-failure
+    deadlock)
+
+Changelog
+---------
+
+ULFM Integrated in Open MPI
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+As of |ompi_ver|, ULFM is now integrated directly in to the community
+release of Open MPI.  The following sections describe previous ULFM
+standlone releases.
+
+ULFM Standalone Release 4.0.2u1
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This is a stability and upstream parity upgrade. It is based on the
+most current Open MPI Release (v4.0.2, October 2019).
+
+* This release is based on Open MPI release v4.0.2 (ompi #cb5f4e737a).
+* This release is based on ULFM master (ulfm #0e249ca1).
+* New features
+
+  * Support for the UCT BTL enters beta stage.
+
+* Bugfixes
+
+  * High sensitivity to noise in the failure detector.
+  * Deadlocks when revoking while BTL progress threads are updating messages.
+  * A case where the failure detector would keep observing a dead
+    process forever.
+  * Disable the use of external pmix/libevent by default (the
+    internals are modified to handle error cases).
+  * Clean error paths leaving some rdma registration dangling.
+  * Do not remove the orte job/proc session dir prematurely upon
+    error.
+
+ULFM Standalone Release 4.0.1u1
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This is a stability and upstream parity upgrade. It improves
+stability, performance and is based on the most current Open MPI
+Release (v4.0.1, May 2019).
+
+* This release is based on Open MPI release v4.0.1 (ompi #b780667).
+* This release is based on ULFM master (ulfm #cf8dc43f).
+* New features
+
+  * Addition of the ``MPI_Comm_is_revoked`` function
+  * Renamed ``ftbasic`` collective component to ``ftagree``
+  * Restored the ``pcollreq`` extension
+
+* Bugfixes
+
+  * Failures of node-local siblings were not always detected
+  * Failure propagation and detection was slowed down by trying to
+    notify known dead processes
+  * There were deadlocks in multithreaded programs
+  * There were issues with PMPI when compiling Fortran Interfaces
+  * There were deadlocks on OS-X
+
+ULFM Standalone Release 2.1
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This release is a bugfix and upstream parity upgrade. It improves
+stability, performance and is based on the most current Open MPI
+main (November 2018).
+
+* ULFM is now based upon Open MPI main branch (#37954b5f).
+* ULFM tuning MCA parameters are exposed by ``ompi_info``.
+* Fortran 90 bindings have been updated
+* Bugfixes:
+
+  * Correct the behavior of process placement during an MPI_COMM_SPAWN
+    when some slots were occcupied by failed processes.
+  * MPI_COMM_SPAWN accepts process placement directives in the Info object.
+  * Fixed deadlocks in some NBC collective operations.
+  * Crashes and deadlocks in MPI_FINALIZE have been resolved.
+  * Any-source requests that returned with an error status of
+    MPIX_PROC_FAILED_PENDING can now correctly complete during later
+    MPI_WAIT/TEST.
+
+ULFM Standalone Release 2.0
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Focus has been toward integration with current Open MPI main
+(November 2017), performance, and stability.
+
+* ULFM is now based upon Open MPI main branch (#689f1be9). It will
+  be regularly updated until it will eventually be merged.
+* Fault Tolerance is enabled by default and is controlled with MCA variables.
+* Added support for multithreaded modes (MPI_THREAD_MULTIPLE, etc.)
+* Added support for non-blocking collective operations (NBC).
+* Added support for CMA shared memory transport (Vader).
+* Added support for advanced failure detection at the MPI level.
+  Implements the algorithm described in "Failure detection and
+  propagation in HPC systems," DOI :doi:`10.1109/SC.2016.26`.
+* Removed the need for special handling of CID allocation.
+* Non-usable components are automatically removed from the build
+  during configure
+* RMA, FILES, and TOPO components are enabled by default, and usage in
+  a fault tolerant execution warns that they may cause undefined
+  behavior after a failure.
+* Bugfixes:
+
+  * Code cleanup and performance cleanup in non-FT builds; --without-ft at
+    configure time gives an almost stock Open MPI.
+  * Code cleanup and performance cleanup in FT builds with FT runtime disabled;
+    --mca ft_enable_mpi false thoroughly disables FT runtime activities.
+  * Some error cases would return ERR_PENDING instead of ERR_PROC_FAILED in
+    collective operations.
+  * Some test could set ERR_PENDING or ERR_PROC_FAILED instead of
+    ERR_PROC_FAILED_PENDING for ANY_SOURCE receptions.
+
+ULFM Standalone Release 1.1
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Focus has been toward improving stability, feature coverage for
+intercomms, and following the updated specification for
+MPI_ERR_PROC_FAILED_PENDING.
+
+* Forked from Open MPI 1.5.5 devel branch
+* Addition of the MPI_ERR_PROC_FAILED_PENDING error code, as per newer
+  specification revision. Properly returned from point-to-point,
+  non-blocking ANY_SOURCE operations.
+* Alias MPI_ERR_PROC_FAILED, MPI_ERR_PROC_FAILED_PENDING and
+  MPI_ERR_REVOKED to the corresponding standard blessed -extension-
+  names MPIX_ERR_xxx.
+* Support for Intercommunicators:
+
+  * Support for the blocking version of the agreement, MPI_COMM_AGREE
+    on Intercommunicators.
+  * MPI_COMM_REVOKE tested on intercommunicators.
+
+* Disabled completely (.ompi_ignore) many untested components.
+* Changed the default ORTE failure notification propagation
+  aggregation delay from 1s to 25ms.
+* Added an Open MPI internal failure propagator; failure propagation
+  between SM domains is now immediate.
+* Bugfixes:
+
+  * SendRecv would not always report MPI_ERR_PROC_FAILED correctly.
+  * SendRecv could incorrectly update the status with errors
+    pertaining to the Send portion of the Sendrecv.
+  * Revoked send operations are now always completed or remote
+    cancelled and may not deadlock anymore.
+  * Cancelled send operations to a dead peer will not trigger an
+    assert when the BTL reports that same failure.
+  * Repeat calls to operations returning MPI_ERR_PROC_FAILED will
+    eventually return MPI_ERR_REVOKED when another process revokes the
+    communicator.
+
+ULFM Standalone Release 1.0
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Focus has been toward improving performance, both before and after the
+occurrence of failures.  The list of new features includes:
+
+* Support for the non-blocking version of the agreement, MPI_COMM_IAGREE.
+* Compliance with the latest ULFM specification draft. In particular,
+  the MPI_COMM_(I)AGREE semantic has changed.
+* New algorithm to perform agreements, with a truly logarithmic
+  complexity in number of ranks, which translates into huge
+  performance boosts in MPI_COMM_(I)AGREE and MPI_COMM_SHRINK.
+* New algorithm to perform communicator revocation. MPI_COMM_REVOKE
+  performs a reliable broadcast with a fixed maximum output degree,
+  which scales logarithmically with the number of ranks.
+* Improved support for our traditional network layer:
+
+  * TCP: fully tested
+  * SM: fully tested (with the exception of XPMEM, which remains unsupported)
+
+* Added support for High Performance networks
+
+  * Open IB: reasonably tested
+  * uGNI: reasonably tested
+
+* The tuned collective module is now enabled by default (reasonably
+  tested), expect a huge performance boost compared to the former
+  basic default setting
+
+  * Back-ported PBS/ALPS fixes from Open MPI
+  * Back-ported OpenIB bug/performance fixes from Open MPI
+  * Improve Context ID allocation algorithm to reduce overheads of
+    Shrink
+  * Miscellaneous bug fixes
+
+Binary Compatibility
+^^^^^^^^^^^^^^^^^^^^
+
+ULFM Open MPI is binary compatible with any version of Open MPI
+compatible with the underlying Open MPI main branch or release (see
+the binary compatibility and version number section in the upstream
+Open MPI README). That is, applications compiled with a compatible
+Open MPI can run with the ULFM Open MPI ``mpirun`` and MPI
+libraries. Conversely, *as long as the application does not employ one
+of the MPIX functions,* which are exclusively defined in ULFM Open
+MPI, an application compiled with ULFM Open MPI can be launched with a
+compatible Open MPI ``mpirun`` and run with the non-fault tolerant MPI
+library.
+
+Contacting the Authors
+----------------------
+
+Found a bug?  Got a question?  Want to make a suggestion?  Want to
+contribute to ULFM Open MPI?  Working on a cool use-case?
+Please let us know!
+
+The best way to report bugs, send comments, or ask questions is to
+sign up on the user's mailing list: ulfm+subscribe@googlegroups.com
+
+Because of spam, only subscribers are allowed to post to these lists
+(ensure that you subscribe with and post from exactly the same e-mail
+address |mdash| joe@example.com is considered different than
+joe@mycomputer.example.com!).  Visit these pages to subscribe to the
+lists: https://groups.google.com/forum/#!forum/ulfm
+
+When submitting questions and problems, be sure to include as much
+extra information as possible.  See the :doc:`Getting help
+</getting-help>` section for more details.
+
+Thanks for your time.
+
+
+ULFM Copyright
+--------------
+
+Copyright (c) 2012-|year| The University of Tennessee and The
+University of Tennessee Research Foundation.  All rights reserved.
diff --git a/docs/generate-mpi-man3-bindings.py b/docs/generate-mpi-man3-bindings.py
new file mode 100755
index 00000000000..cc922a0af42
--- /dev/null
+++ b/docs/generate-mpi-man3-bindings.py
@@ -0,0 +1,282 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2025 Jeffrey M. Squyres.  All rights reserved.
+#
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# Script to create RST files containing the C, F90, and F80 bindings
+# that will be included in each of the MPI API man pages.  We generate
+# the bindings using the official MPI Forum python library to read the
+# API JSON that was generated when building the MPI Forum standard
+# LaTeX document and then emit the bindings.
+#
+# Using this method, we can emit both "regular" and "embiggened"
+# versions of each API (if an "embiggened" version exists).
+
+import re
+import os
+import sys
+import textwrap
+import argparse
+
+from pathlib import Path
+from collections import defaultdict
+from pprint import pprint
+
+#----------------
+
+def setup_cli():
+    parser = argparse.ArgumentParser(description="Generate C/F90/F08 bindings for RST")
+
+    parser.add_argument('--srcdir',
+                        required=True,
+                        help='Build source dir')
+    parser.add_argument('--builddir',
+                        required=True,
+                        help='Build build dir')
+
+    args = parser.parse_args()
+    return args
+
+#----------------
+
+def generate(func_name_arg, output_dir, directives):
+    global std
+
+    # Sanity check
+    func_name = func_name_arg.lower()
+    if not func_name in std.PROCEDURES:
+        print(f"ERROR: Don't know {func_name}")
+        return
+
+    # Do not generate this file if the binding for this function is
+    # going to be included in another file
+    for key, data in directives.items():
+        if key == func_name:
+            continue
+
+        # If this function is in the list of MPI bindings included in
+        # another page, then we don't need to generate this page.
+        if func_name in data:
+            return
+
+    # Also do not generate if we have no man page for this MPI API
+    # function.
+    if func_name not in directives:
+        return
+
+    #----------------
+
+    # If we get here, we're going to generate the bindings for one or
+    # more API functions.
+    func_names_data = []
+    for name in directives[func_name_arg]:
+        func_names_data.append(std.PROCEDURES[name])
+    data = std.PROCEDURES[func_name_arg]
+    if data not in func_names_data:
+        func_names_data.insert(0, data)
+
+    # Make an array of strings to emit into the output RST file.
+    blank = ''
+    out = []
+
+    out.append('SYNTAX')
+    out.append('------')
+    out.append(blank)
+
+    have_binding = False
+
+    # C bindings
+    emitted_header = False
+    for data in func_names_data:
+        binding = str(data.express.iso_c)
+        if binding and len(binding) > 0 and binding != 'None':
+            have_binding = True
+            if not emitted_header:
+                out.append('C Syntax')
+                out.append('^^^^^^^^')
+                out.append(blank)
+                out.append('.. code-block:: c')
+                out.append(blank)
+                emitted_header = True
+
+            # Per
+            # https://github.com/mpi-forum/pympistandard/issues/25,
+            # there's a bug that the binding for MPI_Pcontrol returns
+            # a string containing "\ldots", not "...".  Do a manual
+            # replacement here.
+            binding = binding.replace(r'\ldots', '...')
+
+            line = textwrap.fill(binding, width=72,
+                                 initial_indent='    ',
+                                 subsequent_indent = '        ')
+            out.append(line)
+            out.append(blank)
+
+            if data.has_embiggenment():
+                binding = str(data.express.embiggen.iso_c)
+                binding = binding.replace(r'\ldots', '...')
+                line = textwrap.fill(binding, width=72,
+                                     initial_indent='    ',
+                                     subsequent_indent = '        ')
+                out.append(line)
+                out.append(blank)
+
+    # F90 bindings
+    # Note: the f90 bindings were not embiggened
+    emitted_header = False
+    for data in func_names_data:
+        binding = str(data.express.f90)
+        if binding and len(binding) > 0 and binding != 'None':
+            have_binding = True
+            if not emitted_header:
+                out.append('Fortran Syntax')
+                out.append('^^^^^^^^^^^^^^')
+                out.append(blank)
+                out.append('.. code-block:: fortran')
+                out.append(blank)
+                out.append('    USE MPI')
+                out.append("    ! or the older form: INCLUDE 'mpif.h'")
+                emitted_header = True
+
+            lines = binding.split('\n')
+            for line in lines:
+                out.append(f'    {line}')
+            out.append(blank)
+
+    # F08 bindings
+    emitted_header = False
+    for data in func_names_data:
+        binding = str(data.express.f08)
+        if binding and len(binding) > 0 and binding != 'None':
+            have_binding = True
+            if not emitted_header:
+                out.append('Fortran 2008 Syntax')
+                out.append('^^^^^^^^^^^^^^^^^^^')
+                out.append(blank)
+                out.append('.. code-block:: fortran')
+                out.append(blank)
+                out.append('    USE mpi_f08')
+                emitted_header = True
+
+            lines = binding.split('\n')
+            for line in lines:
+                out.append(f'    {line}')
+            out.append(blank)
+
+            if data.has_embiggenment():
+                binding = str(data.express.embiggen.f08)
+                lines = binding.split('\n')
+                for line in lines:
+                    out.append(f'    {line}')
+                out.append(blank)
+
+    # Sanity check
+    if not have_binding:
+        print(f"NO BINDINGS for {func_name_arg}")
+        return
+
+    # Write the output file -- but only if it has changed
+    old_content = None
+    new_content = '\n'.join(out)
+    output_file = os.path.join(output_dir, f'{func_name}.rst')
+    if os.path.exists(output_file):
+        with open(output_file) as fp:
+            old_content = fp.read()
+
+    if old_content != new_content:
+        with open(output_file, 'w') as fp:
+            fp.write('\n'.join(out))
+
+        # This script is run during "make".  So emit a make-style
+        # message.
+        print(f'  GENERATE RST {func_name_arg} binding')
+
+#----------------
+
+# Some existing .3.rst man pages actually contain the docs for
+# multiple MPI_* API functions.  Read the .3.rst files and look for
+# directives that mean "this file contains documentation for all these
+# MPI API functions".
+def read_rst_man_pages(src_dir):
+    directives = {}
+    prog = re.compile(r'^MPI_.*\.3\.rst$')
+
+    man3_dir = Path(os.path.join(src_dir, 'man-openmpi', 'man3')).resolve()
+    for file in os.listdir(man3_dir):
+        # Only want MPI man pages
+        if not prog.match(file):
+            continue
+
+        with open(os.path.join(man3_dir, file)) as fp:
+            lines = fp.readlines()
+
+        file_api_name = file.replace('.3.rst', '').lower()
+
+        # Make an initial/empty list for every MPI API man page that
+        # we find
+        directives[file_api_name] = list()
+
+        prefix = '.. mpi-bindings:'
+        for line in lines:
+            line = line.strip()
+            if not line.startswith(prefix):
+                continue
+
+            bindings = line[len(prefix):].split(',')
+            for binding in bindings:
+                binding = binding.strip()
+                directives[file_api_name].append(binding.lower())
+
+    return directives
+
+#----------------
+
+def main():
+    args = setup_cli()
+
+    src_dir = os.path.abspath(args.srcdir)
+    build_dir = os.path.abspath(args.builddir)
+
+    # Read existing srcdir/man-openmpi/man3/MPI_*.3.rst files and look
+    # for directives to guide this generation process.
+    directives = read_rst_man_pages(src_dir)
+
+    # A bit of a hack to load the pympistandard module, which is in
+    # the Open MPI '3rd-party" tree in the source dir.
+    pympistandard_dir = Path(os.path.join(src_dir, '..', '3rd-party',
+                                          'pympistandard', 'src')).resolve()
+
+    sys.path.insert(0, str(pympistandard_dir))
+    global std
+    import pympistandard as std
+
+    # This is the JSON file with all the MPI standard APIs.  This is
+    # not currently officially distributed by the MPI Forum, so it was
+    # obtained by checking out the relevant branch from
+    # https://github.com/mpi-forum/mpi-standard/ and doing a build.
+    # This will create a file named apis.json.  Copy that here to this
+    # tree.
+    mpi_standard_json = os.path.abspath(os.path.join(src_dir,
+                                                     'mpi-standard-apis.json'))
+    std.use_api_version(1, given_path=mpi_standard_json)
+
+    # We need to write all of these into the build tree.  See
+    # docs/Makefile.am for a fuller explaination: all RST files are
+    # copied to the build tree, and we build there.
+    output_dir = os.path.join(build_dir, 'man-openmpi', 'man3', 'bindings')
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+
+    # Now we finally generate the files.  Iterate over all the MPI
+    # procedures and generate a binding file for each one of them.
+    for func_name in std.PROCEDURES:
+        generate(func_name, output_dir, directives)
+
+if __name__ == "__main__":
+    main()
diff --git a/docs/getting-help.rst b/docs/getting-help.rst
new file mode 100644
index 00000000000..ca9823f8a57
--- /dev/null
+++ b/docs/getting-help.rst
@@ -0,0 +1,263 @@
+Getting help
+============
+
+If you have a problem or question, it is highly recommended that you
+execute the following steps **in order**.  Many people have similar
+problems with configuration and initial setup of Open MPI |mdash| it
+is possible that your question has already been answered.
+
+#. :doc:`Validate your Open MPI installation </validate>`.  This
+   ensures that you have a nominally-correct Open MPI installation.
+
+#. `Check prior GitHub issues
+   <https://github.com/open-mpi/ompi/issues>`_ and see if others have
+   asked the same question and had it answered.
+
+#. `Check the mailing list archives
+   <https://www.open-mpi.org/community/lists/ompi.php>`_ "search"
+   features (or use Google) to check old posts and see if others have
+   asked the same question and had it answered.
+
+#. If you do not find a solution to your problem in the above
+   resources, proceed to the :ref:`Where to send?
+   <getting-help-where-to-send-label>` section.
+
+.. _getting-help-where-to-send-label:
+
+Where to send?
+--------------
+
+Different types of questions and problems should be sent to different
+places.  If you have:
+
+#. **A general end-user question or problem:** you should probably
+   subscribe to the `Open MPI user's mailing list
+   <https://www.open-mpi.org/community/lists/ompi.php>`_ and post it
+   there.
+
+   .. note:: Because of spam, only subscribers to the mailing list are
+      allowed to post to the mailing list.  Specifically: **you must
+      subscribe to the mailing list before posting.**
+
+   * If you have a compile-time question or problem, see the :ref:`For
+     problems building or installing Open MPI
+     <getting-help-compile-time-label>` section below for the content
+     of what to include in your email.
+
+   * If you have problems launching your MPI or OpenSHMEM application
+     successfully, see the :ref:`For problems launching MPI or
+     OpenSHMEM applications <getting-help-launching-label>` section
+     below for the content of what to include in your email.
+
+   * If you have other questions or problems about running your MPI or
+     OpenSHMEM application, see the :ref:`For problems running MPI or
+     OpenSHMEM applications <getting-help-running-label>` section
+     below for the content of what to include in your email.
+
+   .. important:: The more information you include in your report, the
+      better.  E-mails/bug reports simply stating, "It doesn't work!"
+      are not helpful; we need to know as much information about your
+      environment as possible in order to provide meaningful
+      assistance.
+
+   **The best way to get help** is to provide a "recipe" for
+   reproducing the problem.  This will allow the Open MPI developers
+   to see the error for themselves, and therefore be able to fix it.
+
+   .. important:: Please **use a descriptive "subject" line in your
+      email!** Some Open MPI question-answering people decide whether
+      to read a mail based on its subject line (e.g., to see if it's a
+      question that they can answer).  So please *please* **please**
+      use a good subject line that succinctly describes your problem.
+
+#. **A bug report:** you should probably post it to `Open MPI's Github
+   issue tracker <https://github.com/open-mpi/ompi/issues>`_.  Follow
+   the template to submit all the requested information.
+
+#. **A patch, bug fix, or other code submission:** please post a Github
+   Pull Request to the `Open MPI Github repository
+   <https://github.com/open-mpi/ompi/pulls>`_.
+
+#. **A developer-level / internal question about Open MPI itself:** you
+   should probably subscribe to the `Open MPI developer's mailing list
+   <https://www.open-mpi.org/community/lists/ompi.php>`_ and post it
+   there.
+
+If you're unsure where to send your question, subscribe and send an
+email to the user's mailing list (i.e., option #1, above).
+
+.. _getting-help-compile-time-label:
+
+For problems building or installing Open MPI
+--------------------------------------------
+
+If you cannot successfully configure, build, or install Open MPI,
+please provide *all* of the following information:
+
+#. The version of Open MPI that you're using.
+
+#. The stdout and stderr from running ``configure``.
+
+#. All ``config.log`` files from the Open MPI build tree.
+
+#. Output from when you ran ``make V=1 all`` to build Open MPI.
+
+#. Output from when you ran ``make install`` to install Open MPI.
+
+The script below may be helpful to gather much of the above
+information (adjust as necessary for your specific environment):
+
+.. code-block:: bash
+
+   #!/usr/bin/env bash
+
+   set -euxo pipefail
+
+   # Make a directory for the output files
+   dir="`pwd`/ompi-output"
+   mkdir $dir
+
+   # Fill in the options you want to pass to configure here
+   options=""
+   ./configure $options 2>&1  | tee $dir/config.out
+   tar -cf - `find . -name config.log` | tar -x -C $dir
+
+   # Build and install Open MPI
+   make V=1 all 2>&1          | tee $dir/make.out
+   make install 2>&1          | tee $dir/make-install.out
+
+   # Bundle up all of these files into a tarball
+   filename="ompi-output.tar.bz2"
+   tar -jcf $filename `basename $dir`
+   echo "Tarball $filename created"
+
+Then attach the resulting ``ompi-output.tar.bz2`` file to your report.
+
+.. caution:: The mailing lists have **a 150 KB size limit on
+   messages** (this is a limitation of the mailing list web archives).
+   If attaching the tarball makes your message larger than 150 KB, you
+   may need to post the tarball elsewhere and include a link to that
+   tarball in your mail to the list.
+
+.. _getting-help-launching-label:
+
+For problems launching MPI or OpenSHMEM applications
+----------------------------------------------------
+
+If you cannot successfully launch simple applications across multiple
+nodes (e.g., the non-MPI ``hostname`` command, or the MPI "hello world"
+or "ring" sample applications in the ``examples/`` directory), please
+provide *all* of the information from the :ref:`For problems building
+or installing Open MPI <getting-help-compile-time-label>` section, and
+*all* of the following additional information:
+
+#. The output of the ``ompi_info --all`` command from the node where
+   you are invoking :ref:`mpirun(1) <man1-mpirun>`.
+
+#. If you have questions or problems about process mapping or binding,
+   send the output from running the ``lstopo -v`` and ``lstopo --of
+   xml`` commands from a recent version of `Hwloc
+   <https://www.open-mpi.org/projects/hwloc/>`_.
+
+#. If running on more than one node, also include the output of the
+   ``ompi_info --version`` command **from each node on which you are
+   trying to run**.
+
+#. The output of running ``mpirun --map-by ppr:1:node --prtemca
+   plm_base_verbose 100 --prtemca rmaps_base_verbose 100 --display
+   alloc hostname``.  Add in a ``--hostfile`` argument if needed for
+   your environment.
+
+The script below may be helpful to gather much of the above
+information (adjust as necessary for your specific environment).
+
+.. note:: It is safe to run this script after running the script from
+   the :ref:`building and installing
+   <getting-help-compile-time-label>` section.
+
+.. code-block:: bash
+
+   #!/usr/bin/env bash
+
+   set -euxo pipefail
+
+   # Make a directory for the output files
+   dir="`pwd`/ompi-output"
+   mkdir -p $dir
+
+   # Get installation and system information
+   ompi_info --all 2>&1       | tee $dir/ompi-info-all.out
+   lstopo -v                  | tee $dir/lstopo-v.txt
+   lstopo --of xml            | tee $dir/lstopo.xml
+
+   # Have a text file "my_hostfile.txt" containing the hostnames on
+   # which you are trying to launch
+   for host in `cat my_hostfile.txt`; do
+       ssh $host ompi_info --version 2>&1 | tee $dir/ompi_info-version-$host.out
+       ssh $host lstopo -v                | tee $dir/lstopo-v-$host.txt
+       ssh $host lstopo --of xml          | tee $dir/lstopo-$host.xml
+   done
+
+   # Have a my_hostfile.txt file if needed for your environment, or
+   # remove the --hostfile argument altogether if not needed.
+   set +e
+   mpirun \
+        --hostfile my_hostfile.txt \
+        --map-by ppr:1:node \
+        --prtemca plm_base_verbose 100 \
+        --prtemca rmaps_base_verbose 100 \
+        --display alloc \
+        hostname 2>&1                     | tee $dir/mpirun-hostname.out
+
+   # Bundle up all of these files into a tarball
+   filename="ompi-output.tar.bz2"
+   tar -jcf $filename `basename $dir`
+   echo "Tarball $filename created"
+
+.. _getting-help-running-label:
+
+For problems running MPI or OpenSHMEM applications
+--------------------------------------------------
+
+If you can successfully launch parallel MPI or OpenSHMEM applications,
+but the jobs fail during the run, please provide *all* of the
+information from the :ref:`For problems building or installing Open
+MPI <getting-help-compile-time-label>` section, *all* of the
+information from the :ref:`For problems launching MPI or OpenSHMEM
+applications <getting-help-launching-label>` section, and then *all*
+of the following additional information:
+
+#. A *detailed* description of what is failing.  *The more details
+   that you provide, the better.* Please include at least the
+   following information:
+
+   * The exact command used to run your application.
+
+   * Any relevant MCA parameters that were set (or unset) when
+     you ran (from either the command line, environment,
+     parameter file, etc.).
+
+   * The value of the ``PATH`` and ``LD_LIBRARY_PATH``
+     environment variables (did you set them correctly to point
+     to all relevant executables, the Open MPI libraries, and
+     any required support libraries, such as libraries required
+     for high-speed networks such as InfiniBand).
+
+#. The source code of a short sample program (preferably in C or
+   Fortran) that exhibits the problem.
+
+#. If you are experiencing networking problems, include detailed
+   information about your network.
+
+   .. error:: TODO Update link to IB FAQ entry.
+
+   #. For RoCE- or InfiniBand-based networks, include the information
+      :ref:`in this FAQ entry <faq-ib-troubleshoot-label>`.
+
+   #. For Ethernet-based networks (including RoCE-based networks),
+      include the output of the ``ip addr`` command (or the legacy
+      ``ifconfig`` command) on all relevant nodes.
+
+      .. note:: Some Linux distributions do not put ``ip`` or
+                ``ifconfig`` in the default ``PATH`` of normal users.
+                Try looking for it in ``/sbin`` or ``/usr/sbin``.
diff --git a/docs/history.rst b/docs/history.rst
new file mode 100644
index 00000000000..8cb1c38eb3d
--- /dev/null
+++ b/docs/history.rst
@@ -0,0 +1,75 @@
+History of Open MPI
+===================
+
+Open MPI represents the merger of three prior MPI implementations:
+
+#. LAM/MPI: originally from the Ohio State University supercomputing
+   center and later migrated to the University of Notre Dame.
+#. LA-MPI: from the US Department of Energy Los Alamos National
+   Laboratory.
+#. FT-MPI: from the University of Tennassee at Knoxville.  One of the
+   UTK developers moved back to the University of Stuttgart in late
+   2004, which effectively added their team into the project.
+
+The lead developers of these projects kept bumping into each other at
+various HPC conferences in 2003.  At each conference, our
+lunch/dinner-table conversations got more and more technically
+involved when it finally dawned on us that we are doing a *lot* of the
+same things in each of our respective implementations.  Although each
+MPI implementation focused on different areas of excellence, we all
+shared the same common core values:
+
+* A full MPI implementation
+* Production quality code |mdash| it has to "just work"
+* A desire to explore lots of things that an MPI implementation can do
+  that we've never had the time/resources to investigate because we
+  are bound to bug fixing, etc.
+
+Hence, we decided to collaborate and pool our resources.  At SC2003,
+we decided to start an entire new code base |mdash| leaving all the cruft
+and legacy code of our prior implementations behind.  Take the best,
+leave the rest.  The source tree's first commit was on November 22,
+2003; development work started in earnest on January 5, 2004.  Since
+then, we have met together as a group once a month (for at least a
+week) to meet our goal of a world-class MPI implementation, bar none.
+
+
+Goals of the Open MPI Project
+-----------------------------
+
+We have several top-level goals:
+
+#. Create a free, open source, peer-reviewed, production-quality
+   complete MPI implementation.
+#. Provide extremely high, competitive performance (latency,
+   bandwidth, ...pick your favorite metric).
+#. Directly involve the HPC community with external development
+   and feedback (vendors, 3rd party researchers, users, etc.).
+#. Provide a stable platform for 3rd party research and commercial
+   development.
+#. Support a wide variety of HPC platforms and environments.
+
+In short, we want to work *with* and *for* the HPC community to make a
+world-class MPI implementation that can be used on a huge number and
+kind of systems.
+
+
+Community
+---------
+
+Bringing together smart researchers and developers to work on a common
+product is not only a good idea, it's the open source model.  The Open
+MPI project started by multiple MPI implementation teams, and that
+proved to work *extremely* well; extending this concept to the HPC
+open source community is the next logical step.
+
+The component architecture upon which Open MPI is founded is designed
+to foster 3rd party collaboration by enabling independent developers
+to use Open MPI as a production quality research platform.  Although
+Open MPI is a relatively large code base, it is not necessary to learn
+the entirety of it; it may be sufficient to learn the interfaces for
+the component type which you are implementing and some of the
+surrounding infrastructure.  Specifically, the component architecture
+was designed to allow small, discrete implementations of major
+portions of MPI functionality (e.g., point-to-point messaging,
+collective communications, run-time environment support, etc.).
diff --git a/docs/index.rst b/docs/index.rst
new file mode 100644
index 00000000000..50bd1c412e4
--- /dev/null
+++ b/docs/index.rst
@@ -0,0 +1,92 @@
+.. image:: openmpi_logo.png
+   :align: right
+
+Open MPI |ompi_ver|
+===================
+
+`The Open MPI Project <https://www.open-mpi.org/>`_ is an open source
+implementation of the `Message Passing Interface (MPI) specification
+<https://www.mpi-forum.org/docs/>`_ that is developed and maintained
+by a consortium of academic, research, and industry partners.  Open
+MPI is therefore able to combine the expertise, technologies, and
+resources from all across the High Performance Computing community in
+order to build the best MPI library available.  Open MPI offers
+advantages for system and software vendors, application developers and
+computer science researchers.
+
+Documentation locations
+=======================
+
+Documentation for Open MPI can be found in the following locations:
+
+.. list-table::
+   :header-rows: 1
+
+   * - Open MPI version
+     - Documentation location
+
+   * - v5.0.0 and later
+     - Web: https://docs.open-mpi.org/
+
+       Included in tarball: ``docs/html/index.html``
+
+       Built in source tree (if Sphinx available): ``docs/_build/html/index.html``
+
+       Installed: ``$docdir/html/index.html``
+
+       (which defaults to: ``$prefix/share/doc/openmpi/html/index.html``)
+
+   * - v4.1.x and earlier
+     - See the `legacy Open MPI FAQ <https://www.open-mpi.org/faq/>`_
+       and the README file in the source tarball.
+
+       For example:
+
+       * `v4.1.x README file <https://github.com/open-mpi/ompi/blob/v4.1.x/README>`_
+       * `v4.0.x README file <https://github.com/open-mpi/ompi/blob/v4.0.x/README>`_
+
+Release announcements
+=====================
+
+The best way to hear about new Open MPI releases is via the
+`low-volume announcement mailing list
+<https://lists.open-mpi.org/mailman/listinfo/announce>`_.
+
+Additionally, if you are a downstream packager of Open MPI (e.g., you
+package Open MPI for an operating system distribution), you may wish
+to sign up for the `low-volume packagers list
+<https://lists.open-mpi.org/mailman/listinfo/ompi-packagers>`_.
+
+Table of contents
+=================
+
+.. toctree::
+   :maxdepth: 2
+   :numbered:
+
+   quickstart
+   getting-help
+   release-notes/index
+   installing-open-mpi/index
+   features/index
+   validate
+   version-numbering
+   mca
+   building-apps/index
+   launching-apps/index
+   tuning-apps/index
+   app-debug/index
+   developers/index
+   contributing
+   license/index
+   history
+   man-openmpi/index
+   man-openshmem/index
+
+Contributors
+============
+
+A gigantic "thank you!" to all of our contributors:
+
+.. image:: https://contrib.rocks/image?repo=open-mpi/ompi&max=999
+   :target: https://github.com/open-mpi/ompi/graphs/contributors
diff --git a/docs/installing-open-mpi/compilers-and-flags.rst b/docs/installing-open-mpi/compilers-and-flags.rst
new file mode 100644
index 00000000000..eaca8f1a569
--- /dev/null
+++ b/docs/installing-open-mpi/compilers-and-flags.rst
@@ -0,0 +1,152 @@
+.. _install-configure-compilers-and-flags-label:
+
+Specifying compilers and flags
+==============================
+
+Changing the compilers that Open MPI uses to build itself uses the
+standard GNU Autoconf mechanism of setting special environment variables
+either before invoking ``configure`` or on the ``configure`` command
+line itself.
+
+The following environment variables are recognized by ``configure``:
+
+* ``CC``: C compiler to use
+* ``CFLAGS``: Compile flags to pass to the C compiler
+* ``CPPFLAGS``: Preprocessor flags to pass to the C compiler
+* ``CXX``: C++ compiler to use
+* ``CXXCFLAGS``: Compile flags to pass to the C++ compiler
+* ``CXXCPPFLAGS``: Preprocessor flags to pass to the C++ compiler
+* ``FC``: Fortran compiler to use
+* ``FCFLAGS``: Compile flags to pass to the Fortran compiler
+* ``LDFLAGS``: Linker flags to pass to all compilers
+* ``LIBS``: Libraries to pass to all compilers (it is rarely
+  necessary for users to need to specify additional ``LIBS``)
+* ``PKG_CONFIG``: Path to the ``pkg-config`` utility
+
+.. note:: Open MPI |ompi_ver| does not contain any C++ code.  The only
+          tests that ``configure`` runs with the C++ compiler is for
+          the purposes of determining an appropriate value for ``CXX``
+          to use in the ``mpic++`` :ref:`wrapper compiler
+          <label-quickstart-building-apps>`.  The ``CXXCFLAGS`` and
+          ``CXXCPPFLAGS`` values are *only* used in these
+          ``configure`` checks to ensure that the C++ compiler works.
+
+For example, to build with a specific instance of ``gcc``, ``g++``,
+and ``gfortran``:
+
+.. code-block:: sh
+
+   shell$ ./configure \
+       CC=/opt/gcc-a.b.c/bin/gcc \
+       CXX=/opt/gcc-a.b.c/bin/g++ \
+       FC=/opt/gcc-a.b.c/bin/gfortran ...
+
+Here's another example, this time showing building with the Intel
+compiler suite:
+
+.. code-block:: sh
+
+   shell$ ./configure \
+       CC=icc \
+       CXX=icpc \
+       FC=ifort ...
+
+.. note:: The Open MPI community generally suggests using the above
+   command line form for setting different compilers (vs. setting
+   environment variables and then invoking ``./configure``).  The
+   above form will save all variables and values in the ``config.log``
+   file, which makes post-mortem analysis easier if problems occur.
+
+Note that the flags you specify must be compatible across all the
+compilers.  In particular, flags specified to one language compiler
+must generate code that can be compiled and linked against code that
+is generated by the other language compilers.  For example, on a 64
+bit system where the compiler default is to build 32 bit executables:
+
+.. code-block:: sh
+
+   # Assuming the GNU compiler suite
+   shell$ ./configure CFLAGS=-m64 ...
+
+will produce 64 bit C objects, but 32 bit objects for Fortran.  These
+codes will be incompatible with each other, and Open MPI will not build
+successfully.  Instead, you must specify building 64 bit objects for
+*all* languages:
+
+.. code-block:: sh
+
+   # Assuming the GNU compiler suite
+   shell$ ./configure CFLAGS=-m64 FCFLAGS=-m64 ...
+
+The above command line will pass ``-m64`` to all the compilers, and
+therefore will produce 64 bit objects for all languages.
+
+.. warning:: Note that setting ``CFLAGS`` (etc.) does *not* affect the
+             flags used by the :ref:`wrapper compilers
+             <label-quickstart-building-apps>`.  In the above, example, you may
+             also need to add ``-m64`` to various ``--with-wrapper-FOO``
+             options:
+
+             .. code-block::
+
+                shell$ ./configure CFLAGS=-m64 FCFLAGS=-m64 \
+                     --with-wrapper-cflags=-m64 \
+                     --with-wrapper-cxxflags=-m64 \
+                     --with-wrapper-fcflags=-m64 ...
+
+             Failure to do this will result in MPI applications
+             failing to compile / link properly.
+
+             See the :ref:`Customizing wrapper compiler behavior
+             <label-customizing-wrapper-compiler>` section for more
+             details.
+
+Note that if you intend to compile Open MPI with a ``make`` other than
+the default one in your ``PATH``, then you must either set the ``$MAKE``
+environment variable before invoking Open MPI's ``configure`` script, or
+pass ``MAKE=your_make_prog`` to configure.  For example:
+
+.. code-block:: sh
+
+   shell$ ./configure MAKE=/path/to/my/make ...
+
+This could be the case, for instance, if you have a shell alias for
+``make``, or you always type ``gmake`` out of habit.  Failure to tell
+``configure`` which non-default ``make`` you will use to compile Open MPI
+can result in undefined behavior (meaning: don't do that).
+
+Note that you may also want to ensure that the value of
+``LD_LIBRARY_PATH`` is set appropriately (or not at all) for your build
+(or whatever environment variable is relevant for your operating
+system).  For example, some users have been tripped up by setting to
+use a non-default Fortran compiler via the ``FC`` environment variable,
+but then failing to set ``LD_LIBRARY_PATH`` to include the directory
+containing that non-default Fortran compiler's support libraries.
+This causes Open MPI's ``configure`` script to fail when it tries to
+compile / link / run simple Fortran programs.
+
+It is required that the compilers specified be compile and link
+compatible, meaning that object files created by one compiler must be
+able to be linked with object files from the other compilers and
+produce correctly functioning executables.
+
+Statically linking to the libraries of Intel compiler suite
+-----------------------------------------------------------
+
+The Intel compiler suite, by default, dynamically links its runtime libraries
+against the Open MPI binaries and libraries. This can cause problems if the
+Intel compiler libraries are installed in non-standard locations. For example,
+you might get errors like:
+
+.. code-block::
+
+   error while loading shared libraries: libimf.so: cannot open shared object file:
+   No such file or directory
+
+To avoid such problems, you can pass flags to Open MPI's configure
+script that instruct the Intel compiler suite to statically link its
+runtime libraries with Open MPI:
+
+.. code-block::
+
+   shell$ ./configure CC=icc CXX=icpc FC=ifort LDFLAGS=-Wc,-static-intel ...
diff --git a/docs/installing-open-mpi/configure-cli-options/conventions.rst b/docs/installing-open-mpi/configure-cli-options/conventions.rst
new file mode 100644
index 00000000000..8fefbf1ee29
--- /dev/null
+++ b/docs/installing-open-mpi/configure-cli-options/conventions.rst
@@ -0,0 +1,58 @@
+.. _building-ompi-cli-options-conventions-label:
+
+``configure`` CLI option conventions
+------------------------------------
+
+``configure`` will, by default, search for header files and/or
+libraries for various optional features (e.g., various HPC network
+API/support libraries).  If the relevant files are found, Open MPI
+will built support for that feature.  If they are not found, Open MPI
+will skip building support for that feature.
+
+However, if you specify ``--with-FOO`` (where ``FOO`` is the
+corresponding CLI option name for the feature) on the ``configure``
+command line and Open MPI is unable to find relevant support for
+``FOO``, ``configure`` will assume that it was unable to provide a
+feature that was specifically requested, and will therefore abort so
+that a human can resolve out the issue.
+
+.. note:: Using ``--with-FOO`` to force Open MPI's ``configure``
+          script to abort it if can't find support for a given feature
+          may be preferable to unexpectedly discovering at run-time
+          that Open MPI is missing support for a critical feature.
+
+Additionally, if a search directory is specified for ``FOO`` in the
+form ``--with-FOO=DIR``, Open MPI will:
+
+#. Search for ``FOO``'s header files in ``DIR/include``.
+#. Search for ``FOO``'s library files:
+
+   #. If ``--with-FOO-libdir=LIBDIR`` was specified, search in
+      ``LIBDIR``.
+   #. Otherwise, search in ``DIR/lib``, and if they are not found
+      there, search again in ``DIR/lib64``.
+
+#. If both the relevant header files and libraries are found:
+
+   #. Open MPI will build support for ``FOO``.
+   #. If the root path where the FOO libraries are found is neither
+      ``/usr`` nor ``/usr/local``, Open MPI will compile itself with
+      RPATH flags pointing to the directory where ``FOO``'s libraries
+      are located.
+
+      .. important:: Open MPI does not RPATH ``/usr/lib[64]`` and
+                     ``/usr/local/lib[64]`` because many systems
+                     already search these directories for run-time
+                     libraries by default; adding RPATH for them could
+                     have unintended consequences for the search path
+                     ordering.
+
+.. caution:: The ``--with-FOO-libdir=LIBDIR`` options are not usually
+   needed; they are typically only needed when ``FOO``'s libraries are
+   installed in an "unexpected" location.
+
+   Also note the difference between ``--with-FOO=DIR`` and
+   ``--with-FOO-libdir=LIBDIR``: the former is a directory to which
+   suffixes such as ``/include`` and ``/lib`` are added, whereas the
+   latter is assumed to be a full library directory name (e.g.,
+   ``/opt/some_library/lib``).
diff --git a/docs/installing-open-mpi/configure-cli-options/index.rst b/docs/installing-open-mpi/configure-cli-options/index.rst
new file mode 100644
index 00000000000..e80812e4799
--- /dev/null
+++ b/docs/installing-open-mpi/configure-cli-options/index.rst
@@ -0,0 +1,22 @@
+``configure`` command line options
+==================================
+
+There are many available ``configure`` command line options; see
+``./configure --help`` for a full list.
+
+Conventions for the command line options and many commonly-used
+options (but not all of them!) are described in the sections listed
+below.
+
+.. toctree::
+   :maxdepth: 1
+
+   conventions
+   installation
+   networking
+   runtime
+   required-support-libraries
+   mpi
+   oshmem
+   misc
+   rpath-and-runpath
diff --git a/docs/installing-open-mpi/configure-cli-options/installation.rst b/docs/installing-open-mpi/configure-cli-options/installation.rst
new file mode 100644
index 00000000000..c23877adb6b
--- /dev/null
+++ b/docs/installing-open-mpi/configure-cli-options/installation.rst
@@ -0,0 +1,334 @@
+.. _label-building-installation-cli-options:
+
+Installation options
+^^^^^^^^^^^^^^^^^^^^
+
+The following are general installation command line options that can
+be used with ``configure``:
+
+* ``--prefix=DIR``:
+  Install Open MPI into the base directory named ``DIR``.  Hence, Open
+  MPI will place its executables in ``DIR/bin``, its header files in
+  ``DIR/include``, its libraries in ``DIR/lib``, etc.
+
+  .. note:: Also see the section on :ref:`installation location
+            <building-open-mpi-installation-location-label>` for more
+            information on the installation prefix.
+
+* ``--disable-shared``: By default, Open MPI and OpenSHMEM build
+  shared libraries, and all components are included as part of those
+  shared libraries. This switch disables this default; it is really
+  only useful when used with ``--enable-static``.  Specifically, this
+  option does *not* imply ``--enable-static``; enabling static
+  libraries and disabling shared libraries are two independent
+  options.
+
+  .. tip::
+
+     :ref:`See this section <label-install-packagers-dso-or-not>` for
+     advice to packagers about this CLI option.
+
+* ``--enable-static``:
+  Build MPI and OpenSHMEM as static libraries, and statically link in
+  all components.  Note that this option does *not* imply
+  ``--disable-shared``; enabling static libraries and disabling shared
+  libraries are two independent options.
+
+  Be sure to read the description of ``--without-memory-manager``,
+  below; it may have some effect on ``--enable-static``.
+
+  .. tip::
+
+     :ref:`See this section <label-install-packagers-dso-or-not>` for
+     advice to packagers about this CLI option.
+
+* ``--disable-wrapper-runpath`` / ``--disable-wrapper-rpath``: By
+  default, the wrapper compilers (e.g., ``mpicc``) will explicitly add
+  "runpath" and "rpath" linker flags when linking user executables on
+  systems that support them.  That is, the created executables will
+  include a filesystem path reference to the location of Open MPI's
+  libraries in the application executable itself.  This means that the
+  user does not have to set ``LD_LIBRARY_PATH`` to find Open MPI's
+  libraries, which can be helpful if they are installed in a location
+  that the run-time linker does not search by default.
+
+  Using the ``--disable-wrapper-r*path`` options will prevent the
+  wrappers from explicitly adding one or both of these linker flags.
+
+  .. note:: By default, the wrapper compilers prefer "runpath"
+            behavior over "rpath" behavior.
+
+            * Using ``--disable-wrapper-runpath`` alters this
+              preference: explicit "runpath" linker flags will not be
+              added by the wrappers.  However, "rpath" flags may still
+              be added, if the platform supports them.
+            * Using both ``--disable-wrapper-runpath`` *and*
+              ``--disable-wrapper-rpath`` will prevent the wrappers
+              from explicitly adding "runpath" *and* "rpath" linker
+              flags.
+
+  .. caution:: Even if the wrapper compilers do not explicitly add
+               "runpath" or "rpath" linker flags, the local compiler,
+               linker, and/or operating system may implicitly enable
+               either "runpath" or "rpath" behavior when linking.
+
+  .. important:: The ``--disable-wrapper-runpath`` and
+                 ``--disable-wrapper-rpath`` CLI options *only* affect
+                 the flags that the wrapper compilers use when
+                 building MPI/SHMEM applications.  These options do
+                 not affect how Open MPI or OpenSHMEM are built (to
+                 include the wrapper compilers themselves).
+
+                 See the :ref:`Linker "rpath" and "runpath"
+                 functionality
+                 <building-ompi-cli-options-rpath-and-runpath-label>`
+                 section for details on how "rpath" and "runpath"
+                 affect the building and linking of Open MPI itself.
+
+  When either of "runpath" or "rpath" behaviors are enabled, the
+  applications will have the filesystem path location of the Open MPI
+  libraries hard-coded into the Open MPI/OpenSHMEM application.  The
+  most notable differences between "runpath" and "rpath" behavior are:
+
+  * runpath
+
+     #. The run-time linker first searches the paths in the
+        ``LD_LIBRARY_PATH`` environment variable for the relevant Open
+        MPI/OpenSHMEM libraries.
+     #. If not found there, the run-time linker falls back to checking
+        the hard-coded location for the relevant Open MPI/OpenSHMEM
+        libraries.
+
+  * rpath
+
+     #. The run-time linker first checks the hard-coded location for
+        the relevant Open MPI/OpenSHMEM libraries.
+     #. If not found there, the run-time linker falls back to
+        searching the paths in the ``LD_LIBRARY_PATH`` environment
+        variable for the relevant Open MPI/OpenSHMEM libraries.
+
+  .. warning:: There are other, subtle differences between "runpath"
+               and "rpath" which are out of scope for this
+               documentation.  You may wish to consult other sources
+               for more information.
+
+               For example, a decent set of explanations can be found
+               in the slides for a Linux course entitled "Building and
+               Using Shared Libraries on Linux // `Shared Libraries:
+               The Dynamic Linker
+               <https://man7.org/training/download/shlib_dynlinker_slides.pdf>`_".
+
+  For example, consider that you install Open MPI vA.B.0 and
+  compile/link your MPI/OpenSHMEM application against it.  Later, you
+  install Open MPI vA.B.1 to a different installation prefix (e.g.,
+  ``/opt/openmpi/A.B.1`` vs. ``/opt/openmpi/A.B.0``), and you leave
+  the old installation intact.
+
+  In the runpath case, you can set the ``LD_LIBRARY_PATH`` environment
+  variable to point to the A.B.1 installation, and then your MPI
+  application will use those libraries, since the runtime will search
+  the paths in ``LD_LIBRARY_PATH`` first.
+
+  In the rpath case, since the run-time linker searches the
+  ``/opt/openmpi/A.B.0`` location that is hard-coded in your MPI
+  application first, your application will use the libraries from your
+  A.B.0 installation (regardless of the value of the
+  ``LD_LIBRARY_PATH`` environment variable).
+
+  Note that in both cases, however, if you remove the original A.B.0
+  installation and set ``LD_LIBRARY_PATH`` to point to the A.B.1
+  installation, your application will use the A.B.1 libraries.
+
+  As noted above, both runpath/rpath behaviors can be disabled via
+  ``--disable-wrapper-rpath``.
+
+  .. note:: You can also :ref:`customize the compiler/linker flags
+            that are used by the wrapper compilers
+            <label-customizing-wrapper-compiler>` to build Open
+            MPI/OpenSHMEM applications.
+
+.. _building-ompi-cli-options-diable-dlopen-label:
+
+* ``--enable-dlopen``: Enable Open MPI to load components as
+  standalone Dynamic Shared Objects (DSOs) at run-time.  This option
+  is enabled by default.
+
+  The opposite of this option, ``--disable-dlopen``, causes the following:
+
+  #. Open MPI will not attempt to open any DSOs at run-time.
+  #. configure behaves as if the ``--enable-mca-static`` argument was set.
+  #. configure will ignore the ``--enable-mca-dso`` argument.
+
+  See the description of ``--enable-mca-static`` / ``--enable-mca-dso`` for
+  more information.
+
+  .. note:: This option does *not* change how Open MPI's libraries
+            (``libmpi``, for example) will be built.  You can change
+            whether Open MPI builds static or dynamic libraries via
+            the ``--enable|disable-static`` and
+            ``--enable|disable-shared`` arguments.
+
+.. _building-ompi-cli-options-mca-dso-label:
+
+* ``--enable-mca-dso[=LIST]`` and ``--enable-mca-static[=LIST]``
+  These two options, along with ``--enable-mca-no-build``, govern the
+  behavior of how Open MPI's frameworks and components are built.
+
+  .. tip::
+
+     :ref:`See this section <label-install-packagers-dso-or-not>` for
+     advice to packagers about these CLI options.
+
+  The ``--enable-mca-dso`` option specifies which frameworks and/or
+  components are built as Dynamic Shared Objects (DSOs).
+  Specifically, DSOs are built as "plugins" outside of the core Open
+  MPI libraries, and are loaded by Open MPI at run time.
+
+  The ``--enable-mca-static`` option specifies which frameworks and/or
+  components are built as part of the core Open MPI libraries (i.e.,
+  they are not built as DSOs, and therefore do not need to be
+  separately discovered and opened at run time).
+
+  Both options can be used one of two ways:
+
+  #. ``--enable-mca-OPTION`` (with no value)
+  #. ``--enable-mca-OPTION=LIST``
+
+  ``--enable-mca-OPTION=no`` or ``--disable-mca-OPTION`` are both legal
+  options, but have no impact on the selection logic described below.
+  Only affirmative options change the selection process.
+
+  If ``LIST`` is not specified (e.g., ``--enable-mca-dso`` with no
+  ``LIST``), or if ``LIST`` is the special value ``yes``, then *all*
+  components will be selected.  If ``LIST`` is specified, it is a
+  comma-delimited list of Open MPI frameworks and/or
+  framework+component tuples.  Examples:
+
+  * ``btl`` specifies the entire BTL framework
+  * ``btl-tcp`` specifies just the TCP component in the BTL framework
+  * ``mtl,btl-tcp`` specifies the entire MTL framework and the TCP
+    component in the BTL framework
+
+  Open MPI's ``configure`` script uses the values of these two options
+  when evaluating each component to determine how it should be built
+  by evaluating these conditions in order:
+
+  #. If an individual component's build behavior has been specified
+     via these two options, ``configure`` uses that behavior.
+  #. Otherwise, if the component is in a framework whose build
+     behavior has been specified via these two options, ``configure``
+     uses that behavior.
+  #. Otherwise, ``configure`` uses the global default build behavior.
+
+  At each level of the selection process, if the component is
+  specified to be built as both a static and DSO component, the static
+  option will win.
+
+  .. note:: As of Open MPI |ompi_ver|, ``configure``'s global default
+            is to build all components as static (i.e., part of the
+            Open MPI core libraries, not as DSOs).  Prior to Open MPI
+            v5.0.0, the global default behavior was to build
+            most components as DSOs.
+
+  .. important:: If the ``--disable-dlopen`` option is specified, then
+                 Open MPI will not be able to search for DSOs at run
+                 time, and the value of the ``--enable-mca-dso``
+                 option will be silently ignored.
+
+  Some examples:
+
+  #. Default to building all components as static (i.e., as part of
+     the Open MPI core libraries |mdash| no DSOs)::
+
+        shell$ ./configure
+
+  #. Build all components as DSOs::
+
+        shell$ ./configure --enable-mca-dso
+
+  #. Build all components as static, except the TCP BTL, which will be
+     built as a DSO::
+
+        shell$ ./configure --enable-mca-dso=btl-tcp
+
+  #. Build all components as static, except all BTL components, which
+     will be built as DSOs::
+
+        shell$ ./configure --enable-mca-dso=btl
+
+  #. Build all components as static, except all MTL components and the
+     TCP BTL component, which will be built as DSOs::
+
+        shell$ ./configure --enable-mca-dso=mtl,btl-tcp
+
+  #. Build all BTLs as static, except the TCP BTL, as the
+     ``<framework-component>`` option is more specific than the
+     ``<framework>`` option::
+
+        shell$ ./configure --enable-mca-dso=btl --enable-mca-static=btl-tcp
+
+  #. Build the TCP BTL as static, because the static option at the
+     same level always wins::
+
+        shell$ ./configure --enable-mca-dso=btl-tcp --enable-mca-static=btl-tcp
+
+* ``--enable-mca-no-build=LIST``: Comma-separated list of
+  ``<framework>-<component>`` pairs that will not be built. For
+  example, ``--enable-mca-no-build=threads-qthreads,pml-monitoring`` will
+  disable building both the ``qthreads`` threading component and the
+  ``monitoring`` PML.
+
+  .. note:: This option is typically only useful for components that
+            would otherwise be built.  For example, if you are on a
+            machine without Libfabric support, it is not necessary to
+            specify::
+
+              shell$ ./configure --enable-mca-no-build=cm-ofi
+
+            because the ``configure`` script will naturally see that
+            you do not have support for Libfabric and will
+            automatically skip the ``ofi`` CM component.
+
+* ``--with-show-load-errors=VALUE``:
+  Set the default value of the ``mca_base_component_show_load_errors``
+  MCA variable.  The MCA ``mca_base_component_show_load_errors`` variable
+  can still be overridden at run time via the usual MCA-variable-setting
+  mechanisms; this configure option simply sets the default value.
+
+  The ``no``/``none`` value of this option is intended for Open MPI
+  packagers who tend to enable support for many different types of
+  networks and systems in their packages.  For example, consider a
+  packager who includes support for both the FOO and BAR networks in
+  their Open MPI package, both of which require support libraries
+  (``libFOO.so`` and ``libBAR.so``).  If an end user only has BAR
+  hardware, they likely only have ``libBAR.so`` available on their
+  systems |mdash| not ``libFOO.so``.  Disabling load errors by default will
+  prevent the user from seeing potentially confusing warnings about
+  the FOO components failing to load because ``libFOO.so`` is not
+  available on their systems.
+
+  Conversely, the ``yes``/``all`` value of this option is intended for
+  system administrators who tend to build an Open MPI that is
+  targeted at their specific environment, and contains few (if any)
+  components that are not needed.  In such cases, they might want
+  their users to be warned that the FOO network components failed to
+  load (e.g., if ``libFOO.so`` was mistakenly unavailable), because Open
+  MPI may otherwise silently failover to a slower network path for MPI
+  traffic.
+
+  .. note:: See the section on :ref:`common MCA parameters
+            <label-mca-common-parameters>` for details related to the
+            ``mca_base_component_show_load_errors`` MCA variable.
+
+* ``--with-platform=FILE``:
+  Load configure options for the build from ``FILE``.  Options on the
+  command line that are not in ``FILE`` are also used.  Options on the
+  command line and in ``FILE`` are replaced by what is in ``FILE``.
+
+* ``--with-libmpi-name=STRING``:
+  Replace ``libmpi.*`` and ``libmpi_FOO.*`` (where ``FOO`` is one of the
+  fortran supporting libraries installed in lib) with ``libSTRING.*``
+  and ``libSTRING_FOO.*``. This is provided as a convenience mechanism
+  for third-party packagers of Open MPI that might want to rename
+  these libraries for their own purposes. This option is *not*
+  intended for typical users of Open MPI.
diff --git a/docs/installing-open-mpi/configure-cli-options/misc.rst b/docs/installing-open-mpi/configure-cli-options/misc.rst
new file mode 100644
index 00000000000..30d93f22786
--- /dev/null
+++ b/docs/installing-open-mpi/configure-cli-options/misc.rst
@@ -0,0 +1,97 @@
+.. This file is included by building-open-mpi.rst
+
+Miscellaneous functionality
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The following are command line options that don't fit any any of the
+above categories that can be used with ``configure``:
+
+* ``--without-memory-manager``:
+  Disable building Open MPI's memory manager.  Open MPI's memory
+  manager is usually built on Linux based platforms, and is generally
+  only used for optimizations with some OpenFabrics-based networks (it
+  is not *necessary* for OpenFabrics networks, but some performance
+  loss may be observed without it).
+
+  .. warning:: Open MPI's memory management functionality, which provides
+               important performance optimizations on OS-bypass networks
+               such as InfiniBand, requires the ``dlsym(3)`` interface,
+               and therefore does not work with fully-static applications.
+
+* ``--with-ft=TYPE``:
+  Specify the type of fault tolerance to enable.  The only allowed
+  values are ``ulfm`` and ``none``.  See :ref:`the ULFM section
+  <ulfm-label>` for more details.
+
+* ``--enable-peruse``:
+  Enable the PERUSE MPI data analysis interface.
+
+* ``--enable-heterogeneous``:
+  Enable support for running on heterogeneous clusters (e.g., machines
+  with different endian representations).  Heterogeneous support is
+  disabled by default because it imposes a minor performance penalty.
+
+  .. danger:: The heterogeneous functionality is currently broken |mdash|
+              do not use.
+
+.. _install-wrapper-flags-label:
+
+* ``--with-wrapper-cflags=CFLAGS``
+* ``--with-wrapper-cxxflags=CXXFLAGS``
+* ``--with-wrapper-fcflags=FCFLAGS``
+* ``--with-wrapper-ldflags=LDFLAGS``
+* ``--with-wrapper-libs=LIBS``:
+  Add the specified flags to the default flags that are used in Open
+  MPI's "wrapper" compilers (e.g., ``mpicc`` |mdash| see below for more
+  information about Open MPI's wrapper compilers).  By default, Open
+  MPI's wrapper compilers use the same compilers used to build Open
+  MPI and specify a minimum set of additional flags that are necessary
+  to compile/link MPI applications.  These configure options give
+  system administrators the ability to embed additional flags in
+  OMPI's wrapper compilers (which is a local policy decision).  The
+  meanings of the different flags are:
+
+  * ``CFLAGS``: Flags passed by the ``mpicc`` wrapper to the C
+    compiler
+  * ``CXXFLAGS``: Flags passed by the ``mpic++`` and ``mpiCC``
+    wrappers to the C++ compiler
+  * ``FCFLAGS``: Flags passed by the ``mpifort`` wrapper to the
+    Fortran compiler
+  * ``LDFLAGS``: Flags passed by all the wrappers to the linker
+  * ``LIBS``: Flags passed by all the wrappers to the linker
+
+  See the section on :ref:`customizing wrapper compiler behavior
+  <label-customizing-wrapper-compiler>` to see how to alter the
+  wrapper compiler behavior at run time.
+
+* ``--with-mpi-moduledir``: Specify the directory where the Fortran
+  MPI module files are installed.
+
+  For historical reasons, Open MPI's Fortran MPI modulefiles are
+  installed into ``$libdir`` by default.  This option lets you change
+  where they are installed; some users prefer Fortran module files
+  installed into ``$installdir``, for example.
+
+  .. note:: If you intend to make your Open MPI installation
+            relocatable :ref:`via the OPAL_PREFIX mechanism
+            <install-location-opal-prefix>`, you will want to ensure
+            to specify the module directory relative to the
+            ``prefix``.  For example:
+
+            .. code-block::
+
+               $ ./configure --prefix=/opt/openmpi --with-mpi-moduledir='${includedir}/modules`...
+
+            Note the additional shell quoting that is likely necessary
+            to prevent shell variable expansion, and the additional
+            ``${}`` around ``includedir`` that is necessary for Open MPI
+            to recognize that it is a special name that needs to be
+            expanded.
+
+            Finally, note that the Fortran module installation
+            directory is *not* one of the :ref:`recognized directories
+            that can be specified via environment variable at run time
+            <install-location-overriding-individual-directories>`.
+            Instead, to make the Fortran module directory relocatable,
+            it needs to be relative to one of the other recognized
+            directories.
diff --git a/docs/installing-open-mpi/configure-cli-options/mpi.rst b/docs/installing-open-mpi/configure-cli-options/mpi.rst
new file mode 100644
index 00000000000..dcc35670e72
--- /dev/null
+++ b/docs/installing-open-mpi/configure-cli-options/mpi.rst
@@ -0,0 +1,85 @@
+.. This file is included by building-open-mpi.rst
+
+MPI functionality
+^^^^^^^^^^^^^^^^^
+
+The following are command line options to set the default for various
+MPI API behaviors that can be used with ``configure``:
+
+* ``--with-mpi-param-check[=VALUE]``:
+  Whether or not to check MPI function parameters for errors at
+  runtime.  The following ``VALUE``\s are permitted:
+
+  * ``always``: MPI function parameters are always checked for errors
+  * ``never``: MPI function parameters are never checked for errors
+  * ``runtime``: Whether MPI function parameters are checked depends on
+    the value of the MCA parameter ``mpi_param_check`` (default: yes).
+  * ``yes``: Synonym for "always" (same as ``--with-mpi-param-check``).
+  * ``no``: Synonym for "never" (same as ``--without-mpi-param-check``).
+
+  If ``--with-mpi-param`` is not specified, ``runtime`` is the default.
+
+* ``--disable-mpi-thread-multiple``:
+  Disable the MPI thread level ``MPI_THREAD_MULTIPLE`` (it is enabled by
+  default).
+
+* ``--disable-ft``:
+  Disable the User-Level Fault Mitigation (ULFM) support in Open MPI
+  (it is enabled by default).
+
+  :ref:`See the ULFM section <ulfm-label>` for more information.
+
+* ``--enable-mpi-java``:
+  Enable building of an **EXPERIMENTAL** Java MPI interface (disabled
+  by default).  You may also need to specify ``--with-jdk-dir``,
+  ``--with-jdk-bindir``, and/or ``--with-jdk-headers``.
+
+  .. warning:: Note that this Java interface is **INCOMPLETE**
+     (meaning: it does not support all MPI functionality) and **LIKELY
+     TO CHANGE**.  The Open MPI developers would very much like to
+     hear your feedback about this interface.
+
+  :ref:`See the Java section <open-mpi-java-label>` for many more
+  details.
+
+* ``--enable-mpi-fortran[=VALUE]``:
+  By default, Open MPI will attempt to build all 3 Fortran bindings:
+  ``mpif.h``, the ``mpi`` module, and the ``mpi_f08`` module.  The following
+  ``VALUE``\s are permitted:
+
+  * ``all``: Synonym for ``yes``.
+  * ``yes``: Attempt to build all 3 Fortran bindings; skip
+    any binding that cannot be built (same as
+    ``--enable-mpi-fortran``).
+  * ``mpifh``: Only build ``mpif.h`` support.
+  * ``usempi``: Only build ``mpif.h`` and ``mpi`` module support.
+  * ``usempif08``:  Build ``mpif.h``, ``mpi`` module, and ``mpi_f08``
+    module support.
+  * ``none``: Synonym for ``no``.
+  * ``no``: Do not build any MPI Fortran support (same as
+    ``--disable-mpi-fortran``).  This is mutually exclusive
+    with building the OpenSHMEM Fortran interface.
+
+* ``--with-mpi-moduledir=DIR``:
+  Specify a specific ``DIR`` directory where to install the MPI
+  Fortran bindings modulefiles.  By default, Open MPI will install
+  Fortran modulefiles into ``$libdir``.
+
+* ``--enable-mpi-ext[=LIST]``:
+  Enable Open MPI's non-portable API extensions.  ``LIST`` is a
+  comma-delmited list of extensions.  If no ``LIST`` is specified, all
+  of the extensions are enabled.
+
+  See the "Open MPI API Extensions" section for more details.
+
+* ``--disable-mpi-io``:
+  Disable built-in support for MPI-2 I/O, likely because an
+  externally-provided MPI I/O package will be used.
+
+* ``--disable-io-ompio``:
+  Disable the ompio MPI-IO component
+
+* ``--enable-sparse-groups``:
+  Enable the usage of sparse groups. This would save memory
+  significantly especially if you are creating large
+  communicators. (Disabled by default)
diff --git a/docs/installing-open-mpi/configure-cli-options/networking.rst b/docs/installing-open-mpi/configure-cli-options/networking.rst
new file mode 100644
index 00000000000..cabb723a482
--- /dev/null
+++ b/docs/installing-open-mpi/configure-cli-options/networking.rst
@@ -0,0 +1,108 @@
+.. This file is included by building-open-mpi.rst
+
+.. _install-network-support-label:
+
+Networking support / options
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The following are command line options for various network types that
+can be used with ``configure``:
+
+* ``--with-fca=DIR``:
+  Specify the directory where the Mellanox FCA library and
+  header files are located.
+
+  FCA is the support library for Mellanox switches and HCAs.
+
+* ``--with-hcoll=DIR``:
+  Specify the directory where the Mellanox hcoll library and header
+  files are located.  This option is generally only necessary if the
+  hcoll headers and libraries are not in default compiler/linker
+  search paths.
+
+  hcoll is the support library for MPI collective operation offload on
+  Mellanox ConnectX-3 HCAs (and later).
+
+* ``--with-knem=DIR``:
+  Specify the directory where the knem libraries and header files are
+  located.  This option is generally only necessary if the knem headers
+  and libraries are not in default compiler/linker search paths.
+
+  knem is a Linux kernel module that allows direct process-to-process
+  memory copies (optionally using hardware offload), potentially
+  increasing bandwidth for large messages sent between messages on the
+  same server.  See `the Knem web site
+  <https://knem.gitlabpages.inria.fr/>`_ for details.
+
+* ``--with-libfabric=DIR``:
+  Specify the directory where the OpenFabrics Interfaces ``libfabric``
+  library and header files are located.  This option is generally only
+  necessary if the libfabric headers and libraries are not in default
+  compiler/linker search paths.
+
+  Libfabric is the support library for OpenFabrics Interfaces-based
+  network adapters, such as Cisco usNIC, Intel True Scale PSM, etc.
+
+* ``--with-libfabric-libdir=DIR``:
+  Look in directory for the libfabric libraries.  By default, Open MPI
+  will look in ``DIR/lib`` and ``DIR/lib64``, which covers most cases.
+  This option is only needed for special configurations.
+
+* ``--with-portals4=DIR``:
+  Specify the directory where the Portals4 libraries and header files
+  are located.  This option is generally only necessary if the Portals4
+  headers and libraries are not in default compiler/linker search
+  paths.
+
+  Portals is a low-level network API for high-performance networking
+  on high-performance computing systems developed by Sandia National
+  Laboratories, Intel Corporation, and the University of New Mexico.
+  The Portals 4 Reference Implementation is a complete implementation
+  of Portals 4, with transport over InfiniBand verbs and UDP.
+
+* ``--with-portals4-libdir=DIR``:
+  Location of libraries to link with for Portals4 support.
+
+* ``--with-portals4-max-md-size=SIZE`` and
+  ``--with-portals4-max-va-size=SIZE``:
+  Set configuration values for Portals 4
+
+* ``--with-psm=<directory>``:
+  Specify the directory where the QLogic InfiniPath / Intel True Scale
+  PSM library and header files are located.  This option is generally
+  only necessary if the PSM headers and libraries are not in default
+  compiler/linker search paths.
+
+  PSM is the support library for QLogic InfiniPath and Intel True Scale
+  network adapters.
+
+* ``--with-psm-libdir=DIR``:
+  Look in directory for the PSM libraries.  By default, Open MPI will
+  look in ``DIR/lib`` and ``DIR/lib64``, which covers most cases.  This
+  option is only needed for special configurations.
+
+* ``--with-psm2=DIR``:
+  Specify the directory where the Intel Omni-Path PSM2 library and
+  header files are located.  This option is generally only necessary
+  if the PSM2 headers and libraries are not in default compiler/linker
+  search paths.
+
+  PSM is the support library for Intel Omni-Path network adapters.
+
+* ``--with-psm2-libdir=DIR``:
+  Look in directory for the PSM2 libraries.  By default, Open MPI will
+  look in ``DIR/lib`` and ``DIR/lib64``, which covers most cases.  This
+  option is only needed for special configurations.
+
+* ``--with-ucx=DIR``:
+  Specify the directory where the UCX libraries and header files are
+  located.  This option is generally only necessary if the UCX headers
+  and libraries are not in default compiler/linker search paths.
+
+* ``--with-ucx-libdir=DIR``:
+  Look in directory for the UCX libraries.  By default, Open MPI will
+  look in ``DIR/lib`` and ``DIR/lib64``, which covers most cases.  This
+  option is only needed for special configurations.
+
+* ``--with-usnic``:
+  Abort configure if Cisco usNIC support cannot be built.
diff --git a/docs/installing-open-mpi/configure-cli-options/oshmem.rst b/docs/installing-open-mpi/configure-cli-options/oshmem.rst
new file mode 100644
index 00000000000..73ca3f37ee1
--- /dev/null
+++ b/docs/installing-open-mpi/configure-cli-options/oshmem.rst
@@ -0,0 +1,16 @@
+.. This file is included by building-open-mpi.rst
+
+OpenSHMEM functionality
+^^^^^^^^^^^^^^^^^^^^^^^
+
+The following are command line options to set the default for various
+OpenSHMEM API behaviors that can be used with ``configure``:
+
+* ``--disable-oshmem``:
+  Disable building the OpenSHMEM implementation (by default, it is
+  enabled).
+
+* ``--disable-oshmem-fortran``:
+  Disable building only the Fortran OpenSHMEM bindings. Please see
+  the "Compiler Notes" section herein which contains further
+  details on known issues with various Fortran compilers.
diff --git a/docs/installing-open-mpi/configure-cli-options/required-support-libraries.rst b/docs/installing-open-mpi/configure-cli-options/required-support-libraries.rst
new file mode 100644
index 00000000000..797cba68f26
--- /dev/null
+++ b/docs/installing-open-mpi/configure-cli-options/required-support-libraries.rst
@@ -0,0 +1,46 @@
+.. _label-building-ompi-cli-options-required-support-libraries:
+
+CLI Options for required support libraries
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The following ``configure`` command line options are for Open MPI's
+:ref:`required support libraries
+<label-install-required-support-libraries>`
+
+* ``--with-hwloc[=VALUE]``:
+* ``--with-libevent[=VALUE]``:
+* ``--with-pmix[=VALUE]``:
+* ``--with-prrte[=VALUE]``: These four options specify where to find
+  the headers and libraries for the `Hwloc
+  <https://www.open-mpi.org/projects/hwloc/>`_, `Libevent
+  <https://libevent.org/>`_, `PMIx <https://openpmix.github.io/>`_,
+  and `PRRTE <https://github.com/openpmix/prrte>`_ libraries,
+  respectively.  The following ``VALUE``\s are permitted:
+
+  * ``external``: Use an external installation (rely on default
+    compiler and linker paths to find it).  ``configure`` will abort
+    if it cannot find suitable header files and libraries.
+  * ``internal``: Use Open MPI's internal/bundled copy.
+  * No value specified: Try the ``external`` behavior.  If that fails,
+    fall back to ``internal`` behavior.  *This is the default behavior.*
+  * ``DIR``: Specify the location of a specific installation to use.
+    ``configure`` will abort if it cannot find suitable header files
+    and libraries under ``DIR``.
+
+* ``--with-hwloc-libdir=LIBDIR``:
+* ``--with-libevent-libdir=LIBDIR``:
+* ``--with-prrte-libdir=LIBDIR``:
+* ``--with-pmix-libdir=LIBDIR``:
+  :ref:`See the configure CLI
+  options conventions <building-ompi-cli-options-conventions-label>`
+  for a description of these four options.
+
+* ``--with-valgrind[=DIR]``:
+  Directory where the valgrind software is installed.  If Open MPI
+  finds Valgrind's header files, it will include additional support
+  for Valgrind's memory-checking debugger.
+
+  Valgrind support is disabled by default in Open MPI.  Enabling
+  Valgrind support will eliminate a lot of false positives when
+  running Valgrind on MPI applications.  There is a minor performance
+  penalty for enabling this option.
diff --git a/docs/installing-open-mpi/configure-cli-options/rpath-and-runpath.rst b/docs/installing-open-mpi/configure-cli-options/rpath-and-runpath.rst
new file mode 100644
index 00000000000..6a046967ac5
--- /dev/null
+++ b/docs/installing-open-mpi/configure-cli-options/rpath-and-runpath.rst
@@ -0,0 +1,64 @@
+.. _building-ompi-cli-options-rpath-and-runpath-label:
+
+Linker "rpath" and "runpath" functionality
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   
+Open MPI and OpenSHMEM |ompi_ver| are both composed of multiple
+libraries that depend on each other.
+
+When built from official distribution tarballs (from `the official
+Open MPI download site <https://www.open-mpi.org/software/ompi/>`_),
+Open MPI and OpenSHMEM are built with the following versions of the
+GNU Autotools:
+
+* Autoconf |autoconf_min_version|
+* Automake |automake_min_version|
+* Libtool |libtool_min_version|
+
+This set of GNU Autotools invokes the ``libtool`` executable to build
+Open MPI's libraries and executables with the `-rpath CLI option
+<https://www.gnu.org/software/libtool/manual/html_node/Link-mode.html#Link-mode>`_.
+
+The behavior of ``libtool -rpath ...`` is, unfortunately, highly
+system-dependent.  In conjunction with compiler-, linker-, and other
+system-level settings, the end result may be to effect "rpath"
+behavior, "runpath" behavior, or possibly even neither behavior.
+
+Although the specific behavior of ``libtool -rpath ..`` is outside the
+scope of this documentation, you can run commands such as ``readelf -d
+...`` to find out with which behavior your Open MPI was built.
+
+For example:
+
+.. code::
+
+   shell$ ./configure --prefix=/opt/openmpi/ ...
+   ...
+   shell$ make -j 32 all && make install
+   ...
+   shell$ readelf -d /opt/openmpi/lib/libmpi.so | egrep -i 'rpath|runpath'
+    0x000000000000001d (RUNPATH)            Library runpath: [/opt/openmpi/lib]
+
+The above output indicates that ``libmpi.so`` was built with "runpath"
+support, whereas output like this:
+
+.. code::
+
+   shell$ readelf -d /opt/openmpi/lib/libmpi.so | egrep -i 'rpath|runpath'
+    0x000000000000000f (RPATH)              Library rpath: [/opt/openmpi/lib]
+
+indicates that ``libmpi.so`` was built with "rpath" support.
+          
+.. note:: If you want to utilize additional compiler or linker flags
+          (such as runpath flags) when building Open MPI/OpenSHMEM,
+          you can :ref:`specify these flags on the configure command
+          line <install-configure-compilers-and-flags-label>`.
+
+          For example, if invoking ``libtool -rpath ...`` on your
+          system actually effects "rpath" behavior, and you wish to
+          effect "runpath" behavior, you could set ``LDFLAGS`` when
+          invoking ``configure``, like this:
+
+          .. code-block:: sh
+
+             shell$ ./configure LDFLAGS=-Wl,--enable-new-dtags ...
diff --git a/docs/installing-open-mpi/configure-cli-options/runtime.rst b/docs/installing-open-mpi/configure-cli-options/runtime.rst
new file mode 100644
index 00000000000..c6e825e63d7
--- /dev/null
+++ b/docs/installing-open-mpi/configure-cli-options/runtime.rst
@@ -0,0 +1,57 @@
+.. This file is included by building-open-mpi.rst
+
+Run-time system support
+^^^^^^^^^^^^^^^^^^^^^^^
+
+The following are command line options for various runtime systems that
+can be used with ``configure``:
+
+* ``--enable-mpirun-prefix-by-default``:
+  This option forces the ``mpirun`` command to always behave as if
+  ``--prefix $prefix`` was present on the command line (where ``$prefix``
+  is the value given to the ``--prefix`` option to configure).  This
+  prevents most ``rsh``/``ssh``-based users from needing to modify their
+  shell startup files to set the ``PATH`` and/or ``LD_LIBRARY_PATH`` for
+  Open MPI on remote nodes.  Note, however, that such users may still
+  desire to set ``PATH`` |mdash| perhaps even in their shell startup files |mdash|
+  so that executables such as ``mpicc`` and ``mpirun`` can be found
+  without needing to type long path names.
+
+* ``--with-alps``:
+  Force the building of for the Cray Alps run-time environment.  If
+  Alps support cannot be found, configure will abort.
+
+* ``--with-lsf=DIR``:
+  Specify the directory where the LSF libraries and header files are
+  located.  This option is generally only necessary if the LSF headers
+  and libraries are not in default compiler/linker search paths.
+
+  LSF is a resource manager system, frequently used as a batch
+  scheduler in HPC systems.
+
+* ``--with-lsf-libdir=DIR``:
+  Look in directory for the LSF libraries.  By default, Open MPI will
+  look in ``DIR/lib`` and ``DIR/lib64``, which covers most cases.  This
+  option is only needed for special configurations.
+
+* ``--with-slurm``:
+  Force the building of Slurm scheduler support.
+
+* ``--with-sge``:
+  Specify to build support for the Oracle Grid Engine (OGE) resource
+  manager and/or the Open Grid Engine.  OGE support is disabled by
+  default; this option must be specified to build OMPI's OGE support.
+
+  The Oracle Grid Engine (OGE) and open Grid Engine packages are
+  resource manager systems, frequently used as a batch scheduler in
+  HPC systems.  It used to be called the "Sun Grid Engine", which is
+  why the option is still named ``--with-sge``.
+
+* ``--with-tm=DIR``:
+  Specify the directory where the TM libraries and header files are
+  located.  This option is generally only necessary if the TM headers
+  and libraries are not in default compiler/linker search paths.
+
+  TM is the support library for the Torque and PBS Pro resource
+  manager systems, both of which are frequently used as a batch
+  scheduler in HPC systems.
diff --git a/docs/installing-open-mpi/configure-output-summary.rst b/docs/installing-open-mpi/configure-output-summary.rst
new file mode 100644
index 00000000000..d93ae899893
--- /dev/null
+++ b/docs/installing-open-mpi/configure-output-summary.rst
@@ -0,0 +1,13 @@
+``configure`` output summary
+============================
+
+At the end of ``configure``, a summary of many configuration options
+are shown.
+
+The Open MPI Team *highly* recommends examining the ``configure``
+summary to validate that Open MPI plans to build all the features that
+you expect it to build.
+
+If the summary does not agree with what you think should be done, you
+can look earlier in ``configure``'s output to see why particular
+options and behaviors were selected.
diff --git a/docs/installing-open-mpi/custom-components.rst b/docs/installing-open-mpi/custom-components.rst
new file mode 100644
index 00000000000..e103b355586
--- /dev/null
+++ b/docs/installing-open-mpi/custom-components.rst
@@ -0,0 +1,33 @@
+.. _installing-custom-components-label:
+
+Installing custom components
+============================
+
+By default, Open MPI looks in two places for components at
+run-time (in order):
+
+#. ``$prefix/lib/openmpi/``: This is the system-provided components
+   directory, part of the installation tree of Open MPI itself.
+#. ``$HOME/.openmpi/components/``: This is where users can drop their
+   own components that will automatically be "seen" by Open MPI at
+   run-time.  This is ideal for developmental, private, or otherwise
+   unstable components.
+
+Note that the directories and search ordering used for finding
+components in Open MPI is, itself, an MCA parameter.  Setting the
+``mca_component_path`` changes this value (a colon-delimited list of
+directories).
+
+Note also that components are only used on nodes where they are
+"visible". Hence, if your ``$prefix/lib/openmpi/`` is a directory on a
+local disk that is not shared via a network filesystem to other nodes
+where you run MPI jobs, then components that are installed to that
+directory will *only* be used by MPI jobs running on the local node.
+
+More specifically: components have the same visibility as normal
+files.  If you need a component to be available to all nodes where you
+run MPI jobs, then you need to ensure that it is visible on all nodes
+(typically either by installing it on all nodes for non-networked
+filesystem installs, or by installing them in a directory that is
+visible to all nodes via a networked filesystem).  Open MPI does not
+automatically send components to remote nodes when MPI jobs are run.
diff --git a/docs/installing-open-mpi/definitions.rst b/docs/installing-open-mpi/definitions.rst
new file mode 100644
index 00000000000..a05edb11010
--- /dev/null
+++ b/docs/installing-open-mpi/definitions.rst
@@ -0,0 +1,16 @@
+Definitions
+===========
+
+* **Source tree:** The tree where the Open MPI source code is located.
+  It is typically the result of expanding an Open MPI distribution
+  source code bundle, such as a tarball.
+* **Build tree:** The tree where Open MPI was built.  It is always
+  related to a specific source tree, but may actually be a different
+  tree (:ref:`since Open MPI supports VPATH builds
+  <building-open-mpi-vpath-label>`).  Specifically, this is the tree
+  where you invoked ``configure``, ``make``, etc. to build and install
+  Open MPI.
+* **Installation tree:** The tree where Open MPI was installed.  It is
+  typically the "prefix" argument given to Open MPI's ``configure``
+  script; it is the directory from which you run installed Open MPI
+  executables.
diff --git a/docs/installing-open-mpi/downloading.rst b/docs/installing-open-mpi/downloading.rst
new file mode 100644
index 00000000000..8eb13771dd5
--- /dev/null
+++ b/docs/installing-open-mpi/downloading.rst
@@ -0,0 +1,41 @@
+
+.. _building-open-mpi-downloading-label:
+
+Downloading Open MPI
+====================
+
+Open MPI is generally available two ways:
+
+#. As source code.
+
+   * The best place to get an official Open MPI source code
+     distribution is from `the main Open MPI web site
+     <https://www.open-mpi.org/>`_.
+
+   * Downstream Open MPI packagers (e.g., Linux distributions)
+     sometimes also provide source code distributions.  They may
+     include additional patches or modifications.
+
+     Consult your favorite downstream packager for more details.
+     
+   .. caution:: Do **not** download an Open MPI source code tarball
+               from GitHub.com.  The tarballs automatically generated
+               by GitHub.com are incomplete and will not build
+               properly.
+
+               GitHub.com-generated tarballs are **not** official Open
+               MPI releases.
+
+#. As binary packages.
+
+   * The Open MPI community does not provide binary packages on `the
+     main Open MPI web site <https://www.open-mpi.org/>`_.
+
+   * Various downstream packagers (e.g., Linux distributions,
+     Homebrew, MacPorts, etc.) *do* provide pre-built, binary packages
+     for Open MPI.
+
+     Consult your favorite downstream packager for more details.
+
+The majority of the rest of this section deals with installing Open
+MPI from source.
diff --git a/docs/installing-open-mpi/filesystem-requirements.rst b/docs/installing-open-mpi/filesystem-requirements.rst
new file mode 100644
index 00000000000..d6dbd0690c1
--- /dev/null
+++ b/docs/installing-open-mpi/filesystem-requirements.rst
@@ -0,0 +1,24 @@
+Filesystem requirements
+=======================
+
+.. _install-filesystem-timestamp-warning-label:
+
+.. warning:: If you are building Open MPI on a network filesystem, the
+   machine you on which you are building *must* be time-synchronized
+   with the file server.
+
+Specifically: Open MPI's build system *requires* accurate filesystem
+timestamps.  If your ``make`` output shows that it ran GNU Automake,
+Autoconf, and/or Libtool, or includes warning about timestamps in the
+future, perhaps looking something like this::
+
+   Warning: File `Makefile.am' has modification time 3.6e+04 s in the future
+
+**Know that this is not normal**, and you likely have an invalid
+build.
+
+In this case, you should remove the Open MPI source directory and
+start over (e.g., by re-extracting the Open MPI tarball): either
+switch to build on a local filesystem, or ensure that the time on your
+build machine is synchronized with the time on your file server before
+building again.
diff --git a/docs/installing-open-mpi/index.rst b/docs/installing-open-mpi/index.rst
new file mode 100644
index 00000000000..ee6f1f10db2
--- /dev/null
+++ b/docs/installing-open-mpi/index.rst
@@ -0,0 +1,24 @@
+.. _building-and-installing-section-label:
+
+Building and installing Open MPI
+================================
+
+.. toctree::
+   :maxdepth: 1
+
+   quickstart
+   downloading
+   supported-systems
+   definitions
+   filesystem-requirements
+   vpath-builds
+   compilers-and-flags
+   required-support-libraries
+   configure-cli-options/index
+   configure-output-summary
+   make-targets
+   installation-location
+   custom-components
+   updating
+   packagers
+   sysadmins
diff --git a/docs/installing-open-mpi/installation-location.rst b/docs/installing-open-mpi/installation-location.rst
new file mode 100644
index 00000000000..ee90b5bb432
--- /dev/null
+++ b/docs/installing-open-mpi/installation-location.rst
@@ -0,0 +1,354 @@
+
+.. _building-open-mpi-installation-location-label:
+
+Installation location
+=====================
+
+A common environment to run Open MPI is in a "Beowulf"-class or
+similar cluster (e.g., a bunch of 1U servers in a bunch of racks).
+Simply stated, Open MPI can run on a group of servers or workstations
+connected by a network.  As mentioned in the
+:ref:`prerequisites section <running-prerequisites-label>` there are
+several caveats however (for example, you typically must have an
+account on all the machines, you can ``ssh`` between the
+nodes without using a password, etc.).
+
+Regardless of whether Open MPI is installed on a shared / networked
+filesystem or independently on each node, it is usually easiest if
+Open MPI is available in the same filesystem path on every node.
+For example, if you install Open MPI to ``/opt/openmpi-|ompi_ver|`` on
+one node, ensure that it is available in ``/opt/openmpi-|ompi_ver|``
+on *all* nodes.
+
+.. important:: For simplicity, the Open MPI team *strongly* recommends
+   that you install Open MPI at the same path location on all nodes in
+   your cluster.  This *greatly* simplifies the user experience of
+   running MPI jobs across multiple nodes in your cluster.
+
+   It is *possible* to install Open MPI in unique path locations in
+   the different nodes in your cluster, but it is not *advisable*.
+
+This raises the question for Open MPI system administrators: where to
+install the Open MPI binaries, header files, etc.?  This discussion
+mainly addresses this question for homogeneous clusters (i.e., where
+all nodes and operating systems are the same), although elements of
+this discussion apply to heterogeneous clusters as well.
+
+Filesystem types
+----------------
+
+There are two common approaches.
+
+Network filesystem
+^^^^^^^^^^^^^^^^^^
+
+Have a common filesystem, such as NFS, between all the machines to be
+used.  Install Open MPI such that the installation directory is the
+*same value* on each node.  This will *greatly* simplify user's shell
+startup scripts (e.g., ``.bashrc``, ``.cshrc``, ``.profile`` etc.)
+|mdash| the ``PATH`` can be set without checking which machine the
+user is on.  It also simplifies the system administrator's job; when
+the time comes to patch or otherwise upgrade Open MPI, only one copy
+needs to be modified.
+
+For example, consider a cluster of four machines: ``inky``,
+``blinky``, ``pinky``, and ``clyde``.
+
+* Install Open MPI on ``inky``'s local hard drive in the directory
+  ``/opt/openmpi-VERSION``.  The system administrator then mounts
+  ``inky:/opt/openmpi-VERSION`` on the remaining three machines, such
+  that ``/opt/openmpi-VERSION`` on all machines is effectively "the
+  same".  That is, the following directories all contain the Open MPI
+  installation:
+
+  .. code-block::
+
+     inky:/opt/openmpi-VERSION
+     blinky:/opt/openmpi-VERSION
+     pinky:/opt/openmpi-VERSION
+     clyde:/opt/openmpi-VERSION
+
+* Install Open MPI on ``inky``'s local hard drive in the directory
+  ``/usr/local/openmpi-VERSION``.  The system administrator then
+  mounts ``inky:/usr/local/openmpi-VERSION`` on *all four* machines in
+  some other common location, such as ``/opt/openmpi-VERSION`` (a
+  symbolic link can be installed on ``inky`` instead of a mount point
+  for efficiency).  This strategy is typically used for environments
+  where one tree is NFS exported, but another tree is typically used
+  for the location of actual installation.  For example, the following
+  directories all contain the Open MPI installation:
+
+  .. code-block::
+
+     inky:/opt/openmpi-VERSION
+     blinky:/opt/openmpi-VERSION
+     pinky:/opt/openmpi-VERSION
+      clyde:/opt/openmpi-VERSION
+
+  Notice that there are the same four directories as the previous
+  example, but on ``inky``, the directory is *actually* located in
+  ``/usr/local/openmpi-VERSION``.
+
+There is a bit of a disadvantage in this approach; each of the remote
+nodes have to incur NFS (or whatever filesystem is used) delays to
+access the Open MPI directory tree.  However, both the administration
+ease and low cost (relatively speaking) of using a networked file
+system usually greatly outweighs the cost.  Indeed, once an MPI
+application is past MPI initialization, it doesn't use the Open MPI
+binaries very much.
+
+Local filesystem
+^^^^^^^^^^^^^^^^
+
+If you are concerned with networked filesystem costs of accessing the
+Open MPI binaries, you can install Open MPI on the local hard drive of
+each node in your system.  Again, it is *highly* advisable to install
+Open MPI in the *same* directory on each node so that each user's
+``PATH`` can be set to the same value, regardless of the node that a
+user has logged on to.
+
+This approach will save some network latency of accessing the Open MPI
+binaries, but is typically only used where users are very concerned
+about squeezing every single cycle out of their machines, or are
+running at extreme scale where a networked filesystem may get
+overwhelmed by filesystem requests for Open MPI binaries when running
+very large parallel jobs.
+
+.. _building-open-mpi-install-overwrite-label:
+
+Installing over a prior Open MPI installation
+---------------------------------------------
+
+.. warning:: The Open MPI team does not recommend installing a new
+   version of Open MPI over an existing / older installation of Open
+   MPI.
+
+In its default configuration, an Open MPI installation consists of
+several shared libraries, header files, executables, and plugins
+(dynamic shared objects |mdash| DSOs).  These installation files act
+together as a single entity.  The specific filenames and
+contents of these files are subject to change between different
+versions of Open MPI.
+
+.. important:: Installing one version of Open MPI does *not* uninstall
+   another version.
+
+If you install a new version of Open MPI over an older version, this
+may not overwrite all the files from the older version.  Hence, you
+may end up with an incompatible muddle of files from two different
+installations |mdash| which can cause problems.
+
+See :ref:`updating Open MPI <building-open-mpi-updating-label>` for more
+information about updating or upgrading an installation of Open MPI.
+
+Relocating an Open MPI installation
+-----------------------------------
+
+It can be desirable to initially install Open MPI to one location
+(e.g., ``/path/to/openmpi``) and then later move it to another
+location (e.g., ``/opt/myproduct/bundled-openmpi-a.b.c``).
+
+.. note:: Open MPI hard-codes some directory paths in its executables
+          based on installation paths specified by the ``configure``
+          script.  For example, if you configure with an installation
+          prefix of ``/opt/openmpi/``, Open MPI encodes in its
+          executables that it should be able to find its help files in
+          ``/opt/openmpi/share/openmpi``.
+
+The "installdirs" functionality in Open MPI lets you change any of
+these hard-coded directory paths at run time (*assuming* that you have
+already adjusted your ``PATH`` and/or ``LD_LIBRARY_PATH`` environment
+variables to the new location where Open MPI now resides).
+
+There are three methods.
+
+.. _install-location-opal-prefix:
+
+Move an existing Open MPI installation to a new prefix
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Set the ``OPAL_PREFIX`` environment variable before launching Open
+MPI.  For example, if Open MPI had initially been installed to
+``/opt/openmpi`` and the entire ``openmpi`` tree was later moved to
+``/home/openmpi``, setting ``OPAL_PREFIX`` to ``/home/openmpi`` will
+enable Open MPI to function properly.
+
+.. note:: The ``OPAL_PREFIX`` mechanism relies on all installation
+          directories being specified as relative to the ``prefix``
+          directory specified during ``configure``.
+
+          For example, if Open MPI is configured the following way:
+
+          .. code-block::
+
+             $ ./configure --prefix=/opt/openmpi --libdir=/usr/lib ...
+
+          Then setting ``OPAL_PREFIX`` will not affect the run-time
+          implications of ``libdir``, since ``/usr/lib`` is not
+          specified as relative to ``/opt/openmpi``.
+
+          Instead of specifying absolute directories, you can make
+          them relative to other ``configure``-recognized directories.
+          For example:
+
+          .. code-block::
+
+             $ ./configure --prefix=/opt/openmpi --libdir='${exec_prefix}/x86_64/lib' ...
+
+          Note the additional shell quoting that is likely necessary
+          to prevent shell variable expansion, and the additional
+          ``${}`` around ``exec_prefix`` that is necessary for Open MPI
+          to recognize that it is a special name that needs to be
+          expanded.
+
+          The directory names recognized by Open MPI are listed in the
+          :ref:`Overriding individual directories
+          <install-location-overriding-individual-directories>`
+          section (below), without the ``OPAL_`` prefix, and in lower
+          case.  For example, the ``OPAL_SYSCONFDIR`` environment
+          variable corresponds to ``${sysconfdir}``.
+
+"Stage" an Open MPI installation in a temporary location
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When *creating* self-contained installation packages, systems such as
+RPM install Open MPI into temporary locations.  The package system
+then bundles up everything under the temporary location into a package
+that can be installed into its real location later.  For example, when
+*creating* an RPM that will be installed to ``/opt/openmpi``, the RPM
+system will transparently prepend a "destination directory" (or
+"destdir") to the installation directory.  As such, Open MPI will
+think that it is installed in ``/opt/openmpi``, but it is actually
+temporarily installed in (for example)
+``/var/rpm/build.1234/opt/openmpi``.  If it is necessary to *use* Open
+MPI while it is installed in this staging area, the ``OPAL_DESTDIR``
+environment variable can be used; setting ``OPAL_DESTDIR`` to
+``/var/rpm/build.1234`` will automatically prefix every directory such
+that Open MPI can function properly.
+
+.. _install-location-overriding-individual-directories:
+
+Overriding individual directories
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Open MPI uses the GNU-specified directories (per Autoconf/Automake),
+and can be overridden by setting environment variables directly
+related to their common names.  The list of environment variables that
+can be used is:
+
+* ``OPAL_PREFIX``
+* ``OPAL_EXEC_PREFIX``
+* ``OPAL_BINDIR``
+* ``OPAL_SBINDIR``
+* ``OPAL_LIBEXECDIR``
+* ``OPAL_DATAROOTDIR``
+* ``OPAL_DATADIR``
+* ``OPAL_SYSCONFDIR``
+* ``OPAL_SHAREDSTATEDIR``
+* ``OPAL_LOCALSTATEDIR``
+* ``OPAL_LIBDIR``
+* ``OPAL_INCLUDEDIR``
+* ``OPAL_INFODIR``
+* ``OPAL_MANDIR``
+* ``OPAL_PKGDATADIR``
+* ``OPAL_PKGLIBDIR``
+* ``OPAL_PKGINCLUDEDIR``
+
+Note that not all of the directories listed above are used by Open
+MPI; they are listed here in entirety for completeness.
+
+Also note that several directories listed above are defined in terms
+of other directories.  For example, the ``$bindir`` is defined by
+default as ``$prefix/bin``.  Hence, overriding the ``$prefix`` (via
+``OPAL_PREFIX``) will automatically change the first part of the
+``$bindir`` (which is how method 1 described above works).
+Alternatively, ``OPAL_BINDIR`` can be set to an absolute value that
+ignores ``$prefix`` altogether.
+
+.. _building-open-mpi-installation-location-multiple-copies-label:
+
+Installing Multiple Copies of Open MPI
+--------------------------------------
+
+Open MPI can handle a variety of different run-time environments
+(e.g., ssh, Slurm, PBS, etc.) and a variety of different
+interconnection networks (e.g., ethernet, InfiniBand, etc.)
+in a single installation.  Specifically: because Open MPI is
+fundamentally powered by a component architecture, plug-ins for all
+these different run-time systems and interconnect networks can be
+installed in a single installation tree.  The relevant plug-ins will
+only be used in the environments where they make sense.
+
+Hence, there is no need to have one MPI installation for InfiniBand, one
+MPI installation for ethernet, one MPI installation for PBS, one MPI
+installation for ``ssh``, etc.  Open MPI can handle all of these in a
+single installation.
+
+However, there are some issues that Open MPI cannot solve.  Binary
+compatibility between different compilers is such an issue and may require
+installation of multiple versions of Open MPI.
+
+Let's examine this on a per-language basis (be sure see the big caveat at
+the end):
+
+* *C:* Most C compilers are fairly compatible, such that if you compile
+  Open MPI with one C library and link it to an application that was
+  compiled with a different C compiler, everything should "just work."
+  As such, a single installation of Open MPI should work for most C MPI
+  applications.
+
+* *C++:* The same is not necessarily true for C++.  While Open MPI does not
+  currently contain any C++ code (the MPI C++ bindings were removed in a prior
+  release), and C++ compilers *should* produce ABI-equivalent code for C
+  symbols, obscure problem can sometimes arise when mixing compilers from
+  different suites.  For example, if you compile Open MPI with the XYZ C/C++
+  compiler, you may need to have the XYC C++ run-time libraries
+  installed everywhere you want to run.
+
+* *Fortran:* There are multiple issues with Fortran.
+
+    #. Fortran compilers do something called "symbol mangling," meaning that the
+       back-end symbols may have slightly different names than their corresponding
+       global variables, subroutines, and functions.  There are 4 common name
+       mangling schemes in use by Fortran compilers.  On many systems (e.g.,
+       Linux), Open MPI will automatically support all 4 schemes.  As such, a
+       single Open MPI installation *should* just work with multiple different
+       Fortran compilers.  However, on some systems, this is not possible (e.g.,
+       OS X), and Open MPI will only support the name mangling scheme of the
+       Fortran compiler that was identified during ``configure``.
+
+    #. That being said, there are two notable exceptions that do *not* work
+       across Fortran compilers that are "different enough":
+
+        #. The C constants ``MPI_F_STATUS_IGNORE`` and ``MPI_F_STATUSES_IGNORE``
+             will only compare properly to Fortran applications that were
+             created with Fortran compilers that that use the same
+             name-mangling scheme as the Fortran compiler with which Open MPI was
+             configured.
+
+        #. Fortran compilers may have different values for the logical
+             ``.TRUE.`` constant.  As such, any MPI function that uses the
+             Fortran ``LOGICAL`` type may only get ``.TRUE.`` values back that
+             correspond to the the ``.TRUE.`` value of the Fortran compiler with which
+             Open MPI was configured.
+
+    #. Similar to C++, linking object files that Fortran language features such
+       as modules and/or polymorphism from different
+       Fortran compilers is not likely to work.  The ``mpi`` and ``mpi_f08`` modules that
+       Open MPI creates will likely only work with the Fortran compiler
+       that was identified during ``configure`` (and used to build Open MPI).
+
+The big caveat to all of this is that Open MPI will only work with
+different compilers *if all the datatype sizes are the same.*  For
+example, even though Open MPI supports all 4 name mangling schemes,
+the size of the Fortran ``LOGICAL`` type may be 1 byte in some compilers
+and 4 bytes in others.  This will likely cause Open MPI to perform
+unpredictably.
+
+The bottom line is that Open MPI can support all manner of run-time
+systems and interconnects in a single installation, but supporting
+multiple compilers "sort of" works (i.e., is subject to trial and
+error) in some cases, and definitely does not work in other cases.
+There's unfortunately little that we can do about this |mdash| it's a
+compiler compatibility issue, and one that compiler authors have
+little incentive to resolve.
diff --git a/docs/installing-open-mpi/make-targets.rst b/docs/installing-open-mpi/make-targets.rst
new file mode 100644
index 00000000000..efcf801a6c3
--- /dev/null
+++ b/docs/installing-open-mpi/make-targets.rst
@@ -0,0 +1,13 @@
+``make`` targets
+================
+
+Open MPI supports all the ``make`` targets that are provided by GNU
+Automake, such as:
+
+* ``all``: build the entire Open MPI package
+* ``install``: install Open MPI
+* ``uninstall``: remove all traces of Open MPI from the installation tree
+* ``clean``: clean out the build tree
+
+Once Open MPI has been built and installed, it is safe to run ``make
+clean`` and/or remove the entire build tree.
diff --git a/docs/installing-open-mpi/packagers.rst b/docs/installing-open-mpi/packagers.rst
new file mode 100644
index 00000000000..c0fb13ea23a
--- /dev/null
+++ b/docs/installing-open-mpi/packagers.rst
@@ -0,0 +1,318 @@
+.. _label-install-packagers:
+
+Advice for packagers
+====================
+
+.. _label-install-packagers-do-not-use-internal:
+
+Do not use Open MPI's internal dependent libraries
+--------------------------------------------------
+
+The Open MPI community **strongly** suggests that binary Open MPI
+packages should *not* include Hwloc, Libevent, PMIx, or PRRTE.
+:ref:`Although several of these libraries are required by Open MPI
+<label-install-required-support-libraries>` (and are therefore bundled
+in the Open MPI source code distribution for end-user convenience),
+binary Open MPI packages should limit themselves solely to Open MPI
+artifacts.  Specifically: ensure to configure and build Open MPI
+against external installations of these required packages.
+
+Packagers may therefore wish to configure Open MPI with something like
+the following:
+
+.. code-block:: sh
+
+   # Install Sphinx so that Open MPI can re-build its docs with the
+   # installed PRRTE's docs
+
+   virtualalenv venv
+   . ./venv/bin/activate
+   pip install docs/requirements.txt
+
+   ./configure --with-libevent=external --with-hwloc=external \
+       --with-pmix=external --with-prrte=external ...
+
+.. important:: Note the installation of the Sphinx tool so that Open
+               MPI can re-build its documentation with the external
+               PRRTE's documentation.
+
+               Failure to do this will mean Open MPI's documentation
+               will be correct for the version of PRRTE that is
+               bundled in the Open MPI distribution, but may not be
+               entirely correct for the version of PRRTE that you are
+               building against.
+
+The ``external`` keywords will force Open MPI's ``configure`` to
+ignore all the bundled libraries and only look for external versions
+of these support libraries.  This also has the benefit of causing
+``configure`` to fail if it cannot find the required support libraries
+outside of the Open MPI source tree |mdash| a good sanity check to
+ensure that your package is correctly relying on the
+independently-built and installed versions.
+
+:ref:`See this section
+<label-building-ompi-cli-options-required-support-libraries>` for more
+information about the required support library ``--with-FOO`` command
+line options.
+
+Have Sphinx installed
+---------------------
+
+Since you should be (will be) installing Open MPI against an external
+PRRTE and PMIx, you should have `Sphinx
+<https://www.sphinx-doc.org/>`_ installed before running Open MPI's
+``configure`` script.
+
+This will allow Open MPI to (re-)build its documentation according to
+the PMIx and PRRTE that you are building against.
+
+To be clear: the Open MPI distribution tarball comes with pre-built
+documentation |mdash| rendered in HTML and nroff |mdash| that is
+suitable for the versions of PRRTE and PMIx that are bundled in that
+tarball.
+
+However, if you are building Open MPI against not-bundled versions of
+PRRTE / PMIx (as all packagers should be), Open MPI needs to re-build
+its documentation with specific information from those external PRRTE
+/ PMIx installs.  For that, you need to have Sphinx installed before
+running Open MPI's ``configure`` script.
+
+
+.. _label-install-packagers-dso-or-not:
+
+Components ("plugins"): static or DSO?
+--------------------------------------
+
+Open MPI contains a large number of components (sometimes called
+"plugins") to effect different types of functionality in MPI.  For
+example, some components effect Open MPI's networking functionality:
+they may link against specialized libraries to provide
+highly-optimized network access.
+
+Open MPI can build its components as Dynamic Shared Objects (DSOs) or
+statically included in core libraries (regardless of whether those
+libraries are built as shared or static libraries).
+
+.. note:: As of Open MPI |ompi_ver|, ``configure``'s global default is
+          to build all components as static (i.e., part of the Open
+          MPI core libraries, not as DSOs).  Prior to Open MPI v5.0.0,
+          the global default behavior was to build most components as
+          DSOs.
+
+Why build components as DSOs?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+There are advantages to building components as DSOs:
+
+* Open MPI's core libraries |mdash| and therefore MPI applications
+  |mdash| will have very few dependencies.  For example, if you build
+  Open MPI with support for a specific network stack, the libraries in
+  that network stack will be dependencies of the DSOs, not Open MPI's
+  core libraries (or MPI applications).
+
+* Removing Open MPI functionality that you do not want is as simple as
+  removing a DSO from ``$libdir/open-mpi``.
+
+Why build components as part of Open MPI's core libraries?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The biggest advantage to building the components as part of Open MPI's
+core libraries is when running at (very) large scales when Open MPI is
+installed on a network filesystem (vs. being installed on a local
+filesystem).
+
+For example, consider launching a single MPI process on each of 1,000
+nodes.  In this scenario, the following is accessed from the network
+filesystem:
+
+#. The MPI application
+#. The core Open MPI libraries and their dependencies (e.g.,
+   ``libmpi``)
+
+   * Depending on your configuration, this is probably on the order of
+     10-20 library files.
+
+#. All DSO component files and their dependencies
+
+   * Depending on your configuration, this can be 200+ component
+     files.
+
+If all components are physically located in the libraries, then the
+third step loads zero DSO component files.  When using a networked
+filesystem while launching at scale, this can translate to large
+performance savings.
+
+.. note:: If not using a networked filesystem, or if not launching at
+          scale, loading a large number of DSO files may not consume a
+          noticeable amount of time during MPI process launch.  Put
+          simply: loading DSOs as indvidual files generally only
+          matters when using a networked filesystem while launching at
+          scale.
+
+Direct controls for building components as DSOs or not
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Open MPI |ompi_ver| has two ``configure``-time defaults regarding the
+treatment of components that may be of interest to packagers:
+
+#. Open MPI's libraries default to building as shared libraries
+   (vs. static libraries).  For example, on Linux, Open MPI will
+   default to building ``libmpi.so`` (vs. ``libmpi.a``).
+
+   .. note:: See the descriptions of ``--disable-shared`` and
+             ``--enable-static`` :ref:`in this section
+             <label-building-installation-cli-options>` for more
+             details about how to change this default.
+
+             Also be sure to :ref:`see this warning about building
+             static apps <label-building-fully-static-apps>`.
+
+#. Open MPI will default to including its components in its libraries
+   (as opposed to being compiled as dynamic shared objects, or DSOs).
+   For example, ``libmpi.so`` on Linux systems will contain the UCX
+   PML component, instead of the UCX PML being compiled into
+   ``mca_pml_ucx.so`` and dynamically opened at run time via
+   ``dlopen(3)``.
+
+   .. note:: See the descriptions of ``--enable-mca-dso`` and
+             ``--enable-mca-static`` :ref:`in this section
+             <label-building-installation-cli-options>` for more
+             details about how to change this defaults.
+
+A side effect of these two defaults is that all the components
+included in the Open MPI libraries will bring their dependencies with
+them.  For example (on Linux), if the XYZ PML component in the MPI
+layer requires ``libXYZ.so``, then these defaults mean that
+``libmpi.so`` will depend on ``libXYZ.so``.  This dependency will
+likely be telegraphed into the Open MPI binary package that includes
+``libmpi.so``.
+
+Conversely, if the XYZ PML component was built as a DSO, then |mdash|
+assuming no other parts of Open MPI require ``libXYZ.so`` |mdash|
+``libmpi.so`` would *not* be dependent on ``libXYZ.so``.  Instead, the
+``mca_pml_xyz.so`` DSO would have the dependency upon ``libXYZ.so``.
+
+Packagers can use these facts to potentially create multiple binary
+Open MPI packages, each with different dependencies by, for example,
+using ``--enable-mca-dso`` to selectively build some components as
+DSOs and leave the others included in their respective Open MPI
+libraries.
+
+:ref:`See the section on building accelerator support
+<label-install-packagers-building-accelerator-support-as-dsos>` for a
+practical example where this can be useful.
+
+.. _label-install-packagers-gnu-libtool-dependency-flattening:
+
+GNU Libtool dependency flattening
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When compiling Open MPI's components statically as part of Open MPI's
+core libraries, `GNU Libtool <https://www.gnu.org/software/libtool/>`_
+|mdash| which is used as part of Open MPI's build system |mdash| will
+attempt to "flatten" dependencies.
+
+For example, the :ref:`ompi_info(1) <man1-ompi_info>` command links
+against the Open MPI core library ``libopen-pal``.  This library will
+have dependencies on various HPC-class network stack libraries. For
+simplicity, the discussion below assumes that Open MPI was built with
+support for `Libfabric <https://libfabric.org/>`_ and `UCX
+<https://openucx.org/>`_, and therefore ``libopen-pal`` has direct
+dependencies on ``libfabric`` and ``libucx``.
+
+In this scenario, GNU Libtool will automatically attempt to "flatten"
+these dependencies by linking :ref:`ompi_info(1) <man1-ompi_info>`
+directly to ``libfabric`` and ``libucx`` (vs. letting ``libopen-pal``
+pull the dependencies in at run time).
+
+* In some environments (e.g., Ubuntu 22.04), the compiler and/or
+  linker will automatically utilize the linker CLI flag
+  ``-Wl,--as-needed``, which will effectively cause these dependencies
+  to *not* be flattened: :ref:`ompi_info(1) <man1-ompi_info>` will
+  *not* have a direct dependencies on either ``libfabric`` or
+  ``libucx``.
+
+* In other environments (e.g., Fedora 38), the compiler and linker
+  will *not* utilize the ``-Wl,--as-needed`` linker CLI flag.  As
+  such, :ref:`ompi_info(1) <man1-ompi_info>` will show direct
+  dependencies on ``libfabric`` and ``libucx``.
+
+**Just to be clear:** these flattened dependencies *are not a
+problem*.  Open MPI will function correctly with or without the
+flattened dependencies.  There is no performance impact associated
+with having |mdash| or not having |mdash| the flattened dependencies.
+We mention this situation here in the documentation simply because it
+surprised some Open MPI downstream package managers to see that
+:ref:`ompi_info(1) <man1-ompi_info>` in Open MPI |ompi_ver| had more
+shared library dependencies than it did in prior Open MPI releases.
+
+If packagers want :ref:`ompi_info(1) <man1-ompi_info>` to not have
+these flattened dependencies, use either of the following mechanisms:
+
+#. Use ``--enable-mca-dso`` to force all components to be built as
+   DSOs (this was actually the default behavior before Open MPI v5.0.0).
+
+#. Add ``LDFLAGS=-Wl,--as-needed`` to the ``configure`` command line
+   when building Open MPI.
+
+   .. note:: The Open MPI community specifically chose not to
+             automatically utilize this linker flag for the following
+             reasons:
+
+             #. Having the flattened dependencies does not cause any
+                correctness or performance problems.
+             #. There's multiple mechanisms (see above) for users or
+                packagers to change this behavior, if desired.
+             #. Certain environments have chosen to have |mdash| or
+                not have |mdash| this flattened dependency behavior.
+                It is not Open MPI's place to override these choices.
+             #. In general, Open MPI's ``configure`` script only
+                utilizes compiler and linker flags if they are
+                *needed*.  All other flags should be the user's /
+                packager's choice.
+
+.. _label-install-packagers-building-accelerator-support-as-dsos:
+
+Building accelerator support as DSOs
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you are building a package that includes support for one or more
+accelerators, it may be desirable to build accelerator-related
+components as DSOs (see the :ref:`static or DSO?
+<label-install-packagers-dso-or-not>` section for details).
+
+.. admonition:: Rationale
+   :class: tip
+
+   Accelerator hardware is expensive, and may only be present on some
+   compute nodes in an HPC cluster.  Specifically: there may not be
+   any accelerator hardware on "head" or compile nodes in an HPC
+   cluster.  As such, invoking Open MPI commands on a "head" node with
+   an MPI that was built with static accelerator support but no
+   accelerator hardware may fail to launch because of run-time linker
+   issues (because the accelerator hardware support libraries are
+   likely not present).
+
+   Building Open MPI's accelerator-related components as DSOs allows
+   Open MPI to *try* opening the accelerator components, but proceed
+   if those DSOs fail to open due to the lack of support libraries.
+
+Use the ``--enable-mca-dso`` command line parameter to Open MPI's
+``configure`` command can allow packagers to build all
+accelerator-related components as DSO.  For example:
+
+.. code:: sh
+
+   # Build all the accelerator-related components as DSOs (all other
+   # components will default to being built in their respective
+   # libraries)
+   shell$ ./configure --enable-mca-dso=btl-smcuda,rcache-rgpusm,rcache-gpusm,accelerator
+
+Per the example above, this allows packaging ``$libdir`` as part of
+the "main" Open MPI binary package, but then packaging
+``$libdir/openmpi/mca_accelerator_*.so`` and the other named
+components as sub-packages.  These sub-packages may inherit
+dependencies on the CUDA and/or ROCM packages, for example.  The
+"main" package can be installed on all nodes, and the
+accelerator-specific subpackage can be installed on only the nodes
+with accelerator hardware and support libraries.
diff --git a/docs/installing-open-mpi/quickstart.rst b/docs/installing-open-mpi/quickstart.rst
new file mode 100644
index 00000000000..53ca0b2838f
--- /dev/null
+++ b/docs/installing-open-mpi/quickstart.rst
@@ -0,0 +1,101 @@
+.. _label-quickstart-building-open-mpi:
+
+Quick start: Installing Open MPI
+================================
+
+Although this section skips many details, it offers examples that will
+probably work in many environments.
+
+.. caution:: Note that this section is a "Quick start" |mdash| it does
+   not attempt to be comprehensive or describe how to build Open MPI
+   in all supported environments.  The examples below may therefore
+   not work exactly as shown in your environment.
+
+   Please consult the other sections in this chapter for more details,
+   if necessary.
+
+.. important:: If you have checked out a *developer's copy* of Open MPI
+   (i.e., you cloned from Git), you really need to read :doc:`the
+   Developer's Guide </developers/index>` before attempting to build Open
+   MPI. Really.
+
+Binary packages
+---------------
+
+Although the Open MPI community itself does not distribute binary
+packages for Open MPI, many downstream packagers do.
+
+For example, many Linux distributions include Open MPI packages
+|mdash| even if they are not installed by default.  You should consult
+the documentation and/or package list for your Linux distribution to
+see if you can use its built-in package system to install Open MPI.
+
+The MacOS package managers `Homebrew <https://brew.sh/>`_ and
+`MacPorts <https://macports.org/>`_ both offer binary Open MPI
+packages:
+
+.. code-block:: sh
+
+   # For Homebrew
+   shell$ brew install openmpi
+
+   # For MacPorts
+   shell$ port install openmpi
+
+.. important:: Binary packages may or may not include support for
+               features that are required on your platform (e.g., a
+               specific networking stack).  Or the binary packages
+               available to you may be older / out of date.  As such,
+               it may be better to build and install Open MPI from a
+               source tarball available from `the main Open MPI web
+               site <https://www.open-mpi.org/>`_.
+
+Building from source
+--------------------
+
+Download the Open MPI source code from `the main Open MPI web site
+<https://www.open-mpi.org/>`_.
+
+.. caution:: Do **not** download an Open MPI source code tarball from
+             GitHub.com.  The tarballs automatically generated by
+             GitHub.com are incomplete and will not build properly.
+             They are **not** official Open MPI releases.
+
+Open MPI uses a traditional ``configure`` script paired with ``make``
+to build.  Typical installs can be of the pattern:
+
+.. code-block:: sh
+
+   shell$ tar xf openmpi-<version>.tar.bz2
+   shell$ cd openmpi-<version>
+   shell$ ./configure --prefix=<path> [...options...] 2>&1 | tee config.out
+   <... lots of output ...>
+
+   # Use an integer value of N for parallel builds
+   shell$ make [-j N] all 2>&1 | tee make.out
+
+   # ...lots of output...
+
+   # Depending on the <prefix> chosen above, you may need root access
+   # for the following:
+   shell$ make install 2>&1 | tee install.out
+
+   # ...lots of output...
+
+Note that VPATH builds are fully supported.  For example:
+
+.. code-block:: sh
+
+   shell$ tar xf openmpi-<version>.tar.bz2
+   shell$ cd openmpi-<version>
+   shell$ mkdir build
+   shell$ cd build
+   shell$ ../configure --prefix=<path> 2>&1 | tee config.out
+   # ...etc.
+
+The above patterns can be used in many environments.
+
+Note that there are many, many configuration options available in the
+``./configure`` step.  Some of them may be needed for your particular
+HPC network interconnect type and/or computing environment; see the
+rest of this chapter for descriptions of the available options.
diff --git a/docs/installing-open-mpi/required-support-libraries-dependency-graph.png b/docs/installing-open-mpi/required-support-libraries-dependency-graph.png
new file mode 100644
index 00000000000..c32e3b9eed1
Binary files /dev/null and b/docs/installing-open-mpi/required-support-libraries-dependency-graph.png differ
diff --git a/docs/installing-open-mpi/required-support-libraries-dependency-graph.pptx b/docs/installing-open-mpi/required-support-libraries-dependency-graph.pptx
new file mode 100644
index 00000000000..61a2bb6a484
Binary files /dev/null and b/docs/installing-open-mpi/required-support-libraries-dependency-graph.pptx differ
diff --git a/docs/installing-open-mpi/required-support-libraries.rst b/docs/installing-open-mpi/required-support-libraries.rst
new file mode 100644
index 00000000000..983608a3fb9
--- /dev/null
+++ b/docs/installing-open-mpi/required-support-libraries.rst
@@ -0,0 +1,494 @@
+.. _label-install-required-support-libraries:
+
+Required support libraries
+==========================
+
+
+While Open MPI can be built with support for a wide variety of
+systems, a small set of support libraries are *required* in order to
+build Open MPI in *any* environment.  Several of these packages are
+both fundamental to Open MPI's operation and not universally available
+in all environments.  As such, these "fundamental" packages are both
+embedded in Open MPI's distribution tarballs and also directly
+incorporated into Open MPI's configure, build, and installation
+process.
+
+:ref:`See below
+<required-support-libraries-configure-discovery-label>` for a
+description of how Open MPI chooses whether to use the embedded
+versions of these packages or versions already installed on your
+system.
+
+* `Hardware Locality <https://www.open-mpi.org/projects/hwloc/>`_
+
+  * This library is required; Open MPI will not build without it.
+  * **Minimum version required:** |hwloc_min_version|
+
+    .. note:: While the minimum version of Hwloc will *work*, it may
+              still have bugs and/or have less functionality as
+              compared to later versions.
+
+              Other than the Hwloc restriction about v3.0.0 and beyond
+              (see below), the Open MPI community generally recomends
+              using the latest available version of Hwloc unless there
+              is a specific reason not to.
+
+  * **Version embedded in Open MPI distribution:**
+    |hwloc_embedded_version|
+
+  .. danger:: As of |ompi_ver|, Open MPI does not yet support the
+              Hwloc v3.x series (which may not even be available at
+              the time of Open MPI |ompi_ver|'s release).  Hwloc v3.x
+              is anticipated to break API and/or ABI compared to the
+              Hwloc v2.x series.
+
+              Open MPI will refuse to build if it finds an external
+              Hwloc installation that is >= v3.0.0 on the assumption
+              that other HPC applications and/or libraries may be
+              using it.  Such a configuration could lead to obscure
+              and potentially confusing run-time failures of Open MPI
+              applications.
+
+              If Open MPI's ``configure`` script aborts because it
+              finds an Hwloc installation that is >= v3.0.0, you can
+              either ensure that Open MPI finds a < v3.0.0 Hwloc
+              installation (e.g., by changing the order of paths in
+              ``LD_LIBRARY_PATH``), or force the use of Open MPI's
+              bundled Hwloc via:
+
+              .. code::
+
+                 shell$ ./configure --with-hwloc=internal ...
+
+              Regardless, *it is critically important* that if an MPI
+              application |mdash| or any of its dependencies |mdash|
+              uses Hwloc, it uses the *same* Hwloc with which Open MPI
+              was compiled.
+
+* `Libevent <https://libevent.org/>`_
+
+  * This library is required; Open MPI will not build without it.
+  * **Minimum version required:** |event_min_version|
+  * **Version embedded in Open MPI distribution:**
+    |event_embedded_version|
+
+    .. note:: The Open MPI community has heavily tested Libevent
+              |event_embedded_version|.  Other versions should *work*,
+              but |mdash| unlike Hwloc, OpenPMIx, and PRRTE |mdash|
+              there is not much reason to upgrade to use a later
+              version of Libevent.
+
+* `OpenPMIx <https://docs.openpmix.org/>`_
+
+  * This library is required; Open MPI will not build without it.
+  * **Minimum version required when building without PRRTE:**
+    |pmix_min_version|
+
+    .. note:: While the minimum version of OpenPMIx will *work*, it
+              may still have bugs and/or have less functionality as
+              compared to later versions.
+
+              The Open MPI community generally recomends using the
+              latest available version of OpenPMIx unless there is a
+              specific reason not to.
+
+    .. note:: While `OpenPMIx <https://docs.openpmix.org/>`_ is the
+              formal name of the software that implements the `PMIx
+              <https://pmix.org/>`_ standard, the term "PMIx" is used
+              extensively throughout this documentation to refer to
+              the OpenPMIx software package.
+
+  * **Minimum version required when building with PRRTE:** `See the
+    PRRTE project documentation <https://docs.prrte.org/>`_.
+  * **Version embedded in Open MPI distribution:**
+    |pmix_embedded_version|
+
+* `PRRTE <https://github.com/openpmix/prrte>`_
+
+  * This library is optional in some environments. See below.
+  * **Minimum version required:** |prte_min_version|
+
+    .. note:: While the minimum version of PRRTE will *work*, it may
+              still have bugs and/or have less functionality as
+              compared to later versions.  For example, if you build
+              and run with |prte_min_version|, you will not get a
+              fully-populated ``mpirun(1)`` man page.
+
+              The Open MPI community generally recomends using the
+              latest available version of PRRTE unless there is a
+              specific reason not to.
+
+  * **Version embedded in Open MPI distribution:**
+    |prte_embedded_version|
+
+  PRRTE provides Open MPI's full-featured ``mpirun`` / ``mpiexec`` MPI
+  application launchers (the two commands are identical; they are
+  symbolic links to the same executable).
+
+  .. warning:: If you are building the PRRTE that is embedded in the
+               Open MPI |ompi_ver| distribution:
+
+               * If you are also building the PMIx that is embedded in
+                 the Open MPI |ompi_ver| distribution, that
+                 combination of packages is supported.
+
+               * If you are building against an external PMIx
+                 installation (i.e., a version of PMIx that is not
+                 embedded in the Open MPI |ompi_ver| distribution),
+                 you should check `the PRRTE project documentation
+                 <https://docs.prrte.org/>`_ to see what minimum
+                 version of PMIx is required.
+
+  * If your environment uses another MPI application launcher (e.g.,
+    Slurm users can use the ``srun`` launcher to "direct launch" Open
+    MPI applications), then the use of PRRTE is optional.
+  * If your environment has no other MPI application launcher, then
+    you need to install PRRTE and build Open MPI with PRRTE support.
+  * Open MPI can use the copy of PRRTE embedded in its source code
+    tree, or compile/link against an external PRRTE installation.
+    :ref:`See this section for details about how to specify each
+    method
+    <label-building-ompi-cli-options-required-support-libraries>`.
+
+.. note:: In general, the Open MPI community recommends using the most
+          recent versions of both the :ref:`required support libraries
+          <label-install-required-support-libraries>` and any other
+          optional support libraries.  This is because more recent
+          versions typically tend to include bug fixes, sometimes
+          affecting Open MPI functionality.  As a specific example,
+          there is a known issue with `Hardware Locality
+          <https://www.open-mpi.org/projects/hwloc/>`_ releases older
+          than v2.8.0 on systems with Intel Ponte Vecchio
+          accelerators.  If you run Open MPI on such systems, you need
+          to use Hwloc v2.8.0 or newer, or you will experience
+          undefined behavior.  This effect is not unique to the
+          Hardware Locality library; this is why the Open MPI
+          community recommends using as recent as possible versions of
+          *all* support libraries.
+
+Library dependencies
+--------------------
+
+These support libraries have dependencies upon each other:
+
+.. The "source code" for this figure is simple a PPTX file by the same
+   name in this same directory.  If you ever need to edit this image,
+   edit the PPTX, export it to PNG, and then trim the whitespace from
+   the sides of the image.
+
+.. figure:: required-support-libraries-dependency-graph.png
+   :align: center
+
+   Open MPI required support library dependency graph.
+
+The higher-level boxes depend on the lower-level boxes.  Specifically:
+
+* Open MPI depends on PRRTE, PMIx, Hwloc, and Libevent (i.e.,
+  everything).
+* PRRTE depends on PMIx, Hwloc, and Libevent (i.e., everything except
+  Open MPI).
+* PMIx depends on Hwloc and Libevent.
+* Hwloc does not depend on anything.
+* Libevent does not depend on anything.
+
+At run time, it is critical that the run-time linker loads *exactly
+one copy* of each of these libraries.
+
+.. note:: The required support libraries can have other dependencies,
+          but for simplicity and relevance to building Open MPI,
+          those other dependencies are not discussed here.
+
+Potential problems
+------------------
+
+Problems can (will) arise if multiple different copies of the above
+shared libraries are loaded into a single process.  For example,
+consider if:
+
+* Loading the Open MPI shared library causes the loading of Libevent
+  shared library vA.B.C.
+* But then the subsequent loading of the PMIx shared library causes
+  the loading of Libevent shared library vX.Y.Z.
+
+Since there are now two different versions of the Libevent shared
+library loaded into the same process (yes, this can happen!),
+unpredictable behavior can (will) occur.
+
+Many variations on this same basic erroneous scenario are possible.
+All of them are bad, and can be extremely difficult to diagnose.
+
+Avoiding the problems
+---------------------
+
+A simple way to avoid these problems is to configure your system such
+that it has exactly one copy of each of the required support libraries.
+
+.. important:: If possible, use your OS / environment's package
+   manager to install as many of these support libraries |mdash|
+   including their development headers |mdash| as possible before
+   invoking Open MPI's ``configure`` script.
+
+Not all package managers provide all of the required support
+libraries. But even if your package manager installs |mdash| for
+example |mdash| only Libevent and Hwloc, that somewhat simplifies the
+final Open MPI configuration, and therefore avoids some potentially
+erroneous configurations.
+
+.. _required-support-libraries-configure-discovery-label:
+
+How ``configure`` finds the required libraries
+----------------------------------------------
+
+In an attempt to strike a balance between end-user convenience and
+flexibility, Open MPI bundles these four required support libraries in
+its official distribution tarball.
+
+Generally, if Open MPI cannot find a required support library, it will
+automatically configure, build, install, and use its bundled version
+as part of the main Open MPI configure, build, and installation
+process.
+
+Put differently: Open MPI's ``configure`` script will examine the
+build machine and see if it can find each of the required support
+header files and libraries.  If it cannot find them, it will attempt
+to fall back and use the corresponding bundled support library
+instead.
+
+.. important:: Note, however, that ``configure`` is smart enough to
+   understand the dependencies between the required support libraries.
+
+   Specifically: If ``configure`` finds the development headers and
+   libraries for a given support library already installed on the
+   system, then it will ignore both the corresponding bundled support
+   library, *and it will also ignore all bundled support libraries
+   that are below it in the dependency graph shown above.*
+
+Build example 1
+^^^^^^^^^^^^^^^
+
+``configure`` finds the PRRTE development headers and libraries in
+``/usr/local``.  This will cause the following to occur:
+
+#. ``configure`` will ignore the PRRTE library that is bundled in the
+   Open MPI source tree and will use the PRRTE that is already
+   installed in ``/usr/local``.
+#. ``configure`` will also ignore the bundled PMIx, Hwloc, and
+   Libevent libraries in the Open MPI source tree.
+
+   * If ``configure`` is unable to find header files and libraries for
+     PMIx, Hwloc, and Libevent elsewhere on the build machine (i.e.,
+     assumedly the same PMIx, Hwloc, and Libevent than the PRRTE in
+     ``/usr/local`` is using), this is an error: ``configure`` will
+     abort, and therefore refuse to build Open MPI.
+
+Build example 2
+^^^^^^^^^^^^^^^
+
+``configure`` does *not* find PRRTE on the build machine, but *does*
+find PMIx development headers and libraries in ``/opt/local``.  This
+will cause the following to occur:
+
+#. ``configure`` will set up to build the PRRTE library that is
+   bundled in the Open MPI source tree.
+#. ``configure`` will ignore the PMIx library that is bundled in the
+   Open MPI source tree and will use the PMIx that is already
+   installed in ``/opt/local``.
+#. ``configure`` will also ignore the bundled Hwloc and Libevent
+   libraries in the Open MPI source tree.
+
+   * If ``configure`` is unable to find header files and libraries for
+     Hwloc and Libevent elsewhere on the build machine (i.e.,
+     assumedly the same Hwloc and Libevent than the PMIx in
+     ``/opt/local`` is using), this is an error: ``configure`` will
+     abort, and therefore refuse to build Open MPI.
+
+Build example 3
+^^^^^^^^^^^^^^^
+
+``configure`` only finds the development headers and libraries for
+Libevent on the build machine.  This will cause the following to
+occur:
+
+#. ``configure`` will set up to build the PRRTE, PMIx, and Hwloc
+   libraries that are bundled in the Open MPI source tree.
+#. ``configure`` will ignore the Libevent library that is bundled in
+   the Open MPI source tree and will use the Libevent that is already
+   installed.
+
+Difficulties with C and Fortran
+-------------------------------
+
+Sometimes you may see errors similar to the following when attempting
+to build Open MPI:
+
+.. code-block::
+
+   ...
+   PPFC     profile/pwin_unlock_f08.lo
+   PPFC     profile/pwin_unlock_all_f08.lo
+   PPFC     profile/pwin_wait_f08.lo
+   FCLD     libmpi_usempif08.la
+   ld: library not found for -lhwloc
+   collect2: error: ld returned 1 exit status
+   make``2``: *** ``libmpi_usempif08.la`` Error 1
+
+This error can happen when a number of factors occur together:
+
+#. If Open MPI's ``configure`` script chooses to use an "external"
+   installation of `hwloc <https://www.open-mpi.org/projects/hwloc/>`_
+   and/or `Libevent <https://libevent.org/>`_ (i.e., outside of Open
+   MPI's source tree).
+#. If Open MPI's ``configure`` script chooses C and Fortran compilers
+   from different suites/installations.
+
+Put simply: if the default search library search paths differ between
+the C and Fortran compiler suites, the C linker may find a
+system-installed ``libhwloc`` and/or ``libevent``, but the Fortran linker
+may not.
+
+This may tend to happen more frequently starting with Open MPI v4.0.0
+on Mac OS because:
+
+#. In v4.0.0, Open MPI's ``configure`` script was changed to "prefer"
+   system-installed versions of hwloc and Libevent (vs. preferring the
+   hwloc and Libevent that are bundled in the Open MPI distribution
+   tarballs).
+#. In MacOS, it is common for `Homebrew <https://brew.sh/>`_ or
+   `MacPorts <https://www.macports.org/>`_ to install:
+
+   * `Hardware Locality <https://www.open-mpi.org/projects/hwloc/>`_
+   * `Libevent <https://libevent.org/>`_
+
+For example, as of July 2019, Homebrew:
+
+* Installs hwloc v2.0.4 under ``/usr/local``
+* Installs the Gnu C and Fortran compiler suites v9.1.0 under
+  ``/usr/local``.  *However*, the C compiler executable is named ``gcc-9``
+  (not ``gcc``!), whereas the Fortran compiler executable is
+  named ``gfortran``.
+
+These factors, taken together, result in Open MPI's ``configure``
+script deciding the following:
+
+* The C compiler is ``gcc`` (which is the MacOS-installed C
+  compiler).
+* The Fortran compiler is ``gfortran`` (which is the
+  Homebrew-installed Fortran compiler).
+* There is a suitable system-installed hwloc in ``/usr/local``, which
+  can be found |mdash| by the C compiler/linker |mdash| without specifying any
+  additional linker search paths.
+
+The careful reader will realize that the C and Fortran compilers are
+from two entirely different installations.  Indeed, their default
+library search paths are different:
+
+* The MacOS-installed ``gcc`` will search ``/usr/local/lib`` by
+  default.
+* The Homebrew-installed ``gfortran`` will *not* search
+  ``/usr/local/lib`` by default.
+
+Hence, since the majority of Open MPI's source code base is in C, it
+compiles/links against hwloc successfully.  But when Open MPI's
+Fortran code for the ``mpi_f08`` module is compiled and linked, the
+Homebrew-installed ``gfortran`` |mdash| which does not search
+``/usr/local/lib`` by default |mdash| cannot find ``libhwloc``, and the link
+fails.
+
+There are a few different possible solutions to this issue:
+
+#. The best solution is to always ensure that Open MPI uses a C and
+   Fortran compiler from the same suite/installation.  This will
+   ensure that both compilers/linkers will use the same default
+   library search paths, and all behavior should be consistent.  For
+   example, the following instructs Open MPI's ``configure`` script to
+   use ``gcc-9`` for the C compiler, which (as of July 2019) is the
+   Homebrew executable name for its installed C compiler:
+
+   .. code-block:: sh
+
+      shell$ ./configure CC=gcc-9 ...
+
+      # You can be precise and specify an absolute path for the C
+      # compiler, and/or also specify the Fortran compiler:
+      shell$ ./configure CC=/usr/local/bin/gcc-9 FC=/usr/local/bin/gfortran ...
+
+   Note that this will likely cause ``configure`` to *not* find the
+   Homebrew-installed hwloc, and instead fall back to using the
+   bundled hwloc in the Open MPI source tree.
+
+#. Alternatively, you can simply force ``configure`` to select the
+   bundled versions of hwloc and libevent, which avoids the issue
+   altogether:
+
+   .. code-block:: sh
+
+      shell$ ./configure --with-hwloc=internal --with-libevent=internal ...
+
+#. Finally, you can tell ``configure`` exactly where to find the
+   external hwloc library.  This can have some unintended
+   consequences, however, because it will prefix both the C and
+   Fortran linker's default search paths with ``/usr/local/lib``:
+
+   .. code-block:: sh
+
+      shell$ ./configure --with-hwloc-libdir=/usr/local/lib ...
+
+Overriding ``configure`` behavior
+---------------------------------
+
+If ``configure``'s default searching behavior is not sufficient for
+your environment, you can use :ref:`command line options to override
+its default behavior
+<label-building-ompi-cli-options-required-support-libraries>`.
+
+For example, if PMIx and/or PRRTE are installed such that the default
+header file and linker search paths will not find them, you can
+provide command line options telling Open MPI's ``configure`` where to
+search.  Here's an example ``configure`` invocation where PMIx and
+PRRTE have both been installed to ``/opt/open-mpi-stuff``:
+
+.. code-block:: sh
+
+   ./configure --prefix=$HOME/openmpi-install \
+       --with-pmix=/opt/open-mpi-stuff \
+       --with-prrte=/opt/open-mpi-stuff ...
+
+As another example, if you do not have root-level privileges to use
+the OS / environment package manager, and if you have a simple MPI
+application (e.g., that has no external library dependencies), you may
+wish to configure Open MPI something like this:
+
+.. code-block:: sh
+
+   ./configure --prefix=$HOME/openmpi-install \
+       --with-libevent=internal --with-hwloc=internal \
+       --with-pmix=internal --with-prrte=internal ...
+
+The ``internal`` keywords force ``configure`` to use all four bundled
+versions of the required libraries.
+
+.. danger:: Be very, very careful when overriding ``configure``'s
+   default search behavior for these libraries.  Remember the critical
+   requirement: that Open MPI infrastructure and applications load
+   *exactly one copy* of each support library.  For simplicity, it may
+   be desirable to ensure to use exactly the support libraries that
+   Open MPI was compiled and built against.
+
+   For example, using the Open MPI installed from the sample
+   ``configure`` line (above), you may want to prefix your run-time
+   linker search path (e.g., ``LD_LIBRARY_PATH`` on Linux) with
+   ``$HOME/openmpi-install/lib``.  This will ensure that linker finds
+   the four support libraries from your Open MPI installation tree,
+   even if other copies of the same support libraries are present
+   elsewhere on your system.
+
+(Strong) Advice for packagers
+-----------------------------
+
+If you are an Open MPI packager, we **strongly** suggest that your
+Open MPI package should not include Hwloc, Libevent, PMIx, or PRRTE.
+Instead, it should depend on external, independently-built versions of
+these packages.
+
+See the :ref:`Advice for packagers <label-install-packagers>` section
+for more details.
diff --git a/docs/installing-open-mpi/supported-systems.rst b/docs/installing-open-mpi/supported-systems.rst
new file mode 100644
index 00000000000..e2ccff6853f
--- /dev/null
+++ b/docs/installing-open-mpi/supported-systems.rst
@@ -0,0 +1,67 @@
+Supported Systems
+=================
+
+Operating systems, hardware, and network requirements for this version of
+Open MPI.
+
+Supported Operating Systems
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We primarily develop Open MPI on Linux and MacOS.
+
+Other operating systems are supported, however.  The exact list of
+operating systems supported has changed over time (e.g., native
+Microsoft Windows support was added in v1.3.3, and although it was
+removed prior to v1.8, is still supported through Cygwin).  :ref:`See
+the Platform Notes section <platform-notes-section-label>` for a
+listing of the OSes that that version supports.
+
+Open MPI is fairly POSIX-neutral, so it will run without *too* many
+modifications on most POSIX-like systems.  Hence, if we haven't listed
+your favorite operating system here, it may not be difficult to get
+Open MPI to compile and run properly.  The biggest obstacle is
+typically the assembly language, but that's fairly modular and we're
+happy to provide information about how to port it to new platforms.
+
+It should be noted that we are quite open to accepting patches for
+operating systems that we do not currently support.  If we do not have
+systems to test these on, we probably will only claim to
+"unofficially" support those systems.
+
+Supported Hardware Platforms
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Essentially all the common platforms that the operating
+systems listed in the previous section support.
+
+For example, Linux runs on a *wide* variety of platforms, and we
+certainly don't claim to test all of them.  Open MPI includes
+Linux-compiler-based assembly for support of Intel, AMD, ARM, and
+PowerPC chips, for example.
+
+
+Supported Network Interconnects
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:ref:`See the Platform Notes section <platform-notes-section-label>`
+for a list of networks supported in this specific release of Open MPI.
+
+The set of commonly-available HPC-class network interconnects has
+evolved and changed over time.
+
+Reflecting that evolution, each release of Open MPI supports a
+specific set of such network interconnects.  You will need to check
+the documentation of the version of your Open MPI installation to see
+which interconnects it supports.  A general rule of thumb is that a
+given Open MPI version tends to support the popular HPC-class
+interconnects at the time of its release.
+
+This, unfortunately, does mean that Open MPI removes support for
+networks that are no longer commonly-used in HPC environments.  If you
+still have one of these older interconnects, not all new version of
+Open MPI may support your interconnect |mdash| sorry!  This simply reflects
+the reality of limited development, testing, and maintenance
+resources.
+
+That being said, :doc:`contributions are always welcome!
+</contributing>`.
diff --git a/docs/installing-open-mpi/sysadmins.rst b/docs/installing-open-mpi/sysadmins.rst
new file mode 100644
index 00000000000..4b5c6ef5804
--- /dev/null
+++ b/docs/installing-open-mpi/sysadmins.rst
@@ -0,0 +1,171 @@
+Advice for System Administrators
+================================
+
+Several members of the Open MPI team have strong system
+administrator backgrounds; we recognize the value of having software
+that is friendly to system administrators.  Here are some of the reasons
+that Open MPI is attractive for system administrators:
+
+* Simple, standards-based installation
+* Reduction of the number of MPI installations
+* Ability to set system-level and user-level parameters
+* Scriptable information sources about the Open MPI installation
+
+See the rest of this section for more details.
+
+Setting Global MCA Parameters
+-----------------------------
+
+MCA parameters are a way to tweak Open MPI's behavior at
+run-time.  For example, MCA parameters can specify:
+
+* Which interconnect networks to use
+* Which interconnect networks *not* to use
+* The size difference between eager sends and rendezvous protocol sends
+* How many registered buffers to pre-pin (e.g., for InfiniBand)
+* The size of the pre-pinned registered buffers
+* ...etc.
+
+It can be quite valuable for a system administrator to play with such
+values a bit and find an "optimal" setting for a particular
+operating environment.  These values can then be set in a global text
+file that all users will, by default, inherit when they run Open MPI
+jobs.
+
+For example, say that you have a cluster with 2 ethernet networks |mdash|
+one for NFS and other system-level operations, and one for MPI jobs.
+The system administrator can tell Open MPI to not use the NFS TCP
+network at a system level, such that when users invoke ``mpirun`` or
+``mpiexec`` to launch their jobs, they will automatically only be using
+the network meant for MPI jobs.
+
+:ref:`See how to set MCA params
+<label-running-setting-mca-param-values>` for information on how to
+set global MCA parameters.
+
+Setting MCA Parameters for a Global Open MPI installation
+---------------------------------------------------------
+
+Which global MCA parameters to set depends on both your specific
+cluster setup and the applications that typically run there.
+
+The best thing to do is to use the :ref:`ompi_info(1) <man1-ompi_info>` command
+to see what parameters are available and relevant to you.  Specifically,
+``ompi_info --all`` can be used to show all the parameters that are available
+for each plug-in.  Two common places that system administrators like
+to tweak are:
+
+* *Only allow specific networks:* If you have a cluster with a
+  high-speed interconnect (such as InfiniBand) and a
+  low-speed ethernet network (e.g., 1Gpbps).  The high-speed network is
+  intended for MPI jobs;
+  the control network is intended for NFS and other
+  administrative-level tasks.  In this case, you can simply turn off Open
+  MPI's TCP support.  The ``btl`` framework contains Open MPI's network
+  support; in this case, you want to disable the ``tcp`` plug-in.  You can
+  do this by adding the following line in the file
+  ``$prefix/etc/openmpi-mca-params.conf``:
+
+  .. code-block::
+
+      btl = ^tcp
+
+  This tells Open MPI to load all BTL components *except* ``tcp``.
+
+  Consider another example: your cluster has two TCP networks, one for
+  NFS and administration-level jobs, and another for MPI jobs.  You can
+  tell Open MPI to ignore the TCP network used by NFS by adding the
+  following line in the file ``$prefix/etc/openmpi-mca-params.conf``:
+
+  .. code-block::
+
+      btl_tcp_if_exclude = lo,eth0
+
+  The value of this parameter is the device names to exclude.  In this
+  case, we're excluding ``lo`` (localhost, because Open MPI has its own
+  internal loopback device) and ``eth0``.
+
+* *Tune the parameters for specific networks:* Each network plug-in
+  has a variety of different tunable parameters.  Use the ``ompi_info``
+  command to see what is available.  You show *all* available parameters
+  with:
+
+  .. code-block::
+
+      shell$ ompi_info --param all all
+
+  .. note:: Starting with Open MPI v1.8, ``ompi_info`` categorizes its
+     parameters in *levels*, as defined by the MPI_T interface from
+     the MPI standard.  You will need to specify ``--level 9`` (or
+     ``--all``) to show *all* MCA parameters.  `See this blog entry
+     <https://blogs.cisco.com/performance/open-mpi-and-the-mpi-3-mpi_t-interface>`_
+     for further information.
+
+  .. code-block:: sh
+
+     shell$ ompi_info --level 9
+     # or
+     shell$ ompi_info --all
+
+  Beware: there are *many* variables available.  You can limit the
+  output by showing all the parameters in a specific framework or in a
+  specific plug-in with the command line parameters:
+
+  .. code-block:: sh
+
+     shell$ ompi_info --param btl all --level 9
+
+  Shows all the parameters of all BTL components, and:
+
+  .. code-block:: sh
+
+     shell$ ompi_info --param btl tcp --level 9
+
+  Shows all the parameters of just the ``tcp`` BTL component.
+
+Adding a new plugin to a global Open MPI installation
+-----------------------------------------------------
+
+If you add a new component (such as support for a new network) to a global
+Open MPI installation, Open MPI will
+simply open the new plugin at run-time |mdash| your applications do not need
+to be recompiled or re-linked.
+
+:ref:`See this section <installing-custom-components-label>` for more
+details.
+
+Upgrading network hardware with a global Open MPI installation
+--------------------------------------------------------------
+
+If you upgrade your network hardware and your installation of Open MPI uses
+shared libraries, you simply need to
+recompile the Open MPI components that support that network and
+re-install them.
+
+More specifically, Open MPI shifts the dependency on the underlying
+network away from the MPI applications and to the Open MPI plug-ins.
+This is a major advantage over many other MPI implementations.
+MPI applications will simply open the new plugin when they run.
+
+User customization of a global Open MPI installation
+----------------------------------------------------
+
+It is typically sufficient for a single Open MPI
+installation (or perhaps a small number of Open MPI installations,
+depending on compiler interoperability,
+:ref:`see installing multiple copies of Open MPI
+<building-open-mpi-installation-location-multiple-copies-label>` for
+more information) to serve an entire parallel
+operating environment.
+
+However, a system-wide Open MPI installation can be customized on a
+per-user basis in two important ways:
+
+* *Per-user MCA parameters:* Each user can
+  :ref:`set their own set of MCA parameters, <label-running-setting-mca-param-values>`
+  potentially overriding system-wide defaults.
+* *Per-user plug-ins:* Users can install their own Open MPI
+  plug-ins under ``$HOME/.openmpi/components``.  In this way, developers can
+  experiment with new components without destabilizing Open MPI for the rest
+  of the users on the system.  Or power users can download 3rd party components
+  (perhaps even research-quality components) without affecting other users.
diff --git a/docs/installing-open-mpi/updating.rst b/docs/installing-open-mpi/updating.rst
new file mode 100644
index 00000000000..802a6f1d78e
--- /dev/null
+++ b/docs/installing-open-mpi/updating.rst
@@ -0,0 +1,39 @@
+
+.. _building-open-mpi-updating-label:
+
+Updating or upgrading an Open MPI installation
+==============================================
+
+The Open MPI team recommends one of the following methods for
+upgrading your Open MPI installation:
+
+* Install newer versions of Open MPI into a different directory. For
+  example, install into ``/opt/openmpi-a.b.c`` and
+  ``/opt/openmpi-x.y.z`` for versions a.b.c and x.y.z, respectively.
+* Completely uninstall the old version of Open MPI before installing
+  the new version.  The ``make uninstall`` process from Open MPI a.b.c
+  build tree should completely uninstall that version from the
+  installation tree, making it safe to install a new version (e.g.,
+  version x.y.z) into the same installation tree.
+* Remove the old installation directory entirely and then install the
+  new version.  For example ``rm -rf /opt/openmpi`` *(assuming that
+  there is nothing else of value in this tree!)* The installation of
+  Open MPI x.y.z will safely re-create the ``/opt/openmpi`` tree.
+  This method is preferable if you no longer have the source and build
+  trees to Open MPI a.b.c available from which to ``make
+  uninstall``.
+* Go into the Open MPI a.b.c installation directory and manually
+  remove all old Open MPI files.  Then install Open MPI x.y.z into the
+  same installation directory.  This can be a somewhat painful,
+  annoying, and error-prone process.  *We do not recommend it.*
+  Indeed, if you no longer have access to the original Open MPI a.b.c
+  source and build trees, it may be far simpler to download Open MPI
+  version a.b.c again from the Open MPI web site, configure it with
+  the same installation prefix, and then run ``make uninstall``.  Or
+  use one of the other methods, above.
+
+Depending on the version of Open MPI that applications were initially compiled
+against and the target version of Open MPI to which you upgraded, users may
+need to recompile their applications.
+See the :doc:`section on Open MPI's version numbering scheme </version-numbering>`
+for more information.
diff --git a/docs/installing-open-mpi/vpath-builds.rst b/docs/installing-open-mpi/vpath-builds.rst
new file mode 100644
index 00000000000..e61ee5413e4
--- /dev/null
+++ b/docs/installing-open-mpi/vpath-builds.rst
@@ -0,0 +1,16 @@
+.. _building-open-mpi-vpath-label:
+
+VPATH builds
+============
+
+VPATH build are fully supported. For example:
+
+.. code-block:: sh
+
+   shell$ tar xf openmpi-VERSION.tar.bz2
+   shell$ cd openmpi-VERSION
+   shell$ mkdir build
+   shell$ cd build
+   shell$ ../configure [...options...]
+   <... lots of output ...>
+   shell$ make -j 4
diff --git a/docs/launching-apps/gridengine.rst b/docs/launching-apps/gridengine.rst
new file mode 100644
index 00000000000..462cb26b61e
--- /dev/null
+++ b/docs/launching-apps/gridengine.rst
@@ -0,0 +1,293 @@
+Launching with Grid Engine
+==========================
+
+Open MPI supports the family of run-time schedulers including the Sun
+Grid Engine (SGE), Oracle Grid Engine (OGE), Grid Engine (GE), Son of
+Grid Engine, and others.
+
+This documentation will collectively refer to all of them as "Grid
+Engine", unless a referring to a specific flavor of the Grid Engine
+family.
+
+Verify Grid Engine support
+--------------------------
+
+.. important:: To build Grid Engine support in Open MPI, you will need
+   to explicitly request the SGE support with the ``--with-sge``
+   command line switch to Open MPI's ``configure`` script.
+
+To verify if support for Grid Engine is configured into your Open MPI
+installation, run ``prte_info`` as shown below and look for
+``gridengine``.
+
+.. code-block::
+
+   shell$ prte_info | grep gridengine
+                 MCA ras: gridengine (MCA v2.0, API v2.0, Component v1.3)
+
+.. note:: PRRTE is the software layer that provides run-time
+   environment support to Open MPI.  Open MPI typically hides most
+   PMIx and PRRTE details from the end user, but this is one place
+   that Open MPI is unable to hide the fact that PRRTE provides this
+   functionality, not Open MPI.  Hence, users need to use the
+   ``prte_info`` command to check for Grid Engine support (not
+   ``ompi_info``).
+
+Launching
+---------
+
+When Grid Engine support is included, Open MPI will automatically
+detect when it is running inside SGE and will just "do the Right
+Thing."
+
+Specifically, if you execute an ``mpirun`` command in a Grid Engine
+job, it will automatically use the Grid Engine mechanisms to launch
+and kill processes.  There is no need to specify what nodes to run on
+|mdash| Open MPI will obtain this information directly from Grid
+Engine and default to a number of processes equal to the slot count
+specified.  For example, this will run 4 MPI processes on the nodes
+that were allocated by Grid Engine:
+
+.. code-block:: sh
+
+   # Get the environment variables for Grid Engine
+
+   # (Assuming Grid Engine is installed at /opt/sge and $Grid
+   # Engine_CELL is 'default' in your environment)
+   shell$ . /opt/sge/default/common/settings.sh
+
+   # Allocate an Grid Engine interactive job with 4 slots from a
+   # parallel environment (PE) named 'ompi' and run a 4-process Open
+   # MPI job
+   shell$ qrsh -pe ompi 4 -b y mpirun -n 4 mpi-hello-world
+
+There are also other ways to submit jobs under Grid Engine:
+
+.. code-block:: sh
+
+   # Submit a batch job with the 'mpirun' command embedded in a script
+   shell$ qsub -pe ompi 4 my_mpirun_job.csh
+
+   # Submit a Grid Engine and OMPI job and mpirun in one line
+   shell$ qrsh -V -pe ompi 4 mpirun hostname
+
+   # Use qstat(1) to show the status of Grid Engine jobs and queues
+   shell$ qstat -f
+
+In reference to the setup, be sure you have a Parallel Environment
+(PE) defined for submitting parallel jobs. You don't have to name your
+PE "ompi".  The following example shows a PE named "ompi" that would
+look like:
+
+.. code-block::
+
+   shell$ qconf -sp ompi
+      pe_name            ompi
+      slots              99999
+      user_lists         NONE
+      xuser_lists        NONE
+      start_proc_args    NONE
+      stop_proc_args     NONE
+      allocation_rule    $fill_up
+      control_slaves     TRUE
+      job_is_first_task  FALSE
+      urgency_slots      min
+      accounting_summary FALSE
+      qsort_args         NONE
+
+.. note:: ``qsort_args`` is necessary with the Son of Grid Engine
+   distribution, version 8.1.1 and later, and probably only applicable
+   to it.
+
+.. note:: For very old versions of Sun Grid Engine, omit
+   ``accounting_summary`` too.
+
+You may want to alter other parameters, but the important one is
+``control_slaves``, specifying that the environment has "tight
+integration".  Note also the lack of a start or stop procedure.  The
+tight integration means that mpirun automatically picks up the slot
+count to use as a default in place of the ``-n`` argument, picks up a
+host file, spawns remote processes via ``qrsh`` so that Grid Engine
+can control and monitor them, and creates and destroys a per-job
+temporary directory (``$TMPDIR``), in which Open MPI's directory will
+be created (by default).
+
+Be sure the queue will make use of the PE that you specified:
+
+.. code-block::
+
+   shell$ qconf -sq all.q
+   [...snipped...]
+   pe_list               make cre ompi
+   [...snipped...]
+
+To determine whether the Grid Engine parallel job is successfully
+launched to the remote nodes, you can pass in the MCA parameter
+``--mca plm_base_verbose 1`` to ``mpirun``.
+
+This will add in a ``-verbose`` flag to the ``qrsh -inherit`` command
+that is used to send parallel tasks to the remote Grid Engine
+execution hosts. It will show whether the connections to the remote
+hosts are established successfully or not.
+
+.. error:: TODO is this site still live?  Doesn't look like it..  Jeff
+   emailed Dave Love on 31 Dec 2021 to ask if this is still the
+   correct URL.
+
+   Update March 2022: it doesn't look like this web site is good any
+   more.  Perhaps use https://github.com/grisu48/gridengine instead...?
+
+Various Grid Engine documentation with pointers to more is available
+at `the Son of GridEngine site <http://arc.liv.ac.uk/sge/>`_, and
+configuration instructions can be found at `the Son of GridEngine
+configuration how-to site
+<http://arc.liv.ac.uk/SGE/howto/sge-configs.html>`_.
+
+Grid Engine tight integration support of the ``qsub -notify`` flag
+------------------------------------------------------------------
+
+If you are running SGE 6.2 Update 3 or later, then the ``-notify``
+flag is supported.  If you are running earlier versions, then the
+``-notify`` flag will not work and using it will cause the job to be
+killed.
+
+To use ``-notify``, one has to be careful.  First, let us review what
+``-notify`` does.  Here is an excerpt from the qsub man page for the
+``-notify`` flag.
+
+  The ``-notify`` flag, when set causes Sun Grid Engine to send
+  warning signals to a running job prior to sending the signals
+  themselves. If a SIGSTOP is pending, the job will receive a SIGUSR1
+  several seconds before the SIGSTOP.  If a SIGKILL is pending, the
+  job will receive a SIGUSR2 several seconds before the SIGKILL.  The
+  amount of time delay is controlled by the notify parameter in each
+  queue configuration.
+
+Let us assume the reason you want to use the ``-notify`` flag is to
+get the SIGUSR1 signal prior to getting the SIGTSTP signal.  Something
+like this batch script can be used:
+
+.. code-block:: sh
+
+   #! /bin/bash
+   #$ -S /bin/bash
+   #$ -V
+   #$ -cwd
+   #$ -N Job1
+   #$ -pe ompi 16
+   #$ -j y
+   #$ -l h_rt=00:20:00
+   mpirun -n 16 -mca orte_forward_job_control 1 mpi-hello-world
+
+.. error:: Ralph: Does ``orte_forward_job_control`` still exist?
+
+However, one has to make one of two changes to this script for things
+to work properly.  By default, a SIGUSR1 signal will kill a shell
+script.  So we have to make sure that does not happen. Here is one way
+to handle it:
+
+.. code-block:: sh
+
+   #! /bin/bash
+   #$ -S /bin/bash
+   #$ -V
+   #$ -cwd
+   #$ -N Job1
+   #$ -pe ompi 16
+   #$ -j y
+   #$ -l h_rt=00:20:00
+   exec mpirun -n 16 -mca orte_forward_job_control 1 mpi-hello-world
+
+Alternatively, one can catch the signals in the script instead of doing
+an exec on the mpirun:
+
+.. code-block:: sh
+
+   #! /bin/bash
+   #$ -S /bin/bash
+   #$ -V
+   #$ -cwd
+   #$ -N Job1
+   #$ -pe ompi 16
+   #$ -j y
+   #$ -l h_rt=00:20:00
+
+   function sigusr1handler()
+   {
+       echo "SIGUSR1 caught by shell script" 1>&2
+   }
+
+   function sigusr2handler()
+   {
+       echo "SIGUSR2 caught by shell script" 1>&2
+   }
+
+   trap sigusr1handler SIGUSR1
+   trap sigusr2handler SIGUSR2
+
+   mpirun -n 16 -mca orte_forward_job_control 1 mpi-hello-world
+
+Grid Engine job suspend / resume support
+----------------------------------------
+
+To suspend the job, you send a SIGTSTP (not SIGSTOP) signal to
+``mpirun``.  ``mpirun`` will catch this signal and forward it to the
+``mpi-hello-world`` as a SIGSTOP signal.  To resume the job, you send
+a SIGCONT signal to ``mpirun`` which will be caught and forwarded to
+the ``mpi-hello-world``.
+
+By default, this feature is not enabled.  This means that both the
+SIGTSTP and SIGCONT signals will simply be consumed by the ``mpirun``
+process.  To have them forwarded, you have to run the job with ``--mca
+orte_forward_job_control 1``.  Here is an example on Solaris:
+
+.. error:: TODO Ralph: does ``orte_forward_job_control`` still exist?
+
+.. code-block:: sh
+
+   shell$ mpirun -mca orte_forward_job_control 1 -n 2 mpi-hello-world
+
+In another window, we suspend and continue the job:
+
+.. code-block:: sh
+
+   shell$ prstat -p 15301,15303,15305
+      PID USERNAME  SIZE   RSS STATE  PRI NICE      TIME  CPU PROCESS/NLWP
+    15305 rolfv     158M   22M cpu1     0    0   0:00:21 5.9% mpi-hello-world/1
+    15303 rolfv     158M   22M cpu2     0    0   0:00:21 5.9% mpi-hello-world/1
+    15301 rolfv    8128K 5144K sleep   59    0   0:00:00 0.0% mpirun/1
+
+   shell$ kill -TSTP 15301
+   shell$ prstat -p 15301,15303,15305
+      PID USERNAME  SIZE   RSS STATE  PRI NICE      TIME  CPU PROCESS/NLWP
+    15303 rolfv     158M   22M stop    30    0   0:01:44  21% mpi-hello-world/1
+    15305 rolfv     158M   22M stop    20    0   0:01:44  21% mpi-hello-world/1
+    15301 rolfv    8128K 5144K sleep   59    0   0:00:00 0.0% mpirun/1
+
+   shell$ kill -CONT 15301
+   shell$ prstat -p 15301,15303,15305
+      PID USERNAME  SIZE   RSS STATE  PRI NICE      TIME  CPU PROCESS/NLWP
+    15305 rolfv     158M   22M cpu1     0    0   0:02:06  17% mpi-hello-world/1
+    15303 rolfv     158M   22M cpu3     0    0   0:02:06  17% mpi-hello-world/1
+    15301 rolfv    8128K 5144K sleep   59    0   0:00:00 0.0% mpirun/1
+
+Note that all this does is stop the ``mpi-hello-world`` processes.  It
+does not, for example, free any pinned memory when the job is in the
+suspended state.
+
+To get this to work under the Grid Engine environment, you have to
+change the ``suspend_method`` entry in the queue.  It has to be set to
+SIGTSTP.  Here is an example of what a queue should look like.
+
+.. code-block:: sh
+
+   shell$ qconf -sq all.q
+   qname                 all.q
+   [...snipped...]
+   starter_method        NONE
+   suspend_method        SIGTSTP
+   resume_method         NONE
+
+Note that if you need to suspend other types of jobs with SIGSTOP
+(instead of SIGTSTP) in this queue then you need to provide a script
+that can implement the correct signals for each job type.
diff --git a/docs/launching-apps/index.rst b/docs/launching-apps/index.rst
new file mode 100644
index 00000000000..56b02f525da
--- /dev/null
+++ b/docs/launching-apps/index.rst
@@ -0,0 +1,51 @@
+.. _label-running-mpi-applications:
+
+Launching MPI applications
+==========================
+
+Open MPI can launch MPI processes in a wide variety of environments,
+but they can generally be broken down into two categories:
+
+#. Scheduled environments: these are systems where a resource manager
+   and/or scheduler are used to control access to the compute nodes.
+   Popular resource managers include Slurm, PBS/Pro/Torque, and LSF.
+#. Non-scheduled environments: these are systems where resource
+   managers are not used.  Launches are typically local (e.g., on a
+   single laptop or workstation) or via ``ssh`` (e.g., across a small
+   number of nodes).
+
+Similar to many MPI implementations, Open MPI provides the commands
+:ref:`mpirun(1) <man1-mpirun>` and :ref:`mpiexec(1) <man1-mpiexec>` to
+launch MPI jobs.  This section deals with using these commands.
+
+Note, however, that in Open MPI, :ref:`mpirun(1) <man1-mpirun>` and
+:ref:`mpiexec(1) <man1-mpiexec>` are exactly identical.  Specifically,
+they are symbolic links to a common back-end launcher command.
+
+.. note:: The name of the back-end launcher command has changed over
+          time (it used to be ``orterun``, it is now ``prte``).  This
+          back-end name is largely irrelevant to the user.
+
+The rest of this section usually refers only to :ref:`mpirun(1)
+<man1-mpirun>`, even though the same discussions also apply to
+:ref:`mpiexec(1) <man1-mpiexec>` (because they are both, in fact, the
+same command).
+
+
+.. toctree::
+   :maxdepth: 1
+
+   quickstart
+   prerequisites
+   pmix-and-prrte
+   scheduling
+
+   localhost
+   ssh
+   slurm
+   lsf
+   tm
+   gridengine
+
+   unusual
+   troubleshooting
diff --git a/docs/launching-apps/localhost.rst b/docs/launching-apps/localhost.rst
new file mode 100644
index 00000000000..be5a14c13d5
--- /dev/null
+++ b/docs/launching-apps/localhost.rst
@@ -0,0 +1,40 @@
+Launching only on the local node
+================================
+
+It is common to develop MPI applications on a single workstation or
+laptop, and then move to a larger parallel / HPC environment once the
+MPI application is ready.
+
+Open MPI supports running multi-process MPI jobs on a single machine.
+In such cases, you can simply avoid listing a hostfile or remote
+hosts, and simply list a number of MPI processes to launch.  For
+example:
+
+.. code-block:: sh
+
+   shell$ mpirun -n 6 mpi-hello-world
+   Hello world, I am 0 of 6 (running on my-laptop))
+   Hello world, I am 1 of 6 (running on my-laptop)
+   ...
+   Hello world, I am 5 of 6 (running on my-laptop)
+
+If you do not specify the ``-n`` option, ``mpirun`` will default to
+launching as many MPI processes as there are processor cores (not
+hyperthreads) on the machine.
+
+MPI communication
+-----------------
+
+When running on a single machine, Open MPI will most likely use the
+``ob1`` PML and the following BTLs for MPI communication between
+peers:
+
+* ``self``: used for sending and receiving loopback MPI messages
+  |mdash| where the source and destination MPI process are the same.
+* ``sm``: used for sending and receiving MPI messages where the source
+  and destination MPI processes can share memory (e.g., via SYSV or
+  POSIX shared memory mechanisms).
+
+  .. note:: For more information about using shared memory MPI
+            communication, see the :doc:`Shared Memory
+            </tuning-apps/networking/shared-memory>` page.
diff --git a/docs/launching-apps/lsf.rst b/docs/launching-apps/lsf.rst
new file mode 100644
index 00000000000..7660ca2e2aa
--- /dev/null
+++ b/docs/launching-apps/lsf.rst
@@ -0,0 +1,58 @@
+Launching with LSF
+==================
+
+Open MPI supports the LSF resource manager.
+
+Verify LSF support
+------------------
+
+The ``prte_info`` command can be used to determine whether or not an
+installed Open MPI includes LSF support:
+
+.. code-block::
+
+   shell$ prte_info | grep lsf
+
+If the Open MPI installation includes support for LSF, you
+should see a line similar to that below. Note the MCA version
+information varies depending on which version of Open MPI is
+installed.
+
+.. code-block::
+
+       MCA ras: lsf (MCA v2.1.0, API v2.0.0, Component v3.0.0)
+
+.. note:: PRRTE is the software layer that provides run-time
+   environment support to Open MPI.  Open MPI typically hides most
+   PMIx and PRRTE details from the end user, but this is one place
+   that Open MPI is unable to hide the fact that PRRTE provides this
+   functionality, not Open MPI.  Hence, users need to use the
+   ``prte_info`` command to check for LSF support (not
+   ``ompi_info``).
+
+Launching
+---------
+
+When properly configured, Open MPI obtains both the list of hosts and
+how many processes to start on each host from LSF directly.  Hence, it
+is unnecessary to specify the ``--hostfile``, ``--host``, or ``-n``
+options to ``mpirun``.  Open MPI will use LSF-native mechanisms
+to launch and kill processes (``ssh`` is not required).
+
+For example:
+
+.. code-block:: sh
+
+   # Allocate a job using 4 nodes with 2 processors per node and run the job on the nodes allocated by LSF
+   shell$ bsub -n 8 -R "span[ptile=2]" "mpirun mpi-hello-world"
+
+
+This will run the MPI processes on the nodes that were allocated by
+LSF.  Or, if submitting a script:
+
+.. code-block:: sh
+
+   shell$ cat my_script.sh
+   #!/bin/sh
+   mpirun mpi-hello-world
+   shell$ bsub -n 8 -R "span[ptile=2]" < my_script.sh
diff --git a/docs/launching-apps/pmix-and-prrte.rst b/docs/launching-apps/pmix-and-prrte.rst
new file mode 100644
index 00000000000..11183f57925
--- /dev/null
+++ b/docs/launching-apps/pmix-and-prrte.rst
@@ -0,0 +1,68 @@
+.. _label-running-role-of-pmix-and-prte:
+
+The role of PMIx and PRRTE
+==========================
+
+Prior versions of Open MPI were layered on top of the Open
+Run-Time Environment (ORTE).  ORTE originally started as a small
+portion of the Open MPI code base, but over time, ORTE effectively
+spun off into its own sub-project.  ORTE ultimately evolved into the
+`Process Management Interface Exascale (PMIx) standard and
+corresponding OpenPMIx software project <https://openpmix.github.io/>`_.
+The OpenPMIx project then evolved its own `PMIx Reference Run-Time
+Environment (PRRTE) <https://github.com/openpmix/prrte>`_ project.
+PRRTE has effectively replaced ORTE in the Open MPI implementation.
+
+Open MPI uses both of these external packages for its run-time system support.
+
+Both PMIx and PRRTE have many configure- and run-time options.  Open
+MPI attempts to hide most of these details from the end user, and
+instead present a unified "everything is Open MPI" interface.  Open
+MPI will translate configuration directives to PMIx and PRRTE as
+relevant, hiding such minutia from the end-user.
+
+This is an intentional design decision on the part of the Open MPI
+developer community: HPC and MPI are complicated enough.  We do not
+want to burden the average end user with needing to understand which
+abstractions and configuration options belong to Open MPI vs. PMIx
+vs. PRRTE.
+
+Advanced users can peek into the PMIx and PRRTE internals and tweak
+additional configuration settings if necessary, but we hope that that
+will rarely be necessary.
+
+PMIx
+----
+
+The `Process Management Interface for Exascale (PMIx)
+<https://pmix.org>`_ package is used by Open MPI for the management,
+communication, and coordination of MPI processes with a back-end
+run-time system.
+
+The "back-end run-time system" may range from a low-infrastructure
+system that simply uses ``ssh`` to remotely execute commands (with no
+other infrastructure) to an environment with a full-featured resource
+manager and scheduler such as Slurm, PBS/Pro/Torque, or LSF.
+
+PMIx presents a unified API that hides many of the complexities of
+communication with these back-end run-time environments.  Open MPI
+uses the PMIx API to discover, communicate, and coordinate with any
+supported back-end run-time system without needing to know the
+intimiate details of that system.
+
+PRRTE
+-----
+
+The `PMIx Reference Runtime Environment
+<https://github.com/openpmix/prrte>`_ is, as its name implies, a
+reference run-time environment that utilizes the PMIx API.  It mainly
+provides run-time environment infrastructure for environments that do
+not natively have them.  In practical terms, this typically means
+providing infrastructure for non-scheduled environments that have no
+concept of distributed scheduling, file staging, remote stdout/stderr
+redirection, and only have ``ssh`` to execute commands on remote
+nodes.
+
+Open MPI uses PRRTE to deal with the practical issues of the back-end
+run-time environment such as launching, monitoring, killing, and
+reaping remote processes.
diff --git a/docs/launching-apps/prerequisites.rst b/docs/launching-apps/prerequisites.rst
new file mode 100644
index 00000000000..036ba915a64
--- /dev/null
+++ b/docs/launching-apps/prerequisites.rst
@@ -0,0 +1,256 @@
+.. _running-prerequisites-label:
+
+Prerequisites
+=============
+
+Successful launch of Open MPI jobs requires the ability to
+find Open MPI's executables and shared libraries on all nodes at run
+time.
+
+In general, Open MPI's ``mpicc`` sets the paths to these with the ``runpath`` linker
+option when an application is compiled and linked.
+If the Open MPI executables and libraries can be found via in system-default
+search paths or the ``runpath`` embedded in an application (i.e., without the
+user needing to set or modify ``PATH`` or ``LD_LIBRARY_PATH``), then
+nothing additional needs to be done.  However, if the Open MPI binaries
+are not found, the instructions below may be used to locate them.
+
+If the ``runpath`` embedded in an application is not sufficient to locate
+the Open MPI binaries, then the ``PATH`` and ``LD_LIBRARY_PATH`` can be used
+to locate them.
+In general, Open MPI requires that its executables are in your
+``PATH`` on every node on which you will run and if Open MPI was
+compiled as dynamic libraries (which is the default), the directory
+where its libraries are located must be in your ``LD_LIBRARY_PATH`` on
+every node.
+For example:
+
+* If Open MPI is installed in ``/usr/bin`` and ``/usr/lib``), that is
+  usually sufficient, and the user does not need to do anything extra.
+* If Open MPI is installed in a location that is not searched by
+  default, users may need to add ``$prefix/bin`` to their ``PATH`` and
+  ``$libdir`` (which defaults to ``$prefix/lib``) to their
+  ``LD_LIBRARY_PATH``.
+
+  .. caution:: In scheduled environments, ensuring Open MPI's
+               executables and libraries can be found on the node that
+               executes :ref:`mpirun(1) <man1-mpirun>` may be
+               sufficient.
+
+               In non-scheduled environments, users may need to set
+               the ``PATH`` and ``LD_LIBRARY_PATH`` environment
+               variables in their shell setup files (e.g.,
+               ``$HOME/.bashrc``) so that non-interactive
+               ``ssh``-based logins will be able to find the Open MPI
+               executables and libraries.
+
+               For example, if Open MPI was installed with a prefix of
+               ``/opt/openmpi``, then the following should be in your
+               ``PATH`` and ``LD_LIBRARY_PATH``
+
+               .. list-table::
+                  :header-rows: 1
+
+                  * - Environment variable
+                    - Value to add
+
+                  * - ``PATH``
+                    - ``/opt/openmpi/bin``
+
+                  * - ``LD_LIBRARY_PATH``
+                    - ``/opt/openmpi/lib``
+
+               Depending on your environment, you may need to set these
+               values in your shell startup files (e.g., ``.bashrc``,
+               ``.cshrc``, etc.).
+
+Additionally, Open MPI requires that jobs can be started on remote
+nodes without any input from the keyboard.  For example, if using
+``ssh`` as the remote agent, you must have your environment setup to
+allow execution on remote nodes without entering a password or
+passphrase.
+
+Adding Open MPI to ``PATH`` and ``LD_LIBRARY_PATH``
+---------------------------------------------------
+
+Open MPI *must* be able to find its executables in your ``PATH``
+on every node (if Open MPI was compiled as dynamic libraries, then its
+library path must appear in ``LD_LIBRARY_PATH`` as well).  As such, your
+configuration/initialization files need to add Open MPI to your ``PATH``
+/ ``LD_LIBRARY_PATH`` properly.
+
+How to do this may be highly dependent upon your local configuration;
+you may need to consult with your local system administrator.  Some
+system administrators take care of these details for you, some don't.
+Some common examples are included below, however.
+
+You must have at least a minimum understanding of how your shell works
+to get Open MPI in your ``PATH`` / ``LD_LIBRARY_PATH`` properly.  Note
+that Open MPI must be added to your ``PATH`` and ``LD_LIBRARY_PATH``
+in the following situations:
+
+#. When you login to an interactive shell
+
+   If your interactive login environment is not configured properly,
+   executables like ``mpicc`` will not be found, and it is typically
+   obvious what is wrong.  The Open MPI executable directory can
+   manually be added to the ``PATH``, or the user's startup files can
+   be modified such that the Open MPI executables are added to the
+   ``PATH`` every login.  This latter approach is preferred.
+
+   All shells have some kind of script file that is executed at login
+   time to set things like ``PATH`` and ``LD_LIBRARY_PATH`` and
+   perform other environmental setup tasks.  This startup file is the
+   one that needs to be edited to add Open MPI to the ``PATH`` and
+   ``LD_LIBRARY_PATH``. Consult the manual page for your shell for
+   specific details (some shells are picky about the permissions of
+   the startup file, for example).  The table below lists some common
+   shells and the startup files that they read/execute upon login:
+
+   .. list-table::
+      :header-rows: 1
+      :widths: 10 90
+
+      * - Shell
+        - Interactive login startup files
+
+      * - ``bash``
+        - ``.bash_profile`` if it exists, or ``.bash_login`` if it
+          exists, or ``.profile`` if it exists
+
+          (in that order).  Note that some Linux distributions
+          automatically come with
+
+          ``.bash_profile`` scripts for users that automatically
+          execute ``.bashrc`` as well.
+
+          Consult the ``bash(1)`` man page for more information.
+
+      * - ``zsh``
+        - ``.zshrc`` followed by ``.zshenv``
+
+      * - ``sh`` (or Bash
+
+          named ``sh``)
+        - ``.profile``
+
+      * - ``csh``
+        - ``.cshrc`` followed by ``.login``
+
+      * - ``tcsh``
+        - ``.tcshrc`` if it exists, ``.cshrc`` if it does not, followed by
+          ``.login``
+
+#. When you login to non-interactive shells on remote nodes
+
+   If your non-interactive remote environment is not configured
+   properly, executables like :ref:`mpirun(1) <man1-mpirun>` will not function properly,
+   and it can be somewhat confusing to figure out.
+
+   The startup files in question here are the ones that are
+   automatically executed for a non-interactive login on a remote node
+   (e.g., ``ssh othernode ps``).  Note that not all shells support
+   this, and that some shells use different files for this than listed
+   for interactive logins.  Some shells will supersede non-interactive
+   login startup files with files for interactive logins.  That is,
+   running non-interactive login startup file *may* automatically
+   invoke interactive login startup file.  The following table lists
+   some common shells and the startup file that is automatically
+   executed, either by Open MPI or by the shell itself:
+
+   .. list-table::
+      :header-rows: 1
+      :widths: 10 90
+
+      * - Shell
+        - Non-interactive login startup files
+
+      * - ``bash``
+        - ``.bashrc`` if it exists
+
+      * - ``zsh``
+        - ``.zshrc`` followed by ``.zshenv``
+
+      * - ``sh`` (or Bash
+
+          named ``sh``)
+        - This shell does not execute any file automatically,
+
+          so Open MPI will execute the ``.profile`` script
+
+          before invoking Open MPI executables on remote nodes
+
+      * - ``csh``
+        - ``.cshrc``
+
+      * - ``tcsh``
+        - ``.tcshrc`` if it exists, ``.cshrc`` if it does not
+
+
+Using the ``--prefix`` option with mpirun
+-----------------------------------------
+
+If users are unable to add the relevant directories to ``PATH`` and
+``LD_LIBRARY_PATH``, the :ref:`mpirun(1) <man1-mpirun>` ``--prefix``
+option *may* be sufficient.
+
+There are some situations where you cannot modify the ``PATH`` or
+``LD_LIBRARY_PATH`` |mdash| e.g., some ISV applications prefer to hide
+all parallelism from the user, and therefore do not want to make the
+user modify their shell startup files.  Another case is where you want
+a single user to be able to launch multiple MPI jobs simultaneously,
+each with a different MPI implementation.  Hence, setting shell
+startup files to point to one MPI implementation would be problematic.
+
+In such cases, you have two options:
+
+#. Use the :ref:`mpirun(1) <man1-mpirun>` ``--prefix`` command line
+   option (described below).
+#. Modify the wrapper compilers to include directives to include
+   run-time search locations for the Open MPI libraries.
+
+:ref:`mpirun(1) <man1-mpirun>`'s ``--prefix`` command line option takes as an argument the
+top-level directory where Open MPI was installed.  While relative
+directory names are possible, they can become ambiguous depending on
+the job launcher used; using absolute directory names is strongly
+recommended.
+
+For example, say that Open MPI was installed into
+``/opt/openmpi-VERSION``.  You would use the ``--prefix`` option
+thusly:
+
+.. code-block::
+
+   shell$ mpirun --prefix /opt/openmpi-VERSION -n 4 a.out
+
+This will prefix the ``PATH`` and ``LD_LIBRARY_PATH`` on both the
+local and remote hosts with ``/opt/openmpi-VERSION/bin`` and
+``/opt/openmpi-VERSION/lib``, respectively.  This is *usually*
+unnecessary when using resource managers to launch jobs (e.g., Slurm,
+Torque, etc.) because they tend to copy the entire local environment
+|mdash| to include the ``PATH`` and ``LD_LIBRARY_PATH`` |mdash| to
+remote nodes before execution.  As such, if ``PATH`` and
+``LD_LIBRARY_PATH`` are set properly on the local node, the resource
+manager will automatically propagate those values out to remote nodes.
+The ``--prefix`` option is therefore usually most useful in
+``ssh``-based environments (or similar).
+
+It is possible to make this the default behavior by passing to
+``configure`` the flag ``--enable-mpirun-prefix-by-default``.  This
+will make :ref:`mpirun(1) <man1-mpirun>` behave exactly the same as
+``mpirun --prefix $prefix ...``, where ``$prefix`` is the value given
+to ``--prefix`` in ``configure``.
+
+Finally, note that specifying the absolute pathname to :ref:`mpirun(1)
+<man1-mpirun>` is equivalent to using the ``--prefix`` argument.  For
+example, the following is equivalent to the above command line that
+uses ``--prefix``:
+
+.. code-block::
+
+   shell$ /opt/openmpi-VERSION/bin/mpirun -n 4 a.out
+
+.. error:: TODO Josh H points out that we might also want to mention
+           ``OMPIHOME`` for PRRTE's ``.ini`` file here.  Leaving this
+           as a future to-do item, since PRRTE's ``.ini`` file support
+           does not exist yet.
diff --git a/docs/launching-apps/quickstart.rst b/docs/launching-apps/quickstart.rst
new file mode 100644
index 00000000000..2a3fb37e535
--- /dev/null
+++ b/docs/launching-apps/quickstart.rst
@@ -0,0 +1,252 @@
+.. _label-quickstart-launching-apps:
+
+Quick start: Launching MPI applications
+=======================================
+
+Although this section skips many details, it offers examples that will
+probably work in many environments.
+
+.. caution:: Note that this section is a "Quick start" |mdash| it does
+   not attempt to be comprehensive or describe how to build Open MPI
+   in all supported environments.  The examples below may therefore
+   not work exactly as shown in your environment.
+
+   Please consult the other sections in this chapter for more details,
+   if necessary.
+
+Using ``mpirun`` to launch applications
+---------------------------------------
+
+Open MPI supports both :ref:`mpirun(1) <man1-mpirun>` and
+:ref:`mpiexec(1) <man1-mpiexec>` (they are exactly equivalent) to
+launch MPI applications.  For example:
+
+.. code-block:: sh
+
+   shell$ mpirun -n 2 mpi-hello-world
+   # or
+   shell$ mpiexec -n 2 mpi-hello-world
+   # or
+   shell$ mpiexec -n 1 mpi-hello-world : -n 1 mpi-hello-world
+
+are all equivalent.  For simplicity, the rest of this documentation
+will simply refer to ``mpirun``.
+
+Other ``mpirun`` options
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+``mpirun`` supports the ``--help`` option which provides a usage
+message and a summary of the options that it supports.  It should be
+considered the definitive list of what options are provided.
+
+Several notable options are:
+
+* ``--hostfile``: Specify a hostfile for launchers (such as the
+  ``rsh`` launcher) that need to be told on which hosts to start
+  parallel applications.  Note that for compatibility with other MPI
+  implementations, *--machinefile* is a synonym for ``--hostfile``.
+* ``--host``: Specify a host or list of hosts to run on (see
+  :ref:`this FAQ entry for more details
+  <running-scheduling-host-option-label>`).
+* ``-n``: Indicate the number of processes to start.
+* ``--mca``: Set MCA parameters (see :ref:`how to set MCA params
+  <label-running-setting-mca-param-values>` for more details).
+* ``--wdir DIRECTORY``: Set the working directory of the started
+  applications.  If not supplied, the current working directory is
+  assumed (or ``$HOME``, if the current working directory does not
+  exist on all nodes).
+* ``-x ENV_VARIABLE_NAME``: The name of an environment variable to
+  export to the parallel application.  The ``-x`` option can be
+  specified multiple times to export multiple environment variables to
+  the parallel application.
+
+Note that the :ref:`mpirun(1) <man1-mpirun>` command supports a
+*large* number of options.  Be sure to see the :ref:`mpirun(1)
+<man1-mpirun>` man page for much more information.
+
+Launching on a single host
+--------------------------
+
+It is common to develop MPI applications on a single laptop or
+workstation.  In such simple "single program, multiple data (SPMD)" cases,
+use :ref:`mpirun(1) <man1-mpirun>` and
+specify how many MPI processes you want to launch via the ``-n``
+option:
+
+.. code-block:: sh
+
+   shell$ mpirun -n 6 mpi-hello-world
+   Hello world, I am 0 of 6 (running on my-laptop))
+   Hello world, I am 1 of 6 (running on my-laptop)
+   ...
+   Hello world, I am 5 of 6 (running on my-laptop)
+
+This starts a six-process parallel application, running six copies
+of the executable named ``mpi-hello-world``.
+
+If you do not specify the ``-n`` option, :ref:`mpirun(1)
+<man1-mpirun>` will default to launching as many MPI processes as
+there are processor cores (not hyperthreads) on the machine.
+
+Launching in a non-scheduled environments (via ``ssh``)
+-------------------------------------------------------
+
+In general, Open MPI requires the following to launch and run MPI
+applications:
+
+#. You must be able to login to remote nodes non-interactively (e.g.,
+   without entering a password or passphrase).
+#. Open MPI's executables must be findable (e.g., in your ``PATH``).
+#. Open MPI's libraries must be findable (e.g., in your
+   ``LD_LIBRARY_PATH``).
+
+:ref:`mpirun(1) <man1-mpirun>` accepts a ``--hostfile`` option (and its
+synonym, the ``--machinefile`` option) to specify a hostfile containing one
+hostname per line:
+
+.. code-block:: sh
+
+   shell$ cat my-hostfile.txt
+   node1.example.com
+   node2.example.com
+   node3.example.com slots=2
+   node4.example.com slots=10
+
+The optional ``slots`` attribute tells Open MPI the *maximum* number
+of processes that can be allocated to that node.  If ``slots`` is not
+provided, Open MPI |mdash| by default |mdash| uses the number of
+processor cores (not hyperthreads) on that node.
+
+Assuming that each of the 4 nodes in `my-hostfile.txt` have 16 cores:
+
+.. code-block:: sh
+
+   shell$ mpirun --hostfile my-hostfile.txt mpi-hello-world
+   Hello world, I am 0 of 44 (running on node1.example.com)
+   Hello world, I am 1 of 44 (running on node1.example.com)
+   ...
+   Hello world, I am 15 of 44 (running on node1.example.com)
+   Hello world, I am 16 of 44 (running on node2.example.com)
+   Hello world, I am 17 of 44 (running on node2.example.com)
+   ...
+   Hello world, I am 31 of 44 (running on node2.example.com)
+   Hello world, I am 32 of 44 (running on node3.example.com)
+   Hello world, I am 33 of 44 (running on node3.example.com)
+   Hello world, I am 34 of 44 (running on node4.example.com)
+   ...
+   Hello world, I am 43 of 44 (running on node4.example.com)
+
+You can see the breakdown of how many processes Open MPI launched on
+each node:
+
+* node1: 16, because no ``slots`` was specified
+* node2: 16, because no ``slots`` was specified
+* node3: 2, because ``slots=2`` was specified
+* node2: 10, because ``slots=10`` was specified
+
+Note, however, that not all environments require a hostfile.  For
+example, Open MPI will automatically detect when it is running in
+batch / scheduled environments (such as Slurm, PBS/Torque, SGE,
+LoadLeveler), and will use host information provided by those systems.
+
+Also note that if using a launcher that requires a hostfile and no
+hostfile is specified, all processes are launched on the local host.
+
+Launching in scheduled environments
+-----------------------------------
+
+In scheduled environments (e.g., in a Slurm job, or PBS/Pro, or LSF,
+or any other schedule), the user tells the scheduler how many MPI
+processes to launch, and the scheduler decides which hosts to use.
+The scheduler then passes both pieces of information (the number of
+processes and the hosts to use) to Open MPI.
+
+There are two ways to launch in a scheduled environment.  Nominally,
+they both achieve the same thing: they launch MPI processes.  The
+main user-observable difference between the two methods is that
+:ref:`mpirun(1) <man1-mpirun>` has many more features than scheduler
+direct launchers.
+
+Using Open MPI's :ref:`mpirun(1) <man1-mpirun>`
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. note:: Technically, Open MPI's :ref:`mpirun(1) <man1-mpirun>` is a
+          thin layer around the PRRTE ``prun``.  Hence, most of the
+          functionality described here is really about ``prun``.  For
+          simplicity, however, this docmentation will describe
+          everything in terms of ``mpirun``.
+
+When using the full-featured :ref:`mpirun(1) <man1-mpirun>` in a
+scheduled environment, there is no need to specify a hostfile or
+number of MPI processes to launch.  :ref:`mpirun(1) <man1-mpirun>`
+will receive this information directly from the scheduler.  Hence, if
+you want to launch an MPI job that completely "fills" your scheduled
+allocation (i.e., one MPI process for each slot in the scheduled
+allocation), you can simply:
+
+.. code-block:: sh
+
+   # Write a script that runs your MPI application
+   shell$ cat my-slurm-script.sh
+   #!/bin/sh
+   # There is no need to specify -n or --hostfile because that
+   # information will automatically be provided by Slurm.
+   mpirun mpi-hello-world
+
+You then submit the ``my-slurm-script.sh`` script to Slurm for
+execution:
+
+.. code-block:: sh
+
+   # Use -n to indicate how many MPI processes you want to run.
+   # Slurm will pick the specific hosts which will be used.
+   shell$ sbatch -n 40 my-slurm-script.sh
+   Submitted batch job 1234
+   shell$
+
+After Slurm job 1234 completes, you can look at the output file to see
+what happened:
+
+.. code-block:: sh
+
+   shell$ cat slurm-1234.out
+   Hello world, I am 0 of 40 (running on node37.example.com)
+   Hello world, I am 1 of 40 (running on node37.example.com)
+   Hello world, I am 2 of 40 (running on node37.example.com)
+   ...
+   Hello world, I am 39 of 40 (running on node19.example.com)
+
+Note that the Slurm scheduler picked the hosts on which the processes
+ran.
+
+The above example shows that simply invoking ``mpirun
+mpi-hello-world`` |mdash| with no other CLI options |mdash| obtains
+the number of processes to run and hosts to use from the scheduler.
+
+:ref:`mpirun(1) <man1-mpirun>` has many more features not described in
+this Quick Start section.  For example, while uncommon in scheduled
+environments, you can use ``-n`` and/or ``--hostfile`` to launch in
+subsets of the overall scheduler allocation.  See the :ref:`mpirun(1)
+<man1-mpirun>` man page for more details.
+
+Using the scheduler to "direct launch" (without :ref:`mpirun(1) <man1-mpirun>`)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Some schedulers (such as Slurm) have the ability to "direct launch"
+MPI processes without using Open MPI's :ref:`mpirun(1) <man1-mpirun>`.
+For example:
+
+.. code-block:: sh
+
+   shell$ srun -n 40 mpi-hello-world
+   Hello world, I am 0 of 40 (running on node14.example.com)
+   Hello world, I am 1 of 40 (running on node14.example.com)
+   Hello world, I am 2 of 40 (running on node14.example.com)
+   ...
+   Hello world, I am 39 of 40 (running on node203.example.com)
+   shell$
+
+Similar to the prior example, this example launches 40 copies of
+``mpi-hello-world``, but it does so via the Slurm ``srun`` command
+without using :ref:`mpirun(1) <man1-mpirun>`.
+
diff --git a/docs/launching-apps/scheduling.rst b/docs/launching-apps/scheduling.rst
new file mode 100644
index 00000000000..b39e99bdde8
--- /dev/null
+++ b/docs/launching-apps/scheduling.rst
@@ -0,0 +1,551 @@
+Scheduling processes across hosts
+=================================
+
+Open MPI provides many options for scheduling application processes across
+hosts, including oversubscribing processes to processors.  This section
+describes how to define that mapping.
+
+Scheduling overview
+-------------------
+
+If you are not oversubscribing your hosts
+(i.e., trying to run more processes than slots available on that
+host), scheduling is pretty simple and occurs either on a by-slot or
+by-node round robin schedule.  If you're oversubscribing, the issue
+gets much more complicated |mdash| keep reading.
+
+The more complete answer is: Open MPI schedules processes to nodes by
+asking two questions from each application on the ``mpirun`` command
+line:
+
+#. *How many* processes should be launched?
+#. *Where* should those processes be launched?
+
+The "how many" question is directly answered with the ``-n`` switch
+to ``mpirun``.  If ``-n`` is not specified on the ``mpirun`` command
+line, its value is the sum of the slots on all the nodes.
+
+The "where" question is a little more complicated, and depends on
+three factors:
+
+#. The final node list (e.g., after ``-hostname`` / ``--host``
+   exclusionary or inclusionary processing)
+#. The scheduling policy (which applies to all applications in a
+   single job)
+#. The default and maximum number of slots on each host
+
+Open MPI currently supports two scheduling policies: by slot and by
+node:
+
+#. *By slot:* This is the default scheduling policy, but can also be
+   explicitly requested by using either the ``--map-by slot`` option
+   to ``mpirun`` or by setting the MCA parameter
+   ``rmaps_default_mapping_policy`` to the string ``slot``.
+
+   In this mode, Open MPI will schedule processes on a node until all
+   of its default slots are exhausted before proceeding to the next
+   node.  In MPI terms, this means that Open MPI tries to maximize the
+   number of adjacent ranks in ``MPI_COMM_WORLD`` on the same host
+   without oversubscribing that host.
+
+   For example:
+
+   .. code-block::
+
+      shell$ cat my-hosts
+      node0 slots=2 max_slots=20
+      node1 slots=2 max_slots=20
+      shell$ mpirun --hostfile my-hosts -n 8 --map-by slot hello | sort
+      Hello World I am rank 0 of 8 running on node0
+      Hello World I am rank 1 of 8 running on node0
+      Hello World I am rank 2 of 8 running on node1
+      Hello World I am rank 3 of 8 running on node1
+      Hello World I am rank 4 of 8 running on node0
+      Hello World I am rank 5 of 8 running on node0
+      Hello World I am rank 6 of 8 running on node1
+      Hello World I am rank 7 of 8 running on node1
+
+#. *By node:* This policy can be requested either by using the
+   ``--map-by node`` option to ``mpirun`` or by setting the MCA parameter
+   ``rmaps_default_mapping_policy`` to the string "node".
+
+   In this mode, Open MPI will schedule a single process on each node
+   in a round-robin fashion (looping back to the beginning of the node
+   list as necessary) until all processes have been scheduled.  Nodes
+   are skipped once their default slot counts are exhausted.
+
+   For example:
+
+   .. code-block::
+
+      shell$ cat my-hosts
+      node0 slots=2 max_slots=20
+      node1 slots=2 max_slots=20
+      shell$ mpirun --hostname my-hosts -n 8 --map-by node hello | sort
+      Hello World I am rank 0 of 8 running on node0
+      Hello World I am rank 1 of 8 running on node1
+      Hello World I am rank 2 of 8 running on node0
+      Hello World I am rank 3 of 8 running on node1
+      Hello World I am rank 4 of 8 running on node0
+      Hello World I am rank 5 of 8 running on node1
+      Hello World I am rank 6 of 8 running on node0
+      Hello World I am rank 7 of 8 running on node1
+
+In both policies, if the default slot count is exhausted on all nodes
+while there are still processes to be scheduled, Open MPI will trigger
+an oversubscription condition.
+
+If ``:OVERSUBSCRIBE`` is added as a modifier to the ``--map-by``
+option (e.g., ``mpirun --map-by node:OVERSUBSCRIBE ...`` |mdash| :ref:`see
+this section <running-mpi-apps-oversubscribing-label>` for more
+details), Open MPI will continue to loop through the list of nodes
+again and try to schedule one more process to each node until all
+processes are scheduled.  Nodes are skipped in this process if their
+maximum slot count is exhausted.  If the maximum slot count is
+exhausted on all nodes while there are still processes to be
+scheduled, Open MPI will abort without launching any processes.
+
+If ``:OVERSUBSCRIBE`` is *not* specified and an oversubscription
+condition occurs, Open MPI will abort without launching any processes.
+
+.. _running-scheduling-hostfile-option-label:
+
+Scheduling with the --hostfile option
+-------------------------------------
+
+The ``--hostfile`` option to ``mpirun`` takes a filename that lists
+hosts on which to launch MPI processes.
+
+.. important:: The hosts listed in a hostfile have *nothing* to do
+               with which network interfaces are used for MPI
+               communication.  They are *only* used to specify on
+               which hosts to launch MPI processes.
+
+Hostfiles are simple text files with hosts specified, one per line.
+Each host can also specify a default and maximum number of *slots* to
+be used on that host (i.e., the maximum number of processes that will
+be launched on that node).  Comments are also supported, and blank
+lines are ignored.  For example:
+
+.. code-block::
+
+   # This is an example hostfile.  Comments begin with #.
+   #
+   # Since no slots are specified, the number of slots defaults to the
+   # number of processor cores available on the machine.
+   foo.example.com
+
+   # We want to allow launching a maximum of 2 processes on this host
+   # (e.g., potentially because it has two processor cores):
+   bar.example.com slots=2
+
+Slots are discussed in much more detail :ref:`in this section
+<running-scheduling-slots-label>`.
+
+Hostfiles works in two different ways:
+
+#. *Exclusionary:* If a list of hosts to run on has been provided by
+   another source (e.g., by a hostfile or a batch scheduler such as
+   Slurm, PBS/Torque, SGE, etc.), the hosts provided by the hostfile
+   must be in the already-provided host list.  If the
+   hostfile-specified nodes are *not* in the already-provided host
+   list, ``mpirun`` will abort without launching anything.
+
+   In this case, hostfiles act like an exclusionary filter |mdash|
+   they limit the scope of where processes will be scheduled from the
+   original list of hosts to produce a final list of hosts.
+
+   For example, say that a scheduler job contains hosts ``node01``
+   through ``node04``.  If you run:
+
+   .. code-block::
+
+      shell$ cat my_hosts
+      node03
+      shell$ mpirun -n 1 --hostfile my_hosts hostname
+
+   This will run a single copy of ``hostname`` on the host ``node03``.
+
+   However, presuming your job was allocated only to ``node03`` and
+   you run the following:
+
+   .. code-block::
+
+      shell$ cat my_hosts
+      node17
+      shell$ mpirun -n 1 --hostfile my_hosts hostname
+
+   This is an error (because ``node17`` is not allocated to your job),
+   and ``mpirun`` will abort.
+
+   Finally, note that in exclusionary mode, processes will *only* be
+   executed on the hostfile-specified hosts, If this ends up causing
+   an oversubscription situation, ``mpirun`` will abort by default.
+
+#. *Inclusionary:* If a list of hosts has *not* been provided by
+   another source, then the hosts provided by the ``--hostfile``
+   option will be used as the original and final host list.
+
+   In this case, ``--hostfile`` acts as an inclusionary agent; all
+   ``--hostfile``-supplied hosts become available for scheduling
+   processes.  For example (assume that you are *not* in a scheduling
+   environment where a list of nodes is being transparently supplied):
+
+   .. code-block::
+
+      shell$ cat my_hosts
+      node01.example.com slots=1
+      node02.example.com slots=1
+      node03.example.com slots=1
+      shell$ mpirun -n 3 --hostfile my_hosts hostname
+
+   This will launch a single copy of ``hostname`` on the hosts
+   ``node01.example.com``, ``node02.example.com``, and
+   ``node03.example.com``.
+
+Note, too, that ``--hostfile`` is essentially a per-application switch.
+Hence, if you specify multiple applications (as in an MPMD job),
+``--hostfile`` can be specified multiple times:
+
+.. code-block::
+
+   shell$ cat hostfile_1
+   node01.example.com
+   shell$ cat hostfile_2
+   node02.example.com
+   shell$ mpirun -n 1 --hostfile hostfile_1 hostname : -n 1 --hostfile hostfile_2 uptime
+   node01.example.com
+    06:11:45 up 1 day,  2:32,  0 users,  load average: 21.65, 20.85, 19.84
+
+Notice that ``hostname`` was launched on ``node01.example.com`` and
+``uptime`` was launched on ``node02.example.com``.
+
+.. _running-scheduling-host-option-label:
+
+Scheduling with the --host option
+---------------------------------
+
+The ``--host`` option to ``mpirun`` takes a comma-delimited list of
+hosts on which to run.  For example:
+
+.. code-block::
+
+   shell$ mpirun -n 3 --host a,b,c hostname
+
+Will launch *one* copy of ``hostname`` on each of hosts ``a``, ``b``,
+and ``c``.  Specifically: each host defaults to 1 slot, unless
+specified by the ``:N`` suffix.  For example:
+
+.. code-block::
+
+   shell$ mpirun --host a,b:2,c:3 hostname
+
+Will launch one copy of ``hostname`` on ``a``, two copies of
+``hostname`` on ``b``, and three copies of ``hostname`` and ``c``.
+
+Slots are discussed in much more detail :ref:`in this section
+<running-scheduling-slots-label>`.
+
+.. important:: The hosts specified by the ``--host`` option have
+               *nothing* to do with which network interfaces are used
+               for MPI communication.  They are *only* used to specify
+               on which hosts to launch MPI processes.
+
+``--host`` works in two different ways:
+
+#. *Exclusionary:* If a list of hosts to run on has been provided by
+   another source (e.g., by a hostfile or a batch scheduler such as
+   Slurm, PBS/Torque, SGE, etc.), the hosts provided by the ``--host``
+   option must be in the already-provided host list.  If the
+   ``--host``-specified nodes are *not* in the already-provided host
+   list, ``mpirun`` will abort without launching anything.
+
+   In this case, the ``--host`` option acts like an exclusionary
+   filter |mdash| it limits the scope of where processes will be
+   scheduled from the original list of hosts to produce a final list
+   of hosts.
+
+   For example, say that the hostfile ``my_hosts`` contains the hosts
+   ``node1`` through ``node4``.  If you run:
+
+   .. code-block::
+
+      shell$ mpirun -n 1 --hostfile my_hosts --host node3 hostname
+
+   This will run a single copy of ``hostname`` on the host ``node3``.
+   However, if you run:
+
+   .. code-block::
+
+      shell$ mpirun -n 1 --hostfile my_hosts --host node17 hostname
+
+   This is an error (because ``node17`` is not listed in
+   ``my_hosts``); ``mpirun`` will abort.
+
+   Finally, note that in exclusionary mode, processes will *only* be
+   executed on the ``--host``-specified hosts.  If this ends up
+   causing an oversubscription situation, ``mpirun`` will abort by
+   default.
+
+#. *Inclusionary:* If a list of hosts has *not* been provided by
+   another source, then the hosts provided by the ``--host`` option
+   will be used as the original and final host list.
+
+   In this case, ``--host`` acts as an inclusionary agent; all
+   ``--host``-supplied hosts become available for scheduling
+   processes.  For example (assume that you are *not* in a scheduling
+   environment where a list of nodes is being transparently supplied):
+
+   .. code-block::
+
+      shell$ mpirun -n 3 --host a,b,c hostname
+
+   This will launch a single copy of ``hostname`` on the hosts ``a``,
+   ``b``, and ``c``.
+
+Note, too, that ``--host`` is essentially a per-application switch.
+Hence, if you specify multiple applications (as in an MPMD job),
+``--host`` can be specified multiple times:
+
+.. code-block::
+
+   shell$ mpirun -n 1 --host a hostname : -n 1 --host b uptime
+
+This will launch ``hostname`` on host ``a`` and ``uptime`` on host ``b``.
+
+.. _running-scheduling-slots-label:
+
+Process Slots
+-------------
+
+*Slots* are Open MPI's representation of how many processes can be
+launched on a given host.
+
+Open MPI maintains the number of slots for each host in a given
+parallel job, and |mdash| by default |mdash| will not let you launch
+more processes on a host than it has slots.
+
+.. important:: It is common to set the number of slots on a host to be
+               less than or equal to the number of processor cores on
+               that host.
+
+               **But it is important to realize that Open MPI's concept
+               of slots is actually unrelated to the number of
+               physical processor cores on a host.**
+
+               Specifically: the number of slots on a host can be less
+               than, equal to, or more than the number of processor
+               cores on a host.
+
+If you wish to run more processes on a host than it has slots,
+:ref:`see the section on oversubscription
+<running-mpi-apps-oversubscribing-label>`.
+
+Calculating the number of slots
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The number of slots on a host depends on a few factors:
+
+#. If the host is specified by a job scheduler (e.g., Slurm,
+   PBS/Torque, etc.), the job scheduler specifies the number of slots
+   for that host.
+
+#. If the host is specified in a hostfile:
+
+   #. If the ``slots`` parameter is specified, that value is used for
+      the number of slots on that host.
+   #. Otherwise:
+
+      #. If ``--map-by :HWTCPUS`` was specified, the number of slots
+         defaults to the number of hardware threads on that host.
+      #. Otherwise, the number of slots defaults to the number of
+         processor cores on that host.
+
+#. If the host is specified via the ``--host`` command line option:
+
+   #. If the ``:N`` suffix is specified, ``N`` is used for the number
+      of slots on that host.
+   #. Otherwise, the number of slots defaults to 1.
+   #. If the same host name is specified multiple times, the slots
+      value for that host is increased by ``N`` if ``:N`` is
+      specified, or increased by 1 if ``:N`` is not specified.
+
+.. caution:: The exact scheme used to determine the number of slots
+             has varied between different major versions of Open MPI.
+             The scheme described above is relevant for Open MPI
+             |ompi_series|.
+
+Max slot counts, however, are rarely specified by schedulers.  The max
+slot count for each node will default to "infinite" if it is not
+provided (meaning that Open MPI will oversubscribe the node if you ask
+it to |mdash| see more on oversubscribing in :ref:`this section
+<running-mpi-apps-oversubscribing-label>`).
+
+Here are some examples, all from unscheduled environments:
+
+#. Use a hostfile and specify the ``slots`` parameter.
+
+   .. code-block:: sh
+
+      shell$ cat my-hostfile
+      node01.example.come slots=4
+      shell$ mpirun --hostfile my-hostfile hostname
+      node01
+      node01
+      node01
+      node01
+
+   This launched 4 processes because ``slots=4`` was specified in the
+   hostfile.
+
+#. Use a hostfile and do *not* specify the ``slots`` parameter (assume
+   that ``node01.example.com`` has 2 processor cores):
+
+   .. code-block:: sh
+
+      shell$ cat my-hostfile
+      node01.example.come
+      shell$ mpirun --hostfile my-hostfile hostname
+      node01
+      node01
+
+   This launched 2 processes because ``slots`` was not specified, and
+   ``node02`` has 2 processor cores.
+
+#. Use ``--host``:
+
+   .. code-block:: sh
+
+      shell$ mpirun --host node01.example.com hostname
+      node01
+
+   This launched 1 processes because ``--host`` with no ``:N`` suffix
+   increments the slot count for that host by 1.
+
+#. Use ``--host`` with a ``:N`` suffix:
+
+   .. code-block:: sh
+
+      shell$ mpirun --host node01.example.com:2 hostname
+      node01
+      node01
+
+   This launched 2 processes because ``:2`` was specified on the
+   command line.
+
+#. Use ``--host`` with a ``:N`` suffix, and mention the host multiple times:
+
+   .. code-block:: sh
+
+      shell$ mpirun --host node01.example.com:2,node01.example.com hostname
+      node01
+      node01
+      node01
+
+   This launched 3 processes because ``:2`` was specified on the
+   command line, and then ``node01.example.com`` was specified an
+   additional time, incrementing the slot count for that host to 3.
+
+..  _running-mpi-apps-oversubscribing-label:
+
+Oversubscribing nodes
+---------------------
+
+Running more MPI processes than processors are available on a node is called
+*oversubscribing* the node.   Open MPI can oversubscribe nodes,
+but it very much matters *how* you do it.
+
+Specifically: it is critical that Open MPI *knows* that you are
+oversubscribing the node, or **severe** performance degradation can
+result.
+
+.. important:: Here is a good general rule to follow: **never specify
+               a number of slots that is more than the available
+               number of processors.**
+
+For example, if you want to run 4 processes on a host with 2 processor
+cores, then indicate that you only have 2 slots but want to run 4
+processes.  For example:
+
+.. code-block:: sh
+
+   # In a hostfile, the number of slots will default to the number of
+   # processor cores on the host
+   shell$ cat my-hostfile
+   localhost
+   shell$ mpirun -n 4 --hostfile my-hostfile a.out
+
+Specifically: we strongly suggest that you do **NOT** have a hostfile
+that contains ``slots=4`` (because there are only two available
+processor cores).
+
+That being said, the above command will fail, because you are trying
+to run 4 processes but there are only 2 slots available.  You must
+specifically tell Open MPI that it is ok to oversubscribe via
+``--map-by :OVERSUBSCRIBE``:
+
+.. code-block:: sh
+
+   shell$ cat my-hostfile
+   # For the purposes of this example, explicitly tell Open MPI
+   # that we have 2 slots on the host.
+   localhost slots=2
+   shell$ mpirun -n 4 --hostfile my-hostfile --map-by :OVERSUBSCRIBE a.out
+
+The reason you should tell Open MPI whether you're oversubscribing or
+not (i.e., never specify a ``slots`` value more than the number of
+processor cores available) is because Open MPI basically runs its
+message passing progression engine in two modes: *aggressive* and
+*degraded*.
+
+#. *Degraded:* When Open MPI thinks that it is in an oversubscribed
+   mode (i.e., more processes are running than there are processor
+   cores available), MPI processes will automatically run in
+   *degraded* mode and frequently yield the processor to its peers,
+   thereby allowing all processes to make progress.
+
+   .. note:: Be sure to see :ref:`this FAQ entry
+             <tuning-using-paffinity-label>` that describes how
+             degraded mode affects processor and memory
+             affinity.
+
+#. *Aggressive:* When Open MPI thinks that it is in an exactly- or
+   under-subscribed mode (i.e., the number of running processes is
+   equal to or less than the number of available processor cores), MPI
+   processes will automatically run in *aggressive* mode, meaning that
+   they will never voluntarily give up the processor to other
+   processes.  With some network transports, this means that Open MPI
+   will spin in tight loops attempting to make message passing
+   progress, effectively causing other processes to not get any CPU
+   cycles (and therefore never make any progress).
+
+For example, on a node with a two processor cores:
+
+.. code-block::
+
+   shell$ cat my-hostfile
+   localhost slots=4
+   shell$ mpirun -n 4 --hostfile my-hostfile a.out
+
+This would cause all 4 MPI processes to run in *aggressive* mode
+because Open MPI thinks that there are 4 available processor cores to
+use.  This is actually a lie (there are only 2 processor core |mdash|
+not 4), and can cause extremely bad performance.
+
+Forcing aggressive or degraded performance mode
+-----------------------------------------------
+
+The MCA parameter ``mpi_yield_when_idle`` controls whether an MPI
+process runs in Aggressive or Degraded performance mode.  Setting it
+to 0 forces Aggressive mode; setting it to 1 forces Degraded mode (see
+:ref:`this FAQ entry <label-running-setting-mca-param-values>` to see how
+to set MCA parameters).
+
+Note that this value *only* affects the behavior of MPI processes when
+they are blocking in MPI library calls.  It does not affect behavior
+of non-MPI processes, nor does it affect the behavior of a process
+that is not inside an MPI library call.
+
+Open MPI normally sets this parameter automatically and
+users are cautioned against setting this parameter unless you are
+really, absolutely, positively sure of what you are doing.
diff --git a/docs/launching-apps/slurm.rst b/docs/launching-apps/slurm.rst
new file mode 100644
index 00000000000..56466b219e7
--- /dev/null
+++ b/docs/launching-apps/slurm.rst
@@ -0,0 +1,122 @@
+Launching with Slurm
+====================
+
+Open MPI supports two modes of launching parallel MPI jobs under
+Slurm:
+
+#. Using Open MPI's full-features ``mpirun`` launcher.
+#. Using Slurm's "direct launch" capability.
+
+Unless there is a strong reason to use ``srun`` for direct launch, the
+Open MPI team recommends using ``mpirun`` for launching under Slurm jobs.
+
+.. note:: In versions of Open MPI prior to 5.0.x, using ``srun`` for
+   direct launch could be faster than using ``mpirun``.  **This is no
+   longer true.**
+
+Using ``mpirun``
+----------------
+
+When ``mpirun`` is launched in a Slurm job, ``mpirun`` will
+automatically utilize the Slurm infrastructure for launching and
+controlling the individual MPI processes.
+Hence, it is unnecessary to specify the ``--hostfile``,
+``--host``, or ``-n`` options to ``mpirun``.
+
+.. note:: Using ``mpirun`` is the recommended method for launching Open
+   MPI jobs in Slurm jobs.
+
+   ``mpirun``'s Slurm support should always be available, regardless
+   of how Open MPI or Slurm was installed.
+
+For example:
+
+.. code-block:: sh
+
+   # Allocate a Slurm job with 4 slots
+   shell$ salloc -n 4
+   salloc: Granted job allocation 1234
+
+   # Now run an Open MPI job on all the slots allocated by Slurm
+   shell$ mpirun mpi-hello-world
+
+This will run the 4 MPI processes on the node(s) that were allocated
+by Slurm.
+
+Or, if submitting a script:
+
+.. code-block:: sh
+
+   shell$ cat my_script.sh
+   #!/bin/sh
+   mpirun mpi-hello-world
+   shell$ sbatch -n 4 my_script.sh
+   srun: jobid 1235 submitted
+   shell$
+
+Similar to the ``salloc`` case, no command line options specifying
+number of MPI processes were necessary, since Open MPI will obtain
+that information directly from Slurm at run time.
+
+Using Slurm's "direct launch" functionality
+-------------------------------------------
+
+Assuming that Slurm was configured with its PMIx plugin, you can use
+``srun`` to "direct launch" Open MPI applications without the use of
+Open MPI's ``mpirun`` command.
+
+First, you must ensure that Slurm was built and installed with PMIx
+support.  This can determined as shown below:
+
+.. code-block:: sh
+
+   shell$ srun --mpi=list
+   MPI plugin types are...
+	none
+	pmi2
+	pmix
+   specific pmix plugin versions available: pmix_v4
+
+The output from ``srun`` may vary somewhat depending on the version of Slurm installed.
+If PMIx is not present in the output, then you will not be able to use srun
+to launch Open MPI applications.
+
+.. note:: PMI-2 is not supported in Open MPI 5.0.0 and later releases.
+
+Provided the Slurm installation includes the PMIx plugin, Open MPI applications 
+can then be launched directly via the ``srun`` command.  For example:
+
+.. code-block:: sh
+
+   shell$ srun -N 4 --mpi=pmix mpi-hello-world
+
+Or you can use ``sbatch`` with a script:
+
+.. code-block:: sh
+
+   shell$ cat my_script.sh
+   #!/bin/sh
+   srun --mpi=pmix mpi-hello-world
+   shell$ sbatch -N 4 my_script.sh
+   srun: jobid 1235 submitted
+   shell$
+
+Similar using ``mpirun`` inside of an ``sbatch`` batch script, no
+``srun`` command line options specifying number of processes were
+necessary, because ``sbatch`` set all the relevant Slurm-level
+parameters about number of processes, cores, partition, etc.
+
+Slurm 20.11
+-----------
+
+There were some changes in Slurm behavior that were introduced in
+Slurm 20.11.0 and subsequently reverted out in Slurm 20.11.3.
+
+SchedMD (the makers of Slurm) strongly suggest that all Open MPI users
+avoid using Slurm versions 20.11.0 through 20.11.2.
+
+Indeed, you will likely run into problems using just about any version
+of Open MPI these problematic Slurm releases.
+
+.. important:: Please either downgrade to an older version or upgrade
+               to a newer version of Slurm.
diff --git a/docs/launching-apps/ssh.rst b/docs/launching-apps/ssh.rst
new file mode 100644
index 00000000000..969e55fa826
--- /dev/null
+++ b/docs/launching-apps/ssh.rst
@@ -0,0 +1,240 @@
+Launching with SSH
+==================
+
+When launching Open MPI jobs in a non-scheduled environment, ``ssh``
+is typically used to launch commands on remote nodes.  As listed in
+the :doc:`quick start section </launching-apps/quickstart>`,
+successfully launching MPI applications with ``ssh`` requires the
+following:
+
+#. You must be able to non-interactively login |mdash| without
+   entering a password or passphrase |mdash| to all remote nodes from
+   all remotes nodes.
+#. Open MPI's executables must be findable (e.g., in your ``PATH``).
+#. Open MPI's libraries must be findable (e.g., in your
+   ``LD_LIBRARY_PATH``).
+
+Specifying the hosts for an MPI job
+-----------------------------------
+
+There are three mechanisms for specifying the hosts that an MPI job will run on:
+
+#. The ``--hostfile`` option to :ref:`mpirun(1) <man1-mpirun>`.
+
+   Use this option to specify a list of hosts on which to run.  Note
+   that for compatibility with other MPI implementations,
+   ``--machinefile`` is a synonym for ``--hostfile``.
+   See :ref:`this section <running-scheduling-hostfile-option-label>` for more
+   information about the ``--hostfile`` option.
+
+#. The ``--host`` option to :ref:`mpirun(1) <man1-mpirun>`.
+
+   This option can be used to specify a list of hosts on which to run
+   on the command line.
+   See :ref:`this section <running-scheduling-host-option-label>` for more
+   information about the ``--host`` option.
+
+#. Running in a scheduled environment.
+
+   If you are running in a scheduled environment (e.g., in a Slurm,
+   Torque, or LSF job), Open MPI will automatically get the lists of
+   hosts from the scheduler.  See the next subsections for details about
+   launching MPI jobs in supported scheduled environements.
+
+.. important:: The specification of hosts using any of the above
+               methods has nothing to do with the network interfaces
+               that are used for MPI traffic.  The list of hosts is
+               *only* used for specifying which hosts on which to
+               launch MPI processes.
+
+Non-interactive ``ssh`` logins
+------------------------------
+
+SSH keys must be setup such that the following can be executed without
+being prompted for password or passphrase:
+
+.. code-block:: sh
+
+   shell$ ssh othernode echo hello
+   hello
+   shell$
+
+Consult instructions and tutorials from around the internet to learn
+how to setup SSH keys.  Try Google search terms like "passwordless
+SSH" or "SSH key authentication".
+
+For simplicity, it may be desirable to configure your SSH keys
+without passphrases.  This adds some risk, however (e.g., if your SSH
+keys are compromised).  But it simplifies your SSH setup because you
+will not need to use ``ssh-agent``.  Evaluate the risk level you are
+comfortable with.
+
+.. important:: Open MPI uses a tree-based pattern to launch processes
+   on remote nodes.  This means that Open MPI must be able to
+   non-interactively login |mdash| without being prompted for password
+   or passphrase |mdash| *to any node* in the host list *from any
+   node* in the host list.
+
+   It may *not* be sufficient to only setup an SSH key from the node
+   where you are invoking :ref:`mpirun(1) <man1-mpirun>` to all other
+   nodes.
+
+If you have a shared ``$HOME`` filesystem between your nodes, you can
+setup a single SSH key that is used to login to all nodes.
+
+Finding Open MPI executables and libraries
+------------------------------------------
+
+Once Open MPI is able to use ``ssh`` to invoke executables on a remote
+node, it must be able to find its helper executables and shared
+libraries on that remote node.
+
+If Open MPI is installed in a system-level folder (e.g., in
+``/usr/bin``), Open MPI will likely be able to find its executables
+and libraries on the remote node with no additional assistance.
+
+If, however, Open MPI is installed into a path that is not searched by
+default, you will need to provide assistance so that Open MPI can find
+its executables and libraries.
+
+.. important:: For simplicity, it is *strongly* recommended that you
+   install Open MPI in the same location on all nodes in your job.
+   See the :doc:`Installation location section
+   </installing-open-mpi/installation-location>` for more details.
+
+You can do this in one of two ways.
+
+Use "prefix" behavior
+^^^^^^^^^^^^^^^^^^^^^
+
+.. note:: "Prefix" behavior is only available with :ref:`mpirun(1)
+   <man1-mpirun>`; it is not available via resource manager direct
+   launch mechanisms.  However, this section is about using ``ssh`` to
+   launch MPI jobs, which means that there is no resource manager, and
+   therefore there is no direct launch mechanism available.
+
+When "prefix" behavior is enabled, Open MPI will automatically set the
+``$PATH`` and ``$LD_LIBRARY_PATH`` on remote nodes before executing
+remote commands.
+
+.. important:: Open MPI assumes that the installation ``prefix``,
+   ``bindir``, and ``libdir`` are the same on the remote node as they
+   are on the local node.  If they are not, *then you should not use
+   the "prefix" behavior.*
+
+You can enable "prefix" behavior in one of three ways:
+
+#. Use an absolute path name to invoke :ref:`mpirun(1) <man1-mpirun>`.
+
+   .. code-block:: sh
+
+      shell$ $HOME/my-openmpi/bin/mpirun --hostfile my-hostfile.txt mpi-hello-world
+
+   Simply using the absolute path name to :ref:`mpirun(1)
+   <man1-mpirun>` tells Open MPI to enable "prefix" mode.
+
+
+#. Use the ``--prefix`` option to :ref:`mpirun(1) <man1-mpirun>`.
+
+  .. code-block:: sh
+
+     shell$ $HOME/my-openmpi/bin/mpirun --hostfile my-hostfile.txt \
+         --prefix $HOME/my-openmpi \
+         mpi-hello-world
+
+   The ``-prefix`` option takes a single argument: the prefix path to
+   use for the bindir and libdir on the remote node.
+
+#. Configure Open MPI with ``--enable-mpirun-prefix-by-default``.
+
+   If Open MPI is built this way, :ref:`mpirun(1) <man1-mpirun>` will
+   always enable "prefix" behavior.
+
+Set the ``PATH`` and ``LD_LIBRARY_PATH`` in your shell startup files
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Consider the case where Open MPI was configured with:
+
+.. code-block:: sh
+
+   shell$ ./configure --prefix=$HOME/my-openmpi ...
+
+In this cause, Open MPI will be installed into ``$HOME/my-openmpi``.
+This path is almost certainly not in any system-default search paths,
+so it must be added to the ``$PATH`` and ``$LD_LIBRARY_PATH``
+environment variables.
+
+Specifically: the goal is that the following non-interactive commands
+must be able to execute without error:
+
+.. code-block:: sh
+
+   # First, ensure that this command returns the correct ompi_info
+   # instance (i.e., $HOME/my-openmpi/bin/ompi_info).
+   shell$ ssh remotenode which ompi_info
+   /home/myusername/my-openmpi/bin/ompi_info
+
+   # Next, ensure that you can run that ompi_info command without
+   # error
+   shell$ ssh remotenode ompi_info
+
+   # ... lots of output ...
+
+Ensure that you do not see any errors about libraries that cannot be
+found.
+
+All shells have some kind of script file that is executed at login
+time perform environmental setup tasks.  This startup file is the one
+that needs to be edited to:
+
+#. Add Open MPI's executable path (which is likely ``$prefix/bin``, or
+   ``$HOME/my-openmpi/bin`` in this example) to the ``$PATH``
+   environment variable.
+#. Add Open MPI's library path (which is likely ``$prefix/lib``, or
+   ``$HOME/my-openmpi/lib`` in this example) to the
+   ``$LD_LIBRARY_PATH`` environment variable.
+
+You probably want to add Open MPI's libraries to the *front* of
+``$PATH`` and ``$LD_LIBRARY_PATH`` to ensure that this Open MPI
+installation's files are found *first*.
+
+Consult the manual page for your shell for specific details (some
+shells are picky about the permissions of the startup file, for
+example).  The list below contains some common shells and the startup
+files that they read/execute upon login:
+
+.. list-table::
+   :header-rows: 1
+
+   * - Shell
+     - Non-interactive login
+     - Interactive login
+
+   * - ``bash`` or ``zsh``
+     - ``$HOME/.bashrc`` if it exists.
+     - #. ``$HOME/.bash_profile`` if it exists, or
+       #. ``$HOME/.bash_login`` if it exists, or
+       #. ``$HOME/.profile`` if it exists (in that order).
+
+       Note that some Linux distributions automatically come
+       with ``$HOME/.bash_profile`` scripts for users that
+       automatically execute ``$HOME/.bashrc`` as well. Consult the
+       bash man page for more information.
+
+   * - ``sh``
+     - This shell does not execute any file automatically, so Open MPI
+       will execute the ``$HOME/.profile`` script before invoking Open
+       MPI executables on remote nodes
+     - ``$HOME/.profile``
+
+   * - ``csh``
+     - ``$HOME/.cshrc``
+     - ``$HOME/.cshrc`` followed by ``$HOME/.login``
+
+   * - ``tcsh``
+     - #. ``$HOME/.tcshrc`` if it exists, or
+       #. ``$HOME/.cshrc`` if it does not
+     - #. ``$HOME/.tcshrc`` if it exists, or
+       #. ``$HOME/.cshrc`` if it does not
+
+       Afterwards, execute ``$HOME/.login``
diff --git a/docs/launching-apps/tm.rst b/docs/launching-apps/tm.rst
new file mode 100644
index 00000000000..a19727ce983
--- /dev/null
+++ b/docs/launching-apps/tm.rst
@@ -0,0 +1,72 @@
+Launching with PBS / Torque
+===========================
+
+Open MPI supports PBS, PBS Pro, Torque, and other related resource
+managers.
+
+Verify PBS/Torque support
+-------------------------
+
+The ``prte_info`` command can be used to determine whether or not an
+installed Open MPI includes Torque/PBS Pro support:
+
+.. code-block::
+
+   shell$ prte_info | grep ras
+
+If the Open MPI installation includes support for PBS/Torque, you
+should see a line similar to that below. Note the MCA version
+information varies depending on which version of Open MPI is
+installed.
+
+.. code-block::
+
+       MCA ras: tm (MCA v2.1.0, API v2.0.0, Component v3.0.0)
+
+.. note:: PRRTE is the software layer that provides run-time
+   environment support to Open MPI.  Open MPI typically hides most
+   PMIx and PRRTE details from the end user, but this is one place
+   that Open MPI is unable to hide the fact that PRRTE provides this
+   functionality, not Open MPI.  Hence, users need to use the
+   ``prte_info`` command to check for PBS/Torque support (not
+   ``ompi_info``).
+
+Launching
+---------
+
+When properly configured, Open MPI obtains both the list of hosts and
+how many processes to start on each host from Torque / PBS Pro
+directly.  Hence, it is unnecessary to specify the ``--hostfile``,
+``--host``, or ``-n`` options to ``mpirun``.  Open MPI will use
+PBS/Torque-native mechanisms to launch and kill processes (``ssh`` is
+not required).
+
+For example:
+
+.. code-block:: sh
+
+   # Allocate a PBS job with 4 nodes
+   shell$ qsub -I -lnodes=4
+
+   # Now run an Open MPI job on all the nodes allocated by PBS/Torque
+   shell$ mpirun mpi-hello-world
+
+This will run the MPI processes on the nodes that were allocated by
+PBS/Torque.  Or, if submitting a script:
+
+.. code-block:: sh
+
+   shell$ cat my_script.sh
+   #!/bin/sh
+   mpirun mpi-hello-world
+   shell$ qsub -l nodes=4 my_script.sh
+
+.. warning:: Do not modify ``$PBS_NODEFILE``!
+
+   We've had reports from some sites that system administrators modify
+   the ``$PBS_NODEFILE`` in each job according to local policies.
+   This will currently cause Open MPI to behave in an unpredictable
+   fashion.  As long as no new hosts are added to the hostfile, it
+   *usually* means that Open MPI will incorrectly map processes to
+   hosts, but in some cases it can cause Open MPI to fail to launch
+   processes altogether.
diff --git a/docs/launching-apps/troubleshooting.rst b/docs/launching-apps/troubleshooting.rst
new file mode 100644
index 00000000000..e5ed1618a48
--- /dev/null
+++ b/docs/launching-apps/troubleshooting.rst
@@ -0,0 +1,267 @@
+Troubleshooting
+===============
+
+Launching MPI jobs can be a complex process that involves many moving parts.
+This section attempts to provide solutions to some of the most common
+problems users encounter.
+
+Messages about missing symbols
+------------------------------
+
+Open MPI loads a lot of plugins (sometimes called "components" or
+"modules") at run time.  Sometimes a plugin can fail to load because it
+can't resolve all the symbols that it needs.  There are a few reasons
+why this can happen.
+
+* The plugin is for a different version of Open MPI.  :ref:`See this
+  section <building-open-mpi-install-overwrite-label>` for an
+  explanation of how Open MPI might try to open the "wrong" plugins.
+* An application is trying to manually dynamically open ``libmpi`` in
+  a private symbol space.  For example, if an application is not
+  linked against ``libmpi``, but rather calls something like this:
+
+  .. code-block:: c
+
+     /* This is a Linux example |mdash| the issue is similar/the same on other
+        operating systems */
+     handle = dlopen("libmpi.so", RTLD_NOW | RTLD_LOCAL);
+
+  This is due to some deep run-time linker voodoo |mdash| it is
+  discussed towards the end of `this post to the Open MPI developer's
+  list
+  <https://www.mail-archive.com/devel@lists.open-mpi.org/msg07981.html>`_.
+  Briefly, the issue is this:
+
+  #. The dynamic library ``libmpi`` is opened in a "local" symbol
+     space.
+  #. ``MPI_INIT`` is invoked, which tries to open Open MPI's plugins.
+  #. Open MPI's plugins rely on symbols in ``libmpi`` (and other Open
+     MPI support libraries); these symbols must be resolved when the
+     plugin is loaded.
+  #. However, since ``libmpi`` was opened in a "local" symbol space,
+     its symbols are not available to the plugins that it opens.
+  #. Hence, the plugin fails to load because it can't resolve all of
+     its symbols, and displays a warning message to that effect.
+
+  The ultimate fix for this issue is a bit bigger than Open MPI,
+  unfortunately |mdash| it's a POSIX issue (as briefly described in the
+  devel mailing list posting, above).
+
+  However, there are several common workarounds:
+
+  * Dynamically open ``libmpi`` in a public / global symbol scope
+    |mdash| not a private / local scope.  This will enable
+    ``libmpi``'s symbols to be available for resolution when Open MPI
+    dynamically opens its plugins.
+  * If ``libmpi`` is opened as part of some underlying framework where
+    it is not possible to change the private / local scope to a public
+    / global scope, then dynamically open ``libmpi`` in a public /
+    global scope before invoking the underlying framework.  This
+    sounds a little gross (and it is), but at least the run-time
+    linker is smart enough to not load ``libmpi`` twice |mdash| but it
+    does keeps ``libmpi`` in a public scope.
+  * Use the ``--disable-dlopen`` or ``--disable-mca-dso`` options to
+    Open MPI's ``configure`` script (see this TODO NONEXISTENT FAQ entry
+    for more details on these
+    options).  These options slurp all of Open MPI's plugins up in to
+    ``libmpi`` |mdash| meaning that the plugins physically reside in
+    ``libmpi`` and will not be dynamically opened at run time.
+  * Build Open MPI as a static library by configuring Open MPI with
+    ``--disable-shared`` and ``--enable-static``.  This has the same
+    effect as ``--disable-dlopen``, but it also makes ``libmpi.a`` (as
+    opposed to a shared library).
+
+Errors about missing libraries
+------------------------------
+
+When building Open MPI with the compilers that have libraries in
+non-default search path locations, you may see errors about those
+compiler's support libraries when trying to launch MPI applications if
+their corresponding environments were not setup properly.
+
+For example, you may see warnings similar to the following:
+
+.. code-block:: sh
+
+   # With the Intel compiler suite
+   shell$ mpirun -n 1 --host node1.example.com mpi_hello
+   prted: error while loading shared libraries: libimf.so: cannot open shared object file: No such file or directory
+   --------------------------------------------------------------------------
+   A daemon (pid 11893) died unexpectedly with status 127 while
+   attempting to launch so we are aborting.
+   ...more error messages...
+
+   # With the PGI compiler suite
+   shell$ mpirun -n 1 --host node1.example.com mpi_hello
+   prted: error while loading shared libraries: libpgcc.so: cannot open shared object file: No such file or directory
+   ...more error messages...
+
+   # With the PathScale compiler suite
+   shell$ mpirun -n 1 --host node1.example.com mpi_hello
+   prted: error while loading shared libraries: libmv.so: cannot open shared object file: No such file or directory
+   ...more error messages...
+
+Specifically, Open MPI first attempts to launch a "helper" daemon
+``prted`` on ``node1.example.com``, but it failed because one of
+``prted``'s dependent libraries was not able to be found.  The
+libraries shown above (``libimf.so``, ``libpgcc.so``, and
+``libmv.so``) are specific to their compiler suites (Intel, PGI, and
+PathScale, respectively).  As such, it is likely that the user did not
+setup the compiler library in their environment properly on this node.
+
+Double check that you have setup the appropriate compiler environment
+on the target node, for both interactive and non-interactive logins.
+
+.. note:: It is a common error to ensure that the compiler environment
+          is setup properly for *interactive* logins, but not for
+          *non-interactive* logins.
+
+Here's an example of a user-compiled MPI application working fine
+locally, but failing when invoked non-interactively on a remote node:
+
+.. code-block:: sh
+
+   # Compile a trivial MPI application
+   head_node$ cd $HOME
+   head_node$ mpicc mpi_hello.c -o mpi_hello
+
+   # Run it locally; it works fine
+   head_node$ ./mpi_hello
+   Hello world, I am 0 of 1.
+
+   # Run it remotely interactively; it works fine
+   head_node$ ssh node2.example.com
+
+   Welcome to node2.
+   node2$ ./mpi_hello
+   Hello world, I am 0 of 1.
+   node2$ exit
+
+   # Run it remotely *NON*-interactively; it fails
+   head_node$ ssh node2.example.com $HOME/mpi_hello
+   mpi_hello: error while loading shared libraries: libimf.so: cannot open shared object file: No such file or directory
+
+In cases like this, check your shell script startup files and verify
+that the appropriate compiler environment is setup properly for
+non-interactive logins.
+
+Problems when running across multiple hosts
+-------------------------------------------
+
+When you are able to run MPI jobs on a single host, but fail to run
+them across multiple hosts, try the following:
+
+#. Ensure that your launcher is able to launch across multiple hosts.
+   For example, if you are using ``ssh``, try to ``ssh`` to each
+   remote host and ensure that you are not prompted for a password.
+   For example:
+
+   .. code-block::
+
+      shell$ ssh remotehost hostname
+      remotehost
+
+   If you are unable to launch across multiple hosts, check that your
+   SSH keys are setup properly.  Or, if you are running in a managed
+   environment, such as in a Slurm, Torque, or other job launcher,
+   check that you have reserved enough hosts, are running in an
+   allocated job, etc.
+
+#. Ensure that your ``PATH`` and ``LD_LIBRARY_PATH`` are set correctly
+   on each remote host on which you are trying to run.  For example,
+   with ``ssh``:
+
+   .. code-block::
+
+      shell$ ssh remotehost env | grep -i path
+      PATH=...path on the remote host...
+      LD_LIBRARY_PATH=...LD library path on the remote host...
+
+   If your ``PATH`` or ``LD_LIBRARY_PATH`` are not set properly, see
+   :ref:`this section <running-prerequisites-label>` for
+   the correct values.  Keep in mind that it is fine to have multiple
+   Open MPI installations installed on a machine; the *first* Open MPI
+   installation found by ``PATH`` and ``LD_LIBARY_PATH`` is the one
+   that matters.
+
+#. Run a simple, non-MPI job across multiple hosts.  This verifies
+   that the Open MPI run-time system is functioning properly across
+   multiple hosts.  For example, try running the ``hostname`` command:
+
+   .. code-block::
+
+      shell$ mpirun --host remotehost hostname
+      remotehost
+      shell$ mpirun --host remotehost,otherhost hostname
+      remotehost
+      otherhost
+
+   If you are unable to run non-MPI jobs across multiple hosts, check
+   for common problems such as:
+
+   #. Check your non-interactive shell setup on each remote host to
+      ensure that it is setting up the ``PATH`` and
+      ``LD_LIBRARY_PATH`` properly.
+   #.  Check that Open MPI is finding and launching the correct
+       version of Open MPI on the remote hosts.
+   #. Ensure that you have firewalling disabled between hosts (Open
+      MPI opens random TCP and sometimes random UDP ports between
+      hosts in a single MPI job).
+   #. Try running with the ``plm_base_verbose`` MCA parameter at level
+      10, which will enable extra debugging output to see how Open MPI
+      launches on remote hosts.  For example:
+
+      .. code-block::
+
+         mpirun --mca plm_base_verbose 10 --host remotehost hostname``
+
+#. Now run a simple MPI job across multiple hosts that does not
+   involve MPI communications.  The ``hello_c`` program in the
+   ``examples`` directory in the Open MPI distribution is a good
+   choice.  This verifies that the MPI subsystem is able to initialize
+   and terminate properly.  For example:
+
+   .. code-block::
+
+      shell$ mpirun --host remotehost,otherhost hello_c
+      Hello, world, I am 0 of 1, (Open MPI VERSION, package: Open MPI jsquyres@example.com Distribution, ident: VERSION, DATE)
+      Hello, world, I am 1 of 1, (Open MPI VERSION, package: Open MPI jsquyres@example.com Distribution, ident: VERSION, DATE)
+
+   If you are unable to run simple, non-communication MPI jobs, this
+   can indicate that your Open MPI installation is unable to
+   initialize properly on remote hosts.  Double check your
+   non-interactive login setup on remote hosts.
+
+#. Now run a simple MPI job across multiple hosts that does does some
+   simple MPI communications.  The ``ring_c`` program in the
+   ``examples`` directory in the Open MPI distribution is a good
+   choice.  This verifies that the MPI subsystem is able to pass MPI
+   traffic across your network.  For example:
+
+   .. code-block::
+
+      shell$ mpirun --host remotehost,otherhost ring_c
+      Process 0 sending 10 to 0, tag 201 (1 processes in ring)
+      Process 0 sent to 0
+      Process 0 decremented value: 9
+      Process 0 decremented value: 8
+      Process 0 decremented value: 7
+      Process 0 decremented value: 6
+      Process 0 decremented value: 5
+      Process 0 decremented value: 4
+      Process 0 decremented value: 3
+      Process 0 decremented value: 2
+      Process 0 decremented value: 1
+      Process 0 decremented value: 0
+      Process 0 exiting
+
+   If you are unable to run simple MPI jobs across multiple hosts,
+   this may indicate a problem with the network(s) that Open MPI is
+   trying to use for MPI communications.  Try limiting the networks
+   that it uses, and/or exploring levels 1 through 3 MCA parameters
+   for the communications module that you are using.  For example, if
+   you're using the TCP BTL, see the output of:
+
+   .. code-block::
+
+      ompi_info --level 3 --param btl tcp
diff --git a/docs/launching-apps/unusual.rst b/docs/launching-apps/unusual.rst
new file mode 100644
index 00000000000..81737785393
--- /dev/null
+++ b/docs/launching-apps/unusual.rst
@@ -0,0 +1,164 @@
+Unusual jobs
+============
+
+Open MPI can run many types of applications, including non-MPI programs.
+This section describes some of the less common kinds of programs that can
+be executed.
+
+
+Running non-MPI programs with :ref:`mpirun(1) <man1-mpirun>`
+------------------------------------------------------------
+
+Non-MPI programs can be launched with :ref:`mpirun(1) <man1-mpirun>`,
+for example:
+
+.. code-block::
+
+   shell$ mpirun -n 2 --host a,b uptime
+
+This will launch a copy of the Unix command ``uptime`` on the hosts ``a``
+and ``b``.
+
+:ref:`mpirun(1) <man1-mpirun>` works equally well for MPI and non-MPI
+applications.
+
+Running GUI applications
+------------------------
+
+Running GUI applications depends on your local setup and may require additional
+setup.
+
+You will need to have graphics forwarding (e.g., X11
+forwarding) enabled from the remote processes to the display where you
+want output to appear.  In a secure environment, you can simply allow
+all X requests to be shown on the target display and set the
+``DISPLAY`` environment variable in all MPI processes' environments to
+the target display, perhaps something like this:
+
+.. code-block::
+
+   shell$ hostname
+   my_desktop.secure-cluster.example.com
+   shell$ xhost +
+   shell$ mpirun -n 4 -x DISPLAY=my_desktop.secure-cluster.example.com a.out
+
+However, this technique is not generally suitable for unsecure
+environments (because it allows anyone to read and write to your
+display).  A slightly more secure way is to only allow X connections
+from the nodes where your application will be running:
+
+.. code-block::
+
+   shell$ hostname
+   my_desktop.secure-cluster.example.com
+   shell$ xhost +compute1 +compute2 +compute3 +compute4
+   compute1 being added to access control list
+   compute2 being added to access control list
+   compute3 being added to access control list
+   compute4 being added to access control list
+   shell$ mpirun -n 4 -x DISPLAY=my_desktop.secure-cluster.example.com a.out
+
+(assuming that the four nodes you are running on are ``compute1``
+through ``compute4``).
+
+Other methods are available, but they involve sophisticated X
+forwarding through :ref:`mpirun(1) <man1-mpirun>` and are generally
+more complicated than desirable.
+
+Running curses-based applications
+---------------------------------
+
+Open MPI provides fairly sophisticated stdin / stdout / stderr
+forwarding.  However, it does not work well with curses, ncurses,
+readline, or other sophisticated I/O packages that generally require
+direct control of the terminal.
+
+Every application and I/O library is different |mdash| you should try to
+see if yours is supported.  But chances are that it won't work.
+
+Launching an MPMD MPI job
+-------------------------
+
+Open MPI supports multiple program, multiple data (MPMD) style launches,
+either from the command line or from a file.  For example:
+
+.. code-block::
+
+   shell$ mpirun -n 2 a.out : -n 2 b.out
+
+This will launch a single parallel application, but the first two
+processes will be instances of the ``a.out`` executable, and the
+second two processes will be instances of the ``b.out`` executable.
+In MPI terms, this will be a single ``MPI_COMM_WORLD``, but the
+``a.out`` processes will be ranks 0 and 1 in ``MPI_COMM_WORLD``, while
+the ``b.out`` processes will be ranks 2 and 3 in ``MPI_COMM_WORLD``.
+
+:ref:`mpirun(1) <man1-mpirun>` can also accept a parallel application
+specified in a file instead of on the command line.  For example:
+
+.. code-block::
+
+   shell$ mpirun --app my_appfile
+
+where the file ``my_appfile`` contains the following:
+
+.. code-block:: sh
+
+   # Comments are supported; comments begin with #
+   # Application context files specify each sub-application in the
+   # parallel job, one per line.  The first sub-application is the 2
+   # a.out processes:
+   -n 2 a.out
+   # The second sub-application is the 2 b.out processes:
+   -n 2 b.out
+
+This will result in the same behavior as running ``a.out`` and ``b.out``
+from the command line.
+
+Connecting independent MPI applications
+---------------------------------------
+
+In certain environments, Open MPI supports connecting multiple,
+independent MPI applications using mechanism defined in the MPI
+specification such as ``MPI_Comm_connect() / MPI_Comm_accept()`` and
+publishing connection information using ``MPI_Publish_name() /
+MPI_Lookup_name()``. These mechanisms require a centralized service
+to exchange contact information across multiple jobs.
+
+Beginning with Open MPI v5.0.0 this can be achieved by starting an
+instance of the prte server with the ``report-uri`` option to
+display the contact information of the prte server. This information
+can then be used for launching subsequent MPI applications.
+
+The following commands show an example for launching two MPI jobs
+that will connect to each other at runtime using the MPI-2 based
+functionality.
+
+
+Step 1: start the standalone prte server
+
+.. code-block::
+
+   user@myhost:~/ompi-install/bin$ ./prte --report-uri <filename>
+   DVM ready
+
+Step 2: Launch the first MPI application providing the uri of the
+prte server
+
+.. code-block::
+
+   user@myhost:~/app1-dir$ mpiexec --dvm file:<filename> -np 4 ./mpi_app_1
+
+Step 3: Launch the second MPI application providing the uri of the
+prte server again
+
+.. code-block::
+
+   user@myhost:~/app2-dir$ mpiexec --dvm file:<filename> -np 4 ./mpi_app_2
+
+
+In case the prte server has been started as a system server using the
+``--system-server`` argument (e.g. the nodes used by the MPI
+applications are not shared by multiple jobs), the sequence can be
+simplified by using ``mpiexec --dvm system`` or ``mpiexec --dvm
+system-first`` instead of the uri of the prte server.
diff --git a/docs/license/hwloc.txt b/docs/license/hwloc.txt
new file mode 100644
index 00000000000..e77516e1801
--- /dev/null
+++ b/docs/license/hwloc.txt
@@ -0,0 +1,39 @@
+Copyright © 2004-2006 The Trustees of Indiana University and Indiana University Research and Technology Corporation.  All rights reserved.
+Copyright © 2004-2005 The University of Tennessee and The University of Tennessee Research Foundation.  All rights reserved.
+Copyright © 2004-2005 High Performance Computing Center Stuttgart, University of Stuttgart.  All rights reserved.
+Copyright © 2004-2005 The Regents of the University of California. All rights reserved.
+Copyright © 2009      CNRS
+Copyright © 2009-2016 Inria.  All rights reserved.
+Copyright © 2009-2015 Université Bordeaux
+Copyright © 2009-2015 Cisco Systems, Inc.  All rights reserved.
+Copyright © 2009-2012 Oracle and/or its affiliates.  All rights reserved.
+Copyright © 2010      IBM
+Copyright © 2010      Jirka Hladky
+Copyright © 2012      Aleksej Saushev, The NetBSD Foundation
+Copyright © 2012      Blue Brain Project, EPFL. All rights reserved.
+Copyright © 2013-2014 University of Wisconsin-La Crosse. All rights reserved.
+Copyright © 2015      Research Organization for Information Science and Technology (RIST). All rights reserved.
+Copyright © 2015-2016 Intel, Inc.  All rights reserved.
+See COPYING in top-level directory.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+3. The name of the author may not be used to endorse or promote products
+   derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/docs/license/index.rst b/docs/license/index.rst
new file mode 100644
index 00000000000..b12aad8910d
--- /dev/null
+++ b/docs/license/index.rst
@@ -0,0 +1,55 @@
+License
+=========
+
+Open MPI license
+----------------
+
+The Open MPI license is as follows:
+
+.. literalinclude:: ompi.txt
+   :language: text
+
+Additional licenses
+-------------------
+
+Open MPI includes code from the MPICH project.
+
+.. literalinclude:: mpich.txt
+   :language: text
+
+Redistributed code
+------------------
+
+The Open MPI distribution tarball includes the following other open
+source code bases in their entirety.
+
+Libevent
+^^^^^^^^
+
+.. literalinclude:: libevent.txt
+   :language: text
+
+Hwloc
+^^^^^
+
+.. literalinclude:: hwloc.txt
+   :language: text
+
+PMIx
+^^^^
+
+.. literalinclude:: pmix.txt
+   :language: text
+
+PRRTE
+^^^^^
+
+.. literalinclude:: prrte.txt
+   :language: text
+
+Treematch
+^^^^^^^^^
+
+.. literalinclude:: treematch.txt
+   :language: text
+   :encoding: latin-1
diff --git a/docs/license/libevent.txt b/docs/license/libevent.txt
new file mode 100644
index 00000000000..402ca50896c
--- /dev/null
+++ b/docs/license/libevent.txt
@@ -0,0 +1,99 @@
+Libevent is available for use under the following license, commonly known
+as the 3-clause (or "modified") BSD license:
+
+==============================
+Copyright (c) 2000-2007 Niels Provos <provos@citi.umich.edu>
+Copyright (c) 2007-2012 Niels Provos and Nick Mathewson
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+3. The name of the author may not be used to endorse or promote products
+   derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+==============================
+
+Portions of Libevent are based on works by others, also made available by
+them under the three-clause BSD license above.  The copyright notices are
+available in the corresponding source files; the license is as above.  Here's
+a list:
+
+log.c:
+   Copyright (c) 2000 Dug Song <dugsong@monkey.org>
+   Copyright (c) 1993 The Regents of the University of California.
+
+strlcpy.c:
+   Copyright (c) 1998 Todd C. Miller <Todd.Miller@courtesan.com>
+
+win32select.c:
+   Copyright (c) 2003 Michael A. Davis <mike@datanerds.net>
+
+evport.c:
+   Copyright (c) 2007 Sun Microsystems
+
+ht-internal.h:
+   Copyright (c) 2002 Christopher Clark
+
+minheap-internal.h:
+   Copyright (c) 2006 Maxim Yegorushkin <maxim.yegorushkin@gmail.com>
+
+==============================
+
+The arc4module is available under the following, sometimes called the
+"OpenBSD" license:
+
+   Copyright (c) 1996, David Mazieres <dm@uun.org>
+   Copyright (c) 2008, Damien Miller <djm@openbsd.org>
+
+   Permission to use, copy, modify, and distribute this software for any
+   purpose with or without fee is hereby granted, provided that the above
+   copyright notice and this permission notice appear in all copies.
+
+   THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+   WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+   MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+   ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+   WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+   ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+   OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+==============================
+
+The Windows timer code is based on code from libutp, which is
+distributed under this license, sometimes called the "MIT" license.
+
+
+Copyright (c) 2010 BitTorrent, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/docs/license/mpich.txt b/docs/license/mpich.txt
new file mode 100644
index 00000000000..13bdd8a38d3
--- /dev/null
+++ b/docs/license/mpich.txt
@@ -0,0 +1,36 @@
+The following is a notice of limited availability of the code, and disclaimer
+which must be included in the prologue of the code and in all source listings
+of the code.
+
+Copyright Notice
+ + 2002 University of Chicago
+
+Permission is hereby granted to use, reproduce, prepare derivative works, and
+to redistribute to others.  This software was authored by:
+
+Mathematics and Computer Science Division
+Argonne National Laboratory, Argonne IL 60439
+
+(and)
+
+Department of Computer Science
+University of Illinois at Urbana-Champaign
+
+
+			      GOVERNMENT LICENSE
+
+Portions of this material resulted from work developed under a U.S.
+Government Contract and are subject to the following license: the Government
+is granted for itself and others acting on its behalf a paid-up, nonexclusive,
+irrevocable worldwide license in this computer software to reproduce, prepare
+derivative works, and perform publicly and display publicly.
+
+				  DISCLAIMER
+
+This computer code material was prepared, in part, as an account of work
+sponsored by an agency of the United States Government.  Neither the United
+States, nor the University of Chicago, nor any of their employees, makes any
+warranty express or implied, or assumes any legal liability or responsibility
+for the accuracy, completeness, or usefulness of any information, apparatus,
+product, or process disclosed, or represents that its use would not infringe
+privately owned rights.
diff --git a/docs/license/ompi.txt b/docs/license/ompi.txt
new file mode 100644
index 00000000000..0bd083daaeb
--- /dev/null
+++ b/docs/license/ompi.txt
@@ -0,0 +1,94 @@
+Copyright (c) 2004-2012 The Trustees of Indiana University and Indiana
+                        University Research and Technology
+                        Corporation.  All rights reserved.
+Copyright (c) 2004-2021 The University of Tennessee and The University
+                        of Tennessee Research Foundation.  All rights
+                        reserved.
+Copyright (c) 2004-2018 High Performance Computing Center Stuttgart,
+                        University of Stuttgart.  All rights reserved.
+Copyright (c) 2004-2008 The Regents of the University of California.
+                        All rights reserved.
+Copyright (c) 2006-2018 Los Alamos National Security, LLC.  All rights
+                        reserved.
+Copyright (c) 2006-2021 Cisco Systems, Inc.  All rights reserved.
+Copyright (c) 2006-2010 Voltaire, Inc. All rights reserved.
+Copyright (c) 2006-2021 Sandia National Laboratories. All rights reserved.
+Copyright (c) 2006-2010 Sun Microsystems, Inc.  All rights reserved.
+                        Use is subject to license terms.
+Copyright (c) 2006-2021 The University of Houston. All rights reserved.
+Copyright (c) 2006-2009 Myricom, Inc.  All rights reserved.
+Copyright (c) 2007-2017 UT-Battelle, LLC. All rights reserved.
+Copyright (c) 2007-2021 IBM Corporation.  All rights reserved.
+Copyright (c) 1998-2005 Forschungszentrum Juelich, Juelich Supercomputing
+                        Centre, Federal Republic of Germany
+Copyright (c) 2005-2008 ZIH, TU Dresden, Federal Republic of Germany
+Copyright (c) 2007      Evergrid, Inc. All rights reserved.
+Copyright (c) 2008-2016 Chelsio, Inc.  All rights reserved.
+Copyright (c) 2008-2009 Institut National de Recherche en
+                        Informatique.  All rights reserved.
+Copyright (c) 2007      Lawrence Livermore National Security, LLC.
+                        All rights reserved.
+Copyright (c) 2007-2019 Mellanox Technologies.  All rights reserved.
+Copyright (c) 2006-2010 QLogic Corporation.  All rights reserved.
+Copyright (c) 2008-2017 Oak Ridge National Labs.  All rights reserved.
+Copyright (c) 2006-2012 Oracle and/or its affiliates.  All rights reserved.
+Copyright (c) 2009-2015 Bull SAS.  All rights reserved.
+Copyright (c) 2010      ARM ltd.  All rights reserved.
+Copyright (c) 2016      ARM, Inc.  All rights reserved.
+Copyright (c) 2010-2011 Alex Brick <bricka@ccs.neu.edu>.  All rights reserved.
+Copyright (c) 2012      The University of Wisconsin-La Crosse. All rights reserved.
+Copyright (c) 2013-2020 Intel, Inc. All rights reserved.
+Copyright (c) 2011-2021 NVIDIA Corporation.  All rights reserved.
+Copyright (c) 2016-2018 Broadcom Limited.  All rights reserved.
+Copyright (c) 2011-2021 Fujitsu Limited.  All rights reserved.
+Copyright (c) 2014-2015 Hewlett-Packard Development Company, LP.  All
+                        rights reserved.
+Copyright (c) 2013-2021 Research Organization for Information Science (RIST).
+                        All rights reserved.
+Copyright (c) 2017-2021 Amazon.com, Inc. or its affiliates.  All Rights
+                        reserved.
+Copyright (c) 2018      DataDirect Networks. All rights reserved.
+Copyright (c) 2018-2021 Triad National Security, LLC. All rights reserved.
+Copyright (c) 2019-2021 Hewlett Packard Enterprise Development, LP.
+Copyright (c) 2020-2021 Google, LLC. All rights reserved.
+Copyright (c) 2002      University of Chicago
+Copyright (c) 2001      Argonne National Laboratory
+Copyright (c) 2020-2021 Cornelis Networks, Inc. All rights reserved.
+Copyright (c) 2021      Nanook Consulting
+Copyright (c) 2017-2019 Iowa State University Research Foundation, Inc.
+                        All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+- Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+
+- Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer listed
+  in this license in the documentation and/or other materials
+  provided with the distribution.
+
+- Neither the name of the copyright holders nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+The copyright holders provide no reassurances that the source code
+provided does not infringe any patent, copyright, or any other
+intellectual property rights of third parties.  The copyright holders
+disclaim any liability to any recipient for claims brought against
+recipient by any third party for infringement of that parties
+intellectual property rights.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/docs/license/pmix.txt b/docs/license/pmix.txt
new file mode 100644
index 00000000000..eb5504e3236
--- /dev/null
+++ b/docs/license/pmix.txt
@@ -0,0 +1,87 @@
+Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+                        University Research and Technology
+                        Corporation.  All rights reserved.
+Copyright (c) 2004-2010 The University of Tennessee and The University
+                        of Tennessee Research Foundation.  All rights
+                        reserved.
+Copyright (c) 2004-2010 High Performance Computing Center Stuttgart,
+                        University of Stuttgart.  All rights reserved.
+Copyright (c) 2004-2008 The Regents of the University of California.
+                        All rights reserved.
+Copyright (c) 2006-2010 Los Alamos National Security, LLC.  All rights
+                        reserved.
+Copyright (c) 2006-2010 Cisco Systems, Inc.  All rights reserved.
+Copyright (c) 2006-2010 Voltaire, Inc. All rights reserved.
+Copyright (c) 2006-2011 Sandia National Laboratories. All rights reserved.
+Copyright (c) 2006-2010 Sun Microsystems, Inc.  All rights reserved.
+                        Use is subject to license terms.
+Copyright (c) 2006-2010 The University of Houston. All rights reserved.
+Copyright (c) 2006-2009 Myricom, Inc.  All rights reserved.
+Copyright (c) 2007-2008 UT-Battelle, LLC. All rights reserved.
+Copyright (c) 2007-2019 IBM Corporation.  All rights reserved.
+Copyright (c) 1998-2005 Forschungszentrum Juelich, Juelich Supercomputing
+                        Centre, Federal Republic of Germany
+Copyright (c) 2005-2008 ZIH, TU Dresden, Federal Republic of Germany
+Copyright (c) 2007      Evergrid, Inc. All rights reserved.
+Copyright (c) 2008      Chelsio, Inc.  All rights reserved.
+Copyright (c) 2008-2009 Institut National de Recherche en
+                        Informatique.  All rights reserved.
+Copyright (c) 2007      Lawrence Livermore National Security, LLC.
+                        All rights reserved.
+Copyright (c) 2007-2019 Mellanox Technologies.  All rights reserved.
+Copyright (c) 2006-2010 QLogic Corporation.  All rights reserved.
+Copyright (c) 2008-2010 Oak Ridge National Labs.  All rights reserved.
+Copyright (c) 2006-2010 Oracle and/or its affiliates.  All rights reserved.
+Copyright (c) 2009      Bull SAS.  All rights reserved.
+Copyright (c) 2010      ARM ltd.  All rights reserved.
+Copyright (c) 2010-2011 Alex Brick <bricka@ccs.neu.edu>.  All rights reserved.
+Copyright (c) 2012      The University of Wisconsin-La Crosse. All rights
+                        reserved.
+Copyright (c) 2013-2019 Intel, Inc. All rights reserved.
+Copyright (c) 2011-2014 NVIDIA Corporation.  All rights reserved.
+Copyright (c) 2019      Amazon.com, Inc. or its affiliates.  All Rights
+                        reserved.
+
+$COPYRIGHT$
+
+Additional copyrights may follow
+
+$HEADER$
+
+The following LICENSE pertains to both PMIx and any code ported
+from Open MPI.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+- Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+
+- Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer listed
+  in this license in the documentation and/or other materials
+  provided with the distribution.
+
+- Neither the name of the copyright holders nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+The copyright holders provide no reassurances that the source code
+provided does not infringe any patent, copyright, or any other
+intellectual property rights of third parties.  The copyright holders
+disclaim any liability to any recipient for claims brought against
+recipient by any third party for infringement of that parties
+intellectual property rights.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/docs/license/prrte.txt b/docs/license/prrte.txt
new file mode 100644
index 00000000000..e33930a4733
--- /dev/null
+++ b/docs/license/prrte.txt
@@ -0,0 +1,92 @@
+Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+                        University Research and Technology
+                        Corporation.  All rights reserved.
+Copyright (c) 2004-2017 The University of Tennessee and The University
+                        of Tennessee Research Foundation.  All rights
+                        reserved.
+Copyright (c) 2004-2010 High Performance Computing Center Stuttgart,
+                        University of Stuttgart.  All rights reserved.
+Copyright (c) 2004-2008 The Regents of the University of California.
+                        All rights reserved.
+Copyright (c) 2006-2017 Los Alamos National Security, LLC.  All rights
+                        reserved.
+Copyright (c) 2006-2017 Cisco Systems, Inc.  All rights reserved.
+Copyright (c) 2006-2010 Voltaire, Inc. All rights reserved.
+Copyright (c) 2006-2017 Sandia National Laboratories. All rights reserved.
+Copyright (c) 2006-2010 Sun Microsystems, Inc.  All rights reserved.
+                        Use is subject to license terms.
+Copyright (c) 2006-2017 The University of Houston. All rights reserved.
+Copyright (c) 2006-2009 Myricom, Inc.  All rights reserved.
+Copyright (c) 2007-2017 UT-Battelle, LLC. All rights reserved.
+Copyright (c) 2007-2017 IBM Corporation.  All rights reserved.
+Copyright (c) 1998-2005 Forschungszentrum Juelich, Juelich Supercomputing
+                        Centre, Federal Republic of Germany
+Copyright (c) 2005-2008 ZIH, TU Dresden, Federal Republic of Germany
+Copyright (c) 2007      Evergrid, Inc. All rights reserved.
+Copyright (c) 2008      Chelsio, Inc.  All rights reserved.
+Copyright (c) 2008-2009 Institut National de Recherche en
+                        Informatique.  All rights reserved.
+Copyright (c) 2007      Lawrence Livermore National Security, LLC.
+                        All rights reserved.
+Copyright (c) 2007-2017 Mellanox Technologies.  All rights reserved.
+Copyright (c) 2006-2010 QLogic Corporation.  All rights reserved.
+Copyright (c) 2008-2017 Oak Ridge National Labs.  All rights reserved.
+Copyright (c) 2006-2012 Oracle and/or its affiliates.  All rights reserved.
+Copyright (c) 2009-2015 Bull SAS.  All rights reserved.
+Copyright (c) 2010      ARM ltd.  All rights reserved.
+Copyright (c) 2016      ARM, Inc.  All rights reserved.
+Copyright (c) 2010-2011 Alex Brick <bricka@ccs.neu.edu>.  All rights reserved.
+Copyright (c) 2012      The University of Wisconsin-La Crosse. All rights
+                        reserved.
+Copyright (c) 2013-2018 Intel, Inc.  All rights reserved.
+Copyright (c) 2011-2017 NVIDIA Corporation.  All rights reserved.
+Copyright (c) 2016      Broadcom Limited.  All rights reserved.
+Copyright (c) 2011-2017 Fujitsu Limited.  All rights reserved.
+Copyright (c) 2014-2015 Hewlett-Packard Development Company, LP.  All
+                        rights reserved.
+Copyright (c) 2013-2017 Research Organization for Information Science (RIST).
+                        All rights reserved.
+Copyright (c) 2017-2018 Amazon.com, Inc. or its affiliates.  All Rights
+                        reserved.
+Copyright (c) 2018      DataDirect Networks. All rights reserved.
+
+$COPYRIGHT$
+
+Additional copyrights may follow
+
+$HEADER$
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+- Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+
+- Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer listed
+  in this license in the documentation and/or other materials
+  provided with the distribution.
+
+- Neither the name of the copyright holders nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+The copyright holders provide no reassurances that the source code
+provided does not infringe any patent, copyright, or any other
+intellectual property rights of third parties.  The copyright holders
+disclaim any liability to any recipient for claims brought against
+recipient by any third party for infringement of that parties
+intellectual property rights.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/docs/license/treematch.txt b/docs/license/treematch.txt
new file mode 100644
index 00000000000..186bd9c7d2f
--- /dev/null
+++ b/docs/license/treematch.txt
@@ -0,0 +1,514 @@
+CeCILL-B FREE SOFTWARE LICENSE AGREEMENT
+
+
+    Notice
+
+This Agreement is a Free Software license agreement that is the result
+of discussions between its authors in order to ensure compliance with
+the two main principles guiding its drafting:
+
+    * firstly, compliance with the principles governing the distribution
+      of Free Software: access to source code, broad rights granted to
+      users,
+    * secondly, the election of a governing law, French law, with which
+      it is conformant, both as regards the law of torts and
+      intellectual property law, and the protection that it offers to
+      both authors and holders of the economic rights over software.
+
+The authors of the CeCILL-B (for Ce[a] C[nrs] I[nria] L[ogiciel] L[ibre])
+license are:
+
+Commissariat � l'Energie Atomique - CEA, a public scientific, technical
+and industrial research establishment, having its principal place of
+business at 25 rue Leblanc, immeuble Le Ponant D, 75015 Paris, France.
+
+Centre National de la Recherche Scientifique - CNRS, a public scientific
+and technological establishment, having its principal place of business
+at 3 rue Michel-Ange, 75794 Paris cedex 16, France.
+
+Institut National de Recherche en Informatique et en Automatique -
+INRIA, a public scientific and technological establishment, having its
+principal place of business at Domaine de Voluceau, Rocquencourt, BP
+105, 78153 Le Chesnay cedex, France.
+
+
+    Preamble
+
+This Agreement is an open source software license intended to give users
+significant freedom to modify and redistribute the software licensed
+hereunder.
+
+The exercising of this freedom is conditional upon a strong obligation
+of giving credits for everybody that distributes a software
+incorporating a software ruled by the current license so as all
+contributions to be properly identified and acknowledged.
+
+In consideration of access to the source code and the rights to copy,
+modify and redistribute granted by the license, users are provided only
+with a limited warranty and the software's author, the holder of the
+economic rights, and the successive licensors only have limited liability.
+
+In this respect, the risks associated with loading, using, modifying
+and/or developing or reproducing the software by the user are brought to
+the user's attention, given its Free Software status, which may make it
+complicated to use, with the result that its use is reserved for
+developers and experienced professionals having in-depth computer
+knowledge. Users are therefore encouraged to load and test the
+suitability of the software as regards their requirements in conditions
+enabling the security of their systems and/or data to be ensured and,
+more generally, to use and operate it in the same conditions of
+security. This Agreement may be freely reproduced and published,
+provided it is not altered, and that no provisions are either added or
+removed herefrom.
+
+This Agreement may apply to any or all software for which the holder of
+the economic rights decides to submit the use thereof to its provisions.
+
+
+    Article 1 - DEFINITIONS
+
+For the purpose of this Agreement, when the following expressions
+commence with a capital letter, they shall have the following meaning:
+
+Agreement: means this license agreement, and its possible subsequent
+versions and annexes.
+
+Software: means the software in its Object Code and/or Source Code form
+and, where applicable, its documentation, "as is" when the Licensee
+accepts the Agreement.
+
+Initial Software: means the Software in its Source Code and possibly its
+Object Code form and, where applicable, its documentation, "as is" when
+it is first distributed under the terms and conditions of the Agreement.
+
+Modified Software: means the Software modified by at least one
+Contribution.
+
+Source Code: means all the Software's instructions and program lines to
+which access is required so as to modify the Software.
+
+Object Code: means the binary files originating from the compilation of
+the Source Code.
+
+Holder: means the holder(s) of the economic rights over the Initial
+Software.
+
+Licensee: means the Software user(s) having accepted the Agreement.
+
+Contributor: means a Licensee having made at least one Contribution.
+
+Licensor: means the Holder, or any other individual or legal entity, who
+distributes the Software under the Agreement.
+
+Contribution: means any or all modifications, corrections, translations,
+adaptations and/or new functions integrated into the Software by any or
+all Contributors, as well as any or all Internal Modules.
+
+Module: means a set of sources files including their documentation that
+enables supplementary functions or services in addition to those offered
+by the Software.
+
+External Module: means any or all Modules, not derived from the
+Software, so that this Module and the Software run in separate address
+spaces, with one calling the other when they are run.
+
+Internal Module: means any or all Module, connected to the Software so
+that they both execute in the same address space.
+
+Parties: mean both the Licensee and the Licensor.
+
+These expressions may be used both in singular and plural form.
+
+
+    Article 2 - PURPOSE
+
+The purpose of the Agreement is the grant by the Licensor to the
+Licensee of a non-exclusive, transferable and worldwide license for the
+Software as set forth in Article 5 hereinafter for the whole term of the
+protection granted by the rights over said Software.
+
+
+    Article 3 - ACCEPTANCE
+
+3.1 The Licensee shall be deemed as having accepted the terms and
+conditions of this Agreement upon the occurrence of the first of the
+following events:
+
+    * (i) loading the Software by any or all means, notably, by
+      downloading from a remote server, or by loading from a physical
+      medium;
+    * (ii) the first time the Licensee exercises any of the rights
+      granted hereunder.
+
+3.2 One copy of the Agreement, containing a notice relating to the
+characteristics of the Software, to the limited warranty, and to the
+fact that its use is restricted to experienced users has been provided
+to the Licensee prior to its acceptance as set forth in Article 3.1
+hereinabove, and the Licensee hereby acknowledges that it has read and
+understood it.
+
+
+    Article 4 - EFFECTIVE DATE AND TERM
+
+
+      4.1 EFFECTIVE DATE
+
+The Agreement shall become effective on the date when it is accepted by
+the Licensee as set forth in Article 3.1.
+
+
+      4.2 TERM
+
+The Agreement shall remain in force for the entire legal term of
+protection of the economic rights over the Software.
+
+
+    Article 5 - SCOPE OF RIGHTS GRANTED
+
+The Licensor hereby grants to the Licensee, who accepts, the following
+rights over the Software for any or all use, and for the term of the
+Agreement, on the basis of the terms and conditions set forth hereinafter.
+
+Besides, if the Licensor owns or comes to own one or more patents
+protecting all or part of the functions of the Software or of its
+components, the Licensor undertakes not to enforce the rights granted by
+these patents against successive Licensees using, exploiting or
+modifying the Software. If these patents are transferred, the Licensor
+undertakes to have the transferees subscribe to the obligations set
+forth in this paragraph.
+
+
+      5.1 RIGHT OF USE
+
+The Licensee is authorized to use the Software, without any limitation
+as to its fields of application, with it being hereinafter specified
+that this comprises:
+
+   1. permanent or temporary reproduction of all or part of the Software
+      by any or all means and in any or all form.
+
+   2. loading, displaying, running, or storing the Software on any or
+      all medium.
+
+   3. entitlement to observe, study or test its operation so as to
+      determine the ideas and principles behind any or all constituent
+      elements of said Software. This shall apply when the Licensee
+      carries out any or all loading, displaying, running, transmission
+      or storage operation as regards the Software, that it is entitled
+      to carry out hereunder.
+
+
+      5.2 ENTITLEMENT TO MAKE CONTRIBUTIONS
+
+The right to make Contributions includes the right to translate, adapt,
+arrange, or make any or all modifications to the Software, and the right
+to reproduce the resulting software.
+
+The Licensee is authorized to make any or all Contributions to the
+Software provided that it includes an explicit notice that it is the
+author of said Contribution and indicates the date of the creation thereof.
+
+
+      5.3 RIGHT OF DISTRIBUTION
+
+In particular, the right of distribution includes the right to publish,
+transmit and communicate the Software to the general public on any or
+all medium, and by any or all means, and the right to market, either in
+consideration of a fee, or free of charge, one or more copies of the
+Software by any means.
+
+The Licensee is further authorized to distribute copies of the modified
+or unmodified Software to third parties according to the terms and
+conditions set forth hereinafter.
+
+
+        5.3.1 DISTRIBUTION OF SOFTWARE WITHOUT MODIFICATION
+
+The Licensee is authorized to distribute true copies of the Software in
+Source Code or Object Code form, provided that said distribution
+complies with all the provisions of the Agreement and is accompanied by:
+
+   1. a copy of the Agreement,
+
+   2. a notice relating to the limitation of both the Licensor's
+      warranty and liability as set forth in Articles 8 and 9,
+
+and that, in the event that only the Object Code of the Software is
+redistributed, the Licensee allows effective access to the full Source
+Code of the Software at a minimum during the entire period of its
+distribution of the Software, it being understood that the additional
+cost of acquiring the Source Code shall not exceed the cost of
+transferring the data.
+
+
+        5.3.2 DISTRIBUTION OF MODIFIED SOFTWARE
+
+If the Licensee makes any Contribution to the Software, the resulting
+Modified Software may be distributed under a license agreement other
+than this Agreement subject to compliance with the provisions of Article
+5.3.4.
+
+
+        5.3.3 DISTRIBUTION OF EXTERNAL MODULES
+
+When the Licensee has developed an External Module, the terms and
+conditions of this Agreement do not apply to said External Module, that
+may be distributed under a separate license agreement.
+
+
+        5.3.4 CREDITS
+
+Any Licensee who may distribute a Modified Software hereby expressly
+agrees to:
+
+   1. indicate in the related documentation that it is based on the
+      Software licensed hereunder, and reproduce the intellectual
+      property notice for the Software,
+
+   2. ensure that written indications of the Software intended use,
+      intellectual property notice and license hereunder are included in
+      easily accessible format from the Modified Software interface,
+
+   3. mention, on a freely accessible website describing the Modified
+      Software, at least throughout the distribution term thereof, that
+      it is based on the Software licensed hereunder, and reproduce the
+      Software intellectual property notice,
+
+   4. where it is distributed to a third party that may distribute a
+      Modified Software without having to make its source code
+      available, make its best efforts to ensure that said third party
+      agrees to comply with the obligations set forth in this Article .
+
+If the Software, whether or not modified, is distributed with an
+External Module designed for use in connection with the Software, the
+Licensee shall submit said External Module to the foregoing obligations.
+
+
+        5.3.5 COMPATIBILITY WITH THE CeCILL AND CeCILL-C LICENSES
+
+Where a Modified Software contains a Contribution subject to the CeCILL
+license, the provisions set forth in Article 5.3.4 shall be optional.
+
+A Modified Software may be distributed under the CeCILL-C license. In
+such a case the provisions set forth in Article 5.3.4 shall be optional.
+
+
+    Article 6 - INTELLECTUAL PROPERTY
+
+
+      6.1 OVER THE INITIAL SOFTWARE
+
+The Holder owns the economic rights over the Initial Software. Any or
+all use of the Initial Software is subject to compliance with the terms
+and conditions under which the Holder has elected to distribute its work
+and no one shall be entitled to modify the terms and conditions for the
+distribution of said Initial Software.
+
+The Holder undertakes that the Initial Software will remain ruled at
+least by this Agreement, for the duration set forth in Article 4.2.
+
+
+      6.2 OVER THE CONTRIBUTIONS
+
+The Licensee who develops a Contribution is the owner of the
+intellectual property rights over this Contribution as defined by
+applicable law.
+
+
+      6.3 OVER THE EXTERNAL MODULES
+
+The Licensee who develops an External Module is the owner of the
+intellectual property rights over this External Module as defined by
+applicable law and is free to choose the type of agreement that shall
+govern its distribution.
+
+
+      6.4 JOINT PROVISIONS
+
+The Licensee expressly undertakes:
+
+   1. not to remove, or modify, in any manner, the intellectual property
+      notices attached to the Software;
+
+   2. to reproduce said notices, in an identical manner, in the copies
+      of the Software modified or not.
+
+The Licensee undertakes not to directly or indirectly infringe the
+intellectual property rights of the Holder and/or Contributors on the
+Software and to take, where applicable, vis-�-vis its staff, any and all
+measures required to ensure respect of said intellectual property rights
+of the Holder and/or Contributors.
+
+
+    Article 7 - RELATED SERVICES
+
+7.1 Under no circumstances shall the Agreement oblige the Licensor to
+provide technical assistance or maintenance services for the Software.
+
+However, the Licensor is entitled to offer this type of services. The
+terms and conditions of such technical assistance, and/or such
+maintenance, shall be set forth in a separate instrument. Only the
+Licensor offering said maintenance and/or technical assistance services
+shall incur liability therefor.
+
+7.2 Similarly, any Licensor is entitled to offer to its licensees, under
+its sole responsibility, a warranty, that shall only be binding upon
+itself, for the redistribution of the Software and/or the Modified
+Software, under terms and conditions that it is free to decide. Said
+warranty, and the financial terms and conditions of its application,
+shall be subject of a separate instrument executed between the Licensor
+and the Licensee.
+
+
+    Article 8 - LIABILITY
+
+8.1 Subject to the provisions of Article 8.2, the Licensee shall be
+entitled to claim compensation for any direct loss it may have suffered
+from the Software as a result of a fault on the part of the relevant
+Licensor, subject to providing evidence thereof.
+
+8.2 The Licensor's liability is limited to the commitments made under
+this Agreement and shall not be incurred as a result of in particular:
+(i) loss due the Licensee's total or partial failure to fulfill its
+obligations, (ii) direct or consequential loss that is suffered by the
+Licensee due to the use or performance of the Software, and (iii) more
+generally, any consequential loss. In particular the Parties expressly
+agree that any or all pecuniary or business loss (i.e. loss of data,
+loss of profits, operating loss, loss of customers or orders,
+opportunity cost, any disturbance to business activities) or any or all
+legal proceedings instituted against the Licensee by a third party,
+shall constitute consequential loss and shall not provide entitlement to
+any or all compensation from the Licensor.
+
+
+    Article 9 - WARRANTY
+
+9.1 The Licensee acknowledges that the scientific and technical
+state-of-the-art when the Software was distributed did not enable all
+possible uses to be tested and verified, nor for the presence of
+possible defects to be detected. In this respect, the Licensee's
+attention has been drawn to the risks associated with loading, using,
+modifying and/or developing and reproducing the Software which are
+reserved for experienced users.
+
+The Licensee shall be responsible for verifying, by any or all means,
+the suitability of the product for its requirements, its good working
+order, and for ensuring that it shall not cause damage to either persons
+or properties.
+
+9.2 The Licensor hereby represents, in good faith, that it is entitled
+to grant all the rights over the Software (including in particular the
+rights set forth in Article 5).
+
+9.3 The Licensee acknowledges that the Software is supplied "as is" by
+the Licensor without any other express or tacit warranty, other than
+that provided for in Article 9.2 and, in particular, without any warranty
+as to its commercial value, its secured, safe, innovative or relevant
+nature.
+
+Specifically, the Licensor does not warrant that the Software is free
+from any error, that it will operate without interruption, that it will
+be compatible with the Licensee's own equipment and software
+configuration, nor that it will meet the Licensee's requirements.
+
+9.4 The Licensor does not either expressly or tacitly warrant that the
+Software does not infringe any third party intellectual property right
+relating to a patent, software or any other property right. Therefore,
+the Licensor disclaims any and all liability towards the Licensee
+arising out of any or all proceedings for infringement that may be
+instituted in respect of the use, modification and redistribution of the
+Software. Nevertheless, should such proceedings be instituted against
+the Licensee, the Licensor shall provide it with technical and legal
+assistance for its defense. Such technical and legal assistance shall be
+decided on a case-by-case basis between the relevant Licensor and the
+Licensee pursuant to a memorandum of understanding. The Licensor
+disclaims any and all liability as regards the Licensee's use of the
+name of the Software. No warranty is given as regards the existence of
+prior rights over the name of the Software or as regards the existence
+of a trademark.
+
+
+    Article 10 - TERMINATION
+
+10.1 In the event of a breach by the Licensee of its obligations
+hereunder, the Licensor may automatically terminate this Agreement
+thirty (30) days after notice has been sent to the Licensee and has
+remained ineffective.
+
+10.2 A Licensee whose Agreement is terminated shall no longer be
+authorized to use, modify or distribute the Software. However, any
+licenses that it may have granted prior to termination of the Agreement
+shall remain valid subject to their having been granted in compliance
+with the terms and conditions hereof.
+
+
+    Article 11 - MISCELLANEOUS
+
+
+      11.1 EXCUSABLE EVENTS
+
+Neither Party shall be liable for any or all delay, or failure to
+perform the Agreement, that may be attributable to an event of force
+majeure, an act of God or an outside cause, such as defective
+functioning or interruptions of the electricity or telecommunications
+networks, network paralysis following a virus attack, intervention by
+government authorities, natural disasters, water damage, earthquakes,
+fire, explosions, strikes and labor unrest, war, etc.
+
+11.2 Any failure by either Party, on one or more occasions, to invoke
+one or more of the provisions hereof, shall under no circumstances be
+interpreted as being a waiver by the interested Party of its right to
+invoke said provision(s) subsequently.
+
+11.3 The Agreement cancels and replaces any or all previous agreements,
+whether written or oral, between the Parties and having the same
+purpose, and constitutes the entirety of the agreement between said
+Parties concerning said purpose. No supplement or modification to the
+terms and conditions hereof shall be effective as between the Parties
+unless it is made in writing and signed by their duly authorized
+representatives.
+
+11.4 In the event that one or more of the provisions hereof were to
+conflict with a current or future applicable act or legislative text,
+said act or legislative text shall prevail, and the Parties shall make
+the necessary amendments so as to comply with said act or legislative
+text. All other provisions shall remain effective. Similarly, invalidity
+of a provision of the Agreement, for any reason whatsoever, shall not
+cause the Agreement as a whole to be invalid.
+
+
+      11.5 LANGUAGE
+
+The Agreement is drafted in both French and English and both versions
+are deemed authentic.
+
+
+    Article 12 - NEW VERSIONS OF THE AGREEMENT
+
+12.1 Any person is authorized to duplicate and distribute copies of this
+Agreement.
+
+12.2 So as to ensure coherence, the wording of this Agreement is
+protected and may only be modified by the authors of the License, who
+reserve the right to periodically publish updates or new versions of the
+Agreement, each with a separate number. These subsequent versions may
+address new issues encountered by Free Software.
+
+12.3 Any Software distributed under a given version of the Agreement may
+only be subsequently distributed under the same version of the Agreement
+or a subsequent version.
+
+
+    Article 13 - GOVERNING LAW AND JURISDICTION
+
+13.1 The Agreement is governed by French law. The Parties agree to
+endeavor to seek an amicable solution to any disagreements or disputes
+that may arise during the performance of the Agreement.
+
+13.2 Failing an amicable solution within two (2) months as from their
+occurrence, and unless emergency proceedings are necessary, the
+disagreements or disputes shall be referred to the Paris Courts having
+jurisdiction, by the more diligent Party.
+
+
+Version 1.0 dated 2006-09-05.
diff --git a/docs/man-openmpi/index.rst b/docs/man-openmpi/index.rst
new file mode 100644
index 00000000000..219df50f928
--- /dev/null
+++ b/docs/man-openmpi/index.rst
@@ -0,0 +1,22 @@
+Open MPI manual pages
+=====================
+
+All of the content in this section is also installed as individual
+manual ("man") pages in an Open MPI installation.  If your man page
+search path includes the location where Open MPI installed its man
+pages (which defaults to ``$prefix/share/man``), you can invoke
+commands similar to the following to see the same content that you see
+in these HTML pages:
+
+.. code:: sh
+
+   shell$ man MPI_Send
+
+Open MPI man pages, by section:
+
+.. toctree::
+   :maxdepth: 1
+
+   man1/index
+   man3/index
+   man7/index
diff --git a/docs/man-openmpi/man1/index.rst b/docs/man-openmpi/man1/index.rst
new file mode 100644
index 00000000000..f9afaa10d5c
--- /dev/null
+++ b/docs/man-openmpi/man1/index.rst
@@ -0,0 +1,11 @@
+Commands (section 1)
+====================
+
+.. toctree::
+   :maxdepth: 1
+
+   ompi-wrapper-compiler.1.rst
+   mpirun.1.rst
+   mpisync.1.rst
+   ompi_info.1.rst
+   opal_wrapper.1.rst
diff --git a/docs/man-openmpi/man1/mpic++.1 b/docs/man-openmpi/man1/mpic++.1
new file mode 100644
index 00000000000..7b464ffa102
--- /dev/null
+++ b/docs/man-openmpi/man1/mpic++.1
@@ -0,0 +1 @@
+.so man1/ompi-wrapper-compiler.1
diff --git a/docs/man-openmpi/man1/mpicc.1 b/docs/man-openmpi/man1/mpicc.1
new file mode 100644
index 00000000000..7b464ffa102
--- /dev/null
+++ b/docs/man-openmpi/man1/mpicc.1
@@ -0,0 +1 @@
+.so man1/ompi-wrapper-compiler.1
diff --git a/docs/man-openmpi/man1/mpicxx.1 b/docs/man-openmpi/man1/mpicxx.1
new file mode 100644
index 00000000000..7b464ffa102
--- /dev/null
+++ b/docs/man-openmpi/man1/mpicxx.1
@@ -0,0 +1 @@
+.so man1/ompi-wrapper-compiler.1
diff --git a/docs/man-openmpi/man1/mpif77.1 b/docs/man-openmpi/man1/mpif77.1
new file mode 100644
index 00000000000..7b464ffa102
--- /dev/null
+++ b/docs/man-openmpi/man1/mpif77.1
@@ -0,0 +1 @@
+.so man1/ompi-wrapper-compiler.1
diff --git a/docs/man-openmpi/man1/mpif90.1 b/docs/man-openmpi/man1/mpif90.1
new file mode 100644
index 00000000000..7b464ffa102
--- /dev/null
+++ b/docs/man-openmpi/man1/mpif90.1
@@ -0,0 +1 @@
+.so man1/ompi-wrapper-compiler.1
diff --git a/docs/man-openmpi/man1/mpifort.1 b/docs/man-openmpi/man1/mpifort.1
new file mode 100644
index 00000000000..7b464ffa102
--- /dev/null
+++ b/docs/man-openmpi/man1/mpifort.1
@@ -0,0 +1 @@
+.so man1/ompi-wrapper-compiler.1
diff --git a/docs/man-openmpi/man1/mpijavac.1 b/docs/man-openmpi/man1/mpijavac.1
new file mode 100644
index 00000000000..7b464ffa102
--- /dev/null
+++ b/docs/man-openmpi/man1/mpijavac.1
@@ -0,0 +1 @@
+.so man1/ompi-wrapper-compiler.1
diff --git a/docs/man-openmpi/man1/mpirun.1.rst b/docs/man-openmpi/man1/mpirun.1.rst
new file mode 100644
index 00000000000..4b20b874e36
--- /dev/null
+++ b/docs/man-openmpi/man1/mpirun.1.rst
@@ -0,0 +1,1740 @@
+.. _man1-mpirun:
+.. _man1-mpiexec:
+
+
+mpirun / mpiexec
+================
+
+.. include_body
+
+mpirun, mpiexec |mdash| Execute serial and parallel jobs in Open MPI.
+
+.. note:: ``mpirun`` and ``mpiexec`` are synonyms for each other.
+          Indeed, they are symbolic links to the same executable.
+          Using either of the names will produce the exact same
+          behavior.
+
+SYNOPSIS
+--------
+
+Single Process Multiple Data (SPMD) Model:
+
+.. code::
+
+   mpirun [ options ] <program> [ <args> ]
+
+Multiple Instruction Multiple Data (MIMD) Model:
+
+.. code::
+
+   mpirun [ global_options ]
+          [ local_options1 ] <program1> [ <args1> ] :
+          [ local_options2 ] <program2> [ <args2> ] :
+          ... :
+          [ local_optionsN ] <programN> [ <argsN> ]
+
+Note that in both models, invoking ``mpirun`` via an absolute path
+name is equivalent to specifying the ``--prefix`` option with a
+``<dir>`` value equivalent to the directory where ``mpirun`` resides,
+minus its last subdirectory.  For example:
+
+.. code:: sh
+
+   shell$ /usr/local/bin/mpirun ...
+
+is equivalent to
+
+.. code:: sh
+
+   shell$ mpirun --prefix /usr/local
+
+QUICK SUMMARY
+-------------
+
+If you are simply looking for how to run an MPI application, you
+probably want to use a command line of the following form:
+
+.. code:: sh
+
+   shell$ mpirun [ -n X ] [ --hostfile <filename> ]  <program>
+
+This will run ``X`` copies of ``<program>`` in your current run-time
+environment (if running under a supported resource manager, Open MPI's
+``mpirun`` will usually automatically use the corresponding resource
+manager process starter, as opposed to ``ssh`` (for example), which
+require the use of a hostfile, or will default to running all ``X``
+copies on the localhost), scheduling (by default) in a round-robin
+fashion by CPU slot.  See the rest of this documentation for more
+details.
+
+Please note that ``mpirun`` automatically binds processes to hardware
+resources. Three binding patterns are used in the absence of any
+further directives (See :ref:`map/rank/bind defaults
+<man1-mpirun-map-rank-bind-defaults>` for more details):
+
+* **Bind to core**:     when the number of processes is <= 2
+* **Bind to package**:  when the number of processes is > 2
+* **Bind to none**:     when oversubscribed
+
+If your application uses threads, then you probably want to ensure
+that you are either not bound at all (by specifying ``--bind-to none``),
+or bound to multiple cores using an appropriate binding level or
+specific number of processing elements per application process.
+
+OPEN MPI'S USE OF PRRTE
+-----------------------
+
+Open MPI uses the PMIx Reference Runtime Environment (PRRTE) as the
+main engine for launching, monitoring, and terminating MPI processes.
+
+Much of the documentation below is directly imported from PRRTE.  As
+such, it frequently refers to PRRTE concepts and command line options.
+Except where noted, these concepts and command line argument are all
+applicable to Open MPI as well.  Open MPI extends the available PRRTE
+command line options, and also slightly modifies the PRRTE's default
+behaviors in a few cases.  These will be specifically described in the
+docuemtnation below.
+
+COMMAND LINE OPTIONS
+--------------------
+
+The core of Open MPI's ``mpirun`` processing is performed via the
+`PRRTE <https://docs.prrte.org/>`_.  Specifically: ``mpirun`` is
+effectively a wrapper around ``prterun``, but ``mpirun``'s CLI options
+are slightly different than PRRTE's CLI commands.
+
+.. include:: /schizo-ompi-rst-content/schizo-ompi-cli.rstxt
+
+OPTIONS (OLD / HARD-CODED CONTENT |mdash| TO BE AUDITED
+-------------------------------------------------------
+
+.. admonition:: This is old content
+   :class: error
+
+   This is the old section of manually hard-coded content.  It should
+   probably be read / audited and see what we want to keep and what we
+   want to discard.
+
+   Feel free to refer to https://docs.prrte.org/ rather than
+   replicating content here (e.g., for the definition of a slot and
+   other things).
+
+mpirun will send the name of the directory where it was invoked on the
+local node to each of the remote nodes, and attempt to change to that
+directory.  See the "Current Working Directory" section below for
+further details.
+
+* ``<program>``: The program executable. This is identified as the
+  first non-recognized argument to mpirun.
+
+* ``<args>``: Pass these run-time arguments to every new process.
+  These must always be the last arguments to mpirun. If an app context
+  file is used, ``<args>`` will be ignored.
+
+* ``-h``, ``--help``: Display help for this command
+
+* ``-q``, ``--quiet``: Suppress informative messages from orterun
+  during application execution.
+
+* ``-v``, ``--verbose``:` Be verbose
+
+* ``-V``, ``--version``: Print version number.  If no other arguments
+  are given, this will also cause orterun to exit.
+
+* ``-N <num>``: Launch num processes per node on all allocated nodes
+  (synonym for ``--npernode``).
+
+* ``--display-map``: Display a table showing the mapped location of
+  each process prior to launch.
+
+* ``--display-allocation``: Display the detected resource allocation.
+
+* ``--output-proctable``: Output the debugger proctable after launch.
+
+* ``--dvm``: Create a persistent distributed virtual machine (DVM).
+
+* ``--max-vm-size <size>``: Number of daemons to start.
+
+Use one of the following options to specify which hosts (nodes) of the
+cluster to run on. Note that as of the start of the v1.8 release,
+mpirun will launch a daemon onto each host in the allocation (as
+modified by the following options) at the very beginning of execution,
+regardless of whether or not application processes will eventually be
+mapped to execute there. This is done to allow collection of hardware
+topology information from the remote nodes, thus allowing us to map
+processes against known topology. However, it is a change from the
+behavior in prior releases where daemons were only launched after
+mapping was complete, and thus only occurred on nodes where
+application processes would actually be executing.
+
+* ``-H``, ``--host <host1,host2,...,hostN>``: list of hosts on which to
+  invoke processes.
+
+* ``--hostfile <hostfile>``: Provide a hostfile to use.
+
+* ``--default-hostfile <hostfile>``: Provide a default hostfile.
+
+* ``--machinefile <machinefile>``: Synonym for ``--hostfile``.
+
+* ``--cpu-set <list>``: Restrict launched processes to the specified
+  logical CPUs on each node (comma-separated list). Note that the
+  binding options will still apply within the specified envelope
+  |mdash| e.g., you can elect to bind each process to only one CPU
+  within the specified CPU set.
+
+The following options specify the number of processes to launch. Note
+that none of the options imply a particular binding policy |mdash| e.g.,
+requesting N processes for each package does not imply that the
+processes will be bound to the package.
+
+* ``-n``, ``--n``, ``-c``, ``-np <#>``: Run this many copies of the
+  program on the given nodes.  This option indicates that the
+  specified file is an executable program and not an application
+  context. If no value is provided for the number of copies to execute
+  (i.e., neither the ``-n`` nor its synonyms are provided on the
+  command line), Open MPI will automatically execute a copy of the
+  program on each process slot (see PRRTE's `defintion of "slot"
+  <https://docs.prrte.org/en/latest/placement/overview.html#definition-of-slot>`_
+  for description of a "process slot"). This feature, however, can
+  only be used in the SPMD model and will return an error (without
+  beginning execution of the application) otherwise.
+
+  .. note:: The ``-n`` option is the preferred option to be used to specify the
+            number of copies of the program to be executed, but the alternate
+            options are also accepted.
+
+
+* ``--map-by ppr:N:<object>``: Launch N times the number of objects of
+  the specified type on each node.
+
+* ``--npersocket <#persocket>``: On each node, launch this many
+  processes times the number of processor sockets on the node.
+  The -npersocket option also turns on the ``--bind-to-socket``
+  option.  (deprecated in favor of ``--map-by ppr:n:package``)
+
+* ``--npernode <#pernode>``: On each node, launch this many processes.
+  (deprecated in favor of ``--map-by ppr:n:node``).
+
+* ``--pernode``: On each node, launch one process |mdash| equivalent to
+  ``--npernode 1``.  (deprecated in favor of ``--map-by ppr:1:node``)
+
+To map processes:
+
+* ``--map-by <object>``: Map to the specified object, defaults to
+  ``package``. Supported options include ``slot``, ``hwthread``, ``core``,
+  ``L1cache``, ``L2cache``, ``L3cache``, ``package``, ``numa``,
+  ``node``, ``seq``, ``rankfile``, ``pe-list=#``, and ``ppr``.
+  Any object can include modifiers by adding a ``:`` and any combination
+  of the following:
+
+    * ``pe=n``: bind ``n`` processing elements to each proc
+    * ``span``: load balance the processes across the allocation
+    * ``oversubscribe``: allow more processes on a node than processing elements
+    * ``nooversubscribe``: do *not* allow more processes on a node than processing elements (default)
+    * ``nolocal``: do not place processes on the same host as the ``mpirun`` process
+    * ``hwtcpus``: use hardware threads as CPU slots for mapping
+    * ``corecpus``: use processor cores as CPU slots for mapping (default)
+    * ``file=filename``: used with ``rankfile``; use ``filename`` to specify the file to use
+    * ``ordered``: used with ``pe-list`` to bind each process to one of the specified processing elements
+
+  .. note:: ``socket`` is also accepted as an alias for ``package``.
+
+* ``--bycore``: Map processes by core (deprecated in favor of
+  ``--map-by core``).
+
+* ``--byslot``: Map and rank processes round-robin by slot (deprecated
+  in favor of ``--map-by slot``).
+
+* ``--nolocal``: Do not run any copies of the launched application on
+  the same node as orterun is running.  This option will override
+  listing the localhost with ``--host`` or any other host-specifying
+  mechanism. Alias for ``--map-by :nolocal``.
+
+* ``--nooversubscribe``: Do not oversubscribe any nodes; error
+  (without starting any processes) if the requested number of
+  processes would cause oversubscription.  This option implicitly sets
+  "max_slots" equal to the "slots" value for each node. (Enabled by
+  default). Alias for ``--map-by :nooversubscribe``.
+
+* ``--oversubscribe``: Nodes are allowed to be oversubscribed, even on
+  a managed system, and overloading of processing elements.
+  Alias for ``--map-by :oversubscribe``.
+
+* ``--bynode``: Launch processes one per node, cycling by node in a
+  round-robin fashion.  This spreads processes evenly among nodes and
+  assigns ``MPI_COMM_WORLD`` ranks in a round-robin, "by node" manner.
+  (deprecated in favor of ``--map-by node``)
+
+* ``--cpu-list <cpus>``: Comma-delimited list of processor IDs to
+  which to bind processes [default=``NULL``].  Processor IDs are
+  interpreted as hwloc logical core IDs.
+
+  .. note:: You can run Run the hwloc ``lstopo(1)`` command to see a
+            list of available cores and their logical IDs.
+
+To order processes' ranks in ``MPI_COMM_WORLD``:
+
+* ``--rank-by <mode>``: Rank in round-robin fashion according to the
+  specified mode, defaults to slot. Supported options include
+  ``slot``, ``node``, ``fill``, and ``span``.
+
+For process binding:
+
+* ``--bind-to <object>``: Bind processes to the specified object,
+  defaults to ``core``.  Supported options include ``slot``,
+  ``hwthread``, ``core``, ``l1cache``, ``l2cache``, ``l3cache``,
+  ``package``, ``numa``, and ``none``.
+
+* ``--cpus-per-proc <#perproc>``: Bind each process to the specified
+  number of cpus.  (deprecated in favor of ``--map-by <obj>:PE=n``)
+
+* ``--cpus-per-rank <#perrank>``: Alias for ``--cpus-per-proc``.
+  (deprecated in favor of ``--map-by <obj>:PE=n``)
+
+* ``--bind-to-core`` Bind processes to cores (deprecated in favor of
+  ``--bind-to core``)
+
+* ``--bind-to-socket``: Bind processes to processor sockets
+  (deprecated in favor of ``--bind-to package``)
+
+* ``--report-bindings``: Report any bindings for launched processes.
+
+For rankfiles:
+
+* ``--rankfile <rankfile>``: Provide a rankfile file.
+  (deprecated in favor of ``--map-by rankfile:file=FILE``)
+
+To manage standard I/O:
+
+* ``--output-filename <filename>``: Redirect the stdout, stderr, and
+  stddiag of all processes to a process-unique version of the
+  specified filename. Any directories in the filename will
+  automatically be created.  Each output file will consist of
+  ``filename.id``, where the ``id`` will be the processes' rank in
+  ``MPI_COMM_WORLD``, left-filled with zero's for correct ordering in
+  listings. A relative path value will be converted to an absolute
+  path based on the cwd where mpirun is executed. Note that this will
+  not work on environments where the file system on compute nodes
+  differs from that where :ref:`mpirun(1) <man1-mpirun>` is
+  executed.
+
+* ``--stdin <rank>``: The ``MPI_COMM_WORLD`` rank of the process that is
+  to receive stdin.  The default is to forward stdin to ``MPI_COMM_WORLD``
+  rank 0, but this option can be used to forward stdin to any
+  process. It is also acceptable to specify none, indicating that no
+  processes are to receive stdin.
+
+* ``--merge-stderr-to-stdout``: Merge stderr to stdout for each
+  process.
+
+* ``--tag-output``: Tag each line of output to stdout, stderr, and
+  stddiag with ``[jobid, MCW_rank]<stdxxx>`` indicating the process
+  jobid and ``MPI_COMM_WORLD`` rank of the process that generated the
+  output, and the channel which generated it.
+
+* ``--timestamp-output``: Timestamp each line of output to stdout,
+  stderr, and stddiag.
+
+* ``--xml``: Provide all output to stdout, stderr, and stddiag in an
+  XML format.
+
+* ``--xml-file <filename>`` Provide all output in XML format to the
+  specified file.
+
+* ``--xterm <ranks>``: Display the output from the processes
+  identified by their ``MPI_COMM_WORLD`` ranks in separate xterm
+  windows. The ranks are specified as a comma-separated list of
+  ranges, with a ``-1`` indicating all. A separate window will be created
+  for each specified process.
+
+  .. note:: xterm will normally terminate the window upon termination
+            of the process running within it. However, by adding a
+            ``!`` to the end of the list of specified ranks, the
+            proper options will be provided to ensure that xterm keeps
+            the window open after the process terminates, thus
+            allowing you to see the process' output.  Each xterm
+            window will subsequently need to be manually closed.
+            Note: In some environments, xterm may require that the
+            executable be in the user's path, or be specified in
+            absolute or relative terms. Thus, it may be necessary to
+            specify a local executable as ``./my_mpi_app`` instead of just
+            ``my_mpi_app``. If xterm fails to find the executable, ``mpirun``
+            will hang, but still respond correctly to a ctrl-C.  If
+            this happens, please check that the executable is being
+            specified correctly and try again.
+
+To manage files and runtime environment:
+
+* ``--path <path>``: ``<path>`` that will be used when attempting to
+  locate the requested executables.  This is used prior to using the
+  local ``PATH`` environment variable setting.
+
+* ``--prefix <dir>``: Prefix directory that will be used to set the
+  ``PATH`` and ``LD_LIBRARY_PATH`` on the remote node before invoking
+  Open MPI or the target process.  See the :ref:`Remote Execution
+  <man1-mpirun-remote-execution>` section, below.
+
+* ``--noprefix``: Disable the automatic ``--prefix`` behavior
+
+* ``--preload-binary``: Copy the specified executable(s) to remote
+  machines prior to starting remote processes. The executables will be
+  copied to the Open MPI session directory and will be deleted upon
+  completion of the job.
+
+* ``--preload-files <files>``: Preload the comma-separated list of
+  files to the current working directory of the remote machines where
+  processes will be launched prior to starting those processes.
+
+* ``--set-cwd-to-session-dir``: Set the working directory of the
+  started processes to their session directory.
+
+* ``--wd <dir>``: Synonym for ``-wdir``.
+
+* ``--wdir <dir>``: Change to the directory ``<dir>`` before the
+  user's program executes.  See the :ref:`Current Working Directory
+  <man1-mpirun-current-working-directory>` section for notes on
+  relative paths.  Note: If the ``--wdir`` option appears both on the
+  command line and in an application context, the context will take
+  precedence over the command line. Thus, if the path to the desired
+  wdir is different on the backend nodes, then it must be specified as
+  an absolute path that is correct for the backend node.
+
+* ``-x <env>``: Export the specified environment variables to the
+  remote nodes before executing the program.  Only one environment
+  variable can be specified per ``-x`` option.  Existing environment
+  variables can be specified or new variable names specified with
+  corresponding values.  For example:
+
+  .. code:: sh
+
+     shell$ mpirun -x DISPLAY -x OFILE=/tmp/out ...
+
+  The parser for the ``-x`` option is not very sophisticated; it does
+  not even understand quoted values.  Users are advised to set
+  variables in the environment, and then use ``-x`` to export (not
+  define) them.
+
+Setting MCA parameters:
+
+* ``--gmca <key> <value>``: Pass global MCA parameters that are
+  applicable to all contexts.  ``<key>`` is the parameter name;
+  ``<value>`` is the parameter value.
+
+* ``--mca <key> <value>``: Send arguments to various MCA modules.  See
+  the :ref:`Setting MCA Parameters
+  <man1-mpirun-setting-mca-parameters>` section for more details.
+
+  .. note:: Open MPI will attempt to discern PMIx and PRRTE MCA
+            parameters passed via ``--mca`` and handle them
+            appropriately, but it may not always guess correctly.  It
+            is best to use ``--pmixmca`` and ``--prtemca`` when
+            passing MCA parammeters to PMIx and PRRTE, respectively.
+
+* ``--pmixmca <key> <value>``: Send arguments to MCA modules in the
+  PMIx subsystem.  See the :ref:`Setting MCA Parameters
+  <man1-mpirun-setting-mca-parameters>` section for more details.
+
+* ``--prtemca <key> <value>``: Send arguments to MCA modules in the
+  PMIx Reference Runtime Environment (PRRTE) subsystem.  See the
+  :ref:`Setting MCA Parameters <man1-mpirun-setting-mca-parameters>`
+  section for more details.
+
+* ``--tune <tune_file>``: Specify a tune file to set arguments for
+  various MCA modules and environment variables.  See the :ref:`
+  Setting MCA parameters and environment variables from file
+  <man1-mpirun-setting-mca-params-from-file>`. ``--am <arg>`` is an alias for ``--tune <arg>``.
+
+For debugging:
+
+* ``--get-stack-traces``: When paired with the ``--timeout`` option,
+  ``mpirun`` will obtain and print out stack traces from all launched
+  processes that are still alive when the timeout expires.  Note that
+  obtaining stack traces can take a little time and produce a lot of
+  output, especially for large process-count jobs.
+
+* ``--timeout <seconds>``: The maximum number of seconds that
+  ``mpirun`` will run.  After this many seconds, ``mpirun`` will abort
+  the launched job and exit with a non-zero exit status.  Using
+  ``--timeout`` can be also useful when combined with the
+  ``--get-stack-traces`` option.
+
+There are also other options:
+
+* ``--allow-run-as-root``: Allow ``mpirun`` to run when executed by
+  the root user (``mpirun`` defaults to aborting when launched as the
+  root user).  Be sure to see the :ref:`Running as root
+  <man1-mpirun-running-as-root>` section for more detail.
+
+* ``--app <appfile>``: Provide an appfile, ignoring all other command
+  line options.
+
+* ``--continuous``: Job is to run until explicitly terminated.
+
+* ``--disable-recovery``: Disable recovery (resets all recovery
+  options to off).
+
+* ``--do-not-launch``: Perform all necessary operations to prepare to
+  launch the application, but do not actually launch it.
+
+* ``--enable-recovery``: Enable recovery from process failure (default:
+  disabled)
+
+* ``--leave-session-attached``: Do not detach back-end daemons used by
+  this application. This allows error messages from the daemons as
+  well as the underlying environment (e.g., when failing to launch a
+  daemon) to be output.
+
+* ``--max-restarts <num>``: Max number of times to restart a failed
+  process.
+
+* ``--personality <list>``: Comma-separated list of programming model,
+  languages, and containers being used (default=``ompi``).
+
+* ``--ppr <list>``: Comma-separated list of number of processes on a
+  given resource type (default: none). Alias for ``--map-by ppr:N:OBJ``.
+
+* ``--report-child-jobs-separately``: Return the exit status of the
+  primary job only.
+
+* ``--report-events <URI>``: Report events to a tool listening at the
+  specified URI.
+
+* ``--report-pid <channel>``: Print out ``mpirun``'s PID during
+  startup. The channel must be either a ``-`` to indicate that the PID
+  is to be output to stdout, a ``+`` to indicate that the PID is to be
+  output to stderr, or a filename to which the PID is to be written.
+
+* ``--report-uri <channel>``: Print out ``mpirun``'s URI during
+  startup. The channel must be either a ``-`` to indicate that the URI
+  is to be output to stdout, a ``+`` to indicate that the URI is to be
+  output to stderr, or a filename to which the URI is to be written.
+
+* ``--show-progress``: Output a brief periodic report on launch
+  progress.
+
+* ``--terminate``: Terminate the DVM.
+
+* ``--use-hwthread-cpus``: Use hardware threads as independent CPUs.
+
+  Note that if a number of slots is not provided to Open MPI (e.g.,
+  via the ``slots`` keyword in a hostfile or from a resource manager
+  such as Slurm), the use of this option changes the default
+  calculation of number of slots on a node.  See the PRRTE's
+  `defintion of "slot"
+  <https://docs.prrte.org/en/latest/placement/overview.html#definition-of-slot>`_
+  for more details.
+
+  Also note that the use of this option changes the Open MPI's
+  definition of a "processor element" from a processor core to a
+  hardware thread.  See
+  PRRTE's `defintion of a "processor element"
+  <https://docs.prrte.org/en/latest/placement/overview.html#definition-of-pe>`_
+  for more details.
+
+The following options are useful for developers; they are not
+generally useful to most Open MPI users:
+
+* ``--debug-daemons``: Enable debugging of the run-time daemons used
+  by this application.
+
+* ``--debug-daemons-file``: Enable debugging of the run-time daemons
+  used by this application, storing output in files.
+
+* ``--display-devel-map``: Display a more detailed table showing the
+  mapped location of each process prior to launch.
+
+* ``--display-topo``: Display the topology as part of the process map
+  just before launch.
+
+* ``--launch-agent``: Name of the executable that is to be used to
+  start processes on the remote nodes. The default is ``prted``. This
+  option can be used to test new daemon concepts, or to pass options
+  back to the daemons without having mpirun itself see them. For
+  example, specifying a launch agent of ``prted --prtemca odls_base_verbose
+  5`` allows the developer to ask the ``prted`` for debugging output
+  without clutter from ``mpirun`` itself.
+
+* ``--report-state-on-timeout``: When paired with the ``--timeout``
+  command line option, report the run-time subsystem state of each
+  process when the timeout expires.
+
+There may be other options listed with ``mpirun --help``.
+
+Environment Variables
+^^^^^^^^^^^^^^^^^^^^^
+
+.. admonition:: This is old, hard-coded content
+   :class: error
+
+   Is this content still current / accurate?  Should it be updated and
+   retained, or removed?
+
+* ``MPIEXEC_TIMEOUT``: Synonym for the ``--timeout`` command line option.
+
+DESCRIPTION
+-----------
+
+.. admonition:: This is old, hard-coded content
+   :class: error
+
+   Is this content still current / accurate?  Should it be updated and
+   retained, or removed?
+
+One invocation of ``mpirun`` starts an MPI application running under Open
+MPI. If the application is single process multiple data (SPMD), the
+application can be specified on the ``mpirun`` command line.
+
+If the application is multiple instruction multiple data (MIMD),
+comprising of multiple programs, the set of programs and argument can
+be specified in one of two ways: Extended Command Line Arguments, and
+Application Context.
+
+An application context describes the MIMD program set including all
+arguments in a separate file.  This file essentially contains multiple
+mpirun command lines, less the command name itself.  The ability to
+specify different options for different instantiations of a program is
+another reason to use an application context.
+
+Extended command line arguments allow for the description of the
+application layout on the command line using colons (``:``) to
+separate the specification of programs and arguments. Some options are
+globally set across all specified programs (e.g., ``--hostfile``),
+while others are specific to a single program (e.g., ``-n``).
+
+Specifying Host Nodes
+^^^^^^^^^^^^^^^^^^^^^
+
+.. admonition:: This is old, hard-coded content
+   :class: error
+
+   Is this content still current / accurate?  Should it be updated and
+   retained, or removed?
+
+Host nodes can be identified on the ``mpirun`` command line with the
+``--host`` option or in a hostfile.
+
+For example:
+
+.. code:: sh
+
+   shell$ mpirun -H aa,aa,bb ./a.out
+
+Launches two processes on node ``aa`` and one on ``bb``.
+
+Or, consider the hostfile:
+
+.. code:: sh
+
+   shell$ cat myhostfile
+   aa slots=2
+   bb slots=2
+   cc slots=2
+
+Here, we list both the host names (``aa``, ``bb``, and ``cc``) but
+also how many slots there are for each.
+
+.. code:: sh
+
+   shell$ mpirun --hostfile myhostfile ./a.out
+
+will launch two processes on each of the three nodes.
+
+.. code:: sh
+
+   shell$ mpirun --hostfile myhostfile --host aa ./a.out
+
+will launch two processes, both on node ``aa``.
+
+.. code:: sh
+
+   shell$ mpirun --hostfile myhostfile --host dd ./a.out
+
+will find no hosts to run on and will abort with an error.  That is,
+the specified host ``dd`` is not in the specified hostfile.
+
+When running under resource managers (e.g., Slurm, Torque, etc.), Open
+MPI will obtain both the hostnames and the number of slots directly
+from the resource manager.
+
+Specifying Number of Processes
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. admonition:: This is old, hard-coded content
+   :class: error
+
+   Is this content still current / accurate?  Should it be updated and
+   retained, or removed?
+
+As we have just seen, the number of processes to run can be set using the
+hostfile.  Other mechanisms exist.
+
+The number of processes launched can be specified as a multiple of the
+number of nodes or processor packages available.  For example,
+
+.. code:: sh
+
+   shell$ mpirun -H aa,bb --map-by ppr:2:package ./a.out
+
+launches processes 0-3 on node ``aa`` and process 4-7 on node ``bb``
+(assuming ``aa`` and ``bb`` both contain 4 slots each).
+
+.. code:: sh
+
+   shell$ mpirun -H aa,bb --map-by ppr:2:node ./a.out
+
+launches processes 0-1 on node ``aa`` and processes 2-3 on node ``bb``.
+
+.. code:: sh
+
+   shell$ mpirun -H aa,bb --map-by ppr:1:node ./a.out
+
+launches one process per host node.
+
+.. code:: sh
+
+   mpirun -H aa,bb --pernode ./a.out
+
+is the same as ``--map-by ppr:1:node`` and ``--npernode 1``.
+
+Another alternative is to specify the number of processes with the ``-n``
+option.  Consider now the hostfile:
+
+.. code:: sh
+
+   shell$ cat myhostfile
+   aa slots=4
+   bb slots=4
+   cc slots=4
+
+Now run with ``myhostfile``:
+
+.. code:: sh
+
+   shell$ mpirun --hostfile myhostfile -n 6 ./a.out
+
+will launch processes 0-3 on node ``aa`` and processes 4-5 on node
+``bb``.  The remaining slots in the hostfile will not be used since
+the ``-n`` option indicated that only 6 processes should be launched.
+
+Mapping Processes to Nodes: Using Policies
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. admonition:: This is old, hard-coded content
+   :class: error
+
+   Is this content still current / accurate?  Should it be updated and
+   retained, or removed?
+
+The examples above illustrate the default mapping of process processes
+to nodes.  This mapping can also be controlled with various ``mpirun``
+options that describe mapping policies.
+
+Consider the same hostfile as above, again with ``-n 6``.  The table
+below lists a few ``mpirun`` variations, and shows which
+``MPI_COMM_WORLD`` ranks end up on which node:
+
+.. list-table::
+   :header-rows: 1
+
+   * - Command
+     - Node ``aa``
+     - Node ``bb``
+     - Node ``cc``
+
+   * - ``mpirun``
+     - 0 1 2 3
+     - 4 5
+     -
+
+   * - ``mpirun --map-by node``
+     - 0 3
+     - 1 4
+     - 2 5
+
+   * - ``mpirun --nolocal``
+     -
+     - 0 1 2 3
+     - 4 5
+
+The ``--map-by node`` option will load balance the processes across the
+available nodes, numbering each process in a round-robin fashion.
+
+The ``--nolocal`` option prevents any processes from being mapped onto
+the local host (in this case node ``aa``).  While ``mpirun`` typically
+consumes few system resources, ``--nolocal`` can be helpful for
+launching very large jobs where mpirun may actually need to use
+noticeable amounts of memory and/or processing time.
+
+Just as ``-n`` can specify fewer processes than there are slots, it
+can also oversubscribe the slots.  For example, with the same
+hostfile:
+
+.. code:: sh
+
+   shell$ mpirun --hostfile myhostfile -n 14 ./a.out
+
+will launch processes 0-3 on node ``aa``, 4-7 on ``bb``, and 8-11 on
+``cc``.  It will then add the remaining two processes to whichever
+nodes it chooses.
+
+One can also specify limits to oversubscription.  For example, with the
+same hostfile:
+
+.. code:: sh
+
+   shell$ mpirun --hostfile myhostfile -n 14 --nooversubscribe ./a.out
+
+will produce an error since ``--nooversubscribe`` prevents
+oversubscription.
+
+Limits to oversubscription can also be specified in the hostfile
+itself:
+
+.. code:: sh
+
+   shell$ cat myhostfile
+   aa slots=4 max_slots=4
+   bb         max_slots=4
+   cc slots=4
+
+The ``max_slots`` field specifies such a limit.  When it does, the slots
+value defaults to the limit.  Now:
+
+.. code:: sh
+
+   shell$ mpirun --hostfile myhostfile -n 14 ./a.out
+
+causes the first 12 processes to be launched as before, but the
+remaining two processes will be forced onto node ``cc``.  The other
+two nodes are protected by the hostfile against oversubscription by
+this job.
+
+Using the ``--nooversubscribe`` option can be helpful since Open MPI
+currently does not get ``max_slots`` values from the resource manager.
+
+Of course, ``-n`` can also be used with the ``-H`` or ``-host``
+option.  For example:
+
+.. code:: sh
+
+   shell$ mpirun -H aa,bb -n 8 ./a.out
+
+launches 8 processes.  Since only two hosts are specified, after the
+first two processes are mapped, one to ``aa`` and one to ``bb``, the
+remaining processes oversubscribe the specified hosts.
+
+And here is a MIMD example:
+
+.. code:: sh
+
+   shell$ mpirun -H aa -n 1 hostname : -H bb,cc -n 2 uptime
+
+will launch process 0 running hostname on node ``aa`` and processes 1
+and 2 each running uptime on nodes ``bb`` and ``cc``, respectively.
+
+.. _man1-mpirun-map-rank-bind:
+
+Mapping, Ranking, and Binding: Oh My!
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. admonition:: This is old, hard-coded content
+   :class: error
+
+   Is this content still current / accurate?  Should it be updated and
+   retained, or removed?
+
+Open MPI employs a three-phase procedure for assigning process locations
+and ranks:
+
+#. **Mapping**: Assigns a default location to each process
+#. **Ranking**: Assigns an ``MPI_COMM_WORLD`` rank value to each process
+#. **Binding**: Constrains each process to run on specific processors
+
+The mapping step is used to assign a default location to each process
+based on the mapper being employed. Mapping by slot, node, and
+sequentially results in the assignment of the processes to the node
+level. In contrast, mapping by object, allows the mapper to assign the
+process to an actual object on each node.
+
+Note that the location assigned to the process is independent of where
+it will be bound |mdash| the assignment is used solely as input to the
+binding algorithm.
+
+The mapping of process processes to nodes can be defined not just with
+general policies but also, if necessary, using arbitrary mappings that
+cannot be described by a simple policy.  One can use the "sequential
+mapper," which reads the hostfile line by line, assigning processes to
+nodes in whatever order the hostfile specifies.  Use the ``---map-by seq`` option.  For example, using the same hostfile as before:
+
+.. code:: sh
+
+   shell$ mpirun -hostfile myhostfile --map-by seq ./a.out
+
+will launch three processes, one on each of nodes ``aa``, ``bb``, and ``cc``,
+respectively.  The slot counts don't matter; one process is launched
+per line on whatever node is listed on the line.
+
+Another way to specify arbitrary mappings is with a rankfile, which
+gives you detailed control over process binding as well.  Rankfiles
+are discussed :ref:`below <man1-mpirun-rankfiles>`.
+
+The second phase focuses on the ranking of the process within the
+job's ``MPI_COMM_WORLD``.  Open MPI separates this from the mapping
+procedure to allow more flexibility in the relative placement of MPI
+processes. This is best illustrated by considering the following
+cases where we used the ``--np 8 --map-by ppr:2:package --host aa:4,bb:4`` option:
+
+.. list-table::
+   :header-rows: 1
+
+   * - Option
+     - Node ``aa``
+     - Node ``bb``
+
+   * - ``--rank-by fill`` (i.e., dense packing) Default
+     - 0 1 | 2 3
+     - 4 5 | 6 7
+
+   * - ``--rank-by span`` (i.e., sparse or load balanced packing)
+     - 0 4 | 1 5
+     - 2 6 | 3 7
+
+   * - ``--rank-by node``
+     - 0 2 | 4 6
+     - 1 3 | 5 7
+
+Ranking by ``fill`` assigns MCW ranks in a simple progression across each
+node. Ranking by ``span`` and by ``slot`` provide the identical
+result |mdash| a round-robin progression of the packages across all nodes
+before returning to the first package on the first node. Ranking by
+``node`` assigns MCW ranks iterating first across nodes then by package.
+
+The binding phase actually binds each process to a given set of
+processors. This can improve performance if the operating system is
+placing processes suboptimally.  For example, it might oversubscribe
+some multi-core processor packages, leaving other packages idle; this
+can lead processes to contend unnecessarily for common resources.  Or,
+it might spread processes out too widely; this can be suboptimal if
+application performance is sensitive to interprocess communication
+costs.  Binding can also keep the operating system from migrating
+processes excessively, regardless of how optimally those processes
+were placed to begin with.
+
+The processors to be used for binding can be identified in terms of
+topological groupings |mdash| e.g., binding to an ``l3cache`` will
+bind each process to all processors within the scope of a single L3
+cache within their assigned location. Thus, if a process is assigned
+by the mapper to a certain package, then a ``--bind-to l3cache``
+directive will cause the process to be bound to the processors that
+share a single L3 cache within that package.
+
+Alternatively, processes can be mapped and bound to specified cores using
+the ``--map-by pe-list=`` option. For example, ``--map-by pe-list=0,2,5``
+will map three processes all three of which will be bound to logical cores
+``0,2,5``. If you intend to bind each of the three processes to different
+cores then the ``:ordered`` qualifier can be used like
+``--map-by pe-list=0,2,5:ordered``. In this example, the first process
+on a node will be bound to CPU 0, the second process on the node will
+be bound to CPU 2, and the third process on the node will be bound to
+CPU 5.
+
+Finally, ``--report-bindings`` can be used to report bindings.
+
+As an example, consider a node with two processor packages, each
+comprised of four cores, and each of those cores contains one hardware
+thread.  The ``--report-bindings`` option shows the binding of each process in a
+descriptive manner. Below are some examples.
+
+.. code::
+
+   shell$ mpirun --np 4 --report-bindings --map-by core --bind-to core
+   [...] Rank 0 bound to package[0][core:0]
+   [...] Rank 1 bound to package[0][core:1]
+   [...] Rank 2 bound to package[0][core:2]
+   [...] Rank 3 bound to package[0][core:3]
+
+In the above case, the processes bind to successive cores.
+
+.. code::
+
+   shell$ mpirun --np 4 --report-bindings --map-by package --bind-to package
+   [...] Rank 0 bound to package[0][core:0-3]
+   [...] Rank 1 bound to package[0][core:0-3]
+   [...] Rank 2 bound to package[1][core:4-7]
+   [...] Rank 3 bound to package[1][core:4-7]
+
+In the above case, processes bind to all cores on successive packages.
+The processes cycle through the processor packages in a
+round-robin fashion as many times as are needed. By default, the processes
+are ranked in a ``fill`` manner.
+
+.. code::
+
+   shell$ mpirun --np 4 --report-bindings --map-by package --bind-to package --rank-by span
+   [...] Rank 0 bound to package[0][core:0-3]
+   [...] Rank 1 bound to package[1][core:4-7]
+   [...] Rank 2 bound to package[0][core:0-3]
+   [...] Rank 3 bound to package[1][core:4-7]
+
+The above case demonstrates the difference
+in ranking when the ``span`` qualifier is used instead of the default.
+
+.. code::
+
+   shell$ mpirun --np 4 --report-bindings --map-by slot:PE=2 --bind-to core
+   [...] Rank 0 bound to package[0][core:0-1]
+   [...] Rank 1 bound to package[0][core:2-3]
+   [...] Rank 2 bound to package[0][core:4-5]
+   [...] Rank 3 bound to package[0][core:6-7]
+
+In the above case, the output shows us that 2 cores have been bound per
+process.  Specifically, the mapping by ``slot`` with the ``PE=2`` qualifier
+indicated that each slot (i.e., process) should consume two processor
+elements.  By default, Open MPI defines "processor element" as "core",
+and therefore the ``--bind-to core`` caused each process to be bound to
+both of the cores to which it was mapped.
+
+.. code::
+
+   shell$ mpirun --np 4 --report-bindings --map-by slot:PE=2 --use-hwthread-cpus
+   [...]] Rank 0 bound to package[0][hwt:0-1]
+   [...]] Rank 1 bound to package[0][hwt:2-3]
+   [...]] Rank 2 bound to package[0][hwt:4-5]
+   [...]] Rank 3 bound to package[0][hwt:6-7]
+
+In the above case, we replace the ``--bind-to core`` with
+``--use-hwthread-cpus``. The ``--use-hwthread-cpus`` is converted into
+``--bind-to hwthread`` and tells the ``--report-bindings`` output to show the
+hardware threads to which a process is bound. In this case, processes are
+bound to 2 hardware threads per process.
+
+.. code::
+
+   shell$ mpirun --np 4 --report-bindings --bind-to none
+   [...] Rank 0 is not bound (or bound to all available processors)
+   [...] Rank 1 is not bound (or bound to all available processors)
+   [...] Rank 2 is not bound (or bound to all available processors)
+   [...] Rank 3 is not bound (or bound to all available processors)
+
+In the above case, binding is turned off and are reported as such.
+
+Open MPI's support for process binding depends on the underlying
+operating system.  Therefore, certain process binding options may not
+be available on every system.
+
+Process binding can also be set with MCA parameters.  Their usage is
+less convenient than that of ``mpirun`` options.  On the other hand,
+MCA parameters can be set not only on the mpirun command line, but
+alternatively in a system or user ``mca-params.conf`` file or as
+environment variables, as described in the :ref:`Setting MCA
+Parameters <man1-mpirun-setting-mca-parameters>`. These are MCA parameters for
+the PRRTE runtime so the command line argument ``--prtemca``
+(yes, ``prte`` with a single ``r``, not two ``r``'s) must be used to
+pass the MCA parameter key/value pair. Alternatively, the MCA parameter
+key/value pair may be specific on the command line by prefixing the key with
+``PRTE_MCA_`` (again, that is not a typo: ``PRTE`` not ``PRRTE``).
+Some examples include:
+
+.. list-table::
+   :header-rows: 1
+
+   * - Option
+     - PRRTE MCA parameter key
+     - Value
+
+   * - ``--map-by core``
+     - ``rmaps_default_mapping_policy``
+     - ``core``
+
+   * - ``--map-by package``
+     - ``rmaps_default_mapping_policy``
+     - ``package``
+
+   * - ``--rank-by fill``
+     - ``rmaps_default_ranking_policy``
+     - ``fill``
+
+   * - ``--bind-to core``
+     - ``hwloc_default_binding_policy``
+     - ``core``
+
+   * - ``--bind-to package``
+     - ``hwloc_default_binding_policy``
+     - ``package``
+
+   * - ``--bind-to none``
+     - ``hwloc_default_binding_policy``
+     - ``none``
+
+.. _man1-mpirun-map-rank-bind-defaults:
+
+Defaults for Mapping, Ranking, and Binding
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. admonition:: This is old, hard-coded content
+   :class: error
+
+   Is this content still current / accurate?  Should it be updated and
+   retained, or removed?
+
+If the user does not specify each of ``--map-by``, ``--rank-by``, and ``--bind-to`` option then the default values are as follows:
+
+* If no options are specified then
+
+    * If the number of processes is less than or equal to 2, then:
+
+        * ``--map-by`` is ``core``
+        * ``--bind-to`` is ``core``
+        * ``--rank-by`` is ``span``
+        * Result: ``--map-by core --bind-to core --rank-by span``
+
+    * Otherwise:
+
+        * ``--map-by`` is ``package``
+        * ``--bind-to`` is ``package``
+        * ``--rank-by`` is ``fill``
+        * Result: ``--map-by package --bind-to package --rank-by fill``
+
+* If only ``--map-by OBJ`` (where ``OBJ`` is something like ``core``) is specified, then:
+
+    * ``--map-by`` specified ``OBJ``
+    * ``--bind-to`` uses the same ``OBJ`` as ``--map-by``
+    * ``--rank-by`` defaults to ``fill``
+    * Result: ``--map-by OBJ --bind-to OBJ --rank-by fill``
+
+* If only ``--bind-to OBJ`` (where ``OBJ`` is something like ``core``) is specified, then:
+
+    * ``--map-by`` is either ``core`` or ``package`` depending on the number of processes
+    * ``--bind-to`` specified ``OBJ``
+    * ``--rank-by`` defaults to ``fill``
+    * Result: ``--map-by OBJ --bind-to OBJ --rank-by fill``
+
+* If ``--map-by OBJ1 --bind-to OBJ2``, then:
+
+    * ``--map-by`` specified ``OBJ1``
+    * ``--bind-to`` specified ``OBJ2``
+    * ``--rank-by`` defaults to ``fill``
+    * Result: ``--map-by OBJ2 --bind-to OBJ2 --rank-by fill``
+
+
+Consider 2 identical hosts (``hostA`` and ``hostB``) with 2 packages (denoted by ``[]``) each with 8 cores (denoted by ``/../``) and 2 hardware threads per core (denoted by a ``.``).
+
+Default of ``--map-by core --bind-to core --rank-by span`` when the number of processes is less than or equal to 2.
+
+.. code::
+
+  shell$ mpirun --np 2 --host hostA:4,hostB:2 ./a.out
+  R0  hostA  [BB/../../../../../../..][../../../../../../../..]
+  R1  hostA  [../BB/../../../../../..][../../../../../../../..]
+
+Default of ``--map-by package --bind-to package --rank-by fill`` when the number of processes is greater than 2.
+
+.. code::
+
+  shell$ mpirun --np 4 --host hostA:4,hostB:2 ./a.out
+  R0  hostA  [BB/BB/BB/BB/BB/BB/BB/BB][../../../../../../../..]
+  R1  hostA  [BB/BB/BB/BB/BB/BB/BB/BB][../../../../../../../..]
+  R2  hostA  [../../../../../../../..][BB/BB/BB/BB/BB/BB/BB/BB]
+  R3  hostA  [../../../../../../../..][BB/BB/BB/BB/BB/BB/BB/BB]
+
+If only ``--map-by OBJ`` is specified, then it implies ``--bind-to OBJ --rank-by fill``. The example below results in ``--map-by hwthread --bind-to hwthread --rank-by fill``
+
+.. code::
+
+  shell$ mpirun --np 4 --map-by hwthread --host hostA:4,hostB:2 ./a.out
+  R0  hostA  [B./../../../../../../..][../../../../../../../..]
+  R1  hostA  [.B/../../../../../../..][../../../../../../../..]
+  R0  hostA  [../B./../../../../../..][../../../../../../../..]
+  R1  hostA  [../.B/../../../../../..][../../../../../../../..]
+
+If only ``--bind-to OBJ`` is specified, then ``--map-by`` is determined by the number of processes and ``--rank-by fill``. The example below results in ``--map-by package --bind-to core --rank-by fill``
+
+.. code::
+
+  shell$ mpirun --np 4 --bind-to core --host hostA:4,hostB:2 ./a.out
+  R0  hostA  [BB/../../../../../../..][../../../../../../../..]
+  R1  hostA  [../BB/../../../../../..][../../../../../../../..]
+  R2  hostA  [../../../../../../../..][BB/../../../../../../..]
+  R3  hostA  [../../../../../../../..][../BB/../../../../../..]
+
+The mapping pattern might be better seen if we change the default ``--rank-by`` from ``fill`` to ``span``. First, the processes are mapped by package iterating between the two marking a core at a time. Next, the processes are ranked in a spanning manner that load balances them across the object they were mapped against. Finally, the processes are bound to the core that they were mapped againast.
+
+.. code::
+
+  shell$ mpirun --np 4 --bind-to core --rank-by span --host hostA:4,hostB:2 ./a.out
+  R0  hostA  [BB/../../../../../../..][../../../../../../../..]
+  R1  hostA  [../../../../../../../..][BB/../../../../../../..]
+  R2  hostA  [../BB/../../../../../..][../../../../../../../..]
+  R3  hostA  [../../../../../../../..][../BB/../../../../../..]
+
+
+.. _man1-mpirun-rankfiles:
+
+Rankfiles
+^^^^^^^^^
+
+.. admonition:: This is old, hard-coded content
+   :class: error
+
+   Is this content still current / accurate?  Should it be updated and
+   retained, or removed?
+
+Rankfiles are text files that specify detailed information about how
+individual processes should be mapped to nodes, and to which
+processor(s) they should be bound.  Each line of a rankfile specifies
+the location of one process (for MPI jobs, the process' "rank" refers
+to its rank in ``MPI_COMM_WORLD``).  The general form of each line in the
+rankfile is:
+
+.. code::
+
+   rank <N>=<hostname> slot=<slot list>
+
+For example:
+
+.. code::
+
+   shell$ cat myrankfile
+   rank 0=aa slot=1:0-2
+   rank 1=bb slot=0:0,1
+   rank 2=cc slot=2-3
+   shell$ mpirun -H aa,bb,cc,dd --map-by rankfile:file=myrankfile ./a.out
+
+Means that:
+
+* Rank 0 runs on node aa, bound to logical package 1, cores 0-2.
+* Rank 1 runs on node bb, bound to logical package 0, cores 0 and 1.
+* Rank 2 runs on node cc, bound to logical cores 2 and 3.
+
+Note that only logicical processor locations are supported. By default, the values specifed are assumed to be cores. If you intend to specify specific hardware threads then you must add the ``:hwtcpus`` qualifier to the ``--map-by`` command line option (e.g., ``--map-by rankfile:file=myrankfile:hwtcpus``).
+
+If the binding specification overlaps between any two ranks then an error occurs. If you intend to allow processes to share the same logical processing unit then you must pass the ``--bind-to :overload-allowed`` command line option to tell the runtime to ignore this check.
+
+The hostnames listed above are "absolute," meaning that actual
+resolveable hostnames are specified.  However, hostnames can also be
+specified as "relative," meaning that they are specified in relation
+to an externally-specified list of hostnames (e.g., by ``mpirun``'s
+``--host`` argument, a hostfile, or a job scheduler).
+
+The "relative" specification is of the form ``+n<X>``, where X is an
+integer specifying the Xth hostname in the set of all available
+hostnames, indexed from 0.  For example:
+
+.. code::
+
+   shell$ cat myrankfile
+   rank 0=+n0 slot=1:0-2
+   rank 1=+n1 slot=0:0,1
+   rank 2=+n2 slot=2-3
+   shell$ mpirun -H aa,bb,cc,dd --map-by rankfile:file=myrankfile ./a.out
+
+All package/core slot locations are specified as logical indexes.
+
+.. note:: The Open MPI v1.6 series used physical indexes. Starting in Open MPI v5.0 only logicial indexes are supported and the ``rmaps_rank_file_physical`` MCA parameter is no longer recognized.
+
+You can use tools such as Hwloc's `lstopo(1)` to find the logical
+indexes of package and cores.
+
+Application Context or Executable Program?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. admonition:: This is old, hard-coded content
+   :class: error
+
+   Is this content still current / accurate?  Should it be updated and
+   retained, or removed?
+
+To distinguish the two different forms, mpirun looks on the command
+line for ``--app`` option.  If it is specified, then the file named on
+the command line is assumed to be an application context.  If it is
+not specified, then the file is assumed to be an executable program.
+
+Locating Files
+^^^^^^^^^^^^^^
+
+.. admonition:: This is old, hard-coded content
+   :class: error
+
+   Is this content still current / accurate?  Should it be updated and
+   retained, or removed?
+
+If no relative or absolute path is specified for a file, Open MPI will
+first look for files by searching the directories specified by the
+``--path`` option.  If there is no ``--path`` option set or if the
+file is not found at the ``--path`` location, then Open MPI will
+search the user's ``PATH`` environment variable as defined on the
+source node(s).
+
+If a relative directory is specified, it must be relative to the
+initial working directory determined by the specific starter used. For
+example when using the ssh starter, the initial directory is ``$HOME``
+by default.  Other starters may set the initial directory to the
+current working directory from the invocation of ``mpirun``.
+
+.. _man1-mpirun-current-working-directory:
+
+Current Working Directory
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. admonition:: This is old, hard-coded content
+   :class: error
+
+   Is this content still current / accurate?  Should it be updated and
+   retained, or removed?
+
+The ``--wdir`` ``mpirun`` option (and its synonym, ``--wd``) allows
+the user to change to an arbitrary directory before the program is
+invoked.  It can also be used in application context files to specify
+working directories on specific nodes and/or for specific
+applications.
+
+If the ``--wdir`` option appears both in a context file and on the
+command line, the context file directory will override the command
+line value.
+
+If the ``-wdir`` option is specified, Open MPI will attempt to change
+to the specified directory on all of the remote nodes. If this fails,
+``mpirun`` will abort.
+
+If the ``-wdir`` option is not specified, Open MPI will send the
+directory name where ``mpirun`` was invoked to each of the remote
+nodes.  The remote nodes will try to change to that directory.  If
+they are unable (e.g., if the directory does not exist on that node),
+then Open MPI will use the default directory determined by the
+starter.
+
+All directory changing occurs before the user's program is invoked; it
+does not wait until :ref:`MPI_INIT(3) <mpi_init>` is called.
+
+Standard I/O
+^^^^^^^^^^^^
+
+.. admonition:: This is old, hard-coded content
+   :class: error
+
+   Is this content still current / accurate?  Should it be updated and
+   retained, or removed?
+
+Open MPI directs UNIX standard input to ``/dev/null`` on all processes
+except the ``MPI_COMM_WORLD`` rank 0 process. The ``MPI_COMM_WORLD`` rank 0
+process inherits standard input from ``mpirun``.
+
+.. note:: The node that invoked ``mpirun`` need not be the same as the
+          node where the ``MPI_COMM_WORLD`` rank 0 process resides. Open
+          MPI handles the redirection of ``mpirun``'s standard input
+          to the rank 0 process.
+
+Open MPI directs UNIX standard output and error from remote nodes to
+the node that invoked ``mpirun`` and prints it on the standard
+output/error of ``mpirun``.  Local processes inherit the standard
+output/error of ``mpirun`` and transfer to it directly.
+
+Thus it is possible to redirect standard I/O for Open MPI applications
+by using the typical shell redirection procedure on ``mpirun``.  For
+example:
+
+.. code:: sh
+
+   shell$ mpirun -n 2 my_app < my_input > my_output
+
+Note that in this example only the ``MPI_COMM_WORLD`` rank 0 process will
+receive the stream from ``my_input`` on stdin.  The stdin on all the other
+nodes will be tied to ``/dev/null``.  However, the stdout from all nodes
+will be collected into the ``my_output`` file.
+
+Signal Propagation
+^^^^^^^^^^^^^^^^^^
+
+.. admonition:: This is old, hard-coded content
+   :class: error
+
+   Is this content still current / accurate?  Should it be updated and
+   retained, or removed?
+
+When ``mpirun`` receives a SIGTERM and SIGINT, it will attempt to kill
+the entire job by sending all processes in the job a SIGTERM, waiting
+a small number of seconds, then sending all processes in the job a
+SIGKILL.
+
+SIGUSR1 and SIGUSR2 signals received by ``mpirun`` are propagated to all
+processes in the job.
+
+A SIGTSTOP signal to ``mpirun`` will cause a SIGSTOP signal to be sent
+to all of the programs started by ``mpirun`` and likewise a SIGCONT
+signal to ``mpirun`` will cause a SIGCONT sent.
+
+Other signals are not currently propagated by ``mpirun``.
+
+Process Termination / Signal Handling
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. admonition:: This is old, hard-coded content
+   :class: error
+
+   Is this content still current / accurate?  Should it be updated and
+   retained, or removed?
+
+During the run of an MPI application, if any process dies abnormally
+(either exiting before invoking :ref:`MPI_FINALIZE(3) <mpi_finalize>`,
+or dying as the result of a signal), ``mpirun`` will print out an
+error message and kill the rest of the MPI application.
+
+User signal handlers should probably avoid trying to cleanup MPI state
+(Open MPI is currently not async-signal-safe; see
+:ref:`MPI_INIT_THREAD(3) <mpi_init_thread>` for details about
+MPI_THREAD_MULTIPLE and thread safety).  For example, if a
+segmentation fault occurs in :ref:`MPI_SEND(3) <mpi_send>` (perhaps
+because a bad buffer was passed in) and a user signal handler is
+invoked, if this user handler attempts to invoke :ref:`MPI_FINALIZE(3)
+<mpi_finalize>`, Bad Things could happen since Open MPI was already
+"in" MPI when the error occurred.  Since ``mpirun`` will notice that the
+process died due to a signal, it is probably not necessary (and
+safest) for the user to only clean up non-MPI state.
+
+Process Environment
+^^^^^^^^^^^^^^^^^^^
+
+.. admonition:: This is old, hard-coded content
+   :class: error
+
+   Is this content still current / accurate?  Should it be updated and
+   retained, or removed?
+
+Processes in the MPI application inherit their environment from the
+PRRTE daemon upon the node on which they are running.  The
+environment is typically inherited from the user's shell.  On remote
+nodes, the exact environment is determined by the boot MCA module
+used.  The rsh launch module, for example, uses either rsh/ssh to
+launch the PRRTE daemon on remote nodes, and typically executes one
+or more of the user's shell-setup files before launching the PRRTE
+daemon.  When running dynamically linked applications which require
+the ``LD_LIBRARY_PATH`` environment variable to be set, care must be
+taken to ensure that it is correctly set when booting Open MPI.
+
+See the :ref:`Remote Execution <man1-mpirun-remote-execution>` section
+for more details.
+
+.. _man1-mpirun-remote-execution:
+
+Remote Execution
+^^^^^^^^^^^^^^^^
+
+.. admonition:: This is old, hard-coded content
+   :class: error
+
+   Is this content still current / accurate?  Should it be updated and
+   retained, or removed?
+
+Open MPI requires that the ``PATH`` environment variable be set to
+find executables on remote nodes (this is typically only necessary in
+rsh- or ssh-based environments |mdash| batch/scheduled environments
+typically copy the current environment to the execution of remote
+jobs, so if the current environment has ``PATH`` and/or
+``LD_LIBRARY_PATH`` set properly, the remote nodes will also have it
+set properly).  If Open MPI was compiled with shared library support,
+it may also be necessary to have the ``LD_LIBRARY_PATH`` environment
+variable set on remote nodes as well (especially to find the shared
+libraries required to run user MPI applications).
+
+However, it is not always desirable or possible to edit shell startup
+files to set ``PATH`` and/or ``LD_LIBRARY_PATH``.  The ``--prefix``
+option is provided for some simple configurations where this is not
+possible.
+
+The ``--prefix`` option takes a single argument: the base directory on
+the remote node where Open MPI is installed.  Open MPI will use this
+directory to set the remote ``PATH`` and ``LD_LIBRARY_PATH`` before
+executing any Open MPI or user applications.  This allows running Open
+MPI jobs without having pre-configured the ``PATH`` and
+``LD_LIBRARY_PATH`` on the remote nodes.
+
+Open MPI adds the basename of the current node's ``$bindir`` (the
+directory where Open MPI's executables were installed) to the prefix
+and uses that to set the ``PATH`` on the remote node.  Similarly, Open
+MPI adds the basename of the current node's ``$libdir`` (the directory
+where Open MPI's libraries were installed) to the prefix and uses that
+to set the ``LD_LIBRARY_PATH`` on the remote node.  For example:
+
+* Local bindir: ``/local/node/directory/bin``
+* Local libdir: ``/local/node/directory/lib64``
+
+If the following command line is used:
+
+.. code:: sh
+
+   shell$ mpirun --prefix /remote/node/directory
+
+Open MPI will add ``/remote/node/directory/bin`` to the ``PATH`` and
+``/remote/node/directory/lib64`` to the ``LD_LIBRARY_PATH`` on the
+remote node before attempting to execute anything.
+
+The ``--prefix`` option is not sufficient if the installation paths on
+the remote node are different than the local node (e.g., if ``/lib``
+is used on the local node, but ``/lib64`` is used on the remote node),
+or if the installation paths are something other than a subdirectory
+under a common prefix.
+
+Note that executing ``mpirun`` via an absolute pathname is equivalent
+to specifying ``--prefix`` without the last subdirectory in the
+absolute pathname to ``mpirun``.  For example:
+
+.. code:: sh
+
+   shell$ /usr/local/bin/mpirun ...
+
+is equivalent to
+
+.. code:: sh
+
+   shell$ mpirun --prefix /usr/local
+
+Exported Environment Variables
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. admonition:: This is old, hard-coded content
+   :class: error
+
+   Is this content still current / accurate?  Should it be updated and
+   retained, or removed?
+
+All environment variables that are named in the form ``OMPI_*`` will
+automatically be exported to new processes on the local and remote
+nodes.  Environmental parameters can also be set/forwarded to the new
+processes using the MCA parameter ``mca_base_env_list``. The ``-x``
+option to mpirun has been deprecated, but the syntax of the MCA param
+follows that prior example. While the syntax of the ``-x`` option and
+MCA param allows the definition of new variables, note that the parser
+for these options are currently not very sophisticated |mdash| it does
+not even understand quoted values.  Users are advised to set variables
+in the environment and use the option to export them; not to define
+them.
+
+.. _man1-mpirun-setting-mca-parameters:
+
+Setting MCA Parameters
+^^^^^^^^^^^^^^^^^^^^^^
+
+.. admonition:: This is old, hard-coded content
+   :class: error
+
+   Is this content still current / accurate?  Should it be updated and
+   retained, or removed?
+
+The ``--mca`` switch allows the passing of parameters to various MCA
+(Modular Component Architecture) modules.  MCA modules have direct
+impact on MPI programs because they allow tunable parameters to be set
+at run time (such as which BTL communication device driver to use,
+what parameters to pass to that BTL, etc.).
+
+The ``--mca`` switch takes two arguments: ``<key>`` and ``<value>``.
+The ``<key>`` argument generally specifies which MCA module will
+receive the value.  For example, the ``<key>`` ``btl`` is used to
+select which BTL to be used for transporting MPI messages.  The
+``<value>`` argument is the value that is passed.  For example:
+
+.. code:: sh
+
+   shell$ mpirun --mca btl tcp,self -n 1 my_mpi_app
+
+This tells Open MPI to use the ``tcp`` and ``self`` BTLs, and to run a
+single copy of ``my_mpi_app`` an allocated node.
+
+.. code:: sh
+
+   shell$ mpirun --mca btl self -n 1 my_mpi_app
+
+Tells Open MPI to use the ``self`` BTL, and to run a single copy of
+``my_mpi_app`` an allocated node.
+
+The ``--mca`` switch can be used multiple times to specify different
+<key> and/or ``<value>`` arguments.  If the same ``<key>`` is
+specified more than once, the ``<value>``s are concatenated with a
+comma (``,``) separating them.
+
+Note that the ``--mca`` switch is simply a shortcut for setting
+environment variables.  The same effect may be accomplished by setting
+corresponding environment variables before running ``mpirun``.  The form
+of the environment variables that Open MPI sets is:
+
+.. code:: sh
+
+   OMPI_MCA_<key>=<value>
+
+Thus, the ``--mca`` switch overrides any previously set environment
+variables.  The ``--mca`` settings similarly override MCA parameters
+set in the ``$OPAL_PREFIX/etc/openmpi-mca-params.conf`` or
+``$HOME/.openmpi/mca-params.conf`` file.
+
+Unknown ``<key>`` arguments are still set as environment variable |mdash|
+they are not checked (by mpirun) for correctness.  Illegal or
+incorrect ``<value>`` arguments may or may not be reported |mdash| it
+depends on the specific MCA module.
+
+To find the available component types under the MCA architecture, or
+to find the available parameters for a specific component, use the
+ompi_info command.  See the :ref:`ompi_info(1) <man1-ompi_info>` man
+page for detailed information on this command.
+
+.. _man1-mpirun-setting-mca-params-from-file:
+
+Setting MCA parameters and environment variables from file
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. admonition:: This is old, hard-coded content
+   :class: error
+
+   Is this content still current / accurate?  Should it be updated and
+   retained, or removed?
+
+The ``--tune`` command line option and its synonym ``--mca``
+``mca_base_envar_file_prefix`` allows a user to set MCA parameters and
+environment variables with the syntax described below.  This option
+requires a single file or list of files separated by "," to follow.
+
+A valid line in the file may contain zero or more ``-x`` or
+``--mca``. The following patterns are supported:
+
+* ``--mca var val``
+* ``--mca var "val"``
+* ``-x var=val``
+* ``-x var``
+
+If any argument is duplicated in the file, the last value read will be
+used.
+
+MCA parameters and environment specified on the command line
+have higher precedence than variables specified in the file.
+
+.. _man1-mpirun-running-as-root:
+
+Running as root
+^^^^^^^^^^^^^^^
+
+.. admonition:: This is old, hard-coded content
+   :class: error
+
+   Is this content still current / accurate?  Should it be updated and
+   retained, or removed?
+
+.. warning:: The Open MPI team **strongly** advises against executing
+             ``mpirun`` as the root user.  MPI applications should be
+             run as regular (non-root) users.
+
+``mpirun`` will refuse to run as root by default.
+
+To override this default, you can add the ``--allow-run-as-root``
+option to the mpirun command line, or you can set the environmental
+parameters ``OMPI_ALLOW_RUN_AS_ROOT=1`` and
+``OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1``.  Note that it takes setting two
+environment variables to effect the same behavior as
+``--allow-run-as-root`` in order to stress the Open MPI team's strong
+advice against running as the root user.
+
+After extended discussions with communities who use containers (where
+running as the root user is the default), there was a persistent
+desire to be able to enable root execution of ``mpirun`` via an
+environmental control (vs. the existing ``--allow-run-as-root``
+command line parameter).  The compromise of using two environment
+variables was reached: it allows root execution via an environmental
+control, but it conveys the Open MPI team's strong recommendation
+against this behavior.
+
+Exit status
+^^^^^^^^^^^
+
+.. admonition:: This is old, hard-coded content
+   :class: error
+
+   Is this content still current / accurate?  Should it be updated and
+   retained, or removed?
+
+There is no standard definition for what ``mpirun`` should return as
+an exit status. After considerable discussion, we settled on the
+following method for assigning the ``mpirun`` exit status (note: in
+the following description, the "primary" job is the initial
+application started by mpirun |mdash| all jobs that are spawned by
+that job are designated "secondary" jobs):
+
+* If all processes in the primary job normally terminate with exit
+  status 0, ``mpirun`` returns 0.
+
+* If one or more processes in the primary job normally terminate with
+  non-zero exit status, ``mpirun`` returns the exit status of the
+  process with the lowest ``MPI_COMM_WORLD`` rank to have a non-zero
+  status.
+
+* If all processes in the primary job normally terminate with exit
+  status 0, and one or more processes in a secondary job normally
+  terminate with non-zero exit status, ``mpirun``:
+
+  #. Returns the exit status of the process with the lowest
+     ``MPI_COMM_WORLD`` rank in the lowest jobid to have a non-zero
+     status, and
+  #. Outputs a message summarizing the exit status of the primary and
+     all secondary jobs.
+
+* If the command line option ``--report-child-jobs-separately`` is
+  set, we will return *only* the exit status of the primary job. Any
+  non-zero exit status in secondary jobs will be reported solely in a
+  summary print statement.
+
+By default, the job will abort when any process terminates with
+non-zero status. The MCA parameter ``--prtemca state_base_error_non_zero_exit``
+can be set to "false" (or "0") to cause Open MPI to not abort a job if
+one or more processes return a non-zero status. In that situation the
+Open MPI records and notes that processes exited with non-zero
+termination status to report the appropriate exit status of ``mpirun`` (per
+bullet points above).
+
+EXAMPLES
+--------
+
+.. admonition:: This is old, hard-coded content
+   :class: error
+
+   Is this content still current / accurate?  Should it be updated and
+   retained, or removed?
+
+Be sure also to see the examples throughout the sections above.
+
+.. code:: sh
+
+   shell$ mpirun -n 4 --mca btl tcp,sm,self prog1
+
+Run 4 copies of ``prog1`` using the ``tcp``, ``sm`` (shared memory),
+and ``self`` (process loopback) BTL's for the transport of MPI
+messages.
+
+
+RETURN VALUE
+------------
+
+.. admonition:: This is old, hard-coded content
+   :class: error
+
+   Is this content still current / accurate?  Should it be updated and
+   retained, or removed?
+
+``mpirun`` returns 0 if all processes started by mpirun exit after
+calling :ref:`MPI_FINALIZE(3) <mpi_finalize>`.  A non-zero value is
+returned if an internal error occurred in mpirun, or one or more
+processes exited before calling :ref:`MPI_FINALIZE(3) <mpi_finalize>`.
+If an internal error occurred in mpirun, the corresponding error code
+is returned.  In the event that one or more processes exit before
+calling :ref:`MPI_FINALIZE(3) <mpi_finalize>`, the return value of
+the ``MPI_COMM_WORLD`` rank of the process that mpirun first notices died
+before calling :ref:`MPI_FINALIZE(3) <mpi_finalize>` will be
+returned.  Note that, in general, this will be the first process that
+died but is not guaranteed to be so.
+
+If the ``--timeout`` command line option is used and the timeout
+expires before the job completes (thereby forcing mpirun to kill the
+job) mpirun will return an exit status equivalent to the value of
+ETIMEDOUT (which is typically 110 on Linux and OS X systems).
+
+
+.. seealso::
+   :ref:`MPI_INIT(3) <mpi_init>`,
+   :ref:`MPI_INIT_THREAD(3) <mpi_init_thread>`,
+   :ref:`MPI_FINALIZE(3) <mpi_finalize>`,
+   :ref:`ompi_info(1) <man1-ompi_info>`
diff --git a/docs/man-openmpi/man1/mpisync.1.rst b/docs/man-openmpi/man1/mpisync.1.rst
new file mode 100644
index 00000000000..80195aaaf49
--- /dev/null
+++ b/docs/man-openmpi/man1/mpisync.1.rst
@@ -0,0 +1,57 @@
+.. _man1-mpisync:
+
+
+mpisync
+=======
+
+.. include_body
+
+mpisync |mdash| Open MPI timing tools
+
+
+SYNTAX
+------
+
+``mpisync [options]``
+
+``mpirun_prof [options]``
+
+``ompi_timing_post [<timing-file>] [<processed-file>]``
+
+
+DESCRIPTION
+-----------
+
+``mpisync``: determines clock offsets relative to Head Node Process
+(HNP). It accepts the following options:
+
+* ``-o``, ``--output``: The name of output file where offsets related
+  to HNP will be written
+
+* ``-h``, ``--help``: Print help information
+
+``ompi_timing_post`` takes the timing output file as input parameter.
+The events are sorted by the timestamps. Next, the timestamps are
+replaced with time offsets relative to the ``first`` : ``previous``
+event.
+
+``mpirun_prof`` is a wrapper around :ref:`mpirun <man1-mpirun>` that
+performs clock synchronisation and post-processing of the timing
+output file.
+
+
+NOTES
+-----
+
+The mpisync code was derived from MPIPerf project:
+http://mpiperf.cpct.sibsutis.ru/index.php/Main/Documentation
+
+
+FILES
+-----
+
+The output file has following format:
+
+.. code::
+
+   <hostname> <round-trip-time> <offset-from-hnp>
diff --git a/docs/man-openmpi/man1/ompi-wrapper-compiler.1.rst b/docs/man-openmpi/man1/ompi-wrapper-compiler.1.rst
new file mode 100644
index 00000000000..249de903ad2
--- /dev/null
+++ b/docs/man-openmpi/man1/ompi-wrapper-compiler.1.rst
@@ -0,0 +1,258 @@
+.. _man1-mpicc:
+.. _man1-mpic++:
+.. _man1-mpicxx:
+.. _man1-mpifort:
+.. _man1-mpijavac:
+
+Open MPI Wrapper Compilers
+==========================
+
+.. include_body
+
+mpicc, mpic++, mpicxx, mpifort, mpijavac |mdash| Open MPI wrapper compilers
+
+SYNTAX
+------
+
+``mpicc [--showme | --showme:compile | --showme:link] ...``
+
+``mpic++ [--showme | --showme:compile | --showme:link] ...``
+
+``mpicxx [--showme | --showme:compile | --showme:link] ...``
+
+``mpifort [--showme | --showme:compile | --showme:link] ...``
+
+``mpijavac [--showme | --showme:compile | --showme:link] ...``
+
+The following deprecated commands are also available |mdash| but
+``mpifort`` should be used instead:
+
+``mpif77 [--showme | --showme:compile | --showme:link] ...``
+
+``mpif90 [--showme | --showme:compile | --showme:link] ...``
+
+On case-sensitive filesystems, the following command will also be
+available:
+
+``mpiCC [--showme | --showme:compile | --showme:link] ...``
+
+
+OPTIONS
+-------
+
+The options below apply to all of the wrapper compilers:
+
+* ``--showme``: This option comes in several different variants (see
+  below). None of the variants invokes the underlying compiler; they
+  all provide information on how the underlying compiler would have
+  been invoked had ``--showme`` not been used. The basic ``--showme``
+  option outputs the command line that would be executed to compile
+  the program.
+
+  .. note:: If a non-filename argument is passed on the command line,
+            the ``--showme`` option will *not* display any additional
+            flags. For example, both ``"mpicc --showme`` and
+            ``mpicc --showme my_source.c`` will show all the
+            wrapper-supplied flags. But ``mpicc
+            --showme -v`` will only show the underlying compiler name
+            and ``-v``.
+
+* ``--showme:compile``: Output the compiler flags that would have been
+  supplied to the underlying compiler.
+
+* ``--showme:link``: Output the linker flags that would have been
+  supplied to the underlying compiler.
+
+* ``--showme:command``: Outputs the underlying compiler
+  command (which may be one or more tokens).
+
+* ``--showme:incdirs``: Outputs a space-delimited (but otherwise
+  undecorated) list of directories that the wrapper compiler would
+  have provided to the underlying compiler to indicate
+  where relevant header files are located.
+
+* ``--showme:libdirs``: Outputs a space-delimited (but otherwise
+  undecorated) list of directories that the wrapper compiler would
+  have provided to the underlying linker to indicate where relevant
+  libraries are located.
+
+* ``--showme:libs`` Outputs a space-delimited (but otherwise
+  undecorated) list of library names that the wrapper compiler would
+  have used to link an application. For example: ``mpi open-pal
+  util``.
+
+* ``--showme:version``: Outputs the version number of Open MPI.
+
+* ``--showme:help``: Output a brief usage help message.
+
+See the man page for your underlying compiler for other options that
+can be passed through mpicc.
+
+
+DESCRIPTION
+-----------
+
+Conceptually, the role of these commands is quite simple:
+transparently add relevant compiler and linker flags to the user's
+command line that are necessary to compile / link Open MPI programs,
+and then invoke the underlying compiler to actually perform the
+command.
+
+As such, these commands are frequently referred to as "wrapper"
+compilers because they do not actually compile or link applications
+themselves; they only add in command line flags and invoke the
+back-end compiler.
+
+Background
+----------
+
+Open MPI provides wrapper compilers for several languages:
+
+* ``mpicc``: C
+
+* ``mpic++``, ``mpicxx`` (and on systems with case-sensitive file
+  systems, ``mpiCC``): C++
+
+  .. note:: ``mpic++``, ``mpicxx``, and ``mpiCC`` all invoke the same
+            underlying C++ compiler with the same options. All are
+            provided as compatibility with other MPI implementations.
+
+* ``mpifort`` (and its legacy/deprecated aliaes ``mpif77`` and
+  ``mpif90``): Fortran
+
+* ``mpijavac``: Java
+
+The wrapper compilers for each of the languages are identical; they
+can be use interchangeably.  The different names are provided solely
+for backwards compatibility.
+
+
+Fortran Notes
+-------------
+
+The Fortran wrapper compiler for MPI (``mpifort``, and its
+legacy/deprecated names ``mpif77`` and ``mpif90``) can compile and
+link MPI applications that use any/all of the MPI Fortran bindings:
+``mpif.h``, the ``mpi`` module, and the ``mpi_f08`` module (assuming
+Open MPI was installed with support for each of these Fortran
+bindings). Specifically: it is no longer necessary to use different
+wrapper compilers for applications that use ``mpif.h``
+vs. applications that use the ``mpi`` module |mdash| just use ``mpifort``
+for all Fortran MPI applications.
+
+Note, however, that the Fortran compiler may require additional
+command-line options to enforce a specific Fortran dialect. For
+example, in some versions of the IBM XLF compiler, if ``xlf90`` is the
+underlying Fortran compiler, ``-qfixed`` may be necessary to compile
+fixed-format Fortran source files.
+
+Finally, note that ``mpifort`` will be inoperative and will return an
+error on use if Fortran support was not built into the MPI layer.
+
+Overview
+--------
+
+``mpicc`` is a convenience wrappers for the underlying C compiler.
+Translation of an Open MPI program requires the linkage of the Open
+MPI-specific libraries which may not reside in one of the standard
+search directories of ``ld(1)``. It also often requires the inclusion
+of header files what may also not be found in a standard location.
+
+``mpicc`` passes its arguments to the underlying C compiler along with
+the ``-I``, ``-L`` and ``-l`` options required by Open MPI programs.
+
+The same is true for all the other language wrapper compilers.
+
+The Open MPI Team *strongly* encourages using the wrapper compilers
+instead of attempting to link to the Open MPI libraries manually. This
+allows the specific implementation of Open MPI to change without
+forcing changes to linker directives in users' Makefiles. Indeed, the
+specific set of flags and libraries used by the wrapper compilers
+depends on how Open MPI was configured and built; the values can change
+between different installations of the same version of Open MPI.
+
+Indeed, since the wrappers are simply thin shells on top of an
+underlying compiler, there are very, very few compelling reasons *not*
+to use Open MPI's wrapper compilers. When it is not possible to use
+the wrappers directly, the ``--showme:compile`` and ``--showme:link``
+options should be used to determine what flags the wrappers would have
+used. For example:
+
+.. code:: sh
+
+   shell$ cc -c file1.c `mpicc --showme:compile`
+
+   shell$ cc -c file2.c `mpicc --showme:compile`
+
+   shell$ cc file1.o file2.o `mpicc --showme:link` -o my_mpi_program
+
+
+NOTES
+-----
+
+It is possible to make the wrapper compilers multi-lib aware. That is,
+the libraries and includes specified may differ based on the compiler
+flags specified (for example, with the GNU compilers on Linux, a
+different library path may be used if ``-m32`` is seen versus ``-m64``
+being seen). This is not the default behavior in a standard build, but
+can be activated (for example, in a binary package providing both 32
+and 64 bit support). `More information can be found here
+<https://github.com/open-mpi/ompi/wiki/compilerwrapper3264>`_.
+
+
+.. _man1-ompi-wrapper-compiler-files:
+
+FILES
+-----
+
+The strings that the wrapper compilers insert into the command line
+before invoking the underlying compiler are stored in a text file
+created by Open MPI and installed to
+``$pkgdata/NAME-wrapper-data.txt``, where:
+
+* ``$pkgdata`` is typically ``$prefix/share/openmpi``
+* ``$prefix`` is the top installation directory of Open MPI
+* ``NAME`` is the name of the wrapper compiler (e.g.,
+  ``$pkgdata/mpicc-wrapper-data.txt``)
+
+It is rarely necessary to edit these files, but they can be examined to
+gain insight into what flags the wrappers are placing on the command
+line.
+
+
+ENVIRONMENT VARIABLES
+---------------------
+
+By default, the wrappers use the compilers that were selected when
+Open MPI was configured. These compilers were either found
+automatically by Open MPI's "configure" script, or were selected by
+the user in the ``CC``, ``CXX``, and/or ``FC`` environment variables
+before ``configure`` was invoked. Additionally, other arguments specific
+to the compiler may have been selected by configure.
+
+These values can be selectively overridden by either editing the text
+files containing this configuration information (see the :ref:`FILES
+<man1-ompi-wrapper-compiler-files>` section), or by setting selected
+environment variables of the form ``ompi_value``.
+
+Valid value names are:
+
+* ``CPPFLAGS``: Flags added when invoking the preprocessor (C or C++)
+
+* ``LDFLAGS``: Flags added when invoking the linker (C, C++, or
+  Fortran)
+
+* ``LIBS``: Libraries added when invoking the linker (C, C++, or
+  Fortran)
+
+* ``CC``: C compiler
+
+* ``CFLAGS``: C compiler flags
+
+* ``CXX``: C++ compiler
+
+* ``CXXFLAGS``: C++ compiler flags
+
+* ``FC``: Fortran compiler
+
+* ``FCFLAGS``: Fortran compiler flags
diff --git a/docs/man-openmpi/man1/ompi_info.1.rst b/docs/man-openmpi/man1/ompi_info.1.rst
new file mode 100644
index 00000000000..0699ae5b7d8
--- /dev/null
+++ b/docs/man-openmpi/man1/ompi_info.1.rst
@@ -0,0 +1,227 @@
+.. _man1-ompi_info:
+
+
+ompi_info
+=========
+
+.. include_body
+
+ompi_info |mdash| Display information about the Open MPI installation
+
+
+SYNOPSIS
+--------
+
+``ompi_info [options]``
+
+
+DESCRIPTION
+-----------
+
+``ompi_info`` provides detailed information about the Open MPI
+installation. It can be useful for at least three common scenarios:
+
+#. Checking local configuration and seeing how Open MPI was installed.
+
+#. Submitting bug reports / help requests to the Open MPI community
+   (see :doc:`Getting help </getting-help>`).
+
+#. Seeing a list of installed Open MPI plugins and querying what MCA
+   parameters they support.
+
+.. note:: ``ompi_info`` defaults to only showing a few MCA parameters
+          by default (i.e., level 1 parameters). Use the ``--level``
+          option to enable showing more options (see the :ref:`LEVELS
+          <man1-ompi_info-levels>` section for more information).
+
+
+OPTIONS
+-------
+
+``ompi_info`` accepts the following options:
+
+* ``-a``, ``--all``: Show all configuration options and MCA
+  parameters. Also changes the default MCA parameter level to 9,
+  unless ``--level`` is also specified.
+
+* ``--arch``: Show architecture on which Open MPI was compiled.
+
+* ``-c``, ``--config``: Show configuration options
+
+* ``-gmca``, ``--gmca <param> <value>``: Pass global MCA parameters
+  that are applicable to all contexts.
+
+* ``-h``, ``--help``: Shows help / usage message.
+
+* ``--hostname``: Show the hostname on which Open MPI was configured
+  and built.
+
+* ``--internal``: Show internal MCA parameters (not meant to be
+  modified by users).
+
+* ``--level <level>``: Show only variables with at most this level
+  (1-9). The default is 1 unless ``--all`` is specified without
+  ``--level``, in which case the default is 9. See the :ref:`LEVELS
+  <man1-ompi_info-levels>` section for more information.
+
+* ``-mca``, ``--mca <param> <value>``: Pass context-specific MCA
+  parameters; they are considered global if ``--gmca`` is not used and
+  only one context is specified.
+
+* ``--param <type> <component>``: Show MCA parameters. The first
+  parameter is the type of the component to display; the second
+  parameter is the specific component to display (or the keyword
+  ``all``, meaning "display all components of this type").
+
+* ``-t``, ``--type``: Show MCA parameters of the type specified in the
+  parameter. Accepts the following parameters: ``unsigned_int``,
+  ``unsigned_long``, ``unsigned_long_long``, ``size_t``, ``string``,
+  ``version_string``, ``bool``, ``double``. By default level is 1
+  unless it is specified with ``--level``.
+
+* ``--parsable``: When used in conjunction with other parameters, the
+  output is displayed in a machine-parsable format ``--parseable``
+  Synonym for ``--parsable``.
+
+* ``--path <type>``: Show paths that Open MPI was configured
+  with. Accepts the following parameters: ``prefix``, ``bindir``,
+  ``libdir``, ``incdir``, ``pkglibdir``, ``sysconfdir``.
+
+* ``--pretty``: When used in conjunction with other parameters, the output is
+  displayed in "prettyprint" format (default)
+
+* ``--selected-only``: Show only variables from selected components.
+
+* ``-V``, ``--version``: Show version of Open MPI.
+
+.. _man1-ompi_info-levels:
+
+LEVELS
+------
+
+Open MPI has many, many run-time tunable parameters (called "MCA
+parameters"), and usually only a handful of them are useful to a given
+user.
+
+As such, Open MPI has divided these parameters up into nine distinct
+levels, broken down into three categories, each with three
+sub-categories.
+
+Note that since each MCA parameter is accessible through the MPI_T
+control variable API (introduced in MPI-3.0), these levels exactly
+correspond to the nine MPI_T cvar levels.
+
+The three categories are:
+
+#. **End user**: Generally, these are parameters that are required for
+   correctness, meaning that a user may need to set these just to get
+   their MPI application to run correctly. For example, BTL
+   ``if_include`` and ``if_exclude`` parameters fit into this
+   category.
+
+#. **Application tuner**: Generally, these are parameters that can be
+   used to tweak MPI application performance. This even includes
+   parameters that control resource exhaustion levels (e.g., number of
+   free list entries, size of buffers, etc.), and could be considered
+   "correctness" parameters if they're set too low. But, really |mdash|
+   they're tuning parameters.
+
+#. **Open MPI developer**: Parameters in this category either don't
+   fit in the other two, or are specifically intended for debugging /
+   development of Open MPI itself.
+
+And within each category, there are three sub-categories:
+
+#. **Basic**: This sub-category is for parameters that everyone in
+   this category will want to see |mdash| even less-advanced end users,
+   application tuners, and new OMPI developers.
+
+#. **Detailed**: This sub-category is for parameters that are
+   generally useful, but users probably won't need to change them
+   often.
+
+#. **All**: This sub-category is for all other parameters. Such
+   parameters are likely fairly esoteric.
+
+Combining the categories and sub-categories, here's how Open MPI
+defines all nine levels:
+
+#. Basic information of interest to end users.
+#. Detailed information of interest to end users.
+#. All remaining information of interest to end users.
+#. Basic information required for application tuners.
+#. Detailed information required for application tuners.
+#. All remaining information required for application tuners.
+#. Basic information for Open MPI implementors.
+#. Detailed information for Open MPI implementors.
+#. All remaining information for Open MPI implementors.
+
+By default, ``ompi_info`` only shows level 1 MCA parameters. To see
+more MCA parameters, use the ``--level`` command line option.
+
+
+EXAMPLES
+--------
+
+Show the default output of options and listing of installed
+components in a human-readable / prettyprint format:
+
+.. code-block::
+
+   ompi_info
+
+Show the default output of options and listing of installed components
+in a machine-parsable format:
+
+.. code-block::
+
+   ompi_info --parsable
+
+Show the level 1 MCA parameters of the "tcp" BTL component in a
+human-readable / prettyprint format:
+
+.. code-block::
+
+   ompi_info --param btl tcp
+
+Show the level 1 through level 6 MCA parameters of the "tcp" BTL
+component in a human-readable / prettyprint format:
+
+.. code-block::
+
+   ompi_info --param btl tcp --level 6
+
+Show the level 1 MCA parameters of the "tcp" BTL component in a
+machine-parsable format:
+
+.. code-block::
+
+   ompi_info --param btl tcp --parsable
+
+Show the level 1 through level 3 MCA parameters of string type in a
+human-readable / prettyprint format:
+
+.. code-block::
+
+   ompi_info --type string --pretty-print --level 3
+
+Show the "bindir" that Open MPI was configured with:
+
+.. code-block::
+
+   ompi_info --path bindir
+
+Show the version of Open MPI version numbers in a prettyprint format:
+
+.. code-block::
+
+   ompi_info --version
+
+Show *all* information about the Open MPI installation, including all
+components that can be found, all the MCA parameters that they support
+(i.e., levels 1 through 9), versions of Open MPI and the components,
+etc.:
+
+.. code-block::
+
+   ompi_info --all
diff --git a/docs/man-openmpi/man1/opal_wrapper.1.rst b/docs/man-openmpi/man1/opal_wrapper.1.rst
new file mode 100644
index 00000000000..cd4d4513854
--- /dev/null
+++ b/docs/man-openmpi/man1/opal_wrapper.1.rst
@@ -0,0 +1,33 @@
+.. _man1-opal_wrapper:
+
+
+opal_wrapper
+============
+
+.. include_body
+
+opal_wrapper |mdash| Back-end Open MPI wrapper command
+
+
+DESCRIPTION
+-----------
+
+``opal_wrapper`` is not meant to be called directly by end users. It
+is automatically invoked as the back-end by the Open MPI wrapper
+commands such as: ``mpicc``, ``mpicxx``, and ``mpifort``
+(and its legacy/deprecated aliases ``mpif77`` and ``mpif90``).
+
+Some Open MPI installations may have additional wrapper commands,
+and/or have renamed the wrapper compilers listed above to avoid
+executable name conflicts with other MPI implementations. Hence, you
+may also have wrapper compilers installed including the following
+names: ``mpifort.openmpi`` (and the legacy/deprecated aliases
+``mpif90.openmpi`` and ``mpif77.openmpi``), ``mpicxx.openmpi``,
+``mpicc.openmpi``.
+
+.. seealso::
+   The following may exist depending on your particular Open MPI
+   installation:
+   :ref:`mpicc(1) <man1-mpicc>`,
+   :ref:`mpicxx(1) <man1-mpicxx>`,
+   :ref:`mpifort(1) <man1-mpifort>`
diff --git a/docs/man-openmpi/man3/ERRORS.rst b/docs/man-openmpi/man3/ERRORS.rst
new file mode 100644
index 00000000000..2b7e3c75ff9
--- /dev/null
+++ b/docs/man-openmpi/man3/ERRORS.rst
@@ -0,0 +1,47 @@
+Almost all MPI routines return an error value; C routines as the return result
+of the function and Fortran routines in the last argument.
+
+Before the error value is returned, the current MPI error handler associated
+with the communication object (e.g., communicator, window, file) is called.
+If no communication object is associated with the MPI call, then the call is
+considered attached to MPI_COMM_SELF and will call the associated MPI error
+handler. When MPI_COMM_SELF is not initialized (i.e., before
+:ref:`MPI_Init`/:ref:`MPI_Init_thread`, after :ref:`MPI_Finalize`, or when using the Sessions
+Model exclusively) the error raises the initial error handler. The initial
+error handler can be changed by calling :ref:`MPI_Comm_set_errhandler` on
+MPI_COMM_SELF when using the World model, or the mpi_initial_errhandler CLI
+argument to mpiexec or info key to :ref:`MPI_Comm_spawn`/:ref:`MPI_Comm_spawn_multiple`.
+If no other appropriate error handler has been set, then the MPI_ERRORS_RETURN
+error handler is called for MPI I/O functions and the MPI_ERRORS_ABORT error
+handler is called for all other MPI functions.
+
+Open MPI includes three predefined error handlers that can be used:
+
+* ``MPI_ERRORS_ARE_FATAL``
+  Causes the program to abort all connected MPI processes.
+* ``MPI_ERRORS_ABORT``
+  An error handler that can be invoked on a communicator,
+  window, file, or session. When called on a communicator, it
+  acts as if :ref:`MPI_Abort` was called on that communicator. If
+  called on a window or file, acts as if :ref:`MPI_Abort` was called
+  on a communicator containing the group of processes in the
+  corresponding window or file. If called on a session,
+  aborts only the local process.
+* ``MPI_ERRORS_RETURN``
+  Returns an error code to the application.
+
+MPI applications can also implement their own error handlers by calling:
+
+* :ref:`MPI_Comm_create_errhandler` then :ref:`MPI_Comm_set_errhandler`
+* :ref:`MPI_File_create_errhandler` then :ref:`MPI_File_set_errhandler`
+* :ref:`MPI_Session_create_errhandler` then :ref:`MPI_Session_set_errhandler` or at :ref:`MPI_Session_init`
+* :ref:`MPI_Win_create_errhandler` then :ref:`MPI_Win_set_errhandler`
+
+Note that MPI does not guarantee that an MPI program can continue past
+an error.
+
+See the :ref:`MPI man page <open-mpi>` for a full list of :ref:`MPI error codes <open-mpi-errors>`.
+
+See the Error Handling section of the MPI-|mpi_standard_version| standard for
+more information.
+
diff --git a/docs/man-openmpi/man3/MPIX_Comm_ack_failed.3.rst b/docs/man-openmpi/man3/MPIX_Comm_ack_failed.3.rst
new file mode 100644
index 00000000000..0302b889a5b
--- /dev/null
+++ b/docs/man-openmpi/man3/MPIX_Comm_ack_failed.3.rst
@@ -0,0 +1,128 @@
+.. _mpix_comm_ack_failed:
+
+MPIX_Comm_ack_failed
+====================
+.. include_body
+
+:ref:`MPIX_Comm_get_failed` - acknowledge failed processes in a communicator.
+
+This is part of the User Level Fault Mitigation :ref:`ULFM extension <ulfm-label>`.
+
+SYNTAX
+------
+
+C Syntax
+^^^^^^^^
+
+.. code-block:: c
+
+   #include <mpi.h>
+   #include <mpi-ext.h>
+
+   int MPIX_Comm_ack_failed(MPI_Comm comm, int num_to_ack, int *num_acked)
+
+Fortran Syntax
+^^^^^^^^^^^^^^
+
+.. code-block:: fortran
+
+   USE MPI
+   USE MPI_EXT
+   ! or the older form: INCLUDE 'mpif.h'
+
+   MPIX_COMM_ACK_FAILED(COMM, NUM_TO_ACK, NUM_ACKED, IERROR)
+        INTEGER COMM, NUM_TO_ACK, NUM_ACKED, IERROR
+
+Fortran 2008 Syntax
+^^^^^^^^^^^^^^^^^^^
+
+.. code-block:: fortran
+
+   USE mpi_f08
+   USE mpi_ext_f08
+
+   MPIX_Comm_ack_failed(comm, num_to_ack, num_acked, ierror)
+        TYPE(MPI_Comm), INTENT(IN) :: comm
+        INTEGER, INTENT(IN) :: num_to_ack
+        INTEGER, INTENT(OUT) :: num_acked
+        INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+
+INPUT PARAMETERS
+----------------
+* ``comm``: Communicator (handle).
+* ``num_to_ack``: maximum number of process failures to acknowledge in *comm* (integer)
+
+OUTPUT PARAMETERS
+-----------------
+* ``num_acked``: number of acknowledged failures in *comm* (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+his local operation gives the users a way to **acknowledge**
+locally notified failures on *comm*. The operation acknowledges the first
+*num_to_ack* process failures on *comm*, that is, it acknowledges the
+failure of members with a rank lower than *num_to_ack* in the group that
+would be produced by a concurrent call to :ref:`MPIX_Comm_get_failed` on
+the same *comm*.
+
+The operation also sets the value of *num_acked* to the current number of
+acknowledged process failures in *comm*, that is, a process failure has been
+acknowledged on *comm* if and only if the rank of the process is lower than
+*num_acked* in the group that would be produced by a subsequent call to
+:ref:`MPIX_Comm_get_failed` on the same *comm*.
+
+*num_acked* can be larger than *num_to_ack* when process failures have been
+acknowledged in a prior call to :ref:`MPIX_Comm_ack_failed`.
+
+EFFECT OF ACKNOWLEDGING FAILURES
+--------------------------------
+
+After an MPI process failure is acknowledged on *comm*, unmatched
+MPI_ANY_SOURCE receive operations on the same *comm* that would have raised
+an error of class MPIX_ERR_PROC_FAILED_PENDING proceed without further raising
+errors due to this acknowledged failure.
+
+Also, :ref:`MPIX_Comm_agree` on the same *comm* will not raise an error of
+class MPI_ERR_PROC_FAILED due to this acknowledged failure.
+
+USAGE PATTERNS
+--------------
+
+One may query, without side effect, for the number of currently aknowledged
+process failures *comm* by supplying 0 in *num_to_ack*.
+
+Conversely, one may unconditionally acknowledge all currently known process
+failures in *comm* by supplying the size of the group of *comm* in *num_to_ack*.
+
+Note that the number of acknowledged processes, as returned in *num_acked*,
+can be smaller or larger than the value supplied in *num_to_ack*; It is
+however never larger than the size of the group returned by a subsequent call
+to :ref:`MPIX_Comm_get_failed`.
+
+EFFECT ON COLLECTIVE OPERATIONS
+-------------------------------
+
+Calling :ref:`MPIX_Comm_ack_failed` on a communicator with failed MPI
+processes has no effect on collective operations (except for :ref:`MPIX_Comm_agree`).
+If a collective operation would raise an error due to the communicator
+containing a failed process it will continue to raise an error even after
+the failure has been acknowledged. In order to use collective operations
+between MPI processes of a communicator that contains failed MPI processes,
+users should create a new communicator (e.g., by calling :ref:`MPIX_Comm_shrink`).
+
+WHEN COMMUNICATOR IS AN INTER-COMMUNICATOR
+------------------------------------------
+
+When the communicator is an inter-communicator, the failures of members
+in both the local and the remote groups of *comm* are acknowledged.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPIX_Comm_get_failed`
+   * :ref:`MPIX_Comm_agree`
diff --git a/docs/man-openmpi/man3/MPIX_Comm_agree.3.rst b/docs/man-openmpi/man3/MPIX_Comm_agree.3.rst
new file mode 100644
index 00000000000..3e68135c456
--- /dev/null
+++ b/docs/man-openmpi/man3/MPIX_Comm_agree.3.rst
@@ -0,0 +1,175 @@
+.. _mpix_comm_agree:
+
+MPIX_Comm_agree
+===============
+.. include_body
+
+:ref:`MPIX_Comm_agree`, :ref:`MPIX_Comm_iagree` - Agree on a flag value
+from all live processes and distributes the result back to all live
+processes, even after process failures.
+
+This is part of the User Level Fault Mitigation :ref:`ULFM extension <ulfm-label>`.
+
+SYNTAX
+------
+
+C Syntax
+^^^^^^^^
+
+.. code-block:: c
+
+   #include <mpi.h>
+   #include <mpi-ext.h>
+
+   int MPIX_Comm_agree(MPI_Comm comm, int *flag)
+   
+   int MPIX_Comm_iagree(MPI_Comm comm, int *flag, MPI_Request *request)
+
+
+Fortran Syntax
+^^^^^^^^^^^^^^
+
+.. code-block:: fortran
+
+   USE MPI
+   USE MPI_EXT
+   ! or the older form: INCLUDE 'mpif.h'
+
+   MPIX_COMM_AGREE(COMM, FLAG, IERROR)
+        INTEGER COMM, FLAG, IERROR
+
+   MPIX_COMM_IAGREE(COMM, FLAG, REQUEST, IERROR)
+        INTEGER COMM, FLAG, REQUEST, IERROR
+
+
+Fortran 2008 Syntax
+^^^^^^^^^^^^^^^^^^^
+
+.. code-block:: fortran
+
+   USE mpi_f08
+   USE mpi_ext_f08
+
+   MPIX_Comm_agree(comm, flag, ierror)
+        TYPE(MPI_Comm), INTENT(IN) :: comm
+        INTEGER, INTENT(INOUT) :: flag
+        INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+
+   MPIX_COMM_IAGREE(COMM, FLAG, REQUEST, IERROR)
+        TYPE(MPI_Comm), INTENT(IN) :: comm
+        INTEGER, INTENT(INOUT), ASYNCHRONOUS :: flag
+        TYPE(MPI_Request), INTENT(OUT) :: request
+        INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+
+INPUT PARAMETERS
+----------------
+* ``comm``: Communicator (handle).
+* ``flag``: Binary flags (integer).
+
+OUTPUT PARAMETERS
+-----------------
+* ``flag``: Reduced binary flags (integer).
+* ``request``: Request (handle, non-blocking only).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This collective communication agrees on the integer value *flag* and
+(implicitly) on the group of failed processes in *comm*.
+
+On completion, all non-failed MPI processes have agreed to set the
+output integer value of *flag* to the result of a *bitwise AND*
+operation over the contributed input values of *flag*.
+
+:ref:`MPIX_Comm_iagree` is the non-blocking variant of :ref:`MPIX_Comm_agree`.
+
+PROCESS FAILURES
+----------------
+
+When an MPI process fails before contributing to the agree operation,
+the *flag* is computed ignoring its contribution, and the operation
+raises an error of class MPIX_ERR_PROC_FAILED.
+
+When an error of class MPIX_ERR_PROC_FAILED is raised, it is consistently
+raised at all MPI processes in the group(s) of *comm*.
+
+After :ref:`MPIX_Comm_agree` raised an error of class MPIX_ERR_PROC_FAILED,
+the group produced by a subsequent call to :ref:`MPIX_Comm_get_failed` on
+*comm* contains every MPI process that didn't contribute to the
+computation of *flag*.
+
+WHEN THE COMMUNICATOR CONTAINS ACKNOWLEDGED FAILURES
+----------------------------------------------------
+
+If **all** MPI processes in the group of *comm* have acknowledged the failure
+of an MPI process (using :ref:`MPIX_Comm_ack_failed`) prior to the call to
+:ref:`MPIX_Comm_agree` (or :ref:`MPIX_Comm_iagree`), the MPIX_ERR_PROC_FAILED
+error is not raised when the output value of *flag* ignores the
+contribution of that failed process. Note that this is an uniform property:
+if a non-contributing process is found to be not-acknowledged at any live
+process in *comm*, all processes raise an error of class MPIX_ERR_PROC_FAILED.
+
+**Example 1:** Using a combination of :ref:`MPIX_Comm_ack_failed` and
+:ref:`MPIX_Comm_agree` users can propagate and synchronize the knowledge
+of failures across all MPI processes in *comm*.
+
+.. code-block:: c
+
+    Comm_get_failed_consistent(MPI_Comm c, MPI_Group * g) {
+        int rc; int T=1;
+        int size; int num_acked;
+        MPI_Group gf;
+        int ranges[3] = {0, 0, 1};
+
+        MPI_Comm_size(c, &size);
+
+        do {
+            /* this routine is not pure: calling MPI_Comm_ack_failed
+             * affects the state of the communicator c */
+            MPIX_Comm_ack_failed(c, size, &num_acked);
+            /* we simply ignore the T value in this example */
+            rc = MPIX_Comm_agree(c, &T);
+        } while( rc != MPI_SUCCESS );
+        /* after this loop, MPIX_Comm_agree has returned MPI_SUCCESS at
+         * all processes, so all processes have Acknowledged the same set of
+         * failures. Let's get that set of failures in the g group. */
+        if( 0 == num_acked ) {
+            *g = MPI_GROUP_EMPTY;
+        }
+        else {
+            MPIX_Comm_get_failed(c, &gf);
+            ranges[1] = num_acked - 1;
+            MPI_Group_range_incl(gf, 1, ranges, g);
+            MPI_Group_free(&gf);
+        }
+    }
+
+WHEN THE COMMUNICATOR IS REVOKED
+--------------------------------
+
+This function never raises an error of class MPIX_ERR_REVOKED.
+The defined semantics of :ref:`MPIX_Comm_agree` are maintained when *comm*
+is revoked, or when the group of *comm* contains failed MPI processes.
+In particular, :ref:`MPIX_Comm_agree` is a collective operation, even
+when *comm* is revoked.
+
+WHEN COMMUNICATOR IS AN INTER-COMMUNICATOR
+------------------------------------------
+
+When the communicator is an inter-communicator, the value of *flag* is
+a *bitwise AND* operation over the values contributed by the remote
+group.
+
+When an error of class MPIX_ERR_PROC_FAILED is raised, it is consistently
+raised at all MPI processes in the group(s) of *comm*, that is, both
+the local and remote groups of the inter-communicator.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPIX_Comm_is_revoked`
+   * :ref:`MPIX_Comm_ack_failed`
diff --git a/docs/man-openmpi/man3/MPIX_Comm_get_failed.3.rst b/docs/man-openmpi/man3/MPIX_Comm_get_failed.3.rst
new file mode 100644
index 00000000000..ed72b1462e3
--- /dev/null
+++ b/docs/man-openmpi/man3/MPIX_Comm_get_failed.3.rst
@@ -0,0 +1,98 @@
+.. _mpix_comm_get_failed:
+
+MPIX_Comm_get_failed
+====================
+.. include_body
+
+:ref:`MPIX_Comm_get_failed` - Obtain a group that lists failed processes
+in a communicator.
+
+This is part of the User Level Fault Mitigation :ref:`ULFM extension <ulfm-label>`.
+
+SYNTAX
+------
+
+C Syntax
+^^^^^^^^
+
+.. code-block:: c
+
+   #include <mpi.h>
+   #include <mpi-ext.h>
+
+   int MPIX_Comm_get_failed(MPI_Comm comm, MPI_Group *failedgrp)
+
+Fortran Syntax
+^^^^^^^^^^^^^^
+
+.. code-block:: fortran
+
+   USE MPI
+   USE MPI_EXT
+   ! or the older form: INCLUDE 'mpif.h'
+
+   MPIX_COMM_GET_FAILED(COMM, FAILEDGRP, IERROR)
+        INTEGER COMM, FAILEDGRP, IERROR
+
+Fortran 2008 Syntax
+^^^^^^^^^^^^^^^^^^^
+
+.. code-block:: fortran
+
+   USE mpi_f08
+   USE mpi_ext_f08
+
+   MPIX_Comm_get_failed(comm, failedgrp, ierror)
+        TYPE(MPI_Comm), INTENT(IN) :: comm
+        TYPE(MPI_Group), INTENT(OUT) :: failedgrp
+        INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+
+INPUT PARAMETERS
+----------------
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``failedgrp``: Group (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This local operation returns the group *failedgrp* of processes from the
+communicator *comm* that are locally known to have failed.
+The *failedgrp* can be empty, that is, equal to MPI_GROUP_EMPTY.
+
+For any two groups obtained from calls to that routine at the same MPI
+process, with the same *comm*, the intersection of the largest group with
+the smallest group is MPI_IDENT to the smallest group, that is, the same
+processes have the same ranks in the two groups, up to the size of the
+smallest group.
+
+PROCESS FAILURES
+----------------
+
+MPI makes no assumption about asynchronous progress of the failure detection.
+A valid MPI implementation may choose to update the group of locally known
+failed MPI processes only when it enters a function that must raise a fault
+tolerance error.
+
+It is possible that only the calling MPI process has detected the reported
+failure. If global knowledge is necessary, MPI processes detecting failures
+should call :ref:`MPIX_Comm_revoke` to enforce an error at other ranks.
+
+WHEN COMMUNICATOR IS AN INTER-COMMUNICATOR
+------------------------------------------
+
+When the communicator is an inter-communicator, the value of *failedgrp*
+contains the members known to have failed in both the local and the remote
+groups of *comm*.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPIX_Comm_revoke`
+   * :ref:`MPIX_Comm_ack_failed`
diff --git a/docs/man-openmpi/man3/MPIX_Comm_iagree.3.rst b/docs/man-openmpi/man3/MPIX_Comm_iagree.3.rst
new file mode 100644
index 00000000000..ee4e4656bc2
--- /dev/null
+++ b/docs/man-openmpi/man3/MPIX_Comm_iagree.3.rst
@@ -0,0 +1,8 @@
+.. _mpix_comm_iagree:
+
+MPIX_Comm_iagree
+================
+.. include_body
+
+.. include:: ../man3/MPIX_Comm_agree.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPIX_Comm_is_revoked.3.rst b/docs/man-openmpi/man3/MPIX_Comm_is_revoked.3.rst
new file mode 100644
index 00000000000..787d0dfbd77
--- /dev/null
+++ b/docs/man-openmpi/man3/MPIX_Comm_is_revoked.3.rst
@@ -0,0 +1,115 @@
+.. _mpix_comm_is_revoked:
+
+MPIX_Comm_is_revoked
+====================
+.. include_body
+
+:ref:`MPIX_Comm_is_revoked` - Test if a communicator is revoked.
+
+This is part of the User Level Fault Mitigation :ref:`ULFM extension <ulfm-label>`.
+
+SYNTAX
+------
+
+C Syntax
+^^^^^^^^
+
+.. code-block:: c
+
+   #include <mpi.h>
+   #include <mpi-ext.h>
+
+   int MPIX_Comm_is_revoked(MPI_Comm comm, int *flag)
+
+Fortran Syntax
+^^^^^^^^^^^^^^
+
+.. code-block:: fortran
+
+   USE MPI
+   USE MPI_EXT
+   ! or the older form: INCLUDE 'mpif.h'
+
+   MPIX_COMM_IS_REVOKED(COMM, FLAG, IERROR)
+        INTEGER COMM, IERROR
+        LOGICAL FLAG
+
+Fortran 2008 Syntax
+^^^^^^^^^^^^^^^^^^^
+
+.. code-block:: fortran
+
+   USE mpi_f08
+   USE mpi_ext_f08
+
+   MPIX_Comm_is_revoked(comm, flag, ierror)
+        TYPE(MPI_Comm), INTENT(IN) :: comm
+        LOGICAL, INTENT(OUT) :: flag
+        INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+
+INPUT PARAMETERS
+----------------
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``flag``: *true* if the communicator is revoked.
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Returns *flag = true* if the communicator associated with the handle
+*comm* is revoked at the calling process. It returns *flag = false*
+otherwise. The operation is local.
+
+HOW CAN A COMMUNICATOR BECOME REVOKED
+-------------------------------------
+
+A communicator can become revoked when
+
+1. the user calls the :ref:`MPIX_Comm_revoke` procedure on *comm* to revoke
+   the communicator at the local process;
+2. an MPI operation raised the error class MPIX_ERR_REVOKED because
+   another process called the :ref:`MPIX_Comm_revoke` procedure on *comm*;
+3. the communicator has the info key *mpi_error_range* set to *group* or
+   *universe*, in which case the failure of any process in, respectively,
+   the group of *comm* or the MPI universe caused an operation to raise
+   an error of class MPIX_ERR_REVOKED.
+
+REVOKE PROPAGATION AND ORDERING
+-------------------------------
+
+Note that in a multithreaded application, a thread calling
+:ref:`MPIX_Comm_is_revoked` may return *flag = true* before the operation
+that raises the first exception of class MPIX_ERR_REVOKED has completed
+in a concurrent thread.
+
+EFFECT OF A COMMUNICATOR REVOCATION
+-----------------------------------
+
+Once a communicator has been revoked at an MPI process, all subsequent
+non-local operations on that communicator (with some exceptions listed
+below), are considered local and must complete by raising an error of 
+class MPIX_ERR_REVOKED at that MPI process.
+
+OPERATIONS ON A REVOKED COMMUNICATOR
+------------------------------------
+
+The following operations never raise an error of class MPIX_ERR_REVOKED,
+and complete with their normal semantics on a revoked communicator.
+
+* :ref:`MPIX_Comm_agree`
+* :ref:`MPIX_Comm_iagree`
+* :ref:`MPIX_Comm_shrink`
+* :ref:`MPIX_Comm_ishrink`
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPIX_Comm_revoke`
+   * :ref:`MPIX_Comm_agree`
+   * :ref:`MPIX_Comm_shrink`
diff --git a/docs/man-openmpi/man3/MPIX_Comm_ishrink.3.rst b/docs/man-openmpi/man3/MPIX_Comm_ishrink.3.rst
new file mode 100644
index 00000000000..ff204466e00
--- /dev/null
+++ b/docs/man-openmpi/man3/MPIX_Comm_ishrink.3.rst
@@ -0,0 +1,8 @@
+.. _mpix_comm_ishrink:
+
+MPIX_Comm_ishrink
+=================
+.. include_body
+
+.. include:: ../man3/MPIX_Comm_shrink.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPIX_Comm_revoke.3.rst b/docs/man-openmpi/man3/MPIX_Comm_revoke.3.rst
new file mode 100644
index 00000000000..76fe5bf7b28
--- /dev/null
+++ b/docs/man-openmpi/man3/MPIX_Comm_revoke.3.rst
@@ -0,0 +1,92 @@
+.. _mpix_comm_revoke:
+
+MPIX_Comm_revoke
+================
+.. include_body
+
+:ref:`MPIX_Comm_revoke` - Revoke a communicator, causing errors to be
+raised, at all ranks, for non-local operations on the communicator.
+
+This is part of the User Level Fault Mitigation :ref:`ULFM extension <ulfm-label>`.
+
+SYNTAX
+------
+
+C Syntax
+^^^^^^^^
+
+.. code-block:: c
+
+   #include <mpi.h>
+   #include <mpi-ext.h>
+
+   int MPIX_Comm_revoke(MPI_Comm comm)
+
+Fortran Syntax
+^^^^^^^^^^^^^^
+
+.. code-block:: fortran
+
+   USE MPI
+   USE MPI_EXT
+   ! or the older form: INCLUDE 'mpif.h'
+
+   MPIX_COMM_REVOKE(COMM, IERROR)
+        INTEGER COMM, IERROR
+
+Fortran 2008 Syntax
+^^^^^^^^^^^^^^^^^^^
+
+.. code-block:: fortran
+
+   USE mpi_f08
+   USE mpi_ext_f08
+
+   MPIX_Comm_revoke(comm, ierror)
+        TYPE(MPI_Comm), INTENT(IN) :: comm
+        INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+
+INPUT PARAMETERS
+----------------
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This function marks the communicator *comm* as revoked at all MPI processes
+in the groups (local and remote) associated with the communicator *comm*.
+This function is not collective and therefore does not have a matching call
+on remote MPI processes.
+
+The documentation for :ref:`MPIX_Comm_is_revoked` details the conditions
+for when a communicator becomes revoked locally, and what semantics apply
+on a revoked communicator. In summary, when a communicator is revoked,
+non-local operation raise an exception of class MPIX_ERR_REVOKED,
+except for select fault-tolerant operations.
+
+PROPAGATION OF THE REVOKED STATE AND ORDERING
+---------------------------------------------
+
+The implementation propagates the revoked state in a fault-tolerant manner;
+thus, the communicator becomes revoked at all non-failed MPI processes
+belonging to *comm* despite failed processes.
+
+There is no particular ordering between the revocation call at another
+process and the completion of operations at a local process, for example,
+a receive operation can raise an error of class MPIX_ERR_REVOKED, even if
+the send operation procedure is called before the revoke procedure at the
+sender.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPIX_Comm_is_revoked`
+   * :ref:`MPIX_Comm_agree`
+   * :ref:`MPIX_Comm_shrink`
diff --git a/docs/man-openmpi/man3/MPIX_Comm_shrink.3.rst b/docs/man-openmpi/man3/MPIX_Comm_shrink.3.rst
new file mode 100644
index 00000000000..1bbf94ba44c
--- /dev/null
+++ b/docs/man-openmpi/man3/MPIX_Comm_shrink.3.rst
@@ -0,0 +1,117 @@
+.. _mpix_comm_shrink:
+
+MPIX_Comm_shrink
+================
+.. include_body
+
+:ref:`MPIX_Comm_shrink`, :ref:`MPIX_Comm_ishrink` - Create a new communicator
+that includes all processes from the parent communicator that have not failed.
+
+This is part of the User Level Fault Mitigation :ref:`ULFM extension <ulfm-label>`.
+
+SYNTAX
+------
+
+C Syntax
+^^^^^^^^
+
+.. code-block:: c
+
+   #include <mpi.h>
+   #include <mpi-ext.h>
+
+   int MPIX_Comm_shrink(MPI_Comm comm, MPI_Comm *newcomm)
+   
+   int MPIX_Comm_ishrink(MPI_Comm comm, MPI_Comm *newcomm, MPI_Request *request)
+
+
+Fortran Syntax
+^^^^^^^^^^^^^^
+
+.. code-block:: fortran
+
+   USE MPI
+   USE MPI_EXT
+   ! or the older form: INCLUDE 'mpif.h'
+
+   MPIX_COMM_SHRINK(COMM, NEWCOMM, IERROR)
+        INTEGER COMM, NEWCOMM, IERROR
+
+   MPIX_COMM_ISHRINK(COMM, NEWCOMM, REQUEST, IERROR)
+        INTEGER COMM, NEWCOMM, REQUEST, IERROR
+
+
+Fortran 2008 Syntax
+^^^^^^^^^^^^^^^^^^^
+
+.. code-block:: fortran
+
+   USE mpi_f08
+   USE mpi_ext_f08
+
+   MPIX_Comm_shrink(comm, newcomm, ierror)
+        TYPE(MPI_Comm), INTENT(IN) :: comm
+        TYPE(MPI_Comm), INTENT(OUT) :: newcomm
+        INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+
+   MPIX_Comm_ishrink(comm, newcomm, request, ierror)
+        TYPE(MPI_Comm), INTENT(IN) :: comm
+        TYPE(MPI_Comm), INTENT(OUT), ASYNCHRONOUS :: newcomm
+        TYPE(MPI_Request), INTENT(OUT) :: request
+        INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+
+INPUT PARAMETERS
+----------------
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``newcomm``: Communicator (handle).
+* ``request``: Request (handle, non-blocking only).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This collective operation creates a new intra- or intercommunicator
+*newcomm* from the intra- or intercommunicator *comm*, respectively, by
+excluding the group of failed MPI processes as shrinkd upon during the
+operation.
+
+The groups of *newcomm* must include every MPI process that returns from
+:ref:`MPIX_Comm_shrink`, and it must exclude every MPI process whose failure
+caused an operation on *comm* to raise an MPI error of class
+MPIX_ERR_PROC_FAILED or MPIX_ERR_PROC_FAILED_PENDING at a member of the
+groups of *newcomm*, before that member initiated the shrink operation.
+
+Said otherwise, this procedure is semantically equivalent to an
+:ref:`MPI_Comm_split` operation that would succeed despite failures, where
+members of the groups of *newcomm* participate with the same color and a key
+equal to their rank in *comm*.
+
+:ref:`MPIX_Comm_ishrink` is the non-blocking variant of :ref:`MPIX_Comm_shrink`.
+Note that, as with :ref:`MPI_Comm_idup`, it is erroneous to use *newcomm*
+before *request* has completed.
+
+WHEN THE COMMUNICATOR IS REVOKED OR CONTAINS FAILED PROCESSES
+-------------------------------------------------------------
+
+This function never raises an error of classes MPIX_ERR_REVOKED or
+MPIX_ERR_PROC_FAILED. The defined semantics of :ref:`MPIX_Comm_shrink` and
+:ref:`MPIX_Comm_ishrink` are maintained when *comm* is revoked, or when the
+group of *comm* contains failed MPI processes. In particular,
+:ref:`MPIX_Comm_shrink` and :ref:`MPIX_Comm_ishrink` are collective operations,
+even when *comm* is revoked.
+
+The implementation will strive to detect all failures during the shrink
+operation, but in certain circumpstances, the group of *newcomm* may still
+contain failed MPI processes, whose failure will be detected in subsequent
+MPI operations on *newcomm*.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPIX_Comm_is_revoked`
diff --git a/docs/man-openmpi/man3/MPIX_Query_cuda_support.3.rst b/docs/man-openmpi/man3/MPIX_Query_cuda_support.3.rst
new file mode 100644
index 00000000000..302c478b60b
--- /dev/null
+++ b/docs/man-openmpi/man3/MPIX_Query_cuda_support.3.rst
@@ -0,0 +1,97 @@
+.. _mpix_query_cuda_support:
+
+
+MPIX_Query_cuda_support
+=======================
+
+.. include_body
+
+**MPIX_Query_cuda_support** - Returns 1 if there is CUDA-aware support
+and 0 if there is not.
+
+
+SYNTAX
+------
+
+
+C Syntax
+^^^^^^^^
+
+.. code-block:: c
+
+   #include <mpi.h>
+   #include <mpi-ext.h>
+
+   int MPIX_Query_cuda_support(void)
+
+
+Fortran Syntax
+^^^^^^^^^^^^^^
+
+There is no Fortran binding for this function.
+
+
+C++ Syntax
+^^^^^^^^^^
+
+There is no C++ binding for this function.
+
+
+DESCRIPTION
+-----------
+
+This function is part of an :ref:`Open MPI extension
+<ompi-features-extentions-label>`; it is not part of standard MPI.
+
+This routine returns 1 if both the MPI library was built with the
+NVIDIA CUDA library and the runtime supports CUDA buffers.  Otherwise,
+it returns 0.  This routine must be called after MPI is initialized,
+e.g., by a call to :ref:`MPI_Init(3) <MPI_Init>` or
+:ref:`MPI_Init_thread(3) <MPI_Init_thread>`.
+
+Including the Open MPI-specific file ``<mpi-ext.h>`` will define the C
+preprocessor macro ``OMPI_HAVE_MPI_EXT`` to ``1``.  Otherwise, it will
+be undefined.  This macro can be used by applications as a sentinel to
+know whether ``<mpi-ext.h>`` has been included or not.
+
+The Open MPI CUDA extension is built by default (regardless of whether
+or not Open MPI was built with CUDA support), but *could* have been
+disabled by an administrative action.  It is therefore safest for
+applications to check that the preprocessor macro
+``OMPI_HAVE_MPI_EXT_CUDA`` is defined and is set to 1 to know whether
+the ``MPIX_Query_CUDA_support()`` function is available.  Checking for
+this macro also protects the use of this function when compiling the
+application with older versions of Open MPI or other MPI
+implementations that do not have this function.
+
+
+EXAMPLES
+^^^^^^^^
+
+.. code-block:: c
+
+   #include <stdio.h>
+   #include <mpi.h>
+   #include <mpi-ext.h> /* Needed for CUDA-aware check */
+
+   int main(int argc, char *argv[])
+   {
+       MPI_Init(&argc, &argv);
+
+       bool happy = false;
+   #if defined(OMPI_HAVE_MPI_EXT_CUDA) && OMPI_HAVE_MPI_EXT_CUDA
+       happy = (bool) MPIX_Query_cuda_support();
+   #endif
+
+       if (happy) {
+           printf("This Open MPI installation has CUDA-aware support.\n");
+       } else {
+           printf("This Open MPI installation does not have CUDA-aware support.\n");
+       }
+
+       MPI_Finalize();
+       return 0;
+   }
+
+.. seealso::
+   * :ref:`MPIX_Query_rocm_support`
diff --git a/docs/man-openmpi/man3/MPIX_Query_rocm_support.3.rst b/docs/man-openmpi/man3/MPIX_Query_rocm_support.3.rst
new file mode 100644
index 00000000000..59f6148079a
--- /dev/null
+++ b/docs/man-openmpi/man3/MPIX_Query_rocm_support.3.rst
@@ -0,0 +1,97 @@
+.. _mpix_query_rocm_support:
+
+
+MPIX_Query_rocm_support
+=======================
+
+.. include_body
+
+**MPIX_Query_rocm_support** - Returns 1 if there is AMD ROCm-aware support
+and 0 if there is not.
+
+
+SYNTAX
+------
+
+
+C Syntax
+^^^^^^^^
+
+.. code-block:: c
+
+   #include <mpi.h>
+   #include <mpi-ext.h>
+
+   int MPIX_Query_rocm_support(void)
+
+
+Fortran Syntax
+^^^^^^^^^^^^^^
+
+There is no Fortran binding for this function.
+
+
+C++ Syntax
+^^^^^^^^^^
+
+There is no C++ binding for this function.
+
+
+DESCRIPTION
+-----------
+
+This function is part of an :ref:`Open MPI extension
+<ompi-features-extentions-label>`; it is not part of standard MPI.
+
+This routine returns 1 if both the MPI library was built with the AMD
+ROCm library and the runtime supports ROCm buffers.  Otherwise, it
+returns 0.  This routine must be called after MPI is initialized,
+e.g., by a call to :ref:`MPI_Init(3) <MPI_Init>` or
+:ref:`MPI_Init_thread(3) <MPI_Init_thread>`.
+
+Including the Open MPI-specific file ``<mpi-ext.h>`` will define the C
+preprocessor macro ``OMPI_HAVE_MPI_EXT`` to ``1``.  Otherwise, it will
+be undefined.  This macro can be used by applications as a sentinel to
+know whether ``<mpi-ext.h>`` has been included or not.
+
+The Open MPI ROCm extension is built by default (regardless of whether
+or not Open MPI was built with ROCm support), but *could* have been
+disabled by an administrative action.  It is therefore safest for
+applications to check that the preprocessor macro
+``OMPI_HAVE_MPI_EXT_ROCM`` is defined and is set to 1 to know whether
+the ``MPIX_Query_rocm_support()`` function is available.  Checking for
+this macro also protects the use of this function when compiling the
+application with older versions of Open MPI or other MPI
+implementations that do not have this function.
+
+
+EXAMPLES
+^^^^^^^^
+
+.. code-block:: c
+
+   #include <stdio.h>
+   #include <mpi.h>
+   #include <mpi-ext.h> /* Needed for ROCm-aware check */
+
+   int main(int argc, char *argv[])
+   {
+       MPI_Init(&argc, &argv);
+
+       bool happy = false;
+   #if defined(OMPI_HAVE_MPI_EXT_ROCM) && OMPI_HAVE_MPI_EXT_ROCM
+       happy = (bool) MPIX_Query_rocm_support();
+   #endif
+
+       if (happy) {
+           printf("This Open MPI installation has ROCm-aware support.\n");
+       } else {
+           printf("This Open MPI installation does not have ROCm-aware support.\n");
+       }
+
+       MPI_Finalize();
+       return 0;
+   }
+
+.. seealso::
+   * :ref:`MPIX_Query_cuda_support`
diff --git a/docs/man-openmpi/man3/MPI_Abort.3.rst b/docs/man-openmpi/man3/MPI_Abort.3.rst
new file mode 100644
index 00000000000..02cd8c2a906
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Abort.3.rst
@@ -0,0 +1,42 @@
+.. _mpi_abort:
+
+
+MPI_Abort
+=========
+
+.. include_body
+
+:ref:`MPI_Abort` |mdash| Terminates MPI execution environment.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_abort.rst
+
+INPUT PARAMETERS
+----------------
+* ``comm``: Communicator of tasks to abort.
+* ``errorcode``: Error code to return to invoking environment.
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This routine makes a "best attempt" to abort all tasks in the group of
+comm. This function does not require that the invoking environment take
+any action with the error code. However, a UNIX or POSIX environment
+should handle this as a return errorcode from the main program or an
+abort (errorcode).
+
+The long-term goal of the Open MPI implementation is to terminate all
+processes in all tasks that contain a process in *comm, and the error
+code is not returned to the invoking environment. At the moment, this
+isn't fully implemented and* :ref:`MPI_Abort` *will terminate the entire job.*
+
+Note: All associated processes are sent a ``SIGTERM``.
+
+
+ERRORS
+------
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Accumulate.3.rst b/docs/man-openmpi/man3/MPI_Accumulate.3.rst
new file mode 100644
index 00000000000..3c031d671f2
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Accumulate.3.rst
@@ -0,0 +1,94 @@
+.. _mpi_accumulate:
+
+
+MPI_Accumulate
+==============
+
+.. include_body
+
+:ref:`MPI_Accumulate`, :ref:`MPI_Raccumulate` - Combines the contents of the
+origin buffer with that of a target buffer.
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Accumulate, MPI_Raccumulate
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_accumulate.rst
+
+INPUT PARAMETERS
+----------------
+* ``origin_addr``: Initial address of buffer (choice).
+* ``origin_count``: Number of entries in buffer (nonnegative integer).
+* ``origin_datatype``: Data type of each buffer entry (handle).
+* ``target_rank``: Rank of target (nonnegative integer).
+* ``target_disp``: Displacement from start of window to beginning of target buffer (nonnegative integer).
+* ``target_count``: Number of entries in target buffer (nonnegative integer).
+* ``target_datatype``: Data type of each entry in target buffer (handle).
+* ``op``: Reduce operation (handle).
+* ``win``: Window object (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``MPI_Raccumulate``:  RMA request
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Accumulate` is a function used for one-sided MPI communication
+that adds the contents of the origin buffer (as defined by
+*origin_addr*, *origin_count*, and *origin_datatype*) to the buffer
+specified by the arguments *target_count* and *target_datatype*, at
+offset *target_disp*, in the target window specified by *target_rank*
+and *win*, using the operation *op*. The target window can only be
+accessed by processes within the same node. This is similar to :ref:`MPI_Put`,
+except that data is combined into the target area instead of overwriting
+it.
+
+Any of the predefined operations for :ref:`MPI_Reduce` can be used.
+User-defined functions cannot be used. For example, if *op* is MPI_SUM,
+each element of the origin buffer is added to the corresponding element
+in the target, replacing the former value in the target.
+
+Each datatype argument must be a predefined data type or a derived data
+type, where all basic components are of the same predefined data type.
+Both datatype arguments must be constructed from the same predefined
+data type. The operation *op* applies to elements of that predefined
+type. The *target_datatype* argument must not specify overlapping
+entries, and the target buffer must fit in the target window.
+
+A new predefined operation, MPI_REPLACE, is defined. It corresponds to
+the associative function f(a, b) =b; that is, the current value in the
+target memory is replaced by the value supplied by the origin.
+
+:ref:`MPI_Raccumulate` is similar to :ref:`MPI_Accumulate`, except that it
+allocates a communication request object and associates it with the
+request handle (the argument *request*) that can be used to wait or test
+for completion. The completion of an :ref:`MPI_Raccumulate` operation
+indicates that the *origin_addr* buffer is free to be updated. It does
+not indicate that the operation has completed at the target window.
+
+
+NOTES
+-----
+
+:ref:`MPI_Put` is a special case of :ref:`MPI_Accumulate`, with the operation
+MPI_REPLACE. Note, however, that :ref:`MPI_Put` and :ref:`MPI_Accumulate` have
+different constraints on concurrent updates.
+
+It is the user's responsibility to guarantee that, when using the
+accumulate functions, the target displacement argument is such that
+accesses to the window are properly aligned according to the data type
+arguments in the call to the :ref:`MPI_Accumulate` function.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Put`
+   * :ref:`MPI_Get_accumulate`
+   * :ref:`MPI_Reduce`
diff --git a/docs/man-openmpi/man3/MPI_Add_error_class.3.rst b/docs/man-openmpi/man3/MPI_Add_error_class.3.rst
new file mode 100644
index 00000000000..cee8db554b9
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Add_error_class.3.rst
@@ -0,0 +1,53 @@
+.. _mpi_add_error_class:
+
+
+MPI_Add_error_class
+===================
+
+.. include_body
+
+:ref:`MPI_Add_error_class` |mdash| Creates a new error class and returns its value
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_add_error_class.rst
+
+OUTPUT PARAMETERS
+-----------------
+* ``errorclass``: New error class (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The function :ref:`MPI_Add_error_class` creates a new, local error class.
+
+
+NOTES
+-----
+
+Because this function is local, the same value of *errorclass* may not
+be returned on all processes that make this call, even if they call the
+function concurrently. Thus, same error on different processes may not
+cause the same value of *errorclass* to be returned. To reduce the
+potential for confusion, :ref:`MPI_Add_error_string` may be used on multiple
+processes to associate the same error string with the newly created
+*errorclass*. Even though *errorclass* may not be consistent across
+processes, using :ref:`MPI_Add_error_string` will ensure the error string
+associated with it will be the same everywhere.
+
+No function is provided to free error classes, as it is not expected
+that an application will create them in significant numbers.
+
+The value returned is always greater than or equal to MPI_ERR_LASTCODE.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Add_error_code`
+   * :ref:`MPI_Add_error_string`
+   * :ref:`MPI_Error_class`
+   * :ref:`MPI_Error_string`
diff --git a/docs/man-openmpi/man3/MPI_Add_error_code.3.rst b/docs/man-openmpi/man3/MPI_Add_error_code.3.rst
new file mode 100644
index 00000000000..057efc74d65
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Add_error_code.3.rst
@@ -0,0 +1,47 @@
+.. _mpi_add_error_code:
+
+
+MPI_Add_error_code
+==================
+
+.. include_body
+
+:ref:`MPI_Add_error_code` |mdash| Creates a new error code associated with
+*errorclass*
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_add_error_code.rst
+
+INPUT PARAMETER
+---------------
+* ``errorclass``: MPI error class (integer).
+
+OUTPUT PARAMETERS
+-----------------
+* ``errorcode``: Error code returned by an MPI routine or an MPI error class (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Creates a new error code associated with *errorclass* and returns its
+value in *errorcode*.
+
+
+NOTES
+-----
+
+No function is provided to free error codes, as it is not expected that
+an application will create them in significant numbers.
+
+The value returned is always greater than or equal to MPI_ERR_LASTCODE.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Add_error_class`
+   * :ref:`MPI_Error_class`
diff --git a/docs/man-openmpi/man3/MPI_Add_error_string.3.rst b/docs/man-openmpi/man3/MPI_Add_error_string.3.rst
new file mode 100644
index 00000000000..8b26a3e84ea
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Add_error_string.3.rst
@@ -0,0 +1,43 @@
+.. _mpi_add_error_string:
+
+
+MPI_Add_error_string
+====================
+
+.. include_body
+
+:ref:`MPI_Add_error_string` |mdash| Associates a string with an error code or class
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_add_error_string.rst
+
+INPUT PARAMETERS
+----------------
+* ``errorcode``: MPI error class, or an error code returned by an MPI routine (integer).
+* ``string``: Text that corresponds to the error code or class (string).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This routine associates an error string with an error code or class.
+Calling :ref:`MPI_Add_error_string` for an error code or class that already has
+an associated error string will replace the old string with the new one.
+It is erroneous to call :ref:`MPI_Add_error_string` for an error value not
+generated via :ref:`MPI_Add_error_class` or :ref:`MPI_Add_error_code` (e.g., an error
+code or class with a value not greater than MPI_LAST_ERRCODE).
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Add_error_class`
+   * :ref:`MPI_Add_error_code`
+   * :ref:`MPI_Error_class`
+   * :ref:`MPI_Error_string`
diff --git a/docs/man-openmpi/man3/MPI_Address.3.rst b/docs/man-openmpi/man3/MPI_Address.3.rst
new file mode 100644
index 00000000000..906620ca273
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Address.3.rst
@@ -0,0 +1,66 @@
+.. _mpi_address:
+
+
+MPI_Address
+===========
+
+.. include_body
+
+:ref:`MPI_Address` |mdash| Gets the address of a location in memory |mdash| |deprecated_favor| :ref:`MPI_Get_address`.
+
+
+SYNTAX
+------
+
+.. NOTE: The bindings for this man page are not automatically
+   generated from the official MPI Forum JSON/Python library because
+   this function is deprecated.  Hence, this function is not included
+   in the MPI Forum JSON data, and we therefore have to hard-code the
+   bindings here ourselves.
+
+C Syntax
+^^^^^^^^
+
+.. code-block:: c
+
+   #include <mpi.h>
+
+   int MPI_Address(void *location, MPI_Aint *address)
+
+
+Fortran Syntax
+^^^^^^^^^^^^^^
+
+.. code-block:: fortran
+
+   USE MPI
+   ! or the older form: INCLUDE 'mpif.h'
+
+   MPI_ADDRESS(LOCATION, ADDRESS, IERROR)
+   	<type>	LOCATION (*)
+   	INTEGER	ADDRESS, IERROR
+
+
+INPUT PARAMETER
+---------------
+* ``location``: Location in caller memory (choice).
+
+OUTPUT PARAMETERS
+-----------------
+* ``address``: Address of location (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Note that use of this routine is *deprecated* as of MPI-2. Please use
+:ref:`MPI_Get_address` instead.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Get_address`
diff --git a/docs/man-openmpi/man3/MPI_Aint_add.3.rst b/docs/man-openmpi/man3/MPI_Aint_add.3.rst
new file mode 100644
index 00000000000..f8fa8111adb
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Aint_add.3.rst
@@ -0,0 +1,57 @@
+.. _mpi_aint_add:
+
+
+MPI_Aint_add
+============
+
+.. include_body
+
+:ref:`MPI_Aint_add`, :ref:`MPI_Aint_diff` - Portable functions for arithmetic
+on MPI_Aint values.
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Aint_add, MPI_Aint_diff
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_aint_add.rst
+
+INPUT PARAMETERS
+----------------
+* ``base``: Base address (integer).
+* ``disp``: Displacement (integer).
+* ``addr1``: Minuend address (integer).
+* ``addr2``: Subtrahend address (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Aint_add` produces a new MPI_Aint value that is equivalent to the
+sum of the *base* and *disp* arguments, where *base* represents a base
+address returned by a call to :ref:`MPI_Get_address` and *disp* represents
+a signed integer displacement. The resulting address is valid only at
+the process that generated *base*, and it must correspond to a location
+in the same object referenced by *base*, as described in MPI-3.1 section
+4.1.12. The addition is performed in a manner that results in the
+correct MPI_Aint representation of the output address, as if the process
+that originally produced *base* had called:
+
+.. code-block:: c
+
+   MPI_Get_address ((char *) base + disp, &result);
+
+:ref:`MPI_Aint_diff` produces a new MPI_Aint value that is equivalent to
+the difference between *addr1* and *addr2* arguments, where *addr1*
+and *addr2* represent addresses returned by calls to
+:ref:`MPI_Get_address`.  The resulting address is valid only at the
+process that generated *addr1* and *addr2*, and *addr1* and *addr2*
+must correspond to locations in the same object in the same process,
+as described in MPI-3.1 section 4.1.12. The difference is calculated
+in a manner that results in the signed difference from *addr1* to
+*addr2*, as if the process that originally produced the addresses had
+called ``(char *) addr1`` - ``(char *) addr2`` on the addresses
+initially passed to :ref:`MPI_Get_address`.
+
+
+.. seealso::
+   * :ref:`MPI_Get_address`
diff --git a/docs/man-openmpi/man3/MPI_Aint_diff.3.rst b/docs/man-openmpi/man3/MPI_Aint_diff.3.rst
new file mode 100644
index 00000000000..b231df11d39
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Aint_diff.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_aint_diff:
+
+MPI_Aint_diff
+=============
+    .. include_body
+
+.. include:: ../man3/MPI_Aint_add.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Allgather.3.rst b/docs/man-openmpi/man3/MPI_Allgather.3.rst
new file mode 100644
index 00000000000..2d7926bb5cb
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Allgather.3.rst
@@ -0,0 +1,127 @@
+.. _mpi_allgather:
+
+
+MPI_Allgather
+=============
+
+.. include_body
+
+:ref:`MPI_Allgather`, :ref:`MPI_Iallgather`, :ref:`MPI_Allgather_init` - Gathers data
+from all processes and distributes it to all processes
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Allgather, MPI_Iallgather, MPI_Allgather_init
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_allgather.rst
+
+INPUT PARAMETERS
+----------------
+* ``sendbuf``: Starting address of send buffer (choice).
+* ``sendcount``: Number of elements in send buffer (integer).
+* ``sendtype``: Datatype of send buffer elements (handle).
+* ``recvbuf``: Starting address of recv buffer (choice).
+* ``recvcount``: Number of elements received from any process (integer).
+* ``recvtype``: Datatype of receive buffer elements (handle).
+* ``comm``: Communicator (handle).
+* ``info``: Info (handle, persistent only).
+
+OUTPUT PARAMETERS
+-----------------
+* ``recvbuf``: Address of receive buffer (choice).
+* ``request``: Request (handle, non-blocking and persistent only).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Allgather` is similar to :ref:`MPI_Gather`, except that all processes
+receive the result, instead of just the root. In other words, all
+processes contribute to the result, and all processes receive the
+result.
+
+The type signature associated with sendcount, sendtype at a process must
+be equal to the type signature associated with recvcount, recvtype at
+any other process.
+
+The outcome of a call to :ref:`MPI_Allgather` is as if all processes
+executed n calls to
+
+.. code-block:: c
+
+     MPI_Gather(sendbuf, sendcount, sendtype, recvbuf, recvcount,
+                recvtype, root, comm);
+
+     // for root = 0, ..., n-1.
+
+The rules for correct usage of :ref:`MPI_Allgather`
+are easily found from the corresponding rules for :ref:`MPI_Gather`.
+
+**Example:** The all-gather version of Example 1 in :ref:`MPI_Gather`. Using
+:ref:`MPI_Allgather`, we will gather 100 ints from every process in the group
+to every process.
+
+.. code-block:: c
+
+       MPI_Comm comm;
+       int gsize,sendarray[100];
+       int *rbuf;
+       ...
+       MPI_Comm_size( comm, &gsize);
+       rbuf = (int *)malloc(gsize*100*sizeof(int));
+       MPI_Allgather( sendarray, 100, MPI_INT, rbuf, 100, MPI_INT, comm);
+
+After the call, every process has the group-wide concatenation of the
+sets of data.
+
+
+USE OF IN-PLACE OPTION
+----------------------
+
+When the communicator is an intracommunicator, you can perform an
+all-gather operation in-place (the output buffer is used as the input
+buffer). Use the variable MPI_IN_PLACE as the value of *sendbuf*. In
+this case, *sendcount* and *sendtype* are ignored. The input data of
+each process is assumed to be in the area where that process would
+receive its own contribution to the receive buffer. Specifically, the
+outcome of a call to :ref:`MPI_Allgather` that used the in-place option is
+identical to the case in which all processes executed *n* calls to
+
+.. code-block:: c
+
+      MPI_Gather(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL, recvbuf,
+                 recvcount, recvtype, root, comm);
+
+      // for root = 0, ..., n-1.
+
+Note that MPI_IN_PLACE is a special kind of value; it has the same
+restrictions on its use as MPI_BOTTOM.
+
+Because the in-place option converts the receive buffer into a
+send-and-receive buffer, a Fortran binding that includes INTENT must
+mark these as INOUT, not OUT.
+
+
+WHEN COMMUNICATOR IS AN INTER-COMMUNICATOR
+------------------------------------------
+
+When the communicator is an inter-communicator, the gather operation
+occurs in two phases. The data is gathered from all the members of the
+first group and received by all the members of the second group. Then
+the data is gathered from all the members of the second group and
+received by all the members of the first. The operation, however, need
+not be symmetric. The number of items sent by the processes in first
+group need not be equal to the number of items sent by the the processes
+in the second group. You can move data in only one direction by giving
+*sendcount* a value of 0 for communication in the reverse direction.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Allgatherv`
+   * :ref:`MPI_Gather`
diff --git a/docs/man-openmpi/man3/MPI_Allgather_init.3.rst b/docs/man-openmpi/man3/MPI_Allgather_init.3.rst
new file mode 100644
index 00000000000..c55a8dff817
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Allgather_init.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_allgather_init:
+
+MPI_Allgather_init
+==================
+    .. include_body
+
+.. include:: ../man3/MPI_Allgather.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Allgatherv.3.rst b/docs/man-openmpi/man3/MPI_Allgatherv.3.rst
new file mode 100644
index 00000000000..50b0f78ae2d
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Allgatherv.3.rst
@@ -0,0 +1,110 @@
+.. _mpi_allgatherv:
+
+
+MPI_Allgatherv
+==============
+
+.. include_body
+
+:ref:`MPI_Allgatherv`, :ref:`MPI_Iallgatherv`, :ref:`MPI_Allgatherv_init` - Gathers data
+from all processes and delivers it to all. Each process may contribute a
+different amount of data.
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Allgatherv, MPI_Iallgatherv, MPI_Allgatherv_init
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_allgatherv.rst
+
+INPUT PARAMETERS
+----------------
+* ``sendbuf``: Starting address of send buffer (choice).
+* ``sendcount``: Number of elements in send buffer (integer).
+* ``sendtype``: Datatype of send buffer elements (handle).
+* ``recvcount``: Integer array (of length group size) containing the number of elements that are received from each process.
+* ``displs``: Integer array (of length group size). Entry i specifies the displacement (relative to recvbuf) at which to place the incoming data from process i.
+* ``recvtype``: Datatype of receive buffer elements (handle).
+* ``comm``: Communicator (handle).
+* ``info``: Info (handle, persistent only).
+
+OUTPUT PARAMETERS
+-----------------
+* ``recvbuf``: Address of receive buffer (choice).
+* ``request``: Request (handle, non-blocking and persistent only).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Allgatherv` is similar to :ref:`MPI_Allgather` in that all processes gather
+data from all other processes, except that each process can send a
+different amount of data. The block of data sent from the jth process is
+received by every process and placed in the jth block of the buffer
+*recvbuf.*
+
+The type signature associated with sendcount, sendtype, at process j
+must be equal to the type signature associated with recvcounts[j],
+recvtype at any other process.
+
+The outcome is as if all processes executed calls to
+
+.. code-block:: c
+
+   MPI_Gatherv(sendbuf, sendcount, sendtype, recvbuf, recvcount,
+               displs, recvtype, root, comm);
+
+   // for root = 0, ..., n-1.
+
+The rules for correct usage of :ref:`MPI_Allgatherv`
+are easily found from the corresponding rules for :ref:`MPI_Gatherv`.
+
+
+USE OF IN-PLACE OPTION
+----------------------
+
+When the communicator is an intracommunicator, you can perform an
+all-gather operation in-place (the output buffer is used as the input
+buffer). Use the variable MPI_IN_PLACE as the value of *sendbuf*. In
+this case, *sendcount* and *sendtype* are ignored. The input data of
+each process is assumed to be in the area where that process would
+receive its own contribution to the receive buffer. Specifically, the
+outcome of a call to :ref:`MPI_Allgatherv` that used the in-place option is
+identical to the case in which all processes executed *n* calls to
+
+.. code-block:: c
+
+      MPI_Gatherv(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL, recvbuf,
+                  recvcounts, displs, recvtype, root, comm);
+
+      // for root = 0, ..., n-1.
+
+Note that MPI_IN_PLACE is a special kind of value; it has the same
+restrictions on its use as MPI_BOTTOM.
+
+Because the in-place option converts the receive buffer into a
+send-and-receive buffer, a Fortran binding that includes INTENT must
+mark these as INOUT, not OUT.
+
+
+WHEN COMMUNICATOR IS AN INTER-COMMUNICATOR
+------------------------------------------
+
+When the communicator is an inter-communicator, the gather operation
+occurs in two phases. The data is gathered from all the members of the
+first group, concatenated, and received by all the members of the second
+group. Then the data is gathered from all the members of the second
+group, concatenated, and received by all the members of the first. The
+send buffer arguments in the one group must be consistent with the
+receive buffer arguments in the other group, and vice versa. The
+operation must exhibit symmetric, full-duplex behavior.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Gatherv`
+   * :ref:`MPI_Allgather`
diff --git a/docs/man-openmpi/man3/MPI_Allgatherv_init.3.rst b/docs/man-openmpi/man3/MPI_Allgatherv_init.3.rst
new file mode 100644
index 00000000000..3b4829505a7
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Allgatherv_init.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_allgatherv_init:
+
+MPI_Allgatherv_init
+===================
+    .. include_body
+
+.. include:: ../man3/MPI_Allgatherv.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Alloc_mem.3.rst b/docs/man-openmpi/man3/MPI_Alloc_mem.3.rst
new file mode 100644
index 00000000000..f39c26ee0c5
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Alloc_mem.3.rst
@@ -0,0 +1,72 @@
+.. _mpi_alloc_mem:
+
+
+MPI_Alloc_mem
+=============
+
+.. include_body
+
+:ref:`MPI_Alloc_mem` |mdash| Allocates a specified memory segment.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_alloc_mem.rst
+
+INPUT PARAMETERS
+----------------
+* ``size``: Size of memory segment in bytes (nonnegative integer).
+* ``info``: Info argument (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``baseptr``: Pointer to beginning of memory segment allocated.
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Alloc_mem` allocates *size* bytes of memory. The starting address of
+this memory is returned in the variable *baseptr*.
+
+
+C NOTES
+-------
+
+The parameter *baseptr* is of type ``void *`` to allow passing any
+pointer object for this parameter. The provided argument should be a
+pointer to a pointer of arbitrary type (e.g., ``void **``).
+
+
+Fortran NOTES
+-------------
+
+The :ref:`MPI_Alloc_mem` calls require the use of the ``iso_c_binding`` module
+for due to the use of ``TYPE(C_PTR)``.
+
+.. code-block:: fortran
+
+   use iso_c_binding
+
+   type(c_ptr) :: alloc_ptr
+   integer :: size, ierr
+   double precision, pointer :: array(:,:)
+
+   ! A 2D array of 100 elements
+   size = 10 * 10
+   call MPI_Alloc_Mem(size * 8, MPI_INFO_NULL, alloc_ptr, ierr)
+
+   ! Point to the array
+   call c_f_pointer(alloc_ptr, array, [10, 10])
+
+   ! ... use the array ...
+
+   ! Free the memory, no need for the alloc_ptr
+   call MPI_Free_mem(array, ierr)
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Free_mem`
diff --git a/docs/man-openmpi/man3/MPI_Allreduce.3.rst b/docs/man-openmpi/man3/MPI_Allreduce.3.rst
new file mode 100644
index 00000000000..197fa7032a5
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Allreduce.3.rst
@@ -0,0 +1,136 @@
+.. _mpi_allreduce:
+
+
+MPI_Allreduce
+=============
+
+.. include_body
+
+:ref:`MPI_Allreduce`, :ref:`MPI_Iallreduce`, :ref:`MPI_Allreduce_init` - Combines values
+from all processes and distributes the result back to all processes.
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Allreduce, MPI_Iallreduce, MPI_Allreduce_init
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_allreduce.rst
+
+INPUT PARAMETERS
+----------------
+* ``sendbuf``: Starting address of send buffer (choice).
+* ``count``: Number of elements in send buffer (integer).
+* ``datatype``: Datatype of elements of send buffer (handle).
+* ``op``: Operation (handle).
+* ``comm``: Communicator (handle).
+* ``info``: Info (handle, persistent only).
+
+OUTPUT PARAMETERS
+-----------------
+* ``recvbuf``: Starting address of receive buffer (choice).
+* ``request``: Request (handle, non-blocking and persistent only).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Same as :ref:`MPI_Reduce` except that the result appears in the receive buffer
+of all the group members.
+
+**Example 1:** A routine that computes the product of a vector and an
+array that are distributed across a group of processes and returns the
+answer at all nodes (compare with Example 2, with :ref:`MPI_Reduce`, below).
+
+.. code-block:: fortran
+
+   SUBROUTINE PAR_BLAS2(m, n, a, b, c, comm)
+   REAL a(m), b(m,n)    ! local slice of array
+   REAL c(n)            ! result
+   REAL sum(n)
+   INTEGER n, comm, i, j, ierr
+
+   ! local sum
+   DO j= 1, n
+     sum(j) = 0.0
+     DO i = 1, m
+       sum(j) = sum(j) + a(i)*b(i,j)
+     END DO
+   END DO
+
+   ! global sum
+   CALL MPI_ALLREDUCE(sum, c, n, MPI_REAL, MPI_SUM, comm, ierr)
+
+   ! return result at all nodes
+   RETURN
+
+**Example 2:** A routine that computes the product of a vector and an
+array that are distributed across a group of processes and returns the
+answer at node zero.
+
+.. code-block:: fortran
+
+   SUBROUTINE PAR_BLAS2(m, n, a, b, c, comm)
+   REAL a(m), b(m,n)    ! local slice of array
+   REAL c(n)            ! result
+   REAL sum(n)
+   INTEGER n, comm, i, j, ierr
+
+   ! local sum
+   DO j= 1, n
+     sum(j) = 0.0
+     DO i = 1, m
+       sum(j) = sum(j) + a(i)*b(i,j)
+     END DO
+   END DO
+
+   ! global sum
+   CALL MPI_REDUCE(sum, c, n, MPI_REAL, MPI_SUM, 0, comm, ierr)
+
+   ! return result at node zero (and garbage at the other nodes)
+   RETURN
+
+
+USE OF IN-PLACE OPTION
+----------------------
+
+When the communicator is an intracommunicator, you can perform an
+all-reduce operation in-place (the output buffer is used as the input
+buffer). Use the variable MPI_IN_PLACE as the value of *sendbuf* at all
+processes.
+
+Note that MPI_IN_PLACE is a special kind of value; it has the same
+restrictions on its use as MPI_BOTTOM.
+
+Because the in-place option converts the receive buffer into a
+send-and-receive buffer, a Fortran binding that includes INTENT must
+mark these as INOUT, not OUT.
+
+
+WHEN COMMUNICATOR IS AN INTER-COMMUNICATOR
+------------------------------------------
+
+When the communicator is an inter-communicator, the reduce operation
+occurs in two phases. The data is reduced from all the members of the
+first group and received by all the members of the second group. Then
+the data is reduced from all the members of the second group and
+received by all the members of the first. The operation exhibits a
+symmetric, full-duplex behavior.
+
+When the communicator is an intra-communicator, these groups are the
+same, and the operation occurs in a single phase.
+
+
+NOTES ON COLLECTIVE OPERATIONS
+------------------------------
+
+The reduction functions ( MPI_Op ) do not return an error value. As a
+result, if the functions detect an error, all they can do is either call
+:ref:`MPI_Abort` or silently skip the problem. Thus, if you change the error
+handler from MPI_ERRORS_ARE_FATAL to something else, for example,
+MPI_ERRORS_RETURN , then no error may be indicated.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Allreduce_init.3.rst b/docs/man-openmpi/man3/MPI_Allreduce_init.3.rst
new file mode 100644
index 00000000000..beae09be2bb
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Allreduce_init.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_allreduce_init:
+
+MPI_Allreduce_init
+==================
+
+.. include_body
+
+.. include:: ../man3/MPI_Allreduce.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_Alltoall.3.rst b/docs/man-openmpi/man3/MPI_Alltoall.3.rst
new file mode 100644
index 00000000000..610db373f8c
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Alltoall.3.rst
@@ -0,0 +1,113 @@
+.. _mpi_alltoall:
+
+
+MPI_Alltoall
+============
+
+.. include_body
+
+:ref:`MPI_Alltoall`, :ref:`MPI_Ialltoall`, :ref:`MPI_Alltoall_init` - All processes send
+data to all processes
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Alltoall, MPI_Ialltoall, MPI_Alltoall_init
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_alltoall.rst
+
+INPUT PARAMETERS
+----------------
+* ``sendbuf``: Starting address of send buffer (choice).
+* ``sendcount``: Number of elements to send to each process (integer).
+* ``sendtype``: Datatype of send buffer elements (handle).
+* ``recvcount``: Number of elements to receive from each process (integer).
+* ``recvtype``: Datatype of receive buffer elements (handle).
+* ``comm``: Communicator over which data is to be exchanged (handle).
+* ``info``: Info (handle, persistent only)
+
+OUTPUT PARAMETERS
+-----------------
+* ``recvbuf``: Starting address of receive buffer (choice).
+* ``request``: Request (handle, non-blocking and persistent only).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Alltoall` is a collective operation in which all processes send the
+same amount of data to each other, and receive the same amount of data
+from each other. The operation of this routine can be represented as
+follows, where each process performs 2n (n being the number of processes
+in communicator *comm*) independent point-to-point communications
+(including communication with itself).
+
+::
+
+   	MPI_Comm_size(comm, &n);
+   	for (i = 0, i < n; i++)
+   	    MPI_Send(sendbuf + i * sendcount * extent(sendtype),
+   	        sendcount, sendtype, i, ..., comm);
+   	for (i = 0, i < n; i++)
+   	    MPI_Recv(recvbuf + i * recvcount * extent(recvtype),
+   	        recvcount, recvtype, i, ..., comm);
+
+Each process breaks up its local *sendbuf* into n blocks - each
+containing *sendcount* elements of type *sendtype* - and divides its
+*recvbuf* similarly according to *recvcount* and *recvtype*. Process j
+sends the k-th block of its local *sendbuf* to process k, which places
+the data in the j-th block of its local *recvbuf*. The amount of data
+sent must be equal to the amount of data received, pairwise, between
+every pair of processes.
+
+WHEN COMMUNICATOR IS AN INTER-COMMUNICATOR
+
+When the communicator is an inter-communicator, the gather operation
+occurs in two phases. The data is gathered from all the members of the
+first group and received by all the members of the second group. Then
+the data is gathered from all the members of the second group and
+received by all the members of the first. The operation exhibits a
+symmetric, full-duplex behavior.
+
+The first group defines the root process. The root process uses MPI_ROOT
+as the value of *root*. All other processes in the first group use
+``MPI_PROC_NULL`` as the value of *root*. All processes in the second group
+use the rank of the root process in the first group as the value of
+*root*.
+
+When the communicator is an intra-communicator, these groups are the
+same, and the operation occurs in a single phase.
+
+
+USE OF IN-PLACE OPTION
+----------------------
+
+When the communicator is an intracommunicator, you can perform an
+all-to-all operation in-place (the output buffer is used as the input
+buffer). Use the variable MPI_IN_PLACE as the value of *sendbuf*. In
+this case, *sendcount* and *sendtype* are ignored. The input data of
+each process is assumed to be in the area where that process would
+receive its own contribution to the receive buffer.
+
+
+NOTES
+-----
+
+All arguments on all processes are significant. The *comm* argument, in
+particular, must describe the same communicator on all processes.
+
+There are two MPI library functions that are more general than
+:ref:`MPI_Alltoall`. :ref:`MPI_Alltoallv` allows all-to-all communication to and from
+buffers that need not be contiguous; different processes may send and
+receive different amounts of data. :ref:`MPI_Alltoallw` expands :ref:`MPI_Alltoallv`'s
+functionality to allow the exchange of data with different datatypes.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Alltoallv`
+   * :ref:`MPI_Alltoallw`
diff --git a/docs/man-openmpi/man3/MPI_Alltoall_init.3.rst b/docs/man-openmpi/man3/MPI_Alltoall_init.3.rst
new file mode 100644
index 00000000000..3fe6fe7491d
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Alltoall_init.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_alltoall_init:
+
+MPI_Alltoall_init
+=================
+    .. include_body
+
+.. include:: ../man3/MPI_Alltoall.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Alltoallv.3.rst b/docs/man-openmpi/man3/MPI_Alltoallv.3.rst
new file mode 100644
index 00000000000..d1db508fccd
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Alltoallv.3.rst
@@ -0,0 +1,122 @@
+.. _mpi_alltoallv:
+
+
+MPI_Alltoallv
+=============
+
+.. include_body
+
+:ref:`MPI_Alltoallv`, :ref:`MPI_Ialltoallv`, :ref:`MPI_Alltoallv_init` - All processes
+send different amount of data to, and receive different amount of data
+from, all processes
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Alltoallv, MPI_Ialltoallv, MPI_Alltoallv_init
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_alltoallv.rst
+
+INPUT PARAMETERS
+----------------
+* ``sendbuf``: Starting address of send buffer.
+* ``sendcounts``: Integer array, where entry i specifies the number of elements to send to rank i.
+* ``sdispls``: Integer array, where entry i specifies the displacement (offset from *sendbuf*, in units of *sendtype*) from which to send data to rank i.
+* ``sendtype``: Datatype of send buffer elements.
+* ``recvcounts``: Integer array, where entry j specifies the number of elements to receive from rank j.
+* ``rdispls``: Integer array, where entry j specifies the displacement (offset from *recvbuf*, in units of *recvtype*) to which data from rank j should be written.
+* ``recvtype``: Datatype of receive buffer elements.
+* ``comm``: Communicator over which data is to be exchanged.
+* ``info``: Info (handle, persistent only)
+
+OUTPUT PARAMETERS
+-----------------
+* ``recvbuf``: Address of receive buffer.
+* ``request``: Request (handle, non-blocking and persistent only).
+* ``ierror``: Fortran only: Error status.
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Alltoallv` is a generalized collective operation in which all
+processes send data to and receive data from all other processes. It
+adds flexibility to :ref:`MPI_Alltoall` by allowing the user to specify data to
+send and receive vector-style (via a displacement and element count).
+The operation of this routine can be thought of as follows, where each
+process performs 2n (n being the number of processes in communicator
+*comm*) independent point-to-point communications (including
+communication with itself).
+
+.. code-block:: c
+
+   	MPI_Comm_size(comm, &n);
+   	for (i = 0, i < n; i++)
+   	    MPI_Send(sendbuf + sdispls[i] * extent(sendtype),
+   	        sendcounts[i], sendtype, i, ..., comm);
+   	for (i = 0, i < n; i++)
+   	    MPI_Recv(recvbuf + rdispls[i] * extent(recvtype),
+   	        recvcounts[i], recvtype, i, ..., comm);
+
+Process j sends the k-th block of its local *sendbuf* to process k,
+which places the data in the j-th block of its local *recvbuf*.
+
+When a pair of processes exchanges data, each may pass different element
+count and datatype arguments so long as the sender specifies the same
+amount of data to send (in bytes) as the receiver expects to receive.
+
+Note that process i may send a different amount of data to process j
+than it receives from process j. Also, a process may send entirely
+different amounts of data to different processes in the communicator.
+
+WHEN COMMUNICATOR IS AN INTER-COMMUNICATOR
+
+When the communicator is an inter-communicator, the gather operation
+occurs in two phases. The data is gathered from all the members of the
+first group and received by all the members of the second group. Then
+the data is gathered from all the members of the second group and
+received by all the members of the first. The operation exhibits a
+symmetric, full-duplex behavior.
+
+The first group defines the root process. The root process uses MPI_ROOT
+as the value of *root*. All other processes in the first group use
+``MPI_PROC_NULL`` as the value of *root*. All processes in the second group
+use the rank of the root process in the first group as the value of
+*root*.
+
+When the communicator is an intra-communicator, these groups are the
+same, and the operation occurs in a single phase.
+
+
+USE OF IN-PLACE OPTION
+----------------------
+
+When the communicator is an intracommunicator, you can perform an
+all-to-all operation in-place (the output buffer is used as the input
+buffer). Use the variable MPI_IN_PLACE as the value of *sendbuf*. In
+this case, *sendcounts*, *sdispls*, and *sendtype* are ignored. The
+input data of each process is assumed to be in the area where that
+process would receive its own contribution to the receive buffer.
+
+
+NOTES
+-----
+
+The specification of counts and displacements should not cause any
+location to be written more than once.
+
+All arguments on all processes are significant. The *comm* argument, in
+particular, must describe the same communicator on all processes.
+
+The offsets of *sdispls* and *rdispls* are measured in units of
+*sendtype* and *recvtype*, respectively. Compare this to :ref:`MPI_Alltoallw`,
+where these offsets are measured in bytes.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Alltoall`
+   * :ref:`MPI_Alltoallw`
diff --git a/docs/man-openmpi/man3/MPI_Alltoallv_init.3.rst b/docs/man-openmpi/man3/MPI_Alltoallv_init.3.rst
new file mode 100644
index 00000000000..ac1831de8fd
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Alltoallv_init.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_alltoallv_init:
+
+MPI_Alltoallv_init
+==================
+    .. include_body
+
+.. include:: ../man3/MPI_Alltoallv.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Alltoallw.3.rst b/docs/man-openmpi/man3/MPI_Alltoallw.3.rst
new file mode 100644
index 00000000000..9d3597083c9
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Alltoallw.3.rst
@@ -0,0 +1,123 @@
+.. _mpi_alltoallw:
+
+
+MPI_Alltoallw
+=============
+
+.. include_body
+
+:ref:`MPI_Alltoallw`, :ref:`MPI_Ialltoallw`, :ref:`MPI_Alltoallw_init` - All processes
+send data of different types to, and receive data of different types
+from, all processes
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Alltoallw, MPI_Ialltoallw, MPI_Alltoallw_init
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_alltoallw.rst
+
+INPUT PARAMETERS
+----------------
+* ``sendbuf``: Starting address of send buffer.
+* ``sendcounts``: Integer array, where entry i specifies the number of elements to send to rank i.
+* ``sdispls``: Integer array, where entry i specifies the displacement (in bytes, offset from *sendbuf) from which to send data to rank i.*
+* ``sendtypes``: Datatype array, where entry i specifies the datatype to use when sending data to rank i.
+* ``recvcounts``: Integer array, where entry j specifies the number of elements to receive from rank j.
+* ``rdispls``: Integer array, where entry j specifies the displacement (in bytes, offset from *recvbuf) to which data from rank j should* be written.
+* ``recvtypes``: Datatype array, where entry j specifies the datatype to use when receiving data from rank j.
+* ``comm``: Communicator over which data is to be exchanged.
+* ``info``: Info (handle, persistent only)
+
+OUTPUT PARAMETERS
+-----------------
+* ``recvbuf``: Address of receive buffer.
+* ``request``: Request (handle, non-blocking and persistent only).
+* ``ierror``: Fortran only: Error status.
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Alltoallw` is a generalized collective operation in which all
+processes send data to and receive data from all other processes. It
+adds flexibility to :ref:`MPI_Alltoallv` by allowing the user to specify the
+datatype of individual data blocks (in addition to displacement and
+element count). Its operation can be thought of in the following way,
+where each process performs 2n (n being the number of processes in
+communicator *comm) independent point-to-point communications*
+(including communication with itself).
+
+.. code-block:: c
+
+   	MPI_Comm_size(comm, &n);
+   	for (i = 0, i < n; i++)
+   	    MPI_Send(sendbuf + sdispls[i], sendcounts[i],
+   	        sendtypes[i], i, ..., comm);
+   	for (i = 0, i < n; i++)
+   	    MPI_Recv(recvbuf + rdispls[i], recvcounts[i],
+   	        recvtypes[i], i, ..., comm);
+
+Process j sends the k-th block of its local *sendbuf to process* k,
+which places the data in the j-th block of its local *recvbuf.*
+
+When a pair of processes exchanges data, each may pass different element
+count and datatype arguments so long as the sender specifies the same
+amount of data to send (in bytes) as the receiver expects to receive.
+
+Note that process i may send a different amount of data to process j
+than it receives from process j. Also, a process may send entirely
+different amounts and types of data to different processes in the
+communicator.
+
+WHEN COMMUNICATOR IS AN INTER-COMMUNICATOR
+
+When the communicator is an inter-communicator, the gather operation
+occurs in two phases. The data is gathered from all the members of the
+first group and received by all the members of the second group. Then
+the data is gathered from all the members of the second group and
+received by all the members of the first. The operation exhibits a
+symmetric, full-duplex behavior.
+
+The first group defines the root process. The root process uses MPI_ROOT
+as the value of *root*. All other processes in the first group use
+``MPI_PROC_NULL`` as the value of *root*. All processes in the second group
+use the rank of the root process in the first group as the value of
+*root*.
+
+When the communicator is an intra-communicator, these groups are the
+same, and the operation occurs in a single phase.
+
+
+USE OF IN-PLACE OPTION
+----------------------
+
+When the communicator is an intracommunicator, you can perform an
+all-to-all operation in-place (the output buffer is used as the input
+buffer). Use the variable MPI_IN_PLACE as the value of *sendbuf*. In
+this case, *sendcounts*, *sdispls*, and *sendtypes* are ignored. The
+input data of each process is assumed to be in the area where that
+process would receive its own contribution to the receive buffer.
+
+
+NOTES
+-----
+
+The specification of counts, types, and displacements should not cause
+any location to be written more than once.
+
+All arguments on all processes are significant. The *comm* argument, in
+particular, must describe the same communicator on all processes.
+
+The offsets of *sdispls* and *rdispls* are measured in bytes. Compare
+this to :ref:`MPI_Alltoallv`, where these offsets are measured in units of
+*sendtype* and *recvtype*, respectively.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Alltoall`
+   * :ref:`MPI_Alltoallv`
diff --git a/docs/man-openmpi/man3/MPI_Alltoallw_init.3.rst b/docs/man-openmpi/man3/MPI_Alltoallw_init.3.rst
new file mode 100644
index 00000000000..60db1d0fcfd
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Alltoallw_init.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_alltoallw_init:
+
+MPI_Alltoallw_init
+==================
+    .. include_body
+
+.. include:: ../man3/MPI_Alltoallw.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Attr_delete.3.rst b/docs/man-openmpi/man3/MPI_Attr_delete.3.rst
new file mode 100644
index 00000000000..7b2ff393ae1
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Attr_delete.3.rst
@@ -0,0 +1,38 @@
+.. _mpi_attr_delete:
+
+
+MPI_Attr_delete
+===============
+
+.. include_body
+
+:ref:`MPI_Attr_delete` |mdash| Deletes attribute value associated with a key |mdash| |deprecated_favor| :ref:`MPI_Comm_delete_attr`.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_attr_delete.rst
+
+
+INPUT PARAMETERS
+----------------
+* ``comm``: Communicator to which attribute is attached (handle).
+* ``keyval``: The key value of the deleted attribute (integer).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Note that use of this routine is *deprecated* as of MPI-2, and was
+*deleted* in MPI-3. Please use :ref:`MPI_Comm_delete_attr`. This function does
+not have a mpi_f08 binding.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_delete_attr`
diff --git a/docs/man-openmpi/man3/MPI_Attr_get.3.rst b/docs/man-openmpi/man3/MPI_Attr_get.3.rst
new file mode 100644
index 00000000000..3074bdaff9e
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Attr_get.3.rst
@@ -0,0 +1,39 @@
+.. _mpi_attr_get:
+
+
+MPI_Attr_get
+============
+
+.. include_body
+
+:ref:`MPI_Attr_get` |mdash| Retrieves attribute value by key |mdash| |deprecated_favor| :ref:`MPI_Comm_get_attr`.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_attr_get.rst
+
+INPUT PARAMETERS
+----------------
+* ``comm``: Communicator to which attribute is attached (handle).
+* ``keyval``: Key value (integer).
+
+OUTPUT PARAMETERS
+-----------------
+* ``attribute_val``: Attribute value, unless flag = false.
+* ``flag``: True if an attribute value was extracted; false if no attribute is associated with the key.
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Note that use of this routine is *deprecated* as of MPI-2, and was
+*deleted* in MPI-3. Please use :ref:`MPI_Comm_get_attr`. This function does not
+have a mpi_f08 binding.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_get_attr`
diff --git a/docs/man-openmpi/man3/MPI_Attr_put.3.rst b/docs/man-openmpi/man3/MPI_Attr_put.3.rst
new file mode 100644
index 00000000000..afcf0357331
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Attr_put.3.rst
@@ -0,0 +1,39 @@
+.. _mpi_attr_put:
+
+
+MPI_Attr_put
+============
+
+.. include_body
+
+:ref:`MPI_Attr_put` |mdash| Stores attribute value associated with a key |mdash| |deprecated_favor| :ref:`MPI_Comm_set_attr`.
+
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_attr_put.rst
+
+INPUT PARAMETERS
+----------------
+* ``comm``: Communicator to which attribute will be attached (handle).
+* ``keyval``: Key value, as returned by MPI_KEYVAL_CREATE (integer).
+* ``attribute_val``: Attribute value.
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Note that use of this routine is *deprecated as of MPI-2, and* was
+deleted in MPI-3. Please use :ref:`MPI_Comm_set_attr`. This function does not
+have a mpi_f08 binding.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_set_attr`
diff --git a/docs/man-openmpi/man3/MPI_Barrier.3.rst b/docs/man-openmpi/man3/MPI_Barrier.3.rst
new file mode 100644
index 00000000000..f95b943a69a
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Barrier.3.rst
@@ -0,0 +1,50 @@
+.. _mpi_barrier:
+
+MPI_Barrier
+===========
+
+.. include_body
+
+:ref:`MPI_Barrier`, :ref:`MPI_Ibarrier`, :ref:`MPI_Barrier_init` - Synchronization between MPI processes in a
+group
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Barrier, MPI_Ibarrier, MPI_Barrier_init
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_barrier.rst
+
+INPUT PARAMETER
+---------------
+
+* ``comm`` : Communicator (handle).
+* ``info`` : Info (handle, persistent only).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``request`` : Request (handle, non-blocking and persistent only).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+An MPI barrier completes after all groups members have entered the
+barrier.
+
+When Communicator is an Inter-Communicator
+------------------------------------------
+
+When the communicator is an inter-communicator, the barrier operation is
+performed across all processes in both groups. All processes in the
+first group may exit the barrier when all processes in the second group
+have entered the barrier.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Bcast`
diff --git a/docs/man-openmpi/man3/MPI_Barrier_init.3.rst b/docs/man-openmpi/man3/MPI_Barrier_init.3.rst
new file mode 100644
index 00000000000..85653fba7c7
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Barrier_init.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_barrier_init:
+
+MPI_Barrier_init
+================
+    .. include_body
+
+.. include:: ../man3/MPI_Barrier.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Bcast.3.rst b/docs/man-openmpi/man3/MPI_Bcast.3.rst
new file mode 100644
index 00000000000..e1f187688b8
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Bcast.3.rst
@@ -0,0 +1,86 @@
+.. _mpi_bcast:
+
+MPI_Bcast
+=========
+
+.. include_body
+
+:ref:`MPI_Bcast`, :ref:`MPI_Ibcast`, :ref:`MPI_Bcast_init` - Broadcasts a message from the process
+with rank *root* to all other processes of the group.
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Bcast, MPI_Ibcast, MPI_Bcast_init
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_bcast.rst
+
+INPUT/OUTPUT PARAMETERS
+-----------------------
+
+* ``buffer``: Starting address of buffer (choice).
+* ``count``: Number of entries in buffer (integer).
+* ``datatype``: Data type of buffer (handle).
+* ``root``: Rank of broadcast root (integer).
+* ``comm``: Communicator (handle).
+* ``info``: Info (handle, persistent only).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``request``: Request (handle, non-blocking and persistent only).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Bcast` broadcasts a message from the process with rank root to
+all processes of the group, itself included. It is called by all members
+of group using the same arguments for ``comm``, ``root``. On return, the
+contents of root's communication buffer has been copied to all
+processes.
+
+General, derived datatypes are allowed for datatype. The type signature
+of count, datatype on any process must be equal to the type signature o
+f count, datatype at the root. This implies that the amount of data sent
+must be equal to the amount received, pairwise between each process and
+the root. :ref:`MPI_Bcast` and all other data-movement collective routines
+make this restriction. Distinct type maps between sender and receiver
+are still allowed.
+
+**Example:** Broadcast 100 ints from process 0 to every process in the
+group.
+
+.. code-block:: C
+
+   MPI_Comm comm;
+   int array[100];
+   int root=0;
+   //...
+   MPI_Bcast( array, 100, MPI_INT, root, comm);
+
+As in many of our sample code fragments, we assume that some of the
+variables (such as comm in the example above) have been assigned
+appropriate values.
+
+WHEN COMMUNICATOR IS AN INTER-COMMUNICATOR
+------------------------------------------
+
+When the communicator is an inter-communicator, the root process in the
+first group broadcasts data to all the processes in the second group.
+The first group defines the root process. That process uses MPI_ROOT
+as the value of its ``root`` argument. The remaining processes use
+``MPI_PROC_NULL`` as the value of their ``root`` argument. All processes
+in the second group use the rank of that root process in the first group
+as the value of their ``root`` argument. The receive buffer arguments of
+the processes in the second group must be consistent with the send
+buffer argument of the root process in the first group.
+
+NOTES
+-----
+
+This function does not support the in-place option.
+
+ERRORS
+------
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Bcast_init.3.rst b/docs/man-openmpi/man3/MPI_Bcast_init.3.rst
new file mode 100644
index 00000000000..266a155592e
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Bcast_init.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_bcast_init:
+
+MPI_Bcast_init
+==============
+
+.. include_body
+
+.. include:: ../man3/MPI_Bcast.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_Bsend.3.rst b/docs/man-openmpi/man3/MPI_Bsend.3.rst
new file mode 100644
index 00000000000..38c43dde4a7
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Bsend.3.rst
@@ -0,0 +1,65 @@
+.. _mpi_bsend:
+
+MPI_Bsend
+=========
+
+.. include_body
+
+:ref:`MPI_Bsend` |mdash| Basic send with user-specified buffering.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_bsend.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``buf`` : Initial address of send buffer (choice).
+* ``count`` : Number of entries in send buffer (nonnegative integer).
+* ``datatype`` : Datatype of each send buffer element (handle).
+* ``dest`` : Rank of destination (integer).
+* ``tag`` : Message tag (integer).
+* ``comm`` : Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Bsend` performs a buffered-mode, blocking send.
+
+NOTES
+-----
+
+This send is provided as a convenience function; it allows the user to
+send messages without worrying about where they are buffered (because
+the user must have provided buffer space with :ref:`MPI_Buffer_attach`).
+
+In deciding how much buffer space to allocate, remember that the buffer
+space is not available for reuse by subsequent :ref:`MPI_Bsend`\ s unless
+you are certain that the message has been received (not just that it
+should have been received). For example, this code does not allocate
+enough buffer space:
+
+.. code-block:: c
+
+   MPI_Buffer_attach( b, n*sizeof(double) + MPI_BSEND_OVERHEAD );
+   for (i=0; i<m; i++) {
+       MPI_Bsend( buf, n, MPI_DOUBLE, ... );
+   }
+
+because only enough buffer space is provided for a single send, and the
+loop may start a second ``MPI_Bsend`` before the first is done making
+use of the buffer.
+
+In C, you can force the messages to be delivered by
+``MPI_Buffer_detach( &b, &n );`` ``MPI_Buffer_attach( b, n );`` (The
+``MPI_Buffer_detach`` will not complete until all buffered messages are
+delivered.)
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Bsend_init.3.rst b/docs/man-openmpi/man3/MPI_Bsend_init.3.rst
new file mode 100644
index 00000000000..5edbdb7b7be
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Bsend_init.3.rst
@@ -0,0 +1,43 @@
+.. _mpi_bsend_init:
+
+MPI_Bsend_init
+==============
+
+.. include_body
+
+:ref:`MPI_Bsend_init` |mdash| Builds a handle for a buffered send.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_bsend_init.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``buf`` : Initial address of send buffer (choice).
+* ``count`` : Number of elements sent (integer).
+* ``datatype`` : Type of each element (handle).
+* ``dest`` : Rank of destination (integer).
+* ``tag`` : Message tag (integer).
+* ``comm`` : Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``request`` : Communication request (handle).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Creates a persistent communication request for a buffered mode send, and
+binds to it all the arguments of a send operation.
+
+A communication (send or receive) that uses a persistent request is
+initiated by the function :ref:`MPI_Start`.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso:: :ref:`MPI_Send_init` :ref:`MPI_Start`
diff --git a/docs/man-openmpi/man3/MPI_Buffer_attach.3.rst b/docs/man-openmpi/man3/MPI_Buffer_attach.3.rst
new file mode 100644
index 00000000000..dfd42ed4957
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Buffer_attach.3.rst
@@ -0,0 +1,64 @@
+.. _mpi_buffer_attach:
+
+MPI_Buffer_attach
+=================
+
+.. include_body
+
+:ref:`MPI_Buffer_attach` |mdash| Attaches a user-defined buffer for sending.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_buffer_attach.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``buf`` : Initial buffer address (choice).
+* ``size`` : Buffer size, in bytes (integer).
+
+OUTPUT PARAMETER
+----------------
+
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Provides to MPI a buffer in the user's memory to be used for buffering
+outgoing messages. The buffer is used only by messages sent in buffered
+mode. Only one buffer can be attached to a process at a time.
+
+NOTES
+-----
+
+The size given should be the sum of the sizes of all outstanding Bsends
+that you intend to have, plus MPI_BSEND_OVERHEAD bytes for each Bsend
+that you do. For the purposes of calculating size, you should use
+:ref:`MPI_Pack_size`. In other words, in the code
+
+.. code-block:: c
+
+   MPI_Buffer_attach( buf, size )
+   MPI_Bsend( ..., count=20, datatype=type1, ... );
+   MPI_Bsend( ..., count=40, datatype=type2, ... );
+
+the value of size in the :ref:`MPI_Buffer_attach` call should be greater than
+the value computed by
+
+.. code-block:: c
+
+   MPI_Pack_size( 20, type1, comm, &s1 );
+   MPI_Pack_size( 40, type2, comm, &s2 );
+   size = s1 + s2 + 2 * MPI_BSEND_OVERHEAD;
+
+MPI_BSEND_OVERHEAD gives the maximum amount of buffer space that may be
+used by the Bsend routines. This value is in mpi.h for C and mpif.h for
+Fortran.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Buffer_detach`
diff --git a/docs/man-openmpi/man3/MPI_Buffer_detach.3.rst b/docs/man-openmpi/man3/MPI_Buffer_detach.3.rst
new file mode 100644
index 00000000000..1955974288d
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Buffer_detach.3.rst
@@ -0,0 +1,87 @@
+.. _mpi_buffer_detach:
+
+MPI_Buffer_detach
+=================
+
+.. include_body
+
+:ref:`MPI_Buffer_detach` |mdash| Removes an existing buffer (for use in in :ref:`MPI_Bsend`,
+etc.)
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_buffer_detach.rst
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``buf`` : Initial buffer address (choice).
+* ``size`` : Buffer size, in bytes (integer).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Detach the buffer currently associated with MPI. The call returns the
+address and the size of the detached buffer. This operation will block
+until all messages currently in the buffer have been transmitted. Upon
+return of this function, the user may reuse or deallocate the space
+taken by the buffer.
+
+Example: Calls to attach and detach buffers.
+
+.. code-block:: c
+
+   #define BUFFSIZE 10000
+
+   int size char *buff;
+   MPI_Buffer_attach( malloc(BUFFSIZE), BUFFSIZE);
+
+   // a buffer of 10000 bytes can now be used by MPI_Bsend
+
+   MPI_Buffer_detach( &buff, &size); // Buffer size reduced to zero
+   MPI_Buffer_attach( buff, size); // Buffer of 10000 bytes available again
+
+NOTES
+-----
+
+The reason that :ref:`MPI_Buffer_detach` returns the address and size of the
+buffer being detached is to allow nested libraries to replace and
+restore the buffer. For example, consider
+
+.. code-block:: c
+
+   int size, mysize, idummy;
+   void *ptr, *myptr, *dummy;
+   MPI_Buffer_detach( &ptr, &size );
+   MPI_Buffer_attach( myptr, mysize );
+
+   /* ... library code ... */
+
+   MPI_Buffer_detach( &dummy, &idummy );
+   MPI_Buffer_attach( ptr, size );
+
+This is much like the action of the UNIX signal routine and has the same
+strengths (it's simple) and weak‐nesses (it only works for nested
+usages).
+
+For Fortran: The Fortran binding for this routine is different. Because
+Fortran does not have pointers, it is impossible to provide a way to use
+the output of this routine to exchange buffers. In this case, only the
+size field is set.
+
+For C: Even though the buf argument is declared as void, it is really
+the address of a void pointer. See Rationale, below, for more details.
+
+Even though the C functions :ref:`MPI_Buffer_attach` and :ref:`MPI_Buffer_detach` both
+have a first argument of type void*, these arguments are used
+differently: A pointer to the buffer is passed to :ref:`MPI_Buffer_attach`; the
+address of the pointer is passed to :ref:`MPI_Buffer_detach`, so that this call
+can return the pointer value.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Buffer_attach`
diff --git a/docs/man-openmpi/man3/MPI_Cancel.3.rst b/docs/man-openmpi/man3/MPI_Cancel.3.rst
new file mode 100644
index 00000000000..b667d515777
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Cancel.3.rst
@@ -0,0 +1,85 @@
+.. _mpi_cancel:
+
+MPI_Cancel
+==========
+
+.. include_body
+
+:ref:`MPI_Cancel` |mdash| Cancels a communication request.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_cancel.rst
+
+INPUT PARAMETER
+---------------
+
+* ``request`` : Communication request (handle).
+
+OUTPUT PARAMETER
+----------------
+
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The :ref:`MPI_Cancel` operation allows pending communications to be canceled.
+This is required for cleanup. Posting a send or a receive ties up user
+resources (send or receive buffers), and a cancel may be needed to free
+these resources gracefully.
+
+A call to :ref:`MPI_Cancel` marks for cancellation a pending, nonblocking
+communication operation (send or receive). The cancel call is local. It
+returns immediately, possibly before the communication is actually
+canceled. It is still necessary to complete a communication that has
+been marked for cancellation, using a call to :ref:`MPI_Request_free`,
+:ref:`MPI_Wait`, or :ref:`MPI_Test` (or any of the derived operations).
+
+If a communication is marked for cancellation, then an :ref:`MPI_Wait` call for
+that communication is guaranteed to return, irrespective of the
+activities of other processes (i.e., :ref:`MPI_Wait` behaves as a local
+function); similarly if :ref:`MPI_Test` is repeatedly called in a busy wait
+loop for a canceled communication, then :ref:`MPI_Test` will eventually be
+successful.
+
+:ref:`MPI_Cancel` can be used to cancel a communication that uses a persistent
+request (see Section 3.9 in the `MPI Standard <https://www.mpi-forum.org/docs/>`_, "Persistent
+Communication Requests") in the same way it is used for nonpersistent
+requests. A successful cancellation cancels the active communication,
+but not the request itself. After the call to :ref:`MPI_Cancel` and the
+subsequent call to :ref:`MPI_Wait` or :ref:`MPI_Test`, the request becomes inactive
+and can be activated for a new communication.
+
+The successful cancellation of a buffered send frees the buffer space
+occupied by the pending message.
+
+Either the cancellation succeeds or the communication succeeds, but not
+both. If a send is marked for cancellation, then it must be the case
+that either the send completes normally, in which case the message sent
+is received at the destination process, or that the send is successfully
+canceled, in which case no part of the message is received at the
+destination. Then, any matching receive has to be satisfied by another
+send. If a receive is marked for cancellation, then it must be the case
+that either the receive completes normally, or that the receive is
+successfully canceled, in which case no part of the receive buffer is
+altered. Then, any matching send has to be satisfied by another receive.
+
+If the operation has been canceled, then information to that effect will
+be returned in the status argument of the operation that completes the
+communication.
+
+NOTES
+-----
+
+The primary expected use of :ref:`MPI_Cancel` is in multi-buffering schemes,
+where speculative MPI_Irecvs are made. When the computation completes,
+some of these requests may remain; using :ref:`MPI_Cancel` allows the user to
+cancel these unsatisfied requests.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Probe`
diff --git a/docs/man-openmpi/man3/MPI_Cart_coords.3.rst b/docs/man-openmpi/man3/MPI_Cart_coords.3.rst
new file mode 100644
index 00000000000..1c3a4d73470
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Cart_coords.3.rst
@@ -0,0 +1,39 @@
+.. _mpi_cart_coords:
+
+MPI_Cart_coords
+===============
+
+.. include_body
+
+:ref:`MPI_Cart_coords` |mdash| Determines process coords in Cartesian topology
+given rank in group.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_cart_coords.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``comm`` : Communicator with Cartesian structure (handle).
+* ``rank`` : Rank of a process within group of comm (integer).
+* ``maxdims`` : Length of vector coords in the calling program
+   (integer). Length of vector coords in the calling program (integer).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``coords`` : Integer array (of size ndims,which was defined by
+   :ref:`MPI_Cart_create` call) containing the Cartesian coordinates of
+   specified process (integer).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Cart_coords` provides a mapping of ``rank``\ s to Cartesian
+coordinates.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Cart_create.3.rst b/docs/man-openmpi/man3/MPI_Cart_create.3.rst
new file mode 100644
index 00000000000..52140955205
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Cart_create.3.rst
@@ -0,0 +1,48 @@
+.. _mpi_cart_create:
+
+MPI_Cart_create
+===============
+
+.. include_body
+
+:ref:`MPI_Cart_create` |mdash| Makes a new communicator to which Cartesian
+topology information has been attached.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_cart_create.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``comm_old`` : Input communicator (handle).
+* ``ndims`` : Number of dimensions of Cartesian grid (integer).
+* ``dims`` : Integer array of size ndims specifying the number of
+   processes in each dimension.
+* ``periods`` : Logical array of size ndims specifying whether the grid
+   is periodic (true) or not (false) in each dimension.
+* ``reorder`` : Ranking may be reordered (true) or not (false)
+   (logical).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``comm_cart`` : Communicator with new Cartesian topology (handle).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Cart_create` returns a handle to a new communicator to which the
+Cartesian topology information is attached. If ``reorder`` = false then
+the rank of each process in the new group is identical to its rank in
+the old group. Otherwise, the function may ``reorder`` the processes
+(possibly so as to choose a good embedding of the virtual topology onto
+the physical machine). If the total size of the Cartesian grid is
+smaller than the size of the group of comm, then some processes are
+returned MPI_COMM_NULL, in analogy to :ref:`MPI_Comm_split`. The call
+is erroneous if it specifies a grid that is larger than the group size.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Cart_get.3.rst b/docs/man-openmpi/man3/MPI_Cart_get.3.rst
new file mode 100644
index 00000000000..1fda549d97e
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Cart_get.3.rst
@@ -0,0 +1,45 @@
+.. _mpi_cart_get:
+
+MPI_Cart_get
+============
+
+.. include_body
+
+:ref:`MPI_Cart_get` |mdash| Retrieves Cartesian topology information associated with
+a communicator.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_cart_get.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``comm`` : Communicator with Cartesian structure (handle).
+* ``maxdims`` : Length of vectors dims, periods, and coords in the calling
+   program (integer).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``dims`` : Number of processes for each Cartesian dimension (array of
+   integers).
+* ``periods`` : Periodicity (true/false) for each Cartesian dimension
+   (array of logicals).
+* ``coords`` : Coordinates of calling process in Cartesian structure (array
+   of integers).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The functions :ref:`MPI_Cartdim_get` and :ref:`MPI_Cart_get` return the Cartesian
+topology information that was associated with a communicator by
+:ref:`MPI_Cart_create`.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Cartdim_get`
diff --git a/docs/man-openmpi/man3/MPI_Cart_map.3.rst b/docs/man-openmpi/man3/MPI_Cart_map.3.rst
new file mode 100644
index 00000000000..ba591b0bbed
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Cart_map.3.rst
@@ -0,0 +1,47 @@
+.. _mpi_cart_map:
+
+MPI_Cart_map
+============
+
+.. include_body
+
+:ref:`MPI_Cart_map` |mdash| Maps process to Cartesian topology information.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_cart_map.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``comm`` : Input communicator (handle).
+* ``ndims`` : Number of dimensions of Cartesian structure (integer).
+* ``dims`` : Integer array of size ndims specifying the number of processes
+   in each coordinate direction.
+* ``periods`` : Logical array of size ndims specifying the periodicity
+   specification in each coordinate direction.
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``newrank`` : Reordered rank of the calling process; MPI_UNDEFINED if
+   calling process does not belong to grid (integer).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Cart_map` and :ref:`MPI_Graph_map` can be used to implement all other
+topology functions. In general they will not be called by the user
+directly, unless he or she is creating additional virtual topology
+capability other than that provided by MPI. :ref:`MPI_Cart_map` computes an
+"optimal" placement for the calling process on the physical machine. A
+possible implementation of this function is to always return the rank of
+the calling process, that is, not to perform any reordering.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Graph_map`
diff --git a/docs/man-openmpi/man3/MPI_Cart_rank.3.rst b/docs/man-openmpi/man3/MPI_Cart_rank.3.rst
new file mode 100644
index 00000000000..8c1c3676d99
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Cart_rank.3.rst
@@ -0,0 +1,45 @@
+.. _mpi_cart_rank:
+
+MPI_Cart_rank
+=============
+
+.. include_body
+
+:ref:`MPI_Cart_rank` |mdash| Determines process rank in communicator given Cartesian
+location.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_cart_rank.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``comm`` : Communicator with Cartesian structure (handle).
+* ``coords`` : Integer array (of size ndims, which was defined by
+   :ref:`MPI_Cart_create` call) specifying the Cartesian coordinates of a
+   process.
+
+OUTPUT PARAMETER
+----------------
+
+* ``rank`` : Rank of specified process (integer).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+For a process group with Cartesian structure, the function :ref:`MPI_Cart_rank`
+translates the logical process coordinates to process ranks as they are
+used by the point-to-point routines. For dimension i with periods(i) =
+true, if the coordinate, coords(i), is out of range, that is, coords(i)
+< 0 or coords(i) >= dims(i), it is shifted back to the interval 0 =<
+coords(i) < dims(i) automatically. Out-of-range coordinates are
+erroneous for nonperiodic dimensions.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Cart_create`
diff --git a/docs/man-openmpi/man3/MPI_Cart_shift.3.rst b/docs/man-openmpi/man3/MPI_Cart_shift.3.rst
new file mode 100644
index 00000000000..689da6f10ee
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Cart_shift.3.rst
@@ -0,0 +1,85 @@
+.. _mpi_cart_shift:
+
+MPI_Cart_shift
+==============
+
+.. include_body
+
+:ref:`MPI_Cart_shift` |mdash| Returns the shifted source and destination ranks,
+given a shift direction and amount.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_cart_shift.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``comm`` : Communicator with Cartesian structure (handle).
+* ``direction`` : Coordinate dimension of shift (integer).
+* ``disp`` : Displacement ( > 0: upward shift, < 0: downward shift)
+   (integer).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``rank_source`` : Rank of source process (integer).
+* ``rank_dest`` : Rank of destination process (integer).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+If the process topology is a Cartesian structure, an :ref:`MPI_Sendrecv`
+operation is likely to be used along a coordinate ``direction`` to
+perform a shift of data. As input, :ref:`MPI_Sendrecv` takes the rank of a
+source process for the receive, and the rank of a destination process
+for the send. If the function :ref:`MPI_Cart_shift` is called for a
+Cartesian process group, it provides the calling process with the above
+identifiers, which then can be passed to :ref:`MPI_Sendrecv`. The user
+specifies the coordinate ``direction`` and the size of the step
+(positive or negative). The function is local.
+
+The ``direction`` argument indicates the dimension of the shift, i.e.,
+the coordinate whose value is modified by the shift. The coordinates are
+numbered from 0 to ndims-1, where ndims is the number of dimensions.
+
+Note: The ``direction`` argument is in the range [0, n-1] for an
+n-dimensional Cartesian mesh.
+
+Depending on the periodicity of the Cartesian group in the specified
+coordinate ``direction``, :ref:`MPI_Cart_shift` provides the identifiers
+for a circular or an end-off shift. In the case of an end-off shift, the
+value ``MPI_PROC_NULL`` may be returned in ``rank_source`` or
+``rank_dest``, indicating that the source or the destination for the
+shift is out of range.
+
+Example: The ``comm``\ unicator, ``comm``, has a two-dimensional,
+periodic, Cartesian topology associated with it. A two-dimensional array
+of REALs is stored one element per process, in variable A. One wishes to
+skew this array, by shifting column i (vertically, i.e., along the
+column) by i steps.
+
+.. code-block:: fortran
+
+   ! find process rank
+       CALL MPI_COMM_RANK(comm, rank, ierr)
+   ! find Cartesian coordinates
+       CALL MPI_CART_COORDS(comm, rank, maxdims, coords, ierr)
+   ! compute shift source and destination
+       CALL MPI_CART_SHIFT(comm, 0, coords(2), source, dest, ierr)
+   ! skew array
+       CALL MPI_SENDRECV_REPLACE(A, 1, MPI_REAL, dest, 0, source, 0, comm, status,
+                                 ierr)
+
+NOTE
+----
+
+In Fortran, the dimension indicated by DIRECTION = i has DIMS(i+1)
+nodes, where DIMS is the array that was used to create the grid. In C,
+the dimension indicated by direction = i is the dimension specified by
+dims[i].
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Cart_sub.3.rst b/docs/man-openmpi/man3/MPI_Cart_sub.3.rst
new file mode 100644
index 00000000000..430eeffb184
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Cart_sub.3.rst
@@ -0,0 +1,58 @@
+.. _mpi_cart_sub:
+
+MPI_Cart_sub
+============
+
+.. include_body
+
+:ref:`MPI_Cart_sub` |mdash| Partitions a communicator into subgroups, which form
+lower-dimensional Cartesian subgrids.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_cart_sub.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``comm`` : Communicator with Cartesian structure (handle).
+* ``remain_dims`` : The ith entry of remain_dims specifies whether the ith
+   dimension is kept in the subgrid (true) or is dropped (false)
+   (logical vector).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``comm_new`` : Communicator containing the subgrid that includes the
+   calling process (handle).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+If a Cartesian topology has been created with :ref:`MPI_Cart_create`, the
+function :ref:`MPI_Cart_sub` can be used to partition the communicator group
+into subgroups that form lower-dimensional Cartesian subgrids, and to
+build for each subgroup a communicator with the associated subgrid
+Cartesian topology. (This function is closely related to
+:ref:`MPI_Comm_split`.)
+
+Example: Assume that ``MPI_Cart_create( ..., comm)`` has defined a (2 x 3 x
+4) grid. Let remain_dims = (true, false, true). Then a call to
+
+.. code-block:: c
+
+   MPI_Cart_sub(comm, remain_dims, comm_new);
+
+will create three communicators, each with eight processes in a 2 x 4
+Cartesian topology. If remain_dims = (false, false, true) then the call
+to MPI_Cart_sub(comm, remain_dims, comm_new) will create six
+nonoverlapping communicators, each with four processes, in a
+one-dimensional Cartesian topology.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Cart_create`
diff --git a/docs/man-openmpi/man3/MPI_Cartdim_get.3.rst b/docs/man-openmpi/man3/MPI_Cartdim_get.3.rst
new file mode 100644
index 00000000000..fd179f2bc7c
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Cartdim_get.3.rst
@@ -0,0 +1,37 @@
+.. _mpi_cartdim_get:
+
+MPI_Cartdim_get
+===============
+
+.. include_body
+
+:ref:`MPI_Cartdim_get` |mdash| Retrieves Cartesian topology information associated
+with a communicator.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_cartdim_get.rst
+
+INPUT PARAMETER
+---------------
+
+* ``comm`` : Communicator with Cartesian structure (handle).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``ndims`` : Number of dimensions of the Cartesian structure (integer).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Cartdim_get` returns the number of dimensions of the Cartesian
+structure.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Cart_get`
diff --git a/docs/man-openmpi/man3/MPI_Close_port.3.rst b/docs/man-openmpi/man3/MPI_Close_port.3.rst
new file mode 100644
index 00000000000..9af1b006fa0
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Close_port.3.rst
@@ -0,0 +1,32 @@
+.. _mpi_close_port:
+
+MPI_Close_port
+==============
+
+.. include_body
+
+:ref:`MPI_Close_port` |mdash| Releases the specified network address.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_close_port.rst
+
+INPUT PARAMETER
+---------------
+
+* ``port_name`` : A port (string).
+
+OUTPUT PARAMETER
+----------------
+
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Close_port` releases the network address represented by
+``port_name``.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Comm_accept.3.rst b/docs/man-openmpi/man3/MPI_Comm_accept.3.rst
new file mode 100644
index 00000000000..ab49e6e4ddc
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_accept.3.rst
@@ -0,0 +1,44 @@
+.. _mpi_comm_accept:
+
+MPI_Comm_accept
+===============
+
+.. include_body
+
+:ref:`MPI_Comm_accept` |mdash| Establishes communication with a client.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_accept.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``port_name`` : Port name (string, used only on *root*).
+* ``info`` : Options given by root for the accept (handle, used only on
+   root). No options currently supported.
+* ``root`` : Rank in *comm* of root node (integer).
+* ``comm`` : Intracommunicator over which call is collective (handle).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``newcomm`` : Intercommunicator with client as remote group (handle)
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Comm_accept` establishes communication with a client. It is
+collective over the calling communicator. It returns an
+intercommunicator that allows communication with the client, after the
+client has connected with the :ref:`MPI_Comm_accept` function using the
+:ref:`MPI_Comm_connect` function. The port_name must have been established
+through a call to :ref:`MPI_Open_port` on the root.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_connect`
diff --git a/docs/man-openmpi/man3/MPI_Comm_c2f.3.rst b/docs/man-openmpi/man3/MPI_Comm_c2f.3.rst
new file mode 100644
index 00000000000..11f0136acc1
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_c2f.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_comm_c2f:
+
+MPI_Comm_c2f
+============
+
+.. include_body
+
+.. include:: ../man3/MPI_Comm_f2c.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_Comm_call_errhandler.3.rst b/docs/man-openmpi/man3/MPI_Comm_call_errhandler.3.rst
new file mode 100644
index 00000000000..a982b426251
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_call_errhandler.3.rst
@@ -0,0 +1,46 @@
+.. _mpi_comm_call_errhandler:
+
+MPI_Comm_call_errhandler
+========================
+
+.. include_body
+
+:ref:`MPI_Comm_call_errhandler` |mdash| Passes the supplied error code to the error
+handler assigned to a communicator
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_call_errhandler.rst
+
+INPUT PARAMETER
+---------------
+
+* ``comm`` : communicator with error handler (handle).
+* ``errorcode`` : error code (integer).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This function invokes the error handler assigned to the communicator
+comm with the supplied error code errorcode. If the error handler was
+successfully called, the process is not aborted, and the error handler
+returns, this function returns MPI_SUCCESS.
+
+NOTES
+-----
+
+Users should note that the default error handler is
+MPI_ERRORS_ARE_FATAL. Thus, calling this function will abort the
+processes in comm if the default error handler has not been changed.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_create_errhandler`
diff --git a/docs/man-openmpi/man3/MPI_Comm_compare.3.rst b/docs/man-openmpi/man3/MPI_Comm_compare.3.rst
new file mode 100644
index 00000000000..14da55491b8
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_compare.3.rst
@@ -0,0 +1,38 @@
+.. _mpi_comm_compare:
+
+MPI_Comm_compare
+================
+
+.. include_body
+
+:ref:`MPI_Comm_compare` |mdash| Compares two communicators.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_compare.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``comm1`` : Comm1 (handle).
+* ``comm2`` : Comm2 (handle).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``result`` : Result of comparison (integer).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+MPI_IDENT ``result``\ s if and only if ``comm1`` and ``comm2`` are
+handles for the same object (identical groups and same contexts).
+MPI_CONGRUENT results if the underlying groups are identical in
+constituents and rank order; these communicators differ only by context.
+MPI_SIMILAR results of the group members of both communicators are
+the same but the rank order differs. MPI_UNEQUAL results otherwise.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Comm_connect.3.rst b/docs/man-openmpi/man3/MPI_Comm_connect.3.rst
new file mode 100644
index 00000000000..42fd705d5de
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_connect.3.rst
@@ -0,0 +1,51 @@
+.. _mpi_comm_connect:
+
+MPI_Comm_connect
+================
+
+.. include_body
+
+:ref:`MPI_Comm_connect` |mdash| Establishes communication with a server.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_connect.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``port_name`` : Port name (string, used only on *root*).
+* ``info`` : Options given by root for the connect (handle, used only on
+   root). No options currently supported.
+* ``root`` : Rank in *comm* of root node (integer).
+* ``comm`` : Intracommunicator over which call is collective (handle).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``newcomm`` : Intercommunicator with client as remote group (handle)
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Comm_connect` establishes communication with a server specified by
+port_name. It is collective over the calling communicator and returns an
+intercommunicator in which the remote group participated in an
+:ref:`MPI_Comm_accept`. The :ref:`MPI_Comm_connect` call must only be called after the
+:ref:`MPI_Comm_accept` call has been made by the MPI job acting as the server.
+If the named port does not exist (or has been closed), :ref:`MPI_Comm_connect`
+raises an error of class MPI_ERR_PORT. MPI provides no guarantee of
+fairness in servicing connection attempts. That is, connection attempts
+are not necessarily satisfied in the order in which they were initiated,
+and competition from other connection attempts may prevent a particular
+connection attempt from being satisfied. The port_name parameter is the
+address of the server. It must be the same as the name returned by
+:ref:`MPI_Open_port` on the server.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_accept`
diff --git a/docs/man-openmpi/man3/MPI_Comm_create.3.rst b/docs/man-openmpi/man3/MPI_Comm_create.3.rst
new file mode 100644
index 00000000000..97dcd1e2fab
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_create.3.rst
@@ -0,0 +1,55 @@
+.. _mpi_comm_create:
+
+MPI_Comm_create
+===============
+
+.. include_body
+
+:ref:`MPI_Comm_create` |mdash| Creates a new communicator.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_create.rst
+
+INPUT PARAMETER
+---------------
+
+* ``comm`` : Communicator (handle).
+* ``group`` : Group, which is a subset of the group of comm (handle).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``newcomm`` : New communicator (handle).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This function creates a new communicator newcomm with communication
+group defined by group and a new context. The function sets newcomm to a
+new communicator that spans all the processes that are in the group. It
+sets newcomm to MPI_COMM_NULL for processes that are not in the group.
+Each process must call with a group argument that is a subgroup of the
+group associated with comm; this could be MPI_GROUP_EMPTY. The processes
+may specify different values for the group argument. If a process calls
+with a non-empty group, then all processes in that group must call the
+function with the same group as argument, that is: the same processes in
+the same order. Otherwise the call is erroneous.
+
+NOTES
+-----
+
+:ref:`MPI_Comm_create` provides a means of making a subset of processes for the
+purpose of separate MIMD computation, with separate communication space.
+newcomm, which is created by :ref:`MPI_Comm_create`, can be used in subsequent
+calls to :ref:`MPI_Comm_create` (or other communicator constructors) to further
+subdivide a computation into parallel sub-computations. A more general
+service is provided by :ref:`MPI_Comm_split`.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_split`
diff --git a/docs/man-openmpi/man3/MPI_Comm_create_errhandler.3.rst b/docs/man-openmpi/man3/MPI_Comm_create_errhandler.3.rst
new file mode 100644
index 00000000000..8c4be45f59d
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_create_errhandler.3.rst
@@ -0,0 +1,59 @@
+.. _mpi_comm_create_errhandler:
+
+MPI_Comm_create_errhandler
+==========================
+
+.. include_body
+
+:ref:`MPI_Comm_create_errhandler` |mdash| Creates an error handler that can be
+attached to communicators.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_create_errhandler.rst
+
+Deprecated Type Name Note
+-------------------------
+
+MPI-2.2 deprecated the ``MPI_Comm_errhandler_fn`` type in favor of
+``MPI_Comm_errhandler_function``.
+Open MPI supports both names (indeed, the ``_fn`` names are
+typedefs to the ``_function`` names).
+
+INPUT PARAMETER
+---------------
+
+* ``function`` : User-defined error handling procedure (function).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``errhandler`` : MPI error handler (handle).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Comm_create_errhandler` creates an error handler that can be
+attached to communicators. This ``function`` is identical to
+:ref:`MPI_Errhandler_create`, the use of which is deprecated. In C, the
+user routine should be a ``function`` of type
+``MPI_Comm_errhandler_function``, which is defined as
+
+.. code-block:: c
+
+   typedef void MPI_Comm_errhandler_function(MPI_Comm *, int *, ...);
+
+The first argument is the communicator in use. The second is the error
+code to be returned by the MPI routine that raised the error. This
+typedef replaces ``MPI_Handler_function``, the use of which is
+deprecated. In Fortran, the user routine should be of this form:
+
+.. code-block:: fortran
+
+   SUBROUTINE COMM_ERRHANDLER_FUNCTION(COMM, ERROR_CODE, ...)
+      INTEGER COMM, ERROR_CODE
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Comm_create_from_group.3.rst b/docs/man-openmpi/man3/MPI_Comm_create_from_group.3.rst
new file mode 100644
index 00000000000..ab162857b4f
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_create_from_group.3.rst
@@ -0,0 +1,64 @@
+.. _mpi_comm_create_from_group:
+
+MPI_Comm_create_from_group
+==========================
+
+.. include_body
+
+:ref:`MPI_Comm_create_from_group` |mdash| Creates a new communicator from a group and
+stringtag
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_create_from_group.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``group`` : Group (handler)
+* ``stringtag`` : Unique identifier for this operation (string)
+* ``info`` : info object (handler)
+* ``errhandler`` : error handler to be attached to the new
+   intra-communicator (handle)
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``newcomm`` : New communicator (handle).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Comm_create_from_group` is similar to :ref:`MPI_Comm_create_group`, except
+that the set of MPI processes involved in the creation of the new
+intra-communicator is specified by a group argument, rather than the
+group associated with a pre-existing communicator. If a non-empty group
+is specified, then all MPI processes in that group must call the
+function and each of these MPI processes must provide the same
+arguments, including a group that contains the same members with the
+same ordering, and identical stringtag value. In the event that
+MPI_GROUP_EMPTY is supplied as the group argument, then the call is a
+local operation and MPI_COMM_NULL is returned as newcomm. The stringtag
+argument is analogous to the tag used for :ref:`MPI_Comm_create_group`. If
+multiple threads at a given MPI process perform concurrent
+:ref:`MPI_Comm_create_from_group` operations, the user must distinguish these
+operations by providing different stringtag arguments. The stringtag
+shall not exceed MPI_MAX_STRINGTAG_LEN characters in length. For C, this
+includes space for a null terminating character.
+
+NOTES
+-----
+
+The errhandler argument specifies an error handler to be attached to the
+new intracommunicator. The info argument provides hints and assertions,
+possibly MPI implementation dependent, which indicate desired
+characteristics and guide communicator creation. MPI_MAX_STRINGTAG_LEN
+shall have a value of at least 63.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_create_group`
diff --git a/docs/man-openmpi/man3/MPI_Comm_create_group.3.rst b/docs/man-openmpi/man3/MPI_Comm_create_group.3.rst
new file mode 100644
index 00000000000..93cb6ebf1f5
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_create_group.3.rst
@@ -0,0 +1,65 @@
+.. _mpi_comm_create_group:
+
+MPI_Comm_create_group
+=====================
+
+.. include_body
+
+:ref:`MPI_Comm_create_group` |mdash| Creates a new communicator.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_create_group.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``comm`` : Communicator (handle).
+* ``group`` : Group, which is a subset of the group of comm (handle).
+* ``tag`` : Tag (integer).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``newcomm`` : New communicator (handle).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Comm_create_group` is similar to :ref:`MPI_Comm_create`; however,
+:ref:`MPI_Comm_create` must be called by all processes in the group of comm,
+whereas :ref:`MPI_Comm_create_group` must be called by all processes in group,
+which is a subgroup of the group of comm. In addition,
+:ref:`MPI_Comm_create_group` requires that comm is an intracommunicator.
+:ref:`MPI_Comm_create_group` returns a new intracommunicator, newcomm, for
+which the group argument defines the communication group. No cached
+information propagates from comm to newcomm. Each process must provide a
+group argument that is a subgroup of the group associated with comm;
+this could be MPI_GROUP_EMPTY. If a non-empty group is specified, then
+all processes in that group must call the function, and each of these
+processes must provide the same arguments, including a group that
+contains the same members with the same ordering. Otherwise the call is
+erroneous. If the calling process is a member of the group given as the
+group argument, then newcomm is a communicator with group as its
+associated group. If the calling process is not a member of group, e.g.,
+group is MPI_GROUP_EMPTY, then the call is a local operation and
+MPI_COMM_NULL is returned as newcomm.
+
+NOTES
+-----
+
+:ref:`MPI_Comm_create_group` provides a means of making a subset of processes
+for the purpose of separate MIMD computation, with separate
+communication space. newcomm, which is created by :ref:`MPI_Comm_create_group`,
+can be used in subsequent calls to :ref:`MPI_Comm_create_group` (or other
+communicator constructors) to further subdivide a computation into
+parallel sub-computations. A more general service is provided by
+:ref:`MPI_Comm_split`.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_create`
diff --git a/docs/man-openmpi/man3/MPI_Comm_create_keyval.3.rst b/docs/man-openmpi/man3/MPI_Comm_create_keyval.3.rst
new file mode 100644
index 00000000000..9af3e36701e
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_create_keyval.3.rst
@@ -0,0 +1,80 @@
+.. _mpi_comm_create_keyval:
+
+MPI_Comm_create_keyval
+======================
+
+.. include_body
+
+:ref:`MPI_Comm_create_keyval` |mdash| Generates a new attribute key.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_create_keyval.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``comm_copy_attr_fn`` : Copy callback function for ``comm_keyval``
+   (function).
+* ``comm_delete_attr_fn`` : Delete callback function for
+   ``comm_keyval`` (function).
+* ``extra_state`` : Extra state for callback functions.
+
+OUTPUT PARAMETER
+----------------
+
+* ``comm_keyval`` : Key value for future access (integer).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This function replaces :ref:`MPI_Keyval_create`, the use of which is
+deprecated. The C binding is identical. The Fortran binding differs in
+that ``extra_state`` is an address-sized integer. Also, the copy and
+delete callback functions have Fortran bindings that are consistent with
+address-sized attributes. The argument ``comm_copy_attr_fn`` may be
+specified as ``MPI_COMM_NULL_COPY_FN`` or ``MPI_COMM_DUP_FN`` from C or
+Fortran. ``MPI_COMM_NULL_COPY_FN`` is a function that does nothing more
+than returning ``flag = 0`` and ``MPI_SUCCESS``. ``MPI_COMM_DUP_FN`` is
+a simple-minded copy function that sets ``flag = 1``, returns the value
+of ``attribute_val_in`` in ``attribute_val_out``, and returns
+``MPI_SUCCESS``. These replace the MPI-1 predefined callbacks
+``MPI_NULL_COPY_FN`` and ``MPI_DUP_FN``, the use of which is deprecated.
+The two C callback functions are:
+
+.. code-block:: c
+
+   typedef int MPI_Comm_copy_attr_function(MPI_Comm oldcomm, int comm_keyval,
+                void *extra_state, void *attribute_val_in,
+                void *attribute_val_out, int *flag);
+
+   typedef int MPI_Comm_delete_attr_function(MPI_Comm comm, int comm_keyval,
+                void *attribute_val, void *extra_state);
+
+which are the same as the MPI-1.1 calls but with a new name. The old
+names are deprecated. The two Fortran callback functions are:
+
+.. code-block:: fortran
+
+   SUBROUTINE COMM_COPY_ATTR_FN(OLDCOMM, COMM_KEYVAL, EXTRA_STATE,
+                ATTRIBUTE_VAL_IN, ATTRIBUTE_VAL_OUT, FLAG, IERROR)
+       INTEGER OLDCOMM, COMM_KEYVAL, IERROR
+
+       INTEGER(KIND=MPI_ADDRESS_KIND) EXTRA_STATE, ATTRIBUTE_VAL_IN,
+           ATTRIBUTE_VAL_OUT
+       LOGICAL FLAG
+
+   SUBROUTINE COMM_DELETE_ATTR_FN(COMM, COMM_KEYVAL, ATTRIBUTE_VAL, EXTRA_STATE,
+                IERROR)
+       INTEGER COMM, COMM_KEYVAL, IERROR
+
+       INTEGER(KIND=MPI_ADDRESS_KIND) ATTRIBUTE_VAL, EXTRA_STATE
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_free_keyval`
diff --git a/docs/man-openmpi/man3/MPI_Comm_delete_attr.3.rst b/docs/man-openmpi/man3/MPI_Comm_delete_attr.3.rst
new file mode 100644
index 00000000000..e7c97cafd5c
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_delete_attr.3.rst
@@ -0,0 +1,53 @@
+.. _mpi_comm_delete_attr:
+
+MPI_Comm_delete_attr
+====================
+
+.. include_body
+
+:ref:`MPI_Comm_delete_attr` |mdash| Deletes attribute value associated with a
+key.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_delete_attr.rst
+
+INPUT PARAMETER
+---------------
+
+* ``comm_keyval`` : Key value (integer).
+
+OUTPUT PARAMETER
+----------------
+
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Comm_delete_attr` deletes an attribute from cache by key. This
+function invokes the attribute delete function ``delete_fn`` specified
+when the ``comm_keyval`` was created. The call will fail if the
+``delete_fn`` function returns an error code other than ``MPI_SUCCESS``.
+
+Whenever a communicator is replicated using the function
+:ref:`MPI_Comm_dup`, all callback copy functions for attributes that are
+currently set are invoked (in arbitrary order). Whenever a communicator
+is deleted using the function :ref:`MPI_Comm_free`, all callback delete
+functions for attributes that are currently set are invoked. This
+function is the same as :ref:`MPI_Attr_delete` but is needed to match the
+``comm``\ unicator-specific functions introduced in the MPI-2 standard.
+The use of :ref:`MPI_Attr_delete` is deprecated.
+
+NOTES
+-----
+
+Note that it is not defined by the MPI standard what happens if the
+``delete_fn`` callback invokes other MPI functions. In Open MPI, it is
+not valid for ``delete_fn`` callbacks (or any of their children) to add
+or delete attributes on the same object on which the ``delete_fn``
+callback is being invoked.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Comm_disconnect.3.rst b/docs/man-openmpi/man3/MPI_Comm_disconnect.3.rst
new file mode 100644
index 00000000000..2807495470c
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_disconnect.3.rst
@@ -0,0 +1,48 @@
+.. _mpi_comm_disconnect:
+
+MPI_Comm_disconnect
+===================
+
+.. include_body
+
+:ref:`MPI_Comm_disconnect` |mdash| Deallocates communicator object and sets handle to
+MPI_COMM_NULL.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_disconnect.rst
+
+OUTPUT PARAMETER
+----------------
+
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Comm_disconnect` waits for all pending communication on comm to
+complete internally, deallocates the communicator object, and sets the
+handle to MPI_COMM_NULL. It is a collective operation. It may not be
+called with the communicator MPI_COMM_WORLD or MPI_COMM_SELF.
+:ref:`MPI_Comm_disconnect` may be called only if all communication is complete
+and matched, so that buffered data can be delivered to its destination.
+This requirement is the same as for :ref:`MPI_Finalize`. :ref:`MPI_Comm_disconnect`
+has the same action as :ref:`MPI_Comm_free`, except that it waits for pending
+communication to finish internally and enables the guarantee about the
+behavior of disconnected processes.
+
+NOTES
+-----
+
+To disconnect two processes you may need to call :ref:`MPI_Comm_disconnect`,
+:ref:`MPI_Win_free`, and :ref:`MPI_File_close` to remove all communication paths
+between the two processes. Note that it may be necessary to disconnect
+several communicators (or to free several windows or files) before two
+processes are completely independent.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_connect`
diff --git a/docs/man-openmpi/man3/MPI_Comm_dup.3.rst b/docs/man-openmpi/man3/MPI_Comm_dup.3.rst
new file mode 100644
index 00000000000..3877a7e72d6
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_dup.3.rst
@@ -0,0 +1,63 @@
+.. _mpi_comm_dup:
+
+MPI_Comm_dup
+============
+
+.. include_body
+
+:ref:`MPI_Comm_dup` |mdash| Duplicates an existing communicator with all its cached
+information.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_dup.rst
+
+INPUT PARAMETER
+---------------
+
+* ``comm`` : Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``newcomm`` : Copy of comm (handle).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Comm_dup` duplicates the existing communicator comm with associated
+key values. For each key value, the respective copy callback function
+determines the attribute value associated with this key in the new
+communicator; one particular action that a copy callback may take is to
+delete the attribute from the new communicator. Returns in newcomm a new
+communicator with the same group, any copied cached information, but a
+new context (see the "Functionality" subsection of the "Caching"
+section in the "Groups, Contexts, and Communicators" chapter in the
+`MPI Standard <https://www.mpi-forum.org/docs/>`_).
+
+
+NOTES
+-----
+
+This operation is used to provide a parallel library call with a
+duplicate communication space that has the same properties as the
+original communicator. This includes any attributes (see below) and
+topologies (see the "Process Topologies" chapter in the `MPI Standard
+<https://www.mpi-forum.org/docs/>`_).
+This call is valid even if there are pending point-to-point
+communications involving the communicator comm. A typical call might
+involve an :ref:`MPI_Comm_dup` at the beginning of the parallel call, and an
+:ref:`MPI_Comm_free` of that duplicated communicator at the end of the call.
+Other models of communicator management are also possible. This call
+applies to both intra- and intercommunicators. Note that it is not
+defined by the MPI standard what happens if the attribute copy callback
+invokes other MPI functions. In Open MPI, it is not valid for attribute
+copy callbacks (or any of their children) to add or delete attributes on
+the same object on which the attribute copy callback is being invoked.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso:: :ref:`MPI_Comm_dup_with_info`
diff --git a/docs/man-openmpi/man3/MPI_Comm_dup_with_info.3.rst b/docs/man-openmpi/man3/MPI_Comm_dup_with_info.3.rst
new file mode 100644
index 00000000000..3dd55cf211c
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_dup_with_info.3.rst
@@ -0,0 +1,59 @@
+.. _mpi_comm_dup_with_info:
+
+MPI_Comm_dup_with_info
+======================
+
+.. include_body
+
+:ref:`MPI_Comm_dup_with_info` |mdash| Duplicates an existing communicator using
+provided info.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_dup_with_info.rst
+
+INPUT PARAMETER
+---------------
+
+* ``comm`` : Communicator (handle).
+* ``info`` : Info argument (handle).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``newcomm`` : Copy of comm (handle).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Comm_dup_with_info` acts exactly like :ref:`MPI_Comm_dup` except that the
+info hints associated with the communicator comm are not duplicated in
+newcomm. The hints provided by the argument info are associated with the
+output communicator newcomm instead. See :ref:`MPI_Comm_set_info` for the
+list of recognized info keys.
+
+NOTES
+-----
+
+This operation is used to provide a parallel library call with a
+duplicate communication space that has the same properties as the
+original communicator. This includes any attributes (see below) and
+topologies (see the "Process Topologies" chapter in the `MPI Standard
+<https://www.mpi-forum.org/docs/>`_.
+This call is valid even if there are pending point-to-point
+communications involving the communicator comm. A typical call might
+involve an :ref:`MPI_Comm_dup_with_info` at the beginning of the parallel call,
+and an :ref:`MPI_Comm_free` of that duplicated communicator at the end of the
+call. Other models of communicator management are also possible. This
+call applies to both intra- and intercommunicators. Note that it is not
+defined by the MPI standard what happens if the attribute copy callback
+invokes other MPI functions. In Open MPI, it is not valid for attribute
+copy callbacks (or any of their children) to add or delete attributes on
+the same object on which the attribute copy callback is being invoked.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso:: :ref:`MPI_Comm_dup`
diff --git a/docs/man-openmpi/man3/MPI_Comm_f2c.3.rst b/docs/man-openmpi/man3/MPI_Comm_f2c.3.rst
new file mode 100644
index 00000000000..ac1b3c1600d
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_f2c.3.rst
@@ -0,0 +1,65 @@
+.. _mpi_comm_f2c:
+
+
+MPI_Comm_f2c
+============
+
+.. include_body
+
+:ref:`MPI_Comm_f2c`, :ref:`MPI_Comm_c2f`,
+:ref:`MPI_Errhandler_f2c`, :ref:`MPI_Errhandler_c2f`,
+:ref:`MPI_File_f2c`, :ref:`MPI_File_c2f`,
+:ref:`MPI_Group_f2c`, :ref:`MPI_Group_c2f`,
+:ref:`MPI_Info_f2c`, :ref:`MPI_Info_c2f`,
+:ref:`MPI_Message_f2c`, :ref:`MPI_Message_c2f`,
+:ref:`MPI_Op_f2c`, :ref:`MPI_Op_c2f`,
+:ref:`MPI_Request_f2c`, :ref:`MPI_Request_c2f`,
+:ref:`MPI_Session_f2c`, :ref:`MPI_Session_c2f`,
+:ref:`MPI_Status_f2c`, :ref:`MPI_Status_c2f`,
+:ref:`MPI_Type_f2c`, :ref:`MPI_Type_c2f`,
+:ref:`MPI_Win_f2c`, :ref:`MPI_Win_c2f` - Translates a C handle into a Fortran
+handle, or vice versa.
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Comm_f2c, MPI_Comm_c2f
+.. mpi-bindings: MPI_Errhandler_f2c, MPI_Errhandler_c2f
+.. mpi-bindings: MPI_File_f2c, MPI_File_c2f
+.. mpi-bindings: MPI_Group_f2c, MPI_Group_c2f
+.. mpi-bindings: MPI_Info_f2c, MPI_Info_c2f
+.. mpi-bindings: MPI_Message_f2c, MPI_Message_c2f
+.. mpi-bindings: MPI_Op_f2c, MPI_Op_c2f
+.. mpi-bindings: MPI_Request_f2c, MPI_Request_c2f
+.. mpi-bindings: MPI_Session_f2c, MPI_Session_c2f
+.. mpi-bindings: MPI_Type_f2c, MPI_Type_c2f
+.. mpi-bindings: MPI_Win_f2c, MPI_Win_c2f
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_f2c.rst
+
+DESCRIPTION
+-----------
+
+Handles are passed between Fortran and C by using an explicit C wrapper
+to convert Fortran handles to C handles. There is no direct access to C
+handles in Fortran. The type definition MPI_Fint is provided in C for
+an integer of the size that matches a Fortran *INTEGER*; usually,
+MPI_Fint will be equivalent to *int*. The handle translation functions
+are provided in C to convert from a Fortran handle (which is an integer)
+to a C handle, and vice versa.
+
+For example, if *comm* is a valid Fortran handle to a communicator, then
+:ref:`MPI_Comm_f2c` returns a valid C handle to that same communicator; if
+*comm* = MPI_COMM_NULL (Fortran value), then :ref:`MPI_Comm_f2c` returns a null
+C handle; if *comm* is an invalid Fortran handle, then :ref:`MPI_Comm_f2c`
+returns an invalid C handle.
+
+
+NOTES
+-----
+
+These functions are only available in C; they are not available in any
+of the Fortran MPI interfaces.
+
+This function does not return an error value. Consequently, the result
+of calling it before :ref:`MPI_Init` or after :ref:`MPI_Finalize` is undefined.
diff --git a/docs/man-openmpi/man3/MPI_Comm_free.3.rst b/docs/man-openmpi/man3/MPI_Comm_free.3.rst
new file mode 100644
index 00000000000..bff3959e525
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_free.3.rst
@@ -0,0 +1,52 @@
+.. _mpi_comm_free:
+
+
+MPI_Comm_free
+=============
+
+.. include_body
+
+:ref:`MPI_Comm_free` |mdash| Mark a communicator object for deallocation.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_free.rst
+
+INPUT PARAMETER
+---------------
+* ``comm``: Communicator to be destroyed (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This operation marks the communicator object for deallocation. The
+handle is set to MPI_COMM_NULL. Any pending operations that use this
+communicator will complete normally; the object is actually deallocated
+only if there are no other active references to it. This call applies to
+intracommunicators and intercommunicators. Upon actual deallocation, the
+delete callback functions for all cached attributes (see the "Caching"
+section in the "Groups, Contexts, and Communicators" chapter in the
+`MPI Standard <https://www.mpi-forum.org/docs/>`_) are called in
+arbitrary order.
+
+
+NOTES
+-----
+
+Note that it is not defined by the MPI standard what happens if the
+delete_fn callback invokes other MPI functions. In Open MPI, it is not
+valid for delete_fn callbacks (or any of their children) to add or
+delete attributes on the same object on which the delete_fn callback is
+being invoked.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_delete_attr`
diff --git a/docs/man-openmpi/man3/MPI_Comm_free_keyval.3.rst b/docs/man-openmpi/man3/MPI_Comm_free_keyval.3.rst
new file mode 100644
index 00000000000..d30316f240c
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_free_keyval.3.rst
@@ -0,0 +1,52 @@
+.. _mpi_comm_free_keyval:
+
+
+MPI_Comm_free_keyval
+====================
+
+.. include_body
+
+:ref:`MPI_Comm_free_keyval` |mdash| Frees attribute key for communicator cache
+attribute.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_free_keyval.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``comm_keyval``:
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Comm_free_keyval` frees an extant attribute key. This function sets
+the value of *keyval* to MPI_KEYVAL_INVALID. Note that it is not
+erroneous to free an attribute key that is in use, because the actual
+free does not transpire until after all references (in other
+communicators on the process) to the key have been freed. These
+references need to be explicitly freed by the program, either via calls
+to :ref:`MPI_Comm_delete_attr` that free one attribute instance, or by calls to
+:ref:`MPI_Comm_free` that free all attribute instances associated with the
+freed communicator.
+
+This call is identical to the call :ref:`MPI_Keyval_free` but is needed to
+match the communicator-specific creation function introduced in the
+MPI-2 standard. The use of :ref:`MPI_Keyval_free` is deprecated.
+
+
+NOTES
+-----
+
+Key values are global (they can be used with any and all communicators).
+
+
+ERRORS
+------
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_create_keyval`
diff --git a/docs/man-openmpi/man3/MPI_Comm_get_attr.3.rst b/docs/man-openmpi/man3/MPI_Comm_get_attr.3.rst
new file mode 100644
index 00000000000..f3d67a5f15f
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_get_attr.3.rst
@@ -0,0 +1,42 @@
+.. _mpi_comm_get_attr:
+
+
+MPI_Comm_get_attr
+=================
+
+.. include_body
+
+:ref:`MPI_Comm_get_attr` |mdash| Retrieves attribute value by key.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_get_attr.rst
+
+INPUT PARAMETERS
+----------------
+* ``comm``: Communicator to which the attribute is attached (handle).
+* ``comm_keyval``: Key value (integer).
+
+OUTPUT PARAMETER
+----------------
+* ``attribute_val``: Attribute value, unless f\ *lag* = false.
+* ``flag``: False if no attribute is associated with the key (logical).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Comm_get_attr` retrieves an attribute value by key. The call is
+erroneous if there is no key with value *keyval*. On the other hand, the
+call is correct if the key value exists, but no attribute is attached on
+*comm* for that key; in that case, the call returns *flag* = false. In
+particular, ``MPI_KEYVAL_INVALID`` is an erroneous key value.
+
+This function replaces :ref:`MPI_Attr_get`, the use of which is deprecated. The
+C binding is identical. The Fortran binding differs in that
+*attribute_val* is an address-sized integer.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Comm_get_errhandler.3.rst b/docs/man-openmpi/man3/MPI_Comm_get_errhandler.3.rst
new file mode 100644
index 00000000000..c04b0427523
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_get_errhandler.3.rst
@@ -0,0 +1,48 @@
+.. _mpi_comm_get_errhandler:
+
+
+MPI_Comm_get_errhandler
+=======================
+
+.. include_body
+
+:ref:`MPI_Comm_get_errhandler` |mdash| Retrieves error handler associated with a
+communicator.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_get_errhandler.rst
+
+INPUT PARAMETER
+---------------
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``errhandler``: New error handler for communicator (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Comm_get_errhandler` retrieves the error handler currently associated
+with a communicator. This call is identical to :ref:`MPI_Errhandler_get`, the
+use of which is deprecated.
+
+
+Returns in errhandler (a handle to) the error handler that is currently
+associated with communicator ``comm``.
+
+**Example:** A library function may register at its entry point the
+current error handler for a communicator, set its own private error
+handler for this communicator, and restore before exiting the previous
+error handler.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_create_errhandler`
+   * :ref:`MPI_Comm_set_errhandler`
diff --git a/docs/man-openmpi/man3/MPI_Comm_get_info.3.rst b/docs/man-openmpi/man3/MPI_Comm_get_info.3.rst
new file mode 100644
index 00000000000..6ef8bec6bf5
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_get_info.3.rst
@@ -0,0 +1,41 @@
+.. _mpi_comm_get_info:
+
+
+MPI_Comm_get_info
+=================
+
+.. include_body
+
+:ref:`MPI_Comm_get_info` |mdash| Retrieves active communicator info hints
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_get_info.rst
+
+INPUT PARAMETERS
+----------------
+* ``comm``: Communicator from which to receive active info hints
+
+OUTPUT PARAMETERS
+-----------------
+* ``info_used``: New info object returned with all active hints on this communicator.
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Comm_get_info` returns a new info object containing the hints of the
+communicator associated with *comm*. The current setting of all hints
+actually used by the system related to this communicator is returned in
+*info_used*. If no such hints exist, a handle to a newly created info
+object is returned that contains no key/value pair. The user is
+responsible for freeing info_used via :ref:`MPI_Info_free`.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_get_info`
+   * :ref:`MPI_Info_free`
diff --git a/docs/man-openmpi/man3/MPI_Comm_get_name.3.rst b/docs/man-openmpi/man3/MPI_Comm_get_name.3.rst
new file mode 100644
index 00000000000..6b3cd0cec8a
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_get_name.3.rst
@@ -0,0 +1,64 @@
+.. _mpi_comm_get_name:
+
+
+MPI_Comm_get_name
+=================
+
+.. include_body
+
+:ref:`MPI_Comm_get_name` |mdash| Returns the name that was most recently
+associated with a communicator.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_get_name.rst
+
+INPUT PARAMETER
+---------------
+* ``comm``: Communicator the name of which is to be returned (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``comm_name``: Name previously stored on the communicator, or an empty string if no such name exists (string).
+* ``resultlen``: Length of returned name (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Comm_get_name` returns the last name that was previously associated
+with the given communicator. The name may be set and retrieved from any
+language. The same name will be returned independent of the language
+used. *comm_name* should be allocated so that it can hold a resulting
+string of length MPI_MAX_OBJECT_NAME characters. :ref:`MPI_Comm_get_name`
+returns a copy of the set name in *comm_name*.
+
+If the user has not associated a name with a communicator, or an error
+occurs, :ref:`MPI_Comm_get_name` will return an empty string (all spaces in
+Fortran, "" in C). The three predefined communicators will have
+predefined names associated with them. Thus, the names of
+MPI_COMM_WORLD, MPI_COMM_SELF, and MPI_COMM_PARENT will have the default
+of MPI_COMM_WORLD, MPI_COMM_SELF, and MPI_COMM_PARENT. The fact that the
+system may have chosen to give a default name to a communicator does not
+prevent the user from setting a name on the same communicator; doing
+this removes the old name and assigns the new one.
+
+
+NOTES
+-----
+
+It is safe simply to print the string returned by :ref:`MPI_Comm_get_name`, as
+it is always a valid string even if there was no name.
+
+Note that associating a name with a communicator has no effect on the
+semantics of an MPI program, and will (necessarily) increase the store
+requirement of the program, since the names must be saved. Therefore,
+there is no requirement that users use these functions to associate
+names with communicators. However debugging and profiling MPI
+applications may be made easier if names are associated with
+communicators, since the debugger or profiler should then be able to
+present information in a less cryptic manner.
+
+
+ERRORS
+------
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Comm_get_parent.3.rst b/docs/man-openmpi/man3/MPI_Comm_get_parent.3.rst
new file mode 100644
index 00000000000..b24c29f3d22
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_get_parent.3.rst
@@ -0,0 +1,54 @@
+.. _mpi_comm_get_parent:
+
+
+MPI_Comm_get_parent
+===================
+
+.. include_body
+
+:ref:`MPI_Comm_get_parent` |mdash| Returns the parent intercommunicator of
+current spawned process.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_get_parent.rst
+
+OUTPUT PARAMETERS
+-----------------
+* ``parent``: The parent communicator (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+If a process was started with :ref:`MPI_Comm_spawn` or :ref:`MPI_Comm_spawn_multiple`,
+:ref:`MPI_Comm_get_parent` returns the "parent" intercommunicator of the
+current process. This parent intercommunicator is created implicitly
+inside of :ref:`MPI_Init` and is the same intercommunicator returned by the
+spawn call made in the parents.
+
+If the process was not spawned, :ref:`MPI_Comm_get_parent` returns
+MPI_COMM_NULL.
+
+After the parent communicator is freed or disconnected,
+:ref:`MPI_Comm_get_parent` returns MPI_COMM_NULL.
+
+
+NOTES
+-----
+
+:ref:`MPI_Comm_get_parent` returns a handle to a single intercommunicator.
+Calling :ref:`MPI_Comm_get_parent` a second time returns a handle to the same
+intercommunicator. Freeing the handle with :ref:`MPI_Comm_disconnect` or
+:ref:`MPI_Comm_free` will cause other references to the intercommunicator to
+become invalid (dangling). Note that calling :ref:`MPI_Comm_free` on the parent
+communicator is not useful.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_spawn`
+   * :ref:`MPI_Comm_spawn_multiple`
diff --git a/docs/man-openmpi/man3/MPI_Comm_group.3.rst b/docs/man-openmpi/man3/MPI_Comm_group.3.rst
new file mode 100644
index 00000000000..75c05722063
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_group.3.rst
@@ -0,0 +1,34 @@
+.. _mpi_comm_group:
+
+
+MPI_Comm_group
+==============
+
+.. include_body
+
+:ref:`MPI_Comm_group` |mdash| Returns the group associated with a communicator.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_group.rst
+
+INPUT PARAMETER
+---------------
+* ``comm``: Communicator.
+
+OUTPUT PARAMETERS
+-----------------
+* ``group``: Group in communicator (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+If the communicator is an intercommunicator (enables communication
+between two groups of processes), this function returns the local group.
+To return the remote group, use the :ref:`MPI_Comm_remote_group` function.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Comm_idup.3.rst b/docs/man-openmpi/man3/MPI_Comm_idup.3.rst
new file mode 100644
index 00000000000..901fa0f33c1
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_idup.3.rst
@@ -0,0 +1,75 @@
+.. _mpi_comm_idup:
+
+
+MPI_Comm_idup
+=============
+
+.. include_body
+
+:ref:`MPI_Comm_idup` |mdash| Start the nonblocking duplication of an existing
+communicator with all its cached information.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_idup.rst
+
+INPUT PARAMETER
+---------------
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``newcomm``: Copy of comm (handle).
+* ``request``: Communication request (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Comm_idup` starts the nonblocking duplication of an existing
+communicator comm with associated key values. For each key value, the
+respective copy callback function determines the attribute value
+associated with this key in the new communicator; one particular action
+that a copy callback may take is to delete the attribute from the new
+communicator. Returns in newcomm a new communicator with the same group,
+any copied cached information, but a new context (see the
+"Functionality" subsection of the "Caching" section in the "Groups,
+Contexts, and Communicators" chapter in the `MPI Standard
+<https://www.mpi-forum.org/docs/>`_).  The communicator returned in
+*newcomm* will not be available until the request is complete.
+
+The completion of a communicator duplication request can be determined
+by calling any of :ref:`MPI_Wait`, :ref:`MPI_Waitany`, :ref:`MPI_Test`, or :ref:`MPI_Testany` with
+the request returned by this function.
+
+
+NOTES
+-----
+
+This operation is used to provide a parallel library call with a
+duplicate communication space that has the same properties as the
+original communicator. This includes any attributes (see below) and
+topologies (see the "Process Topologies" chapter in the `MPI Standard
+<https://www.mpi-forum.org/docs/>`_).
+This call is valid even if there are pending point-to-point
+communications involving the communicator comm. A typical call might
+involve an :ref:`MPI_Comm_idup` at the beginning of the parallel call, and an
+:ref:`MPI_Comm_free` of that duplicated communicator at the end of the call.
+Other models of communicator management are also possible.
+
+This call applies to both intra- and intercommunicators.
+
+Note that it is not defined by the MPI standard what happens if the
+attribute copy callback invokes other MPI functions. In Open MPI, it is
+not valid for attribute copy callbacks (or any of their children) to add
+or delete attributes on the same object on which the attribute copy
+callback is being invoked.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_dup`
+   * :ref:`MPI_Comm_dup_with_info`
diff --git a/docs/man-openmpi/man3/MPI_Comm_idup_with_info.3.rst b/docs/man-openmpi/man3/MPI_Comm_idup_with_info.3.rst
new file mode 100644
index 00000000000..45fb6e43a5a
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_idup_with_info.3.rst
@@ -0,0 +1,78 @@
+.. _mpi_comm_idup_with_info:
+
+
+MPI_Comm_idup_with_info
+=======================
+
+.. include_body
+
+:ref:`MPI_Comm_idup_with_info` |mdash| Start the nonblocking duplication of an
+existing communicator with all its cached information.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_idup_with_info.rst
+
+INPUT PARAMETER
+---------------
+* ``comm``: Communicator (handle). info Info object (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``newcomm``: Copy of comm (handle).
+* ``request``: Communication request (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Comm_idup_with_info` starts the nonblocking duplication of an
+existing communicator comm with associated key values. For each key
+value, the respective copy callback function determines the attribute
+value associated with this key in the new communicator; one particular
+action that a copy callback may take is to delete the attribute from the
+new communicator. Returns in newcomm a new communicator with the same
+group, any copied cached information, but a new context
+(see the "Functionality" subsection of the "Caching"
+section in the "Groups, Contexts, and Communicators" chapter in the
+`MPI Standard <https://www.mpi-forum.org/docs/>`_).  The communicator returned
+in *newcomm* will not be available until the request is complete. The
+hints provided by the supplied *info* argument are associated with the
+output communicator.
+
+The completion of a communicator duplication request can be determined
+by calling any of :ref:`MPI_Wait`, :ref:`MPI_Waitany`, :ref:`MPI_Test`, or :ref:`MPI_Testany` with
+the request returned by this function.
+
+
+NOTES
+-----
+
+This operation is used to provide a parallel library call with a
+duplicate communication space that has the same properties as the
+original communicator. This includes any attributes (see below) and
+topologies (see the "Process Topologies" chapter in the `MPI Standard
+<https://www.mpi-forum.org/docs/>`_).
+This call is valid even if there are pending point-to-point
+communications involving the communicator comm. A typical call might
+involve an :ref:`MPI_Comm_idup_with_info` at the beginning of the parallel
+call, and an :ref:`MPI_Comm_free` of that duplicated communicator at the end of
+the call. Other models of communicator management are also possible.
+
+This call applies to both intra- and intercommunicators.
+
+Note that it is not defined by the MPI standard what happens if the
+attribute copy callback invokes other MPI functions. In Open MPI, it is
+not valid for attribute copy callbacks (or any of their children) to add
+or delete attributes on the same object on which the attribute copy
+callback is being invoked.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_dup`
+   * :ref:`MPI_Comm_idup`
+   * :ref:`MPI_Comm_dup_with_info`
diff --git a/docs/man-openmpi/man3/MPI_Comm_join.3.rst b/docs/man-openmpi/man3/MPI_Comm_join.3.rst
new file mode 100644
index 00000000000..736f9a97e9f
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_join.3.rst
@@ -0,0 +1,64 @@
+.. _mpi_comm_join:
+
+
+MPI_Comm_join
+=============
+
+.. include_body
+
+:ref:`MPI_Comm_join` |mdash| Establishes communication between MPI jobs
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_join.rst
+
+INPUT PARAMETER
+---------------
+* ``fd``: socket file descriptor (socket).
+
+OUTPUT PARAMETERS
+-----------------
+* ``intercomm``: Intercommunicator between processes (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Comm_join` creates an intercommunicator from the union of two MPI
+processes that are connected by a socket. *fd* is a file descriptor
+representing a socket of type SOCK_STREAM (a two-way reliable
+byte-stream connection). Nonblocking I/O and asynchronous notification
+via SIGIO must not be enabled for the socket. The socket must be in a
+connected state, and must be quiescent when :ref:`MPI_Comm_join` is called.
+
+:ref:`MPI_Comm_join` must be called by the process at each end of the socket.
+It does not return until both processes have called :ref:`MPI_Comm_join`.
+
+
+NOTES
+-----
+
+There are no MPI library calls for opening and manipulating a socket.
+The socket *fd* can be opened using standard socket API calls. MPI uses
+the socket to bootstrap creation of the intercommunicator, and for
+nothing else. Upon return, the file descriptor will be open and
+quiescent.
+
+In a multithreaded process, the application must ensure that other
+threads do not access the socket while one is in the midst of calling
+:ref:`MPI_Comm_join`.
+
+The returned communicator will contain the two processes connected by
+the socket, and may be used to establish MPI communication with
+additional processes, through the usual MPI communicator-creation
+mechanisms.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * socket(3SOCKET)
+   * :ref:`MPI_Comm_create`
+   * :ref:`MPI_Comm_group`
diff --git a/docs/man-openmpi/man3/MPI_Comm_rank.3.rst b/docs/man-openmpi/man3/MPI_Comm_rank.3.rst
new file mode 100644
index 00000000000..eefb7a6ed8b
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_rank.3.rst
@@ -0,0 +1,47 @@
+.. _mpi_comm_rank:
+
+
+MPI_Comm_rank
+=============
+
+.. include_body
+
+:ref:`MPI_Comm_rank` |mdash| Determines the rank of the calling process in the
+communicator.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_rank.rst
+
+INPUT PARAMETERS
+----------------
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``rank``: Rank of the calling process in group of comm (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This function gives the rank of the process in the particular
+communicator's group. It is equivalent to accessing the communicator's
+group with :ref:`MPI_Comm_group`, computing the rank using :ref:`MPI_Group_rank`, and
+then freeing the temporary group via :ref:`MPI_Group_free`.
+
+Many programs will be written with the manager-worker model, where one
+process (such as the rank-zero process) will play a supervisory role,
+and the other processes will serve as compute nodes. In this framework,
+:ref:`MPI_Comm_size` and :ref:`MPI_Comm_rank` are useful for determining the roles of
+the various processes of a communicator.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_group`
+   * :ref:`MPI_Comm_size`
+   * :ref:`MPI_Comm_compare`
diff --git a/docs/man-openmpi/man3/MPI_Comm_remote_group.3.rst b/docs/man-openmpi/man3/MPI_Comm_remote_group.3.rst
new file mode 100644
index 00000000000..838efad0a71
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_remote_group.3.rst
@@ -0,0 +1,43 @@
+.. _mpi_comm_remote_group:
+
+
+MPI_Comm_remote_group
+=====================
+
+.. include_body
+
+:ref:`MPI_Comm_remote_group` |mdash| Accesses the remote group associated with an
+intercommunicator.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_remote_group.rst
+
+INPUT PARAMETER
+---------------
+* ``comm``: Communicator.
+
+OUTPUT PARAMETERS
+-----------------
+* ``group``: Remote group of communicator.
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Comm_remote_group` accesses the remote group associated with an
+intercommunicator.
+
+The intercommunicator accessors (:ref:`MPI_Comm_test_inter`,
+:ref:`MPI_Comm_remote_size`, MPI_Comm_remote_group) are all local operations.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_test_inter`
+   * :ref:`MPI_Comm_remote_size`
+   * :ref:`MPI_Intercomm_create`
+   * :ref:`MPI_Intercomm_merge`
diff --git a/docs/man-openmpi/man3/MPI_Comm_remote_size.3.rst b/docs/man-openmpi/man3/MPI_Comm_remote_size.3.rst
new file mode 100644
index 00000000000..80ccfb13e8d
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_remote_size.3.rst
@@ -0,0 +1,43 @@
+.. _mpi_comm_remote_size:
+
+
+MPI_Comm_remote_size
+====================
+
+.. include_body
+
+:ref:`MPI_Comm_remote_size` |mdash| Determines the size of the remote group
+associated with an intercommunicator.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_remote_size.rst
+
+INPUT PARAMETER
+---------------
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``size``: Number of processes in the remote group of comm (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Comm_remote_size` determines the size of the remote group associated
+with an intercommunicator.
+
+The intercommunicator accessors (:ref:`MPI_Comm_test_inter`,
+:ref:`MPI_Comm_remote_size`, MPI_Comm_remote_group) are all local operations.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_test_inter`
+   * :ref:`MPI_Comm_remote_group`
+   * :ref:`MPI_Intercomm_create`
+   * :ref:`MPI_Intercomm_merge`
diff --git a/docs/man-openmpi/man3/MPI_Comm_set_attr.3.rst b/docs/man-openmpi/man3/MPI_Comm_set_attr.3.rst
new file mode 100644
index 00000000000..7c4388db682
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_set_attr.3.rst
@@ -0,0 +1,62 @@
+.. _mpi_comm_set_attr:
+
+
+MPI_Comm_set_attr
+=================
+
+.. include_body
+
+:ref:`MPI_Comm_set_attr` |mdash| Stores attribute value associated with a key.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_set_attr.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``comm``: Communicator from which attribute will be attached (handle).
+
+INPUT PARAMETERS
+----------------
+* ``comm_keyval``: Key value (integer).
+* ``attribute_val``: Attribute value.
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Comm_set_attr` stores the stipulated attribute value *attribute_val*
+for subsequent retrieval by :ref:`MPI_Comm_get_attr`. If the value is already
+present, then the outcome is as if :ref:`MPI_Comm_delete_attr` was first called
+to delete the previous value (and the callback function ``delete_fn`` was
+executed), and a new value was next stored. The call is erroneous if
+there is no key with value *comm_keyval*; in particular
+``MPI_KEYVAL_INVALID`` is an erroneous key value. The call will fail if the
+delete_fn function returned an error code other than ``MPI_SUCCESS``.
+
+This function replaces :ref:`MPI_Attr_put`, the use of which is deprecated. The
+C binding is identical. The Fortran binding differs in that
+*attribute_val* is an address-sized integer.
+
+
+NOTES
+-----
+
+Values of the permanent attributes ``MPI_TAG_UB``, ``MPI_HOST``, ``MPI_IO``, and
+``MPI_WTIME_IS_GLOBAL`` may not be changed.
+
+The type of the attribute value depends on whether C or Fortran is being
+used. In C, an attribute value is a pointer (``void *``); in Fortran, it is
+a single, address-size integer system for which a pointer does not fit
+in an integer.
+
+If an attribute is already present, the delete function (specified when
+the corresponding keyval was created) will be called.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Comm_set_errhandler.3.rst b/docs/man-openmpi/man3/MPI_Comm_set_errhandler.3.rst
new file mode 100644
index 00000000000..50ce8f105bd
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_set_errhandler.3.rst
@@ -0,0 +1,40 @@
+.. _mpi_comm_set_errhandler:
+
+
+MPI_Comm_set_errhandler
+=======================
+
+.. include_body
+
+:ref:`MPI_Comm_set_errhandler` |mdash| Attaches a new error handler to a
+communicator.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_set_errhandler.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``errhandler``: New error handler for communicator (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Comm_set_errhandler` attaches a new error handler to a communicator.
+The error handler must be either a predefined error handler or an error
+handler created by a call to :ref:`MPI_Comm_create_errhandler`. This call is
+identical to :ref:`MPI_Errhandler_set`, the use of which is deprecated.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_create_errhandler`
+   * :ref:`MPI_Comm_get_errhandler`
diff --git a/docs/man-openmpi/man3/MPI_Comm_set_info.3.rst b/docs/man-openmpi/man3/MPI_Comm_set_info.3.rst
new file mode 100644
index 00000000000..e94d51e83e4
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_set_info.3.rst
@@ -0,0 +1,66 @@
+.. _mpi_comm_set_info:
+
+
+MPI_Comm_set_info
+=================
+
+.. include_body
+
+:ref:`MPI_Comm_set_info` |mdash| Set communicator info hints
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_set_info.rst
+
+INPUT PARAMETERS
+----------------
+* ``comm``: Communicator on which to set info hints
+* ``info``: Info object containing hints to be set on *comm*
+
+OUTPUT PARAMETERS
+-----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Comm_set_info` sets new values for the hints of the communicator
+associated with *comm*. :ref:`MPI_Comm_set_info` is a collective routine. The
+info object may be different on each process, but any info entries that
+an implementation requires to be the same on all processes must appear
+with the same value in each process's *info* object.
+
+The following info key assertions may be accepted by Open MPI:
+
+*mpi_assert_no_any_tag* (boolean): If set to true, then the
+implementation may assume that the process will not use the ``MPI_ANY_TAG``
+wildcard on the given communicator.
+
+*mpi_assert_no_any_source* (boolean): If set to true, then the
+implementation may assume that the process will not use the
+``MPI_ANY_SOURCE`` wildcard on the given communicator.
+
+*mpi_assert_exact_length* (boolean): If set to true, then the
+implementation may assume that the lengths of messages received by the
+process are equal to the lengths of the corresponding receive buffers,
+for point-to-point communication operations on the given communicator.
+
+*mpi_assert_allow_overtaking* (boolean): If set to true, then the
+implementation may assume that point-to-point communications on the
+given communicator do not rely on the non-overtaking rule specified in
+MPI-3.1 Section 3.5. In other words, the application asserts that send
+operations are not required to be matched at the receiver in the order
+in which the send operations were performed by the sender, and receive
+operations are not required to be matched in the order in which they
+were performed by the receiver.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_get_info`
+   * :ref:`MPI_Info_create`
+   * :ref:`MPI_Info_set`
+   * :ref:`MPI_Info_free`
diff --git a/docs/man-openmpi/man3/MPI_Comm_set_name.3.rst b/docs/man-openmpi/man3/MPI_Comm_set_name.3.rst
new file mode 100644
index 00000000000..7d292e7fbb2
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_set_name.3.rst
@@ -0,0 +1,67 @@
+.. _mpi_comm_set_name:
+
+
+MPI_Comm_set_name
+=================
+
+.. include_body
+
+:ref:`MPI_Comm_set_name` |mdash| Associates a name with a communicator.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_set_name.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``comm``: Communicator whose identifier is to be set (handle).
+
+INPUT PARAMETER
+---------------
+* ``comm_name``: Character string to be used as the identifier for the communicator (string).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Comm_set_name` allows a user to associate a name string with a
+communicator. The character string that is passed to :ref:`MPI_Comm_set_name`
+is saved inside the MPI library (so it can be freed by the caller
+immediately after the call, or allocated on the stack). Leading spaces
+in *name* are significant, but trailing ones are not.
+
+:ref:`MPI_Comm_set_name` is a local (noncollective) operation, which affects
+only the name of the communicator as seen in the process that made the
+:ref:`MPI_Comm_set_name` call. There is no requirement that the same (or any)
+name be assigned to a communicator in every process where it exists.
+
+The length of the name that can be stored is limited to the value of
+MPI_MAX_OBJECT_NAME in Fortran and MPI_MAX_OBJECT_NAME-1 in C (to allow
+for the null terminator). Attempts to set names longer than this will
+result in truncation of the name. MPI_MAX_OBJECT_NAME must have a value
+of at least 64.
+
+
+NOTES
+-----
+
+Since :ref:`MPI_Comm_set_name` is provided to help debug code, it is sensible
+to give the same name to a communicator in all of the processes where it
+exists, to avoid confusion.
+
+Regarding name length, under circumstances of store exhaustion, an
+attempt to set a name of any length could fail; therefore, the value of
+MPI_MAX_OBJECT_NAME should be viewed only as a strict upper bound on the
+name length, not a guarantee that setting names of less than this length
+will always succeed.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_get_name`
diff --git a/docs/man-openmpi/man3/MPI_Comm_size.3.rst b/docs/man-openmpi/man3/MPI_Comm_size.3.rst
new file mode 100644
index 00000000000..7efc1e905fd
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_size.3.rst
@@ -0,0 +1,57 @@
+.. _mpi_comm_size:
+
+
+MPI_Comm_size
+=============
+
+.. include_body
+
+:ref:`MPI_Comm_size` |mdash| Returns the size of the group associated with a
+communicator.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_size.rst
+
+INPUT PARAMETER
+---------------
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``size``: Number of processes in the group of comm (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This function indicates the number of processes involved in a
+communicator. For MPI_COMM_WORLD, it indicates the total number of
+processes available. This function is equivalent to accessing the
+communicator's group with :ref:`MPI_Comm_group`, computing the size using
+:ref:`MPI_Group_size`, and then freeing the temporary group via :ref:`MPI_Group_free`.
+If the communicator is an inter-communicator (enables communication
+between two groups), this function returns the size of the local group.
+To return the size of the remote group, use the :ref:`MPI_Comm_remote_size`
+function.
+
+This call is often used with :ref:`MPI_Comm_rank` to determine the amount of
+concurrency available for a specific library or program. :ref:`MPI_Comm_rank`
+indicates the rank of the process that calls it in the range from 0 . .
+. size-1, where size is the return value of :ref:`MPI_Comm_size`.
+
+
+NOTE
+----
+
+MPI_COMM_NULL is not considered a valid argument to this function.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_group`
+   * :ref:`MPI_Comm_rank`
+   * :ref:`MPI_Comm_compare`
diff --git a/docs/man-openmpi/man3/MPI_Comm_spawn.3.rst b/docs/man-openmpi/man3/MPI_Comm_spawn.3.rst
new file mode 100644
index 00000000000..246d7278738
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_spawn.3.rst
@@ -0,0 +1,234 @@
+.. _mpi_comm_spawn:
+
+
+MPI_Comm_spawn
+==============
+
+.. include_body
+
+:ref:`MPI_Comm_spawn` |mdash| Spawns a number of identical binaries.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_spawn.rst
+
+INPUT PARAMETERS
+----------------
+* ``command``: Name of program to be spawned (string, significant only at *root*).
+* ``argv``: Arguments to *command* (array of strings, significant only at *root*).
+* ``maxprocs``: Maximum number of processes to start (integer, significant only at *root*).
+* ``info``: A set of key-value pairs telling the runtime system where and how to start the processes (handle, significant only at *root*).
+* ``root``: Rank of process in which previous arguments are examined (integer).
+* ``comm``: Intracommunicator containing group of spawning processes (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``intercomm``: Intercommunicator between original group and the newly spawned group (handle).
+* ``array_of_errcodes``: One code per process (array of integers).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Comm_spawn` tries to start *maxprocs* identical copies of the MPI
+program specified by *command*, establishing communication with them and
+returning an intercommunicator. The spawned processes are referred to as
+children. The children have their own MPI_COMM_WORLD, which is separate
+from that of the parents. :ref:`MPI_Comm_spawn` is collective over *comm*, and
+also may not return until :ref:`MPI_Init` has been called in the children.
+Similarly, :ref:`MPI_Init` in the children may not return until all parents
+have called :ref:`MPI_Comm_spawn`. In this sense, :ref:`MPI_Comm_spawn` in the parents
+and :ref:`MPI_Init` in the children form a collective operation over the union
+of parent and child processes. The intercommunicator returned by
+:ref:`MPI_Comm_spawn` contains the parent processes in the local group and the
+child processes in the remote group. The ordering of processes in the
+local and remote groups is the same as the as the ordering of the group
+of the *comm* in the parents and of MPI_COMM_WORLD of the children,
+respectively. This intercommunicator can be obtained in the children
+through the function :ref:`MPI_Comm_get_parent`.
+
+The MPI standard allows an implementation to use the MPI_UNIVERSE_SIZE
+attribute of MPI_COMM_WORLD to specify the number of processes that will
+be active in a program. Although this implementation of the MPI standard
+defines MPI_UNIVERSE_SIZE, it does not allow the user to set its value.
+If you try to set the value of MPI_UNIVERSE_SIZE, you will get an error
+message.
+
+The *command* Argument
+
+The *command* argument is a string containing the name of a program to
+be spawned. The string is null-terminated in C. In Fortran, leading and
+trailing spaces are stripped. MPI looks for the file first in the
+working directory of the spawning process.
+
+The *argv* Argument
+
+*argv* is an array of strings containing arguments that are passed to
+the program. The first element of *argv* is the first argument passed to
+*command*, not, as is conventional in some contexts, the command itself.
+The argument list is terminated by NULL in C and an empty string in
+Fortran (note that it is the MPI application's responsibility to ensure
+that the last entry of the *argv* array is an empty string; the compiler
+will not automatically insert it). In Fortran, leading and trailing
+spaces are always stripped, so that a string consisting of all spaces is
+considered an empty string. The constant MPI_ARGV_NULL may be used in C
+and Fortran to indicate an empty argument list. In C, this constant is
+the same as NULL.
+
+In C, the :ref:`MPI_Comm_spawn` argument *argv* differs from the *argv*
+argument of *main* in two respects. First, it is shifted by one element.
+Specifically, *argv*\ [0] of *main* contains the name of the program
+(given by *command*). *argv*\ [1] of *main* corresponds to *argv*\ [0]
+in :ref:`MPI_Comm_spawn`, *argv*\ [2] of *main* to *argv*\ [1] of
+:ref:`MPI_Comm_spawn`, and so on. Second, *argv* of :ref:`MPI_Comm_spawn` must be
+null-terminated, so that its length can be determined. Passing an *argv*
+of MPI_ARGV_NULL to :ref:`MPI_Comm_spawn` results in *main* receiving *argc* of
+1 and an *argv* whose element 0 is the name of the program.
+
+The *maxprocs* Argument
+
+Open MPI tries to spawn *maxprocs* processes. If it is unable to spawn
+*maxprocs* processes, it raises an error of class MPI_ERR_SPAWN. If MPI
+is able to spawn the specified number of processes, :ref:`MPI_Comm_spawn`
+returns successfully and the number of spawned processes, *m*, is given
+by the size of the remote group of *intercomm*.
+
+A spawn call with the default behavior is called hard. A spawn call for
+which fewer than *maxprocs* processes may be returned is called soft.
+
+The *info* Argument
+
+The *info* argument is an opaque handle of type MPI_Info in C and
+INTEGER in Fortran. It is a container for a number of user-specified
+(*key,value*) pairs. *key* and *value* are strings (null-terminated
+``char *`` in C, ``character*(*)`` in Fortran). Routines to create and
+manipulate the *info* argument are described in Section 4.10 of the
+MPI-2 standard.
+
+For the SPAWN calls, *info* provides additional,
+implementation-dependent instructions to MPI and the runtime system on
+how to start processes. An application may pass MPI_INFO_NULL in C or
+Fortran. Portable programs not requiring detailed control over process
+locations should use MPI_INFO_NULL.
+
+The following keys for *info* are recognized in Open MPI. (The reserved
+values mentioned in Section 5.3.4 of the MPI-2 standard are not
+implemented.)
+
+::
+
+   Key                    Type     Description
+   ---                    ----     -----------
+
+   host                   char *   Host on which the process should be
+                                   spawned.  See the orte_host man
+                                   page for an explanation of how this
+                                   will be used.
+   hostfile               char *   Hostfile containing the hosts on which
+                                   the processes are to be spawned. See
+                                   the orte_hostfile man page for
+                                   an explanation of how this will be
+                                   used.
+   add-host               char *   Add the specified host to the list of
+                                   hosts known to this job and use it for
+                                   the associated process. This will be
+                                   used similarly to the -host option.
+   add-hostfile           char *   Hostfile containing hosts to be added
+                                   to the list of hosts known to this job
+                                   and use it for the associated
+                                   process. This will be used similarly
+                                   to the -hostfile option.
+   wdir                   char *   Directory where the executable is
+                                   located. If files are to be
+                                   pre-positioned, then this location is
+                                   the desired working directory at time
+                                   of execution - if not specified, then
+                                   it will automatically be set to
+                                   ompi_preload_files_dest_dir.
+   ompi_prefix            char *   Same as the --prefix command line
+                                   argument to mpirun.
+   ompi_preload_binary    bool     If set to true, pre-position the
+                                   specified executable onto the remote
+                                   host. A destination directory must
+                                   also be provided.
+   ompi_preload_files     char *   A comma-separated list of files that
+                                   are to be pre-positioned in addition
+                                   to the executable.  Note that this
+                                   option does not depend upon
+                                   ompi_preload_binary - files can
+                                   be moved to the target even if an
+                                   executable is not moved.
+   ompi_stdin_target      char *   Comma-delimited list of ranks to
+                                   receive stdin when forwarded.
+   ompi_non_mpi           bool     If set to true, launching a non-MPI
+                                   application; the returned communicator
+                                   will be MPI_COMM_NULL. Failure to set
+                                   this flag when launching a non-MPI
+                                   application will cause both the child
+                                   and parent jobs to "hang".
+   ompi_param             char *   Pass an OMPI MCA parameter to the
+                                   child job.  If that parameter already
+                                   exists in the environment, the value
+                                   will be overwritten by the provided
+                                   value.
+   mapper                 char *   Mapper to be used for this job
+   map_by                 char *   Mapping directive indicating how
+                                   processes are to be mapped (slot,
+                                   node, socket, etc.).
+   rank_by                char *   Ranking directive indicating how
+                                   processes are to be ranked (slot,
+                                   node, socket, etc.).
+   bind_to                char *   Binding directive indicating how
+                                   processes are to be bound (core, slot,
+                                   node, socket, etc.).
+   path                   char *   List of directories to search for
+                                   the executable
+   npernode               char *   Number of processes to spawn on
+                                   each node of the allocation
+   pernode                bool     Equivalent to npernode of 1
+   ppr                    char *   Spawn specified number of processes
+                                   on each of the identified object type
+   env                    char *   Newline-delimited list of envars to
+                                   be passed to the spawned procs
+
+*bool* info keys are actually strings but are evaluated as follows: if
+the string value is a number, it is converted to an integer and cast to
+a boolean (meaning that zero integers are false and non-zero values are
+true). If the string value is (case-insensitive) "yes" or "true", the
+boolean is true. If the string value is (case-insensitive) "no" or
+"false", the boolean is false. All other string values are unrecognized,
+and therefore false.
+
+The *root* Argument
+
+All arguments before the *root* argument are examined only on the
+process whose rank in *comm* is equal to *root*. The value of these
+arguments on other processes is ignored.
+
+The *array_of_errcodes* Argument
+
+The *array_of_errcodes* is an array of length *maxprocs* in which MPI
+reports the status of the processes that MPI was requested to start. If
+all *maxprocs* processes were spawned, *array_of_errcodes* is filled in
+with the value MPI_SUCCESS. If anyof the processes are *not* spawned,
+*array_of_errcodes* is filled in with the value MPI_ERR_SPAWN. In C or
+Fortran, an application may pass MPI_ERRCODES_IGNORE if it is not
+interested in the error codes.
+
+
+NOTES
+-----
+
+Completion of :ref:`MPI_Comm_spawn` in the parent does not necessarily mean
+that :ref:`MPI_Init` has been called in the children (although the returned
+intercommunicator can be used immediately).
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_spawn_multiple`
+   * :ref:`MPI_Comm_get_parent`
+   * :ref:`mpirun(1) <man1-mpirun>`
diff --git a/docs/man-openmpi/man3/MPI_Comm_spawn_multiple.3.rst b/docs/man-openmpi/man3/MPI_Comm_spawn_multiple.3.rst
new file mode 100644
index 00000000000..d06662442f0
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_spawn_multiple.3.rst
@@ -0,0 +1,229 @@
+.. _mpi_comm_spawn_multiple:
+
+
+MPI_Comm_spawn_multiple
+=======================
+
+.. include_body
+
+:ref:`MPI_Comm_spawn_multiple` |mdash| Spawns multiple binaries, or the same
+binary with multiple sets of arguments.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_spawn_multiple.rst
+
+INPUT PARAMETERS
+----------------
+* ``count``: Number of commands (positive integer, significant to MPI only at *root* |mdash| see NOTES).
+* ``array_of_commands``: Programs to be executed (array of strings, significant only at *root*).
+* ``array_of_argv``: Arguments for *commands* (array of array of strings, significant only at *root*).
+* ``array_of_maxprocs``: Maximum number of processes to start for each command (array of integers, significant only at *root*).
+* ``array_of_info``: Info objects telling the runtime system where and how to start processes (array of handles, significant only at *root*).
+* ``root``: Rank of process in which previous arguments are examined (integer).
+* ``comm``: Intracommunicator containing group of spawning processes (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``intercomm``: Intercommunicator between original group and the newly spawned group (handle).
+* ``array_of_errcodes``: One code per process (array of integers).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Comm_spawn_multiple` is identical to :ref:`MPI_Comm_spawn` except that it
+can specify multiple executables. The first argument, *count*, indicates
+the number of executables. The next three arguments are arrays of the
+corresponding arguments in :ref:`MPI_Comm_spawn`. The next argument,
+*array_of_info*, is an array of *info* arguments, one for each
+executable. See the INFO ARGUMENTS section for more information.
+
+For the Fortran version of *array_of_argv*, the element
+*array_of_argv*\ (i,j) is the jth argument to command number i.
+
+In any language, an application may use the constant MPI_ARGVS_NULL
+(which is likely to be ``(char **)0`` in C) to specify that no arguments
+should be passed to any commands. The effect of setting individual
+elements of *array_of_argv* to MPI_ARGV_NULL is not defined. To specify
+arguments for some commands but not others, the commands without
+arguments should have a corresponding *argv* whose first element is null
+``((char *)0`` in C and empty string in Fortran).
+
+All of the spawned processes have the same MPI_COMM_WORLD. Their ranks
+in MPI_COMM_WORLD correspond directly to the order in which the commands
+are specified in :ref:`MPI_Comm_spawn_multiple`. Assume that m1 processes are
+generated by the first command, m2 by the second, etc. The processes
+corresponding to the first command have ranks 0, 1,..., m1-1. The
+processes in the second command have ranks m1, m1+1, ..., m1+m2-1. The
+processes in the third have ranks m1+m2, m1+m2+1, ..., m1+m2+m3-1, etc.
+
+The *array_of_errcodes* argument is 1-dimensional array of size
+
+::
+
+   	 _ count
+   	\       n ,
+   	/_ i=1   i
+
+where i is the ith element of *array_of_maxprocs*. Command number *i*
+corresponds to the i contiguous slots in this array from element
+
+::
+
+                         _              _
+   	 _ i-1          |   _ i          |
+   	\       n ,  to |  \      n      | -1
+   	/_ j=1   i      |  /_ j=1  j     |
+                        |_              _|
+
+Error codes are treated as for :ref:`MPI_Comm_spawn`.
+
+
+INFO ARGUMENTS
+--------------
+
+The following keys for *info* are recognized in "#PACKAGE_NAME#". (The
+reserved values mentioned in Section 5.3.4 of the MPI-2 standard are not
+implemented.)
+
+::
+
+   Key                    Type     Description
+   ---                    ----     -----------
+
+   host                   char *   Comma-separated list of hosts on which
+                                   the processes should be spawned.  See
+                                   the orte_host man page for an
+                                   explanation of how this will be used.
+   hostfile               char *   Hostfile containing the hosts on which
+                                   the processes are to be spawned. See
+                                   the orte_hostfile man page for
+                                   an explanation of how this will be
+                                   used.
+   add-host               char *   Add the specified hosts to the list of
+                                   hosts known to this job and use it for
+                                   the associated processes. This will be
+                                   used similarly to the -host option.
+   add-hostfile           char *   Hostfile containing hosts to be added
+                                   to the list of hosts known to this job
+                                   and use it for the associated
+                                   process. This will be used similarly
+                                   to the -hostfile option.
+   wdir                   char *   Directory where the executable is
+                                   located. If files are to be
+                                   pre-positioned, then this location is
+                                   the desired working directory at time
+                                   of execution - if not specified, then
+                                   it will automatically be set to
+                                   ompi_preload_files_dest_dir.
+   ompi_prefix            char *   Same as the --prefix command line
+                                   argument to mpirun.
+   ompi_preload_binary    bool     If set to true, pre-position the
+                                   specified executable onto the remote
+                                   host. A destination directory must
+                                   also be provided.
+   ompi_preload_files     char *   A comma-separated list of files that
+                                   are to be pre-positioned in addition
+                                   to the executable.  Note that this
+                                   option does not depend upon
+                                   ompi_preload_binary - files can
+                                   be moved to the target even if an
+                                   executable is not moved.
+   ompi_stdin_target      char *   Comma-delimited list of ranks to
+                                   receive stdin when forwarded.
+   ompi_non_mpi           bool     If set to true, launching a non-MPI
+                                   application; the returned communicator
+                                   will be MPI_COMM_NULL. Failure to set
+                                   this flag when launching a non-MPI
+                                   application will cause both the child
+                                   and parent jobs to "hang".
+   ompi_param             char *   Pass an OMPI MCA parameter to the
+                                   child job.  If that parameter already
+                                   exists in the environment, the value
+                                   will be overwritten by the provided
+                                   value.
+   mapper                 char *   Mapper to be used for this job
+   map_by                 char *   Mapping directive indicating how
+                                   processes are to be mapped (slot,
+                                   node, socket, etc.).
+   rank_by                char *   Ranking directive indicating how
+                                   processes are to be ranked (slot,
+                                   node, socket, etc.).
+   bind_to                char *   Binding directive indicating how
+                                   processes are to be bound (core, slot,
+                                   node, socket, etc.).
+   path                   char *   List of directories to search for
+                                   the executable
+   npernode               char *   Number of processes to spawn on
+                                   each node of the allocation
+   pernode                bool     Equivalent to npernode of 1
+   ppr                    char *   Spawn specified number of processes
+                                   on each of the identified object type
+   env                    char *   Newline-delimited list of envars to
+                                   be passed to the spawned procs
+
+*bool* info keys are actually strings but are evaluated as follows: if
+the string value is a number, it is converted to an integer and cast to
+a boolean (meaning that zero integers are false and non-zero values are
+true). If the string value is (case-insensitive) "yes" or "true", the
+boolean is true. If the string value is (case-insensitive) "no" or
+"false", the boolean is false. All other string values are unrecognized,
+and therefore false.
+
+Note that if any of the info handles have *ompi_non_mpi* set to true,
+then all info handles must have it set to true. If some are set to true,
+but others are set to false (or are unset), MPI_ERR_INFO will be
+returned.
+
+Note that in "#PACKAGE_NAME#", the first array location in
+*array_of_info* is applied to all the commands in *array_of_commands*.
+
+
+NOTES
+-----
+
+The argument *count* is interpreted by MPI only at the root, as is
+*array_of_argv*. Since the leading dimension of *array_of_argv* is
+*count*, a nonpositive value of *count* at a nonroot node could
+theoretically cause a runtime bounds check error, even though
+*array_of_argv* should be ignored by the subroutine. If this happens,
+you should explicitly supply a reasonable value of *count* on the
+nonroot nodes.
+
+Similar to :ref:`MPI_Comm_spawn`, it is the application's responsibility to
+terminate each individual set of argv in the *array_of_argv* argument.
+In C, each argv array is terminated by a NULL pointer. In Fortran, each
+argv array is terminated by an empty string (note that compilers will
+not automatically insert this blank string; the application must ensure
+to have enough space for an empty string entry as the last element of
+the array).
+
+Other restrictions apply to the *array_of_argv* parameter; see
+:ref:`MPI_Comm_spawn`'s description of the *argv* parameter for more
+details.
+
+MPI-3.1 implies (but does not directly state) that the argument
+*array_of_commands* must be an array of strings of length *count*.
+Unlike the *array_of_argv* parameter, *array_of_commands* does not need
+to be terminated with a NULL pointer in C or a blank string in Fortran.
+Older versions of Open MPI required that *array_of_commands* be
+terminated with a blank string in Fortran; that is no longer required in
+this version of Open MPI.
+
+Calling :ref:`MPI_Comm_spawn` many times would create many sets of children
+with different MPI_COMM_WORLDs, whereas :ref:`MPI_Comm_spawn_multiple` creates
+children with a single MPI_COMM_WORLD, so the two methods are not
+completely equivalent. Also if you need to spawn multiple executables,
+you may get better performance by using :ref:`MPI_Comm_spawn_multiple` instead
+of calling :ref:`MPI_Comm_spawn` several times.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_spawn`
+   * :ref:`MPI_Comm_get_parent`
+   * :ref:`mpirun(1) <man1-mpirun>`
diff --git a/docs/man-openmpi/man3/MPI_Comm_split.3.rst b/docs/man-openmpi/man3/MPI_Comm_split.3.rst
new file mode 100644
index 00000000000..83099d44f64
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_split.3.rst
@@ -0,0 +1,94 @@
+.. _mpi_comm_split:
+
+
+MPI_Comm_split
+==============
+
+.. include_body
+
+:ref:`MPI_Comm_split` |mdash| Creates new communicators based on colors and keys.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_split.rst
+
+INPUT PARAMETERS
+----------------
+* ``comm``: Communicator (handle).
+* ``color``: Control of subset assignment (nonnegative integer).
+* ``key``: Control of rank assignment (integer).
+
+OUTPUT PARAMETERS
+-----------------
+* ``newcomm``: New communicator (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This function partitions the group associated with comm into disjoint
+subgroups, one for each value of color. Each subgroup contains all
+processes of the same color. Within each subgroup, the processes are
+ranked in the order defined by the value of the argument key, with ties
+broken according to their rank in the old group. A new communicator is
+created for each subgroup and returned in newcomm. A process may supply
+the color value MPI_UNDEFINED, in which case newcomm returns
+MPI_COMM_NULL. This is a collective call, but each process is permitted
+to provide different values for color and key.
+
+When you call :ref:`MPI_Comm_split` on an inter-communicator, the processes on
+the left with the same color as those on the right combine to create a
+new inter-communicator. The key argument describes the relative rank of
+processes on each side of the inter-communicator. The function returns
+MPI_COMM_NULL for those colors that are specified on only one side of
+the inter-communicator, or for those that specify MPI_UNDEFINED as the
+color.
+
+A call to MPI_Comm_create(comm, *group*, *newcomm*) is equivalent to a
+call to MPI_Comm_split(comm, *color*,\ *key*, *newcomm*), where all
+members of *group* provide *color* = 0 and *key* = rank in group, and
+all processes that are not members of *group* provide *color* =
+MPI_UNDEFINED. The function :ref:`MPI_Comm_split` allows more general
+partitioning of a group into one or more subgroups with optional
+reordering.
+
+The value of *color* must be nonnegative or MPI_UNDEFINED.
+
+
+NOTES
+-----
+
+This is an extremely powerful mechanism for dividing a single
+communicating group of processes into k subgroups, with k chosen
+implicitly by the user (by the number of colors asserted over all the
+processes). Each resulting communicator will be nonoverlapping. Such a
+division could be useful for defining a hierarchy of computations, such
+as for multigrid or linear algebra.
+
+Multiple calls to :ref:`MPI_Comm_split` can be used to overcome the requirement
+that any call have no overlap of the resulting communicators (each
+process is of only one color per call). In this way, multiple
+overlapping communication structures can be created. Creative use of the
+color and key in such splitting operations is encouraged.
+
+Note that, for a fixed color, the keys need not be unique. It is
+:ref:`MPI_Comm_split`'s responsibility to sort processes in ascending order
+according to this key, and to break ties in a consistent way. If all the
+keys are specified in the same way, then all the processes in a given
+color will have the relative rank order as they did in their parent
+group. (In general, they will have different ranks.)
+
+Essentially, making the key value zero for all processes of a given
+color means that one needn't really pay attention to the rank-order of
+the processes in the new communicator.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_create`
+   * :ref:`MPI_Intercomm_create`
+   * :ref:`MPI_Comm_dup`
+   * :ref:`MPI_Comm_free`
diff --git a/docs/man-openmpi/man3/MPI_Comm_split_type.3.rst b/docs/man-openmpi/man3/MPI_Comm_split_type.3.rst
new file mode 100644
index 00000000000..bf0e97a452e
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_split_type.3.rst
@@ -0,0 +1,116 @@
+.. _mpi_comm_split_type:
+
+
+MPI_Comm_split_type
+===================
+
+.. include_body
+
+:ref:`MPI_Comm_split_type` |mdash| Creates new communicators based on colors and
+keys.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_split_type.rst
+
+INPUT PARAMETERS
+----------------
+* ``comm``: Communicator (handle).
+* ``split_type``: Type of processes to be grouped together (integer).
+* ``key``: Control of rank assignment (integer).
+* ``info``: Info argument (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``newcomm``: New communicator (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This function partitions the group associated with *comm* into disjoint
+subgroups, based on the type specified by *split_type*. Each subgroup
+contains all processes of the same type. Within each subgroup, the
+processes are ranked in the order defined by the value of the argument
+*key*, with ties broken according to their rank in the old group. A new
+communicator is created for each subgroup and returned in newcomm. This
+is a collective call; all processes must provide the same *split_type*,
+but each process is permitted to provide different values for key. An
+exception to this rule is that a process may supply the type value
+MPI_UNDEFINED, in which case newcomm returns MPI_COMM_NULL.
+
+
+SPLIT TYPES
+-----------
+
+MPI_COMM_TYPE_SHARED
+   This type splits the communicator into subcommunicators, each of
+   which can create a shared memory region.
+
+OMPI_COMM_TYPE_NODE
+   Synonym for MPI_COMM_TYPE_SHARED.
+
+OMPI_COMM_TYPE_HWTHREAD
+   This type splits the communicator into subcommunicators, each of
+   which belongs to the same hardware thread.
+
+OMPI_COMM_TYPE_CORE
+   This type splits the communicator into subcommunicators, each of
+   which belongs to the same core/processing unit.
+
+OMPI_COMM_TYPE_L1CACHE
+   This type splits the communicator into subcommunicators, each of
+   which belongs to the same L1 cache.
+
+OMPI_COMM_TYPE_L2CACHE
+   This type splits the communicator into subcommunicators, each of
+   which belongs to the same L2 cache.
+
+OMPI_COMM_TYPE_L3CACHE
+   This type splits the communicator into subcommunicators, each of
+   which belongs to the same L3 cache.
+
+OMPI_COMM_TYPE_SOCKET
+   This type splits the communicator into subcommunicators, each of
+   which belongs to the same socket.
+
+OMPI_COMM_TYPE_NUMA
+   This type splits the communicator into subcommunicators, each of
+   which belongs to the same NUMA-node.
+
+OMPI_COMM_TYPE_BOARD
+   This type splits the communicator into subcommunicators, each of
+   which belongs to the same board.
+
+OMPI_COMM_TYPE_HOST
+   This type splits the communicator into subcommunicators, each of
+   which belongs to the same host.
+
+OMPI_COMM_TYPE_CU
+   This type splits the communicator into subcommunicators, each of
+   which belongs to the same computational unit.
+
+OMPI_COMM_TYPE_CLUSTER
+   This type splits the communicator into subcommunicators, each of
+   which belongs to the same cluster.
+
+
+NOTES
+-----
+
+The communicator keys denoted with an *OMPI\_* prefix instead of an
+*MPI\_* prefix are specific to Open MPI, and are not part of the MPI
+standard. Their use should be protected by the *OPEN_MPI* C preprocessor
+macro.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_create`
+   * :ref:`MPI_Intercomm_create`
+   * :ref:`MPI_Comm_dup`
+   * :ref:`MPI_Comm_free`
+   * :ref:`MPI_Comm_split`
diff --git a/docs/man-openmpi/man3/MPI_Comm_test_inter.3.rst b/docs/man-openmpi/man3/MPI_Comm_test_inter.3.rst
new file mode 100644
index 00000000000..d47f4f26938
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Comm_test_inter.3.rst
@@ -0,0 +1,73 @@
+.. _mpi_comm_test_inter:
+
+
+MPI_Comm_test_inter
+===================
+
+.. include_body
+
+:ref:`MPI_Comm_test_inter` |mdash| Tests to see if a comm is an
+intercommunicator.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_comm_test_inter.rst
+
+INPUT PARAMETER
+---------------
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``flag (Logical.)``:
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This local routine allows the calling process to determine the type of a
+communicator. It returns true for an intercommunicator, false for an
+intracommunicator.
+
+The type of communicator also affects the value returned by three other
+functions. When dealing with an intracommunicator (enables communication
+within a single group), the functions listed below return the expected
+values, group size, group, and rank. When dealing with an
+inter-communicator, however, they return the following values:
+
+:ref:`MPI_Comm_size`
+     Returns the size of the local group.
+
+:ref:`MPI_Comm_group`
+     Returns the local group.
+
+:ref:`MPI_Comm_rank`
+     Returns the rank in the local group.
+
+To return the remote group and remote group size of an
+inter-communicator, use the :ref:`MPI_Comm_remote_group` and
+:ref:`MPI_Comm_remote_size` functions.
+
+The operation :ref:`MPI_Comm_compare` is valid for intercommunicators. Both
+communicators must be either intra- or intercommunicators, or else
+MPI_UNEQUAL results. Both corresponding local and remote groups must
+compare correctly to get the results MPI_CONGRUENT and MPI_SIMILAR. In
+particular, it is possible for MPI_SIMILAR to result because either the
+local or remote groups were similar but not identical.
+
+The following accessors provide consistent access to the remote group of
+an intercommunicator: :ref:`MPI_Comm_remote_size`, :ref:`MPI_Comm_remote_group`.
+
+The intercommunicator accessors (:ref:`MPI_Comm_test_inter`,
+:ref:`MPI_Comm_remote_size`, MPI_Comm_remote_group) are all local operations.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_remote_group`
+   * :ref:`MPI_Comm_remote_size`
+   * :ref:`MPI_Intercomm_create`
+   * :ref:`MPI_Intercomm_merge`
diff --git a/docs/man-openmpi/man3/MPI_Compare_and_swap.3.rst b/docs/man-openmpi/man3/MPI_Compare_and_swap.3.rst
new file mode 100644
index 00000000000..53eed0da366
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Compare_and_swap.3.rst
@@ -0,0 +1,57 @@
+.. _mpi_compare_and_swap:
+
+
+MPI_Compare_and_swap
+====================
+
+.. include_body
+
+:ref:`MPI_Compare_and_swap` |mdash| Perform RMA compare-and-swap
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_compare_and_swap.rst
+
+INPUT PARAMETERS
+----------------
+* ``origin_addr``: Initial address of buffer (choice).
+* ``compare_addr``: Initial address of compare buffer (choice).
+* ``result_addr``: Initial address of result buffer (choice).
+* ``datatype``: Data type of the entry in origin, result, and target buffers (handle).
+* ``target_rank``: Rank of target (nonnegative integer).
+* ``target_disp``: Displacement from start of window to beginning of target buffer (nonnegative integer).
+* ``win``: Window object (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This function compares one element of type *datatype* in the compare
+buffer *compare_addr* with the buffer at offset *target_disp* in the
+target window specified by *target_rank* and *win* and replaces the
+value at the target with the value in the origin buffer *origin_addr* if
+the compare buffer and the target buffer are identical. The original
+value at the target is returned in the buffer *result_addr*. The
+parameter *datatype* must belong to one of the following categories of
+predefined datatypes: C integer, Fortran integer, Logical,
+Multi-language types, or Byte as specified in MPI-3 section 5.9.2 on page 176.
+
+The origin and result buffers (*origin_addr* and *result_addr*) must be
+disjoint.
+
+
+NOTES
+-----
+
+It is the user's responsibility to guarantee that, when using the
+accumulate functions, the target displacement argument is such that
+accesses to the window are properly aligned according to the data type
+arguments in the call to the :ref:`MPI_Compare_and_swap` function.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Dims_create.3.rst b/docs/man-openmpi/man3/MPI_Dims_create.3.rst
new file mode 100644
index 00000000000..e7823fb91de
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Dims_create.3.rst
@@ -0,0 +1,73 @@
+.. _mpi_dims_create:
+
+
+MPI_Dims_create
+===============
+
+.. include_body
+
+:ref:`MPI_Dims_create` |mdash| Creates a division of processors in a Cartesian
+grid.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_dims_create.rst
+
+INPUT PARAMETERS
+----------------
+* ``nnodes``: Number of nodes in a grid (integer).
+* ``ndims``: Number of Cartesian dimensions (integer).
+
+IN/OUT PARAMETER
+----------------
+* ``dims``: Integer array of size ndims specifying the number of nodes in each dimension.
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+For Cartesian topologies, the function :ref:`MPI_Dims_create` helps the user
+select a balanced distribution of processes per coordinate direction,
+depending on the number of processes in the group to be balanced and
+optional constraints that can be specified by the user. One use is to
+partition all the processes (the size of MPI_COMM_WORLD's group) into an
+n-dimensional topology.
+
+The entries in the array *dims* are set to describe a Cartesian grid
+with *ndims* dimensions and a total of *nnodes* nodes. The dimensions
+are set to be as close to each other as possible, using an appropriate
+divisibility algorithm. The caller may further constrain the operation
+of this routine by specifying elements of array dims. If dims[i] is set
+to a positive number, the routine will not modify the number of nodes in
+dimension i; only those entries where dims[i] = 0 are modified by the
+call.
+
+Negative input values of dims[i] are erroneous. An error will occur if
+nnodes is not a multiple of ((pi) over (i, dims[i] != 0)) dims[i].
+
+For dims[i] set by the call, dims[i] will be ordered in nonincreasing
+order. Array dims is suitable for use as input to routine
+:ref:`MPI_Cart_create`. :ref:`MPI_Dims_create` is local.
+
+**Example:**
+
+::
+
+
+   dims
+   before					dims
+   call		function call		on return
+   -----------------------------------------------------
+   (0,0)	MPI_Dims_create(6, 2, dims)	(3,2)
+   (0,0)	MPI_Dims_create(7, 2, dims) 	(7,1)
+   (0,3,0)	MPI_Dims_create(6, 3, dims)	(2,3,1)
+   (0,3,0)	MPI_Dims_create(7, 3, dims)	erroneous call
+   ------------------------------------------------------
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Dist_graph_create.3.rst b/docs/man-openmpi/man3/MPI_Dist_graph_create.3.rst
new file mode 100644
index 00000000000..51c98bc3fd2
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Dist_graph_create.3.rst
@@ -0,0 +1,99 @@
+.. _mpi_dist_graph_create:
+
+
+MPI_Dist_graph_create
+=====================
+
+.. include_body
+
+:ref:`MPI_Dist_graph_create` |mdash| Makes a new communicator to which topology
+information has been attached.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_dist_graph_create.rst
+
+INPUT PARAMETERS
+----------------
+* ``comm_old``: Input communicator without topology (handle).
+* ``n``: Number of source nodes for which this process specifies edges (non-negative integer).
+* ``sources``: Array containing the *n* source nodes for which this process specifies edges (array of non-negative integers).
+* ``degrees``: Array specifying the number of destinations for each source node in the source node array (array of non-negative integers).
+* ``destinations``: Destination nodes for the source nodes in the source node array (array of non-negative integers).
+* ``weights``: Weights for source to destination edges (array of non-negative integers).
+* ``info``: Hints on optimization and interpretation of weights (handle).
+* ``reorder``: Ranking may be reordered (true) or not (false) (logical).
+
+OUTPUT PARAMETERS
+-----------------
+* ``comm_dist_graph``: Communicator with distributed graph topology added (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Dist_graph_create` creates a new communicator *comm_dist_graph* with
+distributed graph topology and returns a handle to the new communicator.
+The number of processes in *comm_dist_graph* is identical to the number
+of processes in *comm_old*. Concretely, each process calls the
+constructor with a set of directed (source,destination) communication
+edges as described below. Every process passes an array of *n* source
+nodes in the *sources* array. For each source node, a non-negative
+number of destination nodes is specified in the *degrees* array. The
+destination nodes are stored in the corresponding consecutive segment of
+the *destinations* array. More precisely, if the i-th node in sources is
+s, this specifies *degrees*\ [i] *edges* (s,d) with d of the j-th such
+edge stored in
+*destinations*\ [*degrees*\ [0]+...+\ *degrees*\ [i-1]+j]. The weight of
+this edge is stored in
+*weights*\ [*degrees*\ [0]+...+\ *degrees*\ [i-1]+j]. Both the *sources*
+and the *destinations* arrays may contain the same node more than once,
+and the order in which nodes are listed as destinations or sources is
+not signicant. Similarly, different processes may specify edges with the
+same source and destination nodes. Source and destination nodes must be
+process ranks of comm_old. Different processes may specify different
+numbers of source and destination nodes, as well as different source to
+destination edges. This allows a fully distributed specification of the
+communication graph. Isolated processes (i.e., processes with no
+outgoing or incoming edges, that is, processes that do not occur as
+source or destination node in the graph specification) are allowed. The
+call to :ref:`MPI_Dist_graph_create` is collective.
+
+If reorder = false, all processes will have the same rank in
+comm_dist_graph as in comm_old. If reorder = true then the MPI library
+is free to remap to other processes (of comm_old) in order to improve
+communication on the edges of the communication graph. The weight
+associated with each edge is a hint to the MPI library about the amount
+or intensity of communication on that edge, and may be used to compute a
+
+
+WEIGHTS
+-------
+
+Weights are specified as non-negative integers and can be used to
+influence the process remapping strategy and other internal MPI
+optimizations. For instance, approximate count arguments of later
+communication calls along specific edges could be used as their edge
+weights. Multiplicity of edges can likewise indicate more intense
+communication between pairs of processes. However, the exact meaning of
+edge weights is not specified by the MPI standard and is left to the
+implementation. An application can supply the special value
+MPI_UNWEIGHTED for the weight array to indicate that all edges have the
+same (effectively no) weight. It is erroneous to supply MPI_UNWEIGHTED
+for some but not all processes of comm_old. If the graph is weighted but
+*n* = 0, then MPI_WEIGHTS_EMPTY or any arbitrary array may be passed to
+weights. Note that MPI_UNWEIGHTED and MPI_WEIGHTS_EMPTY are not special
+weight values; rather they are special values for the total array
+argument. In Fortran, MPI_UNWEIGHTED and MPI_WEIGHTS_EMPTY are objects
+like MPI_BOTTOM (not usable for initialization or assignment). See MPI-3
+section 2.5.4.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Dist_graph_create_adjacent`
+   * :ref:`MPI_Dist_graph_neighbors`
+   * :ref:`MPI_Dist_graph_neighbors_count`
diff --git a/docs/man-openmpi/man3/MPI_Dist_graph_create_adjacent.3.rst b/docs/man-openmpi/man3/MPI_Dist_graph_create_adjacent.3.rst
new file mode 100644
index 00000000000..c6545954caa
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Dist_graph_create_adjacent.3.rst
@@ -0,0 +1,87 @@
+.. _mpi_dist_graph_create_adjacent:
+
+
+MPI_Dist_graph_create_adjacent
+==============================
+
+.. include_body
+
+:ref:`MPI_Dist_graph_create_adjacent` |mdash| Makes a new communicator to which
+topology information has been attached.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_dist_graph_create_adjacent.rst
+
+INPUT PARAMETERS
+----------------
+* ``comm_old``: Input communicator without topology (handle).
+* ``indegree``: Size of *sources* and *sourceweights* arrays (non-negative integer).
+* ``sources``: Ranks of processes for which the calling process is a destination (array of non-negative integers).
+* ``sourceweights``: Weights of the edges into the calling process (array of non-negative integers).
+* ``outdegree``: Size of *destinations* and *destweights* arrays (non-negative integer).
+* ``destinations``: Ranks of processes for which the calling process is a source (array of non-negative integers).
+* ``destweights``: Weights of the edges out of the calling process (array of non-negative integers).
+* ``info``: Hints on optimization and interpretation of weights (handle).
+* ``reorder``: Ranking may be reordered (true) or not (false) (logical).
+
+OUTPUT PARAMETERS
+-----------------
+* ``comm_dist_graph``: Communicator with distributed graph topology added (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Dist_graph_create_adjacent` creates a new communicator
+*comm_dist_graph* with distributed graph topology and returns a handle
+to the new communicator. The number of processes in *comm_dist_graph* is
+identical to the number of processes in *comm_old*. Each process passes
+all information about its incoming and outgoing edges in the virtual
+distributed graph topology. The calling processes must ensure that each
+edge of the graph is described in the source and in the destination
+process with the same weights. If there are multiple edges for a given
+(source,dest) pair, then the sequence of the weights of these edges does
+not matter. The complete communication topology is the combination of
+all edges shown in the *sources* arrays of all processes in comm_old,
+which must be identical to the combination of all edges shown in the
+*destinations* arrays. Source and destination ranks must be process
+ranks of comm_old. This allows a fully distributed specification of the
+communication graph. Isolated processes (i.e., processes with no
+outgoing or incoming edges, that is, processes that have specified
+indegree and outdegree as zero and thus do not occur as source or
+destination rank in the graph specification) are allowed. The call to
+:ref:`MPI_Dist_graph_create_adjacent` is collective.
+
+
+WEIGHTS
+-------
+
+Weights are specified as non-negative integers and can be used to
+influence the process remapping strategy and other internal MPI
+optimizations. For instance, approximate count arguments of later
+communication calls along specific edges could be used as their edge
+weights. Multiplicity of edges can likewise indicate more intense
+communication between pairs of processes. However, the exact meaning of
+edge weights is not specified by the MPI standard and is left to the
+implementation. An application can supply the special value
+MPI_UNWEIGHTED for the weight array to indicate that all edges have the
+same (effectively no) weight. It is erroneous to supply MPI_UNWEIGHTED
+for some but not all processes of comm_old. If the graph is weighted but
+*indegree* or *outdegree* is zero, then MPI_WEIGHTS_EMPTY or any
+arbitrary array may be passed to sourceweights or destweights
+respectively. Note that MPI_UNWEIGHTED and MPI_WEIGHTS_EMPTY are not
+special weight values; rather they are special values for the total
+array argument. In Fortran, MPI_UNWEIGHTED and MPI_WEIGHTS_EMPTY are
+objects like MPI_BOTTOM (not usable for initialization or assignment).
+See MPI-3 section 2.5.4.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Dist_graph_create`
+   * :ref:`MPI_Dist_graph_neighbors`
+   * :ref:`MPI_Dist_graph_neighbors_count`
diff --git a/docs/man-openmpi/man3/MPI_Dist_graph_neighbors.3.rst b/docs/man-openmpi/man3/MPI_Dist_graph_neighbors.3.rst
new file mode 100644
index 00000000000..9cc27abc9cc
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Dist_graph_neighbors.3.rst
@@ -0,0 +1,50 @@
+.. _mpi_dist_graph_neighbors:
+
+
+MPI_Dist_graph_neighbors
+========================
+
+.. include_body
+
+:ref:`MPI_Dist_graph_neighbors` |mdash| Returns the neighbors of the calling
+process in a distributed graph topology.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_dist_graph_neighbors.rst
+
+INPUT PARAMETERS
+----------------
+* ``comm``: Communicator with distributed graph topology (handle).
+* ``maxindegree``: Size of *sources* and *sourceweights* arrays (non-negative integer).
+* ``maxoutdegree``: Size of *destinations* and *destweights* arrays (non-negative integer).
+
+OUTPUT PARAMETERS
+-----------------
+* ``sources``: Processes for which the calling process is a destination (array of non-negative integers).
+* ``sourceweights``: Weights of the edges into the calling process (array of non-negative integers).
+* ``destinations``: Processes for which the calling process is a source (array of non-negative integers).
+* ``destweights``: Weights of the edges out of the calling process (array of non-negative integers).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Dist_graph_neighbors` returns the source and destination ranks in a
+distributed graph topology for the calling process. This call will
+return up to *maxindegree* source ranks in the *sources* array and up to
+*maxoutdegree* destination ranks in the *destinations* array. If weights
+were specified at the time of the communicator's creation then the
+associated weights are returned in the *sourceweights* and *destweights*
+arrays. If the communicator was created with
+:ref:`MPI_Dist_graph_create_adjacent` then the order of the values in *sources*
+and *destinations* is identical to the input that was used by the
+process with the same rank in comm_old in the creation call.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Dist_graph_neighbors_count`
diff --git a/docs/man-openmpi/man3/MPI_Dist_graph_neighbors_count.3.rst b/docs/man-openmpi/man3/MPI_Dist_graph_neighbors_count.3.rst
new file mode 100644
index 00000000000..daaef2f391d
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Dist_graph_neighbors_count.3.rst
@@ -0,0 +1,42 @@
+.. _mpi_dist_graph_neighbors_count:
+
+
+MPI_Dist_graph_neighbors_count
+==============================
+
+.. include_body
+
+:ref:`MPI_Dist_graph_neighbors_count` |mdash| Returns the number of in and out
+edges for the calling processes in a distributed graph topology and a
+flag indicating whether the distributed graph is weighted.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_dist_graph_neighbors_count.rst
+
+INPUT PARAMETERS
+----------------
+* ``comm``: Communicator with distributed graph topology (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``indegree``: Number of edges into this process (non-negative integer).
+* ``outdegree``: Number of edges out of this process (non-negative integer).
+* ``weighted``: False if MPI_UNWEIGHTED was supplied during creation, true otherwise (logical).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Dist_graph_neighbors_count` and :ref:`MPI_Graph_neighbors` provide adjacency
+information for a distributed graph topology.
+:ref:`MPI_Dist_graph_neighbors_count` returns the number of sources and
+destinations for the calling process.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Dist_graph_neighbors`
diff --git a/docs/man-openmpi/man3/MPI_Errhandler_c2f.3.rst b/docs/man-openmpi/man3/MPI_Errhandler_c2f.3.rst
new file mode 100644
index 00000000000..4eabf92df8b
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Errhandler_c2f.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_errhandler_c2f:
+
+MPI_Errhandler_c2f
+==================
+
+.. include_body
+
+.. include:: ../man3/MPI_Comm_f2c.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_Errhandler_create.3.rst b/docs/man-openmpi/man3/MPI_Errhandler_create.3.rst
new file mode 100644
index 00000000000..61729ec02bc
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Errhandler_create.3.rst
@@ -0,0 +1,103 @@
+.. _mpi_errhandler_create:
+
+
+MPI_Errhandler_create
+=====================
+
+.. include_body
+
+:ref:`MPI_Errhandler_create` |mdash| Creates an MPI-style error handler |mdash| |deprecated_favor| :ref:`MPI_Comm_create_errhandler`.
+
+
+
+SYNTAX
+------
+
+.. NOTE: The bindings for this man page are not automatically
+   generated from the official MPI Forum JSON/Python library because
+   this function is deprecated.  Hence, this function is not included
+   in the MPI Forum JSON data, and we therefore have to hard-code the
+   bindings here ourselves.
+
+C Syntax
+^^^^^^^^
+
+.. code-block:: c
+
+   #include <mpi.h>
+
+   int MPI_Errhandler_create(MPI_Handler_function *function,
+   	MPI_Errhandler *errhandler)
+
+
+Fortran Syntax
+^^^^^^^^^^^^^^
+
+.. code-block:: fortran
+
+   USE MPI
+   ! or the older form: INCLUDE 'mpif.h'
+
+   MPI_ERRHANDLER_CREATE(FUNCTION, ERRHANDLER, IERROR)
+   	EXTERNAL	FUNCTION
+   	INTEGER	ERRHANDLER, IERROR
+
+
+INPUT PARAMETER
+---------------
+
+* ``function``: User-defined error handling procedure.
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``errhandler``: MPI error handler (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Note that use of this routine is *deprecated* as of MPI-2. Please use
+:ref:`MPI_Comm_create_errhandler` instead.
+
+Registers the user routine function for use as an MPI error handler.
+Returns in errhandler a handle to the registered error handler.
+
+In the C language, the user routine should be a C function of type
+MPI_Handler_function, which is defined as
+
+.. code-block:: c
+
+       typedef void (MPI_Handler_function)(MPI_Comm *, int *, ...);
+
+The first argument is the communicator in use. The second is the error
+code to be returned by the MPI routine that raised the error. If the
+routine would have returned MPI_ERR_IN_STATUS, it is the error code
+returned in the status for the request that caused the error handler to
+be invoked. The remaining arguments are stdargs arguments whose number
+and meaning is implementation-dependent. An implementation should
+clearly document these arguments. Addresses are used so that the handler
+may be written in Fortran.
+
+
+NOTE
+----
+
+The `MPI Standard <https://www.mpi-forum.org/docs/>`_ states that an
+implementation may make the output
+value (errhandler) simply the address of the function. However, the
+action of :ref:`MPI_Errhandler_free` makes this impossible, since it is
+required to set the value of the argument to ``MPI_ERRHANDLER_NULL``. In
+addition, the actual error handler must remain until all communicators
+that use it are freed.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_create_errhandler`
+   * :ref:`MPI_Comm_get_errhandler`
+   * :ref:`MPI_Comm_set_errhandler`
diff --git a/docs/man-openmpi/man3/MPI_Errhandler_f2c.3.rst b/docs/man-openmpi/man3/MPI_Errhandler_f2c.3.rst
new file mode 100644
index 00000000000..e2b5c495eac
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Errhandler_f2c.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_errhandler_f2c:
+
+MPI_Errhandler_f2c
+==================
+
+.. include_body
+
+.. include:: ../man3/MPI_Comm_f2c.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_Errhandler_free.3.rst b/docs/man-openmpi/man3/MPI_Errhandler_free.3.rst
new file mode 100644
index 00000000000..d7993bad695
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Errhandler_free.3.rst
@@ -0,0 +1,39 @@
+.. _mpi_errhandler_free:
+
+
+MPI_Errhandler_free
+===================
+
+.. include_body
+
+:ref:`MPI_Errhandler_free` |mdash| Frees an MPI-style error handler.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_errhandler_free.rst
+
+INPUT PARAMETER
+---------------
+* ``errhandler``: MPI error handler (handle). Set to MPI_ERRHANDLER_NULL on exit.
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Marks the error handler associated with errhandler for deallocation and
+sets errhandler to MPI_ERRHANDLER_NULL. The error handler will be
+deallocated after all communicators associated with it have been
+deallocated.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_create_errhandler`
+   * :ref:`MPI_Comm_get_errhandler`
+   * :ref:`MPI_Comm_set_errhandler`
diff --git a/docs/man-openmpi/man3/MPI_Errhandler_get.3.rst b/docs/man-openmpi/man3/MPI_Errhandler_get.3.rst
new file mode 100644
index 00000000000..4c2e3d4a83c
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Errhandler_get.3.rst
@@ -0,0 +1,67 @@
+.. _mpi_errhandler_get:
+
+
+MPI_Errhandler_get
+==================
+
+.. include_body
+
+:ref:`MPI_Errhandler_get` |mdash| Gets the error handler for a communicator |mdash| |deprecated_favor| :ref:`MPI_Comm_get_errhandler`.
+
+
+SYNTAX
+------
+
+.. NOTE: The bindings for this man page are not automatically
+   generated from the official MPI Forum JSON/Python library because
+   this function is deprecated.  Hence, this function is not included
+   in the MPI Forum JSON data, and we therefore have to hard-code the
+   bindings here ourselves.
+
+C Syntax
+^^^^^^^^
+
+.. code-block:: c
+
+   #include <mpi.h>
+
+   int MPI_Errhandler_get(MPI_Comm comm, MPI_Errhandler *errhandler)
+
+
+Fortran Syntax
+^^^^^^^^^^^^^^
+
+.. code-block:: fortran
+
+   USE MPI
+   ! or the older form: INCLUDE 'mpif.h'
+
+   MPI_ERRHANDLER_GET(COMM, ERRHANDLER, IERROR)
+   	INTEGER	COMM, ERRHANDLER, IERROR
+
+
+INPUT PARAMETER
+---------------
+* ``comm``: Communicator to get the error handler from (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``errhandler``: MPI error handler currently associated with communicator (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Note that use of this routine is *deprecated* as of MPI-2. Please use
+:ref:`MPI_Comm_get_errhandler` instead.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_create_errhandler`
+   * :ref:`MPI_Comm_get_errhandler`
+   * :ref:`MPI_Comm_set_errhandler`
diff --git a/docs/man-openmpi/man3/MPI_Errhandler_set.3.rst b/docs/man-openmpi/man3/MPI_Errhandler_set.3.rst
new file mode 100644
index 00000000000..8fd10de9e6e
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Errhandler_set.3.rst
@@ -0,0 +1,71 @@
+.. _mpi_errhandler_set:
+
+
+MPI_Errhandler_set
+==================
+
+.. include_body
+
+:ref:`MPI_Errhandler_set` |mdash| Sets the error handler for a communicator |mdash| |deprecated_favor| :ref:`MPI_Comm_set_errhandler`.
+
+
+SYNTAX
+------
+
+.. NOTE: The bindings for this man page are not automatically
+   generated from the official MPI Forum JSON/Python library because
+   this function is deprecated.  Hence, this function is not included
+   in the MPI Forum JSON data, and we therefore have to hard-code the
+   bindings here ourselves.
+
+C Syntax
+^^^^^^^^
+
+.. code-block:: c
+
+   #include <mpi.h>
+
+   int MPI_Errhandler_set(MPI_Comm comm, MPI_Errhandler errhandler)
+
+
+Fortran Syntax
+^^^^^^^^^^^^^^
+
+.. code-block:: fortran
+
+   USE MPI
+   ! or the older form: INCLUDE 'mpif.h'
+
+   MPI_ERRHANDLER_SET(COMM, ERRHANDLER, IERROR)
+   	INTEGER	COMM, ERRHANDLER, IERROR
+
+
+INPUT PARAMETERS
+----------------
+* ``comm``: Communicator to set the error handler for (handle).
+* ``errhandler``: New MPI error handler for communicator (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Note that use of this routine is *deprecated* as of MPI-2. Please use
+:ref:`MPI_Comm_set_errhandler` instead.
+
+Associates the new error handler errhandler with communicator comm at
+the calling process. Note that an error handler is always associated
+with the communicator.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_create_errhandler`
+   * :ref:`MPI_Comm_get_errhandler`
+   * :ref:`MPI_Comm_set_errhandler`
diff --git a/docs/man-openmpi/man3/MPI_Error_class.3.rst b/docs/man-openmpi/man3/MPI_Error_class.3.rst
new file mode 100644
index 00000000000..0ad41e1571a
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Error_class.3.rst
@@ -0,0 +1,36 @@
+.. _mpi_error_class:
+
+
+MPI_Error_class
+===============
+
+.. include_body
+
+:ref:`MPI_Error_class` |mdash| Converts an error code into an error class.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_error_class.rst
+
+INPUT PARAMETER
+---------------
+* ``errorcode``: Error code returned by an MPI routine.
+
+OUTPUT PARAMETERS
+-----------------
+* ``errorclass``: Error class associated with errorcode.
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The function :ref:`MPI_Error_class` maps each standard error code (error class)
+onto itself.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Error_string`
diff --git a/docs/man-openmpi/man3/MPI_Error_string.3.rst b/docs/man-openmpi/man3/MPI_Error_string.3.rst
new file mode 100644
index 00000000000..39b8f6f2d45
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Error_string.3.rst
@@ -0,0 +1,41 @@
+.. _mpi_error_string:
+
+
+MPI_Error_string
+================
+
+.. include_body
+
+:ref:`MPI_Error_string` |mdash| Returns a string for a given error code.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_error_string.rst
+
+INPUT PARAMETER
+---------------
+* ``errorcode``: Error code returned by an MPI routine or an MPI error class.
+
+OUTPUT PARAMETERS
+-----------------
+* ``string``: Text that corresponds to the errorcode.
+* ``resultlen``: Length of string.
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Returns the error string associated with an error code or class. The
+argument string must represent storage that is at least
+MPI_MAX_ERROR_STRING characters long.
+
+The number of characters actually written is returned in the output
+argument, resultlen.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Error_class`
diff --git a/docs/man-openmpi/man3/MPI_Errors.3.rst b/docs/man-openmpi/man3/MPI_Errors.3.rst
new file mode 100644
index 00000000000..bb8f99fcf97
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Errors.3.rst
@@ -0,0 +1,351 @@
+.. _mpi_errors:
+.. _open-mpi-errors:
+
+MPI Errors
+==========
+
+.. include_body
+
+
+All MPI routines (except :ref:`MPI_Wtime` and :ref:`MPI_Wtick`) return an
+error value; C routines as the value of the function and Fortran
+routines in the last argument. Before the value is returned, the current
+MPI error handler is called. By default, this error handler aborts the
+MPI job. The error handler may be changed with
+:ref:`MPI_Comm_set_errhandler`; the predefined error handler
+MPI_ERRORS_RETURN may be used to cause error values to be returned.
+Note that MPI does not guarantee that an MPI program can continue past
+an error.
+
+For more information on Open MPI error codes, see ``mpi.h`` in the
+``include`` directory.
+
+Standard error return classes for Open MPI:
+
+.. list-table::
+   :header-rows: 1
+
+   * - Error name
+     - Value
+     - Description
+
+   * - MPI_SUCCESS
+     - 0
+     - Successful return code.
+
+   * - MPI_ERR_BUFFER
+     - 1
+     - Invalid buffer pointer.
+
+   * - MPI_ERR_COUNT
+     - 2
+     - Invalid count argument.
+
+   * - MPI_ERR_TYPE
+     - 3
+     - Invalid datatype argument.
+
+   * - MPI_ERR_TAG
+     - 4
+     - Invalid tag argument.
+
+   * - MPI_ERR_COMM
+     - 5
+     - Invalid communicator.
+
+   * - MPI_ERR_RANK
+     - 6
+     - Invalid rank.
+
+   * - MPI_ERR_REQUEST
+     - 7
+     - Invalid MPI_Request handle.
+
+   * - MPI_ERR_ROOT
+     - 8
+     - Invalid root.
+
+   * - MPI_ERR_GROUP
+     - 9
+     - Null group passed to function.
+
+   * - MPI_ERR_OP
+     - 10
+     - Invalid operation.
+
+   * - MPI_ERR_TOPOLOGY
+     - 11
+     - Invalid topology.
+
+   * - MPI_ERR_DIMS
+     - 12
+     - Illegal dimension argument.
+
+   * - MPI_ERR_ARG
+     - 13
+     - Invalid argument.
+
+   * - MPI_ERR_UNKNOWN
+     - 14
+     - Unknown error.
+
+   * - MPI_ERR_TRUNCATE
+     - 15
+     - Message truncated on receive.
+
+   * - MPI_ERR_OTHER
+     - 16
+     - Other error; use Error_string.
+
+   * - MPI_ERR_INTERN
+     - 17
+     - Internal error code.
+
+   * - MPI_ERR_IN_STATUS
+     - 18
+     - Look in status for error value.
+
+   * - MPI_ERR_PENDING
+     - 19
+     - Pending request.
+
+   * - MPI_ERR_ACCESS
+     - 20
+     - Permission denied.
+
+   * - MPI_ERR_AMODE
+     - 21
+     - Unsupported amode passed to open.
+
+   * - MPI_ERR_ASSERT
+     - 22
+     - Invalid assert.
+
+   * - MPI_ERR_BAD_FILE
+     - 23
+     - Invalid file name (for example, path name too long).
+
+   * - MPI_ERR_BASE
+     - 24
+     - Invalid base.
+
+   * - MPI_ERR_CONVERSION
+     - 25
+     - An error occurred in a user-supplied data-conversion function.
+
+   * - MPI_ERR_DISP
+     - 26
+     - Invalid displacement.
+
+   * - MPI_ERR_DUP_DATAREP
+     - 27
+     - Conversion functions could not be registered because a data
+       representation identifier that was already defined was passed
+       to :ref:`MPI_Register_datarep`.
+
+   * - MPI_ERR_FILE_EXISTS
+     - 28
+     - File exists.
+
+
+   * - MPI_ERR_FILE_IN_USE
+     - 29
+     - File operation could not be completed, as the file is currently
+       open by some process.
+
+   * - MPI_ERR_FILE
+     - 30
+     - Invalid file handle.
+
+   * - MPI_ERR_INFO_KEY
+     - 31
+     - Illegal info key.
+
+   * - MPI_ERR_INFO_NOKEY
+     - 32
+     - No such key.
+
+   * - MPI_ERR_INFO_VALUE
+     - 33
+     - Illegal info value.
+
+   * - MPI_ERR_INFO
+     - 34
+     - Invalid info object.
+
+   * - MPI_ERR_IO
+     - 35
+     - I/O error.
+
+   * - MPI_ERR_KEYVAL
+     - 36
+     - Illegal key value.
+
+   * - MPI_ERR_LOCKTYPE
+     - 37
+     - Invalid locktype.
+
+   * - MPI_ERR_NAME
+     - 38
+     - Name not found.
+
+   * - MPI_ERR_NO_MEM
+     - 39
+     - Memory exhausted.
+
+   * - MPI_ERR_NOT_SAME
+     - 40
+     - Collective argument not identical on all processes, or
+       collective routines called in a different order by different
+       processes.
+
+   * - MPI_ERR_NO_SPACE
+     - 41
+     - Not enough space.
+
+   * - MPI_ERR_NO_SUCH_FILE
+     - 42
+     - File (or directory) does not exist.
+
+   * - MPI_ERR_PORT
+     - 43
+     - Invalid port.
+
+   * - MPI_ERR_PROC_ABORTED
+     - 74
+     - Operation failed because a remote peer has aborted.
+
+   * - MPI_ERR_QUOTA
+     - 44
+     - Quota exceeded.
+
+   * - MPI_ERR_READ_ONLY
+     - 45
+     - Read-only file system.
+
+   * - MPI_ERR_RMA_CONFLICT
+     - 46
+     - Conflicting accesses to window.
+
+   * - MPI_ERR_RMA_SYNC
+     - 47
+     - Erroneous RMA synchronization.
+
+   * - MPI_ERR_SERVICE
+     - 48
+     - Invalid publish/unpublish.
+
+   * - MPI_ERR_SIZE
+     - 49
+     - Invalid size.
+
+   * - MPI_ERR_SPAWN
+     - 50
+     - Error spawning.
+
+   * - MPI_ERR_UNSUPPORTED_DATAREP
+     - 51
+     - Unsupported datarep passed to :ref:`MPI_File_set_view`.
+
+   * - MPI_ERR_UNSUPPORTED_OPERATION
+     - 52
+     - Unsupported operation, such as seeking on a file that supports
+       only sequential access.
+
+   * - MPI_ERR_WIN
+     - 53
+     - Invalid window.
+
+   * - MPI_T_ERR_MEMORY
+     - 54
+     - Out of memory.
+
+   * - MPI_T_ERR_NOT_INITIALIZED
+     - 55
+     - Interface not initialized.
+
+   * - MPI_T_ERR_CANNOT_INIT
+     - 56
+     - Interface not in the state to be initialized.
+
+   * - MPI_T_ERR_INVALID_INDEX
+     - 57
+     - The enumeration index is invalid.
+
+   * - MPI_T_ERR_INVALID_ITEM
+     - 8
+     - The item index queried is out of range.
+
+   * - MPI_T_ERR_INVALID_HANDLE
+     - 59
+     - The handle is invalid.
+
+   * - MPI_T_ERR_OUT_OF_HANDLES
+     - 60
+     - No more handles available.
+
+   * - MPI_T_ERR_OUT_OF_SESSIONS
+     - 61
+     - No more sessions available.
+
+   * - MPI_T_ERR_INVALID_SESSION
+     - 62
+     - Session argument is not a valid session.
+
+   * - MPI_T_ERR_CVAR_SET_NOT_NOW
+     - 63
+     - Variable cannot be set at this moment.
+
+   * - MPI_T_ERR_CVAR_SET_NEVER
+     - 64
+     - Variable cannot be set until end of execution.
+
+   * - MPI_T_ERR_PVAR_NO_STARTSTOP
+     - 65
+     - Variable cannot be started or stopped.
+
+   * - MPI_T_ERR_PVAR_NO_WRITE
+     - 6
+     - Variable cannot be written or reset.
+
+   * - MPI_T_ERR_PVAR_NO_ATOMIC
+     - 67
+     - Variable cannot be read and written atomically.
+
+   * - MPI_ERR_RMA_RANGE
+     - 68
+     - Target memory is not part of the window (in the case of a
+       window created with :ref:`MPI_Win_create_dynamic`, target memory is not attached.
+
+   * - MPI_ERR_RMA_ATTACH
+     - 69
+     - Memory cannot be attached (e.g., because of resource
+       exhaustion).
+
+   * - MPI_ERR_RMA_FLAVOR
+     - 70
+     - Passed window has the wrong flavor for the called function.
+
+   * - MPI_ERR_RMA_SHARED
+     - 71
+     - Memory cannot be shared (e.g., some process in the group of the
+       specified communicator cannot expose shared memory).
+
+   * - MPI_T_ERR_INVALID
+     - 72
+     - Invalid use of the interface or bad parameter values(s).
+
+   * - MPI_T_ERR_INVALID_NAME
+     - 73
+     - The variable or category name is invalid.
+
+   * - MPI_ERR_SESSION
+     - 78
+     - Invalid session
+
+   * - MPI_ERR_LASTCODE
+     - 93
+     - Last error code.
+
+.. seealso:: :ref:`MPI_T` :ref:`mpirun(1) <man1-mpirun>` :ref:`mpiexec(1)
+             <man1-mpiexec>` :ref:`ompi_info(1) <man1-ompi_info>`
diff --git a/docs/man-openmpi/man3/MPI_Exscan.3.rst b/docs/man-openmpi/man3/MPI_Exscan.3.rst
new file mode 100644
index 00000000000..84a8d5bf1b8
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Exscan.3.rst
@@ -0,0 +1,103 @@
+.. _mpi_exscan:
+
+
+MPI_Exscan
+==========
+
+.. include_body
+
+:ref:`MPI_Exscan`, :ref:`MPI_Iexscan`, :ref:`MPI_Exscan_init` - Computes an exclusive scan (partial
+reduction)
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Exscan, MPI_Iexscan, MPI_Exscan_init
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_exscan.rst
+
+INPUT PARAMETERS
+----------------
+* ``sendbuf``: Send buffer (choice).
+* ``count``: Number of elements in input buffer (integer).
+* ``datatype``: Data type of elements of input buffer (handle).
+* ``op``: Operation (handle).
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``recvbuf``: Receive buffer (choice).
+* ``request``: Request (handle, non-blocking and persistent only).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Exscan` is used to perform an exclusive prefix reduction on data
+distributed across the calling processes. The operation returns, in the
+*recvbuf* of the process with rank i, the reduction (calculated
+according to the function *op*) of the values in the *sendbuf*\ s of
+processes with ranks 0, ..., i-1. Compare this with the functionality of
+:ref:`MPI_Scan`, which calculates over the range 0, ..., i (inclusive). The
+type of operations supported, their semantics, and the constraints on
+send and receive buffers are as for :ref:`MPI_Reduce`.
+
+The value in *recvbuf* on process 0 is undefined and unreliable as
+*recvbuf* is not significant for process 0. The value of *recvbuf* on
+process 1 is always the value in *sendbuf* on process 0.
+
+
+USE OF IN-PLACE OPTION
+----------------------
+
+The \`in place' option for intracommunicators is specified by passing
+MPI_IN_PLACE in the *sendbuf* argument. In this case, the input data is
+taken from the receive buffer, and replaced by the output data.
+
+Note that MPI_IN_PLACE is a special kind of value; it has the same
+restrictions on its use as MPI_BOTTOM.
+
+Because the in-place option converts the receive buffer into a
+send-and-receive buffer, a Fortran binding that includes INTENT must
+mark these as INOUT, not OUT.
+
+
+NOTES
+-----
+
+MPI does not specify which process computes which operation. In
+particular, both processes 0 and 1 may participate in the computation
+even though the results for both processes' *recvbuf* are degenerate.
+Therefore, all processes, including 0 and 1, must provide the same *op*.
+
+It can be argued, from a mathematical perspective, that the definition
+of :ref:`MPI_Exscan` is unsatisfactory because the output at process 0 is
+undefined. The "mathematically correct" output for process 0 would be
+the unit element of the reduction operation. However, such a definition
+of an exclusive scan would not work with user-defined *op* functions as
+there is no way for MPI to "know" the unit value for these custom
+operations.
+
+
+NOTES ON COLLECTIVE OPERATIONS
+------------------------------
+
+The reduction functions of type MPI_Op do not return an error value. As
+a result, if the functions detect an error, all they can do is either
+call :ref:`MPI_Abort` or silently skip the problem. Thus, if the error handler
+is changed from MPI_ERRORS_ARE_FATAL to something else (e.g.,
+MPI_ERRORS_RETURN), then no error may be indicated.
+
+The reason for this is the performance problems in ensuring that all
+collective routines return the same error value.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Op_create`
+   * :ref:`MPI_Reduce`
+   * :ref:`MPI_Scan`
diff --git a/docs/man-openmpi/man3/MPI_Exscan_init.3.rst b/docs/man-openmpi/man3/MPI_Exscan_init.3.rst
new file mode 100644
index 00000000000..e0f5f6da016
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Exscan_init.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_exscan_init:
+
+MPI_Exscan_init
+===============
+
+.. include_body
+
+.. include:: ../man3/MPI_Exscan.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_Fetch_and_op.3.rst b/docs/man-openmpi/man3/MPI_Fetch_and_op.3.rst
new file mode 100644
index 00000000000..48b163f0eee
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Fetch_and_op.3.rst
@@ -0,0 +1,70 @@
+.. _mpi_fetch_and_op:
+
+
+MPI_Fetch_and_op
+================
+
+.. include_body
+
+:ref:`MPI_Fetch_and_op` |mdash| Combines the contents of the origin buffer with
+that of a target buffer and returns the target buffer value.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_fetch_and_op.rst
+
+INPUT PARAMETERS
+----------------
+* ``origin_addr``: Initial address of buffer (choice).
+* ``result_addr``: Initial address of result buffer (choice).
+* ``datatype``: Data type of the entry in origin, result, and target buffers (handle).
+* ``target_rank``: Rank of target (nonnegative integer).
+* ``target_disp``: Displacement from start of window to beginning of target buffer (nonnegative integer).
+* ``op``: Reduce operation (handle).
+* ``win``: Window object (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Accumulate one element of type *datatype* from the origin buffer
+(*origin_addr*) to the buffer at offset *target_disp*, in the target
+window specified by *target_rank* and *win*, using the operation *op*
+and return in the result buffer *result_addr* the contents of the target
+buffer before the accumulation.
+
+The origin and result buffers (*origin_addr* and *result_addr*) must be
+disjoint. Any of the predefined operations for MPI_Rreduce, as well
+as MPI_NO_OP or MPI_REPLACE, can be specified as *op*; user-defined
+functions cannot be used. The *datatype* argument must be a predefined
+datatype. The operation is executed atomically.
+
+A new predefined operation, MPI_REPLACE, is defined. It corresponds to
+the associative function f(a, b) =b; that is, the current value in the
+target memory is replaced by the value supplied by the origin.
+
+A new predefined operation, MPI_NO_OP, is defined. It corresponds to the
+assiciative function f(a, b) = a; that is the current value in the
+target memory is returned in the result buffer at the origin and no
+operation is performed on the target buffer.
+
+
+NOTES
+-----
+
+It is the user's responsibility to guarantee that, when using the
+accumulate functions, the target displacement argument is such that
+accesses to the window are properly aligned according to the data type
+arguments in the call to the :ref:`MPI_Fetch_and_op` function.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Get_accumulate`
+   * :ref:`MPI_Reduce`
diff --git a/docs/man-openmpi/man3/MPI_File_c2f.3.rst b/docs/man-openmpi/man3/MPI_File_c2f.3.rst
new file mode 100644
index 00000000000..4dcf7479774
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_c2f.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_file_c2f:
+
+MPI_File_c2f
+============
+
+.. include_body
+
+.. include:: ../man3/MPI_Comm_f2c.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_File_call_errhandler.3.rst b/docs/man-openmpi/man3/MPI_File_call_errhandler.3.rst
new file mode 100644
index 00000000000..8a6a699b593
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_call_errhandler.3.rst
@@ -0,0 +1,43 @@
+.. _mpi_file_call_errhandler:
+
+
+MPI_File_call_errhandler
+========================
+
+.. include_body
+
+:ref:`MPI_File_call_errhandler` |mdash| Passes the supplied error code to the
+error handler assigned to a file
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_call_errhandler.rst
+
+INPUT PARAMETERS
+----------------
+* ``fh``: file with error handler (handle).
+* ``errorcode``: MPI error code (integer).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This function invokes the error handler assigned to the file handle *fh*
+with the supplied error code *errorcode*. If the error handler was
+successfully called, the process is not aborted, and the error handler
+returns, this function returns MPI_SUCCESS.
+
+Unlike errors on communicators and windows, the default errorhandler for
+files is MPI_ERRORS_RETURN.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_File_create_errhandler`
+   * :ref:`MPI_File_set_errhandler`
diff --git a/docs/man-openmpi/man3/MPI_File_close.3.rst b/docs/man-openmpi/man3/MPI_File_close.3.rst
new file mode 100644
index 00000000000..3820ca7e842
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_close.3.rst
@@ -0,0 +1,34 @@
+.. _mpi_file_close:
+
+
+MPI_File_close
+==============
+
+.. include_body
+
+:ref:`MPI_File_close` |mdash| Closes a file (collective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_close.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``fh``: File handle (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_close` first synchronizes file state, then closes the file
+associated with *fh.* :ref:`MPI_File_close` is a collective routine. The user
+is responsible for ensuring that all outstanding requests associated
+with *fh* have completed before calling :ref:`MPI_File_close`.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_create_errhandler.3.rst b/docs/man-openmpi/man3/MPI_File_create_errhandler.3.rst
new file mode 100644
index 00000000000..7336a8fa5b8
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_create_errhandler.3.rst
@@ -0,0 +1,60 @@
+.. _mpi_file_create_errhandler:
+
+
+MPI_File_create_errhandler
+==========================
+
+.. include_body
+
+:ref:`MPI_File_create_errhandler` |mdash| Creates an MPI-style error handler that
+can be attached to a file.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_create_errhandler.rst
+
+DEPRECATED TYPE NAME NOTE
+-------------------------
+
+MPI-2.2 deprecated the ``MPI_File_errhandler_fn`` type in favor of
+``MPI_File_errhandler_function``. Open MPI supports both
+names (indeed, the ``_fn`` names are typedefs to the ``_function`` names).
+
+
+INPUT PARAMETER
+---------------
+
+* ``function``: User-defined error handling procedure (function).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``errhandler``: MPI error handler (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Comm_create_errhandler` creates an error handler that can be
+attached to file operations. In C, the
+user routine should be a ``function`` of type
+``MPI_File_errhandler_function``, which is defined as
+
+.. code-block:: c
+
+       typedef void (MPI_File_errhandler_function)(MPI_File *, int *, ...);
+
+The first argument is the file in use. The second is the
+error code to be returned by the MPI routine that raised the error.
+
+In the Fortran language, the user routine should be of the form:
+
+.. code-block:: fortran
+
+       SUBROUTINE FILE_ERRHANDLER_FUNCTION(FILE, ERROR_CODE, ...)
+           INTEGER FILE, ERROR_CODE
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_delete.3.rst b/docs/man-openmpi/man3/MPI_File_delete.3.rst
new file mode 100644
index 00000000000..92863c2b05c
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_delete.3.rst
@@ -0,0 +1,36 @@
+.. _mpi_file_delete:
+
+
+MPI_File_delete
+===============
+
+.. include_body
+
+:ref:`MPI_File_delete` |mdash| Deletes a file.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_delete.rst
+
+INPUT PARAMETERS
+----------------
+* ``filename``: Name of file to delete (string).
+* ``info``: Info object (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_delete` deletes the file identified by the file name *filename*,
+provided it is not currently open by any process. It is an error to
+delete the file with :ref:`MPI_File_delete` if some process has it open, but
+:ref:`MPI_File_delete` does not check this. If the file does not exist,
+:ref:`MPI_File_delete` returns an error in the class MPI_ERR_NO_SUCH_FILE.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_f2c.3.rst b/docs/man-openmpi/man3/MPI_File_f2c.3.rst
new file mode 100644
index 00000000000..80b2216bbce
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_f2c.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_file_f2c:
+
+MPI_File_f2c
+============
+
+.. include_body
+
+.. include:: ../man3/MPI_Comm_f2c.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_File_get_amode.3.rst b/docs/man-openmpi/man3/MPI_File_get_amode.3.rst
new file mode 100644
index 00000000000..09b149f012b
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_get_amode.3.rst
@@ -0,0 +1,34 @@
+.. _mpi_file_get_amode:
+
+
+MPI_File_get_amode
+==================
+
+.. include_body
+
+:ref:`MPI_File_get_amode` |mdash| Returns access mode associated with an open
+file.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_get_amode.rst
+
+INPUT PARAMETER
+---------------
+* ``fh``: File handle (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``amode``: File access mode used to open the file (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_get_amode` returns, in *amode,* the access mode associated with
+the open file *fh.*
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_get_atomicity.3.rst b/docs/man-openmpi/man3/MPI_File_get_atomicity.3.rst
new file mode 100644
index 00000000000..c01c69f257d
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_get_atomicity.3.rst
@@ -0,0 +1,36 @@
+.. _mpi_file_get_atomicity:
+
+
+MPI_File_get_atomicity
+======================
+
+.. include_body
+
+:ref:`MPI_File_get_atomicity` |mdash| Returns current consistency semantics for
+data-access operations.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_get_atomicity.rst
+
+INPUT PARAMETER
+---------------
+* ``fh``: File handle (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``flag``: true if atomic mode is enabled, false if nonatomic mode is enabled (boolean).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_get_atomicity` returns the current consistency semantics for
+data access operations on the set of file handles created by one
+collective :ref:`MPI_File_open`. If *flag* is *true,* atomic mode is currently
+enabled; if *flag* is *false,* nonatomic mode is currently enabled.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_get_byte_offset.3.rst b/docs/man-openmpi/man3/MPI_File_get_byte_offset.3.rst
new file mode 100644
index 00000000000..4fda137c252
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_get_byte_offset.3.rst
@@ -0,0 +1,37 @@
+.. _mpi_file_get_byte_offset:
+
+
+MPI_File_get_byte_offset
+========================
+
+.. include_body
+
+:ref:`MPI_File_get_byte_offset` |mdash| Converts a view-relative offset into an
+absolute byte position.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_get_byte_offset.rst
+
+INPUT PARAMETERS
+----------------
+* ``fh``: File handle (handle).
+* ``offset``: Offset (integer).
+
+OUTPUT PARAMETERS
+-----------------
+* ``disp``: Absolute byte position of offset (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_get_byte_offset` converts an offset specified for the current
+view to its corresponding displacement value, or absolute byte position,
+from the beginning of the file. The absolute byte position of *offset*
+relative to the current view of *fh* is returned in *disp*.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_get_errhandler.3.rst b/docs/man-openmpi/man3/MPI_File_get_errhandler.3.rst
new file mode 100644
index 00000000000..09c7c1ccda8
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_get_errhandler.3.rst
@@ -0,0 +1,33 @@
+.. _mpi_file_get_errhandler:
+
+
+MPI_File_get_errhandler
+=======================
+
+.. include_body
+
+:ref:`MPI_File_get_errhandler` |mdash| Gets the error handler for a file.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_get_errhandler.rst
+
+INPUT PARAMETER
+---------------
+* ``file``: File (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``errhandler``: MPI error handler currently associated with file (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Returns in *errhandler* (a handle to) the error handler that is
+currently associated with file *file*.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_get_group.3.rst b/docs/man-openmpi/man3/MPI_File_get_group.3.rst
new file mode 100644
index 00000000000..c682d02d9ef
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_get_group.3.rst
@@ -0,0 +1,36 @@
+.. _mpi_file_get_group:
+
+
+MPI_File_get_group
+==================
+
+.. include_body
+
+:ref:`MPI_File_get_group` |mdash| Returns a duplicate of the process group of a
+file.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_get_group.rst
+
+INPUT PARAMETER
+---------------
+* ``fh``: File handle (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``group``: Group that opened the file (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_get_group` returns a duplicate of the group of the communicator
+used to open the file associated with *fh.* The group is returned in
+*group.* The user is responsible for freeing *group,* using
+:ref:`MPI_Group_free`.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_get_info.3.rst b/docs/man-openmpi/man3/MPI_File_get_info.3.rst
new file mode 100644
index 00000000000..24f7a3f2e86
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_get_info.3.rst
@@ -0,0 +1,109 @@
+.. _mpi_file_get_info:
+
+
+MPI_File_get_info
+=================
+
+.. include_body
+
+:ref:`MPI_File_get_info` |mdash| Returns a new info object containing values for
+current hints associated with a file.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_get_info.rst
+
+INPUT PARAMETER
+---------------
+* ``fh``: File handle (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``info_used``: New info object (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_get_info` returns a new info object containing all the hints
+that the system currently associates with the file *fh*. The current
+setting of all hints actually used by the system related to this open
+file is returned in *info_used*. The user is responsible for freeing
+*info_used* via :ref:`MPI_Info_free`.
+
+Note that the set of hints returned in *info_used* may be greater or
+smaller than the set of hints passed in to :ref:`MPI_File_open`,
+:ref:`MPI_File_set_view`, and :ref:`MPI_File_set_info`, as the system
+may not recognize some hints set by the user, and may automatically
+set other hints that the user has not requested to be set. See the
+:ref:`HINTS section <man-openmpi-mpi-file-get-info>` for a list of
+hints that can be set.
+
+.. _man-openmpi-mpi-file-get-info:
+
+
+HINTS
+-----
+
+The following hints can be used as values for the *info_used* argument.
+
+**SETTABLE HINTS**
+
+* ``shared_file_timeout``: Amount of time (in seconds) to wait for
+  access to the shared file pointer before exiting with
+  ``MPI_ERR_TIMEDOUT``.
+
+* ``rwlock_timeout``: Amount of time (in seconds) to wait for
+  obtaining a read or write lock on a contiguous chunk of a UNIX file
+  before exiting with ``MPI_ERR_TIMEDOUT``.
+
+* ``noncoll_read_bufsize``: Maximum size of the buffer used by MPI I/O
+  to satisfy read requests in the noncollective data-access
+  routines.
+
+  .. note:: A buffer size smaller than the distance (in bytes) in a
+            UNIX file between the first byte and the last byte of the
+            access request causes MPI I/O to iterate and perform
+            multiple UNIX ``read()`` or ``write()`` calls. If the
+            request includes multiple noncontiguous chunks of data,
+            and the buffer size is greater than the size of those
+            chunks, then the UNIX ``read()`` or ``write()`` (made at
+            the MPI I/O level) will access data not requested by this
+            process in order to reduce the total number of ``write()``
+            calls made. If this is not desirable behavior, you should
+            reduce this buffer size to equal the size of the
+            contiguous chunks within the aggregate request.
+
+* ``noncoll_write_bufsize``: Maximum size of the buffer used by MPI
+  I/O to satisfy write requests in the noncollective data-access
+  routines.
+
+  See the above note in ``noncoll_read_bufsize``.
+
+* ``coll_read_bufsize``: Maximum size of the buffer used by MPI I/O to
+  satisfy read requests in the collective data-access routines.
+
+  See the above note in ``noncoll_read_bufsize``.
+
+* ``coll_write_bufsize``: Maximum size of the buffer used by MPI I/O
+  to satisfy write requests in the collective data-access
+  routines.
+
+  See the above note in ``noncoll_read_bufsize``.
+
+* ``mpiio_concurrency``: (boolean) controls whether nonblocking
+  I/O routines can bind an extra thread to an LWP.
+
+* ``mpiio_coll_contiguous``: (boolean) controls whether subsequent
+  collective data accesses will request collectively contiguous
+  regions of the file.
+
+**NON-SETTABLE HINTS**
+
+* ``filename``: Access this hint to get the name of the file.
+
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_get_position.3.rst b/docs/man-openmpi/man3/MPI_File_get_position.3.rst
new file mode 100644
index 00000000000..a56ae1820ed
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_get_position.3.rst
@@ -0,0 +1,35 @@
+.. _mpi_file_get_position:
+
+
+MPI_File_get_position
+=====================
+
+.. include_body
+
+:ref:`MPI_File_get_position` |mdash| Returns the current position of the
+individual file pointer.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_get_position.rst
+
+INPUT PARAMETER
+---------------
+* ``fh``: File handle (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``offset``: Offset of the individual file pointer (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_get_position` returns, in *offset,* the current position of the
+individual file pointer in *etype* units relative to the current
+displacement and file type.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_get_position_shared.3.rst b/docs/man-openmpi/man3/MPI_File_get_position_shared.3.rst
new file mode 100644
index 00000000000..2918aa277df
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_get_position_shared.3.rst
@@ -0,0 +1,35 @@
+.. _mpi_file_get_position_shared:
+
+
+MPI_File_get_position_shared
+============================
+
+.. include_body
+
+:ref:`MPI_File_get_position_shared` |mdash| Returns the current position of the
+shared file pointer.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_get_position_shared.rst
+
+INPUT PARAMETER
+---------------
+* ``fh``: File handle (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``offset``: Offset of the shared file pointer (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_get_position_shared` returns, in *offset,* the current position
+of the shared file pointer in *etype* units relative to the current
+displacement and file type.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_get_size.3.rst b/docs/man-openmpi/man3/MPI_File_get_size.3.rst
new file mode 100644
index 00000000000..d519a1d04c7
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_get_size.3.rst
@@ -0,0 +1,39 @@
+.. _mpi_file_get_size:
+
+
+MPI_File_get_size
+=================
+
+.. include_body
+
+:ref:`MPI_File_get_size` |mdash| Returns the current size of the file.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_get_size.rst
+
+INPUT PARAMETERS
+----------------
+* ``fh``: File handle (handle).
+* ``size``: Size of the file in bytes (integer).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_get_size` returns, in *size* , the current size in bytes of the
+file associated with the file handle *fh*. Note that the file size
+returned by Solaris may not represent the number of bytes physically
+allocated for the file in those cases where all bytes in this file have
+not been written at least once.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_File_preallocate`
diff --git a/docs/man-openmpi/man3/MPI_File_get_type_extent.3.rst b/docs/man-openmpi/man3/MPI_File_get_type_extent.3.rst
new file mode 100644
index 00000000000..0893c7ed56d
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_get_type_extent.3.rst
@@ -0,0 +1,56 @@
+.. _mpi_file_get_type_extent:
+
+
+MPI_File_get_type_extent
+========================
+
+.. include_body
+
+:ref:`MPI_File_get_type_extent` |mdash| Returns the extent of the data type in a
+file.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_get_type_extent.rst
+
+INPUT PARAMETERS
+----------------
+* ``fh``: File handle (handle).
+* ``datatype``: Data type (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``extent``: Data type extent (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_get_type_extent` can be used to calculate *extent* for
+*datatype* in the file. The extent is the same for all processes
+accessing the file associated with *fh*. If the current view uses a
+user-defined data representation, :ref:`MPI_File_get_type_extent` uses the
+*dtype_file_extent_fn* callback to calculate the extent.
+
+
+NOTES
+-----
+
+If the file data representation is other than "native," care must be
+taken in constructing etypes and file types. Any of the data-type
+constructor functions may be used; however, for those functions that
+accept displacements in bytes, the displacements must be specified in
+terms of their values in the file for the file data representation being
+used. MPI will interpret these byte displacements as is; no scaling will
+be done. The function :ref:`MPI_File_get_type_extent` can be used to calculate
+the extents of data types in the file. For etypes and file types that
+are portable data types, MPI will scale any displacements in the data
+types to match the file data representation. Data types passed as
+arguments to read/write routines specify the data layout in memory;
+therefore, they must always be constructed using displacements
+corresponding to displacements in memory.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_get_view.3.rst b/docs/man-openmpi/man3/MPI_File_get_view.3.rst
new file mode 100644
index 00000000000..64ca52e8886
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_get_view.3.rst
@@ -0,0 +1,40 @@
+.. _mpi_file_get_view:
+
+
+MPI_File_get_view
+=================
+
+.. include_body
+
+:ref:`MPI_File_get_view` |mdash| Returns the process's view of data in the file.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_get_view.rst
+
+INPUT PARAMETER
+---------------
+* ``fh``: File handle (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``disp``: Displacement (integer).
+* ``etype``: Elementary data type (handle).
+* ``filetype``: File type (handle). See Restrictions, below.
+* ``datarep``: Data representation (string).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The :ref:`MPI_File_get_view` routine returns the process's view of the data in
+the file. The current values of the displacement, etype, and filetype
+are returned in *disp,* *etype,* and *filetype,* respectively.
+
+The :ref:`MPI_File_get_view` interface allows the user to pass a
+data-representation string via the *datarep* argument.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_iread.3.rst b/docs/man-openmpi/man3/MPI_File_iread.3.rst
new file mode 100644
index 00000000000..64bc22cf341
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_iread.3.rst
@@ -0,0 +1,47 @@
+.. _mpi_file_iread:
+
+
+MPI_File_iread
+==============
+
+.. include_body
+
+:ref:`MPI_File_iread` |mdash| Reads a file starting at the location specified by
+the individual file pointer (nonblocking, noncollective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_iread.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``fh``: File handle (handle).
+
+INPUT PARAMETERS
+----------------
+* ``count``: Number of elements in the buffer (integer).
+* ``datatype``: Data type of each buffer element (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``buf``: Initial address of buffer (choice).
+* ``request``: Request object (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_iread` is a nonblocking version of :ref:`MPI_File_read`. It attempts to
+read from the file associated with *fh* at the current individual file
+pointer position maintained by the system in which a total number of
+*count* data items having *datatype* type are read into the user's
+buffer *buf.* The data is taken out of those parts of the file specified
+by the current view. :ref:`MPI_File_iread` stores the number of data-type
+elements actually read in *status.* All other fields of *status* are
+undefined. It is erroneous to call this function if ``MPI_MODE_SEQUENTIAL``
+mode was specified when the file was opened.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_iread_all.3.rst b/docs/man-openmpi/man3/MPI_File_iread_all.3.rst
new file mode 100644
index 00000000000..f78bd31ff1e
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_iread_all.3.rst
@@ -0,0 +1,47 @@
+.. _mpi_file_iread_all:
+
+
+MPI_File_iread_all
+==================
+
+.. include_body
+
+:ref:`MPI_File_iread_all` |mdash| Reads a file starting at the location specified
+by the individual file pointer (nonblocking, collective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_iread_all.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``fh``: File handle (handle).
+
+INPUT PARAMETERS
+----------------
+* ``count``: Number of elements in the buffer (integer).
+* ``datatype``: Data type of each buffer element (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``buf``: Initial address of buffer (choice).
+* ``request``: Request object (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_iread_all` is a nonblocking version of :ref:`MPI_File_read_all`. It
+attempts to read from the file associated with *fh* at the current
+individual file pointer position maintained by the system in which a
+total number of *count* data items having *datatype* type are read into
+the user's buffer *buf.* The data is taken out of those parts of the
+file specified by the current view. :ref:`MPI_File_iread_all` stores the number
+of data-type elements actually read in *status.* All other fields of
+*status* are undefined. It is erroneous to call this function if
+``MPI_MODE_SEQUENTIAL`` mode was specified when the file was opened.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_iread_at.3.rst b/docs/man-openmpi/man3/MPI_File_iread_at.3.rst
new file mode 100644
index 00000000000..d4d5c5e1046
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_iread_at.3.rst
@@ -0,0 +1,46 @@
+.. _mpi_file_iread_at:
+
+
+MPI_File_iread_at
+=================
+
+.. include_body
+
+:ref:`MPI_File_iread_at` |mdash| Reads a file at an explicitly specified offset
+(nonblocking, noncollective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_iread_at.rst
+
+INPUT PARAMETERS
+----------------
+* ``fh``: File handle (handle).
+* ``offset``: File offset (integer).
+* ``count``: Number of elements in the buffer (integer).
+* ``datatype``: Data type of each buffer element (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``buf``: Initial address of the buffer (choice).
+* ``request``: Request object (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_iread_at` is the nonblocking version of :ref:`MPI_File_read_at`.
+
+:ref:`MPI_File_iread_at` is a nonblocking routine that attempts to read from
+the file associated with *fh* at the *offset* position a total number of
+*count* data items having *datatype* type into the user's buffer *buf.*
+The *offset* is in etype units relative to the current view. That is,
+holes are not counted when locating an offset. The data is taken out of
+those parts of the file specified by the current view. :ref:`MPI_File_iread_at`
+stores the number of *datatype* elements actually read in *status.* All
+other fields of *status* are undefined.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_iread_at_all.3.rst b/docs/man-openmpi/man3/MPI_File_iread_at_all.3.rst
new file mode 100644
index 00000000000..3d7708037b3
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_iread_at_all.3.rst
@@ -0,0 +1,47 @@
+.. _mpi_file_iread_at_all:
+
+
+MPI_File_iread_at_all
+=====================
+
+.. include_body
+
+:ref:`MPI_File_iread_at_all` |mdash| Reads a file at an explicitly specified
+offset (nonblocking, collective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_iread_at_all.rst
+
+INPUT PARAMETERS
+----------------
+* ``fh``: File handle (handle).
+* ``offset``: File offset (integer).
+* ``count``: Number of elements in the buffer (integer).
+* ``datatype``: Data type of each buffer element (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``buf``: Initial address of the buffer (choice).
+* ``request``: Request object (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_iread_at_all` is the nonblocking version of
+:ref:`MPI_File_read_at_all`.
+
+:ref:`MPI_File_iread_at_all` is a nonblocking routine that attempts to read
+from the file associated with *fh* at the *offset* position a total
+number of *count* data items having *datatype* type into the user's
+buffer *buf.* The *offset* is in etype units relative to the current
+view. That is, holes are not counted when locating an offset. The data
+is taken out of those parts of the file specified by the current view.
+:ref:`MPI_File_iread_at_all` stores the number of *datatype* elements actually
+read in *status.* All other fields of *status* are undefined.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_iread_shared.3.rst b/docs/man-openmpi/man3/MPI_File_iread_shared.3.rst
new file mode 100644
index 00000000000..9d514e65486
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_iread_shared.3.rst
@@ -0,0 +1,44 @@
+.. _mpi_file_iread_shared:
+
+
+MPI_File_iread_shared
+=====================
+
+.. include_body
+
+:ref:`MPI_File_iread_shared` |mdash| Reads a file using the shared file pointer
+(nonblocking, noncollective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_iread_shared.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``fh``: File handle (handle).
+
+INPUT PARAMETERS
+----------------
+* ``count``: Number of elements in buffer (integer).
+* ``datatype``: Data type of each buffer element (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``buf``: Initial address of buffer (choice).
+* ``request``: Request object (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_iread_shared` is a nonblocking version of the
+:ref:`MPI_File_read_shared` interface. It uses the shared file pointer to read
+files. The order of serialization among the processors is not
+deterministic for this noncollective routine, so you need to use other
+methods of synchronization to impose a particular order among
+processors.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_iwrite.3.rst b/docs/man-openmpi/man3/MPI_File_iwrite.3.rst
new file mode 100644
index 00000000000..1efed44e6a6
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_iwrite.3.rst
@@ -0,0 +1,49 @@
+.. _mpi_file_iwrite:
+
+
+MPI_File_iwrite
+===============
+
+.. include_body
+
+:ref:`MPI_File_iwrite` |mdash| Writes a file starting at the location specified
+by the individual file pointer (nonblocking, noncollective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_iwrite.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``fh``: File handle (handle).
+
+INPUT PARAMETERS
+----------------
+* ``buf``: Initial address of buffer (choice).
+* ``count``: Number of elements in buffer (integer).
+* ``datatype``: Data type of each buffer element (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``request``: Request object (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_iwrite` is a nonblocking version of the :ref:`MPI_File_write`
+interface. It attempts to write into the file associated with *fh* (at
+the current individual file pointer position maintained by the system) a
+total number of *count* data items having *datatype* type from the
+user's buffer *buf.* The data is written into those parts of the file
+specified by the current view. :ref:`MPI_File_iwrite` stores the number of
+*datatype* elements actually written in *status.* All other fields of
+*status* are undefined.
+
+It is erroneous to call this function if ``MPI_MODE_SEQUENTIAL`` mode was
+specified when the file was open.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_iwrite_all.3.rst b/docs/man-openmpi/man3/MPI_File_iwrite_all.3.rst
new file mode 100644
index 00000000000..bbcd60f1240
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_iwrite_all.3.rst
@@ -0,0 +1,49 @@
+.. _mpi_file_iwrite_all:
+
+
+MPI_File_iwrite_all
+===================
+
+.. include_body
+
+:ref:`MPI_File_iwrite_all` |mdash| Writes a file starting at the location
+specified by the individual file pointer (nonblocking, collective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_iwrite_all.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``fh``: File handle (handle).
+
+INPUT PARAMETERS
+----------------
+* ``buf``: Initial address of buffer (choice).
+* ``count``: Number of elements in buffer (integer).
+* ``datatype``: Data type of each buffer element (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``request``: Request object (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_iwrite_all` is a nonblocking version of the :ref:`MPI_File_write_all`
+interface. It attempts to write into the file associated with *fh* (at
+the current individual file pointer position maintained by the system) a
+total number of *count* data items having *datatype* type from the
+user's buffer *buf.* The data is written into those parts of the file
+specified by the current view. :ref:`MPI_File_iwrite_all` stores the number of
+*datatype* elements actually written in *status.* All other fields of
+*status* are undefined.
+
+It is erroneous to call this function if ``MPI_MODE_SEQUENTIAL`` mode was
+specified when the file was open.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_iwrite_at.3.rst b/docs/man-openmpi/man3/MPI_File_iwrite_at.3.rst
new file mode 100644
index 00000000000..2368bf26b4a
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_iwrite_at.3.rst
@@ -0,0 +1,52 @@
+.. _mpi_file_iwrite_at:
+
+
+MPI_File_iwrite_at
+==================
+
+.. include_body
+
+:ref:`MPI_File_iwrite_at` |mdash| Writes a file at an explicitly specified offset
+(nonblocking, noncollective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_iwrite_at.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``fh``: File handle (handle).
+
+INPUT PARAMETERS
+----------------
+* ``offset``: File offset (integer).
+* ``buf``: Initial address of buffer (choice).
+* ``count``: Number of elements in buffer (integer).
+* ``datatype``: Data type of each buffer element (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``request``: Request object (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_iwrite_at` is a nonblocking version of :ref:`MPI_File_write_at`. It
+attempts to write into the file associated with *fh* (at the *offset*
+position) a total number of *count* data items having *datatype* type
+from the user's buffer *buf.* The offset is in *etype* units relative to
+the current view. That is, holes are not counted when locating an
+offset. The data is written into those parts of the file specified by
+the current view. :ref:`MPI_File_iwrite_at` stores the number of *datatype*
+elements actually written in *status.* All other fields of *status* are
+undefined. The request structure can be passed to :ref:`MPI_Wait` or :ref:`MPI_Test`,
+which will return a status with the number of bytes actually accessed.
+
+It is erroneous to call this function if ``MPI_MODE_SEQUENTIAL`` mode was
+specified when the file was open.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_iwrite_at_all.3.rst b/docs/man-openmpi/man3/MPI_File_iwrite_at_all.3.rst
new file mode 100644
index 00000000000..ac2d9109c3f
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_iwrite_at_all.3.rst
@@ -0,0 +1,53 @@
+.. _mpi_file_iwrite_at_all:
+
+
+MPI_File_iwrite_at_all
+======================
+
+.. include_body
+
+:ref:`MPI_File_iwrite_at_all` |mdash| Writes a file at an explicitly specified
+offset (nonblocking, collective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_iwrite_at_all.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``fh``: File handle (handle).
+
+INPUT PARAMETERS
+----------------
+* ``offset``: File offset (integer).
+* ``buf``: Initial address of buffer (choice).
+* ``count``: Number of elements in buffer (integer).
+* ``datatype``: Data type of each buffer element (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``request``: Request object (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_iwrite_at_all` is a nonblocking version of
+:ref:`MPI_File_write_at_all`. It attempts to write into the file associated
+with *fh* (at the *offset* position) a total number of *count* data
+items having *datatype* type from the user's buffer *buf.* The offset is
+in *etype* units relative to the current view. That is, holes are not
+counted when locating an offset. The data is written into those parts of
+the file specified by the current view. :ref:`MPI_File_iwrite_at_all` stores
+the number of *datatype* elements actually written in *status.* All
+other fields of *status* are undefined. The request structure can be
+passed to :ref:`MPI_Wait` or :ref:`MPI_Test`, which will return a status with the
+number of bytes actually accessed.
+
+It is erroneous to call this function if ``MPI_MODE_SEQUENTIAL`` mode was
+specified when the file was open.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_iwrite_shared.3.rst b/docs/man-openmpi/man3/MPI_File_iwrite_shared.3.rst
new file mode 100644
index 00000000000..d11ec397def
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_iwrite_shared.3.rst
@@ -0,0 +1,42 @@
+.. _mpi_file_iwrite_shared:
+
+
+MPI_File_iwrite_shared
+======================
+
+.. include_body
+
+:ref:`MPI_File_iwrite_shared` |mdash| Writes a file using the shared file pointer
+(nonblocking, noncollective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_iwrite_shared.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``fh``: File handle (handle).
+
+INPUT PARAMETERS
+----------------
+* ``count``: Number of elements in buffer (integer).
+* ``datatype``: Data type of each buffer element (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``buf``: Initial address of buffer (choice).
+* ``request``: Request object (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_iwrite_shared` is a nonblocking routine that uses the shared
+file pointer to write files. The order of serialization is not
+deterministic for this noncollective routine, so you need to use other
+methods of synchronization to impose a particular order.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_open.3.rst b/docs/man-openmpi/man3/MPI_File_open.3.rst
new file mode 100644
index 00000000000..0b0be6cba49
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_open.3.rst
@@ -0,0 +1,150 @@
+.. _mpi_file_open:
+
+
+MPI_File_open
+=============
+
+.. include_body
+
+:ref:`MPI_File_open` |mdash| Opens a file (collective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_open.rst
+
+INPUT PARAMETERS
+----------------
+* ``comm``: Communicator (handle).
+* ``filename``: Name of file to open (string).
+* ``amode``: File access mode (integer).
+* ``info``: Info object (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``fh``: New file handle (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_open` opens the file identified by the filename
+*filename* on all processes in the *comm* communicator
+group. :ref:`MPI_File_open` is a collective routine; all processes
+must provide the same value for *amode,* and all processes must
+provide filenames that reference the same file which are textually
+identical (note: Open MPI I/O plugins may have restrictions on
+characters that can be used in filenames.
+A process can open a file independently of other processes
+by using the MPI_COMM_SELF communicator. The file handle returned,
+*fh,* can be subsequently used to access the file until the file is
+closed using :ref:`MPI_File_close`. Before calling
+:ref:`MPI_Finalize`, the user is required to close (via
+MPI_File_close) all files that were opened with
+:ref:`MPI_File_open`. Note that the communicator *comm* is unaffected
+by :ref:`MPI_File_open` and continues to be usable in all MPI
+routines. Furthermore, use of *comm* will not interfere with I/O
+behavior.
+
+Initially, all processes view the file as a linear byte stream; that is,
+the *etype* and *filetype* are both MPI_BYTE. The file view can be
+changed via the :ref:`MPI_File_set_view` routine.
+
+The following access modes are supported (specified in amode, in a
+bit-vector OR in one of the following integer constants):
+
+* ``MPI_MODE_APPEND``
+* ``MPI_MODE_CREATE``: Create the file if it does not exist.
+* ``MPI_MODE_DELETE_ON_CLOSE``
+* ``MPI_MODE_EXCL``: Error creating a file that already exists.
+* ``MPI_MODE_RDONLY``: Read only.
+* ``MPI_MODE_RDWR``: Reading and writing.
+* ``MPI_MODE_SEQUENTIAL``
+* ``MPI__MODE_WRONLY``: Write only.
+* ``MPI_MODE_UNIQUE_OPEN``
+
+The modes ``MPI_MODE_RDONLY``, ``MPI_MODE_RDWR``, ``MPI_MODE_WRONLY``,
+and ``MPI_MODE_CREATE`` have identical semantics to their POSIX
+counterparts. It is erroneous to specify ``MPI_MODE_CREATE`` in
+conjunction with ``MPI_MODE_RDONLY``. Errors related to the access
+mode are raised in the class ``MPI_ERR_AMODE``.
+
+On single-node clusters, files are opened by default using nonatomic
+mode file consistency semantics. The more stringent atomic-mode
+consistency semantics, required for atomicity of overlapping accesses,
+are the default when processors in a communicator group reside on more
+than one node. This setting can be changed using :ref:`MPI_File_set_atomicity`.
+
+The :ref:`MPI_File_open` interface allows the user to pass information
+via the *info* argument. It can be set to ``MPI_INFO_NULL``. See the
+:ref:`HINTS section <man-openmpi-mpi-file-open-hints>` for a list of
+hints that can be set.
+
+.. _man-openmpi-mpi-file-open-hints:
+
+HINTS
+-----
+
+The following hints can be used as values for the *info* argument.
+
+**SETTABLE HINTS**
+
+* ``MPI_INFO_NULL``
+
+* ``shared_file_timeout``: Amount of time (in seconds) to wait for
+  access to the shared file pointer before exiting with
+  ``MPI_ERR_TIMEDOUT``.
+
+* ``rwlock_timeout``: Amount of time (in seconds) to wait for
+  obtaining a read or write lock on a contiguous chunk of a UNIX file
+  before exiting with ``MPI_ERR_TIMEDOUT``.
+
+* ``noncoll_read_bufsize``: Maximum size of the buffer used by MPI I/O
+  to satisfy multiple noncontiguous read requests in the noncollective
+  data-access routines.
+
+  .. note:: A buffer size smaller than the distance (in bytes) in a
+            UNIX file between the first byte and the last byte of the
+            access request causes MPI I/O to iterate and perform
+            multiple UNIX `read()` or `write()` calls. If the request
+            includes multiple noncontiguous chunks of data, and the
+            buffer size is greater than the size of those chunks, then
+            the UNIX `read()` or `write()` (made at the MPI I/O level)
+            will access data not requested by this process in order to
+            reduce the total number of `write()` calls made. If this
+            is not desirable behavior, you should reduce this buffer
+            size to equal the size of the contiguous chunks within the
+            aggregate request.
+
+* ``noncoll_write_bufsize``: Maximum size of the buffer used by MPI
+  I/O to satisfy multiple noncontiguous write requests in the
+  noncollective data-access routines.
+
+  See the above note in ``noncoll_read_bufsize``.
+
+* ``coll_read_bufsize``: Maximum size of the buffer used by MPI I/O to
+  satisfy multiple noncontiguous read requests in the collective
+  data-access routines.
+
+  See the above note in ``noncoll_read_bufsize``.
+
+* ``coll_write_bufsize``: Maximum size of the buffer used by MPI I/O
+  to satisfy multiple noncontiguous write requests in the collective
+  data-access routines.
+
+  See the above note in ``noncoll_read_bufsize``.
+
+* ``mpiio_concurrency``: (boolean) controls whether nonblocking I/O
+  routines can bind an extra thread to an LWP.  .sp
+
+* ``mpiio_coll_contiguous``: (boolean) controls whether subsequent
+  collective data accesses will request collectively contiguous
+  regions of the file.
+
+**NON-SETTABLE HINTS**
+
+* ``filename``: Access this hint to get the name of the file.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_preallocate.3.rst b/docs/man-openmpi/man3/MPI_File_preallocate.3.rst
new file mode 100644
index 00000000000..8ff77274ec0
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_preallocate.3.rst
@@ -0,0 +1,67 @@
+.. _mpi_file_preallocate:
+
+
+MPI_File_preallocate
+====================
+
+.. include_body
+
+:ref:`MPI_File_preallocate` |mdash| Preallocates a specified amount of storage
+space at the beginning of a file (collective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_preallocate.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``fh``: File handle (handle).
+
+INPUT PARAMETER
+---------------
+* ``size``: Size to preallocate file, in bytes (integer).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_preallocate` ensures that storage space is allocated for the
+first *size* bytes of the file associated with *fh*.
+:ref:`MPI_File_preallocate` can be a very time-consuming operation.
+
+:ref:`MPI_File_preallocate` is collective; all processes in the group must pass
+identical values for *size*. Regions of the file that have previously
+been written are unaffected. For newly allocated regions of the file,
+:ref:`MPI_File_preallocate` has the same effect as writing undefined data. If
+size is larger than the current file size, the file size increases to
+*size*. If *size* is less than or equal to the current file size, the
+file size is unchanged.
+
+The treatment of file pointers, pending nonblocking accesses, and file
+consistency is the same as with :ref:`MPI_File_set_size`. If
+``MPI_MODE_SEQUENTIAL`` mode was specified when the file was opened, it is
+erroneous to call this routine.
+
+
+NOTES
+-----
+
+When using the collective routine :ref:`MPI_File_set_size` on a UNIX file, if
+the size that is set is smaller than the current file size, the file is
+truncated at the position defined by size. If the size is set to be
+larger than the current file size, the file size becomes the set size.
+When the file size is increased this way with :ref:`MPI_File_set_size`, new
+regions are created in the file with displacements between the old file
+size and the larger, newly set file size.
+
+Sun MPI I/O does not necessarily allocate file space for such new
+regions. You may reserve file space either by using :ref:`MPI_File_preallocate`
+or by performing a read or write to certain bytes.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_read.3.rst b/docs/man-openmpi/man3/MPI_File_read.3.rst
new file mode 100644
index 00000000000..30cd8a8b911
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_read.3.rst
@@ -0,0 +1,45 @@
+.. _mpi_file_read:
+
+
+MPI_File_read
+=============
+
+.. include_body
+
+:ref:`MPI_File_read` |mdash| Reads a file starting at the location specified by
+the individual file pointer (blocking, noncollective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_read.rst
+
+INPUT PARAMETERS
+----------------
+* ``fh``: File handle (handle).
+* ``count``: Number of elements in buffer (integer).
+* ``datatype``: Data type of each buffer element (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``buf``: Initial address of buffer (integer).
+* ``status``: Status object (status).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_read` attempts to read from the file associated with *fh* (at
+the current individual file pointer position maintained by the system) a
+total number of *count* data items having *datatype* type into the
+user's buffer *buf.* The data is taken out of those parts of the file
+specified by the current view. :ref:`MPI_File_read` stores the number of
+data-type elements actually read in *status.* All other fields of
+*status* are undefined.
+
+It is erroneous to call this function if ``MPI_MODE_SEQUENTIAL`` mode was
+specified when the file was opened.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_read_all.3.rst b/docs/man-openmpi/man3/MPI_File_read_all.3.rst
new file mode 100644
index 00000000000..b6840de2a13
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_read_all.3.rst
@@ -0,0 +1,45 @@
+.. _mpi_file_read_all:
+
+
+MPI_File_read_all
+=================
+
+.. include_body
+
+:ref:`MPI_File_read_all` |mdash| Reads a file starting at the locations specified
+by individual file pointers (blocking, collective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_read_all.rst
+
+INPUT PARAMETERS
+----------------
+* ``fh``: File handle (handle).
+* ``count``: Number of elements in buffer (integer).
+* ``datatype``: Data type of each buffer element (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``buf``: Initial address of buffer (choice).
+* ``status``: Status object (status).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_read_all` is a collective routine that attempts to read from the
+file associated with *fh* (at the current individual file pointer
+position maintained by the system) a total number of *count* data items
+having *datatype* type into the user's buffer *buf.* The data is taken
+out of those parts of the file specified by the current view.
+:ref:`MPI_File_read_all` stores the number of data-type elements actually read
+in *status.* All other fields of *status* are undefined.
+
+It is erroneous to call this function if ``MPI_MODE_SEQUENTIAL`` mode was
+specified when the file was opened.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_read_all_begin.3.rst b/docs/man-openmpi/man3/MPI_File_read_all_begin.3.rst
new file mode 100644
index 00000000000..66c122e11a3
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_read_all_begin.3.rst
@@ -0,0 +1,53 @@
+.. _mpi_file_read_all_begin:
+
+
+MPI_File_read_all_begin
+=======================
+
+.. include_body
+
+:ref:`MPI_File_read_all_begin` |mdash| Reads a file starting at the locations
+specified by individual file pointers; beginning part of a split
+collective routine (nonblocking).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_read_all_begin.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``fh``: File handle (handle).
+
+INPUT PARAMETERS
+----------------
+* ``count``: Number of elements in buffer (integer).
+* ``datatype``: Data type of each buffer element (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``buf``: Initial address of buffer (choice).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_read_all_begin` is the beginning part of a split collective
+operation that attempts to read from the file associated with *fh* (at
+the current individual file pointer position maintained by the system) a
+total number of *count* data items having *datatype* type into the
+user's buffer *buf.* The data is taken out of those parts of the file
+specified by the current view.
+
+
+NOTES
+-----
+
+All the nonblocking collective routines for data access are "split" into
+two routines, each with ``_begin`` or ``_end`` as a suffix. These split
+collective routines are subject to the semantic rules described in
+Section 9.4.5 of the MPI-2 standard.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_read_all_end.3.rst b/docs/man-openmpi/man3/MPI_File_read_all_end.3.rst
new file mode 100644
index 00000000000..8105f639f50
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_read_all_end.3.rst
@@ -0,0 +1,49 @@
+.. _mpi_file_read_all_end:
+
+
+MPI_File_read_all_end
+=====================
+
+.. include_body
+
+:ref:`MPI_File_read_all_end` |mdash| Reads a file starting at the locations
+specified by individual file pointers; ending part of a split collective
+routine (blocking).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_read_all_end.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``fh``: File handle (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``buf``: Initial address of buffer (choice).
+* ``status``: Status object (status).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_read_all_end` is the ending part of a split collective operation
+that stores the number of elements actually read from the file
+associated with *fh* (at the current individual file pointer position
+maintained by the system) into the user's buffer *buf* in *status.* The
+data is taken out of those parts of the file specified by the current
+view. All other fields of *status* are undefined.
+
+
+NOTES
+-----
+
+All the nonblocking collective routines for data access are "split" into
+two routines, each with ``_begin`` or ``_end`` as a suffix. These split
+collective routines are subject to the semantic rules described in
+Section 9.4.5 of the MPI-2 standard.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_read_at.3.rst b/docs/man-openmpi/man3/MPI_File_read_at.3.rst
new file mode 100644
index 00000000000..ec1b255c5b5
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_read_at.3.rst
@@ -0,0 +1,45 @@
+.. _mpi_file_read_at:
+
+
+MPI_File_read_at
+================
+
+.. include_body
+
+:ref:`MPI_File_read_at` |mdash| Reads a file at an explicitly specified offset
+(blocking, noncollective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_read_at.rst
+
+INPUT PARAMETERS
+----------------
+* ``fh``: File handle (handle).
+* ``offset``: File offset (integer).
+* ``count``: Number of elements in buffer (integer).
+* ``datatype``: Data type of each buffer element (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``buf``: Initial address of buffer (choice).
+* ``status``: Status object (status).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_read_at` attempts to read from the file associated with *fh* (at
+the *offset* position) a total number of *count* data items having
+*datatype* type into the user's buffer *buf.* The *offset* is in *etype*
+units relative to the current view. That is, holes are not counted when
+locating an offset. The data is taken out of those parts of the file
+specified by the current view. :ref:`MPI_File_read_at` stores the number of
+*datatype* elements actually read in *status.* All other fields of
+*status* are undefined. It is erroneous to call this function if
+``MPI_MODE_SEQUENTIAL`` mode was specified when the file was opened.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_read_at_all.3.rst b/docs/man-openmpi/man3/MPI_File_read_at_all.3.rst
new file mode 100644
index 00000000000..1d9ddbc01d4
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_read_at_all.3.rst
@@ -0,0 +1,46 @@
+.. _mpi_file_read_at_all:
+
+
+MPI_File_read_at_all
+====================
+
+.. include_body
+
+:ref:`MPI_File_read_at_all` |mdash| Reads a file at explicitly specified offsets
+(blocking, collective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_read_at_all.rst
+
+INPUT PARAMETERS
+----------------
+* ``fh``: File handle (handle).
+* ``offset``: File offset (integer).
+* ``count``: Number of elements in buffer (integer).
+* ``datatype``: Data type of each buffer element (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``buf``: Initial address of buffer (choice).
+* ``status``: Status object (status).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_read_at_all` is a collective routine that attempts to read from
+the file associated with *fh* (at the *offset* position) a total number
+of *count* data items having *datatype* type into the user's buffer
+*buf.* The *offset* is in etype units relative to the current view. That
+is, holes are not counted when locating an offset. The data is taken out
+of those parts of the file specified by the current view.
+:ref:`MPI_File_read_at_all` stores the number of *datatype* elements actually
+read in *status.* All other fields of *status* are undefined. It is
+erroneous to call this function if ``MPI_MODE_SEQUENTIAL`` mode was
+specified when the file was opened.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_read_at_all_begin.3.rst b/docs/man-openmpi/man3/MPI_File_read_at_all_begin.3.rst
new file mode 100644
index 00000000000..5f7a7907b8d
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_read_at_all_begin.3.rst
@@ -0,0 +1,51 @@
+.. _mpi_file_read_at_all_begin:
+
+
+MPI_File_read_at_all_begin
+==========================
+
+.. include_body
+
+:ref:`MPI_File_read_at_all_begin` |mdash| Reads a file at explicitly specified
+offsets; beginning part of a split collective routine (nonblocking).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_read_at_all_begin.rst
+
+INPUT PARAMETERS
+----------------
+* ``fh``: File handle (handle).
+* ``offset``: File offset (integer).
+* ``count``: Number of elements in buffer (integer).
+* ``datatype``: Data type of each buffer element.
+
+OUTPUT PARAMETERS
+-----------------
+* ``buf``: Initial address of buffer (choice).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_read_at_all_begin` is the beginning part of a split collective
+routine that attempts to read from the file associated with *fh* (at the
+*offset* position) a total number of *count* data items having
+*datatype* type into the user's buffer *buf.* The *offset* is in etype
+units relative to the current view. That is, holes are not counted when
+locating an offset. The data is taken out of those parts of the file
+specified by the current view.
+
+
+NOTES
+-----
+
+All the nonblocking collective routines for data access are "split" into
+two routines, each with ``_begin`` or ``_end`` as a suffix. These split
+collective routines are subject to the semantic rules described in
+Section 9.4.5 of the MPI-2 standard.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_read_at_all_end.3.rst b/docs/man-openmpi/man3/MPI_File_read_at_all_end.3.rst
new file mode 100644
index 00000000000..5a2ecc74a50
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_read_at_all_end.3.rst
@@ -0,0 +1,48 @@
+.. _mpi_file_read_at_all_end:
+
+
+MPI_File_read_at_all_end
+========================
+
+.. include_body
+
+:ref:`MPI_File_read_at_all_end` |mdash| Reads a file at explicitly specified
+offsets; ending part of a split collective routine (blocking).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_read_at_all_end.rst
+
+INPUT PARAMETER
+---------------
+* ``fh``: File handle (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``buf``: Initial address of buffer (choice).
+* ``status``: Status object (status).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_read_at_all_end` is a split collective routine that stores the
+number of elements actually read from the file associated with *fh* in
+*status.* :ref:`MPI_File_read_at_all_end` blocks until the operation initiated
+by :ref:`MPI_File_read_at_all_begin` completes. The data is taken out of those
+parts of the file specified by the current view. All other fields of
+*status* are undefined.
+
+
+NOTES
+-----
+
+All the nonblocking collective routines for data access are "split" into
+two routines, each with ``_begin`` or ``_end`` as a suffix. These split
+collective routines are subject to the semantic rules described in
+Section 9.4.5 of the MPI-2 standard.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_read_ordered.3.rst b/docs/man-openmpi/man3/MPI_File_read_ordered.3.rst
new file mode 100644
index 00000000000..45712c1bdaa
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_read_ordered.3.rst
@@ -0,0 +1,47 @@
+.. _mpi_file_read_ordered:
+
+
+MPI_File_read_ordered
+=====================
+
+.. include_body
+
+:ref:`MPI_File_read_ordered` |mdash| Reads a file at a location specified by a
+shared file pointer (blocking, collective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_read_ordered.rst
+
+INPUT PARAMETERS
+----------------
+* ``fh``: File handle (handle).
+* ``count``: Number of elements in buffer (integer).
+* ``datatype``: Data type of each buffer element (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``buf``: Initial address of buffer (choice).
+* ``status``: Status object (Status).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_read_ordered` is a collective routine. This routine must be
+called by all processes in the communicator group associated with the
+file handle *fh.* Each process may pass different argument values for
+the *datatype* and *count* arguments. Each process attempts to read,
+from the file associated with *fh,* a total number of *count* data items
+having *datatype* type into the user's buffer *buf.* For each process,
+the location in the file at which data is read is the position at which
+the shared file pointer would be after all processes whose ranks within
+the group are less than that of this process had read their data.
+:ref:`MPI_File_read_ordered` returns the actual number of *datatype* elements
+read in *status.* The shared file pointer is updated by the amounts of
+data requested by all processes of the group.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_read_ordered_begin.3.rst b/docs/man-openmpi/man3/MPI_File_read_ordered_begin.3.rst
new file mode 100644
index 00000000000..4d62f4f6fc2
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_read_ordered_begin.3.rst
@@ -0,0 +1,57 @@
+.. _mpi_file_read_ordered_begin:
+
+
+MPI_File_read_ordered_begin
+===========================
+
+.. include_body
+
+:ref:`MPI_File_read_ordered_begin` |mdash| Reads a file at a location specified
+by a shared file pointer; beginning part of a split collective routine
+(nonblocking).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_read_ordered_begin.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``fh``: File handle (handle).
+
+INPUT PARAMETERS
+----------------
+* ``count``: Number of elements in buffer (integer).
+* ``datatype``: Data type of each buffer element (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``buf``: Initial address of buffer (choice).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_read_ordered_begin` is the beginning part of a split collective,
+nonblocking routine that must be called by all processes in the
+communicator group associated with the file handle *fh.* Each process
+may pass different argument values for the *datatype* and *count*
+arguments. Each process attempts to read, from the file associated with
+*fh,* a total number of *count* data items having *datatype* type into
+the user's buffer *buf.* For each process, the location in the file at
+which data is read is the position at which the shared file pointer
+would be after all processes whose ranks within the group are less than
+that of this process had read their data.
+
+
+NOTES
+-----
+
+All the nonblocking collective routines for data access are "split" into
+two routines, each with ``_begin`` or ``_end`` as a suffix. These split
+collective routines are subject to the semantic rules described in
+Section 9.4.5 of the MPI-2 standard.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_read_ordered_end.3.rst b/docs/man-openmpi/man3/MPI_File_read_ordered_end.3.rst
new file mode 100644
index 00000000000..549c3ced442
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_read_ordered_end.3.rst
@@ -0,0 +1,53 @@
+.. _mpi_file_read_ordered_end:
+
+
+MPI_File_read_ordered_end
+=========================
+
+.. include_body
+
+:ref:`MPI_File_read_ordered_end` |mdash| Reads a file at a location specified by
+a shared file pointer; ending part of a split collective routine
+(blocking).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_read_ordered_end.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``fh``: File handle (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``buf``: Initial address of buffer (choice).
+* ``status``: Status object (status).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_read_ordered_end` is the ending part of a split collective
+routine that must be called by all processes in the communicator group
+associated with the file handle *fh.* MPI_File_rad_ordered_end blocks
+until the operation initiated by :ref:`MPI_File_read_ordered_begin` completes.
+It attempts to read the file associated with *fh* into the user's buffer
+*buf.* The shared file pointer is updated by the amounts of data
+requested by all processes of the group. For each process, the location
+in the file at which data is read is the position at which the shared
+file pointer would be after all processes whose ranks within the group
+are less than that of this process had read their data.
+
+
+NOTES
+-----
+
+All the nonblocking collective routines for data access are "split" into
+two routines, each with ``_begin`` or ``_end`` as a suffix. These split
+collective routines are subject to the semantic rules described in
+Section 9.4.5 of the MPI-2 standard.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_read_shared.3.rst b/docs/man-openmpi/man3/MPI_File_read_shared.3.rst
new file mode 100644
index 00000000000..6d2ae7d7d5a
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_read_shared.3.rst
@@ -0,0 +1,41 @@
+.. _mpi_file_read_shared:
+
+
+MPI_File_read_shared
+====================
+
+.. include_body
+
+:ref:`MPI_File_read_shared` |mdash| Reads a file using the shared file pointer
+(blocking, noncollective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_read_shared.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``fh``: File handle (handle).
+
+INPUT PARAMETERS
+----------------
+* ``count``: Number of elements in buffer (integer)
+* ``datatype``: Data type of each buffer element (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``buf``: Initial address of buffer (choice).
+* ``status``: Status object (status).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_read_shared` is a blocking routine that uses the shared file
+pointer to read files. The order of serialization is not deterministic
+for this noncollective routine.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_seek.3.rst b/docs/man-openmpi/man3/MPI_File_seek.3.rst
new file mode 100644
index 00000000000..a21f20e3fdb
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_seek.3.rst
@@ -0,0 +1,44 @@
+.. _mpi_file_seek:
+
+
+MPI_File_seek
+=============
+
+.. include_body
+
+:ref:`MPI_File_seek` |mdash| Updates individual file pointers (noncollective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_seek.rst
+
+INPUT PARAMETERS
+----------------
+* ``fh``: File handle (handle).
+* ``offset``: File offset (integer).
+* ``whence``: Update mode (integer).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_seek` updates the individual file pointer according to *whence,*
+which could have the following possible values:
+
+* ``MPI_SEEK_SET`` - The pointer is set to *offset.*
+* ``MPI_SEEK_CUR`` - The pointer is set to the current pointer position plus *offset.*
+* ``MPI_SEEK_END`` - The pointer is set to the end of the file plus *offset.*
+
+The *offset* can be negative, which allows seeking backwards. It is
+erroneous to seek to a negative position in the file. The end of the
+file is defined to be the location of the next elementary data item
+immediately after the last accessed data item, even if that location is
+a hole.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_seek_shared.3.rst b/docs/man-openmpi/man3/MPI_File_seek_shared.3.rst
new file mode 100644
index 00000000000..ae942ac22b4
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_seek_shared.3.rst
@@ -0,0 +1,51 @@
+.. _mpi_file_seek_shared:
+
+
+MPI_File_seek_shared
+====================
+
+.. include_body
+
+:ref:`MPI_File_seek_shared` |mdash| Updates the global shared file pointer
+(collective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_seek_shared.rst
+
+INPUT PARAMETERS
+----------------
+* ``fh``: File handle (handle).
+* ``offset``: File offset (integer).
+* ``whence``: Update mode (integer).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_seek_shared` updates the shared file pointer according to
+*whence,* which could have the following possible values:
+
+* ``MPI_SEEK_SET`` - The pointer is set to *offset.*
+* ``MPI_SEEK_CUR`` - The pointer is set to the current pointer position plus *offset.*
+* ``MPI_SEEK_END`` - The pointer is set to the end of the file plus *offset.*
+
+:ref:`MPI_File_seek_shared` is collective; all the processes in the
+communicator group associated with the file handle *fh* must call
+:ref:`MPI_File_seek_shared` with the same *offset* and *whence.* All processes
+in the communicator group are synchronized before the shared file
+pointer is updated.
+
+The *offset* can be negative, which allows seeking backwards. It is
+erroneous to seek to a negative position in the view. The end of the
+view is defined to be the position of the next elementary data item,
+relative to the current view, following the last whole elementary data
+item accessible.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_set_atomicity.3.rst b/docs/man-openmpi/man3/MPI_File_set_atomicity.3.rst
new file mode 100644
index 00000000000..914715d6d92
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_set_atomicity.3.rst
@@ -0,0 +1,41 @@
+.. _mpi_file_set_atomicity:
+
+
+MPI_File_set_atomicity
+======================
+
+.. include_body
+
+:ref:`MPI_File_set_atomicity` |mdash| Sets consistency semantics for data-access
+operations (collective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_set_atomicity.rst
+
+INPUT PARAMETERS
+----------------
+* ``fh``: File handle (handle).
+* ``flag``: **true** to enable atomic mode, **false** to enable nonatomic mode (boolean).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The consistency semantics for data-access operations using the set of
+file handles created by one collective :ref:`MPI_File_open` is set by
+collectively calling :ref:`MPI_File_set_atomicity`. All processes in the group
+must pass identical values for *fh* and *flag.* If *flag* is *true,*
+atomic mode is set; if *flag* is *false,* nonatomic mode is set.
+
+The default value on a call to :ref:`MPI_File_open` in Open MPI is *true* for
+jobs running on more than one node, *false* for jobs running on a single
+SMP. For more information, see the MPI-2 standard.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_set_errhandler.3.rst b/docs/man-openmpi/man3/MPI_File_set_errhandler.3.rst
new file mode 100644
index 00000000000..685b58b3237
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_set_errhandler.3.rst
@@ -0,0 +1,37 @@
+.. _mpi_file_set_errhandler:
+
+
+MPI_File_set_errhandler
+=======================
+
+.. include_body
+
+:ref:`MPI_File_set_errhandler` |mdash| Sets the error handler for a file.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_set_errhandler.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``file``: File (handle).
+
+INPUT PARAMETER
+---------------
+* ``errhandler``: New error handler for file (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Attaches a new error handler to a file. The error handler must be either
+a predefined error handler or an error handler created by a call to
+:ref:`MPI_File_create_errhandler`.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_set_info.3.rst b/docs/man-openmpi/man3/MPI_File_set_info.3.rst
new file mode 100644
index 00000000000..76253fe2b5a
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_set_info.3.rst
@@ -0,0 +1,107 @@
+.. _mpi_file_set_info:
+
+
+MPI_File_set_info
+=================
+
+.. include_body
+
+:ref:`MPI_File_set_info` |mdash| Sets new values for hints (collective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_set_info.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``fh``: File handle (handle).
+
+INPUT PARAMETER
+---------------
+* ``info``: Info object (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_set_info` is a collective routine that sets new values
+for the hints of the file associated with *fh*. These hints are set
+for each file, using the :ref:`MPI_File_open`, :ref:`MPI_File_delete`,
+:ref:`MPI_File_set_view`, and :ref:`MPI_File_set_info` routines. The
+opaque *info* object, which allows you to provide hints for
+optimization of your code, may be different on each process, but some
+*info* entries are required to be the same on all processes: In these
+cases, they must appear with the same value in each process's info
+object. See the :ref:`HINTS section <man-openmpi-mpi-file-set-info>`
+for a list of hints that can be set.
+
+.. _man-openmpi-mpi-file-set-info:
+
+HINTS
+-----
+
+The following hints can be used as values for the *info* argument.
+
+**SETTABLE HINTS**
+
+* ``shared_file_timeout``: Amount of time (in seconds) to wait for
+  access to the shared file pointer before exiting with
+  ``MPI_ERR_TIMEDOUT``.
+
+* ``rwlock_timeout``: Amount of time (in seconds) to wait for
+  obtaining a read or write lock on a contiguous chunk of a UNIX file
+  before exiting with ``MPI_ERR_TIMEDOUT``.
+
+* ``noncoll_read_bufsize``: Maximum size of the buffer used by MPI I/O
+  to satisfy read requests in the noncollective data-access
+  routines.
+
+  .. note:: A buffer size smaller than the distance (in bytes) in a
+	    UNIX file between the first byte and the last byte of the
+	    access request causes MPI I/O to iterate and perform
+	    multiple UNIX ``read()`` or ``write()`` calls. If the request
+	    includes multiple noncontiguous chunks of data, and the
+	    buffer size is greater than the size of those chunks, then
+	    the UNIX ``read()`` or ``write()`` (made at the MPI I/O level)
+	    will access data not requested by this process in order to
+	    reduce the total number of ``write()`` calls made. If this is
+	    not desirable behavior, you should reduce this buffer size
+	    to equal the size of the contiguous chunks within the
+	    aggregate request.
+
+* ``noncoll_write_bufsize``: Maximum size of the buffer used by MPI
+  I/O to satisfy write requests in the noncollective data-access
+  routines.
+
+  See the above note in ``noncoll_read_bufsize``.
+
+* ``coll_read_bufsize``: Maximum size of the buffer used by MPI I/O to
+  satisfy read requests in the collective data-access routines.
+
+  See the above note in ``noncoll_read_bufsize``.
+
+* ``coll_write_bufsize``: Maximum size of the buffer used by MPI I/O
+  to satisfy write requests in the collective data-access
+  routines.
+
+  See the above note in ``noncoll_read_bufsize``.
+
+* ``mpiio_concurrency``: (boolean) controls whether nonblocking I/O
+  routines can bind an extra thread to an LWP.
+
+* ``mpiio_coll_contiguous``: (boolean) controls whether subsequent
+  collective data accesses will request collectively contiguous
+  regions of the file.
+
+
+  **NON-SETTABLE HINTS**
+
+* ``filename``: Access this hint to get the name of the file.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_set_size.3.rst b/docs/man-openmpi/man3/MPI_File_set_size.3.rst
new file mode 100644
index 00000000000..70ac51228f0
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_set_size.3.rst
@@ -0,0 +1,50 @@
+.. _mpi_file_set_size:
+
+
+MPI_File_set_size
+=================
+
+.. include_body
+
+:ref:`MPI_File_set_size` |mdash| Resizes a file (collective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_set_size.rst
+
+INPUT PARAMETERS
+----------------
+* ``fh``: File handle (handle).
+* ``size``: Size to truncate or expand file (integer).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_set_size` resizes the file associated with the file handle *fh,*
+truncating UNIX files as necessary. :ref:`MPI_File_set_size` is collective; all
+processes in the group must pass identical values for size.
+
+When using :ref:`MPI_File_set_size` on a UNIX file, if *size* is larger than
+the current file size, the file size becomes *size*. If *size* is
+smaller than the current file size, the file is truncated at the
+position defined by *size* (from the beginning of the file and measured
+in bytes). Regions of the file which have been previously written are
+unaffected.
+
+:ref:`MPI_File_set_size` does not affect the individual file pointers or the
+shared file pointer.
+
+Note that the actual amount of storage space cannot be allocated by
+:ref:`MPI_File_set_size`. Use :ref:`MPI_File_preallocate` to accomplish this.
+
+It is erroneous to call this function if ``MPI_MODE_SEQUENTIAL`` mode was
+specified when the file was opened.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_set_view.3.rst b/docs/man-openmpi/man3/MPI_File_set_view.3.rst
new file mode 100644
index 00000000000..d2d00ec229e
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_set_view.3.rst
@@ -0,0 +1,123 @@
+.. _mpi_file_set_view:
+
+
+MPI_File_set_view
+=================
+
+.. include_body
+
+:ref:`MPI_File_set_view` |mdash| Changes process's view of data in file
+(collective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_set_view.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``fh``: File handle (handle).
+
+INPUT PARAMETERS
+----------------
+* ``disp``: Displacement (integer).
+* ``etype``: Elementary data type (handle).
+* ``filetype``: File type (handle). See Restrictions, below.
+* ``datarep``: Data representation (string).
+* ``info``: Info object (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The :ref:`MPI_File_set_view` routine changes the process's view of the data in
+the file |mdash| the beginning of the data accessible in the file through
+that view is set to *disp;* the type of data is set to *etype;* and the
+distribution of data to processes is set to *filetype.* In addition,
+:ref:`MPI_File_set_view` resets the independent file pointers and the shared
+file pointer to zero. :ref:`MPI_File_set_view` is collective across the *fh*;
+all processes in the group must pass identical values for *datarep* and
+provide an *etype* with an identical extent. The values for *disp*,
+*filetype*, and *info* may vary. It is erroneous to use the shared file
+pointer data-access routines unless identical values for *disp* and
+*filetype* are also given. The data types passed in *etype* and
+*filetype* must be committed.
+
+The *disp* displacement argument specifies the position (absolute offset
+in bytes from the beginning of the file) where the view begins.
+
+The :ref:`MPI_File_set_view` interface allows the user to pass a
+data-representation string to MPI I/O via the *datarep* argument. To
+obtain the default value pass the value "native". The user can also
+pass information via the *info* argument. See the :ref:`HINTS section
+<man-openmpi-mpi-file-set-view>` for a list of hints that can be
+set.
+
+.. _man-openmpi-mpi-file-set-view:
+
+HINTS
+-----
+
+The following hints can be used as values for the *info* argument.
+
+**SETTABLE HINTS**
+
+* ``MPI_INFO_NULL``
+
+* ``shared_file_timeout``: Amount of time (in seconds) to wait for
+  access to the shared file pointer before exiting with
+  ``MPI_ERR_TIMEDOUT``.
+
+* ``rwlock_timeout``: Amount of time (in seconds) to wait for
+  obtaining a read or write lock on a contiguous chunk of a UNIX file
+  before exiting with ``MPI_ERR_TIMEDOUT``.
+
+* ``noncoll_read_bufsize``: Maximum size of the buffer used by MPI I/O
+  to satisfy read requests in the noncollective data-access routines.
+
+  .. note:: A buffer size smaller than the distance (in bytes) in a
+            UNIX file between the first byte and the last byte of the
+            access request causes MPI I/O to iterate and perform
+            multiple UNIX ``read()`` or ``write()`` calls. If the
+            request includes multiple noncontiguous chunks of data,
+            and the buffer size is greater than the size of those
+            chunks, then the UNIX ``read()`` or ``write()`` (made at
+            the MPI I/O level) will access data not requested by this
+            process in order to reduce the total number of ``write()``
+            calls made. If this is not desirable behavior, you should
+            reduce this buffer size to equal the size of the
+            contiguous chunks within the aggregate request.
+
+* ``noncoll_write_bufsize``: Maximum size of the buffer used by MPI
+  I/O to satisfy write requests in the noncollective data-access
+  routines.
+
+  See the above note in ``noncoll_read_bufsize``.
+
+* ``coll_read_bufsize``: Maximum size of the buffer used by MPI I/O to
+  satisfy read requests in the collective data-access routines.
+
+  See the above note in ``noncoll_read_bufsize``.
+
+* ``coll_write_bufsize``: Maximum size of the buffer used by MPI I/O
+  to satisfy write requests in the collective data-access routines.
+
+  See the above note in ``noncoll_read_bufsize``.
+
+* ``mpiio_concurrency``: (boolean) controls whether nonblocking I/O
+  routines can bind an extra thread to an LWP.
+
+* ``mpiio_coll_contiguous``: (boolean) controls whether subsequent
+  collective data accesses will request collectively contiguous
+  regions of the file.
+
+**NON-SETTABLE HINTS**
+
+* ``filename``: Access this hint to get the name of the file.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_sync.3.rst b/docs/man-openmpi/man3/MPI_File_sync.3.rst
new file mode 100644
index 00000000000..f4ac4a2cff8
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_sync.3.rst
@@ -0,0 +1,40 @@
+.. _mpi_file_sync:
+
+
+MPI_File_sync
+=============
+
+.. include_body
+
+:ref:`MPI_File_sync` |mdash| Makes semantics consistent for data-access
+operations (collective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_sync.rst
+
+INPUT PARAMETER
+---------------
+* ``fh``: File handle (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Calling :ref:`MPI_File_sync` with *fh* causes all previous writes to *fh* by
+the calling process to be written to permanent storage. If other
+processes have made updates to permanent storage, then all such updates
+become visible to subsequent reads of *fh* by the calling process.
+
+:ref:`MPI_File_sync` is a collective operation. The user is responsible for
+ensuring that all nonblocking requests on *fh* have been completed
+before calling :ref:`MPI_File_sync`. Otherwise, the call to :ref:`MPI_File_sync` is
+erroneous.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_write.3.rst b/docs/man-openmpi/man3/MPI_File_write.3.rst
new file mode 100644
index 00000000000..9d2a5b201a7
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_write.3.rst
@@ -0,0 +1,48 @@
+.. _mpi_file_write:
+
+
+MPI_File_write
+==============
+
+.. include_body
+
+:ref:`MPI_File_write` |mdash| Writes a file starting at the location specified by
+the individual file pointer (blocking, noncollective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_write.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``fh``: File handle (handle).
+
+INPUT PARAMETERS
+----------------
+* ``buf``: Initial address of buffer (choice).
+* ``count``: Number of elements in buffer (integer).
+* ``datatype``: Data type of each buffer element (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``status``: Status object (status).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_write` attempts to write into the file associated with *fh* (at
+the current individual file pointer position maintained by the system) a
+total number of *count* data items having *datatype* type from the
+user's buffer *buf.* The data is written into those parts of the file
+specified by the current view. :ref:`MPI_File_write` stores the number of
+*datatype* elements actually written in *status.* All other fields of
+*status* are undefined.
+
+It is erroneous to call this function if ``MPI_MODE_SEQUENTIAL`` mode was
+specified when the file was opened.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_write_all.3.rst b/docs/man-openmpi/man3/MPI_File_write_all.3.rst
new file mode 100644
index 00000000000..6dd7c3af358
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_write_all.3.rst
@@ -0,0 +1,45 @@
+.. _mpi_file_write_all:
+
+
+MPI_File_write_all
+==================
+
+.. include_body
+
+:ref:`MPI_File_write_all` |mdash| Writes a file starting at the locations
+specified by individual file pointers (blocking, collective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_write_all.rst
+
+INPUT PARAMETERS
+----------------
+* ``fh``: File handle (handle).
+* ``buf``: Initial address of buffer (choice).
+* ``count``: Number of elements in buffer (integer).
+* ``datatype``: Data type of each buffer element (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``status``: Status object (status).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_write_all` is a collective routine that attempts to write into
+the file associated with *fh* (at the current individual file pointer
+position maintained by the system) a total number of *count* data items
+having *datatype* type from the user's buffer *buf.* The data is written
+into those parts of the file specified by the current view.
+:ref:`MPI_File_write_all` stores the number of *datatype* elements actually
+written in *status.* All other fields of *status* are undefined.
+
+It is erroneous to call this function if ``MPI_MODE_SEQUENTIAL`` mode was
+specified when the file was opened.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_write_all_begin.3.rst b/docs/man-openmpi/man3/MPI_File_write_all_begin.3.rst
new file mode 100644
index 00000000000..4434b5618a0
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_write_all_begin.3.rst
@@ -0,0 +1,53 @@
+.. _mpi_file_write_all_begin:
+
+
+MPI_File_write_all_begin
+========================
+
+.. include_body
+
+:ref:`MPI_File_write_all_begin` |mdash| Writes a file starting at the locations
+specified by individual file pointers; beginning part of a split
+collective routine (nonblocking).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_write_all_begin.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``fh``: File handle (handle).
+
+INPUT PARAMETERS
+----------------
+* ``buf``: Initial address of buffer (choice).
+* ``count``: Number of elements in buffer (integer).
+* ``datatype``: Data type of each buffer element (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_write_all_begin` is the beginning part of a split collective,
+nonblocking routine that attempts to write into the file associated with
+*fh* (at the current individual file pointer position maintained by the
+system) a total number of *count* data items having *datatype* type from
+the user's buffer *buf.* The data is written into those parts of the
+file specified by the current view.
+
+
+NOTES
+-----
+
+All the nonblocking collective routines for data access are "split" into
+two routines, each with ``_begin`` or ``_end`` as a suffix. These split
+collective routines are subject to the semantic rules described in
+Section 9.4.5 of the MPI-2 standard.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_write_all_end.3.rst b/docs/man-openmpi/man3/MPI_File_write_all_end.3.rst
new file mode 100644
index 00000000000..964458ef38f
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_write_all_end.3.rst
@@ -0,0 +1,53 @@
+.. _mpi_file_write_all_end:
+
+
+MPI_File_write_all_end
+======================
+
+.. include_body
+
+:ref:`MPI_File_write_all_end` |mdash| Writes a file starting at the locations
+specified by individual file pointers; ending part of a split collective
+routine (blocking).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_write_all_end.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``fh``: File handle (handle).
+
+INPUT PARAMETER
+---------------
+* ``buf``: Initial address of buffer (choice).
+
+OUTPUT PARAMETERS
+-----------------
+* ``status``: Status object (status).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_write_all_end` is the ending part of a split collective routine
+that stores the number of elements actually written into the file
+associated with *fh* from the user's buffer *buf* in *status.*
+:ref:`MPI_File_write_all_end` blocks until the operation initiated by
+:ref:`MPI_File_write_all_begin` completes. The data is written into those parts
+of the file specified by the current view. All other fields of *status*
+are undefined.
+
+
+NOTES
+-----
+
+All the nonblocking collective routines for data access are "split" into
+two routines, each with ``_begin`` or ``_end`` as a suffix. These split
+collective routines are subject to the semantic rules described in
+Section 9.4.5 of the MPI-2 standard.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_write_at.3.rst b/docs/man-openmpi/man3/MPI_File_write_at.3.rst
new file mode 100644
index 00000000000..350861128b0
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_write_at.3.rst
@@ -0,0 +1,53 @@
+.. _mpi_file_write_at:
+
+
+MPI_File_write_at
+=================
+
+.. include_body
+
+:ref:`MPI_File_write_at` |mdash| Writes a file at an explicitly specified offset
+(blocking, noncollective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_write_at.rst
+
+INPUT PARAMETERS
+----------------
+* ``fh``: File handle (handle).
+* ``offset``: File offset (integer).
+* ``buf``: Initial address of buffer (choice).
+* ``count``: Number of elements in buffer (integer).
+* ``datatype``: Data type of each buffer element (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``status``: Status object (status).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_write_at` attempts to write into the file associated with *fh*
+(at the *offset* position) a total number of *count* data items having
+*datatype* type from the user's buffer *buf.* The offset is in *etype*
+units relative to the current view. That is, holes are not counted when
+locating an offset. The data is written into those parts of the file
+specified by the current view. :ref:`MPI_File_write_at` stores the number of
+*datatype* elements actually written in *status.* All other fields of
+*status* are undefined.
+
+It is erroneous to call this function if ``MPI_MODE_SEQUENTIAL`` mode was
+specified when the file was opened.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_File_iwrite_at`
+   * :ref:`MPI_File_write_at_all`
+   * :ref:`MPI_File_write_at_all_begin`
+   * :ref:`MPI_File_write_at_all_end`
diff --git a/docs/man-openmpi/man3/MPI_File_write_at_all.3.rst b/docs/man-openmpi/man3/MPI_File_write_at_all.3.rst
new file mode 100644
index 00000000000..7d2efb7c5cb
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_write_at_all.3.rst
@@ -0,0 +1,47 @@
+.. _mpi_file_write_at_all:
+
+
+MPI_File_write_at_all
+=====================
+
+.. include_body
+
+:ref:`MPI_File_write_at_all` |mdash| Writes a file at explicitly specified
+offsets (blocking, collective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_write_at_all.rst
+
+INPUT PARAMETERS
+----------------
+* ``fh``: File handle (handle).
+* ``offset``: File offset (integer).
+* ``buf``: Initial address of buffer (choice).
+* ``count``: Number of elements in buffer (integer).
+* ``datatype``: Data type of each buffer element (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``status``: Status object (status).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_write_at_all` is a collective routine that attempts to write
+into the file associated with *fh* (at the *offset* position) a total
+number of *count* data items having *datatype* type from the user's
+buffer *buf.* The offset is in etype units relative to the current view.
+That is, holes are not counted when locating an offset. The data is
+written into those parts of the file specified by the current view.
+:ref:`MPI_File_write_at_all` stores the number of *datatype* elements actually
+written in *status.* All other fields of *status* are undefined.
+
+It is erroneous to call this function if ``MPI_MODE_SEQUENTIAL`` mode was
+specified when the file was opened.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_write_at_all_begin.3.rst b/docs/man-openmpi/man3/MPI_File_write_at_all_begin.3.rst
new file mode 100644
index 00000000000..0eae8fd028b
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_write_at_all_begin.3.rst
@@ -0,0 +1,54 @@
+.. _mpi_file_write_at_all_begin:
+
+
+MPI_File_write_at_all_begin
+===========================
+
+.. include_body
+
+:ref:`MPI_File_write_at_all_begin` |mdash| Writes a file at explicitly specified
+offsets; beginning part of a split collective routine (nonblocking).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_write_at_all_begin.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``fh``: File handle (handle).
+
+INPUT PARAMETERS
+----------------
+* ``offset``: File offset (handle).
+* ``buf``: Initial address of buffer (choice).
+* ``count``: Number of elements in buffer (integer).
+* ``datatype``: Data type of each buffer element (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_write_at_all_begin` is the beginning part of a split collective,
+that is, a nonblocking routine that attempts to write into the file
+associated with *fh* (at the *offset* position) a total number of
+*count* data items having *datatype* type from the user's buffer *buf.*
+The offset is in etype units relative to the current view. That is,
+holes are not counted when locating an offset. The data is written into
+those parts of the file specified by the current view.
+
+
+NOTES
+-----
+
+All the nonblocking collective routines for data access are "split" into
+two routines, each with ``_begin`` or ``_end`` as a suffix. These split
+collective routines are subject to the semantic rules described in
+Section 9.4.5 of the MPI-2 standard.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_write_at_all_end.3.rst b/docs/man-openmpi/man3/MPI_File_write_at_all_end.3.rst
new file mode 100644
index 00000000000..fba0d592087
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_write_at_all_end.3.rst
@@ -0,0 +1,50 @@
+.. _mpi_file_write_at_all_end:
+
+
+MPI_File_write_at_all_end
+=========================
+
+.. include_body
+
+:ref:`MPI_File_write_at_all_end` |mdash| Writes a file at explicitly specified
+offsets; ending part of a split collective routine (blocking).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_write_at_all_end.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``fh``: File handle (handle).
+
+INPUT PARAMETER
+---------------
+* ``buf``: Initial address of buffer (choice).
+
+OUTPUT PARAMETERS
+-----------------
+* ``status``: Status object (status).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_write_at_all_end` is the ending part of a split collective
+routine that stores the number of elements actually written into the
+file associated with *fh* in *status.* The data is written into those
+parts of the file specified by the current view. All other fields of
+*status* are undefined.
+
+
+NOTES
+-----
+
+All the nonblocking collective routines for data access are "split" into
+two routines, each with ``_begin`` or ``_end`` as a suffix. These split
+collective routines are subject to the semantic rules described in
+Section 9.4.5 of the MPI-2 standard.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_write_ordered.3.rst b/docs/man-openmpi/man3/MPI_File_write_ordered.3.rst
new file mode 100644
index 00000000000..148da6d4377
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_write_ordered.3.rst
@@ -0,0 +1,48 @@
+.. _mpi_file_write_ordered:
+
+MPI_File_write_ordered
+======================
+
+.. include_body
+
+:ref:`MPI_File_write_ordered` |mdash| Writes a file at a location specified by a
+shared file pointer (blocking, collective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_write_ordered.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``fh`` : File handle (handle).
+* ``buf`` : Initial address of buffer (choice).
+* ``count`` : Number of elements in buffer (integer).
+* ``datatype`` : Data type of each buffer element (handle).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``status`` : Status object (Status).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_write_ordered` is a collective routine. This routine must be
+called by all processes in the communicator group associated with the
+file handle ``fh``. Each process may pass different argument values for
+the ``datatype`` and ``count`` arguments. Each process attempts to
+write, into the file associated with ``fh``, a total number of ``count``
+data items having ``datatype`` type contained in the user's buffer
+``buf``. For each process, the location in the file at which data is
+written is the position at which the shared file pointer would be after
+all processes whose ranks within the group are less than that of this
+process had written their data. :ref:`MPI_File_write_ordered` returns the
+number of ``datatype`` elements written in ``status``. The shared file
+pointer is updated by the amounts of data requested by all processes of
+the group.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_write_ordered_begin.3.rst b/docs/man-openmpi/man3/MPI_File_write_ordered_begin.3.rst
new file mode 100644
index 00000000000..888acb56ec0
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_write_ordered_begin.3.rst
@@ -0,0 +1,59 @@
+.. _mpi_file_write_ordered_begin:
+
+MPI_File_write_ordered_begin
+============================
+
+.. include_body
+
+:ref:`MPI_File_write_ordered_begin` |mdash| Writes a file at a location specified
+by a shared file pointer; beginning part of a split collective routine
+(nonblocking).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_write_ordered_begin.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+
+* ``fh`` : File handle (handle).
+
+INPUT PARAMETERS
+----------------
+
+* ``buf`` : Initial address of buffer (choice).
+* ``count`` : Number of elements in buffer (integer).
+* ``datatype`` : Data type of each buffer element (handle).
+
+OUTPUT PARAMETER
+----------------
+
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_write_ordered_begin` is the beginning part of a split
+collective, nonblocking routine that must be called by all processes in
+the communicator group associated with the file handle ``fh``. Each
+process may pass different argument values for the ``datatype`` and
+``count`` arguments. After all processes of the group have issued their
+respective calls, each process attempts to write, into the file
+associated with ``fh``, a total number of ``count`` data items having
+``datatype`` type contained in the user's buffer ``buf``. For each
+process, the location in the file at which data is written is the
+position at which the shared file pointer would be after all processes
+whose ranks within the group are less than that of this process had
+written their data.
+
+NOTES
+-----
+
+All the nonblocking collective routines for data access are "split" into
+two routines, each with ``_begin`` or ``_end`` as a suffix. These split
+collective routines are subject to the semantic rules described in
+Section 9.4.5 of the MPI-2 standard.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_write_ordered_end.3.rst b/docs/man-openmpi/man3/MPI_File_write_ordered_end.3.rst
new file mode 100644
index 00000000000..5f9b307c781
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_write_ordered_end.3.rst
@@ -0,0 +1,51 @@
+.. _mpi_file_write_ordered_end:
+
+MPI_File_write_ordered_end
+==========================
+
+.. include_body
+
+:ref:`MPI_File_write_ordered_end` |mdash| Writes a file at a location specified
+by a shared file pointer; ending part of a split collective routine
+(blocking).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_write_ordered_end.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+
+* ``fh`` : File handle (handle).
+
+INPUT PARAMETER
+---------------
+
+* ``buf`` : Initial address of buffer (choice).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``status`` : Status object (status).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_write_ordered_end` is the ending part of a split collective
+routine that must be called by all processes in the communicator group
+associated with the file handle ``fh``. :ref:`MPI_File_write_ordered_end`
+returns the number of elements written into the file associated with
+``fh`` in ``status``.
+
+NOTES
+-----
+
+All the nonblocking collective routines for data access are "split" into
+two routines, each with ``_begin`` or ``_end`` as a suffix. These split
+collective routines are subject to the semantic rules described in
+Section 9.4.5 of the MPI-2 standard.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_File_write_shared.3.rst b/docs/man-openmpi/man3/MPI_File_write_shared.3.rst
new file mode 100644
index 00000000000..1402eb5c009
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_File_write_shared.3.rst
@@ -0,0 +1,42 @@
+.. _mpi_file_write_shared:
+
+MPI_File_write_shared
+=====================
+
+.. include_body
+
+:ref:`MPI_File_write_shared` |mdash| Writes a file using the shared file pointer
+(blocking, noncollective).
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_file_write_shared.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+
+* ``fh`` : File handle (handle).
+
+INPUT PARAMETERS
+----------------
+
+* ``buf`` : Initial address of buffer (choice).
+* ``count`` : Number of elements in buffer (integer).
+* ``datatype`` : Data type of each buffer element (handle).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``status`` : Status object (status).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_File_write_shared` is a blocking routine that uses the shared
+file pointer to write files. The order of serialization is not
+deterministic for this noncollective routine.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Finalize.3.rst b/docs/man-openmpi/man3/MPI_Finalize.3.rst
new file mode 100644
index 00000000000..94790f096c8
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Finalize.3.rst
@@ -0,0 +1,73 @@
+.. _mpi_finalize:
+
+MPI_Finalize
+============
+
+.. include_body
+
+:ref:`MPI_Finalize` |mdash| Terminates MPI execution environment.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_finalize.rst
+
+OUTPUT PARAMETER
+----------------
+
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This routine cleans up all MPI states. Once this routine is called, no
+MPI routine (not even MPI_Init) may be called, except for
+:ref:`MPI_Get_version`, :ref:`MPI_Initialized`, and :ref:`MPI_Finalized`. Unless there has
+been a call to :ref:`MPI_Abort`, you must ensure that all pending
+communications involving a process are complete before the process calls
+:ref:`MPI_Finalize`. If the call returns, each process may either continue
+local computations or exit without participating in further
+communication with other processes. At the moment when the last process
+calls :ref:`MPI_Finalize`, all pending sends must be matched by a receive, and
+all pending receives must be matched by a send.
+
+:ref:`MPI_Finalize` is collective over all connected processes. If no processes
+were spawned, accepted, or connected, then this means it is collective
+over MPI_COMM_WORLD. Otherwise, it is collective over the union of all
+processes that have been and continue to be connected.
+
+NOTES
+-----
+
+All processes must call this routine before exiting. All processes will
+still exist but may not make any further MPI calls. :ref:`MPI_Finalize`
+guarantees that all local actions required by communications the user
+has completed will, in fact, occur before it returns. However,
+:ref:`MPI_Finalize` guarantees nothing about pending communications that have
+not been completed; completion is ensured only by :ref:`MPI_Wait`, :ref:`MPI_Test`, or
+:ref:`MPI_Request_free` combined with some other verification of completion.
+
+For example, a successful return from a blocking communication operation
+or from :ref:`MPI_Wait` or :ref:`MPI_Test` means that the communication is completed
+by the user and the buffer can be reused, but does not guarantee that
+the local process has no more work to do. Similarly, a successful return
+from :ref:`MPI_Request_free` with a request handle generated by an :ref:`MPI_Isend`
+nullifies the handle but does not guarantee that the operation has
+completed. The :ref:`MPI_Isend` is complete only when a matching receive has
+completed.
+
+If you would like to cause actions to happen when a process finishes,
+attach an attribute to MPI_COMM_SELF with a callback function. Then,
+when :ref:`MPI_Finalize` is called, it will first execute the equivalent of an
+:ref:`MPI_Comm_free` on MPI_COMM_SELF. This will cause the delete callback
+function to be executed on all keys associated with MPI_COMM_SELF in an
+arbitrary order. If no key has been attached to MPI_COMM_SELF, then no
+callback is invoked. This freeing of MPI_COMM_SELF happens before any
+other parts of MPI are affected. Calling :ref:`MPI_Finalized` will thus return
+"false" in any of these callback functions. Once you have done this with
+MPI_COMM_SELF, the results of :ref:`MPI_Finalize` are not specified.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso:: :ref:`MPI_Init`
diff --git a/docs/man-openmpi/man3/MPI_Finalized.3.rst b/docs/man-openmpi/man3/MPI_Finalized.3.rst
new file mode 100644
index 00000000000..6507503fd44
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Finalized.3.rst
@@ -0,0 +1,32 @@
+.. _mpi_finalized:
+
+MPI_Finalized
+=============
+
+.. include_body
+
+:ref:`MPI_Finalized` |mdash| Checks whether MPI has been finalized
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_finalized.rst
+
+OUTPUT PARAMETER
+----------------
+
+* ``flag`` : True if MPI was finalized, and false otherwise (logical).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This routine may be used to determine whether MPI has been finalized. It
+is one of a small number of routines that may be called before MPI is
+initialized and after MPI has been finalized (:ref:`MPI_Initialized` is
+another).
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso:: :ref:`MPI_Init`
diff --git a/docs/man-openmpi/man3/MPI_Free_mem.3.rst b/docs/man-openmpi/man3/MPI_Free_mem.3.rst
new file mode 100644
index 00000000000..4f27353ebcb
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Free_mem.3.rst
@@ -0,0 +1,34 @@
+.. _mpi_free_mem:
+
+MPI_Free_mem
+============
+
+.. include_body
+
+:ref:`MPI_Free_mem` |mdash| Frees memory that has been allocated using :ref:`MPI_Alloc_mem`.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_free_mem.rst
+
+INPUT PARAMETER
+---------------
+
+* ``base`` : Initial address of memory segment allocated by :ref:`MPI_Alloc_mem`
+   (choice).
+
+OUTPUT PARAMETER
+----------------
+
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Free_mem` frees memory that has been allocated by :ref:`MPI_Alloc_mem`.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso:: :ref:`MPI_Alloc_mem`
diff --git a/docs/man-openmpi/man3/MPI_Gather.3.rst b/docs/man-openmpi/man3/MPI_Gather.3.rst
new file mode 100644
index 00000000000..70b7db6a35e
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Gather.3.rst
@@ -0,0 +1,173 @@
+.. _mpi_gather:
+
+MPI_Gather
+==========
+
+.. include_body
+
+:ref:`MPI_Gather`, :ref:`MPI_Igather`, :ref:`MPI_Gather_init` - Gathers values from a group
+of processes.
+
+Synopsis
+--------
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Gather, MPI_Igather, MPI_Gather_init
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_gather.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``sendbuf`` : Starting address of send buffer (choice).
+* ``sendcount`` : Number of elements in send buffer (integer).
+* ``sendtype`` : Datatype of send buffer elements (handle).
+* ``recvcount`` : Number of elements for any single receive (integer,
+  significant only at root).
+* ``recvtype`` : Datatype of recvbuffer elements (handle, significant only
+  at root).
+* ``root`` : Rank of receiving process (integer).
+* ``comm`` : Communicator (handle).
+* ``info`` : Info (handle, persistent only).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``recvbuf`` : Address of receive buffer (choice, significant only at
+   root).
+* ``request`` : Request (handle, non-blocking and persistent only).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Each process (root process included) sends the contents of its send
+buffer to the root process. The root process receives the messages and
+stores them in rank order. The outcome is as if each of the n processes
+in the group (including the root process) had executed a call to
+
+::
+
+   MPI_Send(sendbuf, sendcount, sendtype, root, ...)
+
+and the root had executed n calls to
+
+::
+
+   MPI_Recv(recfbuf + i * recvcount * extent(recvtype), recvcount,
+            recvtype, i, ...)
+
+where extent(recvtype) is the type extent obtained from a call to
+:ref:`MPI_Type_extent`.
+
+An alternative description is that the n messages sent by the processes
+in the group are concatenated in rank order, and the resulting message
+is received by the root as if by a call to
+``MPI_Recv(recvbuf, recvcount * n, recvtype, ... )``.
+
+The receive buffer is ignored for all nonroot processes.
+
+General, derived datatypes are allowed for both sendtype and recvtype.
+The type signature of sendcount, sendtype on process i must be equal to
+the type signature of recvcount, recvtype at the root. This implies that
+the amount of data sent must be equal to the amount of data received,
+pairwise between each process and the root. Distinct type maps between
+sender and receiver are still allowed.
+
+All arguments to the function are significant on process root, while on
+other processes, only arguments sendbuf, sendcount, sendtype, root, comm
+are significant. The arguments root and comm must have identical values
+on all processes.
+
+The specification of counts and types should not cause any location on
+the root to be written more than once. Such a call is erroneous. Note
+that the recvcount argument at the root indicates the number of items it
+receives from each process, not the total number of items it receives.
+
+Example 1: Gather 100 ints from every process in group to root.
+
+.. code-block:: c
+
+   MPI_Comm comm;
+   int gsize, sendarray[100];
+   int root, *rbuf;
+   ...
+
+   MPI_Comm_size( comm, &gsize);
+   rbuf = (int*)malloc(gsize* 100*sizeof(int));
+
+   MPI_Gather( sendarray, 100, MPI_INT, rbuf, 100, MPI_INT, root, comm);
+
+Example 2: Previous example modified |mdash| only the root allocates memory
+for the receive buffer.
+
+.. code-block:: c
+
+   MPI_Comm comm;
+   int gsize, sendarray[100];
+   int root, myrank, *rbuf;
+   ...
+
+   MPI_Comm_rank( comm, myrank);
+   if ( myrank == root) {
+     MPI_Comm_size( comm, &gsize);
+     rbuf = (int *)malloc(gsize * 100*sizeof(int));
+   }
+   MPI_Gather( sendarray, 100, MPI_INT, rbuf, 100, MPI_INT, root, comm);
+
+Example 3: Do the same as the previous example, but use a derived
+datatype. Note that the type cannot be the entire set of ``gsize * 100``
+ints since type matching is defined pairwise between the root and each
+process in the gather.
+
+.. code-block:: c
+
+   MPI_Comm comm;
+   int gsize, sendarray[100];
+   int root, *rbuf;
+   MPI_Datatype rtype;
+   ...
+
+   MPI_Comm_size( comm, &gsize);
+   MPI_Type_contiguous( 100, MPI_INT, &rtype);
+   MPI_Type_commit( &rtype );
+   rbuf = (int*)malloc(gsize* 100*sizeof(int));
+   MPI_Gather( sendarray, 100, MPI_INT, rbuf, 1, rtype, root, comm);
+
+
+Use Of In-Place Option
+----------------------
+
+When the communicator is an intracommunicator, you can perform a gather
+operation in-place (the output buffer is used as the input buffer). Use
+the variable MPI_IN_PLACE as the value of the root process sendbuf. In
+this case, sendcount and sendtype are ignored, and the contribution of
+the root process to the gathered vector is assumed to already be in the
+correct place in the receive buffer. Note that MPI_IN_PLACE is a special
+kind of value; it has the same restrictions on its use as MPI_BOTTOM.
+Because the in-place option converts the receive buffer into a
+send-and-receive buffer, a Fortran binding that includes INTENT must
+mark these as INOUT, not OUT.
+
+When Communicator Is An Inter-Communicator
+------------------------------------------
+
+When the communicator is an inter-communicator, the root process in the
+first group gathers data from all the processes in the second group. The
+first group defines the root process. That process uses MPI_ROOT as the
+value of its root argument. The remaining processes use ``MPI_PROC_NULL`` as
+the value of their root argument. All processes in the second group use
+the rank of that root process in the first group as the value of their
+root argument. The send buffer argument of the processes in the first
+group must be consistent with the receive buffer argument of the root
+process in the second group.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Gatherv`
diff --git a/docs/man-openmpi/man3/MPI_Gather_init.3.rst b/docs/man-openmpi/man3/MPI_Gather_init.3.rst
new file mode 100644
index 00000000000..bfbb28c0ed3
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Gather_init.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_gather_init:
+
+MPI_Gather_init
+===============
+    .. include_body
+
+.. include:: ../man3/MPI_Gather.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Gatherv.3.rst b/docs/man-openmpi/man3/MPI_Gatherv.3.rst
new file mode 100644
index 00000000000..0460135472d
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Gatherv.3.rst
@@ -0,0 +1,334 @@
+.. _mpi_gatherv:
+
+MPI_Gatherv
+===========
+
+.. include_body
+
+:ref:`MPI_Gatherv`, :ref:`MPI_Igatherv`, :ref:`MPI_Gatherv_init` - Gathers varying amounts of
+data from all processes to the root process
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Gatherv, MPI_Igatherv, MPI_Gatherv_init
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_gatherv.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``sendbuf`` : Starting address of send buffer (choice).
+* ``sendcount`` : Number of elements in send buffer (integer).
+* ``sendtype`` : Datatype of send buffer elements (handle).
+* ``recvcounts`` : Integer array (of length group size) containing the
+   number of elements that are received from each process (significant
+   only at root).
+* ``displs`` : Integer array (of length group size). Entry i specifies the
+   displacement relative to recvbuf at which to place the incoming data
+   from process i (significant only at root).
+* ``recvtype`` : Datatype of recv buffer elements (significant only at
+   root) (handle).
+* ``root`` : Rank of receiving process (integer).
+* ``comm`` : Communicator (handle).
+* ``info`` : Info (handle, persistent only).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``recvbuf`` : Address of receive buffer (choice, significant only at
+   root).
+* ``request`` : Request (handle, non-blocking and persistent only).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Gatherv` extends the functionality of :ref:`MPI_Gather` by allowing a
+varying count of data from each process, since recvcounts is now an
+array. It also allows more flexibility as to where the data is placed on
+the root, by providing the new argument, displs.
+
+The outcome is as if each process, including the root process, sends a
+message to the root,
+
+.. code-block:: c
+
+   MPI_Send(sendbuf, sendcount, sendtype, root, ...);
+
+and the root executes n receives,
+
+.. code-block:: c
+
+   MPI_Recv(recvbuf + disp[i] * extent(recvtype), recvcounts[i],
+            recvtype, i, ...);
+
+Messages are placed in the receive buffer of the root process in rank
+order, that is, the data sent from process j is placed in the jth
+portion of the receive buffer recvbuf on process root. The jth portion
+of recvbuf begins at offset displs[j] elements (in terms of recvtype)
+into recvbuf.
+
+The receive buffer is ignored for all nonroot processes.
+
+The type signature implied by sendcount, sendtype on process i must be
+equal to the type signature implied by recvcounts[i], recvtype at the
+root. This implies that the amount of data sent must be equal to the
+amount of data received, pairwise between each process and the root.
+Distinct type maps between sender and receiver are still allowed, as
+illustrated in Example 2, below.
+
+All arguments to the function are significant on process root, while on
+other processes, only arguments sendbuf, sendcount, sendtype, root, comm
+are significant. The arguments root and comm must have identical values
+on all processes.
+
+The specification of counts, types, and displacements should not cause
+any location on the root to be written more than once. Such a call is
+erroneous.
+
+Example 1: Now have each process send 100 ints to root, but place each
+set (of 100) stride ints apart at receiving end. Use :ref:`MPI_Gatherv` and the
+displs argument to achieve this effect. Assume stride >= 100.
+
+.. code-block:: c
+
+   MPI_Comm comm;
+   int gsize, sendarray[100];
+   int root, *rbuf, stride;
+   int *displs, i, rcounts;
+   ...
+
+   MPI_Comm_size(comm, &gsize);
+   rbuf = (int)malloc(gsize * stride * sizeof(int));
+   displs = (int)malloc(gsize * sizeof(int));
+   rcounts = (int )malloc(gsize * sizeof(int));
+
+   for (i=0; i<gsize; ++i) {
+     displs[i] = i * stride;
+     rcounts[i] = 100;
+   }
+   MPI_Gatherv(sendarray, 100, MPI_INT, rbuf, rcounts, displs, MPI_INT,
+               root, comm);
+
+Note that the program is erroneous if stride < 100.
+
+Example 2: Same as Example 1 on the receiving side, but send the 100
+ints from the 0th column of a 100 150 int array, in C.
+
+.. code-block:: c
+
+   MPI_Comm comm;
+   int gsize, sendarray[100][150];
+   int root, *rbuf, stride;
+   MPI_Datatype stype;
+   int displs,i, rcounts;
+   ...
+
+   MPI_Comm_size(comm, &gsize);
+   rbuf = (int )malloc(gsize * stride * sizeof(int));
+   displs = (int)malloc(gsize * sizeof(int));
+   rcounts = (int )malloc(gsize * sizeof(int));
+
+   for (i=0; i<gsize; ++i) {
+     displs[i] = i * stride;
+     rcounts[i] = 100;
+   }
+
+   // Create datatype for 1 column of array
+   MPI_Type_vector(100, 1, 150, MPI_INT, &stype);
+   MPI_Type_commit( &stype );
+   MPI_Gatherv(sendarray, 1, stype, rbuf, rcounts, displs, MPI_INT,
+               root, comm);
+
+Example 3: Process i sends (100-i) ints from the ith column of a 100 x
+150 int array, in C. It is received into a buffer with stride, as in the
+previous two examples.
+
+.. code-block:: c
+
+   MPI_Comm comm;
+   int gsize, sendarray[100][150], *sptr;
+   int root, *rbuf, stride, myrank;
+   MPI_Datatype stype;
+   int displs, i, rcounts;
+   ...
+
+   MPI_Comm_size(comm, &gsize);
+   MPI_Comm_rank( comm, &myrank );
+   rbuf = (int)malloc(gsize * stride * sizeof(int));
+   displs = (int)malloc(gsize * sizeof(int));
+   rcounts = (int )malloc(gsize * sizeof(int));
+
+   for (i=0; i<gsize; ++i) {
+     displs[i] = i * stride;
+     rcounts[i] = 100-i; // note change from previous example
+   }
+
+   // Create datatype for the column we are sending
+   MPI_Type_vector(100-myrank, 1, 150, MPI_INT, &stype);
+   MPI_Type_commit( &stype );
+   // sptr is the address of start of "myrank" column
+   sptr = &sendarray[0][myrank];
+   MPI_Gatherv(sptr, 1, stype, rbuf, rcounts, displs, MPI_INT,
+               root, comm);
+
+Note that a different amount of data is received from each process.
+
+Example 4: Same as Example 3, but done in a different way at the sending
+end. We create a datatype that causes the correct striding at the
+sending end so that we read a column of a C array.
+
+.. code-block:: c
+
+   MPI_Comm comm;
+   int gsize, sendarray[100][150], *sptr;
+   int root, *rbuf, stride, myrank, disp[2], blocklen[2];
+   MPI_Datatype stype, type[2];
+   int displs, i, rcounts;
+   ...
+
+   MPI_Comm_size(comm, &gsize);
+   MPI_Comm_rank(comm, &myrank );
+   rbuf = (int )alloc(gsize * stride * sizeof(int));
+   displs = (int )malloc(gsize * sizeof(int));
+   rcounts = (int)malloc(gsize * sizeof(int));
+
+   for (i=0; i<gsize; ++i) {
+     displs[i] = i* stride;
+     rcounts[i] = 100-i;
+   }
+   // Create datatype for one int, with extent of entire row
+   disp[0] = 0;
+   disp[1] = 150 * sizeof(int);
+   type[0] = MPI_INT;
+   type[1] = MPI_UB;
+   blocklen[0] = 1;
+   blocklen[1] = 1;
+
+   MPI_Type_struct( 2, blocklen, disp, type, &stype );
+   MPI_Type_commit(&stype );
+   sptr = &sendarray[0][myrank];
+   MPI_Gatherv(sptr, 100-myrank, stype, rbuf, rcounts, displs, MPI_INT,
+               root, comm);
+
+Example 5: Same as Example 3 at sending side, but at receiving side we
+make the stride between received blocks vary from block to block.
+
+.. code-block:: c
+
+   MPI_Comm comm;
+   int gsize, sendarray[100][150], *sptr;
+   int root, *rbuf, *stride, myrank, bufsize;
+   MPI_Datatype stype;
+   int *displs, i, *rcounts, offset;
+   ...
+
+   MPI_Comm_size( comm, &gsize);
+   MPI_Comm_rank( comm, &myrank );
+   de = (int )malloc(gsize * sizeof(int));
+   ...
+   // stride[i] for i = 0 to gsize-1 is set somehow
+
+   // set up displs and rcounts vectors first
+   displs = (int)malloc(gsize * sizeof(int));
+   rcounts = (int )malloc(gsize * sizeof(int));
+   offset = 0;
+
+   for (i=0; i<gsize; ++i) {
+     displs[i] = offset;
+     offset += stride[i];
+     rcounts[i] = 100-i;
+   }
+
+   // the required buffer size for rbuf is now easily obtained
+   bufsize = displs[gsize-1]+rcounts[gsize-1];
+   rbuf = (int )malloc(bufsize * sizeof(int));
+   // Create datatype for the column we are sending
+   MPI_Type_vector(100-myrank, 1, 150, MPI_INT, &stype);
+   MPI_Type_commit( &stype );
+   sptr = &sendarray[0][myrank];
+   MPI_Gatherv(sptr, 1, stype, rbuf, rcounts, displs, MPI_INT,
+               root, comm);
+
+Example 6: Process i sends num ints from the ith column of a 100 x 150
+int array, in C. The complicating factor is that the various values of
+num are not known to root, so a separate gather must first be run to
+find these out. The data is placed contiguously at the receiving end.
+
+.. code-block:: c
+
+   MPI_Comm comm;
+   int gsize, sendarray[100][150], *sptr;
+   int root, *rbuf, stride, myrank, disp[2], blocklen[2];
+   MPI_Datatype stype,types[2];
+   int *displs, i, *rcounts, num;
+   ...
+
+   MPI_Comm_size( comm, &gsize);
+   MPI_Comm_rank( comm, &myrank );
+
+   // First, gather nums to root
+   rcounts = (int )malloc(gsize * sizeof(int));
+   MPI_Gather( &num, 1, MPI_INT, rcounts, 1, MPI_INT, root, comm);
+   // root now has correct rcounts, using these we set
+   // displs[] so that data is placed contiguously (or concatenated) at receive end
+
+   displs = (int)malloc(gsize * sizeof(int));
+   displs[0] = 0;
+   for (i=1; i<gsize; ++i) {
+     displs[i] = displs[i-1]+rcounts[i-1];
+   }
+
+   // And, create receive buffer
+   rbuf = (int *)malloc(gsize * (displs[gsize-1]+rcounts[gsize-1]) * sizeof(int));
+   // Create datatype for one int, with extent of entire row
+   disp[0] = 0;
+   disp[1] = 150 * sizeof(int);
+   type[0] = MPI_INT;
+   type[1] = MPI_UB;
+   blocklen[0] = 1;
+   blocklen[1] = 1;
+   MPI_Type_struct(2, blocklen, disp, type, &stype );
+   MPI_Type_commit( &stype );
+   sptr = &sendarray[0][myrank];
+   MPI_Gatherv(sptr, num, stype, rbuf, rcounts, displs, MPI_INT, root, comm);
+
+
+Use Of In-Place Option
+----------------------
+
+The in-place option operates in the same way as it does for :ref:`MPI_Gather`.
+When the communicator is an intracommunicator, you can perform a gather
+operation in-place (the output buffer is used as the input buffer). Use
+the variable MPI_IN_PLACE as the value of the root process sendbuf. In
+this case, sendcount and sendtype are ignored, and the contribution of
+the root process to the gathered vector is assumed to already be in the
+correct place in the receive buffer.
+
+Note that MPI_IN_PLACE is a special kind of value; it has the same
+restrictions on its use as MPI_BOTTOM.
+
+Because the in-place option converts the receive buffer into a
+send-and-receive buffer, a Fortran binding that includes INTENT must
+mark these as INOUT, not OUT.
+
+When Communicator Is An Inter-Communicator
+------------------------------------------
+
+When the communicator is an inter-communicator, the root process in the
+first group gathers data from all the processes in the second group. The
+first group defines the root process. That process uses MPI_ROOT as the
+value of its root argument. The remaining processes use ``MPI_PROC_NULL`` as
+the value of their root argument. All processes in the second group use
+the rank of that root process in the first group as the value of their
+root argument. The send buffer argument of the processes in the first
+group must be consistent with the receive buffer argument of the root
+process in the second group.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso:: :ref:`MPI_Gather`
diff --git a/docs/man-openmpi/man3/MPI_Gatherv_init.3.rst b/docs/man-openmpi/man3/MPI_Gatherv_init.3.rst
new file mode 100644
index 00000000000..3ec009dca39
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Gatherv_init.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_gatherv_init:
+
+MPI_Gatherv_init
+================
+    .. include_body
+
+.. include:: ../man3/MPI_Gatherv.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Get.3.rst b/docs/man-openmpi/man3/MPI_Get.3.rst
new file mode 100644
index 00000000000..bc215f64e0f
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Get.3.rst
@@ -0,0 +1,61 @@
+.. _mpi_get:
+
+MPI_Get
+=======
+
+.. include_body
+
+:ref:`MPI_Get`, :ref:`MPI_Rget` - Copies data from the target memory to the origin.
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Get, MPI_Rget
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_get.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``origin_addr`` : Initial address of origin buffer (choice).
+* ``origin_count`` : Number of entries in origin buffer (nonnegative
+   integer).
+* ``origin_datatype`` : Data type of each entry in origin buffer (handle).
+* ``target_rank`` : Rank of target (nonnegative integer).
+* ``target_disp`` : Displacement from window start to the beginning of the
+   target buffer (nonnegative integer).
+* ``target_count`` : Number of entries in target buffer (nonnegative
+   integer).
+* target datatype : datatype of each entry in target buffer (handle)
+* ``win`` : window object used for communication (handle)
+
+OUTPUT PARAMETER
+----------------
+
+* ``request`` : :ref:`MPI_Rget`: RMA request
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Get` copies data from the target memory to the origin, similar to
+:ref:`MPI_Put`, except that the direction of data transfer is reversed. The
+origin_datatype may not specify overlapping entries in the origin
+buffer. The target buffer must be contained within the target window,
+and the copied data must fit, without truncation, in the origin buffer.
+Only processes within the same node can access the target window.
+
+:ref:`MPI_Rget` is similar to :ref:`MPI_Get`, except that it allocates a communication
+request object and associates it with the request handle (the argument
+request) that can be used to wait or test for completion. The completion
+of an :ref:`MPI_Rget` operation indicates that the data is available in the
+origin buffer. If origin_addr points to memory attached to a window,
+then the data becomes available in the private copy of this window.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso:: :ref:`MPI_Put`
diff --git a/docs/man-openmpi/man3/MPI_Get_accumulate.3.rst b/docs/man-openmpi/man3/MPI_Get_accumulate.3.rst
new file mode 100644
index 00000000000..5a717ef5836
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Get_accumulate.3.rst
@@ -0,0 +1,109 @@
+.. _mpi_get_accumulate:
+
+MPI_Get_accumulate
+==================
+
+.. include_body
+
+:ref:`MPI_Get_accumulate`, :ref:`MPI_Rget_accumulate` - Combines the contents of the
+origin buffer with that of a target buffer and returns the target buffer
+value.
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Get_accumulate, MPI_Rget_accumulate
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_get_accumulate.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``origin_addr`` : Initial address of buffer (choice).
+* ``origin_count`` : Number of entries in buffer (nonnegative integer).
+* ``origin_datatype`` : Data type of each buffer entry (handle).
+* ``result_addr`` : Initial address of result buffer (choice).
+* ``result_count`` : Number of entries in result buffer (nonnegative
+   integer).
+* ``result_datatype`` : Data type of each result buffer entry (handle).
+* ``target_rank`` : Rank of target (nonnegative integer).
+* ``target_disp`` : Displacement from start of window to beginning of
+   target buffer (nonnegative integer).
+* ``target_count`` : Number of entries in target buffer (nonnegative
+   integer).
+* ``target_datatype`` : Data type of each entry in target buffer (handle).
+* ``op`` : Reduce operation (handle).
+* ``win`` : Window object (handle).
+
+OUTPUT PARAMETER
+----------------
+
+* :ref:`MPI_Rget_accumulate`: RMA request
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Get_accumulate` is a function used for one-sided MPI communication
+that adds the contents of the origin buffer (as defined by origin_addr,
+origin_count, and origin_datatype) to the buffer specified by the
+arguments target_count and target_datatype, at offset target_disp, in
+the target window specified by target_rank and win, using the operation
+op. :ref:`MPI_Get_accumulate` returns in the result buffer result_addr the
+contents of the target buffer before the accumulation.
+
+Any of the predefined operations for :ref:`MPI_Reduce`, as well as MPI_NO_OP,
+can be used. User-defined functions cannot be used. For example, if op
+is MPI_SUM, each element of the origin buffer is added to the
+corresponding element in the target, replacing the former value in the
+target.
+
+Each datatype argument must be a predefined data type or a derived data
+type, where all basic components are of the same predefined data type.
+Both datatype arguments must be constructed from the same predefined
+data type. The operation op applies to elements of that predefined type.
+The target_datatype argument must not specify overlapping entries, and
+the target buffer must fit in the target window.
+
+A new predefined operation, MPI_REPLACE, is defined. It corresponds to
+the associative function f(a, b) =b; that is, the current value in the
+target memory is replaced by the value supplied by the origin.
+
+A new predefined operation, MPI_NO_OP, is defined. It corresponds to the
+assiciative function f(a, b) = a; that is the current value in the
+target memory is returned in the result buffer at the origin and no
+operation is performed on the target buffer.
+
+:ref:`MPI_Rget_accumulate` is similar to :ref:`MPI_Get_accumulate`, except that it
+allocates a communication request object and associates it with the
+request handle (the argument request) that can be used to wait or test
+for completion. The completion of an :ref:`MPI_Rget_accumulate` operation
+indicates that the data is available in the result buffer and the origin
+buffer is free to be updated. It does not indicate that the operation
+has been completed at the target window.
+
+
+NOTES
+-----
+
+The generic functionality of :ref:`MPI_Get_accumulate` might limit the
+performance of fetch-and-increment or fetch-and-add calls that might be
+supported by special hardware operations. :ref:`MPI_Fetch_and_op` thus allows
+for a fast implementation of a commonly used subset of the functionality
+of :ref:`MPI_Get_accumulate`.
+
+:ref:`MPI_Get` is a special case of :ref:`MPI_Get_accumulate`, with the operation
+MPI_NO_OP. Note, however, that :ref:`MPI_Get` and :ref:`MPI_Get_accumulate` have
+different constraints on concurrent updates.
+
+It is the user's responsibility to guarantee that, when using the
+accumulate functions, the target displacement argument is such that
+accesses to the window are properly aligned according to the data type
+arguments in the call to the :ref:`MPI_Get_accumulate` function.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso:: :ref:`MPI_Put` :ref:`MPI_Reduce`
diff --git a/docs/man-openmpi/man3/MPI_Get_address.3.rst b/docs/man-openmpi/man3/MPI_Get_address.3.rst
new file mode 100644
index 00000000000..1d37711d9b1
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Get_address.3.rst
@@ -0,0 +1,73 @@
+.. _mpi_get_address:
+
+MPI_Get_address
+===============
+
+.. include_body
+
+:ref:`MPI_Get_address` |mdash| Gets the address of a location in memory.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_get_address.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``location`` : Location in caller memory (choice).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``address`` : Address of location (integer).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Get_address` returns the byte ``address`` of a location in
+memory.
+
+Example: Using :ref:`MPI_Get_address` for an array.
+
+.. code-block:: fortran
+
+   REAL :: A(100,100)
+   INTEGER(MPI_ADDRESS_KIND) :: I1, I2, DIFF
+
+   CALL MPI_GET_ADDRESS(A(1,1), I1, IERROR)
+   CALL MPI_GET_ADDRESS(A(10,10), I2, IERROR)
+   DIFF = I2 - I1
+   ! The value of DIFF is 909*sizeofreal; the values of I1 and I2 are
+   ! implementation dependent.
+
+
+NOTES
+-----
+
+This routine is provided for both Fortran and C programmers and may be
+useful when writing portable code. In the current release, the address
+returned by this routine will be the same as that produced by the C ``&``
+operator.
+
+C users may be tempted to avoid using :ref:`MPI_Get_address` and rely on the
+availability of the address operator ``&``. Note, however, that &
+cast-expression is a pointer, not an address. ANSI C does not require
+that the value of a pointer (or the pointer cast to int) be the absolute
+address of the object pointed at although this is commonly the case.
+Furthermore, referencing may not have a unique definition on machines
+with a segmented address space. The use of :ref:`MPI_Get_address` to "reference" C
+variables guarantees portability to such machines as well.
+
+Current Fortran MPI codes will run unmodified and will port to any
+system. However, they may fail if ``addresses`` larger than 2^32 - 1 are
+used in the program. New codes should be written so that they use the
+new functions. This provides compatibility with C and avoids errors on
+64-bit architectures. However, such newly written codes may need to be
+(slightly) rewritten to port to old Fortran 77 environments that do not
+support KIND declarations.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Get_count.3.rst b/docs/man-openmpi/man3/MPI_Get_count.3.rst
new file mode 100644
index 00000000000..439585f7075
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Get_count.3.rst
@@ -0,0 +1,57 @@
+.. _mpi_get_count:
+
+MPI_Get_count
+=============
+
+.. include_body
+
+:ref:`MPI_Get_count` |mdash| Gets the number of top-level elements received.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_get_count.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``status`` : Return status of receive operation (status).
+* ``datatype`` : Datatype of each receive buffer element (handle).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``count`` : Number of received elements (integer).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Returns the number of entries received. (We count entries, each of type
+datatype, not bytes.) The datatype argument should match the argument
+provided by the receive call that set the status variable. As explained
+in the "Use of General Datatypes in Communication" section of the `MPI
+Standard <https://www.mpi-forum.org/docs/>`_.
+:ref:`MPI_Get_count` may, in certain situations, return the value
+``MPI_UNDEFINED``.
+
+The datatype argument is passed to :ref:`MPI_Get_count` to improve performance.
+A message might be received without counting the number of elements it
+contains, and the count value is often not needed. Also, this allows the
+same function to be used after a call to :ref:`MPI_Probe`.
+
+NOTES
+-----
+
+If the size of the datatype is zero, this routine will return a count of
+zero. If the amount of data in status is not an exact multiple of the
+size of datatype (so that count would not be integral), a count of
+``MPI_UNDEFINED`` is returned instead.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+If the value to be returned is larger than can fit into the count
+parameter, an ``MPI_ERR_TRUNCATE`` error is raised.
+
+.. seealso:: :ref:`MPI_Get_elements`
diff --git a/docs/man-openmpi/man3/MPI_Get_elements.3.rst b/docs/man-openmpi/man3/MPI_Get_elements.3.rst
new file mode 100644
index 00000000000..0ff29faa06b
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Get_elements.3.rst
@@ -0,0 +1,78 @@
+.. _mpi_get_elements:
+
+MPI_Get_elements
+================
+
+.. include_body
+
+:ref:`MPI_Get_elements`, :ref:`MPI_Get_elements_x` - Returns the number of basic
+elements in a data type.
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Get_elements, MPI_Get_elements_x
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_get_elements.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``status`` : Return status of receive operation (status).
+* ``datatype`` : Datatype used by receive operation (handle).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Get_elements` and :ref:`MPI_Get_elements_x` behave different from
+:ref:`MPI_Get_count`, which returns the number of "top-level entries" received,
+i.e., the number of "copies" of type datatype. :ref:`MPI_Get_count` may return
+any integer value k, where 0 =< k =< count. If :ref:`MPI_Get_count` returns k,
+then the number of basic elements received (and the value returned by
+:ref:`MPI_Get_elements` and MPI_Get_elements_x) is n k, where n is the number
+of basic elements in the type map of datatype. If the number of basic
+elements received is not a multiple of n, that is, if the receive
+operation has not received an integral number of datatype "copies," then
+:ref:`MPI_Get_count` returns the value MPI_UNDEFINED. For both functions, if
+the count parameter cannot express the value to be returned (e.g., if
+the parameter is too small to hold the output value), it is set to
+MPI_UNDEFINED.
+
+Example: Usage of :ref:`MPI_Get_count` and :ref:`MPI_Get_elements`:
+
+.. code-block:: fortran
+
+   call MPI_TYPE_CONTIGUOUS(2, MPI_REAL, Type2, ierr)
+   call MPI_TYPE_COMMIT(Type2, ierr)
+   call MPI_COMM_RANK(comm, rank, ierr)
+
+   IF ( rank == 0 ) THEN
+      CALL MPI_SEND(a, 2, MPI_REAL, 1, 0, comm, ierr)
+      CALL MPI_SEND(a, 3, MPI_REAL, 1, 0, comm, ierr)
+   ELSE
+      CALL MPI_RECV(a, 2, Type2, 0, 0, comm, stat, ierr)
+      CALL MPI_GET_COUNT(stat, Type2, i, ierr) ! returns i=1
+      CALL MPI_GET_ELEMENTS(stat, Type2, i, ierr) ! returns i=2
+      CALL MPI_RECV(a, 2, Type2, 0, 0, comm, stat, ierr)
+      CALL MPI_GET_COUNT(stat, Type2, i, ierr) ! returns i=MPI_UNDEFINED
+      CALL MPI_GET_ELEMENTS(stat, Type2, i, ierr)  ! returns i=3
+   END IF
+
+The function :ref:`MPI_Get_elements` can also be used after a probe to find the
+number of elements in the probed message. Note that the two functions
+:ref:`MPI_Get_count` and :ref:`MPI_Get_elements` return the same values when they are
+used with primitive data types.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+
+.. seealso:: :ref:`MPI_Get_count`
diff --git a/docs/man-openmpi/man3/MPI_Get_elements_x.3.rst b/docs/man-openmpi/man3/MPI_Get_elements_x.3.rst
new file mode 100644
index 00000000000..82ab6efc01b
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Get_elements_x.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_get_elements_x:
+
+MPI_Get_elements_x
+==================
+    .. include_body
+
+.. include:: ../man3/MPI_Get_elements.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Get_library_version.3.rst b/docs/man-openmpi/man3/MPI_Get_library_version.3.rst
new file mode 100644
index 00000000000..0f52083ca4e
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Get_library_version.3.rst
@@ -0,0 +1,50 @@
+.. _mpi_get_library_version:
+
+MPI_Get_library_version
+=======================
+
+.. include_body
+
+:ref:`MPI_Get_library_version` |mdash| Returns a string of the current Open MPI
+version
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_get_library_version.rst
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``version`` : A string containing the Open MPI version (string).
+* ``resultlen`` : Length (in characters) of result returned in version
+   (integer).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This routine returns a string representing the version of the MPI
+library. The version argument is a character string for maximum
+flexibility.
+
+The number of characters actually written is returned in the output
+argument, resultlen. In C, a '0' character is additionally stored at
+version[resultlen]. The resultlen cannot be larger than
+(MPI_MAX_LIBRARY_VERSION_STRING - 1). In Fortran, version is padded on
+the right with blank characters. The resultlen cannot be larger than
+MPI_MAX_LIBRARY_VERSION_STRING.
+
+NOTE
+----
+
+The version string that is passed must be at least
+MPI_MAX_LIBRARY_VERSION_STRING characters long.
+
+:ref:`MPI_Get_library_version` is one of the few functions that can be called
+before :ref:`MPI_Init` and after :ref:`MPI_Finalize`.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso:: :ref:`MPI_Get_version`
diff --git a/docs/man-openmpi/man3/MPI_Get_processor_name.3.rst b/docs/man-openmpi/man3/MPI_Get_processor_name.3.rst
new file mode 100644
index 00000000000..7736b99b3de
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Get_processor_name.3.rst
@@ -0,0 +1,44 @@
+.. _mpi_get_processor_name:
+
+MPI_Get_processor_name
+======================
+
+.. include_body
+
+:ref:`MPI_Get_processor_name` |mdash| Gets the name of the processor.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_get_processor_name.rst
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``name`` : A unique specifier for the actual (as opposed to virtual)
+  node.
+* ``resultlen`` : Length (in characters) of result returned in name.
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This routine returns the ``name`` of the processor on which it was
+called at the moment of the call. The ``name`` is a character string for
+maximum flexibility. From this value it must be possible to identify a
+specific piece of hardware. The argument ``name`` must represent storage
+that is at least MPI_MAX_PROCESSOR_NAME characters long.
+
+The number of characters actually written is returned in the output
+argument, ``resultlen``.
+
+NOTES
+-----
+
+The user must provide at least MPI_MAX_PROCESSOR_NAME space to write
+the processor ``name``; processor ``name``\ s can be this long. The user
+should examine the output argument, ``resultlen``, to determine the
+actual length of the ``name``.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Get_version.3.rst b/docs/man-openmpi/man3/MPI_Get_version.3.rst
new file mode 100644
index 00000000000..9d59d8c5412
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Get_version.3.rst
@@ -0,0 +1,41 @@
+.. _mpi_get_version:
+
+MPI_Get_version
+===============
+
+.. include_body
+
+:ref:`MPI_Get_version` |mdash| Returns the version of the standard corresponding
+to the current implementation.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_get_version.rst
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``version`` : The major version number of the corresponding standard
+  (integer).
+* ``subversion`` : The minor version number of the corresponding
+  standard (integer).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Since Open MPI |ompi_ver| is MPI-|mpi_standard_version| compliant (see
+:ref:`this page for more details about Open MPI standards conformance
+<release-notes-mpi-standard-conformance-label>`) this function will return a
+``version`` value of |mpi_standard_major_version| and a ``subversion``
+value of |mpi_standard_minor_version| for this release.
+
+NOTE
+----
+
+:ref:`MPI_Get_version` is one of the few functions that can be called
+before :ref:`MPI_Init` and after :ref:`MPI_Finalize`.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Graph_create.3.rst b/docs/man-openmpi/man3/MPI_Graph_create.3.rst
new file mode 100644
index 00000000000..150488cd8ac
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Graph_create.3.rst
@@ -0,0 +1,90 @@
+.. _mpi_graph_create:
+
+MPI_Graph_create
+================
+
+.. include_body
+
+:ref:`MPI_Graph_create` |mdash| Makes a new communicator to which topology
+information has been attached.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_graph_create.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``comm_old`` : Input communicator without topology (handle).
+* ``nnodes`` : Number of nodes in graph (integer).
+* ``index`` : Array of integers describing node degrees (see below).
+* ``edges`` : Array of integers describing graph edges (see below).
+* ``reorder`` : Ranking may be reordered (true) or not (false) (logical).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``comm_graph`` : Communicator with graph topology added (handle).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Graph_create` returns a handle to a new communicator to which the
+graph topology information is attached. If reorder = false then the rank
+of each process in the new group is identical to its rank in the old
+group. Otherwise, the function may reorder the processes. If the size,
+nnodes, of the graph is smaller than the size of the group of comm_old,
+then some processes are returned MPI_COMM_NULL, in analogy to
+:ref:`MPI_Cart_create` and :ref:`MPI_Comm_split`. The call is erroneous if it
+specifies a graph that is larger than the group size of the input
+communicator.
+
+The three parameters nnodes, index, and edges define the graph
+structure. nnodes is the number of nodes of the graph. The nodes are
+numbered from 0 to nnodes-1. The ith entry of array index stores the
+total number of neighbors of the first i graph nodes. The lists of
+neighbors of nodes 0, 1, ..., nnodes-1 are stored in consecutive
+locations in array edges. The array edges is a flattened representation
+of the edge lists. The total number of entries in index is nnodes and
+the total number of entries in edges is equal to the number of graph
+edges.
+
+The definitions of the arguments nnodes, index, and edges are
+illustrated with the following simple example.
+
+Example: Assume there are four processes 0, 1, 2, 3 with the following
+adjacency matrix:
+
+------- ---------
+Process Neighbors
+------- ---------
+0       1, 3
+1       0
+2       3
+3       0, 2
+------- ---------
+
+Then, the input arguments are:
+
+* nodes = 4
+* index = 2, 3, 4, 6
+* edges = 1, 3, 0, 3, 0, 2
+
+Thus, in C, index[0] is the degree of node zero, and index[i] -
+index[i-1] is the degree of node i, i=1, . . . , nnodes-1; the list of
+neighbors of node zero is stored in edges[j], for 0 <= j <= index[0] - 1
+and the list of neighbors of node i, i > 0 , is stored in edges[j],
+index[i-1] <= j <= index[i] - 1.
+
+In Fortran, index(1) is the degree of node zero, and index(i+1) -
+index(i) is the degree of node i, i=1, . . . , nnodes-1; the list of
+neighbors of node zero is stored in edges(j), for 1 <= j <= index(1) and
+the list of neighbors of node i, i > 0, is stored in edges(j), index(i)
++ 1 <= j <= index(i + 1).
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso:: :ref:`MPI_Graph_get`
diff --git a/docs/man-openmpi/man3/MPI_Graph_get.3.rst b/docs/man-openmpi/man3/MPI_Graph_get.3.rst
new file mode 100644
index 00000000000..55a9cb723d3
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Graph_get.3.rst
@@ -0,0 +1,44 @@
+.. _mpi_graph_get:
+
+MPI_Graph_get
+=============
+
+.. include_body
+
+:ref:`MPI_Graph_get` |mdash| Retrieves graph topology information associated with a
+communicator.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_graph_get.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``comm`` : Communicator with graph structure (handle).
+* ``maxindex`` : Length of vector index in the calling program (integer).
+* ``maxedges`` : Length of vector edges in the calling program (integer).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``index`` : Array of integers containing the graph structure (for details
+   see the definition of MPI_Graph_create).
+* ``edges`` : Array of integers containing the graph structure.
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Functions :ref:`MPI_Graphdims_get` and :ref:`MPI_Graph_get` retrieve the
+graph-topology information that was associated with a communicator by
+:ref:`MPI_Graph_create`.
+
+The information provided by :ref:`MPI_Graphdims_get` can be used to dimension
+the vectors index and edges correctly for a call to :ref:`MPI_Graph_get`.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso:: :ref:`MPI_Graph_create`
diff --git a/docs/man-openmpi/man3/MPI_Graph_map.3.rst b/docs/man-openmpi/man3/MPI_Graph_map.3.rst
new file mode 100644
index 00000000000..68bfd108eb9
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Graph_map.3.rst
@@ -0,0 +1,42 @@
+.. _mpi_graph_map:
+
+MPI_Graph_map
+=============
+
+.. include_body
+
+:ref:`MPI_Graph_map` |mdash| Maps process to graph topology information.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_graph_map.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``comm`` : Input communicator (handle).
+* ``nnodes`` : Number of graph nodes (integer).
+* ``index`` : Integer array specifying the graph structure, see
+   :ref:`MPI_Graph_create`.
+* ``edges`` : Integer array specifying the graph structure.
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``newrank`` : Reordered rank of the calling process; MPI_UNDEFINED if the
+   calling process does not belong to graph (integer).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Cart_map` and :ref:`MPI_Graph_map` can be used to implement all other
+topology functions. In general they will not be called by the user
+directly, unless he or she is creating additional virtual topology
+capability other than that provided by MPI.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso:: :ref:`MPI_Cart_map`
diff --git a/docs/man-openmpi/man3/MPI_Graph_neighbors.3.rst b/docs/man-openmpi/man3/MPI_Graph_neighbors.3.rst
new file mode 100644
index 00000000000..e997a716c31
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Graph_neighbors.3.rst
@@ -0,0 +1,77 @@
+.. _mpi_graph_neighbors:
+
+
+MPI_Graph_neighbors
+===================
+
+.. include_body
+
+:ref:`MPI_Graph_neighbors` |mdash| Returns the neighbors of a node associated
+with a graph topology.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_graph_neighbors.rst
+
+INPUT PARAMETERS
+----------------
+* ``comm``: Communicator with graph topology (handle).
+* ``rank``: Rank of process in group of comm (integer).
+* ``maxneighbors``: Size of array neighbors (integer).
+
+OUTPUT PARAMETERS
+-----------------
+* ``neighbors``: Ranks of processes that are neighbors to specified process (array of integers).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+**Example:** Suppose that comm is a communicator with a shuffle-exchange
+topology. The group has 2n members. Each process is labeled by a(1),
+..., a(n) with a(i) E{0,1}, and has three neighbors: exchange (a(1),
+..., a(n) = a(1), ..., a(n-1), a(n) (a = 1 - a), shuffle (a(1), ...,
+a(n)) = a(2), ..., a(n), a(1), and unshuffle (a(1), ..., a(n)) = a(n),
+a(1), ..., a(n-1). The graph adjacency list is illustrated below for
+n=3.
+
+::
+
+           		exchange		shuffle		unshuffle
+       node		neighbors(1)	neighbors(2)	neighbors(3)
+       0(000)	    1		    0		    0
+       1(001)	    0		    2		    4
+       2(010)	    3		    4		    1
+       3(011)	    2		    6		    5
+       4(100)	    5		    1		    2
+       5(101)	    4		    3		    6
+       6(110)	    7		    5		    3
+       7(111)	    6		    7		    7
+
+Suppose that the communicator comm has this topology associated with it.
+The following code fragment cycles through the three types of neighbors
+and performs an appropriate permutation for each.
+
+.. code-block:: fortran
+
+   !  assume: each process has stored a real number A.
+   !  extract neighborhood information
+   CALL MPI_COMM_RANK(comm, myrank, ierr)
+   CALL MPI_GRAPH_NEIGHBORS(comm, myrank, 3, neighbors, ierr)
+   !  perform exchange permutation
+   CALL MPI_SENDRECV_REPLACE(A, 1, MPI_REAL, neighbors(1), 0, &
+                             neighbors(1), 0, comm, status, ierr)
+   !  perform shuffle permutation
+   CALL MPI_SENDRECV_REPLACE(A, 1, MPI_REAL, neighbors(2), 0, &
+                             neighbors(3), 0, comm, status, ierr)
+   !  perform unshuffle permutation
+   CALL MPI_SENDRECV_REPLACE(A, 1, MPI_REAL, neighbors(3), 0, &
+                             neighbors(2), 0, comm, status, ierr)
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Graph_neighbors_count`
diff --git a/docs/man-openmpi/man3/MPI_Graph_neighbors_count.3.rst b/docs/man-openmpi/man3/MPI_Graph_neighbors_count.3.rst
new file mode 100644
index 00000000000..d15756e6660
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Graph_neighbors_count.3.rst
@@ -0,0 +1,38 @@
+.. _mpi_graph_neighbors_count:
+
+MPI_Graph_neighbors_count
+=========================
+
+.. include_body
+
+:ref:`MPI_Graph_neighbors_count` |mdash| Returns the number of neighbors of a node
+associated with a graph topology.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_graph_neighbors_count.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``comm`` : Communicator with graph topology (handle).
+* ``rank`` : Rank of process in group of comm (integer).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``nneighbors`` : Number of neighbors of specified process (integer).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Graph_neighbors_count` and :ref:`MPI_Graph_neighbors` provide adjacency
+information for a general, graph topology. :ref:`MPI_Graph_neighbors_count`
+returns the number of neighbors for the process signified by rank.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso:: :ref:`MPI_Graph_neighbors`
diff --git a/docs/man-openmpi/man3/MPI_Graphdims_get.3.rst b/docs/man-openmpi/man3/MPI_Graphdims_get.3.rst
new file mode 100644
index 00000000000..33d12df37d5
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Graphdims_get.3.rst
@@ -0,0 +1,41 @@
+.. _mpi_graphdims_get:
+
+MPI_Graphdims_get
+=================
+
+.. include_body
+
+:ref:`MPI_Graphdims_get` |mdash| Retrieves graph topology information associated with
+a communicator.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_graphdims_get.rst
+
+INPUT PARAMETER
+---------------
+
+* ``comm`` : Communicator for group with graph structure (handle).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``nnodes`` : Number of nodes in graph (integer).
+* ``nedges`` : Number of edges in graph (integer).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Functions :ref:`MPI_Graphdims_get` and :ref:`MPI_Graph_get` retrieve the
+graph-topology information that was associated with a communicator by
+:ref:`MPI_Graph_create`.
+
+The information provided by :ref:`MPI_Graphdims_get` can be used to dimension
+the vectors index and edges correctly for a call to :ref:`MPI_Graph_get`.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso:: :ref:`MPI_Graph_create`
diff --git a/docs/man-openmpi/man3/MPI_Grequest_complete.3.rst b/docs/man-openmpi/man3/MPI_Grequest_complete.3.rst
new file mode 100644
index 00000000000..507922e133d
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Grequest_complete.3.rst
@@ -0,0 +1,51 @@
+.. _mpi_grequest_complete:
+
+MPI_Grequest_complete
+=====================
+
+.. include_body
+
+:ref:`MPI_Grequest_complete` |mdash| Reports that a generalized request is
+complete.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_grequest_complete.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+
+* ``request`` : Generalized request (handle).
+
+OUTPUT PARAMETER
+----------------
+
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Grequest_complete` informs MPI that the operations represented by
+the generalized request ``request`` are complete. A call to
+MPI_Wait(request, status)`` will return, and a call to
+MPI_Test(request, flag, status)`` will return flag=true only after a
+call to :ref:`MPI_Grequest_complete` has declared that these operations are
+complete.
+
+MPI imposes no restrictions on the code executed by the callback
+functions. However, new nonblocking operations should be defined so that
+the general semantic rules about MPI calls such as :ref:`MPI_Test`,
+:ref:`MPI_Request_free`, or :ref:`MPI_Cancel` still hold. For example, all
+these calls are supposed to be local and nonblocking. Therefore, the
+callback functions ``query_fn``, ``free_fn``, or ``cancel_fn`` should
+invoke blocking MPI communication calls only if the context is such that
+these calls are guaranteed to return in finite time. Once :ref:`MPI_Cancel`
+has been invoked, the canceled operation should complete in finite time,
+regardless of the state of other processes (the operation has acquired
+"local" semantics). It should either succeed or fail without
+side-effects. The user should guarantee these same properties for newly
+defined operations.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Grequest_start.3.rst b/docs/man-openmpi/man3/MPI_Grequest_start.3.rst
new file mode 100644
index 00000000000..a5fa78536ae
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Grequest_start.3.rst
@@ -0,0 +1,157 @@
+.. _mpi_grequest_start:
+
+MPI_Grequest_start
+==================
+
+.. include_body
+
+:ref:`MPI_Grequest_start` |mdash| Starts a generalized request and returns a
+handle to it in ``request``.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_grequest_start.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``query_fn`` : Callback function invoked when request status is
+   queried (function).
+* ``free_fn`` : Callback function invoked when request is freed
+   (function).
+* ``cancel_fn`` : Callback function invoked when request is canceled
+   (function).
+* ``extra_state`` : Extra state.
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``request`` : Generalized request (handle).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Grequest_start` starts a generalized ``request`` and returns a
+handle to it in ``request``.
+
+The syntax and meaning of the callback functions are listed below. All
+callback functions are passed the ``extra_state`` argument that was
+associated with the ``request`` by the starting call
+:ref:`MPI_Grequest_start`. This can be used to maintain user-defined state
+for the ``request``. In C, the query function is
+
+.. code-block:: c
+
+   typedef int MPI_Grequest_query_function(void *extra_state,
+       MPI_Status *status);
+
+In Fortran, it is
+
+.. code-block:: fortran
+
+   SUBROUTINE GREQUEST_QUERY_FUNCTION(EXTRA_STATE, STATUS, IERROR)
+       INTEGER STATUS(MPI_STATUS_SIZE), IERROR
+       INTEGER(KIND=MPI_ADDRESS_KIND) EXTRA_STATE
+
+The ``query_fn`` function computes the status that should be returned
+for the generalized request. The status also includes information about
+successful/unsuccessful cancellation of the request (result to be
+returned by ``MPI_Test_cancelled``).
+
+The ``query_fn`` function is invoked by the
+``MPI_{Wait|Test}{any|some|all}`` call that completed the generalized
+request associated with this callback. The callback function is also
+invoked by calls to ``MPI_Request_get_status`` if the request is
+complete when the call occurs. In both cases, the callback is passed a
+reference to the corresponding status variable passed by the user to the
+MPI call. If the user provided ``MPI_STATUS_IGNORE`` or
+``MPI_STATUSES_IGNORE`` to the MPI function that causes ``query_fn`` to
+be called, then MPI will pass a valid status object to ``query_fn``, and
+this status will be ignored upon return of the callback function. Note
+that ``query_fn`` is invoked only after ``MPI_Grequest_complete`` is
+called on the request; it may be invoked several times for the same
+generalized request. Note also that a call to
+``MPI_{Wait|Test}{some|all}`` may cause multiple invocations of
+``query_fn`` callback functions, one for each generalized request that
+is completed by the MPI call. The order of these invocations is not
+specified by MPI.
+
+In C, the free function is
+
+.. code-block:: c
+
+   typedef int MPI_Grequest_free_function(void *extra_state);
+
+And in Fortran, it is
+
+.. code-block:: fortran
+
+   SUBROUTINE GREQUEST_FREE_FUNCTION(EXTRA_STATE, IERROR)
+       INTEGER IERROR
+       INTEGER(KIND=MPI_ADDRESS_KIND) EXTRA_STATE
+
+The ``free_fn`` callback function is invoked to clean up user-allocated
+resources when the generalized request is freed.
+
+The ``free_fn`` function is invoked by the
+``MPI_{Wait|Test}{any|some|all}`` call that completed the generalized
+request associated with this callback. ``free_fn`` is invoked after the
+call to ``query_fn`` for the same request. However, if the MPI call
+completed multiple generalized requests, the order in which ``free_fn``
+callback functions are invoked is not specified by MPI.
+
+The ``free_fn`` callback is also invoked for generalized requests that
+are freed by a call to ``MPI_Request_free`` (no call to
+``MPI_{Wait|Test}{any|some|all}`` will occur for such a request). In
+this case, the callback function will be called either in the MPI call
+``MPI_Request_free(request)`` or in the MPI call
+``MPI_Grequest_complete(request)``, whichever happens last. In other
+words, in this case the actual freeing code is executed as soon as both
+calls (``MPI_Request_free`` and ``MPI_Grequest_complete``) have
+occurred. The ``request`` is not deallocated until after ``free_fn``
+completes. Note that ``free_fn`` will be invoked only once per request
+by a correct program.
+
+In C, the cancel function is
+
+.. code-block:: c
+
+   typedef int MPI_Grequest_cancel_function(void *extra_state, int complete);
+
+In Fortran, the cancel function is
+
+.. code-block:: Fortran
+
+   SUBROUTINE GREQUEST_CANCEL_FUNCTION(EXTRA_STATE, COMPLETE, IERROR)
+        INTEGER IERROR
+        INTEGER(KIND=MPI_ADDRESS_KIND) EXTRA_STATE
+        LOGICAL COMPLETE
+
+The ``cancel_fn`` function is invoked to start the cancellation of a
+generalized request. It is called by ``MPI_Request_cancel(request)``.
+MPI passes to the callback function complete=true if
+``MPI_Grequest_complete`` has already been called on the request, and
+complete=false otherwise.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+All callback functions return an error code. The code is passed back and
+dealt with as appropriate for the error code by the MPI function that
+invoked the callback function. For example, if error codes are returned,
+then the error code returned by the callback function will be returned
+by the MPI function that invoked the callback function. In the case of a
+``MPI_{Wait|Test}any`` call that invokes both ``query_fn`` and
+``free_fn``, the MPI call will return the error code returned by the
+last callback, namely ``free_fn``. If one or more of the ``request``s
+in a call to ``MPI_{Wait|Test}{some|all``} has failed, then the MPI call
+will return ``MPI_ERR_IN_STATUS``. In such a case, if the MPI call was
+passed an array of statuses, then MPI will return in each of the
+statuses that correspond to a completed generalized ``request`` the
+error code returned by the corresponding invocation of its ``free_fn``
+callback function. However, if the MPI function was passed
+``MPI_STATUSES_IGNORE``, then the individual error codes returned by
+each callback function will be lost.
diff --git a/docs/man-openmpi/man3/MPI_Group_c2f.3.rst b/docs/man-openmpi/man3/MPI_Group_c2f.3.rst
new file mode 100644
index 00000000000..c7b56a82553
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Group_c2f.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_group_c2f:
+
+MPI_Group_c2f
+=============
+
+.. include_body
+
+.. include:: ../man3/MPI_Comm_f2c.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_Group_compare.3.rst b/docs/man-openmpi/man3/MPI_Group_compare.3.rst
new file mode 100644
index 00000000000..38055b2c585
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Group_compare.3.rst
@@ -0,0 +1,39 @@
+.. _mpi_group_compare:
+
+MPI_Group_compare
+=================
+
+.. include_body
+
+:ref:`MPI_Group_compare` |mdash| Compares two groups.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_group_compare.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``group1`` : First group (handle).
+* ``group2`` : Second group (handle).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``result`` : Integer which is MPI_IDENT if the order and members of
+   the two groups are the same, MPI_SIMILAR if only the members are the
+   same, and MPI_UNEQUAL otherwise.
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+MPI_IDENT results if the group members and group order is exactly
+the same in both groups. This happens for instance if ``group1`` and
+``group2`` are the same handle. MPI_SIMILAR results if the group
+members are the same but the order is different. MPI_UNEQUAL results
+otherwise.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Group_difference.3.rst b/docs/man-openmpi/man3/MPI_Group_difference.3.rst
new file mode 100644
index 00000000000..5fb363badc5
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Group_difference.3.rst
@@ -0,0 +1,49 @@
+.. _mpi_group_difference:
+
+MPI_Group_difference
+====================
+
+.. include_body
+
+:ref:`MPI_Group_difference` |mdash| Makes a group from the difference of two groups.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_group_difference.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``group1`` : First group (handle).
+* ``group2`` : Second group (handle).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``newgroup`` : Difference group (handle).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The set-like operations are defined as follows:
+
+* union |mdash| All elements of the first group (group1), followed by all
+   elements of second group (group2) that are not in the first group
+* intersect |mdash| all elements of the first group that are also in the
+   second group, ordered as in first group
+* difference |mdash| all elements of the first group that are not in the
+   second group, ordered as in the first group
+
+Note that for these operations the order of processes in the output
+group is determined primarily by order in the first group (if possible)
+and then, if necessary, by order in the second group. Neither union nor
+intersection are commutative, but both are associative.
+
+The new group can be empty, that is, equal to MPI_GROUP_EMPTY.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso:: :ref:`MPI_Group_free`
diff --git a/docs/man-openmpi/man3/MPI_Group_excl.3.rst b/docs/man-openmpi/man3/MPI_Group_excl.3.rst
new file mode 100644
index 00000000000..d1afdc17004
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Group_excl.3.rst
@@ -0,0 +1,50 @@
+.. _mpi_group_excl:
+
+MPI_Group_excl
+==============
+
+.. include_body
+
+:ref:`MPI_Group_excl` |mdash| Produces a group by reordering an existing group and
+taking only unlisted members.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_group_excl.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``group`` : Group (handle).
+* ``n`` : Number of elements in array ranks (integer).
+* ``ranks`` : Array of integer ranks in group not to appear in newgroup.
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``newgroup`` : New group derived from above, preserving the order defined
+   by group (handle).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The function :ref:`MPI_Group_excl` creates a group of processes newgroup that
+is obtained by deleting from group those processes with ranks ranks[0],
+... ranks[n-1]. The ordering of processes in newgroup is identical to
+the ordering in group. Each of the n elements of ranks must be a valid
+rank in group and all elements must be distinct; otherwise, the call is
+erroneous. If n = 0, then newgroup is identical to group.
+
+NOTE
+----
+
+Currently, each of the ranks to exclude must be a valid rank in the
+group and all elements must be distinct or the function is erroneous.
+This restriction is per the draft.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso:: :ref:`MPI_Group_range_excl`
diff --git a/docs/man-openmpi/man3/MPI_Group_f2c.3.rst b/docs/man-openmpi/man3/MPI_Group_f2c.3.rst
new file mode 100644
index 00000000000..31ed36fa760
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Group_f2c.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_group_f2c:
+
+MPI_Group_f2c
+=============
+
+.. include_body
+
+.. include:: ../man3/MPI_Comm_f2c.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_Group_free.3.rst b/docs/man-openmpi/man3/MPI_Group_free.3.rst
new file mode 100644
index 00000000000..18f38c46c0b
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Group_free.3.rst
@@ -0,0 +1,38 @@
+.. _mpi_group_free:
+
+MPI_Group_free
+==============
+
+.. include_body
+
+:ref:`MPI_Group_free` |mdash| Frees a group.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_group_free.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+
+* ``group`` : Group (handle).
+
+OUTPUT PARAMETER
+----------------
+
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This operation marks a ``group`` object for deallocation. The handle
+``group`` is set to MPI_GROUP_NULL by the call. Any ongoing
+operation using this ``group`` will complete normally.
+
+NOTE
+----
+
+On return, ``group`` is set to MPI_GROUP_NULL.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Group_from_session_pset.3.rst b/docs/man-openmpi/man3/MPI_Group_from_session_pset.3.rst
new file mode 100644
index 00000000000..a4efb5045d4
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Group_from_session_pset.3.rst
@@ -0,0 +1,48 @@
+.. _mpi_group_from_session_pset:
+
+MPI_Group_from_session_pset
+===========================
+
+.. include_body
+
+:ref:`MPI_Group_from_session_pset` |mdash| Creates a group using a provided session
+handle and process set.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_group_from_session_pset.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``session`` : Session (handle).
+* ``pset_name`` : name of process set to use to create the new group
+   (string)
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``newgroup`` : New group derived from supplied session and process set
+   (handle).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The function :ref:`MPI_Group_from_session_pset` creates a group newgroup using
+the provided session handle and process set. The process set name must
+be one returned from an invocation of :ref:`MPI_Session_get_nth_pset` using the
+supplied session handle. If the pset_name does not exist, MPI_GROUP_NULL
+will be returned in the newgroup argument.
+
+NOTE
+----
+
+As with other group constructors, :ref:`MPI_Group_from_session_pset` is a local
+function.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso:: :ref:`MPI_Session_init`
diff --git a/docs/man-openmpi/man3/MPI_Group_incl.3.rst b/docs/man-openmpi/man3/MPI_Group_incl.3.rst
new file mode 100644
index 00000000000..8f584b5c528
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Group_incl.3.rst
@@ -0,0 +1,52 @@
+.. _mpi_group_incl:
+
+MPI_Group_incl
+==============
+
+.. include_body
+
+:ref:`MPI_Group_incl` |mdash| Produces a group by reordering an existing group and
+taking only listed members.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_group_incl.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``group`` : Group (handle).
+* ``n`` : Number of elements in array ranks (and size of
+   newgroup)(integer).
+* ``ranks`` : Ranks of processes in group to appear in newgroup (array of
+   integers).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``newgroup`` : New group derived from above, in the order defined by
+   ranks (handle).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The function :ref:`MPI_Group_incl` creates a group group_out that consists of
+the n processes in group with ranks rank[0], ..., rank[n-1]; the process
+with rank i in group_out is the process with rank ranks[i] in group.
+Each of the n elements of ranks must be a valid rank in group and all
+elements must be distinct, or else the program is erroneous. If n = 0,
+then group_out is MPI_GROUP_EMPTY. This function can, for instance, be
+used to reorder the elements of a group.
+
+NOTE
+----
+
+This implementation does not currently check to ensure that there are no
+duplicates in the list of ranks.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso:: :ref:`MPI_Group_compare`
diff --git a/docs/man-openmpi/man3/MPI_Group_intersection.3.rst b/docs/man-openmpi/man3/MPI_Group_intersection.3.rst
new file mode 100644
index 00000000000..7ab53870eb0
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Group_intersection.3.rst
@@ -0,0 +1,50 @@
+.. _mpi_group_intersection:
+
+MPI_Group_intersection
+======================
+
+.. include_body
+
+:ref:`MPI_Group_intersection` |mdash| Produces a group at the intersection of two
+existing groups.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_group_intersection.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``group1`` : First group (handle).
+* ``group2`` : Second group (handle).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``newgroup`` : Intersection group (handle).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The set-like operations are defined as follows:
+
+* union |mdash| All elements of the first group (group1), followed by all
+   elements of second group (group2) not in first.
+* intersect |mdash| all elements of the first group that are also in the
+   second group, ordered as in first group.
+* difference |mdash| all elements of the first group that are not in the
+   second group, ordered as in the first group.
+
+Note that for these operations the order of processes in the output
+group is determined primarily by order in the first group (if possible)
+and then, if necessary, by order in the second group. Neither union nor
+intersection are commutative, but both are associative.
+
+The new group can be empty, that is, equal to MPI_GROUP_EMPTY.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso:: :ref:`MPI_Group_free`
diff --git a/docs/man-openmpi/man3/MPI_Group_range_excl.3.rst b/docs/man-openmpi/man3/MPI_Group_range_excl.3.rst
new file mode 100644
index 00000000000..b628921b305
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Group_range_excl.3.rst
@@ -0,0 +1,47 @@
+.. _mpi_group_range_excl:
+
+
+MPI_Group_range_excl
+====================
+
+.. include_body
+
+:ref:`MPI_Group_range_excl` |mdash| Produces a group by excluding ranges of
+processes from an existing group.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_group_range_excl.rst
+
+INPUT PARAMETERS
+----------------
+* ``group``: Group (handle).
+* ``n``: Number of triplets in array ranges (integer).
+* ``ranges``: A one-dimensional array of integer triplets of the form (first rank, last rank, stride), indicating the ranks in group of processes to be excluded from the output group newgroup.
+
+OUTPUT PARAMETERS
+-----------------
+* ``newgroup``: New group derived from above, preserving the order in group (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Each computed rank must be a valid rank in group and all computed ranks
+must be distinct, or else the program is erroneous.
+
+The functionality of this routine is specified to be equivalent to
+expanding the array of ranges to an array of the excluded ranks and
+passing the resulting array of ranks and other arguments to
+:ref:`MPI_Group_excl`. A call to :ref:`MPI_Group_excl` is equivalent to a call to
+:ref:`MPI_Group_range_excl` with each rank i in ranks replaced by the triplet
+(i,i,1) in the argument ranges.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Group_excl`
+   * :ref:`MPI_Group_free`
diff --git a/docs/man-openmpi/man3/MPI_Group_range_incl.3.rst b/docs/man-openmpi/man3/MPI_Group_range_incl.3.rst
new file mode 100644
index 00000000000..0d927c674cc
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Group_range_incl.3.rst
@@ -0,0 +1,74 @@
+.. _mpi_group_range_incl:
+
+
+MPI_Group_range_incl
+====================
+
+.. include_body
+
+:ref:`MPI_Group_range_incl` |mdash| Creates a new group from ranges of ranks in
+an existing group.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_group_range_incl.rst
+
+INPUT PARAMETERS
+----------------
+* ``group``: Group (handle).
+* ``n``: Number of triplets in array ranges (integer).
+* ``ranges``: A one-dimensional array of integer triplets, of the form (first rank, last rank, stride) indicating ranks in group or processes to be included in newgroup.
+
+OUTPUT PARAMETERS
+-----------------
+* ``newgroup``: New group derived from above, in the order defined by ranges (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+If ranges consist of the triplets
+
+::
+
+       (first1, last1, stride1), ..., (firstn, lastn, striden)
+
+then newgroup consists of the sequence of processes in group with ranks
+
+::
+
+                                                    last(1)-first(1)
+     first(1), first(1) + stride(1),..., first(1) + ---------------- stride(1),...
+                                                        stride(1)
+
+                                                    last(n)-first(n)
+     first(n), first(n) + stride(n),..., first(n) + ---------------- stride(n).
+                                                        stride(n)
+
+Each computed rank must be a valid rank in group and all computed ranks
+must be distinct, or else the program is erroneous. Note that we may
+have first(i) > last(i), and stride(i) may be negative, but cannot be
+zero.
+
+The functionality of this routine is specified to be equivalent to
+expanding the array of ranges to an array of the included ranks and
+passing the resulting array of ranks and other arguments to
+:ref:`MPI_Group_incl`. A call to :ref:`MPI_Group_incl` is equivalent to a call to
+:ref:`MPI_Group_range_incl` with each rank i in ranks replaced by the triplet
+(i,i,1) in the argument ranges.
+
+
+NOTE
+----
+
+This implementation does not currently check to see that the list of
+ranges to include are valid ranks in the group.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Group_incl`
+   * :ref:`MPI_Group_free`
diff --git a/docs/man-openmpi/man3/MPI_Group_rank.3.rst b/docs/man-openmpi/man3/MPI_Group_rank.3.rst
new file mode 100644
index 00000000000..298e8e11f89
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Group_rank.3.rst
@@ -0,0 +1,35 @@
+.. _mpi_group_rank:
+
+
+MPI_Group_rank
+==============
+
+.. include_body
+
+:ref:`MPI_Group_rank` |mdash| Returns the rank of the calling process in the
+given group.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_group_rank.rst
+
+INPUT PARAMETERS
+----------------
+* ``group``: Group (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``rank``: Rank of the calling process in group, or MPI_UNDEFINED if the process is not a member (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Group_rank` returns as the output parameter *rank* the rank of the
+calling process in group. If the process is not a member of group then
+MPI_UNDEFINED is returned.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Group_size.3.rst b/docs/man-openmpi/man3/MPI_Group_size.3.rst
new file mode 100644
index 00000000000..5877af90429
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Group_size.3.rst
@@ -0,0 +1,34 @@
+.. _mpi_group_size:
+
+
+MPI_Group_size
+==============
+
+.. include_body
+
+:ref:`MPI_Group_size` |mdash| Returns the size of a group.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_group_size.rst
+
+INPUT PARAMETERS
+----------------
+* ``group``: Group (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``size``: Number of processes in the group (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Group_size` returns in *size* the number of processes in the group.
+Thus, if group = MPI_GROUP_EMPTY, then the call will return size = 0. On
+the other hand, a call with group = MPI_GROUP_NULL is erroneous.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Group_translate_ranks.3.rst b/docs/man-openmpi/man3/MPI_Group_translate_ranks.3.rst
new file mode 100644
index 00000000000..5c7e06132f4
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Group_translate_ranks.3.rst
@@ -0,0 +1,39 @@
+.. _mpi_group_translate_ranks:
+
+
+MPI_Group_translate_ranks
+=========================
+
+.. include_body
+
+:ref:`MPI_Group_translate_ranks` |mdash| Translates the ranks of processes in one
+group to those in another group.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_group_translate_ranks.rst
+
+INPUT PARAMETERS
+----------------
+* ``group1``: First group (handle).
+* ``n``: Number of ranks in ranks1 and ranks2 arrays (integer).
+* ``ranks1``: Array of zero or more valid ranks in group1.
+* ``group2``: Second group (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``ranks2``: Array of corresponding ranks in group2, MPI_UNDEFINED when no correspondence exists.
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This function is important for determining the relative numbering of the
+same processes in two different groups. For instance, if one knows the
+ranks of certain processes in the group of MPI_COMM_WORLD, one might
+want to know their ranks in a subset of that group.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Group_union.3.rst b/docs/man-openmpi/man3/MPI_Group_union.3.rst
new file mode 100644
index 00000000000..2a76e3b5e0f
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Group_union.3.rst
@@ -0,0 +1,49 @@
+.. _mpi_group_union:
+
+
+MPI_Group_union
+===============
+
+.. include_body
+
+:ref:`MPI_Group_union` |mdash| Produces a group by combining two groups.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_group_union.rst
+
+INPUT PARAMETERS
+----------------
+* ``group1``: First group (handle).
+* ``group2``: Second group (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``newgroup``: Union group (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The set-like operations are defined as follows:
+
+ o    union |mdash| All elements of the first group (group1), followed by all elements of second group (group2) not in first.
+
+ o    intersect |mdash| all elements of the first group that are also in the second group, ordered as in first group.
+
+ o    difference |mdash| all elements of the first group that are not in the second group, ordered as in the first group.
+
+Note that for these operations the order of processes in the output
+group is determined primarily by order in the first group (if possible)
+and then, if necessary, by order in the second group. Neither union nor
+intersection are commutative, but both are associative.
+
+The new group can be empty, that is, equal to MPI_GROUP_EMPTY.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Group_free`
diff --git a/docs/man-openmpi/man3/MPI_Iallgather.3.rst b/docs/man-openmpi/man3/MPI_Iallgather.3.rst
new file mode 100644
index 00000000000..4da5583800e
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Iallgather.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_iallgather:
+
+MPI_Iallgather
+==============
+    .. include_body
+
+.. include:: ../man3/MPI_Allgather.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Iallgatherv.3.rst b/docs/man-openmpi/man3/MPI_Iallgatherv.3.rst
new file mode 100644
index 00000000000..5ac1f364873
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Iallgatherv.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_iallgatherv:
+
+MPI_Iallgatherv
+===============
+    .. include_body
+
+.. include:: ../man3/MPI_Allgatherv.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Iallreduce.3.rst b/docs/man-openmpi/man3/MPI_Iallreduce.3.rst
new file mode 100644
index 00000000000..9a9823a313a
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Iallreduce.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_iallreduce:
+
+MPI_Iallreduce
+==============
+    .. include_body
+
+.. include:: ../man3/MPI_Allreduce.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Ialltoall.3.rst b/docs/man-openmpi/man3/MPI_Ialltoall.3.rst
new file mode 100644
index 00000000000..17fdb92803b
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Ialltoall.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_ialltoall:
+
+MPI_Ialltoall
+=============
+    .. include_body
+
+.. include:: ../man3/MPI_Alltoall.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Ialltoallv.3.rst b/docs/man-openmpi/man3/MPI_Ialltoallv.3.rst
new file mode 100644
index 00000000000..fca249e87db
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Ialltoallv.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_ialltoallv:
+
+MPI_Ialltoallv
+==============
+    .. include_body
+
+.. include:: ../man3/MPI_Alltoallv.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Ialltoallw.3.rst b/docs/man-openmpi/man3/MPI_Ialltoallw.3.rst
new file mode 100644
index 00000000000..43c55403ec1
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Ialltoallw.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_ialltoallw:
+
+MPI_Ialltoallw
+==============
+    .. include_body
+
+.. include:: ../man3/MPI_Alltoallw.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Ibarrier.3.rst b/docs/man-openmpi/man3/MPI_Ibarrier.3.rst
new file mode 100644
index 00000000000..bba95a77166
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Ibarrier.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_ibarrier:
+
+MPI_Ibarrier
+============
+    .. include_body
+
+.. include:: ../man3/MPI_Barrier.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Ibcast.3.rst b/docs/man-openmpi/man3/MPI_Ibcast.3.rst
new file mode 100644
index 00000000000..976689514d4
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Ibcast.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_ibcast:
+
+MPI_Ibcast
+==========
+    .. include_body
+
+.. include:: ../man3/MPI_Bcast.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Ibsend.3.rst b/docs/man-openmpi/man3/MPI_Ibsend.3.rst
new file mode 100644
index 00000000000..e9e4d3809a9
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Ibsend.3.rst
@@ -0,0 +1,49 @@
+.. _mpi_ibsend:
+
+
+MPI_Ibsend
+==========
+
+.. include_body
+
+:ref:`MPI_Ibsend` |mdash| Starts a nonblocking buffered send.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_ibsend.rst
+
+INPUT PARAMETERS
+----------------
+* ``buf``: Initial address of send buffer (choice).
+* ``count``: Number of elements in send buffer (integer).
+* ``datatype``: Data type of each send buffer element (handle).
+* ``dest``: Rank of destination (integer).
+* ``tag``: Message tag (integer).
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``request``: Communication request (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Ibsend` posts a buffered-mode, nonblocking send. Nonblocking calls
+allocate a communication request object and associate it with the
+request handle (the argument request). The request can be used later to
+query the status of the communication or wait for its completion.
+
+A nonblocking send call indicates that the system may start copying data
+out of the send buffer. The sender should not modify any part of the
+send buffer after a nonblocking send operation is called, until the send
+completes.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Test`
+   * :ref:`MPI_Wait`
diff --git a/docs/man-openmpi/man3/MPI_Iexscan.3.rst b/docs/man-openmpi/man3/MPI_Iexscan.3.rst
new file mode 100644
index 00000000000..446256be556
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Iexscan.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_iexscan:
+
+MPI_Iexscan
+===========
+    .. include_body
+
+.. include:: ../man3/MPI_Exscan.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Igather.3.rst b/docs/man-openmpi/man3/MPI_Igather.3.rst
new file mode 100644
index 00000000000..25678e2b6d5
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Igather.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_igather:
+
+MPI_Igather
+===========
+    .. include_body
+
+.. include:: ../man3/MPI_Gather.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Igatherv.3.rst b/docs/man-openmpi/man3/MPI_Igatherv.3.rst
new file mode 100644
index 00000000000..f1cc67dca27
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Igatherv.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_igatherv:
+
+MPI_Igatherv
+============
+    .. include_body
+
+.. include:: ../man3/MPI_Gatherv.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Improbe.3.rst b/docs/man-openmpi/man3/MPI_Improbe.3.rst
new file mode 100644
index 00000000000..4050d139637
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Improbe.3.rst
@@ -0,0 +1,60 @@
+.. _mpi_improbe:
+
+
+MPI_Improbe
+===========
+
+.. include_body
+
+:ref:`MPI_Improbe` |mdash| Non-blocking matched probe for a message.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_improbe.rst
+
+INPUT PARAMETERS
+----------------
+* ``source``: Source rank or ``MPI_ANY_SOURCE`` (integer).
+* ``tag``: Tag value or ``MPI_ANY_TAG`` (integer).
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``flag``: Flag (logical).
+* ``message``: Message (handle).
+* ``status``: Status object (status).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Like :ref:`MPI_Probe` and :ref:`MPI_Iprobe`, the :ref:`MPI_Mprobe` and :ref:`MPI_Improbe` operations
+allow incoming messages to be queried without actually receiving them,
+except that :ref:`MPI_Mprobe` and :ref:`MPI_Improbe` provide a mechanism to receive
+the specific message that was matched regardless of other intervening
+probe or receive operations. This gives the application an opportunity
+to decide how to receive the message, based on the information returned
+by the probe. In particular, the application may allocate memory for the
+receive buffer according to the length of the probed message.
+
+A matching probe with ``MPI_PROC_NULL`` as *source* returns *flag* = true,
+
+*message* = ``MPI_MESSAGE_NO_PROC``, and the *status* object returns source
+^ ``MPI_PROC_NULL``, tag ^ ``MPI_ANY_TAG``, and count ^ 0.
+
+:ref:`MPI_Iprobe` returns a true value in *flag* if a message has been matched
+and can be received by passing the *message* handle to the :ref:`MPI_Mrecv` or
+:ref:`MPI_Imrecv` functions, provided the *source* was not ``MPI_PROC_NULL``.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Mprobe`
+   * :ref:`MPI_Probe`
+   * :ref:`MPI_Iprobe`
+   * :ref:`MPI_Mrecv`
+   * :ref:`MPI_Imrecv`
+   * :ref:`MPI_Cancel`
diff --git a/docs/man-openmpi/man3/MPI_Imrecv.3.rst b/docs/man-openmpi/man3/MPI_Imrecv.3.rst
new file mode 100644
index 00000000000..e5ea4e9803a
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Imrecv.3.rst
@@ -0,0 +1,79 @@
+.. _mpi_imrecv:
+
+
+MPI_Imrecv
+==========
+
+.. include_body
+
+:ref:`MPI_Imrecv` |mdash| Non-blocking receive for a matched message
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_imrecv.rst
+
+INPUT PARAMETERS
+----------------
+* ``count``: Number of elements to receive (nonnegative integer).
+* ``datatype``: Datatype of each send buffer element (handle).
+* ``message``: Message (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``buf``: Initial address of receive buffer (choice).
+* ``request``: Request (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The functions :ref:`MPI_Mrecv` and :ref:`MPI_Imrecv` receive messages that have been
+previously matched by a matching probe.
+
+The *request* returned from :ref:`MPI_Imrecv` can be used with any of the
+:ref:`MPI_Test` and :ref:`MPI_Wait` variants, like any non-blocking receive request.
+
+If :ref:`MPI_Imrecv` is called with ``MPI_MESSAGE_NULL`` as the message argument, a
+call to one of the :ref:`MPI_Test` or :ref:`MPI_Wait` variants will return immediately
+with the *status* object set to *source* = ``MPI_PROC_NULL``, *tag* =
+``MPI_ANY_TAG``, and *count* = 0, as if a receive from ``MPI_PROC_NULL`` was
+issued.
+
+If reception of a matched message is started with :ref:`MPI_Imrecv`, then it is
+possible to cancel the returned request with :ref:`MPI_Cancel`. If :ref:`MPI_Cancel`
+succeeds, the matched message must be found by a subsequent message
+probe (:ref:`MPI_Probe`, :ref:`MPI_Iprobe`, :ref:`MPI_Mprobe`, or :ref:`MPI_Improbe`), received by a
+subsequent receive operation or canceled by the sender.
+
+Note, however, that is it possible for the cancellation of operations
+initiated with :ref:`MPI_Imrecv` to fail. An example of a failing case is when
+canceling the matched message receive would violate MPI message ordering
+rules (e.g., if another message matching the same message signature has
+matched |mdash| and possibly received |mdash| before this :ref:`MPI_Imrecv` is canceled).
+
+If your application does not need to examine the *status* field, you
+can save resources by using the predefined constant
+``MPI_STATUS_IGNORE`` as a special value for the *status* argument.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+Note that per the "Return Status" section in the "Point-to-Point
+Communication" chapter in the `MPI Standard
+<https://www.mpi-forum.org/docs/>`_, MPI errors on messages received
+by :ref:`MPI_Imrecv` do not set the ``status.MPI_ERROR`` field in the
+returned *status*.  The error code is always passed to the back-end
+error handler and may be passed back to the caller through the return
+value of :ref:`MPI_Imrecv` if the back-end error handler returns it.
+The pre-defined MPI error handler ``MPI_ERRORS_RETURN`` exhibits this
+behavior, for example.
+
+.. seealso::
+   * :ref:`MPI_Mprobe`
+   * :ref:`MPI_Improbe`
+   * :ref:`MPI_Probe`
+   * :ref:`MPI_Iprobe`
+   * :ref:`MPI_Imrecv`
+   * :ref:`MPI_Cancel`
diff --git a/docs/man-openmpi/man3/MPI_Ineighbor_allgather.3.rst b/docs/man-openmpi/man3/MPI_Ineighbor_allgather.3.rst
new file mode 100644
index 00000000000..b924b315c74
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Ineighbor_allgather.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_ineighbor_allgather:
+
+MPI_Ineighbor_allgather
+=======================
+    .. include_body
+
+.. include:: ../man3/MPI_Neighbor_allgather.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Ineighbor_allgatherv.3.rst b/docs/man-openmpi/man3/MPI_Ineighbor_allgatherv.3.rst
new file mode 100644
index 00000000000..81885f27692
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Ineighbor_allgatherv.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_ineighbor_allgatherv:
+
+MPI_Ineighbor_allgatherv
+========================
+    .. include_body
+
+.. include:: ../man3/MPI_Neighbor_allgatherv.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Ineighbor_alltoall.3.rst b/docs/man-openmpi/man3/MPI_Ineighbor_alltoall.3.rst
new file mode 100644
index 00000000000..586ee58d08f
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Ineighbor_alltoall.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_ineighbor_alltoall:
+
+MPI_Ineighbor_alltoall
+======================
+    .. include_body
+
+.. include:: ../man3/MPI_Neighbor_alltoall.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Ineighbor_alltoallv.3.rst b/docs/man-openmpi/man3/MPI_Ineighbor_alltoallv.3.rst
new file mode 100644
index 00000000000..cbc7031e887
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Ineighbor_alltoallv.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_ineighbor_alltoallv:
+
+MPI_Ineighbor_alltoallv
+=======================
+    .. include_body
+
+.. include:: ../man3/MPI_Neighbor_alltoallv.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Ineighbor_alltoallw.3.rst b/docs/man-openmpi/man3/MPI_Ineighbor_alltoallw.3.rst
new file mode 100644
index 00000000000..7738bba3e19
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Ineighbor_alltoallw.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_ineighbor_alltoallw:
+
+MPI_Ineighbor_alltoallw
+=======================
+    .. include_body
+
+.. include:: ../man3/MPI_Neighbor_alltoallw.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Info_c2f.3.rst b/docs/man-openmpi/man3/MPI_Info_c2f.3.rst
new file mode 100644
index 00000000000..cf10d0117b3
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Info_c2f.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_info_c2f:
+
+MPI_Info_c2f
+============
+
+.. include_body
+
+.. include:: ../man3/MPI_Comm_f2c.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_Info_create.3.rst b/docs/man-openmpi/man3/MPI_Info_create.3.rst
new file mode 100644
index 00000000000..edbf2326587
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Info_create.3.rst
@@ -0,0 +1,36 @@
+.. _mpi_info_create:
+
+
+MPI_Info_create
+===============
+
+.. include_body
+
+:ref:`MPI_Info_create` |mdash| Creates a new info object.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_info_create.rst
+
+OUTPUT PARAMETERS
+-----------------
+* ``info``: Info object created (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Info_create` creates a new info object. The newly created object
+contains no key/value pairs.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Info_delete`
+   * :ref:`MPI_Info_dup`
+   * :ref:`MPI_Info_free`
+   * :ref:`MPI_Info_get`
+   * :ref:`MPI_Info_set`
diff --git a/docs/man-openmpi/man3/MPI_Info_create_env.3.rst b/docs/man-openmpi/man3/MPI_Info_create_env.3.rst
new file mode 100644
index 00000000000..cb142f13e34
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Info_create_env.3.rst
@@ -0,0 +1,41 @@
+.. _mpi_info_create_env:
+
+
+MPI_Info_create_env
+===================
+
+.. include_body
+
+:ref:`MPI_Info_create_env` |mdash| Creates a new info object with the same construction as :ref:`MPI_INFO_ENV` as created during :ref:`MPI_Init`  or :ref:`MPI_Init_thread` when the same arguments
+are used.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_info_create_env.rst
+
+OUTPUT PARAMETERS
+-----------------
+* ``info``: Info object created (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Info_create_env` creates a new info object with the same construction as :ref:`MPI_INFO_ENV` as created during :ref:`MPI_Init` or :ref:`MPI_Init_thread` when the same arguments are used.
+
+NOTE
+----
+
+:ref:`MPI_Info_create_env` is one of the few functions that can be called
+before :ref:`MPI_Init` and after :ref:`MPI_Finalize`.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Info_delete`
+   * :ref:`MPI_Info_dup`
+   * :ref:`MPI_Info_free`
+   * :ref:`MPI_Info_get`
+   * :ref:`MPI_Info_set`
diff --git a/docs/man-openmpi/man3/MPI_Info_delete.3.rst b/docs/man-openmpi/man3/MPI_Info_delete.3.rst
new file mode 100644
index 00000000000..248e70c3eb4
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Info_delete.3.rst
@@ -0,0 +1,43 @@
+.. _mpi_info_delete:
+
+
+MPI_Info_delete
+===============
+
+.. include_body
+
+:ref:`MPI_Info_delete` |mdash| Deletes a key/value pair from *info*.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_info_delete.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``info``: Info object (handle).
+
+INPUT PARAMETER
+---------------
+* ``key``: Key (string).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Info_delete` deletes a (key,value) pair from *info*. If *key* is not
+defined in *info*, the call raises an error of class MPI_ERR_INFO_NOKEY.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Info_create`
+   * :ref:`MPI_Info_dup`
+   * :ref:`MPI_Info_free`
+   * :ref:`MPI_Info_get`
+   * :ref:`MPI_Info_set`
diff --git a/docs/man-openmpi/man3/MPI_Info_dup.3.rst b/docs/man-openmpi/man3/MPI_Info_dup.3.rst
new file mode 100644
index 00000000000..cf01f5f5b3e
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Info_dup.3.rst
@@ -0,0 +1,40 @@
+.. _mpi_info_dup:
+
+
+MPI_Info_dup
+============
+
+.. include_body
+
+:ref:`MPI_Info_dup` |mdash| Duplicates an info object.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_info_dup.rst
+
+INPUT PARAMETER
+---------------
+* ``info``: Info object (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``newinfo``: Info object (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Info_dup` duplicates an existing info object, creating a new object,
+with the same (key,value) pairs and the same ordering of keys.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Info_create`
+   * :ref:`MPI_Info_delete`
+   * :ref:`MPI_Info_free`
+   * :ref:`MPI_Info_get`
+   * :ref:`MPI_Info_set`
diff --git a/docs/man-openmpi/man3/MPI_Info_env.3.rst b/docs/man-openmpi/man3/MPI_Info_env.3.rst
new file mode 100644
index 00000000000..650b23a610c
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Info_env.3.rst
@@ -0,0 +1,96 @@
+.. _mpi_info_env:
+
+
+MPI_Info_env
+============
+
+.. include_body
+
+:ref:`MPI_INFO_ENV` |mdash| Static MPI_Info object containing info about the
+application
+
+
+DESCRIPTION
+-----------
+
+The MPI-3 standard established a static MPI_Info object named
+:ref:`MPI_INFO_ENV` that can be used to access information about how the
+application was executed from the run-time.
+
+
+SUPPORTED FIELDS
+----------------
+
+command
+   If available, the value will be set to argv[0]. Note that the value
+   may not always be available - e.g., it is valid for a program to call
+   :ref:`MPI_Init` with NULL parameters, in which case argv[0] will not be set
+   if run as a singleton. This value will never be set in a Fortran
+   program as the argv are not available.
+
+argv
+   The argv given for the application. If no arguments are passed to the
+   application, then this value will not be set. It will also not be set
+   in the case of a singleton that calls :ref:`MPI_Init` with NULL parameters,
+   or a Fortran program.
+
+maxprocs
+   The number of processes in the job.
+
+soft
+   Open MPI does not support the *soft* option for specifying the number
+   of processes to be executed, so this value is set to the same as
+   *maxprocs*.
+
+host
+   The name of the host this process is executing upon - the value
+   returned from *gethostname()*.
+
+arch
+   The architecture of the host this process is executing upon. This
+   value indicates the underlying chip architecture (e.g., x86_64), if
+   it can be determined.
+
+wdir
+   The working directory at the time of process launch by mpiexec. Note
+   that this value will not be set for processes launched as singletons
+   as there is no reliable way for the MPI library to determine the
+   location.
+
+file
+   Although specified by the MPI-3 standard, no value is currently set
+   for this field.
+
+thread_level
+   The requested MPI thread level - note that this may differ from the
+   *actual* MPI thread level of the application.
+
+ompi_num_apps
+   The number of application contexts in an MPMD job. This is an Open
+   MPI-specific field and value.
+
+ompi_np
+   The number of processes in each application context, provided as a
+   space-delimited list of integers. This is an Open MPI-specific field
+   and value.
+
+ompi_first_rank
+   The MPI rank of the first process in each application context,
+   provided as a space-delimited list of integers This is an Open
+   MPI-specific field and value.
+
+ompi_positioned_file_dir
+   If Open MPI was asked to pre-position files, this field provides the
+   top-level directory where those files were place. This is an Open
+   MPI-specific field and value.
+
+
+ERRORS
+------
+
+| When calling :ref:`MPI_INFO_GET`, the *flag* parameter will be set to zero
+  (false) if a value for the field has not been set.
+
+
+.. seealso::
+   * :ref:`MPI_Info_get`
diff --git a/docs/man-openmpi/man3/MPI_Info_f2c.3.rst b/docs/man-openmpi/man3/MPI_Info_f2c.3.rst
new file mode 100644
index 00000000000..f947f2da988
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Info_f2c.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_info_f2c:
+
+MPI_Info_f2c
+============
+
+.. include_body
+
+.. include:: ../man3/MPI_Comm_f2c.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_Info_free.3.rst b/docs/man-openmpi/man3/MPI_Info_free.3.rst
new file mode 100644
index 00000000000..caeb475050d
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Info_free.3.rst
@@ -0,0 +1,38 @@
+.. _mpi_info_free:
+
+
+MPI_Info_free
+=============
+
+.. include_body
+
+:ref:`MPI_Info_free` |mdash| Frees an info object.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_info_free.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``info``: Info object (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Info_free` frees *info* and sets it to MPI_INFO_NULL.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Info_create`
+   * :ref:`MPI_Info_delete`
+   * :ref:`MPI_Info_dup`
+   * :ref:`MPI_Info_get`
+   * :ref:`MPI_Info_set`
diff --git a/docs/man-openmpi/man3/MPI_Info_get.3.rst b/docs/man-openmpi/man3/MPI_Info_get.3.rst
new file mode 100644
index 00000000000..288e4f34f52
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Info_get.3.rst
@@ -0,0 +1,54 @@
+.. _mpi_info_get:
+
+
+MPI_Info_get
+============
+
+.. include_body
+
+:ref:`MPI_Info_get` |mdash| Retrieves the value associated with a key in an info
+object.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_info_get.rst
+
+INPUT PARAMETERS
+----------------
+* ``info``: Info object (handle).
+* ``key``: Key (string).
+* ``valuelen``: Length of value arg (integer).
+
+OUTPUT PARAMETER
+----------------
+* ``value``: Value (string).
+* ``flag``: Returns true if key defined, false if not (boolean).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Info_get` retrieves the value associated with *key* in a previous
+call to :ref:`MPI_Info_set`. If such a key exists, it sets *flag* to true and
+returns the value in *value*; otherwise it sets *flag* to false and
+leaves *value* unchanged. *valuelen* is the number of characters
+available in value. If it is less than the actual size of the value, the
+returned value is truncated. In C, *valuelen* should be one less than
+the amount of allocated space to allow for the null terminator.
+
+If *key* is larger than MPI_MAX_INFO_KEY, the call is erroneous.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Info_create`
+   * :ref:`MPI_Info_delete`
+   * :ref:`MPI_Info_dup`
+   * :ref:`MPI_Info_free`
+   * :ref:`MPI_Info_get_valuelen`
+   * :ref:`MPI_Info_get_nkeys`
+   * :ref:`MPI_Info_get_nthkey`
+   * :ref:`MPI_Info_set`
diff --git a/docs/man-openmpi/man3/MPI_Info_get_nkeys.3.rst b/docs/man-openmpi/man3/MPI_Info_get_nkeys.3.rst
new file mode 100644
index 00000000000..9c6e55043a4
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Info_get_nkeys.3.rst
@@ -0,0 +1,39 @@
+.. _mpi_info_get_nkeys:
+
+
+MPI_Info_get_nkeys
+==================
+
+.. include_body
+
+:ref:`MPI_Info_get_nkeys` |mdash| Gets the number of keys currently defined in an
+info object.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_info_get_nkeys.rst
+
+INPUT PARAMETER
+---------------
+* ``info``: Info object (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``nkeys``: Number of defined keys (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Info_get_nkeys` returns the number of currently defined keys in
+*info*.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Info_get`
+   * :ref:`MPI_Info_get_nthkey`
+   * :ref:`MPI_Info_get_valuelen`
diff --git a/docs/man-openmpi/man3/MPI_Info_get_nthkey.3.rst b/docs/man-openmpi/man3/MPI_Info_get_nthkey.3.rst
new file mode 100644
index 00000000000..0e869cece8c
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Info_get_nthkey.3.rst
@@ -0,0 +1,42 @@
+.. _mpi_info_get_nthkey:
+
+
+MPI_Info_get_nthkey
+===================
+
+.. include_body
+
+:ref:`MPI_Info_get_nthkey` |mdash| Returns the *n*\ th defined key in *info*.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_info_get_nthkey.rst
+
+INPUT PARAMETERS
+----------------
+* ``info``: Info object (handle).
+* ``n``: Key number (integer).
+
+OUTPUT PARAMETERS
+-----------------
+* ``key``: Key (string).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Info_get_nthkey` returns the *n*\ th defined key in *info*. Keys are
+numbered 0...\ *N* - 1 where *N* is the value returned by
+:ref:`MPI_Info_get_nkeys`. All keys between 0 and *N* - 1 are guaranteed to be
+defined. The number of a given key does not change as long as *info* is
+not modified with :ref:`MPI_Info_set` or :ref:`MPI_Info_delete`.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Info_get`
+   * :ref:`MPI_Info_get_nkeys`
+   * :ref:`MPI_Info_get_valuelen`
diff --git a/docs/man-openmpi/man3/MPI_Info_get_string.3.rst b/docs/man-openmpi/man3/MPI_Info_get_string.3.rst
new file mode 100644
index 00000000000..cd6a4ac7ffb
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Info_get_string.3.rst
@@ -0,0 +1,59 @@
+.. _mpi_info_get_string:
+
+
+MPI_Info_get_string
+===================
+
+.. include_body
+
+:ref:`MPI_Info_get_string` |mdash| Retrieves the value associated with a key in
+an info object.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_info_get_string.rst
+
+INPUT PARAMETERS
+----------------
+* ``info``: Info object (handle).
+* ``key``: Key (string).
+
+OUTPUT PARAMETER
+----------------
+* ``buflen On entry, length of value arg. On return, set to required size to``: hold value string (integer).
+* ``value``: Value (string).
+* ``flag``: Returns true if key defined, false if not (boolean).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Info_get_string` retrieves the value associated with *key* from
+*info*, if any. If such a key exists in info, it sets *flag* to true and
+returns the value in *value*, otherwise it sets flag to false and leaves
+value unchanged. *buflen* on input is the size of the provided buffer,
+for the output of buflen it is the size of the buffer needed to store
+the value string. If the buflen passed into the function is less than
+the actual size needed to store the value string (including null
+terminator in C), the value is truncated. On return, the value of
+*buflen* will be set to the required buffer size to hold the value
+string. If buflen is set to 0, value is not changed. In C, *buflen*
+includes the required space for the null terminator. In C, this function
+returns a null terminated string in all cases where the *buflen* input
+value is greater than 0.
+
+If *key* is larger than MPI_MAX_INFO_KEY, the call is erroneous.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Info_create`
+   * :ref:`MPI_Info_delete`
+   * :ref:`MPI_Info_dup`
+   * :ref:`MPI_Info_free`
+   * :ref:`MPI_Info_get_nkeys`
+   * :ref:`MPI_Info_get_nthkey`
+   * :ref:`MPI_Info_set`
diff --git a/docs/man-openmpi/man3/MPI_Info_get_valuelen.3.rst b/docs/man-openmpi/man3/MPI_Info_get_valuelen.3.rst
new file mode 100644
index 00000000000..fe5bdf04971
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Info_get_valuelen.3.rst
@@ -0,0 +1,46 @@
+.. _mpi_info_get_valuelen:
+
+
+MPI_Info_get_valuelen
+=====================
+
+.. include_body
+
+:ref:`MPI_Info_get_valuelen` |mdash| Retrieves the length of the key value
+associated with an info object.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_info_get_valuelen.rst
+
+INPUT PARAMETERS
+----------------
+* ``info``: Info object (handle).
+* ``key``: Key (string).
+
+OUTPUT PARAMETERS
+-----------------
+* ``valuelen``: Length of value arg (integer).
+* ``flag``: Returns true if key defined, false if not (boolean).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Info_get_valuelen` retrieves the length of the *value* associated
+with *key*. If *key* is defined, *valuelen* is set to the length of its
+associated value and *flag* is set to true. If *key* is not defined,
+*valuelen* is not touched and *flag* is set to false. The length
+returned in C does not include the end-of-string character.
+
+If *key* is larger than MPI_MAX_INFO_KEY, the call is erroneous.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Info_get`
+   * :ref:`MPI_Info_get_nkeys`
+   * :ref:`MPI_Info_get_nthkey`
diff --git a/docs/man-openmpi/man3/MPI_Info_set.3.rst b/docs/man-openmpi/man3/MPI_Info_set.3.rst
new file mode 100644
index 00000000000..0ad3cb1e289
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Info_set.3.rst
@@ -0,0 +1,48 @@
+.. _mpi_info_set:
+
+
+MPI_Info_set
+============
+
+.. include_body
+
+:ref:`MPI_Info_set` |mdash| Adds a key/value pair to *info*.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_info_set.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``info``: Info object (handle).
+
+INPUT PARAMETERS
+----------------
+* ``key``: Key (string).
+* ``value``: Value (string).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Info_set` adds the (key,value) pair to *info* and overrides the value
+if a value for the same key was previously set. The *key* and *value*
+parameters are null-terminated strings in C. In Fortran, leading and
+trailing spaces in *key* and *value* are stripped. If either *key* or
+*value* is larger than the allowed maximums, the error MPI_ERR_INFO_KEY
+or MPI_ERR_INFO_VALUE is raised, respectively.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Info_create`
+   * :ref:`MPI_Info_delete`
+   * :ref:`MPI_Info_dup`
+   * :ref:`MPI_Info_free`
+   * :ref:`MPI_Info_set`
diff --git a/docs/man-openmpi/man3/MPI_Init.3.rst b/docs/man-openmpi/man3/MPI_Init.3.rst
new file mode 100644
index 00000000000..6938ec423c5
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Init.3.rst
@@ -0,0 +1,66 @@
+.. _mpi_init:
+
+
+MPI_Init
+========
+
+.. include_body
+
+:ref:`MPI_Init` |mdash| Initializes the MPI execution environment
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_init.rst
+
+INPUT PARAMETERS
+----------------
+* ``argc``: C only: Pointer to the number of arguments.
+* ``argv``: C only: Argument vector.
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This routine, or :ref:`MPI_Init_thread`, must be called before most other MPI
+routines are called. There are a small number of errors, such as
+:ref:`MPI_Initialized` and :ref:`MPI_Finalized`. MPI can be initialized at most once;
+subsequent calls to :ref:`MPI_Init` or :ref:`MPI_Init_thread` are erroneous.
+
+All MPI programs must contain a call to :ref:`MPI_Init` or :ref:`MPI_Init_thread`.
+Open MPI accepts the C *argc* and *argv* arguments to main, but neither
+modifies, interprets, nor distributes them:
+
+.. code-block:: c
+
+   /* declare variables */
+   MPI_Init(&argc, &argv);
+   /* parse arguments */
+   /* main program */
+   MPI_Finalize();
+
+
+NOTES
+-----
+
+The Fortran version does not have provisions for *argc* and *argv* and
+takes only IERROR.
+
+The MPI Standard does not say what a program can do before an :ref:`MPI_Init`
+or after an :ref:`MPI_Finalize`. In the Open MPI implementation, it should do
+as little as possible. In particular, avoid anything that changes the
+external state of the program, such as opening files, reading standard
+input, or writing to standard output.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Init_thread`
+   * :ref:`MPI_Initialized`
+   * :ref:`MPI_Finalize`
+   * :ref:`MPI_Finalized`
diff --git a/docs/man-openmpi/man3/MPI_Init_thread.3.rst b/docs/man-openmpi/man3/MPI_Init_thread.3.rst
new file mode 100644
index 00000000000..4bdc3621132
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Init_thread.3.rst
@@ -0,0 +1,119 @@
+.. _mpi_init_thread:
+
+
+MPI_Init_thread
+===============
+
+.. include_body
+
+:ref:`MPI_Init_thread` |mdash| Initializes the MPI execution environment
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_init_thread.rst
+
+INPUT PARAMETERS
+----------------
+* ``argc``: C only: Pointer to the number of arguments.
+* ``argv``: C only: Argument vector.
+* ``required``: Desired level of thread support (integer).
+
+OUTPUT PARAMETERS
+-----------------
+* ``provided``: Available level of thread support (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This routine, or :ref:`MPI_Init`, must be called before most other MPI routines
+are called. There are a small number of exceptions, such as
+:ref:`MPI_Initialized` and :ref:`MPI_Finalized`. MPI can be initialized at most once;
+subsequent calls to :ref:`MPI_Init` or :ref:`MPI_Init_thread` are erroneous.
+
+:ref:`MPI_Init_thread`, as compared to :ref:`MPI_Init`, has a provision to request a
+certain level of thread support in *required*:
+
+MPI_THREAD_SINGLE
+   Only one thread will execute.
+
+MPI_THREAD_FUNNELED
+   If the process is multithreaded, only the thread that called
+   :ref:`MPI_Init_thread` will make MPI calls.
+
+MPI_THREAD_SERIALIZED
+   If the process is multithreaded, only one thread will make MPI
+   library calls at one time.
+
+MPI_THREAD_MULTIPLE
+   If the process is multithreaded, multiple threads may call MPI at
+   once with no restrictions.
+
+The level of thread support available to the program is set in
+*provided*. In Open MPI, the value is dependent on how the library was
+configured and built. Note that there is no guarantee that *provided*
+will be greater than or equal to *required*.
+
+Also note that calling :ref:`MPI_Init_thread` with a *required* value of
+MPI_THREAD_SINGLE is equivalent to calling :ref:`MPI_Init`.
+
+All MPI programs must contain a call to :ref:`MPI_Init` or :ref:`MPI_Init_thread`.
+Open MPI accepts the C *argc* and *argv* arguments to main, but neither
+modifies, interprets, nor distributes them:
+
+.. code-block:: c
+
+   /* declare variables */
+   MPI_Init_thread(&argc, &argv, req, &prov);
+   /* parse arguments */
+   /* main program */
+   MPI_Finalize();
+
+
+NOTES
+-----
+
+The Fortran version does not have provisions for ``argc`` and ``argv`` and
+takes only ``IERROR``.
+
+It is the caller's responsibility to check the value of ``provided``, as
+it may be less than what was requested in ``required``.
+
+The MPI Standard does not say what a program can do before an
+:ref:`MPI_Init_thread` or after an :ref:`MPI_Finalize`. In the Open MPI
+implementation, it should do as little as possible. In particular, avoid
+anything that changes the external state of the program, such as opening
+files, reading standard input, or writing to standard output.
+
+
+MPI_THREAD_MULTIPLE Support
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+``MPI_THREAD_MULTIPLE`` support is included if the environment in which
+Open MPI was built supports threading. You can check the output of
+:ref:`ompi_info(1) <man1-ompi_info>` to see if Open MPI has
+``MPI_THREAD_MULTIPLE`` support:
+
+::
+
+   shell$ ompi_info | grep "Thread support"
+             Thread support: posix (MPI_THREAD_MULTIPLE: yes, OPAL support: yes, OMPI progress: no, Event lib: yes)
+   shell$
+
+The ``MPI_THREAD_MULTIPLE: yes`` portion of the above output indicates
+that Open MPI was compiled with ``MPI_THREAD_MULTIPLE`` support.
+
+Note that there is a small performance penalty for using
+``MPI_THREAD_MULTIPLE`` support; latencies for short messages will be higher
+as compared to when using ``MPI_THREAD_SINGLE``, for example.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Init`
+   * :ref:`MPI_Initialized`
+   * :ref:`MPI_Finalize`
+   * :ref:`MPI_Finalized`
diff --git a/docs/man-openmpi/man3/MPI_Initialized.3.rst b/docs/man-openmpi/man3/MPI_Initialized.3.rst
new file mode 100644
index 00000000000..14b87f2d4f9
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Initialized.3.rst
@@ -0,0 +1,36 @@
+.. _mpi_initialized:
+
+
+MPI_Initialized
+===============
+
+.. include_body
+
+:ref:`MPI_Initialized` |mdash| Checks whether MPI has been initialized
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_initialized.rst
+
+OUTPUT PARAMETERS
+-----------------
+* ``flag``: True if MPI has been initialized, and false otherwise (logical).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This routine may be used to determine whether MPI has been initialized.
+It is one of a small number of routines that may be called before MPI is
+initialized and after MPI has been finalized (:ref:`MPI_Finalized` is another).
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Init`
+   * :ref:`MPI_Init_thread`
+   * :ref:`MPI_Finalize`
+   * :ref:`MPI_Finalized`
diff --git a/docs/man-openmpi/man3/MPI_Intercomm_create.3.rst b/docs/man-openmpi/man3/MPI_Intercomm_create.3.rst
new file mode 100644
index 00000000000..58acd192313
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Intercomm_create.3.rst
@@ -0,0 +1,71 @@
+.. _mpi_intercomm_create:
+
+
+MPI_Intercomm_create
+====================
+
+.. include_body
+
+:ref:`MPI_Intercomm_create` |mdash| Creates an intercommunicator from two
+intracommunicators.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_intercomm_create.rst
+
+INPUT PARAMETERS
+----------------
+* ``local_comm``: The communicator containing the process that initiates the inter-communication (handle).
+* ``local_leader``: Rank of local group leader in local_comm (integer).
+* ``peer_comm``: "Peer" communicator; significant only at the local_leader (handle).
+* ``remote_leader``: Rank of remote group leader in peer_comm; significant only at the local_leader (integer).
+* ``tag``: Message tag used to identify new intercommunicator (integer).
+
+OUTPUT PARAMETERS
+-----------------
+* ``newintercomm``: Created intercommunicator (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This call creates an intercommunicator. It is collective over the union
+of the local and remote groups. Processes should provide identical
+local_comm and local_leader arguments within each group. Wildcards are
+not permitted for remote_leader, local_leader, and tag.
+
+This call uses point-to-point communication with communicator peer_comm,
+and with tag tag between the leaders. Thus, care must be taken that
+there be no pending communication on peer_comm that could interfere with
+this communication.
+
+If multiple MPI_Intercomm_creates are being made, they should use
+different tags (more precisely, they should ensure that the local and
+remote leaders are using different tags for each MPI_intercomm_create).
+
+
+NOTES
+-----
+
+We recommend using a dedicated peer communicator, such as a duplicate of
+MPI_COMM_WORLD, to avoid trouble with peer communicators.
+
+The MPI 1.1 Standard contains two mutually exclusive comments on the
+input intracommunicators. One says that their respective groups must be
+disjoint; the other that the leaders can be the same process. After some
+discussion by the MPI Forum, it has been decided that the groups must be
+disjoint. Note that the **reason** given for this in the standard is
+**not** the reason for this choice; rather, the **other** operations on
+intercommunicators (like :ref:`MPI_Intercomm_merge` ) do not make sense if
+the groups are not disjoint.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Intercomm_merge`
+   * :ref:`MPI_Comm_free`
+   * :ref:`MPI_Comm_remote_group`
+   * :ref:`MPI_Comm_remote_size`
diff --git a/docs/man-openmpi/man3/MPI_Intercomm_create_from_groups.3.rst b/docs/man-openmpi/man3/MPI_Intercomm_create_from_groups.3.rst
new file mode 100644
index 00000000000..774daa97de6
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Intercomm_create_from_groups.3.rst
@@ -0,0 +1,67 @@
+.. _mpi_intercomm_create_from_groups:
+
+MPI_Intercomm_create_from_groups
+================================
+
+.. include_body
+
+:ref:`MPI_Intercomm_create_from_groups` |mdash| Creates a new inter-communicator from
+a local and remote group and stringtag
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_intercomm_create_from_groups.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``local_group`` : Local group (handler)
+* ``local_leader`` : rank of local group leader in local_group (integer)
+* ``remote_group`` : Remote group (handler)
+* ``remote_leader`` : rank of remote leader in remote_group, significant
+   only at local_leader (integer)
+* ``stringtag`` : Unique identifier for this operation (string)
+* ``info`` : info object (handler)
+* ``errhandler`` : error handler to be attached to the new
+   inter-communicator (handle)
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``newintercomm`` : New inter-communicator (handle).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Intercomm_create_from_groups` creates an inter-communicator. Unlike
+:ref:`MPI_Intercomm_create`, this function uses as input previously defined,
+disjoint local and remote groups. The calling MPI process must be a
+member of the local group. The call is collective over the union of the
+local and remote groups. All involved MPI processes shall provide an
+identical value for the stringtag argument. Within each group, all MPI
+processes shall provide identical local_group, local_leader arguments.
+Wildcards are not permitted for the remote_leader or local_leader
+arguments. The stringtag argument serves the same purpose as the
+stringtag used in the :ref:`MPI_Comm_create_from_group` function; it
+differentiates concurrent calls in a multithreaded environment. The
+stringtag shall not exceed MPI_MAX_STRINGTAG_LEN characters in length.
+For C, this includes space for a null terminating character. In the
+event that MPI_GROUP_EMPTY is supplied as the local_group or
+remote_group1 or both, then the call is a local operation and
+MPI_COMM_NULL is returned as the newintercomm.
+
+NOTES
+-----
+
+The errhandler argument specifies an error handler to be attached to the
+new inter-communicator. The info argument provides hints and assertions,
+possibly MPI implementation dependent, which indicate desired
+characteristics and guide communicator creation. MPI_MAX_STRINGTAG_LEN
+shall have a value of at least 63.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso:: :ref:`MPI_Comm_create_from_group`
diff --git a/docs/man-openmpi/man3/MPI_Intercomm_merge.3.rst b/docs/man-openmpi/man3/MPI_Intercomm_merge.3.rst
new file mode 100644
index 00000000000..899497b8696
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Intercomm_merge.3.rst
@@ -0,0 +1,45 @@
+.. _mpi_intercomm_merge:
+
+
+MPI_Intercomm_merge
+===================
+
+.. include_body
+
+:ref:`MPI_Intercomm_merge` |mdash| Creates an intracommunicator from an
+intercommunicator.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_intercomm_merge.rst
+
+INPUT PARAMETERS
+----------------
+* ``intercomm``: Intercommunicator (type indicator).
+* ``high``: Used to order the groups of the two intracommunicators within comm when creating the new communicator (type indicator).
+
+OUTPUT PARAMETERS
+-----------------
+* ``newintracomm``: Created intracommunicator (type indicator).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This function creates an intracommunicator from the union of the two
+groups that are associated with intercomm. All processes should provide
+the same high value within each of the two groups. If processes in one
+group provide the value high = false and processes in the other group
+provide the value high = true, then the union orders the "low" group
+before the "high" group. If all processes provide the same high
+argument, then the order of the union is arbitrary. This call is
+blocking and collective within the union of the two groups.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Intercomm_create`
+   * :ref:`MPI_Comm_free`
diff --git a/docs/man-openmpi/man3/MPI_Iprobe.3.rst b/docs/man-openmpi/man3/MPI_Iprobe.3.rst
new file mode 100644
index 00000000000..6e23d794b9c
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Iprobe.3.rst
@@ -0,0 +1,96 @@
+.. _mpi_iprobe:
+
+
+MPI_Iprobe
+==========
+
+.. include_body
+
+:ref:`MPI_Iprobe` |mdash| Nonblocking test for a message.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_iprobe.rst
+
+INPUT PARAMETERS
+----------------
+* ``source``: Source rank or ``MPI_ANY_SOURCE`` (integer).
+* ``tag``: Tag value or ``MPI_ANY_TAG`` (integer).
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``flag``: Message-waiting flag (logical).
+* ``status``: Status object (status).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The :ref:`MPI_Probe` and :ref:`MPI_Iprobe` operations allow checking of incoming
+messages without actual receipt of them. The user can then decide how to
+receive them, based on the information returned by the probe (basically,
+the information returned by status). In particular, the user may
+allocate memory for the receive buffer, according to the length of the
+probed message.
+
+``MPI_Iprobe(source, tag, comm, flag, status)`` returns *flag* = true if there
+is a message that can be received and that matches the pattern specified
+by the arguments source, tag, and comm. The call matches the same
+message that would have been received by a call to ``MPI_Recv(..., source,
+tag, comm, status)`` executed at the same point in the program, and
+returns in status the same value that would have been returned by
+:ref:`MPI_Recv`. Otherwise, the call returns *flag* = false, and leaves status
+undefined.
+
+If :ref:`MPI_Iprobe` returns *flag* = true, then the content of the
+status object can be subsequently accessed as described in the "Return
+Status" subsection of the "Point-to-Point Communication" chapter in
+the `MPI Standard <https://www.mpi-forum.org/docs/>`_ to find the
+source, tag, and length of the probed message.
+
+A subsequent receive executed with the same context, and the source and
+tag returned in status by :ref:`MPI_Iprobe` will receive the message that was
+matched by the probe if no other intervening receive occurs after the
+probe. If the receiving process is multithreaded, it is the user's
+responsibility to ensure that the last condition holds.
+
+The source argument of :ref:`MPI_Probe` can be ``MPI_ANY_SOURCE``, and the tag
+argument can be ``MPI_ANY_TAG``, so that one can probe for messages from an
+arbitrary source and/or with an arbitrary tag. However, a specific
+communication context must be provided with the comm argument.
+
+If your application does not need to examine the *status* field, you can
+save resources by using the predefined constant ``MPI_STATUS_IGNORE`` as a
+special value for the *status* argument.
+
+It is not necessary to receive a message immediately after it has been
+probed for, and the same message may be probed for several times before
+it is received.
+
+
+NOTE
+----
+
+Multi-threaded application developers should remember that two threads
+calling :ref:`MPI_Iprobe` may return true for the same message in both
+threads.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+Note that per the "Return Status" section in the "Point-to-Point
+Communication" chapter in the `MPI Standard
+<https://www.mpi-forum.org/docs/>`_, MPI errors on messages queried
+by :ref:`MPI_Iprobe` do not set the ``status.MPI_ERROR`` field in the
+returned *status*.  The error code is always passed to the back-end
+error handler and may be passed back to the caller through the return
+value of :ref:`MPI_Iprobe` if the back-end error handler returns it.
+The pre-defined MPI error handler ``MPI_ERRORS_RETURN`` exhibits this
+behavior, for example.
+
+.. seealso::
+   * :ref:`MPI_Probe`
+   * :ref:`MPI_Cancel`
diff --git a/docs/man-openmpi/man3/MPI_Irecv.3.rst b/docs/man-openmpi/man3/MPI_Irecv.3.rst
new file mode 100644
index 00000000000..2563334c16d
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Irecv.3.rst
@@ -0,0 +1,57 @@
+.. _mpi_irecv:
+
+
+MPI_Irecv
+=========
+
+.. include_body
+
+:ref:`MPI_Irecv` |mdash| Starts a standard-mode, nonblocking receive.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_irecv.rst
+
+INPUT PARAMETERS
+----------------
+* ``buf``: Initial address of receive buffer (choice).
+* ``count``: Number of elements in receive buffer (integer).
+* ``datatype``: Datatype of each receive buffer element (handle).
+* ``source``: Rank of source (integer).
+* ``tag``: Message tag (integer).
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``request``: Communication request (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Nonblocking calls allocate a communication request object and associate
+it with the request handle (the argument request). The request can be
+used later to query the status of the communication or wait for its
+completion.
+
+A nonblocking receive call indicates that the system may start writing
+data into the receive buffer. The receiver should not access any part of
+the receive buffer after a nonblocking receive operation is called,
+until the receive completes.
+
+A receive request can be determined being completed by calling the
+:ref:`MPI_Wait`, :ref:`MPI_Waitany`, :ref:`MPI_Test`, or :ref:`MPI_Testany` with request returned by
+this function.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Recv`
+   * :ref:`MPI_Probe`
+   * :ref:`MPI_Test`
+   * :ref:`MPI_Testany`
+   * :ref:`MPI_Wait`
+   * :ref:`MPI_Waitany`
diff --git a/docs/man-openmpi/man3/MPI_Ireduce.3.rst b/docs/man-openmpi/man3/MPI_Ireduce.3.rst
new file mode 100644
index 00000000000..7323cac2cd3
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Ireduce.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_ireduce:
+
+MPI_Ireduce
+===========
+    .. include_body
+
+.. include:: ../man3/MPI_Reduce.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Ireduce_scatter.3.rst b/docs/man-openmpi/man3/MPI_Ireduce_scatter.3.rst
new file mode 100644
index 00000000000..423791f6be1
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Ireduce_scatter.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_ireduce_scatter:
+
+MPI_Ireduce_scatter
+===================
+    .. include_body
+
+.. include:: ../man3/MPI_Reduce_scatter.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Ireduce_scatter_block.3.rst b/docs/man-openmpi/man3/MPI_Ireduce_scatter_block.3.rst
new file mode 100644
index 00000000000..8d65633a3a9
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Ireduce_scatter_block.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_ireduce_scatter_block:
+
+MPI_Ireduce_scatter_block
+=========================
+    .. include_body
+
+.. include:: ../man3/MPI_Reduce_scatter_block.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Irsend.3.rst b/docs/man-openmpi/man3/MPI_Irsend.3.rst
new file mode 100644
index 00000000000..c5015a30501
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Irsend.3.rst
@@ -0,0 +1,48 @@
+.. _mpi_irsend:
+
+
+MPI_Irsend
+==========
+
+.. include_body
+
+:ref:`MPI_Irsend` |mdash| Starts a ready-mode nonblocking send.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_irsend.rst
+
+INPUT PARAMETERS
+----------------
+* ``buf``: Initial address of send buffer (choice).
+* ``count``: Number of elements in send buffer (integer).
+* ``datatype``: Datatype of each send buffer element (handle).
+* ``dest``: Rank of destination (integer).
+* ``tag``: Message tag (integer).
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``request``: Communication request (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Irsend` starts a ready-mode nonblocking send. Nonblocking calls
+allocate a communication request object and associate it with the
+request handle (the argument request). The request can be used later to
+query the status of the communication or to wait for its completion.
+
+A nonblocking send call indicates that the system may start copying data
+out of the send buffer. The sender should not modify any part of the
+send buffer after a nonblocking send operation is called, until the send
+completes.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Rsend`
diff --git a/docs/man-openmpi/man3/MPI_Is_thread_main.3.rst b/docs/man-openmpi/man3/MPI_Is_thread_main.3.rst
new file mode 100644
index 00000000000..2d35feb3d69
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Is_thread_main.3.rst
@@ -0,0 +1,34 @@
+.. _mpi_is_thread_main:
+
+
+MPI_Is_thread_main
+==================
+
+.. include_body
+
+:ref:`MPI_Is_thread_main` |mdash| Determines if thread called :ref:`MPI_Init`
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_is_thread_main.rst
+
+OUTPUT PARAMETERS
+-----------------
+* ``flag``: True if calling thread is main thread (boolean).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Is_thread_main` is called by a thread to find out whether the caller
+is the main thread (that is, the thread that called :ref:`MPI_Init` or
+MPI_Init_thread).
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Init`
+   * :ref:`MPI_Init_thread`
diff --git a/docs/man-openmpi/man3/MPI_Iscan.3.rst b/docs/man-openmpi/man3/MPI_Iscan.3.rst
new file mode 100644
index 00000000000..363a9fb80ac
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Iscan.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_iscan:
+
+MPI_Iscan
+=========
+    .. include_body
+
+.. include:: ../man3/MPI_Scan.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Iscatter.3.rst b/docs/man-openmpi/man3/MPI_Iscatter.3.rst
new file mode 100644
index 00000000000..4928a836fa8
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Iscatter.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_iscatter:
+
+MPI_Iscatter
+============
+    .. include_body
+
+.. include:: ../man3/MPI_Scatter.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Iscatterv.3.rst b/docs/man-openmpi/man3/MPI_Iscatterv.3.rst
new file mode 100644
index 00000000000..c435e0d0d7a
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Iscatterv.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_iscatterv:
+
+MPI_Iscatterv
+=============
+    .. include_body
+
+.. include:: ../man3/MPI_Scatterv.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Isend.3.rst b/docs/man-openmpi/man3/MPI_Isend.3.rst
new file mode 100644
index 00000000000..ac00f05a98c
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Isend.3.rst
@@ -0,0 +1,56 @@
+.. _mpi_isend:
+
+
+MPI_Isend
+=========
+
+.. include_body
+
+:ref:`MPI_Isend` |mdash| Starts a standard-mode, nonblocking send.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_isend.rst
+
+INPUT PARAMETERS
+----------------
+* ``buf``: Initial address of send buffer (choice).
+* ``count``: Number of elements in send buffer (integer).
+* ``datatype``: Datatype of each send buffer element (handle).
+* ``dest``: Rank of destination (integer).
+* ``tag``: Message tag (integer).
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``request``: Communication request (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Isend` starts a standard-mode, nonblocking send. Nonblocking calls
+allocate a communication request object and associate it with the
+request handle (the argument request). The request can be used later to
+query the status of the communication or wait for its completion.
+
+A nonblocking send call indicates that the system may start copying data
+out of the send buffer. The sender should not modify any part of the
+send buffer after a nonblocking send operation is called, until the send
+completes.
+
+A send request can be determined being completed by calling the
+:ref:`MPI_Wait`, :ref:`MPI_Waitany`, :ref:`MPI_Test`, or :ref:`MPI_Testany` with request returned by
+this function.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Send`
+   * :ref:`MPI_Wait`
+   * :ref:`MPI_Waitany`
+   * :ref:`MPI_Test`
+   * :ref:`MPI_Testany`
diff --git a/docs/man-openmpi/man3/MPI_Isendrecv.3.rst b/docs/man-openmpi/man3/MPI_Isendrecv.3.rst
new file mode 100644
index 00000000000..fa97d3e9597
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Isendrecv.3.rst
@@ -0,0 +1,67 @@
+.. _mpi_isendrecv:
+
+
+MPI_Isendrecv
+=============
+
+.. include_body
+
+:ref:`MPI_Isendrecv` |mdash| Sends and receives a message.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_isendrecv.rst
+
+INPUT PARAMETERS
+----------------
+* ``sendbuf``: Initial address of send buffer (choice).
+* ``sendcount``: Number of elements to send (integer).
+* ``sendtype``: Type of elements in send buffer (handle).
+* ``dest``: Rank of destination (integer).
+* ``sendtag``: Send tag (integer).
+* ``recvcount``: Maximum number of elements to receive (integer).
+* ``recvtype``: Type of elements in receive buffer (handle).
+* ``source``: Rank of source (integer).
+* ``recvtag``: Receive tag (integer).
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``recvbuf``: Initial address of receive buffer (choice).
+* ``request``: Communication request (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The non-blocking send-receive operations combine in one call the sending
+of a message to one destination and the receiving of another message,
+from another process. The two (source and destination) are possibly the
+same. This operation is useful for executing a shift operation across a
+chain of processes. The send-receive operation can be used in
+conjunction with the functions described in the "Process Topologies"
+chapter of the MPI Standard in order to perform shifts on various
+logical topologies.
+
+A message sent by a send-receive operation can be received by a regular
+receive operation or probed by a probe operation; a send-receive
+operation can receive a message sent by a regular send operation.
+
+:ref:`MPI_Isendrecv` executes a non-blocking send and receive operation. Both
+send and receive use the same communicator, but possibly different tags.
+The send buffer and receive buffers must be disjoint, and may have
+different lengths and datatypes.
+
+A non-blocking send-receive request can be determined to be completed by
+calling the :ref:`MPI_Wait`, :ref:`MPI_Waitany`, :ref:`MPI_Test`, or :ref:`MPI_Testany` with the
+request returned by this function.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Isendrecv_replace`
+   * :ref:`MPI_Sendrecv`
+   * :ref:`MPI_Sendrecv_replace`
diff --git a/docs/man-openmpi/man3/MPI_Isendrecv_replace.3.rst b/docs/man-openmpi/man3/MPI_Isendrecv_replace.3.rst
new file mode 100644
index 00000000000..1d8635b7adb
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Isendrecv_replace.3.rst
@@ -0,0 +1,68 @@
+.. _mpi_isendrecv_replace:
+
+
+MPI_Isendrecv_replace
+=====================
+
+.. include_body
+
+:ref:`MPI_Isendrecv_replace` |mdash| Sends and receives a message using a single
+buffer.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_isendrecv_replace.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``buf``: Initial address of send and receive buffer (choice).
+
+INPUT PARAMETERS
+----------------
+* ``count``: Number of elements in send and receive buffer (integer).
+* ``datatype``: Type of elements to send and receive (handle).
+* ``dest``: Rank of destination (integer).
+* ``sendtag``: Send message tag (integer).
+* ``source``: Rank of source (integer).
+* ``recvtag``: Receive message tag (integer).
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``request``: Communication request (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The non-blocking send-receive operations combine in one call the sending
+of a message to one destination and the receiving of another message,
+from another process. The two (source and destination) are possibly the
+same. A send-receive operation is useful for executing a shift operation
+across a chain of processes. The send-receive operation can be used in
+conjunction with the functions described in the "Process Topologies"
+chapter of the MPI Standard in order to perform shifts on various
+logical topologies. Also, a send-receive operation is useful for
+implementing remote procedure calls.
+
+A message sent by a send-receive operation can be received by a regular
+receive operation or probed by a probe operation; a send-receive
+operation can receive a message sent by a regular send operation.
+
+:ref:`MPI_Isendrecv_replace` executes a non-blocking send and receive. The same
+buffer is used both for the send and for the receive, so that the
+message sent is replaced by the message received.
+
+A non-blocking send-receive request can be determined to be completed by
+calling the :ref:`MPI_Wait`, :ref:`MPI_Waitany`, :ref:`MPI_Test`, or :ref:`MPI_Testany` with the
+request returned by this function.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Isendrecv`
+   * :ref:`MPI_Sendrecv`
+   * :ref:`MPI_Sendrecv_replace`
diff --git a/docs/man-openmpi/man3/MPI_Issend.3.rst b/docs/man-openmpi/man3/MPI_Issend.3.rst
new file mode 100644
index 00000000000..082c2ba17fd
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Issend.3.rst
@@ -0,0 +1,50 @@
+.. _mpi_issend:
+
+
+MPI_Issend
+==========
+
+.. include_body
+
+:ref:`MPI_Issend` |mdash| Starts a nonblocking synchronous send.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_issend.rst
+
+INPUT PARAMETERS
+----------------
+* ``buf``: Initial address of send buffer (choice).
+* ``count``: Number of elements in send buffer (integer).
+* ``datatype``: Datatype of each send buffer element (handle).
+* ``dest``: Rank of destination (integer).
+* ``tag``: Message tag (integer).
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``request``: Communication request (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Starts a synchronous mode, nonblocking send.
+
+Nonblocking calls allocate a communication request object and associate
+it with the request handle (the argument request). The request can be
+used later to query the status of the communication or wait for its
+completion.
+
+A nonblocking send call indicates that the system may start copying data
+out of the send buffer. The sender should not modify any part of the
+send buffer after a nonblocking send operation is called, until the send
+completes.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Ssend`
diff --git a/docs/man-openmpi/man3/MPI_Keyval_create.3.rst b/docs/man-openmpi/man3/MPI_Keyval_create.3.rst
new file mode 100644
index 00000000000..4af631197df
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Keyval_create.3.rst
@@ -0,0 +1,136 @@
+.. _mpi_keyval_create:
+
+
+MPI_Keyval_create
+=================
+
+.. include_body
+
+:ref:`MPI_Keyval_create` |mdash| Generates a new attribute key |mdash| |deprecated_favor| :ref:`MPI_Comm_create_keyval`.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_keyval_create.rst
+
+INPUT PARAMETERS
+----------------
+* ``copy_fn``: Copy callback function for keyval.
+* ``delete_fn``: Delete callback function for keyval.
+* ``extra_state``: Extra state for callback functions.
+
+OUTPUT PARAMETERS
+-----------------
+* ``keyval``: Key value for future access (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Note that use of this routine is *deprecated* as of MPI-2. Please use
+:ref:`MPI_Comm_create_keyval` instead.
+
+Generates a new attribute key. Keys are locally unique in a process and
+opaque to the user, though they are explicitly stored in integers. Once
+allocated, the key value can be used to associate attributes and access
+them on any locally defined communicator.
+
+The copy_fn function is invoked when a communicator is duplicated by
+:ref:`MPI_Comm_dup`. copy_fn should be of type MPI_Copy_function, which is
+defined as follows:
+
+.. code-block:: c
+
+     typedef int MPI_Copy_function(MPI_Comm oldcomm, int keyval,
+                                   void *extra_state, void *attribute_val_in,
+                                   void *attribute_val_out, int *flag)
+
+A Fortran declaration for such a function is as follows:
+
+.. code-block:: fortran
+
+     SUBROUTINE COPY_FUNCTION(OLDCOMM, KEYVAL, EXTRA_STATE, ATTRIBUTE_VAL_IN,
+                 ATTRIBUTE_VAL_OUT, FLAG, IERR)
+     INTEGER OLDCOMM, KEYVAL, EXTRA_STATE,
+     ATTRIBUTE_VAL_IN, ATTRIBUTE_VAL_OUT, IERR
+     LOGICAL FLAG
+
+The copy callback function is invoked for each key value in oldcomm in
+arbitrary order. Each call to the copy callback is made with a key value
+and its corresponding attribute. If it returns flag = 0, then the
+attribute is deleted in the duplicated communicator. Otherwise ( flag =
+1), the new attribute value is set to the value returned in
+attribute_val_out. The function returns MPI_SUCCESS on success and an
+error code on failure (in which case :ref:`MPI_Comm_dup` will fail).
+
+copy_fn may be specified as MPI_NULL_COPY_FN or MPI_DUP_FN from either C
+or Fortran; MPI_NULL_COPY_FN is a function that does nothing other than
+return flag = 0, and MPI_SUCCESS. MPI_DUP_FN is a simple-minded copy
+function that sets flag = 1, returns the value of attribute_val_in in
+attribute_val_out, and returns MPI_SUCCESS.
+
+
+NOTES
+-----
+
+Key values are global (available for any and all communicators).
+
+There are subtle differences between C and Fortran that require that the
+copy_fn be written in the same language that :ref:`MPI_Keyval_create` is called
+from. This should not be a problem for most users; only programmers
+using both Fortran and C in the same program need to be sure that they
+follow this rule.
+
+Even though both formal arguments attribute_val_in and attribute_val_out
+are of type void*, their usage differs. The C copy function is passed by
+MPI in attribute_val_in the value of the attribute, and in
+attribute_val_out the address of the attribute, so as to allow the
+function to return the (new) attribute value. The use of type ``void *`` for
+both is to avoid messy type casts.
+
+A valid copy function is one that completely duplicates the information
+by making a full duplicate copy of the data structures implied by an
+attribute; another might just make another reference to that data
+structure, while using a reference-count mechanism. Other types of
+attributes might not copy at all (they might be specific to oldcomm
+only).
+
+Analogous to copy_fn is a callback deletion function, defined as
+follows. The delete_fn function is invoked when a communicator is
+deleted by :ref:`MPI_Comm_free` or when a call is made explicitly to
+:ref:`MPI_Attr_delete`. delete_fn should be of type MPI_Delete_function, which
+is defined as follows:
+
+.. code-block:: c
+
+     typedef int MPI_Delete_function(MPI_Comm comm, int keyval,
+         void *attribute_val, void *extra_state);
+
+A Fortran declaration for such a function is as follows:
+
+.. code-block:: fortran
+
+     SUBROUTINE DELETE_FUNCTION(COMM, KEYVAL,ATTRIBUTE_VAL, EXTRA_STATE, IERR)
+         INTEGER COMM, KEYVAL, ATTRIBUTE_VAL, EXTRA_STATE, IERR
+
+This function is called by :ref:`MPI_Comm_free`, :ref:`MPI_Attr_delete`, and
+:ref:`MPI_Attr_put` to do whatever is needed to remove an attribute. The
+function returns MPI_SUCCESS on success and an error code on failure (in
+which case :ref:`MPI_COMM_FREE` will fail).
+
+delete_fn may be specified as MPI_NULL_DELETE_FN from either C or
+FORTRAN; MPI_NULL_DELETE_FN is a function that does nothing, other than
+returning MPI_SUCCESS.
+
+The special key value MPI_KEYVAL_INVALID is never returned by
+:ref:`MPI_Keyval_create`. Therefore, it can be used for static initialization
+of key values.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Keyval_free`
+   * :ref:`MPI_Comm_create_keyval`
+   * :ref:`MPI_Comm_free_keyval`
diff --git a/docs/man-openmpi/man3/MPI_Keyval_free.3.rst b/docs/man-openmpi/man3/MPI_Keyval_free.3.rst
new file mode 100644
index 00000000000..4b36f146939
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Keyval_free.3.rst
@@ -0,0 +1,37 @@
+.. _mpi_keyval_free:
+
+
+MPI_Keyval_free
+===============
+
+.. include_body
+
+:ref:`MPI_Keyval_free` |mdash| Frees attribute key for communicator cache attribute |mdash| |deprecated_favor| :ref:`MPI_Comm_free_keyval`.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_keyval_free.rst
+
+INPUT PARAMETER
+---------------
+* ``keyval``: Frees the integer key value (integer).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Note that use of this routine is *deprecated* as of MPI-2. Please use
+:ref:`MPI_Comm_free_keyval` instead.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Keyval_create`
+   * :ref:`MPI_Comm_create_keyval`
+   * :ref:`MPI_Comm_free_keyval`
diff --git a/docs/man-openmpi/man3/MPI_Lookup_name.3.rst b/docs/man-openmpi/man3/MPI_Lookup_name.3.rst
new file mode 100644
index 00000000000..2bb4dc53cf0
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Lookup_name.3.rst
@@ -0,0 +1,90 @@
+.. _mpi_lookup_name:
+
+
+MPI_Lookup_name
+===============
+
+.. include_body
+
+::
+
+   MPI_Lookup_name - Finds port associated with a service name
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_lookup_name.rst
+
+INPUT PARAMETERS
+----------------
+* ``service_name``: A service name (string).
+* ``info``: Options to the name service functions (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``port_name``: a port name (string).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This function retrieves a *port_name* published under *service_name* by
+a previous invocation of :ref:`MPI_Publish_name`. The application must supply a
+*port_name* buffer large enough to hold the largest possible port name
+(i.e., MPI_MAX_PORT_NAME bytes).
+
+
+INFO ARGUMENTS
+--------------
+
+The following keys for *info* are recognized:
+
+::
+
+   Key                   Type      Description
+   ---                   ----      -----------
+
+   ompi_lookup_order     char *    Resolution order for name lookup.
+
+The *ompi_lookup_order* info key can specify one of four valid string
+values (see the NAME SCOPE section below for more information on name
+scopes):
+
+*local*: Only search the local scope for name resolution.
+
+*global*: Only search the global scope for name resolution.
+
+*local,global*: Search the local scope for name resolution. If
+   not found, try searching the global scope for name resolution. This
+   behavior is the default if the *ompi_lookup_order* info key is not
+   specified.
+
+*global,local*: Search the global scope for name resolution. If
+   not found, try searching the local scope for name resolution.
+
+If no info key is provided, the search will first check to see if a
+global server has been specified and is available. If so, then the
+search will default to global scope first, followed by local. Otherwise,
+the search will default to local.
+
+
+NAME SCOPE
+----------
+
+Open MPI supports two name scopes: *global* and *local*. Local scope
+values are placed in a data store located on the mpirun of the calling
+process' job, while global scope values reside on a central server.
+Calls to :ref:`MPI_Unpublish_name` must correctly specify the scope to be used
+in finding the value to be removed. The function will return an error if
+the specified service name is not found on the indicated location.
+
+For a more detailed description of scoping rules, please see the
+:ref:`MPI_Publish_name` man page.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Publish_name`
+   * :ref:`MPI_Open_port`
diff --git a/docs/man-openmpi/man3/MPI_Message_c2f.3.rst b/docs/man-openmpi/man3/MPI_Message_c2f.3.rst
new file mode 100644
index 00000000000..b7800cef771
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Message_c2f.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_message_c2f:
+
+MPI_Message_c2f
+===============
+
+.. include_body
+
+.. include:: ../man3/MPI_Comm_f2c.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_Message_f2c.3.rst b/docs/man-openmpi/man3/MPI_Message_f2c.3.rst
new file mode 100644
index 00000000000..686fdac1f2b
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Message_f2c.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_message_f2c:
+
+MPI_Message_f2c
+===============
+
+.. include_body
+
+.. include:: ../man3/MPI_Comm_f2c.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_Mprobe.3.rst b/docs/man-openmpi/man3/MPI_Mprobe.3.rst
new file mode 100644
index 00000000000..82cee605636
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Mprobe.3.rst
@@ -0,0 +1,58 @@
+.. _mpi_mprobe:
+
+
+MPI_Mprobe
+==========
+
+.. include_body
+
+:ref:`MPI_Mprobe` |mdash| Blocking matched probe for a message.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_mprobe.rst
+
+INPUT PARAMETERS
+----------------
+* ``source``: Source rank or ``MPI_ANY_SOURCE`` (integer).
+* ``tag``: Tag value or ``MPI_ANY_TAG`` (integer).
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``message``: Message (handle).
+* ``status``: Status object (status).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Like :ref:`MPI_Probe` and :ref:`MPI_Iprobe`, the :ref:`MPI_Mprobe` and :ref:`MPI_Improbe` operations
+allow incoming messages to be queried without actually receiving them,
+except that :ref:`MPI_Mprobe` and :ref:`MPI_Improbe` provide a mechanism to receive
+the specific message that was matched regardless of other intervening
+probe or receive operations. This gives the application an opportunity
+to decide how to receive the message, based on the information returned
+by the probe. In particular, the application may allocate memory for the
+receive buffer according to the length of the probed message.
+
+A matching probe with ``MPI_PROC_NULL`` as *source* returns *message* =
+``MPI_MESSAGE_NO_PROC``, and the *status* object returns source =
+``MPI_PROC_NULL``, tag = ``MPI_ANY_TAG``, and count = 0.
+
+When :ref:`MPI_Mprobe` returns (from a non-``MPI_PROC_NULL`` *source*), the matched
+message can then be received by passing the *message* handle to the
+:ref:`MPI_Mrecv` or :ref:`MPI_Imrecv` functions.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Improbe`
+   * :ref:`MPI_Probe`
+   * :ref:`MPI_Iprobe`
+   * :ref:`MPI_Mrecv`
+   * :ref:`MPI_Imrecv`
+   * :ref:`MPI_Cancel`
diff --git a/docs/man-openmpi/man3/MPI_Mrecv.3.rst b/docs/man-openmpi/man3/MPI_Mrecv.3.rst
new file mode 100644
index 00000000000..12a0a1b2d65
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Mrecv.3.rst
@@ -0,0 +1,63 @@
+.. _mpi_mrecv:
+
+
+MPI_Mrecv
+=========
+
+.. include_body
+
+:ref:`MPI_Mrecv` |mdash| Blocking receive for a matched message
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_mrecv.rst
+
+INPUT PARAMETERS
+----------------
+* ``count``: Number of elements to receive (nonnegative integer).
+* ``datatype``: Datatype of each send buffer element (handle).
+* ``message``: Message (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``buf``: Initial address of receive buffer (choice).
+* ``status``: Status object (status).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The functions :ref:`MPI_Mrecv` and :ref:`MPI_Imrecv` receive messages that have been
+previously matched by a matching probe.
+
+If :ref:`MPI_Mrecv` is called with ``MPI_MESSAGE_NULL`` as the message argument,
+the call returns immediately with the *status* object set to *source* =
+``MPI_PROC_NULL``, *tag* = ``MPI_ANY_TAG``, and *count* = 0, as if a receive
+from ``MPI_PROC_NULL`` was issued.
+
+If your application does not need to examine the *status* field, you
+can save resources by using the predefined constant
+``MPI_STATUS_IGNORE`` as a special value for the *status* argument.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+Note that per the "Return Status" section in the "Point-to-Point
+Communication" chapter in the `MPI Standard
+<https://www.mpi-forum.org/docs/>`_, MPI errors on messages received
+by :ref:`MPI_Mrecv` do not set the ``status.MPI_ERROR`` field in the
+returned *status*.  The error code is always passed to the back-end
+error handler and may be passed back to the caller through the return
+value of :ref:`MPI_Mrecv` if the back-end error handler returns it.
+The pre-defined MPI error handler ``MPI_ERRORS_RETURN`` exhibits this
+behavior, for example.
+
+.. seealso::
+   * :ref:`MPI_Mprobe`
+   * :ref:`MPI_Improbe`
+   * :ref:`MPI_Probe`
+   * :ref:`MPI_Iprobe`
+   * :ref:`MPI_Imrecv`
+   * :ref:`MPI_Cancel`
diff --git a/docs/man-openmpi/man3/MPI_Neighbor_allgather.3.rst b/docs/man-openmpi/man3/MPI_Neighbor_allgather.3.rst
new file mode 100644
index 00000000000..8f5ccffaf8f
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Neighbor_allgather.3.rst
@@ -0,0 +1,94 @@
+.. _mpi_neighbor_allgather:
+
+
+MPI_Neighbor_allgather
+======================
+
+.. include_body
+
+:ref:`MPI_Neighbor_allgather`, :ref:`MPI_Ineighbor_allgather`,
+:ref:`MPI_Neighbor_allgather_init` |mdash| Gathers and distributes data from and to all
+neighbors
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Neighbor_allgather, MPI_Ineighbor_allgather, MPI_Neighbor_allgather_init
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_neighbor_allgather.rst
+
+INPUT PARAMETERS
+----------------
+* ``sendbuf``: Starting address of send buffer (choice).
+* ``sendcount``: Number of elements in send buffer (integer).
+* ``sendtype``: Datatype of send buffer elements (handle).
+* ``recvbuf``: Starting address of recv buffer (choice).
+* ``recvcount``: Number of elements received from any process (integer).
+* ``recvtype``: Datatype of receive buffer elements (handle).
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``recvbuf``: Address of receive buffer (choice).
+* ``request``: Request (handle, non-blocking and persistent only).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Neighbor_allgather` is similar to :ref:`MPI_Allgather`, except that only the
+neighboring processes receive the result, instead of all processes. The
+neighbors and buffer layout is determined by the topology of *comm*.
+
+The type signature associated with sendcount, sendtype at a process must
+be equal to the type signature associated with recvcount, recvtype at
+any other process.
+
+
+NEIGHBOR ORDERING
+-----------------
+
+For a distributed graph topology, created with :ref:`MPI_Dist_graph_create`,
+the sequence of neighbors in the send and receive buffers at each
+process is defined as the sequence returned by :ref:`MPI_Dist_graph_neighbors`
+for destinations and sources, respectively. For a general graph
+topology, created with :ref:`MPI_Graph_create`, the order of neighbors in the
+send and receive buffers is defined as the sequence of neighbors as
+returned by :ref:`MPI_Graph_neighbors`. Note that general graph topologies
+should generally be replaced by the distributed graph topologies.
+
+For a Cartesian topology, created with :ref:`MPI_Cart_create`, the sequence of
+neighbors in the send and receive buffers at each process is defined by
+order of the dimensions, first the neighbor in the negative direction
+and then in the positive direction with displacement 1. The numbers of
+sources and destinations in the communication routines are 2*ndims with
+ndims defined in :ref:`MPI_Cart_create`. If a neighbor does not exist, i.e., at
+the border of a Cartesian topology in the case of a non-periodic virtual
+grid dimension (i.e., periods[...]==false), then this neighbor is
+defined to be ``MPI_PROC_NULL``.
+
+If a neighbor in any of the functions is ``MPI_PROC_NULL``, then the
+neighborhood collective communication behaves like a point-to-point
+communication with ``MPI_PROC_NULL`` in this direction. That is, the buffer
+is still part of the sequence of neighbors but it is neither
+communicated nor updated.
+
+
+NOTES
+-----
+
+The MPI_IN_PLACE option for *sendbuf* is not meaningful for this
+operation.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Neighbor_allgatherv`
+   * :ref:`MPI_Cart_create`
+   * :ref:`MPI_Graph_create`
+   * :ref:`MPI_Dist_graph_create`
+   * :ref:`MPI_Gather`
diff --git a/docs/man-openmpi/man3/MPI_Neighbor_allgather_init.3.rst b/docs/man-openmpi/man3/MPI_Neighbor_allgather_init.3.rst
new file mode 100644
index 00000000000..e6fc7159aaa
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Neighbor_allgather_init.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_neighbor_allgather_init:
+
+MPI_Neighbor_allgather_init
+===========================
+    .. include_body
+
+.. include:: ../man3/MPI_Neighbor_allgather.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Neighbor_allgatherv.3.rst b/docs/man-openmpi/man3/MPI_Neighbor_allgatherv.3.rst
new file mode 100644
index 00000000000..86ad17714fb
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Neighbor_allgatherv.3.rst
@@ -0,0 +1,98 @@
+.. _mpi_neighbor_allgatherv:
+
+
+MPI_Neighbor_allgatherv
+=======================
+
+.. include_body
+
+:ref:`MPI_Neighbor_allgatherv`, :ref:`MPI_Ineighbor_allgatherv`,
+:ref:`MPI_Neighbor_allgatherv_init` |mdash| Gathers and distributes data from and
+to all neighbors. Each process may contribute a different amount of
+data.
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Neighbor_allgatherv, MPI_Ineighbor_allgatherv, MPI_Neighbor_allgatherv_init
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_neighbor_allgatherv.rst
+
+INPUT PARAMETERS
+----------------
+* ``sendbuf``: Starting address of send buffer (choice).
+* ``sendcount``: Number of elements in send buffer (integer).
+* ``sendtype``: Datatype of send buffer elements (handle).
+* ``recvcount``: Integer array (of length group size) containing the number of elements that are received from each neighbor.
+* ``displs``: Integer array (of length group size). Entry i specifies the displacement (relative to recvbuf) at which to place the incoming data from neighbor i.
+* ``recvtype``: Datatype of receive buffer elements (handle).
+* ``comm``: Communicator (handle).
+* ``info Info (handle, persistent only).``:
+
+OUTPUT PARAMETERS
+-----------------
+* ``recvbuf``: Address of receive buffer (choice).
+* ``request``: Request (handle, non-blocking and persistent only).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Neighbor_allgatherv` is similar to :ref:`MPI_Neighbor_allgather` in that all
+processes gather data from all neighbors, except that each process can
+send a different amount of data. The block of data sent from the jth
+neighbor is received by every neighbor and placed in the jth block of
+the buffer. The neighbors and buffer layout is determined by the
+topology of *comm*. *recvbuf.*
+
+The type signature associated with sendcount, sendtype, at process j
+must be equal to the type signature associated with the corresponding
+entry in *recvcounts* on neighboring processes.
+
+
+NEIGHBOR ORDERING
+-----------------
+
+For a distributed graph topology, created with :ref:`MPI_Dist_graph_create`,
+the sequence of neighbors in the send and receive buffers at each
+process is defined as the sequence returned by :ref:`MPI_Dist_graph_neighbors`
+for destinations and sources, respectively. For a general graph
+topology, created with :ref:`MPI_Graph_create`, the order of neighbors in the
+send and receive buffers is defined as the sequence of neighbors as
+returned by :ref:`MPI_Graph_neighbors`. Note that general graph topologies
+should generally be replaced by the distributed graph topologies.
+
+For a Cartesian topology, created with :ref:`MPI_Cart_create`, the sequence of
+neighbors in the send and receive buffers at each process is defined by
+order of the dimensions, first the neighbor in the negative direction
+and then in the positive direction with displacement 1. The numbers of
+sources and destinations in the communication routines are 2*ndims with
+ndims defined in :ref:`MPI_Cart_create`. If a neighbor does not exist, i.e., at
+the border of a Cartesian topology in the case of a non-periodic virtual
+grid dimension (i.e., periods[...]==false), then this neighbor is
+defined to be ``MPI_PROC_NULL``.
+
+If a neighbor in any of the functions is ``MPI_PROC_NULL``, then the
+neighborhood collective communication behaves like a point-to-point
+communication with ``MPI_PROC_NULL`` in this direction. That is, the buffer
+is still part of the sequence of neighbors but it is neither
+communicated nor updated.
+
+
+NOTES
+-----
+
+The MPI_IN_PLACE option for *sendbuf* is not meaningful for this
+operation.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Neighbor_allgather`
+   * :ref:`MPI_Cart_create`
+   * :ref:`MPI_Graph_create`
+   * :ref:`MPI_Dist_graph_create`
diff --git a/docs/man-openmpi/man3/MPI_Neighbor_allgatherv_init.3.rst b/docs/man-openmpi/man3/MPI_Neighbor_allgatherv_init.3.rst
new file mode 100644
index 00000000000..4253f2036ab
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Neighbor_allgatherv_init.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_neighbor_allgatherv_init:
+
+MPI_Neighbor_allgatherv_init
+============================
+    .. include_body
+
+.. include:: ../man3/MPI_Neighbor_allgatherv.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Neighbor_alltoall.3.rst b/docs/man-openmpi/man3/MPI_Neighbor_alltoall.3.rst
new file mode 100644
index 00000000000..73e1e52f66e
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Neighbor_alltoall.3.rst
@@ -0,0 +1,132 @@
+.. _mpi_neighbor_alltoall:
+
+
+MPI_Neighbor_alltoall
+=====================
+
+.. include_body
+
+:ref:`MPI_Neighbor_alltoall`, :ref:`MPI_Ineighbor_alltoall`, :ref:`MPI_Neighbor_alltoall` - All processes send data to neighboring processes in a virtual topology communicator
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Neighbor_alltoall, MPI_Ineighbor_alltoall, MPI_Neighbor_alltoall_init
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_neighbor_alltoall.rst
+
+INPUT PARAMETERS
+----------------
+* ``sendbuf``: Starting address of send buffer (choice).
+* ``sendcount``: Number of elements to send to each process (integer).
+* ``sendtype``: Datatype of send buffer elements (handle).
+* ``recvcount``: Number of elements to receive from each process (integer).
+* ``recvtype``: Datatype of receive buffer elements (handle).
+* ``comm``: Communicator over which data is to be exchanged (handle).
+* ``info``: Info (handle, persistent only).
+
+OUTPUT PARAMETERS
+-----------------
+* ``recvbuf``: Starting address of receive buffer (choice).
+* ``request``: Request (handle, non-blocking and persistent only).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Neighbor_alltoall` is a collective operation in which all processes
+send and receive the same amount of data to each neighbor. The operation
+of this routine can be represented as follows, where each process
+performs 2n (n being the number of neighbors in communicator *comm*)
+independent point-to-point communications. The neighbors and buffer
+layout are determined by the topology of *comm*.
+
+Example of :ref:`MPI_Neighbor_alltoall` semantics for cartesian topologies:
+
+.. code-block:: c
+
+           MPI_Cart_get(comm, maxdims, dims, periods, coords);
+           for (dim = 0, i = 0 ; dim < dims ; ++dim) {
+               MPI_Cart_shift(comm, dim, 1, &r0, &r1);
+               MPI_Isend(sendbuf + i * sendcount * extent(sendtype),
+                         sendcount, sendtype, r0, ..., comm, ...);
+               MPI_Irecv(recvbuf + i * recvcount * extent(recvtype),
+                         recvcount, recvtype, r0, ..., comm, ...);
+               ++i;
+               MPI_Isend(sendbuf + i * sendcount * extent(sendtype),
+                         sendcount, sendtype, r1, ..., comm, &req[i]);
+               MPI_Irecv(recvbuf + i * recvcount * extent(recvtype),
+                         recvcount, recvtype, r1, ..., comm, ...);
+               ++i;
+           }
+
+           MPI_Waitall (...);
+
+Each process breaks up its local *sendbuf* into n blocks - each
+containing *sendcount* elements of type *sendtype* - and divides its
+*recvbuf* similarly according to *recvcount* and *recvtype*. Process j
+sends the k-th block of its local *sendbuf* to neighbor k, which places
+the data in the j-th block of its local *recvbuf*. The amount of data
+sent must be equal to the amount of data received, pairwise, between
+every pair of processes.
+
+
+NEIGHBOR ORDERING
+-----------------
+
+For a distributed graph topology, created with :ref:`MPI_Dist_graph_create`,
+the sequence of neighbors in the send and receive buffers at each
+process is defined as the sequence returned by :ref:`MPI_Dist_graph_neighbors`
+for destinations and sources, respectively. For a general graph
+topology, created with :ref:`MPI_Graph_create`, the order of neighbors in the
+send and receive buffers is defined as the sequence of neighbors as
+returned by :ref:`MPI_Graph_neighbors`. Note that general graph topologies
+should generally be replaced by the distributed graph topologies.
+
+For a Cartesian topology, created with :ref:`MPI_Cart_create`, the sequence of
+neighbors in the send and receive buffers at each process is defined by
+order of the dimensions, first the neighbor in the negative direction
+and then in the positive direction with displacement 1. The numbers of
+sources and destinations in the communication routines are 2*ndims with
+ndims defined in :ref:`MPI_Cart_create`. If a neighbor does not exist, i.e., at
+the border of a Cartesian topology in the case of a non-periodic virtual
+grid dimension (i.e., periods[...]==false), then this neighbor is
+defined to be ``MPI_PROC_NULL``.
+
+If a neighbor in any of the functions is ``MPI_PROC_NULL``, then the
+neighborhood collective communication behaves like a point-to-point
+communication with ``MPI_PROC_NULL`` in this direction. That is, the buffer
+is still part of the sequence of neighbors but it is neither
+communicated nor updated.
+
+
+NOTES
+-----
+
+The MPI_IN_PLACE option for *sendbuf* is not meaningful for this
+function.
+
+All arguments on all processes are significant. The *comm* argument, in
+particular, must describe the same communicator on all processes. *comm*
+must be either a cartesian, graph, or dist graph communicator.
+
+There are two MPI library functions that are more general than
+:ref:`MPI_Neighbor_alltoall`. :ref:`MPI_Neighbor_alltoallv` allows all-to-all
+communication to and from buffers that need not be contiguous; different
+processes may send and receive different amounts of data.
+:ref:`MPI_Neighbor_alltoallw` expands :ref:`MPI_Neighbor_alltoallv`'s functionality to
+allow the exchange of data with different datatypes.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Neighbor_alltoallv`
+   * :ref:`MPI_Neighbor_alltoallw`
+   * :ref:`MPI_Cart_create`
+   * :ref:`MPI_Graph_create`
+   * :ref:`MPI_Dist_graph_create`
+   * :ref:`MPI_Dist_graph_create_adjacent`
diff --git a/docs/man-openmpi/man3/MPI_Neighbor_alltoall_init.3.rst b/docs/man-openmpi/man3/MPI_Neighbor_alltoall_init.3.rst
new file mode 100644
index 00000000000..f1a80c60827
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Neighbor_alltoall_init.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_neighbor_alltoall_init:
+
+MPI_Neighbor_alltoall_init
+==========================
+    .. include_body
+
+.. include:: ../man3/MPI_Neighbor_alltoall.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Neighbor_alltoallv.3.rst b/docs/man-openmpi/man3/MPI_Neighbor_alltoallv.3.rst
new file mode 100644
index 00000000000..7286de763fd
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Neighbor_alltoallv.3.rst
@@ -0,0 +1,136 @@
+.. _mpi_neighbor_alltoallv:
+
+
+MPI_Neighbor_alltoallv
+======================
+
+.. include_body
+
+:ref:`MPI_Neighbor_alltoallv`, :ref:`MPI_Ineighbor_alltoallv`,
+:ref:`MPI_Neighbor_alltoallv_init` |mdash| All processes send different amounts of
+data to, and receive different amounts of data from, all neighbors
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Neighbor_alltoallv, MPI_Ineighbor_alltoallv, MPI_Neighbor_alltoallv_init
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_neighbor_alltoallv.rst
+
+INPUT PARAMETERS
+----------------
+* ``sendbuf``: Starting address of send buffer.
+* ``sendcounts``: Integer array, where entry i specifies the number of elements to send to neighbor i.
+* ``sdispls``: Integer array, where entry i specifies the displacement (offset from *sendbuf*, in units of *sendtype*) from which to send data to neighbor i.
+* ``sendtype``: Datatype of send buffer elements.
+* ``recvcounts``: Integer array, where entry j specifies the number of elements to receive from neighbor j.
+* ``rdispls``: Integer array, where entry j specifies the displacement (offset from *recvbuf*, in units of *recvtype*) to which data from neighbor j should be written.
+* ``recvtype``: Datatype of receive buffer elements.
+* ``comm``: Communicator over which data is to be exchanged.
+* ``info``: Info (handle, persistent only).
+
+OUTPUT PARAMETERS
+-----------------
+* ``recvbuf``: Address of receive buffer.
+* ``request``: Request (handle, non-blocking and persistent only).
+* ``ierror``: Fortran only: Error status.
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Neighbor_alltoallv` is a generalized collective operation in which
+all processes send data to and receive data from all neighbors. It adds
+flexibility to :ref:`MPI_Neighbor_alltoall` by allowing the user to specify
+data to send and receive vector-style (via a displacement and element
+count). The operation of this routine can be thought of as follows,
+where each process performs 2n (n being the number of neighbors in to
+topology of communicator *comm*) independent point-to-point
+communications. The neighbors and buffer layout are determined by the
+topology of *comm*.
+
+.. code-block:: c
+
+           MPI_Cart_get(comm, maxdims, dims, periods, coords);
+           for (dim = 0, i = 0 ; dim < dims ; ++dim) {
+               MPI_Cart_shift(comm, dim, 1, &r0, &r1);
+               MPI_Isend(sendbuf + sdispls[i]  * extent(sendtype),
+                         sendcount, sendtype, r0, ..., comm, ...);
+               MPI_Irecv(recvbuf + rdispls[i] * extent(recvtype),
+                         recvcount, recvtype, r0, ..., comm, ...);
+               ++i;
+               MPI_Isend(sendbuf + sdispls[i] * extent(sendtype),
+                         sendcount, sendtype, r1, ..., comm, &req[i]);
+               MPI_Irecv(recvbuf + rdispls[i] * extent(recvtype),
+                         recvcount, recvtype, r1, ..., comm, ...);
+               ++i;
+           }
+
+Process j sends the k-th block of its local *sendbuf* to neighbor k,
+which places the data in the j-th block of its local *recvbuf*.
+
+When a pair of processes exchanges data, each may pass different element
+count and datatype arguments so long as the sender specifies the same
+amount of data to send (in bytes) as the receiver expects to receive.
+
+Note that process i may send a different amount of data to process j
+than it receives from process j. Also, a process may send entirely
+different amounts of data to different processes in the communicator.
+
+
+NEIGHBOR ORDERING
+-----------------
+
+For a distributed graph topology, created with :ref:`MPI_Dist_graph_create`,
+the sequence of neighbors in the send and receive buffers at each
+process is defined as the sequence returned by :ref:`MPI_Dist_graph_neighbors`
+for destinations and sources, respectively. For a general graph
+topology, created with :ref:`MPI_Graph_create`, the order of neighbors in the
+send and receive buffers is defined as the sequence of neighbors as
+returned by :ref:`MPI_Graph_neighbors`. Note that general graph topologies
+should generally be replaced by the distributed graph topologies.
+
+For a Cartesian topology, created with :ref:`MPI_Cart_create`, the sequence of
+neighbors in the send and receive buffers at each process is defined by
+order of the dimensions, first the neighbor in the negative direction
+and then in the positive direction with displacement 1. The numbers of
+sources and destinations in the communication routines are 2*ndims with
+ndims defined in :ref:`MPI_Cart_create`. If a neighbor does not exist, i.e., at
+the border of a Cartesian topology in the case of a non-periodic virtual
+grid dimension (i.e., periods[...]==false), then this neighbor is
+defined to be ``MPI_PROC_NULL``.
+
+If a neighbor in any of the functions is ``MPI_PROC_NULL``, then the
+neighborhood collective communication behaves like a point-to-point
+communication with ``MPI_PROC_NULL`` in this direction. That is, the buffer
+is still part of the sequence of neighbors but it is neither
+communicated nor updated.
+
+
+NOTES
+-----
+
+The MPI_IN_PLACE option for *sendbuf* is not meaningful for this
+operation.
+
+The specification of counts and displacements should not cause any
+location to be written more than once.
+
+All arguments on all processes are significant. The *comm* argument, in
+particular, must describe the same communicator on all processes.
+
+The offsets of *sdispls* and *rdispls* are measured in units of
+*sendtype* and *recvtype*, respectively. Compare this to
+:ref:`MPI_Neighbor_alltoallw`, where these offsets are measured in bytes.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Neighbor_alltoall`
+   * :ref:`MPI_Neighbor_alltoallw`
+   * :ref:`MPI_Cart_create`
+   * :ref:`MPI_Graph_create`
+   * :ref:`MPI_Dist_graph_create`
diff --git a/docs/man-openmpi/man3/MPI_Neighbor_alltoallv_init.3.rst b/docs/man-openmpi/man3/MPI_Neighbor_alltoallv_init.3.rst
new file mode 100644
index 00000000000..ea22855d346
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Neighbor_alltoallv_init.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_neighbor_alltoallv_init:
+
+MPI_Neighbor_alltoallv_init
+===========================
+    .. include_body
+
+.. include:: ../man3/MPI_Neighbor_alltoallv.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Neighbor_alltoallw.3.rst b/docs/man-openmpi/man3/MPI_Neighbor_alltoallw.3.rst
new file mode 100644
index 00000000000..502bca11d83
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Neighbor_alltoallw.3.rst
@@ -0,0 +1,118 @@
+.. _mpi_neighbor_alltoallw:
+
+
+MPI_Neighbor_alltoallw
+======================
+
+.. include_body
+
+:ref:`MPI_Neighbor_alltoallw`, :ref:`MPI_Ineighbor_alltoallw`,
+:ref:`MPI_Neighbor_alltoallw_init` |mdash| All processes send data of different
+types to, and receive data of different types from, all processes
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Neighbor_alltoallw, MPI_Ineighbor_alltoallw, MPI_Neighbor_alltoallw_init
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_neighbor_alltoallw.rst
+
+INPUT PARAMETERS
+----------------
+* ``sendbuf``: Starting address of send buffer.
+* ``sendcounts``: Integer array, where entry i specifies the number of elements to send to neighbor i.
+* ``sdispls``: Integer array, where entry i specifies the displacement (in bytes, offset from *sendbuf*) from which to send data to neighbor i.
+* ``sendtypes``: Datatype array, where entry i specifies the datatype to use when sending data to neighbor i.
+* ``recvcounts``: Integer array, where entry j specifies the number of elements to receive from neighbor j.
+* ``rdispls``: Integer array, where entry j specifies the displacement (in bytes, offset from *recvbuf*) to which data from neighbor j should be written.
+* ``recvtypes``: Datatype array, where entry j specifies the datatype to use when receiving data from neighbor j.
+* ``comm``: Communicator over which data is to be exchanged.
+* ``info``: Info (handle, persistent only).
+
+OUTPUT PARAMETERS
+-----------------
+* ``recvbuf``: Address of receive buffer.
+* ``request``: Request (handle, non-blocking and persistent only).
+* ``ierror``: Fortran only: Error status.
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Neighbor_alltoallw` is a generalized collective operation in which
+all processes send data to and receive data from all neighbors. It adds
+flexibility to :ref:`MPI_Neighbor_alltoallv` by allowing the user to specify
+the datatype of individual data blocks (in addition to displacement and
+element count). Its operation can be thought of in the following way,
+where each process performs 2n (n being the number of neighbors in the
+topology of communicator *comm*) independent point-to-point
+communications. The neighbors and buffer layout are determined by the
+topology of *comm*.
+
+.. code-block:: c
+
+           MPI_Cart_get(comm, maxdims, dims, periods, coords);
+           for (dim = 0, i = 0 ; dim < dims ; ++dim) {
+               MPI_Cart_shift(comm, dim, 1, &r0, &r1);
+               MPI_Isend(sendbuf + sdispls[i]  * extent(sendtype),
+                         sendcount, sendtypes[i], r0, ..., comm, ...);
+               MPI_Irecv(recvbuf + rdispls[i] * extent(recvtype),
+                         recvcount, recvtypes[i], r0, ..., comm, ...);
+               ++i;
+               MPI_Isend(sendbuf + sdispls[i] * extent(sendtype),
+                         sendcount, sendtypes[i], r1, ..., comm, &req[i]);
+               MPI_Irecv(recvbuf + rdispls[i] * extent(recvtype),
+                         recvcount, recvtypes[i], r1, ..., comm, ...);
+               ++i;
+           }
+
+           MPI_Wait_all (...);
+
+     	   MPI_Comm_size(comm, &n);
+   	   for (i = 0, i < n; i++)
+   	     MPI_Send(sendbuf + sdispls[i], sendcounts[i],
+   	          sendtypes[i], i, ..., comm);
+   	   for (i = 0, i < n; i++)
+	     MPI_Recv(recvbuf + rdispls[i], recvcounts[i],
+   	          recvtypes[i], i, ..., comm);
+
+Process j sends the k-th block of its local *sendbuf* to neighbor k,
+which places the data in the j-th block of its local *recvbuf*.
+
+When a pair of processes exchanges data, each may pass different element
+count and datatype arguments so long as the sender specifies the same
+amount of data to send (in bytes) as the receiver expects to receive.
+
+Note that process i may send a different amount of data to process j
+than it receives from process j. Also, a process may send entirely
+different amounts and types of data to different processes in the
+communicator.
+
+
+NOTES
+-----
+
+The MPI_IN_PLACE option for *sendbuf* is not meaningful for this
+operation
+
+The specification of counts, types, and displacements should not cause
+any location to be written more than once.
+
+All arguments on all processes are significant. The *comm* argument, in
+particular, must describe the same communicator on all processes.
+
+The offsets of *sdispls* and *rdispls* are measured in bytes. Compare
+this to :ref:`MPI_Neighbor_alltoallv`, where these offsets are measured in
+units of *sendtype* and *recvtype*, respectively.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Neighbor_alltoall`
+   * :ref:`MPI_Neighbor_alltoallv`
+   * :ref:`MPI_Cart_create`
+   * :ref:`MPI_Graph_create`
+   * :ref:`MPI_Dist_graph_create`
diff --git a/docs/man-openmpi/man3/MPI_Neighbor_alltoallw_init.3.rst b/docs/man-openmpi/man3/MPI_Neighbor_alltoallw_init.3.rst
new file mode 100644
index 00000000000..0d9c278d3fa
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Neighbor_alltoallw_init.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_neighbor_alltoallw_init:
+
+MPI_Neighbor_alltoallw_init
+===========================
+    .. include_body
+
+.. include:: ../man3/MPI_Neighbor_alltoallw.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Op_c2f.3.rst b/docs/man-openmpi/man3/MPI_Op_c2f.3.rst
new file mode 100644
index 00000000000..2381e74c661
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Op_c2f.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_op_c2f:
+
+MPI_Op_c2f
+==========
+
+.. include_body
+
+.. include:: ../man3/MPI_Comm_f2c.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_Op_commutative.3.rst b/docs/man-openmpi/man3/MPI_Op_commutative.3.rst
new file mode 100644
index 00000000000..b17607efc45
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Op_commutative.3.rst
@@ -0,0 +1,35 @@
+.. _mpi_op_commutative:
+
+
+MPI_Op_commutative
+==================
+
+.. include_body
+
+:ref:`MPI_Op_commutative` |mdash| Query of commutativity of reduction operation.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_op_commutative.rst
+
+INPUT PARAMETER
+---------------
+* ``op``: Operation (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``commute``: True if op is commutative, false otherwise (logical).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Reduction operations can be queried for their commutativity.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Op_create`
diff --git a/docs/man-openmpi/man3/MPI_Op_create.3.rst b/docs/man-openmpi/man3/MPI_Op_create.3.rst
new file mode 100644
index 00000000000..87c48e75362
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Op_create.3.rst
@@ -0,0 +1,185 @@
+.. _mpi_op_create:
+
+
+MPI_Op_create
+=============
+
+.. include_body
+
+:ref:`MPI_Op_create` |mdash| Creates a user-defined combination function handle.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_op_create.rst
+
+INPUT PARAMETERS
+----------------
+* ``function``: User-defined function (function).
+* ``commute``: True if commutative; false otherwise.
+
+OUTPUT PARAMETERS
+-----------------
+* ``op``: Operation (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Op_create` binds a user-defined global operation to an op handle that
+can subsequently be used in :ref:`MPI_Reduce`, :ref:`MPI_Allreduce`,
+:ref:`MPI_Reduce_scatter`, and :ref:`MPI_Scan`. The user-defined operation is assumed
+to be associative. If commute = true, then the operation should be both
+commutative and associative. If commute = false, then the order of
+operands is fixed and is defined to be in ascending, process rank order,
+beginning with process zero. The order of evaluation can be changed,
+taking advantage of the associativity of the operation. If commute =
+true then the order of evaluation can be changed, taking advantage of
+commutativity and associativity.
+
+*function* is the user-defined function, which must have the following
+four arguments: invec, inoutvec, len, and datatype.
+
+The ANSI-C prototype for the function is the following:
+
+.. code-block:: c
+
+     typedef void MPI_User_function(void *invec, void *inoutvec,
+                                    int *len,
+                                    MPI_Datatype *datatype);
+
+The Fortran declaration of the user-defined function appears below.
+
+.. code-block:: fortran
+
+     FUNCTION USER_FUNCTION( INVEC(*), INOUTVEC(*), LEN, TYPE)
+     <type> INVEC(LEN), INOUTVEC(LEN)
+      INTEGER LEN, TYPE
+
+The datatype argument is a handle to the data type that was passed into
+the call to :ref:`MPI_Reduce`. The user reduce function should be written such
+that the following holds: Let u[0], ..., u[len-1] be the len elements in
+the communication buffer described by the arguments invec, len, and
+datatype when the function is invoked; let v[0], ..., v[len-1] be len
+elements in the communication buffer described by the arguments
+inoutvec, len, and datatype when the function is invoked; let w[0], ...,
+w[len-1] be len elements in the communication buffer described by the
+arguments inoutvec, len, and datatype when the function returns; then
+w[i] = u[i] o v[i], for i=0 ,..., len-1, where o is the reduce operation
+that the function computes.
+
+Informally, we can think of invec and inoutvec as arrays of len elements
+that function is combining. The result of the reduction over-writes
+values in inoutvec, hence the name. Each invocation of the function
+results in the pointwise evaluation of the reduce operator on len
+elements: i.e, the function returns in inoutvec[i] the value invec[i] o
+inoutvec[i], for i = 0..., count-1, where o is the combining operation
+computed by the function.
+
+By internally comparing the value of the datatype argument to known,
+global handles, it is possible to overload the use of a single
+user-defined function for several different data types.
+
+General datatypes may be passed to the user function. However, use of
+datatypes that are not contiguous is likely to lead to inefficiencies.
+
+No MPI communication function may be called inside the user function.
+:ref:`MPI_Abort` may be called inside the function in case of an error.
+
+
+NOTES
+-----
+
+Suppose one defines a library of user-defined reduce functions that are
+overloaded: The datatype argument is used to select the right execution
+path at each invocation, according to the types of the operands. The
+user-defined reduce function cannot "decode" the datatype argument that
+it is passed, and cannot identify, by itself, the correspondence between
+the datatype handles and the datatype they represent. This
+correspondence was established when the datatypes were created. Before
+the library is used, a library initialization preamble must be executed.
+This preamble code will define the datatypes that are used by the
+library and store handles to these datatypes in global, static variables
+that are shared by the user code and the library code.
+
+**Example:** Example of user-defined reduce:
+
+Compute the product of an array of complex numbers, in C.
+
+.. code-block:: c
+
+       typedef struct {
+           double real,imag;
+       } Complex;
+
+       /* the user-defined function
+        */
+       void myProd( Complex *in, Complex *inout, int *len,
+                    MPI_Datatype *dptr )
+       {
+           int i;
+           Complex c;
+
+       for (i=0; i< *len; ++i) {
+               c.real = inout->real*in->real -
+                          inout->imag*in->imag;
+               c.imag = inout->real*in->imag +
+                          inout->imag*in->real;
+               *inout = c;
+               in++; inout++;
+           }
+       }
+
+       /* and, to call it...
+        */
+       ...
+
+       /* each process has an array of 100 Complexes
+            */
+           Complex a[100], answer[100];
+           MPI_Op myOp;
+           MPI_Datatype ctype;
+
+       /* explain to MPI how type Complex is defined
+            */
+          MPI_Type_contiguous( 2, MPI_DOUBLE, &ctype );
+           MPI_Type_commit( &ctype );
+           /* create the complex-product user-op
+            */
+           MPI_Op_create( myProd, True, &myOp );
+
+           MPI_Reduce( a, answer, 100, ctype, myOp, root, comm );
+
+           /* At this point, the answer, which consists of 100 Complexes,
+            * resides on process root
+            */
+
+The Fortran version of :ref:`MPI_Reduce` will invoke a user-defined reduce
+function using the Fortran calling conventions and will pass a
+Fortran-type datatype argument; the C version will use C calling
+convention and the C representation of a datatype handle. Users who plan
+to mix languages should define their reduction functions accordingly.
+
+
+NOTES ON COLLECTIVE OPERATIONS
+------------------------------
+
+The reduction functions ( MPI_Op ) do not return an error value. As a
+result, if the functions detect an error, all they can do is either call
+:ref:`MPI_Abort` or silently skip the problem. Thus, if you change the error
+handler from MPI_ERRORS_ARE_FATAL to something else, for example,
+MPI_ERRORS_RETURN , then no error may be indicated.
+
+The reason for this is the performance problems in ensuring that all
+collective routines return the same error value.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Reduce`
+   * :ref:`MPI_Reduce_scatter`
+   * :ref:`MPI_Allreduce`
+   * :ref:`MPI_Scan`
+   * :ref:`MPI_Op_free`
diff --git a/docs/man-openmpi/man3/MPI_Op_f2c.3.rst b/docs/man-openmpi/man3/MPI_Op_f2c.3.rst
new file mode 100644
index 00000000000..14f0b53388c
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Op_f2c.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_op_f2c:
+
+MPI_Op_f2c
+==========
+
+.. include_body
+
+.. include:: ../man3/MPI_Comm_f2c.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_Op_free.3.rst b/docs/man-openmpi/man3/MPI_Op_free.3.rst
new file mode 100644
index 00000000000..e72a6ac683a
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Op_free.3.rst
@@ -0,0 +1,39 @@
+.. _mpi_op_free:
+
+
+MPI_Op_free
+===========
+
+.. include_body
+
+:ref:`MPI_Op_free` |mdash| Frees a user-defined combination function handle.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_op_free.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``op``: Operation (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Marks a user-defined reduction operation for deallocation and sets *op*
+to MPI_OP_NULL.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Op_create`
+   * :ref:`MPI_Reduce`
+   * :ref:`MPI_Allreduce`
+   * :ref:`MPI_Reduce_scatter`
+   * :ref:`MPI_Scan`
diff --git a/docs/man-openmpi/man3/MPI_Open_port.3.rst b/docs/man-openmpi/man3/MPI_Open_port.3.rst
new file mode 100644
index 00000000000..e6084010e5e
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Open_port.3.rst
@@ -0,0 +1,50 @@
+.. _mpi_open_port:
+
+
+MPI_Open_port
+=============
+
+.. include_body
+
+:ref:`MPI_Open_port` |mdash| Establishes a network address for a server to accept
+connections from clients.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_open_port.rst
+
+INPUT PARAMETER
+---------------
+* ``info``: Options on how to establish an address (handle). No options currently supported.
+
+OUTPUT PARAMETERS
+-----------------
+* ``port_name``: Newly established port (string).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Open_port` establishes a network address, encoded in the *port_name*
+string, at which the server will be able to accept connections from
+clients. *port_name* is supplied by the system.
+
+MPI copies a system-supplied port name into *port_name*. *port_name*
+identifies the newly opened port and can be used by a client to contact
+the server. The maximum size string that may be supplied by the system
+is MPI_MAX_PORT_NAME.
+
+
+SUPPORTED INFO KEYS
+-------------------
+
+None.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Comm_accept`
+   * :ref:`MPI_Comm_connect`
diff --git a/docs/man-openmpi/man3/MPI_Pack.3.rst b/docs/man-openmpi/man3/MPI_Pack.3.rst
new file mode 100644
index 00000000000..59bda760377
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Pack.3.rst
@@ -0,0 +1,78 @@
+.. _mpi_pack:
+
+
+MPI_Pack
+========
+
+.. include_body
+
+:ref:`MPI_Pack` |mdash| Packs data of a given datatype into contiguous memory.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_pack.rst
+
+INPUT PARAMETERS
+----------------
+* ``inbuf``: Input buffer start (choice).
+* ``incount``: Number of input data items (integer).
+* ``datatype``: Datatype of each input data item (handle).
+* ``outsize``: Output buffer size, in bytes (integer).
+* ``comm``: Communicator for packed message (handle).
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``position``: Current position in buffer, in bytes (integer).
+
+OUTPUT PARAMETERS
+-----------------
+* ``outbuf``: Output buffer start (choice).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Packs the message in the send buffer specified by *inbuf*, *incount*,
+*datatype* into the buffer space specified by *outbuf* and *outsize*.
+The input buffer can be any communication buffer allowed in :ref:`MPI_Send`.
+The output buffer is a contiguous storage area containing *outsize*
+bytes, starting at the address *outbuf* (length is counted in bytes, not
+elements, as if it were a communication buffer for a message of type
+MPI_Packed).
+
+The input value of *position* is the first location in the output buffer
+to be used for packing. *position* is incremented by the size of the
+packed message, and the output value of *position* is the first location
+in the output buffer following the locations occupied by the packed
+message. The *comm* argument is the communicator that will be
+subsequently used for sending the packed message.
+
+**Example:** An example using :ref:`MPI_Pack`:
+
+.. code-block:: c
+
+   int myrank, position, i, j, a[2];
+   char buff[1000];
+
+   // ...
+
+   MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
+   if (myrank == 0) {
+     /* SENDER CODE */
+     position = 0;
+     MPI_Pack(&i, 1, MPI_INT, buff, 1000, &position, MPI_COMM_WORLD);
+     MPI_Pack(&j, 1, MPI_INT, buff, 1000, &position, MPI_COMM_WORLD);
+     MPI_Send(buff, position, MPI_PACKED, 1, 0, MPI_COMM_WORLD);
+   } else {
+     /* RECEIVER CODE */
+     MPI_Recv(a, 2, MPI_INT, 0, 0, MPI_COMM_WORLD);
+   }
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Unpack`
+   * :ref:`MPI_Pack_size`
diff --git a/docs/man-openmpi/man3/MPI_Pack_external.3.rst b/docs/man-openmpi/man3/MPI_Pack_external.3.rst
new file mode 100644
index 00000000000..89ee2fb3a7b
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Pack_external.3.rst
@@ -0,0 +1,136 @@
+.. _mpi_pack_external:
+
+
+MPI_Pack_external
+=================
+
+.. include_body
+
+:ref:`MPI_Pack_external` |mdash| Writes data to a portable format
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_pack_external.rst
+
+INPUT PARAMETERS
+----------------
+* ``datarep``: Data representation (string).
+* ``inbuf``: Input buffer start (choice).
+* ``incount``: Number of input data items (integer).
+* ``datatype``: Datatype of each input data item (handle).
+* ``outsize``: Output buffer size, in bytes (integer).
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``position``: Current position in buffer, in bytes (integer).
+
+OUTPUT PARAMETERS
+-----------------
+* ``outbuf``: Output buffer start (choice).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Pack_external` packs data into the external32 format, a universal
+data representation defined by the MPI Forum. This format is useful for
+exchanging data between MPI implementations, or when writing data to a
+file.
+
+The input buffer is specified by *inbuf*, *incount* and *datatype*, and
+may be any communication buffer allowed in :ref:`MPI_Send`. The output buffer
+*outbuf* must be a contiguous storage area containing *outsize* bytes.
+
+The input value of *position* is the first position in *outbuf* to be
+used for packing (measured in bytes, not elements, relative to the start
+of the buffer). When the function returns, *position* is incremented by
+the size of the packed message, so that it points to the first location
+in *outbuf* following the packed message. This way it may be used as
+input to a subsequent call to :ref:`MPI_Pack_external`.
+
+**Example:** An example using :ref:`MPI_Pack_external`:
+
+.. code-block:: c
+
+   	int position, i;
+   	double msg[5];
+   	char buf[1000];
+
+   	...
+
+   	MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
+   	if (myrank == 0) {	/* SENDER CODE */
+   		position = 0;
+   		i = 5; /* number of doubles in msg[] */
+   		MPI_Pack_external("external32", &i, 1, MPI_INT,
+   		    buf, 1000, &position);
+   		MPI_Pack_external("external32", &msg, i, MPI_DOUBLE,
+   		    buf, 1000, &position);
+   		MPI_Send(buf, position, MPI_BYTE, 1, 0,
+   		    MPI_COMM_WORLD);
+   	} else {		/* RECEIVER CODE */
+   		MPI_Recv(buf, 1, MPI_BYTE, 0, 0, MPI_COMM_WORLD,
+   		    MPI_STATUS_IGNORE);
+   		MPI_Unpack_external("external32", buf, 1000,
+   		    MPI_INT, &i, 1, &position);
+   		MPI_Unpack_external("external32", buf, 1000,
+   		    MPI_DOUBLE, &msg, i, &position);
+   	}
+
+
+NOTES
+-----
+
+The *datarep* argument specifies the data format. The only valid value
+in the current version of MPI is "external32". The argument is provided
+for future extensibility.
+
+To understand the behavior of pack and unpack, it is convenient to think
+of the data part of a message as being the sequence obtained by
+concatenating the successive values sent in that message. The pack
+operation stores this sequence in the buffer space, as if sending the
+message to that buffer. The unpack operation retrieves this sequence
+from buffer space, as if receiving a message from that buffer. (It is
+helpful to think of internal Fortran files or sscanf in C for a similar
+function.)
+
+Several messages can be successively packed into one packing unit. This
+is effected by several successive related calls to :ref:`MPI_Pack_external`,
+where the first call provides *position*\ =0, and each successive call
+inputs the value of *position* that was output by the previous call,
+along with the same values for *outbuf* and *outcount*. This packing
+unit now contains the equivalent information that would have been stored
+in a message by one send call with a send buffer that is the
+"concatenation" of the individual send buffers.
+
+A packing unit can be sent using type MPI_BYTE. Any point-to-point or
+collective communication function can be used to move the sequence of
+bytes that forms the packing unit from one process to another. This
+packing unit can now be received using any receive operation, with any
+datatype. (The type-matching rules are relaxed for messages sent with
+type MPI_BYTE.)
+
+A packing unit can be unpacked into several successive messages. This is
+effected by several successive related calls to :ref:`MPI_Unpack_external`,
+where the first call provides *position*\ =0, and each successive call
+inputs the value of position that was output by the previous call, and
+the same values for *inbuf* and *insize*.
+
+The concatenation of two packing units is not necessarily a packing
+unit; nor is a substring of a packing unit necessarily a packing unit.
+Thus, one cannot concatenate two packing units and then unpack the
+result as one packing unit; nor can one unpack a substring of a packing
+unit as a separate packing unit. Each packing unit that was created by a
+related sequence of pack calls must be unpacked as a unit by a sequence
+of related unpack calls.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Pack_external_size`
+   * :ref:`MPI_Send`
+   * :ref:`MPI_Unpack_external`
+   * sscanf(3C)
diff --git a/docs/man-openmpi/man3/MPI_Pack_external_size.3.rst b/docs/man-openmpi/man3/MPI_Pack_external_size.3.rst
new file mode 100644
index 00000000000..2cd650cd224
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Pack_external_size.3.rst
@@ -0,0 +1,56 @@
+.. _mpi_pack_external_size:
+
+
+MPI_Pack_external_size
+======================
+
+.. include_body
+
+:ref:`MPI_Pack_external_size` |mdash| Calculates upper bound on space needed to
+write to a portable format
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_pack_external_size.rst
+
+INPUT PARAMETERS
+----------------
+* ``datarep``: Data representation (string).
+* ``incount``: Number of input data items (integer).
+* ``datatype``: Datatype of each input data item (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``size``: Upper bound on size of packed message, in bytes (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Pack_external_size` allows the application to find out how much space
+is needed to pack a message in the portable format defined by the MPI
+Forum. It returns in *size* an upper bound on the increment in
+*position* that would occur in a call to :ref:`MPI_Pack_external` with the same
+values for *datarep*, *incount*, and *datatype*.
+
+The call returns an upper bound, rather than an exact bound, as the
+exact amount of space needed to pack the message may depend on context
+and alignment (e.g., the first message packed in a packing unit may take
+more space).
+
+
+NOTES
+-----
+
+The *datarep* argument specifies the data format. The only valid value
+in the current version of MPI is "external32". The argument is provided
+for future extensibility.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Pack_external`
+   * :ref:`MPI_Unpack_external`
diff --git a/docs/man-openmpi/man3/MPI_Pack_size.3.rst b/docs/man-openmpi/man3/MPI_Pack_size.3.rst
new file mode 100644
index 00000000000..cf224ac8eb4
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Pack_size.3.rst
@@ -0,0 +1,48 @@
+.. _mpi_pack_size:
+
+
+MPI_Pack_size
+=============
+
+.. include_body
+
+:ref:`MPI_Pack_size` |mdash| Returns the upper bound on the amount of space
+needed to pack a message.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_pack_size.rst
+
+INPUT PARAMETERS
+----------------
+* ``incount``: Count argument to packing call (integer).
+* ``datatype``: Datatype argument to packing call (handle).
+* ``comm``: Communicator argument to packing call (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``size``: Upper bound on size of packed message, in bytes (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Pack_size` allows the application to find out how much space is
+needed to pack a message. A call to MPI_Pack_size(incount, datatype,
+comm, size) returns in size an upper bound on the increment in position
+that would occur in a call to :ref:`MPI_Pack`, with the same values for
+*incount*, *datatype*, and *comm*.
+
+**Rationale:** The call returns an upper bound, rather than an exact
+bound, since the exact amount of space needed to pack the message may
+depend on the context (e.g., first message packed in a packing unit may
+take more space).
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Pack`
+   * :ref:`MPI_Unpack`
diff --git a/docs/man-openmpi/man3/MPI_Parrived.3.rst b/docs/man-openmpi/man3/MPI_Parrived.3.rst
new file mode 100644
index 00000000000..b834a1b2f1f
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Parrived.3.rst
@@ -0,0 +1,33 @@
+.. _mpi_parrived:
+
+
+MPI_Parrived
+============
+
+.. include_body
+
+:ref:`MPI_Parrived` |mdash| Tests for completion of a specified receive-side
+partition.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_parrived.rst
+
+INPUT PARAMETERS
+----------------
+* ``request``: Communication request (handle).
+* ``partition``: The number of the partition to test for completion (integer).
+
+OUTPUT PARAMETERS
+-----------------
+* ``flag``: True if partition is completed.
+* ``ierror``: Fortran only: Error status (integer).
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Pready_list`
+   * :ref:`MPI_Pready_range`
+   * :ref:`MPI_Parrived`
diff --git a/docs/man-openmpi/man3/MPI_Pcontrol.3.rst b/docs/man-openmpi/man3/MPI_Pcontrol.3.rst
new file mode 100644
index 00000000000..1e02a6246c6
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Pcontrol.3.rst
@@ -0,0 +1,62 @@
+.. _mpi_pcontrol:
+
+
+MPI_Pcontrol
+============
+
+.. include_body
+
+:ref:`MPI_Pcontrol` |mdash| Controls profiling.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_pcontrol.rst
+
+INPUT PARAMETER
+---------------
+* ``level``: Profiling level.
+
+DESCRIPTION
+-----------
+
+MPI libraries themselves make no use of this routine; they simply return
+immediately to the user code. However the presence of calls to this
+routine allows a profiling package to be explicitly called by the user.
+
+Since MPI has no control of the implementation of the profiling code, we
+are unable to specify precisely the semantics that will be provided by
+calls to :ref:`MPI_Pcontrol`. This vagueness extends to the number of arguments
+to the function, and their datatypes.
+
+However to provide some level of portability of user codes to different
+profiling libraries, we request the following meanings for certain
+values of level:
+
+ o    level==0 Profiling is disabled.
+
+ o    level==1 Profiling is enabled at a normal default level of detail.
+
+ o    level==2 Profile buffers are flushed. (This may be a no-op in some profilers).
+
+ o    All other values of level have profile library-defined effects and additional arguments.
+
+We also request that the default state after :ref:`MPI_Init` has been called is
+for profiling to be enabled at the normal default level (i.e., as if
+:ref:`MPI_Pcontrol` had just been called with the argument 1). This allows
+users to link with a profiling library and obtain profile output without
+having to modify their source code at all.
+
+The provision of :ref:`MPI_Pcontrol` as a no-op in the standard MPI library
+allows users to modify their source code to obtain more detailed
+profiling information, but still be able to link exactly the same code
+against the standard MPI library.
+
+
+NOTES
+-----
+
+This routine provides a common interface for profiling control. The
+interpretation of level and any other arguments is left to the profiling
+library.
+
+This function does not return an error value. Consequently, the result
+of calling it before :ref:`MPI_Init` or after :ref:`MPI_Finalize` is undefined.
diff --git a/docs/man-openmpi/man3/MPI_Pready.3.rst b/docs/man-openmpi/man3/MPI_Pready.3.rst
new file mode 100644
index 00000000000..7ae81905847
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Pready.3.rst
@@ -0,0 +1,32 @@
+.. _mpi_pready:
+
+
+MPI_Pready
+==========
+
+.. include_body
+
+:ref:`MPI_Pready` |mdash| Indicates that a given send-side partition is ready to
+be transferred.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_pready.rst
+
+INPUT PARAMETERS
+----------------
+* ``partition``: The number of the partition to mark ready for transfer (integer).
+* ``request``: Communication request (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Pready_list`
+   * :ref:`MPI_Pready_range`
+   * :ref:`MPI_Parrived`
diff --git a/docs/man-openmpi/man3/MPI_Pready_list.3.rst b/docs/man-openmpi/man3/MPI_Pready_list.3.rst
new file mode 100644
index 00000000000..f1801974f08
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Pready_list.3.rst
@@ -0,0 +1,33 @@
+.. _mpi_pready_list:
+
+
+MPI_Pready_list
+===============
+
+.. include_body
+
+:ref:`MPI_Pready_list` |mdash| Indicates that a list given send-side partitions
+are ready to be transferred.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_pready_list.rst
+
+INPUT PARAMETERS
+----------------
+* ``length``: The length of the given partition array (integer).
+* ``partitions``: An array of numbers of partitions to mark ready for transfer (integer).
+* ``request``: Communication request (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Pready`
+   * :ref:`MPI_Pready_range`
+   * :ref:`MPI_Parrived`
diff --git a/docs/man-openmpi/man3/MPI_Pready_range.3.rst b/docs/man-openmpi/man3/MPI_Pready_range.3.rst
new file mode 100644
index 00000000000..05771648744
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Pready_range.3.rst
@@ -0,0 +1,33 @@
+.. _mpi_pready_range:
+
+
+MPI_Pready_range
+================
+
+.. include_body
+
+:ref:`MPI_Pready_range` |mdash| Indicates that a given range os send-side
+partitions are ready to be transferred.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_pready_range.rst
+
+INPUT PARAMETERS
+----------------
+* ``partition_low``: The lowest of the range of partitions to mark ready for transfer (integer).
+* ``partition_high``: The highest of the range of partitions to mark ready for transfer (integer).
+* ``request``: Communication request (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Pready`
+   * :ref:`MPI_Pready_list`
+   * :ref:`MPI_Parrived`
diff --git a/docs/man-openmpi/man3/MPI_Precv_init.3.rst b/docs/man-openmpi/man3/MPI_Precv_init.3.rst
new file mode 100644
index 00000000000..808fac4986f
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Precv_init.3.rst
@@ -0,0 +1,46 @@
+.. _mpi_precv_init:
+
+
+MPI_Precv_init
+==============
+
+.. include_body
+
+:ref:`MPI_Precv_init` |mdash| Initializes a partitioned receive.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_precv_init.rst
+
+INPUT PARAMETERS
+----------------
+* ``buf``: Initial address of receive buffer (choice).
+* ``partitions``: Number of partitions (integer).
+* ``count``: Number of elements to be received per partition (integer).
+* ``datatype``: Datatype of each element (handle).
+* ``source``: Rank of source (integer).
+* ``tag``: Message tag (integer).
+* ``comm``: Communicator (handle).
+* ``info``: Info argument (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``request``: Communication request (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+NOTE
+----
+
+The current implementation is an early prototype and is not fully
+compliant with the MPI-4.0 specification. Specifically this function and
+it's counterpart (MPI_Psend_init) will block until the partitioned
+communication request is initialized on both ends. This behavior will be
+corrected in future versions.
+
+
+.. seealso::
+   * :ref:`MPI_Psend_init`
diff --git a/docs/man-openmpi/man3/MPI_Probe.3.rst b/docs/man-openmpi/man3/MPI_Probe.3.rst
new file mode 100644
index 00000000000..deb25e40179
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Probe.3.rst
@@ -0,0 +1,122 @@
+.. _mpi_probe:
+
+
+MPI_Probe
+=========
+
+.. include_body
+
+:ref:`MPI_Probe` |mdash| Blocking test for a message.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_probe.rst
+
+INPUT PARAMETERS
+----------------
+* ``source``: Source rank or ``MPI_ANY_SOURCE`` (integer).
+* ``tag``: Tag value or ``MPI_ANY_TAG`` (integer).
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``status``: Status object (status).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The :ref:`MPI_Probe` and :ref:`MPI_Iprobe` operations allow checking of incoming
+messages, without actual receipt of them. The user can then decide how
+to receive them, based on the information returned by the probe in the
+status variable. For example, the user may allocate memory for the
+receive buffer, according to the length of the probed message.
+
+:ref:`MPI_Probe` behaves like :ref:`MPI_Iprobe` except that it is a blocking call that
+returns only after a matching message has been found.
+
+If your application does not need to examine the *status* field, you can
+save resources by using the predefined constant ``MPI_STATUS_IGNORE`` as a
+special value for the *status* argument.
+
+The semantics of :ref:`MPI_Probe` and :ref:`MPI_Iprobe` guarantee progress: If a call
+to :ref:`MPI_Probe` has been issued by a process, and a send that matches the
+probe has been initiated by some process, then the call to :ref:`MPI_Probe`
+will return, unless the message is received by another concurrent
+receive operation (that is executed by another thread at the probing
+process). Similarly, if a process busy waits with :ref:`MPI_Iprobe` and a
+matching message has been issued, then the call to :ref:`MPI_Iprobe` will
+eventually return *flag* = true unless the message is received by another
+concurrent receive operation.
+
+**Example 1:** Use blocking probe to wait for an incoming message.
+
+.. code-block:: fortran
+
+   CALL MPI_COMM_RANK(comm, rank, ierr)
+   IF (rank == 0) THEN
+      CALL MPI_SEND(i, 1, MPI_INTEGER, 2, 0, comm, ierr)
+   ELSE IF(rank == 1) THEN
+      CALL MPI_SEND(x, 1, MPI_REAL, 2, 0, comm, ierr)
+   ELSE   ! rank == 2
+      DO i=1, 2
+         CALL MPI_PROBE(MPI_ANY_SOURCE, 0,
+                        comm, status, ierr)
+	 IF (status(MPI_SOURCE) = 0) THEN
+	    CALL MPI_RECV(i, 1, MPI_INTEGER, 0, 0, status, ierr)
+         ELSE
+	    CALL MPI_RECV(x, 1, MPI_REAL, 1, 0, status, ierr)
+         END IF
+      END DO
+   END IF
+
+Each message is received with the right type.
+
+**Example 2:** A program similar to the previous example, but with a
+problem.
+
+.. code-block:: fortran
+
+   CALL MPI_COMM_RANK(comm, rank, ierr)
+   IF (rank == 0) THEN
+      CALL MPI_SEND(i, 1, MPI_INTEGER, 2, 0, comm, ierr)
+   ELSE IF(rank == 1) THEN
+      CALL MPI_SEND(x, 1, MPI_REAL, 2, 0, comm, ierr)
+   ELSE
+      DO i=1, 2
+         CALL MPI_PROBE(MPI_ANY_SOURCE, 0,
+                        comm, status, ierr)
+	 IF (status(MPI_SOURCE) == 0) THEN
+	    CALL MPI_RECV(i, 1, MPI_INTEGER, MPI_ANY_SOURCE, &
+                          0, status, ierr)
+	 ELSE
+	    CALL MPI_RECV(x, 1, MPI_REAL, MPI_ANY_SOURCE, &
+                          0, status, ierr)
+	 END IF
+      END DO
+   END IF
+
+We slightly modified Example 2, using ``MPI_ANY_SOURCE`` as the source
+argument in the two receive calls in statements labeled 100 and 200. The
+program is now incorrect: The receive operation may receive a message
+that is distinct from the message probed by the preceding call to
+:ref:`MPI_Probe`.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+Note that per the "Return Status" section in the "Point-to-Point
+Communication" chapter in the `MPI Standard
+<https://www.mpi-forum.org/docs/>`_, MPI errors on messages queried
+by :ref:`MPI_Probe` do not set the ``status.MPI_ERROR`` field in the
+returned *status*.  The error code is always passed to the back-end
+error handler and may be passed back to the caller through the return
+value of :ref:`MPI_Probe` if the back-end error handler returns it.
+The pre-defined MPI error handler ``MPI_ERRORS_RETURN`` exhibits this
+behavior, for example.
+
+.. seealso::
+   * :ref:`MPI_Iprobe`
+   * :ref:`MPI_Cancel`
diff --git a/docs/man-openmpi/man3/MPI_Psend_init.3.rst b/docs/man-openmpi/man3/MPI_Psend_init.3.rst
new file mode 100644
index 00000000000..a3b4ccd02b9
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Psend_init.3.rst
@@ -0,0 +1,46 @@
+.. _mpi_psend_init:
+
+
+MPI_Psend_init
+==============
+
+.. include_body
+
+:ref:`MPI_Psend_init` |mdash| Initializes a partitioned send.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_psend_init.rst
+
+INPUT PARAMETERS
+----------------
+* ``buf``: Initial address of send buffer (choice).
+* ``partitions``: Number of partitions (integer).
+* ``count``: Number of elements to be sent per partition (integer).
+* ``datatype``: Datatype of each element (handle).
+* ``dest``: Rank of source (integer).
+* ``tag``: Message tag (integer).
+* ``comm``: Communicator (handle).
+* ``info``: Info argument (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``request``: Communication request (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+NOTE
+----
+
+The current implementation is an early prototype and is not fully
+compliant with the MPI-4.0 specification. Specifically this function and
+it's counterpart (MPI_Precv_init) will block until the partitioned
+communication request is initialized on both ends. This behavior will be
+corrected in future versions.
+
+
+.. seealso::
+   * :ref:`MPI_Precv_init`
diff --git a/docs/man-openmpi/man3/MPI_Publish_name.3.rst b/docs/man-openmpi/man3/MPI_Publish_name.3.rst
new file mode 100644
index 00000000000..bb39e78ac64
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Publish_name.3.rst
@@ -0,0 +1,122 @@
+.. _mpi_publish_name:
+
+
+MPI_Publish_name
+================
+
+.. include_body
+
+:ref:`MPI_Publish_name` |mdash| Publishes a service name associated with a port
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_publish_name.rst
+
+INPUT PARAMETERS
+----------------
+* ``service_name``: A service name (string).
+* ``info``: Options to the name service functions (handle).
+* ``port_name``: A port name (string).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This routine publishes the pair (*service_name, port_name*) so that an
+application may retrieve *port_name* by calling :ref:`MPI_Lookup_name` with
+*service_name* as an argument. It is an error to publish the same
+*service_name* twice, or to use a *port_name* argument that was not
+previously opened by the calling process via a call to :ref:`MPI_Open_port`.
+
+
+INFO ARGUMENTS
+--------------
+
+The following keys for *info* are recognized:
+
+::
+
+   Key                   Type      Description
+   ---                   ----      -----------
+
+   ompi_global_scope     bool      If set to true, publish the name in
+                                   the global scope.  Publish in the local
+                                   scope otherwise.  See the NAME SCOPE
+                                   section for more details.
+
+   ompi_unique           bool      If set to true, return an error if the
+                                   specified service_name already exists.
+                                   Default to overwriting any pre-existing
+                                   value.
+
+*bool* info keys are actually strings but are evaluated as follows: if
+the string value is a number, it is converted to an integer and cast to
+a boolean (meaning that zero integers are false and non-zero values are
+true). If the string value is (case-insensitive) "yes" or "true", the
+boolean is true. If the string value is (case-insensitive) "no" or
+"false", the boolean is false. All other string values are unrecognized,
+and therefore false.
+
+If no info key is provided, the function will first check to see if a
+global server has been specified and is available. If so, then the
+publish function will default to global scope first, followed by local.
+Otherwise, the data will default to publish with local scope.
+
+
+NAME SCOPE
+----------
+
+Open MPI supports two name scopes: *global* and *local*. Local scope
+will place the specified service/port pair in a data store located on
+the mpirun of the calling process' job. Thus, data published with local
+scope will only be accessible to processes in jobs spawned by that
+mpirun - e.g., processes in the calling process' job, or in jobs spawned
+via :ref:`MPI_Comm_spawn`.
+
+Global scope places the specified service/port pair in a data store
+located on a central server that is accessible to all jobs running in
+the cluster or environment. Thus, data published with global scope can
+be accessed by multiple mpiruns and used for :ref:`MPI_Comm_Connect` and
+:ref:`MPI_Comm_accept` between jobs.
+
+Note that global scope operations require both the presence of the
+central server and that the calling process be able to communicate to
+that server. :ref:`MPI_Publish_name` will return an error if global scope is
+specified and a global server is either not specified or cannot be
+found.
+
+Open MPI provides a server called *ompi-server* to support global scope
+operations. Please refer to its manual page for a more detailed
+description of data store/lookup operations.
+
+As an example of the impact of these scoping rules, consider the case
+where a job has been started with mpirun - call this job "job1". A
+process in job1 creates and publishes a service/port pair using a local
+scope. Open MPI will store this data in the data store within mpirun.
+
+A process in job1 (perhaps the same as did the publish, or perhaps some
+other process in the job) subsequently calls :ref:`MPI_Comm_spawn` to start
+another job (call it "job2") under this mpirun. Since the two jobs share
+a common mpirun, both jobs have access to local scope data. Hence, a
+process in job2 can perform an :ref:`MPI_Lookup_name` with a local scope to
+retrieve the information.
+
+However, assume another user starts a job using mpirun - call this job
+"job3". Because the service/port data published by job1 specified local
+scope, processes in job3 cannot access that data. In contrast, if the
+data had been published using global scope, then any process in job3
+could access the data, provided that mpirun was given knowledge of how
+to contact the central server and the process could establish
+communication with it.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Lookup_name`
+   * :ref:`MPI_Open_port`
diff --git a/docs/man-openmpi/man3/MPI_Put.3.rst b/docs/man-openmpi/man3/MPI_Put.3.rst
new file mode 100644
index 00000000000..758f430863e
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Put.3.rst
@@ -0,0 +1,114 @@
+.. _mpi_put:
+
+
+MPI_Put
+=======
+
+.. include_body
+
+:ref:`MPI_Put`, :ref:`MPI_Rput` - Copies data from the origin memory to the
+target.
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Put, MPI_Rput
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_put.rst
+
+INPUT PARAMETERS
+----------------
+* ``origin_addr``: Initial address of origin buffer (choice).
+* ``origin_count``: Number of entries in origin buffer (nonnegative integer).
+* ``origin_datatype``: Data type of each entry in origin buffer (handle).
+* ``target_rank``: Rank of target (nonnegative integer).
+* ``target_disp``: Displacement from start of window to target buffer (nonnegative integer).
+* ``target_count``: Number of entries in target buffer (nonnegative integer).
+* ``target_datatype``: Data type of each entry in target buffer (handle).
+* ``win``: Window object used for communication (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``request``: MPI_Rput: RMA request
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Put` transfers *origin_count* successive entries of the type
+specified by *origin_datatype*, starting at address *origin_addr* on the
+origin node to the target node specified by the *win*, *target_rank*
+
+pair. The data are written in the target buffer at address *target_addr*
+^ *window_base* + *target_disp* x *disp_unit*, where *window_base* and
+*disp_unit* are the base address and window displacement unit specified
+at window initialization, by the target process.
+
+The target buffer is specified by the arguments *target_count* and
+*target_datatype*.
+
+The data transfer is the same as that which would occur if the origin
+process executed a send operation with arguments *origin_addr*,
+*origin_count*, *origin_datatype*, *target_rank*, *tag*, *comm*, and the
+target process executed a receive operation with arguments
+*target_addr*, *target_count*, *target_datatype*, *source*, *tag*,
+*comm*, where *target_addr* is the target buffer address computed as
+explained above, and *comm* is a communicator for the group of *win*.
+
+The communication must satisfy the same constraints as for a similar
+message-passing communication. The *target_datatype* may not specify
+overlapping entries in the target buffer. The message sent must fit,
+without truncation, in the target buffer. Furthermore, the target buffer
+must fit in the target window. In addition, only processes within the
+same buffer can access the target window.
+
+The *target_datatype* argument is a handle to a datatype object defined
+at the origin process. However, this object is interpreted at the target
+process: The outcome is as if the target datatype object were defined at
+the target process, by the same sequence of calls used to define it at
+the origin process. The target data type must contain only relative
+displacements, not absolute addresses. The same holds for get and
+accumulate.
+
+:ref:`MPI_Rput` is similar to :ref:`MPI_Put`, except that it allocates a
+communication request object and associates it with the request handle
+(the argument *request*). The completion of an :ref:`MPI_Rput` operation (i.e.,
+after the corresponding test or wait) indicates that the sender is now
+free to update the locations in the *origin_addr* buffer. It does not
+indicate that the data is available at the target window. If remote
+completion is required, :ref:`MPI_Win_flush`, :ref:`MPI_Win_flush_all`,
+:ref:`MPI_Win_unlock`, or :ref:`MPI_Win_unlock_all` can be used.
+
+
+NOTES
+-----
+
+The *target_datatype* argument is a handle to a datatype object that is
+defined at the origin process, even though it defines a data layout in
+the target process memory. This does not cause problems in a homogeneous
+or heterogeneous environment, as long as only portable data types are
+used (portable data types are defined in Section 2.4 of the MPI-2
+Standard).
+
+The performance of a put transfer can be significantly affected, on some
+systems, from the choice of window location and the shape and location
+of the origin and target buffer: Transfers to a target window in memory
+allocated by :ref:`MPI_Alloc_mem` may be much faster on shared memory systems;
+transfers from contiguous buffers will be faster on most, if not all,
+systems; the alignment of the communication buffers may also impact
+performance.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Get`
+   * :ref:`MPI_Rget`
+   * :ref:`MPI_Accumulate`
+   * :ref:`MPI_Win_flush`
+   * :ref:`MPI_Win_flush_all`
+   * :ref:`MPI_Win_unlock`
+   * :ref:`MPI_Win_unlock_all`
diff --git a/docs/man-openmpi/man3/MPI_Query_thread.3.rst b/docs/man-openmpi/man3/MPI_Query_thread.3.rst
new file mode 100644
index 00000000000..855af907a7f
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Query_thread.3.rst
@@ -0,0 +1,59 @@
+.. _mpi_query_thread:
+
+
+MPI_Query_thread
+================
+
+.. include_body
+
+:ref:`MPI_Query_thread` |mdash| Returns the current level of thread support
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_query_thread.rst
+
+OUTPUT PARAMETERS
+-----------------
+* ``provided``: C/Fortran only: Level of thread support (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This routine returns in *provided* the current level of thread support.
+If MPI was initialized by a call to :ref:`MPI_Init_thread`, *provided* will
+have the same value as was returned by that function.
+
+The possible values of *provided* are as follows:
+
+MPI_THREAD_SINGLE
+   Only one thread may execute.
+
+MPI_THREAD_FUNNELED
+   If the process is multithreaded, only the thread that called
+   MPI_Init[_thread] may make MPI calls.
+
+MPI_THREAD_SERIALIZED
+   If the process is multithreaded, only one thread may make MPI library
+   calls at one time.
+
+MPI_THREAD_MULTIPLE
+   If the process is multithreaded, multiple threads may call MPI at
+   once with no restrictions.
+
+
+NOTES
+-----
+
+In Open MPI, *provided* is always MPI_THREAD_SINGLE, unless the program
+has been linked with the multithreaded library, in which case *provided*
+is MPI_THREAD_MULTIPLE.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Init`
+   * :ref:`MPI_Init_thread`
diff --git a/docs/man-openmpi/man3/MPI_Raccumulate.3.rst b/docs/man-openmpi/man3/MPI_Raccumulate.3.rst
new file mode 100644
index 00000000000..c1c846e018f
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Raccumulate.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_raccumulate:
+
+MPI_Raccumulate
+===============
+
+.. include_body
+
+.. include:: ../man3/MPI_Accumulate.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_Recv.3.rst b/docs/man-openmpi/man3/MPI_Recv.3.rst
new file mode 100644
index 00000000000..14c8c8cdcf3
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Recv.3.rst
@@ -0,0 +1,123 @@
+.. _mpi_recv:
+
+
+MPI_Recv
+========
+
+.. include_body
+
+:ref:`MPI_Recv` |mdash| Performs a standard-mode blocking receive.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_recv.rst
+
+INPUT PARAMETERS
+----------------
+* ``count``: Maximum number of elements to receive (integer).
+* ``datatype``: Datatype of each receive buffer entry (handle).
+* ``source``: Rank of source (integer).
+* ``tag``: Message tag (integer).
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``buf``: Initial address of receive buffer (choice).
+* ``status``: Status object (status).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This basic receive operation, :ref:`MPI_Recv`, is blocking: it returns only
+after the receive buffer contains the newly received message. A receive
+can complete before the matching send has completed (of course, it can
+complete only after the matching send has started).
+
+The blocking semantics of this call are described in the
+"Communication Modes" section of the `MPI Standard
+<https://www.mpi-forum.org/docs/>`_.
+
+The receive buffer contains a number (defined by the value of *count*)
+of consecutive elements. The first element in the set of elements is
+located at *address_buf*. The type of each of these elements is
+specified by *datatype*.
+
+The length of the received message must be less than or equal to the
+length of the receive buffer. An MPI_ERR_TRUNCATE is returned upon the
+overflow condition.
+
+If a message that is shorter than the length of the receive buffer
+arrives, then only those locations corresponding to the (shorter)
+received message are modified.
+
+
+NOTES
+-----
+
+The *count* argument indicates the maximum number of entries of type
+*datatype* that can be received in a message. Once a message is
+received, use the :ref:`MPI_Get_count` function to determine the actual number
+of entries within that message.
+
+To receive messages of unknown length, use the :ref:`MPI_Probe`
+function.  For more information about :ref:`MPI_Probe` and
+:ref:`MPI_Cancel`, see their respective man pages and the "Probe and
+Cancel" section of the `MPI Standard
+<https://www.mpi-forum.org/docs/>`_.
+
+A message can be received by a receive operation only if it is addressed
+to the receiving process, and if its source, tag, and communicator
+(comm) values match the source, tag, and comm values specified by the
+receive operation. The receive operation may specify a wildcard value
+for source and/or tag, indicating that any source and/or tag are
+acceptable. The wildcard value for source is source = ``MPI_ANY_SOURCE``.
+The wildcard value for tag is tag = ``MPI_ANY_TAG``. There is no wildcard
+value for comm. The scope of these wildcards is limited to the processes
+in the group of the specified communicator.
+
+The message tag is specified by the tag argument of the receive
+operation.
+
+The argument source, if different from ``MPI_ANY_SOURCE``, is specified as a
+rank within the process group associated with that same communicator
+(remote process group, for intercommunicators). Thus, the range of valid
+values for the source argument is {0,...,n-1} {``MPI_ANY_SOURCE``}, where n
+is the number of processes in this group.
+
+Note the asymmetry between send and receive operations: A receive
+operation may accept messages from an arbitrary sender; on the other
+hand, a send operation must specify a unique receiver. This matches a
+"push" communication mechanism, where data transfer is effected by the
+sender (rather than a "pull" mechanism, where data transfer is effected
+by the receiver).
+
+Source = destination is allowed, that is, a process can send a message
+to itself. However, it is not recommended for a process to send messages
+to itself using the blocking send and receive operations described
+above, since this may lead to deadlock.  See the "Semantics of
+Point-to-Point Communication" of the `MPI Standard
+<https://www.mpi-forum.org/docs/>`_ for more details.
+
+If your application does not need to examine the *status* field, you can
+save resources by using the predefined constant ``MPI_STATUS_IGNORE`` as a
+special value for the *status* argument.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+Note that per the "Return Status" section in the "Point-to-Point
+Communication" chapter in the `MPI Standard
+<https://www.mpi-forum.org/docs/>`_, MPI errors on messages received
+by :ref:`MPI_Recv` do not set the ``status.MPI_ERROR`` field in the
+returned *status*.  The error code is always passed to the back-end
+error handler and may be passed back to the caller through the return
+value of :ref:`MPI_Recv` if the back-end error handler returns it.
+The pre-defined MPI error handler ``MPI_ERRORS_RETURN`` exhibits this
+behavior, for example.
+
+.. seealso::
+   * :ref:`MPI_Irecv`
+   * :ref:`MPI_Probe`
diff --git a/docs/man-openmpi/man3/MPI_Recv_init.3.rst b/docs/man-openmpi/man3/MPI_Recv_init.3.rst
new file mode 100644
index 00000000000..7b9f7bee01d
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Recv_init.3.rst
@@ -0,0 +1,57 @@
+.. _mpi_recv_init:
+
+
+MPI_Recv_init
+=============
+
+.. include_body
+
+:ref:`MPI_Recv_init` |mdash| Builds a handle for a receive.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_recv_init.rst
+
+INPUT PARAMETERS
+----------------
+* ``count``: Maximum number of elements to receive (integer).
+* ``datatype``: Type of each entry (handle).
+* ``source``: Rank of source (integer).
+* ``tag``: Message tag (integer).
+* ``comm``: Communicator (handle).
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``buf``: Initial address of receive buffer (choice).
+
+OUTPUT PARAMETERS
+-----------------
+* ``request``: Communication request (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Creates a persistent communication request for a receive operation. The
+argument *buf* is marked as OUT because the user gives permission to
+write on the receive buffer by passing the argument to :ref:`MPI_Recv_init`.
+
+A persistent communication request is inactive after it is created |mdash| no
+active communication is attached to the request.
+
+A communication (send or receive) that uses a persistent request is
+initiated by the function :ref:`MPI_Start` or :ref:`MPI_Startall`.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Bsend_init`
+   * :ref:`MPI_Rsend_init`
+   * :ref:`MPI_Send_init`
+   * MPI_Sssend_init
+   * :ref:`MPI_Start`
+   * :ref:`MPI_Startall`
+   * :ref:`MPI_Request_free`
diff --git a/docs/man-openmpi/man3/MPI_Reduce.3.rst b/docs/man-openmpi/man3/MPI_Reduce.3.rst
new file mode 100644
index 00000000000..7b7a19f4ed9
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Reduce.3.rst
@@ -0,0 +1,480 @@
+.. _mpi_reduce:
+
+
+MPI_Reduce
+==========
+
+.. include_body
+
+:ref:`MPI_Reduce`, :ref:`MPI_Ireduce`, :ref:`MPI_Reduce_init` - Reduces values on all
+processes within a group.
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Reduce, MPI_Ireduce, MPI_Reduce_init
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_reduce.rst
+
+INPUT PARAMETERS
+----------------
+* ``sendbuf``: Address of send buffer (choice).
+* ``count``: Number of elements in send buffer (integer).
+* ``datatype``: Data type of elements of send buffer (handle).
+* ``op``: Reduce operation (handle).
+* ``root``: Rank of root process (integer).
+* ``comm``: Communicator (handle).
+* ``info``: Info (handle, persistent only).
+
+OUTPUT PARAMETERS
+-----------------
+* ``recvbuf``: Address of receive buffer (choice, significant only at root).
+* ``request``: Request (handle, non-blocking and persistent only).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The global reduce functions (:ref:`MPI_Reduce`, :ref:`MPI_Op_create`, :ref:`MPI_Op_free`,
+:ref:`MPI_Allreduce`, :ref:`MPI_Reduce_scatter`, MPI_Scan) perform a global reduce
+operation (such as sum, max, logical AND, etc.) across all the members
+of a group. The reduction operation can be either one of a predefined
+list of operations, or a user-defined operation. The global reduction
+functions come in several flavors: a reduce that returns the result of
+the reduction at one node, an all-reduce that returns this result at all
+nodes, and a scan (parallel prefix) operation. In addition, a
+reduce-scatter operation combines the functionality of a reduce and a
+scatter operation.
+
+:ref:`MPI_Reduce` combines the elements provided in the input buffer of each
+process in the group, using the operation op, and returns the combined
+value in the output buffer of the process with rank root. The input
+buffer is defined by the arguments sendbuf, count, and datatype; the
+output buffer is defined by the arguments recvbuf, count, and datatype;
+both have the same number of elements, with the same type. The routine
+is called by all group members using the same arguments for count,
+datatype, op, root, and comm. Thus, all processes provide input buffers
+and output buffers of the same length, with elements of the same type.
+Each process can provide one element, or a sequence of elements, in
+which case the combine operation is executed element-wise on each entry
+of the sequence. For example, if the operation is MPI_MAX and the send
+buffer contains two elements that are floating-point numbers (count = 2
+and datatype = MPI_FLOAT), then recvbuf(1) = global max (sendbuf(1)) and
+recvbuf(2) = global max(sendbuf(2)).
+
+
+USE OF IN-PLACE OPTION
+----------------------
+
+When the communicator is an intracommunicator, you can perform a reduce
+operation in-place (the output buffer is used as the input buffer). Use
+the variable MPI_IN_PLACE as the value of the root process *sendbuf*. In
+this case, the input data is taken at the root from the receive buffer,
+where it will be replaced by the output data.
+
+Note that MPI_IN_PLACE is a special kind of value; it has the same
+restrictions on its use as MPI_BOTTOM.
+
+Because the in-place option converts the receive buffer into a
+send-and-receive buffer, a Fortran binding that includes INTENT must
+mark these as INOUT, not OUT.
+
+
+WHEN COMMUNICATOR IS AN INTER-COMMUNICATOR
+------------------------------------------
+
+When the communicator is an inter-communicator, the root process in the
+first group combines data from all the processes in the second group and
+then performs the *op* operation. The first group defines the root
+process. That process uses MPI_ROOT as the value of its *root* argument.
+The remaining processes use ``MPI_PROC_NULL`` as the value of their *root*
+argument. All processes in the second group use the rank of that root
+process in the first group as the value of their *root* argument. Only
+the send buffer arguments are significant in the second group, and only
+the receive buffer arguments are significant in the root process of the
+first group.
+
+
+PREDEFINED REDUCE OPERATIONS
+----------------------------
+
+The set of predefined operations provided by MPI is listed below
+(Predefined Reduce Operations). That section also enumerates the
+datatypes each operation can be applied to. In addition, users may
+define their own operations that can be overloaded to operate on several
+datatypes, either basic or derived. This is further explained in the
+description of the user-defined operations (see the man pages for
+:ref:`MPI_Op_create` and MPI_Op_free).
+
+The operation op is always assumed to be associative. All predefined
+operations are also assumed to be commutative. Users may define
+operations that are assumed to be associative, but not commutative. The
+\``canonical'' evaluation order of a reduction is determined by the
+ranks of the processes in the group. However, the implementation can
+take advantage of associativity, or associativity and commutativity, in
+order to change the order of evaluation. This may change the result of
+the reduction for operations that are not strictly associative and
+commutative, such as floating point addition.
+
+Predefined operators work only with the MPI types listed below
+(Predefined Reduce Operations, and the section MINLOC and MAXLOC,
+below). User-defined operators may operate on general, derived
+datatypes. In this case, each argument that the reduce operation is
+applied to is one element described by such a datatype, which may
+contain several basic values. This is further explained in Section 4.9.4
+of the MPI Standard, "User-Defined Operations."
+
+The following predefined operations are supplied for :ref:`MPI_Reduce` and
+related functions :ref:`MPI_Allreduce`, :ref:`MPI_Reduce_scatter`, and :ref:`MPI_Scan`. These
+operations are invoked by placing the following in op:
+
+::
+
+   	Name                Meaning
+        ---------           --------------------
+   	MPI_MAX             maximum
+   	MPI_MIN             minimum
+   	MPI_SUM             sum
+   	MPI_PROD            product
+   	MPI_LAND            logical and
+   	MPI_BAND            bit-wise and
+   	MPI_LOR             logical or
+   	MPI_BOR             bit-wise or
+   	MPI_LXOR            logical xor
+   	MPI_BXOR            bit-wise xor
+   	MPI_MAXLOC          max value and location
+   	MPI_MINLOC          min value and location
+
+The two operations MPI_MINLOC and MPI_MAXLOC are discussed separately
+below (MINLOC and MAXLOC). For the other predefined operations, we
+enumerate below the allowed combinations of op and datatype arguments.
+First, define groups of MPI basic datatypes in the following way:
+
+.. code-block:: c
+
+   	C integer:            MPI_INT, MPI_LONG, MPI_SHORT,
+   	                      MPI_UNSIGNED_SHORT, MPI_UNSIGNED,
+   	                      MPI_UNSIGNED_LONG
+   	Fortran integer:      MPI_INTEGER
+   	Floating-point:       MPI_FLOAT, MPI_DOUBLE, MPI_REAL,
+   	                      MPI_DOUBLE_PRECISION, MPI_LONG_DOUBLE
+   	Logical:              MPI_LOGICAL
+   	Complex:              MPI_COMPLEX
+   	Byte:                 MPI_BYTE
+
+Now, the valid datatypes for each option is specified below.
+
+::
+
+   	Op                      	Allowed Types
+        ----------------         ---------------------------
+   	MPI_MAX, MPI_MIN		C integer, Fortran integer,
+   						floating-point
+
+   	MPI_SUM, MPI_PROD 		C integer, Fortran integer,
+   						floating-point, complex
+
+   	MPI_LAND, MPI_LOR,		C integer, logical
+   	MPI_LXOR
+
+   	MPI_BAND, MPI_BOR,		C integer, Fortran integer, byte
+   	MPI_BXOR
+
+**Example 1:** A routine that computes the dot product of two vectors
+that are distributed across a group of processes and returns the answer
+at process zero.
+
+.. code-block:: fortran
+
+       SUBROUTINE PAR_BLAS1(m, a, b, c, comm)
+       REAL a(m), b(m)       ! local slice of array
+       REAL c                ! result (at process zero)
+       REAL sum
+       INTEGER m, comm, i, ierr
+
+       ! local sum
+       sum = 0.0
+       DO i = 1, m
+          sum = sum + a(i)*b(i)
+       END DO
+
+       ! global sum
+       CALL MPI_REDUCE(sum, c, 1, MPI_REAL, MPI_SUM, 0, comm, ierr)
+       RETURN
+
+**Example 2:** A routine that computes the product of a vector and an
+array that are distributed across a group of processes and returns the
+answer at process zero.
+
+.. code-block:: fortran
+
+       SUBROUTINE PAR_BLAS2(m, n, a, b, c, comm)
+       REAL a(m), b(m,n)    ! local slice of array
+       REAL c(n)            ! result
+       REAL sum(n)
+       INTEGER n, comm, i, j, ierr
+
+       ! local sum
+       DO j= 1, n
+         sum(j) = 0.0
+         DO i = 1, m
+           sum(j) = sum(j) + a(i)*b(i,j)
+         END DO
+       END DO
+
+       ! global sum
+       CALL MPI_REDUCE(sum, c, n, MPI_REAL, MPI_SUM, 0, comm, ierr)
+
+       ! return result at process zero (and garbage at the other nodes)
+       RETURN
+
+
+MINLOC AND MAXLOC
+-----------------
+
+The operator MPI_MINLOC is used to compute a global minimum and also an
+index attached to the minimum value. MPI_MAXLOC similarly computes a
+global maximum and index. One application of these is to compute a
+global minimum (maximum) and the rank of the process containing this
+value.
+
+The operation that defines MPI_MAXLOC is
+
+::
+
+            ( u )    (  v )      ( w )
+            (   )  o (    )   =  (   )
+            ( i )    (  j )      ( k )
+
+   where
+
+       w = max(u, v)
+
+   and
+
+            ( i            if u > v
+            (
+      k   = ( min(i, j)    if u = v
+            (
+            (  j           if u < v)
+
+
+MPI_MINLOC is defined similarly:
+
+::
+
+            ( u )    (  v )      ( w )
+            (   )  o (    )   =  (   )
+            ( i )    (  j )      ( k )
+
+   where
+
+       w = min(u, v)
+
+   and
+
+            ( i            if u < v
+            (
+      k   = ( min(i, j)    if u = v
+            (
+            (  j           if u > v)
+
+Both operations are associative and commutative. Note that if MPI_MAXLOC
+is applied to reduce a sequence of pairs (u(0), 0), (u(1), 1), ...,
+(u(n-1), n-1), then the value returned is (u , r), where u= max(i) u(i)
+and r is the index of the first global maximum in the sequence. Thus, if
+each process supplies a value and its rank within the group, then a
+reduce operation with op = MPI_MAXLOC will return the maximum value and
+the rank of the first process with that value. Similarly, MPI_MINLOC can
+be used to return a minimum and its index. More generally, MPI_MINLOC
+computes a lexicographic minimum, where elements are ordered according
+to the first component of each pair, and ties are resolved according to
+the second component.
+
+The reduce operation is defined to operate on arguments that consist of
+a pair: value and index. For both Fortran and C, types are provided to
+describe the pair. The potentially mixed-type nature of such arguments
+is a problem in Fortran. The problem is circumvented, for Fortran, by
+having the MPI-provided type consist of a pair of the same type as
+value, and coercing the index to this type also. In C, the MPI-provided
+pair type has distinct types and the index is an int.
+
+In order to use MPI_MINLOC and MPI_MAXLOC in a reduce operation, one
+must provide a datatype argument that represents a pair (value and
+index). MPI provides nine such predefined datatypes. The operations
+MPI_MAXLOC and MPI_MINLOC can be used with each of the following
+datatypes:
+
+::
+
+       Fortran:
+       Name                     Description
+       MPI_2REAL                pair of REALs
+       MPI_2DOUBLE_PRECISION    pair of DOUBLE-PRECISION variables
+       MPI_2INTEGER             pair of INTEGERs
+
+       C:
+       Name        	    	Description
+       MPI_FLOAT_INT            float and int
+       MPI_DOUBLE_INT           double and int
+       MPI_LONG_INT             long and int
+       MPI_2INT                 pair of ints
+       MPI_SHORT_INT            short and int
+       MPI_LONG_DOUBLE_INT      long double and int
+
+The data type MPI_2REAL is equivalent to:
+
+.. code-block:: fortran
+
+       MPI_TYPE_CONTIGUOUS(2, MPI_REAL, MPI_2REAL)
+
+Similar statements apply for MPI_2INTEGER, MPI_2DOUBLE_PRECISION, and
+MPI_2INT.
+
+The datatype MPI_FLOAT_INT is as if defined by the following sequence of
+instructions.
+
+.. code-block:: c
+
+       type[0] = MPI_FLOAT
+       type[1] = MPI_INT
+       disp[0] = 0
+       disp[1] = sizeof(float)
+       block[0] = 1
+       block[1] = 1
+       MPI_TYPE_STRUCT(2, block, disp, type, MPI_FLOAT_INT)
+
+Similar statements apply for MPI_LONG_INT and MPI_DOUBLE_INT.
+
+**Example 3:** Each process has an array of 30 doubles, in C. For each
+of the 30 locations, compute the value and rank of the process
+containing the largest value.
+
+.. code-block:: c
+
+           ...
+           /* each process has an array of 30 double: ain[30]
+            */
+           double ain[30], aout[30];
+           int  ind[30];
+           struct {
+               double val;
+               int   rank;
+           } in[30], out[30];
+           int i, myrank, root;
+
+           MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
+           for (i=0; i<30; ++i) {
+               in[i].val = ain[i];
+               in[i].rank = myrank;
+           }
+           MPI_Reduce( in, out, 30, MPI_DOUBLE_INT, MPI_MAXLOC, root, comm );
+           /* At this point, the answer resides on process root
+            */
+           if (myrank == root) {
+               /* read ranks out
+                */
+               for (i=0; i<30; ++i) {
+                   aout[i] = out[i].val;
+                   ind[i] = out[i].rank;
+               }
+           }
+
+**Example 4:** Same example, in Fortran.
+
+.. code-block:: fortran
+
+       ...
+       ! each process has an array of 30 double: ain(30)
+
+       DOUBLE PRECISION :: ain(30), aout(30)
+       INTEGER :: ind(30)
+       DOUBLE PRECISION :: in(2,30), out(2,30)
+       INTEGER :: i, myrank, root, ierr
+
+       call MPI_COMM_RANK(MPI_COMM_WORLD, myrank)
+       DO I=1, 30
+           in(1,i) = ain(i)
+           in(2,i) = myrank    ! myrank is coerced to a double
+       END DO
+
+       call MPI_REDUCE( in, out, 30, MPI_2DOUBLE_PRECISION, MPI_MAXLOC, root, &
+                                                                 comm, ierr )
+       ! At this point, the answer resides on process root
+
+       IF (myrank == root) THEN
+           ! read ranks out
+           DO I= 1, 30
+               aout(i) = out(1,i)
+               ind(i) = out(2,i)  ! rank is coerced back to an integer
+           END DO
+       END IF
+
+**Example 5:** Each process has a nonempty array of values. Find the
+minimum global value, the rank of the process that holds it, and its
+index on this process.
+
+.. code-block:: c
+
+       #define  LEN   1000
+
+       float val[LEN];        /* local array of values */
+       int count;             /* local number of values */
+       int myrank, minrank, minindex;
+       float minval;
+
+       struct {
+           float value;
+           int   index;
+       } in, out;
+
+       /* local minloc */
+       in.value = val[0];
+       in.index = 0;
+       for (i=1; i < count; i++)
+           if (in.value > val[i]) {
+               in.value = val[i];
+               in.index = i;
+           }
+
+       /* global minloc */
+       MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
+       in.index = myrank*LEN + in.index;
+       MPI_Reduce( in, out, 1, MPI_FLOAT_INT, MPI_MINLOC, root, comm );
+           /* At this point, the answer resides on process root
+            */
+       if (myrank == root) {
+           /* read answer out
+            */
+           minval = out.value;
+           minrank = out.index / LEN;
+           minindex = out.index % LEN;
+
+All MPI objects (e.g., MPI_Datatype, MPI_Comm) are of type INTEGER in
+Fortran.
+
+
+NOTES ON COLLECTIVE OPERATIONS
+------------------------------
+
+The reduction functions ( MPI_Op ) do not return an error value. As a
+result, if the functions detect an error, all they can do is either call
+:ref:`MPI_Abort` or silently skip the problem. Thus, if you change the error
+handler from MPI_ERRORS_ARE_FATAL to something else, for example,
+MPI_ERRORS_RETURN , then no error may be indicated.
+
+The reason for this is the performance problems in ensuring that all
+collective routines return the same error value.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Allreduce`
+   * :ref:`MPI_Reduce_scatter`
+   * :ref:`MPI_Scan`
+   * :ref:`MPI_Op_create`
+   * :ref:`MPI_Op_free`
diff --git a/docs/man-openmpi/man3/MPI_Reduce_init.3.rst b/docs/man-openmpi/man3/MPI_Reduce_init.3.rst
new file mode 100644
index 00000000000..ccc7cf1280f
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Reduce_init.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_reduce_init:
+
+MPI_Reduce_init
+===============
+    .. include_body
+
+.. include:: ../man3/MPI_Reduce.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Reduce_local.3.rst b/docs/man-openmpi/man3/MPI_Reduce_local.3.rst
new file mode 100644
index 00000000000..9a985bde311
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Reduce_local.3.rst
@@ -0,0 +1,291 @@
+.. _mpi_reduce_local:
+
+
+MPI_Reduce_local
+================
+
+.. include_body
+
+:ref:`MPI_Reduce_local` |mdash| Perform a local reduction
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_reduce_local.rst
+
+INPUT PARAMETERS
+----------------
+* ``inbuf``: Address of input buffer (choice).
+* ``count``: Number of elements in input buffer (integer).
+* ``datatype``: Data type of elements of input buffer (handle).
+* ``op``: Reduce operation (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``inoutbuf``: Address of in/out buffer (choice).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The global reduce functions (:ref:`MPI_Reduce_local`, :ref:`MPI_Op_create`,
+:ref:`MPI_Op_free`, :ref:`MPI_Allreduce`, MPI_Reduce_local_scatter, MPI_Scan) perform
+a global reduce operation (such as sum, max, logical AND, etc.) across
+all the members of a group. The reduction operation can be either one of
+a predefined list of operations, or a user-defined operation. The global
+reduction functions come in several flavors: a reduce that returns the
+result of the reduction at one node, an all-reduce that returns this
+result at all nodes, and a scan (parallel prefix) operation. In
+addition, a reduce-scatter operation combines the functionality of a
+reduce and a scatter operation.
+
+:ref:`MPI_Reduce_local` combines the elements provided in the input and
+input/output buffers of the local process, using the operation op, and
+returns the combined value in the inout/output buffer. The input buffer
+is defined by the arguments inbuf, count, and datatype; the output
+buffer is defined by the arguments inoutbuf, count, and datatype; both
+have the same number of elements, with the same type. The routine is a
+local call. The process can provide one element, or a sequence of
+elements, in which case the combine operation is executed element-wise
+on each entry of the sequence. For example, if the operation is MPI_MAX
+and the input buffer contains two elements that are floating-point
+numbers (count = 2 and datatype = MPI_FLOAT), then inoutbuf(1) = global
+max (inbuf(1)) and inoutbuf(2) = global max(inbuf(2)).
+
+
+USE OF IN-PLACE OPTION
+----------------------
+
+The use of MPI_IN_PLACE is disallowed with :ref:`MPI_Reduce_local`.
+
+
+PREDEFINED REDUCE OPERATIONS
+----------------------------
+
+The set of predefined operations provided by MPI is listed below
+(Predefined Reduce Operations). That section also enumerates the
+datatypes each operation can be applied to. In addition, users may
+define their own operations that can be overloaded to operate on several
+datatypes, either basic or derived. This is further explained in the
+description of the user-defined operations (see the man pages for
+:ref:`MPI_Op_create` and MPI_Op_free).
+
+The operation op is always assumed to be associative. All predefined
+operations are also assumed to be commutative. Users may define
+operations that are assumed to be associative, but not commutative. The
+\``canonical'' evaluation order of a reduction is determined by the
+ranks of the processes in the group. However, the implementation can
+take advantage of associativity, or associativity and commutativity, in
+order to change the order of evaluation. This may change the result of
+the reduction for operations that are not strictly associative and
+commutative, such as floating point addition.
+
+Predefined operators work only with the MPI types listed below
+(Predefined Reduce Operations, and the section MINLOC and MAXLOC,
+below). User-defined operators may operate on general, derived
+datatypes. In this case, each argument that the reduce operation is
+applied to is one element described by such a datatype, which may
+contain several basic values. This is further explained in Section 4.9.4
+of the MPI Standard, "User-Defined Operations."
+
+The following predefined operations are supplied for :ref:`MPI_Reduce_local`
+and related functions :ref:`MPI_Allreduce`, :ref:`MPI_Reduce_scatter`, and :ref:`MPI_Scan`.
+These operations are invoked by placing the following in op:
+
+::
+
+   	Name                Meaning
+        ---------           --------------------
+   	MPI_MAX             maximum
+   	MPI_MIN             minimum
+   	MPI_SUM             sum
+   	MPI_PROD            product
+   	MPI_LAND            logical and
+   	MPI_BAND            bit-wise and
+   	MPI_LOR             logical or
+   	MPI_BOR             bit-wise or
+   	MPI_LXOR            logical xor
+   	MPI_BXOR            bit-wise xor
+   	MPI_MAXLOC          max value and location
+   	MPI_MINLOC          min value and location
+
+The two operations MPI_MINLOC and MPI_MAXLOC are discussed separately
+below (MINLOC and MAXLOC). For the other predefined operations, we
+enumerate below the allowed combinations of op and datatype arguments.
+First, define groups of MPI basic datatypes in the following way:
+
+::
+
+   	C integer:            MPI_INT, MPI_LONG, MPI_SHORT,
+   	                      MPI_UNSIGNED_SHORT, MPI_UNSIGNED,
+   	                      MPI_UNSIGNED_LONG
+   	Fortran integer:      MPI_INTEGER
+   	Floating-point:       MPI_FLOAT, MPI_DOUBLE, MPI_REAL,
+   	                      MPI_DOUBLE_PRECISION, MPI_LONG_DOUBLE
+   	Logical:              MPI_LOGICAL
+   	Complex:              MPI_COMPLEX
+   	Byte:                 MPI_BYTE
+
+Now, the valid datatypes for each option is specified below.
+
+::
+
+   	Op                      	Allowed Types
+        ----------------         ---------------------------
+   	MPI_MAX, MPI_MIN		C integer, Fortran integer,
+   						floating-point
+
+   	MPI_SUM, MPI_PROD 		C integer, Fortran integer,
+   						floating-point, complex
+
+   	MPI_LAND, MPI_LOR,		C integer, logical
+   	MPI_LXOR
+
+   	MPI_BAND, MPI_BOR,		C integer, Fortran integer, byte
+   	MPI_BXOR
+
+
+MINLOC AND MAXLOC
+-----------------
+
+The operator MPI_MINLOC is used to compute a global minimum and also an
+index attached to the minimum value. MPI_MAXLOC similarly computes a
+global maximum and index. One application of these is to compute a
+global minimum (maximum) and the rank of the process containing this
+value.
+
+The operation that defines MPI_MAXLOC is
+
+::
+
+            ( u )    (  v )      ( w )
+            (   )  o (    )   =  (   )
+            ( i )    (  j )      ( k )
+
+   where
+
+       w = max(u, v)
+
+   and
+
+            ( i            if u > v
+            (
+      k   = ( min(i, j)    if u = v
+            (
+            (  j           if u < v)
+
+
+MPI_MINLOC is defined similarly:
+
+::
+
+            ( u )    (  v )      ( w )
+            (   )  o (    )   =  (   )
+            ( i )    (  j )      ( k )
+
+   where
+
+       w = min(u, v)
+
+   and
+
+            ( i            if u < v
+            (
+      k   = ( min(i, j)    if u = v
+            (
+            (  j           if u > v)
+
+Both operations are associative and commutative. Note that if MPI_MAXLOC
+is applied to reduce a sequence of pairs (u(0), 0), (u(1), 1), ...,
+(u(n-1), n-1), then the value returned is (u , r), where u= max(i) u(i)
+and r is the index of the first global maximum in the sequence. Thus, if
+each process supplies a value and its rank within the group, then a
+reduce operation with op = MPI_MAXLOC will return the maximum value and
+the rank of the first process with that value. Similarly, MPI_MINLOC can
+be used to return a minimum and its index. More generally, MPI_MINLOC
+computes a lexicographic minimum, where elements are ordered according
+to the first component of each pair, and ties are resolved according to
+the second component.
+
+The reduce operation is defined to operate on arguments that consist of
+a pair: value and index. For both Fortran and C, types are provided to
+describe the pair. The potentially mixed-type nature of such arguments
+is a problem in Fortran. The problem is circumvented, for Fortran, by
+having the MPI-provided type consist of a pair of the same type as
+value, and coercing the index to this type also. In C, the MPI-provided
+pair type has distinct types and the index is an int.
+
+In order to use MPI_MINLOC and MPI_MAXLOC in a reduce operation, one
+must provide a datatype argument that represents a pair (value and
+index). MPI provides nine such predefined datatypes. The operations
+MPI_MAXLOC and MPI_MINLOC can be used with each of the following
+datatypes:
+
+::
+
+       Fortran:
+       Name                     Description
+       MPI_2REAL                pair of REALs
+       MPI_2DOUBLE_PRECISION    pair of DOUBLE-PRECISION variables
+       MPI_2INTEGER             pair of INTEGERs
+
+       C:
+       Name        	    	Description
+       MPI_FLOAT_INT            float and int
+       MPI_DOUBLE_INT           double and int
+       MPI_LONG_INT             long and int
+       MPI_2INT                 pair of ints
+       MPI_SHORT_INT            short and int
+       MPI_LONG_DOUBLE_INT      long double and int
+
+The data type ``MPI_2REAL`` is equivalent to:
+
+.. code-block:: fortran
+
+       call MPI_TYPE_CONTIGUOUS(2, MPI_REAL, MPI_2REAL)
+
+Similar statements apply for MPI_2INTEGER, MPI_2DOUBLE_PRECISION, and
+MPI_2INT.
+
+The datatype MPI_FLOAT_INT is as if defined by the following sequence of
+instructions.
+
+::.. code-block:: c
+
+       type[0] = MPI_FLOAT
+       type[1] = MPI_INT
+       disp[0] = 0
+       disp[1] = sizeof(float)
+       block[0] = 1
+       block[1] = 1
+       MPI_TYPE_STRUCT(2, block, disp, type, MPI_FLOAT_INT)
+
+Similar statements apply for MPI_LONG_INT and MPI_DOUBLE_INT.
+
+All MPI objects (e.g., MPI_Datatype, MPI_Comm) are of type INTEGER in
+Fortran.
+
+
+NOTES ON COLLECTIVE OPERATIONS
+------------------------------
+
+The reduction operators ( MPI_Op ) do not return an error value. As a
+result, if the functions detect an error, all they can do is either call
+:ref:`MPI_Abort` or silently skip the problem. Thus, if you change the error
+handler from MPI_ERRORS_ARE_FATAL to something else, for example,
+MPI_ERRORS_RETURN , then no error may be indicated.
+
+The reason for this is the performance problems in ensuring that all
+collective routines return the same error value.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Allreduce`
+   * :ref:`MPI_Reduce`
+   * :ref:`MPI_Reduce_scatter`
+   * :ref:`MPI_Scan`
+   * :ref:`MPI_Op_create`
+   * :ref:`MPI_Op_free`
diff --git a/docs/man-openmpi/man3/MPI_Reduce_scatter.3.rst b/docs/man-openmpi/man3/MPI_Reduce_scatter.3.rst
new file mode 100644
index 00000000000..39acf16ab03
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Reduce_scatter.3.rst
@@ -0,0 +1,87 @@
+.. _mpi_reduce_scatter:
+
+
+MPI_Reduce_scatter
+==================
+
+.. include_body
+
+:ref:`MPI_Reduce_scatter`, :ref:`MPI_Ireduce_scatter`, :ref:`MPI_Reduce_scatter_init` -
+Combines values and scatters the results.
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Reduce_scatter, MPI_Ireduce_scatter, MPI_Reduce_scatter_init
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_reduce_scatter.rst
+
+INPUT PARAMETERS
+----------------
+* ``sendbuf``: Starting address of send buffer (choice).
+* ``recvcounts``: Integer array specifying the number of elements in result distributed to each process. Array must be identical on all calling processes.
+* ``datatype``: Datatype of elements of input buffer (handle).
+* ``op``: Operation (handle).
+* ``comm``: Communicator (handle).
+* ``info``: Info (handle, persistent only).
+
+OUTPUT PARAMETERS
+-----------------
+* ``recvbuf``: Starting address of receive buffer (choice).
+* ``request``: Request (handle, non-blocking and persistent only).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Reduce_scatter` first does an element-wise reduction on vector of
+*count* = S(i)\ *recvcounts*\ [i] elements in the send buffer defined by
+*sendbuf*, *count*, and *datatype*. Next, the resulting vector of
+results is split into n disjoint segments, where n is the number of
+processes in the group. Segment i contains *recvcounts*\ [i] elements.
+The ith segment is sent to process i and stored in the receive buffer
+defined by *recvbuf*, *recvcounts*\ [i], and *datatype*.
+
+
+USE OF IN-PLACE OPTION
+----------------------
+
+When the communicator is an intracommunicator, you can perform a
+reduce-scatter operation in-place (the output buffer is used as the
+input buffer). Use the variable MPI_IN_PLACE as the value of the
+*sendbuf*. In this case, the input data is taken from the top of the
+receive buffer. The area occupied by the input data may be either longer
+or shorter than the data filled by the output data.
+
+
+WHEN COMMUNICATOR IS AN INTER-COMMUNICATOR
+------------------------------------------
+
+When the communicator is an inter-communicator, the reduce-scatter
+operation occurs in two phases. First, the result of the reduction
+performed on the data provided by the processes in the first group is
+scattered among the processes in the second group. Then the reverse
+occurs: the reduction performed on the data provided by the processes in
+the second group is scattered among the processes in the first group.
+For each group, all processes provide the same *recvcounts* argument,
+and the sum of the *recvcounts* values should be the same for both
+groups.
+
+
+NOTES ON COLLECTIVE OPERATIONS
+------------------------------
+
+The reduction functions ( MPI_Op ) do not return an error value. As a
+result, if the functions detect an error, all they can do is either call
+:ref:`MPI_Abort` or silently skip the problem. Thus, if you change the error
+handler from MPI_ERRORS_ARE_FATAL to something else, for example,
+MPI_ERRORS_RETURN , then no error may be indicated.
+
+The reason for this is the performance problems in ensuring that all
+collective routines return the same error value.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Reduce_scatter_block.3.rst b/docs/man-openmpi/man3/MPI_Reduce_scatter_block.3.rst
new file mode 100644
index 00000000000..0d4ab22de09
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Reduce_scatter_block.3.rst
@@ -0,0 +1,92 @@
+.. _mpi_reduce_scatter_block:
+
+
+MPI_Reduce_scatter_block
+========================
+
+.. include_body
+
+:ref:`MPI_Reduce_scatter_block`, :ref:`MPI_Ireduce_scatter_block`,
+:ref:`MPI_Reduce_scatter_block_init` |mdash| Combines values and scatters the
+results in blocks.
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Reduce_scatter_block, MPI_Ireduce_scatter_block, MPI_Reduce_scatter_block_init
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_reduce_scatter_block.rst
+
+INPUT PARAMETERS
+----------------
+* ``sendbuf``: Starting address of send buffer (choice).
+* ``recvcount``: lement count per block (non-negative integer).
+* ``datatype``: Datatype of elements of input buffer (handle).
+* ``op``: Operation (handle).
+* ``comm``: Communicator (handle).
+* ``info``: Info (handle, persistent only).
+
+OUTPUT PARAMETERS
+-----------------
+* ``recvbuf``: Starting address of receive buffer (choice).
+* ``request``: Request (handle, non-blocking and persistent only).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Reduce_scatter_block` first does an element-wise reduction on vector
+of ``count = n * recvcount`` elements in the send buffer defined by
+*sendbuf*, *count*, and *datatype*, using the operation *op*, where n is
+the number of processes in the group of *comm*. Next, the resulting
+vector of results is split into n disjoint segments, where n is the
+number of processes in the group. Each segments contains *recvcount*
+elements. The ith segment is sent to process i and stored in the receive
+buffer defined by *recvbuf*, *recvcount*, and *datatype*.
+
+
+USE OF IN-PLACE OPTION
+----------------------
+
+When the communicator is an intracommunicator, you can perform a
+reduce-scatter operation in-place (the output buffer is used as the
+input buffer). Use the variable MPI_IN_PLACE as the value of the
+*sendbuf*. In this case, the input data is taken from the top of the
+receive buffer. The area occupied by the input data may be either longer
+or shorter than the data filled by the output data.
+
+
+WHEN COMMUNICATOR IS AN INTER-COMMUNICATOR
+------------------------------------------
+
+When the communicator is an inter-communicator, the reduce-scatter
+operation occurs in two phases. First, the result of the reduction
+performed on the data provided by the processes in the first group is
+scattered among the processes in the second group. Then the reverse
+occurs: the reduction performed on the data provided by the processes in
+the second group is scattered among the processes in the first group.
+For each group, all processes provide the same *recvcounts* argument,
+and the sum of the *recvcounts* values should be the same for both
+groups.
+
+
+NOTES ON COLLECTIVE OPERATIONS
+------------------------------
+
+The reduction functions ( MPI_Op ) do not return an error value. As a
+result, if the functions detect an error, all they can do is either call
+:ref:`MPI_Abort` or silently skip the problem. Thus, if you change the error
+handler from MPI_ERRORS_ARE_FATAL to something else, for example,
+MPI_ERRORS_RETURN , then no error may be indicated.
+
+The reason for this is the performance problems in ensuring that all
+collective routines return the same error value.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Reduce_scatter`
diff --git a/docs/man-openmpi/man3/MPI_Reduce_scatter_block_init.3.rst b/docs/man-openmpi/man3/MPI_Reduce_scatter_block_init.3.rst
new file mode 100644
index 00000000000..ce685e94ff1
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Reduce_scatter_block_init.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_reduce_scatter_block_init:
+
+MPI_Reduce_scatter_block_init
+=============================
+    .. include_body
+
+.. include:: ../man3/MPI_Reduce_scatter_block.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Reduce_scatter_init.3.rst b/docs/man-openmpi/man3/MPI_Reduce_scatter_init.3.rst
new file mode 100644
index 00000000000..018af740036
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Reduce_scatter_init.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_reduce_scatter_init:
+
+MPI_Reduce_scatter_init
+=======================
+    .. include_body
+
+.. include:: ../man3/MPI_Reduce_scatter.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Register_datarep.3.rst b/docs/man-openmpi/man3/MPI_Register_datarep.3.rst
new file mode 100644
index 00000000000..ffeaf62caae
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Register_datarep.3.rst
@@ -0,0 +1,72 @@
+.. _mpi_register_datarep:
+
+
+MPI_Register_datarep
+====================
+
+.. include_body
+
+:ref:`MPI_Register_datarep` |mdash| Defines data representation.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_register_datarep.rst
+
+INPUT PARAMETERS
+----------------
+* ``datarep``: Data representation identifier (string).
+* ``read_conversion_fn``: Function invoked to convert from file representation to native representation (function).
+* ``write_conversion_fn``: Function invoked to convert from native representation to file representation (function).
+* ``dtype_file_extent_fn``: Function invoked to get the extent of a data type as represented in the file (function).
+* ``extra_state``: Extra state.
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Register_datarep` defines a data representation. It associates the
+data representation's identifier (a string) with the functions that
+convert from file representation to the native representation and vice
+versa, with the function that gets the extent of a data type as
+represented in the file, as well as with "extra state," which is used
+for passing arguments. Once a data representation has been registered
+using this routine, you may specify its identifier as an argument to
+:ref:`MPI_File_set_view`, causing subsequent data-access operations to call the
+specified conversion functions.
+
+The call associates *read_conversion_fn*, *write_conversion_fn*, and
+*dtype_file_extent_fn* with the data representation identifier
+*datarep*. *datarep* can then be used as an argument to
+:ref:`MPI_File_set_view`, causing subsequent data access operations to call the
+conversion functions to convert all data items accessed between file
+data representation and native representation. :ref:`MPI_Register_datarep` is a
+local operation and only registers the data representation for the
+calling MPI process. If *datarep* is already defined, an error in the
+error class MPI_ERR_DUP_DATAREP is raised using the default file error
+handler. The length of a data representation string is limited to the
+value of MPI_MAX_DATAREP_STRING. MPI_MAX_DATAREP_STRING must have a
+value of at least 64. No routines are provided to delete data
+representations and free the associated resources; it is not expected
+that an application will generate them in significant numbers.
+
+
+NOTES
+-----
+
+The Fortran version of each MPI I/O routine includes a final argument,
+IERROR, which is not defined in the PARAMETERS sections. This argument
+is used to return the error status of the routine in the manner typical
+for Fortran library routines.
+
+The C version of each routine returns an error status as an integer
+return value.
+
+Error classes are found in mpi.h (for C) and mpif.h (for Fortran).
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Request_c2f.3.rst b/docs/man-openmpi/man3/MPI_Request_c2f.3.rst
new file mode 100644
index 00000000000..b36d96f010a
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Request_c2f.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_request_c2f:
+
+MPI_Request_c2f
+===============
+
+.. include_body
+
+.. include:: ../man3/MPI_Comm_f2c.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_Request_f2c.3.rst b/docs/man-openmpi/man3/MPI_Request_f2c.3.rst
new file mode 100644
index 00000000000..fe8feb5de98
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Request_f2c.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_request_f2c:
+
+MPI_Request_f2c
+===============
+
+.. include_body
+
+.. include:: ../man3/MPI_Comm_f2c.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_Request_free.3.rst b/docs/man-openmpi/man3/MPI_Request_free.3.rst
new file mode 100644
index 00000000000..a59d81f071e
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Request_free.3.rst
@@ -0,0 +1,106 @@
+.. _mpi_request_free:
+
+
+MPI_Request_free
+================
+
+.. include_body
+
+:ref:`MPI_Request_free` |mdash| Frees a communication request object.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_request_free.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``request``: Communication request (handle).
+
+DESCRIPTION
+-----------
+
+This operation allows a request object to be deallocated without waiting
+for the associated communication to complete.
+
+:ref:`MPI_Request_free` marks the request object for deallocation and sets
+request to MPI_REQUEST_NULL. Any ongoing communication that is
+associated with the request will be allowed to complete. The request
+will be deallocated only after its completion.
+
+
+NOTES
+-----
+
+Once a request is freed by a call to :ref:`MPI_Request_free`, it is not
+possible to check for the successful completion of the associated
+communication with calls to :ref:`MPI_Wait` or :ref:`MPI_Test`. Also, if an error
+occurs subsequently during the communication, an error code cannot be
+returned to the user |mdash| such an error must be treated as fatal.
+Questions arise as to how one knows when the operations have completed
+when using :ref:`MPI_Request_free`. Depending on the program logic, there may
+be other ways in which the program knows that certain operations have
+completed and this makes usage of :ref:`MPI_Request_free` practical. For
+example, an active send request could be freed when the logic of the
+program is such that the receiver sends a reply to the message sent |mdash|
+the arrival of the reply informs the sender that the send has completed
+and the send buffer can be reused. An active receive request should
+never be freed, as the receiver will have no way to verify that the
+receive has completed and the receive buffer can be reused.
+
+**Example:**
+
+.. code-block:: fortran
+
+       CALL MPI_COMM_RANK(MPI_COMM_WORLD, rank)
+       IF(rank == 0) THEN
+           DO i=1, n
+             CALL MPI_ISEND(outval, 1, MPI_REAL, 1, 0, req, ierr)
+             CALL MPI_REQUEST_FREE(req, ierr)
+             CALL MPI_IRECV(inval, 1, MPI_REAL, 1, 0, req, ierr)
+             CALL MPI_WAIT(req, status, ierr)
+           END DO
+       ELSE IF (rank == 1) THEN
+           CALL MPI_IRECV(inval, 1, MPI_REAL, 0, 0, req, ierr)
+           CALL MPI_WAIT(req, status)
+           DO I=1, n-1
+              CALL MPI_ISEND(outval, 1, MPI_REAL, 0, 0, req, ierr)
+              CALL MPI_REQUEST_FREE(req, ierr)
+              CALL MPI_IRECV(inval, 1, MPI_REAL, 0, 0, req, ierr)
+              CALL MPI_WAIT(req, status, ierr)
+           END DO
+           CALL MPI_ISEND(outval, 1, MPI_REAL, 0, 0, req, ierr)
+           CALL MPI_WAIT(req, status)
+       END IF
+
+This routine is normally used to free persistent requests created with
+either :ref:`MPI_Recv_init` or :ref:`MPI_Send_init` and friends. However, it can
+be used to free a request created with :ref:`MPI_Irecv` or :ref:`MPI_Isend` and
+friends; in that case the use can not use the test/wait routines on the
+request.
+
+It **is** permitted to free an active request. However, once freed, you
+can not use the request in a wait or test routine (e.g., :ref:`MPI_Wait` ).
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Isend`
+   * :ref:`MPI_Irecv`
+   * :ref:`MPI_Issend`
+   * :ref:`MPI_Ibsend`
+   * :ref:`MPI_Irsend`
+   * :ref:`MPI_Recv_init`
+   * :ref:`MPI_Send_init`
+   * :ref:`MPI_Ssend_init`
+   * :ref:`MPI_Rsend_init`
+   * :ref:`MPI_Test`
+   * :ref:`MPI_Wait`
+   * :ref:`MPI_Waitall`
+   * :ref:`MPI_Waitany`
+   * :ref:`MPI_Waitsome`
+   * :ref:`MPI_Testall`
+   * :ref:`MPI_Testany`
+   * :ref:`MPI_Testsome`
diff --git a/docs/man-openmpi/man3/MPI_Request_get_status.3.rst b/docs/man-openmpi/man3/MPI_Request_get_status.3.rst
new file mode 100644
index 00000000000..19e953d37bd
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Request_get_status.3.rst
@@ -0,0 +1,41 @@
+.. _mpi_request_get_status:
+
+
+MPI_Request_get_status
+======================
+
+.. include_body
+
+:ref:`MPI_Request_get_status` |mdash| Access information associated with a
+request without freeing the request.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_request_get_status.rst
+
+INPUT PARAMETER
+---------------
+* ``request``: Communication request (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``flag``: Boolean flag, same as from :ref:`MPI_Test` (logical).
+* ``status``: ``MPI_Status`` object if flag is true (status).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Request_get_status` sets ``flag = true`` if the operation is
+complete or sets ``flag = false`` if it is not complete. If the
+operation is complete, it returns in *status* the request status. It
+does not deallocate or deactivate the request; a subsequent call to
+test, wait, or free should be executed with that request.
+
+If your application does not need to examine the *status* field, you can
+save resources by using the predefined constant ``MPI_STATUS_IGNORE`` as a
+special value for the *status* argument.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Rget.3.rst b/docs/man-openmpi/man3/MPI_Rget.3.rst
new file mode 100644
index 00000000000..bc21ad92cf9
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Rget.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_rget:
+
+MPI_Rget
+========
+    .. include_body
+
+.. include:: ../man3/MPI_Get.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Rget_accumulate.3.rst b/docs/man-openmpi/man3/MPI_Rget_accumulate.3.rst
new file mode 100644
index 00000000000..0797871eb1e
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Rget_accumulate.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_rget_accumulate:
+
+MPI_Rget_accumulate
+===================
+    .. include_body
+
+.. include:: ../man3/MPI_Get_accumulate.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Rput.3.rst b/docs/man-openmpi/man3/MPI_Rput.3.rst
new file mode 100644
index 00000000000..d85d8b2c934
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Rput.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_rput:
+
+MPI_Rput
+========
+    .. include_body
+
+.. include:: ../man3/MPI_Put.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Rsend.3.rst b/docs/man-openmpi/man3/MPI_Rsend.3.rst
new file mode 100644
index 00000000000..bb9ed637fbe
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Rsend.3.rst
@@ -0,0 +1,38 @@
+.. _mpi_rsend:
+
+
+MPI_Rsend
+=========
+
+.. include_body
+
+:ref:`MPI_Rsend` |mdash| Ready send.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_rsend.rst
+
+INPUT PARAMETERS
+----------------
+* ``buf``: Initial address of send buffer (choice).
+* ``count``: Number of elements in send buffer (nonnegative integer).
+* ``datatype``: Datatype of each send buffer element (handle).
+* ``dest``: Rank of destination (integer).
+* ``tag``: Message tag (integer).
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+A ready send may only be called if the user can guarantee that a receive
+is already posted. It is an error if the receive is not posted before
+the ready send is called.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Rsend_init.3.rst b/docs/man-openmpi/man3/MPI_Rsend_init.3.rst
new file mode 100644
index 00000000000..8f4ee0a0a29
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Rsend_init.3.rst
@@ -0,0 +1,50 @@
+.. _mpi_rsend_init:
+
+
+MPI_Rsend_init
+==============
+
+.. include_body
+
+:ref:`MPI_Rsend_init` |mdash| Builds a handle for a ready send.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_rsend_init.rst
+
+INPUT PARAMETERS
+----------------
+* ``buf``: Initial address of send buffer (choice).
+* ``count``: Number of elements sent (integer).
+* ``datatype``: Type of each element (handle).
+* ``dest``: Rank of destination (integer).
+* ``tag``: Message tag (integer).
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``request``: Communication request (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Creates a persistent communication object for a ready mode send
+operation, and binds to it all the arguments of a send operation.
+
+A communication (send or receive) that uses a persistent request is
+initiated by the function :ref:`MPI_Start`.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Bsend_init`
+   * :ref:`MPI_Send_init`
+   * MPI_Sssend_init
+   * :ref:`MPI_Recv_init`
+   * :ref:`MPI_Start`
+   * :ref:`MPI_Startall`
+   * :ref:`MPI_Request_free`
diff --git a/docs/man-openmpi/man3/MPI_Scan.3.rst b/docs/man-openmpi/man3/MPI_Scan.3.rst
new file mode 100644
index 00000000000..62a50432686
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Scan.3.rst
@@ -0,0 +1,172 @@
+.. _mpi_scan:
+
+
+MPI_Scan
+========
+
+.. include_body
+
+:ref:`MPI_Scan`, :ref:`MPI_Iscan`, :ref:`MPI_Scan_init` - Computes an inclusive scan
+(partial reduction)
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Scan, MPI_Iscan, MPI_Scan_init
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_scan.rst
+
+INPUT PARAMETERS
+----------------
+* ``sendbuf``: Send buffer (choice).
+* ``count``: Number of elements in input buffer (integer).
+* ``datatype``: Data type of elements of input buffer (handle).
+* ``op``: Operation (handle).
+* ``comm``: Communicator (handle).
+* ``info``: Info (handle, persistent only)
+
+OUTPUT PARAMETERS
+-----------------
+* ``recvbuf``: Receive buffer (choice).
+* ``request``: Request (handle, non-blocking and persistent only).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Scan` is used to perform an inclusive prefix reduction on data
+distributed across the calling processes. The operation returns, in the
+*recvbuf* of the process with rank i, the reduction (calculated
+according to the function *op*) of the values in the *sendbuf*\ s of
+processes with ranks 0, ..., i (inclusive). The type of operations
+supported, their semantics, and the constraints on send and receive
+buffers are as for :ref:`MPI_Reduce`.
+
+
+EXAMPLE
+-------
+
+This example uses a user-defined operation to produce a segmented scan.
+A segmented scan takes, as input, a set of values and a set of logicals,
+where the logicals delineate the various segments of the scan. For
+example,
+
+::
+
+   values     v1      v2      v3      v4      v5      v6      v7      v8
+   logicals   0       0       1       1       1       0       0       1
+   result     v1    v1+v2     v3    v3+v4  v3+v4+v5   v6    v6+v7     v8
+
+The result for rank j is thus the sum v(i) + ... + v(j), where i is the
+lowest rank such that for all ranks n, i <= n <= j, logical(n) =
+logical(j). The operator that produces this effect is
+
+::
+
+         [ u ]     [ v ]     [ w ]
+         [   ]  o  [   ]  =  [   ]
+         [ i ]     [ j ]     [ j ]
+
+   where
+
+       ( u + v if i = j w = ( ( v if i != j
+
+Note that this is a noncommutative operator. C code that implements it
+is given below.
+
+.. code-block:: c
+
+   	typedef struct {
+   		double val;
+   		int log;
+   	} SegScanPair;
+
+   	/*
+   	 * the user-defined function
+   	 */
+   	void segScan(SegScanPair *in, SegScanPair *inout, int *len,
+   		MPI_Datatype *dptr)
+   	{
+   		int i;
+   		SegScanPair c;
+
+   		for (i = 0; i < *len; ++i) {
+   			if (in->log == inout->log)
+   				c.val = in->val + inout->val;
+   			else
+   				c.val = inout->val;
+
+   			c.log = inout->log;
+   			*inout = c;
+   			in++;
+   			inout++;
+   		}
+   	}
+
+Note that the inout argument to the user-defined function corresponds to
+the right-hand operand of the operator. When using this operator, we
+must be careful to specify that it is noncommutative, as in the
+following:
+
+.. code-block:: c
+
+   	int			i, base;
+   	SeqScanPair	a, answer;
+   	MPI_Op		myOp;
+   	MPI_Datatype	type[2] = {MPI_DOUBLE, MPI_INT};
+   	MPI_Aint		disp[2];
+   	int			blocklen[2] = {1, 1};
+   	MPI_Datatype	sspair;
+
+   	/*
+   	 * explain to MPI how type SegScanPair is defined
+   	 */
+   	MPI_Get_address(a, disp);
+   	MPI_Get_address(a.log, disp + 1);
+   	base = disp[0];
+   	for (i = 0; i < 2; ++i)
+   		disp[i] -= base;
+   	MPI_Type_struct(2, blocklen, disp, type, &sspair);
+   	MPI_Type_commit(&sspair);
+
+   	/*
+   	 * create the segmented-scan user-op
+   	 * noncommutative - set commute (arg 2) to 0
+   	 */
+   	MPI_Op_create((MPI_User_function *)segScan, 0, &myOp);
+   	...
+   	MPI_Scan(a, answer, 1, sspair, myOp, comm);
+
+
+USE OF IN-PLACE OPTION
+----------------------
+
+When the communicator is an intracommunicator, you can perform a
+scanning operation in place (the output buffer is used as the input
+buffer). Use the variable MPI_IN_PLACE as the value of the *sendbuf*
+argument. The input data is taken from the receive buffer and replaced
+by the output data.
+
+
+NOTES ON COLLECTIVE OPERATIONS
+------------------------------
+
+The reduction functions of type MPI_Op do not return an error value. As
+a result, if the functions detect an error, all they can do is either
+call :ref:`MPI_Abort` or silently skip the problem. Thus, if the error handler
+is changed from MPI_ERRORS_ARE_FATAL to something else (e.g.,
+MPI_ERRORS_RETURN), then no error may be indicated.
+
+The reason for this is the performance problems in ensuring that all
+collective routines return the same error value.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Exscan`
+   * :ref:`MPI_Op_create`
+   * :ref:`MPI_Reduce`
diff --git a/docs/man-openmpi/man3/MPI_Scan_init.3.rst b/docs/man-openmpi/man3/MPI_Scan_init.3.rst
new file mode 100644
index 00000000000..03a35ecfa6d
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Scan_init.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_scan_init:
+
+MPI_Scan_init
+=============
+    .. include_body
+
+.. include:: ../man3/MPI_Scan.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Scatter.3.rst b/docs/man-openmpi/man3/MPI_Scatter.3.rst
new file mode 100644
index 00000000000..385488c56be
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Scatter.3.rst
@@ -0,0 +1,135 @@
+.. _mpi_scatter:
+
+
+MPI_Scatter
+===========
+
+.. include_body
+
+:ref:`MPI_Scatter`, :ref:`MPI_Iscatter`, :ref:`MPI_Scatter_init` - Sends data from one
+task to all tasks in a group.
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Scatter, MPI_Iscatter, MPI_Scatter_init
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_scatter.rst
+
+INPUT PARAMETERS
+----------------
+* ``sendbuf``: Address of send buffer (choice, significant only at root).
+* ``sendcount``: Number of elements sent to each process (integer, significant only at root).
+* ``sendtype``: Datatype of send buffer elements (handle, significant only at root).
+* ``recvcount``: Number of elements in receive buffer (integer).
+* ``recvtype``: Datatype of receive buffer elements (handle).
+* ``root``: Rank of sending process (integer).
+* ``comm``: Communicator (handle).
+* ``info``: Info (handle, persistent only).
+
+OUTPUT PARAMETERS
+-----------------
+* ``recvbuf``: Address of receive buffer (choice).
+* ``request``: Request (handle, non-blocking and persistent only).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Scatter` is the inverse operation to :ref:`MPI_Gather`.
+
+The outcome is as if the root executed n send operations,
+
+.. code-block:: c
+
+       MPI_Send(sendbuf + i * sendcount * extent(sendtype), sendcount,
+                sendtype, i, ...);
+
+and each process executed a receive,
+
+.. code-block:: c
+
+       MPI_Recv(recvbuf, recvcount, recvtype, i, ...).;
+
+An alternative description is that the root sends a message with
+``MPI_Send(sendbuf, sendcount * n, sendtype, ...)``. This message
+is split into *n* equal segments, the ith segment is sent to the ith
+process in the group, and each process receives this message as above.
+
+The send buffer is ignored for all nonroot processes.
+
+The type signature associated with *sendcount*, *sendtype* at the root
+must be equal to the type signature associated with *recvcount*,
+*recvtype* at all processes (however, the type maps may be different).
+This implies that the amount of data sent must be equal to the amount of
+data received, pairwise between each process and the root. Distinct type
+maps between sender and receiver are still allowed.
+
+All arguments to the function are significant on process *root*, while
+on other processes, only arguments *recvbuf*, *recvcount*, *recvtype*,
+*root*, *comm* are significant. The arguments *root* and *comm* must
+have identical values on all processes.
+
+The specification of counts and types should not cause any location on
+the root to be read more than once.
+
+**Rationale:** Though not needed, the last restriction is imposed so as
+to achieve symmetry with :ref:`MPI_Gather`, where the corresponding restriction
+(a multiple-write restriction) is necessary.
+
+**Example:** The reverse of Example 1 in the :ref:`MPI_Gather` manpage. Scatter
+sets of 100 ints from the root to each process in the group.
+
+.. code-block:: c
+
+           MPI_Comm comm;
+           int gsize,*sendbuf;
+           int root, rbuf[100];
+           ...
+           MPI_Comm_size(comm, &gsize);
+           sendbuf = (int *)malloc(gsize*100*sizeof(int));
+           ...
+           MPI_Scatter(sendbuf, 100, MPI_INT, rbuf, 100,
+                       MPI_INT, root, comm);
+
+
+USE OF IN-PLACE OPTION
+----------------------
+
+When the communicator is an intracommunicator, you can perform a scatter
+operation in-place (the output buffer is used as the input buffer). Use
+the variable MPI_IN_PLACE as the value of the root process *recvbuf*. In
+this case, *recvcount* and *recvtype* are ignored, and the root process
+sends no data to itself.
+
+Note that MPI_IN_PLACE is a special kind of value; it has the same
+restrictions on its use as MPI_BOTTOM.
+
+Because the in-place option converts the receive buffer into a
+send-and-receive buffer, a Fortran binding that includes INTENT must
+mark these as INOUT, not OUT.
+
+
+WHEN COMMUNICATOR IS AN INTER-COMMUNICATOR
+------------------------------------------
+
+When the communicator is an inter-communicator, the root process in the
+first group sends data to all processes in the second group. The first
+group defines the root process. That process uses MPI_ROOT as the value
+of its *root* argument. The remaining processes use ``MPI_PROC_NULL`` as the
+value of their *root* argument. All processes in the second group use
+the rank of that root process in the first group as the value of their
+*root* argument. The receive buffer argument of the root process in the
+first group must be consistent with the receive buffer argument of the
+processes in the second group.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Scatterv`
+   * :ref:`MPI_Gather`
+   * :ref:`MPI_Gatherv`
diff --git a/docs/man-openmpi/man3/MPI_Scatter_init.3.rst b/docs/man-openmpi/man3/MPI_Scatter_init.3.rst
new file mode 100644
index 00000000000..f20b8eff23f
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Scatter_init.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_scatter_init:
+
+MPI_Scatter_init
+================
+    .. include_body
+
+.. include:: ../man3/MPI_Scatter.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Scatterv.3.rst b/docs/man-openmpi/man3/MPI_Scatterv.3.rst
new file mode 100644
index 00000000000..fe455caa5df
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Scatterv.3.rst
@@ -0,0 +1,178 @@
+.. _mpi_scatterv:
+
+
+MPI_Scatterv
+============
+
+.. include_body
+
+:ref:`MPI_Scatterv`, :ref:`MPI_Iscatterv`, :ref:`MPI_Scatterv_init` - Scatters a buffer
+in parts to all tasks in a group.
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Scatterv, MPI_Iscatterv, MPI_Scatterv_init
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_scatterv.rst
+
+INPUT PARAMETERS
+----------------
+* ``sendbuf``: Address of send buffer (choice, significant only at root).
+* ``sendcounts``: Integer array (of length group size) specifying the number of elements to send to each processor.
+* ``displs``: Integer array (of length group size). Entry i specifies the displacement (relative to sendbuf) from which to take the outgoing data to process i.
+* ``sendtype``: Datatype of send buffer elements (handle).
+* ``recvcount``: Number of elements in receive buffer (integer).
+* ``recvtype``: Datatype of receive buffer elements (handle).
+* ``root``: Rank of sending process (integer).
+* ``comm``: Communicator (handle).
+* ``info``: Info (handle, persistent only).
+
+OUTPUT PARAMETERS
+-----------------
+* ``recvbuf``: Address of receive buffer (choice).
+* ``request``: Request (handle, non-blocking and persistent only).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Scatterv` is the inverse operation to :ref:`MPI_Gatherv`.
+
+:ref:`MPI_Scatterv` extends the functionality of :ref:`MPI_Scatter` by allowing a
+varying count of data to be sent to each process, since *sendcounts* is
+now an array. It also allows more flexibility as to where the data is
+taken from on the root, by providing the new argument, *displs*.
+
+The outcome is as if the root executed *n* send operations,
+
+.. code-block:: c
+
+       MPI_Send(sendbuf + displs[i] * extent(sendtype),
+                sendcounts[i], sendtype, i, ...);
+
+       // and each process executed a receive,
+
+       MPI_Recv(recvbuf, recvcount, recvtype, root, ...)
+
+The send buffer is ignored for all nonroot processes.
+
+The type signature implied by *sendcount*\ [*i*], *sendtype* at the root
+must be equal to the type signature implied by *recvcount*, *recvtype*
+at process *i* (however, the type maps may be different). This implies
+that the amount of data sent must be equal to the amount of data
+received, pairwise between each process and the root. Distinct type maps
+between sender and receiver are still allowed.
+
+All arguments to the function are significant on process *root*, while
+on other processes, only arguments *recvbuf*, *recvcount*, *recvtype*,
+*root*, *comm* are significant. The arguments *root* and *comm* must
+have identical values on all processes.
+
+The specification of counts, types, and displacements should not cause
+any location on the root to be read more than once.
+
+**Example 1:** The reverse of Example 5 in the :ref:`MPI_Gatherv` manpage. We
+have a varying stride between blocks at sending (root) side, at the
+receiving side we receive 100 - *i* elements into the *i*\ th column of
+a 100 x 150 C array at process *i*.
+
+.. code-block:: c
+
+           MPI_Comm comm;
+           int gsize,recvarray[100][150],*rptr;
+           int root, *sendbuf, myrank, bufsize, *stride;
+           MPI_Datatype rtype;
+           int i, *displs, *scounts, offset;
+           ...
+           MPI_Comm_size( comm, &gsize);
+           MPI_Comm_rank( comm, &myrank );
+
+           stride = (int *)malloc(gsize*sizeof(int));
+           ...
+           /* stride[i] for i = 0 to gsize-1 is set somehow
+            * sendbuf comes from elsewhere
+            */
+           ...
+           displs = (int *)malloc(gsize*sizeof(int));
+           scounts = (int *)malloc(gsize*sizeof(int));
+           offset = 0;
+           for (i=0; i<gsize; ++i) {
+               displs[i] = offset;
+               offset += stride[i];
+               scounts[i] = 100 - i;
+           }
+           /* Create datatype for the column we are receiving
+            */
+           MPI_Type_vector( 100-myrank, 1, 150, MPI_INT, &rtype);
+           MPI_Type_commit( &rtype );
+           rptr = &recvarray[0][myrank];
+           MPI_Scatterv(sendbuf, scounts, displs, MPI_INT,
+                        rptr, 1, rtype, root, comm);
+
+**Example 2:** The reverse of Example 1 in the MPI_Gather manpage. The
+root process scatters sets of 100 ints to the other processes, but the
+sets of 100 are stride ints apart in the sending buffer. Requires use of
+:ref:`MPI_Scatterv`, where *stride* >= 100.
+
+.. code-block:: c
+
+       MPI_Comm comm;
+       int gsize,*sendbuf;
+       int root, rbuf[100], i, *displs, *scounts;
+
+       ...
+
+       MPI_Comm_size(comm, &gsize);
+       sendbuf = (int *)malloc(gsize*stride*sizeof(int));
+       ...
+       displs = (int *)malloc(gsize*sizeof(int));
+       scounts = (int *)malloc(gsize*sizeof(int));
+       for (i=0; i<gsize; ++i) {
+          displs[i] = i*stride;
+          scounts[i] = 100;
+       }
+       MPI_Scatterv(sendbuf, scounts, displs, MPI_INT,
+                    rbuf, 100, MPI_INT, root, comm);
+
+
+USE OF IN-PLACE OPTION
+----------------------
+
+When the communicator is an intracommunicator, you can perform a scatter
+operation in-place (the output buffer is used as the input buffer). Use
+the variable MPI_IN_PLACE as the value of the root process *recvbuf*. In
+this case, *recvcount* and *recvtype* are ignored, and the root process
+sends no data to itself.
+
+Note that MPI_IN_PLACE is a special kind of value; it has the same
+restrictions on its use as MPI_BOTTOM.
+
+Because the in-place option converts the receive buffer into a
+send-and-receive buffer, a Fortran binding that includes INTENT must
+mark these as INOUT, not OUT.
+
+
+WHEN COMMUNICATOR IS AN INTER-COMMUNICATOR
+------------------------------------------
+
+When the communicator is an inter-communicator, the root process in the
+first group sends data to all processes in the second group. The first
+group defines the root process. That process uses MPI_ROOT as the value
+of its *root* argument. The remaining processes use ``MPI_PROC_NULL`` as the
+value of their *root* argument. All processes in the second group use
+the rank of that root process in the first group as the value of their
+*root* argument. The receive buffer argument of the root process in the
+first group must be consistent with the receive buffer argument of the
+processes in the second group.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Gather`
+   * :ref:`MPI_Gatherv`
+   * :ref:`MPI_Scatter`
diff --git a/docs/man-openmpi/man3/MPI_Scatterv_init.3.rst b/docs/man-openmpi/man3/MPI_Scatterv_init.3.rst
new file mode 100644
index 00000000000..9963769e9f0
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Scatterv_init.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_scatterv_init:
+
+MPI_Scatterv_init
+=================
+    .. include_body
+
+.. include:: ../man3/MPI_Scatterv.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Send.3.rst b/docs/man-openmpi/man3/MPI_Send.3.rst
new file mode 100644
index 00000000000..3d4abd4fbd9
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Send.3.rst
@@ -0,0 +1,49 @@
+.. _mpi_send:
+
+
+MPI_Send
+========
+
+.. include_body
+
+:ref:`MPI_Send` |mdash| Performs a standard-mode blocking send.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_send.rst
+
+INPUT PARAMETERS
+----------------
+* ``buf``: Initial address of send buffer (choice).
+* ``count``: Number of elements in send buffer (nonnegative integer).
+* ``datatype``: Datatype of each send buffer element (handle).
+* ``dest``: Rank of destination (integer).
+* ``tag``: Message tag (integer).
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Send` performs a standard-mode, blocking send.
+
+
+NOTE
+----
+
+This routine will block until the message is sent to the destination.
+For an in-depth explanation of the semantics of the standard-mode send,
+refer to the `MPI Standard <https://www.mpi-forum.org/docs/>`_.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Isend`
+   * :ref:`MPI_Bsend`
+   * :ref:`MPI_Recv`
diff --git a/docs/man-openmpi/man3/MPI_Send_init.3.rst b/docs/man-openmpi/man3/MPI_Send_init.3.rst
new file mode 100644
index 00000000000..26e6cbcd779
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Send_init.3.rst
@@ -0,0 +1,50 @@
+.. _mpi_send_init:
+
+
+MPI_Send_init
+=============
+
+.. include_body
+
+:ref:`MPI_Send_init` |mdash| Builds a handle for a standard send.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_send_init.rst
+
+INPUT PARAMETERS
+----------------
+* ``buf``: Initial address of send buffer (choice).
+* ``count``: Number of elements to send (integer).
+* ``datatype``: Type of each element (handle).
+* ``dest``: Rank of destination (integer).
+* ``tag``: Message tag (integer).
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``request``: Communication request (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Creates a persistent communication request for a standard mode send
+operation, and binds to it all the arguments of a send operation.
+
+A communication (send or receive) that uses a persistent request is
+initiated by the function :ref:`MPI_Start` or :ref:`MPI_Startall`.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Bsend_init`
+   * :ref:`MPI_Ssend_init`
+   * :ref:`MPI_Rsend_init`
+   * :ref:`MPI_Recv_init`
+   * :ref:`MPI_Start`
+   * :ref:`MPI_Startall`
+   * :ref:`MPI_Request_free`
diff --git a/docs/man-openmpi/man3/MPI_Sendrecv.3.rst b/docs/man-openmpi/man3/MPI_Sendrecv.3.rst
new file mode 100644
index 00000000000..d36c9f4a04c
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Sendrecv.3.rst
@@ -0,0 +1,82 @@
+.. _mpi_sendrecv:
+
+
+MPI_Sendrecv
+============
+
+.. include_body
+
+:ref:`MPI_Sendrecv` |mdash| Sends and receives a message.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_sendrecv.rst
+
+INPUT PARAMETERS
+----------------
+* ``sendbuf``: Initial address of send buffer (choice).
+* ``sendcount``: Number of elements to send (integer).
+* ``sendtype``: Type of elements in send buffer (handle).
+* ``dest``: Rank of destination (integer).
+* ``sendtag``: Send tag (integer).
+* ``recvcount``: Maximum number of elements to receive (integer).
+* ``recvtype``: Type of elements in receive buffer (handle).
+* ``source``: Rank of source (integer).
+* ``recvtag``: Receive tag (integer).
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``recvbuf``: Initial address of receive buffer (choice).
+* ``status``: Status object (status). This refers to the receive operation.
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The send-receive operations combine in one call the sending of a message
+to one destination and the receiving of another message, from another
+process. The two (source and destination) are possibly the same. A
+send-receive operation is useful for executing a shift operation across
+a chain of processes. If blocking sends and receives are used for such a
+shift, then one needs to order the sends and receives correctly (for
+example, even processes send, then receive; odd processes receive first,
+then send) in order to prevent cyclic dependencies that may lead to
+deadlock. When a send-receive operation is used, the communication
+subsystem takes care of these issues. The send-receive operation can be
+used in conjunction with the functions described in the "Process
+Topologies" chapter in the `MPI Standard
+<https://www.mpi-forum.org/docs/>`_ in order to perform shifts on
+various logical topologies. Also, a send-receive operation is useful for
+implementing remote procedure calls.
+
+A message sent by a send-receive operation can be received by a regular
+receive operation or probed by a probe operation; a send-receive
+operation can receive a message sent by a regular send operation.
+
+:ref:`MPI_Sendrecv` executes a blocking send and receive operation. Both send
+and receive use the same communicator, but possibly different tags. The
+send buffer and receive buffers must be disjoint, and may have different
+lengths and datatypes.
+
+If your application does not need to examine the *status* field, you can
+save resources by using the predefined constant ``MPI_STATUS_IGNORE`` as a
+special value for the *status* argument.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+Note that per the "Return Status" section in the "Point-to-Point
+Communication" chapter in the `MPI Standard
+<https://www.mpi-forum.org/docs/>`_, MPI errors on messages received
+by :ref:`MPI_Sendrecv` do not set the ``status.MPI_ERROR`` field in
+the returned *status*.  The error code is always passed to the
+back-end error handler and may be passed back to the caller through
+the return value of :ref:`MPI_Sendrecv` if the back-end error handler
+returns it.  The pre-defined MPI error handler ``MPI_ERRORS_RETURN``
+exhibits this behavior, for example.
+
+.. seealso::
+   * :ref:`MPI_Sendrecv_replace`
diff --git a/docs/man-openmpi/man3/MPI_Sendrecv_replace.3.rst b/docs/man-openmpi/man3/MPI_Sendrecv_replace.3.rst
new file mode 100644
index 00000000000..e0dab236687
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Sendrecv_replace.3.rst
@@ -0,0 +1,81 @@
+.. _mpi_sendrecv_replace:
+
+
+MPI_Sendrecv_replace
+====================
+
+.. include_body
+
+:ref:`MPI_Sendrecv_replace` |mdash| Sends and receives a message using a single
+buffer.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_sendrecv_replace.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``buf``: Initial address of send and receive buffer (choice).
+
+INPUT PARAMETERS
+----------------
+* ``count``: Number of elements in send and receive buffer (integer).
+* ``datatype``: Type of elements to send and receive (handle).
+* ``dest``: Rank of destination (integer).
+* ``sendtag``: Send message tag (integer).
+* ``source``: Rank of source (integer).
+* ``recvtag``: Receive message tag (integer).
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``status``: Status object (status).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The send-receive operations combine in one call the sending of a message
+to one destination and the receiving of another message, from another
+process. The two (source and destination) are possibly the same. A
+send-receive operation is useful for executing a shift operation across
+a chain of processes. If blocking sends and receives are used for such a
+shift, then one needs to order the sends and receives correctly (for
+example, even processes send, then receive; odd processes receive first,
+then send) in order to prevent cyclic dependencies that may lead to
+deadlock. When a send-receive operation is used, the communication
+subsystem takes care of these issues. The send-receive operation can be
+used in conjunction with the functions described in Chapter 6 of the MPI
+Standard, "Process Topologies," in order to perform shifts on various
+logical topologies. Also, a send-receive operation is useful for
+implementing remote procedure calls.
+
+A message sent by a send-receive operation can be received by a regular
+receive operation or probed by a probe operation; a send-receive
+operation can receive a message sent by a regular send operation.
+
+:ref:`MPI_Sendrecv_replace` executes a blocking send and receive. The same
+buffer is used both for the send and for the receive, so that the
+message sent is replaced by the message received.
+
+The semantics of a send-receive operation is what would be obtained if
+the caller forked two concurrent threads, one to execute the send, and
+one to execute the receive, followed by a join of these two threads.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+Note that per the "Return Status" section in the "Point-to-Point
+Communication" chapter in the `MPI Standard
+<https://www.mpi-forum.org/docs/>`_, MPI errors on messages received
+by :ref:`MPI_Sendrecv_replace` do not set the ``status.MPI_ERROR``
+field in the returned *status*.  The error code is always passed to
+the back-end error handler and may be passed back to the caller
+through the return value of :ref:`MPI_Sendrecv_replace` if the
+back-end error handler returns it.  The pre-defined MPI error handler
+``MPI_ERRORS_RETURN`` exhibits this behavior, for example.
+
+.. seealso::
+   * :ref:`MPI_Sendrecv`
diff --git a/docs/man-openmpi/man3/MPI_Session_c2f.3.rst b/docs/man-openmpi/man3/MPI_Session_c2f.3.rst
new file mode 100644
index 00000000000..e846396f492
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Session_c2f.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_session_c2f:
+
+MPI_Session_c2f
+===============
+
+.. include_body
+
+.. include:: ../man3/MPI_Comm_f2c.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_Session_call_errhandler.3.rst b/docs/man-openmpi/man3/MPI_Session_call_errhandler.3.rst
new file mode 100644
index 00000000000..c61345d9571
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Session_call_errhandler.3.rst
@@ -0,0 +1,45 @@
+.. _mpi_session_call_errhandler:
+
+MPI_Session_call_errhandler
+===========================
+
+.. include_body
+
+:ref:`MPI_Session_call_errhandler` |mdash| Passes the supplied error code to the error
+handler assigned to a session
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_session_call_errhandler.rst
+
+INPUT PARAMETER
+---------------
+
+* ``session`` : session with error handler (handle).
+* ``errorcode`` : error code (integer).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This function invokes the error handler assigned to the session
+session with the supplied error code errorcode. If the error handler was
+successfully called, the process is not aborted, and the error handler
+returns, this function returns MPI_SUCCESS.
+
+NOTES
+-----
+
+Users should note that the default error handler is
+MPI_ERRORS_ARE_FATAL. Thus, calling this function will abort the
+processes in session if the default error handler has not been changed.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso:: :ref:`MPI_Session_create_errhandler`
diff --git a/docs/man-openmpi/man3/MPI_Session_create_errhandler.3.rst b/docs/man-openmpi/man3/MPI_Session_create_errhandler.3.rst
new file mode 100644
index 00000000000..dab59398e8e
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Session_create_errhandler.3.rst
@@ -0,0 +1,51 @@
+.. _mpi_session_create_errhandler:
+
+MPI_Session_create_errhandler
+=============================
+
+.. include_body
+
+:ref:`MPI_Session_create_errhandler` |mdash| Creates an error handler that can be
+attached to sessions
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_session_create_errhandler.rst
+
+INPUT PARAMETER
+---------------
+
+* ``function`` : User-defined error handling procedure (function).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``errhandler`` : MPI error handler (handle).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Session_create_errhandler` creates an error handler that can be
+attached to sessions. This ``function`` is identical to
+:ref:`MPI_Errhandler_create`, the use of which is deprecated. In C, the
+user routine should be a ``function`` of type
+``MPI_Session_errhandler_function``, which is defined as
+
+.. code-block:: c
+
+   typedef void MPI_Session_errhandler_function(MPI_Session *, int *, ...);
+
+The first argument is the session in use. The second is the error code
+to be returned by the MPI routine that raised the error. This typedef
+replaces ``MPI_Handler_function``, the use of which is deprecated. In
+Fortran, the user routine should be of this form:
+
+.. code-block:: fortran
+
+   SUBROUTINE SESSION_ERRHANDLER_FUNCTION(SESSION, ERROR_CODE, ...)
+      INTEGER SESSION, ERROR_CODE
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Session_f2c.3.rst b/docs/man-openmpi/man3/MPI_Session_f2c.3.rst
new file mode 100644
index 00000000000..43c1ac0207b
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Session_f2c.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_session_f2c:
+
+MPI_Session_f2c
+===============
+
+.. include_body
+
+.. include:: ../man3/MPI_Comm_f2c.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_Session_finalize.3.rst b/docs/man-openmpi/man3/MPI_Session_finalize.3.rst
new file mode 100644
index 00000000000..663f6394315
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Session_finalize.3.rst
@@ -0,0 +1,55 @@
+.. _mpi_session_finalize:
+
+MPI_Session_finalize
+====================
+
+.. include_body
+
+:ref:`MPI_Session_finalize` |mdash| releases all MPI state associated with a session
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_session_finalize.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``session`` : session to be finalized (handle)
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Session_finalize` releases all MPI state associated with the supplied
+session. Every instantiated session must be finalized using
+:ref:`MPI_Session_finalize`. The handle session is set to MPI_SESSION_NULL by
+the call.
+
+NOTES
+-----
+
+Before an MPI process invokes :ref:`MPI_Session_finalize`, the process must
+perform all MPI calls needed to complete its involvement in MPI
+communications: it must locally complete all MPI operations that it
+initiated and it must execute matching calls needed to complete MPI
+communications initiated by other processes. The call to
+:ref:`MPI_Session_finalize` does not free objects created by MPI calls; these
+objects are freed using MPI_XXX_FREE calls. :ref:`MPI_Session_finalize` may be
+synchronizing on any or all of the groups associated with communicators,
+windows, or files derived from the session and not disconnected, freed,
+or closed, respectively, before the call to :ref:`MPI_Session_finalize`
+procedure. :ref:`MPI_Session_finalize` behaves as if all such synchronizations
+occur concurrently. As :ref:`MPI_Comm_free` may mark a communicator for freeing
+later, :ref:`MPI_Session_finalize` may be synchronizing on the group associated
+with a communicator that is only freed (with MPI_Comm_free) rather than
+disconnected (with MPI_Comm_disconnect).
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso:: :ref:`MPI_Session_init`
diff --git a/docs/man-openmpi/man3/MPI_Session_get_errhandler.3.rst b/docs/man-openmpi/man3/MPI_Session_get_errhandler.3.rst
new file mode 100644
index 00000000000..2d466b3037f
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Session_get_errhandler.3.rst
@@ -0,0 +1,34 @@
+.. _mpi_session_get_errhandler:
+
+
+MPI_Session_get_errhandler
+==========================
+
+.. include_body
+
+:ref:`MPI_Session_get_errhandler` |mdash| Retrieves error handler associated with a
+session.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_session_get_errhandler.rst
+
+INPUT PARAMETER
+---------------
+* ``session``: Session (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``errhandler``: New error handler for session (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Session_get_errhandler` retrieves the error handler currently associated
+with a session.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Session_get_info.3.rst b/docs/man-openmpi/man3/MPI_Session_get_info.3.rst
new file mode 100644
index 00000000000..877c3b8d717
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Session_get_info.3.rst
@@ -0,0 +1,48 @@
+.. _mpi_session_get_info:
+
+MPI_Session_get_info
+====================
+
+.. include_body
+
+:ref:`MPI_Session_get_info` |mdash| Returns an info object containing the hints of an
+MPI Session
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_session_get_info.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``session`` : session (handle)
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``info_used`` : info object (handle)
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Session_get_info` returns a new info object containing the hints of
+the MPI Session associated with session. The current setting of all
+hints related to this MPI Session is returned in info_used. An MPI
+implementation is required to return all hints that are supported by the
+implementation and have default values specified; any user-supplied
+hints that were not ignored by the implementation; and any additional
+hints that were set by the implementation. If no such hints exist, a
+handle to a newly created info object is returned that contains no
+key/value pair.
+
+NOTES
+-----
+
+The user is responsible for freeing info_used via :ref:`MPI_Info_free`.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso:: :ref:`MPI_Session_init`
diff --git a/docs/man-openmpi/man3/MPI_Session_get_nth_pset.3.rst b/docs/man-openmpi/man3/MPI_Session_get_nth_pset.3.rst
new file mode 100644
index 00000000000..2eb8f63e446
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Session_get_nth_pset.3.rst
@@ -0,0 +1,57 @@
+.. _mpi_session_get_nth_pset:
+
+MPI_Session_get_nth_pset
+========================
+
+.. include_body
+
+:ref:`MPI_Session_get_nth_pset` |mdash| Query runtime for name of the nth process set
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_session_get_nth_pset.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``session`` : session (handle)
+* ``info`` : info object (handle)
+* ``n`` : index of the desired process set name (integer)
+
+INPUT/OUTPUT PARAMETER
+^^^^^^^^^^^^^^^^^^^^^^
+
+* ``pset_len`` : length of the pset_name argument (integer)
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``pset_name`` : name of the nth process set (string)
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Session_get_nth_pset` returns the name of the nth process set in the
+supplied pset_name buffer. pset_len is the size of the buffer needed to
+store the nth process set name. If the pset_len passed into the function
+is less than the actual buffer size needed for the process set name,
+then the string value returned in pset_name is truncated. If pset_len is
+set to 0, pset_name is not changed. On return, the value of pset_len
+will be set to the required buffer size to hold the process set name. In
+C, pset_len includes the required space for the null terminator. In C,
+this function returns a null terminated string in all cases where the
+pset_len input value is greater than 0.
+
+NOTES
+-----
+
+Process set names have an implementation-defined maximum length of
+MPI_MAX_PSET_NAME_LEN characters. MPI_MAX_PSET_NAME_LEN shall have a
+value of at least 63.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso:: :ref:`MPI_Session_init`
diff --git a/docs/man-openmpi/man3/MPI_Session_get_num_psets.3.rst b/docs/man-openmpi/man3/MPI_Session_get_num_psets.3.rst
new file mode 100644
index 00000000000..032080dd172
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Session_get_num_psets.3.rst
@@ -0,0 +1,53 @@
+.. _mpi_session_get_num_psets:
+
+MPI_Session_get_num_psets
+=========================
+
+.. include_body
+
+:ref:`MPI_Session_get_num_psets` |mdash| Query runtime for number of available
+process sets
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_session_get_num_psets.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``session`` : session (handle)
+* ``info`` : info object (handle)
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``npset_names`` : number of available process sets (non-negtive integer)
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Session_get_num_psets` is used to query the runtime for the number of
+available process sets in which the calling MPI process is a member. An
+MPI implementation is allowed to increase the number of available
+process sets during the execution of an MPI application when new process
+sets become available. However, MPI implementations are not allowed to
+change the index of a particular process set name, or to change the name
+of the process set at a particular index, or to delete a process set
+name once it has been added.
+
+NOTES
+-----
+
+When a process set becomes invalid, for example, when some processes
+become unreachable due to failures in the communication system,
+subsequent usage of the process set name may raise an error. For
+example, creating an MPI_Group from such a process set might succeed
+because it is a local operation, but creating an MPI_Comm from that
+group and attempting collective communication may raise an error.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso:: :ref:`MPI_Session_init`
diff --git a/docs/man-openmpi/man3/MPI_Session_get_pset_info.3.rst b/docs/man-openmpi/man3/MPI_Session_get_pset_info.3.rst
new file mode 100644
index 00000000000..46dec3b75b0
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Session_get_pset_info.3.rst
@@ -0,0 +1,46 @@
+.. _mpi_session_get_pset_info:
+
+MPI_Session_get_pset_info
+=========================
+
+.. include_body
+
+:ref:`MPI_Session_get_pset_info` |mdash| Returns an info object containing properties
+of a specific process set
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_session_get_pset_info.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``session`` : session (handle)
+* ``pset_name`` : name of process set (string)
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``info`` : info object (handle)
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Session_get_pset_info` is used to query properties of a specific
+process set. The returned info object can be queried with existing MPI
+info object query functions. One key/value pair must be defined,
+"mpi_size". The value of the "mpi_size" key specifies the number of MPI
+processes in the process set.
+
+NOTES
+-----
+
+The user is responsible for freeing the returned info object via
+:ref:`MPI_Info_free`.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso:: :ref:`MPI_Session_init`
diff --git a/docs/man-openmpi/man3/MPI_Session_init.3.rst b/docs/man-openmpi/man3/MPI_Session_init.3.rst
new file mode 100644
index 00000000000..3dd3263a020
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Session_init.3.rst
@@ -0,0 +1,49 @@
+.. _mpi_session_init:
+
+MPI_Session_init
+================
+
+.. include_body
+
+:ref:`MPI_Session_init` |mdash| Creates a new session handle
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_session_init.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``info`` : info object (handle)
+* ``errhandler`` : error handler to be attached to the returned session
+   (handle)
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``session`` : New session (handle).
+* ``ierror`` : Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Session_init` is used to instantiate an MPI Session. The returned
+session handle can be used to query the runtime system about
+characteristics of the job within which the process is running, as well
+as other system resources. An application can make multiple calls to
+:ref:`MPI_Session_init` and the related :ref:`MPI_Session_finalize` routine.
+
+NOTES
+-----
+
+The info argument is used to request MPI functionality requirements and
+possible MPI implementation specific capabilities.
+
+The errhandler argument specifies an error handler to invoke in the
+event that the Session instantiation call encounters an error.
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso:: :ref:`MPI_Session_get_num_psets` MPI_Session_group_from_pset
diff --git a/docs/man-openmpi/man3/MPI_Session_set_errhandler.3.rst b/docs/man-openmpi/man3/MPI_Session_set_errhandler.3.rst
new file mode 100644
index 00000000000..7ada2ebb545
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Session_set_errhandler.3.rst
@@ -0,0 +1,35 @@
+.. _mpi_session_set_errhandler:
+
+
+MPI_Session_set_errhandler
+==========================
+
+.. include_body
+
+:ref:`MPI_Session_set_errhandler` |mdash| Attaches a new error handler to a
+session.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_session_set_errhandler.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``session``: Session (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``errhandler``: New error handler for session (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Session_set_errhandler` attaches a new error handler to a session.
+The error handler must be either a predefined error handler or an error
+handler created by a call to :ref:`MPI_Session_create_errhandler`.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Sizeof.3.rst b/docs/man-openmpi/man3/MPI_Sizeof.3.rst
new file mode 100644
index 00000000000..c58d61eb499
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Sizeof.3.rst
@@ -0,0 +1,42 @@
+.. _mpi_sizeof:
+
+
+MPI_Sizeof
+==========
+
+.. include_body
+
+:ref:`MPI_Sizeof` |mdash| Returns the size, in bytes, of the given type
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_sizeof.rst
+
+INPUT PARAMETER
+---------------
+* ``X``: A Fortran variable of numeric intrinsic type (choice).
+
+OUTPUT PARAMETERS
+-----------------
+* ``SIZE``: Size of machine representation of that type (integer).
+* ``ierror``: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_SIZEOF` returns the size (in bytes) of the machine representation of
+the given variable. It is a generic Fortran type and has a Fortran
+binding only. This routine is similar to the sizeof builtin in C.
+However, if given an array argument, it returns the size of the base
+element, not the size of the whole array.
+
+
+NOTES
+-----
+
+This function is not available in C because it is not necessary.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Ssend.3.rst b/docs/man-openmpi/man3/MPI_Ssend.3.rst
new file mode 100644
index 00000000000..e585558f892
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Ssend.3.rst
@@ -0,0 +1,38 @@
+.. _mpi_ssend:
+
+
+MPI_Ssend
+=========
+
+.. include_body
+
+:ref:`MPI_Ssend` |mdash| Standard synchronous send.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_ssend.rst
+
+INPUT PARAMETERS
+----------------
+* ``buf``: Initial address of send buffer (choice).
+* ``count``: Number of elements in send buffer (nonnegative integer).
+* ``datatype``: Datatype of each send buffer element (handle).
+* ``dest``: Rank of destination (integer).
+* ``tag``: Message tag (integer).
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Ssend` performs a synchronous-mode, blocking send. See the
+`MPI Standard <https://www.mpi-forum.org/docs/>`_ for
+more detailed information about such sends.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Ssend_init.3.rst b/docs/man-openmpi/man3/MPI_Ssend_init.3.rst
new file mode 100644
index 00000000000..1d72c43872e
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Ssend_init.3.rst
@@ -0,0 +1,50 @@
+.. _mpi_ssend_init:
+
+
+MPI_Ssend_init
+==============
+
+.. include_body
+
+:ref:`MPI_Ssend_init` |mdash| Builds a handle for a synchronous send.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_ssend_init.rst
+
+INPUT PARAMETERS
+----------------
+* ``buf``: Initial address of send buffer (choice).
+* ``count``: Number of elements to send (integer).
+* ``datatype``: Type of each element (handle).
+* ``dest``: Rank of destination (integer).
+* ``tag``: Message tag (integer).
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``request``: Communication request (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Creates a persistent communication object for a synchronous mode send
+operation, and binds to it all the arguments of a send operation.
+
+A communication (send or receive) that uses a persistent request is
+initiated by the function :ref:`MPI_Start`.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Bsend_init`
+   * :ref:`MPI_Send_init`
+   * :ref:`MPI_Rsend_init`
+   * :ref:`MPI_Recv_init`
+   * :ref:`MPI_Start`
+   * :ref:`MPI_Startall`
+   * :ref:`MPI_Ssend`
diff --git a/docs/man-openmpi/man3/MPI_Start.3.rst b/docs/man-openmpi/man3/MPI_Start.3.rst
new file mode 100644
index 00000000000..4c5b6b44e35
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Start.3.rst
@@ -0,0 +1,61 @@
+.. _mpi_start:
+
+
+MPI_Start
+=========
+
+.. include_body
+
+:ref:`MPI_Start` |mdash| Initiates a communication using a persistent request
+handle.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_start.rst
+
+INPUT PARAMETER
+---------------
+* ``request``: Communication request (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+A communication (send or receive) that uses a persistent request is
+initiated by the function :ref:`MPI_Start`.
+
+The argument, request, is a handle returned by one of the persistent
+communication-request initialization functions (:ref:`MPI_Send_init`,
+:ref:`MPI_Bsend_init`, :ref:`MPI_Ssend_init`, :ref:`MPI_Rsend_init`, MPI_Recv_init). The
+associated request should be inactive and becomes active once the call
+is made.
+
+If the request is for a send with ready mode, then a matching receive
+should be posted before the call is made. From the time the call is made
+until after the operation completes, the communication buffer should not
+be accessed.
+
+The call is local, with semantics similar to the nonblocking
+communication operations (see Section 3.7 in the `MPI Standard
+<https://www.mpi-forum.org/docs/>`_,
+"Nonblocking Communication.") That is, a call to :ref:`MPI_Start` with a
+request created by :ref:`MPI_Send_init` starts a communication in the same
+manner as a call to :ref:`MPI_Isend`; a call to :ref:`MPI_Start` with a request
+created by :ref:`MPI_Bsend_init` starts a communication in the same manner as a
+call to :ref:`MPI_Ibsend`; and so on.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Bsend_init`
+   * :ref:`MPI_Rsend_init`
+   * :ref:`MPI_Send_init`
+   * MPI_Sssend_init
+   * :ref:`MPI_Recv_init`
+   * :ref:`MPI_Startall`
diff --git a/docs/man-openmpi/man3/MPI_Startall.3.rst b/docs/man-openmpi/man3/MPI_Startall.3.rst
new file mode 100644
index 00000000000..8115921e4c8
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Startall.3.rst
@@ -0,0 +1,81 @@
+.. _mpi_startall:
+
+
+MPI_Startall
+============
+
+.. include_body
+
+:ref:`MPI_Startall` |mdash| Starts a collection of requests.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_startall.rst
+
+INPUT PARAMETER
+---------------
+* ``count``: List length (integer).
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``array_of_requests``: Array of requests (array of handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Starts all communications associated with requests in array_of_requests.
+A call to MPI_Startall(count, array_of_requests) has the same effect as
+calls to :ref:`MPI_Start` (&array_of_requests[i]), executed for i=0 ,...,
+count-1, in some arbitrary order.
+
+A communication started with a call to :ref:`MPI_Start` or :ref:`MPI_Startall` is
+completed by a call to :ref:`MPI_Wait`, :ref:`MPI_Test`, or one of the derived
+functions :ref:`MPI_Waitany`, :ref:`MPI_Testany`, :ref:`MPI_Waitall`, :ref:`MPI_Testall`,
+:ref:`MPI_Waitsome`, :ref:`MPI_Testsome` (these are described in Section 3.7.5 of the
+`MPI Standard <https://www.mpi-forum.org/docs/>`_, "Multiple
+Completions"). The request becomes inactive
+after successful completion by such a call. The request is not
+deallocated, and it can be activated anew by another :ref:`MPI_Start` or
+:ref:`MPI_Startall` call.
+
+A persistent request is deallocated by a call to
+:ref:`MPI_Request_free` (see Section 3.7.3 of the `MPI Standard
+<https://www.mpi-forum.org/docs/>`_, "Communication Completion").
+
+| The call to :ref:`MPI_Request_free` can occur at any point in the program
+  after the persistent request was created. However, the request will be
+  deallocated only after it becomes inactive. Active receive requests
+  should not be freed. Otherwise, it will not be possible to check that
+  the receive has completed. It is preferable, in general, to free
+  requests when they are inactive. If this rule is followed, then the
+  persistent communication request functions will be invoked in a
+  sequence of the form,
+
+| Create (Start Complete)\* Free
+
+where \* indicates zero or more repetitions. If the same communication
+object is used in several concurrent threads, it is the user's
+responsibility to coordinate calls so that the correct sequence is
+obeyed.
+
+A send operation initiated with :ref:`MPI_Start` can be matched with any
+receive operation and, likewise, a receive operation initiated with
+:ref:`MPI_Start` can receive messages generated by any send operation.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Bsend_init`
+   * :ref:`MPI_Rsend_init`
+   * :ref:`MPI_Send_init`
+   * :ref:`MPI_Ssend_init`
+   * :ref:`MPI_Recv_init`
+   * :ref:`MPI_Start`
+   * :ref:`MPI_Request_free`
diff --git a/docs/man-openmpi/man3/MPI_Status_c2f.3.rst b/docs/man-openmpi/man3/MPI_Status_c2f.3.rst
new file mode 100644
index 00000000000..68782742f9c
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Status_c2f.3.rst
@@ -0,0 +1,8 @@
+.. _mpi_status_c2f:
+
+MPI_Status_c2f
+==============
+.. include_body
+
+.. include:: ../man3/MPI_Status_f2c.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_Status_c2f08.3.rst b/docs/man-openmpi/man3/MPI_Status_c2f08.3.rst
new file mode 100644
index 00000000000..9ce13dc9cae
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Status_c2f08.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_status_c2f08:
+
+MPI_Status_c2f08
+================
+
+.. include_body
+
+.. include:: ../man3/MPI_Status_f082c.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_Status_f082c.3.rst b/docs/man-openmpi/man3/MPI_Status_f082c.3.rst
new file mode 100644
index 00000000000..f6bab38b092
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Status_f082c.3.rst
@@ -0,0 +1,58 @@
+.. _mpi_status_f082c:
+
+MPI_Status_f082c
+================
+
+.. include_body
+
+:ref:`MPI_Status_f082c`, :ref:`MPI_Status_c2f08` - Translates a C status into a
+Fortran 2008 status, or vice versa.
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Status_f082c, MPI_Status_c2f08
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_status_f082c.rst
+
+PARAMETERS
+----------
+
+* ``f08_status`` : mpi_f08-style MPI status object
+* ``c_status`` : C-style MPI status object
+
+DESCRIPTION
+-----------
+
+These two procedures are provided in C to convert from a Fortran 2008
+status (which is a derived type made of integers) to a C status (which
+is a structure), and vice versa. The conversion occurs on all the
+information in status, including that which is hidden. That is, no
+status information is lost in the conversion.
+
+When using :ref:`MPI_Status_f082c`, if ``f08_status`` is a valid Fortran status,
+but not the Fortran value of ``MPI_F08_STATUS_IGNORE`` or
+``MPI_F08_STATUSES_IGNORE``, then :ref:`MPI_Status_f082c` returns in ``c_status`` a
+valid C status with the same content. If ``f08_status`` is the Fortran value
+of ``MPI_STATUS_IGNORE`` or ``MPI_STATUSES_IGNORE``, or if ``f08_status`` is not a
+valid Fortran status, then the call is erroneous.
+
+When using :ref:`MPI_Status_c2f08`, the opposite conversion is applied. If
+``c_status`` is ``MPI_STATUS_IGNORE`` or ``MPI_STATUSES_IGNORE``, or if ``c_status`` is
+not a valid C status, then the call is erroneous.
+
+The input status has the same source, tag and error code values as the
+output status, and returns the same answers when queried for count,
+elements, and cancellation. The conversion function may be called with
+an input status argument that has an undefined error field, in which
+case the value of the error field in the output status argument is
+undefined.
+
+NOTES
+-----
+
+These functions are only available in C; they are not available in any
+of the Fortran MPI interfaces.
+
+
+.. seealso:: :ref:`MPI_Status_c2f` :ref:`MPI_Status_f2f08`
diff --git a/docs/man-openmpi/man3/MPI_Status_f082f.3.rst b/docs/man-openmpi/man3/MPI_Status_f082f.3.rst
new file mode 100644
index 00000000000..3aa492806e3
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Status_f082f.3.rst
@@ -0,0 +1,61 @@
+.. _mpi_status_f082f:
+
+MPI_Status_f082f
+================
+
+.. include_body
+
+:ref:`MPI_Status_f082f`, :ref:`MPI_Status_f2f08` - Translates a Fortran 2008 status
+into a Fortran INTEGER-style status, or vice versa.
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Status_f082f, MPI_Status_f2f08
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_status_f082f.rst
+
+PARAMETERS
+----------
+
+* ``f08_status`` : mpi_f08-style MPI status object
+* ``f_status`` : mpi-style INTEGER MPI status object
+
+DESCRIPTION
+-----------
+
+These two procedures are provided to convert from a Fortran 2008 status
+(which is a derived datatype made of integers) to a Fortran status
+(which is an array of integers), and vice versa. The conversion occurs
+on all the information in status, including that which is hidden. That
+is, no status information is lost in the conversion.
+
+When using :ref:`MPI_Status_f082f`, if ``f08_status`` is a valid Fortran status,
+but not the Fortran value of ``MPI_F08_STATUS_IGNORE`` (in C),
+``MPI_STATUS_IGNORE`` (in Fortran) or ``MPI_F08_STATUSES_IGNORE`` (in C) or
+``MPI_STATUSES_IGNORE`` (in Fortran), then :ref:`MPI_Status_f082f` returns in
+``f_status`` a valid array with the same content. If ``f08_status`` is the C
+value of ``MPI_F08_STATUS_IGNORE`` or ``MPI_F08_STATUSES_IGNORE`` or the Fortran
+value of ``MPI_STATUS_IGNORE`` or ``MPI_STATUSES_IGNORE``, or if ``f08_status`` is
+not a valid Fortran status, then the call is erroneous.
+
+When using :ref:`MPI_Status_f2f08`, the opposite conversion is applied. If
+``f_status`` is ``MPI_STATUS_IGNORE`` or ``MPI_STATUSES_IGNORE``, or if ``f_status`` is
+not a valid Fortran status, then the call is erroneous.
+
+The input status has the same source, tag and error code values as the
+output status, and returns the same answers when queried for count,
+elements, and cancellation. The conversion function may be called with
+an input status argument that has an undefined error field, in which
+case the value of the error field in the output status argument is
+undefined.
+
+NOTES
+-----
+
+The Fortran subroutines for these MPI routines are only available in the
+mpi and mpi_f08 modules (including the type specification for
+``TYPE(MPI_Status)``; they are (intentionally) not available in ``mpif.h``.
+
+
+.. seealso:: :ref:`MPI_Status_c2f` :ref:`MPI_Status_c2f08`
diff --git a/docs/man-openmpi/man3/MPI_Status_f2c.3.rst b/docs/man-openmpi/man3/MPI_Status_f2c.3.rst
new file mode 100644
index 00000000000..2a898b524b1
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Status_f2c.3.rst
@@ -0,0 +1,58 @@
+.. _mpi_status_f2c:
+
+MPI_Status_f2c
+==============
+
+.. include_body
+
+:ref:`MPI_Status_f2c`, :ref:`MPI_Status_f2c` - Translates a C status into a Fortran
+INTEGER-style status, or vice versa.
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Status_f2c, MPI_Status_c2f
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_status_f2c.rst
+
+PARAMETERS
+----------
+
+* ``f_status`` : mpi-style INTEGER MPI status object
+* ``c_status`` : C-style MPI status object
+
+DESCRIPTION
+-----------
+
+These two procedures are provided in C to convert from a Fortran status
+(which is an array of integers) to a C status (which is a structure),
+and vice versa. The conversion occurs on all the information in status,
+including that which is hidden. That is, no status information is lost
+in the conversion.
+
+When using :ref:`MPI_Status_f2c`, if ``f_status`` is a valid Fortran status, but
+not the Fortran value of ``MPI_STATUS_IGNORE`` or ``MPI_STATUSES_IGNORE``, then
+:ref:`MPI_Status_f2c` returns in ``c_status`` a valid C status with the same
+content. If ``f_status`` is the Fortran value of ``MPI_STATUS_IGNORE`` or
+``MPI_STATUSES_IGNORE``, or if ``f_status`` is not a valid Fortran status, then
+the call is erroneous.
+
+When using :ref:`MPI_Status_c2f`, the opposite conversion is applied. If
+``c_status`` is ``MPI_STATUS_IGNORE`` or ``MPI_STATUSES_IGNORE``, or if ``c_status`` is
+not a valid C status, then the call is erroneous.
+
+The input status has the same source, tag and error code values as the
+output status, and returns the same answers when queried for count,
+elements, and cancellation. The conversion function may be called with
+an input status argument that has an undefined error field, in which
+case the value of the error field in the output status argument is
+undefined.
+
+NOTES
+-----
+
+These functions are only available in C; they are not available in any
+of the Fortran MPI interfaces.
+
+
+.. seealso:: :ref:`MPI_Status_f082c` :ref:`MPI_Status_f2f08`
diff --git a/docs/man-openmpi/man3/MPI_Status_f2f08.3.rst b/docs/man-openmpi/man3/MPI_Status_f2f08.3.rst
new file mode 100644
index 00000000000..4e1b6a0f740
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Status_f2f08.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_status_f2f08:
+
+MPI_Status_f2f08
+================
+
+.. include_body
+
+.. include:: ../man3/MPI_Status_f082f.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_Status_get_error.3.rst b/docs/man-openmpi/man3/MPI_Status_get_error.3.rst
new file mode 100644
index 00000000000..107bb3703fe
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Status_get_error.3.rst
@@ -0,0 +1,54 @@
+.. _mpi_status_get_error:
+
+
+MPI_Status_get_error
+========================
+
+.. include_body
+
+:ref:`MPI_Status_get_error` |mdash| Retrieves the ``MPI_ERROR`` field from ``status``.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_status_get_error.rst
+
+INPUT PARAMETER
+---------------
+* ``status``: Status from which to retrieve the error (status).
+
+OUTPUT PARAMETER
+----------------
+* ``error``: error set in the ``MPI_ERROR`` field (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Returns in error the ``MPI_ERROR`` field from the ``status`` object.
+
+While the ``status`` object members ``MPI_SOURCE``, ``MPI_TAG``, and
+``MPI_ERROR`` are directly accessible in C and Fortran, for
+convenience in other contexts (e.g., when using alternate MPI bindings
+in languages that do not directly translate the ``status`` object),
+users can also access these values via procedure calls such as this
+one.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+
+.. seealso::
+   * :ref:`MPI_Get_count`
+   * :ref:`MPI_Get_elements`
+   * :ref:`MPI_Get_elements_x`
+   * :ref:`MPI_Status_get_source`
+   * :ref:`MPI_Status_get_tag`
+   * :ref:`MPI_Status_set_cancelled`
+   * :ref:`MPI_Status_set_elements`
+   * :ref:`MPI_Status_set_elements_x`
+   * :ref:`MPI_Status_set_error`
+   * :ref:`MPI_Status_set_source`
+   * :ref:`MPI_Status_set_tag`
+   * :ref:`MPI_Test_cancelled`
diff --git a/docs/man-openmpi/man3/MPI_Status_get_source.3.rst b/docs/man-openmpi/man3/MPI_Status_get_source.3.rst
new file mode 100644
index 00000000000..843f8af81d3
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Status_get_source.3.rst
@@ -0,0 +1,54 @@
+.. _mpi_status_get_source:
+
+
+MPI_Status_get_source
+========================
+
+.. include_body
+
+:ref:`MPI_Status_get_source` |mdash| Retrieves the ``MPI_SOURCE`` field from ``status``.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_status_get_source.rst
+
+INPUT PARAMETER
+---------------
+* ``status``: Status from which to retrieve the source rank (status).
+
+OUTPUT PARAMETER
+----------------
+* ``source``: rank set in the ``MPI_SOURCE`` field (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Returns in source the ``MPI_SOURCE`` field from the ``status`` object.
+
+While the ``status`` object members ``MPI_SOURCE``, ``MPI_TAG``, and
+``MPI_ERROR`` are directly accessible in C and Fortran, for
+convenience in other contexts (e.g., when using alternate MPI bindings
+in languages that do not directly translate the ``status`` object),
+users can also access these values via procedure calls such as this
+one.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+
+.. seealso::
+   * :ref:`MPI_Get_count`
+   * :ref:`MPI_Get_elements`
+   * :ref:`MPI_Get_elements_x`
+   * :ref:`MPI_Status_get_error`
+   * :ref:`MPI_Status_get_tag`
+   * :ref:`MPI_Status_set_cancelled`
+   * :ref:`MPI_Status_set_elements`
+   * :ref:`MPI_Status_set_elements_x`
+   * :ref:`MPI_Status_set_error`
+   * :ref:`MPI_Status_set_source`
+   * :ref:`MPI_Status_set_tag`
+   * :ref:`MPI_Test_cancelled`
diff --git a/docs/man-openmpi/man3/MPI_Status_get_tag.3.rst b/docs/man-openmpi/man3/MPI_Status_get_tag.3.rst
new file mode 100644
index 00000000000..168b783ea39
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Status_get_tag.3.rst
@@ -0,0 +1,54 @@
+.. _mpi_status_get_tag:
+
+
+MPI_Status_get_tag
+========================
+
+.. include_body
+
+:ref:`MPI_Status_get_tag` |mdash| Retrieves the ``MPI_TAG`` field from ``status``.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_status_get_tag.rst
+
+INPUT PARAMETER
+---------------
+* ``status``: Status from which to retrieve the tag (status).
+
+OUTPUT PARAMETER
+----------------
+* ``tag``: tag set in the ``MPI_TAG`` field (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Returns in tag the ``MPI_TAG`` field from the ``status`` object.
+
+While the ``status`` object members ``MPI_SOURCE``, ``MPI_TAG``, and
+``MPI_ERROR`` are directly accessible in C and Fortran, for
+convenience in other contexts (e.g., when using alternate MPI bindings
+in languages that do not directly translate the ``status`` object),
+users can also access these values via procedure calls such as this
+one.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+
+.. seealso::
+   * :ref:`MPI_Get_count`
+   * :ref:`MPI_Get_elements`
+   * :ref:`MPI_Get_elements_x`
+   * :ref:`MPI_Status_get_error`
+   * :ref:`MPI_Status_get_source`
+   * :ref:`MPI_Status_set_cancelled`
+   * :ref:`MPI_Status_set_elements`
+   * :ref:`MPI_Status_set_elements_x`
+   * :ref:`MPI_Status_set_error`
+   * :ref:`MPI_Status_set_source`
+   * :ref:`MPI_Status_set_tag`
+   * :ref:`MPI_Test_cancelled`
diff --git a/docs/man-openmpi/man3/MPI_Status_set_cancelled.3.rst b/docs/man-openmpi/man3/MPI_Status_set_cancelled.3.rst
new file mode 100644
index 00000000000..1e03153d90a
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Status_set_cancelled.3.rst
@@ -0,0 +1,52 @@
+.. _mpi_status_set_cancelled:
+
+
+MPI_Status_set_cancelled
+========================
+
+.. include_body
+
+:ref:`MPI_Status_set_cancelled` |mdash| Sets *status* to indicate a request has
+been canceled.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_status_set_cancelled.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``status``: Status with which to associate cancel flag (status).
+
+INPUT PARAMETER
+---------------
+* ``flag``: If true, indicates request was canceled (logical).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+If *flag* is set to true, then a subsequent call to
+MPI_Test_cancelled(status, flag*) will also return *flag* = true;
+otherwise it will return false.
+
+
+NOTES
+-----
+
+Users are advised not to reuse the status fields for values other than
+those for which they were intended. Doing so may lead to unexpected
+results when using the status object. For example, calling
+:ref:`MPI_Get_elements` may cause an error if the value is out of range, or it
+may be impossible to detect such an error. The *extra_state* argument
+provided with a generalized request can be used to return information
+that does not logically belong in *status*. Furthermore, modifying the
+values in a status set internally by MPI, such as :ref:`MPI_Recv`, may lead to
+unpredictable results and is strongly discouraged.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Status_set_elements.3.rst b/docs/man-openmpi/man3/MPI_Status_set_elements.3.rst
new file mode 100644
index 00000000000..9f708f237b8
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Status_set_elements.3.rst
@@ -0,0 +1,63 @@
+.. _mpi_status_set_elements:
+
+
+MPI_Status_set_elements
+=======================
+
+.. include_body
+
+:ref:`MPI_Status_set_elements`, :ref:`MPI_Status_set_elements_x` - Modifies
+opaque part of *status* to allow :ref:`MPI_Get_elements` to return *count*.
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Status_set_elements, MPI_Status_set_elements_x
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_status_set_elements.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``status``: Status to associate with *count (status).*
+
+INPUT PARAMETERS
+----------------
+* ``datatype``: Data type associated with *count (handle).*
+* ``count``: Number of elements to associate with *status (integer).*
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Status_set_elements` modifies the opaque part of *status* so that a
+call to :ref:`MPI_Get_elements` or :ref:`MPI_Get_elements_x` will return count.
+:ref:`MPI_Get_count` will return a compatible value.*
+
+A subsequent call to ``MPI_Get_count(status, datatype, count)``, to
+``MPI_Get_elements(status, datatype, count)``, or to
+``MPI_Get_elements_x(status, datatype, count)`` must use a data-type
+argument that has the same type signature as the data-type argument that
+was used in the call to :ref:`MPI_Status_set_elements`.
+
+
+NOTES
+-----
+
+Users are advised not to reuse the status fields for values other than
+those for which they were intended. Doing so may lead to unexpected
+results when using the status object. For example, calling
+:ref:`MPI_Get_elements` may cause an error if the value is out of range, or it
+may be impossible to detect such an error. The *extra_state* argument
+provided with a generalized request can be used to return information
+that does not logically belong in status. Furthermore, modifying the
+values in a status set internally by MPI, such as :ref:`MPI_Recv`, may lead to
+unpredictable results and is strongly discouraged.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Status_set_elements_x.3.rst b/docs/man-openmpi/man3/MPI_Status_set_elements_x.3.rst
new file mode 100644
index 00000000000..acd13e75172
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Status_set_elements_x.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_status_set_elements_x:
+
+MPI_Status_set_elements_x
+=========================
+    .. include_body
+
+.. include:: ../man3/MPI_Status_set_elements.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Status_set_error.3.rst b/docs/man-openmpi/man3/MPI_Status_set_error.3.rst
new file mode 100644
index 00000000000..ca31946136d
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Status_set_error.3.rst
@@ -0,0 +1,58 @@
+.. _mpi_status_set_error:
+
+
+MPI_Status_set_error
+========================
+
+.. include_body
+
+:ref:`MPI_Status_set_error` |mdash| Sets the ``MPI_ERROR`` field on ``status``.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_status_set_error.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``status``: Status with which to associate error (status).
+
+INPUT PARAMETER
+---------------
+* ``error``: error to set in the ``MPI_ERROR`` field (integer).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Set the ``MPI_ERROR`` field in the ``status`` object to the provided
+error argument.
+
+While the ``status`` object members ``MPI_SOURCE``, ``MPI_TAG``, and
+``MPI_ERROR`` are directly accessible in C and Fortran, for
+convenience in other contexts (e.g., when using alternate MPI bindings
+in languages that do not directly translate the ``status`` object),
+users can also access these values via procedure calls such as this
+one.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+
+.. seealso::
+   * :ref:`MPI_Get_count`
+   * :ref:`MPI_Get_elements`
+   * :ref:`MPI_Get_elements_x`
+   * :ref:`MPI_Status_get_error`
+   * :ref:`MPI_Status_get_source`
+   * :ref:`MPI_Status_get_tag`
+   * :ref:`MPI_Status_set_cancelled`
+   * :ref:`MPI_Status_set_elements`
+   * :ref:`MPI_Status_set_elements_x`
+   * :ref:`MPI_Status_set_source`
+   * :ref:`MPI_Status_set_tag`
+   * :ref:`MPI_Test_cancelled`
diff --git a/docs/man-openmpi/man3/MPI_Status_set_source.3.rst b/docs/man-openmpi/man3/MPI_Status_set_source.3.rst
new file mode 100644
index 00000000000..356f953f828
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Status_set_source.3.rst
@@ -0,0 +1,58 @@
+.. _mpi_status_set_source:
+
+
+MPI_Status_set_source
+========================
+
+.. include_body
+
+:ref:`MPI_Status_set_source` |mdash| Sets the ``MPI_SOURCE`` field on ``status``.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_status_set_source.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``status``: Status with which to associate source rank (status).
+
+INPUT PARAMETER
+---------------
+* ``source``: rank to set in the ``MPI_SOURCE`` field (integer).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Set the ``MPI_SOURCE`` field in the ``status`` object to the provided
+source argument.
+
+While the ``status`` object members ``MPI_SOURCE``, ``MPI_TAG``, and
+``MPI_ERROR`` are directly accessible in C and Fortran, for
+convenience in other contexts (e.g., when using alternate MPI bindings
+in languages that do not directly translate the ``status`` object),
+users can also access these values via procedure calls such as this
+one.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+
+.. seealso::
+   * :ref:`MPI_Get_count`
+   * :ref:`MPI_Get_elements`
+   * :ref:`MPI_Get_elements_x`
+   * :ref:`MPI_Status_get_error`
+   * :ref:`MPI_Status_get_source`
+   * :ref:`MPI_Status_get_tag`
+   * :ref:`MPI_Status_set_cancelled`
+   * :ref:`MPI_Status_set_elements`
+   * :ref:`MPI_Status_set_elements_x`
+   * :ref:`MPI_Status_set_error`
+   * :ref:`MPI_Status_set_tag`
+   * :ref:`MPI_Test_cancelled`
diff --git a/docs/man-openmpi/man3/MPI_Status_set_tag.3.rst b/docs/man-openmpi/man3/MPI_Status_set_tag.3.rst
new file mode 100644
index 00000000000..0e55c85176a
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Status_set_tag.3.rst
@@ -0,0 +1,58 @@
+.. _mpi_status_set_tag:
+
+
+MPI_Status_set_tag
+========================
+
+.. include_body
+
+:ref:`MPI_Status_set_tag` |mdash| Sets the ``MPI_TAG`` field on ``status``.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_status_set_tag.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``status``: Status with which to associate tag (status).
+
+INPUT PARAMETER
+---------------
+* ``tag``: tag to set in the ``MPI_TAG`` field (integer).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Set the ``MPI_TAG`` field in the ``status`` object to the provided tag
+argument.
+
+While the ``status`` object members ``MPI_SOURCE``, ``MPI_TAG``, and
+``MPI_ERROR`` are directly accessible in C and Fortran, for
+convenience in other contexts (e.g., when using alternate MPI bindings
+in languages that do not directly translate the ``status`` object),
+users can also access these values via procedure calls such as this
+one.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+
+.. seealso::
+   * :ref:`MPI_Get_count`
+   * :ref:`MPI_Get_elements`
+   * :ref:`MPI_Get_elements_x`
+   * :ref:`MPI_Status_get_error`
+   * :ref:`MPI_Status_get_source`
+   * :ref:`MPI_Status_get_tag`
+   * :ref:`MPI_Status_set_cancelled`
+   * :ref:`MPI_Status_set_elements`
+   * :ref:`MPI_Status_set_elements_x`
+   * :ref:`MPI_Status_set_error`
+   * :ref:`MPI_Status_set_source`
+   * :ref:`MPI_Test_cancelled`
diff --git a/docs/man-openmpi/man3/MPI_T.3.rst b/docs/man-openmpi/man3/MPI_T.3.rst
new file mode 100644
index 00000000000..5a0a3f83b79
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T.3.rst
@@ -0,0 +1,176 @@
+.. _mpi_t:
+
+MPI_T
+=====
+
+.. include_body
+
+Open MPI's MPI_T interface - General information
+
+DESCRIPTION
+-----------
+
+There are a few Open MPI-specific notes worth mentioning about its MPI_T
+interface implementation.
+
+MPI_T Control Variables
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Open MPI's implementation of the MPI_T Control Variable ("cvar") APIs is
+an interface to Open MPI's underlying Modular Component Architecture
+(MCA) parameters/variables. Simply put: using the MPI_T cvar interface
+is another mechanism to get/set Open MPI MCA parameters.
+
+In order of precedence (highest to lowest), Open MPI provides the
+following mechanisms to set MCA parameters:
+
+1. The MPI_T interface has the highest precedence. Specifically: values
+   set via the MPI_T interface will override all other settings.
+2. The mpirun(1) / mpiexec(1) command line (e.g., via the --mca
+   parameter).
+3. Environment variables.
+4. Parameter files have the lowest precedence. Specifically: values set
+   via parameter files can be overridden by any of the other
+   MCA-variable setting mechanisms.
+
+MPI initialization
+^^^^^^^^^^^^^^^^^^
+
+An application may use the MPI_T interface before MPI is initialized to
+set MCA parameters. Setting MPI-level MCA parameters before MPI is
+initialized may affect *how* MPI is initialized (e.g., by influencing
+which frameworks and components are selected).
+
+The following example sets the pml and btl MCA params before invoking
+:ref:`MPI_Init` in order to force a specific selection of PML and BTL
+components:
+
+.. code:: c
+
+   int provided, index, count;
+   MPI_T_cvar_handle pml_handle, btl_handle;
+   char pml_value[64], btl_value[64];
+
+   MPI_T_init_thread(MPI_THREAD_SINGLE, &provided);
+
+   MPI_T_cvar_get_index("pml", &index);
+   MPI_T_cvar_handle_alloc(index, NULL, &pml_handle, &count);
+   MPI_T_cvar_write(pml_handle, "ob1");
+
+   MPI_T_cvar_get_index("btl", &index);
+   MPI_T_cvar_handle_alloc(index, NULL, &btl_handle, &count);
+   MPI_T_cvar_write(btl_handle, "tcp,vader,self");
+
+   MPI_T_cvar_read(pml_handle, pml_value);
+   MPI_T_cvar_read(btl_handle, btl_value);
+   printf("Set value of cvars: PML: %s, BTL: %s\n",
+          pml_value, btl_value);
+
+   MPI_T_cvar_handle_free(&pml_handle);
+   MPI_T_cvar_handle_free(&btl_handle);
+
+   MPI_Init(NULL, NULL);
+
+   // ...
+
+   MPI_Finalize();
+
+   MPI_T_finalize();
+
+Note that once MPI is initialized, most Open MPI cvars become read-only.
+
+For example, after MPI is initialized, it is no longer possible to set
+the PML and BTL selection mechanisms. This is because many of these MCA
+parameters are only used during MPI initialization; setting them after
+MPI has already been initialized would be meaningless, anyway.
+
+MPI_T Categories
+^^^^^^^^^^^^^^^^
+
+Open MPI's MPI_T categories are organized hierarchically:
+
+1. Layer (or "project"). There are two layers in Open MPI:
+
+   -  ompi: This layer contains cvars, pvars, and sub categories related
+      to MPI characteristics.
+   -  opal: This layer generally contains cvars, pvars, and sub
+      categories of lower-layer constructions, such as operating system
+      issues, networking issues, etc.
+
+2. Framework or section.
+
+   -  In most cases, the next level in the hierarchy is the Open MPI MCA
+      framework.
+
+      -  For example, you can find the btl framework under the opal
+         layer (because it has to do with the underlying networking).
+      -  Additionally, the pml framework is under the ompi layer
+         (because it has to do with MPI semantics of point-to-point
+         messaging).
+
+   -  There are a few non-MCA-framework entities under the layer,
+      however.
+
+      -  For example, there is an mpi section under both the opal and
+         ompi layers for general/core MPI constructs.
+
+3. Component.
+
+   -  If relevant, the third level in the hierarchy is the MCA
+      component.
+   -  For example, the tcp component can be found under the opal
+      framework in the opal layer.
+
+MPI_T Events
+^^^^^^^^^^^^
+
+Open MPI provides a set of stub interfaces for MPI_T events. More complete
+support for MPI_T events will be provided in a future release of Open MPI.
+
+.. seealso::
+   * :ref:`MPI_T_category_changed`
+   * :ref:`MPI_T_category_get_categories`
+   * :ref:`MPI_T_category_get_cvars`
+   * :ref:`MPI_T_category_get_info`
+   * :ref:`MPI_T_category_get_num`
+   * :ref:`MPI_T_category_get_pvars`
+   * :ref:`MPI_T_cvar_get_info`
+   * :ref:`MPI_T_cvar_get_num`
+   * :ref:`MPI_T_cvar_handle_alloc`
+   * :ref:`MPI_T_cvar_handle_free`
+   * :ref:`MPI_T_cvar_read`
+   * :ref:`MPI_T_cvar_write`
+   * :ref:`MPI_T_enum_get_info`
+   * :ref:`MPI_T_enum_get_item`
+   * :ref:`MPI_T_event_callback_get_info`
+   * :ref:`MPI_T_event_callback_set_info`
+   * :ref:`MPI_T_event_copy`
+   * :ref:`MPI_T_event_get_index`
+   * :ref:`MPI_T_event_get_info`
+   * :ref:`MPI_T_event_get_num`
+   * :ref:`MPI_T_event_get_source`
+   * :ref:`MPI_T_event_get_timestamp`
+   * :ref:`MPI_T_event_handle_alloc`
+   * :ref:`MPI_T_event_handle_free`
+   * :ref:`MPI_T_event_handle_get_info`
+   * :ref:`MPI_T_event_handle_set_info`
+   * :ref:`MPI_T_event_read`
+   * :ref:`MPI_T_event_register_callback`
+   * :ref:`MPI_T_event_set_dropped_handler`
+   * :ref:`MPI_T_finalize`
+   * :ref:`MPI_T_init_thread`
+   * :ref:`MPI_T_pvar_get_info`
+   * :ref:`MPI_T_pvar_get_num`
+   * :ref:`MPI_T_pvar_handle_alloc`
+   * :ref:`MPI_T_pvar_handle_free`
+   * :ref:`MPI_T_pvar_read`
+   * :ref:`MPI_T_pvar_readreset`
+   * :ref:`MPI_T_pvar_reset`
+   * :ref:`MPI_T_pvar_session_create`
+   * :ref:`MPI_T_pvar_session_free`
+   * :ref:`MPI_T_pvar_start`
+   * :ref:`MPI_T_pvar_stop`
+   * :ref:`MPI_T_pvar_write`
+   * :ref:`MPI_T_source_get_info`
+   * :ref:`MPI_T_source_get_num`
+   * :ref:`MPI_T_source_get_timestamp`
diff --git a/docs/man-openmpi/man3/MPI_T_ERRORS.rst b/docs/man-openmpi/man3/MPI_T_ERRORS.rst
new file mode 100644
index 00000000000..dd3860ffdeb
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_ERRORS.rst
@@ -0,0 +1,7 @@
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface is not initialized.
+
+* ``MPI_T_ERR_INVALID_HANDLE``: The handle is invalid.
+
+* ``MPI_T_ERR_INVALID_HANDLE``: Invalid use of the interface or bad parameter values(s).
+
+* ``MPI_T_ERR_OTHER``: Other error
diff --git a/docs/man-openmpi/man3/MPI_T_category_changed.3.rst b/docs/man-openmpi/man3/MPI_T_category_changed.3.rst
new file mode 100644
index 00000000000..5694211f083
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_category_changed.3.rst
@@ -0,0 +1,32 @@
+.. _mpi_t_category_changed:
+
+
+MPI_T_category_changed
+======================
+
+.. include_body
+
+:ref:`MPI_T_category_changed` |mdash| Get a timestamp for the categories
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_category_changed.rst
+
+INPUT PARAMETERS
+----------------
+* ``stamp``: A virtual time stamp to indicate the last change to the categories.
+
+DESCRIPTION
+-----------
+
+If two subsequent calls to this routine return the same timestamp, it is
+guaranteed that no categories have been changed or added. If the
+timestamp from the second call is higher than some categories have been
+added or changed.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_category_changed` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
diff --git a/docs/man-openmpi/man3/MPI_T_category_get_categories.3.rst b/docs/man-openmpi/man3/MPI_T_category_get_categories.3.rst
new file mode 100644
index 00000000000..aaa0fa12308
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_category_get_categories.3.rst
@@ -0,0 +1,38 @@
+.. _mpi_t_category_get_categories:
+
+
+MPI_T_category_get_categories
+=============================
+
+.. include_body
+
+:ref:`MPI_T_category_get_categories` |mdash| Query which categories are in a
+category
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_category_get_categories.rst
+
+INPUT PARAMETERS
+----------------
+* ``cat_index``: Index of the category to be queried.
+* ``len``: The length of the indices array.
+
+OUTPUT PARAMETERS
+-----------------
+* ``indices``: An integer array of size len, indicating category indices.
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_category_get_categories` can be used to query which other
+categories are in a category.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_category_get_categories` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
+
+* ``MPI_T_ERR_INVALID_INDEX``: The category index is invalid
diff --git a/docs/man-openmpi/man3/MPI_T_category_get_cvars.3.rst b/docs/man-openmpi/man3/MPI_T_category_get_cvars.3.rst
new file mode 100644
index 00000000000..3a1ffb179da
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_category_get_cvars.3.rst
@@ -0,0 +1,38 @@
+.. _mpi_t_category_get_cvars:
+
+
+MPI_T_category_get_cvars
+========================
+
+.. include_body
+
+:ref:`MPI_T_category_get_cvars` |mdash| Query which control variables are in a
+category
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_category_get_cvars.rst
+
+INPUT PARAMETERS
+----------------
+* ``cat_index``: Index of the category to be queried.
+* ``len``: The length of the indices array.
+
+OUTPUT PARAMETERS
+-----------------
+* ``indices``: An integer array of size len, indicating control variable indices.
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_category_get_cvars` can be used to query which control variables
+are contained in a particular category.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_category_get_cvars` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
+
+* ``MPI_T_ERR_INVALID_INDEX``: The category index is invalid
diff --git a/docs/man-openmpi/man3/MPI_T_category_get_events.3.rst b/docs/man-openmpi/man3/MPI_T_category_get_events.3.rst
new file mode 100644
index 00000000000..af8c50ecca4
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_category_get_events.3.rst
@@ -0,0 +1,38 @@
+.. _mpi_t_category_get_events:
+
+
+MPI_T_category_get_events
+=========================
+
+.. include_body
+
+:ref:`MPI_T_category_get_events` |mdash| Query which events are in a
+category
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_category_get_events.rst
+
+INPUT PARAMETERS
+----------------
+* ``cat_index``: Index of the category to be queried.
+* ``len``: The length of the indices array.
+
+OUTPUT PARAMETERS
+-----------------
+* ``indices``: An integer array of size len, indicating event indices.
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_category_get_events` can be used to query which events
+are contained in a particular category.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_category_get_events` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
+
+* ``MPI_T_ERR_INVALID_INDEX``: The category index is invalid
diff --git a/docs/man-openmpi/man3/MPI_T_category_get_info.3.rst b/docs/man-openmpi/man3/MPI_T_category_get_info.3.rst
new file mode 100644
index 00000000000..d6a0e549d63
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_category_get_info.3.rst
@@ -0,0 +1,60 @@
+.. _mpi_t_category_get_info:
+
+
+MPI_T_category_get_info
+=======================
+
+.. include_body
+
+:ref:`MPI_T_category_get_info` |mdash| Query information from a category
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_category_get_info.rst
+
+INPUT PARAMETERS
+----------------
+* ``cat_index``: Index of the category to be queried.
+
+INPUT/OUTPUT PARAMETERS
+-----------------------
+* ``name_len``: Length of the string and/or buffer for name.
+* ``desc_len``: Length of the string and/or buffer for desc.
+
+OUTPUT PARAMETERS
+-----------------
+* ``name``: Buffer to return the string containing the name of the category.
+* ``desc``: Buffer to return the string containing the description of the category.
+* ``num_cvars``: Number of control variables in the category.
+* ``num_pvars``: Number of performance variables in the category.
+* ``num_categories``: Number of categories contained in the category.
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_category_get_info` can be used to query information from a
+category. The function returns the number of control variables,
+performance variables, and sub-categories in the queried category in the
+arguments *num_cvars*, *num_pvars*, and *num_categories*, respectively.
+
+
+NOTES
+-----
+
+This MPI tool interface function returns two strings. This function
+takes two argument for each string: a buffer to store the string, and a
+length which must initially specify the size of the buffer. If the
+length passed is n then this function will copy at most n - 1 characters
+of the string into the corresponding buffer and set the length to the
+number of characters copied - 1. If the length argument is NULL or the
+value specified in the length is 0 the corresponding string buffer is
+ignored and the string is not returned.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_category_get_info` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
+
+* ``MPI_T_ERR_INVALID_INDEX``: The category index is invalid
diff --git a/docs/man-openmpi/man3/MPI_T_category_get_num.3.rst b/docs/man-openmpi/man3/MPI_T_category_get_num.3.rst
new file mode 100644
index 00000000000..a238d6ce1ad
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_category_get_num.3.rst
@@ -0,0 +1,30 @@
+.. _mpi_t_category_get_num:
+
+
+MPI_T_category_get_num
+======================
+
+.. include_body
+
+:ref:`MPI_T_category_get_num` |mdash| Query the number of categories
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_category_get_num.rst
+
+OUTPUT PARAMETERS
+-----------------
+* ``num_cat``: Current number of categories
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_category_get_num` can be used to query the current number of
+categories.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_category_get_num` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
diff --git a/docs/man-openmpi/man3/MPI_T_category_get_num_events.3.rst b/docs/man-openmpi/man3/MPI_T_category_get_num_events.3.rst
new file mode 100644
index 00000000000..e3294c15675
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_category_get_num_events.3.rst
@@ -0,0 +1,35 @@
+.. _mpi_t_category_get_num_events:
+
+
+MPI_T_category_get_num_events
+=============================
+
+.. include_body
+
+:ref:`MPI_T_category_get_num_events` |mdash| Query returns the number of event types contained
+in the queried category.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_category_get_num_events.rst
+
+INPUT PARAMETERS
+----------------
+* ``cat_index``: Index of the category to be queried
+
+OUTPUT PARAMETERS
+-----------------
+* ``num_events``: Number of event types in the category
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_category_get_num_events` can be used to query the number of events
+contained in the category.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_category_get_num_events` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
diff --git a/docs/man-openmpi/man3/MPI_T_category_get_pvars.3.rst b/docs/man-openmpi/man3/MPI_T_category_get_pvars.3.rst
new file mode 100644
index 00000000000..44090cabe90
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_category_get_pvars.3.rst
@@ -0,0 +1,39 @@
+.. _mpi_t_category_get_pvars:
+
+
+MPI_T_category_get_pvars
+========================
+
+.. include_body
+
+:ref:`MPI_T_category_get_pvars` |mdash| Query which performance variables are in
+a category
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_category_get_pvars.rst
+
+INPUT PARAMETERS
+----------------
+* ``cat_index``: Index of the category to be queried.
+* ``len``: The length of the indices array.
+
+OUTPUT PARAMETERS
+-----------------
+* ``indices``: An integer array of size len, indicating performance variable indices.
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_category_get_pvars` can be used to query which performance
+variables are contained in a particular category. A category contains
+zero or more performance variables.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_category_get_pvars` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
+
+* ``MPI_T_ERR_INVALID_INDEX``: The category index is invalid
diff --git a/docs/man-openmpi/man3/MPI_T_cvar_get_info.3.rst b/docs/man-openmpi/man3/MPI_T_cvar_get_info.3.rst
new file mode 100644
index 00000000000..7b9f0d8dfd4
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_cvar_get_info.3.rst
@@ -0,0 +1,139 @@
+MPI_T see MPI-3 section 14.3.5.
+
+.. _mpi_t_cvar_get_info:
+
+
+MPI_T_cvar_get_info
+===================
+
+.. include_body
+
+:ref:`MPI_T_cvar_get_info` |mdash| Query information from a control variable
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_cvar_get_info.rst
+
+INPUT PARAMETERS
+----------------
+* ``cvar_index``: Index of the control variable to be queried.
+
+INPUT/OUTPUT PARAMETERS
+-----------------------
+* ``name_len``: Length of the string and/or buffer for name.
+* ``desc_len``: Length of the string and/or buffer for desc.
+
+OUTPUT PARAMETERS
+-----------------
+* ``name``: Buffer to return the string containing the name of the control variable.
+* ``verbosity``: Verbosity level of this variable.
+* ``datatype``: MPI datatype of the information stored in the control variable.
+* ``enumtype``: Optional descriptor for enumeration information.
+* ``desc``: Buffer to return the string containing the description of the control variable.
+* ``bind``: Type of MPI object to which this variable must be bound.
+* ``scope``: Scope of when changes to this variable are possible.
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_cvar_get_info` can be used to query information about a control
+variable. The function returns the verbosity, datatype, enumeration
+type, binding, and scope of the queried control variable in the
+arguments *verbosity*, *datatype*, *enumtype*, *bind*, and *scope*,
+respectively. Control variables in Open MPI are the same as MCA
+parameters.
+
+
+VERBOSITY
+---------
+
+As Open MPI exposes a very large number of MCA parameters (control
+variables), control variables are categorized into nine verbosity
+levels corresponding to the equivalent :ref:`ompi_info
+<man1-ompi_info>` level. The nine levels are (in increasing order):
+
+#. ``MPI_T_VERBOSITY_USER_BASIC``: Basic information of interest to users
+
+#. ``MPI_T_VERBOSITY_USER_DETAIL``: Detailed information of interest to users
+
+#. ``MPI_T_VERBOSITY_USER_ALL``: All remaining information of interest to users
+
+#. ``MPI_T_VERBOSITY_TUNER_BASIC``: Basic information required for tuning
+
+#. ``MPI_T_VERBOSITY_TUNER_DETAIL``: Detailed information required for tuning
+
+#. ``MPI_T_VERBOSITY_TUNER_ALL``: All remaining information required for tuning
+
+#. ``MPI_T_VERBOSITY_MPIDEV_BASIC``: Basic information for MPI implementors
+
+#. ``MPI_T_VERBOSITY_MPIDEV_DETAIL``: Detailed information for MPI implementors
+
+#. ``MPI_T_VERBOSITY_MPIDEV_ALL``: All remaining information for MPI implementors
+
+For more information see MPI-3 section 14.3.1.
+
+
+DATATYPE
+--------
+
+The datatype returned by :ref:`MPI_T_cvar_get_info` is restricted to
+one of the following datatypes: ``MPI_INT``, ``MPI_UNSIGNED``,
+``MPI_UNSIGNED_LONG``, ``MPI_UNSIGNED_LONG_LONG``, ``MPI_COUNT``,
+``MPI_CHAR``, and ``MPI_DOUBLE``. For more information on datatypes in
+MPI_T see MPI-3 section 14.3.5.
+
+
+SCOPE
+-----
+
+The scope describes when and how changes can be made to a control
+variable. From MPI-3 section 14.3.6, the scope may be any of the following:
+
+* ``MPI_T_SCOPE_CONSTANT``: read-only, value is constant
+
+* ``MPI_T_SCOPE_READONLY``: read-only, cannot be written, but can change
+
+* ``MPI_T_SCOPE_LOCAL``: may be writeable, writing is a local operation
+
+* ``MPI_T_SCOPE_GROUP``: may be writeable, must be done to a group of
+  processes, all processes in a group must be set to consistent
+  values
+
+* ``MPI_T_SCOPE_GROUP_EQ``: may be writeable, must be done to a group
+  of processes, all processes in a group must be set to the same value
+
+* ``MPI_T_SCOPE_ALL``: may be writeable, must be done to all
+  processes, all connected processes must be set to consistent values
+
+* ``MPI_T_SCOPE_ALL_EQ``: may be writeable, must be done to all
+  processes, all connected processes must be set to the same value
+
+For more information see MPI-3 section 14.3.6 Table 14.4.
+
+
+NOTES
+-----
+
+This MPI tool interface function returns two strings. This function
+takes two argument for each string: a buffer to store the string, and a
+length which must initially specify the size of the buffer. If the
+length passed is n then this function will copy at most n - 1 characters
+of the string into the corresponding buffer and set the length to the
+number of characters copied - 1. If the length argument is NULL or the
+value specified in the length is 0 the corresponding string buffer is
+ignored and the string is not returned.
+
+Open MPI does not currently support binding control variables to MPI
+objects.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_cvar_get_info` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
+
+* ``MPI_T_ERR_INVALID_INDEX``: The control variable index is invalid
+
+
+.. seealso:: :ref:`ompi_info <man1-ompi_info>`
diff --git a/docs/man-openmpi/man3/MPI_T_cvar_get_num.3.rst b/docs/man-openmpi/man3/MPI_T_cvar_get_num.3.rst
new file mode 100644
index 00000000000..eb368d85801
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_cvar_get_num.3.rst
@@ -0,0 +1,31 @@
+.. _mpi_t_cvar_get_num:
+
+
+MPI_T_cvar_get_num
+==================
+
+.. include_body
+
+:ref:`MPI_T_cvar_get_num` |mdash| Query the number of control variables
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_cvar_get_num.rst
+
+OUTPUT PARAMETERS
+-----------------
+* ``num_cvar``: Current number of control variables.
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_cvar_get_num` can be used to query the current number of control
+variables. The number of control variables may increase throughout the
+execution of the process but will never decrease.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_cvar_get_num` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
diff --git a/docs/man-openmpi/man3/MPI_T_cvar_handle_alloc.3.rst b/docs/man-openmpi/man3/MPI_T_cvar_handle_alloc.3.rst
new file mode 100644
index 00000000000..1346e13c269
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_cvar_handle_alloc.3.rst
@@ -0,0 +1,61 @@
+.. _mpi_t_cvar_handle_alloc:
+
+
+MPI_T_cvar_handle_alloc
+=======================
+
+.. include_body
+
+:ref:`MPI_T_cvar_handle_alloc`, :ref:`MPI_T_cvar_handle_free` - Allocate/free
+control variable handles
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_T_cvar_handle_alloc, MPI_T_cvar_handle_free
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_cvar_handle_alloc.rst
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_cvar_handle_alloc` binds the control variable specified in
+*cvar_index* to the MPI object specified in *obj_handle*. If
+:ref:`MPI_T_cvar_get_info` returns MPI_T_BIND_NO_OBJECT as the binding of the
+variable the *obj_handle* argument is ignored. The number of values
+represented by this control variable is returned in the *count*
+parameter. If the control variable represents a string then *count* will
+be the maximum length of the string.
+
+:ref:`MPI_T_cvar_handle_free` frees a handle allocated by
+:ref:`MPI_T_cvar_handle_alloc` and sets the *handle* argument to
+MPI_T_CVAR_HANDLE_NULL.
+
+
+NOTES
+-----
+
+Open MPI does not currently support binding MPI objects to control
+variables so the *obj_handle* argument is always ignored.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_cvar_handle_alloc` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
+
+* ``MPI_T_ERR_INVALID_INDEX``: The control variable index is invalid
+
+* ``MPI_T_ERR_OUT_OF_HANDLES``: No more handles available
+
+:ref:`MPI_T_cvar_handle_free` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
+
+* ``MPI_T_ERR_INVALID_HANDLE``: The handle is invalid
+
+
+.. seealso::
+   * :ref:`MPI_T_cvar_get_info`
diff --git a/docs/man-openmpi/man3/MPI_T_cvar_handle_free.3.rst b/docs/man-openmpi/man3/MPI_T_cvar_handle_free.3.rst
new file mode 100644
index 00000000000..7daa4d6b356
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_cvar_handle_free.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_t_cvar_handle_free:
+
+MPI_T_cvar_handle_free
+======================
+    .. include_body
+
+.. include:: ../man3/MPI_T_cvar_handle_alloc.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_T_cvar_read.3.rst b/docs/man-openmpi/man3/MPI_T_cvar_read.3.rst
new file mode 100644
index 00000000000..cb535fa2be2
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_cvar_read.3.rst
@@ -0,0 +1,41 @@
+.. _mpi_t_cvar_read:
+
+
+MPI_T_cvar_read
+===============
+
+.. include_body
+
+:ref:`MPI_T_cvar_read` |mdash| Read the value of a control variable
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_cvar_read.rst
+
+INPUT PARAMETERS
+----------------
+* ``handle``: Handle of the control variable to be read.
+* ``buf``: Initial address of storage location for variable value.
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_cvar_read` reads the value of the control variable identified by
+the handle specified in *handle* and stores the value in the buffer
+pointed to by *buf*. The caller must ensure that the buffer pointed to
+by *buf* is large enough to hold the entire value of the control
+variable.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_cvar_read` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
+
+* ``MPI_T_ERR_INVALID_HANDLE``: The handle is invalid
+
+
+.. seealso::
+   * :ref:`MPI_T_cvar_handle_alloc`
+   * :ref:`MPI_T_cvar_get_info`
diff --git a/docs/man-openmpi/man3/MPI_T_cvar_write.3.rst b/docs/man-openmpi/man3/MPI_T_cvar_write.3.rst
new file mode 100644
index 00000000000..9ebd4d31773
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_cvar_write.3.rst
@@ -0,0 +1,47 @@
+.. _mpi_t_cvar_write:
+
+
+MPI_T_cvar_write
+================
+
+.. include_body
+
+:ref:`MPI_T_cvar_write` |mdash| Write the value of a bound control variable
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_cvar_write.rst
+
+INPUT PARAMETERS
+----------------
+* ``handle``: Handle of the control variable to be written.
+* ``buf``: Initial address of storage location for variable value.
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_cvar_write` sets the value the control variable identified
+by the handle specified in *handle* from the buffer provided in
+*buf*. The caller must ensure that the buffer specified in *buf* is
+large enough to hold the entire value of the control variable. If the
+variable has global scope, any write call must be issued on all
+connected MPI processes. For more information see MPI-3 section
+14.3.6.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_cvar_write` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
+
+* ``MPI_T_ERR_INVALID_HANDLE``: The handle is invalid
+
+* ``MPI_T_ERR_CVAR_SET_NOT_NOW``: Variable cannot be set at this moment
+
+* ``MPI_T_ERR_CVAR_SET_NEVER``: Variable cannot be set until end of execution
+
+
+.. seealso::
+   * :ref:`MPI_T_cvar_handle_alloc`
+   * :ref:`MPI_T_cvar_get_info`
diff --git a/docs/man-openmpi/man3/MPI_T_enum_get_info.3.rst b/docs/man-openmpi/man3/MPI_T_enum_get_info.3.rst
new file mode 100644
index 00000000000..425fc80b671
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_enum_get_info.3.rst
@@ -0,0 +1,56 @@
+.. _mpi_t_enum_get_info:
+
+
+MPI_T_enum_get_info
+===================
+
+.. include_body
+
+:ref:`MPI_T_enum_get_info` |mdash| Query information about an enumerator
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_enum_get_info.rst
+
+INPUT PARAMETERS
+----------------
+* ``enumtype``: Enumerator to be queried.
+
+INPUT/OUTPUT PARAMETERS
+-----------------------
+* ``name_len``: Length of the string and/or buffer for name.
+
+OUTPUT PARAMETERS
+-----------------
+* ``num``: number of discrete values represented by this enumeration.
+* ``name``: Buffer to return the string containing the name of the category.
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_enum_get_info` can be used to query information about an
+enumerator. The function returns the number of discrete values
+represented by this enumerator in the *num* parameter.
+
+
+NOTES
+-----
+
+This MPI tool interface function returns the name of the enumeration as
+a string. This function takes two argument for the string: *name* which
+specifies a buffer where the name of the should be stored, and
+*name_len* which must initially specify the size of the buffer pointed
+to by *name*. This function will copy at most *name_len* - 1 characters
+of the name and sets *name_len* to the number of characters returned +
+1. If *name_len* is NULL or the value specified in *name_len* is 0 the
+*name* buffer is ignored and the name of the enumeration is not
+returned.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_enum_get_info` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
+
+* ``MPI_T_ERR_INVALID_INDEX``: The enumeration is invalid or has been deleted
diff --git a/docs/man-openmpi/man3/MPI_T_enum_get_item.3.rst b/docs/man-openmpi/man3/MPI_T_enum_get_item.3.rst
new file mode 100644
index 00000000000..1db6351bdbe
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_enum_get_item.3.rst
@@ -0,0 +1,57 @@
+.. _mpi_t_enum_get_item:
+
+
+MPI_T_enum_get_item
+===================
+
+.. include_body
+
+:ref:`MPI_T_enum_get_item` |mdash| Query information about an enumerator
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_enum_get_item.rst
+
+INPUT PARAMETERS
+----------------
+* ``enumtype``: Enumeration to be queried.
+* ``index``: Number of the value to be queried in this enumeration.
+
+INPUT/OUTPUT PARAMETERS
+-----------------------
+* ``name_len``: Length of the string and/or buffer for name.
+
+OUTPUT PARAMETERS
+-----------------
+* ``value``: Variable value.
+* ``name``: Buffer to return the string containing the name of the category.
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_enum_get_item` can be used to query information about an item in an
+enumerator. This function returns the enumeration value in the *value*
+parameter.
+
+
+NOTES
+-----
+
+This MPI tool interface function returns the name of the item as a
+string. This function takes two arguments for the string: a buffer to
+store the string, and a length which must initially specify the size
+of the buffer. If the length passed is n then this function will copy
+at most n - 1 characters of the string into the buffer and sets the
+length to the number of characters copied - 1. If the length argument
+is NULL or the value specified in the length is 0 the string buffer is
+ignored and the string is not returned. For more information see MPI-3
+section 14.3.3.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_enum_get_item` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
+
+* ``MPI_T_ERR_INVALID_INDEX``: The enumeration is invalid or has been deleted
diff --git a/docs/man-openmpi/man3/MPI_T_event_callback_get_info.3.rst b/docs/man-openmpi/man3/MPI_T_event_callback_get_info.3.rst
new file mode 100644
index 00000000000..3c0c0fbaaa4
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_event_callback_get_info.3.rst
@@ -0,0 +1,37 @@
+.. _mpi_t_event_callback_get_info:
+
+
+MPI_T_event_callback_get_info
+=============================
+
+.. include_body
+
+:ref:`MPI_T_event_callback_get_info` |mdash| Returns a new info object containing the hints of the callback function registered for the callback safety level specified by cb_safety of the event-registration handle associated with event_registration.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_event_callback_get_info.rst
+
+INPUT PARAMETERS
+----------------
+* ``event_registration``: Event registration
+* ``cb_safety``: maximum callback safety level
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``info_used``: Info argument
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_event_callback_get_info` returns a new info object containing the
+hints of the callback function registered for the callback safety level specified by ``cb_safety`` of
+the event-registration handle associated with ``event_registration``.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_event_callback_get_info` will fail if:
+
+.. include:: ./MPI_T_ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_T_event_callback_set_info.3.rst b/docs/man-openmpi/man3/MPI_T_event_callback_set_info.3.rst
new file mode 100644
index 00000000000..68e7c5bc5a2
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_event_callback_set_info.3.rst
@@ -0,0 +1,31 @@
+.. _mpi_t_event_callback_set_info:
+
+
+MPI_T_event_callback_set_info
+=============================
+
+.. include_body
+
+:ref:`MPI_T_event_callback_set_info` |mdash| Updates the hints of the callback function registered for the callback safety level specified by cb_safety of the event-registration handle associated with event_registration.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_event_callback_set_info.rst
+
+INPUT PARAMETERS
+----------------
+* ``event_registration``: Event registration
+* ``cb_safety``: maximum callback safety level
+* ``info``: Info argument
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_event_callback_set_info` Updates the hints of the callback function registered for the callback safety level specified by ``cb_safety`` of the event-registration handle associated with event_registration.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_event_callback_set_info` will fail if:
+
+.. include:: ./MPI_T_ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_T_event_copy.3.rst b/docs/man-openmpi/man3/MPI_T_event_copy.3.rst
new file mode 100644
index 00000000000..e9e984567f5
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_event_copy.3.rst
@@ -0,0 +1,35 @@
+.. _mpi_t_event_copy:
+
+
+MPI_T_event_copy
+================
+
+.. include_body
+
+:ref:`MPI_T_event_copy` |mdash| Copy event data as a whole into a user-specified buffer.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_event_copy.rst
+
+INPUT PARAMETERS
+----------------
+* ``event_instance``: event-instance handle provided to the callback function.
+* ``buf``: pointer to a memory location to store the event data.
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_event_copy` copies the event data as a whole into the user-provided buffer.
+The user must assure that the buffer is of at least the size of the extent of the event
+type, which can be computed from the type and displacement information returned by the
+corresponding call to :ref:`MPI_T_event_get_info`.
+
+ERRORS
+------
+
+:ref:`MPI_T_event_copy` will fail if:
+
+.. include:: ./MPI_T_ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_T_event_read`
diff --git a/docs/man-openmpi/man3/MPI_T_event_get_index.3.rst b/docs/man-openmpi/man3/MPI_T_event_get_index.3.rst
new file mode 100644
index 00000000000..5a419b909c0
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_event_get_index.3.rst
@@ -0,0 +1,40 @@
+.. _mpi_t_event_get_index:
+
+
+MPI_T_event_get_index
+=====================
+
+.. include_body
+
+:ref:`MPI_T_event_get_index` |mdash| Query the index of an event type identified by a known event type name
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_event_get_index.rst
+
+INPUT PARAMETERS
+-----------------
+
+* ``name``: name of the event type
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``event_index``: index of the event type
+
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_event_get_index` can be used to query for the index of an event type identified by a known
+event type name.
+
+ERRORS
+------
+
+:ref:`MPI_T_event_get_index` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface is not
+  initialized
+
+* ``MPI_T_ERR_INVALID_NAME``:  The event name is invalid
+  initialized
diff --git a/docs/man-openmpi/man3/MPI_T_event_get_info.3.rst b/docs/man-openmpi/man3/MPI_T_event_get_info.3.rst
new file mode 100644
index 00000000000..d4d66fa405e
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_event_get_info.3.rst
@@ -0,0 +1,52 @@
+.. _mpi_t_event_get_info:
+
+
+MPI_T_event_get_info
+====================
+
+.. include_body
+
+:ref:`MPI_T_event_get_info` |mdash|  Returns additional information about a specific event type
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_event_get_info.rst
+
+INPUT PARAMETERS
+----------------
+* ``event_index``: Index of the event to be queried.
+
+INPUT/OUTPUT PARAMETERS
+-----------------------
+* ``name_len``: Length of the string and/or buffer for name.
+* ``desc_len``: Length of the string and/or buffer for desc.
+* ``num_elements``: length of array of datatypes and displacements.
+
+OUTPUT PARAMETERS
+-----------------
+* ``name``: Buffer to return the string containing the name of the event type.
+* ``verbosity``: Verbosity level of this event type
+* ``array_of_datatypes``:  Array of MPI basic datatypes used to encode the event data
+* ``array_of_displacements``:  Array of byte displacements of the elements in the event buffer
+* ``enumtype``:  Optional descriptor for enumeration information
+* ``info``: Optional info argument.
+* ``desc``: Buffer to return the string containing the description of the event type.
+* ``bind``:  Type of MPI object to which an event of this type must be bound
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_event_get_info` can be used to query information from for a specific event type.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_event_get_info` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
+
+* ``MPI_T_ERR_INVALID_INDEX``: The source index is invalid
+
+* ``MPI_T_ERR_INVALID``: Invalid use of the interface or bad parameter values(s).
+
+* ``MPI_ERR_OTHER``: Other error
diff --git a/docs/man-openmpi/man3/MPI_T_event_get_num.3.rst b/docs/man-openmpi/man3/MPI_T_event_get_num.3.rst
new file mode 100644
index 00000000000..32832c6f500
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_event_get_num.3.rst
@@ -0,0 +1,39 @@
+.. _mpi_t_event_get_num:
+
+
+MPI_T_event_get_num
+===================
+
+.. include_body
+
+:ref:`MPI_T_event_get_num` |mdash| Query the number of MPI_T event types
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_event_get_num.rst
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``num_events``: Returns number of event types
+
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_event_get_num` can be used to query the current number of MPI_T event types.
+
+The number of available event types can be queried with a call to
+:ref:`MPI_T_event_get_num`. An MPI implementation is allowed to increase the number of event types during the
+execution of an MPI process. However, MPI implementations are not allowed to change the
+index of an event type or to delete an event type once it has been made visible to the user.
+
+.. note:: Open MPI will currently return that there are 0 event types.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_event_get_num` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface is not
+  initialized
diff --git a/docs/man-openmpi/man3/MPI_T_event_get_source.3.rst b/docs/man-openmpi/man3/MPI_T_event_get_source.3.rst
new file mode 100644
index 00000000000..ea216e05b62
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_event_get_source.3.rst
@@ -0,0 +1,38 @@
+.. _mpi_t_event_get_source:
+
+
+MPI_T_event_get_source
+=======================
+
+.. include_body
+
+:ref:`MPI_T_event_get_source` |mdash|  Returns the index of the source of the event instance
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_event_get_source.rst
+
+INPUT PARAMETERS
+----------------
+* ``event_instance``:  Event instance provided to the callback function
+
+OUTPUT PARAMETERS
+-----------------
+* ``source_index``: Index identifying the source
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_event_get_source` returns the index of the source of ``event_instance``.
+
+ERRORS
+------
+
+:ref:`MPI_T_event_get_source` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
+
+* ``MPI_T_ERR_INVALID_INDEX``: The handle is invalid
+
+* ``MPI_T_ERR_INVALID``: Invalid use of the interface or bad parameter values(s)
+
+* ``MPI_ERR_OTHER``: Other error
diff --git a/docs/man-openmpi/man3/MPI_T_event_get_timestamp.3.rst b/docs/man-openmpi/man3/MPI_T_event_get_timestamp.3.rst
new file mode 100644
index 00000000000..4762df01bb7
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_event_get_timestamp.3.rst
@@ -0,0 +1,34 @@
+.. _mpi_t_event_get_timestamp:
+
+
+MPI_T_event_get_timestamp
+=========================
+
+.. include_body
+
+:ref:`MPI_T_event_get_timestamp` |mdash|  Returns the timestamp of when the event was initially observed by the implementation
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_event_get_timestamp.rst
+
+INPUT PARAMETERS
+----------------
+* ``event_instance``:  Event instance provided to the callback function
+
+OUTPUT PARAMETERS
+-----------------
+* ``timestamp``: The timestamp when the event was observed
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_event_get_timestamp` returns the timestamp of when the event was initially
+observed by the implementation. The ``event_instance`` argument identifies the event
+instance to query.
+
+ERRORS
+------
+
+:ref:`MPI_T_event_get_timestamp` will fail if:
+
+.. include:: ./MPI_T_ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_T_event_handle_alloc.3.rst b/docs/man-openmpi/man3/MPI_T_event_handle_alloc.3.rst
new file mode 100644
index 00000000000..453aa085ed6
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_event_handle_alloc.3.rst
@@ -0,0 +1,41 @@
+.. _mpi_t_event_handle_alloc:
+
+
+MPI_T_event_handle_alloc
+========================
+
+.. include_body
+
+:ref:`MPI_T_event_handle_alloc` - Allocate event handles
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_event_handle_alloc.rst
+
+INPUT PARAMETERS
+----------------
+* ``event_index``: Index of event type for which the registration handle is to be allocated.
+* ``obj_handle``: reference to a handle of the MPI object to which this event is supposed to be bound.
+* ``info``: Info object.
+
+OUTPUT PARAMETERS
+-----------------
+* ``event_registration``: Event registration.
+
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_event_handle_alloc` binds the control variable specified in
+*event_index* to the MPI object specified in *obj_handle*. If
+:ref:`MPI_T_event_get_info` returns MPI_T_BIND_NO_OBJECT as the binding of the
+variable the *obj_handle* argument is ignored.
+
+ERRORS
+------
+
+:ref:`MPI_T_event_handle_alloc` will fail if:
+
+.. include:: ./MPI_T_ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_T_event_get_info`
diff --git a/docs/man-openmpi/man3/MPI_T_event_handle_free.3.rst b/docs/man-openmpi/man3/MPI_T_event_handle_free.3.rst
new file mode 100644
index 00000000000..0247c9532eb
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_event_handle_free.3.rst
@@ -0,0 +1,38 @@
+.. _mpi_t_event_handle_free:
+
+
+MPI_T_event_handle_free
+=======================
+
+.. include_body
+
+:ref:`MPI_T_event_handle_free` - Free event handles
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_event_handle_free.rst
+
+INPUT PARAMETERS
+----------------
+* ``user_data``: Pointer to a user-controlled buffer.
+* ``free_cb_function``: Pointer to user-defined callback function.
+
+INPUT/OUTPUT PARAMETERS
+-----------------------
+* ``event_registration``: Event registration.
+
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_event_handle_free` frees a handle allocated by
+:ref:`MPI_T_event_handle_alloc`.
+
+ERRORS
+------
+
+:ref:`MPI_T_event_handle_free` will fail if:
+
+.. include:: ./MPI_T_ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_T_event_get_info`
diff --git a/docs/man-openmpi/man3/MPI_T_event_handle_get_info.3.rst b/docs/man-openmpi/man3/MPI_T_event_handle_get_info.3.rst
new file mode 100644
index 00000000000..9b4bb0d1d7f
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_event_handle_get_info.3.rst
@@ -0,0 +1,36 @@
+.. _mpi_t_event_handle_get_info:
+
+
+MPI_T_event_handle_get_info
+===========================
+
+.. include_body
+
+:ref:`MPI_T_event_handle_get_info` |mdash| Returns a new info object containing the hints of the event-registration handle associated with event_registration
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_event_handle_get_info.rst
+
+INPUT PARAMETERS
+----------------
+* ``event_registration``: Event registration to be queried.
+
+OUTPUT PARAMETERS
+-----------------
+* ``info``: Info argument.
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_event_handle_get_info` can be used to query information from a
+source.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_source_get_info` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
+
+* ``MPI_T_ERR_INVALID_INDEX``: The event registration is invalid
diff --git a/docs/man-openmpi/man3/MPI_T_event_handle_set_info.3.rst b/docs/man-openmpi/man3/MPI_T_event_handle_set_info.3.rst
new file mode 100644
index 00000000000..cbf6b0878ff
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_event_handle_set_info.3.rst
@@ -0,0 +1,35 @@
+.. _mpi_t_event_handle_set_info:
+
+
+MPI_T_event_handle_set_info
+===========================
+
+.. include_body
+
+:ref:`MPI_T_event_handle_set_info` |mdash| Updates the hints of the event-registration handle associated with event_registration.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_event_handle_set_info.rst
+
+INPUT PARAMETERS
+----------------
+* ``event_registration``: Event registration to be queried.
+* ``info``: Info argument.
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_event_handle_set_info` updates the hints of the event-registration handle
+associated with event_registration using the hints provided in info.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_source_get_info` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
+
+* ``MPI_T_ERR_INVALID_INDEX``: The event registration is invalid
+
+* ``MPI_T_ERR_INFO``: Invalid info
diff --git a/docs/man-openmpi/man3/MPI_T_event_read.3.rst b/docs/man-openmpi/man3/MPI_T_event_read.3.rst
new file mode 100644
index 00000000000..ed90d78b987
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_event_read.3.rst
@@ -0,0 +1,34 @@
+.. _mpi_t_event_read:
+
+
+MPI_T_event_read
+================
+
+.. include_body
+
+:ref:`MPI_T_event_read` |mdash| Read the value of a event instance.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_event_read.rst
+
+INPUT PARAMETERS
+----------------
+* ``event_instance``: event-instance handle provided to the callback function.
+* ``elem_index``: index into the array of datatypes of the item to be queried.
+* ``buffer``: pointer to a memory location to store the item data
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_event_read` allows users to copy one element of the event data to a userspecified
+buffer at a time.
+
+ERRORS
+------
+
+:ref:`MPI_T_event_read` will fail if:
+
+.. include:: ./MPI_T_ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_T_event_copy`
diff --git a/docs/man-openmpi/man3/MPI_T_event_register_callback.3.rst b/docs/man-openmpi/man3/MPI_T_event_register_callback.3.rst
new file mode 100644
index 00000000000..4cc618af8cd
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_event_register_callback.3.rst
@@ -0,0 +1,41 @@
+.. _mpi_t_event_register_callback:
+
+
+MPI_T_event_register_callback
+=============================
+
+.. include_body
+
+:ref:`MPI_T_event_register_callback` |mdash| Associates a user-defined function with an allocated event-registration handle
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_event_register_callback.rst
+
+INPUT PARAMETERS
+----------------
+* ``event_registration``: Event registration
+* ``cb_safety``: maximum callback safety level
+* ``info``: Info argument
+* ``user_data``: pointer to a user-controlled buffer
+* ``event_cb_function``:  pointer to user-defined callback function
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_event_register_callback` associates a user-defined function pointed to
+by ``event_cb_function`` with an allocated event-registration handle. The maximum callback
+safety level supported by the callback function is passed in the argument ``cb_safety``.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_event_register_callback` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
+
+* ``MPI_T_ERR_INVALID_HANDLE``: The handle is invalid
+
+* ``MPI_T_ERR_INFO``: Invalid info
+
+* ``MPI_T_ERR_OTHER``: Other error
diff --git a/docs/man-openmpi/man3/MPI_T_event_set_dropped_handler.3.rst b/docs/man-openmpi/man3/MPI_T_event_set_dropped_handler.3.rst
new file mode 100644
index 00000000000..a6fdc20cfcf
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_event_set_dropped_handler.3.rst
@@ -0,0 +1,32 @@
+.. _mpi_t_event_set_dropped_handler:
+
+
+MPI_T_event_set_dropped_handler
+===============================
+
+.. include_body
+
+:ref:`MPI_T_event_set_dropped_handler` |mdash|  Registers a function to be called when event information is dropped for the registration handle specified in event_registration
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_event_set_dropped_handler.rst
+
+INPUT PARAMETERS
+----------------
+* ``event_registration``: Event registration
+* ``dropped_cb_function``:  pointer to user-defined callback function
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_event_set_dropped_handler` registers the function
+``dropped_cb_function`` to be called by the MPI implementation when event information is
+dropped for the registration handle specified in ``event_registration``.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_event_set_dropped_handler` will fail if:
+
+.. include:: ./MPI_T_ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_T_finalize.3.rst b/docs/man-openmpi/man3/MPI_T_finalize.3.rst
new file mode 100644
index 00000000000..94153b9691b
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_finalize.3.rst
@@ -0,0 +1,44 @@
+.. _mpi_t_finalize:
+
+
+MPI_T_finalize
+==============
+
+.. include_body
+
+:ref:`MPI_T_finalize` |mdash| Finalize the MPI tool information interface
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_finalize.rst
+
+DESCRIPTION
+-----------
+
+MPI_T_finalize() finalizes the MPI tool information interface and must
+be called the same number of times as MPI_T_init_thread() by the end of
+execution. Calls to MPI tool functions are allowed at any point in
+execution as long as MPI_T_init_thread() has been called at least once
+and the number of calls to MPI_T_init_thread() is greater than the
+number of calls to MPI_T_finalize(). If at any point in execution the
+number of calls to MPI_T_finalize() equals the number of calls to
+MPI_T_init_thread() the MPI tool interface will no longer be available
+until another call to MPI_T_init_thread().
+
+
+NOTES
+-----
+
+Before the end of execution the number of calls to MPI_T_init_thread()
+and :ref:`MPI_T_finalize` must be the same.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_finalize` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
+
+
+.. seealso::
+   * :ref:`MPI_T_init_thread`
diff --git a/docs/man-openmpi/man3/MPI_T_init_thread.3.rst b/docs/man-openmpi/man3/MPI_T_init_thread.3.rst
new file mode 100644
index 00000000000..8a9986495fe
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_init_thread.3.rst
@@ -0,0 +1,71 @@
+.. _mpi_t_init_thread:
+
+MPI_T_init_thread
+=================
+
+.. include_body
+
+:ref:`MPI_T_init_thread` |mdash| Initializes the MPI Tool information interface
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_init_thread.rst
+
+INPUT PARAMETERS
+----------------
+
+* ``required``: Desired level of thread support (integer).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``provided``: Available level of thread support (integer).
+
+DESCRIPTION
+-----------
+
+MPI_T_init_thread() initializes the MPI tool information interface.
+Calls to MPI tool functions are allowed at any point in execution
+(including before MPI_Init() and after MPI_Finalize()) as long
+as MPI_T_init_thread() has been called at least once and the number
+of calls to MPI_T_init_thread() is greater than the number of calls
+to MPI_T_finalize(). If at any point in execution the number of
+calls to MPI_T_finalize() equals the number of calls to
+MPI_T_init_thread() the MPI tool interface will no longer be
+available until another call to MPI_T_init_thread().
+
+MPI_T_init_thread(), like MPI_Init_thread(), has a provision to
+request a certain level of thread support in ``required``:
+
+* ``MPI_THREAD_SINGLE``: Only one thread will execute.
+* ``MPI_THREAD_FUNNELED``: If the process is multithreaded, only the
+  thread that called :ref:`MPI_Init_thread` will make MPI calls.
+* ``MPI_THREAD_SERIALIZED``: If the process is multithreaded, only one
+  thread will make MPI library calls at one time.
+* ``MPI_THREAD_MULTIPLE``: If the process is multithreaded, multiple
+  threads may call MPI at once with no restrictions.
+
+The level of thread support available to the program is set in
+``provided``. In Open MPI, the value is dependent on how the library was
+configured and built. Note that there is no guarantee that ``provided``
+will be greater than or equal to ``required``.
+
+NOTES
+-----
+
+It is the caller's responsibility to check the value of ``provided``, as
+it may be less than what was requested in ``required``.
+
+ERRORS
+------
+
+:ref:`MPI_T_init_thread` will fail if:
+
+* ``MPI_T_ERR_MEMORY``: Out of memory
+* ``MPI_T_ERR_CANNOT_INIT``: Interface not in the state to be
+  initialized
+
+
+.. seealso::
+   * :ref:`MPI_T`
+   * :ref:`MPI_Init`
+   * :ref:`MPI_Init_thread`
diff --git a/docs/man-openmpi/man3/MPI_T_pvar_get_info.3.rst b/docs/man-openmpi/man3/MPI_T_pvar_get_info.3.rst
new file mode 100644
index 00000000000..7d13839bfc5
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_pvar_get_info.3.rst
@@ -0,0 +1,184 @@
+.. _mpi_t_pvar_get_info:
+
+
+MPI_T_pvar_get_info
+===================
+
+.. include_body
+
+:ref:`MPI_T_pvar_get_info` |mdash| Query information from a performance variable
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_pvar_get_info.rst
+
+INPUT PARAMETERS
+----------------
+* ``pvar_index``: Index of the performance variable to be queried.
+
+INPUT/OUTPUT PARAMETERS
+-----------------------
+* ``name_len``: Length of the string and/or buffer for name.
+* ``desc_len``: Length of the string and/or buffer for desc.
+
+OUTPUT PARAMETERS
+-----------------
+* ``name``: Buffer to return the string containing the name of the performance variable.
+* ``verbosity``: Verbosity level of this variable.
+* ``var_class``: Class of performance variable.
+* ``datatype``: MPI datatype of the information stored in the performance variable.
+* ``enumtype``: Optional descriptor for enumeration information.
+* ``desc``: Buffer to return the string containing the description of the performance variable.
+* ``bind``: Type of MPI object to which this variable must be bound.
+* ``readonly``: Flag indicating whether the variable can be written/reset.
+* ``continuous``: Flag indicating whether the variable can be started and stopped or is continuously active.
+* ``atomic``: Flag indicating whether the variable can be atomically read and reset.
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_pvar_get_info` can be used to query information from a performance
+variable. The function returns the verbosity, class, datatype,
+enumeration type, and binding of the queried control variable in the
+arguments *verbosity*, *var_class*, *datatype*, *enumtype*, and *bind*
+respectively. Flags indicating whether the variable is read-only,
+continuous, or atomic are returns in *readonly*, *continuous*, and
+*atomic* accordingly. See MPI-3 section 14.3.7 for more information. See the
+man page for :ref:`MPI_T_cvar_get_info` for information on variable verbosity.
+
+
+VARIABLE CLASS
+--------------
+
+Performance variables are categorized into classes which describe their
+initial value, valid types, and behavior. The class returned in the
+*var_class* parameter may be one of the following:
+
+* ``MPI_T_PVAR_CLASS_STATE``: Variable represents a set of discrete
+  states that may be described by an enumerator. Variables of this
+  class must be represented by an MPI_INT. The starting value is the
+  current state of the variable.
+
+* ``MPI_T_PVAR_CLASS_LEVEL``: Variable represents the current
+  utilization level of a resource.  Variables of this class must be
+  represented by an MPI_UNSIGNED, MPI_UNSIGNED_LONG,
+  MPI_UNSIGNED_LONG_LONG, or MPI_DOUBLE. The starting value is the
+  current utilization level of the resource.
+
+* ``MPI_T_PVAR_CLASS_SIZE``: Variable represents the fixed size of a
+  resource. Variables of this class are represented by an
+  MPI_UNSIGNED, MPI_UNSIGNED_LONG, MPI_UNSIGNED_LONG_LONG, or
+  MPI_DOUBLE. The starting value is the current size of the resource.
+
+* ``MPI_T_PVAR_CLASS_PERCENTAGE``: Variable represents the current
+  percentage utilization level of a resource. Variables of this class
+  are represented by an MPI_DOUBLE.  The starting value is the current
+  percentage utilization of the resource.
+
+* ``MPI_T_PVAR_CLASS_HIGHWATERMARK``: Variable represents the high
+  watermark of the utilization of a resource. Variables of this class
+  are represented by an MPI_UNSIGNED, MPI_UNSIGNED_LONG,
+  MPI_UNSIGNED_LONG_LONG, or MPI_DOUBLE. The starting value is the
+  current utilization of the resource.
+
+* ``MPI_T_PVAR_CLASS_HIGHWATERMARK``: Variable represents the low
+  watermark of the utilization of a resource. Variables of this class
+  are represented by an MPI_UNSIGNED, MPI_UNSIGNED_LONG,
+  MPI_UNSIGNED_LONG_LONG, or MPI_DOUBLE. The starting value is the
+  current utilization of the resource.
+
+* ``MPI_T_PVAR_CLASS_COUNTER``: Variable represents a count of the
+  number of occurrences of a specific event. Variables of this class
+  are represented by an MPI_UNSIGNED, MPI_UNSIGNED_LONG, or
+  MPI_UNSIGNED_LONG_LONG. The starting value is 0.
+
+* ``MPI_T_PVAR_CLASS_COUNTER``: Variable represents an aggregated
+  value that represents a sum of arguments processed during a specific
+  event. Variables of this class are represented by an MPI_UNSIGNED,
+  MPI_UNSIGNED_LONG, MPI_UNSIGNED_LONG_LONG, or MPI_DOUBLE. The
+  starting value is 0.
+
+* ``MPI_T_PVAR_CLASS_TIMER``: Variable represents the aggregated time
+  spent by the MPI implementation while processing an event, type of
+  event, or section of code. Variables of this class are represented
+  by an MPI_UNSIGNED, MPI_UNSIGNED_LONG, MPI_UNSIGNED_LONG_LONG, or
+  MPI_DOUBLE. If the variable is represented by an MPI_DOUBLE the
+  units will be the same as those used by MPI_Wtime(). The starting
+  value is 0.
+
+* ``MPI_T_PVAR_CLASS_GENERIC``: Variable does not fit into any other
+  class. Can by represented by an type supported by the MPI tool
+  information interface (see DATATYPE).  Starting value is variable
+  specific.
+
+For more information see MPI-3 section 14.3.7.
+
+
+DATATYPE
+--------
+
+The datatype returned by :ref:`MPI_T_pvar_get_info` is restricted to one of the
+following datatypes: MPI_INT, MPI_UNSIGNED, MPI_UNSIGNED_LONG,
+MPI_UNSIGNED_LONG_LONG, MPI_COUNT, MPI_CHAR, and MPI_DOUBLE. For more
+information on datatypes in the MPI Tool information interface see MPI-3
+section 14.3.5.
+
+
+BINDING
+-------
+
+Performance variables may be bound to an MPI object. The binding
+returned in the *bind* parameter may be one of the following:
+
+* ``MPI_T_BIND_NO_OBJECT``: No object
+
+* ``MPI_T_BIND_MPI_COMM``: MPI communicator
+
+* ``MPI_T_BIND_MPI_DATATYPE``: MPI datatype
+
+* ``MPI_T_BIND_MPI_ERRHANDLER``: MPI error handler
+
+* ``MPI_T_BIND_MPI_FILE``: MPI file handle
+
+* ``MPI_T_BIND_MPI_GROUP``: MPI group
+
+* ``MPI_T_BIND_MPI_OP``: MPI reduction operator
+
+* ``MPI_T_BIND_MPI_REQUEST``: MPI request
+
+* ``MPI_T_BIND_MPI_WIN``: MPI window for one-sided communication
+
+* ``MPI_T_BIND_MPI_MESSAGE``: MPI message object
+
+* ``MPI_T_BIND_MPI_INFO``: MPI info object
+
+* ``MPI_T_BIND_MPI_SESSION``: MPI session
+
+For more information see MPI-4 section 14.3.2.
+
+
+NOTES
+-----
+
+This MPI tool interface function returns two strings. This function
+takes two argument for each string: a buffer to store the string, and
+a length which must initially specify the size of the buffer. If the
+length passed is n then this function will copy at most n - 1
+characters of the string into the corresponding buffer and set the
+length to the number of characters copied - 1. If the length argument
+is NULL or the value specified in the length is 0 the corresponding
+string buffer is ignored and the string is not returned. For more
+information see MPI-3 section 14.3.3.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_pvar_get_info` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
+
+* ``MPI_T_ERR_INVALID_INDEX``: The performance variable index is invalid
+
+
+.. seealso::
+   * :ref:`MPI_T_cvar_get_info`
diff --git a/docs/man-openmpi/man3/MPI_T_pvar_get_num.3.rst b/docs/man-openmpi/man3/MPI_T_pvar_get_num.3.rst
new file mode 100644
index 00000000000..ea3428b76dc
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_pvar_get_num.3.rst
@@ -0,0 +1,31 @@
+.. _mpi_t_pvar_get_num:
+
+
+MPI_T_pvar_get_num
+==================
+
+.. include_body
+
+:ref:`MPI_T_pvar_get_num` |mdash| Query the number of performance variables
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_pvar_get_num.rst
+
+OUTPUT PARAMETERS
+-----------------
+* ``num_pvar``: Current number of performance variables.
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_pvar_get_num` can be used to query the current number of
+performance variables. The number of performance variables may increase
+throughout the execution of the process but will never decrease.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_pvar_get_num` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
diff --git a/docs/man-openmpi/man3/MPI_T_pvar_handle_alloc.3.rst b/docs/man-openmpi/man3/MPI_T_pvar_handle_alloc.3.rst
new file mode 100644
index 00000000000..35c2838b00a
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_pvar_handle_alloc.3.rst
@@ -0,0 +1,65 @@
+.. _mpi_t_pvar_handle_alloc:
+
+
+MPI_T_pvar_handle_alloc
+=======================
+
+.. include_body
+
+:ref:`MPI_T_pvar_handle_alloc`, :ref:`MPI_T_pvar_handle_free` - Allocate/free
+MPI performance variable handles
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_T_pvar_handle_alloc, MPI_T_pvar_handle_free
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_pvar_handle_alloc.rst
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_pvar_handle_alloc` binds the performance variable specified in
+*pvar_index* to the MPI object specified in *obj_handle* in the session
+identified by the parameter *session*. The object is passed in the
+argument *obj_handle* as an address to a local variable that stores the
+object’s handle. If :ref:`MPI_T_pvar_get_info` returns MPI_T_BIND_NO_OBJECT as
+the binding for the variable the *obj_handle* argument is ignored. The
+handle allocated to reference the variable is returned in the argument
+*handle*. Upon successful return, *count* contains the number of
+elements (of the datatype returned by a previous :ref:`MPI_T_PVAR_GET_INFO`
+call) used to represent this variable.
+
+The value of *pvar_index* should be in the range 0 to *num_pvar - 1*,
+where *num_pvar* is the number of available performance variables as
+determined from a prior call to :ref:`MPI_T_PVAR_GET_NUM`. The type of the
+MPI object it references must be consistent with the type returned in
+the bind argument in a prior call to :ref:`MPI_T_PVAR_GET_INFO`.
+
+:ref:`MPI_T_pvar_handle_free` frees a handle allocated by
+:ref:`MPI_T_pvar_handle_alloc` and sets the *handle* argument to
+MPI_T_PVAR_HANDLE_NULL.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_pvar_handle_alloc` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
+
+* ``MPI_T_ERR_INVALID_INDEX``: The performance variable index is invalid
+
+* ``MPI_T_ERR_OUT_OF_HANDLES``: No more handles available
+
+:ref:`MPI_T_pvar_handle_free` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
+
+* ``MPI_T_ERR_INVALID_HANDLE``: The handle is invalid or the handle
+  argument passed in is not associated with the session argument
+
+
+.. seealso::
+   * :ref:`MPI_T_pvar_get_info`
+   * :ref:`MPI_T_pvar_get_num`
diff --git a/docs/man-openmpi/man3/MPI_T_pvar_handle_free.3.rst b/docs/man-openmpi/man3/MPI_T_pvar_handle_free.3.rst
new file mode 100644
index 00000000000..7ec6ea15763
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_pvar_handle_free.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_t_pvar_handle_free:
+
+MPI_T_pvar_handle_free
+======================
+
+.. include_body
+
+.. include:: ../man3/MPI_T_pvar_handle_alloc.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_T_pvar_read.3.rst b/docs/man-openmpi/man3/MPI_T_pvar_read.3.rst
new file mode 100644
index 00000000000..018f57837cd
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_pvar_read.3.rst
@@ -0,0 +1,45 @@
+.. _mpi_t_pvar_read:
+
+
+MPI_T_pvar_read
+===============
+
+.. include_body
+
+:ref:`MPI_T_pvar_read` |mdash| Read the value of a performance variable
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_pvar_read.rst
+
+INPUT PARAMETERS
+----------------
+* ``session``: Performance experiment session.
+* ``handle``: Performance variable handle.
+* ``buf``: Initial address of storage location for variable value.
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_pvar_read` queries the value of a performance variable identified
+by the handle specified in *handle* in the session specified in
+*session*. The result is stored in the buffer pointed to by *buf*. The
+caller must ensure that the buffer pointed to by *buf* is large enough
+to hold the entire value of the performance variable.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_pvar_read` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
+
+* ``MPI_T_ERR_INVALID_HANDLE``: The handle is invalid or not associated with the session
+
+* ``MPI_T_ERR_INVALID_SESSION``: Session argument is not a valid session
+
+
+.. seealso::
+   * :ref:`MPI_T_pvar_handle_alloc`
+   * :ref:`MPI_T_pvar_get_info`
+   * :ref:`MPI_T_pvar_session_create`
diff --git a/docs/man-openmpi/man3/MPI_T_pvar_readreset.3.rst b/docs/man-openmpi/man3/MPI_T_pvar_readreset.3.rst
new file mode 100644
index 00000000000..eccd979a611
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_pvar_readreset.3.rst
@@ -0,0 +1,54 @@
+.. _mpi_t_pvar_readreset:
+
+
+MPI_T_pvar_readreset
+====================
+
+.. include_body
+
+:ref:`MPI_T_pvar_readreset` |mdash| Atomically read and reset the value of a
+performance variable
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_pvar_readreset.rst
+
+INPUT PARAMETERS
+----------------
+* ``session``: Performance experiment session.
+* ``handle``: Performance variable handle.
+* ``buf``: Initial address of storage location for variable value.
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_pvar_readreset` atomically queries and resets the value of a
+performance variable bound to the handle specified by *handle* in the
+session specified by *session*. The result is stored in the buffer
+pointed to by *buf*. This function can only be used with performance
+variables that are atomic and not readonly. The caller must ensure that
+the buffer pointed to by *buf* is large enough to hold the entire value
+of the performance variable.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_pvar_readreset` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
+
+* ``MPI_T_ERR_INVALID_HANDLE``: The handle is invalid or not associated with the session
+
+* ``MPI_T_ERR_INVALID_SESSION``: Session argument is not a valid session
+
+* ``MPI_T_ERR_PVAR_NO_ATOMIC``: Variable cannot be read and written atomically
+
+* ``MPI_T_ERR_PVAR_NO_WRITE``: Variable cannot be reset
+
+
+.. seealso::
+   * :ref:`MPI_T_pvar_handle_alloc`
+   * :ref:`MPI_T_pvar_get_info`
+   * :ref:`MPI_T_pvar_session_create`
+   * :ref:`MPI_T_pvar_read`
+   * :ref:`MPI_T_pvar_reset`
diff --git a/docs/man-openmpi/man3/MPI_T_pvar_reset.3.rst b/docs/man-openmpi/man3/MPI_T_pvar_reset.3.rst
new file mode 100644
index 00000000000..d64ed025052
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_pvar_reset.3.rst
@@ -0,0 +1,46 @@
+.. _mpi_t_pvar_reset:
+
+
+MPI_T_pvar_reset
+================
+
+.. include_body
+
+:ref:`MPI_T_pvar_reset` |mdash| Reset the value of a performance variable
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_pvar_reset.rst
+
+INPUT PARAMETERS
+----------------
+* ``session``: Performance experiment session.
+* ``handle``: Performance variable handle or MPI_T_PVAR_ALL_HANDLES.
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_pvar_reset` sets the performance variable specified by the handle
+in *handle* to its initial value. The special value
+MPI_T_PVAR_ALL_HANDLES can be passed in *handle* to reset all read-write
+handles in the session specified in *session*.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_pvar_reset` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
+
+* ``MPI_T_ERR_INVALID_HANDLE``: The handle is invalid
+
+* ``MPI_T_ERR_INVALID_SESSION``: Session argument is not a valid session
+
+* ``MPI_T_ERR_PVAR_NO_WRITE``: Variable cannot be reset
+
+
+.. seealso::
+   * :ref:`MPI_T_pvar_handle_alloc`
+   * :ref:`MPI_T_pvar_get_info`
+   * :ref:`MPI_T_pvar_session_create`
+   * :ref:`MPI_T_pvar_write`
diff --git a/docs/man-openmpi/man3/MPI_T_pvar_session_create.3.rst b/docs/man-openmpi/man3/MPI_T_pvar_session_create.3.rst
new file mode 100644
index 00000000000..fa0949fb556
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_pvar_session_create.3.rst
@@ -0,0 +1,45 @@
+.. _mpi_t_pvar_session_create:
+
+
+MPI_T_pvar_session_create
+=========================
+
+.. include_body
+
+:ref:`MPI_T_pvar_session_create`, :ref:`MPI_T_pvar_session_free` - Create/free
+performance variable session
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_T_pvar_session_create, MPI_T_pvar_session_free
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_pvar_session_create.rst
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_pvar_session_create` creates a session for accessing performance
+variables. The new session is returned in the *session* parameter.
+
+:ref:`MPI_T_pvar_session_free` releases a session allocated by
+:ref:`MPI_T_pvar_session_create` and sets the *session* parameter to
+MPI_T_PVAR_SESSION_NULL.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_pvar_session_create` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
+
+* ``MPI_T_ERR_MEMORY``: Out of memory
+
+* ``MPI_T_ERR_OUT_OF_SESSIONS``: No more sessions available
+
+:ref:`MPI_T_pvar_session_free` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
+
+* ``MPI_T_ERR_INVALID_SESSION``: The session parameter is not a valid session
diff --git a/docs/man-openmpi/man3/MPI_T_pvar_session_free.3.rst b/docs/man-openmpi/man3/MPI_T_pvar_session_free.3.rst
new file mode 100644
index 00000000000..ccdc788601a
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_pvar_session_free.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_t_pvar_session_free:
+
+MPI_T_pvar_session_free
+=======================
+
+.. include_body
+
+.. include:: ../man3/MPI_T_pvar_session_create.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_T_pvar_start.3.rst b/docs/man-openmpi/man3/MPI_T_pvar_start.3.rst
new file mode 100644
index 00000000000..5baf4d31f88
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_pvar_start.3.rst
@@ -0,0 +1,55 @@
+.. _mpi_t_pvar_start:
+
+
+MPI_T_pvar_start
+================
+
+.. include_body
+
+:ref:`MPI_T_pvar_start`, :ref:`MPI_T_pvar_stop` - Start/stop a performance
+variable
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_T_pvar_start, MPI_T_pvar_stop
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_pvar_start.rst
+
+INPUT PARAMETERS
+----------------
+* ``session``: Performance experiment session.
+* ``handle``: Performance variable handle.
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_pvar_start` starts the performance variable with the handle
+specified in *handle*. The special value MPI_T_PVAR_ALL_HANDLES can be
+passed in *handle* to start all non-continuous handles in the session
+specified in *session*.
+
+:ref:`MPI_T_pvar_stop` stops the performance variable with the handle specified
+in *handle*. The special value MPI_T_PVAR_ALL_HANDLES can be passed in
+*handle* to stop all non-continuous handles in the session specified in
+*session*.
+
+Continuous performance variables can neither be started nor stopped.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_pvar_start` and MPI_T_pvar_stop() will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
+
+* ``MPI_T_ERR_INVALID_SESSION``: Session parameter is not a valid session
+
+* ``MPI_T_ERR_INVALID_HANDLE``: Invalid handle or handle not associated with the session
+
+* ``MPI_T_ERR_PVAR_NO_STARTSTOP``: The variable cannot be started or stopped
+
+
+.. seealso::
+   * :ref:`MPI_T_pvar_get_info`
diff --git a/docs/man-openmpi/man3/MPI_T_pvar_stop.3.rst b/docs/man-openmpi/man3/MPI_T_pvar_stop.3.rst
new file mode 100644
index 00000000000..b6c530c1d07
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_pvar_stop.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_t_pvar_stop:
+
+MPI_T_pvar_stop
+===============
+
+.. include_body
+
+.. include:: ../man3/MPI_T_pvar_start.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_T_pvar_write.3.rst b/docs/man-openmpi/man3/MPI_T_pvar_write.3.rst
new file mode 100644
index 00000000000..d0f9a52dcad
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_pvar_write.3.rst
@@ -0,0 +1,47 @@
+.. _mpi_t_pvar_write:
+
+
+MPI_T_pvar_write
+================
+
+.. include_body
+
+:ref:`MPI_T_pvar_write` |mdash| Write the value of a control variable
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_pvar_write.rst
+
+INPUT PARAMETERS
+----------------
+* ``session``: Performance experiment session.
+* ``handle``: Performance variable handle.
+* ``buf``: Initial address of storage location for variable value.
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_pvar_write` attempts to set the value of the performance variable
+identified by the handle specified in *handle* in the session specified
+in *session*. The value to be written is specified in *buf*. The caller
+must ensure that the buffer specified in *buf* is large enough to hold
+the entire value of the performance variable.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_pvar_write` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
+
+* ``MPI_T_ERR_INVALID_HANDLE``: The handle is invalid or not associated with the session
+
+* ``MPI_T_ERR_INVALID_SESSION``: Session argument is not a valid session
+
+* ``MPI_T_ERR_PVAR_NO_WRITE``: Variable cannot be written
+
+
+.. seealso::
+   * :ref:`MPI_T_pvar_handle_alloc`
+   * :ref:`MPI_T_pvar_get_info`
+   * :ref:`MPI_T_pvar_session_create`
diff --git a/docs/man-openmpi/man3/MPI_T_source_get_info.3.rst b/docs/man-openmpi/man3/MPI_T_source_get_info.3.rst
new file mode 100644
index 00000000000..819fcaf4402
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_source_get_info.3.rst
@@ -0,0 +1,46 @@
+.. _mpi_t_source_get_info:
+
+
+MPI_T_source_get_info
+=====================
+
+.. include_body
+
+:ref:`MPI_T_source_get_info` |mdash| Query information from a source
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_source_get_info.rst
+
+INPUT PARAMETERS
+----------------
+* ``source_index``: Index of the source to be queried.
+
+INPUT/OUTPUT PARAMETERS
+-----------------------
+* ``name_len``: Length of the string and/or buffer for name.
+* ``desc_len``: Length of the string and/or buffer for desc.
+
+OUTPUT PARAMETERS
+-----------------
+* ``name``: Buffer to return the string containing the name of the source.
+* ``desc``: Buffer to return the string containing the description of the source.
+* ``ordering``: Flag indicating chronological ordering guarantees given by the source.
+* ``ticks_per_second``: The number of ticks per second for the timer of this source.
+* ``max_ticks``: The maximum count of ticks reported by this source before overflow occurs.
+* ``info``: Info argument.
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_source_get_info` can be used to query information from a
+source.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_source_get_info` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
+
+* ``MPI_T_ERR_INVALID_INDEX``: The source index is invalid
diff --git a/docs/man-openmpi/man3/MPI_T_source_get_num.3.rst b/docs/man-openmpi/man3/MPI_T_source_get_num.3.rst
new file mode 100644
index 00000000000..31c478a9a26
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_source_get_num.3.rst
@@ -0,0 +1,45 @@
+.. _mpi_t_source_get_num:
+
+
+MPI_T_source_get_num
+====================
+
+.. include_body
+
+:ref:`MPI_T_source_get_num` |mdash| Query the number of MPI_T event sources
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_source_get_num.rst
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``num_sources``: Current number of sources
+
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_source_get_num` can be used to query the current number of
+MPI_T event sources.
+
+The number of available event sources can be queried with a call to
+:ref:`MPI_T_Source_get_num`. An MPI implementation is allowed to
+increase the number of sources during the execution of an MPI
+process. However, MPI implementations are not allowed to change the
+index of an event source or to delete an event source once it has been
+made visible to the user (e.g., if new event sources become available
+via dynamic loading of additional components in the MPI
+implementation).
+
+.. note:: Open MPI will currently return that there are 0 sources of
+          MPI_T events.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_source_get_num` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface is not
+  initialized
diff --git a/docs/man-openmpi/man3/MPI_T_source_get_timestamp.3.rst b/docs/man-openmpi/man3/MPI_T_source_get_timestamp.3.rst
new file mode 100644
index 00000000000..d7d67bc32da
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_T_source_get_timestamp.3.rst
@@ -0,0 +1,35 @@
+.. _mpi_t_source_get_timestamp:
+
+
+MPI_T_source_get_timestamp
+==========================
+
+.. include_body
+
+:ref:`MPI_T_source_get_timestamp` |mdash| Returns a current timestamp from the source identified by the source_index argument
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_t_source_get_timestamp.rst
+
+INPUT PARAMETERS
+----------------
+* ``source_index``: Index of the source to be queried.
+
+OUTPUT PARAMETERS
+-----------------
+* ``timestamp``: The current timestamp from specified source.
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_T_source_get_timestamp` returns the current timestamp from the specificed source.
+
+
+ERRORS
+------
+
+:ref:`MPI_T_source_get_timestamp` will fail if:
+
+* ``MPI_T_ERR_NOT_INITIALIZED``: The MPI Tools interface not initialized
+
+* ``MPI_T_ERR_INVALID_INDEX``: The source index is invalid
diff --git a/docs/man-openmpi/man3/MPI_Test.3.rst b/docs/man-openmpi/man3/MPI_Test.3.rst
new file mode 100644
index 00000000000..0b8cc8f7681
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Test.3.rst
@@ -0,0 +1,90 @@
+.. _mpi_test:
+
+
+MPI_Test
+========
+
+.. include_body
+
+:ref:`MPI_Test` |mdash| Tests for the completion of a specific send or receive.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_test.rst
+
+INPUT PARAMETER
+---------------
+* ``request``: Communication request (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``flag``: True if operation completed (logical).
+* ``status``: Status object (status).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+A call to :ref:`MPI_Test` returns flag = true if the operation identified by
+request is complete. In such a case, the status object is set to contain
+information on the completed operation; if the communication object was
+created by a nonblocking send or receive, then it is deallocated and the
+request handle is set to ``MPI_REQUEST_NULL``. The call returns flag =
+false, otherwise. In this case, the value of the status object is
+undefined. :ref:`MPI_Test` is a local operation.
+
+The return status object for a receive operation carries information
+that can be accessed as described in the "Return Status" subsection of
+the "Point-to-Point Communication" chapter in the `MPI Standard
+<https://www.mpi-forum.org/docs/>`_. The status object for a send
+operation carries information that can be accessed by a call to
+:ref:`MPI_Test_cancelled` (see the "Probe and Cancel" section of the
+`MPI Standard <https://www.mpi-forum.org/docs/>`_).
+
+If your application does not need to examine the *status* field, you can
+save resources by using the predefined constant ``MPI_STATUS_IGNORE`` as a
+special value for the *status* argument.
+
+One is allowed to call :ref:`MPI_Test` with a null or inactive *request*
+argument. In such a case the operation returns with *flag* = true and
+empty *status*.
+
+The functions :ref:`MPI_Wait` and :ref:`MPI_Test` can be used to complete both sends
+and receives.
+
+
+NOTES
+-----
+
+The use of the nonblocking :ref:`MPI_Test` call allows the user to schedule
+alternative activities within a single thread of execution. An
+event-driven thread scheduler can be emulated with periodic calls to
+:ref:`MPI_Test`.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+Note that per the "Return Status" section in the "Point-to-Point
+Communication" chapter in the `MPI Standard
+<https://www.mpi-forum.org/docs/>`_, MPI errors on requests passed to
+:ref:`MPI_TEST` do not set the ``status.MPI_ERROR`` field in the
+returned *status*.  The error code is always passed to the back-end
+error handler and may be passed back to the caller through the return
+value of :ref:`MPI_TEST` if the back-end error handler returns it.
+The pre-defined MPI error handler ``MPI_ERRORS_RETURN`` exhibits this
+behavior, for example.
+
+
+.. seealso::
+   * :ref:`MPI_Comm_set_errhandler`
+   * :ref:`MPI_File_set_errhandler`
+   * :ref:`MPI_Testall`
+   * :ref:`MPI_Testany`
+   * :ref:`MPI_Testsome`
+   * :ref:`MPI_Wait`
+   * :ref:`MPI_Waitall`
+   * :ref:`MPI_Waitany`
+   * :ref:`MPI_Waitsome`
+   * :ref:`MPI_Win_set_errhandler`
diff --git a/docs/man-openmpi/man3/MPI_Test_cancelled.3.rst b/docs/man-openmpi/man3/MPI_Test_cancelled.3.rst
new file mode 100644
index 00000000000..fd2a7b5873a
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Test_cancelled.3.rst
@@ -0,0 +1,44 @@
+.. _mpi_test_cancelled:
+
+
+MPI_Test_cancelled
+==================
+
+.. include_body
+
+:ref:`MPI_Test_cancelled` |mdash| Tests whether a request was canceled.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_test_cancelled.rst
+
+INPUT PARAMETER
+---------------
+* ``status``: Status object (status).
+
+OUTPUT PARAMETERS
+-----------------
+* ``flag``: True if operation was cancelled (logical).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Returns *flag* = true if the communication associated with the status
+object was canceled successfully. In such a case, all other fields of
+status (such as *count* or *tag*) are undefined. Otherwise, returns
+*flag* = false. If a receive operation might be canceled, one should
+call :ref:`MPI_Test_cancelled` first, to check whether the operation was
+canceled, before checking on the other fields of the return status.
+
+
+NOTES
+-----
+
+Cancel can be an expensive operation that should be used only
+exceptionally.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Testall.3.rst b/docs/man-openmpi/man3/MPI_Testall.3.rst
new file mode 100644
index 00000000000..1d17888bed7
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Testall.3.rst
@@ -0,0 +1,86 @@
+.. _mpi_testall:
+
+
+MPI_Testall
+===========
+
+.. include_body
+
+:ref:`MPI_Testall` |mdash| Tests for the completion of all previously initiated
+communications in a list.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_testall.rst
+
+INPUT PARAMETERS
+----------------
+* ``count``: Lists length (integer).
+* ``array_of_requests``: Array of requests (array of handles).
+
+OUTPUT PARAMETERS
+-----------------
+* ``flag``: True if previously initiated communications are complete (logical.)
+* ``array_of_statuses``: Array of status objects (array of status).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Returns *flag* = true if all communications associated with active
+handles in the array have completed (this includes the case where no
+handle in the list is active). In this case, each status entry that
+corresponds to an active handle request is set to the status of the
+corresponding communication; if the request was allocated by a
+nonblocking communication call then it is deallocated, and the handle is
+set to MPI_REQUEST_NULL. Each status entry that corresponds to a null or
+inactive handle is set to empty.
+
+Otherwise, *flag* = false is returned, no request is modified and the
+values of the status entries are undefined. This is a local operation.
+
+If your application does not need to examine the *array_of_statuses*
+field, you can save resources by using the predefined constant
+MPI_STATUSES_IGNORE can be used as a special value for the
+*array_of_statuses* argument.
+
+Errors that occurred during the execution of :ref:`MPI_Testall` are handled in
+the same manner as errors in :ref:`MPI_Waitall`.
+
+
+NOTE
+----
+
+*flag* is true only if all requests have completed. Otherwise, *flag* is
+false, and neither *array_of_requests* nor *array_of_statuses* is
+modified.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+For each invocation of :ref:`MPI_Testall`, if one or more requests
+generate an MPI error, only the *first* MPI request that caused an
+error will be passed to its corresponding error handler. No other
+error handlers will be invoked (even if multiple requests generated
+errors). However, *all* requests that generate an error will have a
+relevant error code set in the corresponding ``status.MPI_ERROR``
+field (unless ``MPI_STATUSES_IGNORE`` was used).
+
+If the invoked error handler allows :ref:`MPI_Testall` to return to
+the caller, the value ``MPI_ERR_IN_STATUS`` will be returned in the C
+and Fortran bindings.
+
+
+.. seealso::
+   * :ref:`MPI_Comm_set_errhandler`
+   * :ref:`MPI_File_set_errhandler`
+   * :ref:`MPI_Test`
+   * :ref:`MPI_Testany`
+   * :ref:`MPI_Testsome`
+   * :ref:`MPI_Wait`
+   * :ref:`MPI_Waitall`
+   * :ref:`MPI_Waitany`
+   * :ref:`MPI_Waitsome`
+   * :ref:`MPI_Win_set_errhandler`
diff --git a/docs/man-openmpi/man3/MPI_Testany.3.rst b/docs/man-openmpi/man3/MPI_Testany.3.rst
new file mode 100644
index 00000000000..a2326c008cc
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Testany.3.rst
@@ -0,0 +1,83 @@
+.. _mpi_testany:
+
+
+MPI_Testany
+===========
+
+.. include_body
+
+:ref:`MPI_Testany` |mdash| Tests for completion of any one previously initiated
+communication in a list.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_testany.rst
+
+INPUT PARAMETERS
+----------------
+* ``count``: List length (integer).
+* ``array_of_requests``: Array of requests (array of handles).
+
+OUTPUT PARAMETERS
+-----------------
+* ``index``: Index of operation that completed, or ``MPI_UNDEFINED`` if none completed (integer).
+* ``flag``: True if one of the operations is complete (logical).
+* ``status``: Status object (status).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Testany` tests for completion of either one or none of the operations
+associated with active handles. In the former case, it returns *flag* =
+true, returns in *index* the index of this request in the array, and
+returns in *status* the status of that operation; if the request was
+allocated by a nonblocking communication call then the request is
+deallocated and the handle is set to ``MPI_REQUEST_NULL``. (The array is
+indexed from 0 in C, and from 1 in Fortran.) In the latter case (no
+operation completed), it returns *flag* = false, returns a value of
+``MPI_UNDEFINED`` in *index*, and *status* is undefined.
+
+The array may contain null or inactive handles. If the array contains no
+active handles then the call returns immediately with *flag* = true,
+*index* = ``MPI_UNDEFINED``, and an empty *status*.
+
+If the array of requests contains active handles then the execution of
+``MPI_Testany(count, array_of_requests, index, status)`` has the same effect
+as the execution of ``MPI_Test(&array_of_requests[i], flag, status)``,
+for *i*\ =0,1,...,count-1, in some arbitrary order, until one
+call returns ``flag = true``, or all fail. In the former case, *index* is
+set to the last value of *i*, and in the latter case, it is set to
+``MPI_UNDEFINED``. :ref:`MPI_Testany` with an array containing one active entry is
+equivalent to :ref:`MPI_Test`.
+
+If your application does not need to examine the *status* field, you can
+save resources by using the predefined constant ``MPI_STATUS_IGNORE`` as a
+special value for the *status* argument.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+Note that per the "Return Status" section in the "Point-to-Point
+Communication" chapter in the `MPI Standard
+<https://www.mpi-forum.org/docs/>`_, MPI errors on requests passed to
+:ref:`MPI_TESTANY` do not set the ``status.MPI_ERROR`` field in the
+returned *status*.  The error code is always passed to the back-end
+error handler and may be passed back to the caller through the return
+value of :ref:`MPI_TESTANY` if the back-end error handler returns it.
+The pre-defined MPI error handler ``MPI_ERRORS_RETURN`` exhibits this
+behavior, for example.
+
+.. seealso::
+   * :ref:`MPI_Comm_set_errhandler`
+   * :ref:`MPI_File_set_errhandler`
+   * :ref:`MPI_Test`
+   * :ref:`MPI_Testall`
+   * :ref:`MPI_Testsome`
+   * :ref:`MPI_Wait`
+   * :ref:`MPI_Waitall`
+   * :ref:`MPI_Waitany`
+   * :ref:`MPI_Waitsome`
+   * :ref:`MPI_Win_set_errhandler`
diff --git a/docs/man-openmpi/man3/MPI_Testsome.3.rst b/docs/man-openmpi/man3/MPI_Testsome.3.rst
new file mode 100644
index 00000000000..1ccca7ae9e1
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Testsome.3.rst
@@ -0,0 +1,106 @@
+.. _mpi_testsome:
+
+
+MPI_Testsome
+============
+
+.. include_body
+
+:ref:`MPI_Testsome` |mdash| Tests for completion of one or more previously
+initiated communications in a list.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_testsome.rst
+
+INPUT PARAMETERS
+----------------
+* ``incount``: Length of array_of_requests (integer).
+* ``array_of_requests``: Array of requests (array of handles).
+
+OUTPUT PARAMETERS
+-----------------
+* ``outcount``: Number of completed requests (integer).
+* ``array_of_indices``: Array of indices of operations that completed (array of integers).
+* ``array_of_statuses``: Array of status objects for operations that completed (array of status).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Behaves like :ref:`MPI_Waitsome`, except that it returns immediately.
+
+Returns in outcount the number of requests from the list
+array_of_requests that have completed. Returns in the first outcount
+locations of the array array_of_indices the indices of these operations
+(index within the array array_of_requests; the array is indexed from 0
+in C and from 1 in Fortran). Returns in the first outcount locations of
+the array array_of_status the status for these completed operations. If
+a request that completed was allocated by a nonblocking communication
+call, then it is deallocated, and the associated handle is set to
+MPI_REQUEST_NULL.
+
+If no operation has completed it returns outcount = 0. If there is no
+active handle in the list, it returns outcount = MPI_UNDEFINED.
+
+:ref:`MPI_Testsome` is a local operation, which returns immediately, whereas
+:ref:`MPI_Waitsome` blocks until a communication completes, if it was passed a
+list that contains at least one active handle. Both calls fulfill a
+fairness requirement: If a request for a receive repeatedly appears in a
+list of requests passed to :ref:`MPI_Waitsome` or :ref:`MPI_Testsome`, and a matching
+send has been posted, then the receive will eventually succeed unless
+the send is satisfied by another receive; send requests also fulfill
+this fairness requirement.
+
+Errors that occur during the execution of :ref:`MPI_Testsome` are handled as
+for :ref:`MPI_Waitsome`.
+
+If your application does not need to examine the *array_of_statuses*
+field, you can save resources by using the predefined constant
+MPI_STATUSES_IGNORE can be used as a special value for the
+*array_of_statuses* argument.
+
+
+NOTES
+-----
+
+The use of :ref:`MPI_Testsome` is likely to be more efficient than the use of
+:ref:`MPI_Testany`. The former returns information on all completed
+communications; with the latter, a new call is required for each
+communication that completes.
+
+A server with multiple clients can use :ref:`MPI_Waitsome` so as not to starve
+any client. Clients send messages to the server with service requests.
+The server calls :ref:`MPI_Waitsome` with one receive request for each client,
+then handles all receives that have completed. If a call to :ref:`MPI_Waitany`
+is used instead, then one client could starve while requests from
+another client always sneak in first.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+For each invocation of :ref:`MPI_Testsome`, if one or more requests
+generate an MPI error, only the *first* MPI request that caused an
+error will be passed to its corresponding error handler. No other
+error handlers will be invoked (even if multiple requests generated
+errors). However, *all* requests that generate an error will have a
+relevant error code set in the corresponding ``status.MPI_ERROR``
+field (unless ``MPI_STATUSES_IGNORE`` was used).
+
+If the invoked error handler allows :ref:`MPI_Testsome` to return to
+the caller, the value ``MPI_ERR_IN_STATUS`` will be returned in the C
+and Fortran bindings.
+
+.. seealso::
+   * :ref:`MPI_Comm_set_errhandler`
+   * :ref:`MPI_File_set_errhandler`
+   * :ref:`MPI_Test`
+   * :ref:`MPI_Testall`
+   * :ref:`MPI_Testany`
+   * :ref:`MPI_Wait`
+   * :ref:`MPI_Waitall`
+   * :ref:`MPI_Waitany`
+   * :ref:`MPI_Waitsome`
+   * :ref:`MPI_Win_set_errhandler`
diff --git a/docs/man-openmpi/man3/MPI_Topo_test.3.rst b/docs/man-openmpi/man3/MPI_Topo_test.3.rst
new file mode 100644
index 00000000000..d5c88cafc22
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Topo_test.3.rst
@@ -0,0 +1,47 @@
+.. _mpi_topo_test:
+
+
+MPI_Topo_test
+=============
+
+.. include_body
+
+:ref:`MPI_Topo_test` |mdash| Determines the type of topology (if any) associated
+with a communicator.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_topo_test.rst
+
+INPUT PARAMETER
+---------------
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``top_type``: Topology type of communicator comm (choice).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The function :ref:`MPI_Topo_test` returns the type of topology that is assigned
+to a communicator.
+
+The output value *top_type* is one of the following:
+
+::
+
+       MPI_GRAPH        graph topology
+       MPI_CART	        Cartesian topology
+       MPI_DIST_GRAPH   distributed graph topology
+       MPI_UNDEFINED    no topology
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Graph_create`
+   * :ref:`MPI_Cart_create`
diff --git a/docs/man-openmpi/man3/MPI_Type_c2f.3.rst b/docs/man-openmpi/man3/MPI_Type_c2f.3.rst
new file mode 100644
index 00000000000..16436163bfa
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_c2f.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_type_c2f:
+
+MPI_Type_c2f
+============
+    .. include_body
+
+.. include:: ../man3/MPI_Comm_f2c.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Type_commit.3.rst b/docs/man-openmpi/man3/MPI_Type_commit.3.rst
new file mode 100644
index 00000000000..b4238825987
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_commit.3.rst
@@ -0,0 +1,49 @@
+.. _mpi_type_commit:
+
+
+MPI_Type_commit
+===============
+
+.. include_body
+
+:ref:`MPI_Type_commit` |mdash| Commits a data type.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_type_commit.rst
+
+INPUT PARAMETER
+---------------
+* ``datatype``: Data type (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The commit operation commits the data type. A data type is the formal
+description of a communication buffer, not the content of that buffer.
+After a data type has been committed, it can be repeatedly reused to
+communicate the changing content of a buffer or, indeed, the content of
+different buffers, with different starting addresses.
+
+**Example:** The following Fortran code fragment gives examples of using
+:ref:`MPI_Type_commit`.
+
+.. code-block:: fortran
+
+       INTEGER :: type1, type2
+       CALL MPI_TYPE_CONTIGUOUS(5, MPI_REAL, type1, ierr)
+                     ! new type object created
+       CALL MPI_TYPE_COMMIT(type1, ierr)
+                     ! now type1 can be used for communication
+
+If the data type specified in *datatype* is already committed, it is
+equivalent to a no-op.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Type_contiguous.3.rst b/docs/man-openmpi/man3/MPI_Type_contiguous.3.rst
new file mode 100644
index 00000000000..d53a8cf0704
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_contiguous.3.rst
@@ -0,0 +1,69 @@
+.. _mpi_type_contiguous:
+
+
+MPI_Type_contiguous
+===================
+
+.. include_body
+
+:ref:`MPI_Type_contiguous` |mdash| Creates a contiguous datatype.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_type_contiguous.rst
+
+INPUT PARAMETERS
+----------------
+* ``count``: Replication count (nonnegative integer).
+* ``oldtype``: Old datatype (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``newtype``: New datatype (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The simplest datatype constructor is :ref:`MPI_Type_contiguous`, which allows
+replication of a datatype into contiguous locations.
+
+*newtype* is the datatype obtained by concatenating *count* copies of
+*oldtype*. Concatenation is defined using the extent of *oldtype* as the
+size of the concatenated copies.
+
+**Example:** Let oldtype have type map {(double, 0), (char, 8)}, with
+extent 16, and let count = 3. The type map of the datatype returned by
+newtype is
+
+.. code-block:: c
+
+       {(double, 0), (char, 8), (double, 16), (char, 24),
+       (double, 32), (char, 40)];
+
+i.e., alternating double and char elements, with displacements 0, 8, 16,
+24, 32, 40.
+
+In general, assume that the type map of oldtype is
+
+.. code-block:: c
+
+       {(type(0), disp(0)),...,(type(n-1), disp(n-1))},
+
+with extent ex. Then newtype has a type map with count times n entries
+defined by:
+
+.. code-block:: c
+
+       {(type(0), disp(0)), ...,(type(n-1), disp(n-1)),
+       (type(0), disp(0) + ex), ...,(type(n-1),
+       disp(n-1) + ex), ...,(type(0), disp(0) + ex * (count - 1)),
+       ...,(type(n-1), disp(n-1) + ex * (count - 1))}.
+
+For more information about derived datatypes, see the Datatypes
+chapter in the `MPI Standard <https://www.mpi-forum.org/docs/>`_.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Type_create_darray.3.rst b/docs/man-openmpi/man3/MPI_Type_create_darray.3.rst
new file mode 100644
index 00000000000..1543f86e46e
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_create_darray.3.rst
@@ -0,0 +1,114 @@
+.. _mpi_type_create_darray:
+
+
+MPI_Type_create_darray
+======================
+
+.. include_body
+
+:ref:`MPI_Type_create_darray` |mdash| Creates a distributed array datatype;
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_type_create_darray.rst
+
+INPUT PARAMETERS
+----------------
+* ``size``: Size of process group (positive integer).
+* ``rank``: Rank in process group (nonnegative integer).
+* ``ndims``: Number of array dimensions as well as process grid dimensions (positive integer).
+* ``array_of_gsizes``: Number of elements of type *oldtype* in each dimension of global array (array of positive integers).
+* ``array_of_distribs``: Distribution of array in each dimension (array of state).
+* ``array_of_dargs``: Distribution argument in each dimension (array of positive integers).
+* ``array_of_psizes``: Size of process grid in each dimension (array of positive integers).
+* ``order``: Array storage order flag (state).
+* ``oldtype``: Old data type (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``newtype``: New data type (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Type_create_darray` can be used to generate the data types
+corresponding to the distribution of an ndims-dimensional array of
+*oldtype* elements onto an *ndims*-dimensional grid of logical
+processes. Unused dimensions of *array_of_psizes* should be set to 1.
+For a call to :ref:`MPI_Type_create_darray` to be correct, the equation
+
+::
+
+       ndims-1
+     pi              array_of_psizes[i] = size
+       i=0
+
+must be satisfied. The ordering of processes in the process grid is
+assumed to be row-major, as in the case of virtual Cartesian process
+topologies in the `MPI Standard <https://www.mpi-forum.org/docs/>`_.
+
+Each dimension of the array can be distributed in one of three ways:
+
+* ``MPI_DISTRIBUTE_BLOCK`` - Block distribution
+* ``MPI_DISTRIBUTE_CYCLIC`` - Cyclic distribution
+* ``MPI_DISTRIBUTE_NONE`` - Dimension not distributed.
+
+The constant MPI_DISTRIBUTE_DFLT_DARG specifies a default distribution
+argument. The distribution argument for a dimension that is not
+distributed is ignored. For any dimension *i* in which the distribution
+is MPI_DISTRIBUTE_BLOCK, it erroneous to specify *array_of_dargs[i]*
+*\** *array_of_psizes[i]* < *array_of_gsizes[i]*.
+
+For example, the HPF layout ARRAY(CYCLIC(15)) corresponds to
+MPI_DISTRIBUTE_CYCLIC with a distribution argument of 15, and the HPF
+layout ARRAY(BLOCK) corresponds to MPI_DISTRIBUTE_BLOCK with a
+distribution argument of MPI_DISTRIBUTE_DFLT_DARG.
+
+The *order* argument is used as in :ref:`MPI_TYPE_CREATE_SUBARRAY` to specify
+the storage order. Therefore, arrays described by this type constructor
+may be stored in Fortran (column-major) or C (row-major) order. Valid
+values for order are MPI_ORDER_FORTRAN and MPI_ORDER_C.
+
+This routine creates a new MPI data type with a typemap defined in terms
+of a function called "cyclic()" (see below).
+
+Without loss of generality, it suffices to define the typemap for the
+MPI_DISTRIBUTE_CYCLIC case where ``MPI_DISTRIBUTE_DFLT_DARG`` is not used.
+
+MPI_DISTRIBUTE_BLOCK and MPI_DISTRIBUTE_NONE can be reduced to the
+MPI_DISTRIBUTE_CYCLIC case for dimension *i* as follows.
+
+MPI_DISTRIBUTE_BLOCK with *array_of_dargs[i]* equal to
+MPI_DISTRIBUTE_DFLT_DARG is equivalent to MPI_DISTRIBUTE_CYCLIC with
+*array_of_dargs[i]* set to
+
+.. code-block:: c
+
+      (array_of_gsizes[i] + array_of_psizes[i] - 1)/array_of_psizes[i]
+
+If *array_of_dargs[i]* is not MPI_DISTRIBUTE_DFLT_DARG, then
+MPI_DISTRIBUTE_BLOCK and DISTRIBUTE_CYCLIC are equivalent.
+
+MPI_DISTRIBUTE_NONE is equivalent to MPI_DISTRIBUTE_CYCLIC with
+*array_of_dargs[i]* set to *array_of_gsizes[i]*.
+
+Finally, MPI_DISTRIBUTE_CYCLIC with *array_of_dargs[i]* equal to
+MPI_DISTRIBUTE_DFLT_DARG is equivalent to MPI_DISTRIBUTE_CYCLIC with
+*array_of_dargs[i]* set to 1.
+
+
+NOTES
+-----
+
+For both Fortran and C arrays, the ordering of processes in the process
+grid is assumed to be row-major. This is consistent with the ordering
+used in virtual Cartesian process topologies in MPI. To create such
+virtual process topologies, or to find the coordinates of a process in
+the process grid, etc., users may use the corresponding functions
+provided in MPI.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Type_create_f90_complex.3.rst b/docs/man-openmpi/man3/MPI_Type_create_f90_complex.3.rst
new file mode 100644
index 00000000000..e3ba181b65c
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_create_f90_complex.3.rst
@@ -0,0 +1,98 @@
+.. _mpi_type_create_f90_complex:
+
+
+MPI_Type_create_f90_complex
+===========================
+
+.. include_body
+
+::
+
+   MPI_Type_create_f90_complex - Returns a bounded MPI complex datatype
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_type_create_f90_complex.rst
+
+INPUT PARAMETERS
+----------------
+* ``p``: Precision, in decimal digits (integer).
+* ``r``: Decimal exponent range (integer).
+
+OUTPUT PARAMETERS
+-----------------
+* ``newtype``: New data type (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This function provides a way to declare KIND-parameterized COMPLEX MPI
+datatypes. The arguments are interpreted in a similar fashion to the F90
+function SELECTED_REAL_KIND. The parameters *p* and *r* must be scalar
+integers. The argument *p* represents the required level of numerical
+precision, in decimal digits. The *r* parameter indicates the range of
+exponents desired: the returned datatype will have at least one exponent
+between +\ *r* and -*r* (inclusive).
+
+Either *p* or *r*, but not both, may be omitted from calls to
+SELECTED_REAL_KIND. Similarly, either argument to
+:ref:`MPI_Type_create_f90_complex` may be set to MPI_UNDEFINED.
+
+
+NOTES
+-----
+
+It is erroneous to supply values for *p* and *r* not supported by the
+compiler.
+
+The Fortran function SELECTED_REAL_KIND maps a large number of (*p,r*)
+pairs to a much smaller number of KIND parameters supported by the
+compiler. KIND parameters are not specified by the language and are not
+portable. From the point of view of the language, variables of the same
+base type and KIND parameter are equivalent, even if their KIND
+parameters were generated by different (*p,r*) arguments to
+SELECTED_REAL_KIND. However, to help facilitate interoperability in a
+heterogeneous environment, equivalency is more strictly defined for
+datatypes returned by :ref:`MPI_Type_create_f90_complex`. Two MPI datatypes,
+each generated by this function, will match if and only if they have
+identical values for both *p* and *r*.
+
+The interaction between the datatypes returned by this function and the
+external32 data representation - used by :ref:`MPI_Pack_external`,
+:ref:`MPI_Unpack_external`, and many MPI_File functions - is subtle. The
+external32 representation of returned datatypes is as follows.
+
+.. code-block:: c
+
+   	if (p > 33) and/or (r > 4931):
+   		external32 size = n/a (undefined)
+   	else if (p > 15) and/or (r > 307):
+   		external32 size = 32
+   	else if (p > 6) and/or (r > 37):
+   		external32 size = 16
+   	else:
+   		external32 size = 8
+
+If the external32 representation of a datatype is undefined, so are the
+results of using that datatype in operations that require the external32
+format. Care should be taken not to use incompatible datatypes
+indirectly, e.g., as part of another datatype or through a duplicated
+datatype, in these functions.
+
+If a variable is declared specifying a nondefault KIND value that was
+not obtained with SELECTED_REAL_KIND (i.e., *p* and/or *r* are unknown),
+the only way to obtain a matching MPI datatype is to use the functions
+:ref:`MPI_Sizeof` and :ref:`MPI_Type_match_size`.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Pack_external`
+   * :ref:`MPI_Sizeof`
+   * :ref:`MPI_Type_match_size`
+   * :ref:`MPI_Unpack_external`
+   * SELECTED_REAL_KIND
diff --git a/docs/man-openmpi/man3/MPI_Type_create_f90_integer.3.rst b/docs/man-openmpi/man3/MPI_Type_create_f90_integer.3.rst
new file mode 100644
index 00000000000..f3f6f0a1a3a
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_create_f90_integer.3.rst
@@ -0,0 +1,92 @@
+.. _mpi_type_create_f90_integer:
+
+
+MPI_Type_create_f90_integer
+===========================
+
+.. include_body
+
+::
+
+   MPI_Type_create_f90_integer - Returns a bounded MPI integer datatype
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_type_create_f90_integer.rst
+
+INPUT PARAMETER
+---------------
+* ``r``: Precision, in decimal digits (integer).
+
+OUTPUT PARAMETERS
+-----------------
+* ``newtype``: New data type (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This function provides a way to declare KIND-parameterized INTEGER MPI
+datatypes. The argument is interpreted in a similar fashion to the F90
+function SELECTED_INT_KIND: *r* must be a scalar integer, and represents
+the desired level of numerical precision, in decimal digits.
+
+
+NOTES
+-----
+
+It is erroneous to supply a value for *r* not supported by the compiler.
+
+The Fortran function SELECTED_INT_KIND maps a large number of *r* values
+to a much smaller number of KIND parameters supported by the compiler.
+KIND parameters are not specified by the language and are not portable.
+From the point of view of the language, variables of the same base type
+and KIND parameter are equivalent, even if their KIND parameters were
+generated by different *r* arguments to SELECTED_INT_KIND. However, to
+help facilitate interoperability in a heterogeneous environment,
+equivalency is more strictly defined for datatypes returned by
+:ref:`MPI_Type_create_f90_integer`. Two MPI datatypes, each generated by this
+function, will match if and only if they have identical values for *r*.
+
+The interaction between the datatypes returned by this function and the
+external32 data representation - used by :ref:`MPI_Pack_external`,
+:ref:`MPI_Unpack_external` and many MPI_File functions - is subtle. The
+external32 representation of returned datatypes is as follows.
+
+.. code-block:: c
+
+   	if (r > 38):
+   		external32 size = n/a (undefined)
+   	else if (r > 18):
+   		external32 size = 16
+   	else if (r > 9):
+   		external32 size = 8
+   	else if (r > 4):
+   		external32 size = 4
+   	else if (r > 2):
+   		external32 size = 2
+   	else:
+   		external32 size = 1
+
+If the external32 representation of a datatype is undefined, so are the
+results of using that datatype in operations that require the external32
+format. Care should be taken not to use incompatible datatypes
+indirectly, e.g., as part of another datatype or through a duplicated
+datatype, in these functions.
+
+If a variable is declared specifying a nondefault KIND value that was
+not obtained with SELECTED_INT_KIND (i.e., *r* is unknown), the only way
+to obtain a matching MPI datatype is to use the functions :ref:`MPI_Sizeof` and
+:ref:`MPI_Type_match_size`.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Pack_external`
+   * :ref:`MPI_Sizeof`
+   * :ref:`MPI_Type_match_size`
+   * :ref:`MPI_Unpack_external`
+   * SELECTED_INT_KIND
diff --git a/docs/man-openmpi/man3/MPI_Type_create_f90_real.3.rst b/docs/man-openmpi/man3/MPI_Type_create_f90_real.3.rst
new file mode 100644
index 00000000000..b8511371e42
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_create_f90_real.3.rst
@@ -0,0 +1,98 @@
+.. _mpi_type_create_f90_real:
+
+
+MPI_Type_create_f90_real
+========================
+
+.. include_body
+
+::
+
+   MPI_Type_create_f90_real - Returns a bounded MPI real datatype
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_type_create_f90_real.rst
+
+INPUT PARAMETERS
+----------------
+* ``p``: Precision, in decimal digits (integer).
+* ``r``: Decimal exponent range (integer).
+
+OUTPUT PARAMETERS
+-----------------
+* ``newtype``: New data type (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This function provides a way to declare KIND-parameterized REAL MPI
+datatypes. The arguments are interpreted in a similar fashion to the F90
+function SELECTED_REAL_KIND. The parameters *p* and *r* must be scalar
+integers. The argument *p* represents the required level of numerical
+precision, in decimal digits. The *r* parameter indicates the range of
+exponents desired: the returned datatype will have at least one exponent
+between +\ *r* and -*r* (inclusive).
+
+Either *p* or *r*, but not both, may be omitted from calls to
+SELECTED_REAL_KIND. Similarly, either argument to
+:ref:`MPI_Type_create_f90_real` may be set to MPI_UNDEFINED.
+
+
+NOTES
+-----
+
+It is erroneous to supply values for *p* and *r* not supported by the
+compiler.
+
+The Fortran function SELECTED_REAL_KIND maps a large number of (*p,r*)
+pairs to a much smaller number of KIND parameters supported by the
+compiler. KIND parameters are not specified by the language and are not
+portable. From the point of view of the language, variables of the same
+base type and KIND parameter are equivalent, even if their KIND
+parameters were generated by different (*p,r*) arguments to
+SELECTED_REAL_KIND. However, to help facilitate interoperability in a
+heterogeneous environment, equivalency is more strictly defined for
+datatypes returned by :ref:`MPI_Type_create_f90_real`. Two MPI datatypes, each
+generated by this function, will match if and only if they have
+identical values for both *p* and *r*.
+
+The interaction between the datatypes returned by this function and the
+external32 data representation - used by :ref:`MPI_Pack_external`,
+:ref:`MPI_Unpack_external` and many MPI_File functions - is subtle. The
+external32 representation of returned datatypes is as follows.
+
+.. code-block:: c
+
+   	if (p > 33) and/or (r > 4931):
+   		external32 size = n/a (undefined)
+   	else if (p > 15) and/or (r > 307):
+   		external32 size = 16
+   	else if (p > 6) and/or (r > 37):
+   		external32 size = 8
+   	else:
+   		external32 size = 4
+
+If the external32 representation of a datatype is undefined, so are the
+results of using that datatype in operations that require the external32
+format. Care should be taken not to use incompatible datatypes
+indirectly, e.g., as part of another datatype or through a duplicated
+datatype, in these functions.
+
+If a variable is declared specifying a nondefault KIND value that was
+not obtained with SELECTED_REAL_KIND (i.e., *p* and/or *r* are unknown),
+the only way to obtain a matching MPI datatype is to use the functions
+:ref:`MPI_Sizeof` and :ref:`MPI_Type_match_size`.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Pack_external`
+   * :ref:`MPI_Sizeof`
+   * :ref:`MPI_Type_match_size`
+   * :ref:`MPI_Unpack_external`
+   * SELECTED_REAL_KIND
diff --git a/docs/man-openmpi/man3/MPI_Type_create_hindexed.3.rst b/docs/man-openmpi/man3/MPI_Type_create_hindexed.3.rst
new file mode 100644
index 00000000000..758e2fa3ab9
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_create_hindexed.3.rst
@@ -0,0 +1,62 @@
+.. _mpi_type_create_hindexed:
+
+MPI_Type_create_hindexed
+========================
+
+.. include_body
+
+:ref:`MPI_Type_create_hindexed` |mdash| Creates an indexed datatype with offsets in bytes.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_type_create_hindexed.rst
+
+INPUT PARAMETERS
+----------------
+* ``count``: Number of blocks |mdash| also number of entries in array_of_displacements and array_of_blocklengths (integer).
+* ``array_of_blocklengths``: Number of elements in each block (array of nonnegative integers).
+* ``array_of_displacements``: Byte displacement of each block (C: array of *MPI_Aint*, Fortran: array of integer).
+* ``oldtype``: Old datatype (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``newtype``: New datatype (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The function is identical to :ref:`MPI_Type_indexed`, except that block
+displacements in ``array_of_displacements`` are specified in bytes, rather
+than in multiples of the ``oldtype`` extent.
+
+Assume that oldtype has type map
+
+::
+
+       {(type(0), disp(0)), ..., (type(n-1), disp(n-1))},
+
+with extent ex. Let B be the ``array_of_blocklength`` argument and D be the
+``array_of_displacements`` argument. The newly created datatype has
+
+::
+
+   n x S^count-1
+       (i=0)        B[i]  entries:
+
+     {(type(0), disp(0) + D[0]),...,(type(n-1), disp(n-1) + D[0]),...,
+     (type(0), disp(0) + (D[0] + B[0]-1)* ex),...,
+     type(n-1), disp(n-1) + (D[0]+ B[0]-1)* ex),...,
+     (type(0), disp(0) + D[count-1]),...,(type(n-1), disp(n-1) + D[count-1]),...,
+     (type(0), disp(0) +  D[count-1] + (B[count-1] -1)* ex),...,
+     (type(n-1), disp(n-1) + D[count-1] + (B[count-1] -1)* ex)}
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Type_create_struct`
+   * :ref:`MPI_Type_create_hvector`
+   * :ref:`MPI_Type_indexed`
diff --git a/docs/man-openmpi/man3/MPI_Type_create_hindexed_block.3.rst b/docs/man-openmpi/man3/MPI_Type_create_hindexed_block.3.rst
new file mode 100644
index 00000000000..945c2e35a72
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_create_hindexed_block.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_type_create_hindexed_block:
+
+MPI_Type_create_hindexed_block
+==============================
+    .. include_body
+
+.. include:: ../man3/MPI_Type_create_indexed_block.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Type_create_hvector.3.rst b/docs/man-openmpi/man3/MPI_Type_create_hvector.3.rst
new file mode 100644
index 00000000000..a94e151f490
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_create_hvector.3.rst
@@ -0,0 +1,69 @@
+.. _mpi_type_create_hvector:
+
+
+MPI_Type_create_hvector
+=======================
+
+.. include_body
+
+:ref:`MPI_Type_create_hvector` |mdash| Creates a vector (strided) data type with offset in bytes.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_type_create_hvector.rst
+
+INPUT PARAMETERS
+----------------
+* ``count``: Number of blocks (nonnegative integer).
+* ``blocklength``: Number of elements in each block (nonnegative integer).
+* ``stride``: Number of bytes between start of each block (integer).
+* ``oldtype``: Old data type (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``newtype``: New data type (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Type_create_hvector` creates a vector (strided) data type with offset
+in bytes. This routine replaces :ref:`MPI_Type_hvector`, which is deprecated.
+
+The function :ref:`MPI_Type_create_hvector` is identical to :ref:`MPI_Type_vector`, except
+that stride is given in bytes, rather than in elements. The use for both
+types of vector constructors is illustrated in the examples in the
+Datatype chapter of the `MPI Standard
+<https://www.mpi-forum.org/docs/>`_.
+
+Assume that oldtype has type map
+
+::
+
+       {(type(0), disp(0)), ..., (type(n-1), disp(n-1))}
+
+with extent ex. Let bl be the blocklength. The newly created datatype
+has a type map with ``count * bl * n`` entries:
+
+::
+
+     {(type(0), disp(0)), ..., (type(n-1), disp(n-1)),
+     (type(0), disp(0) + ex), ..., (type(n-1), disp(n-1) + ex),
+     ..., (type(0), disp(0) + (bl -1) * ex),...,(type(n-1),
+     disp(n-1) + (bl -1) * ex), (type(0), disp(0) + stride),
+     ...,(type(n-1), disp(n-1) + stride), ..., (type(0),
+     disp(0) + stride + (bl - 1) * ex), ..., (type(n-1),
+     disp(n-1) + stride + (bl -1) * ex), ..., (type(0),
+     disp(0) + stride * (count -1)), ...,(type(n-1),
+     disp(n-1) + stride * (count -1)), ..., (type(0),
+     disp(0) + stride * (count -1) + (bl -1) * ex), ...,
+     (type(n-1), disp(n-1) + stride * (count -1) + (bl -1) * ex)}
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Type_create_hindexed`
+   * :ref:`MPI_Type_vector`
diff --git a/docs/man-openmpi/man3/MPI_Type_create_indexed_block.3.rst b/docs/man-openmpi/man3/MPI_Type_create_indexed_block.3.rst
new file mode 100644
index 00000000000..98fec51683b
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_create_indexed_block.3.rst
@@ -0,0 +1,47 @@
+.. _mpi_type_create_indexed_block:
+
+
+MPI_Type_create_indexed_block
+=============================
+
+.. include_body
+
+:ref:`MPI_Type_create_indexed_block`, :ref:`MPI_Type_create_hindexed_block` -
+Creates an indexed data type with the same block length for all blocks.
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Type_create_indexed_block, MPI_Type_create_hindexed_block
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_type_create_indexed_block.rst
+
+INPUT PARAMETERS
+----------------
+* ``count``: Length of array of displacements (integer).
+* ``blocklength``: Size of block (integer).
+* ``array_of_displacements``: Array of displacements (array of integers). In units of the extent of *oldtype* for MPI_Type_create_indexed_block and bytes for MPI_Type_create_hindexed_block.
+* ``oldtype``: Old data type (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``newtype``: New data type (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Type_create_indexed_block` and :ref:`MPI_Type_create_hindexed_block` create
+an indexed data type with the same block length for all blocks. The only
+difference between the two functions is :ref:`MPI_Type_create_indexed_block`
+takes an array of displacements in units of the extent of *oldtype*
+while :ref:`MPI_Type_create_hindexed_block` takes displacements in bytes.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Type_indexed`
diff --git a/docs/man-openmpi/man3/MPI_Type_create_keyval.3.rst b/docs/man-openmpi/man3/MPI_Type_create_keyval.3.rst
new file mode 100644
index 00000000000..50641535d33
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_create_keyval.3.rst
@@ -0,0 +1,76 @@
+.. _mpi_type_create_keyval:
+
+
+MPI_Type_create_keyval
+======================
+
+.. include_body
+
+:ref:`MPI_Type_create_keyval` |mdash| Generates a new attribute key for caching
+on data types.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_type_create_keyval.rst
+
+INPUT PARAMETERS
+----------------
+* ``type_copy_attr_fn``: Copy callback function for *type_keyval* (function).
+* ``type_delete_attr_fn``: Delete callback function for *type_keyval* (function).
+* ``extra_state``: Extra state for callback functions.
+
+OUTPUT PARAMETERS
+-----------------
+* ``type_keyval``: Key value for future access (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Type_create_keyval` generates a new attribute key for caching on data
+types. This routine partially replaces :ref:`MPI_Keyval_create`.
+
+The argument *type_copy_attr_fn* may be specified as
+MPI_TYPE_NULL_COPY_FN or MPI_TYPE_DUP_FN from C or Fortran.
+MPI_TYPE_NULL_COPY_FN is a function that does nothing other than
+returning *flag* = 0 and MPI_SUCCESS. MPI_TYPE_DUP_FN is a simple-minded
+copy function that sets *flag* = 1, returns the value of
+*attribute_val_in* in *attribute_val_out*, and returns MPI_SUCCESS.
+
+The argument *type_delete_attr_fn* may be specified as
+MPI_TYPE_NULL_DELETE_FN from C or Fortran. MPI_TYPE_NULL_DELETE_FN is a
+function that does nothing beyond returning MPI_SUCCESS. The C callback
+functions are:
+
+.. code-block:: c
+
+   typedef int MPI_Type_copy_attr_function(MPI_Datatype oldtype,
+               int type_keyval, void *extra_state, void *attribute_val_in,
+               void *attribute_val_out, int *flag);
+
+   typedef int MPI_Type_delete_attr_function(MPI_Datatype type, int type_keyval,
+                void *attribute_val, void *extra_state);
+
+The Fortran callback functions are:
+
+.. code-block:: fortran
+
+   SUBROUTINE TYPE_COPY_ATTR_FN(OLDTYPE, TYPE_KEYVAL, EXTRA_STATE,
+                ATTRIBUTE_VAL_IN, ATTRIBUTE_VAL_OUT, FLAG, IERROR)
+       INTEGER OLDTYPE, TYPE KEYVAL, IERROR
+       INTEGER(KIND=MPI_ADDRESS_KIND) EXTRA_STATE,
+           ATTRIBUTE_VAL_IN, ATTRIBUTE_VAL_OUT
+       LOGICAL FLAG
+
+   SUBROUTINE TYPE_DELETE_ATTR_FN(TYPE, TYPE_KEYVAL, ATTRIBUTE_VAL, EXTRA_STATE,
+                IERROR)
+       INTEGER TYPE, TYPE_KEYVAL, IERROR
+       INTEGER(KIND=MPI_ADDRESS_KIND) ATTRIBUTE VAL, EXTRA_STATE
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Type_free_keyval`
diff --git a/docs/man-openmpi/man3/MPI_Type_create_resized.3.rst b/docs/man-openmpi/man3/MPI_Type_create_resized.3.rst
new file mode 100644
index 00000000000..79d0912568d
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_create_resized.3.rst
@@ -0,0 +1,52 @@
+.. _mpi_type_create_resized:
+
+
+MPI_Type_create_resized
+=======================
+
+.. include_body
+
+:ref:`MPI_Type_create_resized` |mdash| Returns a new data type with new extent
+and upper and lower bounds.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_type_create_resized.rst
+
+INPUT PARAMETERS
+----------------
+* ``oldtype``: Input data type (handle).
+* ``lb``: New lower bound of data type (integer).
+* ``extent``: New extent of data type (integer).
+
+OUTPUT PARAMETERS
+-----------------
+* ``newtype``: Output data type (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Type_create_resized` returns in *newtype* a handle to a new data type
+that is identical to *oldtype*, except that the lower bound of this new
+data type is set to be *lb*, and its upper bound is set to be *lb* +
+*extent*. Any previous *lb* and *ub* markers are erased, and a new pair
+of lower bound and upper bound markers are put in the positions
+indicated by the *lb* and *extent* arguments. This affects the behavior
+of the data type when used in communication operations, with *count* >
+1, and when used in the construction of new derived data types.
+
+
+NOTE
+----
+
+Use of :ref:`MPI_Type_create_resized` is strongly recommended over the legacy
+MPI-1 functions :ref:`MPI_Type_extent` and :ref:`MPI_Type_lb`.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Type_get_extent`
diff --git a/docs/man-openmpi/man3/MPI_Type_create_struct.3.rst b/docs/man-openmpi/man3/MPI_Type_create_struct.3.rst
new file mode 100644
index 00000000000..cf8902d5558
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_create_struct.3.rst
@@ -0,0 +1,151 @@
+.. _mpi_type_create_struct:
+
+
+MPI_Type_create_struct
+======================
+
+.. include_body
+
+:ref:`MPI_Type_create_struct` |mdash| Creates a structured data type.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_type_create_struct.rst
+
+INPUT PARAMETERS
+----------------
+* ``count``: Number of blocks (integer) |mdash| also number of entries in arrays *array_of_types*, *array_of_displacements*, and *array_of_blocklengths*.
+* ``array_of_blocklengths``: Number of elements in each block (array of integers).
+* ``array_of_displacements``: Byte displacement of each block (array of integers).
+* ``array_of_types``: Type of elements in each block (array of handles to data-type objects).
+
+OUTPUT PARAMETERS
+-----------------
+* ``newtype``: New data type (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Type_create_struct` creates a structured data type. This routine
+replaces :ref:`MPI_Type_struct`, which is now deprecated.
+
+
+:ref:`MPI_Type_create_struct` is the most general type constructor. It further
+generalizes :ref:`MPI_Type_create_hindexed` in that it allows each block to consist of
+replications of different datatypes.
+
+**Example 1:**
+Let ``type1`` have type map
+
+::
+
+
+       {(double, 0), (char, 8)}
+
+with extent 16. Let ``B = (2, 1, 3)``, ``D = (0, 16, 26)``, and ``T = (MPI_FLOAT, type1, MPI_CHAR)``.
+Then a call to ``MPI_Type_create_struct(3, B, D, T, newtype)``
+returns a datatype with type map
+
+::
+
+
+       {
+        (float, 0), (float,4),             // 2 float
+        (double, 16), (char, 24),          // 1 type1
+        (char, 26), (char, 27), (char, 28) // 3 char
+       }
+
+That is, two copies of ``MPI_FLOAT`` starting at 0, followed by one copy of
+``type1`` starting at 16, followed by three copies of ``MPI_CHAR``, starting at
+26.
+
+
+**Example 2:**
+
+An example of a struct with only some components part of the type
+
+.. code-block:: c
+
+   struct MyStruct {
+       double x[2], y;
+       char a;
+       int n;
+   };
+
+   // create a new type where we only send x, y and n
+   int B[] = {
+       2, // 2 double's
+       1, // 1 double
+       1, // 1 int
+       1  // alignment padding
+   };
+   MPI_Aint D[] = {
+       offsetof(struct MyStruct, x),
+       offsetof(struct MyStruct, y),
+       offsetof(struct MyStruct, n),
+       sizeof(struct MyStruct)
+   };
+   MPI_Datatype T[] = {
+       MPI_DOUBLE,
+       MPI_DOUBLE,
+       MPI_INT,
+       MPI_UB
+   };
+
+   MPI_Datatype mpi_dt_mystruct;
+   MPI_Type_create_struct(4, B, D, T, &mpi_dt_mystruct);
+   MPI_Type_commit(&mpi_dt_mystruct);
+
+   // We can now send a struct (omitting a)
+
+   struct MyStruct values[3];
+
+   if ( rank == 0 ) {
+       // ... initialize values
+       MPI_Send(values, 3, mpi_dt_mystruct, 1, 0, MPI_COMM_WORLD);
+   } else if ( rank == 1 ) {
+       MPI_Recv(values, 3, mpi_dt_mystruct, 0, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+   }
+
+
+For more information, see section 5.1.2 of the MPI-4.0 Standard.
+
+
+NOTES
+-----
+
+If an upper bound is set explicitly by using the MPI datatype ``MPI_UB``, the
+corresponding index must be positive.
+
+The MPI-1 Standard originally made vague statements about padding and
+alignment; this was intended to allow the simple definition of
+structures that could be sent with a count greater than one. For
+example,
+
+.. code-block:: c
+
+       struct {int a; char b;} foo;
+
+may have
+
+.. code-block:: c
+
+       sizeof(foo) = sizeof(int) + sizeof(char);
+
+defining the extent of a datatype as including an epsilon, which would
+have allowed an implementation to make the extent an MPI datatype for
+this structure equal to ``2*sizeof(int)``. However, since different systems
+might define different paddings, a clarification to the standard made
+epsilon zero. Thus, if you define a structure datatype and wish to send
+or receive multiple items, you should explicitly include an ``MPI_UB`` entry
+as the last member of the structure. See the above example.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Type_struct`
+   * :ref:`MPI_Type_create_hindexed`
diff --git a/docs/man-openmpi/man3/MPI_Type_create_subarray.3.rst b/docs/man-openmpi/man3/MPI_Type_create_subarray.3.rst
new file mode 100644
index 00000000000..421c946944c
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_create_subarray.3.rst
@@ -0,0 +1,112 @@
+.. _mpi_type_create_subarray:
+
+
+MPI_Type_create_subarray
+========================
+
+.. include_body
+
+:ref:`MPI_Type_create_subarray` |mdash| Creates a data type describing an
+*n*-dimensional subarray of an *n*-dimensional array.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_type_create_subarray.rst
+
+INPUT PARAMETERS
+----------------
+* ``ndims``: Number of array dimensions (positive integer).
+* ``array_of_sizes``: Number of elements of type *oldtype* in each dimension of the full array (array of positive integers).
+* ``array_of_subsizes``: Number of elements of type *oldtype* in each dimension of the subarray (array of positive integers).
+* ``array_of_starts``: Starting coordinates of the subarray in each dimension (array of nonnegative integers).
+* ``order``: Array storage order flag (state).
+* ``oldtype``: Array element data type (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``newtype``: New data type (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The subarray type constructor creates an MPI data type describing an
+*n*-dimensional subarray of an *n*-dimensional array. The subarray may
+be situated anywhere within the full array, and may be of any nonzero
+size up to the size of the larger array as long as it is confined within
+this array. This type constructor facilitates creating file types to
+access arrays distributed in blocks among processes to a single file
+that contains the global array.
+
+This type constructor can handle arrays with an arbitrary number of
+dimensions and works for both C- and Fortran-ordered matrices (that is,
+row-major or column-major). Note that a C program may use Fortran order
+and a Fortran program may use C order.
+
+The *ndims* parameter specifies the number of dimensions in the full
+data array and gives the number of elements in *array_of_sizes*,
+*array_of_subsizes*, and *array_of_starts*.
+
+The number of elements of type *oldtype* in each dimension of the
+*n*-dimensional array and the requested subarray are specified by
+*array_of_sizes* and *array_of_subsizes*, respectively. For any
+dimension *i*, it is erroneous to specify *array_of_subsizes[i]* < 1 or
+*array_of_subsizes[i]* > *array of sizes[i]*.
+
+The *array_of_starts* contains the starting coordinates of each
+dimension of the subarray. Arrays are assumed to be indexed starting
+from zero. For any dimension *i*, it is erroneous to specify
+
+.. code-block::
+
+   array_of_starts[i] < 0
+
+or
+
+.. code-block::
+
+   array_of_starts[i] > (array_of_sizes[i] - array_of_subsizes[i]).
+
+The *order* argument specifies the storage order for the subarray as
+well as the full array. It must be set to one of the following:
+
+* ``MPI_ORDER_C``: The ordering used by C arrays, (that is, row-major order)
+
+* ``MPI_ORDER_FORTRAN``: The ordering used by Fortran arrays, (that is, column-major order)
+
+A *ndims*-dimensional subarray (*newtype*) with no extra padding can be
+defined by the function Subarray() as follows:
+
+.. code-block::
+
+      newtype = Subarray(ndims, {size_0, size_1,..., size_ndims-1},
+                {subsize_0, subsize_1, ..., subsize_ndims-1},
+                {start_0, start_1, ..., start_ndims-1}, oldtype)
+
+Let the typemap of *oldtype* have the form:
+
+.. code-block::
+
+      {(type_0, disp_0), (type_1, disp_1), ..., (type_n-1, disp_n-1)}
+
+where type\ *i* is a predefined MPI data type, and let *ex* be the
+extent of *oldtype*.
+
+The ``Subarray()`` function is defined recursively in three equations on
+page 72 of the MPI-2 standard.
+
+For an example use of :ref:`MPI_Type_create_subarray` in the context
+of I/O, see Section 9.9.2 of the MPI-2 standard.
+
+
+NOTES
+-----
+
+In a Fortran program with arrays indexed starting from 1, if the
+starting coordinate of a particular dimension of the subarray is *n*,
+then the entry in array of starts for that dimension is *n*-1.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Type_delete_attr.3.rst b/docs/man-openmpi/man3/MPI_Type_delete_attr.3.rst
new file mode 100644
index 00000000000..3e781aabf63
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_delete_attr.3.rst
@@ -0,0 +1,48 @@
+.. _mpi_type_delete_attr:
+
+
+MPI_Type_delete_attr
+====================
+
+.. include_body
+
+:ref:`MPI_Type_delete_attr` |mdash| Deletes a datatype-caching attribute value
+associated with a key.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_type_delete_attr.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``type``: Data type from which the attribute is deleted (handle).n
+
+INPUT PARAMETER
+---------------
+* ``type_keyval``: Key value (integer).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Type_delete_attr` deletes a datatype-caching attribute value
+associated with a key. This routines partially replaces :ref:`MPI_Attr_delete`,
+which is now deprecated.
+
+
+NOTES
+-----
+
+Note that it is not defined by the MPI standard what happens if the
+delete_fn callback invokes other MPI functions. In Open MPI, it is not
+valid for delete_fn callbacks (or any of their children) to add or
+delete attributes on the same object on which the delete_fn callback is
+being invoked.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Type_dup.3.rst b/docs/man-openmpi/man3/MPI_Type_dup.3.rst
new file mode 100644
index 00000000000..36631903056
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_dup.3.rst
@@ -0,0 +1,54 @@
+.. _mpi_type_dup:
+
+
+MPI_Type_dup
+============
+
+.. include_body
+
+:ref:`MPI_Type_dup` |mdash| Duplicates a data type with associated key values.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_type_dup.rst
+
+INPUT PARAMETER
+---------------
+* ``type``: Data type (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``newtype``: Copy of *type* (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Type_dup` is a type constructor that duplicates the existing type
+with associated key values. For each key value, the respective copy
+callback function determines the attribute value associated with this
+key in the new communicator. One particular action that a copy callback
+may take is to delete the attribute from the new data type. Returns in
+*newtype* a new data type with exactly the same properties as *type*, as
+well as any copied cached information. The new data type has identical
+upper bound and lower bound and yields the same net result when fully
+decoded with the functions described in Section 8.6 of the MPI-2
+standard. *newtype* has the same committed state as the old *type*.
+
+
+NOTES
+-----
+
+Note that it is not defined by the MPI standard what happens if the
+attribute copy callback invokes other MPI functions. In Open MPI, it is
+not valid for attribute copy callbacks (or any of their children) to add
+or delete attributes on the same object on which the attribute copy
+callback is being invoked.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Type_create_keyval`
diff --git a/docs/man-openmpi/man3/MPI_Type_extent.3.rst b/docs/man-openmpi/man3/MPI_Type_extent.3.rst
new file mode 100644
index 00000000000..e49da0cb574
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_extent.3.rst
@@ -0,0 +1,100 @@
+.. _mpi_type_extent:
+
+
+MPI_Type_extent
+===============
+
+.. include_body
+
+:ref:`MPI_Type_extent` |mdash| Returns the extent of a data type |mdash| |deprecated_favor| :ref:`MPI_Type_get_extent`.
+
+
+SYNTAX
+------
+
+.. NOTE: The bindings for this man page are not automatically
+   generated from the official MPI Forum JSON/Python library because
+   this function is deprecated.  Hence, this function is not included
+   in the MPI Forum JSON data, and we therefore have to hard-code the
+   bindings here ourselves.
+
+C Syntax
+^^^^^^^^
+
+.. code-block:: c
+
+   #include <mpi.h>
+
+   int MPI_Type_extent(MPI_Datatype datatype, MPI_Aint *extent)
+
+
+Fortran Syntax
+^^^^^^^^^^^^^^
+
+.. code-block:: fortran
+
+   USE MPI
+   ! or the older form: INCLUDE 'mpif.h'
+
+   MPI_TYPE_EXTENT(DATATYPE, EXTENT, IERROR)
+   	INTEGER	DATATYPE, EXTENT, IERROR
+
+
+INPUT PARAMETER
+---------------
+* ``datatype``: Datatype (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``extent``: Datatype extent (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Note that use of this routine is *deprecated* as of MPI-2. Please use
+:ref:`MPI_Type_get_extent` instead.
+
+:ref:`MPI_Type_extent` returns the extent of a data type, the difference
+between the upper and lower bounds of the data type.
+
+In general, if
+
+::
+
+       Typemap = {(type(0), disp(0)), ..., (type(n-1), disp(n-1))}
+
+then the lower bound of Typemap is defined to be
+
+::
+
+                 ( min(j) disp(j)                         if no entry has
+     lb(Typemap)=(                                        basic type lb
+                 (min(j) {disp(j) such that type(j) = lb} otherwise
+
+Similarly, the upper bound of Typemap is defined to be
+
+::
+
+                 (max(j) disp(j) + sizeof(type(j)) + e    if no entry has
+     ub(Typemap)=(                                        basic type ub
+                 (max(j) {disp(j) such that type(j) = ub} otherwise
+
+Then
+
+::
+
+       extent(Typemap) = ub(Typemap) - lb(Typemap)
+
+If type(i) requires alignment to a byte address that is a multiple of
+k(i), then e is the least nonnegative increment needed to round
+extent(Typemap) to the next multiple of max(i) k(i).
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Type_get_extent`
diff --git a/docs/man-openmpi/man3/MPI_Type_f2c.3.rst b/docs/man-openmpi/man3/MPI_Type_f2c.3.rst
new file mode 100644
index 00000000000..ee8c11da09c
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_f2c.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_type_f2c:
+
+MPI_Type_f2c
+============
+
+.. include_body
+
+.. include:: ../man3/MPI_Comm_f2c.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_Type_free.3.rst b/docs/man-openmpi/man3/MPI_Type_free.3.rst
new file mode 100644
index 00000000000..8aeee45aa1d
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_free.3.rst
@@ -0,0 +1,38 @@
+.. _mpi_type_free:
+
+
+MPI_Type_free
+=============
+
+.. include_body
+
+:ref:`MPI_Type_free` |mdash| Frees a data type.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_type_free.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``datatype``: Datatype that is freed (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Marks the datatype object associated with datatype for de-allocation and
+sets datatype to MPI_DATATYPE_NULL. Any communication that is currently
+using this datatype will complete normally. Derived datatypes that were
+defined from the freed datatype are not affected.
+
+Freeing a datatype does not affect any other datatype that was built
+from the freed datatype. The system behaves as if input datatype
+arguments to derived datatype constructors are passed by value.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Type_free_keyval.3.rst b/docs/man-openmpi/man3/MPI_Type_free_keyval.3.rst
new file mode 100644
index 00000000000..049f7386ca4
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_free_keyval.3.rst
@@ -0,0 +1,32 @@
+.. _mpi_type_free_keyval:
+
+
+MPI_Type_free_keyval
+====================
+
+.. include_body
+
+:ref:`MPI_Type_free_keyval` |mdash| Frees a previously created type key value.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_type_free_keyval.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``type_keyval``: Key value to free (integer).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Type_create_keyval`
diff --git a/docs/man-openmpi/man3/MPI_Type_get_attr.3.rst b/docs/man-openmpi/man3/MPI_Type_get_attr.3.rst
new file mode 100644
index 00000000000..c3c5cab7cf1
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_get_attr.3.rst
@@ -0,0 +1,39 @@
+.. _mpi_type_get_attr:
+
+
+MPI_Type_get_attr
+=================
+
+.. include_body
+
+:ref:`MPI_Type_get_attr` |mdash| Returns the attribute associated with a data
+type.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_type_get_attr.rst
+
+INPUT PARAMETERS
+----------------
+* ``type``: Data type to which the attribute is attached (handle).
+* ``type_keyval``: Key value (integer).
+
+OUTPUT PARAMETERS
+-----------------
+* ``attribute_val``: Attribute value, unless *flag* = false
+* ``flag``: "false" if no attribute is associated with the key (logical).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+For the given data type, :ref:`MPI_Type_get_attr` returns an attribute value
+that corresponds to the specified key value.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Type_set_attr`
diff --git a/docs/man-openmpi/man3/MPI_Type_get_contents.3.rst b/docs/man-openmpi/man3/MPI_Type_get_contents.3.rst
new file mode 100644
index 00000000000..85fcb74f3d3
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_get_contents.3.rst
@@ -0,0 +1,85 @@
+.. _mpi_type_get_contents:
+
+
+MPI_Type_get_contents
+=====================
+
+.. include_body
+
+:ref:`MPI_Type_get_contents` |mdash| Returns information about arguments used in
+creation of a data type.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_type_get_contents.rst
+
+INPUT PARAMETERS
+----------------
+* ``datatype``: Data type to access (handle).
+* ``max_integers``: Number of elements in *array_of_integers (nonnegative integer).*
+* ``max_addresses``: Number of elements in *array_of_addresses (nonnegative integer).*
+* ``max_datatypes``: Number of elements in *array_of_datatypes (nonnegative integer).*
+
+OUTPUT PARAMETERS
+-----------------
+* ``array_of_integers``: Contains integer arguments used in constructing *datatype (array of integers).*
+* ``array_of_addresses``: Contains address arguments used in constructing *datatype (array of integers).*
+* ``array_of_datatypes``: Contains data-type arguments used in constructing *datatype (array of integers).*
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+For the given data type, :ref:`MPI_Type_get_envelope` returns information on
+the number and type of input arguments used in the call that created the
+data type. The number-of-arguments values returned can be used to
+provide sufficiently large arrays in the decoding routine
+:ref:`MPI_Type_get_contents`. This call and the meaning of the returned values
+is described below. The combiner reflects the MPI data type constructor
+call that was used in creating *datatype.*
+
+The parameter *datatype must be a predefined unnamed or a derived data
+type. The call is erroneous if datatype is a predefined named data
+type.*
+
+The values given for *max_integers, max_addresses, and max_datatypes
+must be at least as large as the value returned in num_integers,
+num_addresses, and num_datatypes, respectively, in the call
+:ref:`MPI_Type_get_envelope` for the same datatype argument.*
+
+The data types returned in *array_of_datatypes are handles to data-type
+objects that are equivalent to the data types used in the original
+construction call. If these were derived data types, then the returned
+data types are new data-type objects, and the user is responsible for
+freeing these datatypes with :ref:`MPI_Type_free`. If these were predefined
+data types, then the returned data type is equal to that (constant)
+predefined data type and cannot be freed.*
+
+The committed state of returned derived data types is undefined, that
+is, the data types may or may not be committed. Furthermore, the content
+of attributes of returned data types is undefined.
+
+Note that :ref:`MPI_Type_get_contents` can be invoked with a data-type argument
+that was constructed using :ref:`MPI_Type_create_f90_real`,
+:ref:`MPI_Type_create_f90_integer`, or :ref:`MPI_Type_create_f90_complex` (an unnamed
+predefined data type). In such a case, an empty *array_of_datatypes* is
+returned.
+
+In the legacy MPI-1 datatype constructor calls, the address arguments in
+Fortran are of type ``INTEGER``. In subsequent versions of the `MPI
+Standard <https://www.mpi-forum.org/docs/>`_, the address
+arguments are of type ``INTEGER(KIND=MPI_ADDRESS_KIND)``. The call
+:ref:`MPI_Type_get_contents` returns all addresses in an argument of type
+``INTEGER(KIND=MPI_ADDRESS_KIND)``. This is true even if the old MPI-1 calls
+were used. Thus, the location of values returned can be thought of as
+being returned by the C bindings. It can also be determined by examining
+the current MPI calls for datatype constructors for the deprecated MPI-1
+calls that involve addresses.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Type_get_envelope`
diff --git a/docs/man-openmpi/man3/MPI_Type_get_envelope.3.rst b/docs/man-openmpi/man3/MPI_Type_get_envelope.3.rst
new file mode 100644
index 00000000000..551364b7c8f
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_get_envelope.3.rst
@@ -0,0 +1,81 @@
+.. _mpi_type_get_envelope:
+
+
+MPI_Type_get_envelope
+=====================
+
+.. include_body
+
+:ref:`MPI_Type_get_envelope` |mdash| Returns information about input arguments
+associated with a data type.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_type_get_envelope.rst
+
+INPUT PARAMETER
+---------------
+* ``datatype``: Data type to access (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``num_integers``: Number of input integers used in the call constructing *combiner* (nonnegative integer).
+* ``num_addresses``: Number of input addresses used in the call constructing *combiner* (nonnegative integer).
+* ``num_datatypes``: Number of input data types used in the call constructing *combiner* (nonnegative integer).
+* ``combiner``: Combiner (state).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+For the given data type, :ref:`MPI_Type_get_envelope` returns information on
+the number and type of input arguments used in the call that created the
+data type. The number-of-arguments values returned can be used to
+provide sufficiently large arrays in the decoding routine
+:ref:`MPI_Type_get_contents`. This call and the meaning of the returned values
+is described below. The combiner reflects the MPI data type constructor
+call that was used in creating *datatype*.
+
+
+NOTES
+-----
+
+These are the values that can be returned in *combiner* and their
+associated calls:
+
+::
+
+   Values                          Associated Calls
+
+   MPI_COMBINER_NAMED              a named predefined data type
+   MPI_COMBINER_DUP                MPI_Type_dup
+   MPI_COMBINER_CONTIGUOUS         MPI_Type_contiguous
+   MPI_COMBINER_VECTOR             MPI_Type_vector
+   MPI_COMBINER_HVECTOR            MPI_Type_hvector
+                                     and MPI_Type_create_hvector
+   MPI_COMBINER_INDEXED            MPI_Type_indexed
+   MPI_COMBINER_HINDEXED           MPI_Type_hindexed
+                                     and MPI_Type_create_hindexed
+   MPI_COMBINER_INDEXED_BLOCK      MPI_Type_create_indexed_block
+   MPI_COMBINER_STRUCT             MPI_Type_struct
+                                     and MPI_Type_create_struct
+   MPI_COMBINER_SUBARRAY           MPI_Type_create_subarray
+   MPI_COMBINER_DARRAY             MPI_Type_create_darray
+   MPI_COMBINER_F90_REAL           MPI_Type_create_f90_real
+   MPI_COMBINER_F90_COMPLEX        MPI_Type_create_f90_complex
+   MPI_COMBINER_F90_INTEGER        MPI_Type_create_f90_integer
+   MPI_COMBINER_RESIZED            MPI_Type_create_resized
+
+If *combiner* is MPI_COMBINER_NAMED, then *datatype* is a named
+predefined data type.
+
+The actual arguments used in the creation call for a data type can be
+obtained from the call :ref:`MPI_Type_get_contents`.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Type_get_contents`
diff --git a/docs/man-openmpi/man3/MPI_Type_get_extent.3.rst b/docs/man-openmpi/man3/MPI_Type_get_extent.3.rst
new file mode 100644
index 00000000000..23cd01e9afd
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_get_extent.3.rst
@@ -0,0 +1,49 @@
+.. _mpi_type_get_extent:
+
+
+MPI_Type_get_extent
+===================
+
+.. include_body
+
+:ref:`MPI_Type_get_extent`, :ref:`MPI_Type_get_extent_x` - Returns the lower
+bound and extent of a data type.
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Type_get_extent, MPI_Type_get_extent_x
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_type_get_extent.rst
+
+INPUT PARAMETER
+---------------
+* ``datatype``: Data type (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``lb``: Lower bound of data type (integer).
+* ``extent``: Data type extent (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Type_get_extent` returns the lower bound and the extent of
+*datatype*. For either function, if either the *lb* or *extent*
+parameter cannot express the value to be returned (e.g., if the
+parameter is too small to hold the output value), it is set to
+MPI_UNDEFINED.
+
+
+NOTE
+----
+
+Use of :ref:`MPI_Type_get_extent` is strongly recommended over the legacy MPI-1
+functions :ref:`MPI_Type_extent` and :ref:`MPI_Type_lb`.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Type_get_extent_x.3.rst b/docs/man-openmpi/man3/MPI_Type_get_extent_x.3.rst
new file mode 100644
index 00000000000..fedd8847451
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_get_extent_x.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_type_get_extent_x:
+
+MPI_Type_get_extent_x
+=====================
+    .. include_body
+
+.. include:: ../man3/MPI_Type_get_extent.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Type_get_name.3.rst b/docs/man-openmpi/man3/MPI_Type_get_name.3.rst
new file mode 100644
index 00000000000..24a1a1c0d63
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_get_name.3.rst
@@ -0,0 +1,37 @@
+.. _mpi_type_get_name:
+
+
+MPI_Type_get_name
+=================
+
+.. include_body
+
+:ref:`MPI_Type_get_name` |mdash| Gets the name of a data type.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_type_get_name.rst
+
+INPUT PARAMETER
+---------------
+* ``type``: Data type whose name is to be returned (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``type_name``: The name previously stored on the data type, or an empty string if not such name exists (string).
+* ``resultlen``: Length of returned name (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Type_get_name` returns the printable identifier associated with an
+MPI data type.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Type_set_name`
diff --git a/docs/man-openmpi/man3/MPI_Type_get_true_extent.3.rst b/docs/man-openmpi/man3/MPI_Type_get_true_extent.3.rst
new file mode 100644
index 00000000000..ffe4007d215
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_get_true_extent.3.rst
@@ -0,0 +1,53 @@
+.. _mpi_type_get_true_extent:
+
+
+MPI_Type_get_true_extent
+========================
+
+.. include_body
+
+:ref:`MPI_Type_get_true_extent`, :ref:`MPI_Type_get_true_extent_x` - Returns
+the true lower bound and extent of a data type's corresponding typemap,
+ignoring MPI_UB and MPI_LB markers.
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Type_get_true_extent, MPI_Type_get_true_extent_x
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_type_get_true_extent.rst
+
+INPUT PARAMETER
+---------------
+* ``datatype``: Data type for which information is wanted (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``true_lb``: True lower bound of data type (integer).
+* ``true_extent``: True size of data type (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The *true_lb* parameter returns the offset of the lowest unit of store
+that is addressed by the data type, that is, the lower bound of the
+corresponding typemap, ignoring MPI_LB markers. The *true_extent*
+parameter returns the true size of the data type, that is, the extent of
+the corresponding typemap, ignoring MPI_LB and MPI_UB markers, and
+performing no rounding for alignment. For both functions, if either the
+*true_lb* or *true_extent* parameter cannot express the value to be
+returned (e.g., if the parameter is too small to hold the output value),
+it is set to MPI_UNDEFINED.
+
+The *true_extent* is the minimum number of bytes of memory necessary to
+hold a data type, uncompressed.
+
+See section 4.1.8 of the MPI-3 standard for more detailed definitions of these
+parameters in relation to the typemap.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Type_get_true_extent_x.3.rst b/docs/man-openmpi/man3/MPI_Type_get_true_extent_x.3.rst
new file mode 100644
index 00000000000..2650c22bd21
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_get_true_extent_x.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_type_get_true_extent_x:
+
+MPI_Type_get_true_extent_x
+==========================
+    .. include_body
+
+.. include:: ../man3/MPI_Type_get_true_extent.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Type_hindexed.3.rst b/docs/man-openmpi/man3/MPI_Type_hindexed.3.rst
new file mode 100644
index 00000000000..9811b41605b
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_hindexed.3.rst
@@ -0,0 +1,75 @@
+.. _mpi_type_hindexed:
+
+
+MPI_Type_hindexed
+=================
+
+.. include_body
+
+:ref:`MPI_Type_hindexed` |mdash| Creates an indexed datatype with offsets in bytes |mdash| |deprecated_favor| :ref:`MPI_Type_create_hindexed`.
+
+
+SYNTAX
+------
+
+.. NOTE: The bindings for this man page are not automatically
+   generated from the official MPI Forum JSON/Python library because
+   this function is deprecated.  Hence, this function is not included
+   in the MPI Forum JSON data, and we therefore have to hard-code the
+   bindings here ourselves.
+
+C Syntax
+^^^^^^^^
+
+.. code-block:: c
+
+   #include <mpi.h>
+
+   int MPI_Type_hindexed(int count, int *array_of_blocklengths,
+   	MPI_Aint *array_of_displacements, MPI_Datatype oldtype,
+   	MPI_Datatype *newtype)
+
+
+Fortran Syntax
+^^^^^^^^^^^^^^
+
+.. code-block:: fortran
+
+   USE MPI
+   ! or the older form: INCLUDE 'mpif.h'
+
+   MPI_TYPE_HINDEXED(COUNT, ARRAY_OF_BLOCKLENGTHS,
+   		ARRAY_OF_DISPLACEMENTS, OLDTYPE, NEWTYPE, IERROR)
+   	INTEGER	COUNT, ARRAY_OF_BLOCKLENGTHS(*)
+   	INTEGER	ARRAY_OF_DISPLACEMENTS(*), OLDTYPE, NEWTYPE
+   	INTEGER	IERROR
+
+
+INPUT PARAMETERS
+----------------
+* ``count``: Number of blocks |mdash| also number of entries in array_of_displacements and array_of_blocklengths (integer).
+* ``array_of_blocklengths``: Number of elements in each block (array of nonnegative integers).
+* ``array_of_displacements``: Byte displacement of each block (C: array of *MPI_Aint*, Fortran: array of integer).
+* ``oldtype``: Old datatype (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``newtype``: New datatype (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Note that use of this routine is *deprecated* as of MPI-2. Use
+:ref:`MPI_Type_create_hindexed` instead.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Type_create_hindexed`
+   * :ref:`MPI_Type_create_struct`
+   * :ref:`MPI_Type_indexed`
diff --git a/docs/man-openmpi/man3/MPI_Type_hvector.3.rst b/docs/man-openmpi/man3/MPI_Type_hvector.3.rst
new file mode 100644
index 00000000000..deb44f11160
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_hvector.3.rst
@@ -0,0 +1,72 @@
+.. _mpi_type_hvector:
+
+
+MPI_Type_hvector
+================
+
+.. include_body
+
+:ref:`MPI_Type_hvector` |mdash| Creates a vector (strided) datatype with offset in bytes |mdash| |deprecated_favor| :ref:`MPI_Type_create_hvector`.
+
+
+SYNTAX
+------
+
+.. NOTE: The bindings for this man page are not automatically
+   generated from the official MPI Forum JSON/Python library because
+   this function is deprecated.  Hence, this function is not included
+   in the MPI Forum JSON data, and we therefore have to hard-code the
+   bindings here ourselves.
+
+C Syntax
+^^^^^^^^
+
+.. code-block:: c
+
+   #include <mpi.h>
+
+   int MPI_Type_hvector(int count, int blocklength, MPI_Aint stride,
+   	MPI_Datatype oldtype, MPI_Datatype *newtype)
+
+
+Fortran Syntax
+^^^^^^^^^^^^^^
+
+.. code-block:: fortran
+
+   USE MPI
+   ! or the older form: INCLUDE 'mpif.h'
+
+   MPI_TYPE_HVECTOR(COUNT, BLOCKLENGTH, STRIDE, OLDTYPE, NEWTYPE,
+   		IERROR)
+   	INTEGER	COUNT, BLOCKLENGTH, STRIDE, OLDTYPE
+   	INTEGER	NEWTYPE, IERROR
+
+
+INPUT PARAMETERS
+----------------
+* ``count``: Number of blocks (nonnegative integer).
+* ``blocklength``: Number of elements in each block (nonnegative integer).
+* ``stride``: Number of bytes between start of each block (integer).
+* ``oldtype``: Old datatype (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``newtype``: New datatype (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Note that use of this routine is *deprecated* as of MPI-2. Use
+:ref:`MPI_Type_create_hvector` instead.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Type_create_hvector`
+   * :ref:`MPI_Type_vector`
diff --git a/docs/man-openmpi/man3/MPI_Type_indexed.3.rst b/docs/man-openmpi/man3/MPI_Type_indexed.3.rst
new file mode 100644
index 00000000000..feb511bfbee
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_indexed.3.rst
@@ -0,0 +1,95 @@
+.. _mpi_type_indexed:
+
+
+MPI_Type_indexed
+================
+
+.. include_body
+
+:ref:`MPI_Type_indexed` - Creates an indexed datatype.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_type_indexed.rst
+
+INPUT PARAMETERS
+----------------
+* ``count``: Number of blocks |mdash| also number of entries in array_of_displacements and array_of_blocklengths (nonnegative integer).
+* ``array_of_blocklengths``: Number of elements per block (array of nonnegative integers).
+* ``array_of_displacements``: Displacement for each block, in multiples of oldtype extent for MPI_Type_indexed and bytes for MPI_Type_create_hindexed (array of integer for **MPI_TYPE_INDEXED**, array of *MPI_Aint* for **MPI_TYPE_CREATE_HINDEXED**).
+* ``oldtype``: Old datatype (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``newtype``: New datatype (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The function :ref:`MPI_Type_indexed` allows replication of an old datatype into
+a sequence of blocks (each block is a concatenation of the old
+datatype), where each block can contain a different number of copies and
+have a different displacement. All block displacements are multiples of
+the old data type's extent.
+
+**Example:** Let oldtype have type map {(double, 0), (char, 8)}, with
+extent 16. Let B = (3, 1) and let D = (4, 0). A call to
+MPI_Type_indexed(2, B, D, oldtype, newtype) returns a datatype with type
+map
+
+::
+
+       {(double, 64), (char, 72), (double, 80), (char, 88),
+       (double, 96), (char, 104),
+       (double, 0), (char, 8)}
+
+That is, three copies of the old type starting at displacement 4 x 16 =
+64, and one copy starting at displacement 0.
+
+In general, assume that oldtype has type map
+
+::
+
+       {(type(0), disp(0)), ..., (type(n-1), disp(n-1))},
+
+| with extent ex. Let B be the array_of_blocklength argument and D be
+  the array_of_displacements argument. The newly created datatype has
+
+::
+
+   n x S ^count-1
+       i = 0           B[i]  entries:
+
+       {(type(0), disp(0) + D[0]* ex), ...,
+       (type(n-1), disp(n-1) + D[0]* ex), ...,
+       (type(0), disp(0) + (D[0] + B[0]-1)* ex), ...,
+       (type(n-1), disp(n-1) + (D[0]+ B[0]-1)* ex), ...,
+       (type(0), disp(0) + D[count-1]* ex), ...,
+       (type(n-1), disp(n-1) + D[count-1]* ex), ...,
+       (type(0), disp(0) +  (D[count-1] + B[count-1] -1)* ex), ...,
+       (type(n-1), disp(n-1) + (D[count-1] + B[count-1] -1)* ex)}
+
+A call to MPI_Type_vector(count, blocklength, stride, oldtype, newtype)
+is equivalent to a call to MPI_Type_indexed(count, B, D, oldtype,
+newtype) where
+
+::
+
+       D[j] = j * stride, j = 0,..., count-1
+
+   and
+
+       B[j] = blocklength, j = 0, .., count-1
+
+The function :ref:`MPI_Type_create_hindexed` is identical to :ref:`MPI_Type_indexed`,
+except that block displacements in *array_of_displacements* are
+specified in bytes, rather than in multiples of the *oldtype* extent.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Type_hindexed`
diff --git a/docs/man-openmpi/man3/MPI_Type_lb.3.rst b/docs/man-openmpi/man3/MPI_Type_lb.3.rst
new file mode 100644
index 00000000000..35a7b414ef7
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_lb.3.rst
@@ -0,0 +1,108 @@
+.. _mpi_type_lb:
+
+
+MPI_Type_lb
+===========
+
+.. include_body
+
+:ref:`MPI_Type_lb` |mdash| Returns the lower bound of a data type |mdash| |deprecated_favor| :ref:`MPI_Type_get_extent`.
+
+
+SYNTAX
+------
+
+.. NOTE: The bindings for this man page are not automatically
+   generated from the official MPI Forum JSON/Python library because
+   this function is deprecated.  Hence, this function is not included
+   in the MPI Forum JSON data, and we therefore have to hard-code the
+   bindings here ourselves.
+
+C Syntax
+^^^^^^^^
+
+.. code-block:: c
+
+   #include <mpi.h>
+
+   int MPI_Type_lb(MPI_Datatype datatype, MPI_Aint *displacement)
+
+
+Fortran Syntax
+^^^^^^^^^^^^^^
+
+.. code-block:: fortran
+
+   USE MPI
+   ! or the older form: INCLUDE 'mpif.h'
+
+   MPI_TYPE_LB(DATATYPE, DISPLACEMENT, IERROR)
+   	INTEGER	DATATYPE, DISPLACEMENT, IERROR
+
+
+INPUT PARAMETER
+---------------
+* ``datatype``: Datatype (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``displacement``: Displacement of lower bound from origin, in bytes (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Note that use of this routine is *deprecated* as of MPI-2. Please use
+:ref:`MPI_Type_get_extent` instead.
+
+:ref:`MPI_Type_lb` returns the lower bound of a data type. This may differ from
+zero if the type was constructed using MPI_LB.
+
+The "pseudo-datatypes," MPI_LB and MPI_UB, can be used, respectively, to
+mark the lower bound (or the upper bound) of a datatype. These
+pseudo-datatypes occupy no space (extent (MPI_LB) = extent (MPI_UB) =0.
+They do not affect the size or count of a datatype, and do not affect
+the context of a message created with this datatype. However, they do
+affect the definition of the extent of a datatype and, therefore, affect
+the outcome of a replication of this datatype by a datatype constructor.
+
+In general, if
+
+::
+
+       Typemap = {(type0, disp0), ..., (type(n-1), disp(n-1)}
+
+then the lower bound of Typemap is defined to be
+
+::
+
+
+                     (min(j) disp(j)                          if no entry has
+       lb(Typemap) = (                                        basic type lb
+                     (min(j) {disp(j) such that type(j) = lb} otherwise
+
+Similarly, the upper bound of Typemap is defined to be
+
+::
+
+
+                     (max(j) disp(j) + sizeof((type(j)) + e   if no entry has
+       ub(Typemap) = (                                        basic type ub
+                     (max(j) {disp(j) such that type(j) = ub} otherwise
+
+   Then
+
+       extent(Typemap) = ub(Typemap) - lb(Typemap)
+
+If type(i) requires alignment to a byte address that is a multiple of
+k(i), then e is the least nonnegative increment needed to round
+extent(Typemap) to the next multiple of max(i) k(i).
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Type_get_extent`
diff --git a/docs/man-openmpi/man3/MPI_Type_match_size.3.rst b/docs/man-openmpi/man3/MPI_Type_match_size.3.rst
new file mode 100644
index 00000000000..5670845b7f4
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_match_size.3.rst
@@ -0,0 +1,51 @@
+.. _mpi_type_match_size:
+
+
+MPI_Type_match_size
+===================
+
+.. include_body
+
+:ref:`MPI_Type_match_size` |mdash| Returns an MPI datatype of a given type and
+size
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_type_match_size.rst
+
+INPUT PARAMETERS
+----------------
+* ``typeclass``: Generic type specifier (integer).
+* ``size``: Size, in bytes, of representation (integer).
+
+OUTPUT PARAMETERS
+-----------------
+* ``type``: Datatype with correct type and size (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The function returns an MPI datatype matching a local variable of type
+(*typeclass*, *size*). The returned type is a reference (handle) to a
+predefined named datatype, not a duplicate. This type cannot be freed.
+
+The value of *typeclass* may be set to one of MPI_TYPECLASS_REAL,
+MPI_TYPECLASS_INTEGER, or MPI_TYPECLASS_COMPLEX, corresponding to the
+desired datatype.
+
+MPI_type_match_size can be used to obtain a size-specific type that
+matches a Fortran numeric intrinsic type: first call :ref:`MPI_Sizeof` to
+compute the variable size, then call :ref:`MPI_Type_match_size` to find a
+suitable datatype. In C use the sizeof builtin instead of :ref:`MPI_Sizeof`.
+
+It is erroneous to specify a size not supported by the compiler.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Sizeof`
+   * :ref:`MPI_Type_get_extent`
diff --git a/docs/man-openmpi/man3/MPI_Type_set_attr.3.rst b/docs/man-openmpi/man3/MPI_Type_set_attr.3.rst
new file mode 100644
index 00000000000..501f32bf6a7
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_set_attr.3.rst
@@ -0,0 +1,40 @@
+.. _mpi_type_set_attr:
+
+
+MPI_Type_set_attr
+=================
+
+.. include_body
+
+:ref:`MPI_Type_set_attr` |mdash| Sets a key value/attribute pair to a data type.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_type_set_attr.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``type``: Data type to which attribute will be attached (handle).
+
+INPUT PARAMETERS
+----------------
+* ``type_keyval``: Key value (integer).
+* ``attribute_val``: Attribute value.
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+For the given data type, :ref:`MPI_Type_set_attr` sets the key value to the
+value of the specified attribute.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Type_get_attr`
diff --git a/docs/man-openmpi/man3/MPI_Type_set_name.3.rst b/docs/man-openmpi/man3/MPI_Type_set_name.3.rst
new file mode 100644
index 00000000000..3d66d96aacb
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_set_name.3.rst
@@ -0,0 +1,39 @@
+.. _mpi_type_set_name:
+
+
+MPI_Type_set_name
+=================
+
+.. include_body
+
+:ref:`MPI_Type_set_name` |mdash| Sets the name of a data type.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_type_set_name.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``type``: Data type for which the identifier is to be set (handle).
+
+INPUT PARAMETER
+---------------
+* ``type_name``: The character string remembered as the name (string).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Type_set_name` associates a printable identifier with an MPI data
+type.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Type_get_name`
diff --git a/docs/man-openmpi/man3/MPI_Type_size.3.rst b/docs/man-openmpi/man3/MPI_Type_size.3.rst
new file mode 100644
index 00000000000..e969b1d2a69
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_size.3.rst
@@ -0,0 +1,43 @@
+.. _mpi_type_size:
+
+
+MPI_Type_size
+=============
+
+.. include_body
+
+:ref:`MPI_Type_size`, :ref:`MPI_Type_size_x` - Returns the number of bytes
+occupied by entries in a data type.
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Type_size, MPI_Type_size_x
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_type_size.rst
+
+INPUT PARAMETER
+---------------
+* ``datatype``: Datatype (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``size``: Datatype size (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Type_size` returns the total size, in bytes, of the entries in the
+type signature associated with datatype; i.e., the total size of the
+data in a message that would be created with this datatype. Entries that
+occur multiple times in the datatype are counted with their
+multiplicity. For either function, if the *size* parameter cannot
+express the value to be returned (e.g., if the parameter is too small to
+hold the output value), it is set to MPI_UNDEFINED.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Type_size_x.3.rst b/docs/man-openmpi/man3/MPI_Type_size_x.3.rst
new file mode 100644
index 00000000000..f91adb09a5e
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_size_x.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_type_size_x:
+
+MPI_Type_size_x
+===============
+    .. include_body
+
+.. include:: ../man3/MPI_Type_size.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openmpi/man3/MPI_Type_struct.3.rst b/docs/man-openmpi/man3/MPI_Type_struct.3.rst
new file mode 100644
index 00000000000..c39466dd9c5
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_struct.3.rst
@@ -0,0 +1,75 @@
+.. _mpi_type_struct:
+
+
+MPI_Type_struct
+===============
+
+.. include_body
+
+:ref:`MPI_Type_struct` |mdash| Creates a *struct* data type |mdash| |deprecated_favor| :ref:`MPI_Type_create_struct`.
+
+
+SYNTAX
+------
+
+.. NOTE: The bindings for this man page are not automatically
+   generated from the official MPI Forum JSON/Python library because
+   this function is deprecated.  Hence, this function is not included
+   in the MPI Forum JSON data, and we therefore have to hard-code the
+   bindings here ourselves.
+
+C Syntax
+^^^^^^^^
+
+.. code-block:: c
+
+   #include <mpi.h>
+
+   int MPI_Type_struct(int count, int *array_of_blocklengths,
+   	MPI_Aint *array_of_displacements, MPI_Datatype *array_of_types,
+   	MPI_Datatype *newtype)
+
+
+Fortran Syntax
+^^^^^^^^^^^^^^
+
+.. code-block:: fortran
+
+   USE MPI
+   ! or the older form: INCLUDE 'mpif.h'
+
+   MPI_TYPE_STRUCT(COUNT, ARRAY_OF_BLOCKLENGTHS,
+   		ARRAY_OF_DISPLACEMENTS, ARRAY_OF_TYPES,
+   		NEWTYPE, IERROR)
+   	INTEGER	COUNT, ARRAY_OF_BLOCKLENGTHS(*)
+   	INTEGER	ARRAY_OF_DISPLACEMENTS(*)
+   	INTEGER	ARRAY_OF_TYPES(*), NEWTYPE, IERROR
+
+
+INPUT PARAMETERS
+----------------
+* ``count``: Number of blocks (integer) also number of entries in arrays array_of_types, array_of_displacements, and array_of_blocklengths.
+* ``array_of_blocklengths``: Number of elements in each block (array).
+* ``array_of_displacements``: Byte displacement of each block (array).
+* ``array_of_types``: Type of elements in each block (array of handles to datatype objects).
+
+OUTPUT PARAMETERS
+-----------------
+* ``newtype``: New datatype (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Note that use of this routine is *deprecated* as of MPI-2.
+For details of use, see :ref:`MPI_Type_create_struct`.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Type_create_struct`
+   * :ref:`MPI_Type_create_hindexed`
diff --git a/docs/man-openmpi/man3/MPI_Type_ub.3.rst b/docs/man-openmpi/man3/MPI_Type_ub.3.rst
new file mode 100644
index 00000000000..18d07654524
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_ub.3.rst
@@ -0,0 +1,113 @@
+.. _mpi_type_ub:
+
+
+MPI_Type_ub
+===========
+
+.. include_body
+
+:ref:`MPI_Type_ub` |mdash| Returns the upper bound of a datatype |mdash| |deprecated_favor| :ref:`MPI_Type_get_extent`.
+
+
+SYNTAX
+------
+
+.. NOTE: The bindings for this man page are not automatically
+   generated from the official MPI Forum JSON/Python library because
+   this function is deprecated.  Hence, this function is not included
+   in the MPI Forum JSON data, and we therefore have to hard-code the
+   bindings here ourselves.
+
+C Syntax
+^^^^^^^^
+
+.. code-block:: c
+
+   #include <mpi.h>
+
+   int MPI_Type_ub(MPI_Datatype datatype, MPI_Aint *displacement)
+
+
+Fortran Syntax
+^^^^^^^^^^^^^^
+
+.. code-block:: fortran
+
+   USE MPI
+   ! or the older form: INCLUDE 'mpif.h'
+
+   MPI_TYPE_UB(DATATYPE, DISPLACEMENT, IERROR)
+   	INTEGER	DATATYPE, DISPLACEMENT, IERROR
+
+
+INPUT PARAMETER
+---------------
+* ``datatype``: Datatype (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``displacement``: Displacement of upper bound from origin, in bytes (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Note that use of this routine is *deprecated* as of MPI-2. Please use
+:ref:`MPI_Type_get_extent` instead.
+
+:ref:`MPI_Type_ub` returns the upper bound of a data type. This will differ
+from zero if the type was constructed using MPI_UB. The upper bound will
+take into account any alignment considerations.
+
+The "pseudo-datatypes," MPI_LB and MPI_UB, can be used, respectively, to
+mark the upper bound (or the lower bound) of a datatype. These
+pseudo-datatypes occupy no space (extent (MPI_LB) = extent (MPI_UB) =0.
+They do not affect the size or count of a datatype, and do not affect
+the context of a message created with this datatype. However, they do
+affect the definition of the extent of a datatype and, therefore, affect
+the outcome of a replication of this datatype by a datatype constructor.
+
+In general, if
+
+::
+
+
+       Typemap = {(type(0), disp(0)), ..., (type(n-1), disp(n-1))}
+
+then the lower bound of Typemap is defined to be
+
+::
+
+
+                     (min(j) disp(j)                          if no entry has
+       lb(Typemap) = (                                        basic type lb
+                     (min(j) {disp(j) such that type(j) = lb} otherwise
+
+Similarly, the upper bound of Typemap is defined to be
+
+::
+
+
+                     (max(j) disp(j) + sizeof(type(j) = lb}   if no entry has
+       ub(Typemap) = (                                        basic type ub
+                     (max(j) {disp(j) such that type(j) = ub} otherwise
+
+Then
+
+::
+
+
+       extent(Typemap) = ub(Typemap) - lb(Typemap)
+
+If type(i) requires alignment to a byte address that is a multiple of
+k(i), then e is the least nonnegative increment needed to round
+extent(Typemap) to the next multiple of max(i) k(i).
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Type_get_extent`
diff --git a/docs/man-openmpi/man3/MPI_Type_vector.3.rst b/docs/man-openmpi/man3/MPI_Type_vector.3.rst
new file mode 100644
index 00000000000..3666d165da1
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Type_vector.3.rst
@@ -0,0 +1,96 @@
+.. _mpi_type_vector:
+
+
+MPI_Type_vector
+===============
+
+.. include_body
+
+:ref:`MPI_Type_vector` |mdash| Creates a vector (strided) datatype.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_type_vector.rst
+
+INPUT PARAMETERS
+----------------
+* ``count``: Number of blocks (nonnegative integer).
+* ``blocklength``: Number of elements in each block (nonnegative integer).
+* ``stride``: Number of elements between start of each block (integer).
+* ``oldtype``: Old datatype (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``newtype``: New datatype (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The function :ref:`MPI_Type_vector` is a general constructor that allows
+replication of a datatype into locations that consist of equally spaced
+blocks. Each block is obtained by concatenating the same number of
+copies of the old datatype. The spacing between blocks is a multiple of
+the extent of the old datatype.
+
+**Example 1:** Assume, again, that oldtype has type map {(double, 0),
+(char, 8)}, with extent 16. A call to MPI_Type_vector(2, 3, 4, oldtype,
+newtype) will create the datatype with type map
+
+::
+
+       {(double, 0), (char, 8), (double, 16), (char, 24),
+       (double, 32), (char, 40),
+       (double, 64), (char, 72),
+       (double, 80), (char, 88), (double, 96), (char, 104)}
+
+That is, two blocks with three copies each of the old type, with a
+stride of 4 elements (4 x 16 bytes) between the blocks.
+
+**Example 2:** A call to MPI_Type_vector(3, 1, -2, oldtype, newtype)
+will create the datatype
+
+::
+
+
+       {(double, 0), (char, 8), (double, -32), (char, -24),
+       (double, -64), (char, -56)}
+
+In general, assume that oldtype has type map
+
+::
+
+
+       {(type(0), disp(0)), ..., (type(n-1), disp(n-1))},
+
+with extent ex. Let bl be the blocklength. The newly created datatype
+has a type map with count x bl x n entries:
+
+::
+
+
+       {(type(0), disp(0)), ..., (type(n-1), disp(n-1)),
+       (type(0), disp(0) + ex), ..., (type(n-1), disp(n-1) + ex), ...,
+       (type(0), disp(0) + (bl -1) * ex),...,
+       (type(n-1), disp(n-1) + (bl -1)* ex),
+       (type(0), disp(0) + stride * ex),..., (type(n-1),
+       disp(n-1) + stride * ex), ...,
+       (type(0), disp(0) + (stride + bl - 1) * ex), ...,
+       (type(n-1), disp(n-1) + (stride + bl -1) * ex), ...,
+       (type(0), disp(0) + stride * (count -1) * ex), ...,
+       (type(n-1), disp(n-1) + stride * (count -1) * ex), ...,
+       (type(0), disp(0) + (stride * (count -1) + bl -1) * ex), ...,
+       (type(n-1), disp(n-1) + (stride * (count -1) + bl -1) * ex)}
+
+A call to MPI_Type_contiguous(count, oldtype, newtype) is equivalent to
+a call to MPI_Type_vector(count, 1, 1, oldtype, newtype), or to a call
+to MPI_Type_vector(1, count, n, oldtype, newtype), n arbitrary.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Type_create_hvector`
+   * :ref:`MPI_Type_hvector`
diff --git a/docs/man-openmpi/man3/MPI_Unpack.3.rst b/docs/man-openmpi/man3/MPI_Unpack.3.rst
new file mode 100644
index 00000000000..b6a4911c3c6
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Unpack.3.rst
@@ -0,0 +1,111 @@
+.. _mpi_unpack:
+
+
+MPI_Unpack
+==========
+
+.. include_body
+
+:ref:`MPI_Unpack` |mdash| Unpacks a datatype into contiguous memory.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_unpack.rst
+
+INPUT PARAMETERS
+----------------
+* ``inbuf``: Input buffer start (choice).
+* ``insize``: Size of input buffer, in bytes (integer).
+* ``outcount``: Number of items to be unpacked (integer).
+* ``datatype``: Datatype of each output data item (handle).
+* ``comm``: Communicator for packed message (handle).
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``position``: Current position in bytes (integer).
+
+OUTPUT PARAMETERS
+-----------------
+* ``outbuf``: Output buffer start (choice).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Unpacks a message into the receive buffer specified by outbuf, outcount,
+datatype from the buffer space specified by inbuf and insize. The output
+buffer can be any communication buffer allowed in :ref:`MPI_Recv`. The input
+buffer is a contiguous storage area containing insize bytes, starting at
+address inbuf. The input value of position is the first location in the
+input buffer occupied by the packed message. *position* is incremented
+by the size of the packed message, so that the output value of position
+is the first location in the input buffer after the locations occupied
+by the message that was unpacked. *comm* is the communicator used to
+receive the packed message.
+
+
+NOTES
+-----
+
+Note the difference between :ref:`MPI_Recv` and :ref:`MPI_Unpack`: In :ref:`MPI_Recv`, the
+*count* argument specifies the maximum number of items that can be
+received. The actual number of items received is determined by the
+length of the incoming message. In :ref:`MPI_Unpack`, the count argument
+specifies the actual number of items that are to be unpacked; the "size"
+of the corresponding message is the increment in position. The reason
+for this change is that the "incoming message size" is not predetermined
+since the user decides how much to unpack; nor is it easy to determine
+the "message size" from the number of items to be unpacked.
+
+To understand the behavior of pack and unpack, it is convenient to think
+of the data part of a message as being the sequence obtained by
+concatenating the successive values sent in that message. The pack
+operation stores this sequence in the buffer space, as if sending the
+message to that buffer. The unpack operation retrieves this sequence
+from buffer space, as if receiving a message from that buffer. (It is
+helpful to think of internal Fortran files or sscanf in C for a similar
+function.)
+
+Several messages can be successively packed into one packing unit. This
+is effected by several successive related calls to :ref:`MPI_Pack`, where the
+first call provides position = 0, and each successive call inputs the
+value of position that was output by the previous call, and the same
+values for outbuf, outcount, and comm. This packing unit now contains
+the equivalent information that would have been stored in a message by
+one send call with a send buffer that is the "concatenation" of the
+individual send buffers.
+
+A packing unit can be sent using type MPI_Packed. Any point-to-point or
+collective communication function can be used to move the sequence of
+bytes that forms the packing unit from one process to another. This
+packing unit can now be received using any receive operation, with any
+datatype: The type-matching rules are relaxed for messages sent with
+type MPI_Packed.
+
+A message sent with any type (including MPI_Packed) can be received
+using the type MPI_Packed. Such a message can then be unpacked by calls
+to :ref:`MPI_Unpack`.
+
+A packing unit (or a message created by a regular, "typed" send) can be
+unpacked into several successive messages. This is effected by several
+successive related calls to :ref:`MPI_Unpack`, where the first call provides
+position = 0, and each successive call inputs the value of position that
+was output by the previous call, and the same values for inbuf, insize,
+and comm.
+
+The concatenation of two packing units is not necessarily a packing
+unit; nor is a substring of a packing unit necessarily a packing unit.
+Thus, one cannot concatenate two packing units and then unpack the
+result as one packing unit; nor can one unpack a substring of a packing
+unit as a separate packing unit. Each packing unit that was created by a
+related sequence of pack calls or by a regular send must be unpacked as
+a unit, by a sequence of related unpack calls.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Pack`
+   * :ref:`MPI_Pack_size`
diff --git a/docs/man-openmpi/man3/MPI_Unpack_external.3.rst b/docs/man-openmpi/man3/MPI_Unpack_external.3.rst
new file mode 100644
index 00000000000..0fa868d491a
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Unpack_external.3.rst
@@ -0,0 +1,114 @@
+.. _mpi_unpack_external:
+
+
+MPI_Unpack_external
+===================
+
+.. include_body
+
+:ref:`MPI_Unpack_external` |mdash| Reads data from a portable format
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_unpack_external.rst
+
+INPUT PARAMETERS
+----------------
+* ``datarep``: Data Representation (string).
+* ``inbuf``: Input buffer start (choice).
+* ``insize``: Size of input buffer, in bytes (integer).
+* ``outcount``: Number of items to be unpacked (integer).
+* ``datatype``: Datatype of each output data item (handle).
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``position``: Current position in buffer, in bytes (integer).
+
+OUTPUT PARAMETERS
+-----------------
+* ``outbuf``: Output buffer start (choice).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Unpack_external` unpacks data from the external32 format, a universal
+data representation defined by the MPI Forum. This format is useful for
+exchanging data between MPI implementations, or when writing data to a
+file.
+
+The input buffer is a contiguous storage area pointed to by *inbuf*
+containing *insize* bytes. The output buffer can be any communication
+buffer allowed in :ref:`MPI_Recv`, and is specified by *outbuf*, *outcount*,
+and *datatype*.
+
+The input value of *position* is the first position in *inbuf* to be
+read for unpacking (measured in bytes, not elements, relative to the
+start of the buffer). When the function returns, *position* is
+incremented by the size of the packed message, so that it points to the
+first location in *inbuf* following the message that was unpacked. This
+way it may be used as input to a subsequent call to :ref:`MPI_Unpack_external`.
+
+
+NOTES
+-----
+
+Note the difference between :ref:`MPI_Recv` and :ref:`MPI_Unpack_external`: In
+:ref:`MPI_Recv`, the *count* argument specifies the maximum number of items
+that can be received. In :ref:`MPI_Unpack_external`, the *outcount* argument
+specifies the actual number of items that are to be unpacked. With a
+regular receive operation, the incoming message size determines the
+number of components that will be received. With :ref:`MPI_Unpack_external`, it
+is up to the user to specify how many components to unpack, since the
+user may wish to unpack the received message multiple times into various
+buffers.
+
+To understand the behavior of pack and unpack, it is convenient to think
+of the data part of a message as being the sequence obtained by
+concatenating the successive values sent in that message. The pack
+operation stores this sequence in the buffer space, as if sending the
+message to that buffer. The unpack operation retrieves this sequence
+from buffer space, as if receiving a message from that buffer. (It is
+helpful to think of internal Fortran files or sscanf in C for a similar
+function.)
+
+Several messages can be successively packed into one packing unit. This
+is effected by several successive related calls to :ref:`MPI_Pack_external`,
+where the first call provides *position*\ =0, and each successive call
+inputs the value of *position* that was output by the previous call,
+along with the same values for *outbuf* and *outcount*. This packing
+unit now contains the equivalent information that would have been stored
+in a message by one send call with a send buffer that is the
+"concatenation" of the individual send buffers.
+
+A packing unit can be sent using type MPI_BYTE. Any point-to-point or
+collective communication function can be used to move the sequence of
+bytes that forms the packing unit from one process to another. This
+packing unit can now be received using any receive operation, with any
+datatype: The type-matching rules are relaxed for messages sent with
+type MPI_BYTE.
+
+A packing unit can be unpacked into several successive messages. This is
+effected by several successive related calls to :ref:`MPI_Unpack_external`,
+where the first call provides *position*\ =0, and each successive call
+inputs the value of position that was output by the previous call, and
+the same values for *inbuf* and *insize*.
+
+The concatenation of two packing units is not necessarily a packing
+unit; nor is a substring of a packing unit necessarily a packing unit.
+Thus, one cannot concatenate two packing units and then unpack the
+result as one packing unit; nor can one unpack a substring of a packing
+unit as a separate packing unit. Each packing unit that was created by a
+related sequence of pack calls must be unpacked as a unit by a sequence
+of related unpack calls.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Pack_external`
+   * :ref:`MPI_Pack_external_size`
+   * :ref:`MPI_Recv`
+   * sscanf(3C)
diff --git a/docs/man-openmpi/man3/MPI_Unpublish_name.3.rst b/docs/man-openmpi/man3/MPI_Unpublish_name.3.rst
new file mode 100644
index 00000000000..86709c63f3b
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Unpublish_name.3.rst
@@ -0,0 +1,88 @@
+.. _mpi_unpublish_name:
+
+
+MPI_Unpublish_name
+==================
+
+.. include_body
+
+::
+
+   MPI_Unpublish_name - Unpublishes a service name
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_unpublish_name.rst
+
+INPUT PARAMETERS
+----------------
+* ``service_name``: A service name (string).
+* ``info``: Options to the name service functions (handle).
+* ``port_name``: A port name (string).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This routine removes the pair (*service_name, port_name*) so that
+applications may no longer retrieve *port_name* by calling
+:ref:`MPI_Lookup_name`. It is an error to unpublish a *service_name* that was
+not published via :ref:`MPI_Publish_name`. Both the *service_name* and
+*port_name* arguments to :ref:`MPI_Unpublish_name` must be identical to the
+arguments to the previous call to :ref:`MPI_Publish_name`.
+
+
+INFO ARGUMENTS
+--------------
+
+The following keys for *info* are recognized:
+
+::
+
+   Key                   Type      Description
+   ---                   ----      -----------
+
+   ompi_global_scope     bool      If set to true, unpublish the name from
+                                   the global scope.  Unpublish from the local
+                                   scope otherwise.  See the NAME SCOPE
+                                   section for more details.
+
+*bool* info keys are actually strings but are evaluated as follows: if
+the string value is a number, it is converted to an integer and cast to
+a boolean (meaning that zero integers are false and non-zero values are
+true). If the string value is (case-insensitive) "yes" or "true", the
+boolean is true. If the string value is (case-insensitive) "no" or
+"false", the boolean is false. All other string values are unrecognized,
+and therefore false.
+
+If no info key is provided, the function will first check to see if a
+global server has been specified and is available. If so, then the
+unpublish function will default to global scope first, followed by
+local. Otherwise, the data will default to unpublish with local scope.
+
+
+NAME SCOPE
+----------
+
+Open MPI supports two name scopes: *global* and *local*. Local scope
+values are placed in a data store located on the mpirun of the calling
+process' job, while global scope values reside on a central server.
+Calls to :ref:`MPI_Unpublish_name` must correctly specify the scope to be used
+in finding the value to be removed. The function will return an error if
+the specified service name is not found on the indicated location.
+
+For a more detailed description of scoping rules, please see the
+:ref:`MPI_Publish_name` man page.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Publish_name`
+   * :ref:`MPI_Lookup_name`
+   * :ref:`MPI_Open_port`
diff --git a/docs/man-openmpi/man3/MPI_Wait.3.rst b/docs/man-openmpi/man3/MPI_Wait.3.rst
new file mode 100644
index 00000000000..a6abd681bf0
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Wait.3.rst
@@ -0,0 +1,105 @@
+.. _mpi_wait:
+
+
+MPI_Wait
+========
+
+.. include_body
+
+:ref:`MPI_Wait` |mdash| Waits for an MPI send or receive to complete.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_wait.rst
+
+INPUT PARAMETER
+---------------
+* ``request``: Request (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``status``: Status object (status).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+A call to :ref:`MPI_Wait` returns when the operation identified by request is
+complete. If the communication object associated with this request was
+created by a nonblocking send or receive call, then the object is
+deallocated by the call to :ref:`MPI_Wait` and the request handle is set to
+``MPI_REQUEST_NULL``.
+
+The call returns, in status, information on the completed
+operation. The content of the status object for a receive operation
+can be accessed as described in the "Return Status" subsection of the
+"Point-to-Point Communication" chapter in the `MPI Standard
+<https://www.mpi-forum.org/docs/>`_.  The status object for a send
+operation may be queried by a call to :ref:`MPI_Test_cancelled` (see
+the "Probe and Cancel" section in the `MPI Standard
+<https://www.mpi-forum.org/docs/>`_).
+
+If your application does not need to examine the *status* field, you can
+save resources by using the predefined constant ``MPI_STATUS_IGNORE`` as a
+special value for the *status* argument.
+
+One is allowed to call :ref:`MPI_Wait` with a null or inactive request
+argument. In this case the operation returns immediately with empty
+status.
+
+
+NOTES
+-----
+
+Successful return of :ref:`MPI_Wait` after an :ref:`MPI_Ibsend` implies that the user
+send buffer can be reused i.e., data has been sent out or copied into a
+buffer attached with :ref:`MPI_Buffer_attach`. Note that, at this point, we can
+no longer cancel the send (for more information, see the "Probe and
+Cancel" section in the `MPI Standard
+<https://www.mpi-forum.org/docs/>`_).  If a matching receive is never
+posted, then the buffer cannot be freed. This runs somewhat counter to
+the stated goal of :ref:`MPI_Cancel` (always being able to free program space
+that was committed to the communication subsystem).
+
+Example: Simple usage of nonblocking operations and :ref:`MPI_Wait`.
+
+.. code-block:: fortran
+
+       CALL MPI_COMM_RANK(comm, rank, ierr)
+       IF(rank == 0) THEN
+           CALL MPI_ISEND(a(1), 10, MPI_REAL, 1, tag, comm, request, ierr)
+           **** do some computation ****
+           CALL MPI_WAIT(request, status, ierr)
+       ELSE IF (rank == 1) THEN
+           CALL MPI_IRECV(a(1), 15, MPI_REAL, 0, tag, comm, request, ierr)
+           **** do some computation ****
+           CALL MPI_WAIT(request, status, ierr)
+       END IF
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+Note that per the "Return Status" section in the "Point-to-Point
+Communication" chapter in the `MPI Standard
+<https://www.mpi-forum.org/docs/>`_, MPI errors on requests passed to
+:ref:`MPI_WAIT` do not set the ``status.MPI_ERROR`` field in the
+returned *status*.  The error code is always passed to the back-end
+error handler and may be passed back to the caller through the return
+value of :ref:`MPI_WAIT` if the back-end error handler returns it.  The
+pre-defined MPI error handler ``MPI_ERRORS_RETURN`` exhibits this
+behavior, for example.
+
+
+.. seealso::
+   * :ref:`MPI_Comm_set_errhandler`
+   * :ref:`MPI_File_set_errhandler`
+   * :ref:`MPI_Test`
+   * :ref:`MPI_Testall`
+   * :ref:`MPI_Testany`
+   * :ref:`MPI_Testsome`
+   * :ref:`MPI_Waitall`
+   * :ref:`MPI_Waitany`
+   * :ref:`MPI_Waitsome`
+   * :ref:`MPI_Win_set_errhandler`
diff --git a/docs/man-openmpi/man3/MPI_Waitall.3.rst b/docs/man-openmpi/man3/MPI_Waitall.3.rst
new file mode 100644
index 00000000000..32a2550fe7c
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Waitall.3.rst
@@ -0,0 +1,89 @@
+.. _mpi_waitall:
+
+
+MPI_Waitall
+===========
+
+.. include_body
+
+:ref:`MPI_Waitall` |mdash| Waits for all given communications to complete.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_waitall.rst
+
+INPUT PARAMETERS
+----------------
+* ``count``: Lists length (integer).
+* ``array_of_requests``: Array of requests (array of handles).
+
+OUTPUT PARAMETERS
+-----------------
+* ``array_of_statuses``: Array of status objects (array of status).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Blocks until all communication operations associated with active handles
+in the list complete, and returns the status of all these operations
+(this includes the case where no handle in the list is active). Both
+arrays have the same number of valid entries. The ith entry in
+array_of_statuses is set to the return status of the ith operation.
+Requests that were created by nonblocking communication operations are
+deallocated, and the corresponding handles in the array are set to
+MPI_REQUEST_NULL. The list may contain null or inactive handles. The
+call sets to empty the status of each such entry.
+
+The error-free execution of MPI_Waitall(count, array_of_requests,
+array_of_statuses) has the same effect as the execution of
+MPI_Wait(&array_of_request[i], &array_of_statuses[i]), for
+i=0,...,count-1, in some arbitrary order. :ref:`MPI_Waitall` with an array of
+length 1 is equivalent to :ref:`MPI_Wait`.
+
+When one or more of the communications completed by a call to
+:ref:`MPI_Waitall` fail, it is desirable to return specific information on each
+communication. The function :ref:`MPI_Waitall` will return in such case the
+error code MPI_ERR_IN_STATUS and will set the error field of each status
+to a specific error code. This code will be MPI_SUCCESS if the specific
+communication completed; it will be another specific error code if it
+failed; or it can be MPI_ERR_PENDING if it has neither failed nor
+completed. The function :ref:`MPI_Waitall` will return MPI_SUCCESS if no
+request had an error, or will return another error code if it failed for
+other reasons (such as invalid arguments). In such cases, it will not
+update the error fields of the statuses.
+
+If your application does not need to examine the *array_of_statuses*
+field, you can save resources by using the predefined constant
+MPI_STATUSES_IGNORE can be used as a special value for the
+*array_of_statuses* argument.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+For each invocation of :ref:`MPI_Waitall`, if one or more requests
+generate an MPI error, only the *first* MPI request that caused an
+error will be passed to its corresponding error handler. No other
+error handlers will be invoked (even if multiple requests generated
+errors). However, *all* requests that generate an error will have a
+relevant error code set in the corresponding ``status.MPI_ERROR``
+field (unless ``MPI_STATUSES_IGNORE`` was used).
+
+If the invoked error handler allows :ref:`MPI_Waitall` to return to
+the caller, the value ``MPI_ERR_IN_STATUS`` will be returned in the C
+and Fortran bindings.
+
+
+.. seealso::
+   * :ref:`MPI_Comm_set_errhandler`
+   * :ref:`MPI_File_set_errhandler`
+   * :ref:`MPI_Test`
+   * :ref:`MPI_Testall`
+   * :ref:`MPI_Testany`
+   * :ref:`MPI_Testsome`
+   * :ref:`MPI_Wait`
+   * :ref:`MPI_Waitany`
+   * :ref:`MPI_Waitsome`
+   * :ref:`MPI_Win_set_errhandler`
diff --git a/docs/man-openmpi/man3/MPI_Waitany.3.rst b/docs/man-openmpi/man3/MPI_Waitany.3.rst
new file mode 100644
index 00000000000..29bbac44b9d
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Waitany.3.rst
@@ -0,0 +1,97 @@
+.. _mpi_waitany:
+
+
+MPI_Waitany
+===========
+
+.. include_body
+
+:ref:`MPI_Waitany` |mdash| Waits for any specified send or receive to complete.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_waitany.rst
+
+INPUT PARAMETERS
+----------------
+* ``count``: List length (integer).
+* ``array_of_requests``: Array of requests (array of handles).
+
+OUTPUT PARAMETERS
+-----------------
+* ``index``: Index of handle for operation that completed (integer). In the range 0 to count-1. In Fortran, the range is 1 to count.
+* ``status``: Status object (status).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+A call to :ref:`MPI_Waitany` can be used to wait for the completion of one out
+of several requests.
+
+The array_of_requests list may contain null or inactive handles. If the
+list contains no active handles (list has length zero or all entries are
+null or inactive), then the call returns immediately with index =
+``MPI_UNDEFINED``, and an empty status.
+
+The execution of MPI_Waitany(count, array_of_requests, index, status)
+has the same effect as the execution of ``MPI_Wait(&array_of_requests[i], status)``,
+where i is the value returned by index (unless the value of
+index is ``MPI_UNDEFINED``). :ref:`MPI_Waitany` with an array containing one active
+entry is equivalent to :ref:`MPI_Wait`.
+
+If your application does not need to examine the *status* field, you can
+save resources by using the predefined constant ``MPI_STATUS_IGNORE`` as a
+special value for the *status* argument.
+
+**Example:** Client-server code (starvation can occur).
+
+.. code-block:: fortran
+
+       CALL MPI_COMM_SIZE(comm, size, ierr)
+       CALL MPI_COMM_RANK(comm, rank, ierr)
+       IF(rank > 0) THEN         ! client code
+           DO
+              CALL MPI_ISEND(a, n, MPI_REAL, 0, tag, comm, request, ierr)
+              CALL MPI_WAIT(request, status, ierr)
+           END DO
+       ELSE         ! rank=0: server code
+              DO i=1, size-1
+                 CALL MPI_IRECV(a(1,i), n, MPI_REAL, i tag, &
+                          comm, request_list(i), ierr)
+              END DO
+              DO
+                 CALL MPI_WAITANY(size-1, request_list, index, status, ierr)
+                 CALL DO_SERVICE(a(1,index))  ! handle one message
+                 CALL MPI_IRECV(a(1, index), n, MPI_REAL, index, tag, &
+                           comm, request_list(index), ierr)
+              END DO
+       END IF
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+Note that per the "Return Status" section in the "Point-to-Point
+Communication" chapter in the `MPI Standard
+<https://www.mpi-forum.org/docs/>`_, MPI errors on requests passed to
+:ref:`MPI_WAITANY` do not set the ``status.MPI_ERROR`` field in the
+returned *status*.  The error code is always passed to the back-end
+error handler and may be passed back to the caller through the return
+value of :ref:`MPI_WAITANY` if the back-end error handler returns it.
+The pre-defined MPI error handler ``MPI_ERRORS_RETURN`` exhibits this
+behavior, for example.
+
+
+.. seealso::
+   * :ref:`MPI_Comm_set_errhandler`
+   * :ref:`MPI_File_set_errhandler`
+   * :ref:`MPI_Test`
+   * :ref:`MPI_Testall`
+   * :ref:`MPI_Testany`
+   * :ref:`MPI_Testsome`
+   * :ref:`MPI_Wait`
+   * :ref:`MPI_Waitall`
+   * :ref:`MPI_Waitsome`
+   * :ref:`MPI_Win_set_errhandler`
diff --git a/docs/man-openmpi/man3/MPI_Waitsome.3.rst b/docs/man-openmpi/man3/MPI_Waitsome.3.rst
new file mode 100644
index 00000000000..2c0c98f55fd
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Waitsome.3.rst
@@ -0,0 +1,124 @@
+.. _mpi_waitsome:
+
+
+MPI_Waitsome
+============
+
+.. include_body
+
+:ref:`MPI_Waitsome` |mdash| Waits for some given communications to complete.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_waitsome.rst
+
+INPUT PARAMETERS
+----------------
+* ``incount``: Length of array_of_requests (integer).
+* ``array_of_requests``: Array of requests (array of handles).
+
+OUTPUT PARAMETERS
+-----------------
+* ``outcount``: Number of completed requests (integer).
+* ``array_of_indices``: Array of indices of operations that completed (array of integers).
+* ``array_of_statuses``: Array of status objects for operations that completed (array of status).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Waits until at least one of the operations associated with active
+handles in the list have completed. Returns in outcount the number of
+requests from the list array_of_requests that have completed. Returns in
+the first outcount locations of the array array_of_indices the indices
+of these operations (index within the array array_of_requests; the array
+is indexed from 0 in C and from 1 in Fortran). Returns in the first
+outcount locations of the array array_of_status the status for these
+completed operations. If a request that completed was allocated by a
+nonblocking communication call, then it is deallocated, and the
+associated handle is set to MPI_REQUEST_NULL.
+
+If the list contains no active handles, then the call returns
+immediately with outcount = MPI_UNDEFINED.
+
+When one or more of the communications completed by :ref:`MPI_Waitsome` fails,
+then it is desirable to return specific information on each
+communication. The arguments outcount, array_of_indices, and
+array_of_statuses will be adjusted to indicate completion of all
+communications that have succeeded or failed. The call will return the
+error code MPI_ERR_IN_STATUS and the error field of each status returned
+will be set to indicate success or to indicate the specific error that
+occurred. The call will return MPI_SUCCESS if no request resulted in an
+error, and will return another error code if it failed for other reasons
+(such as invalid arguments). In such cases, it will not update the error
+fields of the statuses.
+
+If your application does not need to examine the *array_of_statuses*
+field, you can save resources by using the predefined constant
+MPI_STATUSES_IGNORE can be used as a special value for the
+*array_of_statuses* argument.
+
+**Example:** Same code as the example in the :ref:`MPI_Waitany` man page, but
+using :ref:`MPI_Waitsome`.
+
+.. code-block:: fortran
+
+       CALL MPI_COMM_SIZE(comm, size, ierr)
+       CALL MPI_COMM_RANK(comm, rank, ierr)
+       IF(rank > 0) THEN         ! client code
+           DO
+              CALL MPI_ISEND(a, n, MPI_REAL, 0, tag, comm, request, ierr)
+              CALL MPI_WAIT(request, status, ierr)
+           END DO
+       ELSE         ! rank=0: server code
+           DO i=1, size-1
+              CALL MPI_IRECV(a(1,i), n, MPI_REAL, i, tag, &
+                             comm, requests(i), ierr)
+           END DO
+           DO
+              CALL MPI_WAITSOME(size, request_list, numdone, &
+                               indices, statuses, ierr)
+              DO i=1, numdone
+                 CALL DO_SERVICE(a(1, indices(i)))
+                 CALL MPI_IRECV(a(1, indices(i)), n, MPI_REAL, i, tag, &
+                              comm, requests(indices(i)), ierr)
+              END DO
+           END DO
+       END IF
+
+
+NOTES
+-----
+
+The array of indices are in the range 0 to incount-1 for C and in the
+range 1 to incount for Fortran.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+For each invocation of :ref:`MPI_Waitsome`, if one or more requests
+generate an MPI error, only the *first* MPI request that caused an
+error will be passed to its corresponding error handler. No other
+error handlers will be invoked (even if multiple requests generated
+errors). However, *all* requests that generate an error will have a
+relevant error code set in the corresponding ``status.MPI_ERROR``
+field (unless ``MPI_STATUSES_IGNORE`` was used).
+
+If the invoked error handler allows :ref:`MPI_Waitsome` to return to
+the caller, the value ``MPI_ERR_IN_STATUS`` will be returned in the C
+and Fortran bindings.
+
+
+.. seealso::
+   * :ref:`MPI_Comm_set_errhandler`
+   * :ref:`MPI_File_set_errhandler`
+   * :ref:`MPI_Test`
+   * :ref:`MPI_Testall`
+   * :ref:`MPI_Testany`
+   * :ref:`MPI_Testsome`
+   * :ref:`MPI_Wait`
+   * :ref:`MPI_Waitall`
+   * :ref:`MPI_Waitany`
+   * :ref:`MPI_Win_set_errhandler`
diff --git a/docs/man-openmpi/man3/MPI_Win_allocate.3.rst b/docs/man-openmpi/man3/MPI_Win_allocate.3.rst
new file mode 100644
index 00000000000..091307fbbe2
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_allocate.3.rst
@@ -0,0 +1,86 @@
+.. _mpi_win_allocate:
+
+
+MPI_Win_allocate
+================
+
+.. include_body
+
+:ref:`MPI_Win_allocate` |mdash| One-sided MPI call that allocates memory and
+returns a window object for RMA operations.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_allocate.rst
+
+INPUT PARAMETERS
+----------------
+* ``size``: Size of window in bytes (nonnegative integer).
+* ``disp_unit``: Local unit size for displacements, in bytes (positive integer).
+* ``info``: Info argument (handle).
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``baseptr``: Initial address of window.
+* ``win``: Window object returned by the call (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Win_allocate` is a collective call executed by all processes
+in the group of *comm*. On each process, it allocates memory of at
+least *size* bytes, returns a pointer to it, and returns a window
+object that can be used by all processes in *comm* to perform RMA
+operations. The returned memory consists of *size* bytes local to each
+process, starting at address *baseptr* and is associated with the
+window as if the user called :ref:`MPI_Win_create` on existing
+memory. The *size* argument may be different at each process and
+*size* = 0 is valid; however, a library might allocate and expose more
+memory in order to create a fast, globally symmetric allocation. The
+discussion of and rationales for :ref:`MPI_Alloc_mem` and
+:ref:`MPI_Free_mem` in MPI-3.1 section 8.2 also apply to
+:ref:`MPI_Win_allocate`; in particular, see the rationale in MPI-3.1
+section 8.2 for an explanation of the type used for *baseptr*.
+
+The displacement unit argument is provided to facilitate address
+arithmetic in RMA operations: the target displacement argument of an RMA
+operation is scaled by the factor *disp_unit* specified by the target
+process, at window creation.
+
+For supported info keys see :ref:`MPI_Win_create`.
+
+
+NOTES
+-----
+
+Common choices for *disp_unit are 1 (no scaling), and (in C* syntax)
+*sizeof(type), for a window that consists of an array of* elements of
+type *type. The later choice will allow one to use* array indices in RMA
+calls, and have those scaled correctly to byte displacements, even in a
+heterogeneous environment.
+
+Calling :ref:`MPI_Win_free` will deallocate the memory allocated
+by :ref:`MPI_Win_allocate`. It is thus erroneous to manually free
+baseptr.
+
+
+C NOTES
+-------
+
+While *baseptr* is a ``void *`` type, this is to allow easy use of any
+pointer object for this parameter. This argument is really a ``void **``
+type.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Alloc_mem`
+   * :ref:`MPI_Free_mem`
+   * :ref:`MPI_Win_create`
+   * :ref:`MPI_Win_allocate_shared`
+   * :ref:`MPI_Win_free`
diff --git a/docs/man-openmpi/man3/MPI_Win_allocate_shared.3.rst b/docs/man-openmpi/man3/MPI_Win_allocate_shared.3.rst
new file mode 100644
index 00000000000..9489d463074
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_allocate_shared.3.rst
@@ -0,0 +1,107 @@
+.. _mpi_win_allocate_shared:
+
+
+MPI_Win_allocate_shared
+=======================
+
+.. include_body
+
+:ref:`MPI_Win_allocate_shared` |mdash| One-sided MPI call that allocates shared
+memory and returns a window object for RMA operations.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_allocate_shared.rst
+
+INPUT PARAMETERS
+----------------
+* ``size``: Size of window in bytes (nonnegative integer).
+* ``disp_unit``: Local unit size for displacements, in bytes (positive integer).
+* ``info``: Info argument (handle).
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``baseptr``: Initial address of window.
+* ``win``: Window object returned by the call (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Win_allocate_shared` is a collective call executed by all
+processes in the group of *comm*. On each process, it allocates memory
+of at least *size* bytes that is shared among all processes in *comm*,
+and returns a pointer to the locally allocated segment in *baseptr*
+that can be used for load/store accesses on the calling process. The
+locally allocated memory can be the target of load/store accesses by
+remote processes; the base pointers for other processes can be queried
+using the function :ref:`MPI_Win_shared_query`. The call also returns
+a window object that can be used by all processes in *comm* to perform
+RMA operations. The *size* argument may be different at each process
+and *size* = 0 is valid. It is the user's responsibility to ensure
+that the communicator *comm* represents a group of processes that can
+create a shared memory segment that can be accessed by all processes
+in the group. The discussions of rationales for :ref:`MPI_Alloc_mem`
+and :ref:`MPI_Free_mem` in MPI-3.1 section 8.2 also apply to
+:ref:`MPI_Win_allocate_shared`; in particular, see the rationale in
+MPI-3.1 section 8.2 for an explanation of the type used for
+*baseptr*. The allocated memory is contiguous across process ranks
+unless the info key *alloc_shared_noncontig* is specified. Contiguous
+across process ranks means that the first address in the memory
+segment of process i is consecutive with the last address in the
+memory segment of process i -
+1. This may enable the user to calculate remote address offsets with
+local information only.
+
+The following info keys are supported:
+
+alloc_shared_noncontig
+   If not set to *true*, the allocation strategy is to allocate
+   contiguous memory across process ranks. This may limit the
+   performance on some architectures because it does not allow the
+   implementation to modify the data layout (e.g., padding to reduce
+   access latency).
+
+blocking_fence
+   If set to *true*, the osc/sm component will use :ref:`MPI_Barrier` for
+   :ref:`MPI_Win_fence`. If set to *false* a condition variable and counter
+   will be used instead. The default value is *false*. This info key is
+   Open MPI specific.
+
+For additional supported info keys see :ref:`MPI_Win_create`.
+
+
+NOTES
+-----
+
+Common choices for *disp_unit* are 1 (no scaling), and (in C syntax)
+*sizeof(type)*, for a window that consists of an array of elements of
+type *type*. The later choice will allow one to use array indices in RMA
+calls, and have those scaled correctly to byte displacements, even in a
+heterogeneous environment.
+
+Calling :ref:`MPI_Win_free` will deallocate the memory allocated by
+:ref:`MPI_Win_allocate_shared`. It is thus erroneous to manually free
+*baseptr*.
+
+
+C NOTES
+-------
+
+While *baseptr* is a ``void *`` type, this is to allow easy use of any
+pointer object for this parameter. This argument is really a ``void **``
+type.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Alloc_mem`
+   * :ref:`MPI_Free_mem`
+   * :ref:`MPI_Win_allocate`
+   * :ref:`MPI_Win_create`
+   * :ref:`MPI_Win_shared_query`
+   * :ref:`MPI_Win_free`
diff --git a/docs/man-openmpi/man3/MPI_Win_attach.3.rst b/docs/man-openmpi/man3/MPI_Win_attach.3.rst
new file mode 100644
index 00000000000..2415a4f0b1e
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_attach.3.rst
@@ -0,0 +1,60 @@
+.. _mpi_win_attach:
+
+
+MPI_Win_attach
+==============
+
+.. include_body
+
+:ref:`MPI_Win_attach`, :ref:`MPI_Win_detach` - One-sided MPI call that attaches /
+detaches a memory region to / from a window object for RMA operations.
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Win_attach, MPI_Win_detach
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_attach.rst
+
+INPUT PARAMETERS
+----------------
+* ``win``: A window that was created with *MPI_Win_create_dynamic*
+* ``base``: Initial address of window (choice).
+* ``size``: Size of window in bytes (nonnegative integer).
+
+OUTPUT PARAMETERS
+-----------------
+* ``win``: Window object returned by the call (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Win_attach` is a one-sided MPI communication call used to attach a
+memory region of *size* bytes starting at address *base* to a window for
+RMA access. The window *win* must have been created using
+:ref:`MPI_Win_create_dynamic`. Multiple non-overlapping memory regions may be
+attached to the same dynamic window. Attaching overlapping memory
+regions to the same dynamic window is erroneous.
+
+If the *base* value used by :ref:`MPI_Win_attach` was allocated by
+:ref:`MPI_Alloc_mem`, the size of the window can be no larger than the value
+set by the :ref:`MPI_ALLOC_MEM` function.
+
+:ref:`MPI_Win_detach` can be used to detach a previously attached memory region
+from *win*. The memory address *base* and *win* must match arguments
+passed to a previous call to :ref:`MPI_Win_attach`.
+
+
+NOTES
+-----
+
+Use memory allocated by :ref:`MPI_Alloc_mem` to guarantee properly aligned
+window boundaries (such as word, double-word, cache line, page frame,
+and so on).
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Win_c2f.3.rst b/docs/man-openmpi/man3/MPI_Win_c2f.3.rst
new file mode 100644
index 00000000000..684ccdec1c8
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_c2f.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_win_c2f:
+
+MPI_Win_c2f
+===========
+
+.. include_body
+
+.. include:: ../man3/MPI_Comm_f2c.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_Win_call_errhandler.3.rst b/docs/man-openmpi/man3/MPI_Win_call_errhandler.3.rst
new file mode 100644
index 00000000000..9fd1841fbb9
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_call_errhandler.3.rst
@@ -0,0 +1,49 @@
+.. _mpi_win_call_errhandler:
+
+
+MPI_Win_call_errhandler
+=======================
+
+.. include_body
+
+:ref:`MPI_Win_call_errhandler` |mdash| Passes the supplied error code to the
+error handler assigned to a window
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_call_errhandler.rst
+
+INPUT PARAMETERS
+----------------
+* ``win``: Window with error handler (handle).
+* ``errorcode``: MPI error code (integer).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+This function invokes the error handler assigned to the window *win*
+with the supplied error code *errorcode*. If the error handler was
+successfully called, the process is not aborted, and the error handler
+returns, this function returns MPI_SUCCESS.
+
+
+NOTES
+-----
+
+Users should note that the default error handler is
+MPI_ERRORS_ARE_FATAL. Thus, calling this function will abort the window
+processes if the default error handler has not been changed for this
+window.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Win_create_errhandler`
+   * :ref:`MPI_Win_set_errhandler`
diff --git a/docs/man-openmpi/man3/MPI_Win_complete.3.rst b/docs/man-openmpi/man3/MPI_Win_complete.3.rst
new file mode 100644
index 00000000000..5e0ec5b1806
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_complete.3.rst
@@ -0,0 +1,39 @@
+.. _mpi_win_complete:
+
+
+MPI_Win_complete
+================
+
+.. include_body
+
+:ref:`MPI_Win_complete` |mdash| Completes an RMA access epoch on *win* started by
+a call to :ref:`MPI_Win_start`
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_complete.rst
+
+INPUT PARAMETERS
+----------------
+* ``win``: Window object (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Win_complete` is a one-sided MPI communication synchronization call,
+completing an RMA access epoch on *win* started by a call to
+:ref:`MPI_Win_start`. :ref:`MPI_Win_complete` enforces the completion of preceding RMA
+calls at the origin and not at the target. A put or accumulate call may
+not have completed at the target when it has completed at the origin.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Win_start`
diff --git a/docs/man-openmpi/man3/MPI_Win_create.3.rst b/docs/man-openmpi/man3/MPI_Win_create.3.rst
new file mode 100644
index 00000000000..cba1c33ad4c
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_create.3.rst
@@ -0,0 +1,108 @@
+.. _mpi_win_create:
+
+
+MPI_Win_create
+==============
+
+.. include_body
+
+:ref:`MPI_Win_create` |mdash| One-sided MPI call that returns a window object for
+RMA operations.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_create.rst
+
+INPUT PARAMETERS
+----------------
+* ``base``: Initial address of window (choice).
+* ``size``: Size of window in bytes (nonnegative integer).
+* ``disp_unit``: Local unit size for displacements, in bytes (positive integer).
+* ``info``: Info argument (handle).
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``win``: Window object returned by the call (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Win_create` is a one-sided MPI communication collective call executed
+by all processes in the group of *comm*. It returns a window object that
+can be used by these processes to perform RMA operations. Each process
+specifies a window of existing memory that it exposes to RMA accesses by
+the processes in the group of *comm*. The window consists of *size*
+bytes, starting at address *base*. A process may elect to expose no
+memory by specifying *size* = 0.
+
+If the *base* value used by :ref:`MPI_Win_create` was allocated by
+:ref:`MPI_Alloc_mem`, the size of the window can be no larger than the value
+set by the :ref:`MPI_ALLOC_MEM` function.
+
+The displacement unit argument is provided to facilitate address
+arithmetic in RMA operations: the target displacement argument of an RMA
+operation is scaled by the factor *disp_unit* specified by the target
+process, at window creation.
+
+The following info keys are supported:
+
+no_locks
+   If set to *true*, then the implementation may assume that the local
+   window is never locked (by a call to :ref:`MPI_Win_lock` or
+   MPI_Win_lock_all). Setting this value if only active synchronization
+   may allow the implementation to enable certain optimizations.
+
+accumulate_ordering
+   By default, accumulate operations from one initiator to one target on
+   the same window memory location are strictly ordered. If the info key
+   accumulate_ordering is set to *none*, no ordering of accumulate
+   operations guaranteed. They key can also be a comma-separated list of
+   required orderings consisting of *rar*, *war*, *raw*, and *waw* for
+   read-after-read, write-after-read, read-after-write, and
+   write-after-write, respectively. Looser ordering constraints are
+   likely to result in improved performance.
+
+accumulate_ops
+   If set to *same_op*, the implementation will assume that all
+   concurrent accumulate calls to the same target address will use the
+   same operation. If set to *same_op_no_op*, then the implementation
+   will assume that all concurrent accumulate calls to the same target
+   address will use the same operation or MPI_NO_OP. The default is
+   *same_op_no_op*.
+
+same_size
+   If set to *true*, then the implementation may assume that the
+   argument *size* is identical on all processes, and that all processes
+   have provided this info key with the same value.
+
+same_disp_unit
+   If set to *true*, then the implementation may assume that the
+   argument *disp_unit* is identical on all processes, and that all
+   processes have provided this info key with the same value.
+
+
+NOTES
+-----
+
+Common choices for *disp_unit* are 1 (no scaling), and (in C syntax)
+*sizeof(type)*, for a window that consists of an array of elements of
+type *type*. The later choice will allow one to use array indices in RMA
+calls, and have those scaled correctly to byte displacements, even in a
+heterogeneous environment.
+
+Use memory allocated by :ref:`MPI_Alloc_mem` to guarantee properly aligned
+window boundaries (such as word, double-word, cache line, page frame,
+and so on).
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Alloc_mem`
+   * :ref:`MPI_Free_mem`
+   * :ref:`MPI_Win_allocate`
+   * :ref:`MPI_Win_allocate_shared`
diff --git a/docs/man-openmpi/man3/MPI_Win_create_dynamic.3.rst b/docs/man-openmpi/man3/MPI_Win_create_dynamic.3.rst
new file mode 100644
index 00000000000..3a09dcc0796
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_create_dynamic.3.rst
@@ -0,0 +1,83 @@
+.. _mpi_win_create_dynamic:
+
+
+MPI_Win_create_dynamic
+======================
+
+.. include_body
+
+:ref:`MPI_Win_create_dynamic` |mdash| One-sided MPI call that returns a window
+object for RMA operations.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_create_dynamic.rst
+
+INPUT PARAMETERS
+----------------
+* ``info``: Info argument (handle).
+* ``comm``: Communicator (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``win``: Window object returned by the call (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Win_create_dynamic` is a one-sided MPI communication collective call
+executed by all processes in the group of *comm*. It returns a window
+object without memory attached that can be used by these processes to
+perform RMA operations.
+
+A window created with :ref:`MPI_Win_create_dynamic` requires the
+*target_disp* argument for all RMA communication functions to be the
+actual address at the target.
+
+The following info keys are supported:
+
+no_locks
+   If set to *true*, then the implementation may assume that the local
+   window is never locked (by a call to :ref:`MPI_Win_lock` or
+   MPI_Win_lock_all). Setting this value if only active synchronization
+   may allow the implementation to enable certain optimizations.
+
+accumulate_ordering
+   By default, accumulate operations from one initiator to one target on
+   the same window memory location are strictly ordered. If the info key
+   accumulate_ordering is set to *none*, no ordering of accumulate
+   operations guaranteed. They key can also be a comma-separated list of
+   required orderings consisting of *rar*, *war*, *raw*, and *waw* for
+   read-after-read, write-after-read, read-after-write, and
+   write-after-write, respectively. Looser ordering constraints are
+   likely to result in improved performance.
+
+accumulate_ops
+   If set to *same_op*, the implementation will assume that all
+   concurrent accumulate calls to the same target address will use the
+   same operation. If set to *same_op_no_op*, then the implementation
+   will assume that all concurrent accumulate calls to the same target
+   address will use the same operation or MPI_NO_OP. The default is
+   *same_op_no_op*.
+
+
+NOTES
+-----
+
+Since dynamically attaching memory to a window is a local operation, one
+has to communicate the actual address at the target using
+:ref:`MPI_Get_address` and some communication.
+
+Dynamic memory does not have any *disp_unit* associated and requires
+correct offset calculations with proper type handling.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Win_attach`
+   * :ref:`MPI_Win_detach`
+   * :ref:`MPI_Get_address`
diff --git a/docs/man-openmpi/man3/MPI_Win_create_errhandler.3.rst b/docs/man-openmpi/man3/MPI_Win_create_errhandler.3.rst
new file mode 100644
index 00000000000..eccf86a5043
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_create_errhandler.3.rst
@@ -0,0 +1,57 @@
+.. _mpi_win_create_errhandler:
+
+
+MPI_Win_create_errhandler
+=========================
+
+.. include_body
+
+:ref:`MPI_Win_create_errhandler` |mdash| Creates an error handler for a window.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_create_errhandler.rst
+
+DEPRECATED TYPE NAME NOTE
+-------------------------
+
+MPI-2.2 deprecated the ``MPI_Win_errhandler_fn`` type in favor of
+``MPI_Win_errhandler_function``. Open MPI supports both
+names (indeed, the ``_fn`` names are typedefs to the ``_function`` names).
+
+
+INPUT PARAMETER
+---------------
+
+* ``function``: User-defined error-handling procedure (function).
+
+OUTPUT PARAMETERS
+-----------------
+
+* ``errhandler``: MPI error handler (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Win_create_errhandler` should be, in C, a function of type
+``MPI_Win_errhandler_function``, which is defined as
+
+.. code-block:: c
+
+   typedef void MPI_Win_errhandler_function(MPI Win *, int *, ...);
+
+The first argument is the window in use, the second is the error code to
+be returned.
+
+In Fortran, the user routine should be of the form:
+
+.. code-block:: fortran
+
+   SUBROUTINE WIN_ERRHANDLER_FUNCTION(WIN, ERROR_CODE, ...)
+       INTEGER WIN, ERROR_CODE
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Win_create_keyval.3.rst b/docs/man-openmpi/man3/MPI_Win_create_keyval.3.rst
new file mode 100644
index 00000000000..e1c524f0131
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_create_keyval.3.rst
@@ -0,0 +1,70 @@
+.. _mpi_win_create_keyval:
+
+
+MPI_Win_create_keyval
+=====================
+
+.. include_body
+
+:ref:`MPI_Win_create_keyval` |mdash| Creates a keyval for a window.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_create_keyval.rst
+
+INPUT PARAMETERS
+----------------
+* ``win_copy_attr_fn``: Copy callback function for *win_keyval* (function).
+* ``win_delete_attr_fn``: Delete callback function for *win_keyval* (function).
+* ``extra_state``: Extra state for callback functions.
+
+OUTPUT PARAMETERS
+-----------------
+* ``win_keyval``: Key value for future access (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+The argument *win_copy_attr_fn* may be specified as MPI_WIN_NULL_COPY_FN
+or MPI_WIN_DUP_FN from either C or Fortran. MPI_WIN_NULL_COPY_FN is a
+function that serves only to return *flag* = 0 and MPI_SUCCESS.
+MPI_WIN_DUP_FN is a simple-minded copy function that sets *flag* = 1,
+returns the value of *attribute_val_in* in *attribute_val_out*, and
+returns MPI_SUCCESS.
+
+The argument *win_delete_attr_fn* may be specified as
+MPI_WIN_NULL_DELETE_FN from either C or Fortran. MPI_WIN_NULL_DELETE_FN
+is a function that serves only to return MPI_SUCCESS.
+
+The C callback functions are:
+
+.. code-block:: c
+
+   typedef int MPI_Win_copy_attr_function(MPI_Win oldwin, int win_keyval,
+                void *extra_state, void *attribute_val_in,
+                void *attribute_val_out, int *flag);
+
+   typedef int MPI_Win_delete_attr_function(MPI_Win win, int win_keyval,
+                void *attribute_val, void *extra_state);
+
+The Fortran callback functions are:
+
+.. code-block:: fortran
+
+   SUBROUTINE WIN_COPY_ATTR_FN(OLDWIN, WIN_KEYVAL, EXTRA_STATE,
+   	ATTRIBUTE_VAL_IN, ATTRIBUTE_VAL_OUT, FLAG, IERROR)
+   	INTEGER OLDWIN, WIN_KEYVAL, IERROR
+   	INTEGER(KIND=MPI_ADDRESS_KIND) EXTRA_STATE, ATTRIBUTE_VAL_IN,
+   		ATTRIBUTE_VAL_OUT
+   	LOGICAL FLAG
+
+   SUBROUTINE WIN_DELETE_ATTR_FN(WIN, WIN_KEYVAL, ATTRIBUTE_VAL,
+   	EXTRA_STATE, IERROR)
+   	INTEGER WIN, WIN_KEYVAL, IERROR
+   	INTEGER(KIND=MPI_ADDRESS_KIND) ATTRIBUTE_VAL, EXTRA_STATE
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Win_delete_attr.3.rst b/docs/man-openmpi/man3/MPI_Win_delete_attr.3.rst
new file mode 100644
index 00000000000..ab1379eef7c
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_delete_attr.3.rst
@@ -0,0 +1,39 @@
+.. _mpi_win_delete_attr:
+
+
+MPI_Win_delete_attr
+===================
+
+.. include_body
+
+:ref:`MPI_Win_delete_attr` |mdash| Deletes an attribute from a window.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_delete_attr.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``win``: Window from which the attribute is deleted (handle).
+
+INPUT PARAMETER
+---------------
+* ``win_keyval``: Key value (integer).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+NOTES
+-----
+
+Note that it is not defined by the MPI standard what happens if the
+delete_fn callback invokes other MPI functions. In Open MPI, it is not
+valid for delete_fn callbacks (or any of their children) to add or
+delete attributes on the same object on which the delete_fn callback is
+being invoked.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Win_detach.3.rst b/docs/man-openmpi/man3/MPI_Win_detach.3.rst
new file mode 100644
index 00000000000..0e2ab5d031f
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_detach.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_win_detach:
+
+MPI_Win_detach
+==============
+
+.. include_body
+
+.. include:: ../man3/MPI_Win_attach.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_Win_f2c.3.rst b/docs/man-openmpi/man3/MPI_Win_f2c.3.rst
new file mode 100644
index 00000000000..7faeab2773f
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_f2c.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_win_f2c:
+
+MPI_Win_f2c
+===========
+
+.. include_body
+
+.. include:: ../man3/MPI_Comm_f2c.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_Win_fence.3.rst b/docs/man-openmpi/man3/MPI_Win_fence.3.rst
new file mode 100644
index 00000000000..c37aa7433d8
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_fence.3.rst
@@ -0,0 +1,88 @@
+.. _mpi_win_fence:
+
+
+MPI_Win_fence
+=============
+
+.. include_body
+
+:ref:`MPI_Win_fence` |mdash| Synchronizes RMA calls on a window.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_fence.rst
+
+INPUT PARAMETERS
+----------------
+* ``assert``: Program assertion (integer).
+* ``win``: Window object (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Win_fence` synchronizes RMA calls on *win*. The call is collective on
+the group of *win*. All RMA operations on *win* originating at a given
+process and started before the fence call will complete at that process
+before the fence call returns. They will be completed at their target
+before the fence call returns at the target. RMA operations on *win*
+started by a process after the fence call returns will access their
+target window only after :ref:`MPI_Win_fence` has been called by the target
+process.
+
+The call completes an RMA access epoch if it was preceded by another
+fence call and the local process issued RMA communication calls on *win*
+between these two calls. The call completes an RMA exposure epoch if it
+was preceded by another fence call and the local window was the target
+of RMA accesses between these two calls. The call starts an RMA access
+epoch if it is followed by another fence call and by RMA communication
+calls issued between these two fence calls. The call starts an exposure
+epoch if it is followed by another fence call and the local window is
+the target of RMA accesses between these two fence calls. Thus, the
+fence call is equivalent to calls to a subset of *post, start, complete,
+wait*.
+
+The *assert* argument is used to provide assertions on the context of
+
+the call that may be used for various optimizations. A value of *assert*
+^ 0 is always valid. The following assertion value is supported:
+
+MPI_MODE_NOPRECEDE
+   No local RMA calls have been issued before this fence. This assertion
+   must be provided by all or no members of the group of the window. It
+   may enable faster fence call by avoiding unnecessary synchronization.
+
+MPI_MODE_NOSTORE
+   Informs that the local window was not updated by local stores or get
+   calls in the preceding epoch.
+
+MPI_MODE_NOPUT
+   Informs that the local window will not be updated by any put or
+   accummulate calls in the ensuing epoch (until next fence call).
+
+MPI_MODE_NOSUCCEED
+   No local RMA calls will be issued after this fence. This assertion
+   must be provided by all or no members of the group of the window. It
+   may enable faster fence call by avoiding unnecessary synchronization.
+
+
+NOTE
+----
+
+Calls to :ref:`MPI_Win_fence` should both precede and follow calls to put, get
+or accumulate that are synchronized with fence calls.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Win_create`
+   * :ref:`MPI_Win_start`
+   * :ref:`MPI_Win_post`
+   * :ref:`MPI_Win_complete`
+   * :ref:`MPI_Win_wait`
diff --git a/docs/man-openmpi/man3/MPI_Win_flush.3.rst b/docs/man-openmpi/man3/MPI_Win_flush.3.rst
new file mode 100644
index 00000000000..211a75933fc
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_flush.3.rst
@@ -0,0 +1,48 @@
+.. _mpi_win_flush:
+
+
+MPI_Win_flush
+=============
+
+.. include_body
+
+:ref:`MPI_Win_flush`, :ref:`MPI_Win_flush_all` - Complete all outstanding RMA
+operations at both the origin and the target
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Win_flush, MPI_Win_flush_all
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_flush.rst
+
+INPUT PARAMETERS
+----------------
+* ``rank``: Rank of window (nonnegative integer).
+* ``win``: Window object (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Win_flush` completes all outstanding RMA operations initiated by
+the calling process to the target rank on the specified window. The
+operations are completed both at the origin and at the target.
+:ref:`MPI_Win_flush_all` completes all outstanding RMA operations to all
+targets.
+
+Can only be called from within a passive target epoch.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Win_flush_local`
+   * :ref:`MPI_Win_lock`
+   * :ref:`MPI_Win_lock_all`
diff --git a/docs/man-openmpi/man3/MPI_Win_flush_all.3.rst b/docs/man-openmpi/man3/MPI_Win_flush_all.3.rst
new file mode 100644
index 00000000000..693cc775f64
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_flush_all.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_win_flush_all:
+
+MPI_Win_flush_all
+=================
+
+.. include_body
+
+.. include:: ../man3/MPI_Win_flush.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_Win_flush_local.3.rst b/docs/man-openmpi/man3/MPI_Win_flush_local.3.rst
new file mode 100644
index 00000000000..4740cdd0858
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_flush_local.3.rst
@@ -0,0 +1,49 @@
+.. _mpi_win_flush_local:
+
+
+MPI_Win_flush_local
+===================
+
+.. include_body
+
+:ref:`MPI_Win_flush_local`, :ref:`MPI_Win_flush_local_all` - Complete all
+outstanding RMA operations at both the origin
+
+.. The following directive tells the man page generation script to
+   generate multiple bindings for this file.
+.. mpi-bindings: MPI_Win_flush_local, MPI_Win_flush_local_all
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_flush_local.rst
+
+INPUT PARAMETERS
+----------------
+* ``rank``: Rank of window (nonnegative integer).
+* ``win``: Window object (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Win_flush_local` locally completes at the origin all outstanding
+RMA operations initiated by the calling process to the target process
+specified by rank on the specified window. For example, after this
+routine completes, the user may reuse any buffers provided to put, get,
+or accumulate operations. :ref:`MPI_Win_flush_local_all` locally completes
+at the origin all outstanding RMA operations to all targets.
+
+Can only be called from within a passive target epoch.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Win_flush`
+   * :ref:`MPI_Win_lock`
+   * :ref:`MPI_Win_lock_all`
diff --git a/docs/man-openmpi/man3/MPI_Win_flush_local_all.3.rst b/docs/man-openmpi/man3/MPI_Win_flush_local_all.3.rst
new file mode 100644
index 00000000000..a7841da21d2
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_flush_local_all.3.rst
@@ -0,0 +1,9 @@
+.. _mpi_win_flush_local_all:
+
+MPI_Win_flush_local_all
+=======================
+
+.. include_body
+
+.. include:: ../man3/MPI_Win_flush_local.3.rst
+    :start-after: .. include_body
diff --git a/docs/man-openmpi/man3/MPI_Win_free.3.rst b/docs/man-openmpi/man3/MPI_Win_free.3.rst
new file mode 100644
index 00000000000..ac35718d51e
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_free.3.rst
@@ -0,0 +1,50 @@
+.. _mpi_win_free:
+
+
+MPI_Win_free
+============
+
+.. include_body
+
+:ref:`MPI_Win_free` |mdash| Frees the window object and returns a null handle.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_free.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``win``: Window object (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Win_free` frees the window object *win* and returns a null handle
+(equal to MPI_WIN_NULL). This collective call is executed by all
+processes in the group associated with *win*. It can be invoked by a
+process only after it has completed its involvement in RMA
+communications on window *win*, that is, the process has called
+:ref:`MPI_Win_fence`, or called :ref:`MPI_Win_unlock` to match a previous call to
+:ref:`MPI_Win_lock`. When the call returns, the window memory can be freed.
+
+
+NOTES
+-----
+
+If the window was created through :ref:`MPI_Win_allocate` or
+:ref:`MPI_Win_allocate_shared` then the memory buffer allocated in that
+call will be freed when calling :ref:`MPI_Win_free`.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Win_create`
+   * :ref:`MPI_Win_allocate`
+   * :ref:`MPI_Win_allocate_shared`
diff --git a/docs/man-openmpi/man3/MPI_Win_free_keyval.3.rst b/docs/man-openmpi/man3/MPI_Win_free_keyval.3.rst
new file mode 100644
index 00000000000..42ae9b8f57c
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_free_keyval.3.rst
@@ -0,0 +1,25 @@
+.. _mpi_win_free_keyval:
+
+
+MPI_Win_free_keyval
+===================
+
+.. include_body
+
+:ref:`MPI_Win_free_keyval` |mdash| Frees a window keyval.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_free_keyval.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``win_keyval``: Key value (integer).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Win_get_attr.3.rst b/docs/man-openmpi/man3/MPI_Win_get_attr.3.rst
new file mode 100644
index 00000000000..5871cd4288c
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_get_attr.3.rst
@@ -0,0 +1,35 @@
+.. _mpi_win_get_attr:
+
+
+MPI_Win_get_attr
+================
+
+.. include_body
+
+:ref:`MPI_Win_get_attr` |mdash| Obtains the value of a window attribute.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_get_attr.rst
+
+INPUT PARAMETERS
+----------------
+* ``win``: Window to which the attribute is attached (handle).
+* ``win_keyval``: Key value (integer).
+
+OUTPUT PARAMETERS
+-----------------
+* ``attribute_val``: Attribute value, unless *ag* = false
+* ``flag``: False if no attribute is associated with the key (logical).
+* ``ierror``: Fortran only: Error status (integer).
+
+
+DESCRIPTION
+-----------
+
+Obtains the value of a window attribute.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Win_get_errhandler.3.rst b/docs/man-openmpi/man3/MPI_Win_get_errhandler.3.rst
new file mode 100644
index 00000000000..2480258f22a
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_get_errhandler.3.rst
@@ -0,0 +1,34 @@
+.. _mpi_win_get_errhandler:
+
+
+MPI_Win_get_errhandler
+======================
+
+.. include_body
+
+:ref:`MPI_Win_get_errhandler` |mdash| Retrieves the error handler currently
+associated with a window.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_get_errhandler.rst
+
+INPUT PARAMETER
+---------------
+* ``win``: Window (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``errhandler``: Error handler currently associated with window (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Win_get_errhandler` retrieves the error handler currently associated
+with a window.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Win_get_group.3.rst b/docs/man-openmpi/man3/MPI_Win_get_group.3.rst
new file mode 100644
index 00000000000..bba41bbd3b7
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_get_group.3.rst
@@ -0,0 +1,35 @@
+.. _mpi_win_get_group:
+
+
+MPI_Win_get_group
+=================
+
+.. include_body
+
+:ref:`MPI_Win_get_group` |mdash| Returns a duplicate of the group of the
+communicator used to create the window.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_get_group.rst
+
+INPUT PARAMETER
+---------------
+* ``win``: Window object (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``group``: Group of processes that share access to the window (handle).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Win_get_group` returns a duplicate of the group of the communicator
+used to create the window associated with *win*. The group is returned
+in *group*.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Win_get_info.3.rst b/docs/man-openmpi/man3/MPI_Win_get_info.3.rst
new file mode 100644
index 00000000000..b63e6f233e8
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_get_info.3.rst
@@ -0,0 +1,41 @@
+.. _mpi_win_get_info:
+
+
+MPI_Win_get_info
+================
+
+.. include_body
+
+:ref:`MPI_Win_get_info` |mdash| Retrieves active window info hints
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_get_info.rst
+
+INPUT PARAMETERS
+----------------
+* ``win``: Window from which to receive active info hints
+
+OUTPUT PARAMETERS
+-----------------
+* ``info_used``: New info object returned with all active hints on this window.
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Win_get_info` returns a new info object containing the hints of the
+window associated with *win*. The current setting of all hints actually
+used by the system related to this window is returned in *info_used*. If
+no such hints exist, a handle to a newly created info object is returned
+that contains no key/value pair. The user is responsible for freeing
+info_used via :ref:`MPI_Info_free`.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Win_set_info`
+   * :ref:`MPI_Win_free`
diff --git a/docs/man-openmpi/man3/MPI_Win_get_name.3.rst b/docs/man-openmpi/man3/MPI_Win_get_name.3.rst
new file mode 100644
index 00000000000..b6dff5a6db3
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_get_name.3.rst
@@ -0,0 +1,31 @@
+.. _mpi_win_get_name:
+
+
+MPI_Win_get_name
+================
+
+.. include_body
+
+:ref:`MPI_Win_get_name` |mdash| Obtains the name of a window.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_get_name.rst
+
+INPUT PARAMETER
+---------------
+* ``win``: Window whose name is to be returned (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``win_name``: the name previously stored on the window, or an empty string if no such name exists (string).
+* ``resultlen``: Length of returned name (integer).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Win_lock.3.rst b/docs/man-openmpi/man3/MPI_Win_lock.3.rst
new file mode 100644
index 00000000000..8029c4b9f72
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_lock.3.rst
@@ -0,0 +1,69 @@
+.. _mpi_win_lock:
+
+
+MPI_Win_lock
+============
+
+.. include_body
+
+:ref:`MPI_Win_lock` |mdash| Starts an RMA access epoch locking access to a
+particular rank.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_lock.rst
+
+INPUT PARAMETERS
+----------------
+* ``lock_type``: Either MPI_LOCK_EXCLUSIVE or MPI_LOCK_SHARED (state).
+* ``rank``: Rank of locked window (nonnegative integer).
+* ``assert``: Program assertion (integer).
+* ``win``: Window object (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Starts an RMA access epoch. Locks ensure that only the windows created
+by specific processes can be accessed by those processes (and by no
+other processes) during that epoch.
+
+Locks are used to protect accesses to the locked target window effected
+by RMA calls issued between the lock and unlock call, and to protect
+local load/store accesses to a locked local window executed between the
+lock and unlock call. Accesses that are protected by an exclusive lock
+will not be concurrent at the window site with other accesses to the
+same window that are lock protected. Accesses that are protected by a
+shared lock will not be concurrent at the window site with accesses
+protected by an exclusive lock to the same window.
+
+The *assert* argument is used to provide assertions on the context of
+the call that may be used for various optimizations. (See Section 6.4.4
+of the MPI-2 Standard.) A value of *assert* = 0 is always valid. The
+following assertion value is supported:
+
+MPI_MODE_NOCHECK
+   No other processes will hold or attempt to acquire a conflicting lock
+   while the caller holds the window lock.
+
+
+NOTES
+-----
+
+In a client/server environment in which clients connect to a server and
+create windows that span both the client and the server, if a client or
+server that has obtained a lock on such a window and then terminates
+abnormally, the server or other clients may hang in a :ref:`MPI_Win_lock` call,
+failing to notice that the peer MPI job has terminated.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Win_unlock`
+   * :ref:`MPI_Win_lock_all`
diff --git a/docs/man-openmpi/man3/MPI_Win_lock_all.3.rst b/docs/man-openmpi/man3/MPI_Win_lock_all.3.rst
new file mode 100644
index 00000000000..4155e1c3429
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_lock_all.3.rst
@@ -0,0 +1,70 @@
+.. _mpi_win_lock_all:
+
+
+MPI_Win_lock_all
+================
+
+.. include_body
+
+:ref:`MPI_Win_lock_all` |mdash| Starts an RMA access epoch locking access to all
+processes in the window
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_lock_all.rst
+
+INPUT PARAMETERS
+----------------
+* ``assert``: Program assertion (integer).
+* ``win``: Window object (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Starts an RMA access epoch to all processes in *win*, with a lock type
+of MPI_LOCK_SHARED. During the epoch, the calling process can access the
+window memory on all processes in *win* by using RMA operations. A
+window locked with :ref:`MPI_Win_lock_all` must be unlocked with
+:ref:`MPI_Win_unlock_all`. This routine is not collective — the ALL refers to a
+lock on all members of the group of the window.
+
+Locks are used to protect accesses to the locked target window effected
+by RMA calls issued between the lock and unlock call, and to protect
+local load/store accesses to a locked local window executed between the
+lock and unlock call. Accesses that are protected by an exclusive lock
+will not be concurrent at the window site with other accesses to the
+same window that are lock protected. Accesses that are protected by a
+shared lock will not be concurrent at the window site with accesses
+protected by an exclusive lock to the same window.
+
+The *assert* argument is used to provide assertions on the context of
+the call that may be used for various optimizations. (See Section 6.4.4
+of the MPI-2 Standard.) A value of *assert* = 0 is always valid. The
+following assertion value is supported:
+
+MPI_MODE_NOCHECK
+   No other processes will hold or attempt to acquire a conflicting lock
+   while the caller holds the window lock.
+
+
+NOTES
+-----
+
+In a client/server environment in which clients connect to a server and
+create windows that span both the client and the server, if a client or
+server that has obtained a lock on such a window and then terminates
+abnormally, the server or other clients may hang in a :ref:`MPI_Win_lock_all`
+call, failing to notice that the peer MPI job has terminated.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Win_unlock_all`
+   * :ref:`MPI_Win_lock`
diff --git a/docs/man-openmpi/man3/MPI_Win_post.3.rst b/docs/man-openmpi/man3/MPI_Win_post.3.rst
new file mode 100644
index 00000000000..fbbf122588a
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_post.3.rst
@@ -0,0 +1,59 @@
+.. _mpi_win_post:
+
+
+MPI_Win_post
+============
+
+.. include_body
+
+:ref:`MPI_Win_post` |mdash| Starts an RMA exposure epoch for the local window
+associated with *win*
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_post.rst
+
+INPUT PARAMETERS
+----------------
+* ``group``: The group of origin processes (handle)
+* ``assert``: Program assertion (integer)
+* ``win``: Window object (handle)
+
+OUTPUT PARAMETERS
+-----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+Starts an RMA exposure epoch for the local window associated with *win*.
+Only the processes belonging to *group* should access the window with
+RMA calls on *win* during this epoch. Each process in *group* must issue
+a matching call to :ref:`MPI_Win_start`. :ref:`MPI_Win_post` does not block.
+
+The *assert* argument is used to provide assertions on the context of
+
+the call that may be used for various optimizations. A value of *assert*
+^ 0 is always valid. The following assertion values are supported:
+
+MPI_MODE_NOCHECK
+   The matching calls to :ref:`MPI_Win_start` have not yet occurred on any
+   origin processes when this call is made. This assertion must be
+   present for all matching :ref:`MPI_Win_start` calls if used.
+
+MPI_MODE_NOSTORE
+   Informs that the local window was not updated by local stores or get
+   calls in the preceding epoch.
+
+MPI_MODE_NOPUT
+   Informs that the local window will not be updated by put or
+   accummulate calls until the ensuing wait synchronization.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Win_start`
+   * :ref:`MPI_Win_wait`
diff --git a/docs/man-openmpi/man3/MPI_Win_set_attr.3.rst b/docs/man-openmpi/man3/MPI_Win_set_attr.3.rst
new file mode 100644
index 00000000000..11a92ca615b
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_set_attr.3.rst
@@ -0,0 +1,37 @@
+.. _mpi_win_set_attr:
+
+
+MPI_Win_set_attr
+================
+
+.. include_body
+
+:ref:`MPI_Win_set_attr` |mdash| Sets the value of a window attribute.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_set_attr.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``win``: Window to which attribute will be attached (handle).
+
+INPUT PARAMETERS
+----------------
+* ``win_keyval``: Key value (integer).
+* ``attribute_val``: Attribute value.
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+
+DESCRIPTION
+-----------
+
+Sets the value of a window attribute.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Win_set_errhandler.3.rst b/docs/man-openmpi/man3/MPI_Win_set_errhandler.3.rst
new file mode 100644
index 00000000000..0d8314766b2
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_set_errhandler.3.rst
@@ -0,0 +1,37 @@
+.. _mpi_win_set_errhandler:
+
+
+MPI_Win_set_errhandler
+======================
+
+.. include_body
+
+:ref:`MPI_Win_set_errhandler` |mdash| Attaches a new error handler to a window.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_set_errhandler.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``win``: Window (handle).
+
+INPUT PARAMETER
+---------------
+* ``errhandler``: New error handler for window (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Win_set_errhandler` attaches a new error handler to a window. The
+error handler must be either a predefined error handler or an error
+handler created by a call to :ref:`MPI_Win_create_errhandler`.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Win_set_info.3.rst b/docs/man-openmpi/man3/MPI_Win_set_info.3.rst
new file mode 100644
index 00000000000..69de8bf2894
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_set_info.3.rst
@@ -0,0 +1,42 @@
+.. _mpi_win_set_info:
+
+
+MPI_Win_set_info
+================
+
+.. include_body
+
+:ref:`MPI_Win_set_info` |mdash| Set window info hints
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_set_info.rst
+
+INPUT PARAMETERS
+----------------
+* ``win``: Window on which to set info hints
+* ``info``: Info object containing hints to be set on *win*
+
+OUTPUT PARAMETERS
+-----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_WIN_SET_INFO` sets new values for the hints of the window associated
+with *win.* :ref:`MPI_WIN_SET_INFO` is a collective routine. The info object
+may be different on each process, but any info entries that an
+implementation requires to be the same on all processes must appear with
+the same value in each process's *info* object.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Win_get_info`
+   * :ref:`MPI_Info_create`
+   * :ref:`MPI_Info_set`
+   * :ref:`MPI_Info_free`
diff --git a/docs/man-openmpi/man3/MPI_Win_set_name.3.rst b/docs/man-openmpi/man3/MPI_Win_set_name.3.rst
new file mode 100644
index 00000000000..644eb2e0678
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_set_name.3.rst
@@ -0,0 +1,33 @@
+.. _mpi_win_set_name:
+
+
+MPI_Win_set_name
+================
+
+.. include_body
+
+:ref:`MPI_Win_set_name` |mdash| Sets the name of a window.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_set_name.rst
+
+INPUT/OUTPUT PARAMETER
+----------------------
+* ``win``: Window whose identifier is to be set (handle).
+
+INPUT PARAMETER
+---------------
+* ``win_name``: The character string used as the name (string).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Win_shared_query.3.rst b/docs/man-openmpi/man3/MPI_Win_shared_query.3.rst
new file mode 100644
index 00000000000..94bd58a9ed3
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_shared_query.3.rst
@@ -0,0 +1,61 @@
+.. _mpi_win_shared_query:
+
+
+MPI_Win_shared_query
+====================
+
+.. include_body
+
+:ref:`MPI_Win_shared_query` |mdash| Query a shared memory window
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_shared_query.rst
+
+INPUT PARAMETERS
+----------------
+* ``win``: Shared memory window object (handle).
+* ``rank``: Rank in the group of window *win* (non-negative integer) or ``MPI_PROC_NULL``.
+
+OUTPUT PARAMETERS
+-----------------
+* ``size``: Size of the window segment (non-negative integer).
+* ``disp_unit``: Local unit size for displacements, in bytes (positive integer).
+* ``baseptr``: Address for load/store access to window segment (choice).
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Win_shared_query` queries the process-local address for
+remote memory segments created with
+:ref:`MPI_Win_allocate_shared`. This function can return different
+process-local addresses for the same physical memory on different
+processes. The returned memory can be used for load/store accesses
+subject to the constraints defined in MPI-3.1 section 11.7. This
+function can only be called with windows of flavor
+MPI_WIN_FLAVOR_SHARED. If the passed window is not of flavor
+MPI_WIN_FLAVOR_SHARED, the error MPI_ERR_RMA_FLAVOR is raised. When
+rank is ``MPI_PROC_NULL``, the *pointer*, *disp_unit*, and *size* returned
+are the pointer, disp_unit, and size of the memory segment belonging
+the lowest rank that specified *size* > 0. If all processes in the
+group attached to the window specified *size* = 0, then the call
+returns *size* = 0 and a *baseptr* as if :ref:`MPI_Alloc_mem` was
+called with *size* = 0.
+
+
+C NOTES
+-------
+
+The parameter *baseptr* is of type ``void *`` to allow passing any
+pointer object for this parameter. The provided argument should be a
+pointer to a pointer of arbitrary type (e.g. ``void **``).
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Alloc_mem`
+   * :ref:`MPI_Win_allocate_shared`
diff --git a/docs/man-openmpi/man3/MPI_Win_start.3.rst b/docs/man-openmpi/man3/MPI_Win_start.3.rst
new file mode 100644
index 00000000000..3ce60825ba9
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_start.3.rst
@@ -0,0 +1,52 @@
+.. _mpi_win_start:
+
+
+MPI_Win_start
+=============
+
+.. include_body
+
+:ref:`MPI_Win_start` |mdash| Starts an RMA access epoch for *win*
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_start.rst
+
+INPUT PARAMETERS
+----------------
+* ``group``: The group of target processes (handle).
+* ``assert``: Program assertion (integer).
+* ``win``: Window object (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Win_start` is a one-sided MPI communication synchronization call that
+starts an RMA access epoch for *win*. RMA calls issued on *win* during
+this epoch must access only windows at processes in *group*. Each
+process in *group* must issue a matching call to :ref:`MPI_Win_post`.
+:ref:`MPI_Win_start` is allowed to block until the corresponding :ref:`MPI_Win_post`
+calls have been executed, but is not required to.
+
+The *assert* argument is used to provide assertions on the context of
+the call that may be used for various optimizations. (See Section 6.4.4
+of the MPI-2 Standard.) A value of *assert* = 0 is always valid. The
+following assertion value is supported:
+
+MPI_MODE_NOCHECK
+   When this value is passed in to this call, the library assumes that
+   the post call on the target has been called and it is not necessary
+   for the library to check to see if such a call has been made.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Win_post`
+   * :ref:`MPI_Win_complete`
diff --git a/docs/man-openmpi/man3/MPI_Win_sync.3.rst b/docs/man-openmpi/man3/MPI_Win_sync.3.rst
new file mode 100644
index 00000000000..b5646b3c891
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_sync.3.rst
@@ -0,0 +1,36 @@
+.. _mpi_win_sync:
+
+
+MPI_Win_sync
+============
+
+.. include_body
+
+:ref:`MPI_Win_sync`, - Synchronize the private and public copies of the
+window
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_sync.rst
+
+INPUT PARAMETERS
+----------------
+* ``win``: Window object (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Win_sync` synchronizes the private and public window copies of
+*win*. For the purposes of synchronizing the private and public window,
+:ref:`MPI_Win_sync` has the effect of ending and reopening an access and
+exposure epoch on the window (note that it does not actually end an
+epoch or complete any pending MPI RMA operations).
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
diff --git a/docs/man-openmpi/man3/MPI_Win_test.3.rst b/docs/man-openmpi/man3/MPI_Win_test.3.rst
new file mode 100644
index 00000000000..6147a46ee60
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_test.3.rst
@@ -0,0 +1,46 @@
+.. _mpi_win_test:
+
+
+MPI_Win_test
+============
+
+.. include_body
+
+:ref:`MPI_Win_test` |mdash| Attempts to complete an RMA exposure epoch; a
+nonblocking version of :ref:`MPI_Win_wait`
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_test.rst
+
+INPUT PARAMETERS
+----------------
+* ``win``: Window object (handle)
+
+OUTPUT PARAMETERS
+-----------------
+* ``ierror``: Fortran only: Error status (integer).
+* ``flag``: The returning state of the test for epoch closure.
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Win_test` is a one-sided MPI communication synchronization call, a
+nonblocking version of :ref:`MPI_Win_wait`. It returns *flag = true* if
+:ref:`MPI_Win_wait` would return, *flag = false* otherwise. The effect of
+return of :ref:`MPI_Win_test` with *flag = true* is the same as the effect of a
+return of :ref:`MPI_Win_wait`. If *flag = false* is returned, then the call has
+no visible effect.
+
+Invoke :ref:`MPI_Win_test` only where :ref:`MPI_Win_wait` can be invoked. Once the
+call has returned *flag = true*, it must not be invoked anew, until the
+window is posted anew.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Win_post`
+   * :ref:`MPI_Win_wait`
diff --git a/docs/man-openmpi/man3/MPI_Win_unlock.3.rst b/docs/man-openmpi/man3/MPI_Win_unlock.3.rst
new file mode 100644
index 00000000000..e1f4333b004
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_unlock.3.rst
@@ -0,0 +1,48 @@
+.. _mpi_win_unlock:
+
+
+MPI_Win_unlock
+==============
+
+.. include_body
+
+:ref:`MPI_Win_unlock` |mdash| Completes an RMA access epoch started by a call to
+:ref:`MPI_Win_lock`.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_unlock.rst
+
+INPUT PARAMETERS
+----------------
+* ``rank``: Rank of window (nonnegative integer).
+* ``win``: Window object (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Win_unlock` completes an RMA access epoch started by a call to
+:ref:`MPI_Win_lock`. RMA operations issued during this period will have
+completed both at the origin and at the target when the call returns.
+
+Locks are used to protect accesses to the locked target window effected
+by RMA calls issued between the lock and unlock call, and to protect
+local load/store accesses to a locked local window executed between the
+lock and unlock call. Accesses that are protected by an exclusive lock
+will not be concurrent at the window site with other accesses to the
+same window that are lock protected. Accesses that are protected by a
+shared lock will not be concurrent at the window site with accesses
+protected by an exclusive lock to the same window.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Win_lock`
+   * :ref:`MPI_Win_unlock_all`
diff --git a/docs/man-openmpi/man3/MPI_Win_unlock_all.3.rst b/docs/man-openmpi/man3/MPI_Win_unlock_all.3.rst
new file mode 100644
index 00000000000..c8f13631a26
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_unlock_all.3.rst
@@ -0,0 +1,47 @@
+.. _mpi_win_unlock_all:
+
+
+MPI_Win_unlock_all
+==================
+
+.. include_body
+
+:ref:`MPI_Win_unlock_all` |mdash| Completes an RMA access epoch started by a call
+to :ref:`MPI_Win_lock_all`.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_unlock_all.rst
+
+INPUT PARAMETERS
+----------------
+* ``win``: Window object (handle).
+
+OUTPUT PARAMETER
+----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Win_unlock_all` completes an RMA access epoch started by a call to
+:ref:`MPI_Win_lock_all`. RMA operations issued during this period will have
+completed both at the origin and at the target when the call returns.
+
+Locks are used to protect accesses to the locked target window effected
+by RMA calls issued between the lock and unlock call, and to protect
+local load/store accesses to a locked local window executed between the
+lock and unlock call. Accesses that are protected by an exclusive lock
+will not be concurrent at the window site with other accesses to the
+same window that are lock protected. Accesses that are protected by a
+shared lock will not be concurrent at the window site with accesses
+protected by an exclusive lock to the same window.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Win_lock_all`
+   * :ref:`MPI_Win_unlock`
diff --git a/docs/man-openmpi/man3/MPI_Win_wait.3.rst b/docs/man-openmpi/man3/MPI_Win_wait.3.rst
new file mode 100644
index 00000000000..13a3b3fd7df
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Win_wait.3.rst
@@ -0,0 +1,43 @@
+.. _mpi_win_wait:
+
+
+MPI_Win_wait
+============
+
+.. include_body
+
+:ref:`MPI_Win_wait` |mdash| Completes an RMA exposure epoch started by a call to
+:ref:`MPI_Win_post` on *win*
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_win_wait.rst
+
+INPUT PARAMETERS
+----------------
+* ``win``: Window object (handle).
+
+OUTPUT PARAMETERS
+-----------------
+* ``ierror``: Fortran only: Error status (integer).
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Win_wait` is a one-sided MPI communication synchronization call that
+completes an RMA exposure epoch started by a call to :ref:`MPI_Win_post` on
+*win*. This call matches calls to MPI_Win_complete(win) issued by each
+of the processes that were granted access to the window during this
+epoch. The call to :ref:`MPI_Win_wait` blocks until all matching calls to
+:ref:`MPI_Win_complete` have occurred. This guarantees that all these origin
+processes have completed their RMA accesses to the local window. When
+the call returns, all these RMA accesses will have completed at the
+target window.
+
+
+ERRORS
+------
+
+.. include:: ./ERRORS.rst
+
+.. seealso::
+   * :ref:`MPI_Win_post`
diff --git a/docs/man-openmpi/man3/MPI_Wtick.3.rst b/docs/man-openmpi/man3/MPI_Wtick.3.rst
new file mode 100644
index 00000000000..7ef15a0aa10
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Wtick.3.rst
@@ -0,0 +1,38 @@
+.. _mpi_wtick:
+
+
+MPI_Wtick
+=========
+
+.. include_body
+
+:ref:`MPI_Wtick` |mdash| Returns the resolution of :ref:`MPI_Wtime`.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_wtick.rst
+
+RETURN VALUE
+------------
+
+Time in seconds of resolution of :ref:`MPI_Wtime`.
+
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Wtick` returns the resolution of :ref:`MPI_Wtime` in seconds. That is, it
+returns, as a double-precision value, the number of seconds between
+successive clock ticks. For example, if the clock is implemented by the
+hardware as a counter that is incremented every millisecond, the value
+returned by :ref:`MPI_Wtick` should be 10^-3.
+
+
+NOTE
+----
+
+This function does not return an error value. Consequently, the result
+of calling it before :ref:`MPI_Init` or after :ref:`MPI_Finalize` is undefined.
+
+
+.. seealso::
+   * :ref:`MPI_Wtime`
diff --git a/docs/man-openmpi/man3/MPI_Wtime.3.rst b/docs/man-openmpi/man3/MPI_Wtime.3.rst
new file mode 100644
index 00000000000..9baeef4c46d
--- /dev/null
+++ b/docs/man-openmpi/man3/MPI_Wtime.3.rst
@@ -0,0 +1,84 @@
+.. _mpi_wtime:
+
+
+MPI_Wtime
+=========
+
+.. include_body
+
+:ref:`MPI_Wtime` |mdash| Returns an elapsed time on the calling processor.
+
+.. The following file was automatically generated
+.. include:: ./bindings/mpi_wtime.rst
+
+RETURN VALUE
+------------
+
+Time in seconds since an arbitrary time in the past.
+
+
+DESCRIPTION
+-----------
+
+:ref:`MPI_Wtime` returns a floating-point number of seconds, representing
+elapsed wall-clock time since some time in the past.
+
+The "time in the past" is guaranteed not to change during the life of
+the process. The user is responsible for converting large numbers of
+seconds to other units if they are preferred.
+
+This function is portable (it returns seconds, not "ticks"), it allows
+high resolution, and carries no unnecessary baggage. One would use it
+like this:
+
+.. code-block:: c
+
+       {
+          double starttime, endtime;
+          starttime = MPI_Wtime();
+           ....  stuff to be timed  ...
+          endtime   = MPI_Wtime();
+          printf("That took %f seconds\n",endtime-starttime);
+       }
+
+The times returned are local to the node that called them. There is no
+requirement that different nodes return the "same" time.
+
+
+NOTES
+-----
+
+The boolean variable MPI_WTIME_IS_GLOBAL, a predefined attribute key
+that indicates whether clocks are synchronized, does not have a valid
+value in Open MPI, as the clocks are not guaranteed to be synchronized.
+
+This function is intended to be a high-resolution, elapsed (or wall)
+clock. See :ref:`MPI_Wtick` to determine the resolution of :ref:`MPI_Wtime`.
+
+On POSIX platforms, this function may utilize a timer that is cheaper
+to invoke than the gettimeofday() system call, but will fall back to
+gettimeofday() if a cheap high-resolution timer is not available. The
+:ref:`ompi_info(1) <man1-ompi_info>` command can be consulted to see if
+Open MPI supports a native high-resolution timer on your platform; see
+the value for ":ref:`MPI_WTIME` support" (or "options:mpi-wtime" when
+viewing the parsable output). If this value is "native", a method that
+is likely to be cheaper than gettimeofday() will be used to obtain the
+time when :ref:`MPI_Wtime` is invoked.
+
+For example, on platforms that support it, the *clock_gettime()*
+function will be used to obtain a monotonic clock value with whatever
+precision is supported on that platform (e.g., nanoseconds).
+
+Note, too, that the MCA parameter opal_timer_require_monotonic can
+influcence this behavior. It defaults to true, but if set to false, Open
+MPI may use a finer-grained timing mechanism (e.g., the RDTSC/RDTSCP
+clock ticks on x86_64 platforms), but is not guaranteed to be monotonic
+in some cases (e.g., if the MPI process is not bound to a single
+processor core).
+
+This function does not return an error value. Consequently, the result
+of calling it before :ref:`MPI_Init` or after :ref:`MPI_Finalize` is undefined.
+
+
+.. seealso::
+   * :ref:`MPI_Wtick`
diff --git a/docs/man-openmpi/man3/OMPI_Affinity_str.3.rst b/docs/man-openmpi/man3/OMPI_Affinity_str.3.rst
new file mode 100644
index 00000000000..6e3db7998cd
--- /dev/null
+++ b/docs/man-openmpi/man3/OMPI_Affinity_str.3.rst
@@ -0,0 +1,189 @@
+.. _ompi_affinity_str:
+
+
+OMPI_Affinity_str
+=================
+
+.. include_body
+
+**OMPI_Affinity_str** - Obtain prettyprint strings of processor affinity
+information for this process
+
+
+SYNTAX
+------
+
+
+C Syntax
+^^^^^^^^
+
+.. code-block:: c
+
+   #include <mpi.h>
+   #include <mpi-ext.h>
+
+   int OMPI_Affinity_str(ompi_affinity_fmt_type_t fmt_type,
+                         char ompi_bound[OMPI_AFFINITY_STRING_MAX],
+                         char current_binding[OMPI_AFFINITY_STRING_MAX],
+                         char exists[OMPI_AFFINITY_STRING_MAX])
+
+
+Fortran Syntax
+^^^^^^^^^^^^^^
+
+There is no Fortran binding for this function.
+
+
+Fortran 2008 Syntax
+^^^^^^^^^^^^^^^^^^^
+
+There is no Fortran 2008 binding for this function.
+
+
+C++ Syntax
+^^^^^^^^^^
+
+There is no C++ binding for this function.
+
+
+INPUT PARAMETERS
+----------------
+* ``fmt_type``: An enum indicating how to format the returned ompi_bound and current_binding strings. OMPI_AFFINITY_RSRC_STRING_FMT returns the string as human-readable resource names, such as "socket 0, core 0".
+* ``OMPI_AFFINITY_LAYOUT_FMT returns ASCII art representing where this MPI``: process is bound relative to the machine resource layout. For example "[. B][. .]" shows the process that called the routine is bound to socket 0, core 1 in a system with 2 sockets, each containing 2 cores.
+* ``See below for more output examples.``:
+
+OUTPUT PARAMETERS
+-----------------
+* ``ompi_bound``: A prettyprint string describing what processor(s) Open MPI bound this process to, or a string indicating that Open MPI did not bind this process.
+* ``current_binding``: A prettyprint string describing what processor(s) this process is currently bound to, or a string indicating that the process is bound to all available processors (and is therefore considered "unbound").
+* ``exists``: A prettyprint string describing the available sockets and sockets on this host.
+
+DESCRIPTION
+-----------
+
+Open MPI may bind a process to specific sockets and/or cores at process
+launch time. This non-standard Open MPI function call returns
+prettyprint information about three things:
+
+Where Open MPI bound this process.
+   The string returned in **ompi_bound** will either indicate that Open
+   MPI did not bind this process to anything, or it will contain a
+   prettyprint description of the processor(s) to which Open MPI bound
+   this process.
+
+Where this process is currently bound.
+   Regardless of whether Open MPI bound this process or not, another
+   entity may have bound it. The string returned in **current_binding**
+   will indicate what the *current* binding is of this process,
+   regardless of what Open MPI may have done earlier. The string
+   returned will either indicate that the process is unbound (meaning
+   that it is bound to all available processors) or it will contain a
+   prettyprint description of the sockets and cores to which the process
+   is currently bound.
+
+What processors exist.
+   As a convenience to the user, the **exists** string will contain a
+   prettyprint description of the sockets and cores that this process
+   can see (which is *usually* all processors in the system).
+
+
+EXAMPLES
+^^^^^^^^
+
+**Example 1:** Print out processes binding using resource string format.
+
+.. code-block:: c
+
+       int rank;
+       char ompi_bound[OMPI_AFFINITY_STRING_MAX];
+       char current_binding[OMPI_AFFINITY_STRING_MAX];
+       char exists[OMPI_AFFINITY_STRING_MAX];
+
+       MPI_Init(&argc, &argv);
+       MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+
+       OMPI_Affinity_str(OMPI_AFFINITY_RSRC_STRING_FMT,
+                         ompi_bound, current_binding, exists);
+       printf("rank %d: \n"
+              "       ompi_bound: %s\n"
+              "  current_binding: %s\n"
+              "           exists: %s\n",
+              rank, ompi_bound, current_binding, exists);
+       ...
+
+Output of ``mpirun -n 2 -bind-to-core a.out``:
+
+::
+
+   rank 0:
+          ompi_bound: socket 0[core 0]
+     current_binding: socket 0[core 0]
+              exists: socket 0 has 4 cores
+   rank 1:
+          ompi_bound: socket 0[core 1]
+     current_binding: socket 0[core 1]
+              exists: socket 0 has 4 cores
+
+Output of ``mpirun -n 2 -bind-to-socket a.out``:
+
+::
+
+   rank 0:
+          ompi_bound: socket 0[core 0-3]
+     current_binding: Not bound (or bound to all available processors)
+              exists: socket 0 has 4 cores
+   rank 1:
+          ompi_bound: socket 0[core 0-3]
+     current_binding: Not bound (or bound to all available processors)
+              exists: socket 0 has 4 cores
+
+|
+| **Example 2:** Print out processes binding using layout string format.
+
+.. code-block:: c
+
+       int rank;
+       char ompi_bound[OMPI_AFFINITY_STRING_MAX];
+       char current_binding[OMPI_AFFINITY_STRING_MAX];
+       char exists[OMPI_AFFINITY_STRING_MAX];
+
+       MPI_Init(&argc, &argv);
+       MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+
+       OMPI_Affinity_str(OMPI_AFFINITY_LAYOUT_FMT,
+                         ompi_bound, current_binding, exists);
+       printf("rank %d: \n"
+              "       ompi_bound: %s\n"
+              "  current_binding: %s\n"
+              "           exists: %s\n",
+              rank, ompi_bound, current_binding, exists);
+       ...
+
+Output of ``mpirun -n 2 -bind-to-core a.out``:
+
+::
+
+   rank 0:
+          ompi_bound: [B . . .]
+     current_binding: [B . . .]
+              exists: [. . . .]
+   rank 1:
+          ompi_bound: [. B . .]
+     current_binding: [. B . .]
+              exists: [. . . .]
+
+Output of ``mpirun -n 2 -bind-to-socket a.out``:
+
+::
+
+   rank 0:
+          ompi_bound: [B B B B]
+     current_binding: [B B B B]
+              exists: [. . . .]
+   rank 1:
+          ompi_bound: [B B B B]
+     current_binding: [B B B B]
+              exists: [. . . .]
+
+
+.. seealso:: :ref:`mpirun(1) <man1-mpirun>`
diff --git a/docs/man-openmpi/man3/index.rst b/docs/man-openmpi/man3/index.rst
new file mode 100644
index 00000000000..8b880c33967
--- /dev/null
+++ b/docs/man-openmpi/man3/index.rst
@@ -0,0 +1,513 @@
+MPI API manual pages (section 3)
+================================
+
+.. toctree::
+   :maxdepth: 1
+
+   MPI_Abort.3.rst
+   MPI_Accumulate.3.rst
+   MPI_Add_error_class.3.rst
+   MPI_Add_error_code.3.rst
+   MPI_Add_error_string.3.rst
+   MPI_Address.3.rst
+   MPI_Aint_add.3.rst
+   MPI_Aint_diff.3.rst
+   MPI_Allgather.3.rst
+   MPI_Allgather_init.3.rst
+   MPI_Allgatherv.3.rst
+   MPI_Allgatherv_init.3.rst
+   MPI_Alloc_mem.3.rst
+   MPI_Allreduce.3.rst
+   MPI_Allreduce_init.3.rst
+   MPI_Alltoall.3.rst
+   MPI_Alltoall_init.3.rst
+   MPI_Alltoallv.3.rst
+   MPI_Alltoallv_init.3.rst
+   MPI_Alltoallw.3.rst
+   MPI_Alltoallw_init.3.rst
+   MPI_Attr_delete.3.rst
+   MPI_Attr_get.3.rst
+   MPI_Attr_put.3.rst
+   MPI_Barrier.3.rst
+   MPI_Barrier_init.3.rst
+   MPI_Bcast.3.rst
+   MPI_Bcast_init.3.rst
+   MPI_Bsend.3.rst
+   MPI_Bsend_init.3.rst
+   MPI_Buffer_attach.3.rst
+   MPI_Buffer_detach.3.rst
+   MPI_Cancel.3.rst
+   MPI_Cart_coords.3.rst
+   MPI_Cart_create.3.rst
+   MPI_Cart_get.3.rst
+   MPI_Cart_map.3.rst
+   MPI_Cart_rank.3.rst
+   MPI_Cart_shift.3.rst
+   MPI_Cart_sub.3.rst
+   MPI_Cartdim_get.3.rst
+   MPI_Close_port.3.rst
+   MPI_Comm_accept.3.rst
+   MPI_Comm_c2f.3.rst
+   MPI_Comm_call_errhandler.3.rst
+   MPI_Comm_compare.3.rst
+   MPI_Comm_connect.3.rst
+   MPI_Comm_create.3.rst
+   MPI_Comm_create_errhandler.3.rst
+   MPI_Comm_create_from_group.3.rst
+   MPI_Comm_create_group.3.rst
+   MPI_Comm_create_keyval.3.rst
+   MPI_Comm_delete_attr.3.rst
+   MPI_Comm_disconnect.3.rst
+   MPI_Comm_dup.3.rst
+   MPI_Comm_dup_with_info.3.rst
+   MPI_Comm_f2c.3.rst
+   MPI_Comm_free.3.rst
+   MPI_Comm_free_keyval.3.rst
+   MPI_Comm_get_attr.3.rst
+   MPI_Comm_get_errhandler.3.rst
+   MPI_Comm_get_info.3.rst
+   MPI_Comm_get_name.3.rst
+   MPI_Comm_get_parent.3.rst
+   MPI_Comm_group.3.rst
+   MPI_Comm_idup.3.rst
+   MPI_Comm_idup_with_info.3.rst
+   MPI_Comm_join.3.rst
+   MPI_Comm_rank.3.rst
+   MPI_Comm_remote_group.3.rst
+   MPI_Comm_remote_size.3.rst
+   MPI_Comm_set_attr.3.rst
+   MPI_Comm_set_errhandler.3.rst
+   MPI_Comm_set_info.3.rst
+   MPI_Comm_set_name.3.rst
+   MPI_Comm_size.3.rst
+   MPI_Comm_spawn.3.rst
+   MPI_Comm_spawn_multiple.3.rst
+   MPI_Comm_split.3.rst
+   MPI_Comm_split_type.3.rst
+   MPI_Comm_test_inter.3.rst
+   MPI_Compare_and_swap.3.rst
+   MPI_Dims_create.3.rst
+   MPI_Dist_graph_create.3.rst
+   MPI_Dist_graph_create_adjacent.3.rst
+   MPI_Dist_graph_neighbors.3.rst
+   MPI_Dist_graph_neighbors_count.3.rst
+   MPI_Errhandler_c2f.3.rst
+   MPI_Errhandler_create.3.rst
+   MPI_Errhandler_f2c.3.rst
+   MPI_Errhandler_free.3.rst
+   MPI_Errhandler_get.3.rst
+   MPI_Errhandler_set.3.rst
+   MPI_Errors.3.rst
+   MPI_Error_class.3.rst
+   MPI_Error_string.3.rst
+   MPI_Exscan.3.rst
+   MPI_Exscan_init.3.rst
+   MPI_Fetch_and_op.3.rst
+   MPI_File_c2f.3.rst
+   MPI_File_call_errhandler.3.rst
+   MPI_File_close.3.rst
+   MPI_File_create_errhandler.3.rst
+   MPI_File_delete.3.rst
+   MPI_File_f2c.3.rst
+   MPI_File_get_amode.3.rst
+   MPI_File_get_atomicity.3.rst
+   MPI_File_get_byte_offset.3.rst
+   MPI_File_get_errhandler.3.rst
+   MPI_File_get_group.3.rst
+   MPI_File_get_info.3.rst
+   MPI_File_get_position.3.rst
+   MPI_File_get_position_shared.3.rst
+   MPI_File_get_size.3.rst
+   MPI_File_get_type_extent.3.rst
+   MPI_File_get_view.3.rst
+   MPI_File_iread.3.rst
+   MPI_File_iread_all.3.rst
+   MPI_File_iread_at.3.rst
+   MPI_File_iread_at_all.3.rst
+   MPI_File_iread_shared.3.rst
+   MPI_File_iwrite.3.rst
+   MPI_File_iwrite_all.3.rst
+   MPI_File_iwrite_at.3.rst
+   MPI_File_iwrite_at_all.3.rst
+   MPI_File_iwrite_shared.3.rst
+   MPI_File_open.3.rst
+   MPI_File_preallocate.3.rst
+   MPI_File_read.3.rst
+   MPI_File_read_all.3.rst
+   MPI_File_read_all_begin.3.rst
+   MPI_File_read_all_end.3.rst
+   MPI_File_read_at.3.rst
+   MPI_File_read_at_all.3.rst
+   MPI_File_read_at_all_begin.3.rst
+   MPI_File_read_at_all_end.3.rst
+   MPI_File_read_ordered.3.rst
+   MPI_File_read_ordered_begin.3.rst
+   MPI_File_read_ordered_end.3.rst
+   MPI_File_read_shared.3.rst
+   MPI_File_seek.3.rst
+   MPI_File_seek_shared.3.rst
+   MPI_File_set_atomicity.3.rst
+   MPI_File_set_errhandler.3.rst
+   MPI_File_set_info.3.rst
+   MPI_File_set_size.3.rst
+   MPI_File_set_view.3.rst
+   MPI_File_sync.3.rst
+   MPI_File_write.3.rst
+   MPI_File_write_all.3.rst
+   MPI_File_write_all_begin.3.rst
+   MPI_File_write_all_end.3.rst
+   MPI_File_write_at.3.rst
+   MPI_File_write_at_all.3.rst
+   MPI_File_write_at_all_begin.3.rst
+   MPI_File_write_at_all_end.3.rst
+   MPI_File_write_ordered.3.rst
+   MPI_File_write_ordered_begin.3.rst
+   MPI_File_write_ordered_end.3.rst
+   MPI_File_write_shared.3.rst
+   MPI_Finalize.3.rst
+   MPI_Finalized.3.rst
+   MPI_Free_mem.3.rst
+   MPI_Gather.3.rst
+   MPI_Gather_init.3.rst
+   MPI_Gatherv.3.rst
+   MPI_Gatherv_init.3.rst
+   MPI_Get.3.rst
+   MPI_Get_accumulate.3.rst
+   MPI_Get_address.3.rst
+   MPI_Get_count.3.rst
+   MPI_Get_elements.3.rst
+   MPI_Get_elements_x.3.rst
+   MPI_Get_library_version.3.rst
+   MPI_Get_processor_name.3.rst
+   MPI_Get_version.3.rst
+   MPI_Graph_create.3.rst
+   MPI_Graph_get.3.rst
+   MPI_Graph_map.3.rst
+   MPI_Graph_neighbors.3.rst
+   MPI_Graph_neighbors_count.3.rst
+   MPI_Graphdims_get.3.rst
+   MPI_Grequest_complete.3.rst
+   MPI_Grequest_start.3.rst
+   MPI_Group_c2f.3.rst
+   MPI_Group_compare.3.rst
+   MPI_Group_difference.3.rst
+   MPI_Group_excl.3.rst
+   MPI_Group_f2c.3.rst
+   MPI_Group_free.3.rst
+   MPI_Group_from_session_pset.3.rst
+   MPI_Group_incl.3.rst
+   MPI_Group_intersection.3.rst
+   MPI_Group_range_excl.3.rst
+   MPI_Group_range_incl.3.rst
+   MPI_Group_rank.3.rst
+   MPI_Group_size.3.rst
+   MPI_Group_translate_ranks.3.rst
+   MPI_Group_union.3.rst
+   MPI_Iallgather.3.rst
+   MPI_Iallgatherv.3.rst
+   MPI_Iallreduce.3.rst
+   MPI_Ialltoall.3.rst
+   MPI_Ialltoallv.3.rst
+   MPI_Ialltoallw.3.rst
+   MPI_Ibarrier.3.rst
+   MPI_Ibcast.3.rst
+   MPI_Ibsend.3.rst
+   MPI_Iexscan.3.rst
+   MPI_Igather.3.rst
+   MPI_Igatherv.3.rst
+   MPI_Improbe.3.rst
+   MPI_Imrecv.3.rst
+   MPI_Ineighbor_allgather.3.rst
+   MPI_Ineighbor_allgatherv.3.rst
+   MPI_Ineighbor_alltoall.3.rst
+   MPI_Ineighbor_alltoallv.3.rst
+   MPI_Ineighbor_alltoallw.3.rst
+   MPI_Info_c2f.3.rst
+   MPI_Info_create.3.rst
+   MPI_Info_create_env.3.rst
+   MPI_Info_delete.3.rst
+   MPI_Info_dup.3.rst
+   MPI_Info_env.3.rst
+   MPI_Info_f2c.3.rst
+   MPI_Info_free.3.rst
+   MPI_Info_get.3.rst
+   MPI_Info_get_nkeys.3.rst
+   MPI_Info_get_nthkey.3.rst
+   MPI_Info_get_string.3.rst
+   MPI_Info_get_valuelen.3.rst
+   MPI_Info_set.3.rst
+   MPI_Init.3.rst
+   MPI_Init_thread.3.rst
+   MPI_Initialized.3.rst
+   MPI_Intercomm_create.3.rst
+   MPI_Intercomm_create_from_groups.3.rst
+   MPI_Intercomm_merge.3.rst
+   MPI_Iprobe.3.rst
+   MPI_Irecv.3.rst
+   MPI_Ireduce.3.rst
+   MPI_Ireduce_scatter.3.rst
+   MPI_Ireduce_scatter_block.3.rst
+   MPI_Irsend.3.rst
+   MPI_Is_thread_main.3.rst
+   MPI_Iscan.3.rst
+   MPI_Iscatter.3.rst
+   MPI_Iscatterv.3.rst
+   MPI_Isend.3.rst
+   MPI_Isendrecv.3.rst
+   MPI_Isendrecv_replace.3.rst
+   MPI_Issend.3.rst
+   MPI_Keyval_create.3.rst
+   MPI_Keyval_free.3.rst
+   MPI_Lookup_name.3.rst
+   MPI_Message_c2f.3.rst
+   MPI_Message_f2c.3.rst
+   MPI_Mprobe.3.rst
+   MPI_Mrecv.3.rst
+   MPI_Neighbor_allgather.3.rst
+   MPI_Neighbor_allgather_init.3.rst
+   MPI_Neighbor_allgatherv.3.rst
+   MPI_Neighbor_allgatherv_init.3.rst
+   MPI_Neighbor_alltoall.3.rst
+   MPI_Neighbor_alltoall_init.3.rst
+   MPI_Neighbor_alltoallv.3.rst
+   MPI_Neighbor_alltoallv_init.3.rst
+   MPI_Neighbor_alltoallw.3.rst
+   MPI_Neighbor_alltoallw_init.3.rst
+   MPI_Op_c2f.3.rst
+   MPI_Op_commutative.3.rst
+   MPI_Op_create.3.rst
+   MPI_Op_f2c.3.rst
+   MPI_Op_free.3.rst
+   MPI_Open_port.3.rst
+   MPI_Pack.3.rst
+   MPI_Pack_external.3.rst
+   MPI_Pack_external_size.3.rst
+   MPI_Pack_size.3.rst
+   MPI_Parrived.3.rst
+   MPI_Pcontrol.3.rst
+   MPI_Pready.3.rst
+   MPI_Pready_list.3.rst
+   MPI_Pready_range.3.rst
+   MPI_Precv_init.3.rst
+   MPI_Probe.3.rst
+   MPI_Psend_init.3.rst
+   MPI_Publish_name.3.rst
+   MPI_Put.3.rst
+   MPI_Query_thread.3.rst
+   MPI_Raccumulate.3.rst
+   MPI_Recv.3.rst
+   MPI_Recv_init.3.rst
+   MPI_Reduce.3.rst
+   MPI_Reduce_init.3.rst
+   MPI_Reduce_local.3.rst
+   MPI_Reduce_scatter.3.rst
+   MPI_Reduce_scatter_block.3.rst
+   MPI_Reduce_scatter_block_init.3.rst
+   MPI_Reduce_scatter_init.3.rst
+   MPI_Register_datarep.3.rst
+   MPI_Request_c2f.3.rst
+   MPI_Request_f2c.3.rst
+   MPI_Request_free.3.rst
+   MPI_Request_get_status.3.rst
+   MPI_Rget.3.rst
+   MPI_Rget_accumulate.3.rst
+   MPI_Rput.3.rst
+   MPI_Rsend.3.rst
+   MPI_Rsend_init.3.rst
+   MPI_Scan.3.rst
+   MPI_Scan_init.3.rst
+   MPI_Scatter.3.rst
+   MPI_Scatter_init.3.rst
+   MPI_Scatterv.3.rst
+   MPI_Scatterv_init.3.rst
+   MPI_Send.3.rst
+   MPI_Send_init.3.rst
+   MPI_Sendrecv.3.rst
+   MPI_Sendrecv_replace.3.rst
+   MPI_Session_c2f.3.rst
+   MPI_Session_call_errhandler.3.rst
+   MPI_Session_create_errhandler.3.rst
+   MPI_Session_f2c.3.rst
+   MPI_Session_finalize.3.rst
+   MPI_Session_get_errhandler.3.rst
+   MPI_Session_get_info.3.rst
+   MPI_Session_get_nth_pset.3.rst
+   MPI_Session_get_num_psets.3.rst
+   MPI_Session_get_pset_info.3.rst
+   MPI_Session_init.3.rst
+   MPI_Session_set_errhandler.3.rst
+   MPI_Sizeof.3.rst
+   MPI_Ssend.3.rst
+   MPI_Ssend_init.3.rst
+   MPI_Start.3.rst
+   MPI_Startall.3.rst
+   MPI_Status_c2f.3.rst
+   MPI_Status_c2f08.3.rst
+   MPI_Status_f082c.3.rst
+   MPI_Status_f082f.3.rst
+   MPI_Status_f2c.3.rst
+   MPI_Status_f2f08.3.rst
+   MPI_Status_get_error.3.rst
+   MPI_Status_get_source.3.rst
+   MPI_Status_get_tag.3.rst
+   MPI_Status_set_cancelled.3.rst
+   MPI_Status_set_elements.3.rst
+   MPI_Status_set_elements_x.3.rst
+   MPI_Status_set_error.3.rst
+   MPI_Status_set_source.3.rst
+   MPI_Status_set_tag.3.rst
+   MPI_T.3.rst
+   MPI_T_category_changed.3.rst
+   MPI_T_category_get_categories.3.rst
+   MPI_T_category_get_cvars.3.rst
+   MPI_T_category_get_events.3.rst
+   MPI_T_category_get_info.3.rst
+   MPI_T_category_get_num.3.rst
+   MPI_T_category_get_num_events.3.rst
+   MPI_T_category_get_pvars.3.rst
+   MPI_T_cvar_get_info.3.rst
+   MPI_T_cvar_get_num.3.rst
+   MPI_T_cvar_handle_alloc.3.rst
+   MPI_T_cvar_handle_free.3.rst
+   MPI_T_cvar_read.3.rst
+   MPI_T_cvar_write.3.rst
+   MPI_T_enum_get_info.3.rst
+   MPI_T_enum_get_item.3.rst
+   MPI_T_event_callback_get_info.3.rst
+   MPI_T_event_callback_set_info.3.rst
+   MPI_T_event_copy.3.rst
+   MPI_T_event_get_index.3.rst
+   MPI_T_event_get_info.3.rst
+   MPI_T_event_get_num.3.rst
+   MPI_T_event_get_source.3.rst
+   MPI_T_event_get_timestamp.3.rst
+   MPI_T_event_handle_alloc.3.rst
+   MPI_T_event_handle_free.3.rst
+   MPI_T_event_handle_get_info.3.rst
+   MPI_T_event_handle_set_info.3.rst
+   MPI_T_event_read.3.rst
+   MPI_T_event_register_callback.3.rst
+   MPI_T_event_set_dropped_handler.3.rst
+   MPI_T_finalize.3.rst
+   MPI_T_init_thread.3.rst
+   MPI_T_pvar_get_info.3.rst
+   MPI_T_pvar_get_num.3.rst
+   MPI_T_pvar_handle_alloc.3.rst
+   MPI_T_pvar_handle_free.3.rst
+   MPI_T_pvar_read.3.rst
+   MPI_T_pvar_readreset.3.rst
+   MPI_T_pvar_reset.3.rst
+   MPI_T_pvar_session_create.3.rst
+   MPI_T_pvar_session_free.3.rst
+   MPI_T_pvar_start.3.rst
+   MPI_T_pvar_stop.3.rst
+   MPI_T_pvar_write.3.rst
+   MPI_T_source_get_info.3.rst
+   MPI_T_source_get_num.3.rst
+   MPI_T_source_get_timestamp.3.rst
+   MPI_Test.3.rst
+   MPI_Test_cancelled.3.rst
+   MPI_Testall.3.rst
+   MPI_Testany.3.rst
+   MPI_Testsome.3.rst
+   MPI_Topo_test.3.rst
+   MPI_Type_c2f.3.rst
+   MPI_Type_commit.3.rst
+   MPI_Type_contiguous.3.rst
+   MPI_Type_create_darray.3.rst
+   MPI_Type_create_f90_complex.3.rst
+   MPI_Type_create_f90_integer.3.rst
+   MPI_Type_create_f90_real.3.rst
+   MPI_Type_create_hindexed.3.rst
+   MPI_Type_create_hindexed_block.3.rst
+   MPI_Type_create_hvector.3.rst
+   MPI_Type_create_indexed_block.3.rst
+   MPI_Type_create_keyval.3.rst
+   MPI_Type_create_resized.3.rst
+   MPI_Type_create_struct.3.rst
+   MPI_Type_create_subarray.3.rst
+   MPI_Type_delete_attr.3.rst
+   MPI_Type_dup.3.rst
+   MPI_Type_extent.3.rst
+   MPI_Type_f2c.3.rst
+   MPI_Type_free.3.rst
+   MPI_Type_free_keyval.3.rst
+   MPI_Type_get_attr.3.rst
+   MPI_Type_get_contents.3.rst
+   MPI_Type_get_envelope.3.rst
+   MPI_Type_get_extent.3.rst
+   MPI_Type_get_extent_x.3.rst
+   MPI_Type_get_name.3.rst
+   MPI_Type_get_true_extent.3.rst
+   MPI_Type_get_true_extent_x.3.rst
+   MPI_Type_hindexed.3.rst
+   MPI_Type_hvector.3.rst
+   MPI_Type_indexed.3.rst
+   MPI_Type_lb.3.rst
+   MPI_Type_match_size.3.rst
+   MPI_Type_set_attr.3.rst
+   MPI_Type_set_name.3.rst
+   MPI_Type_size.3.rst
+   MPI_Type_size_x.3.rst
+   MPI_Type_struct.3.rst
+   MPI_Type_ub.3.rst
+   MPI_Type_vector.3.rst
+   MPI_Unpack.3.rst
+   MPI_Unpack_external.3.rst
+   MPI_Unpublish_name.3.rst
+   MPI_Wait.3.rst
+   MPI_Waitall.3.rst
+   MPI_Waitany.3.rst
+   MPI_Waitsome.3.rst
+   MPI_Win_allocate.3.rst
+   MPI_Win_allocate_shared.3.rst
+   MPI_Win_attach.3.rst
+   MPI_Win_c2f.3.rst
+   MPI_Win_call_errhandler.3.rst
+   MPI_Win_complete.3.rst
+   MPI_Win_create.3.rst
+   MPI_Win_create_dynamic.3.rst
+   MPI_Win_create_errhandler.3.rst
+   MPI_Win_create_keyval.3.rst
+   MPI_Win_delete_attr.3.rst
+   MPI_Win_detach.3.rst
+   MPI_Win_f2c.3.rst
+   MPI_Win_fence.3.rst
+   MPI_Win_flush.3.rst
+   MPI_Win_flush_all.3.rst
+   MPI_Win_flush_local.3.rst
+   MPI_Win_flush_local_all.3.rst
+   MPI_Win_free.3.rst
+   MPI_Win_free_keyval.3.rst
+   MPI_Win_get_attr.3.rst
+   MPI_Win_get_errhandler.3.rst
+   MPI_Win_get_group.3.rst
+   MPI_Win_get_info.3.rst
+   MPI_Win_get_name.3.rst
+   MPI_Win_lock.3.rst
+   MPI_Win_lock_all.3.rst
+   MPI_Win_post.3.rst
+   MPI_Win_set_attr.3.rst
+   MPI_Win_set_errhandler.3.rst
+   MPI_Win_set_info.3.rst
+   MPI_Win_set_name.3.rst
+   MPI_Win_shared_query.3.rst
+   MPI_Win_start.3.rst
+   MPI_Win_sync.3.rst
+   MPI_Win_test.3.rst
+   MPI_Win_unlock.3.rst
+   MPI_Win_unlock_all.3.rst
+   MPI_Win_wait.3.rst
+   MPI_Wtick.3.rst
+   MPI_Wtime.3.rst
+   MPIX_Comm_ack_failed.3.rst
+   MPIX_Comm_agree.3.rst
+   MPIX_Comm_get_failed.3.rst
+   MPIX_Comm_iagree.3.rst
+   MPIX_Comm_ishrink.3.rst
+   MPIX_Comm_is_revoked.3.rst
+   MPIX_Comm_revoke.3.rst
+   MPIX_Comm_shrink.3.rst
+   MPIX_Query_cuda_support.3.rst
+   MPIX_Query_rocm_support.3.rst
+   OMPI_Affinity_str.3.rst
diff --git a/docs/man-openmpi/man7/Open-MPI.7.rst b/docs/man-openmpi/man7/Open-MPI.7.rst
new file mode 100644
index 00000000000..6c7b4c61ba7
--- /dev/null
+++ b/docs/man-openmpi/man7/Open-MPI.7.rst
@@ -0,0 +1,64 @@
+.. _open-mpi:
+.. _mpi:
+
+Open MPI
+========
+
+.. include_body
+
+Open MPI - General information
+
+OPEN MPI
+--------
+
+Open MPI is an open source implementation of MPI (message-passing
+interface), the industry-standard specification for writing
+message-passing programs. Message passing is a programming model that
+gives the programmer explicit control over interprocess communication.
+
+The MPI specification was developed by the MPI Forum, a group of
+software developers, computer vendors, academics, and computer-science
+researchers whose goal was to develop a standard for writing
+message-passing programs that would be efficient, flexible, and
+portable.
+
+The outcome, known as the MPI Standard, was first published in 1993; its
+most recent version (MPI-4.1) was published in November 2023. Open MPI
+includes all MPI |mpi_standard_version|-compliant routines.
+
+For more information about Open MPI, see https://www.open-mpi.org.
+
+The MPI standards are available at https://www.mpi-forum.org.
+
+MAN PAGE SYNTAX
+---------------
+
+Man pages for Open MPI and Open MPI I/O routines are named according to
+C syntax, that is, they begin with the prefix ``MPI_``, all in
+uppercase, and the first letter following the ``MPI_`` prefix is also
+uppercase. The rest of the letters in the routine are all lowercase, for
+example, :ref:`MPI_Comm_get_attr`.
+
+ENVIRONMENT
+-----------
+
+To fine-tune your Open MPI environment, you can either use arguments
+to the :ref:`mpirun <man1-mpirun>` or :ref:`mpiexec <man1-mpiexec>`
+commands, or you can use MCA parameters.
+
+For more information on arguments, see the :ref:`mpirun(1)
+<man1-mpirun>` man page.
+
+For a complete listing of MCA parameters and their descriptions, issue
+the command ``ompi_info --all``.  See :ref:`ompi_info(1)
+<man1-ompi_info>` for more information.
+
+
+ERRORS
+------
+
+See :ref:`MPI_Errors` for details about error values.
+
+
+.. seealso:: :ref:`MPI_T` :ref:`mpirun(1) <man1-mpirun>` :ref:`mpiexec(1)
+             <man1-mpiexec>` :ref:`ompi_info(1) <man1-ompi_info>`
diff --git a/docs/man-openmpi/man7/index.rst b/docs/man-openmpi/man7/index.rst
new file mode 100644
index 00000000000..c67605d6c59
--- /dev/null
+++ b/docs/man-openmpi/man7/index.rst
@@ -0,0 +1,7 @@
+General information manual pages (section 7)
+============================================
+
+.. toctree::
+   :maxdepth: 1
+
+   Open-MPI.7.rst
diff --git a/docs/man-openshmem/index.rst b/docs/man-openshmem/index.rst
new file mode 100644
index 00000000000..1dd92431628
--- /dev/null
+++ b/docs/man-openshmem/index.rst
@@ -0,0 +1,21 @@
+OpenSHMEM manual pages
+======================
+
+All of the content in this section is also installed as individual
+manual ("man") pages in an OpenSHMEM installation.  If your man page
+search path includes the location where OpenSHMEM installed its man
+pages (which defaults to ``$prefix/share/man``), you can invoke
+commands similar to the following to see the same content that you see
+in these HTML pages:
+
+.. code:: sh
+
+   shell$ man shmem_init
+
+OpenSHMEM man pages, by section:
+
+.. toctree::
+   :maxdepth: 1
+
+   man1/index
+   man3/index
diff --git a/docs/man-openshmem/man1/index.rst b/docs/man-openshmem/man1/index.rst
new file mode 100644
index 00000000000..7e7d6fd1086
--- /dev/null
+++ b/docs/man-openshmem/man1/index.rst
@@ -0,0 +1,9 @@
+Commands (section 1)
+====================
+
+.. toctree::
+   :maxdepth: 1
+
+   oshmem-wrapper-compiler.1.rst
+   oshmem_info.1.rst
+   oshrun.1.rst
diff --git a/docs/man-openshmem/man1/oshc++.1 b/docs/man-openshmem/man1/oshc++.1
new file mode 100644
index 00000000000..1dbbdca80df
--- /dev/null
+++ b/docs/man-openshmem/man1/oshc++.1
@@ -0,0 +1 @@
+.so man1/oshmem-wrapper-compiler.1
diff --git a/docs/man-openshmem/man1/oshcc.1 b/docs/man-openshmem/man1/oshcc.1
new file mode 100644
index 00000000000..1dbbdca80df
--- /dev/null
+++ b/docs/man-openshmem/man1/oshcc.1
@@ -0,0 +1 @@
+.so man1/oshmem-wrapper-compiler.1
diff --git a/docs/man-openshmem/man1/oshcxx.1 b/docs/man-openshmem/man1/oshcxx.1
new file mode 100644
index 00000000000..1dbbdca80df
--- /dev/null
+++ b/docs/man-openshmem/man1/oshcxx.1
@@ -0,0 +1 @@
+.so man1/oshmem-wrapper-compiler.1
diff --git a/docs/man-openshmem/man1/oshfort.1 b/docs/man-openshmem/man1/oshfort.1
new file mode 100644
index 00000000000..1dbbdca80df
--- /dev/null
+++ b/docs/man-openshmem/man1/oshfort.1
@@ -0,0 +1 @@
+.so man1/oshmem-wrapper-compiler.1
diff --git a/docs/man-openshmem/man1/oshmem-wrapper-compiler.1.rst b/docs/man-openshmem/man1/oshmem-wrapper-compiler.1.rst
new file mode 100644
index 00000000000..4599d3f49bb
--- /dev/null
+++ b/docs/man-openshmem/man1/oshmem-wrapper-compiler.1.rst
@@ -0,0 +1,244 @@
+.. _man1-oshcc:
+.. _man1-oshcxx:
+.. _man1-oshc++:
+.. _man1-oshfort:
+.. _man1-shmemcc:
+.. _man1-shmemcxx:
+.. _man1-shmemc++:
+.. _man1-shmemfort:
+
+
+OpenSHMEM Wrapper Compilers
+===========================
+
+.. include_body
+
+oshcc, oshcxx, oshc++, oshfort, shmemcc, shmemcxx, shmemc++, shmemfort |mdash| OpenSHMEM wrapper compilers
+
+
+SYNTAX
+------
+
+``oshcc [--showme | --showme:compile | --showme:link] ...``
+
+``oshcxx [--showme | --showme:compile | --showme:link] ...``
+
+``oshc++ [--showme | --showme:compile | --showme:link] ...``
+
+``oshfort [--showme | --showme:compile | --showme:link] ...``
+
+``shmemcc [--showme | --showme:compile | --showme:link] ...``
+
+``shmemcxx [--showme | --showme:compile | --showme:link] ...``
+
+``shmemc++ [--showme | --showme:compile | --showme:link] ...``
+
+``shmemfort [--showme | --showme:compile | --showme:link] ...``
+
+
+OPTIONS
+-------
+
+The options below apply to all of the wrapper compilers:
+
+* ``--showme``: This option comes in several different variants (see
+  below). None of the variants invokes the underlying compiler; they
+  all provide information on how the underlying compiler would have
+  been invoked had ``--showme`` not been used. The basic ``--showme``
+  option outputs the command line that would be executed to compile
+  the program.
+
+  .. note:: If a non-filename argument is passed on the command line,
+            the ``--showme`` option will *not* display any additional
+            flags. For example, both ``"oshcc --showme`` and
+            ``oshcc --showme my_source.c`` will show all the
+            wrapper-supplied flags. But ``oshcc
+            --showme -v`` will only show the underlying compiler name
+            and ``-v``.
+
+* ``--showme:compile``: Output the compiler flags that would have been
+  supplied to the underlying compiler.
+
+* ``--showme:link``: Output the linker flags that would have been
+  supplied to the underlying compiler.
+
+* ``--showme:command``: Outputs the underlying compiler
+  command (which may be one or more tokens).
+
+* ``--showme:incdirs``: Outputs a space-delimited (but otherwise
+  undecorated) list of directories that the wrapper compiler would
+  have provided to the underlying compiler to indicate
+  where relevant header files are located.
+
+* ``--showme:libdirs``: Outputs a space-delimited (but otherwise
+  undecorated) list of directories that the wrapper compiler would
+  have provided to the underlying linker to indicate where relevant
+  libraries are located.
+
+* ``--showme:libs`` Outputs a space-delimited (but otherwise
+  undecorated) list of library names that the wrapper compiler would
+  have used to link an application. For example: ``mpi open-pal
+  util``.
+
+* ``--showme:version``: Outputs the version number of Open MPI.
+
+* ``--showme:help``: Output a brief usage help message.
+
+See the man page for your underlying compiler for other
+options that can be passed through oshcc.
+
+
+DESCRIPTION
+-----------
+
+Conceptually, the role of these commands is quite simple: transparently
+add relevant compiler and linker flags to the user's command line that
+are necessary to compile / link OpenSHMEM programs, and then invoke the
+underlying compiler to actually perform the command.
+
+As such, these commands are frequently referred to as "wrapper"
+compilers because they do not actually compile or link applications
+themselves; they only add in command line flags and invoke the back-end
+compiler.
+
+Background
+----------
+
+Open MPI provides wrapper compilers for several languages:
+
+* ``oshcc``, ``shmemcc``: C
+
+* ``oshc++``, ``oshcxx``, ``shmemc++``, shmemcxx`:: C++
+
+* ``oshfort``, ``shmemfort``: Fortran
+
+The wrapper compilers for each of the languages are identical; they
+can be use interchangeably.  The different names are provided solely
+for backwards compatibility.
+
+
+Fortran Notes
+-------------
+
+The Fortran wrapper compiler for OpenSHMEM (``oshfort`` and
+``shmemfort``) can compile and link OpenSHMEM applications that use
+any/all of the OpenSHMEM Fortran bindings.
+
+Note, however, that the Fortran compiler may require additional
+command-line options to enforce a specific Fortran dialect. For
+example, in some versions of the IBM XLF compiler, if ``xlf90`` is the
+underlying Fortran compiler, ``-qfixed`` may be necessary to compile
+fixed-format Fortran source files.
+
+Finally, note that ``oshfort`` will be inoperative and will return an
+error on use if Fortran support was not built into the OpenSHMEM
+layer.
+
+Overview
+--------
+
+``oshcc`` and ``shmemcc`` are convenience wrappers for the underlying
+C compiler.  Translation of an OpenSHMEM program requires the linkage
+of the OpenSHMEM-specific libraries which may not reside in one of the
+standard search directories of ``ld(1)``. It also often requires the
+inclusion of header files what may also not be found in a standard
+location.
+
+``oshcc`` and ``shmemcc`` pass their arguments to the underlying C
+compiler along with the ``-I``, ``-L`` and ``-l`` options required by
+OpenSHMEM programs.
+
+The same is true for all the other language wrapper compilers.
+
+The OpenSHMEM Team *strongly* encourages using the wrapper compilers
+instead of attempting to link to the OpenSHMEM libraries manually. This
+allows the specific implementation of OpenSHMEM to change without
+forcing changes to linker directives in users' Makefiles. Indeed, the
+specific set of flags and libraries used by the wrapper compilers
+depends on how OpenSHMEM was configured and built; the values can change
+between different installations of the same version of OpenSHMEM.
+
+Indeed, since the wrappers are simply thin shells on top of an
+underlying compiler, there are very, very few compelling reasons *not*
+to use ``oshcc``. When it is not possible to use the wrappers
+directly, the ``--showme:compile`` and ``--showme:link`` options should be
+used to determine what flags the wrappers would have used. For example:
+
+.. code:: sh
+
+   shell$ cc -c file1.c `shmemcc --showme:compile`
+
+   shell$ cc -c file2.c `shmemcc --showme:compile`
+
+   shell$ cc file1.o file2.o `shmemcc --showme:link` -o my_oshmem_program
+
+
+NOTES
+-----
+
+It is possible to make the wrapper compilers multi-lib aware. That is,
+the libraries and includes specified may differ based on the compiler
+flags specified (for example, with the GNU compilers on Linux, a
+different library path may be used if ``-m32`` is seen versus ``-m64``
+being seen). This is not the default behavior in a standard build, but
+can be activated (for example, in a binary package providing both 32
+and 64 bit support). `More information can be found here
+<https://github.com/open-mpi/ompi/wiki/compilerwrapper3264>`_.
+
+
+.. _man1-oshmem-wrapper-compiler-files:
+
+FILES
+-----
+
+The strings that the wrapper compilers insert into the command line
+before invoking the underlying compiler are stored in a text file
+created by OpenSHMEM and installed to
+``$pkgdata/NAME-wrapper-data.txt``, where:
+
+* ``$pkgdata`` is typically ``$prefix/share/openmpi``
+* ``$prefix`` is the top installation directory of OpenSHMEM
+* ``NAME`` is the name of the wrapper compiler (e.g.,
+  ``$pkgdata/shmemcc-wrapper-data.txt``)
+
+It is rarely necessary to edit these files, but they can be examined to
+gain insight into what flags the wrappers are placing on the command
+line.
+
+
+ENVIRONMENT VARIABLES
+---------------------
+
+By default, the wrappers use the compilers that were selected when
+OpenSHMEM was configured. These compilers were either found
+automatically by Open MPI's "configure" script, or were selected by
+the user in the ``CC``, ``CXX``, and/or ``FC`` environment variables
+before ``configure`` was invoked. Additionally, other arguments specific
+to the compiler may have been selected by configure.
+
+These values can be selectively overridden by either editing the text
+files containing this configuration information (see the :ref:`FILES
+<man1-oshmem-wrapper-compiler-files>` section), or by setting selected
+environment variables of the form ``oshmem_value``.
+
+Valid value names are:
+
+* ``CPPFLAGS``: Flags added when invoking the preprocessor (C or C++)
+
+* ``LDFLAGS``: Flags added when invoking the linker (C, C++, or
+  Fortran)
+
+* ``LIBS``: Libraries added when invoking the linker (C, C++, or
+  Fortran)
+
+* ``CC``: C compiler
+
+* ``CFLAGS``: C compiler flags
+
+* ``CXX``: C++ compiler
+
+* ``CXXFLAGS``: C++ compiler flags
+
+* ``FC``: Fortran compiler
+
+* ``FCFLAGS``: Fortran compiler flags
diff --git a/docs/man-openshmem/man1/oshmem_info.1.rst b/docs/man-openshmem/man1/oshmem_info.1.rst
new file mode 100644
index 00000000000..b4f19fb9034
--- /dev/null
+++ b/docs/man-openshmem/man1/oshmem_info.1.rst
@@ -0,0 +1,171 @@
+.. _oshmem_info:
+
+
+oshmem_info
+===========
+
+.. include_body
+
+oshmem_info - Display information about the OpenSHMEM installation
+
+
+SYNOPSIS
+--------
+
+``oshmem_info [options]``
+
+
+DESCRIPTION
+-----------
+
+``oshmem_info`` provides detailed information about the OpenSHMEM
+installation. It can be useful for at least three common scenarios:
+
+1. Checking local configuration and seeing how OpenSHMEM was
+   installed.
+
+2. Submitting bug reports / help requests to the OpenSHMEM
+   community
+   (see :doc:`Getting help </getting-help>`).
+
+3. Seeing a list of installed OpenSHMEM plugins and querying what MCA
+   parameters they support.
+
+.. note:: ``oshmem_info`` defaults to only showing a few MCA parameters
+          by default (i.e., level 1 parameters). Use the ``--level``
+          option to enable showing more options (see the :ref:`LEVELS
+          <man1-oshmem_info-levels>` section for more information).
+
+
+OPTIONS
+-------
+
+``oshmem_info`` accepts the following options:
+
+* ``-a``, ``--all``: Show all configuration options and MCA parameters
+
+* ``--arch``: Show architecture OpenSHMEM was compiled on
+
+* ``-c``, ``--config``: Show configuration options
+
+* ``-gmca``, ``--gmca <param> <value>*``: Pass global MCA parameters
+  that are applicable to all contexts.
+
+* ``-h``, ``--help*``: Shows help / usage message
+
+* ``--hostname``: Show the hostname that OpenSHMEM was configured
+  and built on
+
+* ``--internal``: Show internal MCA parameters (not meant to be
+  modified by users)
+
+* ``--level``: Show only variables with at most this level (1-9). The
+  default is 1 unless ``--all`` is specified without ``--level`` in
+  which case the default is 9. See the :ref:`LEVELS
+  <man1-oshmem_info-levels>` section for more information.
+
+* ``-mca``, ``--mca <param> <value>``: Pass context-specific MCA
+  parameters; they are considered global if
+  ``--gmca`` is not used and only one context is specified.
+
+* ``--param <type> <component>``: Show MCA parameters. The first
+  parameter is the type of the component to display; the second
+  parameter is the specific component to display (or the keyword
+  ``all``, meaning "display all components of this type").
+
+* ``--parsable``: When used in conjunction with other parameters,
+  the output is displayed in a machine-parsable format ``--parseable``
+  Synonym for ``--parsable``.
+
+* ``--path <type>``: Show paths that OpenSHMEM was configured
+  with. Accepts the following parameters: ``prefix``, ``bindir``,
+  ``libdir``, ``incdir``, ``pkglibdir``, ``sysconfdir``.
+
+* ``--pretty``: When used in conjunction with other parameters, the
+  output is displayed in prettyprint format (default)
+
+* ``--selected-only``: Show only variables from selected components.
+
+* ``-V``, ``--version*``: Show version of OpenSHMEM.
+
+.. _man1-oshmem_info-levels:
+
+LEVELS
+------
+
+#. Basic information of interest to users
+#. Detailed information of interest to users
+#. All remaining information of interest to users
+#. Basic information required for tuning
+#. Detailed information required for tuning
+#. All remaining information required for tuning
+#. Basic information for OpenSHMEM implementors
+#. Detailed information for OpenSHMEM implementors
+#. All remaining information for OpenSHMEM implementors
+
+
+EXAMPLES
+--------
+
+.. code:: sh
+
+   shell$ oshmem_info
+
+Show the default output of options and listing of installed components
+in a human-readable / prettyprint format.
+
+.. code:: sh
+
+   shell$ oshmem_info --parsable
+
+Show the default output of options and listing of installed components
+in a machine-parsable format.
+
+.. code:: sh
+
+   shell$ oshmem_info --param btl openib
+
+Show the MCA parameters of the "openib" BTL component in a
+human-readable / prettyprint format.
+
+.. code:: sh
+
+   shell$ oshmem_info --param btl openib --parsable
+
+Show the MCA parameters of the "openib" BTL component in a
+machine-parsable format.
+
+.. code:: sh
+
+   shell$ oshmem_info --path bindir
+
+Show the "bindir" that OpenSHMEM was configured with.
+
+.. code:: sh
+
+   shell$ oshmem_info --version ompi full --parsable
+
+Show the full version numbers of OpenSHMEM (including the ORTE
+and OPAL version numbers) in a machine-readable format.
+
+.. code:: sh
+
+   shell$ oshmem_info --version btl major
+
+Show the major version number of all BTL components in a prettyprint
+format.
+
+.. code:: sh
+
+   shell$ oshmem_info --version btl:tcp minor
+
+Show the minor version number of the TCP BTL component in a
+prettyprint format.
+
+.. code:: sh
+
+   shell$ oshmem_info --all
+
+Show *all* information about the OpenSHMEM installation, including all
+components that can be found, the MCA parameters that they support,
+versions of OpenSHMEM and the components, etc.
diff --git a/docs/man-openshmem/man1/oshrun.1.rst b/docs/man-openshmem/man1/oshrun.1.rst
new file mode 100644
index 00000000000..b7cbcf88ddb
--- /dev/null
+++ b/docs/man-openshmem/man1/oshrun.1.rst
@@ -0,0 +1,16 @@
+.. _man1-oshrun:
+
+
+oshrun
+======
+
+.. include_body
+
+oshrun |mdash| Execute serial and OpenSHMEM parallel jobs.
+
+The ``oshrun(1)`` command is functionality identical to the
+:ref:`mpirun(1) <man1-mpirun>` command.  See the :ref:`mpirun(1)
+<man1-mpirun>` man page for details.
+
+.. seealso::
+   :ref:`mpirun(1) <man1-mpirun>`
diff --git a/docs/man-openshmem/man1/shmemc++.1 b/docs/man-openshmem/man1/shmemc++.1
new file mode 100644
index 00000000000..1dbbdca80df
--- /dev/null
+++ b/docs/man-openshmem/man1/shmemc++.1
@@ -0,0 +1 @@
+.so man1/oshmem-wrapper-compiler.1
diff --git a/docs/man-openshmem/man1/shmemcc.1 b/docs/man-openshmem/man1/shmemcc.1
new file mode 100644
index 00000000000..1dbbdca80df
--- /dev/null
+++ b/docs/man-openshmem/man1/shmemcc.1
@@ -0,0 +1 @@
+.so man1/oshmem-wrapper-compiler.1
diff --git a/docs/man-openshmem/man1/shmemcxx.1 b/docs/man-openshmem/man1/shmemcxx.1
new file mode 100644
index 00000000000..1dbbdca80df
--- /dev/null
+++ b/docs/man-openshmem/man1/shmemcxx.1
@@ -0,0 +1 @@
+.so man1/oshmem-wrapper-compiler.1
diff --git a/docs/man-openshmem/man1/shmemfort.1 b/docs/man-openshmem/man1/shmemfort.1
new file mode 100644
index 00000000000..1dbbdca80df
--- /dev/null
+++ b/docs/man-openshmem/man1/shmemfort.1
@@ -0,0 +1 @@
+.so man1/oshmem-wrapper-compiler.1
diff --git a/docs/man-openshmem/man3/OpenSHMEM.3.rst b/docs/man-openshmem/man3/OpenSHMEM.3.rst
new file mode 100644
index 00000000000..f181ec97640
--- /dev/null
+++ b/docs/man-openshmem/man3/OpenSHMEM.3.rst
@@ -0,0 +1,9 @@
+.. _openshmem:
+
+OpenSHMEM
+=========
+    .. include_body
+
+.. include:: ../man3/intro_shmem.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/_my_pe.3.rst b/docs/man-openshmem/man3/_my_pe.3.rst
new file mode 100644
index 00000000000..5e5b1e8b55b
--- /dev/null
+++ b/docs/man-openshmem/man3/_my_pe.3.rst
@@ -0,0 +1,9 @@
+.. _my_pe:
+
+_my_pe
+======
+    .. include_body
+
+.. include:: ../man3/shmem_my_pe.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/_num_pes.3.rst b/docs/man-openshmem/man3/_num_pes.3.rst
new file mode 100644
index 00000000000..36464e6be10
--- /dev/null
+++ b/docs/man-openshmem/man3/_num_pes.3.rst
@@ -0,0 +1,9 @@
+.. _num_pes:
+
+_num_pes
+========
+    .. include_body
+
+.. include:: ../man3/shmem_n_pes.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/index.rst b/docs/man-openshmem/man3/index.rst
new file mode 100644
index 00000000000..37ebd60128d
--- /dev/null
+++ b/docs/man-openshmem/man3/index.rst
@@ -0,0 +1,225 @@
+OpenSHMEM API manual pages (section 3)
+======================================
+
+.. toctree::
+   :maxdepth: 1
+
+   OpenSHMEM.3.rst
+   _my_pe.3.rst
+   _num_pes.3.rst
+   intro_shmem.3.rst
+   shfree.3.rst
+   shmalloc.3.rst
+   shmem_addr_accessible.3.rst
+   shmem_align.3.rst
+   shmem_alltoall32.3.rst
+   shmem_alltoall64.3.rst
+   shmem_alltoalls32.3.rst
+   shmem_alltoalls64.3.rst
+   shmem_barrier.3.rst
+   shmem_barrier_all.3.rst
+   shmem_broadcast32.3.rst
+   shmem_broadcast64.3.rst
+   shmem_char_g.3.rst
+   shmem_char_get.3.rst
+   shmem_char_get_nbi.3.rst
+   shmem_char_p.3.rst
+   shmem_char_put.3.rst
+   shmem_char_put_nbi.3.rst
+   shmem_clear_cache_inv.3.rst
+   shmem_clear_cache_line_inv.3.rst
+   shmem_clear_lock.3.rst
+   shmem_collect32.3.rst
+   shmem_collect64.3.rst
+   shmem_complexd_prod_to_all.3.rst
+   shmem_complexd_sum_to_all.3.rst
+   shmem_complexf_prod_to_all.3.rst
+   shmem_complexf_sum_to_all.3.rst
+   shmem_double_fetch.3.rst
+   shmem_double_g.3.rst
+   shmem_double_get.3.rst
+   shmem_double_get_nbi.3.rst
+   shmem_double_iget.3.rst
+   shmem_double_iput.3.rst
+   shmem_double_max_to_all.3.rst
+   shmem_double_min_to_all.3.rst
+   shmem_double_p.3.rst
+   shmem_double_prod_to_all.3.rst
+   shmem_double_put.3.rst
+   shmem_double_put_nbi.3.rst
+   shmem_double_set.3.rst
+   shmem_double_sum_to_all.3.rst
+   shmem_double_swap.3.rst
+   shmem_fcollect32.3.rst
+   shmem_fcollect64.3.rst
+   shmem_fence.3.rst
+   shmem_finalize.3.rst
+   shmem_float_fetch.3.rst
+   shmem_float_g.3.rst
+   shmem_float_get.3.rst
+   shmem_float_get_nbi.3.rst
+   shmem_float_iget.3.rst
+   shmem_float_iput.3.rst
+   shmem_float_max_to_all.3.rst
+   shmem_float_min_to_all.3.rst
+   shmem_float_p.3.rst
+   shmem_float_prod_to_all.3.rst
+   shmem_float_put.3.rst
+   shmem_float_put_nbi.3.rst
+   shmem_float_set.3.rst
+   shmem_float_sum_to_all.3.rst
+   shmem_float_swap.3.rst
+   shmem_free.3.rst
+   shmem_get128.3.rst
+   shmem_get128_nbi.3.rst
+   shmem_get16_nbi.3.rst
+   shmem_get32.3.rst
+   shmem_get32_nbi.3.rst
+   shmem_get64.3.rst
+   shmem_get64_nbi.3.rst
+   shmem_get8_nbi.3.rst
+   shmem_getmem.3.rst
+   shmem_getmem_nbi.3.rst
+   shmem_global_exit.3.rst
+   shmem_iget128.3.rst
+   shmem_iget32.3.rst
+   shmem_iget64.3.rst
+   shmem_info_get_name.3.rst
+   shmem_info_get_version.3.rst
+   shmem_init.3.rst
+   shmem_int_add.3.rst
+   shmem_int_and_to_all.3.rst
+   shmem_int_cswap.3.rst
+   shmem_int_fadd.3.rst
+   shmem_int_fetch.3.rst
+   shmem_int_finc.3.rst
+   shmem_int_g.3.rst
+   shmem_int_get.3.rst
+   shmem_int_get_nbi.3.rst
+   shmem_int_iget.3.rst
+   shmem_int_inc.3.rst
+   shmem_int_iput.3.rst
+   shmem_int_max_to_all.3.rst
+   shmem_int_min_to_all.3.rst
+   shmem_int_or_to_all.3.rst
+   shmem_int_p.3.rst
+   shmem_int_prod_to_all.3.rst
+   shmem_int_put.3.rst
+   shmem_int_put_nbi.3.rst
+   shmem_int_set.3.rst
+   shmem_int_sum_to_all.3.rst
+   shmem_int_swap.3.rst
+   shmem_int_wait.3.rst
+   shmem_int_wait_until.3.rst
+   shmem_int_xor_to_all.3.rst
+   shmem_iput128.3.rst
+   shmem_iput32.3.rst
+   shmem_iput64.3.rst
+   shmem_long_add.3.rst
+   shmem_long_and_to_all.3.rst
+   shmem_long_cswap.3.rst
+   shmem_long_fadd.3.rst
+   shmem_long_fetch.3.rst
+   shmem_long_finc.3.rst
+   shmem_long_g.3.rst
+   shmem_long_get.3.rst
+   shmem_long_get_nbi.3.rst
+   shmem_long_iget.3.rst
+   shmem_long_inc.3.rst
+   shmem_long_iput.3.rst
+   shmem_long_max_to_all.3.rst
+   shmem_long_min_to_all.3.rst
+   shmem_long_or_to_all.3.rst
+   shmem_long_p.3.rst
+   shmem_long_prod_to_all.3.rst
+   shmem_long_put.3.rst
+   shmem_long_put_nbi.3.rst
+   shmem_long_set.3.rst
+   shmem_long_sum_to_all.3.rst
+   shmem_long_swap.3.rst
+   shmem_long_wait.3.rst
+   shmem_long_wait_until.3.rst
+   shmem_long_xor_to_all.3.rst
+   shmem_longdouble_g.3.rst
+   shmem_longdouble_get.3.rst
+   shmem_longdouble_get_nbi.3.rst
+   shmem_longdouble_iget.3.rst
+   shmem_longdouble_iput.3.rst
+   shmem_longdouble_max_to_all.3.rst
+   shmem_longdouble_min_to_all.3.rst
+   shmem_longdouble_p.3.rst
+   shmem_longdouble_prod_to_all.3.rst
+   shmem_longdouble_put.3.rst
+   shmem_longdouble_put_nbi.3.rst
+   shmem_longdouble_sum_to_all.3.rst
+   shmem_longlong_add.3.rst
+   shmem_longlong_and_to_all.3.rst
+   shmem_longlong_cswap.3.rst
+   shmem_longlong_fadd.3.rst
+   shmem_longlong_fetch.3.rst
+   shmem_longlong_finc.3.rst
+   shmem_longlong_g.3.rst
+   shmem_longlong_get.3.rst
+   shmem_longlong_get_nbi.3.rst
+   shmem_longlong_iget.3.rst
+   shmem_longlong_inc.3.rst
+   shmem_longlong_iput.3.rst
+   shmem_longlong_max_to_all.3.rst
+   shmem_longlong_min_to_all.3.rst
+   shmem_longlong_or_to_all.3.rst
+   shmem_longlong_p.3.rst
+   shmem_longlong_prod_to_all.3.rst
+   shmem_longlong_put.3.rst
+   shmem_longlong_put_nbi.3.rst
+   shmem_longlong_set.3.rst
+   shmem_longlong_sum_to_all.3.rst
+   shmem_longlong_swap.3.rst
+   shmem_longlong_wait.3.rst
+   shmem_longlong_wait_until.3.rst
+   shmem_longlong_xor_to_all.3.rst
+   shmem_malloc.3.rst
+   shmem_my_pe.3.rst
+   shmem_n_pes.3.rst
+   shmem_pe_accessible.3.rst
+   shmem_ptr.3.rst
+   shmem_put128.3.rst
+   shmem_put128_nbi.3.rst
+   shmem_put16_nbi.3.rst
+   shmem_put32.3.rst
+   shmem_put32_nbi.3.rst
+   shmem_put64.3.rst
+   shmem_put64_nbi.3.rst
+   shmem_put8_nbi.3.rst
+   shmem_putmem.3.rst
+   shmem_putmem_nbi.3.rst
+   shmem_quiet.3.rst
+   shmem_realloc.3.rst
+   shmem_set_cache_inv.3.rst
+   shmem_set_cache_line_inv.3.rst
+   shmem_set_lock.3.rst
+   shmem_short_and_to_all.3.rst
+   shmem_short_g.3.rst
+   shmem_short_get.3.rst
+   shmem_short_get_nbi.3.rst
+   shmem_short_iget.3.rst
+   shmem_short_iput.3.rst
+   shmem_short_max_to_all.3.rst
+   shmem_short_min_to_all.3.rst
+   shmem_short_or_to_all.3.rst
+   shmem_short_p.3.rst
+   shmem_short_prod_to_all.3.rst
+   shmem_short_put.3.rst
+   shmem_short_put_nbi.3.rst
+   shmem_short_sum_to_all.3.rst
+   shmem_short_wait.3.rst
+   shmem_short_wait_until.3.rst
+   shmem_short_xor_to_all.3.rst
+   shmem_swap.3.rst
+   shmem_test_lock.3.rst
+   shmem_udcflush.3.rst
+   shmem_udcflush_line.3.rst
+   shmem_wait.3.rst
+   shmem_wait_until.3.rst
+   shmemalign.3.rst
+   shrealloc.3.rst
+   start_pes.3.rst
diff --git a/docs/man-openshmem/man3/intro_shmem.3.rst b/docs/man-openshmem/man3/intro_shmem.3.rst
new file mode 100644
index 00000000000..b810f433559
--- /dev/null
+++ b/docs/man-openshmem/man3/intro_shmem.3.rst
@@ -0,0 +1,738 @@
+.. _intro_shmem:
+
+
+intro_shmem
+===========
+
+.. include_body
+
+intro_shmem - Introduction to the OpenSHMEM programming model
+
+
+DESCRIPTION
+-----------
+
+The SHMEM programming model consists of library routines that provide
+low-latency, high-bandwidth communication for use in highly parallelized
+scalable programs. The routines in the OpenSHMEM application programming
+interface (API) provide a programming model for exchanging data between
+cooperating parallel processes. The resulting programs are similar in
+style to Message Passing Interface (MPI) programs. The SHMEM API can be
+used either alone or in combination with MPI routines in the same
+parallel program.
+
+An OpenSHMEM program is SPMD (single program, multiple data) in style.
+The SHMEM processes, called processing elements or PEs, all start at the
+same time and they all run the same program. Usually the PEs perform
+computation on their own subdomains of the larger problem and
+periodically communicate with other PEs to exchange information on which
+the next computation phase depends.
+
+The OpenSHMEM routines minimize the overhead associated with data
+transfer requests, maximize bandwidth and minimize data latency. Data
+latency is the period of time that starts when a PE initiates a transfer
+of data and ends when a PE can use the data. OpenSHMEM routines support
+remote data transfer through put operations, which transfer data to a
+different PE, get operations, which transfer data from a different PE,
+and remote pointers, which allow direct references to data objects owned
+by another PE. Other operations supported are collective broadcast and
+reduction, barrier synchronization, and atomic memory operations. An
+atomic memory operation is an atomic read-and-update operation, such as
+a fetch-and-increment, on a remote or local data object.
+
+
+OPENSHMEM ROUTINES
+------------------
+
+This section lists the significant OpenSHMEM message-passing routines.
+
+PE queries
+
+..
+
+   * C/C++ only:
+
+      * *\_num_pes*\ (3)
+
+      * *\_my_pe*\ (3)
+
+   * Fortran only:
+
+      * *NUM_PES*\ (3)
+
+      * *MY_PE*\ (3)
+
+Elemental data put routines
+
+..
+
+   * C/C++ only:
+
+      * :ref:`shmem_double_p`\ (3)
+
+      * :ref:`shmem_float_p`\ (3)
+
+      * :ref:`shmem_int_p`\ (3)
+
+      * :ref:`shmem_long_p`\ (3)
+
+      * :ref:`shmem_short_p`.*\ (3)
+
+Block data put routines
+
+..
+
+   * C/C++ and Fortran:
+
+      * :ref:`shmem_put32`\ (3)
+
+      * :ref:`shmem_put64`\ (3)
+
+      * :ref:`shmem_put128`\ (3)
+
+   * C/C++ only:
+
+      * :ref:`shmem_double_put`\ (3)
+
+      * :ref:`shmem_float_put`\ (3)
+
+      * :ref:`shmem_int_put`\ (3)
+
+      * :ref:`shmem_long_put`\ (3)
+
+      * :ref:`shmem_short_put`.*\ (3)
+
+   * Fortran only:
+
+      * shmem_complex_put\ (3)
+
+      * shmem_integer_put\ (3)
+
+      * shmem_logical_put\ (3)
+
+      * shmem_real_put\ (3)
+
+Elemental data get routines
+
+..
+
+   * C/C++ only:
+
+      * :ref:`shmem_double_g`\ (3)
+
+      * :ref:`shmem_float_g`\ (3)
+
+      * :ref:`shmem_int_g`\ (3)
+
+      * :ref:`shmem_long_g`\ (3)
+
+      * :ref:`shmem_short_g`\ (3)
+
+Block data get routines
+
+   * C/C++ and Fortran:
+
+      * :ref:`shmem_get32`\ (3)
+
+      * :ref:`shmem_get64`\ (3)
+
+      * :ref:`shmem_get128`\ (3)
+
+   * C/C++ only:
+
+      * :ref:`shmem_double_get`\ (3)
+
+      * :ref:`shmem_float_get`\ (3)
+
+      * :ref:`shmem_int_get`\ (3)
+
+      * :ref:`shmem_long_get`\ (3)
+
+      * :ref:`shmem_short_get`\ (3)
+
+   * Fortran only:
+
+      * shmem_complex_get\ (3)
+
+      * shmem_integer_get\ (3)
+
+      * shmem_logical_get\ (3)
+
+      * shmem_real_get\ (3)
+
+Strided put routines
+
+   * C/C++ and Fortran:
+
+      * :ref:`shmem_iput32`\ (3)
+
+      * :ref:`shmem_iput64`\ (3)
+
+      * :ref:`shmem_iput128`\ (3)
+
+   * C/C++ only:
+
+      * :ref:`shmem_double_iput`\ (3)
+
+      * :ref:`shmem_float_iput`\ (3)
+
+      * :ref:`shmem_int_iput`\ (3)
+
+      * :ref:`shmem_long_iput`\ (3)
+
+      * :ref:`shmem_short_iput`\ (3)
+
+   * Fortran only:
+
+      * shmem_complex_iput\ (3)
+
+      * shmem_integer_iput\ (3)
+
+      * shmem_logical_iput\ (3)
+
+      * shmem_real_iput\ (3)
+
+Strided get routines
+
+..
+
+   * C/C++ and Fortran:
+
+      * :ref:`shmem_iget32`\ (3)
+
+      * :ref:`shmem_iget64`\ (3)
+
+      * :ref:`shmem_iget128`\ (3)
+
+   * C/C++ only:
+
+      * :ref:`shmem_double_iget`\ (3)
+
+      * :ref:`shmem_float_iget`\ (3)
+
+      * :ref:`shmem_int_iget`\ (3)
+
+      * :ref:`shmem_long_iget`\ (3)
+
+      * :ref:`shmem_short_iget`\ (3)
+
+   * Fortran only:
+
+      * shmem_complex_iget\ (3)
+
+      * shmem_integer_iget\ (3)
+
+      * shmem_logical_iget\ (3)
+
+      * shmem_real_iget\ (3)
+
+Point-to-point synchronization routines
+
+   * C/C++ only:
+
+      * :ref:`shmem_int_wait`\ (3)
+
+      * :ref:`shmem_int_wait_until`\ (3)
+
+      * :ref:`shmem_long_wait`\ (3)
+
+      * :ref:`shmem_long_wait_until`\ (3)
+
+      * :ref:`shmem_longlong_wait`\ (3)
+
+      * :ref:`shmem_longlong_wait_until`\ (3)
+
+      * :ref:`shmem_short_wait`\ (3)
+
+      * :ref:`shmem_short_wait_until`\ (3)
+
+   * Fortran:
+
+      * shmem_int4_wait\ (3)
+
+      * shmem_int4_wait_until\ (3)
+
+      * shmem_int8_wait\ (3)
+
+      * shmem_int8_wait_until\ (3)
+
+Barrier synchronization routines
+
+..
+
+   * C/C++ and Fortran:
+
+      * :ref:`shmem_barrier_all`\ (3)
+
+      * :ref:`shmem_barrier`\ (3)
+
+Atomic memory fetch-and-operate (fetch-op) routines
+
+   * C/C++ and Fortran:
+
+      * :ref:`shmem_swap`
+
+Reduction routines
+
+   * C/C++ only:
+
+      * :ref:`shmem_int_and_to_all`\ (3)
+
+      * :ref:`shmem_long_and_to_all`\ (3)
+
+      * :ref:`shmem_longlong_and_to_all`\ (3)
+
+      * :ref:`shmem_short_and_to_all`\ (3)
+
+      * :ref:`shmem_double_max_to_all`\ (3)
+
+      * :ref:`shmem_float_max_to_all`\ (3)
+
+      * :ref:`shmem_int_max_to_all`\ (3)
+
+      * :ref:`shmem_long_max_to_all`\ (3)
+
+      * :ref:`shmem_longlong_max_to_all`\ (3)
+
+      * :ref:`shmem_short_max_to_all`\ (3)
+
+      * :ref:`shmem_double_min_to_all`\ (3)
+
+      * :ref:`shmem_float_min_to_all`\ (3)
+
+      * :ref:`shmem_int_min_to_all`\ (3)
+
+      * :ref:`shmem_long_min_to_all`\ (3)
+
+      * :ref:`shmem_longlong_min_to_all`\ (3)
+
+      * :ref:`shmem_short_min_to_all`\ (3)
+
+      * :ref:`shmem_double_sum_to_all`\ (3)
+
+      * :ref:`shmem_float_sum_to_all`\ (3)
+
+      * :ref:`shmem_int_sum_to_all`\ (3)
+
+      * :ref:`shmem_long_sum_to_all`\ (3)
+
+      * :ref:`shmem_longlong_sum_to_all`\ (3)
+
+      * :ref:`shmem_short_sum_to_all`\ (3)
+
+      * :ref:`shmem_double_prod_to_all`\ (3)
+
+      * :ref:`shmem_float_prod_to_all`\ (3)
+
+      * :ref:`shmem_int_prod_to_all`\ (3)
+
+      * :ref:`shmem_long_prod_to_all`\ (3)
+
+      * :ref:`shmem_longlong_prod_to_all`\ (3)
+
+      * :ref:`shmem_short_prod_to_all`\ (3)
+
+      * :ref:`shmem_int_or_to_all`\ (3)
+
+      * :ref:`shmem_long_or_to_all`\ (3)
+
+      * :ref:`shmem_longlong_or_to_all`\ (3)
+
+      * :ref:`shmem_short_or_to_all`\ (3)
+
+      * :ref:`shmem_int_xor_to_all`\ (3)
+
+      * :ref:`shmem_long_xor_to_all`\ (3)
+
+      * :ref:`shmem_longlong_xor_to_all`\ (3)
+
+      * :ref:`shmem_short_xor_to_all`\ (3)
+
+   * Fortran only:
+
+      * shmem_int4_and_to_all\ (3)
+
+      * shmem_int8_and_to_all\ (3)
+
+      * shmem_real4_max_to_all\ (3)
+
+      * shmem_real8_max_to_all\ (3)
+
+      * shmem_int4_max_to_all\ (3)
+
+      * shmem_int8_max_to_all\ (3)
+
+      * shmem_real4_min_to_all\ (3)
+
+      * shmem_real8_min_to_all\ (3)
+
+      * shmem_int4_min_to_all\ (3)
+
+      * shmem_int8_min_to_all\ (3)
+
+      * shmem_real4_sum_to_all\ (3)
+
+      * shmem_real8_sum_to_all\ (3)
+
+      * shmem_int4_sum_to_all\ (3)
+
+      * shmem_int8_sum_to_all\ (3)
+
+      * shmem_real4_prod_to_all\ (3)
+
+      * shmem_real8_prod_to_all\ (3)
+
+      * shmem_int4_prod_to_all\ (3)
+
+      * shmem_int8_prod_to_all\ (3)
+
+      * shmem_int4_or_to_all\ (3)
+
+      * shmem_int8_or_to_all\ (3)
+
+      * shmem_int4_xor_to_all\ (3)
+
+      * shmem_int8_xor_to_all\ (3)
+
+Broadcast routines
+
+..
+
+   * C/C++ and Fortran:
+
+      * :ref:`shmem_broadcast32`\ (3)
+
+      * :ref:`shmem_broadcast64`\ (3)
+
+Cache management routines
+
+..
+
+   * C/C++ and Fortran:
+
+      * :ref:`shmem_udcflush`\ (3)
+
+      * :ref:`shmem_udcflush_line`\ (3)
+
+Byte-granularity block put routines
+
+..
+
+   * C/C++ and Fortran
+
+      * :ref:`shmem_putmem`\ (3)
+
+      * :ref:`shmem_getmem`\ (3)
+
+   * Fortran only:
+
+      * shmem_character_put\ (3)
+
+      * shmem_character_get\ (3)
+
+Collect routines
+
+   * C/C++ and Fortran:
+
+      * :ref:`shmem_collect32`\ (3)
+
+      * :ref:`shmem_collect64`\ (3)
+
+      * :ref:`shmem_fcollect32`\ (3)
+
+      * :ref:`shmem_fcollect64`\ (3)
+
+Atomic memory fetch-and-operate (fetch-op) routines
+
+   * C/C++ only:
+
+      * :ref:`shmem_double_swap`\ (3)
+
+      * :ref:`shmem_float_swap`\ (3)
+
+      * :ref:`shmem_int_cswap`\ (3)
+
+      * :ref:`shmem_int_fadd`\ (3)
+
+      * :ref:`shmem_int_finc`\ (3)
+
+      * :ref:`shmem_int_swap`\ (3)
+
+      * :ref:`shmem_long_cswap`\ (3)
+
+      * :ref:`shmem_long_fadd`\ (3)
+
+      * :ref:`shmem_long_finc`\ (3)
+
+      * :ref:`shmem_long_swap`\ (3)
+
+      * :ref:`shmem_longlong_cswap`\ (3)
+
+      * :ref:`shmem_longlong_fadd`\ (3)
+
+      * :ref:`shmem_longlong_finc`\ (3)
+
+      * :ref:`shmem_longlong_swap`\ (3)
+
+   * Fortran only:
+
+      * shmem_int4_cswap\ (3)
+
+      * shmem_int4_fadd\ (3)
+
+      * shmem_int4_finc\ (3)
+
+      * shmem_int4_swap\ (3)
+
+      * shmem_int8_swap\ (3)
+
+      * shmem_real4_swap\ (3)
+
+      * shmem_real8_swap\ (3)
+
+      * shmem_int8_cswap\ (3)
+
+Atomic memory operation routines
+
+   * Fortran only:
+
+      * shmem_int4_add\ (3)
+
+      * shmem_int4_inc\ (3)
+
+Remote memory pointer function
+
+   * C/C++ and Fortran:
+
+      * :ref:`shmem_ptr`\ (3)
+
+Reduction routines
+
+   * C/C++ only:
+
+      * :ref:`shmem_longdouble_max_to_all`\ (3)
+
+      * :ref:`shmem_longdouble_min_to_all`\ (3)
+
+      * :ref:`shmem_longdouble_prod_to_all`\ (3)
+
+      * :ref:`shmem_longdouble_sum_to_all`\ (3)
+
+   * Fortran only:
+
+      * shmem_real16_max_to_all\ (3)
+
+      * shmem_real16_min_to_all\ (3)
+
+      * shmem_real16_prod_to_all\ (3)
+
+      * shmem_real16_sum_to_all\ (3)
+
+Accessibility query routines
+
+   * C/C++ and Fortran:
+
+      * :ref:`shmem_pe_accessible`\ (3)
+
+      * :ref:`shmem_addr_accessible`\ (3)
+
+Symmetric Data Objects
+
+Consistent with the SPMD nature of the OpenSHMEM programming model is
+the concept of symmetric data objects. These are arrays or variables
+that exist with the same size, type, and relative address on all PEs.
+Another term for symmetric data objects is "remotely accessible data
+objects". In the interface definitions for OpenSHMEM data transfer
+routines, one or more of the parameters are typically required to be
+symmetric or remotely accessible.
+
+The following kinds of data objects are symmetric:
+
+   * Fortran data objects in common blocks or with the SAVE
+     attribute. These data objects must not be defined in a dynamic
+     shared object (DSO).
+
+   * Non-stack C and C++ variables. These data objects must not be
+     defined in a DSO.
+
+   * Fortran arrays allocated with *shpalloc*\ (3)
+
+   * C and C++ data allocated by *shmalloc*\ (3)
+
+..
+
+Collective Routines
+   Some SHMEM routines, for example, shmem_broadcast\ (3) and
+   :ref:`shmem_float_sum_to_all`\ (3), are classified as collective routines
+   because they distribute work across a set of PEs. They must be called
+   concurrently by all PEs in the active set defined by the PE_start,
+   logPE_stride, PE_size argument triplet. The following man pages
+   describe the OpenSHMEM collective routines:
+
+   * shmem_and\ (3)
+
+   * :ref:`shmem_barrier`\ (3)
+
+   * shmem_broadcast\ (3)
+
+   * shmem_collect\ (3)
+
+   * shmem_max\ (3)
+
+   * shmem_min\ (3)
+
+   * shmem_or\ (3)
+
+   * shmem_prod\ (3)
+
+   * shmem_sum\ (3)
+
+   * shmem_xor\ (3)
+
+
+USING THE SYMMETRIC WORK ARRAY, PSYNC
+-------------------------------------
+
+Multiple pSync arrays are often needed if a particular PE calls as
+OpenSHMEM collective routine twice without intervening barrier
+synchronization. Problems would occur if some PEs in the active set for
+call 2 arrive at call 2 before processing of call 1 is complete by all
+PEs in the call 1 active set. You can use :ref:`shmem_barrier`\ (3) or
+:ref:`shmem_barrier_all`\ (3) to perform a barrier synchronization between
+consecutive calls to OpenSHMEM collective routines.
+
+There are two special cases:
+
+*
+   The :ref:`shmem_barrier`\ (3) routine allows the same pSync array to be
+   used on consecutive calls as long as the active PE set does not
+   change.
+
+*
+   If the same collective routine is called multiple times with the same
+   active set, the calls may alternate between two pSync arrays. The
+   SHMEM routines guarantee that a first call is completely finished by
+   all PEs by the time processing of a third call begins on any PE.
+
+Because the SHMEM routines restore pSync to its original contents,
+multiple calls that use the same pSync array do not require that pSync
+be reinitialized after the first call.
+
+
+SHMEM ENVIRONMENT VARIABLES
+---------------------------
+
+This section lists the significant SHMEM environment variables.
+
+* **SMA_VERSION** print the library version at start-up.
+
+* **SMA_INFO** print helpful text about all these environment
+  variables.
+
+* **SMA_SYMMETRIC_SIZE** number of bytes to allocate for the symmetric
+  heap.
+
+* **SMA_DEBUG** enable debugging messages.
+
+The first call to SHMEM must be *start_pes*\ (3). This routines
+initialize the SHMEM runtime.
+
+Calling any other SHMEM routines beforehand has undefined behavior.
+Multiple calls to this routine is not allowed.
+
+
+COMPILING AND RUNNING OPENSHMEM PROGRAMS
+----------------------------------------
+
+The OpenSHMEM specification is silent regarding how OpenSHMEM programs
+are compiled, linked and run. This section shows some examples of how
+wrapper programs could be utilized to compile and launch applications.
+The commands are styled after wrapper programs found in many MPI
+implementations.
+
+The following sample command line demonstrates running an OpenSHMEM
+Program using a wrapper script (**oshrun** in this case):
+
+* C/C++:
+
+.. code-block:: c++
+
+   oshcc c_program.c
+
+* FORTRAN:
+
+.. code-block:: fortran
+
+   oshfort fortran_program.f
+
+The following sample command line demonstrates running an OpenSHMEM
+Program assuming that the library provides a wrapper script for such
+purpose (named **oshrun** for this example):
+
+::
+
+   oshrun -n 32 ./a.out
+
+
+EXAMPLES
+--------
+
+**Example 1**: The following Fortran OpenSHMEM program directs all PEs
+to sum simultaneously the numbers in the VALUES variable across all PEs:
+
+.. code-block:: fortran
+
+   PROGRAM REDUCTION
+     REAL VALUES, SUM
+     COMMON /C/ VALUES
+     REAL WORK
+
+     CALL START_PES(0)
+     VALUES = MY_PE()
+     CALL SHMEM_BARRIER_ALL ! Synchronize all PEs
+     SUM = 0.0
+     DO I = 0, NUM_PES()-1
+       CALL SHMEM_REAL_GET(WORK, VALUES, 1, I) ! Get next value
+       SUM = SUM + WORK                ! Sum it
+     ENDDO
+     PRINT *, 'PE ', MY_PE(), ' COMPUTED SUM=', SUM
+     CALL SHMEM_BARRIER_ALL
+   END
+
+**Example 2**: The following C OpenSHMEM program transfers an array of
+10 longs from PE 0 to PE 1:
+
+.. code-block:: c
+
+   #include <mpp/shmem.h>
+
+   main() {
+     long source[10] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
+     static long target[10];
+
+     shmem_init();
+     if (shmem_my_pe() == 0) {
+       /* put 10 elements into target on PE 1 */
+       shmem_long_put(target, source, 10, 1);
+     }
+     shmem_barrier_all(); /* sync sender and receiver */
+     if (shmem_my_pe() == 1)
+       printf("target[0] on PE %d is %d\n", shmem_my_pe(), target[0]);
+   }
+
+
+.. seealso::
+   The following man pages also contain information on OpenSHMEM routines.
+   See the specific man pages for implementation information.
+
+   shmem_add\ (3) shmem_and\ (3) :ref:`shmem_barrier`\ (3)
+   :ref:`shmem_barrier_all`\ (3) shmem_broadcast\ (3) shmem_cache\ (3)
+   shmem_collect\ (3) shmem_cswap\ (3) shmem_fadd\ (3)
+   :ref:`shmem_fence`\ (3) shmem_finc\ (3) shmem_get\ (3) shmem_iget\ (3)
+   shmem_inc\ (3) shmem_iput\ (3) shmem_lock\ (3) shmem_max\ (3)
+   shmem_min\ (3) :ref:`shmem_my_pe`\ (3) shmem_or\ (3) shmem_prod\ (3)
+   shmem_put\ (3) :ref:`shmem_quiet`\ (3) :ref:`shmem_short_g`\ (3)
+   :ref:`shmem_short_p`\ (3) shmem_sum\ (3) :ref:`shmem_swap`\ (3)
+   :ref:`shmem_wait`\ (3) shmem_xor\ (3) :ref:`shmem_pe_accessible`\ (3)
+   :ref:`shmem_addr_accessible`\ (3) :ref:`shmem_init`\ (3) :ref:`shmem_malloc`\ (3)
+   *shmem_my_pe*\ (3) *shmem_n_pes*\ (3)
diff --git a/docs/man-openshmem/man3/shfree.3.rst b/docs/man-openshmem/man3/shfree.3.rst
new file mode 100644
index 00000000000..64f0250d6bf
--- /dev/null
+++ b/docs/man-openshmem/man3/shfree.3.rst
@@ -0,0 +1,9 @@
+.. _shfree:
+
+shfree
+======
+    .. include_body
+
+.. include:: ../man3/shmem_malloc.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmalloc.3.rst b/docs/man-openshmem/man3/shmalloc.3.rst
new file mode 100644
index 00000000000..220f19854a4
--- /dev/null
+++ b/docs/man-openshmem/man3/shmalloc.3.rst
@@ -0,0 +1,101 @@
+.. _shmalloc:
+
+
+shmalloc
+========
+
+.. include_body
+
+*shmalloc*\ (3), *shfree*\ (3), *shmemalign*\ (3), *shrealloc*\ (3) -
+Symmetric heap memory management functions.
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void *shmalloc(size_t size)
+
+   void shfree(void *ptr)
+
+   void *shrealloc(void *ptr, size_t size)
+
+   void *shmemalign(size_t alignment, size_t size)
+
+   extern long malloc_error
+
+
+DESCRIPTION
+-----------
+
+The **shmalloc** function returns a pointer to a block of at least size
+bytes suitably aligned for any use. This space is allocated from the
+symmetric heap (in contrast to *malloc*\ (3), which allocates from the
+private heap).
+
+The **shmemalign** function allocates a block in the symmetric heap that
+has a byte alignment specified by the alignment argument.
+
+The **shfree** function causes the block to which ptr points to, to be
+deallocated, that is, made available for further allocation. If ptr is a
+null pointer, no action occurs; otherwise, if the argument does not
+match a pointer earlier returned by a symmetric heap function, or if the
+space has already been deallocated, malloc_error is set to indicate the
+error, and shfree returns.
+
+The **shrealloc** function changes the size of the block to which ptr
+points to, to the size (in bytes) specified by size.
+
+The contents of the block are unchanged up to the lesser of the new and
+old sizes. If the new size is larger, the value of the newly allocated
+portion of the block is indeterminate. If ptr is a null pointer, the
+shrealloc function behaves like the shmalloc function for the specified
+size. If size is 0 and ptr is not a null pointer, the block to which it
+points to is freed. Otherwise, if ptr does not match a pointer earlier
+returned by a symmetric heap function, or if the space has already been
+deallocated, the malloc_error variable is set to indicate the error, and
+shrealloc returns a null pointer. If the space cannot be allocated, the
+block to which ptr points to is unchanged.
+
+The shmalloc, shfree, and shrealloc functions are provided so that
+multiple PEs in an application can allocate symmetric, remotely
+accessible memory blocks. These memory blocks can then be used with
+(shmem) communication routines. Each of these functions call the
+:ref:`shmem_barrier_all`\ (3) function before returning; this ensures that
+all PEs participate in the memory allocation, and that the memory on
+other PEs can be used as soon as the local PE returns.
+
+The user is responsible for calling these functions with identical
+argument(s) on all PEs; if differing size arguments are used, subsequent
+calls may not return the same symmetric heap address on all PEs.
+
+
+NOTES
+-----
+
+The total size of the symmetric heap is determined at job startup. One
+can adjust the size of the heap using the SHMEM_SYMMETRIC_HEAP_SIZE
+environment variable. See the *intro_shmem*\ (3) man page for further
+details. The shmalloc, shfree, and shrealloc functions differ from the
+private heap allocation functions in that all PEs in an application must
+call them (a barrier is used to ensure this).
+
+
+RETURN VALUES
+-------------
+
+The **shmalloc** function returns a pointer to the allocated space
+(which should be identical on all PEs); otherwise, it returns a null
+pointer (with malloc_error set). The **shfree** function returns no
+value. The **shrealloc** function returns a pointer to the allocated
+space (which may have moved); otherwise, it returns a null pointer (with
+malloc_error set).
+
+
+.. seealso::
+   *intro_shmem*\ (3) *my_pe*\ (3) *start_pes*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_addr_accessible.3.rst b/docs/man-openshmem/man3/shmem_addr_accessible.3.rst
new file mode 100644
index 00000000000..ea423b3ba80
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_addr_accessible.3.rst
@@ -0,0 +1,60 @@
+.. _shmem_addr_accessible:
+
+
+shmem_addr_accessible
+=====================
+
+.. include_body
+
+:ref:`shmem_addr_accessible` - Indicates if an address is accessible via
+OpenSHMEM operations from the specified remote PE.
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   int shmem_addr_accessible(const void *addr, int pe)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   LOGICAL LOG, SHMEM_ADDR_ACCESSIBLE
+   INTEGER pe
+
+   LOG = SHMEM_ADDR_ACCESSIBLE(addr, pe)
+
+
+DESCRIPTION
+-----------
+
+:ref:`shmem_addr_accessible` is a query function that indicates whether a local
+address is accessible via SHMEM operations from the specified remote PE.
+
+This function verifies that the remote PE is accessible via SHMEM data
+transfer functions from the local PE, and that the specified address is
+in a symmetric data segment with respect to the remote PE.
+
+
+RETURN VALUES
+-------------
+
+C: The return value is 1 if addr is a symmetric data object and
+accessible via SHMEM operations from the specified remote PE; otherwise,
+it is 0.
+
+Fortran: The return value is .TRUE. if addr is a symmetric data object
+and accessible via SHMEM operations from the specified remote PE;
+otherwise, it is .FALSE..
+
+
+.. seealso::
+   *intro_shmem*\ (3) *shmem_pe_accessible*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_align.3.rst b/docs/man-openshmem/man3/shmem_align.3.rst
new file mode 100644
index 00000000000..2a6c00854b5
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_align.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_align:
+
+shmem_align
+===========
+    .. include_body
+
+.. include:: ../man3/shmem_malloc.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_alltoall32.3.rst b/docs/man-openshmem/man3/shmem_alltoall32.3.rst
new file mode 100644
index 00000000000..b0da99d54d9
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_alltoall32.3.rst
@@ -0,0 +1,242 @@
+.. _shmem_alltoall32:
+
+
+shmem_alltoall32
+================
+
+.. include_body
+
+:ref:`shmem_alltoall32`\ (3), :ref:`shmem_alltoall64`\ (3),
+:ref:`shmem_alltoalls32`\ (3), :ref:`shmem_alltoalls64`\ (3) - collective routine
+where each PE exchanges a fixed amount of data with all other PEs in the
+Active set
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void shmem_alltoall32(void *target, const void *source,
+     size_t nelems, int PE_start, int logPE_stride,
+     int PE_size, long *pSync)
+
+   void shmem_alltoall64(void *target, const void *source,
+     size_t nelems, int PE_start, int logPE_stride,
+     int PE_size, long *pSync)
+
+   void shmem_alltoalls32(void *target, const void *source,
+     ptrdiff_t dst, ptrdiff_t sst,
+     size_t nelems, int PE_start, int logPE_stride,
+     int PE_size, long *pSync)
+
+   void shmem_alltoalls64(void *target, const void *source,
+     ptrdiff_t dst, ptrdiff_t sst,
+     size_t nelems, int PE_start, int logPE_stride,
+     int PE_size, long *pSync)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER dst, sst, nelems, PE_root, PE_start, logPE_stride, PE_size
+   INTEGER pSync(SHMEM_ALLTOALL_SYNC_SIZE)
+
+   CALL SHMEM_ALLTOALL32(target, source, nelems,
+   & PE_start, logPE_stride, PE_size, fIpSync)
+
+   CALL SHMEM_ALLTOALL64(target, source, nelems,
+   & PE_start, logPE_stride, PE_size, pSync)
+
+   CALL SHMEM_ALLTOALLS32(target, source, dst, sst, nelems,
+   & PE_start, logPE_stride, PE_size, pSync)
+
+   CALL SHMEM_ALLTOALLS64(target, source, dst, sst, nelems,
+   & PE_start, logPE_stride, PE_size, pSync)
+
+
+DESCRIPTION
+-----------
+
+The shmem_alltoalls routines are collective routines. Each PE in the
+Active set exchanges nelems strided data elements of size 32 bits (for
+shmem_alltoalls32) or 64 bits (for shmem_alltoalls64) with all other PEs
+in the set. Both strides, dst and sst, must be greater than or equal to
+1. The sst*jth block sent from PE i to PE j is placed in the dst*ith
+block of the dest data object on PE j. As with all OpenSHMEM collective
+routines, these routines assume that only PEs in the Active set call the
+routine. If a PE not in the Active set calls an OpenSHMEM collective
+routine, undefined behavior results. The values of arguments dst, sst,
+nelems, PE_start, logPE_stride, and PE_size must be equal on all PEs in
+the Active set. The same dest and source data objects, and the same
+pSync work array must be passed to all PEs in the Active set. Before any
+PE calls to a shmem_alltoalls routine, the following conditions must
+exist (synchronization via a barrier or some other method is often
+needed to ensure this): The pSync array on all PEs in the Active set is
+not still in use from a prior call to a shmem_alltoalls routine. The
+dest data object on all PEs in the Active set is ready to accept the
+shmem_alltoalls data. Upon return from a shmem_alltoalls routine, the
+following is true for the local PE: Its dest symmetric data object is
+completely updated and the data has been copied out of the source data
+object. The values in the pSync array are restored to the original
+values.
+
+The arguments are as follows:
+
+A symmetric data object with one of the following data types:
+
+   :ref:`shmem_alltoall32`: Any noncharacter type that
+      has an element size of 32 bits. No Fortran derived types or C/C++
+      structures are allowed.
+
+   :ref:`shmem_alltoall64`: Any noncharacter type that has an element size
+      of 64 bits. No Fortran derived types or C/C++ structures are
+      allowed.
+
+target A symmetric data object large enough to receive the combined
+total of nelems elements from each PE in the Active set.
+
+source
+   A symmetric data object that contains nelems elements of data for
+   each PE in the Active set, ordered according to destination PE.
+
+dst
+   The stride between consecutive elements of the dest data object. The
+   stride is scaled by the element size. A value of 1 indicates
+   contiguous data. dst must be of type ptrdiff_t. If you are using
+   Fortran, it must be a default integer value.
+
+sst
+   The stride between consecutive elements of the source data object.
+   The stride is scaled by the element size. A value of 1 indicates
+   contiguous data. sst must be of type ptrdiff_t. If you are using
+   Fortran, it must be a default integer value.
+
+nelems
+   The number of elements to exchange for each PE. nelems must be of
+   type size_t for C/C++. If you are using Fortran, it must be a default
+   integer value
+
+PE_start
+   The lowest virtual PE number of the active set of PEs. PE_start must
+   be of type integer. If you are using Fortran, it must be a default
+   integer value.
+
+logPE_stride
+   The log (base 2) of the stride between consecutive virtual PE numbers
+   in the active set. log_PE_stride must be of type integer. If you are
+   using Fortran, it must be a default integer value.
+
+PE_size
+   The number of PEs in the active set. PE_size must be of type integer.
+   If you are using Fortran, it must be a default integer value.
+
+pSync
+   A symmetric work array. In C/C++, pSync must be of type long and size
+   \_SHMEM_ALLTOALL_SYNC_SIZE. In Fortran, pSync must be of type integer
+   and size SHMEM_ALLTOALL_SYNC_SIZE. Every element of this array must
+   be initialized with the value \_SHMEM_SYNC_VALUE (in C/C++) or
+   SHMEM_SYNC_VALUE (in Fortran) before any of the PEs in the active set
+   enter shmem_barrier().
+
+Upon return from a shmem_alltoalls routine, the following is true for
+the local PE: Its dest symmetric data object is completely updated and
+the data has been copied out of the source data object. The values in
+the pSync array are restored to the original values.
+
+The values of arguments PE_root, PE_start, logPE_stride, and PE_size
+must be equal on all PEs in the active set. The same target and source
+data objects and the same pSync work array must be passed to all PEs in
+the active set.
+
+Before any PE calls a alltoall routine, you must ensure that the
+following conditions exist (synchronization via a barrier or some other
+method is often needed to ensure this): The pSync array on all PEs in
+the active set is not still in use from a prior call to a alltoall
+routine. The target array on all PEs in the active set is ready to
+accept the alltoall data.
+
+
+NOTES
+-----
+
+The terms collective and symmetric are defined in *intro_shmem*\ (3).
+
+All SHMEM alltoall routines restore pSync to its original contents.
+Multiple calls to SHMEM routines that use the same pSync array do not
+require that pSync be reinitialized after the first call.
+
+You must ensure the that the pSync array is not being updated by any PE
+in the active set while any of the PEs participates in processing of a
+SHMEM broadcast routine. Be careful to avoid these situations: If the
+pSync array is initialized at run time, some type of synchronization is
+needed to ensure that all PEs in the working set have initialized pSync
+before any of them enter a SHMEM routine called with the pSync
+synchronization array. A pSync array may be reused on a subsequent SHMEM
+broadcast routine only if none of the PEs in the active set are still
+processing a prior SHMEM alltoall routine call that used the same pSync
+array. In general, this can be ensured only by doing some type of
+synchronization. However, in the special case of SHMEM routines being
+called with the same active set, you can allocate two pSync arrays and
+alternate between them on successive calls.
+
+
+EXAMPLES
+--------
+
+C/C++ example:
+
+.. code-block:: c++
+
+   #include <shmem.h>
+   #include <stdio.h>
+
+   long pSync[SHMEM_ALLTOALL_SYNC_SIZE];
+   int main(void)
+   {
+   int64_t *source, *dest;
+   int i, count, pe;
+   shmem_init();
+   count = 2;
+   dest = (int64_t*) shmem_malloc(count * shmem_n_pes() * sizeof(int64_t));
+   source = (int64_t*) shmem_malloc(count * shmem_n_pes() * sizeof(int64_t));
+   /* assign source values */
+   for (pe=0; pe <shmem_n_pes(); pe++){
+   for (i=0; i<count; i++){
+   source[(pe*count)+i] = shmem_my_pe() + pe;
+   dest[(pe*count)+i] = 9999;
+   }
+   }
+   for (i=0; i< SHMEM_ALLTOALLS_SYNC_SIZE; i++) {
+   pSync[i] = SHMEM_SYNC_VALUE;
+   }
+   /* wait for all PEs to initialize pSync */
+   shmem_barrier_all();
+   /* alltoalls on all PES */
+   shmem_alltoalls64(dest, source, 1, 1, count, 0, 0, shmem_n_pes(), pSync);
+   /* verify results */
+   for (pe=0; pe<shmem_n_pes(); pe++) {
+   for (i=0; i<count; i++){
+   if (dest[(pe*count)+i] != shmem_my_pe() + pe) {
+   printf("[%d] ERROR: dest[%d]=%ld, should be %d,
+   shmem_my_pe(),(pe*count)+i,dest[(pe*count)+i],
+   shmem_n_pes() + pe);
+   }
+   }
+   }
+   shmem_barrier_all();
+   shmem_free(dest);
+   shmem_free(source);
+   shmem_finalize();
+   return 0;
+   }
+
+
+.. seealso::
+   *intro_shmem*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_alltoall64.3.rst b/docs/man-openshmem/man3/shmem_alltoall64.3.rst
new file mode 100644
index 00000000000..4a8eab1186f
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_alltoall64.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_alltoall64:
+
+shmem_alltoall64
+================
+    .. include_body
+
+.. include:: ../man3/shmem_alltoall32.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_alltoalls32.3.rst b/docs/man-openshmem/man3/shmem_alltoalls32.3.rst
new file mode 100644
index 00000000000..26ce682ab90
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_alltoalls32.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_alltoalls32:
+
+shmem_alltoalls32
+=================
+    .. include_body
+
+.. include:: ../man3/shmem_alltoall32.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_alltoalls64.3.rst b/docs/man-openshmem/man3/shmem_alltoalls64.3.rst
new file mode 100644
index 00000000000..bdf9c87bce8
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_alltoalls64.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_alltoalls64:
+
+shmem_alltoalls64
+=================
+    .. include_body
+
+.. include:: ../man3/shmem_alltoall32.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_barrier.3.rst b/docs/man-openshmem/man3/shmem_barrier.3.rst
new file mode 100644
index 00000000000..e2ca4a64ccc
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_barrier.3.rst
@@ -0,0 +1,120 @@
+.. _shmem_barrier:
+
+
+shmem_barrier
+=============
+
+.. include_body
+
+:ref:`shmem_barrier` - Performs a barrier operation on a subset of processing
+elements (PEs).
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void shmem_barrier(int PE_start, int logPE_stride, int PE_size,
+     long *pSync)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER PE_start, logPE_stride, PE_size
+   INTEGER pSync(SHMEM_BARRIER_SYNC_SIZE)
+
+   CALL SHMEM_BARRIER(PE_start, logPE_stride, PE_size, pSync)
+
+
+DESCRIPTION
+-----------
+
+The :ref:`shmem_barrier` routine does not return until the subset of PEs
+specified by **PE_start**, **logPE_stride** and **PE_size**, has entered
+this routine at the same point of the execution path.
+
+As with all SHMEM collective routines, each of these routines assumes
+that only PEs in the active set call the routine. If a PE not in the
+active set calls a SHMEM collective routine, undefined behavior results.
+
+The arguments are as follows:
+
+PE_start
+   The lowest virtual PE number of the active set of PEs. PE_start must
+   be of type integer. If you are using Fortran, it must be a default
+   integer value.
+
+logPE_stride
+   The log (base 2) of the stride between consecutive virtual PE numbers
+   in the active set. logPE_stride must be of type integer. If you are
+   using Fortran, it must be a default integer value.
+
+PE_size
+   The number of PEs in the active set. PE_size must be of type integer.
+   If you are using Fortran, it must be a default integer value.
+
+pSync
+   A symmetric work array. In C/C++, pSync must be of type int and size
+   \_SHMEM_BARRIER_SYNC_SIZE. In Fortran, pSync must be of type integer
+   and size SHMEM_BARRIER_SYNC_SIZE. If you are using Fortran, it must
+   be a default integer type. Every element of this array must be
+   initialized to 0 before any of the PEs in the active set enter
+   :ref:`shmem_barrier` the first time.
+
+The values of arguments PE_start, logPE_stride, and PE_size must be
+equal on all PEs in the active set. The same work array must be passed
+in pSync to all PEs in the active set.
+
+:ref:`shmem_barrier` ensures that all previously issued local stores and
+previously issued remote memory updates done by any of the PEs in the
+active set (by using SHMEM calls, for example shmem_put\ (3)) are
+complete before returning.
+
+The same pSync array may be reused on consecutive calls to :ref:`shmem_barrier`
+if the same active PE set is used.
+
+
+NOTES
+-----
+
+The term symmetric is defined in *intro_shmem*\ (3).
+
+If the pSync array is initialized at run time, be sure to use some type
+of synchronization, for example, a call to :ref:`shmem_barrier_all`\ (3),
+before calling :ref:`shmem_barrier` for the first time.
+
+If the active set does not change, :ref:`shmem_barrier` can be called
+repeatedly with the same pSync array. No additional synchronization
+beyond that implied by :ref:`shmem_barrier` itself is necessary in this case.
+
+
+EXAMPLES
+--------
+
+C/C++ example:
+
+.. code-block:: c++
+
+   shmem_barrier(PE_start, logPE_stride, size, pSync);
+
+Fortran example:
+
+.. code-block:: fortran
+
+   INTEGER PSYNC(SHMEM_BARRIER_SYNC_SIZE)
+   INTEGER PE_START, LOGPE_STRIDE, PE_SIZE, PSYNC
+   DATA PSYNC /SHMEM_BARRIER_SYNC_SIZE*0/
+
+   CALL SHMEM_BARRIER(PE_START, LOGPE_STRIDE, PE_SIZE, PSYNC)
+
+
+.. seealso::
+   *intro_shmem*\ (3) *shmem_barrier_all*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_barrier_all.3.rst b/docs/man-openshmem/man3/shmem_barrier_all.3.rst
new file mode 100644
index 00000000000..af5ca8ada87
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_barrier_all.3.rst
@@ -0,0 +1,61 @@
+.. _shmem_barrier_all:
+
+
+shmem_barrier_all
+=================
+
+.. include_body
+
+:ref:`shmem_barrier_all` - Suspends the execution of the calling PE until all
+other PEs issue a call to this particular shmem_barrier_all() statement.
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void shmem_barrier_all(void)
+
+Fortran:
+
+.. code-block:: fortran
+
+   include 'mpp/shmem.h'
+
+   CALL SHMEM_BARRIER_ALL
+
+
+DESCRIPTION
+-----------
+
+The :ref:`shmem_barrier_all` routine does not return until all other PEs have
+entered this routine at the same point of the execution path.
+
+Prior to synchronizing with other PEs, :ref:`shmem_barrier_all` ensures
+completion of all previously issued local memory stores and remote
+memory updates issued via SHMEM functions such as :ref:`shmem_put32`\ (3).
+
+
+EXAMPLES
+--------
+
+::
+
+   setup_data()
+   {
+     if (shmem_my_pe() == 0) {
+       setup();
+     }
+
+     /* All PEs wait for PE 0 to complete setup().  */
+     shmem_barrier_all();
+   }
+
+
+.. seealso::
+   *shmem_barrier*\ (3) *shmem_init*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_broadcast32.3.rst b/docs/man-openshmem/man3/shmem_broadcast32.3.rst
new file mode 100644
index 00000000000..a01ea243c88
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_broadcast32.3.rst
@@ -0,0 +1,196 @@
+.. _shmem_broadcast32:
+
+
+shmem_broadcast32
+=================
+
+.. include_body
+
+shmem_broadcast4\ (3), shmem_broadcast8\ (3),
+:ref:`shmem_broadcast32`\ (3), :ref:`shmem_broadcast64`\ (3) - Copy a data object
+from a designated PE to a target location on all other PEs of the active
+set.
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void shmem_broadcast32(void *target, const void *source,
+     size_t nelems, int PE_root, int PE_start, int logPE_stride,
+     int PE_size, long *pSync)
+
+   void shmem_broadcast64(void *target, const void *source,
+     size_t nelems, int PE_root, int PE_start, int logPE_stride,
+     int PE_size, long *pSync)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER nelems, PE_root, PE_start, logPE_stride, PE_size
+   INTEGER pSync(SHMEM_BCAST_SYNC_SIZE)
+
+   CALL SHMEM_BROADCAST4(target, source, nelems, PE_root,
+   & PE_start, logPE_stride, PE_size, fIpSync)
+
+   CALL SHMEM_BROADCAST8(target, source, nelems, PE_root,
+   & PE_start, logPE_stride, PE_size, pSync)
+
+   CALL SHMEM_BROADCAST32(target, source, nelems,
+   & PE_root, PE_start, logPE_stride, PE_size, pSync)
+
+   CALL SHMEM_BROADCAST64(target, source, nelems,
+   & PE_root, PE_start, logPE_stride, PE_size, pSync)
+
+
+DESCRIPTION
+-----------
+
+The broadcast routines write the data at address source of the PE
+specified by **PE_root** to address **target** on all other PEs in the
+active set. The active set of PEs is defined by the triplet
+**PE_start**, **logPE_stride** and **PE_size**. The data is not copied
+to the target address on the PE specified by **PE_root**. Before
+returning, the broadcast routines ensure that the elements of the pSync
+array are restored to their initial values.
+
+As with all SHMEM collective routines, each of these routines assumes
+that only PEs in the active set call the routine. If a PE not in the
+active set calls a SHMEM collective routine, undefined behavior results.
+
+The arguments are as follows:
+
+target
+   A symmetric data object with one of the following data types:
+
+   shmem_broadcast8, :ref:`shmem_broadcast64`: Any noncharacter type that
+      has an element size of 64 bits. No Fortran derived types or C/C++
+      structures are allowed.
+
+   :ref:`shmem_broadcast32`: Any noncharacter type that has an element size
+      of 32 bits. No Fortran derived types or C/C++ structures are
+      allowed.
+
+   shmem_broadcast4: Any noncharacter type that has an element size
+      of 32 bits.
+
+source
+   A symmetric data object that can be of any data type that is
+   permissible for the target argument.
+
+nelems
+   The number of elements in source. For :ref:`shmem_broadcast32` and
+   shmem_broadcast4, this is the number of 32-bit halfwords. nelems must
+   be of type integer. If you are using Fortran, it must be a default
+   integer value.
+
+PE_root
+   Zero-based ordinal of the PE, with respect to the active set, from
+   which the data is copied. Must be greater than or equal to 0 and less
+   than PE_size. PE_root must be of type integer. If you are using
+   Fortran, it must be a default integer value.
+
+PE_start
+   The lowest virtual PE number of the active set of PEs. PE_start must
+   be of type integer. If you are using Fortran, it must be a default
+   integer value.
+
+logPE_stride
+   The log (base 2) of the stride between consecutive virtual PE numbers
+   in the active set. log_PE_stride must be of type integer. If you are
+   using Fortran, it must be a default integer value.
+
+PE_size
+   The number of PEs in the active set. PE_size must be of type integer.
+   If you are using Fortran, it must be a default integer value.
+
+pSync
+   A symmetric work array. In C/C++, pSync must be of type long and size
+   \_SHMEM_BCAST_SYNC_SIZE. In Fortran, pSync must be of type integer
+   and size SHMEM_BCAST_SYNC_SIZE. Every element of this array must be
+   initialized with the value \_SHMEM_SYNC_VALUE (in C/C++) or
+   SHMEM_SYNC_VALUE (in Fortran) before any of the PEs in the active set
+   enter shmem_barrier().
+
+The values of arguments PE_root, PE_start, logPE_stride, and PE_size
+must be equal on all PEs in the active set. The same target and source
+data objects and the same pSync work array must be passed to all PEs in
+the active set.
+
+Before any PE calls a broadcast routine, you must ensure that the
+following conditions exist (synchronization via a barrier or some other
+method is often needed to ensure this): The pSync array on all PEs in
+the active set is not still in use from a prior call to a broadcast
+routine. The target array on all PEs in the active set is ready to
+accept the broadcast data.
+
+Upon return from a broadcast routine, the following are true for the
+local PE: If the current PE is not the root PE, the target data object
+is updated. The values in the pSync array are restored to the original
+values.
+
+
+NOTES
+-----
+
+The terms collective and symmetric are defined in *intro_shmem*\ (3).
+
+All SHMEM broadcast routines restore pSync to its original contents.
+Multiple calls to SHMEM routines that use the same pSync array do not
+require that pSync be reinitialized after the first call.
+
+You must ensure the that the pSync array is not being updated by any PE
+in the active set while any of the PEs participates in processing of a
+SHMEM broadcast routine. Be careful to avoid these situations: If the
+pSync array is initialized at run time, some type of synchronization is
+needed to ensure that all PEs in the working set have initialized pSync
+before any of them enter a SHMEM routine called with the pSync
+synchronization array. A pSync array may be reused on a subsequent SHMEM
+broadcast routine only if none of the PEs in the active set are still
+processing a prior SHMEM broadcast routine call that used the same pSync
+array. In general, this can be ensured only by doing some type of
+synchronization. However, in the special case of SHMEM routines being
+called with the same active set, you can allocate two pSync arrays and
+alternate between them on successive calls.
+
+
+EXAMPLES
+--------
+
+In the following examples, the call to :ref:`shmem_broadcast64` copies source
+on PE 4 to target on PEs 5, 6, and 7.
+
+C/C++ example:
+
+.. code-block:: c++
+
+   for (i=0; i < _SHMEM_BCAST_SYNC_SIZE; i++) {
+     pSync[i] = _SHMEM_SYNC_VALUE;
+   }
+   shmem_barrier_all(); /* Wait for all PEs to initialize pSync */
+   shmem_broadcast64(target, source, nelems, 0, 4, 0, 4, pSync);
+
+Fortran example:
+
+.. code-block:: fortran
+
+   INTEGER PSYNC(SHMEM_BCAST_SYNC_SIZE)
+   INTEGER TARGET, SOURCE, NELEMS, PE_ROOT, PE_START,
+   & LOGPE_STRIDE, PE_SIZE, PSYNC
+   COMMON /COM/ TARGET, SOURCE
+   DATA PSYNC /SHMEM_BCAST_SYNC_SIZE*SHMEM_SYNC_VALUE/
+
+   CALL SHMEM_BROADCAST64(TARGET, SOURCE, NELEMS, 0, 4, 0, 4,
+   & PSYNC)
+
+
+.. seealso::
+   *intro_shmem*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_broadcast64.3.rst b/docs/man-openshmem/man3/shmem_broadcast64.3.rst
new file mode 100644
index 00000000000..1ecfae7a2d6
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_broadcast64.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_broadcast64:
+
+shmem_broadcast64
+=================
+    .. include_body
+
+.. include:: ../man3/shmem_broadcast32.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_char_g.3.rst b/docs/man-openshmem/man3/shmem_char_g.3.rst
new file mode 100644
index 00000000000..e18e8d1a427
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_char_g.3.rst
@@ -0,0 +1,60 @@
+.. _shmem_char_g:
+
+
+shmem_char_g
+============
+
+.. include_body
+
+:ref:`shmem_char_g`\ (3), :ref:`shmem_float_g`\ (3), :ref:`shmem_int_g`\ (3),
+:ref:`shmem_long_g`\ (3), :ref:`shmem_short_g`\ (3), :ref:`shmem_longlong_g`\ (3),
+:ref:`shmem_longdouble_g`\ (3) - These routines provide a low latency
+mechanism to read basic types (char, short, int, float, double, long,
+long long, long double) from symmetric data objects on remote PEs.
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+
+   char shmem_char_g(const char *addr, int pe)
+
+   short shmem_short_g(const short *addr, int pe)
+
+   int shmem_int_g(const int *addr, int pe)
+
+   long shmem_long_g(const long *addr, int pe)
+
+   long shmem_longlong_g(const long long *addr, int pe)
+
+   float shmem_float_g(const float *addr, int pe)
+
+   double shmem_double_g(const double *addr, int pe)
+
+   long shmem_longdouble_g(const long double *addr, int pe)
+
+
+DESCRIPTION
+-----------
+
+These routines provide a very low latency get capability for single
+elements of most basic types.
+
+The arguments are as follows:
+
+addr
+   The remotely accessible array element or scalar data object which
+   will receive the data on the remote PE.
+
+pe
+   The number of the remote PE.
+
+
+.. seealso::
+   *intro_shmem*\ (3) *shmem_get*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_char_get.3.rst b/docs/man-openshmem/man3/shmem_char_get.3.rst
new file mode 100644
index 00000000000..c9ebfbecf44
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_char_get.3.rst
@@ -0,0 +1,202 @@
+.. _shmem_char_get:
+
+
+shmem_char_get
+==============
+
+.. include_body
+
+shmem_character_get\ (3), shmem_complex_get\ (3),
+:ref:`shmem_double_get`\ (3), :ref:`shmem_float_get`\ (3), shmem_get4\
+(3), shmem_get8\ (3), :ref:`shmem_get32`\ (3), :ref:`shmem_get64`\
+(3), :ref:`shmem_get128`\ (3), :ref:`shmem_getmem`\ (3),
+:ref:`shmem_int_get`\ (3), shmem_integer_get\ (3), shmem_logical_get\
+(3), :ref:`shmem_long_get`\ (3), :ref:`shmem_longdouble_get`\ (3),
+:ref:`shmem_longlong_get`\ (3), shmem_real_get\ (3),
+:ref:`shmem_short_get`\ (3) - Transfers data from a specified
+processing element (PE).
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void shmem_get32(void *target, const void *source,
+     size_t len, int pe)
+
+   void shmem_get64(void *target, const void *source,
+     size_t len, int pe)
+
+   void shmem_get128(void *target, const void *source,
+     size_t len, int pe)
+
+   void shmem_getmem(void *target, const void *source,
+     size_t len, int pe)
+
+   void shmem_int_get(int *target, const int *source,
+     size_t len, int pe)
+
+   void shmem_double_get(double *target, const double *source,
+     size_t len, int pe)
+
+   void shmem_float_get(float *target, const float *source,
+     size_t len, int pe)
+
+   void shmem_long_get(long *target, const long *source,
+     size_t len, int pe)
+
+   void shmem_longdouble_get(long double *target,
+     const long double *source, size_t len, int pe)
+
+   void shmem_longlong_get(long long *target,
+     const long long *source, size_t len, int pe)
+
+   void shmem_short_get(short *target,
+     const short *source, size_t len, int pe)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER len, pe
+
+   CALL SHMEM_CHARACTER_GET(target, source, len, pe)
+
+   CALL SHMEM_COMPLEX_GET(target, source, len, pe)
+
+   CALL SHMEM_DOUBLE_GET(target, source, len, pe)
+
+   CALL SHMEM_GET4(target, source, len, pe)
+
+   CALL SHMEM_GET8(target, source, len, pe)
+
+   CALL SHMEM_GET32(target, source, len, pe)
+
+   CALL SHMEM_GET64(target, source, len, pe)
+
+   CALL SHMEM_GET128(target, source, len, pe)
+
+   CALL SHMEM_GETMEM(target, source, len, pe)
+
+   CALL SHMEM_INTEGER_GET(target, source, len, pe)
+
+   CALL SHMEM_LOGICAL_GET(target, source, len, pe)
+
+   CALL SHMEM_REAL_GET(target, source, len, pe)
+
+
+DESCRIPTION
+-----------
+
+The shmem_get routines transfer **nelems** elements of the data object
+at address **source** on the remote PE **pe**, to the data object at
+address **target** on the local PE. These routines return after the data
+has been copied to address **target** on the local PE.
+
+The arguments are as follows:
+
+target
+   Local data object to be updated.
+
+source
+   Data object on the PE identified by pe that contains the data to be
+   copied. This data object must be remotely accessible.
+
+len
+   Number of elements in the target and source arrays. len must be of
+   type integer. If you are using Fortran, it must be a constant,
+   variable, or array element of default integer type.
+
+pe
+   PE number of the remote PE. pe must be of type integer. If you are
+   using Fortran, it must be a constant, variable, or array element of
+   default integer type.
+
+The target and source data objects must conform to typing constraints,
+which are as follows:
+
+:ref:`shmem_getmem`: Fortran: Any noncharacter type. C: Any data type. len is
+   scaled in bytes.
+
+shmem_get4, :ref:`shmem_get32`: Any noncharacter type that has a storage size
+   equal to 32 bits.
+
+{shmem_get8, :ref:`shmem_get64`}: Any noncharacter type that has a storage size equal to
+   64 bits.
+
+:ref:`shmem_get128`: Any noncharacter type that has a storage size equal to 128
+   bits.
+
+:ref:`shmem_short_get`: Elements of type short.
+
+:ref:`shmem_int_get`: Elements of type int.
+
+:ref:`shmem_long_get`: Elements of type long.
+
+:ref:`shmem_longlong_get`: Elements of type long long.
+
+:ref:`shmem_float_get`: Elements of type float.
+
+:ref:`shmem_double_get`: Elements of type double.
+
+:ref:`shmem_longdouble_get`: Elements of type long double.
+
+**SHMEM_CHARACTER_GET**: Elements of type character. len is the number of
+   characters to transfer. The actual character lengths of the source
+   and target variables are ignored.
+
+**SHMEM_COMPLEX_GET**: Elements of type complex of default size.
+
+**SHMEM_DOUBLE_GET**: (Fortran) Elements of type double precision.
+
+**SHMEM_INTEGER_GET**: Elements of type integer.
+
+**SHMEM_LOGICAL_GET**: Elements of type logical.
+
+**SHMEM_REAL_GET**: Elements of type real.
+
+If you are using Fortran, data types must be of default size. For
+example, a real variable must be declared as REAL, REAL*4, or
+REAL(KIND=4).
+
+
+NOTES
+-----
+
+See *intro_shmem*\ (3) for a definition of the term remotely accessible.
+
+
+EXAMPLES
+--------
+
+Consider this simple example for Fortran.
+
+.. code-block:: fortran
+
+   PROGRAM REDUCTION
+     REAL VALUES, SUM
+     COMMON /C/ VALUES
+     REAL WORK
+
+     CALL START_PES(0) ! ALLOW ANY NUMBER OF PES
+     VALUES = MY_PE() ! INITIALIZE IT TO SOMETHING
+     CALL SHMEM_BARRIER_ALL
+     SUM = 0.0
+     DO I = 0,NUM_PES()-1
+       CALL SHMEM_REAL_GET(WORK, VALUES, 1, I)
+       SUM = SUM + WORK
+     ENDDO
+     PRINT *, 'PE ', MY_PE(), ' COMPUTED SUM=', SUM
+     CALL SHMEM_BARRIER_ALL
+   END
+
+
+.. seealso::
+   *intro_shmem*\ (3) *shmem_put*\ (3) *shmem_iget*\ (3) *shmem_quiet*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_char_get_nbi.3.rst b/docs/man-openshmem/man3/shmem_char_get_nbi.3.rst
new file mode 100644
index 00000000000..f4dccc9c02e
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_char_get_nbi.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_char_get_nbi:
+
+shmem_char_get_nbi
+==================
+    .. include_body
+
+.. include:: ../man3/shmem_getmem_nbi.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_char_p.3.rst b/docs/man-openshmem/man3/shmem_char_p.3.rst
new file mode 100644
index 00000000000..24204ac2e76
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_char_p.3.rst
@@ -0,0 +1,67 @@
+.. _shmem_char_p:
+
+
+shmem_char_p
+============
+
+.. include_body
+
+:ref:`shmem_char_p`\ (3), :ref:`shmem_float_p`\ (3), :ref:`shmem_int_p`\ (3),
+:ref:`shmem_long_p`\ (3), :ref:`shmem_short_p`\ (3), :ref:`shmem_longlong_p`\ (3),
+:ref:`shmem_longdouble_p`\ (3) - These routines provide a low latency
+mechanism to write basic types (char, short, int, float, double, long,
+long long, long double) to symmetric data objects on remote PEs.
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+
+   void shmem_char_p(char *addr, char value, int pe)
+
+   void shmem_short_p(short *addr, short value, int pe)
+
+   void shmem_int_p(int *addr, int value, int pe)
+
+   void shmem_long_p(long *addr, long value, int pe)
+
+   void shmem_longlong_p(long long *addr, long long value, int pe)
+
+   void shmem_float_p(float *addr, float value, int pe)
+
+   void shmem_double_p(double *addr, double value, int pe)
+
+   void shmem_longdouble_p(long double *addr, long double value, int pe)
+
+
+DESCRIPTION
+-----------
+
+These routines provide a very low latency put capability for single
+elements of most basic types.
+
+The arguments are as follows:
+
+addr
+   The remotely accessible array element or scalar data object which
+   will receive the data on the remote PE.
+
+value
+   The value to be transferred to addr on the remote PE.
+
+pe
+   The number of the remote PE.
+
+As with shmem_put\ (3), these functions start the remote transfer and
+may return before the data is delivered to the remote PE. Use
+:ref:`shmem_quiet`\ (3) to force completion of all remote PUT transfers.
+
+
+.. seealso::
+   *intro_shmem*\ (3) *shmem_put*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_char_put.3.rst b/docs/man-openshmem/man3/shmem_char_put.3.rst
new file mode 100644
index 00000000000..b283cf0eb3b
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_char_put.3.rst
@@ -0,0 +1,203 @@
+.. _shmem_char_put:
+
+
+shmem_char_put
+==============
+
+.. include_body
+
+shmem_character_put\ (3), shmem_complex_put\ (3),
+:ref:`shmem_double_put`\ (3), :ref:`shmem_float_put`\ (3), :ref:`shmem_int_put`\ (3),
+shmem_integer_put\ (3), shmem_logical_put\ (3),
+:ref:`shmem_long_put`\ (3), :ref:`shmem_longdouble_put`\ (3),
+:ref:`shmem_longlong_put`\ (3), shmem_put4\ (3), shmem_put8\ (3),
+:ref:`shmem_put32`\ (3), :ref:`shmem_put64`\ (3), :ref:`shmem_put128`\ (3),
+:ref:`shmem_putmem`\ (3), shmem_real_put\ (3), :ref:`shmem_short_put`\ (3) -
+Transfers data to a specified processing element (PE)
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void shmem_double_put(double *target, const double *source,
+     size_t len, int pe)
+
+   void shmem_float_put(float *target, const float *source,
+     size_t len, int pe)
+
+   void shmem_int_put(int *target, const int *source, size_t len,
+     int pe)
+
+   void shmem_long_put(long *target, const long *source,
+     size_t len, int pe)
+
+   void shmem_longdouble_put(long double *target,
+     const long double *source, size_t len, int pe)
+
+   void shmem_longlong_put(long long *target,
+     const long long *source, size_t len, int pe)
+
+   void shmem_put32(void *target, const void *source, size_t len,
+     int pe)
+
+   void shmem_put64(void *target, const void *source, size_t len,
+     int pe)
+
+   void shmem_put128(void *target, const void *source, size_t len,
+     int pe)
+
+   void shmem_putmem(void *target, const void *source, size_t len,
+     int pe)
+
+   void shmem_short_put(short *target, const short *source,
+     size_t len, int pe)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER len, pe
+
+   CALL SHMEM_CHARACTER_PUT(target, source, len, pe)
+
+   CALL SHMEM_COMPLEX_PUT(target, source, len, pe)
+
+   CALL SHMEM_DOUBLE_PUT(target, source, len, pe)
+
+   CALL SHMEM_INTEGER_PUT(target, source, len, pe)
+
+   CALL SHMEM_LOGICAL_PUT(target, source, len, pe)
+
+   CALL SHMEM_PUT(target, source, len, pe)
+
+   CALL SHMEM_PUT4(target, source, len, pe)
+
+   CALL SHMEM_PUT8(target, source, len, pe)
+
+   CALL SHMEM_PUT32(target, source, len, pe)
+
+   CALL SHMEM_PUT64(target, source, len, pe)
+
+   CALL SHMEM_PUT128(target, source, len, pe)
+
+   CALL SHMEM_PUTMEM(target, source, len, pe)
+
+   CALL SHMEM_REAL_PUT(target, source, len, pe)
+
+
+DESCRIPTION
+-----------
+
+These routines transfer **nelems** elements of the data object at
+address **source** on the calling PE, to the data object at address
+**target** on the remote PE **pe**. These routines start the remote
+transfer and may return before the data is delivered to the remote PE.
+
+The delivery of data into the data object on the destination PE from
+different put calls may occur in any order. Because of this, two
+successive put operations may deliver data out of order unless a call to
+:ref:`shmem_fence`\ (3) is introduced between the two calls.
+
+The arguments are as follows:
+
+target
+   Data object to be updated on the remote PE. This data object must be
+   remotely accessible.
+
+source
+   Data object containing the data to be copied.
+
+len
+   Number of elements in the target and source arrays. len must be of
+   type integer. If you are using Fortran, it must be a constant,
+   variable, or array element of default integer type.
+
+pe
+   PE number of the remote PE. pe must be of type integer. If you are
+   using Fortran, it must be a constant, variable, or array element of
+   default integer type.
+
+The target and source data objects must conform to certain typing
+constraints, which are as follows:
+
+:ref:`shmem_putmem`: Fortran: Any noncharacter type. C: Any data type. len is scaled in
+   bytes.
+
+shmem_put4, :ref:`shmem_put32`:** Any noncharacter type that has a storage size
+   equal to 32 bits.
+
+shmem_put8, :ref:`shmem_put64`:** Any noncharacter type that has a storage size
+   equal to 64 bits.
+
+:ref:`shmem_put128`:** Any noncharacter type that has a storage size equal to 128
+   bits.
+
+:ref:`shmem_short_put`:** Elements of type short.
+
+:ref:`shmem_int_put`:** Elements of type int.
+
+:ref:`shmem_long_put`:** Elements of type long.
+
+:ref:`shmem_longlong_put`:** Elements of type long long.
+
+:ref:`shmem_float_put`:** Elements of type float.
+
+:ref:`shmem_double_put`:** Elements of type double.
+
+:ref:`shmem_longdouble_put`:** Elements of type long double.
+
+**SHMEM_CHARACTER_PUT:** Elements of type character. len is the number of
+   characters to transfer. The actual character lengths of the source
+   and target variables are ignored.
+
+**SHMEM_COMPLEX_PUT:** Elements of type complex of default size.
+
+**SHMEM_DOUBLE_PUT:** (Fortran) Elements of type double precision.
+
+**SHMEM_INTEGER_PUT:** Elements of type integer.
+
+**SHMEM_LOGICAL_PUT:** Elements of type logical.
+
+**SHMEM_REAL_PUT:** Elements of type real.
+   If you are using Fortran, data types must be of default size. For
+   example, a real variable must be declared as REAL, REAL*4, or
+   REAL(KIND=4).
+
+
+EXAMPLES
+--------
+
+The following shmem_put example is for C/C++ programs:
+
+.. code-block:: c++
+
+   #include <stdio.h>
+   #include <mpp/shmem.h>
+
+   main()
+   {
+     long source[10] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
+     static long target[10];
+     shmem_init();
+
+     if (shmem_my_pe() == 0) {
+       /* put 10 words into target on PE 1 */
+       shmem_long_put(target, source, 10, 1);
+     }
+     shmem_barrier_all();  /* sync sender and receiver */
+     if (shmem_my_pe() == 1)
+       shmem_udcflush();  /* not required on Altix systems */
+     printf("target[0] on PE %d is %d\n", shmem_my_pe(), target[0]);
+   }
+
+
+.. seealso::
+   *intro_shmem*\ (3) *shmem_iput*\ (3) *shmem_quiet*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_char_put_nbi.3.rst b/docs/man-openshmem/man3/shmem_char_put_nbi.3.rst
new file mode 100644
index 00000000000..bcc9e80143c
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_char_put_nbi.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_char_put_nbi:
+
+shmem_char_put_nbi
+==================
+    .. include_body
+
+.. include:: ../man3/shmem_putmem_nbi.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_clear_cache_inv.3.rst b/docs/man-openshmem/man3/shmem_clear_cache_inv.3.rst
new file mode 100644
index 00000000000..33a6e5e3806
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_clear_cache_inv.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_clear_cache_inv:
+
+shmem_clear_cache_inv
+=====================
+    .. include_body
+
+.. include:: ../man3/shmem_udcflush.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_clear_cache_line_inv.3.rst b/docs/man-openshmem/man3/shmem_clear_cache_line_inv.3.rst
new file mode 100644
index 00000000000..fe8a46a1d71
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_clear_cache_line_inv.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_clear_cache_line_inv:
+
+shmem_clear_cache_line_inv
+==========================
+    .. include_body
+
+.. include:: ../man3/shmem_udcflush.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_clear_lock.3.rst b/docs/man-openshmem/man3/shmem_clear_lock.3.rst
new file mode 100644
index 00000000000..15f594ee188
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_clear_lock.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_clear_lock:
+
+shmem_clear_lock
+================
+    .. include_body
+
+.. include:: ../man3/shmem_set_lock.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_collect32.3.rst b/docs/man-openshmem/man3/shmem_collect32.3.rst
new file mode 100644
index 00000000000..d65dcdefa06
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_collect32.3.rst
@@ -0,0 +1,213 @@
+.. _shmem_collect32:
+
+
+shmem_collect32
+===============
+
+.. include_body
+
+shmem_collect4\ (3), shmem_collect8\ (3), :ref:`shmem_collect32`\ (3),
+:ref:`shmem_collect64`\ (3), shmem_fcollect\ (3), shmem_fcollect4\
+(3), shmem_fcollect8\ (3), :ref:`shmem_fcollect32`\ (3),
+:ref:`shmem_fcollect64`\ (3) - Concatenates blocks of data from
+multiple processing elements (PEs) to an array in every PE
+
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void shmem_collect32(void *target, const void *source,
+     size_t nelems, int PE_start, int logPE_stride, int PE_size,
+     long *pSync)
+
+   void shmem_collect64(void *target, const void *source,
+     size_t nelems, int PE_start, int logPE_stride, int PE_size,
+     long *pSync)
+
+   void shmem_fcollect32(void *target, const void *source,
+     size_t nelems, int PE_start, int logPE_stride, int PE_size,
+     long *pSync)
+
+   void shmem_fcollect64(void *target, const void *source,
+     size_t nelems, int PE_start, int logPE_stride, int PE_size,
+     long *pSync)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER nelems
+   INTEGER PE_start, logPE_stride, PE_size
+   INTEGER pSync(SHMEM_COLLECT_SYNC_SIZE)
+
+   CALL SHMEM_COLLECT4(target, source, nelems, PE_start,
+   & logPE_stride, PE_size, pSync)
+
+   CALL SHMEM_COLLECT8(target, source, nelems, PE_start,
+   & logPE_stride, PE_size, pSync)
+
+   CALL SHMEM_FCOLLECT4(target, source, nelems, PE_start,
+   & logPE_stride, PE_size, pSync)
+
+   CALL SHMEM_FCOLLECT8(target, source, nelems, PE_start,
+   & logPE_stride, PE_size, pSync)
+
+
+DESCRIPTION
+-----------
+
+The shared memory (SHMEM) collect and fcollect routines concatenate
+nelems 64-bit or 32-bit data items from the source array into the target
+array, over the set of PEs defined by PE_start, log2PE_stride, and
+PE_size, in processor number order. The resultant target array contains
+the contribution from PE PE_start first, then the contribution from PE
+PE_start + PE_stride second, and so on. The collected result is written
+to the target array for all PEs in the active set.
+
+The fcollect routines require that nelems be the same value in all
+participating PEs, while the collect routines allow nelems to vary from
+PE to PE.
+
+The resulting target array is as follows:
+
+::
+
+   ----------------------------------------------------------
+      source(1..nelems)
+          from PE (PE_start + 0 * (2**logPE_stride))
+   ----------------------------------------------------------
+      source(1..nelems)
+          from PE (PE_start + 1 * (2**logPE_stride))
+   ----------------------------------------------------------
+      ...
+   ----------------------------------------------------------
+      source(1..nelems) from
+          PE (PE_start + (PE_size - 1) * (2**logPE_stride))
+   ----------------------------------------------------------
+
+As with all SHMEM collective routines, each of these routines assumes
+that only PEs in the active set call the routine. If a PE not in the
+active set calls a SHMEM collective routine, undefined behavior results.
+
+The arguments are as follows:
+
+target
+   A symmetric array. The target argument must be large enough to accept
+   the concatenation of the source arrays on all PEs. The data types are
+   as follows:
+
+   [shmem_collect8, :ref:`shmem_collect64`, shmem_fcollect8, and
+      shmem_fcollect64] any data type with an element size of 64 bits.
+      Fortran derived types, Fortran character type, and C/C++
+      structures are not permitted.
+
+   [shmem_collect4, :ref:`shmem_collect32`, shmem_fcollect4, and
+      shmem_fcollect32] any data type with an element size of 32 bits.
+      Fortran derived types, Fortran character type, and C/C++
+      structures are not permitted.
+
+source
+   A symmetric data object that can be of any type permissible for the
+   target argument.
+
+nelems
+   The number of elements in the source array. nelems must be of type
+   integer. If you are using Fortran, it must be a default integer
+   value.
+
+PE_start
+   The lowest virtual PE number of the active set of PEs. PE_start must
+   be of type integer. If you are using Fortran, it must be a default
+   integer value.
+
+logPE_stride
+   The log (base 2) of the stride between consecutive virtual PE numbers
+   in the active set. logPE_stride must be of type integer. If you are
+   using Fortran, it must be a default integer value.
+
+PE_size
+   The number of PEs in the active set. PE_size must be of type integer.
+   If you are using Fortran, it must be a default integer value.
+
+pSync
+   A symmetric work array. In C/C++, pSync must be of type int and size
+   \_SHMEM_COLLECT_SYNC_SIZE. In Fortran, pSync must be of type integer
+   and size SHMEM_COLLECT_SYNC_SIZE. If you are using Fortran, it must
+   be a default integer value. Every element of this array must be
+   initialized with the value \_SHMEM_SYNC_VALUE in C/C++ or
+   SHMEM_SYNC_VALUE in Fortran before any of the PEs in the active set
+   enter shmem_barrier().
+
+The values of arguments PE_start, logPE_stride, and PE_size must be
+equal on all PEs in the active set. The same target and source arrays
+and the same pSync work array must be passed to all PEs in the active
+set.
+
+Upon return from a collective routine, the following are true for the
+local PE: The target array is updated. The values in the pSync array are
+restored to the original values.
+
+
+NOTES
+-----
+
+The terms collective and symmetric are defined in *intro_shmem*\ (3).
+All SHMEM collective routines reset the values in pSync before they
+return, so a particular pSync buffer need only be initialized the first
+time it is used.
+
+You must ensure that the pSync array is not being updated on any PE in
+the active set while any of the PEs participate in processing of a SHMEM
+collective routine. Be careful to avoid these situations: If the pSync
+array is initialized at run time, some type of synchronization is needed
+to ensure that all PEs in the working set have initialized pSync before
+any of them enter a SHMEM routine called with the pSync synchronization
+array. A pSync array can be reused on a subsequent SHMEM collective
+routine only if none of the PEs in the active set are still processing a
+prior SHMEM collective routine call that used the same pSync array. In
+general, this may be ensured only by doing some type of synchronization.
+However, in the special case of SHMEM routines being called with the
+same active set, you can allocate two pSync arrays and alternate between
+them on successive calls.
+
+The collective routines operate on active PE sets that have a
+non-power-of-two PE_size with some performance degradation. They operate
+with no performance degradation when nelems is a non-power-of-two value.
+
+
+EXAMPLES
+--------
+
+C/C++:
+
+.. code-block:: c++
+
+   for (i=0; i < _SHMEM_COLLECT_SYNC_SIZE; i++) {
+     pSync[i] = _SHMEM_SYNC_VALUE;
+   }
+   shmem_barrier_all(); /* Wait for all PEs to initialize pSync */
+   shmem_collect32(target, source, 64, pe_start, logPE_stride,
+      pe_size, pSync);
+
+Fortran:
+
+.. code-block:: fortran
+
+   INTEGER PSYNC(SHMEM_COLLECT_SYNC_SIZE)
+   DATA PSYNC /SHMEM_COLLECT_SYNC_SIZE*SHMEM_SYNC_VALUE/
+
+   CALL SHMEM_COLLECT4(TARGET, SOURCE, 64, PE_START,
+   & LOGPE_STRIDE, PE_SIZE, PSYNC)
+
+
+.. seealso::
+   *intro_shmem*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_collect64.3.rst b/docs/man-openshmem/man3/shmem_collect64.3.rst
new file mode 100644
index 00000000000..ecdc1dbf7bb
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_collect64.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_collect64:
+
+shmem_collect64
+===============
+    .. include_body
+
+.. include:: ../man3/shmem_collect32.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_complexd_prod_to_all.3.rst b/docs/man-openshmem/man3/shmem_complexd_prod_to_all.3.rst
new file mode 100644
index 00000000000..b96cc041802
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_complexd_prod_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_complexd_prod_to_all:
+
+shmem_complexd_prod_to_all
+==========================
+    .. include_body
+
+.. include:: ../man3/shmem_short_prod_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_complexd_sum_to_all.3.rst b/docs/man-openshmem/man3/shmem_complexd_sum_to_all.3.rst
new file mode 100644
index 00000000000..7a24c1b4bd5
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_complexd_sum_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_complexd_sum_to_all:
+
+shmem_complexd_sum_to_all
+=========================
+    .. include_body
+
+.. include:: ../man3/shmem_short_sum_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_complexf_prod_to_all.3.rst b/docs/man-openshmem/man3/shmem_complexf_prod_to_all.3.rst
new file mode 100644
index 00000000000..c658573f5bc
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_complexf_prod_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_complexf_prod_to_all:
+
+shmem_complexf_prod_to_all
+==========================
+    .. include_body
+
+.. include:: ../man3/shmem_short_prod_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_complexf_sum_to_all.3.rst b/docs/man-openshmem/man3/shmem_complexf_sum_to_all.3.rst
new file mode 100644
index 00000000000..879c21dca44
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_complexf_sum_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_complexf_sum_to_all:
+
+shmem_complexf_sum_to_all
+=========================
+    .. include_body
+
+.. include:: ../man3/shmem_short_sum_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_double_fetch.3.rst b/docs/man-openshmem/man3/shmem_double_fetch.3.rst
new file mode 100644
index 00000000000..775aa7b376f
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_double_fetch.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_double_fetch:
+
+shmem_double_fetch
+==================
+    .. include_body
+
+.. include:: ../man3/shmem_int_fetch.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_double_g.3.rst b/docs/man-openshmem/man3/shmem_double_g.3.rst
new file mode 100644
index 00000000000..f0e86f3d898
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_double_g.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_double_g:
+
+shmem_double_g
+==============
+    .. include_body
+
+.. include:: ../man3/shmem_char_g.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_double_get.3.rst b/docs/man-openshmem/man3/shmem_double_get.3.rst
new file mode 100644
index 00000000000..30d5235168a
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_double_get.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_double_get:
+
+shmem_double_get
+================
+    .. include_body
+
+.. include:: ../man3/shmem_char_get.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_double_get_nbi.3.rst b/docs/man-openshmem/man3/shmem_double_get_nbi.3.rst
new file mode 100644
index 00000000000..c740c487320
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_double_get_nbi.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_double_get_nbi:
+
+shmem_double_get_nbi
+====================
+    .. include_body
+
+.. include:: ../man3/shmem_getmem_nbi.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_double_iget.3.rst b/docs/man-openshmem/man3/shmem_double_iget.3.rst
new file mode 100644
index 00000000000..b8829be85e8
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_double_iget.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_double_iget:
+
+shmem_double_iget
+=================
+    .. include_body
+
+.. include:: ../man3/shmem_short_iget.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_double_iput.3.rst b/docs/man-openshmem/man3/shmem_double_iput.3.rst
new file mode 100644
index 00000000000..cd9e6c2e74d
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_double_iput.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_double_iput:
+
+shmem_double_iput
+=================
+    .. include_body
+
+.. include:: ../man3/shmem_short_iput.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_double_max_to_all.3.rst b/docs/man-openshmem/man3/shmem_double_max_to_all.3.rst
new file mode 100644
index 00000000000..b667716495a
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_double_max_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_double_max_to_all:
+
+shmem_double_max_to_all
+=======================
+    .. include_body
+
+.. include:: ../man3/shmem_short_max_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_double_min_to_all.3.rst b/docs/man-openshmem/man3/shmem_double_min_to_all.3.rst
new file mode 100644
index 00000000000..05a24e97d2d
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_double_min_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_double_min_to_all:
+
+shmem_double_min_to_all
+=======================
+    .. include_body
+
+.. include:: ../man3/shmem_short_min_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_double_p.3.rst b/docs/man-openshmem/man3/shmem_double_p.3.rst
new file mode 100644
index 00000000000..e0244906334
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_double_p.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_double_p:
+
+shmem_double_p
+==============
+    .. include_body
+
+.. include:: ../man3/shmem_char_p.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_double_prod_to_all.3.rst b/docs/man-openshmem/man3/shmem_double_prod_to_all.3.rst
new file mode 100644
index 00000000000..f8751b0e4ea
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_double_prod_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_double_prod_to_all:
+
+shmem_double_prod_to_all
+========================
+    .. include_body
+
+.. include:: ../man3/shmem_short_prod_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_double_put.3.rst b/docs/man-openshmem/man3/shmem_double_put.3.rst
new file mode 100644
index 00000000000..a56f0d0b349
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_double_put.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_double_put:
+
+shmem_double_put
+================
+    .. include_body
+
+.. include:: ../man3/shmem_char_put.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_double_put_nbi.3.rst b/docs/man-openshmem/man3/shmem_double_put_nbi.3.rst
new file mode 100644
index 00000000000..729b92eccbe
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_double_put_nbi.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_double_put_nbi:
+
+shmem_double_put_nbi
+====================
+    .. include_body
+
+.. include:: ../man3/shmem_putmem_nbi.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_double_set.3.rst b/docs/man-openshmem/man3/shmem_double_set.3.rst
new file mode 100644
index 00000000000..665b6081899
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_double_set.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_double_set:
+
+shmem_double_set
+================
+    .. include_body
+
+.. include:: ../man3/shmem_int_set.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_double_sum_to_all.3.rst b/docs/man-openshmem/man3/shmem_double_sum_to_all.3.rst
new file mode 100644
index 00000000000..d4dca85ebd3
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_double_sum_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_double_sum_to_all:
+
+shmem_double_sum_to_all
+=======================
+    .. include_body
+
+.. include:: ../man3/shmem_short_sum_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_double_swap.3.rst b/docs/man-openshmem/man3/shmem_double_swap.3.rst
new file mode 100644
index 00000000000..6111ce58509
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_double_swap.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_double_swap:
+
+shmem_double_swap
+=================
+    .. include_body
+
+.. include:: ../man3/shmem_swap.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_fcollect32.3.rst b/docs/man-openshmem/man3/shmem_fcollect32.3.rst
new file mode 100644
index 00000000000..f215b382cca
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_fcollect32.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_fcollect32:
+
+shmem_fcollect32
+================
+    .. include_body
+
+.. include:: ../man3/shmem_collect32.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_fcollect64.3.rst b/docs/man-openshmem/man3/shmem_fcollect64.3.rst
new file mode 100644
index 00000000000..97931c28a4b
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_fcollect64.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_fcollect64:
+
+shmem_fcollect64
+================
+    .. include_body
+
+.. include:: ../man3/shmem_collect32.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_fence.3.rst b/docs/man-openshmem/man3/shmem_fence.3.rst
new file mode 100644
index 00000000000..1a61dde2e43
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_fence.3.rst
@@ -0,0 +1,57 @@
+.. _shmem_fence:
+
+
+shmem_fence
+===========
+
+.. include_body
+
+:ref:`shmem_fence` - Provides a separate ordering on the sequence of puts
+issued by this PE to each destination PE.
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void shmem_fence(void)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   CALL SHMEM_FENCE
+
+
+DESCRIPTION
+-----------
+
+The shmem_fence() routine provides an ordering on the put operations
+issued by the calling PE prior to the call to shmem_fence() relative
+to the put operations issued by the calling PE following the call to
+shmem_fence(). It guarantees that all such prior put operations
+issued to a particular destination PE are fully written to the symmetric
+memory of that destination PE, before any such following put operations
+to that same destination PE are written to the symmetric memory of that
+destination PE. Note that the ordering is provided separately on the
+sequences of puts from the calling PE to each distinct destination PE.
+The shmem_quiet() routine should be used instead if ordering of puts
+is required when multiple destination PEs are involved.
+
+
+NOTES
+-----
+
+The :ref:`shmem_quiet` function should be called if ordering of puts is desired
+when multiple remote PEs are involved.
+
+
+.. seealso::
+   *intro_shmem*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_finalize.3.rst b/docs/man-openshmem/man3/shmem_finalize.3.rst
new file mode 100644
index 00000000000..9b61f934e40
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_finalize.3.rst
@@ -0,0 +1,51 @@
+.. _shmem_finalize:
+
+
+shmem_finalize
+==============
+
+.. include_body
+
+:ref:`shmem_finalize` - A collective operation that releases resources used by
+the OpenSHMEM library. This only terminates the Open-SHMEM portion of a
+program, not the entire program.
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void shmem_finalize(void)
+
+Fortran:
+
+.. code-block:: fortran
+
+   include 'mpp/shmem.fh'
+   CALL SHMEM_FINALIZE
+
+
+DESCRIPTION
+-----------
+
+:ref:`shmem_finalize` is a collective operation that ends the OpenSHMEM portion
+of a program previously initialized by :ref:`shmem_init` and releases resources
+used by the OpenSHMEM library. This collective operation requires all
+PEs to participate in the call. There is an implicit global barrier in
+:ref:`shmem_finalize` so that pending communication is completed, and no
+resources can be released until all PEs have entered :ref:`shmem_finalize`.
+:ref:`shmem_finalize` must be the last OpenSHMEM library call encountered in
+the OpenSHMEM portion of a program. A call to :ref:`shmem_finalize` will
+release any resources initialized by a corresponding call to :ref:`shmem_init`.
+All processes and threads that represent the PEs will still exist after
+the call to :ref:`shmem_finalize` returns, but they will no longer have access
+to any resources that have been released.
+
+
+.. seealso::
+   *intro_shmem*\ (3) *shmem_my_pe*\ (3) *shmem_init*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_float_fetch.3.rst b/docs/man-openshmem/man3/shmem_float_fetch.3.rst
new file mode 100644
index 00000000000..f2050234804
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_float_fetch.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_float_fetch:
+
+shmem_float_fetch
+=================
+    .. include_body
+
+.. include:: ../man3/shmem_int_fetch.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_float_g.3.rst b/docs/man-openshmem/man3/shmem_float_g.3.rst
new file mode 100644
index 00000000000..09be713a31c
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_float_g.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_float_g:
+
+shmem_float_g
+=============
+    .. include_body
+
+.. include:: ../man3/shmem_char_g.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_float_get.3.rst b/docs/man-openshmem/man3/shmem_float_get.3.rst
new file mode 100644
index 00000000000..5a5db10f1b9
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_float_get.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_float_get:
+
+shmem_float_get
+===============
+    .. include_body
+
+.. include:: ../man3/shmem_char_get.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_float_get_nbi.3.rst b/docs/man-openshmem/man3/shmem_float_get_nbi.3.rst
new file mode 100644
index 00000000000..c057ca62c5a
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_float_get_nbi.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_float_get_nbi:
+
+shmem_float_get_nbi
+===================
+    .. include_body
+
+.. include:: ../man3/shmem_getmem_nbi.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_float_iget.3.rst b/docs/man-openshmem/man3/shmem_float_iget.3.rst
new file mode 100644
index 00000000000..e835fd357b7
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_float_iget.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_float_iget:
+
+shmem_float_iget
+================
+    .. include_body
+
+.. include:: ../man3/shmem_short_iget.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_float_iput.3.rst b/docs/man-openshmem/man3/shmem_float_iput.3.rst
new file mode 100644
index 00000000000..ce002bf8278
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_float_iput.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_float_iput:
+
+shmem_float_iput
+================
+    .. include_body
+
+.. include:: ../man3/shmem_short_iput.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_float_max_to_all.3.rst b/docs/man-openshmem/man3/shmem_float_max_to_all.3.rst
new file mode 100644
index 00000000000..7dfe3438420
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_float_max_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_float_max_to_all:
+
+shmem_float_max_to_all
+======================
+    .. include_body
+
+.. include:: ../man3/shmem_short_max_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_float_min_to_all.3.rst b/docs/man-openshmem/man3/shmem_float_min_to_all.3.rst
new file mode 100644
index 00000000000..35c29efc4fd
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_float_min_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_float_min_to_all:
+
+shmem_float_min_to_all
+======================
+    .. include_body
+
+.. include:: ../man3/shmem_short_min_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_float_p.3.rst b/docs/man-openshmem/man3/shmem_float_p.3.rst
new file mode 100644
index 00000000000..79f7cfee3ae
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_float_p.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_float_p:
+
+shmem_float_p
+=============
+    .. include_body
+
+.. include:: ../man3/shmem_char_p.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_float_prod_to_all.3.rst b/docs/man-openshmem/man3/shmem_float_prod_to_all.3.rst
new file mode 100644
index 00000000000..80b6fc9cdd0
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_float_prod_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_float_prod_to_all:
+
+shmem_float_prod_to_all
+=======================
+    .. include_body
+
+.. include:: ../man3/shmem_short_prod_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_float_put.3.rst b/docs/man-openshmem/man3/shmem_float_put.3.rst
new file mode 100644
index 00000000000..a20ae2ed3ae
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_float_put.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_float_put:
+
+shmem_float_put
+===============
+    .. include_body
+
+.. include:: ../man3/shmem_char_put.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_float_put_nbi.3.rst b/docs/man-openshmem/man3/shmem_float_put_nbi.3.rst
new file mode 100644
index 00000000000..59fa90a4522
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_float_put_nbi.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_float_put_nbi:
+
+shmem_float_put_nbi
+===================
+    .. include_body
+
+.. include:: ../man3/shmem_putmem_nbi.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_float_set.3.rst b/docs/man-openshmem/man3/shmem_float_set.3.rst
new file mode 100644
index 00000000000..670cb2f4706
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_float_set.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_float_set:
+
+shmem_float_set
+===============
+    .. include_body
+
+.. include:: ../man3/shmem_int_set.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_float_sum_to_all.3.rst b/docs/man-openshmem/man3/shmem_float_sum_to_all.3.rst
new file mode 100644
index 00000000000..4b6a250fd69
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_float_sum_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_float_sum_to_all:
+
+shmem_float_sum_to_all
+======================
+    .. include_body
+
+.. include:: ../man3/shmem_short_sum_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_float_swap.3.rst b/docs/man-openshmem/man3/shmem_float_swap.3.rst
new file mode 100644
index 00000000000..7abdd21a848
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_float_swap.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_float_swap:
+
+shmem_float_swap
+================
+    .. include_body
+
+.. include:: ../man3/shmem_swap.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_free.3.rst b/docs/man-openshmem/man3/shmem_free.3.rst
new file mode 100644
index 00000000000..94001f871d5
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_free.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_free:
+
+shmem_free
+==========
+    .. include_body
+
+.. include:: ../man3/shmem_malloc.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_get128.3.rst b/docs/man-openshmem/man3/shmem_get128.3.rst
new file mode 100644
index 00000000000..49356ae311c
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_get128.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_get128:
+
+shmem_get128
+============
+    .. include_body
+
+.. include:: ../man3/shmem_char_get.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_get128_nbi.3.rst b/docs/man-openshmem/man3/shmem_get128_nbi.3.rst
new file mode 100644
index 00000000000..05a2115b21d
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_get128_nbi.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_get128_nbi:
+
+shmem_get128_nbi
+================
+    .. include_body
+
+.. include:: ../man3/shmem_getmem_nbi.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_get16_nbi.3.rst b/docs/man-openshmem/man3/shmem_get16_nbi.3.rst
new file mode 100644
index 00000000000..6f55e5b1582
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_get16_nbi.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_get16_nbi:
+
+shmem_get16_nbi
+===============
+    .. include_body
+
+.. include:: ../man3/shmem_getmem_nbi.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_get32.3.rst b/docs/man-openshmem/man3/shmem_get32.3.rst
new file mode 100644
index 00000000000..1ee513475c8
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_get32.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_get32:
+
+shmem_get32
+===========
+    .. include_body
+
+.. include:: ../man3/shmem_char_get.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_get32_nbi.3.rst b/docs/man-openshmem/man3/shmem_get32_nbi.3.rst
new file mode 100644
index 00000000000..595bbdee466
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_get32_nbi.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_get32_nbi:
+
+shmem_get32_nbi
+===============
+    .. include_body
+
+.. include:: ../man3/shmem_getmem_nbi.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_get64.3.rst b/docs/man-openshmem/man3/shmem_get64.3.rst
new file mode 100644
index 00000000000..a6ca649cd04
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_get64.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_get64:
+
+shmem_get64
+===========
+    .. include_body
+
+.. include:: ../man3/shmem_char_get.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_get64_nbi.3.rst b/docs/man-openshmem/man3/shmem_get64_nbi.3.rst
new file mode 100644
index 00000000000..7b992469ba1
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_get64_nbi.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_get64_nbi:
+
+shmem_get64_nbi
+===============
+    .. include_body
+
+.. include:: ../man3/shmem_getmem_nbi.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_get8_nbi.3.rst b/docs/man-openshmem/man3/shmem_get8_nbi.3.rst
new file mode 100644
index 00000000000..1ecc027bb84
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_get8_nbi.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_get8_nbi:
+
+shmem_get8_nbi
+==============
+    .. include_body
+
+.. include:: ../man3/shmem_getmem_nbi.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_getmem.3.rst b/docs/man-openshmem/man3/shmem_getmem.3.rst
new file mode 100644
index 00000000000..452c7d2190f
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_getmem.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_getmem:
+
+shmem_getmem
+============
+    .. include_body
+
+.. include:: ../man3/shmem_char_get.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_getmem_nbi.3.rst b/docs/man-openshmem/man3/shmem_getmem_nbi.3.rst
new file mode 100644
index 00000000000..2f6fad8214e
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_getmem_nbi.3.rst
@@ -0,0 +1,171 @@
+.. _shmem_getmem_nbi:
+
+
+shmem_getmem_nbi
+================
+
+.. include_body
+
+:ref:`shmem_getmem_nbi`\ (3), :ref:`shmem_char_get_nbi`\ (3),
+:ref:`shmem_short_get_nbi`\ (3), :ref:`shmem_int_get_nbi`\ (3),
+:ref:`shmem_long_get_nbi`\ (3), :ref:`shmem_longlong_get_nbi`\ (3),
+:ref:`shmem_float_get_nbi`\ (3), :ref:`shmem_double_get_nbi`\ (3),
+:ref:`shmem_longdouble_get_nbi`\ (3), :ref:`shmem_get8_nbi`\ (3),
+:ref:`shmem_get16_nbi`\ (3), :ref:`shmem_get32_nbi`\ (3), :ref:`shmem_get64_nbi`\ (3),
+:ref:`shmem_get128_nbi`\ (3), - The nonblocking get routines provide a method
+for copying data from a contiguous remote data object on the specified
+PE to the local data object.
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void shmem_getmem_nbi(void *dest, const void *source,
+     size_t nelems, int pe)
+
+   void shmem_char_get(char *dest, const char *source,
+     size_t nelems, int pe)
+
+   void shmem_short_get(short *dest, const short *source,
+     size_t nelems, int pe)
+
+   void shmem_int_get(int *dest, const int *source,
+     size_t nelems, int pe)
+
+   void shmem_long_get(long *dest, const long *source,
+     size_t nelems, int pe)
+
+   void shmem_longlong_get(long long *dest, const long long *source,
+     size_t nelems, int pe)
+
+   void shmem_float_get(float *dest, const float *source,
+     size_t nelems, int pe)
+
+   void shmem_double_get(double *dest, const double *source,
+     size_t nelems, int pe)
+
+   void shmem_longdouble_get(long double *dest, const long double *source,
+     size_t nelems, int pe)
+
+   void shmem_get8(void *dest, const void *source,
+     size_t nelems, int pe)
+
+   void shmem_get16(void *dest, const void *source,
+     size_t nelems, int pe)
+
+   void shmem_get32(void *dest, const void *source,
+     size_t nelems, int pe)
+
+   void shmem_get64(void *dest, const void *source,
+     size_t nelems, int pe)
+
+   void shmem_get128(void *dest, const void *source,
+     size_t nelems, int pe)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER nelems, pe
+
+   CALL SHMEM_GETMEM_NBI(dest, source, nelems, pe)
+
+   CALL SHMEM_CHARACTER_GET_NBI(dest, source, nelems, pe)
+
+   CALL SHMEM_COMPLEX_GET_NBI(dest, source, nelems, pe)
+
+   CALL SHMEM_DOUBLE_GET_NBI(dest, source, nelems, pe)
+
+   CALL SHMEM_INTEGER_GET_NBI(dest, source, nelems, pe)
+
+   CALL SHMEM_LOGICAL_GET_NBI(dest, source, nelems, pe)
+
+   CALL SHMEM_REAL_GET_NBI(dest, source, nelems, pe)
+
+   CALL SHMEM_GET4_NBI(dest, source, nelems, pe)
+
+   CALL SHMEM_GET8_NBI(dest, source, nelems, pe)
+
+   CALL SHMEM_GET32_NBI(dest, source, nelems, pe)
+
+   CALL SHMEM_GET64_NBI(dest, source, nelems, pe)
+
+   CALL SHMEM_GET128_NBI(dest, source, nelems, pe)
+
+
+DESCRIPTION
+-----------
+
+The get routines provide a method for copying a contiguous symmetric
+data object from a different PE to a contiguous data object on the local
+PE. The routines return after posting the operation. The operation is
+considered complete after a subsequent call to :ref:`shmem_quiet`. At the
+completion of :ref:`shmem_quiet`, the data has been delivered to the dest array
+on the local PE.
+
+The arguments are as follows:
+
+dest
+   Local data object to be updated.
+
+source
+   Data object on the PE identified by pe that contains the data to be
+   copied. This data object must be remotely accessible.
+
+nelems
+   Number of elements in the target and source arrays. len must be of
+   type integer. If you are using Fortran, it must be a constant,
+   variable, or array element of default integer type.
+
+pe
+   PE number of the remote PE. pe must be of type integer. If you are
+   using Fortran, it must be a constant, variable, or array element of
+   default integer type.
+
+If you are using Fortran, data types must be of default size. For
+example, a real variable must be declared as REAL, REAL*4, or
+REAL(KIND=4).
+
+
+NOTES
+-----
+
+See *intro_shmem*\ (3) for a definition of the term remotely accessible.
+
+
+EXAMPLES
+--------
+
+Consider this simple example for Fortran.
+
+.. code-block:: fortran
+
+   PROGRAM REDUCTION
+     REAL VALUES, SUM
+     COMMON /C/ VALUES
+     REAL WORK
+
+     CALL START_PES(0) ! ALLOW ANY NUMBER OF PES
+     VALUES = MY_PE() ! INITIALIZE IT TO SOMETHING
+     CALL SHMEM_BARRIER_ALL
+     SUM = 0.0
+     DO I = 0,NUM_PES()-1
+       CALL SHMEM_REAL_GET_NBI(WORK, VALUES, 1, I)
+       CALL SHMEM_QUIET                ! wait for delivery
+       SUM = SUM + WORK
+     ENDDO
+     PRINT *, 'PE ', MY_PE(), ' COMPUTED SUM=', SUM
+     CALL SHMEM_BARRIER_ALL
+   END
+
+
+.. seealso::
+   *intro_shmem*\ (3) *shmem_quiet*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_global_exit.3.rst b/docs/man-openshmem/man3/shmem_global_exit.3.rst
new file mode 100644
index 00000000000..c24fb84532f
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_global_exit.3.rst
@@ -0,0 +1,51 @@
+.. _shmem_global_exit:
+
+
+shmem_global_exit
+=================
+
+.. include_body
+
+:ref:`shmem_global_exit` - A routine that allows any PE to force termination of
+an entire program.
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void shmem_global_exit(int status)
+
+Fortran:
+
+.. code-block:: fortran
+
+   include 'mpp/shmem.fh'
+   INTEGER STATUS
+   CALL SHMEM_GLOBAL_EXIT(status)
+
+
+DESCRIPTION
+-----------
+
+shmem_global_exit() :ref:`shmem_global_exit` is a non-collective routine that
+allows any one PE to force termination of an Open- SHMEM program for all
+PEs, passing an exit status to the execution environment. This routine
+terminates the entire program, not just the OpenSHMEM portion. When any
+PE calls :ref:`shmem_global_exit`, it results in the immediate notification to
+all PEs to terminate. :ref:`shmem_global_exit` flushes I/O and releases
+resources in accordance with C/C++/Fortran language requirements for
+normal program termination. If more than one PE calls :ref:`shmem_global_exit`,
+then the exit status returned to the environment shall be one of the
+values passed to :ref:`shmem_global_exit` as the status argument. There is no
+return to the caller of :ref:`shmem_global_exit`; control is returned from the
+OpenSHMEM program to the execution environment for all PEs.
+
+
+.. seealso::
+   *intro_shmem*\ (3) *shmem_my_pe*\ (3) *shmem_init*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_iget128.3.rst b/docs/man-openshmem/man3/shmem_iget128.3.rst
new file mode 100644
index 00000000000..6b5069a9b53
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_iget128.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_iget128:
+
+shmem_iget128
+=============
+    .. include_body
+
+.. include:: ../man3/shmem_short_iget.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_iget32.3.rst b/docs/man-openshmem/man3/shmem_iget32.3.rst
new file mode 100644
index 00000000000..e709473c808
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_iget32.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_iget32:
+
+shmem_iget32
+============
+    .. include_body
+
+.. include:: ../man3/shmem_short_iget.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_iget64.3.rst b/docs/man-openshmem/man3/shmem_iget64.3.rst
new file mode 100644
index 00000000000..a5710a94d58
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_iget64.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_iget64:
+
+shmem_iget64
+============
+    .. include_body
+
+.. include:: ../man3/shmem_short_iget.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_info_get_name.3.rst b/docs/man-openshmem/man3/shmem_info_get_name.3.rst
new file mode 100644
index 00000000000..c9e685205c5
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_info_get_name.3.rst
@@ -0,0 +1,51 @@
+.. _shmem_info_get_name:
+
+
+shmem_info_get_name
+===================
+
+.. include_body
+
+:ref:`shmem_info_get_name` - This routine returns the vendor defined character
+string.
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void shmem_info_get_name(char *name)
+
+Fortran:
+
+.. code-block:: fortran
+
+   include 'mpp/shmem.fh'
+   SHMEM_INFO_GET_NAME(NAME)
+   CHARACTER *(*)NAME
+
+
+DESCRIPTION
+-----------
+
+shmem_info_get_name() This routine returns the vendor defined character
+string of size defined by the constant SHMEM_MAX_NAME_LEN. The program
+calling this function prepares the memory of size SHMEM_MAX_NAME_LEN,
+and the implementation copies the string of size at most
+SHMEM_MAX_NAME_LEN. In C, the string is terminated by a null character.
+In Fortran, the string of size less than SHMEM_MAX_NAME_LEN is padded
+with blank characters up to size SHMEM_MAX_NAME_LEN. The implementation
+copying a string of size greater than SHMEM_MAX_NAME_LEN results in an
+undefined behavior. Multiple invocations of the routine in an OpenSHMEM
+program always return the same string. For a given library
+implementation, the major and minor version returned by these calls is
+consistent with the compile-time constants defined in its shmem.h.
+
+
+.. seealso::
+   *intro_shmem*\ (3) *shmem_my_pe*\ (3) *shmem_init*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_info_get_version.3.rst b/docs/man-openshmem/man3/shmem_info_get_version.3.rst
new file mode 100644
index 00000000000..6f646c97b2b
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_info_get_version.3.rst
@@ -0,0 +1,45 @@
+.. _shmem_info_get_version:
+
+
+shmem_info_get_version
+======================
+
+.. include_body
+
+:ref:`shmem_info_get_version` - Returns the major and minor version of the
+library implementation.
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void shmem_info_get_version(int *major, int *minor)
+
+Fortran:
+
+.. code-block:: fortran
+
+   include 'mpp/shmem.fh'
+   SHMEM_INFO_GET_VERSION(MAJOR, MINOR)
+   INTEGER MAJOR, MINOR
+
+
+DESCRIPTION
+-----------
+
+shmem_info_get_version() This routine returns the major and minor
+version of the OpenSHMEM standard in use. For a given library
+implementation, the major and minor version returned by these calls is
+consistent with the compile-time constants, SHMEM_MAJOR_VERSION and
+SHMEM_MINOR_VERSION, defined in its shmem.h. The valid major version
+value is 1, and the valid minor version value is 2.
+
+
+.. seealso::
+   *intro_shmem*\ (3) *shmem_my_pe*\ (3) *shmem_init*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_init.3.rst b/docs/man-openshmem/man3/shmem_init.3.rst
new file mode 100644
index 00000000000..9e51afdfe8e
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_init.3.rst
@@ -0,0 +1,91 @@
+.. _shmem_init:
+
+
+shmem_init
+==========
+
+.. include_body
+
+:ref:`shmem_init`, start_pes - Allocates a block of memory from the symmetric
+heap.
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   void shmem_init(void)
+   void start_pes(int npes)
+
+Fortran:
+
+.. code-block:: fortran
+
+   CALL SHMEM_INIT()
+   CALL START_PES(npes)
+
+
+DESCRIPTION
+-----------
+
+The start_pes routine should be the first statement in a SHMEM parallel
+program.
+
+The start_pes routine accepts the following argument:
+
+npes
+   Unused. Should be set to 0.
+
+This routine initializes the SHMEM API, therefore it must be called
+before calling any other SHMEM routine. This routine is responsible
+inter alia for setting up the symmetric heap on the calling PE, and the
+creation of the virtual PE numbers. Upon successful return from this
+routine, the calling PE will be able to communicate with and transfer
+data to other PEs.
+
+Multiple calls to this function are not allowed.
+
+For an overview of programming with SHMEM communication routines,
+example SHMEM programs, and instructions for compiling SHMEM programs,
+see the *intro_shmem*\ (3) man page.
+
+
+EXAMPLES
+--------
+
+This is a simple program that calls shmem_integer_put\ (3):
+
+::
+
+   PROGRAM PUT
+     INCLUDE "mpp/shmem.fh"
+
+     INTEGER TARG, SRC, RECEIVER, BAR
+     COMMON /T/ TARG
+     PARAMETER (RECEIVER=1)
+
+     CALL SHMEM_INIT()
+     IF (MY_PE() .EQ. 0) THEN
+       SRC = 33
+       CALL SHMEM_INTEGER_PUT(TARG, SRC, 1, RECEIVER)
+     ENDIF
+     CALL SHMEM_BARRIER_ALL ! SYNCHRONIZES SENDER AND RECEIVER
+     IF (MY_PE() .EQ. RECEIVER) THEN
+       PRINT *,'PE ', MY_PE(),' TARG=',TARG,' (expect 33)'
+     ENDIF
+   END
+
+
+NOTES
+-----
+
+If the start_pes call is not the first statement in a program,
+unexpected results may occur on some architectures.
+
+
+.. seealso::
+   *intro_shmem*\ (3) :ref:`shmem_barrier`\ (3) :ref:`shmem_barrier_all`\ (3)
+   *shmem_put*\ (3) *my_pe*\ (3) *shmem_n_pes*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_int_add.3.rst b/docs/man-openshmem/man3/shmem_int_add.3.rst
new file mode 100644
index 00000000000..1864e7a83ef
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_int_add.3.rst
@@ -0,0 +1,75 @@
+.. _shmem_int_add:
+
+
+shmem_int_add
+=============
+
+.. include_body
+
+:ref:`shmem_int_add`\ (3), shmem_int4_add\ (3), shmem_int8_add\ (3),
+:ref:`shmem_long_add`\ (3), :ref:`shmem_longlong_add`\ (3) - Performs an atomic
+add operation.
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void shmem_int_add(int *target, int value, int pe)
+   void shmem_long_add(long *target, long value, int pe)
+   void shmem_longlong_add(long long *target, long long value,
+     int pe)
+
+Fortran:
+
+.. code-block:: fortran
+
+   include 'mpp/shmem.h'
+
+   INTEGER pe
+
+   CALL SHMEM_INT4_ADD(target, value, pe)
+   CALL SHMEM_INT8_ADD(target, value, pe)
+
+
+DESCRIPTION
+-----------
+
+The atomic add routines add **value** to the data at address **target**
+on PE **pe**. The operation completes without the possibility of another
+process updating target between the time of the fetch and the update.
+
+The arguments are as follows:
+
+target
+   The remotely accessible integer data object to be updated on the
+   remote PE. If you are using C/C++, the type of target should match
+   that implied in the SYNOPSIS section. If you are using the Fortran
+   compiler, it must be of type integer with an element size of 4 bytes
+   for SHMEM_INT4_ADD and 8 bytes for SHMEM_INT8_ADD.
+
+value
+   The value to be atomically added to target. If you are using C/C++,
+   the type of value should match that implied in the SYNOPSIS section.
+   If you are using Fortran, it must be of type integer with an element
+   size of target.
+
+pe
+   An integer that indicates the PE number upon which target is to be
+   updated. If you are using Fortran, it must be a default integer
+   value.
+
+
+NOTES
+-----
+
+The term remotely accessible is defined in *intro_shmem*\ (3).
+
+
+.. seealso::
+   *intro_shmem*\ (3) *shmem_cache*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_int_and_to_all.3.rst b/docs/man-openshmem/man3/shmem_int_and_to_all.3.rst
new file mode 100644
index 00000000000..419f00702dd
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_int_and_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_int_and_to_all:
+
+shmem_int_and_to_all
+====================
+    .. include_body
+
+.. include:: ../man3/shmem_short_and_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_int_cswap.3.rst b/docs/man-openshmem/man3/shmem_int_cswap.3.rst
new file mode 100644
index 00000000000..85b48b6e016
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_int_cswap.3.rst
@@ -0,0 +1,135 @@
+.. _shmem_int_cswap:
+
+
+shmem_int_cswap
+===============
+
+.. include_body
+
+:ref:`shmem_int_cswap`\ (3), shmem_int4_cswap\ (3),
+shmem_int8_cswap\ (3), :ref:`shmem_long_cswap`\ (3),
+:ref:`shmem_longlong_cswap`\ (3) - Performs an atomic conditional swap to a
+remote data object
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   int shmem_int_cswap(int *target, int cond, int value, int pe)
+
+   long shmem_long_cswap(long *target, long cond, long value,
+     int pe)
+
+   long long shmem_longlong_cswap(longlong *target,
+     longlong cond, longlong value, int pe)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER pe
+
+   INTEGER(KIND=4) SHMEM_INT4_CSWAP
+   ires = SHMEM_INT4_CSWAP(target, cond, value, pe)
+
+   INTEGER(KIND=8) SHMEM_INT8_CSWAP
+   ires = SHMEM_INT8_CSWAP(target, cond, value, pe)
+
+
+DESCRIPTION
+-----------
+
+The conditional swap routines conditionally update a target data object
+on an arbitrary processing element (PE) and return the prior contents of
+the data object in one atomic operation.
+
+The arguments are as follows:
+
+target
+   The remotely accessible integer data object to be updated on the
+   remote PE. If you are using C/C++, the type of target should match
+   that implied in the SYNOPSIS section. If you are using the Fortran
+   compiler, it must be of type integer with an element size of 4 bytes
+   for SHMEM_INT4_ADD and 8 bytes for SHMEM_INT8_ADD.
+
+value
+   The value to be atomically added to target. If you are using C/C++,
+   the type of value should match that implied in the SYNOPSIS section.
+   If you are using Fortran, it must be of type integer with an element
+   size of target.
+
+pe
+   An integer that indicates the PE number upon which target is to be
+   updated. If you are using Fortran, it must be a default integer
+   value.
+
+target
+   The remotely accessible integer data object to be updated on the
+   remote PE. If you are using C/C++, the data type of target should
+   match that implied in the SYNOPSIS section. If you are using Fortran,
+   it must be of the following type:
+
+   **SHMEM_INT4_CSWAP**: 4-byte integer
+
+   **SHMEM_INT8_CSWAP**: 8-byte integer
+
+cond
+   cond is compared to the remote target value. If cond and the remote
+   target are equal, then value is swapped into the remote target.
+   Otherwise, the remote target is unchanged. In either case, the old
+   value of the remote target is returned as the function return value.
+   cond must be of the same data type as target.
+
+value
+   The value to be atomically written to the remote PE. value must be
+   the same data type as target.
+
+pe
+   An integer that indicates the PE number upon which target is to be
+   updated. If you are using Fortran, it must be a default integer
+   value.
+
+
+NOTES
+-----
+
+The term remotely accessible is defined in *intro_shmem*\ (3).
+
+
+RETURN VALUES
+-------------
+
+The contents that had been in the target data object on the remote PE
+prior to the conditional swap.
+
+
+EXAMPLES
+--------
+
+The following call ensures that the first PE to execute the conditional
+swap will successfully write its PE number to race_winner on PE 0.
+
+::
+
+   main()
+   {
+     static int race_winner = -1;
+     int oldval;
+
+     shmem_init();
+     oldval = shmem_int_cswap(&race_winner, -1, shmem_my_pe(), 0);
+     if (oldval == -1)
+       printf("pe %d was first\n",shmem_my_pe());
+   }
+
+
+.. seealso::
+   *intro_shmem*\ (3) *shmem_cache*\ (3) *shmem_swap*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_int_fadd.3.rst b/docs/man-openshmem/man3/shmem_int_fadd.3.rst
new file mode 100644
index 00000000000..a61b4c308bd
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_int_fadd.3.rst
@@ -0,0 +1,86 @@
+.. _shmem_int_fadd:
+
+
+shmem_int_fadd
+==============
+
+.. include_body
+
+shmem_int4_fadd\ (3), shmem_int8_fadd\ (3), :ref:`shmem_int_fadd`\ (3),
+:ref:`shmem_long_fadd`\ (3), :ref:`shmem_longlong_fadd`\ (3) - Performs an atomic
+fetch-and-add operation on a remote data object
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   int shmem_int_fadd(int *target, int value, int pe)
+
+   long shmem_long_fadd(long *target, long value, int pe)
+
+   long long shmem_longlong_fadd(long long *target, longlong value,
+     int pe)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER pe
+
+   INTEGER(KIND=4) SHMEM_INT4_FADD, ires, target, value
+   ires = SHMEM_INT4_FADD(target, value, pe)
+
+   INTEGER(KIND=8) SHMEM_INT8_FADD, ires, target, value
+   ires = SHMEM_INT8_FADD(target, value, pe)
+
+
+DESCRIPTION
+-----------
+
+shmem_fadd functions perform an atomic fetch-and-add operation. An
+atomic fetch-and-add operation fetches the old target and adds value to
+target without the possibility of another process updating target
+between the time of the fetch and the update. These routines add value
+to target on Processing Element (PE) pe and return the previous contents
+of target as an atomic operation.
+
+The arguments are as follows:
+
+target
+   The remotely accessible integer data object to be updated on the
+   remote PE. The type of target should match that implied in the
+   SYNOPSIS section.
+
+value
+   The value to be atomically added to target. The type of value should
+   match that implied in the SYNOPSIS section.
+
+pe
+   An integer that indicates the PE number on which target is to be
+   updated. If you are using Fortran, it must be a default integer
+   value.
+
+
+NOTES
+-----
+
+The term remotely accessible is defined in *intro_shmem*\ (3).
+
+
+RETURN VALUES
+-------------
+
+The contents that had been at the target address on the remote PE prior
+to the atomic addition operation.
+
+
+.. seealso::
+   *intro_shmem*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_int_fetch.3.rst b/docs/man-openshmem/man3/shmem_int_fetch.3.rst
new file mode 100644
index 00000000000..29f8d0aaf31
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_int_fetch.3.rst
@@ -0,0 +1,82 @@
+.. _shmem_int_fetch:
+
+
+shmem_int_fetch
+===============
+
+.. include_body
+
+shmem_int4_fetch\ (3), shmem_int8_fetch\ (3),
+:ref:`shmem_int_fetch`\ (3), :ref:`shmem_long_fetch`\ (3),
+:ref:`shmem_longlong_fetch`\ (3) :ref:`shmem_double_fetch`\ (3)
+:ref:`shmem_float_fetch`\ (3) - Atomically fetches the value of a remote data
+object
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   int shmem_int_fetch(int *target, int pe)
+
+   long shmem_long_fetch(long *target, int pe)
+
+   long long shmem_longlong_fetch(long long *target, int pe)
+
+   double shmem_double_fetch(long long *target, int pe)
+
+   float  shmem_float_fetch(float *target, int pe)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER pe
+
+   INTEGER(KIND=4) SHMEM_INT4_FETCH, ires, target
+   ires = SHMEM_INT4_FETCH(target, pe)
+
+   INTEGER(KIND=8) SHMEM_INT8_FETCH, ires, target
+   ires = SHMEM_INT8_FETCH(target, pe)
+
+
+   REAL(KIND=4) SHMEM_INT4_FETCH, ires, target
+   ires = SHMEM_REAL4_FETCH(target, pe)
+
+   REAL(KIND=8) SHMEM_INT8_FETCH, ires, target
+   ires = SHMEM_REAL8_FETCH(target, pe)
+
+
+DESCRIPTION
+-----------
+
+The shmem_fetch functions perform an atomic fetch operation. They return
+the contents of the **target** as an atomic operation.
+
+The arguments are as follows:
+
+target
+   The remotely accessible data object to be fetched from the remote PE.
+
+pe
+   An integer that indicates the PE number from which *target* is to be
+   fetched. If you are using Fortran, it must be a default integer
+   value.
+
+
+RETURN VALUES
+-------------
+
+The contents at the *target* address on the remote PE. The data type of
+the return value is the same as the the type of the remote data object.
+
+
+.. seealso::
+   *intro_shmem*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_int_finc.3.rst b/docs/man-openshmem/man3/shmem_int_finc.3.rst
new file mode 100644
index 00000000000..c7dc75c03db
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_int_finc.3.rst
@@ -0,0 +1,80 @@
+.. _shmem_int_finc:
+
+
+shmem_int_finc
+==============
+
+.. include_body
+
+shmem_int4_finc\ (3), shmem_int8_finc\ (3), :ref:`shmem_int_finc`\ (3),
+:ref:`shmem_long_finc`\ (3), :ref:`shmem_longlong_finc`\ (3) - Performs an atomic
+fetch-and-increment operation on a remote data object
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   int shmem_int_finc(int *target, int pe)
+
+   long shmem_long_finc(long *target, int pe)
+
+   long long shmem_longlong_finc(long long *target, int pe)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER pe
+   INTEGER(KIND=4) SHMEM_INT4_FINC, target4
+   INTEGER(KIND=8) SHMEM_INT8_FINC, target8
+
+   ires4 = SHMEM_INT4_FINC(target4, pe)
+
+   ires8 = SHMEM_INT8_FINC(target8, pe)
+
+
+DESCRIPTION
+-----------
+
+The fetch and increment routines retrieve the value at address
+**target** on PE **pe**, and update **target** with the result of
+incrementing the retrieved value by one. The operation must be completed
+without the possibility of another process updating **target** between
+the time of the fetch and the update.
+
+The arguments are as follows:
+
+target
+   The remotely accessible integer data object to be updated on the
+   remote PE. The type of target should match that implied in the
+   SYNOPSIS section.
+
+pe
+   An integer that indicates the PE number upon which target is to be
+   updated. If you are using Fortran, it must be a default integer
+   value.
+
+
+NOTES
+-----
+
+The term remotely accessible is defined in *intro_shmem*\ (3).
+
+
+RETURN VALUES
+-------------
+
+The contents that had been at the target address on the remote PE prior
+to the increment.
+
+
+.. seealso::
+   *intro_shmem*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_int_g.3.rst b/docs/man-openshmem/man3/shmem_int_g.3.rst
new file mode 100644
index 00000000000..e13ad7be1ab
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_int_g.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_int_g:
+
+shmem_int_g
+===========
+    .. include_body
+
+.. include:: ../man3/shmem_char_g.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_int_get.3.rst b/docs/man-openshmem/man3/shmem_int_get.3.rst
new file mode 100644
index 00000000000..176ea58aa58
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_int_get.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_int_get:
+
+shmem_int_get
+=============
+    .. include_body
+
+.. include:: ../man3/shmem_char_get.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_int_get_nbi.3.rst b/docs/man-openshmem/man3/shmem_int_get_nbi.3.rst
new file mode 100644
index 00000000000..c559f06d33a
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_int_get_nbi.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_int_get_nbi:
+
+shmem_int_get_nbi
+=================
+    .. include_body
+
+.. include:: ../man3/shmem_getmem_nbi.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_int_iget.3.rst b/docs/man-openshmem/man3/shmem_int_iget.3.rst
new file mode 100644
index 00000000000..14f67e3bc95
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_int_iget.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_int_iget:
+
+shmem_int_iget
+==============
+    .. include_body
+
+.. include:: ../man3/shmem_short_iget.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_int_inc.3.rst b/docs/man-openshmem/man3/shmem_int_inc.3.rst
new file mode 100644
index 00000000000..08020e38ab7
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_int_inc.3.rst
@@ -0,0 +1,78 @@
+.. _shmem_int_inc:
+
+
+shmem_int_inc
+=============
+
+.. include_body
+
+shmem_int4_inc\ (3), shmem_int8_inc\ (3), :ref:`shmem_int_inc`\ (3),
+:ref:`shmem_long_inc`\ (3), :ref:`shmem_longlong_inc`\ (3) - These routines
+perform an atomic increment operation on a remote data object.
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   int shmem_int_inc(int *target, int pe)
+
+   long shmem_long_inc(long *target, int pe)
+
+   long long shmem_longlong_inc(long long *target, int pe)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER pe
+   INTEGER(KIND=4) SHMEM_INT4_INC, target4
+   INTEGER(KIND=8) SHMEM_INT8_INC, target8
+
+   ires4 = SHMEM_INT4_INC(target4, pe)
+
+   ires8 = SHMEM_INT8_INC(target8, pe)
+
+
+DESCRIPTION
+-----------
+
+The atomic increment routines replace the value of **target** with its
+value incremented by one. The operation must be completed without the
+possibility of another process updating **target** between the time of
+the fetch and the update.
+
+The arguments are as follows:
+
+target
+   The remotely accessible integer data object to be updated on the
+   remote PE. The type of target should match that implied in the
+   SYNOPSIS section.
+
+pe
+   An integer that indicates the PE number upon which target is to be
+   updated. If you are using Fortran, it must be a default integer
+   value.
+
+
+NOTES
+-----
+
+The term remotely accessible is defined in *intro_shmem*\ (3).
+
+
+RETURN VALUES
+-------------
+
+None.
+
+
+.. seealso::
+   *intro_shmem*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_int_iput.3.rst b/docs/man-openshmem/man3/shmem_int_iput.3.rst
new file mode 100644
index 00000000000..3e3db6ed0d6
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_int_iput.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_int_iput:
+
+shmem_int_iput
+==============
+    .. include_body
+
+.. include:: ../man3/shmem_short_iput.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_int_max_to_all.3.rst b/docs/man-openshmem/man3/shmem_int_max_to_all.3.rst
new file mode 100644
index 00000000000..2fbe1a21492
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_int_max_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_int_max_to_all:
+
+shmem_int_max_to_all
+====================
+    .. include_body
+
+.. include:: ../man3/shmem_short_max_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_int_min_to_all.3.rst b/docs/man-openshmem/man3/shmem_int_min_to_all.3.rst
new file mode 100644
index 00000000000..2d2d59fec08
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_int_min_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_int_min_to_all:
+
+shmem_int_min_to_all
+====================
+    .. include_body
+
+.. include:: ../man3/shmem_short_min_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_int_or_to_all.3.rst b/docs/man-openshmem/man3/shmem_int_or_to_all.3.rst
new file mode 100644
index 00000000000..5daabc33c5a
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_int_or_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_int_or_to_all:
+
+shmem_int_or_to_all
+===================
+    .. include_body
+
+.. include:: ../man3/shmem_short_or_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_int_p.3.rst b/docs/man-openshmem/man3/shmem_int_p.3.rst
new file mode 100644
index 00000000000..f6c91b43e16
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_int_p.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_int_p:
+
+shmem_int_p
+===========
+    .. include_body
+
+.. include:: ../man3/shmem_char_p.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_int_prod_to_all.3.rst b/docs/man-openshmem/man3/shmem_int_prod_to_all.3.rst
new file mode 100644
index 00000000000..ba26d664913
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_int_prod_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_int_prod_to_all:
+
+shmem_int_prod_to_all
+=====================
+    .. include_body
+
+.. include:: ../man3/shmem_short_prod_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_int_put.3.rst b/docs/man-openshmem/man3/shmem_int_put.3.rst
new file mode 100644
index 00000000000..173b43f4227
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_int_put.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_int_put:
+
+shmem_int_put
+=============
+    .. include_body
+
+.. include:: ../man3/shmem_char_put.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_int_put_nbi.3.rst b/docs/man-openshmem/man3/shmem_int_put_nbi.3.rst
new file mode 100644
index 00000000000..10405f05631
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_int_put_nbi.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_int_put_nbi:
+
+shmem_int_put_nbi
+=================
+    .. include_body
+
+.. include:: ../man3/shmem_putmem_nbi.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_int_set.3.rst b/docs/man-openshmem/man3/shmem_int_set.3.rst
new file mode 100644
index 00000000000..38fd1763f69
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_int_set.3.rst
@@ -0,0 +1,75 @@
+.. _shmem_int_set:
+
+
+shmem_int_set
+=============
+
+.. include_body
+
+:ref:`shmem_double_set`\ (3), :ref:`shmem_float_set`\ (3), :ref:`shmem_int_set`\ (3),
+:ref:`shmem_long_set`\ (3), :ref:`shmem_longlong_set`\ (3) shmem_int4_set\ (3),
+shmem_int8_set\ (3), shmem_real4_set\ (3), shmem_real8_set\ (3), -
+Atomically sets the value of a remote data object
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void shmem_double_set(double *target, double value, int pe)
+
+   void shmem_float_set(float *target, float value, int pe)
+
+   void shmem_int_set(int *target, int value, int pe)
+
+   void shmem_long_set(long *target, long value, int pe)
+
+   void shmem_longlong_set(long long *target, long long value, int pe)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER pe
+
+   CALL SHMEM_INT4_SET(target, value, pe)
+   CALL SHMEM_INT8_SET(target, value, pe)
+   CALL SHMEM_REAL4_SET(target, value, pe)
+   CALL SHMEM_REAL8_SET(target, value, pe)
+
+
+DESCRIPTION
+-----------
+
+The set routines write the **value** into the address **target** on
+**pe** as an atomic operation.
+
+The arguments are as follows:
+
+target
+   The remotely accessible data object to be set on the remote PE.
+
+value
+   The value to be atomically written to the remote PE.
+
+pe
+   An integer that indicates the PE number upon which target is to be
+   updated. If you are using Fortran, it must be a default integer
+   value.
+
+
+RETURN VALUES
+-------------
+
+NONE
+
+
+.. seealso::
+   *intro_shmem*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_int_sum_to_all.3.rst b/docs/man-openshmem/man3/shmem_int_sum_to_all.3.rst
new file mode 100644
index 00000000000..1f8348870dd
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_int_sum_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_int_sum_to_all:
+
+shmem_int_sum_to_all
+====================
+    .. include_body
+
+.. include:: ../man3/shmem_short_sum_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_int_swap.3.rst b/docs/man-openshmem/man3/shmem_int_swap.3.rst
new file mode 100644
index 00000000000..cd4617076b3
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_int_swap.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_int_swap:
+
+shmem_int_swap
+==============
+    .. include_body
+
+.. include:: ../man3/shmem_swap.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_int_wait.3.rst b/docs/man-openshmem/man3/shmem_int_wait.3.rst
new file mode 100644
index 00000000000..742d35a4d9b
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_int_wait.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_int_wait:
+
+shmem_int_wait
+==============
+    .. include_body
+
+.. include:: ../man3/shmem_wait.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_int_wait_until.3.rst b/docs/man-openshmem/man3/shmem_int_wait_until.3.rst
new file mode 100644
index 00000000000..273734127a0
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_int_wait_until.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_int_wait_until:
+
+shmem_int_wait_until
+====================
+    .. include_body
+
+.. include:: ../man3/shmem_wait.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_int_xor_to_all.3.rst b/docs/man-openshmem/man3/shmem_int_xor_to_all.3.rst
new file mode 100644
index 00000000000..965538548d3
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_int_xor_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_int_xor_to_all:
+
+shmem_int_xor_to_all
+====================
+    .. include_body
+
+.. include:: ../man3/shmem_short_xor_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_iput128.3.rst b/docs/man-openshmem/man3/shmem_iput128.3.rst
new file mode 100644
index 00000000000..7ec004c7929
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_iput128.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_iput128:
+
+shmem_iput128
+=============
+    .. include_body
+
+.. include:: ../man3/shmem_short_iput.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_iput32.3.rst b/docs/man-openshmem/man3/shmem_iput32.3.rst
new file mode 100644
index 00000000000..2bea9b98c93
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_iput32.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_iput32:
+
+shmem_iput32
+============
+    .. include_body
+
+.. include:: ../man3/shmem_short_iput.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_iput64.3.rst b/docs/man-openshmem/man3/shmem_iput64.3.rst
new file mode 100644
index 00000000000..1f001ed68a0
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_iput64.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_iput64:
+
+shmem_iput64
+============
+    .. include_body
+
+.. include:: ../man3/shmem_short_iput.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_long_add.3.rst b/docs/man-openshmem/man3/shmem_long_add.3.rst
new file mode 100644
index 00000000000..f9f11c774e0
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_long_add.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_long_add:
+
+shmem_long_add
+==============
+    .. include_body
+
+.. include:: ../man3/shmem_int_add.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_long_and_to_all.3.rst b/docs/man-openshmem/man3/shmem_long_and_to_all.3.rst
new file mode 100644
index 00000000000..18e40324f63
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_long_and_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_long_and_to_all:
+
+shmem_long_and_to_all
+=====================
+    .. include_body
+
+.. include:: ../man3/shmem_short_and_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_long_cswap.3.rst b/docs/man-openshmem/man3/shmem_long_cswap.3.rst
new file mode 100644
index 00000000000..fbff3fb9e5a
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_long_cswap.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_long_cswap:
+
+shmem_long_cswap
+================
+    .. include_body
+
+.. include:: ../man3/shmem_int_cswap.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_long_fadd.3.rst b/docs/man-openshmem/man3/shmem_long_fadd.3.rst
new file mode 100644
index 00000000000..770244c81ea
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_long_fadd.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_long_fadd:
+
+shmem_long_fadd
+===============
+    .. include_body
+
+.. include:: ../man3/shmem_int_fadd.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_long_fetch.3.rst b/docs/man-openshmem/man3/shmem_long_fetch.3.rst
new file mode 100644
index 00000000000..6d055f48eec
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_long_fetch.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_long_fetch:
+
+shmem_long_fetch
+================
+    .. include_body
+
+.. include:: ../man3/shmem_int_fetch.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_long_finc.3.rst b/docs/man-openshmem/man3/shmem_long_finc.3.rst
new file mode 100644
index 00000000000..ccb6005f1a7
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_long_finc.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_long_finc:
+
+shmem_long_finc
+===============
+    .. include_body
+
+.. include:: ../man3/shmem_int_finc.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_long_g.3.rst b/docs/man-openshmem/man3/shmem_long_g.3.rst
new file mode 100644
index 00000000000..8777b2a4a0a
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_long_g.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_long_g:
+
+shmem_long_g
+============
+    .. include_body
+
+.. include:: ../man3/shmem_char_g.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_long_get.3.rst b/docs/man-openshmem/man3/shmem_long_get.3.rst
new file mode 100644
index 00000000000..3f2cfc5f90f
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_long_get.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_long_get:
+
+shmem_long_get
+==============
+    .. include_body
+
+.. include:: ../man3/shmem_char_get.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_long_get_nbi.3.rst b/docs/man-openshmem/man3/shmem_long_get_nbi.3.rst
new file mode 100644
index 00000000000..ada774a82fa
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_long_get_nbi.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_long_get_nbi:
+
+shmem_long_get_nbi
+==================
+    .. include_body
+
+.. include:: ../man3/shmem_getmem_nbi.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_long_iget.3.rst b/docs/man-openshmem/man3/shmem_long_iget.3.rst
new file mode 100644
index 00000000000..bf285314b49
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_long_iget.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_long_iget:
+
+shmem_long_iget
+===============
+    .. include_body
+
+.. include:: ../man3/shmem_short_iget.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_long_inc.3.rst b/docs/man-openshmem/man3/shmem_long_inc.3.rst
new file mode 100644
index 00000000000..e30c9ff1194
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_long_inc.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_long_inc:
+
+shmem_long_inc
+==============
+    .. include_body
+
+.. include:: ../man3/shmem_int_inc.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_long_iput.3.rst b/docs/man-openshmem/man3/shmem_long_iput.3.rst
new file mode 100644
index 00000000000..a8ec99f3aba
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_long_iput.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_long_iput:
+
+shmem_long_iput
+===============
+    .. include_body
+
+.. include:: ../man3/shmem_short_iput.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_long_max_to_all.3.rst b/docs/man-openshmem/man3/shmem_long_max_to_all.3.rst
new file mode 100644
index 00000000000..391968d66ee
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_long_max_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_long_max_to_all:
+
+shmem_long_max_to_all
+=====================
+    .. include_body
+
+.. include:: ../man3/shmem_short_max_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_long_min_to_all.3.rst b/docs/man-openshmem/man3/shmem_long_min_to_all.3.rst
new file mode 100644
index 00000000000..30b086b0279
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_long_min_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_long_min_to_all:
+
+shmem_long_min_to_all
+=====================
+    .. include_body
+
+.. include:: ../man3/shmem_short_min_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_long_or_to_all.3.rst b/docs/man-openshmem/man3/shmem_long_or_to_all.3.rst
new file mode 100644
index 00000000000..174198c6539
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_long_or_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_long_or_to_all:
+
+shmem_long_or_to_all
+====================
+    .. include_body
+
+.. include:: ../man3/shmem_short_or_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_long_p.3.rst b/docs/man-openshmem/man3/shmem_long_p.3.rst
new file mode 100644
index 00000000000..de485c24ac7
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_long_p.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_long_p:
+
+shmem_long_p
+============
+    .. include_body
+
+.. include:: ../man3/shmem_char_p.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_long_prod_to_all.3.rst b/docs/man-openshmem/man3/shmem_long_prod_to_all.3.rst
new file mode 100644
index 00000000000..473882e3c56
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_long_prod_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_long_prod_to_all:
+
+shmem_long_prod_to_all
+======================
+    .. include_body
+
+.. include:: ../man3/shmem_short_prod_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_long_put.3.rst b/docs/man-openshmem/man3/shmem_long_put.3.rst
new file mode 100644
index 00000000000..94224bf2d37
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_long_put.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_long_put:
+
+shmem_long_put
+==============
+    .. include_body
+
+.. include:: ../man3/shmem_char_put.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_long_put_nbi.3.rst b/docs/man-openshmem/man3/shmem_long_put_nbi.3.rst
new file mode 100644
index 00000000000..fe7b63dbe6e
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_long_put_nbi.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_long_put_nbi:
+
+shmem_long_put_nbi
+==================
+    .. include_body
+
+.. include:: ../man3/shmem_putmem_nbi.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_long_set.3.rst b/docs/man-openshmem/man3/shmem_long_set.3.rst
new file mode 100644
index 00000000000..960161bf071
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_long_set.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_long_set:
+
+shmem_long_set
+==============
+    .. include_body
+
+.. include:: ../man3/shmem_int_set.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_long_sum_to_all.3.rst b/docs/man-openshmem/man3/shmem_long_sum_to_all.3.rst
new file mode 100644
index 00000000000..db4b62aafbc
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_long_sum_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_long_sum_to_all:
+
+shmem_long_sum_to_all
+=====================
+    .. include_body
+
+.. include:: ../man3/shmem_short_sum_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_long_swap.3.rst b/docs/man-openshmem/man3/shmem_long_swap.3.rst
new file mode 100644
index 00000000000..18ea1440f25
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_long_swap.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_long_swap:
+
+shmem_long_swap
+===============
+    .. include_body
+
+.. include:: ../man3/shmem_swap.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_long_wait.3.rst b/docs/man-openshmem/man3/shmem_long_wait.3.rst
new file mode 100644
index 00000000000..4efb39b6e18
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_long_wait.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_long_wait:
+
+shmem_long_wait
+===============
+    .. include_body
+
+.. include:: ../man3/shmem_wait.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_long_wait_until.3.rst b/docs/man-openshmem/man3/shmem_long_wait_until.3.rst
new file mode 100644
index 00000000000..c2ed977ddce
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_long_wait_until.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_long_wait_until:
+
+shmem_long_wait_until
+=====================
+    .. include_body
+
+.. include:: ../man3/shmem_wait.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_long_xor_to_all.3.rst b/docs/man-openshmem/man3/shmem_long_xor_to_all.3.rst
new file mode 100644
index 00000000000..d8699055a54
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_long_xor_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_long_xor_to_all:
+
+shmem_long_xor_to_all
+=====================
+    .. include_body
+
+.. include:: ../man3/shmem_short_xor_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longdouble_g.3.rst b/docs/man-openshmem/man3/shmem_longdouble_g.3.rst
new file mode 100644
index 00000000000..e58b3acfb9f
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longdouble_g.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longdouble_g:
+
+shmem_longdouble_g
+==================
+    .. include_body
+
+.. include:: ../man3/shmem_char_g.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longdouble_get.3.rst b/docs/man-openshmem/man3/shmem_longdouble_get.3.rst
new file mode 100644
index 00000000000..e27db8ca149
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longdouble_get.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longdouble_get:
+
+shmem_longdouble_get
+====================
+    .. include_body
+
+.. include:: ../man3/shmem_char_get.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longdouble_get_nbi.3.rst b/docs/man-openshmem/man3/shmem_longdouble_get_nbi.3.rst
new file mode 100644
index 00000000000..0051d0ebb2c
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longdouble_get_nbi.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longdouble_get_nbi:
+
+shmem_longdouble_get_nbi
+========================
+    .. include_body
+
+.. include:: ../man3/shmem_getmem_nbi.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longdouble_iget.3.rst b/docs/man-openshmem/man3/shmem_longdouble_iget.3.rst
new file mode 100644
index 00000000000..d3b53e405e4
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longdouble_iget.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longdouble_iget:
+
+shmem_longdouble_iget
+=====================
+    .. include_body
+
+.. include:: ../man3/shmem_short_iget.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longdouble_iput.3.rst b/docs/man-openshmem/man3/shmem_longdouble_iput.3.rst
new file mode 100644
index 00000000000..4b48141c61b
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longdouble_iput.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longdouble_iput:
+
+shmem_longdouble_iput
+=====================
+    .. include_body
+
+.. include:: ../man3/shmem_short_iput.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longdouble_max_to_all.3.rst b/docs/man-openshmem/man3/shmem_longdouble_max_to_all.3.rst
new file mode 100644
index 00000000000..8193419e90b
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longdouble_max_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longdouble_max_to_all:
+
+shmem_longdouble_max_to_all
+===========================
+    .. include_body
+
+.. include:: ../man3/shmem_short_max_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longdouble_min_to_all.3.rst b/docs/man-openshmem/man3/shmem_longdouble_min_to_all.3.rst
new file mode 100644
index 00000000000..ac6772c10fd
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longdouble_min_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longdouble_min_to_all:
+
+shmem_longdouble_min_to_all
+===========================
+    .. include_body
+
+.. include:: ../man3/shmem_short_min_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longdouble_p.3.rst b/docs/man-openshmem/man3/shmem_longdouble_p.3.rst
new file mode 100644
index 00000000000..648f82b0dfa
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longdouble_p.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longdouble_p:
+
+shmem_longdouble_p
+==================
+    .. include_body
+
+.. include:: ../man3/shmem_char_p.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longdouble_prod_to_all.3.rst b/docs/man-openshmem/man3/shmem_longdouble_prod_to_all.3.rst
new file mode 100644
index 00000000000..df4965541ad
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longdouble_prod_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longdouble_prod_to_all:
+
+shmem_longdouble_prod_to_all
+============================
+    .. include_body
+
+.. include:: ../man3/shmem_short_prod_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longdouble_put.3.rst b/docs/man-openshmem/man3/shmem_longdouble_put.3.rst
new file mode 100644
index 00000000000..e66d04b2427
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longdouble_put.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longdouble_put:
+
+shmem_longdouble_put
+====================
+    .. include_body
+
+.. include:: ../man3/shmem_char_put.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longdouble_put_nbi.3.rst b/docs/man-openshmem/man3/shmem_longdouble_put_nbi.3.rst
new file mode 100644
index 00000000000..901ce172afa
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longdouble_put_nbi.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longdouble_put_nbi:
+
+shmem_longdouble_put_nbi
+========================
+    .. include_body
+
+.. include:: ../man3/shmem_putmem_nbi.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longdouble_sum_to_all.3.rst b/docs/man-openshmem/man3/shmem_longdouble_sum_to_all.3.rst
new file mode 100644
index 00000000000..9ff8ee5394f
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longdouble_sum_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longdouble_sum_to_all:
+
+shmem_longdouble_sum_to_all
+===========================
+    .. include_body
+
+.. include:: ../man3/shmem_short_sum_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longlong_add.3.rst b/docs/man-openshmem/man3/shmem_longlong_add.3.rst
new file mode 100644
index 00000000000..83a0f10eec7
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longlong_add.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longlong_add:
+
+shmem_longlong_add
+==================
+    .. include_body
+
+.. include:: ../man3/shmem_int_add.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longlong_and_to_all.3.rst b/docs/man-openshmem/man3/shmem_longlong_and_to_all.3.rst
new file mode 100644
index 00000000000..88e1ab5044c
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longlong_and_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longlong_and_to_all:
+
+shmem_longlong_and_to_all
+=========================
+    .. include_body
+
+.. include:: ../man3/shmem_short_and_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longlong_cswap.3.rst b/docs/man-openshmem/man3/shmem_longlong_cswap.3.rst
new file mode 100644
index 00000000000..6b48b3c3b0e
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longlong_cswap.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longlong_cswap:
+
+shmem_longlong_cswap
+====================
+    .. include_body
+
+.. include:: ../man3/shmem_int_cswap.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longlong_fadd.3.rst b/docs/man-openshmem/man3/shmem_longlong_fadd.3.rst
new file mode 100644
index 00000000000..96057e956d1
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longlong_fadd.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longlong_fadd:
+
+shmem_longlong_fadd
+===================
+    .. include_body
+
+.. include:: ../man3/shmem_int_fadd.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longlong_fetch.3.rst b/docs/man-openshmem/man3/shmem_longlong_fetch.3.rst
new file mode 100644
index 00000000000..846da2ebc37
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longlong_fetch.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longlong_fetch:
+
+shmem_longlong_fetch
+====================
+    .. include_body
+
+.. include:: ../man3/shmem_int_fetch.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longlong_finc.3.rst b/docs/man-openshmem/man3/shmem_longlong_finc.3.rst
new file mode 100644
index 00000000000..ad64e9eba1c
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longlong_finc.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longlong_finc:
+
+shmem_longlong_finc
+===================
+    .. include_body
+
+.. include:: ../man3/shmem_int_finc.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longlong_g.3.rst b/docs/man-openshmem/man3/shmem_longlong_g.3.rst
new file mode 100644
index 00000000000..4d709852c5d
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longlong_g.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longlong_g:
+
+shmem_longlong_g
+================
+    .. include_body
+
+.. include:: ../man3/shmem_char_g.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longlong_get.3.rst b/docs/man-openshmem/man3/shmem_longlong_get.3.rst
new file mode 100644
index 00000000000..a5cf2c9210f
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longlong_get.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longlong_get:
+
+shmem_longlong_get
+==================
+    .. include_body
+
+.. include:: ../man3/shmem_char_get.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longlong_get_nbi.3.rst b/docs/man-openshmem/man3/shmem_longlong_get_nbi.3.rst
new file mode 100644
index 00000000000..7d01b61fb96
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longlong_get_nbi.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longlong_get_nbi:
+
+shmem_longlong_get_nbi
+======================
+    .. include_body
+
+.. include:: ../man3/shmem_getmem_nbi.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longlong_iget.3.rst b/docs/man-openshmem/man3/shmem_longlong_iget.3.rst
new file mode 100644
index 00000000000..f05429b9f4e
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longlong_iget.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longlong_iget:
+
+shmem_longlong_iget
+===================
+    .. include_body
+
+.. include:: ../man3/shmem_short_iget.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longlong_inc.3.rst b/docs/man-openshmem/man3/shmem_longlong_inc.3.rst
new file mode 100644
index 00000000000..71b4fa9bfa8
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longlong_inc.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longlong_inc:
+
+shmem_longlong_inc
+==================
+    .. include_body
+
+.. include:: ../man3/shmem_int_inc.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longlong_iput.3.rst b/docs/man-openshmem/man3/shmem_longlong_iput.3.rst
new file mode 100644
index 00000000000..50e119d604b
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longlong_iput.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longlong_iput:
+
+shmem_longlong_iput
+===================
+    .. include_body
+
+.. include:: ../man3/shmem_short_iput.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longlong_max_to_all.3.rst b/docs/man-openshmem/man3/shmem_longlong_max_to_all.3.rst
new file mode 100644
index 00000000000..e2b5b66fe07
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longlong_max_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longlong_max_to_all:
+
+shmem_longlong_max_to_all
+=========================
+    .. include_body
+
+.. include:: ../man3/shmem_short_max_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longlong_min_to_all.3.rst b/docs/man-openshmem/man3/shmem_longlong_min_to_all.3.rst
new file mode 100644
index 00000000000..70b1ee34c66
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longlong_min_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longlong_min_to_all:
+
+shmem_longlong_min_to_all
+=========================
+    .. include_body
+
+.. include:: ../man3/shmem_short_min_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longlong_or_to_all.3.rst b/docs/man-openshmem/man3/shmem_longlong_or_to_all.3.rst
new file mode 100644
index 00000000000..458310642ee
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longlong_or_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longlong_or_to_all:
+
+shmem_longlong_or_to_all
+========================
+    .. include_body
+
+.. include:: ../man3/shmem_short_or_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longlong_p.3.rst b/docs/man-openshmem/man3/shmem_longlong_p.3.rst
new file mode 100644
index 00000000000..4ecc68d2632
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longlong_p.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longlong_p:
+
+shmem_longlong_p
+================
+    .. include_body
+
+.. include:: ../man3/shmem_char_p.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longlong_prod_to_all.3.rst b/docs/man-openshmem/man3/shmem_longlong_prod_to_all.3.rst
new file mode 100644
index 00000000000..762206907e3
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longlong_prod_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longlong_prod_to_all:
+
+shmem_longlong_prod_to_all
+==========================
+    .. include_body
+
+.. include:: ../man3/shmem_short_prod_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longlong_put.3.rst b/docs/man-openshmem/man3/shmem_longlong_put.3.rst
new file mode 100644
index 00000000000..cb1e93e6aa3
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longlong_put.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longlong_put:
+
+shmem_longlong_put
+==================
+    .. include_body
+
+.. include:: ../man3/shmem_char_put.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longlong_put_nbi.3.rst b/docs/man-openshmem/man3/shmem_longlong_put_nbi.3.rst
new file mode 100644
index 00000000000..a0c1c032fc1
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longlong_put_nbi.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longlong_put_nbi:
+
+shmem_longlong_put_nbi
+======================
+    .. include_body
+
+.. include:: ../man3/shmem_putmem_nbi.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longlong_set.3.rst b/docs/man-openshmem/man3/shmem_longlong_set.3.rst
new file mode 100644
index 00000000000..073767df78a
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longlong_set.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longlong_set:
+
+shmem_longlong_set
+==================
+    .. include_body
+
+.. include:: ../man3/shmem_int_set.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longlong_sum_to_all.3.rst b/docs/man-openshmem/man3/shmem_longlong_sum_to_all.3.rst
new file mode 100644
index 00000000000..8885b5c98c1
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longlong_sum_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longlong_sum_to_all:
+
+shmem_longlong_sum_to_all
+=========================
+    .. include_body
+
+.. include:: ../man3/shmem_short_sum_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longlong_swap.3.rst b/docs/man-openshmem/man3/shmem_longlong_swap.3.rst
new file mode 100644
index 00000000000..7a8c5bbf273
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longlong_swap.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longlong_swap:
+
+shmem_longlong_swap
+===================
+    .. include_body
+
+.. include:: ../man3/shmem_swap.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longlong_wait.3.rst b/docs/man-openshmem/man3/shmem_longlong_wait.3.rst
new file mode 100644
index 00000000000..dc88f95d2f7
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longlong_wait.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longlong_wait:
+
+shmem_longlong_wait
+===================
+    .. include_body
+
+.. include:: ../man3/shmem_wait.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longlong_wait_until.3.rst b/docs/man-openshmem/man3/shmem_longlong_wait_until.3.rst
new file mode 100644
index 00000000000..eb39d351875
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longlong_wait_until.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longlong_wait_until:
+
+shmem_longlong_wait_until
+=========================
+    .. include_body
+
+.. include:: ../man3/shmem_wait.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_longlong_xor_to_all.3.rst b/docs/man-openshmem/man3/shmem_longlong_xor_to_all.3.rst
new file mode 100644
index 00000000000..e6c31588397
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_longlong_xor_to_all.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_longlong_xor_to_all:
+
+shmem_longlong_xor_to_all
+=========================
+    .. include_body
+
+.. include:: ../man3/shmem_short_xor_to_all.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_malloc.3.rst b/docs/man-openshmem/man3/shmem_malloc.3.rst
new file mode 100644
index 00000000000..5e95af39978
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_malloc.3.rst
@@ -0,0 +1,106 @@
+.. _shmem_malloc:
+
+
+shmem_malloc
+============
+
+.. include_body
+
+:ref:`shmem_malloc`\ (3), :ref:`shmem_free`\ (3), :ref:`shmem_align`\ (3),
+:ref:`shmem_realloc`\ (3) *shmalloc*\ (3), *shfree*\ (3), *shmemalign*\ (3),
+*shrealloc*\ (3) - Symmetric heap memory management functions.
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void *shmem_malloc(size_t size)
+   void *shmalloc(size_t size)
+
+   void shmem_free(void *ptr)
+   void shfree(void *ptr)
+
+   void *shmem_realloc(void *ptr, size_t size)
+   void *shrealloc(void *ptr, size_t size)
+
+   void *shmem_align(size_t alignment, size_t size)
+   void *shmemalign(size_t alignment, size_t size)
+
+   extern long malloc_error
+
+
+DESCRIPTION
+-----------
+
+The :ref:`shmem_malloc` function returns a pointer to a block of at least
+size bytes suitably aligned for any use. This space is allocated from
+the symmetric heap (in contrast to *malloc*\ (3), which allocates from
+the private heap).
+
+The :ref:`shmem_align` function allocates a block in the symmetric heap
+that has a byte alignment specified by the alignment argument.
+
+The :ref:`shmem_free` function causes the block to which ptr points to, to
+be deallocated, that is, made available for further allocation. If ptr
+is a null pointer, no action occurs; otherwise, if the argument does not
+match a pointer earlier returned by a symmetric heap function, or if the
+space has already been deallocated, malloc_error is set to indicate the
+error, and shfree returns.
+
+The :ref:`shmem_realloc` function changes the size of the block to which
+ptr points to, to the size (in bytes) specified by size.
+
+The contents of the block are unchanged up to the lesser of the new and
+old sizes. If the new size is larger, the value of the newly allocated
+portion of the block is indeterminate. If ptr is a null pointer, the
+shrealloc function behaves like the shmalloc function for the specified
+size. If size is 0 and ptr is not a null pointer, the block to which it
+points to is freed. Otherwise, if ptr does not match a pointer earlier
+returned by a symmetric heap function, or if the space has already been
+deallocated, the malloc_error variable is set to indicate the error, and
+shrealloc returns a null pointer. If the space cannot be allocated, the
+block to which ptr points to is unchanged.
+
+The :ref:`shmem_malloc`, :ref:`shmem_free`, and :ref:`shmem_realloc` functions are provided
+so that multiple PEs in an application can allocate symmetric, remotely
+accessible memory blocks. These memory blocks can then be used with
+(shmem) communication routines. Each of these functions call the
+:ref:`shmem_barrier_all`\ (3) function before returning; this ensures that
+all PEs participate in the memory allocation, and that the memory on
+other PEs can be used as soon as the local PE returns.
+
+The user is responsible for calling these functions with identical
+argument(s) on all PEs; if differing size arguments are used, subsequent
+calls may not return the same symmetric heap address on all PEs.
+
+
+NOTES
+-----
+
+The total size of the symmetric heap is determined at job startup. One
+can adjust the size of the heap using the SHMEM_SYMMETRIC_HEAP_SIZE
+environment variable. See the *intro_shmem*\ (3) man page for further
+details. The :ref:`shmem_malloc`, :ref:`shmem_free`, and :ref:`shmem_realloc` functions
+differ from the private heap allocation functions in that all PEs in an
+application must call them (a barrier is used to ensure this).
+
+
+RETURN VALUES
+-------------
+
+The :ref:`shmem_malloc` function returns a pointer to the allocated space
+(which should be identical on all PEs); otherwise, it returns a null
+pointer (with malloc_error set). The :ref:`shmem_free` function returns no
+value. The :ref:`shmem_realloc` function returns a pointer to the allocated
+space (which may have moved); otherwise, it returns a null pointer (with
+malloc_error set).
+
+
+.. seealso::
+   *intro_shmem*\ (3) *shmem_my_pe*\ (3) *shmem_init*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_my_pe.3.rst b/docs/man-openshmem/man3/shmem_my_pe.3.rst
new file mode 100644
index 00000000000..4d8178aa8bc
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_my_pe.3.rst
@@ -0,0 +1,44 @@
+.. _shmem_my_pe:
+
+
+shmem_my_pe
+===========
+
+.. include_body
+
+:ref:`shmem_my_pe`, my_pe, \_my_pe - Returns the virtual PE number of the
+calling PE.
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   int shmem_my_pe (void)
+   int my_pe (void)
+
+Fortran:
+
+.. code-block:: fortran
+
+   include 'mpp/shmem.fh'
+   I = SHMEM_MY_PE ()
+   I = MY_PE ()
+
+
+DESCRIPTION
+-----------
+
+my_pe() or shmem_my_pe() return the processing element (PE) number of
+the calling PE. It accepts no arguments. The result is an integer
+between 0 and npes - 1, where npes is the total number of PEs executing
+the current program.
+
+
+.. seealso::
+   *intro_shmem*\ (3) *shmem_n_pes*\ (3) *shmem_init*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_n_pes.3.rst b/docs/man-openshmem/man3/shmem_n_pes.3.rst
new file mode 100644
index 00000000000..68d6e1bb207
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_n_pes.3.rst
@@ -0,0 +1,42 @@
+.. _shmem_n_pes:
+
+
+shmem_n_pes
+===========
+
+.. include_body
+
+:ref:`num_pes`, \_num_pes  :ref:`shmem_n_pes` - Returns the number of processing
+elements (PEs) used to run the application.
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   int _num_pes (void)
+   int shmem_n_pes (void)
+
+Fortran:
+
+.. code-block:: fortran
+
+   include 'mpp/shmem.fh'
+   I = NUM_PES ()
+   I = SHMEM_N_PES ()
+
+
+DESCRIPTION
+-----------
+
+num_pes() or shmem_n_pes() return the total number of PEs running in an
+application.
+
+
+.. seealso::
+   *intro_shmem*\ (3) *shmem_my_pe*\ (3) *shmem_init*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_pe_accessible.3.rst b/docs/man-openshmem/man3/shmem_pe_accessible.3.rst
new file mode 100644
index 00000000000..275ea4c2fb2
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_pe_accessible.3.rst
@@ -0,0 +1,57 @@
+.. _shmem_pe_accessible:
+
+
+shmem_pe_accessible
+===================
+
+.. include_body
+
+:ref:`shmem_pe_accessible` - Determines whether a processing element (PE) is
+accessible via SHMEM data transfer operations.
+
+
+SYNOPSIS
+--------
+
+C:
+
+.. code-block:: c
+
+   #include <mpp/shmem.h>
+
+   int shmem_pe_accessible(int pe)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   LOGICAL LOG, SHMEM_PE_ACCESSIBLE
+   INTEGER pe
+
+   LOG = SHMEM_PE_ACCESSIBLE(pe)
+
+
+DESCRIPTION
+-----------
+
+:ref:`shmem_pe_accessible` returns a value that indicates whether the calling
+PE is able to perform OpenSHMEM communication operations with the remote
+PE.
+
+
+RETURN VALUES
+-------------
+
+C/C++
+   The return value is 1 if the specified PE is a valid remote PE for
+   SHMEM functions; otherwise,it is 0.
+
+Fortran
+   The return value is .TRUE. if the specified PE is a valid remote PE
+   for SHMEM functions; otherwise, it is .FALSE..
+
+
+.. seealso::
+   *intro_shmem*\ (3) *shmem_addr_accessible*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_ptr.3.rst b/docs/man-openshmem/man3/shmem_ptr.3.rst
new file mode 100644
index 00000000000..a86ebe137a6
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_ptr.3.rst
@@ -0,0 +1,137 @@
+.. _shmem_ptr:
+
+
+shmem_ptr
+=========
+
+.. include_body
+
+:ref:`shmem_ptr`\ (3) - Returns a pointer to a data object on a specified
+processing element (PE).
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void *shmem_ptr(const void *target, int pe)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   POINTER (PTR, POINTEE)
+   INTEGER pe
+
+   PTR = SHMEM_PTR(target, pe)
+
+
+DESCRIPTION
+-----------
+
+The :ref:`shmem_ptr` routine returns an address that can be used to directly
+reference **target** on the remote PE **pe**. With this address we can
+perform ordinary loads and stores to the remote address.
+
+When a sequence of loads (gets) and stores (puts) to a data object on a
+remote PE does not match the access pattern provided in a SHMEM data
+transfer routine like :ref:`shmem_put32`\ (3) or shmem_real_iget\ (3), the
+:ref:`shmem_ptr` function can provide an efficient means to accomplish the
+communication.
+
+The arguments are as follows:
+
+target
+   The symmetric data object to be referenced.
+
+pe
+   An integer that indicates the PE number on which target is to be
+   accessed. If you are using Fortran, it must be a default integer
+   value.
+
+
+EXAMPLES
+--------
+
+This Fortran program calls :ref:`shmem_ptr` and then PE 0 writes to the BIGD
+array on PE 1:
+
+::
+
+   PROGRAM REMOTEWRITE
+     INCLUDE 'mpp/shmem.fh'
+
+     INTEGER BIGD(100)
+     SAVE BIGD
+     INTEGER POINTEE(*)
+
+     POINTER (PTR,POINTEE)
+     CALL START_PES(0)
+     IF (MY_PE() .EQ. 0) THEN
+                                ! initialize PE 1's BIGD array
+       PTR = SHMEM_PTR(BIGD, 1) ! get address of PE 1's BIGD
+                                ! array
+       DO I=1,100
+         POINTEE(I) = I
+       ENDDO
+     ENDIF
+     CALL SHMEM_BARRIER_ALL
+     IF (MY_PE() .EQ. 1) THEN
+       PRINT *, 'BIGD on PE 1 is: '
+       PRINT *, BIGD
+     ENDIF
+   END
+
+This is the equivalent program written in C:
+
+.. code-block:: c
+
+   #include <mpp/shmem.h>
+
+   main()
+   {
+     static int bigd[100];
+     int *ptr;
+     int i;
+
+     shmem_init();
+     if (shmem_my_pe() == 0) {
+     /* initialize PE 1's bigd array */
+       ptr = shmem_ptr(bigd, 1);
+       for (i=0; i<100; i++)
+         *ptr++ = i+1;
+     }
+     shmem_barrier_all();
+     if (shmem_my_pe() == 1) {
+       printf("bigd on PE 1 is:\n");
+       for (i=0; i<100; i++)
+         printf(" %d\n",bigd[i]);
+       printf("\n");
+     }
+   }
+
+
+NOTES
+-----
+
+The :ref:`shmem_ptr` function is available only on systems where ordinary
+memory loads and stores are used to implement SHMEM put and get
+operations.
+
+
+RETURN VALUES
+-------------
+
+:ref:`shmem_ptr` returns a pointer to the data object on the specified remote
+PE. If target is not remotely accessible, a NULL pointer is returned.
+
+
+.. seealso::
+   *intro_shmem*\ (3) *shmem_put*\ (3) *shmem_get*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_put128.3.rst b/docs/man-openshmem/man3/shmem_put128.3.rst
new file mode 100644
index 00000000000..f5ebf7d723c
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_put128.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_put128:
+
+shmem_put128
+============
+    .. include_body
+
+.. include:: ../man3/shmem_char_put.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_put128_nbi.3.rst b/docs/man-openshmem/man3/shmem_put128_nbi.3.rst
new file mode 100644
index 00000000000..4167d3deb0f
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_put128_nbi.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_put128_nbi:
+
+shmem_put128_nbi
+================
+    .. include_body
+
+.. include:: ../man3/shmem_putmem_nbi.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_put16_nbi.3.rst b/docs/man-openshmem/man3/shmem_put16_nbi.3.rst
new file mode 100644
index 00000000000..1e81e1a57ae
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_put16_nbi.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_put16_nbi:
+
+shmem_put16_nbi
+===============
+    .. include_body
+
+.. include:: ../man3/shmem_putmem_nbi.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_put32.3.rst b/docs/man-openshmem/man3/shmem_put32.3.rst
new file mode 100644
index 00000000000..b194fb80266
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_put32.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_put32:
+
+shmem_put32
+===========
+    .. include_body
+
+.. include:: ../man3/shmem_char_put.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_put32_nbi.3.rst b/docs/man-openshmem/man3/shmem_put32_nbi.3.rst
new file mode 100644
index 00000000000..20749322643
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_put32_nbi.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_put32_nbi:
+
+shmem_put32_nbi
+===============
+    .. include_body
+
+.. include:: ../man3/shmem_putmem_nbi.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_put64.3.rst b/docs/man-openshmem/man3/shmem_put64.3.rst
new file mode 100644
index 00000000000..bae39401a65
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_put64.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_put64:
+
+shmem_put64
+===========
+    .. include_body
+
+.. include:: ../man3/shmem_char_put.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_put64_nbi.3.rst b/docs/man-openshmem/man3/shmem_put64_nbi.3.rst
new file mode 100644
index 00000000000..e8c17837443
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_put64_nbi.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_put64_nbi:
+
+shmem_put64_nbi
+===============
+    .. include_body
+
+.. include:: ../man3/shmem_putmem_nbi.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_put8_nbi.3.rst b/docs/man-openshmem/man3/shmem_put8_nbi.3.rst
new file mode 100644
index 00000000000..63bfa22aed3
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_put8_nbi.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_put8_nbi:
+
+shmem_put8_nbi
+==============
+    .. include_body
+
+.. include:: ../man3/shmem_putmem_nbi.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_putmem.3.rst b/docs/man-openshmem/man3/shmem_putmem.3.rst
new file mode 100644
index 00000000000..dcbfa99eb18
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_putmem.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_putmem:
+
+shmem_putmem
+============
+    .. include_body
+
+.. include:: ../man3/shmem_char_put.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_putmem_nbi.3.rst b/docs/man-openshmem/man3/shmem_putmem_nbi.3.rst
new file mode 100644
index 00000000000..70913f1e60a
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_putmem_nbi.3.rst
@@ -0,0 +1,174 @@
+.. _shmem_putmem_nbi:
+
+
+shmem_putmem_nbi
+================
+
+.. include_body
+
+:ref:`shmem_putmem_nbi`\ (3), :ref:`shmem_char_put_nbi`\ (3),
+:ref:`shmem_short_put_nbi`\ (3), :ref:`shmem_int_put_nbi`\ (3),
+:ref:`shmem_long_put_nbi`\ (3), :ref:`shmem_longlong_put_nbi`\ (3),
+:ref:`shmem_float_put_nbi`\ (3), :ref:`shmem_double_put_nbi`\ (3),
+:ref:`shmem_longdouble_put_nbi`\ (3), :ref:`shmem_put8_nbi`\ (3),
+:ref:`shmem_put16_nbi`\ (3), :ref:`shmem_put32_nbi`\ (3), :ref:`shmem_put64_nbi`\ (3),
+:ref:`shmem_put128_nbi`\ (3), - The nonblocking put routines provide a method
+for copying data from a contiguous local data object to a data object on
+a specified PE.
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void shmem_putmem_nbi(void *dest, const void *source,
+     size_t nelems, int pe)
+
+   void shmem_char_put(char *dest, const char *source,
+     size_t nelems, int pe)
+
+   void shmem_short_put(short *dest, const short *source,
+     size_t nelems, int pe)
+
+   void shmem_int_put(int *dest, const int *source,
+     size_t nelems, int pe)
+
+   void shmem_long_put(long *dest, const long *source,
+     size_t nelems, int pe)
+
+   void shmem_longlong_put(long long *dest, const long long *source,
+     size_t nelems, int pe)
+
+   void shmem_float_put(float *dest, const float *source,
+     size_t nelems, int pe)
+
+   void shmem_double_put(double *dest, const double *source,
+     size_t nelems, int pe)
+
+   void shmem_longdouble_put(long double *dest, const long double *source,
+     size_t nelems, int pe)
+
+   void shmem_put8(void *dest, const void *source,
+     size_t nelems, int pe)
+
+   void shmem_put16(void *dest, const void *source,
+     size_t nelems, int pe)
+
+   void shmem_put32(void *dest, const void *source,
+     size_t nelems, int pe)
+
+   void shmem_put64(void *dest, const void *source,
+     size_t nelems, int pe)
+
+   void shmem_put128(void *dest, const void *source,
+     size_t nelems, int pe)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER nelems, pe
+
+   CALL SHMEM_PUTMEM_NBI(dest, source, nelems, pe)
+
+   CALL SHMEM_CHARACTER_PUT_NBI(dest, source, nelems, pe)
+
+   CALL SHMEM_COMPLEX_PUT_NBI(dest, source, nelems, pe)
+
+   CALL SHMEM_DOUBLE_PUT_NBI(dest, source, nelems, pe)
+
+   CALL SHMEM_INTEGER_PUT_NBI(dest, source, nelems, pe)
+
+   CALL SHMEM_LOGICAL_PUT_NBI(dest, source, nelems, pe)
+
+   CALL SHMEM_REAL_PUT_NBI(dest, source, nelems, pe)
+
+   CALL SHMEM_PUT4_NBI(dest, source, nelems, pe)
+
+   CALL SHMEM_PUT8_NBI(dest, source, nelems, pe)
+
+   CALL SHMEM_PUT32_NBI(dest, source, nelems, pe)
+
+   CALL SHMEM_PUT64_NBI(dest, source, nelems, pe)
+
+   CALL SHMEM_PUT128_NBI(dest, source, nelems, pe)
+
+
+DESCRIPTION
+-----------
+
+The routines return after posting the operation. The operation is
+considered complete after a subsequent call to :ref:`shmem_quiet`. At the
+completion of :ref:`shmem_quiet`, the data has been copied into the dest array
+on the destination PE. The delivery of data words into the data object
+on the destination PE may occur in any order. Furthermore, two
+successive put routines may deliver data out of order unless a call to
+:ref:`shmem_fence` is introduced between the two calls.
+
+The arguments are as follows:
+
+dest
+   Data object to be updated on the remote PE. This data object must be
+   remotely accessible.
+
+source
+   Data object containing the data to be copied.
+
+nelems
+   Number of elements in the dest and source arrays. nelems must be of
+   type size_t for C. If you are using Fortran, it must be a constant,
+   variable, or array element of default integer type.
+
+pe
+   PE number of the remote PE. pe must be of type integer. If you are
+   using Fortran, it must be a constant, variable, or array element of
+   default integer type.
+
+If you are using Fortran, data types must be of default size. For
+example, a real variable must be declared as REAL, REAL*4, or
+REAL(KIND=4).
+
+
+NOTES
+-----
+
+See *intro_shmem*\ (3) for a definition of the term remotely accessible.
+
+
+EXAMPLES
+--------
+
+Consider this simple example for C.
+
+.. code-block:: c
+
+   #include <stdio.h>
+   #include <mpp/shmem.h>
+
+   main()
+   {
+     long source[10] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
+     static long target[10];
+     shmem_init();
+
+     if (shmem_my_pe() == 0) {
+       /* put 10 words into target on PE 1 */
+       shmem_long_put_nbi(target, source, 10, 1);
+       shmem_quiet();
+     }
+     shmem_barrier_all();  /* sync sender and receiver */
+     if (shmem_my_pe() == 1)
+       shmem_udcflush();  /* not required on Altix systems */
+     printf("target[0] on PE %d is %d\n", shmem_my_pe(), target[0]);
+   }
+
+
+.. seealso::
+   *intro_shmem*\ (3) *shmem_quiet*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_quiet.3.rst b/docs/man-openshmem/man3/shmem_quiet.3.rst
new file mode 100644
index 00000000000..6bbfa4e6cb0
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_quiet.3.rst
@@ -0,0 +1,86 @@
+.. _shmem_quiet:
+
+
+shmem_quiet
+===========
+
+.. include_body
+
+:ref:`shmem_quiet`\ (3) - Waits for completion of all outstanding remote
+writes issued by a processing element (PE).
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void shmem_quiet(void)
+
+Fortran:
+
+.. code-block:: fortran
+
+   CALL SHMEM_QUIET
+
+
+DESCRIPTION
+-----------
+
+:ref:`shmem_quiet` ensures ordering of put (remote write) operations. All put
+operations issued to any processing element (PE) prior to the call to
+:ref:`shmem_quiet` are guaranteed to be visible to all other PEs no later than
+any subsequent memory load or store, remote put or get, or
+synchronization operations that follow the call to :ref:`shmem_quiet`.
+
+
+NOTES
+-----
+
+| :ref:`shmem_quiet` is most useful as a way of ensuring ordering of delivery
+  of several put operations. For example, you might use :ref:`shmem_quiet` to
+  await delivery of a block of data before issuing another put, which
+  sets a completion flag on another PE.
+| :ref:`shmem_quiet` is not usually needed if :ref:`shmem_barrier_all`\ (3) or
+  :ref:`shmem_barrier`\ (3) are called. The barrier routines all wait for the
+  completion of outstanding remote writes (puts).
+
+
+EXAMPLES
+--------
+
+::
+
+   PROGRAM COMPFLAG
+     INCLUDE "mpp/shmem.fh"
+
+     INTEGER FLAG_VAR, ARRAY(100), RECEIVER, SENDER
+     COMMON/FLAG/FLAG_VAR
+     COMMON/DATA/ARRAY
+     INTRINSIC MY_PE
+
+     FLAG_VAR = 0
+     CALL SHMEM_BARRIER_ALL ! wait for FLAG_VAR to be initialized
+     SENDER = 0                        ! PE 0 sends the data
+     RECEIVER = 1                      ! PE 1 receives the data
+
+     IF (MY_PE() .EQ. 0) THEN
+       ARRAY = 33
+       CALL SHMEM_PUT(ARRAY, ARRAY, 100, RECEIVER) ! start sending data
+       CALL SHMEM_QUIET                ! wait for delivery
+       CALL SHMEM_PUT(FLAG_VAR, 1, 1, RECEIVER) ! send completion flag
+     ELSE IF (MY_PE() .EQ. RECEIVER) THEN
+       CALL SHMEM_UDCFLUSH
+       CALL SHMEM_WAIT(FLAG_VAR, 0)
+       PRINT *,ARRAY                       ! ARRAY has been delivered
+     ENDIF
+   END
+
+
+.. seealso::
+   *intro_shmem*\ (3) :ref:`shmem_barrier`\ (3) :ref:`shmem_barrier_all`\ (3)
+   *shmem_fence*\ (3) *shmem_put*\ (3) *shmem_wait*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_realloc.3.rst b/docs/man-openshmem/man3/shmem_realloc.3.rst
new file mode 100644
index 00000000000..b71c1548359
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_realloc.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_realloc:
+
+shmem_realloc
+=============
+    .. include_body
+
+.. include:: ../man3/shmem_malloc.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_set_cache_inv.3.rst b/docs/man-openshmem/man3/shmem_set_cache_inv.3.rst
new file mode 100644
index 00000000000..8a96b58025c
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_set_cache_inv.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_set_cache_inv:
+
+shmem_set_cache_inv
+===================
+    .. include_body
+
+.. include:: ../man3/shmem_udcflush.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_set_cache_line_inv.3.rst b/docs/man-openshmem/man3/shmem_set_cache_line_inv.3.rst
new file mode 100644
index 00000000000..59131fb82e6
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_set_cache_line_inv.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_set_cache_line_inv:
+
+shmem_set_cache_line_inv
+========================
+    .. include_body
+
+.. include:: ../man3/shmem_udcflush.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_set_lock.3.rst b/docs/man-openshmem/man3/shmem_set_lock.3.rst
new file mode 100644
index 00000000000..4105cb321be
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_set_lock.3.rst
@@ -0,0 +1,85 @@
+.. _shmem_set_lock:
+
+
+shmem_set_lock
+==============
+
+.. include_body
+
+:ref:`shmem_set_lock`\ (3), :ref:`shmem_clear_lock`\ (3), :ref:`shmem_test_lock`\ (3) -
+Releases, locks, and tests a mutual exclusion memory lock.
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void shmem_clear_lock(volatile long *lock)
+
+   void shmem_set_lock(volatile long *lock)
+
+   int shmem_test_lock(volatile long *lock)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER lock, SHMEM_TEST_LOCK
+
+   CALL SHMEM_CLEAR_LOCK(lock)
+
+   CALL SHMEM_SET_LOCK(lock)
+
+   I = SHMEM_TEST_LOCK(lock)
+
+
+DESCRIPTION
+-----------
+
+The :ref:`shmem_set_lock` routine sets a mutual exclusion lock after waiting
+for the lock to be freed by any other PE currently holding the lock.
+Waiting PEs are assured of getting the lock in a first-come,
+first-served manner.
+
+The :ref:`shmem_clear_lock` routine releases a lock previously set by
+:ref:`shmem_set_lock` after ensuring that all local and remote stores initiated
+in the critical region are complete.
+
+The :ref:`shmem_test_lock` function sets a mutual exclusion lock only if it is
+currently cleared. By using this function, a PE can avoid blocking on a
+set lock. If the lock is currently set, the function returns without
+waiting. These routines are appropriate for protecting a critical region
+from simultaneous update by multiple PEs. They accept the following
+arguments:
+
+lock
+   A symmetric data object that is a scalar variable or an array of
+   length 1. This data object must be set to 0 on all processing
+   elements (PEs) prior to the first use. lock must be of type integer.
+   If you are using Fortran, it must be of default kind.
+
+
+NOTES
+-----
+
+The term symmetric data object is defined on *intro_shmem*\ (3).
+
+
+RETURN VALUES
+-------------
+
+The :ref:`shmem_test_lock` function returns 0 if the lock was originally
+cleared and this call was able to set the lock. A value of 1 is returned
+if the lock had been set and the call returned without waiting to set
+the lock.
+
+
+.. seealso::
+   *intro_shmem*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_short_and_to_all.3.rst b/docs/man-openshmem/man3/shmem_short_and_to_all.3.rst
new file mode 100644
index 00000000000..7f4b598b659
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_short_and_to_all.3.rst
@@ -0,0 +1,219 @@
+.. _shmem_short_and_to_all:
+
+
+shmem_short_and_to_all
+======================
+
+.. include_body
+
+:ref:`shmem_int_and_to_all`\ (3), shmem_int4_and_to_all\ (3),
+shmem_int8_and_to_all\ (3), :ref:`shmem_long_and_to_all`\ (3),
+:ref:`shmem_longlong_and_to_all`\ (3), :ref:`shmem_short_and_to_all`\ (3) -
+Performs a bitwise AND operation on symmetric arrays over the active set
+of PEs.
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void shmem_int_and_to_all(int *target, const int *source,
+     int nreduce, int PE_start, int logPE_stride, int PE_size,
+     int *pWrk, long *pSync)
+
+   void shmem_long_and_to_all(long *target, const long *source,
+     int nreduce, int PE_start, int  logPE_stride, int PE_size,
+     long *pWrk, long *pSync)
+
+   void shmem_longlong_and_to_all(long long *target,
+     const long long *source, int nreduce, int PE_start, int logPE_stride,
+     int PE_size, long long *pWrk, long *pSync)
+
+   void shmem_short_and_to_all(short *target, const short *source,
+     int nreduce, int PE_start, int logPE_stride, int PE_size,
+     short *pWrk, long *pSync)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER pSync(SHMEM_REDUCE_SYNC_SIZE)
+   INTEGER nreduce, PE_start, logPE_stride, PE_size
+
+   CALL SHMEM_INT4_AND_TO_ALL(target, source, nreduce,
+   & PE_start, logPE_stride, PE_size, pWrk, pSync)
+
+   CALL SHMEM_INT8_AND_TO_ALL(target, source, nreduce,
+   & PE_start, logPE_stride, PE_size, pWrk, pSync)
+
+
+DESCRIPTION
+-----------
+
+The shared memory (SHMEM) reduction routines compute one or more
+reductions across symmetric arrays on multiple virtual PEs. A reduction
+performs an associative binary operation across a set of values. For a
+list of other SHMEM reduction routines, see *intro_shmem*\ (3).
+
+The nreduce argument determines the number of separate reductions to
+perform. The source array on all PEs in the active set provides one
+element for each reduction. The results of the reductions are placed in
+the target array on all PEs in the active set. The active set is defined
+by the PE_start, logPE_stride, PE_size triplet.
+
+The source and target arrays may be the same array, but they may not be
+overlapping arrays. As with all SHMEM collective routines, each of these
+routines assumes that only PEs in the active set call the routine. If a
+PE not in the active set calls a SHMEM collective routine, undefined
+behavior results.
+
+The arguments are as follows:
+
+target
+   A symmetric array, of length nreduce elements, to receive the result
+   of the reduction operations. The data type of target varies with the
+   version of the reduction routine being called. When calling from
+   C/C++, refer to the SYNOPSIS section for data type information. When
+   calling from Fortran, the target data types are as follows:
+
+   shmem_int8_and_to_all: Integer, with an element size of 8 bytes
+
+   shmem_int4_and_to_all: Integer, with an element size of 4 bytes
+
+source
+   A symmetric array, of length nreduce elements, that contains one
+   element for each separate reduction operation. The source argument
+   must have the same data type as target.
+
+nreduce
+   The number of elements in the target and source arrays. nreduce must
+   be of type integer. If you are using Fortran, it must be a default
+   integer value.
+
+PE_start
+   The lowest virtual PE number of the active set of PEs. PE_start must
+   be of type integer. If you are using Fortran, it must be a default
+   integer value.
+
+logPE_stride
+   The log (base 2) of the stride between consecutive virtual PE numbers
+   in the active set. logPE_stride must be of type integer. If you are
+   using Fortran, it must be a default integer value.
+
+PE_size
+   The number of PEs in the active set. PE_size must be of type integer.
+   If you are using Fortran, it must be a default integer value.
+
+pWrk
+   A symmetric work array. The pWrk argument must have the same data
+   type as target. In C/C++, this contains max(nreduce/2 + 1,
+   \_SHMEM_REDUCE_MIN_WRKDATA_SIZE) elements. In Fortran, this contains
+   max(nreduce/2 + 1, SHMEM_REDUCE_MIN_WRKDATA_SIZE) elements.
+
+pSync
+   A symmetric work array. In C/C++, pSync must be of type long and size
+   \_SHMEM_REDUCE_SYNC_SIZE. In Fortran, pSync must be of type integer
+   and size SHMEM_REDUCE_SYNC_SIZE. If you are using Fortran, it must be
+   a default integer value. Every element of this array must be
+   initialized with the value \_SHMEM_SYNC_VALUE (in C/C++) or
+   SHMEM_SYNC_VALUE (in Fortran) before any of the PEs in the active set
+   enter the reduction routine.
+
+The values of arguments nreduce, PE_start, logPE_stride, and PE_size
+must be equal on all PEs in the active set. The same target and source
+arrays, and the same pWrk and pSync work arrays, must be passed to all
+PEs in the active set.
+
+Before any PE calls a reduction routine, you must ensure that the
+following conditions exist (synchronization via a barrier or some other
+method is often needed to ensure this): The pWrk and pSync arrays on all
+PEs in the active set are not still in use from a prior call to a
+collective SHMEM routine. The target array on all PEs in the active set
+is ready to accept the results of the reduction.
+
+Upon return from a reduction routine, the following are true for the
+local PE: The target array is updated. The values in the pSync array are
+restored to the original values.
+
+
+NOTES
+-----
+
+The terms collective, symmetric, and cache aligned are defined in
+*intro_shmem*\ (3). All SHMEM reduction routines reset the values in
+pSync before they return, so a particular pSync buffer need only be
+initialized the first time it is used.
+
+You must ensure that the pSync array is not being updated on any PE in
+the active set while any of the PEs participate in processing of a SHMEM
+reduction routine. Be careful to avoid the following situations: If the
+pSync array is initialized at run time, some type of synchronization is
+needed to ensure that all PEs in the working set have initialized pSync
+before any of them enter a SHMEM routine called with the pSync
+synchronization array. A pSync or pWrk array can be reused in a
+subsequent reduction routine call only if none of the PEs in the active
+set are still processing a prior reduction routine call that used the
+same pSync or pWrk arrays. In general, this can be assured only by doing
+some type of synchronization. However, in the special case of reduction
+routines being called with the same active set, you can allocate two
+pSync and pWrk arrays and alternate between them on successive calls.
+
+
+EXAMPLES
+--------
+
+**Example 1**: This Fortran example statically initializes the pSync
+array and finds the logical AND of the integer variable FOO across all
+even PEs.
+
+::
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER PSYNC(SHMEM_REDUCE_SYNC_SIZE)
+   DATA PSYNC /SHMEM_REDUCE_SYNC_SIZE*SHMEM_SYNC_VALUE/
+   PARAMETER (NR=1)
+   REAL PWRK(MAX(NR/2+1, SHMEM_REDUCE_MIN_WRKDATA_SIZE))
+   INTEGER FOO, FOOAND
+   COMMON /COM/ FOO, FOOAND, PWRK
+   INTRINSIC MY_PE
+
+   IF ( MOD(MY_PE(),2) .EQ. 0) THEN
+     CALL SHMEM_INT8_AND_TO_ALL(FOOAND, FOO, NR, 0, 1, N$PES/2,
+     & PWRK, PSYNC)
+     PRINT *, 'Result on PE ', MY_PE(), ' is ', FOOAND
+   ENDIF
+
+**Example 2**: Consider the following C call:
+
+.. code-block:: c
+
+   shmem_int_and_to_all( target, source, 3, 0, 0, 8, pwrk, psync );
+
+The preceding call is more efficient, but semantically equivalent to,
+the combination of the following calls:
+
+::
+
+   shmem_int_and_to_all(&(target[0]), &(source[0]), 1, 0, 0, 8,
+     pwrk1, psync1);
+
+   shmem_int_and_to_all(&(target[1]), &(source[1]), 1, 0, 0, 8,
+     pwrk2, psync2);
+
+   shmem_int_and_to_all(&(target[2]), &(source[2]), 1, 0, 0, 8,
+     pwrk1, psync1);
+
+Note that two sets of pWrk and pSync arrays are used alternately because
+no synchronization is done between calls.
+
+
+.. seealso::
+   *f90*\ (1) *intro_shmem*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_short_g.3.rst b/docs/man-openshmem/man3/shmem_short_g.3.rst
new file mode 100644
index 00000000000..8075e40fb62
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_short_g.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_short_g:
+
+shmem_short_g
+=============
+    .. include_body
+
+.. include:: ../man3/shmem_char_g.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_short_get.3.rst b/docs/man-openshmem/man3/shmem_short_get.3.rst
new file mode 100644
index 00000000000..c1047a441fc
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_short_get.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_short_get:
+
+shmem_short_get
+===============
+    .. include_body
+
+.. include:: ../man3/shmem_char_get.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_short_get_nbi.3.rst b/docs/man-openshmem/man3/shmem_short_get_nbi.3.rst
new file mode 100644
index 00000000000..1a97b203b62
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_short_get_nbi.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_short_get_nbi:
+
+shmem_short_get_nbi
+===================
+    .. include_body
+
+.. include:: ../man3/shmem_getmem_nbi.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_short_iget.3.rst b/docs/man-openshmem/man3/shmem_short_iget.3.rst
new file mode 100644
index 00000000000..e77dfd5cb88
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_short_iget.3.rst
@@ -0,0 +1,213 @@
+.. _shmem_short_iget:
+
+
+shmem_short_iget
+================
+
+.. include_body
+
+shmem_complex_iget\ (3), :ref:`shmem_double_iget`\ (3),
+:ref:`shmem_float_iget`\ (3), shmem_iget4\ (3), shmem_iget8\ (3),
+:ref:`shmem_iget32`\ (3), :ref:`shmem_iget64`\ (3), :ref:`shmem_iget128`\ (3),
+:ref:`shmem_int_iget`\ (3), shmem_integer_iget\ (3),
+shmem_logical_iget\ (3), :ref:`shmem_long_iget`\ (3),
+:ref:`shmem_longdouble_iget`\ (3), :ref:`shmem_longlong_iget`\ (3),
+shmem_real_iget\ (3), :ref:`shmem_short_iget`\ (3) - Transfers strided data
+from a specified processing element (PE)
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void shmem_iget32(void *target, const void *source,
+     ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe)
+
+   void shmem_iget64(void *target, const void *source,
+     ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe)
+
+   void shmem_iget128(void *target, const void *source,
+     ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe)
+
+   void shmem_int_iget(int *target, const int *source,
+     ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe)
+
+   void shmem_double_iget(double *target, const double *source,
+     ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe)
+
+   void shmem_float_iget(float *target, const float *source,
+     ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe)
+
+   void shmem_long_iget(long *target, const long *source,
+     ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe)
+
+   void shmem_longdouble_iget(long double *target,
+     const long double *source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe)
+
+   void shmem_longlong_iget(long long *target,
+     const long long *source, ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe)
+
+   void shmem_short_iget(short *target,
+     const short *source, ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER tst, sst, len, pe
+
+   CALL SHMEM_COMPLEX_IGET(target, source, tst, sst, len,
+   & pe)
+
+   CALL SHMEM_DOUBLE_IGET(target, source, tst, sst, len,
+   & pe)
+
+   CALL SHMEM_IGET4(target, source, tst, sst, len, pe)
+
+   CALL SHMEM_IGET8(target, source, tst, sst, len, pe)
+
+   CALL SHMEM_IGET32(target, source, tst, sst, len, pe)
+
+   CALL SHMEM_IGET64(target, source, tst, sst, len, pe)
+
+   CALL SHMEM_IGET128(target, source, tst, sst, len, pe)
+
+   CALL SHMEM_INTEGER_IGET(target, source, tst, sst, len,
+   & pe)
+
+   CALL SHMEM_LOGICAL_IGET(target, source, tst, sst, len,
+   & pe)
+
+   CALL SHMEM_REAL_IGET(target, source, tst, sst, len, pe)
+
+
+DESCRIPTION
+-----------
+
+The strided get routines retrieve array data available at address source
+on remote PE (pe). The elements of the **source** array are separated by
+a stride **sst**. Once the data is received, it is stored at the local
+memory address **target**, separated by stride **tst**. The routines
+return when the data has been copied into the local **target** array.
+
+The arguments are as follows:
+
+target
+   Array to be updated on the local PE.
+
+source
+   Array containing the data to be copied on the remote PE.
+
+tst
+   The stride between consecutive elements of the target array. The
+   stride is scaled by the element size of the target array. A value of
+   1 indicates contiguous data. tst must be of type integer. If you are
+   calling from Fortran, it must be a default integer value.
+
+sst
+   The stride between consecutive elements of the source array. The
+   stride is scaled by the element size of the source array. A value of
+   1 indicates contiguous data. sst must be of type integer. If you are
+   calling from Fortran, it must be a default integer value.
+
+len
+   Number of elements in the target and source arrays. len must be of
+   type integer. If you are using Fortran, it must be a constant,
+   variable, or array element of default integer type.
+
+pe
+   PE number of the remote PE. pe must be of type integer. If you are
+   using Fortran, it must be a constant, variable, or array element of
+   default integer type.
+
+The target and source data objects must conform to typing constraints,
+which are as follows:
+
+:ref:`shmem_iget32`, shmem_iget4: Any noncharacter type that has a storage size
+   equal to 32 bits.
+
+:ref:`shmem_iget64`, shmem_iget8: Any noncharacter type that has a storage size
+   equal to 64 bits.
+
+:ref:`shmem_iget128`: Any noncharacter type that has a storage size equal to
+   128 bits.
+
+:ref:`shmem_short_iget`: Elements of type short.
+
+:ref:`shmem_int_iget`: Elements of type int.
+
+:ref:`shmem_long_iget`: Elements of type long.
+
+:ref:`shmem_longlong_iget`: Elements of type long long.
+
+:ref:`shmem_float_iget`: Elements of type float.
+
+:ref:`shmem_double_iget`: Elements of type double.
+
+:ref:`shmem_longdouble_iget`: Elements of type long double.
+
+**SHMEM_COMPLEX_IGET**: Elements of type complex of default size.
+
+**SHMEM_DOUBLE_IGET**: (Fortran) Elements of type double precision.
+
+**SHMEM_INTEGER_IGET**: Elements of type integer.
+
+**SHMEM_LOGICAL_IGET**: Elements of type logical.
+
+**SHMEM_REAL_IGET**: Elements of type real.
+
+:ref:`shmem_longdouble_iget`: Elements of type long double.
+
+**SHMEM_COMPLEX_IGET**: Elements of type complex of default size.
+
+**SHMEM_DOUBLE_IGET**: (Fortran) Elements of type double precision.
+
+**SHMEM_INTEGER_IGET**: Elements of type integer.
+
+**SHMEM_LOGICAL_IGET**: Elements of type logical.
+
+**SHMEM_REAL_IGET**: Elements of type real.
+
+If you are using Fortran, data types must be of default size. For
+example, a real variable must be declared as REAL, REAL*4, or
+REAL(KIND=4).
+
+
+NOTES
+-----
+
+See *intro_shmem*\ (3) for a definition of the term remotely accessible.
+
+
+EXAMPLES
+--------
+
+The following simple example uses shmem_logical_iget in a Fortran
+program. Compile this example with the -lsma compiler option.
+
+::
+
+   PROGRAM STRIDELOGICAL
+     LOGICAL SOURCE(10), TARGET(5)
+     SAVE SOURCE ! SAVE MAKES IT REMOTELY ACCESSIBLE
+     DATA SOURCE /.T.,.F.,.T.,.F.,.T.,.F.,.T.,.F.,.T.,.F./
+     DATA TARGET / 5*.F. /
+
+     CALL START_PES(2)
+     IF (MY_PE() .EQ. 0) THEN
+       CALL SHMEM_LOGICAL_IGET(TARGET, SOURCE, 1, 2, 5, 1)
+       PRINT*,'TARGET AFTER SHMEM_LOGICAL_IGET:',TARGET
+     ENDIF
+     CALL SHMEM_BARRIER_ALL
+   END
+
+
+.. seealso::
+   *intro_shmem*\ (3) *shmem_get*\ (3) *shmem_quiet*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_short_iput.3.rst b/docs/man-openshmem/man3/shmem_short_iput.3.rst
new file mode 100644
index 00000000000..c52ef01608a
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_short_iput.3.rst
@@ -0,0 +1,216 @@
+.. _shmem_short_iput:
+
+
+shmem_short_iput
+================
+
+.. include_body
+
+shmem_complex_iput\ (3), :ref:`shmem_double_iput`\ (3),
+:ref:`shmem_float_iput`\ (3), :ref:`shmem_int_iput`\ (3),
+shmem_integer_iput\ (3), shmem_iput4\ (3), shmem_iput8\ (3),
+:ref:`shmem_iput32`\ (3), :ref:`shmem_iput64`\ (3), :ref:`shmem_iput128`\ (3),
+shmem_logical_iput\ (3), :ref:`shmem_long_iput`\ (3),
+:ref:`shmem_longdouble_iput`\ (3), :ref:`shmem_longlong_iput`\ (3),
+shmem_real_iput\ (3), :ref:`shmem_short_iput`\ (3) - Transfer strided data
+to a specified processing element (PE).
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void shmem_double_iput(double *target, const double *source,
+     ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe)
+
+   void shmem_float_iput(float *target, const float *source,
+     ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe)
+
+   void shmem_int_iput(int *target, const int *source,
+     ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe)
+
+   void shmem_iput32(void *target, const void *source,
+     ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe)
+
+   void shmem_iput64(void *target, const void *source,
+     ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe)
+
+   void shmem_iput128(void *target, const void *source,
+     ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe)
+
+   void shmem_long_iput(long *target, const long *source,
+     ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe)
+
+   void shmem_longdouble_iput(long double *target,
+     const long double *source, ptrdiff_t tst, ptrdiff_t sst,
+     size_t len, int pe)
+
+   void shmem_longlong_iput(long long *target,
+     const long long *source, ptrdiff_t tst, ptrdiff_t sst,
+     size_t len, int pe)
+
+   void shmem_short_iput(short *target, const short *source,
+     ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER tst, sst, len, pe
+
+   CALL SHMEM_COMPLEX_IPUT(target, source, tst, sst, len,
+   & pe)
+
+   CALL SHMEM_DOUBLE_IPUT(target, source, tst, sst, len,
+   & pe)
+
+   CALL SHMEM_INTEGER_IPUT(target, source, tst, sst, len,
+   & pe)
+
+   CALL SHMEM_IPUT4(target, source, tst, sst, len, pe)
+
+   CALL SHMEM_IPUT8(target, source, tst, sst, len, pe)
+
+   CALL SHMEM_IPUT32(target, source, tst, sst, len, pe)
+
+   CALL SHMEM_IPUT64(target, source, tst, sst, len, pe)
+
+   CALL SHMEM_IPUT128(target, source, tst, sst, len, pe)
+
+   CALL SHMEM_LOGICAL_IPUT(target, source, tst, sst, len,
+   & pe)
+
+   CALL SHMEM_REAL_IPUT(target, source, tst, sst, len, pe)
+
+
+DESCRIPTION
+-----------
+
+The shmem_iput routines read the elements of a local array (**source**)
+and write them to a remote array (**target**) on the PE indicated by
+**pe**. These routines return when the data has been copied out of the
+source array on the local PE but not necessarily before the data has
+been delivered to the remote data object.
+
+The arguments are as follows:
+
+target
+   Array to be updated on the remote PE. This data object must be
+   remotely accessible.
+
+source
+   Array containing the data to be copied.
+
+tst
+   The stride between consecutive elements of the target array. The
+   stride is scaled by the element size of the target array. A value of
+   1 indicates contiguous data. tst must be of type integer. If you are
+   using Fortran, it must be a default integer value.
+
+sst
+   The stride between consecutive elements of the source array. The
+   stride is scaled by the element size of the source array. A value of
+   1 indicates contiguous data. sst must be of type integer. If you are
+   using Fortran, it must be a default integer value.
+
+len
+   Number of elements in the target and source arrays. len must be of
+   type integer. If you are using Fortran, it must be a constant,
+   variable, or array element of default integer type.
+
+pe
+   PE number of the remote PE. pe must be of type integer. If you are
+   using Fortran, it must be a constant, variable, or array element of
+   default integer type.
+
+The target and source data objects must conform to typing constraints,
+which are as follows:
+
+:ref:`shmem_iput32`, shmem_iput4: Any noncharacter type that has a storage size equal
+   to 32 bits.
+
+:ref:`shmem_iput64`, shmem_iput8: Any noncharacter type that has a storage size equal
+   to 64 bits.
+
+:ref:`shmem_iput128`: Any noncharacter type that has a storage size equal to 128 bits.
+
+:ref:`shmem_short_iput`: Elements of type short.
+
+:ref:`shmem_int_iput`: Elements of type int.
+
+:ref:`shmem_long_iput`: Elements of type long.
+
+:ref:`shmem_longlong_iput`: Elements of type long long.
+
+:ref:`shmem_float_iput`: Elements of type float.
+
+:ref:`shmem_double_iput`: Elements of type double.
+
+:ref:`shmem_longdouble_iput`: Elements of type long double.
+
+**SHMEM_COMPLEX_IPUT**: Elements of type complex of default size.
+
+**SHMEM_DOUBLE_IPUT**: (Fortran) Elements of type double precision.
+
+**SHMEM_INTEGER_IPUT**: Elements of type integer.
+
+**SHMEM_LOGICAL_IPUT**: Elements of type logical.
+
+**SHMEM_REAL_IPUT**: Elements of type real.
+
+**SHMEM_LOGICAL_IPUT**: Elements of type logical.
+
+**SHMEM_REAL_IPUT**: Elements of type real.
+
+If you are using Fortran, data types must be of default size. For
+example, a real variable must be declared as REAL, REAL*4 or
+REAL(KIND=4).
+
+
+NOTES
+-----
+
+See *intro_shmem*\ (3) for a definition of the term remotely accessible.
+
+
+EXAMPLES
+--------
+
+Consider the following simple :ref:`shmem_long_iput` example for C/C++
+programs.
+
+::
+
+   #include <mpp/shmem.h>
+
+   main()
+   {
+     short source[10] = { 1, 2, 3, 4, 5,
+     6, 7, 8, 9, 10 };
+     static short target[10];
+
+     shmem_init();
+     if (shmem_my_pe() == 0) {
+       /* put 10 words into target on PE 1 */
+       shmem_short_iput(target, source, 1, 2, 5, 1);
+     }
+     shmem_barrier_all(); /* sync sender and receiver */
+     if (shmem_my_pe() == 1) {
+       shmem_udcflush(); /* not required on IRIX systems */
+       printf("target on PE %d is %d %d %d %d %d0, shmem_my_pe(),
+       (int)target[0], (int)target[1], (int)target[2],
+       (int)target[3], (int)target[4] );
+     }
+     shmem_barrier_all(); /* sync before exiting */
+   }
+
+
+.. seealso::
+   *intro_shmem*\ (3) *shmem_iget*\ (3) *shmem_put*\ (3) *shmem_quiet*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_short_max_to_all.3.rst b/docs/man-openshmem/man3/shmem_short_max_to_all.3.rst
new file mode 100644
index 00000000000..b4c72779e42
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_short_max_to_all.3.rst
@@ -0,0 +1,253 @@
+.. _shmem_short_max_to_all:
+
+
+shmem_short_max_to_all
+======================
+
+.. include_body
+
+:ref:`shmem_double_max_to_all`\ (3), :ref:`shmem_float_max_to_all`\ (3),
+:ref:`shmem_int_max_to_all`\ (3), shmem_int4_max_to_all\ (3),
+shmem_int8_max_to_all\ (3), :ref:`shmem_long_max_to_all`\ (3),
+:ref:`shmem_longdouble_max_to_all`\ (3), :ref:`shmem_longlong_max_to_all`\ (3),
+shmem_real4_max_to_all\ (3), shmem_real8_max_to_all\ (3),
+shmem_real16_max_to_all\ (3), :ref:`shmem_short_max_to_all`\ (3) - Performs
+a maximum function reduction across a set of processing elements (PEs).
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void shmem_double_max_to_all(double *target, const double *source,
+     int nreduce, int PE_start, int logPE_stride, int PE_size,
+     double *pWrk, long *pSync)
+
+   void shmem_float_max_to_all(float *target, const float *source,
+     int nreduce, int PE_start, int logPE_stride, int PE_size,
+     float *pWrk, long *pSync)
+
+   void shmem_int_max_to_all(int *target, const int *source,
+     int nreduce, int PE_start, int logPE_stride, int PE_size,
+     int *pWrk, long *pSync)
+
+   void shmem_long_max_to_all(long *target, const long *source,
+     int nreduce, int PE_start, int logPE_stride, int PE_size,
+     long *pWrk, long *pSync)
+
+   void shmem_longdouble_max_to_all(long double *target,
+     const long double *source, int nreduce, int PE_start,
+     int logPE_stride, int PE_size, long double *pWrk, long *pSync)
+
+   void shmem_longlong_max_to_all(long long *target,
+     const long long *source, int nreduce,  int PE_start,
+     int logPE_stride, int PE_size, long long *pWrk, long *pSync)
+
+    void shmem_short_max_to_all(short *target, const short *source,
+     int nreduce, int PE_start, int logPE_stride, int PE_size,
+     short *pWrk, long *pSync)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER pSync(SHMEM_REDUCE_SYNC_SIZE)
+
+   INTEGER nreduce, PE_start, logPE_stride, PE_size
+
+   CALL SHMEM_INT4_MAX_TO_ALL(target, source, nreduce,
+   & PE_start, logPE_stride, PE_size, pWrk, pSync)
+
+   CALL SHMEM_INT8_MAX_TO_ALL(target, source, nreduce,
+   & PE_start, logPE_stride, PE_size, pWrk, pSync)
+
+   CALL SHMEM_REAL4_MAX_TO_ALL(target, source, nreduce,
+   & PE_start, logPE_stride, PE_size, pWrk, pSync)
+
+   CALL SHMEM_REAL8_MAX_TO_ALL(target, source, nreduce,
+   & PE_start, logPE_stride, PE_size, pWrk, pSync)
+
+   CALL SHMEM_REAL16_MAX_TO_ALL(target, source, nreduce,
+   & PE_start, logPE_stride, PE_size, pWrk, pSync)
+
+
+DESCRIPTION
+-----------
+
+The shared memory (SHMEM) reduction routines compute one or more
+reductions across symmetric arrays on multiple virtual PEs. A reduction
+performs an associative binary operation across a set of values. For a
+list of other SHMEM reduction routines, see *intro_shmem*\ (3).
+
+As with all SHMEM collective routines, each of these routines assumes
+that only PEs in the active set call the routine. If a PE not in the
+active set calls a SHMEM collective routine, undefined behavior results.
+
+The nreduce argument determines the number of separate reductions to
+perform. The source array on all PEs in the active set provides one
+element for each reduction. The results of the reductions are placed in
+the target array on all PEs in the active set. The active set is defined
+by the PE_start, logPE_stride, PE_size triplet.
+
+The source and target arrays may be the same array, but they may not be
+overlapping arrays.
+
+The arguments are as follows:
+
+target
+   A symmetric array of length nreduce elements to receive the results
+   of the reduction operations. The data type of target varies with the
+   version of the reduction routine being called. When calling from C,
+   refer to the SYNOPSIS section for data type information.
+
+When calling from Fortran, the target data types are as follows:
+
+   shmem_comp8_max_to_all: Complex, with an element size equal to two
+      8-byte real values.
+
+   shmem_int4_max_to_all: Integer, with an element size of 4 bytes.
+
+   shmem_int8_max_to_all: Integer, with an element size of 8 bytes.
+
+   shmem_real4_max_to_all: Real, with an element size of 4 bytes.
+
+   shmem_real16_max_to_all: Real, with an element size of 16 bytes.
+
+..
+
+source
+   A symmetric array of length nreduce elements that contains one
+   element for each separate reduction operation. The source argument
+   must have the same data type as target.
+
+nreduce
+   The number of elements in the target and source arrays. nreduce must
+   be of type integer. If you are using Fortran, it must be a default
+   integer value.
+
+PE_start
+   The lowest virtual PE number of the active set of PEs. PE_start must
+   be of type integer. If you are using Fortran, it must be a default
+   integer value.
+
+logPE_stride
+   The log (base 2) of the stride between consecutive virtual PE numbers
+   in the active set. logPE_stride must be of type integer. If you are
+   using Fortran, it must be a default integer value.
+
+PE_size
+   The number of PEs in the active set. PE_size must be of type integer.
+   If you are using Fortran, it must be a default integer value.
+
+pWrk
+   A symmetric work array. The pWrk argument must have the same data
+   type as target. In C/C++, this contains max(nreduce/2 + 1,
+   \_SHMEM_REDUCE_MIN_WRKDATA_SIZE) elements. In Fortran, this contains
+   max(nreduce/2 + 1, SHMEM_REDUCE_MIN_WRKDATA_SIZE) elements.
+
+pSync
+   A symmetric work array. In C/C++, pSync is of type long and size
+   \_SHMEM_REDUCE_SYNC_SIZE. In Fortran, pSync is of type integer and
+   size SHMEM_REDUCE_SYNC_SIZE. If you are using Fortran, it must be a
+   default integer value. Every element of this array must be
+   initialized with the value \_SHMEM_SYNC_VALUE (in C/C++) or
+   SHMEM_SYNC_VALUE (in Fortran) before any of the PEs in the active set
+   enter the reduction routine.
+
+The values of arguments nreduce, PE_start, logPE_stride, and PE_size
+must be equal on all PEs in the active set. The same target and source
+arrays, and the same pWrk and pSync work arrays, must be passed to all
+PEs in the active set.
+
+Before any PE calls a reduction routine, you must ensure that the
+following conditions exist (synchronization via a barrier or some other
+method is often needed to ensure this): The pWrk and pSync arrays on all
+PEs in the active set are not still in use from a prior call to a
+collective SHMEM routine. The target array on all PEs in the active set
+is ready to accept the results of the reduction.
+
+Upon return from a reduction routine, the following are true for the
+local PE: The target array is updated. The values in the pSync array are
+restored to the original values.
+
+
+NOTES
+-----
+
+The terms collective, symmetric, and cache aligned are defined in
+*intro_shmem*\ (3). All SHMEM reduction routines reset the values in
+pSync before they return, so a particular pSync buffer need only be
+initialized the first time it is used.
+
+You must ensure that the pSync array is not being updated on any PE in
+the active set while any of the PEs participate in processing of a SHMEM
+reduction routine. Be careful of the following situations: If the pSync
+array is initialized at run time, some type of synchronization is needed
+to ensure that all PEs in the working set have initialized pSync before
+any of them enter a SHMEM routine called with the pSync synchronization
+array. A pSync or pWrk array can be reused in a subsequent reduction
+routine call only if none of the PEs in the active set are still
+processing a prior reduction routine call that used the same pSync or
+pWrk arrays.
+
+In general, this can be assured only by doing some type of
+synchronization. However, in the special case of reduction routines
+being called with the same active set, you can allocate two pSync and
+pWrk arrays and alternate between them on successive calls.
+
+
+EXAMPLES
+--------
+
+**Example 1:** This Fortran example statically initializes the pSync
+array and finds the maximum value of real variable FOO across all even
+PEs.
+
+::
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER PSYNC(SHMEM_REDUCE_SYNC_SIZE)
+   DATA PSYNC /SHMEM_REDUCE_SYNC_SIZE*SHMEM_SYNC_VALUE/
+   PARAMETER (NR=1)
+   REAL FOO, FOOMAX, PWRK(MAX(NR/2+1,SHMEM_REDUCE_MIN_WRKDATA_SIZE))
+   COMMON /COM/ FOO, FOOMAX, PWRK
+   INTRINSIC MY_PE
+
+   IF ( MOD(MY_PE(),2) .EQ. 0) THEN
+     CALL SHMEM_REAL8_MAX_TO_ALL(FOOMAX, FOO, NR, 0, 1, N$PES/2,
+     & PWRK, PSYNC)
+     PRINT *, 'Result on PE ', MY_PE(), ' is ', FOOMAX
+   ENDIF
+
+**Example 2:** Consider the following C/C++ call:
+
+.. code-block:: c++
+
+   shmem_int_max_to_all( target, source, 3, 0, 0, 8, pwrk, psync );
+
+The preceding call is more efficient, but semantically equivalent to,
+the combination of the following calls:
+
+::
+
+   shmem_int_max_to_all(&(target[0]), &(source[0]), 1, 0, 0, 8,
+     pwrk1, psync1);
+   shmem_int_max_to_all(&(target[1]), &(source[1]), 1, 0, 0, 8,
+     pwrk2, psync2);
+   shmem_int_max_to_all(&(target[2]), &(source[2]), 1, 0, 0, 8,
+     pwrk1, psync1);
+
+Note that two sets of pWrk and pSync arrays are used alternately because
+no synchronization is done between calls.
+
+
+.. seealso::
+   *intro_shmem*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_short_min_to_all.3.rst b/docs/man-openshmem/man3/shmem_short_min_to_all.3.rst
new file mode 100644
index 00000000000..1ce89284494
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_short_min_to_all.3.rst
@@ -0,0 +1,246 @@
+.. _shmem_short_min_to_all:
+
+
+shmem_short_min_to_all
+======================
+
+.. include_body
+
+:ref:`shmem_double_min_to_all`\ (3), :ref:`shmem_float_min_to_all`\ (3),
+:ref:`shmem_int_min_to_all`\ (3), shmem_int4_min_to_all\ (3),
+shmem_int8_min_to_all\ (3), :ref:`shmem_long_min_to_all`\ (3),
+:ref:`shmem_longdouble_min_to_all`\ (3), :ref:`shmem_longlong_min_to_all`\ (3),
+shmem_real4_min_to_all\ (3), shmem_real8_min_to_all\ (3),
+shmem_real16_min_to_all\ (3), :ref:`shmem_short_min_to_all`\ (3) - Performs
+a minimum function reduction across a set of processing elements (PEs)
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void shmem_double_min_to_all(double *target, const double *source,
+     int nreduce, int PE_start, int logPE_stride, int PE_size,
+     double *pWrk, long *pSync)
+
+   void shmem_float_min_to_all(float *target, const float *source,
+     int nreduce, int PE_start, int logPE_stride, int PE_size,
+     float *pWrk, long *pSync)
+
+   void shmem_int_min_to_all(int *target, const int *source,
+     int nreduce, int PE_start, int logPE_stride, int PE_size,
+     int *pWrk, long *pSync)
+
+   void shmem_long_min_to_all(long *target, const long *source,
+     int nreduce, int PE_start, int logPE_stride, int PE_size,
+     long *pWrk, long *pSync)
+
+   void shmem_longdouble_min_to_all(long double *target,
+     const long double *source, int nreduce, int PE_start,
+     int logPE_stride, int PE_size, long double *pWrk,
+     long *pSync)
+
+   void shmem_longlong_min_to_all(long long *target,
+     const long long *source, int nreduce, int PE_start, int logPE_stride,
+     int PE_size, long long *pWrk, long *pSync)
+
+   void shmem_short_min_to_all(short *target, const short *source,
+     int nreduce, int PE_start, int logPE_stride, int PE_size,
+     short *pWrk, long *pSync)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER pSync(SHMEM_REDUCE_SYNC_SIZE)
+   INTEGER nreduce, PE_start, logPE_stride, PE_size
+
+   CALL SHMEM_INT4_MIN_TO_ALL(target, source, nreduce, PE_start,
+   & logPE_stride, PE_size, pWrk, pSync)
+
+   CALL SHMEM_INT8_MIN_TO_ALL(target, source, nreduce, PE_start,
+   & logPE_stride, PE_size, pWrk, pSync)
+
+   CALL SHMEM_REAL4_MIN_TO_ALL(target, source, nreduce, PE_start,
+   & logPE_stride, PE_size, pWrk, pSync)
+
+   CALL SHMEM_REAL8_MIN_TO_ALL(target, source, nreduce, PE_start,
+   & logPE_stride, PE_size, pWrk, pSync)
+
+   CALL SHMEM_REAL16_MIN_TO_ALL(target, source, nreduce, PE_start,
+   & logPE_stride, PE_size, pWrk, pSync)
+
+
+DESCRIPTION
+-----------
+
+The shared memory (SHMEM) reduction routines compute one or more
+reductions across symmetric arrays on multiple virtual PEs. A reduction
+performs an associative binary operation across a set of values. For a
+list of other SHMEM reduction routines, see *intro_shmem*\ (3).
+
+As with all SHMEM collective routines, each of these routines assumes
+that only PEs in the active set call the routine. If a PE not in the
+active set calls a SHMEM collective routine, undefined behavior results.
+
+The nreduce argument determines the number of separate reductions to
+perform. The source array on all PEs in the active set provides one
+element for each reduction. The results of the reductions are placed in
+the target array on all PEs in the active set. The active set is defined
+by the PE_start, logPE_stride, PE_size triplet.
+
+The source and target arrays may be the same array, but they may not be
+overlapping arrays.
+
+The arguments are as follows:
+
+target
+   A symmetric array of length nreduce elements to receive the results
+   of the reduction operations. The data type of target varies with the
+   version of the reduction routine being called. When calling from
+   C/C++, refer to the SYNOPSIS section for data type information. When
+   calling from Fortran, the target data types are as follows:
+
+   shmem_int4_min_to_all: Integer, with an element size of 4 bytes
+
+   shmem_int8_min_to_all: Integer, with an element size of 8 bytes
+
+   shmem_real4_min_to_all: Real, with an element size of 4 bytes
+
+   shmem_real8_min_to_all: Real, with an element size of 8 bytes
+
+   shmem_real16_min_to_all: Real, with an element size of 16 bytes
+
+   **source A symmetric array**: of length nreduce elements, that contains one
+      element for each separate reduction operation. The source argument
+      must have the same data type as target.
+
+nreduce
+   The number of elements in the target and source arrays. nreduce must
+   be of type integer. If you are using Fortran, it must be a default
+   integer value.
+
+PE_start
+   The lowest virtual PE number of the active set of PEs. PE_start must
+   be of type integer. If you are using Fortran, it must be a default
+   integer value.
+
+logPE_stride
+   The log (base 2) of the stride between consecutive virtual PE numbers
+   in the active set. logPE_stride must be of type integer. If you are
+   using Fortran, it must be a default integer value.
+
+PE_size
+   The number of PEs in the active set. PE_size must be of type integer.
+   If you are using Fortran, it must be a default integer value.
+
+pWrk
+   A symmetric work array. The pWrk argument must have the same data
+   type as target. In C/C++, this contains max(nreduce/2 + 1,
+   \_SHMEM_REDUCE_MIN_WRKDATA_SIZE) elements. In Fortran, this contains
+   max(nreduce/2 + 1, SHMEM_REDUCE_MIN_WRKDATA_SIZE) elements.
+
+pSync
+   A symmetric work array. In C/C++, pSync is of type long and size
+   \_SHMEM_REDUCE_SYNC_SIZE. In Fortran, pSync is of type integer and
+   size SHMEM_REDUCE_SYNC_SIZE. If you are using Fortran, it must be a
+   default integer value. Every element of this array must be
+   initialized with the value \_SHMEM_SYNC_VALUE (in C/C++) or
+   SHMEM_SYNC_VALUE (in Fortran) before any of the PEs in the active set
+   enter the reduction routine.
+
+The values of arguments nreduce, PE_start, logPE_stride, and PE_size
+must be equal on all PEs in the active set. The same target and source
+arrays, and the same pWrk and pSync work arrays, must be passed to all
+PEs in the active set.
+
+Before any PE calls a reduction routine, you must ensure that the
+following conditions exist (synchronization via a barrier or some other
+method is often needed to ensure this): The pWrk and pSync arrays on all
+PEs in the active set are not still in use from a prior call to a
+collective SHMEM routine. The target array on all PEs in the active set
+is ready to accept the results of the reduction.
+
+Upon return from a reduction routine, the following are true for the
+local PE: The target array is updated. The values in the pSync array are
+restored to the original values.
+
+
+NOTES
+-----
+
+The terms collective, symmetric, and cache aligned are defined in
+*intro_shmem*\ (3). All SHMEM reduction routines reset the values in
+pSync before they return, so a particular pSync buffer need only be
+initialized the first time it is used.
+
+You must ensure that the pSync array is not being updated on any PE in
+the active set while any of the PEs participate in processing of a SHMEM
+reduction routine. Be careful of the following situations: If the pSync
+array is initialized at run time, some type of synchronization is needed
+to ensure that all PEs in the working set have initialized pSync before
+any of them enter a SHMEM routine called with the pSync synchronization
+array. A pSync or pWrk array can be reused in a subsequent reduction
+routine call only if none of the PEs in the active set are still
+processing a prior reduction routine call that used the same pSync or
+pWrk arrays. In general, this can be assured only by doing some type of
+synchronization. However, in the special case of reduction routines
+being called with the same active set, you can allocate two pSync and
+pWrk arrays and alternate between them on successive calls.
+
+
+EXAMPLES
+--------
+
+**Example 1:** This Fortran example statically initializes the pSync
+array and finds the minimum value of real variable FOO across all the
+even PEs.
+
+::
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER PSYNC(SHMEM_REDUCE_SYNC_SIZE)
+   DATA PSYNC /SHMEM_REDUCE_SYNC_SIZE*SHMEM_SYNC_VALUE/
+   PARAMETER (NR=1)
+   REAL FOO, FOOMIN, PWRK(MAX(NR/2+1,SHMEM_REDUCE_MIN_WRKDATA_SIZE))
+   COMMON /COM/ FOO, FOOMIN, PWRK
+   INTRINSIC MY_PE
+
+   IF ( MOD(MY_PE(),2) .EQ. 0) THEN
+     CALL SHMEM_REAL8_MIN_TO_ALL(FOOMIN, FOO, NR, 0, 1, N$PES/2,
+     & PWRK, PSYNC)
+     PRINT *, 'Result on PE ', MY_PE(), ' is ', FOOMIN
+   ENDIF
+
+**Example 2:** Consider the following C/C++ call:
+
+.. code-block:: c++
+
+   shmem_int_min_to_all( target, source, 3, 0, 0, 8, pwrk, psync );
+
+The preceding call is more efficient, but semantically equivalent to,
+the combination of the following calls:
+
+::
+
+   shmem_int_min_to_all(&(target[0]), &(source[0]), 1, 0, 0, 8,
+     pwrk1, psync1);
+   shmem_int_min_to_all(&(target[1]), &(source[1]), 1, 0, 0, 8,
+     pwrk2, psync2);
+   shmem_int_min_to_all(&(target[2]), &(source[2]), 1, 0, 0, 8,
+     pwrk1, psync1);
+
+Note that two sets of pWrk and pSync arrays are used alternately because
+no synchronization is done between calls.
+
+
+.. seealso::
+   *intro_shmem*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_short_or_to_all.3.rst b/docs/man-openshmem/man3/shmem_short_or_to_all.3.rst
new file mode 100644
index 00000000000..912cf399d54
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_short_or_to_all.3.rst
@@ -0,0 +1,218 @@
+.. _shmem_short_or_to_all:
+
+
+shmem_short_or_to_all
+=====================
+
+.. include_body
+
+:ref:`shmem_int_or_to_all`\ (3), shmem_int4_or_to_all\ (3),
+shmem_int8_or_to_all\ (3), :ref:`shmem_long_or_to_all`\ (3),
+:ref:`shmem_longlong_or_to_all`\ (3), :ref:`shmem_short_or_to_all`\ (3) - Performs
+a bitwise OR function reduction across a set of processing elements
+(PEs)
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void shmem_int_or_to_all(int *target, const int *source,
+     int nreduce, int PE_start, int logPE_stride, int PE_size,
+     int *pWrk, long *pSync)
+
+   void shmem_long_or_to_all(long *target, const long *source,
+     int nreduce, int PE_start, int logPE_stride, int PE_size,
+     long *pWrk, long *pSync)
+
+   void shmem_longlong_or_to_all(long long *target,
+     const long long *source, int nreduce, int PE_start, int logPE_stride,
+     int PE_size, long long *pWrk, long *pSync)
+
+   void shmem_short_or_to_all(short *target, const short *source,
+     int nreduce, int PE_start, int logPE_stride, int PE_size,
+     short *pWrk, long *pSync)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER pSync(SHMEM_REDUCE_SYNC_SIZE)
+   INTEGER nreduce, PE_start, logPE_stride, PE_size
+
+   CALL SHMEM_INT4_OR_TO_ALL(target, source, nreduce, PE_start,
+   & logPE_stride, PE_size, pWrk, pSync)
+
+   CALL SHMEM_INT8_OR_TO_ALL(target, source, nreduce, PE_start,
+   & logPE_stride, PE_size, pWrk, pSync)
+
+
+DESCRIPTION
+-----------
+
+The shared memory (SHMEM) reduction routines compute one or more
+reductions across symmetric arrays on multiple virtual PEs. A reduction
+performs an associative binary operation across a set of values. For a
+list of other SHMEM reduction routines, see intro_shmem(3).
+
+As with all SHMEM collective routines, each of these routines assumes
+that only PEs in the active set call the routine. If a PE not in the
+active set calls a SHMEM collective routine, undefined behavior results.
+
+The nreduce argument determines the number of separate reductions to
+perform. The source array on all PEs in the active set provides one
+element for each reduction. The results of the reductions are placed in
+the target array on all PEs in the active set. The active set is defined
+by the PE_start, logPE_stride, PE_size triplet.
+
+The source and target arrays may be the same array, but they may not be
+overlapping arrays.
+
+The arguments are as follows:
+
+target
+   A symmetric array of length nreduce elements to receive the results
+   of the reduction operations. The data type of target varies with the
+   version of the reduction routine being called. When calling from
+   C/C++, refer to the SYNOPSIS section for data type information. When
+   calling from Fortran, the target data types are as follows:
+
+   shmem_int8_or_to_all Integer, with an element size of 8 bytes.
+
+   shmem_int4_or_to_all Integer, with an element size of 4 bytes.
+
+source
+   A symmetric array, of length nreduce elements, that contains one
+   element for each separate reduction operation. The source argument
+   must have the same data type as target.
+
+nreduce
+   The number of elements in the target and source arrays. nreduce must
+   be of type integer. If you are using Fortran, it must be a default
+   integer value.
+
+PE_start
+   The lowest virtual PE number of the active set of PEs. PE_start must
+   be of type integer. If you are using Fortran, it must be a default
+   integer value.
+
+logPE_stride
+   The log (base 2) of the stride between consecutive virtual PE numbers
+   in the active set. logPE_stride must be of type integer. If you are
+   using Fortran, it must be a default integer value.
+
+PE_size
+   The number of PEs in the active set. PE_size must be of type integer.
+   If you are using Fortran, it must be a default integer value.
+
+pWrk
+   A symmetric work array. The pWrk argument must have the same data
+   type as target. In C/C++, this contains max(nreduce/2 + 1,
+   \_SHMEM_REDUCE_MIN_WRKDATA_SIZE) elements. In Fortran, this contains
+   max(nreduce/2 + 1, SHMEM_REDUCE_MIN_WRKDATA_SIZE) elements.
+
+pSync
+   A symmetric work array. In C/C++, pSync is of type long and size
+   \_SHMEM_REDUCE_SYNC_SIZE. In Fortran, pSync is of type integer and
+   size SHMEM_REDUCE_SYNC_SIZE. If you are using Fortran, it must be a
+   default integer value. Every element of this array must be
+   initialized with the value \_SHMEM_SYNC_VALUE (in C/C++) or
+   SHMEM_SYNC_VALUE (in Fortran) before any of the PEs in the active set
+   enter the reduction routine.
+
+The values of arguments nreduce, PE_start, logPE_stride, and PE_size
+must be equal on all PEs in the active set. The same target and source
+arrays, and the same pWrk and pSync work arrays, must be passed to all
+PEs in the active set.
+
+Before any PE calls a reduction routine, you must ensure that the
+following conditions exist (synchronization via a barrier or some other
+method is often needed to ensure this): The pWrk and pSync arrays on all
+PEs in the active set are not still in use from a prior call to a
+collective SHMEM routine. The target array on all PEs in the active set
+is ready to accept the results of the reduction.
+
+Upon return from a reduction routine, the following are true: The target
+array is updated. The values in the pSync array are restored to the
+original values.
+
+
+NOTES
+-----
+
+The terms collective, symmetric, and cache aligned are defined in
+*intro_shmem*\ (3). All SHMEM reduction routines reset the values in
+pSync before they return, so a particular pSync buffer need only be
+initialized the first time it is used.
+
+You must ensure that the pSync array is not being updated on any PE in
+the active set while any of the PEs participate in processing of a SHMEM
+reduction routine. Be careful to avoid these situations: If the pSync
+array is initialized at run time, some type of synchronization is needed
+to ensure that all PEs in the working set have initialized pSync before
+any of them enter a SHMEM routine called with the pSync synchronization
+array. A pSync or pWrk array can be reused in a subsequent reduction
+routine call only if none of the PEs in the active set are still
+processing a prior reduction routine call that used the same pSync or
+pWrk arrays. In general, this can be assured only by doing some type of
+synchronization. However, in the special case of reduction routines
+being called with the same active set, you can allocate two pSync and
+pWrk arrays and alternate between them on successive calls.
+
+
+EXAMPLES
+--------
+
+**Example 1:** This Fortran example statically initializes the pSync
+array and finds the logical OR of the integer variable FOO across all
+even PEs.
+
+::
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER PSYNC(SHMEM_REDUCE_SYNC_SIZE)
+   DATA PSYNC /SHMEM_REDUCE_SYNC_SIZE*SHMEM_SYNC_VALUE/
+   PARAMETER (NR=1)
+   REAL PWRK(MAX(NR/2+1,SHMEM_REDUCE_MIN_WRKDATA_SIZE))
+   INTEGER FOO, FOOOR
+   COMMON /COM/ FOO, FOOOR, PWRK
+   INTRINSIC MY_PE
+
+   IF ( MOD(MY_PE(),2) .EQ. 0) THEN
+     CALL SHMEM_INT8_OR_TO_ALL(FOOOR, FOO, NR, 0, 1, N$PES/2,
+     & PWRK, PSYNC)
+     PRINT *,'Result on PE ',MY_PE(),' is ',FOOOR
+   ENDIF
+
+**Example 2:** Consider the following C/C++ call:
+
+.. code-block:: c++
+
+   shmem_int_or_to_all( target, source, 3, 0, 0, 8, pwrk, psync );
+
+The preceding call is more efficient, but semantically equivalent to,
+the combination of the following calls:
+
+::
+
+   shmem_int_or_to_all(&(target[0]), &(source[0]), 1, 0, 0, 8,
+     pwrk1, psync1);
+   shmem_int_or_to_all(&(target[1]), &(source[1]), 1, 0, 0, 8,
+     pwrk2, psync2);
+   shmem_int_or_to_all(&(target[2]), &(source[2]), 1, 0, 0, 8,
+     pwrk1, psync1);
+
+Note that two sets of pWrk and pSync arrays are used alternately because
+no synchronization is done between calls.
+
+
+.. seealso::
+   *intro_shmem*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_short_p.3.rst b/docs/man-openshmem/man3/shmem_short_p.3.rst
new file mode 100644
index 00000000000..bc37e9a526f
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_short_p.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_short_p:
+
+shmem_short_p
+=============
+    .. include_body
+
+.. include:: ../man3/shmem_char_p.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_short_prod_to_all.3.rst b/docs/man-openshmem/man3/shmem_short_prod_to_all.3.rst
new file mode 100644
index 00000000000..cf79160d0ff
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_short_prod_to_all.3.rst
@@ -0,0 +1,271 @@
+.. _shmem_short_prod_to_all:
+
+
+shmem_short_prod_to_all
+=======================
+
+.. include_body
+
+shmem_comp4_prod_to_all\ (3), shmem_comp8_prod_to_all\ (3),
+:ref:`shmem_complexd_prod_to_all`\ (3), :ref:`shmem_complexf_prod_to_all`\ (3),
+:ref:`shmem_double_prod_to_all`\ (3), :ref:`shmem_float_prod_to_all`\ (3),
+:ref:`shmem_int_prod_to_all`\ (3), shmem_int4_prod_to_all\ (3),
+shmem_int8_prod_to_all\ (3), :ref:`shmem_long_prod_to_all`\ (3),
+:ref:`shmem_longdouble_prod_to_all`\ (3), :ref:`shmem_longlong_prod_to_all`\ (3),
+shmem_real8_prod_to_all\ (3), shmem_real16_prod_to_all\ (3),
+shmem_real4_prod_to_all\ (3), :ref:`shmem_short_prod_to_all`\ (3) -
+Performs a product reduction across a set of processing elements (PEs)
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void shmem_complexd_prod_to_all(double complex *target,
+     const double complex *source, int nreduce, int PE_start,
+     int logPE_stride, int PE_size, double complex *pWrk,
+     long *pSync)
+
+   void shmem_complexf_prod_to_all(float complex *target,
+     const float complex *source, int nreduce, int PE_start,
+     int logPE_stride, int PE_size, float complex *pWrk,
+     long *pSync)
+
+   void shmem_double_prod_to_all(double *target, const double *source,
+     int nreduce, int PE_start, int logPE_stride, int PE_size,
+     double *pWrk, long *pSync)
+
+   void shmem_float_prod_to_all(float *target, const float *source,
+     int nreduce, int PE_start, int logPE_stride, int PE_size,
+     float *pWrk, long *pSync)
+
+   void shmem_int_prod_to_all(int *target, const int *source,
+     int nreduce, int PE_start, int logPE_stride, int PE_size,
+     int *pWrk, long *pSync)
+
+   void shmem_long_prod_to_all(long *target, const long *source,
+     int nreduce, int PE_start, int logPE_stride, int PE_size,
+     long *pWrk, long *pSync)
+
+   void shmem_longdouble_prod_to_all(long double *target,
+     const long double *source, int nreduce, int PE_start,
+     int logPE_stride, int PE_size, long double *pWrk,
+     long *pSync)
+
+   void shmem_longlong_prod_to_all(long long *target,
+     const long long *source, int nreduce, int PE_start,
+     int logPE_stride, int PE_size, long long *pWrk,
+     long *pSync)
+
+   void shmem_short_prod_to_all(short *target, const short *source,
+     int nreduce, int PE_start, int logPE_stride, int PE_size,
+     short *pWrk, long *pSync)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER pSync(SHMEM_REDUCE_SYNC_SIZE)
+   INTEGER nreduce, PE_start, logPE_stride, PE_size
+
+   CALL SHMEM_COMP4_PROD_TO_ALL(target, source, nreduce, PE_start,
+   & logPE_stride, PE_size, pWrk, pSync)
+
+   CALL SHMEM_COMP8_PROD_TO_ALL(target, source, nreduce, PE_start,
+   & logPE_stride, PE_size, pWrk, pSync)
+
+   CALL SHMEM_INT4_PROD_TO_ALL(target, source, nreduce, PE_start,
+   & logPE_stride, PE_size, pWrk, pSync)
+
+   CALL SHMEM_INT8_PROD_TO_ALL(target, source, nreduce, PE_start,
+   & logPE_stride, PE_size, pWrk, pSync)
+
+   CALL SHMEM_REAL4_PROD_TO_ALL(target, source, nreduce, PE_start,
+   & logPE_stride, PE_size, pWrk, pSync)
+
+   CALL SHMEM_REAL8_PROD_TO_ALL(target, source, nreduce, PE_start,
+   & logPE_stride, PE_size, pWrk, pSync)
+
+   CALL SHMEM_REAL16_PROD_TO_ALL(target, source, nreduce, PE_start,
+   & logPE_stride, PE_size, pWrk, pSync)
+
+
+DESCRIPTION
+-----------
+
+The shared memory (SHMEM) reduction routines compute one or more
+reductions across symmetric arrays on multiple virtual PEs. A reduction
+performs an associative binary operation across a set of values. For a
+list of other SHMEM reduction routines, see *intro_shmem*\ (3).
+
+As with all SHMEM collective routines, each of these routines assumes
+that only PEs in the active set call the routine. If a PE not in the
+active set calls a SHMEM collective routine, undefined behavior results.
+
+The nreduce argument determines the number of separate reductions to
+perform. The source array on all PEs in the active set provides one
+element for each reduction. The results of the reductions are placed in
+the target array on all PEs in the active set. The active set is defined
+by the PE_start, logPE_stride, PE_size triplet.
+
+The source and target arrays may be the same array, but they may not be
+overlapping arrays.
+
+The arguments are as follows:
+
+target
+   A symmetric array of length nreduce elements to receive the results
+   of the reduction operations. The data type of target varies with the
+   version of the reduction routine being called and the language used.
+   When calling from C/C++, refer to the SYNOPSIS section for data type
+   information. When calling from Fortran, the target data types are as
+   follows:
+
+   shmem_comp4_prod_to_all: Complex, with an element size equal to two
+      4-byte real values.
+
+   shmem_comp8_prod_to_all: Complex, with an element size equal to two
+      8-byte real values.
+
+   shmem_int4_prod_to_all: Integer, with an element size of 4 bytes
+
+   shmem_int8_prod_to_all: Integer, with an element size of 8 bytes
+
+   shmem_real4_prod_to_all: Real, with an element size of 4 bytes
+
+   shmem_real8_prod_to_all: Real, with an element size of 8 bytes
+
+   shmem_real16_prod_to_all: Real, with an element size of 16 bytes
+
+source
+   A symmetric array, of length nreduce elements, that contains one
+   element for each separate reduction operation. The source argument
+   must have the same data type as target.
+
+nreduce
+   The number of elements in the target and source arrays. nreduce must
+   be of type integer. If you are using Fortran, it must be a default
+   integer value.
+
+PE_start
+   The lowest virtual PE number of the active set of PEs. PE_start must
+   be of type integer. If you are using Fortran, it must be a default
+   integer value.
+
+logPE_stride
+   The log (base 2) of the stride between consecutive virtual PE numbers
+   in the active set. logPE_stride must be of type integer. If you are
+   using Fortran, it must be a default integer value.
+
+PE_size
+   The number of PEs in the active set. PE_size must be of type integer.
+   If you are using Fortran, it must be a default integer value.
+
+pWrk
+   A symmetric work array. The pWrk argument must have the same data
+   type as target. In C/C++, this contains max(nreduce/2 + 1,
+   \_SHMEM_REDUCE_MIN_WRKDATA_SIZE) elements. In Fortran, this contains
+   max(nreduce/2 + 1, SHMEM_REDUCE_MIN_WRKDATA_SIZE) elements.
+
+pSync
+   A symmetric work array. In C/C++, pSync is of type long and size
+   \_SHMEM_REDUCE_SYNC_SIZE. In Fortran, pSync is of type integer and
+   size SHMEM_REDUCE_SYNC_SIZE. If you are using Fortran, it must be a
+   default integer value. Before any of the PEs in the active set enter
+   the reduction routine, every element of this array must be
+   initialized with the value \_SHMEM_SYNC_VALUE (in C/C++) or
+   SHMEM_SYNC_VALUE (in Fortran).
+
+The values of arguments nreduce, PE_start, logPE_stride, and PE_size
+must be equal on all PEs in the active set. The same target and source
+arrays, and the same pWrk and pSync work arrays, must be passed to all
+PEs in the active set. Before any PE calls a reduction routine, you must
+ensure that the following conditions exist (synchronization via a
+barrier or some other method is often needed to ensure this): The pWrk
+and pSync arrays on all PEs in the active set are not still in use from
+a prior call to a collective SHMEM routine. The target array on all PEs
+in the active set is ready to accept the results of the reduction.
+
+Upon return from a reduction routine, the following are true for the
+local PE: The target array is updated. The values in the pSync array are
+restored to the original values.
+
+
+NOTES
+-----
+
+The terms collective, symmetric, and cache aligned are defined in
+*intro_shmem*\ (3). All SHMEM reduction routines reset the values in
+pSync before they return, so a particular pSync buffer need only be
+initialized the first time it is used.
+
+You must ensure that the pSync array is not being updated on any PE in
+the active set while any of the PEs participate in processing of a SHMEM
+reduction routine. Be careful of the following situations: If the pSync
+array is initialized at run time, some type of synchronization is needed
+to ensure that all PEs in the working set have initialized pSync before
+any of them enter a SHMEM routine called with the pSync synchronization
+array. A pSync or pWrk array can be reused in a subsequent reduction
+routine call only if none of the PEs in the active set are still
+processing a prior reduction routine call that used the same pSync or
+pWrk arrays. In general, this can be assured only by doing some type of
+synchronization. However, in the special case of reduction routines
+being called with the same active set, you can allocate two pSync and
+pWrk arrays and alternate between them on successive calls.
+
+
+EXAMPLES
+--------
+
+**Example 1:** This Fortran example statically initializes the pSync
+array and finds the product of the real variable FOO across all the even
+PEs.
+
+::
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER PSYNC(SHMEM_REDUCE_SYNC_SIZE)
+   DATA PSYNC /SHMEM_REDUCE_SYNC_SIZE*SHMEM_SYNC_VALUE/
+   PARAMETER (NR=1)
+   REAL FOO, FOOPROD, PWRK(MAX(NR/2+1,SHMEM_REDUCE_MIN_WRKDATA_SIZE))
+   COMMON /COM/ FOO, FOOPROD, PWRK
+   INTRINSIC MY_PE
+
+   IF ( MOD(MY_PE(),2) .EQ. 0) THEN
+     CALL SHMEM_COMP8_PROD_TO_ALL(FOOPROD, FOO, NR, 0, 1, N$PES/2,
+     & PWRK, PSYNC)
+     PRINT *, 'Result on PE ', MY_PE(), ' is ', FOOPROD
+   ENDIF
+
+**Example 2:** Consider the following C/C++ call:
+
+.. code-block:: c++
+
+   shmem_short_prod_to_all(target, source, 3, 0, 0, 8, pwrk, psync);
+
+The preceding call is more efficient, but semantically equivalent to,
+the combination of the following calls:
+
+::
+
+   shmem_short_prod_to_all(&(target[0]), &(source[0]), 1, 0, 0, 8,
+     pwrk1, psync1);
+   shmem_short_prod_to_all(&(target[1]), &(source[1]), 1, 0, 0, 8,
+     pwrk2, psync2);
+   shmem_short_prod_to_all(&(target[2]), &(source[2]), 1, 0, 0, 8,
+     pwrk1, psync1);
+
+Note that two sets of pWrk and pSync arrays are used alternately because
+no synchronization is done between calls.
+
+
+.. seealso::
+   *intro_shmem*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_short_put.3.rst b/docs/man-openshmem/man3/shmem_short_put.3.rst
new file mode 100644
index 00000000000..5214eac60da
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_short_put.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_short_put:
+
+shmem_short_put
+===============
+    .. include_body
+
+.. include:: ../man3/shmem_char_put.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_short_put_nbi.3.rst b/docs/man-openshmem/man3/shmem_short_put_nbi.3.rst
new file mode 100644
index 00000000000..660f0227f79
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_short_put_nbi.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_short_put_nbi:
+
+shmem_short_put_nbi
+===================
+    .. include_body
+
+.. include:: ../man3/shmem_putmem_nbi.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_short_sum_to_all.3.rst b/docs/man-openshmem/man3/shmem_short_sum_to_all.3.rst
new file mode 100644
index 00000000000..03f43ac59ac
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_short_sum_to_all.3.rst
@@ -0,0 +1,292 @@
+.. _shmem_short_sum_to_all:
+
+
+shmem_short_sum_to_all
+======================
+
+.. include_body
+
+shmem_comp4_sum_to_all\ (3), shmem_comp8_sum_to_all\ (3),
+:ref:`shmem_complexd_sum_to_all`\ (3), :ref:`shmem_complexf_sum_to_all`\ (3),
+:ref:`shmem_double_sum_to_all`\ (3), :ref:`shmem_float_sum_to_all`\ (3),
+:ref:`shmem_int_sum_to_all`\ (3), shmem_int4_sum_to_all\ (3),
+shmem_int8_sum_to_all\ (3), :ref:`shmem_long_sum_to_all`\ (3),
+:ref:`shmem_longdouble_sum_to_all`\ (3), :ref:`shmem_longlong_sum_to_all`\ (3),
+shmem_real4_sum_to_all\ (3), shmem_real8_sum_to_all\ (3),
+shmem_real16_sum_to_all\ (3), :ref:`shmem_short_sum_to_all`\ (3) - Performs
+a sum reduction across a set of processing elements (PEs)
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void shmem_complexd_sum_to_all(double complex *target,
+     const double complex *source, int nreduce, int PE_start,
+     int logPE_stride, int PE_size, double complex *pWrk,
+     long *pSync)
+
+   void shmem_complexf_sum_to_all(float complex *target,
+     const float complex *source, int nreduce, int PE_start,
+     int logPE_stride, int PE_size, float complex *pWrk,
+     long *pSync)
+
+   void shmem_double_sum_to_all(double *target,
+     const double *source, int nreduce, int PE_start, int logPE_stride,
+     int PE_size, double *pWrk, long *pSync)
+
+   void shmem_float_sum_to_all(float *target, const float *source,
+     int nreduce, int PE_start, int logPE_stride, int PE_size,
+     float *pWrk, long *pSync)
+
+   void shmem_int_sum_to_all(int *target, const int *source,
+     int nreduce, int PE_start, int logPE_stride, int PE_size,
+     int *pWrk, long *pSync)
+
+   void shmem_long_sum_to_all(long *target, const long *source,
+     int nreduce, int PE_start, int logPE_stride, int PE_size,
+     long *pWrk, long *pSync)
+
+   void shmem_longdouble_sum_to_all(long double *target,
+     const long double *source, int nreduce, int PE_start, int
+     logPE_stride, int PE_size, long double *pWrk, long *pSync)
+
+   void shmem_longlong_sum_to_all(long long *target,
+     const long long *source, int nreduce, int PE_start,
+     int logPE_stride, int PE_size, long long *pWrk,
+     long *pSync)
+
+   void shmem_short_sum_to_all(short *target, const short *source,
+     int nreduce, int PE_start, int logPE_stride, int PE_size,
+     short *pWrk, long *pSync)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER pSync(SHMEM_REDUCE_SYNC_SIZE)
+   INTEGER nreduce, PE_start, logPE_stride, PE_size
+
+   CALL SHMEM_COMP4_SUM_TO_ALL(target, source, nreduce,
+   & PE_start, logPE_stride, PE_size, pWrk, pSync)
+
+   CALL SHMEM_COMP8_SUM_TO_ALL(target, source, nreduce,
+   & PE_start, logPE_stride, PE_size, pWrk, pSync)
+
+   CALL SHMEM_INT4_SUM_TO_ALL(target, source, nreduce,
+   & PE_start, logPE_stride, PE_size, pWrk, pSync)
+
+   CALL SHMEM_INT8_SUM_TO_ALL(target, source, nreduce,
+   & PE_start, logPE_stride, PE_size, pWrk, pSync)
+
+   CALL SHMEM_REAL4_SUM_TO_ALL(target, source, nreduce,
+   & PE_start, logPE_stride, PE_size, pWrk, pSync)
+
+   CALL SHMEM_REAL8_SUM_TO_ALL(target, source, nreduce,
+   & PE_start, logPE_stride, PE_size, pWrk, pSync)
+
+   CALL SHMEM_REAL16_SUM_TO_ALL(target, source, nreduce,
+   & PE_start, logPE_stride, PE_size, pWrk, pSync)
+
+
+DESCRIPTION
+-----------
+
+The shared memory (SHMEM) reduction routines compute one or more
+reductions across symmetric arrays on multiple virtual PEs. A reduction
+performs an associative binary operation across a set of values. For a
+list of other SHMEM reduction routines, see *intro_shmem*\ (3).
+
+As with all SHMEM collective routines, each of these routines assumes
+that only PEs in the active set call the routine. If a PE not in the
+active set calls a SHMEM collective routine, undefined behavior results.
+
+The nreduce argument determines the number of separate reductions to
+perform. The source array on all PEs in the active set provides one
+element for each reduction. The results of the reductions are placed in
+the target array on all PEs in the active set. The active set is defined
+by the PE_start, logPE_stride, PE_size triplet.
+
+The source and target arrays may be the same array, but they may not be
+overlapping arrays.
+
+The arguments are as follows:
+
+target
+   The remotely accessible integer data object to be updated on the
+   remote PE. If you are using C/C++, the type of target should match
+   that implied in the SYNOPSIS section. If you are using the Fortran
+   compiler, it must be of type integer with an element size of 4 bytes
+   for SHMEM_INT4_ADD and 8 bytes for SHMEM_INT8_ADD.
+
+value
+   The value to be atomically added to target. If you are using C/C++,
+   the type of value should match that implied in the SYNOPSIS section.
+   If you are using Fortran, it must be of type integer with an element
+   size of target.
+
+pe
+   An integer that indicates the PE number upon which target is to be
+   updated. If you are using Fortran, it must be a default integer
+   value.
+
+target
+   | A symmetric array of length nreduce elements to receive the results
+     of the reduction operations.
+   | The data type of target varies with the version of the reduction
+     routine being called and the language used. When calling from
+     C/C++, refer to the SYNOPSIS section for data type information.
+     When calling from Fortran, the target data types are as follows:
+
+   shmem_comp4_sum_to_all:** COMPLEX(KIND=4).
+
+   shmem_comp8_sum_to_all:** Complex. If you are using Fortran, it must be
+      a default complex value.
+
+   shmem_int4_sum_to_all:** INTEGER(KIND=4).
+
+   shmem_int8_sum_to_all:** Integer. If you are using Fortran, it must be a
+      default integer value.
+
+   shmem_real4_sum_to_all:** REAL(KIND=4).
+
+   shmem_real8_sum_to_all:** Real. If you are using Fortran, it must be a
+      default real value.
+
+   shmem_real16_sum_to_all:** Real. If you are using Fortran, it must be a
+      default real value.
+
+source
+   A symmetric array, of length nreduce elements, that contains one
+   element for each separate reduction operation. The source argument
+   must have the same data type as target.
+
+nreduce
+   The number of elements in the target and source arrays. nreduce must
+   be of type integer. If you are using Fortran, it must be a default
+   integer value.
+
+PE_start
+   The lowest virtual PE number of the active set of PEs. PE_start must
+   be of type integer. If you are using Fortran, it must be a default
+   integer value.
+
+logPE_stride
+   The log (base 2) of the stride between consecutive virtual PE numbers
+   in the active set. logPE_stride must be of type integer. If you are
+   using Fortran, it must be a default integer value.
+
+PE_size
+   The number of PEs in the active set. PE_size must be of type integer.
+   If you are using Fortran, it must be a default integer value.
+
+pWrk
+   A symmetric work array. The pWrk argument must have the same data
+   type as target. In C/C++, this contains max(nreduce/2 + 1,
+   \_SHMEM_REDUCE_MIN_WRKDATA_SIZE) elements. In Fortran, this contains
+   max(nreduce/2 + 1, SHMEM_REDUCE_MIN_WRKDATA_SIZE) elements.
+
+pSync
+   A symmetric work array. In C/C++, pSync is of type long and size
+   \_SHMEM_REDUCE_SYNC_SIZE. In Fortran, pSync is of type integer and
+   size SHMEM_REDUCE_SYNC_SIZE. It must be a default integer value.
+   Every element of this array must be initialized with the value
+   \_SHMEM_SYNC_VALUE (in C/C++) or SHMEM_SYNC_VALUE (in Fortran) before
+   any of the PEs in the active set enter the reduction routine.
+
+The values of arguments nreduce, PE_start, logPE_stride, and PE_size
+must be equal on all PEs in the active set. The same target and source
+arrays, and the same pWrk and pSync work arrays, must be passed to all
+PEs in the active set.
+
+Before any PE calls a reduction routine, you must ensure that the
+following conditions exist (synchronization via a barrier or some other
+method is often needed to ensure this): The pWrk and pSync arrays on all
+PEs in the active set are not still in use from a prior call to a
+collective SHMEM routine. The target array on all PEs in the active set
+is ready to accept the results of the reduction.
+
+Upon return from a reduction routine, the following are true for the
+local PE: The target array is updated. The values in the pSync array are
+restored to the original values.
+
+
+NOTES
+-----
+
+The terms collective, symmetric, and cache aligned are defined in
+*intro_shmem*\ (3).
+
+All SHMEM reduction routines reset the values in pSync before they
+return, so a particular pSync buffer need only be initialized the first
+time it is used.
+
+You must ensure that the pSync array is not being updated on any PE in
+the active set while any of the PEs participate in processing of a SHMEM
+reduction routine. Be careful of the following situations: If the pSync
+array is initialized at run time, some type of synchronization is needed
+to ensure that all PEs in the working set have initialized pSync before
+any of them enter a SHMEM routine called with the pSync synchronization
+array. A pSync or pWrk array can be reused in a subsequent reduction
+routine call only if none of the PEs in the active set are still
+processing a prior reduction routine call that used the same pSync or
+pWrk arrays. In general, this can be assured only by doing some type of
+synchronization. However, in the special case of reduction routines
+being called with the same active set, you can allocate two pSync and
+pWrk arrays and alternate between them on successive calls.
+
+
+EXAMPLES
+--------
+
+**Example 1:** This Fortran example statically initializes the pSync
+array and finds the sum of the real variable FOO across all even PEs.
+
+::
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER PSYNC(SHMEM_REDUCE_SYNC_SIZE)
+   DATA PSYNC /SHMEM_REDUCE_SYNC_SIZE*SHMEM_SYNC_VALUE/
+   PARAMETER (NR=1)
+   REAL FOO, FOOSUM, PWRK(MAX(NR/2+1,SHMEM_REDUCE_MIN_WRKDATA_SIZE))
+   COMMON /COM/ FOO, FOOSUM, PWRK
+   INTRINSIC MY_PE
+
+   IF ( MOD(MY_PE(),2) .EQ. 0) THEN
+     CALL SHMEM_INT4_SUM_TO_ALL(FOOSUM, FOO, NR, 0, 1, N$PES/2,
+     & PWRK, PSYNC)
+     PRINT *, 'Result on PE ', MY_PE(), ' is ', FOOSUM
+   ENDIF
+
+**Example 2:** Consider the following C/C++ call:
+
+.. code-block:: c++
+
+   shmem_int_sum_to_all( target, source, 3, 0, 0, 8, pwrk, psync );
+
+The preceding call is more efficient, but semantically equivalent to,
+the combination of the following calls:
+
+::
+
+   shmem_int_sum_to_all(&(target[0]), &(source[0]), 1, 0, 0, 8,
+     pwrk1, psync1);
+   shmem_int_sum_to_all(&(target[1]), &(source[1]), 1, 0, 0, 8,
+     pwrk2, psync2);
+   shmem_int_sum_to_all(&(target[2]), &(source[2]), 1, 0, 0, 8,
+     pwrk1, psync1);
+
+   Note that two sets of pWrk and pSync arrays are used alternately because no
+   synchronization is done between calls.
+
+
+.. seealso::
+   *intro_shmem*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_short_wait.3.rst b/docs/man-openshmem/man3/shmem_short_wait.3.rst
new file mode 100644
index 00000000000..10c8d650466
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_short_wait.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_short_wait:
+
+shmem_short_wait
+================
+    .. include_body
+
+.. include:: ../man3/shmem_wait.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_short_wait_until.3.rst b/docs/man-openshmem/man3/shmem_short_wait_until.3.rst
new file mode 100644
index 00000000000..c5c7a9f3628
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_short_wait_until.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_short_wait_until:
+
+shmem_short_wait_until
+======================
+    .. include_body
+
+.. include:: ../man3/shmem_wait.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_short_xor_to_all.3.rst b/docs/man-openshmem/man3/shmem_short_xor_to_all.3.rst
new file mode 100644
index 00000000000..8b4c47b6ccd
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_short_xor_to_all.3.rst
@@ -0,0 +1,230 @@
+.. _shmem_short_xor_to_all:
+
+
+shmem_short_xor_to_all
+======================
+
+.. include_body
+
+shmem_comp4_xor_to_all\ (3), :ref:`shmem_int_xor_to_all`\ (3),
+shmem_int4_xor_to_all\ (3), shmem_int8_xor_to_all\ (3),
+:ref:`shmem_long_xor_to_all`\ (3), :ref:`shmem_longlong_xor_to_all`\ (3),
+:ref:`shmem_short_xor_to_all`\ (3) - Performs a bitwise XOR operation on
+symmetric arrays over the active set of PEs.
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void shmem_int_xor_to_all(int *target, const int *source,
+     int nreduce, int PE_start, int logPE_stride, int PE_size,
+     int *pWrk, long *pSync)
+
+   void shmem_long_xor_to_all(long *target, const long *source,
+     int nreduce, int PE_start, int logPE_stride, int PE_size,
+     long *pWrk, long *pSync)
+
+   void shmem_longlong_xor_to_all(long long *target,
+     const long long *source, int nreduce, int PE_start, int logPE_stride,
+     int PE_size, long long *pWrk, long *pSync)
+
+   void shmem_short_xor_to_all(short *target, const short *source,
+     int nreduce, int PE_start, int logPE_stride, int PE_size,
+     short *pWrk, long *pSync)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER pSync(SHMEM_REDUCE_SYNC_SIZE)
+   INTEGER nreduce, PE_start, logPE_stride, PE_size
+
+   CALL SHMEM_COMP4_XOR_TO_ALL(target, source, nreduce,
+   & PE_start, logPE_stride, PE_size, pWrk, pSync)
+
+   CALL SHMEM_INT4_XOR_TO_ALL(target, source, nreduce,
+   & PE_start, logPE_stride, PE_size, pWrk, pSync)
+
+   CALL SHMEM_INT8_XOR_TO_ALL(target, source, nreduce,
+   & PE_start, logPE_stride, PE_size, pWrk, pSync)
+
+
+DESCRIPTION
+-----------
+
+The shared memory (SHMEM) reduction routines compute one or more
+reductions across symmetric arrays on multiple virtual PEs. A reduction
+performs an associative binary operation across a set of values. For a
+list of other SHMEM reduction routines, see *intro_shmem*\ (3).
+
+As with all SHMEM collective routines, each of these routines assumes
+that only PEs in the active set call the routine. If a PE not in the
+active set calls a SHMEM collective routine, undefined behavior results.
+
+The nreduce argument determines the number of separate reductions to
+perform. The source array on all PEs in the active set provides one
+element for each reduction. The results of the reductions are placed in
+the target array on all PEs in the active set. The active set is defined
+by the PE_start, logPE_stride, PE_size triplet.
+
+The source and target arrays may be the same array, but they may not be
+overlapping arrays.
+
+The arguments are as follows:
+
+target
+   A symmetric array of length nreduce elements to receive the results
+   of the reduction operations. The data type of target varies with the
+   version of the reduction routine being called and the language used.
+   When calling from C/C++, refer to the SYNOPSIS section for data type
+   information. When calling from Fortran, the target data types are as
+   follows:
+
+   shmem_comp8_xor_to_all:** Complex, with an element size equal to two 8-
+      byte real values
+
+   shmem_comp4_xor_to_all:** Complex, with an element size equal to two 4-
+      byte real values
+
+   shmem_int8_xor_to_all:** Integer, with an element size of 8 bytes
+
+   shmem_int4_xor_to_all:** Integer, with an element size of 4 bytes
+
+   shmem_real8_xor_to_all:** Real, with an element size of 8 bytes
+
+   shmem_real4_xor_to_all:** Real, with an element size of 4 bytes
+
+source
+   A symmetric array, of length nreduce elements, that contains one
+   element for each separate reduction operation. The source argument
+   must have the same data type as target.
+
+nreduce
+   The number of elements in the target and source arrays. nreduce must
+   be of type integer. If you are using Fortran, it must be a default
+   integer value.
+
+PE_start
+   The lowest virtual PE number of the active set of PEs. PE_start must
+   be of type integer. If you are using Fortran, it must be a default
+   integer value.
+
+logPE_stride
+   The log (base 2) of the stride between consecutive virtual PE numbers
+   in the active set. logPE_stride must be of type integer. If you are
+   using Fortran, it must be a default integer value.
+
+PE_size
+   The number of PEs in the active set. PE_size must be of type integer.
+   If you are using Fortran, it must be a default integer value.
+
+pWrk
+   A symmetric work array. The pWrk argument must have the same data
+   type as target. In C/C++, this contains max(nreduce/2 + 1,
+   \_SHMEM_REDUCE_MIN_WRKDATA_SIZE) elements. In Fortran, this contains
+   max(nreduce/2 + 1, SHMEM_REDUCE_MIN_WRKDATA_SIZE) elements.
+
+pSync
+   A symmetric work array. In C/C++, pSync is of type long and size
+   \_SHMEM_REDUCE_SYNC_SIZE. In Fortran, pSync is of type integer and
+   size SHMEM_REDUCE_SYNC_SIZE. If you are using Fortran, it must be a
+   default integer value. Every element of this array must be
+   initialized with the value \_SHMEM_SYNC_VALUE (in C/C++) or
+   SHMEM_SYNC_VALUE (in Fortran) before any of the PEs in the active set
+   enter the reduction routine.
+
+The values of arguments nreduce, PE_start, logPE_stride, and PE_size
+must be equal on all PEs in the active set. The same target and source
+arrays, and the same pWrk and pSync work arrays, must be passed to all
+PEs in the active set.
+
+Before any PE calls a reduction routine, you must ensure that the
+following conditions exist (synchronization via a barrier or some other
+method is often needed to ensure this): The pWrk and pSync arrays on all
+PEs in the active set are not still in use from a prior call to a
+collective SHMEM routine. The target array on all PEs in the active set
+is ready to accept the results of the reduction.
+
+Upon return from a reduction routine, the following are true for the
+local PE: The target array is updated. The values in the pSync array are
+restored to the original values.
+
+
+NOTES
+-----
+
+The terms collective, symmetric, and cache aligned are defined in
+*intro_shmem*\ (3). All SHMEM reduction routines reset the values in
+pSync before they return, so a particular pSync buffer need only be
+initialized the first time it is used.
+
+You must ensure that the pSync array is not being updated on any PE in
+the active set while any of the PEs participate in processing of a SHMEM
+reduction routine. Be careful of the following situations: If the pSync
+array is initialized at run time, some type of synchronization is needed
+to ensure that all PEs in the working set have initialized pSync before
+any of them enter a SHMEM routine called with the pSync synchronization
+array. A pSync or pWrk array can be reused in a subsequent reduction
+routine call only if none of the PEs in the active set are still
+processing a prior reduction routine call that used the same pSync or
+pWrk arrays. In general, this can be assured only by doing some type of
+synchronization. However, in the special case of reduction routines
+being called with the same active set, you can allocate two pSync and
+pWrk arrays and alternate between them on successive calls.
+
+
+EXAMPLES
+--------
+
+**Example 1:** This Fortran example statically initializes the pSync
+array and computes the exclusive OR of variable FOO across all even PEs.
+
+::
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER PSYNC(SHMEM_REDUCE_SYNC_SIZE)
+   DATA PSYNC /SHMEM_REDUCE_SYNC_SIZE*SHMEM_SYNC_VALUE/
+   PARAMETER (NR=1)
+   REAL FOO, FOOXOR, PWRK(MAX(NR/2+1,SHMEM_REDUCE_MIN_WRKDATA_SIZE))
+   COMMON /COM/ FOO, FOOXOR, PWRK
+   INTRINSIC MY_PE
+
+   IF ( MOD(MY_PE(),2) .EQ. 0) THEN
+     CALL SHMEM_REAL8_XOR_TO_ALL(FOOXOR, FOO, NR, 0, 1, N$PES/2,
+     & PWRK, PSYNC)
+     PRINT *, 'Result on PE ', MY_PE(), ' is ', FOOXOR
+   ENDIF
+
+**Example 2:** Consider the following C/C++ call:
+
+.. code-block:: c++
+
+   shmem_short_xor_to_all( target, source, 3, 0, 0, 8, pwrk, psync );
+
+The preceding call is more efficient, but semantically equivalent to,
+the combination of the following calls:
+
+::
+
+   shmem_short_xor_to_all(&(target[0]), &(source[0]), 1, 0, 0, 8,
+     pwrk1, psync1);
+   shmem_short_xor_to_all(&(target[1]), &(source[1]), 1, 0, 0, 8,
+     pwrk2, psync2);
+   shmem_short_xor_to_all(&(target[2]), &(source[2]), 1, 0, 0, 8,
+     pwrk1, psync1);
+
+Note that two sets of pWrk and pSync arrays are used alternately because
+no synchronization is done between calls.
+
+
+.. seealso::
+   *intro_shmem*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_swap.3.rst b/docs/man-openshmem/man3/shmem_swap.3.rst
new file mode 100644
index 00000000000..d273eae4964
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_swap.3.rst
@@ -0,0 +1,111 @@
+.. _shmem_swap:
+
+
+shmem_swap
+==========
+
+.. include_body
+
+:ref:`shmem_double_swap`\ (3), :ref:`shmem_float_swap`\ (3),
+:ref:`shmem_int_swap`\ (3), :ref:`shmem_long_swap`\ (3), :ref:`shmem_swap`\ (3),
+shmem_int4_swap\ (3), shmem_int8_swap\ (3), shmem_real4_swap\ (3),
+shmem_real8_swap\ (3), :ref:`shmem_longlong_swap`\ (3) - Performs an atomic
+swap to a remote data object
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   double shmem_double_swap(double *target, double value,
+     int pe)
+
+   float shmem_float_swap(float *target, float value, int pe)
+
+   int shmem_int_swap(int *target, int value, int pe)
+
+   long shmem_long_swap(long *target, long value, int pe)
+
+   long long shmem_longlong_swap(long long *target,
+     long long value, int pe)
+
+   long shmem_swap(long *target, long value, int pe)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   INTEGER pe
+
+   INTEGER SHMEM_SWAP
+   ires = SHMEM_SWAP(target, value, pe)
+
+   INTEGER(KIND=4) SHMEM_INT4_SWAP
+   ires = SHMEM_INT4_SWAP(target, value, pe)
+
+   INTEGER(KIND=8) SHMEM_INT8_SWAP
+   ires = SHMEM_INT8_SWAP(target, value, pe)
+
+   REAL(KIND=4) SHMEM_REAL4_SWAP
+   res = SHMEM_REAL4_SWAP(target, value, pe)
+
+   REAL(KIND=8) SHMEM_REAL8_SWAP
+   res = SHMEM_REAL8_SWAP(target, value, pe)
+
+
+DESCRIPTION
+-----------
+
+The atomic swap routines write **value** to address target on PE **pe**,
+and return the previous contents of **target** in one atomic operation.
+
+The arguments are as follows:
+
+target
+   The remotely accessible integer data object to be updated on the
+   remote PE. If you are using C/C++, the type of target should match
+   that implied in the SYNOPSIS section. If you are using Fortran, it
+   must be of the following type:
+
+   **SHMEM_SWAP:** Integer of default kind
+
+   **SHMEM_INT4_SWAP:** 4-byte integer
+
+   **SHMEM_INT8_SWAP:** 8-byte integer
+
+   **SHMEM_REAL4_SWAP:** 4-byte real
+
+   **SHMEM_REAL8_SWAP:** 8-byte real
+
+value
+   Value to be atomically written to the remote PE. value is the same
+   type as target.
+
+pe
+   An integer that indicates the PE number on which target is to be
+   updated. If you are using Fortran, it must be a default integer
+   value.
+
+
+NOTES
+-----
+
+The term remotely accessible is defined in *intro_shmem*\ (3).
+
+
+RETURN VALUES
+-------------
+
+The contents that had been at the target address on the remote PE prior
+to the swap is returned.
+
+
+.. seealso::
+   *intro_shmem*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_test_lock.3.rst b/docs/man-openshmem/man3/shmem_test_lock.3.rst
new file mode 100644
index 00000000000..9d37314fb2d
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_test_lock.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_test_lock:
+
+shmem_test_lock
+===============
+    .. include_body
+
+.. include:: ../man3/shmem_set_lock.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_udcflush.3.rst b/docs/man-openshmem/man3/shmem_udcflush.3.rst
new file mode 100644
index 00000000000..9a225b03256
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_udcflush.3.rst
@@ -0,0 +1,84 @@
+.. _shmem_udcflush:
+
+
+shmem_udcflush
+==============
+
+.. include_body
+
+:ref:`shmem_clear_cache_inv`\ (3), :ref:`shmem_set_cache_inv`\ (3),
+:ref:`shmem_set_cache_line_inv`\ (3), :ref:`shmem_udcflush`\ (3),
+:ref:`shmem_udcflush_line`\ (3) - Controls data cache utilities
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void shmem_clear_cache_inv(void)
+   void shmem_clear_cache_line_inv(void *target)
+   void shmem_set_cache_inv(void)
+   void shmem_set_cache_line_inv(void *target)
+   void shmem_udcflush(void)
+   void shmem_udcflush_line(void *target)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   CALL SHMEM_CLEAR_CACHE_INV
+   CALL SHMEM_CLEAR_CACHE_LINE_INV(target)
+   CALL SHMEM_SET_CACHE_INV
+   CALL SHMEM_SET_CACHE_LINE_INV(target)
+
+   CALL SHMEM_UDCFLUSH
+   CALL SHMEM_UDCFLUSH_LINE(target)
+
+
+DESCRIPTION
+-----------
+
+The following argument is passed to the cache line control routines:
+
+target
+   A data object that is local to the processing element (PE). target
+   can be of any noncharacter type. If you are using Fortran, it can be
+   of any kind.
+
+:ref:`shmem_clear_cache_inv` disables automatic cache coherency mode
+previously enabled by :ref:`shmem_set_cache_inv` or :ref:`shmem_set_cache_line_inv`.
+
+:ref:`shmem_clear_cache_line_inv` disables automatic cache coherency mode
+for the cache line associated with the address of **target** only.
+
+:ref:`shmem_set_cache_inv` enables the OpenSHMEM API to automatically
+decide the best strategy for cache coherency.
+
+:ref:`shmem_set_cache_line_inv` enables automatic cache coherency mode for
+the cache line associated with the address of **target** only.
+
+:ref:`shmem_clear_cache_inv` disables automatic cache coherency mode
+previously enabled by :ref:`shmem_set_cache_inv` or :ref:`shmem_set_cache_line_inv`.
+
+:ref:`shmem_udcflush` makes the entire user data cache coherent.
+
+:ref:`shmem_udcflush_line` makes coherent the cache line that corresponds
+with the address specified by target.
+
+
+NOTES
+-----
+
+These routines have been retained for improved backward compatibility
+with legacy architectures.
+
+
+.. seealso::
+   *intro_shmem*\ (3) *shmem_put*\ (3) *shmem_swap*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_udcflush_line.3.rst b/docs/man-openshmem/man3/shmem_udcflush_line.3.rst
new file mode 100644
index 00000000000..d6e089e49c0
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_udcflush_line.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_udcflush_line:
+
+shmem_udcflush_line
+===================
+    .. include_body
+
+.. include:: ../man3/shmem_udcflush.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmem_wait.3.rst b/docs/man-openshmem/man3/shmem_wait.3.rst
new file mode 100644
index 00000000000..5b5fd861e65
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_wait.3.rst
@@ -0,0 +1,201 @@
+.. _shmem_wait:
+
+
+shmem_wait
+==========
+
+.. include_body
+
+:ref:`shmem_int_wait`\ (3), :ref:`shmem_int_wait`\ (3)_until,
+shmem_int4_wait\ (3), shmem_int4_wait\ (3)_until,
+shmem_int8_wait\ (3), shmem_int8_wait\ (3)_until,
+:ref:`shmem_long_wait`\ (3), :ref:`shmem_long_wait`\ (3)_until,
+:ref:`shmem_longlong_wait`\ (3), :ref:`shmem_longlong_wait`\ (3)_until,
+:ref:`shmem_short_wait`\ (3), :ref:`shmem_short_wait`\ (3)_until,
+:ref:`shmem_wait`\ (3), :ref:`shmem_wait`\ (3)_until - Waits for a variable on the
+local processing element (PE) to change
+
+
+SYNOPSIS
+--------
+
+C or C++:
+
+.. code-block:: c++
+
+   #include <mpp/shmem.h>
+
+   void shmem_int_wait(volatile int *var, int value)
+
+   void shmem_int_wait_until(volatile int *var, int cond, int value)
+
+   void shmem_long_wait(volatile long *var, long value)
+
+   void shmem_long_wait_until(volatile long *var, int cond, long value)
+
+   void shmem_longlong_wait(volatile long long *var, long long value)
+
+   void shmem_longlong_wait_until(volatile long long *var, int cond,
+     long long value)
+
+   void shmem_short_wait(volatile short *var, short value)
+
+   void shmem_short_wait_until(volatile short *var, int cond,
+     short value)
+
+   void shmem_wait(volatile long *ivar, long cmp_value)
+
+   void shmem_wait_until(volatile long *ivar, int cmp, long value)
+
+Fortran:
+
+.. code-block:: fortran
+
+   INCLUDE "mpp/shmem.fh"
+
+   CALL SHMEM_INT4_WAIT(ivar, cmp_value)
+
+   CALL SHMEM_INT4_WAIT_UNTIL(ivar, cmp, cmp_value)
+
+   CALL SHMEM_INT8_WAIT(ivar, cmp_value)
+
+   CALL SHMEM_INT8_WAIT_UNTIL(ivar, cmp, cmp_value)
+
+   CALL SHMEM_WAIT(ivar, cmp_value)
+
+   CALL SHMEM_WAIT_UNTIL(ivar, cmp, cmp_value)
+
+
+DESCRIPTION
+-----------
+
+:ref:`shmem_wait` and :ref:`shmem_wait_until` wait for **ivar** to be changed by a
+remote write or atomic swap issued by a different processor. These
+routines can be used for point-to- point directed synchronization. A
+call to :ref:`shmem_wait` does not return until some other processor writes a
+value, not equal to cmp_value, into **ivar** on the waiting processor. A
+call to :ref:`shmem_wait_until` does not return until some other processor
+changes **ivar** to satisfy the condition implied by cmp and cmp_value.
+This mechanism is useful when a processor needs to tell another
+processor that it has completed some action.
+
+The arguments are as follows:
+
+target
+   The remotely accessible integer data object to be updated on the
+   remote PE. If you are using C/C++, the type of target should match
+   that implied in the SYNOPSIS section. If you are using the Fortran
+   compiler, it must be of type integer with an element size of 4 bytes
+   for SHMEM_INT4_ADD and 8 bytes for SHMEM_INT8_ADD.
+
+value
+   The value to be atomically added to target. If you are using C/C++,
+   the type of value should match that implied in the SYNOPSIS section.
+   If you are using Fortran, it must be of type integer with an element
+   size of target.
+
+pe
+   An integer that indicates the PE number upon which target is to be
+   updated. If you are using Fortran, it must be a default integer
+   value.
+
+ivar
+   A remotely accessible integer variable that is being updated by
+   another PE. If you are using C/C++, the type of ivar should match
+   that implied in the SYNOPSIS section. If you are using Fortran, ivar
+   must be a specific sized integer type according to the function being
+   called, as follows:
+
+   :ref:`shmem_wait`, :ref:`shmem_wait_until`:** default INTEGER
+
+   shmem_int4_wait, shmem_int4_wait_until:** INTEGER*4
+
+   shmem_int8_wait, shmem_int8_wait_until:** INTEGER*8
+
+cmp
+   The compare operator that compares ivar with cmp_value. cmp must be
+   of type integer. If you are using Fortran, it must be of default
+   kind. If you are using C/C++, the type of cmp should match that
+   implied in the SYNOPSIS section. The following cmp values are
+   supported:
+
+   SHMEM_CMP_EQ
+      Equal
+
+   SHMEM_CMP_NE
+      Not equal
+
+   SHMEM_CMP_GT
+      Greater than
+
+   SHMEM_CMP_LE
+      Less than or equal to
+
+   SHMEM_CMP_LT
+      Less than
+
+   SHMEM_CMP_GE
+      Greater than or equal to
+
+cmp_value
+   cmp_value must be of type integer. If you are using C/C++, the type
+   of cmp_value should match thatimplied in the SYNOPSIS section. If you
+   are using Fortran, cmp_value must be an integer of the same size and
+   kind as ivar. The :ref:`shmem_wait` routines return when ivar is no longer
+   equal to cmp_value. The :ref:`shmem_wait_until` routines return when the
+   compare condition is true. The compare condition is defined by the
+   ivar argument compared with the cmp_value using the comparison
+   operator, cmp.
+
+
+EXAMPLES
+--------
+
+**Example 1:** The following call returns when variable ivar is not
+equal to 100:
+
+::
+
+   INTEGER*8 IVAR
+
+   CALL SHMEM_INT8_WAIT(IVAR, INT8(100))
+
+**Example 2:** The following call to SHMEM_INT8_WAIT_UNTIL is equivalent
+to the call to SHMEM_INT8_WAIT in example 1:
+
+::
+
+   INTEGER*8 IVAR
+
+   CALL SHMEM_INT8_WAIT_UNTIL(IVAR, SHMEM_CMP_NE, INT8(100))
+
+**Example 3:** The following C/C++ call waits until the sign bit in ivar
+is set by a transfer from a remote PE:
+
+::
+
+   int ivar;
+
+   shmem_int_wait_until(&ivar, SHMEM_CMP_LT, 0);
+
+**Example 4:** The following Fortran example is in the context of a
+subroutine:
+
+::
+
+   SUBROUTINE EXAMPLE()
+     INTEGER FLAG_VAR
+     COMMON/FLAG/FLAG_VAR
+     . . .
+     FLAG_VAR = FLAG_VALUE ! initialize the event variable
+     . . .
+     IF (FLAG_VAR .EQ. FLAG_VALUE) THEN
+       CALL SHMEM_WAIT(FLAG_VAR, FLAG_VALUE)
+     ENDIF
+     FLAG_VAR = FLAG_VALUE ! reset the event variable for next time
+     . . .
+   END
+
+
+.. seealso::
+   *intro_shmem*\ (3) *shmem_put*\ (3)
diff --git a/docs/man-openshmem/man3/shmem_wait_until.3.rst b/docs/man-openshmem/man3/shmem_wait_until.3.rst
new file mode 100644
index 00000000000..4dd514955e2
--- /dev/null
+++ b/docs/man-openshmem/man3/shmem_wait_until.3.rst
@@ -0,0 +1,9 @@
+.. _shmem_wait_until:
+
+shmem_wait_until
+================
+    .. include_body
+
+.. include:: ../man3/shmem_wait.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shmemalign.3.rst b/docs/man-openshmem/man3/shmemalign.3.rst
new file mode 100644
index 00000000000..f60aebb64e9
--- /dev/null
+++ b/docs/man-openshmem/man3/shmemalign.3.rst
@@ -0,0 +1,9 @@
+.. _shmemalign:
+
+shmemalign
+==========
+    .. include_body
+
+.. include:: ../man3/shmem_malloc.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/shrealloc.3.rst b/docs/man-openshmem/man3/shrealloc.3.rst
new file mode 100644
index 00000000000..82a74a99404
--- /dev/null
+++ b/docs/man-openshmem/man3/shrealloc.3.rst
@@ -0,0 +1,9 @@
+.. _shrealloc:
+
+shrealloc
+=========
+    .. include_body
+
+.. include:: ../man3/shmem_malloc.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/man-openshmem/man3/start_pes.3.rst b/docs/man-openshmem/man3/start_pes.3.rst
new file mode 100644
index 00000000000..da178a852c8
--- /dev/null
+++ b/docs/man-openshmem/man3/start_pes.3.rst
@@ -0,0 +1,9 @@
+.. _start_pes:
+
+start_pes
+=========
+    .. include_body
+
+.. include:: ../man3/shmem_init.3.rst
+    :start-after: .. include_body
+
diff --git a/docs/mca.rst b/docs/mca.rst
new file mode 100644
index 00000000000..beffb6b8ac9
--- /dev/null
+++ b/docs/mca.rst
@@ -0,0 +1,1149 @@
+.. _label-mca:
+
+The Modular Component Architecture (MCA)
+========================================
+
+Open MPI is a highly-customizable system; it can be configured via
+configuration files, command line parameters, and environment
+variables.  The main functionality of Open MPI's configuration system
+is through the Modular Component Architecture (MCA).
+
+* This section describes the MCA itself and how to set MCA parameters at
+  run time.
+* Later sections in this documentation describe different parts of
+  Open MPI's functionality, and the specific names and values of MCA
+  parameters that can be used to affect Open MPI's behavior.
+
+.. note:: :ref:`The PMIx and PRRTE software packages
+          <label-running-role-of-pmix-and-prte>` also use the MCA for
+          their configuration, composition, and run-time tuning.
+
+/////////////////////////////////////////////////////////////////////////
+
+Terminology
+-----------
+
+The Modular Component Architecture (MCA) is the backbone for much of
+Open MPI's functionality.  It is a series of *projects*, *frameworks*,
+*components*, and *modules* that are assembled at run-time to create
+an MPI implementation.
+
+MCA *parameters* (also known as MCA *variables*) are used to customize
+Open MPI's behavior at run-time.
+
+Each of these entities are described below.
+
+Projects
+^^^^^^^^
+
+A *project* is essentially the highest abstraction layer division in
+the Open MPI code base.
+
+.. note:: The word "project" is unfortunately overloaded.  It can be
+          used to mean the code/resources/people in the greater Open
+          MPI community associated with the development of a
+          particular software package, but it can also be used to mean
+          a major, top-level section of code within the Open MPI code
+          base.
+
+          For the purposes of this documentation, "project" means the
+          latter: a major, top-level section of code within the Open
+          MPI code base.
+
+The following *projects* exist in Open MPI |ompi_ver|:
+
+* **Open Portability Access Layer (OPAL):** Low-level, operating
+  system and architecture portability code.
+* **Open MPI (OMPI):** The MPI API and supporting infrastructure.
+* **OpenSHMEM (OSHMEM):** The OpenSHMEM API and supporting
+  infrastructure.
+
+.. note:: Prior versions of Open MPI also included an Open MPI
+          Runtime Environment (ORTE) project.  ORTE essentially
+          evolved into the standalone `PMIx Runtime Reference
+          Environment (PRRTE) <https://github.com/openpmix/prrte>`_
+          and is now considered a 3rd-party dependency of Open MPI
+          |mdash| not one of its included projects.
+
+          See :ref:`the role of PMIx and PRRTE
+          <label-running-role-of-pmix-and-prte>` for more information.
+
+.. _label-mca-frameworks:
+
+Frameworks
+^^^^^^^^^^
+
+An MCA framework manages zero or more components at run-time and is
+targeted at a specific task (e.g., providing MPI collective operation
+functionality).  Although each MCA framework supports only a single
+type of component, it may support multiple components of that type.
+
+Some of the more common frameworks that users may want or need to
+customize include the following:
+
+* ``btl``: Byte Transport Layer; these components are used as the
+  underlying transports for the ``ob1`` PML component and ``rdma``
+  OSC component.
+* ``coll``: MPI collective algorithms
+* ``io``: MPI I/O
+* ``mtl``: MPI Matching Transport Layer (MTL); these components are
+  exclusively used as the underlying transports for the ``cm`` PML
+  component.
+* ``pml``: Point-to-point Messaging Layer (PML).  These components are
+  used to implement MPI point-to-point messaging functionality.
+
+There are many frameworks within Open MPI; the exact set varies
+between different versions of Open MPI.  You can use the
+:ref:`ompi_info(1) <man1-ompi_info>` command to see the full list of
+frameworks that are included in Open MPI |ompi_ver|.
+
+.. _label-mca-components:
+
+Components
+^^^^^^^^^^
+
+An MCA component is an implementation of a framework's formal
+interface.  It is a standalone collection of code that can be bundled
+into a plugin that can be inserted into the Open MPI code base, either
+at run-time and/or compile-time.
+
+.. note:: Good synonyms for Open MPI's "component" concept are
+          "plugin", or "add-on".
+
+The exact set of components varies between different versions of Open
+MPI.  Open MPI's code base includes support for many components, but
+not all of them may be present or available on your system.  You can
+use the :ref:`ompi_info(1) <man1-ompi_info>` command to see what
+components are included in Open MPI |ompi_ver| on your system.
+
+.. _label-mca-modules:
+
+Modules
+^^^^^^^
+
+An MCA module is an instance of a component (in the C++ sense of the
+word "instance"; an MCA component is analogous to a C++ class).  For
+example, if a node running an Open MPI application has two Ethernet
+NICs, the Open MPI application will contain one TCP MPI point-to-point
+*component*, but two TCP point-to-point *modules*.
+
+Parameters (variables)
+^^^^^^^^^^^^^^^^^^^^^^
+
+MCA *parameters* (sometimes called MCA *variables*) are the basic unit
+of run-time tuning for Open MPI.  They are simple "key = value" pairs
+that are used extensively throughout Open MPI.  The general rules of
+thumb that the developers use are:
+
+#. Instead of using a constant for an important value, make it an MCA
+   parameter.
+#. If a task can be implemented in multiple, user-discernible ways,
+   implement as many as possible, and use an an MCA parameter to
+   choose between them at run-time.
+
+For example, an easy MCA parameter to describe is the boundary between
+short and long messages in TCP wire-line transmissions.  "Short"
+messages are sent eagerly whereas "long" messages use a rendezvous
+protocol.  The decision point between these two protocols is the
+overall size of the message (in bytes).  By making this value an MCA
+parameter, it can be changed at run-time by the user or system
+administrator to use a sensible value for a particular environment or
+set of hardware (e.g., a value suitable for 1Gpbs Ethernet is probably
+not suitable for 100 Gigabit Ethernet, and may require even a third
+different value for 25 Gigabit Ethernet).
+
+/////////////////////////////////////////////////////////////////////////
+
+.. _label-running-setting-mca-param-values:
+
+Setting MCA parameter values
+----------------------------
+
+MCA parameters may be set in several different ways.
+
+.. admonition:: Rationale
+   :class: tip
+
+   Having multiple methods to set MCA parameters allows, for example,
+   system administrators to fine-tune the Open MPI installation for
+   their hardware / environment such that normal users can simply use
+   the default values (that were set by the system administrators).
+
+   HPC environments |mdash| and the applications that run on them
+   |mdash| tend to be unique.  Providing extensive run-time tuning
+   capabilities through MCA parameters allows the customization of
+   Open MPI to each system's / user's / application's particular
+   needs.
+
+The following are the different methods to set MCA parameters, listed
+in priority order:
+
+#. Command line parameters
+#. Environment variables
+#. Tuning MCA parameter files
+#. Configuration files
+
+.. danger:: Due to how the PMIx and PRRTE projects both evolved to
+            become independent projects from Open MPI (:ref:`see this
+            section for more detail
+            <label-running-role-of-pmix-and-prte>`), they both have
+            their own MCA system for setting MCA parameters.
+
+            Hence, all the information about MCA parameters below
+            *also* applies to PMIx and PRRTE.
+
+Command line parameters
+^^^^^^^^^^^^^^^^^^^^^^^
+
+The highest-precedence method is setting MCA parameters on the command
+line.  For example:
+
+.. code-block:: sh
+
+   shell$ mpirun --mca mpi_show_handle_leaks 1 -np 4 a.out
+
+This sets the MCA parameter ``mpi_show_handle_leaks`` to the value of
+1 before running ``a.out`` with four processes.  In general, the
+format used on the command line is ``--mca <param_name> <value>``.
+
+.. note:: When setting a value that includes spaces, you need to use
+          quotes to ensure that the shell understands that the
+          multiple tokens are a single value.  For example:
+
+          .. code-block:: sh
+
+             shell$ mpirun --mca param "value with multiple words" ...
+
+.. warning:: Setting Open MPI MCA parameters via the command line
+             entails using the ``--mca`` CLI option.  When setting
+             PMIx- and PRRTE-specific MCA parameters via configuration
+             files, use a different CLI option:
+
+             +----------+----------------+
+             | Open MPI | ``--mca``      |
+             +----------+----------------+
+             | PMIx     | ``--pmixmca``  |
+             +----------+----------------+
+             | PRRTE    | ``--prtemca``  |
+             +----------+----------------+
+
+Environment variables
+^^^^^^^^^^^^^^^^^^^^^
+
+Next, environment variables are searched.  Any environment variable
+named ``OMPI_MCA_<param_name>`` will be used.  For example, the
+following has the same effect as the previous example (for sh-flavored
+shells):
+
+.. code-block:: sh
+
+   shell$ export OMPI_MCA_mpi_show_handle_leaks=1
+   shell$ mpirun -np 4 a.out
+
+.. note:: Just like with command line values, setting environment
+          variables to values with multiple words requires shell
+          quoting, such as:
+
+          .. code-block:: sh
+
+             shell$ export OMPI_MCA_param="value with multiple words"
+
+.. warning:: Setting Open MPI MCA parameters via environment variables
+             entails prefixing the parameter name with ``OMPI_MCA_``.
+             When setting PMIx- and PRRTE-specific MCA parameters via
+             environment variables, use a different prefix:
+
+             +----------+----------------+
+             | Open MPI | ``OMPI_MCA_``  |
+             +----------+----------------+
+             | PMIx     | ``PMIX_MCA_``  |
+             +----------+----------------+
+             | PRRTE    | ``PRRTE_MCA_`` |
+             +----------+----------------+
+
+Tuning MCA parameter files
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. error:: TODO This entire section needs to be checked for correctness.
+
+Simple text files can be used to set MCA parameter values for a
+specific application.
+
+The ``mpirun --tune`` CLI option allows users to specify both MCA
+parameters and environment variables from within a single file.
+
+MCA parameters set in tuned parameter files will override any MCA
+parameters supplied in global parameter files (e.g.,
+``$HOME/.openmpi/mca-params.conf``), but not command line or
+environment parameters.
+
+Consider a tuned parameter file name ``foo.conf`` that is placed in
+the same directory as the application ``a.out``. A user will typically
+run the application as:
+
+.. code-block:: sh
+
+   shell$ mpirun -np 2 a.out
+
+To use the ``foo.conf`` tuned parameter file, this command line
+changes to:
+
+.. code-block:: sh
+
+   shell$ mpirun -np 2 --tune foo.conf a.out
+
+Tuned parameter files can be coupled if more than one file is to be
+used. If there is another tuned parameter file called ``bar.conf``, it
+can be added to the command line as follows:
+
+.. code-block:: sh
+
+   shell$ mpirun -np 2 --tune foo.conf,bar.conf a.out
+
+The contents of tuned files consist of one or more lines, each of
+which contain zero or more `-x` and `--mca` options.  Comments are not
+allowed.  For example, the following tuned file:
+
+.. code-block::
+
+   -x envvar1=value1 -mca param1 value1 -x envvar2
+   -mca param2 value2
+   -x envvar3
+
+is equivalent to:
+
+.. code-block:: sh
+
+   shell$ mpirun \
+       -x envvar1=value1 -mca param1 value1 -x envvar2 \
+       -mca param2 value2
+       -x envvar3 \
+       ...rest of mpirun command line...
+
+Although the typical use case for tuned parameter files is to be
+specified on the command line, they can also be set as MCA parameters
+in the environment.  The MCA parameter ``mca_base_envvar_file_prefix``
+contains a comma-delimited list of tuned parameter files exactly as
+they would be passed to the ``--tune`` command line option.  The MCA
+parameter ``mca_base_envvar_file_path`` specifies the path to search
+for tuned files with relative paths.
+
+.. error:: TODO Check that these MCA var names ^^ are correct.
+
+Configuration files
+^^^^^^^^^^^^^^^^^^^
+
+Finally, simple configuration text files can be used to set MCA
+parameter values.  Parameters are set one per line (comments are
+permitted).  For example:
+
+.. code-block:: ini
+
+   # This is a comment
+   # Set the same MCA parameter as in previous examples
+   mpi_show_handle_leaks = 1
+
+Note that quotes are *not* necessary for setting multi-word values
+in MCA parameter files.  Indeed, if you use quotes in the MCA
+parameter file, they will be used as part of the value itself.  For
+example:
+
+.. code-block:: ini
+
+   # The following two values are different:
+   param1 = value with multiple words
+   param2 = "value with multiple words"
+
+By default, two files are searched (in order):
+
+#. ``$HOME/.openmpi/mca-params.conf``: The user-supplied set of
+   values takes the highest precedence.
+#. ``$prefix/etc/openmpi-mca-params.conf``: The system-supplied set
+   of values has a lower precedence.
+
+More specifically, the MCA parameter ``mca_param_files`` specifies a
+colon-delimited path of files to search for MCA parameters.  Files to
+the left have lower precedence; files to the right are higher
+precedence.
+
+.. note:: Keep in mind that, just like components, these parameter
+          files are *only* relevant where they are "visible"
+          (:ref:`see this FAQ entry
+          <installing-custom-components-label>`).  Specifically,
+          Open MPI does not read all the values from these files
+          during startup and then send them to all nodes in the job.
+          Instead, the files are read on each node during each
+          process' startup.
+
+          *This is intended behavior:* it allows for per-node
+          customization, which is especially relevant in heterogeneous
+          environments.
+
+.. error:: TODO This table needs to be checked for correctness.
+
+.. warning:: Setting Open MPI MCA parameters via configuration files
+             entails editing (by default) the ``mca-params.conf`` or
+             ``openmpi-mca-params.conf`` files.  When setting PMIx-
+             and PRRTE-specific MCA parameters via configuration
+             files, set them (by default) in different files:
+
+             +----------+------------------------------------------+
+             | Open MPI | ``$HOME/.openmpi/mca-params.conf`` or    |
+             |          | ``$prefix/etc/openmpi-mca-params.conf``  |
+             +----------+------------------------------------------+
+             | PMIx     | ``$HOME/.pmix/mca-params.conf`` or       |
+             |          | ``$prefix/etc/openpmix-mca-params.conf`` |
+             +----------+------------------------------------------+
+             | PRRTE    | ``$HOME/.prrte/mca-params.conf`` or      |
+             |          | ``$prefix/etc/prte-mca-params.conf``     |
+             +----------+------------------------------------------+
+
+/////////////////////////////////////////////////////////////////////////
+
+.. _label-running-displaying-mca-param-values:
+
+Displaying MCA parameter values
+-------------------------------
+
+MCA parameters are the "life's blood" of Open MPI. MCA parameters are
+used to control both detailed and large-scale behavior of Open MPI and
+are present throughout the code base.
+
+This raises an important question: since MCA parameters can be set from a
+file, the environment, the command line, and even internally within Open MPI,
+how do I actually know what MCA params my job is seeing, and their value?
+
+One way, of course, is to use the :ref:`ompi_info(1) <man1-ompi_info>`
+command, which is documented elsewhere (you can use ``man ompi_info``,
+or ``ompi_info --help`` to get more info on this command). However,
+this still doesn't fully answer the question since ``ompi_info`` isn't
+an MPI process.
+
+To help relieve this problem, Open MPI provides the MCA parameter
+``mpi_show_mca_params`` that directs the ``MPI_COMM_WORLD`` rank 0
+process to report the name of MCA parameters, their current value as
+seen by that process, and the source that set that value.  The
+parameter can take several values that define which MCA parameters to
+report:
+
+* ``all``: report all MCA params. Note that this typically generates a
+  rather long list of parameters since it includes all of the default
+  parameters defined inside Open MPI
+* ``default``: MCA params that are at their default settings - i.e.,
+  all MCA params that are at the values set as default within Open MPI
+* ``file``: MCA params that had their value set by a file
+* ``api``: MCA params set using Open MPI's internal APIs, perhaps to
+  override an incompatible set of conditions specified by the user
+* ``enviro``: MCA params that obtained their value either from the
+  local environment or the command line. Open MPI treats environmental
+  and command line parameters as equivalent, so there currently is no
+  way to separate these two sources
+
+These options can be combined in any order by separating them with
+commas.
+
+Here is an example of the output generated by this parameter:
+
+.. code-block:: sh
+
+   shell$ mpirun --mca mpi_show_mca_params enviro hello_c
+   [local-hostname:12345] mpi_show_mca_params=enviro (environment)
+   Hello, World, I am 0 of 1
+
+Note that several MCA parameters set by Open MPI itself for internal
+uses are displayed in addition to the ones actually set by the user.
+
+Since the output from this option can be long, and since it can be
+helpful to have a more permanent record of the MCA parameters used for
+a job, a companion MCA parameter ``mpi_show_mca_params_file`` is
+provided. If ``mpi_show_mca_params_file`` is *also* set, the output
+listing of MCA parameters will be directed into the specified file
+instead of being printed to stdout.  For example:
+
+.. code-block:: sh
+
+   shell$ mpirun --mca mpi_show_mca_params enviro \
+       --mca mpi_show_mca_param_file /tmp/foo.txt hello_c
+   Hello, World, I am 0 of 1
+   shell$ cat /tmp/foo.txt
+   #
+   # This file was automatically generated on Sun Feb  7 14:34:31 2021
+   # by MPI_COMM_WORLD rank 0 (out of a total of 16) on savbu-usnic-a
+   #
+   mpi_show_mca_params=enviro (environment)
+   mpi_show_mca_params_file=/tmp/foo.txt (environment)
+
+/////////////////////////////////////////////////////////////////////////
+
+.. _label-running-selecting-framework-components:
+
+Selecting which Open MPI components are used at run time
+--------------------------------------------------------
+
+Each MCA framework has a top-level MCA parameter that helps guide
+which components are selected to be used at run-time.  Specifically,
+every framework has an MCA parameter of the same name that can be used
+to *include* or *exclude* components from a given run.
+
+For example, the ``btl`` MCA parameter is used to control which BTL
+components are used.  It takes a comma-delimited list of component
+names, and may be optionally prefixed with ``^``.  For example:
+
+.. note:: The Byte Transfer Layer (BTL) framework is used as the
+          underlying network transports with the `ob1` Point-to-point
+          Messaging Layer (PML) component.
+
+.. code-block:: sh
+
+   # Tell Open MPI to include *only* the BTL components listed here and
+   # implicitly ignore all the rest:
+   shell$ mpirun --mca btl self,sm,usnic ...
+
+   # Tell Open MPI to exclude the tcp and uct BTL components
+   # and implicitly include all the rest
+   shell$ mpirun --mca btl ^tcp,uct ...
+
+Note that ``^`` can *only* be the prefix of the *entire*
+comma-delimited list because the inclusive and exclusive behavior are
+mutually exclusive.  Specifically, since the exclusive behavior means
+"use all components *except* these", it does not make sense to mix it
+with the inclusive behavior of not specifying it (i.e., "use all of
+these components").  Hence, something like this:
+
+.. code-block:: sh
+
+   shell$ mpirun --mca btl self,sm,usnic,^tcp ...
+
+does not make sense |mdash| and will cause an error |mdash| because it
+says "use only the ``self``, ``sm``, and ``usnic`` components" but
+also "use all components except ``tcp``".  These two statements
+clearly contradict each other.
+
+/////////////////////////////////////////////////////////////////////////
+
+.. _label-mca-common-parameters:
+
+Common MCA parameters
+---------------------
+
+Open MPI has a *large* number of MCA parameters available.  Users can
+use the :ref:`ompi_info(1) <man1-ompi_info>` command to see *all*
+available MCA parameters.
+
+.. note:: Similarly, you can use the ``pmix_info(1)`` and
+          ``prte_info(1)`` commands to see all the MCA parameters
+          available for the PMIx and PRRTE projects, respectively.
+
+          The documentation for these commands are not included in the
+          Open MPI docs, but they are both quite similar to
+          :ref:`ompi_info(1) <man1-ompi_info>`.
+
+The vast majority of these MCA parameters, however, are not useful to
+most users.  Indeed, there only are a handful of MCA parameters that
+are commonly used by end users.  :ref:`As described in the
+ompi_info(1) man page <man1-ompi_info-levels>`, MCA parameters are
+grouped into nine levels, corresponding to the MPI standard's tool
+support verbosity levels.  In general:
+
+* Levels 1-3 are intended for the end user.
+
+  * These parameters are generally used to effect whether an Open MPI
+    job will be able to run correctly.
+
+  .. tip:: Parameters in levels 1-3 are probably applicable to
+           most end users.
+
+* Levels 4-6 are intended for the application tuner.
+
+  * These parameters are generally used to tune the performance of an
+    Open MPI job.
+
+* Levels 7-9 are intended for the MPI implementer.
+
+  * These parameters are esoteric and really only intended for those
+    who work deep within the implementation of Open MPI code base
+    itself.
+
+Although the full list of MCA parameters can be found in the output of
+``ompi_info(1)``, the following list of commonly-used parameters is
+presented here so that they can easily be found via internet searches:
+
+* Individual framework names are used as MCA parameters to
+  :ref:`select which components will be used
+  <label-running-selecting-framework-components>`.  For example, the
+  ``btl`` MCA parameter is used to select which components will be
+  used from the ``btl`` framework.  The ``coll`` MCA parameter is used
+  to select which ``coll`` components are used.  And so on.
+
+* Individual framework names with the ``_base_verbose`` suffix
+  appended (e.g., ``btl_base_verbose``, ``coll_base_verbose``, etc.)
+  can be used to set the general verbosity level of all the components
+  in that framework.
+
+  * This can be helpful when troubleshooting why certain components
+    are or are not being selected at run time.
+
+* Many network-related components support "include" and "exclude"
+  types of components (e.g., ``btl_tcp_if_include`` and
+  ``btl_tcp_if_exclude``).  The "include" parameters specify an
+  explicit set of network interfaces to use; the "exclude" parameters
+  specify an explicit set of network interfaces to ignore.  Check the
+  output from :ref:`ompi_info(1)'s <man1-ompi_info>` full list to see
+  if the network-related component you are using has "include" and
+  "exclude" network interface parameters.
+
+  .. important:: You can only use the "include" *or* the "exclude"
+                 parameter |mdash| they are mutually exclusive from each
+                 other.
+* ``mca_base_component_show_load_errors``: By default, Open MPI
+  emits a warning message if it fails to open a DSO component at run
+  time.  This typically happens when a shared library that the DSO
+  requires is not available.
+
+  .. admonition:: Rationale
+     :class: tip
+
+     In prior versions of Open MPI, components defaulted to building
+     as DSOs (vs. being included in their parent libraries, such as
+     ``libmpi.so``).  On misconfigured systems, sometimes network
+     acceleration libraries would not be present, meaning that
+     HPC-class networking components failed to open at run time.  As
+     such, Open MPI would typically fall back to TCP as a network
+     transport, which usually led to poor performance of end-user
+     applications.
+
+     Having Open MPI warn about such failures to load was useful
+     because it alerted users to the misconfiguration.
+
+  .. note:: By default, Open MPI |ompi_ver| includes all components in
+            its base libraries (e.g., on Linux, ``libmpi.so`` includes
+            all the components that were built with Open MPI, and
+            therefore no component need to be opened dynamically), and
+            does not build its components as DSOs.
+
+            This MCA parameter *only* affects the behavior of when a
+            component DSO fails to open.
+
+  This MCA parameter can take four general values:
+
+  #. ``yes`` or a boolean "true" value (e.g., ``1``): Open MPI will
+     emit a warning about every component DSO that fails to load.
+
+  #. ``no`` or a boolean "false" value (e.g., ``0``): Open MPI will
+     never emit warnings about component DSOs that fail to load.
+
+  #. A comma-delimited list of frameworks and/or components: Open MPI
+     will emit a warning about any dynamic component that fails to
+     open and matches a token in the list. "Match" is defined as:
+
+     * If a token in the list is only a framework name, then any
+       component in that framework will match.
+     * If a token in the list specifies both a framework name and a
+       component name (in the form ``framework/component``), then
+       only the specified component in the specified framework will
+       match.
+
+     For example, if the value of this MCA parameter is
+     ``accelerator,btl/uct``, then Open MPI warn if any component in
+     the accelerator framework or if the UCT BTL fails to load at run
+     time.
+
+  #. The value can also be a ``^`` character followed by a
+     comma-delimited list of ``framework[/component]`` values: This
+     is similar to the comma-delimited list of tokens, except it will
+     only emit warnings about dynamic components that fail to load
+     and do *not* match a token in the list.
+
+     For example, if the value of this MCA parameter is
+     ``^accelerator,btl/uct``, then Open MPI will only warn about the
+     failure to load DSOs that are neither in the accelerator
+     framework nor are the UCT BTL.
+
+.. _label-mca-backward-compat:
+
+MCA Parameter Changes Between Open MPI 4.x and 5.x
+--------------------------------------------------
+
+When Open MPI :ref:`switched from using ORTE to PRRTE as its run-time
+environment, <label-running-role-of-pmix-and-prte>` some MCA
+parameters were renamed to be more consistent and/or allow more
+flexible behavior.  The deprecated Open MPI MCA parameters listed
+below are currently replaced by a corresponding new PRRTE parameter,
+but may be removed in future releases.
+
+.. note:: In all cases listed below, the deprecated MCA parameter is
+          an Open MPI MCA parameter, meaning that its corresponding
+          environment variable was prefixed with ``OMPI_MCA_`` (e.g.,
+          ``OMPI_MCA_orte_xml_output``).  However, the corresponding
+          new MCA parameter is a PRRTE MCA parameter, meaning that its
+          corresponding environment variable is prefixed with
+          ``PRTE_MCA_`` (e.g., ``PRTE_MCA_output``).
+
+          .. important:: Yes, that's a single ``R`` in the
+                         ``PRTE_MCA_`` environment variable prefix.
+                         `See this explanation
+                         <https://docs.prrte.org/>`_ for the when one
+                         R or two R's are used in the PRRTE name.
+
+
+.. list-table::
+    :header-rows: 1
+
+    * - Behavior
+      - Deprecated MCA parameter
+      - Replaced with
+
+    * - Control buffering of stream output
+      - ``orte_ess_base_stream_buffering``
+
+        Values:  0 | 1 | 2
+
+      - ``ompi_stream_buffering``
+
+        Values: same
+
+    * - Output a brief periodic report on launch progress
+      - ``orte_report_launch_progress``
+
+        Values: boolean
+      - ``state_base_show_launch_progress``
+
+        Values: same
+
+    * - Provide all output in XML format
+      - ``orte_xml_output``
+
+        Values: boolean
+      - ``output``
+
+        Value: ``xml``
+
+    * - Tag all output with [job,rank]
+      - ``orte_tag_output``
+
+        Values: boolean
+      - ``output``
+
+        Value: ``tag``
+
+    * - Timestamp all application process output
+      - ``orte_timestamp_output``
+
+        Values: boolean
+      - ``output``
+
+        Value: ``timestamp``
+
+    * - Redirect output from application processes into filename / job
+        / rank / stdout / stderr / stdddiag.
+      - ``orte_output_filename``
+
+        Value: ``<filenname>``
+      - ``output``
+
+        Value: ``file=<filename>``
+
+    * - Display a detailed process map just before launch
+      - ``rmaps_base_display_devel_map``
+
+        Values: boolean
+      - ``display``
+
+        Value: ``map-devel``
+
+    * - Display the topology as part of the process map just before
+        launch
+      - ``rmaps_base_display_topo_with_map``
+
+        Values: ``<value>``
+      - ``display``
+
+        Value: ``topo=<value>``
+
+    * - Whether to report process bindings to stderr
+      - ``hwloc_base_report_bindings``
+
+        Values: boolean
+      - ``display``
+
+        Value: ``bind``
+
+    * - Display the process map just before launch
+      - ``rmaps_base_display_map``
+
+        Values: boolean
+      - ``display``
+
+        Value: ``map``
+
+    * - Display the allocation being used by this job
+      - ``orte_display_alloc``
+
+        Values: boolean
+      - ``display``
+
+        Value: ``allocation``
+
+    * - Do not run any MPI applications on the local node
+      - ``rmaps_base_no_schedule_local``
+
+        Values: boolean
+      - ``rmaps_default_mapping_policy``
+
+        Value: ``[<mapping>]:nolocal``
+
+    * - Nodes are allowed to be oversubscribed, even on a managed
+        system, and overloading of processing elements
+      - ``rmaps_base_oversubscribe``
+
+        Values: boolean
+      - ``rmaps_default_mapping_policy``
+
+        Value: ``[<mapping>]:oversubscribe``
+
+    * - Nodes are not to be oversubscribed, even if the system
+        supports such operation
+      - ``rmaps_base_no_oversubscribe``
+
+        Values: boolean
+      - ``rmaps_default_mapping_policy``
+
+        Value: ``[<mapping>]:nooversubscribe``
+
+    * - Use hardware threads as independent CPUs
+      - ``hwloc_base_use_hwthreads_as_cpus``
+
+        Values: boolean
+      - ``rmaps_default_mapping_policy``
+
+        Value: ``[<mapping>]:hwtcpus``
+
+    * - Comma-separated list of ranges specifying logical cpus
+        allocated to this job
+      - ``hwloc_base_cpu_set``
+
+        Value: ``<value>``
+      - ``rmaps_default_mapping_policy``
+
+        Value: ``pe-list=<value>``
+
+    * - List of processor IDs to bind processes to
+      - ``hwloc_base_cpu_list``
+
+        Value: ``<value>``
+      - ``rmaps_default_mapping_policy``
+
+        Value: ``pe-list=<value>``
+
+    * - Bind processes to cores
+      - ``hwloc_base_bind_to_core``
+
+        Values: boolean
+      - ``hwloc_default_binding_policy``
+
+        Value: ``core``
+
+    * - Bind processes to sockets
+      - ``hwloc_base_bind_to_socket``
+
+        Values: boolean
+      - ``hwloc_default_binding_policy``
+
+        Value: ``package``
+
+    * - Whether to map and rank processes round-robin by node
+      - ``rmaps_base_bynode``
+
+        Values: boolean
+      - ``rmaps_default_mapping_policy``
+
+        Value: ``node``
+
+    * - Whether to map and rank processes round-robin by core
+      - ``rmaps_base_bycore``
+
+        Values: boolean
+      - ``rmaps_default_mapping_policy``
+
+        Value: ``core``
+
+    * - Whether to map and rank processes round-robin by slot
+      - ``rmaps_base_byslot``
+
+        Values: boolean
+      - ``rmaps_default_mapping_policy``
+
+        Value: ``slot``
+
+    * - Number of cpus to use for each process
+      - ``rmaps_base_cpus_per_rank``
+
+        Value: ``<X>``
+      - ``rmaps_default_mapping_policy``
+
+        Value: ``[<mapping>]:pe=<X>``
+
+    * - Launch n processes per node on all allocated nodes
+      - ``rmaps_ppr_n_pernode``
+
+        Value: ``<X>``
+      - ``rmaps_default_mapping_policy``
+
+        Value: ``ppr:<X>:node``
+
+    * - Launch one process per available node
+      - ``rmaps_ppr_pernode``
+
+        Values: boolean
+      - ``rmaps_default_mapping_policy``
+
+        Value: ``ppr:1:node``
+
+    * - Launch n processes per socket on all allocated nodes
+      - ``rmaps_ppr_n_persocket``
+
+        Value: integer ``<X>``
+      - ``rmaps_default_mapping_policy``
+
+        Value: ``ppr:<X>:package``
+
+    * - Comma-separated list of number of processes on a given
+        resource type
+      - ``rmaps_ppr_pattern``
+
+        Value: ``<value>``
+      - ``rmaps_default_mapping_policy``
+
+        Value: ``ppr:<value>``
+
+    * - Provide a rankfile file
+      - ``orte_rankfile``
+
+        Value: ``<filename>``
+      - ``rmaps_default_mapping_policy``
+
+        Value: ``rankfile:file=<filename>``
+
+Examples
+^^^^^^^^^^^^^^^^^^^
+
+Converting many parameters in the table above are straightforward, where an
+integer or boolean value is involved, but some of the conversions require
+substituting a boolean with a value to the new parameter, or even constructing
+a more complicated composite value for the new parameter.  Examples of all
+of these types of conversions are given below.
+
+Simple values, where only the name of the MCA parameter changed
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+.. code-block:: ini
+
+   # Old environment variable: (integer value)
+   export OMPI_MCA_orte_ess_base_stream_buffering=2
+
+   # New environment variable: (integer value)
+   export PRTE_MCA_ompi_stream_buffering=2
+
+.. code-block:: ini
+
+   # Old environment variable: (boolean value)
+   export OMPI_MCA_orte_report_launch_progress=1
+
+   # New environment variable: (boolean value)
+   export PRTE_MCA_state_base_show_launch_progress=1
+
+Convert from boolean value to parameter for variable
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+.. code-block:: ini
+
+   # Old environment variable: (boolean value)
+   export OMPI_MCA_orte_xml_output=1
+
+   # New environment variable: (parameter value)
+   export PRTE_MCA_output=xml
+
+.. code-block:: ini
+
+   # Old environment variables: (boolean value)
+   export OMPI_MCA_orte_xml_output=1
+   export OMPI_MCA_orte_timestamp_output=1
+
+   # New environment variable: (parameter value)
+   export PRTE_MCA_output=xml,timestamp
+
+.. code-block:: ini
+
+   # Old environment variable: (boolean value)
+   export OMPI_MCA_rmaps_base_display_devel_map=1
+
+   # New environment variable: (parameter value)
+   export PRTE_MCA_display=map-devel
+
+.. code-block:: ini
+
+   # Old environment variables: (boolean value)
+   export OMPI_MCA_rmaps_base_display_devel_map=1
+   export OMPI_MCA_rmaps_base_report_bindings=1
+
+   # New environment variable: (parameter value)
+   export PRTE_MCA_display=map-devel,bind
+
+Convert from string value to parameter for variable
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+.. code-block:: ini
+
+   # Old environment variable: (string value)
+   export OMPI_MCA_orte_output_filename=output.txt
+
+   # New environment variable: (parameter value)
+   export PRTE_MCA_output=file=output.txt
+
+.. code-block:: ini
+
+   # Old environment variables: (boolean value)
+   export OMPI_MCA_orte_xml_output=1
+   export OMPI_MCA_orte_timestamp_output=1
+
+   # Old environment variable: (string value)
+   export OMPI_MCA_orte_output_filename=output.txt
+
+   # New environment variable: (parameter value)
+   export PRTE_MCA_output=xml,timestamp,file=output.txt
+
+.. code-block:: ini
+
+   # Old environment variable: (string value)
+   export OMPI_MCA_rmaps_base_display_topo_with_map=node
+
+   # New environment variable: (parameter value)
+   export PRTE_MCA_display=topo=node
+
+.. code-block:: ini
+
+   # Old environment variables: (boolean value)
+   export OMPI_MCA_rmaps_base_display_devel_map=1
+   export OMPI_MCA_rmaps_base_report_bindings=1
+
+   # Old environment variable: (string value)
+   export OMPI_MCA_rmaps_base_display_topo_with_map=node
+
+   # New environment variable: (parameter value)
+   export PRTE_MCA_display=map-devel,bind,topo=node
+
+Converting mapping parameters
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+Mapping parameters were previously prefixed with ``rmaps_base_`` or ``hwloc_base_``
+(and also the ``orte_rankfile`` parameter).  These have been updated
+to the ``rmaps_default_mapping_policy`` and ``hwloc_default_binding_policy``
+parameters to be more consistent and indicate that they are the *default*
+mapping for processes.  Some of the old parameters are now values for a
+new parameter and some are now suffixes, as shown in the examples below.
+
+The examples below show conversions from old boolean parameters to new
+parameter values:
+
+.. code-block:: ini
+
+   # Old environment variable: (boolean value)
+   export OMPI_MCA_rmaps_base_bycore=1
+
+   # New environment variable: (parameter value)
+   export PRTE_MCA_rmaps_default_mapping_policy=core
+
+.. code-block:: ini
+
+   # Old environment variable: (boolean value)
+   export OMPI_MCA_hwloc_base_bind_to_socket=1
+
+   # New environment variable: (parameter value)
+   export PRTE_MCA_hwloc_default_binding_policy=package
+
+
+The examples below show conversions from old parameters that have integer or
+string values to new parameter values with those same values:
+
+.. code-block:: ini
+
+   # Old environment variable: (string value)
+   export OMPI_MCA_hwloc_base_cpu_set=1,3,8
+
+   # New environment variable: (parameter value)
+   export PRTE_MCA_rmaps_default_mapping_policy=pe-list=1,3,8
+
+.. code-block:: ini
+
+   # Old environment variable: (integer value)
+   export OMPI_MCA_rmaps_ppr_n_persocket=4
+
+   # New environment variable: (parameter value)
+   export PRTE_MCA_rmaps_default_mapping_policy=ppr:4:package
+
+.. code-block:: ini
+
+   # Old environment variable: (string value)
+   export OMPI_MCA_orte_rankfile=rankfile.txt
+
+   # New environment variable: (parameter value)
+   export PRTE_MCA_rmaps_default_mapping_policy=rankfile:file=rankfile.txt
+
+The examples below show conversions from old parameters that map to suffixes
+for new parameter values:
+
+.. code-block:: ini
+
+   # Old environment variable: (boolean value)
+   export OMPI_MCA_hwloc_base_use_hwthreads_as_cpus=1
+
+   # New environment variable: (standalone suffix)
+   export PRTE_MCA_rmaps_default_mapping_policy=:hwtcpus
+
+.. code-block:: ini
+
+   # Old environment variable: (boolean value)
+   export OMPI_MCA_rmaps_base_oversubscribe=1
+
+   # New environment variable: (standalone suffix)
+   export PRTE_MCA_rmaps_default_mapping_policy=:oversubscribe
+
+The examples below show conversions from old parameters that map to suffixes
+combined with parameters that have values:
+
+.. code-block:: ini
+
+   # Old environment variable: (string value)
+   export OMPI_MCA_hwloc_base_cpu_set=1,3,8
+
+   # Old environment variable: (boolean value)
+   export OMPI_MCA_rmaps_base_oversubscribe=1
+
+   # New environment variable: (suffix on value)
+   export PRTE_MCA_rmaps_default_mapping_policy=pe-list=1,3,8:oversubscribe
+
+.. code-block:: ini
+
+   # Old environment variable: (integer value)
+   export OMPI_MCA_rmaps_ppr_n_persocket=4
+
+   # Old environment variable: (boolean value)
+   export OMPI_MCA_hwloc_base_use_hwthreads_as_cpus=1
+
+   # New environment variable: (suffix on value)
+   export PRTE_MCA_rmaps_default_mapping_policy=ppr:4:package:hwtcpus
+
+Multiple suffixes may be appended to a mapping value:
+
+.. code-block:: ini
+
+   # Old environment variable: (integer value)
+   export OMPI_MCA_rmaps_ppr_n_persocket=4
+
+   # Old environment variables: (boolean value)
+   export OMPI_MCA_hwloc_base_use_hwthreads_as_cpus=1
+   export OMPI_MCA_rmaps_base_oversubscribe=1
+
+   # New environment variable: (suffix on value)
+   export PRTE_MCA_rmaps_default_mapping_policy=ppr:4:package:hwtcpus:oversubscribe
+
diff --git a/docs/mpi-standard-4.1-apis.json b/docs/mpi-standard-4.1-apis.json
new file mode 100644
index 00000000000..1f422842bc4
--- /dev/null
+++ b/docs/mpi-standard-4.1-apis.json
@@ -0,0 +1,58961 @@
+{
+    "mpi_abort": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Abort",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator of \\MPI/ processes to abort",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "error code to return to invoking environment",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "errorcode",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_accumulate": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Accumulate",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "origin_addr",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of entries in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "origin_count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each entry",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "origin_datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of target",
+                "func_type": "",
+                "kind": "RANK_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_rank",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "displacement from start of window to beginning of target buffer",
+                "func_type": "",
+                "kind": "RMA_DISPLACEMENT_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_disp",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of entries in target buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each entry in target buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "accumulate operator",
+                "func_type": "",
+                "kind": "OPERATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "op",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_add_error_class": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Add_error_class",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "value for the new error class",
+                "func_type": "",
+                "kind": "ERROR_CLASS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "errorclass",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_add_error_code": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Add_error_code",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CLASS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "errorclass",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new error code to be associated with \\mpiarg{errorclass}",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "errorcode",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_add_error_string": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Add_error_string",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "error code or class",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "errorcode",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "text corresponding to \\mpiarg{errorcode}",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "string",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_aint_add": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Aint_add",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "base address",
+                "func_type": "",
+                "kind": "LOCATION_SMALL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "base",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "displacement",
+                "func_type": "",
+                "kind": "DISPLACEMENT",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "disp",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            }
+        ],
+        "return_kind": "LOCATION_SMALL"
+    },
+    "mpi_aint_diff": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Aint_diff",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "minuend address",
+                "func_type": "",
+                "kind": "LOCATION_SMALL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "addr1",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "subtrahend address",
+                "func_type": "",
+                "kind": "LOCATION_SMALL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "addr2",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            }
+        ],
+        "return_kind": "DISPLACEMENT"
+    },
+    "mpi_allgather": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Allgather",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in send buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements received from any \\MPI/ process",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of receive buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_allgather_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Allgather_init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in send buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements received from any \\MPI/ process",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of receive buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info argument",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_allgatherv": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Allgatherv",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in send buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "nonnegative integer array (of length group size) containing the number of elements that are received from each \\MPI/ process",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "recvcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "integer array (of length group size). Entry \\mpicode{i} specifies the displacement (relative to \\mpiarg{recvbuf}) at which to place the incoming data from \\MPI/ process \\mpicode{i}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "displs",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of receive buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_allgatherv_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Allgatherv_init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in send buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "nonnegative integer array (of length group size) containing the number of elements that are received from each \\MPI/ process",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "recvcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "integer array (of length group size). Entry \\mpicode{i} specifies the displacement (relative to \\mpiarg{recvbuf}) at which to place the incoming data from \\MPI/ process \\mpicode{i}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "displs",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of receive buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info argument",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_alloc_mem": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": "\\>INTERFACE MPI\\_ALLOC\\_MEM \\\\ \\>\\>SUBROUTINE MPI\\_ALLOC\\_MEM(SIZE, INFO, BASEPTR, IERROR) \\\\ \\>\\>\\>IMPORT :: MPI\\_ADDRESS\\_KIND \\\\ \\>\\>\\>INTEGER :: INFO, IERROR \\\\ \\>\\>\\>INTEGER(KIND=MPI\\_ADDRESS\\_KIND) :: SIZE, BASEPTR \\\\ \\>\\>END SUBROUTINE \\\\ \\>\\>SUBROUTINE MPI\\_ALLOC\\_MEM\\_CPTR(SIZE, INFO, BASEPTR, IERROR) \\\\ \\>\\>\\>USE, INTRINSIC :: ISO\\_C\\_BINDING, ONLY : C\\_PTR \\\\ \\>\\>\\>IMPORT :: MPI\\_ADDRESS\\_KIND \\\\ \\>\\>\\>INTEGER :: INFO, IERROR \\\\ \\>\\>\\>INTEGER(KIND=MPI\\_ADDRESS\\_KIND) :: SIZE \\\\ \\>\\>\\>TYPE(C\\_PTR) :: BASEPTR \\\\ \\>\\>END SUBROUTINE \\\\ \\>END INTERFACE",
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Alloc_mem",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "size of memory segment in bytes",
+                "func_type": "",
+                "kind": "ALLOC_MEM_NUM_BYTES",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "size",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "pointer to beginning of memory segment allocated",
+                "func_type": "",
+                "kind": "C_BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "baseptr",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_allreduce": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Allreduce",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "starting address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in send buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of elements of send buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "operation",
+                "func_type": "",
+                "kind": "OPERATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "op",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_allreduce_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Allreduce_init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "starting address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in send buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of elements of send buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "operation",
+                "func_type": "",
+                "kind": "OPERATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "op",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info argument",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_alltoall": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Alltoall",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements sent to each \\MPI/ process",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements received from any \\MPI/ process",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of receive buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_alltoall_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Alltoall_init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements sent to each \\MPI/ process",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements received from any \\MPI/ process",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of receive buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info argument",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_alltoallv": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Alltoallv",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "nonnegative integer array (of length group size) specifying the number of elements to send to each rank",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sendcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "integer array (of length group size). Entry \\mpicode{j} specifies the displacement (relative to \\mpiarg{sendbuf}) from which to take the outgoing data destined for \\MPI/ process \\mpicode{j}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sdispls",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "nonnegative integer array (of length group size) specifying the number of elements that can be received from each rank",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "recvcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "integer array (of length group size). Entry \\mpicode{i} specifies the displacement (relative to \\mpiarg{recvbuf}) at which to place the incoming data from \\MPI/ process \\mpicode{i}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "rdispls",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of receive buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_alltoallv_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Alltoallv_init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "nonnegative integer array (of length group size) specifying the number of elements to send to each \\MPI/ process",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sendcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "integer array (of length group size). Entry \\mpicode{j} specifies the displacement (relative to \\mpiarg{sendbuf}) from which to take the outgoing data destined for \\MPI/ process \\mpicode{j}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sdispls",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "nonnegative integer array (of length group size) specifying the number of elements that can be received from each \\MPI/ process",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "recvcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "integer array (of length group size). Entry \\mpicode{i} specifies the displacement (relative to \\mpiarg{recvbuf}) at which to place the incoming data from \\MPI/ process \\mpicode{i}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "rdispls",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of receive buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info argument",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_alltoallw": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Alltoallw",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "nonnegative integer array (of length group size) specifying the number of elements to send to each rank",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sendcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "integer array (of length group size). Entry \\mpicode{j} specifies the displacement in bytes (relative to \\mpiarg{sendbuf}) from which to take the outgoing data destined for \\MPI/ process \\mpicode{j}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sdispls",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "array of datatypes (of length group size). Entry \\mpicode{j} specifies the type of data to send to \\MPI/ process \\mpicode{j}",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sendtypes",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "nonnegative integer array (of length group size) specifying the number of elements that can be received from each rank",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "recvcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "integer array (of length group size). Entry \\mpicode{i} specifies the displacement in bytes (relative to \\mpiarg{recvbuf}) at which to place the incoming data from \\MPI/ process \\mpicode{i}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "rdispls",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "array of datatypes (of length group size). Entry \\mpicode{i} specifies the type of data received from \\MPI/ process \\mpicode{i}",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "recvtypes",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_alltoallw_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Alltoallw_init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "integer array (of length group size) specifying the number of elements to send to each \\MPI/ process",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sendcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "integer array (of length group size). Entry \\mpicode{j} specifies the displacement in bytes (relative to \\mpiarg{sendbuf}) from which to take the outgoing data destined for \\MPI/ process \\mpicode{j}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sdispls",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "array of datatypes (of length group size). Entry \\mpicode{j} specifies the type of data to send to \\MPI/ process \\mpicode{j}",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sendtypes",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "integer array (of length group size) specifying the number of elements that can be received from each \\MPI/ process",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "recvcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "integer array (of length group size). Entry \\mpicode{i} specifies the displacement in bytes (relative to \\mpiarg{recvbuf}) at which to place the incoming data from \\MPI/ process \\mpicode{i}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "rdispls",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "array of datatypes (of length group size). Entry \\mpicode{i} specifies the type of data received from \\MPI/ process \\mpicode{i}",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "recvtypes",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info argument",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_attr_delete": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": true
+        },
+        "name": "MPI_Attr_delete",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator to which attribute is attached",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "The key value of the deleted attribute",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_attr_get": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": true
+        },
+        "name": "MPI_Attr_get",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator to which attribute is attached",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "key value",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "attribute value, unless \\mpiarg{flag} \\mpicode{= false}",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL_10",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "attribute_val",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "\\mpicode{true} if an attribute value was extracted; \\mpicode{false} if no attribute is associated with the key",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_attr_put": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": true
+        },
+        "name": "MPI_Attr_put",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator to which attribute will be attached",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": false,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "key value, as returned by \\mpifunc{MPI_KEYVAL_CREATE}",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "attribute value",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL_10",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_barrier": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Barrier",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_barrier_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Barrier_init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info argument",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_bcast": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Bcast",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "starting address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "buffer",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of entries in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of the root",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "root",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_bcast_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Bcast_init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "starting address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "buffer",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of entries in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of broadcast root",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "root",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info argument",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_bsend": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Bsend",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "initial address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in send buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each send buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of destination",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "dest",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "message tag",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "tag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_bsend_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Bsend_init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "initial address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements sent",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "type of each element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of destination",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "dest",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "message tag",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "tag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_buffer_attach": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Buffer_attach",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "initial buffer address",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buffer",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "buffer size, in bytes",
+                "func_type": "",
+                "kind": "POLYNUM_BYTES_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "size",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_buffer_detach": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Buffer_detach",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "initial buffer address",
+                "func_type": "",
+                "kind": "C_BUFFER2",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "buffer_addr",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "buffer size, in bytes",
+                "func_type": "",
+                "kind": "POLYNUM_BYTES",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "size",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_buffer_flush": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Buffer_flush",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_buffer_iflush": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Buffer_iflush",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_cancel": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Cancel",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "request",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": true,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_cart_coords": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Cart_coords",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator with associated Cartesian topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of an \\MPI/ process within group of \\mpiarg{comm}",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "rank",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "length of vector  \\mpiarg{coords} in the calling program",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "maxdims",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "coordinates of the \\MPI/ process with the rank \\mpiarg{rank} in Cartesian structure",
+                "func_type": "",
+                "kind": "COORDINATE",
+                "large_only": false,
+                "length": "maxdims",
+                "lis_direction": "out",
+                "name": "coords",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_cart_create": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Cart_create",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "input communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm_old",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of dimensions of Cartesian grid",
+                "func_type": "",
+                "kind": "NUM_DIMS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "ndims",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "integer array of size \\mpiarg{ndims} specifying the number of processes in each dimension",
+                "func_type": "",
+                "kind": "DIMENSION",
+                "large_only": false,
+                "length": "ndims",
+                "lis_direction": "in",
+                "name": "dims",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "logical array of size \\mpiarg{ndims} specifying whether the grid is periodic (\\mpicode{true}) or not (\\mpicode{false}) in each dimension",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": "ndims",
+                "lis_direction": "in",
+                "name": "periods",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "ranks may be reordered (\\mpicode{true}) or not (\\mpicode{false})",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "reorder",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new communicator with associated Cartesian topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "comm_cart",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_cart_get": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Cart_get",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator with associated Cartesian topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "length of vectors \\mpiarg{dims}, \\mpiarg{periods}, and \\mpiarg{coords} in the calling program",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "maxdims",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of \\MPI/ processes for each Cartesian dimension",
+                "func_type": "",
+                "kind": "DIMENSION",
+                "large_only": false,
+                "length": "maxdims",
+                "lis_direction": "out",
+                "name": "dims",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "periodicity (\\mpicode{true}/\\mpicode{false}) for each Cartesian dimension",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": "maxdims",
+                "lis_direction": "out",
+                "name": "periods",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "coordinates of calling \\MPI/ process in Cartesian structure",
+                "func_type": "",
+                "kind": "COORDINATE",
+                "large_only": false,
+                "length": "maxdims",
+                "lis_direction": "out",
+                "name": "coords",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_cart_map": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Cart_map",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "input communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of dimensions of Cartesian structure",
+                "func_type": "",
+                "kind": "NUM_DIMS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "ndims",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "integer array of size \\mpiarg{ndims} specifying the number of processes in each coordinate direction",
+                "func_type": "",
+                "kind": "DIMENSION",
+                "large_only": false,
+                "length": "ndims",
+                "lis_direction": "in",
+                "name": "dims",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "logical array of size \\mpiarg{ndims} specifying the periodicity specification in each coordinate direction",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": "ndims",
+                "lis_direction": "in",
+                "name": "periods",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "reordered rank of the calling \\MPI/ process; \\mpiconst{MPI_UNDEFINED} if calling \\MPI/ process does not belong to grid",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newrank",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_cart_rank": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Cart_rank",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator with associated Cartesian topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "integer array (of size \\mpiarg{ndims}) specifying the Cartesian coordinates of an \\MPI/ process",
+                "func_type": "",
+                "kind": "COORDINATE",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "coords",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of specified \\MPI/ process within group of \\mpiarg{comm}",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "rank",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_cart_shift": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Cart_shift",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator with associated Cartesian topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "coordinate dimension of shift",
+                "func_type": "",
+                "kind": "INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "direction",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "displacement ($> 0$: upwards shift, $< 0$: downwards shift)",
+                "func_type": "",
+                "kind": "DIMENSION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "disp",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of source \\MPI/ process",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "rank_source",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of destination \\MPI/ process",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "rank_dest",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_cart_sub": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Cart_sub",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator with associated Cartesian topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "the \\mpicode{i}-th entry of \\mpiarg{remain_dims} specifies whether the \\mpicode{i}-th dimension is kept in the subgrid (\\mpicode{true}) or is dropped (\\mpicode{false})",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "remain_dims",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new communicator with associated Cartesian topology containing the subgrid that includes the calling \\MPI/ process",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newcomm",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_cartdim_get": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Cartdim_get",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator with associated Cartesian topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of dimensions of the Cartesian structure",
+                "func_type": "",
+                "kind": "NUM_DIMS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ndims",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_close_port": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Close_port",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "a port",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "port_name",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_accept": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_accept",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "port name",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "port_name",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "implementation-dependent information",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of root in \\mpiarg{comm}",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "root",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "intra-communicator over which call is collective",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "inter-communicator with client as remote group",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newcomm",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_attach_buffer": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_attach_buffer",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "initial buffer address",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buffer",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "buffer size, in bytes",
+                "func_type": "",
+                "kind": "POLYNUM_BYTES_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "size",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_c2f": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": true,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_c2f",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": false,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "F90_COMM"
+    },
+    "mpi_comm_call_errhandler": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_call_errhandler",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator with error handler",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "errorcode",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_compare": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_compare",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "first communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm1",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "second communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm2",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "result",
+                "func_type": "",
+                "kind": "COMM_COMPARISON",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "result",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_connect": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_connect",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "network address",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "port_name",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "implementation-dependent information",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of root in \\mpiarg{comm}",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "root",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "intra-communicator over which call is collective",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "inter-communicator with server as remote group",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newcomm",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_copy_attr_function": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": true,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_copy_attr_function",
+        "name_f90": "COMM_COPY_ATTR_FUNCTION",
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "oldcomm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm_keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val_in",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "attribute_val_out",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "ierror",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_create": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_create",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "group, which is a subset of the group of \\mpiarg{comm}",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "group",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newcomm",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_create_errhandler": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_create_errhandler",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "user defined error handling procedure",
+                "func_type": "MPI_Comm_errhandler_function",
+                "kind": "FUNCTION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm_errhandler_fn",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERRHANDLER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "errhandler",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_create_from_group": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_create_from_group",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "group",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "group",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "unique identifier for this operation",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "stringtag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "error handler to be attached to new intra-communicator",
+                "func_type": "",
+                "kind": "ERRHANDLER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "errhandler",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newcomm",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_create_group": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_create_group",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "intra-communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "group, which is a subset of the group of \\mpiarg{comm}",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "group",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "tag",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "tag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newcomm",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_create_keyval": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_create_keyval",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "copy callback function for \\mpiarg{comm_keyval}",
+                "func_type": "MPI_Comm_copy_attr_function",
+                "kind": "FUNCTION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm_copy_attr_fn",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "delete callback function for \\mpiarg{comm_keyval}",
+                "func_type": "MPI_Comm_delete_attr_function",
+                "kind": "FUNCTION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm_delete_attr_fn",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "key value for future access",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "comm_keyval",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "extra state for callback function",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_delete_attr": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_delete_attr",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator from which the attribute is deleted",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "key value",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm_keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_delete_attr_function": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": true,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_delete_attr_function",
+        "name_f90": "COMM_DELETE_ATTR_FUNCTION",
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm_keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "attribute_val",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "ierror",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_detach_buffer": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_detach_buffer",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "initial buffer address",
+                "func_type": "",
+                "kind": "C_BUFFER2",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "buffer_addr",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "buffer size, in bytes",
+                "func_type": "",
+                "kind": "POLYNUM_BYTES",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "size",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_disconnect": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_disconnect",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_dup": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_dup",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "copy of \\mpiarg{comm}",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newcomm",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_dup_fn": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": "MPI_Comm_copy_attr_function",
+            "proxy_render": false
+        },
+        "name": "MPI_COMM_DUP_FN",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "oldcomm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm_keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val_in",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val_out",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "ierror",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_dup_with_info": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_dup_with_info",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "copy of \\mpiarg{comm}",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newcomm",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_errhandler_function": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": true,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_errhandler_function",
+        "name_f90": "COMM_ERRHANDLER_FUNCTION",
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": true,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "error_code",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": true,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "VARARGS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "varargs",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            }
+        ],
+        "return_kind": "NOTHING"
+    },
+    "mpi_comm_f2c": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": true,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_f2c",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "F90_COMM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": false,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "COMMUNICATOR"
+    },
+    "mpi_comm_flush_buffer": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_flush_buffer",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_free": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_free",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator to be destroyed",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_free_keyval": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_free_keyval",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "key value",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "comm_keyval",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_get_attr": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_get_attr",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator to which the attribute is attached",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "key value",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm_keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "attribute value, unless \\mpiarg{flag}\\mpicode{ = false}",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "attribute_val",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "\\mpicode{false} if no attribute is associated with the key",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_get_errhandler": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_get_errhandler",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "error handler currently associated with communicator",
+                "func_type": "",
+                "kind": "ERRHANDLER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "errhandler",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_get_info": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_get_info",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator object",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "info_used",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_get_name": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_get_name",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator whose name is to be returned",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "the name previously stored on the communicator, or an empty string if no such name exists",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": "MPI_MAX_OBJECT_NAME",
+                "lis_direction": "out",
+                "name": "comm_name",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "length of returned name",
+                "func_type": "",
+                "kind": "STRING_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "resultlen",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_get_parent": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_get_parent",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "the parent communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "parent",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_group": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_group",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "group corresponding to \\mpiarg{comm}",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "group",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_idup": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_idup",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "copy of \\mpiarg{comm}",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newcomm",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_idup_with_info": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_idup_with_info",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "copy of \\mpiarg{comm}",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newcomm",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_iflush_buffer": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_iflush_buffer",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_join": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_join",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "socket file descriptor",
+                "func_type": "",
+                "kind": "FILE_DESCRIPTOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "fd",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new inter-communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "intercomm",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_null_copy_fn": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": "MPI_Comm_copy_attr_function",
+            "proxy_render": false
+        },
+        "name": "MPI_COMM_NULL_COPY_FN",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "oldcomm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm_keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val_in",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val_out",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "ierror",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_null_delete_fn": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": "MPI_Comm_delete_attr_function",
+            "proxy_render": false
+        },
+        "name": "MPI_COMM_NULL_DELETE_FN",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm_keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "ierror",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_rank": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_rank",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of the calling \\MPI/ process in group of \\mpiarg{comm}",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "rank",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_remote_group": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_remote_group",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "inter-communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "remote group corresponding to \\mpiarg{comm}",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "group",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_remote_size": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_remote_size",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "inter-communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of \\MPI/ processes in the remote group of \\mpiarg{comm}",
+                "func_type": "",
+                "kind": "COMM_SIZE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "size",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_set_attr": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_set_attr",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator to which attribute will be attached",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "key value",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm_keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "attribute value",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_set_errhandler": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_set_errhandler",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new error handler for communicator",
+                "func_type": "",
+                "kind": "ERRHANDLER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "errhandler",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_set_info": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_set_info",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_set_name": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_set_name",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator whose identifier is to be set",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "the character string that is remembered as the name",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm_name",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_size": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_size",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of \\MPI/ processes in the group of \\mpiarg{comm}",
+                "func_type": "",
+                "kind": "COMM_SIZE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "size",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_spawn": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_spawn",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "name of program to be spawned",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "command",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "arguments to \\mpiarg{command}",
+                "func_type": "",
+                "kind": "STRING_ARRAY",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "argv",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "maximum number of processes to start",
+                "func_type": "",
+                "kind": "COMM_SIZE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "maxprocs",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "a set of key-value pairs telling the runtime system where and how to start the processes",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of process in which previous arguments are examined",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "root",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "intra-communicator containing group of spawning processes",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "inter-communicator between original group and the newly spawned group",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "intercomm",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "one code per process",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": "",
+                "lis_direction": "out",
+                "name": "array_of_errcodes",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_spawn_multiple": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_spawn_multiple",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of commands",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH_PI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "programs to be executed",
+                "func_type": "",
+                "kind": "STRING_ARRAY",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "array_of_commands",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "arguments for \\mpiarg{commands}",
+                "func_type": "",
+                "kind": "STRING_2DARRAY",
+                "large_only": false,
+                "length": "count",
+                "lis_direction": "in",
+                "name": "array_of_argv",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "hidden",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "maximum number of processes to start for each command",
+                "func_type": "",
+                "kind": "COMM_SIZE",
+                "large_only": false,
+                "length": "count",
+                "lis_direction": "in",
+                "name": "array_of_maxprocs",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "hidden",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "info objects telling the runtime system where and how to start processes",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": "count",
+                "lis_direction": "in",
+                "name": "array_of_info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of process in which previous arguments are examined",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "root",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "intra-communicator containing group of spawning processes",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "inter-communicator between original group and the newly spawned group",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "intercomm",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "one error code per process",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": "",
+                "lis_direction": "out",
+                "name": "array_of_errcodes",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_split": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_split",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "control of subset assignment",
+                "func_type": "",
+                "kind": "COLOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "color",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "control of rank assignment",
+                "func_type": "",
+                "kind": "KEY",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "key",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newcomm",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_split_type": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_split_type",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "type of processes to be grouped together",
+                "func_type": "",
+                "kind": "SPLIT_TYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "split_type",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "control of rank assignment",
+                "func_type": "",
+                "kind": "KEY",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "key",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info argument",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newcomm",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_comm_test_inter": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Comm_test_inter",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "\\mpicode{true} if \\mpiarg{comm} is an inter-communicator",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_compare_and_swap": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Compare_and_swap",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "origin_addr",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "initial address of compare buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "compare_addr",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "initial address of result buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "result_addr",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of the element in all buffers",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of target",
+                "func_type": "",
+                "kind": "RANK_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_rank",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "displacement from start of window to beginning of target buffer",
+                "func_type": "",
+                "kind": "RMA_DISPLACEMENT_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_disp",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_conversion_fn_null": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": true,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": false,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": "MPI_Datarep_conversion_function",
+            "proxy_render": false
+        },
+        "name": "MPI_CONVERSION_FN_NULL",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "C_BUFFER3",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "userbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "C_BUFFER3",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "filebuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "OFFSET",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "position",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "ierror",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_copy_function": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": true,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": true
+        },
+        "name": "MPI_Copy_function",
+        "name_f90": "COPY_FUNCTION",
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "oldcomm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EXTRA_STATE2",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL_10",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val_in",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL_10",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "attribute_val_out",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "ierr",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_datarep_conversion_function": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": true,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Datarep_conversion_function",
+        "name_f90": "DATAREP_CONVERSION_FUNCTION",
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "C_BUFFER3",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "userbuf",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "C_BUFFER3",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "filebuf",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "OFFSET",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "position",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "ierror",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_datarep_extent_function": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": true,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Datarep_extent_function",
+        "name_f90": "DATAREP_EXTENT_FUNCTION",
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "DISPOFFSET_SMALL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "extent",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "ierror",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_delete_function": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": true,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": true
+        },
+        "name": "MPI_Delete_function",
+        "name_f90": "DELETE_FUNCTION",
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL_10",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EXTRA_STATE2",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "ierr",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_dims_create": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Dims_create",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of nodes in a grid",
+                "func_type": "",
+                "kind": "COMM_SIZE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "nnodes",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of Cartesian dimensions",
+                "func_type": "",
+                "kind": "NUM_DIMS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "ndims",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "integer array of size \\mpiarg{ndims} specifying the number of nodes in each dimension",
+                "func_type": "",
+                "kind": "DIMENSION",
+                "large_only": false,
+                "length": "ndims",
+                "lis_direction": "inout",
+                "name": "dims",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_dist_graph_create": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Dist_graph_create",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "input communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm_old",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of source nodes for which this \\MPI/ process specifies edges",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "n",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "array containing the \\mpiarg{n} source nodes for which this \\MPI/ process specifies edges",
+                "func_type": "",
+                "kind": "RANK_NNI",
+                "large_only": false,
+                "length": "n",
+                "lis_direction": "in",
+                "name": "sources",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "array specifying the number of destinations for each source node in the source node array",
+                "func_type": "",
+                "kind": "DEGREE",
+                "large_only": false,
+                "length": "n",
+                "lis_direction": "in",
+                "name": "degrees",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "destination nodes for the source nodes in the source node array",
+                "func_type": "",
+                "kind": "RANK_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "destinations",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "weights for source to destination edges",
+                "func_type": "",
+                "kind": "WEIGHT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "weights",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "hints on optimization and interpretation of weights",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "ranks may be reordered (\\mpicode{true}) or not~(\\mbox{\\mpicode{false}})",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "reorder",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new communicator with associated distributed graph topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "comm_dist_graph",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_dist_graph_create_adjacent": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Dist_graph_create_adjacent",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "input communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm_old",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "size of \\mpiarg{sources} and \\mpiarg{sourceweights} arrays",
+                "func_type": "",
+                "kind": "DEGREE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "indegree",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "ranks of \\MPI/ processes for which the calling process is a destination",
+                "func_type": "",
+                "kind": "RANK_NNI",
+                "large_only": false,
+                "length": "indegree",
+                "lis_direction": "in",
+                "name": "sources",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "weights of the edges into the calling \\MPI/ process",
+                "func_type": "",
+                "kind": "WEIGHT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sourceweights",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "size of \\mpiarg{destinations} and \\mpiarg{destweights} arrays",
+                "func_type": "",
+                "kind": "DEGREE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "outdegree",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "ranks of \\MPI/ processes for which the calling \\MPI/ process is a source",
+                "func_type": "",
+                "kind": "RANK_NNI",
+                "large_only": false,
+                "length": "outdegree",
+                "lis_direction": "in",
+                "name": "destinations",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "weights of the edges out of the calling \\MPI/ process",
+                "func_type": "",
+                "kind": "WEIGHT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "destweights",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "hints on optimization and interpretation of weights",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "ranks may be reordered (\\mpicode{true}) or not~(\\mbox{\\mpicode{false}})",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "reorder",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new communicator with associated distributed graph topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "comm_dist_graph",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_dist_graph_neighbors": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Dist_graph_neighbors",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator with associated distributed graph topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "size of sources and sourceweights arrays",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "maxindegree",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "ranks of \\MPI/ processes for which the calling \\MPI/ process is a destination",
+                "func_type": "",
+                "kind": "RANK_NNI",
+                "large_only": false,
+                "length": "maxindegree",
+                "lis_direction": "out",
+                "name": "sources",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "weights of the edges into the calling \\MPI/ process",
+                "func_type": "",
+                "kind": "WEIGHT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "out",
+                "name": "sourceweights",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "size of destinations and destweights arrays",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "maxoutdegree",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "ranks of \\MPI/ processes for which the calling \\MPI/ process is a source",
+                "func_type": "",
+                "kind": "RANK_NNI",
+                "large_only": false,
+                "length": "maxoutdegree",
+                "lis_direction": "out",
+                "name": "destinations",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "weights of the edges out of the calling \\MPI/ process",
+                "func_type": "",
+                "kind": "WEIGHT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "out",
+                "name": "destweights",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_dist_graph_neighbors_count": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Dist_graph_neighbors_count",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator with associated distributed graph topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of edges into this \\MPI/ process",
+                "func_type": "",
+                "kind": "DEGREE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "indegree",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of edges out of this \\MPI/ process",
+                "func_type": "",
+                "kind": "DEGREE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "outdegree",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "\\mpicode{false} if \\mpiconst{MPI_UNWEIGHTED} was supplied during creation, \\mpicode{true} otherwise",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "weighted",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_dup_fn": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": "MPI_Copy_function",
+            "proxy_render": false
+        },
+        "name": "MPI_DUP_FN",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "oldcomm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EXTRA_STATE2",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL_10",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val_in",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL_10",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val_out",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "ierr",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_errhandler_c2f": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": true,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Errhandler_c2f",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERRHANDLER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "errhandler",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": false,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "F90_ERRHANDLER"
+    },
+    "mpi_errhandler_f2c": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": true,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Errhandler_f2c",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "F90_ERRHANDLER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "errhandler",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": false,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERRHANDLER"
+    },
+    "mpi_errhandler_free": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Errhandler_free",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERRHANDLER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "errhandler",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_error_class": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Error_class",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "Error code returned by an \\MPI/ routine",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "errorcode",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "Error class associated with \\mpiarg{errorcode}",
+                "func_type": "",
+                "kind": "ERROR_CLASS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "errorclass",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_error_string": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Error_string",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "Error code returned by an \\MPI/ routine",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "errorcode",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "Text that corresponds to the \\mpiarg{errorcode}",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": "MPI_MAX_ERROR_STRING",
+                "lis_direction": "out",
+                "name": "string",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "Length (in printable characters) of the result returned in \\mpiarg{string}",
+                "func_type": "",
+                "kind": "STRING_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "resultlen",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_exscan": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Exscan",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "starting address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in input buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of elements of input buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "operation",
+                "func_type": "",
+                "kind": "OPERATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "op",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "intra-communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_exscan_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Exscan_init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "starting address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in input buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of elements of input buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "operation",
+                "func_type": "",
+                "kind": "OPERATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "op",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "intra-communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info argument",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_f_sync_reg": {
+        "attributes": {
+            "c_expressible": false,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_F_sync_reg",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_fetch_and_op": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Fetch_and_op",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "origin_addr",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "initial address of result buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "result_addr",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of the entry in origin, result, and target buffers",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of target",
+                "func_type": "",
+                "kind": "RANK_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_rank",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "displacement from start of window to beginning of target buffer",
+                "func_type": "",
+                "kind": "RMA_DISPLACEMENT_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_disp",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "accumulate operator",
+                "func_type": "",
+                "kind": "OPERATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "op",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_c2f": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": true,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_c2f",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "file",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": false,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "F90_FILE"
+    },
+    "mpi_file_call_errhandler": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_call_errhandler",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file with error handler",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "errorcode",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_close": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_close",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_create_errhandler": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_create_errhandler",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "user defined error handling procedure",
+                "func_type": "MPI_File_errhandler_function",
+                "kind": "FUNCTION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "file_errhandler_fn",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERRHANDLER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "errhandler",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_delete": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_delete",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "name of file to delete",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "filename",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_errhandler_function": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": true,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_errhandler_function",
+        "name_f90": "FILE_ERRHANDLER_FUNCTION",
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "file",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": true,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "error_code",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": true,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "VARARGS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "varargs",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            }
+        ],
+        "return_kind": "NOTHING"
+    },
+    "mpi_file_f2c": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": true,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_f2c",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "F90_FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "file",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": false,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "FILE"
+    },
+    "mpi_file_get_amode": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_get_amode",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file access mode used to open the file",
+                "func_type": "",
+                "kind": "ACCESS_MODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "amode",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_get_atomicity": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_get_atomicity",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "\\mpicode{true} if atomic mode, \\mpicode{false} if nonatomic mode",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_get_byte_offset": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_get_byte_offset",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "offset",
+                "func_type": "",
+                "kind": "OFFSET",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "offset",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "absolute byte position of offset",
+                "func_type": "",
+                "kind": "OFFSET",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "disp",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_get_errhandler": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_get_errhandler",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "file",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "error handler currently associated with file",
+                "func_type": "",
+                "kind": "ERRHANDLER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "errhandler",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_get_group": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_get_group",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "group that opened the file",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "group",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_get_info": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_get_info",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "info_used",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_get_position": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_get_position",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "offset of individual pointer",
+                "func_type": "",
+                "kind": "OFFSET",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "offset",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_get_position_shared": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_get_position_shared",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "offset of shared pointer",
+                "func_type": "",
+                "kind": "OFFSET",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "offset",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_get_size": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_get_size",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "size of the file in bytes",
+                "func_type": "",
+                "kind": "OFFSET",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "size",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_get_type_extent": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_get_type_extent",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype extent",
+                "func_type": "",
+                "kind": "POLYDISPOFFSET",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "extent",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_get_view": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_get_view",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "displacement",
+                "func_type": "",
+                "kind": "OFFSET",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "disp",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "elementary datatype",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "etype",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "filetype",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "filetype",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "data representation",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "datarep",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_iread": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_iread",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "request object",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_iread_all": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_iread_all",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "request object",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_iread_at": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_iread_at",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file offset",
+                "func_type": "",
+                "kind": "OFFSET",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "offset",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "request object",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_iread_at_all": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_iread_at_all",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file offset",
+                "func_type": "",
+                "kind": "OFFSET",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "offset",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "request object",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_iread_shared": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_iread_shared",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "request object",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_iwrite": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_iwrite",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "request object",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_iwrite_all": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_iwrite_all",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "request object",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_iwrite_at": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_iwrite_at",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file offset",
+                "func_type": "",
+                "kind": "OFFSET",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "offset",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "request object",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_iwrite_at_all": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_iwrite_at_all",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file offset",
+                "func_type": "",
+                "kind": "OFFSET",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "offset",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "request object",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_iwrite_shared": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_iwrite_shared",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "request object",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_open": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_open",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "name of file to open",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "filename",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file access mode",
+                "func_type": "",
+                "kind": "ACCESS_MODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "amode",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_preallocate": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_preallocate",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "size to preallocate file",
+                "func_type": "",
+                "kind": "OFFSET",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "size",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_read": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_read",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status object",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_read_all": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_read_all",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status object",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_read_all_begin": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_read_all_begin",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_read_all_end": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_read_all_end",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status object",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_read_at": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_read_at",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file offset",
+                "func_type": "",
+                "kind": "OFFSET",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "offset",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status object",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_read_at_all": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_read_at_all",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file offset",
+                "func_type": "",
+                "kind": "OFFSET",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "offset",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status object",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_read_at_all_begin": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_read_at_all_begin",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file offset",
+                "func_type": "",
+                "kind": "OFFSET",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "offset",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_read_at_all_end": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_read_at_all_end",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status object",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_read_ordered": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_read_ordered",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status object",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_read_ordered_begin": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_read_ordered_begin",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_read_ordered_end": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_read_ordered_end",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status object",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_read_shared": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_read_shared",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status object",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_seek": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_seek",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file offset",
+                "func_type": "",
+                "kind": "OFFSET",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "offset",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "update mode",
+                "func_type": "",
+                "kind": "UPDATE_MODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "whence",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_seek_shared": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_seek_shared",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file offset",
+                "func_type": "",
+                "kind": "OFFSET",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "offset",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "update mode",
+                "func_type": "",
+                "kind": "UPDATE_MODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "whence",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_set_atomicity": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_set_atomicity",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "\\mpicode{true} to set atomic mode, \\mpicode{false} to set nonatomic mode",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_set_errhandler": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_set_errhandler",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "file",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new error handler for file",
+                "func_type": "",
+                "kind": "ERRHANDLER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "errhandler",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_set_info": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_set_info",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_set_size": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_set_size",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "size to truncate or expand file",
+                "func_type": "",
+                "kind": "OFFSET",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "size",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_set_view": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_set_view",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "displacement",
+                "func_type": "",
+                "kind": "OFFSET",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "disp",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "elementary datatype",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "etype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "filetype",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "filetype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "data representation",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datarep",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_sync": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_sync",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_write": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_write",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status object",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_write_all": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_write_all",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status object",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_write_all_begin": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_write_all_begin",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_write_all_end": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_write_all_end",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status object",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_write_at": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_write_at",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file offset",
+                "func_type": "",
+                "kind": "OFFSET",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "offset",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status object",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_write_at_all": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_write_at_all",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file offset",
+                "func_type": "",
+                "kind": "OFFSET",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "offset",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status object",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_write_at_all_begin": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_write_at_all_begin",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file offset",
+                "func_type": "",
+                "kind": "OFFSET",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "offset",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_write_at_all_end": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_write_at_all_end",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status object",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_write_ordered": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_write_ordered",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status object",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_write_ordered_begin": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_write_ordered_begin",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_write_ordered_end": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_write_ordered_end",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status object",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_file_write_shared": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_File_write_shared",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "file handle",
+                "func_type": "",
+                "kind": "FILE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "fh",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status object",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_finalize": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Finalize",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_finalized": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Finalized",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "true if \\mpifunc{MPI\\_FINALIZE} has been called and false otherwise.",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_free_mem": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Free_mem",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "initial address of memory segment allocated by \\mpifunc{MPI_ALLOC_MEM}",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "base",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_gather": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Gather",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in send buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements for any single receive",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of recv buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of receiving \\MPI/ process",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "root",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_gather_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Gather_init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in send buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": true,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements for any single receive",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of recv buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of receiving \\MPI/ process",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "root",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info argument",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_gatherv": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Gatherv",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in send buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "nonnegative integer array (of length group size) containing the number of elements that are received from each \\MPI/ process",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "recvcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "integer array (of length group size). Entry \\mpicode{i} specifies the displacement relative to \\mpiarg{recvbuf} at which to place the incoming data from \\MPI/ process \\mpicode{i}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "displs",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of recv buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of receiving \\MPI/ process",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "root",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_gatherv_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Gatherv_init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in send buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": true,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "nonnegative integer array (of length group size) containing the number of elements that are received from each \\MPI/ process",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "recvcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "integer array (of length group size). Entry \\mpicode{i} specifies the displacement relative to \\mpiarg{recvbuf} at which to place the incoming data from \\MPI/ process \\mpicode{i}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "displs",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of recv buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of receiving \\MPI/ process",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "root",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info argument",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_get": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Get",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "initial address of origin buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "origin_addr",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of entries in origin buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "origin_count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each entry in origin buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "origin_datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of target",
+                "func_type": "",
+                "kind": "RANK_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_rank",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "displacement from window start to the beginning of the target buffer",
+                "func_type": "",
+                "kind": "RMA_DISPLACEMENT_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_disp",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of entries in target buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each entry in target buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "window used for communication",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_get_accumulate": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Get_accumulate",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "origin_addr",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of entries in origin buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "origin_count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each entry in origin buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "origin_datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "initial address of result buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "result_addr",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of entries in result buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "result_count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each entry in result buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "result_datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of target",
+                "func_type": "",
+                "kind": "RANK_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_rank",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "displacement from start of window to beginning of target buffer",
+                "func_type": "",
+                "kind": "RMA_DISPLACEMENT_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_disp",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of entries in target buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each entry in target buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "accumulate operator",
+                "func_type": "",
+                "kind": "OPERATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "op",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_get_address": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Get_address",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "location in caller memory",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "location",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "address of location",
+                "func_type": "",
+                "kind": "DISPLACEMENT",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "address",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_get_count": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Get_count",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "return status of receive operation",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "status",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each receive buffer entry",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of received entries",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "count",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_get_elements": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Get_elements",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "return status of receive operation",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "status",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype used by receive operation",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of received basic elements",
+                "func_type": "",
+                "kind": "POLYDTYPE_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "count",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_get_elements_x": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Get_elements_x",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "return status of receive operation",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "status",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype used by receive operation",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of received basic elements",
+                "func_type": "",
+                "kind": "NUM_BYTES",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "count",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_get_hw_resource_info": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Get_hw_resource_info",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object created",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "hw_info",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_get_library_version": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Get_library_version",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "version number",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": "MPI_MAX_LIBRARY_VERSION_STRING",
+                "lis_direction": "out",
+                "name": "version",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "Length (in printable characters) of the result returned in \\mpiarg{version}",
+                "func_type": "",
+                "kind": "STRING_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "resultlen",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_get_processor_name": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Get_processor_name",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "A unique specifier for the actual (as opposed to virtual) node.",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": "MPI_MAX_PROCESSOR_NAME",
+                "lis_direction": "out",
+                "name": "name",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "Length (in printable characters) of the result returned in \\mpiarg{name}",
+                "func_type": "",
+                "kind": "STRING_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "resultlen",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_get_version": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Get_version",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "version number",
+                "func_type": "",
+                "kind": "VERSION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "version",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "subversion number",
+                "func_type": "",
+                "kind": "VERSION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "subversion",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_graph_create": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Graph_create",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "input communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm_old",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of nodes in graph",
+                "func_type": "",
+                "kind": "COMM_SIZE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "nnodes",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "array of integers describing node degrees (see below)",
+                "func_type": "",
+                "kind": "INDEX",
+                "large_only": false,
+                "length": "nnodes",
+                "lis_direction": "in",
+                "name": "index",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "array of integers describing graph edges (see below)",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "edges",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "ranks may be reordered (\\mpicode{true}) or not (\\mpicode{false})",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "reorder",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new communicator with associated graph topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "comm_graph",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_graph_get": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Graph_get",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator with associated graph topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "length of vector \\mpiarg{index} in the calling program",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "maxindex",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "length of vector \\mpiarg{edges} in the calling program",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "maxedges",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "array of integers containing the graph structure (for details see the definition of \\mpifunc{MPI_GRAPH_CREATE})",
+                "func_type": "",
+                "kind": "INDEX",
+                "large_only": false,
+                "length": "maxindex",
+                "lis_direction": "out",
+                "name": "index",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "array of integers containing the graph structure",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": "maxedges",
+                "lis_direction": "out",
+                "name": "edges",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_graph_map": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Graph_map",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "input communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of graph nodes",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "nnodes",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "integer array specifying the graph structure (for details see the definition of \\mpifunc{MPI_GRAPH_CREATE})",
+                "func_type": "",
+                "kind": "INDEX",
+                "large_only": false,
+                "length": "nnodes",
+                "lis_direction": "in",
+                "name": "index",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "integer array specifying the graph structure",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "edges",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "reordered rank of the calling \\MPI/ process; \\mpiconst{MPI_UNDEFINED} if the calling \\MPI/ process does not belong to graph",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newrank",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_graph_neighbors": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Graph_neighbors",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator with associated graph topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of \\MPI/ process in group of \\mpiarg{comm}",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "rank",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "size of array \\mpiarg{neighbors}",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "maxneighbors",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "ranks of \\MPI/ processes that are neighbors to specified \\MPI/ process",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": "maxneighbors",
+                "lis_direction": "out",
+                "name": "neighbors",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_graph_neighbors_count": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Graph_neighbors_count",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator with associated graph topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of \\MPI/ process in group of \\mpiarg{comm}",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "rank",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of neighbors of specified \\MPI/ process",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "nneighbors",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_graphdims_get": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Graphdims_get",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator with associated graph topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of nodes in graph (same as number of \\MPI/ processes in the group of \\mpiarg{comm})",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "nnodes",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of edges in graph",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "nedges",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_grequest_cancel_function": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": true,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Grequest_cancel_function",
+        "name_f90": "GREQUEST_CANCEL_FUNCTION",
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "complete",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "ierror",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_grequest_complete": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Grequest_complete",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "generalized request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "request",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_grequest_free_function": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": true,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Grequest_free_function",
+        "name_f90": "GREQUEST_FREE_FUNCTION",
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "ierror",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_grequest_query_function": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": true,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Grequest_query_function",
+        "name_f90": "GREQUEST_QUERY_FUNCTION",
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "ierror",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_grequest_start": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Grequest_start",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "callback function invoked when request status is queried",
+                "func_type": "MPI_Grequest_query_function",
+                "kind": "FUNCTION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "query_fn",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "callback function invoked when request is freed",
+                "func_type": "MPI_Grequest_free_function",
+                "kind": "FUNCTION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "free_fn",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "callback function invoked when request is cancelled",
+                "func_type": "MPI_Grequest_cancel_function",
+                "kind": "FUNCTION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "cancel_fn",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "extra state",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "generalized request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_group_c2f": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": true,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Group_c2f",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "group",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": false,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "F90_GROUP"
+    },
+    "mpi_group_compare": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Group_compare",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "first group",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "group1",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "second group",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "group2",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "result",
+                "func_type": "",
+                "kind": "GROUP_COMPARISON",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "result",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_group_difference": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Group_difference",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "first group",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "group1",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "second group",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "group2",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "difference group",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newgroup",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_group_excl": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Group_excl",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "group",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "group",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in array \\mpiarg{ranks}",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "n",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "array of integer ranks of \\MPI/ processes in \\mpiarg{group} not to appear in \\mpiarg{newgroup}",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": "n",
+                "lis_direction": "in",
+                "name": "ranks",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new group derived from above, preserving the order defined by \\mpiarg{group}",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newgroup",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_group_f2c": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": true,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Group_f2c",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "F90_GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "group",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": false,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "GROUP"
+    },
+    "mpi_group_free": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Group_free",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "group",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "group",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_group_from_session_pset": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Group_from_session_pset",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "session",
+                "func_type": "",
+                "kind": "SESSION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "session",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "name of process set to use to create the new group",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "pset_name",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new group derived from supplied session and process set",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newgroup",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_group_incl": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Group_incl",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "group",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "group",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in array \\mpiarg{ranks} (and size of \\mpiarg{newgroup})",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "n",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "ranks of processes in \\mpiarg{group} to appear in \\mpiarg{newgroup}",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": "n",
+                "lis_direction": "in",
+                "name": "ranks",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new group derived from above, in the order defined by \\mpiarg{ranks}",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newgroup",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_group_intersection": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Group_intersection",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "first group",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "group1",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "second group",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "group2",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "intersection group",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newgroup",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_group_range_excl": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Group_range_excl",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "group",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "group",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of triplets in array \\mpiarg{ranges}",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "n",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "a one-dimensional array of integer triplets, of the form (first rank, last rank, stride) indicating ranks in \\mpiarg{group} of \\MPI/ processes to be excluded from the output group \\mpiarg{newgroup}",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": [
+                    "n",
+                    "3"
+                ],
+                "lis_direction": "in",
+                "name": "ranges",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new group derived from above, preserving the order in \\mpiarg{group}",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newgroup",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_group_range_incl": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Group_range_incl",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "group",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "group",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of triplets in array \\mpiarg{ranges}",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "n",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "a one-dimensional array of integer triplets, of the form (first rank, last rank, stride) indicating ranks in \\mpiarg{group} of \\MPI/ processes to be included in \\mpiarg{newgroup}",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": [
+                    "n",
+                    "3"
+                ],
+                "lis_direction": "in",
+                "name": "ranges",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new group derived from above, in the order defined by \\mpiarg{ranges}",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newgroup",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_group_rank": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Group_rank",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "group",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "group",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of the calling \\MPI/ process in group, or \\mpiconst{MPI_UNDEFINED} if the \\MPI/ process is not a member",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "rank",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_group_size": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Group_size",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "group",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "group",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of \\MPI/ processes in the group",
+                "func_type": "",
+                "kind": "COMM_SIZE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "size",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_group_translate_ranks": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Group_translate_ranks",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "group1",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "group1",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in \\mpiarg{ranks1} and \\mpiarg{ranks2} arrays",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "n",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "array of zero or more valid ranks in group1",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": "n",
+                "lis_direction": "in",
+                "name": "ranks1",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "group2",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "group2",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "array of corresponding ranks in group2, \\mpiconst{MPI_UNDEFINED} when no correspondence exists.",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": "n",
+                "lis_direction": "out",
+                "name": "ranks2",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_group_union": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Group_union",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "first group",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "group1",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "second group",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "group2",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "union group",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newgroup",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_iallgather": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Iallgather",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in send buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements received from any \\MPI/ process",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of receive buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_iallgatherv": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Iallgatherv",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in send buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "nonnegative integer array (of length group size) containing the number of elements that are received from each \\MPI/ process",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "recvcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "integer array (of length group size). Entry \\mpicode{i} specifies the displacement (relative to \\mpiarg{recvbuf}) at which to place the incoming data from \\MPI/ process \\mpicode{i}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "displs",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of receive buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_iallreduce": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Iallreduce",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "starting address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in send buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of elements of send buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "operation",
+                "func_type": "",
+                "kind": "OPERATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "op",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_ialltoall": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Ialltoall",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements sent to each \\MPI/ process",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements received from any \\MPI/ process",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of receive buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_ialltoallv": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Ialltoallv",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "nonnegative integer array (of length group size) specifying the number of elements to send to each \\MPI/ process",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sendcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "integer array (of length group size). Entry \\mpicode{j} specifies the displacement (relative to \\mpiarg{sendbuf}) from which to take the outgoing data destined for \\MPI/ process \\mpicode{j}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sdispls",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "nonnegative integer array (of length group size) specifying the number of elements that can be received from each \\MPI/ process",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "recvcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "integer array (of length group size). Entry \\mpicode{i} specifies the displacement (relative to \\mpiarg{recvbuf}) at which to place the incoming data from \\MPI/ process \\mpicode{i}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "rdispls",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of receive buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_ialltoallw": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Ialltoallw",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "integer array (of length group size) specifying the number of elements to send to each \\MPI/ process",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sendcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "integer array (of length group size). Entry \\mpicode{j} specifies the displacement in bytes (relative to \\mpiarg{sendbuf}) from which to take the outgoing data destined for \\MPI/ process \\mpicode{j}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sdispls",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "array of datatypes (of length group size). Entry \\mpicode{j} specifies the type of data to send to \\MPI/ process \\mpicode{j}",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sendtypes",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "integer array (of length group size) specifying the number of elements that can be received from each \\MPI/ process",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "recvcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "integer array (of length group size). Entry \\mpicode{i} specifies the displacement in bytes (relative to \\mpiarg{recvbuf}) at which to place the incoming data from \\MPI/ process \\mpicode{i}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "rdispls",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "array of datatypes (of length group size). Entry \\mpicode{i} specifies the type of data received from \\MPI/ process \\mpicode{i}",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "recvtypes",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_ibarrier": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Ibarrier",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_ibcast": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Ibcast",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "starting address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "buffer",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of entries in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of broadcast root",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "root",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_ibsend": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Ibsend",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "initial address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in send buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each send buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of destination",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "dest",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "message tag",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "tag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_iexscan": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Iexscan",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "starting address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in input buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of elements of input buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "operation",
+                "func_type": "",
+                "kind": "OPERATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "op",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "intra-communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_igather": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Igather",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in send buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements for any single receive",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of recv buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of receiving \\MPI/ process",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "root",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_igatherv": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Igatherv",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in send buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": true,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "nonnegative integer array (of length group size) containing the number of elements that are received from each \\MPI/ process",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "recvcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "integer array (of length group size). Entry \\mpicode{i} specifies the displacement relative to \\mpiarg{recvbuf} at which to place the incoming data from \\MPI/ process \\mpicode{i}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "displs",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of recv buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of receiving \\MPI/ process",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "root",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_improbe": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Improbe",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of source or \\mpiconst{MPI_ANY_SOURCE}",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "source",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "message tag or \\mpiconst{MPI_ANY_TAG}",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "tag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "\\mpicode{true} if there is a matching message that can be received",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "returned message",
+                "func_type": "",
+                "kind": "MESSAGE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "message",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_imrecv": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Imrecv",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "initial address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in receive buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each receive buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "message",
+                "func_type": "",
+                "kind": "MESSAGE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "message",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_ineighbor_allgather": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Ineighbor_allgather",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements sent to each neighbor",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "starting address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements received from each neighbor",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of receive buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator with associated virtual topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_ineighbor_allgatherv": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Ineighbor_allgatherv",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements sent to each neighbor",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "starting address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "nonnegative integer array (of length indegree) containing the number of elements that are received from each neighbor",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "recvcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "integer array (of length indegree). Entry \\mpicode{i} specifies the displacement (relative to \\mpiarg{recvbuf}) at which to place the incoming data from neighbor \\mpicode{i}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "displs",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of receive buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator with associated virtual topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_ineighbor_alltoall": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Ineighbor_alltoall",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements sent to each neighbor",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "starting address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements received from each neighbor",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of receive buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator with associated virtual topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_ineighbor_alltoallv": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Ineighbor_alltoallv",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "nonnegative integer array (of length outdegree) specifying the number of elements to send to each neighbor",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sendcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "integer array (of length outdegree). Entry \\mpicode{j} specifies the displacement (relative to \\mpiarg{sendbuf}) from which send the outgoing data to neighbor \\mpicode{j}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sdispls",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "starting address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "nonnegative integer array (of length indegree) specifying the number of elements that are received from each neighbor",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "recvcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "integer array (of length indegree). Entry \\mpicode{i} specifies the displacement (relative to \\mpiarg{recvbuf}) at which to place the incoming data from neighbor \\mpicode{i}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "rdispls",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of receive buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator with associated virtual topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_ineighbor_alltoallw": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Ineighbor_alltoallw",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "nonnegative integer array (of length outdegree) specifying the number of elements to send to each neighbor",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sendcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "integer array (of length outdegree). Entry \\mpicode{j} specifies the displacement in bytes (relative to \\mpiarg{sendbuf}) from which to take the outgoing data destined for neighbor \\mpicode{j}",
+                "func_type": "",
+                "kind": "DISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sdispls",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "array of datatypes (of length outdegree). Entry \\mpicode{j} specifies the type of data to send to neighbor \\mpicode{j}",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sendtypes",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "starting address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "nonnegative integer array (of length indegree) specifying the number of elements that are received from each neighbor",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "recvcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "integer array (of length indegree). Entry \\mpicode{i} specifies the displacement in bytes (relative to \\mpiarg{recvbuf}) at which to place the incoming data from neighbor \\mpicode{i}",
+                "func_type": "",
+                "kind": "DISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "rdispls",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "array of datatypes (of length indegree). Entry \\mpicode{i} specifies the type of data received from neighbor \\mpicode{i}",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "recvtypes",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator with associated virtual topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_info_c2f": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": true,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Info_c2f",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": false,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "F90_INFO"
+    },
+    "mpi_info_create": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Info_create",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object created",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "info",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_info_create_env": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Info_create_env",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ARGUMENT_COUNT",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "argc",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": false,
+                "root_only": false,
+                "suppress": "lis_parameter f08_parameter f90_parameter"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ARGUMENT_LIST",
+                "large_only": false,
+                "length": "argc",
+                "lis_direction": "inout",
+                "name": "argv",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": false,
+                "root_only": false,
+                "suppress": "lis_parameter f08_parameter f90_parameter"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "info",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_info_delete": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Info_delete",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "key",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "key",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_info_dup": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Info_dup",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object created",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newinfo",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_info_f2c": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": true,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Info_f2c",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "F90_INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": false,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "INFO"
+    },
+    "mpi_info_free": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Info_free",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "info",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_info_get": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Info_get",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "key",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "key",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "length of value associated with \\mpiarg{key}",
+                "func_type": "",
+                "kind": "INFO_VALUE_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "valuelen",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "value",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": "valuelen",
+                "lis_direction": "out",
+                "name": "value",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "\\mpicode{true} if \\mpiarg{key} defined, \\mpicode{false} if not",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_info_get_nkeys": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Info_get_nkeys",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of defined keys",
+                "func_type": "",
+                "kind": "KEY_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "nkeys",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_info_get_nthkey": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Info_get_nthkey",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "key number",
+                "func_type": "",
+                "kind": "KEY_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "n",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "key",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "key",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_info_get_string": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Info_get_string",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "key",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "key",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "length of buffer",
+                "func_type": "",
+                "kind": "INFO_VALUE_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "buflen",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "value",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "value",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "\\mpicode{true} if \\mpiarg{key} defined, \\mpicode{false} if not",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_info_get_valuelen": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Info_get_valuelen",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "key",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "key",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "length of value associated with \\mpiarg{key}",
+                "func_type": "",
+                "kind": "INFO_VALUE_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "valuelen",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "\\mpicode{true} if \\mpiarg{key} defined, \\mpicode{false} if not",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_info_set": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Info_set",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "key",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "key",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "value",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "value",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ARGUMENT_COUNT",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "argc",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_parameter f08_parameter f90_parameter"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ARGUMENT_LIST",
+                "large_only": false,
+                "length": "argc",
+                "lis_direction": "inout",
+                "name": "argv",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": true,
+                "root_only": false,
+                "suppress": "lis_parameter f08_parameter f90_parameter"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_init_thread": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Init_thread",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ARGUMENT_COUNT",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "argc",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_parameter f08_parameter f90_parameter"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ARGUMENT_LIST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "argv",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_parameter f08_parameter f90_parameter"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "desired level of thread support",
+                "func_type": "",
+                "kind": "THREAD_LEVEL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "required",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "provided level of thread support",
+                "func_type": "",
+                "kind": "THREAD_LEVEL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "provided",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_initialized": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Initialized",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "Flag is true if \\mpifunc{MPI_INIT} or \\mpifunc{MPI_INIT_THREAD} has been called and false otherwise",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_intercomm_create": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Intercomm_create",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "local intra-communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "local_comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of local group leader in \\mpiarg{local_comm}",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "local_leader",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "``peer'' communicator; significant only at the \\mpiarg{local_leader}",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "peer_comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of remote group leader in \\mpiarg{peer_comm}; significant only at the \\mpiarg{local_leader}",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "remote_leader",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "tag",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "tag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new inter-communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newintercomm",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_intercomm_create_from_groups": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Intercomm_create_from_groups",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "local group",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "local_group",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of local group leader in \\mpiarg{local\\_group}",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "local_leader",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "remote group, significant only at \\mpiarg{local\\_leader}",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "remote_group",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of remote group leader in \\mpiarg{remote\\_group}, significant only at \\mpiarg{local\\_leader}",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "remote_leader",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "unique idenitifier for this operation",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "stringtag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "error handler to be attached to new inter-communicator",
+                "func_type": "",
+                "kind": "ERRHANDLER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "errhandler",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new inter-communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newintercomm",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_intercomm_merge": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Intercomm_merge",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "inter-communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "intercomm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "ordering of the local and remote groups in the new intra-communicator",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "high",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new intra-communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newintracomm",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_iprobe": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Iprobe",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of source or \\mpiconst{MPI_ANY_SOURCE}",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "source",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "message tag or \\mpiconst{MPI_ANY_TAG}",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "tag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "\\mpicode{true} if there is a matching message that can be received",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_irecv": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Irecv",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "initial address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in receive buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each receive buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of source or \\mpiconst{MPI_ANY_SOURCE}",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "source",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "message tag or \\mpiconst{MPI_ANY_TAG}",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "tag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_ireduce": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Ireduce",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": true,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in send buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of elements of send buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "reduce operation",
+                "func_type": "",
+                "kind": "OPERATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "op",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of the root",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "root",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_ireduce_scatter": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Ireduce_scatter",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "starting address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "nonnegative integer array specifying the number of elements in result distributed to each \\MPI/ process. This array must be identical on all calling \\MPI/ processes.",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "recvcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of elements of input buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "operation",
+                "func_type": "",
+                "kind": "OPERATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "op",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_ireduce_scatter_block": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Ireduce_scatter_block",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "starting address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "element count per block",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of elements of send and receive buffers",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "operation",
+                "func_type": "",
+                "kind": "OPERATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "op",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_irsend": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Irsend",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "initial address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in send buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each send buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of destination",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "dest",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "message tag",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "tag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_is_thread_main": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Is_thread_main",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "true if calling thread is main thread, false otherwise",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_iscan": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Iscan",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "starting address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in input buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of elements of input buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "operation",
+                "func_type": "",
+                "kind": "OPERATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "op",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_iscatter": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Iscatter",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements sent to each \\MPI/ process",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in receive buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of receive buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of sending \\MPI/ process",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "root",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_iscatterv": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Iscatterv",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "nonnegative integer array (of length group size) specifying the number of elements to send to each \\MPI/ process",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sendcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "integer array (of length group size). Entry \\mpicode{i} specifies the displacement (relative to \\mpiarg{sendbuf}) from which to take the outgoing data to \\MPI/ process \\mpicode{i}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "displs",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in receive buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of receive buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of sending \\MPI/ process",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "root",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_isend": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Isend",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "initial address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in send buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each send buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of destination",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "dest",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "message tag",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "tag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_isendrecv": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Isendrecv",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "initial address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in send buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each send buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of destination",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "dest",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "send tag",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "initial address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in receive buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each receive buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of source or \\mpiconst{MPI_ANY_SOURCE}",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "source",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "receive tag or \\mpiconst{MPI_ANY_TAG}",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_isendrecv_replace": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Isendrecv_replace",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "initial address of send and receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in send and receive buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "type of elements in send and receive buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of destination",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "dest",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "send message tag",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of source or \\mpiconst{MPI_ANY_SOURCE}",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "source",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "receive message tag or \\mpiconst{MPI_ANY_TAG}",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_issend": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Issend",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "initial address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in send buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each send buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of destination",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "dest",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "message tag",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "tag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_keyval_create": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": true
+        },
+        "name": "MPI_Keyval_create",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "Copy callback function for \\mpiarg{keyval}",
+                "func_type": "MPI_Copy_function",
+                "kind": "FUNCTION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "copy_fn",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "Delete callback function for \\mpiarg{keyval}",
+                "func_type": "MPI_Delete_function",
+                "kind": "FUNCTION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "delete_fn",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "key value for future access",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "keyval",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "Extra state for callback functions",
+                "func_type": "",
+                "kind": "EXTRA_STATE2",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_keyval_free": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": true
+        },
+        "name": "MPI_Keyval_free",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "Frees the integer key value",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "keyval",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_lookup_name": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Lookup_name",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "a service name",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "service_name",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "implementation-specific information",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "a port name",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": "MPI_MAX_PORT_NAME",
+                "lis_direction": "out",
+                "name": "port_name",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_message_c2f": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": true,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Message_c2f",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "MESSAGE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "message",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": false,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "F90_MESSAGE"
+    },
+    "mpi_message_f2c": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": true,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Message_f2c",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "F90_MESSAGE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "message",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": false,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "MESSAGE"
+    },
+    "mpi_mprobe": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Mprobe",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of source or \\mpiconst{MPI_ANY_SOURCE}",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "source",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "message tag or \\mpiconst{MPI_ANY_TAG}",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "tag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "returned message",
+                "func_type": "",
+                "kind": "MESSAGE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "message",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_mrecv": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Mrecv",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "initial address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in receive buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each receive buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "message",
+                "func_type": "",
+                "kind": "MESSAGE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "message",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_neighbor_allgather": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Neighbor_allgather",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements sent to each neighbor",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "starting address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements received from each neighbor",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of receive buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator with associated virtual topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_neighbor_allgather_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Neighbor_allgather_init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements sent to each neighbor",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "starting address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements received from each neighbor",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of receive buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator with associated virtual topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info argument",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_neighbor_allgatherv": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Neighbor_allgatherv",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements sent to each neighbor",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "starting address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "nonnegative integer array (of length indegree) containing the number of elements that are received from each neighbor",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "recvcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "integer array (of length indegree). Entry \\mpicode{i} specifies the displacement (relative to \\mpiarg{recvbuf}) at which to place the incoming data from neighbor \\mpicode{i}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "displs",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of receive buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator with associated virtual topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_neighbor_allgatherv_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Neighbor_allgatherv_init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements sent to each neighbor",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "starting address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "nonnegative integer array (of length indegree) containing the number of elements that are received from each neighbor",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "recvcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "integer array (of length indegree). Entry \\mpicode{i} specifies the displacement (relative to \\mpiarg{recvbuf}) at which to place the incoming data from neighbor \\mpicode{i}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "displs",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of receive buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator with associated virtual topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info argument",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_neighbor_alltoall": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Neighbor_alltoall",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements sent to each neighbor",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "starting address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements received from each neighbor",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of receive buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator with associated virtual topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_neighbor_alltoall_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Neighbor_alltoall_init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements sent to each neighbor",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "starting address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements received from each neighbor",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of receive buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator with associated virtual topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info argument",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_neighbor_alltoallv": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Neighbor_alltoallv",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "nonnegative integer array (of length outdegree) specifying the number of elements to send to each neighbor",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sendcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "integer array (of length outdegree). Entry \\mpicode{j} specifies the displacement (relative to \\mpiarg{sendbuf}) from which to send the outgoing data to neighbor \\mpicode{j}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sdispls",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "starting address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "nonnegative integer array (of length indegree) specifying the number of elements that are received from each neighbor",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "recvcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "integer array (of length indegree). Entry \\mpicode{i} specifies the displacement (relative to \\mpiarg{recvbuf}) at which to place the incoming data from neighbor \\mpicode{i}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "rdispls",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of receive buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator with associated virtual topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_neighbor_alltoallv_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Neighbor_alltoallv_init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "nonnegative integer array (of length outdegree) specifying the number of elements to send to each neighbor",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sendcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "integer array (of length outdegree). Entry \\mpicode{j} specifies the displacement (relative to \\mpiarg{sendbuf}) from which send the outgoing data to neighbor \\mpicode{j}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sdispls",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "starting address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "nonnegative integer array (of length indegree) specifying the number of elements that are received from each neighbor",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "recvcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "integer array (of length indegree). Entry \\mpicode{i} specifies the displacement (relative to \\mpiarg{recvbuf}) at which to place the incoming data from neighbor \\mpicode{i}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "rdispls",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of receive buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator with associated virtual topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info argument",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_neighbor_alltoallw": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Neighbor_alltoallw",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "nonnegative integer array (of length outdegree) specifying the number of elements to send to each neighbor",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sendcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "integer array (of length outdegree). Entry \\mpicode{j} specifies the displacement in bytes (relative to \\mpiarg{sendbuf}) from which to take the outgoing data destined for neighbor \\mpicode{j}",
+                "func_type": "",
+                "kind": "DISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sdispls",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "array of datatypes (of length outdegree). Entry \\mpicode{j} specifies the type of data to send to neighbor \\mpicode{j}",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sendtypes",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "starting address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "nonnegative integer array (of length indegree) specifying the number of elements that are received from each neighbor",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "recvcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "integer array (of length indegree). Entry \\mpicode{i} specifies the displacement in bytes (relative to \\mpiarg{recvbuf}) at which to place the incoming data from neighbor \\mpicode{i}",
+                "func_type": "",
+                "kind": "DISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "rdispls",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "array of datatypes (of length indegree). Entry \\mpicode{i} specifies the type of data received from neighbor \\mpicode{i}",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "recvtypes",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator with associated virtual topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_neighbor_alltoallw_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Neighbor_alltoallw_init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "nonnegative integer array (of length outdegree) specifying the number of elements to send to each neighbor",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sendcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "integer array (of length outdegree). Entry \\mpicode{j} specifies the displacement in bytes (relative to \\mpiarg{sendbuf}) from which to take the outgoing data destined for neighbor \\mpicode{j}",
+                "func_type": "",
+                "kind": "DISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sdispls",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "array of datatypes (of length outdegree). Entry \\mpicode{j} specifies the type of data to send to neighbor \\mpicode{j}",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sendtypes",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "starting address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "nonnegative integer array (of length indegree) specifying the number of elements that are received from each neighbor",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "recvcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "integer array (of length indegree). Entry \\mpicode{i} specifies the displacement in bytes (relative to \\mpiarg{recvbuf}) at which to place the incoming data from neighbor \\mpicode{i}",
+                "func_type": "",
+                "kind": "DISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "rdispls",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "array of datatypes (of length indegree). Entry \\mpicode{i} specifies the type of data received from neighbor \\mpicode{i}",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "recvtypes",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator with associated virtual topology",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info argument",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_null_copy_fn": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": "MPI_Copy_function",
+            "proxy_render": false
+        },
+        "name": "MPI_NULL_COPY_FN",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "oldcomm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EXTRA_STATE2",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL_10",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val_in",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL_10",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val_out",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "ierr",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_null_delete_fn": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": "MPI_Delete_function",
+            "proxy_render": false
+        },
+        "name": "MPI_NULL_DELETE_FN",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL_10",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EXTRA_STATE2",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "ierror",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_op_c2f": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": true,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Op_c2f",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "OPERATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "op",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": false,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "F90_OP"
+    },
+    "mpi_op_commutative": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Op_commutative",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "operation",
+                "func_type": "",
+                "kind": "OPERATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "op",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "\\mpicode{true} if \\mpiarg{op} is commutative, \\mpicode{false} otherwise",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "commute",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_op_create": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": false,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Op_create",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "user defined function",
+                "func_type": "MPI_User_function",
+                "kind": "POLYFUNCTION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "user_fn",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "\\mpicode{true} if commutative; \\mpicode{false} otherwise.",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "commute",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "operation",
+                "func_type": "",
+                "kind": "OPERATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "op",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_op_f2c": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": true,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Op_f2c",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "F90_OP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "op",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": false,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "OPERATION"
+    },
+    "mpi_op_free": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Op_free",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "operation",
+                "func_type": "",
+                "kind": "OPERATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "op",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_open_port": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Open_port",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "implementation-specific information on how to establish an address",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "newly established port",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": "MPI_MAX_PORT_NAME",
+                "lis_direction": "out",
+                "name": "port_name",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_pack": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Pack",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "input buffer start",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "inbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of input data items",
+                "func_type": "",
+                "kind": "POLYDTYPE_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "incount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each input data item",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "output buffer start",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "outbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "output buffer size, in bytes",
+                "func_type": "",
+                "kind": "POLYNUM_BYTES_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "outsize",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "current position in buffer, in bytes",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT_COUNT",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "position",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator for packed message",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_pack_external": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Pack_external",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "data representation",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "datarep",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "input buffer start",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "inbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of input data items",
+                "func_type": "",
+                "kind": "POLYDTYPE_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "incount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each input data item",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "output buffer start",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "outbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "output buffer size, in bytes",
+                "func_type": "",
+                "kind": "POLYDTYPE_PACK_SIZE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "outsize",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "current position in buffer, in bytes",
+                "func_type": "",
+                "kind": "POLYLOCATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "position",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_pack_external_size": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Pack_external_size",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "data representation",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "datarep",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of input data items",
+                "func_type": "",
+                "kind": "POLYDTYPE_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "incount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each input data item",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "output buffer size, in bytes",
+                "func_type": "",
+                "kind": "POLYDTYPE_PACK_SIZE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "size",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_pack_size": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Pack_size",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "count argument to packing call",
+                "func_type": "",
+                "kind": "POLYDTYPE_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "incount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype argument to packing call",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator argument to packing call",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "upper bound on size of packed message, in bytes",
+                "func_type": "",
+                "kind": "POLYNUM_BYTES_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "size",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_parrived": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Parrived",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "partitioned communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "request",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "partition to be tested",
+                "func_type": "",
+                "kind": "PARTITION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "partition",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "\\mpicode{true} if operation completed on the specified partition, \\mpicode{false} if not",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_pcontrol": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Pcontrol",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "Profiling level",
+                "func_type": "",
+                "kind": "PROFILE_LEVEL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "level",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "VARARGS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "varargs",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_pready": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Pready",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "partition to mark ready for transfer",
+                "func_type": "",
+                "kind": "PARTITION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "partition",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "partitioned communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "request",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_pready_list": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Pready_list",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "list length",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "length",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": "true",
+                "desc": "array of partitions",
+                "func_type": "",
+                "kind": "PARTITION",
+                "large_only": false,
+                "length": "length",
+                "lis_direction": "in",
+                "name": "array_of_partitions",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "partitioned communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "request",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_pready_range": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Pready_range",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "lowest partition ready for transfer",
+                "func_type": "",
+                "kind": "PARTITION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "partition_low",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "highest partition ready for transfer",
+                "func_type": "",
+                "kind": "PARTITION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "partition_high",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "partitioned communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "request",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_precv_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Precv_init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "initial address of recv buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of partitions",
+                "func_type": "",
+                "kind": "PARTITION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "partitions",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements received per partition",
+                "func_type": "",
+                "kind": "XFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "type of each element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of source",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "source",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "message tag",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "tag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info argument",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_probe": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Probe",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of source or \\mpiconst{MPI_ANY_SOURCE}",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "source",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "message tag or \\mpiconst{MPI_ANY_TAG}",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "tag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_psend_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Psend_init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": "true",
+                "desc": "initial address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of partitions",
+                "func_type": "",
+                "kind": "PARTITION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "partitions",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements sent per partition",
+                "func_type": "",
+                "kind": "XFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "type of each element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of destination",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "dest",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "message tag",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "tag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info argument",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_publish_name": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Publish_name",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "a service name to associate with the port",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "service_name",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "implementation-specific information",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "a port name",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "port_name",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_put": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Put",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "initial address of origin buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "origin_addr",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of entries in origin buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "origin_count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each entry in origin buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "origin_datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of target",
+                "func_type": "",
+                "kind": "RANK_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_rank",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "displacement from start of window to target buffer",
+                "func_type": "",
+                "kind": "RMA_DISPLACEMENT_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_disp",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of entries in target buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each entry in target buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "window used for communication",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_query_thread": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Query_thread",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "provided level of thread support",
+                "func_type": "",
+                "kind": "THREAD_LEVEL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "provided",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_raccumulate": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Raccumulate",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "origin_addr",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of entries in buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "origin_count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each entry in origin buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "origin_datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of target",
+                "func_type": "",
+                "kind": "RANK_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_rank",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "displacement from start of window to beginning of target buffer",
+                "func_type": "",
+                "kind": "RMA_DISPLACEMENT_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_disp",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of entries in target buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each entry in target buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "accumulate operator",
+                "func_type": "",
+                "kind": "OPERATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "op",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "\\RMA/ request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_recv": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Recv",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "initial address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in receive buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each receive buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of source or \\mpiconst{MPI_ANY_SOURCE}",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "source",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "message tag or \\mpiconst{MPI_ANY_TAG}",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "tag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_recv_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Recv_init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "initial address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements received",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "type of each element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of source or \\mpiconst{MPI_ANY_SOURCE}",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "source",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "message tag or \\mpiconst{MPI_ANY_TAG}",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "tag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_reduce": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Reduce",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in send buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of elements of send buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "reduce operation",
+                "func_type": "",
+                "kind": "OPERATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "op",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of the root",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "root",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_reduce_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Reduce_init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": true,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in send buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of elements of send buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "reduce operation",
+                "func_type": "",
+                "kind": "OPERATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "op",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of the root",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "root",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info argument",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_reduce_local": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Reduce_local",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "input buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "inbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "combined input and output buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "inoutbuf",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in \\mpiarg{inbuf} and \\mpiarg{inoutbuf} buffers",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of elements of \\mpiarg{inbuf} and \\mpiarg{inoutbuf} buffers",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "operation",
+                "func_type": "",
+                "kind": "OPERATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "op",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_reduce_scatter": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Reduce_scatter",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "starting address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "nonnegative integer array (of length group size) specifying the number of elements of the result distributed to each \\MPI/ process.",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "recvcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of elements of send and receive buffers",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "operation",
+                "func_type": "",
+                "kind": "OPERATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "op",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_reduce_scatter_block": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Reduce_scatter_block",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "starting address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "element count per block",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of elements of send and receive buffers",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "operation",
+                "func_type": "",
+                "kind": "OPERATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "op",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_reduce_scatter_block_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Reduce_scatter_block_init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "starting address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "element count per block",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of elements of send and receive buffers",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "operation",
+                "func_type": "",
+                "kind": "OPERATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "op",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info argument",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_reduce_scatter_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Reduce_scatter_init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "starting address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "nonnegative integer array specifying the number of elements in result distributed to each \\MPI/ process. This array must be identical on all calling \\MPI/ processes.",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "recvcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of elements of input buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "operation",
+                "func_type": "",
+                "kind": "OPERATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "op",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info argument",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_register_datarep": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": false,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Register_datarep",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "data representation identifier",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datarep",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "function invoked to convert from file representation to native representation",
+                "func_type": "MPI_Datarep_conversion_function",
+                "kind": "POLYFUNCTION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "read_conversion_fn",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "function invoked to convert from native representation to file representation",
+                "func_type": "MPI_Datarep_conversion_function",
+                "kind": "POLYFUNCTION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "write_conversion_fn",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "function invoked to get the extent of a datatype as represented in the file",
+                "func_type": "MPI_Datarep_extent_function",
+                "kind": "FUNCTION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "dtype_file_extent_fn",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "extra state",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_remove_error_class": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Remove_error_class",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "value for the error class to remove",
+                "func_type": "",
+                "kind": "ERROR_CLASS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "errorclass",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_remove_error_code": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Remove_error_code",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "error code to be removed",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "errorcode",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_remove_error_string": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Remove_error_string",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "error code or class",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "errorcode",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_request_c2f": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": true,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Request_c2f",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "request",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": false,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "F90_REQUEST"
+    },
+    "mpi_request_f2c": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": true,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Request_f2c",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "F90_REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "request",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": false,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "REQUEST"
+    },
+    "mpi_request_free": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Request_free",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "request",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_request_get_status": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Request_get_status",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "request",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "boolean flag, same as from \\mpifunc{MPI_TEST}",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status object if flag is true",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_request_get_status_all": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Request_get_status_all",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "list length",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "array of requests",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": "count",
+                "lis_direction": "in",
+                "name": "array_of_requests",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "true if all of the operations are complete",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "array of status objects",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "out",
+                "name": "array_of_statuses",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": false,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_request_get_status_any": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Request_get_status_any",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "list length",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "array of requests",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": "count",
+                "lis_direction": "in",
+                "name": "array_of_requests",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "index of operation that completed or \\mpiconst{MPI_UNDEFINED} if none completed",
+                "func_type": "",
+                "kind": "INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "index",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "\\mpicode{true} if one of the operations is complete",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status object if flag is true",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_request_get_status_some": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Request_get_status_some",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "length of array_of_requests",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "incount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "array of requests",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": "incount",
+                "lis_direction": "in",
+                "name": "array_of_requests",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of completed requests",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "outcount",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "array of indices of operations that completed",
+                "func_type": "",
+                "kind": "INDEX",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "out",
+                "name": "array_of_indices",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "array of status objects for operations that completed",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "out",
+                "name": "array_of_statuses",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": false,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_rget": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Rget",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "initial address of origin buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "origin_addr",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of entries in origin buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "origin_count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each entry in origin buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "origin_datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of target",
+                "func_type": "",
+                "kind": "RANK_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_rank",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "displacement from window start to the beginning of the target buffer",
+                "func_type": "",
+                "kind": "RMA_DISPLACEMENT_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_disp",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of entries in target buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each entry in target buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "window used for communication",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "\\RMA/ request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_rget_accumulate": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Rget_accumulate",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "initial address of buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "origin_addr",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of entries in origin buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "origin_count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each entry in origin buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "origin_datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "initial address of result buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "result_addr",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of entries in result buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "result_count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of entries in result buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "result_datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of target",
+                "func_type": "",
+                "kind": "RANK_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_rank",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "displacement from start of window to beginning of target buffer",
+                "func_type": "",
+                "kind": "RMA_DISPLACEMENT_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_disp",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of entries in target buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each entry in target buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "accumulate operator",
+                "func_type": "",
+                "kind": "OPERATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "op",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "\\RMA/ request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_rput": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Rput",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "initial address of origin buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "origin_addr",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of entries in origin buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "origin_count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each entry in origin buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "origin_datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of target",
+                "func_type": "",
+                "kind": "RANK_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_rank",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "displacement from start of window to target buffer",
+                "func_type": "",
+                "kind": "RMA_DISPLACEMENT_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_disp",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of entries in target buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each entry in target buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "target_datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "window used for communication",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "\\RMA/ request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_rsend": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Rsend",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "initial address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in send buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each send buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of destination",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "dest",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "message tag",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "tag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_rsend_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Rsend_init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "initial address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements sent",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "type of each element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of destination",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "dest",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "message tag",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "tag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_scan": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Scan",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "starting address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in input buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of elements of input buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "operation",
+                "func_type": "",
+                "kind": "OPERATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "op",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_scan_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Scan_init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "starting address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "starting address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in input buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of elements of input buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "operation",
+                "func_type": "",
+                "kind": "OPERATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "op",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info argument",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_scatter": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Scatter",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements sent to each \\MPI/ process",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in receive buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of receive buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of sending \\MPI/ process",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "root",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_scatter_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Scatter_init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements sent to each \\MPI/ process",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in receive buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of receive buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of sending \\MPI/ process",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "root",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info argument",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_scatterv": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Scatterv",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "nonnegative integer array (of length group size) specifying the number of elements to send to each rank",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sendcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "integer array (of length group size). Entry \\mpicode{i} specifies the displacement (relative to \\mpiarg{sendbuf}) from which to take the outgoing data to \\MPI/ process \\mpicode{i}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "displs",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in receive buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of receive buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of sending \\MPI/ process",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "root",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_scatterv_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Scatterv_init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "nonnegative integer array (of length group size) specifying the number of elements to send to each \\MPI/ process",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "sendcounts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "integer array (of length group size). Entry \\mpicode{i} specifies the displacement (relative to \\mpiarg{sendbuf}) from which to take the outgoing data to \\MPI/ process \\mpicode{i}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "displs",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": "lis_paren"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of send buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": true,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": true,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in receive buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of receive buffer elements",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of sending \\MPI/ process",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "root",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info argument",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communication request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_send": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Send",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "initial address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in send buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each send buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of destination",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "dest",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "message tag",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "tag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_send_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Send_init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "initial address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements sent",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "type of each element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of destination",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "dest",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "message tag",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "tag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_sendrecv": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Sendrecv",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "initial address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in send buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "type of elements in send buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of destination",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "dest",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "send tag",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "initial address of receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "recvbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in receive buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "type of elements receive buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of source or \\mpiconst{MPI_ANY_SOURCE}",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "source",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "receive tag or \\mpiconst{MPI_ANY_TAG}",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_sendrecv_replace": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Sendrecv_replace",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "initial address of send and receive buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in send and receive buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "type of elements in send and receive buffer",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of destination",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "dest",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "send message tag",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "sendtag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of source or \\mpiconst{MPI_ANY_SOURCE}",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "source",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "receive message tag or \\mpiconst{MPI_ANY_TAG}",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "recvtag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_session_attach_buffer": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Session_attach_buffer",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "SESSION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "session",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "initial buffer address",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buffer",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "buffer size, in bytes",
+                "func_type": "",
+                "kind": "POLYNUM_BYTES_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "size",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_session_c2f": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": true,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Session_c2f",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "SESSION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "session",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": false,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "F90_SESSION"
+    },
+    "mpi_session_call_errhandler": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Session_call_errhandler",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "session with error handler",
+                "func_type": "",
+                "kind": "SESSION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "session",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "errorcode",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_session_create_errhandler": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Session_create_errhandler",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "user defined error handling procedure",
+                "func_type": "MPI_Session_errhandler_function",
+                "kind": "FUNCTION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "session_errhandler_fn",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "tilda"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERRHANDLER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "errhandler",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_session_detach_buffer": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Session_detach_buffer",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "SESSION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "session",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "initial buffer address",
+                "func_type": "",
+                "kind": "C_BUFFER2",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "buffer_addr",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "buffer size, in bytes",
+                "func_type": "",
+                "kind": "POLYNUM_BYTES",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "size",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_session_errhandler_function": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": true,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Session_errhandler_function",
+        "name_f90": "SESSION_ERRHANDLER_FUNCTION",
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "SESSION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "session",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": true,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "error_code",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": true,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "VARARGS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "varargs",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            }
+        ],
+        "return_kind": "NOTHING"
+    },
+    "mpi_session_f2c": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": true,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Session_f2c",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "F90_SESSION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "session",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": false,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "SESSION"
+    },
+    "mpi_session_finalize": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Session_finalize",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "session to be finalized",
+                "func_type": "",
+                "kind": "SESSION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "session",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_session_flush_buffer": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Session_flush_buffer",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "SESSION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "session",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_session_get_errhandler": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Session_get_errhandler",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "session",
+                "func_type": "",
+                "kind": "SESSION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "session",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "error handler currently associated with session",
+                "func_type": "",
+                "kind": "ERRHANDLER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "errhandler",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_session_get_info": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Session_get_info",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "session",
+                "func_type": "",
+                "kind": "SESSION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "session",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "see explanation below",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "info_used",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_session_get_nth_pset": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Session_get_nth_pset",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "session",
+                "func_type": "",
+                "kind": "SESSION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "session",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "index of the desired process set name",
+                "func_type": "",
+                "kind": "INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "n",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "length of the pset\\_name argument",
+                "func_type": "",
+                "kind": "STRING_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "pset_len",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "name of the \\mpiarg{n}th process set",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "pset_name",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_session_get_num_psets": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Session_get_num_psets",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "session",
+                "func_type": "",
+                "kind": "SESSION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "session",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of available process sets",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "npset_names",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_session_get_pset_info": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Session_get_pset_info",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "session",
+                "func_type": "",
+                "kind": "SESSION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "session",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "name of process set",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "pset_name",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object containing information about the given process set",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "info",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_session_iflush_buffer": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Session_iflush_buffer",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "SESSION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "session",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_session_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Session_init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object to specify thread support level and \\MPI/ implementation specific resources",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "error handler to invoke in the event that an error is encountered during this function call",
+                "func_type": "",
+                "kind": "ERRHANDLER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "errhandler",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new session",
+                "func_type": "",
+                "kind": "SESSION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "session",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_session_set_errhandler": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Session_set_errhandler",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "session",
+                "func_type": "",
+                "kind": "SESSION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "session",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new error handler for session",
+                "func_type": "",
+                "kind": "ERRHANDLER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "errhandler",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_sizeof": {
+        "attributes": {
+            "c_expressible": false,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Sizeof",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "a Fortran variable of numeric intrinsic type",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "x",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent f90_parenthesis"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "size of machine representation of that type",
+                "func_type": "",
+                "kind": "NUM_BYTES_SMALL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "size",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_ssend": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Ssend",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "initial address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in send buffer",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each send buffer element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of destination",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "dest",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "message tag",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "tag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_ssend_init": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Ssend_init",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "initial address of send buffer",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements sent",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "type of each element",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of destination",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "dest",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "message tag",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "tag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "request",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_start": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Start",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "request",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_startall": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Startall",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "list length",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "array of requests",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": "count",
+                "lis_direction": "inout",
+                "name": "array_of_requests",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_status_c2f": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": true,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Status_c2f",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "c_status",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "F90_STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "f_status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CLASS"
+    },
+    "mpi_status_c2f08": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": true,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Status_c2f08",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "c_status",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "F08_STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "f08_status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CLASS"
+    },
+    "mpi_status_f082c": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": true,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Status_f082c",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "",
+                "func_type": "",
+                "kind": "F08_STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "f08_status",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "c_status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CLASS"
+    },
+    "mpi_status_f082f": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": true,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": true,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Status_f082f",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "status object declared as named type",
+                "func_type": "",
+                "kind": "F08_STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "f08_status",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": true,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status object declared as array",
+                "func_type": "",
+                "kind": "F90_STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "f_status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_status_f2c": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": true,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Status_f2c",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "",
+                "func_type": "",
+                "kind": "F90_STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "f_status",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "c_status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CLASS"
+    },
+    "mpi_status_f2f08": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": true,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": true,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Status_f2f08",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "status object declared as array",
+                "func_type": "",
+                "kind": "F90_STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "f_status",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": true,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status object declared as named type",
+                "func_type": "",
+                "kind": "F08_STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "f08_status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_status_get_error": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Status_get_error",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status from which to retrieve error",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "status",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "error set in the \\mpiconst{MPI\\_ERROR} field",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "err",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_status_get_source": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Status_get_source",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status from which to retrieve source rank",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "status",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank set in the \\mpiconst{MPI\\_SOURCE} field",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "source",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_status_get_tag": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Status_get_tag",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status from which to retrieve tag",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "status",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "tag set in the \\mpiconst{MPI\\_TAG} field",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "tag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_status_set_cancelled": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Status_set_cancelled",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status with which to associate cancel flag",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "status",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "if true, indicates request was cancelled",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_status_set_elements": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Status_set_elements",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status with which to associate count",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "status",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype associated with count",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements to associate with status",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_status_set_elements_x": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Status_set_elements_x",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status with which to associate count",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "status",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype associated with count",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements to associate with status",
+                "func_type": "",
+                "kind": "XFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_status_set_error": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Status_set_error",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status with which to associate error",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "status",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "error to set in the \\mpiconst{MPI\\_ERROR} field",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "err",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_status_set_source": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Status_set_source",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status with which to associate source rank",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "status",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank to set in the \\mpiconst{MPI\\_SOURCE} field",
+                "func_type": "",
+                "kind": "RANK",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "source",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_status_set_tag": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Status_set_tag",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "status with which to associate tag",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "status",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "tag to set in the \\mpiconst{MPI\\_TAG} field",
+                "func_type": "",
+                "kind": "TAG",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "tag",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_category_changed": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_category_changed",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "update number",
+                "func_type": "",
+                "kind": "UPDATE_NUMBER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "update_number",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_category_get_categories": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_category_get_categories",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "index of the category to be queried, in the range from $0$ to $\\mpiarg{num_cat}-1$",
+                "func_type": "",
+                "kind": "CAT_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "cat_index",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "the length of the indices array",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "len",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "an integer array of size \\mpiarg{len}, indicating category indices",
+                "func_type": "",
+                "kind": "CAT_INDEX",
+                "large_only": false,
+                "length": "len",
+                "lis_direction": "out",
+                "name": "indices",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_category_get_cvars": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_category_get_cvars",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "index of the category to be queried, in the range from $0$ to $\\mpiarg{num_cat}-1$",
+                "func_type": "",
+                "kind": "CAT_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "cat_index",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "the length of the indices array",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "len",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "an integer array of size \\mpiarg{len}, indicating control variable indices",
+                "func_type": "",
+                "kind": "CVAR_INDEX",
+                "large_only": false,
+                "length": "len",
+                "lis_direction": "out",
+                "name": "indices",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_category_get_events": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_category_get_events",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "index of the category to be queried, in the range from $0$ to $\\mpiarg{num_cat}-1$",
+                "func_type": "",
+                "kind": "CAT_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "cat_index",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "the length of the indices array",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "len",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "an integer array of size \\mpiarg{len}, indicating event type indices",
+                "func_type": "",
+                "kind": "CAT_INDEX",
+                "large_only": false,
+                "length": "len",
+                "lis_direction": "out",
+                "name": "indices",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_category_get_index": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_category_get_index",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "the name of the category",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "name",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "the index of the category",
+                "func_type": "",
+                "kind": "CAT_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "cat_index",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_category_get_info": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_category_get_info",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "index of the category to be queried",
+                "func_type": "",
+                "kind": "CAT_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "cat_index",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "buffer to return the string containing the name of the category",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "name",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "length of the string and/or buffer for \\mpiarg{name}",
+                "func_type": "",
+                "kind": "STRING_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "name_len",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "buffer to return the string containing the description of the category",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "desc",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "length of the string and/or buffer for \\mpiarg{desc}",
+                "func_type": "",
+                "kind": "STRING_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "desc_len",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of control variables in the category",
+                "func_type": "",
+                "kind": "CVAR_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "num_cvars",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of performance variables in the category",
+                "func_type": "",
+                "kind": "PVAR_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "num_pvars",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of  categories contained in the category",
+                "func_type": "",
+                "kind": "CAT_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "num_categories",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_category_get_num": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_category_get_num",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "current number of categories",
+                "func_type": "",
+                "kind": "CAT_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "num_cat",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_category_get_num_events": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_category_get_num_events",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "index of the category to be queried",
+                "func_type": "",
+                "kind": "CAT_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "cat_index",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of event types in the category",
+                "func_type": "",
+                "kind": "EVENT_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "num_events",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_category_get_pvars": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_category_get_pvars",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "index of the category to be queried, in the range from $0$ to $\\mpiarg{num_cat}-1$",
+                "func_type": "",
+                "kind": "CAT_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "cat_index",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "the length of the indices array",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "len",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "an integer array of size \\mpiarg{len}, indicating performance variable indices",
+                "func_type": "",
+                "kind": "PVAR_INDEX",
+                "large_only": false,
+                "length": "len",
+                "lis_direction": "out",
+                "name": "indices",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_cvar_get_index": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_cvar_get_index",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "name of the control variable",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "name",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "index of the control variable",
+                "func_type": "",
+                "kind": "CVAR_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "cvar_index",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_cvar_get_info": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_cvar_get_info",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "index of the control variable to be queried, value between $0$ and $\\mpiarg{num_cvar}-1$",
+                "func_type": "",
+                "kind": "CVAR_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "cvar_index",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "buffer to return the string containing the name of the control variable",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "name",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "length of the string and/or buffer for \\mpiarg{name}",
+                "func_type": "",
+                "kind": "STRING_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "name_len",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "verbosity level of this variable",
+                "func_type": "",
+                "kind": "TOOL_VAR_VERBOSITY",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "verbosity",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "\\mpi/ datatype of the information stored in the control variable",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "optional descriptor for enumeration information",
+                "func_type": "",
+                "kind": "TOOLS_ENUM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "enumtype",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "buffer to return the string containing a description of the control variable",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "desc",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "length of the string and/or buffer for \\mpiarg{desc}",
+                "func_type": "",
+                "kind": "STRING_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "desc_len",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "type of \\mpi/ object to which this variable must be bound",
+                "func_type": "",
+                "kind": "BIND_TYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "bind",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "scope of when changes to this variable are possible",
+                "func_type": "",
+                "kind": "VARIABLE_SCOPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "scope",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_cvar_get_num": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_cvar_get_num",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "returns number of control variables",
+                "func_type": "",
+                "kind": "CVAR_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "num_cvar",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_cvar_handle_alloc": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_cvar_handle_alloc",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "index of control variable for which handle is to be allocated",
+                "func_type": "",
+                "kind": "CVAR_INDEX_SPECIAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "cvar_index",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "reference to a handle of the \\mpi/ object to which this variable is supposed to be bound",
+                "func_type": "",
+                "kind": "TOOL_MPI_OBJ",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "obj_handle",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "allocated handle",
+                "func_type": "",
+                "kind": "CVAR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "handle",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements used to represent this variable",
+                "func_type": "",
+                "kind": "TOOLS_NUM_ELEM_SMALL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "count",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_cvar_handle_free": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_cvar_handle_free",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "handle to be freed",
+                "func_type": "",
+                "kind": "CVAR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "handle",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_cvar_read": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_cvar_read",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "handle to the control variable to be read",
+                "func_type": "",
+                "kind": "CVAR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "handle",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "initial address of storage location for variable value",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_cvar_write": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_cvar_write",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "handle to the control variable to be written",
+                "func_type": "",
+                "kind": "CVAR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "handle",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "initial address of storage location for variable value",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_enum_get_info": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_enum_get_info",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "enumeration to be queried",
+                "func_type": "",
+                "kind": "TOOLS_ENUM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "enumtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of discrete values represented by this enumeration",
+                "func_type": "",
+                "kind": "TOOLENUM_SIZE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "num",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "buffer to return the string containing the name of the enumeration item",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "name",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "length of the string and/or buffer for \\mpiarg{name}",
+                "func_type": "",
+                "kind": "STRING_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "name_len",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_enum_get_item": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_enum_get_item",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "enumeration to be queried",
+                "func_type": "",
+                "kind": "TOOLS_ENUM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "enumtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of the value to be queried in this enumeration",
+                "func_type": "",
+                "kind": "TOOLENUM_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "index",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "variable value",
+                "func_type": "",
+                "kind": "TOOL_VAR_VALUE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "value",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "buffer to return the string containing the name of the enumeration item",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "name",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "length of the string and/or buffer for \\mpiarg{name}",
+                "func_type": "",
+                "kind": "STRING_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "name_len",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_event_callback_get_info": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_event_callback_get_info",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "event registration",
+                "func_type": "",
+                "kind": "EVENT_REGISTRATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "event_registration",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "callback safety level",
+                "func_type": "",
+                "kind": "CALLBACK_SAFETY",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "cb_safety",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "info_used",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_event_callback_set_info": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_event_callback_set_info",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "event registration",
+                "func_type": "",
+                "kind": "EVENT_REGISTRATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "event_registration",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "callback safety level",
+                "func_type": "",
+                "kind": "CALLBACK_SAFETY",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "cb_safety",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_event_cb_function": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": true,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_event_cb_function",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EVENT_INSTANCE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "event_instance",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EVENT_REGISTRATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "event_registration",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "CALLBACK_SAFETY",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "cb_safety",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "user_data",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "NOTHING"
+    },
+    "mpi_t_event_copy": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_event_copy",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "event instance provided to the callback function",
+                "func_type": "",
+                "kind": "EVENT_INSTANCE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "event_instance",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "user-allocated buffer for event data",
+                "func_type": "",
+                "kind": "C_BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "buffer",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_event_dropped_cb_function": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": true,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_event_dropped_cb_function",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "DROPPED_COUNT",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EVENT_REGISTRATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "event_registration",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "SOURCE_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "source_index",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "CALLBACK_SAFETY",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "cb_safety",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "user_data",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "NOTHING"
+    },
+    "mpi_t_event_free_cb_function": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": true,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_event_free_cb_function",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EVENT_REGISTRATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "event_registration",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "CALLBACK_SAFETY",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "cb_safety",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "user_data",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "NOTHING"
+    },
+    "mpi_t_event_get_index": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_event_get_index",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "name of the event type",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "name",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "index of the event type",
+                "func_type": "",
+                "kind": "EVENT_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "event_index",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_event_get_info": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_event_get_info",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "index of the event type to be queried between $0$ and $\\mpiarg{num_events}-1$",
+                "func_type": "",
+                "kind": "EVENT_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "event_index",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "buffer to return the string containing the name of the event type",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "name",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "length of the string and/or buffer for \\mpiarg{name}",
+                "func_type": "",
+                "kind": "STRING_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "name_len",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "verbosity level of this event type",
+                "func_type": "",
+                "kind": "TOOL_VAR_VERBOSITY",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "verbosity",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "array of \\mpi/ basic datatypes used to encode the event data",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": "num_elements",
+                "lis_direction": "out",
+                "name": "array_of_datatypes",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "array of byte displacements of the elements in the event buffer",
+                "func_type": "",
+                "kind": "DISPLACEMENT_NNI",
+                "large_only": false,
+                "length": "num_elements",
+                "lis_direction": "out",
+                "name": "array_of_displacements",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "length of \\mpiarg{array\\_of\\_datatypes} and \\mpiarg{array\\_of\\_displacements} arrays",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "num_elements",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "optional descriptor for enumeration information",
+                "func_type": "",
+                "kind": "TOOLS_ENUM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "enumtype",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "optional info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "info",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "buffer to return the string containing a description of the event type",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "desc",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "length of the string and/or buffer for \\mpiarg{desc}",
+                "func_type": "",
+                "kind": "STRING_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "desc_len",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "type of \\mpi/ object to which an event of this type must be bound",
+                "func_type": "",
+                "kind": "BIND_TYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "bind",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_event_get_num": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_event_get_num",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "returns number of event types",
+                "func_type": "",
+                "kind": "EVENT_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "num_events",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_event_get_source": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_event_get_source",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "event instance provided to the callback function",
+                "func_type": "",
+                "kind": "EVENT_INSTANCE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "event_instance",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "index identifying the source",
+                "func_type": "",
+                "kind": "SOURCE_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "source_index",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_event_get_timestamp": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_event_get_timestamp",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "event instance provided to the callback function",
+                "func_type": "",
+                "kind": "EVENT_INSTANCE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "event_instance",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "timestamp the event was observed",
+                "func_type": "",
+                "kind": "TOOLS_TICK_COUNT",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "event_timestamp",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_event_handle_alloc": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_event_handle_alloc",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "index of event type for which the registration handle is to be allocated",
+                "func_type": "",
+                "kind": "EVENT_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "event_index",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "reference to a handle of the \\mpi/ object to which this event is supposed to be bound",
+                "func_type": "",
+                "kind": "TOOL_MPI_OBJ",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "obj_handle",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "event registration",
+                "func_type": "",
+                "kind": "EVENT_REGISTRATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "event_registration",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_event_handle_free": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_event_handle_free",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "event registration",
+                "func_type": "",
+                "kind": "EVENT_REGISTRATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "event_registration",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "pointer to a user-controlled buffer",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "user_data",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "pointer to user-defined callback function",
+                "func_type": "",
+                "kind": "EVENT_FREE_CB_FUNCTION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "free_cb_function",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_event_handle_get_info": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_event_handle_get_info",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "event registration",
+                "func_type": "",
+                "kind": "EVENT_REGISTRATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "event_registration",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "info_used",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_event_handle_set_info": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_event_handle_set_info",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "event registration",
+                "func_type": "",
+                "kind": "EVENT_REGISTRATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "event_registration",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_event_read": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_event_read",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "event-instance handle provided to the callback function",
+                "func_type": "",
+                "kind": "EVENT_INSTANCE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "event_instance",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "index into the array of datatypes of the item to be queried",
+                "func_type": "",
+                "kind": "INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "element_index",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "pointer to a memory location to store the item data",
+                "func_type": "",
+                "kind": "C_BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "buffer",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_event_register_callback": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_event_register_callback",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "event registration",
+                "func_type": "",
+                "kind": "EVENT_REGISTRATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "event_registration",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "maximum callback safety level",
+                "func_type": "",
+                "kind": "CALLBACK_SAFETY",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "cb_safety",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "pointer to a user-controlled buffer",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "user_data",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "pointer to user-defined callback function",
+                "func_type": "",
+                "kind": "EVENT_CB_FUNCTION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "event_cb_function",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_event_set_dropped_handler": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_event_set_dropped_handler",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "valid event registration",
+                "func_type": "",
+                "kind": "EVENT_REGISTRATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "event_registration",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "pointer to user-defined callback function",
+                "func_type": "",
+                "kind": "EVENT_DROP_CB_FUNCTION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "dropped_cb_function",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_finalize": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_finalize",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_init_thread": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_init_thread",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "desired level of thread support",
+                "func_type": "",
+                "kind": "THREAD_LEVEL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "required",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "provided level of thread support",
+                "func_type": "",
+                "kind": "THREAD_LEVEL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "provided",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_pvar_get_index": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_pvar_get_index",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "the name of the performance variable",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "name",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "the class of the performance variable",
+                "func_type": "",
+                "kind": "PVAR_CLASS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "var_class",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "the index of the performance variable",
+                "func_type": "",
+                "kind": "PVAR_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "pvar_index",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_pvar_get_info": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_pvar_get_info",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "index of the performance variable to be queried between $0$ and $\\mpiarg{num_pvar}-1$",
+                "func_type": "",
+                "kind": "PVAR_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "pvar_index",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "buffer to return the string containing the name of the performance variable",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "name",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "length of the string and/or buffer for \\mpiarg{name}",
+                "func_type": "",
+                "kind": "STRING_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "name_len",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "verbosity level of this variable",
+                "func_type": "",
+                "kind": "TOOL_VAR_VERBOSITY",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "verbosity",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "class of performance variable",
+                "func_type": "",
+                "kind": "PVAR_CLASS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "var_class",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "\\mpi/ datatype of the information stored in the performance variable",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "optional descriptor for enumeration information",
+                "func_type": "",
+                "kind": "TOOLS_ENUM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "enumtype",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "buffer to return the string containing a description of the performance variable",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "desc",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "length of the string and/or buffer for \\mpiarg{desc}",
+                "func_type": "",
+                "kind": "STRING_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "desc_len",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "type of \\mpi/ object to which this variable must be bound",
+                "func_type": "",
+                "kind": "BIND_TYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "bind",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "flag indicating whether the variable can be written/reset",
+                "func_type": "",
+                "kind": "LOGICAL_OPTIONAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "readonly",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "flag indicating whether the variable can be started and stopped or is continuously active",
+                "func_type": "",
+                "kind": "LOGICAL_OPTIONAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "continuous",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "flag indicating whether the variable can be atomically read and reset",
+                "func_type": "",
+                "kind": "LOGICAL_OPTIONAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "atomic",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_pvar_get_num": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_pvar_get_num",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "returns number of performance variables",
+                "func_type": "",
+                "kind": "PVAR_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "num_pvar",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_pvar_handle_alloc": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_pvar_handle_alloc",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "identifier of performance experiment session",
+                "func_type": "",
+                "kind": "PVAR_SESSION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "pe_session",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "index of performance variable for which handle is to be allocated",
+                "func_type": "",
+                "kind": "PVAR_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "pvar_index",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "reference to a handle of the \\mpi/ object to which this variable is supposed to be bound",
+                "func_type": "",
+                "kind": "TOOL_MPI_OBJ",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "obj_handle",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "allocated handle",
+                "func_type": "",
+                "kind": "PVAR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "handle",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements used to represent this variable",
+                "func_type": "",
+                "kind": "TOOLS_NUM_ELEM_SMALL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "count",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_pvar_handle_free": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_pvar_handle_free",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "identifier of performance experiment session",
+                "func_type": "",
+                "kind": "PVAR_SESSION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "pe_session",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "handle to be freed",
+                "func_type": "",
+                "kind": "PVAR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "handle",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_pvar_read": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_pvar_read",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "identifier of performance experiment session",
+                "func_type": "",
+                "kind": "PVAR_SESSION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "pe_session",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "handle of a performance variable",
+                "func_type": "",
+                "kind": "PVAR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "handle",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "initial address of storage location for variable value",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_pvar_readreset": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_pvar_readreset",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "identifier of performance experiment session",
+                "func_type": "",
+                "kind": "PVAR_SESSION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "pe_session",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "handle of a performance variable",
+                "func_type": "",
+                "kind": "PVAR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "handle",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "initial address of storage location for variable value",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_pvar_reset": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_pvar_reset",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "identifier of performance experiment session",
+                "func_type": "",
+                "kind": "PVAR_SESSION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "pe_session",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "handle of a performance variable",
+                "func_type": "",
+                "kind": "PVAR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "handle",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_pvar_session_create": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_pvar_session_create",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "identifier of performance experiment session",
+                "func_type": "",
+                "kind": "PVAR_SESSION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "pe_session",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_pvar_session_free": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_pvar_session_free",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "identifier of performance experiment session",
+                "func_type": "",
+                "kind": "PVAR_SESSION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "pe_session",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_pvar_start": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_pvar_start",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "identifier of performance experiment session",
+                "func_type": "",
+                "kind": "PVAR_SESSION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "pe_session",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "handle of a performance variable",
+                "func_type": "",
+                "kind": "PVAR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "handle",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_pvar_stop": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_pvar_stop",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "identifier of performance experiment session",
+                "func_type": "",
+                "kind": "PVAR_SESSION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "pe_session",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "handle of a performance variable",
+                "func_type": "",
+                "kind": "PVAR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "handle",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_pvar_write": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_pvar_write",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "identifier of performance experiment session",
+                "func_type": "",
+                "kind": "PVAR_SESSION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "pe_session",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "handle of a performance variable",
+                "func_type": "",
+                "kind": "PVAR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "handle",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "initial address of storage location for variable value",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "buf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_source_get_info": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_source_get_info",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "index of the source to be queried between $0$ and $\\mpiarg{num_sources}-1$",
+                "func_type": "",
+                "kind": "SOURCE_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "source_index",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "buffer to return the string containing the name of the source",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "name",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "length of the string and/or buffer for \\mpiarg{name}",
+                "func_type": "",
+                "kind": "STRING_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "name_len",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "buffer to return the string containing the description of the source",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "desc",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "length of the string and/or buffer for \\mpiarg{desc}",
+                "func_type": "",
+                "kind": "STRING_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "desc_len",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "flag indicating chronological ordering guarantees given by the source",
+                "func_type": "",
+                "kind": "SOURCE_ORDERING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ordering",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "the number of ticks per second for the timer of this source",
+                "func_type": "",
+                "kind": "TOOLS_TICK_COUNT",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ticks_per_second",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "the maximum count of ticks reported by this source before overflow occurs",
+                "func_type": "",
+                "kind": "TOOLS_TICK_COUNT",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "max_ticks",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "optional info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "info",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_source_get_num": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_source_get_num",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "returns number of event sources",
+                "func_type": "",
+                "kind": "SOURCE_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "num_sources",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_t_source_get_timestamp": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_T_source_get_timestamp",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "index of the source",
+                "func_type": "",
+                "kind": "SOURCE_INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "source_index",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "current timestamp from specified source",
+                "func_type": "",
+                "kind": "TOOLS_TICK_COUNT",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "timestamp",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_test": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Test",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "request",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "\\mpicode{true} if operation completed",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_test_cancelled": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Test_cancelled",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "status",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "\\mpicode{true} if the operation has been cancelled",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_testall": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Testall",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "list length",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "array of requests",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": "count",
+                "lis_direction": "inout",
+                "name": "array_of_requests",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "\\mpicode{true} if all of the operations are complete",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "array of status objects",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "out",
+                "name": "array_of_statuses",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": false,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_testany": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Testany",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "list length",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "array of requests",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": "count",
+                "lis_direction": "inout",
+                "name": "array_of_requests",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "index of operation that completed or \\mpiconst{MPI_UNDEFINED} if none completed",
+                "func_type": "",
+                "kind": "INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "index",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "\\mpicode{true} if one of the operations is complete",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_testsome": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Testsome",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "length of array_of_requests",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "incount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "array of requests",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": "incount",
+                "lis_direction": "inout",
+                "name": "array_of_requests",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of completed requests",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "outcount",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "array of indices of operations that completed",
+                "func_type": "",
+                "kind": "INDEX",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "out",
+                "name": "array_of_indices",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "array of status objects for operations that completed",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "out",
+                "name": "array_of_statuses",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": false,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_topo_test": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Topo_test",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "topology type of communicator \\mpiarg{comm}",
+                "func_type": "",
+                "kind": "TOPOLOGY_TYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_c2f": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": true,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_c2f",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": false,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "F90_DATATYPE"
+    },
+    "mpi_type_commit": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_commit",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype that is committed",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_contiguous": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_contiguous",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "replication count",
+                "func_type": "",
+                "kind": "POLYDTYPE_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "old datatype",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "oldtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new datatype",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newtype",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_copy_attr_function": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": true,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_copy_attr_function",
+        "name_f90": "TYPE_COPY_ATTR_FUNCTION",
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "oldtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "type_keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val_in",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val_out",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "ierror",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_create_darray": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_create_darray",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "size of process group",
+                "func_type": "",
+                "kind": "COMM_SIZE_PI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "size",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank in process group",
+                "func_type": "",
+                "kind": "RANK_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "rank",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of array dimensions as well as process grid dimensions",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH_PI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "ndims",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "number of elements of type \\mpiarg{oldtype} in each dimension of global array",
+                "func_type": "",
+                "kind": "POLYDTYPE_NUM_ELEM_PI",
+                "large_only": false,
+                "length": "ndims",
+                "lis_direction": "in",
+                "name": "array_of_gsizes",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "distribution of array in each dimension",
+                "func_type": "",
+                "kind": "DISTRIB_ENUM",
+                "large_only": false,
+                "length": "ndims",
+                "lis_direction": "in",
+                "name": "array_of_distribs",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "distribution argument in each dimension",
+                "func_type": "",
+                "kind": "DTYPE_DISTRIBUTION",
+                "large_only": false,
+                "length": "ndims",
+                "lis_direction": "in",
+                "name": "array_of_dargs",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "size of process grid in each dimension",
+                "func_type": "",
+                "kind": "PROCESS_GRID_SIZE",
+                "large_only": false,
+                "length": "ndims",
+                "lis_direction": "in",
+                "name": "array_of_psizes",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "array storage order flag",
+                "func_type": "",
+                "kind": "ORDER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "order",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "old datatype",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "oldtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new datatype",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newtype",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_create_f90_complex": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_create_f90_complex",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "precision, in decimal digits",
+                "func_type": "",
+                "kind": "MATH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "p",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "decimal exponent range",
+                "func_type": "",
+                "kind": "MATH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "r",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "the requested \\MPI/ datatype",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newtype",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_create_f90_integer": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_create_f90_integer",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "decimal exponent range, i.e., number of decimal digits",
+                "func_type": "",
+                "kind": "MATH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "r",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "the requested \\MPI/ datatype",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newtype",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_create_f90_real": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_create_f90_real",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "precision, in decimal digits",
+                "func_type": "",
+                "kind": "MATH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "p",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "decimal exponent range",
+                "func_type": "",
+                "kind": "MATH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "r",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "the requested \\MPI/ datatype",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newtype",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_create_hindexed": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_create_hindexed",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of blocks---also number of entries in \\mpiarg{array_of_displacements} and \\mpiarg{array_of_blocklengths}",
+                "func_type": "",
+                "kind": "POLYDTYPE_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "number of elements in each block",
+                "func_type": "",
+                "kind": "POLYDTYPE_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "count",
+                "lis_direction": "in",
+                "name": "array_of_blocklengths",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "byte displacement of each block",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT_AINT_COUNT",
+                "large_only": false,
+                "length": "count",
+                "lis_direction": "in",
+                "name": "array_of_displacements",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "old datatype",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "oldtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new datatype",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newtype",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_create_hindexed_block": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_create_hindexed_block",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of blocks---also number of entries in \\mpiarg{array_of_displacements}",
+                "func_type": "",
+                "kind": "POLYDTYPE_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in each block",
+                "func_type": "",
+                "kind": "POLYDTYPE_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "blocklength",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "byte displacement of each block",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT_AINT_COUNT",
+                "large_only": false,
+                "length": "count",
+                "lis_direction": "in",
+                "name": "array_of_displacements",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "old datatype",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "oldtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new datatype",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newtype",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_create_hvector": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_create_hvector",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of blocks",
+                "func_type": "",
+                "kind": "POLYDTYPE_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in each block",
+                "func_type": "",
+                "kind": "POLYDTYPE_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "blocklength",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of bytes between start of each block",
+                "func_type": "",
+                "kind": "POLYDTYPE_STRIDE_BYTES",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "stride",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "old datatype",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "oldtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new datatype",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newtype",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_create_indexed_block": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_create_indexed_block",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of blocks---also number of entries in \\mpiarg{array_of_displacements}",
+                "func_type": "",
+                "kind": "POLYDTYPE_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in each block",
+                "func_type": "",
+                "kind": "POLYDTYPE_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "blocklength",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "array of displacements, in multiples of \\mpiarg{oldtype}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT_COUNT",
+                "large_only": false,
+                "length": "count",
+                "lis_direction": "in",
+                "name": "array_of_displacements",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "old datatype",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "oldtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new datatype",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newtype",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_create_keyval": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_create_keyval",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "copy callback function for \\mpiarg{type_keyval}",
+                "func_type": "MPI_Type_copy_attr_function",
+                "kind": "FUNCTION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "type_copy_attr_fn",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "delete callback function for \\mpiarg{type_keyval}",
+                "func_type": "MPI_Type_delete_attr_function",
+                "kind": "FUNCTION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "type_delete_attr_fn",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "key value for future access",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "type_keyval",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "extra state for callback function",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_create_resized": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_create_resized",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "input datatype",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "oldtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new lower bound of datatype",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT_AINT_COUNT",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "lb",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new extent of datatype",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT_AINT_COUNT",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extent",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "output datatype",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newtype",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_create_struct": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_create_struct",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of blocks---also number of entries in arrays \\mpiarg{array_of_types}, \\mpiarg{array_of_displacements}, and \\mpiarg{array_of_blocklengths}",
+                "func_type": "",
+                "kind": "POLYDTYPE_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "number of elements in each block",
+                "func_type": "",
+                "kind": "POLYDTYPE_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "count",
+                "lis_direction": "in",
+                "name": "array_of_blocklengths",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "byte displacement of each block",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT_AINT_COUNT",
+                "large_only": false,
+                "length": "count",
+                "lis_direction": "in",
+                "name": "array_of_displacements",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "type of elements in each block",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": "count",
+                "lis_direction": "in",
+                "name": "array_of_types",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new datatype",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newtype",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_create_subarray": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_create_subarray",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of array dimensions",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH_PI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "ndims",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "number of elements of type \\mpiarg{oldtype} in each dimension of the full array",
+                "func_type": "",
+                "kind": "POLYDTYPE_NUM_ELEM_PI",
+                "large_only": false,
+                "length": "ndims",
+                "lis_direction": "in",
+                "name": "array_of_sizes",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "number of elements of type \\mpiarg{oldtype} in each dimension of the subarray",
+                "func_type": "",
+                "kind": "POLYDTYPE_NUM_ELEM_PI",
+                "large_only": false,
+                "length": "ndims",
+                "lis_direction": "in",
+                "name": "array_of_subsizes",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "starting coordinates of the subarray in each dimension",
+                "func_type": "",
+                "kind": "POLYDTYPE_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "ndims",
+                "lis_direction": "in",
+                "name": "array_of_starts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "array storage order flag",
+                "func_type": "",
+                "kind": "ORDER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "order",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "old datatype",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "oldtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new datatype",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newtype",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_delete_attr": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_delete_attr",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype from which the attribute is deleted",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "key value",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "type_keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_delete_attr_function": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": true,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_delete_attr_function",
+        "name_f90": "TYPE_DELETE_ATTR_FUNCTION",
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "type_keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "ierror",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_dup": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_dup",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "oldtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "copy of \\mpiarg{oldtype}",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newtype",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_dup_fn": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": "MPI_Type_copy_attr_function",
+            "proxy_render": false
+        },
+        "name": "MPI_TYPE_DUP_FN",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "oldtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "type_keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val_in",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val_out",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "ierror",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_f2c": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": true,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_f2c",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "F90_DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": false,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "DATATYPE"
+    },
+    "mpi_type_free": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_free",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype that is freed",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_free_keyval": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_free_keyval",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "key value",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "type_keyval",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_get_attr": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_get_attr",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype to which the attribute is attached",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "key value",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "type_keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "attribute value, unless \\mpiarg{flag}\\mpicode{ = false}",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "attribute_val",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "\\mpicode{false} if no attribute is associated with the key",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_get_contents": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_get_contents",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype to decode",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in \\mpiarg{array_of_integers}",
+                "func_type": "",
+                "kind": "POLYNUM_PARAM_VALUES",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "max_integers",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in \\mpiarg{array_of_addresses}",
+                "func_type": "",
+                "kind": "POLYNUM_PARAM_VALUES",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "max_addresses",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in \\mpiarg{array_of_large_counts}",
+                "func_type": "",
+                "kind": "POLYNUM_PARAM_VALUES",
+                "large_only": true,
+                "length": null,
+                "lis_direction": "in",
+                "name": "max_large_counts",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in \\mpiarg{array_of_datatypes}",
+                "func_type": "",
+                "kind": "POLYNUM_PARAM_VALUES",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "max_datatypes",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "contains integer arguments used in constructing \\mpiarg{datatype}",
+                "func_type": "",
+                "kind": "GENERIC_DTYPE_INT",
+                "large_only": false,
+                "length": "max_integers",
+                "lis_direction": "out",
+                "name": "array_of_integers",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "contains address arguments used in constructing \\mpiarg{datatype}",
+                "func_type": "",
+                "kind": "DISPLACEMENT",
+                "large_only": false,
+                "length": "max_addresses",
+                "lis_direction": "out",
+                "name": "array_of_addresses",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "contains large count arguments used in constructing \\mpiarg{datatype}",
+                "func_type": "",
+                "kind": "GENERIC_DTYPE_COUNT",
+                "large_only": true,
+                "length": "max_large_counts",
+                "lis_direction": "out",
+                "name": "array_of_large_counts",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "contains datatype arguments used in constructing \\mpiarg{datatype}",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": "max_datatypes",
+                "lis_direction": "out",
+                "name": "array_of_datatypes",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_get_envelope": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_get_envelope",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype to decode",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of input integers used in call constructing \\mpiarg{combiner}",
+                "func_type": "",
+                "kind": "POLYNUM_PARAM_VALUES",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "num_integers",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of input addresses used in call constructing \\mpiarg{combiner}",
+                "func_type": "",
+                "kind": "POLYNUM_PARAM_VALUES",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "num_addresses",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of input large counts used in call constructing \\mpiarg{combiner}",
+                "func_type": "",
+                "kind": "POLYNUM_PARAM_VALUES",
+                "large_only": true,
+                "length": null,
+                "lis_direction": "out",
+                "name": "num_large_counts",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of input datatypes used in call constructing \\mpiarg{combiner}",
+                "func_type": "",
+                "kind": "POLYNUM_PARAM_VALUES",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "num_datatypes",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "combiner",
+                "func_type": "",
+                "kind": "COMBINER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "combiner",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_get_extent": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_get_extent",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype to get information on",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "lower bound of datatype",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT_AINT_COUNT",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "lb",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "extent of datatype",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT_AINT_COUNT",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "extent",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_get_extent_x": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_get_extent_x",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype to get information on",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "lower bound of datatype",
+                "func_type": "",
+                "kind": "XFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "lb",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "extent of datatype",
+                "func_type": "",
+                "kind": "XFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "extent",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_get_name": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_get_name",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype whose name is to be returned",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "the name previously stored on the datatype, or an empty string if no such name exists",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": "MPI_MAX_OBJECT_NAME",
+                "lis_direction": "out",
+                "name": "type_name",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "length of returned name",
+                "func_type": "",
+                "kind": "STRING_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "resultlen",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_get_true_extent": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_get_true_extent",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype to get information on",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "true lower bound of datatype",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT_AINT_COUNT",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "true_lb",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "true extent of datatype",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT_AINT_COUNT",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "true_extent",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_get_true_extent_x": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_get_true_extent_x",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype to get information on",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "true lower bound of datatype",
+                "func_type": "",
+                "kind": "XFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "true_lb",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "true extent of datatype",
+                "func_type": "",
+                "kind": "XFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "true_extent",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_get_value_index": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_get_value_index",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of the value in pair",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "value_type",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of the index in pair",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "index_type",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of the value-index pair",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "pair_type",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_indexed": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_indexed",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of blocks---also number of entries in \\mpiarg{array_of_displacements} and \\mpiarg{array_of_blocklengths}",
+                "func_type": "",
+                "kind": "POLYDTYPE_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "number of elements per block",
+                "func_type": "",
+                "kind": "POLYDTYPE_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": "count",
+                "lis_direction": "in",
+                "name": "array_of_blocklengths",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "displacement for each block, in multiples of \\mpiarg{oldtype}",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT_COUNT",
+                "large_only": false,
+                "length": "count",
+                "lis_direction": "in",
+                "name": "array_of_displacements",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "old datatype",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "oldtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new datatype",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newtype",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_match_size": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_match_size",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "generic type specifier",
+                "func_type": "",
+                "kind": "TYPECLASS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "typeclass",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "size, in bytes, of representation",
+                "func_type": "",
+                "kind": "TYPECLASS_SIZE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "size",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype with correct type, size",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_null_copy_fn": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": "MPI_Type_copy_attr_function",
+            "proxy_render": false
+        },
+        "name": "MPI_TYPE_NULL_COPY_FN",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "oldtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "type_keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val_in",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val_out",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "ierror",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_null_delete_fn": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": "MPI_Type_delete_attr_function",
+            "proxy_render": false
+        },
+        "name": "MPI_TYPE_NULL_DELETE_FN",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "type_keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE_SHOW_INTENT",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_set_attr": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_set_attr",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype to which attribute will be attached",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "key value",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "type_keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "attribute value",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_set_name": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_set_name",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype whose identifier is to be set",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "the character string that is remembered as the name",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "type_name",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_size": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_size",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype to get information on",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype size",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "size",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_size_x": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_size_x",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype to get information on",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype size",
+                "func_type": "",
+                "kind": "XFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "size",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_type_vector": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Type_vector",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of blocks",
+                "func_type": "",
+                "kind": "POLYDTYPE_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements in each block",
+                "func_type": "",
+                "kind": "POLYDTYPE_NUM_ELEM_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "blocklength",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of elements between start of each block",
+                "func_type": "",
+                "kind": "POLYDTYPE_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "stride",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "old datatype",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "oldtype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new datatype",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "newtype",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_unpack": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Unpack",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "input buffer start",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "inbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "size of input buffer, in bytes",
+                "func_type": "",
+                "kind": "POLYNUM_BYTES_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "insize",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "current position in bytes",
+                "func_type": "",
+                "kind": "POLYDISPLACEMENT_COUNT",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "position",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "output buffer start",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "outbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of items to be unpacked",
+                "func_type": "",
+                "kind": "POLYDTYPE_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "outcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of each output data item",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "communicator for packed message",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_unpack_external": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Unpack_external",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "data representation",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "in",
+                "name": "datarep",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": false,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "input buffer start",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "inbuf",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "input buffer size, in bytes",
+                "func_type": "",
+                "kind": "POLYDTYPE_PACK_SIZE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "insize",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "current position in buffer, in bytes",
+                "func_type": "",
+                "kind": "POLYLOCATION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "position",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "output buffer start",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "outbuf",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of output data items",
+                "func_type": "",
+                "kind": "POLYDTYPE_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "outcount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "datatype of output data item",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_unpublish_name": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Unpublish_name",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "a service name",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "service_name",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "implementation-specific information",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "a port name",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "port_name",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_user_function": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": true,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_User_function",
+        "name_f90": "USER_FUNCTION",
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "C_BUFFER4",
+                "large_only": false,
+                "length": "len",
+                "lis_direction": "in",
+                "name": "invec",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "C_BUFFER4",
+                "large_only": false,
+                "length": "len",
+                "lis_direction": "in",
+                "name": "inoutvec",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "POLYXFER_NUM_ELEM",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "len",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": true,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "DATATYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "datatype",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": true,
+                "root_only": false,
+                "suppress": ""
+            }
+        ],
+        "return_kind": "NOTHING"
+    },
+    "mpi_wait": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Wait",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "request",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "request",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_waitall": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Waitall",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "list length",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "array of requests",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": "count",
+                "lis_direction": "inout",
+                "name": "array_of_requests",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "array of status objects",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "out",
+                "name": "array_of_statuses",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": false,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_waitany": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Waitany",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "list length",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "count",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "array of requests",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": "count",
+                "lis_direction": "inout",
+                "name": "array_of_requests",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "index of handle for operation that completed",
+                "func_type": "",
+                "kind": "INDEX",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "index",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "status",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_waitsome": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Waitsome",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "length of array_of_requests",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "incount",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "array of requests",
+                "func_type": "",
+                "kind": "REQUEST",
+                "large_only": false,
+                "length": "incount",
+                "lis_direction": "inout",
+                "name": "array_of_requests",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "number of completed requests",
+                "func_type": "",
+                "kind": "ARRAY_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "outcount",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "array of indices of operations that completed",
+                "func_type": "",
+                "kind": "INDEX",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "out",
+                "name": "array_of_indices",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "array of status objects for operations that completed",
+                "func_type": "",
+                "kind": "STATUS",
+                "large_only": false,
+                "length": "*",
+                "lis_direction": "out",
+                "name": "array_of_statuses",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": false,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_allocate": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": "\\>INTERFACE MPI\\_WIN\\_ALLOCATE \\\\ \\>\\>SUBROUTINE MPI\\_WIN\\_ALLOCATE(SIZE, DISP\\_UNIT, INFO, COMM, BASEPTR, \\& \\\\ \\>\\>\\>\\>WIN, IERROR) \\\\ \\>\\>\\>IMPORT :: MPI\\_ADDRESS\\_KIND \\\\ \\>\\>\\>INTEGER :: DISP\\_UNIT, INFO, COMM, WIN, IERROR \\\\ \\>\\>\\>INTEGER(KIND=MPI\\_ADDRESS\\_KIND) :: SIZE, BASEPTR \\\\ \\>\\>END SUBROUTINE \\\\ \\>\\>SUBROUTINE MPI\\_WIN\\_ALLOCATE\\_CPTR(SIZE, DISP\\_UNIT, INFO, COMM, BASEPTR, \\& \\\\ \\>\\>\\>\\>WIN, IERROR) \\\\ \\>\\>\\>USE, INTRINSIC :: ISO\\_C\\_BINDING, ONLY : C\\_PTR \\\\ \\>\\>\\>IMPORT :: MPI\\_ADDRESS\\_KIND \\\\ \\>\\>\\>INTEGER :: DISP\\_UNIT, INFO, COMM, WIN, IERROR \\\\ \\>\\>\\>INTEGER(KIND=MPI\\_ADDRESS\\_KIND) :: SIZE \\\\ \\>\\>\\>TYPE(C\\_PTR) :: BASEPTR \\\\ \\>\\>END SUBROUTINE \\\\ \\>END INTERFACE",
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_allocate",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "size of window in bytes",
+                "func_type": "",
+                "kind": "WINDOW_SIZE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "size",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "local unit size for displacements, in bytes",
+                "func_type": "",
+                "kind": "POLYRMA_DISPLACEMENT",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "disp_unit",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "intra-communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "initial address of window",
+                "func_type": "",
+                "kind": "C_BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "baseptr",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "win",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_allocate_shared": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": "\\>INTERFACE MPI\\_WIN\\_ALLOCATE\\_SHARED \\\\ \\>\\>SUBROUTINE MPI\\_WIN\\_ALLOCATE\\_SHARED(SIZE, DISP\\_UNIT, INFO, COMM, \\& \\\\ \\>\\>\\>\\>BASEPTR, WIN, IERROR) \\\\ \\>\\>\\>IMPORT :: MPI\\_ADDRESS\\_KIND \\\\ \\>\\>\\>INTEGER :: DISP\\_UNIT, INFO, COMM, WIN, IERROR \\\\ \\>\\>\\>INTEGER(KIND=MPI\\_ADDRESS\\_KIND) :: SIZE, BASEPTR \\\\ \\>\\>END SUBROUTINE \\\\ \\>\\>SUBROUTINE MPI\\_WIN\\_ALLOCATE\\_SHARED\\_CPTR(SIZE, DISP\\_UNIT, INFO, COMM, \\& \\\\ \\>\\>\\>\\>BASEPTR, WIN, IERROR) \\\\ \\>\\>\\>USE, INTRINSIC :: ISO\\_C\\_BINDING, ONLY : C\\_PTR \\\\ \\>\\>\\>IMPORT :: MPI\\_ADDRESS\\_KIND \\\\ \\>\\>\\>INTEGER :: DISP\\_UNIT, INFO, COMM, WIN, IERROR \\\\ \\>\\>\\>INTEGER(KIND=MPI\\_ADDRESS\\_KIND) :: SIZE \\\\ \\>\\>\\>TYPE(C\\_PTR) :: BASEPTR \\\\ \\>\\>END SUBROUTINE \\\\ \\>END INTERFACE",
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_allocate_shared",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "size of local window in bytes",
+                "func_type": "",
+                "kind": "WINDOW_SIZE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "size",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "local unit size for displacements, in bytes",
+                "func_type": "",
+                "kind": "POLYRMA_DISPLACEMENT",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "disp_unit",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "intra-communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "address of local allocated window segment",
+                "func_type": "",
+                "kind": "C_BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "baseptr",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "win",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_attach": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_attach",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "initial address of memory to be attached",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "base",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "size of memory to be attached in bytes",
+                "func_type": "",
+                "kind": "WIN_ATTACH_SIZE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "size",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_c2f": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": true,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_c2f",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": false,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "F90_WIN"
+    },
+    "mpi_win_call_errhandler": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_call_errhandler",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "window with error handler",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "errorcode",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_complete": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_complete",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_copy_attr_function": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": true,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_copy_attr_function",
+        "name_f90": "WIN_COPY_ATTR_FUNCTION",
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "oldwin",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win_keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val_in",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "attribute_val_out",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "ierror",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_create": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_create",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": false,
+                "desc": "initial address of window",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "base",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "size of window in bytes",
+                "func_type": "",
+                "kind": "WINDOW_SIZE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "size",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "local unit size for displacements, in bytes",
+                "func_type": "",
+                "kind": "POLYRMA_DISPLACEMENT",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "disp_unit",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "intra-communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "win",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_create_dynamic": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_create_dynamic",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "intra-communicator",
+                "func_type": "",
+                "kind": "COMMUNICATOR",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "comm",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "win",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_create_errhandler": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_create_errhandler",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "user defined error handling procedure",
+                "func_type": "MPI_Win_errhandler_function",
+                "kind": "FUNCTION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win_errhandler_fn",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERRHANDLER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "errhandler",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_create_keyval": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_create_keyval",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "copy callback function for \\mpiarg{win_keyval}",
+                "func_type": "MPI_Win_copy_attr_function",
+                "kind": "FUNCTION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win_copy_attr_fn",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "delete callback function for \\mpiarg{win_keyval}",
+                "func_type": "MPI_Win_delete_attr_function",
+                "kind": "FUNCTION",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win_delete_attr_fn",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "key value for future access",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "win_keyval",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "extra state for callback function",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_delete_attr": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_delete_attr",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "window from which the attribute is deleted",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "key value",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win_keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_delete_attr_function": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": true,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_delete_attr_function",
+        "name_f90": "WIN_DELETE_ATTR_FUNCTION",
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win_keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "ierror",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_detach": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_detach",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": true,
+                "constant": true,
+                "desc": "initial address of memory to be detached",
+                "func_type": "",
+                "kind": "BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "base",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "f08_intent"
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_dup_fn": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": "MPI_Win_copy_attr_function",
+            "proxy_render": false
+        },
+        "name": "MPI_WIN_DUP_FN",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "oldwin",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win_keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val_in",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val_out",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "ierror",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_errhandler_function": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": true,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_errhandler_function",
+        "name_f90": "WIN_ERRHANDLER_FUNCTION",
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": true,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "error_code",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": true,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "VARARGS",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "varargs",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            }
+        ],
+        "return_kind": "NOTHING"
+    },
+    "mpi_win_f2c": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": false,
+            "f90_expressible": false,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": true,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_f2c",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "F90_WIN",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": false,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "WINDOW"
+    },
+    "mpi_win_fence": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_fence",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ASSERT",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "assert",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_flush": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_flush",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of target \\MPI/ process in the group of the window \\mpiarg{win}",
+                "func_type": "",
+                "kind": "RANK_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "rank",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_flush_all": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_flush_all",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_flush_local": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_flush_local",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of target \\MPI/ process in the group of the window \\mpiarg{win}",
+                "func_type": "",
+                "kind": "RANK_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "rank",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_flush_local_all": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_flush_local_all",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_free": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_free",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "win",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_free_keyval": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_free_keyval",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "key value",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "win_keyval",
+                "optional": false,
+                "param_direction": "inout",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_get_attr": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_get_attr",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "window to which the attribute is attached",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "key value",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win_keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "attribute value, unless \\mpiarg{flag}\\mpicode{ = false}",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "attribute_val",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "\\mpicode{false} if no attribute is associated with the key",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_get_errhandler": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_get_errhandler",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "error handler currently associated with window",
+                "func_type": "",
+                "kind": "ERRHANDLER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "errhandler",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_get_group": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_get_group",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "group of \\mpi/ processes that share access to the window",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "group",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_get_info": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_get_info",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new info object",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "info_used",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_get_name": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_get_name",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "window whose name is to be returned",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "the name previously stored on the window, or an empty string if no such name exists",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": "MPI_MAX_OBJECT_NAME",
+                "lis_direction": "out",
+                "name": "win_name",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "length of returned name",
+                "func_type": "",
+                "kind": "STRING_LENGTH",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "resultlen",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_lock": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_lock",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "either \\mpiconst{MPI_LOCK_EXCLUSIVE} or \\mpiconst{MPI_LOCK_SHARED}",
+                "func_type": "",
+                "kind": "LOCK_TYPE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "lock_type",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of locked window",
+                "func_type": "",
+                "kind": "RANK_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "rank",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ASSERT",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "assert",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_lock_all": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_lock_all",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ASSERT",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "assert",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_null_copy_fn": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": "MPI_Win_copy_attr_function",
+            "proxy_render": false
+        },
+        "name": "MPI_WIN_NULL_COPY_FN",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "oldwin",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win_keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val_in",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val_out",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "ierror",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_null_delete_fn": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": false,
+            "not_with_mpif": false,
+            "predefined_function": "MPI_Win_delete_attr_function",
+            "proxy_render": false
+        },
+        "name": "MPI_WIN_NULL_DELETE_FN",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win_keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "EXTRA_STATE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "extra_state",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "ierror",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_post": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_post",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "group of origin processes",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "group",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ASSERT",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "assert",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_set_attr": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_set_attr",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "window to which attribute will be attached",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "key value",
+                "func_type": "",
+                "kind": "KEYVAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win_keyval",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "attribute value",
+                "func_type": "",
+                "kind": "ATTRIBUTE_VAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "attribute_val",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_set_errhandler": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_set_errhandler",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "new error handler for window",
+                "func_type": "",
+                "kind": "ERRHANDLER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "errhandler",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_set_info": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_set_info",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "INFO",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "info",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_set_name": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_set_name",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "window whose identifier is to be set",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "inout",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": true,
+                "desc": "the character string that is remembered as the name",
+                "func_type": "",
+                "kind": "STRING",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win_name",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_shared_query": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": "\\>INTERFACE MPI\\_WIN\\_SHARED\\_QUERY \\\\ \\>\\>SUBROUTINE MPI\\_WIN\\_SHARED\\_QUERY(WIN, RANK, SIZE, DISP\\_UNIT, \\&\\\\ \\>\\>\\>\\>BASEPTR, IERROR) \\\\ \\>\\>\\>IMPORT :: MPI\\_ADDRESS\\_KIND \\\\ \\>\\>\\>INTEGER :: WIN, RANK, DISP\\_UNIT, IERROR \\\\ \\>\\>\\>INTEGER(KIND=MPI\\_ADDRESS\\_KIND) :: SIZE, BASEPTR \\\\ \\>\\>END SUBROUTINE \\\\ \\>\\>SUBROUTINE MPI\\_WIN\\_SHARED\\_QUERY\\_CPTR(WIN, RANK, SIZE, DISP\\_UNIT, \\&\\\\ \\>\\>\\>\\>BASEPTR, IERROR) \\\\ \\>\\>\\>USE, INTRINSIC :: ISO\\_C\\_BINDING, ONLY : C\\_PTR \\\\ \\>\\>\\>IMPORT :: MPI\\_ADDRESS\\_KIND \\\\ \\>\\>\\>INTEGER :: WIN, RANK, DISP\\_UNIT, IERROR \\\\ \\>\\>\\>INTEGER(KIND=MPI\\_ADDRESS\\_KIND) :: SIZE \\\\ \\>\\>\\>TYPE(C\\_PTR) :: BASEPTR \\\\ \\>\\>END SUBROUTINE \\\\ \\>END INTERFACE",
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_shared_query",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "shared memory window",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank in the group of window win or \\mpiconst{MPI_PROC_NULL}",
+                "func_type": "",
+                "kind": "RANK_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "rank",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "size of the window segment",
+                "func_type": "",
+                "kind": "WINDOW_SIZE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "size",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "local unit size for displacements, in bytes",
+                "func_type": "",
+                "kind": "POLYRMA_DISPLACEMENT",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "disp_unit",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "address for load/store access to window segment",
+                "func_type": "",
+                "kind": "C_BUFFER",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "baseptr",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_start": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_start",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "group of target processes",
+                "func_type": "",
+                "kind": "GROUP",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "group",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ASSERT",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "assert",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_sync": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_sync",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_test": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_test",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "success flag",
+                "func_type": "",
+                "kind": "LOGICAL",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "flag",
+                "optional": false,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_unlock": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_unlock",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "rank of window",
+                "func_type": "",
+                "kind": "RANK_NNI",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "rank",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_unlock_all": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_unlock_all",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_win_wait": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Win_wait",
+        "name_f90": null,
+        "parameters": [
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "WINDOW",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "in",
+                "name": "win",
+                "optional": false,
+                "param_direction": "in",
+                "pointer": null,
+                "root_only": false,
+                "suppress": ""
+            },
+            {
+                "array_type": "",
+                "asynchronous": false,
+                "constant": false,
+                "desc": "",
+                "func_type": "",
+                "kind": "ERROR_CODE",
+                "large_only": false,
+                "length": null,
+                "lis_direction": "out",
+                "name": "ierror",
+                "optional": true,
+                "param_direction": "out",
+                "pointer": null,
+                "root_only": false,
+                "suppress": "c_parameter lis_parameter"
+            }
+        ],
+        "return_kind": "ERROR_CODE"
+    },
+    "mpi_wtick": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Wtick",
+        "name_f90": null,
+        "parameters": [],
+        "return_kind": "TICK_RESOLUTION"
+    },
+    "mpi_wtime": {
+        "attributes": {
+            "c_expressible": true,
+            "callback": false,
+            "capitalized": false,
+            "deprecated": false,
+            "execute_once": false,
+            "f08_abstract_interface": true,
+            "f08_expressible": true,
+            "f90_expressible": true,
+            "f90_index_overload": null,
+            "f90_use_colons": false,
+            "index_upper": false,
+            "lis_expressible": true,
+            "not_with_mpif": false,
+            "predefined_function": null,
+            "proxy_render": false
+        },
+        "name": "MPI_Wtime",
+        "name_f90": null,
+        "parameters": [],
+        "return_kind": "WALL_TIME"
+    }
+}
diff --git a/docs/mpi-standard-apis.json b/docs/mpi-standard-apis.json
new file mode 120000
index 00000000000..795396cb200
--- /dev/null
+++ b/docs/mpi-standard-apis.json
@@ -0,0 +1 @@
+mpi-standard-4.1-apis.json
\ No newline at end of file
diff --git a/docs/no-prrte-content.rst.txt b/docs/no-prrte-content.rst.txt
new file mode 100644
index 00000000000..ea034952d31
--- /dev/null
+++ b/docs/no-prrte-content.rst.txt
@@ -0,0 +1,24 @@
+.. This file is only used in certain cases.  Hence, the original file
+   in the Open MPI "docs" source tree ends in ".txt", so that Sphinx
+   will not complain if it is not used.  If it *is* used, it is copied
+   to another file (that ends in ".rst") so that it can be properly
+   found / used by Sphinx.
+
+No content
+^^^^^^^^^^
+
+There is no meaningful content in this file because Open MPI was either:
+
+* Built without PRRTE support.
+
+* Built with a PRRTE that was too old to include machine-readable
+  documentation that could be incorporated into Open MPI's
+  documentation.
+
+If you build Open MPI with a newer version of PRRTE (and have the
+Sphinx tool available when you run Open MPI's ``configure`` command),
+you should get more meaningful documentation here.
+
+Hence, there is no documentation for this section.
+
+Sorry!
diff --git a/docs/openmpi_logo.png b/docs/openmpi_logo.png
new file mode 100644
index 00000000000..16a2e0cbab2
Binary files /dev/null and b/docs/openmpi_logo.png differ
diff --git a/docs/quickstart.rst b/docs/quickstart.rst
new file mode 100644
index 00000000000..745469b530c
--- /dev/null
+++ b/docs/quickstart.rst
@@ -0,0 +1,20 @@
+.. _label-quickstart:
+
+Quick start
+===========
+
+There are three general phases of using Open MPI: installing Open MPI,
+building MPI applications, and running MPI applications.
+
+The links below take you to "quick start" sections at the beginning of
+each chapter.  These "quick start" sections provide a good starting
+point, and may be all that many users need to read.  However, if the
+"quick start" sections are not enough, the remainder of each chapter
+provides significantly more detailed information.
+
+#. :doc:`Quick start: Installing Open MPI
+   </installing-open-mpi/quickstart>`
+#. :doc:`Quick start: Building MPI applications
+   </building-apps/quickstart>`
+#. :doc:`Quick start: Launching MPI applications
+   </launching-apps/quickstart>`
diff --git a/docs/release-notes/changelog/index.rst b/docs/release-notes/changelog/index.rst
new file mode 100644
index 00000000000..18dbd073f8d
--- /dev/null
+++ b/docs/release-notes/changelog/index.rst
@@ -0,0 +1,18 @@
+Changelog
+=========
+
+This section contains the main features as well as overviews of
+specific bug fixes (and other actions) for each version of Open MPI
+since v1.0.0.
+
+.. toctree::
+   :maxdepth: 1
+
+   v6.0.x
+   v5.0.x
+   v4.1.x
+   v4.0.x
+   v3.1.x
+   v3.0.x
+   v2.x
+   v1.x
diff --git a/docs/release-notes/changelog/v1.x.rst b/docs/release-notes/changelog/v1.x.rst
new file mode 100644
index 00000000000..b945177598c
--- /dev/null
+++ b/docs/release-notes/changelog/v1.x.rst
@@ -0,0 +1,3161 @@
+Open MPI v1.x series
+====================
+
+This file contains all the NEWS updates for all the Open MPI v1.x
+series, in reverse chronological order.
+
+Open MPI 1.10.x series
+----------------------
+
+Open MPI version 1.10.7
+^^^^^^^^^^^^^^^^^^^^^^^
+:Date: 16 May 2017
+
+- Fix bug in TCP BTL that impacted performance on 10GbE (and faster)
+  networks by not adjusting the TCP send/recv buffer sizes and using
+  system default values
+- Add missing MPI_AINT_ADD and MPI_AINT_DIFF function delcarations in
+  mpif.h
+- Fixed time reported by MPI_WTIME; it was previously reported as
+  dependent upon the CPU frequency.
+- Fix platform detection on FreeBSD
+- Fix a bug in the handling of ``MPI_TYPE_CREATE_DARRAY`` in
+  ``MPI_(R)(GET_)ACCUMULATE``
+- Fix openib memory registration limit calculation
+- Add missing MPI_T_PVAR_SESSION_NULL in mpi.h
+- Fix "make distcheck" when using external hwloc and/or libevent packages
+- Add latest ConnectX-5 vendor part id to OpenIB device params
+- Fix race condition in the UCX PML
+- Fix signal handling for rsh launcher
+- Fix Fortran compilation errors by removing MPI_SIZEOF in the Fortran
+  interfaces when the compiler does not support it
+- Fixes for the pre-ignore-TKR "mpi" Fortran module implementation
+  (i.e., for older Fortran compilers |mdash| these problems did not exist
+  in the "mpi" module implementation for modern Fortran compilers):
+  - Add ``PMPI_*`` interfaces
+  - Fix typo in MPI_FILE_WRITE_AT_ALL_BEGIN interface name
+  - Fix typo in MPI_FILE_READ_ORDERED_BEGIN interface name
+- Fixed the type of MPI_DISPLACEMENT_CURRENT in all Fortran interfaces
+  to be an INTEGER(KIND=MPI_OFFSET_KIND).
+- Fixed typos in ``MPI_INFO_GET_*`` man pages.  Thanks to Nicolas Joly for
+  the patch
+- Fix typo bugs in wrapper compiler script
+
+
+Open MPI version 1.10.6
+^^^^^^^^^^^^^^^^^^^^^^^
+:Date: 17 Feb 2017
+
+- Fix bug in timer code that caused problems at optimization settings
+  greater than 2
+- OSHMEM: make mmap allocator the default instead of sysv or verbs
+- Support MPI_Dims_create with dimension zero
+- Update USNIC support
+- Prevent 64-bit overflow on timer counter
+- Add support for forwarding signals
+- Fix bug that caused truncated messages on large sends over TCP BTL
+- Fix potential infinite loop when printing a stacktrace
+
+
+Open MPI version 1.10.5
+^^^^^^^^^^^^^^^^^^^^^^^
+:Date: 19 Dec 2016
+
+- Update UCX APIs
+- Fix bug in darray that caused MPI/IO failures
+- Use a MPI_Get_library_version() like string to tag the debugger DLL.
+  Thanks to Alastair McKinstry for the report
+- Fix multi-threaded race condition in coll/libnbc
+- Several fixes to OSHMEM
+- Fix bug in UCX support due to uninitialized field
+- Fix MPI_Ialltoallv with MPI_IN_PLACE and without MPI param check
+- Correctly reset receive request type before init. Thanks Chris Pattison
+  for the report and test case.
+- Fix bug in iallgather[v]
+- Fix concurrency issue with MPI_Comm_accept. Thanks to Pieter Noordhuis
+  for the patch
+- Fix ompi_coll_base_{gather,scatter}_intra_binomial
+- Fixed an issue with MPI_Type_get_extent returning the wrong extent
+  for distributed array datatypes.
+- Re-enable use of rtdtsc instruction as a monotonic clock source if
+  the processor has a core-invariant tsc. This is a partial fix for a
+  performance regression introduced in Open MPI v1.10.3.
+
+
+Open MPI version 1.10.4
+^^^^^^^^^^^^^^^^^^^^^^^
+:Date: 10 Sep 2016
+
+- Fix assembler support for MIPS
+- Improve memory handling for temp buffers in collectives
+- Fix [all]reduce with non-zero lower bound datatypes
+  Thanks Hristo Iliev for the report
+- Fix non-standard ddt handling. Thanks Yuki Matsumoto for the report
+- Various libnbc fixes. Thanks Yuki Matsumoto for the report
+- Fix typos in request RMA bindings for Fortran. Thanks to @alazzaro
+  and @vondele for the assist
+- Various bug fixes and enhancements to collective support
+- Fix predefined types mapping in hcoll
+- Revive the coll/sync component to resolve unexpected message issues
+  during tight loops across collectives
+- Fix typo in wrapper compiler for Fortran static builds
+
+
+Open MPI version 1.10.3
+^^^^^^^^^^^^^^^^^^^^^^^
+:Date: 15 June 2016
+
+- Fix zero-length datatypes.  Thanks to Wei-keng Liao for reporting
+  the issue.
+- Minor manpage cleanups
+- Implement atomic support in OSHMEM/UCX
+- Fix support of MPI_COMBINER_RESIZED. Thanks to James Ramsey
+  for the report
+- Fix computation of #cpus when ``--use-hwthread-cpus`` is used
+- Add entry points for Allgatherv, iAllgatherv, Reduce, and iReduce
+  for the HCOLL library
+- Fix an HCOLL integration bug that could signal completion of request
+  while still being worked
+- Fix computation of cores when SMT is enabled. Thanks to Ben Menadue
+  for the report
+- Various USNIC fixes
+- Create a datafile in the per-proc directory in order to make it
+  unique per communicator. Thanks to Peter Wind for the report
+- Fix zero-size malloc in one-sided pt-to-pt code. Thanks to Lisandro
+  Dalcín for the report
+- Fix MPI_Get_address when passed MPI_BOTTOM to not return an error.
+  Thanks to Lisandro Dalcín for the report
+- Fix MPI_TYPE_SET_ATTR with NULL value. Thanks to Lisandro Dalcín for
+  the report
+- Fix various Fortran08 binding issues
+- Fix memchecker no-data case. Thanks to Clinton Stimpson for the report
+- Fix CUDA support under OS-X
+- Fix various OFI/MTL integration issues
+- Add MPI_T man pages
+- Fix one-sided pt-to-pt issue by preventing communication from happening
+  before a target enters a fence, even in the no-precede case
+- Fix a bug that disabled Totalview for MPMD use-case
+- Correctly support MPI_UNWEIGHTED in topo-graph-neighbors. Thanks to
+  Jun Kudo for the report
+- Fix singleton operations under SLURM when PMI2 is enabled
+- Do not use MPI_IN_PLACE in neighborhood collectives for non-blocking
+  collectives (libnbc). Thanks to Jun Kudo for the report
+- Silence autogen deprecation warnings for newer versions of Perl
+- Do not return MPI_ERR_PENDING from collectives
+- Use type ``int*`` for MPI_WIN_DISP_UNIT, MPI_WIN_CREATE_FLAVOR, and MPI_WIN_MODEL.
+  Thanks to Alastair McKinstry for the report
+- Fix register_datarep stub function in IO/OMPIO. Thanks to Eric
+  Chamberland for the report
+- Fix a bus error on MPI_WIN_[POST,START] in the shared memory one-sided component
+- Add several missing MPI_WIN_FLAVOR constants to the Fortran support
+- Enable connecting processes from different subnets using the openib BTL
+- Fix bug in basic/barrier algorithm in OSHMEM
+- Correct process binding for the ``--map-by node`` case
+- Include support for subnet-to-subnet routing over InfiniBand networks
+- Fix usnic resource check
+- AUTHORS - Fix an errant reference to Subversion IDs
+- Fix affinity for MPMD jobs running under LSF
+- Fix many Fortran binding bugs
+- Fix ``MPI_IN_PLACE`` related bugs
+- Fix PSM/PSM2 support for singleton operations
+- Ensure MPI transports continue to progress during RTE barriers
+- Update HWLOC to 1.9.1 end-of-series
+- Fix a bug in the Java command line parser when the
+  ``-Djava.library.path`` options was given by the user
+- Update the MTL/OFI provider selection behavior
+- Add support for clock_gettime on Linux.
+- Correctly detect and configure for Solaris Studio 12.5
+  beta compilers
+- Correctly compute #slots when -host is used for MPMD case
+- Fix a bug in the hcoll collectives due to an uninitialized field
+- Do not set a binding policy when oversubscribing a node
+- Fix hang in intercommunicator operations when oversubscribed
+- Speed up process termination during MPI_Abort
+- Disable backtrace support by default in the PSM/PSM2 libraries to
+  prevent unintentional conflicting behavior.
+
+
+Open MPI version 1.10.2
+^^^^^^^^^^^^^^^^^^^^^^^
+:Date: 26 Jan 2016
+
+- OSHMEM is now 1.2 compliant
+- Fix NBC_Copy for legitimate zero-size messages
+- Fix multiple bugs in OSHMEM
+- Correctly handle ``mpirun --host <user>@<ip-address>``
+- Centralize two MCA params to avoid duplication between OMPI and
+  OSHMEM layers: opal_abort_delay and opal_abort_print_stack
+- Add support for Fujitsu compilers
+- Add UCX support for OMPI and OSHMEM
+- Correctly handle oversubscription when not given directives
+  to permit it. Thanks to @ammore1 for reporting it
+- Fix rpm spec file to not include the /usr directory
+- Add Intel HFI1 default parameters for the openib BTL
+- Resolve symbol conflicts in the PSM2 library
+- Add ability to empty the rgpusm cache when full if requested
+- Fix another libtool bug when -L requires a space between it
+  and the path. Thanks to Eric Schnetter for the patch.
+- Add support for OSHMEM v1.2 APIs
+- Improve efficiency of oshmem_preconnect_all algorithm
+- Fix bug in buffered sends support
+- Fix double free in edge case of mpirun. Thanks to @jsharpe for
+  the patch
+- Multiple one-sided support fixes
+- Fix integer overflow in the tuned "reduce" collective when
+  using buffers larger than INT_MAX in size
+- Fix parse of user environment variables in mpirun. Thanks to
+  Stefano Garzarella for the patch
+- Performance improvements in PSM2 support
+- Fix NBS iBarrier for inter-communicators
+- Fix bug in vader BTL during finalize
+- Improved configure support for Fortran compilers
+- Fix rank_file mapper to support default ``--slot-set``. Thanks
+  to Matt Thompson for reporting it
+- Update MPI_Testsome man page. Thanks to Eric Schnetter for
+  the suggestion
+- Fix missing resize of the returned type for subarray and
+  darray types. Thanks to Keith Bennett and Dan Garmann for
+  reporting it
+- Fix Java support on OSX 10.11. Thanks to Alexander Daryin
+  for reporting the problem
+- Fix some compilation issues on Solaris 11.2. Thanks to
+  Paul Hargrove for his continued help in such areas
+
+
+Open MPI version 1.10.1
+^^^^^^^^^^^^^^^^^^^^^^^
+:Date: 4 Nov 2015
+
+- Workaround an optimization problem with gcc compilers >= 4.9.2 that
+  causes problems with memory registration, and forced
+  mpi_leave_pinned to default to 0 (i.e., off).  Thanks to @oere for
+  the fix.
+- Fix use of MPI_LB and MPI_UB in subarray and darray datatypes.
+  Thanks to Gus Correa and Dimitar Pashov for pointing out the issue.
+- Minor updates to mpi_show_mpi_alloc_mem_leaks and
+  ompi_debug_show_handle_leaks functionality.
+- Fix segv when invoking non-blocking reductions with a user-defined
+  operation.  Thanks to Rupert Nash and Georg Geiser for identifying
+  the issue.
+- No longer probe for PCI topology on Solaris (unless running as root).
+- Fix for Intel Parallel Studio 2016 ifort partial support of the
+  !GCC$ pragma.  Thanks to Fabrice Roy for reporting the problem.
+- Bunches of Coverity / static analysis fixes.
+- Fixed ROMIO to look for lstat in <sys/stat.h>.  Thanks to William
+  Throwe for submitting the patch both upstream and to Open MPI.
+- Fixed minor memory leak when attempting to open plugins.
+- Fixed type in MPI_IBARRIER C prototype.  Thanks to Harald Servat for
+  reporting the issue.
+- Add missing man pages for MPI_WIN_CREATE_DYNAMIC, MPI_WIN_ATTACH,
+  MPI_WIN_DETACH, MPI_WIN_ALLOCATE, MPI_WIN_ALLOCATE_SHARED.
+- When mpirun-launching new applications, only close file descriptors
+  that are actually open (resulting in a faster launch in some
+  environments).
+- Fix "test ==" issues in Open MPI's configure script.  Thank to Kevin
+  Buckley for pointing out the issue.
+- Fix performance issue in usnic BTL: ensure progress thread is
+  throttled back to not aggressively steal CPU cycles.
+- Fix cache line size detection on POWER architectures.
+- Add missing #include in a few places.  Thanks to Orion Poplawski for
+  supplying the patch.
+- When OpenSHMEM building is disabled, no longer install its header
+  files, help files, or man pages.  Add man pages for oshrun, oshcc,
+  and oshfort.
+- Fix mpi_f08 implementations of MPI_COMM_SET_INFO, and profiling
+  versions of MPI_BUFFER_DETACH, MPI_WIN_ALLOCATE,
+  MPI_WIN_ALLOCATE_SHARED, MPI_WTICK, and MPI_WTIME.
+- Add orte_rmaps_dist_device MCA param, allowing users to map near a
+  specific device.
+- Various updates/fixes to the openib BTL.
+- Add missing defaults for the Mellanox ConnectX 3 card to the openib BTL.
+- Minor bug fixes in the OFI MTL.
+- Various updates to Mellanox's MXM, hcoll, and FCA components.
+- Add OpenSHMEM man pages.  Thanks to Tony Curtis for sharing the man
+  pages files from openshmem.org.
+- Add missing "const" attributes to MPI_COMPARE_AND_SWAP,
+  MPI_FETCH_AND_OP, MPI_RACCUMULATE, and MPI_WIN_DETACH prototypes.
+  Thanks to Michael Knobloch and Takahiro Kawashima for bringing this
+  to our attention.
+- Fix linking issues on some platforms (e.g., SLES 12).
+- Fix hang on some corner cases when MPI applications abort.
+- Add missing options to mpirun man page. Thanks to Daniel Letai
+  for bringing this to our attention.
+- Add new ``--with-platform-patches-dir`` configure option
+- Adjust relative selection priorities to ensure that MTL
+  support is favored over BTL support when both are available
+- Use CUDA IPC for all sized messages for performance
+
+
+Open MPI version 1.10.0
+^^^^^^^^^^^^^^^^^^^^^^^
+:Date: 25 Aug 2015
+
+.. important::
+   NOTE: The v1.10.0 release marks the transition to Open MPI's new
+   version numbering scheme.  The v1.10.x release series is based on
+   the v1.8.x series, but with a few new features.  v2.x will be the
+   next series after the v1.10.x series, and complete the transition
+   to the new version numbering scheme.  See README for more details
+   on the new versioning scheme.
+
+.. note::
+   In accordance with OMPI version numbering, the v1.10 is **not**
+   API compatible with the v1.8 release series.
+
+- Added libfabric support (see README for more details):
+- usNIC BTL updated to use libfabric.
+- Added OFI MTL (usable with PSM in libfabric v1.1.0).
+- Added Intel Omni-Path support via new PSM2 MTL.
+- Added "yalla" PML for faster MXM support.
+- Removed support for MX
+- Added persistent distributed virtual machine (pDVM) support for fast
+  workflow executions.
+- Fixed typo in GCC inline assembly introduced in Open MPI v1.8.8.
+  Thanks to Paul Hargrove for pointing out the issue.
+- Add missing man pages for MPI_Win_get|set_info(3).
+- Ensure that session directories are cleaned up at the end of a run.
+- Fixed linking issues on some OSs where symbols of dependent
+  libraries are not automatically publicly available.
+- Improve hcoll and fca configury library detection.  Thanks to David
+  Shrader for helping track down the issue.
+- Removed the LAMA mapper (for use in setting affinity).  Its
+  functionality has been largely superseded by other mpirun CLI
+  options.
+- CUDA: Made the asynchronous copy mode be the default.
+- Fix a malloc(0) warning in MPI_IREDUCE_SCATTER_BLOCK.  Thanks to
+  Lisandro Dalcín for reporting the issue.
+- Fix typo in MPI_Scatter(3) man page.  Thanks to Akshay Venkatesh for
+  noticing the mistake.
+- Add rudimentary protection from TCP port scanners.
+- Fix typo in Open MPI error handling.  Thanks to Åke Sandgren for
+  pointing out the error.
+- Increased the performance of the CM PML (i.e., the Portals, PSM,
+  PSM2, MXM, and OFI transports).
+- Restored visibility of blocking send requests in message queue
+  debuggers (e.g., TotalView, DDT).
+- Fixed obscure IPv6-related bug in the TCP BTL.
+- Add support for the "no_locks" MPI_Info key for one-sided
+  functionality.
+- Fixed ibv_fork support for verbs-based networks.
+- Fixed a variety of small bugs in OpenSHMEM.
+- Fixed MXM configure with additional CPPFLAGS and LDFLAGS.  Thanks to
+  David Shrader for the patch.
+- Fixed incorrect memalign threshhold in the openib BTL.  Thanks to
+  Xavier Besseron for pointing out the issue.
+
+
+Open MPI 1.8.x series
+---------------------
+
+Open MPI version 1.8.8
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 5 Aug 2015
+
+- Fix a segfault in MPI_FINALIZE with the PSM MTL.
+- Fix mpi_f08 sentinels (e.g., MPI_STATUS_IGNORE) handling.
+- Set some additional MXM default values for OSHMEM.
+- Fix an invalid memory access in MPI_MRECV and MPI_IMRECV.
+- Include two fixes that were mistakenly left out of the official
+  v1.8.7 tarball:
+- Fixed MPI_WIN_POST and MPI_WIN_START for zero-size messages
+- Protect the OOB TCP ports from segfaulting when accessed by port
+  scanners
+
+
+Open MPI version 1.8.7
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 15 Jul 2015
+
+.. note:: v1.8.7 technically breaks ABI with prior versions
+   in the 1.8 series because it repairs two incorrect API
+   signatures. However, users will only need to recompile
+   if they were using those functions - which they couldn't
+   have been, because the signatures were wrong :-)
+
+- Plugged a memory leak that impacted blocking sends
+- Fixed incorrect declaration for MPI_T_pvar_get_index and added
+  missing return code MPI_T_INVALID_NAME.
+- Fixed an uninitialized variable in PMI2 support
+- Added new vendor part id for Mellanox ConnectX4-LX
+- Fixed NBC_Copy for legitimate zero-size messages
+- Fixed MPI_Win_post and MPI_Win_start for zero-size messages
+- Protect the OOB ports from segfaulting when accessed by port scanners
+- Fixed several Fortran typos
+- Fixed configure detection of XRC support
+- Fixed support for highly heterogeneous systems to avoid
+  memory corruption when printing out the bindings
+
+
+Open MPI version 1.8.6
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 17 Jun 2015
+
+- Fixed memory leak on Mac OS-X exposed by TCP keepalive
+- Fixed keepalive support to ensure that daemon/node failure
+  results in complete job cleanup
+- Update Java binding support
+- Fixed MPI_THREAD_MULTIPLE bug in vader shared memory BTL
+- Fixed issue during shutdown when CUDA initialization wasn't complete
+- Fixed orted environment when no prefix given
+- Fixed trivial typo in MPI_Neighbor_allgather manpage
+- Fixed tree-spawn support for sh and ksh shells
+- Several data type fixes
+- Fixed IPv6 support bug
+- Cleaned up an unlikely build issue
+- Fixed PMI2 process map parsing for cyclic mappings
+- Fixed memalign threshold in openib BTL
+- Fixed debugger access to message queues for blocking send/recv
+
+
+Open MPI version 1.8.5
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 5 May 2015
+
+- Fixed configure problems in some cases when using an external hwloc
+  installation.  Thanks to Erick Schnetter for reporting the error and
+  helping track down the source of the problem.
+- Fixed linker error on OS X when using the clang compiler.  Thanks to
+  Erick Schnetter for reporting the error and helping track down the
+  source of the problem.
+- Fixed MPI_THREAD_MULTIPLE deadlock error in the vader BTL.  Thanks
+  to Thomas Klimpel for reporting the issue.
+- Fixed several Valgrind warnings.  Thanks for Lisandro Dalcín for
+  contributing a patch fixing some one-sided code paths.
+- Fixed version compatibility test in OOB that broke ABI within the
+  1.8 series. NOTE: this will not resolve the problem between pre-1.8.5
+  versions, but will fix it going forward.
+- Fix some issues related to running on Intel Xeon Phi coprocessors.
+- Opportunistically switch away from using GNU Libtool's libltdl
+  library when possible (by default).
+- Fix some VampirTrace errors.  Thanks to Paul Hargrove for reporting
+  the issues.
+- Correct default binding patterns when ``--use-hwthread-cpus`` was
+  specified and nprocs <= 2.
+- Fix warnings about -finline-functions when compiling with clang.
+- Updated the embedded hwloc with several bug fixes, including the
+  "duplicate Lhwloc1 symbol" that multiple users reported on some
+  platforms.
+- Do not error when mpirun is invoked with with default bindings
+  (i.e., no binding was specified), and one or more nodes do not
+  support bindings.  Thanks to Annu Desari for pointing out the
+  problem.
+- Let root invoke ``mpirun --version`` to check the version without
+  printing the "Don't run as root!" warnings.  Thanks to Robert McLay
+  for the suggestion.
+- Fixed several bugs in OpenSHMEM support.
+- Extended vader shared memory support to 32-bit architectures.
+- Fix handling of very large datatypes.  Thanks to Bogdan Sataric for
+  the bug report.
+- Fixed a bug in handling subarray MPI datatypes, and a bug when using
+  MPI_LB and MPI_UB.  Thanks to Gus Correa for pointing out the issue.
+- Restore user-settable bandwidth and latency PML MCA variables.
+- Multiple bug fixes for cleanup during MPI_FINALIZE in unusual
+  situations.
+- Added support for TCP keepalive signals to ensure timely termination
+  when sockets between daemons cannot be created (e.g., due to a
+  firewall).
+- Added MCA parameter to allow full use of a SLURM allocation when
+  started from a tool (supports LLNL debugger).
+- Fixed several bugs in the configure logic for PMI and hwloc.
+- Fixed incorrect interface index in TCP communications setup.  Thanks
+  to Mark Kettenis for spotting the problem and providing a patch.
+- Fixed MPI_IREDUCE_SCATTER with single-process communicators when
+  MPI_IN_PLACE was not used.
+- Added XRC support for OFED v3.12 and higher.
+- Various updates and bug fixes to the Mellanox hcoll collective
+  support.
+- Fix problems with Fortran compilers that did not support
+  ``REAL*16``/``COMPLEX*32`` types.  Thanks to Orion Poplawski for
+  identifying the issue.
+- Fixed problem with rpath/runpath support in pkg-config files.
+  Thanks to Christoph Junghans for notifying us of the issue.
+- Man page fixes:
+
+   - Removed erroneous "color" discussion from MPI_COMM_SPLIT_TYPE.
+     Thanks to Erick Schnetter for spotting the outdated text.
+   - Fixed prototypes for MPI_IBARRIER.  Thanks to Maximilian for
+     finding the issue.
+   - Updated docs about buffer usage in non-blocking communications.
+     Thanks to Alexander Pozdneev for citing the outdated text.
+   - Added documentation about the 'ompi_unique' MPI_Info key with
+     MPI_PUBLISH_NAME.
+   - Fixed typo in MPI_INTERCOMM_MERGE.  Thanks to Harald Servat for
+     noticing and sending a patch.
+   - Updated configure paths in HACKING.  Thanks to Maximilien Levesque
+     for the fix.
+   - Fixed Fortran typo in MPI_WIN_LOCK_ALL.  Thanks to Thomas Jahns
+     for pointing out the issue.
+
+- Fixed a number of MPI one-sided bugs.
+- Fixed MPI_COMM_SPAWN when invoked from a singleton job.
+- Fixed a number of minor issues with CUDA support, including
+  registering of shared memory and supporting reduction support for
+  GPU buffers.
+- Improved support for building OMPI on Cray platforms.
+- Fixed performance regression introduced by the inadvertent default
+  enabling of MPI_THREAD_MULTIPLE support.
+
+
+Open MPI version 1.8.4
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 19 Dec 2014
+
+- Fix MPI_SIZEOF; now available in mpif.h for modern Fortran compilers
+  (see README for more details).  Also fixed various compiler/linker
+  errors.
+- Fixed inadvertant Fortran ABI break between v1.8.1 and v1.8.2 in the
+  mpi interface module when compiled with gfortran >= v4.9.
+- Fix various MPI_THREAD_MULTIPLE issues in the TCP BTL.
+- mpirun no longer requires the ``--hetero-nodes`` switch; it will
+  automatically detect when running in heterogeneous scenarios.
+- Update LSF support, to include revamped affinity functionality.
+- Update embedded hwloc to v1.9.1.
+- Fixed max registerable memory computation in the openib BTL.
+- Updated error message when debuggers are unable to find various
+  symbols/types to be more clear.  Thanks to Dave Love for raising the
+  issue.
+- Added proper support for LSF and PBS/Torque libraries in static builds.
+- Rankfiles now support physical processor IDs.
+- Fixed potential hang in MPI_ABORT.
+- Fixed problems with the PSM MTL and "re-connect" scenarios, such as
+  MPI_INTERCOMM_CREATE.
+- Fix MPI_IREDUCE_SCATTER with a single process.
+- Fix (rare) race condition in stdout/stderr funneling to mpirun where
+  some trailing output could get lost when a process terminated.
+- Removed inadvertent change that set ``--enable-mpi-thread-multiple``
+  "on" by default, thus impacting performance for non-threaded apps.
+- Significantly reduced startup time by optimizing internal hash table
+  implementation.
+- Fixed OS X linking with the Fortran mpi module when used with
+  gfortran >= 4.9.  Thanks to Github user yafshar for raising the
+  issue.
+- Fixed memory leak on Cygwin platforms.  Thanks for Marco Atzeri for
+  reporting the issue.
+- Fixed seg fault in neighborhood collectives when the degree of the
+  topology is higher than the communicator size.  Thanks to Lisandro
+  Dalcín for reporting the issue.
+- Fixed segfault in neighborhood collectives under certain use-cases.
+- Fixed various issues regarding Solaris support.  Thanks to Siegmar
+  Gross for patiently identifying all the issues.
+- Fixed PMI configure tests for certain Slurm installation patterns.
+- Fixed param registration issue in Java bindings.  Thanks to Takahiro
+  Kawashima and Siegmar Gross for identifying the issue.
+- Several man page fixes.
+- Silence several warnings and close some memory leaks (more remain,
+  but it's better than it was).
+- Re-enabled the use of CMA and knem in the shared memory BTL.
+- Updated mpirun manpage to correctly explain new map/rank/binding options.
+- Fixed MPI_IALLGATHER problem with intercommunicators.  Thanks for
+  Takahiro Kawashima for the patch.
+- Numerous updates and performance improvements to OpenSHMEM.
+- Turned off message coalescing in the openib BTL until a proper fix
+  for that capability can be provided (tentatively expected for 1.8.5)
+- Fix a bug in iof output that dates back to the dinosaurs which would
+  output extra bytes if the system was very heavily loaded
+- Fix a bug where specifying mca_component_show_load_errors=0 could
+  cause ompi_info to segfault
+- Updated valgrind suppression file
+
+
+Open MPI version 1.8.3
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 26 Sep 2014
+
+- Fixed application abort bug to ensure that MPI_Abort exits appropriately
+  and returns the provided exit status
+- Fixed some alignment (not all) issues identified by Clang
+- Allow CUDA-aware to work with nonblocking collectives. Forces packing to
+  happen when using GPU buffers.
+- Fixed configure test issue with Intel 2015 Fortran compiler
+- Fixed some PGI-related errors
+- Provide better help message when encountering a firewall
+- Fixed MCA parameter quoting to protect multi-word params and params
+  that contain special characters
+- Improved the bind-to help message to clarify the defaults
+- Add new MPI-3.1 tools interface
+- Several performance optimizations and memory leak cleanups
+- Turn off the coll/ml plugin unless specifically requested as it
+  remains in an experimental state
+- Fix LSF support by adding required libraries for the latest LSF
+  releases.  Thanks to Joshua Randal for supplying the initial
+  patches.
+
+
+Open MPI version 1.8.2
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 25 Aug 2014
+
+- Fix auto-wireup of OOB, allowing ORTE to automatically
+  test all available NICs
+- "Un-deprecate" pernode, npernode, and npersocket options
+  by popular demand
+- Add missing Fortran bindings for MPI_WIN_LOCK_ALL,
+  MPI_WIN_UNLOCK_ALL, and MPI_WIN_SYNC.
+- Fix cascading/over-quoting in some cases with the rsh/ssh-based
+  launcher.  Thanks to multiple users for raising the issue.
+- Properly add support for gfortran 4.9 ignore TKR pragma (it was
+  erroneously only partially added in v1.7.5).  Thanks to Marcus
+  Daniels for raising the issue.
+- Update/improve help messages in the usnic BTL.
+- Resolve a race condition in MPI_Abort.
+- Fix obscure cases where static linking from wrapper compilers would
+  fail.
+- Clarify the configure ``--help`` message about when OpenSHMEM is
+  enabled/disabled by default.  Thanks to Paul Hargrove for the
+  suggestion.
+- Align pages properly where relevant.  Thanks to Paul Hargrove for
+  identifying the issue.
+- Various compiler warning and minor fixes for OpenBSD, FreeBSD, and
+  Solaris/SPARC.  Thanks to Paul Hargrove for the patches.
+- Properly pass function pointers from Fortran to C in the mpi_f08
+  module, thereby now supporting gfortran 4.9.  Thanks to Tobias
+  Burnus for assistance and testing with this issue.
+- Improve support for Cray CLE 5.
+- Fix mpirun regression: ensure exit status is non-zero if mpirun is
+  terminated due to signal.
+- Improved CUDA efficiency of asynchronous copies.
+- Fix to parameter type in MPI_Type_indexed.3.  Thanks to Bastian
+  Beischer for reporting the mistake.
+- Fix NUMA distance calculations in the openib BTL.
+- Decrease time required to shut down mpirun at the end of a job.
+- More RMA fixes.
+- More hostfile fixes from Tetsuya Mishima.
+- Fix darray issue where UB was not computed correctly.
+- Fix mpi_f08 parameter name for MPI_GET_LIBRARY_VERSION.  Thanks to
+  Junchao Zhang for pointing out the issue.
+- Ensure mpirun aborts properly when unable to map processes in
+  scheduled environments.
+- Ensure that MPI RMA error codes show up properly.  Thanks to
+  Lisandro Dalcín for reporting the issue.
+- Minor bug fixes and improvements to the bash and zsh mpirun
+  autocompletion scripts.
+- Fix sequential mpirun process mapper.  Thanks to Bill Chen for
+  reporting the issue.
+- Correct SLURM stdout/stderr redirection.
+- Added missing portals 4 files.
+- Performance improvements for blocking sends and receives.
+- Lots of cleanup to the ml collective component
+- Added new Java methods to provide full MPI coverage
+- Many OSHMEM cleanups
+- Prevent comm_spawn from automatically launching a VM across
+  all available nodes
+- Close many memory leaks to achieve valgrind-clean operation
+- Better handling of TCP connection discovery for mismatched networks
+  where we don't have a direct 1:1 subnet match between nodes
+- Prevent segfault when OMPI info tools are used in pipes and user
+  exits one step of that pipe before completing output
+
+
+Open MPI version 1.8.1
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 23 Apr 2014
+
+- Fix for critical bug: mpirun removed files (but not directories)
+  from / when run as root.  Thanks to Jay Fenlason and Orion Poplawski
+  for bringing the issue to our attention and helping identify the
+  fix.
+
+
+Open MPI version 1.8.0
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 31 Mar 2014
+
+- Commit upstream ROMIO fix for mixed NFS+local filesystem environments.
+- Several fixes for MPI-3 one-sided support.  For example,
+  arbitrary-length datatypes are now supported.
+- Add config support for the Mellanox ConnectX 4 card.
+- Add missing MPI_COMM_GET|SET_INFO functions, and missing
+  MPI_WEIGHTS_EMPTY and MPI_ERR_RMA_SHARED constants.  Thanks to
+  Lisandro Dalcín for pointing out the issue.
+- Update some help messages in OSHMEM, the usnic BTL, the TCP BTL, and
+  ORTE, and update documentation about ompi_info's ``--level`` option.
+- Fix some compiler warnings.
+- Ensure that ORTE daemons are not bound to a single processor
+  if TaskAffinity is set on by default in Slurm. Thanks to Artem Polyakov
+  for identifying the problem and providing a patch
+
+
+Open MPI 1.7.x series
+---------------------
+
+Open MPI version 1.7.5
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 20 Mar 2014
+
+.. attention::
+   Open MPI is now fully MPI-3.0 compliant
+
+- Add Linux OpenSHMEM support built on top of Open MPI's MPI
+  layer. Thanks to Mellanox for contributing this new feature.
+- Allow restricting ORTE daemons to specific cores using the
+  orte_daemon_cores MCA param.
+- Ensure to properly set "locality" flags for processes launched via
+  MPI dynamic functions such as MPI_COMM_SPAWN.
+- Fix MPI_GRAPH_CREATE when nnodes is smaller than the size of the old
+  communicator.
+- usnic BTL now supports underlying UDP transport.
+- usnic BTL now checks for common connectivty errors at first send to
+  a remote server.
+- Minor scalability improvements in the usnic BTL.
+- ompi_info now lists whether the Java MPI bindings are available or not.
+- MPI-3: mpi.h and the Fortran interfaces now report MPI_VERSION==3
+  and MPI_SUBVERSION==0.
+- MPI-3: Added support for new RMA functions and functionality.
+- Fix MPI_Info "const buglet.  Thanks to Orion Poplawski for
+  identifying the issue.
+- Multiple fixes to mapping/binding options. Thanks to Tetsuya Mishima
+  for his assistance.
+- Multiple fixes for normal and abnormal process termination,
+  including singleton MPI_Abort and ensuring to kill entire process
+  groups when abnormally terminating a job.
+- Fix DESTDIR install for javadocs.  Thanks to Orion Poplawski for
+  pointing out the issue.
+- Various performance improvements for the MPI Java bindings.
+- OMPI now uses its own internal random number generator and will not
+  perturb srand() and friends.
+- Some cleanups for Cygwin builds.  Thanks to Marco Atzeri for the
+  patches.
+- Add a new collective component (coll/ml) that provides substantially
+  improved performance.  It is still experimental, and requires
+  setting coll_ml_priority > 0 to become active.
+- Add version check during startup to ensure you are using the same
+  version of Open MPI on all nodes in a job.
+- Significantly improved the performance of MPI_DIMS_CREATE for large
+  values.  Thanks to Andreas Schäfer for the contribution.
+- Removed ASYNCHRONOUS keyword from the "ignore TKR" mpi_f08 module.
+- Deprecated the following mpirun options:
+  ``--bynode, --bycore, --byslot``: replaced with ``--map-by node|core|slot``.
+  ``--npernode, --npersocket``: replaced with ``--map-by ppr:N:node`` and
+  ``--map-by ppr:N:socket``, respectively
+- Pick NFS "infinitely stale" fix from ROMIO upstream.
+- Various PMI2 fixes and extension to support broader range of mappings.
+- Improve launch performance at large scale.
+- Add support for PBS/Torque environments that set environment
+  variables to indicate the number of slots available on each nodes.
+  Set the ras_tm_smp MCA parameter to "1" to enable this mode.
+- Add new, more scalable endpoint exchange (commonly called "modex")
+  method that only exchanges endpoint data on a per-peer basis
+  on first message. Not all transports have been updated to use
+  this feature. Set the rte_orte_direct_modex parameter to "1"
+  to enable this mode.
+
+
+Open MPI version 1.7.4
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 5 Feb 2014
+
+.. important::
+   As of release 1.7.4, OpenMPI's default mapping, ranking, and binding
+   settings have changed:
+
+   - Mapping:
+
+     * if #procs <= 2, default to map-by core
+     * if #procs > 2, default to map-by socket
+
+   - Ranking:
+
+     * if default mapping is used, then default to rank-by slot
+     * if map-by <obj> is given, then default to rank-by <obj>,
+       where <obj> is whatever object we mapped against
+
+   - Binding:
+
+     * default to bind-to core
+
+   - Users can override any of these settings individually using the
+     corresponding MCA parameter. Note that multi-threaded applications
+     in particular may want to override at least the binding default
+     to allow threads to use multiple cores.
+
+- Restore version number output in ``ompi_info --all``.
+- Various bug fixes for the mpi_f08 Fortran bindings.
+- Fix ROMIO compile error with Lustre 2.4.  Thanks to Adam Moody for
+  reporting the issue.
+- Various fixes for 32 bit platforms.
+- Add ability to selectively disable building the mpi or mpi_f08
+  module.  See the README file for details.
+- Fix MX MTL finalization issue.
+- Fix ROMIO issue when opening a file with MPI_MODE_EXCL.
+- Fix PowerPC and MIPS assembly issues.
+- Various fixes to the hcoll and FCA collective offload modules.
+- Prevent integer overflow when creating datatypes.  Thanks to
+  original patch from Gilles Gouaillardet.
+- Port some upstream hwloc fixes to Open MPI's embedded copy for
+  working around buggy NUMA node cpusets and including mising header
+  files.  Thanks to Jeff Becker and Paul Hargrove for reporting the
+  issues.
+- Fix recursive invocation issues in the MXM MTL.
+- Various bug fixes to the new MCA parameter back-end system.
+- Have the posix fbtl module link against -laio on NetBSD platforms.
+  Thanks to Paul Hargrove for noticing the issue.
+- Various updates and fixes to network filesystem detection to support
+  more operating systems.
+- Add gfortran v4.9 "ignore TKR" syntax to the mpi Fortran module.
+- Various compiler fixes for several BSD-based platforms.  Thanks to
+  Paul Hargrove for reporting the issues.
+- Fix when MPI_COMM_SPAWN[_MULTIPLE] is used on oversubscribed
+  systems.
+- Change the output from ``--report`` bindings to simply state that a
+  process is not bound, instead of reporting that it is bound to all
+  processors.
+- Per MPI-3.0 guidance, remove support for all MPI subroutines with
+  choice buffers from the TKR-based mpi Fortran module.  Thanks to Jed
+  Brown for raising the issue.
+- Only allow the usnic BTL to build on 64 bit platforms.
+- Various bug fixes to SLURM support, to include ensuring proper
+  exiting on abnormal termination.
+- Ensure that MPI_COMM_SPAWN[_MULTIPLE] jobs get the same mapping
+  directives that were used with mpirun.
+- Fixed the application of TCP_NODELAY.
+- Change the TCP BTL to not warn if a non-existent interface is
+  ignored.
+- Restored the "--bycore" mpirun option for backwards compatibility.
+- Fixed debugger attach functionality.  Thanks to Ashley Pittman for
+  reporting the issue and suggesting the fix.
+- Fixed faulty MPI_IBCAST when invoked on a communicator with only
+  one process.
+- Add new Mellanox device IDs to the openib BTL.
+- Progress towards cleaning up various internal memory leaks as
+  reported by Valgrind.
+- Fixed some annoying flex-generated warnings that have been there for
+  years.  Thanks to Tom Fogal for the initial patch.
+- Support user-provided environment variables via the "env" info key
+  to MPI_COMM_SPAWN[_MULTIPLE].  Thanks to Tom Fogal for the feature
+  request.
+- Fix uninitialized variable in MPI_DIST_GRAPH_CREATE.
+- Fix a variety of memory errors on SPARC platforms.  Thanks to
+  Siegmar Gross for reporting and testing all the issues.
+- Remove Solaris threads support.  When building on Solaris, pthreads
+  will be used.
+- Correctly handle the convertor internal stack for persistent
+  receives.  Thanks to Guillaume Gouaillardet for identifying the
+  problem.
+- Add support for using an external libevent via ``--with-libevent``.
+  See the README for more details.
+- Various OMPIO updates and fixes.
+- Add support for the MPIEXEC_TIMEOUT environment variable.  If set,
+  mpirun will terminate the job after this many seconds.
+- Update the internal copy of ROMIO to that which shipped in MPICH
+  3.0.4.
+- Various performance tweaks and improvements in the usnic BTL,
+  including now reporting MPI_T performance variables for each usnic
+  device.
+- Fix to not access send datatypes for non-root processes with
+  MPI_ISCATTER[V] and MPI_IGATHER[V].  Thanks to Pierre Jolivet for
+  supplying the initial patch.
+- Update VampirTrace to 5.14.4.9.
+- Fix ptmalloc2 hook disable when used with ummunotify.
+- Change the default connection manager for the openib BTL to be based
+  on UD verbs data exchanges instead of ORTE OOB data exchanges.
+- Fix Fortran compile error when compiling with 8-byte INTEGERs and
+  4-byte ints.
+- Fix C++11 issue identified by Jeremiah Willcock.
+- Many changes, updates, and bug fixes to the ORTE run-time layer.
+- Correctly handle MPI_REDUCE_SCATTER with recvcounts of 0.
+- Update man pages for MPI-3, and add some missing man pages for
+  MPI-2.x functions.
+- Updated mpi_f08 module in accordance with post-MPI-3.0 errata which
+  basically removed BIND(C) from all interfaces.
+- Fixed MPI_IN_PLACE detection for MPI_SCATTER[V] in Fortran
+  routines.  Thanks to Charles Gerlach for identifying the issue.
+- Added support for routable RoCE to the openib BTL.
+- Update embedded hwloc to v1.7.2.
+- ErrMgr framework redesigned to better support fault tolerance development
+  activities. See the following RFC for details:
+  https://www.open-mpi.org/community/lists/devel/2010/03/7589.php
+- Added database framework to OPAL and changed all modex operations
+  to flow thru it, also included additional system info in the
+  available data
+- Added staged state machine to support sequential work flows
+- Added distributed file system support for accessing files across
+  nodes that do not have networked file systems
+- Extended filem framework to support scalable pre-positioning of
+  files for use by applications, adding new "raw" component that
+  transmits files across the daemon network
+- Native Windows support has been removed. A cygwin package is
+  available from that group for Windows-based use.
+- Added new MPI Java bindings.  See the Javadocs for more details on
+  the API.
+- Wrapper compilers now add rpath support by default to generated
+  executables on systems that support it.  This behavior can be
+  disabled via ``--disable-wrapper-rpath``.  See note in README about ABI
+  issues when using rpath in MPI applications.
+- Added a new parallel I/O component and multiple new frameworks to
+  support parallel I/O operations.
+- Fixed MPI_STATUS_SIZE Fortran issue when used with 8-byte Fortran
+  INTEGERs and 4-byte C ints.  Since this issue affects ABI, it is
+  only enabled if Open MPI is configured with
+  ``--enable-abi-breaking-fortran-status-i8-fix``.  Thanks to Jim Parker
+  for supplying the initial patch.
+- Add support for Intel Phi SCIF transport.
+- For CUDA-aware MPI configured with CUDA 6.0, use new pointer
+  attribute to avoid extra synchronization in stream 0 when using
+  CUDA IPC between GPUs on the same node.
+- For CUDA-aware MPI configured with CUDA 6.0, compile in support
+  of GPU Direct RDMA in openib BTL to improve small message latency.
+- Updated ROMIO from MPICH v3.0.4.
+- MPI-3: Added support for remaining non-blocking collectives.
+- MPI-3: Added support for neighborhood collectives.
+- MPI-3: Updated C bindings with consistent use of [].
+- MPI-3: Added the const keyword to read-only buffers.
+- MPI-3: Added support for non-blocking communicator duplication.
+- MPI-3: Added support for non-collective communicator creation.
+
+
+Open MPI version 1.7.3
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 17 Oct 2013
+
+- Make CUDA-aware support dynamically load libcuda.so so CUDA-aware
+  MPI library can run on systems without CUDA software.
+- Fix various issues with dynamic processes and intercommunicator
+  operations under Torque.  Thanks to Suraj Prabhakaran for reporting
+  the problem.
+- Enable support for the Mellanox MXM2 library by default.
+- Improve support for Portals 4.
+- Various Solaris fixes.  Many thanks to Siegmar Gross for his
+  incredible patience in reporting all the issues.
+- MPI-2.2: Add reduction support for ``MPI_C_*COMPLEX`` and ``MPI::*COMPLEX``.
+- Fixed internal accounting when openpty() fails.  Thanks to Michal
+  Peclo for reporting the issue and providing a patch.
+- Fixed too-large memory consumption in XRC mode of the openib BTL.
+  Thanks to Alexey Ryzhikh for the patch.
+- Add bozo check for negative np values to mpirun to prevent a
+  deadlock.  Thanks to Upinder Malhi for identifying the issue.
+- Fixed MPI_IS_THREAD_MAIN behavior.  Thanks to Lisandro Dalcín for
+  pointing out the problem.
+- Various rankfile fixes.
+- Fix functionality over iWARP devices.
+- Various memory and performance optimizations and tweaks.
+- Fix MPI_Cancel issue identified by Fujitsu.
+- Add missing support for MPI_Get_address in the "use mpi" TKR
+  implementation.  Thanks to Hugo Gagnon for identifying the issue.
+- MPI-3: Add support for MPI_Count.
+- MPI-2.2: Add missing MPI_IN_PLACE support for MPI_ALLTOALL.
+- Added new usnic BTL to support the Cisco usNIC device.
+- Minor VampirTrace update to 5.14.4.4.
+- Removed support for ancient OS X systems (i.e., prior to 10.5).
+- Fixed obscure packing/unpacking datatype bug.  Thanks to Takahiro
+  Kawashima for identifying the issue.
+- Add run-time support for PMI2 environments.
+- Update openib BTL default parameters to include support for Mellanox
+  ConnectX3-Pro devices.
+- Update libevent to v2.0.21.
+- ``ompi_info --param <TYPE> <PLUGIN>`` now only shows a small number
+  of MCA parameters by default.  Add ``--level 9`` or ``--all`` to see
+  **all** MCA parameters.  See README for more details.
+- Add support for asynchronous CUDA-aware copies.
+- Add support for Mellanox MPI collective operation offload via the
+  "hcoll" library.
+- MPI-3: Add support for the MPI_T interface.  Open MPI's MCA
+  parameters are now accessible via the MPI_T control variable
+  interface.  Support has been added for a small number of MPI_T
+  performance variables.
+- Add Gentoo memory hooks override.  Thanks to Justin Bronder for the
+  patch.
+- Added new "mindist" process mapper, allowing placement of processes
+  via PCI locality information reported by the BIOS.
+- MPI-2.2: Add support for MPI_Dist_graph functionality.
+- Enable generic, client-side support for PMI2 implementations. Can
+  be leveraged by any resource manager that implements PMI2; e.g. SLURM,
+  versions 2.6 and higher.
+
+
+Open MPI version 1.7.2
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 26 Jun 2013
+
+- Major VampirTrace update to 5.14.4.2.
+  (** also appeared: 1.6.5)
+- Fix to set flag==1 when MPI_IPROBE is called with MPI_PROC_NULL.
+  (** also appeared: 1.6.5)
+- Set the Intel Phi device to be ignored by default by the openib BTL.
+  (** also appeared: 1.6.5)
+- Decrease the internal memory storage used by intrinsic MPI datatypes
+  for Fortran types.  Thanks to Takahiro Kawashima for the initial
+  patch.
+  (** also appeared: 1.6.5)
+- Fix total registered memory calculation for Mellanox ConnectIB and
+  OFED 2.0.
+  (** also appeared: 1.6.5)
+- Fix possible data corruption in the MXM MTL component.
+  (** also appeared: 1.6.5)
+- Remove extraneous -L from hwloc's embedding.  Thanks to Stefan
+  Friedel for reporting the issue.
+  (** also appeared: 1.6.5)
+- Fix contiguous datatype memory check.  Thanks to Eric Chamberland
+  for reporting the issue.
+  (** also appeared: 1.6.5)
+- Make the openib BTL more friendly to ignoring verbs devices that are
+  not RC-capable.
+  (** also appeared: 1.6.5)
+- Fix some MPI datatype engine issues.  Thanks to Thomas Jahns for
+  reporting the issue.
+  (** also appeared: 1.6.5)
+- Add INI information for Chelsio T5 device.
+  (** also appeared: 1.6.5)
+- Integrate MXM STREAM support for MPI_ISEND and MPI_IRECV, and other
+  minor MXM fixes.
+  (** also appeared: 1.6.5)
+- Fix to not show amorphous "MPI was already finalized" error when
+  failing to MPI_File_close an open file.  Thanks to Brian Smith for
+  reporting the issue.
+  (** also appeared: 1.6.5)
+- Add a distance-based mapping component to find the socket "closest"
+  to the PCI bus.
+- Fix an error that caused epoll to automatically be disabled
+  in libevent.
+- Upgrade hwloc to 1.5.2.
+- **Really** fixed XRC compile issue in Open Fabrics support.
+- Fix MXM connection establishment flow.
+- Fixed parallel debugger ability to attach to MPI jobs.
+- Fixed some minor memory leaks.
+- Fixed datatype corruption issue when combining datatypes of specific
+  formats.
+- Added Location Aware Mapping Algorithm (LAMA) mapping component.
+- Fixes for MPI_STATUS handling in corner cases.
+- Add a distance-based mapping component to find the socket "closest"
+  to the PCI bus.
+
+
+Open MPI version 1.7.1
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 16 Apr 2013
+
+- Fixed compile error when ``--without-memory-manager`` was specified
+  on Linux
+- Fixed XRC compile issue in Open Fabrics support.
+
+
+Open MPI version 1.7.0
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 1 Apr 2013
+
+- Added MPI-3 functionality:
+
+    - MPI_GET_LIBRARY_VERSION
+    - Matched probe
+    - MPI_TYPE_CREATE_HINDEXED_BLOCK
+    - Non-blocking collectives
+    - MPI_INFO_ENV support
+    - Fortran '08 bindings (see below)
+
+- Dropped support for checkpoint/restart due to loss of maintainer :-(
+- Enabled compile-time warning of deprecated MPI functions by default
+  (in supported compilers).
+- Revamped Fortran MPI bindings (see the README for details):
+
+   - "mpifort" is now the preferred wrapper compiler for Fortran
+   - Added "use mpi_f08" bindings (for compilers that support it)
+   - Added better "use mpi" support (for compilers that support it)
+   - Removed incorrect MPI_SCATTERV interface from "mpi" module that
+     was added in the 1.5.x series for ABI reasons.
+
+- Lots of VampirTrace upgrades and fixes; upgrade to v5.14.3.
+- Modified process affinity system to provide warning when bindings
+  result in being "bound to all", which is equivalent to not being
+  bound.
+- Removed maffinity, paffinity, and carto frameworks (and associated
+  MCA params).
+- Upgraded to hwloc v1.5.1.
+- Added performance improvements to the OpenIB (OpenFabrics) BTL.
+- Made malloc hooks more friendly to IO interprosers.  Thanks to the
+  bug report and suggested fix from Darshan maintainer Phil Carns.
+- Added support for the DMTCP checkpoint/restart system.
+- Added support for the Cray uGNI interconnect.
+- Fixed header file problems on OpenBSD.
+- Fixed issue with MPI_TYPE_CREATE_F90_REAL.
+- Wrapper compilers now explicitly list/link all Open MPI libraries if
+  they detect static linking CLI arguments.
+- Open MPI now requires a C99 compiler to build.  Please upgrade your
+  C compiler if you do not have a C99-compliant compiler.
+- Fix MPI_GET_PROCESSOR_NAME Fortran binding to set ierr properly.
+  Thanks to LANL for spotting the error.
+- Many MXM and FCA updates.
+- Fixed erroneous free of putenv'ed string that showed up in Valgrind
+  reports.
+- Fixed MPI_IN_PLACE case for MPI_ALLGATHER.
+- Fixed a bug that prevented MCA params from being forwarded to
+  daemons upon launch.
+- Fixed issues with VT and CUDA ``--with-cuda[-libdir]`` configuration
+  CLI parameters.
+- Entirely new implementation of many MPI collective routines focused
+  on better performance.
+- Revamped autogen / build system.
+- Add new sensor framework to ORTE that includes modules for detecting
+  stalled applications and processes that consume too much memory.
+- Added new state machine framework to ORTE that converts ORTE into an
+  event-driven state machine using the event library.
+- Added a new MCA parameter (ess_base_stream_buffering) that allows the user
+  to override the system default for buffering of stdout/stderr streams
+  (via setvbuf). Parameter is not visible via ompi_info.
+- Revamped the launch system to allow consideration of node hardware
+  in assigning process locations and bindings.
+- Added the -novm option to preserve the prior launch behavior.
+- Revamped the process mapping system to utilize node hardware by adding
+  new map-by, rank-by, and bind-to cmd line options.
+- Added new MCA parameter to provide protection against IO forwarding
+  backlog.
+- Dropped support for native Windows due to loss of maintainers. :-(
+- Added a new parallel I/O component and multiple new frameworks to
+  support parallel I/O operations.
+- Fix typo in orte_setup_hadoop.m4. Thanks to Aleksej Saushev for
+  reporting it
+- Fix a very old error in opal_path_access(). Thanks to Marco Atzeri
+  for chasing it down.
+
+
+Open MPI v1.6.x series
+----------------------
+
+Open MPI version 1.6.6
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: Not released
+
+.. important::
+
+   v1.6.6 was not released
+
+- Prevent integer overflow in datatype creation.  Thanks to Gilles
+  Gouaillardet for identifying the problem and providing a preliminary
+  version of the patch.
+- Ensure help-opal-hwloc-base.txt is included in distribution
+  tarballs.  Thanks to Gilles Gouaillardet for supplying the patch.
+- Correctly handle the invalid status for NULL and inactive requests.
+  Thanks to KAWASHIMA Takahiro for submitting the initial patch.
+- Fixed MPI_STATUS_SIZE Fortran issue when used with 8-byte Fortran
+  INTEGERs and 4-byte C ints.  Since this issue affects ABI, it is
+  only enabled if Open MPI is configured with
+  ``--enable-abi-breaking-fortran-status-i8-fix``.  Thanks to Jim Parker
+  for supplying the initial patch.
+- Fix datatype issue for sending from the middle of non-contiguous
+  data.
+- Fixed failure error with pty support.  Thanks to Michal Pecio for
+  the patch.
+- Fixed debugger support for direct-launched jobs.
+- Fix MPI_IS_THREAD_MAIN to return the correct value.  Thanks to
+  Lisandro Dalcín for pointing out the issue.
+- Update VT to 5.14.4.4:
+
+  - Fix C++-11 issue.
+  - Fix support for building RPMs on Fedora with CUDA libraries.
+
+- Add openib part number for ConnectX3-Pro HCA.
+- Ensure to check that all resolved IP addresses are local.
+- Fix MPI_COMM_SPAWN via rsh when mpirun is on a different server.
+- Add Gentoo "sandbox" memory hooks override.
+
+
+Open MPI version 1.6.5
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 26 Jun 2013
+
+- Updated default SRQ parameters for the openib BTL.
+  (** also to appear: 1.7.2)
+- Major VampirTrace update to 5.14.4.2.
+  (** also to appear: 1.7.2)
+- Fix to set flag==1 when MPI_IPROBE is called with MPI_PROC_NULL.
+  (** also to appear: 1.7.2)
+- Set the Intel Phi device to be ignored by default by the openib BTL.
+  (** also to appear: 1.7.2)
+- Decrease the internal memory storage used by intrinsic MPI datatypes
+  for Fortran types.  Thanks to Takahiro Kawashima for the initial
+  patch.
+  (** also to appear: 1.7.2)
+- Fix total registered memory calculation for Mellanox ConnectIB and
+  OFED 2.0.
+  (** also to appear: 1.7.2)
+- Fix possible data corruption in the MXM MTL component.
+  (** also to appear: 1.7.2)
+- Remove extraneous -L from hwloc's embedding.  Thanks to Stefan
+  Friedel for reporting the issue.
+  (** also to appear: 1.7.2)
+- Fix contiguous datatype memory check.  Thanks to Eric Chamberland
+  for reporting the issue.
+  (** also to appear: 1.7.2)
+- Make the openib BTL more friendly to ignoring verbs devices that are
+  not RC-capable.
+  (** also to appear: 1.7.2)
+- Fix some MPI datatype engine issues.  Thanks to Thomas Jahns for
+  reporting the issue.
+  (** also to appear: 1.7.2)
+- Add INI information for Chelsio T5 device.
+  (** also to appear: 1.7.2)
+- Integrate MXM STREAM support for MPI_ISEND and MPI_IRECV, and other
+  minor MXM fixes.
+  (** also to appear: 1.7.2)
+- Improved alignment for OpenFabrics buffers.
+- Fix to not show amorphous "MPI was already finalized" error when
+  failing to MPI_File_close an open file.  Thanks to Brian Smith for
+  reporting the issue.
+  (** also to appear: 1.7.2)
+
+
+Open MPI version 1.6.4
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 21 Feb 2013
+
+- Fix Cygwin shared memory and debugger plugin support.  Thanks to
+  Marco Atzeri for reporting the issue and providing initial patches.
+- Fix to obtaining the correct available nodes when a rankfile is
+  providing the allocation.  Thanks to Siegmar Gross for reporting the
+  problem.
+- Fix process binding issue on Solaris.  Thanks to Siegmar Gross for
+  reporting the problem.
+- Updates for MXM 2.0.
+- Major VT update to 5.14.2.3.
+- Fixed F77 constants for Cygwin/Cmake build.
+- Fix a linker error when configuring ``--without-hwloc``.
+- Automatically provide compiler flags that compile properly on some
+  types of ARM systems.
+- Fix slot_list behavior when multiple sockets are specified.  Thanks
+  to Siegmar Gross for reporting the problem.
+- Fixed memory leak in one-sided operations.  Thanks to Victor
+  Vysotskiy for letting us know about this one.
+- Added performance improvements to the OpenIB (OpenFabrics) BTL.
+- Improved error message when process affinity fails.
+- Fixed MPI_MINLOC on man pages for MPI_REDUCE(_LOCAL).  Thanks to Jed
+  Brown for noticing the problem and supplying a fix.
+- Made malloc hooks more friendly to IO interprosers.  Thanks to the
+  bug report and suggested fix from Darshan maintainer Phil Carns.
+- Restored ability to direct launch under SLURM without PMI support.
+- Fixed MPI datatype issues on OpenBSD.
+- Major VT update to 5.14.2.3.
+- Support FCA v3.0+.
+- Fixed header file problems on OpenBSD.
+- Fixed issue with MPI_TYPE_CREATE_F90_REAL.
+- Fix an issue with using external libltdl installations.  Thanks to
+  opolawski for identifying the problem.
+- Fixed MPI_IN_PLACE case for MPI_ALLGATHER for FCA.
+- Allow SLURM PMI support to look in lib64 directories.  Thanks to
+  Guillaume Papaure for the patch.
+- Restore "use mpi" ABI compatibility with the rest of the 1.5/1.6
+  series (except for v1.6.3, where it was accidentally broken).
+- Fix a very old error in opal_path_access(). Thanks to Marco Atzeri
+  for chasing it down.
+
+
+Open MPI version 1.6.3
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 30 Oct 2012
+
+- Fix mpirun ``--launch-agent`` behavior when a prefix is specified.
+  Thanks to Reuti for identifying the issue.
+- Fixed memchecker configury.
+- Brought over some compiler warning squashes from the development trunk.
+- Fix spawning from a singleton to multiple hosts when the "add-host"
+  MPI_Info key is used.  Thanks to Brian Budge for pointing out the
+  problem.
+- Add Mellanox ConnextIB IDs and max inline value.
+- Fix rankfile when no -np is given.
+- FreeBSD detection improvement.  Thanks to Brooks Davis for the
+  patch.
+- Removed TCP warnings on Windows.
+- Improved collective algorithm selection for very large messages.
+- Fix PSM MTL affinity settings.
+- Fix issue with MPI_OP_COMMUTATIVE in the mpif.h bindings.  Thanks to
+  Åke Sandgren for providing a patch to fix the issue.
+- Fix issue with MPI_SIZEOF when using CHARACTER and LOGICAL types in
+  the mpi module.  Thanks to Åke Sandgren for providing a patch to fix
+  the issue.
+
+
+Open MPI version 1.6.2
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 25 Sep 2012
+
+- Fix issue with MX MTL.  Thanks to Doug Eadline for raising the issue.
+- Fix singleton MPI_COMM_SPAWN when the result job spans multiple nodes.
+- Fix MXM hang, and update for latest version of MXM.
+- Update to support Mellanox FCA 2.5.
+- Fix startup hang for large jobs.
+- Ensure MPI_TESTANY / MPI_WAITANY properly set the empty status when
+  count==0.
+- Fix MPI_CART_SUB behavior of not copying periods to the new
+  communicator properly.  Thanks to John Craske for the bug report.
+- Add btl_openib_abort_not_enough_reg_mem MCA parameter to cause Open
+  MPI to abort MPI jobs if there is not enough registered memory
+  available on the system (vs. just printing a warning).  Thanks to
+  Brock Palen for raising the issue.
+- Minor fix to Fortran MPI_INFO_GET: only copy a value back to the
+  user's buffer if the flag is .TRUE.
+- Fix VampirTrace compilation issue with the PGI compiler suite.
+
+
+Open MPI version 1.6.1
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 22 Aug 2012
+
+- A bunch of changes to eliminate hangs on OpenFabrics-based networks.
+  Users with Mellanox hardware are **STRONGLY ENCOURAGED** to check
+  their registered memory kernel module settings to ensure that the OS
+  will allow registering more than 8GB of memory.  See this FAQ item
+  for details:
+
+  https://www.open-mpi.org/faq/?category=openfabrics#ib-low-reg-mem
+
+   - Fall back to send/receive semantics if registered memory is
+     unavilable for RDMA.
+   - Fix two fragment leaks when registered memory is exhausted.
+   - Hueristically determine how much registered memory is available
+     and warn if it's significantly less than all of RAM.
+   - Artifically limit the amount of registered memory each MPI process
+     can use to about 1/Nth to total registered memory available.
+   - Improve error messages when events occur that are likely due to
+     unexpected registered memory exhaustion.
+
+- Fix double semicolon error in the C++ in <mpi.h>.  Thanks to John
+  Foster for pointing out the issue.
+- Allow -Xclang to be specified multiple times in CFLAGS.  Thanks to
+  A. Martin for raising the issue.
+- Break up a giant ``print *`` statement in the ABI-preserving incorrect
+  MPI_SCATTER interface in the "large" Fortran "mpi" module.  Thanks
+  to Juan Escobar for the initial patch.
+- Switch the MPI_ALLTOALLV default algorithm to a pairwise exchange.
+- Increase the openib BTL default CQ length to handle more types of
+  OpenFabrics devices.
+- Lots of VampirTrace fixes; upgrade to v5.13.0.4.
+- Map MPI_2INTEGER to underlying MPI_INTEGERs, not MPI_INTs.
+- Ensure that the OMPI version number is toleant of handling spaces.
+  Thanks to dragonboy for identifying the issue.
+- Fixed IN parameter marking on Fortran "mpi" module
+  MPI_COMM_TEST_INTER interface.
+- Various MXM improvements.
+- Make the output of ``mpirun --report-bindings`` much more friendly /
+  human-readable.
+- Properly handle MPI_COMPLEX8|16|32.
+- More fixes for mpirun's processor affinity options (--bind-to-core
+  and friends).
+- Use aligned memory for OpenFabrics registered memory.
+- Multiple fixes for parameter checking in MPI_ALLGATHERV,
+  MPI_REDUCE_SCATTER, MPI_SCATTERV, and MPI_GATHERV.  Thanks to the
+  mpi4py community (Bennet Fauber, Lisandro Dalcín, Jonathan Dursi).
+- Fixed file positioning overflows in MPI_FILE_GET_POSITION,
+  MPI_FILE_GET_POSITION_SHARED, FILE_GET_SIZE, FILE_GET_VIEW.
+- Removed the broken ``mpirun --cpu-set`` option.
+- Fix cleanup of MPI errorcodes.  Thanks to Alexey Bayduraev for the
+  patch.
+- Fix default hostfile location.  Thanks to Götz Waschk for noticing
+  the issue.
+- Improve several error messages.
+
+
+Open MPI version 1.6.0
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 14 May 2012
+
+- Fix some process affinity issues.  When binding a process, Open MPI
+  will now bind to all available hyperthreads in a core (or socket,
+  depending on the binding options specified).
+
+  .. note::
+     Note that ``mpirun --bind-to-socket ...`` does not work on POWER6-
+     and POWER7-based systems with some Linux kernel versions.  See
+     the FAQ on the Open MPI web site for more information.
+
+- Add support for ARM5 and ARM6 (in addition to the existing ARM7
+  support).  Thanks to Evan Clinton for the patch.
+- Minor Mellanox MXM fixes.
+- Properly detect FDR10, FDR, and EDR OpenFabrics devices.
+- Minor fixes to the mpirun(1) and MPI_Comm_create(3) man pages.
+- Prevent segv if COMM_SPAWN_MULTIPLE fails.  Thanks to Fujitsu for
+  the patch.
+- Disable interposed memory management in fakeroot environments.  This
+  fixes a problem in some build environments.
+- Minor hwloc updates.
+- Array versions of MPI_TEST and MPI_WAIT with a count==0 will now
+  return immediately with MPI_SUCCESS.  Thanks to Jeremiah Willcock
+  for the suggestion.
+- Update VampirTrace to v5.12.2.
+- Properly handle forwarding stdin to all processes when ``mpirun
+  --stdin all`` is used.
+- Workaround XLC assembly bug.
+- OS X Tiger (10.4) has not been supported for a while, so forcibly
+  abort configure if we detect it.
+- Fix segv in the openib BTL when running on SPARC 64 systems.
+- Fix some include file ordering issues on some BSD-based platforms.
+  Thanks to Paul Hargove for this (and many, many other) fixes.
+- Properly handle .FALSE. return parameter value to attribute copy
+  callback functions.
+- Fix a bunch of minor C++ API issues; thanks to Fujitsu for the patch.
+- Fixed the default hostfile MCA parameter behavior.
+- Per the MPI spec, ensure not to touch the port_name parameter to
+  MPI_CLOSE_PORT (it's an IN parameter).
+
+
+Open MPI v1.5.x series
+----------------------
+
+Open MPI version 1.5.5
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 27 Mar 2012
+
+- Many, many portability configure/build fixes courtesy of Paul
+  Hargrove.  Thanks, Paul!
+- Fixed shared memory fault tolerance support compiler errors.
+- Removed not-production-quality rshd and tmd PLM launchers.
+- Minor updates to the Open MPI SRPM spec file.
+- Fixed mpirun's ``--bind-to-socket`` option.
+- A few MPI_THREAD_MULTIPLE fixes in the shared memory BTL.
+- Upgrade the GNU Autotools used to bootstrap the 1.5/1.6 series to
+  all the latest versions at the time of this release.
+- Categorically state in the README that if you're having a problem
+  with Open MPI with the Linux Intel 12.1 compilers, **upgrade your
+  Intel Compiler Suite to the latest patch version**, and the problems
+  will go away. :-)
+- Fix the ``--without-memory-manager`` configure option.
+- Fixes for Totalview/DDT MPI-capable debuggers.
+- Update rsh/ssh support to properly handle the Mac OS X library path
+  (i.e., DYLD_LIBRARY_PATH).
+- Make warning about shared memory backing files on a networked file
+  system be optional (i.e., can be disabled via MCA parameter).
+- Several fixes to processor and memory affinity.
+- Various shared memory infrastructure improvements.
+- Various checkpoint/restart fixes.
+- Fix MPI_IN_PLACE (and other MPI sentinel values) on OS X.  Thanks to
+  Dave Goodell for providing the magic OS X gcc linker flags necessary.
+- Various man page corrections and typo fixes.  Thanks to Fujitsu for
+  the patch.
+- Updated wrapper compiler man pages to list the various ``--showme``
+  options that are available.
+- Add PMI direct-launch support (e.g., "srun mpi_application" under
+  SLURM).
+- Correctly compute the aligned address when packing the
+  datatype description. Thanks to Fujitsu for the patch.
+- Fix MPI obscure corner case handling in packing MPI datatypes.
+  Thanks to Fujitsu for providing the patch.
+- Workaround an Intel compiler v12.1.0 2011.6.233 vector optimization
+  bug.
+- Output the MPI API in ompi_info output.
+- Major VT update to 5.12.1.4.
+- Upgrade embedded Hardware Locality (hwloc) v1.3.2, plus some
+  post-1.3.2-release bug fixes.  All processor and memory binding is
+  now done through hwloc.  Woo hoo!  Note that this fixes core binding
+  on AMD Opteron 6200 and 4200 series-based systems (sometimes known
+  as Interlagos, Valencia, or other Bulldozer-based chips).
+- New MCA parameters to control process-wide memory binding policy:
+  hwloc_base_mem_alloc_policy, hwloc_base_mem_bind_failure_action (see
+  ``ompi_info --param hwloc base``).
+- Removed direct support for libnuma.  Libnuma support may now be
+  picked up through hwloc.
+- Added MPI_IN_PLACE support to MPI_EXSCAN.
+- Various fixes for building on Windows, including MinGW support.
+- Removed support for the OpenFabrics IBCM connection manager.
+- Updated Chelsio T4 and Intel NE OpenFabrics default buffer settings.
+- Increased the default RDMA CM timeout to 30 seconds.
+- Issue a warning if both btl_tcp_if_include and btl_tcp_if_exclude
+  are specified.
+- Many fixes to the Mellanox MXM transport.
+
+
+Open MPI version 1.5.4
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 18 Aug 2011
+
+- Add support for the (as yet unreleased) Mellanox MXM transport.
+- Add support for dynamic service levels (SLs) in the openib BTL.
+- Fixed C++ bindings cosmetic/warnings issue with
+  MPI::Comm::NULL_COPY_FN and MPI::Comm::NULL_DELETE_FN.  Thanks to
+  Júlio Hoffimann for identifying the issues.
+- Also allow the word "slots" in rankfiles (i.e., not just "slot").
+  (** also to appear in 1.4.4)
+- Add Mellanox ConnectX 3 device IDs to the openib BTL defaults.
+  (** also to appear in 1.4.4)
+- Various FCA updates.
+- Fix 32 bit SIGBUS errors on Solaris SPARC platforms.
+- Add missing ARM assembly code files.
+- Update to allow more than 128 entries in an appfile.
+  (** also to appear in 1.4.4)
+- Various VT updates and bug fixes.
+- Update description of btl_openib_cq_size to be more accurate.
+  (** also to appear in 1.4.4)
+- Various assembly "clobber" fixes.
+- Fix a hang in carto selection in obscure situations.
+- Guard the inclusion of execinfo.h since not all platforms have it.  Thanks
+  to Aleksej Saushev for identifying this issue.
+  (** also to appear in 1.4.4)
+- Support Solaris legacy munmap prototype changes.
+  (** also to appear in 1.4.4)
+- Updated to Automake 1.11.1 per
+  https://www.open-mpi.org/community/lists/devel/2011/07/9492.php.
+- Fix compilation of LSF support.
+- Update MPI_Comm_spawn_multiple.3 man page to reflect what it
+  actually does.
+- Fix for possible corruption of the environment.  Thanks to Peter
+  Thompson for the suggestion.  (** also to appear in 1.4.4)
+- Enable use of PSM on direct-launch SLURM jobs.
+- Update paffinity hwloc to v1.2, and to fix minor bugs affinity
+  assignment bugs on PPC64/Linux platforms.
+- Let the openib BTL auto-detect its bandwidth.
+- Support new MPI-2.2 datatypes.
+- Updates to support more datatypes in MPI one-sided communication.
+- Fix recursive locking bug when MPI-IO was used with
+  MPI_THREAD_MULTIPLE.  (** also to appear in 1.4.4)
+- Fix mpirun handling of prefix conflicts.
+- Ensure mpirun's ``--xterm`` options leaves sessions attached.
+  (** also to appear in 1.4.4)
+- Fixed type of sendcounts and displs in the "use mpi" F90 module.
+  ABI is preserved, but applications may well be broken.  See the
+  README for more details.  Thanks to Stanislav Sazykin for
+  identifying the issue.  (** also to appear in 1.4.4)
+- Fix indexed datatype leaks.  Thanks to Pascal Deveze for supplying
+  the initial patch.  (** also to appear in 1.4.4)
+- Fix debugger mapping when mpirun's -npernode option is used.
+- Fixed support for configure's ``--disable-dlopen`` option when
+  used with ``make distclean``.
+- Fix segv associated with MPI_Comm_create with MPI_GROUP_EMPTY.
+  Thanks to Dominik Goeddeke for finding this.
+  (** also to appear in 1.4.4)
+- Improved LoadLeveler ORTE support.
+- Add new WinVerbs BTL plugin, supporting native OpenFabrics verbs on
+  Windows (the "wv" BTL).
+- Add new btl_openib_gid_index MCA parameter to allow selecting which
+  GID to use on an OpenFabrics device's GID table.
+- Add support for PCI relaxed ordering in the OpenFabrics BTL (when
+  available).
+- Update rsh logic to allow correct SGE operation.
+- Ensure that the mca_paffinity_alone MCA parameter only appears once
+  in the ompi_info output.  Thanks to Gus Correa for identifying the
+  issue.
+- Fixed return codes from MPI_PROBE and MPI_IPROBE.
+  (** also to appear in 1.4.4)
+- Remove ``--enable-progress-thread`` configure option; it doesn't work on
+  the v1.5 branch.  Rename ``--enable-mpi-threads`` to
+  ``--enable-mpi-thread-multiple``.  Add new ``--enable-opal-multi-threads``
+  option.
+- Updates for Intel Fortran compiler version 12.
+- Remove bproc support.  Farewell bproc!
+- If something goes wrong during MPI_INIT, fix the error
+  message to say that it's illegal to invoke MPI_INIT before
+  MPI_INIT.
+
+
+Open MPI version 1.5.3
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 16 Mar 2011
+
+- Add missing "affinity" MPI extension (i.e., the OMPI_Affinity_str()
+  API) that was accidentally left out of the 1.5.2 release.
+
+
+Open MPI version 1.5.2
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 9 Mar 2011
+
+- Replaced all custom topology / affinity code with initial support
+  for hwloc v1.1.1 (PLPA has been removed |mdash| long live hwloc!).  Note
+  that hwloc is bundled with Open MPI, but an external hwloc can be
+  used, if desired.  See README for more details.
+- Many CMake updates for Windows builds.
+- Updated opal_cr_thread_sleep_wait MCA param default value to make it
+  less aggressive.
+- Updated debugger support to allow Totalview attaching from jobs
+  launched directly via srun (not mpirun).  Thanks to Nikolay Piskun
+  for the patch.
+- Added more FTB/CIFTS support.
+- Fixed compile error with the PGI compiler.
+- Portability fixes to allow the openib BTL to run on the Solaris
+  verbs stack.
+- Fixed multi-token command-line issues when using the mpirun
+  ``--debug`` switch.  For example:
+
+  .. code-block:: sh
+
+     mpirun --debug -np 2 a.out "foo bar"
+
+  Thanks to Gabriele Fatigati for reporting the issue.
+- Added ARM support.
+- Added the MPI_ROOT environment variable in the Open MPI Linux SRPM
+  for customers who use the BPS and LSF batch managers.
+- Updated ROMIO from MPICH v1.3.1 (plus one additional patch).
+- Fixed some deprecated MPI API function notification messages.
+- Added new "bfo" PML that provides failover on OpenFabrics networks.
+- Fixed some buffer memcheck issues in ``MPI_*_init``.
+- Added Solaris-specific chip detection and performance improvements.
+- Fix some compile errors on Solaris.
+- Updated the "rmcast" framework with bug fixes, new functionality.
+- Updated the Voltaire FCA component with bug fixes, new
+  functionality.  Support for FCA version 2.1.
+- Fix gcc 4.4.x and 4.5.x over-aggressive warning notifications on
+  possibly freeing stack variables.  Thanks to the Gentoo packagers
+  for reporting the issue.
+- Make the openib component be verbose when it disqualifies itself due
+  to MPI_THREAD_MULTIPLE.
+- Minor man page fixes.
+- Various checkpoint / restart fixes.
+- Fix race condition in the one-sided unlock code.  Thanks to
+  Guillaume Thouvenin for finding the issue.
+- Improve help message aggregation.
+- Add OMPI_Affinity_str() optional user-level API function (i.e., the
+  "affinity" MPI extension).  See README for more details.
+- Added btl_tcp_if_seq MCA parameter to select a different ethernet
+  interface for each MPI process on a node.  This parameter is only
+  useful when used with virtual ethernet interfaces on a single
+  network card (e.g., when using virtual interfaces give dedicated
+  hardware resources on the NIC to each process).
+- Changed behavior of mpirun to terminate if it receives 10 (or more)
+  SIGPIPEs.
+- Fixed oversubscription detection.
+- Added new mtl_mx_board and mtl_mx_endpoint MCA parameters.
+- Added ummunotify support for OpenFabrics-based transports.  See the
+  README for more details.
+
+
+Open MPI version 1.5.1
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 15 Dec 2010
+
+- Fixes for the Oracle Studio 12.2 Fortran compiler.
+- Fix SPARC and SPARCv9 atomics.  Thanks to Nicola Stange for the
+  initial patch.
+- Fix Libtool issues with the IBM XL compiler in 64-bit mode.
+- Restore the reset of the libevent progress counter to avoid
+  over-sampling the event library.
+- Update memory barrier support.
+- Use memmove (instead of memcpy) when necessary (e.g., source and
+  destination overlap).
+- Fixed ompi-top crash.
+- Fix to handle Autoconf ``--program-transforms`` properly and other
+  m4/configury updates.  Thanks to the GASNet project for the
+  ``--program`` transforms fix.
+- Allow hostfiles to specify usernames on a per-host basis.
+- Update wrapper compiler scripts to search for perl during configure,
+  per request from the BSD maintainers.
+- Minor man page fixes.
+- Added ``--with-libltdl`` option to allow building Open MPI with an
+  external installation of libltdl.
+- Fixed various issues with -D_FORTIFY_SOURCE=2.
+- Various VT fixes and updates.
+
+
+Open MPI version 1.5.0
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 10 Oct 2010
+
+- Added "knem" support: direct process-to-process copying for shared
+  memory message passing.  See https://knem.gitlabpages.inria.fr/
+  and the README file for more details.
+- Updated shared library versioning scheme and linking style of MPI
+  applications.  The MPI application ABI has been broken from the
+  v1.3/v1.4 series.  MPI applications compiled against any prior
+  version of Open MPI will need to, at a minimum, re-link.  See the
+  README file for more details.
+- Added "fca" collective component, enabling MPI collective offload
+  support for Voltaire switches.
+- Fixed MPI one-sided operations with large target displacements.
+  Thanks to Brian Price and Jed Brown for reporting the issue.
+- Fixed MPI_GET_COUNT when used with large counts.  Thanks to Jed
+  Brown for reporting the issue.
+- Made the openib BTL safer if extremely low SRQ settings are used.
+- Fixed handling of the array_of_argv parameter in the Fortran
+  binding of MPI_COMM_SPAWN_MULTIPLE (** also to appear: 1.4.3).
+- Fixed malloc(0) warnings in some collectives.
+- Fixed a problem with the Fortran binding for
+  MPI_FILE_CREATE_ERRHANDLER.  Thanks to Secretan Yves for identifying
+  the issue (** also to appear: 1.4.3).
+- Updates to the LSF PLM to ensure that the path is correctly passed.
+  Thanks to Teng Lin for the patch (** also to appear: 1.4.3).
+- Fixes for the F90 MPI_COMM_SET_ERRHANDLER and MPI_WIN_SET_ERRHANDLER
+  bindings.  Thanks to Paul Kapinos for pointing out the issue
+  (** also to appear: 1.4.3).
+- Fixed extra_state parameter types in F90 prototypes for
+  MPI_COMM_CREATE_KEYVAL, MPI_GREQUEST_START, MPI_REGISTER_DATAREP,
+  MPI_TYPE_CREATE_KEYVAL, and MPI_WIN_CREATE_KEYVAL.
+- Fixes for Solaris oversubscription detection.
+- If the PML determines it can't reach a peer process, print a
+  slightly more helpful message.  Thanks to Nick Edmonds for the
+  suggestion.
+- Make btl_openib_if_include/exclude function the same way
+  btl_tcp_if_include/exclude works (i.e., supplying an _include list
+  overrides supplying an _exclude list).
+- Apply more scalable reachability algorithm on platforms with more
+  than 8 TCP interfaces.
+- Various assembly code updates for more modern platforms / compilers.
+- Relax restrictions on using certain kinds of MPI datatypes with
+  one-sided operations.  Users beware; not all MPI datatypes are valid
+  for use with one-sided operations!
+- Improve behavior of MPI_COMM_SPAWN with regards to ``--bynode``.
+- Various threading fixes in the openib BTL and other core pieces of
+  Open MPI.
+- Various help file and man pages updates.
+- Various FreeBSD and NetBSD updates and fixes.  Thanks to Kevin
+  Buckley and Aleksej Saushev for their work.
+- Fix case where freeing communicators in MPI_FINALIZE could cause
+  process failures.
+- Print warnings if shared memory state files are opened on what look
+  like networked filesystems.
+- Update libevent to v1.4.13.
+- Allow propagating signals to processes that call fork().
+- Fix bug where MPI_GATHER was sometimes incorrectly examining the
+  datatype on non-root processes.  Thanks to Michael Hofmann for
+  investigating the issue.
+- Various Microsoft Windows fixes.
+- Various Catamount fixes.
+- Various checkpoint / restart fixes.
+- Xgrid support has been removed until it can be fixed (patches
+  would be welcome).
+- Added simplistic "libompitrace" contrib package.  Using the MPI
+  profiling interface, it essentially prints out to stderr when select
+  MPI functions are invoked.
+- Update bundled VampirTrace to v5.8.2.
+- Add pkg-config(1) configuration files for ompi, ompi-c, ompi-cxx,
+  ompi-f77, ompi-f90.  See the README for more details.
+- Removed the libopenmpi_malloc library (added in the v1.3 series)
+  since it is no longer necessary
+- Add several notifier plugins (generally used when Open MPI detects
+  system/network administrator-worthy problems); each have their own
+  MCA parameters to govern their usage.  See ``ompi_info --param
+  notifier <name>`` for more details.
+
+   - command to execute arbitrary commands (e.g., run a script).
+   - file to send output to a file.
+   - ftb to send output to the Fault Tolerant Backplane (see:
+     https://wiki.mcs.anl.gov/cifts/index.php/CIFTS)
+   - hnp to send the output to mpirun.
+   - smtp (requires libesmtp) to send an email.
+
+
+Open MPI v1.4.x series
+----------------------
+
+Open MPI version 1.4.5
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 12 Feb 2012
+
+- Fixed the ``--disable-memory-manager`` configure switch.
+  (** also to appear in 1.5.5)
+- Fix typos in code and man pages.  Thanks to Fujitsu for these fixes.
+  (** also to appear in 1.5.5)
+- Improve management of the registration cache; when full, try freeing
+  old entries and attempt to re-register.
+- Fixed a data packing pointer alignment issue.  Thanks to Fujitsu
+  for the patch.
+  (** also to appear in 1.5.5)
+- Add ability to turn off warning about having the shared memory backing
+  store over a networked filesystem.  Thanks to Chris Samuel for this
+  suggestion.
+  (** also to appear in 1.5.5)
+- Removed an unnecessary memmove() and plugged a couple of small memory leaks
+  in the openib OOB connection setup code.
+- Fixed some QLogic bugs. Thanks to Mark Debbage from QLogic for the patches.
+- Fixed problem with MPI_IN_PLACE and other sentinel Fortran constants
+  on OS X.
+  (** also to appear in 1.5.5)
+- Fix SLURM cpus-per-task allocation.
+  (** also to appear in 1.5.5)
+- Fix the datatype engine for when data left over from the previous
+  pack was larger than the allowed space in the pack buffer. Thanks to
+  Yuki Matsumoto and Takahiro Kawashima for the bug report and the
+  patch.
+- Fix Fortran value for MPI_MAX_PORT_NAME.  Thanks to Enzo Dari for
+  raising the issue.
+- Workaround an Intel compiler v12.1.0 2011.6.233 vector optimization
+  bug.
+- Fix issues on Solaris with the openib BTL.
+- Fixes for the Oracle Studio 12.2 Fortran compiler.
+- Update iWARP parameters for the Intel NICs.
+  (** also to appear in 1.5.5)
+- Fix obscure cases where MPI_ALLGATHER could crash.  Thanks to Andrew
+  Senin for reporting the problem.
+  (** also to appear in 1.5.5)
+
+
+Open MPI version 1.4.4
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 11 Oct 2011
+
+- Modified a memcpy() call in the openib btl connection setup to use
+  memmove() instead because of the possibility of an overlapping
+  copy (as identified by valgrind).
+- Changed use of sys_timer_get_cycles() to the more appropriate
+  wrapper: opal_timer_base_get_cycles().  Thanks to Jani Monoses
+  for this fix.
+- Corrected the reported default value of btl_openib_ib_timeout
+  in the "IB retries exceeded" error message.  Thanks to Kevin Buckley
+  for this correction.
+- Increased rdmacm address resolution timeout from 1s to 30s &
+  updated Chelsio T4 openib BTL defaults.  Thanks to Steve Wise
+  for these updates.
+  (** also to appear in 1.5.5)
+- Ensure that MPI_Accumulate error return in 1.4 is consistent with
+  1.5.x and trunk.
+- Allow the word "slots" in rankfiles (i.e., not just "slot").
+  (** also appeared in 1.5.4)
+- Add Mellanox ConnectX 3 device IDs to the openib BTL defaults.
+  (** also appeared in 1.5.4)
+- Update description of btl_openib_cq_size to be more accurate.
+- Ensure mpirun's ``--xterm`` options leaves sessions attached.
+  (** also appeared in 1.5.4)
+- Update to allow more than 128 entries in an appfile.
+  (** also appeared in 1.5.4)
+- Update description of btl_openib_cq_size to be more accurate.
+  (** also appeared in 1.5.4)
+- Fix for deadlock when handling recursive attribute keyval deletions
+  (e.g., when using ROMIO with MPI_THREAD_MULTIPLE).
+- Fix indexed datatype leaks.  Thanks to Pascal Deveze for supplying
+  the initial patch.  (** also appeared in 1.5.4)
+- Fixed the F90 types of the sendcounts and displs parameters to
+  MPI_SCATTERV.  Thanks to Stanislav Sazykin for identifying the issue.
+  (** also appeared in 1.5.4)
+- Exclude opal/libltdl from "make distclean" when ``--disable-dlopen`` is
+  used.  Thanks to David Gunter for reporting the issue.
+- Fixed a segv in MPI_Comm_create when called with GROUP_EMPTY.
+  Thanks to Dominik Goeddeke for finding this.
+  (** also appeared in 1.5.4)
+- Fixed return codes from MPI_PROBE and MPI_IPROBE.
+  (** also appeared in 1.5.4)
+- Fixed undefined symbol error when using the vtf90 profiling tool.
+- Fix for referencing an uninitialized variable in DPM ORTE.  Thanks
+  to Avinash Malik for reporting the issue.
+- Fix for correctly handling multi-token args when using debuggers.
+- Eliminated the unneeded ``u_int*_t`` datatype definitions.
+- Change in ORTE DPM to get around gcc 4.[45].x compiler wanrings
+  about possibly calling free() on a non-heap variable, even though it
+  will never happen because the refcount will never go to zero.
+- Fixed incorrect text in MPI_File_set_view man page.
+- Fix in MPI_Init_thread for checkpoint/restart.
+- Fix for libtool issue when using pgcc to compile ompi in conjunction
+  with the -tp option.
+- Fixed a race condition in osc_rdma_sync.  Thanks to Guillaume
+  Thouvenin for finding this issue.
+- Clarification of MPI_Init_thread man page.
+- Fixed an indexing problem in precondition_transports.
+- Fixed a problem in which duplicated libs were being specified for
+  linking.  Thanks to Hicham Mouline for noticing it.
+- Various autogen.sh fixes.
+- Fix for memchecking buffers during ``MPI_*INIT``.
+- Man page cleanups.  Thanks to Jeremiah Willcock and Jed Brown.
+- Fix for VT rpmbuild on RHEL5.
+- Support Solaris legacy munmap prototype changes.
+  (** also appeared in 1.5.4)
+- Expands app_idx to int32_t to allow more than 127 app_contexts.
+- Guard the inclusion of execinfo.h since not all platforms have it.  Thanks
+  to Aleksej Saushev for identifying this issue.
+  (** also appeared in 1.5.4)
+- Fix to avoid possible environment corruption.  Thanks to Peter Thompson
+  for identifying the issue and supplying a patch.
+  (** also appeared in 1.5.4)
+- Fixed paffinity base MCA duplicate registrations.  Thanks to Gus
+  Correa for bringing this to our attention.
+- Fix recursive locking bug when MPI-IO was used with
+  MPI_THREAD_MULTIPLE.  (** also appeared in 1.5.4)
+- F90 MPI API fixes.
+- Fixed a misleading MPI_Bcast error message.  Thanks to Jeremiah
+  Willcock for reporting this.
+- Added <sys/stat.h> to ptmalloc's hooks.c (it's not always included
+  by default on some systems).
+- Libtool patch to get around a build problem when using the IBM XL
+  compilers.
+- Fix to detect and avoid overlapping memcpy().  Thanks to
+  Francis Pellegrini for identifying the issue.
+- Fix to allow ompi to work on top of RoCE vLANs.
+- Restored a missing debugger flag to support TotalView.  Thanks to
+  David Turner and the TV folks for supplying the fix.
+- Updated SLURM support to 1.5.1.
+- Removed an extraneous #include from the TCP BTL.
+- When specifying OOB ports, fix to convert the ports into network
+  byte order before binding.
+- Fixed use of memory barriers in the SM BTL.  This fixed segv's when
+  compiling with Intel 10.0.025 or PGI 9.0-3.
+- Fix to prevent the SM BTL from creating its mmap'd file in
+  directories that are remotely mounted.
+
+
+Open MPI version 1.4.3
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 6 Sep 2010
+
+- Fixed handling of the array_of_argv parameter in the Fortran
+  binding of MPI_COMM_SPAWN_MULTIPLE (** also to appear: 1.5).
+- Fixed a problem with the Fortran binding for
+  MPI_FILE_CREATE_ERRHANDLER.  Thanks to Secretan Yves for identifying
+  the issue (** also to appear: 1.5).
+- Updates to the LSF PLM to ensure that the path is correctly passed.
+  Thanks to Teng Lin for the patch (** also to appear: 1.5).
+- Fixes for the F90 MPI_COMM_SET_ERRHANDLER and MPI_WIN_SET_ERRHANDLER
+  bindings.  Thanks to Paul Kapinos for pointing out the issue.
+  (** also to appear: 1.5).
+- Fixed various MPI_THREAD_MULTIPLE race conditions.
+- Fixed an issue with an undeclared variable from ptmalloc2 munmap on
+  BSD systems.
+- Fixes for BSD interface detection.
+- Various other BSD fixes.  Thanks to Kevin Buckley helping to track.
+  all of this down.
+- Fixed issues with the use of the -nper* mpirun command line arguments.
+- Fixed an issue with coll tuned dynamic rules.
+- Fixed an issue with the use of OPAL_DESTDIR being applied too aggressively.
+- Fixed an issue with one-sided xfers when the displacement exceeds 2GBytes.
+- Change to ensure TotalView works properly on Darwin.
+- Added support for Visual Studio 2010.
+- Fix to ensure proper placement of VampirTrace header files.
+- Needed to add volatile keyword to a varialbe used in debugging
+  (MPIR_being_debugged).
+- Fixed a bug in inter-allgather.
+- Fixed malloc(0) warnings.
+- Corrected a typo the MPI_Comm_size man page (intra -> inter).  Thanks
+  to Simon number.cruncher for pointing this out.
+- Fixed a SegV in orted when given more than 127 app_contexts.
+- Removed xgrid source code from the 1.4 branch since it is no longer
+  supported in the 1.4 series.
+- Removed the ``--enable-opal-progress-threads`` config option since
+  opal progress thread support does not work in 1.4.x.
+- Fixed a defect in VampirTrace's vtfilter.
+- Fixed wrong Windows path in hnp_contact.
+- Removed the requirement for a paffinity component.
+- Removed a hardcoded limit of 64 interconnected jobs.
+- Fix to allow singletons to use ompi-server for rendezvous.
+- Fixed bug in output-filename option.
+- Fix to correctly handle failures in mx_init().
+- Fixed a potential Fortran memory leak.
+- Fixed an incorrect branch in some ppc32 assembly code.  Thanks
+  to Matthew Clark for this fix.
+- Remove use of undocumented AS_VAR_GET macro during configuration.
+- Fixed an issue with VampirTrace's wrapper for MPI_init_thread.
+- Updated mca-btl-openib-device-params.ini file with various new vendor id's.
+- Configuration fixes to ensure CPPFLAGS in handled properly if a non-standard
+  valgrind location was specified.
+- Various man page updates
+
+
+Open MPI version 1.4.2
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 4 May 2010
+
+- Fixed problem when running in heterogeneous environments.  Thanks to
+  Timur Magomedov for helping to track down this issue.
+- Update LSF support to ensure that the path is passed correctly.
+  Thanks to Teng Lin for submitting a patch.
+- Fixed some miscellaneous oversubscription detection bugs.
+- IBM re-licensed its LoadLeveler code to be BSD-compliant.
+- Various OpenBSD and NetBSD build and run-time fixes.  Many thanks to
+  the OpenBSD community for their time, expertise, and patience
+  getting these fixes incorporated into Open MPI's main line.
+- Various fixes for multithreading deadlocks, race conditions, and
+  other nefarious things.
+- Fixed ROMIO's handling of "nearly" contiguous issues (e.g., with
+  non-zero true_lb).  Thanks for Pascal Deveze for the patch.
+- Bunches of Windows build fixes.  Many thanks to several Windows
+  users for their help in improving our support on Windows.
+- Now allow the graceful failover from MTLs to BTLs if no MTLs can
+  initialize successfully.
+- Added "clobber" information to various atomic operations, fixing
+  erroneous behavior in some newer versions of the GNU compiler suite.
+- Update various iWARP and InfiniBand device specifications in the
+  OpenFabrics .ini support file.
+- Fix the use of hostfiles when a username is supplied.
+- Various fixes for rankfile support.
+- Updated the internal version of VampirTrace to 5.4.12.
+- Fixed OS X TCP wireup issues having to do with IPv4/IPv6 confusion
+  (see https://svn.open-mpi.org/trac/ompi/changeset/22788 for more
+  details).
+- Fixed some problems in processor affinity support, including when
+  there are "holes" in the processor namespace (e.g., offline
+  processors).
+- Ensure that Open MPI's "session directory" (usually located in /tmp)
+  is cleaned up after process termination.
+- Fixed some problems with the collective "hierarch" implementation
+  that could occur in some obscure conditions.
+- Various MPI_REQUEST_NULL, API parameter checking, and attribute
+  error handling fixes.  Thanks to Lisandro Dalcín for reporting the
+  issues.
+- Fix case where MPI_GATHER erroneously used datatypes on non-root
+  nodes.  Thanks to Michael Hofmann for investigating the issue.
+- Patched ROMIO support for PVFS2 > v2.7 (patch taken from MPICH2
+  version of ROMIO).
+- Fixed ``mpirun`` ``--report-bindings`` behavior when used with
+  mpi_paffinity_alone=1.  Also fixed mpi_paffinity_alone=1 behavior
+  with non-MPI applications.  Thanks to Brice Goglin for noticing the
+  problem.
+- Ensure that all OpenFabrics devices have compatible receive_queues
+  specifications before allowing them to communicate.  See the lengthy
+  comment in https://svn.open-mpi.org/trac/ompi/changeset/22592 for
+  more details.
+- Fix some issues with checkpoint/restart.
+- Improve the pre-MPI_INIT/post-MPI_FINALIZE error messages.
+- Ensure that loopback addresses are never advertised to peer
+  processes for RDMA/OpenFabrics support.
+- Fixed a CSUM PML false positive.
+- Various fixes for Catamount support.
+- Minor update to wrapper compilers in how user-specific argv is
+  ordered on the final command line.  Thanks to Jed Brown for the
+  suggestions.
+- Removed flex.exe binary from Open MPI tarballs; now generate flex
+  code from a newer (Windows-friendly) flex when we make official
+  tarballs.
+
+
+Open MPI version 1.4.1
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 15 Jan 2010
+
+- Update to PLPA v1.3.2, addressing a licensing issue identified by
+  the Fedora project.  See
+  https://svn.open-mpi.org/trac/plpa/changeset/262 for details.
+- Add check for malformed checkpoint metadata files (Ticket #2141).
+- Fix error path in ompi-checkpoint when not able to checkpoint
+  (Ticket #2138).
+- Cleanup component release logic when selecting checkpoint/restart
+  enabled components (Ticket #2135).
+- Fixed VT node name detection for Cray XT platforms, and fixed some
+  broken VT documentation files.
+- Fix a possible race condition in tearing down RDMA CM-based
+  connections.
+- Relax error checking on MPI_GRAPH_CREATE.  Thanks to David Singleton
+  for pointing out the issue.
+- Fix a shared memory "hang" problem that occurred on x86/x86_64
+  platforms when used with the GNU >=4.4.x compiler series.
+- Add fix for Libtool 2.2.6b's problems with the PGI 10.x compiler
+  suite.  Inspired directly from the upstream Libtool patches that fix
+  the issue (but we need something working before the next Libtool
+  release).
+
+
+Open MPI version 1.4.0
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 8 Dec 2009
+
+- The **only** change in the Open MPI v1.4 release (as compared to v1.3.4)
+  was to update the embedded version of Libtool's libltdl to address a
+  potential security vulnerability.  Specifically: Open MPI v1.3.4 was
+  created with GNU Libtool 2.2.6a; Open MPI v1.4 was created with GNU
+  Libtool 2.2.6b.  There are no other changes between Open MPI v1.3.4
+  and v1.4.
+
+
+Open MPI v1.3.x series
+----------------------
+
+Open MPI version 1.3.4
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 13 Feb 2010
+
+- Fix some issues in OMPI's SRPM with regard to shell_scripts_basename
+  and its use with mpi-selector.  Thanks to Bill Johnstone for
+  pointing out the problem.
+- Added many new MPI job process affinity options to mpirun.  See the
+  newly-updated mpirun(1) man page for details.
+- Several updates to mpirun's XML output.
+- Update to fix a few Valgrind warnings with regards to the ptmalloc2
+  allocator and Open MPI's use of PLPA.
+- Many updates and fixes to the (non-default) "sm" collective
+  component (i.e., native shared memory MPI collective operations).
+- Updates and fixes to some MPI_COMM_SPAWN_MULTIPLE corner cases.
+- Fix some internal copying functions in Open MPI's use of PLPA.
+- Correct some SLURM nodelist parsing logic that may have interfered
+  with large jobs.  Additionally, per advice from the SLURM team,
+  change the environment variable that we use for obtaining the job's
+  allocation.
+- Revert to an older, safer (but slower) communicator ID allocation
+  algorithm.
+- Fixed minimum distance finding for OpenFabrics devices in the openib
+  BTL.
+- Relax the parameter checking MPI_CART_CREATE a bit.
+- Fix MPI_COMM_SPAWN[_MULTIPLE] to only error-check the info arguments
+  on the root process.  Thanks to Federico Golfre Andreasi for
+  reporting the problem.
+- Fixed some BLCR configure issues.
+- Fixed a potential deadlock when the openib BTL was used with
+  MPI_THREAD_MULTIPLE.
+- Fixed dynamic rules selection for the "tuned" coll component.
+- Added a launch progress meter to mpirun (useful for large jobs; set
+  the orte_report_launch_progress MCA parameter to 1 to see it).
+- Reduced the number of file descriptors consumed by each MPI process.
+- Add new device IDs for Chelsio T3 RNICs to the openib BTL config file.
+- Fix some CRS self component issues.
+- Added some MCA parameters to the PSM MTL to tune its run-time
+  behavior.
+- Fix some VT issues with MPI_BOTTOM/MPI_IN_PLACE.
+- Man page updates from the Debain Open MPI package maintainers.
+- Add cycle counter support for the Alpha and Sparc platforms.
+- Pass visibility flags to libltdl's configure script, resulting in
+  those symbols being hidden.  This appears to mainly solve the
+  problem of applications attempting to use different versions of
+  libltdl from that used to build Open MPI.
+
+
+Open MPI version 1.3.3
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 14 Jul 2009
+
+- Fix a number of issues with the openib BTL (OpenFabrics) RDMA CM,
+  including a memory corruption bug, a shutdown deadlock, and a route
+  timeout.  Thanks to David McMillen and Hal Rosenstock for help in
+  tracking down the issues.
+- Change the behavior of the EXTRA_STATE parameter that is passed to
+  Fortran attribute callback functions: this value is now stored
+  internally in MPI |mdash| it no longer references the original value
+  passed by ``MPI_*_CREATE_KEYVAL``.
+- Allow the overriding RFC1918 and RFC3330 for the specification of
+  "private" networks, thereby influencing Open MPI's TCP
+  "reachability" computations.
+- Improve flow control issues in the sm btl, by both tweaking the
+  shared memory progression rules and by enabling the "sync" collective
+  to barrier every 1,000th collective.
+- Various fixes for the IBM XL C/C++ v10.1 compiler.
+- Allow explicit disabling of ptmalloc2 hooks at runtime (e.g., enable
+  support for Debian's builtroot system).  Thanks to Manuel Prinz and
+  the rest of the Debian crew for helping identify and fix this issue.
+- Various minor fixes for the I/O forwarding subsystem.
+- Big endian iWARP fixes in the Open Fabrics RDMA CM support.
+- Update support for various OpenFabrics devices in the openib BTL's
+  .ini file.
+- Fixed undefined symbol issue with Open MPI's parallel debugger
+  message queue support so it can be compiled by Sun Studio compilers.
+- Update MPI_SUBVERSION to 1 in the Fortran bindings.
+- Fix MPI_GRAPH_CREATE Fortran 90 binding.
+- Fix MPI_GROUP_COMPARE behavior with regards to MPI_IDENT.  Thanks to
+  Geoffrey Irving for identifying the problem and supplying the fix.
+- Silence gcc 4.1 compiler warnings about type punning.  Thanks to
+  Number Cruncher for the fix.
+- Added more Valgrind and other memory-cleanup fixes.  Thanks to
+  various Open MPI users for help with these issues.
+- Miscellaneous VampirTrace fixes.
+- More fixes for openib credits in heavy-congestion scenarios.
+- Slightly decrease the latency in the openib BTL in some conditions
+  (add "send immediate" support to the openib BTL).
+- Ensure to allow MPI_REQUEST_GET_STATUS to accept an
+  MPI_STATUS_IGNORE parameter.  Thanks to Shaun Jackman for the bug
+  report.
+- Added Microsoft Windows support.  See README.WINDOWS file for
+  details.
+
+
+Open MPI version 1.3.2
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 22 Apr 2009
+
+- Fixed a potential infinite loop in the openib BTL that could occur
+  in senders in some frequent-communication scenarios.  Thanks to Don
+  Wood for reporting the problem.
+- Add a new checksum PML variation on ob1 (main MPI point-to-point
+  communication engine) to detect memory corruption in node-to-node
+  messages
+- Add a new configuration option to add padding to the openib
+  header so the data is aligned
+- Add a new configuration option to use an alternative checksum algo
+  when using the checksum PML
+- Fixed a problem reported by multiple users on the mailing list that
+  the LSF support would fail to find the appropriate libraries at
+  run-time.
+- Allow empty shell designations from getpwuid().  Thanks to Sergey
+  Koposov for the bug report.
+- Ensure that mpirun exits with non-zero status when applications die
+  due to user signal.  Thanks to Geoffroy Pignot for suggesting the
+  fix.
+- Ensure that MPI_VERSION / MPI_SUBVERSION match what is returned by
+  MPI_GET_VERSION.  Thanks to Rob Egan for reporting the error.
+- Updated ``MPI_*KEYVAL_CREATE`` functions to properly handle Fortran
+  extra state.
+- A variety of ob1 (main MPI point-to-point communication engine) bug
+  fixes that could have caused hangs or seg faults.
+- Do not install Open MPI's signal handlers in MPI_INIT if there are
+  already signal handlers installed.  Thanks to Kees Verstoep for
+  bringing the issue to our attention.
+- Fix GM support to not seg fault in MPI_INIT.
+- Various VampirTrace fixes.
+- Various PLPA fixes.
+- No longer create BTLs for invalid (TCP) devices.
+- Various man page style and lint cleanups.
+- Fix critical OpenFabrics-related bug noted here:
+  https://www.open-mpi.org/community/lists/announce/2009/03/0029.php.
+  Open MPI now uses a much more robust memory intercept scheme that is
+  quite similar to what is used by MX.  The use of "-lopenmpi-malloc"
+  is no longer necessary, is deprecated, and is expected to disappear
+  in a future release.  -lopenmpi-malloc will continue to work for the
+  duration of the Open MPI v1.3 and v1.4 series.
+- Fix some OpenFabrics shutdown errors, both regarding iWARP and SRQ.
+- Allow the udapl BTL to work on Solaris platforms that support
+  relaxed PCI ordering.
+- Fix problem where the mpirun would sometimes use rsh/ssh to launch on
+  the localhost (instead of simply forking).
+- Minor SLURM stdin fixes.
+- Fix to run properly under SGE jobs.
+- Scalability and latency improvements for shared memory jobs: convert
+  to using one message queue instead of N queues.
+- Automatically size the shared-memory area (mmap file) to match
+  better what is needed;  specifically, so that large-np jobs will start.
+- Use fixed-length MPI predefined handles in order to provide ABI
+  compatibility between Open MPI releases.
+- Fix building of the posix paffinity component to properly get the
+  number of processors in loosely tested environments (e.g.,
+  FreeBSD).  Thanks to Steve Kargl for reporting the issue.
+- Fix ``--with-libnuma`` handling in configure.  Thanks to Gus Correa for
+  reporting the problem.
+
+
+Open MPI version 1.3.1
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 19 Mar 2009
+
+- Added "sync" coll component to allow users to synchronize every N
+  collective operations on a given communicator.
+- Increased the default values of the IB and RNR timeout MCA parameters.
+- Fix a compiler error noted by Mostyn Lewis with the PGI 8.0 compiler.
+- Fix an error that prevented stdin from being forwarded if the
+  rsh launcher was in use.  Thanks to Branden Moore for pointing out
+  the problem.
+- Correct a case where the added datatype is considered as contiguous but
+  has gaps in the beginning.
+- Fix an error that limited the number of comm_spawns that could
+  simultaneously be running in some environments
+- Correct a corner case in OB1's GET protocol for long messages; the
+  error could sometimes cause MPI jobs using the openib BTL to hang.
+- Fix a bunch of bugs in the IO forwarding (IOF) subsystem and add some
+  new options to output to files and redirect output to xterm.  Thanks to
+  Jody Weissmann for helping test out many of the new fixes and
+  features.
+- Fix SLURM race condition.
+- Fix MPI_File_c2f(MPI_FILE_NULL) to return 0, not -1.  Thanks to
+  Lisandro Dalcín for the bug report.
+- Fix the DSO build of tm PLM.
+- Various fixes for size disparity between C int's and Fortran
+  INTEGER's.  Thanks to Christoph van Wullen for the bug report.
+- Ensure that mpirun exits with a non-zero exit status when daemons or
+  processes abort or fail to launch.
+- Various fixes to work around Intel (NetEffect) RNIC behavior.
+- Various fixes for mpirun's ``--preload-files`` and ``--preload-binary``
+  options.
+- Fix the string name in MPI::ERRORS_THROW_EXCEPTIONS.
+- Add ability to forward SIFTSTP and SIGCONT to MPI processes if you
+  set the MCA parameter orte_forward_job_control to 1.
+- Allow the sm BTL to allocate larger amounts of shared memory if
+  desired (helpful for very large multi-core boxen).
+- Fix a few places where we used PATH_MAX instead of OPAL_PATH_MAX,
+  leading to compile problems on some platforms.  Thanks to Andrea Iob
+  for the bug report.
+- Fix mca_btl_openib_warn_no_device_params_found MCA parameter; it
+  was accidentally being ignored.
+- Fix some run-time issues with the sctp BTL.
+- Ensure that RTLD_NEXT exists before trying to use it (e.g., it
+  doesn't exist on Cygwin).  Thanks to Gustavo Seabra for reporting
+  the issue.
+- Various fixes to VampirTrace, including fixing compile errors on
+  some platforms.
+- Fixed missing MPI_Comm_accept.3 man page; fixed minor issue in
+  orterun.1 man page.  Thanks to Dirk Eddelbuettel for identifying the
+  problem and submitting a patch.
+- Implement the XML formatted output of stdout/stderr/stddiag.
+- Fixed mpirun's -wdir switch to ensure that working directories for
+  multiple app contexts are properly handled.  Thanks to Geoffroy
+  Pignot for reporting the problem.
+- Improvements to the MPI C++ integer constants:
+
+   - Allow ``MPI::SEEK_*`` constants to be used as constants
+   - Allow other MPI C++ constants to be used as array sizes
+
+- Fix minor problem with orte-restart's command line options.  See
+  ticket #1761 for details.  Thanks to Gregor Dschung for reporting
+  the problem.
+
+
+Open MPI version 1.3.0
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 19 Jan 2009
+
+- Extended the OS X 10.5.x (Leopard) workaround for a problem when
+  assembly code is compiled with -g[0-9].  Thanks to Barry Smith for
+  reporting the problem.  See ticket #1701.
+- Disabled MPI_REAL16 and MPI_COMPLEX32 support on platforms where the
+  bit representation of ``REAL*16`` is different than that of the C type
+  of the same size (usually long double).  Thanks to Julien Devriendt
+  for reporting the issue.  See ticket #1603.
+- Increased the size of MPI_MAX_PORT_NAME to 1024 from 36. See ticket #1533.
+- Added "notify debugger on abort" feature. See tickets #1509 and #1510.
+  Thanks to Seppo Sahrakropi for the bug report.
+- Upgraded Open MPI tarballs to use Autoconf 2.63, Automake 1.10.1,
+  Libtool 2.2.6a.
+- Added missing MPI::Comm::Call_errhandler() function.  Thanks to Dave
+  Goodell for bringing this to our attention.
+- Increased MPI_SUBVERSION value in mpi.h to 1 (i.e., MPI 2.1).
+- Changed behavior of MPI_GRAPH_CREATE, MPI_TOPO_CREATE, and several
+  other topology functions per MPI-2.1.
+- Fix the type of the C++ constant MPI::IN_PLACE.
+- Various enhancements to the openib BTL:
+
+   - Added ``btl_openib_if_[in|ex]clude`` MCA parameters for
+     including/excluding comma-delimited lists of HCAs and ports.
+   - Added RDMA CM support, includng ``btl_openib_cpc_[in|ex]clude``
+     MCA parameters
+   - Added NUMA support to only use "near" network adapters
+   - Added "Bucket SRQ" (BSRQ) support to better utilize registered
+     memory, including btl_openib_receive_queues MCA parameter
+   - Added ConnectX XRC support (and integrated with BSRQ)
+   - Added btl_openib_ib_max_inline_data MCA parameter
+   - Added iWARP support
+   - Revamped flow control mechansisms to be more efficient
+   - ``mpi_leave_pinned=1`` is now the default when possible,
+     automatically improving performance for large messages when
+     application buffers are re-used
+
+- Elimiated duplicated error messages when multiple MPI processes fail
+  with the same error.
+- Added NUMA support to the shared memory BTL.
+- Add Valgrind-based memory checking for MPI-semantic checks.
+- Add support for some optional Fortran datatypes (MPI_LOGICAL1,
+  MPI_LOGICAL2, MPI_LOGICAL4 and MPI_LOGICAL8).
+- Remove the use of the STL from the C++ bindings.
+- Added support for Platform/LSF job launchers.  Must be Platform LSF
+  v7.0.2 or later.
+- Updated ROMIO with the version from MPICH2 1.0.7.
+- Added RDMA capable one-sided component (called rdma), which
+  can be used with BTL components that expose a full one-sided
+  interface.
+- Added the optional datatype MPI_REAL2. As this is added to the "end of"
+  predefined datatypes in the fortran header files, there will not be
+  any compatibility issues.
+- Added Portable Linux Processor Affinity (PLPA) for Linux.
+- Addition of a finer symbols export control via the visibiliy feature
+  offered by some compilers.
+- Added checkpoint/restart process fault tolerance support. Initially
+  support a LAM/MPI-like protocol.
+- Removed "mvapi" BTL; all InfiniBand support now uses the OpenFabrics
+  driver stacks ("openib" BTL).
+- Added more stringent MPI API parameter checking to help user-level
+  debugging.
+- The ptmalloc2 memory manager component is now by default built as
+  a standalone library named libopenmpi-malloc.  Users wanting to
+  use leave_pinned with ptmalloc2 will now need to link the library
+  into their application explicitly.  All other users will use the
+  libc-provided allocator instead of Open MPI's ptmalloc2.  This change
+  may be overriden with the configure option enable-ptmalloc2-internal
+- The leave_pinned options will now default to using mallopt on
+  Linux in the cases where ptmalloc2 was not linked in.  mallopt
+  will also only be available if munmap can be intercepted (the
+  default whenever Open MPI is not compiled with ``--without-memory-manager``.
+- Open MPI will now complain and refuse to use leave_pinned if
+  no memory intercept / mallopt option is available.
+- Add option of using Perl-based wrapper compilers instead of the
+  C-based wrapper compilers.  The Perl-based version does not
+  have the features of the C-based version, but does work better
+  in cross-compile environments.
+
+
+Open MPI v1.2.x series
+----------------------
+
+Open MPI version 1.2.9
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 14 Feb 2009
+
+- Fix a segfault when using one-sided communications on some forms of derived
+  datatypes.  Thanks to Dorian Krause for reporting the bug. See #1715.
+- Fix an alignment problem affecting one-sided communications on
+  some architectures (e.g., SPARC64). See #1738.
+- Fix compilation on Solaris when thread support is enabled in Open MPI
+  (e.g., when using ``--with-threads``). See #1736.
+- Correctly take into account the MTU that an OpenFabrics device port
+  is using. See #1722 and
+  https://bugs.openfabrics.org/show_bug.cgi?id=1369.
+- Fix two datatype engine bugs. See #1677.
+  Thanks to Peter Kjellstrom for the bugreport.
+- Fix the bml r2 help filename so the help message can be found. See #1623.
+- Fix a compilation problem on RHEL4U3 with the PGI 32 bit compiler
+  caused by <infiniband/driver.h>.  See ticket #1613.
+- Fix the ``--enable-cxx-exceptions`` configure option. See ticket #1607.
+- Properly handle when the MX BTL cannot open an endpoint. See ticket #1621.
+- Fix a double free of events on the tcp_events list. See ticket #1631.
+- Fix a buffer overun in opal_free_list_grow (called by MPI_Init).
+  Thanks to Patrick Farrell for the bugreport and Stephan Kramer for
+  the bugfix.  See ticket #1583.
+- Fix a problem setting OPAL_PREFIX for remote sh-based shells.
+  See ticket #1580.
+
+
+Open MPI version 1.2.8
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 14 Oct 2008
+
+- Tweaked one memory barrier in the openib component to be more conservative.
+  May fix a problem observed on PPC machines.  See ticket #1532.
+- Fix OpenFabrics IB partition support. See ticket #1557.
+- Restore v1.1 feature that sourced .profile on remote nodes if the default
+  shell will not do so (e.g. ``/bin/sh`` and ``/bin/ksh``).  See ticket #1560.
+- Fix segfault in MPI_Init_thread() if ompi_mpi_init() fails. See ticket #1562.
+- Adjust SLURM support to first look for $SLURM_JOB_CPUS_PER_NODE instead of
+  the deprecated $SLURM_TASKS_PER_NODE environment variable.  This change
+  may be **required** when using SLURM v1.2 and above.  See ticket #1536.
+- Fix the MPIR_Proctable to be in process rank order. See ticket #1529.
+- Fix a regression introduced in 1.2.6 for the IBM eHCA. See ticket #1526.
+
+
+Open MPI version 1.2.7
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 28 Aug 2008
+
+- Add some Sun HCA vendor IDs.  See ticket #1461.
+- Fixed a memory leak in MPI_Alltoallw when called from Fortran.
+  Thanks to Dave Grote for the bugreport.  See ticket #1457.
+- Only link in libutil when it is needed/desired.  Thanks to
+  Brian Barret for diagnosing and fixing the problem.  See ticket #1455.
+- Update some QLogic HCA vendor IDs.  See ticket #1453.
+- Fix F90 binding for MPI_CART_GET.  Thanks to Scott Beardsley for
+  bringing it to our attention. See ticket #1429.
+- Remove a spurious warning message generated in/by ROMIO. See ticket #1421.
+- Fix a bug where command-line MCA parameters were not overriding
+  MCA parameters set from environment variables.  See ticket #1380.
+- Fix a bug in the AMD64 atomics assembly.  Thanks to Gabriele Fatigati
+  for the bug report and bugfix.  See ticket #1351.
+- Fix a gather and scatter bug on intercommunicators when the datatype
+  being moved is 0 bytes. See ticket #1331.
+- Some more man page fixes from the Debian maintainers.
+  See tickets #1324 and #1329.
+- Have openib BTL (OpenFabrics support) check for the presence of
+  /sys/class/infiniband before allowing itself to be used.  This check
+  prevents spurious "OMPI did not find RDMA hardware!" notices on
+  systems that have the software drivers installed, but no
+  corresponding hardware.  See tickets #1321 and #1305.
+- Added vendor IDs for some ConnectX openib HCAs. See ticket #1311.
+- Fix some RPM specfile inconsistencies.  See ticket #1308.
+  Thanks to Jim Kusznir for noticing the problem.
+- Removed an unused function prototype that caused warnings on
+  some systems (e.g., OS X).  See ticket #1274.
+- Fix a deadlock in inter-communicator scatter/gather operations.
+  Thanks to Martin Audet for the bug report.  See ticket #1268.
+
+
+Open MPI version 1.2.6
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 7 Apr 2008
+
+- Fix a bug in the inter-allgather for asymmetric inter-communicators.
+  Thanks to Martin Audet for the bug report. See ticket #1247.
+- Fix a bug in the openib BTL when setting the CQ depth.  Thanks
+  to Jon Mason for the bug report and fix.  See ticket #1245.
+- On Mac OS X Leopard, the execinfo component will be used for
+  backtraces, making for a more durable solution.  See ticket #1246.
+- Added vendor IDs for some QLogic DDR openib HCAs. See ticket #1227.
+- Updated the URL to get the latest config.guess and config.sub files.
+  Thanks to Ralf Wildenhues for the bug report. See ticket #1226.
+- Added shared contexts support to PSM MTL.  See ticket #1225.
+- Added pml_ob1_use_early_completion MCA parameter to allow users
+  to turn off the OB1 early completion semantic and avoid "stall"
+  problems seen on InfiniBand in some cases.  See ticket #1224.
+- Sanitized some #define macros used in mpi.h to avoid compiler warnings
+  caused by MPI programs built with different autoconf versions.
+  Thanks to Ben Allan for reporting the problem, and thanks to
+  Brian Barrett for the fix. See ticket #1220.
+- Some man page fixes from the Debian maintainers. See ticket #1219.
+- Made the openib BTL a bit more resilient in the face of driver
+  errors.  See ticket #1217.
+- Fixed F90 interface for MPI_CART_CREATE.  See ticket #1208.
+  Thanks to Michal Charemza for reporting the problem.
+- Fixed some C++ compiler warnings. See ticket #1203.
+- Fixed formatting of the orterun man page.  See ticket #1202.
+  Thanks to Peter Breitenlohner for the patch.
+
+
+Open MPI version 1.2.5
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 8 Jan 2008
+
+- Fixed compile issue with open() on Fedora 8 (and newer) platforms.
+  Thanks to Sebastian Schmitzdorff for noticing the problem.
+- Added run-time warnings during MPI_INIT when MPI_THREAD_MULTIPLE
+  and/or progression threads are used (the OMPI v1.2 series does not
+  support these well at all).
+- Better handling of ECONNABORTED from connect on Linux.  Thanks to
+  Bob Soliday for noticing the problem; thanks to Brian Barrett for
+  submitting a patch.
+- Reduce extraneous output from OOB when TCP connections must
+  be retried.  Thanks to Brian Barrett for submitting a patch.
+- Fix for ConnectX devices and OFED 1.3.  See ticket #1190.
+- Fixed a configure problem for Fortran 90 on Cray systems.  Ticket #1189.
+- Fix an uninitialized variable in the error case in opal_init.c.
+  Thanks to Åke Sandgren for pointing out the mistake.
+- Fixed a hang in configure if $USER was not defined.  Thanks to
+  Darrell Kresge for noticing the problem.  See ticket #900.
+- Added support for parallel debuggers even when we have an optimized build.
+  See ticket #1178.
+- Worked around a bus error in the Mac OS X 10.5.X (Leopard) linker when
+  compiling Open MPI with -g.  See ticket #1179.
+- Removed some warnings about 'rm' from Mac OS X 10.5 (Leopard) builds.
+- Fix the handling of mx_finalize().  See ticket #1177.
+  Thanks to Åke Sandgren for bringing this issue to our attention.
+- Fixed minor file descriptor leak in the Altix timer code.  Thanks to
+  Paul Hargrove for noticing the problem and supplying the fix.
+- Fix a problem when using a different compiler for C and Objective C.
+  See ticket #1153.
+- Fix segfault in MPI_COMM_SPAWN when the user specified a working
+  directory.  Thanks to Murat Knecht for reporting this and suggesting
+  a fix.
+- A few manpage fixes from the Debian Open MPI maintainers.  Thanks to
+  Tilman Koschnick, Sylvestre Ledru, and Dirk Eddelbuettel.
+- Fixed issue with pthread detection when compilers are not all
+  from the same vendor.  Thanks to Åke Sandgren for the bug
+  report.  See ticket #1150.
+- Fixed vector collectives in the self module.  See ticket #1166.
+- Fixed some data-type engine bugs: an indexing bug, and an alignment bug.
+  See ticket #1165.
+- Only set the MPI_APPNUM attribute if it is defined.  See ticket
+  #1164.
+
+
+Open MPI version 1.2.4
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 26 Sep 2007
+
+- Really added support for TotalView/DDT parallel debugger message queue
+  debugging (it was mistakenly listed as "added" in the 1.2 release).
+- Fixed a build issue with GNU/kFreeBSD. Thanks to Petr Salinger for
+  the patch.
+- Added missing MPI_FILE_NULL constant in Fortran.  Thanks to
+  Bernd Schubert for bringing this to our attention.
+- Change such that the UDAPL BTL is now only built in Linux when
+  explicitly specified via the ``--with-udapl`` configure command line
+  switch.
+- Fixed an issue with umask not being propagated when using the TM
+  launcher.
+- Fixed behavior if number of slots is not the same on all bproc nodes.
+- Fixed a hang on systems without GPR support (ex. Cray XT3/4).
+- Prevent users of 32-bit MPI apps from requesting >= 2GB of shared
+  memory.
+- Added a Portals MTL.
+- Fix 0 sized MPI_ALLOC_MEM requests.  Thanks to Lisandro Dalcín for
+  pointing out the problem.
+- Fixed a segfault crash on large SMPs when doing collectives.
+- A variety of fixes for Cray XT3/4 class of machines.
+- Fixed which error handler is used when MPI_COMM_SELF is passed
+  to MPI_COMM_FREE.  Thanks to Lisandro Dalcín for the bug report.
+- Fixed compilation on platforms that don't have hton/ntoh.
+- Fixed a logic problem in the fortran binding for MPI_TYPE_MATCH_SIZE.
+  Thanks to Jeff Dusenberry for pointing out the problem and supplying
+  the fix.
+- Fixed a problem with MPI_BOTTOM in various places of the f77-interface.
+  Thanks to Daniel Spangberg for bringing this up.
+- Fixed problem where MPI-optional Fortran datatypes were not
+  correctly initialized.
+- Fixed several problems with stdin/stdout forwarding.
+- Fixed overflow problems with the sm mpool MCA parameters on large SMPs.
+- Added support for the DDT parallel debugger via orterun's ``--debug``
+  command line option.
+- Added some sanity/error checks to the openib MCA parameter parsing
+  code.
+- Updated the udapl BTL to use RDMA capabilities.
+- Allow use of the BProc head node if it was allocated to the user.
+  Thanks to Sean Kelly for reporting the problem and helping debug it.
+- Fixed a ROMIO problem where non-blocking I/O errors were not properly
+  reported to the user.
+- Made remote process launch check the $SHELL environment variable if
+  a valid shell was not otherwise found for the user.
+  Thanks to Alf Wachsmann for the bugreport and suggested fix.
+- Added/updated some vendor IDs for a few openib HCAs.
+- Fixed a couple of failures that could occur when specifying devices
+  for use by the OOB.
+- Removed dependency on sysfsutils from the openib BTL for
+  libibverbs >=v1.1 (i.e., OFED 1.2 and beyond).
+
+
+Open MPI version 1.2.3
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 20 Jun 2007
+
+- Fix a regression in comm_spawn functionality that inadvertently
+  caused the mapping of child processes to always start at the same
+  place.  Thanks to Prakash Velayutham for helping discover the
+  problem.
+- Fix segfault when a user's home directory is unavailable on a remote
+  node.  Thanks to Guillaume Thomas-Collignon for bringing the issue
+  to our attention.
+- Fix MPI_IPROBE to properly handle MPI_STATUS_IGNORE on mx and psm
+  MTLs. Thanks to Sophia Corwell for finding this and supplying a
+  reproducer.
+- Fix some error messages in the tcp BTL.
+- Use _NSGetEnviron instead of environ on Mac OS X so that there
+  are no undefined symbols in the shared libraries.
+- On OS X, when MACOSX_DEPLOYMENT_TARGET is 10.3 or higher, support
+  building the Fortran 90 bindings as a shared library.  Thanks to
+  Jack Howarth for his advice on making this work.
+- No longer require extra include flag for the C++ bindings.
+- Fix detection of weak symbols support with Intel compilers.
+- Fix issue found by Josh England: ompi_info would not show framework
+  MCA parameters set in the environment properly.
+- Rename the oob_tcp_include/exclude MCA params to oob_tcp_if_include/exclude
+  so that they match the naming convention of the btl_tcp_if_include/exclude
+  params.  The old names are depreciated, but will still work.
+- Add -wd as a synonym for the -wdir orterun/mpirun option.
+- Fix the mvapi BTL to compile properly with compilers that do not support
+  anonymous unions.  Thanks to Luis Kornblueh for reporting the bug.
+
+
+Open MPI version 1.2.2
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 16 May 2007
+
+- Fix regression in 1.2.1 regarding the handling of $CC with both
+  absolute and relative path names.
+- Fix F90 array of status dimensions.  Thanks to Randy Bramley for
+  noticing the problem.
+- Add btl_openib_ib_pkey_value MCA parameter for controlling IB port selection.
+- Fixed a variety of threading/locking bugs.
+- Fixed some compiler warnings associated with ROMIO, OS X, and gridengine.
+- If pbs-config can be found, use it to look for TM support.  Thanks
+  to Bas van der Vlies for the inspiration and preliminary work.
+- Fixed a deadlock in orterun when the rsh PLS encounters some errors.
+
+
+Open MPI version 1.2.1
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 25 Apr 2007
+
+- Fixed a number of connection establishment errors in the TCP out-
+  of-band messaging system.
+- Fixed a memory leak when using mpi_comm calls.
+  Thanks to Bas van der Vlies for reporting the problem.
+- Fixed various memory leaks in OPAL and ORTE.
+- Improved launch times when using TM (PBS Pro, Torque, Open PBS).
+- Fixed mpi_leave_pinned to work for all datatypes.
+- Fix functionality allowing users to disable sbrk() (the
+  mpool_base_disable_sbrk MCA parameter) on platforms that support it.
+- Fixed a pair of problems with the TCP "listen_thread" mode for the
+  oob_tcp_listen_mode MCA parameter that would cause failures when
+  attempting to launch applications.
+- Fixed a segfault if there was a failure opening a BTL MX endpoint.
+- Fixed a problem with mpirun's ``--nolocal`` option introduced in 1.2.
+- Re-enabled MPI_COMM_SPAWN_MULTIPLE from singletons.
+- LoadLeveler and TM configure fixes, Thanks to Martin Audet for the
+  bug report.
+- Various C++ MPI attributes fixes.
+- Fixed issues with backtrace code on 64 bit Intel & PPC OS X builds.
+- Fixed issues with multi-word CC variables and libtool.
+  Thanks to Bert Wesarg for the bug reports.
+- Fix issue with non-uniform node naming schemes in SLURM.
+- Fix file descriptor leak in the Grid Engine/N1GE support.
+- Fix compile error on OS X 10.3.x introduced with Open MPI 1.1.5.
+- Implement MPI_TYPE_CREATE_DARRAY function (was in 1.1.5 but not 1.2).
+- Recognize zsh shell when using rsh/ssh for launching MPI jobs.
+- Ability to set the OPAL_DESTDIR or OPAL_PREFIX environment
+  variables to "re-root" an existing Open MPI installation.
+- Always include -I for Fortran compiles, even if the prefix is
+  /usr/local.
+- Support for "fork()" in MPI applications that use the
+  OpenFabrics stack (OFED v1.2 or later).
+- Support for setting specific limits on registered memory.
+
+
+Open MPI version 1.2.0
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 15 Mar 2007
+
+- Fixed race condition in the shared memory fifo's, which led to
+  orphaned messages.
+- Corrected the size of the shared memory file - subtracted out the
+  space the header was occupying.
+- Add support for MPI_2COMPLEX and MPI_2DOUBLE_COMPLEX.
+- Always ensure to create $(includedir)/openmpi, even if the C++
+  bindings are disabled so that the wrapper compilers don't point to
+  a directory that doesn't exist.  Thanks to Martin Audet for
+  identifying the problem.
+- Fixes for endian handling in MPI process startup.
+- Openib BTL initialization fixes for cases where MPI processes in the
+  same job has different numbers of active ports on the same physical
+  fabric.
+- Print more descriptive information when displaying backtraces on
+  OS's that support this functionality, such as the hostname and PID
+  of the process in question.
+- Fixes to properly handle MPI exceptions in C++ on communicators,
+  windows, and files.
+- Much more reliable runtime support, particularly with regards to MPI
+  job startup scalability, BProc support, and cleanup in failure
+  scenarios (e.g., MPI_ABORT, MPI processes abnormally terminating,
+  etc.).
+- Significant performance improvements for MPI collectives,
+  particularly on high-speed networks.
+- Various fixes in the MX BTL component.
+- Fix C++ typecast problems with MPI_ERRCODES_IGNORE.  Thanks to
+  Satish Balay for bringing this to our attention.
+- Allow run-time specification of the maximum amount of registered
+  memory for OpenFabrics and GM.
+- Users who utilize the wrapper compilers (e.g., mpicc and mpif77)
+  will not notice, but the underlying library names for ORTE and OPAL
+  have changed to libopen-rte and libopen-pal, respectively (listed
+  here because there are undoubtedly some users who are not using the
+  wrapper compilers).
+- Many bug fixes to MPI-2 one-sided support.
+- Added support for TotalView message queue debugging.
+- Fixes for MPI_STATUS_SET_ELEMENTS.
+- Print better error messages when mpirun's "-nolocal" is used when
+  there is only one node available.
+- Added man pages for several Open MPI executables and the MPI API
+  functions.
+- A number of fixes for Alpha platforms.
+- A variety of Fortran API fixes.
+- Build the Fortran MPI API as a separate library to allow these
+  functions to be profiled properly.
+- Add new ``--enable-mpirun-prefix-by-default`` configure option to always
+  imply the ``--prefix`` option to mpirun, preventing many rsh/ssh-based
+  users from needing to modify their shell startup files.
+- Add a number of missing constants in the C++ bindings.
+- Added tight integration with Sun N1 Grid Engine (N1GE) 6 and the
+  open source Grid Engine.
+- Allow building the F90 MPI bindings as shared libraries for most
+  compilers / platforms.  Explicitly disallow building the F90
+  bindings as shared libraries on OS X because of complicated
+  situations with Fortran common blocks and lack of support for
+  unresolved common symbols in shared libraries.
+- Added stacktrace support for Solaris and Mac OS X.
+- Update event library to libevent-1.1b.
+- Fixed standards conformance issues with MPI_ERR_TRUNCATED and
+  setting MPI_ERROR during MPI_TEST/MPI_WAIT.
+- Addition of "cm" PML to better support library-level matching
+  interconnects, with support for Myrinet/MX, and QLogic PSM-based
+  networks.
+- Addition of "udapl" BTL for transport across uDAPL interconnects.
+- Really check that the $CXX given to configure is a C++ compiler
+  (not a C compiler that "sorta works" as a C++ compiler).
+- Properly check for local host only addresses properly, looking
+  for 127.0.0.0/8, rather than just 127.0.0.1.
+
+
+Open MPI v1.1.x series
+----------------------
+
+Open MPI version 1.1.5
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 19 Mar 2007
+
+- Implement MPI_TYPE_CREATE_DARRAY function.
+- Fix race condition in shared memory BTL startup that could cause MPI
+  applications to hang in MPI_INIT.
+- Fix syntax error in a corner case of the event library.  Thanks to
+  Bert Wesarg for pointing this out.
+- Add new MCA parameter (mpi_preconnect_oob) for pre-connecting the
+  "out of band" channels between all MPI processes.  Most helpful for
+  MPI applications over InfiniBand where process A sends an initial
+  message to process B, but process B does not enter the MPI library
+  for a long time.
+- Fix for a race condition in shared memory locking semantics.
+- Add major, minor, and release version number of Open MPI to mpi.h.
+  Thanks to Martin Audet for the suggestion.
+- Fix the "restrict" compiler check in configure.
+- Fix a problem with argument checking in MPI_TYPE_CREATE_SUBARRAY.
+- Fix a problem with compiling the XGrid components with non-gcc
+  compilers.
+
+
+Open MPI version 1.1.4
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 30 Jan 2007
+
+- Fixed 64-bit alignment issues with TCP interface detection on
+  intel-based OS X machines.
+- Adjusted TCP interface selection to automatically ignore Linux
+  channel-bonded slave interfaces.
+- Fixed the type of the first parameter to the MPI F90 binding for
+  MPI_INITIALIZED.  Thanks to Tim Campbell for pointing out the
+  problem.
+- Fix a bunch of places in the Fortran MPI bindings where (``MPI_Fint*``)
+  was mistakenly being used instead of (``MPI_Aint*``).
+- Fixes for fortran MPI_STARTALL, which could sometimes return
+  incorrect request values.  Thanks to Tim Campbell for pointing out
+  the problem.
+- Include both pre- and post-MPI-2 errata bindings for
+  MPI::Win::Get_attr.
+- Fix math error on Intel OS X platforms that would greatly increase
+  shared memory latency.
+- Fix type casting issue with MPI_ERRCODES_IGNORE that would cause
+  errors when using a C++ compiler.  Thanks to Barry Smith for
+  bringing this to our attention.
+- Fix possible segmentation fault during shutdown when using the
+  MX BTL.
+
+
+Open MPI version 1.1.3
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 26 Jan 2007
+
+- Remove the "hierarch" coll component; it was not intended to be
+  included in stable releases yet.
+- Fix a race condition with stdout/stderr not appearing properly from
+  all processes upon termination of an MPI job.
+- Fix internal accounting errors with the self BTL.
+- Fix typos in the code path for when sizeof(int) != sizeof(INTEGER)
+  in the MPI F77 bindings functions.  Thanks to Pierre-Matthieu
+  Anglade for bringing this problem to our attention.
+- Fix for a memory leak in the derived datatype function
+  ompi_ddt_duplicate().  Thanks to Andreas Schäfer for reporting,
+  diagnosing, and patching the leak.
+- Used better performing basic algorithm for MPI_ALLGATHERV.
+- Added a workaround for a bug in the Intel 9.1 C++ compiler (all
+  versions up to and including 20060925) in the MPI C++ bindings that
+  caused run-time failures.  Thanks to Scott Weitzenkamp for reporting
+  this problem.
+- Fix MPI_SIZEOF implementation in the F90 bindings for COMPLEX
+  variable types.
+- Fixes for persistent requests involving MPI_PROC_NULL.  Thanks to
+  Lisandro Dalcín for reporting the problem.
+- Fixes to ``MPI_TEST*`` and ``MPI_WAIT*`` for proper MPI exception reporting.
+  Thanks to Lisandro Dalcín for finding the issue.
+- Various fixes for MPI generalized request handling; addition of
+  missing MPI::Grequest functionality to the C++ bindings.
+- Add "mpi_preconnect_all" MCA parameter to force wireup of all MPI
+  connections during MPI_INIT (vs. making connections lazily whenever
+  the first MPI communication occurs between a pair of peers).
+- Fix a problem for when $FC and/or $F77 were specified as multiple
+  tokens.  Thanks to Orion Poplawski for identifying the problem and
+  to Ralf Wildenhues for suggesting the fix.
+- Fix several ``MPI_*ERRHANDLER*`` functions and MPI_GROUP_TRANSLATE_RANKS
+  with respect to what arguments they allowed and the behavior that
+  they effected.  Thanks to Lisandro Dalcín for reporting the
+  problems.
+
+
+Open MPI version 1.1.2
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 18 Oct 2006
+
+- Really fix Fortran status handling in MPI_WAITSOME and MPI_TESTSOME.
+- Various datatype fixes, reported by several users as causing
+  failures in the BLACS testing suite.  Thanks to Harald Forbert, Åke
+  Sandgren and, Michael Kluskens for reporting the problem.
+- Correctness and performance fixes for heterogeneous environments.
+- Fixed a error in command line parsing on some platforms (causing
+  mpirun to crash without doing anything).
+- Fix for initialization hangs on 64 bit Mac OS X PowerPC systems.
+- Fixed some memory allocation problems in mpirun that could cause
+  random problems if "-np" was not specified on the command line.
+- Add Kerberos authentication support for XGrid.
+- Added LoadLeveler support for jobs larger than 128 tasks.
+- Fix for large-sized Fortran LOGICAL datatypes.
+- Fix various error checking in MPI_INFO_GET_NTHKEY and
+  MPI_GROUP_TRANSLATE_RANKS, and some collective operations
+  (particularly with regards to MPI_IN_PLACE).  Thanks to Lisandro
+  Dalcín for reporting the problems.
+- Fix receiving messages to buffers allocated by MPI_ALLOC_MEM.
+- Fix a number of race conditions with the MPI-2 Onesided
+  interface.
+- Fix the "tuned" collective componenete where some cases where
+  MPI_BCAST could hang.
+- Update TCP support to support non-uniform TCP environments.
+- Allow the "poe" RAS component to be built on AIX or Linux.
+- Only install mpif.h if the rest of the Fortran bindings are
+  installed.
+- Fixes for BProc node selection.
+- Add some missing Fortran MPI-2 IO constants.
+
+
+Open MPI version 1.1.1
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 28 Aug 2006
+
+- Fix for Fortran string handling in various MPI API functions.
+- Fix for Fortran status handling in MPI_WAITSOME and MPI_TESTSOME.
+- Various fixes for the XL compilers.
+- Automatically disable using mallot() on AIX.
+- Memory fixes for 64 bit platforms with registering MCA parameters in
+  the self and MX BTL components.
+- Fixes for BProc to support oversubscription and changes to the
+  mapping algorithm so that mapping processes "by slot" works as
+  expected.
+- Fixes for various abort cases to not hang and clean up nicely.
+- If using the Intel 9.0 v20051201 compiler on an IA64 platform, the
+  ptmalloc2 memory manager component will automatically disable
+  itself.  Other versions of the Intel compiler on this platform seem
+  to work fine (e.g., 9.1).
+- Added "host" MPI_Info key to MPI_COMM_SPAWN and
+  MPI_COMM_SPAWN_MULTIPLE.
+- Add missing C++ methods: MPI::Datatype::Create_indexed_block,
+  MPI::Datatype::Create_resized, MPI::Datatype::Get_true_extent.
+- Fix OSX linker issue with Fortran bindings.
+- Fixed MPI_COMM_SPAWN to start spawning new processes in slots that
+  (according to Open MPI) are not already in use.
+- Added capability to "mpirun a.out" (without specifying -np) that
+  will run on all currently-allocated resources (e.g., within a batch
+  job such as SLURM, Torque, etc.).
+- Fix a bug with one particular case of MPI_BCAST.  Thanks to Doug
+  Gregor for identifying the problem.
+- Ensure that the shared memory mapped file is only created when there
+  is more than one process on a node.
+- Fixed problems with BProc stdin forwarding.
+- Fixed problem with MPI_TYPE_INDEXED datatypes.  Thanks to Yven
+  Fournier for identifying this problem.
+- Fix some thread safety issues in MPI attributes and the openib BTL.
+- Fix the BProc allocator to not potentially use the same resources
+  across multiple ORTE universes.
+- Fix gm resource leak.
+- More latency reduction throughout the code base.
+- Make the TM PLS (PBS Pro, Torque, Open PBS) more scalable, and fix
+  some latent bugs that crept in v1.1.  Thanks to the Thunderbird crew
+  at Sandia National Laboratories and Martin Schaffoner for access to
+  testing facilities to make this happen.
+- Added new command line options to mpirun:
+
+  - ``--nolocal``: Do not run any MPI processes on the same node as mpirun
+    (compatibility with the OSC mpiexec launcher)
+  - ``--nooversubscribe``: Abort if the number of processes requested would
+    cause oversubscription
+  - ``--quiet / -q``: do not show spurious status messages
+  - ``--version / -V``: show the version of Open MPI
+
+- Fix bus error in XGrid process starter.  Thanks to Frank from the
+  Open MPI user's list for identifying the problem.
+- Fix data size mismatches that caused memory errors on PPC64
+  platforms during the startup of the openib BTL.
+- Allow propagation of SIGUSR1 and SIGUSR2 signals from mpirun to
+  back-end MPI processes.
+- Add missing MPI::Is_finalized() function.
+
+
+Open MPI version 1.1.0
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 23 Jun 2006
+
+- Various MPI datatype fixes, optimizations.
+- Fixed various problems on the SPARC architecture (e.g., not
+  correctly aligning addresses within structs).
+- Improvements in various run-time error messages to be more clear
+  about what they mean and where the errors are occurring.
+- Various fixes to mpirun's handling of ``--prefix``.
+- Updates and fixes for Cray/Red Storm support.
+- Major improvements to the Fortran 90 MPI bindings:
+
+   - General improvements in compile/linking time and portability
+     between different F90 compilers.
+   - Addition of "trivial", "small" (the default), and "medium"
+     Fortran 90 MPI module sizes (v1.0.x's F90 module was
+     equivalent to "medium").  See the README file for more
+     explanation.
+   - Fix various MPI F90 interface functions and constant types to
+     match.  Thanks to Michael Kluskens for pointing out the problems
+     to us.
+
+- Allow short messagees to use RDMA (vs. send/receive semantics) to a
+  limited number peers in both the mvapi and openib BTL components.
+  This reduces communication latency over IB channels.
+- Numerous performance improvements throughout the entire code base.
+- Many minor threading fixes.
+- Add a define OMPI_SKIP_CXX to allow the user to skip the mpicxx.h from
+  being included in mpi.h. It allows the user to compile C code with a CXX
+  compiler without including the CXX bindings.
+- PERUSE support has been added. In order to activate it add
+  ``--enable-peruse`` to the configure options. All events described in
+  the PERUSE 2.0 draft are supported, plus one Open MPI
+  extension. PERUSE_COMM_REQ_XFER_CONTINUE allow to see how the data
+  is segmented internally, using multiple interfaces or the pipeline
+  engine. However, this version only support one event of each type
+  simultaneously attached to a communicator.
+- Add support for running jobs in heterogeneous environments.
+  Currently supports environments with different endianness and
+  different representations of C++ bool and Fortran LOGICAL.
+  Mismatched sizes for other datatypes is not supported.
+- Open MPI now includes an implementation of the MPI-2 One-Sided
+  Communications specification.
+- Open MPI is now configurable in cross-compilation environments.
+  Several Fortran 77 and Fortran 90 tests need to be pre-seeded with
+  results from a config.cache-like file.
+- Add ``--debug`` option to mpirun to generically invoke a parallel debugger.
+
+
+Open MPI v1.0.x series
+----------------------
+
+Open MPI version 1.0.3
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: Not released (all fixes included in 1.1.0)
+
+.. important::
+   v1.0.3 was not released
+
+- Fix a problem noted by Chris Hennes where MPI_INFO_SET incorrectly
+  disallowed long values.
+- Fix a problem in the launch system that could cause inconsistent
+  launch behavior, particularly when launching large jobs.
+- Require that the openib BTL find <sysfs/libsysfs.h>.  Thanks to Josh
+  Aune for the suggestion.
+- Include updates to support the upcoming Autoconf 2.60 and Libtool
+  2.0.  Thanks to Ralf Wildenhues for all the work!
+- Fix bug with infinite loop in the "round robin" process mapper.
+  Thanks to Paul Donohue for reporting the problem.
+- Enusre that memory hooks are removed properly during MPI_FINALIZE.
+  Thanks to Neil Ludban for reporting the problem.
+- Various fixes to the included support for ROMIO.
+- Fix to ensure that MPI_LONG_LONG and MPI_LONG_LONG_INT are actually
+  synonyms, as defined by the MPI standard.  Thanks to Martin Audet
+  for reporting this.
+- Fix Fortran 90 configure tests to properly utilize LDFLAGS and LIBS.
+  Thanks to Terry Reeves for reporting the problem.
+- Fix shared memory progression in asynchronous progress scenarios.
+  Thanks to Mykael Bouquey for reporting the problem.
+- Fixed back-end operations for predefined MPI_PROD for some
+  datatypes.  Thanks to Bert Wesarg for reporting this.
+- Adapted configure to be able to handle Torque 2.1.0p0's (and above)
+  new library name.  Thanks to Brock Palen for pointing this out and
+  providing access to a Torque 2.1.0p0 cluster to test with.
+- Fixed situation where mpirun could set a shell pipeline's stdout
+  to non-blocking, causing the shell pipeline to prematurely fail.
+  Thanks to Darrell Kresge for figuring out what was happening.
+- Fixed problems with leave_pinned that could cause Badness with the
+  mvapi BTL.
+- Fixed problems with MPI_FILE_OPEN and non-blocking MPI-2 IO access.
+- Fixed various InfiniBand port matching issues during startup.
+  Thanks to Scott Weitzenkamp for identifying these problems.
+- Fixed various configure, build and run-time issues with ROMIO.
+  Thanks to Dries Kimpe for bringing them to our attention.
+- Fixed error in MPI_COMM_SPLIT when dealing with intercommunicators.
+  Thanks to Bert Wesarg for identifying the problem.
+- Fixed backwards handling of "high" parameter in MPI_INTERCOMM_MERGE.
+  Thanks to Michael Kluskens for pointing this out to us.
+- Fixed improper handling of string arguments in Fortran bindings
+  for MPI-IO functionality
+- Fixed segmentation fault with 64 bit applications on Solaris when
+  using the shared memory transports.
+- Fixed MPI_COMM_SELF attributes to free properly at the beginning of
+  MPI_FINALIZE.  Thanks to Martin Audet for bringing this to our
+  attention.
+- Fixed alignment tests for cross-compiling to not cause errors with
+  recent versions of GCC.
+
+
+Open MPI version 1.0.2
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 7 Apr 2006
+
+- Fixed assembly race condition on AMD64 platforms.
+- Fixed residual .TRUE. issue with copying MPI attributes set from
+  Fortran.
+- Remove unnecessary logic from Solaris pty I/O forwarding.  Thanks to
+  Francoise Roch for bringing this to our attention.
+- Fixed error when count = 0 was given for multiple completion MPI
+  functions (MPI_TESTSOME, MPI_TESTANY, MPI_TESTALL, MPI_WAITSOME,
+  MPI_WAITANY, MPI_WAITALL).
+- Better handling in MPI_ABORT for when peer processes have already
+  died, especially under some resource managers.
+- Random updates to README file, to include notes about the Portland
+  compilers.
+- Random, small threading fixes to prevent deadlock.
+- Fixed a problem with handling long mpirun app files.  Thanks to Ravi
+  Manumachu for identifying the problem.
+- Fix handling of strings in several of the Fortran 77 bindings.
+- Fix LinuxPPC assembly issues.  Thanks to Julian Seward for reporting
+  the problem.
+- Enable pty support for standard I/O forwarding on platforms that
+  have ptys but do not have openpty().  Thanks to Pierre Valiron for
+  bringing this to our attention.
+- Disable inline assembly for PGI compilers to avoid compiler errors.
+  Thanks to Troy Telford for bringing this to our attention.
+- Added MPI_UNSIGNED_CHAR and MPI_SIGNED_CHAR to the allowed reduction
+  types.
+- Fix a segv in variable-length message displays on Opterons running
+  Solaris.  Thanks to Pierre Valiron for reporting the issue.
+- Added MPI_BOOL to the intrinsic reduction operations MPI_LAND,
+  MPI_LOR, MPI_LXOR.  Thanks to Andy Selle for pointing this out to us.
+- Fixed TCP BTL network matching logic during MPI_INIT; in some cases
+  on multi-NIC nodes, a NIC could get paired with a NIC on another
+  network (typically resulting in deadlock).  Thanks to Ken Mighell
+  for pointing this out to us.
+- Change the behavior of orterun (mpirun, mpirexec) to search for
+  argv[0] and the cwd on the target node (i.e., the node where the
+  executable will be running in all systems except BProc, where the
+  searches are run on the node where orterun is invoked).
+- Fix race condition in shared memory transport that could cause
+  crashes on machines with weak memory consistency models (including
+  POWER/PowerPC machines).
+- Fix warnings about setting read-only MCA parameters on bproc systems.
+- Change the exit status set by mpirun when an application process is
+  killed by a signal.  The exit status is now set to signo + 128, which
+  conforms with the behavior of (almost) all shells.
+- Correct a datatype problem with the convertor when partially
+  unpacking data. Now we can position the convertor to any position
+  not only on the predefined types boundaries. Thanks to Yvan Fournier
+  for reporting this to us.
+- Fix a number of standard I/O forwarding issues, including the
+  ability to background mpirun and a loss of data issue when
+  redirecting mpirun's standard input from a file.
+- Fixed bug in ompi_info where rcache and bml MCA parameters would not
+  be displayed.
+- Fixed umask issues in the session directory.  Thanks to Glenn Morris
+  for reporting this to us.
+- Fixed tcsh-based LD_LIBRARY_PATH issues with ``--prefix``.  Thanks to
+  Glen Morris for identifying the problem and suggesting the fix.
+- Removed extraneous \n's when setting PATH and LD_LIBRARY_PATH in the
+  rsh startup.  Thanks to Glen Morris for finding these typos.
+- Fixed missing constants in MPI C++ bindings.
+- Fixed some errors caused by threading issues.
+- Fixed openib BTL flow control logic to not overrun the number of
+  send wqes available.
+- Update to match newest OpenIB user-level library API.  Thanks to
+  Roland Dreier for submitting this patch.
+- Report errors properly when failing to register memory in the openib
+  BTL.
+- Reduce memory footprint of openib BTL.
+- Fix parsing problem with mpirun's "-tv" switch.  Thanks to Chris
+  Gottbrath for supplying the fix.
+- Fix Darwin net/if.h configure warning.
+- The GNU assembler unbelievably defaults to making stacks executable.
+  So when using gas, add flags to explicitly tell it to not make
+  stacks executable (lame but necessary).
+- Add missing MPI::Request::Get_status() methods.  Thanks to Bill
+  Saphir for pointing this out to us.
+- Improved error messages on memory registration errors (e.g., when
+  using high-speed networks).
+- Open IB support now checks firmware for how many outstanding RDMA
+  requests are supported.  Thanks to Mellanox for pointing this out to
+  us.
+- Enable printing of stack traces in MPI processes upon SIGBUS,
+  SIGSEGV, and SIGFPE if the platform supports it.
+- Fixed F90 compilation support for the Lahey compiler.
+- Fixed issues with ROMIO shared library support.
+- Fixed internal accounting problems with rsh support.
+- Update to GNU Libtool 1.5.22.
+- Fix error in configure script when setting CCAS to ias (the Intel
+  assembler).
+- Added missing MPI::Intercomm collectives.
+- Fixed MPI_IN_PLACE handling for Fortran collectives.
+- Fixed some more C++ const_cast<> issues.  Thanks for Martin Audet
+  (again) for bringing this to our attention.
+- Updated ROMIO with the version from MPICH 1.2.7p1, marked as version
+  2005-06-09.
+- Fixes for some cases where the use of MPI_BOTTOM could cause
+  problems.
+- Properly handle the case where an mVAPI does not have shared receive
+  queue support (such as the one shipped by SilverStorm / Infinicon
+  for OS X).
+
+
+Open MPI version 1.0.1
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 12 Dec 2005
+
+- Fixed assembly on Solaris AMD platforms.  Thanks to Pierre Valiron
+  for bringing this to our attention.
+- Fixed long messages in the send-to-self case.
+- Ensure that when the "leave_pinned" option is used, the memory hooks
+  are also enabled.  Thanks to Gleb Natapov for pointing this out.
+- Fixed compile errors for IRIX.
+- Allow hostfiles to have integer host names (for BProc clusters).
+- Fixed a problem with message matching of out-of-order fragments in
+  multiple network device scenarios.
+- Converted all the C++ MPI bindings to use proper const_cast<>'s
+  instead of old C-style casts to get rid of const-ness.  Thanks to
+  Martin Audet for raising the issue with us.
+- Converted MPI_Offset to be a typedef instead of a #define because it
+  causes problems for some C++ parsers.  Thanks to Martin Audet for
+  bringing this to our attention.
+- Improved latency of TCP BTL.
+- Fixed index value in MPI_TESTANY to be MPI_UNDEFINED if some
+  requests were not MPI_REQUEST_NULL, but no requests finished.
+- Fixed several Fortran MPI API implementations that incorrectly used
+  integers instead of logicals or address-sized integers.
+- Fix so that Open MPI correctly handles the Fortran value for .TRUE.,
+  regardless of what the Fortran compiler's value for .TRUE. is.
+- Improved scalability of MX startup.
+- Fix datatype offset handling in the coll basic component's
+  MPI_SCATTERV implementation.
+- Fix EOF handling on stdin.
+- Fix missing MPI_F_STATUS_IGNORE and MPI_F_STATUSES_IGNORE
+  instanatiations.  Thanks to Anthony Chan for pointing this out.
+- Add a missing value for MPI_WIN_NULL in mpif.h.
+- Bring over some fixes for the sm btl that somehow didn't make it
+  over from the trunk before v1.0.  Thanks to Beth Tibbitts and Bill
+  Chung for helping identify this issue.
+- Bring over some fixes for the iof that somehow didn't make it over
+  from the trunk before v1.0.
+- Fix for ``--with-wrapper-ldflags`` handling.  Thanks to Dries Kimpe for
+  pointing this out to us.
+
+
+Open MPI version 1.0.0
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 17 Nov 2005
+
+- Initial public release.
diff --git a/docs/release-notes/changelog/v2.x.rst b/docs/release-notes/changelog/v2.x.rst
new file mode 100644
index 00000000000..896055a43b6
--- /dev/null
+++ b/docs/release-notes/changelog/v2.x.rst
@@ -0,0 +1,836 @@
+Open MPI v2.x series
+====================
+
+This file contains all the NEWS updates for all the Open MPI v2.x
+series, in reverse chronological order.
+
+Open MPI v2.1.x series
+----------------------
+
+Open MPI version 2.1.5
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: August 2018
+
+- A subtle race condition bug was discovered in the "vader" BTL
+  (shared memory communications) that, in rare instances, can cause
+  MPI processes to crash or incorrectly classify (or effectively drop)
+  an MPI message sent via shared memory.  If you are using the "ob1"
+  PML with "vader" for shared memory communication (note that vader is
+  the default for shared memory communication with ob1), you need to
+  upgrade to v2.1.5 to fix this issue.  You may also upgrade to the
+  following versions to fix this issue:
+
+  - Open MPI v3.0.1 (released March, 2018) or later in the v3.0.x
+    series
+  - Open MPI v3.1.2 (expected end of August, 2018) or later
+
+- A link issue was fixed when the UCX library was not located in the
+  linker-default search paths.
+
+Open MPI version 2.1.4
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: August, 2018
+
+- Disable the POWER 7/BE block in configure.  Note that POWER 7/BE is
+  still not a supported platform, but it is no longer automatically
+  disabled.  See
+  https://github.com/open-mpi/ompi/issues/4349#issuecomment-374970982
+  for more information.
+- Fix bug with request-based one-sided MPI operations when using the
+  "rdma" component.
+- Fix issue with large data structure in the TCP BTL causing problems
+  in some environments.  Thanks to @lgarithm for reporting the issue.
+- Minor Cygwin build fixes.
+- Minor fixes for the openib BTL:
+  1. Support for the QLogic RoCE HCA
+  2. Support for the Boradcom Cumulus RoCE HCA
+  3. Enable support for HDR link speeds
+- Fix MPI_FINALIZED hang if invoked from an attribute destructor
+  during the MPI_COMM_SELF destruction in MPI_FINALIZE.  Thanks to
+  @AndrewGaspar for reporting the issue.
+- Java fixes:
+
+  - Modernize Java framework detection, especially on OS X/MacOS.
+    Thanks to Bryce Glover for reporting and submitting the fixes.
+  - Prefer "javac -h" to "javah" to support newer Java frameworks.
+
+- Fortran fixes:
+
+  - Use conformant dummy parameter names for Fortran bindings.  Thanks
+    to Themos Tsikas for reporting and submitting the fixes.
+  - Build the MPI_SIZEOF() interfaces in the "TKR"-style "mpi" module
+    whenever possible.  Thanks to Themos Tsikas for reporting the
+    issue.
+  - Fix array of argv handling for the Fortran bindings of
+    MPI_COMM_SPAWN_MULTIPLE (and its associated man page).
+  - Make NAG Fortran compiler support more robust in configure.
+
+- Disable the "pt2pt" one-sided MPI component when MPI_THREAD_MULTIPLE
+  is used.  This component is simply not safe in MPI_THREAD_MULTIPLE
+  scenarios, and will not be fixed in the v2.1.x series.
+- Make the "external" hwloc component fail gracefully if it is tries
+  to use an hwloc v2.x.y installation.  hwloc v2.x.y will not be
+  supported in the Open MPI v2.1.x series.
+- Fix "vader" shared memory support for messages larger than 2GB.
+  Thanks to Heiko Bauke for the bug report.
+- Configure fixes for external PMI directory detection.  Thanks to
+  Davide Vanzo for the report.
+
+
+Open MPI version 2.1.3
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: March, 2018
+
+- Update internal PMIx version to 1.2.5.
+- Fix a problem with ompi_info reporting using param option.
+  Thanks to Alexander Pozdneev for reporting.
+- Correct PMPI_Aint_{add|diff} to be functions (not subroutines)
+  in the Fortran mpi_f08 module.
+- Fix a problem when doing MPI I/O using data types with large
+  extents in conjunction with MPI_TYPE_CREATE_SUBARRAY.  Thanks to
+  Christopher Brady for reporting.
+- Fix a problem when opening many files using MPI_FILE_OPEN.
+  Thanks to William Dawson for reporting.
+- Fix a problem with debuggers failing to attach to a running job.
+  Thanks to Dirk Schubert for reporting.
+- Fix a problem when using madvise and the OpenIB BTL.  Thanks to
+  Timo Bingmann for reporting.
+- Fix a problem in the Vader BTL that resulted in failures of
+  IMB under certain circumstances.  Thanks to Nicolas Morey-
+  Chaisemartin for reporting.
+- Fix a problem preventing Open MPI from working under Cygwin.
+  Thanks to Marco Atzeri for reporting.
+- Reduce some verbosity being emitted by the USNIC BTL under certain
+  circumstances.  Thanks to Peter Forai for reporting.
+- Fix a problem with misdirection of SIGKILL.  Thanks to Michael Fern
+  for reporting.
+- Replace use of posix_memalign with malloc for small allocations.  Thanks
+  to Ben Menaude for reporting.
+- Fix a problem with Open MPI's out of band TCP network for file descriptors
+  greater than 32767.  Thanks to Wojtek Wasko for reporting and fixing.
+- Plug a memory leak in MPI_Mem_free().  Thanks to Philip Blakely for reporting.
+
+
+Open MPI version 2.1.2
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: September, 2017
+
+- Update internal PMIx version to 1.2.3.
+- Fix some problems when using the NAG Fortran compiler to build Open MPI
+  and when using the compiler wrappers.  Thanks to Neil Carlson for reporting.
+- Fix a compilation problem with the SM BTL.  Thanks to Paul Hargrove for
+  reporting.
+- Fix a problem with MPI_IALLTOALLW when using zero-length messages.
+  Thanks to Dahai Guo for reporting.
+- Fix a problem with C11 generic type interface for SHMEM_G.  Thanks
+  to Nick Park for reporting.
+- Switch to using the lustreapi.h include file when building Open MPI
+  with Lustre support.
+- Fix a problem in the OB1 PML that led to hangs with OSU collective tests.
+- Fix a progression issue with MPI_WIN_FLUSH_LOCAL.  Thanks to
+  Joseph Schuchart for reporting.
+- Fix an issue with recent versions of PBSPro requiring libcrypto.
+  Thanks to Petr Hanousek for reporting.
+- Fix a problem when using MPI_ANY_SOURCE with MPI_SENDRECV.
+- Fix an issue that prevented signals from being propagated to ORTE
+  daemons.
+- Ensure that signals are forwarded from ORTE daemons to all processes
+  in the process group created by the daemons.  Thanks to Ted Sussman
+  for reporting.
+- Fix a problem with launching a job under a debugger. Thanks to
+  Greg Lee for reporting.
+- Fix a problem with Open MPI native I/O MPI_FILE_OPEN when using
+  a communicator having an associated topology.  Thanks to
+  Wei-keng Liao for reporting.
+- Fix an issue when using MPI_ACCUMULATE with derived datatypes.
+- Fix a problem with Fortran bindings that led to compilation errors
+  for user defined reduction operations.  Thanks to Nathan Weeks for
+  reporting.
+- Fix ROMIO issues with large writes/reads when using NFS file systems.
+- Fix definition of Fortran MPI_ARGV_NULL and MPI_ARGVS_NULL.
+- Enable use of the head node of a SLURM allocation on Cray XC systems.
+- Fix a problem with synchronous sends when using the UCX PML.
+- Use default socket buffer size to improve TCP BTL performance.
+- Add a mca parameter ras_base_launch_orted_on_hn to allow for launching
+  MPI processes on the same node where mpirun is executing using a separate
+  orte daemon, rather than the mpirun process.   This may be useful to set to
+  true when using SLURM, as it improves interoperability with SLURM's signal
+  propagation tools.  By default it is set to false, except for Cray XC systems.
+- Fix ``--without-lsf`` when lsf is installed in the default search path.
+- Remove support for big endian PowerPC.
+- Remove support for XL compilers older than v13.1
+- Remove IB XRC support from the OpenIB BTL due to loss of maintainer.
+
+
+Open MPI version 2.1.1
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: April, 2017
+
+- Fix a problem with one of Open MPI's fifo data structures which led to
+  hangs in a make check test.  Thanks to Nicolas Morey-Chaisemartin for
+  reporting.
+- Add missing MPI_AINT_ADD/MPI_AINT_DIFF function definitions to mpif.h.
+  Thanks to Aboorva Devarajan for reporting.
+- Fix the error return from MPI_WIN_LOCK when rank argument is invalid.
+  Thanks to Jeff Hammond for reporting and fixing this issue.
+- Fix a problem with mpirun/orterun when started under a debugger. Thanks
+  to Gregory Leff for reporting.
+- Add configury option to disable use of CMA by the vader BTL.  Thanks
+  to Sascha Hunold for reporting.
+- Add configury check for MPI_DOUBLE_COMPLEX datatype support.
+  Thanks to Alexander Klein for reporting.
+- Fix memory allocated by MPI_WIN_ALLOCATE_SHARED to
+  be 64 bit aligned.  Thanks to Joseph Schuchart for
+  reporting.
+- Update MPI_WTICK man page to reflect possibly higher
+  resolution than 10e-6.  Thanks to Mark Dixon for
+  reporting
+- Add missing MPI_T_PVAR_SESSION_NULL definition to mpi.h
+  include file.  Thanks to Omri Mor for this contribution.
+- Enhance the Open MPI spec file to install modulefile in ``/opt``
+  if installed in a non-default location.  Thanks to Kevin
+  Buckley for reporting and supplying a fix.
+- Fix a problem with conflicting PMI symbols when linking statically.
+  Thanks to Kilian Cavalotti for reporting.
+
+.. note:: Known issues (to be addressed in v2.1.2):
+
+   - See the list of fixes slated for v2.1.2 here:
+     https://github.com/open-mpi/ompi/milestone/28?closed=1
+
+
+Open MPI version 2.1.0
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: March, 2017
+
+.. important:: Major new features:
+
+   - The main focus of the Open MPI v2.1.0 release was to update to PMIx
+     v1.2.1.  When using PMIx (e.g., via mpirun-based launches, or via
+     direct launches with recent versions of popular resource managers),
+     launch time scalability is improved, and the run time memory
+     footprint is greatly decreased when launching large numbers of MPI /
+     OpenSHMEM processes.
+   - Update OpenSHMEM API conformance to v1.3.
+   - The usnic BTL now supports MPI_THREAD_MULTIPLE.
+   - General/overall performance improvements to MPI_THREAD_MULTIPLE.
+   - Add a summary message at the bottom of configure that tells you many
+     of the configuration options specified and/or discovered by Open MPI.
+
+.. attention:: Removed legacy support:
+
+   - The ptmalloc2 hooks have been removed from the Open MPI code base.
+     This is not really a user-noticable change; it is only mentioned
+     here because there was much rejoycing in the Open MPI developer
+     community.
+
+- New MCA parameters:
+
+  - **iof_base_redirect_app_stderr_to_stdout**: as its name implies, it
+    combines MPI / OpenSHMEM applications' stderr into its stdout
+    stream.
+  - **opal_event_include**: allow the user to specify which FD selection
+    mechanism is used by the underlying event engine.
+  - opal_stacktrace_output: indicate where stacktraces should be sent
+    upon MPI / OpenSHMEM process crashes (``none``, ``stdout``,
+    ``stderr``, ``file:filename``).
+  - **orte_timeout_for_stack_trace**: number of seconds to wait for stack
+    traces to be reported (or ``<=0`` to wait forever).
+  - **mtl_ofi_control_prog_type**/**mtl_ofi_data_prog_type**: specify
+    libfabric progress model to be used for control and data.
+
+- Fix MPI_WTICK regression where the time reported may be inaccurate
+  on systems with processor frequency scalaing enabled.
+- Fix regression that lowered the memory maximum message bandwidth for
+  large messages on some BTL network transports, such as openib, sm,
+  and vader.
+- Fix a name collision in the shared file pointer MPI IO file locking
+  scheme.  Thanks to Nicolas Joly for reporting the issue.
+- Fix datatype extent/offset errors in MPI_PUT and MPI_RACCUMULATE
+  when using the Portals 4 one-sided component.
+- Add support for non-contiguous datatypes to the Portals 4 one-sided
+  component.
+- Various updates for the UCX PML.
+- Updates to the following man pages:
+
+  - mpirun(1)
+  - MPI_COMM_CONNECT(3)
+  - MPI_WIN_GET_NAME(3). Thanks to Nicolas Joly for reporting the
+    typo.
+  - MPI_INFO_GET_[NKEYS|NTHKEY](3). Thanks to Nicolas Joly for
+    reporting the typo.
+
+- Fixed a problem in the TCP BTL when using MPI_THREAD_MULTIPLE.
+  Thanks to Evgueni Petrov for reporting.
+- Fixed external32 representation in the romio314 module.  Note that
+  for now, external32 representation is not correctly supported by the
+  ompio module.  Thanks to Thomas Gastine for bringing this to our
+  attention.
+- Add note how to disable a warning message about when a high-speed
+  MPI transport is not found.  Thanks to Susan Schwarz for reporting
+  the issue.
+- Ensure that sending SIGINT when using the rsh/ssh launcher does not
+  orphan children nodes in the launch tree.
+- Fix the help message when showing deprecated MCA param names to show
+  the correct (i.e., deprecated) name.
+- Enable support for the openib BTL to use multiple different
+  InfiniBand subnets.
+- Fix a minor error in MPI_AINT_DIFF.
+- Fix bugs with MPI_IN_PLACE handling in:
+
+  - MPI_ALLGATHER[V]
+  - MPI_[I][GATHER|SCATTER][V]
+  - MPI_IREDUCE[_SCATTER]
+  - Thanks to all the users who helped diagnose these issues.
+
+- Allow qrsh to tree spawn (if the back-end system supports it).
+- Fix MPI_T_PVAR_GET_INDEX to return the correct index.
+- Correctly position the shared file pointer in append mode in the
+  OMPIO component.
+- Add some deprecated names into shmem.h for backwards compatibility
+  with legacy codes.
+- Fix MPI_MODE_NOCHECK support.
+- Fix a regression in PowerPC atomics support.  Thanks to Orion
+  Poplawski for reporting the issue.
+- Fixes for assembly code with aggressively-optimized compilers on
+  x86_64/AMD64 platforms.
+- Fix one more place where configure was mangling custom CFLAGS.
+  Thanks to Phil Tooley (@Telemin) for reporting the issue.
+- Better handle builds with external installations of hwloc.
+- Fixed a hang with MPI_PUT and MPI_WIN_LOCK_ALL.
+- Fixed a bug when using MPI_GET on non-contiguous datatypes and
+  MPI_LOCK/MPI_UNLOCK.
+- Fixed a bug when using POST/START/COMPLETE/WAIT after a fence.
+- Fix configure portability by cleaning up a few uses of "==" with
+  "test".  Thanks to Kevin Buckley for pointing out the issue.
+- Fix bug when using darrays with lib and extent of darray datatypes.
+- Updates to make Open MPI binary builds more bit-for-bit
+  reproducable.  Thanks to Alastair McKinstry for the suggestion.
+- Fix issues regarding persistent request handling.
+- Ensure that shmemx.h is a standalone OpenSHMEM header file.  Thanks
+  to Nick Park (@nspark) for the report.
+- Ensure that we always send SIGTERM prior to SIGKILL.  Thanks to Noel
+  Rycroft for the report.
+- Added ConnectX-5 and Chelsio T6 device defaults for the openib BTL.
+- OpenSHMEM no longer supports MXM less than v2.0.
+- Plug a memory leak in ompi_osc_sm_free.  Thanks to Joseph Schuchart
+  for the report.
+- The "self" BTL now uses less memory.
+- The vader BTL is now more efficient in terms of memory usage when
+  using XPMEM.
+- Removed the ``--enable-openib-failover`` configure option.  This is not
+  considered backwards-incompatible because this option was stale and
+  had long-since stopped working, anyway.
+- Allow jobs launched under Cray aprun to use hyperthreads if
+  opal_hwloc_base_hwthreads_as_cpus MCA parameter is set.
+- Add support for 32-bit and floating point Cray Aries atomic
+  operations.
+- Add support for network AMOs for MPI_ACCUMULATE, MPI_FETCH_AND_OP,
+  and MPI_COMPARE_AND_SWAP if the "ompi_single_intrinsic" info key is
+  set on the window or the "acc_single_intrinsic" MCA param is set.
+- Automatically disqualify RDMA CM support in the openib BTL if
+  MPI_THREAD_MULTIPLE is used.
+- Make configure smarter/better about auto-detecting Linux CMA
+  support.
+- Improve the scalability of MPI_COMM_SPLIT_TYPE.
+- Fix the mixing of C99 and C++ header files with the MPI C++
+  bindings.  Thanks to Alastair McKinstry for the bug report.
+- Add support for ARM v8.
+- Several MCA parameters now directly support MPI_T enumerator
+  semantics (i.e., they accept a limited set of values |mdash| e.g., MCA
+  parameters that accept boolean values).
+- Added ``--with-libmpi-name=<STRING>`` configure option for vendor
+  releases of Open MPI.  See the README for more detail.
+- Fix a problem with Open MPI's internal memory checker.  Thanks to Yvan
+  Fournier for reporting.
+- Fix a multi-threaded issue with MPI_WAIT.  Thanks to Pascal Deveze for
+  reporting.
+
+.. note:: Known issues (to be addressed in v2.1.1):
+
+   - See the list of fixes slated for v2.1.1 here:
+     https://github.com/open-mpi/ompi/milestone/26?closed=1
+
+
+Open MPI v2.0.x series
+----------------------
+
+Open MPI version 2.0.4
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: November, 2017
+
+- Fix an issue with visibility of functions defined in the built-in PMIx.
+  Thanks to Siegmar Gross for reporting this issue.
+- Add configure check to prevent trying to build this release of
+  Open MPI with an external hwloc 2.0 or newer release.
+- Add ability to specify layered providers for OFI MTL.
+- Fix a correctness issue with Open MPI's memory manager code
+  that could result in corrupted message data.  Thanks to
+  Valentin Petrov for reporting.
+- Fix issues encountered when using newer versions of PBS Pro.
+  Thanks to Petr Hanousek for reporting.
+- Fix a problem with MPI_GET when using the vader BTL.  Thanks
+  to Dahai Guo for reporting.
+- Fix a problem when using MPI_ANY_SOURCE with MPI_SENDRECV_REPLACE.
+  Thanks to Dahai Guo for reporting.
+- Fix a problem using MPI_FILE_OPEN with a communicator with an
+  attached cartesian topology.  Thanks to Wei-keng Liao for reporting.
+- Remove IB XRC support from the OpenIB BTL due to lack of support.
+- Remove support for big endian PowerPC.
+- Remove support for XL compilers older than v13.1
+
+
+Open MPI version 2.0.3
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: June 2017
+
+- Fix a problem with MPI_IALLTOALLW when zero size messages are present.
+  Thanks to @mathbird for reporting.
+- Add missing MPI_USER_FUNCTION definition to the mpi_f08 module.
+  Thanks to Nathan Weeks for reporting this issue.
+- Fix a problem with MPI_WIN_LOCK not returning an error code when
+  a negative rank is supplied.  Thanks to Jeff Hammond for reporting and
+  providing a fix.
+- Fix a problem with make check that could lead to hangs.  Thanks to
+  Nicolas Morey-Chaisemartin for reporting.
+- Resolve a symbol conflict problem with PMI-1 and PMI-2 PMIx components.
+  Thanks to Kilian Cavalotti for reporting this issue.
+- Insure that memory allocations returned from MPI_WIN_ALLOCATE_SHARED are
+  64 byte aligned.  Thanks to Joseph Schuchart for reporting this issue.
+- Make use of DOUBLE_COMPLEX, if available, for Fortran bindings.  Thanks
+  to Alexander Klein for reporting this issue.
+- Add missing MPI_T_PVAR_SESSION_NULL definition to Open MPI mpi.h include
+  file.  Thanks to Omri Mor for reporting and fixing.
+- Fix a problem with use of MPI shared file pointers when accessing
+  a file from independent jobs.  Thanks to Nicolas Joly for reporting
+  this issue.
+- Optimize zero size MPI_IALLTOALL{V,W} with MPI_IN_PLACE.  Thanks to
+  Lisandro Dalcín for the report.
+- Fix a ROMIO buffer overflow problem for large transfers when using NFS
+  filesystems.
+- Fix type of MPI_ARGV[S]_NULL which prevented it from being used
+  properly with MPI_COMM_SPAWN[_MULTIPLE] in the mpi_f08 module.
+- Ensure to add proper linker flags to the wrapper compilers for
+  dynamic libraries on platforms that need it (e.g., RHEL 7.3 and
+  later).
+- Get better performance on TCP-based networks 10Gbps and higher by
+  using OS defaults for buffer sizing.
+- Fix a bug with ``MPI_[R][GET_]ACCUMULATE`` when using DARRAY datatypes.
+- Fix handling of ``--with-lustre`` configure command line argument.
+  Thanks to Prentice Bisbal and Tim Mattox for reporting the issue.
+- Added MPI_AINT_ADD and MPI_AINT_DIFF declarations to mpif.h.  Thanks
+  to Aboorva Devarajan (@AboorvaDevarajan) for the bug report.
+- Fix a problem in the TCP BTL when Open MPI is initialized with
+  MPI_THREAD_MULTIPLE support.  Thanks to Evgueni Petro for analyzing and
+  reporting this issue.
+- Fix yalla PML to properly handle underflow errors, and fixed a
+  memory leak with blocking non-contiguous sends.
+- Restored ability to run autogen.pl on official distribution tarballs
+  (although this is still not recommended for most users!).
+- Fix accuracy problems with MPI_WTIME on some systems by always using
+  either clock_gettime(3) or gettimeofday(3).
+- Fix a problem where MPI_WTICK was not returning a higher time resolution
+  when available.  Thanks to Mark Dixon for reporting this issue.
+- Restore SGE functionality.  Thanks to Kevin Buckley for the initial
+  report.
+- Fix external hwloc compilation issues, and extend support to allow
+  using external hwloc installations as far back as v1.5.0.  Thanks to
+  Orion Poplawski for raising the issue.
+- Added latest Mellanox Connect-X and Chelsio T-6 adapter part IDs to
+  the openib list of default values.
+- Do a better job of cleaning up session directories (e.g., in ``/tmp``).
+- Update a help message to indicate how to suppress a warning about
+  no high performance networks being detected by Open MPI.  Thanks to
+  Susan Schwarz for reporting this issue.
+- Fix a problem with mangling of custom CFLAGS when configuring Open MPI.
+  Thanks to Phil Tooley for reporting.
+- Fix some minor memory leaks and remove some unused variables.
+  Thanks to Joshua Gerrard for reporting.
+- Fix MPI_ALLGATHERV bug with MPI_IN_PLACE.
+
+.. note:: Known issues (to be addressed in v2.0.4):
+
+   - See the list of fixes slated for v2.0.4 here:
+     https://github.com/open-mpi/ompi/milestone/29?closed=1
+
+
+Open MPI version 2.0.2
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 26 Jan 2017
+
+- Fix a problem with MPI_FILE_WRITE_SHARED when using MPI_MODE_APPEND and
+  Open MPI's native MPI-IO implementation.  Thanks to Nicolas Joly for
+  reporting.
+- Fix a typo in the MPI_WIN_GET_NAME man page.  Thanks to Nicolas Joly
+  for reporting.
+- Fix a race condition with ORTE's session directory setup.  Thanks to
+  @tbj900 for reporting this issue.
+- Fix a deadlock issue arising from Open MPI's approach to catching calls to
+  munmap. Thanks to Paul Hargrove for reporting and helping to analyze this
+  problem.
+- Fix a problem with PPC atomics which caused make check to fail unless builtin
+  atomics configure option was enabled.  Thanks to Orion Poplawski for reporting.
+- Fix a problem with use of x86_64 cpuid instruction which led to segmentation
+  faults when Open MPI was configured with -O3 optimization.  Thanks to Mark
+  Santcroos for reporting this problem.
+- Fix a problem when using built in atomics configure options on PPC platforms
+  when building 32 bit applications.  Thanks to Paul Hargrove for reporting.
+- Fix a problem with building Open MPI against an external hwloc installation.
+  Thanks to Orion Poplawski for reporting this issue.
+- Remove use of DATE in the message queue version string reported to debuggers to
+  insure bit-wise reproducibility of binaries.  Thanks to Alastair McKinstry
+  for help in fixing this problem.
+- Fix a problem with early exit of a MPI process without calling MPI_FINALIZE
+  or MPI_ABORT that could lead to job hangs.  Thanks to Christof Koehler for
+  reporting.
+- Fix a problem with forwarding of SIGTERM signal from mpirun to MPI processes
+  in a job.  Thanks to Noel Rycroft for reporting this problem
+- Plug some memory leaks in MPI_WIN_FREE discovered using Valgrind.  Thanks
+  to Joseph Schuchart for reporting.
+- Fix a problems  MPI_NEIGHOR_ALLTOALL when using a communicator with an empty topology
+  graph.  Thanks to Daniel Ibanez for reporting.
+- Fix a typo in a PMIx component help file.  Thanks to @njoly for reporting this.
+- Fix a problem with Valgrind false positives when using Open MPI's internal memchecker.
+  Thanks to Yvan Fournier for reporting.
+- Fix a problem with MPI_FILE_DELETE returning MPI_SUCCESS when
+  deleting a non-existent file. Thanks to Wei-keng Liao for reporting.
+- Fix a problem with MPI_IMPROBE that could lead to hangs in subsequent MPI
+  point to point or collective calls.  Thanks to Chris Pattison for reporting.
+- Fix a problem when configure Open MPI for powerpc with ``--enable-mpi-cxx``
+  enabled.  Thanks to Alastair McKinstry for reporting.
+- Fix a problem using MPI_IALLTOALL with MPI_IN_PLACE argument.  Thanks to
+  Chris Ward for reporting.
+- Fix a problem using MPI_RACCUMULATE with the Portals4 transport.  Thanks to
+  @PDeveze for reporting.
+- Fix an issue with static linking and duplicate symbols arising from PMIx
+  Slurm components.  Thanks to Limin Gu for reporting.
+- Fix a problem when using MPI dynamic memory windows.  Thanks to
+  Christoph Niethammer for reporting.
+- Fix a problem with Open MPI's pkgconfig files.  Thanks to Alastair McKinstry
+  for reporting.
+- Fix a problem with MPI_IREDUCE when the same buffer is supplied for the
+  send and recv buffer arguments.  Thanks to Valentin Petrov for reporting.
+- Fix a problem with atomic operations on PowerPC.  Thanks to Paul
+  Hargrove for reporting.
+
+.. note:: Known issues (to be addressed in v2.0.3):
+
+   - See the list of fixes slated for v2.0.3 here:
+     https://github.com/open-mpi/ompi/milestone/23?closed=1
+
+
+Open MPI version 2.0.1
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 2 Sep 2016
+
+- Short message latency and message rate performance improvements for
+  all transports.
+- Fix shared memory performance when using RDMA-capable networks.
+  Thanks to Tetsuya Mishima and Christoph Niethammer for reporting.
+- Fix bandwith performance degredation in the yalla (MXM) PML.  Thanks
+  to Andreas Kempf for reporting the issue.
+- Fix OpenSHMEM crash when running on non-Mellanox MXM-based networks.
+  Thanks to Debendra Das for reporting the issue.
+- Fix a crash occuring after repeated calls to MPI_FILE_SET_VIEW with
+  predefined datatypes.  Thanks to Eric Chamberland and Matthew
+  Knepley for reporting and helping chase down this issue.
+- Fix stdin propagation to MPI processes.  Thanks to Jingchao Zhang
+  for reporting the issue.
+- Fix various runtime and portability issues by updating the PMIx
+  internal component to v1.1.5.
+- Fix process startup failures on Intel MIC platforms due to very
+  large entries in ``/proc/mounts``.
+- Fix a problem with use of relative path for specifing executables to
+  mpirun / oshrun.  Thanks to David Schneider for reporting.
+- Various improvements when running over portals-based networks.
+- Fix thread-based race conditions with GNI-based networks.
+- Fix a problem with MPI_FILE_CLOSE and MPI_FILE_SET_SIZE.  Thanks
+  to Cihan Altinay for reporting.
+- Remove all use of rand(3) from within Open MPI so as not to perturb
+  applications use of it.  Thanks to Matias Cabral and Noel Rycroft
+  for reporting.
+- Fix crash in MPI_COMM_SPAWN.
+- Fix types for MPI_UNWEIGHTED and MPI_WEIGHTS_EMPTY.  Thanks to
+  Lisandro Dalcín for reporting.
+- Correctly report the name of MPI_INTEGER16.
+- Add some missing MPI constants to the Fortran bindings.
+- Fixed compile error when configuring Open MPI with ``--enable-timing``.
+- Correctly set the shared library version of libompitrace.so.  Thanks
+  to Alastair McKinstry for reporting.
+- Fix errors in the MPI_RPUT, MPI_RGET, MPI_RACCUMULATE, and
+  MPI_RGET_ACCUMULATE Fortran bindings.  Thanks to Alfio Lazzaro and
+  Joost VandeVondele for tracking this down.
+- Fix problems with use of derived datatypes in non-blocking
+  collectives.  Thanks to Yuki Matsumoto for reporting.
+- Fix problems with OpenSHMEM header files when using CMake.  Thanks to
+  Paul Kapinos for reporting the issue.
+- Fix problem with use use of non-zero lower bound datatypes in
+  collectives.  Thanks to Hristo Iliev for reporting.
+- Fix a problem with memory allocation within MPI_GROUP_INTERSECTION.
+  Thanks to Lisandro Dalcín for reporting.
+- Fix an issue with MPI_ALLGATHER for communicators that don't consist
+  of two ranks.  Thanks to David Love for reporting.
+- Various fixes for collectives when used with esoteric MPI datatypes.
+- Fixed corner cases of handling DARRAY and HINDEXED_BLOCK datatypes.
+- Fix a problem with filesystem type check for OpenBSD.
+  Thanks to Paul Hargrove for reporting.
+- Fix some debug input within Open MPI internal functions.  Thanks to
+  Durga Choudhury for reporting.
+- Fix a typo in a configury help message.  Thanks to Paul Hargrove for
+  reporting.
+- Correctly support MPI_IN_PLACE in MPI_[I]ALLTOALL[V|W] and
+  MPI_[I]EXSCAN.
+- Fix alignment issues on SPARC platforms.
+
+.. note:: Known issues (to be addressed in v2.0.2):
+
+   - See the list of fixes slated for v2.0.2 here:
+     https://github.com/open-mpi/ompi/milestone/20?closed=1, and
+     https://github.com/open-mpi/ompi-release/milestone/19?closed=1
+     (note that the "ompi-release" Github repo will be folded/absorbed
+     into the "ompi" Github repo at some point in the future)
+
+
+Open MPI version 2.0.0
+^^^^^^^^^^^^^^^^^^^^^^
+:Date: 12 Jul 2016
+
+.. attention:: Open MPI is now fully MPI-3.1 compliant
+
+.. important:: Major new features:
+
+   - Many enhancements to MPI RMA.  Open MPI now maps MPI RMA operations
+     on to native RMA operations for those networks which support this
+     capability.
+   - Greatly improved support for MPI_THREAD_MULTIPLE (when configured
+     with ``--enable-mpi-thread-multiple``).
+   - Enhancements to reduce the memory footprint for jobs at scale.  A
+     new MCA parameter, "mpi_add_procs_cutoff", is available to set the
+     threshold for using this feature.
+   - Completely revamped support for memory registration hooks when using
+     OS-bypass network transports.
+   - Significant OMPIO performance improvements and many bug fixes.
+   - Add support for PMIx - Process Management Interface for Exascale.
+     Version 1.1.2 of PMIx is included internally in this release.
+   - Add support for PLFS file systems in Open MPI I/O.
+   - Add support for UCX transport.
+   - Simplify build process for Cray XC systems.  Add support for
+     using native SLURM.
+   - Add a ``--tune`` mpirun command line option to simplify setting many
+     environment variables and MCA parameters.
+   - Add a new MCA parameter "orte_default_dash_host" to offer an analogue
+     to the existing "orte_default_hostfile" MCA parameter.
+   - Add the ability to specify the number of desired slots in the mpirun
+     ``--host`` option.
+
+.. note:: Known issues (to be addressed in v2.0.1):
+
+   - See the list of fixes slated for v2.0.1 here:
+     https://github.com/open-mpi/ompi/milestone/16?closed=1, and
+     https://github.com/open-mpi/ompi-release/milestone/16?closed=1
+     (note that the "ompi-release" Github repo will be folded/absorbed
+     into the "ompi" Github repo at some point in the future)
+
+   - ompi-release#986: Fix data size counter for large ops with fcoll/static
+   - ompi-release#987: Fix OMPIO performance on Lustre
+   - ompi-release#1013: Fix potential inconsistency in btl/openib default settings
+   - ompi-release#1014: Do not return MPI_ERR_PENDING from collectives
+   - ompi-release#1056: Remove dead profile code from oshmem
+   - ompi-release#1081: Fix MPI_IN_PLACE checking for IALLTOALL{V|W}
+   - ompi-release#1081: Fix memchecker in MPI_IALLTOALLW
+   - ompi-release#1081: Support MPI_IN_PLACE in MPI_(I)ALLTOALLW and MPI_(I)EXSCAN
+   - ompi-release#1107: Allow future PMIx support for RM spawn limits
+   - ompi-release#1108: Fix sparse group process reference counting
+   - ompi-release#1109: If specified to be oversubcribed, disable binding
+   - ompi-release#1122: Allow NULL arrays for empty datatypes
+   - ompi-release#1123: Fix signed vs. unsigned compiler warnings
+   - ompi-release#1123: Make max hostname length uniform across code base
+   - ompi-release#1127: Fix MPI_Compare_and_swap
+   - ompi-release#1127: Fix MPI_Win_lock when used with MPI_Win_fence
+   - ompi-release#1132: Fix typo in help message for ``--enable-mca-no-build``
+   - ompi-release#1154: Ensure pairwise coll algorithms disqualify themselves properly
+   - ompi-release#1165: Fix typos in debugging/verbose message output
+   - ompi-release#1178: Fix ROMIO filesystem check on OpenBSD 5.7
+   - ompi-release#1197: Fix Fortran pthread configure check
+   - ompi-release#1205: Allow using external PMIx 1.1.4 and 2.0
+   - ompi-release#1215: Fix configure to support the NAG Fortran compiler
+   - ompi-release#1220: Fix combiner args for MPI_HINDEXED_BLOCK
+   - ompi-release#1225: Fix combiner args for MPI_DARRAY
+   - ompi-release#1226: Disable old memory hooks with recent gcc versions
+   - ompi-release#1231: Fix new "patcher" support for some XLC platforms
+   - ompi-release#1244: Fix Java error handling
+   - ompi-release#1250: Ensure TCP is not selected for RDMA operations
+   - ompi-release#1252: Fix verbose output in coll selection
+   - ompi-release#1253: Set a default name for user-defined MPI_Op
+   - ompi-release#1254: Add count==0 checks in some non-blocking colls
+   - ompi-release#1258: Fix ``make distclean`` when using external pmix/hwloc/libevent
+   - ompi-release#1260: Clean up/uniform mca/coll/base memory management
+   - ompi-release#1261: Remove "patcher" warning message for static builds
+   - ompi-release#1263: Fix IO MPI_Request for 0-size read/write
+   - ompi-release#1264: Add blocking fence for SLURM operations
+
+- In environments where mpirun cannot automatically determine the
+  number of slots available (e.g., when using a hostfile that does not
+  specify "slots", or when using ``--host`` without specifying a ":N"
+  suffix to hostnames), mpirun now requires the use of "-np N" to
+  specify how many MPI processes to launch.
+- The MPI C++ bindings (which were removed from the MPI standard in
+  v3.0) are no longer built by default and will be removed in some
+  future version of Open MPI.  Use the ``--enable-mpi-cxx-bindings``
+  configure option to build the deprecated/removed MPI C++ bindings.
+- ompi_info now shows all components, even if they do not have MCA
+  parameters.  The prettyprint output now separates groups with a
+  dashed line.
+- OMPIO is now the default implementation of parallel I/O, with the
+  exception for Lustre parallel filesystems (where ROMIO is still the
+  default).  The default selection of OMPI vs. ROMIO can be controlled
+  via the "--mca io ompi|romio" command line switch to mpirun.
+- Per Open MPI's versioning scheme (see the README), increasing the
+  major version number to 2 indicates that this version is not
+  ABI-compatible with prior versions of Open MPI.  You will need to
+  recompile MPI and OpenSHMEM applications to work with this version
+  of Open MPI.
+- Removed checkpoint/restart code due to loss of maintainer. :-(
+- Change the behavior for handling certain signals when using PSM and
+  PSM2 libraries.  Previously, the PSM and PSM2 libraries would trap
+  certain signals in order to generate tracebacks.  The mechanism was
+  found to cause issues with Open MPI's own error reporting mechanism.
+  If not already set, Open MPI now sets the IPATH_NO_BACKTRACE and
+  HFI_NO_BACKTRACE environment variables to disable PSM/PSM2's
+  handling these signals.
+
+.. attention:: Removed legacy support:
+
+   - Removed support for OS X Leopard.
+   - Removed support for Cray XT systems.
+   - Removed VampirTrace.
+   - Removed support for Myrinet/MX.
+   - Removed legacy collective module:ML.
+   - Removed support for Alpha processors.
+   - Removed ``--enable-mpi-profiling`` configure option.
+
+- Updated internal/embedded copies of third-party software:
+
+  - Update the internal copy of ROMIO to that which shipped in MPICH
+    3.1.4.
+  - Update internal copy of libevent to v2.0.22.
+  - Update internal copy of hwloc to v1.11.2.
+
+- Notable new MCA parameters:
+
+  - opal_progress_lp_call_ration: Control how often low-priority
+    callbacks are made during Open MPI's main progress loop.
+  - opal_common_verbs_want_fork_support: This replaces the
+    btl_openib_want_fork_support parameter.
+
+- Add ``--with-platform-patches-dir`` configure option.
+- Add ``--with-pmi-libdir`` configure option for environments that install
+  PMI libs in a non-default location.
+- Various configure-related compatibility updates for newer versions
+  of libibverbs and OFED.
+- Numerous fixes/improvements to orte-dvm.  Special thanks to Mark
+  Santcroos for his help.
+- Fix a problem with timer code on ia32 platforms.  Thanks to
+  Paul Hargrove for reporting this and providing a patch.
+- Fix a problem with use of a 64 bit atomic counter.  Thanks to
+  Paul Hargrove for reporting.
+- Fix a problem with singleton job launching.  Thanks to Lisandro
+  Dalcín for reporting.
+- Fix a problem with use of MPI_UNDEFINED with MPI_COMM_SPLIT_TYPE.
+  Thanks to Lisandro Dalcín for reporting.
+- Silence a compiler warning in PSM MTL.  Thanks to Adrian Reber for
+  reporting this.
+- Properly detect Intel TrueScale and OmniPath devices in the ACTIVE
+  state.  Thanks to Durga Choudhury for reporting the issue.
+- Fix detection and use of Solaris Studio 12.5 (beta) compilers.
+  Thanks to Paul Hargrove for reporting and debugging.
+- Fix various small memory leaks.
+- Allow NULL arrays when creating empty MPI datatypes.
+- Replace use of alloca with malloc for certain datatype creation
+  functions.  Thanks to Bogdan Sataric for reporting this.
+- Fix use of MPI_LB and MPI_UB in creation of of certain MPI datatypes.
+  Thanks to Gus Correa for helping to fix this.
+- Implement a workaround for a GNU Libtool problem.  Thanks to Eric
+  Schnetter for reporting and fixing.
+- Improve hcoll library detection in configure.  Thanks to David
+  Shrader and Åke Sandgren for reporting this.
+- Miscellaneous minor bug fixes in the hcoll component.
+- Miscellaneous minor bug fixes in the ugni component.
+- Fix problems with XRC detection in OFED 3.12 and older releases.
+  Thanks to Paul Hargrove for his analysis of this problem.
+- Update (non-standard/experimental) Java MPI interfaces to support
+  MPI-3.1 functionality.
+- Fix an issue with MCA parameters for Java bindings.  Thanks to
+  Takahiro Kawashima and Siegmar Gross for reporting this issue.
+- Fix a problem when using persistent requests in the Java bindings.
+  Thanks to Nate Chambers for reporting.
+- Fix problem with Java bindings on OX X 10.11.  Thanks to Alexander
+  Daryin for reporting this issue.
+- Fix a performance problem for large messages for Cray XC systems.
+  Thanks to Jerome Vienne for reporting this.
+- Fix an issue with MPI_WIN_LOCK_ALL.  Thanks to Thomas Jahns for
+  reporting.
+- Fix an issue with passing a parameter to configure multiple times.
+  Thanks to QuesarVII for reporting and supplying a fix.
+- Add support for ALPS resource allocation system on Cray CLE 5.2 and
+  later.  Thanks to Mark Santcroos.
+- Corrections to the HACKING file.  Thanks to Maximilien Levesque.
+- Fix an issue with user supplied reduction operator functions.
+  Thanks to Rupert Nash for reporting this.
+- Fix an issue with an internal list management function.  Thanks to
+  Adrian Reber for reporting this.
+- Fix a problem with MPI-RMA PSCW epochs.  Thanks to Berk Hess for
+  reporting this.
+- Fix a problem in neighborhood collectives.  Thanks to Lisandro
+  Dalcín for reporting.
+- Fix MPI_IREDUCE_SCATTER_BLOCK for a one-process communicator. Thanks
+  to Lisandro Dalcín for reporting.
+- Add (Open MPI-specific) additional flavors to MPI_COMM_SPLIT_TYPE.
+  See MPI_Comm_split_type(3) for details.  Thanks to Nick Andersen for
+  supplying this enhancement.
+- Improve closing of file descriptors during the job launch phase.
+  Thanks to Piotr Lesnicki for reporting and providing this
+  enhancement.
+- Fix a problem in MPI_GET_ACCUMULATE and MPI_RGET_ACCUMULATE when
+  using Portals4.  Thanks to Nicolas Chevalier for reporting.
+- Use correct include file for lstat prototype in ROMIO. Thanks to
+  William Throwe for finding and providing a fix.
+- Add missing Fortran bindings for MPI_WIN_ALLOCATE.  Thanks to Christoph
+  Niethammer for reporting and fixing.
+- Fortran related fixes to handle Intel 2016 compiler.  Thanks to
+  Fabrice Roy for reporting this.
+- Fix a Fortran linkage issue.  Thanks to Macro Atzeri for finding and
+  suggesting a fix.
+- Fix problem with using BIND(C) for Fortran bindings with logical
+  parameters.  Thanks to Paul Romano for reporting.
+- Fix an issue with use of DL-related macros in opal library.  Thanks to
+  Scott Atchley for finding this.
+- Fix an issue with parsing mpirun command line options which contain
+  colons.  Thanks to Lev Given for reporting.
+- Fix a problem with Open MPI's package configury files.  Thanks to
+  Christoph Junghans for reporting.
+- Fix a typo in the MPI_INTERCOMM_MERGE man page.  Thanks To Harald
+  Servat for reporting and correcting.
+- Update man pages for non-blocking sends per MPI 3.1 standard.
+  Thanks to Alexander Pozdneev for reporting.
+- Fix problem when compiling against PVFS2.  Thanks to Dave Love for
+  reporting.
+- Fix problems with MPI_NEIGHBOR_ALLTOALL{V,W}.  Thanks to Willem
+  Vermin for reporting this issue.
+- Fix various compilation problems on Cygwin.  Thanks to Marco Atzeri
+  for supplying these fixes.
+- Fix problem with resizing of subarray and darray data types.  Thanks
+  to Keith Bennett and Dan Garmann for reporting.
+- Fix a problem with MPI_COMBINER_RESIZED.  Thanks to James Ramsey for
+  the report.
+- Fix an hwloc binding issue.  Thanks to Ben Menadue for reporting.
+- Fix a problem with the shared memory (sm) BTL.  Thanks to Peter Wind
+  for the report.
+- Fixes for heterogeneous support. Thanks to Siegmar Gross for reporting.
+- Fix a problem with memchecker.  Thanks to Clinton Simpson for reporting.
+- Fix a problem with MPI_UNWEIGHTED in topology functions.  Thanks to
+  Jun Kudo for reporting.
+- Fix problem with a MCA parameter base filesystem types.  Thanks to
+  Siegmar Gross for reporting.
+- Fix a problem with some windows info argument types.  Thanks to
+  Alastair McKinstry for reporting.
diff --git a/docs/release-notes/changelog/v3.0.x.rst b/docs/release-notes/changelog/v3.0.x.rst
new file mode 100644
index 00000000000..27578071121
--- /dev/null
+++ b/docs/release-notes/changelog/v3.0.x.rst
@@ -0,0 +1,252 @@
+Open MPI v3.0.x series
+======================
+
+This file contains all the NEWS updates for the Open MPI v3.0.x
+series, in reverse chronological order.
+
+Open MPI version 3.0.6
+----------------------
+:Date: March, 2020
+
+- Fix one-sided shared memory window configuration bug.
+- Fix support for PGI'18 compiler.
+- Fix run-time linker issues with OMPIO on newer Linux distros.
+- Allow the user to override modulefile_path in the Open MPI SRPM,
+  even if ``install_in_opt`` is set to 1.
+- Properly detect ConnectX-6 HCAs in the openib BTL.
+- Fix segfault in the MTL/OFI initialization for large jobs.
+- Fix various portals4 control flow bugs.
+- Fix communications ordering for alltoall and Cartesian neighborhood
+  collectives.
+- Fix an infinite recursion crash in the memory patcher on systems
+  with glibc v2.26 or later (e.g., Ubuntu 18.04) when using certain
+  OS-bypass interconnects.
+
+
+Open MPI version 3.0.5
+----------------------
+:Date: November, 2019
+
+- Fix OMPIO issue limiting file reads/writes to 2GB.  Thanks to
+  Richard Warren for reporting the issue.
+- At run time, automatically disable Linux cross-memory attach (CMA)
+  for vader BTL (shared memory) copies when running in user namespaces
+  (i.e., containers).  Many thanks to Adrian Reber for raising the
+  issue and providing the fix.
+- Sending very large MPI messages using the ofi MTL will fail with
+  some of the underlying Libfabric transports (e.g., PSM2 with
+  messages >=4GB, verbs with messages >=2GB).  Prior version of Open
+  MPI failed silently; this version of Open MPI invokes the
+  appropriate MPI error handler upon failure.  See
+  https://github.com/open-mpi/ompi/issues/7058 for more details.
+  Thanks to Emmanuel Thomé for raising the issue.
+- Fix case where 0-extent datatypes might be eliminated during
+  optimization.  Thanks to Github user @tjahns for raising the issue.
+- Ensure that the MPIR_Breakpoint symbol is not optimized out on
+  problematic platforms.
+- Fix OMPIO offset calculations with ``SEEK_END`` and ``SEEK_CUR`` in
+  ``MPI_FILE_GET_POSITION``.  Thanks to Wei-keng Liao for raising the
+  issue.
+- Fix corner case for datatype extent computations.  Thanks to David
+  Dickenson for raising the issue.
+- Fix MPI buffered sends with the "cm" PML.
+- Update to PMIx v2.2.3.
+- Fix ssh-based tree-based spawning at scale.  Many thanks to Github
+  user @zrss for the report and diagnosis.
+- Fix the Open MPI RPM spec file to not abort when grep fails.  Thanks
+  to Daniel Letai for bringing this to our attention.
+- Handle new SLURM CLI options (SLURM 19 deprecated some options that
+  Open MPI was using).  Thanks to Jordan Hayes for the report and the
+  initial fix.
+- OMPI: fix division by zero with an empty file view.
+- Also handle ``shmat()``/``shmdt()`` memory patching with OS-bypass networks.
+- Add support for unwinding info to all files that are present in the
+  stack starting from ``MPI_Init``, which is helpful with parallel
+  debuggers.  Thanks to James Clark for the report and initial fix.
+- Fixed inadvertant use of bitwise operators in the MPI C++ bindings
+  header files.  Thanks to Bert Wesarg for the report and the fix.
+- Added configure option ``--disable-wrappers-runpath`` (alongside the
+  already-existing ``--disable-wrappers-rpath`` option) to prevent Open
+  MPI's configure script from automatically adding runpath CLI options
+  to the wrapper compilers.
+
+
+Open MPI version 3.0.4
+----------------------
+:Date: April, 2019
+
+- Fix compile error when configured with ``--enable-mpi-java`` and
+  ``--with-devel-headers``.  Thanks to @g-raffy for reporting the issue.
+- Fix possible floating point rounding and division issues in OMPIO
+  which led to crashes and/or data corruption with very large data.
+  Thanks to Axel Huebl and René Widera for identifing the issue,
+  supplying and testing the fix (** also appeared: v3.0.4).
+- Use ``static_cast<>`` in ``mpi.h`` where appropriate.  Thanks to @shadow-fx
+  for identifying the issue.
+- Fix datatype issue with RMA accumulate.  Thanks to Jeff Hammond for
+  raising the issue.
+- Fix RMA accumulate of non-predefined datatypes with predefined
+  operators.  Thanks to Jeff Hammond for raising the issue.
+- Fix race condition when closing open file descriptors when launching
+  MPI processes.  Thanks to Jason Williams for identifying the issue and
+  supplying the fix.
+- Fix Valgrind warnings for some ``MPI_TYPE_CREATE_*`` functions.  Thanks
+  to Risto Toijala for identifying the issue and supplying the fix.
+- Fix ``MPI_TYPE_CREATE_F90_{REAL,COMPLEX}`` for r=38 and r=308.
+- Fix assembly issues with old versions of gcc (<6.0.0) that affected
+  the stability of shared memory communications (e.g., with the vader
+  BTL).
+- Fix the OFI MTL handling of ``MPI_ANY_SOURCE``.
+- Fix noisy errors in the openib BTL with regards to
+  ``ibv_exp_query_device()``.  Thanks to Angel Beltre and others who
+  reported the issue.
+
+
+Open MPI version 3.0.3
+----------------------
+:Date: October, 2018
+
+- Fix race condition in ``MPI_THREAD_MULTIPLE`` support of non-blocking
+  send/receive path.
+- Fix error handling ``SIGCHLD`` forwarding.
+- Add support for ``CHARACTER`` and ``LOGICAL`` Fortran datatypes for ``MPI_SIZEOF``.
+- Fix compile error when using OpenJDK 11 to compile the Java bindings.
+- Fix crash when using a hostfile with a 'user@host' line.
+- Numerous Fortran '08 interface fixes.
+- TCP BTL error message fixes.
+- OFI MTL now will use any provider other than shm, sockets, tcp, udp, or
+  rstream, rather than only supporting gni, psm, and psm2.
+- Disable async receive of CUDA buffers by default, fixing a hang
+  on large transfers.
+- Support the BCM57XXX and BCM58XXX Broadcomm adapters.
+- Fix minmax datatype support in ROMIO.
+- Bug fixes in vader shared memory transport.
+- Support very large buffers with ``MPI_TYPE_VECTOR``.
+- Fix hang when launching with mpirun on Cray systems.
+- Bug fixes in OFI MTL.
+- Assorted Portals 4.0 bug fixes.
+- Fix for possible data corruption in ``MPI_BSEND``.
+- Move shared memory file for vader btl into ``/dev/shm`` on Linux.
+- Fix for ``MPI_ISCATTER``/``MPI_ISCATTERV`` Fortran interfaces with ``MPI_IN_PLACE``.
+- Upgrade PMIx to v2.1.4.
+- Fix for Power9 built-in atomics.
+- Numerous One-sided bug fixes.
+- Fix for race condition in uGNI BTL.
+- Improve handling of large number of interfaces with TCP BTL.
+- Numerous UCX bug fixes.
+- Add support for QLogic and Broadcom Cumulus RoCE HCAs to Open IB BTL.
+- Add patcher support for aarch64.
+- Fix hang on Power and ARM when Open MPI was built with low compiler
+  optimization settings.
+
+
+Open MPI version 3.0.2
+----------------------
+:Date: June, 2018
+
+- Disable osc/pt2pt when using ``MPI_THREAD_MULTIPLE`` due to numerous
+  race conditions in the component.
+- Fix dummy variable names for the mpi and mpi_f08 Fortran bindings to
+  match the MPI standard.  This may break applications which use
+  name-based parameters in Fortran which used our internal names
+  rather than those documented in the MPI standard.
+- Fixed ``MPI_SIZEOF`` in the "mpi" Fortran module for the NAG compiler.
+- Fix RMA function signatures for ``use-mpi-f08`` bindings to have the
+  asynchonous property on all buffers.
+- Fix Fortran ``MPI_COMM_SPAWN_MULTIPLE`` to properly follow the count
+  length argument when parsing the array_of_commands variable.
+- Revamp Java detection to properly handle new Java versions which do
+  not provide a javah wrapper.
+- Improved configure logic for finding the UCX library.
+- Add support for HDR InfiniBand link speeds.
+- Disable the POWER 7/BE block in configure.  Note that POWER 7/BE is
+  still not a supported platform, but it is no longer automatically
+  disabled.  See
+  https://github.com/open-mpi/ompi/issues/4349#issuecomment-374970982
+  for more information.
+
+
+Open MPI version 3.0.1
+----------------------
+:Date: March, 2018
+
+- Fix ability to attach parallel debuggers to MPI processes.
+- Fix a number of issues in MPI I/O found by the HDF5 test suite.
+- Fix (extremely) large message transfers with shared memory.
+- Fix out of sequence bug in multi-NIC configurations.
+- Fix stdin redirection bug that could result in lost input.
+- Disable the LSF launcher if CSM is detected.
+- Plug a memory leak in ``MPI_Mem_free()``.  Thanks to Philip Blakely for reporting.
+- Fix the tree spawn operation when the number of nodes is larger than the radix.
+  Thanks to Carlos Eduardo de Andrade for reporting.
+- Fix Fortran 2008 macro in MPI extensions.  Thanks to Nathan T. Weeks for
+  reporting.
+- Add UCX to list of interfaces that OpenSHMEM will use by default.
+- Add ``--{enable|disable}-show-load-errors-by-default`` to control
+  default behavior of the load errors option.
+- OFI MTL improvements: handle empty completion queues properly, fix
+  incorrect error message around ``fi_getinfo()``, use default progress
+  option for provider by default, Add support for reading multiple
+  CQ events in ofi_progress.
+- PSM2 MTL improvements: Allow use of GPU buffers, thread fixes.
+- Numerous corrections to memchecker behavior.
+- Add a mca parameter ``ras_base_launch_orted_on_hn`` to allow for launching
+  MPI processes on the same node where mpirun is executing using a separate
+  orte daemon, rather than the mpirun process.   This may be useful to set to
+  true when using SLURM, as it improves interoperability with SLURM's signal
+  propagation tools.  By default it is set to false, except for Cray XC systems.
+- Fix a problem reported on the mailing separately by Kevin McGrattan and Stephen
+  Guzik about consistency issues on NFS file systems when using OMPIO. This fix
+  also introduces a new mca parameter ``fs_ufs_lock_algorithm`` which allows to
+  control the locking algorithm used by ompio for read/write operations. By
+  default, ompio does not perfom locking on local UNIX file systems, locks the
+  entire file per operation on NFS file systems, and selective byte-range
+  locking on other distributed file systems.
+- Add an mca parameter ``pmix_server_usock_connections`` to allow mpirun to
+  support applications statically built against the Open MPI v2.x release,
+  or installed in a container along with the Open MPI v2.x libraries. It is
+  set to false by default.
+
+
+Open MPI version 3.0.0
+----------------------
+:Date: September, 2017
+
+.. important:: Major new features:
+
+   - Use UCX allocator for OSHMEM symmetric heap allocations to optimize intra-node
+     data transfers.  UCX SPML only.
+   - Use UCX multi-threaded API in the UCX PML.  Requires UCX 1.0 or later.
+   - Added support for Flux PMI
+   - Update embedded PMIx to version 2.1.0
+   - Update embedded hwloc to version 1.11.7
+
+- Per Open MPI's versioning scheme (see the README), increasing the
+  major version number to 3 indicates that this version is not
+  ABI-compatible with prior versions of Open MPI. In addition, there may
+  be differences in MCA parameter names and defaults from previous releases.
+  Command line options for mpirun and other commands may also differ from
+  previous versions. You will need to recompile MPI and OpenSHMEM applications
+  to work with this version of Open MPI.
+- With this release, Open MPI supports ``MPI_THREAD_MULTIPLE`` by default.
+- New configure options have been added to specify the locations of libnl
+  and zlib.
+- A new configure option has been added to request Flux PMI support.
+- The help menu for mpirun and related commands is now context based.
+  ``mpirun --help compatibility`` generates the help menu in the same format
+  as previous releases.
+
+.. attention:: Removed legacy support:
+
+   - AIX is no longer supported.
+   - Loadlever is no longer supported.
+   - OpenSHMEM currently supports the UCX and MXM transports via the ucx and ikrit
+     SPMLs respectively.
+   - Remove IB XRC support from the OpenIB BTL due to lack of support.
+   - Remove support for big endian PowerPC.
+   - Remove support for XL compilers older than v13.1
+
+.. note:: Known issues:
+
+   - MPI_Connect/accept between applications started by different mpirun
+     commands will fail, even if ompi-server is running.
diff --git a/docs/release-notes/changelog/v3.1.x.rst b/docs/release-notes/changelog/v3.1.x.rst
new file mode 100644
index 00000000000..de265cf19ac
--- /dev/null
+++ b/docs/release-notes/changelog/v3.1.x.rst
@@ -0,0 +1,258 @@
+Open MPI v3.1.x series
+======================
+
+This file contains all the NEWS updates for the Open MPI v3.1.x
+series, in reverse chronological order.
+
+Open MPI version 3.1.6
+----------------------
+:Date: March 2020
+
+- Fix one-sided shared memory window configuration bug.
+- Fix support for PGI'18 compiler.
+- Fix issue with zero-length blockLength in ``MPI_TYPE_INDEXED``.
+- Fix run-time linker issues with OMPIO on newer Linux distros.
+- Fix PMIX dstore locking compilation issue.  Thanks to Marco Atzeri
+  for reporting the issue.
+- Allow the user to override modulefile_path in the Open MPI SRPM,
+  even if ``install_in_opt`` is set to 1.
+- Properly detect ConnectX-6 HCAs in the openib BTL.
+- Fix segfault in the MTL/OFI initialization for large jobs.
+- Fix issue to guarantee to properly release MPI one-sided lock when
+  using UCX transports to avoid a deadlock.
+- Fix potential deadlock when processing outstanding transfers with
+  uGNI transports.
+- Fix various portals4 control flow bugs.
+- Fix communications ordering for alltoall and Cartesian neighborhood
+  collectives.
+- Fix an infinite recursion crash in the memory patcher on systems
+  with glibc v2.26 or later (e.g., Ubuntu 18.04) when using certain
+  OS-bypass interconnects.
+
+
+Open MPI version 3.1.5
+----------------------
+:Date: November, 2019
+
+- Fix OMPIO issue limiting file reads/writes to 2GB.  Thanks to
+  Richard Warren for reporting the issue.
+- At run time, automatically disable Linux cross-memory attach (CMA)
+  for vader BTL (shared memory) copies when running in user namespaces
+  (i.e., containers).  Many thanks to Adrian Reber for raising the
+  issue and providing the fix.
+- Sending very large MPI messages using the ofi MTL will fail with
+  some of the underlying Libfabric transports (e.g., PSM2 with
+  messages >=4GB, verbs with messages >=2GB).  Prior version of Open
+  MPI failed silently; this version of Open MPI invokes the
+  appropriate MPI error handler upon failure.  See
+  https://github.com/open-mpi/ompi/issues/7058 for more details.
+  Thanks to Emmanuel Thomé for raising the issue.
+- Fix case where 0-extent datatypes might be eliminated during
+  optimization.  Thanks to Github user @tjahns for raising the issue.
+- Ensure that the ``MPIR_Breakpoint`` symbol is not optimized out on
+  problematic platforms.
+- Fix MPI one-sided 32 bit atomic support.
+- Fix OMPIO offset calculations with ``SEEK_END`` and ``SEEK_CUR`` in
+  ``MPI_FILE_GET_POSITION``.  Thanks to Wei-keng Liao for raising the
+  issue.
+- Add "naive" regx component that will never fail, no matter how
+  esoteric the hostnames are.
+- Fix corner case for datatype extent computations.  Thanks to David
+  Dickenson for raising the issue.
+- Allow individual jobs to set their map/rank/bind policies when
+  running LSF.  Thanks to Nick R. Papior for assistance in solving the
+  issue.
+- Fix MPI buffered sends with the "cm" PML.
+- Properly propagate errors to avoid deadlocks in MPI one-sided operations.
+- Update to PMIx v2.2.3.
+- Fix data corruption in non-contiguous MPI accumulates over UCX.
+- Fix ssh-based tree-based spawning at scale.  Many thanks to Github
+  user @zrss for the report and diagnosis.
+- Fix the Open MPI RPM spec file to not abort when grep fails.  Thanks
+  to Daniel Letai for bringing this to our attention.
+- Handle new SLURM CLI options (SLURM 19 deprecated some options that
+  Open MPI was using).  Thanks to Jordan Hayes for the report and the
+  initial fix.
+- OMPI: fix division by zero with an empty file view.
+- Also handle ``shmat()``/``shmdt()`` memory patching with OS-bypass networks.
+- Add support for unwinding info to all files that are present in the
+  stack starting from MPI_Init, which is helpful with parallel
+  debuggers.  Thanks to James Clark for the report and initial fix.
+- Fixed inadvertant use of bitwise operators in the MPI C++ bindings
+  header files.  Thanks to Bert Wesarg for the report and the fix.
+
+
+Open MPI version 3.1.4
+----------------------
+:Date: April, 2019
+
+- Fix compile error when configured with ``--enable-mpi-java`` and
+  ``--with-devel-headers``.  Thanks to @g-raffy for reporting the issue
+  (** also appeared: v3.0.4).
+- Only use hugepages with appropriate permissions.  Thanks to Hunter
+  Easterday for the fix.
+- Fix possible floating point rounding and division issues in OMPIO
+  which led to crashes and/or data corruption with very large data.
+  Thanks to Axel Huebl and René Widera for identifing the issue,
+  supplying and testing the fix (** also appeared: v3.0.4).
+- Use ``static_cast<>`` in ``mpi.h`` where appropriate.  Thanks to @shadow-fx
+  for identifying the issue (** also appeared: v3.0.4).
+- Fix RMA accumulate of non-predefined datatypes with predefined
+  operators.  Thanks to Jeff Hammond for raising the issue (** also
+  appeared: v3.0.4).
+- Fix race condition when closing open file descriptors when launching
+  MPI processes.  Thanks to Jason Williams for identifying the issue and
+  supplying the fix (** also appeared: v3.0.4).
+- Fix support for external PMIx v3.1.x.
+- Fix Valgrind warnings for some ``MPI_TYPE_CREATE_*`` functions.  Thanks
+  to Risto Toijala for identifying the issue and supplying the fix (**
+  also appeared: v3.0.4).
+- Fix ``MPI_TYPE_CREATE_F90_{REAL,COMPLEX}`` for r=38 and r=308 (** also
+  appeared: v3.0.4).
+- Fix assembly issues with old versions of gcc (<6.0.0) that affected
+  the stability of shared memory communications (e.g., with the vader
+  BTL) (** also appeared: v3.0.4).
+- Fix ``MPI_Allreduce`` crashes with some cases in the coll/spacc module.
+- Fix the OFI MTL handling of ``MPI_ANY_SOURCE`` (** also appeared:
+  v3.0.4).
+- Fix noisy errors in the openib BTL with regards to
+  ``ibv_exp_query_device()``.  Thanks to Angel Beltre and others who
+  reported the issue (** also appeared: v3.0.4).
+- Fix zero-size MPI one-sided windows with UCX.
+
+
+Open MPI version 3.1.3
+----------------------
+:Date: October, 2018
+
+- Fix race condition in ``MPI_THREAD_MULTIPLE`` support of non-blocking
+  send/receive path.
+- Fix error handling ``SIGCHLD`` forwarding.
+- Add support for ``CHARACTER`` and ``LOGICAL`` Fortran datatypes for ``MPI_SIZEOF``.
+- Fix compile error when using OpenJDK 11 to compile the Java bindings.
+- Fix crash when using a hostfile with a 'user@host' line.
+- Numerous Fortran '08 interface fixes.
+- TCP BTL error message fixes.
+- OFI MTL now will use any provider other than shm, sockets, tcp, udp, or
+  rstream, rather than only supporting gni, psm, and psm2.
+- Disable async receive of CUDA buffers by default, fixing a hang
+  on large transfers.
+- Support the BCM57XXX and BCM58XXX Broadcomm adapters.
+- Fix minmax datatype support in ROMIO.
+- Bug fixes in vader shared memory transport.
+- Support very large buffers with ``MPI_TYPE_VECTOR``.
+- Fix hang when launching with mpirun on Cray systems.
+
+
+Open MPI version 3.1.2
+----------------------
+:Date: August, 2018
+
+- A subtle race condition bug was discovered in the "vader" BTL
+  (shared memory communications) that, in rare instances, can cause
+  MPI processes to crash or incorrectly classify (or effectively drop)
+  an MPI message sent via shared memory.  If you are using the "ob1"
+  PML with "vader" for shared memory communication (note that vader is
+  the default for shared memory communication with ob1), you need to
+  upgrade to v3.1.2 or later to fix this issue.  You may also upgrade
+  to the following versions to fix this issue:
+
+  - Open MPI v2.1.5 (expected end of August, 2018) or later in the
+    v2.1.x series
+  - Open MPI v3.0.1 (released March, 2018) or later in the v3.0.x
+    series
+
+- Assorted Portals 4.0 bug fixes.
+- Fix for possible data corruption in MPI_BSEND.
+- Move shared memory file for vader btl into /dev/shm on Linux.
+- Fix for ``MPI_ISCATTER``/``MPI_ISCATTERV`` Fortran interfaces with ``MPI_IN_PLACE``.
+- Upgrade PMIx to v2.1.3.
+- Numerous One-sided bug fixes.
+- Fix for race condition in uGNI BTL.
+- Improve handling of large number of interfaces with TCP BTL.
+- Numerous UCX bug fixes.
+
+
+Open MPI version 3.1.1
+----------------------
+:Date: June, 2018
+
+- Fix potential hang in UCX PML during ``MPI_Finalize()``
+- Update internal PMIx to v2.1.2rc2 to fix forward version compatibility.
+- Add new MCA parameter ``osc_sm_backing_store`` to allow users to specify
+  where in the filesystem the backing file for the shared memory
+  one-sided component should live.  Defaults to ``/dev/shm`` on Linux.
+- Fix potential hang on non-x86 platforms when using builds with
+  optimization flags turned off.
+- Disable osc/pt2pt when using ``MPI_THREAD_MULTIPLE`` due to numerous
+  race conditions in the component.
+- Fix dummy variable names for the mpi and mpi_f08 Fortran bindings to
+  match the MPI standard.  This may break applications which use
+  name-based parameters in Fortran which used our internal names
+  rather than those documented in the MPI standard.
+- Revamp Java detection to properly handle new Java versions which do
+  not provide a javah wrapper.
+- Fix RMA function signatures for use-mpi-f08 bindings to have the
+  asynchonous property on all buffers.
+- Improved configure logic for finding the UCX library.
+
+
+Open MPI version 3.1.0
+----------------------
+:Date: May, 2018
+
+- Various OpenSHMEM bug fixes.
+- Properly handle array_of_commands argument to Fortran version of
+  ``MPI_COMM_SPAWN_MULTIPLE``.
+- Fix bug with ``MODE_SEQUENTIAL`` and the sharedfp MPI-IO component.
+- Use ``javac -h`` instead of ``javah`` when building the Java bindings
+  with a recent version of Java.
+- Fix mis-handling of jostepid under SLURM that could cause problems
+  with PathScale/OmniPath NICs.
+- Disable the POWER 7/BE block in configure.  Note that POWER 7/BE is
+  still not a supported platform, but it is no longer automatically
+  disabled.  See
+  https://github.com/open-mpi/ompi/issues/4349#issuecomment-374970982
+  for more information.
+- The output-filename option for ``mpirun`` is now converted to an
+  absolute path before being passed to other nodes.
+- Add monitoring component for PML, OSC, and COLL to track data
+  movement of MPI applications.  See
+  ompi/mca/commmon/monitoring/HowTo_pml_monitoring.tex for more
+  information about the monitoring framework.
+- Add support for communicator assertions: ``mpi_assert_no_any_tag``,
+  ``mpi_assert_no_any_source``, ``mpi_assert_exact_length``, and
+  ``mpi_assert_allow_overtaking``.
+- Update PMIx to version 2.1.1.
+- Update hwloc to 1.11.7.
+- Many one-sided behavior fixes.
+- Improved performance for Reduce and Allreduce using Rabenseifner's algorithm.
+- Revamped mpirun ``--help`` output to make it a bit more manageable.
+- Portals4 MTL improvements: Fix race condition in rendezvous protocol and
+  retry logic.
+- UCX OSC: initial implementation.
+- UCX PML improvements: add multi-threading support.
+- Yalla PML improvements: Fix error with irregular contiguous datatypes.
+- Openib BTL: disable XRC support by default.
+- TCP BTL: Add check to detect and ignore connections from processes
+  that aren't MPI (such as IDS probes) and verify that source and
+  destination are using the same version of Open MPI, fix issue with very
+  large message transfer.
+- ompi_info parsable output now escapes double quotes in values, and
+  also quotes values can contains colons.  Thanks to Lev Givon for the
+  suggestion.
+- CUDA-aware support can now handle GPUs within a node that do not
+  support CUDA IPC.  Earlier versions would get error and abort.
+- Add a mca parameter ras_base_launch_orted_on_hn to allow for launching
+  MPI processes on the same node where mpirun is executing using a separate
+  orte daemon, rather than the mpirun process.   This may be useful to set to
+  true when using SLURM, as it improves interoperability with SLURM's signal
+  propagation tools.  By default it is set to false, except for Cray XC systems.
+- Remove LoadLeveler RAS support.
+- Remove IB XRC support from the OpenIB BTL due to lack of support.
+- Add functionality for IBM s390 platforms.  Note that regular
+  regression testing does not occur on the s390 and it is not
+  considered a supported platform.
+- Remove support for big endian PowerPC.
+- Remove support for XL compilers older than v13.1.
+- Remove support for atomic operations using MacOS atomics library.
diff --git a/docs/release-notes/changelog/v4.0.x.rst b/docs/release-notes/changelog/v4.0.x.rst
new file mode 100644
index 00000000000..70d452c0619
--- /dev/null
+++ b/docs/release-notes/changelog/v4.0.x.rst
@@ -0,0 +1,286 @@
+Open MPI v4.0.x series
+======================
+
+This file contains all the NEWS updates for the Open MPI v4.0.x
+series, in reverse chronological order.
+
+Open MPI version 4.0.6
+----------------------
+:Date: March, 2021
+
+- Update embedded PMIx to 3.2.2.  This update addresses several
+  ``MPI_COMM_SPAWN`` problems.
+- Fix a problem when using Flux PMI and UCX.  Thanks to Sami Ilvonen
+  for reporting and supplying a fix.
+- Fix a problem with MPIR breakpoint being compiled out using PGI
+  compilers.  Thanks to @louisespellacy-arm for reporting.
+- Fix some ROMIO issues when using Lustre.  Thanks to Mark Dixon for
+  reporting.
+- Fix a problem using an external PMIx 4 to build Open MPI 4.0.x.
+- Fix a compile problem when using the enable-timing configure option
+  and UCX.  Thanks to Jan Bierbaum for reporting.
+- Fix a symbol name collision when using the Cray compiler to build
+  Open SHMEM.  Thanks to Pak Lui for reporting and fixing.
+- Correct an issue encountered when building Open MPI under OSX Big Sur.
+  Thanks to FX Coudert for reporting.
+- Various fixes to the OFI MTL.
+- Fix an issue with allocation of sufficient memory for parsing long
+  environment variable values.  Thanks to @zrss for reporting.
+- Improve reproducibility of builds to assist Open MPI packages.
+  Thanks to Bernhard Wiedmann for bringing this to our attention.
+
+
+Open MPI version 4.0.5
+----------------------
+:Date: August, 2020
+
+- Fix a problem with MPI RMA compare and swap operations.  Thanks
+  to Wojciech Chlapek for reporting.
+- Disable binding of MPI processes to system resources by Open MPI
+  if an application is launched using SLURM's srun command.
+- Disable building of the Fortran mpi_f08 module when configuring
+  Open MPI with default 8 byte Fortran integer size.  Thanks to
+  @ahcien for reporting.
+- Fix a problem with mpirun when the ``--map-by`` option is used.
+  Thanks to Wenbin Lyu for reporting.
+- Fix some issues with MPI one-sided operations uncovered using Global
+  Arrays regression test-suite.  Thanks to @bjpalmer for reporting.
+- Fix a problem with make check when using the PGI compiler.  Thanks to
+  Carl Ponder for reporting.
+- Fix a problem with ``MPI_FILE_READ_AT_ALL`` that could lead to application
+  hangs under certain circumstances.  Thanks to Scot Breitenfeld for
+  reporting.
+- Fix a problem building C++ applications with newer versions of GCC.
+  Thanks to Constantine Khrulev for reporting.
+
+
+Open MPI version 4.0.4
+----------------------
+:Date: June, 2020
+
+- Fix a memory patcher issue intercepting shmat and shmdt.  This was
+  observed on RHEL 8.x ppc64le (see README for more info).
+- Fix an illegal access issue caught using gcc's address sanitizer.
+  Thanks to  Georg Geiser for reporting.
+- Add checks to avoid conflicts with a libevent library shipped with LSF.
+- Switch to linking against libevent_core rather than libevent, if present.
+- Add improved support for UCX 1.9 and later.
+- Fix an ABI compatibility issue with the Fortran 2008 bindings.
+  Thanks to Alastair McKinstry for reporting.
+- Fix an issue with rpath of ``/usr/lib64`` when building OMPI on
+  systems with Lustre.  Thanks to David Shrader for reporting.
+- Fix a memory leak occurring with certain MPI RMA operations.
+- Fix an issue with ORTE's mapping of MPI processes to resources.
+  Thanks to Alex Margolin for reporting and providing a fix.
+- Correct a problem with incorrect error codes being returned
+  by OMPI MPI_T functions.
+- Fix an issue with debugger tools not being able to attach
+  to mpirun more than once.  Thanks to Gregory Lee for reporting.
+- Fix an issue with the Fortran compiler wrappers when using
+  NAG compilers.  Thanks to Peter Brady for reporting.
+- Fix an issue with the ORTE ssh based process launcher at scale.
+  Thanks to Benjamín Hernández for reporting.
+- Address an issue when using shared MPI I/O operations.  OMPIO will
+  now successfully return from the file open statement but will
+  raise an error if the file system does not supported shared I/O
+  operations.  Thanks to Romain Hild for reporting.
+- Fix an issue with ``MPI_WIN_DETACH.``  Thanks to Thomas Naughton for reporting.
+
+
+Open MPI version 4.0.3
+----------------------
+:Date: March, 2020
+
+- Update embedded PMIx to 3.1.5
+- Add support for Mellanox ConnectX-6.
+- Fix an issue in OpenMPI IO when using shared file pointers.
+  Thanks to Romain Hild for reporting.
+- Fix a problem with Open MPI using a previously installed
+  Fortran mpi module during compilation.  Thanks to Marcin
+  Mielniczuk for reporting
+- Fix a problem with Fortran compiler wrappers ignoring use of
+  disable-wrapper-runpath configure option.  Thanks to David
+  Shrader for reporting.
+- Fixed an issue with trying to use mpirun on systems where neither
+  ssh nor rsh is installed.
+- Address some problems found when using XPMEM for intra-node message
+  transport.
+- Improve dimensions returned by MPI_Dims_create for certain
+  cases.  Thanks to @aw32 for reporting.
+- Fix an issue when sending messages larger than 4GB. Thanks to
+  Philip Salzmann for reporting this issue.
+- Add ability to specify alternative module file path using
+  Open MPI's RPM spec file.  Thanks to @jschwartz-cray for reporting.
+- Clarify use of ``--with-hwloc`` configuration option in the README.
+  Thanks to Marcin Mielniczuk for raising this documentation issue.
+- Fix an issue with shmem_atomic_set.  Thanks to Sameh Sharkawi for reporting.
+- Fix a problem with ``MPI_Neighbor_alltoall(v,w)`` for cartesian communicators
+  with cyclic boundary conditions.  Thanks to Ralph Rabenseifner and
+  Tony Skjellum for reporting.
+- Fix an issue using Open MPIO on 32 bit systems.  Thanks to
+  Orion Poplawski for reporting.
+- Fix an issue with NetCDF test deadlocking when using the vulcan
+  Open MPIO component.  Thanks to Orion Poplawski for reporting.
+- Fix an issue with the ``mpi_yield_when_idle`` parameter being ignored
+  when set in the Open MPI MCA parameter configuration file.
+  Thanks to @iassiour for reporting.
+- Address an issue with Open MPIO when writing/reading more than 2GB
+  in an operation.  Thanks to Richard Warren for reporting.
+
+
+Open MPI version 4.0.2
+----------------------
+:Date: September, 2019
+
+- Update embedded PMIx to 3.1.4
+- Enhance Open MPI to detect when processes are running in
+  different name spaces on the same node, in which case the
+  vader CMA single copy mechanism is disabled.  Thanks
+  to Adrian Reber for reporting and providing a fix.
+- Fix an issue with ORTE job tree launch mechanism.  Thanks
+  to @lanyangyang for reporting.
+- Fix an issue with env processing when running as root.
+  Thanks to Simon Byrne for reporting and providing a fix.
+- Fix Fortran ``MPI_FILE_GET_POSITION`` return code bug.
+  Thanks to Wei-Keng Liao for reporting.
+- Fix user defined datatypes/ops leak in nonblocking base collective
+  component.  Thanks to Andrey Maslennikov for verifying fix.
+- Fixed shared memory not working with spawned processes.
+  Thanks to @rodarima for reporting.
+- Fix data corruption of overlapping datatypes on sends.
+  Thanks to DKRZ for reporting.
+- Fix segfault in oob_tcp component on close with active listeners.
+  Thanks to Orivej Desh for reporting and providing a fix.
+- Fix divide by zero segfault in ompio.
+  Thanks to @haraldkl for reporting and providing a fix.
+- Fix finalize of flux compnents.
+  Thanks to Stephen Herbein and Jim Garlick for providing a fix.
+- Fix osc_rdma_acc_single_intrinsic regression.
+  Thanks to Joseph Schuchart for reporting and providing a fix.
+- Fix hostnames with large integers.
+  Thanks to @perrynzhou for reporting and providing a fix.
+- Fix Deadlock in ``MPI_Fetch_and_op`` when using UCX
+  Thanks to Joseph Schuchart for reporting.
+- Fix the SLURM plm for mpirun-based launching.
+  Thanks to Jordon Hayes for reporting and providing a fix.
+- Prevent grep failure in rpmbuild from aborting.
+  Thanks to Daniel Letai for reporting.
+- Fix btl/vader finalize sequence.
+  Thanks to Daniel Vollmer for reporting.
+- Fix pml/ob1 local handle sent during PUT control message.
+  Thanks to @EmmanuelBRELLE for reporting and providing a fix.
+- Fix Memory leak with persistent MPI sends and the ob1 "get" protocol.
+  Thanks to @s-kuberski for reporting.
+- v4.0.x: mpi: mark ``MPI_COMBINER_{HVECTOR,HINDEXED,STRUCT}_INTEGER``
+  removed unless configured with ``--enable-mpi1-compatibility``
+- Fix make-authors.pl when run in a git submodule.
+  Thanks to Michael Heinz for reporting and providing a fix.
+- Fix deadlock with ``mpi_assert_allow_overtaking`` in MPI_Issend.
+  Thanks to Joseph Schuchart and George Bosilca for reporting.
+- Add compilation flag to allow unwinding through files that are
+  present in the stack when attaching with MPIR.
+  Thanks to James A Clark for reporting and providing a fix.
+
+.. admonition:: Known Issue:
+   :class: knownissue
+
+   There is a known issue with the OFI libfabric and PSM2 MTLs when trying to send
+   very long (> 4 GBytes) messages.  In this release, these MTLs will catch
+   this case and abort the transfer.  A future release will provide a
+   better solution to this issue.
+
+
+Open MPI version 4.0.1
+----------------------
+:Date: March, 2019
+
+- Update embedded PMIx to 3.1.2.
+- Fix an issue with Vader (shared-memory) transport on OS-X. Thanks
+  to Daniel Vollmer for reporting.
+- Fix a problem with the usNIC BTL Makefile.  Thanks to George Marselis
+  for reporting.
+- Fix an issue when using ``--enable-visibility`` configure option
+  and older versions of hwloc.  Thanks to Ben Menadue for reporting
+  and providing a fix.
+- Fix an issue with ``MPI_WIN_CREATE_DYNAMIC`` and ``MPI_GET`` from self.
+  Thanks to Bart Janssens for reporting.
+- Fix an issue of excessive compiler warning messages from mpi.h
+  when using newer C++ compilers.  Thanks to @Shadow-fax for
+  reporting.
+- Fix a problem when building Open MPI using clang 5.0.
+- Fix a problem with ``MPI_WIN_CREATE`` when using UCX.  Thanks
+  to Adam Simpson for reporting.
+- Fix a memory leak encountered for certain MPI datatype
+  destructor operations.  Thanks to Axel Huebl for reporting.
+- Fix several problems with MPI RMA accumulate operations.
+  Thanks to Jeff Hammond for reporting.
+- Fix possible race condition in closing some file descriptors
+  during job launch using mpirun.  Thanks to Jason Williams
+  for reporting and providing a fix.
+- Fix a problem in OMPIO for large individual write operations.
+  Thanks to Axel Huebl for reporting.
+- Fix a problem with parsing of map-by ppr options to mpirun.
+  Thanks to David Rich for reporting.
+- Fix a problem observed when using the mpool hugepage component.  Thanks
+  to Hunter Easterday for reporting and fixing.
+- Fix valgrind warning generated when invoking certain MPI Fortran
+  data type creation functions.  Thanks to @rtoijala for reporting.
+- Fix a problem when trying to build with a PMIX 3.1 or newer
+  release.  Thanks to Alastair McKinstry for reporting.
+- Fix a problem encountered with building MPI F08 module files.
+  Thanks to Igor Andriyash and Axel Huebl for reporting.
+- Fix two memory leaks encountered for certain MPI-RMA usage patterns.
+  Thanks to Joseph Schuchart for reporting and fixing.
+- Fix a problem with the ORTE ``rmaps_base_oversubscribe`` MCA paramater.
+  Thanks to @iassiour for reporting.
+- Fix a problem with UCX PML default error handler for MPI communicators.
+  Thanks to Marcin Krotkiewski for reporting.
+- Fix various issues with OMPIO uncovered by the testmpio test suite.
+
+
+Open MPI version 4.0.0
+----------------------
+:Date: September, 2018
+
+- OSHMEM updated to the OpenSHMEM 1.4 API.
+- Do not build OpenSHMEM layer when there are no SPMLs available.
+  Currently, this means the OpenSHMEM layer will only build if
+  a MXM or UCX library is found.
+- A UCX BTL was added for enhanced MPI RMA support using UCX
+- With this release,  OpenIB BTL now only supports iWarp and RoCE by default.
+- Updated internal HWLOC to 2.0.2
+- Updated internal PMIx to 3.0.2
+- Change the priority for selecting external verses internal HWLOC
+  and PMIx packages to build.  Starting with this release, configure
+  by default selects available external HWLOC and PMIx packages over
+  the internal ones.
+- Updated internal ROMIO to 3.2.1.
+- Removed support for the MXM MTL.
+- Removed support for SCIF.
+- Improved CUDA support when using UCX.
+- Enable use of CUDA allocated buffers for OMPIO.
+- Improved support for two phase MPI I/O operations when using OMPIO.
+- Added support for Software-based Performance Counters, see
+  https://github.com/davideberius/ompi/wiki/How-to-Use-Software-Based-Performance-Counters-(SPCs)-in-Open-MPI
+- Change MTL OFI from opting-IN on "psm,psm2,gni" to opting-OUT on
+  "shm,sockets,tcp,udp,rstream"
+- Various improvements to MPI RMA performance when using RDMA
+  capable interconnects.
+- Update memkind component to use the memkind 1.6 public API.
+- Fix a problem with javadoc builds using OpenJDK 11.  Thanks to
+  Siegmar Gross for reporting.
+- Fix a memory leak using UCX.  Thanks to Charles Taylor for reporting.
+- Fix hangs in ``MPI_Finalize`` when using UCX.
+- Fix a problem with building Open MPI using an external PMIx 2.1.2
+  library.  Thanks to Marcin Krotkiewski for reporting.
+- Fix race conditions in Vader (shared memory) transport.
+- Fix problems with use of newer map-by mpirun options.  Thanks to
+  Tony Reina for reporting.
+- Fix rank-by algorithms to properly rank by object and span
+- Allow for running as root of two environment variables are set.
+  Requested by Axel Huebl.
+- Fix a problem with building the Java bindings when using Java 10.
+  Thanks to Bryce Glover for reporting.
+- Fix a problem with ORTE not reporting error messages if an application
+  terminated normally but exited with non-zero error code.  Thanks to
+  Emre Brookes for reporting.
diff --git a/docs/release-notes/changelog/v4.1.x.rst b/docs/release-notes/changelog/v4.1.x.rst
new file mode 100644
index 00000000000..f5540505189
--- /dev/null
+++ b/docs/release-notes/changelog/v4.1.x.rst
@@ -0,0 +1,343 @@
+Open MPI v4.1.x series
+======================
+
+This file contains all the NEWS updates for the Open MPI v4.1.x
+series, in reverse chronological order.
+
+Open MPI version 4.1.6
+----------------------
+:Date: September, 2023
+
+- Fix configure issue with XCode 15.
+- Update embedded PMIx to 3.2.5.  PMIx 3.2.5 addresses CVE-2023-41915.
+
+  .. note:: Prior versions of Open MPI (and their associated PMIx
+            implementations) are **not** impacted by this CVE, because
+            Open MPI never uses escalated privileges on behalf of an
+            unprivileged user.  We are backporting this change both
+            because it is low risk and to avoid alarms from CVE
+            scanners.
+
+- Fix issue with buffered sends and MTL-based interfaces (Libfabric,
+  PSM, Portals).
+- Add missing ``MPI_F_STATUS_SIZE`` to ``mpi.h``.  Thanks to @jprotze for
+  reporting the issue.
+- Update Fortran ``mpi`` module configure check to be more correct.
+  Thanks to Sergey Kosukhin for identifying the issue and supplying
+  the fix.
+- Update to properly handle PMIx v>=4.2.3.  Thanks to Bruno Chareyre,
+  Github user @sukanka, and Christof Koehler for raising the
+  compatibility issues and helping test the fixes.
+- Fix minor issues and add some minor performance optimizations with
+  OFI support.
+- Support the ``striping_factor`` and ``striping_unit`` MPI_Info names
+  recomended by the MPI standard for parallel IO.
+- Fixed some minor issues with UCX support.
+- Minor optimization for 0-byte MPI_Alltoallw (i.e., make it a no-op).
+
+
+Open MPI version 4.1.5
+----------------------
+:Date: February, 2023
+
+- Fix crash in one-sided applications for certain process layouts.
+- Update embedded OpenPMIx to version 3.2.4
+- Fix issue building with ``ifort`` on MacOS.
+- Backport patches to Libevent for CVE-2016-10195, CVE-2016-10196, and
+  CVE-2016-10197.
+
+  .. note:: Open MPI's internal libevent does not use the impacted
+            portions of the Libevent code base.
+
+- SHMEM improvements:
+
+  - Fix initializer bugs in SHMEM interface.
+  - Fix unsigned type comparisons generating warnings.
+  - Fix use after clear issue in ``shmem_ds_reset``.
+
+- UCX improvements
+
+  - Fix memory registration bug that could occur when UCX was built
+    but not selected.
+  - Reduce overhead of ``add_procs`` with intercommunicators.
+  - Enable ``multi_send_nb`` by default.
+  - Call ``opal_progress`` while waiting for a UCX fence to complete.
+
+- Fix data corruption bug in osc/rdma component.
+- Fix overflow bug in alltoall collective
+- Fix crash when displaying topology.
+- Add some ``MPI_F_XXX`` constants that were missing from ``mpi.h``.
+- coll/ucc bug fixes.
+
+
+Open MPI version 4.1.4
+----------------------
+:Date: May, 2022
+
+- Fix possible length integer overflow in numerous non-blocking collective
+  operations.
+- Fix segmentation fault in UCX if MPI Tool interface is finalized before
+  ``MPI_Init`` is called.
+- Remove ``/usr/bin/python`` dependency in configure.
+- Fix OMPIO issue with long double etypes.
+- Update treematch topology component to fix numerous correctness issues.
+- Fix memory leak in UCX MCA parameter registration.
+- Fix long operation closing file descriptors on non-Linux systems that
+  can appear as a hang to users.
+- Fix for attribute handling on GCC 11 due to pointer aliasing.
+- Fix multithreaded race in UCX PML's datatype handling.
+- Fix a correctness issue in CUDA Reduce algorithm.
+- Fix compilation issue with CUDA GPUDirect RDMA support.
+- Fix to make ``shmem_calloc(..., 0)`` conform to the OpenSHMEM
+  specification.
+- Add UCC collectives component.
+- Fix divide by zero issue in OMPI IO component.
+- Fix compile issue with libnl when not in standard search locations.
+
+
+Open MPI version 4.1.3
+----------------------
+:Date: March, 2022
+
+- Fixed a seg fault in the ``smcuda`` BTL.  Thanks to Moritz Kreutzer
+  and @Stadik for reporting the issue.
+- Added support for ``ELEMENTAL`` to the MPI handle comparison
+  functions in the ``mpi_f08`` module.  Thanks to Salvatore Filippone
+  for raising the issue.
+- Minor datatype performance improvements in the CUDA-based code paths.
+- Fix ``MPI_ALLTOALLV`` when used with ``MPI_IN_PLACE``.
+- Fix ``MPI_BOTTOM`` handling for non-blocking collectives.  Thanks to
+  Lisandro Dalcin for reporting the problem.
+- Enable OPAL memory hooks by default for UCX.
+- Many compiler warnings fixes, particularly for newer versions of
+  GCC.
+- Fix intercommunicator overflow with large payload collectives.  Also
+  fixed ``MPI_REDUCE_SCATTER_BLOCK`` for similar issues with large
+  payload collectives.
+- Back-port ROMIO 3.3 fix to use stat64() instead of stat() on GPFS.
+- Fixed several non-blocking MPI collectives to not round fractions
+  based on float precision.
+- Fix compile failure for ``--enable-heterogeneous``.  Also updated
+  the README to clarify that ``--enable-heterogeneous`` is functional,
+  but still not recomended for most environments.
+- Minor fixes to OMPIO, including:
+
+  - Fixing the open behavior of shared memory shared file pointers.
+    Thanks to Axel Huebl for reporting the issue
+  - Fixes to clean up lockfiles when closing files.  Thanks to Eric
+    Chamberland for reporting the issue.
+
+- Update LSF configure failure output to be more clear (e.g., on RHEL
+  platforms).
+- Update ``if_[in|ex]clude`` behavior in ``btl_tcp`` and ``oob_tcp``
+  to select *all* interfaces that fall within the specified subnet
+  range.
+
+
+Open MPI version 4.1.2
+----------------------
+:Date: November, 2021
+
+- ROMIO portability fix for OpenBSD
+- Fix handling of ``MPI_IN_PLACE`` with ``MPI_ALLTOALLW`` and improve performance
+  of ``MPI_ALLTOALL`` and ``MPI_ALLTOALLV`` for ``MPI_IN_PLACE.``
+- Fix one-sided issue with empty groups in Post-Start-Wait-Complete
+  synchronization mode.
+- Fix Fortran status returns in certain use cases involving
+  Generalized Requests
+- Romio datatype bug fixes.
+- Fix ``oshmem_shmem_finalize()`` when ``main()`` returns non-zero value.
+- Fix wrong affinity under LSF with the membind option.
+- Fix ``count==0`` cases in ``MPI_REDUCE`` and ``MPI_IREDUCE.``
+- Fix ssh launching on Bourne-flavored shells when the user has ``set -u``
+  set in their shell startup files.
+- Correctly process 0 slots with the ``mpirun --host`` option.
+- Ensure to unlink and rebind socket when the Open MPI session
+  directory already exists.
+- Fix a segv in ``mpirun --disable-dissable-map``.
+- Fix a potential hang in the memory hook handling.
+- Slight performance improvement in ``MPI_WAITALL`` when running in
+  ``MPI_THREAD_MULTIPLE``.
+- Fix hcoll datatype mapping and rooted operation behavior.
+- Correct some operations modifying ``MPI_Status``.  ``MPI_ERROR`` when it is
+  disallowed by the MPI standard.
+- UCX updates:
+
+   - Fix datatype reference count issues.
+   - Detach dynamic window memory when freeing a window.
+   - Fix memory leak in datatype handling.
+
+- Fix various atomic operations issues.
+- mpirun: try to set the curses winsize to the pty of the spawned
+  task.  Thanks to Stack Overflow user @Seriously for reporting the
+  issue.
+- PMIx updates:
+
+   - Fix compatibility with external PMIx v4.x installations.
+   - Fix handling of PMIx v3.x compiler/linker flags.  Thanks to Erik
+     Schnetter for reporting the issue.
+   - Skip SLURM-provided PMIx detection when appropriate.  Thanks to
+     Alexander Grund for reporting the issue.
+
+- Fix handling by C++ compilers when they #include the STL "<version>"
+  header file, which ends up including Open MPI's text VERSION file
+  (which is not C code).  Thanks to @srpgilles for reporting the
+  issue.
+- Fix ``MPI_Op`` support for ``MPI_LONG``.
+- Make the MPI C++ bindings library (libmpi_cxx) explicitly depend on
+  the OPAL internal library (libopen-pal).  Thanks to Ye Luo for
+  reporting the issue.
+- Fix configure handling of ``--with-libevent=/usr``.
+- Fix memory leak when opening Lustre files.  Thanks to Bert Wesarg
+  for submitting the fix.
+- Fix ``MPI_SENDRECV_REPLACE`` to correctly process datatype errors.
+  Thanks to Lisandro Dalcin for reporting the issue.
+- Fix ``MPI_SENDRECV_REPLACE`` to correctly handle large data.  Thanks
+  Jakub Benda for reporting this issue and suggesting a fix.
+- Add workaround for TCP "dropped connection" errors to drastically
+  reduce the possibility of this happening.
+- OMPIO updates:
+
+   - Fix handling when AMODE is not set.  Thanks to Rainer Keller for
+     reporting the issue and supplying the fix.
+   - Fix FBTL "posix" component linking issue.  Thanks for Honggang Li
+     for reporting the issue.
+   - Fixed segv with ``MPI_FILE_GET_BYTE_OFFSET`` on 0-sized file view.
+   - Thanks to GitHub user @shanedsnyder for submitting the issue.
+
+- OFI updates:
+
+   - Multi-plane / Multi-Nic nic selection cleanups
+   - Add support for exporting Open MPI memory monitors into
+     Libfabric.
+   - Ensure that Cisco usNIC devices are never selected by the OFI
+     MTL.
+   - Fix buffer overflow in OFI networking setup.  Thanks to Alexander
+     Grund for reporting the issue and supplying the fix.
+
+- Fix SSEND on tag matching networks.
+- Fix error handling in several MPI collectives.
+- Fix the ordering of ``MPI_COMM_SPLIT_TYPE``.  Thanks to Wolfgang
+  Bangerth for raising the issue.
+- No longer install the orted-mpir library (it's an internal / Libtool
+  convenience library).  Thanks to Andrew Hesford for the fix.
+- PSM2 updates:
+
+   - Allow advanced users to disable PSM2 version checking.
+   - Fix to allow non-default installation locations of psm2.h.
+
+Open MPI version 4.1.1
+----------------------
+:Date: April, 2021
+
+- Fix a number of datatype issues, including an issue with
+  improper handling of partial datatypes that could lead to
+  an unexpected application failure.
+- Change UCX PML to not warn about MPI_Request leaks during
+  ``MPI_Finalize()`` by default.  The old behavior can be restored with
+  the mca_pml_ucx_request_leak_check MCA parameter.
+- Reverted temporary solution that worked around launch issues in
+  SLURM v20.11.{0,1,2}. SchedMD encourages users to avoid these
+  versions and to upgrade to v20.11.3 or newer.
+- Updated PMIx to v3.2.2.
+- Fixed configuration issue on Apple Silicon observed with
+  Homebrew. Thanks to François-Xavier Coudert for reporting the issue.
+- Disabled gcc built-in atomics by default on aarch64 platforms.
+- Disabled UCX PML when UCX v1.8.0 is detected. UCX version 1.8.0 has a bug that
+  may cause data corruption when its TCP transport is used in conjunction with
+  the shared memory transport. UCX versions prior to v1.8.0 are not affected by
+  this issue. Thanks to @ksiazekm for reporting the issue.
+- Fixed detection of available UCX transports/devices to better inform PML
+  prioritization.
+- Fixed SLURM support to mark ORTE daemons as non-MPI tasks.
+- Improved AVX detection to more accurately detect supported
+  platforms.  Also improved the generated AVX code, and switched to
+  using word-based MCA params for the op/avx component (vs. numeric
+  big flags).
+- Improved OFI compatibility support and fixed memory leaks in error
+  handling paths.
+- Improved HAN collectives with support for Barrier and Scatter. Thanks
+  to @EmmanuelBRELLE for these changes and the relevant bug fixes.
+- Fixed MPI debugger support (i.e., the ``MPIR_Breakpoint()`` symbol).
+  Thanks to @louisespellacy-arm for reporting the issue.
+- Fixed ORTE bug that prevented debuggers from reading MPIR_Proctable.
+- Removed PML uniformity check from the UCX PML to address performance
+  regression.
+- Fixed ``MPI_Init_thread(3)`` statement about C++ binding and update
+  references about ``MPI_THREAD_MULTIPLE.``  Thanks to Andreas Lösel for
+  bringing the outdated docs to our attention.
+- Added ``fence_nb`` to Flux PMIx support to address segmentation faults.
+- Ensured progress of AIO requests in the POSIX FBTL component to
+  prevent exceeding maximum number of pending requests on MacOS.
+- Used OPAL's mutli-thread support in the orted to leverage atomic
+  operations for object refcounting.
+- Fixed segv when launching with static TCP ports.
+- Fixed ``--debug-daemons`` mpirun CLI option.
+- Fixed bug where mpirun did not honor ``--host`` in a managed job
+  allocation.
+- Made a managed allocation filter a hostfile/hostlist.
+- Fixed bug to marked a generalized request as pending once initiated.
+- Fixed external PMIx v4.x check.
+- Fixed OSHMEM build with ``--enable-mem-debug``.
+- Fixed a performance regression observed with older versions of GCC when
+  ``__ATOMIC_SEQ_CST`` is used. Thanks to @BiplabRaut for reporting the issue.
+- Fixed buffer allocation bug in the binomial tree scatter algorithm when
+  non-contiguous datatypes are used. Thanks to @sadcat11 for reporting the issue.
+- Fixed bugs related to the accumulate and atomics functionality in the
+  osc/rdma component.
+- Fixed race condition in MPI group operations observed with
+  ``MPI_THREAD_MULTIPLE`` threading level.
+- Fixed a deadlock in the TCP BTL's connection matching logic.
+- Fixed pml/ob1 compilation error when CUDA support is enabled.
+- Fixed a build issue with Lustre caused by unnecessary header includes.
+- Fixed a build issue with IMB LSF workload manager.
+- Fixed linker error with UCX SPML.
+
+
+Open MPI version 4.1.0
+----------------------
+:Date: December, 2020
+
+- collectives: Add HAN and ADAPT adaptive collectives components.
+  Both components are off by default and can be enabled by specifying
+  ``mpirun --mca coll_adapt_priority 100 --mca coll_han_priority 100 ...``.
+  We intend to enable both by default in Open MPI 5.0.
+- OMPIO is now the default for MPI-IO on all filesystems, including
+  Lustre (prior to this, ROMIO was the default for Lustre).  Many
+  thanks to Mark Dixon for identifying MPI I/O issues and providing
+  access to Lustre systems for testing.
+- Updates for macOS Big Sur.  Thanks to FX Coudert for reporting this
+  issue and pointing to a solution.
+- Minor MPI one-sided RDMA performance improvements.
+- Fix hcoll ``MPI_SCATTERV`` with ``MPI_IN_PLACE``.
+- Add AVX support for MPI collectives.
+- Updates to mpirun(1) about "slots" and PE=x values.
+- Fix buffer allocation for large environment variables.  Thanks to
+  @zrss for reporting the issue.
+- Upgrade the embedded OpenPMIx to v3.2.2.
+- Take more steps towards creating fully Reproducible builds (see
+  https://reproducible-builds.org/).  Thanks Bernhard M. Wiedemann for
+  bringing this to our attention.
+- Fix issue with extra-long values in MCA files.  Thanks to GitHub
+  user @zrss for bringing the issue to our attention.
+- UCX: Fix zero-sized datatype transfers.
+- Fix ``--cpu-list`` for non-uniform modes.
+- Fix issue in PMIx callback caused by missing memory barrier on Arm platforms.
+- OFI MTL: Various bug fixes.
+- Fixed issue where ``MPI_TYPE_CREATE_RESIZED`` would create a datatype
+  with unexpected extent on oddly-aligned datatypes.
+- collectives: Adjust default tuning thresholds for many collective
+  algorithms
+- runtime: fix situation where rank-by argument does not work
+- Portals4: Clean up error handling corner cases
+- runtime: Remove ``--enable-install-libpmix`` option, which has not
+  worked since it was added
+- opal: Disable memory patcher component on MacOS
+- UCX: Allow UCX 1.8 to be used with the btl uct
+- UCX: Replace usage of the deprecated NB API of UCX with NBX
+- OMPIO: Add support for the IME file system
+- OFI/libfabric: Added support for multiple NICs
+- OFI/libfabric: Added support for Scalable Endpoints
+- OFI/libfabric: Added btl for one-sided support
+- OFI/libfabric: Multiple small bugfixes
+- libnbc: Adding numerous performance-improving algorithms
diff --git a/docs/release-notes/changelog/v5.0.x.rst b/docs/release-notes/changelog/v5.0.x.rst
new file mode 100644
index 00000000000..71e95861392
--- /dev/null
+++ b/docs/release-notes/changelog/v5.0.x.rst
@@ -0,0 +1,361 @@
+Open MPI v5.0.x series
+======================
+
+This file contains all the NEWS updates for the Open MPI v5.0.x
+series, in reverse chronological order.
+
+Open MPI version v5.0.2
+--------------------------
+:Date: 6 February 2024
+
+- Internal PMIx and PRRTe versions:
+
+  - PMIx (v4.2.8). Commit hash: ``d1c1ed0c2e64f19ad49291241a43630ea7fdce28``.
+  - PRRTE (v3.0.3). Commit hash: ``e383f5ad70c2633420d3425e9fb67e69b6bfd9c4``.
+
+  .. note:: Due to the timing of Open MPI v5.0.2 release, the PMIx and PRRTE versions
+            that are internally bundled in the Open MPI distribution are the same
+            versions as in the Open MPI v5.0.1 release: PMIx v4.2.8 and PRRTE v3.0.3.
+            As a result, changes and bug fixes in newer PMIx and PRRTE releases are
+            not included in Open MPI v5.0.2, notably:
+
+            PRRTE
+
+            - Fix the map-by pe-list option
+            - Use pmix_path_nfs to detect shared file systems
+            - psets: fix some problems with PMIX_QUERY_PSET_MEMBERSHIP query.
+              PMIX_QUERY_PSET_MEMBERSHIP query bugfixes.
+
+            PMIx
+
+            - Restore default to enable-devel-check in Git repos
+            - Protect against empty envar definition for mca_base_param_files
+            - PMIx_Query_info: removed duplicated PMIX_RELEASE
+            - Do not add no-unused-parameter for non-devel-check builds
+            - Cast a few parameters when translating macros to functions
+            - Fix one more spot for "get" from rank undefined
+            - Fix "get" of key with undefined rank
+            - Resolve problem of stack variables and realloc
+            - Restore support for detecting shared file systems
+            - Fix broken link in README
+
+- Bugfixes and changes (in chronological order)
+
+  - Fix the fs/lustre component build
+  - Fix a mtl/ofi multi-threaded race condition bug
+  - Add missing fortran profiling interfaces according to MPI 4.0 standard
+  - Allow 0-size data copy in opal/accelerator
+  - Fix a btl/ofi and mtl/ofi provider selection bug
+  - Properly initialize mca_rcache_base_module_t members
+  - Fix a singleton launch segfault
+  - Add GCC13 support with MacOS compatibility
+
+- Many thanks to Open MPI community including:
+  Erik Schnetter
+
+Open MPI version v5.0.1
+--------------------------
+:Date: 20 December 2023
+
+- Internal PMIx and PRRTe versions:
+
+  - PMIx (v4.2.8). Commit hash: ``d1c1ed0c2e64f19ad49291241a43630ea7fdce28``.
+  - PRRTE (v3.0.3). Commit hash: ``e383f5ad70c2633420d3425e9fb67e69b6bfd9c4``.
+
+- Bugfixes and changes (in chronological order)
+
+  - Various documentation related updates and changes
+  - Fix a bunch of compiler warnings
+  - Sessions: fix a problem with fortran MPI_Session_get_nth_pset interface
+  - Correctly access the communicator name is MSGQ
+  - accelerator/cuda: fix bug in makefile.am preventing correct linkage
+    in non-standard location
+  - btlsmcuda: fix problem with makefile
+  - Fix rcache/gpusm and rcache/rgpsum
+  - Correctly handle attributes on MPI_COMM_WORLD.
+  - Minor memory leak fixes in:
+    btl/tcp, mca_base_framework, ob1
+  - Fix static initialization of recursive mutexes
+  - Spack: fix for dlopen missing symbol problem
+  - opal/mca/accelerator: ROCm 6.0 incompatibility fix
+  - opal_var_dump_color_keys: fix an array overflow
+  - SHMEM_LOCKS: MCS implementation of SHMEM LOCKS
+  - configury: support flang-new
+  - Update processing of "display_map" info key
+  - dpm: update PMIX attribute
+
+- Many thanks to Open MPI community including:
+  Tony Curtis, David Edwards Linaro, Niv Shpak for their contribution.
+
+Open MPI version 5.0.0
+--------------------------
+:Date: 25 October 2023
+
+.. admonition:: The MPIR API has been removed
+   :class: warning
+
+   As was announced in the summer of 2017, Open MPI has removed
+   support for MPIR-based tools beginning with the release of Open MPI
+   v5.0.0.
+
+   Open MPI now uses the `PRRTE <https://github.com/openpmix/prrte>`_
+   runtime environment, which supports the `PMIx <https://pmix.org/>`_
+   tools API |mdash| instead of the legacy MPIR API |mdash| for
+   debugging parallel jobs.
+
+   Users who still need legacy MPIR support should see
+   https://github.com/hpc/mpir-to-pmix-guide for more information.
+
+.. admonition:: Zlib is suggested for better performance
+   :class: note
+
+   `PMIx <https://pmix.org/>`_ will optionally use `Zlib
+   <https://github.com/madler/zlib>`_ to compress large data streams.
+   This may result in faster startup times and smaller memory
+   footprints (compared to not using compression).
+
+   The Open MPI community recommends building PMIx with Zlib support,
+   regardless of whether you are using an externally-installed PMIx or
+   the bundled PMIx that is included with Open MPI distribution
+   tarballs.
+
+   Note that while the Zlib library *may* be present on many systems
+   by default, the Zlib header files |mdash| which are needed to build
+   PMIx with Zlib support |mdash| may need to be installed separately
+   before building PMIx.
+
+.. caution:: Open MPI has changed the default behavior of how it
+             builds and links against its :ref:`required 3rd-party
+             packages <label-install-required-support-libraries>`:
+             `Libevent <https://libevent.org/>`_, `Hardware Locality
+             <https://www.open-mpi.org/projects/hwloc/>`_, `PMIx
+             <https://pmix.org/>`_, and `PRRTE
+             <https://github.com/openpmix/prrte>`_.
+
+             #. Unlike previous versions of Open MPI, Open MPI 5.0 and
+                later will prefer an external package that meets our
+                version requirements, even if it is older than our
+                internal version.
+             #. To simplify managing dependencies, any required
+                packages that Open MPI |ompi_series| bundles will be
+                installed in Open MPI's installation prefix, without
+                name mangling.
+
+                For example, if a valid Libevent installation cannot
+                be found and Open MPI therefore builds its bundled
+                version, a ``libevent.so`` will be installed in Open
+                MPI's installation tree. This is different from
+                previous releases, where Open MPI name-mangled the
+                Libevent symbols and then statically pulled the
+                library into ``libmpi.so``.
+
+- Internal PMIx and PRRTe versions:
+
+  - PMIx release tag v4.2.7. Commit hash: ``57c405c52ad76bab0be9f95e29a6df660673081e``.
+  - PRRTE release tag v3.0.2. Commit hash: ``1552e36f0852bbc6d901ec95983369f0a3c283f6``.
+
+- All other notable updates for v5.0.0:
+
+  - MPI-4.0 updates and additions:
+
+    - Support for MPI Sessions has been added.
+    - Added partitioned communication using persistent sends
+      and persistent receives.
+    - Added persistent collectives to the ``MPI_`` namespace
+      (they were previously available via the ``MPIX_`` prefix).
+    - Added ``MPI_Isendrecv()`` and its variants.
+    - Added support for ``MPI_Comm_idup_with_info()``.
+    - Added support for ``MPI_Info_get_string()``.
+    - Added support for ``initial_error_handler`` and the
+      ``ERRORS_ABORT`` infrastructure.
+    - Added error handling for unbound errors to ``MPI_COMM_SELF``.
+    - Made ``MPI_Comm_get_info()``, ``MPI_File_get_info()``, and
+      ``MPI_Win_get_info()`` compliant to the standard.
+    - Droped unknown/ignored info keys on communicators, files,
+      and windows.
+    - Initial implementations of ``MPI_COMM_TYPE_HW_GUIDED`` and
+      ``MPI_COMM_TYPE_HW_GUIDED`` added.
+    - ``MPI_Info_get()`` and ``MPI_Info_get_valuelen()`` are now
+      deprecated.
+    - Issue a deprecation warning when ``MPI_Cancel()`` is called for
+      a non-blocking send request.
+
+  - New Features:
+
+    - ULFM Fault Tolerance support has been added. See :ref:`the ULFM
+      section <ulfm-label>`.
+    - CUDA is now supported in the ``ofi`` MTL.
+    - A threading framework has been added to allow building Open MPI
+      with different threading libraries. It currently supports
+      `Argobots <https://www.argobots.org/>`_, `Qthreads
+      <https://github.com/Qthreads/qthreads>`_, and Pthreads.  See the
+      ``--with-threads`` option in the ``configure`` command.  Thanks
+      to Shintaro Iwasaki and Jan Ciesko for their contributions to
+      this effort.
+    - New Thread Local Storage API: Removes global visibility of TLS
+      structures and allows for dynamic TLS handling.
+    - Added new ``Accelerator`` framework. CUDA-specific code
+      was replaced with a generic framework that standardizes various
+      device features such as copies or pointer type detection. This
+      allows for modularized implementation of various devices such as
+      the newly introduced ROCm Accelerator component. The redesign
+      also allows for Open MPI builds to be shipped with CUDA
+      support enabled without requiring CUDA libraries.
+    - Added load-linked, store-conditional atomics support for
+      AArch64.
+    - Added atomicity support to the ``ompio`` component.
+    - ``osc/rdma``: Added support for MPI minimum alignment key.
+    - Add ability to detect patched memory to
+      ``memory_patcher``. Thanks to Rich Welch for the contribution.
+    - ``coll/ucc``: Added support for the ``MPI_Scatter()`` and
+      ``MPI_Iscatter()`` collectives.
+    - New algorithm for Allgather and Allgatherv has been added, based
+      on the paper *"Sparbit: a new logarithmic-cost and data
+      locality-aware MPI Allgather algorithm"*. Default algorithm
+      selection rules are unchanged; to use these algorithms add:
+      ``--mca coll_tuned_allgather_algorithm sparbit`` and/or ``--mca
+      coll_tuned_allgatherv_algorithm sparbit`` to your ``mpirun``
+      command.  Thanks to Wilton Jaciel Loch and Guilherme Koslovski
+      for their contribution.
+
+  - Transport updates and improvements
+
+    - One-sided Communication:
+
+      - Many MPI one-sided and RDMA emulation fixes for the ``tcp`` BTL.
+
+        This patch series fixs many issues when running with ``--mca
+        osc rdma --mca btl tcp``, i.e., TCP support for one sided
+        MPI calls.
+
+      - Many MPI one-sided fixes for the ``uct`` BTL.
+      - Added support for ``acc_single_intrinsic`` to the one-sided
+        ``ucx`` component.
+      - Removed the legacy ``pt2pt`` one-sided component. Users should
+        now utilize the ``rdma`` one-sided component instead.  The
+        ``rdma`` component will use BTL components |mdash| such as the
+        TCP BTL |mdash| to effect one-sided communications.
+
+    - Updated the ``tcp`` BTL to use graph solving for global
+      interface matching between peers in order to improve
+      ``MPI_Init()`` wireup performance.
+
+    - OFI
+
+      - Improved support for the HPE SS11 network.
+      - Added cache bypass mechanism. This fixes conflicts with
+        `Libfabric <https://libfabric.org/>`_, which has its own
+        registration cache. This adds a bypass flag which can be used
+        for providers known to have their own registration cache.
+
+    - Shared Memory:
+
+      - Update the new ``sm`` BTL to not use Linux Cross Memory Attach
+        (CMA) in user namespaces.
+      - Fixed a crash when using the new ``sm`` BTL when compiled with
+        Linux Cross Memory Attach (``XPMEM``).  Thanks to George
+        Katevenis for reporting this issue.
+
+    - Updated the ``-mca pml`` option to only accept one PML, not a list.
+
+  - Deprecations and removals:
+
+    - The legacy ``sm`` (shared memory) BTL has been removed.  The
+      next-generation shared memory BTL ``vader`` replaces it, and
+      has been renamed to be ``sm`` (``vader`` will still work as an
+      alias).
+    - ORTE, the underlying Open MPI launcher has been removed, and
+      replaced with the `PMIx Reference RunTime Environment
+      <https://github.com/openpmix/prrte>`_ (``PRTE``).
+    - PMI support has been removed from Open MPI; now only PMIx is
+      supported.  Thanks to Zach Osman for contributing.
+    - The following components have been removed, and are replaced by
+      UCX support: PML ``yalla``, PML ``mxm``, SPML ``ikrit``.
+    - The MTL ``psm`` component has been removed and is no longer
+      supported.
+    - Removed all vestiges of Checkpoint Restart (C/R) support.
+    - 32 bit atomics are now only supported via C11 compliant compilers.
+    - Explicitly disable support for GNU gcc < v4.8.1 (note: the
+      default gcc compiler that is included in RHEL 7 is v4.8.5).
+    - Various atomics support removed: S390/s390x, Sparc v9, ARMv4 and
+      ARMv5 with CMA support.
+    - The MPI C++ bindings have been removed.
+    - The ``mpirun`` options ``--am`` and ``--amca`` options have been
+      deprecated.
+    - The ``libompitrace`` contributed library has been removed.
+      This library was incomplete and unmaintained. If needed, it
+      is available in the v4.x series.
+    - The rankfile format no longer supports physical processor
+      locations. Only logical processor locations are supported.
+    - 32-bit builds have been disabled. Building Open MPI in a 32-bit
+      environment is no longer supported.  32 bit support is still
+      available in the v4.x series.
+
+  - Other updates and bug fixes:
+
+    - Updated Open MPI to use ``ROMIO`` v3.4.1.
+    - Add missing ``MPI_Status`` conversion subroutines:
+      ``MPI_Status_c2f08()``, ``MPI_Status_f082c()``,
+      ``MPI_Status_f082f()``, ``MPI_Status_f2f08()`` and the
+      ``PMPI_*`` related subroutines.
+    - MPI module: added the ``mpi_f08`` ``TYPE(MPI_*)`` types for
+      Fortran.  Thanks to George Katevenis for the report and their
+      contribution to the patch.
+    - The default atomics have been changed to be GCC, with C11 as a
+      fallback. C11 atomics incurs sequential memory ordering, which
+      in most cases is not desired.
+    - The default build mode has changed from building Open MPI's
+      components as Dynamic Shared Objects (DSOs) to being statically
+      included in their respective libraries.
+
+      .. important:: This has consequences for packagers.  Be sure to
+                     read the :ref:`GNU Libtool dependency flattening
+                     <label-install-packagers-gnu-libtool-dependency-flattening>`
+                     subsection.
+
+    - Various datatype bugfixes and performance improvements.
+    - Various pack/unpack bugfixes and performance improvements.
+    - Various OSHMEM bugfixes and performance improvements.
+    - Thanks to Jeff Hammond, Pak Lui, Felix Uhl, Naribayashi Akira,
+      Julien Emmanuel, and Yaz Saito for their invaluable contributions.
+
+  - Documentation updates and improvements:
+
+    - Open MPI has consolidated and converted all of its documentation
+      to use `ReStructured Text.
+      <https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html>`_
+      and `Sphinx <https://www.sphinx-doc.org/>`_.
+
+      - The resulting documentation is now hosted on
+        https://docs.open-mpi.org (via `ReadTheDocs
+        <https://ReadTheDocs.io/>`_).
+      - The documentation is also wholly available offline via Open
+        MPI distribution tarballs, in the ``docs/_build/html``
+        directory.
+
+    - Many, many people from the Open MPI community contributed to the
+      overall documentation effort |mdash| not just those who are
+      listed in the Git commit logs.  Indeed, many Open MPI core
+      developers contributed their time and effort, as did a fairly
+      large group of non-core developers (e.g., those who participated
+      just to help the documentation revamp), including (but not
+      limited to):
+
+      - Lachlan Bell
+      - Simon Byrne
+      - Samuel Cho
+      - Tony Curtis
+      - Lisandro Dalcin
+      - Sophia Fang
+      - Rick Gleitz
+      - Colton Kammes
+      - Robert Langfield
+      - Nick Papior
+      - Luz Paz
+      - Alex Ross
+      - Hao Tong
+      - Mitchell Topaloglu
+      - Siyu Wu
+      - Fangcong Yin
+      - Seth Zegelstein
+      - Yixin Zhang
diff --git a/docs/release-notes/changelog/v6.0.x.rst b/docs/release-notes/changelog/v6.0.x.rst
new file mode 100644
index 00000000000..fbebf74d329
--- /dev/null
+++ b/docs/release-notes/changelog/v6.0.x.rst
@@ -0,0 +1,13 @@
+Open MPI v6.0.x series
+======================
+
+This file contains all the NEWS updates for the Open MPI v6.0.x
+series, in reverse chronological order.
+
+Open MPI version v6.0.0
+--------------------------
+:Date: ...fill me in...
+
+- Open MPI now requires a C11-compliant compiler to build.
+- Removed ROMIO package.
+- Added MPI-4.1 ``MPI_Status_*`` functions.
diff --git a/docs/release-notes/compilers.rst b/docs/release-notes/compilers.rst
new file mode 100644
index 00000000000..b52b38eb000
--- /dev/null
+++ b/docs/release-notes/compilers.rst
@@ -0,0 +1,324 @@
+.. _compiler-notes-section-label:
+
+Compiler Notes
+==============
+
+* Open MPI requires a C11-capable compiler to build.
+
+* On platforms other than x86-64, AArc64 (64-bit ARM), and PPC, Open
+  MPI requires a compiler that either supports C11 atomics or the GCC
+  ``__atomic`` atomics (e.g., GCC >= v4.8.x).
+
+* 32-bit platforms are only supported with a recent compiler that
+  supports C11 atomics. This includes GCC 4.9.x+ (although GCC 6.x or
+  newer is recommended), the Intel compiler suite 16, and clang 3.1.
+
+* Mixing compilers from different vendors when building Open MPI
+  (e.g., using the C/C++ compiler from one vendor and the Fortran
+  compiler from a different vendor) has been successfully employed by
+  some Open MPI users (discussed on the Open MPI user's mailing list),
+  but such configurations are not tested and not documented.  For
+  example, such configurations may require additional compiler /
+  linker flags to make Open MPI build properly.
+
+  A not-uncommon case for this is when building on MacOS with the
+  system-default GCC compiler (i.e., ``/usr/bin/gcc``), but a 3rd party
+  gfortran (e.g., provided by Homebrew, in ``/usr/local/bin/gfortran``).
+  Since these compilers are provided by different organizations, they
+  have different default search paths.  For example, if Homebrew has
+  also installed a local copy of Libevent (a 3rd party package that
+  Open MPI requires), the MacOS-default ``gcc`` linker will find it
+  without any additional command line flags, but the Homebrew-provided
+  gfortran linker will not.  In this case, it may be necessary to
+  provide the following on the configure command line:
+
+  .. code-block:: sh
+
+     shell$ ./configure FCFLAGS=-L/usr/local/lib ...
+
+  This ``-L`` flag will then be passed to the Fortran linker when
+  creating Open MPI's Fortran libraries, and it will therefore be able
+  to find the installed Libevent.
+
+* In general, the latest versions of compilers of a given vendor's
+  series have the least bugs.  We have seen cases where Vendor XYZ's
+  compiler version A.B fails to compile Open MPI, but version A.C
+  (where C>B) works just fine.  If you run into a compile failure, you
+  might want to double check that you have the latest bug fixes and
+  patches for your compiler.
+
+* Users have reported issues with older versions of the Fortran PGI
+  compiler suite when using Open MPI's (non-default) ``--enable-debug``
+  configure option.  Per the above advice of using the most recent
+  version of a compiler series, the Open MPI team recommends using the
+  latest version of the PGI suite, and/or not using the ``--enable-debug``
+  configure option.  If it helps, here's what we have found with some
+  (not comprehensive) testing of various versions of the PGI compiler
+  suite:
+
+  * pgi-8 : NO known good version with ``--enable-debug``
+  * pgi-9 : 9.0-4 known GOOD
+  * pgi-10: 10.0-0 known GOOD
+  * pgi-11: NO known good version with ``--enable-debug``
+  * pgi-12: 12.10 known BAD with ``-m32``, but known GOOD without ``-m32``
+    (and 12.8 and 12.9 both known BAD with ``--enable-debug``)
+  * pgi-13: 13.9 known BAD with ``-m32``, 13.10 known GOOD without ``-m32``
+  * pgi-15: 15.10 known BAD with ``-m32``
+
+* Similarly, there is a known Fortran PGI compiler issue with long
+  source directory path names that was resolved in 9.0-4 (9.0-3 is
+  known to be broken in this regard).
+
+* Open MPI does not support the PGI compiler suite on OS X or MacOS.
+  See issues below for more details:
+
+  * https://github.com/open-mpi/ompi/issues/2604
+  * https://github.com/open-mpi/ompi/issues/2605
+
+* OpenSHMEM Fortran bindings do not support the "no underscore"
+  Fortran symbol convention. IBM's ``xlf`` compilers build in that mode
+  by default.  As such, IBM's ``xlf`` compilers cannot build/link the
+  OpenSHMEM Fortran bindings by default. A workaround is to pass
+  ``FC="xlf -qextname"`` at configure time to force a trailing
+  underscore. See https://github.com/open-mpi/ompi/issues/3612 for
+  more details.
+
+* MPI applications that use the ``mpi_f08`` module on PowerPC platforms
+  (tested ppc64le) will likely experience runtime failures if:
+
+   * they are using a GNU linker (ld) version after v2.25.1 and before
+     v2.28,
+     *and*
+   * they compiled with PGI (tested 17.5) or XL (tested v15.1.5)
+     compilers.  This was noticed on Ubuntu 16.04 which uses the
+     2.26.1 version of ``ld`` by default. However, this issue impacts
+     any OS using a version of ``ld`` noted above. This GNU linker
+     regression will be fixed in version 2.28.  `Here is a link to the
+     GNU bug on this issue
+     <https://sourceware.org/bugzilla/show_bug.cgi?id=21306>`_.  The
+     XL compiler will include a fix for this issue in a future
+     release.
+
+* On NetBSD-6 (at least AMD64 and i386), and possibly on OpenBSD,
+  Libtool misidentifies properties of f95/g95, leading to obscure
+  compile-time failures if used to build Open MPI.  You can work
+  around this issue by ensuring that libtool will not use f95/g95
+  (e.g., by specifying ``FC=<some_other_compiler>``, or otherwise ensuring
+  a different Fortran compiler will be found earlier in the path than
+  ``f95``/``g95``), or by disabling the Fortran MPI bindings with
+  ``--disable-mpi-fortran``.
+
+* On OpenBSD/i386, if you configure with
+  ``--enable-mca-no-build=patcher``, you will also need to add
+  ``--disable-dlopen``.  Otherwise, odd crashes can occur
+  nondeterministically.
+
+* Absoft 11.5.2 plus a service pack from September 2012 (which Absoft
+  says is available upon request), or a version later than 11.5.2
+  (e.g., 11.5.3), is required to compile the Fortran ``mpi_f08``
+  module.
+
+* Open MPI does not support the Sparc v8 CPU target.  However,
+  as of Solaris Studio 12.1, and later compilers, one should not
+  specify ``-xarch=v8plus`` or ``-xarch=v9``.  The use of the options
+  ``-m32`` and ``-m64`` for producing 32 and 64 bit targets, respectively,
+  are now preferred by the Solaris Studio compilers.  GCC may
+  require either ``-m32`` or ``-mcpu=v9 -m32``, depending on GCC version.
+
+* If one tries to build OMPI on Ubuntu with Solaris Studio using the C++
+  compiler and the ``-m32`` option, you might see a warning:
+
+  .. code-block::
+
+     CC: Warning: failed to detect system linker version, falling back to custom linker usage
+
+  And the build will fail.  One can overcome this error by either
+  setting ``LD_LIBRARY_PATH`` to the location of the 32 bit libraries
+  (most likely ``/lib32``), or giving ``LDFLAGS="-L/lib32 -R/lib32"``
+  to the ``configure`` command.  Officially, Solaris Studio is not
+  supported on Ubuntu Linux distributions, so additional problems
+  might occur.
+
+* Open MPI does not support the ``gccfss`` compiler (GCC For SPARC
+  Systems; a now-defunct compiler project from Sun).
+
+* At least some versions of the Intel 8.1 compiler seg fault while
+  compiling certain Open MPI source code files.  As such, it is not
+  supported.
+
+* It has been reported that the Intel 9.1 and 10.0 compilers fail to
+  compile Open MPI on IA64 platforms.  As of 12 Sep 2012, there is
+  very little (if any) testing performed on IA64 platforms (with any
+  compiler).  Support is "best effort" for these platforms, but it is
+  doubtful that any effort will be expended to fix the Intel 9.1 /
+  10.0 compiler issuers on this platform.
+
+* Early versions of the Intel 12.1 Linux compiler suite on x86_64 seem
+  to have a bug that prevents Open MPI from working.  Symptoms
+  including immediate segv of the wrapper compilers (e.g., ``mpicc``) and
+  MPI applications.  As of 1 Feb 2012, if you upgrade to the latest
+  version of the Intel 12.1 Linux compiler suite, the problem will go
+  away.
+
+* `Users have reported
+  <https://github.com/open-mpi/ompi/issues/7615>`_ that the Intel
+  Fortran compiler will fail to link Fortran-based MPI applications on
+  macOS with linker errors similar to this:
+
+  .. code-block:: text
+
+     Undefined symbols for architecture x86_64:
+       "_ompi_buffer_detach_f08", referenced from:
+           import-atom in libmpi_usempif08.dylib
+     ld: symbol(s) not found for architecture x86_64
+
+  It appears that setting the environment variable
+  ``lt_cx_ld_force_load=no`` before invoking Open MPI's ``configure``
+  script works around the issue.  For example:
+
+  .. code-block:: sh
+
+     shell$ lt_cv_ld_force_load=no ./configure ...
+
+* The Portland Group compilers prior to version 7.0 require the
+  ``-Msignextend`` compiler flag to extend the sign bit when converting
+  from a shorter to longer integer.  This is different than other
+  compilers (such as GNU).  When compiling Open MPI with the Portland
+  compiler suite, the following flags should be passed to Open MPI's
+  ``configure`` script:
+
+  .. code-block:: sh
+
+     shell$ ./configure CFLAGS=-Msignextend CXXFLAGS=-Msignextend \
+            --with-wrapper-cflags=-Msignextend \
+            --with-wrapper-cxxflags=-Msignextend ...
+
+  This will both compile Open MPI with the proper compile flags and
+  also automatically add ``-Msignextend`` when the C and C++ MPI wrapper
+  compilers are used to compile user MPI applications.
+
+* It has been reported that Pathscale 5.0.5 and 6.0.527 compilers
+  give an internal compiler error when trying to build Open MPI.
+
+* As of July 2017, the Pathscale compiler suite apparently has no
+  further commercial support, and it does not look like there will be
+  further releases.  Any issues discovered regarding building /
+  running Open MPI with the Pathscale compiler suite therefore may not
+  be able to be resolved.
+
+* Using the Absoft compiler to build the MPI Fortran bindings on Suse
+  9.3 is known to fail due to a Libtool compatibility issue.
+
+* There is now only a single Fortran MPI wrapper compiler and a
+  single Fortran OpenSHMEM wrapper compiler: ``mpifort`` and ``oshfort``,
+  respectively.
+
+  .. caution:: The legacy executable names ``mpif77`` and ``mpif90``
+               still exist, but they are symbolic links to
+               ``mpifort``.  Users should immediately stop using the
+               legacy names, and should always use ``mpifort``.
+
+  Similarly, Open MPI's ``configure`` script only recognizes the ``FC``
+  and ``FCFLAGS`` environment variables (to specify the Fortran
+  compiler and compiler flags, respectively).  The ``F77`` and ``FFLAGS``
+  environment variables are **IGNORED**.
+
+  .. important:: As a direct result, it is **STRONGLY** recommended
+     that you specify a Fortran compiler that uses file suffixes to
+     determine Fortran code layout (e.g., free form vs. fixed).  For
+     example, with some versions of the IBM XLF compiler, it is
+     preferable to use ``FC=xlf`` instead of ``FC=xlf90``, because
+     ``xlf`` will automatically determine the difference between free
+     form and fixed Fortran source code.
+
+  However, many Fortran compilers allow specifying additional
+  command-line arguments to indicate which Fortran dialect to use.
+  For example, if ``FC=xlf90``, you may need to use ``mpifort --qfixed ...``
+  to compile fixed format Fortran source files.
+
+  You can use either ``ompi_info`` or ``oshmem_info`` to see with which
+  Fortran compiler Open MPI was configured and compiled.
+
+  There are up to three sets of Fortran MPI bindings that may be
+  provided (depending on your Fortran compiler):
+
+  #. ``mpif.h``: This is the first MPI Fortran interface that was
+     defined in MPI-1.  It is a file that is included in Fortran
+     source code.  The only interface declared in Open MPI's
+     ``mpif.h`` is ``MPI_SIZEOF`` (because of its polymorphism).  All
+     other interfaces are implicit.
+
+  #. ``mpi`` module: The ``mpi`` module file was added in MPI-2.  It
+     provides strong compile-time parameter type checking for MPI all
+     interfaces.
+
+  #. ``mpi_f08`` module: The ``mpi_f08`` module was added in MPI-3.
+     It provides many advantages over the ``mpif.h`` file and ``mpi``
+     module.  For example, MPI handles have distinct types (vs. all
+     being integers).  See the `MPI-3.0 (or later) standard
+     <https://www.mpi-forum.org/docs/>`_ for more details.
+
+  .. important:: The ``mpi_f08`` module is **STRONGLY** recommended
+     for all new MPI Fortran subroutines and applications.  Note that
+     the ``mpi_f08`` module can be used in conjunction with the other
+     two Fortran MPI bindings in the same application (only one
+     binding can be used per subroutine/function, however).  Full
+     interoperability between ``mpif.h``/``mpi`` module and
+     ``mpi_f08`` module MPI handle types is provided, allowing
+     ``mpi_f08`` to be used in new subroutines in legacy MPI
+     applications.
+
+  Per the OpenSHMEM specification, there is only one Fortran OpenSHMEM
+  binding provided:
+
+  * ``shmem.fh``: All Fortran OpenSHMEM programs should include
+    ``shmem.f``, and Fortran OpenSHMEM programs that use constants
+    defined by OpenSHMEM **MUST** include ``shmem.fh``.
+
+  The following notes apply to the above-listed Fortran bindings:
+
+  * All Fortran compilers support the ``mpif.h``/``shmem.fh``-based
+    bindings, with one exception: the ``MPI_SIZEOF`` interfaces will
+    only be present when Open MPI is built with a Fortran compiler
+    that supports the ``INTERFACE`` keyword and ``ISO_FORTRAN_ENV``.  Most
+    notably, this excludes the GNU Fortran compiler suite before
+    version 4.9.
+
+  * The level of support provided by the ``mpi`` module is based on your
+    Fortran compiler.
+
+    If Open MPI is built with a non-GNU Fortran compiler, or if Open
+    MPI is built with the GNU Fortran compiler >= v4.9, all MPI
+    subroutines will be prototyped in the ``mpi`` module.  All calls to
+    MPI subroutines will therefore have their parameter types checked
+    at compile time.
+
+    If Open MPI is built with an old ``gfortran`` (i.e., < v4.9), a
+    limited ``mpi`` module will be built.  Due to the limitations of
+    these compilers, and per guidance from the MPI-3.0 (and later)
+    specification, all MPI subroutines with "choice" buffers are
+    specifically *not* included in the ``mpi`` module, and their
+    parameters will not be checked at compile time.  Specifically, all
+    MPI subroutines with no "choice" buffers are prototyped and will
+    receive strong parameter type checking at run-time (e.g.,
+    ``MPI_INIT``, ``MPI_COMM_RANK``, etc.).
+
+    Similar to the ``mpif.h`` interface, ``MPI_SIZEOF`` is only
+    supported on Fortran compilers that support ``INTERFACE`` and
+    ``ISO_FORTRAN_ENV``.
+
+  * The ``mpi_f08`` module has been tested with the Intel Fortran
+    compiler and gfortran >= 4.9.  Other modern Fortran compilers
+    likely also work.
+
+    Many older Fortran compilers do not provide enough modern Fortran
+    features to support the ``mpi_f08`` module.  For example, ``gfortran``
+    < v4.9 does provide enough support for the ``mpi_f08`` module.
+
+  You can examine the output of the following command to see all
+  the Fortran features that are/are not enabled in your Open MPI
+  installation:
+
+  .. code-block:: sh
+
+     shell$ ompi_info | grep -i fort
diff --git a/docs/release-notes/extensions.rst b/docs/release-notes/extensions.rst
new file mode 100644
index 00000000000..36021c427d3
--- /dev/null
+++ b/docs/release-notes/extensions.rst
@@ -0,0 +1,8 @@
+Open MPI Extensions
+===================
+
+An MPI "extensions" framework is included in Open MPI, providing
+non-standardized / Open MPI-specific extensions for MPI applications.
+
+:doc:`See the Open MPI-specific features </features/index>` section
+for more information on compiling and using MPI extensions.
diff --git a/docs/release-notes/general.rst b/docs/release-notes/general.rst
new file mode 100644
index 00000000000..e23e55cfd61
--- /dev/null
+++ b/docs/release-notes/general.rst
@@ -0,0 +1,59 @@
+General notes
+=============
+
+The following list of release notes applies to this code base as of
+this writing:
+
+* Open MPI now includes two public software layers: MPI and OpenSHMEM.
+  Throughout this document, references to Open MPI implicitly include
+  both of these layers. When distinction between these two layers is
+  necessary, we will reference them as the "MPI" and "OpenSHMEM"
+  layers respectively.
+
+* OpenSHMEM is a collaborative effort between academia, industry, and
+  the U.S. Government to create a specification for a standardized API
+  for parallel programming in the Partitioned Global Address Space
+  (PGAS).  For more information about the OpenSHMEM project, including
+  access to the current OpenSHMEM specification, please visit
+  http://openshmem.org/.
+
+  .. note:: This OpenSHMEM implementation will only work in Linux
+            environments with a restricted set of supported networks.
+
+* Open MPI includes support for a wide variety of supplemental
+  hardware and software packages.  When configuring Open MPI, you may
+  need to supply additional flags to the ``configure`` script in order
+  to tell Open MPI where the header files, libraries, and any other
+  required files are located.  As such, running ``configure`` by itself
+  may not include support for all the devices (etc.) that you expect,
+  especially if their support headers / libraries are installed in
+  non-standard locations.  Network interconnects are an easy example
+  to discuss |mdash| Libfabric and OpenFabrics networks, for example, both
+  have supplemental headers and libraries that must be found before
+  Open MPI can build support for them.  You must specify where these
+  files are with the appropriate options to configure.  See the
+  listing of configure command-line switches, below, for more details.
+
+* The majority of Open MPI's documentation is here in this document.
+  The man pages are also installed by default.
+
+* Note that Open MPI documentation uses the word "component"
+  frequently; the word "plugin" is probably more familiar to most
+  users.  As such, end users can probably completely substitute the
+  word "plugin" wherever you see "component" in our documentation.
+  For what it's worth, we use the word "component" for historical
+  reasons, mainly because it is part of our acronyms and internal API
+  function calls.
+
+* Open MPI has taken some steps towards `Reproducible Builds
+  <https://reproducible-builds.org/>`_.  Specifically, Open MPI's
+  ``configure`` and ``make`` process, by default, records the build date
+  and some system-specific information such as the hostname where Open
+  MPI was built and the username who built it.  If you desire a
+  Reproducible Build, set the ``$SOURCE_DATE_EPOCH``, ``$USER`` and
+  ``$HOSTNAME`` environment variables before invoking ``configure`` and
+  ``make``, and Open MPI will use those values instead of invoking
+  ``whoami`` and/or ``hostname``, respectively.  See
+  https://reproducible-builds.org/docs/source-date-epoch/ for
+  information on the expected format and content of the
+  ``$SOURCE_DATE_EPOCH`` variable.
diff --git a/docs/release-notes/index.rst b/docs/release-notes/index.rst
new file mode 100644
index 00000000000..8ddc9932d32
--- /dev/null
+++ b/docs/release-notes/index.rst
@@ -0,0 +1,17 @@
+Release notes
+=============
+
+.. toctree::
+   :maxdepth: 1
+
+   changelog/index
+   general
+   platform
+   compilers
+   run-time
+   mpi
+   openshmem
+   mpi-collectives
+   openshmem-collectives
+   networks
+   extensions
diff --git a/docs/release-notes/mpi-collectives.rst b/docs/release-notes/mpi-collectives.rst
new file mode 100644
index 00000000000..4a723a74f58
--- /dev/null
+++ b/docs/release-notes/mpi-collectives.rst
@@ -0,0 +1,9 @@
+MPI Collectives
+===============
+
+* The ``cuda`` coll component provides CUDA-aware support for the
+  reduction type collectives with GPU buffers. This component is only
+  compiled into the library when the library has been configured with
+  CUDA-aware support.  It intercepts calls to the reduction
+  collectives, copies the data to staging buffers if GPU buffers, then
+  calls underlying collectives to do the work.
diff --git a/docs/release-notes/mpi.rst b/docs/release-notes/mpi.rst
new file mode 100644
index 00000000000..caccfd3573a
--- /dev/null
+++ b/docs/release-notes/mpi.rst
@@ -0,0 +1,195 @@
+MPI Functionality and Features
+==============================
+
+.. _release-notes-mpi-standard-conformance-label:
+
+MPI Standard conformance
+------------------------
+
+In the Open MPI |ompi_series| series, all MPI-|mpi_standard_version|
+functionality is supported.  *Some* MPI-4.0 functionality is
+supported.
+
+As such, ``MPI_VERSION`` is set to 3 and ``MPI_SUBVERSION`` is set
+to 1.
+
+For historical reference:
+
+.. list-table::
+   :header-rows: 1
+
+   * - MPI standards conformance
+     - Introduced in Open MPI version
+
+   * - MPI-2.0
+     - Open MPI v1.2
+
+   * - MPI-2.1
+     - Open MPI v1.3
+
+   * - MPI-3.0
+     - Open MPI v1.8
+
+   * - MPI-3.1
+     - Open MPI v2.0
+
+MPI-4.0 partial compliance
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In the Open MPI |ompi_series| series, only partial MPI-4.0 functionality is
+supported. This section contains a list of features added for the release.
+
+* Added support for MPI Sessions.
+* Added partitioned communication using persistent sends and persistent receives.
+* Added persistent collectives to the ``MPI_`` namespace (they were previously
+  available via the ``MPIX_`` prefix).
+* Added support for :ref:`MPI_Isendrecv()<mpi_isendrecv>` and its variants.
+* Added support for :ref:`MPI_Comm_idup_with_info()<mpi_comm_idup_with_info>`.
+* Added support for :ref:`MPI_Info_get_string()<mpi_info_get_string>`.
+* Added support for ``initial_error_handler`` info key and the
+  ``MPI_ERRORS_ABORT`` infrastructure.
+* Added support for ``mpi_minimum_alignment`` info key.
+* Added support for ``MPI_COMM_TYPE_HW_GUIDED`` and
+  ``MPI_COMM_TYPE_HW_UNGUIDED``.
+* Added support for :ref:`MPI_Info_create_env()<mpi_info_create_env>`.
+* Added error handling for "unbound" errors to ``MPI_COMM_SELF``.
+* Added ``MPI_F_STATUS_SIZE``, ``MPI_F_SOURCE``, ``MPI_F_TAG``, and
+  ``MPI_F_ERROR``.
+* Made :ref:`MPI_Comm_get_info()<mpi_comm_get_info>`,
+  :ref:`MPI_File_get_info()<mpi_file_get_info>`, and
+  :ref:`MPI_Win_get_info()<mpi_win_get_info>` MPI-4.0 compliant.
+* Info keys that are not understood by Open MPI will be silently ignored and
+  dropped on communicators, files, and windows.
+* Deprecated :ref:`MPI_Sizeof()<mpi_sizeof>`.
+* Deprecated :ref:`MPI_Cancel()<mpi_cancel>` on send requests.
+* Deprecated :ref:`MPI_Info_get()<mpi_info_get>` and
+  :ref:`MPI_Info_get_valuelen()<mpi_info_get_valuelen>`.
+
+Removed MPI APIs
+----------------
+
+Note that starting with Open MPI v4.0.0, prototypes for several
+legacy MPI-1 symbols that were deleted in the MPI-3.0 specification
+are no longer available by default in ``mpi.h``.  Specifically,
+several MPI-1 symbols were deprecated in the 1996 publishing of the
+MPI-2.0 specification.  These deprecated symbols were eventually
+removed from the MPI-3.0 specification in
+2012.
+
+The symbols that now no longer appear by default in Open MPI's
+``mpi.h`` are:
+
+* ``MPI_Address`` (replaced by ``MPI_Get_address``)
+* ``MPI_Errhandler_create`` (replaced by ``MPI_Comm_create_errhandler``)
+* ``MPI_Errhandler_get`` (replaced by ``MPI_Comm_get_errhandler``)
+* ``MPI_Errhandler_set`` (replaced by ``MPI_Comm_set_errhandler``)
+* ``MPI_Type_extent`` (replaced by ``MPI_Type_get_extent``)
+* ``MPI_Type_hindexed`` (replaced by ``MPI_Type_create_hindexed``)
+* ``MPI_Type_hvector`` (replaced by ``MPI_Type_create_hvector``)
+* ``MPI_Type_lb`` (replaced by ``MPI_Type_get_extent``)
+* ``MPI_Type_struct`` (replaced by ``MPI_Type_create_struct``)
+* ``MPI_Type_ub`` (replaced by ``MPI_Type_get_extent``)
+* ``MPI_LB`` (replaced by ``MPI_Type_create_resized``)
+* ``MPI_UB`` (replaced by ``MPI_Type_create_resized``)
+* ``MPI_COMBINER_HINDEXED_INTEGER``
+* ``MPI_COMBINER_HVECTOR_INTEGER``
+* ``MPI_COMBINER_STRUCT_INTEGER``
+* ``MPI_Handler_function`` (replaced by ``MPI_Comm_errhandler_function``)
+
+Although these symbols are no longer prototyped in ``mpi.h``, they
+are still present in the MPI library in Open MPI |ompi_series|. This
+enables legacy MPI applications to link and run successfully with
+Open MPI |ompi_series|, even though they will fail to compile.
+
+.. warning:: Future releases of Open MPI beyond the |ompi_series|
+   series may remove these symbols altogether.
+
+.. warning:: The Open MPI team **STRONGLY** encourages all MPI
+   application developers to stop using these constructs that were
+   first deprecated over 20 years ago, and finally removed from the
+   MPI specification in MPI-3.0 (in 2012).
+
+.. important:: :ref:`The "Removed MPI constructs" section
+   <label-removed-mpi-constructs>` contains examples of how to update
+   legacy MPI applications using these deleted symbols to use the
+   "new" symbols.
+
+All that being said, if you are unable to immediately update your
+application to stop using these legacy MPI-1 symbols, you can
+re-enable them in ``mpi.h`` by configuring Open MPI with the
+``--enable-mpi1-compatibility`` flag.
+
+Other MPI features
+------------------
+
+* Rank reordering support is available using the TreeMatch library. It
+  is activated for the graph and ``dist_graph`` communicator topologies.
+
+* When using MPI deprecated functions, some compilers will emit
+  warnings.  For example:
+
+  .. code-block::
+
+     shell$ cat deprecated_example.c
+     #include <mpi.h>
+     void foo(void) {
+         MPI_Datatype type;
+         MPI_Type_struct(1, NULL, NULL, NULL, &type);
+     }
+     shell$ mpicc -c deprecated_example.c
+     deprecated_example.c: In function 'foo':
+     deprecated_example.c:4: warning: 'MPI_Type_struct' is deprecated (declared at /opt/openmpi/include/mpi.h:1522)
+     shell$
+
+* ``MPI_THREAD_MULTIPLE`` is supported with some exceptions.
+
+  The following PMLs support ``MPI_THREAD_MULTIPLE``:
+
+  #. ``cm``, when used with the following MTLs:
+
+     #. ``ofi`` (Libfabric)
+     #. ``portals4``
+
+  #. ``ob1``, when used with the following BTLs:
+
+     #. ``self``
+     #. ``sm``
+     #. ``smcuda``
+     #. ``tcp``
+     #. ``usnic``
+
+  #. ``ucx``
+
+  Currently, MPI File operations are not thread safe even if MPI is
+  initialized for ``MPI_THREAD_MULTIPLE`` support.
+
+* ``MPI_REAL16`` and ``MPI_COMPLEX32`` are only supported on platforms
+  where a portable C datatype can be found that matches the Fortran
+  type ``REAL*16``, both in size and bit representation.
+
+* The "libompitrace" library is bundled in Open MPI and is installed
+  by default (it can be disabled via the ``--disable-libompitrace``
+  flag).  This library provides a simplistic tracing of select MPI
+  function calls via the MPI profiling interface.  Linking it in to
+  your application via (e.g., via ``-lompitrace``) will automatically
+  output to stderr when some MPI functions are invoked:
+
+  .. code-block::
+
+     shell$ cd examples/
+     shell$ mpicc hello_c.c -o hello_c -lompitrace
+     shell$ mpirun -n 1 hello_c
+     MPI_INIT: argc 1
+     Hello, world, I am 0 of 1
+     MPI_BARRIER[0]: comm MPI_COMM_WORLD
+     MPI_FINALIZE[0]
+     shell$
+
+  Keep in mind that the output from the trace library is going to
+  ``stderr``, so it may output in a slightly different order than the
+  ``stdout`` from your application.
+
+  This library is being offered as a "proof of concept" / convenience
+  from Open MPI.  If there is interest, it is trivially easy to extend
+  it to printf for other MPI functions.  Pull requests on github.com
+  would be greatly appreciated.
diff --git a/docs/release-notes/networks.rst b/docs/release-notes/networks.rst
new file mode 100644
index 00000000000..400bf21f9cd
--- /dev/null
+++ b/docs/release-notes/networks.rst
@@ -0,0 +1,124 @@
+Network Support
+===============
+
+Main network support models
+---------------------------
+
+There are multiple MPI network models available in this release:
+
+* ``ob1`` supports a variety of networks using BTL ("Byte Transfer
+  Layer") plugins that can be used in
+  combination with each other:
+
+  * ``self``: Loopback (send-to-self)
+  * ``sm``: Shared memory, including single-copy technologies:
+    XPMEM, Linux CMA, as Linux KNEM, as well as traditional
+    copy-in/copy-out shared memory.
+  * ``tcp``: TCP
+  * ``smcuda``: CUDA-enabled shared memory
+  * ``usnic``: Cisco usNIC
+
+* ``cm`` supports a smaller number of networks (and they cannot be
+  used together), but may provide better overall MPI performance by
+  utilizing MTL ("Matching Transport Layer") plugins:
+
+  * OpenFabrics Interfaces ("libfabric" tag matching)
+  * Intel Omni-Path PSM2 (version 11.2.173 or later)
+  * Intel True Scale PSM (QLogic InfiniPath)
+  * Portals 4
+
+* ``ucx`` uses the `Unified Communication X (UCX) communication
+  library <https://www.openucx.org/>`_.  This is an open-source
+  project developed in collaboration between industry, laboratories,
+  and academia to create an open-source production grade
+  communication framework for data centric and high-performance
+  applications.  The UCX library can be downloaded from repositories
+  (e.g., Fedora/RedHat yum repositories).  The UCX library is also
+  part of Mellanox OFED and Mellanox HPC-X binary distributions.
+
+  UCX currently supports:
+
+  * OpenFabrics Verbs (including InfiniBand and RoCE)
+  * TCP
+  * Shared memory
+  * NVIDIA CUDA drivers
+
+While users can manually select any of the above transports at run
+time, Open MPI will select a default transport as follows:
+
+#. If InfiniBand devices are available, use the UCX PML.
+#. If PSM, PSM2, or other tag-matching-supporting Libfabric
+   transport devices are available, use the ``cm``
+   PML and a single appropriate corresponding ``mtl`` module.
+#. Otherwise, use the ``ob1`` PML and one or more appropriate ``btl``
+   modules.
+
+Users can override Open MPI's default selection algorithms and force
+the use of a specific transport if desired by setting the ``pml`` MCA
+parameter (and potentially the ``btl`` and/or ``mtl`` MCA parameters) at
+run-time:
+
+.. code-block:: sh
+
+   shell$ mpirun --mca pml ob1 --mca btl [comma-delimted-BTLs] ...
+   # or
+   shell$ mpirun --mca pml cm --mca mtl [MTL] ...
+   # or
+   shell$ mpirun --mca pml ucx ...
+
+There is a known issue when using UCX with very old Mellanox
+Infiniband HCAs, in particular HCAs preceding the introduction of
+the ConnectX product line, which can result in Open MPI crashing in
+MPI_Finalize.  This issue is addressed by UCX release 1.9.0 and
+newer.
+
+Miscellaneous network notes
+---------------------------
+
+* The main OpenSHMEM network model is ``ucx``; it interfaces directly
+  with UCX.
+
+* In prior versions of Open MPI, InfiniBand and RoCE support was
+  provided through the ``openib`` BTL and ``ob1`` PML plugins.  Starting
+  with Open MPI 4.0.0, InfiniBand support through the ``openib`` plugin
+  is both deprecated and superseded by the ``ucx`` PML component.  The
+  ``openib`` BTL was removed in Open MPI v5.0.0.
+
+  While the ``openib`` BTL depended on ``libibverbs``, the UCX PML depends
+  on the UCX library.
+
+  Once installed, Open MPI can be built with UCX support by adding
+  ``--with-ucx`` to the Open MPI configure command. Once Open MPI is
+  configured to use UCX, the runtime will automatically select the
+  ``ucx`` PML if one of the supported networks is detected (e.g.,
+  InfiniBand).  It's possible to force using UCX in the ``mpirun`` or
+  ``oshrun`` command lines by specifying any or all of the following mca
+  parameters: ``--mca pml ucx`` for MPI point-to-point operations,
+  ``--mca spml ucx`` for OpenSHMEM support, and ``--mca osc ucx`` for MPI
+  RMA (one-sided) operations.
+
+* The ``usnic`` BTL is support for Cisco's usNIC device ("userspace NIC")
+  on Cisco UCS servers with the Virtualized Interface Card (VIC).
+  Although the usNIC is accessed via the OpenFabrics Libfabric API
+  stack, this BTL is specific to Cisco usNIC devices.
+
+
+* Linux ``knem`` support is used when the ``sm`` (shared memory) BTL is
+  compiled with knem support (see the ``--with-knem`` configure option)
+  and the ``knem`` Linux module is loaded in the running kernel.  If the
+  ``knem`` Linux kernel module is not loaded, the ``knem`` support is (by
+  default) silently deactivated during Open MPI jobs.
+
+  See https://knem.gitlabpages.inria.fr/ for details on Knem.
+
+* Linux Cross-Memory Attach (CMA) or XPMEM is used by the ``sm`` shared
+  memory BTL when the CMA/XPMEM libraries are installed,
+  respectively.  Linux CMA and XPMEM are similar (but different)
+  mechanisms for Open MPI to utilize single-copy semantics for shared
+  memory.
+
+* The OFI MTL does not support sending messages larger than the active
+  Libfabric provider's ``max_msg_size``.  If you receive an error
+  message about sending too large of a message when using the OFI MTL,
+  please reach out to your networking vendor to ask them to support a
+  larger ``max_msg_size`` for tagged messages.
diff --git a/docs/release-notes/openshmem-collectives.rst b/docs/release-notes/openshmem-collectives.rst
new file mode 100644
index 00000000000..38e491cdcb2
--- /dev/null
+++ b/docs/release-notes/openshmem-collectives.rst
@@ -0,0 +1,10 @@
+OpenSHMEM Collectives
+=====================
+
+* The ``fca`` scoll component: the Mellanox Fabric Collective
+  Accelerator (FCA) is a solution for offloading collective operations
+  from the MPI process onto Mellanox QDR InfiniBand switch CPUs and
+  HCAs.
+
+* The ``basic`` scoll component: Reference implementation of all
+  OpenSHMEM collective operations.
diff --git a/docs/release-notes/openshmem.rst b/docs/release-notes/openshmem.rst
new file mode 100644
index 00000000000..46ace7d48e8
--- /dev/null
+++ b/docs/release-notes/openshmem.rst
@@ -0,0 +1,4 @@
+OpenSHMEM Functionality and Features
+====================================
+
+All OpenSHMEM-1.3 functionality is supported.
diff --git a/docs/release-notes/platform.rst b/docs/release-notes/platform.rst
new file mode 100644
index 00000000000..f576da867c9
--- /dev/null
+++ b/docs/release-notes/platform.rst
@@ -0,0 +1,55 @@
+.. _platform-notes-section-label:
+
+Platform Notes
+==============
+
+.. error:: **TODO We should have a canonical list of:**
+
+   *  *required* 3rd-party package versions supported (PRRTE, hwloc,
+      libevent)
+   * back-end run-time systems supported (behind PRRTE)
+   * OS's and compilers supported
+   * network interconnects supported.
+
+Open MPI uses both `OpenPMIx <https://openpmix.github.io/>`_ and
+`PRRTE <https://github.com/openpmix/prrte>`_ for its run-time system support,
+and therefore supports whatever run-time systems they support.
+
+.. note:: See the :ref:`PMIx and PRRTE section
+          <label-running-role-of-pmix-and-prte>` of the Open MPI
+          documentation for more details about those projects and
+          their relationship to Open MPI.
+
+Each version of Open MPI supports a specific set of versions of
+OpenPMIx and PRRTE.  Those versions therefore determine which run-time systems
+that a release of Open MPI supports.
+
+* Systems that have been tested are:
+
+  * Linux (various flavors/distros), 64 bit (x86, ppc, aarch64),
+    with gcc/gfortran (>=7.x+), clang (>=10.x), Intel,
+    and Portland (be sure to also see :ref:`the Compiler Notes
+    section <compiler-notes-section-label>`)
+  * macOS (14.x, 15.x), 64 bit (x86_64) with XCode compilers
+
+* Other systems have been lightly (but not fully) tested:
+
+  * Cygwin 64 bit with gcc
+  * ARMv6, ARMv7, ARMv9
+  * Other 64 bit platforms.
+  * OpenBSD.  Requires configure options ``--enable-mca-no-build=patcher``
+    and ``--disable-dlopen`` with this release.
+  * Problems have been reported when building Open MPI on FreeBSD 11.1
+    using the clang-4.0 system compiler. A workaround is to build
+    Open MPI using the GNU compiler.
+
+.. note:: 32-bit environments are no longer supported.
+
+* The run-time systems that are currently supported are:
+
+  * ssh / rsh
+  * PBS Pro, Torque
+  * Platform LSF (tested with v9.1.1 and later)
+  * Slurm
+  * Cray XE, XC, and XK
+  * Oracle Grid Engine (OGE) 6.1, 6.2 and open source Grid Engine
diff --git a/docs/release-notes/run-time.rst b/docs/release-notes/run-time.rst
new file mode 100644
index 00000000000..b3ca74414f1
--- /dev/null
+++ b/docs/release-notes/run-time.rst
@@ -0,0 +1,30 @@
+General Run-Time Support Notes
+==============================
+
+* The Open MPI installation must be in your ``PATH`` on all nodes (and
+  potentially ``LD_LIBRARY_PATH`` or ``DYLD_LIBRARY_PATH``, if
+  ``libmpi``/``libshmem`` is a shared library), unless using the
+  ``--prefix`` or ``--enable-mpirun-prefix-by-default`` functionality (see
+  below).
+
+* Open MPI's run-time behavior can be customized via Modular Component
+  Architecture (MCA) parameters (see :ref:`this section
+  <label-running-setting-mca-param-values>`) for more information on
+  how to get/set MCA parameter values).  Some MCA parameters can be
+  set in a way that renders Open MPI inoperable.  In particular, some
+  parameters have required options that must be included:
+
+  * If specified, the ``btl`` parameter must include the ``self``
+    component, or Open MPI will not be able to deliver messages to the
+    same rank as the sender.  For example: ``mpirun --mca btl tcp,self
+    ...``
+  * If specified, the ``btl_tcp_if_exclude`` parameter must include the
+    loopback device (``lo`` on many Linux platforms), or Open MPI will
+    not be able to route MPI messages using the TCP BTL.  For example:
+    ``mpirun --mca btl_tcp_if_exclude lo,eth1 ...``
+
+* Running on nodes with different endian and/or different datatype
+  sizes within a single parallel job is supported in this release.
+  However, Open MPI does not resize data when datatypes differ in size
+  (for example, sending a 4 byte ``MPI_DOUBLE`` and receiving an 8 byte
+  ``MPI_DOUBLE`` will fail).
diff --git a/docs/requirements.txt b/docs/requirements.txt
new file mode 100644
index 00000000000..80798525f3d
--- /dev/null
+++ b/docs/requirements.txt
@@ -0,0 +1,4 @@
+sphinx>=4.2.0
+recommonmark
+docutils
+sphinx-rtd-theme
diff --git a/docs/tuning-apps/affinity.rst b/docs/tuning-apps/affinity.rst
new file mode 100644
index 00000000000..856a18b9fe2
--- /dev/null
+++ b/docs/tuning-apps/affinity.rst
@@ -0,0 +1,64 @@
+Processor and memory affinity
+=============================
+
+.. _tuning-using-paffinity-label:
+
+Processor affinity
+------------------
+
+Open MPI supports processor affinity on a variety of systems through
+process binding, in which each MPI process, along with its threads, is
+"bound" to a specific subset of processing resources (cores, packages,
+etc.).  That is, the operating system will constrain that process to
+run on only that subset.
+
+.. note:: The operating system may allow other processes to run on the
+          same resources.
+
+Affinity can improve performance by inhibiting excessive process
+movement |mdash| for example, away from "hot" caches or NUMA memory.
+Judicious bindings can improve performance by reducing resource
+contention (by spreading processes apart from one another) or
+improving interprocess communications (by placing processes close to
+one another).  Binding can also improve performance reproducibility by
+eliminating variable process placement.
+
+.. warning:: Processor affinity probably should *not* be used when a
+             node is over-subscribed (i.e., more processes are
+             launched than there are processors).
+
+             This can lead to a serious degradation in performance
+             (even more than simply oversubscribing the node).  Open
+             MPI will usually detect this situation and automatically
+             disable the use of processor affinity (and display
+             run-time warnings to this effect).
+
+Memory affinity
+---------------
+
+Memory affinity is critically important on modern servers
+because most architectures exhibit Non-Uniform Memory Access (NUMA)
+architectures.  In a NUMA architecture, memory is physically
+distributed throughout the machine even though it is virtually treated
+as a single address space.  That is, memory may be physically local to
+one or more processors |mdash| and therefore remote to other processors.
+
+Simply put: some memory will be faster to access (for a given process)
+than others.
+
+Open MPI supports general and specific memory affinity, meaning that
+it generally tries to allocate all memory local to the processor that
+asked for it.  When shared memory is used for communication, Open MPI
+uses memory affinity to make certain pages local to specific
+processes in order to minimize memory network/bus traffic.
+
+Open MPI supports memory affinity on a variety of systems, and is
+controlled through the `Hardware Locality (hwloc)
+<https://www.open-mpi.org/projects/hwloc/>`_ library.
+
+Note that memory affinity support is enabled
+*only when processor affinity is enabled.* Specifically: using memory
+affinity does not make sense if processor affinity is not enabled
+because processes may allocate local memory and then move to a
+different processor, potentially remote from the memory that it just
+allocated.
diff --git a/docs/tuning-apps/benchmarking.rst b/docs/tuning-apps/benchmarking.rst
new file mode 100644
index 00000000000..0a555a2fdbb
--- /dev/null
+++ b/docs/tuning-apps/benchmarking.rst
@@ -0,0 +1,111 @@
+Benchmarking Open MPI applications
+==================================
+
+Running benchmarks is an extremely difficult task to do correctly.
+There are many, many factors to take into account; it is *not* as
+simple as just compiling and running a stock benchmark application.
+This documentation is by no means a definitive guide, but it does try
+to offer some suggestions for generating accurate, meaningful
+benchmarks.
+
+#. Decide *exactly* what you are benchmarking and setup your system
+   accordingly.  For example, if you are trying to benchmark maximum
+   performance, then many of the suggestions listed below are
+   extremely relevant (be the only user on the systems and network in
+   question, be the only software running, use processor affinity,
+   etc.).  If you're trying to benchmark average performance, some of
+   the suggestions below may be less relevant.  Regardless, it is
+   critical to *know* exactly what you're trying to benchmark, and
+   *know* (not guess) both your system and the benchmark application
+   itself well enough to understand what the results mean.
+
+   To be specific, many benchmark applications are not well understood
+   for exactly what they are testing.  There have been many cases
+   where users run a given benchmark application and wrongfully
+   conclude that their system's performance is bad |mdash| solely on
+   the basis of a single benchmark that they did not understand.  Read
+   the documentation of the benchmark carefully, and possibly even
+   look into the code itself to see exactly what it is testing.
+
+   Case in point: not all ping-pong benchmarks are created equal.
+   Most users assume that a ping-pong benchmark is a ping-pong
+   benchmark is a ping-pong benchmark.  But this is not true; the
+   common ping-pong benchmarks tend to test subtly different things
+   (e.g., NetPIPE, TCP bench, IMB, OSU, etc.).  *Make sure you
+   understand what your benchmark is actually testing.*
+
+#. Make sure that you are the *only* user on the systems where you are
+   running the benchmark to eliminate contention from other
+   processes.
+
+#. Make sure that you are the *only* user on the entire network /
+   interconnect to eliminate network traffic contention from other
+   processes.  This is usually somewhat difficult to do, especially in
+   larger, shared systems.  But your most accurate, repeatable results
+   will be achieved when you are the only user on the entire network.
+
+#. Disable all services and daemons that are not being used.  Even
+   "harmless" daemons consume system resources (such as RAM) and cause
+   "jitter" by occasionally waking up, consuming CPU cycles, reading
+   or writing to disk, etc.  The optimum benchmark system has an
+   absolute minimum number of system services running.
+
+#. Ensure that processor and memory affinity are properly utilized to
+   disallow the operating system from swapping MPI processes between
+   processors (and causing unnecessary cache thrashing, for example).
+
+   .. warning:: On NUMA architectures, having the processes getting
+                bumped from one socket to another is more expensive in
+                terms of cache locality (with all of the cache
+                coherency overhead that comes with the lack of it)
+                than in terms of memory transfer routing (see below).
+
+#. Be sure to understand your system's architecture, particularly with
+   respect to the memory, disk, and network characteristics, and test
+   accordingly.  For example, on NUMA architectures, memory accesses
+   may be routed through a memory interconnect; remote device and/or
+   memory accesses will be noticeably slower than local device and/or
+   memory accesses.
+
+#. Compile your benchmark with the appropriate compiler optimization
+   flags.  With some MPI implementations, the compiler wrappers (like
+   ``mpicc``, ``mpifort``, etc.) add optimization flags
+   automatically.  Open MPI does not.  Add ``-O`` or other flags
+   explicitly.
+
+#. Make sure your benchmark runs for a sufficient amount of time.
+   Short-running benchmarks are generally less accurate because they
+   take fewer samples; longer-running jobs tend to take more samples.
+
+#. If your benchmark is trying to benchmark extremely short events
+   (such as the time required for a single ping-pong of messages):
+
+   * Perform some "warmup" events first.  Many MPI implementations
+     (including Open MPI) |mdash| and other subsystems upon which the
+     MPI uses |mdash| may use "lazy" semantics to setup and maintain
+     streams of communications.  Hence, the first event (or first few
+     events) may well take significantly longer than subsequent
+     events.
+
+   * Use a high-resolution timer if possible |mdash|
+     ``gettimeofday()`` only returns millisecond precision (sometimes
+     on the order of several microseconds).
+
+   * Run the event many, many times (hundreds or thousands, depending
+     on the event and the time it takes).  Not only does this provide
+     more samples, it may also be necessary, especially when the
+     precision of the timer you're using may be several orders of
+     magnitude less precise than the event you're trying to
+     benchmark.
+
+#. Decide whether you are reporting minimum, average, or maximum
+   numbers, and have good reasons why.
+
+#. Accurately label and report all results.  Reproducibility is a
+   major goal of benchmarking; benchmark results are effectively
+   useless if they are not precisely labeled as to exactly what they
+   are reporting.  Keep a log and detailed notes about the ''exact''
+   system configuration that you are benchmarking.  Note, for example,
+   all hardware and software characteristics (to include hardware,
+   firmware, and software versions as appropriate).
+
diff --git a/docs/tuning-apps/coll-tuned.rst b/docs/tuning-apps/coll-tuned.rst
new file mode 100644
index 00000000000..fa9c7ba7236
--- /dev/null
+++ b/docs/tuning-apps/coll-tuned.rst
@@ -0,0 +1,476 @@
+Tuning Collectives
+==================
+
+Open MPI's ``coll`` framework provides a number of components implementing
+collective communication, including: ``han``, ``libnbc``, ``self``, ``ucc`` ``base``,
+``hcoll``, ``sync``, ``xhc``, ``accelerator``, ``basic``, ``ftagree``, ``inter``, ``portals4``,
+and ``tuned``.  Some of these components may not be available depending on how
+Open MPI was compiled and what hardware is available on the system.  A run-time
+decision based on each component's self reported priority, selects which
+component will be used.  These priorities may be adjusted on the command line
+or with any of the other usual ways of setting MCA variables, giving us a way
+to influence or override component selection.  In the end, which of the
+available components is selected depends on a number of factors such as the
+underlying hardware and the whether or not a specific collective is provided by
+the component as not all components implement all collectives.  However, there
+is always a fallback ``base`` component that steps in and takes over when another
+component fails to provide an implementation.
+
+The remainder of this section describes the tuning options available in the
+``tuned`` collective component.
+
+Fixed, Forced, and Dynamic Decisions
+------------------------------------
+Open MPI's ``tuned`` collective component has three modes of operation, *fixed
+decision*, *forced algorithm*, and *dynamic decision*.  Since different
+collective algorithms perform better in different situations, the purpose of
+these modes is, when a collective is called, to select an appropriate
+algorithm.
+
+In the *fixed decision* mode a decision tree, essentially a large set of nested
+if-then-else-if blocks with baked in comm and message size thresholds derived
+by measuring performance on existing clusters, selects one of a number of
+available algorithms for the specific collective being called.  *Fixed
+decision* is the default.  The *fixed decision* mode can work well if your
+cluster hardware is similar to the systems profiled when constructing the
+decision tree.  However, in some cases the *fixed decision* rules can yield
+less than ideal performance. For instance, when your hardware differs
+substantially from that which was used to derive the fixed rules decision
+tree.
+
+In the *dynamic decision* mode a user can provide a set of rules encoded in an
+ASCII file that tell the ``tuned`` component which algorithm should be for one or
+more collectives as a function of communicator and message size.  For the
+collectives which rules are not provided the *fixed decision* mode is used.
+More detail about the dynamic decision mode and rules file can be found in
+section :ref:`Rules File<RulesFile>`.
+
+In the *forced algorithm* mode one can specify an algorithm for a specific
+collective on the command line. This is discussed in detail in section
+:ref:`Forcing an Algorithm<ForceAlg>`.
+
+.. _ForceAlg:
+
+Forcing an Algorithm
+--------------------
+
+The simplest method of tuning is to force the ``tuned`` collective component to
+use a specific algorithm for all calls to a specific collective. This can be
+done on the command line, or by any of the other usual ways of setting MCA
+variables.  For example, the following command line sets the algorithm to
+linear for all MPI_Alltoall calls in the entire run.
+
+.. code-block:: sh
+
+   shell$ mpirun ... --mca coll_tuned_use_dynamic_rules 1 \
+                     --mca coll_tuned_alltoall_algorithm 1 ...
+
+When an algorithm is selected this way, the *fixed decision* rules associated
+with that collective are short circuited.  This is an easy way to select a
+collective algorithm and is most effective when there is a single communicator
+and message sizes are static with in the run. On the other hand, when the
+message size for a given collective varies during the run, or there are
+multiple communicators of different sizes, the effectiveness of this approach
+may be limited. The :ref:`rules file <RulesFile>` provides a more comprehensive
+and flexible approach to tuning.  For collectives where an algorithm is not
+forced on the command line, the *fixed decision* rules are used.
+
+.. _RulesFile:
+
+Dynamic Decisions and the Rules File
+------------------------------------
+
+Given that the best choice of algorithm for a given collective depends on a
+number of factors only known at run time, and that some of these factors may
+vary within a run, setting an algorithm on the command line often is an
+ineffective means of tuning.  The rules file provides a means of choosing
+an algorithm at run time based on communicator and message size.  The rules
+file can be specified on the command line, or the other usual ways to set MCA
+variables. The file is parsed during initialization and takes affect there
+after.
+
+.. code-block:: sh
+
+   shell$ mpirun ... --mca coll_tuned_use_dynamic_rules 1 \
+                     --mca coll_tuned_dynamic_rules_filename /path/to/my_rules.json ...
+
+The loaded set of rules then are used to select the algorithm
+to use based on the collective, the communicator size, and the message size.
+Collectives for which rules have not be specified in the file will make use of
+the *fixed decision* rules as usual.
+
+Starting with Open MPI 6.0, dynamic tuning files can be specified in JSON
+format, although the classic format will still be accepted.  A converter script
+is also available to transfer classic format files into JSON.
+
+The JSON format can be checked using the schema in
+`docs/tuning-apps/tuned_dynamic_file_schema.json`.  If your editor supports it,
+this schema may provide validation of your file along with helpful tooltips for
+each variable.
+
+An example file is shown here:
+
+.. code-block:: json
+
+    {
+        "$schema": "tuned_schema.json",
+        "rule_file_version" : 3,
+        "module" : "tuned",
+        "collectives" : {
+            "allreduce" :
+            [
+                {
+                    "comm_size_min" : 64,
+                    "comm_size_max" : 128,
+                    "rules" : [
+                        {
+                            "msg_size_min" : 512,
+                            "msg_size_max" : 511999,
+                            "alg" : 2,
+                        },
+                        {
+                            "msg_size_min" : 512000,
+                            "msg_size_max" : "inf",
+                            "alg" : "recursive_doubling",
+                            "reqs" : 8
+                        }
+                    ]
+                }
+            ]
+        }
+    }
+
+In this toy example the MPI_Allreduce collective (indicated by the `allreduce`
+field) has two algorithms that will only be used on communicators with between
+64 and 128 ranks.  Additionally, those rules only apply to certain message
+sizes.  All others communicator sizes or message sizes fall back to the default
+set of rules, and collectives other than MPI_Allreduce are not affected.
+
+Unlike in the classic file format, there is no need to specify a default rule or
+specify rules in increasing order.  Overlapping message sizes or communicator
+sizes are allowed, and won't emit warnings.
+
+The process for selecting the matching rule is a simple first-match principle.
+During communicator creation, the first set of communicator-rules which
+satisfies the requirements (`comm_size_min`/`comm_size_max`) is selected. Then,
+during each collective call, the message size is used to find the first matching
+entry in the "rules" list.
+
+The algorithm selected is indicated by the `alg` field.  It may be either an
+integer mapping to the classic file format, or a string.  In both cases, the
+value is checked against the appropriate coll_tuned_<collectived>_algorithm MCA
+parameter, and un-recognized values will cause the rule to be ignored.
+
+
+Classic file format:
+
+.. code-block:: sh
+   :linenos:
+
+   rule-file-version-2
+   1   # num of collectives
+   3   # collective ID
+   1   # number of comm sizes
+   #=====================
+   64   # comm size
+   14   # number of rules
+   # Bytes   alg topo segs reqs
+   #----------------------
+   0            0 0 0 0
+   512000       4 0 0 64
+   1536000      4 0 0 64
+   3072000      4 0 0 64
+   6144000      4 0 0 64
+   12288000     4 0 0 16
+   24576000     4 0 0 16
+   49152000     4 0 0 16
+   98304000     4 0 0 16
+   196608000    4 0 0 8
+   393216000    4 0 0 8
+   786432000    4 0 0 1
+   1572864000   4 0 0 1
+   2621440000   0 0 0 0
+
+The rules file effectively defines, for one or more collectives, a function of
+two variables, which given communicator and message size, returns an algorithm
+id to use for the call.  This mechanism allows one to specify for each
+collective, an algorithm for any number of ranges of message and communicator
+sizes.  As communicators are constructed, a search of the rules table is made
+using the communicator size to select a set of message size rules to be used
+with that communicator.  Later as the collective is invoked, a search of the
+message size rules associated with the communicator is made. The rule with the
+nearest (less than) matching message size specifies the algorithm that is used.
+The actual definition of *message size* is dependent on the collective in
+question, see the section on :ref:`collectives and algorithms<CollectivesAndAlgorithms>`
+for details.
+
+One may provide rules for as many collectives, communicator sizes, and message
+sizes as desired. Simply repeat the sections as needed and adjust the relevant
+count parameters.  One must always provide a rule for message size of zero.
+Message size rules are expected in ascending order. The last parameters in the
+message size rule may or may not be used and have different meaning depending
+on the collective and algorithm. The first two parameters in the rule following
+the algorithm ID, `topo` and `segment size`, are always required. In version 2
+of the file format a third parameter, `max requests`, may also be provided. A
+release of Open MPI at least v5.0.7 is required for version 2 features.
+
+The file format version specifier, `rule-file-version-N` where N is an integer
+greater or equal to 1, should appear on the first line of the file.  If the
+version specifier is not present, the file format is assumed to be version 1.
+Version 2 or greater is required to use the `max requests` parameter. Open MPI
+releases older than v5.0.7 do not support the file format version
+identifier. When using older releases of Open MPI do not include a version
+specifier and do not use the `max requests` parameter in message size rules.
+
+.. _CollectivesAndAlgorithms:
+
+Collectives and their Algorithms
+--------------------------------
+The following table lists the collectives
+implemented by the ``tuned`` collective component along with the enumeration value identifying it.
+It is this value that must be used in the rules file when specifying a set of rules.
+The definition of *message size* is dependent on the collective and is given in the table.
+Tables describing the algorithms available for each collective, and there identifiers, are linked.
+
+.. csv-table:: Collectives
+   :header: "Collective", "Id", "Message Size"
+   :widths: 20, 10, 65
+
+   :ref:`Allgather<Allgather>`, 0, "datatype size * comm size * number of elements in send buffer"
+   :ref:`Allgatherv<Allgatherv>`, 1, "datatype size * sum of number of elements that are to be received from each process (sum of recvcounts)"
+   :ref:`Allreduce<Allreduce>`, 2, "datatype size * number of elements in send buffer"
+   :ref:`Alltoall<Alltoall>`, 3, "datatype size * comm size * number of elements to send to each process"
+   :ref:`Alltoallv<Alltoallv>`, 4, "not used"
+   :ref:`Barrier<Barrier>`, 6, "not used"
+   :ref:`Bcast<Bcast>`, 7, "datatype size * number of entries in buffer"
+   :ref:`Exscan<Exscan>`, 8, "datatype size * comm size"
+   :ref:`Gather<Gather>`, 9, "datatype size * comm size * number of elements in send buffer"
+   :ref:`Reduce<Reduce>`, 11, "datatype size * number of elements in send buffer"
+   :ref:`Reduce_scatter<Reduce_scatter>`, 12, "datatype size * sum of number of elements in result distributed to each process (sum of recvcounts)"
+   :ref:`Reduce_scatter_block<Reduce_scatter_block>`, 13, "datatype size * comm size * element count per block"
+   :ref:`Scan<Scan>`, 14, "datatype size * comm size"
+   :ref:`Scatter<Scatter>`, 15, "datatype size * number of elements in send buffer"
+
+.. _Allgather:
+
+Allgather (Id=0)
+~~~~~~~~~~~~~~~~
+
+.. csv-table:: Allgather Algorithms
+   :header: "Id", "Name", "Description"
+   :widths: 10, 25, 60
+
+   0, "ignore", "Use fixed rules"
+   1, "linear", "..."
+   2, "bruck-k-fanout", "..."
+   3, "recursive_doubling", "..."
+   4, "ring", "..."
+   5, "neighbor", "..."
+   6, "two_proc", "..."
+   7, "sparbit", "..."
+   8, "direct-messaging", "..."
+
+.. _Allgatherv:
+
+Allgatherv (Id=1)
+~~~~~~~~~~~~~~~~~
+
+.. csv-table:: Allgatherv Algorithms
+   :header: "Id", "Name", "Description"
+   :widths: 10, 25, 60
+
+   0, "ignore", "Use fixed rules"
+   1, "default", "..."
+   2, "bruck", "..."
+   3, "ring", "..."
+   4, "neighbor", "..."
+   5, "two_proc", "..."
+   6, "sparbit", "..."
+
+.. _Allreduce:
+
+Allreduce (Id=2)
+~~~~~~~~~~~~~~~~
+
+.. csv-table:: Allreduce Algorithms
+   :header: "Id", "Name", "Description"
+   :widths: 10, 25, 60
+
+   0, "ignore", "Use fixed rules"
+   1, "basic_linear", "..."
+   2, "nonoverlapping", "..."
+   3, "recursive_doubling", "..."
+   4, "ring", "..."
+   5, "segmented_ring", "..."
+   6, "rabenseifner", "..."
+   7, "allgather_reduce", "..."
+
+.. _Alltoall:
+
+Alltoall (Id=3)
+~~~~~~~~~~~~~~~
+
+.. csv-table:: Alltoall Algorithms
+   :header: "Id", "Name", "Description"
+   :widths: 10, 25, 60
+
+   0, "ignore", "Use fixed rules"
+   1, "linear", "Launches all non-blocking send/recv pairs and wait for them to complete."
+   2, "pairwise", "For comm size P, implemented as P rounds of blocking MPI_Sendrecv"
+   3, "modified_bruck", "An algorithm exploiting network packet quantization to achieve O(log) time complexity. Typically best for very small message sizes."
+   4, "linear_sync", "Keep N non-blocking MPI_Isend/Irecv pairs in flight at all times. N is set by the coll_tuned_alltoall_max_requests MCA variable."
+   5, "two_proc", "An implementation tailored for alltoall between 2 ranks, otherwise it is not used."
+
+.. _Alltoallv:
+
+Alltoallv (Id=4)
+~~~~~~~~~~~~~~~~
+
+.. csv-table:: Alltoallv Algorithms
+   :header: "Id", "Name", "Description"
+   :widths: 10, 25, 60
+
+   0, "ignore", "Use fixed rules"
+   1, "basic_linear", "..."
+   2, "pairwise", "..."
+
+.. _Barrier:
+
+Barrier (Id=6)
+~~~~~~~~~~~~~~
+
+.. csv-table:: Barrier Algorithms
+   :header: "Id", "Name", "Description"
+   :widths: 10, 25, 60
+
+   0, "ignore", "Use fixed rules"
+   1, "linear", "..."
+   2, "double_ring", "..."
+   3, "recursive_doubling", "..."
+   4, "bruck", "..."
+   5, "two_proc", "..."
+   6, "tree", "..."
+
+.. _Bcast:
+
+Bcast (Id=7)
+~~~~~~~~~~~~
+
+.. csv-table:: Bcast Algorithms
+   :header: "Id", "Name", "Description"
+   :widths: 10, 25, 60
+
+   0, "ignore", "Use fixed rules"
+   1, "basic_linear", "..."
+   2, "chain", "..."
+   3, "pipeline", "..."
+   4, "split_binary_tree", "..."
+   5, "binary_tree", "..."
+   6, "binomial", "..."
+   7, "knomial", "..."
+   8, "scatter_allgather", "..."
+   9, "scatter_allgather_ring", "..."
+
+.. _Exscan:
+
+Exscan (Id=8)
+~~~~~~~~~~~~~
+
+.. csv-table:: Exscan Algorithms
+   :header: "Id", "Name", "Description"
+   :widths: 10, 25, 60
+
+   0, "ignore", "Use fixed rules"
+   1, "linear", "..."
+   2, "recursive_doubling", "..."
+
+.. _Gather:
+
+Gather (Id=9)
+~~~~~~~~~~~~~
+
+.. csv-table:: Gather Algorithms
+   :header: "Id", "Name", "Description"
+   :widths: 10, 25, 60
+
+   0, "ignore", "Use fixed rules"
+   1, "basic_linear", "..."
+   2, "binomial", "..."
+   3, "linear_sync", "..."
+
+.. _Reduce:
+
+Reduce (Id=11)
+~~~~~~~~~~~~~~
+
+.. csv-table:: Reduce Algorithms
+   :header: "Id", "Name", "Description"
+   :widths: 10, 25, 60
+
+   0, "ignore", "Use fixed rules"
+   1, "linear", "..."
+   2, "chain", "..."
+   3, "pipeline", "..."
+   4, "binary", "..."
+   5, "binomial", "..."
+   6, "in-order_binary", "..."
+   7, "rabenseifner", "..."
+   8, "knomial", "..."
+
+.. _Reduce_scatter:
+
+Reduce_scatter (Id=12)
+~~~~~~~~~~~~~~~~~~~~~~
+
+.. csv-table:: Reduce_scatter Algorithms
+   :header: "Id", "Name", "Description"
+   :widths: 10, 25, 60
+
+   0, "ignore", "Use fixed rules"
+   1, "non-overlapping", "..."
+   2, "recursive_halving", "..."
+   3, "ring", "..."
+   4, "butterfly", "..."
+
+.. _Reduce_scatter_block:
+
+Reduce_scatter_block (Id=13)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. csv-table:: Reduce_scatter_block Algorithms
+   :header: "Id", "Name", "Description"
+   :widths: 10, 25, 60
+
+   0, "ignore", "Use fixed rules"
+   1, "basic_linear", "..."
+   2, "recursive_doubling", "..."
+   3, "recursive_halving", "..."
+   4, "butterfly", "..."
+
+.. _Scan:
+
+Scan (Id=14)
+~~~~~~~~~~~~
+
+.. csv-table:: Scan Algorithms
+   :header: "Id", "Name", "Description"
+   :widths: 10, 25, 60
+
+   0, "ignore", "Use fixed rules"
+   1, "linear", "..."
+   2, "recursive_doubling", "..."
+
+.. _Scatter:
+
+Scatter (Id=15)
+~~~~~~~~~~~~~~~
+
+.. csv-table:: Scatter Algorithms
+   :header: "Id", "Name", "Description"
+   :widths: 10, 25, 60
+
+   0, "ignore", "Use fixed rules"
+   1, "basic_linear", "..."
+   2, "binomial", "..."
+   3, "linear_nb", "..."
+
diff --git a/docs/tuning-apps/dynamic-loading.rst b/docs/tuning-apps/dynamic-loading.rst
new file mode 100644
index 00000000000..3f81bda2639
--- /dev/null
+++ b/docs/tuning-apps/dynamic-loading.rst
@@ -0,0 +1,44 @@
+.. _label-running-dynamic-loading-libmpi:
+
+Dynamically loading ``libmpi`` at runtime
+=========================================
+
+If you want to explicitly load Open MPI's shared library ``libmpi`` at
+runtime by using ``dlopen()`` from C/C ++ or something like the
+``ctypes`` package from Python, extra care is required.  The
+default configuration of Open MPI uses ``dlopen()`` internally to load
+its support components.  These components rely on symbols available in
+``libmpi``.  In order to make the symbols in ``libmpi`` available to
+the components loaded by Open MPI at runtime, your application must load
+``libmpi`` with the ``RTLD_GLOBAL`` option.
+
+In C/C++, this option is specified as the second parameter to the
+POSIX ``dlopen(3)`` function.
+
+When using ``ctypes`` with Python, this can be done with the second
+(optional) parameter to ``CDLL()``.  For example (shown below in Mac OS
+X, where Open MPI's shared library name ends in ``.dylib``; other
+operating systems use other suffixes, such as ``.so``):
+
+.. code-block:: python
+
+   from ctypes import *
+
+   mpi = CDLL('libmpi.0.dylib', RTLD_GLOBAL)
+
+   f = pythonapi.Py_GetArgcArgv
+   argc = c_int()
+   argv = POINTER(c_char_p)()
+   f(byref(argc), byref(argv))
+   mpi.MPI_Init(byref(argc), byref(argv))
+
+   # Your MPI program here
+
+   mpi.MPI_Finalize()
+
+.. note:: The above is just an example showing dynamic loading.  If
+          you want to use MPI in Python, you are much better off using
+          `MPI4Py <https://mpi4py.github.io/>`_.
+
+Other scripting languages should have similar options when dynamically
+loading shared libraries.
diff --git a/docs/tuning-apps/environment-var.rst b/docs/tuning-apps/environment-var.rst
new file mode 100644
index 00000000000..b003c1b3f32
--- /dev/null
+++ b/docs/tuning-apps/environment-var.rst
@@ -0,0 +1,21 @@
+Environment variables set for MPI applications
+==============================================
+
+Open MPI provides the following environment variables that will be
+defined on every MPI process:
+
+* ``OMPI_COMM_WORLD_SIZE``: the number of processes in this process's
+  MPI_COMM_WORLD
+* ``OMPI_COMM_WORLD_RANK``: the MPI rank of this process in
+  MPI_COMM_WORLD
+* ``OMPI_COMM_WORLD_LOCAL_SIZE``: the number of ranks from this job
+  that are running on this node.
+* ``OMPI_COMM_WORLD_LOCAL_RANK``: the relative rank of this process on
+  this node within its job. For example, if four processes in a job
+  share a node, they will each be given a local rank ranging from 0 to
+  3.
+* ``OMPI_UNIVERSE_SIZE``: the number of :ref:`process slots
+  <running-scheduling-slots-label>` allocated to this job. Note that
+  this may be different than the number of processes in the job.
+* ``OMPI_COMM_WORLD_NODE_RANK``: the relative rank of this process on
+  this node looking across *all* jobs.
diff --git a/docs/tuning-apps/fault-tolerance/checkpoint-restart.rst b/docs/tuning-apps/fault-tolerance/checkpoint-restart.rst
new file mode 100644
index 00000000000..d904c7b1efa
--- /dev/null
+++ b/docs/tuning-apps/fault-tolerance/checkpoint-restart.rst
@@ -0,0 +1,27 @@
+.. _ft-checkpoint-restart-label:
+
+Checkpoint and restart of parallel jobs
+=======================================
+
+Old versions of Open MPI (starting from v1.3 series) had support for
+the transparent, coordinated checkpointing and restarting of MPI
+processes (similar to LAM/MPI).
+
+Open MPI supported both the the `BLCR <http://ftg.lbl.gov/checkpoint/>`_
+checkpoint/restart system and a "self" checkpointer that allows
+applications to perform their own checkpoint/restart functionality while taking
+advantage of the Open MPI checkpoint/restart infrastructure.
+For both of these, Open MPI provides a coordinated checkpoint/restart protocol
+and integration with a variety of network interconnects including shared memory,
+Ethernet, and InfiniBand.
+
+The implementation introduced a series of new frameworks and
+components designed to support a variety of checkpoint and restart
+techniques. This enabled support for the methods described above
+(application-directed, BLCR, etc.) as well as other kinds of
+checkpoint/restart systems (e.g., Condor, libckpt) and protocols
+(e.g., uncoordinated, message induced).
+
+.. note:: The checkpoint/restart support was last included as part of
+          the v1.6 series.  It was removed from Open MPI after failing
+          to gain adoption and lack of maintenance.
diff --git a/docs/tuning-apps/fault-tolerance/data-reliability.rst b/docs/tuning-apps/fault-tolerance/data-reliability.rst
new file mode 100644
index 00000000000..690af483a26
--- /dev/null
+++ b/docs/tuning-apps/fault-tolerance/data-reliability.rst
@@ -0,0 +1,35 @@
+.. _ft-data-reliability-label:
+
+End-to-end data reliability for MPI messages
+============================================
+
+Current Open MPI releases have no support for end-to-end data
+reliability, at least not more than currently provided by the
+underlying network.
+
+Previous releases included a data reliability PML component (``dr``) that
+assumed that the underlying
+network is unreliable.  It could drop / restart connections, retransmit
+corrupted or lost data, etc.  The end effect was that data sent through MPI API
+functions would be guaranteed to be reliable.
+
+For example, if you're using TCP as a message transport, chances of
+data corruption are fairly low.  However, other interconnects do *not*
+guarantee that data will be uncorrupted when traveling across the
+network.  Additionally, there are non-zero possibilities that data can
+be corrupted while traversing PCI buses, etc. (some corruption errors
+at this level can be caught/fixed, others cannot).  Such errors are
+not uncommon at high altitudes (!).
+
+Note that such added reliability does incur a performance cost |mdash|
+latency and bandwidth suffer when Open MPI performs the consistency
+checks that are necessary to provide such guarantees.
+
+Most clusters/networks do not need data reliability.  But some do
+(e.g., those operating at high altitudes).  The ``dr`` PML was intended for
+these rare environments where reliability was an issue; and users were
+willing to tolerate slightly slower applications in order to guarantee
+that their job does not crash (or worse, produce wrong answers).
+
+.. note:: The ``dr`` component has been removed from current releases
+          of Open MPI, but is still available in older releases.
diff --git a/docs/tuning-apps/fault-tolerance/index.rst b/docs/tuning-apps/fault-tolerance/index.rst
new file mode 100644
index 00000000000..11edfa12f11
--- /dev/null
+++ b/docs/tuning-apps/fault-tolerance/index.rst
@@ -0,0 +1,32 @@
+Fault tolerance
+===============
+
+The phrase "fault tolerance" means many things to many
+people.  Typical definitions include user processes dumping vital
+state to disk periodically, checkpoint/restart of running processes,
+elaborate recreate-process-state-from-incremental-pieces schemes,
+and many others.
+
+In the scope of Open MPI, we typically define "fault tolerance" to
+mean the ability to recover from one or more component failures in a
+well defined manner with either a transparent or application-directed
+mechanism.  Component failures may exhibit themselves as a corrupted
+transmission over a faulty network interface or the failure of one or
+more serial or parallel processes due to a processor or node failure.
+Open MPI strives to provide the application with a consistent system
+view while still providing a production quality, high performance
+implementation.
+
+Yes, that's pretty much as all-inclusive as possible |mdash| intentionally
+so!  Remember that in addition to being a production-quality MPI
+implementation, Open MPI is also a vehicle for research.  So while
+some forms of "fault tolerance" are more widely accepted and used,
+others are certainly of valid academic interest.
+
+
+.. toctree::
+   :maxdepth: 1
+
+   supported
+   checkpoint-restart
+   data-reliability
diff --git a/docs/tuning-apps/fault-tolerance/supported.rst b/docs/tuning-apps/fault-tolerance/supported.rst
new file mode 100644
index 00000000000..90c393eddfa
--- /dev/null
+++ b/docs/tuning-apps/fault-tolerance/supported.rst
@@ -0,0 +1,45 @@
+Supported fault tolerance techniques
+====================================
+
+Open MPI is a vehicle for research in fault tolerance and over the years
+provided support for a wide range of resilience techniques:
+
+* Currently supported
+  
+    * User Level Fault Mitigation techniques similar to those defined
+      in the context of the MPI Forum (this is the closest match when
+      migrating from FT-MPI); :ref:`see its documentation section
+      <ulfm-label>`.
+
+* Only for research / non-production usage
+      
+    * Message logging techniques. Similar to those implemented in
+      MPICH-V.
+
+* Deprecated / no longer available
+  
+    * Coordinated and uncoordinated process checkpoint and
+      restart. Similar to those implemented in LAM/MPI and MPICH-V,
+      respectively.
+    * Data Reliability and network fault tolerance. Similar to those
+      implemented in LA-MPI.
+
+Current fault tolerance development
+-----------------------------------
+
+The only active work in resilience in Open MPI targets the User Level Fault
+Mitigation (ULFM) approach, a technique discussed in the context of the MPI
+standardization body.
+
+For information on the Fault Tolerant MPI prototype in Open MPI see the
+links below:
+
+*  :ref:`Open MPI's ULFM documentation section <ulfm-label>`
+* `MPI Forum's Fault Tolerance Working Group <https://github.com/mpiwg-ft/ft-issues/wiki>`_
+* `Information, examples, and support <https://fault-tolerance.org/>`_
+
+Support for other types of resilience (e.g., :ref:`data reliability <ft-data-reliability-label>`,
+:ref:`checkpoint <ft-checkpoint-restart-label>`) has been deprecated over the
+years due to lack of adoption and lack of maintenance. If you are interested
+in doing some archeological work, traces are still available on the main
+repository.
diff --git a/docs/tuning-apps/fork-system-popen.rst b/docs/tuning-apps/fork-system-popen.rst
new file mode 100644
index 00000000000..6e2c0c00cf7
--- /dev/null
+++ b/docs/tuning-apps/fork-system-popen.rst
@@ -0,0 +1,16 @@
+Calling fork(), system(), or popen() in MPI processes
+=====================================================
+
+It may be possible to call ``fork()``, ``system()``, ``popen()``, etc. calls
+from an MPI process, but it depends on a lot of factors, including (but not
+limited to):
+
+* The operating system
+* The underlying compute hardware
+* The network stack
+* Interactions with other middleware in the MPI process
+
+Users are encouraged to avoid invoking these operations in their MPI applications
+as they may cause issues with Open MPI's operation or the operation of underlying
+components, such as `UCX <https://openucx.org/>`_
+and `Libfabric <https://libfabric.org/>`_.
diff --git a/docs/tuning-apps/heterogeneity.rst b/docs/tuning-apps/heterogeneity.rst
new file mode 100644
index 00000000000..1eadfb68067
--- /dev/null
+++ b/docs/tuning-apps/heterogeneity.rst
@@ -0,0 +1,21 @@
+.. _label-heterogeneous-mpi-apps:
+
+Building heterogeneous MPI applications
+=======================================
+
+Heterogeneous support (specifically: supporting different sized and/or
+represented data types in a single MPI application run) within a
+single MPI job is technically required by the MPI standard.
+
+However, there are both theoretical and practical problems with
+supporting true data heterogeneity at run-time.
+
+Indeed, it is quite uncommon for production HPC environments to be
+data-heterogeneous (e.g., natively support little-endian on some nodes
+and big-endian on other nodes in the same MPI application job).
+
+As such, supporting data heterogeneity is a feature that has fallen
+into disrepair and is currently known to be broken in this release of
+Open MPI.
+
+:doc:`Contributions to fix it would be welcome! </contributing>`
diff --git a/docs/tuning-apps/index.rst b/docs/tuning-apps/index.rst
new file mode 100644
index 00000000000..debc86a0e5e
--- /dev/null
+++ b/docs/tuning-apps/index.rst
@@ -0,0 +1,21 @@
+Run-time operation and tuning MPI applications
+==============================================
+
+Once your MPI application is running, there are many different
+components that can be tuned to affect behavior at run time.
+
+.. toctree::
+   :maxdepth: 1
+
+   environment-var
+   networking/index
+   multithreaded
+   dynamic-loading
+   fork-system-popen
+   fault-tolerance/index
+   large-clusters/index
+   affinity
+   mpi-io
+   coll-tuned
+   benchmarking
+   heterogeneity
diff --git a/docs/tuning-apps/large-clusters/index.rst b/docs/tuning-apps/large-clusters/index.rst
new file mode 100644
index 00000000000..ff58b2b7e13
--- /dev/null
+++ b/docs/tuning-apps/large-clusters/index.rst
@@ -0,0 +1,14 @@
+Large Clusters
+==============
+
+Setting up a large cluster to run MPI applications can be challenging and
+full of many decisions.  The following sections include guidance for
+installing and tuning Open MPI on large clusters.
+
+.. toctree::
+   :maxdepth: 1
+
+   reduce-startup-time
+   libraries
+   reduce-wireup-time
+   static-cluster-config
diff --git a/docs/tuning-apps/large-clusters/libraries.rst b/docs/tuning-apps/large-clusters/libraries.rst
new file mode 100644
index 00000000000..8673c45de14
--- /dev/null
+++ b/docs/tuning-apps/large-clusters/libraries.rst
@@ -0,0 +1,56 @@
+Libraries
+=========
+
+Location: network vs. local filesystems
+---------------------------------------
+
+Open MPI itself doesn't really care where its libraries and plugins
+are stored.  However, where they are stored does have an impact on
+startup times, particularly for large clusters, which can be mitigated
+somewhat through use of Open MPI's configuration options.
+
+Startup times will always be minimized by storing the libraries and
+plugins local to each node, either on local disk or in ramdisk. The
+latter is sometimes problematic since the libraries do consume some
+space, thus potentially reducing memory that would have been available
+for MPI processes.
+
+There are two main considerations for large clusters that need to
+place the Open MPI libraries on networked file systems:
+
+* While dynamic shared objects ("DSO") are more flexible, you
+  definitely do *not* want to use them when the Open MPI libraries
+  will be mounted on a network file system that is shared to a large
+  cluster!  Doing so will lead to significant network traffic and
+  delayed start times, especially on clusters with a large number of
+  nodes.  Instead, be sure to :ref:`configure your build
+  <building-ompi-cli-options-diable-dlopen-label>` with
+  ``--disable-dlopen``.  This will include the DSO's in the main
+  libraries, resulting in much faster startup times.
+
+  .. note:: As of the Open MPI v5.0.x series, ``--disable-dlopen`` is
+            now the default.
+
+* Many networked file systems use automount for user level
+  directories, as well as for some locally administered system
+  directories. There are many reasons why system administrators may
+  choose to automount such directories. MPI jobs, however, tend to
+  launch very quickly, thereby creating a situation wherein a large
+  number of nodes will nearly simultaneously demand automount of a
+  specific directory. This can overload NFS servers, resulting in
+  delayed response or even failed automount requests.
+
+  Note that this applies to both automount of directories containing
+  Open MPI libraries as well as directories containing user
+  applications. Since these are unlikely to be the same location,
+  multiple automount requests from each node are possible, thus
+  increasing the level of traffic.
+
+Static vs. shared
+-----------------
+
+It is perfectly fine to use either shared or static
+libraries. Shared libraries will save memory when operating multiple
+processes per node, especially on clusters with high numbers of cores
+on a node, but can also take longer to launch on networked file
+systems.
diff --git a/docs/tuning-apps/large-clusters/reduce-startup-time.rst b/docs/tuning-apps/large-clusters/reduce-startup-time.rst
new file mode 100644
index 00000000000..01751330bf6
--- /dev/null
+++ b/docs/tuning-apps/large-clusters/reduce-startup-time.rst
@@ -0,0 +1,58 @@
+Reducing startup time for jobs
+==============================
+
+.. error:: TODO This whole section needs to be checked.
+
+There are several ways to reduce the startup time on large
+clusters. Some of them are described on this page. We continue to work
+on making startup even faster, especially on the large clusters coming
+in future years.
+
+Open MPI |ompi_ver| is significantly faster and more robust than its
+predecessors. We recommend that anyone running large jobs and/or on
+large clusters make the upgrade to the |ompi_series| series.
+
+Several major launch time enhancements have been made starting with the
+v3.0 release. Most of these take place in the background |mdash| i.e., there
+is nothing you (as a user) need do to take advantage of them. However,
+there are a few that are left as options until we can assess any potential
+negative impacts on different applications.
+
+Some options are available when launching via ``mpirun`` or when launching using
+the native resource manager launcher (e.g., ``srun`` in a Slurm environment).
+These are activated by setting the corresponding MCA parameter, and include:
+
+* Setting the ``pmix_base_async_modex`` MCA parameter will eliminate a
+  global out-of-band collective operation during ``MPI_INIT``. This
+  operation is performed in order to share endpoint information prior
+  to communication. At scale, this operation can take some time and
+  scales at best logarithmically. Setting the parameter bypasses the
+  operation and causes the system to lookup the endpoint information
+  for a peer only at first message. Thus, instead of collecting
+  endpoint information for all processes, only the endpoint
+  information for those processes this peer communicates with will be
+  retrieved. The parameter is especially effective for applications
+  with sparse communication patterns |mdash| i.e., where a process
+  only communicates with a few other peers. Applications that use
+  dense communication patterns (i.e., where a peer communicates
+  directly to all other peers in the job) will probably see a negative
+  impact of this option.
+
+  .. note:: This option is only available in PMIx-supporting
+            environments, or when launching via ``mpirun``
+
+* The ``async_mpi_init`` parameter is automatically set to ``true``
+  when the ``pmix_base_async_modex`` parameter has been set, but can
+  also be independently controlled. When set to ``true``, this parameter
+  causes ``MPI_Init`` to skip an out-of-band barrier operation at the end
+  of the procedure that is not required whenever direct retrieval of
+  endpoint information is being used.
+
+* Similarly, the ``async_mpi_finalize`` parameter skips an out-of-band
+  barrier operation usually performed at the beginning of
+  ``MPI_FINALIZE``. Some transports (e.g., the ``usnic`` BTL) require this
+  barrier to ensure that all MPI messages are completed prior to
+  finalizing, while other transports handle this internally and thus
+  do not require the additional barrier. Check with your transport
+  provider to be sure, or you can experiment to determine the proper
+  setting.
diff --git a/docs/tuning-apps/large-clusters/reduce-wireup-time.rst b/docs/tuning-apps/large-clusters/reduce-wireup-time.rst
new file mode 100644
index 00000000000..349f0500a77
--- /dev/null
+++ b/docs/tuning-apps/large-clusters/reduce-wireup-time.rst
@@ -0,0 +1,33 @@
+Reducing wireup time
+====================
+
+Open MPI's run-time uses an *out-of-band* (OOB) communication
+subsystem to pass messages during the launch, initialization, and
+termination stages for the job. These messages allow ``mpirun`` to tell
+its daemons what processes to launch, and allow the daemons in turn to
+forward stdio to ``mpirun``, update ``mpirun`` on process status, etc.
+
+The OOB uses TCP sockets for its communication, with each daemon
+opening a socket back to ``mpirun`` upon startup. In a large cluster,
+this can mean thousands of connections being formed on the node where
+``mpirun`` resides, and requires that ``mpirun`` actually process all
+these connection requests. ``mpirun`` defaults to processing
+connection requests sequentially |mdash| so on large clusters, a
+backlog can be created that can cause remote daemons to timeout
+waiting for a response.
+
+Fortunately, Open MPI provides an alternative mechanism for processing
+connection requests that helps alleviate this problem. Setting the MCA
+parameter ``oob_tcp_listen_mode`` to ``listen_thread`` causes
+``mpirun`` to startup a separate thread dedicated to responding to
+connection requests. Thus, remote daemons receive a quick response to
+their connection request, allowing ``mpirun`` to deal with the message
+as soon as possible.
+
+.. error:: TODO This seems very out of date.  We should have content
+           about PMIx instant on.
+
+This parameter can be included in the default MCA parameter file,
+placed in the user's environment, or added to the ``mpirun`` command
+line.  See :ref:`this FAQ entry <label-running-setting-mca-param-values>`
+for more details on how to set MCA parameters.
diff --git a/docs/tuning-apps/large-clusters/static-cluster-config.rst b/docs/tuning-apps/large-clusters/static-cluster-config.rst
new file mode 100644
index 00000000000..b40cdc7bffe
--- /dev/null
+++ b/docs/tuning-apps/large-clusters/static-cluster-config.rst
@@ -0,0 +1,43 @@
+Static cluster configurations
+=============================
+
+.. error:: This entire section needs to be checked.
+
+Clusters rarely change from day-to-day, and large clusters rarely
+change at all.  If you know your cluster's configuration, there are
+several steps you can take to both reduce Open MPI's memory footprint
+and reduce the launch time of large-scale applications.  These steps
+use a combination of build-time configuration options to eliminate
+components |mdash| thus eliminating their libraries and avoiding
+unnecessary component open/close operations |mdash| as well as
+run-time MCA parameters to specify what modules to use by default for
+most users.
+
+One way to save memory is to avoid building components that will
+actually never be selected by the system. Unless MCA parameters
+specify which components to open, built components are always opened
+and tested as to whether or not they should be selected for use. If
+you know that a component can build on your system, but due to your
+cluster's configuration will never actually be selected, then it is
+best to simply configure OMPI to not build that component by using the
+``--enable-mca-no-build`` configure option.
+
+For example, if you know that your system will only utilize the
+``ob1`` component of the PML framework, then you can ``no_build`` all
+the others. This not only reduces memory in the libraries, but also
+reduces memory footprint that is consumed by Open MPI opening all the
+built components to see which of them can be selected to run.
+
+In some cases, however, a user may optionally choose to use a
+component other than the default.  For example, you may want to build
+all of the PRRTE ``routed`` framework components, even though the vast
+majority of users will simply use the default ``debruijn``
+component.  This means you have to allow the system to build the other
+components, even though they may rarely be used.
+
+You can still save launch time and memory, though, by setting the
+``routed=debruijn`` MCA parameter in the default MCA parameter file.
+This causes OMPI to not open the other components during startup, but
+allows users to override this on their command line or in their
+environment so no functionality is lost |mdash| you just save some
+memory and time.
diff --git a/docs/tuning-apps/mpi-io.rst b/docs/tuning-apps/mpi-io.rst
new file mode 100644
index 00000000000..ddb84d62874
--- /dev/null
+++ b/docs/tuning-apps/mpi-io.rst
@@ -0,0 +1,256 @@
+Open MPI IO ("OMPIO")
+=====================
+
+OMPIO is an Open MPI-native implementation of the MPI I/O functions
+defined in the MPI specification.
+
+The main goals of OMPIO are:
+
+#. Increase the modularity of the parallel I/O library by separating
+   MPI I/O functionality into sub-frameworks.
+
+#. Allow frameworks to utilize different run-time decision algorithms
+   to determine which module to use in a particular scenario, enabling
+   non-file-system-specific decisions.
+
+#. Improve the integration of parallel I/O functions with other
+   components of Open MPI, most notably the derived data type engine
+   and the progress engine. The integration with the derived data type
+   engine allows for faster decoding of derived data types and the
+   usage of optimized data type to data type copy operations.
+
+OMPIO is fundamentally a component of the ``io`` framework in Open
+MPI.  Upon opening a file, the OMPIO component initializes a number of
+sub-frameworks and their components, namely:
+
+*  ``fs``: responsible for all file management operations
+* ``fbtl``: support for blocking and non-blocking individual
+  I/O operations
+* ``fcoll``: support for blocking and non-blocking collective I/O
+  operations
+* ``sharedfp``: support for all shared file pointer operations.
+
+
+MCA parameters of OMPIO and associated frameworks
+-------------------------------------------------
+
+The :ref:`ompi_info(1) <man1-ompi_info>` command can display all the
+parameters available for the OMPIO ``io``, ``fcoll``, ``fs``,
+``fbtl``, and ``sharedfp`` components:
+
+.. code-block:: sh
+
+   shell$ ompi_info --param io       ompio --level 9
+   shell$ ompi_info --param fcoll    all --level 9
+   shell$ ompi_info --param fs       all --level 9
+   shell$ ompi_info --param fbtl     all --level 9
+   shell$ ompi_info --param sharedfp all --level 9
+
+OMPIO sub-framework components
+------------------------------
+
+The OMPIO architecture is designed around sub-frameworks, which allow
+you to develop a relatively small amount of code optimized for a
+particular environment, application, or infrastructure.  Although
+significant efforts have been invested into making good decisions for
+default values and switching points between components, users and/or
+system administrators might occasionally want to tune the selection
+logic of the components and force the utilization of a particular
+component.
+
+The simplest way to force the usage of a component is to simply
+restrict the list of available components for that framework. For
+example, an application wanting to use the ``dynamic`` ``fcoll``
+component simply has to pass the name of the component as a value to
+the corresponding MCA parameter during ``mpirun`` or any other
+mechanism available in Open MPI to influence a parameter value, e.g.:
+
+.. code-block:: sh
+
+   shell$ mpirun --mca fcoll dynamic -n 64 ./a.out
+
+``fs`` and ``fbtl`` components are typically chosen based on the file
+system type utilized (e.g. the ``pvfs2`` component is chosen when the
+file is located on an PVFS2/OrangeFS file system, the ``lustre``
+component is chosen for Lustre file systems, etc.). The ``ufs`` ``fs``
+component is used if no file system specific component is availabe
+(e.g. local file systems, NFS, BeefFS, etc.), and the ``posix``
+``fbtl`` component is used as the default component for read/write
+operations.
+
+The ``fcoll`` framework provides several different components. The
+current decision logic in OMPIO uses the file view provided by the
+application as well as file system level characteristics (e.g. file
+system, stripe width) to determine which component to use. The most
+important ``fcoll`` components are:
+
+* ``dynamic_gen2``: the default component used on lustre file
+  system. This component is based on the two-phase I/O algorithm with
+  a static file partioning strategy, i.e. an aggregator processes will
+  by default only write data to a single storage server.
+
+* ``vulcan``: the default component used on all other file
+  systems. This component is based on the two-phase I/O algorithm with
+  an even file partitioning strategy, i.e. each of the n aggregators
+  will write 1/n th of the overall file.
+
+* ``individual``: this components executes all collective I/O
+  operations in terms of individual I/O operations.
+
+The ``sharedfp`` framework provides a different implementation of the
+shared file pointer operations depending on file system features.
+
+* ``lockedfile``: this component will be used on file systems which
+  support file locking.
+
+* ``sm``: component used in scenarios in which all processes of the
+  communicator are on the same physical node.
+
+* ``individual``: a component that can be used if neither of the other
+  two components are available. This component provides however only
+  limited functionality (i.e. write operations only).
+
+  .. note:: See :ref:`the section on the individual sharedfp component
+            <label-ompio-individual-sharedfp>` to understand
+            functionality and limitations.
+
+Tuning OMPIO performance
+------------------------
+
+The most important parameters influencing the performance of an I/O
+operation are listed below:
+
+#. ``io_ompio_cycle_buffer_size``: Data size issued by individual
+   reads/writes per call. By default, an individual read/write
+   operation will be executed as one chunk. Splitting the operation up
+   into multiple, smaller chunks can lead to performance improvements
+   in certain scenarios.
+
+#. ``io_ompio_bytes_per_agg``: Size of temporary buffer for collective
+   I/O operations on aggregator processes. Default value is 32MB.
+   Tuning this parameter has a very high impact on the performance of
+   collective operations.
+
+#. ``io_ompio_num_aggregators``: Number of aggregators used in
+   collective I/O operations.  Setting this parameter to a value
+   larger zero disables the internal automatic aggregator selection
+   logic of OMPIO.  Tuning this parameter has a very high impact on
+   the performance of collective operations.
+
+#. ``io_ompio_grouping_option``: Algorithm used to automatically
+   decide the number of aggregators used. Applications working with
+   regular 2-D or 3-D data decomposition can try changing this
+   parameter to 4 (hybrid) algorithm.
+
+#. ``fs_ufs_lock_algorithm``: Parameter used to determing what part of
+   a file needs to be locked for a file operation. Since the ``ufs``
+   ``fs`` component is used on multiple file systems, OMPIO
+   automatically chooses the value required for correctness on all
+   file systems, e.g. enforcing locking on an NFS file system, while
+   disabling locking on a local file system. Users can adjust the
+   required locking behavior based on their use case, since the
+   default value might often be too restrictive for their application.
+
+Setting stripe size and stripe width on parallel file systems
+-------------------------------------------------------------
+
+Many ``fs`` components allow you to manipulate the layout of a new
+file on a parallel file system.  Note, that many file systems only
+allow changing these setting upon file creation, i.e. modifying these
+values for an already existing file might not be possible.
+
+#. ``fs_pvfs2_stripe_size``: Sets the number of storage servers for a
+   new file on a PVFS2/OrangeFS  file system. If not set, system default will be
+   used. Note that this parameter can also be set through the
+   ``stripe_size`` MPI Info value.
+
+#. ``fs_pvfs2_stripe_width``: Sets the size of an individual block for
+   a new file on a PVFS2 file system. If not set, system default will
+   be used. Note that this parameter can also be set through the
+   ``stripe_width`` MPI Info value.
+
+#. ``fs_lustre_stripe_size``: Sets the number of storage servers for a
+   new file on a Lustre file system. If not set, system default will
+   be used. Note that this parameter can also be set through the
+   ``stripe_size`` MPI Info value.
+
+#. ``fs_lustre_stripe_width``: Sets the size of an individual block
+   for a new file on a Lustre file system. If not set, system default
+   will be used. Note that this parameter can also be set through the
+   ``stripe_width`` MPI Info value.
+
+Using GPU device buffers in MPI File I/O operations
+----------------------------------------------------
+
+OMPIO supports reading and writing directly to/from GPU buffers using
+the MPI I/O interfaces. Using this feature simplifies managing buffers
+that are exclusively used on GPU devices, and hence there is no need to
+implement staging through host memory for file I/O operations.
+
+Internally, OMPIO splits a user buffer into chunks for performing the
+read/write operation. The chunk-size used by OMPIO can have a
+significant influence on the performance of the file I/O operation
+from device buffers, and can be controlled using the
+``io_ompio_pipeline_buffer_size`` MCA parameter.
+
+.. _label-ompio-individual-sharedfp:
+
+Using the ``individual`` ``sharedfp`` component and its limitations
+-------------------------------------------------------------------
+
+The ``individual`` sharedfp component provides an approximation of
+shared file pointer operations that can be used for *write operations
+only*. It is only recommended in scenarios, where neither the ``sm``
+nor the ``lockedfile`` component can be used, e.g., due to the fact
+that more than one node are being used and the file system does not
+support locking.
+
+Conceptually, each process writes the data of a write_shared operation
+into a separate file along with a time stamp. In every collective
+operation (or during the file_close operation), data from all
+individual files are merged into the actual output file, using the
+time stamps as the main criteria.
+
+The component has certain limitations and restrictions, such as its
+relience on the synchronization clocks on the individual cluster nodes
+to determine the order between entries in the final file, which might
+lead to some deviations compared to the actual calling sequence.
+
+Furthermore, the component only supports ``write`` operations, read
+operations are not supported.
+
+Other features of OMPIO
+-----------------------
+
+OMPIO has a number of additional features, mostly directed towards
+developers, which could occasionally also be useful to interested
+end-users. These can typically be controlled through MCA parameters.
+
+* ``io_ompio_coll_timing_info``: Setting this parameter will lead to a
+  short report upon closing a file indicating the amount of time spent
+  in communication and I/O operations of collective I/O operations
+  only.
+
+* ``io_ompio_record_file_offset_info``: Setting this parameter will
+  report neighborhood relationship of processes based on the file view
+  used. This is occasionally important for understanding performance
+  characteristics of I/O operations.  Note, that using this features
+  requires an additional compile time flag when compiling ``ompio``.
+
+  The output file generated as a result of this flag provides the
+  access pattern of processes to the file recorded as neighborhood
+  relationships of processes as a matrix. For example, if the first
+  four bytes of a file are being accessed by process 0 and the next
+  four bytes by process 1, processes 0 and 1 are said to have a
+  neighborhood relationship since they access neighboring elements of
+  the file.  For each neighborhood relation detected in the file, the
+  value for the corresponding pair of processes is increased by one.
+
+  Data is provided in compressed row storage format. To minimize the
+  amount of data written using this feature, only non-zero values are
+  output.  The first row in the output file indicates the number of
+  non-zero elements in the matrix; the second number is the number of
+  elements in the row index.  The third row of the output file gives
+  all the column indexes. The fourth row lists all the values and the
+  fifth row gives the row index. A row index represents the position
+  in the value array where a new row starts.
diff --git a/docs/tuning-apps/multithreaded.rst b/docs/tuning-apps/multithreaded.rst
new file mode 100644
index 00000000000..00439c6a358
--- /dev/null
+++ b/docs/tuning-apps/multithreaded.rst
@@ -0,0 +1,15 @@
+.. _label-running-multithreaded-apps:
+
+Running multi-threaded MPI applications
+=======================================
+
+Support for ``MPI_THREAD_MULTIPLE`` (i.e., multiple threads
+executing within the MPI library) and asynchronous message passing
+progress (i.e., continuing message passing operations even while no
+user threads are in the MPI library) has been designed into Open MPI
+from its first planning meetings.
+
+``MPI_THREAD_MULTIPLE`` was included in the first version of
+Open MPI, but it only became robust around v3.0.0.  Subsequent
+releases continually improve reliability and performance of
+multi-threaded MPI applications.
diff --git a/docs/tuning-apps/networking/cuda.rst b/docs/tuning-apps/networking/cuda.rst
new file mode 100644
index 00000000000..b133258175e
--- /dev/null
+++ b/docs/tuning-apps/networking/cuda.rst
@@ -0,0 +1,725 @@
+CUDA
+====
+
+.. error:: TODO This section needs to be converted from FAQ Q&A style
+           to regular documentation style.
+
+How do I build Open MPI with CUDA-aware support?
+------------------------------------------------
+
+CUDA-aware support means that the MPI library can send and receive GPU
+buffers directly.  CUDA support is being continuously updated so
+different levels of support exist in different versions.  We recommend
+you use the latest version of Open MPI for best support.
+
+Open MPI offers two flavors of CUDA support:
+
+#. Via `UCX <https://openucx.org/>`_.
+
+   This is the preferred mechanism.  Since UCX will be providing the
+   CUDA support, it is important to ensure that UCX itself is built
+   with CUDA support.
+
+   To see if your ucx was built with CUDA support run the following
+   command:
+
+   .. code-block:: sh
+
+      # Check if ucx was built with CUDA support
+      shell$ ucx_info -v
+
+      # configured with: --build=powerpc64le-redhat-linux-gnu --host=powerpc64le-redhat-linux-gnu --program-prefix= --disable-dependency-tracking --prefix=/usr --exec-prefix=/usr --bindir=/usr/bin --sbindir=/usr/sbin --sysconfdir=/etc --datadir=/usr/share --includedir=/usr/include --libdir=/usr/lib64 --libexecdir=/usr/libexec --localstatedir=/var --sharedstatedir=/var/lib --mandir=/usr/share/man --infodir=/usr/share/info --disable-optimizations --disable-logging --disable-debug --disable-assertions --enable-mt --disable-params-check --enable-cma --without-cuda --without-gdrcopy --with-verbs --with-cm --with-knem --with-rdmacm --without-rocm --without-xpmem --without-java
+
+   If you need to build ucx yourself to include CUDA support, please
+   see the UCX documentation for `building ucx with Open MPI: <https://openucx.readthedocs.io/en/master/running.html#openmpi-with-ucx>`_
+
+   It should look something like:
+
+   .. code-block:: sh
+
+      # Configure UCX this way
+      shell$ ./configure --prefix=/path/to/ucx-cuda-install --with-cuda=/usr/local/cuda --with-gdrcopy=/usr
+
+      # Configure Open MPI this way
+      shell$ ./configure --with-cuda=/usr/local/cuda --with-ucx=/path/to/ucx-cuda-install <other configure params>
+
+#. Via internal Open MPI CUDA support
+
+Regardless of which flavor of CUDA support (or both) you plan to use,
+Open MPI should be configured using the ``--with-cuda=<path-to-cuda>``
+configure option to build CUDA support into Open MPI. The configure
+script will automatically search the path given for ``libcuda.so``. If it cannot
+be found, please also pass ``--with-cuda-libdir``. For example:
+``--with-cuda=<path-to-cuda> --with-cuda-libdir=/usr/local/cuda/lib64/stubs``.
+
+Open MPI supports building with CUDA libraries and running on systems
+without CUDA libraries or hardware.
+
+For releases v5.0.2 and newer no special steps are required to get this behavior.
+
+In order to realize this behavior for the v5.0.0 and v5.0.1 releases,
+when configuring Open MPI, you have to specify the CUDA dependent components to be built as DSOs using the
+``--enable-mca-dso=<comma-delimited-list-of-cuda-components.``
+configure option.
+
+This affects the ``smcuda`` shared memory and ``uct`` BTLs, as well
+as the ``rgpusm`` and ``gpusm`` rcache components.
+
+An example configure command would look like the following:
+
+   .. code-block:: sh
+
+      # Configure Open MPI this way
+      shell$ ./configure --with-cuda=/usr/local/cuda \
+             --enable-mca-dso=btl-smcuda,rcache-rgpusm,rcache-gpusm,accelerator-cuda <other configure params>
+
+/////////////////////////////////////////////////////////////////////////
+
+How do I verify that Open MPI has been built with CUDA support?
+---------------------------------------------------------------
+
+Verify that Open MPI has been built with cuda using ``ompi_info``
+with one of the following commands.
+
+.. code-block:: sh
+
+   # Use ompi_info to verify cuda support in Open MPI
+   shell$ ompi_info | grep "MPI extensions"
+          MPI extensions: affinity, cuda, pcollreq
+   shell$ ompi_info --parsable --all | grep mpi_built_with_cuda_support:value
+          mca:mpi:base:param:mpi_built_with_cuda_support:value:true
+
+/////////////////////////////////////////////////////////////////////////
+
+How do I run Open MPI with applications that pass CUDA buffers to MPI?
+----------------------------------------------------------------------
+
+Open MPI will detect and enable CUDA enabled components at runtime with
+no additional mpirun parameters.
+
+/////////////////////////////////////////////////////////////////////////
+
+How do I build Open MPI with CUDA-aware support using PGI?
+----------------------------------------------------------
+
+With CUDA 6.5, you can build all versions of CUDA-aware Open MPI
+without doing anything special.  However, with CUDA 7.0 and CUDA 7.5,
+you need to pass in some specific compiler flags for things to work
+correctly.  Add the following to your configure line.
+
+.. code-block:: sh
+
+   # For PGI 15.9 and later (Also called NVCC):
+   shell$ ./configure --with-wrapper-cflags=-ta:tesla
+
+   # For earlier versions of PGI:
+   shell$ ./configure CFLAGS=-D__LP64__ --with-wrapper-cflags="-D__LP64__ -ta:tesla"
+
+/////////////////////////////////////////////////////////////////////////
+
+What kind of CUDA support exists in Open MPI?
+---------------------------------------------
+
+CUDA-aware support is defined as Open MPI automatically detecting that
+the argument pointer being passed to an MPI routine is a CUDA device
+memory pointer.
+
+See :ref:`this FAQ entry <faq-cuda-mpi-apis-cuda-label>`
+for more details on which APIs are CUDA-aware.
+
+
+.. error:: CUDA 4.0 is SUPER OLD!  End users dont care about the
+   differences between cuda-aware, cuda-ipc, gpu-direct, and gpu-direct-rdma
+
+Open MPI depends on various features of CUDA 4.0, so one needs to have
+at least the CUDA 4.0 driver and toolkit.  The new features of
+interest are the Unified Virtual Addressing (UVA) so that all pointers
+within a program have unique addresses.  In addition, there is a new
+API that allows one to determine if a pointer is a CUDA device pointer
+or host memory pointer.  This API is used by the library to decide
+what needs to be done with each buffer.  In addition, CUDA 4.1 also
+provides the ability to register host memory with the CUDA driver,
+which can improve performance.  CUDA 4.1 also added CUDA IPC support
+for fast communication between GPUs on the same node.
+
+Note that derived datatypes |mdash| both contiguous and non-contiguous
+|mdash| are supported.  However, the non-contiguous datatypes
+currently have high overhead because of the many calls to the CUDA
+function ``cuMemcpy()`` to copy all the pieces of the buffer into the
+intermediate buffer.
+
+CUDA-aware support is available in:
+
+* The UCX (``ucx``) PML
+* The PSM2 (``psm2``) MTL with the CM (``cm``) PML.
+* The OFI (``ofi``) MTL with the CM (``cm``) PML.
+* Both CUDA-ized shared memory (``smcuda``) and TCP (``tcp``) BTLs
+  with the OB1 (``ob1``) PML.
+* The HCOLL (``hcoll``) COLL
+
+/////////////////////////////////////////////////////////////////////////
+
+PSM2 support for CUDA
+---------------------
+
+CUDA-aware support is present in PSM2 MTL.  When running CUDA-aware
+Open MPI on Cornelis Networks Omni-Path, the PSM2 MTL will automatically set
+``PSM2_CUDA`` environment variable which enables PSM2 to handle GPU
+buffers.  If the user wants to use host buffers with a CUDA-aware Open
+MPI, it is recommended to set ``PSM2_CUDA`` to ``0`` in the execution
+environment. PSM2 also has support for the NVIDIA GPUDirect support
+feature. To enable this, users will need to set ``PSM2_GPUDIRECT``
+to ``1`` in the execution environment.
+
+Note: The PSM2 library and ``hfi1`` driver with CUDA support are requirements
+to use GPUDirect support on Cornelis Networks Omni-Path. The minimum
+PSM2 build version required is `PSM2 10.2.175
+<https://github.com/01org/opa-psm2/releases/tag/PSM2_10.2-175>`_.
+
+For more information refer to the `Cornelis Networks Customer Center
+<https://customercenter.cornelisnetworks.com/>`_.
+
+/////////////////////////////////////////////////////////////////////////
+
+OFI support for CUDA
+---------------------
+
+CUDA-aware support is present in OFI MTL.  When running CUDA-aware
+Open MPI over Libfabric, the OFI MTL will check if there are any
+providers capable of handling GPU (or other accelerator) memory
+through the ``hmem``-related flags. If a CUDA-capable provider is
+available, the OFI MTL will directly send GPU buffers through
+Libfabric's API after registering the memory. If there are no
+CUDA-capable providers available, the buffers will automatically
+be copied to host buffers before being transferred through
+Libfabric's API.
+
+/////////////////////////////////////////////////////////////////////////
+
+Can I get additional CUDA debug-level information at run-time?
+--------------------------------------------------------------
+
+Yes, by enabling some vebosity flags.
+
+* The ``opal_cuda_verbose`` parameter has only one level of verbosity:
+
+  .. code-block::
+
+     shell$ mpirun --mca opal_cuda_verbose 10 ...
+
+
+* The ``mpi_common_cuda_verbose`` parameter provides additional
+  information about CUDA-aware related activities.  This can be set to
+  a variety of different values.  There is really no need to use these
+  unless you have strange problems:
+
+  .. code-block:: sh
+
+     # A bunch of CUDA debug information
+     shell$ mpirun --mca mpi_common_cuda_verbose 10 ...
+     # Even more CUDA debug information
+     shell$ mpirun --mca mpi_common_cuda_verbose 20 ...
+     # Yet more CUDA debug information
+     shell$ mpirun --mca mpi_common_cuda_verbose 100 ...
+
+* The ``smcuda`` BTL has three MCA parameters related to the use of
+  CUDA IPC.  By default, CUDA IPC is used where possible.  But the
+  user can now turn it off if they prefer.
+
+  .. code-block:: sh
+
+     shell$ mpirun --mca btl_smcuda_use_cuda_ipc 0 ...
+
+  In addition, it is assumed that CUDA IPC is possible when running on
+  the same GPU, and this is typically true.  However, there is the
+  ability to turn it off.
+
+  .. code-block:: sh
+
+     shell$ mpirun --mca btl_smcuda_use_cuda_ipc_same_gpu 0 ...
+
+  Last, to get some insight into whether CUDA IPC is being used, you
+  can turn on some verbosity that shows whether CUDA IPC gets enabled
+  between two GPUs.
+
+  .. code-block:: sh
+
+     shell$ mpirun --mca btl_smcuda_cuda_ipc_verbose 100 ...
+
+/////////////////////////////////////////////////////////////////////////
+
+.. _faq-cuda-mpi-cuda-numa-issues-label:
+
+NUMA Node Issues
+----------------
+
+When running on a node that has multiple GPUs, you may want to select
+the GPU that is closest to the NUMA node on which your process is
+running.  One way to do this is to make use of the ``hwloc`` library.
+The following is a C code snippet that can be used in your application
+to select a GPU that is close.  It will determine on which CPU it is
+running and then look for the closest GPU.  There could be multiple
+GPUs that are the same distance away.  This is dependent on having
+``hwloc`` somewhere on your system.
+
+.. code-block:: c
+
+   /**
+    * Test program to show the use of hwloc to select the GPU closest to the CPU
+    * that the MPI program is running on.  Note that this works even without
+    * any libpciaccess or libpci support as it keys off the NVIDIA vendor ID.
+    * There may be other ways to implement this but this is one way.
+    * January 10, 2014
+    */
+   #include <assert.h>
+   #include <stdio.h>
+   #include "cuda.h"
+   #include "mpi.h"
+   #include "hwloc.h"
+
+   #define ABORT_ON_ERROR(func) \
+     { CUresult res; \
+       res = func; \
+       if (CUDA_SUCCESS != res) { \
+           printf("%s returned error=%d\n", #func, res); \
+           abort(); \
+       } \
+     }
+   static hwloc_topology_t topology = NULL;
+   static int gpuIndex = 0;
+   static hwloc_obj_t gpus[16] = {0};
+
+   /**
+    * This function searches for all the GPUs that are hanging off a NUMA
+    * node.  It walks through each of the PCI devices and looks for ones
+    * with the NVIDIA vendor ID.  It then stores them into an array.
+    * Note that there can be more than one GPU on the NUMA node.
+    */
+   static void find_gpus(hwloc_topology_t topology, hwloc_obj_t parent, hwloc_obj_t child) {
+       hwloc_obj_t pcidev;
+       pcidev = hwloc_get_next_child(topology, parent, child);
+       if (NULL == pcidev) {
+           return;
+       } else if (0 != pcidev->arity) {
+           /* This device has children so need to look recursively at them */
+           find_gpus(topology, pcidev, NULL);
+           find_gpus(topology, parent, pcidev);
+       } else {
+           if (pcidev->attr->pcidev.vendor_id == 0x10de) {
+               gpus[gpuIndex++] = pcidev;
+           }
+           find_gpus(topology, parent, pcidev);
+       }
+   }
+
+   int main(int argc, char *argv[])
+   {
+       int rank, retval, length;
+       char procname[MPI_MAX_PROCESSOR_NAME+1];
+       const unsigned long flags = HWLOC_TOPOLOGY_FLAG_IO_DEVICES | HWLOC_TOPOLOGY_FLAG_IO_BRIDGES;
+       hwloc_cpuset_t newset;
+       hwloc_obj_t node, bridge;
+       char pciBusId[16];
+       CUdevice dev;
+       char devName[256];
+
+       MPI_Init(&argc, &argv);
+       MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+       if (MPI_SUCCESS != MPI_Get_processor_name(procname, &length)) {
+           strcpy(procname, "unknown");
+       }
+
+       /* Now decide which GPU to pick.  This requires hwloc to work properly.
+        * We first see which CPU we are bound to, then try and find a GPU nearby.
+        */
+       retval = hwloc_topology_init(&topology);
+       assert(retval == 0);
+       retval = hwloc_topology_set_flags(topology, flags);
+       assert(retval == 0);
+       retval = hwloc_topology_load(topology);
+       assert(retval == 0);
+       newset = hwloc_bitmap_alloc();
+       retval = hwloc_get_last_cpu_location(topology, newset, 0);
+       assert(retval == 0);
+
+       /* Get the object that contains the cpuset */
+       node = hwloc_get_first_largest_obj_inside_cpuset(topology, newset);
+
+       /* Climb up from that object until we find the HWLOC_OBJ_NODE */
+       while (node->type != HWLOC_OBJ_NODE) {
+           node = node->parent;
+       }
+
+       /* Now look for the HWLOC_OBJ_BRIDGE.  All PCI busses hanging off the
+        * node will have one of these */
+       bridge = hwloc_get_next_child(topology, node, NULL);
+       while (bridge->type != HWLOC_OBJ_BRIDGE) {
+           bridge = hwloc_get_next_child(topology, node, bridge);
+       }
+
+       /* Now find all the GPUs on this NUMA node and put them into an array */
+       find_gpus(topology, bridge, NULL);
+
+       ABORT_ON_ERROR(cuInit(0));
+       /* Now select the first GPU that we find */
+       if (gpus[0] == 0) {
+           printf("No GPU found\n");
+       } else {
+           sprintf(pciBusId, "%.2x:%.2x:%.2x.%x", gpus[0]->attr->pcidev.domain, gpus[0]->attr->pcidev.bus,
+           gpus[0]->attr->pcidev.dev, gpus[0]->attr->pcidev.func);
+           ABORT_ON_ERROR(cuDeviceGetByPCIBusId(&dev, pciBusId));
+           ABORT_ON_ERROR(cuDeviceGetName(devName, 256, dev));
+           printf("rank=%d (%s): Selected GPU=%s, name=%s\n", rank, procname, pciBusId, devName);
+       }
+
+       MPI_Finalize();
+       return 0;
+   }
+
+/////////////////////////////////////////////////////////////////////////
+
+How do I develop CUDA-aware Open MPI applications?
+--------------------------------------------------
+
+Developing CUDA-aware applications is a complex topic, and beyond the
+scope of this document. CUDA-aware applications often have to take
+machine-specific considerations into account, including the number of
+GPUs installed on each node and how the GPUs are connected to the CPUs
+and to each other. Often, when using a particular transport layer
+(such as OPA/PSM2) there will be run-time decisions to make about
+which CPU cores will be used with which GPUs.
+
+A good place to start is the `NVIDIA CUDA Toolkit Documentation
+<https://docs.nvidia.com/cuda/>`_ including the `Programming Guide
+<https://docs.nvidia.com/cuda/cuda-c-programming-guide/>`_ and the
+`Best Practices Guide
+<https://docs.nvidia.com/cuda/cuda-c-best-practices-guide/>`_.  For
+examples of how to write CUDA-aware MPI applications, the `NVIDIA
+developers blog
+<https://github.com/NVIDIA-developer-blog/code-samples/tree/master/posts/cuda-aware-mpi-example>`_
+offers examples and the `OSU Micro-Benchmarks
+<https://mvapich.cse.ohio-state.edu/benchmarks/>`_ offer an excellent
+example of how to write CUDA-aware MPI applications.
+
+/////////////////////////////////////////////////////////////////////////
+
+.. _faq-cuda-mpi-apis-cuda-label:
+
+Which MPI APIs work with CUDA-aware?
+------------------------------------
+
+* MPI_Allgather
+* MPI_Allgatherv
+* MPI_Allreduce
+* MPI_Alltoall
+* MPI_Alltoallv
+* MPI_Alltoallw
+* MPI_Bcast
+* MPI_Bsend
+* MPI_Bsend_init
+* MPI_Exscan
+* MPI_Ibsend
+* MPI_Irecv
+* MPI_Isend
+* MPI_Irsend
+* MPI_Issend
+* MPI_Gather
+* MPI_Gatherv
+* MPI_Get
+* MPI_Put
+* MPI_Rsend
+* MPI_Rsend_init
+* MPI_Recv
+* MPI_Recv_init
+* MPI_Reduce
+* MPI_Reduce_scatter
+* MPI_Reduce_scatter_block
+* MPI_Scan
+* MPI_Scatter
+* MPI_Scatterv
+* MPI_Send
+* MPI_Send_init
+* MPI_Sendrecv
+* MPI_Ssend
+* MPI_Ssend_init
+* MPI_Win_create
+
+.. FIXME: We need to verify the above list.
+
+/////////////////////////////////////////////////////////////////////////
+
+Which MPI APIs do NOT work with CUDA-aware?
+-------------------------------------------
+
+* MPI_Accumulate
+* MPI_Compare_and_swap
+* MPI_Fetch_and_op
+* MPI_Get_Accumulate
+* MPI_Iallgather
+* MPI_Iallgatherv
+* MPI_Iallreduce
+* MPI_Ialltoall
+* MPI_Ialltoallv
+* MPI_Ialltoallw
+* MPI_Ibcast
+* MPI_Iexscan
+* MPI_Rget
+* MPI_Rput
+
+.. FIXME: We need to verify the above list.
+
+/////////////////////////////////////////////////////////////////////////
+
+How do I use CUDA-aware UCX for Open MPI?
+-----------------------------------------
+
+Example of running ``osu_latency`` from the `OSU benchmarks
+<https://mvapich.cse.ohio-state.edu/benchmarks>`_ with CUDA buffers
+using Open MPI and UCX CUDA support:
+
+.. code-block::
+
+   shell$ mpirun -n 2 --mca pml ucx \
+       -x UCX_TLS=rc,sm,cuda_copy,gdr_copy,cuda_ipc ./osu_latency D D
+
+/////////////////////////////////////////////////////////////////////////
+
+Which MPI APIs work with CUDA-aware UCX?
+----------------------------------------
+
+* MPI_Send
+* MPI_Bsend
+* MPI_Ssend
+* MPI_Rsend
+* MPI_Isend
+* MPI_Ibsend
+* MPI_Issend
+* MPI_Irsend
+* MPI_Send_init
+* MPI_Bsend_init
+* MPI_Ssend_init
+* MPI_Rsend_init
+* MPI_Recv
+* MPI_Irecv
+* MPI_Recv_init
+* MPI_Sendrecv
+* MPI_Bcast
+* MPI_Gather
+* MPI_Gatherv
+* MPI_Allgather
+* MPI_Reduce
+* MPI_Reduce_scatter
+* MPI_Reduce_scatter_block
+* MPI_Allreduce
+* MPI_Scan
+* MPI_Exscan
+* MPI_Allgatherv
+* MPI_Alltoall
+* MPI_Alltoallv
+* MPI_Alltoallw
+* MPI_Scatter
+* MPI_Scatterv
+* MPI_Iallgather
+* MPI_Iallgatherv
+* MPI_Ialltoall
+* MPI_Iialltoallv
+* MPI_Ialltoallw
+* MPI_Ibcast
+* MPI_Iexscan
+
+.. FIXME: We need to verify the above list.  These _SHOULD_ be the same
+   as above.
+
+/////////////////////////////////////////////////////////////////////////
+
+Which MPI APIs do NOT work with CUDA-aware UCX?
+-----------------------------------------------
+
+* All one-sided operations such as MPI_Put, MPI_Get, MPI_Accumulate,
+  MPI_Rget, MPI_Rput, MPI_Get_Accumulate, MPI_Fetch_and_op,
+  MPI_Compare_and_swap, etc
+* All window creation calls such as MPI_Win_create
+* All non-blocking reduction collectives like MPI_Ireduce,
+  MPI_Iallreduce, etc
+
+.. FIXME: Checking with nVidia.  This may be more of an issue of OSC_UCX
+   not supporting CUDA, though perhaps it's just performance.
+
+/////////////////////////////////////////////////////////////////////////
+
+Can I tell at compile time or runtime whether I have CUDA-aware support?
+------------------------------------------------------------------------
+
+There is both a compile time check and a run-time check available.
+You can use whichever is the most convenient for your program.  To
+access them, you need to include ``mpi-ext.h``. Note that
+``mpi-ext.h`` is specific to Open MPI. The following program shows an
+example of using the CUDA-aware macro and run-time check.
+
+.. code-block:: c
+
+   /*
+    * Program that shows the use of CUDA-aware macro and runtime check.
+    */
+   #include <stdio.h>
+   #include "mpi.h"
+
+   #if !defined(OPEN_MPI) || !OPEN_MPI
+   #error This source code uses an Open MPI-specific extension
+   #endif
+
+   /* Needed for MPIX_Query_cuda_support(), below */
+   #include "mpi-ext.h"
+
+   int main(int argc, char *argv[])
+   {
+       MPI_Init(&argc, &argv);
+
+       printf("Compile time check:\n");
+   #if defined(MPIX_CUDA_AWARE_SUPPORT) && MPIX_CUDA_AWARE_SUPPORT
+       printf("This MPI library has CUDA-aware support.\n", MPIX_CUDA_AWARE_SUPPORT);
+   #elif defined(MPIX_CUDA_AWARE_SUPPORT) && !MPIX_CUDA_AWARE_SUPPORT
+       printf("This MPI library does not have CUDA-aware support.\n");
+   #else
+       printf("This MPI library cannot determine if there is CUDA-aware support.\n");
+   #endif /* MPIX_CUDA_AWARE_SUPPORT */
+
+       printf("Run time check:\n");
+   #if defined(MPIX_CUDA_AWARE_SUPPORT)
+       if (1 == MPIX_Query_cuda_support()) {
+           printf("This MPI library has CUDA-aware support.\n");
+       } else {
+           printf("This MPI library does not have CUDA-aware support.\n");
+       }
+   #else /* !defined(MPIX_CUDA_AWARE_SUPPORT) */
+       printf("This MPI library cannot determine if there is CUDA-aware support.\n");
+   #endif /* MPIX_CUDA_AWARE_SUPPORT */
+
+       MPI_Finalize();
+
+       return 0;
+   }
+
+/////////////////////////////////////////////////////////////////////////
+
+How do I limit how much CUDA IPC memory is held in the registration cache?
+--------------------------------------------------------------------------
+
+As mentioned earlier, the Open MPI library will make use of CUDA IPC support where
+possible to move the GPU data quickly between GPUs that are on the same node and
+same PCI root complex. The library holds on to registrations even after the data
+transfer is complete as it is expensive to make some of the CUDA IPC registration
+calls. If you want to limit how much memory is registered, you can use the
+``mpool_rgpusm_rcache_size_limit`` MCA parameter. For example, this sets the limit
+to 1000000 bytes:
+
+.. code-block::
+
+   shell$ mpirun --mca mpool_rgpusm_rcache_size_limit 1000000 ...
+
+When the cache reaches this size, it will kick out the least recently used until
+it can fit the new registration in.
+
+There also is the ability to have the cache empty itself out when the
+limit is reached:
+
+.. code-block::
+
+   shell$ mpirun --mca mpool_rgpusm_rcache_empty_cache 1 ...
+
+/////////////////////////////////////////////////////////////////////////
+
+What are some guidelines for using CUDA and Open MPI with Omni-Path?
+--------------------------------------------------------------------
+
+When developing CUDA-aware Open MPI applications for OPA-based fabrics, the
+PSM2 transport is preferred and a CUDA-aware version of PSM2 is provided with
+all versions of the Cornelis Networks Omni-Path OPXS software suite.
+
+.. error:: TODO Are Intel/OPA references still correct?
+
+The PSM2 library provides a number of settings that will govern how it
+will interact with CUDA, including ``PSM2_CUDA`` and ``PSM2_GPUDIRECT``,
+which should be set in the environment before ``MPI_Init()`` is called. For
+example:
+
+.. code-block::
+
+   shell$ mpirun -x PSM2_CUDA=1 -x PSM2_GPUDIRECT=1 --mca mtl psm2 mpi_hello
+
+In addition, each process of the application should select a specific
+GPU card to use before calling ``MPI_Init()``, by using
+``cudaChooseDevice()``, ``cudaSetDevice()`` and similar. The chosen
+GPU should be within the same NUMA node as the CPU the MPI process is
+running on. You will also want to use the ``mpirun``
+``--bind-to-core`` or ``--bind-to-socket`` option to ensure that MPI
+processes do not move between NUMA nodes. See the section on
+:ref:`NUMA Node Issues <faq-cuda-mpi-cuda-numa-issues-label>`, for
+more information.
+
+For more information see the *Cornelis Networks Performance Scaled Messaging 2
+(PSM2) Programmer's Guide* and the *Cornelis Networks Omni-Path Performance
+Tuning Guide*, which can be found in the `Cornelis Networks Customer Center
+<https://customercenter.cornelisnetworks.com/>`_.
+
+.. error:: TODO Are Intel/OPA references still correct?
+
+/////////////////////////////////////////////////////////////////////////
+
+When do I need to select a CUDA device?
+---------------------------------------
+
+"mpi-cuda-dev-selection"
+
+OpenMPI requires CUDA resources allocated for internal use.  These
+are allocated lazily when they are first needed, e.g. CUDA IPC mem handles
+are created when a communication routine first requires them during a
+transfer.  So, the CUDA device needs to be selected before the first MPI
+call requiring a CUDA resource. MPI_Init and most communicator related
+operations do not create any CUDA resources (guaranteed for MPI_Init,
+MPI_Comm_rank, MPI_Comm_size, MPI_Comm_split_type and MPI_Comm_free).  It
+is thus possible to use those routines to query rank information and use
+those to select a GPU, e.g. using
+
+.. code-block:: c
+
+    int local_rank = -1;
+    {
+        MPI_Comm local_comm;
+        MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, rank, MPI_INFO_NULL, &local_comm);
+        MPI_Comm_rank(local_comm, &local_rank);
+        MPI_Comm_free(&local_comm);
+    }
+    int num_devices = 0;
+    cudaGetDeviceCount(&num_devices);
+    cudaSetDevice(local_rank % num_devices);
+
+MPI internal CUDA resources are released during MPI_Finalize. Thus it is an
+application error to call cudaDeviceReset before MPI_Finalize is called.
+
+
+/////////////////////////////////////////////////////////////////////////
+
+How do I enable CUDA support in HCOLL collective component
+----------------------------------------------------------
+
+HCOLL component supports CUDA GPU buffers for the following
+collectives:
+
+MPI_Allreduce
+MPI_Bcast
+MPI_Allgather
+MPI_Ibarrier
+MPI_Ibcast
+MPI_Iallgather
+MPI_Iallreduce
+
+To enable CUDA GPU buffer support in these collectives pass the
+following environment variables via mpirun:
+
+.. code-block::
+
+   shell$ mpirun -x HCOLL_GPU_ENABLE=1 -x HCOLL_ENABLE_NBC=1 ..
+
+See `nVidia HCOLL documentation <https://docs.nvidia.com/networking/display/HPCXv29/HCOLL>`_
+for more information.
diff --git a/docs/tuning-apps/networking/ib-and-roce.rst b/docs/tuning-apps/networking/ib-and-roce.rst
new file mode 100644
index 00000000000..32ba7936cc2
--- /dev/null
+++ b/docs/tuning-apps/networking/ib-and-roce.rst
@@ -0,0 +1,188 @@
+InifiniBand / RoCE support
+==========================
+
+.. error:: TODO This section needs to be converted from FAQ Q&A style
+           to regular documentation style.
+
+How are InfiniBand / RoCE devices supported in Open MPI?
+--------------------------------------------------------
+
+Open MPI's support for InfiniBand and RoCE devices has changed over
+time.
+
+In the Open MPI |ompi_series| series, InfiniBand and RoCE devices are
+supported via the UCX (``ucx``) PML.
+
+.. note:: Prior versions of Open MPI also included the ``openib`` BTL
+          for InfiniBand and RoCE devices.  Open MPI |ompi_series| no
+          longer includes the ``openib`` BTL.
+
+/////////////////////////////////////////////////////////////////////////
+
+What is UCX?
+------------
+
+`UCX <https://openucx.org/>`_ is an open-source optimized
+communication library which supports multiple networks, including
+RoCE, InfiniBand, uGNI, TCP, shared memory, and others. UCX
+mixes-and-matches transports and protocols which are available on the
+system to provide optimal performance. It also has built-in support
+for GPU transports (with CUDA and ROCm providers) which lets
+RDMA-capable transports access the GPU memory directly.
+
+/////////////////////////////////////////////////////////////////////////
+
+How do I use UCX with Open MPI?
+-------------------------------
+
+If Open MPI includes UCX support, then UCX is enabled and selected by
+default for InfiniBand and RoCE network devices; typically, no
+additional parameters are required.  In this case, the network port
+with the highest bandwidth on the system will be used for inter-node
+communication, and shared memory will be used for intra-node
+communication.  To select a specific network device to use (for
+example, ``mlx5_0`` device port 1):
+
+.. code-block::
+
+   shell$ mpirun -x UCX_NET_DEVICES=mlx5_0:1 ...
+
+It's also possible to force using UCX for MPI point-to-point and
+one-sided operations:
+
+.. code-block::
+
+   shell$ mpirun --mca pml ucx --mca osc ucx ...
+
+For OpenSHMEM, in addition to the above, it's possible to force using
+UCX for remote memory access and atomic memory operations:
+
+.. code-block::
+
+   shell$ mpirun --mca pml ucx --mca osc ucx --mca scoll ucx --mca atomic ucx ...
+
+/////////////////////////////////////////////////////////////////////////
+
+What is RDMA over Converged Ethernet (RoCE)?
+--------------------------------------------
+
+RoCE (which stands for *RDMA over Converged Ethernet*) provides
+InfiniBand native RDMA transport on top of lossless Ethernet data
+links.
+
+Since we're talking about Ethernet, there's no Subnet Manager, no
+Subnet Administrator, no InfiniBand SL, nor any other InfiniBand
+Subnet Administration parameters.
+
+Connection management in RoCE is based on the OFED RDMACM (RDMA
+Connection Manager) service:
+
+* The OS IP stack is used to resolve remote (IP,hostname) tuples to
+  a DMAC.
+* The outgoing Ethernet interface and VLAN are determined according
+  to this resolution.
+* The appropriate RoCE device is selected accordingly.
+* Network parameters (such as MTU, SL, timeout) are set locally by
+  the RDMACM in accordance with kernel policy.
+
+/////////////////////////////////////////////////////////////////////////
+
+How do I know what MCA parameters are available for tuning MPI performance?
+---------------------------------------------------------------------------
+
+The ``ompi_info`` command can display all the parameters available for
+any Open MPI component.  For example:
+
+.. code-block::
+
+   shell$ ompi_info --param pml ucx --level 9
+
+.. important:: Unlike most other Open MPI components, the UCX PML
+               mainly uses environment variables for run-time tuning
+               |mdash| not Open MPI MCA parameters.  Consult `the UCX
+               documentation
+               <https://openucx.org/documentation/>`_ for details
+               about what environment variables are available.
+
+/////////////////////////////////////////////////////////////////////////
+
+How do I tell Open MPI which IB Service Level to use?
+-----------------------------------------------------
+
+In order to tell the UCX PML which SL to use, the IB SL must be
+specified using the ``UCX_IB_SL`` environment variable.  For example:
+
+.. code-block::
+
+   shell$ mpirun --mca pml ucx -x UCX_IB_SL=N ...
+
+The value of IB SL ``N`` should be between 0 and 15, where 0 is the
+default value.
+
+/////////////////////////////////////////////////////////////////////////
+
+How do I run Open MPI over RoCE?
+--------------------------------
+
+In order to use RoCE with the UCX PML, the relevant Ethernet port must
+be specified using the ``UCX_NET_DEVICES`` environment variable.  For
+example:
+
+.. code-block::
+
+   shell$ mpirun --mca pml ucx -x UCX_NET_DEVICES=mlx5_0:1 ...
+
+UCX selects IPv4 RoCEv2 by default. If different behavior is needed,
+you can set a specific GID index:
+
+.. code-block::
+
+   shell$ mpirun --mca pml ucx -x UCX_NET_DEVICES=mlx5_0:1 -x UCX_IB_GID_INDEX=1 ...
+
+/////////////////////////////////////////////////////////////////////////
+
+.. _faq-ib-troubleshoot-label:
+
+I'm experiencing a problem with Open MPI on my InfiniBand / RoCE network; how do I troubleshoot and get help?
+-------------------------------------------------------------------------------------------------------------
+
+In order for us to help you, it is *most* helpful if you can run a few
+steps before sending an e-mail to both perform some basic
+troubleshooting and provide us with enough information about your
+environment to help you.  Please include answers to the following
+questions in your e-mail:
+
+#. Which UCX and OpenFabrics version are you running?  Please specify
+   where you got the software from (e.g., from the OpenFabrics and/or
+   UCX community web sites, already included in your Linux
+   distribution, downloade from NVIDIA's web site, etc.).
+
+#. What distro and version of Linux are you running?  What is your
+   kernel version?
+
+#. What is the output of the ``ibv_devinfo`` command on a known "good"
+   node and a known "bad" node?
+
+   .. note:: There must be at least one port listed as "PORT_ACTIVE"
+             for Open MPI to work.  If there is not at least one
+             PORT_ACTIVE port, something is wrong with your InfiniBand
+             / RoCE environment and Open MPI will not be able to run.
+
+#. What is the output of the ``ifconfig`` command on a known "good"
+   node and a known "bad" node?
+
+   .. note:: Note that some Linux distributions do not put
+             ``ifconfig`` in the default path for normal users; look
+             for it at ``/sbin/ifconfig`` or ``/usr/sbin/ifconfig``.
+
+#. If running under Bourne shells, what is the output of the ``ulimit
+   -l`` command?
+
+   If running under C shells, what is the output of the ``limit | grep
+   memorylocked`` command?
+
+   .. note:: If the value is not ``unlimited``, .................
+
+   .. error:: TODO Would be good to point to some UCX/vendor docs here
+              about setting memory limits (rather than reproducing this
+              information ourselves).
diff --git a/docs/tuning-apps/networking/index.rst b/docs/tuning-apps/networking/index.rst
new file mode 100644
index 00000000000..00aa0f39df5
--- /dev/null
+++ b/docs/tuning-apps/networking/index.rst
@@ -0,0 +1,28 @@
+Networking support
+==================
+
+Open MPI supports a variety of different networking transports for
+off-node communication.  Not all transports are supported or available
+on every platform.  Many require specialized hardware, operating
+system drivers, and/or userspace network transport libraries.
+
+When Open MPI is being configured, it will search for a variety of
+network transport libraries (and corresponding development header
+files).  By default, if ``configure`` can find a network transport
+library and its development header files, it will include support for
+that library.  If ``configure`` does not find a library or its header
+files, it will simply skip that library (and Open MPI will simply not
+build support for that library).
+
+.. note:: The sections listed below are by no means comprehensive.
+
+.. toctree::
+   :maxdepth: 1
+
+   ofi
+   tcp
+   shared-memory
+   ib-and-roce
+   iwarp
+   cuda
+   rocm
diff --git a/docs/tuning-apps/networking/iwarp.rst b/docs/tuning-apps/networking/iwarp.rst
new file mode 100644
index 00000000000..a24b3c73fbb
--- /dev/null
+++ b/docs/tuning-apps/networking/iwarp.rst
@@ -0,0 +1,23 @@
+iWARP Support
+=============
+
+Open MPI's support for iWARP devices has changed over time.
+
+In the Open MPI |ompi_series| series, iWARP devices are
+supported via the OFI (``ofi``) MTL via the CM (``cm``) PML.
+
+.. note:: Prior versions of Open MPI supported iWARP devies via the
+          ``openib`` BTL.  Open MPI |ompi_series| no longer includes
+          the ``openib`` BTL.
+
+Specifically, iWARP support is provides through the ``rxm`` provider
+of the OpenFabrics Interfaces library ``libfabric``.  There is
+software emulation involved in the MPI support of iWARP devices (and
+therefore at least some level of performance degradation), but the
+current iWARP vendors have chosen not to provide a higher-performance
+option.
+
+.. important:: iWARP support is not well tested or maintained in Open
+               MPI.  The Open MPI community would love to have a
+               maintainer who can develop and provide support for
+               iWARP devices over time.
diff --git a/docs/tuning-apps/networking/ofi.rst b/docs/tuning-apps/networking/ofi.rst
new file mode 100644
index 00000000000..3f7563985d1
--- /dev/null
+++ b/docs/tuning-apps/networking/ofi.rst
@@ -0,0 +1,135 @@
+OpenFabrics Interfaces (OFI) / Libfabric support
+================================================
+
+.. error:: TODO This section needs to be converted from FAQ Q&A style
+           to regular documentation style.
+
+What is OFI / Libfabric?
+------------------------
+
+"OFI" stands for the `OpenFabrics Interfaces
+<https://libfabric.org/>`_, which are implemented in the ``libfabric``
+library.  These two terms are typically used interchangeably.
+
+Open MPI supports many different underlying networks via Libfabric,
+including (but not limited to):
+
+* AWS EFA
+* Cisco usNIC
+* Cornelis Networks Omni-Path
+* HPE Slingshot 11
+
+In general, the OFI-based components in Open MPI will auto-select
+themselves as appropriate at run time.
+
+That being said, additional questions are available in this FAQ
+section to provide more information about specific OFI-based network
+types and support.
+
+/////////////////////////////////////////////////////////////////////////
+
+What are the Libfabric (OFI) components in Open MPI?
+----------------------------------------------------
+
+Open MPI has three main components for Libfabric (a.k.a., OFI)
+communications:
+
+#. ``ofi`` MTL: Available since Open MPI v1.10, this component is used
+   with the ``cm`` PML and is used for two-sided MPI communication
+   (e.g., ``MPI_SEND`` and ``MPI_RECV``).
+
+  The ``ofi`` MTL requires that the Libfabric provider support
+  reliable datagrams with ordered tagged messaging (specifically:
+  ``FI_EP_RDM`` endpoints, ``FI_TAGGED`` capabilities, and
+  ``FI_ORDER_SAS`` ordering).
+
+#. ``ofi`` BTL: Available since Open MPI v4.0.0, this component is primarily
+   intended for one-sided MPI communications (e.g., ``MPI_PUT``). It
+   can also support BTL send/recv operations.
+   ``ofi`` BTL requires that the Libfabric provider support reliable
+   datagrams, RMA and atomic operations, and remote atomic completion
+   notifications (specifically: ``FI_EP_RDM`` endpoints, ``FI_RMA``
+   and ``FI_ATOMIC`` capabilities, and ``FI_DELIVERY_COMPLETE`` op
+   flags).
+
+#. ``usnic`` BTL: This BTL is used exclusively with Cisco usNIC-based
+   networks.  It will auto-select itself over the other OFI-based
+   components when run with Cisco usNIC-based networks.
+
+See each Lifabric provider man page (e.g., fi_sockets(7)) to understand which
+provider will work for each of the above-listed Open MPI components. Some
+providers may require to be used with one of the Libfabric utility providers;
+for example, the verbs provider needs to be paired with utility provider
+``ofi_rxm`` to provide reliable datagram endpoint support (``verbs;ofi_rxm``).
+
+Both components have MCA parameters to specify the Libfabric provider(s) that
+will be included/excluded in the selection process. For example:
+
+.. code-block::
+
+   shell$ mpirun --mca pml cm --mca mtl ofi --mca mtl_ofi_provider_include psm2 mpi_hello
+
+In addition, each component has specific parameters for each one; see
+``ompi_info --param <framework> <component> -level 9`` for a full
+list. For example:
+
+.. code-block::
+
+   shell$ ompi_info --param mtl ofi --level 9
+
+.. important:: When using the HPE CXI provider and ``mpirun`` as the job launcher,
+          it is recommended that the PRTE ``ras_base_launch_orted_on_hn`` MCA parameter be set to 1.
+          This can be done by adding ``--prtemca ras_base_launch_orted_on_hn 1`` to the job launch
+          command line.  This ensures that MPI processes launched on the first node of
+          an allocation are able to use the CXI provider.
+          
+For more information refer to the `Libfabric web site
+<https://libfabric.org/>`_.
+
+/////////////////////////////////////////////////////////////////////////
+
+Omni-Path: How can the multi-rail settings be adjusted if multiple HFI (Host Fabric Interface) cards are installed on the system?
+---------------------------------------------------------------------------------------------------------------------------------
+
+Multi-Rail feature allows a process to use multiple HFIs to transfer a message
+to improve message bandwidth. The PSM2 library handles the support for multi-rail
+which is off by default. The multi-rail settings can be modified using the
+following environment variables:
+
+* ``PSM2_MULTIRAIL=[0,1,2] ]``: 0=Disabled, 1=Enable across all HFIs in the
+  system, 2=Enable multi-rail within a NUMA node.
+* ``PSM2_MULTIRAIL_MAP=unit:port,unit:port...``
+
+The variables above may be included in the ``mpirun`` command line or in
+the environment. For example:
+
+.. code-block::
+
+   shell$ mpirun -mca mtl [psm2|ofi] -x PSM2_MULTIRAIL=1 -n 2 -H host1,host2 ./a.out
+
+.. note:: When using the OFI MTL, please ensure that the PSM2 OFI
+          provider is used for communication with OPA devices.
+
+/////////////////////////////////////////////////////////////////////////
+
+Omni-Path: What is Multi-HFI support in PSM2 and how does it differ from multi-rail?
+------------------------------------------------------------------------------------
+
+Multi-HFI support is intended to describe the use of multiple HFIs in
+a system among MPI ranks local to a node in order to load-balance the
+hardware resources. It differs from the Multi-Rail feature, which is
+intended to allow a single process to use all HFIs in the system. For
+an MPI job with multiple processes on a single node, the default PSM2
+behavior depends on the affinity settings of the MPI process. The PSM2
+library defaults to using the HFI (Host Fabric Interface) that is in
+the same NUMA node as that of the MPI process.
+
+Users can restrict access to a single HFI using the environment variable:
+
+* ``HFI_UNIT=N``: valid values of N are 0,1,2 and 3
+
+More details can be found on the PSM2 Programmer's Guide and the Omni-Path
+Fabric Performance Tuning Guide.
+
+Please see the `Cornelis Networks Customer Center <https://customercenter.cornelisnetworks.com/>`_
+for more details.
diff --git a/docs/tuning-apps/networking/rocm.rst b/docs/tuning-apps/networking/rocm.rst
new file mode 100644
index 00000000000..10ee12fe9e2
--- /dev/null
+++ b/docs/tuning-apps/networking/rocm.rst
@@ -0,0 +1,134 @@
+ROCm
+====
+
+ROCm is the name of the software stack used by AMD GPUs. It includes
+the ROCm Runtime (ROCr), the HIP programming model, and numerous
+numerical and machine learning libraries tuned for the AMD Instinct
+accelerators. More information can be found at the following
+`AMD webpages <https://www.amd.com/en/graphics/servers-solutions-rocm>`_
+
+
+Building Open MPI with ROCm support
+-----------------------------------
+
+ROCm-aware support means that the MPI library can send and receive
+data from AMD GPU device buffers directly. As of today, ROCm support
+is available through UCX. While other communication transports might
+work as well, UCX is the only transport formally supported in Open MPI
+|ompi_ver| for ROCm devices.
+
+Since UCX will be providing the ROCm support, it is important to
+ensure that UCX itself is built with ROCm support.
+
+To see if your UCX library was built with ROCm support, run the
+following command:
+
+.. code-block:: sh
+
+ # Check if ucx was built with ROCm support
+ shell$ ucx_info -v
+
+ # configured with: --with-rocm=/opt/rocm --without-knem --without-cuda
+
+If you need to build the UCX library yourself to include ROCm support,
+please see the UCX documentation for `building UCX with Open MPI:
+<https://openucx.readthedocs.io/en/master/running.html#openmpi-with-ucx>`_
+
+It should look something like:
+
+.. code-block:: sh
+
+   # Configure UCX with ROCm support
+   shell$ cd ucx
+   shell$ ./configure --prefix=/path/to/ucx-rocm-install \
+                     --with-rocm=/opt/rocm --without-knem
+
+   # Configure Open MPI with UCX and ROCm support
+   shell$ cd ompi
+   shell$ ./configure --with-rocm=/opt/rocm    \
+          --with-ucx=/path/to/ucx-rocm-install \
+          <other configure params>
+
+/////////////////////////////////////////////////////////////////////////
+
+Checking that Open MPI has been built with ROCm support
+-------------------------------------------------------
+
+Verify that Open MPI has been built with ROCm using the
+:ref:`ompi_info(1) <man1-ompi_info>` command:
+
+.. code-block:: sh
+
+   # Use ompi_info to verify ROCm support in Open MPI
+   shell$ ./ompi_info | grep "MPI extensions"
+          MPI extensions: affinity, cuda, ftmpi, rocm
+
+/////////////////////////////////////////////////////////////////////////
+
+
+Using ROCm-aware UCX with Open MPI
+--------------------------------------------------------------------------
+
+If UCX and Open MPI have been configured with ROCm support, specifying
+the UCX pml component is sufficient to take advantage of the ROCm
+support in the libraries. For example, the command to execute the
+``osu_latency`` benchmark from the `OSU benchmarks
+<https://mvapich.cse.ohio-state.edu/benchmarks>`_ with ROCm buffers
+using Open MPI and UCX ROCm support is something like this:
+
+.. code-block::
+
+   shell$ mpirun -n 2 --mca pml ucx \
+           ./osu_latency D D
+
+Note: some additional configure flags are required to compile the OSU
+benchmark to support ROCm buffers. Please refer to the `UCX ROCm
+instructions
+<https://github.com/openucx/ucx/wiki/Build-and-run-ROCM-UCX-OpenMPI>`_
+for details.
+
+
+/////////////////////////////////////////////////////////////////////////
+
+Runtime querying of ROCm support in Open MPI
+--------------------------------------------
+
+Starting with Open MPI v5.0.0 :ref:`MPIX_Query_rocm_support(3)
+<mpix_query_rocm_support>` is available as an extension to check
+the availability of ROCm support in the library. To use the
+function, the code needs to include ``mpi-ext.h``. Note that
+``mpi-ext.h`` is an Open MPI specific header file.
+
+/////////////////////////////////////////////////////////////////////////
+
+Collective component supporting ROCm device memory
+--------------------------------------------------
+
+The `UCC <https://github.com/openucx/ucc>`_ based collective component
+in Open MPI can be configured and compiled to include ROCm support.
+
+An example for configure UCC and Open MPI with ROCm is shown below:
+
+.. code-block::
+
+   # Configure and compile UCC with ROCm support
+   shell$ cd ucc
+   shell$ ./configure --with-rocm=/opt/rocm                \
+                      --with-ucx=/path/to/ucx-rocm-install \
+                      --prefix=/path/to/ucc-rocm-install
+   shell$ make -j && make install
+
+   # Configure and compile Open MPI with UCX, UCC, and ROCm support
+   shell$ cd ompi
+   shell$ ./configure --with-rocm=/opt/rocm                \
+                      --with-ucx=/path/to/ucx-rocm-install \
+                      --with-ucc=/path/to/ucc-rocm-install 
+   
+To use the UCC component in an applicatin requires setting some
+additional parameters:
+
+.. code-block::
+
+   shell$ mpirun --mca pml ucx --mca osc ucx \
+                 --mca coll_ucc_enable 1     \
+                 --mca coll_ucc_priority 100 -np 64 ./my_mpi_app
diff --git a/docs/tuning-apps/networking/shared-memory.rst b/docs/tuning-apps/networking/shared-memory.rst
new file mode 100644
index 00000000000..7c40693cd76
--- /dev/null
+++ b/docs/tuning-apps/networking/shared-memory.rst
@@ -0,0 +1,174 @@
+Shared Memory
+=============
+
+
+The sm BTL
+----------
+
+The ``sm`` BTL is a low-latency, high-bandwidth mechanism for
+transferring data between two processes via shared memory.  This BTL
+can only be used between processes executing on the same node.
+
+.. note:: Between Open MPI version 1.8.0 and 4.1.x, the shared memory
+          BTL was named ``vader``.  As of Open MPI version 5.0.0, the
+          BTL has been renamed ``sm``.
+
+.. warning:: In Open MPI version 5.0.x, the name ``vader`` is simply
+             an alias for the ``sm`` BTL.  Similarly, all
+             ``vader_``-prefixed MCA parameters are automatically
+             aliased to their corresponding ``sm_``-prefixed MCA
+             parameter.
+
+             This alias mechanism is a legacy transition device, and
+             will likely disappear in a future release of Open MPI.
+
+/////////////////////////////////////////////////////////////////////////
+
+Specifying the Use of sm for MPI Messages
+-----------------------------------------
+
+Typically, it is unnecessary to do so;  OMPI will use the best BTL available
+for each communication.
+
+Nevertheless, you may use the MCA parameter ``btl``.  You should also
+specify the ``self`` BTL for communications between a process and
+itself.  Furthermore, if not all processes in your job will run on the
+same, single node, then you also need to specify a BTL for internode
+communications.  For example:
+
+.. code-block:: sh
+
+   shell$ mpirun --mca btl self,sm,tcp -n 16 ./a.out
+
+/////////////////////////////////////////////////////////////////////////
+
+Tuning Parameters to Improve Performance
+----------------------------------------
+
+Mostly, the default values of the MCA parameters have already
+been chosen to give good performance.  To improve performance further
+is a little bit of an art.  Sometimes, it's a matter of trading off
+performance for memory.
+
+* ``btl_sm_eager_limit``: If message data plus header information fits
+  within this limit, the message is sent "eagerly" |mdash| that is, a
+  sender attempts to write its entire message to shared buffers
+  without waiting for a receiver to be ready.  Above this size, a
+  sender will only write the first part of a message, then wait for
+  the receiver to acknowledge its readiness before continuing.  Eager
+  sends *can* improve performance by decoupling senders from
+  receivers.
+
+* ``btl_sm_max_send_size``: Large messages are sent in fragments of
+  this size.  Larger segments *can* lead to greater efficiencies,
+  though they could perhaps also inhibit pipelining between sender and
+  receiver.
+
+* ``btl_sm_free_list_num``: This is the initial number of fragments on
+  each (eager and max) free list.  The free lists can grow in response
+  to resource congestion, but you can increase this parameter to
+  pre-reserve space for more fragments.
+
+* ``btl_sm_backing_directory``: Directory to place backing files for
+  shared memory communication. This directory should be on a local
+  filesystem such as ``/tmp`` or ``/dev/shm`` (default: (linux) ``/dev/shm``,
+  (others) session directory)
+
+/////////////////////////////////////////////////////////////////////////
+
+Shared Memory Mechanisms
+------------------------
+
+The ``sm`` BTL supports two modes of shared memory communication:
+
+#. **Two-copy:** Otherwise known as "copy-in / copy-out", this mode is
+   where the sender copies data into shared memory and the receiver
+   copies the data out.
+
+   This mechanism is always available.
+
+#. **Single copy:** In this mode, the sender or receiver makes a
+   single copy of the message data from the source buffer in one
+   process to the destination buffer in another process.  Open MPI
+   supports three flavors of shared memory single-copy transfers:
+
+   * `Linux KNEM <https://knem.gitlabpages.inria.fr/>`_.  This is a
+     standalone Linux kernel module, made specifically for HPC and MPI
+     libraries to enable high-performance single-copy message
+     transfers.
+
+     Open MPI must be able to find the KNEM header files in order to
+     build support for KNEM.
+
+   * `Linux XPMEM <https://github.com/hjelmn/xpmem>`_.  Similar to
+     KNEM, this is a standalone Linux kernel module, made specifically
+     for HPC and MPI libraries to enable high-performance single-copy
+     message transfers.  It is derived from the Cray XPMEM system.
+
+     Open MPI must be able to find the XPMEM header files in order to
+     build support for XPMEM.
+
+   * Linux Cross-Memory Attach (CMA).  This mechanism is built-in to
+     modern versions of the Linux kernel.  Although more performance
+     than the two-copy shared memory transfer mechanism, CMA is the
+     lowest performance of the single-copy mechanisms.  However, CMA
+     is likely the most widely available because it is enabled by
+     default in several modern Linux distributions.
+
+     Open MPI must be built on a Linux system with a recent enough
+     Glibc and kernel version in order to build support for Linux CMA.
+
+Which mechanism is used at run time depends both on how Open MPI was
+built and how your system is configured.  You can check to see which
+single-copy mechanisms Open MPI was built with via two mechanisms:
+
+#. At the end of running ``configure``, Open MPI emits a list of
+   transports for which it found relevant header files and libraries
+   such that it will be able to build support for them.  You might see
+   lines like this, for example:
+
+   .. code-block:: text
+
+      Shared memory/copy in+copy out: yes
+      Shared memory/Linux CMA: yes
+      Shared memory/Linux KNEM: no
+      Shared memory/XPMEM: no
+
+   The above output indicates that Open MPI will be built with 2-copy
+   (as mentioned above, 2-copy is *always* available) and with Linux
+   CMA support.  KNEM and XPMEM support will *not* be built.
+
+#. After Open MPI is installed, the ``ompi_info`` command can show
+   which ``smsc`` (shared memory single copy) components are
+   available:
+
+   .. code-block:: text
+
+      shell$ ompi_info | grep smsc
+                MCA smsc: cma (MCA v2.1.0, API v1.0.0, Component v5.1.0)
+
+   This Open MPI installation only supports the Linux CMA single-copy
+   mechanism.
+
+.. note:: As implied by the SMSC component names, none of them are
+   supported on macOS.  macOS users will use the two-copy mechanism.
+
+/////////////////////////////////////////////////////////////////////////
+
+Shared Memory Mapping on the Filesystem
+---------------------------------------
+
+The default location of the file is in the ``/dev/shm`` directory. If ``/dev/shm``
+does not exist on the system, the default location will be the OMPI session
+directory. The path is typically something like:
+``/dev/shm/sm_segment.nodename.user_id.job_id.my_node_rank``.
+For example, the full path could be: ``/dev/shm/sm_segment.x.1000.23c70000.0``.
+
+You can use the MCA parameter ``btl_sm_backing_directory`` to place the
+directory in a non-default location.
+
+.. note:: The session directory can be customized via
+          PRRTE using ``--prtemca prte_tmpdir_base /path/to/somewhere``.
+
+.. note:: Even when using single-copy methods like CMA, a shared memory file is still
+          created for managing connection metadata.
diff --git a/docs/tuning-apps/networking/tcp.rst b/docs/tuning-apps/networking/tcp.rst
new file mode 100644
index 00000000000..07dc893e036
--- /dev/null
+++ b/docs/tuning-apps/networking/tcp.rst
@@ -0,0 +1,521 @@
+TCP
+===
+
+.. error:: TODO This section needs to be converted from FAQ Q&A style
+           to regular documentation style.
+
+How do I specify to use the IP network for MPI messages?
+--------------------------------------------------------
+
+Open MPI will generally automatically use the ``tcp`` BTL when:
+
+#. The ``tcp`` BTL is  available at run time (which it should be on
+   most POSIX-like systems), and
+#. A higher-performance network is not available
+
+When the ``tcp`` BTL is used, it is typically also (automatically)
+used with the ``self`` and ``sm`` BTLs for process-loopback and
+node-loopback communication, respectively.
+
+If you want to guarantee that the ``tcp``, ``sm``, and ``self`` BTLs
+are used, you can explicitly specify them on the ``mpirun`` command
+line:
+
+.. code-block:: sh
+
+   shell$ mpirun --mca pml ob1 --mca btl tcp,sm,self ...
+
+.. warning:: Failure to specify the ``sm`` BTL will likely result in
+             lower performance when Open MPI uses the TCP network
+             stack to send to peers on the same host.
+
+.. warning:: Failure to specify the ``self`` BTL may result in Open
+             MPI being unable to complete send-to-self scenarios
+             (meaning that your program will run fine until a process
+             tries to send to itself).
+
+/////////////////////////////////////////////////////////////////////////
+
+But wait |mdash| I'm using a high-speed network.  Do I have to disable the TCP BTL?
+-----------------------------------------------------------------------------------
+
+No.  Following the so-called "Law of Least Astonishment", Open MPI
+assumes that if you have both an IP network and at least one
+high-speed network (such InfiniBand), you will likely only want to use
+the high-speed network(s) for MPI message passing.  Hence, the ``tcp``
+BTL component will sense this and automatically deactivate itself.
+
+That being said, Open MPI may still use TCP for setup and teardown
+information |mdash| so you'll see traffic across your IP network during
+startup and shutdown of your MPI job.  This is normal and does not
+affect the MPI message passing channels.
+
+/////////////////////////////////////////////////////////////////////////
+
+How do I know what MCA parameters are available for tuning MPI performance?
+---------------------------------------------------------------------------
+
+The ``ompi_info`` command can display all the parameters
+available for the ``tcp`` BTL component (i.e., the component that uses
+TCP for MPI communications):
+
+.. code-block:: sh
+
+   shell$ ompi_info --param btl tcp --level 9
+
+/////////////////////////////////////////////////////////////////////////
+
+Does Open MPI use the IP loopback interface?
+--------------------------------------------
+
+Usually not.
+
+In general message passing usage, there are two scenarios where using
+the operating system IP loopback interface could be used:
+
+#. Sending a message from one process to itself
+#. Sending a message from one process to another process on the same
+   machine
+
+The TCP BTL does not handle "send-to-self" scenarios in Open MPI;
+indeed, it is not even capable of doing so.  Instead, the ``self`` BTL
+component is used for all send-to-self MPI communications.  Not only
+does this allow all Open MPI BTL components to avoid special case code
+for send-to-self scenarios, it also allows avoiding using inefficient
+loopback network stacks (such as the IP loopback device).
+
+Specifically: the ``self`` component uses its own mechanisms for
+send-to-self scenarios; it does not use operating system network
+interfaces such as the IP loopback interface.
+
+When sending to other processes on the same machine, Open MPI will
+default to using a shared memory BTL (``sm``).  If the user has
+deactivated these BTLs, depending on what other BTL components are
+available, it is possible that the TCP BTL will be chosen for message
+passing to processes on the same node, in which case the IP loopback
+device will likely be used.  But this is not the default; either
+shared memory has to fail to startup properly or the user must
+specifically request not to use the shared memory BTL.
+
+/////////////////////////////////////////////////////////////////////////
+
+I have multiple IP networks on some/all of my cluster nodes.  Which ones will Open MPI use?
+-------------------------------------------------------------------------------------------
+
+In general, Open MPI will greedily use all IP networks that
+it finds per its :ref:`reachability computations <faq-tcp-routability>`.
+
+To change this behavior, you can either specifically include certain
+networks or specifically exclude certain networks.  :ref:`See this FAQ
+entry <faq-tcp-selection>` for more details.
+
+/////////////////////////////////////////////////////////////////////////
+
+I'm getting TCP-related errors.  What do they mean?
+
+TCP-related errors are usually reported by Open MPI in a message
+similar to these:
+
+.. code-block::
+
+   btl_tcp_endpoint.c:572:mca_btl_tcp_endpoint_complete_connect: connect() failed with errno=113
+   mca_btl_tcp_frag_send: writev failed with errno=104
+
+If an `errno` number is displayed with no explanation string, you can
+see what that specific error number means on your operating system.
+On Linux, you can use the ``perror`` command:
+
+.. code-block:: sh
+
+   # See what errno 113 is
+   shell$ perror 113
+   OS error code 113:  No route to host
+
+   # See what errno 104 is
+   shell$ perror 104
+   OS error code 104:  Connection reset by peer
+
+Two types of errors are commonly reported to the Open MPI user's
+mailing list:
+
+#. **No route to host:** These types of errors *usually* mean that
+   there are multiple IP interfaces available and they do not obey
+   Open MPI's assumptions about routability.  See :ref:`the TCP
+   routability assumptions FAQ entry <faq-tcp-routability>` and
+   :ref:`the TCP selection FAQ entry <faq-tcp-selection>` for more
+   information.
+
+#. **Connection reset by peer:** These types of errors *usually* occur
+   after ``MPI_INIT`` has completed, and typically indicate that an
+   MPI process has died unexpectedly (e.g., due to a catastrophic error
+   such as a segmentation fault).  The specific error message
+   indicates that a peer MPI process tried to write to the now-dead
+   MPI process and failed.
+
+/////////////////////////////////////////////////////////////////////////
+
+.. _faq-tcp-selection:
+
+How do I tell Open MPI which IP interfaces / networks to use?
+-------------------------------------------------------------
+
+In some HPC environments, it is not uncommon to have multiple IP
+interfaces on each node |mdash| for example, one IP network may be
+"slow" and used for control information such as a batch scheduler, a
+networked filesystem, and/or interactive logins.  Another IP network
+(or networks) may be "fast" and be intended for parallel applications
+to use during their runs.  As another example, some operating systems
+may also have virtual interfaces for communicating with virtual
+machines.
+
+Unless otherwise specified, Open MPI will greedily use all "up" IP
+networks that it can find and try to connect to all peers *upon
+demand* (i.e., Open MPI does not open sockets to all of its MPI peers
+during ``MPI_INIT`` |mdash| see :ref:`this FAQ entry
+<faq-tcp-sockets>` for more details).  Hence, if you want MPI jobs to
+not use specific IP networks |mdash| or not use any IP networks at all
+|mdash| then you need to tell Open MPI.
+
+.. warning:: Aggressively using all "up" interfaces can cause problems
+             in some cases.  For example, if you have a machine with a
+             local-only interface (e.g., the loopback device, or a
+             virtual-machine bridge device that can only be used *on
+             that machine*, and cannot be used to communicate with MPI
+             processes on other machines), you will likely need to
+             tell Open MPI to ignore these networks.
+
+             Open MPI usually ignores loopback devices by default, but
+             *other local-only devices must be manually ignored.*
+             Users have reported cases where RHEL6 automatically
+             installed a ``virbr0`` device for Xen virtualization.
+             This interface was automatically given an IP address in
+             the 192.168.1.0/24 subnet and marked as "up".  Since Open
+             MPI saw this 192.168.1.0/24 "up" interface in all MPI
+             processes on all nodes, it assumed that that network was
+             usable for MPI communications.  This is obviously
+             incorrect, and it led to MPI applications hanging when
+             they tried to send or receive MPI messages.
+
+#. To disable Open MPI from using TCP for MPI communications, the
+   ``tcp`` MCA parameter should be set accordingly.  You can either
+   *exclude* the TCP component or *include* all other components.
+   Specifically:
+
+   .. code-block:: sh
+
+      # This says to exclude the TCP BTL component
+      # (implicitly including all others)
+      shell$ mpirun --mca btl ^tcp...
+
+      # This says to include only the listed BTL components
+      # (tcp is not listed, and therefore will not be used)
+      shell$ mpirun --mca btl self,vader,openib ...
+
+#. If you want to use TCP for MPI communications, but want to restrict
+   it from certain networks, use the ``btl_tcp_if_include`` or
+   ``btl_tcp_if_exclude`` MCA parameters (only one of the two should
+   be set).  The values of these parameters can be a comma-delimited
+   list of network interfaces.  For example:
+
+   .. code-block:: sh
+
+      # This says to not use the eth0 and lo interfaces.
+      # (and implicitly use all the rest).  Per the description
+      # above, IP loopback and all local-only devices *must*
+      # be included if the exclude list is specified.
+      shell$ mpirun --mca btl_tcp_if_exclude lo,eth0 ...
+
+      # This says to only use the eth1 and eth2 interfaces
+      # (and implicitly ignore the rest)
+      shell$ mpirun --mca btl_tcp_if_include eth1,eth2 ...
+
+#. You can  also specify subnets  in the  include or exclude  lists in
+   CIDR notation.  For example:
+
+   .. code-block:: sh
+
+      # Only use the 192.168.1.0/24 and 10.10.0.0/16 subnets for MPI
+      # communications:
+      shell$ mpirun --mca btl_tcp_if_include 192.168.1.0/24,10.10.0.0/16 ...
+
+
+   .. note:: You must specify the CIDR notation for a given network
+             precisely.  For example, if you have two IP networks
+             10.10.0.0/24 and 10.10.1.0/24, Open MPI will not
+             recognize either of them if you specify "10.10.0.0/16".
+
+.. warning:: If you use the ``btl_tcp_if_include`` and
+             ``btl_tcp_if_exclude`` MCA parameters to shape the
+             behavior of the TCP BTL for MPI communications, you may
+             also need/want to investigate the corresponding MCA
+             parameters ``oob_tcp_if_include`` and
+             ``oob_tcp_if_exclude``, which are used to shape non-MPI
+             TCP-based communication (e.g., communications setup and
+             coordination during ``MPI_INIT`` and ``MPI_FINALIZE``).
+
+.. error:: TODO do corresponding OOB TCP params still exist in PMIx?
+
+Note that Open MPI will still use TCP for control messages, such as
+data between ``mpirun`` and the MPI processes, rendezvous information
+during ``MPI_INIT``, etc.  To disable TCP altogether, you also need to
+disable the ``tcp`` component from the OOB framework.
+
+.. error:: TODO Is this possible in PMIx?  I doubt it...?
+
+/////////////////////////////////////////////////////////////////////////
+
+.. _faq-tcp-sockets:
+
+Does Open MPI open a bunch of sockets during ``MPI_INIT``?
+----------------------------------------------------------
+
+Although Open MPI is likely to open multiple TCP sockets during
+``MPI_INIT``, the ``tcp`` BTL component *does not open one socket per
+MPI peer process during MPI_INIT.*  Open MPI opens sockets as they
+are required |mdash| so the first time a process sends a message to a
+peer and there is no TCP connection between the two, Open MPI will
+automatically open a new socket.
+
+Hence, you should not have scalability issues with running large
+numbers of processes (e.g., running out of per-process file
+descriptors) if your parallel application is sparse in its
+communication with peers.
+
+/////////////////////////////////////////////////////////////////////////
+
+Are there any Linux kernel TCP parameters that I should set?
+------------------------------------------------------------
+
+Everyone has different opinions on this, and it also depends
+on your exact hardware and environment.  Below are general guidelines
+that some users have found helpful.
+
+#. ``net.ipv4.tcp_syn_retries``: Some Linux systems have very large
+   initial connection timeouts |mdash| they retry sending SYN packets
+   many times before determining that a connection cannot be made.  If
+   MPI is going to fail to make socket connections, it would be better
+   for them to fail somewhat quickly (minutes vs. hours).  You might
+   want to reduce this value to a smaller value; YMMV.
+
+#. ``net.ipv4.tcp_keepalive_time``: Some MPI applications send an
+   initial burst of MPI messages (over TCP) and then send nothing for
+   long periods of time (e.g., embarrassingly parallel applications).
+   Linux may decide that these dormant TCP sockets are dead because it
+   has seen no traffic on them for long periods of time.  You might
+   therefore need to lengthen the TCP inactivity timeout.  Many Linux
+   systems default to 7,200 seconds; increase it if necessary.
+
+#. Increase TCP buffering for 10G or 40G Ethernet.  Many Linux
+   distributions come with good buffering presets for 1G Ethernet.  In
+   a datacenter/HPC cluster with 10G or 40G Ethernet NICs, this amount
+   of kernel buffering is typically insufficient.  Here's a set of
+   parameters that some have used for good 10G/40G TCP bandwidth:
+
+   * ``net.core.rmem_max``: 16777216
+   * ``net.core.wmem_max``: 16777216
+   * ``net.ipv4.tcp_rmem``: 4096 87380 16777216
+   * ``net.ipv4.tcp_wmem``: 4096 65536 16777216
+   * ``net.core.netdev_max_backlog``: 30000
+   * ``net.core.rmem_default``: 16777216
+   * ``net.core.wmem_default``: 16777216
+   * ``net.ipv4.tcp_mem``: '16777216 16777216 16777216'
+   * ``net.ipv4.route.flush``: 1
+
+   Each of the above items is a Linux kernel parameter that can be set
+   in multiple different ways.
+
+   #. You can change the running kernel via the ``/proc`` filesystem:
+
+      .. code-block:: sh
+
+         shell# cat /proc/sys/net/ipv4/tcp_syn_retries
+         5
+         shell# echo 6 > /proc/sys/net/ipv4/tcp_syn_retries
+
+   #. You can also use the ``sysctl`` command:
+
+      .. code-block:: sh
+
+         shell# sysctl net.ipv4.tcp_syn_retries
+         net.ipv4.tcp_syn_retries = 5
+         shell# sysctl -w net.ipv4.tcp_syn_retries=6
+         net.ipv4.tcp_syn_retries = 6
+
+   #. Or you can set them by adding entries in ``/etc/sysctl.conf``,
+      which are persistent across reboots:
+
+      .. code-block:: sh
+
+         shell$ grep tcp_syn_retries /etc/sysctl.conf
+         net.ipv4.tcp_syn_retries = 6
+
+   #. Your Linux distro may also support putting individual files in
+      ``/etc/sysctl.d`` (even if that directory does not yet exist),
+      which is actually better practice than putting them in
+      ``/etc/sysctl.conf``.  For example:
+
+      .. code-block:: sh
+
+         shell$ cat /etc/sysctl.d/my-tcp-settings
+         net.ipv4.tcp_syn_retries = 6
+
+/////////////////////////////////////////////////////////////////////////
+
+.. _faq-tcp-routability:
+
+How does Open MPI know which IP addresses are routable to each other?
+---------------------------------------------------------------------
+
+Open MPI assumes that all interfaces are routable as long as they have
+the same address family, IPv4 or IPv6.  We use graph theory and give
+each possible connection a weight depending on the quality of the
+connection.  This allows the library to select the best connections
+between nodes.  This method also supports striping but prevents more
+than one connection to any interface.
+
+The quality of the connection is defined as follows, with a higher
+number meaning better connection.  Note that when giving a weight to a
+connection consisting of a private address and a public address, it
+will give it the weight of ``PRIVATE_DIFFERENT_NETWORK``.
+
+.. code-block::
+
+               NO_CONNECTION = 0
+   PRIVATE_DIFFERENT_NETWORK = 1
+   PRIVATE_SAME_NETWORK      = 2
+   PUBLIC_DIFFERENT_NETWORK  = 3
+   PUBLIC_SAME_NETWORK       = 4
+
+An example will best illustrate how two processes on two different
+nodes would connect up.  Here we have two nodes with a variety of
+interfaces:
+
+.. code-block::
+
+            Node A                Node B
+      ----------------       ----------------
+     |      lo0       |     |      lo0       |
+     | 127.0.0.1/8    |     | 127.0.0.1/8    |
+     |                |     |                |
+     |      eth0      |     |      eth0      |
+     | 10.8.47.1/24   |     | 10.8.47.2/24   |
+     |                |     |                |
+     |      eth1      |     |      eth1      |
+     | 192.168.1.1/24 |     | 192.168.1.2/24 |
+     |                |     |                |
+     |      eth2      |     |                |
+     | 192.168.2.2/24 |     |                |
+      ----------------      ------------------
+
+From these two nodes, the software builds up a bipartite graph that
+shows all the possible connections with all the possible weights.  The
+*lo0* interfaces are excluded as the ``btl_tcp_if_exclude`` MCA parameter
+is set to *lo* by default.  Here is what all the possible connections
+with their weights look like.
+
+.. code-block::
+
+         Node A       Node B
+   eth0 --------- 2 -------- eth0
+          ------- 1 -------- eth1
+
+   eth1 --------- 1 -------- eth0
+          ------- 2 -------- eth1
+
+   eth2 --------- 1 -------- eth0
+          ------- 1 -------- eth1
+
+The library then examines all the connections and picks the optimal
+ones.  This leaves us with two connections being established between
+the two nodes.
+
+If you are curious about the actual ``connect()`` calls being made by
+the processes, then you can run with ``--mca btl_base_verbose 30``.
+This can be useful if you notice your job hanging and believe it may
+be the library trying to make connections to unreachable hosts.
+
+.. code-block:: sh
+
+   # Here is an example with some of the output deleted for clarity.
+   # One can see the connections that are attempted.
+   shell$ mpirun --mca btl self,sm,tcp --mca btl_base_verbose 30 -n 2 -host NodeA,NodeB a.out
+   [...snip...]
+   [NodeA:18003] btl: tcp: attempting to connect() to address 10.8.47.2 on port 59822
+   [NodeA:18003] btl: tcp: attempting to connect() to address 192.168.1.2 on port 59822
+   [NodeB:16842] btl: tcp: attempting to connect() to address 192.168.1.1 on port 44500
+   [...snip...]
+
+In case you want more details about the theory behind the connection
+code, you can find the background story in `this IEEE paper
+<https://ieeexplore.ieee.org/document/4476565>`_.
+
+/////////////////////////////////////////////////////////////////////////
+
+Does Open MPI ever close TCP sockets?
+-------------------------------------
+
+In general, no.
+
+Although TCP sockets are opened "lazily" (meaning that MPI
+connections / TCP sockets are only opened upon demand |mdash| as opposed to
+opening all possible sockets between MPI peer processes during
+``MPI_INIT``), they are never closed.
+
+/////////////////////////////////////////////////////////////////////////
+
+Does Open MPI support IP interfaces that have more than one IP address?
+-----------------------------------------------------------------------
+
+In general, no.
+
+For example, if the output from your ``ifconfig`` has a single IP device
+with multiple IP addresses like this:
+
+.. code-block::
+
+   0: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP qlen 1000
+      link/ether 00:18:ae:f4:d2:29 brd ff:ff:ff:ff:ff:ff
+      inet 192.168.0.3/24 brd 192.168.0.255 scope global eth0:1
+      inet 10.10.0.3/24 brf 10.10.0.255 scope global eth0
+      inet6 fe80::218:aef2:29b4:2c4/64 scope link
+         valid_lft forever preferred_lft forever
+
+(note the two ``inet`` lines in there)
+
+Then Open MPI will be unable to use this device.
+
+/////////////////////////////////////////////////////////////////////////
+
+Does Open MPI support virtual IP interfaces?
+--------------------------------------------
+
+No.
+
+For example, if the output of your ``ifconfig`` has both ``eth0`` and
+``eth0:0``, Open MPI will get confused if you use the TCP BTL, and
+may hang or otherwise act unpredictably.
+
+Note that using ``btl_tcp_if_include`` or ``btl_tcp_if_exclude`` to avoid
+using the virtual interface will *not* solve the issue.
+
+/////////////////////////////////////////////////////////////////////////
+
+Can I use multiple TCP connections to improve network performance?
+------------------------------------------------------------------
+
+Open MPI can use multiple TCP connections between any pair of MPI
+processes, striping large messages across the connections.  The
+``btl_tcp_links`` parameter can be used to set how many TCP
+connections should be established between MPI processes.
+
+Note that
+this may not improve application performance for common use cases of
+nearest-neighbor exchanges when there many MPI processes on each host.  In
+these cases, there are already many TCP connections between any two
+hosts (because of the many processes all communicating), so the extra TCP
+connections are likely just consuming extra resources and adding work
+to the MPI implementation.
+
+However, for highly multi-threaded applications, where there are only
+one or two MPI processes per host, the ``btl_tcp_links`` option may
+improve TCP throughput considerably.
diff --git a/docs/tuning-apps/tuned_dynamic_file_schema.json b/docs/tuning-apps/tuned_dynamic_file_schema.json
new file mode 100644
index 00000000000..844bcd2444c
--- /dev/null
+++ b/docs/tuning-apps/tuned_dynamic_file_schema.json
@@ -0,0 +1,130 @@
+{
+    "$schema": "https://json-schema.org/draft/2019-09/schema#",
+    "title": "OpenMPITunedRules",
+    "description": "Defines configuration for the Open MPI Tuned module to select which collective algorithms will be used depending on comm size, message size, etc.",
+    "type": "object",
+    "required": ["rule_file_version","module","collectives"],
+    "additionalProperties" : false,
+    "properties": {
+      "rule_file_version": {
+        "description": "The version of this configuration file",
+        "type": "number"
+      },
+      "module": {
+        "description": "The collective module intended to use these rules (tuned)",
+        "type": "string"
+      },
+      "$schema": {
+        "description": "The schema used for validation",
+        "type": "string"
+      },
+      "collectives" : {
+        "description": "The collectives, each with their own rules.  Each collective is indicated by a lowercase property such as \"allgather\"",
+        "type": "object",
+        "additionalProperties" : false,
+        "patternProperties": {
+            "^(allgather|allreduce|alltoall|alltoallv|alltoallw|barrier)$": {
+                "type" : "array",
+                "items": { "$ref" : "#/$defs/comm_size_rule" }
+            },
+            "^(bcast|exscan|gather|gatherv|reduce|reducescatter|reducescatterblock)$": {
+                "type" : "array",
+                "items": { "$ref" : "#/$defs/comm_size_rule" }
+            },
+            "^(scan|scatter|scatterv|neighbor_allgather|neighbor_allgatherv)$": {
+                "type" : "array",
+                "items": { "$ref" : "#/$defs/comm_size_rule" }
+            },
+            "^(neighbor_alltoall|neighbor_alltoallv|neighbor_alltoallw)$": {
+                "type" : "array",
+                "items": { "$ref" : "#/$defs/comm_size_rule" }
+            }
+        }
+      }
+    },
+
+    "$defs": {
+        "msg_size_rule": {
+            "type": "object",
+            "required": ["alg"],
+            "additionalProperties" : false,
+            "properties" : {
+                "msg_size_min" : {
+                    "description" : "The smallest message size in bytes this rule applies to",
+                    "anyOf" : { "$ref" : "#/$defs/int_or_inf" }
+                },
+                "msg_size_max" : {
+                    "description" : "The largest message size (inclusive) in bytes this rule applies to",
+                    "anyOf" : { "$ref" : "#/$defs/int_or_inf" }
+                },
+                "alg" : {
+                    "description" : "The algorithm to use for this collective. Integer or name, see coll_tuned_<collective>_algorithm for options.",
+                    "type" : [ "string", "integer"]
+                },
+                "reqs" : {
+                    "description" : "Algorithm parameter: Use this many requests.  Some algorithms may ignore this option.",
+                    "type" : [ "integer"]
+                },
+                "faninout" : {
+                    "description" : "Algorithm parameter: Fan in and/or out by this much.  Some algorithms may ignore this option.",
+                    "type" : [ "integer"]
+                }
+            }
+        },
+
+        "comm_size_rule": {
+            "type": "object",
+            "required": ["rules"],
+            "additionalProperties" : false,
+            "properties" : {
+                "comm_size_min" : {
+                    "description" : "The smallest size communicator these rules apply to",
+                    "anyOf" :  { "$ref" : "#/$defs/int_or_inf" }
+                },
+                "comm_size_max" : {
+                    "description" : "The largest (inclusive) size communicator these rules apply to",
+                    "anyOf" :  { "$ref" : "#/$defs/int_or_inf" }
+                },
+                "comm_rank_distribution" : {
+                    "description" : "A description of how the ranks are distributed within the communicator",
+                    "enum" : ["any", "one-per-node", "single-node"]
+                },
+
+                "rules" : {
+                    "description" : "A list of rules.  The first matching rule is selected.  If no match is found, defaults are used.",
+                    "type" : "array",
+                    "items":  { "$ref" : "#/$defs/msg_size_rule" }
+                }
+            }
+        },
+        "collective_identifier": {
+            "enum" : [
+                "allgather",
+                "allreduce",
+                "alltoall",
+                "alltoallv",
+                "alltoallw",
+                "barrier",
+                "bcast",
+                "exscan",
+                "gather",
+                "gatherv",
+                "reduce",
+                "reducescatter",
+                "reducescatterblock",
+                "scan",
+                "scatter",
+                "scatterv",
+                "neighbor_allgather",
+                "neighbor_allgatherv",
+                "neighbor_alltoall",
+                "neighbor_alltoallv",
+                "neighbor_alltoallw"
+            ]
+        },
+        "int_or_inf": [
+            { "type" : "integer" },
+            { "enum": ["inf","INF","Inf"] }
+        ]
+    }
+}
\ No newline at end of file
diff --git a/docs/validate.rst b/docs/validate.rst
new file mode 100644
index 00000000000..ea32a11902c
--- /dev/null
+++ b/docs/validate.rst
@@ -0,0 +1,71 @@
+Validating your installation
+============================
+
+Checking your Open MPI configuration
+------------------------------------
+
+The :ref:`ompi_info(1) <man1-ompi_info>` command can be used to check
+the status of your Open MPI installation (located in
+``$prefix/bin/ompi_info``).  Running it with no arguments provides a
+summary of information about your Open MPI installation.
+
+Note that the :ref:`ompi_info(1) <man1-ompi_info>` command is
+extremely helpful in determining which components are installed as
+well as listing all the run-time settable parameters that are
+available in each component (as well as their default values).
+
+The following :ref:`ompi_info(1) <man1-ompi_info>` options may be
+helpful:
+
+* ``--all``: Show a *lot* of information about your Open MPI
+  installation.
+* ``--parsable``: Display all the information in a machine-parsable
+  format.
+* ``--param FRAMEWORK COMPONENT``:
+  A ``FRAMEWORK`` value of ``all`` and a ``COMPONENT`` value of ``all`` will
+  show all parameters to all components.  Otherwise, the parameters of
+  all the components in a specific framework, or just the parameters
+  of a specific component can be displayed by using an appropriate
+  FRAMEWORK and/or COMPONENT name.
+* ``--level LEVEL``:
+  By default, ``ompi_info`` only shows "Level 1" MCA parameters |mdash|
+  parameters that can affect whether MPI processes can run
+  successfully or not (e.g., determining which network interfaces to
+  use).  The ``--level`` option will display all MCA parameters from
+  level 1 to ``LEVEL`` (the max ``LEVEL`` value is 9).  Use ``ompi_info
+  --param FRAMEWORK COMPONENT --level 9`` to see *all* MCA parameters
+  for a given component.  See "The Modular Component Architecture
+  (MCA)" section, below, for a fuller explanation.
+
+Changing the values of these MCA parameters is explained in the
+:ref:`Tuning section <label-running-setting-mca-param-values>`.
+
+
+Testing your Open MPI installation
+----------------------------------
+
+When verifying a new Open MPI installation, we recommend running the
+following tests in order (the tests build upon each other):
+
+#. Use :ref:`mpirun(1) <man1-mpirun>` to launch a non-MPI program
+   (e.g., ``hostname`` or ``uptime``) across multiple nodes.
+#. Use :ref:`mpirun(1) <man1-mpirun>` to launch a trivial MPI program
+   that does no MPI communication (e.g., the ``hello_c`` program in
+   the ``examples/`` directory in the Open MPI distribution).
+#. Use :ref:`mpirun(1) <man1-mpirun>` to launch a trivial MPI program
+   that sends and receives a few MPI messages (e.g., the ``ring_c``
+   program in the ``examples/`` directory in the Open MPI
+   distribution).
+#. Use :ref:`oshrun(1) <man1-oshrun>` to launch a non-OpenSHMEM
+   program across multiple nodes.
+#. Use :ref:`oshrun(1) <man1-oshrun>` to launch a trivial MPI program
+   that does no OpenSHMEM communication (e.g., ``hello_shmem.c``
+   program in the ``examples/`` directory in the Open MPI
+   distribution.)
+#. Use :ref:`oshrun <man1-oshrun>` to launch a trivial OpenSHMEM
+   program that puts and gets a few messages (e.g., the
+   ``ring_shmem.c`` in the ``examples/`` directory in the Open MPI
+   distribution.)
+
+If you can run all of these tests successfully, that is a good
+indication that Open MPI built and installed properly.
diff --git a/docs/version-numbering.rst b/docs/version-numbering.rst
new file mode 100644
index 00000000000..b96da23de91
--- /dev/null
+++ b/docs/version-numbering.rst
@@ -0,0 +1,204 @@
+.. _version_numbers_section_label:
+
+Version numbers and compatibility
+=================================
+
+Open MPI has two sets of version numbers that are likely of interest
+to end users / system administrators:
+
+#. Software version number
+#. Shared library version numbers
+
+Both are predicated on Open MPI's definition of "backward compatibility."
+
+Backward Compatibility
+----------------------
+
+Open MPI version Y is backward compatible with Open MPI version X
+(where Y>X) if users can:
+
+* Compile an MPI/OpenSHMEM application with version X,
+  ``mpirun``/``oshrun`` it with version Y, and get the same
+  user-observable behavior.
+* Invoke ``ompi_info`` with the same CLI options in versions X and Y and
+  get the same user-observable behavior.
+
+Note that this definition encompasses several things:
+
+* Application Binary Interface (ABI)
+* MPI / OpenSHMEM run time system
+* ``mpirun`` / ``oshrun`` command line options
+* MCA parameter names / values / meanings
+
+However, this definition only applies when the same version of Open
+MPI is used with all instances of the runtime and MPI / OpenSHMEM
+processes in a single MPI job.  If the versions are not exactly the
+same everywhere, Open MPI is not guaranteed to work properly in any
+scenario.
+
+Backward compatibility tends to work best when user applications are
+dynamically linked to one version of the Open MPI / OSHMEM libraries,
+and can be updated at run time to link to a new version of the Open
+MPI / OSHMEM libraries.
+
+For example, if an MPI / OSHMEM application links statically against
+the libraries from Open MPI vX, then attempting to launch that
+application with ``mpirun`` / ``oshrun`` from Open MPI vY is not guaranteed to
+work (because it is mixing vX and vY of Open MPI in a single job).
+
+Similarly, if using a container technology that internally bundles all
+the libraries from Open MPI vX, attempting to launch that container
+with ``mpirun`` / ``oshrun`` from Open MPI vY is not guaranteed to work.
+
+Open MPI |ompi_series| MCA parameter compatibility
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Several MCA parameters have been deprecated in Open MPI |ompi_series|, please
+see this :ref:`table <label-mca-backward-compat>` for the full list.
+
+Software Version Number
+-----------------------
+
+Official Open MPI releases use the common "A.B.C" version identifier
+format.  Each of the three numbers has a specific meaning:
+
+* Major: The major number is the first integer in the version string
+  Changes in the major number typically indicate a significant
+  change in the code base and/or end-user functionality, and also
+  indicate a break from backward compatibility.  Specifically: Open
+  MPI releases with different major version numbers are not
+  backward compatibale with each other.
+
+  .. important:: This rule does not extend to versions prior to
+     v1.10.0.  Specifically: v1.10.x is not guaranteed to be backward
+     compatible with other v1.x releases.
+
+* Minor: The minor number is the second integer in the version string.
+  Changes in the minor number indicate a user-observable change in the
+  code base and/or end-user functionality.  Backward compatibility
+  will still be preserved with prior releases that have the same major
+  version number (e.g., v2.5.3 is backward compatible with v2.3.1).
+
+* Release: The release number is the third integer in the version
+  string.  Changes in the release number typically indicate a bug fix
+  in the code base and/or end-user functionality.  For example, if
+  there is a release that only contains bug fixes and no other
+  user-observable changes or new features, only the third integer will
+  be increased (e.g., from v4.3.0 to v4.3.1).
+
+The "A.B.C" version number may optionally be followed by a quantifier
+string:
+
+* ``aX``: Indicates an alpha release. X is an integer indicating the
+  number of the alpha release (e.g., v1.10.3a5 indicates the 5th alpha
+  release of version 1.10.3).
+* ``bX``: Indicates a beta release. X is an integer indicating the
+  number of the beta release (e.g., v1.10.3b3 indicates the 3rd beta
+  release of version 1.10.3).
+* ``rcX``: Indicates a release candidate. X is an integer indicating
+  the number of the release candidate (e.g., v1.10.3rc4 indicates the
+  4th release candidate of version 1.10.3).
+
+Nightly development snapshot tarballs use a different version number
+scheme; they contain three distinct values:
+
+* The git branch name from which the tarball was created.
+* The date/timestamp, in ``YYYYMMDDHHMM`` format.
+* The hash of the git commit from which the tarball was created.
+
+For example, a snapshot tarball filename of
+``openmpi-v2.x-201703070235-e4798fb.tar.bz2`` indicates that this tarball
+was created from the v2.x branch, on March 7, 2017, at 2:35am GMT,
+from git hash e4798fb.
+
+
+Shared Library Version Number
+-----------------------------
+
+The `GNU Libtool official documentation
+<https://www.gnu.org/software/libtool/manual/>`_ details how the
+versioning scheme works.  The quick version is that the shared library
+versions are a triple of integers: (current,revision,age), or
+``c:r:a``.  This triple is not related to the Open MPI software
+version number.  There are six simple rules for updating the values
+(taken almost verbatim from the Libtool docs):
+
+#. Start with version information of ``0:0:0`` for each shared library.
+#. Update the version information only immediately before a public
+   release of your software. More frequent updates are unnecessary,
+   and only guarantee that the current interface number gets larger
+   faster.
+#. If the library source code has changed at all since the last
+   update, then increment revision (``c:r:a`` becomes ``c:r+1:a``).
+#. If any interfaces have been added, removed, or changed since the
+   last update, increment current, and set revision to 0.
+#. If any interfaces have been added since the last public release,
+   then increment age.
+#. If any interfaces have been removed since the last public release,
+   then set age to 0.
+
+Here's how we apply those rules specifically to Open MPI:
+
+#. The above rules do not apply to MCA components (a.k.a. "plugins");
+   MCA component ``.so`` versions stay unspecified.
+#. The above rules apply exactly as written to the following libraries
+   starting with Open MPI version v1.5:
+
+    * ``libopen-pal``
+    * ``libmca_common_*``
+
+#. The following libraries use a slightly modified version of the
+   above rules: rules 4, 5, and 6 only apply to the official MPI and
+   OpenSHMEM interfaces (functions, global variables).  The rationale
+   for this decision is that the vast majority of our users only care
+   about the official/public MPI/OpenSHMEM interfaces; we therefore
+   want the ``.so`` version number to reflect only changes to the
+   official MPI/OpenSHMEM APIs.  Put simply: non-MPI/OpenSHMEM API /
+   internal changes to the MPI-application-facing libraries are
+   irrelevant to pure MPI/OpenSHMEM applications.
+
+   * ``libmpi``
+   * ``libmpi_mpifh``
+   * ``libmpi_usempi_tkr``
+   * ``libmpi_usempi_ignore_tkr``
+   * ``libmpi_usempif08``
+   * ``libmpi_cxx``
+   * ``libmpi_java``
+   * ``liboshmem``
+
+API and ABI Compatibility
+-------------------------
+
+Open MPI provides the following Application Programming Interface
+(API) and Application Binary Interface (ABI) compatibility guarantees
+for applications:
+
+#. Open MPI is source code compatible (i.e., API compatible) across all
+   versions.  This means that you can compile and link your compliant MPI
+   application against :ref:`any version of Open MPI that supports the version
+   of the MPI standard <release-notes-mpi-standard-conformance-label>` to
+   which your application was written.
+
+#. Open MPI provided forward application binary interface (ABI)
+   compatibility within a major series for MPI applications starting
+   with v1.3.2.  Prior to that version, no ABI guarantees were
+   provided.
+
+#. Open MPI reserves the right to break ABI compatibility at new major
+   release series.
+
+Open MPI |ompi_series| ABI compatibility
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The Open MPI |ompi_series| series shared libraries are ABI compatible
+with Open MPI v4.0.x and v4.1.x, with a few exceptions for Fortran.
+
+* If your Fortran application was compiled in such a way that the size
+  of an integer in C is different than the size of an integer in
+  Fortran, you will need to rebuild and relink your application
+  against Open MPI |ompi_series|.
+
+* There are also Fortran API changes involving intents and
+  ``ASYNCHRONOUS``, and some interfaces changed from named to unnamed.
+  These may require changes to an application's source code, followed
+  by recompilation and relinking.
diff --git a/examples/Hello.java b/examples/Hello.java
index 4f2ac4cfb15..cd7c5268fa2 100644
--- a/examples/Hello.java
+++ b/examples/Hello.java
@@ -3,7 +3,7 @@
     you may not use this file except in compliance with the License.
     You may obtain a copy of the License at
 
-       http://www.apache.org/licenses/LICENSE-2.0
+       https://www.apache.org/licenses/LICENSE-2.0
 
     Unless required by applicable law or agreed to in writing, software
     distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/examples/Makefile b/examples/Makefile
index 5d32bb00973..269ed954175 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -65,14 +65,15 @@ EXAMPLES = \
 	 oshmem_max_reduction \
 	 oshmem_strided_puts \
 	 oshmem_symmetric_data \
-	 spc_example
+	 spc_example \
+         hello_sessions_c
 
 
 # Default target.  Always build the C MPI examples.  Only build the
 # others if we have the appropriate Open MPI / OpenSHMEM language
 # bindings.
 
-all: hello_c ring_c connectivity_c spc_example
+all: hello_c ring_c connectivity_c spc_example hello_sessions_c
 	@ if which ompi_info >/dev/null 2>&1 ; then \
 	    $(MAKE) mpi; \
 	fi
@@ -86,7 +87,7 @@ mpi:
 	@ if ompi_info --parsable | grep -q bindings:mpif.h:yes >/dev/null; then \
 	    $(MAKE) hello_mpifh ring_mpifh; \
 	fi
-	@ if ompi_info --parsable | egrep -q bindings:use_mpi:\"\?yes >/dev/null; then \
+	@ if ompi_info --parsable | grep -E -q bindings:use_mpi:\"\?yes >/dev/null; then \
 	    $(MAKE) hello_usempi ring_usempi; \
 	fi
 	@ if ompi_info --parsable | grep -q bindings:use_mpi_f08:yes >/dev/null; then \
@@ -129,6 +130,8 @@ connectivity_c: connectivity_c.c
 	$(MPICC) $(CFLAGS) $(LDFLAGS) $? $(LDLIBS) -o $@
 spc_example: spc_example.c
 	$(MPICC) $(CFLAGS) $(LDFLAGS) $? $(LDLIBS) -o $@
+hello_sessions_c: hello_sessions_c.c
+	$(MPICC) $(CFLAGS) $(LDFLAGS) $? $(LDLIBS) -o $@
 
 hello_mpifh: hello_mpifh.f
 	$(MPIFC) $(FCFLAGS) $(LDFLAGS) $? $(LDLIBS) -o $@
diff --git a/examples/Makefile.include b/examples/Makefile.include
index 92afc1175bc..8da106cb507 100644
--- a/examples/Makefile.include
+++ b/examples/Makefile.include
@@ -56,4 +56,5 @@ EXTRA_DIST += \
         examples/oshmem_symmetric_data.c \
         examples/Hello.java \
         examples/Ring.java \
-        examples/spc_example.c
+        examples/spc_example.c \
+        examples/hello_sessions_c.c
diff --git a/examples/README.md b/examples/README.md
index ad924d2e792..6a1ab9528cb 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -4,7 +4,7 @@ Open MPI installation is working properly.
 
 If you are looking for a comprehensive MPI tutorial, these samples are
 not enough.  [Excellent MPI tutorials are available
-here](http://www.citutor.org/login.php).
+here](http://www.hpc-training.org/).
 
 Get a free account and login; you can then browse to the list of
 available courses.  Look for the ones with "MPI" in the title.
@@ -50,7 +50,7 @@ processes:
 
 The `Makefile` in this directory will build as many of the examples as
 you have language support (e.g., if you do not have the Fortran `use
-mpi` bindings compiled as part of Open MPI, the those examples will be
+mpi` bindings compiled as part of Open MPI, those examples will be
 skipped).
 
 The `Makefile` assumes that the wrapper compilers `mpicc`, `mpic++`, and
diff --git a/examples/Ring.java b/examples/Ring.java
index 08cd864663e..6aa3770e21b 100644
--- a/examples/Ring.java
+++ b/examples/Ring.java
@@ -29,7 +29,7 @@ static public void main(String[] args) throws MPIException {
 	next = (myrank + 1) % size;
 	prev = (myrank + size - 1) % size;
 
-	/* If we are the "master" process (i.e., MPI_COMM_WORLD rank 0),
+	/* If we are the "manager" process (i.e., MPI_COMM_WORLD rank 0),
 	   put the number of times to go around the ring in the
 	   message. */
 
diff --git a/examples/connectivity_c.c b/examples/connectivity_c.c
index 14f3f546d53..7e650d739f3 100644
--- a/examples/connectivity_c.c
+++ b/examples/connectivity_c.c
@@ -6,26 +6,21 @@
  * Test the connectivity between all processes.
  */
 
-#include <errno.h>
+#include <mpi.h>
 #include <stdio.h>
-#include <stdlib.h>
 #include <string.h>
-#include <netdb.h>
-#include <unistd.h>
-#include <mpi.h>
 
-int
-main(int argc, char **argv)
+int main(int argc, char **argv)
 {
-    MPI_Status  status;
-    int         verbose = 0;
-    int         rank;
-    int         np;        /* number of processes in job */
-    int         peer;
-    int         i;
-    int         j;
-    int         length;
-    char        name[MPI_MAX_PROCESSOR_NAME+1];
+    MPI_Status status;
+    int verbose = 0;
+    int rank;
+    int np; /* number of processes in job */
+    int peer;
+    int i;
+    int j;
+    int length;
+    char name[MPI_MAX_PROCESSOR_NAME + 1];
 
     MPI_Init(&argc, &argv);
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
@@ -38,20 +33,19 @@ main(int argc, char **argv)
         strcpy(name, "unknown");
     }
 
-    if (argc>1 && strcmp(argv[1], "-v")==0)
+    if (argc > 1 && strcmp(argv[1], "-v") == 0)
         verbose = 1;
 
-    for (i=0; i<np; i++) {
-        if (rank==i) {
+    for (i = 0; i < np; i++) {
+        if (rank == i) {
             /* rank i sends to and receives from each higher rank */
-            for(j=i+1; j<np; j++) {
+            for (j = i + 1; j < np; j++) {
                 if (verbose)
-                    printf("checking connection between rank %d on %s and rank %-4d\n",
-                           i, name, j);
+                    printf("checking connection between rank %d on %s and rank %-4d\n", i, name, j);
                 MPI_Send(&rank, 1, MPI_INT, j, rank, MPI_COMM_WORLD);
                 MPI_Recv(&peer, 1, MPI_INT, j, j, MPI_COMM_WORLD, &status);
             }
-        } else if (rank>i) {
+        } else if (rank > i) {
             /* receive from and reply to rank i */
             MPI_Recv(&peer, 1, MPI_INT, i, i, MPI_COMM_WORLD, &status);
             MPI_Send(&rank, 1, MPI_INT, i, rank, MPI_COMM_WORLD);
@@ -59,7 +53,7 @@ main(int argc, char **argv)
     }
 
     MPI_Barrier(MPI_COMM_WORLD);
-    if (rank==0)
+    if (rank == 0)
         printf("Connectivity test on %d processes PASSED.\n", np);
 
     MPI_Finalize();
diff --git a/examples/dtrace/README b/examples/dtrace/README
index 8df35acca8a..42e3378eaae 100644
--- a/examples/dtrace/README
+++ b/examples/dtrace/README
@@ -61,7 +61,7 @@ perform the following steps:
   2.  Type the following command but replace the hostnames in the example with the
       names of the hosts in your cluster.
 
-	% mpirun -np 2 --host burl-ct-v440-4,burl-ct-v440-5 myppriv.sh
+	% mpirun -n 2 --host burl-ct-v440-4,burl-ct-v440-5 myppriv.sh
 
 
 If the output of ppriv shows that the E privilege set has the dtrace
@@ -109,7 +109,7 @@ There are two ways to use Dynamic Tracing with MPI programs:
 For illustration purposes, assume you have a program named mpiapp. To trace
 the program mpiapp using the mpitrace.d script, type the following command:
 
-% mpirun -np 4 dtrace -s mpitrace.d -c mpiapp
+% mpirun -n 4 dtrace -s mpitrace.d -c mpiapp
 
 The advantage of tracing an MPI program in this way is that all the processes
 in the job will be traced from the beginning.  This method is probably most
@@ -131,7 +131,7 @@ dtrace -s $1 -c $2 -o $2.$OMPI_COMM_WORLD_RANK.trace
 
 Type the following command to run the partrace.sh shell script:
 
-% mpirun -np 4 partrace.sh mpitrace.d mpiapp
+% mpirun -n 4 partrace.sh mpitrace.d mpiapp
 
 This will run mpiapp under dtrace using the mpitrace.d script.  The script
 saves the trace output for each process in a job under a separate file name,
diff --git a/examples/dtrace/mpicommleak.c b/examples/dtrace/mpicommleak.c
index b2fcb8a25cf..2a5c737ab09 100644
--- a/examples/dtrace/mpicommleak.c
+++ b/examples/dtrace/mpicommleak.c
@@ -9,10 +9,10 @@
  */
 
 #include <errno.h>
+#include <netdb.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <netdb.h>
 #include <unistd.h>
 
 #include <mpi.h>
@@ -23,39 +23,40 @@ static void allocate_comms(void);
 static void deallocate_comms(void);
 static MPI_Comm communicator_a, communicator_b, communicator_c;
 
-int
-main(int argc, char *argv[])
+int main(int argc, char *argv[])
 {
-    int		rank;                         /* COMM_WORLD rank of process */
-    int		np;	                      /* number of processes in job */
-    int         i;
-    int         unslept;
+    int rank; /* COMM_WORLD rank of process */
+    int np;   /* number of processes in job */
+    int i;
+    int unslept;
 
     MPI_Init(&argc, &argv);
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &np);
 
-    for (i=0; i < ITERATIONS; i++) {
-	/* allocate our communicators */
-	allocate_comms();
+    for (i = 0; i < ITERATIONS; i++) {
+        /* allocate our communicators */
+        allocate_comms();
 
-	/* simulate doing work */
-	unslept = sleep(5);
+        /* simulate doing work */
+        unslept = sleep(5);
 
-	/* deallocate communicators forgetting one of the communicators */
-	deallocate_comms();
+        /* deallocate communicators forgetting one of the communicators */
+        deallocate_comms();
     }
     MPI_Finalize();
     return 0;
 }
 
-void allocate_comms(void) {
+void allocate_comms(void)
+{
     MPI_Comm_dup(MPI_COMM_WORLD, &communicator_a);
     MPI_Comm_dup(MPI_COMM_WORLD, &communicator_b);
     MPI_Comm_dup(MPI_COMM_WORLD, &communicator_c);
 }
 
-void deallocate_comms(void) {
+void deallocate_comms(void)
+{
     MPI_Comm_free(&communicator_a);
     MPI_Comm_free(&communicator_c);
 }
diff --git a/examples/hello_c.c b/examples/hello_c.c
index e44f6843702..ba30e167062 100644
--- a/examples/hello_c.c
+++ b/examples/hello_c.c
@@ -7,10 +7,10 @@
  * Sample MPI "hello world" application in C
  */
 
-#include <stdio.h>
 #include "mpi.h"
+#include <stdio.h>
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int rank, size, len;
     char version[MPI_MAX_LIBRARY_VERSION_STRING];
@@ -19,8 +19,7 @@ int main(int argc, char* argv[])
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &size);
     MPI_Get_library_version(version, &len);
-    printf("Hello, world, I am %d of %d, (%s, %d)\n",
-           rank, size, version, len);
+    printf("Hello, world, I am %d of %d, (%s, %d)\n", rank, size, version, len);
     MPI_Finalize();
 
     return 0;
diff --git a/examples/hello_oshmem_c.c b/examples/hello_oshmem_c.c
index 6bb0d08c062..a485e2e99d6 100644
--- a/examples/hello_oshmem_c.c
+++ b/examples/hello_oshmem_c.c
@@ -9,14 +9,14 @@
  * $HEADER$
  */
 
-#include <stdio.h>
 #include "shmem.h"
+#include <stdio.h>
 
 #if !defined(OSHMEM_SPEC_VERSION) || OSHMEM_SPEC_VERSION < 10200
-#error This application uses API 1.2 and up
+#    error This application uses API 1.2 and up
 #endif
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int proc, nproc;
     char name[SHMEM_MAX_NAME_LEN];
@@ -28,8 +28,7 @@ int main(int argc, char* argv[])
     shmem_info_get_name(name);
     shmem_info_get_version(&major, &minor);
 
-    printf("Hello, world, I am %d of %d: %s (version: %d.%d)\n",
-           proc, nproc, name, major, minor);
+    printf("Hello, world, I am %d of %d: %s (version: %d.%d)\n", proc, nproc, name, major, minor);
     shmem_finalize();
 
     return 0;
diff --git a/examples/hello_sessions_c.c b/examples/hello_sessions_c.c
new file mode 100644
index 00000000000..8d84c39ba6b
--- /dev/null
+++ b/examples/hello_sessions_c.c
@@ -0,0 +1,21 @@
+#include "mpi.h"
+#include <stdio.h>
+
+/*
+ * Simple test to demonstrate several aspects of MPI 4 Sessions and related
+ * functionality.  See sections 11.3 and 11.4 of the MPI 4 standard for more
+ * details.
+ */
+
+int main(int argc, char** argv) {
+  MPI_Info info;
+  MPI_Session s1, s2;
+
+  MPI_Info_create(&info);
+  MPI_Session_init(MPI_INFO_NULL, MPI_ERRORS_RETURN, &s1);
+  MPI_Session_finalize(&s1);
+  MPI_Session_init(MPI_INFO_NULL, MPI_ERRORS_RETURN, &s2);
+  MPI_Session_finalize(&s2);
+  MPI_Info_free(&info);
+  return 0;
+}
diff --git a/examples/oshmem_circular_shift.c b/examples/oshmem_circular_shift.c
index 3ff2d6d0c87..1cc26466dae 100644
--- a/examples/oshmem_circular_shift.c
+++ b/examples/oshmem_circular_shift.c
@@ -8,10 +8,10 @@
  * $HEADER$
  */
 
-#include <stdio.h>
 #include <shmem.h>
+#include <stdio.h>
 
-int main (void)
+int main(void)
 {
     static int aaa, bbb;
     int num_pes, my_pe, peer;
@@ -32,4 +32,3 @@ int main (void)
 
     return 0;
 }
-
diff --git a/examples/oshmem_max_reduction.c b/examples/oshmem_max_reduction.c
index d0dec673af2..c9bef246d11 100644
--- a/examples/oshmem_max_reduction.c
+++ b/examples/oshmem_max_reduction.c
@@ -23,7 +23,7 @@ long src[N];
 long dst[N];
 long pWrk[_SHMEM_REDUCE_SYNC_SIZE];
 
-int  main(void)
+int main(void)
 {
     int i;
     int my_pe, num_pes;
@@ -47,7 +47,7 @@ int  main(void)
 
     printf("%d/%d dst =", my_pe, num_pes);
 
-    for (i = 0; i < N; i+= 1) {
+    for (i = 0; i < N; i += 1) {
         printf(" %ld", dst[i]);
     }
 
@@ -56,4 +56,3 @@ int  main(void)
 
     return 0;
 }
-
diff --git a/examples/oshmem_shmalloc.c b/examples/oshmem_shmalloc.c
index e5f9ab90907..aac04d9ac94 100644
--- a/examples/oshmem_shmalloc.c
+++ b/examples/oshmem_shmalloc.c
@@ -27,4 +27,3 @@ int main(void)
 
     shmem_finalize();
 }
-
diff --git a/examples/oshmem_strided_puts.c b/examples/oshmem_strided_puts.c
index e9f31ddb749..ea547c6a66a 100644
--- a/examples/oshmem_strided_puts.c
+++ b/examples/oshmem_strided_puts.c
@@ -24,12 +24,12 @@
  *
  */
 
-#include <stdio.h>
 #include <shmem.h>
+#include <stdio.h>
 
 int main(void)
 {
-    short source[10] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
+    short source[10] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10};
     static short target[10];
     int me;
 
@@ -44,9 +44,8 @@ int main(void)
     shmem_barrier_all(); /* sync sender and receiver */
 
     if (me == 1) {
-        printf("target on PE %d is %hd %hd %hd %hd %hd\n", me,
-        target[0], target[1], target[2],
-        target[3], target[4] );
+        printf("target on PE %d is %hd %hd %hd %hd %hd\n", me, target[0], target[1], target[2],
+               target[3], target[4]);
     }
     shmem_barrier_all(); /* sync before exiting */
     shmem_finalize();
diff --git a/examples/oshmem_symmetric_data.c b/examples/oshmem_symmetric_data.c
index 7c5a1db6b53..42155aa06b1 100644
--- a/examples/oshmem_symmetric_data.c
+++ b/examples/oshmem_symmetric_data.c
@@ -8,12 +8,12 @@
  * $HEADER$
  */
 
-#include <stdio.h>
 #include <shmem.h>
+#include <stdio.h>
 
 #define SIZE 16
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     short source[SIZE];
     static short target[SIZE];
@@ -27,13 +27,13 @@ int main(int argc, char* argv[])
 
     if (my_pe == 0) {
         /* initialize array */
-        for(i = 0; i < SIZE; i++) {
+        for (i = 0; i < SIZE; i++) {
             source[i] = i;
         }
         /* local, not symmetric */
         /* static makes it symmetric */
         /* put "size" words into target on each PE */
-        for(i = 1; i < num_pe; i++) {
+        for (i = 1; i < num_pe; i++) {
             shmem_short_put(target, source, SIZE, i);
         }
     }
@@ -43,7 +43,7 @@ int main(int argc, char* argv[])
     if (my_pe != 0) {
         printf("Target on PE %d is \t", my_pe);
 
-        for(i = 0; i < SIZE; i++) {
+        for (i = 0; i < SIZE; i++) {
             printf("%hd \t", target[i]);
         }
         printf("\n");
diff --git a/examples/ring_c.c b/examples/ring_c.c
index 86c205b3d21..f3b037ead76 100644
--- a/examples/ring_c.c
+++ b/examples/ring_c.c
@@ -7,8 +7,8 @@
  * Simple ring test program in C.
  */
 
-#include <stdio.h>
 #include "mpi.h"
+#include <stdio.h>
 
 int main(int argc, char *argv[])
 {
@@ -27,15 +27,15 @@ int main(int argc, char *argv[])
     next = (rank + 1) % size;
     prev = (rank + size - 1) % size;
 
-    /* If we are the "master" process (i.e., MPI_COMM_WORLD rank 0),
+    /* If we are the "manager" process (i.e., MPI_COMM_WORLD rank 0),
        put the number of times to go around the ring in the
        message. */
 
     if (0 == rank) {
         message = 10;
 
-        printf("Process 0 sending %d to %d, tag %d (%d processes in ring)\n",
-               message, next, tag, size);
+        printf("Process 0 sending %d to %d, tag %d (%d processes in ring)\n", message, next, tag,
+               size);
         MPI_Send(&message, 1, MPI_INT, next, tag, MPI_COMM_WORLD);
         printf("Process 0 sent to %d\n", next);
     }
@@ -49,8 +49,7 @@ int main(int argc, char *argv[])
        and can quit normally. */
 
     while (1) {
-        MPI_Recv(&message, 1, MPI_INT, prev, tag, MPI_COMM_WORLD,
-                 MPI_STATUS_IGNORE);
+        MPI_Recv(&message, 1, MPI_INT, prev, tag, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
 
         if (0 == rank) {
             --message;
@@ -68,8 +67,7 @@ int main(int argc, char *argv[])
        to be received before the program can exit */
 
     if (0 == rank) {
-        MPI_Recv(&message, 1, MPI_INT, prev, tag, MPI_COMM_WORLD,
-                 MPI_STATUS_IGNORE);
+        MPI_Recv(&message, 1, MPI_INT, prev, tag, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
     }
 
     /* All done */
diff --git a/examples/ring_mpifh.f b/examples/ring_mpifh.f
index 64e6498fe7f..2d09d752f3b 100644
--- a/examples/ring_mpifh.f
+++ b/examples/ring_mpifh.f
@@ -2,7 +2,7 @@
 C Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
 C                         University Research and Technology
 C                         Corporation.  All rights reserved.
-C Copyright (c) 2006-2015 Cisco Systems, Inc.  All rights reserved.
+C Copyright (c) 2006-2022 Cisco Systems, Inc.  All rights reserved
 C $COPYRIGHT$
 C
 C Simple ring test program using the mpif.h Fortran bindings.
@@ -26,7 +26,7 @@ program ring_f77
       next = mod((rank + 1), size)
       from = mod((rank + size - 1), size)
 
-C     If we are the "master" process (i.e., MPI_COMM_WORLD rank 0), put
+C     If we are the "manager" process (i.e., MPI_COMM_WORLD rank 0), put
 C     the number of times to go around the ring in the message.
 
       if (rank .eq. 0) then
diff --git a/examples/ring_oshmem_c.c b/examples/ring_oshmem_c.c
index 5fe1c7f65dc..17ba57d3264 100644
--- a/examples/ring_oshmem_c.c
+++ b/examples/ring_oshmem_c.c
@@ -12,10 +12,10 @@
 #include <stdio.h>
 
 #if !defined(OSHMEM_SPEC_VERSION) || OSHMEM_SPEC_VERSION < 10200
-#error This application uses API 1.2 and up
+#    error This application uses API 1.2 and up
 #endif
 
-int main (int argc, char * argv[])
+int main(int argc, char *argv[])
 {
     static int rbuf = -1;
     int proc, nproc, next;
@@ -30,8 +30,7 @@ int main (int argc, char * argv[])
 
     next = (proc + 1) % nproc;
 
-    if(proc == 0)
-    {
+    if (proc == 0) {
         printf("Process 0 puts message %d to %d (%d processes in ring)\n", message, next, nproc);
         shmem_int_put(&rbuf, &message, 1, next);
     }
@@ -44,14 +43,14 @@ int main (int argc, char * argv[])
        message first, every process gets the 0 message and can quit
        normally. */
 
-    while(message > 0) {
+    while (message > 0) {
         shmem_int_wait_until(&rbuf, SHMEM_CMP_EQ, message);
-        if(proc == 0) {
+        if (proc == 0) {
             --message;
             printf("Process 0 decremented value: %d\n", message);
         }
         shmem_int_put(&rbuf, &message, 1, next);
-        if(proc != 0) {
+        if (proc != 0) {
             --message;
         }
     }
diff --git a/examples/ring_usempi.f90 b/examples/ring_usempi.f90
index 6c5b0b17bd0..f104e1e8b44 100644
--- a/examples/ring_usempi.f90
+++ b/examples/ring_usempi.f90
@@ -2,7 +2,7 @@
 ! Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
 !                         University Research and Technology
 !                         Corporation.  All rights reserved.
-! Copyright (c) 2006-2015 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2006-2022 Cisco Systems, Inc.  All rights reserved
 ! $COPYRIGHT$
 !
 ! Simple ring test program using the Fortran mpi module bindings.
@@ -25,7 +25,7 @@ program ring
   next = mod((rank + 1), size)
   from = mod((rank + size - 1), size)
 
-! If we are the "master" process (i.e., MPI_COMM_WORLD rank 0), put
+! If we are the "manager" process (i.e., MPI_COMM_WORLD rank 0), put
 ! the number of times to go around the ring in the message.
 
   if (rank .eq. 0) then
diff --git a/examples/ring_usempif08.f90 b/examples/ring_usempif08.f90
index 753b89531e6..b35c9b5fa36 100644
--- a/examples/ring_usempif08.f90
+++ b/examples/ring_usempif08.f90
@@ -3,7 +3,7 @@
 ! Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
 !                         University Research and Technology
 !                         Corporation.  All rights reserved.
-! Copyright (c) 2006-2015 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2006-2022 Cisco Systems, Inc.  All rights reserved
 ! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
 !                         All rights reserved.
 ! $COPYRIGHT$
@@ -29,7 +29,7 @@ program ring
   next = mod((rank + 1), size)
   from = mod((rank + size - 1), size)
 
-! If we are the "master" process (i.e., MPI_COMM_WORLD rank 0), put
+! If we are the "manager" process (i.e., MPI_COMM_WORLD rank 0), put
 ! the number of times to go around the ring in the message.
 
   if (rank .eq. 0) then
diff --git a/examples/spc_example.c b/examples/spc_example.c
index ba0bb38996d..7c98955c828 100644
--- a/examples/spc_example.c
+++ b/examples/spc_example.c
@@ -2,6 +2,8 @@
  * Copyright (c) 2018-2020 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2023      High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
  *
  * Simple example usage of SPCs through MPI_T.
  */
@@ -19,17 +21,16 @@ void message_exchange(int num_messages, int message_size)
 {
     int i, rank;
     /* Use calloc to initialize data to 0's */
-    char *data = (char*)calloc(message_size, sizeof(char));
-    MPI_Status status;
+    char *data = (char *) calloc(message_size, sizeof(char));
 
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
 
-    if(rank == 0) {
-        for(i = 0; i < num_messages; i++)
+    if (rank == 0) {
+        for (i = 0; i < num_messages; i++)
             MPI_Send(data, message_size, MPI_BYTE, 1, 123, MPI_COMM_WORLD);
-    } else if(rank == 1) {
-        for(i = 0; i < num_messages; i++)
-            MPI_Recv(data, message_size, MPI_BYTE, 0, 123, MPI_COMM_WORLD, &status);
+    } else if (rank == 1) {
+        for (i = 0; i < num_messages; i++)
+            MPI_Recv(data, message_size, MPI_BYTE, 0, 123, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
     }
 
     free(data);
@@ -39,15 +40,17 @@ int main(int argc, char **argv)
 {
     int num_messages, message_size, rc;
 
-    if(argc < 3) {
-        printf("Usage: mpirun -np 2 --mca mpi_spc_attach all --mca mpi_spc_dump_enabled true ./spc_example [num_messages] [message_size]\n");
+    if (argc < 3) {
+        printf("Usage: mpirun -n 2 --mca mpi_spc_attach all --mca mpi_spc_dump_enabled true "
+               "./spc_example [num_messages] [message_size]\n");
         return -1;
     } else {
         num_messages = atoi(argv[1]);
         message_size = atoi(argv[2]);
     }
 
-    int i, rank, size, provided, num, name_len, desc_len, verbosity, bind, var_class, readonly, continuous, atomic, count, index;
+    int i, rank, size, provided, num, name_len, desc_len, verbosity, bind, var_class, readonly,
+        continuous, atomic, count, index;
     MPI_Datatype datatype;
     MPI_T_enum enumtype;
     MPI_Comm comm;
@@ -56,65 +59,63 @@ int main(int argc, char **argv)
     /* Counter names to be read by ranks 0 and 1 */
     /* (See also: ompi_spc_counters_t for list) */
     char *counter_names[] = {"runtime_spc_OMPI_SPC_BYTES_SENT_USER",
-                             "runtime_spc_OMPI_SPC_BYTES_RECEIVED_USER" };
+                             "runtime_spc_OMPI_SPC_BYTES_RECEIVED_USER"};
 
     MPI_Init(NULL, NULL);
     MPI_T_init_thread(MPI_THREAD_SINGLE, &provided);
 
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &size);
-    if(size != 2) {
+    if (size != 2) {
         fprintf(stderr, "ERROR: This test should be run with two MPI processes.\n");
         MPI_Abort(MPI_COMM_WORLD, -1);
     }
 
-    /* Determine the MPI_T pvar indices for the OMPI_BYTES_SENT/RECIEVED_USER SPCs */
+    /* Determine the MPI_T pvar indices for the OMPI_BYTES_SENT/RECEIVED_USER SPCs */
     index = -1;
     MPI_T_pvar_get_num(&num);
-    for(i = 0; i < num; i++) {
+    for (i = 0; i < num; i++) {
         name_len = desc_len = 256;
-        rc = PMPI_T_pvar_get_info(i, name, &name_len, &verbosity,
-                                  &var_class, &datatype, &enumtype, description, &desc_len, &bind,
-                                  &readonly, &continuous, &atomic);
-        if( MPI_SUCCESS != rc )
+        rc = PMPI_T_pvar_get_info(i, name, &name_len, &verbosity, &var_class, &datatype, &enumtype,
+                                  description, &desc_len, &bind, &readonly, &continuous, &atomic);
+        if (MPI_SUCCESS != rc)
             continue;
-        if(strcmp(name, counter_names[rank]) == 0) {
+        if (strcmp(name, counter_names[rank]) == 0) {
             index = i;
             printf("[%d] %s -> %s\n", rank, name, description);
         }
     }
 
     /* Make sure we found the counters */
-    if(index == -1) {
+    if (index == -1) {
         fprintf(stderr, "ERROR: Couldn't find the appropriate SPC counter in the MPI_T pvars.\n");
         MPI_Abort(MPI_COMM_WORLD, -1);
     }
 
-    int ret;
     long long value;
 
     MPI_T_pvar_session session;
     MPI_T_pvar_handle handle;
     /* Create the MPI_T sessions/handles for the counters and start the counters */
-    ret = MPI_T_pvar_session_create(&session);
-    ret = MPI_T_pvar_handle_alloc(session, index, NULL, &handle, &count);
-    ret = MPI_T_pvar_start(session, handle);
+    MPI_T_pvar_session_create(&session);
+    MPI_T_pvar_handle_alloc(session, index, NULL, &handle, &count);
+    MPI_T_pvar_start(session, handle);
 
     message_exchange(num_messages, message_size);
 
-    ret = MPI_T_pvar_read(session, handle, &value);
+    MPI_T_pvar_read(session, handle, &value);
     /* Print the counter values in order by rank */
-    for(i = 0; i < 2; i++) {
-        if(i == rank) {
+    for (i = 0; i < 2; i++) {
+        if (i == rank) {
             printf("[%d] Value Read: %lld\n", rank, value);
             fflush(stdout);
         }
         MPI_Barrier(MPI_COMM_WORLD);
     }
     /* Stop the MPI_T session, free the handle, and then free the session */
-    ret = MPI_T_pvar_stop(session, handle);
-    ret = MPI_T_pvar_handle_free(session, &handle);
-    ret = MPI_T_pvar_session_free(&session);
+    MPI_T_pvar_stop(session, handle);
+    MPI_T_pvar_handle_free(session, &handle);
+    MPI_T_pvar_session_free(&session);
 
     MPI_T_finalize();
     MPI_Finalize();
diff --git a/ompi/Makefile.am b/ompi/Makefile.am
index b3952e375a5..f855492ef15 100644
--- a/ompi/Makefile.am
+++ b/ompi/Makefile.am
@@ -9,7 +9,7 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2008-2019 Cisco Systems, Inc.  All rights reserved
+# Copyright (c) 2008-2022 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
 # Copyright (c) 2010-2011 Sandia National Laboratories. All rights reserved.
 # Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights
@@ -19,6 +19,9 @@
 #                         and Technology (RIST).  All rights reserved.
 # Copyright (c) 2016      IBM Corporation.  All rights reserved.
 # Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+# Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+#                         reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -26,15 +29,6 @@
 # $HEADER$
 #
 
-# We always have C bindings, but do we have profiling?
-
-if BUILD_MPI_BINDINGS_LAYER
-c_mpi_lib = mpi/c/libmpi_c_mpi.la mpi/tool/libmpi_mpit.la
-else
-c_mpi_lib =
-endif
-c_pmpi_lib = mpi/c/profile/libmpi_c_pmpi.la mpi/tool/profile/libmpi_pmpit.la
-
 # See if we have Fortran mpif.h MPI bindings
 
 if OMPI_BUILD_FORTRAN_MPIFH_BINDINGS
@@ -90,7 +84,7 @@ SUBDIRS = \
         mpi/fortran/mpif-h \
         mpi/fortran/use-mpi \
         $(OMPI_MPIEXT_USEMPI_DIR) \
-        $(OMPI_FORTRAN_USEMPI_DIR) \
+        mpi/fortran/use-mpi-ignore-tkr \
         mpi/fortran/mpiext-use-mpi \
         mpi/fortran/use-mpi-f08/base \
         mpi/fortran/use-mpi-f08/mod \
@@ -98,10 +92,7 @@ SUBDIRS = \
         $(OMPI_MPIEXT_USEMPIF08_DIRS) \
         mpi/fortran/use-mpi-f08 \
         mpi/fortran/mpiext-use-mpi-f08 \
-        $(MCA_ompi_FRAMEWORK_COMPONENT_DSO_SUBDIRS) \
-        $(OMPI_CONTRIB_SUBDIRS) \
-        mpi/man/man3 \
-        mpi/man/man5
+        $(MCA_ompi_FRAMEWORK_COMPONENT_DSO_SUBDIRS)
 
 if OMPI_WANT_JAVA_BINDINGS
 SUBDIRS += \
@@ -123,7 +114,6 @@ DIST_SUBDIRS = \
         mpi/fortran/base \
         mpi/fortran/mpif-h \
         mpi/fortran/use-mpi \
-        mpi/fortran/use-mpi-tkr \
         mpi/fortran/use-mpi-ignore-tkr \
         mpi/fortran/mpiext-use-mpi \
         mpi/fortran/use-mpi-f08 \
@@ -134,10 +124,7 @@ DIST_SUBDIRS = \
         mpi/java \
         $(OMPI_MPIEXT_ALL_SUBDIRS) \
         $(MCA_ompi_FRAMEWORKS_SUBDIRS) \
-        $(MCA_ompi_FRAMEWORK_COMPONENT_ALL_SUBDIRS) \
-        $(OMPI_CONTRIB_DIST_SUBDIRS) \
-        mpi/man/man3 \
-        mpi/man/man5
+        $(MCA_ompi_FRAMEWORK_COMPONENT_ALL_SUBDIRS)
 
 # Build the main MPI library
 
@@ -147,7 +134,7 @@ lib@OMPI_LIBMPI_NAME@_la_LIBADD = \
         datatype/libdatatype.la \
         debuggers/libdebuggers.la \
         mpi/c/libmpi_c.la \
-        mpi/tool/libmpi_mpit_common.la \
+        mpi/tool/libmpi_mpit.la \
         $(c_mpi_lib) \
         $(c_pmpi_lib) \
         $(mpi_fortran_base_lib) \
@@ -157,7 +144,7 @@ lib@OMPI_LIBMPI_NAME@_la_LIBADD = \
 
 
 lib@OMPI_LIBMPI_NAME@_la_LIBADD += \
-        $(OMPI_TOP_BUILDDIR)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+        $(OMPI_TOP_BUILDDIR)/opal/lib@OPAL_LIB_NAME@.la
 lib@OMPI_LIBMPI_NAME@_la_DEPENDENCIES = $(lib@OMPI_LIBMPI_NAME@_la_LIBADD)
 lib@OMPI_LIBMPI_NAME@_la_LDFLAGS = \
        -version-info $(libmpi_so_version) \
@@ -167,7 +154,7 @@ lib@OMPI_LIBMPI_NAME@_la_LDFLAGS = \
 headers =
 noinst_LTLIBRARIES =
 include_HEADERS =
-dist_ompidata_DATA =
+EXTRA_DIST =
 lib@OMPI_LIBMPI_NAME@_la_SOURCES += $(headers)
 
 # Conditionally install the header files
@@ -196,10 +183,9 @@ include win/Makefile.am
 include tools/Makefile.am
 include mpi/Makefile.am
 include mpiext/Makefile.am
-include patterns/net/Makefile.am
-include patterns/comm/Makefile.am
 include mca/Makefile.am
 include util/Makefile.am
+include instance/Makefile.am
 
 distclean-local:
 	rm -f mpiext/static-components.h
diff --git a/ompi/attribute/attribute.c b/ompi/attribute/attribute.c
index aaf04a685e0..d224f4b8793 100644
--- a/ompi/attribute/attribute.c
+++ b/ompi/attribute/attribute.c
@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -14,6 +15,10 @@
  *                         reserved.
  * Copyright (c) 2017      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018-2022 Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -124,15 +129,15 @@
  *          ompi_set_attr_int(..., foo, ...)
  *
  * 4. C reads the attribute value.  The value returned is a pointer
- *    that points to an int that has a value
- *    of 7.
+ *    that points to an int that has a value of 7.
  *
  * Example: int *ret;
  *          MPI_Attr_get(..., &ret);
  *          -> *ret will equal 7.
  *
- * 5. Fortran MPI-1 reads the attribute value.  This is the unity
- *    case; the same value is returned.
+ * 5. Fortran MPI-1 reads the attribute value.  The C int value is
+ *    cast to a fortran INTEGER (i.e., MPI_Fint) -- potentially being
+ *    truncated if sizeof(int) > sizeof(INTEGER).
  *
  * Example: INTEGER ret
  *          CALL MPI_ATTR_GET(..., ret, ierr)
@@ -158,7 +163,7 @@
  *    that points to an INTEGER (i.e., an MPI_Fint) that has a value
  *    of 7.
  *    --> NOTE: The external MPI interface does not distinguish between
- *        this case and case 7.  It is the programer's responsibility
+ *        this case and case 10.  It is the programer's responsibility
  *        to code accordingly.
  *
  * Example: MPI_Fint *ret;
@@ -197,7 +202,7 @@
  *    that points to an INTEGER(KIND=MPI_ADDRESS_KIND) (i.e., a void*)
  *    that has a value of 12.
  *    --> NOTE: The external MPI interface does not distinguish between
- *        this case and case 4.  It is the programer's responsibility
+ *        this case and case 7.  It is the programer's responsibility
  *        to code accordingly.
  *
  * Example A: MPI_Aint *ret;
@@ -240,8 +245,10 @@
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/communicator/communicator.h"  /* ompi_communicator_t generated in [COPY|DELETE]_ATTR_CALLBACKS */
 #include "ompi/win/win.h"                    /* ompi_win_t generated in [COPY|DELETE]_ATTR_CALLBACKS */
+#include "ompi/instance/instance.h"
 #include "ompi/mpi/fortran/base/fint_2_int.h"
 
+
 /*
  * Macros
  */
@@ -256,9 +263,9 @@
 #define attr_datatype_f d_f_to_c_index
 #define attr_win_f w_f_to_c_index
 
-#define CREATE_KEY(key) opal_bitmap_find_and_set_first_unset_bit(key_bitmap, (key))
+#define CREATE_KEY(key) opal_bitmap_find_and_set_first_unset_bit(attr_subsys->key_bitmap, (key))
 
-#define FREE_KEY(key) opal_bitmap_clear_bit(key_bitmap, (key))
+#define FREE_KEY(key) opal_bitmap_clear_bit(attr_subsys->key_bitmap, (key))
 
 
 /* Not checking for NULL_DELETE_FN here, since according to the
@@ -326,7 +333,7 @@ do { \
 /* See the big, long comment above from DELETE_ATTR_CALLBACKS -- most of
    that text applies here, too. */
 
-#define COPY_ATTR_CALLBACKS(type, old_object, keyval_obj, in_attr, new_object, out_attr, err) \
+#define COPY_ATTR_CALLBACKS(type, old_object, keyval_obj, in_attr, out_attr, err) \
 do { \
     OPAL_THREAD_UNLOCK(&attribute_lock); \
     if (0 != (keyval_obj->attr_flag & OMPI_KEYVAL_F77)) { \
@@ -346,8 +353,8 @@ do { \
             if (MPI_SUCCESS != OMPI_FINT_2_INT(f_err)) { \
                 err = OMPI_FINT_2_INT(f_err);           \
             } else {                                    \
-                out_attr->av_value = (void*) 0;         \
-                *out_attr->av_fint_pointer = out;    \
+                out_attr->av_value.av_fint = out;          \
+                out_attr->av_set_from = OMPI_ATTRIBUTE_FINT; \
                 flag = OMPI_LOGICAL_2_INT(f_flag);      \
             }                                           \
         } \
@@ -364,7 +371,8 @@ do { \
             if (MPI_SUCCESS != OMPI_FINT_2_INT(f_err)) { \
                 err = OMPI_FINT_2_INT(f_err);           \
             } else {                                    \
-                out_attr->av_value = (void *) out;      \
+                out_attr->av_value.av_aint = out;       \
+                out_attr->av_set_from = OMPI_ATTRIBUTE_AINT; \
                 flag = OMPI_LOGICAL_2_INT(f_flag);      \
             }                                           \
         } \
@@ -375,9 +383,10 @@ do { \
         in = translate_to_c(in_attr); \
         if ((err = (*((keyval_obj->copy_attr_fn).attr_##type##_copy_fn)) \
               ((ompi_##type##_t *)old_object, key, keyval_obj->extra_state.c_ptr, \
-               in, &out, &flag, (ompi_##type##_t *)(new_object))) == MPI_SUCCESS) { \
-            out_attr->av_value = out;                                   \
-        }                                                               \
+               in, &out, &flag)) == MPI_SUCCESS) { \
+            out_attr->av_value.av_pointer = out;   \
+            out_attr->av_set_from = OMPI_ATTRIBUTE_C; \
+        }                                          \
     } \
     OPAL_THREAD_LOCK(&attribute_lock); \
 } while (0)
@@ -386,52 +395,75 @@ do { \
  * Cases for attribute values
  */
 typedef enum ompi_attribute_translate_t {
+    OMPI_ATTRIBUTE_INVALID = -1,
     OMPI_ATTRIBUTE_C,
     OMPI_ATTRIBUTE_INT,
     OMPI_ATTRIBUTE_FINT,
-    OMPI_ATTRIBUTE_AINT
+    OMPI_ATTRIBUTE_AINT,
 } ompi_attribute_translate_t;
 
+typedef union attribute_value_t {
+    MPI_Fint av_fint;
+    MPI_Aint av_aint;
+    int av_int;
+    void *av_pointer;
+} attribute_value_t;
+
 /*
  * struct to hold attribute values on each MPI object
  */
-typedef struct attribute_value_t {
+typedef struct attribute_key_value_t {
     opal_object_t super;
     int av_key;
-    void *av_value;
-    int *av_int_pointer;
-    MPI_Fint *av_fint_pointer;
-    MPI_Aint *av_aint_pointer;
-    int av_set_from;
+    attribute_value_t av_value;
+    ompi_attribute_translate_t av_set_from;
     int av_sequence;
-} attribute_value_t;
+} attribute_key_value_t;
 
+/*
+ * struct to hold state of attr subsys
+ */
+
+typedef  struct attr_subsys_t {
+    opal_object_t            super;
+    opal_hash_table_t *keyval_hash;
+    opal_bitmap_t *key_bitmap;
+ } attr_subsys_t;
 
 /*
  * Local functions
  */
-static void attribute_value_construct(attribute_value_t *item);
+static void attribute_key_value_construct(attribute_key_value_t *item);
 static void ompi_attribute_keyval_construct(ompi_attribute_keyval_t *keyval);
 static void ompi_attribute_keyval_destruct(ompi_attribute_keyval_t *keyval);
+static void attr_subsys_construct(attr_subsys_t *subsys);
+static void attr_subsys_destruct(attr_subsys_t *subsys);
 static int set_value(ompi_attribute_type_t type, void *object,
                      opal_hash_table_t **attr_hash, int key,
-                     attribute_value_t *new_attr,
+                     attribute_key_value_t *new_attr,
                      bool predefined);
 static int get_value(opal_hash_table_t *attr_hash, int key,
-                     attribute_value_t **attribute, int *flag);
-static void *translate_to_c(attribute_value_t *val);
-static MPI_Fint translate_to_fint(attribute_value_t *val);
-static MPI_Aint translate_to_aint(attribute_value_t *val);
+                     attribute_key_value_t **attribute, int *flag);
+static void *translate_to_c(attribute_key_value_t *val);
+static MPI_Fint translate_to_fint(attribute_key_value_t *val);
+static MPI_Aint translate_to_aint(attribute_key_value_t *val);
 
 static int compare_attr_sequence(const void *attr1, const void *attr2);
 
+/*
+ * attribute_subsys_t class
+ */
+static OBJ_CLASS_INSTANCE(attr_subsys_t,
+                          opal_object_t,
+                          attr_subsys_construct,
+                          attr_subsys_destruct);
 
 /*
- * attribute_value_t class
+ * attribute_key_value_t class
  */
-static OBJ_CLASS_INSTANCE(attribute_value_t,
+static OBJ_CLASS_INSTANCE(attribute_key_value_t,
                           opal_object_t,
-                          attribute_value_construct,
+                          attribute_key_value_construct,
                           NULL);
 
 
@@ -443,35 +475,28 @@ static OBJ_CLASS_INSTANCE(ompi_attribute_keyval_t,
                           ompi_attribute_keyval_construct,
                           ompi_attribute_keyval_destruct);
 
-
 /*
  * Static variables
  */
 
-static opal_hash_table_t *keyval_hash;
-static opal_bitmap_t *key_bitmap;
+static attr_subsys_t *attr_subsys = NULL;
 static int attr_sequence;
-static unsigned int int_pos = 12345;
-static unsigned int integer_pos = 12345;
 
 /*
  * MPI attributes are *not* high performance, so just use a One Big Lock
  * approach. However, this lock is released before a user provided callback is
  * triggered and acquired right after, allowing for recursive behaviors.
  */
-static opal_mutex_t attribute_lock;
-
+static opal_mutex_t attribute_lock = OPAL_MUTEX_STATIC_INIT;
 
 /*
- * attribute_value_t constructor function
+ * attribute_key_value_t constructor function
  */
-static void attribute_value_construct(attribute_value_t *item)
+static void attribute_key_value_construct(attribute_key_value_t *item)
 {
     item->av_key = MPI_KEYVAL_INVALID;
-    item->av_aint_pointer = (MPI_Aint*) &item->av_value;
-    item->av_int_pointer = (int *)&item->av_value + int_pos;
-    item->av_fint_pointer = (MPI_Fint *)&item->av_value + integer_pos;
-    item->av_set_from = 0;
+    item->av_value.av_aint = 0;
+    item->av_set_from = OMPI_ATTRIBUTE_INVALID;
     item->av_sequence = -1;
 }
 
@@ -507,74 +532,87 @@ ompi_attribute_keyval_destruct(ompi_attribute_keyval_t *keyval)
             free(keyval->bindings_extra_state);
         }
 
-        opal_hash_table_remove_value_uint32(keyval_hash, keyval->key);
+        opal_hash_table_remove_value_uint32(attr_subsys->keyval_hash, keyval->key);
         FREE_KEY(keyval->key);
     }
 }
 
 
-/*
- * This will initialize the main list to store key- attribute
- * items. This will be called one time, during MPI_INIT().
- */
-int ompi_attr_init(void)
+int ompi_attr_get_ref(void)
 {
-    int ret;
-    void *bogus = (void*) 1;
-    int *p = (int *) &bogus;
+    int ret = OMPI_SUCCESS;
 
-    keyval_hash = OBJ_NEW(opal_hash_table_t);
-    if (NULL == keyval_hash) {
-        return OMPI_ERR_OUT_OF_RESOURCE;
-    }
-    key_bitmap = OBJ_NEW(opal_bitmap_t);
-    /*
-     * Set the max size to OMPI_FORTRAN_HANDLE_MAX to enforce bound
-     */
-    opal_bitmap_set_max_size (key_bitmap, OMPI_FORTRAN_HANDLE_MAX);
-    if (0 != opal_bitmap_init(key_bitmap, 32)) {
-        return OMPI_ERR_OUT_OF_RESOURCE;
-    }
+    OPAL_THREAD_LOCK(&attribute_lock);
 
-    for (int_pos = 0; int_pos < (sizeof(void*) / sizeof(int));
-         ++int_pos) {
-        if (p[int_pos] == 1) {
-            break;
+    if (NULL == attr_subsys) {
+        attr_subsys = OBJ_NEW(attr_subsys_t);
+        if (NULL == attr_subsys) {
+            ret = OMPI_ERR_OUT_OF_RESOURCE;
+            goto fn_exit;
+        }
+        if ((NULL == attr_subsys->keyval_hash) || (NULL == attr_subsys->key_bitmap)) {
+            OBJ_RELEASE(attr_subsys);
+            attr_subsys = NULL;
+            ret = OMPI_ERR_OUT_OF_RESOURCE;
+            goto fn_exit;
         }
+    } else {
+        OBJ_RETAIN(attr_subsys);
     }
 
-    for (integer_pos = 0; integer_pos < (sizeof(void*) / sizeof(MPI_Fint));
-         ++integer_pos) {
-        if (p[integer_pos] == 1) {
-            break;
-        }
+fn_exit:
+    OPAL_THREAD_UNLOCK(&attribute_lock);
+
+    return ret;
+}
+
+int ompi_attr_put_ref(void)
+{
+    if (NULL != attr_subsys) {
+        OBJ_RELEASE(attr_subsys);
     }
+    return OMPI_SUCCESS;
+}
+
+static void attr_subsys_construct(attr_subsys_t *subsys)
+{
+    int ret;
+
+    subsys->keyval_hash = OBJ_NEW(opal_hash_table_t);
 
-    OBJ_CONSTRUCT(&attribute_lock, opal_mutex_t);
+    subsys->key_bitmap = OBJ_NEW(opal_bitmap_t);
 
-    if (OMPI_SUCCESS != (ret = opal_hash_table_init(keyval_hash,
-                                                    ATTR_TABLE_SIZE))) {
-        return ret;
+    /*
+     * Set the max size to OMPI_FORTRAN_HANDLE_MAX to enforce bound
+     */
+    opal_bitmap_set_max_size (subsys->key_bitmap,
+                              OMPI_FORTRAN_HANDLE_MAX);
+    ret = opal_bitmap_init(subsys->key_bitmap, 32);
+    if (OPAL_SUCCESS != ret) {
+        abort();
     }
-    if (OMPI_SUCCESS != (ret = ompi_attr_create_predefined())) {
-        return ret;
+
+    for (int i = 0; i < MPI_ATTR_PREDEFINED_KEY_MAX; i++) {
+        opal_bitmap_set_bit(subsys->key_bitmap, i);
     }
 
-    return OMPI_SUCCESS;
+    ret = opal_hash_table_init(subsys->keyval_hash, ATTR_TABLE_SIZE);
+    if (OPAL_SUCCESS != ret) {
+        abort();
+    }
+
+    attr_sequence = 0;
 }
 
 
 /*
- * Cleanup everything during MPI_Finalize().
+ * Cleanup everything when no more refs to the attr subsys
  */
-int ompi_attr_finalize(void)
+static void attr_subsys_destruct(attr_subsys_t *subsys)
 {
     ompi_attr_free_predefined();
-    OBJ_DESTRUCT(&attribute_lock);
-    OBJ_RELEASE(keyval_hash);
-    OBJ_RELEASE(key_bitmap);
-
-    return OMPI_SUCCESS;
+    OBJ_RELEASE(subsys->keyval_hash);
+    OBJ_RELEASE(subsys->key_bitmap);
 }
 
 /*****************************************************************************/
@@ -609,10 +647,15 @@ static int ompi_attr_create_keyval_impl(ompi_attribute_type_t type,
 
     /* Create a new unique key and fill the hash */
     OPAL_THREAD_LOCK(&attribute_lock);
-    ret = CREATE_KEY(key);
+    ret = MPI_SUCCESS;
+    if (!(flags & OMPI_KEYVAL_PREDEFINED)) {
+        ret = CREATE_KEY(key);
+    }
+
     if (OMPI_SUCCESS == ret) {
         keyval->key = *key;
-        ret = opal_hash_table_set_value_uint32(keyval_hash, *key, keyval);
+        ret = opal_hash_table_set_value_uint32(attr_subsys->keyval_hash,
+                                               *key, keyval);
     }
 
     if (OMPI_SUCCESS != ret) {
@@ -635,11 +678,27 @@ int ompi_attr_create_keyval(ompi_attribute_type_t type,
                             void *bindings_extra_state)
 {
     ompi_attribute_fortran_ptr_t es_tmp;
+    int rc;
+    bool is_predefined = flags & OMPI_KEYVAL_PREDEFINED;
+
+    /* Predefined attributes are created as part of the instance creation,
+     * so do not retain the instance to avoid a circular dependency */
+    if (!is_predefined) {
+        rc = ompi_mpi_instance_retain ();
+        if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+            return rc;
+        }
+    }
 
     es_tmp.c_ptr = extra_state;
-    return ompi_attr_create_keyval_impl(type, copy_attr_fn, delete_attr_fn,
-                                        key, &es_tmp, flags,
-                                        bindings_extra_state);
+    rc = ompi_attr_create_keyval_impl(type, copy_attr_fn, delete_attr_fn,
+                                      key, &es_tmp, flags,
+                                      bindings_extra_state);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc) && !is_predefined) {
+        ompi_mpi_instance_release ();
+    }
+
+    return rc;
 }
 
 int ompi_attr_create_keyval_fint(ompi_attribute_type_t type,
@@ -651,6 +710,12 @@ int ompi_attr_create_keyval_fint(ompi_attribute_type_t type,
                                  void *bindings_extra_state)
 {
     ompi_attribute_fortran_ptr_t es_tmp;
+    int rc;
+
+    rc = ompi_mpi_instance_retain ();
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+        return rc;
+    }
 
     es_tmp.f_integer = extra_state;
 #if SIZEOF_INT == OMPI_SIZEOF_FORTRAN_INTEGER
@@ -670,6 +735,12 @@ int ompi_attr_create_keyval_aint(ompi_attribute_type_t type,
                                  void *bindings_extra_state)
 {
     ompi_attribute_fortran_ptr_t es_tmp;
+    int rc;
+
+    rc = ompi_mpi_instance_retain ();
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+        return rc;
+    }
 
     es_tmp.f_address = extra_state;
     return ompi_attr_create_keyval_impl(type, copy_attr_fn, delete_attr_fn,
@@ -687,7 +758,7 @@ int ompi_attr_free_keyval(ompi_attribute_type_t type, int *key,
 
     /* Find the key-value pair */
     OPAL_THREAD_LOCK(&attribute_lock);
-    ret = opal_hash_table_get_value_uint32(keyval_hash, *key,
+    ret = opal_hash_table_get_value_uint32(attr_subsys->keyval_hash, *key,
                                            (void **) &keyval);
     if ((OMPI_SUCCESS != ret) || (NULL == keyval) ||
         (keyval->attr_type != type) ||
@@ -707,6 +778,13 @@ int ompi_attr_free_keyval(ompi_attribute_type_t type, int *key,
     opal_atomic_wmb();
     OPAL_THREAD_UNLOCK(&attribute_lock);
 
+    /* balance out retain in keyval_create
+     * predefined attributes do not retain the instance so do not release it either
+     */
+    if (!predefined) {
+        ompi_mpi_instance_release ();
+    }
+
     return MPI_SUCCESS;
 }
 
@@ -720,15 +798,15 @@ int ompi_attr_set_c(ompi_attribute_type_t type, void *object,
                     opal_hash_table_t **attr_hash,
                     int key, void *attribute, bool predefined)
 {
-    int ret;
-    attribute_value_t *new_attr = OBJ_NEW(attribute_value_t);
+    int ret = MPI_SUCCESS;
+    attribute_key_value_t *new_attr = OBJ_NEW(attribute_key_value_t);
     if (NULL == new_attr) {
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
 
     OPAL_THREAD_LOCK(&attribute_lock);
 
-    new_attr->av_value = attribute;
+    new_attr->av_value.av_pointer = attribute;
     new_attr->av_set_from = OMPI_ATTRIBUTE_C;
     ret = set_value(type, object, attr_hash, key, new_attr, predefined);
     if (OMPI_SUCCESS != ret) {
@@ -751,15 +829,14 @@ int ompi_attr_set_int(ompi_attribute_type_t type, void *object,
                       int key, int attribute, bool predefined)
 {
     int ret;
-    attribute_value_t *new_attr = OBJ_NEW(attribute_value_t);
+    attribute_key_value_t *new_attr = OBJ_NEW(attribute_key_value_t);
     if (NULL == new_attr) {
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
 
     OPAL_THREAD_LOCK(&attribute_lock);
 
-    new_attr->av_value = (void *) 0;
-    *new_attr->av_int_pointer = attribute;
+    new_attr->av_value.av_int = attribute;
     new_attr->av_set_from = OMPI_ATTRIBUTE_INT;
     ret = set_value(type, object, attr_hash, key, new_attr, predefined);
     if (OMPI_SUCCESS != ret) {
@@ -783,15 +860,14 @@ int ompi_attr_set_fint(ompi_attribute_type_t type, void *object,
                        bool predefined)
 {
     int ret;
-    attribute_value_t *new_attr = OBJ_NEW(attribute_value_t);
+    attribute_key_value_t *new_attr = OBJ_NEW(attribute_key_value_t);
     if (NULL == new_attr) {
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
 
     OPAL_THREAD_LOCK(&attribute_lock);
 
-    new_attr->av_value = (void *) 0;
-    *new_attr->av_fint_pointer = attribute;
+    new_attr->av_value.av_fint = attribute;
     new_attr->av_set_from = OMPI_ATTRIBUTE_FINT;
     ret = set_value(type, object, attr_hash, key, new_attr, predefined);
     if (OMPI_SUCCESS != ret) {
@@ -815,14 +891,14 @@ int ompi_attr_set_aint(ompi_attribute_type_t type, void *object,
                        bool predefined)
 {
     int ret;
-    attribute_value_t *new_attr = OBJ_NEW(attribute_value_t);
+    attribute_key_value_t *new_attr = OBJ_NEW(attribute_key_value_t);
     if (NULL == new_attr) {
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
 
     OPAL_THREAD_LOCK(&attribute_lock);
 
-    new_attr->av_value = (void *) attribute;
+    new_attr->av_value.av_aint = attribute;
     new_attr->av_set_from = OMPI_ATTRIBUTE_AINT;
     ret = set_value(type, object, attr_hash, key, new_attr, predefined);
     if (OMPI_SUCCESS != ret) {
@@ -844,7 +920,7 @@ int ompi_attr_set_aint(ompi_attribute_type_t type, void *object,
 int ompi_attr_get_c(opal_hash_table_t *attr_hash, int key,
                     void **attribute, int *flag)
 {
-    attribute_value_t *val = NULL;
+    attribute_key_value_t *val = NULL;
     int ret;
 
     OPAL_THREAD_LOCK(&attribute_lock);
@@ -867,7 +943,7 @@ int ompi_attr_get_c(opal_hash_table_t *attr_hash, int key,
 int ompi_attr_get_fint(opal_hash_table_t *attr_hash, int key,
                        MPI_Fint *attribute, int *flag)
 {
-    attribute_value_t *val = NULL;
+    attribute_key_value_t *val = NULL;
     int ret;
 
     OPAL_THREAD_LOCK(&attribute_lock);
@@ -890,7 +966,7 @@ int ompi_attr_get_fint(opal_hash_table_t *attr_hash, int key,
 int ompi_attr_get_aint(opal_hash_table_t *attr_hash, int key,
                        MPI_Aint *attribute, int *flag)
 {
-    attribute_value_t *val = NULL;
+    attribute_key_value_t *val = NULL;
     int ret;
 
     OPAL_THREAD_LOCK(&attribute_lock);
@@ -921,7 +997,7 @@ int ompi_attr_copy_all(ompi_attribute_type_t type, void *old_object,
     uint32_t key;
     int flag;
     void *node, *in_node;
-    attribute_value_t *old_attr, *new_attr;
+    attribute_key_value_t *old_attr, *new_attr;
     ompi_attribute_keyval_t *hash_value;
 
     /* If there's nothing to do, just return */
@@ -942,7 +1018,7 @@ int ompi_attr_copy_all(ompi_attribute_type_t type, void *old_object,
 
         /* Get the keyval in the main keyval hash - so that we know
            what the copy_attr_fn is */
-        err = opal_hash_table_get_value_uint32(keyval_hash, key,
+        err = opal_hash_table_get_value_uint32(attr_subsys->keyval_hash, key,
                                                (void **) &hash_value);
         if (OMPI_SUCCESS != err) {
             /* This should not happen! */
@@ -951,24 +1027,24 @@ int ompi_attr_copy_all(ompi_attribute_type_t type, void *old_object,
         }
 
         err = 0;
-        new_attr = OBJ_NEW(attribute_value_t);
+        new_attr = OBJ_NEW(attribute_key_value_t);
         switch (type) {
         case COMM_ATTR:
             /* Now call the copy_attr_fn */
             COPY_ATTR_CALLBACKS(communicator, old_object, hash_value,
-                                old_attr, new_object, new_attr, err);
+                                old_attr, new_attr, err);
             break;
 
         case TYPE_ATTR:
             /* Now call the copy_attr_fn */
             COPY_ATTR_CALLBACKS(datatype, old_object, hash_value,
-                                old_attr, new_object, new_attr, err);
+                                old_attr, new_attr, err);
             break;
 
         case WIN_ATTR:
             /* Now call the copy_attr_fn */
             COPY_ATTR_CALLBACKS(win, old_object, hash_value,
-                                old_attr, new_object, new_attr, err);
+                                old_attr, new_attr, err);
             break;
 
         default:
@@ -1034,10 +1110,10 @@ static int ompi_attr_delete_impl(ompi_attribute_type_t type, void *object,
 {
     ompi_attribute_keyval_t *keyval;
     int ret = OMPI_SUCCESS;
-    attribute_value_t *attr;
+    attribute_key_value_t *attr;
 
     /* Check if the key is valid in the master keyval hash */
-    ret = opal_hash_table_get_value_uint32(keyval_hash, key,
+    ret = opal_hash_table_get_value_uint32(attr_subsys->keyval_hash, key,
                                            (void **) &keyval);
 
     if ((OMPI_SUCCESS != ret) || (NULL == keyval) ||
@@ -1053,7 +1129,7 @@ static int ompi_attr_delete_impl(ompi_attribute_type_t type, void *object,
         goto exit;
     }
 
-    /* Check if the key is valid for the communicator/window/dtype. If
+    /* Check if the key is valid for the communicator/window/dtype/instance. If
        yes, then delete the attribute and key entry from the object's
        hash */
     ret = opal_hash_table_get_value_uint32(attr_hash, key, (void**) &attr);
@@ -1124,7 +1200,7 @@ int ompi_attr_delete_all(ompi_attribute_type_t type, void *object,
     int ret, i, num_attrs;
     uint32_t key;
     void *node, *in_node, *attr;
-    attribute_value_t **attrs;
+    attribute_key_value_t **attrs;
 
     /* Ensure that the table is not empty */
 
@@ -1141,7 +1217,7 @@ int ompi_attr_delete_all(ompi_attribute_type_t type, void *object,
         return MPI_SUCCESS;
     }
 
-    attrs = malloc(sizeof(attribute_value_t *) * num_attrs);
+    attrs = malloc(sizeof(attribute_key_value_t *) * num_attrs);
     if (NULL == attrs) {
         OPAL_THREAD_UNLOCK(&attribute_lock);
         return OMPI_ERR_OUT_OF_RESOURCE;
@@ -1156,7 +1232,7 @@ int ompi_attr_delete_all(ompi_attribute_type_t type, void *object,
     }
 
     /* Sort attributes in the order that they were set */
-    qsort(attrs, num_attrs, sizeof(attribute_value_t *), compare_attr_sequence);
+    qsort(attrs, num_attrs, sizeof(attribute_key_value_t *), compare_attr_sequence);
 
     /* Delete attributes in the reverse order that they were set.
        Actually this ordering is required only for MPI_COMM_SELF, as
@@ -1187,18 +1263,18 @@ int ompi_attr_delete_all(ompi_attribute_type_t type, void *object,
  */
 static int set_value(ompi_attribute_type_t type, void *object,
                      opal_hash_table_t **attr_hash, int key,
-                     attribute_value_t *new_attr,
+                     attribute_key_value_t *new_attr,
                      bool predefined)
 {
     ompi_attribute_keyval_t *keyval;
     int ret;
-    attribute_value_t *old_attr;
+    attribute_key_value_t *old_attr;
     bool had_old = false;
 
     /* Note that this function can be invoked by ompi_attr_copy_all()
        to set attributes on the new object (in addition to the
        top-level MPI_* functions that set attributes). */
-    ret = opal_hash_table_get_value_uint32(keyval_hash, key,
+    ret = opal_hash_table_get_value_uint32(attr_subsys->keyval_hash, key,
                                            (void **) &keyval);
 
     /* If key not found */
@@ -1242,7 +1318,7 @@ static int set_value(ompi_attribute_type_t type, void *object,
         had_old = true;
     }
 
-    ret = opal_hash_table_get_value_uint32(keyval_hash, key,
+    ret = opal_hash_table_get_value_uint32(attr_subsys->keyval_hash, key,
                                            (void **) &keyval);
     if ((OMPI_SUCCESS != ret ) || (NULL == keyval)) {
         /* Keyval has disappeared underneath us -- this shouldn't
@@ -1257,7 +1333,7 @@ static int set_value(ompi_attribute_type_t type, void *object,
     ret = opal_hash_table_set_value_uint32(*attr_hash, key, new_attr);
 
     /* Increase the reference count of the object, only if there was no
-       old atribute/no old entry in the object's key hash */
+       old attribute/no old entry in the object's key hash */
     if (OMPI_SUCCESS == ret && !had_old) {
         OBJ_RETAIN(keyval);
     }
@@ -1276,7 +1352,7 @@ static int set_value(ompi_attribute_type_t type, void *object,
  * Assumes that you do NOT already have the attribute lock.
  */
 static int get_value(opal_hash_table_t *attr_hash, int key,
-                     attribute_value_t **attribute, int *flag)
+                     attribute_key_value_t **attribute, int *flag)
 {
     int ret;
     void *attr;
@@ -1288,7 +1364,7 @@ static int get_value(opal_hash_table_t *attr_hash, int key,
        with the key, then the call is valid and returns FALSE in the
        flag argument */
     *flag = 0;
-    ret = opal_hash_table_get_value_uint32(keyval_hash, key,
+    ret = opal_hash_table_get_value_uint32(attr_subsys->keyval_hash, key,
                                            (void**) &keyval);
     if (OMPI_ERR_NOT_FOUND == ret) {
         return MPI_KEYVAL_INVALID;
@@ -1302,7 +1378,7 @@ static int get_value(opal_hash_table_t *attr_hash, int key,
 
     ret = opal_hash_table_get_value_uint32(attr_hash, key, &attr);
     if (OMPI_SUCCESS == ret) {
-        *attribute = (attribute_value_t*)attr;
+        *attribute = (attribute_key_value_t*)attr;
         *flag = 1;
     }
 
@@ -1318,25 +1394,25 @@ static int get_value(opal_hash_table_t *attr_hash, int key,
  * This function does not fail -- it is only invoked in "safe"
  * situations.
  */
-static void *translate_to_c(attribute_value_t *val)
+static void *translate_to_c(attribute_key_value_t *val)
 {
     switch (val->av_set_from) {
     case OMPI_ATTRIBUTE_C:
         /* Case 1: wrote a C pointer, read a C pointer
            (unity) */
-        return val->av_value;
+        return val->av_value.av_pointer;
 
     case OMPI_ATTRIBUTE_INT:
         /* Case 4: wrote an int, read a C pointer */
-        return (void *) val->av_int_pointer;
+        return &val->av_value.av_int;
 
     case OMPI_ATTRIBUTE_FINT:
         /* Case 7: wrote a MPI_Fint, read a C pointer */
-        return (void *) val->av_fint_pointer;
+        return &val->av_value.av_fint;
 
     case OMPI_ATTRIBUTE_AINT:
         /* Case 10: wrote a MPI_Aint, read a C pointer */
-        return (void *) val->av_aint_pointer;
+        return &val->av_value.av_aint;
 
     default:
         /* Should never reach here */
@@ -1352,25 +1428,25 @@ static void *translate_to_c(attribute_value_t *val)
  * This function does not fail -- it is only invoked in "safe"
  * situations.
  */
-static MPI_Fint translate_to_fint(attribute_value_t *val)
+static MPI_Fint translate_to_fint(attribute_key_value_t *val)
 {
     switch (val->av_set_from) {
     case OMPI_ATTRIBUTE_C:
         /* Case 2: wrote a C pointer, read a MPI_Fint */
-        return (MPI_Fint)*val->av_int_pointer;
+        return (MPI_Fint)(intptr_t)val->av_value.av_pointer;
 
     case OMPI_ATTRIBUTE_INT:
         /* Case 5: wrote an int, read a MPI_Fint */
-        return (MPI_Fint)*val->av_int_pointer;
+        return (MPI_Fint)val->av_value.av_int;
 
     case OMPI_ATTRIBUTE_FINT:
         /* Case 8: wrote a MPI_Fint, read a MPI_Fint
            (unity) */
-        return *val->av_fint_pointer;
+        return val->av_value.av_fint;
 
     case OMPI_ATTRIBUTE_AINT:
         /* Case 11: wrote a MPI_Aint, read a MPI_Fint */
-        return (MPI_Fint)*val->av_fint_pointer;
+        return (MPI_Fint)val->av_value.av_aint;
 
     default:
         /* Should never reach here */
@@ -1386,25 +1462,25 @@ static MPI_Fint translate_to_fint(attribute_value_t *val)
  * This function does not fail -- it is only invoked in "safe"
  * situations.
  */
-static MPI_Aint translate_to_aint(attribute_value_t *val)
+static MPI_Aint translate_to_aint(attribute_key_value_t *val)
 {
     switch (val->av_set_from) {
     case OMPI_ATTRIBUTE_C:
        /* Case 3: wrote a C pointer, read a MPI_Aint */
-        return (MPI_Aint) val->av_value;
+        return (MPI_Aint) val->av_value.av_pointer;
 
     case OMPI_ATTRIBUTE_INT:
         /* Case 6: wrote an int, read a MPI_Aint */
-        return (MPI_Aint) *val->av_int_pointer;
+        return (MPI_Aint) val->av_value.av_int;
 
     case OMPI_ATTRIBUTE_FINT:
         /* Case 9: wrote a MPI_Fint, read a MPI_Aint */
-        return (MPI_Aint) *val->av_fint_pointer;
+        return (MPI_Aint) val->av_value.av_fint;
 
     case OMPI_ATTRIBUTE_AINT:
         /* Case 12: wrote a MPI_Aint, read a MPI_Aint
            (unity) */
-        return (MPI_Aint) val->av_value;
+        return val->av_value.av_aint;
 
     default:
         /* Should never reach here */
@@ -1417,6 +1493,6 @@ static MPI_Aint translate_to_aint(attribute_value_t *val)
  */
 static int compare_attr_sequence(const void *attr1, const void *attr2)
 {
-    return (*(attribute_value_t **)attr1)->av_sequence -
-           (*(attribute_value_t **)attr2)->av_sequence;
+    return (*(attribute_key_value_t **)attr1)->av_sequence -
+           (*(attribute_key_value_t **)attr2)->av_sequence;
 }
diff --git a/ompi/attribute/attribute.h b/ompi/attribute/attribute.h
index 2bec4387dad..6292f1b0c55 100644
--- a/ompi/attribute/attribute.h
+++ b/ompi/attribute/attribute.h
@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -12,6 +13,11 @@
  * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2017      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018-2019 Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * Copyright (c) 2023      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -55,7 +61,7 @@ enum ompi_attribute_type_t {
                       * with 1 so that we can have it initialized to 0
                       * using memset in the constructor */
     TYPE_ATTR,       /**< The attribute belongs to datatype object */
-    WIN_ATTR         /**< The attribute belongs to a win object */
+    WIN_ATTR,        /**< The attribute belongs to a win object */
 };
 typedef enum ompi_attribute_type_t ompi_attribute_type_t;
 
@@ -64,50 +70,22 @@ typedef enum ompi_attribute_type_t ompi_attribute_type_t;
    delete. These will only be used here and not in the front end
    functions. */
 
-typedef void (ompi_fint_copy_attr_function)(MPI_Fint *oldobj,
-                                                    MPI_Fint *keyval,
-                                                    MPI_Fint *extra_state,
-                                                    MPI_Fint *attr_in,
-                                                    MPI_Fint *attr_out,
-                                                    ompi_fortran_logical_t *flag,
-                                                    MPI_Fint *ierr);
-typedef void (ompi_fint_delete_attr_function)(MPI_Fint *obj,
-                                                      MPI_Fint *keyval,
-                                                      MPI_Fint *attr_in,
-                                                      MPI_Fint *extra_state,
-                                                      MPI_Fint *ierr);
+typedef void (*ompi_fint_copy_attr_function)(MPI_Fint *oldobj, MPI_Fint *keyval,
+                                             MPI_Fint *extra_state, MPI_Fint *attr_in,
+                                             MPI_Fint *attr_out, ompi_fortran_logical_t *flag,
+                                             MPI_Fint *ierr);
+typedef void (*ompi_fint_delete_attr_function)(MPI_Fint *obj, MPI_Fint *keyval, MPI_Fint *attr_in,
+                                               MPI_Fint *extra_state, MPI_Fint *ierr);
 
 /* New-style MPI-2 Fortran function pointer declarations for copy and
    delete. These will only be used here and not in the front end
    functions. */
 
-typedef void (ompi_aint_copy_attr_function)(MPI_Fint *oldobj,
-                                            MPI_Fint *keyval,
-                                            void *extra_state,
-                                            void *attr_in,
-                                            void *attr_out,
-                                            ompi_fortran_logical_t *flag,
-                                            MPI_Fint *ierr);
-typedef void (ompi_aint_delete_attr_function)(MPI_Fint *obj,
-                                              MPI_Fint *keyval,
-                                              void *attr_in,
-                                              void *extra_state,
-                                              MPI_Fint *ierr);
-/*
- * Internally the copy function for all kinds of MPI objects has one more
- * argument, the pointer to the new object. Therefore, we can do on the
- * flight modifications of the new communicator based on attributes stored
- * on the main communicator.
- */
-typedef int (MPI_Comm_internal_copy_attr_function)(MPI_Comm, int, void *,
-                                                   void *, void *, int *,
-                                                   MPI_Comm);
-typedef int (MPI_Type_internal_copy_attr_function)(MPI_Datatype, int, void *,
-                                                   void *, void *, int *,
-                                                   MPI_Datatype);
-typedef int (MPI_Win_internal_copy_attr_function)(MPI_Win, int, void *,
-                                                  void *, void *, int *,
-                                                  MPI_Win);
+typedef void (*ompi_aint_copy_attr_function)(MPI_Fint *oldobj, MPI_Fint *keyval, void *extra_state,
+                                             void *attr_in, void *attr_out,
+                                             ompi_fortran_logical_t *flag, MPI_Fint *ierr);
+typedef void (*ompi_aint_delete_attr_function)(MPI_Fint *obj, MPI_Fint *keyval, void *attr_in,
+                                               void *extra_state, MPI_Fint *ierr);
 
 typedef void (ompi_attribute_keyval_destructor_fn_t)(int);
 
@@ -120,19 +98,17 @@ union ompi_attribute_fn_ptr_union_t {
     MPI_Type_delete_attr_function          *attr_datatype_delete_fn;
     MPI_Win_delete_attr_function           *attr_win_delete_fn;
 
-    MPI_Comm_internal_copy_attr_function   *attr_communicator_copy_fn;
-    MPI_Type_internal_copy_attr_function   *attr_datatype_copy_fn;
-    MPI_Win_internal_copy_attr_function    *attr_win_copy_fn;
+    MPI_Comm_copy_attr_function            *attr_communicator_copy_fn;
+    MPI_Type_copy_attr_function            *attr_datatype_copy_fn;
+    MPI_Win_copy_attr_function             *attr_win_copy_fn;
 
     /* For Fortran old MPI-1 callback functions */
-
-    ompi_fint_delete_attr_function *attr_fint_delete_fn;
-    ompi_fint_copy_attr_function   *attr_fint_copy_fn;
+    ompi_fint_delete_attr_function          attr_fint_delete_fn;
+    ompi_fint_copy_attr_function            attr_fint_copy_fn;
 
     /* For Fortran new MPI-2 callback functions */
-
-    ompi_aint_delete_attr_function *attr_aint_delete_fn;
-    ompi_aint_copy_attr_function   *attr_aint_copy_fn;
+    ompi_aint_delete_attr_function          attr_aint_delete_fn;
+    ompi_aint_copy_attr_function            attr_aint_copy_fn;
 };
 
 typedef union ompi_attribute_fn_ptr_union_t ompi_attribute_fn_ptr_union_t;
@@ -201,22 +177,26 @@ int ompi_attr_hash_init(opal_hash_table_t **hash)
 }
 
 /**
- * Initialize the main attribute hash that stores the keyvals and meta data
+ * Increase the reference count on the attributes subsystem.  Instantiate subsys if
+ * not yet instantiated.
  *
  * @return OMPI return code
  */
 
-int ompi_attr_init(void);
+int ompi_attr_get_ref(void);
+
 
 /**
- * Destroy the main attribute hash that stores the keyvals and meta data
+ * Decrease the reference count on the attributes subsystem.  Attributes subsystem
+ * resources are released when the count drops to zero.
+ *
+ * @return OMPI return code
  */
 
-int ompi_attr_finalize(void);
-
+int ompi_attr_put_ref(void);
 
 /**
- * Create a new key for use by attribute of Comm/Win/Datatype
+ * Create a new key for use by attribute of Comm/Win/Datatype/Instance
  *
  * @param type           Type of attribute (COMM/WIN/DTYPE) (IN)
  * @param copy_attr_fn   Union variable containing the function pointer
@@ -289,7 +269,7 @@ int ompi_attr_free_keyval(ompi_attribute_type_t type, int *key,
  * Set an attribute on the comm/win/datatype in a form valid for C.
  *
  * @param type           Type of attribute (COMM/WIN/DTYPE) (IN)
- * @param object         The actual Comm/Win/Datatype object (IN)
+ * @param object         The actual Comm/Win/Datatype/Instance object (IN)
  * @param attr_hash      The attribute hash table hanging on the object(IN/OUT)
  * @param key            Key val for the attribute (IN)
  * @param attribute      The actual attribute pointer (IN)
@@ -318,7 +298,7 @@ int ompi_attr_set_c(ompi_attribute_type_t type, void *object,
  * Set an int predefined attribute in a form valid for C.
  *
  * @param type           Type of attribute (COMM/WIN/DTYPE) (IN)
- * @param object         The actual Comm/Win/Datatype object (IN)
+ * @param object         The actual Comm/Win/Datatype/Instance object (IN)
  * @param attr_hash      The attribute hash table hanging on the object(IN/OUT)
  * @param key            Key val for the attribute (IN)
  * @param attribute      The actual attribute value (IN)
@@ -348,7 +328,7 @@ int ompi_attr_set_int(ompi_attribute_type_t type, void *object,
  * Fortran MPI-1.
  *
  * @param type           Type of attribute (COMM/WIN/DTYPE) (IN)
- * @param object         The actual Comm/Win/Datatype object (IN)
+ * @param object         The actual Comm/Win/Datatype/Instance object (IN)
  * @param attr_hash      The attribute hash table hanging on the object(IN/OUT)
  * @param key            Key val for the attribute (IN)
  * @param attribute      The actual attribute pointer (IN)
@@ -379,7 +359,7 @@ OMPI_DECLSPEC int ompi_attr_set_fint(ompi_attribute_type_t type, void *object,
  * Fortran MPI-2.
  *
  * @param type           Type of attribute (COMM/WIN/DTYPE) (IN)
- * @param object         The actual Comm/Win/Datatype object (IN)
+ * @param object         The actual Comm/Win/Datatype/Instance object (IN)
  * @param attr_hash      The attribute hash table hanging on the object(IN/OUT)
  * @param key            Key val for the attribute (IN)
  * @param attribute      The actual attribute pointer (IN)
@@ -488,7 +468,7 @@ OMPI_DECLSPEC int ompi_attr_get_aint(opal_hash_table_t *attr_hash, int key,
 /**
  * Delete an attribute on the comm/win/datatype
  * @param type           Type of attribute (COMM/WIN/DTYPE) (IN)
- * @param object         The actual Comm/Win/Datatype object (IN)
+ * @param object         The actual Comm/Win/Datatype/Instance object (IN)
  * @param attr_hash      The attribute hash table hanging on the object(IN)
  * @param key            Key val for the attribute (IN)
  * @param predefined     Whether the key is predefined or not 0/1 (IN)
@@ -503,7 +483,7 @@ int ompi_attr_delete(ompi_attribute_type_t type, void *object,
 
 /**
  * This to be used from functions like MPI_*_DUP in order to copy all
- * the attributes from the old Comm/Win/Dtype object to a new
+ * the attributes from the old Comm/Win/Dtype/Instance object to a new
  * object.
  * @param type         Type of attribute (COMM/WIN/DTYPE) (IN)
  * @param old_object   The old COMM/WIN/DTYPE object (IN)
@@ -520,7 +500,7 @@ int ompi_attr_copy_all(ompi_attribute_type_t type, void *old_object,
 
 
 /**
- * This to be used to delete all the attributes from the Comm/Win/Dtype
+ * This to be used to delete all the attributes from the Comm/Win/Dtype/Instance
  * object in one shot
  * @param type         Type of attribute (COMM/WIN/DTYPE) (IN)
  * @param object       The COMM/WIN/DTYPE object (IN)
@@ -536,11 +516,23 @@ int ompi_attr_delete_all(ompi_attribute_type_t type, void *object,
 /**
  * \internal
  *
- * Create all the predefined attributes
+ * Create all the predefined attribute keys
+ * @note This routine is invoked when creating a session 
+ *       so must be thread safe.
+ *
+ * @returns OMPI_SUCCESS
+ */
+int ompi_attr_create_predefined_keyvals(void);
+
+/**
+ * \internal
+ *
+ * Cache and release the predefined attribute keys used in the World Model
  *
  * @returns OMPI_SUCCESS
  */
-int ompi_attr_create_predefined(void);
+int ompi_attr_set_predefined_keyvals_for_wm(void);
+void ompi_attr_delete_predefined_keyvals_for_wm(void);
 
 /**
  * \internal
diff --git a/ompi/attribute/attribute_predefined.c b/ompi/attribute/attribute_predefined.c
index df948378bb7..3bc1849dc52 100644
--- a/ompi/attribute/attribute_predefined.c
+++ b/ompi/attribute/attribute_predefined.c
@@ -14,6 +14,11 @@
  * Copyright (c) 2017      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2020      Intel, Inc.  All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * Copyright (c) 2022      Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2023      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -48,7 +53,7 @@
  *
  * MPI_HOST is set to MPI_PROC_NULL (per MPI-1, see 7.1.1, p192).
  *
- * MPI_IO is set to MPI_ANY_SOURCE.  We may need to revist this.
+ * MPI_IO is set to MPI_ANY_SOURCE.  We may need to revisit this.
  *
  * MPI_WTIME_IS_GLOBAL is set to 0 (a conservative answer).
  *
@@ -74,7 +79,7 @@
  * integer-valued attributes, *BUT* at least the MPI_WIN_SIZE is an
  * MPI_Aint, so in terms of consistency, both should be the same --
  * hence, we treat them as MPI-2 Fortran integer-valued attributes.
- * All three of these atrributes have NULL_COPY_FN copy functions; it
+ * All three of these attributes have NULL_COPY_FN copy functions; it
  * doesn't make sense to copy them to new windows (because they're
  * values specific and unique to each window) -- especially when
  * WIN_CREATE will explicitly set them on new windows anyway.
@@ -93,6 +98,8 @@
 #include "ompi/mca/pml/pml.h"
 #include "ompi/runtime/ompi_rte.h"
 
+static bool attrs_predefined_initialized = false;
+
 /*
  * Private functions
  */
@@ -103,34 +110,55 @@ static int create_win(int target_keyval);
 static int free_win(int keyval);
 
 static int set_f(int keyval, MPI_Fint value);
+static int unset_f(int keyval);
 
-
-int ompi_attr_create_predefined(void)
+/*
+ * We do not need a lock here as this function is invoked when the 
+ * instance_lock mutex is held.
+ */
+int ompi_attr_create_predefined_keyvals(void)
 {
-    int ret;
-
-    /* Create all the keyvals */
-
-    /* DO NOT CHANGE THE ORDER OF CREATING THESE KEYVALS!  This order
-       strictly adheres to the order in mpi.h.  If you change the
-       order here, you must change the order in mpi.h as well! */
-
-    if (OMPI_SUCCESS != (ret = create_comm(MPI_TAG_UB, true)) ||
-        OMPI_SUCCESS != (ret = create_comm(MPI_HOST, true)) ||
-        OMPI_SUCCESS != (ret = create_comm(MPI_IO, true)) ||
-        OMPI_SUCCESS != (ret = create_comm(MPI_WTIME_IS_GLOBAL, true)) ||
-        OMPI_SUCCESS != (ret = create_comm(MPI_APPNUM, true)) ||
-        OMPI_SUCCESS != (ret = create_comm(MPI_LASTUSEDCODE, false)) ||
-        OMPI_SUCCESS != (ret = create_comm(MPI_UNIVERSE_SIZE, true)) ||
-        OMPI_SUCCESS != (ret = create_win(MPI_WIN_BASE)) ||
-        OMPI_SUCCESS != (ret = create_win(MPI_WIN_SIZE)) ||
-        OMPI_SUCCESS != (ret = create_win(MPI_WIN_DISP_UNIT)) ||
-        OMPI_SUCCESS != (ret = create_win(MPI_WIN_CREATE_FLAVOR)) ||
-        OMPI_SUCCESS != (ret = create_win(MPI_WIN_MODEL)) ||
-        OMPI_SUCCESS != (ret = create_comm(MPI_FT, false)) || /* not #if conditional on OPAL_ENABLE_FT_MPI for ABI */
-        0) {
-        return ret;
+    int ret = OMPI_SUCCESS, rc;
+
+    if (false == attrs_predefined_initialized) {
+
+        attrs_predefined_initialized = true;
+
+        /* Create all the keyvals */
+
+        /* DO NOT CHANGE THE ORDER OF CREATING THESE KEYVALS!  This order
+            strictly adheres to the order in mpi.h.  If you change the
+            order here, you must change the order in mpi.h as well! */
+
+        if (OMPI_SUCCESS != (rc = create_comm(MPI_TAG_UB, true)) ||
+            OMPI_SUCCESS != (rc = create_comm(MPI_HOST, true)) ||
+            OMPI_SUCCESS != (rc = create_comm(MPI_IO, true)) ||
+            OMPI_SUCCESS != (rc = create_comm(MPI_WTIME_IS_GLOBAL, true)) ||
+            OMPI_SUCCESS != (rc = create_comm(MPI_APPNUM, true)) ||
+            OMPI_SUCCESS != (rc = create_comm(MPI_LASTUSEDCODE, false)) ||
+            OMPI_SUCCESS != (rc = create_comm(MPI_UNIVERSE_SIZE, true)) ||
+            OMPI_SUCCESS != (rc = create_win(MPI_WIN_BASE)) ||
+            OMPI_SUCCESS != (rc = create_win(MPI_WIN_SIZE)) ||
+            OMPI_SUCCESS != (rc = create_win(MPI_WIN_DISP_UNIT)) ||
+            OMPI_SUCCESS != (rc = create_win(MPI_WIN_CREATE_FLAVOR)) ||
+            OMPI_SUCCESS != (rc = create_win(MPI_WIN_MODEL)) ||
+            OMPI_SUCCESS != (rc = create_comm(MPI_FT, false)) || /* not #if conditional on OPAL_ENABLE_FT_MPI for ABI */
+            0) {
+            ret = rc;
+        }
+
     }
+ 
+    return ret;
+}
+
+/*
+ * This method is only invoked during MPI initialization using the world model
+ * (MPI_Init/MPI_Init_thread) so does not need to be thread safe.
+ */
+int ompi_attr_set_predefined_keyvals_for_wm(void)
+{
+    int ret = OMPI_SUCCESS;
 
     /* Set default values for everything except MPI_UNIVERSE_SIZE */
 
@@ -162,27 +190,50 @@ int ompi_attr_create_predefined(void)
     return ret;
 }
 
+void ompi_attr_delete_predefined_keyvals_for_wm(void)
+{
+    unset_f(MPI_TAG_UB);
+    unset_f(MPI_HOST);
+    unset_f(MPI_IO);
+    unset_f(MPI_WTIME_IS_GLOBAL);
+    unset_f(MPI_FT);
+    unset_f(MPI_LASTUSEDCODE);
+    unset_f(MPI_UNIVERSE_SIZE);
+    unset_f(MPI_APPNUM);
+}
+
+/*
+ * We do not need a lock here as this function is invoked when the 
+ * destructor for attr_subsys is invoked.
+ */
 
 int ompi_attr_free_predefined(void)
 {
-    int ret;
-
-    if (OMPI_SUCCESS != (ret = free_comm(MPI_TAG_UB)) ||
-        OMPI_SUCCESS != (ret = free_comm(MPI_HOST)) ||
-        OMPI_SUCCESS != (ret = free_comm(MPI_IO)) ||
-        OMPI_SUCCESS != (ret = free_comm(MPI_WTIME_IS_GLOBAL)) ||
-        OMPI_SUCCESS != (ret = free_comm(MPI_APPNUM)) ||
-        OMPI_SUCCESS != (ret = free_comm(MPI_LASTUSEDCODE)) ||
-        OMPI_SUCCESS != (ret = free_comm(MPI_UNIVERSE_SIZE)) ||
-        OMPI_SUCCESS != (ret = free_comm(MPI_FT)) || /* not #if conditional on OPAL_ENABLE_FT_MPI for ABI */
-        OMPI_SUCCESS != (ret = free_win(MPI_WIN_BASE)) ||
-        OMPI_SUCCESS != (ret = free_win(MPI_WIN_SIZE)) ||
-        OMPI_SUCCESS != (ret = free_win(MPI_WIN_DISP_UNIT)) ||
-        OMPI_SUCCESS != (ret = free_win(MPI_WIN_CREATE_FLAVOR)) ||
-        OMPI_SUCCESS != (ret = free_win(MPI_WIN_MODEL))) {
-        return ret;
+    int ret = OMPI_SUCCESS, rc;
+
+    if (true == attrs_predefined_initialized) {
+
+        attrs_predefined_initialized = false;
+
+        if (OMPI_SUCCESS != (rc = free_comm(MPI_TAG_UB)) ||
+            OMPI_SUCCESS != (rc = free_comm(MPI_HOST)) ||
+            OMPI_SUCCESS != (rc = free_comm(MPI_IO)) ||
+            OMPI_SUCCESS != (rc = free_comm(MPI_WTIME_IS_GLOBAL)) ||
+            OMPI_SUCCESS != (rc = free_comm(MPI_APPNUM)) ||
+            OMPI_SUCCESS != (rc = free_comm(MPI_LASTUSEDCODE)) ||
+            OMPI_SUCCESS != (rc = free_comm(MPI_UNIVERSE_SIZE)) ||
+            OMPI_SUCCESS != (rc = free_comm(MPI_FT)) || /* not #if conditional on OPAL_ENABLE_FT_MPI for ABI */
+            OMPI_SUCCESS != (rc = free_win(MPI_WIN_BASE)) ||
+            OMPI_SUCCESS != (rc = free_win(MPI_WIN_SIZE)) ||
+            OMPI_SUCCESS != (rc = free_win(MPI_WIN_DISP_UNIT)) ||
+            OMPI_SUCCESS != (rc = free_win(MPI_WIN_CREATE_FLAVOR)) ||
+            OMPI_SUCCESS != (rc = free_win(MPI_WIN_MODEL))) {
+            ret = rc;
+        }
+
     }
-    return OMPI_SUCCESS;
+
+    return ret;
 }
 
 
@@ -194,9 +245,10 @@ static int create_comm(int target_keyval, bool want_inherit)
     ompi_attribute_fn_ptr_union_t del;
 
     keyval = -1;
-    copy.attr_communicator_copy_fn = (MPI_Comm_internal_copy_attr_function*)
-        (want_inherit ? MPI_COMM_DUP_FN : MPI_COMM_NULL_COPY_FN);
+    copy.attr_communicator_copy_fn =
+        want_inherit ? MPI_COMM_DUP_FN : MPI_COMM_NULL_COPY_FN;
     del.attr_communicator_delete_fn = MPI_COMM_NULL_DELETE_FN;
+    keyval = target_keyval;
     err = ompi_attr_create_keyval(COMM_ATTR, copy, del,
                                   &keyval, NULL, OMPI_KEYVAL_PREDEFINED, NULL);
     if (MPI_SUCCESS != err) {
@@ -224,8 +276,9 @@ static int create_win(int target_keyval)
     ompi_attribute_fn_ptr_union_t del;
 
     keyval = -1;
-    copy.attr_win_copy_fn = (MPI_Win_internal_copy_attr_function*)MPI_WIN_NULL_COPY_FN;
+    copy.attr_win_copy_fn = MPI_WIN_NULL_COPY_FN;
     del.attr_win_delete_fn = MPI_WIN_NULL_DELETE_FN;
+    keyval = target_keyval;
     err = ompi_attr_create_keyval(WIN_ATTR, copy, del,
                                   &keyval, NULL, OMPI_KEYVAL_PREDEFINED, NULL);
     if (MPI_SUCCESS != err) {
@@ -252,3 +305,10 @@ static int set_f(int keyval, MPI_Fint value)
                               keyval, value,
                               true);
 }
+
+static int unset_f(int keyval)
+{
+    return ompi_attr_delete(COMM_ATTR, MPI_COMM_WORLD,
+                            MPI_COMM_WORLD->c_keyhash,
+                            keyval, true);
+}
diff --git a/ompi/communicator/Makefile.am b/ompi/communicator/Makefile.am
index dcff49460b8..e1d25ad1026 100644
--- a/ompi/communicator/Makefile.am
+++ b/ompi/communicator/Makefile.am
@@ -15,6 +15,7 @@
 # Copyright (c) 2014      Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
 # Copyright (c) 2016      IBM Corporation.  All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -39,3 +40,4 @@ lib@OMPI_LIBMPI_NAME@_la_SOURCES += \
 	communicator/ft/comm_ft.c communicator/ft/comm_ft_reliable_bcast.c communicator/ft/comm_ft_propagator.c communicator/ft/comm_ft_detector.c communicator/ft/comm_ft_revoke.c
 endif # WANT_FT_MPI
 
+EXTRA_DIST += communicator/help-comm.txt
diff --git a/ompi/communicator/comm.c b/ompi/communicator/comm.c
index 3240f3db346..e7cdc68de63 100644
--- a/ompi/communicator/comm.c
+++ b/ompi/communicator/comm.c
@@ -3,10 +3,10 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2019 The University of Tennessee and The University
+ * Copyright (c) 2004-2023 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ * Copyright (c) 2004-2022 High Performance Computing Center Stuttgart,
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
@@ -22,8 +22,11 @@
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Mellanox Technologies. All rights reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2017-2022 IBM Corporation.  All rights reserved.
  * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -44,6 +47,7 @@
 #include "opal/mca/threads/mutex.h"
 #include "opal/util/bit_ops.h"
 #include "opal/util/output.h"
+#include "opal/util/show_help.h"
 #include "ompi/mca/topo/topo.h"
 #include "ompi/mca/topo/base/base.h"
 #include "ompi/dpm/dpm.h"
@@ -52,6 +56,51 @@
 #include "ompi/communicator/communicator.h"
 #include "ompi/mca/pml/pml.h"
 #include "ompi/request/request.h"
+#include "ompi/info/info_memkind.h"
+
+#include "ompi/runtime/params.h"
+
+struct ompi_comm_split_type_hw_guided_t {
+    const char *info_value;
+    int split_type;
+};
+typedef struct ompi_comm_split_type_hw_guided_t ompi_comm_split_type_hw_guided_t;
+
+/*
+ * The ompi_comm_split_unguided function uses this array to determine the next
+ * topology to test for a MPI_COMM_TYPE_HW_UNGUIDED communicator split. Therefore,
+ * the order in this array must be from largest topology class to smallest.
+ */
+static const ompi_comm_split_type_hw_guided_t ompi_comm_split_type_hw_guided_support[] = {
+    {.info_value = "cluster",  .split_type = OMPI_COMM_TYPE_CLUSTER},
+    {.info_value = "cu",       .split_type = OMPI_COMM_TYPE_CU},
+    {.info_value = "host",     .split_type = OMPI_COMM_TYPE_HOST},
+    {.info_value = "mpi_shared_memory", .split_type = MPI_COMM_TYPE_SHARED},
+    {.info_value = "board",    .split_type = OMPI_COMM_TYPE_BOARD},
+    {.info_value = "numanode", .split_type = OMPI_COMM_TYPE_NUMA},
+    {.info_value = "socket",   .split_type = OMPI_COMM_TYPE_SOCKET},
+    {.info_value = "l3cache",  .split_type = OMPI_COMM_TYPE_L3CACHE},
+    {.info_value = "l2cache",  .split_type = OMPI_COMM_TYPE_L2CACHE},
+    {.info_value = "l1cache",  .split_type = OMPI_COMM_TYPE_L1CACHE},
+    {.info_value = "core",     .split_type = OMPI_COMM_TYPE_CORE},
+    {.info_value = "hwthread", .split_type = OMPI_COMM_TYPE_HWTHREAD},
+    {.info_value = NULL},
+};
+
+static const char * ompi_comm_split_type_to_str(int split_type) {
+    for (int i = 0; NULL != ompi_comm_split_type_hw_guided_support[i].info_value; ++i) {
+        if (split_type == ompi_comm_split_type_hw_guided_support[i].split_type) {
+            return ompi_comm_split_type_hw_guided_support[i].info_value;
+        }
+    }
+    if (MPI_COMM_TYPE_HW_GUIDED == split_type) {
+        return "MPI_COMM_TYPE_HW_GUIDED";
+    }
+    else if (MPI_COMM_TYPE_HW_UNGUIDED == split_type) {
+        return "MPI_COMM_TYPE_HW_UNGUIDED";
+    }
+    return "Unknown";
+}
 
 /*
 ** sort-function for MPI_Comm_split
@@ -91,6 +140,10 @@ static int ompi_comm_idup_internal (ompi_communicator_t *comm, ompi_group_t *gro
                                     opal_info_t *info, ompi_communicator_t **newcomm, ompi_request_t **req);
 
 
+static int ompi_comm_get_rprocs (ompi_communicator_t *local_comm, ompi_communicator_t *bridge_comm,
+                                 int local_leader, int remote_leader, int tag, int rsize,
+                                 ompi_proc_t ***rprocs);
+
 /**********************************************************************/
 /**********************************************************************/
 /**********************************************************************/
@@ -107,15 +160,15 @@ int ompi_comm_set ( ompi_communicator_t **ncomm,
                     int *remote_ranks,
                     opal_hash_table_t *attr,
                     ompi_errhandler_t *errh,
-                    bool copy_topocomponent,
                     ompi_group_t *local_group,
-                    ompi_group_t *remote_group )
+                    ompi_group_t *remote_group,
+                    uint32_t flags)
 {
     ompi_request_t *req;
     int rc;
 
     rc = ompi_comm_set_nb (ncomm, oldcomm, local_size, local_ranks, remote_size, remote_ranks,
-                           attr, errh, copy_topocomponent, local_group, remote_group, &req);
+                           attr, errh, local_group, remote_group, flags, &req);
     if (OMPI_SUCCESS != rc) {
         return rc;
     }
@@ -127,23 +180,25 @@ int ompi_comm_set ( ompi_communicator_t **ncomm,
     return rc;
 }
 
+static int ompi_comm_set_simple (ompi_communicator_t **ncomm, ompi_errhandler_t *errhandler,
+                                 ompi_group_t *local_group)
+{
+    return ompi_comm_set (ncomm, NULL, local_group->grp_proc_count, NULL, 0, NULL, NULL, errhandler,
+			  local_group, NULL, 0);
+}
+
+
 /*
  * if remote_group == &ompi_mpi_group_null, then the new communicator
  * is forced to be an inter communicator.
  */
-int ompi_comm_set_nb ( ompi_communicator_t **ncomm,
-                       ompi_communicator_t *oldcomm,
-                       int local_size,
-                       int *local_ranks,
-                       int remote_size,
-                       int *remote_ranks,
-                       opal_hash_table_t *attr,
-                       ompi_errhandler_t *errh,
-                       bool copy_topocomponent,
-                       ompi_group_t *local_group,
-                       ompi_group_t *remote_group,
-                       ompi_request_t **req )
+int ompi_comm_set_nb (ompi_communicator_t **ncomm, ompi_communicator_t *oldcomm, int local_size,
+                      int *local_ranks, int remote_size, int *remote_ranks, opal_hash_table_t *attr,
+                      ompi_errhandler_t *errh, ompi_group_t *local_group, ompi_group_t *remote_group,
+                      uint32_t flags, ompi_request_t **req)
 {
+    bool copy_topocomponent = !!(flags & OMPI_COMM_SET_FLAG_COPY_TOPOLOGY);
+    bool dup_comm = !(flags & OMPI_COMM_SET_FLAG_LOCAL_COMM_NODUP);
     ompi_communicator_t *newcomm = NULL;
     int ret;
 
@@ -162,11 +217,14 @@ int ompi_comm_set_nb ( ompi_communicator_t **ncomm,
     if (NULL == newcomm) {
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
+    newcomm->c_name = (char*) malloc (OPAL_MAX_OBJECT_NAME);
+    if (NULL == newcomm->c_name) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+    newcomm->c_name[0]    = '\0';
     newcomm->super.s_info = NULL;
     /* fill in the inscribing hyper-cube dimensions */
     newcomm->c_cube_dim = opal_cube_dim(local_size);
-    newcomm->c_id_available   = MPI_UNDEFINED;
-    newcomm->c_id_start_index = MPI_UNDEFINED;
 
     if (NULL == local_group) {
         /* determine how the list of local_rank can be stored most
@@ -181,8 +239,19 @@ int ompi_comm_set_nb ( ompi_communicator_t **ncomm,
         OBJ_RETAIN(newcomm->c_local_group);
     }
     newcomm->c_my_rank = newcomm->c_local_group->grp_my_rank;
+    newcomm->c_assertions = 0;
 
     /* Set remote group and duplicate the local comm, if applicable */
+    if ((NULL == remote_group) && (NULL != remote_ranks)) {
+        /* determine how the list of local_rank can be stored most
+           efficiently */
+        ret = ompi_group_incl(oldcomm->c_remote_group, remote_size,
+                              remote_ranks, &newcomm->c_remote_group);
+        if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
+            return ret;
+        }
+        remote_group = newcomm->c_remote_group;
+    }
     if ( NULL != remote_group ) {
         ompi_communicator_t *old_localcomm;
 
@@ -199,11 +268,17 @@ int ompi_comm_set_nb ( ompi_communicator_t **ncomm,
 
         newcomm->c_flags |= OMPI_COMM_INTER;
 
-        old_localcomm = OMPI_COMM_IS_INTRA(oldcomm) ? oldcomm : oldcomm->c_local_comm;
+        if (dup_comm) {
+            old_localcomm = OMPI_COMM_IS_INTRA(oldcomm) ? oldcomm : oldcomm->c_local_comm;
 
-        /* NTH: use internal idup function that takes a local group argument */
-        ompi_comm_idup_internal (old_localcomm, newcomm->c_local_group, NULL, NULL,
-                                 &newcomm->c_local_comm, req);
+            /* NTH: use internal idup function that takes a local group argument */
+            ompi_comm_idup_internal (old_localcomm, newcomm->c_local_group, NULL, NULL,
+                                     &newcomm->c_local_comm, req);
+        } else {
+            /* take ownership of the old communicator (it must be an intracommunicator) */
+            assert (OMPI_COMM_IS_INTRA(oldcomm));
+            newcomm->c_local_comm = oldcomm;
+        }
     } else {
         newcomm->c_remote_group = newcomm->c_local_group;
         OBJ_RETAIN(newcomm->c_remote_group);
@@ -221,7 +296,7 @@ int ompi_comm_set_nb ( ompi_communicator_t **ncomm,
     OBJ_RETAIN ( newcomm->error_handler );
 
     /* Set Topology, if required and if available */
-    if ( copy_topocomponent && (NULL != oldcomm->c_topo) ) {
+    if (NULL != oldcomm && copy_topocomponent && (NULL != oldcomm->c_topo) ) {
         /**
          * The MPI standard is pretty clear on this, the topology information
          * behave as info keys, and is copied only on MPI_Comm_dup.
@@ -233,7 +308,7 @@ int ompi_comm_set_nb ( ompi_communicator_t **ncomm,
     }
 
     /* Copy attributes and call according copy functions, if required */
-    if (NULL != oldcomm->c_keyhash) {
+    if (NULL != oldcomm && NULL != oldcomm->c_keyhash) {
         if (NULL != attr) {
             ompi_attr_hash_init(&newcomm->c_keyhash);
             if (OMPI_SUCCESS != (ret = ompi_attr_copy_all (COMM_ATTR, oldcomm,
@@ -245,6 +320,10 @@ int ompi_comm_set_nb ( ompi_communicator_t **ncomm,
         }
     }
 
+    if (NULL != oldcomm) {
+        newcomm->instance = oldcomm->instance;
+    }
+
     *ncomm = newcomm;
     return (OMPI_SUCCESS);
 }
@@ -271,8 +350,8 @@ int ompi_comm_group ( ompi_communicator_t* comm, ompi_group_t **group )
 /*
 ** Counterpart to MPI_Comm_create. To be used within OMPI.
 */
-int ompi_comm_create ( ompi_communicator_t *comm, ompi_group_t *group,
-                       ompi_communicator_t **newcomm )
+int ompi_comm_create_w_info (ompi_communicator_t *comm, ompi_group_t *group, opal_info_t *info,
+                             ompi_communicator_t **newcomm)
 {
     ompi_communicator_t *newcomp = NULL;
     int rsize;
@@ -350,9 +429,9 @@ int ompi_comm_create ( ompi_communicator_t *comm, ompi_group_t *group,
                          rranks,                   /* remote_ranks */
                          NULL,                     /* attrs */
                          comm->error_handler,      /* error handler */
-                         false,                    /* dont copy the topo */
                          group,                    /* local group */
-                         remote_group);            /* remote group */
+                         remote_group,             /* remote group */
+                         0);                       /* flags */
 
     if ( OMPI_SUCCESS != rc ) {
         goto exit;
@@ -364,9 +443,20 @@ int ompi_comm_create ( ompi_communicator_t *comm, ompi_group_t *group,
         goto exit;
     }
 
+    /* Copy info if there is one. */
+    newcomp->super.s_info = OBJ_NEW(opal_info_t);
+    if (info) {
+        opal_info_dup(info, &(newcomp->super.s_info));
+    }
+    ompi_info_memkind_assert_type type;
+    ompi_info_memkind_copy_or_set (&comm->instance->super, &newcomp->super, info, &type);
+    if (OMPI_INFO_MEMKIND_ASSERT_NO_ACCEL == type) {
+        newcomp->c_assertions |= OMPI_COMM_ASSERT_NO_ACCEL_BUF;
+    }
+
     /* Set name for debugging purposes */
-    snprintf(newcomp->c_name, MPI_MAX_OBJECT_NAME, "MPI COMMUNICATOR %d CREATE FROM %d",
-             newcomp->c_contextid, comm->c_contextid );
+    snprintf(newcomp->c_name, MPI_MAX_OBJECT_NAME, "MPI COMMUNICATOR %s CREATE FROM %s",
+	     ompi_comm_print_cid (newcomp), ompi_comm_print_cid (comm));
 
     /* Activate the communicator and init coll-component */
     rc = ompi_comm_activate (&newcomp, comm, NULL, NULL, NULL, false, mode);
@@ -397,6 +487,11 @@ int ompi_comm_create ( ompi_communicator_t *comm, ompi_group_t *group,
     return ( rc );
 }
 
+int ompi_comm_create ( ompi_communicator_t *comm, ompi_group_t *group,
+                       ompi_communicator_t **newcomm )
+{
+    return ompi_comm_create_w_info (comm, group, NULL, newcomm);
+}
 
 /**********************************************************************/
 /**********************************************************************/
@@ -574,9 +669,9 @@ int ompi_comm_split_with_info( ompi_communicator_t* comm, int color, int key,
                          rranks,             /* remote_ranks */
                          NULL,               /* attrs */
                          comm->error_handler,/* error handler */
-                         pass_on_topo,
-                         local_group,       /* local group */
-                         remote_group);     /* remote group */
+                         local_group,        /* local group */
+                         remote_group,       /* remote group */
+                         pass_on_topo ? OMPI_COMM_SET_FLAG_COPY_TOPOLOGY : 0); /* flags */
 
     if ( OMPI_SUCCESS != rc  ) {
         goto exit;
@@ -586,9 +681,8 @@ int ompi_comm_split_with_info( ompi_communicator_t* comm, int color, int key,
         OBJ_RELEASE(local_group);
         if (NULL != newcomp->c_local_comm) {
             snprintf(newcomp->c_local_comm->c_name, MPI_MAX_OBJECT_NAME,
-                     "MPI COMMUNICATOR %d SPLIT FROM %d",
-                     newcomp->c_local_comm->c_contextid,
-                     comm->c_local_comm->c_contextid );
+                     "MPI COMM %s SPLIT FROM %s", ompi_comm_print_cid (newcomp),
+		     ompi_comm_print_cid (comm));
         }
     }
 
@@ -607,15 +701,19 @@ int ompi_comm_split_with_info( ompi_communicator_t* comm, int color, int key,
     }
 
     /* Set name for debugging purposes */
-    snprintf(newcomp->c_name, MPI_MAX_OBJECT_NAME, "MPI COMMUNICATOR %d SPLIT FROM %d",
-             newcomp->c_contextid, comm->c_contextid );
+    snprintf(newcomp->c_name, MPI_MAX_OBJECT_NAME, "MPI COMM %s SPLIT FROM %s",
+	     ompi_comm_print_cid (newcomp), ompi_comm_print_cid (comm));
 
     /* Copy info if there is one */
+    newcomp->super.s_info = OBJ_NEW(opal_info_t);
     if (info) {
-        newcomp->super.s_info = OBJ_NEW(opal_info_t);
         opal_info_dup(info, &(newcomp->super.s_info));
     }
-
+    ompi_info_memkind_assert_type type;
+    ompi_info_memkind_copy_or_set (&comm->instance->super, &newcomp->super, info, &type);
+    if (OMPI_INFO_MEMKIND_ASSERT_NO_ACCEL == type) {
+        newcomp->c_assertions |= OMPI_COMM_ASSERT_NO_ACCEL_BUF;
+    }
     /* Activate the communicator and init coll-component */
     rc = ompi_comm_activate (&newcomp, comm, NULL, NULL, NULL, false, mode);
 
@@ -730,6 +828,20 @@ static int ompi_comm_split_type_get_part (ompi_group_t *group, const int split_t
         case OMPI_COMM_TYPE_CLUSTER:
             include = OPAL_PROC_ON_LOCAL_CLUSTER(locality);
             break;
+        case MPI_COMM_TYPE_HW_GUIDED:
+        case MPI_COMM_TYPE_HW_UNGUIDED:
+            /*
+             * MPI_COMM_TYPE_HW_(UN)GUIDED handled in calling function.
+             * We should not get here as the split type will be changed
+             * at a higher level.
+             */
+            opal_show_help("help-comm.txt",
+                           "unexpected-split-type",
+                           true,
+                           ompi_comm_split_type_to_str(split_type),
+                           split_type);
+            free (ranks);
+            return OMPI_ERR_BAD_PARAM;
         }
 
         if (include) {
@@ -785,7 +897,7 @@ static int ompi_comm_split_verify (ompi_communicator_t *comm, int split_type, in
     }
 
     for (int i = 0 ; i < size ; ++i) {
-        if (MPI_UNDEFINED == results[i * 2] || (i > 1 && results[i * 2 + 1] < results[i * 2 - 1])) {
+        if (MPI_UNDEFINED == results[i * 2] || (i >= 1 && results[i * 2 + 1] < results[i * 2 - 1])) {
             *need_split = true;
             break;
         }
@@ -796,76 +908,38 @@ static int ompi_comm_split_verify (ompi_communicator_t *comm, int split_type, in
     return OMPI_SUCCESS;
 }
 
-int ompi_comm_split_type (ompi_communicator_t *comm, int split_type, int key,
-                          opal_info_t *info, ompi_communicator_t **newcomm)
+/**
+ * ompi_comm_split_type_core: Perform common processing for a MPI_Comm_type_split
+ *                            function call.
+ *
+ * comm(in) i            : input communicator
+ * global_split_type(in) : Split type defined by all ranks in the communicator
+ * local_split_type(in)  : Split type defined by this rank in the communicator
+ *                         (might be MPI_UNDEFINED)
+ * key(in)               : original key from the user
+ * need_split            : If a split is needed to separate ranks supplying
+ *                         MPI_UNDEFINED from others
+ * no_reorder(in)        : Did all of the ranks specify the same key (optimization)
+ * no_undefined(in)      : None of the ranks specified MPI_UNDEFINED (optimization)
+ * info(out)             : info guiding the split operation
+ * newcomm(out)          : Pointer to the newly created communicator, or pointer to MPI_COMM_NULL
+ *                         if no communicator created.
+ */
+static int ompi_comm_split_type_core(ompi_communicator_t *comm,
+                                     int global_split_type, int local_split_type,
+                                     int key, bool need_split, bool no_reorder,
+                                     bool no_undefined, opal_info_t *info,
+                                     ompi_communicator_t **newcomm)
 {
-    bool need_split = false, no_reorder = false, no_undefined = false;
-    ompi_communicator_t *newcomp = MPI_COMM_NULL;
-    int my_size, my_rsize = 0, mode, inter;
     int *lranks = NULL, *rranks = NULL;
-    int global_split_type, ok, tmp[4];
+    ompi_communicator_t *newcomp = MPI_COMM_NULL;
+    int my_size, my_rsize = 0, mode;
     int rc;
+    int inter = OMPI_COMM_IS_INTER(comm);
 
-    /* silence clang warning. newcomm should never be NULL */
-    if (OPAL_UNLIKELY(NULL == newcomm)) {
-        return OMPI_ERR_BAD_PARAM;
-    }
-
-    inter = OMPI_COMM_IS_INTER(comm);
-
-    /* Step 1: verify all ranks have supplied the same value for split type. All split types
-     * must be the same or MPI_UNDEFINED (which is negative). */
-    tmp[0] = split_type;
-    tmp[1] = -split_type;
-    tmp[2] = key;
-    tmp[3] = -key;
-
-    rc = comm->c_coll->coll_allreduce (MPI_IN_PLACE, &tmp, 4, MPI_INT, MPI_MAX, comm,
-                                      comm->c_coll->coll_allreduce_module);
-    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
-        return rc;
-    }
-
-    global_split_type = tmp[0];
-
-    if (tmp[0] != -tmp[1] || inter) {
-        /* at least one rank supplied a different split type check if our split_type is ok */
-        ok = (MPI_UNDEFINED == split_type) || global_split_type == split_type;
-
-        rc = comm->c_coll->coll_allreduce (MPI_IN_PLACE, &ok, 1, MPI_INT, MPI_MIN, comm,
-                                          comm->c_coll->coll_allreduce_module);
-        if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
-            return rc;
-        }
-
-        if (inter) {
-            /* need an extra allreduce to ensure that all ranks have the same result */
-            rc = comm->c_coll->coll_allreduce (MPI_IN_PLACE, &ok, 1, MPI_INT, MPI_MIN, comm,
-                                              comm->c_coll->coll_allreduce_module);
-            if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
-                return rc;
-            }
-        }
-
-        if (OPAL_UNLIKELY(!ok)) {
-            return OMPI_ERR_BAD_PARAM;
-        }
-
-        need_split = tmp[0] == -tmp[1];
-    } else {
-        /* intracommunicator and all ranks specified the same split type */
-        no_undefined = true;
-        /* check if all ranks specified the same key */
-        no_reorder = tmp[2] == -tmp[3];
-    }
-
-    if (MPI_UNDEFINED == global_split_type) {
-        /* short-circut. every rank provided MPI_UNDEFINED */
-        *newcomm = MPI_COMM_NULL;
-        return OMPI_SUCCESS;
-    }
-
-    /* Step 2: Build potential communicator groups. If any ranks will not be part of
+    /* Perform the rest of the processing for a communicator split.
+     *
+     * Step 1: Build potential communicator groups. If any ranks will not be part of
      * the ultimate communicator we will drop them later. This saves doing an extra
      * allgather on the whole communicator. By using ompi_comm_split() later only
      * if needed we 1) optimized the common case (no MPI_UNDEFINED and no reorder),
@@ -896,7 +970,7 @@ int ompi_comm_split_type (ompi_communicator_t *comm, int split_type, int key,
         return rc;
     }
 
-    /* Step 3: determine all the information for the remote group */
+    /* Step 2: determine all the information for the remote group */
     /* --------------------------------------------------------- */
     if (inter) {
         rc = ompi_comm_split_type_get_part (comm->c_remote_group, global_split_type, &rranks, &my_rsize);
@@ -909,63 +983,68 @@ int ompi_comm_split_type (ompi_communicator_t *comm, int split_type, int key,
     /* set the CID allgather mode to the appropriate one for the communicator */
     mode = inter ? OMPI_COMM_CID_INTER : OMPI_COMM_CID_INTRA;
 
-    /* Step 4: set up the communicator                           */
+    /* Step 3: set up the communicator                           */
     /* --------------------------------------------------------- */
     /* Create the communicator finally */
 
-    do {
-        rc = ompi_comm_set (&newcomp, comm, my_size, lranks, my_rsize,
-                            rranks, NULL, comm->error_handler, false,
-                            NULL, NULL);
-        if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
-            break;
-        }
+    rc = ompi_comm_set (&newcomp, comm, my_size, lranks, my_rsize,
+                        rranks, NULL, comm->error_handler, NULL, NULL, 0);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+       goto exit;
+    }
 
-        /* Determine context id. It is identical to f_2_c_handle */
-        rc = ompi_comm_nextcid (newcomp, comm, NULL, NULL, NULL, false, mode);
-        if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
-            break;
-        }
+    /* Determine context id. It is identical to f_2_c_handle */
+    rc = ompi_comm_nextcid (newcomp, comm, NULL, NULL, NULL, false, mode);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+        goto exit;
+    }
 
-        // Copy info if there is one.
-        newcomp->super.s_info = OBJ_NEW(opal_info_t);
-        if (info) {
-            opal_info_dup(info, &(newcomp->super.s_info));
-        }
+    ompi_comm_assert_subscribe (newcomp, OMPI_COMM_ASSERT_LAZY_BARRIER);
+    ompi_comm_assert_subscribe (newcomp, OMPI_COMM_ASSERT_ACTIVE_POLL);
+    if (info) {
+        opal_infosubscribe_change_info(&newcomp->super, info);
+    }
+    ompi_info_memkind_assert_type type;
+    ompi_info_memkind_copy_or_set (&comm->instance->super, &newcomp->super, info, &type);
+    if (OMPI_INFO_MEMKIND_ASSERT_NO_ACCEL == type) {
+        newcomp->c_assertions |= OMPI_COMM_ASSERT_NO_ACCEL_BUF;
+    }
 
-        /* Activate the communicator and init coll-component */
-        rc = ompi_comm_activate (&newcomp, comm, NULL, NULL, NULL, false, mode);
-        if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
-            break;
-        }
+    /* Activate the communicator and init coll-component */
+    rc = ompi_comm_activate (&newcomp, comm, NULL, NULL, NULL, false, mode);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+        goto exit;
+    }
 
-        /* Step 5: Check if we need to remove or reorder ranks in the communicator */
-        if (!(no_reorder && no_undefined)) {
-            rc = ompi_comm_split_verify (newcomp, split_type, key, &need_split);
+    /* Step 4: Check if we need to remove or reorder ranks in the communicator */
+    if (!(no_reorder && no_undefined)) {
+        rc = ompi_comm_split_verify (newcomp, local_split_type, key, &need_split);
 
-            if (inter) {
-                /* verify that no local ranks need to be removed or reordered */
-                rc = ompi_comm_split_verify (newcomp->c_local_comm, split_type, key, &need_split);
-            }
+        if (inter) {
+            /* verify that no local ranks need to be removed or reordered */
+            rc = ompi_comm_split_verify (newcomp->c_local_comm, local_split_type, key,
+                                         &need_split);
         }
+    }
 
-        if (!need_split) {
-            /* common case. no reordering and no MPI_UNDEFINED */
-            *newcomm = newcomp;
+    if (!need_split) {
 
-            /* Set name for debugging purposes */
-            snprintf(newcomp->c_name, MPI_MAX_OBJECT_NAME, "MPI COMMUNICATOR %d SPLIT_TYPE FROM %d",
-                     newcomp->c_contextid, comm->c_contextid );
-            break;
-        }
+        /* common case. no reordering and no MPI_UNDEFINED */
+        *newcomm = newcomp;
 
-        /* TODO: there probably is better way to handle this case without throwing away the
-         * intermediate communicator. */
-        rc = ompi_comm_split (newcomp, split_type, key, newcomm, false);
-        /* get rid of the intermediate communicator */
-        ompi_comm_free (&newcomp);
-    } while (0);
+        /* Set name for debugging purposes */
+        snprintf(newcomp->c_name, MPI_MAX_OBJECT_NAME, "MPI COMM %s SPLIT_TYPE FROM %s",
+        ompi_comm_print_cid (newcomp), ompi_comm_print_cid (comm));
+        goto exit;
+    }
 
+    /* TODO: there probably is better way to handle this case without throwing away the
+     * intermediate communicator. */
+    rc = ompi_comm_split (newcomp, local_split_type, key, newcomm, false);
+    /* get rid of the intermediate communicator */
+    ompi_comm_free (&newcomp);
+
+ exit:
     if (OPAL_UNLIKELY(OMPI_SUCCESS != rc && MPI_COMM_NULL != newcomp)) {
         ompi_comm_free (&newcomp);
         *newcomm = MPI_COMM_NULL;
@@ -977,6 +1056,256 @@ int ompi_comm_split_type (ompi_communicator_t *comm, int split_type, int key,
     return rc;
 }
 
+/**
+ * ompi_comm_split_unguided: Process an MPI_Comm_split_type function call where the
+ *                           split is performed at the next lower topology level where
+ *                           the new communicator would contain fewer ranks than the
+ *                           input communicator.
+ *
+ * comm(in)         : Input communicator
+ * split_type(in)   : Communicator split type for this task, may be MPI_UNDEFINED
+ * key(in)          : Original key from the user
+ * need_split(in)   : If a split is needed to separate ranks supplying MPI_UNDEFINED
+ *                    from others
+ * no_reorder(in)   : Did all of the ranks specify the same key (optimization)
+ * no_undefined(in) : None of the ranks specified MPI_UNDEFINED (optimization)
+ * info(out)        : Info object to update with selected topology class
+ * newcomm(out)     : Pointer to newly created communicator, or pointer to
+ *                    MPI_COMM_NULL if no communicator generated.
+ */
+static int ompi_comm_split_unguided(ompi_communicator_t *comm, int split_type, int key,
+                                    bool need_split, bool no_reorder,
+                                    bool no_undefined, struct opal_info_t *info,
+                                    ompi_communicator_t** newcomm)
+{
+    int i, rc;
+    struct opal_info_t *split_info = NULL;
+    int new_size, original_size;
+    ompi_communicator_t *unguided_comm = NULL;
+
+    if (1 == ompi_comm_size(comm)) {
+        /* Communicator cannot be split any smaller */
+        *newcomm = MPI_COMM_NULL;
+        return OMPI_SUCCESS;
+    }
+
+    /*
+     * Split out the MPI_UNDEFINED
+     */
+    rc = ompi_comm_split(comm,
+                         MPI_UNDEFINED == split_type ? MPI_UNDEFINED : 0,
+                         key,
+                         &unguided_comm, false);
+    if (OMPI_SUCCESS != rc) {
+        *newcomm = MPI_COMM_NULL;
+        return rc;
+    }
+    if (MPI_UNDEFINED == split_type) {
+        ompi_comm_free(&unguided_comm);
+        *newcomm = MPI_COMM_NULL;
+        return OMPI_SUCCESS;
+    }
+
+    /*
+     * Attempt to split the communicator based on topology class by iteratively
+     * calling ompi_comm_split_type specifying the split type as
+     * MPI_COMM_TYPE_HW_GUIDED using the next lower topology class until a
+     * split results in a smaller size communicator than the input communicator.
+     * The search starts with OMPI_COMM_TYPE_CU since that is the highest possible
+     * topology class where the communicator size can be smaller than MPI_COMM_WORLD.
+     */
+    original_size = ompi_comm_size(unguided_comm);
+    split_info = OBJ_NEW(opal_info_t);
+    i = 1;
+    while (NULL != ompi_comm_split_type_hw_guided_support[i].info_value) {
+        /* MPI_COMM_TYPE_HW_GUIDED splits require mpi_hw_resource_type to be set */
+        opal_info_set(split_info, "mpi_hw_resource_type",
+                      ompi_comm_split_type_hw_guided_support[i].info_value);
+
+        rc = ompi_comm_split_type_core(unguided_comm,
+                                       ompi_comm_split_type_hw_guided_support[i].split_type,
+                                       ompi_comm_split_type_hw_guided_support[i].split_type,
+                                       key, need_split, no_reorder, no_undefined,
+                                       split_info, newcomm);
+        if (OMPI_SUCCESS != rc) {
+            break;
+        }
+
+        // Everyone else will get a new communicator.
+        // Check the size to see if we need to iterate again.
+        new_size = ompi_comm_size(*newcomm);
+        if (new_size < original_size) {
+            /* If a valid info object was passed, set the selected topology */
+            if (NULL != info) {
+                opal_info_set(info, "mpi_hw_resource_type", 
+                              ompi_comm_split_type_hw_guided_support[i].info_value);
+            }
+            ompi_comm_free(&unguided_comm);
+            OBJ_RELEASE(split_info);
+            return OMPI_SUCCESS;
+        }
+
+        // Free the new communicator, and prepare for the next iteration.
+        ompi_comm_free(newcomm);
+        i = i + 1;
+    }
+
+    ompi_comm_free(&unguided_comm);
+    OBJ_RELEASE(split_info);
+    *newcomm = MPI_COMM_NULL;
+    return rc;
+}
+
+/*
+ * ompi_comm_split_type: Performs a communicator split. This function performs initial
+ *                       processing to set up a  MPI_COMM_TYPE_HW_GUIDED split and
+ *                       validation of input parameters.
+ * comm(in)              : Input communicator
+ * split_type(in)        : Split type to be performed by this rank, may be MPI_UNDEFINED
+ * key(in)               : Original key from the user
+ * info(in/out)          : Info guiding the split operation
+ * newcomm(out)          : Pointer to the newly created communicator, or pointer to MPI_COMM_NULL
+ *                         if no communicator created.
+ */                         
+int ompi_comm_split_type (ompi_communicator_t *comm, int split_type, int key,
+                          opal_info_t *info, ompi_communicator_t **newcomm)
+{
+    bool need_split = false, no_reorder = false, no_undefined = false;
+    int inter;
+    int global_split_type, global_orig_split_type, ok[2], tmp[6];
+    int rc;
+    int orig_split_type = split_type;
+    int flag;
+    opal_cstring_t *value = NULL;
+
+    /* silence clang warning. newcomm should never be NULL */
+    if (OPAL_UNLIKELY(NULL == newcomm)) {
+        return OMPI_ERR_BAD_PARAM;
+    }
+
+    inter = OMPI_COMM_IS_INTER(comm);
+
+    /* Step 0: Convert MPI_COMM_TYPE_HW_GUIDED to the internal type */
+    if (MPI_COMM_TYPE_HW_GUIDED == split_type) {
+        opal_info_get(info, "mpi_hw_resource_type", &value, &flag);
+        /* If key is not in the 'info', then return MPI_COMM_NULL.
+         * This is caught at the MPI interface level, but it doesn't hurt to
+         * check it again.
+         */
+        if (!flag) {
+            *newcomm = MPI_COMM_NULL;
+            return OMPI_SUCCESS;
+        }
+
+        /* Verify the value associated with the "mpi_hw_resource_type" key
+         * - is supported, and
+         * - is the same value at all ranks
+         *
+         * If not supported, then return MPI_COMM_NULL.
+         * If not the same at all ranks, throw an error.
+         */
+        flag = 0;
+        for (int i = 0; NULL != ompi_comm_split_type_hw_guided_support[i].info_value; ++i) {
+            if (0 == strncasecmp(value->string,
+                                 ompi_comm_split_type_hw_guided_support[i].info_value,
+                                 strlen(ompi_comm_split_type_hw_guided_support[i].info_value))) {
+                split_type = ompi_comm_split_type_hw_guided_support[i].split_type;
+                flag = 1;
+                break;
+            }
+        }
+        /* If not supported, then return MPI_COMM_NULL. */
+        if (0 == flag) {
+            *newcomm = MPI_COMM_NULL;
+            return OMPI_SUCCESS;
+        }
+    }
+
+    /* Step 1: verify all ranks have supplied the same value for split type. All split types
+     * must be the same or MPI_UNDEFINED (which is negative). */
+    tmp[0] = orig_split_type;
+    tmp[1] = -orig_split_type;
+    tmp[2] = key;
+    tmp[3] = -key;
+    /* For MPI_COMM_TYPE_HW_GUIDED, verify all ranks have supplied the same
+     * split_type (represented by orig_split_type) and info 'value' (represented by split_type).
+     *
+     * For split_type != MPI_COMM_TYPE_HW_GUIDED then orig_split_type == split_type.
+     */
+    tmp[4] = split_type;
+    tmp[5] = -split_type;
+
+    rc = comm->c_coll->coll_allreduce (MPI_IN_PLACE, &tmp, 6, MPI_INT, MPI_MAX, comm,
+                                      comm->c_coll->coll_allreduce_module);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+        return rc;
+    }
+
+    global_orig_split_type = tmp[0];
+    global_split_type = tmp[4];
+
+    if (tmp[0] != -tmp[1] || tmp[4] != -tmp[5] || inter) {
+        /* at least one rank supplied a different split type check if our split_type is ok */
+        ok[0] = (MPI_UNDEFINED == orig_split_type) || global_orig_split_type == orig_split_type;
+        ok[1] = (MPI_UNDEFINED == orig_split_type) || global_split_type == split_type;
+
+        rc = comm->c_coll->coll_allreduce (MPI_IN_PLACE, &ok, 2, MPI_INT, MPI_MIN, comm,
+                                          comm->c_coll->coll_allreduce_module);
+        if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+            return rc;
+        }
+
+        if (inter) {
+            /* need an extra allreduce to ensure that all ranks have the same result */
+            rc = comm->c_coll->coll_allreduce (MPI_IN_PLACE, &ok, 2, MPI_INT, MPI_MIN, comm,
+                                              comm->c_coll->coll_allreduce_module);
+            if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+                return rc;
+            }
+        }
+
+        if (OPAL_UNLIKELY(!ok[0] || !ok[1])) {
+            if (0 == ompi_comm_rank(comm)) {
+                opal_info_get(info, "mpi_hw_resource_type", &value, &flag);
+                if (!flag) {
+                    value = NULL;
+                }
+                opal_show_help("help-comm.txt",
+                               "mismatched-split_type-values",
+                               true,
+                               ompi_comm_split_type_to_str(orig_split_type),
+                               orig_split_type,
+                               NULL == value ? "" : value->string);
+            }
+            return OMPI_ERR_BAD_PARAM;
+        }
+
+        need_split = tmp[0] == -tmp[1];
+    } else {
+        /* intracommunicator and all ranks specified the same split type */
+        no_undefined = true;
+        /* check if all ranks specified the same key */
+        no_reorder = tmp[2] == -tmp[3];
+    }
+
+    if (MPI_UNDEFINED == global_orig_split_type) {
+        /* short-circut. every rank provided MPI_UNDEFINED */
+        *newcomm = MPI_COMM_NULL;
+        return OMPI_SUCCESS;
+    }
+
+    if (MPI_COMM_TYPE_HW_UNGUIDED == global_orig_split_type) {
+        /* Handle MPI_COMM_TYPE_HW_UNGUIDED communicator split. */
+        return ompi_comm_split_unguided( comm, split_type,
+                                         key, need_split, no_reorder,
+                                         no_undefined, info, newcomm );
+    } else {
+        return ompi_comm_split_type_core( comm, global_split_type, split_type,
+                                          key, need_split, no_reorder,
+                                          no_undefined, info, newcomm);
+    }
+}
+
 /**********************************************************************/
 /**********************************************************************/
 /**********************************************************************/
@@ -1009,9 +1338,9 @@ int ompi_comm_dup_with_info ( ompi_communicator_t * comm, opal_info_t *info, omp
                           NULL,                                   /* remote_procs */
                           comm->c_keyhash,                        /* attrs */
                           comm->error_handler,                    /* error handler */
-                          true,                                   /* copy the topo */
                           comm->c_local_group,                    /* local group */
-                          remote_group );                         /* remote group */
+                          remote_group,                           /* remote group */
+                          OMPI_COMM_SET_FLAG_COPY_TOPOLOGY);      /* flags */
     if ( OMPI_SUCCESS != rc) {
         return rc;
     }
@@ -1024,13 +1353,19 @@ int ompi_comm_dup_with_info ( ompi_communicator_t * comm, opal_info_t *info, omp
     }
 
     /* Set name for debugging purposes */
-    snprintf(newcomp->c_name, MPI_MAX_OBJECT_NAME, "MPI COMMUNICATOR %d DUP FROM %d",
-             newcomp->c_contextid, comm->c_contextid );
+    snprintf(newcomp->c_name, MPI_MAX_OBJECT_NAME, "MPI COMM %s DUP FROM %s",
+	     ompi_comm_print_cid (newcomp), ompi_comm_print_cid (comm));
 
     // Copy info if there is one.
-    newcomp->super.s_info = OBJ_NEW(opal_info_t);
+    ompi_comm_assert_subscribe (newcomp, OMPI_COMM_ASSERT_LAZY_BARRIER);
+    ompi_comm_assert_subscribe (newcomp, OMPI_COMM_ASSERT_ACTIVE_POLL);
     if (info) {
-        opal_info_dup(info, &(newcomp->super.s_info));
+        opal_infosubscribe_change_info(&newcomp->super, info);
+    }
+    ompi_info_memkind_assert_type type;
+    ompi_info_memkind_copy_or_set (&comm->instance->super, &newcomp->super, info, &type);
+    if (OMPI_INFO_MEMKIND_ASSERT_NO_ACCEL == type) {
+        newcomp->c_assertions |= OMPI_COMM_ASSERT_NO_ACCEL_BUF;
     }
 
     /* activate communicator and init coll-module */
@@ -1106,9 +1441,9 @@ static int ompi_comm_idup_internal (ompi_communicator_t *comm, ompi_group_t *gro
                             NULL,                                   /* remote_procs */
                             comm->c_keyhash,                        /* attrs */
                             comm->error_handler,                    /* error handler */
-                            true,                                   /* copy the topo */
                             group,                                  /* local group */
                             remote_group,                           /* remote group */
+                            OMPI_COMM_SET_FLAG_COPY_TOPOLOGY,       /* flags */
                             subreq);                                /* new subrequest */
     if (OMPI_SUCCESS != rc) {
         ompi_comm_request_return (request);
@@ -1122,6 +1457,12 @@ static int ompi_comm_idup_internal (ompi_communicator_t *comm, ompi_group_t *gro
         if (info) {
             opal_info_dup(info, &(newcomp->super.s_info));
         }
+
+        ompi_info_memkind_assert_type type;
+        ompi_info_memkind_copy_or_set (&comm->instance->super, &newcomp->super, info, &type);
+        if (OMPI_INFO_MEMKIND_ASSERT_NO_ACCEL == type) {
+            newcomp->c_assertions |= OMPI_COMM_ASSERT_NO_ACCEL_BUF;
+        }
     }
 
     ompi_comm_request_schedule_append (request, ompi_comm_idup_getcid, subreq, subreq[0] ? 1 : 0);
@@ -1177,8 +1518,8 @@ static int ompi_comm_idup_with_info_activate (ompi_comm_request_t *request)
     }
 
     /* Set name for debugging purposes */
-    snprintf(context->newcomp->c_name, MPI_MAX_OBJECT_NAME, "MPI COMMUNICATOR %d DUP FROM %d",
-             context->newcomp->c_contextid, context->comm->c_contextid );
+    snprintf(context->newcomp->c_name, MPI_MAX_OBJECT_NAME, "MPI COMM %s DUP FROM %s",
+	     ompi_comm_print_cid (context->newcomp), ompi_comm_print_cid (context->comm));
 
     /* activate communicator and init coll-module */
     rc = ompi_comm_activate_nb (&context->newcomp, context->comm, NULL, NULL, NULL, false, mode, subreq);
@@ -1194,7 +1535,7 @@ static int ompi_comm_idup_with_info_activate (ompi_comm_request_t *request)
 
 static int ompi_comm_idup_with_info_finish (ompi_comm_request_t *request)
 {
-    /* done */
+    /* nothing to be done */
     return MPI_SUCCESS;
 }
 
@@ -1216,9 +1557,9 @@ int ompi_comm_create_group (ompi_communicator_t *comm, ompi_group_t *group, int
                           NULL,                                   /* remote_procs */
                           comm->c_keyhash,                        /* attrs */
                           comm->error_handler,                    /* error handler */
-                          true,                                   /* copy the topo */
                           group,                                  /* local group */
-                          NULL);                                  /* remote group */
+                          NULL,                                   /* remote group */
+                          OMPI_COMM_SET_FLAG_COPY_TOPOLOGY);      /* flags */
     if ( OMPI_SUCCESS != rc) {
         return rc;
     }
@@ -1231,8 +1572,8 @@ int ompi_comm_create_group (ompi_communicator_t *comm, ompi_group_t *group, int
     }
 
     /* Set name for debugging purposes */
-    snprintf(newcomp->c_name, MPI_MAX_OBJECT_NAME, "MPI COMMUNICATOR %d GROUP FROM %d",
-             newcomp->c_contextid, comm->c_contextid );
+    snprintf(newcomp->c_name, MPI_MAX_OBJECT_NAME, "MPI COMM %s GROUP FROM %s",
+	     ompi_comm_print_cid (newcomp), ompi_comm_print_cid (comm));
 
     /* activate communicator and init coll-module */
     rc = ompi_comm_activate (&newcomp, comm, NULL, &tag, NULL, false, mode);
@@ -1245,6 +1586,372 @@ int ompi_comm_create_group (ompi_communicator_t *comm, ompi_group_t *group, int
     return MPI_SUCCESS;
 }
 
+int ompi_comm_create_from_group (ompi_group_t *group, const char *tag, opal_info_t *info,
+                                 ompi_errhandler_t *errhandler, ompi_communicator_t **newcomm)
+{
+    ompi_communicator_t *newcomp = NULL;
+    int rc;
+
+    *newcomm = MPI_COMM_NULL;
+
+    rc = ompi_comm_set_simple (&newcomp, errhandler, group);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+        return rc;
+    }
+
+    /* Determine context id. It is identical to f_2_c_handle */
+    rc = ompi_comm_nextcid (newcomp, NULL, NULL, (void *) tag, NULL, false,
+                            OMPI_COMM_CID_GROUP_NEW);
+    if ( OMPI_SUCCESS != rc ) {
+        return rc;
+    }
+
+    /* Set name for debugging purposes */
+    snprintf(newcomp->c_name, MPI_MAX_OBJECT_NAME, "MPI COMM %s FROM GROUP",
+	     ompi_comm_print_cid (newcomp));
+
+    newcomp->super.s_info = OBJ_NEW(opal_info_t);
+    if (NULL == newcomp->super.s_info) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+    ompi_info_memkind_assert_type type;
+    ompi_info_memkind_copy_or_set (&group->grp_instance->super, &newcomp->super, info, &type);
+    if (OMPI_INFO_MEMKIND_ASSERT_NO_ACCEL == type) {
+        newcomp->c_assertions |= OMPI_COMM_ASSERT_NO_ACCEL_BUF;
+    }
+
+    /* activate communicator and init coll-module. use the group allreduce implementation as
+     * no collective module has yet been selected. the tag does not matter as any tag will
+     * be unique on the new communicator. */
+    rc = ompi_comm_activate (&newcomp, newcomp, NULL, &(int) {0xfeed}, NULL,
+                             false, OMPI_COMM_CID_GROUP);
+    if ( OMPI_SUCCESS != rc ) {
+        return rc;
+    }
+
+    newcomp->instance = group->grp_instance;
+
+    /*
+     * setup predefined keyvals - see MPI Standard for predefined keyvals cached on 
+     * communicators created via MPI_Comm_create_from_group or MPI_Intercomm_create_from_groups
+     */
+    ompi_attr_hash_init(&newcomp->c_keyhash);
+    ompi_attr_set_int(COMM_ATTR,
+                      newcomp,
+                      &newcomp->c_keyhash,
+                      MPI_TAG_UB, mca_pml.pml_max_tag,
+                      true);
+
+    *newcomm = newcomp;
+    return MPI_SUCCESS;
+}
+
+int ompi_intercomm_create (ompi_communicator_t *local_comm, int local_leader, ompi_communicator_t *bridge_comm,
+                           int remote_leader, int tag, ompi_communicator_t **newintercomm)
+{
+    int local_size = 0, local_rank = 0, lleader = 0, rleader = 0, rc, rsize = 0;
+    struct ompi_proc_t **rprocs;
+    ompi_communicator_t *newcomp;
+    ompi_group_t *new_group_pointer;
+
+    *newintercomm = MPI_COMM_NULL;
+
+    local_size = ompi_comm_size ( local_comm );
+    local_rank = ompi_comm_rank ( local_comm );
+    lleader = local_leader;
+    rleader = remote_leader;
+
+    if ( MPI_PARAM_CHECK ) {
+        if ( (0 > local_leader) || (local_leader >= local_size) ) {
+            return OMPI_ERR_BAD_PARAM;
+        }
+
+        /* remember that the remote_leader and bridge_comm arguments
+           just have to be valid at the local_leader */
+        if ( local_rank == local_leader ) {
+            if (ompi_comm_invalid (bridge_comm) || (bridge_comm->c_flags & OMPI_COMM_INTER)) {
+                return MPI_ERR_COMM;
+            }
+
+            if ((remote_leader < 0) || (remote_leader >= ompi_comm_size(bridge_comm))) {
+                return OMPI_ERR_BAD_PARAM;
+            }
+        } /* if ( local_rank == local_leader ) */
+    }
+
+    if (local_rank == local_leader) {
+        MPI_Request req;
+
+        /* local leader exchange group sizes lists */
+        rc = MCA_PML_CALL(irecv (&rsize, 1, MPI_INT, rleader, tag, bridge_comm, &req));
+        if ( rc != MPI_SUCCESS ) {
+            return rc;
+        }
+        rc = MCA_PML_CALL(send (&local_size, 1, MPI_INT, rleader, tag,
+                                MCA_PML_BASE_SEND_STANDARD, bridge_comm));
+        if ( rc != MPI_SUCCESS ) {
+            return rc;
+        }
+        rc = ompi_request_wait (&req, MPI_STATUS_IGNORE);
+        if ( rc != MPI_SUCCESS ) {
+            return rc;
+        }
+    }
+
+    /* bcast size and list of remote processes to all processes in local_comm */
+    rc = local_comm->c_coll->coll_bcast (&rsize, 1, MPI_INT, lleader, local_comm,
+                                         local_comm->c_coll->coll_bcast_module);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+        return rc;
+    }
+
+    rc = ompi_comm_get_rprocs (local_comm, bridge_comm, lleader, remote_leader, tag, rsize, &rprocs);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+        return rc;
+    }
+
+    /* put group elements in the list */
+    new_group_pointer = ompi_group_allocate_plist_w_procs (local_comm->c_local_group, rprocs, rsize);
+    if (OPAL_UNLIKELY(NULL == new_group_pointer)) {
+        free (rprocs);
+        return MPI_ERR_GROUP;
+    }
+
+    if (MPI_PARAM_CHECK) {
+        bool overlap = ompi_group_overlap (local_comm->c_local_group, new_group_pointer);
+        if (overlap && MPI_THREAD_MULTIPLE != ompi_mpi_thread_provided) {
+            ompi_group_free (&new_group_pointer);
+            return OMPI_ERR_BAD_PARAM;
+        }
+    }
+
+    rc = ompi_comm_set (&newcomp,                                     /* new comm */
+                        local_comm,                                   /* old comm */
+                        local_comm->c_local_group->grp_proc_count,    /* local_size */
+                        NULL,                                         /* local_procs*/
+                        rsize,                                        /* remote_size */
+                        NULL,                                         /* remote_procs */
+                        NULL,                                         /* attrs */
+                        local_comm->error_handler,                    /* error handler*/
+                        local_comm->c_local_group,                    /* local group */
+                        new_group_pointer,                            /* remote group */
+                        0);                                           /* flags */
+
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+        ompi_group_free (&new_group_pointer);
+        return rc;
+    }
+
+    /* Determine context id. It is identical to f_2_c_handle */
+    rc = ompi_comm_nextcid (newcomp, local_comm, bridge_comm, &lleader,
+                            &rleader, false, OMPI_COMM_CID_INTRA_BRIDGE);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+        ompi_comm_free (&newcomp);
+        return rc;
+    }
+
+    /* activate comm and init coll-module */
+    rc = ompi_comm_activate (&newcomp, local_comm, bridge_comm, &lleader, &rleader,
+                             false, OMPI_COMM_CID_INTRA_BRIDGE);
+    if ( MPI_SUCCESS != rc ) {
+        ompi_comm_free (&newcomp);
+        return rc;
+    }
+
+    // Copy info if there is one.
+    newcomp->super.s_info = OBJ_NEW(opal_info_t);
+    ompi_info_memkind_assert_type type;
+    ompi_info_memkind_copy_or_set (&local_comm->instance->super, &newcomp->super,
+                                   &ompi_mpi_info_null.info.super, &type);
+    if (OMPI_INFO_MEMKIND_ASSERT_NO_ACCEL == type) {
+        newcomp->c_assertions |= OMPI_COMM_ASSERT_NO_ACCEL_BUF;
+    }
+
+    *newintercomm = newcomp;
+
+    return OMPI_SUCCESS;
+}
+
+int ompi_intercomm_create_from_groups (ompi_group_t *local_group, int local_leader,
+                                       ompi_group_t *remote_group, int remote_leader, const char *tag,
+                                       opal_info_t *info, ompi_errhandler_t *errhandler,
+                                       ompi_communicator_t **newintercomm)
+{
+    ompi_communicator_t *newcomp = NULL, *local_comm, *leader_comm = MPI_COMM_NULL;
+    ompi_comm_extended_cid_block_t new_block = {0};
+    bool i_am_leader = local_leader == local_group->grp_my_rank;
+    ompi_proc_t **rprocs;
+    uint64_t data[4];
+    int leader_comm_remote_leader;
+    char *sub_tag = NULL;
+    size_t rsize;
+    int rc;
+
+    *newintercomm = MPI_COMM_NULL;
+
+    /* create a local communicator first. create a unique tag for this communicator */
+    opal_asprintf (&sub_tag, "%s-OMPIi-%s", tag, OPAL_NAME_PRINT(ompi_group_get_proc_name (local_group, local_leader)));
+    if (OPAL_UNLIKELY(NULL == sub_tag)) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    rc = ompi_comm_create_from_group (local_group, sub_tag, info, errhandler, &local_comm);
+    free (sub_tag);
+    sub_tag = NULL;
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+        return rc;
+    }
+
+    if (i_am_leader) {
+        /* create a bridge communicator for the leaders (so we can use the existing collectives
+         * for activation). there are probably more efficient ways to do this but for intercommunicator
+         * creation is not considered a performance critical operation. */
+        ompi_proc_t **leader_procs, *my_proc;
+        ompi_group_t *leader_group;
+
+        leader_procs = calloc (2, sizeof (*leader_procs));
+
+        my_proc = leader_procs[0] = ompi_group_get_proc_ptr (local_group, local_leader, true);
+        leader_procs[1] = ompi_group_get_proc_ptr (remote_group, remote_leader, true);
+
+        if (leader_procs[0] != leader_procs[1]) {
+            /* NTH: they are definitely different (can the ever be the same) */
+            if (leader_procs[0]->super.proc_name.jobid > leader_procs[1]->super.proc_name.jobid ||
+                (leader_procs[0]->super.proc_name.jobid == leader_procs[1]->super.proc_name.jobid &&
+                 leader_procs[0]->super.proc_name.vpid > leader_procs[1]->super.proc_name.vpid)) {
+                ompi_proc_t *tmp = leader_procs[0];
+                leader_procs[0] = leader_procs[1];
+                leader_procs[1] = tmp;
+            }
+
+            leader_group = ompi_group_allocate_plist_w_procs (NULL, leader_procs, 2);
+            ompi_set_group_rank (leader_group, my_proc);
+            if (OPAL_UNLIKELY(NULL == leader_group)) {
+                free(leader_procs);
+                ompi_comm_free (&local_comm);
+                return OMPI_ERR_OUT_OF_RESOURCE;
+            }
+            leader_group->grp_instance = local_group->grp_instance;
+
+            /* create a unique tag for allocating the leader communicator. we can eliminate this step
+             * if we take a CID from the newly allocated block belonging to local_comm. this is
+             * a note to make this change at a later time. */
+            opal_asprintf (&sub_tag, "%s-OMPIi-LC-%s", tag, OPAL_NAME_PRINT(ompi_group_get_proc_name (leader_group, 0)));
+            if (OPAL_UNLIKELY(NULL == sub_tag)) {
+                free(leader_procs);
+                ompi_comm_free (&local_comm);
+                OBJ_RELEASE(leader_group);
+                return OMPI_ERR_OUT_OF_RESOURCE;
+            }
+
+            /* remote leader is whichever rank I am not */
+            leader_comm_remote_leader = !(leader_group->grp_my_rank);
+
+            rc = ompi_comm_create_from_group (leader_group, sub_tag, info, errhandler, &leader_comm);
+            OBJ_RELEASE(leader_group);
+            free (sub_tag);
+            sub_tag = NULL;
+            if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+                free(leader_procs);
+                ompi_comm_free (&local_comm);
+                return rc;
+            }
+
+            /* grab a CID for the intercomm while we are at it */
+            ompi_comm_extended_cid_block_new (&leader_comm->c_contextidb, &new_block, false);
+
+            data[0] = remote_group->grp_proc_count;
+            /* store the relevant new_block data */
+            data[1] = new_block.block_cid.cid_base;
+            data[2] = new_block.block_cid.cid_sub.u64;
+            data[3] = new_block.block_level;
+        } else {
+            free (leader_procs);
+        }
+    }
+
+    /* bcast size and list of remote processes to all processes in local_comm */
+    rc = local_comm->c_coll->coll_bcast (data, 4, MPI_UINT64_T, local_leader, local_comm,
+                                         local_comm->c_coll->coll_bcast_module);
+    rsize = data[0];
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
+        ompi_comm_free (&local_comm);
+        return rc;
+    }
+
+    /* using 0 for the tag because we control both local_comm and leader_comm */
+    rc = ompi_comm_get_rprocs (local_comm, leader_comm, local_leader, leader_comm_remote_leader, 0, rsize, &rprocs);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+        ompi_comm_free (&local_comm);
+        return rc;
+    }
+
+    if (!i_am_leader) {
+        /* create a new group containing the remote processes for non-leader ranks */
+        remote_group = ompi_group_allocate_plist_w_procs (local_group, rprocs, rsize);
+        if (OPAL_UNLIKELY(NULL == remote_group)) {
+            free (rprocs);
+            ompi_comm_free (&local_comm);
+            return OMPI_ERR_OUT_OF_RESOURCE;
+        }
+    } else {
+        OBJ_RETAIN(remote_group);
+    }
+
+    rc = ompi_comm_set (&newcomp, local_comm, local_group->grp_proc_count, NULL, remote_group->grp_proc_count,
+                        NULL, NULL, errhandler, local_group, remote_group, OMPI_COMM_SET_FLAG_LOCAL_COMM_NODUP);
+    OBJ_RELEASE(remote_group);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+        ompi_comm_free (&local_comm);
+        return rc;
+    }
+
+    /*
+     * append the pmix CONTEXT_ID obtained when creating the leader comm as discriminator
+     */
+    opal_asprintf (&sub_tag, "%s-%ld", tag, data[1]);
+    if (OPAL_UNLIKELY(NULL == sub_tag)) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    rc = ompi_comm_nextcid (newcomp, NULL, NULL, (void *) sub_tag, NULL, false, OMPI_COMM_CID_GROUP_NEW);
+    free (sub_tag);
+    if ( OMPI_SUCCESS != rc ) {
+        OBJ_RELEASE(newcomp);
+        return rc;
+    }
+
+    /* Set name for debugging purposes */
+    snprintf(newcomp->c_name, MPI_MAX_OBJECT_NAME, "MPI INTERCOMM %s FROM GROUP", ompi_comm_print_cid (newcomp));
+
+    // Copy info if there is one.
+    newcomp->super.s_info = OBJ_NEW(opal_info_t);
+    if (info) {
+        opal_info_dup(info, &(newcomp->super.s_info));
+    }
+    ompi_info_memkind_assert_type type;
+    ompi_info_memkind_copy_or_set (&local_group->grp_instance->super, &newcomp->super, info, &type);
+    if (OMPI_INFO_MEMKIND_ASSERT_NO_ACCEL == type) {
+        newcomp->c_assertions |= OMPI_COMM_ASSERT_NO_ACCEL_BUF;
+    }
+
+    /* activate communicator and init coll-module */
+    rc = ompi_comm_activate (&newcomp, local_comm, leader_comm, &local_leader, &leader_comm_remote_leader,
+                             false, OMPI_COMM_CID_INTRA_BRIDGE);
+    if (MPI_COMM_NULL != leader_comm) {
+        ompi_comm_free (&leader_comm);
+    }
+
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+        ompi_comm_free (&newcomp);
+        return rc;
+    }
+
+    *newintercomm = newcomp;
+
+    return MPI_SUCCESS;
+}
+
 /**********************************************************************/
 /**********************************************************************/
 /**********************************************************************/
@@ -1255,10 +1962,14 @@ int ompi_comm_compare(ompi_communicator_t *comm1, ompi_communicator_t *comm2, in
     int lresult, rresult=MPI_CONGRUENT;
     int cmp_result;
 
+    if (comm1->instance != comm2->instance) {
+        return OMPI_ERR_BAD_PARAM;
+    }
+
     comp1 = (ompi_communicator_t *) comm1;
     comp2 = (ompi_communicator_t *) comm2;
 
-    if ( comp1->c_contextid == comp2->c_contextid ) {
+    if (ompi_comm_compare_cids(comp1,comp2)) {
         *result = MPI_IDENT;
         return MPI_SUCCESS;
     }
@@ -1452,7 +2163,7 @@ static int ompi_comm_allgather_emulate_intra( void *inbuf, int incount,
 int ompi_comm_free( ompi_communicator_t **comm )
 {
     int ret;
-    int cid = (*comm)->c_contextid;
+    int cid = (*comm)->c_index;
     int is_extra_retain = OMPI_COMM_IS_EXTRA_RETAIN(*comm);
 
     /* Release attributes.  We do this now instead of during the
@@ -1522,7 +2233,7 @@ int ompi_comm_free( ompi_communicator_t **comm )
          * makes sure that the pointer to the dependent communicator
          * still contains a valid object.
          */
-        ompi_communicator_t *tmpcomm = (ompi_communicator_t *) opal_pointer_array_get_item(&ompi_mpi_communicators, cid);
+        ompi_communicator_t *tmpcomm = ompi_comm_lookup(cid);
         if ( NULL != tmpcomm ){
             ompi_comm_free(&tmpcomm);
         }
@@ -1535,22 +2246,22 @@ int ompi_comm_free( ompi_communicator_t **comm )
 /**********************************************************************/
 /**********************************************************************/
 /**********************************************************************/
-int ompi_comm_get_rprocs ( ompi_communicator_t *local_comm,
-                           ompi_communicator_t *bridge_comm,
-                           int local_leader,
-                           int remote_leader,
-                           int tag,
-                           int rsize,
-                           ompi_proc_t ***prprocs )
+/**
+ * This is a short-hand routine used in intercomm_create.
+ * The routine makes sure, that all processes have afterwards
+ * a list of ompi_proc_t pointers for the remote group.
+ */
+int ompi_comm_get_rprocs (ompi_communicator_t *local_comm, ompi_communicator_t *bridge_comm,
+			  int local_leader, int remote_leader, int tag, int rsize, ompi_proc_t ***prprocs)
 {
     MPI_Request req;
     int rc = OMPI_SUCCESS;
     int local_rank, local_size;
     ompi_proc_t **rprocs=NULL;
-    int32_t size_len;
+    size_t size_len;
     int int_len=0, rlen;
     pmix_data_buffer_t *sbuf=NULL, *rbuf=NULL;
-    void *sendbuf=NULL;
+    char *sendbuf=NULL;
     char *recvbuf;
     ompi_proc_t **proc_list = NULL;
     int i;
@@ -1656,7 +2367,7 @@ int ompi_comm_get_rprocs ( ompi_communicator_t *local_comm,
         PMIX_DATA_BUFFER_RELEASE(sbuf);
     }
 
-    /* broadcast name list to all proceses in local_comm */
+    /* broadcast name list to all processes in local_comm */
     rc = local_comm->c_coll->coll_bcast( recvbuf, rlen, MPI_BYTE,
                                         local_leader, local_comm,
                                         local_comm->c_coll->coll_bcast_module);
@@ -1705,7 +2416,7 @@ int ompi_comm_get_rprocs ( ompi_communicator_t *local_comm,
        since it is used in the communicator */
     if ( OMPI_SUCCESS != rc ) {
         OMPI_ERROR_LOG(rc);
-        opal_output(0, "%d: Error in ompi_get_rprocs\n", local_rank);
+        opal_output(0, "%d: Error in ompi_comm_get_rprocs\n", local_rank);
         if ( NULL != rprocs ) {
             free ( rprocs );
             rprocs=NULL;
@@ -1731,37 +2442,14 @@ int ompi_comm_get_rprocs ( ompi_communicator_t *local_comm,
 /**********************************************************************/
 /**********************************************************************/
 /**********************************************************************/
-/**
- * This routine verifies, whether local_group and remote group are overlapping
- * in intercomm_create
- */
-int ompi_comm_overlapping_groups (int size, ompi_proc_t **lprocs,
-                                  int rsize, ompi_proc_t ** rprocs)
-
-{
-    int rc=OMPI_SUCCESS;
-    int i,j;
-
-    for (i=0; i<size; i++) {
-        for ( j=0; j<rsize; j++) {
-            if ( lprocs[i] == rprocs[j] ) {
-                rc = MPI_ERR_COMM;
-                return rc;
-            }
-        }
-    }
-
-    return rc;
-}
-/**********************************************************************/
-/**********************************************************************/
-/**********************************************************************/
 int ompi_comm_determine_first ( ompi_communicator_t *intercomm, int high )
 {
     int flag, rhigh;
     int rank, rsize;
     int *rcounts;
     int *rdisps;
+    ompi_count_array_t rcounts_desc;
+    ompi_disp_array_t rdisps_desc;
     int scount=0;
     int rc;
 
@@ -1789,8 +2477,10 @@ int ompi_comm_determine_first ( ompi_communicator_t *intercomm, int high )
         scount = 1;
     }
 
+    OMPI_COUNT_ARRAY_INIT(&rcounts_desc, rcounts);
+    OMPI_DISP_ARRAY_INIT(&rdisps_desc, rdisps);
     rc = intercomm->c_coll->coll_allgatherv(&high, scount, MPI_INT,
-                                           &rhigh, rcounts, rdisps,
+                                           &rhigh, rcounts_desc, rdisps_desc,
                                            MPI_INT, intercomm,
                                            intercomm->c_coll->coll_allgatherv_module);
     if ( NULL != rdisps ) {
@@ -1838,7 +2528,7 @@ int ompi_comm_determine_first_auto ( ompi_communicator_t* intercomm )
 /********************************************************************************/
 int ompi_comm_dump ( ompi_communicator_t *comm )
 {
-    opal_output(0, "Dumping information for comm_cid %d\n", comm->c_contextid);
+    opal_output(0, "Dumping information for comm_cid %s\n", ompi_comm_print_cid (comm));
     opal_output(0,"  f2c index:%d cube_dim: %d\n", comm->c_f_to_c_index,
                 comm->c_cube_dim);
     opal_output(0,"  Local group: size = %d my_rank = %d\n",
@@ -1973,6 +2663,10 @@ static int ompi_comm_fill_rest(ompi_communicator_t *comm,
                                int my_rank,
                                ompi_errhandler_t *errh)
 {
+    ompi_group_t *new_group;
+
+    new_group = ompi_group_allocate_plist_w_procs(comm->c_local_group, proc_pointers, num_procs);
+
     /* properly decrement the ref counts on the groups.
        We are doing this because this function is sort of a redo
        of what is done in comm.c. No need to decrement the ref
@@ -1988,7 +2682,7 @@ static int ompi_comm_fill_rest(ompi_communicator_t *comm,
     }
 
     /* allocate a group structure for the new communicator */
-    comm->c_local_group = ompi_group_allocate_plist_w_procs (proc_pointers, num_procs);
+    comm->c_local_group = new_group;
 
     /* set the remote group to be the same as local group */
     comm->c_remote_group = comm->c_local_group;
@@ -2012,8 +2706,8 @@ static int ompi_comm_fill_rest(ompi_communicator_t *comm,
     /* there is no cid at this stage ... make this right and make edgars
      * code call this function and remove dupli cde
      */
-    snprintf (comm->c_name, MPI_MAX_OBJECT_NAME, "MPI_COMMUNICATOR %d",
-              comm->c_contextid);
+    snprintf (comm->c_name, MPI_MAX_OBJECT_NAME, "MPI_COMMUNICATOR %s",
+	      ompi_comm_print_cid (comm));
 
     /* determine the cube dimensions */
     comm->c_cube_dim = opal_cube_dim(comm->c_local_group->grp_proc_count);
@@ -2032,3 +2726,28 @@ static int ompi_comm_copy_topo(ompi_communicator_t *oldcomm,
     newcomm->c_flags |= newcomm->c_topo->type;
     return OMPI_SUCCESS;
 }
+
+char *ompi_comm_print_cid (const ompi_communicator_t *comm)
+{
+#if OPAL_HAVE_THREAD_LOCAL
+    static opal_thread_local char cid_buffer[2][20];
+    static opal_thread_local int cid_buffer_index = 0;
+#else
+    /* no thread local == you get what you get. upgrade your compiler */
+    static char cid_buffer[2][20];
+    static int cid_buffer_index = 0;
+#endif
+    int bindex = cid_buffer_index;
+
+    if (mca_pml_base_supports_extended_cid () && !OMPI_COMM_IS_GLOBAL_INDEX(comm)) {
+	snprintf (cid_buffer[bindex], sizeof (cid_buffer[0]), "0x%" PRIx64 "%08" PRIx64,
+	    comm->c_contextid.cid_base,
+	    comm->c_contextid.cid_sub.u64);
+    } else {
+	snprintf (cid_buffer[bindex], sizeof (cid_buffer[0]), "%d", comm->c_index);
+    }
+
+    cid_buffer_index = cid_buffer_index ? 0 : 1;
+
+    return cid_buffer[bindex];
+}
diff --git a/ompi/communicator/comm_cid.c b/ompi/communicator/comm_cid.c
index 9015f26bbeb..8546b700401 100644
--- a/ompi/communicator/comm_cid.c
+++ b/ompi/communicator/comm_cid.c
@@ -18,12 +18,14 @@
  *                         reserved.
  * Copyright (c) 2012      Oak Ridge National Labs.  All rights reserved.
  * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
- * Copyright (c) 2014-2016 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2014-2023 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2016      IBM Corporation.  All rights reserved.
  * Copyright (c) 2017      Mellanox Technologies. All rights reserved.
  * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2020-2025 Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -31,10 +33,13 @@
  * $HEADER$
  */
 
+
 #include "ompi_config.h"
 
 #include "opal/mca/pmix/base/base.h"
+#include "opal/mca/pmix/pmix-internal.h"
 #include "opal/util/printf.h"
+#include "opal/util/show_help.h"
 
 #include "ompi/proc/proc.h"
 #include "ompi/communicator/communicator.h"
@@ -44,15 +49,25 @@
 #include "opal/class/opal_list.h"
 #include "ompi/mca/pml/pml.h"
 #include "ompi/runtime/ompi_rte.h"
+#include "ompi/mca/pml/base/base.h"
 #include "ompi/mca/coll/base/base.h"
 #include "ompi/request/request.h"
 #include "ompi/runtime/mpiruntime.h"
+#include "ompi/runtime/ompi_rte.h"
+
+#include "pmix.h"
+
+/* for use when we don't have a PMIx that supports CID generation */
+opal_atomic_int64_t ompi_comm_next_base_cid = 1;
+
+/* A macro comparing two CIDs */
+#define OMPI_COMM_CID_IS_LOWER(comm1,comm2) ( ((comm1)->c_index < (comm2)->c_index)? 1:0)
 
 struct ompi_comm_cid_context_t;
 
-typedef int (*ompi_comm_allreduce_impl_fn_t) (int *inbuf, int *outbuf, int count, struct ompi_op_t *op,
-                                              struct ompi_comm_cid_context_t *cid_context,
-                                              ompi_request_t **req);
+typedef int (*ompi_comm_iallreduce_impl_fn_t) (int *inbuf, int *outbuf, int count, struct ompi_op_t *op,
+                                               struct ompi_comm_cid_context_t *cid_context,
+                                               ompi_request_t **req);
 
 
 struct ompi_comm_cid_context_t {
@@ -63,7 +78,7 @@ struct ompi_comm_cid_context_t {
     ompi_communicator_t *comm;
     ompi_communicator_t *bridgecomm;
 
-    ompi_comm_allreduce_impl_fn_t allreduce_fn;
+    ompi_comm_iallreduce_impl_fn_t iallreduce_fn;
 
     int nextcid;
     int nextlocal_cid;
@@ -83,7 +98,8 @@ struct ompi_comm_cid_context_t {
     int remote_leader;
     int iter;
     /** storage for activate barrier */
-    int ok;
+    int local_peers;
+    int max_local_peers;
     char *port_string;
     bool send_first;
     int pml_tag;
@@ -210,17 +226,18 @@ static ompi_comm_cid_context_t *mca_comm_cid_context_alloc (ompi_communicator_t
      * for the current mode. */
     switch (mode) {
     case OMPI_COMM_CID_INTRA:
-        context->allreduce_fn = ompi_comm_allreduce_intra_nb;
+        context->iallreduce_fn = ompi_comm_allreduce_intra_nb;
         break;
     case OMPI_COMM_CID_INTER:
-        context->allreduce_fn = ompi_comm_allreduce_inter_nb;
+        context->iallreduce_fn = ompi_comm_allreduce_inter_nb;
         break;
     case OMPI_COMM_CID_GROUP:
-        context->allreduce_fn = ompi_comm_allreduce_group_nb;
+    case OMPI_COMM_CID_GROUP_NEW:
+        context->iallreduce_fn = ompi_comm_allreduce_group_nb;
         context->pml_tag = ((int *) arg0)[0];
         break;
     case OMPI_COMM_CID_INTRA_PMIX:
-        context->allreduce_fn = ompi_comm_allreduce_intra_pmix_nb;
+        context->iallreduce_fn = ompi_comm_allreduce_intra_pmix_nb;
         context->local_leader = ((int *) arg0)[0];
         if (arg1) {
             context->port_string = strdup ((char *) arg1);
@@ -228,19 +245,19 @@ static ompi_comm_cid_context_t *mca_comm_cid_context_alloc (ompi_communicator_t
         context->pmix_tag = strdup ((char *) pmix_tag);
         break;
     case OMPI_COMM_CID_INTRA_BRIDGE:
-        context->allreduce_fn = ompi_comm_allreduce_intra_bridge_nb;
+        context->iallreduce_fn = ompi_comm_allreduce_intra_bridge_nb;
         context->local_leader = ((int *) arg0)[0];
         context->remote_leader = ((int *) arg1)[0];
         break;
 #if OPAL_ENABLE_FT_MPI
     case OMPI_COMM_CID_INTRA_FT:
-        context->allreduce_fn = ompi_comm_ft_allreduce_intra_nb;
+        context->iallreduce_fn = ompi_comm_ft_allreduce_intra_nb;
         break;
     case OMPI_COMM_CID_INTER_FT:
-        context->allreduce_fn = ompi_comm_ft_allreduce_inter_nb;
+        context->iallreduce_fn = ompi_comm_ft_allreduce_inter_nb;
         break;
     case OMPI_COMM_CID_INTRA_PMIX_FT:
-        context->allreduce_fn = ompi_comm_ft_allreduce_intra_pmix_nb;
+        context->iallreduce_fn = ompi_comm_ft_allreduce_intra_pmix_nb;
         break;
 #endif /* OPAL_ENABLE_FT_MPI */
     default:
@@ -250,7 +267,8 @@ static ompi_comm_cid_context_t *mca_comm_cid_context_alloc (ompi_communicator_t
 
     context->send_first = send_first;
     context->iter = 0;
-    context->ok = 1;
+    context->local_peers = ompi_group_count_local_peers(newcomm->c_local_group);
+    context->max_local_peers = -1;
 
     return context;
 }
@@ -287,6 +305,261 @@ static volatile int64_t ompi_comm_cid_lowest_id = INT64_MAX;
 static int ompi_comm_cid_epoch = INT_MAX;
 #endif /* OPAL_ENABLE_FT_MPI */
 
+static int ompi_comm_ext_cid_new_block (ompi_communicator_t *newcomm, ompi_communicator_t *comm,
+                                        ompi_comm_extended_cid_block_t *new_block,
+                                        const void *arg0, const void *arg1, bool send_first, int mode,
+                                        ompi_request_t **req)
+{
+    pmix_info_t *pinfo, *results = NULL;
+    size_t nresults;
+    opal_process_name_t opal_proc_name;
+    bool cid_base_set = false;
+    char *tag = NULL;
+    size_t proc_count = 0, rproc_count = 0, tproc_count = 0, cid_base = 0UL, ninfo;
+    int rc, leader_rank;
+    pmix_proc_t *procs = NULL;
+    void *grpinfo = NULL, *list = NULL;
+    pmix_data_array_t darray;
+    pmix_info_t tinfo;
+
+    switch (mode) {
+    case OMPI_COMM_CID_GROUP_NEW:
+        tag = (char *) arg0;
+        break;
+    case OMPI_COMM_CID_GROUP:
+        ompi_group_translate_ranks (newcomm->c_local_group, 1, &(int){0},
+                                    comm->c_local_group, &leader_rank);
+
+        tag = ompi_comm_extended_cid_get_unique_tag (&comm->c_contextidb, *((int *) arg0), leader_rank);
+        break;
+    case OMPI_COMM_CID_INTRA:
+        tag = ompi_comm_extended_cid_get_unique_tag (&comm->c_contextidb, -1, 0);
+        break;
+    }
+
+    grpinfo = PMIx_Info_list_start();
+    if (NULL == grpinfo) {
+        rc = OMPI_ERR_OUT_OF_RESOURCE;
+        goto fn_exit;
+    }
+
+    rc = PMIx_Info_list_add(grpinfo, PMIX_GROUP_ASSIGN_CONTEXT_ID, NULL, PMIX_BOOL);
+    if (PMIX_SUCCESS != rc) {
+        OPAL_OUTPUT_VERBOSE((10, ompi_comm_output, "PMIx_Info_list_add failed %s %d", PMIx_Error_string(rc), __LINE__));
+        rc = OMPI_ERR_OUT_OF_RESOURCE;
+        goto fn_exit;
+    }
+
+    rc = PMIx_Info_list_add(grpinfo, PMIX_TIMEOUT, &ompi_pmix_connect_timeout, PMIX_UINT32);
+    if (PMIX_SUCCESS != rc) {
+        OPAL_OUTPUT_VERBOSE((10, ompi_comm_output, "PMIx_Info_list_add failed %s %d", PMIx_Error_string(rc), __LINE__));
+        rc = OMPI_ERR_OUT_OF_RESOURCE;
+        goto fn_exit;
+    }
+
+    list = PMIx_Info_list_start();
+
+    size_t c_index = (size_t)newcomm->c_index;
+    rc = PMIx_Info_list_add(list, PMIX_GROUP_LOCAL_CID, &c_index, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        OPAL_OUTPUT_VERBOSE((10, ompi_comm_output, "PMIx_Info_list_add failed %s %d", PMIx_Error_string(rc), __LINE__));
+        rc = OMPI_ERR_OUT_OF_RESOURCE;
+        goto fn_exit;
+    }
+
+    rc = PMIx_Info_list_convert(list, &darray);
+    if (PMIX_SUCCESS != rc) {
+        OPAL_OUTPUT_VERBOSE((10, ompi_comm_output, "PMIx_Info_list_convert failed %s %d", PMIx_Error_string(rc), __LINE__));
+        rc = OMPI_ERR_OUT_OF_RESOURCE;
+        goto fn_exit;
+    }
+    rc = PMIx_Info_list_add(grpinfo, PMIX_GROUP_INFO, &darray, PMIX_DATA_ARRAY);
+    PMIX_DATA_ARRAY_DESTRUCT(&darray);
+    if (PMIX_SUCCESS != rc) {
+        OPAL_OUTPUT_VERBOSE((10, ompi_comm_output, "PMIx_Info_list_add failed %s %d", PMIx_Error_string(rc), __LINE__));
+        rc = OMPI_ERR_OUT_OF_RESOURCE;
+        goto fn_exit;
+    }
+
+    rc = PMIx_Info_list_convert(grpinfo, &darray);
+    if (PMIX_SUCCESS != rc) {
+        OPAL_OUTPUT_VERBOSE((10, ompi_comm_output, "PMIx_Info_list_convert failed %s %d", PMIx_Error_string(rc), __LINE__));
+        rc = OMPI_ERR_OUT_OF_RESOURCE;
+        goto fn_exit;
+    }
+
+    pinfo = (pmix_info_t*)darray.array;
+    ninfo = darray.size;
+
+    proc_count = newcomm->c_local_group->grp_proc_count;
+    if ( OMPI_COMM_IS_INTER (newcomm) ){
+        rproc_count = newcomm->c_remote_group->grp_proc_count;
+    }
+
+    PMIX_PROC_CREATE(procs, proc_count + rproc_count);
+
+    for (size_t i = 0 ; i < proc_count; ++i) {
+        opal_proc_name = ompi_group_get_proc_name(newcomm->c_local_group, i);
+        OPAL_PMIX_CONVERT_NAME(&procs[i],&opal_proc_name);
+    }
+    for (size_t i = 0; i < rproc_count; ++i) {
+        opal_proc_name = ompi_group_get_proc_name(newcomm->c_remote_group, i);
+        OPAL_PMIX_CONVERT_NAME(&procs[proc_count+i],&opal_proc_name);
+    }
+
+    tproc_count = proc_count + rproc_count;
+
+    OPAL_OUTPUT_VERBOSE((10, ompi_comm_output, "calling PMIx_Group_construct - tag %s size %ld ninfo %ld cid_base %ld\n",
+                         tag, tproc_count, ninfo, cid_base));
+    rc = PMIx_Group_construct(tag, procs, tproc_count, pinfo, ninfo, &results, &nresults);
+    PMIX_DATA_ARRAY_DESTRUCT(&darray);
+    if(PMIX_SUCCESS != rc) {
+       char msg_string[1024];
+        switch (rc) {
+        case PMIX_ERR_UNREACH:
+            sprintf(msg_string,"PMIx server unreachable");
+            opal_show_help("help-comm.txt",
+                           "MPI function not supported",
+                           true,
+                           "MPI_Comm_create_from_group/MPI_Intercomm_create_from_groups",
+                           msg_string);
+
+            rc = MPI_ERR_UNSUPPORTED_OPERATION;
+            break;
+        case PMIX_ERR_NOT_SUPPORTED:
+            sprintf(msg_string,"PMIx server does not support PMIx Group operations");
+            opal_show_help("help-comm.txt",
+                           "MPI function not supported",
+                           true,
+                           "MPI_Comm_create_from_group/MPI_Intercomm_create_from_groups",
+                           msg_string);
+            rc = MPI_ERR_UNSUPPORTED_OPERATION;
+            break;
+        default:
+            rc = opal_pmix_convert_status(rc);
+            break;
+        } 
+        goto fn_exit;
+    }
+
+    for (size_t i=0; i<nresults; i++) {
+        if (PMIX_CHECK_KEY(&results[i], PMIX_GROUP_CONTEXT_ID)) {
+            PMIX_VALUE_GET_NUMBER(rc, &results[i].value, cid_base, size_t);
+            if(PMIX_SUCCESS != rc) {
+                rc = opal_pmix_convert_status(rc);
+                goto fn_exit;
+            }
+            cid_base_set = true;
+            break;
+        }
+    }
+
+    OPAL_OUTPUT_VERBOSE((10, ompi_comm_output, "PMIx_Group_construct - tag %s size %ld ninfo %ld cid_base %ld\n",
+                         tag, tproc_count, ninfo, cid_base));
+
+    /* destruct the group */
+    PMIX_INFO_CONSTRUCT(&tinfo);
+    PMIX_INFO_LOAD(&tinfo, PMIX_TIMEOUT, &ompi_pmix_connect_timeout, PMIX_UINT32);
+    rc = PMIx_Group_destruct (tag, &tinfo, 0);
+    PMIX_INFO_DESTRUCT(&tinfo);
+    if(PMIX_SUCCESS != rc) {
+        OPAL_OUTPUT_VERBOSE((10, ompi_comm_output, "PMIx_Group_destruct failed %s", PMIx_Error_string(rc)));
+        rc = opal_pmix_convert_status(rc);
+        goto fn_exit;
+    }
+
+    if (!cid_base_set) {
+        opal_show_help("help-comm.txt", "cid-base-not-set", true);
+        rc = OMPI_ERROR;
+        goto fn_exit;
+    }
+
+    ompi_comm_extended_cid_block_initialize (new_block, cid_base, 0, 0);
+
+fn_exit:
+    if (NULL != results) {
+        PMIX_INFO_FREE(results, nresults);
+        results = NULL;
+    }
+
+    if(NULL != procs) {
+        PMIX_PROC_FREE(procs, tproc_count);
+        procs = NULL;
+    }
+
+    if (NULL != grpinfo) {
+        PMIx_Info_list_release(grpinfo);
+    }
+
+    if (NULL != list) {
+        PMIx_Info_list_release(list);
+    }
+
+    return rc;
+}
+
+static int ompi_comm_nextcid_ext_nb (ompi_communicator_t *newcomm, ompi_communicator_t *comm,
+                                     ompi_communicator_t *bridgecomm, const void *arg0, const void *arg1,
+                                     bool send_first, int mode, ompi_request_t **req)
+{
+    ompi_comm_extended_cid_block_t *block;
+    bool is_new_block = false;
+    int rc;
+
+    /*
+     * sanity check and coverity pacifier
+     */
+    if (!(OMPI_COMM_CID_GROUP == mode || OMPI_COMM_CID_GROUP_NEW == mode) && (NULL == comm)) {
+        return OMPI_ERROR;
+    }
+
+    if (OMPI_COMM_CID_GROUP == mode || OMPI_COMM_CID_GROUP_NEW == mode) {
+        /* new block belongs to the new communicator */
+        block = &newcomm->c_contextidb;
+    } else {
+        block = &comm->c_contextidb;
+    }
+
+    for (unsigned int i = ompi_mpi_communicators.lowest_free ; i < mca_pml.pml_max_contextid ; ++i) {
+        bool flag = opal_pointer_array_test_and_set_item (&ompi_mpi_communicators, i, newcomm);
+        if (true == flag) {
+            newcomm->c_index = i;
+            break;
+        }
+    }
+    assert(newcomm->c_index > 2);
+
+    if (NULL == arg1) {
+        if (OMPI_COMM_CID_GROUP == mode || OMPI_COMM_CID_GROUP_NEW == mode ||
+            !ompi_comm_extended_cid_block_available (&comm->c_contextidb)) {
+            /* need a new block. it will be either assigned the the new communicator (MPI_Comm_create*_group)
+             * or the parent (which has no more CIDs in its block) */
+            rc = ompi_comm_ext_cid_new_block (newcomm, comm, block, arg0, arg1, send_first, mode, req);
+            if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+                return rc;
+            }
+
+            is_new_block = true;
+        }
+    } else {
+        /* got a block already */
+        *block = *((ompi_comm_extended_cid_block_t *) arg1);
+        is_new_block = true;
+    }
+
+    if (block != &newcomm->c_contextidb) {
+        (void) ompi_comm_extended_cid_block_new (block, &newcomm->c_contextidb, is_new_block);
+    }
+
+    newcomm->c_contextid = newcomm->c_contextidb.block_cid;
+
+    opal_hash_table_set_value_ptr (&ompi_comm_hash, &newcomm->c_contextid,
+                                   sizeof (newcomm->c_contextid), (void *) newcomm);
+    *req = &ompi_request_empty;
+    /* nothing more to do here */
+    return OMPI_SUCCESS;
+}
+
 int ompi_comm_nextcid_nb (ompi_communicator_t *newcomm, ompi_communicator_t *comm,
                           ompi_communicator_t *bridgecomm, const void *arg0, const void *arg1,
                           bool send_first, int mode, ompi_request_t **req)
@@ -294,6 +567,29 @@ int ompi_comm_nextcid_nb (ompi_communicator_t *newcomm, ompi_communicator_t *com
     ompi_comm_cid_context_t *context;
     ompi_comm_request_t *request;
 
+    if (mca_pml_base_supports_extended_cid() && NULL == comm) {
+        return ompi_comm_nextcid_ext_nb (newcomm, comm, bridgecomm, arg0, arg1, send_first, mode, req);
+    }
+
+    /* old CID algorighm */
+
+    /* if we got here and comm is NULL then that means the app is invoking MPI-4 Sessions or later
+       functions but the pml does not support these functions so return not supported */
+    if (NULL == comm) {
+       char msg_string[1024];
+       sprintf(msg_string,"The PML being used - %s - does not support MPI sessions related features",
+               mca_pml_base_selected_component.pmlm_version.mca_component_name);
+       opal_show_help("help-comm.txt",
+                      "MPI function not supported",
+                      true,
+                      "MPI_Comm_create_from_group/MPI_Intercomm_create_from_groups",
+                      msg_string);
+
+        return MPI_ERR_UNSUPPORTED_OPERATION;
+    }
+
+    newcomm->c_flags |= OMPI_COMM_GLOBAL_INDEX;
+
     context = mca_comm_cid_context_alloc (newcomm, comm, bridgecomm, arg0, arg1,
                                           "nextcid", send_first, mode);
     if (NULL == context) {
@@ -332,9 +628,11 @@ int ompi_comm_nextcid (ompi_communicator_t *newcomm, ompi_communicator_t *comm,
         return rc;
     }
 
-    ompi_request_wait_completion (req);
-    rc = req->req_status.MPI_ERROR;
-    ompi_comm_request_return ((ompi_comm_request_t *) req);
+    if (&ompi_request_empty != req) {
+        ompi_request_wait_completion (req);
+        rc = req->req_status.MPI_ERROR;
+        ompi_comm_request_return ((ompi_comm_request_t *) req);
+    }
 
     return rc;
 }
@@ -342,7 +640,7 @@ int ompi_comm_nextcid (ompi_communicator_t *newcomm, ompi_communicator_t *comm,
 static int ompi_comm_allreduce_getnextcid (ompi_comm_request_t *request)
 {
     ompi_comm_cid_context_t *context = (ompi_comm_cid_context_t *) request->context;
-    int64_t my_id = ((int64_t) ompi_comm_get_cid (context->comm) << 32 | context->pml_tag);
+    int64_t my_id = ((int64_t) ompi_comm_get_local_cid (context->comm) << 32 | context->pml_tag);
     ompi_request_t *subreq;
     bool flag = false;
     int ret = OMPI_SUCCESS;
@@ -367,7 +665,7 @@ static int ompi_comm_allreduce_getnextcid (ompi_comm_request_t *request)
         context->nextlocal_cid = mca_pml.pml_max_contextid;
         for (unsigned int i = context->start ; i < mca_pml.pml_max_contextid ; ++i) {
             flag = opal_pointer_array_test_and_set_item (&ompi_mpi_communicators, i,
-                                                         context->comm);
+                                                         (void *)OMPI_COMM_SENTINEL);
             if (true == flag) {
                 context->nextlocal_cid = i;
                 break;
@@ -387,8 +685,8 @@ static int ompi_comm_allreduce_getnextcid (ompi_comm_request_t *request)
 #endif /* OPAL_ENABLE_FT_MPI */
     }
 
-    ret = context->allreduce_fn (&context->nextlocal_cid, &context->nextcid, 1, MPI_MAX,
-                                 context, &subreq);
+    ret = context->iallreduce_fn (&context->nextlocal_cid, &context->nextcid, 1, MPI_MAX,
+                                  context, &subreq);
     /* there was a failure during non-blocking collective
      * all we can do is abort
      */
@@ -441,7 +739,8 @@ static int ompi_comm_checkcid (ompi_comm_request_t *request)
             opal_pointer_array_set_item(&ompi_mpi_communicators, context->nextlocal_cid, NULL);
 
             context->flag = opal_pointer_array_test_and_set_item (&ompi_mpi_communicators,
-                                                                  context->nextcid, context->comm);
+                                                                  context->nextcid, 
+                                                                  (void *)OMPI_COMM_SENTINEL);
         }
     }
 
@@ -453,7 +752,7 @@ static int ompi_comm_checkcid (ompi_comm_request_t *request)
 
     ++context->iter;
 
-    ret = context->allreduce_fn (&context->flag, &context->rflag, 1, MPI_MIN, context, &subreq);
+    ret = context->iallreduce_fn (&context->flag, &context->rflag, 1, MPI_MIN, context, &subreq);
     if (OMPI_SUCCESS == ret) {
         ompi_comm_request_schedule_append (request, ompi_comm_nextcid_check_flag, &subreq, 1);
     } else {
@@ -493,7 +792,7 @@ static int ompi_comm_nextcid_check_flag (ompi_comm_request_t *request)
             for (unsigned int i = context->start ; i < mca_pml.pml_max_contextid ; ++i) {
                 bool flag;
                 flag = opal_pointer_array_test_and_set_item (&ompi_mpi_communicators, i,
-                                                                context->comm);
+                                                             (void *)OMPI_COMM_SENTINEL);
                 if (true == flag) {
                     context->nextlocal_cid = i;
                     break;
@@ -503,11 +802,16 @@ static int ompi_comm_nextcid_check_flag (ompi_comm_request_t *request)
         }
 
         /* set the according values to the newcomm */
-        context->newcomm->c_contextid = context->nextcid;
 #if OPAL_ENABLE_FT_MPI
         context->newcomm->c_epoch = INT_MAX - context->rflag; /* reorder for simpler debugging */
         ompi_comm_cid_epoch -= 1; /* protected by the cid_lock */
 #endif /* OPAL_ENABLE_FT_MPI */
+        context->newcomm->c_index = context->nextcid;
+
+        /* to simplify coding always set the global CID even if it isn't used by the
+         * active PML */
+        context->newcomm->c_contextid.cid_base = 0;
+        context->newcomm->c_contextid.cid_sub.u64 = context->nextcid;
         opal_pointer_array_set_item (&ompi_mpi_communicators, context->nextcid, context->newcomm);
 
         /* unlock the cid generator */
@@ -553,6 +857,103 @@ static int ompi_comm_nextcid_check_flag (ompi_comm_request_t *request)
 /* Non-blocking version of ompi_comm_activate */
 static int ompi_comm_activate_nb_complete (ompi_comm_request_t *request);
 
+/* Callback function to set communicator disjointness flags */
+static inline void ompi_comm_set_disjointness_nb_complete(ompi_comm_cid_context_t *context)
+{
+    /* Only set the disjoint flags when it is intra-communicator */
+    if (OMPI_COMM_IS_INTER(*context->newcommp)) {
+        return;
+    }
+
+    if (OMPI_COMM_IS_DISJOINT_SET(*context->newcommp)) {
+        opal_show_help("help-comm.txt", "disjointness-set-again", true);
+        return;
+    }
+
+    if (1 == context->max_local_peers) {
+        (*context->newcommp)->c_flags |= OMPI_COMM_DISJOINT;
+    } else {
+        (*context->newcommp)->c_flags &= ~OMPI_COMM_DISJOINT;
+    }
+    (*context->newcommp)->c_flags |= OMPI_COMM_DISJOINT_SET;
+}
+
+static int ompi_comm_activate_complete (ompi_comm_cid_context_t *context)
+{
+    int ret;
+    ompi_communicator_t **newcomm = context->newcommp, *comm = context->comm;
+
+    /**
+     * Determine the new communicator's disjointness based on
+     * context->max_local_peers. It is reduced on the communicator
+     * before ompi_comm_activate_nb_complete is called.
+     */
+    ompi_comm_set_disjointness_nb_complete(context);
+
+    /**
+     * Check to see if this process is in the new communicator.
+     *
+     * Specifically, this function is invoked by all processes in the
+     * old communicator, regardless of whether they are in the new
+     * communicator or not.  This is because it is far simpler to use
+     * MPI collective functions on the old communicator to determine
+     * some data for the new communicator (e.g., remote_leader) than
+     * to kludge up our own pseudo-collective routines over just the
+     * processes in the new communicator.  Hence, *all* processes in
+     * the old communicator need to invoke this function.
+     *
+     * That being said, only processes in the new communicator need to
+     * select a coll module for the new communicator.  More
+     * specifically, processes who are not in the new communicator
+     * should *not* select a coll module -- for example,
+     * ompi_comm_rank(newcomm) returns MPI_UNDEFINED for processes who
+     * are not in the new communicator.  This can cause errors in the
+     * selection / initialization of a coll module.  Plus, it's
+     * wasteful -- processes in the new communicator will end up
+     * freeing the new communicator anyway, so we might as well leave
+     * the coll selection as NULL (the coll base comm unselect code
+     * handles that case properly).
+     */
+    if (MPI_UNDEFINED == (*newcomm)->c_local_group->grp_my_rank) {
+        return OMPI_SUCCESS;
+    }
+
+    /* Let the collectives components fight over who will do
+       collective on this new comm.  */
+    if (OMPI_SUCCESS != (ret = mca_coll_base_comm_select(*newcomm))) {
+        OBJ_RELEASE(*newcomm);
+        *newcomm = MPI_COMM_NULL;
+        return ret;
+    }
+
+    /* For an inter communicator, we have to deal with the potential
+     * problem of what is happening if the local_comm that we created
+     * has a lower CID than the parent comm. This is not a problem
+     * as long as the user calls MPI_Comm_free on the inter communicator.
+     * However, if the communicators are not freed by the user but released
+     * by Open MPI in MPI_Finalize, we walk through the list of still available
+     * communicators and free them one by one. Thus, local_comm is freed before
+     * the actual inter-communicator. However, the local_comm pointer in the
+     * inter communicator will still contain the 'previous' address of the local_comm
+     * and thus this will lead to a segmentation violation. In order to prevent
+     * that from happening, we increase the reference counter local_comm
+     * by one if its CID is lower than the parent. We cannot increase however
+     *  its reference counter if the CID of local_comm is larger than
+     * the CID of the inter communicators, since a regular MPI_Comm_free would
+     * leave in that the case the local_comm hanging around and thus we would not
+     * recycle CID's properly, which was the reason and the cause for this trouble.
+     */
+    if (OMPI_COMM_IS_INTER(*newcomm)) {
+        if (OMPI_COMM_CID_IS_LOWER(*newcomm, comm)) {
+            OMPI_COMM_SET_EXTRA_RETAIN (*newcomm);
+            OBJ_RETAIN (*newcomm);
+        }
+    }
+
+    /* done */
+    return OMPI_SUCCESS;
+}
+
 int ompi_comm_activate_nb (ompi_communicator_t **newcomm, ompi_communicator_t *comm,
                            ompi_communicator_t *bridgecomm, const void *arg0,
                            const void *arg1, bool send_first, int mode, ompi_request_t **req)
@@ -560,8 +961,11 @@ int ompi_comm_activate_nb (ompi_communicator_t **newcomm, ompi_communicator_t *c
     ompi_comm_cid_context_t *context;
     ompi_comm_request_t *request;
     ompi_request_t *subreq;
+    uint32_t comm_size;
     int ret = 0;
 
+    /* the caller should not pass NULL for comm (it may be the same as *newcomm) */
+    assert (NULL != comm);
     context = mca_comm_cid_context_alloc (*newcomm, comm, bridgecomm, arg0, arg1, "activate",
                                           send_first, mode);
     if (NULL == context) {
@@ -579,6 +983,25 @@ int ompi_comm_activate_nb (ompi_communicator_t **newcomm, ompi_communicator_t *c
 
     request->context = &context->super;
 
+    /* Prep communicator for handling remote cids if needed */
+
+    if (!OMPI_COMM_IS_GLOBAL_INDEX(*newcomm)) {
+        if (OMPI_COMM_IS_INTER(*newcomm)) {
+            comm_size = ompi_comm_remote_size(*newcomm);
+        } else {
+            comm_size = ompi_comm_size(*newcomm);
+        }
+
+        (*newcomm)->c_index_vec = (uint32_t *)calloc(comm_size, sizeof(uint32_t));
+        if (NULL == (*newcomm)->c_index_vec) {
+            return OMPI_ERR_OUT_OF_RESOURCE;
+        }
+
+        if (OMPI_COMM_IS_INTRA(*newcomm)) {
+            (*newcomm)->c_index_vec[(*newcomm)->c_my_rank] = (*newcomm)->c_index;
+        }
+    }
+
     if (MPI_UNDEFINED != (*newcomm)->c_local_group->grp_my_rank) {
         /* Initialize the PML stuff in the newcomm  */
         if ( OMPI_SUCCESS != (ret = MCA_PML_CALL(add_comm(*newcomm))) ) {
@@ -590,22 +1013,28 @@ int ompi_comm_activate_nb (ompi_communicator_t **newcomm, ompi_communicator_t *c
         OMPI_COMM_SET_PML_ADDED(*newcomm);
     }
 
-    /* Step 1: the barrier, after which it is allowed to
-     * send messages over the new communicator
-     */
-    ret = context->allreduce_fn (&context->ok, &context->ok, 1, MPI_MIN, context,
-                                 &subreq);
-    if (OMPI_SUCCESS != ret) {
-        ompi_comm_request_return (request);
-        return ret;
+    if (OMPI_COMM_IS_INTRA(*newcomm)) {
+        /**
+         * The communicator's disjointness is inferred from max_local_peers.
+         * Note: MPI_IN_PLACE cannot be used here because the parent could be an
+         * inter-communicator
+         */
+        ret = context->iallreduce_fn (&context->local_peers, &context->max_local_peers, 1, MPI_MAX, context,
+                                      &subreq);
+        if (OMPI_SUCCESS != ret) {
+            ompi_comm_request_return (request);
+            return ret;
+        }
+        ompi_comm_request_schedule_append (request, ompi_comm_activate_nb_complete, &subreq, 1);
+    } else {
+        ompi_comm_request_schedule_append (request, ompi_comm_activate_nb_complete, NULL, 0);
     }
 
-    ompi_comm_request_schedule_append (request, ompi_comm_activate_nb_complete, &subreq, 1);
     ompi_comm_request_start (request);
 
     *req = &request->super;
 
-    return OMPI_SUCCESS;
+    return ret;
 }
 
 int ompi_comm_activate (ompi_communicator_t **newcomm, ompi_communicator_t *comm,
@@ -620,80 +1049,74 @@ int ompi_comm_activate (ompi_communicator_t **newcomm, ompi_communicator_t *comm
         return rc;
     }
 
-    ompi_request_wait_completion (req);
-    rc = req->req_status.MPI_ERROR;
-    ompi_comm_request_return ((ompi_comm_request_t *) req);
+    if (&ompi_request_empty != req) {
+        ompi_request_wait_completion (req);
+        rc = req->req_status.MPI_ERROR;
+        ompi_comm_request_return ((ompi_comm_request_t *) req);
+    }
 
     return rc;
 }
 
-static int ompi_comm_activate_nb_complete (ompi_comm_request_t *request)
+int ompi_comm_get_remote_cid_from_pmix (ompi_communicator_t *comm, int dest, uint32_t *remote_cid)
 {
-    ompi_comm_cid_context_t *context = (ompi_comm_cid_context_t *) request->context;
-    int ret;
+    ompi_proc_t *ompi_proc;
+    pmix_proc_t  pmix_proc;
+    pmix_info_t tinfo[2];
+    pmix_value_t *val = NULL;
+    ompi_comm_extended_cid_t excid;
+    int rc = OMPI_SUCCESS;
+    size_t remote_cid64;
 
-    /**
-     * Check to see if this process is in the new communicator.
-     *
-     * Specifically, this function is invoked by all proceses in the
-     * old communicator, regardless of whether they are in the new
-     * communicator or not.  This is because it is far simpler to use
-     * MPI collective functions on the old communicator to determine
-     * some data for the new communicator (e.g., remote_leader) than
-     * to kludge up our own pseudo-collective routines over just the
-     * processes in the new communicator.  Hence, *all* processes in
-     * the old communicator need to invoke this function.
-     *
-     * That being said, only processes in the new communicator need to
-     * select a coll module for the new communicator.  More
-     * specifically, proceses who are not in the new communicator
-     * should *not* select a coll module -- for example,
-     * ompi_comm_rank(newcomm) returns MPI_UNDEFINED for processes who
-     * are not in the new communicator.  This can cause errors in the
-     * selection / initialization of a coll module.  Plus, it's
-     * wasteful -- processes in the new communicator will end up
-     * freeing the new communicator anyway, so we might as well leave
-     * the coll selection as NULL (the coll base comm unselect code
-     * handles that case properly).
-     */
-    if (MPI_UNDEFINED == (context->newcomm)->c_local_group->grp_my_rank) {
-        return OMPI_SUCCESS;
+    assert(NULL != remote_cid);
+
+    ompi_proc = ompi_comm_peer_lookup(comm, dest);
+    OPAL_PMIX_CONVERT_NAME(&pmix_proc, &ompi_proc->super.proc_name);
+
+    PMIx_Info_construct(&tinfo[0]);
+    PMIX_INFO_LOAD(&tinfo[0], PMIX_TIMEOUT, &ompi_pmix_connect_timeout, PMIX_UINT32);
+
+    excid = ompi_comm_get_extended_cid(comm);
+
+    PMIX_INFO_CONSTRUCT(&tinfo[1]);
+    PMIX_INFO_LOAD(&tinfo[1], PMIX_GROUP_CONTEXT_ID, &excid.cid_base, PMIX_SIZE);
+    PMIX_INFO_SET_QUALIFIER(&tinfo[1]);
+    if (PMIX_SUCCESS != (rc = PMIx_Get(&pmix_proc, PMIX_GROUP_LOCAL_CID, tinfo, 2, &val))) {
+        OPAL_OUTPUT_VERBOSE((10, ompi_comm_output, "PMIx_Get failed for PMIX_GROUP_LOCAL_CID cid_base %ld %s", excid.cid_base, PMIx_Error_string(rc)));
+        rc = OMPI_ERR_NOT_FOUND;
+        goto done;
     }
 
-    /* Let the collectives components fight over who will do
-       collective on this new comm.  */
-    if (OMPI_SUCCESS != (ret = mca_coll_base_comm_select(context->newcomm))) {
-        OBJ_RELEASE(context->newcomm);
-        *context->newcommp = MPI_COMM_NULL;
-        return ret;
+    if (NULL == val) {
+        OPAL_OUTPUT_VERBOSE((10, ompi_comm_output, "PMIx_Get failed for PMIX_GROUP_LOCAL_CID val returned NULL"));
+        rc = OMPI_ERR_NOT_FOUND;
+        goto done;
     }
 
-    /* For an inter communicator, we have to deal with the potential
-     * problem of what is happening if the local_comm that we created
-     * has a lower CID than the parent comm. This is not a problem
-     * as long as the user calls MPI_Comm_free on the inter communicator.
-     * However, if the communicators are not freed by the user but released
-     * by Open MPI in MPI_Finalize, we walk through the list of still available
-     * communicators and free them one by one. Thus, local_comm is freed before
-     * the actual inter-communicator. However, the local_comm pointer in the
-     * inter communicator will still contain the 'previous' address of the local_comm
-     * and thus this will lead to a segmentation violation. In order to prevent
-     * that from happening, we increase the reference counter local_comm
-     * by one if its CID is lower than the parent. We cannot increase however
-     *  its reference counter if the CID of local_comm is larger than
-     * the CID of the inter communicators, since a regular MPI_Comm_free would
-     * leave in that the case the local_comm hanging around and thus we would not
-     * recycle CID's properly, which was the reason and the cause for this trouble.
-     */
-    if (OMPI_COMM_IS_INTER(context->newcomm)) {
-        if (OMPI_COMM_CID_IS_LOWER(context->newcomm, context->comm)) {
-            OMPI_COMM_SET_EXTRA_RETAIN (context->newcomm);
-            OBJ_RETAIN (context->newcomm);
-        }
+    if (val->type != PMIX_SIZE) {
+        OPAL_OUTPUT_VERBOSE((10, ompi_comm_output, "PMIx_Get failed for PMIX_GROUP_LOCAL_CID type mismatch"));
+        rc = OMPI_ERR_TYPE_MISMATCH;
+        goto done;
     }
 
-    /* done */
-    return OMPI_SUCCESS;
+    PMIX_VALUE_GET_NUMBER(rc, val, remote_cid64, size_t);
+    rc = OMPI_SUCCESS;
+    *remote_cid = (uint32_t)remote_cid64;
+    comm->c_index_vec[dest] = (uint32_t)remote_cid64;
+    OPAL_OUTPUT_VERBOSE((10, ompi_comm_output, "PMIx_Get PMIX_GROUP_LOCAL_CID %d for cid_base %ld", *remote_cid, excid.cid_base));
+
+done:
+    if (NULL != val) {
+        PMIX_VALUE_RELEASE(val);
+    }
+
+    return rc;
+}
+
+static int ompi_comm_activate_nb_complete (ompi_comm_request_t *request)
+{
+    ompi_comm_cid_context_t *context = (ompi_comm_cid_context_t *) request->context;
+    return ompi_comm_activate_complete (context);
 }
 
 /**************************************************************************/
@@ -1305,15 +1728,17 @@ static int ompi_comm_ft_allreduce_intra_nb(int *inbuf, int *outbuf, int count,
 static int ompi_comm_ft_allreduce_inter_nb(int *inbuf, int *outbuf, int count,
                                            struct ompi_op_t *op, ompi_comm_cid_context_t *cid_context,
                                            ompi_request_t **req) {
-    return MPI_ERR_UNSUPPORTED_OPERATION;
+    //TODO: CID_INTER_FT needs an implementation, using the non-ft for now...
+    int rc = ompi_comm_allreduce_inter_nb(inbuf, outbuf, count, op, cid_context, req);
+    return (rc == MPI_ERR_REVOKED  || rc == MPI_ERR_PROC_FAILED) ? MPI_ERR_UNSUPPORTED_OPERATION: rc;
 }
 
 static int ompi_comm_ft_allreduce_intra_pmix_nb(int *inbuf, int *outbuf, int count,
                                                 struct ompi_op_t *op, ompi_comm_cid_context_t *cid_context,
                                                 ompi_request_t **req) {
     //TODO: CID_INTRA_PMIX_FT needs an implementation, using the non-ft for now...
-    return ompi_comm_allreduce_intra_pmix_nb(inbuf, outbuf, count, op, cid_context, req);
+    int rc = ompi_comm_allreduce_intra_pmix_nb(inbuf, outbuf, count, op, cid_context, req);
+    return (rc == MPI_ERR_REVOKED || rc == MPI_ERR_PROC_FAILED) ? MPI_ERR_UNSUPPORTED_OPERATION: rc;
 }
 
 #endif /* OPAL_ENABLE_FT_MPI */
-
diff --git a/ompi/communicator/comm_init.c b/ompi/communicator/comm_init.c
index a88ba29b6d3..86e260c536d 100644
--- a/ompi/communicator/comm_init.c
+++ b/ompi/communicator/comm_init.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
+ * Copyright (c) 2004-2023 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -23,6 +23,10 @@
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2015-2019 Intel, Inc.  All rights reserved.
  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2023-2024 Advanced Micro Devices, Inc. All rights reserved.
+ * Copyright (c) 2023      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -48,6 +52,8 @@
 #include "ompi/attribute/attribute.h"
 #include "ompi/dpm/dpm.h"
 #include "ompi/memchecker.h"
+#include "ompi/instance/instance.h"
+#include "ompi/info/info_memkind.h"
 
 /*
 ** Table for Fortran <-> C communicator handle conversion
@@ -57,12 +63,17 @@
 */
 opal_pointer_array_t ompi_mpi_communicators = {{0}};
 opal_pointer_array_t ompi_comm_f_to_c_table = {{0}};
+opal_hash_table_t ompi_comm_hash = {{0}};
 
 ompi_predefined_communicator_t  ompi_mpi_comm_world = {{{{0}}}};
 ompi_predefined_communicator_t  ompi_mpi_comm_self = {{{{0}}}};
 ompi_predefined_communicator_t  ompi_mpi_comm_null = {{{{0}}}};
 ompi_communicator_t  *ompi_mpi_comm_parent = NULL;
 
+int ompi_comm_output = -1;
+
+static bool ompi_comm_intrinsic_init;
+
 ompi_predefined_communicator_t *ompi_mpi_comm_world_addr =
     &ompi_mpi_comm_world;
 ompi_predefined_communicator_t *ompi_mpi_comm_self_addr =
@@ -78,17 +89,24 @@ OBJ_CLASS_INSTANCE(ompi_communicator_t, opal_infosubscriber_t,
                    ompi_comm_destruct);
 
 /* This is the counter for the number of communicators, which contain
-   process with more than one jobid. This counter is a usefull
+   process with more than one jobid. This counter is a useful
    shortcut for finalize and abort. */
 int ompi_comm_num_dyncomm=0;
 
+static int ompi_comm_finalize (void);
+
 /*
  * Initialize comm world/self/null/parent.
  */
 int ompi_comm_init(void)
 {
-    ompi_group_t *group;
-    size_t size;
+
+    /* create output stream */
+
+    if (ompi_comm_output == -1) {
+        ompi_comm_output = opal_output_open(NULL);
+        opal_output_set_verbosity(ompi_comm_output, ompi_comm_verbose_level);
+    }
 
     /* Setup communicator array */
     OBJ_CONSTRUCT(&ompi_mpi_communicators, opal_pointer_array_t);
@@ -97,58 +115,134 @@ int ompi_comm_init(void)
         return OMPI_ERROR;
     }
 
+    OBJ_CONSTRUCT(&ompi_comm_hash, opal_hash_table_t);
+    if (OPAL_SUCCESS != opal_hash_table_init (&ompi_comm_hash, 1024)) {
+        return OMPI_ERROR;
+    }
+
     /* Setup f to c table (we can no longer use the cid as the fortran handle) */
     OBJ_CONSTRUCT(&ompi_comm_f_to_c_table, opal_pointer_array_t);
-    if( OPAL_SUCCESS != opal_pointer_array_init(&ompi_comm_f_to_c_table, 8,
-                                                OMPI_FORTRAN_HANDLE_MAX, 32) ) {
+    if( OPAL_SUCCESS != opal_pointer_array_init (&ompi_comm_f_to_c_table, 8,
+                                                 OMPI_FORTRAN_HANDLE_MAX, 32) ) {
+        return OMPI_ERROR;
+    }
+
+    /*
+     * reserve indices in the F to C table for:
+     * MPI_COMM_WORLD
+     * MPI_COMM_SELF
+     * MPI_COMM_NULL
+     */
+
+    if (OPAL_SUCCESS != opal_pointer_array_set_item(&ompi_comm_f_to_c_table,
+                                                      0,
+                                                      (void *)-1L)) {
+        return OMPI_ERROR;
+    }
+
+    if (OPAL_SUCCESS != opal_pointer_array_set_item(&ompi_comm_f_to_c_table,
+                                                      1,
+                                                      (void *)-1L)) {
+        return OMPI_ERROR;
+    }
+
+    if (OPAL_SUCCESS != opal_pointer_array_set_item(&ompi_comm_f_to_c_table,
+                                                      2,
+                                                      (void *)-1L)) {
         return OMPI_ERROR;
     }
 
+    /* Setup MPI_COMM_NULL */
+    OBJ_CONSTRUCT(&ompi_mpi_comm_null, ompi_communicator_t);
+    assert(ompi_mpi_comm_null.comm.c_f_to_c_index == 2);
+    ompi_mpi_comm_null.comm.c_local_group  = &ompi_mpi_group_null.group;
+    ompi_mpi_comm_null.comm.c_remote_group = &ompi_mpi_group_null.group;
+    OBJ_RETAIN(&ompi_mpi_group_null.group);
+    OBJ_RETAIN(&ompi_mpi_group_null.group);
+
+    (void) ompi_comm_extended_cid_block_new (&ompi_mpi_comm_world.comm.c_contextidb,
+                                             &ompi_mpi_comm_null.comm.c_contextidb, false);
+    ompi_mpi_comm_null.comm.c_contextid    = ompi_mpi_comm_null.comm.c_contextidb.block_cid;
+    ompi_mpi_comm_null.comm.c_index        = 2;
+    ompi_mpi_comm_null.comm.c_my_rank      = MPI_PROC_NULL;
+
+    ompi_mpi_comm_null.comm.error_handler  = &ompi_mpi_errors_are_fatal.eh;
+    OBJ_RETAIN( &ompi_mpi_errors_are_fatal.eh );
+    opal_pointer_array_set_item (&ompi_mpi_communicators, 2, &ompi_mpi_comm_null);
+
+    /*
+     * first three places in ompi_mpi_communicators are reserved for
+     * MPI world model communicators MPI_COMM_WORLD, MPI_COMM_SELF, in addition to
+     * MPI_COMM_NULL which is needed both for world and sessions models.
+     * We conditionally insert ompi_mpi_comm_null into slots 0 and 1.  If MPI_Init was
+     * already called then these two slots are already occupied.  If
+     * MPI_Init is called after one or more MPI_Session_init, then
+     * slots 0 and 1 will be filled in with MPI_COMM_WORLD and MPI_COMM_SELF.
+     */
+    (void)opal_pointer_array_test_and_set_item(&ompi_mpi_communicators, 0, &ompi_mpi_comm_null);
+    (void)opal_pointer_array_test_and_set_item(&ompi_mpi_communicators, 1, &ompi_mpi_comm_null);
+
+    ompi_mpi_comm_null.comm.c_name = strdup ("MPI_COMM_NULL");
+    ompi_mpi_comm_null.comm.c_flags |= OMPI_COMM_NAMEISSET | OMPI_COMM_INTRINSIC |
+        OMPI_COMM_GLOBAL_INDEX;
+
+    /* Initialize the parent communicator to MPI_COMM_NULL */
+    ompi_mpi_comm_parent = &ompi_mpi_comm_null.comm;
+    OBJ_RETAIN(&ompi_mpi_comm_null);
+    OBJ_RETAIN(&ompi_mpi_group_null.group);
+
+    /* initialize communicator requests (for ompi_comm_idup) */
+    ompi_comm_request_init ();
+
+    /* get a reference on the attributes subsys */
+    ompi_attr_get_ref();
+
+    ompi_mpi_instance_append_finalize (ompi_comm_finalize);
+
+    return OMPI_SUCCESS;
+}
+
+int ompi_comm_init_mpi3 (void)
+{
+    int ret;
+    ompi_group_t *group = NULL;
+
+    /* the intrinsic communicators have been initialized */
+    ompi_comm_intrinsic_init = true;
+
     /* Setup MPI_COMM_WORLD */
     OBJ_CONSTRUCT(&ompi_mpi_comm_world, ompi_communicator_t);
     assert(ompi_mpi_comm_world.comm.c_f_to_c_index == 0);
-    group = OBJ_NEW(ompi_group_t);
-
-    size = ompi_process_info.num_procs;
-    group->grp_proc_pointers = (ompi_proc_t **) calloc (size, sizeof (ompi_proc_t *));
-    group->grp_proc_count = size;
-
-    for (size_t i = 0 ; i < size ; ++i) {
-        opal_process_name_t name = {.vpid = i, .jobid = OMPI_PROC_MY_NAME->jobid};
-        /* look for existing ompi_proc_t that matches this name */
-        group->grp_proc_pointers[i] = (ompi_proc_t *) ompi_proc_lookup (name);
-        if (NULL == group->grp_proc_pointers[i]) {
-            /* set sentinel value */
-            group->grp_proc_pointers[i] = (ompi_proc_t *) ompi_proc_name_to_sentinel (name);
-        } else {
-            OBJ_RETAIN (group->grp_proc_pointers[i]);
-        }
+
+    ret = ompi_group_from_pset (ompi_mpi_instance_default, "mpi://WORLD", &group);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
+        return ret;
     }
 
     OMPI_GROUP_SET_INTRINSIC (group);
-    OMPI_GROUP_SET_DENSE (group);
-    ompi_set_group_rank(group, ompi_proc_local());
-
-    ompi_mpi_comm_world.comm.c_contextid    = 0;
-    ompi_mpi_comm_world.comm.c_id_start_index = 4;
-    ompi_mpi_comm_world.comm.c_id_available = 4;
+    ompi_comm_extended_cid_block_initialize (&ompi_mpi_comm_world.comm.c_contextidb, 0, 0, 0);
+    ompi_mpi_comm_world.comm.c_contextid = ompi_mpi_comm_world.comm.c_contextidb.block_cid;
+    ompi_mpi_comm_world.comm.c_index          = 0;
     ompi_mpi_comm_world.comm.c_my_rank      = group->grp_my_rank;
     ompi_mpi_comm_world.comm.c_local_group  = group;
     ompi_mpi_comm_world.comm.c_remote_group = group;
     OBJ_RETAIN(ompi_mpi_comm_world.comm.c_remote_group);
-    ompi_mpi_comm_world.comm.c_cube_dim     = opal_cube_dim((int)size);
+    ompi_mpi_comm_world.comm.c_cube_dim     = opal_cube_dim ((int) group->grp_proc_count);
     ompi_mpi_comm_world.comm.error_handler  = ompi_initial_error_handler_eh;
     OBJ_RETAIN( ompi_mpi_comm_world.comm.error_handler );
     OMPI_COMM_SET_PML_ADDED(&ompi_mpi_comm_world.comm);
     opal_pointer_array_set_item (&ompi_mpi_communicators, 0, &ompi_mpi_comm_world);
 
-    opal_string_copy(ompi_mpi_comm_world.comm.c_name, "MPI_COMM_WORLD",
-                     sizeof(ompi_mpi_comm_world.comm.c_name));
-    ompi_mpi_comm_world.comm.c_flags |= OMPI_COMM_NAMEISSET;
-    ompi_mpi_comm_world.comm.c_flags |= OMPI_COMM_INTRINSIC;
+    ompi_mpi_comm_world.comm.c_name = strdup("MPI_COMM_WORLD");
+    ompi_mpi_comm_world.comm.c_flags |= OMPI_COMM_NAMEISSET | OMPI_COMM_INTRINSIC |
+        OMPI_COMM_GLOBAL_INDEX;
+    ompi_mpi_comm_world.comm.instance = group->grp_instance;
+
+    /* get a reference on the attributes subsys */
+    ompi_attr_get_ref();
 
     /* We have to create a hash (although it is legal to leave this
-       filed NULL -- the attribute accessor functions will intepret
+       filed NULL -- the attribute accessor functions will interpret
        this as "there are no attributes cached on this object")
        because MPI_COMM_WORLD has some predefined attributes. */
     ompi_attr_hash_init(&ompi_mpi_comm_world.comm.c_keyhash);
@@ -173,19 +267,22 @@ int ompi_comm_init(void)
             free(str);
         }
     }
+
     /* Setup MPI_COMM_SELF */
     OBJ_CONSTRUCT(&ompi_mpi_comm_self, ompi_communicator_t);
     assert(ompi_mpi_comm_self.comm.c_f_to_c_index == 1);
-    group = OBJ_NEW(ompi_group_t);
-    group->grp_proc_pointers = ompi_proc_self(&size);
-    group->grp_my_rank       = 0;
-    group->grp_proc_count    = (int)size;
+
+    ret = ompi_group_from_pset (ompi_mpi_instance_default, "mpi://SELF", &group);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
+        return ret;
+    }
+
     OMPI_GROUP_SET_INTRINSIC (group);
-    OMPI_GROUP_SET_DENSE (group);
 
-    ompi_mpi_comm_self.comm.c_contextid    = 1;
-    ompi_mpi_comm_self.comm.c_id_start_index = 20;
-    ompi_mpi_comm_self.comm.c_id_available = 20;
+    (void) ompi_comm_extended_cid_block_new (&ompi_mpi_comm_world.comm.c_contextidb,
+                                             &ompi_mpi_comm_self.comm.c_contextidb, false);
+    ompi_mpi_comm_self.comm.c_contextid = ompi_mpi_comm_self.comm.c_contextidb.block_cid;
+    ompi_mpi_comm_self.comm.c_index    = 1;
     ompi_mpi_comm_self.comm.c_my_rank      = group->grp_my_rank;
     ompi_mpi_comm_self.comm.c_local_group  = group;
     ompi_mpi_comm_self.comm.c_remote_group = group;
@@ -195,101 +292,63 @@ int ompi_comm_init(void)
     OMPI_COMM_SET_PML_ADDED(&ompi_mpi_comm_self.comm);
     opal_pointer_array_set_item (&ompi_mpi_communicators, 1, &ompi_mpi_comm_self);
 
-    opal_string_copy(ompi_mpi_comm_self.comm.c_name, "MPI_COMM_SELF",
-                     sizeof(ompi_mpi_comm_self.comm.c_name));
-    ompi_mpi_comm_self.comm.c_flags |= OMPI_COMM_NAMEISSET;
-    ompi_mpi_comm_self.comm.c_flags |= OMPI_COMM_INTRINSIC;
+    ompi_mpi_comm_self.comm.c_name = strdup("MPI_COMM_SELF");
+    ompi_mpi_comm_self.comm.c_flags |= OMPI_COMM_NAMEISSET | OMPI_COMM_INTRINSIC |
+        OMPI_COMM_GLOBAL_INDEX;
+    ompi_mpi_comm_self.comm.instance = group->grp_instance;
 
     /* We can set MPI_COMM_SELF's keyhash to NULL because it has no
        predefined attributes.  If a user defines an attribute on
        MPI_COMM_SELF, the keyhash will automatically be created. */
     ompi_mpi_comm_self.comm.c_keyhash = NULL;
 
-    /* Setup MPI_COMM_NULL */
-    OBJ_CONSTRUCT(&ompi_mpi_comm_null, ompi_communicator_t);
-    assert(ompi_mpi_comm_null.comm.c_f_to_c_index == 2);
-    ompi_mpi_comm_null.comm.c_local_group  = &ompi_mpi_group_null.group;
-    ompi_mpi_comm_null.comm.c_remote_group = &ompi_mpi_group_null.group;
-    OBJ_RETAIN(&ompi_mpi_group_null.group);
-    OBJ_RETAIN(&ompi_mpi_group_null.group);
-
-    ompi_mpi_comm_null.comm.c_contextid    = 2;
-    ompi_mpi_comm_null.comm.c_my_rank      = MPI_PROC_NULL;
-
-    /* unlike world, self, and parent, comm_null does not inherit the initial error
-     * handler */
-    ompi_mpi_comm_null.comm.error_handler  = &ompi_mpi_errors_are_fatal.eh;
-    OBJ_RETAIN( ompi_mpi_comm_null.comm.error_handler );
-    opal_pointer_array_set_item (&ompi_mpi_communicators, 2, &ompi_mpi_comm_null);
+    char *memkind_requested = getenv ("OMPI_MCA_mpi_memory_alloc_kinds");
+    if (NULL != memkind_requested) {
+        char *memkind_provided;
+        ompi_info_memkind_assert_type type;
+
+        ompi_info_memkind_process (memkind_requested, &memkind_provided, &type);
+        opal_infosubscribe_subscribe (&ompi_mpi_comm_world.comm.super, "mpi_memory_alloc_kinds", memkind_provided, ompi_info_memkind_cb);
+        opal_infosubscribe_subscribe (&ompi_mpi_comm_self.comm.super, "mpi_memory_alloc_kinds", memkind_provided, ompi_info_memkind_cb);
+        opal_infosubscribe_subscribe (&ompi_mpi_comm_world.comm.instance->super, "mpi_memory_alloc_kinds", memkind_provided, ompi_info_memkind_cb);
+        if (OMPI_INFO_MEMKIND_ASSERT_NO_ACCEL == type) {
+            ompi_mpi_comm_world.comm.c_assertions |= OMPI_COMM_ASSERT_NO_ACCEL_BUF;
+            ompi_mpi_comm_self.comm.c_assertions |= OMPI_COMM_ASSERT_NO_ACCEL_BUF;
+        }
+        free (memkind_provided);
+    }
 
-    opal_string_copy(ompi_mpi_comm_null.comm.c_name, "MPI_COMM_NULL",
-                     sizeof(ompi_mpi_comm_null.comm.c_name));
-    ompi_mpi_comm_null.comm.c_flags |= OMPI_COMM_NAMEISSET;
-    ompi_mpi_comm_null.comm.c_flags |= OMPI_COMM_INTRINSIC;
+    /*
+     * finally here we set the predefined attribute keyvals
+     */
+    ompi_attr_set_predefined_keyvals_for_wm();
 
-    /* Initialize the parent communicator to MPI_COMM_NULL */
-    ompi_mpi_comm_parent = &ompi_mpi_comm_null.comm;
-    OBJ_RETAIN(&ompi_mpi_comm_null);
-    OBJ_RETAIN(&ompi_mpi_group_null.group);
     OBJ_RETAIN(&ompi_mpi_errors_are_fatal.eh);
     /* During dyn_init, the comm_parent error handler will be set to the same
      * as comm_world (thus, the initial error handler). */
 
-    /* initialize communicator requests (for ompi_comm_idup) */
-    ompi_comm_request_init ();
-
     return OMPI_SUCCESS;
 }
 
-
-ompi_communicator_t *ompi_comm_allocate ( int local_size, int remote_size )
-{
-    ompi_communicator_t *new_comm;
-
-    /* create new communicator element */
-    new_comm = OBJ_NEW(ompi_communicator_t);
-    new_comm->super.s_info = NULL;
-    new_comm->c_local_group = ompi_group_allocate ( local_size );
-    if ( 0 < remote_size ) {
-        new_comm->c_remote_group = ompi_group_allocate (remote_size);
-        new_comm->c_flags |= OMPI_COMM_INTER;
-    } else {
-        /*
-         * simplifies some operations (e.g. p2p), if
-         * we can always use the remote group
-         */
-        new_comm->c_remote_group = new_comm->c_local_group;
-        OBJ_RETAIN(new_comm->c_remote_group);
-    }
-
-    /* fill in the inscribing hyper-cube dimensions */
-    new_comm->c_cube_dim = opal_cube_dim(local_size);
-
-    return new_comm;
-}
-
-int ompi_comm_finalize(void)
+static int ompi_comm_finalize (void)
 {
     int max, i;
     ompi_communicator_t *comm;
 
-    /* Shut down MPI_COMM_SELF */
-    OBJ_DESTRUCT( &ompi_mpi_comm_self );
-
     /* disconnect all dynamic communicators */
     ompi_dpm_dyn_finalize();
 
-    /* Free the attributes on comm world. This is not done in the
-     * destructor as we delete attributes in ompi_comm_free (which
-     * is not called for comm world) */
-    if (NULL != ompi_mpi_comm_world.comm.c_keyhash) {
-        /* Ignore errors when deleting attributes on comm_world */
-        (void) ompi_attr_delete_all(COMM_ATTR, &ompi_mpi_comm_world.comm, ompi_mpi_comm_world.comm.c_keyhash);
-        OBJ_RELEASE(ompi_mpi_comm_world.comm.c_keyhash);
-    }
+    if (ompi_comm_intrinsic_init) {
+        /* tear down MPI-3 predefined communicators (not initialized unless using MPI_Init) */
+        OBJ_DESTRUCT( &ompi_mpi_comm_self );
+        ompi_attr_delete_predefined_keyvals_for_wm();
+        /* Destroy the keyhash even is user defined attributes are still attached. */
+        OBJ_DESTRUCT(ompi_mpi_comm_world.comm.c_keyhash);
+        ompi_mpi_comm_world.comm.c_keyhash = NULL;
+        OBJ_DESTRUCT( &ompi_mpi_comm_world );
 
-    /* Shut down MPI_COMM_WORLD */
-    OBJ_DESTRUCT( &ompi_mpi_comm_world );
+        ompi_comm_intrinsic_init = false;
+    }
 
     /* Shut down the parent communicator, if it exists */
     if( ompi_mpi_comm_parent != &ompi_mpi_comm_null.comm ) {
@@ -324,13 +383,13 @@ int ompi_comm_finalize(void)
     OBJ_DESTRUCT( &ompi_mpi_comm_null );
 
     /* Check whether we have some communicators left */
-    max = opal_pointer_array_get_size(&ompi_mpi_communicators);
+    max = ompi_comm_get_num_communicators();
     for ( i=3; i<max; i++ ) {
-        comm = (ompi_communicator_t *)opal_pointer_array_get_item(&ompi_mpi_communicators, i);
+        comm = ompi_comm_lookup(i);
         if ( NULL != comm ) {
             /* Communicator has not been freed before finalize */
             OBJ_RELEASE(comm);
-            comm=(ompi_communicator_t *)opal_pointer_array_get_item(&ompi_mpi_communicators, i);
+            comm = ompi_comm_lookup(i);
             if ( NULL != comm ) {
                 /* Still here ? */
                 if ( !OMPI_COMM_IS_EXTRA_RETAIN(comm)) {
@@ -355,12 +414,19 @@ int ompi_comm_finalize(void)
     }
 
     OBJ_DESTRUCT (&ompi_mpi_communicators);
+    OBJ_DESTRUCT (&ompi_comm_hash);
     OBJ_DESTRUCT (&ompi_comm_f_to_c_table);
 
     /* finalize communicator requests */
     ompi_comm_request_fini ();
 
-    return OMPI_SUCCESS;
+    /* close output stream */
+
+    opal_output_close(ompi_comm_output);
+    ompi_comm_output = -1;
+
+    /* release a reference to the attributes subsys */
+    return ompi_attr_put_ref();
 }
 
 /********************************************************************************/
@@ -370,11 +436,9 @@ int ompi_comm_finalize(void)
 
 static void ompi_comm_construct(ompi_communicator_t* comm)
 {
-    comm->c_f_to_c_index = opal_pointer_array_add(&ompi_comm_f_to_c_table, comm);
-    comm->c_name[0]      = '\0';
-    comm->c_contextid    = MPI_UNDEFINED;
-    comm->c_id_available = MPI_UNDEFINED;
-    comm->c_id_start_index = MPI_UNDEFINED;
+    int idx;
+    comm->c_name         = NULL;
+    comm->c_index        = MPI_UNDEFINED;
     comm->c_flags        = 0;
     comm->c_my_rank      = 0;
     comm->c_cube_dim     = 0;
@@ -385,12 +449,28 @@ static void ompi_comm_construct(ompi_communicator_t* comm)
     comm->c_topo         = NULL;
     comm->c_coll         = NULL;
     comm->c_nbc_tag      = MCA_COLL_BASE_TAG_NONBLOCKING_BASE;
+    comm->instance       = NULL;
+    comm->c_index_vec    = NULL;
+
+    /*
+     * magic numerology - see TOPDIR/ompi/include/mpif-values.pl
+     */
+    idx = (comm == (ompi_communicator_t*)ompi_mpi_comm_world_addr) ? 0 :
+              (comm == (ompi_communicator_t*)ompi_mpi_comm_self_addr) ? 1 :
+                  (comm == (ompi_communicator_t*)ompi_mpi_comm_null_addr) ? 2 : -1;
+    if (-1 == idx) {
+        comm->c_f_to_c_index = opal_pointer_array_add(&ompi_comm_f_to_c_table,
+                                                      comm);
+    } else {
+        opal_pointer_array_set_item(&ompi_comm_f_to_c_table, idx, comm);
+        comm->c_f_to_c_index = idx;
+    }
 
     /* A keyhash will be created if/when an attribute is cached on
        this communicator */
     comm->c_keyhash      = NULL;
-
-    comm->errhandler_type  = OMPI_ERRHANDLER_TYPE_COMM;
+    comm->errhandler_type = OMPI_ERRHANDLER_TYPE_COMM;
+    comm->error_handler  = &ompi_mpi_errors_are_fatal.eh;
 #ifdef OMPI_WANT_PERUSE
     comm->c_peruse_handles = NULL;
 #endif
@@ -399,6 +479,7 @@ static void ompi_comm_construct(ompi_communicator_t* comm)
 #if OPAL_ENABLE_FT_MPI
     comm->any_source_enabled  = true;
     comm->any_source_offset   = 0;
+    comm->num_acked           = 0;
     comm->comm_revoked        = false;
     comm->coll_revoked        = false;
     comm->c_epoch             = 0;
@@ -466,6 +547,16 @@ static void ompi_comm_destruct(ompi_communicator_t* comm)
         comm->error_handler = NULL;
     }
 
+    if (NULL != comm->c_name) {
+        free (comm->c_name);
+        comm->c_name = NULL;
+    }
+
+    if (NULL != comm->c_index_vec) {
+        free (comm->c_index_vec);
+        comm->c_index_vec = NULL;
+    }
+
 #if OPAL_ENABLE_FT_MPI
     if( NULL != comm->agreement_specific ) {
         OBJ_RELEASE( comm->agreement_specific );
@@ -473,11 +564,15 @@ static void ompi_comm_destruct(ompi_communicator_t* comm)
 #endif  /* OPAL_ENABLE_FT_MPI */
 
     /* mark this cid as available */
-    if ( MPI_UNDEFINED != (int)comm->c_contextid &&
+    if ( MPI_UNDEFINED != (int)comm->c_index &&
          NULL != opal_pointer_array_get_item(&ompi_mpi_communicators,
-                                             comm->c_contextid)) {
+                                             comm->c_index)) {
         opal_pointer_array_set_item ( &ompi_mpi_communicators,
-                                      comm->c_contextid, NULL);
+                                      comm->c_index, NULL);
+        if (!OMPI_COMM_IS_GLOBAL_INDEX(comm)) {
+            opal_hash_table_remove_value_ptr (&ompi_comm_hash, &comm->c_contextid,
+                                              sizeof (comm->c_contextid));
+        }
     }
 
     /* reset the ompi_comm_f_to_c_table entry */
@@ -492,7 +587,7 @@ static void ompi_comm_destruct(ompi_communicator_t* comm)
 }
 
 #define OMPI_COMM_SET_INFO_FN(name, flag)       \
-    static char *ompi_comm_set_ ## name (opal_infosubscriber_t *obj, char *key, char *value) \
+    static const char *ompi_comm_set_ ## name (opal_infosubscriber_t *obj, const char *key, const char *value) \
     {                                                                   \
         ompi_communicator_t *comm = (ompi_communicator_t *) obj;        \
                                                                         \
@@ -509,6 +604,8 @@ OMPI_COMM_SET_INFO_FN(no_any_source, OMPI_COMM_ASSERT_NO_ANY_SOURCE)
 OMPI_COMM_SET_INFO_FN(no_any_tag, OMPI_COMM_ASSERT_NO_ANY_TAG)
 OMPI_COMM_SET_INFO_FN(allow_overtake, OMPI_COMM_ASSERT_ALLOW_OVERTAKE)
 OMPI_COMM_SET_INFO_FN(exact_length, OMPI_COMM_ASSERT_EXACT_LENGTH)
+OMPI_COMM_SET_INFO_FN(lazy_barrier, OMPI_COMM_ASSERT_LAZY_BARRIER)
+OMPI_COMM_SET_INFO_FN(active_poll, OMPI_COMM_ASSERT_ACTIVE_POLL)
 
 void ompi_comm_assert_subscribe (ompi_communicator_t *comm, int32_t assert_flag)
 {
@@ -525,5 +622,11 @@ void ompi_comm_assert_subscribe (ompi_communicator_t *comm, int32_t assert_flag)
     case OMPI_COMM_ASSERT_EXACT_LENGTH:
         opal_infosubscribe_subscribe (&comm->super, "mpi_assert_exact_length", "false", ompi_comm_set_exact_length);
         break;
+    case OMPI_COMM_ASSERT_LAZY_BARRIER:
+        opal_infosubscribe_subscribe (&comm->super, "ompi_assert_lazy_barrier", "false", ompi_comm_set_lazy_barrier);
+        break;
+    case OMPI_COMM_ASSERT_ACTIVE_POLL:
+        opal_infosubscribe_subscribe (&comm->super, "ompi_assert_active_poll", "true", ompi_comm_set_active_poll);
+        break;
     }
 }
diff --git a/ompi/communicator/comm_request.c b/ompi/communicator/comm_request.c
index e1092deb400..89ff7fff7af 100644
--- a/ompi/communicator/comm_request.c
+++ b/ompi/communicator/comm_request.c
@@ -1,13 +1,15 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2013-2018 Los Alamos National Security, LLC.  All rights
- *                         reseved.
+ *                         reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2004-2016 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2016      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2021      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -30,6 +32,7 @@ typedef struct ompi_comm_request_item_t {
     opal_list_item_t super;
     ompi_comm_request_callback_fn_t callback;
     ompi_request_t *subreqs[OMPI_COMM_REQUEST_MAX_SUBREQ];
+    uint32_t flags;
     int subreq_count;
 } ompi_comm_request_item_t;
 OBJ_CLASS_DECLARATION(ompi_comm_request_item_t);
@@ -70,6 +73,12 @@ void ompi_comm_request_fini (void)
 
 int ompi_comm_request_schedule_append (ompi_comm_request_t *request, ompi_comm_request_callback_fn_t callback,
                             ompi_request_t *subreqs[], int subreq_count)
+{
+    return ompi_comm_request_schedule_append_w_flags(request, callback, subreqs, subreq_count, 0);
+}
+
+int ompi_comm_request_schedule_append_w_flags(ompi_comm_request_t *request, ompi_comm_request_callback_fn_t callback,
+                            ompi_request_t *subreqs[], int subreq_count, uint32_t flags)
 {
     ompi_comm_request_item_t *request_item;
     int i;
@@ -84,6 +93,7 @@ int ompi_comm_request_schedule_append (ompi_comm_request_t *request, ompi_comm_r
     }
 
     request_item->callback = callback;
+    request_item->flags = flags;
 
     for (i = 0 ; i < subreq_count ; ++i) {
         request_item->subreqs[i] = subreqs[i];
@@ -125,7 +135,9 @@ static int ompi_comm_request_progress (void)
                          * that it does some subreqs cleanup */
                         request->super.req_status.MPI_ERROR = subreq->req_status.MPI_ERROR;
                     }
-                    ompi_request_free (&subreq);
+                    if (!(request_item->flags & OMPI_COMM_REQ_FLAG_RETAIN_SUBREQ)) {
+                        ompi_request_free (&subreq);
+                    }
                     request_item->subreq_count--;
                     completed++;
                 } else {
@@ -269,6 +281,10 @@ ompi_comm_request_t *ompi_comm_request_get (void)
 
 void ompi_comm_request_return (ompi_comm_request_t *request)
 {
+    if ((void *) &ompi_request_empty == (void *) request) {
+        return;
+    }
+
     if (request->context) {
         OBJ_RELEASE (request->context);
         request->context = NULL;
diff --git a/ompi/communicator/comm_request.h b/ompi/communicator/comm_request.h
index 43082d60935..5f85aa6ed4f 100644
--- a/ompi/communicator/comm_request.h
+++ b/ompi/communicator/comm_request.h
@@ -1,7 +1,9 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2013-2016 Los Alamos National Security, LLC.  All rights
- *                         reseved.
+ *                         reserved.
+ * Copyright (c) 2021      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -9,7 +11,7 @@
  * $HEADER$
  */
 
-#if !defined(OMPI_COMM_REQ_H)
+#ifndef OMPI_COMM_REQ_H
 #define OMPI_COMM_REQ_H
 
 #include "opal/class/opal_list.h"
@@ -18,6 +20,9 @@
 /* increase this number if more subrequests are needed */
 #define OMPI_COMM_REQUEST_MAX_SUBREQ 2
 
+/* indicate that the caller will free subrequests */
+#define OMPI_COMM_REQ_FLAG_RETAIN_SUBREQ  0x00000001
+
 typedef struct ompi_comm_request_t {
     ompi_request_t super;
 
@@ -32,6 +37,8 @@ void ompi_comm_request_init (void);
 void ompi_comm_request_fini (void);
 int ompi_comm_request_schedule_append (ompi_comm_request_t *request, ompi_comm_request_callback_fn_t callback,
                             ompi_request_t *subreqs[], int subreq_count);
+int ompi_comm_request_schedule_append_w_flags(ompi_comm_request_t *request, ompi_comm_request_callback_fn_t callback,
+                            ompi_request_t *subreqs[], int subreq_count, uint32_t flags);
 void ompi_comm_request_start (ompi_comm_request_t *request);
 ompi_comm_request_t *ompi_comm_request_get (void);
 void ompi_comm_request_return (ompi_comm_request_t *request);
diff --git a/ompi/communicator/communicator.h b/ompi/communicator/communicator.h
index d1e82a7dcfd..4a1b42ab0c4 100644
--- a/ompi/communicator/communicator.h
+++ b/ompi/communicator/communicator.h
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
+ * Copyright (c) 2004-2023 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -16,12 +16,16 @@
  * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
  * Copyright (c) 2011-2013 Inria.  All rights reserved.
  * Copyright (c) 2011-2013 Universite Bordeaux 1
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
+ * Copyright (c) 2012-2018 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2014-2015 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2023      Advanced Micro Devices, Inc. All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -47,6 +51,8 @@
 #include "ompi/info/info.h"
 #include "ompi/proc/proc.h"
 
+#include "opal/util/printf.h"
+
 BEGIN_C_DECLS
 
 OMPI_DECLSPEC OBJ_CLASS_DECLARATION(ompi_communicator_t);
@@ -57,12 +63,15 @@ OMPI_DECLSPEC OBJ_CLASS_DECLARATION(ompi_communicator_t);
 #define OMPI_COMM_DYNAMIC      0x00000008
 #define OMPI_COMM_ISFREED      0x00000010
 #define OMPI_COMM_INVALID      0x00000020
+#define OMPI_COMM_DISJOINT_SET 0x00000040
+#define OMPI_COMM_DISJOINT     0x00000080
 #define OMPI_COMM_CART         0x00000100
 #define OMPI_COMM_GRAPH        0x00000200
 #define OMPI_COMM_DIST_GRAPH   0x00000400
 #define OMPI_COMM_PML_ADDED    0x00001000
 #define OMPI_COMM_EXTRA_RETAIN 0x00004000
 #define OMPI_COMM_MAPBY_NODE   0x00008000
+#define OMPI_COMM_GLOBAL_INDEX 0x00010000
 
 /* some utility #defines */
 #define OMPI_COMM_IS_INTER(comm) ((comm)->c_flags & OMPI_COMM_INTER)
@@ -74,12 +83,15 @@ OMPI_DECLSPEC OBJ_CLASS_DECLARATION(ompi_communicator_t);
 #define OMPI_COMM_IS_FREED(comm) ((comm)->c_flags & OMPI_COMM_ISFREED)
 #define OMPI_COMM_IS_DYNAMIC(comm) ((comm)->c_flags & OMPI_COMM_DYNAMIC)
 #define OMPI_COMM_IS_INVALID(comm) ((comm)->c_flags & OMPI_COMM_INVALID)
+#define OMPI_COMM_IS_DISJOINT_SET(comm) ((comm)->c_flags & OMPI_COMM_DISJOINT_SET)
+#define OMPI_COMM_IS_DISJOINT(comm) ((comm)->c_flags & OMPI_COMM_DISJOINT)
 #define OMPI_COMM_IS_PML_ADDED(comm) ((comm)->c_flags & OMPI_COMM_PML_ADDED)
 #define OMPI_COMM_IS_EXTRA_RETAIN(comm) ((comm)->c_flags & OMPI_COMM_EXTRA_RETAIN)
 #define OMPI_COMM_IS_TOPO(comm) (OMPI_COMM_IS_CART((comm)) || \
                                  OMPI_COMM_IS_GRAPH((comm)) || \
                                  OMPI_COMM_IS_DIST_GRAPH((comm)))
 #define OMPI_COMM_IS_MAPBY_NODE(comm) ((comm)->c_flags & OMPI_COMM_MAPBY_NODE)
+#define OMPI_COMM_IS_GLOBAL_INDEX(comm) ((comm)->c_flags & OMPI_COMM_GLOBAL_INDEX)
 
 #define OMPI_COMM_SET_DYNAMIC(comm) ((comm)->c_flags |= OMPI_COMM_DYNAMIC)
 #define OMPI_COMM_SET_INVALID(comm) ((comm)->c_flags |= OMPI_COMM_INVALID)
@@ -92,12 +104,18 @@ OMPI_DECLSPEC OBJ_CLASS_DECLARATION(ompi_communicator_t);
 #define OMPI_COMM_ASSERT_NO_ANY_SOURCE  0x00000002
 #define OMPI_COMM_ASSERT_EXACT_LENGTH   0x00000004
 #define OMPI_COMM_ASSERT_ALLOW_OVERTAKE 0x00000008
+#define OMPI_COMM_ASSERT_LAZY_BARRIER   0x00000010
+#define OMPI_COMM_ASSERT_ACTIVE_POLL    0x00000020
+#define OMPI_COMM_ASSERT_NO_ACCEL_BUF   0x00000040
 
 #define OMPI_COMM_CHECK_ASSERT(comm, flag) !!((comm)->c_assertions & flag)
 #define OMPI_COMM_CHECK_ASSERT_NO_ANY_TAG(comm)     OMPI_COMM_CHECK_ASSERT(comm, OMPI_COMM_ASSERT_NO_ANY_TAG)
 #define OMPI_COMM_CHECK_ASSERT_NO_ANY_SOURCE(comm)  OMPI_COMM_CHECK_ASSERT(comm, OMPI_COMM_ASSERT_NO_ANY_SOURCE)
 #define OMPI_COMM_CHECK_ASSERT_EXACT_LENGTH(comm)   OMPI_COMM_CHECK_ASSERT(comm, OMPI_COMM_ASSERT_EXACT_LENGTH)
 #define OMPI_COMM_CHECK_ASSERT_ALLOW_OVERTAKE(comm) OMPI_COMM_CHECK_ASSERT(comm, OMPI_COMM_ASSERT_ALLOW_OVERTAKE)
+#define OMPI_COMM_CHECK_ASSERT_LAZY_BARRIER(comm) OMPI_COMM_CHECK_ASSERT(comm, OMPI_COMM_ASSERT_LAZY_BARRIER)
+#define OMPI_COMM_CHECK_ASSERT_ACTIVE_POLL(comm) OMPI_COMM_CHECK_ASSERT(comm, OMPI_COMM_ASSERT_ACTIVE_POLL)
+#define OMPI_COMM_CHECK_ASSERT_NO_ACCEL_BUF(comm) OMPI_COMM_CHECK_ASSERT(comm, OMPI_COMM_ASSERT_NO_ACCEL_BUF)
 
 /**
  * Modes required for acquiring the new comm-id.
@@ -111,10 +129,11 @@ OMPI_DECLSPEC OBJ_CLASS_DECLARATION(ompi_communicator_t);
 #define OMPI_COMM_CID_INTRA_BRIDGE 0x00000080
 #define OMPI_COMM_CID_INTRA_PMIX   0x00000100
 #define OMPI_COMM_CID_GROUP        0x00000200
+#define OMPI_COMM_CID_GROUP_NEW    0x00000400
 #if OPAL_ENABLE_FT_MPI
-#define OMPI_COMM_CID_INTRA_FT        0x00000400
-#define OMPI_COMM_CID_INTER_FT        0x00000800
-#define OMPI_COMM_CID_INTRA_PMIX_FT   0x00001000
+#define OMPI_COMM_CID_INTRA_FT        0x00000800
+#define OMPI_COMM_CID_INTER_FT        0x00001000
+#define OMPI_COMM_CID_INTRA_PMIX_FT   0x00002000
 #endif /* OPAL_ENABLE_FT_MPI */
 
 /**
@@ -124,11 +143,111 @@ OMPI_DECLSPEC OBJ_CLASS_DECLARATION(ompi_communicator_t);
 #define OMPI_COMM_BLOCK_WORLD      16
 #define OMPI_COMM_BLOCK_OTHERS     8
 
+/**
+ * Placeholder to use in array of ompi communicators during CID allocation 
+ */
+#define OMPI_COMM_SENTINEL        0x00000001
+
 /* A macro comparing two CIDs */
-#define OMPI_COMM_CID_IS_LOWER(comm1,comm2) ( ((comm1)->c_contextid < (comm2)->c_contextid)? 1:0)
+#define OMPI_COMM_CID_IS_LOWER(comm1,comm2) ( ((comm1)->c_index < (comm2)->c_index)? 1:0)
 
+OMPI_DECLSPEC extern opal_hash_table_t ompi_comm_hash;
 OMPI_DECLSPEC extern opal_pointer_array_t ompi_mpi_communicators;
 OMPI_DECLSPEC extern opal_pointer_array_t ompi_comm_f_to_c_table;
+
+OMPI_DECLSPEC extern int ompi_comm_output;
+
+struct ompi_comm_extended_cid_t {
+    uint64_t  cid_base;
+    union {
+        uint64_t u64;
+	uint8_t  u8[8];
+    } cid_sub;
+};
+typedef struct ompi_comm_extended_cid_t ompi_comm_extended_cid_t;
+
+struct ompi_comm_extended_cid_block_t {
+    ompi_comm_extended_cid_t block_cid;
+    /** can be used to get a unique string tag for pmix context creation */
+    uint64_t                 block_nexttag;
+    uint8_t                  block_nextsub;
+    uint8_t                  block_level;
+};
+typedef struct ompi_comm_extended_cid_block_t ompi_comm_extended_cid_block_t;
+
+static inline void ompi_comm_extended_cid_block_initialize (ompi_comm_extended_cid_block_t *block, uint64_t cid_base,
+							    uint64_t cid_sub, uint8_t block_level)
+{
+    block->block_cid.cid_base = cid_base;
+    block->block_cid.cid_sub.u64 = cid_sub;
+    block->block_level = block_level;
+    block->block_nextsub = 0;
+    block->block_nexttag = 0;
+}
+
+static inline bool ompi_comm_extended_cid_block_available (ompi_comm_extended_cid_block_t *block)
+{
+    return (4 > block->block_level && 0xff > block->block_nextsub);
+}
+
+static inline char *ompi_comm_extended_cid_get_unique_tag (ompi_comm_extended_cid_block_t *block, int tag,
+                                                           int leader)
+{
+    char *id;
+
+    /* create a unique ID for this */
+    if (-1 == tag) {
+        opal_asprintf (&id, "ALL:%" PRIx64 "-%" PRIx64 "-%" PRIx64, block->block_cid.cid_base,
+                       block->block_cid.cid_sub.u64, ++block->block_nexttag);
+    } else {
+        opal_asprintf (&id, "GROUP:%" PRIx64 "-%" PRIx64 "-%d-%d", block->block_cid.cid_base,
+                       block->block_cid.cid_sub.u64, tag, leader);
+    }
+
+    return id;
+}
+
+/**
+ * Create a new sub-block from an existing block
+ *
+ * @param[in]  block       block
+ * @param[out] new_block   new CID block
+ * @param[in]  use_current use the current CID of the existing block as the base
+ *
+ * This function creates a new CID block from an existing block. The use_current flag
+ * can be used to indicate that the new block should use the existing CID. This can
+ * be used to assign the first CID in a new block.
+ */
+static inline int ompi_comm_extended_cid_block_new (ompi_comm_extended_cid_block_t *block,
+						    ompi_comm_extended_cid_block_t *new_block,
+                                                    bool use_current)
+{
+    if (!ompi_comm_extended_cid_block_available (block)) {
+	/* a new block is needed */
+	return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    new_block->block_cid = block->block_cid;
+    if (!use_current) {
+        new_block->block_cid.cid_sub.u8[3 - block->block_level] = ++block->block_nextsub;
+    }
+
+    new_block->block_level = block->block_level + 1;
+    new_block->block_nextsub = 0;
+
+    return OMPI_SUCCESS;
+}
+
+struct ompi_comm_cid_t {
+    opal_object_t            super;
+    ompi_group_t             cid_group;
+    ompi_comm_extended_cid_t cid_value;
+    uint8_t                  cid_sublevel;
+};
+typedef struct ompi_comm_cid_t ompi_comm_cid_t;
+
+OBJ_CLASS_DECLARATION(ompi_comm_cid_t);
+
 #if OPAL_ENABLE_FT_MPI
 /**
  * This array holds the number of time each id has been used. In the case where a communicator
@@ -157,19 +276,34 @@ struct ompi_communicator_t {
     opal_infosubscriber_t      super;
     opal_mutex_t               c_lock; /* mutex for name and potentially
                                           attributes */
-    char  c_name[MPI_MAX_OBJECT_NAME];
-    uint32_t              c_contextid;
-    int                     c_my_rank;
-    uint32_t                  c_flags; /* flags, e.g. intercomm,
-                                          topology, etc. */
-    uint32_t                  c_assertions; /* info assertions */
-
-    int c_id_available; /* the currently available Cid for allocation
-               to a child*/
-    int c_id_start_index; /* the starting index of the block of cids
-                 allocated to this communicator*/
-    uint32_t c_epoch;  /* Identifier used to differenciate between two communicators
+    char                      *c_name;
+    ompi_comm_extended_cid_t      c_contextid;
+    ompi_comm_extended_cid_block_t c_contextidb;
+    uint32_t                      c_index;
+    int                           c_my_rank;
+    uint32_t                      c_flags; /* flags, e.g. intercomm,
+                                              topology, etc. */
+    uint32_t                      c_assertions; /* info assertions */
+#if OPAL_ENABLE_FT_MPI
+    uint32_t c_epoch;  /* Identifier used to differentiate between two communicators
                           using the same c_contextid (not at the same time, obviously) */
+#endif
+    /* vector used to store remote cid values for communicators not using
+     * a global cid, i.e. when OMPI_COMM_IS_GLOBAL_INDEX(comm) returns 0.
+     */
+    uint32_t                     *c_index_vec;
+    /* Non-blocking collective tag. These tags might be shared between
+     * all non-blocking collective modules (to avoid message collision
+     * between them in the case where multiple outstanding non-blocking
+     * collective coexists using multiple backends).
+     */
+    opal_atomic_int32_t c_nbc_tag;
+
+    /**< inscribing cube dimension */
+    int c_cube_dim;
+
+    /* index in Fortran <-> C translation array */
+    int c_f_to_c_index;
 
     ompi_group_t        *c_local_group;
     ompi_group_t       *c_remote_group;
@@ -182,16 +316,10 @@ struct ompi_communicator_t {
     /* Attributes */
     struct opal_hash_table_t       *c_keyhash;
 
-    /**< inscribing cube dimension */
-    int c_cube_dim;
-
     /* Standard information about the selected topology module (or NULL
        if this is not a cart, graph or dist graph communicator) */
     struct mca_topo_base_module_t* c_topo;
 
-    /* index in Fortran <-> C translation array */
-    int c_f_to_c_index;
-
 #ifdef OMPI_WANT_PERUSE
     /*
      * Place holder for the PERUSE events.
@@ -202,28 +330,28 @@ struct ompi_communicator_t {
     /* Error handling.  This field does not have the "c_" prefix so
        that the OMPI_ERRHDL_* macros can find it, regardless of whether
        it's a comm, window, or file. */
-
     ompi_errhandler_t                  *error_handler;
     ompi_errhandler_type_t             errhandler_type;
 
     /* Hooks for PML to hang things */
     struct mca_pml_comm_t  *c_pml_comm;
 
+    /* Hooks for MTL to hang things */
+    struct mca_mtl_comm_t  *c_mtl_comm;
+
     /* Collectives module interface and data */
     mca_coll_base_comm_coll_t *c_coll;
 
-    /* Non-blocking collective tag. These tags might be shared between
-     * all non-blocking collective modules (to avoid message collision
-     * between them in the case where multiple outstanding non-blocking
-     * collective coexists using multiple backends).
-     */
-    opal_atomic_int32_t c_nbc_tag;
+    /* instance that this comm belongs to */
+    ompi_instance_t* instance;
 
 #if OPAL_ENABLE_FT_MPI
-    /** MPI_ANY_SOURCE Failed Group Offset - OMPI_Comm_failure_get_acked */
-    int                      any_source_offset;
     /** agreement caching info for topology and previous returned decisions */
     opal_object_t           *agreement_specific;
+    /** num_acked - OMPI_Comm_ack_failed */
+    int                      num_acked;
+    /** MPI_ANY_SOURCE Failed Group Offset - OMPI_Comm_failure_get_acked */
+    int                      any_source_offset;
     /** Are MPI_ANY_SOURCE operations enabled? - OMPI_Comm_failure_ack */
     bool                     any_source_enabled;
     /** Has this communicator been revoked - OMPI_Comm_revoke() */
@@ -235,7 +363,7 @@ struct ompi_communicator_t {
 typedef struct ompi_communicator_t ompi_communicator_t;
 
 /**
- * Padded struct to maintain back compatibiltiy.
+ * Padded struct to maintain back compatibility.
  *
  * The following ompi_predefined_xxx_t structure is used to maintain
  * backwards binary compatibility for MPI applications compiled
@@ -284,7 +412,7 @@ typedef struct ompi_communicator_t ompi_communicator_t;
  *
  * - union of struct and padding - Similar to current implementation
  *   except using a union for the parent.  This worked except in cases
- *   where the compilers did not support C99 union static initalizers.
+ *   where the compilers did not support C99 union static initializers.
  *   It would have been a pain to convert a bunch of the code to use
  *   non-static initializers (e.g., MPI datatypes).
  */
@@ -341,7 +469,7 @@ OMPI_DECLSPEC extern ompi_predefined_communicator_t ompi_mpi_comm_null;
 
 /*
  * These variables are for the MPI F03 bindings (F03 must bind Fortran
- * varaiables to symbols; it cannot bind Fortran variables to the
+ * variables to symbols; it cannot bind Fortran variables to the
  * address of a C variable).
  */
 OMPI_DECLSPEC extern ompi_predefined_communicator_t *ompi_mpi_comm_world_addr;
@@ -369,10 +497,10 @@ OMPI_DECLSPEC extern ompi_predefined_communicator_t *ompi_mpi_comm_null_addr;
  * ompi_comm_invalid() as originally coded -- per the MPI-1
  * definition, where MPI_COMM_NULL is an invalid communicator.
  * The MPI_Comm_c2f() function, therefore, calls
- * ompi_comm_invalid() but also explictily checks to see if the
+ * ompi_comm_invalid() but also explicitly checks to see if the
  * handle is MPI_COMM_NULL.
  */
-static inline int ompi_comm_invalid(ompi_communicator_t* comm)
+static inline int ompi_comm_invalid (const ompi_communicator_t* comm)
 {
     if ((NULL == comm) || (MPI_COMM_NULL == comm) ||
         (OMPI_COMM_IS_FREED(comm)) || (OMPI_COMM_IS_INVALID(comm)) )
@@ -384,7 +512,7 @@ static inline int ompi_comm_invalid(ompi_communicator_t* comm)
 /**
  * rank w/in the communicator
  */
-static inline int ompi_comm_rank(ompi_communicator_t* comm)
+static inline int ompi_comm_rank (const ompi_communicator_t* comm)
 {
     return comm->c_my_rank;
 }
@@ -392,7 +520,7 @@ static inline int ompi_comm_rank(ompi_communicator_t* comm)
 /**
  * size of the communicator
  */
-static inline int ompi_comm_size(ompi_communicator_t* comm)
+static inline int ompi_comm_size (const ompi_communicator_t* comm)
 {
     return comm->c_local_group->grp_proc_count;
 }
@@ -401,7 +529,7 @@ static inline int ompi_comm_size(ompi_communicator_t* comm)
  * size of the remote group for inter-communicators.
  * returns zero for an intra-communicator
  */
-static inline int ompi_comm_remote_size(ompi_communicator_t* comm)
+static inline int ompi_comm_remote_size (const ompi_communicator_t* comm)
 {
     return (comm->c_flags & OMPI_COMM_INTER ? comm->c_remote_group->grp_proc_count : 0);
 }
@@ -410,20 +538,103 @@ static inline int ompi_comm_remote_size(ompi_communicator_t* comm)
  * Context ID for the communicator, suitable for passing to
  * ompi_comm_lookup for getting the communicator back
  */
-static inline uint32_t ompi_comm_get_cid(ompi_communicator_t* comm)
+static inline uint32_t ompi_comm_get_local_cid (const ompi_communicator_t* comm)
+{
+    return comm->c_index;
+}
+
+int ompi_comm_get_remote_cid_from_pmix (ompi_communicator_t *comm, int dest, uint32_t *remote_cid);
+
+/**
+ * Get remote cid for the communicator.  In the case of communicators created
+ * using methods that don't supply an input communicator, i.e.
+ * MPI_Comm_create_from_group, the remote cid may be different from the local cid.
+ */
+static inline int ompi_comm_get_remote_cid (ompi_communicator_t *comm, int dest, uint32_t *remote_cid)
+{
+    int rc = OMPI_SUCCESS;
+
+    assert(NULL != remote_cid);
+
+    if (OPAL_LIKELY(OMPI_COMM_IS_GLOBAL_INDEX(comm))) {
+        *remote_cid = comm->c_index;
+    } else if (0 != comm->c_index_vec[dest]) {
+        *remote_cid = comm->c_index_vec[dest];
+    } else {
+        rc = ompi_comm_get_remote_cid_from_pmix(comm, dest, remote_cid);
+    }
+
+    return rc;
+}
+
+/**
+ * Get the extended context ID for the communicator, suitable for passing
+ * to ompi_comm_lookup_cid for getting the communicator back
+ */
+static inline ompi_comm_extended_cid_t ompi_comm_get_extended_cid (const ompi_communicator_t *comm)
 {
     return comm->c_contextid;
 }
 
+static inline bool ompi_comm_cid_compare (const ompi_communicator_t *comm, const ompi_comm_extended_cid_t cid)
+{
+    return comm->c_contextid.cid_base == cid.cid_base && comm->c_contextid.cid_sub.u64 == cid.cid_sub.u64;
+}
+
+static inline bool ompi_comm_compare_cids (const ompi_communicator_t *comm1, const ompi_communicator_t *comm2)
+{
+    return comm1->c_contextid.cid_base == comm2->c_contextid.cid_base && comm1->c_contextid.cid_sub.u64 == comm2->c_contextid.cid_sub.u64;
+}
+
 /* return pointer to communicator associated with context id cid,
  * No error checking is done*/
-static inline ompi_communicator_t *ompi_comm_lookup(uint32_t cid)
+static inline ompi_communicator_t *ompi_comm_lookup (const uint32_t c_index)
 {
+    ompi_communicator_t *comm = NULL;
     /* array of pointers to communicators, indexed by context ID */
-    return (ompi_communicator_t*)opal_pointer_array_get_item(&ompi_mpi_communicators, cid);
+    comm = (ompi_communicator_t *) opal_pointer_array_get_item (&ompi_mpi_communicators, c_index);
+    /*
+     * OMPI_COMM_SENTINEL indicates the slot is being used for CID allocation
+     * and is not a valid communicator
+     */
+    if ((ompi_communicator_t *)OMPI_COMM_SENTINEL == comm) {
+        comm = NULL;
+    }
+
+    /*
+     * return NULL if comm doesn't yet have an associated PML
+     */
+    if ((NULL != comm) && !OMPI_COMM_IS_PML_ADDED(comm)) {
+        comm = NULL;
+    }
+
+    return comm;
+}
+
+/**
+ * Number of entries in the ompi_mpi_communicators pointer array.
+ * Note this includes entries which may have NULL or OMPI_COMM_SENTINEL values.
+ */
+static inline int ompi_comm_get_num_communicators(void)
+{
+    return opal_pointer_array_get_size(&ompi_mpi_communicators);
+}
+
+static inline ompi_communicator_t *ompi_comm_lookup_cid (const ompi_comm_extended_cid_t cid)
+{
+    ompi_communicator_t *comm = NULL;
+    (void) opal_hash_table_get_value_ptr (&ompi_comm_hash, &cid, sizeof (cid), (void **) &comm);
+    /*
+     * return NULL if the comm does not yet have an asociated PML
+     */
+    if ((NULL != comm) && !OMPI_COMM_IS_PML_ADDED(comm)) {
+        comm = NULL;
+    }
+
+    return comm;
 }
 
-static inline struct ompi_proc_t* ompi_comm_peer_lookup(ompi_communicator_t* comm, int peer_id)
+static inline struct ompi_proc_t* ompi_comm_peer_lookup (const ompi_communicator_t* comm, const int peer_id)
 {
 #if OPAL_ENABLE_DEBUG
     if(peer_id >= comm->c_remote_group->grp_proc_count) {
@@ -435,6 +646,12 @@ static inline struct ompi_proc_t* ompi_comm_peer_lookup(ompi_communicator_t* com
     return ompi_group_peer_lookup(comm->c_remote_group,peer_id);
 }
 
+static inline bool ompi_comm_instances_same(const ompi_communicator_t *comm1,
+                                            const ompi_communicator_t *comm2)
+{
+    return comm1->instance == comm2->instance;
+}
+
 #if OPAL_ENABLE_FT_MPI
 /*
  * Support for MPI_ANY_SOURCE point-to-point operations
@@ -460,6 +677,17 @@ static inline bool ompi_comm_is_revoked(ompi_communicator_t* comm)
     return (comm->comm_revoked);
 }
 
+/*
+ * Obtain the group of locally known failed processes on comm
+ */
+OMPI_DECLSPEC int ompi_comm_get_failed_internal(ompi_communicator_t* comm, ompi_group_t** failed_group);
+
+/*
+ * Acknowledge failures and re-enable MPI_ANY_SOURCE
+ * Related to OMPI_Comm_failure_ack() and OMPI_Comm_failure_get_acked()
+ */
+OMPI_DECLSPEC int ompi_comm_ack_failed_internal(ompi_communicator_t* comm, int num_to_ack, int *num_acked);
+
 /*
  * Acknowledge failures and re-enable MPI_ANY_SOURCE
  * Related to OMPI_Comm_failure_ack() and OMPI_Comm_failure_get_acked()
@@ -482,6 +710,11 @@ OMPI_DECLSPEC int ompi_comm_revoke_internal(ompi_communicator_t* comm);
  */
 OMPI_DECLSPEC int ompi_comm_shrink_internal(ompi_communicator_t* comm, ompi_communicator_t** newcomm);
 
+/*
+ * Shrink the communicator
+ */
+OMPI_DECLSPEC int ompi_comm_ishrink_internal(ompi_communicator_t* comm, ompi_communicator_t** newcomm, ompi_request_t** request);
+
 /*
  * Check if the process is active
  */
@@ -616,7 +849,7 @@ OMPI_DECLSPEC int ompi_comm_revoke_finalize(void);
 
 #endif /* OPAL_ENABLE_FT_MPI */
 
-static inline bool ompi_comm_peer_invalid(ompi_communicator_t* comm, int peer_id)
+static inline bool ompi_comm_peer_invalid (const ompi_communicator_t* comm, const int peer_id)
 {
     if(peer_id < 0 || peer_id >= comm->c_remote_group->grp_proc_count) {
         return true;
@@ -624,12 +857,18 @@ static inline bool ompi_comm_peer_invalid(ompi_communicator_t* comm, int peer_id
     return false;
 }
 
+char *ompi_comm_print_cid (const ompi_communicator_t *comm);
 
 /**
- * Initialise MPI_COMM_WORLD and MPI_COMM_SELF
+ * @brief Initialize the communicator subsystem as well as MPI_COMM_NULL.
  */
 int ompi_comm_init(void);
 
+/**
+ * Initialise MPI_COMM_WORLD and MPI_COMM_SELF
+ */
+int ompi_comm_init_mpi3 (void);
+
 /**
  * extract the local group from a communicator
  */
@@ -641,6 +880,9 @@ OMPI_DECLSPEC int ompi_comm_group (ompi_communicator_t *comm, ompi_group_t **gro
 int ompi_comm_create (ompi_communicator_t* comm, ompi_group_t *group,
                       ompi_communicator_t** newcomm);
 
+int ompi_comm_create_w_info (ompi_communicator_t *comm, ompi_group_t *group, opal_info_t *info,
+                             ompi_communicator_t **newcomm);
+
 
 /**
  * Non-collective create communicator based on a group
@@ -648,6 +890,26 @@ int ompi_comm_create (ompi_communicator_t* comm, ompi_group_t *group,
 int ompi_comm_create_group (ompi_communicator_t *comm, ompi_group_t *group, int tag,
                             ompi_communicator_t **newcomm);
 
+/**
+ * Non-collective create communicator based on a group with no base communicator
+ */
+int ompi_comm_create_from_group (ompi_group_t *group, const char *tag, opal_info_t *info,
+                                 ompi_errhandler_t *errhandler, ompi_communicator_t **newcomm);
+
+/**
+ * create an intercommunicator
+ */
+int ompi_intercomm_create (ompi_communicator_t *local_comm, int local_leader, ompi_communicator_t *bridge_comm,
+                           int remote_leader, int tag, ompi_communicator_t **newintercomm);
+
+/**
+ * Non-collective create intercommunicator based on a group with no base communicator
+ */
+int ompi_intercomm_create_from_groups (ompi_group_t *local_group, int local_leader,
+                                       ompi_group_t *remote_group, int remote_leader, const char *tag,
+                                       opal_info_t *info, ompi_errhandler_t *errhandler,
+                                       ompi_communicator_t **newintercomm);
+
 /**
  * Take an almost complete communicator and reserve the CID as well
  * as activate it (initialize the collective and the topologies).
@@ -764,19 +1026,6 @@ int ompi_comm_compare(ompi_communicator_t *comm1, ompi_communicator_t *comm2, in
  */
 OMPI_DECLSPEC int ompi_comm_free (ompi_communicator_t **comm);
 
-/**
- * allocate a new communicator structure
- * @param local_group_size
- * @param remote_group_size
- *
- * This routine allocates the structure, the according local and
- * remote groups, the proc-arrays in the local and remote group.
- * It furthermore sets the fortran index correctly,
- * and sets all other elements to zero.
- */
-ompi_communicator_t* ompi_comm_allocate (int local_group_size,
-                                         int remote_group_size);
-
 /**
  * allocate new communicator ID
  * @param newcomm:    pointer to the new communicator
@@ -821,11 +1070,6 @@ OMPI_DECLSPEC int ompi_comm_nextcid_nb (ompi_communicator_t *newcomm, ompi_commu
                                         ompi_communicator_t *bridgecomm, const void *arg0, const void *arg1,
                                         bool send_first, int mode, ompi_request_t **req);
 
-/**
- * shut down the communicator infrastructure.
- */
-int ompi_comm_finalize (void);
-
 /**
  * This is THE routine, where all the communicator stuff
  * is really set.
@@ -841,6 +1085,7 @@ int ompi_comm_finalize (void);
  * @param[in]  copy_topocomponent whether to copy the topology
  * @param[in]  local_group        local process group (may be NULL if local_ranks array supplied)
  * @param[in]  remote_group       remote process group (may be NULL)
+ * @param[in]  flags              flags to control the behavior of ompi_comm_set_nb
  */
 OMPI_DECLSPEC int ompi_comm_set ( ompi_communicator_t** newcomm,
                                   ompi_communicator_t* oldcomm,
@@ -850,9 +1095,20 @@ OMPI_DECLSPEC int ompi_comm_set ( ompi_communicator_t** newcomm,
                                   int *remote_ranks,
                                   opal_hash_table_t *attr,
                                   ompi_errhandler_t *errh,
-                                  bool copy_topocomponent,
                                   ompi_group_t *local_group,
-                                  ompi_group_t *remote_group   );
+                                  ompi_group_t *remote_group,
+                                  uint32_t flags);
+
+/**
+ * @brief Don't duplicate the local communicator. just reference it directly. This
+ *        flag passes ownership to the new communicator.
+ */
+#define OMPI_COMM_SET_FLAG_LOCAL_COMM_NODUP 0x00000001
+
+/**
+ * @brief Copy the topology from the old communicator
+ */
+#define OMPI_COMM_SET_FLAG_COPY_TOPOLOGY    0x00000002
 
 /**
  * This is THE routine, where all the communicator stuff
@@ -869,6 +1125,7 @@ OMPI_DECLSPEC int ompi_comm_set ( ompi_communicator_t** newcomm,
  * @param[in]  copy_topocomponent whether to copy the topology
  * @param[in]  local_group        local process group (may be NULL if local_ranks array supplied)
  * @param[in]  remote_group       remote process group (may be NULL)
+ * @param[in]  flags              flags to control the behavior of ompi_comm_set_nb
  * @param[out] req                ompi_request_t object for tracking completion
  */
 OMPI_DECLSPEC int ompi_comm_set_nb ( ompi_communicator_t **ncomm,
@@ -879,30 +1136,10 @@ OMPI_DECLSPEC int ompi_comm_set_nb ( ompi_communicator_t **ncomm,
                                      int *remote_ranks,
                                      opal_hash_table_t *attr,
                                      ompi_errhandler_t *errh,
-                                     bool copy_topocomponent,
                                      ompi_group_t *local_group,
                                      ompi_group_t *remote_group,
-                                     ompi_request_t **req );
-
-/**
- * This is a short-hand routine used in intercomm_create.
- * The routine makes sure, that all processes have afterwards
- * a list of ompi_proc_t pointers for the remote group.
- */
-int ompi_comm_get_rprocs ( ompi_communicator_t *local_comm,
-                           ompi_communicator_t *bridge_comm,
-                           int local_leader,
-                           int remote_leader,
-                           int tag,
-                           int rsize,
-                           struct ompi_proc_t ***prprocs );
-
-/**
- * This routine verifies, whether local_group and remote group are overlapping
- * in intercomm_create
- */
-int ompi_comm_overlapping_groups (int size, struct ompi_proc_t ** lprocs,
-                                  int rsize, struct ompi_proc_t ** rprocs);
+                                     uint32_t flags,
+                                     ompi_request_t **req);
 
 /**
  * This is a routine determining whether the local or the
@@ -913,7 +1150,7 @@ int ompi_comm_determine_first ( ompi_communicator_t *intercomm,
                                 int high );
 
 /**
- * This is a routine determining wether the local or the
+ * This is a routine determining whether the local or the
  * remote group will be first in the new intra-comm.
  * It does not communicate to exchange the "high" values; used in Agree
  */
diff --git a/ompi/communicator/ft/comm_ft.c b/ompi/communicator/ft/comm_ft.c
index 3532954f06b..bd2627c778c 100644
--- a/ompi/communicator/ft/comm_ft.c
+++ b/ompi/communicator/ft/comm_ft.c
@@ -1,11 +1,12 @@
 /*
  * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2011-2020 The University of Tennessee and The University
+ * Copyright (c) 2011-2022 The University of Tennessee and The University
  *
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  *
  * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -30,7 +31,7 @@ ompi_comm_rank_failure_callback_t *ompi_rank_failure_cbfunc = NULL;
  * The handling of known failed processes is based on a two level process. On one
  * side the MPI library itself must know the failed processes (in order to be able
  * to correctly handle complex operations such as shrink). On the other side, the
- * failed processes acknowledged by the users shuould not be altered during any of
+ * failed processes acknowledged by the users should not be altered during any of
  * the internal calls, as they must only be updated upon user request.
  * Thus, the global list (ompi_group_all_failed_procs) is the list of all known
  * failed processes (by the MPI library internals), and it is allegedly updated
@@ -39,6 +40,7 @@ ompi_comm_rank_failure_callback_t *ompi_rank_failure_cbfunc = NULL;
  * in the context of a communicator. Thus, using a single index to know the user-level
  * acknowledged failure is the simplest solution.
  */
+/* deprecated ulfm v1 API */
 int ompi_comm_failure_ack_internal(ompi_communicator_t* comm)
 {
     opal_mutex_lock(&ompi_group_afp_mutex);
@@ -49,14 +51,16 @@ int ompi_comm_failure_ack_internal(ompi_communicator_t* comm)
     /* use the AFP lock implicit memory barrier to propagate the update to
      * any_source_enabled at the same time.
      */
+    comm->num_acked = -1; /* compat with v2 API: force recompute next time ack_failed is called */
     opal_mutex_unlock(&ompi_group_afp_mutex);
 
     return OMPI_SUCCESS;
 }
 
+/* deprecated ulfm v1 API; used internally in MPI_COMM_AGREE as well */
 int ompi_comm_failure_get_acked_internal(ompi_communicator_t* comm, ompi_group_t **group )
 {
-    int ret, exit_status = OMPI_SUCCESS;
+    int rc, exit_status = OMPI_SUCCESS;
     int range[3];
     ompi_group_t *tmp_sub_group = NULL;
 
@@ -80,10 +84,10 @@ int ompi_comm_failure_get_acked_internal(ompi_communicator_t* comm, ompi_group_t
     range[1] = comm->any_source_offset - 1;
     range[2] = 1;
 
-    ret = ompi_group_range_incl(ompi_group_all_failed_procs, 1, &range, &tmp_sub_group);
+    rc = ompi_group_range_incl(ompi_group_all_failed_procs, 1, &range, &tmp_sub_group);
     opal_mutex_unlock(&ompi_group_afp_mutex);
-    if( OMPI_SUCCESS != ret ) {
-        exit_status = ret;
+    if( OMPI_SUCCESS != rc ) {
+        exit_status = rc;
         goto cleanup;
     }
 
@@ -91,17 +95,17 @@ int ompi_comm_failure_get_acked_internal(ompi_communicator_t* comm, ompi_group_t
      * Access the intersection between the failed subgroup and our group
      */
     if( OMPI_COMM_IS_INTER(comm) ) {
-        ret = ompi_group_intersection(tmp_sub_group,
-                                      comm->c_local_group,
-                                      group);
+        rc = ompi_group_intersection(tmp_sub_group,
+                                     comm->c_local_group,
+                                     group);
     } else {
-        ret = ompi_group_intersection(tmp_sub_group,
-                                      comm->c_remote_group,
-                                      group);
+        rc = ompi_group_intersection(tmp_sub_group,
+                                     comm->c_remote_group,
+                                     group);
     }
 
-    if( OMPI_SUCCESS != ret ) {
-        exit_status = ret;
+    if( OMPI_SUCCESS != rc ) {
+        exit_status = rc;
         goto cleanup;
     }
 
@@ -114,14 +118,115 @@ int ompi_comm_failure_get_acked_internal(ompi_communicator_t* comm, ompi_group_t
     return exit_status;
 }
 
-int ompi_comm_shrink_internal(ompi_communicator_t* comm, ompi_communicator_t** newcomm)
+/* New v2 interface get_failed/ack_failed.
+ * This interface uses a cached value comm->num_acked to track how many
+ * processes in the group of this comm have been acknowledged in prior calls.
+ * For compatibility with v1 interface (failure_get_acked), it still updates
+ * the comm->any_source_offset, and the v1 interface failure_ack may erase the
+ * cached value comm->num_acked with -1 to force recomputing this value in mixed
+ * use cases (that is calling failure_ack will force full recomputation of
+ * comm->num_acked during the next ack_failed call). */
+int ompi_comm_ack_failed_internal(ompi_communicator_t* comm, int num_to_ack, int *num_acked) {
+    int ret, exit_status = MPI_SUCCESS;
+    int nf = -1, na = -1;
+    ompi_group_t *c_group = OMPI_COMM_IS_INTER(comm)? comm->c_local_group: comm->c_remote_group;
+    ompi_group_t *failed_group = NULL;
+
+    opal_mutex_lock(&ompi_group_afp_mutex);
+
+    /* shortcut when reading only */
+    if(num_to_ack <= comm->num_acked)
+        goto return_num_acked;
+
+    /* shortcut when no new faults */
+    if(comm->any_source_offset == ompi_group_size(ompi_group_all_failed_procs)
+    && comm->num_acked >= 0 /* reset by a call to v1 API? */)
+        goto return_num_acked;
+
+    /* compute num_acked */
+    ret = ompi_group_intersection(ompi_group_all_failed_procs,
+                                  c_group, &failed_group);
+    if(OMPI_SUCCESS != ret) {
+        exit_status = ret;
+        goto cleanup;
+    }
+    nf = ompi_group_size(failed_group);
+    na = (num_to_ack < nf)? num_to_ack: nf; /* never ack more than requested */
+
+    if(comm->num_acked < 0) { /* reset by a call to the v1 API: recompute it */
+        if(0 == comm->any_source_offset) comm->num_acked = 0;
+        else {
+            int aso = comm->any_source_offset - 1;
+            ret = ompi_group_translate_ranks(ompi_group_all_failed_procs, 1, &aso,
+                                             failed_group, &comm->num_acked);
+            comm->num_acked++; /* make it a group size again */
+            if(OMPI_SUCCESS != ret) {
+                exit_status = ret;
+                goto cleanup;
+            }
+        }
+    }
+
+    if(comm->num_acked < na) { /* comm->num_acked needs to be updated during this call */
+        comm->num_acked = na;
+        if(nf == na) {
+            /* all faults on comm acknowledged, resume any source then */
+            comm->any_source_enabled = true;
+            comm->any_source_offset = ompi_group_size(ompi_group_all_failed_procs); // compat with v1 interface
+        }
+        else {
+            /* some faults not acknowledged, do not resume any source then, but
+             * still update any_source_offset */
+            assert(comm->num_acked > 0);
+            int cna = comm->num_acked - 1;
+            ret = ompi_group_translate_ranks(failed_group, 1, &cna,
+                                             ompi_group_all_failed_procs, &comm->any_source_offset); // compat with v1 interface
+            comm->any_source_offset++; /* make it a group size again */
+            if(OMPI_SUCCESS != ret) {
+                exit_status = ret;
+                goto cleanup;
+            }
+        }
+    }
+
+return_num_acked:
+    *num_acked = comm->num_acked;
+
+cleanup:
+    if(NULL != failed_group) OBJ_RELEASE(failed_group);
+    /* use the AFP lock implicit memory barrier to propagate the update to
+     * any_source_enabled, num_acked, etc. at the same time.
+     */
+    opal_mutex_unlock(&ompi_group_afp_mutex);
+
+    return exit_status;
+}
+
+int ompi_comm_get_failed_internal(ompi_communicator_t* comm, ompi_group_t **group)
 {
     int ret, exit_status = OMPI_SUCCESS;
+    ompi_group_t *c_group = OMPI_COMM_IS_INTER(comm)? comm->c_local_group: comm->c_remote_group;
+    opal_mutex_lock(&ompi_group_afp_mutex);
+    ret = ompi_group_intersection(ompi_group_all_failed_procs,
+                                  c_group,
+                                  group);
+    opal_mutex_unlock(&ompi_group_afp_mutex);
+    if( OMPI_SUCCESS != ret ) {
+        exit_status = ret;
+    }
+    return exit_status;
+}
+
+int ompi_comm_shrink_internal(ompi_communicator_t* comm, ompi_communicator_t** newcomm)
+{
+    int rc, exit_status = OMPI_SUCCESS;
     int flag = 1;
     ompi_group_t *failed_group = NULL, *comm_group = NULL, *alive_group = NULL, *alive_rgroup = NULL;
     ompi_communicator_t *newcomp = NULL;
     int mode;
+#if OPAL_ENABLE_DEBUG
     double start, stop;
+#endif
 
     *newcomm = MPI_COMM_NULL;
 
@@ -132,35 +237,41 @@ int ompi_comm_shrink_internal(ompi_communicator_t* comm, ompi_communicator_t** n
     OPAL_OUTPUT_VERBOSE((5, ompi_ftmpi_output_handle,
                          "%s ompi: comm_shrink: Agreement on failed processes",
                          OMPI_NAME_PRINT(OMPI_PROC_MY_NAME) ));
+#if OPAL_ENABLE_DEBUG
     start = PMPI_Wtime();
+#endif
     opal_mutex_lock(&ompi_group_afp_mutex);
     ompi_group_intersection(comm->c_remote_group, ompi_group_all_failed_procs, &failed_group);
     opal_mutex_unlock(&ompi_group_afp_mutex);
+#if OPAL_ENABLE_DEBUG
     stop = PMPI_Wtime();
     OPAL_OUTPUT_VERBOSE((10, ompi_ftmpi_output_handle,
                          "%s ompi: comm_shrink: group_inter: %g seconds",
                          OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), stop-start));
     start = PMPI_Wtime();
+#endif
     do {
         /* We need to create the list of alive processes. Thus, we don't care about
          * the value of flag, instead we are only using the globally consistent
          * return value.
          */
-        ret = comm->c_coll->coll_agree( &flag,
-                                        1,
-                                        &ompi_mpi_int.dt,
-                                        &ompi_mpi_op_band.op,
-                                        &failed_group, true,
-                                        comm,
-                                        comm->c_coll->coll_agree_module);
-    } while( MPI_ERR_PROC_FAILED == ret );
+        rc = comm->c_coll->coll_agree( &flag,
+                                       1,
+                                       &ompi_mpi_int.dt,
+                                       &ompi_mpi_op_band.op,
+                                       &failed_group, true,
+                                       comm,
+                                       comm->c_coll->coll_agree_module);
+    } while( MPI_ERR_PROC_FAILED == rc );
+#if OPAL_ENABLE_DEBUG
     stop = PMPI_Wtime();
     OPAL_OUTPUT_VERBOSE((10, ompi_ftmpi_output_handle,
                          "%s ompi: comm_shrink: AGREE: %g seconds",
                          OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), stop-start));
-    if(OMPI_SUCCESS != ret) {
-        opal_output(0, "%s:%d Agreement failure: %d\n", __FILE__, __LINE__, ret);
-        exit_status = ret;
+#endif
+    if(OMPI_SUCCESS != rc) {
+        opal_output(0, "%s:%d Agreement failure: %d\n", __FILE__, __LINE__, rc);
+        exit_status = rc;
         goto cleanup;
     }
 
@@ -169,51 +280,55 @@ int ompi_comm_shrink_internal(ompi_communicator_t* comm, ompi_communicator_t** n
      */
     /* --------------------------------------------------------- */
     OPAL_OUTPUT_VERBOSE((5, ompi_ftmpi_output_handle,
-                         "%s ompi: comm_shrink: Determine ranking for new communicator",
-                         OMPI_NAME_PRINT(OMPI_PROC_MY_NAME) ));
+                         "%s ompi: comm_shrink: Determine ranking for new communicator intra %d",
+                         OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), OMPI_COMM_IS_INTRA(comm)));
+#if OPAL_ENABLE_DEBUG
     start = PMPI_Wtime();
+#endif
 
     /* Create 'alive' groups */
     mode = OMPI_COMM_CID_INTRA_FT;
     comm_group = comm->c_local_group;
-    ret = ompi_group_difference(comm_group, failed_group, &alive_group);
-    if( OMPI_SUCCESS != ret ) {
-        exit_status = ret;
+    rc = ompi_group_difference(comm_group, failed_group, &alive_group);
+    if( OMPI_SUCCESS != rc ) {
+        exit_status = rc;
         goto cleanup;
     }
     if( OMPI_COMM_IS_INTER(comm) ) {
         mode = OMPI_COMM_CID_INTER_FT;
         comm_group = comm->c_remote_group;
-        ret = ompi_group_difference(comm_group, failed_group, &alive_rgroup);
-        if( OMPI_SUCCESS != ret ) {
-            exit_status = ret;
+        rc = ompi_group_difference(comm_group, failed_group, &alive_rgroup);
+        if( OMPI_SUCCESS != rc ) {
+            exit_status = rc;
             goto cleanup;
         }
     }
-    ret = ompi_comm_set( &newcomp,                 /* new comm */
-                         comm,                     /* old comm */
-                         0,                        /* local_size */
-                         NULL,                     /* local_ranks */
-                         0,                        /* remote_size */
-                         NULL,                     /* remote_ranks */
-                         comm->c_keyhash,          /* attrs */
-                         comm->error_handler,      /* error handler */
-                         NULL,                     /* topo component */
-                         alive_group,              /* local group */
-                         alive_rgroup              /* remote group */
-                       );
-    if( OMPI_SUCCESS != ret ) {
-        exit_status = ret;
+    rc = ompi_comm_set( &newcomp,                 /* new comm */
+                        comm,                     /* old comm */
+                        0,                        /* local_size */
+                        NULL,                     /* local_ranks */
+                        0,                        /* remote_size */
+                        NULL,                     /* remote_ranks */
+                        comm->c_keyhash,          /* attrs */
+                        comm->error_handler,      /* error handler */
+                        alive_group,              /* local group */
+                        alive_rgroup,             /* remote group */
+                        0                         /* flags */
+                      );
+    if( OMPI_SUCCESS != rc ) {
+        exit_status = rc;
         goto cleanup;
     }
     if( NULL == newcomp ) {
         exit_status = MPI_ERR_INTERN;
         goto cleanup;
     }
+#if OPAL_ENABLE_DEBUG
     stop = PMPI_Wtime();
     OPAL_OUTPUT_VERBOSE((10, ompi_ftmpi_output_handle,
                          "%s ompi: comm_shrink: GRP COMPUTATION: %g seconds\n",
                          OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), stop-start));
+#endif
     /*
      * Step 3: Determine context id
      */
@@ -221,46 +336,55 @@ int ompi_comm_shrink_internal(ompi_communicator_t* comm, ompi_communicator_t** n
     OPAL_OUTPUT_VERBOSE((5, ompi_ftmpi_output_handle,
                          "%s ompi: comm_shrink: Determine context id",
                          OMPI_NAME_PRINT(OMPI_PROC_MY_NAME) ));
+#if OPAL_ENABLE_DEBUG
     start = PMPI_Wtime();
-    ret = ompi_comm_nextcid( newcomp,  /* new communicator */
-                             comm,     /* old comm */
-                             NULL,     /* bridge comm */
-                             NULL,     /* local leader */
-                             NULL,     /* remote_leader */
-                             -1,       /* send_first */
-                             mode);    /* mode */
-    if( OMPI_SUCCESS != ret ) {
+#endif
+    rc = ompi_comm_nextcid( newcomp,  /* new communicator */
+                            comm,     /* old comm */
+                            NULL,     /* bridge comm */
+                            NULL,     /* local leader */
+                            NULL,     /* remote_leader */
+                            -1,       /* send_first */
+                            mode);    /* mode */
+    if( OMPI_SUCCESS != rc ) {
         opal_output_verbose(1, ompi_ftmpi_output_handle,
                             "%s ompi: comm_shrink: Determine context id failed with error %d",
-                            OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), ret);
-        exit_status = ret;
+                            OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), rc);
+        exit_status = rc;
         goto cleanup;
     }
+#if OPAL_ENABLE_DEBUG
     stop = PMPI_Wtime();
     OPAL_OUTPUT_VERBOSE((10, ompi_ftmpi_output_handle,
                          "%s ompi: comm_shrink: NEXT CID: %g seconds\n",
                          OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), stop-start));
+#endif
     /*
      * Step 4: Activate the communicator
      */
     /* --------------------------------------------------------- */
     /* Set name for debugging purposes */
     snprintf(newcomp->c_name, MPI_MAX_OBJECT_NAME, "MPI COMMUNICATOR %d SHRUNK FROM %d",
-             newcomp->c_contextid, comm->c_contextid );
+             ompi_comm_get_local_cid(newcomp),
+             ompi_comm_get_local_cid(comm));
+#if OPAL_ENABLE_DEBUG
     start = PMPI_Wtime();
+#endif
     /* activate communicator and init coll-module */
-    ret = ompi_comm_activate( &newcomp, /* new communicator */
-                              comm,
-                              NULL,
-                              NULL,
-                              NULL,
-                              -1,
-                              mode);
-    if( OMPI_SUCCESS != ret ) {
-        exit_status = ret;
+    rc = ompi_comm_activate( &newcomp, /* new communicator */
+                             comm,
+                             NULL,
+                             NULL,
+                             NULL,
+                             -1,
+                             mode);
+    if( OMPI_SUCCESS != rc ) {
+        exit_status = rc;
         goto cleanup;
     }
+#if OPAL_ENABLE_DEBUG
     stop = PMPI_Wtime();
+#endif
     OPAL_OUTPUT_VERBOSE((10, ompi_ftmpi_output_handle,
                          "%s ompi: comm_shrink: COLL SELECT: %g seconds\n",
                          OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), stop-start));
@@ -285,6 +409,358 @@ int ompi_comm_shrink_internal(ompi_communicator_t* comm, ompi_communicator_t** n
     return exit_status;
 }
 
+struct ompi_comm_ishrink_context_t {
+    opal_object_t super;
+    ompi_communicator_t *comm;
+    ompi_communicator_t **newcomm;
+    ompi_group_t *failed_group;
+    ompi_group_t *alive_group;
+    ompi_group_t *alive_rgroup;
+    int flag;
+    double start;
+};
+typedef struct ompi_comm_ishrink_context_t ompi_comm_ishrink_context_t;
+OBJ_CLASS_INSTANCE(ompi_comm_ishrink_context_t, opal_object_t, NULL, NULL);
+
+static int ompi_comm_ishrink_check_agree(ompi_comm_request_t *request);
+static int ompi_comm_ishrink_check_setrank(ompi_comm_request_t *request);
+static int ompi_comm_ishrink_check_cid(ompi_comm_request_t *request);
+static int ompi_comm_ishrink_check_activate(ompi_comm_request_t *request);
+
+int ompi_comm_ishrink_internal(ompi_communicator_t* comm, ompi_communicator_t** newcomm, ompi_request_t** req)
+{
+    int rc;
+#if OPAL_ENABLE_DEBUG
+    double stop;
+#endif
+    ompi_comm_request_t *request;
+    ompi_comm_ishrink_context_t *context;
+    ompi_request_t *subreq[1];
+
+    *newcomm = MPI_COMM_NULL;
+
+    request = ompi_comm_request_get();
+    if(OPAL_UNLIKELY( NULL == request )) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    context = OBJ_NEW(ompi_comm_ishrink_context_t);
+    if(OPAL_UNLIKELY( NULL == context )) {
+        ompi_comm_request_return(request);
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+    context->comm = comm;
+    context->newcomm = newcomm;
+    context->failed_group = NULL;
+    context->alive_group = NULL;
+    context->alive_rgroup = NULL;
+    request->context = &context->super;
+    request->super.req_mpi_object.comm = comm;
+
+    /*
+     * Step 1: Agreement on failed group in comm
+     */
+    /* --------------------------------------------------------- */
+    OPAL_OUTPUT_VERBOSE((5, ompi_ftmpi_output_handle,
+                         "%s ompi: comm_ishrink: Agreement on failed processes",
+                         OMPI_NAME_PRINT(OMPI_PROC_MY_NAME) ));
+#if OPAL_ENABLE_DEBUG
+    context->start = PMPI_Wtime();
+#endif
+    ompi_group_intersection(comm->c_remote_group, ompi_group_all_failed_procs, &context->failed_group);
+#if OPAL_ENABLE_DEBUG
+    stop = PMPI_Wtime();
+    OPAL_OUTPUT_VERBOSE((10, ompi_ftmpi_output_handle,
+                         "%s ompi: comm_ishrink: group_inter: %g seconds",
+                         OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), stop-context->start));
+    context->start = PMPI_Wtime();
+#endif
+    /* We need to create the list of alive processes. Thus, we don't care about
+     * the value of flag, instead we are only using the globally consistent
+     * return value.
+     */
+    context->flag = 1;
+    rc = comm->c_coll->coll_iagree( &context->flag,
+                                    1,
+                                    &ompi_mpi_int.dt,
+                                    &ompi_mpi_op_band.op,
+                                    &context->failed_group, true,
+                                    comm,
+                                    subreq,
+                                    comm->c_coll->coll_iagree_module );
+    if( OMPI_SUCCESS != rc ) {
+        OBJ_RELEASE(context->failed_group);
+        ompi_comm_request_return(request);
+        return rc;
+    }
+
+    ompi_comm_request_schedule_append(request, ompi_comm_ishrink_check_agree, subreq, 1);
+
+    /* kick off the request */
+    ompi_comm_request_start(request);
+    *req = &request->super;
+
+    return OMPI_SUCCESS;
+}
+
+static int ompi_comm_ishrink_check_agree(ompi_comm_request_t *request) {
+    ompi_comm_ishrink_context_t *context =
+        (ompi_comm_ishrink_context_t *)request->context;
+    ompi_communicator_t *comm = context->comm;
+    ompi_request_t *subreq[1];
+    ompi_group_t *comm_group = NULL;
+    int rc;
+#if OPAL_ENABLE_DEBUG
+    double stop;
+#endif
+
+#if OPAL_ENABLE_DEBUG
+    stop = PMPI_Wtime();
+    OPAL_OUTPUT_VERBOSE((10, ompi_ftmpi_output_handle,
+                         "%s ompi: comm_ishrink: AGREE: %g seconds",
+                         OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), stop-context->start));
+#endif
+    rc = request->super.req_status.MPI_ERROR;
+    if( (OMPI_SUCCESS != rc) && (MPI_ERR_PROC_FAILED != rc) ) {
+        opal_output(0, "%s:%d Agreement failure: %d\n", __FILE__, __LINE__, rc);
+        OBJ_RELEASE(context->failed_group);
+        ompi_comm_request_return(request);
+        return rc;
+    }
+
+    if( MPI_ERR_PROC_FAILED == rc ) {
+        /* previous round found more failures, redo */
+        OBJ_RELEASE(context->failed_group);
+        request->super.req_status.MPI_ERROR = MPI_SUCCESS;
+        context->flag = 1;
+        rc = comm->c_coll->coll_iagree( &context->flag,
+                                        1,
+                                        &ompi_mpi_int.dt,
+                                        &ompi_mpi_op_band.op,
+                                        &context->failed_group, true,
+                                        comm,
+                                        subreq,
+                                        comm->c_coll->coll_iagree_module );
+        if( OMPI_SUCCESS != rc ) {
+            OBJ_RELEASE(context->failed_group);
+            ompi_comm_request_return(request);
+            return rc;
+        }
+        ompi_comm_request_schedule_append(request, ompi_comm_ishrink_check_agree, subreq, 1);
+        return OMPI_SUCCESS;
+    }
+
+    /*
+     * Step 2: Determine ranks for new communicator
+     */
+    /* --------------------------------------------------------- */
+    OPAL_OUTPUT_VERBOSE((5, ompi_ftmpi_output_handle,
+                         "%s ompi: comm_ishrink: Determine ranking for new communicator",
+                         OMPI_NAME_PRINT(OMPI_PROC_MY_NAME) ));
+#if OPAL_ENABLE_DEBUG
+    context->start = PMPI_Wtime();
+#endif
+
+    /* Create 'alive' groups */
+    comm_group = comm->c_local_group;
+    rc = ompi_group_difference(comm_group, context->failed_group, &context->alive_group);
+    if( OMPI_SUCCESS != rc ) {
+        OBJ_RELEASE(context->failed_group);
+        ompi_comm_request_return(request);
+        return rc;
+    }
+    if( OMPI_COMM_IS_INTER(comm) ) {
+        comm_group = comm->c_remote_group;
+        rc = ompi_group_difference(comm_group, context->failed_group, &context->alive_rgroup);
+        if( OMPI_SUCCESS != rc ) {
+            OBJ_RELEASE(context->alive_group);
+            OBJ_RELEASE(context->failed_group);
+            ompi_comm_request_return(request);
+            return rc;
+        }
+    }
+    OBJ_RELEASE(context->failed_group);
+
+    rc = ompi_comm_set_nb( context->newcomm,         /* new comm */
+                           comm,                     /* old comm */
+                           0,                        /* local_size */
+                           NULL,                     /* local_ranks */
+                           0,                        /* remote_size */
+                           NULL,                     /* remote_ranks */
+                           comm->c_keyhash,          /* attrs */
+                           comm->error_handler,      /* error handler */
+                           context->alive_group,     /* local group */
+                           context->alive_rgroup,    /* remote group */
+                           0,                        /* flags */
+                           subreq
+                         );
+    if( OMPI_SUCCESS != rc ) {
+        OBJ_RELEASE(context->alive_group);
+        if( NULL != context->alive_rgroup ) {
+            OBJ_RELEASE(context->alive_rgroup);
+        }
+        ompi_comm_request_return(request);
+        return rc;
+    }
+
+    ompi_comm_request_schedule_append(request, ompi_comm_ishrink_check_setrank, subreq, subreq[0] ? 1 : 0);
+    return OMPI_SUCCESS;
+}
+
+static int ompi_comm_ishrink_check_setrank(ompi_comm_request_t *request) {
+    ompi_comm_ishrink_context_t *context =
+        (ompi_comm_ishrink_context_t *)request->context;
+    ompi_request_t *subreq[1];
+    int rc, mode;
+#if OPAL_ENABLE_DEBUG
+    double stop;
+#endif
+
+    /* cleanup temporary groups */
+    OBJ_RELEASE(context->alive_group);
+    if( NULL != context->alive_rgroup ) {
+        OBJ_RELEASE(context->alive_rgroup);
+    }
+
+    /* check errors in prior step */
+    rc = request->super.req_status.MPI_ERROR;
+    if( OMPI_SUCCESS != rc ) {
+        opal_output_verbose(1, ompi_ftmpi_output_handle,
+                            "%s ompi: comm_ishrink: Construction failed with error %d",
+                            OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), rc);
+        OBJ_RELEASE(*context->newcomm);
+        ompi_comm_request_return(request);
+        return rc;
+    }
+
+#if OPAL_ENABLE_DEBUG
+    stop = PMPI_Wtime();
+    OPAL_OUTPUT_VERBOSE((10, ompi_ftmpi_output_handle,
+                         "%s ompi: comm_ishrink: GRP COMPUTATION: %g seconds\n",
+                         OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), stop-context->start));
+#endif
+
+    /*
+     * Step 3: Determine context id
+     */
+    mode = OMPI_COMM_CID_INTRA_FT;
+    if( OMPI_COMM_IS_INTER(context->comm) ) {
+        mode = OMPI_COMM_CID_INTER_FT;
+    }
+    /* --------------------------------------------------------- */
+    OPAL_OUTPUT_VERBOSE((5, ompi_ftmpi_output_handle,
+                         "%s ompi: comm_ishrink: Determine context id",
+                         OMPI_NAME_PRINT(OMPI_PROC_MY_NAME) ));
+#if OPAL_ENABLE_DEBUG
+    context->start = PMPI_Wtime();
+#endif
+    rc = ompi_comm_nextcid_nb( *context->newcomm, /* new communicator */
+                               context->comm,     /* old comm */
+                               NULL,              /* bridge comm */
+                               NULL,              /* local leader */
+                               NULL,              /* remote_leader */
+                               false,             /* send_first */
+                               mode,              /* mode */
+                               subreq );
+    if( OMPI_SUCCESS != rc ) {
+        OBJ_RELEASE(*context->newcomm);
+        ompi_comm_request_return(request);
+        return rc;
+    }
+
+    ompi_comm_request_schedule_append(request, ompi_comm_ishrink_check_cid, subreq, 1);
+
+    return OMPI_SUCCESS;
+}
+
+static int ompi_comm_ishrink_check_cid(ompi_comm_request_t *request) {
+    ompi_comm_ishrink_context_t *context =
+        (ompi_comm_ishrink_context_t *)request->context;
+    ompi_request_t *subreq[1];
+    int rc, mode;
+#if OPAL_ENABLE_DEBUG
+    double stop;
+#endif
+
+    rc = request->super.req_status.MPI_ERROR;
+    if( OMPI_SUCCESS != rc ) {
+        opal_output_verbose(1, ompi_ftmpi_output_handle,
+                            "%s ompi: comm_ishrink: Determine context id failed with error %d",
+                            OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), rc);
+        OBJ_RELEASE(*context->newcomm);
+        ompi_comm_request_return(request);
+        return rc;
+    }
+#if OPAL_ENABLE_DEBUG
+    stop = PMPI_Wtime();
+    OPAL_OUTPUT_VERBOSE((10, ompi_ftmpi_output_handle,
+                         "%s ompi: comm_ishrink: NEXT CID: %g seconds\n",
+                         OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), stop-context->start));
+#endif
+
+    /*
+     * Step 4: Activate the communicator
+     */
+    mode = OMPI_COMM_CID_INTRA_FT;
+    if( OMPI_COMM_IS_INTER(context->comm) ) {
+        mode = OMPI_COMM_CID_INTER_FT;
+    }
+    /* --------------------------------------------------------- */
+    /* Set name for debugging purposes */
+    ompi_communicator_t *newcomp = *context->newcomm;
+    snprintf(newcomp->c_name, MPI_MAX_OBJECT_NAME, "MPI COMMUNICATOR %d SHRUNK FROM %d",
+             ompi_comm_get_local_cid(newcomp),
+             ompi_comm_get_local_cid(context->comm));
+#if OPAL_ENABLE_DEBUG
+    context->start = PMPI_Wtime();
+#endif
+    /* activate communicator and init coll-module */
+    rc = ompi_comm_activate_nb( context->newcomm, /* new communicator */
+                                context->comm,
+                                NULL,
+                                NULL,
+                                NULL,
+                                -1,
+                                mode,
+                                subreq );
+    if( OMPI_SUCCESS != rc ) {
+        OBJ_RELEASE(*context->newcomm);
+        ompi_comm_request_return(request);
+        return rc;
+    }
+
+    ompi_comm_request_schedule_append(request, ompi_comm_ishrink_check_activate, subreq, 1);
+
+    return OMPI_SUCCESS;
+}
+
+static int ompi_comm_ishrink_check_activate(ompi_comm_request_t *request) {
+    ompi_comm_ishrink_context_t *context =
+        (ompi_comm_ishrink_context_t *)request->context;
+    int rc;
+#if OPAL_ENABLE_DEBUG
+    double stop;
+#endif
+
+    rc = request->super.req_status.MPI_ERROR;
+    if( OMPI_SUCCESS != rc ) {
+        opal_output_verbose(1, ompi_ftmpi_output_handle,
+                            "%s ompi: comm_ishrink: Activation failed with error %d",
+                            OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), rc);
+        OBJ_RELEASE(*context->newcomm);
+        ompi_comm_request_return(request);
+        return rc;
+    }
+#if OPAL_ENABLE_DEBUG
+    stop = MPI_Wtime();
+    OPAL_OUTPUT_VERBOSE((10, ompi_ftmpi_output_handle,
+                         "%s ompi: comm_ishrink: COLL SELECT: %g seconds\n",
+                         OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), stop-context->start));
+#endif
+
+    return OMPI_SUCCESS;
+}
+
 
 bool ompi_comm_is_proc_active(ompi_communicator_t *comm, int peer_id, bool remote)
 {
@@ -316,6 +792,14 @@ bool ompi_comm_is_proc_active(ompi_communicator_t *comm, int peer_id, bool remot
 
 int ompi_comm_set_rank_failed(ompi_communicator_t *comm, int peer_id, bool remote)
 {
+#if OPAL_ENABLE_DEBUG
+    /* populate the proc in the comm's group array so that it is not a
+       sentinel and can be read as failed */
+    ompi_proc_t *ompi_proc = ompi_group_get_proc_ptr((remote ? comm->c_remote_group : comm->c_local_group),
+                                                     peer_id, true);
+    assert(NULL != ompi_proc);
+#endif
+
     /* Disable ANY_SOURCE */
     comm->any_source_enabled = false;
     opal_atomic_wmb(); /* non-locked update needs a memory barrier to propagate */
diff --git a/ompi/communicator/ft/comm_ft_detector.c b/ompi/communicator/ft/comm_ft_detector.c
index 9fccfaa4d6f..7b835e84f81 100644
--- a/ompi/communicator/ft/comm_ft_detector.c
+++ b/ompi/communicator/ft/comm_ft_detector.c
@@ -1,9 +1,12 @@
 /*
- * Copyright (c) 2016-2020 The University of Tennessee and The University
+ * Copyright (c) 2016-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
- *
  * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2021      Triad National Security, LLC. All rights
+ *                         reserved.
+ *
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -337,8 +340,8 @@ static int fd_heartbeat_request(comm_detector_t* detector) {
         /* if everybody else is dead, I don't need to monitor myself. */
         if( rank == comm->c_my_rank ) {
             OPAL_OUTPUT_VERBOSE((2, ompi_ftmpi_output_handle,
-                             "%s %s: Every other node is dead on communicator %3d:%d",
-                             OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), __func__, comm->c_contextid, comm->c_epoch));
+                             "%s %s: Every other node is dead on communicator %s:%d",
+                             OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), __func__, ompi_comm_print_cid(comm), comm->c_epoch));
             detector->hb_observer = detector->hb_observing = MPI_PROC_NULL;
             detector->hb_rstamp = INFINITY;
             detector->hb_period = INFINITY;
@@ -354,8 +357,8 @@ static int fd_heartbeat_request(comm_detector_t* detector) {
 #endif
 
         OPAL_OUTPUT_VERBOSE((2, ompi_ftmpi_output_handle,
-                             "%s %s: Sending observe request to %d on communicator %3d:%d stamp %g",
-                             OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), __func__, rank, comm->c_contextid, comm->c_epoch, detector->hb_rstamp-startdate ));
+                             "%s %s: Sending observe request to %d on communicator %s:%d stamp %g",
+                             OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), __func__, rank, ompi_comm_print_cid(comm), comm->c_epoch, detector->hb_rstamp-startdate ));
 
         if( comm_detector_use_rdma_hb ) {
             mca_bml_base_endpoint_t* endpoint = mca_bml_base_get_endpoint(proc);
@@ -380,7 +383,7 @@ static int fd_heartbeat_request(comm_detector_t* detector) {
         detector->hb_observing = rank;
 
         ompi_comm_heartbeat_req_t* msg = calloc(sizeof(*msg)+regsize, 1);
-        msg->super.cid = comm->c_contextid;
+        msg->super.cid = ompi_comm_get_local_cid(comm);
         msg->super.epoch = comm->c_epoch;
         msg->super.type = comm_heartbeat_request_cb_type;
         msg->from = comm->c_my_rank;
@@ -389,7 +392,7 @@ static int fd_heartbeat_request(comm_detector_t* detector) {
             memcpy(&msg->rdma_rreg[0], detector->hb_rdma_flag_lreg, regsize);
             msg->rdma_raddr = (uint64_t)&detector->hb_rdma_flag;
         }
-        ret = ompi_comm_rbcast_send_msg(proc, &msg->super, sizeof(*msg)+regsize);
+        ret = ompi_comm_rbcast_send_msg(proc, (ompi_comm_rbcast_message_t*)msg, sizeof(*msg)+regsize);
         free(msg);
         break;
     }
@@ -407,19 +410,18 @@ static int fd_heartbeat_request_cb(ompi_communicator_t* comm, ompi_comm_heartbea
     ro = (np-comm->c_my_rank+detector->hb_observer) % np; /* same for the observer rank */
     if( rr < ro ) {
         opal_output_verbose(1, ompi_ftmpi_output_handle,
-                             "%s %s: Received heartbeat request from %d on communicator %3d:%d but I am monitored by %d -- this is stall information, ignoring.",
-                             OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), __func__, msg->from, comm->c_contextid, comm->c_epoch, detector->hb_observer );
+                             "%s %s: Received heartbeat request from %d on communicator %s:%d but I am monitored by %d -- this is stall information, ignoring.",
+                             OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), __func__, msg->from, ompi_comm_print_cid(comm), comm->c_epoch, detector->hb_observer );
         return false; /* never forward on the rbcast */
     }
     OPAL_OUTPUT_VERBOSE((2, ompi_ftmpi_output_handle,
-                         "%s %s: Recveived heartbeat request from %d on communicator %3d:%d",
-                         OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), __func__, msg->from, comm->c_contextid, comm->c_epoch));
+                         "%s %s: Recveived heartbeat request from %d on communicator %s:%d",
+                         OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), __func__, msg->from, ompi_comm_print_cid(comm), comm->c_epoch));
     detector->hb_observer = msg->from;
     detector->hb_sstamp = 0.;
 
     if( comm_detector_use_rdma_hb ) {
-        ompi_communicator_t* comm = detector->comm;
-        ompi_proc_t* proc = ompi_comm_peer_lookup(comm, msg->from);
+        ompi_proc_t* proc = ompi_comm_peer_lookup(detector->comm, msg->from);
         assert( NULL != proc );
         mca_bml_base_endpoint_t* endpoint = mca_bml_base_get_endpoint(proc);
         assert( NULL != endpoint );
@@ -460,7 +462,7 @@ static int fd_heartbeat_request_cb(ompi_communicator_t* comm, ompi_comm_heartbea
 
 static void fd_event_cb(int fd, short flags, void* pdetector)
 {
-    comm_detector_t* detector = pdetector;;
+    comm_detector_t* detector = pdetector;
     double stamp = PMPI_Wtime();
     double lastpeek = detector->hb_lastpeek;
     detector->hb_lastpeek = stamp;
@@ -578,7 +580,7 @@ static void fd_event_cb(int fd, short flags, void* pdetector)
 }
 
 void* fd_progress(opal_object_t* obj) {
-    int ret;
+    int __opal_attribute_unused__ ret;
     MPI_Request req;
     if( OMPI_SUCCESS != ompi_comm_start_detector(&ompi_mpi_comm_world.comm)) {
         OPAL_THREAD_ADD_FETCH32(&fd_thread_active, -1);
@@ -668,19 +670,19 @@ static int fd_heartbeat_send(comm_detector_t* detector) {
     }
     detector->hb_sstamp = now;
     OPAL_OUTPUT_VERBOSE((9, ompi_ftmpi_output_handle,
-                         "%s %s: Sending heartbeat to %d on communicator %3d:%d stamp %g",
-                         OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), __func__, detector->hb_observer, comm->c_contextid, comm->c_epoch, detector->hb_sstamp-startdate ));
+                         "%s %s: Sending heartbeat to %d on communicator %s:%d stamp %g",
+                         OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), __func__, detector->hb_observer, ompi_comm_print_cid(comm), comm->c_epoch, detector->hb_sstamp-startdate ));
 
     if( comm_detector_use_rdma_hb ) return fd_heartbeat_rdma_put(detector);
 
     /* send the heartbeat with eager send */
     ompi_comm_heartbeat_message_t msg;
-    msg.super.cid = comm->c_contextid;
+    msg.super.cid = ompi_comm_get_local_cid(comm);
     msg.super.epoch = comm->c_epoch;
     msg.super.type = comm_heartbeat_recv_cb_type;
     msg.from = detector->hb_rdma_rank; /* comm->c_my_rank; except during finalize when it is equal to detector->hb_observer */
     ompi_proc_t* proc = ompi_comm_peer_lookup(comm, detector->hb_observer);
-    ompi_comm_rbcast_send_msg(proc, &msg.super, sizeof(msg));
+    ompi_comm_rbcast_send_msg(proc, (ompi_comm_rbcast_message_t*)&msg, sizeof(msg));
     return OMPI_SUCCESS;
 }
 
@@ -702,15 +704,15 @@ static int fd_heartbeat_recv_cb(ompi_communicator_t* comm, ompi_comm_heartbeat_m
 
     if( msg->from != detector->hb_observing ) {
         OPAL_OUTPUT_VERBOSE((2, ompi_ftmpi_output_handle,
-                             "%s %s: Received heartbeat from %d on communicator %3d:%d but I am now monitoring %d -- ignored.",
-                             OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), __func__, msg->from, comm->c_contextid, comm->c_epoch, detector->hb_observing ));
+                             "%s %s: Received heartbeat from %d on communicator %s:%d but I am now monitoring %d -- ignored.",
+                             OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), __func__, msg->from, ompi_comm_print_cid(comm), comm->c_epoch, detector->hb_observing ));
     }
     else {
         double stamp = PMPI_Wtime();
         double grace = detector->hb_timeout - (stamp - detector->hb_rstamp);
         OPAL_OUTPUT_VERBOSE((9, ompi_ftmpi_output_handle,
-                             "%s %s: Received heartbeat from %d on communicator %3d:%d at timestamp %g (remained %.1e of %.1e before suspecting)",
-                             OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), __func__, msg->from, comm->c_contextid, comm->c_epoch, stamp-startdate, grace, detector->hb_timeout ));
+                             "%s %s: Received heartbeat from %d on communicator %s:%d at timestamp %g (remained %.1e of %.1e before suspecting)",
+                             OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), __func__, msg->from, ompi_comm_print_cid(comm), comm->c_epoch, stamp-startdate, grace, detector->hb_timeout ));
         detector->hb_rstamp = stamp;
         if( grace < 0.0 ) {
             opal_output_verbose(1, ompi_ftmpi_output_handle,
diff --git a/ompi/communicator/ft/comm_ft_propagator.c b/ompi/communicator/ft/comm_ft_propagator.c
index dbb20275d53..d203f11c7c5 100644
--- a/ompi/communicator/ft/comm_ft_propagator.c
+++ b/ompi/communicator/ft/comm_ft_propagator.c
@@ -2,7 +2,8 @@
  * Copyright (c) 2011-2020 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
- *
+ * Copyright (c) 2021      Triad National Security, LLC. All rights
+ *                         reserved.
  *
  * $COPYRIGHT$
  *
@@ -68,12 +69,12 @@ int ompi_comm_failure_propagate(ompi_communicator_t* comm, ompi_proc_t* proc, in
     if( -1 == comm_failure_propagator_cb_type ) return OMPI_SUCCESS;
 
     OPAL_OUTPUT_VERBOSE((2, ompi_ftmpi_output_handle,
-                         "%s %s: Initiate a propagation for failure of %s (state %d) on communicator %3d:%d",
-                         OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), __func__, OMPI_NAME_PRINT(&proc->super.proc_name), state, comm->c_contextid, comm->c_epoch ));
+                         "%s %s: Initiate a propagation for failure of %s (state %d) on communicator %s:%d",
+                         OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), __func__, OMPI_NAME_PRINT(&proc->super.proc_name), state, ompi_comm_print_cid(comm), comm->c_epoch ));
 
     ompi_comm_failure_propagator_message_t msg;
     /* Broadcast the 'failure_propagator' signal to all other processes. */
-    msg.rbcast_msg.cid   = comm->c_contextid;
+    msg.rbcast_msg.cid   = ompi_comm_get_local_cid(comm);
     msg.rbcast_msg.epoch = comm->c_epoch;
     msg.rbcast_msg.type  = comm_failure_propagator_cb_type;
     msg.proc_name        = proc->super.proc_name;
@@ -90,13 +91,13 @@ static int ompi_comm_failure_propagator_local(ompi_communicator_t* comm, ompi_co
     ompi_proc_t* proc = (ompi_proc_t*)ompi_proc_for_name(msg->proc_name);
     if( !ompi_proc_is_active(proc) ) {
         OPAL_OUTPUT_VERBOSE((9, ompi_ftmpi_output_handle,
-                "%s %s: failure of %s has already been propagated on comm %3d:%d",
-                OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), __func__, OMPI_NAME_PRINT(&msg->proc_name), comm->c_contextid, comm->c_epoch));
+                "%s %s: failure of %s has already been propagated on comm %s:%d",
+                OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), __func__, OMPI_NAME_PRINT(&msg->proc_name), ompi_comm_print_cid(comm), comm->c_epoch));
         return false; /* already propagated, done. */
     }
     OPAL_OUTPUT_VERBOSE((9, ompi_ftmpi_output_handle,
-            "%s %s: failure of %s needs to be propagated on comm %3d:%d",
-            OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), __func__, OMPI_NAME_PRINT(&msg->proc_name), comm->c_contextid, comm->c_epoch));
+            "%s %s: failure of %s needs to be propagated on comm %s:%d",
+            OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), __func__, OMPI_NAME_PRINT(&msg->proc_name), ompi_comm_print_cid(comm), comm->c_epoch));
     ompi_errhandler_proc_failed_internal(proc, msg->proc_state, false);
     return true;
 }
diff --git a/ompi/communicator/ft/comm_ft_reliable_bcast.c b/ompi/communicator/ft/comm_ft_reliable_bcast.c
index 7302737f1bd..1450a2a5924 100644
--- a/ompi/communicator/ft/comm_ft_reliable_bcast.c
+++ b/ompi/communicator/ft/comm_ft_reliable_bcast.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013-2020 The University of Tennessee and The University
+ * Copyright (c) 2013-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  *
@@ -73,6 +73,7 @@ static int ompi_comm_rbcast_bmg(ompi_communicator_t* comm, ompi_comm_rbcast_mess
             proc = ompi_group_peer_lookup(lgrp, idx);
         }
         else {
+            assert(NULL != hgrp); assert(OMPI_COMM_IS_INTER(comm));
             proc = ompi_group_peer_lookup(hgrp, idx-ompi_group_size(lgrp));
         }
         if( ompi_proc_is_active(proc) ) {
@@ -160,17 +161,16 @@ static void ompi_comm_rbcast_bml_recv_cb(
         const mca_btl_base_receive_descriptor_t* descriptor)
 {
     ompi_comm_rbcast_message_t* msg;
-    mca_btl_base_tag_t tag = descriptor->tag;
     ompi_communicator_t* comm;
 
     /* Parse the rbcast fragment */
-    assert( MCA_BTL_TAG_FT_RBCAST == tag );
+    assert( MCA_BTL_TAG_FT_RBCAST == descriptor->tag );
     assert( 1 == descriptor->des_segment_count );
     assert( sizeof(ompi_comm_rbcast_message_t) <= descriptor->des_segments->seg_len );
     msg = (ompi_comm_rbcast_message_t*) descriptor->des_segments->seg_addr.pval;
 
     OPAL_OUTPUT_VERBOSE((10, ompi_ftmpi_output_handle,
-                         "%s %s: Info: Recieved rbcast tag for communicator with CID %3d:%d",
+                         "%s %s: Info: Received rbcast tag for communicator with CID %3d:%d",
                          OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), __func__, msg->cid, msg->epoch));
 
     /* Find the target comm */
@@ -181,7 +181,7 @@ static void ompi_comm_rbcast_bml_recv_cb(
                              OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), __func__, msg->cid, msg->epoch));
         return;
     }
-    if(OPAL_UNLIKELY( msg->cid != comm->c_contextid )) {
+    if(OPAL_UNLIKELY( msg->cid != ompi_comm_get_local_cid(comm))) {
         OPAL_OUTPUT_VERBOSE((2, ompi_ftmpi_output_handle,
                              "%s %s: Info: received a late rbcast message with CID %3d:%d during an MPI_COMM_DUP that is trying to reuse that CID (thus increasing the epoch) - ignoring, nothing to do",
                              OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), __func__, msg->cid, msg->epoch));
@@ -190,13 +190,13 @@ static void ompi_comm_rbcast_bml_recv_cb(
     /* Check if this is a delayed rbcast for an old communicator whose CID has been reused */
     if(OPAL_UNLIKELY( comm->c_epoch != msg->epoch )) {
         OPAL_OUTPUT_VERBOSE((2, ompi_ftmpi_output_handle,
-                             "%s %s: Info: Received a late rbcast order for the communicator with CID %3d:%d when is is now at epoch %d - ignoring, nothing to do",
+                             "%s %s: Info: Received a late rbcast order for the communicator with CID %3d:%d when it is now at epoch %d - ignoring, nothing to do",
                              OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), __func__, msg->cid, msg->epoch, comm->c_epoch));
         return;
     }
 
     /* invoke the local registered callback for the type */
-    assert( 0 <= msg->type && RBCAST_CB_TYPE_MAX >= msg->type );
+    assert( RBCAST_CB_TYPE_MAX >= msg->type );
     if( NULL != ompi_comm_rbcast_cb[msg->type] ) {
         if( ompi_comm_rbcast_cb[msg->type](comm, msg) ) {
             /* forward the rbcast */
@@ -204,7 +204,7 @@ static void ompi_comm_rbcast_bml_recv_cb(
         }
     }
     else {
-        /* During finalize, we loosly synchronize so it may happen 
+        /* During finalize, we loosely synchronize so it may happen 
          * that we keep receiving messages after we deregistered the type.
          * Any other time, this is indicative of a problem.
          */
diff --git a/ompi/communicator/ft/comm_ft_revoke.c b/ompi/communicator/ft/comm_ft_revoke.c
index 027e659814a..0e4c3158afa 100644
--- a/ompi/communicator/ft/comm_ft_revoke.c
+++ b/ompi/communicator/ft/comm_ft_revoke.c
@@ -3,6 +3,8 @@
  * Copyright (c) 2011-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2021      Triad National Security, LLC. All rights
+ *                         reserved.
  *
  *
  * $COPYRIGHT$
@@ -49,14 +51,14 @@ int ompi_comm_revoke_internal(ompi_communicator_t* comm)
     int ret = OMPI_SUCCESS;;
 
     OPAL_OUTPUT_VERBOSE((1, ompi_ftmpi_output_handle,
-                         "%s %s: Initiate a revoke on communicator %3d:%d",
-                         OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), __func__, comm->c_contextid, comm->c_epoch ));
+                         "%s %s: Initiate a revoke on communicator %s:%d",
+                         OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), __func__, ompi_comm_print_cid(comm), comm->c_epoch ));
 
     /* Mark locally revoked */
     if( ompi_comm_revoke_local(comm, NULL) ) {
         /* Broadcast the 'revoke' signal to all other processes. */
         ompi_comm_rbcast_message_t msg;
-        msg.cid   = comm->c_contextid;
+        msg.cid   = ompi_comm_get_local_cid(comm);
         msg.epoch = comm->c_epoch;
         msg.type  = comm_revoke_cb_type;
         ret = ompi_comm_rbcast(comm, &msg, sizeof(msg));
@@ -71,13 +73,13 @@ static int ompi_comm_revoke_local(ompi_communicator_t* comm, ompi_comm_rbcast_me
 {
     if( comm->comm_revoked ) {
         OPAL_OUTPUT_VERBOSE((9, ompi_ftmpi_output_handle,
-                             "%s %s: comm %3d:%d is already revoked, nothing to do",
-                             OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), __func__, comm->c_contextid, comm->c_epoch));
+                             "%s %s: comm %s:%d is already revoked, nothing to do",
+                             OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), __func__, ompi_comm_print_cid(comm), comm->c_epoch));
         return false;
     }
     OPAL_OUTPUT_VERBOSE((9, ompi_ftmpi_output_handle,
-                         "%s %s: comm %3d:%d is marked revoked locally",
-                         OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), __func__, comm->c_contextid, comm->c_epoch));
+                         "%s %s: comm %s:%d is marked revoked locally",
+                         OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), __func__, ompi_comm_print_cid(comm), comm->c_epoch));
     /*
      * Locally revoke the communicator
      *
diff --git a/ompi/communicator/help-comm.txt b/ompi/communicator/help-comm.txt
new file mode 100644
index 00000000000..736718f76e8
--- /dev/null
+++ b/ompi/communicator/help-comm.txt
@@ -0,0 +1,48 @@
+# -*- text -*-
+#
+# Copyright (c) 2006      High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2006-2015 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2018 IBM Corporation. All rights reserved.
+# Copyright (c) 2020      The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+# This is the US/English general help file for Open MPI.
+#
+[MPI function not supported]
+Your application has invoked an MPI function that is not supported in
+this environment.
+
+  MPI function: %s
+  Reason:       %s
+[info-set-with-reserved-prefix]
+Comments
+MPI_Info_set warning, key is using a reserved prefix.
+  Key:             %s
+  Reserved prefix: %s
+[mismatched-split_type-values]
+Detected mismatched split_type and/or info "mpi_hw_resource_type" values
+in a call to MPI_Comm_split_type between peers in the communicator.
+  split_type:  %s (%d)
+  info:        %s
+[unexpected-split-type]
+Detected an unexpected split type in a call to MPI_Comm_split_type.
+  split_type:  %s (%d)
+[disjointness-set-again]
+Communicator disjointness should only be set once at initialization.
+Attempts to modify the state are illegal and shall be ignored.
+[cid-base-not-set]
+An internal error has occurred when creating a new communicator.  This
+should probably never happen:
+
+* cid_base is not set.  Either nresults
+  is zero or no returned info matches PMIX_GROUP_CONTEXT_ID.
+
+Please contact the Open MPI developer community with this message and
+a small example code that reproduces the issue.
diff --git a/ompi/contrib/README.md b/ompi/contrib/README.md
deleted file mode 100644
index bcaae6d43d9..00000000000
--- a/ompi/contrib/README.md
+++ /dev/null
@@ -1,19 +0,0 @@
-This is the OMPI contrib system.  It is (far) less functional and
-flexible than the OMPI MCA framework/component system.
-
-Each contrib package must have a `configure.m4`.  It may optionally also
-have an `autogen.subdirs` file.
-
-If it has a `configure.m4` file, it must specify its own relevant
-files to `AC_CONFIG_FILES` to create during `AC_OUTPUT` -- just like
-MCA components (at a minimum, usually its own `Makefile`).  The
-`configure.m4` file will be slurped up into the main `configure`
-script, just like other MCA components.  Note that there is currently
-no "no configure" option for contrib packages -- you *must* have a
-`configure.m4` (even if all it does it call `$1`).  Feel free to fix
-this situation if you want -- it probably won't not be too difficult
-to extend `autogen.pl` to support this scenario, similar to how it is
-done for MCA components.  :smile:
-
-If it has an `autogen.subdirs` file, then it needs to be a
-subdirectory that is autogen-able.
diff --git a/ompi/contrib/libompitrace/Makefile.am b/ompi/contrib/libompitrace/Makefile.am
deleted file mode 100644
index be35171d42a..00000000000
--- a/ompi/contrib/libompitrace/Makefile.am
+++ /dev/null
@@ -1,46 +0,0 @@
-# -*- makefile -*-
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2009-2016 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2016      IBM Corporation.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-lib_LTLIBRARIES = libompitrace.la
-
-libompitrace_la_SOURCES = \
-        abort.c \
-        accumulate.c \
-        add_error_class.c \
-        add_error_code.c \
-        add_error_string.c \
-        allgather.c \
-        allgatherv.c \
-        alloc_mem.c \
-        allreduce.c \
-        barrier.c \
-        bcast.c \
-	get_address.c \
-        finalize.c \
-        init.c \
-        isend.c \
-        recv.c \
-        reduce.c \
-        request_free.c \
-        send.c \
-        sendrecv.c
-
-libompitrace_la_LDFLAGS = -version-info $(libompitrace_so_version)
diff --git a/ompi/contrib/libompitrace/abort.c b/ompi/contrib/libompitrace/abort.c
deleted file mode 100644
index 246e4bea335..00000000000
--- a/ompi/contrib/libompitrace/abort.c
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007-2009 Cisco Systems, Inc.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-
-int MPI_Abort(MPI_Comm comm, int errorcode)
-{
-    char commname[MPI_MAX_OBJECT_NAME];
-    int len;
-    int rank;
-
-    PMPI_Comm_rank(MPI_COMM_WORLD, &rank);
-    PMPI_Comm_get_name(comm, commname, &len);
-
-    fprintf(stderr, "MPI_ABORT[%d]: comm %s errorcode %d\n", rank, commname, errorcode);
-    fflush(stderr);
-
-    return PMPI_Abort(comm, errorcode);
-}
diff --git a/ompi/contrib/libompitrace/accumulate.c b/ompi/contrib/libompitrace/accumulate.c
deleted file mode 100644
index 1cee619c6fd..00000000000
--- a/ompi/contrib/libompitrace/accumulate.c
+++ /dev/null
@@ -1,55 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2009      Sun Microsystmes, Inc.  All rights reserved.
- * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "opal_stdint.h"
-#include "ompi/op/op.h"
-#include "ompi/mpi/c/bindings.h"
-
-int MPI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype,
-                   int target_rank, MPI_Aint target_disp, int target_count,
-                   MPI_Datatype target_datatype, MPI_Op op, MPI_Win win)
-{
-
-    char typename[MPI_MAX_OBJECT_NAME], target_dt[MPI_MAX_OBJECT_NAME];
-    char winname[MPI_MAX_OBJECT_NAME];
-    int len;
-    int rank;
-
-    PMPI_Comm_rank(MPI_COMM_WORLD, &rank);
-    PMPI_Type_get_name(origin_datatype, typename, &len);
-    PMPI_Type_get_name(target_datatype, target_dt, &len);
-    PMPI_Win_get_name(win, winname, &len);
-
-    fprintf(stderr, "MPI_ACCUMULATE[%d]: origin_addr %0" PRIxPTR " origin_count %d origin_datatype %s\n"
-            "\ttarget_rank %d target_disp %" PRIdPTR " target_count %d target_datatype %s op %s win %s\n",
-            rank, (uintptr_t)origin_addr, origin_count, typename, target_rank, (intptr_t) target_disp,
-            target_count, target_dt, op->o_name, winname);
-    fflush(stderr);
-
-    return PMPI_Accumulate(origin_addr, origin_count, origin_datatype,
-                           target_rank, target_disp, target_count,
-                           target_datatype, op, win);
-}
diff --git a/ompi/contrib/libompitrace/add_error_class.c b/ompi/contrib/libompitrace/add_error_class.c
deleted file mode 100644
index 721aa3b1fd8..00000000000
--- a/ompi/contrib/libompitrace/add_error_class.c
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2006 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      University of Houston. All rights reserved.
- * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "opal_stdint.h"
-#include "ompi/mpi/c/bindings.h"
-
-int MPI_Add_error_class(int *errorclass)
-{
-    int rank;
-
-    PMPI_Comm_rank(MPI_COMM_WORLD, &rank);
-
-    fprintf(stderr, "MPI_ADD_ERROR_CLASS[%d]: errorclass %0" PRIxPTR "\n", rank, (uintptr_t)errorclass);
-    fflush(stderr);
-
-    return PMPI_Add_error_class(errorclass);
-}
-
diff --git a/ompi/contrib/libompitrace/add_error_code.c b/ompi/contrib/libompitrace/add_error_code.c
deleted file mode 100644
index f1465ab3ba8..00000000000
--- a/ompi/contrib/libompitrace/add_error_code.c
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      University of Houston. All rights reserved.
- * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "opal_stdint.h"
-#include "ompi/mpi/c/bindings.h"
-
-int MPI_Add_error_code(int errorclass, int *errorcode)
-{
-    int rank;
-
-    PMPI_Comm_rank(MPI_COMM_WORLD, &rank);
-
-    fprintf(stderr, "MPI_ADD_ERROR_CODE[%d]: errorclass %d errcode %0" PRIxPTR "\n", rank, errorclass, (uintptr_t)errorcode);
-    fflush(stderr);
-
-    return PMPI_Add_error_code(errorclass, errorcode);
-}
diff --git a/ompi/contrib/libompitrace/add_error_string.c b/ompi/contrib/libompitrace/add_error_string.c
deleted file mode 100644
index 499607dd9f9..00000000000
--- a/ompi/contrib/libompitrace/add_error_string.c
+++ /dev/null
@@ -1,39 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      University of Houston. All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-
-int MPI_Add_error_string(int errorcode, const char *string)
-{
-    int rank;
-
-    PMPI_Comm_rank(MPI_COMM_WORLD, &rank);
-
-    fprintf(stderr, "MPI_ADD_ERROR_STRING[%d]: errorcode %d string %s\n",
-            rank, errorcode, string);
-    fflush(stderr);
-
-    return PMPI_Add_error_string(errorcode, string);
-}
diff --git a/ompi/contrib/libompitrace/allgather.c b/ompi/contrib/libompitrace/allgather.c
deleted file mode 100644
index 400464742b0..00000000000
--- a/ompi/contrib/libompitrace/allgather.c
+++ /dev/null
@@ -1,51 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007-2009 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "opal_stdint.h"
-
-#include "ompi/mpi/c/bindings.h"
-
-int MPI_Allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                  void *recvbuf, int recvcount, MPI_Datatype recvtype,
-                  MPI_Comm comm)
-{
-    char sendtypename[MPI_MAX_OBJECT_NAME], recvtypename[MPI_MAX_OBJECT_NAME];
-    char commname[MPI_MAX_OBJECT_NAME];
-    int len;
-    int rank;
-
-    PMPI_Comm_rank(MPI_COMM_WORLD, &rank);
-    PMPI_Type_get_name(sendtype, sendtypename, &len);
-    PMPI_Type_get_name(recvtype, recvtypename, &len);
-    PMPI_Comm_get_name(comm, commname, &len);
-
-    fprintf(stderr, "MPI_ALLGATHER[%d]: sendbuf %0" PRIxPTR " sendcount %d sendtype %s\n\trecvbuf %0" PRIxPTR " recvcount %d recvtype %s comm %s\n",
-           rank, (uintptr_t) sendbuf, sendcount, sendtypename, (uintptr_t) recvbuf, recvcount, recvtypename, commname);
-    fflush(stderr);
-
-    return PMPI_Allgather(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm);
-}
-
diff --git a/ompi/contrib/libompitrace/allgatherv.c b/ompi/contrib/libompitrace/allgatherv.c
deleted file mode 100644
index 4dbff470479..00000000000
--- a/ompi/contrib/libompitrace/allgatherv.c
+++ /dev/null
@@ -1,51 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "opal_stdint.h"
-
-#include "ompi/mpi/c/bindings.h"
-
-int MPI_Allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                   void *recvbuf, const int recvcounts[], const int displs[],
-                   MPI_Datatype recvtype, MPI_Comm comm)
-{
-    char sendtypename[MPI_MAX_OBJECT_NAME], recvtypename[MPI_MAX_OBJECT_NAME];
-    char commname[MPI_MAX_OBJECT_NAME];
-    int len;
-    int rank;
-
-    PMPI_Comm_rank(MPI_COMM_WORLD, &rank);
-    PMPI_Type_get_name(sendtype, sendtypename, &len);
-    PMPI_Type_get_name(recvtype, recvtypename, &len);
-    PMPI_Comm_get_name(comm, commname, &len);
-
-    fprintf(stderr, "MPI_ALLGATHERV[%d]: sendbuf %0" PRIxPTR " sendcount %d sendtype %s\n\trecvbuf %0" PRIxPTR " recvtype %s comm %s\n",
-           rank, (uintptr_t) sendbuf, sendcount, sendtypename, (uintptr_t) recvbuf, recvtypename, commname);
-    fflush(stderr);
-
-    return PMPI_Allgatherv(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, comm);
-}
-
diff --git a/ompi/contrib/libompitrace/alloc_mem.c b/ompi/contrib/libompitrace/alloc_mem.c
deleted file mode 100644
index 59c0de39a32..00000000000
--- a/ompi/contrib/libompitrace/alloc_mem.c
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-
-int MPI_Alloc_mem(MPI_Aint size, MPI_Info info, void *baseptr)
-{
-
-    int rank;
-
-    PMPI_Comm_rank(MPI_COMM_WORLD, &rank);
-
-    fprintf(stderr, "MPI_Alloc_mem[%d]: size %0ld\n", rank, (long)size);
-    fflush(stderr);
-
-    return PMPI_Alloc_mem(size, info, baseptr);
-}
-
diff --git a/ompi/contrib/libompitrace/allreduce.c b/ompi/contrib/libompitrace/allreduce.c
deleted file mode 100644
index 5382f312b3e..00000000000
--- a/ompi/contrib/libompitrace/allreduce.c
+++ /dev/null
@@ -1,47 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "opal_stdint.h"
-#include "ompi/op/op.h"
-#include "ompi/mpi/c/bindings.h"
-
-int MPI_Allreduce(const void *sendbuf, void *recvbuf, int count,
-                  MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
-{
-    char typename[MPI_MAX_OBJECT_NAME], commname[MPI_MAX_OBJECT_NAME];
-    int len;
-    int rank;
-
-    PMPI_Comm_rank(MPI_COMM_WORLD, &rank);
-    PMPI_Type_get_name(datatype, typename, &len);
-    PMPI_Comm_get_name(comm, commname, &len);
-
-    fprintf(stderr, "MPI_ALLREDUCE[%d]: sendbuf %0" PRIxPTR " recvbuf %0" PRIxPTR " count %d datatype %s op %s comm %s\n",
-           rank, (uintptr_t)sendbuf, (uintptr_t)recvbuf, count, typename, op->o_name, commname);
-    fflush(stderr);
-
-    return PMPI_Allreduce(sendbuf, recvbuf, count, datatype, op, comm);
-}
diff --git a/ompi/contrib/libompitrace/barrier.c b/ompi/contrib/libompitrace/barrier.c
deleted file mode 100644
index 0a1c58ed179..00000000000
--- a/ompi/contrib/libompitrace/barrier.c
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-
-int MPI_Barrier(MPI_Comm comm)
-{
-    char commname[MPI_MAX_OBJECT_NAME];
-    int len;
-    int rank;
-
-    PMPI_Comm_rank(MPI_COMM_WORLD, &rank);
-    PMPI_Comm_get_name(comm, commname, &len);
-
-    fprintf(stderr, "MPI_BARRIER[%d]: comm %s\n", rank, commname);
-    fflush(stderr);
-
-    return PMPI_Barrier(comm);
-}
diff --git a/ompi/contrib/libompitrace/bcast.c b/ompi/contrib/libompitrace/bcast.c
deleted file mode 100644
index ef6bcdf63a5..00000000000
--- a/ompi/contrib/libompitrace/bcast.c
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "opal_stdint.h"
-
-#include "ompi/mpi/c/bindings.h"
-
-
-int MPI_Bcast(void *buffer, int count, MPI_Datatype datatype,
-              int root, MPI_Comm comm)
-{
-    char typename[MPI_MAX_OBJECT_NAME], commname[MPI_MAX_OBJECT_NAME];
-    int len;
-    int rank;
-
-    PMPI_Comm_rank(MPI_COMM_WORLD, &rank);
-    PMPI_Type_get_name(datatype, typename, &len);
-    PMPI_Comm_get_name(comm, commname, &len);
-
-    fprintf(stderr, "MPI_BCAST[%d]: buffer %0" PRIxPTR " count %d datatype %s root %d comm %s\n",
-           rank, (uintptr_t) buffer, count, typename, root, commname);
-    fflush(stderr);
-
-    return PMPI_Bcast(buffer, count, datatype, root, comm);
-}
diff --git a/ompi/contrib/libompitrace/configure.m4 b/ompi/contrib/libompitrace/configure.m4
deleted file mode 100644
index a55b6590785..00000000000
--- a/ompi/contrib/libompitrace/configure.m4
+++ /dev/null
@@ -1,27 +0,0 @@
-# -*- shell-script -*-
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2007-2010 Cisco Systems, Inc.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-# OMPI_contrib_libompitrace_CONFIG([action-if-can-compile],
-#                                  [action-if-cant-compile])
-# ------------------------------------------------
-AC_DEFUN([OMPI_contrib_libompitrace_CONFIG],[
-    AC_CONFIG_FILES([ompi/contrib/libompitrace/Makefile])
-    $1
-])dnl
diff --git a/ompi/contrib/libompitrace/finalize.c b/ompi/contrib/libompitrace/finalize.c
deleted file mode 100644
index 30b4df110cd..00000000000
--- a/ompi/contrib/libompitrace/finalize.c
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-
-int MPI_Finalize(void)
-{
-    int rank;
-
-    PMPI_Comm_rank(MPI_COMM_WORLD, &rank);
-
-    fprintf(stderr, "MPI_FINALIZE[%d]\n", rank);
-    fflush(stderr);
-
-    return PMPI_Finalize();
-}
diff --git a/ompi/contrib/libompitrace/get_address.c b/ompi/contrib/libompitrace/get_address.c
deleted file mode 100644
index bef3d347967..00000000000
--- a/ompi/contrib/libompitrace/get_address.c
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2018      Los Alamos National Security, LLC. All
- *                         rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "opal_stdint.h"
-#include "ompi/mpi/c/bindings.h"
-
-int MPI_Get_address(const void *location, MPI_Aint *address)
-{
-
-    int rank;
-
-    PMPI_Comm_rank(MPI_COMM_WORLD, &rank);
-
-    fprintf(stderr, "MPI_GET_ADDRESS[%d]: location %0" PRIxPTR " address %0" PRIxPTR "\n",
-            rank, (uintptr_t)location, (uintptr_t)address);
-    fflush(stderr);
-
-    return PMPI_Get_address(location, address);
-}
diff --git a/ompi/contrib/libompitrace/init.c b/ompi/contrib/libompitrace/init.c
deleted file mode 100644
index a92f384048c..00000000000
--- a/ompi/contrib/libompitrace/init.c
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2006 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007-2009 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2007-2008 Sun Microsystems, Inc.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-
-int MPI_Init(int *argc, char ***argv)
-{
-    fprintf(stderr, "MPI_INIT: argc %d\n", (0 < *argc) ? *argc : 0);
-    fflush(stderr);
-
-    return PMPI_Init(argc, argv);
-}
diff --git a/ompi/contrib/libompitrace/isend.c b/ompi/contrib/libompitrace/isend.c
deleted file mode 100644
index 0ef6e508294..00000000000
--- a/ompi/contrib/libompitrace/isend.c
+++ /dev/null
@@ -1,47 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2009 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "opal_stdint.h"
-
-#include "ompi/mpi/c/bindings.h"
-
-int MPI_Isend(const void *buf, int count, MPI_Datatype type, int dest,
-              int tag, MPI_Comm comm, MPI_Request *request)
-{
-    char typename[MPI_MAX_OBJECT_NAME], commname[MPI_MAX_OBJECT_NAME];
-    int len;
-    int rank;
-
-    PMPI_Comm_rank(MPI_COMM_WORLD, &rank);
-    PMPI_Type_get_name(type, typename, &len);
-    PMPI_Comm_get_name(comm, commname, &len);
-
-    fprintf(stderr, "MPI_ISEND[%d]: buf %0" PRIxPTR " count %d datatype %s dest %d tag %d comm %s\n",
-           rank, (uintptr_t) buf, count, typename, dest, tag, commname);
-    fflush(stderr);
-
-    return PMPI_Isend(buf, count, type, dest, tag, comm, request);
-}
diff --git a/ompi/contrib/libompitrace/recv.c b/ompi/contrib/libompitrace/recv.c
deleted file mode 100644
index 0fc5a9665c7..00000000000
--- a/ompi/contrib/libompitrace/recv.c
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "opal_stdint.h"
-
-#include "ompi/mpi/c/bindings.h"
-
-int MPI_Recv(void *buf, int count, MPI_Datatype type, int source,
-             int tag, MPI_Comm comm, MPI_Status *status)
-{
-    char typename[MPI_MAX_OBJECT_NAME], commname[MPI_MAX_OBJECT_NAME];
-    int len;
-    int rank;
-
-    PMPI_Comm_rank(MPI_COMM_WORLD, &rank);
-    PMPI_Type_get_name(type, typename, &len);
-    PMPI_Comm_get_name(comm, commname, &len);
-
-    fprintf(stderr, "MPI_RECV[%d]: buf %0" PRIxPTR " count %d datatype %s source %d tag %d comm %s\n",
-           rank, (uintptr_t) buf, count, typename, source, tag, commname);
-    fflush(stderr);
-
-    return PMPI_Recv(buf, count, type, source, tag, comm, status);
-}
diff --git a/ompi/contrib/libompitrace/reduce.c b/ompi/contrib/libompitrace/reduce.c
deleted file mode 100644
index 24fb60f7b85..00000000000
--- a/ompi/contrib/libompitrace/reduce.c
+++ /dev/null
@@ -1,48 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2009 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "opal_stdint.h"
-
-#include "ompi/op/op.h"
-#include "ompi/mpi/c/bindings.h"
-
-int MPI_Reduce(const void *sendbuf, void *recvbuf, int count,
-               MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm)
-{
-    char typename[MPI_MAX_OBJECT_NAME], commname[MPI_MAX_OBJECT_NAME];
-    int len;
-    int rank;
-
-    PMPI_Comm_rank(MPI_COMM_WORLD, &rank);
-    PMPI_Type_get_name(datatype, typename, &len);
-    PMPI_Comm_get_name(comm, commname, &len);
-
-    fprintf(stderr,"MPI_REDUCE[%d]: sendbuf %0" PRIxPTR " recvbuf %0" PRIxPTR " count %d datatype %s op %s root %d comm %s\n",
-           rank, (uintptr_t) sendbuf, (uintptr_t) recvbuf, count, typename, op->o_name, root, commname);
-    fflush(stderr);
-
-    return PMPI_Reduce(sendbuf, recvbuf, count, datatype, op, root, comm);
-}
diff --git a/ompi/contrib/libompitrace/request_free.c b/ompi/contrib/libompitrace/request_free.c
deleted file mode 100644
index cc740582e8e..00000000000
--- a/ompi/contrib/libompitrace/request_free.c
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2009 Cisco Systems, Inc.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-
-int MPI_Request_free(MPI_Request *request)
-{
-    int rank;
-
-    PMPI_Comm_rank(MPI_COMM_WORLD, &rank);
-
-    fprintf(stderr, "MPI_REQUEST_FREE[%d]\n", rank);
-    fflush(stderr);
-
-    return PMPI_Request_free(request);
-}
diff --git a/ompi/contrib/libompitrace/send.c b/ompi/contrib/libompitrace/send.c
deleted file mode 100644
index b9eddda57f4..00000000000
--- a/ompi/contrib/libompitrace/send.c
+++ /dev/null
@@ -1,47 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "opal_stdint.h"
-
-#include "ompi/mpi/c/bindings.h"
-
-int MPI_Send(const void *buf, int count, MPI_Datatype type, int dest,
-             int tag, MPI_Comm comm)
-{
-    char typename[MPI_MAX_OBJECT_NAME], commname[MPI_MAX_OBJECT_NAME];
-    int len;
-    int rank;
-
-    PMPI_Comm_rank(MPI_COMM_WORLD, &rank);
-    PMPI_Type_get_name(type, typename, &len);
-    PMPI_Comm_get_name(comm, commname, &len);
-
-    fprintf(stderr, "MPI_SEND[%d]: : buf %0" PRIxPTR " count %d datatype %s dest %d tag %d comm %s\n",
-           rank, (uintptr_t) buf, count, typename, dest, tag, commname);
-    fflush(stderr);
-
-    return PMPI_Send(buf, count, type, dest, tag, comm);
-}
diff --git a/ompi/contrib/libompitrace/sendrecv.c b/ompi/contrib/libompitrace/sendrecv.c
deleted file mode 100644
index 5fd84cbc11b..00000000000
--- a/ompi/contrib/libompitrace/sendrecv.c
+++ /dev/null
@@ -1,60 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <stdio.h>
-#include <string.h>
-
-#include "opal_stdint.h"
-
-#include "ompi/mpi/c/bindings.h"
-
-int MPI_Sendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                 int dest, int sendtag, void *recvbuf, int recvcount,
-                 MPI_Datatype recvtype, int source, int recvtag,
-                 MPI_Comm comm,  MPI_Status *status)
-{
-    char sendtypename[MPI_MAX_OBJECT_NAME], recvtypename[MPI_MAX_OBJECT_NAME];
-    char commname[MPI_MAX_OBJECT_NAME];
-    int len;
-    int rank;
-    int size;
-
-    PMPI_Comm_rank(MPI_COMM_WORLD, &rank);
-    PMPI_Type_get_name(sendtype, sendtypename, &len);
-    PMPI_Type_get_name(sendtype, recvtypename, &len);
-    PMPI_Comm_get_name(comm, commname, &len);
-    PMPI_Type_size(recvtype, &size);
-
-    fprintf(stderr, "MPI_SENDRECV[%d]: sendbuf %0" PRIxPTR " sendcount %d sendtype %s dest %d sendtag %d\n\t"
-           "recvbuf %0" PRIxPTR " recvcount %d recvtype %s source %d recvtag %d comm %s\n",
-            rank, (uintptr_t) sendbuf, sendcount, sendtypename, dest, sendtag,
-            (uintptr_t) recvbuf, recvcount, recvtypename, source, recvtag, commname);
-    fflush(stderr);
-
-    memset(recvbuf, 0, recvcount*size);
-
-    return PMPI_Sendrecv(sendbuf, sendcount, sendtype, dest, sendtag,
-                         recvbuf, recvcount, recvtype, source, recvtag,
-                         comm, status);
-}
diff --git a/ompi/datatype/ompi_datatype.h b/ompi/datatype/ompi_datatype.h
index e2ee2a79c01..87de1e30aca 100644
--- a/ompi/datatype/ompi_datatype.h
+++ b/ompi/datatype/ompi_datatype.h
@@ -11,6 +11,7 @@
  *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
  * Copyright (c) 2021      IBM Corporation. All rights reserved.
+ * Copyright (c) 2025      Triad National Security, LLC. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -59,7 +60,7 @@ BEGIN_C_DECLS
 #define OMPI_DATATYPE_FLAG_DATA_FORTRAN  0xC000
 #define OMPI_DATATYPE_FLAG_DATA_LANGUAGE 0xC000
 
-#define OMPI_DATATYPE_MAX_PREDEFINED 50
+#define OMPI_DATATYPE_MAX_PREDEFINED 52
 
 #if OMPI_DATATYPE_MAX_PREDEFINED > OPAL_DATATYPE_MAX_SUPPORTED
 #error Need to increase the number of supported dataypes by OPAL (value OPAL_DATATYPE_MAX_SUPPORTED).
@@ -91,7 +92,7 @@ typedef struct ompi_datatype_t ompi_datatype_t;
 OMPI_DECLSPEC OBJ_CLASS_DECLARATION(ompi_datatype_t);
 
 /**
- * Padded struct to maintain back compatibiltiy.
+ * Padded struct to maintain back compatibility.
  * See opal/communicator/communicator.h comments with struct opal_communicator_t
  * for full explanation why we chose the following padding construct for predefines.
  */
@@ -118,7 +119,6 @@ OMPI_DECLSPEC extern opal_convertor_t* ompi_mpi_local_convertor;
 extern struct opal_pointer_array_t ompi_datatype_f_to_c_table;
 
 OMPI_DECLSPEC int32_t ompi_datatype_init( void );
-OMPI_DECLSPEC int32_t ompi_datatype_finalize( void );
 
 OMPI_DECLSPEC int32_t ompi_datatype_default_convertors_init( void );
 OMPI_DECLSPEC int32_t ompi_datatype_default_convertors_fini( void );
@@ -378,15 +378,15 @@ OMPI_DECLSPEC int ompi_datatype_safeguard_pointer_debug_breakpoint( const void*
                                                                     int count );
 #endif  /* OPAL_ENABLE_DEBUG */
 
-OMPI_DECLSPEC int ompi_datatype_pack_external( const char datarep[], const void *inbuf, int incount,
+OMPI_DECLSPEC int ompi_datatype_pack_external( const char datarep[], const void *inbuf, size_t incount,
                                                ompi_datatype_t *datatype, void *outbuf,
                                                MPI_Aint outsize, MPI_Aint *position);
 
 OMPI_DECLSPEC int ompi_datatype_unpack_external( const char datarep[], const void *inbuf, MPI_Aint insize,
-                                                 MPI_Aint *position, void *outbuf, int outcount,
+                                                 MPI_Aint *position, void *outbuf, size_t outcount,
                                                  ompi_datatype_t *datatype);
 
-OMPI_DECLSPEC int ompi_datatype_pack_external_size( const char datarep[], int incount,
+OMPI_DECLSPEC int ompi_datatype_pack_external_size( const char datarep[], size_t incount,
                                                     ompi_datatype_t *datatype, MPI_Aint *size);
 
 #define OMPI_DATATYPE_RETAIN(ddt)                                       \
@@ -424,7 +424,7 @@ OMPI_DECLSPEC int ompi_datatype_pack_external_size( const char datarep[], int in
  * converted to (1,larger_datatype).  This comes up in pack/unpack if
  * the datatype is [int4b,empty4b] for example.  With that datatype the
  * (count,datatype) path has to loop over the count processing each
- * occurrance of the datatype, but a larger type created via
+ * occurrence of the datatype, but a larger type created via
  * MPI_Type_contiguous(count,datatype,) will have a single description
  * entry describing the whole vector and go through pack/unpack much
  * faster.
diff --git a/ompi/datatype/ompi_datatype_args.c b/ompi/datatype/ompi_datatype_args.c
index 2063839ef33..22e3c3f51f2 100644
--- a/ompi/datatype/ompi_datatype_args.c
+++ b/ompi/datatype/ompi_datatype_args.c
@@ -71,57 +71,45 @@ typedef struct __dt_args {
 #define OMPI_DATATYPE_ALIGN_PTR(PTR, TYPE)
 #endif  /* OPAL_ALIGN_WORD_SIZE_INTEGERS */
 
-/**
- * Some architectures require 64 bits pointers (to pointers) to
- * be 64 bits aligned. As in the ompi_datatype_args_t structure we have
- * 2 such array of pointers and one to an array of ints, if we start by
- * setting the 64 bits aligned one we will not have any trouble. Problem
- * originally reported on SPARC 64.
- */
-#define ALLOC_ARGS(PDATA, IC, AC, DC)                                   \
-    do {                                                                \
-        int length = sizeof(ompi_datatype_args_t) + (IC) * sizeof(int) + \
-            (AC) * sizeof(ptrdiff_t) + (DC) * sizeof(MPI_Datatype); \
-        char* buf = (char*)malloc( length );                            \
-        ompi_datatype_args_t* pArgs = (ompi_datatype_args_t*)buf;       \
-        pArgs->ci = (IC);                                               \
-        pArgs->ca = (AC);                                               \
-        pArgs->cd = (DC);                                               \
-        buf += sizeof(ompi_datatype_args_t);                            \
-        if( pArgs->ca == 0 ) pArgs->a = NULL;                           \
-        else {                                                          \
-            pArgs->a = (ptrdiff_t*)buf;                         \
-            buf += pArgs->ca * sizeof(ptrdiff_t);               \
-        }                                                               \
-        if( pArgs->cd == 0 ) pArgs->d = NULL;                           \
-        else {                                                          \
-            pArgs->d = (ompi_datatype_t**)buf;                          \
-            buf += pArgs->cd * sizeof(MPI_Datatype);                    \
-        }                                                               \
-        if( pArgs->ci == 0 ) pArgs->i = NULL;                           \
-        else pArgs->i = (int*)buf;                                      \
-        pArgs->ref_count = 1;                                           \
-        pArgs->total_pack_size = (4 + (IC) + (DC)) * sizeof(int) +      \
-            (AC) * sizeof(ptrdiff_t);                                   \
-        (PDATA)->args = (void*)pArgs;                                   \
-        (PDATA)->packed_description = 0;                                \
-    } while(0)
-
-
 int32_t ompi_datatype_set_args( ompi_datatype_t* pData,
                                 int32_t ci, const int32_t** i,
                                 int32_t ca, const ptrdiff_t* a,
                                 int32_t cd, ompi_datatype_t* const * d, int32_t type)
 {
     int pos;
-    ompi_datatype_args_t* pArgs;
 
     assert( NULL == pData->args );
-    ALLOC_ARGS( pData, ci, ca, cd );
-
-    pArgs = (ompi_datatype_args_t*)pData->args;
+    int length = sizeof(ompi_datatype_args_t) + ci * sizeof(int) +
+                 ca * sizeof(ptrdiff_t) + cd * sizeof(MPI_Datatype);
+    char* buf = (char*)malloc( length );
+    ompi_datatype_args_t* pArgs = (ompi_datatype_args_t*)buf;
+    pArgs->ci = ci; pArgs->i = NULL;
+    pArgs->ca = ca; pArgs->a = NULL;
+    pArgs->cd = cd; pArgs->d = NULL;
     pArgs->create_type = type;
 
+    /**
+     * Some architectures require 64 bits pointers (to pointers) to
+     * be 64 bits aligned. As in the ompi_datatype_args_t structure we have
+     * 2 such array of pointers and one to an array of ints, if we start by
+     * setting the 64 bits aligned one we will not have any trouble. Problem
+     * originally reported on SPARC 64.
+     */
+    buf += sizeof(ompi_datatype_args_t);
+    if( 0 != pArgs->ca ) {
+        pArgs->a = (ptrdiff_t*)buf;
+        buf += pArgs->ca * sizeof(ptrdiff_t);
+    }
+    if( 0 != pArgs->cd ) {
+        pArgs->d = (ompi_datatype_t**)buf;
+        buf += pArgs->cd * sizeof(MPI_Datatype);
+    }
+    if( 0 != pArgs->ci ) pArgs->i = (int*)buf;
+
+    pArgs->ref_count = 1;
+    pArgs->total_pack_size = (4 + ci) * sizeof(int) +
+                             cd * sizeof(MPI_Datatype) +
+                             ca * sizeof(ptrdiff_t);
     switch(type) {
 
     case MPI_COMBINER_DUP:
@@ -229,7 +217,7 @@ int32_t ompi_datatype_set_args( ompi_datatype_t* pData,
         pArgs->d[pos] = d[pos];
         if( !(ompi_datatype_is_predefined(d[pos])) ) {
             /* We handle a user defined datatype. We should make sure that the
-             * user will not have the oportunity to destroy it before all derived
+             * user will not have the opportunity to destroy it before all derived
              * datatypes are destroyed. As we keep pointers to every datatype
              * (for MPI_Type_get_content and MPI_Type_get_envelope) we have to make
              * sure that those datatype will be available if the user ask for them.
@@ -243,10 +231,12 @@ int32_t ompi_datatype_set_args( ompi_datatype_t* pData,
         pArgs->total_pack_size += sizeof(int);  /* each data has an ID */
     }
 
+    pData->args = (void*)pArgs;
+    pData->packed_description = 0;
+
     return OMPI_SUCCESS;
 }
 
-
 int32_t ompi_datatype_print_args( const ompi_datatype_t* pData )
 {
     int32_t i;
@@ -375,8 +365,8 @@ int32_t ompi_datatype_copy_args( const ompi_datatype_t* source_data,
 {
     ompi_datatype_args_t* pArgs = (ompi_datatype_args_t*)source_data->args;
 
-    /* Increase the reference count of the datatype enveloppe. This
-     * prevent us from making extra copies for the enveloppe (which is mostly
+    /* Increase the reference count of the datatype envelope. This
+     * prevent us from making extra copies for the envelope (which is mostly
      * a read only memory).
      */
     if( NULL != pArgs ) {
@@ -401,7 +391,7 @@ int32_t ompi_datatype_release_args( ompi_datatype_t* pData )
     OPAL_THREAD_ADD_FETCH32(&pArgs->ref_count, -1);
     if( 0 == pArgs->ref_count ) {
         /* There are some duplicated datatypes around that have a pointer to this
-         * args. We will release them only when the last datatype will dissapear.
+         * args. We will release them only when the last datatype will disappear.
          */
         for( i = 0; i < pArgs->cd; i++ ) {
             if( !(ompi_datatype_is_predefined(pArgs->d[i])) ) {
@@ -458,7 +448,7 @@ static inline int __ompi_datatype_pack_description( ompi_datatype_t* datatype,
     position = (int*)next_packed;
     next_packed += sizeof(int) * args->cd;
 
-    /* copy the aray of counts (32 bits aligned) */
+    /* copy the array of counts (32 bits aligned) */
     memcpy( next_packed, args->i, sizeof(int) * args->ci );
     next_packed += args->ci * sizeof(int);
 
@@ -841,9 +831,7 @@ ompi_datatype_t* ompi_datatype_get_single_predefined_type_from_args( ompi_dataty
                 return NULL;
             }
         }
-#if OMPI_ENABLE_MPI1_COMPAT
         if (current_predef != MPI_LB && current_predef != MPI_UB) {
-#endif
             if( NULL == predef ) {  /* This is the first iteration */
                 predef = current_predef;
             } else {
@@ -857,9 +845,7 @@ ompi_datatype_t* ompi_datatype_get_single_predefined_type_from_args( ompi_dataty
                     return NULL;
                 }
             }
-#if OMPI_ENABLE_MPI1_COMPAT
         }
-#endif
     }
     return predef;
 }
diff --git a/ompi/datatype/ompi_datatype_create_struct.c b/ompi/datatype/ompi_datatype_create_struct.c
index 9c78f53fee3..72d3251b936 100644
--- a/ompi/datatype/ompi_datatype_create_struct.c
+++ b/ompi/datatype/ompi_datatype_create_struct.c
@@ -33,8 +33,8 @@ int32_t ompi_datatype_create_struct( int count, const int* pBlockLength, const p
 {
     ptrdiff_t disp = 0, endto, lastExtent, lastDisp;
     ompi_datatype_t *pdt, *lastType;
-    int lastBlock;
     int i, start_from;
+    size_t lastBlock;
 
     /* Find first non-zero length element */
     for( i = 0; (i < count) && (0 == pBlockLength[i]); i++ );
diff --git a/ompi/datatype/ompi_datatype_external.c b/ompi/datatype/ompi_datatype_external.c
index 53b907218cf..090dda0d26a 100644
--- a/ompi/datatype/ompi_datatype_external.c
+++ b/ompi/datatype/ompi_datatype_external.c
@@ -15,6 +15,7 @@
  *                         reserved.
  * Copyright (c) 2015-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2025      Triad National Security, LLC. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -29,7 +30,7 @@
 #include "ompi/datatype/ompi_datatype.h"
 #include "opal/datatype/opal_convertor.h"
 
-int ompi_datatype_pack_external(const char datarep[], const void *inbuf, int incount,
+int ompi_datatype_pack_external(const char datarep[], const void *inbuf, size_t incount,
                                 ompi_datatype_t *datatype, void *outbuf,
                                 MPI_Aint outsize, MPI_Aint *position)
 {
@@ -57,7 +58,7 @@ int ompi_datatype_pack_external(const char datarep[], const void *inbuf, int inc
         return MPI_ERR_TRUNCATE;
     }
 
-    /* Prepare the iovec with all informations */
+    /* Prepare the iovec with all information */
     invec.iov_base = (char*) outbuf + (*position);
     invec.iov_len = size;
 
@@ -73,7 +74,7 @@ int ompi_datatype_pack_external(const char datarep[], const void *inbuf, int inc
 }
 
 int ompi_datatype_unpack_external (const char datarep[], const void *inbuf, MPI_Aint insize,
-                                   MPI_Aint *position, void *outbuf, int outcount,
+                                   MPI_Aint *position, void *outbuf, size_t outcount,
                                    ompi_datatype_t *datatype)
 {
     int rc = MPI_SUCCESS;
@@ -92,12 +93,12 @@ int ompi_datatype_unpack_external (const char datarep[], const void *inbuf, MPI_
 
     /* Check for truncation */
     opal_convertor_get_packed_size( &local_convertor, &size );
-    if( (*position + size) > (unsigned int)insize ) {
+    if( (*position + (MPI_Aint)size) > insize ) {
         OBJ_DESTRUCT( &local_convertor );
         return MPI_ERR_TRUNCATE;
     }
 
-    /* Prepare the iovec with all informations */
+    /* Prepare the iovec with all information */
     outvec.iov_base = (char*) inbuf + (*position);
     outvec.iov_len = size;
 
@@ -112,7 +113,7 @@ int ompi_datatype_unpack_external (const char datarep[], const void *inbuf, MPI_
     return (rc == 1) ? OMPI_SUCCESS : MPI_ERR_UNKNOWN;
 }
 
-int ompi_datatype_pack_external_size(const char datarep[], int incount,
+int ompi_datatype_pack_external_size(const char datarep[], size_t incount,
                                      ompi_datatype_t *datatype, MPI_Aint *size)
 {
     opal_convertor_t local_convertor;
@@ -126,7 +127,7 @@ int ompi_datatype_pack_external_size(const char datarep[], int incount,
                                               CONVERTOR_SEND_CONVERSION,
                                               &local_convertor );
 
-    opal_convertor_get_unpacked_size( &local_convertor, &length );
+    opal_convertor_get_packed_size( &local_convertor, &length );
     *size = (MPI_Aint)length;
     OBJ_DESTRUCT( &local_convertor );
 
diff --git a/ompi/datatype/ompi_datatype_external32.c b/ompi/datatype/ompi_datatype_external32.c
index 108e14258b7..d8eb81dc897 100644
--- a/ompi/datatype/ompi_datatype_external32.c
+++ b/ompi/datatype/ompi_datatype_external32.c
@@ -26,39 +26,74 @@
 
 /* From the MPI standard. external32 use the following types:
  *   Type Length
- * MPI_PACKED               1
- * MPI_BYTE                 1
- * MPI_CHAR                 1
- * MPI_UNSIGNED_CHAR        1
- * MPI_SIGNED_CHAR          1
- * MPI_WCHAR                2
- * MPI_SHORT                2
- * MPI_UNSIGNED_SHORT       2
- * MPI_INT                  4
- * MPI_UNSIGNED             4
- * MPI_LONG                 4
- * MPI_UNSIGNED_LONG        4
- * MPI_FLOAT                4
- * MPI_DOUBLE               8
- * MPI_LONG_DOUBLE         16
+ * MPI_PACKED                   1
+ * MPI_BYTE                     1
+ * MPI_CHAR                     1
+ * MPI_UNSIGNED_CHAR            1
+ * MPI_SIGNED_CHAR              1
+ * MPI_WCHAR                    2
+ * MPI_SHORT                    2
+ * MPI_UNSIGNED_SHORT           2
+ * MPI_INT                      4
+ * MPI_LONG                     4
+ * MPI_UNSIGNED                 4
+ * MPI_UNSIGNED_LONG            4
+ * MPI_LONG_LONG_INT            8
+ * MPI_UNSIGNED_LONG_LONG       8
+ * MPI_FLOAT                    4
+ * MPI_DOUBLE                   8
+ * MPI_LONG_DOUBLE             16
+ *
+ * MPI_C_BOOL                   1
+ * MPI_INT8_T                   1
+ * MPI_INT16_T                  2
+ * MPI_INT32_T                  4
+ * MPI_INT64_T                  8
+ * MPI_UINT8_T                  1
+ * MPI_UINT16_T                 2
+ * MPI_UINT32_T                 4
+ * MPI_UINT64_T                 8
+ * MPI_AINT                     8
+ * MPI_COUNT                    8
+ * MPI_OFFSET                   8
+ * MPI_C_COMPLEX                2*4
+ * MPI_C_FLOAT_COMPLEX          2*4
+ * MPI_C_DOUBLE_COMPLEX         2*8
+ * MPI_C_LONG_DOUBLE_COMPLEX    2*16
+ *
  * Fortran types
- * MPI_CHARACTER            1
- * MPI_LOGICAL              4
- * MPI_INTEGER              4
- * MPI_REAL                 4
- * MPI_DOUBLE_PRECISION     8
- * MPI_COMPLEX              2*4
- * MPI_DOUBLE_COMPLEX       2*8
+ * MPI_CHARACTER                1
+ * MPI_LOGICAL                  4
+ * MPI_INTEGER                  4
+ * MPI_REAL                     4
+ * MPI_DOUBLE_PRECISION         8
+ * MPI_COMPLEX                  2*4
+ * MPI_DOUBLE_COMPLEX           2*8
+ *
+ * MPI_CXX_BOOL                 1
+ * MPI_CXX_FLOAT_COMPLEX        2*4
+ * MPI_CXX_DOUBLE_COMPLEX       2*8
+ * MPI_CXX_LONG_DOUBLE_COMPLEX  2*16
+ *
  * Optional types
- * MPI_INTEGER1             1
- * MPI_INTEGER2             2
- * MPI_INTEGER4             4
- * MPI_INTEGER8             8
- * MPI_LONG_LONG_INT        8
- * MPI_UNSIGNED_LONG_LONG   8
- * MPI_REAL4                4
- * MPI_REAL8                8
- * MPI_REAL16              16
+ * MPI_INTEGER1                 1
+ * MPI_INTEGER2                 2
+ * MPI_INTEGER4                 4
+ * MPI_INTEGER8                 8
+ * MPI_INTEGER16               16
+ * MPI_REAL2                    2
+ * MPI_REAL4                    4
+ * MPI_REAL8                    8
+ * MPI_REAL16                  16
+ * MPI_COMPLEX4                 2*2
+ * MPI_COMPLEX8                 2*4
+ * MPI_COMPLEX16                2*8
+ * MPI_COMPLEX32                2*16
+ *
+ * MPI_CXX_BOOL                 1
+ * MPI_CXX_FLOAT_COMPLEX        2*4
+ * MPI_CXX_DOUBLE_COMPLEX       2*8
+ * MPI_CXX_LONG_DOUBLE_COMPLEX  2*16
  *
  * All floating point values are in big-endian IEEE format. Double extended use 16 bytes, with
  * 15 exponent bits (bias = 10383), 112 mantissa bits and the same encoding as double. All
@@ -90,8 +125,12 @@ int32_t ompi_datatype_default_convertors_init( void )
 
 int32_t ompi_datatype_default_convertors_fini( void )
 {
-    OBJ_RELEASE( ompi_mpi_external32_convertor );
-    OBJ_RELEASE( ompi_mpi_local_convertor );
+    if (NULL != ompi_mpi_external32_convertor) {
+        OBJ_RELEASE( ompi_mpi_external32_convertor );
+    }
+    if (NULL != ompi_mpi_local_convertor) {
+        OBJ_RELEASE( ompi_mpi_local_convertor );
+    }
 
     return OMPI_SUCCESS;
 }
diff --git a/ompi/datatype/ompi_datatype_internal.h b/ompi/datatype/ompi_datatype_internal.h
index 0c175c4266c..31f3cf287c4 100644
--- a/ompi/datatype/ompi_datatype_internal.h
+++ b/ompi/datatype/ompi_datatype_internal.h
@@ -29,6 +29,7 @@
 #define OMPI_DATATYPE_INTERNAL_H
 
 #include "opal/datatype/opal_datatype_internal.h"
+#include "ompi/datatype/ompi_datatype.h"
 
 /*
  * This is the OMPI-layered numbering of ALL supported MPI types
@@ -108,8 +109,14 @@
 #define OMPI_DATATYPE_MPI_SHORT_FLOAT             0x30
 #define OMPI_DATATYPE_MPI_C_SHORT_FLOAT_COMPLEX   0x31
 
+/*
+ * Datatypes that have a different external32 length.
+ */
+#define OMPI_DATATYPE_MPI_LONG                    0x32
+#define OMPI_DATATYPE_MPI_UNSIGNED_LONG           0x33
+
 /* This should __ALWAYS__ stay last  */
-#define OMPI_DATATYPE_MPI_UNAVAILABLE             0x32
+#define OMPI_DATATYPE_MPI_UNAVAILABLE             0x34
 
 
 #define OMPI_DATATYPE_MPI_MAX_PREDEFINED          (OMPI_DATATYPE_MPI_UNAVAILABLE+1)
@@ -126,27 +133,14 @@
  * of being redefined as independent types, they will be made synonyms to
  * the most basic type.
  */
-#if SIZEOF_CHAR == 1
+#if SIZEOF_CHAR != 1
+#error Open MPI asserts that sizeof(char) is 1. This seems not to be the case here. Please report on Open MPI github (see issue #11815)
+#endif
+
 #define OMPI_DATATYPE_MPI_CHAR                    OMPI_DATATYPE_MPI_INT8_T
 #define OMPI_DATATYPE_MPI_SIGNED_CHAR             OMPI_DATATYPE_MPI_INT8_T
 #define OMPI_DATATYPE_MPI_UNSIGNED_CHAR           OMPI_DATATYPE_MPI_UINT8_T
 #define OMPI_DATATYPE_MPI_BYTE                    OMPI_DATATYPE_MPI_UINT8_T
-#elif SIZEOF_CHAR == 2
-#define OMPI_DATATYPE_MPI_CHAR                    OMPI_DATATYPE_MPI_INT16_T
-#define OMPI_DATATYPE_MPI_SIGNED_CHAR             OMPI_DATATYPE_MPI_INT16_T
-#define OMPI_DATATYPE_MPI_UNSIGNED_CHAR           OMPI_DATATYPE_MPI_UINT16_T
-#define OMPI_DATATYPE_MPI_BYTE                    OMPI_DATATYPE_MPI_UINT16_T
-#elif SIZEOF_CHAR == 4
-#define OMPI_DATATYPE_MPI_CHAR                    OMPI_DATATYPE_MPI_INT32_T
-#define OMPI_DATATYPE_MPI_SIGNED_CHAR             OMPI_DATATYPE_MPI_INT32_T
-#define OMPI_DATATYPE_MPI_UNSIGNED_CHAR           OMPI_DATATYPE_MPI_UINT32_T
-#define OMPI_DATATYPE_MPI_BYTE                    OMPI_DATATYPE_MPI_UINT32_T
-#elif SIZEOF_CHAR == 8
-#define OMPI_DATATYPE_MPI_CHAR                    OMPI_DATATYPE_MPI_INT64_T
-#define OMPI_DATATYPE_MPI_SIGNED_CHAR             OMPI_DATATYPE_MPI_INT64_T
-#define OMPI_DATATYPE_MPI_UNSIGNED_CHAR           OMPI_DATATYPE_MPI_UINT64_T
-#define OMPI_DATATYPE_MPI_BYTE                    OMPI_DATATYPE_MPI_UINT64_T
-#endif
 
 #if SIZEOF_SHORT == 1
 #define OMPI_DATATYPE_MPI_SHORT                   OMPI_DATATYPE_MPI_INT8_T
@@ -176,20 +170,6 @@
 #define OMPI_DATATYPE_MPI_UNSIGNED                OMPI_DATATYPE_MPI_UINT64_T
 #endif
 
-#if SIZEOF_LONG == 1
-#define OMPI_DATATYPE_MPI_LONG                    OMPI_DATATYPE_MPI_INT8_T
-#define OMPI_DATATYPE_MPI_UNSIGNED_LONG           OMPI_DATATYPE_MPI_UINT8_T
-#elif SIZEOF_LONG == 2
-#define OMPI_DATATYPE_MPI_LONG                    OMPI_DATATYPE_MPI_INT16_T
-#define OMPI_DATATYPE_MPI_UNSIGNED_LONG           OMPI_DATATYPE_MPI_UINT16_T
-#elif SIZEOF_LONG == 4
-#define OMPI_DATATYPE_MPI_LONG                    OMPI_DATATYPE_MPI_INT32_T
-#define OMPI_DATATYPE_MPI_UNSIGNED_LONG           OMPI_DATATYPE_MPI_UINT32_T
-#elif SIZEOF_LONG == 8
-#define OMPI_DATATYPE_MPI_LONG                    OMPI_DATATYPE_MPI_INT64_T
-#define OMPI_DATATYPE_MPI_UNSIGNED_LONG           OMPI_DATATYPE_MPI_UINT64_T
-#endif
-
 #if SIZEOF_LONG_LONG == 1
 #define OMPI_DATATYPE_MPI_LONG_LONG_INT           OMPI_DATATYPE_MPI_INT8_T
 #define OMPI_DATATYPE_MPI_UNSIGNED_LONG_LONG      OMPI_DATATYPE_MPI_UINT8_T
@@ -462,7 +442,7 @@ extern const ompi_datatype_t* ompi_datatype_basicDatatypes[OMPI_DATATYPE_MPI_MAX
 /*
  * Initialization for these types is deferred until runtime.
  *
- * Using this macro implies that at this point not all informations needed
+ * Using this macro implies that at this point not all information needed
  * to fill up the datatype are known. We fill them with zeros and then later
  * when the datatype engine will be initialized we complete with the
  * correct information. This macro should be used for all composed types.
@@ -526,27 +506,10 @@ extern const ompi_datatype_t* ompi_datatype_basicDatatypes[OMPI_DATATYPE_MPI_MAX
 #define OMPI_DATATYPE_INITIALIZER_INT64_T             OPAL_DATATYPE_INITIALIZER_INT8
 #define OMPI_DATATYPE_INITIALIZER_UINT64_T            OPAL_DATATYPE_INITIALIZER_UINT8
 
-#if SIZEOF_CHAR == 1
 #define OMPI_DATATYPE_INITIALIZER_CHAR                OPAL_DATATYPE_INITIALIZER_INT1
 #define OMPI_DATATYPE_INITIALIZER_UNSIGNED_CHAR       OPAL_DATATYPE_INITIALIZER_UINT1
 #define OMPI_DATATYPE_INITIALIZER_SIGNED_CHAR         OPAL_DATATYPE_INITIALIZER_INT1
 #define OMPI_DATATYPE_INITIALIZER_BYTE                OPAL_DATATYPE_INITIALIZER_UINT1
-#elif SIZEOF_CHAR == 2
-#define OMPI_DATATYPE_INITIALIZER_CHAR                OPAL_DATATYPE_INITIALIZER_INT2
-#define OMPI_DATATYPE_INITIALIZER_UNSIGNED_CHAR       OPAL_DATATYPE_INITIALIZER_UINT2
-#define OMPI_DATATYPE_INITIALIZER_SIGNED_CHAR         OPAL_DATATYPE_INITIALIZER_INT2
-#define OMPI_DATATYPE_INITIALIZER_BYTE                OPAL_DATATYPE_INITIALIZER_UINT2
-#elif SIZEOF_CHAR == 4
-#define OMPI_DATATYPE_INITIALIZER_CHAR                OPAL_DATATYPE_INITIALIZER_INT4
-#define OMPI_DATATYPE_INITIALIZER_UNSIGNED_CHAR       OPAL_DATATYPE_INITIALIZER_UINT4
-#define OMPI_DATATYPE_INITIALIZER_SIGNED_CHAR         OPAL_DATATYPE_INITIALIZER_INT4
-#define OMPI_DATATYPE_INITIALIZER_BYTE                OPAL_DATATYPE_INITIALIZER_UINT4
-#elif SIZEOF_CHAR == 8
-#define OMPI_DATATYPE_INITIALIZER_CHAR                OPAL_DATATYPE_INITIALIZER_INT8
-#define OMPI_DATATYPE_INITIALIZER_UNSIGNED_CHAR       OPAL_DATATYPE_INITIALIZER_UINT8
-#define OMPI_DATATYPE_INITIALIZER_SIGNED_CHAR         OPAL_DATATYPE_INITIALIZER_INT8
-#define OMPI_DATATYPE_INITIALIZER_BYTE                OPAL_DATATYPE_INITIALIZER_UINT8
-#endif
 
 #if SIZEOF_SHORT == 2
 #define OMPI_DATATYPE_INITIALIZER_SHORT               OPAL_DATATYPE_INITIALIZER_INT2
@@ -570,16 +533,8 @@ extern const ompi_datatype_t* ompi_datatype_basicDatatypes[OMPI_DATATYPE_MPI_MAX
 #define OMPI_DATATYPE_INITIALIZER_UNSIGNED            OPAL_DATATYPE_INITIALIZER_UINT8
 #endif
 
-#if SIZEOF_LONG == 4
-#define OMPI_DATATYPE_INITIALIZER_LONG                OPAL_DATATYPE_INITIALIZER_INT4
-#define OMPI_DATATYPE_INITIALIZER_UNSIGNED_LONG       OPAL_DATATYPE_INITIALIZER_UINT4
-#elif SIZEOF_LONG == 8
-#define OMPI_DATATYPE_INITIALIZER_LONG                OPAL_DATATYPE_INITIALIZER_INT8
-#define OMPI_DATATYPE_INITIALIZER_UNSIGNED_LONG       OPAL_DATATYPE_INITIALIZER_UINT8
-#elif SIZEOF_LONG == 16
-#define OMPI_DATATYPE_INITIALIZER_LONG                OPAL_DATATYPE_INITIALIZER_INT16
-#define OMPI_DATATYPE_INITIALIZER_UNSIGNED_LONG       OPAL_DATATYPE_INITIALIZER_UINT16
-#endif
+#define OMPI_DATATYPE_INITIALIZER_LONG                OPAL_DATATYPE_INITIALIZER_LONG
+#define OMPI_DATATYPE_INITIALIZER_UNSIGNED_LONG       OPAL_DATATYPE_INITIALIZER_UNSIGNED_LONG
 
 #if SIZEOF_LONG_LONG == 4
 #define OMPI_DATATYPE_INITIALIZER_LONG_LONG_INT       OPAL_DATATYPE_INITIALIZER_INT4
diff --git a/ompi/datatype/ompi_datatype_module.c b/ompi/datatype/ompi_datatype_module.c
index 4330308cc2c..77747100d90 100644
--- a/ompi/datatype/ompi_datatype_module.c
+++ b/ompi/datatype/ompi_datatype_module.c
@@ -18,6 +18,8 @@
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016-2018 FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2018-2021 Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -34,11 +36,16 @@
 #include "opal/util/output.h"
 #include "opal/util/string_copy.h"
 #include "opal/class/opal_pointer_array.h"
+#include "ompi/attribute/attribute.h"
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/datatype/ompi_datatype_internal.h"
+#include "ompi/instance/instance.h"
+#include "ompi/attribute/attribute.h"
 
 #include "mpi.h"
 
+static int ompi_datatype_finalize (void);
+
 /**
  * This is the number of predefined datatypes. It is different than the MAX_PREDEFINED
  * as it include all the optional datatypes (such as MPI_INTEGER?, MPI_REAL?).
@@ -366,6 +373,8 @@ const ompi_datatype_t* ompi_datatype_basicDatatypes[OMPI_DATATYPE_MPI_MAX_PREDEF
     [OMPI_DATATYPE_MPI_LB] = &ompi_mpi_lb.dt,
     [OMPI_DATATYPE_MPI_UB] = &ompi_mpi_ub.dt,
 
+    [OMPI_DATATYPE_MPI_LONG] = &ompi_mpi_long.dt,
+    [OMPI_DATATYPE_MPI_UNSIGNED_LONG] = &ompi_mpi_unsigned_long.dt,
     /* MPI 3.0 types */
     [OMPI_DATATYPE_MPI_COUNT] = &ompi_mpi_count.dt,
 
@@ -470,6 +479,7 @@ opal_pointer_array_t ompi_datatype_f_to_c_table = {{0}};
 int32_t ompi_datatype_init( void )
 {
     int32_t i;
+    int ret = OMPI_SUCCESS;
 
     opal_datatype_init();
 
@@ -544,6 +554,9 @@ int32_t ompi_datatype_init( void )
 #define MOOG(name, index)                                               \
     do {                                                                \
         int rc;                                                         \
+         /* Silence 'unused' compiler warning in optimized builds,      \
+            where assert() is removed. */                               \
+        (void) rc;                                                      \
         ompi_mpi_##name.dt.d_f_to_c_index = index;                      \
         rc = opal_pointer_array_set_item(&ompi_datatype_f_to_c_table,   \
                                          index, &ompi_mpi_##name);      \
@@ -664,24 +677,26 @@ int32_t ompi_datatype_init( void )
             datatype->flags &= ~OPAL_DATATYPE_FLAG_NO_GAPS;
         }
     }
+
+    /* get a reference to the attributes subsys */
+    ret = ompi_attr_get_ref();
+    if (OMPI_SUCCESS != ret) {
+        return ret;
+    }
+
     ompi_datatype_default_convertors_init();
+
+    ompi_mpi_instance_append_finalize (ompi_datatype_finalize);
     return OMPI_SUCCESS;
 }
 
 
-int32_t ompi_datatype_finalize( void )
+static int ompi_datatype_finalize (void)
 {
     /* As the synonyms are just copies of the internal data we should not free them.
      * Anyway they are over the limit of OMPI_DATATYPE_MPI_MAX_PREDEFINED so they will never get freed.
      */
 
-    /* As they are statically allocated they cannot be released.
-     * But we can call OBJ_DESTRUCT, just to free all internally allocated ressources.
-     */
-    for( int i = 0; i < ompi_datatype_number_of_predefined_data; i++ ) {
-        opal_datatype_t* datatype = (opal_datatype_t*)opal_pointer_array_get_item(&ompi_datatype_f_to_c_table, i );
-        OBJ_DESTRUCT(datatype);
-    }
 
     /* Get rid of the Fortran2C translation table */
     OBJ_DESTRUCT(&ompi_datatype_f_to_c_table);
@@ -692,7 +707,8 @@ int32_t ompi_datatype_finalize( void )
     /* don't call opal_datatype_finalize () as it no longer exists. the function will be called
      * opal_finalize_util (). */
 
-    return OMPI_SUCCESS;
+    /* release a reference to the attributes subsys */
+    return ompi_attr_put_ref();
 }
 
 
diff --git a/ompi/debuggers/MPI_Handles_interface.txt b/ompi/debuggers/MPI_Handles_interface.txt
index 863425bd221..4f479e36d1e 100644
--- a/ompi/debuggers/MPI_Handles_interface.txt
+++ b/ompi/debuggers/MPI_Handles_interface.txt
@@ -507,7 +507,7 @@ create their own requests.
   - Whether the underlying object has been "freed" by the MPI
     implementation (i.e., made inactive, and would have actually been
     freed if not running under a debugger)
-  - Peer process(es) invovled with the request (if available)
+  - Peer process(es) involved with the request (if available)
   - If pt2pt, communicator associated with the request
   - If file, file associated with the request
   - If pt2pt or file, whether the data transfer has started yet
@@ -570,7 +570,7 @@ C++: MPI::Status
 
 A user-accessible struct that contains information about a completed
 communication.  The MPI status is a little different from other MPI
-handles in that it is the object itselt; not a handle to an underlying
+handles in that it is the object itself; not a handle to an underlying
 MPI status.  For example, if a point-to-point communication was
 started with a wildcard receive, the status will contain information
 about the peer to whom the communication completed.  There are no
diff --git a/ompi/debuggers/Makefile.am b/ompi/debuggers/Makefile.am
index 8dafba9c9b6..72e5769db76 100644
--- a/ompi/debuggers/Makefile.am
+++ b/ompi/debuggers/Makefile.am
@@ -46,7 +46,7 @@ headers = \
 dlopen_test_SOURCES = dlopen_test.c
 dlopen_test_LDADD = \
         $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 dlopen_test_DEPENDENCIES = $(ompi_predefined_LDADD)
 
 predefined_gap_test_SOURCES = predefined_gap_test.c
diff --git a/ompi/debuggers/dlopen_test.c b/ompi/debuggers/dlopen_test.c
index 47b5ae5a854..c58a236547f 100644
--- a/ompi/debuggers/dlopen_test.c
+++ b/ompi/debuggers/dlopen_test.c
@@ -117,7 +117,7 @@ static int do_test(void)
     /* If we're using libltdl, it will find the .la file and may
        discover that it needs to open the actual file in the .libs
        directory.  If we're not using libltdl, then we won't know
-       about the magic .la file / .libs directory.  Hueristic: if we
+       about the magic .la file / .libs directory.  Heuristic: if we
        get here, manually prefix the filename with .libs/ and try
        again. */
     char *rel_filename;
diff --git a/ompi/debuggers/mpihandles_interface.h b/ompi/debuggers/mpihandles_interface.h
index 66412414e5f..092415863e9 100644
--- a/ompi/debuggers/mpihandles_interface.h
+++ b/ompi/debuggers/mpihandles_interface.h
@@ -153,7 +153,7 @@ struct mpidbg_process_t {
     int mpi_comm_world_rank;
 };
 /* ==> JMS Should we just use mqs_process_location instead?  George
-   thinks that this is unncessary -- perhaps due to the fact that we
+   thinks that this is unnecessary -- perhaps due to the fact that we
    could use mqs_process_location...?  Need to get some feedback from
    others on this one.  Need to check Euro PVM/MPI '06 paper... */
 
@@ -503,8 +503,8 @@ extern char mpidbg_dll_is_big_endian;
    the DLL. */
 extern char mpidbg_dll_bitness;
 
-/* Global variable *in the DLL* describing the DLL's capabilties with
-   regards to communicators.  This value is valid after a successfull
+/* Global variable *in the DLL* describing the DLL's capabilities with
+   regards to communicators.  This value is valid after a successful
    call to mpidbg_init_per_process(). */
 extern enum mpidbg_comm_capabilities_t mpidbg_comm_capabilities;
 
@@ -515,12 +515,12 @@ extern enum mpidbg_comm_capabilities_t mpidbg_comm_capabilities;
    be able to map this name to a valid handle.  MPI implementations
    not requiring this mapping can either have a NULL value for this
    variable or have a single entry that has a NULL string value.  This
-   variable is not valid until after a successfull call to
+   variable is not valid until after a successful call to
    mpidbg_init_per_process().  */
 extern struct mpidbg_name_map_t *mpidbg_comm_name_map;
 
-/* Global variable *in the DLL* describing the DLL's capabilties with
-   regards to error handlers.  This value is valid after a successfull
+/* Global variable *in the DLL* describing the DLL's capabilities with
+   regards to error handlers.  This value is valid after a successful
    call to mpidbg_init_per_process(). */
 extern enum mpidbg_errhandler_capabilities_t mpidbg_errhandler_capabilities;
 
diff --git a/ompi/debuggers/ompi_common_dll.c b/ompi/debuggers/ompi_common_dll.c
index 9395d93241a..1c014e5731d 100644
--- a/ompi/debuggers/ompi_common_dll.c
+++ b/ompi/debuggers/ompi_common_dll.c
@@ -67,9 +67,9 @@ static int host_is_big_endian = 0;
     {                                                                   \
         out_name = mqs_field_offset((qh_type), #field_name);            \
         if (out_name < 0) {                                             \
-            fprintf(stderr, "WARNING: Open MPI is unable to find "      \
+            fprintf(stderr, "WARNING: " OMPI_IDENT_STRING " is unable to find "      \
                     "field " #field_name " in the " #struct_name        \
-                    " type.  This can happen can if Open MPI is built " \
+                    " type.  This can happen if " OMPI_IDENT_STRING " is built " \
                     "without debugging information, or is stripped "    \
                     "after building.\n");                               \
         }                                                               \
@@ -324,6 +324,10 @@ int ompi_fill_in_type_info(mqs_image *image, char **message)
                           qh_type, ompi_communicator_t, c_name);
         ompi_field_offset(i_info->ompi_communicator_t.offset.c_contextid,
                           qh_type, ompi_communicator_t, c_contextid);
+        ompi_field_offset(i_info->ompi_communicator_t.offset.c_contextidb,
+                          qh_type, ompi_communicator_t, c_contextidb);
+        ompi_field_offset(i_info->ompi_communicator_t.offset.c_index,
+                          qh_type, ompi_communicator_t, c_index);
         ompi_field_offset(i_info->ompi_communicator_t.offset.c_my_rank,
                           qh_type, ompi_communicator_t, c_my_rank);
         ompi_field_offset(i_info->ompi_communicator_t.offset.c_local_group,
@@ -501,7 +505,7 @@ int ompi_fill_in_type_info(mqs_image *image, char **message)
         }
     }
 
-    /* All the types are here. Let's succesfully return. */
+    /* All the types are here. Let's successfully return. */
     *message = NULL;
     return mqs_ok;
 
@@ -512,7 +516,7 @@ int ompi_fill_in_type_info(mqs_image *image, char **message)
      * did our best but here we're at our limit. Give up!
      */
     *message = missing_in_action;
-    fprintf(stderr, "WARNING: Open MPI is unable to find debugging information about the \"%s\" type.  This can happen if Open MPI was built without debugging information, or was stripped after building.\n",
+    fprintf(stderr, "WARNING: " OMPI_IDENT_STRING " is unable to find debugging information about the \"%s\" type.  This can happen if " OMPI_IDENT_STRING " was built without debugging information, or was stripped after building.\n",
            missing_in_action);
     return err_missing_type;
 }
@@ -634,7 +638,7 @@ int ompi_fetch_opal_pointer_array_item(mqs_process *proc, mqs_taddr_t addr,
 
 int ompi_get_lib_version(char * buf, int size) {
     int ret;
-    ret = snprintf(buf, size-1, "Open MPI v%d.%d.%d%s%s%s%s%s%s%s%s%s",
+    ret = snprintf(buf, size-1, OMPI_IDENT_STRING " v%d.%d.%d%s%s%s%s%s%s%s%s%s",
                    OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION, OMPI_RELEASE_VERSION,
                    (strlen(OMPI_GREEK_VERSION) > 0)?OMPI_GREEK_VERSION:"",
                    (strlen(OPAL_PACKAGE_STRING) > 0)?", package: ":"",
diff --git a/ompi/debuggers/ompi_common_dll_defs.h b/ompi/debuggers/ompi_common_dll_defs.h
index 6f4e6b89381..5fe11d3986e 100644
--- a/ompi/debuggers/ompi_common_dll_defs.h
+++ b/ompi/debuggers/ompi_common_dll_defs.h
@@ -198,6 +198,8 @@ typedef struct
         struct {
             int c_name;
             int c_contextid;
+            int c_contextidb;
+            int c_index;
             int c_my_rank;
             int c_local_group;
             int c_remote_group;
diff --git a/ompi/debuggers/ompi_debuggers.c b/ompi/debuggers/ompi_debuggers.c
index f9d7c28bb18..cb3f87f49c2 100644
--- a/ompi/debuggers/ompi_debuggers.c
+++ b/ompi/debuggers/ompi_debuggers.c
@@ -26,8 +26,6 @@
  * MPI portion of debugger support: initially based on the
  * TotalView/Etnus API for debuggers to attach to MPI jobs.
  *
- * There is a lengthy explanation of how OMPI handles parallel
- * debuggers attaching to MPI jobs in orte/tools/orterun/debuggers.c.
  */
 
 #include "ompi_config.h"
@@ -60,7 +58,7 @@
  * information on optimized build, and as a result we are unable to access
  * the fields structure (i.e. to get their displacement). This file is
  * included in the optimized build just to provide us with this missing
- * informations. Therefore, it always have to be compiled with the -g flag,
+ * information. Therefore, it always have to be compiled with the -g flag,
  * otherwise the type information will be missing and the parallel
  * debuggers will be unable to initialize the Open MPI debug library.
  */
diff --git a/ompi/debuggers/ompi_mpihandles_dll.c b/ompi/debuggers/ompi_mpihandles_dll.c
index c0aef28c657..aabb803a8bb 100644
--- a/ompi/debuggers/ompi_mpihandles_dll.c
+++ b/ompi/debuggers/ompi_mpihandles_dll.c
@@ -7,7 +7,9 @@
  * Copyright (c) 2012-2013 Inria.  All rights reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017 IBM Corp.  All rights reserved.
+ * Copyright (c) 2017      IBM Corp.  All rights reserved.
+ * Copyright (c) 2023      NVIDIA Corporation. All rights reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -286,7 +288,7 @@ int mpidbg_init_per_image(mqs_image *image, const mqs_image_callbacks *icb,
     handle_types->hi_cxx_win =
         mqs_find_type(image, "MPI::Win", mqs_lang_cplus);
 
-    /* Tell the debuger what capabilities we have */
+    /* Tell the debugger what capabilities we have */
     mpidbg_comm_capabilities =
         MPIDBG_COMM_CAP_BASIC |
         MPIDBG_COMM_CAP_STRING_NAMES |
@@ -452,14 +454,26 @@ int mpidbg_comm_query(mqs_image *image, mqs_image_info *image_info,
     if (NULL == *info) {
         return MPIDBG_ERR_NO_MEM;
     }
-    /* JMS temporarily zero everything out.  Remove this when we fill
-       in all the fields */
+    /* Zero everything out.  Remove this when we fill in all the
+       fields */
     memset(*info, 0, sizeof(struct mpidbg_comm_info_t));
     (*info)->comm_c_handle = c_comm;
 
-    printf("mpidbg_comm_query: %p\n", (void*) c_comm);
-    mqs_fetch_data(process, c_comm + i_info->ompi_communicator_t.offset.c_name,
+    mqs_taddr_t name_addr = ompi_fetch_pointer( process,
+                                                c_comm + i_info->ompi_communicator_t.offset.c_name,
+                                                p_info );
+    mqs_fetch_data(process, name_addr,
                    MPIDBG_MAX_OBJECT_NAME, (*info)->comm_name);
+    (*info)->comm_name[MPIDBG_MAX_OBJECT_NAME-1] = '\0';
+    /* Defensively zero anything beyond the actual name.
+
+       We know that MPIDBG_MAX_OBJECT_NAME == MPI_MAX_OBJECT_NAME (per
+       mpihandles_interface.h), and OMPI *guarantees* that
+       (*info)->comm_name will be both \0-terminated, and have a
+       maximum of (MPI_MAX_OBJECT_NAME-1) non-NULL characters.  So the
+       memset length expression below is guaranted be >=0. */
+    memset((*info)->comm_name + strlen((*info)->comm_name), 0,
+           MPIDBG_MAX_OBJECT_NAME - 1 - strlen((*info)->comm_name));
 
     /* Get this process' rank in the comm */
     (*info)->comm_rank = ompi_fetch_int(process,
diff --git a/ompi/debuggers/ompi_msgq_dll.c b/ompi/debuggers/ompi_msgq_dll.c
index f5afab5d2e7..5779140979c 100644
--- a/ompi/debuggers/ompi_msgq_dll.c
+++ b/ompi/debuggers/ompi_msgq_dll.c
@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2007-2018 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2007-2022 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2004-2010 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
@@ -10,6 +10,8 @@
  * Copyright (c) 2016      Intel, Inc. All rights reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2023      NVIDIA Corporation. All rights reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -79,7 +81,7 @@
 
 /* Notice to developers!!!!
  * The following include files with _dbg.h suffixes contains definitions
- * that are shared between the debuggger plugins and the OMPI code base.
+ * that are shared between the debugger plugins and the OMPI code base.
  * This is done instead of including the non-_dbg suffixed files because
  * of the different way compilers may handle extern definitions. The
  * particular case that is causing problems is when there is an extern
@@ -650,11 +652,13 @@ static int rebuild_communicator_list (mqs_process *proc)
         if( 0 == comm_ptr ) continue;
         commcount++;
         /* Now let's grab the data we want from inside */
+        /* NTH:  XXXXXXXXXXXXX FIXME!!!!!!!!!!!!!! c_index is local but MSGQ needs a global identifier
+         * that is sizeof (void *) or smaller. */
         DEBUG(VERBOSE_GENERAL, ("Retrieve context_id from 0x%llx and local_rank from 0x%llx\n",
-                                (long long)(comm_ptr + i_info->ompi_communicator_t.offset.c_contextid),
+                                (long long)(comm_ptr + i_info->ompi_communicator_t.offset.c_index),
                                 (long long)(comm_ptr + i_info->ompi_communicator_t.offset.c_my_rank)));
         context_id = ompi_fetch_int( proc,
-                                     comm_ptr + i_info->ompi_communicator_t.offset.c_contextid,
+                                     comm_ptr + i_info->ompi_communicator_t.offset.c_index,
                                      p_info );
         /* Do we already have this communicator ? */
         old = find_communicator(p_info, context_id);
@@ -680,8 +684,18 @@ static int rebuild_communicator_list (mqs_process *proc)
                                     p_info );
             old->group = find_or_create_group( proc, group_base );
         }
-        mqs_fetch_data( proc, comm_ptr + i_info->ompi_communicator_t.offset.c_name,
-                        64, old->comm_info.name );
+        mqs_taddr_t name_addr = ompi_fetch_pointer( proc,
+                                                    comm_ptr + i_info->ompi_communicator_t.offset.c_name,
+                                                    p_info );
+        /* c_name can be up to MPI_MAX_OBJECT_NAME bytes, but we only
+         * copy the first (sizeof(old->comm_info.name)-1) here.  Make
+         * sure the string is correctly terminated. */
+        size_t target_size = sizeof(old->comm_info.name);
+        mqs_fetch_data( proc, name_addr, target_size, old->comm_info.name );
+        old->comm_info.name[target_size - 1] = '\0';
+        /* Defensively zero anything beyond the actual name */
+        size_t src_strlen = strlen(old->comm_info.name);
+        memset(old->comm_info.name + src_strlen, 0, target_size - 1 - src_strlen);
 
         if( NULL != old->group ) {
             old->comm_info.size = old->group->entries;
@@ -1154,8 +1168,9 @@ static int fetch_request( mqs_process *proc, mpi_process_info *p_info,
                                ompi_datatype + i_info->ompi_datatype_t.offset.size,
                                p_info );
         /* Be user friendly, show the datatype name */
+        size_t data_name_size = sizeof(data_name);
         mqs_fetch_data( proc, ompi_datatype + i_info->ompi_datatype_t.offset.name,
-                        64, data_name );
+                        data_name_size, data_name );
         if( '\0' != data_name[0] ) {
             // res->extra_text[x] is only 64 chars long -- same as
             // data_name.  If you try to snprintf it into
@@ -1165,10 +1180,15 @@ static int fetch_request( mqs_process *proc, mpi_process_info *p_info,
             // data_name in res->extra_text[2] (vs. extra_text[1]),
             // where it is guaranteed to fit.
             data_name[4] = '\0';
-            snprintf( (char*)res->extra_text[1], 64, "Data: %d",
+            snprintf( (char*)res->extra_text[1], 64, "Data: %d instances of MPI datatype",
                       (int)res->desired_length);
             snprintf( (char*)res->extra_text[2], 64, "%s",
                       data_name );
+        } else {
+            data_name[data_name_size - 1] = '\0';
+            /* Be nice and zero anything beyond the actual name */
+            size_t data_name_strlen = strlen(data_name);
+            memset(data_name + data_name_strlen, 0, data_name_size - 1 - data_name_strlen);
         }
         /* And now compute the real length as specified by the user */
         res->desired_length *=
diff --git a/ompi/debuggers/predefined_gap_test.c b/ompi/debuggers/predefined_gap_test.c
index 0129eb63a23..36adc09fb7b 100644
--- a/ompi/debuggers/predefined_gap_test.c
+++ b/ompi/debuggers/predefined_gap_test.c
@@ -1,7 +1,7 @@
 /*
  * Copyright (c) 2009      Sun Microsystems, Inc  All rights reserved.
  * Copyright (c) 2009-2013 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 The University of Tennessee and The University
+ * Copyright (c) 2012-2023 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2012-2013 Inria.  All rights reserved.
@@ -59,8 +59,6 @@ int main(int argc, char **argv) {
     GAP_CHECK("c_contextid", test_comm, c_contextid, c_name, 1);
     GAP_CHECK("c_my_rank", test_comm, c_my_rank, c_contextid, 1);
     GAP_CHECK("c_flags", test_comm, c_flags, c_my_rank, 1);
-    GAP_CHECK("c_id_available", test_comm, c_id_available, c_flags, 1);
-    GAP_CHECK("c_id_start_index", test_comm, c_id_start_index, c_id_available,  1);
     GAP_CHECK("c_remote_group", test_comm, c_remote_group, c_local_group, 1);
     GAP_CHECK("c_local_comm", test_comm, c_local_comm,  c_remote_group,  1);
     GAP_CHECK("c_keyhash", test_comm, c_keyhash, c_local_comm,  1);
@@ -131,7 +129,6 @@ int main(int argc, char **argv) {
     GAP_CHECK("error_handler", test_win, error_handler, w_f_to_c_index, 1);
     GAP_CHECK("errhandler_type", test_win, errhandler_type, error_handler, 1);
     GAP_CHECK("w_osc_module", test_win, w_osc_module, errhandler_type, 1);
-
     /* Test Predefined info sizes */
     printf("=============================================\n");
     printf("ompi_predefined_info_t = %lu bytes\n", sizeof(ompi_predefined_info_t));
diff --git a/ompi/dpm/Makefile.am b/ompi/dpm/Makefile.am
index 002b050b568..02562525b9f 100644
--- a/ompi/dpm/Makefile.am
+++ b/ompi/dpm/Makefile.am
@@ -2,6 +2,7 @@
 #
 # Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
 # Copyright (c) 2016      IBM Corporation.  All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -11,11 +12,10 @@
 
 # This makefile.am does not stand on its own - it is included from ompi/Makefile.am
 
-dist_ompidata_DATA += dpm/help-dpm.txt
+EXTRA_DIST += dpm/help-dpm.txt
 
 headers += \
         dpm/dpm.h
 
 lib@OMPI_LIBMPI_NAME@_la_SOURCES += \
         dpm/dpm.c
-
diff --git a/ompi/dpm/dpm.c b/ompi/dpm/dpm.c
index 63d1e220009..4b5dbf623e1 100644
--- a/ompi/dpm/dpm.c
+++ b/ompi/dpm/dpm.c
@@ -3,14 +3,14 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2007-2020 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2007-2022 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2006-2009 University of Houston.  All rights reserved.
  * Copyright (c) 2009      Sun Microsystems, Inc.  All rights reserved.
  * Copyright (c) 2011-2015 Los Alamos National Security, LLC.  All rights
@@ -20,7 +20,11 @@
  * Copyright (c) 2014-2020 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
- * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2021-2024 Nanook Consulting  All rights reserved.
+ * Copyright (c) 2018-2022 Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -49,6 +53,7 @@
 #include "opal/util/show_help.h"
 #include "opal/util/printf.h"
 #include "opal/mca/hwloc/base/base.h"
+#include "opal/mca/installdirs/base/base.h"
 #include "opal/mca/pmix/base/base.h"
 
 #include "ompi/communicator/communicator.h"
@@ -104,7 +109,7 @@ int ompi_dpm_connect_accept(ompi_communicator_t *comm, int root,
     bool dense, isnew;
     opal_process_name_t pname;
     opal_list_t ilist, mlist, rlist;
-    pmix_info_t info;
+    pmix_info_t info, tinfo;
     pmix_value_t pval;
     pmix_pdata_t pdat;
     pmix_proc_t *procs, pxproc;
@@ -244,6 +249,7 @@ int ompi_dpm_connect_accept(ompi_communicator_t *comm, int root,
      * fail. */
     if (0 >= rportlen) {
         rc = rportlen;
+        /* no need to free here, the root has already done it and everyone else has not yet allocated the rport array */
         goto exit;
     }
 
@@ -266,6 +272,7 @@ int ompi_dpm_connect_accept(ompi_communicator_t *comm, int root,
     /* initiate a list of participants for the connect,
      * starting with our own members */
     OBJ_CONSTRUCT(&mlist, opal_list_t);
+    assert(NULL != members /* would mean comm had 0-sized group! */);
     for (i=0; NULL != members[i]; i++) {
         OPAL_PMIX_CONVERT_STRING_TO_PROCT(&pxproc, members[i]);
         plt = OBJ_NEW(opal_proclist_t);
@@ -373,7 +380,11 @@ int ompi_dpm_connect_accept(ompi_communicator_t *comm, int root,
     /* tell the host RTE to connect us - this will download
      * all known data for the nspace's of participating procs
      * so that add_procs will not result in a slew of lookups */
-    pret = PMIx_Connect(procs, nprocs, NULL, 0);
+    PMIX_INFO_CONSTRUCT(&tinfo);
+    PMIX_INFO_LOAD(&tinfo, PMIX_TIMEOUT, &ompi_pmix_connect_timeout, PMIX_UINT32);
+
+    pret = PMIx_Connect(procs, nprocs, &tinfo, 1);
+    PMIX_INFO_DESTRUCT(&tinfo);
     PMIX_PROC_FREE(procs, nprocs);
     rc = opal_pmix_convert_status(pret);
     if (OPAL_SUCCESS != rc) {
@@ -382,72 +393,85 @@ int ompi_dpm_connect_accept(ompi_communicator_t *comm, int root,
         OPAL_LIST_DESTRUCT(&rlist);
         goto exit;
     }
-    if (0 < opal_list_get_size(&ilist)) {
-        uint32_t *peer_ranks = NULL;
+    if (!opal_list_is_empty(&ilist)) {
         int prn, nprn = 0;
         char *val;
-        uint16_t u16;
         opal_process_name_t wildcard_rank;
+        i = 0;  /* start from the begining */
+
         /* convert the list of new procs to a proc_t array */
         new_proc_list = (ompi_proc_t**)calloc(opal_list_get_size(&ilist),
                                               sizeof(ompi_proc_t *));
-        /* get the list of local peers for the new procs */
-        cd = (ompi_dpm_proct_caddy_t*)opal_list_get_first(&ilist);
-        proc = cd->p;
-        wildcard_rank.jobid = proc->super.proc_name.jobid;
-        wildcard_rank.vpid = OMPI_NAME_WILDCARD->vpid;
-        /* retrieve the local peers */
-        OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_LOCAL_PEERS,
-                                       &wildcard_rank, &val, PMIX_STRING);
-        if (OPAL_SUCCESS == rc && NULL != val) {
-            char **peers = opal_argv_split(val, ',');
-            free(val);
-            nprn = opal_argv_count(peers);
-            peer_ranks = (uint32_t*)calloc(nprn, sizeof(uint32_t));
-            for (prn = 0; NULL != peers[prn]; prn++) {
-                peer_ranks[prn] = strtoul(peers[prn], NULL, 10);
-            }
-            opal_argv_free(peers);
-        }
-
-        i = 0;
-        OPAL_LIST_FOREACH(cd, &ilist, ompi_dpm_proct_caddy_t) {
+        /* Extract the modex info for the first proc on the ilist, and then
+         * remove all processors in the same jobid from the list by getting
+         * their connection information and moving them into the proc array.
+         */
+        do {
+            uint32_t *local_ranks_in_jobid = NULL;
+            ompi_dpm_proct_caddy_t* next = NULL;
+            cd = (ompi_dpm_proct_caddy_t*)opal_list_get_first(&ilist);
             proc = cd->p;
-            new_proc_list[i] = proc ;
-            /* ompi_proc_complete_init_single() initializes and optionally retrieves
-             * OPAL_PMIX_LOCALITY and OPAL_PMIX_HOSTNAME. since we can live without
-             * them, we are just fine */
-            ompi_proc_complete_init_single(proc);
-            /* if this proc is local, then get its locality */
-            if (NULL != peer_ranks) {
-                for (prn=0; prn < nprn; prn++) {
-                    if (peer_ranks[prn] == proc->super.proc_name.vpid) {
-                        /* get their locality string */
-                        val = NULL;
-                        OPAL_MODEX_RECV_VALUE_IMMEDIATE(rc, PMIX_LOCALITY_STRING,
-                                                       &proc->super.proc_name, &val, PMIX_STRING);
-                        if (OPAL_SUCCESS == rc && NULL != ompi_process_info.locality) {
-                            u16 = opal_hwloc_compute_relative_locality(ompi_process_info.locality, val);
-                            free(val);
-                        } else {
-                            /* all we can say is that it shares our node */
-                            u16 = OPAL_PROC_ON_CLUSTER | OPAL_PROC_ON_CU | OPAL_PROC_ON_NODE;
+            wildcard_rank.jobid = proc->super.proc_name.jobid;
+            wildcard_rank.vpid = OMPI_NAME_WILDCARD->vpid;
+            /* retrieve the local peers for the specified jobid */
+            OPAL_MODEX_RECV_VALUE_IMMEDIATE(rc, PMIX_LOCAL_PEERS,
+                                           &wildcard_rank, &val, PMIX_STRING);
+            if (OPAL_SUCCESS == rc && NULL != val) {
+                char **peers = opal_argv_split(val, ',');
+                free(val);
+                nprn = opal_argv_count(peers);
+                local_ranks_in_jobid = (uint32_t*)calloc(nprn, sizeof(uint32_t));
+                for (prn = 0; NULL != peers[prn]; prn++) {
+                    local_ranks_in_jobid[prn] = strtoul(peers[prn], NULL, 10);
+                }
+                opal_argv_free(peers);
+            }
+
+            OPAL_LIST_FOREACH_SAFE(cd, next, &ilist, ompi_dpm_proct_caddy_t) {
+                proc = cd->p;
+                if( proc->super.proc_name.jobid != wildcard_rank.jobid )
+                    continue;  /* not a proc from this jobid */
+
+                new_proc_list[i] = proc;
+                opal_list_remove_item(&ilist, (opal_list_item_t*)cd);  // TODO: do we need to release cd ?
+                OBJ_RELEASE(cd);
+                /* ompi_proc_complete_init_single() initializes and optionally retrieves
+                 * OPAL_PMIX_LOCALITY and OPAL_PMIX_HOSTNAME. since we can live without
+                 * them, we are just fine */
+                ompi_proc_complete_init_single(proc);
+                /* if this proc is local, then get its locality */
+                if (NULL != local_ranks_in_jobid) {
+                    uint16_t u16;
+                    for (prn=0; prn < nprn; prn++) {
+                        if (local_ranks_in_jobid[prn] == proc->super.proc_name.vpid) {
+                            /* get their locality string */
+                            val = NULL;
+                            OPAL_MODEX_RECV_VALUE_IMMEDIATE(rc, PMIX_LOCALITY_STRING,
+                                                           &proc->super.proc_name, &val, PMIX_STRING);
+                            if (OPAL_SUCCESS == rc && NULL != ompi_process_info.locality) {
+                                u16 = opal_hwloc_compute_relative_locality(ompi_process_info.locality, val);
+                                free(val);
+                            } else {
+                                /* all we can say is that it shares our node */
+                                u16 = OPAL_PROC_ON_CLUSTER | OPAL_PROC_ON_CU | OPAL_PROC_ON_NODE;
+                            }
+                            proc->super.proc_flags = u16;
+                            /* save the locality for later */
+                            OPAL_PMIX_CONVERT_NAME(&pxproc, &proc->super.proc_name);
+                            pval.type = PMIX_UINT16;
+                            pval.data.uint16 = proc->super.proc_flags;
+                            PMIx_Store_internal(&pxproc, PMIX_LOCALITY, &pval);
+                            break;
                         }
-                        proc->super.proc_flags = u16;
-                        /* save the locality for later */
-                        OPAL_PMIX_CONVERT_NAME(&pxproc, &proc->super.proc_name);
-                        pval.type = PMIX_UINT16;
-                        pval.data.uint16 = proc->super.proc_flags;
-                        PMIx_Store_internal(&pxproc, PMIX_LOCALITY, &pval);
-                        break;
                     }
                 }
+                ++i;
             }
-            ++i;
-        }
-        if (NULL != peer_ranks) {
-            free(peer_ranks);
-        }
+            if (NULL != local_ranks_in_jobid) {
+                free(local_ranks_in_jobid);
+            }
+        } while (!opal_list_is_empty(&ilist));
+
         /* call add_procs on the new ones */
         rc = MCA_PML_CALL(add_procs(new_proc_list, opal_list_get_size(&ilist)));
         free(new_proc_list);
@@ -462,7 +486,7 @@ int ompi_dpm_connect_accept(ompi_communicator_t *comm, int root,
 
     /* now deal with the remote group */
     rsize = opal_list_get_size(&rlist);
-    new_group_pointer=ompi_group_allocate(rsize);
+    new_group_pointer=ompi_group_allocate(NULL, rsize);
     if (NULL == new_group_pointer) {
         rc = OMPI_ERR_OUT_OF_RESOURCE;
         OPAL_LIST_DESTRUCT(&rlist);
@@ -486,10 +510,9 @@ int ompi_dpm_connect_accept(ompi_communicator_t *comm, int root,
                          NULL  ,                   /* remote_procs */
                          NULL,                     /* attrs */
                          comm->error_handler,      /* error handler */
-                         NULL,                     /* topo component */
                          group,                    /* local group */
-                         new_group_pointer         /* remote group */
-                         );
+                         new_group_pointer,        /* remote group */
+                         0);                       /* flags */
     if (OMPI_SUCCESS != rc) {
         goto exit;
     }
@@ -654,7 +677,7 @@ static dpm_conflicts_t bindto_modifiers[] = {
     {.name = ""}
 };
 
-static int check_modifiers(char *modifier, char **checks, dpm_conflicts_t *conflicts)
+static int check_modifiers(const char *modifier, char **checks, dpm_conflicts_t *conflicts)
 {
     int n, m, k;
 
@@ -674,17 +697,17 @@ static int check_modifiers(char *modifier, char **checks, dpm_conflicts_t *confl
 }
 
 static int dpm_convert(opal_list_t *infos,
-                       char *infokey,
-                       char *option,
-                       char *directive,
-                       char *modifier,
+                       const char *infokey,
+                       const char *option,
+                       const char *directive,
+                       const char *modifier,
                        bool deprecated)
 {
     opal_info_item_t *iptr;
     char *ck, *ptr, *help_str = NULL;
     int rc;
     char **tmp;
-    dpm_conflicts_t *modifiers;
+    dpm_conflicts_t *modifiers = NULL;
     const char *attr;
 
     /* pick the modifiers to be checked */
@@ -745,7 +768,10 @@ static int dpm_convert(opal_list_t *infos,
                         /* we have a conflict */
                         opal_asprintf(&ptr, "  Option %s\n  Conflicting modifiers \"%s %s\"", option, infokey, modifier);
 #if PMIX_NUMERIC_VERSION >= 0x00040000
-                        attr = PMIx_Get_attribute_string(option);
+                        /* TODO: remove strdup if PMIx_Get_attribute_string takes const char* */
+                        char *option_dup = strdup(option);
+                        attr = PMIx_Get_attribute_string(option_dup);
+                        free(option_dup);
 #else
                         attr = option;
 #endif
@@ -765,14 +791,16 @@ static int dpm_convert(opal_list_t *infos,
                     return OMPI_SUCCESS;
                 }
             }
+            free(ck);
         }
     }
 
     /**** Get here if the specified option is not found in the
      **** current list - add it
      ****/
-
-    if (NULL == directive) {
+    if (NULL == directive && NULL == modifier) {
+        return OMPI_ERR_BAD_PARAM;
+    } else if (NULL == directive) {
         opal_asprintf(&ptr, ":%s", modifier);
     } else if (NULL == modifier) {
         ptr = strdup(directive);
@@ -805,14 +833,7 @@ int ompi_dpm_spawn(int count, const char *array_of_commands[],
     int rc, i, j;
     int have_wdir=0;
     int flag=0;
-    char cwd[OPAL_PATH_MAX];
-    char host[OPAL_MAX_INFO_VAL];  /*** should define OMPI_HOST_MAX ***/
-    char init_errh[OPAL_MAX_INFO_VAL];
-    char prefix[OPAL_MAX_INFO_VAL];
-    char stdin_target[OPAL_MAX_INFO_VAL];
-    char params[OPAL_MAX_INFO_VAL];
-    char mapper[OPAL_MAX_INFO_VAL];
-    char slot_list[OPAL_MAX_INFO_VAL];
+    opal_cstring_t *info_str;
     uint32_t ui32;
     bool personality = false;
     char *tmp;
@@ -906,36 +927,35 @@ int ompi_dpm_spawn(int count, const char *array_of_commands[],
         /* Check for well-known info keys */
         have_wdir = 0;
         if ( array_of_info != NULL && array_of_info[i] != MPI_INFO_NULL ) {
-
             /* check for personality - this is a job-level key */
-            ompi_info_get (array_of_info[i], "personality", sizeof(host) - 1, host, &flag);
+            ompi_info_get (array_of_info[i], "personality", &info_str, &flag);
             if ( flag ) {
                 /* deprecate --> PMIX_PERSONALITY */
                 opal_show_help("help-dpm.txt", "deprecated-converted", true,
                                 "personality", "PMIX_PERSONALITY");
                 personality = true;
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_PERSONALITY, host, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_PERSONALITY, info_str->string, PMIX_STRING);
                 opal_list_append(&job_info, &info->super);
-                continue;
+                OBJ_RELEASE(info_str);
             }
-            ompi_info_get (array_of_info[i], "PMIX_PERSONALITY", sizeof(host) - 1, host, &flag);
+            ompi_info_get (array_of_info[i], "PMIX_PERSONALITY", &info_str, &flag);
             if ( flag ) {
                 personality = true;
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_PERSONALITY, host, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_PERSONALITY, info_str->string, PMIX_STRING);
                 opal_list_append(&job_info, &info->super);
-                continue;
+                OBJ_RELEASE(info_str);
             }
 #if PMIX_NUMERIC_VERSION >= 0x00040000
             checkkey = PMIx_Get_attribute_string("PMIX_PERSONALITY");
-            ompi_info_get (array_of_info[i], checkkey, sizeof(host) - 1, host, &flag);
+            ompi_info_get (array_of_info[i], checkkey, &info_str, &flag);
             if ( flag ) {
                 personality = true;
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_PERSONALITY, host, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_PERSONALITY, info_str->string, PMIX_STRING);
                 opal_list_append(&job_info, &info->super);
-                continue;
+                OBJ_RELEASE(info_str);
             }
 #endif
 
@@ -943,69 +963,74 @@ int ompi_dpm_spawn(int count, const char *array_of_commands[],
              * MPI standard ch. 10.3.4 */
 
             /* check for 'host' */
-            ompi_info_get (array_of_info[i], "host", sizeof(host) - 1, host, &flag);
+            ompi_info_get (array_of_info[i], "host", &info_str, &flag);
             if ( flag ) {
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_HOST, host, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_HOST, info_str->string, PMIX_STRING);
                 opal_list_append(&app_info, &info->super);
-                opal_argv_append_nosize(&dash_host, host);
-                continue;
+                opal_argv_append_nosize(&dash_host, info_str->string);
+                OBJ_RELEASE(info_str);
             }
-            ompi_info_get (array_of_info[i], "PMIX_HOST", sizeof(host) - 1, host, &flag);
+            ompi_info_get (array_of_info[i], "PMIX_HOST", &info_str, &flag);
             if ( flag ) {
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_HOST, host, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_HOST, info_str->string, PMIX_STRING);
                 opal_list_append(&app_info, &info->super);
-                opal_argv_append_nosize(&dash_host, host);
-                continue;
+                opal_argv_append_nosize(&dash_host, info_str->string);
+                OBJ_RELEASE(info_str);
             }
 #if PMIX_NUMERIC_VERSION >= 0x00040000
             checkkey = PMIx_Get_attribute_string("PMIX_HOST");
-            ompi_info_get (array_of_info[i], checkkey, sizeof(host) - 1, host, &flag);
+            ompi_info_get (array_of_info[i], checkkey, &info_str, &flag);
             if ( flag ) {
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_HOST, host, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_HOST, info_str->string, PMIX_STRING);
                 opal_list_append(&app_info, &info->super);
-                opal_argv_append_nosize(&dash_host, host);
-                continue;
+                opal_argv_append_nosize(&dash_host, info_str->string);
+                OBJ_RELEASE(info_str);
             }
 #endif
 
             /* check for 'wdir' */
-            ompi_info_get (array_of_info[i], "wdir", sizeof(cwd) - 1, cwd, &flag);
+            ompi_info_get (array_of_info[i], "wdir", &info_str, &flag);
             if ( flag ) {
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_WDIR, cwd, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_WDIR, info_str->string, PMIX_STRING);
                 opal_list_append(&app_info, &info->super);
+                OBJ_RELEASE(info_str);
                 have_wdir = 1;
             }
-            ompi_info_get (array_of_info[i], "PMIX_WDIR", sizeof(cwd) - 1, cwd, &flag);
-            if ( flag ) {
-                info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_WDIR, cwd, PMIX_STRING);
-                opal_list_append(&app_info, &info->super);
-                have_wdir = 1;
-                continue;
+            if (!have_wdir) {
+                ompi_info_get (array_of_info[i], "PMIX_WDIR", &info_str, &flag);
+                if ( flag ) {
+                    info = OBJ_NEW(opal_info_item_t);
+                    PMIX_INFO_LOAD(&info->info, PMIX_WDIR, info_str->string, PMIX_STRING);
+                    opal_list_append(&app_info, &info->super);
+                    OBJ_RELEASE(info_str);
+                    have_wdir = 1;
+                }
             }
 #if PMIX_NUMERIC_VERSION >= 0x00040000
-            checkkey = PMIx_Get_attribute_string("PMIX_WDIR");
-            ompi_info_get (array_of_info[i], checkkey, sizeof(cwd) - 1, cwd, &flag);
-            if ( flag ) {
-                info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_WDIR, cwd, PMIX_STRING);
-                opal_list_append(&app_info, &info->super);
-                have_wdir = 1;
-                continue;
+            if (!have_wdir) {
+                checkkey = PMIx_Get_attribute_string("PMIX_WDIR");
+                ompi_info_get (array_of_info[i], checkkey, &info_str, &flag);
+                if ( flag ) {
+                    info = OBJ_NEW(opal_info_item_t);
+                    PMIX_INFO_LOAD(&info->info, PMIX_WDIR, info_str->string, PMIX_STRING);
+                    opal_list_append(&app_info, &info->super);
+                    OBJ_RELEASE(info_str);
+                    have_wdir = 1;
+                }
             }
 #endif
 
             /* check for 'mpi_initial_errhandler' */
-            ompi_info_get (array_of_info[i], "mpi_initial_errhandler", sizeof(init_errh) - 1, init_errh, &flag);
+            ompi_info_get (array_of_info[i], "mpi_initial_errhandler", &info_str, &flag);
             if ( flag ) {
                 /* this is set as an environment because it must be available
                  * before pmix_init */
-                opal_setenv("OMPI_MCA_mpi_initial_errhandler", init_errh, true, &app->env);
-                continue;
+                opal_setenv("OMPI_MCA_mpi_initial_errhandler", info_str->string, true, &app->env);
+                OBJ_RELEASE(info_str);
             }
 
             /* 'path', 'arch', 'file', 'soft'  -- to be implemented */
@@ -1018,220 +1043,208 @@ int ompi_dpm_spawn(int count, const char *array_of_commands[],
              * deprecated in the non-prefixed form */
 
             /* check for 'hostfile' */
-            ompi_info_get (array_of_info[i], "hostfile", sizeof(host) - 1, host, &flag);
+            ompi_info_get (array_of_info[i], "hostfile", &info_str, &flag);
             if ( flag ) {
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_HOSTFILE, host, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_HOSTFILE, info_str->string, PMIX_STRING);
                 opal_list_append(&app_info, &info->super);
-                opal_argv_append_nosize(&hostfiles, host);
-                continue;
+                opal_argv_append_nosize(&hostfiles, info_str->string);
+                OBJ_RELEASE(info_str);
             }
-            ompi_info_get (array_of_info[i], "PMIX_HOSTFILE", sizeof(host) - 1, host, &flag);
+            ompi_info_get (array_of_info[i], "PMIX_HOSTFILE", &info_str, &flag);
             if ( flag ) {
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_HOSTFILE, host, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_HOSTFILE, info_str->string, PMIX_STRING);
                 opal_list_append(&app_info, &info->super);
-                opal_argv_append_nosize(&hostfiles, host);
-                continue;
+                opal_argv_append_nosize(&hostfiles, info_str->string);
+                OBJ_RELEASE(info_str);
             }
 #if PMIX_NUMERIC_VERSION >= 0x00040000
             checkkey = PMIx_Get_attribute_string("PMIX_HOSTFILE");
-            ompi_info_get (array_of_info[i], checkkey, sizeof(host) - 1, host, &flag);
+            ompi_info_get (array_of_info[i], checkkey, &info_str, &flag);
             if ( flag ) {
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_HOSTFILE, host, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_HOSTFILE, info_str->string, PMIX_STRING);
                 opal_list_append(&app_info, &info->super);
-                opal_argv_append_nosize(&hostfiles, host);
-                continue;
+                opal_argv_append_nosize(&hostfiles, info_str->string);
+                OBJ_RELEASE(info_str);
             }
 #endif
 
             /* check for 'add-hostfile' */
-            ompi_info_get (array_of_info[i], "add-hostfile", sizeof(host) - 1, host, &flag);
+            ompi_info_get (array_of_info[i], "add-hostfile", &info_str, &flag);
             if ( flag ) {
                 /* deprecate --> PMIX_ADD_HOSTFILE */
                 opal_show_help("help-dpm.txt", "deprecated-converted", true,
                                 "add-hostfile", "PMIX_ADD_HOSTFILE");
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_ADD_HOSTFILE, host, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_ADD_HOSTFILE, info_str->string, PMIX_STRING);
                 opal_list_append(&app_info, &info->super);
-                continue;
+                OBJ_RELEASE(info_str);
             }
-            ompi_info_get (array_of_info[i], "PMIX_ADD_HOSTFILE", sizeof(host) - 1, host, &flag);
+            ompi_info_get (array_of_info[i], "PMIX_ADD_HOSTFILE", &info_str, &flag);
             if ( flag ) {
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_ADD_HOSTFILE, host, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_ADD_HOSTFILE, info_str->string, PMIX_STRING);
                 opal_list_append(&app_info, &info->super);
-                continue;
+                OBJ_RELEASE(info_str);
             }
 #if PMIX_NUMERIC_VERSION >= 0x00040000
             checkkey = PMIx_Get_attribute_string("PMIX_ADD_HOSTFILE");
-            ompi_info_get (array_of_info[i], checkkey, sizeof(host) - 1, host, &flag);
+            ompi_info_get (array_of_info[i], checkkey, &info_str, &flag);
             if ( flag ) {
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_ADD_HOSTFILE, host, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_ADD_HOSTFILE, info_str->string, PMIX_STRING);
                 opal_list_append(&app_info, &info->super);
-                continue;
+                OBJ_RELEASE(info_str);
             }
 #endif
 
             /* check for 'add-host' */
-            ompi_info_get (array_of_info[i], "add-host", sizeof(host) - 1, host, &flag);
+            ompi_info_get (array_of_info[i], "add-host", &info_str, &flag);
             if ( flag ) {
                 /* deprecate --> PMIX_ADD_HOST */
                 opal_show_help("help-dpm.txt", "deprecated-converted", true,
                                 "add-host", "PMIX_ADD_HOST");
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_ADD_HOST, host, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_ADD_HOST, info_str->string, PMIX_STRING);
                 opal_list_append(&app_info, &info->super);
-                continue;
+                OBJ_RELEASE(info_str);
             }
-            ompi_info_get (array_of_info[i], "PMIX_ADD_HOST", sizeof(host) - 1, host, &flag);
+            ompi_info_get (array_of_info[i], "PMIX_ADD_HOST", &info_str, &flag);
             if ( flag ) {
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_ADD_HOST, host, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_ADD_HOST, info_str->string, PMIX_STRING);
                 opal_list_append(&app_info, &info->super);
-                continue;
+                OBJ_RELEASE(info_str);
             }
 #if PMIX_NUMERIC_VERSION >= 0x00040000
             checkkey = PMIx_Get_attribute_string("PMIX_ADD_HOST");
-            ompi_info_get (array_of_info[i], checkkey, sizeof(host) - 1, host, &flag);
+            ompi_info_get (array_of_info[i], checkkey, &info_str, &flag);
             if ( flag ) {
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_ADD_HOST, host, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_ADD_HOST, info_str->string, PMIX_STRING);
                 opal_list_append(&app_info, &info->super);
-                continue;
+                OBJ_RELEASE(info_str);
             }
 #endif
 
             /* check for env */
-            ompi_info_get (array_of_info[i], "env", sizeof(host)-1, host, &flag);
+            ompi_info_get (array_of_info[i], "env", &info_str, &flag);
             if ( flag ) {
                 /* deprecate --> PMIX_ENVAR */
                 opal_show_help("help-dpm.txt", "deprecated-converted", true,
                                 "env", "PMIX_ENVAR");
-                envars = opal_argv_split(host, '\n');
+                envars = opal_argv_split(info_str->string, '\n');
+                OBJ_RELEASE(info_str);
                 for (j=0; NULL != envars[j]; j++) {
                     opal_argv_append_nosize(&app->env, envars[j]);
                 }
                 opal_argv_free(envars);
-                continue;
             }
-            ompi_info_get (array_of_info[i], "PMIX_ENVAR", sizeof(host)-1, host, &flag);
+            ompi_info_get (array_of_info[i], "PMIX_ENVAR", &info_str, &flag);
             if ( flag ) {
-                envars = opal_argv_split(host, '\n');
+                envars = opal_argv_split(info_str->string, '\n');
+                OBJ_RELEASE(info_str);
                 for (j=0; NULL != envars[j]; j++) {
                     opal_argv_append_nosize(&app->env, envars[j]);
                 }
                 opal_argv_free(envars);
-                continue;
             }
 #if PMIX_NUMERIC_VERSION >= 0x00040000
             checkkey = PMIx_Get_attribute_string("PMIX_ENVAR");
-            ompi_info_get (array_of_info[i], "PMIX_ENVAR", sizeof(host)-1, host, &flag);
+            ompi_info_get (array_of_info[i], "PMIX_ENVAR", &info_str, &flag);
             if ( flag ) {
-                envars = opal_argv_split(host, '\n');
+                envars = opal_argv_split(info_str->string, '\n');
+                OBJ_RELEASE(info_str);
                 for (j=0; NULL != envars[j]; j++) {
                     opal_argv_append_nosize(&app->env, envars[j]);
                 }
                 opal_argv_free(envars);
-                continue;
             }
 #endif
 
             /* check for 'ompi_prefix' (OMPI-specific -- to effect the same
-             * behavior as --prefix option to orterun)
+             * behavior as --prefix option to prun)
              *
              * This is a job-level key
              */
-            ompi_info_get (array_of_info[i], "ompi_prefix", sizeof(prefix) - 1, prefix, &flag);
+            ompi_info_get (array_of_info[i], "ompi_prefix", &info_str, &flag);
             if ( flag ) {
                 /* deprecate --> PMIX_PREFIX */
                 opal_show_help("help-dpm.txt", "deprecated-converted", true,
                                 "ompi_prefix", "PMIX_PREFIX");
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_PREFIX, prefix, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_PREFIX, info_str->string, PMIX_STRING);
                 opal_list_append(&job_info, &info->super);
-                continue;
+                OBJ_RELEASE(info_str);
             }
-            ompi_info_get (array_of_info[i], "PMIX_PREFIX", sizeof(prefix) - 1, prefix, &flag);
+            ompi_info_get (array_of_info[i], "PMIX_PREFIX", &info_str, &flag);
             if ( flag ) {
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_PREFIX, prefix, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_PREFIX, info_str->string, PMIX_STRING);
                 opal_list_append(&job_info, &info->super);
-                continue;
+                OBJ_RELEASE(info_str);
             }
 #if PMIX_NUMERIC_VERSION >= 0x00040000
             checkkey = PMIx_Get_attribute_string("PMIX_PREFIX");
-            ompi_info_get (array_of_info[i], checkkey, sizeof(prefix) - 1, prefix, &flag);
+            ompi_info_get (array_of_info[i], checkkey, &info_str, &flag);
             if ( flag ) {
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_PREFIX, prefix, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_PREFIX, info_str->string, PMIX_STRING);
                 opal_list_append(&job_info, &info->super);
-                continue;
+                OBJ_RELEASE(info_str);
             }
 #endif
 
             /* check for 'mapper' - a job-level key */
-            ompi_info_get(array_of_info[i], "mapper", sizeof(mapper) - 1, mapper, &flag);
+            ompi_info_get(array_of_info[i], "mapper", &info_str, &flag);
             if ( flag ) {
                 /* deprecate --> PMIX_MAPPER */
                 opal_show_help("help-dpm.txt", "deprecated-converted", true,
                                 "mapper", "PMIX_MAPPER");
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_MAPPER, mapper, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_MAPPER, info_str->string, PMIX_STRING);
                 opal_list_append(&job_info, &info->super);
-                continue;
+                OBJ_RELEASE(info_str);
             }
-            ompi_info_get(array_of_info[i], "PMIX_MAPPER", sizeof(mapper) - 1, mapper, &flag);
+            ompi_info_get(array_of_info[i], "PMIX_MAPPER", &info_str, &flag);
             if ( flag ) {
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_MAPPER, mapper, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_MAPPER, info_str->string, PMIX_STRING);
                 opal_list_append(&job_info, &info->super);
-                continue;
+                OBJ_RELEASE(info_str);
             }
 #if PMIX_NUMERIC_VERSION >= 0x00040000
             checkkey = PMIx_Get_attribute_string("PMIX_MAPPER");
-            ompi_info_get(array_of_info[i], checkkey, sizeof(mapper) - 1, mapper, &flag);
+            ompi_info_get(array_of_info[i], checkkey, &info_str, &flag);
             if ( flag ) {
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_MAPPER, mapper, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_MAPPER, info_str->string, PMIX_STRING);
                 opal_list_append(&job_info, &info->super);
-                continue;
+                OBJ_RELEASE(info_str);
             }
 #endif
 
             /* check for 'display_map' - a job-level key */
             ompi_info_get_bool(array_of_info[i], "display_map", &local_spawn, &flag);
             if ( flag ) {
-                rc = dpm_convert(&job_info, "display_map", PMIX_MAPBY, NULL, "DISPLAYMAP", true);
-                if (OMPI_SUCCESS != rc) {
-                    OPAL_LIST_DESTRUCT(&job_info);
-                    OPAL_LIST_DESTRUCT(&app_info);
-                    PMIX_APP_FREE(apps, scount);
-                    opal_progress_event_users_decrement();
-                    if (NULL != hostfiles) {
-                        opal_argv_free(hostfiles);
-                    }
-                    if (NULL != dash_host) {
-                        opal_argv_free(dash_host);
-                    }
-                    return MPI_ERR_SPAWN;
-                }
-                continue;
+                info = OBJ_NEW(opal_info_item_t);
+                PMIX_INFO_LOAD(&info->info, PMIX_DISPLAY_MAP, &local_spawn, PMIX_BOOL);
+                opal_list_append(&job_info, &info->super);
             }
 
             /* check for 'npernode' and 'ppr' - job-level key */
-            ompi_info_get (array_of_info[i], "npernode", sizeof(slot_list) - 1, slot_list, &flag);
+            ompi_info_get (array_of_info[i], "npernode", &info_str, &flag);
             if ( flag ) {
-                opal_asprintf(&tmp, "PPR:%s:NODE", slot_list);
+                opal_asprintf(&tmp, "PPR:%s:NODE", info_str->string);
                 rc = dpm_convert(&job_info, "npernode", PMIX_MAPBY, tmp, NULL, true);
                 free(tmp);
+                OBJ_RELEASE(info_str);
                 if (OMPI_SUCCESS != rc) {
                     OPAL_LIST_DESTRUCT(&job_info);
                     OPAL_LIST_DESTRUCT(&app_info);
                     PMIX_APP_FREE(apps, scount);
-                    opal_progress_event_users_decrement();
                     if (NULL != hostfiles) {
                         opal_argv_free(hostfiles);
                     }
@@ -1240,17 +1253,15 @@ int ompi_dpm_spawn(int count, const char *array_of_commands[],
                     }
                     return MPI_ERR_SPAWN;
                 }
-                continue;
             }
-            ompi_info_get (array_of_info[i], "pernode", sizeof(slot_list) - 1, slot_list, &flag);
+            ompi_info_get (array_of_info[i], "pernode", &info_str, &flag);
             if ( flag ) {
                 rc = dpm_convert(&job_info, "pernode", PMIX_MAPBY, "PPR:1:NODE", NULL, true);
-                free(tmp);
+                OBJ_RELEASE(info_str);
                 if (OMPI_SUCCESS != rc) {
                     OPAL_LIST_DESTRUCT(&job_info);
                     OPAL_LIST_DESTRUCT(&app_info);
                     PMIX_APP_FREE(apps, scount);
-                    opal_progress_event_users_decrement();
                     if (NULL != hostfiles) {
                         opal_argv_free(hostfiles);
                     }
@@ -1259,47 +1270,45 @@ int ompi_dpm_spawn(int count, const char *array_of_commands[],
                     }
                     return MPI_ERR_SPAWN;
                 }
-                continue;
             }
-            ompi_info_get (array_of_info[i], "ppr", sizeof(slot_list) - 1, slot_list, &flag);
+            ompi_info_get (array_of_info[i], "ppr", &info_str, &flag);
             if ( flag ) {
                 /* must have correct syntax with two colons */
-                if (NULL == (tmp = strchr(slot_list, ':'))) {
-                    opal_show_help("help-dpm.txt", "bad-ppr", true, slot_list);
+                if (NULL == (tmp = strchr(info_str->string, ':'))) {
+                    opal_show_help("help-dpm.txt", "bad-ppr", true, info_str->string);
                     OPAL_LIST_DESTRUCT(&job_info);
                     OPAL_LIST_DESTRUCT(&app_info);
                     PMIX_APP_FREE(apps, scount);
-                    opal_progress_event_users_decrement();
                     if (NULL != hostfiles) {
                         opal_argv_free(hostfiles);
                     }
                     if (NULL != dash_host) {
                         opal_argv_free(dash_host);
                     }
+                    OBJ_RELEASE(info_str);
                     return MPI_ERR_SPAWN;
                 }
                 ++tmp; // step over first colon
                 if (NULL == strchr(tmp, ':')) {
-                    opal_show_help("help-dpm.txt", "bad-ppr", true, slot_list);
+                    opal_show_help("help-dpm.txt", "bad-ppr", true, info_str->string);
                     OPAL_LIST_DESTRUCT(&job_info);
                     OPAL_LIST_DESTRUCT(&app_info);
                     PMIX_APP_FREE(apps, scount);
-                    opal_progress_event_users_decrement();
                     if (NULL != hostfiles) {
                         opal_argv_free(hostfiles);
                     }
                     if (NULL != dash_host) {
                         opal_argv_free(dash_host);
                     }
+                    OBJ_RELEASE(info_str);
                     return MPI_ERR_SPAWN;
                 }
-                rc = dpm_convert(&job_info, "ppr", PMIX_MAPBY, slot_list, NULL, true);
-                free(tmp);
+                rc = dpm_convert(&job_info, "ppr", PMIX_MAPBY, info_str->string, NULL, true);
+                OBJ_RELEASE(info_str);
                 if (OMPI_SUCCESS != rc) {
                     OPAL_LIST_DESTRUCT(&job_info);
                     OPAL_LIST_DESTRUCT(&app_info);
                     PMIX_APP_FREE(apps, scount);
-                    opal_progress_event_users_decrement();
                     if (NULL != hostfiles) {
                         opal_argv_free(hostfiles);
                     }
@@ -1308,19 +1317,17 @@ int ompi_dpm_spawn(int count, const char *array_of_commands[],
                     }
                     return MPI_ERR_SPAWN;
                 }
-                continue;
             }
 
             /* check for 'map_by' - job-level key */
-            ompi_info_get(array_of_info[i], "map_by", sizeof(slot_list) - 1, slot_list, &flag);
+            ompi_info_get(array_of_info[i], "map_by", &info_str, &flag);
             if ( flag ) {
-                rc = dpm_convert(&job_info, "map_by", PMIX_MAPBY, slot_list, NULL, false);
-                free(tmp);
+                rc = dpm_convert(&job_info, "map_by", PMIX_MAPBY, info_str->string, NULL, false);
+                OBJ_RELEASE(info_str);
                 if (OMPI_SUCCESS != rc) {
                     OPAL_LIST_DESTRUCT(&job_info);
                     OPAL_LIST_DESTRUCT(&app_info);
                     PMIX_APP_FREE(apps, scount);
-                    opal_progress_event_users_decrement();
                     if (NULL != hostfiles) {
                         opal_argv_free(hostfiles);
                     }
@@ -1329,87 +1336,82 @@ int ompi_dpm_spawn(int count, const char *array_of_commands[],
                     }
                     return MPI_ERR_SPAWN;
                 }
-                continue;
             }
-            ompi_info_get(array_of_info[i], "PMIX_MAPBY", sizeof(slot_list) - 1, slot_list, &flag);
+            ompi_info_get(array_of_info[i], "PMIX_MAPBY", &info_str, &flag);
             if ( flag ) {
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_MAPBY, slot_list, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_MAPBY, info_str->string, PMIX_STRING);
                 opal_list_append(&job_info, &info->super);
-                continue;
+                OBJ_RELEASE(info_str);
             }
 #if PMIX_NUMERIC_VERSION >= 0x00040000
             checkkey = PMIx_Get_attribute_string("PMIX_MAPBY");
-            ompi_info_get(array_of_info[i], checkkey, sizeof(slot_list) - 1, slot_list, &flag);
+            ompi_info_get(array_of_info[i], checkkey, &info_str, &flag);
             if ( flag ) {
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_MAPBY, slot_list, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_MAPBY, info_str->string, PMIX_STRING);
                 opal_list_append(&job_info, &info->super);
-                continue;
+                OBJ_RELEASE(info_str);
             }
 #endif
 
             /* check for 'rank_by' - job-level key */
-            ompi_info_get(array_of_info[i], "rank_by", sizeof(slot_list) - 1, slot_list, &flag);
+            ompi_info_get(array_of_info[i], "rank_by", &info_str, &flag);
             if ( flag ) {
-                rc = dpm_convert(&job_info, "rank_by", PMIX_RANKBY, slot_list, NULL, false);
-                free(tmp);
+                rc = dpm_convert(&job_info, "rank_by", PMIX_RANKBY, info_str->string, NULL, false);
+                OBJ_RELEASE(info_str);
                 if (OMPI_SUCCESS != rc) {
                     OPAL_LIST_DESTRUCT(&job_info);
                     OPAL_LIST_DESTRUCT(&app_info);
                     PMIX_APP_FREE(apps, scount);
-                    opal_progress_event_users_decrement();
                     return MPI_ERR_SPAWN;
                 }
-                continue;
             }
-            ompi_info_get(array_of_info[i], "PMIX_RANKBY", sizeof(slot_list) - 1, slot_list, &flag);
+            ompi_info_get(array_of_info[i], "PMIX_RANKBY", &info_str, &flag);
             if ( flag ) {
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_RANKBY, slot_list, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_RANKBY, info_str->string, PMIX_STRING);
                 opal_list_append(&job_info, &info->super);
-                continue;
+                OBJ_RELEASE(info_str);
             }
 #if PMIX_NUMERIC_VERSION >= 0x00040000
             checkkey = PMIx_Get_attribute_string("PMIX_RANKBY");
-            ompi_info_get(array_of_info[i], checkkey, sizeof(slot_list) - 1, slot_list, &flag);
+            ompi_info_get(array_of_info[i], checkkey, &info_str, &flag);
             if ( flag ) {
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_RANKBY, slot_list, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_RANKBY, info_str->string, PMIX_STRING);
                 opal_list_append(&job_info, &info->super);
-                continue;
+                OBJ_RELEASE(info_str);
             }
 #endif
 
             /* check for 'bind_to' - job-level key */
-            ompi_info_get(array_of_info[i], "bind_to", sizeof(slot_list) - 1, slot_list, &flag);
+            ompi_info_get(array_of_info[i], "bind_to", &info_str, &flag);
             if ( flag ) {
-                rc = dpm_convert(&job_info, "bind_to", PMIX_BINDTO, slot_list, NULL, false);
-                free(tmp);
+                rc = dpm_convert(&job_info, "bind_to", PMIX_BINDTO, info_str->string, NULL, false);
+                OBJ_RELEASE(info_str);
                 if (OMPI_SUCCESS != rc) {
                     OPAL_LIST_DESTRUCT(&job_info);
                     OPAL_LIST_DESTRUCT(&app_info);
                     PMIX_APP_FREE(apps, scount);
-                    opal_progress_event_users_decrement();
                     return MPI_ERR_SPAWN;
                 }
-                continue;
             }
-            ompi_info_get(array_of_info[i], "PMIX_BINDTO", sizeof(slot_list) - 1, slot_list, &flag);
+            ompi_info_get(array_of_info[i], "PMIX_BINDTO", &info_str, &flag);
             if ( flag ) {
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_BINDTO, slot_list, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_BINDTO, info_str->string, PMIX_STRING);
                 opal_list_append(&job_info, &info->super);
-                continue;
+                OBJ_RELEASE(info_str);
             }
 #if PMIX_NUMERIC_VERSION >= 0x00040000
             checkkey = PMIx_Get_attribute_string("PMIX_BINDTO");
-            ompi_info_get(array_of_info[i], checkkey, sizeof(slot_list) - 1, slot_list, &flag);
+            ompi_info_get(array_of_info[i], checkkey, &info_str, &flag);
             if ( flag ) {
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_BINDTO, slot_list, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_BINDTO, info_str->string, PMIX_STRING);
                 opal_list_append(&job_info, &info->super);
-                continue;
+                OBJ_RELEASE(info_str);
             }
 #endif
 
@@ -1422,14 +1424,12 @@ int ompi_dpm_spawn(int count, const char *array_of_commands[],
                 info = OBJ_NEW(opal_info_item_t);
                 PMIX_INFO_LOAD(&info->info, PMIX_PRELOAD_BIN, &local_spawn, PMIX_BOOL);
                 opal_list_append(&job_info, &info->super);
-                continue;
             }
             ompi_info_get_bool(array_of_info[i], "PMIX_PRELOAD_BIN", &local_spawn, &flag);
             if ( flag ) {
                 info = OBJ_NEW(opal_info_item_t);
                 PMIX_INFO_LOAD(&info->info, PMIX_PRELOAD_BIN, &local_spawn, PMIX_BOOL);
                 opal_list_append(&job_info, &info->super);
-                continue;
             }
 #if PMIX_NUMERIC_VERSION >= 0x00040000
             checkkey = PMIx_Get_attribute_string("PMIX_PRELOAD_BIN");
@@ -1438,106 +1438,105 @@ int ompi_dpm_spawn(int count, const char *array_of_commands[],
                 info = OBJ_NEW(opal_info_item_t);
                 PMIX_INFO_LOAD(&info->info, PMIX_PRELOAD_BIN, &local_spawn, PMIX_BOOL);
                 opal_list_append(&job_info, &info->super);
-                continue;
             }
 #endif
 
             /* check for 'preload_files' - job-level key */
-            ompi_info_get (array_of_info[i], "ompi_preload_files", sizeof(cwd) - 1, cwd, &flag);
+            ompi_info_get (array_of_info[i], "ompi_preload_files", &info_str, &flag);
             if ( flag ) {
                 /* deprecate --> PMIX_PRELOAD_FILES */
                 opal_show_help("help-dpm.txt", "deprecated-converted", true,
                                 "ompi_preload_files", "PMIX_PRELOAD_FILES");
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_PRELOAD_FILES, cwd, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_PRELOAD_FILES, info_str->string, PMIX_STRING);
                 opal_list_append(&job_info, &info->super);
-                continue;
+                OBJ_RELEASE(info_str);
             }
-            ompi_info_get (array_of_info[i], "PMIX_PRELOAD_FILES", sizeof(cwd) - 1, cwd, &flag);
+            ompi_info_get (array_of_info[i], "PMIX_PRELOAD_FILES", &info_str, &flag);
             if ( flag ) {
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_PRELOAD_FILES, cwd, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_PRELOAD_FILES, info_str->string, PMIX_STRING);
                 opal_list_append(&job_info, &info->super);
-                continue;
+                OBJ_RELEASE(info_str);
             }
 #if PMIX_NUMERIC_VERSION >= 0x00040000
             checkkey = PMIx_Get_attribute_string("PMIX_PRELOAD_FILES");
-            ompi_info_get (array_of_info[i], checkkey, sizeof(cwd) - 1, cwd, &flag);
+            ompi_info_get (array_of_info[i], checkkey, &info_str, &flag);
             if ( flag ) {
                 info = OBJ_NEW(opal_info_item_t);
-                PMIX_INFO_LOAD(&info->info, PMIX_PRELOAD_FILES, cwd, PMIX_STRING);
+                PMIX_INFO_LOAD(&info->info, PMIX_PRELOAD_FILES, info_str->string, PMIX_STRING);
                 opal_list_append(&job_info, &info->super);
-                continue;
+                OBJ_RELEASE(info_str);
             }
 #endif
 
-            /* see if this is a non-mpi job - if so, then set the flag so ORTE
+            /* see if this is a non-mpi job - if so, then set the flag so PRTE
              * knows what to do - job-level key
              */
             ompi_info_get_bool(array_of_info[i], "ompi_non_mpi", &non_mpi, &flag);
             if (flag && non_mpi) {
                 opal_show_help("help-dpm.txt", "deprecated-inform", true,
                                 "ompi_non_mpi", "No longer relevant as RTE automatically detects this scenario");
-                continue;
             }
 
             /* see if this is an MCA param that the user wants applied to the child job */
-            ompi_info_get (array_of_info[i], "ompi_param", sizeof(params) - 1, params, &flag);
+            ompi_info_get (array_of_info[i], "ompi_param", &info_str, &flag);
             if ( flag ) {
                 opal_show_help("help-dpm.txt", "deprecated-converted", true,
                                 "ompi_param", "PMIX_ENVAR");
-                opal_argv_append_unique_nosize(&app->env, params, true);
+                opal_argv_append_unique_nosize(&app->env, info_str->string, true);
+                OBJ_RELEASE(info_str);
             }
 
             /* see if user specified what to do with stdin - defaults to
              * not forwarding stdin to child processes - job-level key
              */
-            ompi_info_get (array_of_info[i], "ompi_stdin_target", sizeof(stdin_target) - 1, stdin_target, &flag);
+            ompi_info_get (array_of_info[i], "ompi_stdin_target", &info_str, &flag);
             if ( flag ) {
                 /* deprecate --> PMIX_STDIN_TGT */
                 opal_show_help("help-dpm.txt", "deprecated-converted", true,
                                 "ompi_stdin_target", "PMIX_STDIN_TGT");
-                if (0 == strcmp(stdin_target, "all")) {
+                if (0 == strcmp(info_str->string, "all")) {
                     ui32 = OPAL_VPID_WILDCARD;
-                } else if (0 == strcmp(stdin_target, "none")) {
+                } else if (0 == strcmp(info_str->string, "none")) {
                     ui32 = OPAL_VPID_INVALID;
                 } else {
-                    ui32 = strtoul(stdin_target, NULL, 10);
+                    ui32 = strtoul(info_str->string, NULL, 10);
                 }
                 info = OBJ_NEW(opal_info_item_t);
                 PMIX_INFO_LOAD(&info->info, PMIX_STDIN_TGT, &ui32, PMIX_UINT32);
                 opal_list_append(&job_info, &info->super);
-                continue;
+                OBJ_RELEASE(info_str);
             }
-            ompi_info_get (array_of_info[i], "PMIX_STDIN_TGT", sizeof(stdin_target) - 1, stdin_target, &flag);
+            ompi_info_get (array_of_info[i], "PMIX_STDIN_TGT", &info_str, &flag);
             if ( flag ) {
-                if (0 == strcmp(stdin_target, "all")) {
+                if (0 == strcmp(info_str->string, "all")) {
                     ui32 = OPAL_VPID_WILDCARD;
-                } else if (0 == strcmp(stdin_target, "none")) {
+                } else if (0 == strcmp(info_str->string, "none")) {
                     ui32 = OPAL_VPID_INVALID;
                 } else {
-                    ui32 = strtoul(stdin_target, NULL, 10);
+                    ui32 = strtoul(info_str->string, NULL, 10);
                 }
                 info = OBJ_NEW(opal_info_item_t);
                 PMIX_INFO_LOAD(&info->info, PMIX_STDIN_TGT, &ui32, PMIX_UINT32);
                 opal_list_append(&job_info, &info->super);
-                continue;
+                OBJ_RELEASE(info_str);
             }
 #if PMIX_NUMERIC_VERSION >= 0x00040000
             checkkey = PMIx_Get_attribute_string("PMIX_STDIN_TGT");
-            ompi_info_get (array_of_info[i], checkkey, sizeof(stdin_target) - 1, stdin_target, &flag);
+            ompi_info_get (array_of_info[i], checkkey, &info_str, &flag);
             if ( flag ) {
-                if (0 == strcmp(stdin_target, "all")) {
+                if (0 == strcmp(info_str->string, "all")) {
                     ui32 = OPAL_VPID_WILDCARD;
-                } else if (0 == strcmp(stdin_target, "none")) {
+                } else if (0 == strcmp(info_str->string, "none")) {
                     ui32 = OPAL_VPID_INVALID;
                 } else {
-                    ui32 = strtoul(stdin_target, NULL, 10);
+                    ui32 = strtoul(info_str->string, NULL, 10);
                 }
                 info = OBJ_NEW(opal_info_item_t);
                 PMIX_INFO_LOAD(&info->info, PMIX_STDIN_TGT, &ui32, PMIX_UINT32);
                 opal_list_append(&job_info, &info->super);
-                continue;
+                OBJ_RELEASE(info_str);
             }
 #endif
         }
@@ -1546,10 +1545,10 @@ int ompi_dpm_spawn(int count, const char *array_of_commands[],
          * executable, we assume the current working directory
          */
         if ( !have_wdir ) {
+            char cwd[OPAL_PATH_MAX];
             if (OMPI_SUCCESS != (rc = opal_getcwd(cwd, OPAL_PATH_MAX))) {
                 OMPI_ERROR_LOG(rc);
                 PMIX_APP_FREE(apps, (size_t)count);
-                opal_progress_event_users_decrement();
                 if (NULL != hostfiles) {
                     opal_argv_free(hostfiles);
                 }
@@ -1587,26 +1586,18 @@ int ompi_dpm_spawn(int count, const char *array_of_commands[],
         opal_list_append(&job_info, &info->super);
     }
 
-    /* spawn procs */
-    ninfo = opal_list_get_size(&job_info);
-    if (0 < ninfo) {
-        PMIX_INFO_CREATE(pinfo, ninfo);
-        n = 0;
-        OPAL_LIST_FOREACH(info, &job_info, opal_info_item_t) {
-            PMIX_INFO_XFER(&pinfo[n], &info->info);
-            ++n;
-        }
-    }
-    OPAL_LIST_DESTRUCT(&job_info);
-
-    if (ompi_singleton) {
+    if (opal_process_info.is_singleton) {
+        /* The GDS 'hash' component is known to work for singleton, so
+         * recommend it. The user may set this envar to override the setting.
+         */
+        setenv("PMIX_MCA_gds", "hash", 0);
+        /* Start the DVM */
         rc = start_dvm(hostfiles, dash_host);
         if (OPAL_SUCCESS != rc) {
             if (NULL != pinfo) {
                 PMIX_INFO_FREE(pinfo, ninfo);
             }
             PMIX_APP_FREE(apps, scount);
-            opal_progress_event_users_decrement();
             if (NULL != hostfiles) {
                 opal_argv_free(hostfiles);
             }
@@ -1615,6 +1606,16 @@ int ompi_dpm_spawn(int count, const char *array_of_commands[],
             }
             return MPI_ERR_SPAWN;
         }
+        /* tell it to forward output to us */
+        info = OBJ_NEW(opal_info_item_t);
+        PMIX_INFO_LOAD(&info->info, PMIX_FWD_STDOUT, NULL, PMIX_BOOL);
+        opal_list_append(&job_info, &info->super);
+        info = OBJ_NEW(opal_info_item_t);
+        PMIX_INFO_LOAD(&info->info, PMIX_FWD_STDERR, NULL, PMIX_BOOL);
+        opal_list_append(&job_info, &info->super);
+        info = OBJ_NEW(opal_info_item_t);
+        PMIX_INFO_LOAD(&info->info, PMIX_FWD_STDDIAG, NULL, PMIX_BOOL);
+        opal_list_append(&job_info, &info->super);
     }
     if (NULL != hostfiles) {
         opal_argv_free(hostfiles);
@@ -1623,6 +1624,18 @@ int ompi_dpm_spawn(int count, const char *array_of_commands[],
         opal_argv_free(dash_host);
     }
 
+    /* spawn procs */
+    ninfo = opal_list_get_size(&job_info);
+    if (0 < ninfo) {
+        PMIX_INFO_CREATE(pinfo, ninfo);
+        n = 0;
+        OPAL_LIST_FOREACH(info, &job_info, opal_info_item_t) {
+            PMIX_INFO_XFER(&pinfo[n], &info->info);
+            ++n;
+        }
+    }
+    OPAL_LIST_DESTRUCT(&job_info);
+
     pret = PMIx_Spawn(pinfo, ninfo, apps, count, nspace);
     rc = opal_pmix_convert_status(pret);
     if (NULL != pinfo) {
@@ -1631,7 +1644,6 @@ int ompi_dpm_spawn(int count, const char *array_of_commands[],
     PMIX_APP_FREE(apps, scount);
 
     if (OPAL_SUCCESS != rc) {
-        opal_progress_event_users_decrement();
         return MPI_ERR_SPAWN;
     }
 
@@ -1684,6 +1696,9 @@ int ompi_dpm_dyn_init(void)
         ptr = &tmp[0];
     }
     port_name = strdup(ptr);
+    if (NULL == port_name) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
 
     rc = ompi_dpm_connect_accept(MPI_COMM_WORLD, root, port_name, send_first, &newcomm);
     free(port_name);
@@ -1709,15 +1724,6 @@ int ompi_dpm_dyn_init(void)
     return OMPI_SUCCESS;
 }
 
-
-/*
- * finalize the module
- */
-int ompi_dpm_finalize(void)
-{
-    return OMPI_SUCCESS;
-}
-
 static void cleanup_dpm_disconnect_objs(ompi_dpm_disconnect_obj **objs, int count)
 {
     for(int i = 0; i < count; i++) {
@@ -1747,9 +1753,9 @@ int ompi_dpm_dyn_finalize(void)
             return OMPI_ERR_OUT_OF_RESOURCE;
         }
 
-        max = opal_pointer_array_get_size(&ompi_mpi_communicators);
+        max = ompi_comm_get_num_communicators();
         for (i=3; i<max; i++) {
-            comm = (ompi_communicator_t*)opal_pointer_array_get_item(&ompi_mpi_communicators,i);
+            comm = ompi_comm_lookup(i);
             if (NULL != comm &&  OMPI_COMM_IS_DYNAMIC(comm)) {
                 objs[j++] = disconnect_init(comm);
             }
@@ -1761,6 +1767,7 @@ int ompi_dpm_dyn_finalize(void)
         }
 
         disconnect_waitall(ompi_comm_num_dyncomm, objs);
+        cleanup_dpm_disconnect_objs(objs, ompi_comm_num_dyncomm);
     }
 
     return OMPI_SUCCESS;
@@ -1878,8 +1885,6 @@ static int disconnect_waitall (int count, ompi_dpm_disconnect_obj **objs)
     /* force all non-blocking all-to-alls to finish */
     ret = ompi_request_wait_all(2*totalcount, reqs, MPI_STATUSES_IGNORE);
 
-    /* Finally, free everything */
-    cleanup_dpm_disconnect_objs(objs, count);
     free(reqs);
 
     return ret;
@@ -1954,6 +1959,46 @@ static void set_handler_default(int sig)
     sigaction(sig, &act, (struct sigaction *)0);
 }
 
+static char *find_prte(void)
+{
+    char *filename = NULL;
+#if !OMPI_USING_INTERNAL_PRRTE
+    char *prrte_prefix = NULL;
+#endif
+
+    /* 1) Did the user tell us exactly where to find prte? */
+    filename = getenv("OMPI_PRTERUN");
+    if (NULL != filename) {
+        return strdup(filename);
+    }
+
+#if OMPI_USING_INTERNAL_PRRTE
+    /* 2) If using internal PRRTE, use our bindir.  Note that this
+     * will obey OPAL_PREFIX and OPAL_DESTDIR */
+    opal_asprintf(&filename, "%s%sprte", opal_install_dirs.bindir, OPAL_PATH_SEP);
+    return filename;
+#else
+
+    /* 3) Look in ${PRTE_PREFIX}/bin */
+    prrte_prefix = getenv("PRTE_PREFIX");
+    if (NULL != prrte_prefix) {
+        opal_asprintf(&filename, "%s%sbin%sprte", prrte_prefix, OPAL_PATH_SEP, OPAL_PATH_SEP);
+        return filename;
+    }
+
+    /* 4) See if configure told us where to look, if set */
+#if defined(OMPI_PRTERUN_PATH)
+    return strdup(OMPI_PRTERUN_PATH);
+#else
+
+    /* 5) Use path search */
+    filename = opal_find_absolute_path("prte");
+
+    return filename;
+#endif
+#endif
+}
+
 static int start_dvm(char **hostfiles, char **dash_host)
 {
     pmix_status_t pret;
@@ -1967,11 +2012,23 @@ static int start_dvm(char **hostfiles, char **dash_host)
     pmix_info_t info;
     int buffer_length, num_chars_read, chunk;
     char *uri;
+    char *opal_prefix = getenv("OPAL_PREFIX");
+
+    /* as a special case, if OPAL_PREFIX was set and either PRRTE or
+     * PMIx are internal builds, set their prefix variables as well */
+    if (NULL != opal_prefix) {
+#if OMPI_USING_INTERNAL_PRRTE
+        setenv("PRTE_PREFIX", opal_prefix, 1);
+#endif
+#if OPAL_USING_INTERNAL_PMIX
+        setenv("PMIX_PREFIX", opal_prefix, 1);
+#endif
+    }
 
     /* find the prte binary using the install_dirs support - this also
      * checks to ensure that we can see this executable and it *is* executable by us
      */
-    cmd = opal_find_absolute_path("prte");
+    cmd = find_prte();
     if (NULL == cmd) {
         /* guess we couldn't do it - best to abort */
         OMPI_ERROR_LOG(OMPI_ERROR);
@@ -1984,6 +2041,7 @@ static int start_dvm(char **hostfiles, char **dash_host)
      */
     if (pipe(p) < 0) {
         OMPI_ERROR_LOG(OMPI_ERROR);
+        free(cmd);
         return OMPI_ERROR;
     }
 
@@ -1996,6 +2054,7 @@ static int start_dvm(char **hostfiles, char **dash_host)
         OMPI_ERROR_LOG(OMPI_ERROR);
         close(p[0]);
         close(p[1]);
+        free(cmd);
         return OMPI_ERROR;
     }
 
@@ -2003,6 +2062,10 @@ static int start_dvm(char **hostfiles, char **dash_host)
      * spawn processes - see if they gave us any hostfile
      * or dash-host options we should pass along */
     opal_argv_append_nosize(&args, "prte");
+    /* ensure we use the PRRTE personality */
+    opal_argv_append_nosize(&args, "--prtemca");
+    opal_argv_append_nosize(&args, "schizo");
+    opal_argv_append_nosize(&args, "prte");
     if (NULL != hostfiles) {
         tmp = opal_argv_join(hostfiles, ',');
         opal_argv_append_nosize(&args, "--hostfile");
@@ -2026,10 +2089,6 @@ static int start_dvm(char **hostfiles, char **dash_host)
     opal_asprintf(&tmp, "%d", death_pipe[0]);
     opal_argv_append_nosize(&args, tmp);
     free(tmp);
-    opal_argv_append_nosize(&args, "&");
-
-    tmp = opal_argv_join(args, ' ');
-    free(tmp);
 
     /* Fork off the child */
     pid = fork();
@@ -2054,7 +2113,7 @@ static int start_dvm(char **hostfiles, char **dash_host)
          will) reset them.  If we don't do this, the event
          library may have left some set that, at least on some
          OS's, don't get reset via fork() or exec().  Hence, the
-         orted could be unkillable (for example). */
+         prted could be unkillable (for example). */
         set_handler_default(SIGTERM);
         set_handler_default(SIGINT);
         set_handler_default(SIGHUP);
@@ -2062,11 +2121,11 @@ static int start_dvm(char **hostfiles, char **dash_host)
         set_handler_default(SIGCHLD);
 
         /* Unblock all signals, for many of the same reasons that
-         we set the default handlers, above.  This is noticable
+         we set the default handlers, above.  This is noticeable
          on Linux where the event library blocks SIGTERM, but we
-         don't want that blocked by the orted (or, more
+         don't want that blocked by the prted (or, more
          specifically, we don't want it to be blocked by the
-         orted and then inherited by the ORTE processes that it
+         prted and then inherited by the PRTE processes that it
          forks, making them unkillable by SIGTERM). */
         sigprocmask(0, 0, &sigs);
         sigprocmask(SIG_UNBLOCK, &sigs, 0);
@@ -2074,7 +2133,7 @@ static int start_dvm(char **hostfiles, char **dash_host)
         execv(cmd, args);
 
         /* if I get here, the execv failed! */
-        opal_show_help("help-ess-base.txt", "ess-base:execv-error",
+        opal_show_help("help-dpm.txt", "execv-error",
                        true, cmd, strerror(errno));
         exit(1);
 
@@ -2127,7 +2186,6 @@ static int start_dvm(char **hostfiles, char **dash_host)
     pret = PMIx_Init(NULL, &info, 1);
     rc = opal_pmix_convert_status(pret);
     if (OPAL_SUCCESS != rc) {
-        opal_progress_event_users_decrement();
         return MPI_ERR_SPAWN;
     }
     /* decrement the PMIx init refcount */
@@ -2137,12 +2195,12 @@ static int start_dvm(char **hostfiles, char **dash_host)
     PMIx_Commit();
 
     /* we are no longer a singleton */
-    ompi_singleton = false;
+     opal_process_info.is_singleton = false;
 
     return OMPI_SUCCESS;
 }
 #else
-static int start_dvm(char *hostfile, char *dash_host)
+static int start_dvm(char **hostfiles, char **dash_host)
 {
     return OMPI_ERROR;
 }
diff --git a/ompi/dpm/dpm.h b/ompi/dpm/dpm.h
index 34084480f87..f954f141ac2 100644
--- a/ompi/dpm/dpm.h
+++ b/ompi/dpm/dpm.h
@@ -13,6 +13,8 @@
  * Copyright (c) 2013-2015 Intel, Inc. All rights reserved
  * Copyright (c) 2013-2015 Los Alamos National Security, LLC.  All rights
  *                         reserved.
+ * Copyright (c) 2018      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -96,11 +98,6 @@ int ompi_dpm_open_port(char *port_name);
  */
 int ompi_dpm_close_port(const char *port_name);
 
-/*
- * Finalize the DPM
- */
-int ompi_dpm_finalize(void);
-
 END_C_DECLS
 
 #endif /* OMPI_DPM_H */
diff --git a/ompi/dpm/help-dpm.txt b/ompi/dpm/help-dpm.txt
index 43923cdd8e2..ed7a868b866 100644
--- a/ompi/dpm/help-dpm.txt
+++ b/ompi/dpm/help-dpm.txt
@@ -1,6 +1,7 @@
 # -*- text -*-
 #
 # Copyright (c) 2020      Intel, Inc.  All rights reserved.
+# Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -44,3 +45,20 @@ the number of procs for each resource and the type of resource to be used.
   Specified value:  %s
 
 We are not able to proceed. Please correct your application.
+#
+[execv-error]
+
+The singleton application was not able to find an executable in your
+PATH or in the directory where PRTE was initially installed, and
+therefore cannot continue.
+
+For reference, we tried the following command:
+
+  %s
+
+and got the error %s.
+
+This could mean that your PATH or executable name is wrong, or that
+you do not have the necessary permissions.  Please ensure that the
+executable is able to be found and executed as it is required for
+singleton operations.
diff --git a/ompi/errhandler/Makefile.am b/ompi/errhandler/Makefile.am
index 6f7264135c2..9f94623a9ff 100644
--- a/ompi/errhandler/Makefile.am
+++ b/ompi/errhandler/Makefile.am
@@ -12,6 +12,7 @@
 #                         All rights reserved.
 # Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2016      IBM Corporation.  All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -21,7 +22,7 @@
 
 # This makefile.am does not stand on its own - it is included from ompi/Makefile.am
 
-dist_ompidata_DATA += errhandler/help-mpi-errors.txt
+EXTRA_DIST += errhandler/help-mpi-errors.txt
 
 headers += \
         errhandler/errcode.h \
diff --git a/ompi/errhandler/errcode-internal.c b/ompi/errhandler/errcode-internal.c
index 8d76030a6c6..e1b8a0abf27 100644
--- a/ompi/errhandler/errcode-internal.c
+++ b/ompi/errhandler/errcode-internal.c
@@ -13,8 +13,10 @@
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
- *                         reseved.
+ *                         reserved.
  * Copyright (c) 2018      Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2018      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -31,6 +33,7 @@
 #include "opal/util/string_copy.h"
 
 #include "ompi/errhandler/errcode-internal.h"
+#include "ompi/instance/instance.h"
 
 /* Table holding all error codes */
 opal_pointer_array_t ompi_errcodes_intern = {{0}};
@@ -62,6 +65,7 @@ static ompi_errcode_intern_t ompi_err_rma_flavor_intern;
 
 static void ompi_errcode_intern_construct(ompi_errcode_intern_t* errcode);
 static void ompi_errcode_intern_destruct(ompi_errcode_intern_t* errcode);
+static int ompi_errcode_intern_finalize (void);
 
 OBJ_CLASS_INSTANCE(ompi_errcode_intern_t,opal_object_t,ompi_errcode_intern_construct, ompi_errcode_intern_destruct);
 
@@ -286,10 +290,21 @@ int ompi_errcode_intern_init (void)
                                 &ompi_err_rma_flavor_intern);
 
     ompi_errcode_intern_lastused=pos;
+
+    ompi_mpi_instance_append_finalize (ompi_errcode_intern_finalize);
+
     return OMPI_SUCCESS;
 }
 
-int ompi_errcode_intern_finalize(void)
+/**
+ * Finalize the error codes.
+ *
+ * @returns OMPI_SUCCESS Always
+ *
+ * Invoked from instance teardown if ompi_errcode_intern_init() was called;
+ * tears down the error code array.
+ */
+static int ompi_errcode_intern_finalize (void)
 {
 
     OBJ_DESTRUCT(&ompi_success_intern);
diff --git a/ompi/errhandler/errcode-internal.h b/ompi/errhandler/errcode-internal.h
index 745098b5a0d..ec64faa9fc0 100644
--- a/ompi/errhandler/errcode-internal.h
+++ b/ompi/errhandler/errcode-internal.h
@@ -14,6 +14,8 @@
  * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved.
  * Copyright (c) 2012      Los Alamos National Security, LLC.  All rights
  *                         reserved.
+ * Copyright (c) 2018      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -87,15 +89,6 @@ static inline int ompi_errcode_get_mpi_code(int errcode)
  */
 int ompi_errcode_intern_init(void);
 
-/**
- * Finalize the error codes.
- *
- * @returns OMPI_SUCCESS Always
- *
- * Invokes from ompi_mpi_finalize(); tears down the error code array.
- */
-int ompi_errcode_intern_finalize(void);
-
 END_C_DECLS
 
 #endif /* OMPI_ERRCODE_INTERNAL_H */
diff --git a/ompi/errhandler/errcode.c b/ompi/errhandler/errcode.c
index c52c5789c16..63b55be9fea 100644
--- a/ompi/errhandler/errcode.c
+++ b/ompi/errhandler/errcode.c
@@ -13,10 +13,12 @@
  * Copyright (c) 2006      University of Houston. All rights reserved.
  * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
  * Copyright (c) 2013-2018 Cisco Systems, Inc.  All rights reserved
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ * Copyright (c) 2013-2018 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -35,6 +37,7 @@
 
 #include "ompi/errhandler/errcode.h"
 #include "ompi/constants.h"
+#include "ompi/instance/instance.h"
 
 /* Table holding all error codes */
 opal_pointer_array_t ompi_mpi_errcodes = {{0}};
@@ -122,6 +125,8 @@ static ompi_mpi_errcode_t ompi_err_proc_fail_stop;
 static ompi_mpi_errcode_t ompi_err_proc_fail_pending;
 static ompi_mpi_errcode_t ompi_err_revoked;
 #endif
+static ompi_mpi_errcode_t ompi_err_session;
+static ompi_mpi_errcode_t ompi_err_value_too_large;
 
 static void ompi_mpi_errcode_construct(ompi_mpi_errcode_t* errcode);
 static void ompi_mpi_errcode_destruct(ompi_mpi_errcode_t* errcode);
@@ -137,14 +142,14 @@ do {                                                                      \
     opal_pointer_array_set_item(&ompi_mpi_errcodes, (ERRCODE), &(VAR));   \
 } while (0)
 
-static opal_mutex_t errcode_init_lock = OPAL_MUTEX_STATIC_INIT;
+static opal_mutex_t errcode_lock = OPAL_MUTEX_STATIC_INIT;
 
 int ompi_mpi_errcode_init (void)
 {
-    opal_mutex_lock(&errcode_init_lock);
+    opal_mutex_lock(&errcode_lock);
     if ( 0 != ompi_mpi_errcode_lastpredefined ) {
         /* Already initialized (presumably by an API call before MPI_init */
-        opal_mutex_unlock(&errcode_init_lock);
+        opal_mutex_unlock(&errcode_lock);
         return OMPI_SUCCESS;
     }
 
@@ -186,7 +191,7 @@ int ompi_mpi_errcode_init (void)
     CONSTRUCT_ERRCODE( ompi_err_conversion, MPI_ERR_CONVERSION, "MPI_ERR_CONVERSION: error in data conversion" );
     CONSTRUCT_ERRCODE( ompi_err_disp, MPI_ERR_DISP, "MPI_ERR_DISP: invalid displacement" );
     CONSTRUCT_ERRCODE( ompi_err_dup_datarep, MPI_ERR_DUP_DATAREP, "MPI_ERR_DUP_DATAREP: error duplicating data representation" );
-    CONSTRUCT_ERRCODE( ompi_err_file_exists, MPI_ERR_FILE_EXISTS, "MPI_ERR_FILE_EXISTS: file exists alreay" );
+    CONSTRUCT_ERRCODE( ompi_err_file_exists, MPI_ERR_FILE_EXISTS, "MPI_ERR_FILE_EXISTS: file exists already" );
     CONSTRUCT_ERRCODE( ompi_err_file_in_use, MPI_ERR_FILE_IN_USE, "MPI_ERR_FILE_IN_USE: file already in use" );
     CONSTRUCT_ERRCODE( ompi_err_file, MPI_ERR_FILE, "MPI_ERR_FILE: invalid file" );
     CONSTRUCT_ERRCODE( ompi_err_info_key, MPI_ERR_INFO_KEY, "MPI_ERR_INFO_KEY: invalid key argument for info object" );
@@ -238,21 +243,34 @@ int ompi_mpi_errcode_init (void)
     CONSTRUCT_ERRCODE( ompi_err_proc_fail_pending,  MPI_ERR_PROC_FAILED_PENDING,  "MPI_ERR_PROC_FAILED_PENDING: Process Failure during an MPI_ANY_SOURCE non-blocking receive, request is still active" );
     CONSTRUCT_ERRCODE( ompi_err_revoked,  MPI_ERR_REVOKED,  "MPI_ERR_REVOKED: Communication Object Revoked" );
 #endif
+    CONSTRUCT_ERRCODE( ompi_err_session,  MPI_ERR_SESSION,  "MPI_ERR_SESSION: Invalid session handle" );
+    CONSTRUCT_ERRCODE( ompi_err_value_too_large,  MPI_ERR_VALUE_TOO_LARGE,  "MPI_ERR_VALUE_TOO_LARGE: Value is too large to store" );
 
     /* Per MPI-3 p353:27-32, MPI_LASTUSEDCODE must be >=
        MPI_ERR_LASTCODE.  So just start it as == MPI_ERR_LASTCODE. */
     ompi_mpi_errcode_lastused = MPI_ERR_LASTCODE;
     ompi_mpi_errcode_lastpredefined = MPI_ERR_LASTCODE;
-    opal_mutex_unlock(&errcode_init_lock);
+
+    opal_mutex_unlock(&errcode_lock);
+
+    ompi_mpi_instance_append_finalize (ompi_mpi_errcode_finalize);
+
     return OMPI_SUCCESS;
 }
 
-int ompi_mpi_errcode_finalize(void)
+/**
+ * Finalize the error codes.
+ *
+ * @returns OMPI_SUCCESS Always
+ *
+ * Invoked from instance teardown if ompi_mpi_errcode_init() was called; tears down the error code array.
+ */
+int ompi_mpi_errcode_finalize (void)
 {
     int i;
     ompi_mpi_errcode_t *errc;
 
-    opal_mutex_lock(&errcode_init_lock);
+    opal_mutex_lock(&errcode_lock);
     for (i=ompi_mpi_errcode_lastpredefined+1; i<=ompi_mpi_errcode_lastused; i++) {
         /*
          * there are some user defined error-codes, which
@@ -342,10 +360,11 @@ int ompi_mpi_errcode_finalize(void)
     OBJ_DESTRUCT(&ompi_err_proc_fail_pending);
     OBJ_DESTRUCT(&ompi_err_revoked);
 #endif
-
+    OBJ_DESTRUCT(&ompi_err_session);
+    OBJ_DESTRUCT(&ompi_err_value_too_large);
     OBJ_DESTRUCT(&ompi_mpi_errcodes);
     ompi_mpi_errcode_lastpredefined = 0;
-    opal_mutex_unlock(&errcode_init_lock);
+    opal_mutex_unlock(&errcode_lock);
     return OMPI_SUCCESS;
 }
 
@@ -354,11 +373,14 @@ int ompi_mpi_errcode_add(int errclass )
     ompi_mpi_errcode_t *newerrcode;
 
     newerrcode = OBJ_NEW(ompi_mpi_errcode_t);
+
+    opal_mutex_lock(&errcode_lock);
     newerrcode->code = (ompi_mpi_errcode_lastused+1);
     newerrcode->cls = errclass;
     opal_pointer_array_set_item(&ompi_mpi_errcodes, newerrcode->code, newerrcode);
-
     ompi_mpi_errcode_lastused++;
+    opal_mutex_unlock(&errcode_lock);
+
     return newerrcode->code;
 }
 
@@ -367,10 +389,13 @@ int ompi_mpi_errclass_add(void)
     ompi_mpi_errcode_t *newerrcode;
 
     newerrcode = OBJ_NEW(ompi_mpi_errcode_t);
+
+    opal_mutex_lock(&errcode_lock);
     newerrcode->cls = ( ompi_mpi_errcode_lastused+1);
     opal_pointer_array_set_item(&ompi_mpi_errcodes, newerrcode->cls, newerrcode);
-
     ompi_mpi_errcode_lastused++;
+    opal_mutex_unlock(&errcode_lock);
+
     return newerrcode->cls;
 }
 
diff --git a/ompi/errhandler/errcode.h b/ompi/errhandler/errcode.h
index 033abd24167..24d070fb4f3 100644
--- a/ompi/errhandler/errcode.h
+++ b/ompi/errhandler/errcode.h
@@ -14,6 +14,8 @@
  * Copyright (c) 2007-2015 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
  *                         reserved.
+ * Copyright (c) 2018      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -215,6 +217,50 @@ static inline char* ompi_mpi_errnum_get_string (int errnum)
 }
 
 
+/**
+ * Initialize the error codes
+ *
+ * @returns OMPI_SUCCESS Upon success
+ * @returns OMPI_ERROR Otherwise
+ *
+ * Invoked from ompi_mpi_init(); sets up all static MPI error codes,
+ */
+int ompi_mpi_errcode_init(void);
+
+/**
+ * Add an error code
+ *
+ * @param: error class to which this new error code belongs to
+ *
+ * @returns the new error code on SUCCESS (>0)
+ * @returns OMPI_ERROR otherwise
+ *
+ */
+int ompi_mpi_errcode_add (int errclass);
+
+/**
+ * Add an error class
+ *
+ * @param: none
+ *
+ * @returns the new error class on SUCCESS (>0)
+ * @returns OMPI_ERROR otherwise
+ *
+ */
+int ompi_mpi_errclass_add (void);
+
+/**
+ * Add an error string to an error code
+ *
+ * @param: error code for which the string is defined
+ * @param: error string to add
+ * @param: length of the string
+ *
+ * @returns OMPI_SUCCESS on success
+ * @returns OMPI_ERROR on error
+ */
+int ompi_mpi_errnum_add_string (int errnum, const char* string, int len);
+
 END_C_DECLS
 
 #endif /* OMPI_MPI_ERRCODE_H */
diff --git a/ompi/errhandler/errhandler.c b/ompi/errhandler/errhandler.c
index 3a2d2eab162..17fd48dbe9b 100644
--- a/ompi/errhandler/errhandler.c
+++ b/ompi/errhandler/errhandler.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
+ * Copyright (c) 2004-2023 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -17,6 +17,8 @@
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2015-2019 Intel, Inc.  All rights reserved.
  * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2018-2021 Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -36,7 +38,7 @@
 #include "opal/mca/pmix/pmix-internal.h"
 #include "opal/util/string_copy.h"
 #include "opal/mca/backtrace/backtrace.h"
-
+#include "ompi/runtime/mpiruntime.h"
 
 /*
  * Table for Fortran <-> C errhandler handle conversion
@@ -137,56 +139,60 @@ int ompi_initial_errhandler_init(void) {
     return OMPI_SUCCESS;
 }
 
+static int ompi_errhandler_finalize (void);
+
 /*
  * Initialize OMPI errhandler infrastructure
  */
 int ompi_errhandler_init(void)
 {
-  /* initialize ompi_errhandler_f_to_c_table */
+    OBJ_CONSTRUCT( &ompi_errhandler_f_to_c_table, opal_pointer_array_t);
+    if( OPAL_SUCCESS != opal_pointer_array_init(&ompi_errhandler_f_to_c_table, 8,
+                                                OMPI_FORTRAN_HANDLE_MAX, 16) ) {
+        return OMPI_ERROR;
+    }
 
-  OBJ_CONSTRUCT( &ompi_errhandler_f_to_c_table, opal_pointer_array_t);
-  if( OPAL_SUCCESS != opal_pointer_array_init(&ompi_errhandler_f_to_c_table, 8,
-                                              OMPI_FORTRAN_HANDLE_MAX, 16) ) {
-    return OMPI_ERROR;
-  }
+    /* Initialize the predefined error handlers */
+    OBJ_CONSTRUCT( &ompi_mpi_errhandler_null.eh, ompi_errhandler_t );
+    if( ompi_mpi_errhandler_null.eh.eh_f_to_c_index != OMPI_ERRHANDLER_NULL_FORTRAN ) {
+        return OMPI_ERROR;
+    }
 
-  /* Initialize the predefined error handlers */
-  OBJ_CONSTRUCT( &ompi_mpi_errhandler_null.eh, ompi_errhandler_t );
-  if( ompi_mpi_errhandler_null.eh.eh_f_to_c_index != OMPI_ERRHANDLER_NULL_FORTRAN )
-      return OMPI_ERROR;
-  ompi_mpi_errhandler_null.eh.eh_mpi_object_type = OMPI_ERRHANDLER_TYPE_PREDEFINED;
-  ompi_mpi_errhandler_null.eh.eh_lang = OMPI_ERRHANDLER_LANG_C;
-  ompi_mpi_errhandler_null.eh.eh_comm_fn = NULL;
-  ompi_mpi_errhandler_null.eh.eh_file_fn = NULL;
-  ompi_mpi_errhandler_null.eh.eh_win_fn  = NULL ;
-  ompi_mpi_errhandler_null.eh.eh_fort_fn = NULL;
-  opal_string_copy(ompi_mpi_errhandler_null.eh.eh_name, "MPI_ERRHANDLER_NULL",
-                   sizeof(ompi_mpi_errhandler_null.eh.eh_name));
-
-  OBJ_CONSTRUCT( &ompi_mpi_errors_are_fatal.eh, ompi_errhandler_t );
-  if( ompi_mpi_errors_are_fatal.eh.eh_f_to_c_index != OMPI_ERRORS_ARE_FATAL_FORTRAN )
-      return OMPI_ERROR;
-  ompi_mpi_errors_are_fatal.eh.eh_mpi_object_type = OMPI_ERRHANDLER_TYPE_PREDEFINED;
-  ompi_mpi_errors_are_fatal.eh.eh_lang = OMPI_ERRHANDLER_LANG_C;
-  ompi_mpi_errors_are_fatal.eh.eh_comm_fn = ompi_mpi_errors_are_fatal_comm_handler;
-  ompi_mpi_errors_are_fatal.eh.eh_file_fn = ompi_mpi_errors_are_fatal_file_handler;
-  ompi_mpi_errors_are_fatal.eh.eh_win_fn  = ompi_mpi_errors_are_fatal_win_handler ;
-  ompi_mpi_errors_are_fatal.eh.eh_fort_fn = NULL;
-  opal_string_copy(ompi_mpi_errors_are_fatal.eh.eh_name,
-                   "MPI_ERRORS_ARE_FATAL",
-                   sizeof(ompi_mpi_errors_are_fatal.eh.eh_name));
-
-  OBJ_CONSTRUCT( &ompi_mpi_errors_return.eh, ompi_errhandler_t );
-  if( ompi_mpi_errors_return.eh.eh_f_to_c_index != OMPI_ERRORS_RETURN_FORTRAN )
-      return OMPI_ERROR;
-  ompi_mpi_errors_return.eh.eh_mpi_object_type  = OMPI_ERRHANDLER_TYPE_PREDEFINED;
-  ompi_mpi_errors_return.eh.eh_lang = OMPI_ERRHANDLER_LANG_C;
-  ompi_mpi_errors_return.eh.eh_comm_fn = ompi_mpi_errors_return_comm_handler;
-  ompi_mpi_errors_return.eh.eh_file_fn = ompi_mpi_errors_return_file_handler;
-  ompi_mpi_errors_return.eh.eh_win_fn  = ompi_mpi_errors_return_win_handler;
-  ompi_mpi_errors_return.eh.eh_fort_fn = NULL;
-  opal_string_copy(ompi_mpi_errors_return.eh.eh_name, "MPI_ERRORS_RETURN",
-                   sizeof(ompi_mpi_errors_return.eh.eh_name));
+    ompi_mpi_errhandler_null.eh.eh_mpi_object_type = OMPI_ERRHANDLER_TYPE_PREDEFINED;
+    ompi_mpi_errhandler_null.eh.eh_lang = OMPI_ERRHANDLER_LANG_C;
+    ompi_mpi_errhandler_null.eh.eh_comm_fn = NULL;
+    ompi_mpi_errhandler_null.eh.eh_file_fn = NULL;
+    ompi_mpi_errhandler_null.eh.eh_win_fn  = NULL ;
+    ompi_mpi_errhandler_null.eh.eh_fort_fn = NULL;
+    opal_string_copy (ompi_mpi_errhandler_null.eh.eh_name, "MPI_ERRHANDLER_NULL",
+                      sizeof(ompi_mpi_errhandler_null.eh.eh_name));
+
+    OBJ_CONSTRUCT( &ompi_mpi_errors_are_fatal.eh, ompi_errhandler_t );
+    if( ompi_mpi_errors_are_fatal.eh.eh_f_to_c_index != OMPI_ERRORS_ARE_FATAL_FORTRAN )
+        return OMPI_ERROR;
+    ompi_mpi_errors_are_fatal.eh.eh_mpi_object_type = OMPI_ERRHANDLER_TYPE_PREDEFINED;
+    ompi_mpi_errors_are_fatal.eh.eh_lang = OMPI_ERRHANDLER_LANG_C;
+    ompi_mpi_errors_are_fatal.eh.eh_comm_fn = ompi_mpi_errors_are_fatal_comm_handler;
+    ompi_mpi_errors_are_fatal.eh.eh_file_fn = ompi_mpi_errors_are_fatal_file_handler;
+    ompi_mpi_errors_are_fatal.eh.eh_win_fn  = ompi_mpi_errors_are_fatal_win_handler;
+    ompi_mpi_errors_are_fatal.eh.eh_instance_fn = ompi_mpi_errors_are_fatal_instance_handler;
+    ompi_mpi_errors_are_fatal.eh.eh_fort_fn = NULL;
+    opal_string_copy(ompi_mpi_errors_are_fatal.eh.eh_name,
+                     "MPI_ERRORS_ARE_FATAL",
+                     sizeof(ompi_mpi_errors_are_fatal.eh.eh_name));
+
+    OBJ_CONSTRUCT( &ompi_mpi_errors_return.eh, ompi_errhandler_t );
+    if( ompi_mpi_errors_return.eh.eh_f_to_c_index != OMPI_ERRORS_RETURN_FORTRAN )
+        return OMPI_ERROR;
+    ompi_mpi_errors_return.eh.eh_mpi_object_type  = OMPI_ERRHANDLER_TYPE_PREDEFINED;
+    ompi_mpi_errors_return.eh.eh_lang = OMPI_ERRHANDLER_LANG_C;
+    ompi_mpi_errors_return.eh.eh_comm_fn = ompi_mpi_errors_return_comm_handler;
+    ompi_mpi_errors_return.eh.eh_file_fn = ompi_mpi_errors_return_file_handler;
+    ompi_mpi_errors_return.eh.eh_win_fn  = ompi_mpi_errors_return_win_handler;
+    ompi_mpi_errors_return.eh.eh_instance_fn = ompi_mpi_errors_return_instance_handler;
+    ompi_mpi_errors_return.eh.eh_fort_fn = NULL;
+    opal_string_copy(ompi_mpi_errors_return.eh.eh_name, "MPI_ERRORS_RETURN",
+                     sizeof(ompi_mpi_errors_return.eh.eh_name));
 
   OBJ_CONSTRUCT( &ompi_mpi_errors_abort.eh, ompi_errhandler_t );
   if( ompi_mpi_errors_abort.eh.eh_f_to_c_index != OMPI_ERRORS_ABORT_FORTRAN )
@@ -206,14 +212,23 @@ int ompi_errhandler_init(void)
   if( NULL != env ) {
     ompi_process_info.initial_errhandler = strndup(env, MPI_MAX_INFO_VAL);
   }
-  return ompi_initial_errhandler_init();
+
+  ompi_initial_errhandler_init();
+  ompi_mpi_instance_append_finalize (ompi_errhandler_finalize);
+
+  return OMPI_SUCCESS;
 }
 
 
-/*
- * Clean up the errorhandler resources
+/**
+ * Finalize the error handler interface.
+ *
+ * @returns OMPI_SUCCESS Always
+ *
+ * Invoked on instance teardown if ompi_errhandler_init() was called; tears down the error handler
+ * interface, and destroys the F2C translation table.
  */
-int ompi_errhandler_finalize(void)
+static int ompi_errhandler_finalize (void)
 {
     OBJ_DESTRUCT(&ompi_mpi_errhandler_null.eh);
     OBJ_DESTRUCT(&ompi_mpi_errors_return.eh);
@@ -232,46 +247,64 @@ int ompi_errhandler_finalize(void)
     return OMPI_SUCCESS;
 }
 
+void ompi_errhandler_free (ompi_errhandler_t *errhandler)
+{
+    OBJ_RELEASE(errhandler);
+    ompi_mpi_instance_release ();
+}
 
 ompi_errhandler_t *ompi_errhandler_create(ompi_errhandler_type_t object_type,
-					                      ompi_errhandler_generic_handler_fn_t *func,
+                                          ompi_errhandler_generic_handler_fn_t *func,
                                           ompi_errhandler_lang_t lang)
 {
-  ompi_errhandler_t *new_errhandler;
-
-  /* Create a new object and ensure that it's valid */
-
-  new_errhandler = OBJ_NEW(ompi_errhandler_t);
-  if (NULL != new_errhandler) {
-    if (0 > new_errhandler->eh_f_to_c_index) {
-      OBJ_RELEASE(new_errhandler);
-      new_errhandler = NULL;
-    } else {
-
-      /* We cast the user's callback function to any one of the
-         function pointer types in the union; it doesn't matter which.
-         It only matters that we dereference/use the right member when
-         invoking the callback. */
-
-      new_errhandler->eh_mpi_object_type = object_type;
-      new_errhandler->eh_lang = lang;
-      switch (object_type ) {
-	  case (OMPI_ERRHANDLER_TYPE_COMM):
-	      new_errhandler->eh_comm_fn = (MPI_Comm_errhandler_function *)func;
-	      break;
-	  case (OMPI_ERRHANDLER_TYPE_FILE):
-	      new_errhandler->eh_file_fn = (ompi_file_errhandler_function *)func;
-	      break;
-	  case (OMPI_ERRHANDLER_TYPE_WIN):
-	      new_errhandler->eh_win_fn = (MPI_Win_errhandler_function *)func;
-	      break;
-	  default:
-	      break;
-      }
-
-      new_errhandler->eh_fort_fn = (ompi_errhandler_fortran_handler_fn_t *)func;
+    ompi_errhandler_t *new_errhandler;
+    int ret;
+
+    /* make sure the infrastructure is initialized */
+    ret = ompi_mpi_instance_retain ();
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
+        return NULL;
+    }
+
+    /* Create a new object and ensure that it's valid */
+
+    new_errhandler = OBJ_NEW(ompi_errhandler_t);
+    if (NULL != new_errhandler) {
+        if (0 > new_errhandler->eh_f_to_c_index) {
+            OBJ_RELEASE(new_errhandler);
+            new_errhandler = NULL;
+        } else {
+
+        /* We cast the user's callback function to any one of the
+           function pointer types in the union; it doesn't matter which.
+           It only matters that we dereference/use the right member when
+           invoking the callback. */
+
+            new_errhandler->eh_mpi_object_type = object_type;
+            new_errhandler->eh_lang = lang;
+            switch (object_type ) {
+            case OMPI_ERRHANDLER_TYPE_COMM:
+                new_errhandler->eh_comm_fn = (MPI_Comm_errhandler_function *)func;
+                break;
+            case OMPI_ERRHANDLER_TYPE_FILE:
+                new_errhandler->eh_file_fn = (ompi_file_errhandler_function *)func;
+                break;
+            case OMPI_ERRHANDLER_TYPE_WIN:
+                new_errhandler->eh_win_fn = (MPI_Win_errhandler_function *)func;
+                break;
+            case OMPI_ERRHANDLER_TYPE_INSTANCE:
+                new_errhandler->eh_instance_fn = (MPI_Session_errhandler_function *)func;
+                break;
+            default:
+                break;
+            }
+        }
+
+        if (NULL != new_errhandler) {
+            new_errhandler->eh_fort_fn = (ompi_errhandler_fortran_handler_fn_t *)func;
+        }
+
     }
-  }
 
   /* All done */
 
@@ -310,10 +343,10 @@ int ompi_errhandler_proc_failed_internal(ompi_proc_t* ompi_proc, int status, boo
     opal_mutex_unlock(&errhandler_ftmpi_lock);
 
     opal_output_verbose(1, ompi_ftmpi_output_handle,
-                        "%s ompi: Process %s failed (state = %d).",
+                        "%s ompi: Process %s failed (state = %d %s).",
                         OMPI_NAME_PRINT(OMPI_PROC_MY_NAME),
                         OMPI_NAME_PRINT(&ompi_proc->super.proc_name),
-                        status );
+                        status, PMIx_Error_string(status) );
 
     if(90 < opal_output_get_verbosity(ompi_ftmpi_output_handle)) {
         /* how did we get there? */
@@ -322,9 +355,9 @@ int ompi_errhandler_proc_failed_internal(ompi_proc_t* ompi_proc, int status, boo
 
     /* Communicator State:
      * Let them know about the failure. */
-    max_num_comm = opal_pointer_array_get_size(&ompi_mpi_communicators);
+    max_num_comm = ompi_comm_get_num_communicators();
     for( i = 0; i < max_num_comm; ++i ) {
-        comm = (ompi_communicator_t *)opal_pointer_array_get_item(&ompi_mpi_communicators, i);
+        comm = ompi_comm_lookup(i);
         if( NULL == comm ) {
             continue;
         }
@@ -350,10 +383,10 @@ int ompi_errhandler_proc_failed_internal(ompi_proc_t* ompi_proc, int status, boo
             if(OPAL_UNLIKELY( OMPI_SUCCESS != rc )) goto cleanup;
         }
         OPAL_OUTPUT_VERBOSE((10, ompi_ftmpi_output_handle,
-                             "%s ompi: Process %s is in comm (%d) with rank %d. [%s]",
+                             "%s ompi: Process %s is in comm (%s) with rank %d. [%s]",
                              OMPI_NAME_PRINT(OMPI_PROC_MY_NAME),
                              OMPI_NAME_PRINT(&ompi_proc->super.proc_name),
-                             comm->c_contextid,
+                             ompi_comm_print_cid(comm),
                              proc_rank,
                              (OMPI_ERRHANDLER_TYPE_PREDEFINED == comm->errhandler_type ? "P" :
                               (OMPI_ERRHANDLER_TYPE_COMM == comm->errhandler_type ? "C" :
@@ -380,14 +413,14 @@ int ompi_errhandler_proc_failed_internal(ompi_proc_t* ompi_proc, int status, boo
      * The wait function has a check, so all we need to do here is
      * signal it so it will check again.
      */
-    wait_sync_global_wakeup(MPI_ERR_PROC_FAILED);
+    wait_sync_global_wakeup(PMIX_ERR_PROC_ABORTED == status? MPI_ERR_PROC_ABORTED: MPI_ERR_PROC_FAILED);
 
     /* Collectives:
      * Propagate the error (this has been selected rather than the "roll
      * forward through errors in collectives" as this is less intrusive to the
      * code base.) */
     if( forward ) {
-        /* TODO: this to become redundand when pmix has rbcast */
+        /* TODO: this to become redundant when pmix has rbcast */
         ompi_comm_failure_propagate(&ompi_mpi_comm_world.comm, ompi_proc, status);
         /* Let pmix know: flush modex information, propagate to connect/accept
          * jobs; we will tell our local daemon, and it will do the proper thing */
@@ -401,7 +434,7 @@ int ompi_errhandler_proc_failed_internal(ompi_proc_t* ompi_proc, int status, boo
         OPAL_PMIX_CONVERT_NAME(&pmix_proc, &ompi_proc->super.proc_name);
         PMIX_INFO_CONSTRUCT(&pmix_info[0]);
         PMIX_INFO_LOAD(&pmix_info[0], PMIX_EVENT_AFFECTED_PROC, &pmix_proc, PMIX_PROC);
-        prc = PMIx_Notify_event(status, &pmix_source, PMIX_RANGE_LOCAL,
+        prc = PMIx_Notify_event(PMIX_ERR_PROC_TERM_WO_SYNC, &pmix_source, PMIX_RANGE_LOCAL,
                                 pmix_info, 1, NULL, &active);
         if( PMIX_SUCCESS != prc &&
             PMIX_OPERATION_SUCCEEDED != prc ) {
@@ -434,17 +467,29 @@ static void *ompi_errhandler_event_cb(int fd, int flags, void *context) {
     opal_process_name_t prc;
     int rc;
 #if OPAL_ENABLE_FT_MPI
-    if( PMIX_ERR_PROC_ABORTED == status ) {
-        int i;
-        for(i = 0; i < event->nvalue; i++) {
-            if (PMIX_PROC != event->info[i].value.type) {
+    switch( status ) {
+    case PMIX_ERR_PROC_TERM_WO_SYNC:
+    case PMIX_ERR_PROC_ABORTED_BY_SIG:
+    case PMIX_ERR_PROC_ABORTED: /* that is, proc aborted by pmix_abort */
+        for(int i = 0; i < event->nvalue; i++) {
+            if (strcmp(PMIX_EVENT_AFFECTED_PROC, event->info[i].key)) {
+                OPAL_OUTPUT_VERBOSE((70, ompi_ftmpi_output_handle,
+                    "%s ompi: ignoring the following key for a PMIx fault event: %s",
+                    OMPI_NAME_PRINT(OMPI_PROC_MY_NAME),
+                    event->info[i].key));
                 continue;
             }
+            assert(event->info[i].value.type == PMIX_PROC);
             OPAL_PMIX_CONVERT_PROCT(rc, &prc, event->info[i].value.data.proc);
             if (OPAL_SUCCESS != rc) {
                 OPAL_ERROR_LOG(rc);
                 break;
             }
+            OPAL_OUTPUT_VERBOSE((5, ompi_ftmpi_output_handle,
+                "%s ompi: proc %s reported dead by PMIx event handler (found at %s): %s",
+                OMPI_NAME_PRINT(OMPI_PROC_MY_NAME),
+                OPAL_NAME_PRINT(prc), OPAL_NAME_PRINT(source),
+                PMIx_Error_string(status)));
             ompi_proc_t *proc = (ompi_proc_t*)ompi_proc_for_name(prc);
             if( NULL == proc ) {
                 continue; /* we are not 'MPI connected' with this proc. */
@@ -455,15 +500,21 @@ static void *ompi_errhandler_event_cb(int fd, int flags, void *context) {
         opal_event_del(&event->super);
         free(event);
         return NULL;
+    case PMIX_ERR_LOST_CONNECTION:
+        opal_output_verbose(1, ompi_ftmpi_output_handle,
+            "%s ompi: Error event PMIX_ERR_LOST_CONNECTION reported, that usually means that my daemon died thus I need to go away.",
+            OMPI_NAME_PRINT(OMPI_PROC_MY_NAME));
+        break;
+    default:
+        /* An unmanaged type of failure, let it do its thing. */
+        opal_output_verbose(1, ompi_ftmpi_output_handle,
+            "%s ompi: Error event reported through PMIx from %s (state = %s). "
+            "This error type is not handled by the fault tolerant layer "
+            "and the application will now presumably abort.",
+            OMPI_NAME_PRINT(OMPI_PROC_MY_NAME),
+            OPAL_NAME_PRINT(source),
+            PMIx_Error_string(status));
     }
-    /* An unmanaged type of failure, let it do its thing. */
-    opal_output_verbose(1, ompi_ftmpi_output_handle,
-        "%s ompi: Error event reported through PMIx from %s (state = %d). "
-        "This error type is not handled by the fault tolerant layer "
-        "and the application will now presumably abort.",
-        OMPI_NAME_PRINT(OMPI_PROC_MY_NAME),
-        OPAL_NAME_PRINT(source),
-        status );
 #endif /* OPAL_ENABLE_FT_MPI */
     opal_event_del(&event->super);
     free(event);
diff --git a/ompi/errhandler/errhandler.h b/ompi/errhandler/errhandler.h
index 572deeb9bf3..40909f3f140 100644
--- a/ompi/errhandler/errhandler.h
+++ b/ompi/errhandler/errhandler.h
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2021 The University of Tennessee and The University
+ * Copyright (c) 2004-2023 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -17,6 +17,8 @@
  *                         reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018-2021 Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -83,7 +85,8 @@ enum ompi_errhandler_type_t {
     OMPI_ERRHANDLER_TYPE_PREDEFINED,
     OMPI_ERRHANDLER_TYPE_COMM,
     OMPI_ERRHANDLER_TYPE_WIN,
-    OMPI_ERRHANDLER_TYPE_FILE
+    OMPI_ERRHANDLER_TYPE_FILE,
+    OMPI_ERRHANDLER_TYPE_INSTANCE,
 };
 typedef enum ompi_errhandler_type_t ompi_errhandler_type_t;
 
@@ -109,6 +112,7 @@ struct ompi_errhandler_t {
     MPI_Comm_errhandler_function *eh_comm_fn;
     ompi_file_errhandler_function *eh_file_fn;
     MPI_Win_errhandler_function *eh_win_fn;
+    MPI_Session_errhandler_function *eh_instance_fn;
     ompi_errhandler_fortran_handler_fn_t *eh_fort_fn;
 
     /* index in Fortran <-> C translation array */
@@ -117,7 +121,7 @@ struct ompi_errhandler_t {
 typedef struct ompi_errhandler_t ompi_errhandler_t;
 
 /**
- * Padded struct to maintain back compatibiltiy.
+ * Padded struct to maintain back compatibility.
  * See ompi/communicator/communicator.h comments with struct ompi_communicator_t
  * for full explanation why we chose the following padding construct for predefines.
  */
@@ -188,6 +192,10 @@ OMPI_DECLSPEC extern void (*ompi_initial_error_handler)(struct ompi_communicator
 struct ompi_request_t;
 
 
+/* declared here because we can't include instance.h from this header
+ * because it would create a circular dependency */
+extern opal_atomic_int32_t ompi_instance_count;
+
 /**
  * This is the macro to check the state of MPI and determine whether
  * it was properly initialized and not yet finalized.
@@ -203,15 +211,13 @@ struct ompi_request_t;
  * potentially-performance-critical code paths) before reading the
  * variable.
  */
-#define OMPI_ERR_INIT_FINALIZE(name)                                    \
-    {                                                                   \
-        int32_t state = ompi_mpi_state;                                 \
-        if (OPAL_UNLIKELY(state < OMPI_MPI_STATE_INIT_COMPLETED ||      \
-                          state > OMPI_MPI_STATE_FINALIZE_PAST_COMM_SELF_DESTRUCT)) { \
-            ompi_errhandler_invoke(NULL, NULL, -1,                       \
+#define OMPI_ERR_INIT_FINALIZE(name)                                       \
+    {                                                                      \
+        if (OPAL_UNLIKELY(0 == ompi_instance_count)) {                     \
+            ompi_errhandler_invoke(NULL, NULL, -1,                         \
                                    ompi_errcode_get_mpi_code(MPI_ERR_ARG), \
-                                   name);                               \
-        }                                                               \
+                                   name);                                  \
+        }                                                                  \
     }
 
 /**
@@ -327,16 +333,6 @@ struct ompi_request_t;
    */
   int ompi_errhandler_init(void);
 
-  /**
-   * Finalize the error handler interface.
-   *
-   * @returns OMPI_SUCCESS Always
-   *
-   * Invokes from ompi_mpi_finalize(); tears down the error handler
-   * interface, and destroys the F2C translation table.
-   */
-  int ompi_errhandler_finalize(void);
-
   /**
    * \internal
    *
@@ -382,8 +378,9 @@ struct ompi_request_t;
   /**
    * Create a ompi_errhandler_t
    *
-   * @param object_type Enum of the type of MPI object
-   * @param func Function pointer of the error handler
+   * @param[in]  object_type    Enum of the type of MPI object
+   * @param[in]  func           Function pointer of the error handler
+   * @param[in]  language       Calling language
    *
    * @returns errhandler Pointer to the ompi_errorhandler_t that will be
    *   created and returned
@@ -402,9 +399,11 @@ struct ompi_request_t;
    * same as sizeof(void(*)).
    */
   OMPI_DECLSPEC ompi_errhandler_t *ompi_errhandler_create(ompi_errhandler_type_t object_type,
-					    ompi_errhandler_generic_handler_fn_t *func,
+                                            ompi_errhandler_generic_handler_fn_t *func,
                                             ompi_errhandler_lang_t language);
 
+  OMPI_DECLSPEC void ompi_errhandler_free (ompi_errhandler_t *errhandler);
+
 /**
  * Callback function to alert the MPI layer of an error or notification
  * from the internal RTE and/or the resource manager.
@@ -454,7 +453,7 @@ struct ompi_proc_t;
 
 OMPI_DECLSPEC int ompi_errhandler_proc_failed_internal(struct ompi_proc_t *ompi_proc, int status, bool forward);
 static inline int ompi_errhandler_proc_failed(struct ompi_proc_t* ompi_proc) {
-    return ompi_errhandler_proc_failed_internal(ompi_proc, OPAL_ERR_PROC_ABORTED, true);
+    return ompi_errhandler_proc_failed_internal(ompi_proc, PMIX_ERR_PROC_TERM_WO_SYNC, true);
 }
 #endif /* OPAL_ENABLE_FT_MPI */
 
diff --git a/ompi/errhandler/errhandler_invoke.c b/ompi/errhandler/errhandler_invoke.c
index fd8bacf195e..d9d3ede4677 100644
--- a/ompi/errhandler/errhandler_invoke.c
+++ b/ompi/errhandler/errhandler_invoke.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
+ * Copyright (c) 2004-2023 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -16,6 +16,8 @@
  *                         reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2023      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -40,6 +42,7 @@ int ompi_errhandler_invoke(ompi_errhandler_t *errhandler, void *mpi_object,
     ompi_communicator_t *comm;
     ompi_win_t *win;
     ompi_file_t *file;
+    ompi_instance_t *instance;
 
     /* If we got no errorhandler, then route the error to the appropriate
      * predefined error handler */
@@ -121,6 +124,22 @@ int ompi_errhandler_invoke(ompi_errhandler_t *errhandler, void *mpi_object,
             break;
         }
         break;
+
+    case OMPI_ERRHANDLER_TYPE_INSTANCE:
+        instance = (ompi_instance_t *) mpi_object;
+        switch (errhandler->eh_lang) {
+        case OMPI_ERRHANDLER_LANG_C:
+            errhandler->eh_instance_fn(&instance, &err_code, message, NULL);
+            break;
+
+        case OMPI_ERRHANDLER_LANG_FORTRAN:
+            fortran_handle = OMPI_INT_2_FINT(instance->i_f_to_c_index);
+            errhandler->eh_fort_fn(&fortran_handle, &fortran_err_code);
+            err_code = OMPI_FINT_2_INT(fortran_err_code);
+            break;
+        }
+        break;
+
     }
 
     /* All done */
@@ -200,9 +219,9 @@ int ompi_errhandler_request_invoke(int count,
         break;
     default:
         /* Covers REQUEST_GEN, REQUEST_NULL, REQUEST_MAX */
-        return ompi_errhandler_invoke(MPI_COMM_WORLD->error_handler,
-                                      MPI_COMM_WORLD,
-                                      MPI_COMM_WORLD->errhandler_type,
+        return ompi_errhandler_invoke(NULL,
+                                      NULL,
+                                      0,
                                       ec, message);
         break;
     }
diff --git a/ompi/errhandler/errhandler_predefined.c b/ompi/errhandler/errhandler_predefined.c
index 4c9353465f2..759025d4bd8 100644
--- a/ompi/errhandler/errhandler_predefined.c
+++ b/ompi/errhandler/errhandler_predefined.c
@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -10,13 +11,15 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2006      University of Houston. All rights reserved.
- * Copyright (c) 2008-2018 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2008-2022 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2009      Sun Microsystems, Inc.  All rights reserved.
  * Copyright (c) 2010-2011 Oak Ridge National Labs.  All rights reserved.
  * Copyright (c) 2012      Los Alamos National Security, LLC.
  *                         All rights reserved.
  * Copyright (c) 2016-2019 Intel, Inc.  All rights reserved.
  * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2018-2021 Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -41,14 +44,24 @@
 #include "ompi/communicator/communicator.h"
 #include "ompi/file/file.h"
 #include "ompi/win/win.h"
+#include "ompi/instance/instance.h"
 #include "opal/util/printf.h"
 #include "opal/util/output.h"
+#include "ompi/runtime/mpiruntime.h"
 
 /*
  * Local functions
  */
 static void backend_abort(int fatal, char *type, struct ompi_communicator_t *comm,
                           char *name, int *error_code, va_list arglist);
+static void backend_abort_aggregate(int fatal, char *type,
+                                    struct ompi_communicator_t *comm,
+                                    char *name, int *error_code,
+                                    va_list arglist);
+static void backend_abort_no_aggregate(int fatal, char *type,
+                                       struct ompi_communicator_t *comm,
+                                       char *name, int *error_code,
+                                       va_list arglist);
 static void out(char *str, char *arg);
 
 
@@ -172,6 +185,36 @@ void ompi_mpi_errors_abort_win_handler(struct ompi_win_t **win,
   va_end(arglist);
 }
 
+void ompi_mpi_errors_are_fatal_instance_handler (struct ompi_instance_t **instance,
+                                                 int *error_code, ...)
+{
+  char *name;
+  va_list arglist;
+  int err = MPI_ERR_UNKNOWN;
+
+  va_start(arglist, error_code);
+
+  if (NULL != instance) {
+      name = (*instance)->i_name;
+  } else {
+      name = NULL;
+  }
+
+  if (NULL != error_code) {
+     err = *error_code;
+  }
+
+  /* We only want aggregation while the rte is initialized */
+  if (ompi_rte_initialized) {
+      backend_abort_aggregate(true, "session", NULL, name, error_code, arglist);
+  } else {
+      backend_abort_no_aggregate(true, "session", NULL, name, error_code, arglist);
+  }
+  va_end(arglist);
+
+  ompi_mpi_abort(NULL, err);
+}
+
 void ompi_mpi_errors_return_comm_handler(struct ompi_communicator_t **comm,
                                          int *error_code, ...)
 {
@@ -208,6 +251,18 @@ void ompi_mpi_errors_return_win_handler(struct ompi_win_t **win,
 }
 
 
+void ompi_mpi_errors_return_instance_handler (struct ompi_instance_t **instance,
+                                              int *error_code, ...)
+{
+    /* Don't need anything more -- just need this function to exist */
+    /* Silence some compiler warnings */
+
+    va_list arglist;
+    va_start(arglist, error_code);
+    va_end(arglist);
+}
+
+
 static void out(char *str, char *arg)
 {
     if (ompi_rte_initialized &&
@@ -283,6 +338,9 @@ static void backend_abort_aggregate(int fatal, char *type,
     usable_err_msg = (NULL == err_msg) ? unknown_error  : err_msg;
 
     if (NULL != name) {
+        // For the show_help topic checker script
+        // SHOW_HELP:"help-mpi-errors.txt","mpi_errors_are_fatal"
+        // SHOW_HELP:"help-mpi-errors.txt","mpi_errors_abort"
         opal_show_help("help-mpi-errors.txt",
                        fatal? "mpi_errors_are_fatal": "mpi_errors_abort",
                        false,
@@ -403,11 +461,11 @@ static void backend_abort_no_aggregate(int fatal, char *type,
 
             len -= strlen(type);
             if (len > 0) {
-                strncat(str, " ", len);
+                strncat(str, " ", len - 1);
 
                 --len;
                 if (len > 0) {
-                    strncat(str, name, len);
+                    strncat(str, name, len - 1);
                 }
             }
             out("*** on %s", str);
diff --git a/ompi/errhandler/errhandler_predefined.h b/ompi/errhandler/errhandler_predefined.h
index 07e306e9a08..c663d962220 100644
--- a/ompi/errhandler/errhandler_predefined.h
+++ b/ompi/errhandler/errhandler_predefined.h
@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -9,6 +10,8 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2018      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -24,6 +27,7 @@
 struct ompi_communicator_t;
 struct ompi_file_t;
 struct ompi_win_t;
+struct ompi_instance_t;
 
 /**
  * Handler function for MPI_ERRORS_ARE_FATAL
@@ -34,6 +38,8 @@ OMPI_DECLSPEC void ompi_mpi_errors_are_fatal_file_handler(struct ompi_file_t **f
 					    int *error_code, ...);
 OMPI_DECLSPEC void ompi_mpi_errors_are_fatal_win_handler(struct ompi_win_t **win,
 					    int *error_code, ...);
+OMPI_DECLSPEC void ompi_mpi_errors_are_fatal_instance_handler(struct ompi_instance_t **win,
+					    int *error_code, ...);
 
 /**
  * Handler function for MPI_ERRORS_ABORT
@@ -54,6 +60,8 @@ OMPI_DECLSPEC void ompi_mpi_errors_return_file_handler(struct ompi_file_t **file
                                    int *error_code, ...);
 OMPI_DECLSPEC void ompi_mpi_errors_return_win_handler(struct ompi_win_t **win,
                                    int *error_code, ...);
+OMPI_DECLSPEC void ompi_mpi_errors_return_instance_handler(struct ompi_instance_t **win,
+                                   int *error_code, ...);
 
 
 #endif /* OMPI_ERRHANDLER_PREDEFINED_H */
diff --git a/ompi/etc/Makefile.am b/ompi/etc/Makefile.am
index 964ce8c7c6d..521a0033b0a 100644
--- a/ompi/etc/Makefile.am
+++ b/ompi/etc/Makefile.am
@@ -47,7 +47,7 @@ install-data-local:
 
 # Only remove if exactly the same as what in our tree
 # NOTE TO READER: Bourne shell if ... fi evaluates the body if
-#    the return of the evaluted command is 0 (as opposed to non-zero
+#    the return of the evaluated command is 0 (as opposed to non-zero
 #    as used by everyone else)
 uninstall-local:
 	@ p="$(ompi_config_files)"; \
diff --git a/ompi/file/file.c b/ompi/file/file.c
index bf546a55694..c7e3759da16 100644
--- a/ompi/file/file.c
+++ b/ompi/file/file.c
@@ -16,6 +16,9 @@
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016      University of Houston. All rights reserved.
  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -32,7 +35,7 @@
 #include "ompi/runtime/params.h"
 #include "ompi/mca/io/base/base.h"
 #include "ompi/info/info.h"
-
+#include "ompi/info/info_memkind.h"
 
 opal_mutex_t ompi_mpi_file_bootstrap_mutex = OPAL_MUTEX_STATIC_INIT;
 
@@ -54,6 +57,7 @@ ompi_predefined_file_t  *ompi_mpi_file_null_addr = &ompi_mpi_file_null;
  */
 static void file_constructor(ompi_file_t *obj);
 static void file_destructor(ompi_file_t *obj);
+static int ompi_file_finalize (void);
 
 
 /*
@@ -66,7 +70,7 @@ OBJ_CLASS_INSTANCE(ompi_file_t,
 
 
 /*
- * Initialize file handling bookeeping
+ * Initialize file handling bookkeeping
  */
 int ompi_file_init(void)
 {
@@ -89,6 +93,7 @@ int ompi_file_init(void)
                                 &ompi_mpi_file_null.file);
 
     /* All done */
+    ompi_mpi_instance_append_finalize (ompi_file_finalize);
 
     return OMPI_SUCCESS;
 }
@@ -119,7 +124,11 @@ int ompi_file_open(struct ompi_communicator_t *comm, const char *filename,
     if (info) {
         opal_info_dup(info, &(file->super.s_info));
     }
-
+    ompi_info_memkind_assert_type type;
+    ompi_info_memkind_copy_or_set (&comm->instance->super, &file->super, info, &type);
+    if (OMPI_INFO_MEMKIND_ASSERT_NO_ACCEL == type) {
+        file->f_flags |= OMPI_FILE_ASSERT_NO_ACCEL_BUF;
+    }
     file->f_amode = amode;
     file->f_filename = strdup(filename);
     if (NULL == file->f_filename) {
@@ -127,9 +136,6 @@ int ompi_file_open(struct ompi_communicator_t *comm, const char *filename,
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
 
-    /* Create the mutex */
-    OBJ_CONSTRUCT(&file->f_lock, opal_mutex_t);
-
     /* Select a module and actually open the file */
 
     if (OMPI_SUCCESS != (ret = mca_io_base_file_select(file, NULL))) {
@@ -149,9 +155,6 @@ int ompi_file_open(struct ompi_communicator_t *comm, const char *filename,
  */
 int ompi_file_close(ompi_file_t **file)
 {
-
-    OBJ_DESTRUCT(&(*file)->f_lock);
-
     (*file)->f_flags |= OMPI_FILE_ISCLOSED;
     OBJ_RELEASE(*file);
     *file = &ompi_mpi_file_null.file;
@@ -160,10 +163,14 @@ int ompi_file_close(ompi_file_t **file)
 }
 
 
-/*
- * Shut down the MPI_File bookkeeping
+/**
+ * Tear down MPI_File handling.
+ *
+ * @retval OMPI_SUCCESS Always.
+ *
+ * Invoked during instance teardown if ompi_file_init() is called.
  */
-int ompi_file_finalize(void)
+static int ompi_file_finalize (void)
 {
     int i, max;
     size_t num_unnamed;
@@ -230,6 +237,7 @@ static void file_constructor(ompi_file_t *file)
     file->f_comm = NULL;
     file->f_filename = NULL;
     file->f_amode = 0;
+    OBJ_CONSTRUCT(&file->f_lock, opal_mutex_t);
 
     /* Initialize flags */
 
@@ -278,8 +286,8 @@ static void file_destructor(ompi_file_t *file)
     /* Finalize the module */
 
     switch (file->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        file->f_io_selected_module.v2_0_0.io_module_file_close(file);
+    case MCA_IO_BASE_V_3_0_0:
+        file->f_io_selected_module.v3_0_0.io_module_file_close(file);
         break;
     default:
         /* Should never get here */
@@ -324,4 +332,6 @@ static void file_destructor(ompi_file_t *file)
         opal_pointer_array_set_item(&ompi_file_f_to_c_table,
                                     file->f_f_to_c_index, NULL);
     }
+
+    OBJ_DESTRUCT(&(file->f_lock));
 }
diff --git a/ompi/file/file.h b/ompi/file/file.h
index bb50903ae5d..4ba7e0e3506 100644
--- a/ompi/file/file.h
+++ b/ompi/file/file.h
@@ -1,4 +1,4 @@
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -16,6 +16,8 @@
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016      University of Houston. All rights reserved.
  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
+ * Copyright (c) 2018      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -37,9 +39,9 @@
 /*
  * Flags
  */
-#define OMPI_FILE_ISCLOSED     0x00000001
-#define OMPI_FILE_HIDDEN       0x00000002
-
+#define OMPI_FILE_ISCLOSED            0x00000001
+#define OMPI_FILE_HIDDEN              0x00000002
+#define OMPI_FILE_ASSERT_NO_ACCEL_BUF 0x00000004
 BEGIN_C_DECLS
 
 /**
@@ -99,7 +101,7 @@ struct ompi_file_t {
 typedef struct ompi_file_t ompi_file_t;
 
 /**
- * Padded struct to maintain back compatibiltiy.
+ * Padded struct to maintain back compatibility.
  * See ompi/communicator/communicator.h comments with struct ompi_communicator_t
  * for full explanation why we chose the following padding construct for predefines.
  */
@@ -134,7 +136,7 @@ int ompi_file_init(void);
 
 /**
  * Back-end to MPI_FILE_OPEN: create a file handle, select an io
- * component to use, and have that componet open the file.
+ * component to use, and have that component open the file.
  *
  * @param comm Communicator
  * @param filename String filename
@@ -155,7 +157,7 @@ int ompi_file_open(struct ompi_communicator_t *comm, const char *filename,
                    ompi_file_t **fh);
 
 /**
- * Atomicly set a name on a file handle.
+ * Atomically set a name on a file handle.
  *
  * @param file MPI_File handle to set the name on
  * @param name NULL-terminated string to use
@@ -185,15 +187,6 @@ int ompi_file_set_name(ompi_file_t *file, char *name);
  */
 int ompi_file_close(ompi_file_t **file);
 
-/**
- * Tear down MPI_File handling.
- *
- * @retval OMPI_SUCCESS Always.
- *
- * Invoked during ompi_mpi_finalize().
- */
-int ompi_file_finalize(void);
-
 /**
  * Check to see if an MPI_File handle is valid.
  *
diff --git a/ompi/group/group.c b/ompi/group/group.c
index 9e368c96da9..3d092433762 100644
--- a/ompi/group/group.c
+++ b/ompi/group/group.c
@@ -18,6 +18,7 @@
  *                         reserved.
  * Copyright (c) 2015-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2020      Triad National Security, LLC. All rights
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -102,7 +103,7 @@ int ompi_group_translate_ranks ( ompi_group_t *group1,
 
     /* loop over all ranks */
     for (int proc = 0; proc < n_ranks; ++proc) {
-        struct ompi_proc_t *proc1_pointer, *proc2_pointer;
+        ompi_process_name_t proc1_name, proc2_name;
         int rank = ranks1[proc];
 
         if ( MPI_PROC_NULL == rank) {
@@ -110,12 +111,12 @@ int ompi_group_translate_ranks ( ompi_group_t *group1,
             continue;
         }
 
-        proc1_pointer = ompi_group_get_proc_ptr_raw (group1, rank);
+        proc1_name = ompi_group_get_proc_name(group1, rank);
         /* initialize to no "match" */
         ranks2[proc] = MPI_UNDEFINED;
         for (int proc2 = 0; proc2 < group2->grp_proc_count; ++proc2) {
-            proc2_pointer = ompi_group_get_proc_ptr_raw (group2, proc2);
-            if ( proc1_pointer == proc2_pointer) {
+            proc2_name = ompi_group_get_proc_name(group2, proc2);
+            if(0 == opal_compare_proc(proc1_name, proc2_name)) {
                 ranks2[proc] = proc2;
                 break;
             }
@@ -201,7 +202,7 @@ int ompi_group_incl(ompi_group_t* group, int n, const int *ranks, ompi_group_t *
         len[2] = ompi_group_calc_sporadic ( n ,ranks );
         len[3] = ompi_group_calc_bmap     ( n , group->grp_proc_count ,ranks );
 
-        /* determin minimum length */
+        /* determine minimum length */
         method = ompi_group_minloc ( len, 4 );
     }
 #endif
@@ -446,7 +447,7 @@ int ompi_group_intersection(ompi_group_t* group1,ompi_group_t* group2,
     int proc1,proc2,k, result;
     int *ranks_included=NULL;
     ompi_group_t *group1_pointer, *group2_pointer;
-    ompi_proc_t *proc1_pointer, *proc2_pointer;
+    ompi_process_name_t proc1_name, proc2_name;
 
     group1_pointer=(ompi_group_t *)group1;
     group2_pointer=(ompi_group_t *)group2;
@@ -462,14 +463,14 @@ int ompi_group_intersection(ompi_group_t* group1,ompi_group_t* group2,
     /* determine the list of included processes for the incl-method */
     k = 0;
     for (proc1 = 0; proc1 < group1_pointer->grp_proc_count; proc1++) {
-        proc1_pointer = ompi_group_peer_lookup (group1_pointer , proc1);
+        proc1_name = ompi_group_get_proc_name(group1_pointer , proc1);
 
         /* check to see if this proc is in group2 */
 
         for (proc2 = 0; proc2 < group2_pointer->grp_proc_count; proc2++) {
-            proc2_pointer = ompi_group_peer_lookup (group2_pointer ,proc2);
+            proc2_name = ompi_group_get_proc_name(group2_pointer ,proc2);
 
-            if( proc1_pointer == proc2_pointer ) {
+            if(0 == opal_compare_proc(proc1_name, proc2_name)) {
                 ranks_included[k] = proc1;
                 k++;
                 break;
@@ -494,7 +495,7 @@ int ompi_group_compare(ompi_group_t *group1,
     int proc1, proc2, match;
     bool similar, identical;
     ompi_group_t *group1_pointer, *group2_pointer;
-    ompi_proc_t *proc1_pointer, *proc2_pointer;
+    opal_process_name_t proc1_name, proc2_name;
 
     /* check for same groups */
     if( group1 == group2 ) {
@@ -524,12 +525,12 @@ int ompi_group_compare(ompi_group_t *group1,
     similar=true;
     identical=true;
     for(proc1=0 ; proc1 < group1_pointer->grp_proc_count ; proc1++ ) {
-        proc1_pointer= ompi_group_peer_lookup(group1_pointer,proc1);
+        proc1_name=ompi_group_get_proc_name(group1_pointer,proc1);
         /* loop over group2 processes to find "match" */
         match=-1;
         for(proc2=0 ; proc2 < group2_pointer->grp_proc_count ; proc2++ ) {
-            proc2_pointer=ompi_group_peer_lookup(group2_pointer,proc2);
-            if( proc1_pointer == proc2_pointer ) {
+            proc2_name=ompi_group_get_proc_name(group2_pointer,proc2);
+            if(0 == opal_compare_proc(proc1_name, proc2_name)) {
                 if(proc1 != proc2 ) {
                     identical=false;
                 }
@@ -606,3 +607,36 @@ int ompi_group_count_local_peers (ompi_group_t *group)
 
     return local_peers;
 }
+
+int ompi_group_to_proc_name_array (ompi_group_t *group, opal_process_name_t **name_array, size_t *name_array_size)
+{
+    opal_process_name_t *array = calloc (group->grp_proc_count, sizeof (array[0]));
+
+    if (NULL == array) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    for (int i = 0 ; i < group->grp_proc_count ; ++i) {
+        array[i] = ompi_group_get_proc_name (group, i);
+    }
+
+    *name_array = array;
+    *name_array_size = group->grp_proc_count;
+
+    return OMPI_SUCCESS;
+}
+
+bool ompi_group_overlap (const ompi_group_t *group1, const ompi_group_t *group2)
+{
+    for (int i = 0 ; i < group1->grp_proc_count ; ++i) {
+        opal_process_name_t proc1 = ompi_group_get_proc_name (group1, i);
+        for (int j = 0 ; j < group2->grp_proc_count ; ++j) {
+            opal_process_name_t proc2 = ompi_group_get_proc_name (group2, j);
+            if (0 == opal_compare_proc (proc1, proc2)) {
+                return true;
+            }
+        }
+    }
+
+    return false;
+}
diff --git a/ompi/group/group.h b/ompi/group/group.h
index 966ab5f8306..c188e98f02f 100644
--- a/ompi/group/group.h
+++ b/ompi/group/group.h
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
+ * Copyright (c) 2004-2023 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -14,10 +14,12 @@
  * Copyright (c) 2007-2017 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2009      Sun Microsystems, Inc. All rights reserved.
  * Copyright (c) 2012      Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013-2017 Los Alamos National Security, LLC.  All rights
+ * Copyright (c) 2013-2018 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018-2022 Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -39,6 +41,7 @@
 #include "opal/class/opal_pointer_array.h"
 #include "opal/mca/threads/threads.h"
 #include "opal/util/output.h"
+#include "ompi/instance/instance.h"
 
 BEGIN_C_DECLS
 
@@ -98,13 +101,15 @@ struct ompi_group_t {
         struct ompi_group_strided_data_t  grp_strided;
         struct ompi_group_bitmap_data_t   grp_bitmap;
     } sparse_data;
+
+    ompi_instance_t *grp_instance; /**< instance this group was allocated within */
 };
 
 typedef struct ompi_group_t ompi_group_t;
 OMPI_DECLSPEC OBJ_CLASS_DECLARATION(ompi_group_t);
 
 /**
- * Padded struct to maintain back compatibiltiy.
+ * Padded struct to maintain back compatibility.
  * See ompi/communicator/communicator.h comments with struct ompi_communicator_t
  * for full explanation why we chose the following padding construct for predefines.
  */
@@ -158,15 +163,29 @@ OMPI_DECLSPEC extern struct opal_mutex_t ompi_group_afp_mutex;
 /**
  * Allocate a new group structure.
  *
+ * @param orig_group Original group 
  * @param group_size Number of MPI processes in the group
  *
  * @return Pointer to new group structure
  */
-OMPI_DECLSPEC ompi_group_t *ompi_group_allocate(int group_size);
-ompi_group_t *ompi_group_allocate_plist_w_procs (ompi_proc_t **procs, int group_size);
-ompi_group_t *ompi_group_allocate_sporadic(int group_size);
-ompi_group_t *ompi_group_allocate_strided(void);
-ompi_group_t *ompi_group_allocate_bmap(int orig_group_size, int group_size);
+OMPI_DECLSPEC ompi_group_t *ompi_group_allocate(ompi_group_t *orig_group, int group_size);
+ompi_group_t *ompi_group_allocate_plist_w_procs (ompi_group_t *orig_group, ompi_proc_t **procs, int group_size);
+ompi_group_t *ompi_group_allocate_sporadic(ompi_group_t *orig_group, int group_size);
+ompi_group_t *ompi_group_allocate_strided(ompi_group_t *orig_group);
+ompi_group_t *ompi_group_allocate_bmap(ompi_group_t *orig_group, int group_size);
+
+/**
+ * @brief Allocate a dense group from a group
+ *
+ * @param[in] group   group
+ *
+ * @returns new group pointer on success
+ * @returns NULL on error
+ *
+ * This function duplicates a group. The new group will have a dense process
+ * table.
+ */
+ompi_group_t *ompi_group_flatten (ompi_group_t *group, int max_procs);
 
 /**
  * Increment the reference count of the proc structures.
@@ -193,14 +212,6 @@ OMPI_DECLSPEC void ompi_group_decrement_proc_count(ompi_group_t *group);
 int ompi_group_init(void);
 
 
-/**
- * Clean up OMPI group infrastructure.
- *
- * @return Error code
- */
-int ompi_group_finalize(void);
-
-
 /**
  * Get group size.
  *
@@ -334,7 +345,7 @@ int ompi_group_calc_bmap ( int n, int orig_size , const int *ranks );
 int ompi_group_minloc (int list[], int length);
 
 /**
- * @brief Helper function for retreiving the proc of a group member in a dense group
+ * @brief Helper function for retrieving the proc of a group member in a dense group
  *
  * This function exists to handle the translation of sentinel group members to real
  * ompi_proc_t's. If a sentinel value is found and allocate is true then this function
@@ -384,15 +395,15 @@ static inline ompi_proc_t *ompi_group_get_proc_ptr (ompi_group_t *group, int ran
 #if OMPI_GROUP_SPARSE
     do {
         if (OMPI_GROUP_IS_DENSE(group)) {
-            return ompi_group_dense_lookup (group, rank, allocate);
+            break;
         }
         int ranks1 = rank;
         ompi_group_translate_ranks (group, 1, &ranks1, group->grp_parent_group_ptr, &rank);
         group = group->grp_parent_group_ptr;
     } while (1);
-#else
-    return ompi_group_dense_lookup (group, rank, allocate);
 #endif
+
+    return ompi_group_dense_lookup (group, rank, allocate);
 }
 
 /**
@@ -402,9 +413,23 @@ static inline ompi_proc_t *ompi_group_get_proc_ptr (ompi_group_t *group, int ran
  * or cached in the proc hash table) or a sentinel value representing the proc. This
  * differs from ompi_group_get_proc_ptr() which returns the ompi_proc_t or NULL.
  */
-ompi_proc_t *ompi_group_get_proc_ptr_raw (ompi_group_t *group, int rank);
+static inline ompi_proc_t *ompi_group_get_proc_ptr_raw (const ompi_group_t *group, int rank)
+{
+#if OMPI_GROUP_SPARSE
+    do {
+        if (OMPI_GROUP_IS_DENSE(group)) {
+            break;
+        }
+        int ranks1 = rank;
+        ompi_group_translate_ranks (group, 1, &ranks1, group->grp_parent_group_ptr, &rank);
+        group = group->grp_parent_group_ptr;
+    } while (1);
+#endif
 
-static inline opal_process_name_t ompi_group_get_proc_name (ompi_group_t *group, int rank)
+    return group->grp_proc_pointers[rank];
+}
+
+static inline opal_process_name_t ompi_group_get_proc_name (const ompi_group_t *group, int rank)
 {
     ompi_proc_t *proc = ompi_group_get_proc_ptr_raw (group, rank);
     if (ompi_proc_is_sentinel (proc)) {
@@ -434,19 +459,17 @@ static inline struct ompi_proc_t *ompi_group_peer_lookup_existing (ompi_group_t
  */
 static inline int ompi_group_proc_lookup_rank (ompi_group_t* group, ompi_proc_t* proc)
 {
-    int i, np, v;
+    int i, np, rank;
+    opal_vpid_t v;
     assert( NULL != proc );
     assert( !ompi_proc_is_sentinel(proc) );
     np = ompi_group_size(group);
     if( 0 == np ) return MPI_PROC_NULL;
     /* heuristic: On comm_world, start the lookup from v=vpid, so that
-     * when working on comm_world, the search is O(1);
-     * Otherwise, wild guess: start from a proportional position
-     * compared to comm_world position. */
+     * when working on comm_world, on average, the search remains O(1). */
     v = proc->super.proc_name.vpid;
-    v = (v<np)? v: v*ompi_proc_world_size()/np;
     for( i = 0; i < np; i++ ) {
-        int rank = (i+v)%np;
+        rank = (i+v)%np;
         /* procs are lazy initialized and may be a sentinel. Handle both cases. */
         ompi_proc_t* p = ompi_group_get_proc_ptr_raw(group, rank);
         if(OPAL_LIKELY(!ompi_proc_is_sentinel(p))) {
@@ -472,6 +495,17 @@ bool ompi_group_have_remote_peers (ompi_group_t *group);
  */
 int ompi_group_count_local_peers (ompi_group_t *group);
 
+/**
+ * @brief Check if groups overlap
+ *
+ * @param[in] group1    ompi group
+ * @param[in] group2    ompi group
+ *
+ * @returns true if any proc in group1 is also in group2
+ * @returns false otherwise
+ */
+bool ompi_group_overlap (const ompi_group_t *group1, const ompi_group_t *group2);
+
 /**
  *  Function to print the group info
  */
@@ -482,5 +516,19 @@ int ompi_group_dump (ompi_group_t* group);
  */
 int ompi_group_div_ceil (int num, int den);
 
+/**
+ * Create a process name array from a group
+ */
+int ompi_group_to_proc_name_array (ompi_group_t *group, opal_process_name_t **name_array, size_t *name_array_size);
+
+/**
+ * Return instance from a group
+ */
+
+static inline ompi_instance_t *ompi_group_get_instance(ompi_group_t *group)
+{
+    return group->grp_instance;
+}
+
 END_C_DECLS
 #endif /* OMPI_GROUP_H */
diff --git a/ompi/group/group_bitmap.c b/ompi/group/group_bitmap.c
index c93184f162c..c70e6cde51f 100644
--- a/ompi/group/group_bitmap.c
+++ b/ompi/group/group_bitmap.c
@@ -14,6 +14,8 @@
  * Copyright (c) 2007      Cisco Systems, Inc. All rights reserved.
  * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
  *                         reserved.
+ * Copyright (c) 2022      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -54,13 +56,13 @@ int ompi_group_translate_ranks_bmap ( ompi_group_t *parent_group,
             m = ranks1[j];
             count = 0;
             tmp = ( 1 << (m % BSIZE) );
-            /* check if the bit that correponds to the parent rank is set in the bitmap */
+            /* check if the bit that corresponds to the parent rank is set in the bitmap */
             if ( tmp == (child_group->sparse_data.grp_bitmap.grp_bitmap_array[(int)(m/BSIZE)]
                          & (1 << (m % BSIZE)))) {
                 /*
                  * add up how many bits are set, till we get to the bit of parent
                  * rank that we want. The rank in the child will be the sum of the bits
-                 * that are set on the way till we get to the correponding bit
+                 * that are set on the way till we get to the corresponding bit
                  */
                 for (i=0 ; i<=(int)(m/BSIZE) ; i++) {
                     for (k=0 ; k<BSIZE ; k++) {
@@ -131,7 +133,7 @@ int ompi_group_div_ceil (int num, int den)
 /*
  * This functions is to check that all ranks in the included list of ranks
  * are monotonically increasing. If not, the bitmap format can not be used
- * since we won't be able to translate the ranks corrently since the algorithms
+ * since we won't be able to translate the ranks correctly since the algorithms
  * assume that the ranks are in order in the bitmap list.
  */
 static bool check_ranks (int n, const int *ranks) {
@@ -159,7 +161,7 @@ int ompi_group_incl_bmap(ompi_group_t* group, int n, const int *ranks,
         return OMPI_SUCCESS;
     }
 
-    new_group_pointer = ompi_group_allocate_bmap(group->grp_proc_count, n);
+    new_group_pointer = ompi_group_allocate_bmap(group, n);
     if( NULL == new_group_pointer ) {
         return MPI_ERR_GROUP;
     }
diff --git a/ompi/group/group_init.c b/ompi/group/group_init.c
index fed47997218..77fd697aa22 100644
--- a/ompi/group/group_init.c
+++ b/ompi/group/group_init.c
@@ -16,6 +16,8 @@
  * Copyright (c) 2012      Oak Ridge National Labs.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018-2022 Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -32,6 +34,8 @@
 static void ompi_group_construct(ompi_group_t *);
 static void ompi_group_destruct(ompi_group_t *);
 
+static int ompi_group_finalize (void);
+
 OBJ_CLASS_INSTANCE(ompi_group_t,
                    opal_object_t,
                    ompi_group_construct,
@@ -67,7 +71,7 @@ opal_mutex_t ompi_group_afp_mutex = OPAL_MUTEX_STATIC_INIT;
 /*
  * Allocate a new group structure
  */
-ompi_group_t *ompi_group_allocate(int group_size)
+ompi_group_t *ompi_group_allocate(ompi_group_t *orig_group, int group_size)
 {
     /* local variables */
     ompi_proc_t **procs = calloc (group_size, sizeof (ompi_proc_t *));
@@ -77,7 +81,7 @@ ompi_group_t *ompi_group_allocate(int group_size)
         return NULL;
     }
 
-    new_group = ompi_group_allocate_plist_w_procs (procs, group_size);
+    new_group = ompi_group_allocate_plist_w_procs (orig_group, procs, group_size);
     if (NULL == new_group) {
         free (procs);
     }
@@ -85,7 +89,7 @@ ompi_group_t *ompi_group_allocate(int group_size)
     return new_group;
 }
 
-ompi_group_t *ompi_group_allocate_plist_w_procs (ompi_proc_t **procs, int group_size)
+ompi_group_t *ompi_group_allocate_plist_w_procs (ompi_group_t *orig_group, ompi_proc_t **procs, int group_size)
 {
     /* local variables */
     ompi_group_t * new_group = NULL;
@@ -117,12 +121,18 @@ ompi_group_t *ompi_group_allocate_plist_w_procs (ompi_proc_t **procs, int group_
     new_group->grp_my_rank = MPI_UNDEFINED;
     OMPI_GROUP_SET_DENSE(new_group);
 
+    if (NULL != orig_group) {
+        new_group->grp_instance = orig_group->grp_instance;
+    } else {
+        new_group->grp_instance = NULL;
+    }
+
     ompi_group_increment_proc_count (new_group);
 
     return new_group;
 }
 
-ompi_group_t *ompi_group_allocate_sporadic(int group_size)
+ompi_group_t *ompi_group_allocate_sporadic(ompi_group_t *orig_group, int group_size)
 {
     /* local variables */
     ompi_group_t *new_group = NULL;
@@ -161,13 +171,14 @@ ompi_group_t *ompi_group_allocate_sporadic(int group_size)
     /* initialize our rank to MPI_UNDEFINED */
     new_group->grp_my_rank       = MPI_UNDEFINED;
     new_group->grp_proc_pointers = NULL;
+    new_group->grp_instance = orig_group->grp_instance;
     OMPI_GROUP_SET_SPORADIC(new_group);
 
  error_exit:
     return new_group;
 }
 
-ompi_group_t *ompi_group_allocate_strided(void)
+ompi_group_t *ompi_group_allocate_strided(ompi_group_t *orig_group)
 {
     ompi_group_t *new_group = NULL;
 
@@ -184,6 +195,7 @@ ompi_group_t *ompi_group_allocate_strided(void)
     /* initialize our rank to MPI_UNDEFINED */
     new_group->grp_my_rank    = MPI_UNDEFINED;
     new_group->grp_proc_pointers     = NULL;
+    new_group->grp_instance = orig_group->grp_instance;
     OMPI_GROUP_SET_STRIDED(new_group);
     new_group->sparse_data.grp_strided.grp_strided_stride         = -1;
     new_group->sparse_data.grp_strided.grp_strided_offset         = -1;
@@ -192,9 +204,13 @@ ompi_group_t *ompi_group_allocate_strided(void)
     /* return */
     return new_group;
 }
-ompi_group_t *ompi_group_allocate_bmap(int orig_group_size , int group_size)
+
+ompi_group_t *ompi_group_allocate_bmap(ompi_group_t *orig_group , int group_size)
 {
     ompi_group_t *new_group = NULL;
+    int orig_group_size;
+
+    orig_group_size = orig_group->grp_proc_count;
 
     assert (group_size >= 0);
 
@@ -220,6 +236,7 @@ ompi_group_t *ompi_group_allocate_bmap(int orig_group_size , int group_size)
     /* initialize our rank to MPI_UNDEFINED */
     new_group->grp_my_rank    = MPI_UNDEFINED;
     new_group->grp_proc_pointers     = NULL;
+    new_group->grp_instance = orig_group->grp_instance;
     OMPI_GROUP_SET_BITMAP(new_group);
 
  error_exit:
@@ -227,6 +244,71 @@ ompi_group_t *ompi_group_allocate_bmap(int orig_group_size , int group_size)
     return new_group;
 }
 
+/**
+ * @brief Allocate a dense group from a group
+ *
+ * @param[in] group   group
+ *
+ * @returns new group pointer on success
+ * @returns NULL on error
+ *
+ * This function duplicates a group. The new group will have a dense process
+ * table.
+ */
+ompi_group_t *ompi_group_flatten (ompi_group_t *group, int max_procs)
+{
+    int proc_count = (max_procs > group->grp_proc_count) ? group->grp_proc_count : max_procs;
+    size_t proc_pointer_array_size = proc_count * sizeof (group->grp_proc_pointers[0]);
+    ompi_group_t *new_group = OBJ_NEW(ompi_group_t);;
+    if (NULL == new_group) {
+        return NULL;
+    }
+
+    if (0 > new_group->grp_f_to_c_index) {
+        OBJ_RELEASE (new_group);
+        return NULL;
+    }
+
+    if (0 != proc_count) {
+        new_group->grp_proc_pointers = malloc (proc_pointer_array_size);
+        if (OPAL_UNLIKELY(NULL == new_group->grp_proc_pointers)) {
+            OBJ_RELEASE(new_group);
+            return NULL;
+        }
+
+        /*
+         * Allocate array of (ompi_proc_t *)'s, one for each
+         * process in the group.
+         */
+        if (!OMPI_GROUP_IS_DENSE(group)) {
+            for (int i = 0 ; i < proc_count ; i++) {
+                new_group->grp_proc_pointers[i] = ompi_group_peer_lookup (group, i);
+            }
+        } else {
+            memcpy (new_group->grp_proc_pointers, group->grp_proc_pointers, proc_pointer_array_size);
+        }
+    }
+
+    /* set the group size */
+    new_group->grp_proc_count = proc_count;
+
+    if (group->grp_my_rank >= max_procs) {
+        /* initialize our rank to MPI_UNDEFINED */
+        new_group->grp_my_rank = MPI_UNDEFINED;
+    } else {
+        /* rank is the same as in the old group */
+        new_group->grp_my_rank = group->grp_my_rank;
+    }
+
+    new_group->grp_instance = group->grp_instance;
+
+    OMPI_GROUP_SET_DENSE(new_group);
+
+    ompi_group_increment_proc_count (new_group);
+
+    return new_group;
+}
+
 /*
  * increment the reference count of the proc structures
  */
@@ -265,7 +347,7 @@ static void ompi_group_construct(ompi_group_t *new_group)
 
     /* Note that we do *NOT* increase the refcount on all the included
        procs here because that is handled at a different level (e.g.,
-       the proc counts are not decreased during the desstructor,
+       the proc counts are not decreased during the destructor,
        either). */
 
     /* assign entry in fortran <-> c translation array */
@@ -337,16 +419,6 @@ int ompi_group_init(void)
         return OMPI_ERROR;
     }
 
-#if OPAL_ENABLE_FT_MPI
-    /* Setup global list of failed processes */
-    ompi_group_all_failed_procs = OBJ_NEW(ompi_group_t);
-    ompi_group_all_failed_procs->grp_proc_count     = 0;
-    ompi_group_all_failed_procs->grp_my_rank        = MPI_UNDEFINED;
-    ompi_group_all_failed_procs->grp_proc_pointers  = NULL;
-    ompi_group_all_failed_procs->grp_flags         |= OMPI_GROUP_DENSE;
-    ompi_group_all_failed_procs->grp_flags         |= OMPI_GROUP_INTRINSIC;
-#endif
-
     /* add MPI_GROUP_NULL to table */
     OBJ_CONSTRUCT(&ompi_mpi_group_null, ompi_group_t);
     ompi_mpi_group_null.group.grp_proc_count        = 0;
@@ -363,6 +435,19 @@ int ompi_group_init(void)
     ompi_mpi_group_empty.group.grp_flags            |= OMPI_GROUP_DENSE;
     ompi_mpi_group_empty.group.grp_flags            |= OMPI_GROUP_INTRINSIC;
 
+#if OPAL_ENABLE_FT_MPI
+    /* Setup global list of failed processes */
+    ompi_group_all_failed_procs = OBJ_NEW(ompi_group_t);
+    ompi_group_all_failed_procs->grp_proc_count     = 0;
+    ompi_group_all_failed_procs->grp_my_rank        = MPI_UNDEFINED;
+    ompi_group_all_failed_procs->grp_proc_pointers  = NULL;
+    ompi_group_all_failed_procs->grp_flags         |= OMPI_GROUP_DENSE;
+    ompi_group_all_failed_procs->grp_flags         |= OMPI_GROUP_INTRINSIC;
+#endif
+
+
+    ompi_mpi_instance_append_finalize (ompi_group_finalize);
+
     return OMPI_SUCCESS;
 }
 
@@ -370,7 +455,7 @@ int ompi_group_init(void)
 /*
  * Clean up group infrastructure
  */
-int ompi_group_finalize(void)
+static int ompi_group_finalize (void)
 {
     ompi_mpi_group_null.group.grp_flags = 0;
     OBJ_DESTRUCT(&ompi_mpi_group_null);
diff --git a/ompi/group/group_plist.c b/ompi/group/group_plist.c
index 244cd17385e..15ad075e9c0 100644
--- a/ompi/group/group_plist.c
+++ b/ompi/group/group_plist.c
@@ -17,6 +17,8 @@
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2017      Intel, Inc. All rights reserved.
+ * Copyright (c) 2022      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -35,18 +37,18 @@
 
 static int ompi_group_dense_overlap (ompi_group_t *group1, ompi_group_t *group2, opal_bitmap_t *bitmap)
 {
-    ompi_proc_t *proc1_pointer, *proc2_pointer;
+    ompi_process_name_t proc1_name, proc2_name;
     int rc, overlap_count;
 
     overlap_count = 0;
 
     for (int proc1 = 0 ; proc1 < group1->grp_proc_count ; ++proc1) {
-        proc1_pointer = ompi_group_get_proc_ptr_raw (group1, proc1);
+        proc1_name = ompi_group_get_proc_name(group1, proc1);
 
         /* check to see if this proc is in group2 */
         for (int proc2 = 0 ; proc2 < group2->grp_proc_count ; ++proc2) {
-            proc2_pointer = ompi_group_get_proc_ptr_raw (group2, proc2);
-            if( proc1_pointer == proc2_pointer ) {
+            proc2_name = ompi_group_get_proc_name(group2, proc2);
+            if(0 == opal_compare_proc(proc1_name, proc2_name)) {
                 rc = opal_bitmap_set_bit (bitmap, proc2);
                 if (OPAL_SUCCESS != rc) {
                     return rc;
@@ -61,38 +63,6 @@ static int ompi_group_dense_overlap (ompi_group_t *group1, ompi_group_t *group2,
     return overlap_count;
 }
 
-static struct ompi_proc_t *ompi_group_dense_lookup_raw (ompi_group_t *group, const int peer_id)
-{
-    if (OPAL_UNLIKELY(ompi_proc_is_sentinel (group->grp_proc_pointers[peer_id]))) {
-        ompi_proc_t *proc =
-            (ompi_proc_t *) ompi_proc_lookup (ompi_proc_sentinel_to_name ((uintptr_t) group->grp_proc_pointers[peer_id]));
-        if (NULL != proc) {
-            /* replace sentinel value with an actual ompi_proc_t */
-            group->grp_proc_pointers[peer_id] = proc;
-            /* retain the proc */
-            OBJ_RETAIN(group->grp_proc_pointers[peer_id]);
-        }
-    }
-
-    return group->grp_proc_pointers[peer_id];
-}
-
-ompi_proc_t *ompi_group_get_proc_ptr_raw (ompi_group_t *group, int rank)
-{
-#if OMPI_GROUP_SPARSE
-    do {
-        if (OMPI_GROUP_IS_DENSE(group)) {
-            return ompi_group_dense_lookup_raw (group, rank);
-        }
-        int ranks1 = rank;
-        ompi_group_translate_ranks (group, 1, &ranks1, group->grp_parent_group_ptr, &rank);
-        group = group->grp_parent_group_ptr;
-    } while (1);
-#else
-    return ompi_group_dense_lookup_raw (group, rank);
-#endif
-}
-
 int ompi_group_calc_plist ( int n , const int *ranks ) {
     return sizeof(char *) * n ;
 }
@@ -113,7 +83,7 @@ int ompi_group_incl_plist(ompi_group_t* group, int n, const int *ranks,
     }
 
     /* get new group struct */
-    new_group_pointer=ompi_group_allocate(n);
+    new_group_pointer=ompi_group_allocate(group,n);
     if( NULL == new_group_pointer ) {
         return MPI_ERR_GROUP;
     }
@@ -135,6 +105,8 @@ int ompi_group_incl_plist(ompi_group_t* group, int n, const int *ranks,
         new_group_pointer->grp_my_rank = MPI_UNDEFINED;
     }
 
+    new_group_pointer->grp_instance = group->grp_instance;
+
     *new_group = (MPI_Group)new_group_pointer;
 
     return OMPI_SUCCESS;
@@ -180,7 +152,7 @@ int ompi_group_union (ompi_group_t* group1, ompi_group_t* group2,
     }
 
     /* get new group struct */
-    new_group_pointer = ompi_group_allocate(new_group_size);
+    new_group_pointer = ompi_group_allocate(group1, new_group_size);
     if (NULL == new_group_pointer) {
         OBJ_DESTRUCT(&bitmap);
         return MPI_ERR_GROUP;
@@ -229,7 +201,7 @@ int ompi_group_union (ompi_group_t* group1, ompi_group_t* group2,
 int ompi_group_difference(ompi_group_t* group1, ompi_group_t* group2,
                           ompi_group_t **new_group) {
 
-    /* local varibles */
+    /* local variables */
     int new_group_size, overlap_count, rc;
     ompi_group_t *new_group_pointer;
     ompi_proc_t *proc1_pointer;
@@ -262,7 +234,7 @@ int ompi_group_difference(ompi_group_t* group1, ompi_group_t* group2,
     }
 
     /* allocate a new ompi_group_t structure */
-    new_group_pointer = ompi_group_allocate(new_group_size);
+    new_group_pointer = ompi_group_allocate(group1, new_group_size);
     if( NULL == new_group_pointer ) {
         OBJ_DESTRUCT(&bitmap);
         return MPI_ERR_GROUP;
diff --git a/ompi/group/group_sporadic.c b/ompi/group/group_sporadic.c
index 46e0d6bc9f8..ac630a76086 100644
--- a/ompi/group/group_sporadic.c
+++ b/ompi/group/group_sporadic.c
@@ -14,6 +14,8 @@
  * Copyright (c) 2007      Cisco Systems, Inc. All rights reserved.
  * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
  *                         reserved.
+ * Copyright (c) 2022      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -144,7 +146,7 @@ int ompi_group_incl_spor(ompi_group_t* group, int n, const int *ranks,
         }
     }
 
-    new_group_pointer = ompi_group_allocate_sporadic(l);
+    new_group_pointer = ompi_group_allocate_sporadic(group,l);
     if( NULL == new_group_pointer ) {
         return MPI_ERR_GROUP;
     }
diff --git a/ompi/group/group_strided.c b/ompi/group/group_strided.c
index d5f7f2848a8..fa6185204f0 100644
--- a/ompi/group/group_strided.c
+++ b/ompi/group/group_strided.c
@@ -14,6 +14,8 @@
  * Copyright (c) 2007      Cisco Systems, Inc. All rights reserved.
  * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
  *                         reserved.
+ * Copyright (c) 2022      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -117,7 +119,7 @@ int ompi_group_incl_strided(ompi_group_t* group, int n, const int *ranks,
     }
 
     stride = check_stride(ranks,n);
-    new_group_pointer = ompi_group_allocate_strided();
+    new_group_pointer = ompi_group_allocate_strided(group);
     if( NULL == new_group_pointer ) {
         return MPI_ERR_GROUP;
     }
diff --git a/ompi/include/Makefile.am b/ompi/include/Makefile.am
index 8384b864bef..5a27d1b68bd 100644
--- a/ompi/include/Makefile.am
+++ b/ompi/include/Makefile.am
@@ -11,9 +11,10 @@
 #                         All rights reserved.
 # Copyright (c) 2006-2015 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2009-2011 Oak Ridge National Labs.  All rights reserved.
-# Copyright (c) 2014-2015 Research Organization for Information Science
-#                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2014-2025 Research Organization for Information Science
+#                         and Technology (RIST).  All rights reserved.
 # Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+# Copyright (c) 2022      IBM Corporation.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -46,8 +47,9 @@ include_HEADERS += \
 endif
 
 # These files are always installed in $(includedir), but shouldn't be
-# shipped since they are generated by configure from their .in
-# counterparts (which AM automatically ships).
+# shipped since they are generated by configure from .in counterparts
+# (which AM automatically ships) or other sources (which are shipped
+# by other Makefile.am's).
 nodist_include_HEADERS = \
         mpi.h \
         mpi-ext.h \
@@ -88,9 +90,12 @@ mpif-sizeof.h:
 	    --maxrank=$(OMPI_FORTRAN_MAX_ARRAY_RANK) \
 	    --generate=$(OMPI_FORTRAN_BUILD_SIZEOF) \
 	    --real2=$(OMPI_HAVE_FORTRAN_REAL2) \
+	    --iso_real16=$(OMPI_FORTRAN_HAVE_ISO_FORTRAN_ENV_REAL16) \
 	    --real16=$(OMPI_HAVE_FORTRAN_REAL16) \
 	    --complex4=$(OMPI_HAVE_FORTRAN_COMPLEX4) \
-	    --complex32=$(OMPI_HAVE_FORTRAN_COMPLEX32)
+	    --complex32=$(OMPI_HAVE_FORTRAN_COMPLEX32) \
+	    --mpi_version=$(MPI_VERSION) \
+	    --request_deprecate=$(OMPI_FORTRAN_HAVE_ATTR_DEPRECATED)
 
 #
 # mpif-c-constants-decl.h, among other files, is generated based on some
@@ -105,7 +110,10 @@ mpif-c-constants-decl.h:
 	    --caps $(OMPI_FORTRAN_CAPS) \
 	    --plain $(OMPI_FORTRAN_PLAIN) \
 	    --single $(OMPI_FORTRAN_SINGLE_UNDERSCORE) \
-	    --double $(OMPI_FORTRAN_DOUBLE_UNDERSCORE)
+	    --double $(OMPI_FORTRAN_DOUBLE_UNDERSCORE) \
+	    --status-size $(OMPI_FORTRAN_STATUS_SIZE) \
+	    --align $(OMPI_FORTRAN_COMMON_ALIGNMENT)
+
 
 if WANT_INSTALL_HEADERS
 ompidir = $(ompiincludedir)
@@ -126,6 +134,6 @@ distclean-local:
 	    mpif-sizeof.h \
 	    mpif-c-constants-decl.h mpif-c-constants.h mpif-f08-types.h
 
-mpi_portable_platform.h: $(top_srcdir)/opal/include/opal/opal_portable_platform.h
+mpi_portable_platform.h: $(top_srcdir)/opal/include/opal/opal_portable_platform_real.h
 	-@rm -f mpi_portable_platform.h
-	$(OMPI_V_LN_S) $(LN_S) $(top_srcdir)/opal/include/opal/opal_portable_platform.h mpi_portable_platform.h
+	$(OMPI_V_LN_S) $(LN_S) $(top_srcdir)/opal/include/opal/opal_portable_platform_real.h mpi_portable_platform.h
diff --git a/ompi/include/mpi.h.in b/ompi/include/mpi.h.in
index 7fb7274620e..4886868c37c 100644
--- a/ompi/include/mpi.h.in
+++ b/ompi/include/mpi.h.in
@@ -9,18 +9,25 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2007-2020 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2007-2022 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2008-2009 Sun Microsystems, Inc.  All rights reserved.
  * Copyright (c) 2009-2012 Oak Rigde National Laboratory.  All rights reserved.
- * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2011-2020 Sandia National Laboratories. All rights reserved.
  * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2011-2013 INRIA.  All rights reserved.
  * Copyright (c) 2015      University of Houston. All rights reserved.
- * Copyright (c) 2015-2020 Research Organization for Information Science
+ * Copyright (c) 2015-2023 Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2017-2019 IBM Corporation.  All rights reserved.
+ * Copyright (c) 2017-2022 IBM Corporation.  All rights reserved.
  * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2021-2022 Google, LLC. All rights reserved.
+ * Copyright (c) 2021-2022 Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
+ * Copyright (c) 2021      Bull S.A.S. All rights reserved.
+ * Copyright (c) 2018-2022 Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2025      Advanced Micro Devices, Inc. All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -77,6 +84,9 @@
 /* The size of a `int', as computed by sizeof. */
 #undef OPAL_SIZEOF_INT
 
+/* The size of a `void*', as computed by sizeof. */
+#undef OPAL_SIZEOF_VOID_P
+
 /* Maximum length of datarep string (default is 128) */
 #undef OPAL_MAX_DATAREP_STRING
 
@@ -98,6 +108,12 @@
 /* Maximum length of processor names (default is 256) */
 #undef OPAL_MAX_PROCESSOR_NAME
 
+/* Maximum length of processor names (default is 1024) */
+#undef OPAL_MAX_PSET_NAME_LEN
+
+/* Maximum length of from group tag (default is 256) */
+#undef OPAL_MAX_STRINGTAG_LEN
+
 /* The number or Fortran INTEGER in MPI Status */
 #undef OMPI_FORTRAN_STATUS_SIZE
 
@@ -183,6 +199,12 @@
 #undef OMPI_MINOR_VERSION
 #undef OMPI_RELEASE_VERSION
 
+/*
+ * MPI version
+ */
+#undef MPI_VERSION
+#undef MPI_SUBVERSION
+
 /* A  type that allows us to have sentinel type values that are still
    valid */
 #undef ompi_fortran_bogus_type_t
@@ -233,12 +255,6 @@
  */
 #define OPEN_MPI 1
 
-/*
- * MPI version
- */
-#define MPI_VERSION 3
-#define MPI_SUBVERSION 1
-
 
 /*
  * Do we want MPI interface deprecated function warnings?  This is
@@ -248,10 +264,22 @@
 #if !OMPI_BUILDING
      /*
       * Figure out which compiler is being invoked (in order to compare if
-      * it was different than what OMPI was built with).
+      * it was different than what OMPI was built with).  Do some preprocessor
+      * hacks to eliminate warnings in the portable_platform.h file.
       */
+#    ifndef SIZEOF_VOID_P
+#    define OMPI_CLEANUP_SIZEOF_VOID_P 1
+#    define SIZEOF_VOID_P OPAL_SIZEOF_VOID_P
+#    else
+#    define OMPI_CLEANUP_SIZEOF_VOID_P 0
+#    endif
+
 #    include "mpi_portable_platform.h"
 
+#if OMPI_CLEANUP_SIZEOF_VOID_P
+#undef SIZEOF_VOID_P
+#endif
+
      /*
       * If we're currently using the same compiler that was used to
       * build Open MPI, enable compile-time warning of user-level code
@@ -295,7 +323,9 @@
           * Don't do MACRO magic for building Profiling library as it
           * interferes with the above.
           */
-#        if (OMPI_ENABLE_MPI1_COMPAT || OMPI_BUILDING)
+#        if defined(OMPI_OMIT_MPI1_COMPAT_DECLS)
+             /* The user set OMPI_OMIT_MPI1_COMPAT_DECLS, do what he commands */
+#        elif OMPI_ENABLE_MPI1_COMPAT
 #            define OMPI_OMIT_MPI1_COMPAT_DECLS 0
 #            define OMPI_REMOVED_USE_STATIC_ASSERT 0
 #            define __mpi_interface_removed__(func, newfunc) __mpi_interface_deprecated__(#func " was removed in MPI-3.0.  Use " #newfunc " instead.  continuing...")
@@ -352,12 +382,18 @@
 #    define THIS_FUNCTION_WAS_REMOVED_IN_MPI30(func, newfunc)
 #endif
 
+#if MPI_VERSION >= 4
+#    define __mpi_interface_deprecated_in_mpi40__(msg) __mpi_interface_deprecated__(msg)
+#else
+#    define __mpi_interface_deprecated_in_mpi40__(msg)
+#endif
+
 #if !defined(OMPI_REMOVED_USE_STATIC_ASSERT)
 #    define OMPI_REMOVED_USE_STATIC_ASSERT 0
 #endif
 
 #if !defined(OMPI_OMIT_MPI1_COMPAT_DECLS)
-#    define OMPI_OMIT_MPI1_COMPAT_DECLS !OMPI_ENABLE_MPI1_COMPAT
+#    define OMPI_OMIT_MPI1_COMPAT_DECLS (!(OMPI_ENABLE_MPI1_COMPAT || OMPI_BUILDING))
 #endif
 
 /*
@@ -417,6 +453,9 @@ typedef struct mca_base_var_enum_t *MPI_T_enum;
 typedef struct ompi_mpit_cvar_handle_t *MPI_T_cvar_handle;
 typedef struct mca_base_pvar_handle_t *MPI_T_pvar_handle;
 typedef struct mca_base_pvar_session_t *MPI_T_pvar_session;
+typedef struct ompi_instance_t *MPI_Session;
+typedef unsigned long *MPI_T_event_instance;
+typedef unsigned long *MPI_T_event_registration;
 
 /*
  * MPI_Status
@@ -455,7 +494,10 @@ typedef struct ompi_f08_status_public_t ompi_f08_status_public_t;
 typedef int (MPI_Datarep_extent_function)(MPI_Datatype, MPI_Aint *, void *);
 typedef int (MPI_Datarep_conversion_function)(void *, MPI_Datatype,
                                               int, void *, MPI_Offset, void *);
+typedef int (MPI_Datarep_conversion_function_c)(void *, MPI_Datatype,
+                                              MPI_Count, void *, MPI_Offset, void *);
 typedef void (MPI_Comm_errhandler_function)(MPI_Comm *, int *, ...);
+typedef void (MPI_Session_errhandler_function) (MPI_Session *, int *, ...);
 
     /* This is a little hackish, but errhandler.h needs space for a
        MPI_File_errhandler_function.  While it could just be removed, this
@@ -464,6 +506,7 @@ typedef void (MPI_Comm_errhandler_function)(MPI_Comm *, int *, ...);
 typedef void (ompi_file_errhandler_function)(MPI_File *, int *, ...);
 typedef void (MPI_Win_errhandler_function)(MPI_Win *, int *, ...);
 typedef void (MPI_User_function)(void *, void *, int *, MPI_Datatype *);
+typedef void (MPI_User_function_c)(void *, void *, MPI_Count *, MPI_Datatype *);
 typedef int (MPI_Comm_copy_attr_function)(MPI_Comm, int, void *,
                                             void *, void *, int *);
 typedef int (MPI_Comm_delete_attr_function)(MPI_Comm, int, void *, void *);
@@ -493,6 +536,11 @@ typedef MPI_Win_errhandler_function MPI_Win_errhandler_fn
 
 /*
  * Miscellaneous constants
+ *
+ * NOTE: Many of the integer constants below *also* appear in
+ * ompi/include/mpif-values.pl.  If you change any of these integer
+ * values below, make sure to also change the corresponding values in
+ * mpif-values.pl.
  */
 #define MPI_ANY_SOURCE         -1                      /* match any source rank */
 #define MPI_PROC_NULL          -2                      /* rank of null process */
@@ -528,6 +576,20 @@ typedef MPI_Win_errhandler_function MPI_Win_errhandler_fn
 #define MPI_DISTRIBUTE_CYCLIC    1                     /* cyclic distribution */
 #define MPI_DISTRIBUTE_NONE      2                     /* not distributed */
 #define MPI_DISTRIBUTE_DFLT_DARG (-1)                  /* default distribution arg */
+#define MPI_MAX_PSET_NAME_LEN    OPAL_MAX_PSET_NAME_LEN /* max pset name len */
+#define MPI_MAX_STRINGTAG_LEN    OPAL_MAX_STRINGTAG_LEN /* max length of string arg to comm from group funcs*/
+
+/*
+ * Constants for C code to access elements in Fortran MPI status array.
+ *
+ * NOTE: The MPI_F_SOURCE, MPI_F_TAG, MPI_F_ERROR are intentionally 1
+ * smaller than their Fortran equivalents in mpif-values.pl (because C
+ * is 0-indexed and Fortran is 1-indexed).
+ */
+#define MPI_F_STATUS_SIZE OMPI_FORTRAN_STATUS_SIZE    /* Size of Fortran MPI status array */
+#define MPI_F_SOURCE      0                           /* Index for MPI_SOURCE */
+#define MPI_F_TAG         1                           /* Index for MPI_TAG */
+#define MPI_F_ERROR       2                           /* Index for MPI_ERROR */
 
 /*
  * Since these values are arbitrary to Open MPI, we might as well make
@@ -577,7 +639,8 @@ typedef MPI_Win_errhandler_function MPI_Win_errhandler_fn
  * Predefined attribute keyvals
  *
  * DO NOT CHANGE THE ORDER WITHOUT ALSO CHANGING THE ORDER IN
- * src/attribute/attribute_predefined.c and mpif.h.in.
+ * ompi/attribute/attribute_predefined.c and mpif-constants.h (which
+ * is included by mpif.h.in).
  */
 enum {
     /* MPI-1 */
@@ -598,6 +661,7 @@ enum {
 
     /* MPI-4 */
     MPI_FT, /* used by OPAL_ENABLE_FT_MPI */
+    MPI_ATTR_PREDEFINED_KEY_MAX,
 };
 
 /*
@@ -685,6 +749,8 @@ enum {
 #define MPI_ERR_PROC_FAILED           75
 #define MPI_ERR_PROC_FAILED_PENDING   76
 #define MPI_ERR_REVOKED               77
+#define MPI_ERR_SESSION               78
+#define MPI_ERR_VALUE_TOO_LARGE       79
 
 /* Per MPI-3 p349 47, MPI_ERR_LASTCODE must be >= the last predefined
    MPI_ERR_<foo> code. Set the last code to allow some room for adding
@@ -694,7 +760,8 @@ enum {
 /*
  * Comparison results.  Don't change the order of these, the group
  * comparison functions rely on it.
- * Do not change the order of these without also modifying mpif.h.in.
+ * Do not change the order of these without also modifying
+ * mpif-values.pl.
  */
 enum {
   MPI_IDENT,
@@ -705,7 +772,8 @@ enum {
 
 /*
  * MPI_Init_thread constants
- * Do not change the order of these without also modifying mpif.h.in.
+ * Do not change the order of these without also modifying
+ * mpif-values.pl.
  */
 enum {
   MPI_THREAD_SINGLE,
@@ -716,21 +784,33 @@ enum {
 
 /*
  * Datatype combiners.
- * Do not change the order of these without also modifying mpif.h.in.
- * (see also mpif-common.h.fin).
+ * Do not change the order of these without also modifying
+ * mpif-values.pl.
  */
 enum {
   MPI_COMBINER_NAMED,
   MPI_COMBINER_DUP,
   MPI_COMBINER_CONTIGUOUS,
   MPI_COMBINER_VECTOR,
+#if (!OMPI_OMIT_MPI1_COMPAT_DECLS)
   MPI_COMBINER_HVECTOR_INTEGER,
+#else
+  OMPI_WAS_MPI_COMBINER_HVECTOR_INTEGER, /* preserve ABI compatibility */
+#endif
   MPI_COMBINER_HVECTOR,
   MPI_COMBINER_INDEXED,
+#if (!OMPI_OMIT_MPI1_COMPAT_DECLS)
   MPI_COMBINER_HINDEXED_INTEGER,
+#else
+  OMPI_WAS_MPI_COMBINER_HINDEXED_INTEGER, /* preserve ABI compatibility */
+#endif
   MPI_COMBINER_HINDEXED,
   MPI_COMBINER_INDEXED_BLOCK,
+#if (!OMPI_OMIT_MPI1_COMPAT_DECLS)
   MPI_COMBINER_STRUCT_INTEGER,
+#else
+  OMPI_WAS_MPI_COMBINER_STRUCT_INTEGER, /* preserve ABI compatibility */
+#endif
   MPI_COMBINER_STRUCT,
   MPI_COMBINER_SUBARRAY,
   MPI_COMBINER_DARRAY,
@@ -741,10 +821,23 @@ enum {
   MPI_COMBINER_HINDEXED_BLOCK
 };
 
+#if (OMPI_OMIT_MPI1_COMPAT_DECLS)
+/* If not building or configured --enable-mpi1-compatibility, then
+ * we don't want these datatypes, instead we define MPI_COMBINER_*_INTEGER
+ * to our Static Assert message if the compiler supports
+ * that static assert with a nice message.
+ */
+#    if (OMPI_REMOVED_USE_STATIC_ASSERT)
+#        define MPI_COMBINER_HVECTOR_INTEGER THIS_SYMBOL_WAS_REMOVED_IN_MPI30(MPI_COMBINER_HVECTOR_INTEGER, MPI_COMBINER_HVECTOR);
+#        define MPI_COMBINER_HINDEXED_INTEGER THIS_SYMBOL_WAS_REMOVED_IN_MPI30(MPI_COMBINER_HINDEXED_INTEGER, MPI_COMBINER_HINDEXED);
+#        define MPI_COMBINER_STRUCT_INTEGER THIS_SYMBOL_WAS_REMOVED_IN_MPI30(MPI_COMBINER_STRUCT_INTEGER, MPI_COMBINER_STRUCT);
+#    endif /* OMPI_REMOVED_USE_STATIC_ASSERT */
+#endif /* Removed datatypes */
+
 /*
  * Communicator split type constants.
- * Do not change the order of these without also modifying mpif.h.in
- * (see also mpif-common.h.fin).
+ * Do not change the order of these without also modifying
+ * mpif-values.pl.
  */
 enum {
   MPI_COMM_TYPE_SHARED,
@@ -758,7 +851,9 @@ enum {
   OMPI_COMM_TYPE_BOARD,
   OMPI_COMM_TYPE_HOST,
   OMPI_COMM_TYPE_CU,
-  OMPI_COMM_TYPE_CLUSTER
+  OMPI_COMM_TYPE_CLUSTER,
+  MPI_COMM_TYPE_HW_UNGUIDED,
+  MPI_COMM_TYPE_HW_GUIDED
 };
 #define OMPI_COMM_TYPE_NODE MPI_COMM_TYPE_SHARED
 
@@ -804,7 +899,8 @@ enum {
   MPI_T_BIND_MPI_REQUEST,
   MPI_T_BIND_MPI_WIN,
   MPI_T_BIND_MPI_MESSAGE,
-  MPI_T_BIND_MPI_INFO
+  MPI_T_BIND_MPI_INFO,
+  MPI_T_BIND_MPI_SESSION
 };
 
 /*
@@ -823,9 +919,52 @@ enum {
   MPI_T_PVAR_CLASS_GENERIC
 };
 
+/*
+ * MPIT callback safety levels
+ *
+ * Values are set in configure.ac for consistency with mca_base_event.h
+ */
+#undef OPAL_MCA_BASE_CB_REQUIRE_NONE
+#undef OPAL_MCA_BASE_CB_REQUIRE_MPI_RESTRICTED
+#undef OPAL_MCA_BASE_CB_REQUIRE_THREAD_SAFE
+#undef OPAL_MCA_BASE_CB_REQUIRE_ASYNC_SIGNAL_SAFE
+typedef enum {
+    MPI_T_CB_REQUIRE_NONE = OPAL_MCA_BASE_CB_REQUIRE_NONE,
+    MPI_T_CB_REQUIRE_MPI_RESTRICTED = OPAL_MCA_BASE_CB_REQUIRE_MPI_RESTRICTED,
+    MPI_T_CB_REQUIRE_THREAD_SAFE = OPAL_MCA_BASE_CB_REQUIRE_THREAD_SAFE,
+    MPI_T_CB_REQUIRE_ASYNC_SIGNAL_SAFE = OPAL_MCA_BASE_CB_REQUIRE_ASYNC_SIGNAL_SAFE
+} MPI_T_cb_safety;
+
+/*
+ * MPIT source ordering
+ */
+enum ompi_mpi_t_source_order_t {
+  MPI_T_ORDERED,
+  MPI_T_UNORDERED,
+};
+
+typedef enum ompi_mpi_t_source_order_t MPI_T_source_order;
+
+/*
+ * MPI Tool event functions
+ */
+typedef void (*MPI_T_event_free_cb_function) (MPI_T_event_registration handle,
+                                              MPI_T_cb_safety cb_safety,
+                                              void *user_data);
+typedef void (*MPI_T_event_dropped_cb_function) (MPI_Count count,
+                                                 MPI_T_event_registration handle,
+                                                 int source_index,
+                                                 MPI_T_cb_safety cb_safety,
+                                                 void *user_data);
+typedef void (*MPI_T_event_cb_function) (MPI_T_event_instance event,
+                                         MPI_T_event_registration handle,
+                                         MPI_T_cb_safety cb_safety,
+                                         void *user_data);
+
 /*
  * NULL handles
  */
+#define MPI_SESSION_NULL OMPI_PREDEFINED_GLOBAL(MPI_Session, ompi_mpi_instance_null)
 #define MPI_GROUP_NULL OMPI_PREDEFINED_GLOBAL(MPI_Group, ompi_mpi_group_null)
 #define MPI_COMM_NULL OMPI_PREDEFINED_GLOBAL(MPI_Comm, ompi_mpi_comm_null)
 #define MPI_REQUEST_NULL OMPI_PREDEFINED_GLOBAL(MPI_Request, ompi_request_null)
@@ -857,6 +996,7 @@ enum {
 #define MPI_T_PVAR_HANDLE_NULL ((MPI_T_pvar_handle) 0)
 #define MPI_T_PVAR_SESSION_NULL ((MPI_T_pvar_session) 0)
 #define MPI_T_CVAR_HANDLE_NULL ((MPI_T_cvar_handle) 0)
+#define MPI_T_EVENT_REGISTRATION_NULL ((MPI_T_event_registration) 0)
 
 /* MPI-2 specifies that the name "MPI_TYPE_NULL_DELETE_FN" (and all
    related friends) must be accessible in C, C++, and Fortran. This is
@@ -896,6 +1036,7 @@ enum {
    must be able to be present, and therefore has to be in this
    conditional block in mpi.h. */
 #define MPI_CONVERSION_FN_NULL ((MPI_Datarep_conversion_function*) 0)
+#define MPI_CONVERSION_FN_NULL_C ((MPI_Datarep_conversion_function_c*) 0)
 #endif
 
 OMPI_DECLSPEC int OMPI_C_MPI_TYPE_NULL_DELETE_FN( MPI_Datatype datatype,
@@ -959,6 +1100,8 @@ OMPI_DECLSPEC extern struct ompi_predefined_communicator_t ompi_mpi_comm_null;
 OMPI_DECLSPEC extern struct ompi_predefined_group_t ompi_mpi_group_empty;
 OMPI_DECLSPEC extern struct ompi_predefined_group_t ompi_mpi_group_null;
 
+OMPI_DECLSPEC extern struct ompi_predefined_instance_t ompi_mpi_instance_null;
+
 OMPI_DECLSPEC extern struct ompi_predefined_request_t ompi_request_null;
 
 OMPI_DECLSPEC extern struct ompi_predefined_message_t ompi_message_null;
@@ -1085,6 +1228,9 @@ OMPI_DECLSPEC extern struct ompi_predefined_info_t ompi_mpi_info_env;
 OMPI_DECLSPEC extern MPI_Fint *MPI_F_STATUS_IGNORE;
 OMPI_DECLSPEC extern MPI_Fint *MPI_F_STATUSES_IGNORE;
 
+OMPI_DECLSPEC extern MPI_Fint *MPI_F08_STATUS_IGNORE;
+OMPI_DECLSPEC extern MPI_Fint *MPI_F08_STATUSES_IGNORE;
+
 /*
  * Removed datatypes.  These datatypes are only available if Open MPI
  * was configured with --enable-mpi1-compatibility.
@@ -1092,7 +1238,7 @@ OMPI_DECLSPEC extern MPI_Fint *MPI_F_STATUSES_IGNORE;
  * These datatypes were formally removed from the MPI specification
  * and should no longer be used in MPI applications.
  */
-#if (OMPI_ENABLE_MPI1_COMPAT || OMPI_BUILDING)
+#if (!OMPI_OMIT_MPI1_COMPAT_DECLS)
 #    define MPI_UB OMPI_PREDEFINED_GLOBAL(MPI_Datatype, ompi_mpi_ub)
 #    define MPI_LB OMPI_PREDEFINED_GLOBAL(MPI_Datatype, ompi_mpi_lb)
 
@@ -1103,7 +1249,7 @@ OMPI_DECLSPEC extern struct ompi_predefined_datatype_t ompi_mpi_ub;
 /* If not building or configured --enable-mpi1-compatibility, then
  * we don't want these datatypes, instead we define MPI_UB and
  * MPI_LB to our Static Assert message if the compiler supports
- * that staticly assert with a nice message.
+ * that static assert with a nice message.
  */
 #    if (OMPI_REMOVED_USE_STATIC_ASSERT)
 #        define MPI_UB THIS_SYMBOL_WAS_REMOVED_IN_MPI30(MPI_UB, MPI_Type_create_resized);
@@ -1159,13 +1305,7 @@ OMPI_DECLSPEC extern struct ompi_predefined_datatype_t ompi_mpi_ub;
 #define MPI_LONG_INT OMPI_PREDEFINED_GLOBAL(MPI_Datatype, ompi_mpi_long_int)
 #define MPI_SHORT_INT OMPI_PREDEFINED_GLOBAL(MPI_Datatype, ompi_mpi_short_int)
 #define MPI_2INT OMPI_PREDEFINED_GLOBAL(MPI_Datatype, ompi_mpi_2int)
-#if !OMPI_OMIT_MPI1_COMPAT_DECLS
-/*
- * Removed datatypes
- */
-#define MPI_UB OMPI_PREDEFINED_GLOBAL(MPI_Datatype, ompi_mpi_ub)
-#define MPI_LB OMPI_PREDEFINED_GLOBAL(MPI_Datatype, ompi_mpi_lb)
-#endif
+
 #define MPI_WCHAR OMPI_PREDEFINED_GLOBAL(MPI_Datatype, ompi_mpi_wchar)
 #if OPAL_HAVE_LONG_LONG
 #define MPI_LONG_LONG_INT OMPI_PREDEFINED_GLOBAL(MPI_Datatype, ompi_mpi_long_long_int)
@@ -1285,65 +1425,154 @@ OMPI_DECLSPEC extern struct ompi_predefined_datatype_t ompi_mpi_ub;
 #define PMPI_Aint_diff(addr1, addr2) MPI_Aint_diff(addr1, addr2)
 
 /*
- * MPI API
+ * Predefined info keys
  */
+#define MPI_INFO_KEY_SESSION_PSET_SIZE  "mpi_size"
 
+/*
+ * MPI API
+ */
 OMPI_DECLSPEC  int MPI_Abort(MPI_Comm comm, int errorcode);
 OMPI_DECLSPEC  int MPI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype,
                                   int target_rank, MPI_Aint target_disp, int target_count,
                                   MPI_Datatype target_datatype, MPI_Op op, MPI_Win win);
+OMPI_DECLSPEC  int MPI_Accumulate_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype,
+                                    int target_rank, MPI_Aint target_disp, MPI_Count target_count,
+                                    MPI_Datatype target_datatype, MPI_Op op, MPI_Win win);
 OMPI_DECLSPEC  int MPI_Add_error_class(int *errorclass);
 OMPI_DECLSPEC  int MPI_Add_error_code(int errorclass, int *errorcode);
 OMPI_DECLSPEC  int MPI_Add_error_string(int errorcode, const char *string);
 OMPI_DECLSPEC  int MPI_Allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                                  void *recvbuf, int recvcount,
                                  MPI_Datatype recvtype, MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Allgather_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                   void *recvbuf, MPI_Count recvcount,
+                                   MPI_Datatype recvtype, MPI_Comm comm);
 OMPI_DECLSPEC  int MPI_Iallgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                                 void *recvbuf, int recvcount,
-                                 MPI_Datatype recvtype, MPI_Comm comm, MPI_Request *request);
+                                  void *recvbuf, int recvcount,
+                                  MPI_Datatype recvtype, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Iallgather_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                    void *recvbuf, MPI_Count recvcount,
+                                    MPI_Datatype recvtype, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Allgather_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
+                                      void *recvbuf, int recvcount, MPI_Datatype recvtype,
+                                      MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC int MPI_Allgather_init_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                         void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                         MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                                  void *recvbuf, const int recvcounts[],
-                                  const int displs[], MPI_Datatype recvtype, MPI_Comm comm);
+                                  void *recvbuf, const int recvcounts[], const int displs[], MPI_Datatype recvtype,
+                                  MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Allgatherv_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                    void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint displs[], MPI_Datatype recvtype,
+                                    MPI_Comm comm);
 OMPI_DECLSPEC  int MPI_Iallgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                                  void *recvbuf, const int recvcounts[],
-                                  const int displs[], MPI_Datatype recvtype, MPI_Comm comm, MPI_Request *request);
+                                   void *recvbuf, const int recvcounts[], const int displs[], MPI_Datatype recvtype,
+                                   MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Iallgatherv_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                     void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint displs[], MPI_Datatype recvtype,
+                                     MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Allgatherv_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
+                                       void *recvbuf, const int recvcounts[], const int displs[], MPI_Datatype recvtype,
+                                       MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Allgatherv_init_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                         void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint displs[], MPI_Datatype recvtype,
+                                         MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Alloc_mem(MPI_Aint size, MPI_Info info,
                                  void *baseptr);
-OMPI_DECLSPEC  int MPI_Allreduce(const void *sendbuf, void *recvbuf, int count,
-                                 MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
-OMPI_DECLSPEC  int MPI_Iallreduce(const void *sendbuf, void *recvbuf, int count,
-                                 MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Allreduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
+                                 MPI_Op op, MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Allreduce_c(const void *sendbuf, void *recvbuf, MPI_Count count, MPI_Datatype datatype,
+                                   MPI_Op op, MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Iallreduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
+                                  MPI_Op op, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Iallreduce_c(const void *sendbuf, void *recvbuf, MPI_Count count, MPI_Datatype datatype,
+                                    MPI_Op op, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Allreduce_init(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
+                                      MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Allreduce_init_c(const void *sendbuf, void *recvbuf, MPI_Count count, MPI_Datatype datatype,
+                                        MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Alltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                                 void *recvbuf, int recvcount,
                                 MPI_Datatype recvtype, MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Alltoall_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                  void *recvbuf, MPI_Count recvcount,
+                                  MPI_Datatype recvtype, MPI_Comm comm);
 OMPI_DECLSPEC  int MPI_Ialltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                                void *recvbuf, int recvcount,
-                                MPI_Datatype recvtype, MPI_Comm comm, MPI_Request *request);
-OMPI_DECLSPEC  int MPI_Alltoallv(const void *sendbuf, const int sendcounts[], const int sdispls[],
-                                 MPI_Datatype sendtype, void *recvbuf, const int recvcounts[],
-                                 const int rdispls[], MPI_Datatype recvtype, MPI_Comm comm);
-OMPI_DECLSPEC  int MPI_Ialltoallv(const void *sendbuf, const int sendcounts[], const int sdispls[],
-                                 MPI_Datatype sendtype, void *recvbuf, const int recvcounts[],
-                                 const int rdispls[], MPI_Datatype recvtype, MPI_Comm comm, MPI_Request *request);
+                                 void *recvbuf, int recvcount, MPI_Datatype recvtype,
+                                 MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Ialltoall_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                   void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                   MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Alltoall_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
+                                     void *recvbuf, int recvcount, MPI_Datatype recvtype,
+                                     MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Alltoall_init_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                       void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                       MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Alltoallv(const void *sendbuf, const int sendcounts[], const int sdispls[], MPI_Datatype sendtype,
+                                 void *recvbuf, const int recvcounts[], const int rdispls[], MPI_Datatype recvtype,
+                                 MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Alltoallv_c(const void *sendbuf, const MPI_Count sendcounts[], const MPI_Aint sdispls[], MPI_Datatype sendtype,
+                                   void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint rdispls[], MPI_Datatype recvtype,
+                                   MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Ialltoallv(const void *sendbuf, const int sendcounts[], const int sdispls[], MPI_Datatype sendtype,
+                                  void *recvbuf, const int recvcounts[], const int rdispls[], MPI_Datatype recvtype,
+                                  MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Ialltoallv_c(const void *sendbuf, const MPI_Count sendcounts[], const MPI_Aint sdispls[], MPI_Datatype sendtype,
+                                    void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint rdispls[], MPI_Datatype recvtype,
+                                    MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Alltoallv_init(const void *sendbuf, const int sendcounts[], const int sdispls[], MPI_Datatype sendtype,
+                                      void *recvbuf, const int recvcounts[], const int rdispls[], MPI_Datatype recvtype,
+                                      MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Alltoallv_init_c(const void *sendbuf, const MPI_Count sendcounts[], const MPI_Aint sdispls[], MPI_Datatype sendtype,
+                                        void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint rdispls[], MPI_Datatype recvtype,
+                                        MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Alltoallw(const void *sendbuf, const int sendcounts[], const int sdispls[], const MPI_Datatype sendtypes[],
                                  void *recvbuf, const int recvcounts[], const int rdispls[], const MPI_Datatype recvtypes[],
                                  MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Alltoallw_c(const void *sendbuf, const MPI_Count sendcounts[], const MPI_Aint sdispls[], const MPI_Datatype sendtypes[],
+                                   void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint rdispls[], const MPI_Datatype recvtypes[],
+                                   MPI_Comm comm);
 OMPI_DECLSPEC  int MPI_Ialltoallw(const void *sendbuf, const int sendcounts[], const int sdispls[], const MPI_Datatype sendtypes[],
-                                 void *recvbuf, const int recvcounts[], const int rdispls[], const MPI_Datatype recvtypes[],
-                                 MPI_Comm comm, MPI_Request *request);
+                                  void *recvbuf, const int recvcounts[], const int rdispls[], const MPI_Datatype recvtypes[],
+                                  MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Ialltoallw_c(const void *sendbuf, const MPI_Count sendcounts[], const MPI_Aint sdispls[], const MPI_Datatype sendtypes[],
+                                    void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint rdispls[], const MPI_Datatype recvtypes[],
+                                    MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Alltoallw_init(const void *sendbuf, const int sendcounts[], const int sdispls[], const MPI_Datatype sendtypes[],
+                                      void *recvbuf, const int recvcounts[], const int rdispls[], const MPI_Datatype recvtypes[],
+                                      MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Alltoallw_init_c(const void *sendbuf, const MPI_Count sendcounts[], const MPI_Aint sdispls[], const MPI_Datatype sendtypes[],
+                                        void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint rdispls[], const MPI_Datatype recvtypes[],
+                                        MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Barrier(MPI_Comm comm);
 OMPI_DECLSPEC  int MPI_Ibarrier(MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Barrier_init(MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Bcast(void *buffer, int count, MPI_Datatype datatype,
                              int root, MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Bcast_c(void *buffer, MPI_Count count, MPI_Datatype datatype,
+                               int root, MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Ibcast(void *buffer, int count, MPI_Datatype datatype,
+                              int root, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Ibcast_c(void *buffer, MPI_Count count, MPI_Datatype datatype,
+                                int root, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Bcast_init(void *buffer, int count, MPI_Datatype datatype,
+                                  int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Bcast_init_c(void *buffer, MPI_Count count, MPI_Datatype datatype,
+                                    int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Bsend(const void *buf, int count, MPI_Datatype datatype,
                              int dest, int tag, MPI_Comm comm);
-OMPI_DECLSPEC  int MPI_Ibcast(void *buffer, int count, MPI_Datatype datatype,
-				                              int root, MPI_Comm comm,
-											  MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Bsend_c(const void *buf, MPI_Count count, MPI_Datatype datatype,
+                               int dest, int tag, MPI_Comm comm);
 OMPI_DECLSPEC  int MPI_Bsend_init(const void *buf, int count, MPI_Datatype datatype,
                                   int dest, int tag, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Bsend_init_c(const void *buf, MPI_Count count, MPI_Datatype datatype,
+                                  int dest, int tag, MPI_Comm comm, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Buffer_attach(void *buffer, int size);
+OMPI_DECLSPEC  int MPI_Buffer_attach_c(void *buffer, MPI_Count size);
 OMPI_DECLSPEC  int MPI_Buffer_detach(void *buffer, int *size);
+OMPI_DECLSPEC  int MPI_Buffer_detach_c(void *buffer, MPI_Count *size);
 OMPI_DECLSPEC  int MPI_Cancel(MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Cart_coords(MPI_Comm comm, int rank, int maxdims, int coords[]);
 OMPI_DECLSPEC  int MPI_Cart_create(MPI_Comm old_comm, int ndims, const int dims[],
@@ -1371,12 +1600,16 @@ OMPI_DECLSPEC  int MPI_Comm_create_keyval(MPI_Comm_copy_attr_function *comm_copy
                                           MPI_Comm_delete_attr_function *comm_delete_attr_fn,
                                           int *comm_keyval, void *extra_state);
 OMPI_DECLSPEC  int MPI_Comm_create_group(MPI_Comm comm, MPI_Group group, int tag, MPI_Comm *newcomm);
+OMPI_DECLSPEC  int MPI_Comm_create_from_group(MPI_Group group, const char *tag, MPI_Info info,
+					      MPI_Errhandler errhandler, MPI_Comm *newcomm);
 OMPI_DECLSPEC  int MPI_Comm_create(MPI_Comm comm, MPI_Group group, MPI_Comm *newcomm);
 OMPI_DECLSPEC  int MPI_Comm_delete_attr(MPI_Comm comm, int comm_keyval);
 OMPI_DECLSPEC  int MPI_Comm_disconnect(MPI_Comm *comm);
 OMPI_DECLSPEC  int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm);
 OMPI_DECLSPEC  int MPI_Comm_idup(MPI_Comm comm, MPI_Comm *newcomm, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Comm_dup_with_info(MPI_Comm comm, MPI_Info info, MPI_Comm *newcomm);
+OMPI_DECLSPEC  int MPI_Comm_idup_with_info(MPI_Comm comm, MPI_Info info, MPI_Comm *newcomm,
+                                           MPI_Request *request);
 OMPI_DECLSPEC  MPI_Comm MPI_Comm_f2c(MPI_Fint comm);
 OMPI_DECLSPEC  int MPI_Comm_free_keyval(int *comm_keyval);
 OMPI_DECLSPEC  int MPI_Comm_free(MPI_Comm *comm);
@@ -1436,12 +1669,20 @@ OMPI_DECLSPEC  MPI_Errhandler MPI_Errhandler_f2c(MPI_Fint errhandler);
 OMPI_DECLSPEC  int MPI_Errhandler_free(MPI_Errhandler *errhandler);
 OMPI_DECLSPEC  int MPI_Error_class(int errorcode, int *errorclass);
 OMPI_DECLSPEC  int MPI_Error_string(int errorcode, char *string, int *resultlen);
-OMPI_DECLSPEC  int MPI_Exscan(const void *sendbuf, void *recvbuf, int count,
-                              MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Exscan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
+                              MPI_Op op, MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Exscan_c(const void *sendbuf, void *recvbuf, MPI_Count count, MPI_Datatype datatype,
+                                MPI_Op op, MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Iexscan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
+                               MPI_Op op, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Iexscan_c(const void *sendbuf, void *recvbuf, MPI_Count count, MPI_Datatype datatype,
+                                 MPI_Op op, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Exscan_init(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
+                                   MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Exscan_init_c(const void *sendbuf, void *recvbuf, MPI_Count count, MPI_Datatype datatype,
+                                     MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Fetch_and_op(const void *origin_addr, void *result_addr, MPI_Datatype datatype,
                                     int target_rank, MPI_Aint target_disp, MPI_Op op, MPI_Win win);
-OMPI_DECLSPEC  int MPI_Iexscan(const void *sendbuf, void *recvbuf, int count,
-                              MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, MPI_Request *request);
 OMPI_DECLSPEC  MPI_Fint MPI_File_c2f(MPI_File file);
 OMPI_DECLSPEC  MPI_File MPI_File_f2c(MPI_Fint file);
 OMPI_DECLSPEC  int MPI_File_call_errhandler(MPI_File fh, int errorcode);
@@ -1467,74 +1708,132 @@ OMPI_DECLSPEC  int MPI_File_get_view(MPI_File fh, MPI_Offset *disp,
                                      MPI_Datatype *filetype, char *datarep);
 OMPI_DECLSPEC  int MPI_File_read_at(MPI_File fh, MPI_Offset offset, void *buf,
                                     int count, MPI_Datatype datatype, MPI_Status *status);
+OMPI_DECLSPEC  int MPI_File_read_at_c(MPI_File fh, MPI_Offset offset, void *buf,
+                                      MPI_Count count, MPI_Datatype datatype, MPI_Status *status);
 OMPI_DECLSPEC  int MPI_File_read_at_all(MPI_File fh, MPI_Offset offset, void *buf,
                                         int count, MPI_Datatype datatype, MPI_Status *status);
+OMPI_DECLSPEC  int MPI_File_read_at_all_c(MPI_File fh, MPI_Offset offset, void *buf,
+                                          MPI_Count count, MPI_Datatype datatype, MPI_Status *status);
 OMPI_DECLSPEC  int MPI_File_write_at(MPI_File fh, MPI_Offset offset, const void *buf,
                                      int count, MPI_Datatype datatype, MPI_Status *status);
+OMPI_DECLSPEC  int MPI_File_write_at_c(MPI_File fh, MPI_Offset offset, const void *buf,
+                                       MPI_Count count, MPI_Datatype datatype, MPI_Status *status);
 OMPI_DECLSPEC  int MPI_File_write_at_all(MPI_File fh, MPI_Offset offset, const void *buf,
                                          int count, MPI_Datatype datatype, MPI_Status *status);
+OMPI_DECLSPEC  int MPI_File_write_at_all_c(MPI_File fh, MPI_Offset offset, const void *buf,
+                                           MPI_Count count, MPI_Datatype datatype, MPI_Status *status);
 OMPI_DECLSPEC  int MPI_File_iread_at(MPI_File fh, MPI_Offset offset, void *buf,
                                      int count, MPI_Datatype datatype, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_File_iread_at_c(MPI_File fh, MPI_Offset offset, void *buf,
+                                       MPI_Count count, MPI_Datatype datatype, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_File_iwrite_at(MPI_File fh, MPI_Offset offset, const void *buf,
                                       int count, MPI_Datatype datatype, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_File_iwrite_at_c(MPI_File fh, MPI_Offset offset, const void *buf,
+                                        MPI_Count count, MPI_Datatype datatype, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_File_iread_at_all(MPI_File fh, MPI_Offset offset, void *buf,
-                                     int count, MPI_Datatype datatype, MPI_Request *request);
+                                         int count, MPI_Datatype datatype, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_File_iread_at_all_c(MPI_File fh, MPI_Offset offset, void *buf,
+                                           MPI_Count count, MPI_Datatype datatype, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_File_iwrite_at_all(MPI_File fh, MPI_Offset offset, const void *buf,
-                                      int count, MPI_Datatype datatype, MPI_Request *request);
+                                          int count, MPI_Datatype datatype, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_File_iwrite_at_all_c(MPI_File fh, MPI_Offset offset, const void *buf,
+                                            MPI_Count count, MPI_Datatype datatype, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_File_read(MPI_File fh, void *buf, int count,
                                  MPI_Datatype datatype, MPI_Status *status);
+OMPI_DECLSPEC  int MPI_File_read_c(MPI_File fh, void *buf, MPI_Count count,
+                                   MPI_Datatype datatype, MPI_Status *status);
 OMPI_DECLSPEC  int MPI_File_read_all(MPI_File fh, void *buf, int count,
                                      MPI_Datatype datatype, MPI_Status *status);
+OMPI_DECLSPEC  int MPI_File_read_all_c(MPI_File fh, void *buf, MPI_Count count,
+                                       MPI_Datatype datatype, MPI_Status *status);
 OMPI_DECLSPEC  int MPI_File_write(MPI_File fh, const void *buf, int count,
                                   MPI_Datatype datatype, MPI_Status *status);
+OMPI_DECLSPEC  int MPI_File_write_c(MPI_File fh, const void *buf, MPI_Count count,
+                                    MPI_Datatype datatype, MPI_Status *status);
 OMPI_DECLSPEC  int MPI_File_write_all(MPI_File fh, const void *buf, int count,
                                       MPI_Datatype datatype, MPI_Status *status);
+OMPI_DECLSPEC  int MPI_File_write_all_c(MPI_File fh, const void *buf, MPI_Count count,
+                                        MPI_Datatype datatype, MPI_Status *status);
 OMPI_DECLSPEC  int MPI_File_iread(MPI_File fh, void *buf, int count,
                                   MPI_Datatype datatype, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_File_iread_c(MPI_File fh, void *buf, MPI_Count count,
+                                    MPI_Datatype datatype, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_File_iwrite(MPI_File fh, const void *buf, int count,
                                    MPI_Datatype datatype, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_File_iwrite_c(MPI_File fh, const void *buf, MPI_Count count,
+                                     MPI_Datatype datatype, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_File_iread_all(MPI_File fh, void *buf, int count,
-                                  MPI_Datatype datatype, MPI_Request *request);
+                                      MPI_Datatype datatype, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_File_iread_all_c(MPI_File fh, void *buf, MPI_Count count,
+                                        MPI_Datatype datatype, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_File_iwrite_all(MPI_File fh, const void *buf, int count,
-                                   MPI_Datatype datatype, MPI_Request *request);
+                                       MPI_Datatype datatype, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_File_iwrite_all_c(MPI_File fh, const void *buf, MPI_Count count,
+                                         MPI_Datatype datatype, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_File_seek(MPI_File fh, MPI_Offset offset, int whence);
 OMPI_DECLSPEC  int MPI_File_get_position(MPI_File fh, MPI_Offset *offset);
 OMPI_DECLSPEC  int MPI_File_get_byte_offset(MPI_File fh, MPI_Offset offset,
                                             MPI_Offset *disp);
 OMPI_DECLSPEC  int MPI_File_read_shared(MPI_File fh, void *buf, int count,
                                         MPI_Datatype datatype, MPI_Status *status);
+OMPI_DECLSPEC  int MPI_File_read_shared_c(MPI_File fh, void *buf, MPI_Count count,
+                                          MPI_Datatype datatype, MPI_Status *status);
 OMPI_DECLSPEC  int MPI_File_write_shared(MPI_File fh, const void *buf, int count,
-					 MPI_Datatype datatype, MPI_Status *status);
+                                         MPI_Datatype datatype, MPI_Status *status);
+OMPI_DECLSPEC  int MPI_File_write_shared_c(MPI_File fh, const void *buf, MPI_Count count,
+                                           MPI_Datatype datatype, MPI_Status *status);
 OMPI_DECLSPEC  int MPI_File_iread_shared(MPI_File fh, void *buf, int count,
                                          MPI_Datatype datatype, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_File_iread_shared_c(MPI_File fh, void *buf, MPI_Count count,
+                                           MPI_Datatype datatype, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_File_iwrite_shared(MPI_File fh, const void *buf, int count,
                                           MPI_Datatype datatype, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_File_iwrite_shared_c(MPI_File fh, const void *buf, MPI_Count count,
+                                            MPI_Datatype datatype, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_File_read_ordered(MPI_File fh, void *buf, int count,
                                          MPI_Datatype datatype, MPI_Status *status);
+OMPI_DECLSPEC  int MPI_File_read_ordered_c(MPI_File fh, void *buf, MPI_Count count,
+                                           MPI_Datatype datatype, MPI_Status *status);
 OMPI_DECLSPEC  int MPI_File_write_ordered(MPI_File fh, const void *buf, int count,
                                           MPI_Datatype datatype, MPI_Status *status);
+OMPI_DECLSPEC  int MPI_File_write_ordered_c(MPI_File fh, const void *buf, MPI_Count count,
+                                            MPI_Datatype datatype, MPI_Status *status);
 OMPI_DECLSPEC  int MPI_File_seek_shared(MPI_File fh, MPI_Offset offset, int whence);
 OMPI_DECLSPEC  int MPI_File_get_position_shared(MPI_File fh, MPI_Offset *offset);
 OMPI_DECLSPEC  int MPI_File_read_at_all_begin(MPI_File fh, MPI_Offset offset, void *buf,
                                               int count, MPI_Datatype datatype);
+OMPI_DECLSPEC  int MPI_File_read_at_all_begin_c(MPI_File fh, MPI_Offset offset, void *buf,
+                                                MPI_Count count, MPI_Datatype datatype);
 OMPI_DECLSPEC  int MPI_File_read_at_all_end(MPI_File fh, void *buf, MPI_Status *status);
 OMPI_DECLSPEC  int MPI_File_write_at_all_begin(MPI_File fh, MPI_Offset offset, const void *buf,
                                                int count, MPI_Datatype datatype);
+OMPI_DECLSPEC  int MPI_File_write_at_all_begin_c(MPI_File fh, MPI_Offset offset, const void *buf,
+                                                 MPI_Count count, MPI_Datatype datatype);
 OMPI_DECLSPEC  int MPI_File_write_at_all_end(MPI_File fh, const void *buf, MPI_Status *status);
 OMPI_DECLSPEC  int MPI_File_read_all_begin(MPI_File fh, void *buf, int count,
                                            MPI_Datatype datatype);
+OMPI_DECLSPEC  int MPI_File_read_all_begin_c(MPI_File fh, void *buf, MPI_Count count,
+                                             MPI_Datatype datatype);
 OMPI_DECLSPEC  int MPI_File_read_all_end(MPI_File fh, void *buf, MPI_Status *status);
 OMPI_DECLSPEC  int MPI_File_write_all_begin(MPI_File fh, const void *buf, int count,
                                             MPI_Datatype datatype);
+OMPI_DECLSPEC  int MPI_File_write_all_begin_c(MPI_File fh, const void *buf, MPI_Count count,
+                                              MPI_Datatype datatype);
 OMPI_DECLSPEC  int MPI_File_write_all_end(MPI_File fh, const void *buf, MPI_Status *status);
 OMPI_DECLSPEC  int MPI_File_read_ordered_begin(MPI_File fh, void *buf, int count,
                                                MPI_Datatype datatype);
+OMPI_DECLSPEC  int MPI_File_read_ordered_begin_c(MPI_File fh, void *buf, MPI_Count count,
+                                                 MPI_Datatype datatype);
 OMPI_DECLSPEC  int MPI_File_read_ordered_end(MPI_File fh, void *buf, MPI_Status *status);
 OMPI_DECLSPEC  int MPI_File_write_ordered_begin(MPI_File fh, const void *buf, int count,
                                                 MPI_Datatype datatype);
+OMPI_DECLSPEC  int MPI_File_write_ordered_begin_c(MPI_File fh, const void *buf, MPI_Count count,
+                                                  MPI_Datatype datatype);
 OMPI_DECLSPEC  int MPI_File_write_ordered_end(MPI_File fh, const void *buf, MPI_Status *status);
 OMPI_DECLSPEC  int MPI_File_get_type_extent(MPI_File fh, MPI_Datatype datatype,
                                             MPI_Aint *extent);
+OMPI_DECLSPEC  int MPI_File_get_type_extent_c(MPI_File fh, MPI_Datatype datatype,
+                                              MPI_Count *extent);
 OMPI_DECLSPEC  int MPI_File_set_atomicity(MPI_File fh, int flag);
 OMPI_DECLSPEC  int MPI_File_get_atomicity(MPI_File fh, int *flag);
 OMPI_DECLSPEC  int MPI_File_sync(MPI_File fh);
@@ -1544,27 +1843,61 @@ OMPI_DECLSPEC  int MPI_Free_mem(void *base);
 OMPI_DECLSPEC  int MPI_Gather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                               void *recvbuf, int recvcount, MPI_Datatype recvtype,
                               int root, MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Gather_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                int root, MPI_Comm comm);
 OMPI_DECLSPEC  int MPI_Igather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                              void *recvbuf, int recvcount, MPI_Datatype recvtype,
-                              int root, MPI_Comm comm, MPI_Request *request);
+                               void *recvbuf, int recvcount, MPI_Datatype recvtype,
+                               int root, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Igather_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                 void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                 int root, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Gather_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
+                                   void *recvbuf, int recvcount, MPI_Datatype recvtype,
+                                   int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Gather_init_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                     void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                     int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Gatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                                void *recvbuf, const int recvcounts[], const int displs[],
                                MPI_Datatype recvtype, int root, MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Gatherv_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                 void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint displs[],
+                                 MPI_Datatype recvtype, int root, MPI_Comm comm);
 OMPI_DECLSPEC  int MPI_Igatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                               void *recvbuf, const int recvcounts[], const int displs[],
-                               MPI_Datatype recvtype, int root, MPI_Comm comm, MPI_Request *request);
+                                void *recvbuf, const int recvcounts[], const int displs[], MPI_Datatype recvtype,
+                                int root, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Igatherv_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                  void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint displs[], MPI_Datatype recvtype,
+                                  int root, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Gatherv_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
+                                    void *recvbuf, const int recvcounts[], const int displs[], MPI_Datatype recvtype,
+                                    int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Gatherv_init_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                      void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint displs[], MPI_Datatype recvtype,
+                                      int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Get_address(const void *location, MPI_Aint *address);
 OMPI_DECLSPEC  int MPI_Get_count(const MPI_Status *status, MPI_Datatype datatype, int *count);
+OMPI_DECLSPEC  int MPI_Get_count_c(const MPI_Status *status, MPI_Datatype datatype, MPI_Count *count);
 OMPI_DECLSPEC  int MPI_Get_elements(const MPI_Status *status, MPI_Datatype datatype, int *count);
+OMPI_DECLSPEC  int MPI_Get_elements_c(const MPI_Status *status, MPI_Datatype datatype, MPI_Count *count);
 OMPI_DECLSPEC  int MPI_Get_elements_x(const MPI_Status *status, MPI_Datatype datatype, MPI_Count *count);
 OMPI_DECLSPEC  int MPI_Get(void *origin_addr, int origin_count,
                            MPI_Datatype origin_datatype, int target_rank,
                            MPI_Aint target_disp, int target_count,
                            MPI_Datatype target_datatype, MPI_Win win);
+OMPI_DECLSPEC  int MPI_Get_c(void *origin_addr, MPI_Count origin_count,
+                             MPI_Datatype origin_datatype, int target_rank,
+                             MPI_Aint target_disp, MPI_Count target_count,
+                             MPI_Datatype target_datatype, MPI_Win win);
 OMPI_DECLSPEC  int MPI_Get_accumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype,
                                       void *result_addr, int result_count, MPI_Datatype result_datatype,
                                       int target_rank, MPI_Aint target_disp, int target_count,
                                       MPI_Datatype target_datatype, MPI_Op op, MPI_Win win);
+OMPI_DECLSPEC  int MPI_Get_accumulate_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype,
+                                        void *result_addr, MPI_Count result_count, MPI_Datatype result_datatype,
+                                        int target_rank, MPI_Aint target_disp, MPI_Count target_count,
+                                        MPI_Datatype target_datatype, MPI_Op op, MPI_Win win);
 OMPI_DECLSPEC  int MPI_Get_library_version(char *version, int *resultlen);
 OMPI_DECLSPEC  int MPI_Get_processor_name(char *name, int *resultlen);
 OMPI_DECLSPEC  int MPI_Get_version(int *version, int *subversion);
@@ -1591,6 +1924,7 @@ OMPI_DECLSPEC  int MPI_Group_excl(MPI_Group group, int n, const int ranks[],
                                   MPI_Group *newgroup);
 OMPI_DECLSPEC  MPI_Group MPI_Group_f2c(MPI_Fint group);
 OMPI_DECLSPEC  int MPI_Group_free(MPI_Group *group);
+OMPI_DECLSPEC  int MPI_Group_from_session_pset (MPI_Session session, const char *pset_name, MPI_Group *newgroup);
 OMPI_DECLSPEC  int MPI_Group_incl(MPI_Group group, int n, const int ranks[],
                                   MPI_Group *newgroup);
 OMPI_DECLSPEC  int MPI_Group_intersection(MPI_Group group1, MPI_Group group2,
@@ -1607,23 +1941,26 @@ OMPI_DECLSPEC  int MPI_Group_union(MPI_Group group1, MPI_Group group2,
                                    MPI_Group *newgroup);
 OMPI_DECLSPEC  int MPI_Ibsend(const void *buf, int count, MPI_Datatype datatype, int dest,
                               int tag, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Ibsend_c(const void *buf, MPI_Count count, MPI_Datatype datatype, int dest,
+                              int tag, MPI_Comm comm, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Improbe(int source, int tag, MPI_Comm comm,
                                int *flag, MPI_Message *message,
                                MPI_Status *status);
 OMPI_DECLSPEC  int MPI_Imrecv(void *buf, int count, MPI_Datatype type,
                               MPI_Message *message, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Imrecv_c(void *buf, MPI_Count count, MPI_Datatype type,
+                                MPI_Message *message, MPI_Request *request);
 OMPI_DECLSPEC  MPI_Fint MPI_Info_c2f(MPI_Info info);
 OMPI_DECLSPEC  int MPI_Info_create(MPI_Info *info);
+OMPI_DECLSPEC  int MPI_Info_create_env(int argc, char *argv[], MPI_Info *info);
 OMPI_DECLSPEC  int MPI_Info_delete(MPI_Info info, const char *key);
 OMPI_DECLSPEC  int MPI_Info_dup(MPI_Info info, MPI_Info *newinfo);
 OMPI_DECLSPEC  MPI_Info MPI_Info_f2c(MPI_Fint info);
 OMPI_DECLSPEC  int MPI_Info_free(MPI_Info *info);
-OMPI_DECLSPEC  int MPI_Info_get(MPI_Info info, const char *key, int valuelen,
-                                char *value, int *flag);
 OMPI_DECLSPEC  int MPI_Info_get_nkeys(MPI_Info info, int *nkeys);
 OMPI_DECLSPEC  int MPI_Info_get_nthkey(MPI_Info info, int n, char *key);
-OMPI_DECLSPEC  int MPI_Info_get_valuelen(MPI_Info info, const char *key, int *valuelen,
-                                         int *flag);
+OMPI_DECLSPEC  int MPI_Info_get_string(MPI_Info info, const char *key, int *buflen,
+                                       char *value, int *flag);
 OMPI_DECLSPEC  int MPI_Info_set(MPI_Info info, const char *key, const char *value);
 OMPI_DECLSPEC  int MPI_Init(int *argc, char ***argv);
 OMPI_DECLSPEC  int MPI_Initialized(int *flag);
@@ -1632,18 +1969,43 @@ OMPI_DECLSPEC  int MPI_Init_thread(int *argc, char ***argv, int required,
 OMPI_DECLSPEC  int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader,
                                         MPI_Comm bridge_comm, int remote_leader,
                                         int tag, MPI_Comm *newintercomm);
+OMPI_DECLSPEC  int MPI_Intercomm_create_from_groups (MPI_Group local_group, int local_leader, MPI_Group remote_group,
+                                                     int remote_leader, const char *tag, MPI_Info info, MPI_Errhandler errhandler,
+                                                     MPI_Comm *newintercomm);
 OMPI_DECLSPEC  int MPI_Intercomm_merge(MPI_Comm intercomm, int high,
-                                       MPI_Comm *newintercomm);
+                                       MPI_Comm *newintracomm);
 OMPI_DECLSPEC  int MPI_Iprobe(int source, int tag, MPI_Comm comm, int *flag,
                               MPI_Status *status);
 OMPI_DECLSPEC  int MPI_Irecv(void *buf, int count, MPI_Datatype datatype, int source,
                              int tag, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Irecv_c(void *buf, MPI_Count count, MPI_Datatype datatype, int source,
+                               int tag, MPI_Comm comm, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Irsend(const void *buf, int count, MPI_Datatype datatype, int dest,
                               int tag, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Irsend_c(const void *buf, MPI_Count count, MPI_Datatype datatype, int dest,
+                                int tag, MPI_Comm comm, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Isend(const void *buf, int count, MPI_Datatype datatype, int dest,
                              int tag, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Isend_c(const void *buf, MPI_Count count, MPI_Datatype datatype, int dest,
+                               int tag, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Isendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
+                                 int dest, int sendtag, void *recvbuf, int recvcount,
+                                 MPI_Datatype recvtype, int source, int recvtag,
+                                 MPI_Comm comm,  MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Isendrecv_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                   int dest, int sendtag, void *recvbuf, MPI_Count recvcount,
+                                   MPI_Datatype recvtype, int source, int recvtag,
+                                   MPI_Comm comm,  MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Isendrecv_replace(void * buf, int count, MPI_Datatype datatype,
+                                         int dest, int sendtag, int source, int recvtag,
+                                         MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Isendrecv_replace_c(void * buf, MPI_Count count, MPI_Datatype datatype,
+                                           int dest, int sendtag, int source, int recvtag,
+                                           MPI_Comm comm, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Issend(const void *buf, int count, MPI_Datatype datatype, int dest,
                               int tag, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Issend_c(const void *buf, MPI_Count count, MPI_Datatype datatype, int dest,
+                                int tag, MPI_Comm comm, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Is_thread_main(int *flag);
 OMPI_DECLSPEC  int MPI_Lookup_name(const char *service_name, MPI_Info info, char *port_name);
 OMPI_DECLSPEC  MPI_Fint MPI_Message_c2f(MPI_Message message);
@@ -1653,85 +2015,211 @@ OMPI_DECLSPEC  int MPI_Mprobe(int source, int tag, MPI_Comm comm,
                                MPI_Status *status);
 OMPI_DECLSPEC  int MPI_Mrecv(void *buf, int count, MPI_Datatype type,
                              MPI_Message *message, MPI_Status *status);
+OMPI_DECLSPEC  int MPI_Mrecv_c(void *buf, MPI_Count count, MPI_Datatype type,
+                               MPI_Message *message, MPI_Status *status);
 OMPI_DECLSPEC  int MPI_Neighbor_allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                                           void *recvbuf, int recvcount, MPI_Datatype recvtype,
                                           MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Neighbor_allgather_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                            void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                            MPI_Comm comm);
 OMPI_DECLSPEC  int MPI_Ineighbor_allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                                            void *recvbuf, int recvcount, MPI_Datatype recvtype,
                                            MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Ineighbor_allgather_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                             void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                             MPI_Comm comm, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Neighbor_allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                                            void *recvbuf, const int recvcounts[], const int displs[],
                                            MPI_Datatype recvtype, MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Neighbor_allgatherv_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                             void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint displs[],
+                                             MPI_Datatype recvtype, MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Neighbor_allgather_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
+                                               void *recvbuf, int recvcount, MPI_Datatype recvtype,
+                                               MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Neighbor_allgather_init_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                                 void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                                 MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Ineighbor_allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                                            void *recvbuf, const int recvcounts[], const int displs[],
-                                            MPI_Datatype recvtype, MPI_Comm comm, MPI_Request *request);
+                                            void *recvbuf, const int recvcounts[], const int displs[], MPI_Datatype recvtype,
+                                            MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Ineighbor_allgatherv_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                              void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint displs[], MPI_Datatype recvtype,
+                                              MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Neighbor_allgatherv_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
+                                                void *recvbuf, const int recvcounts[], const int displs[], MPI_Datatype recvtype,
+                                                MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Neighbor_allgatherv_init_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                                  void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint displs[], MPI_Datatype recvtype,
+                                                  MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Neighbor_alltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                                          void *recvbuf, int recvcount, MPI_Datatype recvtype,
                                          MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Neighbor_alltoall_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                           void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                           MPI_Comm comm);
 OMPI_DECLSPEC  int MPI_Ineighbor_alltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                                           void *recvbuf, int recvcount, MPI_Datatype recvtype,
                                           MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Ineighbor_alltoall_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                            void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                            MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Neighbor_alltoall_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
+                                              void *recvbuf, int recvcount, MPI_Datatype recvtype,
+                                              MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Neighbor_alltoall_init_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                                void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                                MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Neighbor_alltoallv(const void *sendbuf, const int sendcounts[], const int sdispls[],  MPI_Datatype sendtype,
                                           void *recvbuf, const int recvcounts[], const int rdispls[], MPI_Datatype recvtype,
                                           MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Neighbor_alltoallv_c(const void *sendbuf, const MPI_Count sendcounts[], const MPI_Aint sdispls[],  MPI_Datatype sendtype,
+                                            void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint rdispls[], MPI_Datatype recvtype,
+                                            MPI_Comm comm);
 OMPI_DECLSPEC  int MPI_Ineighbor_alltoallv(const void *sendbuf, const int sendcounts[], const int sdispls[], MPI_Datatype sendtype,
                                            void *recvbuf, const int recvcounts[], const int rdispls[], MPI_Datatype recvtype,
                                            MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Ineighbor_alltoallv_c(const void *sendbuf, const MPI_Count sendcounts[], const MPI_Aint sdispls[], MPI_Datatype sendtype,
+                                             void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint rdispls[], MPI_Datatype recvtype,
+                                             MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Neighbor_alltoallv_init(const void *sendbuf, const int sendcounts[], const int sdispls[], MPI_Datatype sendtype,
+                                               void *recvbuf, const int recvcounts[], const int rdispls[], MPI_Datatype recvtype,
+                                               MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Neighbor_alltoallv_init_c(const void *sendbuf, const MPI_Count sendcounts[], const MPI_Aint sdispls[], MPI_Datatype sendtype,
+                                                 void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint rdispls[], MPI_Datatype recvtype,
+                                                 MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Neighbor_alltoallw(const void *sendbuf, const int sendcounts[], const MPI_Aint sdispls[], const MPI_Datatype sendtypes[],
                                           void *recvbuf, const int recvcounts[], const MPI_Aint rdispls[], const MPI_Datatype recvtypes[],
                                           MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Neighbor_alltoallw_c(const void *sendbuf, const MPI_Count sendcounts[], const MPI_Aint sdispls[], const MPI_Datatype sendtypes[],
+                                            void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint rdispls[], const MPI_Datatype recvtypes[],
+                                            MPI_Comm comm);
 OMPI_DECLSPEC  int MPI_Ineighbor_alltoallw(const void *sendbuf, const int sendcounts[], const MPI_Aint sdispls[], const MPI_Datatype sendtypes[],
                                            void *recvbuf, const int recvcounts[], const MPI_Aint rdispls[], const MPI_Datatype recvtypes[],
                                            MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Ineighbor_alltoallw_c(const void *sendbuf, const MPI_Count sendcounts[], const MPI_Aint sdispls[], const MPI_Datatype sendtypes[],
+                                             void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint rdispls[], const MPI_Datatype recvtypes[],
+                                             MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Neighbor_alltoallw_init(const void *sendbuf, const int sendcounts[], const MPI_Aint sdispls[], const MPI_Datatype sendtypes[],
+                                               void *recvbuf, const int recvcounts[], const MPI_Aint rdispls[], const MPI_Datatype recvtypes[],
+                                               MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Neighbor_alltoallw_init_c(const void *sendbuf, const MPI_Count sendcounts[], const MPI_Aint sdispls[], const MPI_Datatype sendtypes[],
+                                                 void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint rdispls[], const MPI_Datatype recvtypes[],
+                                                 MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  MPI_Fint MPI_Op_c2f(MPI_Op op);
 OMPI_DECLSPEC  int MPI_Op_commutative(MPI_Op op, int *commute);
 OMPI_DECLSPEC  int MPI_Op_create(MPI_User_function *function, int commute, MPI_Op *op);
+OMPI_DECLSPEC  int MPI_Op_create_c(MPI_User_function_c *function, int commute, MPI_Op *op);
 OMPI_DECLSPEC  int MPI_Open_port(MPI_Info info, char *port_name);
 OMPI_DECLSPEC  MPI_Op MPI_Op_f2c(MPI_Fint op);
 OMPI_DECLSPEC  int MPI_Op_free(MPI_Op *op);
 OMPI_DECLSPEC  int MPI_Pack_external(const char datarep[], const void *inbuf, int incount,
                                      MPI_Datatype datatype, void *outbuf,
                                      MPI_Aint outsize, MPI_Aint *position);
+OMPI_DECLSPEC  int MPI_Pack_external_c(const char datarep[], const void *inbuf, MPI_Count incount,
+                                       MPI_Datatype datatype, void *outbuf,
+                                       MPI_Count outsize, MPI_Count *position);
 OMPI_DECLSPEC  int MPI_Pack_external_size(const char datarep[], int incount,
                                           MPI_Datatype datatype, MPI_Aint *size);
+OMPI_DECLSPEC  int MPI_Pack_external_size_c(const char datarep[], MPI_Count incount,
+                                            MPI_Datatype datatype, MPI_Count *size);
 OMPI_DECLSPEC  int MPI_Pack(const void *inbuf, int incount, MPI_Datatype datatype,
                             void *outbuf, int outsize, int *position, MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Pack_c(const void *inbuf, MPI_Count incount, MPI_Datatype datatype,
+                              void *outbuf, MPI_Count outsize, MPI_Count *position, MPI_Comm comm);
 OMPI_DECLSPEC  int MPI_Pack_size(int incount, MPI_Datatype datatype, MPI_Comm comm,
                                  int *size);
+OMPI_DECLSPEC  int MPI_Pack_size_c(MPI_Count incount, MPI_Datatype datatype, MPI_Comm comm,
+                                   MPI_Count *size);
+OMPI_DECLSPEC  int MPI_Parrived(MPI_Request request, int partition, int *flag);
 OMPI_DECLSPEC  int MPI_Pcontrol(const int level, ...);
+OMPI_DECLSPEC  int MPI_Pready(int partitions, MPI_Request request);
+OMPI_DECLSPEC  int MPI_Pready_range(int partition_low, int partition_high,
+                                    MPI_Request request);
+OMPI_DECLSPEC  int MPI_Pready_list(int length, const int partition_list[], MPI_Request request);
+OMPI_DECLSPEC  int MPI_Precv_init(void* buf, int partitions, MPI_Count count,
+                                  MPI_Datatype datatype, int source, int tag, MPI_Comm comm,
+                                  MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Probe(int source, int tag, MPI_Comm comm, MPI_Status *status);
+OMPI_DECLSPEC  int MPI_Psend_init(const void* buf, int partitions, MPI_Count count,
+                                  MPI_Datatype datatype, int dest, int tag, MPI_Comm comm,
+                                  MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Publish_name(const char *service_name, MPI_Info info,
                                     const char *port_name);
 OMPI_DECLSPEC  int MPI_Put(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype,
                            int target_rank, MPI_Aint target_disp, int target_count,
                            MPI_Datatype target_datatype, MPI_Win win);
+OMPI_DECLSPEC  int MPI_Put_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype,
+                             int target_rank, MPI_Aint target_disp, MPI_Count target_count,
+                             MPI_Datatype target_datatype, MPI_Win win);
 OMPI_DECLSPEC  int MPI_Query_thread(int *provided);
 OMPI_DECLSPEC  int MPI_Raccumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype,
                                    int target_rank, MPI_Aint target_disp, int target_count,
                                    MPI_Datatype target_datatype, MPI_Op op, MPI_Win win, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Raccumulate_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype,
+                                     int target_rank, MPI_Aint target_disp, MPI_Count target_count,
+                                     MPI_Datatype target_datatype, MPI_Op op, MPI_Win win, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Recv_init(void *buf, int count, MPI_Datatype datatype, int source,
                                  int tag, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Recv_init_c(void *buf, MPI_Count count, MPI_Datatype datatype, int source,
+                                   int tag, MPI_Comm comm, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Recv(void *buf, int count, MPI_Datatype datatype, int source,
                             int tag, MPI_Comm comm, MPI_Status *status);
-OMPI_DECLSPEC  int MPI_Reduce(const void *sendbuf, void *recvbuf, int count,
-                              MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm);
-OMPI_DECLSPEC  int MPI_Ireduce(const void *sendbuf, void *recvbuf, int count,
-                              MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Recv_c(void *buf, MPI_Count count, MPI_Datatype datatype, int source,
+                              int tag, MPI_Comm comm, MPI_Status *status);
+OMPI_DECLSPEC  int MPI_Recv_c(void *buf, MPI_Count count, MPI_Datatype datatype, int source,
+                              int tag, MPI_Comm comm, MPI_Status *status);
+OMPI_DECLSPEC  int MPI_Reduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
+                              MPI_Op op, int root, MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Reduce_c(const void *sendbuf, void *recvbuf, MPI_Count count, MPI_Datatype datatype,
+                                MPI_Op op, int root, MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Ireduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
+                               MPI_Op op, int root, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Ireduce_c(const void *sendbuf, void *recvbuf, MPI_Count count, MPI_Datatype datatype,
+                                 MPI_Op op, int root, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Reduce_init(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
+                                   MPI_Op op, int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Reduce_init_c(const void *sendbuf, void *recvbuf, MPI_Count count, MPI_Datatype datatype,
+                                     MPI_Op op, int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Reduce_local(const void *inbuf, void *inoutbuf, int count,
                                     MPI_Datatype datatype, MPI_Op op);
-OMPI_DECLSPEC  int MPI_Reduce_scatter(const void *sendbuf, void *recvbuf, const int recvcounts[],
-                                      MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
-OMPI_DECLSPEC  int MPI_Ireduce_scatter(const void *sendbuf, void *recvbuf, const int recvcounts[],
-                                      MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, MPI_Request *request);
-OMPI_DECLSPEC  int MPI_Reduce_scatter_block(const void *sendbuf, void *recvbuf, int recvcount,
-                                      MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
-OMPI_DECLSPEC  int MPI_Ireduce_scatter_block(const void *sendbuf, void *recvbuf, int recvcount,
-                                      MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Reduce_local_c(const void *inbuf, void *inoutbuf, MPI_Count count,
+                                      MPI_Datatype datatype, MPI_Op op);
+OMPI_DECLSPEC  int MPI_Reduce_scatter(const void *sendbuf, void *recvbuf, const int recvcounts[], MPI_Datatype datatype,
+                                      MPI_Op op, MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Reduce_scatter_c(const void *sendbuf, void *recvbuf, const MPI_Count recvcounts[], MPI_Datatype datatype,
+                                        MPI_Op op, MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Ireduce_scatter(const void *sendbuf, void *recvbuf, const int recvcounts[], MPI_Datatype datatype,
+                                       MPI_Op op, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Ireduce_scatter_c(const void *sendbuf, void *recvbuf, const MPI_Count recvcounts[], MPI_Datatype datatype,
+                                         MPI_Op op, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Reduce_scatter_init(const void *sendbuf, void *recvbuf, const int recvcounts[], MPI_Datatype datatype,
+                                           MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Reduce_scatter_init_c(const void *sendbuf, void *recvbuf, const MPI_Count recvcounts[], MPI_Datatype datatype,
+                                             MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Reduce_scatter_block(const void *sendbuf, void *recvbuf, int recvcount, MPI_Datatype datatype,
+                                            MPI_Op op, MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Reduce_scatter_block_c(const void *sendbuf, void *recvbuf, MPI_Count recvcount, MPI_Datatype datatype,
+                                              MPI_Op op, MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Ireduce_scatter_block(const void *sendbuf, void *recvbuf, int recvcount, MPI_Datatype datatype,
+                                             MPI_Op op, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Ireduce_scatter_block_c(const void *sendbuf, void *recvbuf, MPI_Count recvcount, MPI_Datatype datatype,
+                                               MPI_Op op, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Reduce_scatter_block_init(const void *sendbuf, void *recvbuf, int recvcount, MPI_Datatype datatype,
+                                                 MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Reduce_scatter_block_init_c(const void *sendbuf, void *recvbuf, MPI_Count recvcount, MPI_Datatype datatype,
+                                                   MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Register_datarep(const char *datarep,
                                         MPI_Datarep_conversion_function *read_conversion_fn,
                                         MPI_Datarep_conversion_function *write_conversion_fn,
                                         MPI_Datarep_extent_function *dtype_file_extent_fn,
                                         void *extra_state);
+OMPI_DECLSPEC  int MPI_Register_datarep_c(const char *datarep,
+                                          MPI_Datarep_conversion_function_c *read_conversion_fn,
+                                          MPI_Datarep_conversion_function_c *write_conversion_fn,
+                                          MPI_Datarep_extent_function *dtype_file_extent_fn,
+                                          void *extra_state);
 OMPI_DECLSPEC  MPI_Fint MPI_Request_c2f(MPI_Request request);
 OMPI_DECLSPEC  MPI_Request MPI_Request_f2c(MPI_Fint request);
 OMPI_DECLSPEC  int MPI_Request_free(MPI_Request *request);
@@ -1740,52 +2228,132 @@ OMPI_DECLSPEC  int MPI_Request_get_status(MPI_Request request, int *flag,
 OMPI_DECLSPEC  int MPI_Rget(void *origin_addr, int origin_count, MPI_Datatype origin_datatype,
                             int target_rank, MPI_Aint target_disp, int target_count, MPI_Datatype target_datatype,
                             MPI_Win win, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Rget_c(void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype,
+                              int target_rank, MPI_Aint target_disp, MPI_Count target_count, MPI_Datatype target_datatype,
+                              MPI_Win win, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Rget_accumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype,
                                        void *result_addr, int result_count, MPI_Datatype result_datatype,
                                        int target_rank, MPI_Aint target_disp, int target_count,
                                        MPI_Datatype target_datatype, MPI_Op op,
                                        MPI_Win win, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Rget_accumulate_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype,
+                                         void *result_addr, MPI_Count result_count, MPI_Datatype result_datatype,
+                                         int target_rank, MPI_Aint target_disp, MPI_Count target_count,
+                                         MPI_Datatype target_datatype, MPI_Op op,
+                                         MPI_Win win, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Rput(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype,
                             int target_rank, MPI_Aint target_disp, int target_cout,
                             MPI_Datatype target_datatype, MPI_Win win, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Rput_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype,
+                              int target_rank, MPI_Aint target_disp, MPI_Count target_cout,
+                              MPI_Datatype target_datatype, MPI_Win win, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Rsend(const void *ibuf, int count, MPI_Datatype datatype, int dest,
                              int tag, MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Rsend_c(const void *ibuf, MPI_Count count, MPI_Datatype datatype, int dest,
+                               int tag, MPI_Comm comm);
 OMPI_DECLSPEC  int MPI_Rsend_init(const void *buf, int count, MPI_Datatype datatype,
                                   int dest, int tag, MPI_Comm comm,
                                   MPI_Request *request);
-OMPI_DECLSPEC  int MPI_Scan(const void *sendbuf, void *recvbuf, int count,
-                            MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
-OMPI_DECLSPEC  int MPI_Iscan(const void *sendbuf, void *recvbuf, int count,
-                            MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Rsend_init_c(const void *buf, MPI_Count count, MPI_Datatype datatype,
+                                    int dest, int tag, MPI_Comm comm,
+                                    MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Scan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
+                            MPI_Op op, MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Scan_c(const void *sendbuf, void *recvbuf, MPI_Count count, MPI_Datatype datatype,
+                              MPI_Op op, MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Iscan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
+                             MPI_Op op, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Iscan_c(const void *sendbuf, void *recvbuf, MPI_Count count, MPI_Datatype datatype,
+                               MPI_Op op, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Scan_init(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
+                                 MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Scan_init_c(const void *sendbuf, void *recvbuf, MPI_Count count, MPI_Datatype datatype,
+                                   MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Scatter(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                                void *recvbuf, int recvcount, MPI_Datatype recvtype,
                                int root, MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Scatter_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                 void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                 int root, MPI_Comm comm);
 OMPI_DECLSPEC  int MPI_Iscatter(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                               void *recvbuf, int recvcount, MPI_Datatype recvtype,
-                               int root, MPI_Comm comm, MPI_Request *request);
-OMPI_DECLSPEC  int MPI_Scatterv(const void *sendbuf, const int sendcounts[], const int displs[],
-                                MPI_Datatype sendtype, void *recvbuf, int recvcount,
-                                MPI_Datatype recvtype, int root, MPI_Comm comm);
-OMPI_DECLSPEC  int MPI_Iscatterv(const void *sendbuf, const int sendcounts[], const int displs[],
-                                MPI_Datatype sendtype, void *recvbuf, int recvcount,
-                                MPI_Datatype recvtype, int root, MPI_Comm comm, MPI_Request *request);
+                                void *recvbuf, int recvcount, MPI_Datatype recvtype,
+                                int root, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Iscatter_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                  void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                  int root, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Scatter_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
+                                    void *recvbuf, int recvcount, MPI_Datatype recvtype,
+                                    int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Scatter_init_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                      void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                      int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Scatterv(const void *sendbuf, const int sendcounts[], const int displs[], MPI_Datatype sendtype,
+                                void *recvbuf, int recvcount, MPI_Datatype recvtype,
+                                int root, MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Scatterv_c(const void *sendbuf, const MPI_Count sendcounts[], const MPI_Aint displs[], MPI_Datatype sendtype,
+                                  void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                  int root, MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Iscatterv(const void *sendbuf, const int sendcounts[], const int displs[], MPI_Datatype sendtype,
+                                 void *recvbuf, int recvcount, MPI_Datatype recvtype,
+                                 int root, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Iscatterv_c(const void *sendbuf, const MPI_Count sendcounts[], const MPI_Aint displs[], MPI_Datatype sendtype,
+                                   void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                   int root, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Scatterv_init(const void *sendbuf, const int sendcounts[], const int displs[], MPI_Datatype sendtype,
+                                     void *recvbuf, int recvcount, MPI_Datatype recvtype,
+                                     int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Scatterv_init_c(const void *sendbuf, const MPI_Count sendcounts[], const MPI_Aint displs[], MPI_Datatype sendtype,
+                                       void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                       int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Send_init(const void *buf, int count, MPI_Datatype datatype,
                                  int dest, int tag, MPI_Comm comm,
                                  MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Send_init_c(const void *buf, MPI_Count count, MPI_Datatype datatype,
+                                   int dest, int tag, MPI_Comm comm,
+                                   MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Send(const void *buf, int count, MPI_Datatype datatype, int dest,
                             int tag, MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Send_c(const void *buf, MPI_Count count, MPI_Datatype datatype, int dest,
+                              int tag, MPI_Comm comm);
 OMPI_DECLSPEC  int MPI_Sendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                                 int dest, int sendtag, void *recvbuf, int recvcount,
                                 MPI_Datatype recvtype, int source, int recvtag,
                                 MPI_Comm comm,  MPI_Status *status);
+OMPI_DECLSPEC  int MPI_Sendrecv_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                  int dest, int sendtag, void *recvbuf, MPI_Count recvcount,
+                                  MPI_Datatype recvtype, int source, int recvtag,
+                                  MPI_Comm comm,  MPI_Status *status);
 OMPI_DECLSPEC  int MPI_Sendrecv_replace(void * buf, int count, MPI_Datatype datatype,
                                         int dest, int sendtag, int source, int recvtag,
                                         MPI_Comm comm, MPI_Status *status);
+OMPI_DECLSPEC  int MPI_Sendrecv_replace_c(void * buf, MPI_Count count, MPI_Datatype datatype,
+                                          int dest, int sendtag, int source, int recvtag,
+                                          MPI_Comm comm, MPI_Status *status);
+OMPI_DECLSPEC  MPI_Fint MPI_Session_c2f (const MPI_Session session);
+OMPI_DECLSPEC  int MPI_Session_call_errhandler(MPI_Session session, int errorcode);
+OMPI_DECLSPEC  int MPI_Session_create_errhandler (MPI_Session_errhandler_function *session_errhandler_fn,
+                                                  MPI_Errhandler *errhandler);
+OMPI_DECLSPEC  int MPI_Session_finalize (MPI_Session *session);
+OMPI_DECLSPEC  int MPI_Session_get_errhandler(MPI_Session session, MPI_Errhandler *erhandler);
+OMPI_DECLSPEC  int MPI_Session_get_info (MPI_Session session, MPI_Info *info_used);
+OMPI_DECLSPEC  int MPI_Session_get_num_psets (MPI_Session session, MPI_Info info, int *npset_names);
+OMPI_DECLSPEC  int MPI_Session_get_nth_pset (MPI_Session session, MPI_Info info, int n, int *len, char *pset_name);
+OMPI_DECLSPEC  int MPI_Session_get_pset_info (MPI_Session session, const char *pset_name, MPI_Info *info_used);
+OMPI_DECLSPEC  int MPI_Session_init (MPI_Info info, MPI_Errhandler errhandler,
+                                     MPI_Session *session);
+OMPI_DECLSPEC  MPI_Session MPI_Session_f2c (MPI_Fint session);
+OMPI_DECLSPEC  int MPI_Session_set_errhandler(MPI_Session session, MPI_Errhandler errhandler);
+OMPI_DECLSPEC  int MPI_Session_set_info (MPI_Session session, MPI_Info info);
 OMPI_DECLSPEC  int MPI_Ssend_init(const void *buf, int count, MPI_Datatype datatype,
                                   int dest, int tag, MPI_Comm comm,
                                   MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Ssend_init_c(const void *buf, MPI_Count count, MPI_Datatype datatype,
+                                    int dest, int tag, MPI_Comm comm,
+                                    MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Ssend(const void *buf, int count, MPI_Datatype datatype, int dest,
                              int tag, MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Ssend_c(const void *buf, MPI_Count count, MPI_Datatype datatype, int dest,
+                               int tag, MPI_Comm comm);
 OMPI_DECLSPEC  int MPI_Start(MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Startall(int count, MPI_Request array_of_requests[]);
 OMPI_DECLSPEC  int MPI_Status_c2f(const MPI_Status *c_status, MPI_Fint *f_status);
@@ -1794,9 +2362,17 @@ OMPI_DECLSPEC  int MPI_Status_f082c(const MPI_F08_status *f08_status, MPI_Status
 OMPI_DECLSPEC  int MPI_Status_f082f(const MPI_F08_status *f08_status, MPI_Fint *f_status);
 OMPI_DECLSPEC  int MPI_Status_f2c(const MPI_Fint *f_status, MPI_Status *c_status);
 OMPI_DECLSPEC  int MPI_Status_f2f08(const MPI_Fint *f_status, MPI_F08_status *f08_status);
+OMPI_DECLSPEC  int MPI_Status_get_error(const MPI_Status *status, int *error);
+OMPI_DECLSPEC  int MPI_Status_get_source(const MPI_Status *status, int *source);
+OMPI_DECLSPEC  int MPI_Status_get_tag(const MPI_Status *status, int *tag);
 OMPI_DECLSPEC  int MPI_Status_set_cancelled(MPI_Status *status, int flag);
+OMPI_DECLSPEC  int MPI_Status_set_error(MPI_Status *status, int error);
+OMPI_DECLSPEC  int MPI_Status_set_source(MPI_Status *status, int source);
+OMPI_DECLSPEC  int MPI_Status_set_tag(MPI_Status *status, int tag);
 OMPI_DECLSPEC  int MPI_Status_set_elements(MPI_Status *status, MPI_Datatype datatype,
                                            int count);
+OMPI_DECLSPEC  int MPI_Status_set_elements_c(MPI_Status *status, MPI_Datatype datatype,
+                                             MPI_Count count);
 OMPI_DECLSPEC  int MPI_Status_set_elements_x(MPI_Status *status, MPI_Datatype datatype,
                                              MPI_Count count);
 OMPI_DECLSPEC  int MPI_Testall(int count, MPI_Request array_of_requests[], int *flag,
@@ -1813,11 +2389,18 @@ OMPI_DECLSPEC  MPI_Fint MPI_Type_c2f(MPI_Datatype datatype);
 OMPI_DECLSPEC  int MPI_Type_commit(MPI_Datatype *type);
 OMPI_DECLSPEC  int MPI_Type_contiguous(int count, MPI_Datatype oldtype,
                                        MPI_Datatype *newtype);
+OMPI_DECLSPEC  int MPI_Type_contiguous_c(MPI_Count count, MPI_Datatype oldtype,
+                                         MPI_Datatype *newtype);
 OMPI_DECLSPEC  int MPI_Type_create_darray(int size, int rank, int ndims,
                                           const int gsize_array[], const int distrib_array[],
                                           const int darg_array[], const int psize_array[],
                                           int order, MPI_Datatype oldtype,
                                           MPI_Datatype *newtype);
+OMPI_DECLSPEC  int MPI_Type_create_darray_c(int size, int rank, int ndims,
+                                            const MPI_Count gsize_array[], const int distrib_array[],
+                                            const int darg_array[], const int psize_array[],
+                                            int order, MPI_Datatype oldtype,
+                                            MPI_Datatype *newtype);
 OMPI_DECLSPEC  int MPI_Type_create_f90_complex(int p, int r, MPI_Datatype *newtype);
 OMPI_DECLSPEC  int MPI_Type_create_f90_integer(int r, MPI_Datatype *newtype);
 OMPI_DECLSPEC  int MPI_Type_create_f90_real(int p, int r, MPI_Datatype *newtype);
@@ -1825,13 +2408,24 @@ OMPI_DECLSPEC  int MPI_Type_create_hindexed_block(int count, int blocklength,
                                                   const MPI_Aint array_of_displacements[],
                                                   MPI_Datatype oldtype,
                                                   MPI_Datatype *newtype);
+OMPI_DECLSPEC  int MPI_Type_create_hindexed_block_c(MPI_Count count, MPI_Count blocklength,
+                                                    const MPI_Count array_of_displacements[],
+                                                    MPI_Datatype oldtype,
+                                                    MPI_Datatype *newtype);
 OMPI_DECLSPEC  int MPI_Type_create_hindexed(int count, const int array_of_blocklengths[],
                                             const MPI_Aint array_of_displacements[],
                                             MPI_Datatype oldtype,
                                             MPI_Datatype *newtype);
+OMPI_DECLSPEC  int MPI_Type_create_hindexed_c(MPI_Count count, const MPI_Count array_of_blocklengths[],
+                                              const MPI_Count array_of_displacements[],
+                                              MPI_Datatype oldtype,
+                                              MPI_Datatype *newtype);
 OMPI_DECLSPEC  int MPI_Type_create_hvector(int count, int blocklength, MPI_Aint stride,
                                            MPI_Datatype oldtype,
                                            MPI_Datatype *newtype);
+OMPI_DECLSPEC  int MPI_Type_create_hvector_c(MPI_Count count, MPI_Count blocklength, MPI_Count stride,
+                                             MPI_Datatype oldtype,
+                                             MPI_Datatype *newtype);
 OMPI_DECLSPEC  int MPI_Type_create_keyval(MPI_Type_copy_attr_function *type_copy_attr_fn,
                                           MPI_Type_delete_attr_function *type_delete_attr_fn,
                                           int *type_keyval, void *extra_state);
@@ -1839,15 +2433,28 @@ OMPI_DECLSPEC  int MPI_Type_create_indexed_block(int count, int blocklength,
                                                  const int array_of_displacements[],
                                                  MPI_Datatype oldtype,
                                                  MPI_Datatype *newtype);
+OMPI_DECLSPEC  int MPI_Type_create_indexed_block_c(MPI_Count count, MPI_Count blocklength,
+                                                   const MPI_Count array_of_displacements[],
+                                                   MPI_Datatype oldtype,
+                                                   MPI_Datatype *newtype);
 OMPI_DECLSPEC  int MPI_Type_create_struct(int count, const int array_of_block_lengths[],
                                           const MPI_Aint array_of_displacements[],
                                           const MPI_Datatype array_of_types[],
                                           MPI_Datatype *newtype);
+OMPI_DECLSPEC  int MPI_Type_create_struct_c(MPI_Count count, const MPI_Count array_of_block_lengths[],
+                                            const MPI_Count array_of_displacements[],
+                                            const MPI_Datatype array_of_types[],
+                                            MPI_Datatype *newtype);
 OMPI_DECLSPEC  int MPI_Type_create_subarray(int ndims, const int size_array[], const int subsize_array[],
                                             const int start_array[], int order,
                                             MPI_Datatype oldtype, MPI_Datatype *newtype);
+OMPI_DECLSPEC  int MPI_Type_create_subarray_c(int ndims, const MPI_Count size_array[], const MPI_Count subsize_array[],
+                                              const MPI_Count start_array[], int order,
+                                              MPI_Datatype oldtype, MPI_Datatype *newtype);
 OMPI_DECLSPEC  int MPI_Type_create_resized(MPI_Datatype oldtype, MPI_Aint lb,
                                            MPI_Aint extent, MPI_Datatype *newtype);
+OMPI_DECLSPEC  int MPI_Type_create_resized_c(MPI_Datatype oldtype, MPI_Count lb,
+                                             MPI_Count extent, MPI_Datatype *newtype);
 OMPI_DECLSPEC  int MPI_Type_delete_attr(MPI_Datatype type, int type_keyval);
 OMPI_DECLSPEC  int MPI_Type_dup(MPI_Datatype type, MPI_Datatype *newtype);
 OMPI_DECLSPEC  int MPI_Type_free(MPI_Datatype *type);
@@ -1860,37 +2467,63 @@ OMPI_DECLSPEC  int MPI_Type_get_contents(MPI_Datatype mtype, int max_integers,
                                          int array_of_integers[],
                                          MPI_Aint array_of_addresses[],
                                          MPI_Datatype array_of_datatypes[]);
+OMPI_DECLSPEC  int MPI_Type_get_contents_c(MPI_Datatype mtype, MPI_Count max_integers,
+                                           MPI_Count max_addresses, MPI_Count max_large_counts,
+                                           MPI_Count max_datatypes,
+                                           int array_of_integers[],
+                                           MPI_Aint array_of_addresses[],
+                                           MPI_Count array_of_large_counts[],
+                                           MPI_Datatype array_of_datatypes[]);
 OMPI_DECLSPEC  int MPI_Type_get_envelope(MPI_Datatype type, int *num_integers,
                                          int *num_addresses, int *num_datatypes,
                                          int *combiner);
+OMPI_DECLSPEC  int MPI_Type_get_envelope_c(MPI_Datatype type, MPI_Count *num_integers,
+                                         MPI_Count *num_addresses, MPI_Count *num_large_counts,
+					 MPI_Count *num_datatypes, int *combiner);
 OMPI_DECLSPEC  int MPI_Type_get_extent(MPI_Datatype type, MPI_Aint *lb,
                                        MPI_Aint *extent);
+OMPI_DECLSPEC  int MPI_Type_get_extent_c(MPI_Datatype type, MPI_Count *lb,
+                                         MPI_Count *extent);
 OMPI_DECLSPEC  int MPI_Type_get_extent_x(MPI_Datatype type, MPI_Count *lb,
                                          MPI_Count *extent);
 OMPI_DECLSPEC  int MPI_Type_get_name(MPI_Datatype type, char *type_name,
                                      int *resultlen);
 OMPI_DECLSPEC  int MPI_Type_get_true_extent(MPI_Datatype datatype, MPI_Aint *true_lb,
                                             MPI_Aint *true_extent);
+OMPI_DECLSPEC  int MPI_Type_get_true_extent_c(MPI_Datatype datatype, MPI_Count *true_lb,
+                                             MPI_Count *true_extent);
 OMPI_DECLSPEC  int MPI_Type_get_true_extent_x(MPI_Datatype datatype, MPI_Count *true_lb,
                                               MPI_Count *true_extent);
 OMPI_DECLSPEC  int MPI_Type_indexed(int count, const int array_of_blocklengths[],
                                     const int array_of_displacements[],
                                     MPI_Datatype oldtype, MPI_Datatype *newtype);
+OMPI_DECLSPEC  int MPI_Type_indexed_c(MPI_Count count, const MPI_Count array_of_blocklengths[],
+                                      const MPI_Count array_of_displacements[],
+                                      MPI_Datatype oldtype, MPI_Datatype *newtype);
 OMPI_DECLSPEC  int MPI_Type_match_size(int typeclass, int size, MPI_Datatype *type);
 OMPI_DECLSPEC  int MPI_Type_set_attr(MPI_Datatype type, int type_keyval,
                                      void *attr_val);
 OMPI_DECLSPEC  int MPI_Type_set_name(MPI_Datatype type, const char *type_name);
 OMPI_DECLSPEC  int MPI_Type_size(MPI_Datatype type, int *size);
+OMPI_DECLSPEC  int MPI_Type_size_c(MPI_Datatype type, MPI_Count *size);
 OMPI_DECLSPEC  int MPI_Type_size_x(MPI_Datatype type, MPI_Count *size);
 OMPI_DECLSPEC  int MPI_Type_vector(int count, int blocklength, int stride,
                                    MPI_Datatype oldtype, MPI_Datatype *newtype);
+OMPI_DECLSPEC  int MPI_Type_vector_c(MPI_Count count, MPI_Count blocklength, MPI_Count stride,
+                                     MPI_Datatype oldtype, MPI_Datatype *newtype);
 OMPI_DECLSPEC  int MPI_Unpack(const void *inbuf, int insize, int *position,
                               void *outbuf, int outcount, MPI_Datatype datatype,
                               MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Unpack_c(const void *inbuf, MPI_Count insize, MPI_Count *position,
+                                void *outbuf, MPI_Count outcount, MPI_Datatype datatype,
+                                MPI_Comm comm);
 OMPI_DECLSPEC  int MPI_Unpublish_name(const char *service_name, MPI_Info info, const char *port_name);
 OMPI_DECLSPEC  int MPI_Unpack_external (const char datarep[], const void *inbuf, MPI_Aint insize,
                                         MPI_Aint *position, void *outbuf, int outcount,
                                         MPI_Datatype datatype);
+OMPI_DECLSPEC  int MPI_Unpack_external_c (const char datarep[], const void *inbuf, MPI_Count insize,
+                                          MPI_Count *position, void *outbuf, MPI_Count outcount,
+                                          MPI_Datatype datatype);
 OMPI_DECLSPEC  int MPI_Waitall(int count, MPI_Request array_of_requests[],
                                MPI_Status *array_of_statuses);
 OMPI_DECLSPEC  int MPI_Waitany(int count, MPI_Request array_of_requests[],
@@ -1901,14 +2534,20 @@ OMPI_DECLSPEC  int MPI_Waitsome(int incount, MPI_Request array_of_requests[],
                                 MPI_Status array_of_statuses[]);
 OMPI_DECLSPEC  int MPI_Win_allocate(MPI_Aint size, int disp_unit, MPI_Info info,
                                     MPI_Comm comm, void *baseptr, MPI_Win *win);
+OMPI_DECLSPEC  int MPI_Win_allocate_c(MPI_Aint size, MPI_Aint disp_unit, MPI_Info info,
+                                      MPI_Comm comm, void *baseptr, MPI_Win *win);
 OMPI_DECLSPEC  int MPI_Win_allocate_shared(MPI_Aint size, int disp_unit, MPI_Info info,
                                            MPI_Comm comm, void *baseptr, MPI_Win *win);
+OMPI_DECLSPEC  int MPI_Win_allocate_shared_c(MPI_Aint size, MPI_Aint disp_unit, MPI_Info info,
+                                             MPI_Comm comm, void *baseptr, MPI_Win *win);
 OMPI_DECLSPEC  int MPI_Win_attach(MPI_Win win, void *base, MPI_Aint size);
 OMPI_DECLSPEC  MPI_Fint MPI_Win_c2f(MPI_Win win);
 OMPI_DECLSPEC  int MPI_Win_call_errhandler(MPI_Win win, int errorcode);
 OMPI_DECLSPEC  int MPI_Win_complete(MPI_Win win);
 OMPI_DECLSPEC  int MPI_Win_create(void *base, MPI_Aint size, int disp_unit,
                                   MPI_Info info, MPI_Comm comm, MPI_Win *win);
+OMPI_DECLSPEC  int MPI_Win_create_c(void *base, MPI_Aint size, MPI_Aint disp_unit,
+                                    MPI_Info info, MPI_Comm comm, MPI_Win *win);
 OMPI_DECLSPEC  int MPI_Win_create_dynamic(MPI_Info info, MPI_Comm comm, MPI_Win *win);
 OMPI_DECLSPEC  int MPI_Win_create_errhandler(MPI_Win_errhandler_function *function,
                                              MPI_Errhandler *errhandler);
@@ -1918,7 +2557,7 @@ OMPI_DECLSPEC  int MPI_Win_create_keyval(MPI_Win_copy_attr_function *win_copy_at
 OMPI_DECLSPEC  int MPI_Win_delete_attr(MPI_Win win, int win_keyval);
 OMPI_DECLSPEC  int MPI_Win_detach(MPI_Win win, const void *base);
 OMPI_DECLSPEC  MPI_Win MPI_Win_f2c(MPI_Fint win);
-OMPI_DECLSPEC  int MPI_Win_fence(int assert, MPI_Win win);
+OMPI_DECLSPEC  int MPI_Win_fence(int mpi_assert, MPI_Win win);
 OMPI_DECLSPEC  int MPI_Win_flush(int rank, MPI_Win win);
 OMPI_DECLSPEC  int MPI_Win_flush_all(MPI_Win win);
 OMPI_DECLSPEC  int MPI_Win_flush_local(int rank, MPI_Win win);
@@ -1931,15 +2570,16 @@ OMPI_DECLSPEC  int MPI_Win_get_errhandler(MPI_Win win, MPI_Errhandler *errhandle
 OMPI_DECLSPEC  int MPI_Win_get_group(MPI_Win win, MPI_Group *group);
 OMPI_DECLSPEC  int MPI_Win_get_info(MPI_Win win, MPI_Info *info_used);
 OMPI_DECLSPEC  int MPI_Win_get_name(MPI_Win win, char *win_name, int *resultlen);
-OMPI_DECLSPEC  int MPI_Win_lock(int lock_type, int rank, int assert, MPI_Win win);
-OMPI_DECLSPEC  int MPI_Win_lock_all(int assert, MPI_Win win);
-OMPI_DECLSPEC  int MPI_Win_post(MPI_Group group, int assert, MPI_Win win);
+OMPI_DECLSPEC  int MPI_Win_lock(int lock_type, int rank, int mpi_assert, MPI_Win win);
+OMPI_DECLSPEC  int MPI_Win_lock_all(int mpi_assert, MPI_Win win);
+OMPI_DECLSPEC  int MPI_Win_post(MPI_Group group, int mpi_assert, MPI_Win win);
 OMPI_DECLSPEC  int MPI_Win_set_attr(MPI_Win win, int win_keyval, void *attribute_val);
 OMPI_DECLSPEC  int MPI_Win_set_errhandler(MPI_Win win, MPI_Errhandler errhandler);
 OMPI_DECLSPEC  int MPI_Win_set_info(MPI_Win win, MPI_Info info);
 OMPI_DECLSPEC  int MPI_Win_set_name(MPI_Win win, const char *win_name);
 OMPI_DECLSPEC  int MPI_Win_shared_query(MPI_Win win, int rank, MPI_Aint *size, int *disp_unit, void *baseptr);
-OMPI_DECLSPEC  int MPI_Win_start(MPI_Group group, int assert, MPI_Win win);
+OMPI_DECLSPEC  int MPI_Win_shared_query_c(MPI_Win win, int rank, MPI_Aint *size, MPI_Aint *disp_unit, void *baseptr);
+OMPI_DECLSPEC  int MPI_Win_start(MPI_Group group, int mpi_assert, MPI_Win win);
 OMPI_DECLSPEC  int MPI_Win_sync(MPI_Win win);
 OMPI_DECLSPEC  int MPI_Win_test(MPI_Win win, int *flag);
 OMPI_DECLSPEC  int MPI_Win_unlock(int rank, MPI_Win win);
@@ -1955,79 +2595,143 @@ OMPI_DECLSPEC  int PMPI_Abort(MPI_Comm comm, int errorcode);
 OMPI_DECLSPEC  int PMPI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype,
                                    int target_rank, MPI_Aint target_disp, int target_count,
                                    MPI_Datatype target_datatype, MPI_Op op, MPI_Win win);
+OMPI_DECLSPEC  int PMPI_Accumulate_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype,
+                                     int target_rank, MPI_Aint target_disp, MPI_Count target_count,
+                                     MPI_Datatype target_datatype, MPI_Op op, MPI_Win win);
 OMPI_DECLSPEC  int PMPI_Add_error_class(int *errorclass);
 OMPI_DECLSPEC  int PMPI_Add_error_code(int errorclass, int *errorcode);
 OMPI_DECLSPEC  int PMPI_Add_error_string(int errorcode, const char *string);
 OMPI_DECLSPEC  int PMPI_Allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                                   void *recvbuf, int recvcount,
                                   MPI_Datatype recvtype, MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Allgather_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                    void *recvbuf, MPI_Count recvcount,
+                                    MPI_Datatype recvtype, MPI_Comm comm);
 OMPI_DECLSPEC  int PMPI_Iallgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                                  void *recvbuf, int recvcount,
-                                  MPI_Datatype recvtype, MPI_Comm comm, MPI_Request *request);
+                                   void *recvbuf, int recvcount,
+                                   MPI_Datatype recvtype, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Iallgather_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                     void *recvbuf, MPI_Count recvcount,
+                                     MPI_Datatype recvtype, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Allgather_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
+                                       void *recvbuf, int recvcount, MPI_Datatype recvtype,
+                                       MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC int PMPI_Allgather_init_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                        void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                        MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                                   void *recvbuf, const int recvcounts[],
-                                   const int displs[], MPI_Datatype recvtype, MPI_Comm comm);
+                                   void *recvbuf, const int recvcounts[], const int displs[], MPI_Datatype recvtype,
+                                   MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Allgatherv_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                     void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint displs[], MPI_Datatype recvtype,
+                                     MPI_Comm comm);
 OMPI_DECLSPEC  int PMPI_Iallgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                                   void *recvbuf, const int recvcounts[],
-                                   const int displs[], MPI_Datatype recvtype, MPI_Comm comm, MPI_Request *request);
+                                    void *recvbuf, const int recvcounts[], const int displs[], MPI_Datatype recvtype,
+                                    MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Iallgatherv_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                      void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint displs[], MPI_Datatype recvtype,
+                                      MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Allgatherv_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
+                                        void *recvbuf, const int recvcounts[], const int displs[], MPI_Datatype recvtype,
+                                        MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Allgatherv_init_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                          void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint displs[], MPI_Datatype recvtype,
+                                          MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Alloc_mem(MPI_Aint size, MPI_Info info,
                                   void *baseptr);
-OMPI_DECLSPEC  int PMPI_Allreduce(const void *sendbuf, void *recvbuf, int count,
-                                  MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
-OMPI_DECLSPEC  int PMPI_Iallreduce(const void *sendbuf, void *recvbuf, int count,
-                                  MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Allreduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
+                                  MPI_Op op, MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Allreduce_c(const void *sendbuf, void *recvbuf, MPI_Count count, MPI_Datatype datatype,
+                                    MPI_Op op, MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Iallreduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
+                                   MPI_Op op, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Iallreduce_c(const void *sendbuf, void *recvbuf, MPI_Count count, MPI_Datatype datatype,
+                                     MPI_Op op, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Allreduce_init(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
+                                       MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Allreduce_init_c(const void *sendbuf, void *recvbuf, MPI_Count count, MPI_Datatype datatype,
+                                         MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Alltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                                  void *recvbuf, int recvcount,
                                  MPI_Datatype recvtype, MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Alltoall_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                   void *recvbuf, MPI_Count recvcount,
+                                   MPI_Datatype recvtype, MPI_Comm comm);
 OMPI_DECLSPEC  int PMPI_Ialltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                                 void *recvbuf, int recvcount,
-                                 MPI_Datatype recvtype, MPI_Comm comm, MPI_Request *request);
-OMPI_DECLSPEC  int PMPI_Alltoallv(const void *sendbuf, const int sendcounts[], const int sdispls[],
-                                  MPI_Datatype sendtype, void *recvbuf, const int recvcounts[],
-                                  const int rdispls[], MPI_Datatype recvtype, MPI_Comm comm);
-OMPI_DECLSPEC  int PMPI_Ialltoallv(const void *sendbuf, const int sendcounts[], const int sdispls[],
-                                  MPI_Datatype sendtype, void *recvbuf, const int recvcounts[],
-                                  const int rdispls[], MPI_Datatype recvtype, MPI_Comm comm, MPI_Request *request);
+                                  void *recvbuf, int recvcount, MPI_Datatype recvtype,
+                                  MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Ialltoall_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                    void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                    MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Alltoall_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
+                                      void *recvbuf, int recvcount, MPI_Datatype recvtype,
+                                      MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Alltoall_init_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                        void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                        MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Alltoallv(const void *sendbuf, const int sendcounts[], const int sdispls[], MPI_Datatype sendtype,
+                                  void *recvbuf, const int recvcounts[], const int rdispls[], MPI_Datatype recvtype,
+                                  MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Alltoallv_c(const void *sendbuf, const MPI_Count sendcounts[], const MPI_Aint sdispls[], MPI_Datatype sendtype,
+                                    void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint rdispls[], MPI_Datatype recvtype,
+                                    MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Ialltoallv(const void *sendbuf, const int sendcounts[], const int sdispls[], MPI_Datatype sendtype,
+                                   void *recvbuf, const int recvcounts[], const int rdispls[], MPI_Datatype recvtype,
+                                   MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Ialltoallv_c(const void *sendbuf, const MPI_Count sendcounts[], const MPI_Aint sdispls[], MPI_Datatype sendtype,
+                                     void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint rdispls[], MPI_Datatype recvtype,
+                                     MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Alltoallv_init(const void *sendbuf, const int sendcounts[], const int sdispls[], MPI_Datatype sendtype,
+                                       void *recvbuf, const int recvcounts[], const int rdispls[], MPI_Datatype recvtype,
+                                       MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Alltoallv_init_c(const void *sendbuf, const MPI_Count sendcounts[], const MPI_Aint sdispls[], MPI_Datatype sendtype,
+                                         void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint rdispls[], MPI_Datatype recvtype,
+                                         MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Alltoallw(const void *sendbuf, const int sendcounts[], const int sdispls[], const MPI_Datatype sendtypes[],
                                   void *recvbuf, const int recvcounts[], const int rdispls[], const MPI_Datatype recvtypes[],
                                   MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Alltoallw_c(const void *sendbuf, const MPI_Count sendcounts[], const MPI_Aint sdispls[], const MPI_Datatype sendtypes[],
+                                    void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint rdispls[], const MPI_Datatype recvtypes[],
+                                    MPI_Comm comm);
 OMPI_DECLSPEC  int PMPI_Ialltoallw(const void *sendbuf, const int sendcounts[], const int sdispls[], const MPI_Datatype sendtypes[],
-                                  void *recvbuf, const int recvcounts[], const int rdispls[], const MPI_Datatype recvtypes[],
-                                  MPI_Comm comm, MPI_Request *request);
-OMPI_DECLSPEC  int PMPI_Dist_graph_create(MPI_Comm comm_old, int n, const int nodes[],
-                                          const int degrees[], const int targets[],
-                                          const int weights[], MPI_Info info,
-                                          int reorder, MPI_Comm * newcomm);
-OMPI_DECLSPEC  int PMPI_Dist_graph_create_adjacent(MPI_Comm comm_old,
-                                                   int indegree, const int sources[],
-                                                   const int sourceweights[],
-                                                   int outdegree,
-                                                   const int destinations[],
-                                                   const int destweights[],
-                                                   MPI_Info info, int reorder,
-                                                   MPI_Comm *comm_dist_graph);
-OMPI_DECLSPEC int PMPI_Dist_graph_neighbors(MPI_Comm comm, int maxindegree,
-                                            int sources[], int sourceweights[],
-                                            int maxoutdegree,
-                                             int destinations[],
-                                            int destweights[]);
-OMPI_DECLSPEC  int PMPI_Dist_graph_neighbors_count(MPI_Comm comm,
-                                                   int *inneighbors,
-                                                   int *outneighbors,
-                                                   int *weighted);
+                                   void *recvbuf, const int recvcounts[], const int rdispls[], const MPI_Datatype recvtypes[],
+                                   MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Ialltoallw_c(const void *sendbuf, const MPI_Count sendcounts[], const MPI_Aint sdispls[], const MPI_Datatype sendtypes[],
+                                     void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint rdispls[], const MPI_Datatype recvtypes[],
+                                     MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Alltoallw_init(const void *sendbuf, const int sendcounts[], const int sdispls[], const MPI_Datatype sendtypes[],
+                                       void *recvbuf, const int recvcounts[], const int rdispls[], const MPI_Datatype recvtypes[],
+                                       MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Alltoallw_init_c(const void *sendbuf, const MPI_Count sendcounts[], const MPI_Aint sdispls[], const MPI_Datatype sendtypes[],
+                                         void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint rdispls[], const MPI_Datatype recvtypes[],
+                                         MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Barrier(MPI_Comm comm);
 OMPI_DECLSPEC  int PMPI_Ibarrier(MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Barrier_init(MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Bcast(void *buffer, int count, MPI_Datatype datatype,
                               int root, MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Bcast_c(void *buffer, MPI_Count count, MPI_Datatype datatype,
+                                int root, MPI_Comm comm);
 OMPI_DECLSPEC  int PMPI_Ibcast(void *buffer, int count, MPI_Datatype datatype,
-				                              int root, MPI_Comm comm,
-											  MPI_Request *request);
+                               int root, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Ibcast_c(void *buffer, MPI_Count count, MPI_Datatype datatype,
+                                 int root, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Bcast_init(void *buffer, int count, MPI_Datatype datatype,
+                                   int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Bcast_init_c(void *buffer, MPI_Count count, MPI_Datatype datatype,
+                                     int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Bsend(const void *buf, int count, MPI_Datatype datatype,
                               int dest, int tag, MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Bsend_c(const void *buf, MPI_Count count, MPI_Datatype datatype,
+                                int dest, int tag, MPI_Comm comm);
 OMPI_DECLSPEC  int PMPI_Bsend_init(const void *buf, int count, MPI_Datatype datatype,
                                    int dest, int tag, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Bsend_init_c(const void *buf, MPI_Count count, MPI_Datatype datatype,
+                                   int dest, int tag, MPI_Comm comm, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Buffer_attach(void *buffer, int size);
+OMPI_DECLSPEC  int PMPI_Buffer_attach_c(void *buffer, MPI_Count size);
 OMPI_DECLSPEC  int PMPI_Buffer_detach(void *buffer, int *size);
+OMPI_DECLSPEC  int PMPI_Buffer_detach_c(void *buffer, MPI_Count *size);
 OMPI_DECLSPEC  int PMPI_Cancel(MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Cart_coords(MPI_Comm comm, int rank, int maxdims, int coords[]);
 OMPI_DECLSPEC  int PMPI_Cart_create(MPI_Comm old_comm, int ndims, const int dims[],
@@ -2055,17 +2759,42 @@ OMPI_DECLSPEC  int PMPI_Comm_create_keyval(MPI_Comm_copy_attr_function *comm_cop
                                            MPI_Comm_delete_attr_function *comm_delete_attr_fn,
                                            int *comm_keyval, void *extra_state);
 OMPI_DECLSPEC  int PMPI_Comm_create_group(MPI_Comm comm, MPI_Group group, int tag, MPI_Comm *newcomm);
+OMPI_DECLSPEC  int PMPI_Comm_create_from_group(MPI_Group group, const char *tag, MPI_Info info,
+ 					      MPI_Errhandler errhandler, MPI_Comm *newcomm);
 OMPI_DECLSPEC  int PMPI_Comm_create(MPI_Comm comm, MPI_Group group, MPI_Comm *newcomm);
 OMPI_DECLSPEC  int PMPI_Comm_delete_attr(MPI_Comm comm, int comm_keyval);
 OMPI_DECLSPEC  int PMPI_Comm_disconnect(MPI_Comm *comm);
 OMPI_DECLSPEC  int PMPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm);
 OMPI_DECLSPEC  int PMPI_Comm_idup(MPI_Comm comm, MPI_Comm *newcomm, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Comm_dup_with_info(MPI_Comm comm, MPI_Info info, MPI_Comm *newcomm);
+OMPI_DECLSPEC  int PMPI_Comm_idup_with_info(MPI_Comm comm, MPI_Info info, MPI_Comm *newcomm,
+                                            MPI_Request *request);
 OMPI_DECLSPEC  MPI_Comm PMPI_Comm_f2c(MPI_Fint comm);
 OMPI_DECLSPEC  int PMPI_Comm_free_keyval(int *comm_keyval);
 OMPI_DECLSPEC  int PMPI_Comm_free(MPI_Comm *comm);
 OMPI_DECLSPEC  int PMPI_Comm_get_attr(MPI_Comm comm, int comm_keyval,
                                       void *attribute_val, int *flag);
+OMPI_DECLSPEC  int PMPI_Dist_graph_create(MPI_Comm comm_old, int n, const int nodes[],
+                                          const int degrees[], const int targets[],
+                                          const int weights[], MPI_Info info,
+                                          int reorder, MPI_Comm * newcomm);
+OMPI_DECLSPEC  int PMPI_Dist_graph_create_adjacent(MPI_Comm comm_old,
+                                                   int indegree, const int sources[],
+                                                   const int sourceweights[],
+                                                   int outdegree,
+                                                   const int destinations[],
+                                                   const int destweights[],
+                                                   MPI_Info info, int reorder,
+                                                   MPI_Comm *comm_dist_graph);
+OMPI_DECLSPEC int PMPI_Dist_graph_neighbors(MPI_Comm comm, int maxindegree,
+                                            int sources[], int sourceweights[],
+                                            int maxoutdegree,
+                                            int destinations[],
+                                            int destweights[]);
+OMPI_DECLSPEC  int PMPI_Dist_graph_neighbors_count(MPI_Comm comm,
+                                                   int *inneighbors,
+                                                   int *outneighbors,
+                                                   int *weighted);
 OMPI_DECLSPEC  int PMPI_Comm_get_errhandler(MPI_Comm comm, MPI_Errhandler *erhandler);
 OMPI_DECLSPEC  int PMPI_Comm_get_info(MPI_Comm comm, MPI_Info *info_used);
 OMPI_DECLSPEC  int PMPI_Comm_get_name(MPI_Comm comm, char *comm_name, int *resultlen);
@@ -2099,12 +2828,20 @@ OMPI_DECLSPEC  MPI_Errhandler PMPI_Errhandler_f2c(MPI_Fint errhandler);
 OMPI_DECLSPEC  int PMPI_Errhandler_free(MPI_Errhandler *errhandler);
 OMPI_DECLSPEC  int PMPI_Error_class(int errorcode, int *errorclass);
 OMPI_DECLSPEC  int PMPI_Error_string(int errorcode, char *string, int *resultlen);
-OMPI_DECLSPEC  int PMPI_Exscan(const void *sendbuf, void *recvbuf, int count,
-                               MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Exscan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
+                               MPI_Op op, MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Exscan_c(const void *sendbuf, void *recvbuf, MPI_Count count, MPI_Datatype datatype,
+                                 MPI_Op op, MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Iexscan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
+                                MPI_Op op, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Iexscan_c(const void *sendbuf, void *recvbuf, MPI_Count count, MPI_Datatype datatype,
+                                  MPI_Op op, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Exscan_init(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
+                                    MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Exscan_init_c(const void *sendbuf, void *recvbuf, MPI_Count count, MPI_Datatype datatype,
+                                      MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Fetch_and_op(const void *origin_addr, void *result_addr, MPI_Datatype datatype,
                                      int target_rank, MPI_Aint target_disp, MPI_Op op, MPI_Win win);
-OMPI_DECLSPEC  int PMPI_Iexscan(const void *sendbuf, void *recvbuf, int count,
-                               MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, MPI_Request *request);
 OMPI_DECLSPEC  MPI_Fint PMPI_File_c2f(MPI_File file);
 OMPI_DECLSPEC  MPI_File PMPI_File_f2c(MPI_Fint file);
 OMPI_DECLSPEC  int PMPI_File_call_errhandler(MPI_File fh, int errorcode);
@@ -2130,74 +2867,132 @@ OMPI_DECLSPEC  int PMPI_File_get_view(MPI_File fh, MPI_Offset *disp,
                                       MPI_Datatype *filetype, char *datarep);
 OMPI_DECLSPEC  int PMPI_File_read_at(MPI_File fh, MPI_Offset offset, void *buf,
                                      int count, MPI_Datatype datatype, MPI_Status *status);
+OMPI_DECLSPEC  int PMPI_File_read_at_c(MPI_File fh, MPI_Offset offset, void *buf,
+                                       MPI_Count count, MPI_Datatype datatype, MPI_Status *status);
 OMPI_DECLSPEC  int PMPI_File_read_at_all(MPI_File fh, MPI_Offset offset, void *buf,
                                          int count, MPI_Datatype datatype, MPI_Status *status);
+OMPI_DECLSPEC  int PMPI_File_read_at_all_c(MPI_File fh, MPI_Offset offset, void *buf,
+                                           MPI_Count count, MPI_Datatype datatype, MPI_Status *status);
 OMPI_DECLSPEC  int PMPI_File_write_at(MPI_File fh, MPI_Offset offset, const void *buf,
                                       int count, MPI_Datatype datatype, MPI_Status *status);
+OMPI_DECLSPEC  int PMPI_File_write_at_c(MPI_File fh, MPI_Offset offset, const void *buf,
+                                        MPI_Count count, MPI_Datatype datatype, MPI_Status *status);
 OMPI_DECLSPEC  int PMPI_File_write_at_all(MPI_File fh, MPI_Offset offset, const void *buf,
                                           int count, MPI_Datatype datatype, MPI_Status *status);
+OMPI_DECLSPEC  int PMPI_File_write_at_all_c(MPI_File fh, MPI_Offset offset, const void *buf,
+                                            MPI_Count count, MPI_Datatype datatype, MPI_Status *status);
 OMPI_DECLSPEC  int PMPI_File_iread_at(MPI_File fh, MPI_Offset offset, void *buf,
                                       int count, MPI_Datatype datatype, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_File_iread_at_c(MPI_File fh, MPI_Offset offset, void *buf,
+                                        MPI_Count count, MPI_Datatype datatype, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_File_iwrite_at(MPI_File fh, MPI_Offset offset, const void *buf,
                                        int count, MPI_Datatype datatype, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_File_iwrite_at_c(MPI_File fh, MPI_Offset offset, const void *buf,
+                                         MPI_Count count, MPI_Datatype datatype, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_File_iread_at_all(MPI_File fh, MPI_Offset offset, void *buf,
                                           int count, MPI_Datatype datatype, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_File_iread_at_all_c(MPI_File fh, MPI_Offset offset, void *buf,
+                                            MPI_Count count, MPI_Datatype datatype, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_File_iwrite_at_all(MPI_File fh, MPI_Offset offset, const void *buf,
                                            int count, MPI_Datatype datatype, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_File_iwrite_at_all_c(MPI_File fh, MPI_Offset offset, const void *buf,
+                                             MPI_Count count, MPI_Datatype datatype, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_File_read(MPI_File fh, void *buf, int count,
                                   MPI_Datatype datatype, MPI_Status *status);
+OMPI_DECLSPEC  int PMPI_File_read_c(MPI_File fh, void *buf, MPI_Count count,
+                                    MPI_Datatype datatype, MPI_Status *status);
 OMPI_DECLSPEC  int PMPI_File_read_all(MPI_File fh, void *buf, int count,
                                       MPI_Datatype datatype, MPI_Status *status);
+OMPI_DECLSPEC  int PMPI_File_read_all_c(MPI_File fh, void *buf, MPI_Count count,
+                                        MPI_Datatype datatype, MPI_Status *status);
 OMPI_DECLSPEC  int PMPI_File_write(MPI_File fh, const void *buf, int count,
                                    MPI_Datatype datatype, MPI_Status *status);
+OMPI_DECLSPEC  int PMPI_File_write_c(MPI_File fh, const void *buf, MPI_Count count,
+                                     MPI_Datatype datatype, MPI_Status *status);
 OMPI_DECLSPEC  int PMPI_File_write_all(MPI_File fh, const void *buf, int count,
                                        MPI_Datatype datatype, MPI_Status *status);
+OMPI_DECLSPEC  int PMPI_File_write_all_c(MPI_File fh, const void *buf, MPI_Count count,
+                                         MPI_Datatype datatype, MPI_Status *status);
 OMPI_DECLSPEC  int PMPI_File_iread(MPI_File fh, void *buf, int count,
                                    MPI_Datatype datatype, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_File_iread_c(MPI_File fh, void *buf, MPI_Count count,
+                                     MPI_Datatype datatype, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_File_iwrite(MPI_File fh, const void *buf, int count,
                                     MPI_Datatype datatype, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_File_iwrite_c(MPI_File fh, const void *buf, MPI_Count count,
+                                      MPI_Datatype datatype, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_File_iread_all(MPI_File fh, void *buf, int count,
                                        MPI_Datatype datatype, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_File_iread_all_c(MPI_File fh, void *buf, MPI_Count count,
+                                         MPI_Datatype datatype, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_File_iwrite_all(MPI_File fh, const void *buf, int count,
                                         MPI_Datatype datatype, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_File_iwrite_all_c(MPI_File fh, const void *buf, MPI_Count count,
+                                          MPI_Datatype datatype, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_File_seek(MPI_File fh, MPI_Offset offset, int whence);
 OMPI_DECLSPEC  int PMPI_File_get_position(MPI_File fh, MPI_Offset *offset);
 OMPI_DECLSPEC  int PMPI_File_get_byte_offset(MPI_File fh, MPI_Offset offset,
                                              MPI_Offset *disp);
 OMPI_DECLSPEC  int PMPI_File_read_shared(MPI_File fh, void *buf, int count,
                                          MPI_Datatype datatype, MPI_Status *status);
+OMPI_DECLSPEC  int PMPI_File_read_shared_c(MPI_File fh, void *buf, MPI_Count count,
+                                           MPI_Datatype datatype, MPI_Status *status);
 OMPI_DECLSPEC  int PMPI_File_write_shared(MPI_File fh, const void *buf, int count,
                                           MPI_Datatype datatype, MPI_Status *status);
+OMPI_DECLSPEC  int PMPI_File_write_shared_c(MPI_File fh, const void *buf, MPI_Count count,
+                                            MPI_Datatype datatype, MPI_Status *status);
 OMPI_DECLSPEC  int PMPI_File_iread_shared(MPI_File fh, void *buf, int count,
                                           MPI_Datatype datatype, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_File_iread_shared_c(MPI_File fh, void *buf, MPI_Count count,
+                                            MPI_Datatype datatype, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_File_iwrite_shared(MPI_File fh, const void *buf, int count,
                                            MPI_Datatype datatype, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_File_iwrite_shared_c(MPI_File fh, const void *buf, MPI_Count count,
+                                             MPI_Datatype datatype, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_File_read_ordered(MPI_File fh, void *buf, int count,
                                           MPI_Datatype datatype, MPI_Status *status);
+OMPI_DECLSPEC  int PMPI_File_read_ordered_c(MPI_File fh, void *buf, MPI_Count count,
+                                            MPI_Datatype datatype, MPI_Status *status);
 OMPI_DECLSPEC  int PMPI_File_write_ordered(MPI_File fh, const void *buf, int count,
                                            MPI_Datatype datatype, MPI_Status *status);
+OMPI_DECLSPEC  int PMPI_File_write_ordered_c(MPI_File fh, const void *buf, MPI_Count count,
+                                             MPI_Datatype datatype, MPI_Status *status);
 OMPI_DECLSPEC  int PMPI_File_seek_shared(MPI_File fh, MPI_Offset offset, int whence);
 OMPI_DECLSPEC  int PMPI_File_get_position_shared(MPI_File fh, MPI_Offset *offset);
 OMPI_DECLSPEC  int PMPI_File_read_at_all_begin(MPI_File fh, MPI_Offset offset, void *buf,
                                                int count, MPI_Datatype datatype);
+OMPI_DECLSPEC  int PMPI_File_read_at_all_begin_c(MPI_File fh, MPI_Offset offset, void *buf,
+                                                 MPI_Count count, MPI_Datatype datatype);
 OMPI_DECLSPEC  int PMPI_File_read_at_all_end(MPI_File fh, void *buf, MPI_Status *status);
 OMPI_DECLSPEC  int PMPI_File_write_at_all_begin(MPI_File fh, MPI_Offset offset, const void *buf,
                                                 int count, MPI_Datatype datatype);
+OMPI_DECLSPEC  int PMPI_File_write_at_all_begin_c(MPI_File fh, MPI_Offset offset, const void *buf,
+                                                  MPI_Count count, MPI_Datatype datatype);
 OMPI_DECLSPEC  int PMPI_File_write_at_all_end(MPI_File fh, const void *buf, MPI_Status *status);
 OMPI_DECLSPEC  int PMPI_File_read_all_begin(MPI_File fh, void *buf, int count,
                                             MPI_Datatype datatype);
+OMPI_DECLSPEC  int PMPI_File_read_all_begin_c(MPI_File fh, void *buf, MPI_Count count,
+                                              MPI_Datatype datatype);
 OMPI_DECLSPEC  int PMPI_File_read_all_end(MPI_File fh, void *buf, MPI_Status *status);
 OMPI_DECLSPEC  int PMPI_File_write_all_begin(MPI_File fh, const void *buf, int count,
                                              MPI_Datatype datatype);
+OMPI_DECLSPEC  int PMPI_File_write_all_begin_c(MPI_File fh, const void *buf, MPI_Count count,
+                                               MPI_Datatype datatype);
 OMPI_DECLSPEC  int PMPI_File_write_all_end(MPI_File fh, const void *buf, MPI_Status *status);
 OMPI_DECLSPEC  int PMPI_File_read_ordered_begin(MPI_File fh, void *buf, int count,
                                                 MPI_Datatype datatype);
+OMPI_DECLSPEC  int PMPI_File_read_ordered_begin_c(MPI_File fh, void *buf, MPI_Count count,
+                                                  MPI_Datatype datatype);
 OMPI_DECLSPEC  int PMPI_File_read_ordered_end(MPI_File fh, void *buf, MPI_Status *status);
 OMPI_DECLSPEC  int PMPI_File_write_ordered_begin(MPI_File fh, const void *buf, int count,
                                                  MPI_Datatype datatype);
+OMPI_DECLSPEC  int PMPI_File_write_ordered_begin_c(MPI_File fh, const void *buf, MPI_Count count,
+                                                   MPI_Datatype datatype);
 OMPI_DECLSPEC  int PMPI_File_write_ordered_end(MPI_File fh, const void *buf, MPI_Status *status);
 OMPI_DECLSPEC  int PMPI_File_get_type_extent(MPI_File fh, MPI_Datatype datatype,
                                              MPI_Aint *extent);
+OMPI_DECLSPEC  int PMPI_File_get_type_extent_c(MPI_File fh, MPI_Datatype datatype,
+                                               MPI_Count *extent);
 OMPI_DECLSPEC  int PMPI_File_set_atomicity(MPI_File fh, int flag);
 OMPI_DECLSPEC  int PMPI_File_get_atomicity(MPI_File fh, int *flag);
 OMPI_DECLSPEC  int PMPI_File_sync(MPI_File fh);
@@ -2207,29 +3002,61 @@ OMPI_DECLSPEC  int PMPI_Free_mem(void *base);
 OMPI_DECLSPEC  int PMPI_Gather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                                void *recvbuf, int recvcount, MPI_Datatype recvtype,
                                int root, MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Gather_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                 void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                 int root, MPI_Comm comm);
 OMPI_DECLSPEC  int PMPI_Igather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                               void *recvbuf, int recvcount, MPI_Datatype recvtype,
-                               int root, MPI_Comm comm, MPI_Request *request);
+                                void *recvbuf, int recvcount, MPI_Datatype recvtype,
+                                int root, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Igather_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                  void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                  int root, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Gather_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
+                                    void *recvbuf, int recvcount, MPI_Datatype recvtype,
+                                    int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Gather_init_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                      void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                      int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Gatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                                 void *recvbuf, const int recvcounts[], const int displs[],
                                 MPI_Datatype recvtype, int root, MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Gatherv_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                  void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint displs[],
+                                  MPI_Datatype recvtype, int root, MPI_Comm comm);
 OMPI_DECLSPEC  int PMPI_Igatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                                void *recvbuf, const int recvcounts[], const int displs[],
-                                MPI_Datatype recvtype, int root, MPI_Comm comm, MPI_Request *request);
+                                 void *recvbuf, const int recvcounts[], const int displs[], MPI_Datatype recvtype,
+                                 int root, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Igatherv_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                   void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint displs[], MPI_Datatype recvtype,
+                                   int root, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Gatherv_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
+                                     void *recvbuf, const int recvcounts[], const int displs[], MPI_Datatype recvtype,
+                                     int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Gatherv_init_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                       void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint displs[], MPI_Datatype recvtype,
+                                       int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Get_address(const void *location, MPI_Aint *address);
 OMPI_DECLSPEC  int PMPI_Get_count(const MPI_Status *status, MPI_Datatype datatype, int *count);
-OMPI_DECLSPEC  int PMPI_Get_elements(const MPI_Status *status, MPI_Datatype datatype,
-                                     int *count);
-OMPI_DECLSPEC  int PMPI_Get_elements_x(const MPI_Status *status, MPI_Datatype datatype,
-                                       MPI_Count *count);
+OMPI_DECLSPEC  int PMPI_Get_count_c(const MPI_Status *status, MPI_Datatype datatype, MPI_Count *count);
+OMPI_DECLSPEC  int PMPI_Get_elements(const MPI_Status *status, MPI_Datatype datatype, int *count);
+OMPI_DECLSPEC  int PMPI_Get_elements_c(const MPI_Status *status, MPI_Datatype datatype, MPI_Count *count);
+OMPI_DECLSPEC  int PMPI_Get_elements_x(const MPI_Status *status, MPI_Datatype datatype, MPI_Count *count);
 OMPI_DECLSPEC  int PMPI_Get(void *origin_addr, int origin_count,
                             MPI_Datatype origin_datatype, int target_rank,
                             MPI_Aint target_disp, int target_count,
                             MPI_Datatype target_datatype, MPI_Win win);
+OMPI_DECLSPEC  int PMPI_Get_c(void *origin_addr, MPI_Count origin_count,
+                              MPI_Datatype origin_datatype, int target_rank,
+                              MPI_Aint target_disp, MPI_Count target_count,
+                              MPI_Datatype target_datatype, MPI_Win win);
 OMPI_DECLSPEC  int PMPI_Get_accumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype,
                                        void *result_addr, int result_count, MPI_Datatype result_datatype,
                                        int target_rank, MPI_Aint target_disp, int target_count,
                                        MPI_Datatype target_datatype, MPI_Op op, MPI_Win win);
+OMPI_DECLSPEC  int PMPI_Get_accumulate_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype,
+                                         void *result_addr, MPI_Count result_count, MPI_Datatype result_datatype,
+                                         int target_rank, MPI_Aint target_disp, MPI_Count target_count,
+                                         MPI_Datatype target_datatype, MPI_Op op, MPI_Win win);
 OMPI_DECLSPEC  int PMPI_Get_library_version(char *version, int *resultlen);
 OMPI_DECLSPEC  int PMPI_Get_processor_name(char *name, int *resultlen);
 OMPI_DECLSPEC  int PMPI_Get_version(int *version, int *subversion);
@@ -2256,6 +3083,7 @@ OMPI_DECLSPEC  int PMPI_Group_excl(MPI_Group group, int n, const int ranks[],
                                    MPI_Group *newgroup);
 OMPI_DECLSPEC  MPI_Group PMPI_Group_f2c(MPI_Fint group);
 OMPI_DECLSPEC  int PMPI_Group_free(MPI_Group *group);
+OMPI_DECLSPEC  int PMPI_Group_from_session_pset (MPI_Session session, const char *pset_name, MPI_Group *newgroup);
 OMPI_DECLSPEC  int PMPI_Group_incl(MPI_Group group, int n, const int ranks[],
                                    MPI_Group *newgroup);
 OMPI_DECLSPEC  int PMPI_Group_intersection(MPI_Group group1, MPI_Group group2,
@@ -2272,23 +3100,26 @@ OMPI_DECLSPEC  int PMPI_Group_union(MPI_Group group1, MPI_Group group2,
                                     MPI_Group *newgroup);
 OMPI_DECLSPEC  int PMPI_Ibsend(const void *buf, int count, MPI_Datatype datatype, int dest,
                                int tag, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Ibsend_c(const void *buf, MPI_Count count, MPI_Datatype datatype, int dest,
+                               int tag, MPI_Comm comm, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Improbe(int source, int tag, MPI_Comm comm,
                                 int *flag, MPI_Message *message,
                                 MPI_Status *status);
 OMPI_DECLSPEC  int PMPI_Imrecv(void *buf, int count, MPI_Datatype type,
                                MPI_Message *message, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Imrecv_c(void *buf, MPI_Count count, MPI_Datatype type,
+                                 MPI_Message *message, MPI_Request *request);
 OMPI_DECLSPEC  MPI_Fint PMPI_Info_c2f(MPI_Info info);
 OMPI_DECLSPEC  int PMPI_Info_create(MPI_Info *info);
+OMPI_DECLSPEC  int PMPI_Info_create_env(int argc, char *argv[], MPI_Info *info);
 OMPI_DECLSPEC  int PMPI_Info_delete(MPI_Info info, const char *key);
 OMPI_DECLSPEC  int PMPI_Info_dup(MPI_Info info, MPI_Info *newinfo);
 OMPI_DECLSPEC  MPI_Info PMPI_Info_f2c(MPI_Fint info);
 OMPI_DECLSPEC  int PMPI_Info_free(MPI_Info *info);
-OMPI_DECLSPEC  int PMPI_Info_get(MPI_Info info, const char *key, int valuelen,
-                                 char *value, int *flag);
 OMPI_DECLSPEC  int PMPI_Info_get_nkeys(MPI_Info info, int *nkeys);
 OMPI_DECLSPEC  int PMPI_Info_get_nthkey(MPI_Info info, int n, char *key);
-OMPI_DECLSPEC  int PMPI_Info_get_valuelen(MPI_Info info, const char *key, int *valuelen,
-                                          int *flag);
+OMPI_DECLSPEC  int PMPI_Info_get_string(MPI_Info info, const char *key, int *buflen,
+                                        char *value, int *flag);
 OMPI_DECLSPEC  int PMPI_Info_set(MPI_Info info, const char *key, const char *value);
 OMPI_DECLSPEC  int PMPI_Init(int *argc, char ***argv);
 OMPI_DECLSPEC  int PMPI_Initialized(int *flag);
@@ -2297,106 +3128,257 @@ OMPI_DECLSPEC  int PMPI_Init_thread(int *argc, char ***argv, int required,
 OMPI_DECLSPEC  int PMPI_Intercomm_create(MPI_Comm local_comm, int local_leader,
                                          MPI_Comm bridge_comm, int remote_leader,
                                          int tag, MPI_Comm *newintercomm);
+OMPI_DECLSPEC  int PMPI_Intercomm_create_from_groups (MPI_Group local_group, int local_leader, MPI_Group remote_group,
+                                                      int remote_leader, const char *tag, MPI_Info info, MPI_Errhandler errhandler,
+                                                      MPI_Comm *newintercomm);
 OMPI_DECLSPEC  int PMPI_Intercomm_merge(MPI_Comm intercomm, int high,
-                                        MPI_Comm *newintercomm);
+                                        MPI_Comm *newintracomm);
 OMPI_DECLSPEC  int PMPI_Iprobe(int source, int tag, MPI_Comm comm, int *flag,
                                MPI_Status *status);
 OMPI_DECLSPEC  int PMPI_Irecv(void *buf, int count, MPI_Datatype datatype, int source,
                               int tag, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Irecv_c(void *buf, MPI_Count count, MPI_Datatype datatype, int source,
+                                int tag, MPI_Comm comm, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Irsend(const void *buf, int count, MPI_Datatype datatype, int dest,
                                int tag, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Irsend_c(const void *buf, MPI_Count count, MPI_Datatype datatype, int dest,
+                                 int tag, MPI_Comm comm, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Isend(const void *buf, int count, MPI_Datatype datatype, int dest,
                               int tag, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Isend_c(const void *buf, MPI_Count count, MPI_Datatype datatype, int dest,
+                                int tag, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Isendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
+                                  int dest, int sendtag, void *recvbuf, int recvcount,
+                                  MPI_Datatype recvtype, int source, int recvtag,
+                                  MPI_Comm comm,  MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Isendrecv_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                    int dest, int sendtag, void *recvbuf, MPI_Count recvcount,
+                                    MPI_Datatype recvtype, int source, int recvtag,
+                                    MPI_Comm comm,  MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Isendrecv_replace(void * buf, int count, MPI_Datatype datatype,
+                                          int dest, int sendtag, int source, int recvtag,
+                                          MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Isendrecv_replace_c(void * buf, MPI_Count count, MPI_Datatype datatype,
+                                            int dest, int sendtag, int source, int recvtag,
+                                            MPI_Comm comm, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Issend(const void *buf, int count, MPI_Datatype datatype, int dest,
                                int tag, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Issend_c(const void *buf, MPI_Count count, MPI_Datatype datatype, int dest,
+                                 int tag, MPI_Comm comm, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Is_thread_main(int *flag);
 OMPI_DECLSPEC  int PMPI_Lookup_name(const char *service_name, MPI_Info info, char *port_name);
 OMPI_DECLSPEC  MPI_Fint PMPI_Message_c2f(MPI_Message message);
 OMPI_DECLSPEC  MPI_Message PMPI_Message_f2c(MPI_Fint message);
 OMPI_DECLSPEC  int PMPI_Mprobe(int source, int tag, MPI_Comm comm,
-                               MPI_Message *message,
-                               MPI_Status *status);
+                                MPI_Message *message,
+                                MPI_Status *status);
 OMPI_DECLSPEC  int PMPI_Mrecv(void *buf, int count, MPI_Datatype type,
                               MPI_Message *message, MPI_Status *status);
+OMPI_DECLSPEC  int PMPI_Mrecv_c(void *buf, MPI_Count count, MPI_Datatype type,
+                                MPI_Message *message, MPI_Status *status);
 OMPI_DECLSPEC  int PMPI_Neighbor_allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                                            void *recvbuf, int recvcount, MPI_Datatype recvtype,
                                            MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Neighbor_allgather_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                             void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                             MPI_Comm comm);
 OMPI_DECLSPEC  int PMPI_Ineighbor_allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                                             void *recvbuf, int recvcount, MPI_Datatype recvtype,
                                             MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Ineighbor_allgather_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                              void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                              MPI_Comm comm, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Neighbor_allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                                             void *recvbuf, const int recvcounts[], const int displs[],
                                             MPI_Datatype recvtype, MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Neighbor_allgatherv_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                              void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint displs[],
+                                              MPI_Datatype recvtype, MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Neighbor_allgather_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
+                                                void *recvbuf, int recvcount, MPI_Datatype recvtype,
+                                                MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Neighbor_allgather_init_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                                  void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                                  MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Ineighbor_allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                                             void *recvbuf, const int recvcounts[], const int displs[],
-                                             MPI_Datatype recvtype, MPI_Comm comm, MPI_Request *request);
+                                             void *recvbuf, const int recvcounts[], const int displs[], MPI_Datatype recvtype,
+                                             MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Ineighbor_allgatherv_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                               void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint displs[], MPI_Datatype recvtype,
+                                               MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Neighbor_allgatherv_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
+                                                 void *recvbuf, const int recvcounts[], const int displs[], MPI_Datatype recvtype,
+                                                 MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Neighbor_allgatherv_init_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                                   void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint displs[], MPI_Datatype recvtype,
+                                                   MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Neighbor_alltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                                           void *recvbuf, int recvcount, MPI_Datatype recvtype,
                                           MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Neighbor_alltoall_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                            void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                            MPI_Comm comm);
 OMPI_DECLSPEC  int PMPI_Ineighbor_alltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                                            void *recvbuf, int recvcount, MPI_Datatype recvtype,
                                            MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Ineighbor_alltoall_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                             void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                             MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Neighbor_alltoall_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
+                                               void *recvbuf, int recvcount, MPI_Datatype recvtype,
+                                               MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Neighbor_alltoall_init_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                                 void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                                 MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Neighbor_alltoallv(const void *sendbuf, const int sendcounts[], const int sdispls[],  MPI_Datatype sendtype,
                                            void *recvbuf, const int recvcounts[], const int rdispls[], MPI_Datatype recvtype,
                                            MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Neighbor_alltoallv_c(const void *sendbuf, const MPI_Count sendcounts[], const MPI_Aint sdispls[],  MPI_Datatype sendtype,
+                                             void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint rdispls[], MPI_Datatype recvtype,
+                                             MPI_Comm comm);
 OMPI_DECLSPEC  int PMPI_Ineighbor_alltoallv(const void *sendbuf, const int sendcounts[], const int sdispls[], MPI_Datatype sendtype,
                                             void *recvbuf, const int recvcounts[], const int rdispls[], MPI_Datatype recvtype,
                                             MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Ineighbor_alltoallv_c(const void *sendbuf, const MPI_Count sendcounts[], const MPI_Aint sdispls[], MPI_Datatype sendtype,
+                                              void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint rdispls[], MPI_Datatype recvtype,
+                                              MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Neighbor_alltoallv_init(const void *sendbuf, const int sendcounts[], const int sdispls[], MPI_Datatype sendtype,
+                                                void *recvbuf, const int recvcounts[], const int rdispls[], MPI_Datatype recvtype,
+                                                MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Neighbor_alltoallv_init_c(const void *sendbuf, const MPI_Count sendcounts[], const MPI_Aint sdispls[], MPI_Datatype sendtype,
+                                                  void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint rdispls[], MPI_Datatype recvtype,
+                                                  MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Neighbor_alltoallw(const void *sendbuf, const int sendcounts[], const MPI_Aint sdispls[], const MPI_Datatype sendtypes[],
                                            void *recvbuf, const int recvcounts[], const MPI_Aint rdispls[], const MPI_Datatype recvtypes[],
                                            MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Neighbor_alltoallw_c(const void *sendbuf, const MPI_Count sendcounts[], const MPI_Aint sdispls[], const MPI_Datatype sendtypes[],
+                                             void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint rdispls[], const MPI_Datatype recvtypes[],
+                                             MPI_Comm comm);
 OMPI_DECLSPEC  int PMPI_Ineighbor_alltoallw(const void *sendbuf, const int sendcounts[], const MPI_Aint sdispls[], const MPI_Datatype sendtypes[],
                                             void *recvbuf, const int recvcounts[], const MPI_Aint rdispls[], const MPI_Datatype recvtypes[],
                                             MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Ineighbor_alltoallw_c(const void *sendbuf, const MPI_Count sendcounts[], const MPI_Aint sdispls[], const MPI_Datatype sendtypes[],
+                                              void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint rdispls[], const MPI_Datatype recvtypes[],
+                                              MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Neighbor_alltoallw_init(const void *sendbuf, const int sendcounts[], const MPI_Aint sdispls[], const MPI_Datatype sendtypes[],
+                                                void *recvbuf, const int recvcounts[], const MPI_Aint rdispls[], const MPI_Datatype recvtypes[],
+                                                MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Neighbor_alltoallw_init_c(const void *sendbuf, const MPI_Count sendcounts[], const MPI_Aint sdispls[], const MPI_Datatype sendtypes[],
+                                                  void *recvbuf, const MPI_Count recvcounts[], const MPI_Aint rdispls[], const MPI_Datatype recvtypes[],
+                                                  MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  MPI_Fint PMPI_Op_c2f(MPI_Op op);
 OMPI_DECLSPEC  int PMPI_Op_commutative(MPI_Op op, int *commute);
 OMPI_DECLSPEC  int PMPI_Op_create(MPI_User_function *function, int commute, MPI_Op *op);
+OMPI_DECLSPEC  int PMPI_Op_create_c(MPI_User_function_c *function, int commute, MPI_Op *op);
 OMPI_DECLSPEC  int PMPI_Open_port(MPI_Info info, char *port_name);
 OMPI_DECLSPEC  MPI_Op PMPI_Op_f2c(MPI_Fint op);
 OMPI_DECLSPEC  int PMPI_Op_free(MPI_Op *op);
 OMPI_DECLSPEC  int PMPI_Pack_external(const char datarep[], const void *inbuf, int incount,
                                       MPI_Datatype datatype, void *outbuf,
                                       MPI_Aint outsize, MPI_Aint *position);
+OMPI_DECLSPEC  int PMPI_Pack_external_c(const char datarep[], const void *inbuf, MPI_Count incount,
+                                        MPI_Datatype datatype, void *outbuf,
+                                        MPI_Count outsize, MPI_Count *position);
 OMPI_DECLSPEC  int PMPI_Pack_external_size(const char datarep[], int incount,
                                            MPI_Datatype datatype, MPI_Aint *size);
+OMPI_DECLSPEC  int PMPI_Pack_external_size_c(const char datarep[], MPI_Count incount,
+                                             MPI_Datatype datatype, MPI_Count *size);
 OMPI_DECLSPEC  int PMPI_Pack(const void *inbuf, int incount, MPI_Datatype datatype,
                              void *outbuf, int outsize, int *position, MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Pack_c(const void *inbuf, MPI_Count incount, MPI_Datatype datatype,
+                               void *outbuf, MPI_Count outsize, MPI_Count *position, MPI_Comm comm);
 OMPI_DECLSPEC  int PMPI_Pack_size(int incount, MPI_Datatype datatype, MPI_Comm comm,
                                   int *size);
+OMPI_DECLSPEC  int PMPI_Pack_size_c(MPI_Count incount, MPI_Datatype datatype, MPI_Comm comm,
+                                    MPI_Count *size);
+OMPI_DECLSPEC  int PMPI_Parrived(MPI_Request request, int partition, int *flag);
 OMPI_DECLSPEC  int PMPI_Pcontrol(const int level, ...);
+OMPI_DECLSPEC  int PMPI_Pready(int partitions, MPI_Request request);
+OMPI_DECLSPEC  int PMPI_Pready_range(int partition_low, int partition_high,
+                                     MPI_Request request);
+OMPI_DECLSPEC  int PMPI_Pready_list(int length, const int partition_list[], MPI_Request request);
+OMPI_DECLSPEC  int PMPI_Precv_init(void* buf, int partitions, MPI_Count count,
+                                   MPI_Datatype datatype, int source, int tag, MPI_Comm comm,
+                                   MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Probe(int source, int tag, MPI_Comm comm, MPI_Status *status);
+OMPI_DECLSPEC  int PMPI_Psend_init(const void* buf, int partitions, MPI_Count count,
+                                   MPI_Datatype datatype, int dest, int tag, MPI_Comm comm,
+                                   MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Publish_name(const char *service_name, MPI_Info info,
                                      const char *port_name);
 OMPI_DECLSPEC  int PMPI_Put(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype,
                             int target_rank, MPI_Aint target_disp, int target_count,
                             MPI_Datatype target_datatype, MPI_Win win);
+OMPI_DECLSPEC  int PMPI_Put_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype,
+                              int target_rank, MPI_Aint target_disp, MPI_Count target_count,
+                              MPI_Datatype target_datatype, MPI_Win win);
 OMPI_DECLSPEC  int PMPI_Query_thread(int *provided);
 OMPI_DECLSPEC  int PMPI_Raccumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype,
                                     int target_rank, MPI_Aint target_disp, int target_count,
                                     MPI_Datatype target_datatype, MPI_Op op, MPI_Win win, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Raccumulate_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype,
+                                      int target_rank, MPI_Aint target_disp, MPI_Count target_count,
+                                      MPI_Datatype target_datatype, MPI_Op op, MPI_Win win, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Recv_init(void *buf, int count, MPI_Datatype datatype, int source,
                                   int tag, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Recv_init_c(void *buf, MPI_Count count, MPI_Datatype datatype, int source,
+                                    int tag, MPI_Comm comm, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Recv(void *buf, int count, MPI_Datatype datatype, int source,
                              int tag, MPI_Comm comm, MPI_Status *status);
-OMPI_DECLSPEC  int PMPI_Reduce(const void *sendbuf, void *recvbuf, int count,
-                               MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm);
-OMPI_DECLSPEC  int PMPI_Ireduce(const void *sendbuf, void *recvbuf, int count,
-                               MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Recv_c(void *buf, MPI_Count count, MPI_Datatype datatype, int source,
+                               int tag, MPI_Comm comm, MPI_Status *status);
+OMPI_DECLSPEC  int PMPI_Recv_c(void *buf, MPI_Count count, MPI_Datatype datatype, int source,
+                               int tag, MPI_Comm comm, MPI_Status *status);
+OMPI_DECLSPEC  int PMPI_Reduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
+                               MPI_Op op, int root, MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Reduce_c(const void *sendbuf, void *recvbuf, MPI_Count count, MPI_Datatype datatype,
+                                 MPI_Op op, int root, MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Ireduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
+                                MPI_Op op, int root, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Ireduce_c(const void *sendbuf, void *recvbuf, MPI_Count count, MPI_Datatype datatype,
+                                  MPI_Op op, int root, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Reduce_init(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
+                                    MPI_Op op, int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Reduce_init_c(const void *sendbuf, void *recvbuf, MPI_Count count, MPI_Datatype datatype,
+                                      MPI_Op op, int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Reduce_local(const void *inbuf, void *inoutbuf, int count,
                                      MPI_Datatype datatype, MPI_Op op);
-OMPI_DECLSPEC  int PMPI_Reduce_scatter(const void *sendbuf, void *recvbuf, const int recvcounts[],
-                                       MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
-OMPI_DECLSPEC  int PMPI_Ireduce_scatter(const void *sendbuf, void *recvbuf, const int recvcounts[],
-                                        MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, MPI_Request *request);
-OMPI_DECLSPEC  int PMPI_Reduce_scatter_block(const void *sendbuf, void *recvbuf, int recvcount,
-                                             MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
-OMPI_DECLSPEC  int PMPI_Ireduce_scatter_block(const void *sendbuf, void *recvbuf, int recvcount,
-                                              MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Reduce_local_c(const void *inbuf, void *inoutbuf, MPI_Count count,
+                                       MPI_Datatype datatype, MPI_Op op);
+OMPI_DECLSPEC  int PMPI_Reduce_scatter(const void *sendbuf, void *recvbuf, const int recvcounts[], MPI_Datatype datatype,
+                                       MPI_Op op, MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Reduce_scatter_c(const void *sendbuf, void *recvbuf, const MPI_Count recvcounts[], MPI_Datatype datatype,
+                                         MPI_Op op, MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Ireduce_scatter(const void *sendbuf, void *recvbuf, const int recvcounts[], MPI_Datatype datatype,
+                                        MPI_Op op, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Ireduce_scatter_c(const void *sendbuf, void *recvbuf, const MPI_Count recvcounts[], MPI_Datatype datatype,
+                                          MPI_Op op, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Reduce_scatter_init(const void *sendbuf, void *recvbuf, const int recvcounts[], MPI_Datatype datatype,
+                                            MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Reduce_scatter_init_c(const void *sendbuf, void *recvbuf, const MPI_Count recvcounts[], MPI_Datatype datatype,
+                                              MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Reduce_scatter_block(const void *sendbuf, void *recvbuf, int recvcount, MPI_Datatype datatype,
+                                             MPI_Op op, MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Reduce_scatter_block_c(const void *sendbuf, void *recvbuf, MPI_Count recvcount, MPI_Datatype datatype,
+                                               MPI_Op op, MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Ireduce_scatter_block(const void *sendbuf, void *recvbuf, int recvcount, MPI_Datatype datatype,
+                                              MPI_Op op, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Ireduce_scatter_block_c(const void *sendbuf, void *recvbuf, MPI_Count recvcount, MPI_Datatype datatype,
+                                                MPI_Op op, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Reduce_scatter_block_init(const void *sendbuf, void *recvbuf, int recvcount, MPI_Datatype datatype,
+                                                  MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Reduce_scatter_block_init_c(const void *sendbuf, void *recvbuf, MPI_Count recvcount, MPI_Datatype datatype,
+                                                    MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Register_datarep(const char *datarep,
                                          MPI_Datarep_conversion_function *read_conversion_fn,
                                          MPI_Datarep_conversion_function *write_conversion_fn,
                                          MPI_Datarep_extent_function *dtype_file_extent_fn,
                                          void *extra_state);
+OMPI_DECLSPEC  int PMPI_Register_datarep_c(const char *datarep,
+                                           MPI_Datarep_conversion_function_c *read_conversion_fn,
+                                           MPI_Datarep_conversion_function_c *write_conversion_fn,
+                                           MPI_Datarep_extent_function *dtype_file_extent_fn,
+                                           void *extra_state);
 OMPI_DECLSPEC  MPI_Fint PMPI_Request_c2f(MPI_Request request);
 OMPI_DECLSPEC  MPI_Request PMPI_Request_f2c(MPI_Fint request);
 OMPI_DECLSPEC  int PMPI_Request_free(MPI_Request *request);
@@ -2405,68 +3387,157 @@ OMPI_DECLSPEC  int PMPI_Request_get_status(MPI_Request request, int *flag,
 OMPI_DECLSPEC  int PMPI_Rget(void *origin_addr, int origin_count, MPI_Datatype origin_datatype,
                              int target_rank, MPI_Aint target_disp, int target_count, MPI_Datatype target_datatype,
                              MPI_Win win, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Rget_c(void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype,
+                               int target_rank, MPI_Aint target_disp, MPI_Count target_count, MPI_Datatype target_datatype,
+                               MPI_Win win, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Rget_accumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype,
                                         void *result_addr, int result_count, MPI_Datatype result_datatype,
                                         int target_rank, MPI_Aint target_disp, int target_count,
                                         MPI_Datatype target_datatype, MPI_Op op,
                                         MPI_Win win, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Rget_accumulate_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype,
+                                          void *result_addr, MPI_Count result_count, MPI_Datatype result_datatype,
+                                          int target_rank, MPI_Aint target_disp, MPI_Count target_count,
+                                          MPI_Datatype target_datatype, MPI_Op op,
+                                          MPI_Win win, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Rput(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype,
                              int target_rank, MPI_Aint target_disp, int target_cout,
                              MPI_Datatype target_datatype, MPI_Win win, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Rput_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype,
+                               int target_rank, MPI_Aint target_disp, MPI_Count target_cout,
+                               MPI_Datatype target_datatype, MPI_Win win, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Rsend(const void *ibuf, int count, MPI_Datatype datatype, int dest,
                               int tag, MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Rsend_c(const void *ibuf, MPI_Count count, MPI_Datatype datatype, int dest,
+                                int tag, MPI_Comm comm);
 OMPI_DECLSPEC  int PMPI_Rsend_init(const void *buf, int count, MPI_Datatype datatype,
                                    int dest, int tag, MPI_Comm comm,
                                    MPI_Request *request);
-OMPI_DECLSPEC  int PMPI_Scan(const void *sendbuf, void *recvbuf, int count,
-                             MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
-OMPI_DECLSPEC  int PMPI_Iscan(const void *sendbuf, void *recvbuf, int count,
-                             MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Rsend_init_c(const void *buf, MPI_Count count, MPI_Datatype datatype,
+                                     int dest, int tag, MPI_Comm comm,
+                                     MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Scan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
+                             MPI_Op op, MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Scan_c(const void *sendbuf, void *recvbuf, MPI_Count count, MPI_Datatype datatype,
+                               MPI_Op op, MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Iscan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
+                              MPI_Op op, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Iscan_c(const void *sendbuf, void *recvbuf, MPI_Count count, MPI_Datatype datatype,
+                                MPI_Op op, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Scan_init(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
+                                  MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Scan_init_c(const void *sendbuf, void *recvbuf, MPI_Count count, MPI_Datatype datatype,
+                                    MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Scatter(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                                 void *recvbuf, int recvcount, MPI_Datatype recvtype,
                                 int root, MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Scatter_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                  void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                  int root, MPI_Comm comm);
 OMPI_DECLSPEC  int PMPI_Iscatter(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                                void *recvbuf, int recvcount, MPI_Datatype recvtype,
-                                int root, MPI_Comm comm, MPI_Request *request);
-OMPI_DECLSPEC  int PMPI_Scatterv(const void *sendbuf, const int sendcounts[], const int displs[],
-                                 MPI_Datatype sendtype, void *recvbuf, int recvcount,
-                                 MPI_Datatype recvtype, int root, MPI_Comm comm);
-OMPI_DECLSPEC  int PMPI_Iscatterv(const void *sendbuf, const int sendcounts[], const int displs[],
-                                 MPI_Datatype sendtype, void *recvbuf, int recvcount,
-                                 MPI_Datatype recvtype, int root, MPI_Comm comm, MPI_Request *request);
+                                 void *recvbuf, int recvcount, MPI_Datatype recvtype,
+                                 int root, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Iscatter_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                   void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                   int root, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Scatter_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
+                                     void *recvbuf, int recvcount, MPI_Datatype recvtype,
+                                     int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Scatter_init_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                       void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                       int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Scatterv(const void *sendbuf, const int sendcounts[], const int displs[], MPI_Datatype sendtype,
+                                 void *recvbuf, int recvcount, MPI_Datatype recvtype,
+                                 int root, MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Scatterv_c(const void *sendbuf, const MPI_Count sendcounts[], const MPI_Aint displs[], MPI_Datatype sendtype,
+                                   void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                   int root, MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Iscatterv(const void *sendbuf, const int sendcounts[], const int displs[], MPI_Datatype sendtype,
+                                  void *recvbuf, int recvcount, MPI_Datatype recvtype,
+                                  int root, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Iscatterv_c(const void *sendbuf, const MPI_Count sendcounts[], const MPI_Aint displs[], MPI_Datatype sendtype,
+                                    void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                    int root, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Scatterv_init(const void *sendbuf, const int sendcounts[], const int displs[], MPI_Datatype sendtype,
+                                      void *recvbuf, int recvcount, MPI_Datatype recvtype,
+                                      int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Scatterv_init_c(const void *sendbuf, const MPI_Count sendcounts[], const MPI_Aint displs[], MPI_Datatype sendtype,
+                                        void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
+                                        int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Send_init(const void *buf, int count, MPI_Datatype datatype,
                                   int dest, int tag, MPI_Comm comm,
                                   MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Send_init_c(const void *buf, MPI_Count count, MPI_Datatype datatype,
+                                    int dest, int tag, MPI_Comm comm,
+                                    MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Send(const void *buf, int count, MPI_Datatype datatype, int dest,
                              int tag, MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Send_c(const void *buf, MPI_Count count, MPI_Datatype datatype, int dest,
+                               int tag, MPI_Comm comm);
 OMPI_DECLSPEC  int PMPI_Sendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                                  int dest, int sendtag, void *recvbuf, int recvcount,
                                  MPI_Datatype recvtype, int source, int recvtag,
                                  MPI_Comm comm,  MPI_Status *status);
+OMPI_DECLSPEC  int PMPI_Sendrecv_c(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
+                                   int dest, int sendtag, void *recvbuf, MPI_Count recvcount,
+                                   MPI_Datatype recvtype, int source, int recvtag,
+                                   MPI_Comm comm,  MPI_Status *status);
 OMPI_DECLSPEC  int PMPI_Sendrecv_replace(void * buf, int count, MPI_Datatype datatype,
                                          int dest, int sendtag, int source, int recvtag,
                                          MPI_Comm comm, MPI_Status *status);
+OMPI_DECLSPEC  int PMPI_Sendrecv_replace_c(void * buf, MPI_Count count, MPI_Datatype datatype,
+                                           int dest, int sendtag, int source, int recvtag,
+                                           MPI_Comm comm, MPI_Status *status);
+OMPI_DECLSPEC  MPI_Fint PMPI_Session_c2f (const MPI_Session session);
+OMPI_DECLSPEC  int PMPI_Session_call_errhandler(MPI_Session session, int errorcode);
+OMPI_DECLSPEC  int PMPI_Session_create_errhandler (MPI_Session_errhandler_function *session_errhandler_fn,
+                                                   MPI_Errhandler *errhandler);
+OMPI_DECLSPEC  int PMPI_Session_finalize (MPI_Session *session);
+OMPI_DECLSPEC  int PMPI_Session_get_errhandler(MPI_Session session, MPI_Errhandler *erhandler);
+OMPI_DECLSPEC  int PMPI_Session_get_info (MPI_Session session, MPI_Info *info_used);
+OMPI_DECLSPEC  int PMPI_Session_get_num_psets (MPI_Session session, MPI_Info info, int *npset_names);
+OMPI_DECLSPEC  int PMPI_Session_get_nth_pset (MPI_Session session, MPI_Info info, int n, int *len, char *pset_name);
+OMPI_DECLSPEC  int PMPI_Session_get_pset_info (MPI_Session session, const char *pset_name, MPI_Info *info_used);
+OMPI_DECLSPEC  int PMPI_Session_init (MPI_Info info, MPI_Errhandler errhandler,
+                                      MPI_Session *session);
+OMPI_DECLSPEC  MPI_Session PMPI_Session_f2c (MPI_Fint session);
+OMPI_DECLSPEC  int PMPI_Session_set_errhandler(MPI_Session session, MPI_Errhandler errhandler);
+OMPI_DECLSPEC  int PMPI_Session_set_info (MPI_Session session, MPI_Info info);
 OMPI_DECLSPEC  int PMPI_Ssend_init(const void *buf, int count, MPI_Datatype datatype,
                                    int dest, int tag, MPI_Comm comm,
                                    MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Ssend_init_c(const void *buf, MPI_Count count, MPI_Datatype datatype,
+                                     int dest, int tag, MPI_Comm comm,
+                                     MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Ssend(const void *buf, int count, MPI_Datatype datatype, int dest,
                               int tag, MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Ssend_c(const void *buf, MPI_Count count, MPI_Datatype datatype, int dest,
+                                int tag, MPI_Comm comm);
 OMPI_DECLSPEC  int PMPI_Start(MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Startall(int count, MPI_Request array_of_requests[]);
 OMPI_DECLSPEC  int PMPI_Status_c2f(const MPI_Status *c_status, MPI_Fint *f_status);
 OMPI_DECLSPEC  int PMPI_Status_c2f08(const MPI_Status *c_status, MPI_F08_status *f08_status);
-OMPI_DECLSPEC  int PMPI_Status_f082f(const MPI_F08_status *f08_status, MPI_Fint *f_status);
 OMPI_DECLSPEC  int PMPI_Status_f082c(const MPI_F08_status *f08_status, MPI_Status *c_status);
+OMPI_DECLSPEC  int PMPI_Status_f082f(const MPI_F08_status *f08_status, MPI_Fint *f_status);
 OMPI_DECLSPEC  int PMPI_Status_f2c(const MPI_Fint *f_status, MPI_Status *c_status);
 OMPI_DECLSPEC  int PMPI_Status_f2f08(const MPI_Fint *f_status, MPI_F08_status *f08_status);
+OMPI_DECLSPEC  int PMPI_Status_get_error(const MPI_Status *status, int *error);
+OMPI_DECLSPEC  int PMPI_Status_get_source(const MPI_Status *status, int *source);
+OMPI_DECLSPEC  int PMPI_Status_get_tag(const MPI_Status *status, int *tag);
 OMPI_DECLSPEC  int PMPI_Status_set_cancelled(MPI_Status *status, int flag);
+OMPI_DECLSPEC  int PMPI_Status_set_error(MPI_Status *status, int error);
+OMPI_DECLSPEC  int PMPI_Status_set_source(MPI_Status *status, int source);
+OMPI_DECLSPEC  int PMPI_Status_set_tag(MPI_Status *status, int tag);
 OMPI_DECLSPEC  int PMPI_Status_set_elements(MPI_Status *status, MPI_Datatype datatype,
                                             int count);
+OMPI_DECLSPEC  int PMPI_Status_set_elements_c(MPI_Status *status, MPI_Datatype datatype,
+                                              MPI_Count count);
 OMPI_DECLSPEC  int PMPI_Status_set_elements_x(MPI_Status *status, MPI_Datatype datatype,
                                               MPI_Count count);
 OMPI_DECLSPEC  int PMPI_Testall(int count, MPI_Request array_of_requests[], int *flag,
                                 MPI_Status array_of_statuses[]);
-OMPI_DECLSPEC  int PMPI_Testany(int count, MPI_Request array_of_requests[], int *index, int *flag, MPI_Status *status);
+OMPI_DECLSPEC  int PMPI_Testany(int count, MPI_Request array_of_requests[], int *index,
+                                int *flag, MPI_Status *status);
 OMPI_DECLSPEC  int PMPI_Test(MPI_Request *request, int *flag, MPI_Status *status);
 OMPI_DECLSPEC  int PMPI_Test_cancelled(const MPI_Status *status, int *flag);
 OMPI_DECLSPEC  int PMPI_Testsome(int incount, MPI_Request array_of_requests[],
@@ -2477,41 +3548,72 @@ OMPI_DECLSPEC  MPI_Fint PMPI_Type_c2f(MPI_Datatype datatype);
 OMPI_DECLSPEC  int PMPI_Type_commit(MPI_Datatype *type);
 OMPI_DECLSPEC  int PMPI_Type_contiguous(int count, MPI_Datatype oldtype,
                                         MPI_Datatype *newtype);
+OMPI_DECLSPEC  int PMPI_Type_contiguous_c(MPI_Count count, MPI_Datatype oldtype,
+                                          MPI_Datatype *newtype);
 OMPI_DECLSPEC  int PMPI_Type_create_darray(int size, int rank, int ndims,
                                            const int gsize_array[], const int distrib_array[],
                                            const int darg_array[], const int psize_array[],
                                            int order, MPI_Datatype oldtype,
                                            MPI_Datatype *newtype);
+OMPI_DECLSPEC  int PMPI_Type_create_darray_c(int size, int rank, int ndims,
+                                             const MPI_Count gsize_array[], const int distrib_array[],
+                                             const int darg_array[], const int psize_array[],
+                                             int order, MPI_Datatype oldtype,
+                                             MPI_Datatype *newtype);
 OMPI_DECLSPEC  int PMPI_Type_create_f90_complex(int p, int r, MPI_Datatype *newtype);
 OMPI_DECLSPEC  int PMPI_Type_create_f90_integer(int r, MPI_Datatype *newtype);
 OMPI_DECLSPEC  int PMPI_Type_create_f90_real(int p, int r, MPI_Datatype *newtype);
+OMPI_DECLSPEC  int PMPI_Type_create_hindexed_block(int count, int blocklength,
+                                                   const MPI_Aint array_of_displacements[],
+                                                   MPI_Datatype oldtype,
+                                                   MPI_Datatype *newtype);
+OMPI_DECLSPEC  int PMPI_Type_create_hindexed_block_c(MPI_Count count, MPI_Count blocklength,
+                                                     const MPI_Count array_of_displacements[],
+                                                     MPI_Datatype oldtype,
+                                                     MPI_Datatype *newtype);
 OMPI_DECLSPEC  int PMPI_Type_create_hindexed(int count, const int array_of_blocklengths[],
                                              const MPI_Aint array_of_displacements[],
                                              MPI_Datatype oldtype,
                                              MPI_Datatype *newtype);
+OMPI_DECLSPEC  int PMPI_Type_create_hindexed_c(MPI_Count count, const MPI_Count array_of_blocklengths[],
+                                               const MPI_Count array_of_displacements[],
+                                               MPI_Datatype oldtype,
+                                               MPI_Datatype *newtype);
 OMPI_DECLSPEC  int PMPI_Type_create_hvector(int count, int blocklength, MPI_Aint stride,
                                             MPI_Datatype oldtype,
                                             MPI_Datatype *newtype);
+OMPI_DECLSPEC  int PMPI_Type_create_hvector_c(MPI_Count count, MPI_Count blocklength, MPI_Count stride,
+                                              MPI_Datatype oldtype,
+                                              MPI_Datatype *newtype);
 OMPI_DECLSPEC  int PMPI_Type_create_keyval(MPI_Type_copy_attr_function *type_copy_attr_fn,
                                            MPI_Type_delete_attr_function *type_delete_attr_fn,
                                            int *type_keyval, void *extra_state);
-OMPI_DECLSPEC  int PMPI_Type_create_hindexed_block(int count, int blocklength,
-                                                   const MPI_Aint array_of_displacements[],
-                                                   MPI_Datatype oldtype,
-                                                   MPI_Datatype *newtype);
 OMPI_DECLSPEC  int PMPI_Type_create_indexed_block(int count, int blocklength,
                                                   const int array_of_displacements[],
                                                   MPI_Datatype oldtype,
                                                   MPI_Datatype *newtype);
+OMPI_DECLSPEC  int PMPI_Type_create_indexed_block_c(MPI_Count count, MPI_Count blocklength,
+                                                    const MPI_Count array_of_displacements[],
+                                                    MPI_Datatype oldtype,
+                                                    MPI_Datatype *newtype);
 OMPI_DECLSPEC  int PMPI_Type_create_struct(int count, const int array_of_block_lengths[],
                                            const MPI_Aint array_of_displacements[],
                                            const MPI_Datatype array_of_types[],
                                            MPI_Datatype *newtype);
+OMPI_DECLSPEC  int PMPI_Type_create_struct_c(MPI_Count count, const MPI_Count array_of_block_lengths[],
+                                             const MPI_Count array_of_displacements[],
+                                             const MPI_Datatype array_of_types[],
+                                             MPI_Datatype *newtype);
 OMPI_DECLSPEC  int PMPI_Type_create_subarray(int ndims, const int size_array[], const int subsize_array[],
                                              const int start_array[], int order,
                                              MPI_Datatype oldtype, MPI_Datatype *newtype);
+OMPI_DECLSPEC  int PMPI_Type_create_subarray_c(int ndims, const MPI_Count size_array[], const MPI_Count subsize_array[],
+                                               const MPI_Count start_array[], int order,
+                                               MPI_Datatype oldtype, MPI_Datatype *newtype);
 OMPI_DECLSPEC  int PMPI_Type_create_resized(MPI_Datatype oldtype, MPI_Aint lb,
                                             MPI_Aint extent, MPI_Datatype *newtype);
+OMPI_DECLSPEC  int PMPI_Type_create_resized_c(MPI_Datatype oldtype, MPI_Count lb,
+                                              MPI_Count extent, MPI_Datatype *newtype);
 OMPI_DECLSPEC  int PMPI_Type_delete_attr(MPI_Datatype type, int type_keyval);
 OMPI_DECLSPEC  int PMPI_Type_dup(MPI_Datatype type, MPI_Datatype *newtype);
 OMPI_DECLSPEC  int PMPI_Type_free(MPI_Datatype *type);
@@ -2524,40 +3626,65 @@ OMPI_DECLSPEC  int PMPI_Type_get_contents(MPI_Datatype mtype, int max_integers,
                                           int array_of_integers[],
                                           MPI_Aint array_of_addresses[],
                                           MPI_Datatype array_of_datatypes[]);
+OMPI_DECLSPEC  int PMPI_Type_get_contents_c(MPI_Datatype mtype, MPI_Count max_integers,
+                                            MPI_Count max_addresses, MPI_Count max_large_counts,
+                                            MPI_Count max_datatypes,
+                                            int array_of_integers[],
+                                            MPI_Aint array_of_addresses[],
+                                            MPI_Count array_of_large_counts[],
+                                            MPI_Datatype array_of_datatypes[]);
 OMPI_DECLSPEC  int PMPI_Type_get_envelope(MPI_Datatype type, int *num_integers,
                                           int *num_addresses, int *num_datatypes,
                                           int *combiner);
+OMPI_DECLSPEC  int PMPI_Type_get_envelope_c(MPI_Datatype type, MPI_Count *num_integers,
+                                          MPI_Count *num_addresses, MPI_Count *num_large_counts, 
+        				  MPI_Count *num_datatypes, int *combiner);
 OMPI_DECLSPEC  int PMPI_Type_get_extent(MPI_Datatype type, MPI_Aint *lb,
                                         MPI_Aint *extent);
+OMPI_DECLSPEC  int PMPI_Type_get_extent_c(MPI_Datatype type, MPI_Count *lb,
+                                          MPI_Count *extent);
 OMPI_DECLSPEC  int PMPI_Type_get_extent_x(MPI_Datatype type, MPI_Count *lb,
                                           MPI_Count *extent);
 OMPI_DECLSPEC  int PMPI_Type_get_name(MPI_Datatype type, char *type_name,
                                       int *resultlen);
 OMPI_DECLSPEC  int PMPI_Type_get_true_extent(MPI_Datatype datatype, MPI_Aint *true_lb,
                                              MPI_Aint *true_extent);
+OMPI_DECLSPEC  int PMPI_Type_get_true_extent_c(MPI_Datatype datatype, MPI_Count *true_lb,
+                                              MPI_Count *true_extent);
 OMPI_DECLSPEC  int PMPI_Type_get_true_extent_x(MPI_Datatype datatype, MPI_Count *true_lb,
                                                MPI_Count *true_extent);
 OMPI_DECLSPEC  int PMPI_Type_indexed(int count, const int array_of_blocklengths[],
                                      const int array_of_displacements[],
                                      MPI_Datatype oldtype, MPI_Datatype *newtype);
+OMPI_DECLSPEC  int PMPI_Type_indexed_c(MPI_Count count, const MPI_Count array_of_blocklengths[],
+                                       const MPI_Count array_of_displacements[],
+                                       MPI_Datatype oldtype, MPI_Datatype *newtype);
 OMPI_DECLSPEC  int PMPI_Type_match_size(int typeclass, int size, MPI_Datatype *type);
 OMPI_DECLSPEC  int PMPI_Type_set_attr(MPI_Datatype type, int type_keyval,
                                       void *attr_val);
 OMPI_DECLSPEC  int PMPI_Type_set_name(MPI_Datatype type, const char *type_name);
 OMPI_DECLSPEC  int PMPI_Type_size(MPI_Datatype type, int *size);
+OMPI_DECLSPEC  int PMPI_Type_size_c(MPI_Datatype type, MPI_Count *size);
 OMPI_DECLSPEC  int PMPI_Type_size_x(MPI_Datatype type, MPI_Count *size);
 OMPI_DECLSPEC  int PMPI_Type_vector(int count, int blocklength, int stride,
                                     MPI_Datatype oldtype, MPI_Datatype *newtype);
+OMPI_DECLSPEC  int PMPI_Type_vector_c(MPI_Count count, MPI_Count blocklength, MPI_Count stride,
+                                      MPI_Datatype oldtype, MPI_Datatype *newtype);
 OMPI_DECLSPEC  int PMPI_Unpack(const void *inbuf, int insize, int *position,
                                void *outbuf, int outcount, MPI_Datatype datatype,
                                MPI_Comm comm);
-OMPI_DECLSPEC  int PMPI_Unpublish_name(const char *service_name, MPI_Info info,
-                                       const char *port_name);
+OMPI_DECLSPEC  int PMPI_Unpack_c(const void *inbuf, MPI_Count insize, MPI_Count *position,
+                                 void *outbuf, MPI_Count outcount, MPI_Datatype datatype,
+                                 MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Unpublish_name(const char *service_name, MPI_Info info, const char *port_name);
 OMPI_DECLSPEC  int PMPI_Unpack_external (const char datarep[], const void *inbuf, MPI_Aint insize,
                                          MPI_Aint *position, void *outbuf, int outcount,
                                          MPI_Datatype datatype);
+OMPI_DECLSPEC  int PMPI_Unpack_external_c (const char datarep[], const void *inbuf, MPI_Count insize,
+                                           MPI_Count *position, void *outbuf, MPI_Count outcount,
+                                           MPI_Datatype datatype);
 OMPI_DECLSPEC  int PMPI_Waitall(int count, MPI_Request array_of_requests[],
-                                MPI_Status array_of_statuses[]);
+                                MPI_Status *array_of_statuses);
 OMPI_DECLSPEC  int PMPI_Waitany(int count, MPI_Request array_of_requests[],
                                 int *index, MPI_Status *status);
 OMPI_DECLSPEC  int PMPI_Wait(MPI_Request *request, MPI_Status *status);
@@ -2566,14 +3693,20 @@ OMPI_DECLSPEC  int PMPI_Waitsome(int incount, MPI_Request array_of_requests[],
                                  MPI_Status array_of_statuses[]);
 OMPI_DECLSPEC  int PMPI_Win_allocate(MPI_Aint size, int disp_unit, MPI_Info info,
                                      MPI_Comm comm, void *baseptr, MPI_Win *win);
+OMPI_DECLSPEC  int PMPI_Win_allocate_c(MPI_Aint size, MPI_Aint disp_unit, MPI_Info info,
+                                       MPI_Comm comm, void *baseptr, MPI_Win *win);
 OMPI_DECLSPEC  int PMPI_Win_allocate_shared(MPI_Aint size, int disp_unit, MPI_Info info,
                                             MPI_Comm comm, void *baseptr, MPI_Win *win);
+OMPI_DECLSPEC  int PMPI_Win_allocate_shared_c(MPI_Aint size, MPI_Aint disp_unit, MPI_Info info,
+                                              MPI_Comm comm, void *baseptr, MPI_Win *win);
 OMPI_DECLSPEC  int PMPI_Win_attach(MPI_Win win, void *base, MPI_Aint size);
 OMPI_DECLSPEC  MPI_Fint PMPI_Win_c2f(MPI_Win win);
 OMPI_DECLSPEC  int PMPI_Win_call_errhandler(MPI_Win win, int errorcode);
 OMPI_DECLSPEC  int PMPI_Win_complete(MPI_Win win);
 OMPI_DECLSPEC  int PMPI_Win_create(void *base, MPI_Aint size, int disp_unit,
                                    MPI_Info info, MPI_Comm comm, MPI_Win *win);
+OMPI_DECLSPEC  int PMPI_Win_create_c(void *base, MPI_Aint size, MPI_Aint disp_unit,
+                                     MPI_Info info, MPI_Comm comm, MPI_Win *win);
 OMPI_DECLSPEC  int PMPI_Win_create_dynamic(MPI_Info info, MPI_Comm comm, MPI_Win *win);
 OMPI_DECLSPEC  int PMPI_Win_create_errhandler(MPI_Win_errhandler_function *function,
                                               MPI_Errhandler *errhandler);
@@ -2583,7 +3716,7 @@ OMPI_DECLSPEC  int PMPI_Win_create_keyval(MPI_Win_copy_attr_function *win_copy_a
 OMPI_DECLSPEC  int PMPI_Win_delete_attr(MPI_Win win, int win_keyval);
 OMPI_DECLSPEC  int PMPI_Win_detach(MPI_Win win, const void *base);
 OMPI_DECLSPEC  MPI_Win PMPI_Win_f2c(MPI_Fint win);
-OMPI_DECLSPEC  int PMPI_Win_fence(int assert, MPI_Win win);
+OMPI_DECLSPEC  int PMPI_Win_fence(int mpi_assert, MPI_Win win);
 OMPI_DECLSPEC  int PMPI_Win_flush(int rank, MPI_Win win);
 OMPI_DECLSPEC  int PMPI_Win_flush_all(MPI_Win win);
 OMPI_DECLSPEC  int PMPI_Win_flush_local(int rank, MPI_Win win);
@@ -2596,15 +3729,16 @@ OMPI_DECLSPEC  int PMPI_Win_get_errhandler(MPI_Win win, MPI_Errhandler *errhandl
 OMPI_DECLSPEC  int PMPI_Win_get_group(MPI_Win win, MPI_Group *group);
 OMPI_DECLSPEC  int PMPI_Win_get_info(MPI_Win win, MPI_Info *info_used);
 OMPI_DECLSPEC  int PMPI_Win_get_name(MPI_Win win, char *win_name, int *resultlen);
-OMPI_DECLSPEC  int PMPI_Win_lock(int lock_type, int rank, int assert, MPI_Win win);
-OMPI_DECLSPEC  int PMPI_Win_lock_all(int assert, MPI_Win win);
-OMPI_DECLSPEC  int PMPI_Win_post(MPI_Group group, int assert, MPI_Win win);
+OMPI_DECLSPEC  int PMPI_Win_lock(int lock_type, int rank, int mpi_assert, MPI_Win win);
+OMPI_DECLSPEC  int PMPI_Win_lock_all(int mpi_assert, MPI_Win win);
+OMPI_DECLSPEC  int PMPI_Win_post(MPI_Group group, int mpi_assert, MPI_Win win);
 OMPI_DECLSPEC  int PMPI_Win_set_attr(MPI_Win win, int win_keyval, void *attribute_val);
 OMPI_DECLSPEC  int PMPI_Win_set_errhandler(MPI_Win win, MPI_Errhandler errhandler);
 OMPI_DECLSPEC  int PMPI_Win_set_info(MPI_Win win, MPI_Info info);
 OMPI_DECLSPEC  int PMPI_Win_set_name(MPI_Win win, const char *win_name);
 OMPI_DECLSPEC  int PMPI_Win_shared_query(MPI_Win win, int rank, MPI_Aint *size, int *disp_unit, void *baseptr);
-OMPI_DECLSPEC  int PMPI_Win_start(MPI_Group group, int assert, MPI_Win win);
+OMPI_DECLSPEC  int PMPI_Win_shared_query_c(MPI_Win win, int rank, MPI_Aint *size, MPI_Aint *disp_unit, void *baseptr);
+OMPI_DECLSPEC  int PMPI_Win_start(MPI_Group group, int mpi_assert, MPI_Win win);
 OMPI_DECLSPEC  int PMPI_Win_sync(MPI_Win win);
 OMPI_DECLSPEC  int PMPI_Win_test(MPI_Win win, int *flag);
 OMPI_DECLSPEC  int PMPI_Win_unlock(int rank, MPI_Win win);
@@ -2626,11 +3760,13 @@ OMPI_DECLSPEC  int PMPI_T_cvar_handle_free (MPI_T_cvar_handle *handle);
 OMPI_DECLSPEC  int PMPI_T_cvar_read (MPI_T_cvar_handle handle, void *buf);
 OMPI_DECLSPEC  int PMPI_T_cvar_write (MPI_T_cvar_handle handle, const void *buf);
 OMPI_DECLSPEC  int PMPI_T_category_get_num(int *num_cat);
+OMPI_DECLSPEC  int PMPI_T_category_get_num_events (int cat_index, int *num_events);
 OMPI_DECLSPEC  int PMPI_T_category_get_info(int cat_index, char *name, int *name_len,
                                             char *desc, int *desc_len, int *num_cvars,
                                             int *num_pvars, int *num_categories);
 OMPI_DECLSPEC  int PMPI_T_category_get_index (const char *name, int *category_index);
 OMPI_DECLSPEC  int PMPI_T_category_get_cvars(int cat_index, int len, int indices[]);
+OMPI_DECLSPEC  int PMPI_T_category_get_events(int cat_index, int len, int indices[]);
 OMPI_DECLSPEC  int PMPI_T_category_get_pvars(int cat_index, int len, int indices[]);
 OMPI_DECLSPEC  int PMPI_T_category_get_categories(int cat_index, int len, int indices[]);
 OMPI_DECLSPEC  int PMPI_T_category_changed(int *stamp);
@@ -2659,6 +3795,41 @@ OMPI_DECLSPEC  int PMPI_T_enum_get_info(MPI_T_enum enumtype, int *num, char *nam
 OMPI_DECLSPEC  int PMPI_T_enum_get_item(MPI_T_enum enumtype, int index, int *value, char *name,
                                         int *name_len);
 
+OMPI_DECLSPEC  int PMPI_T_event_get_num (int *num_events);
+OMPI_DECLSPEC  int PMPI_T_event_get_info (int event_index, char *name, int *name_len,
+                                          int *verbosity, MPI_Datatype *array_of_datatypes,
+                                          MPI_Aint *array_of_displacements, int *num_elements,
+                                          MPI_T_enum *enumtype, MPI_Info *info,
+                                          char *desc, int *desc_len, int *bind);
+OMPI_DECLSPEC  int PMPI_T_event_get_index (const char *name, int *event_index);
+OMPI_DECLSPEC  int PMPI_T_event_handle_alloc (int event_index, void *obj_handle,
+                                              MPI_Info info, MPI_T_event_registration *event_registration);
+OMPI_DECLSPEC  int PMPI_T_event_handle_set_info (MPI_T_event_registration event_registration, MPI_Info info);
+OMPI_DECLSPEC  int PMPI_T_event_handle_get_info (MPI_T_event_registration event_registration,
+                                                    MPI_Info *info_used);
+OMPI_DECLSPEC  int PMPI_T_event_register_callback (MPI_T_event_registration event_registration,
+                                                   MPI_T_cb_safety cb_safety, MPI_Info info, void *user_data,
+                                                   MPI_T_event_cb_function event_cb_function);
+OMPI_DECLSPEC  int PMPI_T_event_callback_set_info (MPI_T_event_registration event_registration,
+                                                    MPI_T_cb_safety cb_safety, MPI_Info info);
+OMPI_DECLSPEC  int PMPI_T_event_callback_get_info (MPI_T_event_registration event_registration,
+                                                   MPI_T_cb_safety cb_safety, MPI_Info *info_used);
+OMPI_DECLSPEC  int PMPI_T_event_handle_free (MPI_T_event_registration event_registration,
+                                             void *user_data,
+                                             MPI_T_event_free_cb_function free_cb_function);
+OMPI_DECLSPEC  int PMPI_T_event_set_dropped_handler (MPI_T_event_registration handle,
+                                                     MPI_T_event_dropped_cb_function dropped_cb_function);
+OMPI_DECLSPEC  int PMPI_T_event_read (MPI_T_event_instance event, int element_index, void *buffer);
+OMPI_DECLSPEC  int PMPI_T_event_copy (MPI_T_event_instance event, void *buffer);
+OMPI_DECLSPEC  int PMPI_T_event_get_timestamp (MPI_T_event_instance event, MPI_Count *event_time);
+OMPI_DECLSPEC  int PMPI_T_event_get_source (MPI_T_event_instance event, int *source_index);
+OMPI_DECLSPEC  int PMPI_T_source_get_num (int *num_source);
+OMPI_DECLSPEC  int PMPI_T_source_get_info (int source_id, char *name, int *name_len,
+                                           char *desc, int *desc_len, MPI_T_source_order *ordering,
+                                           MPI_Count *ticks_per_second, MPI_Count *max_timestamp,
+                                           MPI_Info *info);
+OMPI_DECLSPEC  int PMPI_T_source_get_timestamp (int source_id, MPI_Count *timestamp);
+
   /*
    * Tool MPI API
    */
@@ -2676,11 +3847,13 @@ OMPI_DECLSPEC  int MPI_T_cvar_handle_free (MPI_T_cvar_handle *handle);
 OMPI_DECLSPEC  int MPI_T_cvar_read (MPI_T_cvar_handle handle, void *buf);
 OMPI_DECLSPEC  int MPI_T_cvar_write (MPI_T_cvar_handle handle, const void *buf);
 OMPI_DECLSPEC  int MPI_T_category_get_num(int *num_cat);
+OMPI_DECLSPEC  int MPI_T_category_get_num_events (int cat_index, int *num_events);
 OMPI_DECLSPEC  int MPI_T_category_get_info(int cat_index, char *name, int *name_len,
                                            char *desc, int *desc_len, int *num_cvars,
                                            int *num_pvars, int *num_categories);
 OMPI_DECLSPEC  int MPI_T_category_get_index (const char *name, int *category_index);
 OMPI_DECLSPEC  int MPI_T_category_get_cvars(int cat_index, int len, int indices[]);
+OMPI_DECLSPEC  int MPI_T_category_get_events(int cat_index, int len, int indices[]);
 OMPI_DECLSPEC  int MPI_T_category_get_pvars(int cat_index, int len, int indices[]);
 OMPI_DECLSPEC  int MPI_T_category_get_categories(int cat_index, int len, int indices[]);
 OMPI_DECLSPEC  int MPI_T_category_changed(int *stamp);
@@ -2723,7 +3896,56 @@ OMPI_DECLSPEC  int PMPI_Attr_get(MPI_Comm comm, int keyval, void *attribute_val,
 OMPI_DECLSPEC  int MPI_Attr_put(MPI_Comm comm, int keyval, void *attribute_val)
             __mpi_interface_deprecated__("MPI_Attr_put was deprecated in MPI-2.0; use MPI_Comm_set_attr instead");
 OMPI_DECLSPEC  int PMPI_Attr_put(MPI_Comm comm, int keyval, void *attribute_val)
-            __mpi_interface_deprecated__("PMPI_Attr_put was deprecated in MPI-2.0; use PMPI_Comm_set_attr instead");
+    __mpi_interface_deprecated__("PMPI_Attr_put was deprecated in MPI-2.0; use PMPI_Comm_set_attr instead");
+
+OMPI_DECLSPEC  int MPI_Info_get(MPI_Info info, const char *key, int valuelen,
+                                char *value, int *flag)
+            __mpi_interface_deprecated_in_mpi40__("MPI_Info_get was deprecated in MPI-4.0; use MPI_Info_get_string instead");
+OMPI_DECLSPEC  int PMPI_Info_get(MPI_Info info, const char *key, int valuelen,
+                                 char *value, int *flag)
+            __mpi_interface_deprecated_in_mpi40__("PMPI_Info_get was deprecated in MPI-4.0; use PMPI_Info_get_string instead");
+OMPI_DECLSPEC  int MPI_Info_get_valuelen(MPI_Info info, const char *key, int *valuelen,
+                                         int *flag)
+            __mpi_interface_deprecated_in_mpi40__("MPI_Info_get_valuelen was deprecated in MPI-4.0; use MPI_Info_get_string instead");
+OMPI_DECLSPEC  int PMPI_Info_get_valuelen(MPI_Info info, const char *key, int *valuelen,
+                                          int *flag)
+            __mpi_interface_deprecated_in_mpi40__("PMPI_Info_get_valuelen was deprecated in MPI-4.0; use PMPI_Info_get_string instead");
+
+OMPI_DECLSPEC  int MPI_T_event_get_num (int *num_events);
+OMPI_DECLSPEC  int MPI_T_event_get_info (int event_index, char *name, int *name_len,
+                                         int *verbosity, MPI_Datatype *array_of_datatypes,
+                                         MPI_Aint *array_of_displacements, int *num_elements,
+                                         MPI_T_enum *enumtype, MPI_Info *info,
+                                         char *desc, int *desc_len, int *bind);
+OMPI_DECLSPEC  int MPI_T_event_get_index (const char *name, int *event_index);
+OMPI_DECLSPEC  int MPI_T_event_handle_alloc (int event_index, void *obj_handle,
+                                             MPI_Info info, MPI_T_event_registration *event_registration);
+OMPI_DECLSPEC  int MPI_T_event_handle_set_info (MPI_T_event_registration event_registration, MPI_Info info);
+OMPI_DECLSPEC  int MPI_T_event_handle_get_info (MPI_T_event_registration event_registration,
+                                                MPI_Info *info_used);
+OMPI_DECLSPEC  int MPI_T_event_handle_free (MPI_T_event_registration event_registration,
+                                             void *user_data,
+                                            MPI_T_event_free_cb_function free_cb_function);
+OMPI_DECLSPEC  int MPI_T_event_register_callback (MPI_T_event_registration event_registration,
+                                                  MPI_T_cb_safety cb_safety, MPI_Info info, void *user_data,
+                                                  MPI_T_event_cb_function event_cb_function);
+OMPI_DECLSPEC  int MPI_T_event_callback_set_info (MPI_T_event_registration event_registration,
+                                                  MPI_T_cb_safety cb_safety, MPI_Info info);
+OMPI_DECLSPEC  int MPI_T_event_callback_get_info (MPI_T_event_registration event_registration,
+                                                  MPI_T_cb_safety cb_safety, MPI_Info *info_used);
+OMPI_DECLSPEC  int MPI_T_event_set_dropped_handler (MPI_T_event_registration handle,
+                                                    MPI_T_event_dropped_cb_function dropped_cb_function);
+OMPI_DECLSPEC  int MPI_T_event_read (MPI_T_event_instance event, int element_index, void *buffer);
+OMPI_DECLSPEC  int MPI_T_event_copy (MPI_T_event_instance event, void *buffer);
+OMPI_DECLSPEC  int MPI_T_event_get_timestamp (MPI_T_event_instance event, MPI_Count *event_time);
+OMPI_DECLSPEC  int MPI_T_event_get_source (MPI_T_event_instance event, int *source_index);
+
+OMPI_DECLSPEC  int MPI_T_source_get_num (int *num_source);
+OMPI_DECLSPEC  int MPI_T_source_get_info (int source_id, char *name, int *name_len,
+                                          char *desc, int *desc_len, MPI_T_source_order *ordering,
+                                          MPI_Count *ticks_per_second, MPI_Count *max_timestamp,
+                                          MPI_Info *info);
+OMPI_DECLSPEC  int MPI_T_source_get_timestamp (int source_id, MPI_Count *timestamp);
 
 /*
  * Even though MPI_Copy_function and MPI_Delete_function are
@@ -2777,7 +3999,7 @@ OMPI_DECLSPEC int OMPI_C_MPI_NULL_DELETE_FN( MPI_Comm comm, int comm_keyval,
                                              void* extra_state )
             __mpi_interface_deprecated__("MPI_NULL_DELETE_FN was deprecated in MPI-2.0; use MPI_COMM_NULL_DELETE_FN instead.");
 
-#if !OMPI_OMIT_MPI1_COMPAT_DECLS
+#if (!OMPI_OMIT_MPI1_COMPAT_DECLS)
 /*
  * Removed typedefs.  These typedefs are only available if Open MPI
  * was configured with --enable-mpi1-compatibility.
diff --git a/ompi/include/mpif-config.h.in b/ompi/include/mpif-config.h.in
index a3a6d7b0c1e..fc2054df637 100644
--- a/ompi/include/mpif-config.h.in
+++ b/ompi/include/mpif-config.h.in
@@ -13,6 +13,8 @@
 ! Copyright (c) 2006-2017 Cisco Systems, Inc.  All rights reserved
 ! Copyright (c) 2013      Los Alamos National Security, LLC. All rights
 !                         reserved.
+! Copyright (c) 2019      Triad National Security, LLC. All rights
+!                         reserved.
 ! $COPYRIGHT$
 !
 ! Additional copyrights may follow
@@ -60,6 +62,8 @@
       integer MPI_MAX_INFO_VAL
       integer MPI_MAX_PORT_NAME
       integer MPI_MAX_DATAREP_STRING
+      integer MPI_MAX_PSET_NAME_LEN
+      integer MPI_MAX_STRINGTAG_LEN
       parameter (MPI_MAX_PROCESSOR_NAME=@OPAL_MAX_PROCESSOR_NAME@-1)
       parameter (MPI_MAX_ERROR_STRING=@OPAL_MAX_ERROR_STRING@-1)
       parameter (MPI_MAX_OBJECT_NAME=@OPAL_MAX_OBJECT_NAME@-1)
@@ -68,6 +72,8 @@
       parameter (MPI_MAX_INFO_VAL=@OPAL_MAX_INFO_VAL@-1)
       parameter (MPI_MAX_PORT_NAME=@OPAL_MAX_PORT_NAME@-1)
       parameter (MPI_MAX_DATAREP_STRING=@OPAL_MAX_DATAREP_STRING@-1)
+      parameter (MPI_MAX_PSET_NAME_LEN=@OPAL_MAX_PSET_NAME_LEN@-1)
+      parameter (MPI_MAX_STRINGTAG_LEN=@OPAL_MAX_STRINGTAG_LEN@-1)
 
 !
 ! MPI F08 conformance
diff --git a/ompi/include/mpif-sentinels.h b/ompi/include/mpif-sentinels.h
index 082154cdbbc..99384d072eb 100644
--- a/ompi/include/mpif-sentinels.h
+++ b/ompi/include/mpif-sentinels.h
@@ -36,7 +36,7 @@
 !     MPI_IN_PLACE has the same rationale as MPI_BOTTOM.
       integer MPI_IN_PLACE
 !     Making MPI_ARGV_NULL be the same type as the parameter that is
-!     exepected in the F90 binding for MPI_COMM_SPAWN means that we
+!     expected in the F90 binding for MPI_COMM_SPAWN means that we
 !     don't need another interface for MPI_COMM_SPAWN.
       character MPI_ARGV_NULL(1)
 !     Ditto for MPI_ARGVS_NULL / MPI_COMM_SPAWN_MULTIPLE.
diff --git a/ompi/include/mpif-values.pl b/ompi/include/mpif-values.pl
index b5334f06355..21f69530cde 100755
--- a/ompi/include/mpif-values.pl
+++ b/ompi/include/mpif-values.pl
@@ -7,6 +7,7 @@
 # Copyright (c) 2020      The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
+# Copyright (c) 2022      IBM Corporation.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -70,6 +71,30 @@ sub write_file {
 
 #----------------------------------------------------------------------------
 
+# Read a value for a specified key from the file specified.
+
+sub read_value_from_file {
+    my ($filename, $key) = @_;
+    my $value;
+
+    open(FILE_IN, $filename) || die "Couldn't open $filename";
+    while(my $line = <FILE_IN>) {
+        if( $line =~ /^$key=(.+)/ ) {
+            $value = $1;
+            last;
+        }
+    }
+    close(FILE_IN);
+
+    if(!defined($value)) {
+        die "Did not find the string \"$key\" in the file $filename"
+    }
+
+    return $value;
+}
+
+#----------------------------------------------------------------------------
+
 print "creating Fortran header files (with common constants)...\n";
 
 # Find the OMPI topdir.  It is likely the pwd.
@@ -123,6 +148,7 @@ sub write_file {
 $handles->{MPI_REQUEST_NULL} = 0;
 $handles->{MPI_WIN_NULL} = 0;
 $handles->{MPI_MESSAGE_NULL} = 0;
+$handles->{MPI_SESSION_NULL} = 0;
 
 $handles->{MPI_BYTE} =  1;
 $handles->{MPI_PACKED} =  2;
@@ -215,8 +241,8 @@ sub write_file {
 
 my $constants;
 
-$constants->{MPI_VERSION} = 3;
-$constants->{MPI_SUBVERSION} = 1;
+$constants->{MPI_VERSION} = read_value_from_file("$topdir/VERSION", "mpi_standard_version");
+$constants->{MPI_SUBVERSION} = read_value_from_file("$topdir/VERSION", "mpi_standard_subversion");
 
 $constants->{MPI_ANY_SOURCE} = -1;
 $constants->{MPI_ANY_TAG} = -1;
@@ -354,6 +380,8 @@ sub write_file {
 $constants->{MPI_ERR_RMA_FLAVOR} = 70;
 $constants->{MPI_ERR_RMA_SHARED} = 71;
 $constants->{MPI_T_ERR_INVALID} = 72;
+$constants->{MPI_ERR_SESSION} = 78;
+$constants->{MPI_ERR_VALUE_TOO_LARGE} = 79;
 $constants->{MPI_ERR_LASTCODE} = 92;
 
 $constants->{MPI_IDENT} = 0;
@@ -394,6 +422,8 @@ sub write_file {
 $constants->{OMPI_COMM_TYPE_HOST} = 9;
 $constants->{OMPI_COMM_TYPE_CU} = 10;
 $constants->{OMPI_COMM_TYPE_CLUSTER} = 11;
+$constants->{MPI_COMM_TYPE_HW_UNGUIDED} = 12;
+$constants->{MPI_COMM_TYPE_HW_GUIDED} = 13;
 
 #----------------------------------------------------------------------------
 
diff --git a/ompi/include/ompi/constants.h b/ompi/include/ompi/constants.h
index 34826b8fd93..c545ae7a596 100644
--- a/ompi/include/ompi/constants.h
+++ b/ompi/include/ompi/constants.h
@@ -27,7 +27,7 @@
 
 /* error codes */
 enum {
-    /* Error codes inherited from ORTE/OPAL.  Still enum values so
+    /* Error codes inherited from OPAL.  Still enum values so
        that we might get nice debugger help */
     OMPI_SUCCESS                    = OPAL_SUCCESS,
     OMPI_ERROR                      = OPAL_ERROR,
@@ -62,6 +62,7 @@ enum {
     OMPI_ERR_BUFFER                 = OPAL_ERR_BUFFER,
     OMPI_ERR_SILENT                 = OPAL_ERR_SILENT,
     OMPI_ERR_HANDLERS_COMPLETE      = OPAL_ERR_HANDLERS_COMPLETE,
+    OMPI_ERR_NOT_BOUND              = OPAL_ERR_NOT_BOUND,
 
     OMPI_ERR_REQUEST                = OMPI_ERR_BASE - 1,
     OMPI_ERR_RMA_SYNC               = OMPI_ERR_BASE - 2,
diff --git a/ompi/include/ompi/memchecker.h b/ompi/include/ompi/memchecker.h
index cbd4d6eb028..15b7111877f 100644
--- a/ompi/include/ompi/memchecker.h
+++ b/ompi/include/ompi/memchecker.h
@@ -1,7 +1,7 @@
 /*
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
  *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2010-2017 The University of Tennessee and The University
+ * Copyright (c) 2010-2023 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2009      Oak Ridge National Labs.  All rights reserved.
@@ -55,7 +55,7 @@ static inline int memchecker_convertor_call (int (*f)(void *, size_t), opal_conv
         /*  We have a contiguous type. */
         f( (void *)pConvertor->pBaseBuf , pConvertor->local_size);
     } else {
-        /* Now we got a noncontigous data. */
+        /* Now we got a non-contiguous data. */
         uint32_t elem_pos = 0, i;
         ptrdiff_t  stack_disp  = 0;
         dt_elem_desc_t*  description = pConvertor->use_desc->desc;
@@ -109,7 +109,7 @@ static inline int memchecker_call (int (*f)(void *, size_t), const void * addr,
         /*  We have a contiguous type. */
         f( (void*)((char *)addr+datatype->super.true_lb), datatype->super.size * count );
     } else {
-        /* Now we got a noncontigous type. */
+        /* Now we got a non-contiguous type. */
         uint32_t         elem_pos = 0, i;
         ptrdiff_t        stack_disp  = 0;
         dt_elem_desc_t*  description = datatype->super.opt_desc.desc;
@@ -220,8 +220,6 @@ static inline int memchecker_comm(MPI_Comm comm)
     opal_memchecker_base_isdefined (&comm->c_name, MPI_MAX_OBJECT_NAME);
     opal_memchecker_base_isdefined (&comm->c_my_rank, sizeof(int));
     opal_memchecker_base_isdefined (&comm->c_flags, sizeof(uint32_t));
-    opal_memchecker_base_isdefined (&comm->c_id_available, sizeof(int));
-    opal_memchecker_base_isdefined (&comm->c_id_start_index, sizeof(int));
     opal_memchecker_base_isdefined (&comm->c_local_group, sizeof(ompi_group_t *));
     opal_memchecker_base_isdefined (&comm->c_remote_group, sizeof(ompi_group_t *));
     opal_memchecker_base_isdefined (&comm->c_keyhash, sizeof(struct opal_hash_table_t *));
@@ -308,7 +306,7 @@ static inline int memchecker_request(MPI_Request *request)
     opal_memchecker_base_isdefined (&(*request)->req_status._ucount, sizeof(size_t));
 #endif
 
-    opal_memchecker_base_isdefined ((void*)&(*request)->req_complete, sizeof(volatile _Bool));
+    opal_memchecker_base_isdefined ((void*)&(*request)->req_complete, sizeof(volatile void*));
     opal_memchecker_base_isdefined ((void*)&(*request)->req_state, sizeof(volatile ompi_request_state_t));
     opal_memchecker_base_isdefined (&(*request)->req_persistent, sizeof(_Bool));
     opal_memchecker_base_isdefined (&(*request)->req_f_to_c_index, sizeof(int));
diff --git a/ompi/include/ompi_config.h b/ompi/include/ompi_config.h
index a7a2c1fa085..b8b5ad38098 100644
--- a/ompi/include/ompi_config.h
+++ b/ompi/include/ompi_config.h
@@ -10,7 +10,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2007-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2007-2022 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2007      Sun Microsystems, Inc.  All rights reserved.
  * Copyright (c) 2014      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -44,16 +44,17 @@
 #    ifndef OMPI_DECLSPEC
 #      define OMPI_DECLSPEC            __opal_attribute_visibility__("default")
 #    endif
-#    ifndef OMPI_MODULE_DECLSPEC
-#      define OMPI_MODULE_DECLSPEC     __opal_attribute_visibility__("default")
+#    ifndef OMPI_HIDDEN
+#      define OMPI_HIDDEN              __opal_attribute_visibility__("hidden")
 #    endif
 #  else
 #    ifndef OMPI_DECLSPEC
 #      define OMPI_DECLSPEC
 #    endif
-#    ifndef OMPI_MODULE_DECLSPEC
-#      define OMPI_MODULE_DECLSPEC
+#    ifndef OMPI_HIDDEN
+#      define OMPI_HIDDEN
 #    endif
 #  endif
 
+
 #endif
diff --git a/ompi/info/Makefile.am b/ompi/info/Makefile.am
index e4af170dcf8..171d7877185 100644
--- a/ompi/info/Makefile.am
+++ b/ompi/info/Makefile.am
@@ -21,7 +21,9 @@
 # This makefile.am does not stand on its own - it is included from ompi/Makefile.am
 
 headers += \
-        info/info.h
+        info/info.h                 \
+        info/info_memkind.h
 
 lib@OMPI_LIBMPI_NAME@_la_SOURCES += \
-        info/info.c
+        info/info.c                 \
+        info/info_memkind.c
diff --git a/ompi/info/info.c b/ompi/info/info.c
index ba51bdc2d39..15b4d50033f 100644
--- a/ompi/info/info.c
+++ b/ompi/info/info.c
@@ -17,7 +17,7 @@
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
- * Copyright (c) 2019      Triad National Security, LLC. All rights
+ * Copyright (c) 2019-2021 Triad National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2020      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
@@ -51,9 +51,11 @@
 #include "opal/util/info.h"
 
 #include "ompi/info/info.h"
+#include "ompi/info/info_memkind.h"
 #include "ompi/runtime/mpiruntime.h"
 #include "ompi/runtime/params.h"
 #include "ompi/runtime/ompi_rte.h"
+#include "ompi/instance/instance.h"
 
 /*
  * Global variables
@@ -86,9 +88,9 @@ opal_pointer_array_t ompi_info_f_to_c_table = {{0}};
  * fortran to C translation table. It also fills in the values
  * for the MPI_INFO_GET_ENV object
  */
+
 int ompi_mpiinfo_init(void)
 {
-    char *cptr, **tmp;
 
     /* initialize table */
 
@@ -102,56 +104,87 @@ int ompi_mpiinfo_init(void)
     OBJ_CONSTRUCT(&ompi_mpi_info_null.info, ompi_info_t);
     assert(ompi_mpi_info_null.info.i_f_to_c_index == 0);
 
-    /* Create MPI_INFO_ENV */
+    /* Create MPI_INFO_ENV  - we create here for the f_to_c.  Can't fill in 
+       here because most info needed is only available after a call to
+       ompi_rte_init. */
     OBJ_CONSTRUCT(&ompi_mpi_info_env.info, ompi_info_t);
     assert(ompi_mpi_info_env.info.i_f_to_c_index == 1);
 
+    ompi_mpi_instance_append_finalize (ompi_mpiinfo_finalize);
+
+    /* All done */
+
+    return OMPI_SUCCESS;
+}
+
+/**
+ * Fill in an info object with ENV info.  Used for setting
+ * MPI_INFO_ENV and by invocation of MPI_Info_create_env.
+ */
+
+int ompi_mpiinfo_init_env(int argc, char *argv[], ompi_info_t *info)
+{
+    char *cptr = NULL, **tmp = NULL;
+
     /* fill the env info object */
 
     /* command for this app_context */
-    if (NULL != ompi_process_info.command) {
+    if (NULL != argv) {
+        tmp = argv;
+    } else if (NULL != ompi_process_info.command) {
         tmp = opal_argv_split(ompi_process_info.command, ' ');
-        opal_info_set(&ompi_mpi_info_env.info.super, "command", tmp[0]);
+    }
+
+    if (NULL != tmp) {
+        if (NULL != tmp[0]) {
+            opal_info_set(&info->super, "command", tmp[0]);
+        }
 
         /* space-separated list of argv for this command */
         if (1 < opal_argv_count(tmp)) {
             cptr = opal_argv_join(&tmp[1], ' ');
         } else {
-            cptr = strdup(tmp[0]);
+            if (NULL != tmp[0]) {
+                cptr = strdup(tmp[0]);
+            }
+        }
+	if (NULL == argv) {
+            opal_argv_free(tmp);
+        }
+        opal_info_set(&info->super, "argv", cptr);
+        if (NULL != cptr) {
+            free(cptr);
         }
-        opal_argv_free(tmp);
-        opal_info_set(&ompi_mpi_info_env.info.super, "argv", cptr);
-        free(cptr);
     }
 
     /* max procs for the entire job */
     opal_asprintf(&cptr, "%u", ompi_process_info.num_procs);
-    opal_info_set(&ompi_mpi_info_env.info.super, "maxprocs", cptr);
+    opal_info_set(&info->super, "maxprocs", cptr);
     /* Open MPI does not support the "soft" option, so set it to maxprocs */
-    opal_info_set(&ompi_mpi_info_env.info.super, "soft", cptr);
+    opal_info_set(&info->super, "soft", cptr);
     free(cptr);
 
     /* the initial error handler, set it as requested (nothing if not
      * requested) */
     if (NULL != ompi_process_info.initial_errhandler) {
-        opal_info_set(&ompi_mpi_info_env.info.super, "mpi_initial_errhandler", ompi_process_info.initial_errhandler);
+        opal_info_set(&info->super, "mpi_initial_errhandler", ompi_process_info.initial_errhandler);
     }
 
     /* local host name */
-    opal_info_set(&ompi_mpi_info_env.info.super, "host", ompi_process_info.nodename);
+    opal_info_set(&info->super, "host", ompi_process_info.nodename);
 
 #ifdef HAVE_SYS_UTSNAME_H
     {
         struct utsname sysname;
         uname(&sysname);
         cptr = sysname.machine;
-        opal_info_set(&ompi_mpi_info_env.info.super, "arch", cptr);
+        opal_info_set(&info->super, "arch", cptr);
     }
 #endif
 
     /* initial working dir of this process, if provided */
     if (NULL != ompi_process_info.initial_wdir) {
-        opal_info_set(&ompi_mpi_info_env.info.super, "wdir", ompi_process_info.initial_wdir);
+        opal_info_set(&info->super, "wdir", ompi_process_info.initial_wdir);
     }
 
     /* provide the REQUESTED thread level - may be different
@@ -159,16 +192,16 @@ int ompi_mpiinfo_init(void)
      * ugly, but have to do a switch to find the string representation */
     switch (ompi_mpi_thread_requested) {
     case MPI_THREAD_SINGLE:
-        opal_info_set(&ompi_mpi_info_env.info.super, "thread_level", "MPI_THREAD_SINGLE");
+        opal_info_set(&info->super, "thread_level", "MPI_THREAD_SINGLE");
         break;
     case MPI_THREAD_FUNNELED:
-        opal_info_set(&ompi_mpi_info_env.info.super, "thread_level", "MPI_THREAD_FUNNELED");
+        opal_info_set(&info->super, "thread_level", "MPI_THREAD_FUNNELED");
         break;
     case MPI_THREAD_SERIALIZED:
-        opal_info_set(&ompi_mpi_info_env.info.super, "thread_level", "MPI_THREAD_SERIALIZED");
+        opal_info_set(&info->super, "thread_level", "MPI_THREAD_SERIALIZED");
         break;
     case MPI_THREAD_MULTIPLE:
-        opal_info_set(&ompi_mpi_info_env.info.super, "thread_level", "MPI_THREAD_MULTIPLE");
+        opal_info_set(&info->super, "thread_level", "MPI_THREAD_MULTIPLE");
         break;
     default:
         /* do nothing - don't know the value */
@@ -179,24 +212,24 @@ int ompi_mpiinfo_init(void)
 
     /* the number of app_contexts in this job */
     opal_asprintf(&cptr, "%u", ompi_process_info.num_apps);
-    opal_info_set(&ompi_mpi_info_env.info.super, "ompi_num_apps", cptr);
+    opal_info_set(&info->super, "ompi_num_apps", cptr);
     free(cptr);
 
     /* space-separated list of first MPI rank of each app_context */
     if (NULL != ompi_process_info.app_ldrs) {
-        opal_info_set(&ompi_mpi_info_env.info.super, "ompi_first_rank", ompi_process_info.app_ldrs);
+        opal_info_set(&info->super, "ompi_first_rank", ompi_process_info.app_ldrs);
     }
 
     /* space-separated list of num procs for each app_context */
     if (NULL != ompi_process_info.app_sizes) {
-        opal_info_set(&ompi_mpi_info_env.info.super, "ompi_np", ompi_process_info.app_sizes);
+        opal_info_set(&info->super, "ompi_np", ompi_process_info.app_sizes);
     }
 
     /* location of the directory containing any prepositioned files
      * the user may have requested
      */
     if (NULL != ompi_process_info.proc_session_dir) {
-        opal_info_set(&ompi_mpi_info_env.info.super, "ompi_positioned_file_dir", ompi_process_info.proc_session_dir);
+        opal_info_set(&info->super, "ompi_positioned_file_dir", ompi_process_info.proc_session_dir);
     }
 
     /* All done */
@@ -211,10 +244,7 @@ int ompi_mpiinfo_init(void)
 // related calls:
 
 int ompi_info_dup (ompi_info_t *info, ompi_info_t **newinfo) {
-    return opal_info_dup (&(info->super), (opal_info_t **)newinfo);
-}
-int ompi_info_dup_mpistandard (ompi_info_t *info, ompi_info_t **newinfo) {
-    return opal_info_dup_mpistandard (&(info->super), (opal_info_t **)newinfo);
+    return opal_info_dup_public (&(info->super), (opal_info_t **)newinfo);
 }
 int ompi_info_set (ompi_info_t *info, const char *key, const char *value) {
     return opal_info_set (&(info->super), key, value);
@@ -224,10 +254,10 @@ int ompi_info_set_value_enum (ompi_info_t *info, const char *key, int value,
 {
     return opal_info_set_value_enum (&(info->super), key, value, var_enum);
 }
-int ompi_info_get (ompi_info_t *info, const char *key, int valuelen,
-                   char *value, int *flag)
+int ompi_info_get (ompi_info_t *info, const char *key,
+                   opal_cstring_t **value, int *flag)
 {
-    return opal_info_get (&(info->super), key, valuelen, value, flag);
+    return opal_info_get (&(info->super), key, value, flag);
 }
 int ompi_info_get_value_enum (ompi_info_t *info, const char *key, int *value,
                               int default_value, mca_base_var_enum_t *var_enum,
@@ -247,7 +277,7 @@ int ompi_info_get_valuelen (ompi_info_t *info, const char *key, int *valuelen,
 {
     return opal_info_get_valuelen (&(info->super), key, valuelen, flag);
 }
-int ompi_info_get_nthkey (ompi_info_t *info, int n, char *key) {
+int ompi_info_get_nthkey (ompi_info_t *info, int n, opal_cstring_t **key) {
     return opal_info_get_nthkey (&(info->super), n, key);
 }
 int ompi_info_get_nkeys(ompi_info_t *info, int *nkeys)
@@ -304,8 +334,8 @@ int ompi_mpiinfo_finalize(void)
                          item = opal_list_get_next(item)) {
                         entry = (opal_info_entry_t *) item;
                         opal_output(0, "WARNING:   key=\"%s\", value=\"%s\"",
-                                    entry->ie_key,
-                                    NULL != entry->ie_value ? entry->ie_value : "(null)");
+                                    entry->ie_key->string,
+                                    NULL != entry->ie_value ? entry->ie_value->string : "(null)");
                         found = true;
                     }
                 }
@@ -322,6 +352,9 @@ int ompi_mpiinfo_finalize(void)
         }
     }
 
+    /* Release the array of available memkind objects */
+    ompi_info_memkind_free_available();
+
     /* All done -- destroy the table */
 
     OBJ_DESTRUCT(&ompi_info_f_to_c_table);
@@ -368,6 +401,31 @@ static void info_destructor(ompi_info_t *info)
 
 }
 
+ompi_info_t *ompi_info_allocate (void)
+{
+    ompi_info_t *new_info;
+    int rc;
+
+    rc = ompi_mpi_instance_retain ();
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+        /* NTH: seriously, what can we do other than abort () or return? we failed to
+         * set up the most basic infrastructure! */
+        return NULL;
+    }
+
+    /*
+     * Call the object create function. This function not only
+     * allocates the space for MPI_Info, but also calls all the
+     * relevant init functions. Should I check if the fortran
+     * handle is valid
+     */
+    new_info = OBJ_NEW(ompi_info_t);
+    if (NULL == new_info) {
+        return NULL;
+    }
+
+    return new_info;
+}
 
 /*
  * Free an info handle and all of its keys and values.
@@ -377,5 +435,9 @@ int ompi_info_free (ompi_info_t **info)
     (*info)->i_freed = true;
     OBJ_RELEASE(*info);
     *info = MPI_INFO_NULL;
+
+    /* release the retain() from info create/dup */
+    ompi_mpi_instance_release ();
+
     return MPI_SUCCESS;
 }
diff --git a/ompi/info/info.h b/ompi/info/info.h
index 8b5a861de8e..71563eb4e68 100644
--- a/ompi/info/info.h
+++ b/ompi/info/info.h
@@ -16,6 +16,9 @@
  *                         reserved.
  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
  * Copyright (c) 2020      Intel, Inc.  All rights reserved.
+ * Copyright (c) 2020      Triad National Security, LLC. All rights
+ *                         reserved.
+ *
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -52,7 +55,7 @@ struct ompi_info_t {
 typedef struct ompi_info_t ompi_info_t;
 
 /**
- * Padded struct to maintain back compatibiltiy.
+ * Padded struct to maintain back compatibility.
  * See ompi/communicator/communicator.h comments with struct ompi_communicator_t
  * for full explanation why we chose the following padding construct for predefines.
  */
@@ -83,11 +86,17 @@ OMPI_DECLSPEC extern ompi_predefined_info_t *ompi_mpi_info_null_addr;
 OMPI_DECLSPEC OBJ_CLASS_DECLARATION(ompi_info_t);
 
 /**
- * This function is invoked during ompi_mpi_init() and sets up
+ * This function is invoked during ompi_instance_retain() and sets up
  * MPI_Info handling.
  */
 int ompi_mpiinfo_init(void);
 
+/**
+ * Fill in an info object with ENV info.  Used for setting
+ * MPI_INFO_ENV and by invocation of MPI_Info_create_env.
+ */
+int ompi_mpiinfo_init_env(int argc, char *argv[], ompi_info_t *info);
+
 /**
  * This function is used to free a ompi level info
  */
@@ -130,8 +139,8 @@ OMPI_DECLSPEC int ompi_info_get_value_enum (ompi_info_t *info, const char *key,
 /**
  * ompi_info_foo() wrapper around various opal_info_foo() calls
  */
-OMPI_DECLSPEC int ompi_info_get (ompi_info_t *info, const char *key, int valuelen,
-                                 char *value, int *flag);
+OMPI_DECLSPEC int ompi_info_get (ompi_info_t *info, const char *key,
+                                 opal_cstring_t **value, int *flag);
 /**
  * ompi_info_foo() wrapper around various opal_info_foo() calls
  */
@@ -144,7 +153,7 @@ OMPI_DECLSPEC int ompi_info_get_valuelen (ompi_info_t *info, const char *key, in
 /**
  * ompi_info_foo() wrapper around various opal_info_foo() calls
  */
-OMPI_DECLSPEC int ompi_info_get_nthkey (ompi_info_t *info, int n, char *key);
+OMPI_DECLSPEC int ompi_info_get_nthkey (ompi_info_t *info, int n, opal_cstring_t **key);
 /**
  * ompi_info_foo() wrapper around various opal_info_foo() calls
  */
@@ -155,6 +164,15 @@ OMPI_DECLSPEC int ompi_info_value_to_bool(char *value, bool *interp);
 OMPI_DECLSPEC int ompi_info_get_nkeys(ompi_info_t *info, int *nkeys);
 
 
+/**
+ * @brief Allocate a new info object
+ *
+ * This helper function ensures that the minimum infrastructure is initialized
+ * for creation/modification/destruction of an info object. Do not call
+ * OBJ_NEW(opal_info_t) directly.
+ */
+OMPI_DECLSPEC ompi_info_t *ompi_info_allocate (void);
+
 END_C_DECLS
 
 /**
diff --git a/ompi/info/info_memkind.c b/ompi/info/info_memkind.c
new file mode 100644
index 00000000000..da5bb9c0bd2
--- /dev/null
+++ b/ompi/info/info_memkind.c
@@ -0,0 +1,594 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2025      Advanced Micro Devices, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "info/info_memkind.h"
+#include "opal/util/argv.h"
+#include "opal/mca/accelerator/accelerator.h"
+#include "opal/mca/accelerator/base/base.h"
+#include "opal/util/printf.h"
+#include "ompi/errhandler/errcode.h"
+#include "ompi/constants.h"
+
+static opal_mutex_t ompi_info_memkind_mutex = OPAL_MUTEX_STATIC_INIT;
+static ompi_memkind_t *ompi_info_memkind_available;
+static int ompi_info_memkind_num_available = 0;
+
+#if 0
+static void ompi_info_memkind_dump (const char *var_name, int num_memkinds, ompi_memkind_t *memkinds)
+{
+    for (int i = 0; i < num_memkinds; i++) {
+        printf("[%d] %s memkind[%d].name: %s ", getpid(), var_name, i, memkinds[i].im_name);
+        if (memkinds[i].im_num_restrictors > 0) {
+            printf("restrictors: ");
+            for (int j = 0; j < memkinds[i].im_num_restrictors; j++) {
+                printf("%c %s", (j == 0 ? ' ': ','), memkinds[i].im_restrictors[j]);
+            }
+        } else {
+            printf("num restrictors = 0");
+        }
+        printf(" no_restrictors = %s", memkinds[i].im_no_restrictors ? "true" : "false");
+        printf("\n");
+    }
+}
+#endif
+
+static void ompi_info_memkind_extract (const char* memkind_str, int *num_memkinds, ompi_memkind_t **memkinds)
+{
+    /* The memkind string is a comma-separated list of memkinds, which can have two forms:
+    **   - standalone memkind type, which implies that all restrictors of the memkind are requested
+    **     (or looking at it the other way around, no restrictions are imposed on that memory kind)
+    **   - memkind:restrictor
+    ** The same memkind type can appear multiple times, e.g.
+    **    memkind_a:restrictor_1,memkind_a:restrictor_2;
+    */
+
+    /* Separate requested_str into an array of individual entries */
+    int current_max = 0;
+    char **memkind_combos = opal_argv_split(memkind_str, ',');
+    int max_num_memkinds = opal_argv_count(memkind_combos);
+
+    ompi_memkind_t *memkind_arr = NULL;
+    memkind_arr = (ompi_memkind_t *) malloc(max_num_memkinds * sizeof(ompi_memkind_t));
+    if (NULL == memkind_arr) {
+        goto err_exit;
+    }
+    for (int i = 0; i < max_num_memkinds; i++) {
+        memkind_arr[i].im_num_restrictors = 0;
+        memkind_arr[i].im_no_restrictors = true;
+    }
+
+    int iter = 0;
+    char *m = memkind_combos[iter];
+    while (m != NULL) {
+        bool name_found = false;
+        char **tmp_str = opal_argv_split (m, ':');
+        int pos;
+
+        // Try to remove duplicates of the exact same name:restrictor appearance
+        for (int i = 0; i < current_max; i++) {
+            if (!strncmp(tmp_str[0], memkind_arr[i].im_name, strlen(tmp_str[0]))) {
+                name_found = true;
+                pos = i;
+                break;
+            }
+        }
+
+        if (name_found)  {
+            // check whether restrictor matches (if any present at either m
+            // or at memkind_arr[pos] )
+            if (NULL != tmp_str[1] ) {
+                if (memkind_arr[pos].im_num_restrictors > 0) {
+                    if (!strncmp(tmp_str[1], memkind_arr[pos].im_restrictors[0], strlen(tmp_str[1]))) {
+                        // We have seen this exact name:restrictor combination already                    
+                        m = memkind_combos[++iter];
+                        continue;
+                    }
+                } 
+            } else {
+                if (0 == memkind_arr[pos].im_num_restrictors) {
+                    // neither the memkind that we want to add nor the
+                    // element in the list of known memkinds have a restrictors, so skip
+                    m = memkind_combos[++iter];
+                    continue;
+                }
+            }
+            
+        }
+        pos = current_max;
+        memkind_arr[pos].im_name = strdup (tmp_str[0]);
+        if (NULL != tmp_str[1]) {
+            memkind_arr[pos].im_restrictors[0] = strdup(tmp_str[1]);
+            memkind_arr[pos].im_num_restrictors = 1;
+            memkind_arr[pos].im_no_restrictors = false;
+        }
+        current_max++;
+
+        opal_argv_free(tmp_str);
+        m = memkind_combos[++iter];
+    }
+
+ err_exit:
+    *num_memkinds = current_max;
+    *memkinds = memkind_arr;
+
+    return;
+}
+
+static int ompi_info_memkind_get_available(int *num_memkinds, ompi_memkind_t **memkinds)
+{
+    int ret = OMPI_SUCCESS;
+    if (ompi_info_memkind_num_available > 0) {
+        goto exit_no_lock;
+    }
+
+    OPAL_THREAD_LOCK (&ompi_info_memkind_mutex);
+    if (ompi_info_memkind_num_available > 0) {
+        goto exit;
+    }
+
+    int tmp_num = 2;
+    if (0 != strcmp(opal_accelerator_base_selected_component.base_version.mca_component_name, "null")) {
+        tmp_num++;
+    }
+
+    ompi_info_memkind_available = (ompi_memkind_t *) malloc (tmp_num * sizeof(ompi_memkind_t));
+    if (NULL == ompi_info_memkind_available) {
+        *num_memkinds = 0;
+        *memkinds = NULL;
+        OPAL_THREAD_UNLOCK(&ompi_info_memkind_mutex);
+        return OMPI_ERROR;
+    }
+
+    /* The system and mpi memory kinds are defined in MPI 4.1 section 12.4.3 */
+    ompi_info_memkind_available[0].im_name = strdup ("system");
+    ompi_info_memkind_available[0].im_num_restrictors = 0;
+    ompi_info_memkind_available[0].im_no_restrictors = true;
+
+    ompi_info_memkind_available[1].im_name = strdup ("mpi");
+    ompi_info_memkind_available[1].im_num_restrictors = 3;
+    ompi_info_memkind_available[1].im_no_restrictors = false;
+    ompi_info_memkind_available[1].im_restrictors[0] = strdup ("alloc_mem");
+    ompi_info_memkind_available[1].im_restrictors[1] = strdup ("win_allocate");
+    ompi_info_memkind_available[1].im_restrictors[2] = strdup ("win_allocate_shared");
+
+    if (tmp_num > 2) {
+        opal_accelerator.get_memkind (&ompi_info_memkind_available[2]);
+    }
+    ompi_info_memkind_num_available = tmp_num;
+
+ exit:
+    OPAL_THREAD_UNLOCK(&ompi_info_memkind_mutex);
+ exit_no_lock:
+    *num_memkinds = ompi_info_memkind_num_available;
+    *memkinds = ompi_info_memkind_available;
+    return ret;
+}
+
+static void ompi_info_memkind_free (int num, ompi_memkind_t *memkind_arr)
+{
+    for (int i = 0; i < num; i++) {
+        free (memkind_arr[i].im_name);
+        for (int j = 0; j < memkind_arr[i].im_num_restrictors; j++) {
+            free (memkind_arr[i].im_restrictors[j]);
+        }
+    }
+    free (memkind_arr);
+}
+
+static void ompi_info_memkind_str_create (int num_memkinds, ompi_memkind_t *memkinds, char** memkind_str)
+{
+    int num_elems = 0;
+
+    for (int i = 0; i < num_memkinds; i++) {
+        if (memkinds[i].im_no_restrictors || (memkinds[i].im_num_restrictors == 0)) {
+            num_elems++;
+        } else {
+            num_elems += memkinds[i].im_num_restrictors;
+        }
+    }
+
+    char **tmp_str_arr = (char**) malloc ((num_elems+1) * sizeof (char**));
+    if (NULL == tmp_str_arr) {
+        *memkind_str = NULL;
+        return;
+    }
+
+    int c = 0;
+    for (int i = 0; i < num_memkinds; i++) {
+        if (memkinds[i].im_no_restrictors || (memkinds[i].im_num_restrictors == 0)) {
+            opal_asprintf(&tmp_str_arr[c++], "%s",memkinds[i].im_name);
+        } else {
+            for (int j = 0; j < memkinds[i].im_num_restrictors; j++) {
+                opal_asprintf(&tmp_str_arr[c++], "%s:%s",memkinds[i].im_name,
+                              memkinds[i].im_restrictors[j]);
+            }
+        }
+    }
+    tmp_str_arr[num_elems] = NULL;
+
+    char *tmp_str = opal_argv_join(tmp_str_arr, ',');
+    opal_argv_free(tmp_str_arr);
+
+    *memkind_str = tmp_str;
+    return;
+}
+
+#define COPY_MEMKIND(_to,_from,_copy_restrictors) {                     \
+    _to.im_name = strdup(_from.im_name);                                \
+    _to.im_no_restrictors = _from.im_no_restrictors;                    \
+    _to.im_num_restrictors = 0;                                         \
+    if (_copy_restrictors) {                                            \
+        _to.im_num_restrictors = _from.im_num_restrictors;              \
+        for (int _i = 0; _i < _from.im_num_restrictors; _i++) {         \
+            _to.im_restrictors[_i] = strdup (_from.im_restrictors[_i]); \
+        }                                                               \
+    }                                                                   \
+}
+
+static int ompi_info_memkind_remove_unsupported (int num_requested, ompi_memkind_t *requested_memkinds,
+                                                 int num_available, ompi_memkind_t *available_memkinds,
+                                                 int *num_provided, ompi_memkind_t **provided_memkinds)
+{
+    bool have_system_memkind = false;
+    bool have_mpi_memkind = false;
+    int pos = 0;
+    int *apos = malloc (num_requested *sizeof(int));
+    if (NULL == apos) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    /*
+    ** Check whether we support the memkinds requested by the user
+    ** In addition, keep track whether user requested "system" and "mpi"
+    ** memory_alloc_kinds, since we always add those to the list
+    ** of support memory_alloc_kinds
+    */
+    for (int i = 0; i < num_requested; i++) {
+        bool found_name = false;
+        bool found_all_requested_restrictors = true;
+        int j = -1;
+
+        if (!have_system_memkind && !strncmp(requested_memkinds[i].im_name, "system", strlen("system"))) {
+            have_system_memkind = true;
+        }
+        if ( (!have_mpi_memkind && !strncmp(requested_memkinds[i].im_name, "mpi", strlen("mpi"))) &&
+             (0 == requested_memkinds[i].im_num_restrictors)) {
+            have_mpi_memkind = true;
+        }
+
+        // Check for memory_alloc_kind name first
+        for (j = 0; j < num_available; j++) {
+            if (!strncmp(requested_memkinds[i].im_name, available_memkinds[j].im_name,
+                         strlen(requested_memkinds[i].im_name))) {
+                found_name = true;
+                break;
+            }
+        }
+        if (found_name) {
+            // Check whether we recognize all restrictors requested by user for
+            // this memory_alloc_kind
+            bool found_this_restrictor = false;
+            for (int k = 0; k < requested_memkinds[i].im_num_restrictors; k++) {
+                for (int l = 0; l < available_memkinds[j].im_num_restrictors; l++) {
+                    if (!strncmp(requested_memkinds[i].im_restrictors[k], available_memkinds[j].im_restrictors[l],
+                                 strlen(requested_memkinds[i].im_restrictors[k]))) {
+                        found_this_restrictor = true;
+                        break;
+                    }
+                }
+                if (!found_this_restrictor) {
+                    found_all_requested_restrictors = false;
+                    break;
+                }
+            }
+            if (found_all_requested_restrictors) {
+                apos[pos++] = i;
+            }
+        }
+    }
+
+    // Add "system" and "mpi" memkinds as supported, even if not requested by user
+    int total_len = pos;
+    if (!have_system_memkind) {
+        total_len++;
+    }
+    if (!have_mpi_memkind) {
+        total_len++;
+    }
+
+    ompi_memkind_t *final = (ompi_memkind_t*) malloc (total_len * sizeof(ompi_memkind_t));
+    if (NULL == final) {
+        free (apos);
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    int offset = 0;
+    // assert (!strncmp(available_memkinds[0].im_name, "system", strlen("system")));
+    COPY_MEMKIND(final[0], available_memkinds[0], false);
+    offset++;
+
+    // assert (!strncmp(available_memkinds[1].im_name, "mpim", strlen("mpi")));
+    COPY_MEMKIND(final[offset], available_memkinds[1], false);
+    offset++;
+
+    for (int i = 0; i < pos; i++) {
+        if (!strncmp(requested_memkinds[apos[i]].im_name, "system", strlen("system"))) {
+            continue;
+        }
+        if ( (!strncmp(requested_memkinds[apos[i]].im_name, "mpi", strlen("mpi"))) &&
+             (0 == requested_memkinds[apos[i]].im_num_restrictors)) {
+            continue;
+        }
+        COPY_MEMKIND (final[offset], requested_memkinds[apos[i]], true);
+        offset++;
+    }
+
+    *num_provided = total_len;
+    *provided_memkinds = final;
+    return OMPI_SUCCESS;
+}
+
+static bool ompi_info_memkind_is_subset (int num_subset, ompi_memkind_t *subset,
+                                         int num_superset, ompi_memkind_t *superset)
+{
+    bool ret = true;
+
+    for (int i = 0; i < num_subset; i++) {
+        bool found_name = false;
+        int j = -1;
+
+        // Check for memory_alloc_kind name first
+        for (j = 0; j < num_superset; j++) {
+            if (!strncmp(subset[i].im_name, superset[j].im_name,
+                         strlen(subset[i].im_name))) {
+                found_name = true;
+                break;
+            }
+        }
+        if (found_name) {
+            /* Check whether we recognize all restrictors requested listed in
+            ** the subset in the superset. Note, that the superset might not
+            ** have any restrictors set, in which case all restrictors are accepted
+            */
+            if ((0 == superset[j].im_num_restrictors) || superset[j].im_no_restrictors) {
+                continue;
+            }
+            for (int k = 0; k < subset[i].im_num_restrictors; k++) {
+                bool found_this_restrictor = false;
+                for (int l = 0; l < superset[j].im_num_restrictors; l++) {
+                    if (!strncmp(subset[i].im_restrictors[k], superset[j].im_restrictors[l],
+                                 strlen(subset[i].im_restrictors[k]))) {
+                        found_this_restrictor = true;
+                        break;
+                    }
+                }
+                if (!found_this_restrictor) {
+                    ret = false;
+                    goto exit;
+                }
+            }
+        } else {
+            ret = false;
+            goto exit;
+        }
+    }
+
+ exit:
+    return ret;
+}
+
+static bool ompi_info_memkind_validate (const char *assert_str, const char *parent_str)
+{
+    int num_assert_memkinds = 0, num_parent_memkinds = 0;
+    ompi_memkind_t *assert_memkinds = NULL;
+    ompi_memkind_t *parent_memkinds = NULL;
+    bool ret;
+
+    ompi_info_memkind_extract (assert_str, &num_assert_memkinds, &assert_memkinds);
+    ret = ompi_info_memkind_is_subset (num_assert_memkinds, assert_memkinds,
+                                       ompi_info_memkind_num_available, ompi_info_memkind_available);
+    if (!ret) {
+        goto exit;
+    }
+
+    ompi_info_memkind_extract (parent_str, &num_parent_memkinds, &parent_memkinds);
+    ret = ompi_info_memkind_is_subset (num_assert_memkinds, assert_memkinds,
+                                       num_parent_memkinds, parent_memkinds);
+
+ exit:
+    if (NULL != assert_memkinds) {
+        ompi_info_memkind_free(num_assert_memkinds, assert_memkinds);
+    }
+    if (NULL != parent_memkinds) {
+        ompi_info_memkind_free(num_parent_memkinds, parent_memkinds);
+    }
+
+    return ret;
+}
+
+static bool ompi_info_memkind_check_no_accel (int num_memkinds, ompi_memkind_t *memkinds)
+{
+    bool result = true;
+
+    for (int i = 0; i < num_memkinds; i++) {
+        if (!strncmp(memkinds[i].im_name, "system", strlen("system"))) {
+            continue;
+        }
+        if (!strncmp(memkinds[i].im_name, "mpi", strlen("mpi"))) {
+            continue;
+        }
+        result = false;
+        break;
+    }
+
+    return result;
+}
+
+static bool ompi_info_memkind_check_no_accel_from_string (char *mstring)
+{
+    bool ret = false;
+    int num_memkinds;
+    ompi_memkind_t *memkinds = NULL;
+
+    ompi_info_memkind_extract (mstring, &num_memkinds, &memkinds);
+    if (NULL != memkinds) {
+        ret = ompi_info_memkind_check_no_accel (num_memkinds, memkinds);
+        ompi_info_memkind_free(num_memkinds, memkinds);
+    }
+
+    return ret;
+}
+int ompi_info_memkind_process (const char* requested_str, char **provided_str,
+                               ompi_info_memkind_assert_type *type)
+{
+    int err;
+    char *tmp_str = NULL;
+    int num_requested_memkinds, num_available_memkinds, num_provided_memkinds;
+    ompi_memkind_t *requested_memkinds = NULL ;
+    ompi_memkind_t *available_memkinds = NULL;
+    ompi_memkind_t *provided_memkinds = NULL;
+    ompi_info_memkind_assert_type assert_type = OMPI_INFO_MEMKIND_ASSERT_UNDEFINED;
+
+    if (NULL == requested_str) {
+        *provided_str = NULL;
+        *type = assert_type;
+        return OMPI_SUCCESS;
+    }
+
+    ompi_info_memkind_extract (requested_str, &num_requested_memkinds, &requested_memkinds);
+    err = ompi_info_memkind_get_available (&num_available_memkinds, &available_memkinds);
+    if (OMPI_SUCCESS != err) {
+        goto exit;
+    }
+
+    err = ompi_info_memkind_remove_unsupported (num_requested_memkinds, requested_memkinds,
+                                                num_available_memkinds, available_memkinds,
+                                                &num_provided_memkinds, &provided_memkinds);
+    if (OMPI_SUCCESS != err) {
+        goto exit;
+    }
+
+    if (ompi_info_memkind_check_no_accel (num_provided_memkinds, provided_memkinds)) {
+        assert_type = OMPI_INFO_MEMKIND_ASSERT_NO_ACCEL;
+    }
+    
+    ompi_info_memkind_str_create(num_provided_memkinds, provided_memkinds, &tmp_str);
+
+ exit:
+    if (NULL != requested_memkinds) {
+        ompi_info_memkind_free(num_requested_memkinds, requested_memkinds);
+    }
+    if (NULL != provided_memkinds) {
+        ompi_info_memkind_free(num_provided_memkinds, provided_memkinds);
+    }
+    // Don't free the available_memkinds, they will be released in info_finalize;
+    
+    *type = assert_type;
+    *provided_str = tmp_str;
+    return err;
+}
+
+/**
+ * Callback invoked by the info subscriber mechanism.
+ * Accepts only the first value set.
+ */
+const char *ompi_info_memkind_cb (opal_infosubscriber_t *obj, const char *key, const char *value)
+{
+    opal_cstring_t *existing_val;
+    int flag;
+    char *ret_string;
+
+    opal_info_get(obj->s_info, key, &existing_val, &flag);
+    if (0 == flag) {
+        ret_string = (char *)value;
+    } else {
+        ret_string = (char *)existing_val->string;
+        OBJ_RELEASE(existing_val);
+    }
+    return ret_string;
+}
+
+/*
+** Algorithm is a bit convoluted:
+**
+** - retrieve mpi_memory_alloc_kinds from parent instance.
+** - if info object passed in as argument to this routine contains
+**   mpi_assert_memory_alloc_kinds key/value pair:
+**     - validate that we recognize all memory kinds listed
+**     - if that is the case, use the value of the of mpi_assert_memory_alloc_kinds
+**       when setting mpi_memory_alloc_kinds on the child object
+**     - else ignore the mpi_assert_memory_alloc_kinds. (Quote:
+**       "If the MPI library does not support one or more of the allocation kinds associated
+**        with the mpi_assert_memory_alloc_kinds info key, it will ignore this info key".
+**       So we are supposed to drop the entire key, not just the memory kinds that we did
+**       recognize.)
+** - else use the same memkinds as in mpi_memory_alloc_kinds on the parent object on the
+**   child object (i.e. we just copy it over)
+**
+** To summerize, the value of one info key (mpi_assert_memory_alloc_kinds) can influence the
+** value of another info key (mpi_memory_alloc_kinds).
+*/
+int ompi_info_memkind_copy_or_set (opal_infosubscriber_t *parent, opal_infosubscriber_t *child,
+                                   opal_info_t *info, ompi_info_memkind_assert_type *type)
+{
+    opal_cstring_t *parent_val;
+    opal_cstring_t *assert_val;
+    ompi_info_memkind_assert_type assert_type = OMPI_INFO_MEMKIND_ASSERT_UNDEFINED;
+    char *final_str = NULL;
+    int flag;
+
+    opal_info_get(parent->s_info, "mpi_memory_alloc_kinds", &parent_val, &flag);
+    if (0 == flag) {
+        *type = assert_type;
+        return OMPI_SUCCESS;
+    }
+    final_str = (char*) parent_val->string;
+
+    if (NULL != info) {
+        opal_info_get(info, "mpi_assert_memory_alloc_kinds", &assert_val, &flag);
+        if (0 == flag) {
+            // assert_memory_alloc_kinds was not set by code
+            goto exit;
+        }
+
+        // Validate asserted memory kind
+        bool ret = ompi_info_memkind_validate (assert_val->string, parent_val->string);
+        if (ret) {
+            final_str = (char*) assert_val->string;
+        }
+        OBJ_RELEASE(assert_val);
+
+        opal_infosubscribe_subscribe (child, "mpi_assert_memory_alloc_kinds", final_str,
+                                      ompi_info_memkind_cb);
+    }
+
+ exit:
+    opal_infosubscribe_subscribe (child, "mpi_memory_alloc_kinds", final_str,
+                                  ompi_info_memkind_cb);
+    OBJ_RELEASE(parent_val);
+
+    if (ompi_info_memkind_check_no_accel_from_string(final_str)) {
+        assert_type = OMPI_INFO_MEMKIND_ASSERT_NO_ACCEL;
+    }
+
+    *type = assert_type;
+    return OMPI_SUCCESS;
+}
+
+void ompi_info_memkind_free_available (void)
+{
+    ompi_info_memkind_free (ompi_info_memkind_num_available, ompi_info_memkind_available);
+}
diff --git a/ompi/info/info_memkind.h b/ompi/info/info_memkind.h
new file mode 100644
index 00000000000..a23a55ea27f
--- /dev/null
+++ b/ompi/info/info_memkind.h
@@ -0,0 +1,81 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2025      Advanced Micro Devices, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef OMPI_INFO_MEMKIND_H
+#define OMPI_INFO_MEMKIND_H
+
+#include "ompi_config.h"
+#include "opal/util/info_subscriber.h"
+
+BEGIN_C_DECLS
+
+#define OMPI_MAX_NUM_MEMKIND_RESTRICTORS 3
+struct ompi_memkind_t {
+    char *im_name;
+    bool  im_no_restrictors;
+    int   im_num_restrictors;
+    char *im_restrictors[OMPI_MAX_NUM_MEMKIND_RESTRICTORS];
+};
+typedef struct ompi_memkind_t ompi_memkind_t;
+
+typedef enum {
+    OMPI_INFO_MEMKIND_ASSERT_UNDEFINED = 0,     // no statement on memkind usage
+    OMPI_INFO_MEMKIND_ASSERT_NO_ACCEL,          // no accelerator memory is used
+    OMPI_INFO_MEMKIND_ASSERT_ACCEL_DEVICE_ONLY, // only accelerator device memory used
+    OMPI_INFO_MEMKIND_ASSERT_ACCEL_ALL          // only accelerator memory (no restrictors) used
+} ompi_info_memkind_assert_type;
+
+/*
+** Given a string of user requested memory alloc kinds, create
+** a string with the actually support memory kinds by the library.
+**
+** @param[IN]: requested_str     input string
+** @param[OUT]: provided_str     result string
+** @param[OUT]: type             guarantuees given on memkind utilization
+**
+** @return:                      OMPI_SUCCESS or error on failure
+*/
+OMPI_DECLSPEC int ompi_info_memkind_process (const char* requested_str,
+                                             char **provided_str,
+                                             ompi_info_memkind_assert_type *type);
+/*
+** Set the memory_alloc_kind info object on the child object, either
+** by copying it from the parent object, or adjusting it based
+** on the assert_memory_alloc_kind info object provided by the code
+** during object creation
+**
+** @param[IN]:     parent        parent object (e.g. comm->super, file->super, etc.)
+** @param [INOUT]: child         child object
+** @param[IN]:     info          info object provided by code during object creation
+**                               (e.g. MPI_Comm_dup_with_info, MPI_File_open, etc.)
+** @param[OUT]: type             guarantuees given on memkind utilization
+**
+** @return:                      OMPI_SUCCESS or error on failure
+*/
+OMPI_DECLSPEC int ompi_info_memkind_copy_or_set (opal_infosubscriber_t *parent,
+                                                 opal_infosubscriber_t *child,
+                                                 opal_info_t *info,
+                                                 ompi_info_memkind_assert_type *type);
+
+/*
+** free the array of available memkinds when shutting down the info
+** infrastructure.
+*/
+OMPI_DECLSPEC void ompi_info_memkind_free_available (void);
+
+/*
+** Callback function used when registering memkind info object
+*/
+OMPI_DECLSPEC const char *ompi_info_memkind_cb (opal_infosubscriber_t *obj, const char *key, const char *value);
+
+END_C_DECLS
+
+#endif /* OMPI_INFO_MEMKIND_H */
+
diff --git a/ompi/instance/Makefile.am b/ompi/instance/Makefile.am
new file mode 100644
index 00000000000..2ee7f5d59a3
--- /dev/null
+++ b/ompi/instance/Makefile.am
@@ -0,0 +1,26 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2016      IBM Corporation.  All rights reserved.
+# Copyright (c) 2018      Triad National Security, LLC. All rights
+#                         reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# This makefile.am does not stand on its own - it is included from ompi/Makefile.am
+
+headers += instance/instance.h
+
+lib@OMPI_LIBMPI_NAME@_la_SOURCES += instance/instance.c
diff --git a/ompi/instance/instance.c b/ompi/instance/instance.c
new file mode 100644
index 00000000000..0b83e442b0c
--- /dev/null
+++ b/ompi/instance/instance.c
@@ -0,0 +1,1462 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2022      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "instance.h"
+
+#include "opal/util/arch.h"
+
+#include "opal/util/show_help.h"
+#include "opal/util/argv.h"
+#include "opal/runtime/opal_params.h"
+
+#include "ompi/mca/pml/pml.h"
+#include "ompi/runtime/params.h"
+
+#include "ompi/interlib/interlib.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/errhandler/errcode.h"
+#include "ompi/message/message.h"
+#include "ompi/info/info.h"
+#include "ompi/info/info_memkind.h"
+#include "ompi/attribute/attribute.h"
+#include "ompi/op/op.h"
+#include "ompi/dpm/dpm.h"
+#include "ompi/file/file.h"
+#include "ompi/mpiext/mpiext.h"
+
+#include "ompi/mca/hook/base/base.h"
+#include "ompi/mca/op/base/base.h"
+#include "opal/mca/allocator/base/base.h"
+#include "opal/mca/rcache/base/base.h"
+#include "opal/mca/mpool/base/base.h"
+#include "opal/mca/smsc/base/base.h"
+#include "ompi/mca/bml/base/base.h"
+#include "ompi/mca/pml/base/base.h"
+#include "ompi/mca/coll/base/base.h"
+#include "ompi/mca/osc/base/base.h"
+#include "ompi/mca/part/base/base.h"
+#include "ompi/mca/io/base/base.h"
+#include "ompi/mca/topo/base/base.h"
+#include "opal/mca/pmix/base/base.h"
+
+#include "ompi/mca/pml/base/pml_base_bsend.h"
+#include "ompi/util/timings.h"
+#include "opal/mca/mpool/base/mpool_base_tree.h"
+#include "opal/mca/pmix/pmix-internal.h"
+#include "opal/util/clock_gettime.h"
+
+ompi_predefined_instance_t ompi_mpi_instance_null = {{{{0}}}};
+
+#if defined(OPAL_RECURSIVE_MUTEX_STATIC_INIT)
+static opal_recursive_mutex_t instance_lock = OPAL_RECURSIVE_MUTEX_STATIC_INIT;
+#elif defined(OPAL_HAVE_ATTRIBUTE_CONSTRUCTOR)
+static opal_recursive_mutex_t instance_lock;
+__opal_attribute_constructor__ static void instance_lock_init(void) {
+    OBJ_CONSTRUCT(&instance_lock, opal_recursive_mutex_t);
+}
+#else
+#error "No support for recursive mutexes available on this platform."
+#endif  /* defined(OPAL_RECURSIVE_MUTEX_STATIC_INIT) */
+
+/** MPI_Init instance */
+ompi_instance_t *ompi_mpi_instance_default = NULL;
+
+/**
+ * @brief: Base timer initialization. All timers returned to the user via MPI_Wtime
+ *         are relative to this timer. Setting it early in during the common
+ *         initialization (world or session model) allows for measuring the cost of
+ *         the MPI initialization.
+ */
+struct timespec ompi_wtime_time_origin = {.tv_sec = 0};
+
+enum {
+    OMPI_INSTANCE_INITIALIZING = -1,
+    OMPI_INSTANCE_FINALIZING   = -2,
+};
+
+opal_atomic_int32_t ompi_instance_count = 0;
+
+static const char *ompi_instance_builtin_psets[] = {
+    "mpi://WORLD",
+    "mpi://SELF",
+    "mpix://SHARED",
+};
+
+static const int32_t ompi_instance_builtin_count = 3;
+
+/** finalization functions that need to be called on teardown */
+static opal_finalize_domain_t ompi_instance_basic_domain;
+static opal_finalize_domain_t ompi_instance_common_domain;
+
+static void ompi_instance_construct (ompi_instance_t *instance)
+{
+    instance->i_f_to_c_index = opal_pointer_array_add (&ompi_instance_f_to_c_table, instance);
+    instance->i_name[0] = '\0';
+    instance->i_flags = 0;
+    instance->i_keyhash = NULL;
+    OBJ_CONSTRUCT(&instance->s_lock, opal_mutex_t);
+    instance->errhandler_type = OMPI_ERRHANDLER_TYPE_INSTANCE;
+}
+
+static void ompi_instance_destruct(ompi_instance_t *instance)
+{
+    OBJ_DESTRUCT(&instance->s_lock);
+}
+
+OBJ_CLASS_INSTANCE(ompi_instance_t, opal_infosubscriber_t, ompi_instance_construct, ompi_instance_destruct);
+
+/* NTH: frameworks needed by MPI */
+static mca_base_framework_t *ompi_framework_dependencies[] = {
+    &ompi_hook_base_framework, &ompi_op_base_framework,
+    &opal_allocator_base_framework, &opal_rcache_base_framework, &opal_mpool_base_framework, &opal_smsc_base_framework,
+    &ompi_bml_base_framework, &ompi_pml_base_framework, &ompi_coll_base_framework,
+    &ompi_osc_base_framework, &ompi_part_base_framework, NULL,
+};
+
+static mca_base_framework_t *ompi_lazy_frameworks[] = {
+    &ompi_io_base_framework, &ompi_topo_base_framework, NULL,
+};
+
+
+static int ompi_mpi_instance_finalize_common (void);
+
+/*
+ * Hash tables for MPI_Type_create_f90* functions
+ */
+opal_hash_table_t ompi_mpi_f90_integer_hashtable = {{0}};
+opal_hash_table_t ompi_mpi_f90_real_hashtable = {{0}};
+opal_hash_table_t ompi_mpi_f90_complex_hashtable = {{0}};
+
+static size_t  ompi_mpi_instance_num_pmix_psets;
+static char  **ompi_mpi_instance_pmix_psets;
+/*
+ * Per MPI-2:9.5.3, MPI_REGISTER_DATAREP is a memory leak.  There is
+ * no way to *de*register datareps once they've been registered.  So
+ * we have to track all registrations here so that they can be
+ * de-registered during MPI_FINALIZE so that memory-tracking debuggers
+ * don't show Open MPI as leaking memory.
+ */
+opal_list_t ompi_registered_datareps = {{0}};
+
+opal_pointer_array_t ompi_instance_f_to_c_table = {{0}};
+
+/*
+ * PMIx event handlers
+ */
+
+static size_t ompi_default_pmix_err_handler = 0;
+static size_t ompi_ulfm_pmix_err_handler = 0;
+
+static int ompi_instance_print_error (const char *error, int ret)
+{
+    /* Only print a message if one was not already printed */
+    if (NULL != error && OMPI_ERR_SILENT != ret) {
+        const char *err_msg = opal_strerror(ret);
+        opal_show_help("help-mpi-runtime.txt",
+                       "mpi_init:startup:internal-failure", true,
+                       "MPI_INIT", "MPI_INIT", error, err_msg, ret);
+    }
+
+    return ret;
+}
+
+static int ompi_mpi_instance_cleanup_pml (void)
+{
+    /* call del_procs on all allocated procs even though some may not be known
+     * to the pml layer. the pml layer is expected to be resilient and ignore
+     * any unknown procs. */
+    size_t nprocs = 0;
+    ompi_proc_t **procs;
+
+    procs = ompi_proc_get_allocated (&nprocs);
+    MCA_PML_CALL(del_procs(procs, nprocs));
+    free(procs);
+
+    return OMPI_SUCCESS;
+}
+
+/**
+ * Static functions used to configure the interactions between the OPAL and
+ * the runtime.
+ */
+static char *_process_name_print_for_opal (const opal_process_name_t procname)
+{
+    ompi_process_name_t *rte_name = (ompi_process_name_t*)&procname;
+    return OMPI_NAME_PRINT(rte_name);
+}
+
+static int _process_name_compare (const opal_process_name_t p1, const opal_process_name_t p2)
+{
+    ompi_process_name_t *o1 = (ompi_process_name_t *) &p1;
+    ompi_process_name_t *o2 = (ompi_process_name_t *) &p2;
+    return ompi_rte_compare_name_fields(OMPI_RTE_CMP_ALL, o1, o2);
+}
+
+static int _convert_string_to_process_name (opal_process_name_t *name, const char* name_string)
+{
+    return ompi_rte_convert_string_to_process_name(name, name_string);
+}
+
+static int _convert_process_name_to_string (char **name_string, const opal_process_name_t *name)
+{
+    return ompi_rte_convert_process_name_to_string(name_string, name);
+}
+
+static int32_t ompi_mpi_instance_init_basic_count;
+static bool ompi_instance_basic_init;
+
+void ompi_mpi_instance_release (void)
+{
+    opal_mutex_lock (&instance_lock);
+
+    if (0 != --ompi_mpi_instance_init_basic_count) {
+        opal_mutex_unlock (&instance_lock);
+        return;
+    }
+
+    opal_argv_free (ompi_mpi_instance_pmix_psets);
+    ompi_mpi_instance_pmix_psets = NULL;
+
+    OBJ_DESTRUCT(&ompi_mpi_instance_null);
+
+    opal_finalize_cleanup_domain (&ompi_instance_basic_domain);
+    OBJ_DESTRUCT(&ompi_instance_basic_domain);
+
+    opal_finalize_util ();
+
+    opal_mutex_unlock (&instance_lock);
+}
+
+int ompi_mpi_instance_retain (void)
+{
+    int ret;
+
+    opal_mutex_lock (&instance_lock);
+
+    if (0 < ompi_mpi_instance_init_basic_count++) {
+        opal_mutex_unlock (&instance_lock);
+        return OMPI_SUCCESS;
+    }
+
+    /* Setup enough to check get/set MCA params */
+    if (OPAL_SUCCESS != (ret = opal_init_util (NULL, NULL))) {
+        opal_mutex_unlock (&instance_lock);
+        return ompi_instance_print_error ("ompi_mpi_instance_init: opal_init_util failed", ret);
+    }
+
+    ompi_instance_basic_init = true;
+
+    OBJ_CONSTRUCT(&ompi_instance_basic_domain, opal_finalize_domain_t);
+    opal_finalize_domain_init (&ompi_instance_basic_domain, "ompi_mpi_instance_retain");
+    opal_finalize_set_domain (&ompi_instance_basic_domain);
+
+    /* Setup f to c table */
+    OBJ_CONSTRUCT(&ompi_instance_f_to_c_table, opal_pointer_array_t);
+    if (OPAL_SUCCESS != opal_pointer_array_init (&ompi_instance_f_to_c_table, 8,
+                                                 OMPI_FORTRAN_HANDLE_MAX, 32)) {
+        opal_mutex_unlock (&instance_lock);
+        return OMPI_ERROR;
+    }
+
+    /* setup the default error handler on instance_null */
+    OBJ_CONSTRUCT(&ompi_mpi_instance_null, ompi_instance_t);
+    ompi_mpi_instance_null.instance.error_handler = &ompi_mpi_errors_return.eh;
+
+    /* Convince OPAL to use our naming scheme */
+    opal_process_name_print = _process_name_print_for_opal;
+    opal_compare_proc = _process_name_compare;
+    opal_convert_string_to_process_name = _convert_string_to_process_name;
+    opal_convert_process_name_to_string = _convert_process_name_to_string;
+    opal_proc_for_name = ompi_proc_for_name;
+
+    /* Register MCA variables */
+    if (OPAL_SUCCESS != (ret = ompi_mpi_register_params ())) {
+        opal_mutex_unlock (&instance_lock);
+        return ompi_instance_print_error ("ompi_mpi_init: ompi_register_mca_variables failed", ret);
+    }
+
+    /* initialize error handlers */
+    if (OMPI_SUCCESS != (ret = ompi_errhandler_init ())) {
+        opal_mutex_unlock (&instance_lock);
+        return ompi_instance_print_error ("ompi_errhandler_init() failed", ret);
+    }
+
+    /* initialize error codes */
+    if (OMPI_SUCCESS != (ret = ompi_mpi_errcode_init ())) {
+        opal_mutex_unlock (&instance_lock);
+        return ompi_instance_print_error ("ompi_mpi_errcode_init() failed", ret);
+    }
+
+    /* initialize internal error codes */
+    if (OMPI_SUCCESS != (ret = ompi_errcode_intern_init ())) {
+        opal_mutex_unlock (&instance_lock);
+        return ompi_instance_print_error ("ompi_errcode_intern_init() failed", ret);
+    }
+
+    /* initialize info */
+    if (OMPI_SUCCESS != (ret = ompi_mpiinfo_init ())) {
+        opal_mutex_unlock (&instance_lock);
+        return ompi_instance_print_error ("ompi_info_init() failed", ret);
+    }
+
+    ompi_instance_basic_init = false;
+
+    opal_mutex_unlock (&instance_lock);
+
+    return OMPI_SUCCESS;
+}
+
+static void fence_release(pmix_status_t status, void *cbdata)
+{
+    volatile bool *active = (volatile bool*)cbdata;
+    OPAL_ACQUIRE_OBJECT(active);
+    *active = false;
+    OPAL_POST_OBJECT(active);
+}
+
+static void evhandler_reg_callbk(pmix_status_t status,
+                                 size_t evhandler_ref,
+                                 void *cbdata)
+{
+    opal_pmix_lock_t *lock = (opal_pmix_lock_t*)cbdata;
+
+    lock->status = status;
+    lock->errhandler_ref = evhandler_ref;
+
+    OPAL_PMIX_WAKEUP_THREAD(lock);
+}
+
+static void evhandler_dereg_callbk(pmix_status_t status,
+                                 void *cbdata)
+{
+    opal_pmix_lock_t *lock = (opal_pmix_lock_t*)cbdata;
+    
+    lock->status = status;
+
+    OPAL_PMIX_WAKEUP_THREAD(lock);
+}       
+
+
+
+/**
+ * @brief Function that starts up the common components needed by all instances
+ */
+static int ompi_mpi_instance_init_common (int argc, char **argv)
+{
+    int ret;
+    ompi_proc_t **procs;
+    size_t nprocs;
+    volatile bool active;
+    bool background_fence = false;
+    pmix_info_t info[2];
+    pmix_status_t rc;
+    opal_pmix_lock_t mylock;
+    OMPI_TIMING_INIT(64);
+
+    // We intentionally don't use the OPAL timer framework here.  See
+    // https://github.com/open-mpi/ompi/issues/3003 for more details.
+    (void) opal_clock_gettime(&ompi_wtime_time_origin);
+
+    ret = ompi_mpi_instance_retain ();
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
+        return ret;
+    }
+
+    OBJ_CONSTRUCT(&ompi_instance_common_domain, opal_finalize_domain_t);
+    opal_finalize_domain_init (&ompi_instance_common_domain, "ompi_mpi_instance_init_common");
+    opal_finalize_set_domain (&ompi_instance_common_domain);
+
+    if (OPAL_SUCCESS != (ret = opal_arch_set_fortran_logical_size(sizeof(ompi_fortran_logical_t)))) {
+        return ompi_instance_print_error ("ompi_mpi_init: opal_arch_set_fortran_logical_size failed", ret);
+    }
+
+    /* _After_ opal_init_util() but _before_ orte_init(), we need to
+       set an MCA param that tells libevent that it's ok to use any
+       mechanism in libevent that is available on this platform (e.g.,
+       epoll and friends).  Per opal/event/event.s, we default to
+       select/poll -- but we know that MPI processes won't be using
+       pty's with the event engine, so it's ok to relax this
+       constraint and let any fd-monitoring mechanism be used. */
+
+    ret = mca_base_var_find("opal", "event", "*", "event_include");
+    if (ret >= 0) {
+        char *allvalue = "all";
+        /* We have to explicitly "set" the MCA param value here
+           because libevent initialization will re-register the MCA
+           param and therefore override the default. Setting the value
+           here puts the desired value ("all") in different storage
+           that is not overwritten if/when the MCA param is
+           re-registered. This is unless the user has specified a different
+           value for this MCA parameter. Make sure we check to see if the
+           default is specified before forcing "all" in case that is not what
+           the user desires. Note that we do *NOT* set this value as an
+           environment variable, just so that it won't be inherited by
+           any spawned processes and potentially cause unintended
+           side-effects with launching RTE tools... */
+        mca_base_var_set_value(ret, allvalue, 4, MCA_BASE_VAR_SOURCE_DEFAULT, NULL);
+    }
+
+    OMPI_TIMING_NEXT("initialization");
+
+    /* Setup RTE */
+    if (OMPI_SUCCESS != (ret = ompi_rte_init (&argc, &argv))) {
+        return ompi_instance_print_error ("ompi_mpi_init: ompi_rte_init failed", ret);
+    }
+
+    /* open the ompi hook framework */
+    for (int i = 0 ; ompi_framework_dependencies[i] ; ++i) {
+        ret = mca_base_framework_open (ompi_framework_dependencies[i], 0);
+        if (OPAL_UNLIKELY(OPAL_SUCCESS != ret)) {
+            char error_msg[256];
+            snprintf (error_msg, sizeof(error_msg), "mca_base_framework_open on %s_%s failed",
+                      ompi_framework_dependencies[i]->framework_project,
+                      ompi_framework_dependencies[i]->framework_name);
+            return ompi_instance_print_error (error_msg, ret);
+        }
+    }
+
+    OMPI_TIMING_NEXT("rte_init");
+    OMPI_TIMING_IMPORT_OPAL("orte_ess_base_app_setup");
+    OMPI_TIMING_IMPORT_OPAL("rte_init");
+
+    ompi_rte_initialized = true;
+    /* if we are oversubscribed, then set yield_when_idle
+     * accordingly */
+    if (ompi_mpi_oversubscribed) {
+        ompi_mpi_yield_when_idle = true;
+    }
+
+
+    /* Register the default errhandler callback  */
+    /* give it a name so we can distinguish it */
+    PMIX_INFO_LOAD(&info[0], PMIX_EVENT_HDLR_NAME, "MPI-Default", PMIX_STRING);
+    OPAL_PMIX_CONSTRUCT_LOCK(&mylock);
+    PMIx_Register_event_handler(NULL, 0, info, 1, ompi_errhandler_callback, evhandler_reg_callbk, (void*)&mylock);
+    OPAL_PMIX_WAIT_THREAD(&mylock);
+    rc = mylock.status;
+    ompi_default_pmix_err_handler = mylock.errhandler_ref;
+    OPAL_PMIX_DESTRUCT_LOCK(&mylock);
+    PMIX_INFO_DESTRUCT(&info[0]);
+    if (PMIX_SUCCESS != rc) {
+        ompi_default_pmix_err_handler = 0;
+        ret = opal_pmix_convert_status(rc);
+        return ret;
+    }
+
+    /* Register the ULFM errhandler callback  */
+    /* we want to go first */
+    PMIX_INFO_LOAD(&info[0], PMIX_EVENT_HDLR_PREPEND, NULL, PMIX_BOOL);
+    /* give it a name so we can distinguish it */
+    PMIX_INFO_LOAD(&info[1], PMIX_EVENT_HDLR_NAME, "ULFM-Event-handler", PMIX_STRING);
+    OPAL_PMIX_CONSTRUCT_LOCK(&mylock);
+    pmix_status_t codes[3] = {
+        PMIX_ERR_PROC_TERM_WO_SYNC,
+        PMIX_ERR_PROC_ABORTED_BY_SIG,
+        PMIX_ERR_PROC_ABORTED
+    };
+    PMIx_Register_event_handler(codes, 3, info, 2, ompi_errhandler_callback, evhandler_reg_callbk, (void*)&mylock);
+    OPAL_PMIX_WAIT_THREAD(&mylock);
+    rc = mylock.status;
+    ompi_ulfm_pmix_err_handler = mylock.errhandler_ref;
+    OPAL_PMIX_DESTRUCT_LOCK(&mylock);
+    PMIX_INFO_DESTRUCT(&info[0]);
+    PMIX_INFO_DESTRUCT(&info[1]);
+    if (PMIX_SUCCESS != rc) {
+        ompi_ulfm_pmix_err_handler = 0;
+        ret = opal_pmix_convert_status(rc);
+        return ret;
+    }
+
+    /* initialize MPI_INFO_ENV */
+    if (OMPI_SUCCESS != (ret = ompi_mpiinfo_init_env(0, NULL, &ompi_mpi_info_env.info))) {
+        return ompi_instance_print_error ("ompi_info_init_env() failed", ret);
+    }
+
+    /* declare our presence for interlib coordination, and
+     * register for callbacks when other libs declare. XXXXXX -- TODO -- figure out how
+     * to specify the thread level when different instances may request different levels. */
+    if (OMPI_SUCCESS != (ret = ompi_interlib_declare(MPI_THREAD_MULTIPLE, OMPI_IDENT_STRING))) {
+        return ompi_instance_print_error ("ompi_interlib_declare", ret);
+    }
+
+    /* initialize datatypes. This step should be done early as it will
+     * create the local convertor and local arch used in the proc
+     * init.
+     */
+    if (OMPI_SUCCESS != (ret = ompi_datatype_init())) {
+        return ompi_instance_print_error ("ompi_datatype_init() failed", ret);
+    }
+
+    /* Initialize OMPI procs */
+    if (OMPI_SUCCESS != (ret = ompi_proc_init())) {
+        return ompi_instance_print_error ("mca_proc_init() failed", ret);
+    }
+
+    /* Initialize the op framework. This has to be done *after*
+       ddt_init, but before mca_coll_base_open, since some collective
+       modules (e.g., the hierarchical coll component) may need ops in
+       their query function. */
+    if (OMPI_SUCCESS != (ret = ompi_op_base_find_available (OPAL_ENABLE_PROGRESS_THREADS, ompi_mpi_thread_multiple))) {
+        return ompi_instance_print_error ("ompi_op_base_find_available() failed", ret);
+    }
+
+    if (OMPI_SUCCESS != (ret = ompi_op_init())) {
+        return ompi_instance_print_error ("ompi_op_init() failed", ret);
+    }
+
+    /* In order to reduce the common case for MPI apps (where they
+       don't use MPI-2 IO or MPI-1/3 topology functions), the io and
+       topo frameworks are initialized lazily, at the first use of
+       relevant functions (e.g., MPI_FILE_*, MPI_CART_*, MPI_GRAPH_*),
+       so they are not opened here. */
+
+    /* Select which MPI components to use */
+
+    if (OPAL_SUCCESS != (ret = mca_smsc_base_select())) {
+        return ompi_instance_print_error ("mca_smsc_base_select() failed", ret);
+    }
+
+    if (OMPI_SUCCESS != (ret = mca_pml_base_select (OPAL_ENABLE_PROGRESS_THREADS, ompi_mpi_thread_multiple))) {
+        return ompi_instance_print_error ("mca_pml_base_select() failed", ret);
+    }
+
+    OMPI_TIMING_IMPORT_OPAL("orte_init");
+    OMPI_TIMING_NEXT("rte_init-commit");
+
+    /* exchange connection info - this function may also act as a barrier
+     * if data exchange is required. The modex occurs solely across procs
+     * in our job. If a barrier is required, the "modex" function will
+     * perform it internally */
+    rc = PMIx_Commit();
+    if (PMIX_SUCCESS != rc) {
+        ret = opal_pmix_convert_status(rc);
+        return ret;  /* TODO: need to fix this */
+    }
+
+   OMPI_TIMING_NEXT("commit");
+#if (OPAL_ENABLE_TIMING)
+    if (OMPI_TIMING_ENABLED && !opal_pmix_base_async_modex &&
+            opal_pmix_collect_all_data && !opal_process_info.is_singleton) {
+        if (PMIX_SUCCESS != (rc = PMIx_Fence(NULL, 0, NULL, 0))) {
+            ret = opal_pmix_convert_status(rc);
+            return ompi_instance_print_error ("timing: pmix-barrier-1 failed", ret);
+        }
+        OMPI_TIMING_NEXT("pmix-barrier-1");
+        if (PMIX_SUCCESS != (rc = PMIx_Fence(NULL, 0, NULL, 0))) {
+            return ompi_instance_print_error ("timing: pmix-barrier-2 failed", ret);
+        }
+        OMPI_TIMING_NEXT("pmix-barrier-2");
+    }
+#endif
+
+   if (! opal_process_info.is_singleton) {
+        if (opal_pmix_base_async_modex) {
+            /* if we are doing an async modex, but we are collecting all
+             * data, then execute the non-blocking modex in the background.
+             * All calls to modex_recv will be cached until the background
+             * modex completes. If collect_all_data is false, then we skip
+             * the fence completely and retrieve data on-demand from the
+             * source node.
+             */
+            if (opal_pmix_collect_all_data) {
+                /* execute the fence_nb in the background to collect
+                 * the data */
+                background_fence = true;
+                active = true;
+                OPAL_POST_OBJECT(&active);
+                PMIX_INFO_LOAD(&info[0], PMIX_COLLECT_DATA, &opal_pmix_collect_all_data, PMIX_BOOL);
+                if( PMIX_SUCCESS != (rc = PMIx_Fence_nb(NULL, 0, NULL, 0,
+                                                        fence_release,
+                                                        (void*)&active))) {
+                    ret = opal_pmix_convert_status(rc);
+                    return ompi_instance_print_error ("PMIx_Fence_nb() failed", ret);
+                }
+            }
+        } else {
+            /* we want to do the modex - we block at this point, but we must
+             * do so in a manner that allows us to call opal_progress so our
+             * event library can be cycled as we have tied PMIx to that
+             * event base */
+            active = true;
+            OPAL_POST_OBJECT(&active);
+            PMIX_INFO_LOAD(&info[0], PMIX_COLLECT_DATA, &opal_pmix_collect_all_data, PMIX_BOOL);
+            rc = PMIx_Fence_nb(NULL, 0, info, 1, fence_release, (void*)&active);
+            if( PMIX_SUCCESS != rc) {
+                ret = opal_pmix_convert_status(rc);
+                return ompi_instance_print_error ("PMIx_Fence() failed", ret);
+            }
+            /* cannot just wait on thread as we need to call opal_progress */
+            OMPI_LAZY_WAIT_FOR_COMPLETION(active);
+        }
+    }
+
+    OMPI_TIMING_NEXT("modex");
+
+    /* select buffered send allocator component to be used */
+    if (OMPI_SUCCESS != (ret = mca_pml_base_bsend_init ())) {
+        return ompi_instance_print_error ("mca_pml_base_bsend_init() failed", ret);
+    }
+
+    if (OMPI_SUCCESS != (ret = mca_coll_base_find_available (OPAL_ENABLE_PROGRESS_THREADS, ompi_mpi_thread_multiple))) {
+        return ompi_instance_print_error ("mca_coll_base_find_available() failed", ret);
+    }
+
+    if (OMPI_SUCCESS != (ret = ompi_osc_base_find_available (OPAL_ENABLE_PROGRESS_THREADS, ompi_mpi_thread_multiple))) {
+        return ompi_instance_print_error ("ompi_osc_base_find_available() failed", ret);
+    }
+
+    /* io and topo components are not selected here -- see comment
+       above about the io and topo frameworks being loaded lazily */
+
+    /* Initialize each MPI handle subsystem */
+    /* initialize requests */
+    if (OMPI_SUCCESS != (ret = ompi_request_init ())) {
+        return ompi_instance_print_error ("ompi_request_init() failed", ret);
+    }
+
+    if (OMPI_SUCCESS != (ret = ompi_message_init ())) {
+        return ompi_instance_print_error ("ompi_message_init() failed", ret);
+    }
+
+    /* initialize groups  */
+    if (OMPI_SUCCESS != (ret = ompi_group_init ())) {
+        return ompi_instance_print_error ("ompi_group_init() failed", ret);
+    }
+
+    ompi_mpi_instance_append_finalize (ompi_mpi_instance_cleanup_pml);
+
+    /* initialize communicator subsystem */
+    if (OMPI_SUCCESS != (ret = ompi_comm_init ())) {
+        opal_mutex_unlock (&instance_lock);
+        return ompi_instance_print_error ("ompi_comm_init() failed", ret);
+    }
+
+    /* Construct predefined keyvals */
+
+    if (OMPI_SUCCESS != (ret = ompi_attr_create_predefined_keyvals())) {
+        opal_mutex_unlock (&instance_lock);
+        return ompi_instance_print_error ("ompi_attr_create_predefined_keyvals() failed", ret);
+    }
+
+    if (mca_pml_base_requires_world ()) {
+        /* need to set up comm world for this instance -- XXX -- FIXME -- probably won't always
+         * be the case. */
+        if (OMPI_SUCCESS != (ret = ompi_comm_init_mpi3 ())) {
+            return ompi_instance_print_error ("ompi_comm_init_mpi3 () failed", ret);
+        }
+    }
+
+    /* initialize file handles */
+    if (OMPI_SUCCESS != (ret = ompi_file_init ())) {
+        return ompi_instance_print_error ("ompi_file_init() failed", ret);
+    }
+
+    /* initialize windows */
+    if (OMPI_SUCCESS != (ret = ompi_win_init ())) {
+        return ompi_instance_print_error ("ompi_win_init() failed", ret);
+    }
+
+    /* select part component to use */
+    if (OMPI_SUCCESS != (ret = mca_part_base_select (true, true))) {
+        return ompi_instance_print_error ("mca_part_base_select() failed", ret);
+    }
+
+    /* Setup the dynamic process management (DPM) subsystem */
+    if (OMPI_SUCCESS != (ret = ompi_dpm_init ())) {
+        return ompi_instance_print_error ("ompi_dpm_init() failed", ret);
+    }
+
+
+    /* identify the architectures of remote procs and setup
+     * their datatype convertors, if required
+     */
+    if (OMPI_SUCCESS != (ret = ompi_proc_complete_init())) {
+        return ompi_instance_print_error ("ompi_proc_complete_init failed", ret);
+    }
+
+    /* start PML/BTL's */
+    ret = MCA_PML_CALL(enable(true));
+    if( OMPI_SUCCESS != ret ) {
+        return ompi_instance_print_error ("PML control failed", ret);
+    }
+
+    /* some btls/mtls require we call add_procs with all procs in the job.
+     * since the btls/mtls have no visibility here it is up to the pml to
+     * convey this requirement */
+    if (mca_pml_base_requires_world ()) {
+        if (NULL == (procs = ompi_proc_world (&nprocs))) {
+            return ompi_instance_print_error ("ompi_proc_get_allocated () failed", ret);
+        }
+    } else {
+        /* add all allocated ompi_proc_t's to PML (below the add_procs limit this
+         * behaves identically to ompi_proc_world ()) */
+        if (NULL == (procs = ompi_proc_get_allocated (&nprocs))) {
+            return ompi_instance_print_error ("ompi_proc_get_allocated () failed", ret);
+        }
+    }
+
+    ret = MCA_PML_CALL(add_procs(procs, nprocs));
+    free(procs);
+    /* If we got "unreachable", then print a specific error message.
+       Otherwise, if we got some other failure, fall through to print
+       a generic message. */
+    if (OMPI_ERR_UNREACH == ret) {
+        opal_show_help("help-mpi-runtime.txt",
+                       "mpi_init:startup:pml-add-procs-fail", true);
+        return ret;
+    } else if (OMPI_SUCCESS != ret) {
+        return ompi_instance_print_error ("PML add procs failed", ret);
+    }
+
+    /* Determine the overall threadlevel support of all processes
+       in MPI_COMM_WORLD. This has to be done before calling
+       coll_base_comm_select, since some of the collective components
+       e.g. hierarch, might create subcommunicators. The threadlevel
+       requested by all processes is required in order to know
+       which cid allocation algorithm can be used. */
+    if (OMPI_SUCCESS != ( ret = ompi_comm_cid_init ())) {
+        return ompi_instance_print_error ("ompi_mpi_init: ompi_comm_cid_init failed", ret);
+    }
+
+    /* Do we need to wait for a debugger? */
+    ompi_rte_wait_for_debugger();
+
+    /* Next timing measurement */
+    OMPI_TIMING_NEXT("modex-barrier");
+
+    if (!opal_process_info.is_singleton) {
+        /* if we executed the above fence in the background, then
+         * we have to wait here for it to complete. However, there
+         * is no reason to do two barriers! */
+        if (background_fence) {
+            OMPI_LAZY_WAIT_FOR_COMPLETION(active);
+        } else if (!ompi_async_mpi_init) {
+            /* wait for everyone to reach this point - this is a hard
+             * barrier requirement at this time, though we hope to relax
+             * it at a later point */
+            bool flag = false;
+            active = true;
+            OPAL_POST_OBJECT(&active);
+            PMIX_INFO_LOAD(&info[0], PMIX_COLLECT_DATA, &flag, PMIX_BOOL);
+            if (PMIX_SUCCESS != (rc = PMIx_Fence_nb(NULL, 0, info, 1,
+                                                    fence_release, (void*)&active))) {
+                ret = opal_pmix_convert_status(rc);
+                return ompi_instance_print_error ("PMIx_Fence_nb() failed", ret);
+            }
+            OMPI_LAZY_WAIT_FOR_COMPLETION(active);
+        }
+    }
+
+    /* check for timing request - get stop time and report elapsed
+       time if so, then start the clock again */
+    OMPI_TIMING_NEXT("barrier");
+
+#if OPAL_ENABLE_PROGRESS_THREADS == 0
+    /* Start setting up the event engine for MPI operations.  Don't
+       block in the event library, so that communications don't take
+       forever between procs in the dynamic code.  This will increase
+       CPU utilization for the remainder of MPI_INIT when we are
+       blocking on RTE-level events, but may greatly reduce non-TCP
+       latency. */
+    opal_progress_set_event_flag(OPAL_EVLOOP_NONBLOCK);
+#endif
+
+    /* Undo OPAL calling opal_progress_event_users_increment() during
+       opal_init, to get better latency when not using TCP.  Do
+       this *after* dyn_init, as dyn init uses lots of RTE
+       communication and we don't want to hinder the performance of
+       that code. */
+    opal_progress_event_users_decrement();
+
+    /* see if yield_when_idle was specified - if so, use it */
+    opal_progress_set_yield_when_idle (ompi_mpi_yield_when_idle);
+
+    /* negative value means use default - just don't do anything */
+    if (ompi_mpi_event_tick_rate >= 0) {
+        opal_progress_set_event_poll_rate(ompi_mpi_event_tick_rate);
+    }
+
+    /* At this point, we are fully configured and in MPI mode.  Any
+       communication calls here will work exactly like they would in
+       the user's code.  Setup the connections between procs and warm
+       them up with simple sends, if requested */
+
+    if (OMPI_SUCCESS != (ret = ompi_mpiext_init())) {
+        return ompi_instance_print_error ("ompi_mpiext_init", ret);
+    }
+
+    /* Initialize the registered datarep list to be empty */
+    OBJ_CONSTRUCT(&ompi_registered_datareps, opal_list_t);
+
+    /* Initialize the arrays used to store the F90 types returned by the
+     *  MPI_Type_create_f90_XXX functions.
+     */
+    OBJ_CONSTRUCT( &ompi_mpi_f90_integer_hashtable, opal_hash_table_t);
+    opal_hash_table_init(&ompi_mpi_f90_integer_hashtable, 16 /* why not? */);
+
+    OBJ_CONSTRUCT( &ompi_mpi_f90_real_hashtable, opal_hash_table_t);
+    opal_hash_table_init(&ompi_mpi_f90_real_hashtable, FLT_MAX_10_EXP);
+
+    OBJ_CONSTRUCT( &ompi_mpi_f90_complex_hashtable, opal_hash_table_t);
+    opal_hash_table_init(&ompi_mpi_f90_complex_hashtable, FLT_MAX_10_EXP);
+
+    return OMPI_SUCCESS;
+}
+
+int ompi_mpi_instance_init (int ts_level,  opal_info_t *info, ompi_errhandler_t *errhandler, ompi_instance_t **instance, int argc, char **argv)
+{
+    ompi_instance_t *new_instance;
+    int ret;
+
+    *instance = &ompi_mpi_instance_null.instance;
+
+    /* If thread support was enabled, then setup OPAL to allow for them by default. This must be done
+     * early to prevent a race condition that can occur with orte_init(). */
+    if (ts_level == MPI_THREAD_MULTIPLE) {
+        opal_set_using_threads(true);
+    }
+
+    opal_mutex_lock (&instance_lock);
+    if (0 == opal_atomic_fetch_add_32 (&ompi_instance_count, 1)) {
+        ret = ompi_mpi_instance_init_common (argc, argv);
+        if (OPAL_UNLIKELY(OPAL_SUCCESS != ret)) {
+            opal_mutex_unlock (&instance_lock);
+            return ret;
+        }
+    }
+
+    new_instance = OBJ_NEW(ompi_instance_t);
+    if (OPAL_UNLIKELY(NULL == new_instance)) {
+        if (0 == opal_atomic_add_fetch_32 (&ompi_instance_count, -1)) {
+            // We can't do anything if an error occurs here because
+            // we're already in an error path, so don't even bother to
+            // look at the return value.
+            (void) ompi_mpi_instance_finalize_common ();
+        }
+        opal_mutex_unlock (&instance_lock);
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    new_instance->error_handler = errhandler;
+    OBJ_RETAIN(new_instance->error_handler);
+
+    /* Copy info if there is one. */
+    if (OPAL_UNLIKELY(NULL != info)) {
+        opal_cstring_t *memkind_requested;
+        ompi_info_memkind_assert_type type;
+        int flag;
+        
+        new_instance->super.s_info = OBJ_NEW(opal_info_t);
+        opal_info_get(info, "mpi_memory_alloc_kinds", &memkind_requested, &flag);
+        if (1 == flag) {
+            char *memkind_provided;
+            ompi_info_memkind_process (memkind_requested->string, &memkind_provided, &type);
+            opal_infosubscribe_subscribe (&new_instance->super, "mpi_memory_alloc_kinds",
+                                          memkind_provided, ompi_info_memkind_cb);
+            free (memkind_provided);
+            OBJ_RELEASE(memkind_requested);
+        }
+
+        if (info) {
+            opal_info_dup(info, &new_instance->super.s_info);
+        }
+    }
+
+    *instance = new_instance;
+    opal_mutex_unlock (&instance_lock);
+
+    return OMPI_SUCCESS;
+}
+
+static int ompi_mpi_instance_finalize_common (void)
+{
+    uint32_t key;
+    ompi_datatype_t *datatype;
+    int ret;
+    opal_pmix_lock_t mylock;
+
+    /* As finalize is the last legal MPI call, we are allowed to force the release
+     * of the user buffer used for bsend, before going anywhere further.
+     */
+    (void) mca_pml_base_bsend_detach (NULL, NULL);
+
+    /* Shut down any bindings-specific issues: C++, F77, F90 */
+
+    /* Remove all memory associated by MPI_REGISTER_DATAREP (per
+       MPI-2:9.5.3, there is no way for an MPI application to
+       *un*register datareps, but we don't want the OMPI layer causing
+       memory leaks). */
+    OPAL_LIST_DESTRUCT(&ompi_registered_datareps);
+
+    /* Remove all F90 types from the hash tables */
+    OPAL_HASH_TABLE_FOREACH(key, uint32, datatype, &ompi_mpi_f90_integer_hashtable)
+        OBJ_RELEASE(datatype);
+    OBJ_DESTRUCT(&ompi_mpi_f90_integer_hashtable);
+    OPAL_HASH_TABLE_FOREACH(key, uint32, datatype, &ompi_mpi_f90_real_hashtable)
+        OBJ_RELEASE(datatype);
+    OBJ_DESTRUCT(&ompi_mpi_f90_real_hashtable);
+    OPAL_HASH_TABLE_FOREACH(key, uint32, datatype, &ompi_mpi_f90_complex_hashtable)
+        OBJ_RELEASE(datatype);
+    OBJ_DESTRUCT(&ompi_mpi_f90_complex_hashtable);
+
+    /* If requested, print out a list of memory allocated by ALLOC_MEM
+       but not freed by FREE_MEM */
+    if (0 != ompi_debug_show_mpi_alloc_mem_leaks) {
+        mca_mpool_base_tree_print (ompi_debug_show_mpi_alloc_mem_leaks);
+    }
+
+    opal_finalize_cleanup_domain (&ompi_instance_common_domain);
+
+    if (NULL != ompi_mpi_main_thread) {
+        OBJ_RELEASE(ompi_mpi_main_thread);
+        ompi_mpi_main_thread = NULL;
+    }
+
+    if (0 != ompi_default_pmix_err_handler) {
+        OPAL_PMIX_CONSTRUCT_LOCK(&mylock);
+        PMIx_Deregister_event_handler(ompi_default_pmix_err_handler, evhandler_dereg_callbk, &mylock);
+        OPAL_PMIX_WAIT_THREAD(&mylock);
+        OPAL_PMIX_DESTRUCT_LOCK(&mylock);
+        ompi_default_pmix_err_handler = 0;
+    }
+
+    if (0 != ompi_ulfm_pmix_err_handler) {
+        OPAL_PMIX_CONSTRUCT_LOCK(&mylock);
+        PMIx_Deregister_event_handler(ompi_ulfm_pmix_err_handler, evhandler_dereg_callbk, &mylock);
+        OPAL_PMIX_WAIT_THREAD(&mylock);
+        OPAL_PMIX_DESTRUCT_LOCK(&mylock);
+        ompi_ulfm_pmix_err_handler = 0;
+    }
+
+    /* Leave the RTE */
+    if (OMPI_SUCCESS != (ret = ompi_rte_finalize())) {
+        return ret;
+    }
+
+    ompi_rte_initialized = false;
+
+    for (int i = 0 ; ompi_lazy_frameworks[i] ; ++i) {
+        if (0 < ompi_lazy_frameworks[i]->framework_refcnt) {
+            /* May have been "opened" multiple times. We want it closed now! */
+            ompi_lazy_frameworks[i]->framework_refcnt = 1;
+
+            ret = mca_base_framework_close (ompi_lazy_frameworks[i]);
+            if (OPAL_UNLIKELY(OPAL_SUCCESS != ret)) {
+                return ret;
+            }
+        }
+    }
+
+    int last_framework = 0;
+    for (int i = 0 ; ompi_framework_dependencies[i] ; ++i) {
+        last_framework = i;
+    }
+
+    for (int j = last_framework ; j >= 0; --j) {
+        ret = mca_base_framework_close (ompi_framework_dependencies[j]);
+        if (OPAL_UNLIKELY(OPAL_SUCCESS != ret)) {
+            return ret;
+        }
+    }
+
+    ompi_proc_finalize();
+
+    ompi_mpi_instance_release ();
+
+    return OMPI_SUCCESS;
+}
+
+int ompi_mpi_instance_finalize (ompi_instance_t **instance)
+{
+    int ret = OMPI_SUCCESS;
+
+    OBJ_RELEASE(*instance);
+
+    opal_mutex_lock (&instance_lock);
+    if (0 == opal_atomic_add_fetch_32 (&ompi_instance_count, -1)) {
+        ret = ompi_mpi_instance_finalize_common ();
+    }
+    opal_mutex_unlock (&instance_lock);
+
+    *instance = &ompi_mpi_instance_null.instance;
+
+    return ret;
+}
+
+static void ompi_instance_get_num_psets_complete (pmix_status_t status, 
+		                                  pmix_info_t *info,
+		                                  size_t ninfo,
+                                                  void *cbdata, 
+                                                  pmix_release_cbfunc_t release_fn,
+                                                  void *release_cbdata)
+{
+    size_t n;
+    pmix_status_t rc;
+    size_t sz;
+    size_t num_pmix_psets = 0, *num_pmix_psets_ptr;
+    char *pset_names = NULL;
+
+    opal_pmix_lock_t *lock = (opal_pmix_lock_t *) cbdata;
+
+    num_pmix_psets_ptr = &num_pmix_psets;
+
+    for (n=0; n < ninfo; n++) {
+        if (0 == strcmp(info[n].key,PMIX_QUERY_NUM_PSETS)) {
+            PMIX_VALUE_UNLOAD(rc,
+                              &info[n].value,
+                              (void **)&num_pmix_psets_ptr,
+                              &sz);
+            if (rc != PMIX_SUCCESS) {
+                opal_argv_free (ompi_mpi_instance_pmix_psets);
+                ompi_mpi_instance_pmix_psets = NULL;
+                goto done;
+            }
+            if (num_pmix_psets != ompi_mpi_instance_num_pmix_psets) {
+                opal_argv_free (ompi_mpi_instance_pmix_psets);
+                ompi_mpi_instance_pmix_psets = NULL;
+            }
+            ompi_mpi_instance_num_pmix_psets = num_pmix_psets;
+        } else if (0 == strcmp (info[n].key, PMIX_QUERY_PSET_NAMES)) {
+            if (ompi_mpi_instance_pmix_psets) {
+                opal_argv_free (ompi_mpi_instance_pmix_psets);
+            }
+            if (NULL != pset_names) {
+                free(pset_names);
+                pset_names = NULL;
+            }
+            PMIX_VALUE_UNLOAD(rc,
+                              &info[n].value,
+                              (void **)&pset_names,
+                              &sz);
+            if (rc != PMIX_SUCCESS) {
+                opal_argv_free (ompi_mpi_instance_pmix_psets);
+                ompi_mpi_instance_pmix_psets = NULL;
+                goto done;
+            }
+            ompi_mpi_instance_pmix_psets = opal_argv_split (pset_names, ',');
+            ompi_mpi_instance_num_pmix_psets = opal_argv_count (ompi_mpi_instance_pmix_psets);
+        }
+    }
+
+done:
+    if (NULL != pset_names) {
+        free(pset_names);
+    }
+
+    if (NULL != release_fn) {
+        release_fn(release_cbdata);
+    }
+    OPAL_PMIX_WAKEUP_THREAD(lock);
+}
+
+static void ompi_instance_refresh_pmix_psets (const char *key)
+{
+    pmix_status_t rc;
+    pmix_query_t query;
+    opal_pmix_lock_t lock;
+    bool refresh = true;
+
+    opal_mutex_lock (&instance_lock);
+
+    PMIX_QUERY_CONSTRUCT(&query);
+    PMIX_ARGV_APPEND(rc, query.keys, key);
+    PMIX_INFO_CREATE(query.qualifiers, 1);
+    query.nqual = 1;
+    PMIX_INFO_LOAD(&query.qualifiers[0], PMIX_QUERY_REFRESH_CACHE, &refresh, PMIX_BOOL);
+
+    OPAL_PMIX_CONSTRUCT_LOCK(&lock);
+
+    /*
+     * TODO: need to handle this better
+     */
+    if (PMIX_SUCCESS != (rc = PMIx_Query_info_nb(&query, 1, 
+                                                 ompi_instance_get_num_psets_complete,
+                                                 (void*)&lock))) {
+       opal_mutex_unlock (&instance_lock);
+       return;
+    }
+
+    OPAL_PMIX_WAIT_THREAD(&lock);
+    OPAL_PMIX_DESTRUCT_LOCK(&lock);
+
+    opal_mutex_unlock (&instance_lock);
+}
+
+
+int ompi_instance_get_num_psets (ompi_instance_t *instance, int *npset_names)
+{
+    ompi_instance_refresh_pmix_psets (PMIX_QUERY_NUM_PSETS);
+    *npset_names = ompi_instance_builtin_count + ompi_mpi_instance_num_pmix_psets;
+
+    return OMPI_SUCCESS;
+}
+
+int ompi_instance_get_nth_pset (ompi_instance_t *instance, int n, int *len, char *pset_name)
+{
+    if (NULL == ompi_mpi_instance_pmix_psets ||
+        (size_t) n >= (ompi_instance_builtin_count + ompi_mpi_instance_num_pmix_psets)) {
+        ompi_instance_refresh_pmix_psets (PMIX_QUERY_PSET_NAMES);
+    }
+
+    if ((size_t) n >= (ompi_instance_builtin_count + ompi_mpi_instance_num_pmix_psets) || n < 0) {
+        return OMPI_ERR_BAD_PARAM;
+    }
+
+    if (0 == *len) {
+        if (n < ompi_instance_builtin_count) {
+            *len = strlen(ompi_instance_builtin_psets[n]) + 1;
+        } else {
+            *len = strlen (ompi_mpi_instance_pmix_psets[n - ompi_instance_builtin_count]) + 1;
+        }
+        return OMPI_SUCCESS;
+    }
+
+    if (n < ompi_instance_builtin_count) {
+        strncpy (pset_name, ompi_instance_builtin_psets[n], *len);
+    } else {
+        strncpy (pset_name, ompi_mpi_instance_pmix_psets[n - ompi_instance_builtin_count], *len);
+    }
+
+    return OMPI_SUCCESS;
+}
+
+static int ompi_instance_group_world (ompi_instance_t *instance, ompi_group_t **group_out)
+{
+    ompi_group_t *group;
+    size_t size;
+
+    size = ompi_process_info.num_procs;
+
+    group = ompi_group_allocate (NULL,size);
+    if (OPAL_UNLIKELY(NULL == group)) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    for (size_t i = 0 ; i < size ; ++i) {
+        opal_process_name_t name = {.vpid = i, .jobid = OMPI_PROC_MY_NAME->jobid};
+        /* look for existing ompi_proc_t that matches this name */
+        group->grp_proc_pointers[i] = (ompi_proc_t *) ompi_proc_lookup (name);
+        if (NULL == group->grp_proc_pointers[i]) {
+            /* set sentinel value */
+            group->grp_proc_pointers[i] = (ompi_proc_t *) ompi_proc_name_to_sentinel (name);
+        } else {
+            OBJ_RETAIN (group->grp_proc_pointers[i]);
+        }
+    }
+
+    ompi_set_group_rank (group, ompi_proc_local());
+
+    group->grp_instance = instance;
+
+    *group_out = group;
+    return OMPI_SUCCESS;
+}
+
+static int ompi_instance_group_shared (ompi_instance_t *instance, ompi_group_t **group_out)
+{
+    ompi_group_t *group;
+    opal_process_name_t wildcard_rank;
+    int ret;
+    size_t size;
+    char **peers;
+    char *val;
+
+    /* Find out which processes are local */
+    wildcard_rank.jobid = OMPI_PROC_MY_NAME->jobid;
+    wildcard_rank.vpid = OMPI_NAME_WILDCARD->vpid;
+
+    OPAL_MODEX_RECV_VALUE(ret, PMIX_LOCAL_PEERS, &wildcard_rank, &val, PMIX_STRING);
+    if (OPAL_SUCCESS != ret || NULL == val) {
+        return OMPI_ERROR;
+    }
+
+    peers = opal_argv_split(val, ',');
+    free (val);
+    if (OPAL_UNLIKELY(NULL == peers)) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    size = opal_argv_count (peers);
+
+    group = ompi_group_allocate (NULL,size);
+    if (OPAL_UNLIKELY(NULL == group)) {
+        opal_argv_free (peers);
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    for (size_t i = 0 ; NULL != peers[i] ; ++i) {
+        opal_process_name_t name = {.vpid = strtoul(peers[i], NULL, 10), .jobid = OMPI_PROC_MY_NAME->jobid};
+        /* look for existing ompi_proc_t that matches this name */
+        group->grp_proc_pointers[i] = (ompi_proc_t *) ompi_proc_lookup (name);
+        if (NULL == group->grp_proc_pointers[i]) {
+            /* set sentinel value */
+            group->grp_proc_pointers[i] = (ompi_proc_t *) ompi_proc_name_to_sentinel (name);
+        } else {
+            OBJ_RETAIN (group->grp_proc_pointers[i]);
+        }
+    }
+
+    opal_argv_free (peers);
+
+    /* group is dense */
+    ompi_set_group_rank (group, ompi_proc_local());
+
+    group->grp_instance = instance;
+
+    *group_out = group;
+    return OMPI_SUCCESS;
+}
+
+static int ompi_instance_group_self (ompi_instance_t *instance, ompi_group_t **group_out)
+{
+    ompi_group_t *group;
+    size_t size;
+
+    group = OBJ_NEW(ompi_group_t);
+    if (OPAL_UNLIKELY(NULL == group)) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    group->grp_proc_pointers = ompi_proc_self(&size);
+    group->grp_my_rank       = 0;
+    group->grp_proc_count   = size;
+
+    /* group is dense */
+    OMPI_GROUP_SET_DENSE (group);
+
+    group->grp_instance = instance;
+
+    *group_out = group;
+    return OMPI_SUCCESS;
+}
+
+static int ompi_instance_group_pmix_pset (ompi_instance_t *instance, const char *pset_name, ompi_group_t **group_out)
+{
+    int ret = OMPI_SUCCESS;
+    size_t i,n;
+    bool isnew, try_again = false, refresh = true;
+    pmix_status_t rc;
+    ompi_group_t *group = NULL;
+    pmix_query_t query;
+    pmix_info_t *info = NULL;
+    size_t ninfo;
+    opal_process_name_t pname;
+
+    PMIX_QUERY_CONSTRUCT(&query);
+    PMIX_ARGV_APPEND(rc, query.keys, PMIX_QUERY_PSET_MEMBERSHIP);
+    PMIX_INFO_CREATE(query.qualifiers, 1);
+    query.nqual = 1;
+    PMIX_INFO_LOAD(&query.qualifiers[0], PMIX_PSET_NAME, pset_name, PMIX_STRING);
+
+    /*
+     * First try finding in the local PMIx cache, if not found, try a refresh
+     */
+fn_try_again:
+    if (PMIX_SUCCESS != (rc = PMIx_Query_info(&query, 1, &info, &ninfo)) || 0 == ninfo) {
+        if ((PMIX_ERR_NOT_FOUND == rc) && (false == try_again)) {
+            try_again = true;
+            PMIX_QUERY_DESTRUCT(&query);
+            PMIX_QUERY_CONSTRUCT(&query);
+            PMIX_ARGV_APPEND(rc, query.keys, PMIX_QUERY_PSET_MEMBERSHIP);
+            PMIX_INFO_CREATE(query.qualifiers, 2);
+            PMIX_INFO_LOAD(&query.qualifiers[0], PMIX_PSET_NAME, pset_name, PMIX_STRING);
+            PMIX_INFO_LOAD(&query.qualifiers[1], PMIX_QUERY_REFRESH_CACHE, &refresh, PMIX_BOOL);
+            query.nqual = 2;
+            goto fn_try_again;
+        }	    
+    }
+
+    if (PMIX_SUCCESS != rc) {
+        char msg_string[1024];
+        switch (rc) {
+        case PMIX_ERR_NOT_FOUND:
+            ret = MPI_ERR_ARG; /* pset_name not valid */
+            break;
+        case PMIX_ERR_UNREACH:
+            sprintf(msg_string,"PMIx server unreachable");
+            opal_show_help("help-comm.txt",
+                           "MPI function not supported",
+                           true,
+                           "MPI_Group_from_session_pset",
+                           msg_string);
+            ret = MPI_ERR_UNSUPPORTED_OPERATION;
+            break;
+        case PMIX_ERR_NOT_SUPPORTED:
+            sprintf(msg_string,"PMIx server does not support PMIX_QUERY_PSET_MEMBERSHIP operation");
+            opal_show_help("help-comm.txt",
+                           "MPI function not supported",
+                           true,
+                           "MPI_Group_from_session_pset",
+                           msg_string);
+            ret = MPI_ERR_UNSUPPORTED_OPERATION;
+            break;
+        default:
+            ret = opal_pmix_convert_status(rc);
+            break;
+        }
+        ompi_instance_print_error ("PMIx_Query_info() failed", ret);
+        goto fn_w_query;
+    }
+
+    for(n = 0; n < ninfo; n++){
+        if(0 == strcmp(info[n].key, PMIX_QUERY_PSET_MEMBERSHIP)){
+            
+            pmix_data_array_t *data_array = info[n].value.data.darray;
+            pmix_proc_t *members_array = (pmix_proc_t*) data_array->array;
+
+            group = ompi_group_allocate (NULL, data_array->size);
+            if (OPAL_UNLIKELY(NULL == group)) {
+                ret = OMPI_ERR_OUT_OF_RESOURCE;
+                goto fn_w_info;
+            }
+
+            for(i = 0; i < data_array->size; i++){
+                OPAL_PMIX_CONVERT_PROCT(ret, &pname, &members_array[i]);
+		if (OPAL_SUCCESS == rc) {
+                    group->grp_proc_pointers[i] = ompi_proc_find_and_add(&pname,&isnew);
+                } else {
+                    ompi_instance_print_error ("OPAL_PMIX_CONVERT_PROCT failed %d", ret);
+                    ompi_group_free(&group);
+                    goto fn_w_info;
+                }
+            }
+            break;
+        }
+    }
+
+    if (NULL != group) {
+        ompi_set_group_rank (group, ompi_proc_local());
+        group->grp_instance = instance;
+        *group_out = group;
+    } else {
+        ret = OMPI_ERR_NOT_FOUND;
+    }
+
+fn_w_info:
+    PMIX_INFO_DESTRUCT(info);
+fn_w_query:
+    PMIX_QUERY_DESTRUCT(&query);
+
+    return ret;
+}
+
+static int ompi_instance_get_pmix_pset_size (ompi_instance_t *instance, const char *pset_name, size_t *size_out)
+{
+    pmix_status_t rc;
+    pmix_proc_t p;
+    pmix_value_t *pval = NULL;
+    size_t size = 0;
+    char *stmp = NULL;
+
+    for (size_t i = 0 ; i < ompi_process_info.num_procs ; ++i) {
+        opal_process_name_t name = {.vpid = i, .jobid = OMPI_PROC_MY_NAME->jobid};
+
+        OPAL_PMIX_CONVERT_NAME(&p, &name);
+        rc = PMIx_Get(&p, PMIX_PSET_NAME, NULL, 0, &pval);
+        if (OPAL_UNLIKELY(PMIX_SUCCESS != rc)) {
+            return opal_pmix_convert_status(rc);
+        }
+
+        PMIX_VALUE_UNLOAD(rc,
+                          pval,
+                          (void **)&stmp,
+                          &size);
+
+        size += (0 == strcmp (pset_name, stmp));
+        PMIX_VALUE_RELEASE(pval);
+        free(stmp);
+        stmp = NULL;
+
+        ++size;
+    }
+
+    *size_out = size;
+
+    return OMPI_SUCCESS;
+}
+
+int ompi_group_from_pset (ompi_instance_t *instance, const char *pset_name, ompi_group_t **group_out)
+{
+    if (NULL == group_out) {
+        return OMPI_ERR_BAD_PARAM;
+    }
+
+    if (0 == strncmp (pset_name, "mpi://", 6)) {
+        pset_name += 6;
+        if (0 == strcasecmp (pset_name, "WORLD")) {
+            return ompi_instance_group_world (instance, group_out);
+        }
+        if (0 == strcasecmp (pset_name, "SELF")) {
+            return ompi_instance_group_self (instance, group_out);
+        }
+    }
+
+    if (0 == strncmp (pset_name, "mpix://", 7)) {
+        pset_name += 7;
+        if (0 == strcasecmp (pset_name, "SHARED")) {
+            return ompi_instance_group_shared (instance, group_out);
+        }
+    }
+
+    return ompi_instance_group_pmix_pset (instance, pset_name, group_out);
+}
+
+int ompi_instance_get_pset_info (ompi_instance_t *instance, const char *pset_name, opal_info_t **info_used)
+{
+    ompi_info_t *info = ompi_info_allocate ();
+    char tmp[16];
+    size_t size = 0UL;
+    int ret = OMPI_SUCCESS ;
+
+    *info_used = (opal_info_t *) MPI_INFO_NULL;
+
+    if (OPAL_UNLIKELY(NULL == info)) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    if (0 == strncmp (pset_name, "mpi://", 6)) {
+        pset_name += 6;
+        if (0 == strcasecmp (pset_name, "WORLD")) {
+            size = ompi_process_info.num_procs;
+        } else if (0 == strcasecmp (pset_name, "SELF")) {
+            size = 1;
+        } 
+    } else if (0 == strncmp (pset_name, "mpix://", 7)) {
+        pset_name += 7;
+        if (0 == strcasecmp (pset_name, "SHARED")) {
+            size = ompi_process_info.num_local_peers + 1;
+        }
+    } else {
+        ret = ompi_instance_get_pmix_pset_size (instance, pset_name, &size);
+    }
+
+    if (OMPI_SUCCESS == ret) {
+        snprintf (tmp, 16, "%" PRIsize_t, size);
+        ret = opal_info_set (&info->super, MPI_INFO_KEY_SESSION_PSET_SIZE, tmp);
+        if (OPAL_UNLIKELY(OPAL_SUCCESS != ret)) {
+            ompi_info_free (&info);
+            return ret;
+        }
+        *info_used = &info->super;
+    } else { 
+        ompi_info_free (&info);
+    }
+
+    return ret;
+}
diff --git a/ompi/instance/instance.h b/ompi/instance/instance.h
new file mode 100644
index 00000000000..f72f19525ae
--- /dev/null
+++ b/ompi/instance/instance.h
@@ -0,0 +1,167 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2018      Triad National Security, LLC.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#if !defined(OMPI_INSTANCE_H)
+#define OMPI_INSTANCE_H
+
+#include "opal/class/opal_object.h"
+#include "opal/class/opal_hash_table.h"
+#include "opal/util/info_subscriber.h"
+#include "ompi/errhandler/errhandler.h"
+#include "opal/mca/threads/mutex.h"
+#include "ompi/communicator/comm_request.h"
+
+#include "mpi.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/info/info.h"
+#include "ompi/proc/proc.h"
+
+struct ompi_group_t;
+
+struct ompi_instance_t {
+    opal_infosubscriber_t  super;
+    opal_mutex_t           s_lock;
+    int                    i_thread_level;
+    char                   i_name[MPI_MAX_OBJECT_NAME];
+    uint32_t               i_flags;
+
+    /* Attributes */
+    opal_hash_table_t     *i_keyhash;
+
+    /* index in Fortran <-> C translation array (for when I get around
+     * to implementing fortran support-- UGH) */
+    int                    i_f_to_c_index;
+
+    ompi_errhandler_t     *error_handler;
+    ompi_errhandler_type_t errhandler_type;
+};
+
+typedef struct ompi_instance_t ompi_instance_t;
+
+OBJ_CLASS_DECLARATION(ompi_instance_t);
+
+/* Define for the preallocated size of the predefined handle.
+ * Note that we are using a pointer type as the base memory chunk
+ * size so when the bitness changes the size of the handle changes.
+ * This is done so we don't end up needing a structure that is
+ * incredibly larger than necessary because of the bitness.
+ *
+ * This padding mechanism works as a (likely) compile time check for when the
+ * size of the ompi_communicator_t exceeds the predetermined size of the
+ * ompi_predefined_communicator_t. It also allows us to change the size of
+ * the ompi_communicator_t without impacting the size of the
+ * ompi_predefined_communicator_t structure for some number of additions.
+ *
+ * Note: we used to define the PAD as a multiple of sizeof(void*).
+ * However, this makes a different size PAD, depending on
+ * sizeof(void*).  In some cases
+ * (https://github.com/open-mpi/ompi/issues/3610), 32 bit builds can
+ * run out of space when 64 bit builds are still ok.  So we changed to
+ * use just a naked byte size.  As a rule of thumb, however, the size
+ * should probably still be a multiple of 8 so that it has the
+ * possibility of being nicely aligned.
+ *
+ * As an example:
+ * If the size of ompi_communicator_t is less than the size of the _PAD then
+ * the _PAD ensures that the size of the ompi_predefined_communicator_t is
+ * whatever size is defined below in the _PAD macro.
+ * However, if the size of the ompi_communicator_t grows larger than the _PAD
+ * (say by adding a few more function pointers to the structure) then the
+ * 'padding' variable will be initialized to a large number often triggering
+ * a 'array is too large' compile time error. This signals two things:
+ * 1) That the _PAD should be increased.
+ * 2) That users need to be made aware of the size change for the
+ *    ompi_predefined_communicator_t structure.
+ *
+ * Q: So you just made a change to communicator structure, do you need to adjust
+ * the PREDEFINED_COMMUNICATOR_PAD macro?
+ * A: Most likely not, but it would be good to check.
+ */
+#define PREDEFINED_INSTANCE_PAD 512
+
+struct ompi_predefined_instance_t {
+    ompi_instance_t instance;
+    char padding[PREDEFINED_INSTANCE_PAD - sizeof(ompi_instance_t)];
+};
+typedef struct ompi_predefined_instance_t ompi_predefined_instance_t;
+
+/**
+ * @brief NULL instance
+ */
+OMPI_DECLSPEC extern ompi_predefined_instance_t ompi_mpi_instance_null;
+
+OMPI_DECLSPEC extern opal_pointer_array_t ompi_instance_f_to_c_table;
+
+extern ompi_instance_t *ompi_mpi_instance_default;
+
+/**
+ * @brief Bring up the bare minimum infrastructure to support pre-session_init functions.
+ *
+ * List of subsystems initialized:
+ *  - OPAL (including class system)
+ *  - Error handlers
+ *  - MPI Info
+ */
+int ompi_mpi_instance_retain (void);
+
+/**
+ * @brief Release (and possibly teardown) pre-session_init infrastructure.
+ */
+void ompi_mpi_instance_release (void);
+
+/**
+ * @brief Create a new MPI instance
+ *
+ * @param[in]    ts_level  thread support level (see mpi.h)
+ * @param[in]    info      info object
+ * @param[in]    errhander errhandler to set on the instance
+ */
+OMPI_DECLSPEC int ompi_mpi_instance_init (int ts_level, opal_info_t *info, ompi_errhandler_t *errhandler,
+                                          ompi_instance_t **instance, int argc, char **argv);
+
+/**
+ * @brief Destroy an MPI instance and set it to MPI_SESSION_NULL
+ */
+OMPI_DECLSPEC int ompi_mpi_instance_finalize (ompi_instance_t **instance);
+
+
+/**
+ * @brief Add a function to the finalize chain. Note this function will be called
+ *        when the last instance has been destroyed.
+ */
+#define ompi_mpi_instance_append_finalize opal_finalize_register_cleanup
+
+/**
+ * @brief Get an MPI group object for a named process set.
+ *
+ * @param[in] instance   MPI instance (session)
+ * @param[in] pset_name  Name of process set (includes mpi://world, mpi://self)
+ * @param[out group_out  New MPI group
+ */
+OMPI_DECLSPEC int ompi_group_from_pset (ompi_instance_t *instance, const char *pset_name, struct ompi_group_t **group_out);
+
+OMPI_DECLSPEC int ompi_instance_get_num_psets (ompi_instance_t *instance, int *npset_names);
+OMPI_DECLSPEC int ompi_instance_get_nth_pset (ompi_instance_t *instance, int n, int *len, char *pset_name);
+OMPI_DECLSPEC int ompi_instance_get_pset_info (ompi_instance_t *instance, const char *pset_name, opal_info_t **info_used);
+
+/**
+ * @brief current number of active instances
+ */
+extern opal_atomic_int32_t ompi_instance_count;
+
+static inline int ompi_instance_invalid (const ompi_instance_t* instance)
+{
+    if ((NULL == instance) || (MPI_SESSION_NULL == instance))
+        return true;
+    else
+        return false;
+}
+
+#endif /* !defined(OMPI_INSTANCE_H) */
diff --git a/ompi/interlib/interlib.c b/ompi/interlib/interlib.c
index 1071fe6ed0a..255346949c7 100644
--- a/ompi/interlib/interlib.c
+++ b/ompi/interlib/interlib.c
@@ -17,6 +17,8 @@
  * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2017      IBM Corporation. All rights reserved.
  * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2022      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -29,6 +31,7 @@
 #include <string.h>
 
 #include "opal/mca/pmix/pmix-internal.h"
+#include "opal/util/proc.h"
 #include "ompi/runtime/ompi_rte.h"
 #include "ompi/interlib/interlib.h"
 
@@ -122,7 +125,7 @@ int ompi_interlib_declare(int threadlevel, char *version)
     PMIX_INFO_DESTRUCT(&info[3]);
     /* account for our refcount on pmix_init */
     PMIx_Finalize(NULL, 0);
-    if (ompi_singleton && PMIX_ERR_UNREACH == rc) {
+    if (opal_process_info.is_singleton && PMIX_ERR_UNREACH == rc) {
         ret = OMPI_SUCCESS;
     } else {
         ret = opal_pmix_convert_status(rc);
diff --git a/ompi/mca/bml/base/base.h b/ompi/mca/bml/base/base.h
index b7a226ac6ec..723f905cc8c 100644
--- a/ompi/mca/bml/base/base.h
+++ b/ompi/mca/bml/base/base.h
@@ -61,6 +61,7 @@ OMPI_DECLSPEC extern mca_bml_base_component_t mca_bml_component;
 OMPI_DECLSPEC extern mca_bml_base_module_t mca_bml;
 OMPI_DECLSPEC extern mca_base_framework_t ompi_bml_base_framework;
 OMPI_DECLSPEC extern opal_mutex_t mca_bml_lock;
+OMPI_DECLSPEC extern bool mca_bml_component_init_called;
 
 static inline struct mca_bml_base_endpoint_t *mca_bml_base_get_endpoint (struct ompi_proc_t *proc) {
     if (OPAL_UNLIKELY(NULL == proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML])) {
diff --git a/ompi/mca/bml/base/bml_base_frame.c b/ompi/mca/bml/base/bml_base_frame.c
index b5a63dd9a2e..a76a891e49d 100644
--- a/ompi/mca/bml/base/bml_base_frame.c
+++ b/ompi/mca/bml/base/bml_base_frame.c
@@ -127,5 +127,11 @@ static int mca_bml_base_close( void )
         return ret;
     }
 
-    return mca_base_framework_close(&opal_btl_base_framework);
+    ret = mca_base_framework_close(&opal_btl_base_framework);
+    if (OMPI_SUCCESS != ret) {
+        return ret;
+    }
+
+    mca_bml_component_init_called = false;
+    return OMPI_SUCCESS;
 }
diff --git a/ompi/mca/bml/base/bml_base_init.c b/ompi/mca/bml/base/bml_base_init.c
index 9a2efec8ccc..6d1060d4690 100644
--- a/ompi/mca/bml/base/bml_base_init.c
+++ b/ompi/mca/bml/base/bml_base_init.c
@@ -42,12 +42,12 @@ mca_bml_base_module_t mca_bml = {
 };
 mca_bml_base_component_t mca_bml_component = {{0}};
 
-static bool init_called = false;
+bool mca_bml_component_init_called = false;
 
 bool
 mca_bml_base_inited(void)
 {
-    return init_called;
+    return mca_bml_component_init_called;
 }
 
 int mca_bml_base_init( bool enable_progress_threads,
@@ -57,11 +57,11 @@ int mca_bml_base_init( bool enable_progress_threads,
     int priority = 0, best_priority = -1;
     mca_base_component_list_item_t *cli = NULL;
 
-    if (init_called) {
+    if (true == mca_bml_component_init_called) {
         return OPAL_SUCCESS;
     }
 
-    init_called = true;
+    mca_bml_component_init_called = true;
 
     OPAL_LIST_FOREACH(cli, &ompi_bml_base_framework.framework_components, mca_base_component_list_item_t) {
         component = (mca_bml_base_component_t*) cli->cli_component;
diff --git a/ompi/mca/bml/bml.h b/ompi/mca/bml/bml.h
index d5a70928ecd..e50384db0d3 100644
--- a/ompi/mca/bml/bml.h
+++ b/ompi/mca/bml/bml.h
@@ -22,7 +22,7 @@
 /**
  * @file
  *
- * BML Management Layer (BML)
+ * BTL Management Layer (BML)
  *
  */
 
diff --git a/ompi/mca/bml/r2/Makefile.am b/ompi/mca/bml/r2/Makefile.am
index a37dba65668..0fdb3e636f3 100644
--- a/ompi/mca/bml/r2/Makefile.am
+++ b/ompi/mca/bml/r2/Makefile.am
@@ -9,6 +9,7 @@
 #						 All rights reserved.
 # Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -21,7 +22,7 @@ r2_sources  = \
 	bml_r2.h \
 	bml_r2_component.c
 
-dist_ompidata_DATA = help-mca-bml-r2.txt
+EXTRA_DIST = help-mca-bml-r2.txt
 
 if MCA_BUILD_ompi_bml_r2_DSO
 component_noinst =
diff --git a/ompi/mca/bml/r2/bml_r2.c b/ompi/mca/bml/r2/bml_r2.c
index da3413c2525..6b5a5493678 100644
--- a/ompi/mca/bml/r2/bml_r2.c
+++ b/ompi/mca/bml/r2/bml_r2.c
@@ -220,7 +220,7 @@ static int mca_bml_r2_endpoint_add_btl (struct ompi_proc_t *proc, mca_bml_base_e
 
     if ((btl_flags & (MCA_BTL_FLAGS_PUT | MCA_BTL_FLAGS_GET | MCA_BTL_FLAGS_SEND)) == 0) {
         /* If no protocol specified, we have 2 choices: we ignore the BTL
-         * as we don't know which protocl to use, or we suppose that all
+         * as we don't know which protocol to use, or we suppose that all
          * BTLs support the send protocol. This is really a btl error as
          * these flags should have been sanitized by the btl. */
         btl_flags |= MCA_BTL_FLAGS_SEND;
@@ -634,7 +634,7 @@ static int mca_bml_r2_del_procs(size_t nprocs,
             }
 
             /* The reference stored in btl_eager and btl_rdma will automatically
-             * dissapear once the btl_array destructor is called. Thus, there is
+             * disappear once the btl_array destructor is called. Thus, there is
              * no need for extra cleaning here.
              */
         }
@@ -916,7 +916,7 @@ static int mca_bml_r2_register( mca_btl_base_tag_t tag,
 {
     mca_btl_base_active_message_trigger[tag].cbfunc = cbfunc;
     mca_btl_base_active_message_trigger[tag].cbdata = data;
-    /* Give an oportunity to the BTLs to do something special
+    /* Give an opportunity to the BTLs to do something special
      * for each registration.
      */
     {
diff --git a/ompi/mca/coll/accelerator/Makefile.am b/ompi/mca/coll/accelerator/Makefile.am
new file mode 100644
index 00000000000..635d4cb9886
--- /dev/null
+++ b/ompi/mca/coll/accelerator/Makefile.am
@@ -0,0 +1,41 @@
+#
+# Copyright (c) 2014      The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2014-2024 NVIDIA Corporation.  All rights reserved.
+# Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+sources = coll_accelerator_reduce.c coll_accelerator_allreduce.c \
+          coll_accelerator_reduce_scatter_block.c coll_accelerator_reduce_scatter.c       \
+	  coll_accelerator_allgather.c coll_accelerator_alltoall.c coll_accelerator_bcast.c \
+	  coll_accelerator_component.c coll_accelerator_module.c \
+          coll_accelerator_scan.c coll_accelerator_exscan.c coll_accelerator.h
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_ompi_coll_accelerator_DSO
+component_noinst =
+component_install = mca_coll_accelerator.la
+else
+component_noinst = libmca_coll_accelerator.la
+component_install =
+endif
+
+mcacomponentdir = $(ompilibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_coll_accelerator_la_SOURCES = $(sources)
+mca_coll_accelerator_la_LDFLAGS = -module -avoid-version
+mca_coll_accelerator_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_coll_accelerator_la_SOURCES =$(sources)
+libmca_coll_accelerator_la_LDFLAGS = -module -avoid-version
+
diff --git a/ompi/mca/coll/accelerator/coll_accelerator.h b/ompi/mca/coll/accelerator/coll_accelerator.h
new file mode 100644
index 00000000000..ad3163b937a
--- /dev/null
+++ b/ompi/mca/coll/accelerator/coll_accelerator.h
@@ -0,0 +1,179 @@
+/*
+ * Copyright (c) 2024      NVIDIA Corporation. All rights reserved.
+ * Copyright (c) 2014      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2024 NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights reserved.
+ * Copyright (c) 2024      Advanced Micro Devices, Inc. All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef MCA_COLL_ACCELERATOR_EXPORT_H
+#define MCA_COLL_ACCELERATOR_EXPORT_H
+
+#include "ompi_config.h"
+
+#include "mpi.h"
+
+#include "opal/class/opal_object.h"
+#include "ompi/mca/mca.h"
+
+#include "ompi/constants.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/coll/base/base.h"
+#include "ompi/communicator/communicator.h"
+#include "opal/mca/accelerator/accelerator.h"
+#include "opal/mca/accelerator/base/base.h"
+
+BEGIN_C_DECLS
+
+/* API functions */
+
+
+extern int mca_coll_accelerator_bcast_thresh;
+extern int mca_coll_accelerator_allgather_thresh;
+extern int mca_coll_accelerator_alltoall_thresh;
+
+int mca_coll_accelerator_init_query(bool enable_progress_threads,
+                             bool enable_mpi_threads);
+mca_coll_base_module_t
+*mca_coll_accelerator_comm_query(struct ompi_communicator_t *comm,
+                          int *priority);
+
+int
+mca_coll_accelerator_allreduce(const void *sbuf, void *rbuf, size_t count,
+                        struct ompi_datatype_t *dtype,
+                        struct ompi_op_t *op,
+                        struct ompi_communicator_t *comm,
+                        mca_coll_base_module_t *module);
+
+int mca_coll_accelerator_reduce_local(const void *sbuf, void *rbuf, size_t count,
+                         struct ompi_datatype_t *dtype,
+                         struct ompi_op_t *op,
+                         mca_coll_base_module_t *module);
+
+int mca_coll_accelerator_reduce(const void *sbuf, void *rbuf, size_t count,
+                         struct ompi_datatype_t *dtype,
+                         struct ompi_op_t *op,
+                         int root,
+                         struct ompi_communicator_t *comm,
+                         mca_coll_base_module_t *module);
+
+int mca_coll_accelerator_exscan(const void *sbuf, void *rbuf, size_t count,
+                         struct ompi_datatype_t *dtype,
+                         struct ompi_op_t *op,
+                         struct ompi_communicator_t *comm,
+                         mca_coll_base_module_t *module);
+
+int mca_coll_accelerator_scan(const void *sbuf, void *rbuf, size_t count,
+                       struct ompi_datatype_t *dtype,
+                       struct ompi_op_t *op,
+                       struct ompi_communicator_t *comm,
+                       mca_coll_base_module_t *module);
+
+int
+mca_coll_accelerator_reduce_scatter_block(const void *sbuf, void *rbuf, size_t rcount,
+                                   struct ompi_datatype_t *dtype,
+                                   struct ompi_op_t *op,
+                                   struct ompi_communicator_t *comm,
+                                   mca_coll_base_module_t *module);
+
+int
+mca_coll_accelerator_reduce_scatter(const void *sbuf, void *rbuf, ompi_count_array_t rcounts,
+                                   struct ompi_datatype_t *dtype,
+                                   struct ompi_op_t *op,
+                                   struct ompi_communicator_t *comm,
+                                   mca_coll_base_module_t *module);
+
+int
+mca_coll_accelerator_allgather(const void *sbuf, size_t scount,
+			       struct ompi_datatype_t *sdtype,
+			       void *rbuf, size_t rcount,
+			       struct ompi_datatype_t *rdtype,
+			       struct ompi_communicator_t *comm,
+			       mca_coll_base_module_t *module);
+
+int
+mca_coll_accelerator_alltoall(const void *sbuf, size_t scount,
+			      struct ompi_datatype_t *sdtype,
+			      void *rbuf, size_t rcount,
+			      struct ompi_datatype_t *rdtype,
+			      struct ompi_communicator_t *comm,
+			      mca_coll_base_module_t *module);
+
+int
+mca_coll_accelerator_bcast(void *buff, size_t count,
+			   struct ompi_datatype_t *datatype,
+			   int root,
+			   struct ompi_communicator_t *comm,
+			   mca_coll_base_module_t *module);
+
+/* Checks the type of pointer
+ *
+ * @param addr   One pointer to check
+ * @retval <0                An error has occurred.
+ * @retval 0                 The buffer is NULL or it does not belong to a managed buffer
+ *                           in device memory.
+ * @retval >0                The buffer belongs to a managed buffer in
+ *                           device memory.
+ */
+static inline int mca_coll_accelerator_check_buf(void *addr, int *dev_id)
+{
+    uint64_t flags;
+
+    if (OPAL_LIKELY(NULL != addr)) {
+        return opal_accelerator.check_addr(addr, dev_id, &flags);
+    } else {
+        *dev_id = MCA_ACCELERATOR_NO_DEVICE_ID;
+        return 0;
+    }
+}
+
+static inline void *mca_coll_accelerator_memcpy(void *dest, int dest_dev, const void *src, int src_dev, size_t size,
+						opal_accelerator_transfer_type_t type)
+{
+    int res;
+
+    res = opal_accelerator.mem_copy(dest_dev, src_dev, dest, src, size, type);
+    if (res != 0) {
+        opal_output(0, "coll/accelerator: Error in mem_copy: res=%d, dest=%p, src=%p, size=%d", res, dest, src,
+                    (int) size);
+        abort();
+    } else {
+        return dest;
+    }
+}
+
+/* Types */
+/* Module */
+
+typedef struct mca_coll_accelerator_module_t {
+    mca_coll_base_module_t super;
+
+    /* Pointers to all the "real" collective functions */
+    mca_coll_base_comm_coll_t c_coll;
+} mca_coll_accelerator_module_t;
+
+OBJ_CLASS_DECLARATION(mca_coll_accelerator_module_t);
+
+/* Component */
+
+typedef struct mca_coll_accelerator_component_t {
+    mca_coll_base_component_3_0_0_t super;
+
+    int priority; /* Priority of this component */
+    int disable_accelerator_coll;  /* Force disable of the accelerator collective component */
+} mca_coll_accelerator_component_t;
+
+/* Globally exported variables */
+
+OMPI_DECLSPEC extern mca_coll_accelerator_component_t mca_coll_accelerator_component;
+
+END_C_DECLS
+
+#endif /* MCA_COLL_ACCELERATOR_EXPORT_H */
diff --git a/ompi/mca/coll/accelerator/coll_accelerator_allgather.c b/ompi/mca/coll/accelerator/coll_accelerator_allgather.c
new file mode 100644
index 00000000000..af87b98eebc
--- /dev/null
+++ b/ompi/mca/coll/accelerator/coll_accelerator_allgather.c
@@ -0,0 +1,100 @@
+/*
+ * Copyright (c) 2014-2017 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2015 NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights reserved.
+ * Copyright (c) 2024      Advanced Micro Devices, Inc. All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "coll_accelerator.h"
+
+#include <stdio.h>
+
+#include "ompi/op/op.h"
+#include "opal/datatype/opal_convertor.h"
+
+/*
+ *	Function:	- allgather for device buffers through temp CPU buffer
+ *	Accepts:	- same as MPI_Allgather()
+ *	Returns:	- MPI_SUCCESS or error code
+ */
+int
+mca_coll_accelerator_allgather(const void *sbuf, size_t scount,
+                       struct ompi_datatype_t *sdtype,
+                       void *rbuf, size_t rcount,
+                       struct ompi_datatype_t *rdtype,
+                       struct ompi_communicator_t *comm,
+                       mca_coll_base_module_t *module)
+{
+    mca_coll_accelerator_module_t *s = (mca_coll_accelerator_module_t*) module;
+    ptrdiff_t sgap, rgap;
+    char *rbuf1 = NULL, *sbuf1 = NULL, *rbuf2 = NULL;
+    int sbuf_dev, rbuf_dev;
+    size_t sbufsize, rbufsize;
+    int rc;
+    int comm_size = ompi_comm_size(comm);
+    
+    sbufsize = opal_datatype_span(&sdtype->super, scount, &sgap);
+    rc = mca_coll_accelerator_check_buf((void *)sbuf, &sbuf_dev);
+    if (rc < 0) {
+        return rc;
+    }
+    if ((MPI_IN_PLACE != sbuf) && (rc > 0) &&
+        (sbufsize <= (size_t)mca_coll_accelerator_allgather_thresh)) {
+        sbuf1 = (char*)malloc(sbufsize);
+        if (NULL == sbuf1) {
+            return OMPI_ERR_OUT_OF_RESOURCE;
+        }
+        mca_coll_accelerator_memcpy(sbuf1, MCA_ACCELERATOR_NO_DEVICE_ID, sbuf, sbuf_dev,
+                                    sbufsize, MCA_ACCELERATOR_TRANSFER_DTOH);
+        sbuf = sbuf1 - sgap;
+    }
+
+    rbufsize = opal_datatype_span(&rdtype->super, rcount, &rgap);
+    rc = mca_coll_accelerator_check_buf(rbuf, &rbuf_dev);
+    if (rc < 0) {
+        goto exit;
+    }
+    /* Using sbufsize here on purpose to ensure symmetric decision for handling of GPU vs
+       CPU buffers. The two buffer sizes are expected to be the same for pre-defined datatypes,
+       but could vary due to layout issues/gaps for derived datatypes */
+    if ((rc > 0) && (sbufsize <= (size_t)mca_coll_accelerator_allgather_thresh)) {
+        rbuf1 = (char*)malloc(rbufsize * comm_size);
+        if (NULL == rbuf1) {
+            rc = OMPI_ERR_OUT_OF_RESOURCE;
+            goto exit;
+        }
+        mca_coll_accelerator_memcpy(rbuf1, MCA_ACCELERATOR_NO_DEVICE_ID, rbuf, rbuf_dev,
+                                    rbufsize * comm_size, MCA_ACCELERATOR_TRANSFER_DTOH);
+        rbuf2 = rbuf; /* save original buffer */
+        rbuf = rbuf1 - rgap;
+    }
+    rc = s->c_coll.coll_allgather(sbuf, scount, sdtype, rbuf, rcount, rdtype,
+                                  comm, s->c_coll.coll_allgather_module);
+    if (rc < 0) {
+        goto exit;
+    }
+
+    if (NULL != rbuf1) {
+        mca_coll_accelerator_memcpy(rbuf2, rbuf_dev, rbuf1, MCA_ACCELERATOR_NO_DEVICE_ID, rbufsize * comm_size,
+                                    MCA_ACCELERATOR_TRANSFER_HTOD);
+    }
+
+ exit:
+    if (NULL != sbuf1) {
+        free(sbuf1);
+    }
+    if (NULL != rbuf1) {
+        free(rbuf1);
+    }
+
+    return rc;
+}
diff --git a/ompi/mca/coll/accelerator/coll_accelerator_allreduce.c b/ompi/mca/coll/accelerator/coll_accelerator_allreduce.c
new file mode 100644
index 00000000000..75e9051ce0a
--- /dev/null
+++ b/ompi/mca/coll/accelerator/coll_accelerator_allreduce.c
@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2014-2017 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2015 NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights reserved.
+ * Copyright (c) 2024      Advanced Micro Devices, Inc. All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "coll_accelerator.h"
+
+#include <stdio.h>
+
+#include "ompi/op/op.h"
+#include "opal/datatype/opal_convertor.h"
+
+/*
+ *	allreduce_intra
+ *
+ *	Function:	- allreduce using other MPI collectives
+ *	Accepts:	- same as MPI_Allreduce()
+ *	Returns:	- MPI_SUCCESS or error code
+ */
+int
+mca_coll_accelerator_allreduce(const void *sbuf, void *rbuf, size_t count,
+                        struct ompi_datatype_t *dtype,
+                        struct ompi_op_t *op,
+                        struct ompi_communicator_t *comm,
+                        mca_coll_base_module_t *module)
+{
+    mca_coll_accelerator_module_t *s = (mca_coll_accelerator_module_t*) module;
+    ptrdiff_t gap;
+    char *rbuf1 = NULL, *sbuf1 = NULL, *rbuf2 = NULL;
+    int sbuf_dev, rbuf_dev;
+    size_t bufsize;
+    int rc;
+
+    bufsize = opal_datatype_span(&dtype->super, count, &gap);
+    rc = mca_coll_accelerator_check_buf((void *)sbuf, &sbuf_dev);
+    if (rc < 0) {
+        return rc;
+    }
+    if ((MPI_IN_PLACE != sbuf) && (rc > 0)) {
+        sbuf1 = (char*)malloc(bufsize);
+        if (NULL == sbuf1) {
+            return OMPI_ERR_OUT_OF_RESOURCE;
+        }
+        mca_coll_accelerator_memcpy(sbuf1, MCA_ACCELERATOR_NO_DEVICE_ID, sbuf, sbuf_dev,
+                                    bufsize, MCA_ACCELERATOR_TRANSFER_DTOH);
+        sbuf = sbuf1 - gap;
+    }
+    rc = mca_coll_accelerator_check_buf(rbuf, &rbuf_dev);
+    if (rc < 0) {
+        return rc;
+    }
+    if (rc > 0) {
+        rbuf1 = (char*)malloc(bufsize);
+        if (NULL == rbuf1) {
+            if (NULL != sbuf1) free(sbuf1);
+            return OMPI_ERR_OUT_OF_RESOURCE;
+        }
+        mca_coll_accelerator_memcpy(rbuf1, MCA_ACCELERATOR_NO_DEVICE_ID, rbuf, rbuf_dev,
+                                    bufsize, MCA_ACCELERATOR_TRANSFER_DTOH);
+        rbuf2 = rbuf; /* save away original buffer */
+        rbuf = rbuf1 - gap;
+    }
+    rc = s->c_coll.coll_allreduce(sbuf, rbuf, count, dtype, op, comm, s->c_coll.coll_allreduce_module);
+    if (NULL != sbuf1) {
+        free(sbuf1);
+    }
+    if (NULL != rbuf1) {
+        rbuf = rbuf2;
+        mca_coll_accelerator_memcpy(rbuf, rbuf_dev, rbuf1, MCA_ACCELERATOR_NO_DEVICE_ID, bufsize,
+                                    MCA_ACCELERATOR_TRANSFER_HTOD);
+        free(rbuf1);
+    }
+    return rc;
+}
+
diff --git a/ompi/mca/coll/accelerator/coll_accelerator_alltoall.c b/ompi/mca/coll/accelerator/coll_accelerator_alltoall.c
new file mode 100644
index 00000000000..9fd89e58f2c
--- /dev/null
+++ b/ompi/mca/coll/accelerator/coll_accelerator_alltoall.c
@@ -0,0 +1,99 @@
+/*
+ * Copyright (c) 2014-2017 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2015 NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights reserved.
+ * Copyright (c) 2024      Advanced Micro Devices, Inc. All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "coll_accelerator.h"
+
+#include <stdio.h>
+
+#include "ompi/op/op.h"
+#include "opal/datatype/opal_convertor.h"
+
+/*
+ *	Function:	- alltoall for device buffers using temp. CPU buffer
+ *	Accepts:	- same as MPI_Alltoall()
+ *	Returns:	- MPI_SUCCESS or error code
+ */
+int
+mca_coll_accelerator_alltoall(const void *sbuf, size_t scount,
+                      struct ompi_datatype_t *sdtype,
+                      void *rbuf, size_t rcount,
+                      struct ompi_datatype_t *rdtype,
+                      struct ompi_communicator_t *comm,
+                      mca_coll_base_module_t *module)
+{
+    mca_coll_accelerator_module_t *s = (mca_coll_accelerator_module_t*) module;
+    ptrdiff_t sgap, rgap;
+    char *rbuf1 = NULL, *sbuf1 = NULL, *rbuf2 = NULL;
+    int sbuf_dev, rbuf_dev;
+    size_t sbufsize, rbufsize;
+    int rc;
+    int comm_size = ompi_comm_size(comm);
+
+    sbufsize = opal_datatype_span(&sdtype->super, scount, &sgap);
+    rc = mca_coll_accelerator_check_buf((void *)sbuf, &sbuf_dev);
+    if (rc < 0) {
+        return rc;
+    }
+    if ((MPI_IN_PLACE != sbuf) && (rc > 0) &&
+        (sbufsize <= (size_t)mca_coll_accelerator_alltoall_thresh)) {
+        sbuf1 = (char*)malloc(sbufsize * comm_size);
+        if (NULL == sbuf1) {
+            return OMPI_ERR_OUT_OF_RESOURCE;
+        }
+        mca_coll_accelerator_memcpy(sbuf1, MCA_ACCELERATOR_NO_DEVICE_ID, sbuf, sbuf_dev,
+                                    sbufsize * comm_size, MCA_ACCELERATOR_TRANSFER_DTOH);
+        sbuf = sbuf1 - sgap;
+    }
+
+    rbufsize = opal_datatype_span(&rdtype->super, rcount, &rgap);
+    rc = mca_coll_accelerator_check_buf(rbuf, &rbuf_dev);
+    if (rc < 0) {
+        goto exit;
+    }
+    /* Using sbufsize here on purpose to ensure symmetric decision for handling of GPU vs
+       CPU buffers. The two buffer sizes are expected to be the same for pre-defined datatypes,
+       but could vary due to layout issues/gaps for derived datatypes */
+    if ((rc > 0) && (sbufsize <= (size_t)mca_coll_accelerator_alltoall_thresh)) {
+        rbuf1 = (char*)malloc(rbufsize * comm_size);
+        if (NULL == rbuf1) {
+            rc = OMPI_ERR_OUT_OF_RESOURCE;
+            goto exit;
+        }
+        mca_coll_accelerator_memcpy(rbuf1, MCA_ACCELERATOR_NO_DEVICE_ID, rbuf, rbuf_dev,
+                                    rbufsize * comm_size, MCA_ACCELERATOR_TRANSFER_DTOH);
+        rbuf2 = rbuf; /* save away original buffer */
+        rbuf = rbuf1 - rgap;
+    }
+    rc = s->c_coll.coll_alltoall(sbuf, scount, sdtype, rbuf, rcount, rdtype,
+                                  comm, s->c_coll.coll_alltoall_module);
+    if (rc < 0) {
+        goto exit;;
+    }
+    if (NULL != rbuf1) {
+         mca_coll_accelerator_memcpy(rbuf2, rbuf_dev, rbuf1, MCA_ACCELERATOR_NO_DEVICE_ID,
+				     rbufsize * comm_size, MCA_ACCELERATOR_TRANSFER_HTOD);
+    }
+
+ exit:
+    if (NULL != sbuf1) {
+        free(sbuf1);
+    }
+    if (NULL != rbuf1) {
+        free(rbuf1);
+    }
+
+    return rc;
+}
diff --git a/ompi/mca/coll/accelerator/coll_accelerator_bcast.c b/ompi/mca/coll/accelerator/coll_accelerator_bcast.c
new file mode 100644
index 00000000000..996230ac80d
--- /dev/null
+++ b/ompi/mca/coll/accelerator/coll_accelerator_bcast.c
@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2004-2023 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2015 NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights reserved.
+ * Copyright (c) 2024      Advanced Micro Devices, Inc. All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "coll_accelerator.h"
+
+#include <stdio.h>
+
+#include "ompi/op/op.h"
+#include "opal/datatype/opal_convertor.h"
+
+/*
+ *
+ *	Function:	- Bcast for device buffers through temp CPU buffer. 
+ *	Accepts:	- same as MPI_Bcast()
+ *	Returns:	- MPI_SUCCESS or error code
+ */
+int
+mca_coll_accelerator_bcast(void *orig_buf, size_t count,
+			   struct ompi_datatype_t *datatype,
+			   int root,
+			   struct ompi_communicator_t *comm,
+			   mca_coll_base_module_t *module)
+{
+    mca_coll_accelerator_module_t *s = (mca_coll_accelerator_module_t*) module;
+    ptrdiff_t gap;
+    char *buf1 = NULL;
+    char *sbuf = (char*) orig_buf;
+    int buf_dev;
+    size_t bufsize;
+    int rc;
+
+    bufsize = opal_datatype_span(&datatype->super, count, &gap);
+
+    rc = mca_coll_accelerator_check_buf((void *)orig_buf, &buf_dev);
+    if (rc < 0) {
+        return rc;
+    }
+    if ((rc > 0) && (bufsize <= (size_t)mca_coll_accelerator_bcast_thresh)) {
+        buf1 = (char*)malloc(bufsize);
+        if (NULL == buf1) {
+            return OMPI_ERR_OUT_OF_RESOURCE;
+        }
+        mca_coll_accelerator_memcpy(buf1, MCA_ACCELERATOR_NO_DEVICE_ID, orig_buf, buf_dev, bufsize,
+                                    MCA_ACCELERATOR_TRANSFER_DTOH);
+        sbuf = buf1 - gap;
+    }
+
+    rc = s->c_coll.coll_bcast((void *) sbuf, count, datatype, root, comm,
+                               s->c_coll.coll_bcast_module);
+    if (rc < 0) {
+        goto exit;
+    }
+    if (NULL != buf1) {
+        mca_coll_accelerator_memcpy((void*)orig_buf, buf_dev, buf1, MCA_ACCELERATOR_NO_DEVICE_ID, bufsize,
+                                    MCA_ACCELERATOR_TRANSFER_HTOD);
+    }
+
+ exit:
+    if (NULL != buf1) {
+        free(buf1);
+    }
+
+    return rc;
+}
diff --git a/ompi/mca/coll/accelerator/coll_accelerator_component.c b/ompi/mca/coll/accelerator/coll_accelerator_component.c
new file mode 100644
index 00000000000..be0452668b2
--- /dev/null
+++ b/ompi/mca/coll/accelerator/coll_accelerator_component.c
@@ -0,0 +1,124 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2014      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2015 NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include <string.h>
+
+#include "mpi.h"
+#include "ompi/constants.h"
+#include "coll_accelerator.h"
+
+
+int mca_coll_accelerator_bcast_thresh = 256;
+int mca_coll_accelerator_allgather_thresh = 65536;
+int mca_coll_accelerator_alltoall_thresh = 65536;
+
+/*
+ * Public string showing the coll ompi_accelerator component version number
+ */
+const char *mca_coll_accelerator_component_version_string =
+    "Open MPI accelerator collective MCA component version " OMPI_VERSION;
+
+/*
+ * Local function
+ */
+static int accelerator_register(void);
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+
+mca_coll_accelerator_component_t mca_coll_accelerator_component = {
+    {
+        /* First, the mca_component_t struct containing meta information
+         * about the component itself */
+
+        .collm_version = {
+            MCA_COLL_BASE_VERSION_3_0_0,
+
+            /* Component name and version */
+            .mca_component_name = "accelerator",
+            MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION,
+                                  OMPI_RELEASE_VERSION),
+
+            /* Component open and close functions */
+            .mca_register_component_params = accelerator_register,
+        },
+        .collm_data = {
+            /* The component is checkpoint ready */
+            MCA_BASE_METADATA_PARAM_CHECKPOINT
+        },
+
+        /* Initialization / querying functions */
+
+        .collm_init_query = mca_coll_accelerator_init_query,
+        .collm_comm_query = mca_coll_accelerator_comm_query,
+    },
+
+    /* accelerator-specific component information */
+
+    /* Priority: make it above all point to point collectives including self */
+    .priority = 78,
+};
+
+
+static int accelerator_register(void)
+{
+    (void) mca_base_component_var_register(&mca_coll_accelerator_component.super.collm_version,
+                                           "priority", "Priority of the accelerator coll component; only relevant if barrier_before or barrier_after is > 0",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           OPAL_INFO_LVL_6,
+                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           &mca_coll_accelerator_component.priority);
+
+    (void) mca_base_component_var_register(&mca_coll_accelerator_component.super.collm_version,
+                                           "disable_accelerator_coll", "Automatically handle the accelerator buffers for the MPI collective.",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           OPAL_INFO_LVL_2,
+                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           &mca_coll_accelerator_component.disable_accelerator_coll);
+
+    mca_coll_accelerator_bcast_thresh = 256;
+    (void) mca_base_component_var_register(&mca_coll_accelerator_component.super.collm_version,
+                                           "bcast_thresh",
+                                           "max. msg length for which to copy accelerator buffer to CPU for bcast operation",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           &mca_coll_accelerator_bcast_thresh);
+
+    mca_coll_accelerator_allgather_thresh = 65536;
+    (void) mca_base_component_var_register(&mca_coll_accelerator_component.super.collm_version,
+                                           "allgather_thresh",
+                                           "max. msg length for which to copy accelerator buffer to CPU for allgather operation",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           &mca_coll_accelerator_allgather_thresh);
+
+    mca_coll_accelerator_alltoall_thresh = 65536;
+    (void) mca_base_component_var_register(&mca_coll_accelerator_component.super.collm_version,
+                                           "alltoall_thresh",
+                                           "max. msg length for which to copy accelerator buffer to CPU for alltoall operation",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           &mca_coll_accelerator_alltoall_thresh);
+
+    return OMPI_SUCCESS;
+}
diff --git a/ompi/mca/coll/accelerator/coll_accelerator_exscan.c b/ompi/mca/coll/accelerator/coll_accelerator_exscan.c
new file mode 100644
index 00000000000..d77dd5a9b46
--- /dev/null
+++ b/ompi/mca/coll/accelerator/coll_accelerator_exscan.c
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2014-2017 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2015 NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights reserved.
+ * Copyright (c) 2024      Advanced Micro Devices, Inc. All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "coll_accelerator.h"
+
+#include <stdio.h>
+
+#include "ompi/op/op.h"
+#include "opal/datatype/opal_convertor.h"
+
+int mca_coll_accelerator_exscan(const void *sbuf, void *rbuf, size_t count,
+                         struct ompi_datatype_t *dtype,
+                         struct ompi_op_t *op,
+                         struct ompi_communicator_t *comm,
+                         mca_coll_base_module_t *module)
+{
+    mca_coll_accelerator_module_t *s = (mca_coll_accelerator_module_t*) module;
+    ptrdiff_t gap;
+    char *rbuf1 = NULL, *sbuf1 = NULL, *rbuf2 = NULL;
+    int sbuf_dev, rbuf_dev;
+    size_t bufsize;
+    int rc;
+
+    bufsize = opal_datatype_span(&dtype->super, count, &gap);
+    rc = mca_coll_accelerator_check_buf((void *)sbuf, &sbuf_dev);
+    if (rc < 0) {
+        return rc;
+    }
+
+    if ((MPI_IN_PLACE != sbuf) && rc > 0) {
+        sbuf1 = (char*)malloc(bufsize);
+        if (NULL == sbuf1) {
+            return OMPI_ERR_OUT_OF_RESOURCE;
+        }
+        mca_coll_accelerator_memcpy(sbuf1, MCA_ACCELERATOR_NO_DEVICE_ID, sbuf, sbuf_dev, bufsize,
+                                    MCA_ACCELERATOR_TRANSFER_DTOH);
+        sbuf = sbuf1 - gap;
+    }
+    rc = mca_coll_accelerator_check_buf(rbuf, &rbuf_dev);
+    if (rc < 0) {
+        return rc;
+    }
+    if (rc > 0) {
+        rbuf1 = (char*)malloc(bufsize);
+        if (NULL == rbuf1) {
+            if (NULL != sbuf1) free(sbuf1);
+            return OMPI_ERR_OUT_OF_RESOURCE;
+        }
+        mca_coll_accelerator_memcpy(rbuf1, MCA_ACCELERATOR_NO_DEVICE_ID, rbuf, rbuf_dev, bufsize,
+                                    MCA_ACCELERATOR_TRANSFER_DTOH);
+        rbuf2 = rbuf; /* save away original buffer */
+        rbuf = rbuf1 - gap;
+    }
+
+    rc = s->c_coll.coll_exscan(sbuf, rbuf, count, dtype, op, comm,
+                               s->c_coll.coll_exscan_module);
+    if (NULL != sbuf1) {
+        free(sbuf1);
+    }
+    if (NULL != rbuf1) {
+        rbuf = rbuf2;
+        mca_coll_accelerator_memcpy(rbuf, rbuf_dev, rbuf1, MCA_ACCELERATOR_NO_DEVICE_ID, bufsize,
+                                    MCA_ACCELERATOR_TRANSFER_HTOD);
+        free(rbuf1);
+    }
+    return rc;
+}
diff --git a/ompi/mca/coll/accelerator/coll_accelerator_module.c b/ompi/mca/coll/accelerator/coll_accelerator_module.c
new file mode 100644
index 00000000000..6e242f2c353
--- /dev/null
+++ b/ompi/mca/coll/accelerator/coll_accelerator_module.c
@@ -0,0 +1,193 @@
+/*
+ * Copyright (c) 2024      NVIDIA Corporation. All rights reserved.
+ * Copyright (c) 2014-2017 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2024 NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2019      Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2023-2024 Advanced Micro Devices, Inc. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include <string.h>
+#include <stdio.h>
+
+#include "coll_accelerator.h"
+
+#include "mpi.h"
+
+
+#include "opal/util/show_help.h"
+
+#include "ompi/constants.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/coll/base/base.h"
+#include "coll_accelerator.h"
+
+static int
+mca_coll_accelerator_module_enable(mca_coll_base_module_t *module,
+                                   struct ompi_communicator_t *comm);
+static int
+mca_coll_accelerator_module_disable(mca_coll_base_module_t *module,
+                                    struct ompi_communicator_t *comm);
+
+static void mca_coll_accelerator_module_construct(mca_coll_accelerator_module_t *module)
+{
+    memset(&(module->c_coll), 0, sizeof(module->c_coll));
+}
+
+OBJ_CLASS_INSTANCE(mca_coll_accelerator_module_t, mca_coll_base_module_t,
+                   mca_coll_accelerator_module_construct,
+                   NULL);
+
+
+/*
+ * Initial query function that is invoked during MPI_INIT, allowing
+ * this component to disqualify itself if it doesn't support the
+ * required level of thread support.
+ */
+int mca_coll_accelerator_init_query(bool enable_progress_threads,
+                             bool enable_mpi_threads)
+{
+    /* Nothing to do */
+
+    return OMPI_SUCCESS;
+}
+
+
+/*
+ * Invoked when there's a new communicator that has been created.
+ * Look at the communicator and decide which set of functions and
+ * priority we want to return.
+ */
+mca_coll_base_module_t *
+mca_coll_accelerator_comm_query(struct ompi_communicator_t *comm,
+                         int *priority)
+{
+    mca_coll_accelerator_module_t *accelerator_module;
+
+    if (0 == strcmp(opal_accelerator_base_selected_component.base_version.mca_component_name,
+                    "null")) {
+        opal_output_verbose(10, ompi_coll_base_framework.framework_output,
+                          "coll:accelerator:comm_query: accelerator component is null: disqualifying myself");
+        return NULL;
+    }
+
+    if (OMPI_COMM_CHECK_ASSERT_NO_ACCEL_BUF(comm)) {
+        opal_output_verbose(10, ompi_coll_base_framework.framework_output,
+			    "coll:accelerator:comm_query: NO_ACCEL_BUF assertion set: disqualifying myself");
+	return NULL;
+    }
+
+    accelerator_module = OBJ_NEW(mca_coll_accelerator_module_t);
+    if (NULL == accelerator_module) {
+        return NULL;
+    }
+
+    *priority = mca_coll_accelerator_component.priority;
+
+    /* Choose whether to use [intra|inter] */
+    accelerator_module->super.coll_module_enable = mca_coll_accelerator_module_enable;
+    accelerator_module->super.coll_module_disable = mca_coll_accelerator_module_disable;
+
+    accelerator_module->super.coll_allgather  = mca_coll_accelerator_allgather;
+    accelerator_module->super.coll_allreduce  = mca_coll_accelerator_allreduce;
+    accelerator_module->super.coll_alltoall   = mca_coll_accelerator_alltoall;
+    accelerator_module->super.coll_reduce     = mca_coll_accelerator_reduce;
+    accelerator_module->super.coll_bcast      = mca_coll_accelerator_bcast;
+    accelerator_module->super.coll_reduce_local         = mca_coll_accelerator_reduce_local;
+    accelerator_module->super.coll_reduce_scatter       = mca_coll_accelerator_reduce_scatter;
+    accelerator_module->super.coll_reduce_scatter_block = mca_coll_accelerator_reduce_scatter_block;
+    if (!OMPI_COMM_IS_INTER(comm)) {
+        accelerator_module->super.coll_scan       = mca_coll_accelerator_scan;
+        accelerator_module->super.coll_exscan     = mca_coll_accelerator_exscan;
+    }
+
+    return &(accelerator_module->super);
+}
+
+
+#define  ACCELERATOR_INSTALL_COLL_API(__comm, __module, __api) \
+    do \
+    { \
+        if ((__comm)->c_coll->coll_##__api) \
+        { \
+            MCA_COLL_SAVE_API(__comm, __api, (__module)->c_coll.coll_##__api, (__module)->c_coll.coll_##__api##_module, "accelerator"); \
+            MCA_COLL_INSTALL_API(__comm, __api, mca_coll_accelerator_##__api, &__module->super, "accelerator"); \
+        } \
+        else \
+        { \
+            opal_show_help("help-mca-coll-base.txt", "comm-select:missing collective", true, \
+                           "cuda", #__api, ompi_process_info.nodename, \
+                           mca_coll_accelerator_component.priority); \
+        } \
+    } while (0)
+
+#define ACCELERATOR_UNINSTALL_COLL_API(__comm, __module, __api) \
+    do \
+    { \
+        if (&(__module)->super == (__comm)->c_coll->coll_##__api##_module) { \
+            MCA_COLL_INSTALL_API(__comm, __api, (__module)->c_coll.coll_##__api, (__module)->c_coll.coll_##__api##_module, "accelerator"); \
+            (__module)->c_coll.coll_##__api##_module = NULL; \
+            (__module)->c_coll.coll_##__api = NULL; \
+        } \
+    } while (0)
+
+/*
+ * Init/Fini module on the communicator
+ */
+static int
+mca_coll_accelerator_module_enable(mca_coll_base_module_t *module,
+                                   struct ompi_communicator_t *comm)
+{
+    mca_coll_accelerator_module_t *s = (mca_coll_accelerator_module_t*) module;
+
+    ACCELERATOR_INSTALL_COLL_API(comm, s, allgather);
+    ACCELERATOR_INSTALL_COLL_API(comm, s, allreduce);
+    ACCELERATOR_INSTALL_COLL_API(comm, s, alltoall);
+    ACCELERATOR_INSTALL_COLL_API(comm, s, bcast);
+    ACCELERATOR_INSTALL_COLL_API(comm, s, reduce);
+    ACCELERATOR_INSTALL_COLL_API(comm, s, reduce_local);
+    ACCELERATOR_INSTALL_COLL_API(comm, s, reduce_scatter);
+    ACCELERATOR_INSTALL_COLL_API(comm, s, reduce_scatter_block);
+    if (!OMPI_COMM_IS_INTER(comm)) {
+        /* MPI does not define scan/exscan on intercommunicators */
+        ACCELERATOR_INSTALL_COLL_API(comm, s, exscan);
+        ACCELERATOR_INSTALL_COLL_API(comm, s, scan);
+    }
+
+    return OMPI_SUCCESS;
+}
+
+static int
+mca_coll_accelerator_module_disable(mca_coll_base_module_t *module,
+                                    struct ompi_communicator_t *comm)
+{
+    mca_coll_accelerator_module_t *s = (mca_coll_accelerator_module_t*) module;
+
+    ACCELERATOR_UNINSTALL_COLL_API(comm, s, allgather);
+    ACCELERATOR_UNINSTALL_COLL_API(comm, s, allreduce);
+    ACCELERATOR_UNINSTALL_COLL_API(comm, s, alltoall);
+    ACCELERATOR_UNINSTALL_COLL_API(comm, s, reduce);
+    ACCELERATOR_UNINSTALL_COLL_API(comm, s, bcast);
+    ACCELERATOR_UNINSTALL_COLL_API(comm, s, reduce_local);
+    ACCELERATOR_UNINSTALL_COLL_API(comm, s, reduce_scatter);
+    ACCELERATOR_UNINSTALL_COLL_API(comm, s, reduce_scatter_block);
+    if (!OMPI_COMM_IS_INTER(comm))
+    {
+        /* MPI does not define scan/exscan on intercommunicators */
+        ACCELERATOR_UNINSTALL_COLL_API(comm, s, exscan);
+        ACCELERATOR_UNINSTALL_COLL_API(comm, s, scan);
+    }
+
+    return OMPI_SUCCESS;
+}
diff --git a/ompi/mca/coll/accelerator/coll_accelerator_reduce.c b/ompi/mca/coll/accelerator/coll_accelerator_reduce.c
new file mode 100644
index 00000000000..509ea0961d4
--- /dev/null
+++ b/ompi/mca/coll/accelerator/coll_accelerator_reduce.c
@@ -0,0 +1,153 @@
+/*
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2004-2023 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2015 NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights reserved.
+ * Copyright (c) 2024      Advanced Micro Devices, Inc. All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "coll_accelerator.h"
+
+#include <stdio.h>
+
+#include "ompi/op/op.h"
+#include "opal/datatype/opal_convertor.h"
+
+/*
+ *	reduce_log_inter
+ *
+ *	Function:	- reduction using O(N) algorithm
+ *	Accepts:	- same as MPI_Reduce()
+ *	Returns:	- MPI_SUCCESS or error code
+ */
+int
+mca_coll_accelerator_reduce(const void *sbuf, void *rbuf, size_t count,
+                     struct ompi_datatype_t *dtype,
+                     struct ompi_op_t *op,
+                     int root, struct ompi_communicator_t *comm,
+                     mca_coll_base_module_t *module)
+{
+    mca_coll_accelerator_module_t *s = (mca_coll_accelerator_module_t*) module;
+    int rank = ompi_comm_rank(comm);
+    ptrdiff_t gap;
+    char *rbuf1 = NULL, *sbuf1 = NULL, *rbuf2 = NULL;
+    int rbuf_dev, sbuf_dev;
+    size_t bufsize;
+    int rc;
+
+    bufsize = opal_datatype_span(&dtype->super, count, &gap);
+
+    rc = mca_coll_accelerator_check_buf((void *)sbuf, &sbuf_dev);
+    if (rc < 0) {
+        return rc;
+    }
+    if ((MPI_IN_PLACE != sbuf) && (rc > 0)) {
+        sbuf1 = (char*)malloc(bufsize);
+        if (NULL == sbuf1) {
+            return OMPI_ERR_OUT_OF_RESOURCE;
+        }
+        mca_coll_accelerator_memcpy(sbuf1, MCA_ACCELERATOR_NO_DEVICE_ID, sbuf, sbuf_dev, bufsize,
+                                    MCA_ACCELERATOR_TRANSFER_DTOH);
+        sbuf = sbuf1 - gap;
+    }
+
+    rc = mca_coll_accelerator_check_buf(rbuf, &rbuf_dev);
+    if (rc < 0) {
+        return rc;
+    }
+    if ((rank == root) && (rc > 0)) {
+        rbuf1 = (char*)malloc(bufsize);
+        if (NULL == rbuf1) {
+            if (NULL != sbuf1) free(sbuf1);
+            return OMPI_ERR_OUT_OF_RESOURCE;
+        }
+        mca_coll_accelerator_memcpy(rbuf1, MCA_ACCELERATOR_NO_DEVICE_ID, rbuf, rbuf_dev, bufsize,
+                                    MCA_ACCELERATOR_TRANSFER_DTOH);
+        rbuf2 = rbuf; /* save away original buffer */
+        rbuf = rbuf1 - gap;
+    }
+    rc = s->c_coll.coll_reduce((void *) sbuf, rbuf, count,
+                               dtype, op, root, comm,
+                               s->c_coll.coll_reduce_module);
+
+    if (NULL != sbuf1) {
+        free(sbuf1);
+    }
+    if (NULL != rbuf1) {
+        rbuf = rbuf2;
+        mca_coll_accelerator_memcpy(rbuf, rbuf_dev, rbuf1, MCA_ACCELERATOR_NO_DEVICE_ID, bufsize,
+                                    MCA_ACCELERATOR_TRANSFER_HTOD);
+        free(rbuf1);
+    }
+    return rc;
+}
+
+int
+mca_coll_accelerator_reduce_local(const void *sbuf, void *rbuf, size_t count,
+                                  struct ompi_datatype_t *dtype,
+                                  struct ompi_op_t *op,
+                                  mca_coll_base_module_t *module)
+{
+    ptrdiff_t gap;
+    char *rbuf1 = NULL, *sbuf1 = NULL, *rbuf2 = NULL;
+    int sbuf_dev, rbuf_dev;
+    size_t bufsize;
+    int rc;
+
+    bufsize = opal_datatype_span(&dtype->super, count, &gap);
+
+    rc = mca_coll_accelerator_check_buf((void *)sbuf, &sbuf_dev);
+    if (rc < 0) {
+        return rc;
+    }
+
+    if ((MPI_IN_PLACE != sbuf) && (rc > 0)) {
+        sbuf1 = (char*)malloc(bufsize);
+        if (NULL == sbuf1) {
+            return OMPI_ERR_OUT_OF_RESOURCE;
+        }
+        mca_coll_accelerator_memcpy(sbuf1, MCA_ACCELERATOR_NO_DEVICE_ID, sbuf, sbuf_dev, bufsize,
+                                    MCA_ACCELERATOR_TRANSFER_DTOH);
+        sbuf = sbuf1 - gap;
+    }
+
+    rc = mca_coll_accelerator_check_buf(rbuf, &rbuf_dev);
+    if (rc < 0) {
+        return rc;
+    }
+
+    if (rc > 0) {
+        rbuf1 = (char*)malloc(bufsize);
+        if (NULL == rbuf1) {
+            if (NULL != sbuf1) free(sbuf1);
+            return OMPI_ERR_OUT_OF_RESOURCE;
+        }
+        mca_coll_accelerator_memcpy(rbuf1, MCA_ACCELERATOR_NO_DEVICE_ID, rbuf, rbuf_dev, bufsize,
+                                    MCA_ACCELERATOR_TRANSFER_DTOH);
+        rbuf2 = rbuf; /* save away original buffer */
+        rbuf = rbuf1 - gap;
+    }
+
+    ompi_op_reduce(op, (void *)sbuf, rbuf, count, dtype);
+    rc = OMPI_SUCCESS;
+
+    if (NULL != sbuf1) {
+        free(sbuf1);
+    }
+    if (NULL != rbuf1) {
+        rbuf = rbuf2;
+        mca_coll_accelerator_memcpy(rbuf, rbuf_dev, rbuf1, MCA_ACCELERATOR_NO_DEVICE_ID, bufsize,
+                                    MCA_ACCELERATOR_TRANSFER_HTOD);
+        free(rbuf1);
+    }
+    return rc;
+}
diff --git a/ompi/mca/coll/accelerator/coll_accelerator_reduce_scatter.c b/ompi/mca/coll/accelerator/coll_accelerator_reduce_scatter.c
new file mode 100644
index 00000000000..222e9401910
--- /dev/null
+++ b/ompi/mca/coll/accelerator/coll_accelerator_reduce_scatter.c
@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2014-2017 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2015 NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights reserved.
+ * Copyright (c) 2024      Advanced Micro Devices, Inc. All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "coll_accelerator.h"
+
+#include <stdio.h>
+
+#include "ompi/op/op.h"
+#include "opal/datatype/opal_convertor.h"
+
+/*
+ *	reduce_scatter
+ *
+ *	Function:	- reduce_scatter for device buffers through temp. CPU buffer
+ *	Accepts:	- same as MPI_Reduce_scatter()
+ *	Returns:	- MPI_SUCCESS or error code
+ *
+ * Algorithm:
+ *     reduce and scatter (needs to be cleaned
+ *     up at some point)
+ */
+int
+mca_coll_accelerator_reduce_scatter(const void *sbuf, void *rbuf, ompi_count_array_t rcounts,
+                                   struct ompi_datatype_t *dtype,
+                                   struct ompi_op_t *op,
+                                   struct ompi_communicator_t *comm,
+                                   mca_coll_base_module_t *module)
+{
+    mca_coll_accelerator_module_t *s = (mca_coll_accelerator_module_t*) module;
+    ptrdiff_t gap;
+    char *rbuf1 = NULL, *sbuf1 = NULL, *rbuf2 = NULL;
+    int sbuf_dev, rbuf_dev;
+    size_t sbufsize, rbufsize, elemsize;
+    int rc, i;
+    int comm_size = ompi_comm_size(comm);
+    int total_count = 0;
+    
+    elemsize = opal_datatype_span(&dtype->super, 1, &gap);
+    for (i = 0; i < comm_size; i++) {
+	total_count += ompi_count_array_get(rcounts, i);
+    }
+    sbufsize = elemsize * total_count;
+
+    rc = mca_coll_accelerator_check_buf((void *)sbuf, &sbuf_dev);
+    if (0 > rc) {
+        return rc;
+    }
+    if ((MPI_IN_PLACE != sbuf) && (0 < rc)) {
+        sbuf1 = (char*)malloc(sbufsize);
+        if (NULL == sbuf1) {
+            return OMPI_ERR_OUT_OF_RESOURCE;
+        }
+        mca_coll_accelerator_memcpy(sbuf1, MCA_ACCELERATOR_NO_DEVICE_ID, sbuf, sbuf_dev, sbufsize,
+                                    MCA_ACCELERATOR_TRANSFER_DTOH);
+        sbuf = sbuf1 - gap;
+    }
+
+    rc = mca_coll_accelerator_check_buf(rbuf, &rbuf_dev);
+    if (0 > rc) {
+        goto exit;
+    }
+    rbufsize = elemsize * ompi_count_array_get(rcounts, ompi_comm_rank(comm));
+    if (0 < rc) {
+        rbuf1 = (char*)malloc(rbufsize);
+        if (NULL == rbuf1) {
+            rc = OMPI_ERR_OUT_OF_RESOURCE;
+	    goto exit;
+        }
+        mca_coll_accelerator_memcpy(rbuf1, MCA_ACCELERATOR_NO_DEVICE_ID, rbuf, rbuf_dev, rbufsize,
+                                    MCA_ACCELERATOR_TRANSFER_DTOH);
+        rbuf2 = rbuf; /* save away original buffer */
+        rbuf = rbuf1 - gap;
+    }
+    rc = s->c_coll.coll_reduce_scatter(sbuf, rbuf, rcounts, dtype, op, comm,
+                                       s->c_coll.coll_reduce_scatter_block_module);
+    if (0 > rc) {
+        goto exit;
+    }
+
+    if (NULL != rbuf1) {
+        mca_coll_accelerator_memcpy(rbuf2, rbuf_dev, rbuf1, MCA_ACCELERATOR_NO_DEVICE_ID, rbufsize,
+                                    MCA_ACCELERATOR_TRANSFER_HTOD);
+    }
+
+ exit:
+    if (NULL != sbuf1) {
+        free(sbuf1);
+    }
+    if (NULL != rbuf1) {
+        free(rbuf1);
+    }
+
+    return rc;
+}
+
diff --git a/ompi/mca/coll/accelerator/coll_accelerator_reduce_scatter_block.c b/ompi/mca/coll/accelerator/coll_accelerator_reduce_scatter_block.c
new file mode 100644
index 00000000000..5f0fd61914f
--- /dev/null
+++ b/ompi/mca/coll/accelerator/coll_accelerator_reduce_scatter_block.c
@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2014-2017 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2015 NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights reserved.
+ * Copyright (c) 2024      Advanced Micro Devices, Inc. All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "coll_accelerator.h"
+
+#include <stdio.h>
+
+#include "ompi/op/op.h"
+#include "opal/datatype/opal_convertor.h"
+
+/*
+ *	reduce_scatter_block
+ *
+ *	Function:	- reduce then scatter
+ *	Accepts:	- same as MPI_Reduce_scatter_block()
+ *	Returns:	- MPI_SUCCESS or error code
+ *
+ * Algorithm:
+ *     reduce and scatter (needs to be cleaned
+ *     up at some point)
+ */
+int
+mca_coll_accelerator_reduce_scatter_block(const void *sbuf, void *rbuf, size_t rcount,
+                                   struct ompi_datatype_t *dtype,
+                                   struct ompi_op_t *op,
+                                   struct ompi_communicator_t *comm,
+                                   mca_coll_base_module_t *module)
+{
+    mca_coll_accelerator_module_t *s = (mca_coll_accelerator_module_t*) module;
+    ptrdiff_t gap;
+    char *rbuf1 = NULL, *sbuf1 = NULL, *rbuf2 = NULL;
+    int sbuf_dev, rbuf_dev;
+    size_t sbufsize, rbufsize;
+    int rc;
+
+    rbufsize = opal_datatype_span(&dtype->super, rcount, &gap);
+
+    sbufsize = rbufsize * ompi_comm_size(comm);
+    rc = mca_coll_accelerator_check_buf((void *)sbuf, &sbuf_dev);
+    if (rc < 0) {
+        return rc;
+    }
+    if ((MPI_IN_PLACE != sbuf) && (rc > 0)) {
+        sbuf1 = (char*)malloc(sbufsize);
+        if (NULL == sbuf1) {
+            return OMPI_ERR_OUT_OF_RESOURCE;
+        }
+        mca_coll_accelerator_memcpy(sbuf1, MCA_ACCELERATOR_NO_DEVICE_ID, sbuf, sbuf_dev, sbufsize,
+                                    MCA_ACCELERATOR_TRANSFER_DTOH);
+        sbuf = sbuf1 - gap;
+    }
+    rc = mca_coll_accelerator_check_buf(rbuf, &rbuf_dev);
+    if (rc < 0) {
+        return rc;
+    }
+    if (rc > 0) {
+        rbuf1 = (char*)malloc(rbufsize);
+        if (NULL == rbuf1) {
+            if (NULL != sbuf1) free(sbuf1);
+            return OMPI_ERR_OUT_OF_RESOURCE;
+        }
+        mca_coll_accelerator_memcpy(rbuf1, MCA_ACCELERATOR_NO_DEVICE_ID, rbuf, rbuf_dev, rbufsize,
+                                    MCA_ACCELERATOR_TRANSFER_DTOH);
+        rbuf2 = rbuf; /* save away original buffer */
+        rbuf = rbuf1 - gap;
+    }
+    rc = s->c_coll.coll_reduce_scatter_block(sbuf, rbuf, rcount, dtype, op, comm,
+                                             s->c_coll.coll_reduce_scatter_block_module);
+    if (NULL != sbuf1) {
+        free(sbuf1);
+    }
+    if (NULL != rbuf1) {
+        rbuf = rbuf2;
+        mca_coll_accelerator_memcpy(rbuf, rbuf_dev, rbuf1, MCA_ACCELERATOR_NO_DEVICE_ID, rbufsize,
+                                    MCA_ACCELERATOR_TRANSFER_HTOD);
+        free(rbuf1);
+    }
+    return rc;
+}
+
diff --git a/ompi/mca/coll/accelerator/coll_accelerator_scan.c b/ompi/mca/coll/accelerator/coll_accelerator_scan.c
new file mode 100644
index 00000000000..7018744c1c8
--- /dev/null
+++ b/ompi/mca/coll/accelerator/coll_accelerator_scan.c
@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2014-2017 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2015 NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights reserved.
+ * Copyright (c) 2024      Advanced Micro Devices, Inc. All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "coll_accelerator.h"
+
+#include <stdio.h>
+
+#include "ompi/op/op.h"
+#include "opal/datatype/opal_convertor.h"
+
+/*
+ *	scan
+ *
+ *	Function:	- scan
+ *	Accepts:	- same arguments as MPI_Scan()
+ *	Returns:	- MPI_SUCCESS or error code
+ */
+int mca_coll_accelerator_scan(const void *sbuf, void *rbuf, size_t count,
+                       struct ompi_datatype_t *dtype,
+                       struct ompi_op_t *op,
+                       struct ompi_communicator_t *comm,
+                       mca_coll_base_module_t *module)
+{
+    mca_coll_accelerator_module_t *s = (mca_coll_accelerator_module_t*) module;
+    ptrdiff_t gap;
+    char *rbuf1 = NULL, *sbuf1 = NULL, *rbuf2 = NULL;
+    int sbuf_dev, rbuf_dev;
+    size_t bufsize;
+    int rc;
+
+    bufsize = opal_datatype_span(&dtype->super, count, &gap);
+    rc = mca_coll_accelerator_check_buf((void *)sbuf, &sbuf_dev);
+    if (rc < 0) {
+        return rc;
+    }
+    if ((MPI_IN_PLACE != sbuf) && (rc > 0)) {
+        sbuf1 = (char*)malloc(bufsize);
+        if (NULL == sbuf1) {
+            return OMPI_ERR_OUT_OF_RESOURCE;
+        }
+        mca_coll_accelerator_memcpy(sbuf1, MCA_ACCELERATOR_NO_DEVICE_ID, sbuf, sbuf_dev, bufsize,
+                                    MCA_ACCELERATOR_TRANSFER_DTOH);
+        sbuf = sbuf1 - gap;
+    }
+    rc = mca_coll_accelerator_check_buf(rbuf, &rbuf_dev);
+    if (rc < 0) {
+        return rc;
+    }
+    if (rc > 0) {
+        rbuf1 = (char*)malloc(bufsize);
+        if (NULL == rbuf1) {
+            if (NULL != sbuf1) free(sbuf1);
+            return OMPI_ERR_OUT_OF_RESOURCE;
+        }
+        mca_coll_accelerator_memcpy(rbuf1, MCA_ACCELERATOR_NO_DEVICE_ID, rbuf, rbuf_dev, bufsize,
+                                    MCA_ACCELERATOR_TRANSFER_DTOH);
+        rbuf2 = rbuf; /* save away original buffer */
+        rbuf = rbuf1 - gap;
+    }
+    rc = s->c_coll.coll_scan(sbuf, rbuf, count, dtype, op, comm,
+                             s->c_coll.coll_scan_module);
+    if (NULL != sbuf1) {
+        free(sbuf1);
+    }
+    if (NULL != rbuf1) {
+        rbuf = rbuf2;
+        mca_coll_accelerator_memcpy(rbuf, rbuf_dev, rbuf1, MCA_ACCELERATOR_NO_DEVICE_ID, bufsize,
+                                    MCA_ACCELERATOR_TRANSFER_HTOD);
+        free(rbuf1);
+    }
+    return rc;
+}
diff --git a/ompi/mca/coll/accelerator/owner.txt b/ompi/mca/coll/accelerator/owner.txt
new file mode 100644
index 00000000000..6db9f5e4715
--- /dev/null
+++ b/ompi/mca/coll/accelerator/owner.txt
@@ -0,0 +1,7 @@
+#
+# owner/status file
+# owner: institution that is responsible for this package
+# status: e.g. active, maintenance, unmaintained
+#
+owner: NVIDIA/AMD
+status: active
diff --git a/ompi/mca/coll/acoll/Makefile.am b/ompi/mca/coll/acoll/Makefile.am
new file mode 100644
index 00000000000..705ed5bf479
--- /dev/null
+++ b/ompi/mca/coll/acoll/Makefile.am
@@ -0,0 +1,46 @@
+#
+# Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+AM_CPPFLAGS = $(coll_acoll_CPPFLAGS)
+
+sources = \
+        coll_acoll.h \
+        coll_acoll_utils.h \
+        coll_acoll_allgather.c \
+        coll_acoll_bcast.c \
+        coll_acoll_gather.c \
+        coll_acoll_alltoall.c \
+        coll_acoll_reduce.c \
+        coll_acoll_allreduce.c \
+        coll_acoll_barrier.c \
+        coll_acoll_component.c \
+        coll_acoll_module.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_ompi_coll_acoll_DSO
+component_noinst =
+component_install = mca_coll_acoll.la
+else
+component_noinst = libmca_coll_acoll.la
+component_install =
+endif
+
+mcacomponentdir = $(ompilibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_coll_acoll_la_SOURCES = $(sources)
+mca_coll_acoll_la_LDFLAGS = -module -avoid-version $(coll_acoll_LDFLAGS)
+mca_coll_acoll_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la $(coll_acoll_LIBS)
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_coll_acoll_la_SOURCES =$(sources)
+libmca_coll_acoll_la_LIBADD = $(coll_acoll_LIBS)
+libmca_coll_acoll_la_LDFLAGS = -module -avoid-version $(coll_acoll_LDFLAGS)
diff --git a/ompi/mca/coll/acoll/README b/ompi/mca/coll/acoll/README
new file mode 100644
index 00000000000..d5b5acae8f1
--- /dev/null
+++ b/ompi/mca/coll/acoll/README
@@ -0,0 +1,16 @@
+Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved.
+
+$COPYRIGHT$
+
+Additional copyrights may follow
+
+$HEADER$
+
+===========================================================================
+
+The collective component, AMD Coll (“acoll”), is a high-performant MPI collective component for the OpenMPI library that is optimized for AMD "Zen"-based processors. “acoll” is optimized for communications within a single node of AMD “Zen”-based processors and provides the following commonly used collective algorithms: boardcast (MPI_Bcast), allreduce (MPI_Allreduce), reduce (MPI_Reduce), gather (MPI_Gather), allgather (MPI_Allgather), and barrier (MPI_Barrier).
+
+At present, “acoll” has been tested with OpenMPI v5.0.2 and can be built as part of OpenMPI.
+
+To run an application with acoll, use the following command line parameters
+-              mpirun <common mpi runtime parameters> --mca coll acoll,tuned,libnbc,basic --mca coll_acoll_priority 40 <executable>
diff --git a/ompi/mca/coll/acoll/coll_acoll.h b/ompi/mca/coll/acoll/coll_acoll.h
new file mode 100644
index 00000000000..e5ec8a381a8
--- /dev/null
+++ b/ompi/mca/coll/acoll/coll_acoll.h
@@ -0,0 +1,263 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2024 - 2025 Advanced Micro Devices, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef MCA_COLL_ACOLL_EXPORT_H
+#define MCA_COLL_ACOLL_EXPORT_H
+
+#include "ompi_config.h"
+
+#include "mpi.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/mca/coll/base/coll_base_functions.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/mca.h"
+#include "ompi/request/request.h"
+
+#ifdef HAVE_XPMEM_H
+#include "opal/mca/rcache/base/base.h"
+#include "opal/class/opal_hash_table.h"
+#include <xpmem.h>
+#endif
+
+#include "opal/mca/shmem/base/base.h"
+#include "opal/mca/shmem/shmem.h"
+
+BEGIN_C_DECLS
+
+/* Globally exported variables */
+OMPI_DECLSPEC extern const mca_coll_base_component_3_0_0_t mca_coll_acoll_component;
+extern int mca_coll_acoll_priority;
+extern int mca_coll_acoll_max_comms;
+extern int mca_coll_acoll_sg_size;
+extern int mca_coll_acoll_sg_scale;
+extern int mca_coll_acoll_node_size;
+extern int mca_coll_acoll_force_numa;
+extern int mca_coll_acoll_use_dynamic_rules;
+extern int mca_coll_acoll_mnode_enable;
+extern int mca_coll_acoll_bcast_lin0;
+extern int mca_coll_acoll_bcast_lin1;
+extern int mca_coll_acoll_bcast_lin2;
+extern int mca_coll_acoll_bcast_nonsg;
+extern int mca_coll_acoll_bcast_socket;
+extern int mca_coll_acoll_allgather_lin;
+extern int mca_coll_acoll_allgather_ring_1;
+
+/* API functions */
+int mca_coll_acoll_init_query(bool enable_progress_threads, bool enable_mpi_threads);
+mca_coll_base_module_t *mca_coll_acoll_comm_query(struct ompi_communicator_t *comm, int *priority);
+
+int mca_coll_acoll_module_enable(mca_coll_base_module_t *module, struct ompi_communicator_t *comm);
+
+int mca_coll_acoll_allgather(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                             void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
+                             struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+int mca_coll_acoll_bcast(void *buff, size_t count, struct ompi_datatype_t *datatype, int root,
+                         struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+int mca_coll_acoll_gather_intra(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                                void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype, int root,
+                                struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+int mca_coll_acoll_alltoall(const void *sbuf, size_t scount,
+                            struct ompi_datatype_t *sdtype,
+                            void* rbuf, size_t rcount,
+                            struct ompi_datatype_t *rdtype,
+                            struct ompi_communicator_t *comm,
+                            mca_coll_base_module_t *module);
+
+int mca_coll_acoll_reduce_intra(const void *sbuf, void *rbuf, size_t count,
+                                struct ompi_datatype_t *dtype, struct ompi_op_t *op, int root,
+                                struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+int mca_coll_acoll_allreduce_intra(const void *sbuf, void *rbuf, size_t count,
+                                   struct ompi_datatype_t *dtype, struct ompi_op_t *op,
+                                   struct ompi_communicator_t *comm,
+                                   mca_coll_base_module_t *module);
+
+int mca_coll_acoll_barrier_intra(struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+END_C_DECLS
+
+#define MCA_COLL_ACOLL_ROOT_CHANGE_THRESH 10
+#define MCA_COLL_ACOLL_SPLIT_FACTOR_LIST_LEN 6
+#define MCA_COLL_ACOLL_SPLIT_FACTOR_LIST {2, 4, 8, 16, 32, 64}
+
+typedef enum MCA_COLL_ACOLL_SG_SIZES {
+    MCA_COLL_ACOLL_SG_SIZE_1 = 8,
+    MCA_COLL_ACOLL_SG_SIZE_2 = 16
+} MCA_COLL_ACOLL_SG_SIZES;
+
+typedef enum MCA_COLL_ACOLL_SG_SCALES {
+    MCA_COLL_ACOLL_SG_SCALE_1 = 1,
+    MCA_COLL_ACOLL_SG_SCALE_2 = 2,
+    MCA_COLL_ACOLL_SG_SCALE_3 = 4,
+    MCA_COLL_ACOLL_SG_SCALE_4 = 8,
+    MCA_COLL_ACOLL_SG_SCALE_5 = 16
+} MCA_COLL_ACOLL_SG_SCALES;
+
+typedef enum MCA_COLL_ACOLL_SUBCOMMS {
+    MCA_COLL_ACOLL_NODE_L = 0,
+    MCA_COLL_ACOLL_INTRA,
+    MCA_COLL_ACOLL_SOCK_L,
+    MCA_COLL_ACOLL_NUMA_L,
+    MCA_COLL_ACOLL_L3_L,
+    MCA_COLL_ACOLL_LEAF,
+    MCA_COLL_ACOLL_NUM_SC
+} MCA_COLL_ACOLL_SUBCOMMS;
+
+typedef enum MCA_COLL_ACOLL_LAYERS {
+    MCA_COLL_ACOLL_LYR_NODE = 0,
+    MCA_COLL_ACOLL_LYR_SOCKET,
+    MCA_COLL_ACOLL_NUM_LAYERS
+} MCA_COLL_ACOLL_LAYERS;
+
+typedef enum MCA_COLL_ACOLL_BASE_LYRS {
+    MCA_COLL_ACOLL_L3CACHE = 0,
+    MCA_COLL_ACOLL_NUMA,
+    MCA_COLL_ACOLL_NUM_BASE_LYRS
+} MCA_COLL_ACOLL_BASE_LYRS;
+
+typedef struct coll_acoll_data {
+#ifdef HAVE_XPMEM_H
+    xpmem_segid_t *allseg_id;
+    xpmem_apid_t *all_apid;
+    void **allshm_sbuf;
+    void **allshm_rbuf;
+    void **xpmem_saddr;
+    void **xpmem_raddr;
+    mca_rcache_base_module_t **rcache;
+    void *scratch;
+    opal_hash_table_t **xpmem_reg_tracker_ht;
+#endif
+    opal_shmem_ds_t *allshmseg_id;
+    void **allshmmmap_sbuf;
+
+    int comm_size;
+    int l1_local_rank;
+    int l2_local_rank;
+    int l1_gp_size;
+    int *l1_gp;
+    int *l2_gp;
+    int l2_gp_size;
+    int offset[4];
+    int sync[2];
+} coll_acoll_data_t;
+
+/* The enum literals are used as indices into arrays and values are
+ * assigned to the enum literals so as to ensure it is valid irrespective
+ * of what the compiler assigns. */
+typedef enum MCA_COLL_ACOLL_R2R_DIST {
+    DIST_CORE = 0,
+    DIST_L3CACHE,
+    DIST_NUMA,
+    DIST_SOCKET,
+    DIST_NODE,
+    DIST_END
+} MCA_COLL_ACOLL_R2R_DIST_T;
+
+typedef struct coll_acoll_subcomms {
+    ompi_communicator_t *local_comm;
+    ompi_communicator_t *local_r_comm;
+    ompi_communicator_t *leader_comm;
+    ompi_communicator_t *subgrp_comm;
+    ompi_communicator_t *numa_comm;
+    ompi_communicator_t *base_comm[MCA_COLL_ACOLL_NUM_BASE_LYRS][MCA_COLL_ACOLL_NUM_LAYERS];
+    ompi_communicator_t *orig_comm;
+    ompi_communicator_t *socket_comm;
+    ompi_communicator_t *socket_ldr_comm;
+    ompi_communicator_t *split_comm[MCA_COLL_ACOLL_SPLIT_FACTOR_LIST_LEN]; // AllToAll odd even split comm
+    int num_nodes;
+    int derived_node_size;
+    int is_root_node;
+    int is_root_sg;
+    int is_root_numa;
+    int is_root_socket;
+    int local_root[MCA_COLL_ACOLL_NUM_LAYERS];
+    int outer_grp_root;
+    int subgrp_root;
+    int numa_root;
+    int socket_ldr_root;
+    int base_root[MCA_COLL_ACOLL_NUM_BASE_LYRS][MCA_COLL_ACOLL_NUM_LAYERS];
+    int base_rank[MCA_COLL_ACOLL_NUM_BASE_LYRS][MCA_COLL_ACOLL_NUM_LAYERS];
+    int socket_rank;
+    int subgrp_size;
+    int initialized;
+    int prev_init_root;
+    int num_root_change;
+    MCA_COLL_ACOLL_R2R_DIST_T r2r_dist;
+
+    ompi_communicator_t *numa_comm_ldrs;
+    ompi_communicator_t *node_comm;
+    ompi_communicator_t *inter_comm;
+    int cid;
+    coll_acoll_data_t *data;
+    bool initialized_data;
+    bool initialized_shm_data;
+#ifdef HAVE_XPMEM_H
+    uint64_t xpmem_buf_size;
+    int without_xpmem;
+    int xpmem_use_sr_buf;
+#endif
+
+} coll_acoll_subcomms_t;
+
+typedef struct coll_acoll_reserve_mem {
+    void *reserve_mem;
+    uint64_t reserve_mem_size;
+    bool reserve_mem_allocate;
+    bool reserve_mem_in_use;
+} coll_acoll_reserve_mem_t;
+
+typedef struct {
+    int split_factor;
+    size_t psplit_msg_thresh;
+    size_t xpmem_msg_thresh;
+} coll_acoll_alltoall_attr_t;
+
+struct mca_coll_acoll_module_t {
+    mca_coll_base_module_t super;
+    MCA_COLL_ACOLL_SG_SIZES sg_size;
+    MCA_COLL_ACOLL_SG_SCALES sg_scale;
+    int sg_cnt;
+    // Todo: Remove log2 variables
+    int log2_sg_cnt;
+    int node_cnt;
+    int log2_node_cnt;
+    int force_numa;
+    int use_dyn_rules;
+    // Todo: Use substructure for every API related ones
+    int use_mnode;
+    int use_lin0;
+    int use_lin1;
+    int use_lin2;
+    int use_socket;
+    int mnode_sg_size;
+    int mnode_log2_sg_size;
+    int allg_lin;
+    int allg_ring;
+    int max_comms;
+    coll_acoll_subcomms_t **subc;
+    coll_acoll_reserve_mem_t reserve_mem_s;
+    int num_subc;
+    coll_acoll_alltoall_attr_t alltoall_attr;
+};
+
+#ifdef HAVE_XPMEM_H
+struct acoll_xpmem_rcache_reg_t {
+    mca_rcache_base_registration_t base;
+    void *xpmem_vaddr;
+};
+#endif
+
+typedef struct mca_coll_acoll_module_t mca_coll_acoll_module_t;
+OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_coll_acoll_module_t);
+
+#endif /* MCA_COLL_ACOLL_EXPORT_H */
diff --git a/ompi/mca/coll/acoll/coll_acoll_allgather.c b/ompi/mca/coll/acoll/coll_acoll_allgather.c
new file mode 100644
index 00000000000..3fc2167193f
--- /dev/null
+++ b/ompi/mca/coll/acoll/coll_acoll_allgather.c
@@ -0,0 +1,625 @@
+/* -*- Mode: C; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+
+#include "mpi.h"
+#include "ompi/constants.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_functions.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/mca/coll/base/coll_tags.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/pml/pml.h"
+#include "opal/util/bit_ops.h"
+#include "coll_acoll.h"
+#include "coll_acoll_utils.h"
+
+static inline int log_sg_bcast_intra(void *buff, size_t count, struct ompi_datatype_t *datatype,
+                                     int rank, int dim, int size, int sg_size, int cur_base,
+                                     int sg_start, struct ompi_communicator_t *comm,
+                                     mca_coll_base_module_t *module, ompi_request_t **preq,
+                                     int *nreqs)
+{
+    int msb_pos, sub_rank, peer, err;
+    int i, mask;
+    int end_sg, end_peer;
+
+    end_sg = sg_start + sg_size - 1;
+    if (end_sg >= size) {
+        end_sg = size - 1;
+    }
+    end_peer = (end_sg - cur_base) % sg_size;
+    sub_rank = (rank - cur_base + sg_size) % sg_size;
+
+    msb_pos = opal_hibit(sub_rank, dim);
+    --dim;
+
+    /* Receive data from parent in the sg tree. */
+    if (sub_rank > 0) {
+        assert(msb_pos >= 0);
+        peer = (sub_rank & ~(1 << msb_pos));
+        if (peer > end_peer) {
+            peer = (((peer + cur_base - sg_start) % sg_size) + sg_start);
+        } else {
+            peer = peer + cur_base;
+        }
+
+        err = MCA_PML_CALL(
+            recv(buff, count, datatype, peer, MCA_COLL_BASE_TAG_ALLGATHER, comm, MPI_STATUS_IGNORE));
+        if (MPI_SUCCESS != err) {
+            return err;
+        }
+    }
+
+    for (i = msb_pos + 1, mask = 1 << i; i <= dim; ++i, mask <<= 1) {
+        peer = sub_rank | mask;
+        if (peer >= sg_size) {
+            continue;
+        }
+        if (peer >= end_peer) {
+            peer = (((peer + cur_base - sg_start) % sg_size) + sg_start);
+        } else {
+            peer = peer + cur_base;
+        }
+        /* Checks to ensure that the sends are limited to the necessary ones.
+           It also ensures 'preq' not exceeding the max allocated. */
+        if ((peer < size) && (peer != rank) && (peer != cur_base)) {
+            *nreqs = *nreqs + 1;
+            err = MCA_PML_CALL(isend(buff, count, datatype, peer, MCA_COLL_BASE_TAG_ALLGATHER,
+                                     MCA_PML_BASE_SEND_STANDARD, comm, preq++));
+            if (MPI_SUCCESS != err) {
+                return err;
+            }
+        }
+    }
+
+    return err;
+}
+
+static inline int lin_sg_bcast_intra(void *buff, size_t count, struct ompi_datatype_t *datatype,
+                                     int rank, int dim, int size, int sg_size, int cur_base,
+                                     int sg_start, struct ompi_communicator_t *comm,
+                                     mca_coll_base_module_t *module, ompi_request_t **preq,
+                                     int *nreqs)
+{
+    int peer;
+    int err;
+    int sg_end;
+
+    sg_end = sg_start + sg_size - 1;
+    if (sg_end >= size) {
+        sg_end = size - 1;
+    }
+
+    if (rank == cur_base) {
+        for (peer = sg_start; peer <= sg_end; peer++) {
+            if (peer == cur_base) {
+                continue;
+            }
+            *nreqs = *nreqs + 1;
+            err = MCA_PML_CALL(isend(buff, count, datatype, peer, MCA_COLL_BASE_TAG_ALLGATHER,
+                                     MCA_PML_BASE_SEND_STANDARD, comm, preq++));
+            if (MPI_SUCCESS != err) {
+                return err;
+            }
+        }
+    } else {
+        err = MCA_PML_CALL(recv(buff, count, datatype, cur_base, MCA_COLL_BASE_TAG_ALLGATHER, comm,
+                                MPI_STATUS_IGNORE));
+        if (MPI_SUCCESS != err) {
+            return err;
+        }
+    }
+
+    return err;
+}
+
+/*
+ * sg_bcast_intra
+ *
+ * Function:    broadcast operation within a subgroup
+ * Accepts:     Arguments of MPI_Bcast() plus subgroup params
+ * Returns:     MPI_SUCCESS or error code
+ *
+ * Description: O(N) or O(log(N)) algorithm based on count.
+ *
+ * Memory:      No additional memory requirements beyond user-supplied buffers.
+ *
+ */
+static inline int sg_bcast_intra(void *buff, size_t count, struct ompi_datatype_t *datatype, int rank,
+                                 int dim, int size, int sg_size, int cur_base, int sg_start,
+                                 struct ompi_communicator_t *comm, mca_coll_base_module_t *module,
+                                 ompi_request_t **preq, int *nreqs)
+{
+    int err;
+    size_t total_dsize, dsize;
+
+    ompi_datatype_type_size(datatype, &dsize);
+    total_dsize = dsize * count;
+
+    if (total_dsize <= 8192) {
+        err = log_sg_bcast_intra(buff, count, datatype, rank, dim, size, sg_size, cur_base,
+                                 sg_start, comm, module, preq, nreqs);
+    } else {
+        err = lin_sg_bcast_intra(buff, count, datatype, rank, dim, size, sg_size, cur_base,
+                                 sg_start, comm, module, preq, nreqs);
+    }
+    return err;
+}
+
+/*
+ * coll_allgather_decision_fixed
+ *
+ * Function:    Choose optimal allgather algorithm
+ *
+ * Description: Based on no. of processes and message size, chooses whether
+ *              or not to use subgroups. If subgroup based algorithm is not,
+ *              chosen, further decides if [ring|lin] allgather is to be used.
+ *
+ */
+static inline void coll_allgather_decision_fixed(int size, size_t total_dsize, int sg_size,
+                                                 int *use_ring, int *use_lin)
+{
+    *use_ring = 0;
+    *use_lin = 0;
+    if (size <= (sg_size << 1)) {
+        if (total_dsize >= 1048576) {
+            *use_lin = 1;
+        }
+    } else if (size <= (sg_size << 2)) {
+        if ((total_dsize >= 4096) && (total_dsize < 32768)) {
+            *use_ring = 1;
+        } else if (total_dsize >= 1048576) {
+            *use_lin = 1;
+        }
+    } else if (size <= (sg_size << 3)) {
+        if ((total_dsize >= 4096) && (total_dsize < 32768)) {
+            *use_ring = 1;
+        }
+    } else {
+        if (total_dsize >= 4096) {
+            *use_ring = 1;
+        }
+    }
+}
+
+/*
+ * rd_allgather_sub
+ *
+ * Function:    Uses recursive doubling based allgather for the group.
+ *              Group can be all ranks in a subgroup or base ranks across
+ *              subgroups.
+ *
+ * Description: Implementation logic of recursive doubling reused from
+ *              ompi_coll_base_allgather_intra_recursivedoubling().
+ *
+ */
+static inline int rd_allgather_sub(void *rbuf, struct ompi_datatype_t *rdtype,
+                                   struct ompi_communicator_t *comm, size_t count, int send_blk_loc,
+                                   int rank, int virtual_rank, int grp_size, const int across_sg,
+                                   int sg_start, int sg_size, ptrdiff_t rext)
+{
+    int err;
+    /* At step i, rank r exchanges message with rank (r ^ 2^i) */
+    for (int dist = 0x1, i = 0; dist < grp_size; dist <<= 1, i++) {
+        int remote = virtual_rank ^ dist;
+        int recv_blk_loc = virtual_rank < remote ? send_blk_loc + dist : send_blk_loc - dist;
+        size_t sr_cnt = count << i;
+        char *tmpsend = (char *) rbuf + (ptrdiff_t) send_blk_loc * (ptrdiff_t) count * rext;
+        char *tmprecv = (char *) rbuf + (ptrdiff_t) recv_blk_loc * (ptrdiff_t) count * rext;
+        int peer = across_sg ? remote * sg_size : remote + sg_start;
+        if (virtual_rank >= remote) {
+            send_blk_loc -= dist;
+        }
+
+        /* Sendreceive */
+        err = ompi_coll_base_sendrecv(tmpsend, sr_cnt, rdtype, peer, MCA_COLL_BASE_TAG_ALLGATHER,
+                                      tmprecv, sr_cnt, rdtype, peer, MCA_COLL_BASE_TAG_ALLGATHER,
+                                      comm, MPI_STATUS_IGNORE, rank);
+        if (MPI_SUCCESS != err) {
+            return err;
+        }
+    }
+
+    return err;
+}
+
+static inline int mca_coll_acoll_allgather_intra(const void *sbuf, size_t scount,
+                                                 struct ompi_datatype_t *sdtype, void *rbuf,
+                                                 size_t rcount, struct ompi_datatype_t *rdtype,
+                                                 struct ompi_communicator_t *comm,
+                                                 mca_coll_base_module_t *module)
+{
+    int i;
+    int err;
+    int size;
+    int rank, adj_rank;
+    int sg_id, num_sgs, is_pow2_num_sgs;
+    int sg_start, sg_end;
+    int sg_size, log2_sg_size;
+    int subgrp_size, last_subgrp_size;
+    ptrdiff_t rlb, rext;
+    char *tmpsend = NULL, *tmprecv = NULL;
+    int sendto, recvfrom;
+    int num_data_blks;
+    size_t data_blk_size[2] = {0}, blk_ofst[2] = {0};
+    size_t bcount;
+    size_t last_subgrp_rcnt;
+    int brank, last_brank;
+    int use_rd_base, use_ring_sg;
+    int use_ring = 0, use_lin = 0;
+    int nreqs;
+    ompi_request_t **preq, **reqs;
+    size_t dsize;
+    mca_coll_acoll_module_t *acoll_module = (mca_coll_acoll_module_t *) module;
+
+    err = ompi_datatype_get_extent(rdtype, &rlb, &rext);
+    if (MPI_SUCCESS != err) {
+        return err;
+    }
+
+    ompi_datatype_type_size(rdtype, &dsize);
+    size = ompi_comm_size(comm);
+    rank = ompi_comm_rank(comm);
+    sg_size = acoll_module->sg_cnt;
+    log2_sg_size = acoll_module->log2_sg_cnt;
+
+    /* Handle non MPI_IN_PLACE */
+    tmprecv = (char *) rbuf + (ptrdiff_t) rank * (ptrdiff_t) rcount * rext;
+    if (MPI_IN_PLACE != sbuf) {
+        tmpsend = (char *) sbuf;
+        err = ompi_datatype_sndrcv(tmpsend, scount, sdtype, tmprecv, rcount, rdtype);
+        if (MPI_SUCCESS != err) {
+            return err;
+        }
+    }
+
+    /* Derive subgroup parameters */
+    sg_id = rank >> log2_sg_size;
+    num_sgs = (size + sg_size - 1) >> log2_sg_size;
+    sg_start = sg_id << log2_sg_size;
+    sg_end = sg_start + sg_size;
+    if (sg_end > size) {
+        sg_end = size;
+    }
+    subgrp_size = sg_end - sg_start;
+    last_subgrp_size = size - ((num_sgs - 1) << log2_sg_size);
+    last_subgrp_rcnt = rcount * last_subgrp_size;
+    use_ring_sg = (subgrp_size != sg_size) ? 1 : 0;
+    bcount = rcount << log2_sg_size;
+
+    /* Override subgroup params based on data size */
+    coll_allgather_decision_fixed(size, dsize * rcount, sg_size, &use_ring, &use_lin);
+
+    if (use_lin) {
+        err = ompi_coll_base_allgather_intra_basic_linear(sbuf, scount, sdtype, rbuf, rcount,
+                                                          rdtype, comm, module);
+        return err;
+    }
+    if (use_ring) {
+        sg_size = sg_end = subgrp_size = size;
+        num_sgs = 1;
+        use_ring_sg = 1;
+        sg_start = 0;
+    }
+
+    /* Do ring/recursive doubling based allgather within subgroup */
+    adj_rank = rank - sg_start;
+    if (use_ring_sg) {
+        recvfrom = ((adj_rank - 1 + subgrp_size) % subgrp_size) + sg_start;
+        sendto = ((adj_rank + 1) % subgrp_size) + sg_start;
+
+        /* Loop over ranks in subgroup */
+        for (i = 0; i < (subgrp_size - 1); i++) {
+            int recv_peer = ((adj_rank - i - 1 + subgrp_size) % subgrp_size) + sg_start;
+            int send_peer = ((adj_rank - i + subgrp_size) % subgrp_size) + sg_start;
+
+            tmprecv = (char *) rbuf + (ptrdiff_t) recv_peer * (ptrdiff_t) rcount * rext;
+            tmpsend = (char *) rbuf + (ptrdiff_t) send_peer * (ptrdiff_t) rcount * rext;
+
+            /* Sendreceive */
+            err = ompi_coll_base_sendrecv(tmpsend, rcount, rdtype, sendto,
+                                          MCA_COLL_BASE_TAG_ALLGATHER, tmprecv, rcount, rdtype,
+                                          recvfrom, MCA_COLL_BASE_TAG_ALLGATHER, comm,
+                                          MPI_STATUS_IGNORE, rank);
+            if (MPI_SUCCESS != err) {
+                return err;
+            }
+        }
+    } else {
+        err = rd_allgather_sub(rbuf, rdtype, comm, rcount, rank, rank, adj_rank, sg_size, 0,
+                               sg_start, sg_size, rext);
+        if (MPI_SUCCESS != err) {
+            return err;
+        }
+    }
+
+    /* Return if all ranks belong to single subgroup */
+    if (1 == num_sgs) {
+        /* All done */
+        return err;
+    }
+
+    /* Do ring/rd based allgather across start ranks of subgroups */
+    is_pow2_num_sgs = 0;
+    if (num_sgs == (1 << opal_hibit(num_sgs, comm->c_cube_dim))) {
+        is_pow2_num_sgs = 1;
+    }
+    use_rd_base = is_pow2_num_sgs ? ((last_subgrp_rcnt == bcount) ? 1 : 0) : 0;
+
+    brank = sg_id;
+    last_brank = num_sgs - 1;
+
+    /* Use ring for non-power of 2 cases */
+    if (!(rank & (sg_size - 1)) && !use_rd_base) {
+        recvfrom = ((brank - 1 + num_sgs) % num_sgs) << log2_sg_size;
+        sendto = ((brank + 1) % num_sgs) << log2_sg_size;
+
+        /* Loop over subgroups */
+        for (i = 0; i < (num_sgs - 1); i++) {
+            int recv_peer = ((brank - i - 1 + num_sgs) % num_sgs);
+            int send_peer = ((brank - i + num_sgs) % num_sgs);
+            size_t scnt = (send_peer == last_brank) ? last_subgrp_rcnt : bcount;
+            size_t rcnt = (recv_peer == last_brank) ? last_subgrp_rcnt : bcount;
+
+            tmprecv = (char *) rbuf + (ptrdiff_t) recv_peer * (ptrdiff_t) bcount * rext;
+            tmpsend = (char *) rbuf + (ptrdiff_t) send_peer * (ptrdiff_t) bcount * rext;
+
+            recv_peer <<= log2_sg_size;
+            send_peer <<= log2_sg_size;
+
+            /* Sendreceive */
+            err = ompi_coll_base_sendrecv(tmpsend, scnt, rdtype, sendto,
+                                          MCA_COLL_BASE_TAG_ALLGATHER, tmprecv, rcnt, rdtype,
+                                          recvfrom, MCA_COLL_BASE_TAG_ALLGATHER, comm,
+                                          MPI_STATUS_IGNORE, rank);
+            if (MPI_SUCCESS != err) {
+                return err;
+            }
+        }
+    } else if (!(rank & (sg_size - 1))) {
+        /* Use recursive doubling for power of 2 cases */
+        err = rd_allgather_sub(rbuf, rdtype, comm, bcount, brank, rank, brank, num_sgs, 1, sg_start,
+                               sg_size, rext);
+        if (MPI_SUCCESS != err) {
+            return err;
+        }
+    }
+    /* Now all base ranks have the full data */
+    /* Do broadcast within subgroups from the base ranks for the extra data */
+    if (0 == sg_id) {
+        num_data_blks = 1;
+        data_blk_size[0] = bcount * (num_sgs - 2) + last_subgrp_rcnt;
+        blk_ofst[0] = bcount;
+    } else if (sg_id == num_sgs - 1) {
+        if (last_subgrp_size < 2) {
+            return err;
+        }
+        num_data_blks = 1;
+        data_blk_size[0] = bcount * (num_sgs - 1);
+        blk_ofst[0] = 0;
+    } else {
+        num_data_blks = 2;
+        data_blk_size[0] = bcount * sg_id;
+        data_blk_size[1] = bcount * (num_sgs - sg_id - 2) + last_subgrp_rcnt;
+        blk_ofst[0] = 0;
+        blk_ofst[1] = bcount * (sg_id + 1);
+    }
+    reqs = ompi_coll_base_comm_get_reqs(module->base_data, size);
+    if (NULL == reqs) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+    nreqs = 0;
+    preq = reqs;
+    /* Loop over data blocks */
+    for (i = 0; i < num_data_blks; i++) {
+        char *buff = (char *) rbuf + (ptrdiff_t) blk_ofst[i] * rext;
+        int sg_dim = opal_hibit(subgrp_size - 1, comm->c_cube_dim);
+        if ((1 << sg_dim) < subgrp_size) {
+            sg_dim++;
+        }
+        /* The size parameters to sg_bcast_intra ensures that the no. of send
+           requests do not exceed the max allocated. */
+        err = sg_bcast_intra(buff, data_blk_size[i], rdtype, rank, sg_dim, size, sg_size, sg_start,
+                             sg_start, comm, module, preq, &nreqs);
+        if (MPI_SUCCESS != err) {
+            ompi_coll_base_free_reqs(reqs, nreqs);
+            return err;
+        }
+    }
+    /* Start and wait on all requests. */
+    if (nreqs > 0) {
+        err = ompi_request_wait_all(nreqs, reqs, MPI_STATUSES_IGNORE);
+        if (MPI_SUCCESS != err) {
+            ompi_coll_base_free_reqs(reqs, nreqs);
+        }
+    }
+
+    /* All done */
+    return err;
+}
+
+/*
+ * mca_coll_acoll_allgather
+ *
+ * Function:    Allgather operation using subgroup based algorithm
+ * Accepts:     Same arguments as MPI_Allgather()
+ * Returns:     MPI_SUCCESS or error code
+ *
+ * Description: Allgather is performed across and within subgroups.
+ *              Subgroups can be 1 or more based on size and count.
+ *
+ * Memory:      No additional memory requirements beyond user-supplied buffers.
+ *
+ */
+int mca_coll_acoll_allgather(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                             void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
+                             struct ompi_communicator_t *comm, mca_coll_base_module_t *module)
+{
+    int i;
+    int err;
+    int size;
+    int rank;
+    int num_nodes, node_start, node_end, node_id;
+    int node_size, last_node_size;
+    ptrdiff_t rlb, rext;
+    char *tmpsend = NULL, *tmprecv = NULL;
+    int sendto, recvfrom;
+    int num_data_blks;
+    size_t data_blk_size[2] = {0}, blk_ofst[2] = {0};
+    size_t bcount;
+    size_t last_subgrp_rcnt;
+    int brank, last_brank;
+    int use_rd_base;
+    mca_coll_acoll_module_t *acoll_module = (mca_coll_acoll_module_t *) module;
+    coll_acoll_subcomms_t *subc = NULL;
+    char *local_rbuf;
+    ompi_communicator_t *intra_comm;
+
+    /* Obtain the subcomms structure */
+    err = check_and_create_subc(comm, acoll_module, &subc);
+    /* Fallback to ring if subc is not obtained */
+    if (NULL == subc) {
+        return ompi_coll_base_allgather_intra_ring(sbuf, scount, sdtype, rbuf, rcount, rdtype, comm,
+                                                   module);
+    }
+
+    size = ompi_comm_size(comm);
+    if (!subc->initialized && size > 2) {
+        err = mca_coll_acoll_comm_split_init(comm, acoll_module, subc, 0);
+        if (MPI_SUCCESS != err) {
+            return err;
+        }
+    }
+
+    err = ompi_datatype_get_extent(rdtype, &rlb, &rext);
+    if (MPI_SUCCESS != err) {
+        return err;
+    }
+
+    rank = ompi_comm_rank(comm);
+    node_size = size > 2 ? subc->derived_node_size : size;
+
+    /* Derive node parameters */
+    num_nodes = (size + node_size - 1) / node_size;
+    node_id = rank / node_size;
+    node_start = node_id * node_size;
+    node_end = node_start + node_size;
+    if (node_end > size) {
+        node_end = size;
+    }
+    last_node_size = size - (num_nodes - 1) * node_size;
+
+    /* Call intra */
+    local_rbuf = (char *) rbuf + (ptrdiff_t) node_start * (ptrdiff_t) rcount * rext;
+    if (size <= 2) {
+        intra_comm = comm;
+    } else {
+        if (num_nodes > 1) {
+            assert(subc->local_r_comm != NULL);
+        }
+        intra_comm = 1 == num_nodes ? comm : subc->local_r_comm;
+    }
+    err = mca_coll_acoll_allgather_intra(sbuf, scount, sdtype, local_rbuf, rcount, rdtype,
+                                         intra_comm, module);
+    if (MPI_SUCCESS != err) {
+        return err;
+    }
+
+    /* Return if intra-node communicator */
+    if ((1 == num_nodes) || (size <= 2)) {
+        /* All done */
+        return err;
+    }
+
+    /* Handle inter-case by first doing allgather across node leaders */
+    bcount = node_size * rcount;
+    last_subgrp_rcnt = last_node_size * rcount;
+
+    /* Perform allgather across node leaders */
+    if (rank == node_start) {
+        int is_pow2_num_nodes = 0;
+        if (num_nodes == (1 << opal_hibit(num_nodes, comm->c_cube_dim))) {
+            is_pow2_num_nodes = 1;
+        }
+        use_rd_base = is_pow2_num_nodes ? ((last_node_size == node_size) ? 1 : 0) : 0;
+        brank = node_id;
+        last_brank = num_nodes - 1;
+
+        /* Use ring for non-power of 2 cases */
+        if (!use_rd_base) {
+            recvfrom = ((brank - 1 + num_nodes) % num_nodes) * node_size;
+            sendto = ((brank + 1) % num_nodes) * node_size;
+
+            /* Loop over nodes */
+            for (i = 0; i < (num_nodes - 1); i++) {
+                int recv_peer = ((brank - i - 1 + num_nodes) % num_nodes);
+                int send_peer = ((brank - i + num_nodes) % num_nodes);
+                size_t scnt = (send_peer == last_brank) ? last_subgrp_rcnt : bcount;
+                size_t rcnt = (recv_peer == last_brank) ? last_subgrp_rcnt : bcount;
+
+                tmprecv = (char *) rbuf + (ptrdiff_t) recv_peer * (ptrdiff_t) bcount * rext;
+                tmpsend = (char *) rbuf + (ptrdiff_t) send_peer * (ptrdiff_t) bcount * rext;
+                recv_peer *= node_size;
+                send_peer *= node_size;
+
+                /* Sendreceive */
+                err = ompi_coll_base_sendrecv(tmpsend, scnt, rdtype, sendto,
+                                              MCA_COLL_BASE_TAG_ALLGATHER, tmprecv, rcnt, rdtype,
+                                              recvfrom, MCA_COLL_BASE_TAG_ALLGATHER, comm,
+                                              MPI_STATUS_IGNORE, rank);
+                if (MPI_SUCCESS != err) {
+                    return err;
+                }
+            }
+        } else {
+            /* Use recursive doubling for power of 2 cases */
+            err = rd_allgather_sub(rbuf, rdtype, comm, bcount, brank, rank, brank, num_nodes, 1,
+                                   node_start, node_size, rext);
+            if (MPI_SUCCESS != err) {
+                return err;
+            }
+        }
+    } /* End of if inter leader */
+
+    /* Do intra node broadcast */
+    if (0 == node_id) {
+        num_data_blks = 1;
+        data_blk_size[0] = bcount * (num_nodes - 2) + last_subgrp_rcnt;
+        blk_ofst[0] = bcount;
+    } else if (node_id == num_nodes - 1) {
+        if (last_node_size < 2) {
+            return err;
+        }
+        num_data_blks = 1;
+        data_blk_size[0] = bcount * (num_nodes - 1);
+        blk_ofst[0] = 0;
+    } else {
+        num_data_blks = 2;
+        data_blk_size[0] = bcount * node_id;
+        data_blk_size[1] = bcount * (num_nodes - node_id - 2) + last_subgrp_rcnt;
+        blk_ofst[0] = 0;
+        blk_ofst[1] = bcount * (node_id + 1);
+    }
+    /* Loop over data blocks */
+    for (i = 0; i < num_data_blks; i++) {
+        char *buff = (char *) rbuf + (ptrdiff_t) blk_ofst[i] * rext;
+        err = ompi_coll_base_bcast_intra_basic_linear(buff, data_blk_size[i], rdtype, 0, subc->local_r_comm,
+                                         module);
+        if (MPI_SUCCESS != err) {
+            return err;
+        }
+    }
+
+    /* All done */
+    return err;
+}
diff --git a/ompi/mca/coll/acoll/coll_acoll_allreduce.c b/ompi/mca/coll/acoll/coll_acoll_allreduce.c
new file mode 100644
index 00000000000..79ef9c4807e
--- /dev/null
+++ b/ompi/mca/coll/acoll/coll_acoll_allreduce.c
@@ -0,0 +1,550 @@
+/* -*- Mode: C; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+
+#include "mpi.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/constants.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_tags.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/op/op.h"
+#include "opal/util/bit_ops.h"
+#include "coll_acoll.h"
+#include "coll_acoll_utils.h"
+
+
+void mca_coll_acoll_sync(coll_acoll_data_t *data, int offset, int *group, int gp_size, int rank, int up);
+int mca_coll_acoll_allreduce_small_msgs_h(const void *sbuf, void *rbuf, size_t count,
+                                          struct ompi_datatype_t *dtype, struct ompi_op_t *op,
+                                          struct ompi_communicator_t *comm,
+                                          mca_coll_base_module_t *module,
+                                          coll_acoll_subcomms_t *subc, int intra);
+
+
+static inline int coll_allreduce_decision_fixed(int comm_size, size_t msg_size)
+{
+    int alg = 3;
+    if (msg_size <= 256) {
+        alg = 1;
+    } else if (msg_size <= 1045876) {
+        alg = 2;
+    } else if (msg_size <= 4194304) {
+        alg = 3;
+    } else if (msg_size <= 8388608) {
+        alg = 0;
+    } else {
+        alg = 3;
+    }
+    return alg;
+}
+
+#ifdef HAVE_XPMEM_H
+static inline int mca_coll_acoll_reduce_xpmem_h(const void *sbuf, void *rbuf, size_t count,
+                                                struct ompi_datatype_t *dtype, struct ompi_op_t *op,
+                                                struct ompi_communicator_t *comm,
+                                                mca_coll_base_module_t *module,
+                                                coll_acoll_subcomms_t *subc)
+{
+    int size;
+    size_t total_dsize, dsize;
+
+    coll_acoll_init(module, comm, subc->data, subc, 0);
+    coll_acoll_data_t *data = subc->data;
+    if (NULL == data) {
+        return -1;
+    }
+
+    size = ompi_comm_size(comm);
+    int rank = ompi_comm_rank(comm);
+    ompi_datatype_type_size(dtype, &dsize);
+    total_dsize = dsize * count;
+
+    int l1_gp_size = data->l1_gp_size;
+    int *l1_gp = data->l1_gp;
+    int *l2_gp = data->l2_gp;
+    int l2_gp_size = data->l2_gp_size;
+
+    int l1_local_rank = data->l1_local_rank;
+    int l2_local_rank = data->l2_local_rank;
+    char *tmp_sbuf = NULL;
+    char *tmp_rbuf = NULL;
+    if (!subc->xpmem_use_sr_buf) {
+        tmp_rbuf = (char *) data->scratch;
+        tmp_sbuf = (char *) data->scratch + (subc->xpmem_buf_size) / 2;
+        if ((MPI_IN_PLACE == sbuf)) {
+            memcpy(tmp_sbuf, rbuf, total_dsize);
+        } else {
+            memcpy(tmp_sbuf, sbuf, total_dsize);
+        }
+    } else {
+        tmp_sbuf = (char *) sbuf;
+        tmp_rbuf = (char *) rbuf;
+        if (MPI_IN_PLACE == sbuf) {
+            tmp_sbuf = (char *) rbuf;
+        }
+    }
+    void *sbuf_vaddr[1] = {tmp_sbuf};
+    void *rbuf_vaddr[1] = {tmp_rbuf};
+    int err = MPI_SUCCESS;
+
+    err = comm->c_coll->coll_allgather(sbuf_vaddr, sizeof(void *), MPI_BYTE, data->allshm_sbuf,
+                                       sizeof(void *), MPI_BYTE, comm,
+                                       comm->c_coll->coll_allgather_module);
+    if (err != MPI_SUCCESS) {
+        return err;
+    }
+
+    err = comm->c_coll->coll_allgather(rbuf_vaddr, sizeof(void *), MPI_BYTE, data->allshm_rbuf,
+                                       sizeof(void *), MPI_BYTE, comm,
+                                       comm->c_coll->coll_allgather_module);
+    if (err != MPI_SUCCESS) {
+        return err;
+    }
+
+    register_and_cache(size, total_dsize, rank, data);
+
+    /* reduce to the local group leader */
+    size_t chunk = count / l1_gp_size;
+    size_t my_count_size = (l1_local_rank == (l1_gp_size - 1)) ? chunk + count % l1_gp_size : chunk;
+
+    if (rank == l1_gp[0]) {
+        if (sbuf != MPI_IN_PLACE)
+            memcpy(tmp_rbuf, sbuf, my_count_size * dsize);
+
+        for (int i = 1; i < l1_gp_size; i++) {
+            ompi_op_reduce(op, (char *) data->xpmem_saddr[l1_gp[i]] + chunk * l1_local_rank * dsize,
+                           (char *) tmp_rbuf + chunk * l1_local_rank * dsize, my_count_size, dtype);
+        }
+    } else {
+        ompi_3buff_op_reduce(op,
+                             (char *) data->xpmem_saddr[l1_gp[0]] + chunk * l1_local_rank * dsize,
+                             (char *) tmp_sbuf + chunk * l1_local_rank * dsize,
+                             (char *) data->xpmem_raddr[l1_gp[0]] + chunk * l1_local_rank * dsize,
+                             my_count_size, dtype);
+        for (int i = 1; i < l1_gp_size; i++) {
+            if (i == l1_local_rank) {
+                continue;
+            }
+            ompi_op_reduce(op, (char *) data->xpmem_saddr[l1_gp[i]] + chunk * l1_local_rank * dsize,
+                           (char *) data->xpmem_raddr[l1_gp[0]] + chunk * l1_local_rank * dsize,
+                           my_count_size, dtype);
+        }
+    }
+    err = ompi_coll_base_barrier_intra_tree(comm, module);
+    if (err != MPI_SUCCESS) {
+        return err;
+    }
+
+    /* perform reduce to 0 */
+    int local_size = l2_gp_size;
+    if ((rank == l1_gp[0]) && (local_size > 1)) {
+        chunk = count / local_size;
+
+        my_count_size = (l2_local_rank == (local_size - 1)) ? chunk + (count % local_size) : chunk;
+
+        if (0 == l2_local_rank) {
+            for (int i = 1; i < local_size; i++) {
+                ompi_op_reduce(op, (char *) data->xpmem_raddr[l2_gp[i]], (char *) tmp_rbuf,
+                               my_count_size, dtype);
+            }
+        } else {
+            for (int i = 1; i < local_size; i++) {
+                if (i == l2_local_rank) {
+                    continue;
+                }
+
+                ompi_op_reduce(op,
+                               (char *) data->xpmem_raddr[l2_gp[i]] + chunk * l2_local_rank * dsize,
+                               (char *) data->xpmem_raddr[0] + chunk * l2_local_rank * dsize,
+                               my_count_size, dtype);
+            }
+            ompi_op_reduce(op, (char *) tmp_rbuf + chunk * l2_local_rank * dsize,
+                           (char *) data->xpmem_raddr[0] + chunk * l2_local_rank * dsize,
+                           my_count_size, dtype);
+        }
+    }
+
+    err = ompi_coll_base_barrier_intra_tree(comm, module);
+    if (!subc->xpmem_use_sr_buf) {
+        memcpy(rbuf, tmp_rbuf, total_dsize);
+    }
+    return err;
+}
+
+static inline int mca_coll_acoll_allreduce_xpmem_f(const void *sbuf, void *rbuf, size_t count,
+                                                   struct ompi_datatype_t *dtype,
+                                                   struct ompi_op_t *op,
+                                                   struct ompi_communicator_t *comm,
+                                                   mca_coll_base_module_t *module,
+                                                   coll_acoll_subcomms_t *subc)
+{
+    int size;
+    size_t total_dsize, dsize;
+
+    coll_acoll_init(module, comm, subc->data, subc, 0);
+    coll_acoll_data_t *data = subc->data;
+    if (NULL == data) {
+        return -1;
+    }
+
+    size = ompi_comm_size(comm);
+    ompi_datatype_type_size(dtype, &dsize);
+    total_dsize = dsize * count;
+
+    char *tmp_sbuf = NULL;
+    char *tmp_rbuf = NULL;
+    if (!subc->xpmem_use_sr_buf) {
+        tmp_rbuf = (char *) data->scratch;
+        tmp_sbuf = (char *) data->scratch + (subc->xpmem_buf_size) / 2;
+        if ((MPI_IN_PLACE == sbuf)) {
+            memcpy(tmp_sbuf, rbuf, total_dsize);
+        } else {
+            memcpy(tmp_sbuf, sbuf, total_dsize);
+        }
+    } else {
+        tmp_sbuf = (char *) sbuf;
+        tmp_rbuf = (char *) rbuf;
+        if (MPI_IN_PLACE == sbuf) {
+            tmp_sbuf = (char *) rbuf;
+        }
+    }
+    void *sbuf_vaddr[1] = {tmp_sbuf};
+    void *rbuf_vaddr[1] = {tmp_rbuf};
+    int err = MPI_SUCCESS;
+    int rank = ompi_comm_rank(comm);
+
+    err = comm->c_coll->coll_allgather(sbuf_vaddr, sizeof(void *), MPI_BYTE, data->allshm_sbuf,
+                                       sizeof(void *), MPI_BYTE, comm,
+                                       comm->c_coll->coll_allgather_module);
+    if (err != MPI_SUCCESS) {
+        return err;
+    }
+    err = comm->c_coll->coll_allgather(rbuf_vaddr, sizeof(void *), MPI_BYTE, data->allshm_rbuf,
+                                       sizeof(void *), MPI_BYTE, comm,
+                                       comm->c_coll->coll_allgather_module);
+
+    if (err != MPI_SUCCESS) {
+        return err;
+    }
+
+    register_and_cache(size, total_dsize, rank, data);
+
+    size_t chunk = count / size;
+    size_t my_count_size = (rank == (size - 1)) ? (count / size) + count % size : count / size;
+    if (0 == rank) {
+        if (sbuf != MPI_IN_PLACE)
+            memcpy(tmp_rbuf, sbuf, my_count_size * dsize);
+    } else {
+        ompi_3buff_op_reduce(op, (char *) data->xpmem_saddr[0] + chunk * rank * dsize,
+                             (char *) tmp_sbuf + chunk * rank * dsize,
+                             (char *) tmp_rbuf + chunk * rank * dsize, my_count_size, dtype);
+    }
+
+    err = ompi_coll_base_barrier_intra_tree(comm, module);
+    if (err != MPI_SUCCESS) {
+        return err;
+    }
+
+    for (int i = 1; i < size; i++) {
+        if (rank == i) {
+            continue;
+        }
+        ompi_op_reduce(op, (char *) data->xpmem_saddr[i] + chunk * rank * dsize,
+                       (char *) tmp_rbuf + chunk * rank * dsize, my_count_size, dtype);
+    }
+    err = ompi_coll_base_barrier_intra_tree(comm, module);
+    if (err != MPI_SUCCESS) {
+        return err;
+    }
+
+    size_t tmp = chunk * dsize;
+    for (int i = 0; i < size; i++) {
+        if (subc->xpmem_use_sr_buf && (rank == i)) {
+            continue;
+        }
+        my_count_size = (i == (size - 1)) ? (count / size) + count % size : count / size;
+        size_t tmp1 = i * tmp;
+        char *dst = (char *) rbuf + tmp1;
+        char *src = (char *) data->xpmem_raddr[i] + tmp1;
+        memcpy(dst, src, my_count_size * dsize);
+    }
+
+    err = ompi_coll_base_barrier_intra_tree(comm, module);
+
+    return err;
+}
+#endif
+
+void mca_coll_acoll_sync(coll_acoll_data_t *data, int offset, int *group, int gp_size, int rank,
+                         int up)
+{
+    volatile int *tmp, tmp0;
+    tmp = (int *) ((char *) data->allshmmmap_sbuf[group[0]] + offset
+                   + CACHE_LINE_SIZE * rank);
+    tmp0 = __atomic_load_n((int *) ((char *) data->allshmmmap_sbuf[group[0]] + offset
+                                    + CACHE_LINE_SIZE * group[0]),
+                           __ATOMIC_RELAXED);
+
+    opal_atomic_wmb();
+
+    int val;
+    if (1 == up) {
+        val = data->sync[0];
+    } else {
+        val = data->sync[1];
+    }
+
+    if (rank == group[0]) {
+        __atomic_store_n((int *) ((char *) data->allshmmmap_sbuf[group[0]] + offset
+                                  + CACHE_LINE_SIZE * group[0]),
+                         val, __ATOMIC_RELAXED);
+    }
+
+    while (tmp0 != val) {
+        tmp0 = __atomic_load_n((int *) ((char *) data->allshmmmap_sbuf[group[0]] + offset
+                                        + CACHE_LINE_SIZE * group[0]),
+                               __ATOMIC_RELAXED);
+    }
+
+    if (rank != group[0]) {
+        val++;
+        __atomic_store_n(tmp, val, __ATOMIC_RELAXED);
+    }
+    opal_atomic_wmb();
+    if (rank == group[0]) {
+        for (int i = 1; i < gp_size; i++) {
+            volatile int tmp1 = __atomic_load_n(
+                (int *) ((char *) data->allshmmmap_sbuf[group[0]] + offset + CACHE_LINE_SIZE * group[i]),
+                __ATOMIC_RELAXED);
+            while (tmp1 == val) {
+                tmp1 = __atomic_load_n((int *) ((char *) data->allshmmmap_sbuf[group[0]] + offset
+                                                + CACHE_LINE_SIZE * group[i]),
+                                       __ATOMIC_RELAXED);
+            }
+            opal_atomic_wmb();
+        }
+        ++val;
+        __atomic_store_n(tmp, val, __ATOMIC_RELAXED);
+    } else {
+        volatile int tmp1 = __atomic_load_n(
+            (int *) ((char *) data->allshmmmap_sbuf[group[0]] + offset + CACHE_LINE_SIZE * group[0]),
+            __ATOMIC_RELAXED);
+        while (tmp1 != val) {
+            tmp1 = __atomic_load_n((int *) ((char *) data->allshmmmap_sbuf[group[0]] + offset
+                                            + CACHE_LINE_SIZE * group[0]),
+                                   __ATOMIC_RELAXED);
+        }
+    }
+    if (1 == up) {
+        data->sync[0] = val;
+    } else {
+        data->sync[1] = val;
+    }
+}
+
+int mca_coll_acoll_allreduce_small_msgs_h(const void *sbuf, void *rbuf, size_t count,
+                                          struct ompi_datatype_t *dtype, struct ompi_op_t *op,
+                                          struct ompi_communicator_t *comm,
+                                          mca_coll_base_module_t *module,
+                                          coll_acoll_subcomms_t *subc, int intra)
+{
+    size_t dsize;
+    int err = MPI_SUCCESS;
+    coll_acoll_init(module, comm, subc->data, subc, 0);
+    coll_acoll_data_t *data = subc->data;
+    if (NULL == data) {
+        return -1;
+    }
+
+    int rank = ompi_comm_rank(comm);
+    ompi_datatype_type_size(dtype, &dsize);
+
+    int l1_gp_size = data->l1_gp_size;
+    int *l1_gp = data->l1_gp;
+    int *l2_gp = data->l2_gp;
+    int l2_gp_size = data->l2_gp_size;
+
+    int l1_local_rank = data->l1_local_rank;
+    int l2_local_rank = data->l2_local_rank;
+
+    int offset1 = data->offset[0];
+    int offset2 = data->offset[1];
+    int tshm_offset = data->offset[2];
+    int shm_offset = data->offset[3];
+    const int per_rank_shm_size = 8 * 1024;
+
+    int local_size;
+
+    if (rank == l1_gp[0]) {
+        if (l2_gp_size > 1) {
+            mca_coll_acoll_sync(data, offset2, l2_gp, l2_gp_size, rank, 3);
+        }
+    }
+
+    if (MPI_IN_PLACE == sbuf) {
+        memcpy((char *) data->allshmmmap_sbuf[l1_gp[0]] + shm_offset, rbuf, count * dsize);
+    } else {
+        memcpy((char *) data->allshmmmap_sbuf[l1_gp[0]] + shm_offset, sbuf, count * dsize);
+    }
+
+    mca_coll_acoll_sync(data, offset1, l1_gp, l1_gp_size, rank, 1);
+
+    if (rank == l1_gp[0]) {
+        memcpy((char *) data->allshmmmap_sbuf[l1_gp[l1_local_rank]],
+               (char *) data->allshmmmap_sbuf[l1_gp[0]] + shm_offset, count * dsize);
+        for (int i = 1; i < l1_gp_size; i++) {
+            ompi_op_reduce(op,
+                           (char *) data->allshmmmap_sbuf[l1_gp[0]] + tshm_offset
+                               + l1_gp[i] * per_rank_shm_size,
+                           (char *) data->allshmmmap_sbuf[l1_gp[l1_local_rank]], count, dtype);
+        }
+        memcpy(rbuf, data->allshmmmap_sbuf[l1_gp[l1_local_rank]], count * dsize);
+    }
+
+    if (rank == l1_gp[0]) {
+        if (l2_gp_size > 1) {
+            mca_coll_acoll_sync(data, offset2, l2_gp, l2_gp_size, rank, 3);
+        }
+    }
+
+    /* perform allreduce across leaders */
+    local_size = l2_gp_size;
+    if (local_size > 1) {
+        if (rank == l1_gp[0]) {
+            for (int i = 0; i < local_size; i++) {
+                if (i == l2_local_rank) {
+                    continue;
+                }
+                ompi_op_reduce(op, (char *) data->allshmmmap_sbuf[l2_gp[i]], (char *) rbuf, count,
+                               dtype);
+            }
+        }
+    }
+
+    if (intra && (ompi_comm_size(subc->numa_comm) > 1)) {
+        err = ompi_coll_base_bcast_intra_basic_linear(rbuf, count, dtype, 0, subc->numa_comm, module);
+    }
+    return err;
+}
+
+int mca_coll_acoll_allreduce_intra(const void *sbuf, void *rbuf, size_t count,
+                                   struct ompi_datatype_t *dtype, struct ompi_op_t *op,
+                                   struct ompi_communicator_t *comm, mca_coll_base_module_t *module)
+{
+    int size, alg, err;
+    int num_nodes;
+    size_t total_dsize, dsize;
+    mca_coll_acoll_module_t *acoll_module = (mca_coll_acoll_module_t *) module;
+    size = ompi_comm_size(comm);
+    ompi_datatype_type_size(dtype, &dsize);
+    total_dsize = dsize * count;
+
+    if (1 == size) {
+        if (MPI_IN_PLACE != sbuf) {
+            memcpy((char *) rbuf, sbuf, total_dsize);
+        }
+        return MPI_SUCCESS;
+    }
+
+    /* Falling back to recursivedoubling for non-commutative operators to be safe */
+    if (!ompi_op_is_commute(op)) {
+        return ompi_coll_base_allreduce_intra_recursivedoubling(sbuf, rbuf, count, dtype, op, comm,
+                                                                module);
+    }
+
+    /* Obtain the subcomms structure */
+    coll_acoll_subcomms_t *subc = NULL;
+    err = check_and_create_subc(comm, acoll_module, &subc);
+
+    /* Fallback to knomial if subc is not obtained */
+    if (NULL == subc) {
+        return ompi_coll_base_allreduce_intra_redscat_allgather(sbuf, rbuf, count, dtype, op, comm,
+                                                                module);
+    }
+    if (!subc->initialized) {
+        err = mca_coll_acoll_comm_split_init(comm, acoll_module, subc, 0);
+        if (MPI_SUCCESS != err)
+            return err;
+    }
+
+    num_nodes = subc->num_nodes;
+
+    alg = coll_allreduce_decision_fixed(size, total_dsize);
+
+    if (1 == num_nodes) {
+        if (total_dsize < 32) {
+            return ompi_coll_base_allreduce_intra_recursivedoubling(sbuf, rbuf, count, dtype, op,
+                                                                    comm, module);
+        } else if (total_dsize < 512) {
+            return mca_coll_acoll_allreduce_small_msgs_h(sbuf, rbuf, count, dtype, op, comm, module,
+                                                         subc, 1);
+        } else if (total_dsize <= 2048) {
+            return ompi_coll_base_allreduce_intra_recursivedoubling(sbuf, rbuf, count, dtype, op,
+                                                                    comm, module);
+        } else if (total_dsize < 65536) {
+            if (1 == alg) {
+                return ompi_coll_base_allreduce_intra_recursivedoubling(sbuf, rbuf, count, dtype,
+                                                                        op, comm, module);
+            } else if (2 == alg) {
+                return ompi_coll_base_allreduce_intra_redscat_allgather(sbuf, rbuf, count, dtype,
+                                                                        op, comm, module);
+            } else { /*alg == 3 */
+                return ompi_coll_base_allreduce_intra_ring_segmented(sbuf, rbuf, count, dtype, op,
+                                                                     comm, module, 0);
+            }
+        } else if (total_dsize < 4194304) {
+#ifdef HAVE_XPMEM_H
+            if (((subc->xpmem_use_sr_buf != 0) || (subc->xpmem_buf_size > 2 * total_dsize)) && (subc->without_xpmem != 1)) {
+                return mca_coll_acoll_allreduce_xpmem_f(sbuf, rbuf, count, dtype, op, comm, module, subc);
+            } else {
+                return ompi_coll_base_allreduce_intra_redscat_allgather(sbuf, rbuf, count, dtype,
+                                                                        op, comm, module);
+            }
+#else
+            return ompi_coll_base_allreduce_intra_redscat_allgather(sbuf, rbuf, count, dtype, op,
+                                                                    comm, module);
+#endif
+        } else if (total_dsize <= 16777216) {
+#ifdef HAVE_XPMEM_H
+            if (((subc->xpmem_use_sr_buf != 0) || (subc->xpmem_buf_size > 2 * total_dsize)) && (subc->without_xpmem != 1)) {
+                mca_coll_acoll_reduce_xpmem_h(sbuf, rbuf, count, dtype, op, comm, module, subc);
+                return mca_coll_acoll_bcast(rbuf, count, dtype, 0, comm, module);
+            } else {
+                return ompi_coll_base_allreduce_intra_redscat_allgather(sbuf, rbuf, count, dtype,
+                                                                        op, comm, module);
+            }
+#else
+            return ompi_coll_base_allreduce_intra_redscat_allgather(sbuf, rbuf, count, dtype, op,
+                                                                    comm, module);
+#endif
+        } else {
+#ifdef HAVE_XPMEM_H
+            if (((subc->xpmem_use_sr_buf != 0) || (subc->xpmem_buf_size > 2 * total_dsize)) && (subc->without_xpmem != 1)) {
+                return mca_coll_acoll_allreduce_xpmem_f(sbuf, rbuf, count, dtype, op, comm, module, subc);
+            } else {
+                return ompi_coll_base_allreduce_intra_redscat_allgather(sbuf, rbuf, count, dtype,
+                                                                        op, comm, module);
+            }
+#else
+            return ompi_coll_base_allreduce_intra_redscat_allgather(sbuf, rbuf, count, dtype, op,
+                                                                    comm, module);
+#endif
+        }
+
+    } else {
+        return ompi_coll_base_allreduce_intra_redscat_allgather(sbuf, rbuf, count, dtype, op, comm,
+                                                                module);
+    }
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mca/coll/acoll/coll_acoll_alltoall.c b/ompi/mca/coll/acoll/coll_acoll_alltoall.c
new file mode 100644
index 00000000000..ab58a38d503
--- /dev/null
+++ b/ompi/mca/coll/acoll/coll_acoll_alltoall.c
@@ -0,0 +1,568 @@
+/*
+ * Copyright (c) 2024 - 2025 Advanced Micro Devices, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "mpi.h"
+#include "ompi/constants.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/coll/base/coll_tags.h"
+#include "ompi/mca/coll/base/coll_base_functions.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "coll_acoll.h"
+#include "ompi/mca/pml/pml.h"
+#include "opal/util/bit_ops.h"
+#include "coll_acoll_utils.h"
+
+static void mca_coll_acoll_get_split_factor_and_base_algo
+                            (size_t scount, struct ompi_datatype_t *sdtype,
+                             size_t rcount, struct ompi_datatype_t *rdtype,
+                             bool is_inplace,
+                             struct ompi_communicator_t *comm,
+                             bool* sync_enable,
+                             int* split_factor)
+{
+    (*sync_enable) = false;
+    (*split_factor) = 2;
+
+    size_t dsize = 0;
+    size_t total_dsize = 0;
+
+    int comm_size = ompi_comm_size(comm);
+
+    if (false == is_inplace) {
+        ompi_datatype_type_size(sdtype, &dsize);
+        total_dsize = dsize * (ptrdiff_t)scount;
+    } else {
+        ompi_datatype_type_size(rdtype, &dsize);
+        total_dsize = dsize * (ptrdiff_t)rcount;
+    }
+
+    if (comm_size <= 8) {
+        if (total_dsize <= 128) {
+            (*sync_enable) = true;
+        } else {
+            (*sync_enable) = false;
+        }
+        (*split_factor) = 2;
+    } else if (comm_size <= 16) {
+        if (total_dsize <= 192) {
+            (*sync_enable) = true;
+            (*split_factor) = 4;
+        } else if (total_dsize <= 512) {
+            (*sync_enable) = false;
+            (*split_factor) = 4;
+        } else if (total_dsize <= 4096) {
+            (*sync_enable) = false;
+            (*split_factor) = 2;
+        } else {
+            (*sync_enable) = true;
+            (*split_factor) = 2;
+        }
+    } else if (comm_size <= 24) {
+        if (total_dsize <= 64) {
+            (*sync_enable) = true;
+            (*split_factor) = 4;
+        } else if (total_dsize <= 1024) {
+            (*sync_enable) = false;
+            (*split_factor) = 4;
+        } else {
+            (*sync_enable) = false;
+            (*split_factor) = 2;
+        }
+    } else if (comm_size <= 32) {
+        if (total_dsize <= 64) {
+            (*sync_enable) = true;
+            (*split_factor) = 4;
+        } else if (total_dsize <= 1024) {
+            (*sync_enable) = false;
+            (*split_factor) = 4;
+        } else if (total_dsize <= 4096) {
+            (*sync_enable) = false;
+            (*split_factor) = 2;
+        } else {
+            (*sync_enable) = true;
+            (*split_factor) = 2;
+        }
+    } else if (comm_size <= 48) {
+        if (total_dsize <= 64) {
+            (*sync_enable) = true;
+            (*split_factor) = 4;
+        } else if (total_dsize <= 1024) {
+            (*sync_enable) = false;
+            (*split_factor) = 4;
+        } else {
+            (*sync_enable) = false;
+            (*split_factor) = 2;
+        }
+    } else if (comm_size <= 64) {
+        if (total_dsize <= 64) {
+            (*sync_enable) = true;
+            (*split_factor) = 4;
+        } else if (total_dsize <= 1024) {
+            (*sync_enable) = false;
+            (*split_factor) = 4;
+        } else {
+            (*sync_enable) = false;
+            (*split_factor) = 2;
+        }
+    } else if (comm_size <= 72) {
+        if (total_dsize <= 64) {
+            (*sync_enable) = true;
+            (*split_factor) = 4;
+        } else if (total_dsize <= 1024) {
+            (*sync_enable) = false;
+            (*split_factor) = 4;
+        } else {
+            (*sync_enable) = false;
+            (*split_factor) = 2;
+        }
+    } else if (comm_size <= 96) {
+        if (total_dsize <= 64) {
+            (*sync_enable) = true;
+            (*split_factor) = 4;
+        } else if (total_dsize <= 1024) {
+            (*sync_enable) = false;
+            (*split_factor) = 4;
+        } else {
+            (*sync_enable) = false;
+            (*split_factor) = 2;
+        }
+    } else if (comm_size <= 128) {
+        if (total_dsize <= 64) {
+            (*sync_enable) = true;
+            (*split_factor) = 8;
+        } else if (total_dsize <= 512) {
+            (*sync_enable) = false;
+            (*split_factor) = 8;
+        } else {
+            (*sync_enable) = false;
+            (*split_factor) = 2;
+        }
+    } else {
+        if (total_dsize <= 32) {
+            (*sync_enable) = true;
+            (*split_factor) = 8;
+        } else if (total_dsize <= 2048) {
+            (*sync_enable) = false;
+            (*split_factor) = 8;
+        } else if (total_dsize <= 8192) {
+            (*sync_enable) = false;
+            (*split_factor) = 2;
+        } else {
+            (*sync_enable) = true;
+            (*split_factor) = 2;
+        }
+    }
+
+    /* Non-multiple size of comm only supported for split factor where
+     * comm_size % split factor = 1. Split factor should always be a
+     * power of 2, else undefined behavior. */
+    while ((2 < (*split_factor)) &&
+        (1 < (comm_size % (*split_factor)))) {
+        (*split_factor) = (*split_factor) / 2;
+    }
+}
+
+static inline size_t mca_coll_acoll_get_msg_thresh(coll_acoll_subcomms_t *subc,
+                mca_coll_acoll_module_t *acoll_module)
+{
+    size_t msg_thres[DIST_END] = {4096, 2048, 1024, 1024, 512};
+    size_t dsize_thresh = msg_thres[subc->r2r_dist];
+
+    /* Override if associated mca param is set. */
+    if (0 < (acoll_module->alltoall_attr).psplit_msg_thresh) {
+        dsize_thresh = (acoll_module->alltoall_attr).psplit_msg_thresh;
+    }
+
+    return dsize_thresh;
+}
+
+static int mca_coll_acoll_last_rank_scatter_gather
+                        (const void *sbuf, size_t scount,
+                        struct ompi_datatype_t *sdtype,
+                        void* rbuf, size_t rcount,
+                        struct ompi_datatype_t *rdtype,
+                        char* work_buf,
+                        struct ompi_communicator_t *comm,
+                        mca_coll_acoll_module_t *acoll_module)
+{
+    int error;
+    int rank = ompi_comm_rank(comm);
+    int size = ompi_comm_size(comm);
+    MPI_Aint sbext, sblb;
+    MPI_Aint rext, rlb;
+    error = ompi_datatype_get_extent (sdtype, &sblb, &sbext);
+    if (MPI_SUCCESS != error) {
+        return error;
+    }
+    error = ompi_datatype_get_extent (rdtype, &rlb, &rext);
+    if (MPI_SUCCESS != error) {
+        return error;
+    }
+
+    MPI_Status status;
+    int subgrp_size = acoll_module->sg_cnt;
+
+    /* Scatter/Gather fused code. */
+    /* Last rank does a scatter and gather to the sub group leaders. */
+    if ((size - 1) == rank) {
+        /* The last block of data belongs to this last rank, and copying it
+         * to rbuf from sbuf suffices.*/
+        error = ompi_datatype_sndrcv(
+                    (char*)sbuf + ((size - 1) * scount * sbext),
+                    scount, sdtype,
+                    (char*)rbuf + ((size - 1) * rcount * rext),
+                    rcount, rdtype);
+        if (MPI_SUCCESS != error) { goto error_handler; }
+
+        /* Scatterring data to the sub group leaders, with sub group size worth
+         * of data.*/
+        for (int cur_rank = 0; cur_rank < (size - 1); cur_rank += subgrp_size) {
+            int sg_scount = ((cur_rank + subgrp_size) >= size) ?
+                                 ((size - (cur_rank + 1)) * scount) :
+                                 (scount * subgrp_size);
+            error = MCA_PML_CALL(send((char*)sbuf + (cur_rank * scount * sbext),
+                                      sg_scount, sdtype, cur_rank,
+                                      MCA_COLL_BASE_TAG_ALLTOALL,
+                                      MCA_PML_BASE_SEND_STANDARD, comm));
+            if (MPI_SUCCESS != error) { goto error_handler; }
+        }
+
+        /* Gathering data from the sub group leaders, with sub group size worth
+         * of data. */
+        for (int cur_rank = 0; cur_rank < (size - 1); cur_rank += subgrp_size) {
+            int sg_rcount = ((cur_rank + subgrp_size) >= size) ?
+                                 ((size - (cur_rank + 1)) * rcount) :
+                                 (rcount * subgrp_size);
+            error = MCA_PML_CALL(recv((char*)rbuf + (cur_rank * rcount * rext),
+                                      sg_rcount, rdtype, cur_rank,
+                                      MCA_COLL_BASE_TAG_ALLTOALL, comm, &status));
+            if (MPI_SUCCESS != error) { goto error_handler; }
+        }
+    } else {
+        /* The 0th rank within a sub group is considered as sub group leader. */
+        if (0 == (rank % subgrp_size)) {
+            /* Receive sub group specific data from last rank. */
+            int sg_rcount = ((rank + subgrp_size) >= size) ?
+                                 ((size - (rank + 1)) * rcount) :
+                                 (rcount * subgrp_size);
+            error = MCA_PML_CALL(recv(work_buf,
+                                      sg_rcount, rdtype, size - 1,
+                                      MCA_COLL_BASE_TAG_ALLTOALL, comm, &status));
+            if (MPI_SUCCESS != error) { goto error_handler; }
+
+            int end_rank = ((rank + subgrp_size) >= size) ?
+                                  (size - 1) : ( rank + subgrp_size);
+
+            /* The data received from last rank is distributed in the sub group. */
+            error = ompi_datatype_copy_content_same_ddt(rdtype, rcount,
+                                       (char*)rbuf + ((size - 1) * rcount * rext),
+                                       (char*)work_buf);
+            if (MPI_SUCCESS != error) { goto error_handler; }
+
+            for (int cur_rank = rank + 1; cur_rank < end_rank; ++cur_rank) {
+                error = MCA_PML_CALL(send(((char*)work_buf +
+                                ((cur_rank % subgrp_size) * rcount * rext)),
+                                rcount, rdtype, cur_rank,
+                                MCA_COLL_BASE_TAG_ALLTOALL,
+                                MCA_PML_BASE_SEND_STANDARD, comm));
+                if (MPI_SUCCESS != error) { goto error_handler; }
+            }
+
+            /* The sub group leader gathers the data for the last rank from the
+             * sub group and then sends it to the last rank. */
+            error = ompi_datatype_sndrcv(
+                        (char*)sbuf + ((size - 1) * scount * sbext),
+                        scount, sdtype,
+                        (char*)work_buf,
+                        rcount, rdtype);
+            if (MPI_SUCCESS != error) { goto error_handler; }
+
+            for (int cur_rank = rank + 1; cur_rank < end_rank; ++cur_rank) {
+                error = MCA_PML_CALL(recv(((char*)work_buf +
+                                ((cur_rank % subgrp_size) * rcount * rext)),
+                                rcount, rdtype, cur_rank,
+                                MCA_COLL_BASE_TAG_ALLTOALL, comm, &status));
+                if (MPI_SUCCESS != error) { goto error_handler; }
+            }
+
+            int sg_rscount = ((rank + subgrp_size) >= size) ?
+                                 ((size - (rank + 1)) * rcount) :
+                                 (rcount * subgrp_size);
+            error = MCA_PML_CALL(send(work_buf,
+                                      sg_rscount, rdtype, size - 1,
+                                      MCA_COLL_BASE_TAG_ALLTOALL,
+                                      MCA_PML_BASE_SEND_STANDARD, comm));
+            if (MPI_SUCCESS != error) { goto error_handler; }
+        } else {
+            /* The leaf ranks send/receive the data for/from the last rank
+             * to/from the sub group leader. */
+            int sg_ldr_rank = ((rank / subgrp_size) * subgrp_size);
+
+            error = MCA_PML_CALL(recv((char*)rbuf + ((size - 1) * rcount * rext),
+                                      rcount, rdtype, sg_ldr_rank,
+                                      MCA_COLL_BASE_TAG_ALLTOALL, comm, &status));
+            if (MPI_SUCCESS != error) { goto error_handler; }
+
+            error = MCA_PML_CALL(send((char*)sbuf + ((size - 1) * scount * sbext),
+                                      scount, sdtype, sg_ldr_rank,
+                                      MCA_COLL_BASE_TAG_ALLTOALL,
+                                      MCA_PML_BASE_SEND_STANDARD, comm));
+            if (MPI_SUCCESS != error) { goto error_handler; }
+        }
+    }
+
+error_handler:
+
+    return error;
+}
+
+static inline int mca_coll_acoll_base_alltoall_dispatcher
+                        (const void *sbuf, size_t scount,
+                        struct ompi_datatype_t *sdtype,
+                        void* rbuf, size_t rcount,
+                        struct ompi_datatype_t *rdtype,
+                        struct ompi_communicator_t *comm,
+                        mca_coll_acoll_module_t *acoll_module,
+                        bool sync_enable)
+{
+    int error;
+
+    if (sync_enable) {
+        error = ompi_coll_base_alltoall_intra_linear_sync
+                        ((char*)sbuf, scount, sdtype,
+                         (char*)rbuf, rcount, rdtype,
+                         comm, &acoll_module->super, 0);
+    } else {
+        error = ompi_coll_base_alltoall_intra_basic_linear
+                        ((char*)sbuf, scount, sdtype,
+                         (char*)rbuf, rcount, rdtype,
+                         comm, &acoll_module->super);
+    }
+    return error;
+}
+
+static inline int mca_coll_acoll_exchange_data
+                        (const void *sbuf, size_t scount,
+                        struct ompi_datatype_t *sdtype,
+                        void* rbuf, size_t rcount,
+                        struct ompi_datatype_t *rdtype,
+                        char* work_buf,
+                        struct ompi_communicator_t *comm,
+                        mca_coll_acoll_module_t *acoll_module,
+                        int grp_split_f)
+{
+    /* sbuf is not used, but added to maintain uniform arguments. */
+    (void) sbuf;
+    (void) scount;
+    (void) sdtype;
+
+    int error;
+    int rank = ompi_comm_rank(comm);
+    int size = ompi_comm_size(comm);
+    MPI_Aint rext, rlb;
+    error = ompi_datatype_get_extent (rdtype, &rlb, &rext);
+    if (MPI_SUCCESS != error) {
+        return error;
+    }
+
+    /* Exchange data among groups with split factor (2 or 4 or 8) number of
+     * ranks. */
+    int ps_grp_size = grp_split_f;
+    int ps_grp_start_rank = (rank / ps_grp_size) * ps_grp_size;
+    int ps_grp_num_ranks = size / ps_grp_size;
+    size_t ps_grp_rcount = ps_grp_num_ranks * rcount;
+    size_t ps_grp_rcount_ext = ps_grp_rcount * rext;
+    size_t ps_grp_buf_copy_stride = ps_grp_size * rcount * rext;
+
+    /* Create a new datatype that iterates over the send buffer in strides
+     * of ps_grp_size * rcount. */
+    struct ompi_datatype_t *new_ddt;
+    ompi_datatype_create_vector(ps_grp_num_ranks, rcount,
+                    (rcount * ps_grp_size),
+                    rdtype, &new_ddt);
+    error = ompi_datatype_commit(&new_ddt);
+    if (MPI_SUCCESS != error) { goto error_handler; }
+
+    for (int iter = 1; iter < ps_grp_size; ++iter) {
+        int next_rank = ps_grp_start_rank + ((rank + iter) % ps_grp_size);
+        int prev_rank = ps_grp_start_rank +
+                        ((rank + ps_grp_size - iter) % ps_grp_size);
+        int read_pos = ((rank + iter) % ps_grp_size);
+
+        error = ompi_coll_base_sendrecv
+                    ((char*)rbuf + ((ptrdiff_t)read_pos * rcount * rext),
+                     1, new_ddt, next_rank,
+                     MCA_COLL_BASE_TAG_ALLTOALL,
+                     (char*)work_buf + ((iter - 1) * ps_grp_rcount_ext),
+                     ps_grp_rcount, rdtype, prev_rank,
+                     MCA_COLL_BASE_TAG_ALLTOALL,
+                     comm, MPI_STATUS_IGNORE, rank);
+        if (MPI_SUCCESS != error) { goto error_handler; }
+    }
+
+    error = ompi_datatype_destroy(&new_ddt);
+    if (MPI_SUCCESS != error) { goto error_handler; }
+
+    /* Copy received data to the correct blocks. */
+    for (int iter = 1; iter < ps_grp_size; ++iter) {
+        int write_pos = ((rank + ps_grp_size - iter) % ps_grp_size);
+        char* dst = (char*)rbuf + (write_pos * rcount * rext);
+        char* src = (char*)work_buf + ((iter - 1) * ps_grp_rcount_ext);
+
+        for (int i = 0; i < ps_grp_num_ranks; ++i) {
+            error = ompi_datatype_copy_content_same_ddt(rdtype, rcount,
+                                                        dst, src);
+            if (MPI_SUCCESS != error) { goto error_handler; }
+
+            dst = dst + ps_grp_buf_copy_stride;
+            src = src + (1 * rcount * rext);
+        }
+    }
+
+error_handler:
+
+    return error;
+}
+
+/* Parallel Split AllToAll algorithm in a nutshell:
+ * 1. Divide the ranks into split factor number of parallel groups.
+ *      -Rank r is part of parallel group i if r % split_factor == i.
+ * 2. Perform all_to_all among the split groups in parallel.
+ * 3. Divide the ranks into exchange groups, where each group contains
+      split factor number of consecutive ranks.
+        -Rank r is part of exchange group i if r / split_factor == i.
+ * 4. Exchange data among the ranks in each exchange group to complete
+ *    all_to_all. */
+int mca_coll_acoll_alltoall
+                        (const void *sbuf, size_t scount,
+                        struct ompi_datatype_t *sdtype,
+                        void* rbuf, size_t rcount,
+                        struct ompi_datatype_t *rdtype,
+                        struct ompi_communicator_t *comm,
+                        mca_coll_base_module_t *module)
+{
+    int rank = ompi_comm_rank(comm);
+    int size = ompi_comm_size(comm);
+    int error = MPI_SUCCESS;
+
+    MPI_Aint rext, rlb;
+    error = ompi_datatype_get_extent (rdtype, &rlb, &rext);
+    if (MPI_SUCCESS != error) { return error; }
+
+    mca_coll_acoll_module_t *acoll_module = (mca_coll_acoll_module_t *)module;
+    coll_acoll_subcomms_t *subc = NULL;
+
+    /* Obtain the subcomms structure */
+    error = check_and_create_subc(comm, acoll_module, &subc);
+    /* Fallback to knomial if subcomms is not obtained */
+    if ((NULL == subc) || (size < 4)) {
+        return mca_coll_acoll_base_alltoall_dispatcher
+                        (sbuf, scount, sdtype,
+                         rbuf, rcount, rdtype,
+                         comm, acoll_module, false);
+    }
+
+    coll_acoll_reserve_mem_t* reserve_mem_gather = &(acoll_module->reserve_mem_s);
+
+    if (!subc->initialized && (size > 2)) {
+        error = mca_coll_acoll_comm_split_init(comm, acoll_module, subc, 0);
+        if (MPI_SUCCESS != error) { return error; }
+    }
+
+    size_t dsize = 0;
+    ompi_datatype_type_size(rdtype, &dsize);
+
+    /* Derive upper bound on message size where this algorithm is applicable. */
+    size_t dsize_thresh = mca_coll_acoll_get_msg_thresh(subc, acoll_module);
+
+    if (dsize_thresh < (rcount * rext)) {
+        return mca_coll_acoll_base_alltoall_dispatcher
+                        (sbuf, scount, sdtype,
+                         rbuf, rcount, rdtype,
+                         comm, acoll_module, false);
+    }
+
+    bool sync_enable = false;
+    int grp_split_f = 2;
+    if ((acoll_module->alltoall_attr).split_factor > 0 ) {
+        grp_split_f = (acoll_module->alltoall_attr).split_factor;
+
+        /* Non-multiple size of comm only supported for split factor where
+         * comm_size % split factor = 1. Split factor should always be a
+         * power of 2, else undefined behavior. */
+        while ((2 < grp_split_f) &&
+               (1 < (size % grp_split_f))) {
+            grp_split_f = grp_split_f / 2;
+        }
+    } else {
+        mca_coll_acoll_get_split_factor_and_base_algo
+                    (scount, sdtype, rcount, rdtype,
+                     (MPI_IN_PLACE == sbuf), comm,
+                     &sync_enable, &grp_split_f);
+    }
+
+    char* work_buf_free = NULL;
+    char* work_buf = NULL;
+    MPI_Aint rgap = 0, ssize;
+
+    ssize = opal_datatype_span(&rdtype->super, size * rcount, &rgap);
+    work_buf_free = (char*)coll_acoll_buf_alloc(reserve_mem_gather, ssize);
+    if (NULL == work_buf_free) {
+        error = OMPI_ERR_OUT_OF_RESOURCE;
+        goto error_handler;
+    }
+    work_buf = work_buf_free - rgap;
+
+     /* In case size is odd, the data to and from the last rank is handled as
+      * a separate case. */
+    if ((0 == (size % 2)) || (rank != (size - 1))) {
+        /* Perform all_to_all among the parallel-split groups. */
+        struct ompi_communicator_t *split_comm;
+
+        /* Select the right split_comm. */
+        int pow2_idx = -2;
+        int tmp_grp_split_f = grp_split_f;
+        while (tmp_grp_split_f > 0)
+        {
+            pow2_idx += 1;
+            tmp_grp_split_f = tmp_grp_split_f / 2;
+        }
+        split_comm = subc->split_comm[pow2_idx];
+
+        error = mca_coll_acoll_base_alltoall_dispatcher
+                        (sbuf, (grp_split_f * scount), sdtype,
+                         rbuf, (grp_split_f * rcount), rdtype,
+                         split_comm, acoll_module, sync_enable);
+        if (MPI_SUCCESS != error) { goto error_handler; }
+
+        /* Exchange data among consecutive blocks of split factor ranks. */
+        error = mca_coll_acoll_exchange_data
+                        (sbuf, scount, sdtype,
+                         rbuf, rcount, rdtype,
+                         work_buf, comm, acoll_module, grp_split_f);
+        if (MPI_SUCCESS != error) { goto error_handler; }
+    }
+
+    /* Data transfer for the last rank. */
+    if (0 != (size % 2)) {
+        error = mca_coll_acoll_last_rank_scatter_gather
+                        (sbuf, scount, sdtype,
+                         rbuf, rcount, rdtype,
+                         work_buf, comm, acoll_module);
+        if (MPI_SUCCESS != error) { goto error_handler; }
+    }
+
+error_handler:
+    coll_acoll_buf_free(reserve_mem_gather, work_buf_free);
+
+    return error;
+}
diff --git a/ompi/mca/coll/acoll/coll_acoll_barrier.c b/ompi/mca/coll/acoll/coll_acoll_barrier.c
new file mode 100644
index 00000000000..d57db48b91f
--- /dev/null
+++ b/ompi/mca/coll/acoll/coll_acoll_barrier.c
@@ -0,0 +1,224 @@
+/* -*- Mode: C; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+
+#include "mpi.h"
+#include "ompi/constants.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_functions.h"
+#include "ompi/mca/coll/base/coll_tags.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/pml/pml.h"
+#include "opal/util/bit_ops.h"
+#include "coll_acoll.h"
+#include "coll_acoll_utils.h"
+
+static int mca_coll_acoll_barrier_recv_subc(struct ompi_communicator_t *comm,
+                                            mca_coll_base_module_t *module, ompi_request_t **reqs,
+                                            int *nreqs, int root)
+{
+    int rank = ompi_comm_rank(comm);
+    int size = ompi_comm_size(comm);
+    int err = MPI_SUCCESS;
+
+    if (rank < 0) {
+        return err;
+    }
+
+    /* Non-zero ranks receive zero-byte message from rank 0 */
+    if (rank != root) {
+        err = MCA_PML_CALL(
+            recv(NULL, 0, MPI_BYTE, root, MCA_COLL_BASE_TAG_BARRIER, comm, MPI_STATUS_IGNORE));
+        if (MPI_SUCCESS != err) {
+            return err;
+        }
+    } else if (rank == root) {
+        ompi_request_t **preq = reqs;
+        *nreqs = 0;
+        for (int i = 0; i < size; i++) {
+            if (i == root) {
+                continue;
+            }
+            *nreqs = *nreqs + 1;
+            err = MCA_PML_CALL(isend(NULL, 0, MPI_BYTE, i, MCA_COLL_BASE_TAG_BARRIER,
+                                     MCA_PML_BASE_SEND_STANDARD, comm, preq++));
+            if (MPI_SUCCESS != err) {
+                return err;
+            }
+        }
+        err = ompi_request_wait_all(*nreqs, reqs, MPI_STATUSES_IGNORE);
+        if (MPI_SUCCESS != err) {
+            return err;
+        }
+    }
+
+    return err;
+}
+
+static int mca_coll_acoll_barrier_send_subc(struct ompi_communicator_t *comm,
+                                            mca_coll_base_module_t *module, ompi_request_t **reqs,
+                                            int *nreqs, int root)
+{
+    int rank = ompi_comm_rank(comm);
+    int size = ompi_comm_size(comm);
+    int err = MPI_SUCCESS;
+
+    if (rank < 0) {
+        return err;
+    }
+
+    /* Non-zero ranks send zero-byte message to rank 0 */
+    if (rank != root) {
+        err = MCA_PML_CALL(send(NULL, 0, MPI_BYTE, root, MCA_COLL_BASE_TAG_BARRIER,
+                                MCA_PML_BASE_SEND_STANDARD, comm));
+        if (MPI_SUCCESS != err) {
+            return err;
+        }
+    } else if (rank == root) {
+        ompi_request_t **preq = reqs;
+        *nreqs = 0;
+        for (int i = 0; i < size; i++) {
+            if (i == root) {
+                continue;
+            }
+            *nreqs = *nreqs + 1;
+            err = MCA_PML_CALL(
+                irecv(NULL, 0, MPI_BYTE, i, MCA_COLL_BASE_TAG_BARRIER, comm, preq++));
+            if (MPI_SUCCESS != err) {
+                return err;
+            }
+        }
+        err = ompi_request_wait_all(*nreqs, reqs, MPI_STATUSES_IGNORE);
+        if (MPI_SUCCESS != err) {
+            return err;
+        }
+    }
+
+    return err;
+}
+
+/*
+ * mca_coll_acoll_barrier_intra
+ *
+ * Function:    Barrier operation using subgroup based algorithm
+ * Accepts:     Same arguments as MPI_Barrier()
+ * Returns:     MPI_SUCCESS or error code
+ *
+ * Description: Step 1 - All leaf ranks of a subgroup send to base rank.
+ *              Step 2 - All base ranks send to rank 0.
+ *              Step 3 - Base rank sends to leaf ranks.
+ *
+ * Limitations: None
+ *
+ * Memory:      No additional memory requirements beyond user-supplied buffers.
+ *
+ */
+int mca_coll_acoll_barrier_intra(struct ompi_communicator_t *comm, mca_coll_base_module_t *module)
+{
+    int size, ssize, bsize;
+    int err = MPI_SUCCESS;
+    int nreqs = 0;
+    ompi_request_t **reqs;
+    int num_nodes;
+    mca_coll_acoll_module_t *acoll_module = (mca_coll_acoll_module_t *) module;
+    coll_acoll_subcomms_t *subc = NULL;
+
+    /* Obtain the subcomms structure */
+    err = check_and_create_subc(comm, acoll_module, &subc);
+
+    /* Fallback to linear if subcomms structure is not obtained */
+    if (NULL == subc) {
+        return ompi_coll_base_barrier_intra_basic_linear(comm, module);
+    }
+
+    size = ompi_comm_size(comm);
+    if (1 == size) {
+        return err;
+    }
+    if (!subc->initialized && size > 1) {
+        err = mca_coll_acoll_comm_split_init(comm, acoll_module, subc, 0);
+        if (MPI_SUCCESS != err) {
+            return err;
+        }
+    }
+    num_nodes = size > 1 ? subc->num_nodes : 1;
+
+    reqs = ompi_coll_base_comm_get_reqs(module->base_data, size);
+    if (NULL == reqs) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    ssize = ompi_comm_size(subc->subgrp_comm);
+    bsize = ompi_comm_size(subc->base_comm[MCA_COLL_ACOLL_L3CACHE][MCA_COLL_ACOLL_LYR_NODE]);
+
+    /* Sends from leaf ranks at subgroup level */
+    if (ssize > 1) {
+        err = mca_coll_acoll_barrier_send_subc(subc->subgrp_comm, module, reqs, &nreqs,
+                                               subc->subgrp_root);
+        if (MPI_SUCCESS != err) {
+            ompi_coll_base_free_reqs(reqs, nreqs);
+            return err;
+        }
+    }
+    /* Sends from leaf ranks at base rank level */
+    if ((bsize > 1) && (subc->base_root[MCA_COLL_ACOLL_L3CACHE][MCA_COLL_ACOLL_LYR_NODE] != -1)) {
+        err = mca_coll_acoll_barrier_send_subc(
+            subc->base_comm[MCA_COLL_ACOLL_L3CACHE][MCA_COLL_ACOLL_LYR_NODE], module, reqs, &nreqs,
+            subc->base_root[MCA_COLL_ACOLL_L3CACHE][MCA_COLL_ACOLL_LYR_NODE]);
+        if (MPI_SUCCESS != err) {
+            ompi_coll_base_free_reqs(reqs, nreqs);
+            return err;
+        }
+    }
+    /* Sends from leaf ranks at node leader level */
+    if ((num_nodes > 1) && (subc->outer_grp_root != -1)) {
+        err = mca_coll_acoll_barrier_send_subc(subc->leader_comm, module, reqs, &nreqs,
+                                               subc->outer_grp_root);
+        if (MPI_SUCCESS != err) {
+            ompi_coll_base_free_reqs(reqs, nreqs);
+            return err;
+        }
+    }
+
+    /* Leaf ranks at node leader level receive from root */
+    if ((num_nodes > 1) && (subc->outer_grp_root != -1)) {
+        err = mca_coll_acoll_barrier_recv_subc(subc->leader_comm, module, reqs, &nreqs,
+                                               subc->outer_grp_root);
+        if (MPI_SUCCESS != err) {
+            ompi_coll_base_free_reqs(reqs, nreqs);
+            return err;
+        }
+    }
+    /* Leaf ranks at base rank level receive from inter leader */
+    if ((bsize > 1) && (subc->base_root[MCA_COLL_ACOLL_L3CACHE][MCA_COLL_ACOLL_LYR_NODE] != -1)) {
+        err = mca_coll_acoll_barrier_recv_subc(
+            subc->base_comm[MCA_COLL_ACOLL_L3CACHE][MCA_COLL_ACOLL_LYR_NODE], module, reqs, &nreqs,
+            subc->base_root[MCA_COLL_ACOLL_L3CACHE][MCA_COLL_ACOLL_LYR_NODE]);
+        if (MPI_SUCCESS != err) {
+            ompi_coll_base_free_reqs(reqs, nreqs);
+            return err;
+        }
+    }
+    /* Leaf ranks at subgroup level to receive from base ranks */
+    if (ssize > 1) {
+        err = mca_coll_acoll_barrier_recv_subc(subc->subgrp_comm, module, reqs, &nreqs,
+                                               subc->subgrp_root);
+        if (MPI_SUCCESS != err) {
+            ompi_coll_base_free_reqs(reqs, nreqs);
+            return err;
+        }
+    }
+
+    /* All done */
+    ompi_coll_base_free_reqs(reqs, nreqs);
+    return err;
+}
diff --git a/ompi/mca/coll/acoll/coll_acoll_bcast.c b/ompi/mca/coll/acoll/coll_acoll_bcast.c
new file mode 100644
index 00000000000..9a9d447a167
--- /dev/null
+++ b/ompi/mca/coll/acoll/coll_acoll_bcast.c
@@ -0,0 +1,562 @@
+/* -*- Mode: C; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "mpi.h"
+#include "ompi/constants.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_functions.h"
+#include "ompi/mca/coll/base/coll_tags.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/pml/pml.h"
+#include "opal/util/bit_ops.h"
+#include "coll_acoll.h"
+#include "coll_acoll_utils.h"
+
+typedef int (*bcast_subc_func)(void *buff, size_t count, struct ompi_datatype_t *datatype, int root,
+                               struct ompi_communicator_t *comm, ompi_request_t **preq, int *nreqs,
+                               int world_rank);
+
+/*
+ * bcast_binomial
+ *
+ * Function:    Broadcast operation using balanced binomial tree
+ *
+ * Description: Core logic of implementation is derived from that in
+ *              "basic" component.
+ */
+static int bcast_binomial(void *buff, size_t count, struct ompi_datatype_t *datatype, int root,
+                          struct ompi_communicator_t *comm, ompi_request_t **preq, int *nreqs,
+                          int world_rank)
+{
+    int msb_pos, sub_rank, peer, err = MPI_SUCCESS;
+    int size, rank, dim;
+    int i, mask;
+
+    size = ompi_comm_size(comm);
+    rank = ompi_comm_rank(comm);
+    dim = comm->c_cube_dim;
+    sub_rank = (rank - root + size) % size;
+
+    msb_pos = opal_hibit(sub_rank, dim);
+    --dim;
+
+    /* Receive data from parent in the subgroup tree. */
+    if (sub_rank > 0) {
+        assert(msb_pos >= 0);
+        peer = ((sub_rank & ~(1 << msb_pos)) + root) % size;
+
+        err = MCA_PML_CALL(
+            recv(buff, count, datatype, peer, MCA_COLL_BASE_TAG_BCAST, comm, MPI_STATUS_IGNORE));
+        if (MPI_SUCCESS != err) {
+            return err;
+        }
+    }
+
+    for (i = msb_pos + 1, mask = 1 << i; i <= dim; ++i, mask <<= 1) {
+        peer = sub_rank | mask;
+        if (peer < size) {
+            peer = (peer + root) % size;
+            *nreqs = *nreqs + 1;
+
+            err = MCA_PML_CALL(isend(buff, count, datatype, peer, MCA_COLL_BASE_TAG_BCAST,
+                                     MCA_PML_BASE_SEND_STANDARD, comm, preq++));
+            if (MPI_SUCCESS != err) {
+                return err;
+            }
+        }
+    }
+
+    return err;
+}
+
+static int bcast_flat_tree(void *buff, size_t count, struct ompi_datatype_t *datatype, int root,
+                           struct ompi_communicator_t *comm, ompi_request_t **preq, int *nreqs,
+                           int world_rank)
+{
+    int peer;
+    int err = MPI_SUCCESS;
+    int rank = ompi_comm_rank(comm);
+    int size = ompi_comm_size(comm);
+
+    if (rank == root) {
+        for (peer = 0; peer < size; peer++) {
+            if (peer == root) {
+                continue;
+            }
+            *nreqs = *nreqs + 1;
+            err = MCA_PML_CALL(isend(buff, count, datatype, peer, MCA_COLL_BASE_TAG_BCAST,
+                                     MCA_PML_BASE_SEND_STANDARD, comm, preq++));
+            if (MPI_SUCCESS != err) {
+                return err;
+            }
+        }
+    } else {
+        err = MCA_PML_CALL(
+            recv(buff, count, datatype, root, MCA_COLL_BASE_TAG_BCAST, comm, MPI_STATUS_IGNORE));
+        if (MPI_SUCCESS != err) {
+            return err;
+        }
+    }
+
+    return err;
+}
+
+/*
+ * coll_bcast_decision_fixed
+ *
+ * Function:    Choose optimal broadcast algorithm
+ *
+ * Description: Based on no. of processes and message size, chooses [log|lin]
+ *              broadcast and subgroup size to be used.
+ *
+ */
+
+#define SET_BCAST_PARAMS(l0, l1, l2) \
+    *lin_0 = l0;                     \
+    *lin_1 = l1;                     \
+    *lin_2 = l2;
+
+static inline void coll_bcast_decision_fixed(int size, size_t total_dsize, int node_size,
+                                             int *sg_cnt, int *use_0, int *use_numa,
+                                             int *use_socket, int *lin_0,
+                                             int *lin_1, int *lin_2, int num_nodes,
+                                             mca_coll_acoll_module_t *acoll_module,
+                                             coll_acoll_subcomms_t *subc)
+{
+    int sg_size = *sg_cnt;
+    *use_0 = 0;
+    *lin_0 = 0;
+    *use_numa = 0;
+    *use_socket = 0;
+    if (size <= node_size) {
+        if (acoll_module->use_dyn_rules) {
+            *sg_cnt = (acoll_module->mnode_sg_size == acoll_module->sg_cnt) ? acoll_module->sg_cnt : node_size;
+            *use_0 = 0;
+            SET_BCAST_PARAMS(acoll_module->use_lin0, acoll_module->use_lin1, acoll_module->use_lin2)
+        } else if (size <= sg_size) {
+            *sg_cnt = sg_size;
+            if (total_dsize <= 8192) {
+                SET_BCAST_PARAMS(0, 0, 0)
+            } else {
+                SET_BCAST_PARAMS(0, 1, 1)
+            }
+        } else if (size <= (sg_size << 1)) {
+            if (total_dsize <= 1024) {
+                *sg_cnt = size;
+                SET_BCAST_PARAMS(0, 0, 0)
+            } else if (total_dsize <= 8192) {
+                *sg_cnt = sg_size;
+                SET_BCAST_PARAMS(0, 0, 0)
+            } else if (total_dsize <= 2097152) {
+                *sg_cnt = size;
+                SET_BCAST_PARAMS(0, 1, 1)
+            } else {
+                *sg_cnt = sg_size;
+                SET_BCAST_PARAMS(0, 0, 0)
+            }
+        } else if (size <= (sg_size << 2)) {
+            if (total_dsize <= 1024) {
+                *sg_cnt = size;
+                SET_BCAST_PARAMS(0, 0, 0)
+            } else if (total_dsize <= 8192) {
+                *sg_cnt = sg_size;
+                SET_BCAST_PARAMS(0, 0, 0)
+            } else if (total_dsize <= 32768) {
+                *sg_cnt = sg_size;
+                SET_BCAST_PARAMS(0, 1, 1)
+            } else if (total_dsize <= 4194304) {
+                *sg_cnt = size;
+                SET_BCAST_PARAMS(0, 1, 1)
+            } else {
+                *sg_cnt = sg_size;
+                SET_BCAST_PARAMS(0, 0, 0)
+            }
+        } else if (size <= (sg_size << 3)) {
+            if (total_dsize <= 1024) {
+                *sg_cnt = size;
+                SET_BCAST_PARAMS(0, 0, 0)
+            } else if (total_dsize <= 8192) {
+                *sg_cnt = sg_size;
+                SET_BCAST_PARAMS(0, 0, 0)
+            } else if (total_dsize <= 262144) {
+                *sg_cnt = sg_size;
+                SET_BCAST_PARAMS(0, 1, 1)
+            } else {
+                *sg_cnt = size;
+                SET_BCAST_PARAMS(0, 1, 1)
+            }
+        } else if (size <= (sg_size << 4)) {
+            if (total_dsize <= 512) {
+                *sg_cnt = size;
+                SET_BCAST_PARAMS(0, 0, 0)
+            } else if (total_dsize <= 8192) {
+                *sg_cnt = sg_size;
+                SET_BCAST_PARAMS(0, 0, 0)
+            } else if (total_dsize <= 262144) {
+                *sg_cnt = sg_size;
+                SET_BCAST_PARAMS(0, 1, 1)
+            } else {
+                *sg_cnt = size;
+                SET_BCAST_PARAMS(0, 1, 1)
+            }
+        } else {
+            if (total_dsize <= 512) {
+                *sg_cnt = size;
+                SET_BCAST_PARAMS(0, 0, 0)
+            } else if (total_dsize <= 8192) {
+                *sg_cnt = sg_size;
+                SET_BCAST_PARAMS(0, 0, 0)
+            } else if (total_dsize <= 262144) {
+                *sg_cnt = sg_size;
+                SET_BCAST_PARAMS(0, 1, 1)
+            } else if (total_dsize <= 16777216) {
+                *sg_cnt = size;
+                SET_BCAST_PARAMS(0, 1, 1)
+            } else {
+                *sg_cnt = sg_size;
+                *use_numa = 1;
+                SET_BCAST_PARAMS(0, 1, 1)
+            }
+        }
+    } else {
+        if (acoll_module->use_dyn_rules) {
+            *sg_cnt = (acoll_module->mnode_sg_size == acoll_module->sg_cnt) ? acoll_module->sg_cnt : node_size;
+            *use_0 = acoll_module->use_mnode;
+            SET_BCAST_PARAMS(acoll_module->use_lin0, acoll_module->use_lin1, acoll_module->use_lin2)
+        } else {
+            *use_0 = 1;
+            *sg_cnt = sg_size;
+            if (2 == num_nodes) {
+                SET_BCAST_PARAMS(1, 1, 1)
+                *use_socket = 1;
+                *use_numa = (total_dsize <= 2097152) ? 0 : 1;
+            } else if (num_nodes <= 4) {
+                if (total_dsize <= 512) {
+                    *use_socket = 1;
+                    SET_BCAST_PARAMS(1, 1, 0)
+                } else if (total_dsize <= 2097152) {
+                    *use_socket = 1;
+                    SET_BCAST_PARAMS(1, 1, 1)
+                } else {
+                    *use_numa = 1;
+                    *use_socket = (total_dsize <= 4194304) ? 0 : 1;
+                    SET_BCAST_PARAMS(1, 1, 1)
+                }
+            } else if (num_nodes <= 6) {
+                SET_BCAST_PARAMS(1, 1, 1)
+                if (total_dsize <= 524288) {
+                    *use_socket = 1;
+                } else {
+                    *use_numa = 1;
+                }
+            } else if (num_nodes <= 8) {
+                SET_BCAST_PARAMS(1, 1, 1)
+                if (total_dsize <= 8192) {
+                    *use_numa = 0;
+                } else {
+                    *use_numa = 1;
+                }
+            } else if (num_nodes <= 10) {
+                *use_numa = 1;
+                if (total_dsize <= 32768) {
+                    SET_BCAST_PARAMS(1, 1, 0)
+                } else {
+                    SET_BCAST_PARAMS(1, 1, 1)
+                }
+            } else {
+                *use_numa = 1;
+                if (total_dsize <= 64) {
+                    SET_BCAST_PARAMS(1, 0, 1)
+                } else if (total_dsize <= 2097152) {
+                    SET_BCAST_PARAMS(1, 1, 1)
+                } else {
+                    *use_socket = 1;
+                    SET_BCAST_PARAMS(0, 1, 1)
+                }
+            }
+        }
+    }
+    if (-1 != acoll_module->force_numa) {
+        *use_numa = acoll_module->force_numa;
+        if (acoll_module->force_numa) {
+            *sg_cnt = sg_size;
+        }
+    }
+    if (-1 != acoll_module->use_socket) {
+        *use_socket = acoll_module->use_socket;
+    }
+}
+
+static inline void coll_acoll_bcast_subcomms(struct ompi_communicator_t *comm,
+                                             coll_acoll_subcomms_t *subc,
+                                             struct ompi_communicator_t **subcomms, int *subc_roots,
+                                             int root, int num_nodes, int use_0, int no_sg,
+                                             int use_numa, int use_socket)
+{
+    int lyr_id = use_socket ? MCA_COLL_ACOLL_LYR_SOCKET : MCA_COLL_ACOLL_LYR_NODE;
+    /* Node leaders */
+    if (use_0) {
+        subcomms[MCA_COLL_ACOLL_NODE_L] = subc->leader_comm;
+        subc_roots[MCA_COLL_ACOLL_NODE_L] = subc->outer_grp_root;
+    }
+    /* Socket leaders */
+    if (use_socket) {
+        subcomms[MCA_COLL_ACOLL_NODE_L] = subc->socket_ldr_comm;
+        subc_roots[MCA_COLL_ACOLL_NODE_L] = subc->socket_ldr_root;
+    }
+    /* Intra comm */
+    if (((num_nodes > 1) && use_0) || use_socket) {
+        int is_root = use_socket ? subc->is_root_socket : subc->is_root_node;
+        subc_roots[MCA_COLL_ACOLL_INTRA] = is_root ? subc->local_root[lyr_id] : 0;
+        subcomms[MCA_COLL_ACOLL_INTRA] = use_socket ? subc->socket_comm : subc->local_comm;
+    } else {
+        subc_roots[MCA_COLL_ACOLL_INTRA] = root;
+        subcomms[MCA_COLL_ACOLL_INTRA] = comm;
+    }
+    /* Base ranks comm */
+    int parent = lyr_id;
+    if (no_sg) {
+        subcomms[MCA_COLL_ACOLL_L3_L] = subcomms[MCA_COLL_ACOLL_INTRA];
+        subc_roots[MCA_COLL_ACOLL_L3_L] = subc_roots[MCA_COLL_ACOLL_INTRA];
+    } else {
+        subcomms[MCA_COLL_ACOLL_L3_L] = subc->base_comm[MCA_COLL_ACOLL_L3CACHE]
+                                                       [parent];
+        subc_roots[MCA_COLL_ACOLL_L3_L] = subc->base_root[MCA_COLL_ACOLL_L3CACHE]
+                                                         [parent];
+    }
+    /* Subgroup comm */
+    subcomms[MCA_COLL_ACOLL_LEAF] = subc->subgrp_comm;
+    subc_roots[MCA_COLL_ACOLL_LEAF] = subc->subgrp_root;
+
+    /* Override with numa when needed */
+    if (use_numa) {
+        subcomms[MCA_COLL_ACOLL_L3_L] = subc->base_comm[MCA_COLL_ACOLL_NUMA]
+                                                       [parent];
+        subc_roots[MCA_COLL_ACOLL_L3_L] = subc->base_root[MCA_COLL_ACOLL_NUMA]
+                                                         [parent];
+        subcomms[MCA_COLL_ACOLL_LEAF] = subc->numa_comm;
+        subc_roots[MCA_COLL_ACOLL_LEAF] = subc->numa_root;
+    }
+}
+
+static int mca_coll_acoll_bcast_intra_node(void *buff, size_t count, struct ompi_datatype_t *datatype,
+                                           mca_coll_base_module_t *module,
+                                           coll_acoll_subcomms_t *subc,
+                                           struct ompi_communicator_t **subcomms, int *subc_roots,
+                                           int lin_1, int lin_2, int no_sg, int use_numa,
+                                           int use_socket, int world_rank)
+{
+    int size;
+    int rank;
+    int err;
+    int subgrp_size;
+    int is_base = 0;
+    int nreqs;
+    ompi_request_t **preq, **reqs;
+    struct ompi_communicator_t *comm = subcomms[MCA_COLL_ACOLL_INTRA];
+    bcast_subc_func bcast_intra[2] = {&bcast_binomial, &bcast_flat_tree};
+
+    rank = ompi_comm_rank(comm);
+    size = ompi_comm_size(comm);
+
+    reqs = ompi_coll_base_comm_get_reqs(module->base_data, size);
+    if (NULL == reqs) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+    nreqs = 0;
+    preq = reqs;
+    err = MPI_SUCCESS;
+    if (no_sg) {
+        is_base = 1;
+    } else {
+        int ind1 = use_numa ? MCA_COLL_ACOLL_NUMA : MCA_COLL_ACOLL_L3CACHE;
+        int ind2 = use_socket ? MCA_COLL_ACOLL_LYR_SOCKET : MCA_COLL_ACOLL_LYR_NODE;
+        is_base = rank == subc->base_rank[ind1][ind2] ? 1 : 0;
+    }
+
+    /* All base ranks receive from root */
+    if (is_base) {
+        err = bcast_intra[lin_1](buff, count, datatype, subc_roots[MCA_COLL_ACOLL_L3_L],
+                                 subcomms[MCA_COLL_ACOLL_L3_L], preq, &nreqs, world_rank);
+        if (MPI_SUCCESS != err) {
+            ompi_coll_base_free_reqs(reqs, nreqs);
+            return err;
+        }
+    }
+
+    /* Start and wait on all requests. */
+    if (nreqs > 0) {
+        err = ompi_request_wait_all(nreqs, reqs, MPI_STATUSES_IGNORE);
+        if (MPI_SUCCESS != err) {
+            ompi_coll_base_free_reqs(reqs, nreqs);
+        }
+    }
+
+    /* If single stage, return */
+    if (no_sg) {
+        ompi_coll_base_free_reqs(reqs, nreqs);
+        return err;
+    }
+
+    subgrp_size = use_numa ? ompi_comm_size(subc->numa_comm) : subc->subgrp_size;
+    /* All leaf ranks receive from the respective base rank */
+    if ((subgrp_size > 1) && !no_sg) {
+        err = bcast_intra[lin_2](buff, count, datatype, subc_roots[MCA_COLL_ACOLL_LEAF],
+                                 subcomms[MCA_COLL_ACOLL_LEAF], preq, &nreqs, world_rank);
+    }
+
+    /* Start and wait on all requests. */
+    if (nreqs > 0) {
+        err = ompi_request_wait_all(nreqs, reqs, MPI_STATUSES_IGNORE);
+        if (MPI_SUCCESS != err) {
+            ompi_coll_base_free_reqs(reqs, nreqs);
+        }
+    }
+
+    /* All done */
+    ompi_coll_base_free_reqs(reqs, nreqs);
+    return err;
+}
+
+/*
+ * mca_coll_acoll_bcast
+ *
+ * Function:    Broadcast operation using subgroup based algorithm
+ * Accepts:     Same arguments as MPI_Bcast()
+ * Returns:     MPI_SUCCESS or error code
+ *
+ * Description: Broadcast is performed across and within subgroups.
+ *              O(N) or O(log(N)) algorithm within sunbgroup based on count.
+ *              Subgroups can be 1 or more based on size and count.
+ *
+ * Limitations: None
+ *
+ * Memory:      No additional memory requirements beyond user-supplied buffers.
+ *
+ */
+int mca_coll_acoll_bcast(void *buff, size_t count, struct ompi_datatype_t *datatype, int root,
+                         struct ompi_communicator_t *comm, mca_coll_base_module_t *module)
+{
+    int size;
+    int rank;
+    int err;
+    int nreqs;
+    ompi_request_t **preq, **reqs;
+    int sg_cnt, node_size;
+    int num_nodes;
+    int use_0 = 0;
+    int lin_0 = 0, lin_1 = 0, lin_2 = 0;
+    int use_numa = 0, use_socket = 0;
+    int no_sg;
+    size_t total_dsize, dsize;
+    mca_coll_acoll_module_t *acoll_module = (mca_coll_acoll_module_t *) module;
+    bcast_subc_func bcast_func[2] = {&bcast_binomial, &bcast_flat_tree};
+    coll_acoll_subcomms_t *subc = NULL;
+    struct ompi_communicator_t *subcomms[MCA_COLL_ACOLL_NUM_SC] = {NULL};
+    int subc_roots[MCA_COLL_ACOLL_NUM_SC] = {-1};
+
+    /* For small communicators, use linear bcast */
+    size = ompi_comm_size(comm);
+    if (size < 8) {
+        return ompi_coll_base_bcast_intra_basic_linear(buff, count, datatype, root, comm, module);
+    }
+
+    /* Obtain the subcomms structure */
+    err = check_and_create_subc(comm, acoll_module, &subc);
+    /* Fallback to knomial if subcomms is not obtained */
+    if (NULL == subc) {
+        return ompi_coll_base_bcast_intra_knomial(buff, count, datatype, root, comm, module, 0, 4);
+    }
+
+    /* Fallback to knomial if no. of root changes is beyond a threshold */
+    if ((subc->num_root_change > MCA_COLL_ACOLL_ROOT_CHANGE_THRESH)
+        && (root != subc->prev_init_root)) {
+        return ompi_coll_base_bcast_intra_knomial(buff, count, datatype, root, comm, module, 0, 4);
+    }
+    if ((!subc->initialized || (root != subc->prev_init_root)) && size > 2) {
+        err = mca_coll_acoll_comm_split_init(comm, acoll_module, subc, root);
+        if (MPI_SUCCESS != err) {
+            return err;
+        }
+    }
+
+    ompi_datatype_type_size(datatype, &dsize);
+    total_dsize = dsize * count;
+    rank = ompi_comm_rank(comm);
+    sg_cnt = acoll_module->sg_cnt;
+    if (size > 2) {
+        num_nodes = subc->num_nodes;
+        node_size = ompi_comm_size(subc->local_comm);
+    } else {
+        num_nodes = 1;
+        node_size = size;
+    }
+
+    /* Use knomial for nodes 8 and above and non-large messages */
+    if ((num_nodes >= 8 && total_dsize <= 65536)
+        || (1 == num_nodes && size >= 256 && total_dsize < 16384)) {
+        return ompi_coll_base_bcast_intra_knomial(buff, count, datatype, root, comm, module, 0, 4);
+    }
+
+    /* Determine the algorithm to be used based on size and count */
+    /* sg_cnt determines subgroup based communication */
+    /* lin_1 and lin_2 indicate whether to use linear or log based
+     sends/receives across and within subgroups respectively. */
+    coll_bcast_decision_fixed(size, total_dsize, node_size, &sg_cnt, &use_0, &use_numa, &use_socket, &lin_0,
+                              &lin_1, &lin_2, num_nodes, acoll_module, subc);
+    no_sg = (sg_cnt == node_size) ? 1 : 0;
+    if (size <= 2)
+        no_sg = 1;
+
+    coll_acoll_bcast_subcomms(comm, subc, subcomms, subc_roots, root, num_nodes, use_0, no_sg,
+                              use_numa, use_socket);
+
+    reqs = ompi_coll_base_comm_get_reqs(module->base_data, size);
+    if (NULL == reqs) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+    nreqs = 0;
+    preq = reqs;
+    err = MPI_SUCCESS;
+
+    if (use_0 || use_socket) {
+        if (subc_roots[MCA_COLL_ACOLL_NODE_L] != -1) {
+            err = bcast_func[lin_0](buff, count, datatype, subc_roots[MCA_COLL_ACOLL_NODE_L],
+                                    subcomms[MCA_COLL_ACOLL_NODE_L], preq, &nreqs, rank);
+            if (MPI_SUCCESS != err) {
+                ompi_coll_base_free_reqs(reqs, nreqs);
+                return err;
+            }
+        }
+    }
+
+    /* Start and wait on all requests. */
+    if (nreqs > 0) {
+        err = ompi_request_wait_all(nreqs, reqs, MPI_STATUSES_IGNORE);
+        if (MPI_SUCCESS != err) {
+            ompi_coll_base_free_reqs(reqs, nreqs);
+            return err;
+        }
+    }
+
+    err = mca_coll_acoll_bcast_intra_node(buff, count, datatype, module, subc, subcomms, subc_roots,
+                                          lin_1, lin_2, no_sg, use_numa, use_socket, rank);
+
+    if (MPI_SUCCESS != err) {
+        ompi_coll_base_free_reqs(reqs, nreqs);
+        return err;
+    }
+
+    /* All done */
+    ompi_coll_base_free_reqs(reqs, nreqs);
+    return err;
+}
diff --git a/ompi/mca/coll/acoll/coll_acoll_component.c b/ompi/mca/coll/acoll/coll_acoll_component.c
new file mode 100644
index 00000000000..d3c8d2469b1
--- /dev/null
+++ b/ompi/mca/coll/acoll/coll_acoll_component.c
@@ -0,0 +1,410 @@
+/* -*- Mode: C; c-acoll-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2024 - 2025 Advanced Micro Devices, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include "ompi_config.h"
+
+#include "mpi.h"
+#include "ompi/mca/coll/coll.h"
+#include "coll_acoll.h"
+
+/*
+ * Public string showing the coll ompi_acoll component version number
+ */
+const char *mca_coll_acoll_component_version_string
+    = "Open MPI acoll collective MCA component version " OMPI_VERSION;
+
+/*
+ * Global variables
+ */
+int mca_coll_acoll_priority = 0;
+int mca_coll_acoll_max_comms = 10;
+int mca_coll_acoll_sg_size = 8;
+int mca_coll_acoll_sg_scale = 1;
+int mca_coll_acoll_node_size = 128;
+int mca_coll_acoll_force_numa = -1;
+int mca_coll_acoll_use_dynamic_rules = 0;
+int mca_coll_acoll_mnode_enable = 1;
+int mca_coll_acoll_bcast_lin0 = 0;
+int mca_coll_acoll_bcast_lin1 = 0;
+int mca_coll_acoll_bcast_lin2 = 0;
+int mca_coll_acoll_bcast_nonsg = 0;
+int mca_coll_acoll_bcast_socket = -1;
+int mca_coll_acoll_allgather_lin = 0;
+int mca_coll_acoll_allgather_ring_1 = 0;
+int mca_coll_acoll_reserve_memory_for_algo = 0;
+uint64_t mca_coll_acoll_reserve_memory_size_for_algo = 128 * 32768; // 4 MB
+uint64_t mca_coll_acoll_xpmem_buffer_size = 128 * 32768;
+int mca_coll_acoll_alltoall_split_factor = 0;
+size_t mca_coll_acoll_alltoall_psplit_msg_thres = 0;
+size_t mca_coll_acoll_alltoall_xpmem_msg_thres = 0;
+
+/* By default utilize xpmem based algorithms applicable when built with xpmem. */
+int mca_coll_acoll_without_xpmem = 0;
+int mca_coll_acoll_xpmem_use_sr_buf = 1;
+
+/*
+ * Local function
+ */
+static int acoll_register(void);
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+
+const mca_coll_base_component_3_0_0_t mca_coll_acoll_component = {
+
+    /* First, the mca_component_t struct containing meta information
+     * about the component itself */
+
+    .collm_version = {
+        MCA_COLL_BASE_VERSION_3_0_0,
+
+        /* Component name and version */
+        .mca_component_name = "acoll",
+        MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION,
+                              OMPI_RELEASE_VERSION),
+
+        /* Component open and close functions */
+        .mca_register_component_params = acoll_register,
+    },
+    .collm_data = {
+        /* The component is checkpoint ready */
+        MCA_BASE_METADATA_PARAM_CHECKPOINT
+    },
+
+    /* Initialization / querying functions */
+
+    .collm_init_query = mca_coll_acoll_init_query,
+    .collm_comm_query = mca_coll_acoll_comm_query,
+};
+
+static int acoll_register(void)
+{
+    /* Use a low priority, but allow other components to be lower */
+    mca_coll_acoll_priority = 0;
+    (void) mca_base_component_var_register(&mca_coll_acoll_component.collm_version, "priority",
+                                           "Priority of the acoll coll component",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_READONLY, &mca_coll_acoll_priority);
+
+    /* Defaults on topology */
+    (void)
+        mca_base_component_var_register(&mca_coll_acoll_component.collm_version, "max_comms",
+                                        "Maximum no. of communicators using subgroup based algorithms",
+                                        MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9,
+                                        MCA_BASE_VAR_SCOPE_READONLY, &mca_coll_acoll_max_comms);
+    (void)
+        mca_base_component_var_register(&mca_coll_acoll_component.collm_version, "sg_size",
+                                        "Size of subgroup to be used for subgroup based algorithms",
+                                        MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9,
+                                        MCA_BASE_VAR_SCOPE_READONLY, &mca_coll_acoll_sg_size);
+
+    (void) mca_base_component_var_register(
+        &mca_coll_acoll_component.collm_version, "sg_scale",
+        "Scale factor for effective subgroup size for subgroup based algorithms",
+        MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+        &mca_coll_acoll_sg_scale);
+    (void) mca_base_component_var_register(&mca_coll_acoll_component.collm_version, "node_size",
+                                           "Size of node for multinode cases",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_READONLY, &mca_coll_acoll_node_size);
+    (void) mca_base_component_var_register(&mca_coll_acoll_component.collm_version, "force_numa",
+                                           "Force enable/disable NUMA based comm split",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_READONLY, &mca_coll_acoll_force_numa);
+    (void)
+        mca_base_component_var_register(&mca_coll_acoll_component.collm_version,
+                                        "use_dynamic_rules",
+                                        "Use dynamic selection of algorithms for multinode cases",
+                                        MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9,
+                                        MCA_BASE_VAR_SCOPE_READONLY,
+                                        &mca_coll_acoll_use_dynamic_rules);
+    (void) mca_base_component_var_register(&mca_coll_acoll_component.collm_version, "mnode_enable",
+                                           "Enable separate algorithm for multinode cases",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           &mca_coll_acoll_mnode_enable);
+    (void) mca_base_component_var_register(&mca_coll_acoll_component.collm_version, "bcast_lin0",
+                                           "Use lin/log for stage 0 of multinode algorithm",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_READONLY, &mca_coll_acoll_bcast_lin0);
+    (void) mca_base_component_var_register(&mca_coll_acoll_component.collm_version, "bcast_lin1",
+                                           "Use lin/log for stage 1 of multinode algorithm",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_READONLY, &mca_coll_acoll_bcast_lin1);
+    (void) mca_base_component_var_register(&mca_coll_acoll_component.collm_version, "bcast_lin2",
+                                           "Use lin/log for stage 2 of multinode algorithm",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_READONLY, &mca_coll_acoll_bcast_lin2);
+    (void) mca_base_component_var_register(
+        &mca_coll_acoll_component.collm_version, "bcast_nonsg",
+        "Flag to turn on/off subgroup based algorithms for multinode", MCA_BASE_VAR_TYPE_INT, NULL,
+        0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &mca_coll_acoll_bcast_nonsg);
+    (void) mca_base_component_var_register(&mca_coll_acoll_component.collm_version, "bcast_socket",
+                                           "Flag to turn on/off socket based algorithms for bcast",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_READONLY, &mca_coll_acoll_bcast_socket);
+    (void) mca_base_component_var_register(&mca_coll_acoll_component.collm_version, "allgather_lin",
+                                           "Flag to indicate use of linear allgather for multinode",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           &mca_coll_acoll_allgather_lin);
+    (void)
+        mca_base_component_var_register(&mca_coll_acoll_component.collm_version, "allgather_ring_1",
+                                        "Flag to indicate use of ring/rd allgather for multinode",
+                                        MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9,
+                                        MCA_BASE_VAR_SCOPE_READONLY,
+                                        &mca_coll_acoll_allgather_ring_1);
+    (void) mca_base_component_var_register(
+        &mca_coll_acoll_component.collm_version, "reserve_memory_for_algo",
+        "Flag to inform the acoll component to reserve/pre-allocate memory"
+        " for use inside collective algorithms.",
+        MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+        &mca_coll_acoll_reserve_memory_for_algo);
+    (void) mca_base_component_var_register(
+        &mca_coll_acoll_component.collm_version, "reserve_memory_size_for_algo",
+        "Size of memory to be allocated by acoll component to use as reserve"
+        "memory inside collective algorithms.",
+        MCA_BASE_VAR_TYPE_UINT64_T, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+        &mca_coll_acoll_reserve_memory_size_for_algo);
+    (void) mca_base_component_var_register(
+        &mca_coll_acoll_component.collm_version, "without_xpmem",
+        "By default, xpmem-based algorithms are used when applicable. "
+        "When this flag is set to 1, xpmem-based algorithms are disabled.",
+        MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+        &mca_coll_acoll_without_xpmem);
+    (void) mca_base_component_var_register(
+        &mca_coll_acoll_component.collm_version, "xpmem_buffer_size",
+        "Maximum size of memory that can be used for temporary buffers for "
+        "xpmem-based algorithms. By default these buffers are not created or "
+        "used unless xpmem_use_sr_buf is set to 0.",
+        MCA_BASE_VAR_TYPE_UINT64_T, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+        &mca_coll_acoll_xpmem_buffer_size);
+    (void) mca_base_component_var_register(
+        &mca_coll_acoll_component.collm_version, "xpmem_use_sr_buf",
+        "Uses application provided send/recv buffers during xpmem registration "
+        "when set to 1 instead of temporary buffers. The send/recv buffers are "
+        "assumed to persist for the duration of the application.",
+        MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+        &mca_coll_acoll_xpmem_use_sr_buf);
+    (void) mca_base_component_var_register(
+        &mca_coll_acoll_component.collm_version, "alltoall_split_factor",
+        "Split factor value to be used in alltoall parallel split algorithm,"
+        "valid values are 2, 4, 8, 16, 32, 64.",
+        MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+        &mca_coll_acoll_alltoall_split_factor);
+    (void) mca_base_component_var_register(
+        &mca_coll_acoll_component.collm_version, "alltoall_psplit_msg_thresh",
+        "Message threshold above which parallel split alltoall algorithm "
+        "should not be used.",
+        MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+        &mca_coll_acoll_alltoall_psplit_msg_thres);
+    (void) mca_base_component_var_register(
+        &mca_coll_acoll_component.collm_version, "alltoall_xpmem_msg_thresh",
+        "Message threshold above which xpmem based linear alltoall algorithm "
+        "should be used for intra node cases.",
+        MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+        &mca_coll_acoll_alltoall_xpmem_msg_thres);
+
+    return OMPI_SUCCESS;
+}
+
+/*
+ * Module constructor
+ */
+static void mca_coll_acoll_module_construct(mca_coll_acoll_module_t *module)
+{
+
+    /* Set number of subcomms to 0 */
+    module->num_subc = 0;
+    module->subc = NULL;
+
+    /* Reserve memory init. Lazy allocation of memory when needed. */
+    (module->reserve_mem_s).reserve_mem = NULL;
+    (module->reserve_mem_s).reserve_mem_size = 0;
+    (module->reserve_mem_s).reserve_mem_allocate = false;
+    (module->reserve_mem_s).reserve_mem_in_use = false;
+    if ((0 != mca_coll_acoll_reserve_memory_for_algo)
+        && (0 < mca_coll_acoll_reserve_memory_size_for_algo)
+        && (false == ompi_mpi_thread_multiple)) {
+        (module->reserve_mem_s).reserve_mem_allocate = true;
+        (module->reserve_mem_s).reserve_mem_size = mca_coll_acoll_reserve_memory_size_for_algo;
+    }
+
+    /* Ensure valid split factor is given. */
+    int8_t valid_sf = 0;
+    const int split_factor_list[MCA_COLL_ACOLL_SPLIT_FACTOR_LIST_LEN] =
+                MCA_COLL_ACOLL_SPLIT_FACTOR_LIST;
+    for (int ii = 0; ii < MCA_COLL_ACOLL_SPLIT_FACTOR_LIST_LEN; ++ii) {
+        if (split_factor_list[ii] == mca_coll_acoll_alltoall_split_factor) {
+            valid_sf = 1;
+            break;
+        }
+    }
+    (module->alltoall_attr).split_factor = 0;
+    if (1 == valid_sf) {
+        (module->alltoall_attr).split_factor = mca_coll_acoll_alltoall_split_factor;
+    }
+
+    (module->alltoall_attr).psplit_msg_thresh = 0;
+    if (0 < mca_coll_acoll_alltoall_psplit_msg_thres) {
+        (module->alltoall_attr).psplit_msg_thresh =
+            mca_coll_acoll_alltoall_psplit_msg_thres;
+    }
+
+    (module->alltoall_attr).xpmem_msg_thresh = 0;
+    if (0 < mca_coll_acoll_alltoall_xpmem_msg_thres) {
+        (module->alltoall_attr).xpmem_msg_thresh =
+            mca_coll_acoll_alltoall_xpmem_msg_thres;
+    }
+}
+
+/*
+ * Module destructor
+ */
+static void mca_coll_acoll_module_destruct(mca_coll_acoll_module_t *module)
+{
+    for (int i = 0; i < module->num_subc; i++) {
+        coll_acoll_subcomms_t *subc = module->subc[i];
+        if (subc->initialized_data) {
+            if (subc->initialized_shm_data) {
+                if (subc->orig_comm != NULL) {
+                    opal_shmem_unlink(
+                        &((subc->data)->allshmseg_id[ompi_comm_rank(subc->orig_comm)]));
+                    opal_shmem_segment_detach(
+                        &((subc->data)->allshmseg_id[ompi_comm_rank(subc->orig_comm)]));
+                }
+            }
+            coll_acoll_data_t *data = subc->data;
+            if (NULL != data) {
+#ifdef HAVE_XPMEM_H
+                for (int j = 0; j < data->comm_size; j++) {
+                    if (ompi_comm_rank(subc->orig_comm) == j) {
+                        continue;
+                    }
+                    // Dereg all rcache regs.
+                    uint64_t key = 0;
+                    uint64_t value = 0;
+                    uint64_t zero_value = 0;
+                    OPAL_HASH_TABLE_FOREACH(key,uint64,value,(data->xpmem_reg_tracker_ht[j])) {
+                        mca_rcache_base_registration_t* reg =
+                            (mca_rcache_base_registration_t*) key;
+
+                        for (uint64_t d_i = 0; d_i < value; ++d_i) {
+                            (data->rcache[j])->rcache_deregister(data->rcache[j], reg);
+                        }
+                        opal_hash_table_set_value_uint64(data->xpmem_reg_tracker_ht[j],
+                                 key, (void*)(zero_value));
+                    }
+                    xpmem_release(data->all_apid[j]);
+                    mca_rcache_base_module_destroy(data->rcache[j]);
+                    opal_hash_table_remove_all(data->xpmem_reg_tracker_ht[j]);
+                    OBJ_RELEASE(data->xpmem_reg_tracker_ht[j]);
+                }
+                xpmem_remove(data->allseg_id[ompi_comm_rank(subc->orig_comm)]);
+
+                free(data->allseg_id);
+                data->allseg_id = NULL;
+                free(data->all_apid);
+                data->all_apid = NULL;
+                free(data->allshm_sbuf);
+                data->allshm_sbuf = NULL;
+                free(data->allshm_rbuf);
+                data->allshm_rbuf = NULL;
+                free(data->xpmem_saddr);
+                data->xpmem_saddr = NULL;
+                free(data->xpmem_raddr);
+                data->xpmem_raddr = NULL;
+                free(data->scratch);
+                data->scratch = NULL;
+                free(data->xpmem_reg_tracker_ht);
+                data->xpmem_reg_tracker_ht = NULL;
+                free(data->rcache);
+                data->rcache = NULL;
+#endif
+                free(data->allshmseg_id);
+                data->allshmseg_id = NULL;
+                free(data->allshmmmap_sbuf);
+                data->allshmmmap_sbuf = NULL;
+                free(data->l1_gp);
+                data->l1_gp = NULL;
+                free(data->l2_gp);
+                data->l2_gp = NULL;
+                free(data);
+                data = NULL;
+            }
+        }
+
+        if (subc->local_comm != NULL) {
+            ompi_comm_free(&(subc->local_comm));
+            subc->local_comm = NULL;
+        }
+
+        if (subc->local_r_comm != NULL) {
+            ompi_comm_free(&(subc->local_r_comm));
+            subc->local_r_comm = NULL;
+        }
+
+        if (subc->leader_comm != NULL) {
+            ompi_comm_free(&(subc->leader_comm));
+            subc->leader_comm = NULL;
+        }
+
+        if (subc->subgrp_comm != NULL) {
+            ompi_comm_free(&(subc->subgrp_comm));
+            subc->subgrp_comm = NULL;
+        }
+        if (subc->socket_comm != NULL) {
+            ompi_comm_free(&(subc->socket_comm));
+            subc->socket_comm = NULL;
+        }
+
+        if (subc->socket_ldr_comm != NULL) {
+            ompi_comm_free(&(subc->socket_ldr_comm));
+            subc->socket_ldr_comm = NULL;
+        }
+        for (int k = 0; k < MCA_COLL_ACOLL_NUM_BASE_LYRS; k++) {
+            for (int j = 0; j < MCA_COLL_ACOLL_NUM_LAYERS; j++) {
+                if (subc->base_comm[k][j] != NULL) {
+                    ompi_comm_free(&(subc->base_comm[k][j]));
+                    subc->base_comm[k][j] = NULL;
+                }
+            }
+        }
+
+        for (int k = 0; k < MCA_COLL_ACOLL_SPLIT_FACTOR_LIST_LEN; ++k) {
+            if (subc->split_comm[k] != NULL) {
+                ompi_comm_free(&(subc->split_comm[k]));
+                subc->split_comm[k] = NULL;
+            }
+        }
+        subc->initialized = 0;
+        free(subc);
+        module->subc[i] = NULL;
+    }
+
+    module->num_subc = 0;
+    free(module->subc);
+    module->subc = NULL;
+
+    if ((true == (module->reserve_mem_s).reserve_mem_allocate)
+        && (NULL != (module->reserve_mem_s).reserve_mem)) {
+        free((module->reserve_mem_s).reserve_mem);
+    }
+
+    (module->alltoall_attr).split_factor = 0;
+    (module->alltoall_attr).psplit_msg_thresh = 0;
+    (module->alltoall_attr).xpmem_msg_thresh = 0;
+}
+
+OBJ_CLASS_INSTANCE(mca_coll_acoll_module_t, mca_coll_base_module_t, mca_coll_acoll_module_construct,
+                   mca_coll_acoll_module_destruct);
diff --git a/ompi/mca/coll/acoll/coll_acoll_gather.c b/ompi/mca/coll/acoll/coll_acoll_gather.c
new file mode 100644
index 00000000000..429b61296aa
--- /dev/null
+++ b/ompi/mca/coll/acoll/coll_acoll_gather.c
@@ -0,0 +1,217 @@
+/* -*- Mode: C; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "mpi.h"
+#include "ompi/constants.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_functions.h"
+#include "ompi/mca/coll/base/coll_tags.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/pml/pml.h"
+#include "opal/util/bit_ops.h"
+#include "coll_acoll.h"
+#include "coll_acoll_utils.h"
+
+/*
+ * mca_coll_acoll_gather_intra
+ *
+ * Function:    Gather operation using subgroup based algorithm
+ * Accepts:     Same arguments as MPI_Gather()
+ * Returns:     MPI_SUCCESS or error code
+ *
+ * Description: Gather is performed across and within subgroups.
+ *              Subgroups can be 1 or more based on size and count.
+ *
+ * Limitations: Current implementation is optimal only for map-by core.
+ *
+ * Memory:      The base rank of each subgroup may create temporary buffer.
+ *
+ */
+int mca_coll_acoll_gather_intra(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                                void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype, int root,
+                                struct ompi_communicator_t *comm, mca_coll_base_module_t *module)
+{
+    int i, err, rank, size;
+    char *wkg = NULL, *workbuf = NULL;
+    MPI_Status status;
+    MPI_Aint sextent, sgap = 0, ssize;
+    MPI_Aint rextent = 0;
+    size_t total_recv = 0;
+    int sg_cnt, node_cnt;
+    int cur_sg, root_sg;
+    int cur_node, root_node;
+    int is_base, is_local_root;
+    int startr, endr, inc;
+    int startn, endn;
+    int num_nodes;
+    mca_coll_acoll_module_t *acoll_module = (mca_coll_acoll_module_t *) module;
+    coll_acoll_reserve_mem_t *reserve_mem_gather = &(acoll_module->reserve_mem_s);
+
+    size = ompi_comm_size(comm);
+    rank = ompi_comm_rank(comm);
+
+    sg_cnt = acoll_module->sg_cnt;
+    node_cnt = acoll_module->node_cnt;
+    num_nodes = (size + node_cnt - 1) / node_cnt;
+    /* For small messages for nodes 8 and above, fall back to normal */
+    if (num_nodes >= 8 && (rcount < 262144)) {
+        node_cnt = size;
+        sg_cnt = size;
+        num_nodes = 1;
+    }
+
+    /* Setup root for receive */
+    if (rank == root) {
+        ompi_datatype_type_extent(rdtype, &rextent);
+        /* Just use the recv buffer */
+        wkg = (char *) rbuf;
+        if (sbuf != MPI_IN_PLACE) {
+            MPI_Aint root_ofst = rextent * (ptrdiff_t) (rcount * root);
+            err = ompi_datatype_sndrcv((void *) sbuf, scount, sdtype, wkg + (ptrdiff_t) root_ofst,
+                                       rcount, rdtype);
+            if (MPI_SUCCESS != err) {
+                return err;
+            }
+        }
+        total_recv = rcount;
+    }
+
+    /* Setup base ranks of non-root subgroups for receive */
+    cur_sg = rank / sg_cnt;
+    root_sg = root / sg_cnt;
+    is_base = (rank % sg_cnt == 0) && (cur_sg != root_sg);
+    startr = (rank / sg_cnt) * sg_cnt;
+    cur_node = rank / node_cnt;
+    root_node = root / node_cnt;
+    is_local_root = (rank % node_cnt == 0) && (cur_node != root_node);
+    startn = (rank / node_cnt) * node_cnt;
+
+    if (is_base) {
+        size_t buf_size = is_local_root ? (size_t) scount * node_cnt : (size_t) scount * sg_cnt;
+        ompi_datatype_type_extent(sdtype, &sextent);
+        ssize = opal_datatype_span(&sdtype->super, buf_size, &sgap);
+        if (cur_sg != root_sg) {
+            char *tmprecv = NULL;
+            workbuf = (char *) coll_acoll_buf_alloc(reserve_mem_gather, ssize + sgap);
+            if (NULL == workbuf) {
+                return OMPI_ERR_OUT_OF_RESOURCE;
+            }
+            wkg = workbuf - sgap;
+            tmprecv = wkg + sextent * (ptrdiff_t) (rcount * (rank - startr));
+            /* local copy to workbuf */
+            err = ompi_datatype_sndrcv((void *) sbuf, scount, sdtype, tmprecv, scount, sdtype);
+            if (MPI_SUCCESS != err) {
+
+                return err;
+            }
+        }
+        rdtype = sdtype;
+        rcount = scount;
+        rextent = sextent;
+        total_recv = rcount;
+    } else if (rank != root) {
+        wkg = (char *) sbuf;
+        total_recv = scount;
+    }
+
+    /* All base ranks receive from other ranks in their respective subgroup */
+    endr = startr + sg_cnt;
+    if (endr > size) {
+        endr = size;
+    }
+    inc = (rank == root) ? ((root != 0) ? 0 : 1) : 1;
+    if (is_base || (rank == root)) {
+        for (i = startr + inc; i < endr; i++) {
+            char *tmprecv = NULL;
+            if (i == root) {
+                continue;
+            }
+            if (rank == root) {
+                tmprecv = wkg + rextent * (ptrdiff_t) (rcount * i);
+            } else {
+                tmprecv = wkg + rextent * (ptrdiff_t) (rcount * (i - startr));
+            }
+            err = MCA_PML_CALL(
+                recv(tmprecv, rcount, rdtype, i, MCA_COLL_BASE_TAG_GATHER, comm, &status));
+            total_recv += rcount;
+        }
+    } else {
+        int peer = (cur_sg == root_sg) ? root : startr;
+        err = MCA_PML_CALL(send(sbuf, scount, sdtype, peer, MCA_COLL_BASE_TAG_GATHER,
+                                MCA_PML_BASE_SEND_STANDARD, comm));
+        return err;
+    }
+
+    /* All base ranks send to local root */
+    endn = startn + node_cnt;
+    if (endn > size) {
+        endn = size;
+    }
+    if (sg_cnt < size) {
+        int local_root = (root_node == cur_node) ? root : startn;
+        for (i = startn; i < endn; i += sg_cnt) {
+            int i_sg = i / sg_cnt;
+            if ((rank != local_root) && (rank == i) && is_base) {
+                err = MCA_PML_CALL(send(workbuf - sgap, total_recv, sdtype, local_root,
+                                        MCA_COLL_BASE_TAG_GATHER, MCA_PML_BASE_SEND_STANDARD,
+                                        comm));
+            }
+            if ((rank == local_root) && (rank != i) && (i_sg != root_sg)) {
+                size_t recv_amt = (i + sg_cnt > size) ? rcount * (size - i) : rcount * sg_cnt;
+                MPI_Aint rcv_ofst = rextent * (ptrdiff_t) (rcount * (i - startn));
+
+                err = MCA_PML_CALL(recv(wkg + (ptrdiff_t) rcv_ofst, recv_amt, rdtype, i,
+                                        MCA_COLL_BASE_TAG_GATHER, comm, &status));
+                total_recv += recv_amt;
+            }
+            if (MPI_SUCCESS != err) {
+                if (NULL != workbuf) {
+                    coll_acoll_buf_free(reserve_mem_gather, workbuf);
+                }
+                return err;
+            }
+        }
+    }
+
+    /* All local roots ranks send to root */
+    if (node_cnt < size && num_nodes > 1) {
+        for (i = 0; i < size; i += node_cnt) {
+            int i_node = i / node_cnt;
+            if ((rank != root) && (rank == i) && is_base) {
+                err = MCA_PML_CALL(send(workbuf - sgap, total_recv, sdtype, root,
+                                        MCA_COLL_BASE_TAG_GATHER, MCA_PML_BASE_SEND_STANDARD,
+                                        comm));
+            }
+            if ((rank == root) && (rank != i) && (i_node != root_node)) {
+                size_t recv_amt = (i + node_cnt > size) ? rcount * (size - i) : rcount * node_cnt;
+                MPI_Aint rcv_ofst = rextent * (ptrdiff_t) (rcount * i);
+
+                err = MCA_PML_CALL(recv((char *) rbuf + (ptrdiff_t) rcv_ofst, recv_amt, rdtype, i,
+                                        MCA_COLL_BASE_TAG_GATHER, comm, &status));
+                total_recv += recv_amt;
+            }
+            if (MPI_SUCCESS != err) {
+                if (NULL != workbuf) {
+                    coll_acoll_buf_free(reserve_mem_gather, workbuf);
+                }
+                return err;
+            }
+        }
+    }
+
+    if (NULL != workbuf) {
+        coll_acoll_buf_free(reserve_mem_gather, workbuf);
+    }
+
+    /* All done */
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mca/coll/acoll/coll_acoll_module.c b/ompi/mca/coll/acoll/coll_acoll_module.c
new file mode 100644
index 00000000000..697f47bb8d0
--- /dev/null
+++ b/ompi/mca/coll/acoll/coll_acoll_module.c
@@ -0,0 +1,213 @@
+/*
+ * Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include <stdio.h>
+
+#include "mpi.h"
+#include "ompi/mca/coll/base/base.h"
+#include "ompi/mca/coll/coll.h"
+#include "coll_acoll.h"
+
+
+static int acoll_module_enable(mca_coll_base_module_t *module,  struct ompi_communicator_t *comm);
+static int acoll_module_disable(mca_coll_base_module_t *module, struct ompi_communicator_t *comm);
+
+/*
+ * Initial query function that is invoked during MPI_INIT, allowing
+ * this component to disqualify itself if it doesn't support the
+ * required level of thread support.
+ */
+int mca_coll_acoll_init_query(bool enable_progress_threads, bool enable_mpi_threads)
+{
+    /* Nothing to do */
+    return OMPI_SUCCESS;
+}
+
+
+
+#define ACOLL_INSTALL_COLL_API(__comm, __module, __api)                                                     \
+    do                                                                                                      \
+    {                                                                                                       \
+        if (__module->super.coll_##__api)                                                                   \
+        {                                                                                                   \
+            MCA_COLL_INSTALL_API(__comm, __api, __module->super.coll_##__api, &__module->super, "acoll");  \
+        }                                                                                                   \
+    } while (0)
+
+#define ACOLL_UNINSTALL_COLL_API(__comm, __module, __api)               \
+    do                                                                  \
+    {                                                                   \
+        if (__comm->c_coll->coll_##__api##_module == &__module->super)  \
+        {                                                               \
+            MCA_COLL_INSTALL_API(__comm, __api, NULL, NULL, "acoll");   \
+        }                                                               \
+    } while (0)
+
+/*
+ * Invoked when there's a new communicator that has been created.
+ * Look at the communicator and decide which set of functions and
+ * priority we want to return.
+ */
+mca_coll_base_module_t *mca_coll_acoll_comm_query(struct ompi_communicator_t *comm, int *priority)
+{
+    mca_coll_acoll_module_t *acoll_module;
+
+    if (OMPI_COMM_IS_INTER(comm)) {
+        *priority = 0;
+        return NULL;
+    }
+    if (OMPI_COMM_IS_INTRA(comm) && ompi_comm_size(comm) < 2) {
+        *priority = 0;
+        return NULL;
+    }
+
+    acoll_module = OBJ_NEW(mca_coll_acoll_module_t);
+    if (NULL == acoll_module) {
+        return NULL;
+    }
+
+    *priority = mca_coll_acoll_priority;
+
+    /* Set topology params */
+    acoll_module->max_comms = mca_coll_acoll_max_comms;
+    acoll_module->sg_scale = mca_coll_acoll_sg_scale;
+    acoll_module->sg_size = mca_coll_acoll_sg_size;
+    acoll_module->sg_cnt = mca_coll_acoll_sg_size / mca_coll_acoll_sg_scale;
+    acoll_module->node_cnt = mca_coll_acoll_node_size;
+    if (mca_coll_acoll_sg_size == MCA_COLL_ACOLL_SG_SIZE_1) {
+        assert((acoll_module->sg_cnt == 1) || (acoll_module->sg_cnt == 2)
+               || (acoll_module->sg_cnt == 4) || (acoll_module->sg_cnt == 8));
+    }
+    if (mca_coll_acoll_sg_size == MCA_COLL_ACOLL_SG_SIZE_2) {
+        assert((acoll_module->sg_cnt == 1) || (acoll_module->sg_cnt == 2)
+               || (acoll_module->sg_cnt == 4) || (acoll_module->sg_cnt == 8)
+               || (acoll_module->sg_cnt == 16));
+    }
+
+    switch (acoll_module->sg_cnt) {
+    case 1:
+        acoll_module->log2_sg_cnt = 0;
+        break;
+    case 2:
+        acoll_module->log2_sg_cnt = 1;
+        break;
+    case 4:
+        acoll_module->log2_sg_cnt = 2;
+        break;
+    case 8:
+        acoll_module->log2_sg_cnt = 3;
+        break;
+    case 16:
+        acoll_module->log2_sg_cnt = 4;
+        break;
+    default:
+        assert(0);
+        break;
+    }
+
+    switch (acoll_module->node_cnt) {
+    case 96:
+    case 128:
+        acoll_module->log2_node_cnt = 7;
+        break;
+    case 192:
+        acoll_module->log2_node_cnt = 8;
+        break;
+    case 64:
+        acoll_module->log2_node_cnt = 6;
+        break;
+    case 32:
+        acoll_module->log2_node_cnt = 5;
+        break;
+    default:
+        assert(0);
+        break;
+    }
+
+    acoll_module->force_numa = mca_coll_acoll_force_numa;
+    acoll_module->use_dyn_rules = mca_coll_acoll_use_dynamic_rules;
+    acoll_module->use_mnode = mca_coll_acoll_mnode_enable;
+    /* Value of 0 is currently unsupported for mnode_enable */
+    acoll_module->use_mnode = 1;
+    acoll_module->use_lin0 = mca_coll_acoll_bcast_lin0;
+    acoll_module->use_lin1 = mca_coll_acoll_bcast_lin1;
+    acoll_module->use_lin2 = mca_coll_acoll_bcast_lin2;
+    acoll_module->use_socket = mca_coll_acoll_bcast_socket;
+    if (mca_coll_acoll_bcast_nonsg) {
+        acoll_module->mnode_sg_size = acoll_module->node_cnt;
+        acoll_module->mnode_log2_sg_size = acoll_module->log2_node_cnt;
+    } else {
+        acoll_module->mnode_sg_size = acoll_module->sg_cnt;
+        acoll_module->mnode_log2_sg_size = acoll_module->log2_sg_cnt;
+    }
+    acoll_module->allg_lin = mca_coll_acoll_allgather_lin;
+    acoll_module->allg_ring = mca_coll_acoll_allgather_ring_1;
+
+    /* Choose whether to use [intra|inter], and [subgroup|normal]-based
+     * algorithms. */
+    acoll_module->super.coll_module_enable = acoll_module_enable;
+    acoll_module->super.coll_module_disable = acoll_module_disable;
+
+    acoll_module->super.coll_allgather = mca_coll_acoll_allgather;
+    acoll_module->super.coll_allreduce = mca_coll_acoll_allreduce_intra;
+    acoll_module->super.coll_alltoall = mca_coll_acoll_alltoall;
+    acoll_module->super.coll_barrier = mca_coll_acoll_barrier_intra;
+    acoll_module->super.coll_bcast = mca_coll_acoll_bcast;
+    acoll_module->super.coll_gather = mca_coll_acoll_gather_intra;
+    acoll_module->super.coll_reduce = mca_coll_acoll_reduce_intra;
+
+    return &(acoll_module->super);
+}
+
+/*
+ * Init module on the communicator
+ */
+static int acoll_module_enable(mca_coll_base_module_t *module, struct ompi_communicator_t *comm)
+{
+    mca_coll_acoll_module_t *acoll_module = (mca_coll_acoll_module_t *) module;
+
+    /* prepare the placeholder for the array of request* */
+    module->base_data = OBJ_NEW(mca_coll_base_comm_t);
+    if (NULL == module->base_data) {
+        return OMPI_ERROR;
+    }
+
+   ACOLL_INSTALL_COLL_API(comm, acoll_module, allgather);
+   ACOLL_INSTALL_COLL_API(comm, acoll_module, allreduce);
+   ACOLL_INSTALL_COLL_API(comm, acoll_module, alltoall);
+   ACOLL_INSTALL_COLL_API(comm, acoll_module, barrier);
+   ACOLL_INSTALL_COLL_API(comm, acoll_module, bcast);
+   ACOLL_INSTALL_COLL_API(comm, acoll_module, gather);
+   ACOLL_INSTALL_COLL_API(comm, acoll_module, reduce);
+
+   /* Initialize k-nomial tree */
+    module->base_data->cached_kmtree = NULL;
+    module->base_data->cached_kmtree_root = -1;
+    module->base_data->cached_kmtree_radix = 4;
+
+    /* All done */
+    return OMPI_SUCCESS;
+}
+
+static int acoll_module_disable(mca_coll_base_module_t *module, struct ompi_communicator_t *comm)
+{
+    mca_coll_acoll_module_t *acoll_module = (mca_coll_acoll_module_t *) module;
+
+    ACOLL_UNINSTALL_COLL_API(comm, acoll_module, allgather);
+    ACOLL_UNINSTALL_COLL_API(comm, acoll_module, allreduce);
+    ACOLL_UNINSTALL_COLL_API(comm, acoll_module, alltoall);
+    ACOLL_UNINSTALL_COLL_API(comm, acoll_module, barrier);
+    ACOLL_UNINSTALL_COLL_API(comm, acoll_module, bcast);
+    ACOLL_UNINSTALL_COLL_API(comm, acoll_module, gather);
+    ACOLL_UNINSTALL_COLL_API(comm, acoll_module, reduce);
+
+    return OMPI_SUCCESS;
+}
diff --git a/ompi/mca/coll/acoll/coll_acoll_reduce.c b/ompi/mca/coll/acoll/coll_acoll_reduce.c
new file mode 100644
index 00000000000..32442fc4889
--- /dev/null
+++ b/ompi/mca/coll/acoll/coll_acoll_reduce.c
@@ -0,0 +1,394 @@
+/* -*- Mode: C; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "mpi.h"
+#include "ompi/constants.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_tags.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/op/op.h"
+#include "opal/util/bit_ops.h"
+#include "coll_acoll.h"
+#include "coll_acoll_utils.h"
+
+static inline int coll_reduce_decision_fixed(int comm_size, size_t msg_size)
+{
+    /* Set default to topology aware algorithm */
+    int alg = 0;
+    if (comm_size <= 8) {
+        /* Linear */
+        alg = 1;
+    } else if (msg_size <= 8192) {
+        alg = 0;
+    } else if (msg_size <= 262144) {
+        /* Binomial */
+        alg = 2;
+    } else if (msg_size <= 8388608 && comm_size < 64) {
+        alg = 1;
+    } else if (msg_size <= 8388608 && comm_size <= 128) {
+        /* In order binary */
+        alg = 3;
+    } else {
+        alg = 2;
+    }
+    return alg;
+}
+
+static inline int coll_acoll_reduce_topo(const void *sbuf, void *rbuf, size_t count,
+                                         struct ompi_datatype_t *dtype, struct ompi_op_t *op,
+                                         int root, struct ompi_communicator_t *comm,
+                                         mca_coll_base_module_t *module,
+                                         coll_acoll_subcomms_t *subc)
+{
+    int ret = MPI_SUCCESS, rank, sz;
+
+    ptrdiff_t dsize, gap = 0;
+    char *free_buffer = NULL;
+    char *pml_buffer = NULL;
+    char *tmp_rbuf = NULL;
+    char *tmp_sbuf = NULL;
+
+    mca_coll_acoll_module_t *acoll_module = (mca_coll_acoll_module_t *) module;
+    coll_acoll_reserve_mem_t *reserve_mem_rbuf_reduce = &(acoll_module->reserve_mem_s);
+
+    rank = ompi_comm_rank(comm);
+
+    tmp_sbuf = (char *) sbuf;
+    if ((MPI_IN_PLACE == sbuf) && (rank == root)) {
+        tmp_sbuf = (char *) rbuf;
+    }
+
+    int i;
+    int ind1 = MCA_COLL_ACOLL_L3CACHE;
+    int ind2 = MCA_COLL_ACOLL_LYR_NODE;
+    int is_base = rank == subc->base_rank[ind1][ind2] ? 1 : 0;
+    int bound = subc->subgrp_size;
+
+    sz = ompi_comm_size(subc->base_comm[ind1][ind2]);
+    dsize = opal_datatype_span(&dtype->super, count, &gap);
+    if (rank == root) {
+        tmp_rbuf = rbuf;
+    } else if (is_base) {
+        tmp_rbuf = (char *) coll_acoll_buf_alloc(reserve_mem_rbuf_reduce, dsize);
+        if (NULL == tmp_rbuf) {
+            return OMPI_ERR_OUT_OF_RESOURCE;
+        }
+    }
+
+    if (is_base) {
+        ret = ompi_datatype_copy_content_same_ddt(dtype, count, (char *) tmp_rbuf,
+                                                  (char *) tmp_sbuf);
+        free_buffer = (char *) malloc(dsize);
+        if (NULL == free_buffer) {
+            return OMPI_ERR_OUT_OF_RESOURCE;
+        }
+        pml_buffer = free_buffer - gap;
+    }
+
+    /* if not a local root, send the message to the local root */
+    if (!is_base) {
+        ret = MCA_PML_CALL(send(tmp_sbuf, count, dtype, subc->subgrp_root, MCA_COLL_BASE_TAG_REDUCE,
+                                MCA_PML_BASE_SEND_STANDARD, subc->subgrp_comm));
+    }
+
+    /* if local root, receive the message from other ranks within that group */
+    if (is_base) {
+        for (i = 0; i < bound; i++) {
+            if (i == subc->subgrp_root) {
+                continue;
+            }
+            ret = MCA_PML_CALL(recv(pml_buffer, count, dtype, i, MCA_COLL_BASE_TAG_REDUCE,
+                                    subc->subgrp_comm, MPI_STATUS_IGNORE));
+            ompi_op_reduce(op, pml_buffer, tmp_rbuf, count, dtype);
+        }
+    }
+    /* perform reduction at root */
+    if (is_base && (sz > 1)) {
+        if (rank != root) {
+            ret = MCA_PML_CALL(send(tmp_rbuf, count, dtype, subc->base_root[ind1][ind2],
+                                    MCA_COLL_BASE_TAG_REDUCE, MCA_PML_BASE_SEND_STANDARD,
+                                    subc->base_comm[ind1][ind2]));
+            if (ret != MPI_SUCCESS) {
+                free(pml_buffer);
+                if (NULL != tmp_rbuf) {
+                    coll_acoll_buf_free(reserve_mem_rbuf_reduce, tmp_rbuf);
+                }
+                return ret;
+            }
+        }
+        if (rank == root) {
+            for (i = 0; i < sz; i++) {
+                if (i == subc->base_root[ind1][ind2]) {
+                    continue;
+                }
+                ret = MCA_PML_CALL(recv(pml_buffer, count, dtype, i, MCA_COLL_BASE_TAG_REDUCE,
+                                        subc->base_comm[ind1][ind2], MPI_STATUS_IGNORE));
+                if (ret != MPI_SUCCESS) {
+                    free(pml_buffer);
+                    return ret;
+                }
+                ompi_op_reduce(op, pml_buffer, rbuf, count, dtype);
+            }
+        }
+    }
+
+    /* if local root, reduce at root */
+    if (is_base && (sz > 1)) {
+        free(pml_buffer);
+        if (rank != root && NULL != tmp_rbuf) {
+            coll_acoll_buf_free(reserve_mem_rbuf_reduce, tmp_rbuf);
+        }
+    }
+
+    return ret;
+}
+
+#ifdef HAVE_XPMEM_H
+static inline int mca_coll_acoll_reduce_xpmem(const void *sbuf, void *rbuf, size_t count,
+                                              struct ompi_datatype_t *dtype, struct ompi_op_t *op,
+                                              int root, struct ompi_communicator_t *comm,
+                                              mca_coll_base_module_t *module,
+                                              coll_acoll_subcomms_t *subc)
+{
+    int size;
+    size_t total_dsize, dsize;
+    ptrdiff_t gap = 0;
+
+    mca_coll_acoll_module_t *acoll_module = (mca_coll_acoll_module_t *) module;
+
+    coll_acoll_init(module, comm, subc->data, subc, 0);
+    coll_acoll_reserve_mem_t *reserve_mem_rbuf_reduce = NULL;
+    if (subc->xpmem_use_sr_buf != 0) {
+        reserve_mem_rbuf_reduce = &(acoll_module->reserve_mem_s);
+    }
+    coll_acoll_data_t *data = subc->data;
+    if (NULL == data) {
+        return -1;
+    }
+
+    size = ompi_comm_size(comm);
+    int rank = ompi_comm_rank(comm);
+    ompi_datatype_type_size(dtype, &dsize);
+    total_dsize = opal_datatype_span(&dtype->super, count, &gap);
+
+    int l1_gp_size = data->l1_gp_size;
+    int *l1_gp = data->l1_gp;
+    int *l2_gp = data->l2_gp;
+    int l2_gp_size = data->l2_gp_size;
+
+    int l1_local_rank = data->l1_local_rank;
+    int l2_local_rank = data->l2_local_rank;
+
+    char *tmp_sbuf = NULL;
+    char *tmp_rbuf = NULL;
+
+    if (0 == subc->xpmem_use_sr_buf) {
+        tmp_rbuf = (char *) data->scratch;
+        tmp_sbuf = (char *) data->scratch + (subc->xpmem_buf_size) / 2;
+        if ((MPI_IN_PLACE == sbuf) && (rank == root)) {
+            memcpy(tmp_sbuf, rbuf, total_dsize);
+        } else {
+            memcpy(tmp_sbuf, sbuf, total_dsize);
+        }
+    } else {
+        tmp_sbuf = (char *) sbuf;
+        if ((MPI_IN_PLACE == sbuf) && (rank == root)) {
+            tmp_sbuf = (char *) rbuf;
+        }
+
+        if (rank == root) {
+            tmp_rbuf = rbuf;
+        } else {
+            tmp_rbuf = (char *) coll_acoll_buf_alloc(reserve_mem_rbuf_reduce, total_dsize);
+            if (NULL == tmp_rbuf) {
+                return OMPI_ERR_OUT_OF_RESOURCE;
+            }
+        }
+    }
+    void *sbuf_vaddr[1] = {tmp_sbuf};
+    void *rbuf_vaddr[1] = {tmp_rbuf};
+
+    int ret;
+
+    ret = comm->c_coll->coll_allgather(sbuf_vaddr, sizeof(void *), MPI_BYTE, data->allshm_sbuf,
+                                       sizeof(void *), MPI_BYTE, comm,
+                                       comm->c_coll->coll_allgather_module);
+    if (ret != MPI_SUCCESS) {
+        return ret;
+    }
+    ret = comm->c_coll->coll_allgather(rbuf_vaddr, sizeof(void *), MPI_BYTE, data->allshm_rbuf,
+                                       sizeof(void *), MPI_BYTE, comm,
+                                       comm->c_coll->coll_allgather_module);
+
+    if (ret != MPI_SUCCESS) {
+        return ret;
+    }
+
+    register_and_cache(size, total_dsize, rank, data);
+
+    /* reduce to the group leader */
+    size_t chunk = count / l1_gp_size;
+    size_t my_count_size = (l1_local_rank == (l1_gp_size - 1)) ? chunk + count % l1_gp_size : chunk;
+
+    if (rank == l1_gp[0]) {
+        if (sbuf != MPI_IN_PLACE)
+            memcpy(tmp_rbuf, sbuf, my_count_size * dsize);
+        for (int i = 1; i < l1_gp_size; i++) {
+            ompi_op_reduce(op, (char *) data->xpmem_saddr[l1_gp[i]] + chunk * l1_local_rank * dsize,
+                           (char *) tmp_rbuf + chunk * l1_local_rank * dsize, my_count_size, dtype);
+        }
+    } else {
+        ompi_3buff_op_reduce(op,
+                             (char *) data->xpmem_saddr[l1_gp[0]] + chunk * l1_local_rank * dsize,
+                             (char *) tmp_sbuf + chunk * l1_local_rank * dsize,
+                             (char *) data->xpmem_raddr[l1_gp[0]] + chunk * l1_local_rank * dsize,
+                             my_count_size, dtype);
+        for (int i = 1; i < l1_gp_size; i++) {
+            if (i == l1_local_rank) {
+                continue;
+            }
+            ompi_op_reduce(op, (char *) data->xpmem_saddr[l1_gp[i]] + chunk * l1_local_rank * dsize,
+                           (char *) data->xpmem_raddr[l1_gp[0]] + chunk * l1_local_rank * dsize,
+                           my_count_size, dtype);
+        }
+    }
+    ompi_coll_base_barrier_intra_tree(comm, module);
+
+    /* perform reduce to 0 */
+    int local_size = l2_gp_size;
+    if ((rank == l1_gp[0]) && (local_size > 1)) {
+        chunk = count / local_size;
+        my_count_size = (l2_local_rank == (local_size - 1)) ? chunk + (count % local_size) : chunk;
+
+        if (0 == l2_local_rank) {
+            for (int i = 1; i < local_size; i++) {
+                ompi_op_reduce(op, (char *) data->xpmem_raddr[l2_gp[i]], (char *) tmp_rbuf,
+                               my_count_size, dtype);
+            }
+        } else {
+            for (int i = 1; i < local_size; i++) {
+                if (i == l2_local_rank) {
+                    continue;
+                }
+                ompi_op_reduce(op,
+                               (char *) data->xpmem_raddr[l2_gp[i]] + chunk * l2_local_rank * dsize,
+                               (char *) data->xpmem_raddr[0] + chunk * l2_local_rank * dsize,
+                               my_count_size, dtype);
+            }
+            ompi_op_reduce(op, (char *) tmp_rbuf + chunk * l2_local_rank * dsize,
+                           (char *) data->xpmem_raddr[0] + chunk * l2_local_rank * dsize,
+                           my_count_size, dtype);
+        }
+    }
+    ompi_coll_base_barrier_intra_tree(comm, module);
+    if (0 == subc->xpmem_use_sr_buf) {
+        if (rank == root) {
+            memcpy(rbuf, tmp_rbuf, total_dsize);
+        }
+    } else {
+        if ((rank != root) && (subc->xpmem_use_sr_buf != 0)) {
+            coll_acoll_buf_free(reserve_mem_rbuf_reduce, tmp_rbuf);
+        }
+    }
+
+    return MPI_SUCCESS;
+}
+#endif
+
+int mca_coll_acoll_reduce_intra(const void *sbuf, void *rbuf, size_t count,
+                                struct ompi_datatype_t *dtype, struct ompi_op_t *op, int root,
+                                struct ompi_communicator_t *comm, mca_coll_base_module_t *module)
+{
+    int size, alg;
+    int num_nodes, ret;
+    size_t total_dsize, dsize;
+    mca_coll_acoll_module_t *acoll_module = (mca_coll_acoll_module_t *) module;
+
+    size = ompi_comm_size(comm);
+    if (size < 4)
+        return ompi_coll_base_reduce_intra_basic_linear(sbuf, rbuf, count, dtype, op, root, comm,
+                                                        module);
+
+    /* Falling back to inorder binary for non-commutative operators to be safe */
+    if (!ompi_op_is_commute(op)) {
+        return ompi_coll_base_reduce_intra_in_order_binary(sbuf, rbuf, count, dtype, op, root, comm,
+                                                           module, 0, 0);
+    }
+    if (root != 0) { // ToDo: support non-zero root
+        return ompi_coll_base_reduce_intra_binomial(sbuf, rbuf, count, dtype, op, root, comm,
+                                                    module, 0, 0);
+    }
+
+    ompi_datatype_type_size(dtype, &dsize);
+    total_dsize = dsize * count;
+
+    alg = coll_reduce_decision_fixed(size, total_dsize);
+
+    /* Obtain the subcomms structure */
+    coll_acoll_subcomms_t *subc = NULL;
+    ret = check_and_create_subc(comm, acoll_module, &subc);
+
+    /* Fallback to knomial if subc is not obtained */
+    if (NULL == subc) {
+        return ompi_coll_base_reduce_intra_binomial(sbuf, rbuf, count, dtype, op, root, comm,
+                                                    module, 0, 0);
+    }
+
+    if (!subc->initialized || (root != subc->prev_init_root)) {
+        ret = mca_coll_acoll_comm_split_init(comm, acoll_module, subc, 0);
+        if (MPI_SUCCESS != ret) {
+            return ret;
+        }
+    }
+
+    num_nodes = subc->num_nodes;
+
+    if (1 == num_nodes) {
+        if (total_dsize < 262144) {
+            if (-1 == alg /* interaction with xpmem implementation causing issues 0*/) {
+                return coll_acoll_reduce_topo(sbuf, rbuf, count, dtype, op, root, comm, module,
+                                              subc);
+            } else if (1 == alg) {
+                return ompi_coll_base_reduce_intra_basic_linear(sbuf, rbuf, count, dtype, op, root,
+                                                                comm, module);
+            } else if (2 == alg) {
+                return ompi_coll_base_reduce_intra_binomial(sbuf, rbuf, count, dtype, op, root,
+                                                            comm, module, 0, 0);
+            } else { /*(alg == 3)*/
+                return ompi_coll_base_reduce_intra_in_order_binary(sbuf, rbuf, count, dtype, op,
+                                                                   root, comm, module, 0, 0);
+            }
+        } else {
+#ifdef HAVE_XPMEM_H
+            if ((((subc->xpmem_use_sr_buf != 0)
+                  && (acoll_module->reserve_mem_s).reserve_mem_allocate
+                  && ((acoll_module->reserve_mem_s).reserve_mem_size >= total_dsize))
+                 || ((0 == subc->xpmem_use_sr_buf) && (subc->xpmem_buf_size > 2 * total_dsize)))
+                && (subc->without_xpmem != 1)) {
+                return mca_coll_acoll_reduce_xpmem(sbuf, rbuf, count, dtype, op, root, comm,
+                                                   module, subc);
+            } else {
+                return ompi_coll_base_reduce_intra_binomial(sbuf, rbuf, count, dtype, op,
+                                                                   root, comm, module, 0, 0);
+            }
+#else
+            return ompi_coll_base_reduce_intra_binomial(sbuf, rbuf, count, dtype, op, root,
+                                                               comm, module, 0, 0);
+#endif
+        }
+    } else {
+        return ompi_coll_base_reduce_intra_binomial(sbuf, rbuf, count, dtype, op, root, comm,
+                                                    module, 0, 0);
+    }
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mca/coll/acoll/coll_acoll_utils.h b/ompi/mca/coll/acoll/coll_acoll_utils.h
new file mode 100644
index 00000000000..20b1f26df42
--- /dev/null
+++ b/ompi/mca/coll/acoll/coll_acoll_utils.h
@@ -0,0 +1,1088 @@
+/* -*- Mode: C; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2024 - 2025 Advanced Micro Devices, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "mpi.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/mca/coll/base/coll_base_functions.h"
+#include "opal/include/opal/align.h"
+
+#ifdef HAVE_XPMEM_H
+#include "opal/mca/rcache/base/base.h"
+#include <xpmem.h>
+#endif
+
+
+/* shared memory structure */
+/* first 16 * 1024 bytes (16KB) are used for the leader */
+/* next 2* 64 * comm_size bytes are used for  sync variables */
+/* next 8 * 1024 * comm_size are used for per_rank data (8KB per rank) */
+/* offsets for the shared memory region */
+#define CACHE_LINE_SIZE 64
+#define LEADER_SHM_SIZE 16384
+#define PER_RANK_SHM_SIZE 8192
+
+extern uint64_t mca_coll_acoll_xpmem_buffer_size;
+extern int mca_coll_acoll_without_xpmem;
+extern int mca_coll_acoll_xpmem_use_sr_buf;
+
+
+/* Function to allocate scratch buffer */
+static inline void *coll_acoll_buf_alloc(coll_acoll_reserve_mem_t *reserve_mem_ptr, uint64_t size)
+{
+    void *temp_ptr = NULL;
+    /* If requested size is within the pre-allocated range, use the
+       pre-allocated buffer if not in use. */
+    if ((true == reserve_mem_ptr->reserve_mem_allocate)
+        && (size <= reserve_mem_ptr->reserve_mem_size)
+        && (false == reserve_mem_ptr->reserve_mem_in_use)) {
+        if (NULL == reserve_mem_ptr->reserve_mem) {
+            reserve_mem_ptr->reserve_mem = malloc(reserve_mem_ptr->reserve_mem_size);
+        }
+        temp_ptr = reserve_mem_ptr->reserve_mem;
+
+        /* Mark the buffer as "in use" */
+        if (NULL != temp_ptr) {
+            reserve_mem_ptr->reserve_mem_in_use = true;
+        }
+    } else {
+        /* If requested size if greater than that of the pre-allocated
+           buffer or if the pre-allocated buffer is in use, create new buffer */
+        temp_ptr = malloc(size);
+    }
+
+    return temp_ptr;
+}
+
+/* Function to free scratch buffer */
+static inline void coll_acoll_buf_free(coll_acoll_reserve_mem_t *reserve_mem_ptr, void *ptr)
+{
+    /* Free the buffer only if it is not the reserved (pre-allocated) one */
+    if ((false == reserve_mem_ptr->reserve_mem_allocate)
+        || (false == reserve_mem_ptr->reserve_mem_in_use)) {
+        if (NULL != ptr) {
+            free(ptr);
+        }
+    } else if (reserve_mem_ptr->reserve_mem == ptr) {
+        /* Mark the reserved buffer as free to be used */
+        reserve_mem_ptr->reserve_mem_in_use = false;
+    }
+}
+
+/* Function to check if subcomms structure is allocated and initialized */
+static inline int check_and_create_subc(ompi_communicator_t *comm,
+                                        mca_coll_acoll_module_t *acoll_module,
+                                        coll_acoll_subcomms_t **subc_ptr)
+{
+    int cid = ompi_comm_get_local_cid(comm);
+    int num_subc = acoll_module->num_subc;
+    coll_acoll_subcomms_t *subc;
+
+    /* Return if max comms is not positive */
+    if (acoll_module->max_comms <= 0) {
+        OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
+                     "coll:acoll WARNING Set mca_coll_acoll_max_comms to positive value to use acoll!"));
+        *subc_ptr = NULL;
+        return MPI_SUCCESS;
+    }
+
+    /* Allocate memory for subcomms array */
+    if (NULL == acoll_module->subc) {
+        acoll_module->subc = malloc(sizeof(coll_acoll_subcomms_t**) * acoll_module->max_comms);
+        if (NULL == acoll_module->subc) {
+            return OMPI_ERR_OUT_OF_RESOURCE;
+        }
+    }
+
+    /* Check if subcomms structure is already created for the communicator */
+    for (int i = 0; i < num_subc; i++) {
+        if (acoll_module->subc[i]->cid == cid) {
+            *subc_ptr = acoll_module->subc[i];
+            return MPI_SUCCESS;
+        }
+    }
+
+    /* Subcomms structure is not present, create one if within limit*/
+    if (num_subc == acoll_module->max_comms) {
+        OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
+                     "coll:acoll WARNING Falling back to base since max communicators limit %d exceeded, set mca_coll_acoll_max_comms to higher value to use acoll!", acoll_module->max_comms));
+        *subc_ptr = NULL;
+        return MPI_SUCCESS;
+    }
+    *subc_ptr = (coll_acoll_subcomms_t *)malloc(sizeof(coll_acoll_subcomms_t));
+    if (NULL == *subc_ptr) {
+        return MPI_SUCCESS;
+    }
+    /* Update the module with the new subc */
+    acoll_module->subc[num_subc] = *subc_ptr;
+    acoll_module->num_subc++;
+
+    /* Initialize elements of subc */
+    subc = *subc_ptr;
+    subc->cid = cid;
+    subc->initialized = 0;
+    subc->is_root_node = 0;
+    subc->is_root_sg = 0;
+    subc->is_root_numa = 0;
+    subc->outer_grp_root = -1;
+    subc->subgrp_root = 0;
+    subc->num_nodes = 1;
+    subc->prev_init_root = -1;
+    subc->num_root_change = 0;
+    subc->numa_root = 0;
+    subc->socket_ldr_root = -1;
+    subc->orig_comm = comm;
+    subc->local_comm = NULL;
+    subc->local_r_comm = NULL;
+    subc->leader_comm = NULL;
+    subc->subgrp_comm = NULL;
+    subc->socket_comm = NULL;
+    subc->socket_ldr_comm = NULL;
+    for (int j = 0; j < MCA_COLL_ACOLL_NUM_LAYERS; j++) {
+        for (int k = 0; k < MCA_COLL_ACOLL_NUM_BASE_LYRS; k++) {
+            subc->base_comm[k][j] = NULL;
+            subc->base_root[k][j] = -1;
+        }
+        subc->local_root[j] = 0;
+    }
+
+    for (int k = 0; k < MCA_COLL_ACOLL_SPLIT_FACTOR_LIST_LEN; ++k) {
+        subc->split_comm[k] = NULL;
+    }
+
+    subc->numa_comm = NULL;
+    subc->numa_comm_ldrs = NULL;
+    subc->node_comm = NULL;
+    subc->inter_comm = NULL;
+    subc->initialized_data = false;
+    subc->initialized_shm_data = false;
+    subc->data = NULL;
+#ifdef HAVE_XPMEM_H
+    subc->xpmem_buf_size = mca_coll_acoll_xpmem_buffer_size;
+    subc->without_xpmem = mca_coll_acoll_without_xpmem;
+    subc->xpmem_use_sr_buf = mca_coll_acoll_xpmem_use_sr_buf;
+#endif
+    return MPI_SUCCESS;
+
+}
+
+/* Function to compare integer elements */
+static int compare_values(const void *ptra, const void *ptrb)
+{
+    int a = *((int *) ptra);
+    int b = *((int *) ptrb);
+
+    if (a < b) {
+        return -1;
+    } else if (a > b) {
+        return 1;
+    }
+
+    return 0;
+}
+
+/* Function to map ranks from parent communicator to sub-communicator */
+static inline int comm_grp_ranks_local(ompi_communicator_t *comm, ompi_communicator_t *local_comm,
+                                       int *is_root_node, int *local_root, int **ranks_buf,
+                                       int root)
+{
+    ompi_group_t *local_grp, *grp;
+    int local_size = ompi_comm_size(local_comm);
+    int *ranks = malloc(local_size * sizeof(int));
+    int *local_ranks = malloc(local_size * sizeof(int));
+    int i, err;
+
+    /* Create parent (comm) and sub-comm (local_comm) groups */
+    err = ompi_comm_group(comm, &grp);
+    err = ompi_comm_group(local_comm, &local_grp);
+    /* Initialize ranks for sub-communicator (local_comm) */
+    for (i = 0; i < local_size; i++) {
+        local_ranks[i] = i;
+    }
+
+    /* Translate the ranks among the 2 communicators */
+    err = ompi_group_translate_ranks(local_grp, local_size, local_ranks, grp, ranks);
+    if (ranks_buf != NULL) {
+        *ranks_buf = malloc(local_size * sizeof(int));
+        memcpy(*ranks_buf, ranks, local_size * sizeof(int));
+    }
+
+    /* Derive the 'local_root' which is the equivalent rank for 'root' of
+       'comm' in 'local_comm' */
+    for (i = 0; i < local_size; i++) {
+        if (ranks[i] == root) {
+            *is_root_node = 1;
+            *local_root = i;
+            break;
+        }
+    }
+
+    err = ompi_group_free(&grp);
+    err = ompi_group_free(&local_grp);
+    free(ranks);
+    free(local_ranks);
+
+    return err;
+}
+
+static inline int mca_coll_acoll_create_base_comm(ompi_communicator_t **parent_comm,
+                                                  coll_acoll_subcomms_t *subc, int color, int *rank,
+                                                  int *root, int base_lyr)
+{
+    int i;
+    int err;
+
+    for (i = 0; i < MCA_COLL_ACOLL_NUM_LAYERS; i++) {
+        int is_root_node = 0;
+
+        /* Create base comm */
+        err = ompi_comm_split(parent_comm[i], color, rank[i], &subc->base_comm[base_lyr][i], false);
+        if (MPI_SUCCESS != err)
+            return err;
+
+        /* Find out local rank of root in base comm */
+        err = comm_grp_ranks_local(parent_comm[i], subc->base_comm[base_lyr][i], &is_root_node,
+                                   &subc->base_root[base_lyr][i], NULL, root[i]);
+    }
+    return err;
+}
+
+static inline int mca_coll_acoll_is_adj_rank_same_sub_comm
+                            (ompi_communicator_t* sub_comm,
+                             int sub_comm_size, int* sub_comm_ranks,
+                             ompi_group_t* parent_grp,
+                             int parent_comm_size, int* parent_comm_ranks,
+                             int par_comm_rank, bool* is_adj)
+{
+    (*is_adj) = false;
+
+    ompi_group_t *sub_comm_grp;
+    int error = ompi_comm_group(sub_comm, &sub_comm_grp);
+    if (MPI_SUCCESS != error) {
+        return error;
+    }
+
+    for (int i = 0; i < sub_comm_size; ++i) {
+        sub_comm_ranks[i] = i;
+    }
+
+    /* Current rank is guaranteed to be in all the accessed subcomms. */
+    error = ompi_group_translate_ranks(sub_comm_grp, sub_comm_size, sub_comm_ranks,
+                                       parent_grp, parent_comm_ranks);
+    if (MPI_SUCCESS != error) {
+        return error;
+    }
+
+    for (int ii = 0; ii < sub_comm_size; ++ii)
+    {
+        if (((par_comm_rank + 1) % parent_comm_size) == parent_comm_ranks[ii])
+        {
+            (*is_adj) = true;
+            break;
+        }
+    }
+
+    return MPI_SUCCESS;
+}
+
+static inline int mca_coll_acoll_derive_r2r_latency
+                                (ompi_communicator_t *comm,
+                                 coll_acoll_subcomms_t *subc,
+                                 mca_coll_acoll_module_t *acoll_module)
+{
+    int size = ompi_comm_size(comm);
+    int rank = ompi_comm_rank(comm);
+    subc->r2r_dist = DIST_NODE;
+
+    coll_acoll_reserve_mem_t *rsv_mem = &(acoll_module->reserve_mem_s);
+    int* workbuf = (int *) coll_acoll_buf_alloc(rsv_mem,
+                                                2 * size * sizeof(int));
+    if (NULL == workbuf) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    int* comm_ranks = workbuf + size;
+    ompi_group_t *comm_grp;
+    int error = ompi_comm_group(comm, &comm_grp);
+    if (MPI_SUCCESS != error) { goto error_handler; }
+
+    bool is_same_l3 = false;
+    int distance = DIST_CORE; /* map-by core distance. */
+    int l3_comm_size = subc->subgrp_size;
+    error = mca_coll_acoll_is_adj_rank_same_sub_comm(subc->subgrp_comm,
+                                l3_comm_size, workbuf, comm_grp,
+                                size, comm_ranks, rank, &is_same_l3);
+    if (MPI_SUCCESS != error) { goto error_handler; }
+
+    bool is_same_numa = false;
+    if (!is_same_l3) {
+        distance = DIST_L3CACHE; /* map-by l3 distance. */
+        int numa_comm_size = ompi_comm_size(subc->numa_comm);
+        error = mca_coll_acoll_is_adj_rank_same_sub_comm(subc->numa_comm,
+                                    numa_comm_size, workbuf, comm_grp,
+                                    size, comm_ranks, rank, &is_same_numa);
+        if (MPI_SUCCESS != error) { goto error_handler; }
+    }
+
+    bool is_same_socket = false;
+    if ((!is_same_l3) && (!is_same_numa)) {
+        distance = DIST_NUMA; /* map-by numa distance. */
+        int socket_comm_size = ompi_comm_size(subc->socket_comm);
+        error = mca_coll_acoll_is_adj_rank_same_sub_comm(subc->socket_comm,
+                                    socket_comm_size, workbuf, comm_grp,
+                                    size, comm_ranks, rank, &is_same_socket);
+        if (MPI_SUCCESS != error) { goto error_handler; }
+    }
+
+    bool is_same_node = false;
+    if ((!is_same_l3) && (!is_same_numa) && (!is_same_socket)) {
+        distance = DIST_SOCKET; /* map-by socket distance. */
+        int local_comm_size = ompi_comm_size(subc->local_comm);
+        error = mca_coll_acoll_is_adj_rank_same_sub_comm(subc->local_comm,
+                                    local_comm_size, workbuf, comm_grp,
+                                    size, comm_ranks, rank, &is_same_node);
+        if (MPI_SUCCESS != error) { goto error_handler; }
+    }
+
+    if ((!is_same_l3) && (!is_same_numa) && (!is_same_socket) && (!is_same_node)) {
+        distance = DIST_NODE; /* map-by node distance. */
+    }
+
+    error = (comm)->c_coll->coll_allgather(&distance, 1, MPI_INT,
+                                           workbuf, 1, MPI_INT,
+                                           comm, &acoll_module->super);
+    if (MPI_SUCCESS != error) { goto error_handler; }
+
+    int dist_count_array[DIST_END] = {0};
+    for (int ii = 0; ii < size; ++ii) {
+        dist_count_array[workbuf[ii]] += 1;
+    }
+
+    int max_idx = DIST_CORE;
+    for (int ii = (max_idx + 1); ii < DIST_END; ++ii) {
+        if (dist_count_array[ii] > dist_count_array[max_idx]) {
+            max_idx = ii;
+        }
+    }
+    subc->r2r_dist = max_idx;
+
+    coll_acoll_buf_free(rsv_mem, workbuf);
+    return MPI_SUCCESS;
+
+error_handler:
+    coll_acoll_buf_free(rsv_mem, workbuf);
+    return error;
+}
+
+static inline int mca_coll_acoll_comm_split_init(ompi_communicator_t *comm,
+                                                 mca_coll_acoll_module_t *acoll_module,
+                                                 coll_acoll_subcomms_t *subc,
+                                                 int root)
+{
+    opal_info_t comm_info;
+    mca_coll_base_module_allreduce_fn_t coll_allreduce_org = (comm)->c_coll->coll_allreduce;
+    mca_coll_base_module_allgather_fn_t coll_allgather_org = (comm)->c_coll->coll_allgather;
+    mca_coll_base_module_bcast_fn_t coll_bcast_org = (comm)->c_coll->coll_bcast;
+    mca_coll_base_module_allreduce_fn_t coll_allreduce_loc, coll_allreduce_soc;
+    mca_coll_base_module_allgather_fn_t coll_allgather_loc, coll_allgather_soc;
+    mca_coll_base_module_bcast_fn_t coll_bcast_loc, coll_bcast_soc;
+    int err;
+    int size = ompi_comm_size(comm);
+    int rank = ompi_comm_rank(comm);
+
+    (comm)->c_coll->coll_allgather = ompi_coll_base_allgather_intra_ring;
+    (comm)->c_coll->coll_allreduce = ompi_coll_base_allreduce_intra_recursivedoubling;
+    (comm)->c_coll->coll_bcast = ompi_coll_base_bcast_intra_basic_linear;
+    if (!subc->initialized) {
+        OBJ_CONSTRUCT(&comm_info, opal_info_t);
+        opal_info_set(&comm_info, "ompi_comm_coll_preference", "libnbc,basic,^acoll");
+        /* Create node-level subcommunicator */
+        err = ompi_comm_split_type(comm, MPI_COMM_TYPE_SHARED, 0, &comm_info, &(subc->local_comm));
+        if (MPI_SUCCESS != err) {
+            return err;
+        }
+        /* Create socket-level subcommunicator */
+        err = ompi_comm_split_type(comm, OMPI_COMM_TYPE_SOCKET, 0, &comm_info,
+                                   &(subc->socket_comm));
+        if (MPI_SUCCESS != err) {
+            return err;
+        }
+        OBJ_DESTRUCT(&comm_info);
+        OBJ_CONSTRUCT(&comm_info, opal_info_t);
+        opal_info_set(&comm_info, "ompi_comm_coll_preference", "libnbc,basic,^acoll");
+        /* Create subgroup-level subcommunicator */
+        err = ompi_comm_split_type(comm, OMPI_COMM_TYPE_L3CACHE, 0, &comm_info,
+                                   &(subc->subgrp_comm));
+        if (MPI_SUCCESS != err) {
+            return err;
+        }
+        err = ompi_comm_split_type(comm, OMPI_COMM_TYPE_NUMA, 0, &comm_info, &(subc->numa_comm));
+        if (MPI_SUCCESS != err) {
+            return err;
+        }
+        subc->subgrp_size = ompi_comm_size(subc->subgrp_comm);
+        OBJ_DESTRUCT(&comm_info);
+
+        /* Derive the no. of nodes */
+        if (size == ompi_comm_size(subc->local_comm)) {
+            subc->num_nodes = 1;
+        } else {
+            int *size_list_buf = (int *) malloc(size * sizeof(int));
+            int num_nodes = 0;
+            int local_size = ompi_comm_size(subc->local_comm);
+            /* Perform allgather so that all ranks know the sizes of the nodes
+               to which all other ranks belong */
+            err = (comm)->c_coll->coll_allgather(&local_size, 1, MPI_INT, size_list_buf, 1, MPI_INT,
+                                                 comm, &acoll_module->super);
+            if (MPI_SUCCESS != err) {
+                free(size_list_buf);
+                return err;
+            }
+            /* Find the no. of nodes by counting each node only once.
+             * E.g., if there are 3 nodes with 2, 3 and 4 ranks on each node,
+             * first sort the size array so that the array elements are
+             * {2,2,3,3,3,4,4,4,4}. Read the value at the start of the array,
+             * offset the array by the read value, increment the counter,
+             * and repeat the process till end of array is reached. */
+            qsort(size_list_buf, size, sizeof(int), compare_values);
+            for (int i = 0; i < size;) {
+                int ofst = size_list_buf[i];
+                num_nodes++;
+                i += ofst;
+            }
+            subc->num_nodes = num_nodes;
+            free(size_list_buf);
+        }
+    }
+    /* Common initializations */
+    {
+        subc->outer_grp_root = -1;
+        subc->subgrp_root = 0;
+        subc->is_root_sg = 0;
+        subc->is_root_numa = 0;
+        subc->numa_root = 0;
+        subc->is_root_socket = 0;
+        subc->socket_ldr_root = -1;
+        subc->is_root_node = 0;
+
+        if (subc->initialized) {
+            if (subc->num_nodes > 1) {
+                ompi_comm_free(&(subc->leader_comm));
+                subc->leader_comm = NULL;
+            }
+            ompi_comm_free(&(subc->socket_ldr_comm));
+            subc->socket_ldr_comm = NULL;
+        }
+        for (int i = 0; i < MCA_COLL_ACOLL_NUM_LAYERS; i++) {
+            if (subc->initialized) {
+                ompi_comm_free(&(subc->base_comm[MCA_COLL_ACOLL_L3CACHE][i]));
+                subc->base_comm[MCA_COLL_ACOLL_L3CACHE][i] = NULL;
+                ompi_comm_free(&(subc->base_comm[MCA_COLL_ACOLL_NUMA][i]));
+                subc->base_comm[MCA_COLL_ACOLL_NUMA][i] = NULL;
+            }
+            subc->base_root[MCA_COLL_ACOLL_L3CACHE][i] = -1;
+            subc->base_root[MCA_COLL_ACOLL_NUMA][i] = -1;
+        }
+        /* Store original collectives for local and socket comms */
+        coll_allreduce_loc = (subc->local_comm)->c_coll->coll_allreduce;
+        coll_allgather_loc = (subc->local_comm)->c_coll->coll_allgather;
+        coll_bcast_loc = (subc->local_comm)->c_coll->coll_bcast;
+        (subc->local_comm)->c_coll->coll_allgather = ompi_coll_base_allgather_intra_ring;
+        (subc->local_comm)->c_coll->coll_allreduce
+            = ompi_coll_base_allreduce_intra_recursivedoubling;
+        (subc->local_comm)->c_coll->coll_bcast = ompi_coll_base_bcast_intra_basic_linear;
+        coll_allreduce_soc = (subc->socket_comm)->c_coll->coll_allreduce;
+        coll_allgather_soc = (subc->socket_comm)->c_coll->coll_allgather;
+        coll_bcast_soc = (subc->socket_comm)->c_coll->coll_bcast;
+        (subc->socket_comm)->c_coll->coll_allgather = ompi_coll_base_allgather_intra_ring;
+        (subc->socket_comm)->c_coll->coll_allreduce
+            = ompi_coll_base_allreduce_intra_recursivedoubling;
+        (subc->socket_comm)->c_coll->coll_bcast = ompi_coll_base_bcast_intra_basic_linear;
+    }
+
+    /* Further subcommunicators based on root */
+    int *subgrp_ranks = NULL, *numa_ranks = NULL, *socket_ranks = NULL;
+    ompi_communicator_t *parent_comm[MCA_COLL_ACOLL_NUM_LAYERS];
+    int parent_rank[MCA_COLL_ACOLL_NUM_LAYERS];
+    if (subc->num_nodes > 1) { /* Multinode case */
+        int local_rank = ompi_comm_rank(subc->local_comm);
+        int color = MPI_UNDEFINED;
+        int is_root_node = 0, is_root_socket = 0;
+        int local_root = 0;
+
+        /* Initializations */
+        subc->local_root[MCA_COLL_ACOLL_LYR_NODE] = 0;
+        subc->local_root[MCA_COLL_ACOLL_LYR_SOCKET] = 0;
+
+        /* Find out the local rank of root */
+        err = comm_grp_ranks_local(comm, subc->local_comm, &subc->is_root_node,
+                                   &subc->local_root[MCA_COLL_ACOLL_LYR_NODE], NULL, root);
+
+        /* Create subcommunicator with leader ranks */
+        color = 1;
+        if (!subc->is_root_node && (0 == local_rank)) {
+            color = 0;
+        }
+        if (rank == root) {
+            color = 0;
+        }
+        err = ompi_comm_split(comm, color, rank, &subc->leader_comm, false);
+        if (MPI_SUCCESS != err) {
+            return err;
+        }
+
+        /* Find out local rank of root in leader comm */
+        err = comm_grp_ranks_local(comm, subc->leader_comm, &is_root_node, &subc->outer_grp_root,
+                                   NULL, root);
+
+        /* Find out local rank of root in socket comm */
+        if (subc->is_root_node) {
+            local_root = subc->local_root[MCA_COLL_ACOLL_LYR_NODE];
+        }
+        err = comm_grp_ranks_local(subc->local_comm, subc->socket_comm, &subc->is_root_socket,
+                                   &subc->local_root[MCA_COLL_ACOLL_LYR_SOCKET], &socket_ranks,
+                                   local_root);
+
+        /* Create subcommunicator with socket leaders */
+        subc->socket_rank = 1 == subc->is_root_socket ? local_root : socket_ranks[0];
+        color = local_rank == subc->socket_rank ? 0 : 1;
+        err = ompi_comm_split(comm, color, rank, &subc->socket_ldr_comm, false);
+        if (MPI_SUCCESS != err)
+            return err;
+
+        /* Find out local rank of root in socket leader comm */
+        err = comm_grp_ranks_local(comm, subc->socket_ldr_comm, &is_root_socket,
+                                   &subc->socket_ldr_root, NULL, root);
+
+        /* Find out local rank of root in subgroup comm */
+        err = comm_grp_ranks_local(subc->local_comm, subc->subgrp_comm, &subc->is_root_sg,
+                                   &subc->subgrp_root, &subgrp_ranks, local_root);
+
+        subc->base_rank[MCA_COLL_ACOLL_L3CACHE][MCA_COLL_ACOLL_LYR_NODE] =
+                1 == subc->is_root_sg ? local_root : subgrp_ranks[0];
+        /* Find out socket rank of root in subgroup comm */
+        int tmp_root;
+        err = comm_grp_ranks_local(subc->socket_comm, subc->subgrp_comm, &subc->is_root_sg,
+                                   &tmp_root, &subgrp_ranks,
+                                   subc->local_root[MCA_COLL_ACOLL_LYR_SOCKET]);
+        subc->base_rank[MCA_COLL_ACOLL_L3CACHE][MCA_COLL_ACOLL_LYR_SOCKET] =
+            1 == subc->is_root_sg ? subc->local_root[MCA_COLL_ACOLL_LYR_SOCKET] : subgrp_ranks[0];
+
+        /* Create subcommunicator with base ranks */
+        color = local_rank == subc->base_rank[MCA_COLL_ACOLL_L3CACHE][MCA_COLL_ACOLL_LYR_NODE] ? 0 : 1;
+        parent_comm[MCA_COLL_ACOLL_LYR_NODE] = subc->local_comm;
+        parent_comm[MCA_COLL_ACOLL_LYR_SOCKET] = subc->socket_comm;
+        parent_rank[MCA_COLL_ACOLL_LYR_NODE] = local_rank;
+        parent_rank[MCA_COLL_ACOLL_LYR_SOCKET] = ompi_comm_rank(subc->socket_comm);
+        err = mca_coll_acoll_create_base_comm(parent_comm, subc, color, parent_rank,
+                                              subc->local_root, MCA_COLL_ACOLL_L3CACHE);
+
+        /* Find out local rank of root in numa comm */
+        err = comm_grp_ranks_local(subc->local_comm, subc->numa_comm, &subc->is_root_numa,
+                                   &subc->numa_root, &numa_ranks, local_root);
+
+        subc->base_rank[MCA_COLL_ACOLL_NUMA][MCA_COLL_ACOLL_LYR_NODE] =
+                1 == subc->is_root_numa ? local_root : numa_ranks[0];
+        /* Find out socket rank of root in numa comm */
+        err = comm_grp_ranks_local(subc->socket_comm, subc->numa_comm, &subc->is_root_numa,
+                                   &tmp_root, &numa_ranks,
+                                   subc->local_root[MCA_COLL_ACOLL_LYR_SOCKET]);
+        subc->base_rank[MCA_COLL_ACOLL_NUMA][MCA_COLL_ACOLL_LYR_SOCKET] =
+            1 == subc->is_root_numa ? subc->local_root[MCA_COLL_ACOLL_LYR_SOCKET] : numa_ranks[0];
+
+        color = local_rank == subc->base_rank[MCA_COLL_ACOLL_NUMA][MCA_COLL_ACOLL_LYR_NODE] ? 0 : 1;
+        err = mca_coll_acoll_create_base_comm(parent_comm, subc, color, parent_rank,
+                                              subc->local_root, MCA_COLL_ACOLL_NUMA);
+    } else {
+        /* Intra node case */
+        int color;
+        int is_root_socket = 0;
+
+        /* Initializations */
+        subc->local_root[MCA_COLL_ACOLL_LYR_NODE] = root;
+        subc->local_root[MCA_COLL_ACOLL_LYR_SOCKET] = 0;
+
+        /* Find out local rank of root in socket comm */
+        err = comm_grp_ranks_local(comm, subc->socket_comm, &subc->is_root_socket,
+                                   &subc->local_root[MCA_COLL_ACOLL_LYR_SOCKET], &socket_ranks,
+                                   root);
+
+        /* Create subcommunicator with socket leaders */
+        subc->socket_rank = 1 == subc->is_root_socket ? root : socket_ranks[0];
+        color = rank == subc->socket_rank ? 0 : 1;
+        err = ompi_comm_split(comm, color, rank, &subc->socket_ldr_comm, false);
+        if (MPI_SUCCESS != err) {
+            return err;
+        }
+
+        /* Find out local rank of root in socket leader comm */
+        err = comm_grp_ranks_local(comm, subc->socket_ldr_comm, &is_root_socket,
+                                   &subc->socket_ldr_root, NULL, root);
+
+        /* Find out local rank of root in subgroup comm */
+        err = comm_grp_ranks_local(comm, subc->subgrp_comm, &subc->is_root_sg, &subc->subgrp_root,
+                                   &subgrp_ranks, root);
+
+        subc->base_rank[MCA_COLL_ACOLL_L3CACHE][MCA_COLL_ACOLL_LYR_NODE] =
+                        1 == subc->is_root_sg ? root : subgrp_ranks[0];
+        /* Find out socket rank of root in subgroup comm */
+        int tmp_root;
+        err = comm_grp_ranks_local(subc->socket_comm, subc->subgrp_comm, &subc->is_root_sg,
+                                   &tmp_root, &subgrp_ranks,
+                                   subc->local_root[MCA_COLL_ACOLL_LYR_SOCKET]);
+        subc->base_rank[MCA_COLL_ACOLL_L3CACHE][MCA_COLL_ACOLL_LYR_SOCKET] =
+            1 == subc->is_root_sg ? subc->local_root[MCA_COLL_ACOLL_LYR_SOCKET] : subgrp_ranks[0];
+
+        /* Create subcommunicator with base ranks */
+        color = rank == subc->base_rank[MCA_COLL_ACOLL_L3CACHE][MCA_COLL_ACOLL_LYR_NODE] ? 0 : 1;
+        parent_comm[MCA_COLL_ACOLL_LYR_NODE] = subc->local_comm;
+        parent_comm[MCA_COLL_ACOLL_LYR_SOCKET] = subc->socket_comm;
+        parent_rank[MCA_COLL_ACOLL_LYR_NODE] = rank;
+        parent_rank[MCA_COLL_ACOLL_LYR_SOCKET] = ompi_comm_rank(subc->socket_comm);
+        err = mca_coll_acoll_create_base_comm(parent_comm, subc, color, parent_rank, subc->local_root,
+                                              MCA_COLL_ACOLL_L3CACHE);
+
+        int numa_rank;
+        numa_rank = ompi_comm_rank(subc->numa_comm);
+        color = (0 == numa_rank) ? 0 : 1;
+        err = ompi_comm_split(subc->local_comm, color, rank, &subc->numa_comm_ldrs, false);
+
+        /* Find out local rank of root in numa comm */
+        err = comm_grp_ranks_local(comm, subc->numa_comm, &subc->is_root_numa, &subc->numa_root,
+                                   &numa_ranks, root);
+
+        subc->base_rank[MCA_COLL_ACOLL_NUMA][MCA_COLL_ACOLL_LYR_NODE] =
+                            1 == subc->is_root_numa ? root : numa_ranks[0];
+        /* Find out socket rank of root in numa comm */
+        err = comm_grp_ranks_local(subc->socket_comm, subc->numa_comm, &subc->is_root_numa,
+                                   &tmp_root, &numa_ranks,
+                                   subc->local_root[MCA_COLL_ACOLL_LYR_SOCKET]);
+        subc->base_rank[MCA_COLL_ACOLL_NUMA][MCA_COLL_ACOLL_LYR_SOCKET] =
+            1 == subc->is_root_numa ? subc->local_root[MCA_COLL_ACOLL_LYR_SOCKET] : numa_ranks[0];
+
+        color = rank == subc->base_rank[MCA_COLL_ACOLL_NUMA][MCA_COLL_ACOLL_LYR_NODE] ? 0 : 1;
+        err = mca_coll_acoll_create_base_comm(parent_comm, subc, color, parent_rank, subc->local_root,
+                                              MCA_COLL_ACOLL_NUMA);
+    }
+
+    if (socket_ranks != NULL) {
+        free(socket_ranks);
+        socket_ranks = NULL;
+    }
+    if (subgrp_ranks != NULL) {
+        free(subgrp_ranks);
+        subgrp_ranks = NULL;
+    }
+    if (numa_ranks != NULL) {
+        free(numa_ranks);
+        numa_ranks = NULL;
+    }
+
+    /* Restore originals for local and socket comms */
+    (subc->local_comm)->c_coll->coll_allreduce = coll_allreduce_loc;
+    (subc->local_comm)->c_coll->coll_allgather = coll_allgather_loc;
+    (subc->local_comm)->c_coll->coll_bcast = coll_bcast_loc;
+    (subc->socket_comm)->c_coll->coll_allreduce = coll_allreduce_soc;
+    (subc->socket_comm)->c_coll->coll_allgather = coll_allgather_soc;
+    (subc->socket_comm)->c_coll->coll_bcast = coll_bcast_soc;
+
+    /* For collectives where order is important (like gather, allgather),
+     * split based on ranks. This is optimal for global communicators with
+     * equal split among nodes, but suboptimal for other cases.
+     */
+    if (!subc->initialized) {
+        if (subc->num_nodes > 1) {
+            int node_size = (size + subc->num_nodes - 1) / subc->num_nodes;
+            int color = rank / node_size;
+            err = ompi_comm_split(comm, color, rank, &subc->local_r_comm, false);
+            if (MPI_SUCCESS != err) {
+                return err;
+            }
+        }
+
+        err = mca_coll_acoll_derive_r2r_latency(comm, subc, acoll_module);
+        if (MPI_SUCCESS != err) {
+            return err;
+        }
+
+        const int split_factor_list_len = MCA_COLL_ACOLL_SPLIT_FACTOR_LIST_LEN;
+        const int split_factor_list[MCA_COLL_ACOLL_SPLIT_FACTOR_LIST_LEN] =
+                    MCA_COLL_ACOLL_SPLIT_FACTOR_LIST;
+        for (int ii = 0; ii < split_factor_list_len; ++ii) {
+            int split_comm_color = rank % split_factor_list[ii];
+
+            /* If comm size is not a perfect multiple of split factor, then
+             * unless comm size % split factor <= 1, the split_comm
+             * for split factor 2 is used.*/
+            if ((0 != (size % split_factor_list[ii])) &&
+                (rank >= (size - (size % split_factor_list[ii])))) {
+                split_comm_color = split_factor_list[ii];
+            }
+            err = ompi_comm_split(comm, split_comm_color, rank,
+                            &subc->split_comm[ii], false);
+            if (MPI_SUCCESS != err) {
+                return err;
+            }
+        }
+
+        subc->derived_node_size = (size + subc->num_nodes - 1) / subc->num_nodes;
+    }
+
+    /* Restore originals */
+    (comm)->c_coll->coll_allreduce = coll_allreduce_org;
+    (comm)->c_coll->coll_allgather = coll_allgather_org;
+    (comm)->c_coll->coll_bcast = coll_bcast_org;
+
+    /* Init done */
+    subc->initialized = 1;
+    if (root != subc->prev_init_root) {
+        subc->num_root_change++;
+    }
+    subc->prev_init_root = root;
+
+    return err;
+}
+
+#ifdef HAVE_XPMEM_H
+static inline int mca_coll_acoll_xpmem_register(void *xpmem_apid, void *base, size_t size,
+                                                mca_rcache_base_registration_t *reg)
+{
+    struct xpmem_addr xpmem_addr;
+    xpmem_addr.apid = *((xpmem_apid_t *) xpmem_apid);
+    xpmem_addr.offset = (uintptr_t) base;
+    struct acoll_xpmem_rcache_reg_t *xpmem_reg = (struct acoll_xpmem_rcache_reg_t *) reg;
+    xpmem_reg->xpmem_vaddr = xpmem_attach(xpmem_addr, size, NULL);
+
+    if ((void *) -1 == xpmem_reg->xpmem_vaddr) {
+        return -1;
+    }
+    return 0;
+}
+
+static inline int mca_coll_acoll_xpmem_deregister(void *xpmem_apid,
+                                                  mca_rcache_base_registration_t *reg)
+{
+    int status = xpmem_detach(((struct acoll_xpmem_rcache_reg_t *) reg)->xpmem_vaddr);
+    return status;
+}
+#endif
+
+static inline int coll_acoll_init(mca_coll_base_module_t *module, ompi_communicator_t *comm,
+                                  coll_acoll_data_t *data, coll_acoll_subcomms_t *subc, int root)
+{
+    int size, ret = 0, rank, line;
+
+    int cid = ompi_comm_get_local_cid(comm);
+    if (subc->initialized_data) {
+        return ret;
+    }
+    data = (coll_acoll_data_t *) malloc(sizeof(coll_acoll_data_t));
+    if (NULL == data) {
+        line = __LINE__;
+        ret = OMPI_ERR_OUT_OF_RESOURCE;
+        goto error_hndl;
+    }
+    size = ompi_comm_size(comm);
+    rank = ompi_comm_rank(comm);
+    data->comm_size = size;
+
+#ifdef HAVE_XPMEM_H
+    if (0 == subc->xpmem_use_sr_buf) {
+        data->scratch = (char *) malloc(subc->xpmem_buf_size);
+        if (NULL == data->scratch) {
+            line = __LINE__;
+            ret = OMPI_ERR_OUT_OF_RESOURCE;
+            goto error_hndl;
+        }
+    } else {
+        data->scratch = NULL;
+    }
+
+    xpmem_segid_t seg_id;
+    data->allseg_id = (xpmem_segid_t *) malloc(sizeof(xpmem_segid_t) * size);
+    if (NULL == data->allseg_id) {
+        line = __LINE__;
+        ret = OMPI_ERR_OUT_OF_RESOURCE;
+        goto error_hndl;
+    }
+    data->all_apid = (xpmem_apid_t *) malloc(sizeof(xpmem_apid_t) * size);
+    if (NULL == data->all_apid) {
+        line = __LINE__;
+        ret = OMPI_ERR_OUT_OF_RESOURCE;
+        goto error_hndl;
+    }
+    data->allshm_sbuf = (void **) malloc(sizeof(void *) * size);
+    if (NULL == data->allshm_sbuf) {
+        line = __LINE__;
+        ret = OMPI_ERR_OUT_OF_RESOURCE;
+        goto error_hndl;
+    }
+    data->allshm_rbuf = (void **) malloc(sizeof(void *) * size);
+    if (NULL == data->allshm_rbuf) {
+        line = __LINE__;
+        ret = OMPI_ERR_OUT_OF_RESOURCE;
+        goto error_hndl;
+    }
+    data->xpmem_saddr = (void **) malloc(sizeof(void *) * size);
+    if (NULL == data->xpmem_saddr) {
+        line = __LINE__;
+        ret = OMPI_ERR_OUT_OF_RESOURCE;
+        goto error_hndl;
+    }
+    data->xpmem_raddr = (void **) malloc(sizeof(void *) * size);
+    if (NULL == data->xpmem_raddr) {
+        line = __LINE__;
+        ret = OMPI_ERR_OUT_OF_RESOURCE;
+        goto error_hndl;
+    }
+    data->rcache = (mca_rcache_base_module_t **) malloc(sizeof(mca_rcache_base_module_t *) * size);
+    if (NULL == data->rcache) {
+        line = __LINE__;
+        ret = OMPI_ERR_OUT_OF_RESOURCE;
+        goto error_hndl;
+    }
+    data->xpmem_reg_tracker_ht = NULL;
+    data->xpmem_reg_tracker_ht = (opal_hash_table_t **) malloc(sizeof(opal_hash_table_t*) * size);
+    if (NULL == data->xpmem_reg_tracker_ht) {
+        line = __LINE__;
+        ret = OMPI_ERR_OUT_OF_RESOURCE;
+        goto error_hndl;
+    }
+
+    seg_id = xpmem_make(0, XPMEM_MAXADDR_SIZE, XPMEM_PERMIT_MODE, (void *) 0666);
+    if (-1 == seg_id) {
+        line = __LINE__;
+        ret = -1;
+        goto error_hndl;
+    }
+
+    ret = comm->c_coll->coll_allgather(&seg_id, sizeof(xpmem_segid_t), MPI_BYTE, data->allseg_id,
+                                       sizeof(xpmem_segid_t), MPI_BYTE, comm,
+                                       comm->c_coll->coll_allgather_module);
+
+    /* Assuming the length of rcache name is less than 50 characters */
+    char rc_name[50];
+    for (int i = 0; i < size; i++) {
+        if (rank != i) {
+            data->all_apid[i] = xpmem_get(data->allseg_id[i], XPMEM_RDWR, XPMEM_PERMIT_MODE,
+                                          (void *) 0666);
+            if (-1 == data->all_apid[i]) {
+                line = __LINE__;
+                ret = -1;
+                goto error_hndl;
+            }
+            if (-1 == data->all_apid[i]) {
+                line = __LINE__;
+                ret = -1;
+                goto error_hndl;
+            }
+            sprintf(rc_name, "acoll_%d_%d_%d", cid, rank, i);
+            mca_rcache_base_resources_t rcache_element
+                = {.cache_name = rc_name,
+                   .reg_data = &data->all_apid[i],
+                   .sizeof_reg = sizeof(struct acoll_xpmem_rcache_reg_t),
+                   .register_mem = mca_coll_acoll_xpmem_register,
+                   .deregister_mem = mca_coll_acoll_xpmem_deregister};
+
+            data->rcache[i] = mca_rcache_base_module_create("grdma", NULL, &rcache_element);
+            if (NULL == data->rcache[i]) {
+                ret = -1;
+                line = __LINE__;
+                goto error_hndl;
+            }
+            data->xpmem_reg_tracker_ht[i] = OBJ_NEW(opal_hash_table_t);
+            opal_hash_table_init(data->xpmem_reg_tracker_ht[i], 2048);
+        }
+    }
+#endif
+
+    /* temporary variables */
+    int tmp1, tmp2, tmp3 = root;
+    comm_grp_ranks_local(comm, subc->numa_comm, &tmp1, &tmp2, &data->l1_gp, tmp3);
+    data->l1_gp_size = ompi_comm_size(subc->numa_comm);
+    data->l1_local_rank = ompi_comm_rank(subc->numa_comm);
+
+    comm_grp_ranks_local(comm, subc->numa_comm_ldrs, &tmp1, &tmp2, &data->l2_gp, tmp3);
+    data->l2_gp_size = ompi_comm_size(subc->numa_comm_ldrs);
+    data->l2_local_rank = ompi_comm_rank(subc->numa_comm_ldrs);
+    data->offset[0] = LEADER_SHM_SIZE;
+    data->offset[1] = data->offset[0] + size * CACHE_LINE_SIZE;
+    data->offset[2] = data->offset[1] + size * CACHE_LINE_SIZE;
+    data->offset[3] = data->offset[2] + rank * PER_RANK_SHM_SIZE;
+    data->allshmseg_id = (opal_shmem_ds_t *) malloc(sizeof(opal_shmem_ds_t) * size);
+    data->allshmmmap_sbuf = (void **) malloc(sizeof(void *) * size);
+    data->sync[0] = 0;
+    data->sync[1] = 0;
+    char *shfn;
+
+    /* Only the leaders need to allocate shared memory */
+    /* remaining ranks move their data into their leader's shm */
+    if (data->l1_gp[0] == rank) {
+        subc->initialized_shm_data = true;
+        ret = asprintf(&shfn, "/dev/shm/acoll_coll_shmem_seg.%u.%x.%d:%d-%d", geteuid(),
+                       OPAL_PROC_MY_NAME.jobid, ompi_comm_rank(MPI_COMM_WORLD),
+                       ompi_comm_get_local_cid(comm), ompi_comm_size(comm));
+    }
+
+    if (ret < 0) {
+        line = __LINE__;
+        goto error_hndl;
+    }
+
+    opal_shmem_ds_t seg_ds;
+    if (data->l1_gp[0] == rank) {
+        /* Assuming cacheline size is 64 */
+        long memsize
+            = (LEADER_SHM_SIZE /* scratch leader */ + CACHE_LINE_SIZE * size /* sync variables l1 group*/
+               + CACHE_LINE_SIZE * size /* sync variables l2 group*/ + PER_RANK_SHM_SIZE * size /*data from ranks*/);
+        ret = opal_shmem_segment_create(&seg_ds, shfn, memsize);
+        free(shfn);
+    }
+
+    if (ret != OPAL_SUCCESS) {
+        opal_output_verbose(MCA_BASE_VERBOSE_ERROR, ompi_coll_base_framework.framework_output,
+                            "coll:acoll: Error: Could not create shared memory segment");
+        line = __LINE__;
+        goto error_hndl;
+    }
+
+    ret = comm->c_coll->coll_allgather(&seg_ds, sizeof(opal_shmem_ds_t), MPI_BYTE,
+                                       data->allshmseg_id, sizeof(opal_shmem_ds_t), MPI_BYTE, comm,
+                                       comm->c_coll->coll_allgather_module);
+
+    if (data->l1_gp[0] != rank) {
+        data->allshmmmap_sbuf[data->l1_gp[0]] = opal_shmem_segment_attach(
+            &data->allshmseg_id[data->l1_gp[0]]);
+    } else {
+        for (int i = 0; i < data->l2_gp_size; i++) {
+            data->allshmmmap_sbuf[data->l2_gp[i]] = opal_shmem_segment_attach(
+                &data->allshmseg_id[data->l2_gp[i]]);
+        }
+    }
+
+    data->allshmmmap_sbuf[root] = opal_shmem_segment_attach(&data->allshmseg_id[0]);
+
+    int offset = LEADER_SHM_SIZE;
+    memset(((char *) data->allshmmmap_sbuf[data->l1_gp[0]]) + offset + CACHE_LINE_SIZE * rank, 0, CACHE_LINE_SIZE);
+    if (data->l1_gp[0] == rank) {
+        memset(((char *) data->allshmmmap_sbuf[data->l2_gp[0]]) + (offset + CACHE_LINE_SIZE * size) + CACHE_LINE_SIZE * rank,
+               0, CACHE_LINE_SIZE);
+    }
+
+    subc->initialized_data = true;
+    subc->data = data;
+    ompi_coll_base_barrier_intra_tree(comm, module);
+
+    return MPI_SUCCESS;
+error_hndl:
+    (void) line;
+    if (NULL != data) {
+#ifdef HAVE_XPMEM_H
+        free(data->allseg_id);
+        data->allseg_id = NULL;
+        free(data->all_apid);
+        data->all_apid = NULL;
+        free(data->allshm_sbuf);
+        data->allshm_sbuf = NULL;
+        free(data->allshm_rbuf);
+        data->allshm_rbuf = NULL;
+        free(data->xpmem_saddr);
+        data->xpmem_saddr = NULL;
+        free(data->xpmem_raddr);
+        data->xpmem_raddr = NULL;
+        free(data->xpmem_reg_tracker_ht);
+        data->xpmem_reg_tracker_ht = NULL;
+        free(data->rcache);
+        data->rcache = NULL;
+        free(data->scratch);
+        data->scratch = NULL;
+#endif
+        free(data->allshmseg_id);
+        data->allshmseg_id = NULL;
+        free(data->allshmmmap_sbuf);
+        data->allshmmmap_sbuf = NULL;
+        free(data->l1_gp);
+        data->l1_gp = NULL;
+        free(data->l2_gp);
+        data->l2_gp = NULL;
+        free(data);
+        data = NULL;
+    }
+    return ret;
+}
+
+#ifdef HAVE_XPMEM_H
+static inline void update_rcache_reg_hashtable_entry
+                            (struct acoll_xpmem_rcache_reg_t *reg,
+                             opal_hash_table_t* ht)
+{
+    // Converting pointer to uint64 to use as key.
+    uint64_t key = (uint64_t)reg;
+    // Converting uint64_t to pointer type to use for value.
+    uint64_t value = 1;
+    int ht_ret = opal_hash_table_get_value_uint64(ht, key, (void**)(&value));
+
+    if (OPAL_ERR_NOT_FOUND == ht_ret) {
+        value = 1;
+        opal_hash_table_set_value_uint64(ht, key, (void*)(value));
+    } else if (OPAL_SUCCESS == ht_ret) {
+        value += 1;
+        opal_hash_table_set_value_uint64(ht, key, (void*)(value));
+    }
+}
+
+static inline void register_and_cache(int size, size_t total_dsize, int rank,
+                                      coll_acoll_data_t *data)
+{
+    uintptr_t base, bound;
+    for (int i = 0; i < size; i++) {
+        if (rank != i) {
+            mca_rcache_base_module_t *rcache_i = data->rcache[i];
+            int access_flags = 0;
+            struct acoll_xpmem_rcache_reg_t *sbuf_reg = NULL, *rbuf_reg = NULL;
+            base = OPAL_DOWN_ALIGN((uintptr_t) data->allshm_sbuf[i], 4096, uintptr_t);
+            bound = OPAL_ALIGN((uintptr_t) data->allshm_sbuf[i] + total_dsize, 4096, uintptr_t);
+            int ret = rcache_i->rcache_register(rcache_i, (void *) base, bound - base, access_flags,
+                                                MCA_RCACHE_ACCESS_ANY,
+                                                (mca_rcache_base_registration_t **) &sbuf_reg);
+
+            if (ret != 0) {
+                sbuf_reg = NULL;
+                return;
+            }
+            update_rcache_reg_hashtable_entry(sbuf_reg, data->xpmem_reg_tracker_ht[i]);
+
+            data->xpmem_saddr[i] = (void *) ((uintptr_t) sbuf_reg->xpmem_vaddr
+                                             + ((uintptr_t) data->allshm_sbuf[i]
+                                                - (uintptr_t) sbuf_reg->base.base));
+
+            base = OPAL_DOWN_ALIGN((uintptr_t) data->allshm_rbuf[i], 4096, uintptr_t);
+            bound = OPAL_ALIGN((uintptr_t) data->allshm_rbuf[i] + total_dsize, 4096, uintptr_t);
+            ret = rcache_i->rcache_register(rcache_i, (void *) base, bound - base, access_flags,
+                                            MCA_RCACHE_ACCESS_ANY,
+                                            (mca_rcache_base_registration_t **) &rbuf_reg);
+
+            if (ret != 0) {
+                rbuf_reg = NULL;
+                return;
+            }
+            update_rcache_reg_hashtable_entry(rbuf_reg, data->xpmem_reg_tracker_ht[i]);
+
+            data->xpmem_raddr[i] = (void *) ((uintptr_t) rbuf_reg->xpmem_vaddr
+                                             + ((uintptr_t) data->allshm_rbuf[i]
+                                                - (uintptr_t) rbuf_reg->base.base));
+        } else {
+            data->xpmem_saddr[i] = data->allshm_sbuf[i];
+            data->xpmem_raddr[i] = data->allshm_rbuf[i];
+        }
+    }
+}
+#endif
diff --git a/ompi/mca/coll/acoll/configure.m4 b/ompi/mca/coll/acoll/configure.m4
new file mode 100644
index 00000000000..339b34c567c
--- /dev/null
+++ b/ompi/mca/coll/acoll/configure.m4
@@ -0,0 +1,18 @@
+#
+# Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+AC_DEFUN([MCA_ompi_coll_acoll_CONFIG],[
+        AC_CONFIG_FILES([ompi/mca/coll/acoll/Makefile])
+
+        OPAL_CHECK_XPMEM([coll_acoll], [should_build=1], [should_build=1])
+
+        AC_SUBST([coll_acoll_CPPFLAGS])
+        AC_SUBST([coll_acoll_LDFLAGS])
+        AC_SUBST([coll_acoll_LIBS])
+])dnl
diff --git a/ompi/mca/coll/acoll/owner.txt b/ompi/mca/coll/acoll/owner.txt
new file mode 100644
index 00000000000..6bd0a386110
--- /dev/null
+++ b/ompi/mca/coll/acoll/owner.txt
@@ -0,0 +1,7 @@
+#
+# owner/status file
+# owner: institution that is responsible for this package
+# status: e.g. active, maintenance, unmaintained
+#
+owner: AMD
+status: active
\ No newline at end of file
diff --git a/ompi/mca/coll/adapt/coll_adapt.h b/ompi/mca/coll/adapt/coll_adapt.h
index 4da6fe775d4..5bd1623f266 100644
--- a/ompi/mca/coll/adapt/coll_adapt.h
+++ b/ompi/mca/coll/adapt/coll_adapt.h
@@ -44,7 +44,7 @@ typedef enum {
  */
 typedef struct mca_coll_adapt_component_t {
     /* Base coll component */
-    mca_coll_base_component_2_4_0_t super;
+    mca_coll_base_component_3_0_0_t super;
 
     /* MCA parameter: Priority of this component */
     int adapt_priority;
@@ -133,7 +133,7 @@ struct mca_coll_adapt_module_t {
 OBJ_CLASS_DECLARATION(mca_coll_adapt_module_t);
 
 /* Global component instance */
-OMPI_MODULE_DECLSPEC extern mca_coll_adapt_component_t mca_coll_adapt_component;
+OMPI_DECLSPEC extern mca_coll_adapt_component_t mca_coll_adapt_component;
 
 /* ADAPT module functions */
 int ompi_coll_adapt_init_query(bool enable_progress_threads, bool enable_mpi_threads);
diff --git a/ompi/mca/coll/adapt/coll_adapt_bcast.c b/ompi/mca/coll/adapt/coll_adapt_bcast.c
index 9cfebd97859..df39537e81b 100644
--- a/ompi/mca/coll/adapt/coll_adapt_bcast.c
+++ b/ompi/mca/coll/adapt/coll_adapt_bcast.c
@@ -12,7 +12,7 @@
 #include "coll_adapt.h"
 #include "coll_adapt_algorithms.h"
 
-int ompi_coll_adapt_bcast(void *buff, int count, struct ompi_datatype_t *datatype, int root,
+int ompi_coll_adapt_bcast(void *buff, size_t count, struct ompi_datatype_t *datatype, int root,
                          struct ompi_communicator_t *comm, mca_coll_base_module_t * module)
 {
     ompi_request_t *request = NULL;
diff --git a/ompi/mca/coll/adapt/coll_adapt_component.c b/ompi/mca/coll/adapt/coll_adapt_component.c
index 5ca59238cfb..3d7d7e16cbe 100644
--- a/ompi/mca/coll/adapt/coll_adapt_component.c
+++ b/ompi/mca/coll/adapt/coll_adapt_component.c
@@ -2,6 +2,7 @@
  * Copyright (c) 2014-2020 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2024      NVIDIA CORPORATION. All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -41,7 +42,7 @@ mca_coll_adapt_component_t mca_coll_adapt_component = {
         /* First, the mca_component_t struct containing meta
            information about the component itself */
         .collm_version = {
-            MCA_COLL_BASE_VERSION_2_4_0,
+            MCA_COLL_BASE_VERSION_3_0_0,
 
             /* Component name and version */
             .mca_component_name = "adapt",
@@ -114,39 +115,39 @@ static int adapt_register(void)
        we should have a high priority */
     cs->adapt_priority = 0;
     (void) mca_base_component_var_register(c, "priority", "Priority of the adapt coll component",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                            OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY, &cs->adapt_priority);
+                                           MCA_BASE_VAR_SCOPE_ALL, &cs->adapt_priority);
 
     cs->adapt_verbose = ompi_coll_base_framework.framework_verbose;
     (void) mca_base_component_var_register(c, "verbose",
                                            "Verbose level (default set to the collective framework verbosity)",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                            OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY, &cs->adapt_verbose);
+                                           MCA_BASE_VAR_SCOPE_ALL, &cs->adapt_verbose);
 
     cs->adapt_context_free_list_min = 64;
     (void) mca_base_component_var_register(c, "context_free_list_min",
                                            "Minimum number of segments in context free list",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                            OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           MCA_BASE_VAR_SCOPE_ALL,
                                            &cs->adapt_context_free_list_min);
 
     cs->adapt_context_free_list_max = 1024;
     (void) mca_base_component_var_register(c, "context_free_list_max",
                                            "Maximum number of segments in context free list",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                            OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           MCA_BASE_VAR_SCOPE_ALL,
                                            &cs->adapt_context_free_list_max);
 
     cs->adapt_context_free_list_inc = 32;
     (void) mca_base_component_var_register(c, "context_free_list_inc",
                                            "Increasement number of segments in context free list",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                            OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           MCA_BASE_VAR_SCOPE_ALL,
                                            &cs->adapt_context_free_list_inc);
     ompi_coll_adapt_ibcast_register();
     ompi_coll_adapt_ireduce_register();
diff --git a/ompi/mca/coll/adapt/coll_adapt_context.h b/ompi/mca/coll/adapt/coll_adapt_context.h
index 5d729423fbe..ec8ed8516a9 100644
--- a/ompi/mca/coll/adapt/coll_adapt_context.h
+++ b/ompi/mca/coll/adapt/coll_adapt_context.h
@@ -32,7 +32,7 @@ struct ompi_coll_adapt_constant_bcast_context_s {
     opal_mutex_t *mutex;
     int *recv_array;
     int *send_array;
-    /* Length of the fragment array, which is the number of recevied segments */
+    /* Length of the fragment array, which is the number of received segments */
     int num_recv_segs;
     /* Number of segments that is finishing recving */
     int num_recv_fini;
@@ -81,7 +81,7 @@ struct ompi_coll_adapt_constant_reduce_context_s {
     int ireduce_tag;
     /* How many sends are posted but not finished */
     int32_t ongoing_send;
-    /* Length of the fragment array, which is the number of recevied segments */
+    /* Length of the fragment array, which is the number of received segments */
     int32_t num_recv_segs;
     /* Number of sent segments */
     int32_t num_sent_segs;
diff --git a/ompi/mca/coll/adapt/coll_adapt_ibcast.c b/ompi/mca/coll/adapt/coll_adapt_ibcast.c
index 605d6262303..00bc79fbd46 100644
--- a/ompi/mca/coll/adapt/coll_adapt_ibcast.c
+++ b/ompi/mca/coll/adapt/coll_adapt_ibcast.c
@@ -2,6 +2,8 @@
  * Copyright (c) 2014-2020 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
+ * Copyright (c) 2024      NVIDIA CORPORATION. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -33,8 +35,9 @@ int ompi_coll_adapt_ibcast_register(void)
 
     mca_coll_adapt_component.adapt_ibcast_algorithm = 1;
     mca_base_component_var_register(c, "bcast_algorithm",
-                                    "Algorithm of broadcast, 0: tuned, 1: binomial, 2: in_order_binomial, 3: binary, 4: pipeline, 5: chain, 6: linear", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                    OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_READONLY,
+                                    "Algorithm of broadcast, 0: tuned, 1: binomial, 2: in_order_binomial, 3: binary, 4: pipeline, 5: chain, 6: linear",
+                                    MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
+                                    OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_ALL,
                                     &mca_coll_adapt_component.adapt_ibcast_algorithm);
     if( (mca_coll_adapt_component.adapt_ibcast_algorithm < 0) ||
         (mca_coll_adapt_component.adapt_ibcast_algorithm >= OMPI_COLL_ADAPT_ALGORITHM_COUNT) ) {
@@ -44,33 +47,33 @@ int ompi_coll_adapt_ibcast_register(void)
     mca_coll_adapt_component.adapt_ibcast_segment_size = 0;
     mca_base_component_var_register(c, "bcast_segment_size",
                                     "Segment size in bytes used by default for bcast algorithms. Only has meaning if algorithm is forced and supports segmenting. 0 bytes means no segmentation.",
-                                    MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                    MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                     OPAL_INFO_LVL_5,
-                                    MCA_BASE_VAR_SCOPE_READONLY,
+                                    MCA_BASE_VAR_SCOPE_ALL,
                                     &mca_coll_adapt_component.adapt_ibcast_segment_size);
 
     mca_coll_adapt_component.adapt_ibcast_max_send_requests = 2;
     mca_base_component_var_register(c, "bcast_max_send_requests",
                                     "Maximum number of send requests",
-                                    MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                    MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                     OPAL_INFO_LVL_5,
-                                    MCA_BASE_VAR_SCOPE_READONLY,
+                                    MCA_BASE_VAR_SCOPE_ALL,
                                     &mca_coll_adapt_component.adapt_ibcast_max_send_requests);
 
     mca_coll_adapt_component.adapt_ibcast_max_recv_requests = 3;
     mca_base_component_var_register(c, "bcast_max_recv_requests",
                                     "Maximum number of receive requests",
-                                    MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                    MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                     OPAL_INFO_LVL_5,
-                                    MCA_BASE_VAR_SCOPE_READONLY,
+                                    MCA_BASE_VAR_SCOPE_ALL,
                                     &mca_coll_adapt_component.adapt_ibcast_max_recv_requests);
 
     mca_coll_adapt_component.adapt_ibcast_synchronous_send = true;
     (void) mca_base_component_var_register(c, "bcast_synchronous_send",
                                            "Whether to use synchronous send operations during setup of bcast operations",
-                                           MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
+                                           MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                            OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           MCA_BASE_VAR_SCOPE_ALL,
                                            &mca_coll_adapt_component.adapt_ibcast_synchronous_send);
 
     mca_coll_adapt_component.adapt_ibcast_context_free_list = NULL;
@@ -318,7 +321,7 @@ static int recv_cb(ompi_request_t * req)
     return 1;
 }
 
-int ompi_coll_adapt_ibcast(void *buff, int count, struct ompi_datatype_t *datatype, int root,
+int ompi_coll_adapt_ibcast(void *buff, size_t count, struct ompi_datatype_t *datatype, int root,
                           struct ompi_communicator_t *comm, ompi_request_t ** request,
                           mca_coll_base_module_t * module)
 {
@@ -335,12 +338,12 @@ int ompi_coll_adapt_ibcast(void *buff, int count, struct ompi_datatype_t *dataty
     }
 
     return ompi_coll_adapt_ibcast_generic(buff, count, datatype, root, comm, request, module,
-                                          adapt_module_cached_topology(module, comm, root, mca_coll_adapt_component.adapt_ibcast_algorithm),
+                                          ompi_coll_adapt_module_cached_topology(module, comm, root, mca_coll_adapt_component.adapt_ibcast_algorithm),
                                           mca_coll_adapt_component.adapt_ibcast_segment_size);
 }
 
 
-int ompi_coll_adapt_ibcast_generic(void *buff, int count, struct ompi_datatype_t *datatype, int root,
+int ompi_coll_adapt_ibcast_generic(void *buff, size_t count, struct ompi_datatype_t *datatype, int root,
                                    struct ompi_communicator_t *comm, ompi_request_t ** request,
                                    mca_coll_base_module_t * module, ompi_coll_tree_t * tree,
                                    size_t seg_size)
@@ -350,7 +353,7 @@ int ompi_coll_adapt_ibcast_generic(void *buff, int count, struct ompi_datatype_t
     int min;
 
     /* Number of datatype in a segment */
-    int seg_count = count;
+    size_t seg_count = count;
     /* Size of a datatype */
     size_t type_size;
     /* Real size of a segment */
@@ -414,7 +417,7 @@ int ompi_coll_adapt_ibcast_generic(void *buff, int count, struct ompi_datatype_t
     num_segs = (count + seg_count - 1) / seg_count;
     real_seg_size = (ptrdiff_t) seg_count *extent;
 
-    /* Set memory for recv_array and send_array, created on heap becasue they are needed to be accessed by other functions (callback functions) */
+    /* Set memory for recv_array and send_array, created on heap because they are needed to be accessed by other functions (callback functions) */
     if (num_segs != 0) {
         recv_array = (int *) malloc(sizeof(int) * num_segs);
     }
@@ -445,7 +448,7 @@ int ompi_coll_adapt_ibcast_generic(void *buff, int count, struct ompi_datatype_t
                          "[%d]: Ibcast, root %d, tag %d\n", rank, root,
                          con->ibcast_tag));
     OPAL_OUTPUT_VERBOSE((30, mca_coll_adapt_component.adapt_output,
-                         "[%d]: con->mutex = %p, num_children = %d, num_segs = %d, real_seg_size = %d, seg_count = %d, tree_adreess = %p\n",
+                         "[%d]: con->mutex = %p, num_children = %d, num_segs = %d, real_seg_size = %d, seg_count = %zu, tree_adreess = %p\n",
                          rank, (void *) con->mutex, tree->tree_nextsize, num_segs,
                          (int) real_seg_size, seg_count, (void *) con->tree));
 
@@ -485,7 +488,7 @@ int ompi_coll_adapt_ibcast_generic(void *buff, int count, struct ompi_datatype_t
                 context->frag_id = i;
                 /* The id of peer in in children_list */
                 context->child_id = j;
-                /* Actural rank of the peer */
+                /* Actual rank of the peer */
                 context->peer = tree->tree_next[j];
                 context->con = con;
                 OBJ_RETAIN(con);
@@ -557,11 +560,11 @@ int ompi_coll_adapt_ibcast_generic(void *buff, int count, struct ompi_datatype_t
                 MCA_PML_CALL(irecv
                              (recv_buff, recv_count, datatype, context->peer,
                               con->ibcast_tag - i, comm, &recv_req));
+            /* Set receive callback */
+            OPAL_THREAD_UNLOCK(mutex);
             if (MPI_SUCCESS != err) {
                 return err;
             }
-            /* Set receive callback */
-            OPAL_THREAD_UNLOCK(mutex);
             ompi_request_set_callback(recv_req, recv_cb, context);
             OPAL_THREAD_LOCK(mutex);
         }
diff --git a/ompi/mca/coll/adapt/coll_adapt_ireduce.c b/ompi/mca/coll/adapt/coll_adapt_ireduce.c
index 4755b7730af..15bd586901a 100644
--- a/ompi/mca/coll/adapt/coll_adapt_ireduce.c
+++ b/ompi/mca/coll/adapt/coll_adapt_ireduce.c
@@ -3,6 +3,9 @@
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
+ * Copyright (c) 2024      NVIDIA CORPORATION. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -36,8 +39,9 @@ int ompi_coll_adapt_ireduce_register(void)
 
     mca_coll_adapt_component.adapt_ireduce_algorithm = 1;
     mca_base_component_var_register(c, "reduce_algorithm",
-                                    "Algorithm of reduce, 1: binomial, 2: in_order_binomial, 3: binary, 4: pipeline, 5: chain, 6: linear", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                    OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_READONLY,
+                                    "Algorithm of reduce, 1: binomial, 2: in_order_binomial, 3: binary, 4: pipeline, 5: chain, 6: linear",
+                                    MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
+                                    OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_ALL,
                                     &mca_coll_adapt_component.adapt_ireduce_algorithm);
     if( (mca_coll_adapt_component.adapt_ireduce_algorithm < 0) ||
         (mca_coll_adapt_component.adapt_ireduce_algorithm > OMPI_COLL_ADAPT_ALGORITHM_COUNT) ) {
@@ -47,58 +51,58 @@ int ompi_coll_adapt_ireduce_register(void)
     mca_coll_adapt_component.adapt_ireduce_segment_size = 163740;
     mca_base_component_var_register(c, "reduce_segment_size",
                                     "Segment size in bytes used by default for reduce algorithms. Only has meaning if algorithm is forced and supports segmenting. 0 bytes means no segmentation.",
-                                    MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0,
+                                    MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                     OPAL_INFO_LVL_5,
-                                    MCA_BASE_VAR_SCOPE_READONLY,
+                                    MCA_BASE_VAR_SCOPE_ALL,
                                     &mca_coll_adapt_component.adapt_ireduce_segment_size);
 
     mca_coll_adapt_component.adapt_ireduce_max_send_requests = 2;
     mca_base_component_var_register(c, "reduce_max_send_requests",
                                     "Maximum number of send requests",
-                                    MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                    MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                     OPAL_INFO_LVL_5,
-                                    MCA_BASE_VAR_SCOPE_READONLY,
+                                    MCA_BASE_VAR_SCOPE_ALL,
                                     &mca_coll_adapt_component.adapt_ireduce_max_send_requests);
 
     mca_coll_adapt_component.adapt_ireduce_max_recv_requests = 3;
     mca_base_component_var_register(c, "reduce_max_recv_requests",
                                     "Maximum number of receive requests per peer",
-                                    MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                    MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                     OPAL_INFO_LVL_5,
-                                    MCA_BASE_VAR_SCOPE_READONLY,
+                                    MCA_BASE_VAR_SCOPE_ALL,
                                     &mca_coll_adapt_component.adapt_ireduce_max_recv_requests);
 
     mca_coll_adapt_component.adapt_inbuf_free_list_min = 10;
     mca_base_component_var_register(c, "inbuf_free_list_min",
                                     "Minimum number of segment in inbuf free list",
-                                    MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                    MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                     OPAL_INFO_LVL_5,
-                                    MCA_BASE_VAR_SCOPE_READONLY,
+                                    MCA_BASE_VAR_SCOPE_ALL,
                                     &mca_coll_adapt_component.adapt_inbuf_free_list_min);
 
     mca_coll_adapt_component.adapt_inbuf_free_list_max = 10000;
     mca_base_component_var_register(c, "inbuf_free_list_max",
                                     "Maximum number of segment in inbuf free list",
-                                    MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                    MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                     OPAL_INFO_LVL_5,
-                                    MCA_BASE_VAR_SCOPE_READONLY,
+                                    MCA_BASE_VAR_SCOPE_ALL,
                                     &mca_coll_adapt_component.adapt_inbuf_free_list_max);
 
 
     mca_coll_adapt_component.adapt_inbuf_free_list_inc = 10;
     mca_base_component_var_register(c, "inbuf_free_list_inc",
                                     "Number of segments to allocate when growing the inbuf free list",
-                                    MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                    MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                     OPAL_INFO_LVL_5,
-                                    MCA_BASE_VAR_SCOPE_READONLY,
+                                    MCA_BASE_VAR_SCOPE_ALL,
                                     &mca_coll_adapt_component.adapt_inbuf_free_list_inc);
 
     mca_coll_adapt_component.adapt_ireduce_synchronous_send = true;
     (void) mca_base_component_var_register(c, "reduce_synchronous_send",
                                            "Whether to use synchronous send operations during setup of reduce operations",
-                                           MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
+                                           MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                            OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           MCA_BASE_VAR_SCOPE_ALL,
                                            &mca_coll_adapt_component.adapt_ireduce_synchronous_send);
 
     mca_coll_adapt_component.adapt_ireduce_context_free_list = NULL;
@@ -474,7 +478,7 @@ static int recv_cb(ompi_request_t * req)
     return 1;
 }
 
-int ompi_coll_adapt_ireduce(const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype,
+int ompi_coll_adapt_ireduce(const void *sbuf, void *rbuf, size_t count, struct ompi_datatype_t *dtype,
                            struct ompi_op_t *op, int root, struct ompi_communicator_t *comm,
                            ompi_request_t ** request, mca_coll_base_module_t * module)
 {
@@ -504,13 +508,14 @@ int ompi_coll_adapt_ireduce(const void *sbuf, void *rbuf, int count, struct ompi
 
 
     return ompi_coll_adapt_ireduce_generic(sbuf, rbuf, count, dtype, op, root, comm, request, module,
-                                           adapt_module_cached_topology(module, comm, root, mca_coll_adapt_component.adapt_ireduce_algorithm),
+                                           ompi_coll_adapt_module_cached_topology(module, comm, root,
+                                        		                                            mca_coll_adapt_component.adapt_ireduce_algorithm),
                                            mca_coll_adapt_component.adapt_ireduce_segment_size);
 
 }
 
 
-int ompi_coll_adapt_ireduce_generic(const void *sbuf, void *rbuf, int count,
+int ompi_coll_adapt_ireduce_generic(const void *sbuf, void *rbuf, size_t count,
                                     struct ompi_datatype_t *dtype, struct ompi_op_t *op, int root,
                                     struct ompi_communicator_t *comm, ompi_request_t ** request,
                                     mca_coll_base_module_t * module, ompi_coll_tree_t * tree,
@@ -520,7 +525,8 @@ int ompi_coll_adapt_ireduce_generic(const void *sbuf, void *rbuf, int count,
     ptrdiff_t extent, lower_bound, segment_increment;
     ptrdiff_t true_lower_bound, true_extent, real_seg_size;
     size_t typelng;
-    int seg_count = count, num_segs, rank, recv_count, send_count, err, min;
+    size_t seg_count = count, recv_count, send_count;
+    int num_segs, rank, err, min;
     /* Used to store the accumuate result, pointer to every segment */
     char **accumbuf = NULL;
     opal_mutex_t *mutex_op_list;
@@ -661,7 +667,6 @@ int ompi_coll_adapt_ireduce_generic(const void *sbuf, void *rbuf, int count,
             con->next_recv_segs[i] = min - 1;
         }
 
-        int num_recvs = 0;
         for (int32_t seg_index = 0; seg_index < min; seg_index++)
         {
             /* For each child */
@@ -694,7 +699,7 @@ int ompi_coll_adapt_ireduce_generic(const void *sbuf, void *rbuf, int count,
                 context->inbuf = inbuf;
 
                 OPAL_OUTPUT_VERBOSE((30, mca_coll_adapt_component.adapt_output,
-                                      "[%d]: In ireduce, create irecv for seg %d, peer %d, recv_count %d, inbuf %p tag %d\n",
+                                      "[%d]: In ireduce, create irecv for seg %d, peer %d, recv_count %zu, inbuf %p tag %d\n",
                                       context->con->rank, context->seg_index, context->peer,
                                       recv_count, (void *) inbuf,
                                       con->ireduce_tag - seg_index));
@@ -709,8 +714,6 @@ int ompi_coll_adapt_ireduce_generic(const void *sbuf, void *rbuf, int count,
                 }
                 /* Set the recv callback */
                 ompi_request_set_callback(recv_req, recv_cb, context);
-
-                ++num_recvs;
             }
         }
     }
@@ -741,13 +744,13 @@ int ompi_coll_adapt_ireduce_generic(const void *sbuf, void *rbuf, int count,
                 (ompi_coll_adapt_reduce_context_t *)opal_free_list_wait(mca_coll_adapt_component.adapt_ireduce_context_free_list);
             context->buff = (char *) sbuf + (ptrdiff_t) seg_index * (ptrdiff_t) segment_increment;
             context->seg_index = seg_index;
-            /* Actural rank of the peer */
+            /* Actual rank of the peer */
             context->peer = tree->tree_prev;
             context->con = con;
             context->inbuf = NULL;
 
             OPAL_OUTPUT_VERBOSE((30, mca_coll_adapt_component.adapt_output,
-                                  "[%d]: In ireduce, create isend to seg %d, peer %d, send_count %d tag %d\n",
+                                  "[%d]: In ireduce, create isend to seg %d, peer %d, send_count %zu tag %d\n",
                                   context->con->rank, context->seg_index, context->peer,
                                   send_count, con->ireduce_tag - context->seg_index));
 
diff --git a/ompi/mca/coll/adapt/coll_adapt_module.c b/ompi/mca/coll/adapt/coll_adapt_module.c
index 8b5fda9bf60..abd27d09870 100644
--- a/ompi/mca/coll/adapt/coll_adapt_module.c
+++ b/ompi/mca/coll/adapt/coll_adapt_module.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2014-2020 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2021      Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
+ *
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -63,8 +68,8 @@ static void adapt_module_construct(mca_coll_adapt_module_t * module)
 static void adapt_module_destruct(mca_coll_adapt_module_t * module)
 {
     if (NULL != module->topo_cache) {
-        adapt_topology_cache_item_t *item;
-        while (NULL != (item = (adapt_topology_cache_item_t*)opal_list_remove_first(module->topo_cache))) {
+        ompi_coll_adapt_topology_cache_item_t *item;
+        while (NULL != (item = (ompi_coll_adapt_topology_cache_item_t*)opal_list_remove_first(module->topo_cache))) {
             OBJ_RELEASE(item);
         }
         OBJ_RELEASE(module->topo_cache);
@@ -79,25 +84,41 @@ OBJ_CLASS_INSTANCE(mca_coll_adapt_module_t,
             adapt_module_construct,
             adapt_module_destruct);
 
-/*
- * In this macro, the following variables are supposed to have been declared
- * in the caller:
- * . ompi_communicator_t *comm
- * . mca_coll_adapt_module_t *adapt_module
- */
-#define ADAPT_SAVE_PREV_COLL_API(__api)                                 \
-    do {                                                                \
-        adapt_module->previous_ ## __api            = comm->c_coll->coll_ ## __api; \
-        adapt_module->previous_ ## __api ## _module = comm->c_coll->coll_ ## __api ## _module; \
-        if (!comm->c_coll->coll_ ## __api || !comm->c_coll->coll_ ## __api ## _module) { \
-            opal_output_verbose(1, ompi_coll_base_framework.framework_output, \
-                                "(%d/%s): no underlying " # __api"; disqualifying myself", \
-                                comm->c_contextid, comm->c_name); \
-            return OMPI_ERROR;                                  \
-        }                                                       \
-        OBJ_RETAIN(adapt_module->previous_ ## __api ## _module);  \
-    } while(0)
-
+#define ADAPT_INSTALL_COLL_API(__comm, __module, __api)                                                   \
+    do                                                                                                    \
+    {                                                                                                     \
+        if (__module->super.coll_##__api)                                                                 \
+        {                                                                                                 \
+            MCA_COLL_INSTALL_API(__comm, __api, __module->super.coll_##__api, &__module->super, "adapt"); \
+        }                                                                                                 \
+    } while (0)
+#define ADAPT_UNINSTALL_COLL_API(__comm, __module, __api)                                 \
+    do                                                                                    \
+    {                                                                                     \
+        if (__comm->c_coll->coll_##__api##_module == &__module->super)                    \
+        {                                                                                 \
+            MCA_COLL_INSTALL_API(__comm, __api, NULL, NULL, "adapt");                     \
+        }                                                                                 \
+    } while (0)
+#define ADAPT_INSTALL_AND_SAVE_COLL_API(__comm, __module, __api)                                                         \
+    do                                                                                                                   \
+    {                                                                                                                    \
+        if (__comm->c_coll->coll_##__api && __comm->c_coll->coll_##__api##_module)                                       \
+        {                                                                                                                \
+            MCA_COLL_SAVE_API(__comm, __api, __module->previous_##__api, __module->previous_##__api##_module, "adapt");  \
+            MCA_COLL_INSTALL_API(__comm, __api, __module->super.coll_##__api, &__module->super, "adapt");                \
+        }                                                                                                                \
+    } while (0)
+#define ADAPT_UNINSTALL_AND_RESTORE_COLL_API(__comm, __module, __api)                                                      \
+    do                                                                                                                     \
+    {                                                                                                                      \
+        if (__comm->c_coll->coll_##__api##_module == &__module->super)                                                     \
+        {                                                                                                                  \
+            MCA_COLL_INSTALL_API(__comm, __api, __module->previous_##__api, __module->previous_##__api##_module, "adapt"); \
+            __module->previous_##__api = NULL;                                                                             \
+            __module->previous_##__api##_module = NULL;                                                                    \
+        }                                                                                                                  \
+    } while (0)
 
 /*
  * Init module on the communicator
@@ -107,12 +128,25 @@ static int adapt_module_enable(mca_coll_base_module_t * module,
 {
     mca_coll_adapt_module_t * adapt_module = (mca_coll_adapt_module_t*) module;
 
-    ADAPT_SAVE_PREV_COLL_API(reduce);
-    ADAPT_SAVE_PREV_COLL_API(ireduce);
+    ADAPT_INSTALL_AND_SAVE_COLL_API(comm, adapt_module, reduce);
+    ADAPT_INSTALL_COLL_API(comm, adapt_module, bcast);
+    ADAPT_INSTALL_AND_SAVE_COLL_API(comm, adapt_module, ireduce);
+    ADAPT_INSTALL_COLL_API(comm, adapt_module, ibcast);
 
     return OMPI_SUCCESS;
 }
+static int adapt_module_disable(mca_coll_base_module_t *module,
+                                struct ompi_communicator_t *comm)
+{
+    mca_coll_adapt_module_t *adapt_module = (mca_coll_adapt_module_t *)module;
+
+    ADAPT_UNINSTALL_AND_RESTORE_COLL_API(comm, adapt_module, reduce);
+    ADAPT_UNINSTALL_COLL_API(comm, adapt_module, bcast);
+    ADAPT_UNINSTALL_AND_RESTORE_COLL_API(comm, adapt_module, ireduce);
+    ADAPT_UNINSTALL_COLL_API(comm, adapt_module, ibcast);
 
+    return OMPI_SUCCESS;
+}
 /*
  * Initial query function that is invoked during MPI_INIT, allowing
  * this component to disqualify itself if it doesn't support the
@@ -137,9 +171,9 @@ mca_coll_base_module_t *ompi_coll_adapt_comm_query(struct ompi_communicator_t *
     /* If we're intercomm, or if there's only one process in the communicator */
     if (OMPI_COMM_IS_INTER(comm) || 1 == ompi_comm_size(comm)) {
         opal_output_verbose(10, ompi_coll_base_framework.framework_output,
-                            "coll:adapt:comm_query (%d/%s): intercomm, "
+                            "coll:adapt:comm_query (%s/%s): intercomm, "
                             "comm is too small; disqualifying myself",
-                            comm->c_contextid, comm->c_name);
+                            ompi_comm_print_cid(comm), comm->c_name);
         return NULL;
     }
 
@@ -148,9 +182,9 @@ mca_coll_base_module_t *ompi_coll_adapt_comm_query(struct ompi_communicator_t *
     *priority = mca_coll_adapt_component.adapt_priority;
     if (mca_coll_adapt_component.adapt_priority < 0) {
         opal_output_verbose(10, ompi_coll_base_framework.framework_output,
-                            "coll:adapt:comm_query (%d/%s): priority too low; "
+                            "coll:adapt:comm_query (%s/%s): priority too low; "
                             "disqualifying myself",
-                            comm->c_contextid, comm->c_name);
+                            ompi_comm_print_cid(comm), comm->c_name);
         return NULL;
     }
 
@@ -161,28 +195,15 @@ mca_coll_base_module_t *ompi_coll_adapt_comm_query(struct ompi_communicator_t *
 
     /* All is good -- return a module */
     adapt_module->super.coll_module_enable = adapt_module_enable;
-    adapt_module->super.coll_allgather = NULL;
-    adapt_module->super.coll_allgatherv = NULL;
-    adapt_module->super.coll_allreduce = NULL;
-    adapt_module->super.coll_alltoall = NULL;
-    adapt_module->super.coll_alltoallw = NULL;
-    adapt_module->super.coll_barrier = NULL;
+    adapt_module->super.coll_module_disable = adapt_module_disable;
     adapt_module->super.coll_bcast = ompi_coll_adapt_bcast;
-    adapt_module->super.coll_exscan = NULL;
-    adapt_module->super.coll_gather = NULL;
-    adapt_module->super.coll_gatherv = NULL;
     adapt_module->super.coll_reduce = ompi_coll_adapt_reduce;
-    adapt_module->super.coll_reduce_scatter = NULL;
-    adapt_module->super.coll_scan = NULL;
-    adapt_module->super.coll_scatter = NULL;
-    adapt_module->super.coll_scatterv = NULL;
     adapt_module->super.coll_ibcast = ompi_coll_adapt_ibcast;
     adapt_module->super.coll_ireduce = ompi_coll_adapt_ireduce;
-    adapt_module->super.coll_iallreduce = NULL;
 
     opal_output_verbose(10, ompi_coll_base_framework.framework_output,
-                        "coll:adapt:comm_query (%d/%s): pick me! pick me!",
-                        comm->c_contextid, comm->c_name);
+                        "coll:adapt:comm_query (%s/%s): pick me! pick me!",
+                        ompi_comm_print_cid(comm), comm->c_name);
     return &(adapt_module->super);
 }
 
diff --git a/ompi/mca/coll/adapt/coll_adapt_reduce.c b/ompi/mca/coll/adapt/coll_adapt_reduce.c
index d0ad26d6e68..f206538b810 100644
--- a/ompi/mca/coll/adapt/coll_adapt_reduce.c
+++ b/ompi/mca/coll/adapt/coll_adapt_reduce.c
@@ -15,7 +15,7 @@
 #include "coll_adapt_algorithms.h"
 
 /* MPI_Reduce and MPI_Ireduce in the ADAPT module only work for commutative operations */
-int ompi_coll_adapt_reduce(const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype,
+int ompi_coll_adapt_reduce(const void *sbuf, void *rbuf, size_t count, struct ompi_datatype_t *dtype,
                           struct ompi_op_t *op, int root, struct ompi_communicator_t *comm,
                           mca_coll_base_module_t * module)
 {
diff --git a/ompi/mca/coll/adapt/coll_adapt_topocache.c b/ompi/mca/coll/adapt/coll_adapt_topocache.c
index 93c9a6043d4..e114456d12c 100644
--- a/ompi/mca/coll/adapt/coll_adapt_topocache.c
+++ b/ompi/mca/coll/adapt/coll_adapt_topocache.c
@@ -2,6 +2,7 @@
  * Copyright (c) 2014-2020 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -14,14 +15,14 @@
 
 #include "ompi/communicator/communicator.h"
 
-static void destruct_topology_cache(adapt_topology_cache_item_t *item)
+static void destruct_topology_cache(ompi_coll_adapt_topology_cache_item_t *item)
 {
     if (NULL != item->tree) {
         ompi_coll_base_topo_destroy_tree(&item->tree);
     }
 }
 
-OBJ_CLASS_INSTANCE(adapt_topology_cache_item_t, opal_list_item_t,
+OBJ_CLASS_INSTANCE(ompi_coll_adapt_topology_cache_item_t, opal_list_item_t,
                    NULL, &destruct_topology_cache);
 
 static ompi_coll_tree_t *create_topology(
@@ -58,7 +59,7 @@ static ompi_coll_tree_t *create_topology(
         {
             int fanout = ompi_comm_size(comm) - 1;
             ompi_coll_tree_t *tree;
-            if (fanout < 1) {
+            if (fanout <= 1) {
                 tree = ompi_coll_base_topo_build_chain(1, comm, root);
             } else if (fanout <= MAXTREEFANOUT) {
                 tree = ompi_coll_base_topo_build_tree(ompi_comm_size(comm) - 1, comm, root);
@@ -73,17 +74,17 @@ static ompi_coll_tree_t *create_topology(
     }
 }
 
-ompi_coll_tree_t* adapt_module_cached_topology(
+ompi_coll_tree_t* ompi_coll_adapt_module_cached_topology(
     mca_coll_base_module_t *module,
     struct ompi_communicator_t *comm,
     int root,
     ompi_coll_adapt_algorithm_t algorithm)
 {
     mca_coll_adapt_module_t *adapt_module = (mca_coll_adapt_module_t*)module;
-    adapt_topology_cache_item_t *item;
+    ompi_coll_adapt_topology_cache_item_t *item;
     ompi_coll_tree_t * tree;
     if (NULL != adapt_module->topo_cache) {
-        OPAL_LIST_FOREACH(item, adapt_module->topo_cache, adapt_topology_cache_item_t) {
+        OPAL_LIST_FOREACH(item, adapt_module->topo_cache, ompi_coll_adapt_topology_cache_item_t) {
             if (item->root == root && item->algorithm == algorithm) {
                 return item->tree;
             }
@@ -95,7 +96,7 @@ ompi_coll_tree_t* adapt_module_cached_topology(
     /* topology not found, create one */
     tree = create_topology(algorithm, root, comm);
 
-    item = OBJ_NEW(adapt_topology_cache_item_t);
+    item = OBJ_NEW(ompi_coll_adapt_topology_cache_item_t);
     item->tree = tree;
     item->root = root;
     item->algorithm = algorithm;
diff --git a/ompi/mca/coll/adapt/coll_adapt_topocache.h b/ompi/mca/coll/adapt/coll_adapt_topocache.h
index 3fae7562d15..b0e79e1c490 100644
--- a/ompi/mca/coll/adapt/coll_adapt_topocache.h
+++ b/ompi/mca/coll/adapt/coll_adapt_topocache.h
@@ -3,6 +3,7 @@
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -13,6 +14,8 @@
 #ifndef MCA_COLL_ADAPT_TOPOCACHE_H
 #define MCA_COLL_ADAPT_TOPOCACHE_H
 
+#include "ompi/communicator/communicator.h"
+#include "ompi/mca/coll/adapt/coll_adapt.h"
 #include "opal/class/opal_list.h"
 #include "ompi/mca/coll/coll.h"
 #include "ompi/mca/coll/base/coll_base_topo.h"
@@ -22,12 +25,12 @@ typedef struct adapt_topology_cache_item_t {
     ompi_coll_tree_t *tree;
     int root;
     ompi_coll_adapt_algorithm_t algorithm;
-} adapt_topology_cache_item_t;
+} ompi_coll_adapt_topology_cache_item_t;
 
-OBJ_CLASS_DECLARATION(adapt_topology_cache_item_t);
+OBJ_CLASS_DECLARATION(ompi_coll_adapt_topology_cache_item_t);
 
 
-OMPI_DECLSPEC ompi_coll_tree_t* adapt_module_cached_topology(
+OMPI_DECLSPEC ompi_coll_tree_t* ompi_coll_adapt_module_cached_topology(
     mca_coll_base_module_t *module,
     struct ompi_communicator_t *comm,
     int root,
diff --git a/ompi/mca/coll/base/Makefile.am b/ompi/mca/coll/base/Makefile.am
index bf98e476411..5c164d579e0 100644
--- a/ompi/mca/coll/base/Makefile.am
+++ b/ompi/mca/coll/base/Makefile.am
@@ -9,6 +9,7 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -16,7 +17,7 @@
 # $HEADER$
 #
 
-dist_ompidata_DATA = base/help-mca-coll-base.txt
+EXTRA_DIST = base/help-mca-coll-base.txt
 
 headers += \
         base/base.h \
@@ -51,4 +52,3 @@ if WANT_FT_MPI
 libmca_coll_la_SOURCES += \
         base/coll_base_agree_noft.c
 endif # WANT_FT_MPI
-
diff --git a/ompi/mca/coll/base/coll_base_agree_noft.c b/ompi/mca/coll/base/coll_base_agree_noft.c
index 7f7bc112a71..08591e1e897 100644
--- a/ompi/mca/coll/base/coll_base_agree_noft.c
+++ b/ompi/mca/coll/base/coll_base_agree_noft.c
@@ -19,20 +19,21 @@
 
 int
 ompi_coll_base_agree_noft(void *contrib,
-                         int dt_count,
+                         size_t dt_count,
                          struct ompi_datatype_t *dt,
                          struct ompi_op_t *op,
                          struct ompi_group_t **group, bool update_grp,
                          struct ompi_communicator_t* comm,
                          mca_coll_base_module_t *module)
 {
-    return comm->c_coll->coll_allreduce(MPI_IN_PLACE, contrib, dt_count, dt, op,
+    void *sendbuf = OMPI_COMM_IS_INTER(comm) ? contrib : MPI_IN_PLACE;
+    return comm->c_coll->coll_allreduce(sendbuf, contrib, dt_count, dt, op,
                                        comm, comm->c_coll->coll_allreduce_module);
 }
 
 int
 ompi_coll_base_iagree_noft(void *contrib,
-                          int dt_count,
+                          size_t dt_count,
                           struct ompi_datatype_t *dt,
                           struct ompi_op_t *op,
                           struct ompi_group_t **group, bool update_grp,
@@ -40,6 +41,7 @@ ompi_coll_base_iagree_noft(void *contrib,
                           ompi_request_t **request,
                           mca_coll_base_module_t *module)
 {
-    return comm->c_coll->coll_iallreduce(MPI_IN_PLACE, contrib, dt_count, dt, op,
+    void *sendbuf = OMPI_COMM_IS_INTER(comm) ? contrib : MPI_IN_PLACE;
+    return comm->c_coll->coll_iallreduce(sendbuf, contrib, dt_count, dt, op,
                                         comm, request, comm->c_coll->coll_iallreduce_module);
 }
diff --git a/ompi/mca/coll/base/coll_base_allgather.c b/ompi/mca/coll/base/coll_base_allgather.c
index a6d0794b6aa..d3e27884106 100644
--- a/ompi/mca/coll/base/coll_base_allgather.c
+++ b/ompi/mca/coll/base/coll_base_allgather.c
@@ -14,6 +14,7 @@
  *                         reserved.
  * Copyright (c) 2014-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -23,6 +24,8 @@
 
 #include "ompi_config.h"
 
+#include "math.h"
+
 #include "mpi.h"
 #include "opal/util/bit_ops.h"
 #include "ompi/constants.h"
@@ -34,176 +37,6 @@
 #include "coll_base_topo.h"
 #include "coll_base_util.h"
 
-/*
- * ompi_coll_base_allgather_intra_bruck
- *
- * Function:     allgather using O(log(N)) steps.
- * Accepts:      Same arguments as MPI_Allgather
- * Returns:      MPI_SUCCESS or error code
- *
- * Description:  Variation to All-to-all algorithm described by Bruck et al.in
- *               "Efficient Algorithms for All-to-all Communications
- *                in Multiport Message-Passing Systems"
- * Memory requirements:  non-zero ranks require shift buffer to perform final
- *               step in the algorithm.
- *
- * Example on 6 nodes:
- *   Initialization: everyone has its own buffer at location 0 in rbuf
- *                   This means if user specified MPI_IN_PLACE for sendbuf
- *                   we must copy our block from recvbuf to begining!
- *    #     0      1      2      3      4      5
- *         [0]    [1]    [2]    [3]    [4]    [5]
- *   Step 0: send message to (rank - 2^0), receive message from (rank + 2^0)
- *    #     0      1      2      3      4      5
- *         [0]    [1]    [2]    [3]    [4]    [5]
- *         [1]    [2]    [3]    [4]    [5]    [0]
- *   Step 1: send message to (rank - 2^1), receive message from (rank + 2^1)
- *           message contains all blocks from location 0 to 2^1*block size
- *    #     0      1      2      3      4      5
- *         [0]    [1]    [2]    [3]    [4]    [5]
- *         [1]    [2]    [3]    [4]    [5]    [0]
- *         [2]    [3]    [4]    [5]    [0]    [1]
- *         [3]    [4]    [5]    [0]    [1]    [2]
- *   Step 2: send message to (rank - 2^2), receive message from (rank + 2^2)
- *           message size is "all remaining blocks"
- *    #     0      1      2      3      4      5
- *         [0]    [1]    [2]    [3]    [4]    [5]
- *         [1]    [2]    [3]    [4]    [5]    [0]
- *         [2]    [3]    [4]    [5]    [0]    [1]
- *         [3]    [4]    [5]    [0]    [1]    [2]
- *         [4]    [5]    [0]    [1]    [2]    [3]
- *         [5]    [0]    [1]    [2]    [3]    [4]
- *    Finalization: Do a local shift to get data in correct place
- *    #     0      1      2      3      4      5
- *         [0]    [0]    [0]    [0]    [0]    [0]
- *         [1]    [1]    [1]    [1]    [1]    [1]
- *         [2]    [2]    [2]    [2]    [2]    [2]
- *         [3]    [3]    [3]    [3]    [3]    [3]
- *         [4]    [4]    [4]    [4]    [4]    [4]
- *         [5]    [5]    [5]    [5]    [5]    [5]
- */
-int ompi_coll_base_allgather_intra_bruck(const void *sbuf, int scount,
-                                          struct ompi_datatype_t *sdtype,
-                                          void* rbuf, int rcount,
-                                          struct ompi_datatype_t *rdtype,
-                                          struct ompi_communicator_t *comm,
-                                          mca_coll_base_module_t *module)
-{
-    int line = -1, rank, size, sendto, recvfrom, distance, blockcount, err = 0;
-    ptrdiff_t rlb, rext;
-    char *tmpsend = NULL, *tmprecv = NULL;
-
-    size = ompi_comm_size(comm);
-    rank = ompi_comm_rank(comm);
-
-    OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
-                 "coll:base:allgather_intra_bruck rank %d", rank));
-
-    err = ompi_datatype_get_extent (rdtype, &rlb, &rext);
-    if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
-
-    /* Initialization step:
-       - if send buffer is not MPI_IN_PLACE, copy send buffer to block 0 of
-       receive buffer, else
-       - if rank r != 0, copy r^th block from receive buffer to block 0.
-    */
-    tmprecv = (char*) rbuf;
-    if (MPI_IN_PLACE != sbuf) {
-        tmpsend = (char*) sbuf;
-        err = ompi_datatype_sndrcv(tmpsend, scount, sdtype, tmprecv, rcount, rdtype);
-        if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl;  }
-
-    } else if (0 != rank) {  /* non root with MPI_IN_PLACE */
-        tmpsend = ((char*)rbuf) + (ptrdiff_t)rank * (ptrdiff_t)rcount * rext;
-        err = ompi_datatype_copy_content_same_ddt(rdtype, rcount, tmprecv, tmpsend);
-        if (err < 0) { line = __LINE__; goto err_hndl; }
-    }
-
-    /* Communication step:
-       At every step i, rank r:
-       - doubles the distance
-       - sends message which starts at begining of rbuf and has size
-       (blockcount * rcount) to rank (r - distance)
-       - receives message of size blockcount * rcount from rank (r + distance)
-       at location (rbuf + distance * rcount * rext)
-       - blockcount doubles until last step when only the remaining data is
-       exchanged.
-    */
-    blockcount = 1;
-    tmpsend = (char*) rbuf;
-    for (distance = 1; distance < size; distance<<=1) {
-
-        recvfrom = (rank + distance) % size;
-        sendto = (rank - distance + size) % size;
-
-        tmprecv = tmpsend + (ptrdiff_t)distance * (ptrdiff_t)rcount * rext;
-
-        if (distance <= (size >> 1)) {
-            blockcount = distance;
-        } else {
-            blockcount = size - distance;
-        }
-
-        /* Sendreceive */
-        err = ompi_coll_base_sendrecv(tmpsend, blockcount * rcount, rdtype,
-                                       sendto, MCA_COLL_BASE_TAG_ALLGATHER,
-                                       tmprecv, blockcount * rcount, rdtype,
-                                       recvfrom, MCA_COLL_BASE_TAG_ALLGATHER,
-                                       comm, MPI_STATUS_IGNORE, rank);
-        if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
-
-    }
-
-    /* Finalization step:
-       On all nodes except 0, data needs to be shifted locally:
-       - create temporary shift buffer,
-       see discussion in coll_basic_reduce.c about the size and begining
-       of temporary buffer.
-       - copy blocks [0 .. (size - rank - 1)] from rbuf to shift buffer
-       - move blocks [(size - rank) .. size] from rbuf to begining of rbuf
-       - copy blocks from shift buffer starting at block [rank] in rbuf.
-    */
-    if (0 != rank) {
-        char *free_buf = NULL, *shift_buf = NULL;
-        ptrdiff_t span, gap = 0;
-
-        span = opal_datatype_span(&rdtype->super, (int64_t)(size - rank) * rcount, &gap);
-
-        free_buf = (char*)calloc(span, sizeof(char));
-        if (NULL == free_buf) {
-            line = __LINE__; err = OMPI_ERR_OUT_OF_RESOURCE; goto err_hndl;
-        }
-        shift_buf = free_buf - gap;
-
-        /* 1. copy blocks [0 .. (size - rank - 1)] from rbuf to shift buffer */
-        err = ompi_datatype_copy_content_same_ddt(rdtype, ((ptrdiff_t)(size - rank) * (ptrdiff_t)rcount),
-                                                  shift_buf, rbuf);
-        if (err < 0) { line = __LINE__; free(free_buf); goto err_hndl;  }
-
-        /* 2. move blocks [(size - rank) .. size] from rbuf to the begining of rbuf */
-        tmpsend = (char*) rbuf + (ptrdiff_t)(size - rank) * (ptrdiff_t)rcount * rext;
-        err = ompi_datatype_copy_content_same_ddt(rdtype, (ptrdiff_t)rank * (ptrdiff_t)rcount,
-                                                  rbuf, tmpsend);
-        if (err < 0) { line = __LINE__; free(free_buf); goto err_hndl;  }
-
-        /* 3. copy blocks from shift buffer back to rbuf starting at block [rank]. */
-        tmprecv = (char*) rbuf + (ptrdiff_t)rank * (ptrdiff_t)rcount * rext;
-        err = ompi_datatype_copy_content_same_ddt(rdtype, (ptrdiff_t)(size - rank) * (ptrdiff_t)rcount,
-                                                  tmprecv, shift_buf);
-        if (err < 0) { line = __LINE__; free(free_buf); goto err_hndl;  }
-
-        free(free_buf);
-    }
-
-    return OMPI_SUCCESS;
-
- err_hndl:
-    OPAL_OUTPUT((ompi_coll_base_framework.framework_output,  "%s:%4d\tError occurred %d, rank %2d",
-                 __FILE__, line, err, rank));
-    (void)line;  // silence compiler warning
-    return err;
-}
-
 /*
  * ompi_coll_base_allgather_intra_recursivedoubling
  *
@@ -250,9 +83,9 @@ int ompi_coll_base_allgather_intra_bruck(const void *sbuf, int scount,
  *          step, and send them appropriate messages.
  */
 int
-ompi_coll_base_allgather_intra_recursivedoubling(const void *sbuf, int scount,
+ompi_coll_base_allgather_intra_recursivedoubling(const void *sbuf, size_t scount,
                                                   struct ompi_datatype_t *sdtype,
-                                                  void* rbuf, int rcount,
+                                                  void* rbuf, size_t rcount,
                                                   struct ompi_datatype_t *rdtype,
                                                   struct ompi_communicator_t *comm,
                                                   mca_coll_base_module_t *module)
@@ -277,9 +110,11 @@ ompi_coll_base_allgather_intra_recursivedoubling(const void *sbuf, int scount,
                      "coll:base:allgather_intra_recursivedoubling WARNING: non-pow-2 size %d, switching to bruck algorithm",
                      size));
 
-        return ompi_coll_base_allgather_intra_bruck(sbuf, scount, sdtype,
-                                                     rbuf, rcount, rdtype,
-                                                     comm, module);
+        // fall back to the bruck method with radix 2
+        int k = 2;
+        return ompi_coll_base_allgather_intra_k_bruck(sbuf, scount, sdtype,
+                                                      rbuf, rcount, rdtype,
+                                                      comm, module, k);
     }
 
     OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
@@ -338,7 +173,145 @@ ompi_coll_base_allgather_intra_recursivedoubling(const void *sbuf, int scount,
     return err;
 }
 
+/*
+ * ompi_coll_base_allgather_intra_sparbit
+ *
+ * Function:     allgather using O(log(N)) steps.
+ * Accepts:      Same arguments as MPI_Allgather
+ * Returns:      MPI_SUCCESS or error code
+ *
+ * Description: Proposal of an allgather algorithm similar to Bruck but with inverted distances
+ *              and non-decreasing exchanged data sizes. Described in "Sparbit: a new
+ *              logarithmic-cost and data locality-aware MPI Allgather algorithm".
+ *
+ * Memory requirements:  
+ *              Additional memory for N requests. 
+ *
+ * Example on 6 nodes, with l representing the highest power of two smaller than N, in this case l =
+ * 4 (more details can be found on the paper):
+ *  Initial state
+ *    #     0      1      2      3      4      5
+ *         [0]    [ ]    [ ]    [ ]    [ ]    [ ]
+ *         [ ]    [1]    [ ]    [ ]    [ ]    [ ]
+ *         [ ]    [ ]    [2]    [ ]    [ ]    [ ]
+ *         [ ]    [ ]    [ ]    [3]    [ ]    [ ]
+ *         [ ]    [ ]    [ ]    [ ]    [4]    [ ]
+ *         [ ]    [ ]    [ ]    [ ]    [ ]    [5]
+ *   Step 0: Each process sends its own block to process r + l and receives another from r - l.
+ *    #     0      1      2      3      4      5
+ *         [0]    [ ]    [ ]    [ ]    [0]    [ ]
+ *         [ ]    [1]    [ ]    [ ]    [ ]    [1]
+ *         [2]    [ ]    [2]    [ ]    [ ]    [ ]
+ *         [ ]    [3]    [ ]    [3]    [ ]    [ ]
+ *         [ ]    [ ]    [4]    [ ]    [4]    [ ]
+ *         [ ]    [ ]    [ ]    [5]    [ ]    [5]
+ *   Step 1: Each process sends its own block to process r + l/2 and receives another from r - l/2.
+ *   The block received on the previous step is ignored to avoid a future double-write.  
+ *    #     0      1      2      3      4      5
+ *         [0]    [ ]    [0]    [ ]    [0]    [ ]
+ *         [ ]    [1]    [ ]    [1]    [ ]    [1]
+ *         [2]    [ ]    [2]    [ ]    [2]    [ ]
+ *         [ ]    [3]    [ ]    [3]    [ ]    [3]
+ *         [4]    [ ]    [4]    [ ]    [4]    [ ]
+ *         [ ]    [5]    [ ]    [5]    [ ]    [5]
+ *   Step 1: Each process sends all the data it has (3 blocks) to process r + l/4 and similarly
+ *   receives all the data from process r - l/4. 
+ *    #     0      1      2      3      4      5
+ *         [0]    [0]    [0]    [0]    [0]    [0]
+ *         [1]    [1]    [1]    [1]    [1]    [1]
+ *         [2]    [2]    [2]    [2]    [2]    [2]
+ *         [3]    [3]    [3]    [3]    [3]    [3]
+ *         [4]    [4]    [4]    [4]    [4]    [4]
+ *         [5]    [5]    [5]    [5]    [5]    [5]
+ */
+
+int ompi_coll_base_allgather_intra_sparbit(const void *sbuf, size_t scount,
+                                                  struct ompi_datatype_t *sdtype,
+                                                  void* rbuf, size_t rcount,
+                                                  struct ompi_datatype_t *rdtype,
+                                                  struct ompi_communicator_t *comm,
+                                                  mca_coll_base_module_t *module)
+{
+    /* ################# VARIABLE DECLARATION, BUFFER CREATION AND PREPARATION FOR THE ALGORITHM ######################## */
+
+    /* list of variable declaration */
+    int rank = 0, comm_size = 0, comm_log = 0, exclusion = 0, data_expected = 1, transfer_count = 0;
+    int sendto, recvfrom, send_disp, recv_disp;
+    uint32_t last_ignore, ignore_steps, distance = 1;
 
+    int err = 0;
+    int line = -1;
+
+    ptrdiff_t rlb, rext;
+
+    char *tmpsend = NULL, *tmprecv = NULL;
+
+    MPI_Request *requests = NULL;
+
+    /* algorithm choice information printing */
+    OPAL_OUTPUT((ompi_coll_base_framework.framework_output, 
+                 "coll:base:allgather_intra_sparbit rank %d", rank));
+
+    comm_size = ompi_comm_size(comm);
+    rank = ompi_comm_rank(comm);
+
+    err = ompi_datatype_get_extent(rdtype, &rlb, &rext);
+    if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
+
+    /* if the MPI_IN_PLACE condition is not set, copy the send buffer to the receive buffer to perform the sends (all the data is extracted and forwarded from the recv buffer)*/
+    /* tmprecv and tmpsend are used as abstract pointers to simplify send and receive buffer choice */
+    tmprecv = (char *) rbuf;
+    if(MPI_IN_PLACE != sbuf){
+        tmpsend = (char *) sbuf; 
+        err = ompi_datatype_sndrcv(tmpsend, scount, sdtype, tmprecv + (ptrdiff_t) rank * rcount * rext, rcount, rdtype);
+        if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl;  }
+    }
+    tmpsend = tmprecv;
+
+    requests = (MPI_Request *) malloc(comm_size * sizeof(MPI_Request));
+    
+    /* ################# ALGORITHM LOGIC ######################## */
+
+    /* calculate log2 of the total process count */
+    comm_log = ceil(log(comm_size)/log(2));
+    distance <<= comm_log - 1;
+
+    last_ignore = __builtin_ctz(comm_size);
+    ignore_steps = (~((uint32_t) comm_size >> last_ignore) | 1) << last_ignore;
+
+    /* perform the parallel binomial tree distribution steps */
+    for (int i = 0; i < comm_log; ++i) {
+       sendto = (rank + distance) % comm_size;  
+       recvfrom = (rank - distance + comm_size) % comm_size;  
+       exclusion = (distance & ignore_steps) == distance;
+
+       for (transfer_count = 0; transfer_count < data_expected - exclusion; transfer_count++) {
+           send_disp = (rank - 2 * transfer_count * distance + comm_size) % comm_size;
+           recv_disp = (rank - (2 * transfer_count + 1) * distance + comm_size) % comm_size;
+
+           /* Since each process sends several non-contiguos blocks of data, each block sent (and therefore each send and recv call) needs a different tag. */
+           /* As base OpenMPI only provides one tag for allgather, we are forced to use a tag space from other components in the send and recv calls */
+           MCA_PML_CALL(isend(tmpsend + (ptrdiff_t) send_disp * scount * rext, scount, rdtype, sendto, MCA_COLL_BASE_TAG_HCOLL_BASE - send_disp, MCA_PML_BASE_SEND_STANDARD, comm, requests + transfer_count));
+           MCA_PML_CALL(irecv(tmprecv + (ptrdiff_t) recv_disp * rcount * rext, rcount, rdtype, recvfrom, MCA_COLL_BASE_TAG_HCOLL_BASE - recv_disp, comm, requests + data_expected - exclusion + transfer_count));
+       }
+       ompi_request_wait_all(transfer_count * 2, requests, MPI_STATUSES_IGNORE);
+
+       distance >>= 1; 
+       /* calculates the data expected for the next step, based on the current number of blocks and eventual exclusions */
+       data_expected = (data_expected << 1) - exclusion;
+       exclusion = 0;
+    }
+    
+    free(requests);
+
+    return OMPI_SUCCESS;
+
+err_hndl:
+    OPAL_OUTPUT((ompi_coll_base_framework.framework_output,  "%s:%4d\tError occurred %d, rank %2d",
+                 __FILE__, line, err, rank));
+    (void)line;  // silence compiler warning
+    return err;
+}
 
 /*
  * ompi_coll_base_allgather_intra_ring
@@ -355,9 +328,9 @@ ompi_coll_base_allgather_intra_recursivedoubling(const void *sbuf, int scount,
  *               No additional memory requirements.
  *
  */
-int ompi_coll_base_allgather_intra_ring(const void *sbuf, int scount,
+int ompi_coll_base_allgather_intra_ring(const void *sbuf, size_t scount,
                                          struct ompi_datatype_t *sdtype,
-                                         void* rbuf, int rcount,
+                                         void* rbuf, size_t rcount,
                                          struct ompi_datatype_t *rdtype,
                                          struct ompi_communicator_t *comm,
                                          mca_coll_base_module_t *module)
@@ -392,7 +365,7 @@ int ompi_coll_base_allgather_intra_ring(const void *sbuf, int scount,
        [(r - i - 1 + size) % size]
        - sends message to rank [(r + 1) % size] containing data from rank
        [(r - i + size) % size]
-       - sends message which starts at begining of rbuf and has size
+       - sends message which starts at beginning of rbuf and has size
     */
     sendto = (rank + 1) % size;
     recvfrom  = (rank - 1 + size) % size;
@@ -481,9 +454,9 @@ int ompi_coll_base_allgather_intra_ring(const void *sbuf, int scount,
  *         [5]    [5]    [5]    [5]    [5]    [5]
  */
 int
-ompi_coll_base_allgather_intra_neighborexchange(const void *sbuf, int scount,
+ompi_coll_base_allgather_intra_neighborexchange(const void *sbuf, size_t scount,
                                                  struct ompi_datatype_t *sdtype,
-                                                 void* rbuf, int rcount,
+                                                 void* rbuf, size_t rcount,
                                                  struct ompi_datatype_t *rdtype,
                                                  struct ompi_communicator_t *comm,
                                                  mca_coll_base_module_t *module)
@@ -545,7 +518,7 @@ ompi_coll_base_allgather_intra_neighborexchange(const void *sbuf, int scount,
        - Rest of the steps:
        update recv_data_from according to offset, and
        exchange two blocks with appropriate neighbor.
-       the send location becomes previous receve location.
+       the send location becomes previous receive location.
     */
     tmprecv = (char*)rbuf + (ptrdiff_t)neighbor[0] * (ptrdiff_t)rcount * rext;
     tmpsend = (char*)rbuf + (ptrdiff_t)rank * (ptrdiff_t)rcount * rext;
@@ -595,9 +568,9 @@ ompi_coll_base_allgather_intra_neighborexchange(const void *sbuf, int scount,
 }
 
 
-int ompi_coll_base_allgather_intra_two_procs(const void *sbuf, int scount,
+int ompi_coll_base_allgather_intra_two_procs(const void *sbuf, size_t scount,
                                               struct ompi_datatype_t *sdtype,
-                                              void* rbuf, int rcount,
+                                              void* rbuf, size_t rcount,
                                               struct ompi_datatype_t *rdtype,
                                               struct ompi_communicator_t *comm,
                                               mca_coll_base_module_t *module)
@@ -678,10 +651,10 @@ int ompi_coll_base_allgather_intra_two_procs(const void *sbuf, int scount,
  *    Returns:    - MPI_SUCCESS or error code
  */
 int
-ompi_coll_base_allgather_intra_basic_linear(const void *sbuf, int scount,
+ompi_coll_base_allgather_intra_basic_linear(const void *sbuf, size_t scount,
                                              struct ompi_datatype_t *sdtype,
                                              void *rbuf,
-                                             int rcount,
+                                             size_t rcount,
                                              struct ompi_datatype_t *rdtype,
                                              struct ompi_communicator_t *comm,
                                              mca_coll_base_module_t *module)
@@ -726,4 +699,321 @@ ompi_coll_base_allgather_intra_basic_linear(const void *sbuf, int scount,
     return err;
 }
 
+/*
+ * ompi_coll_base_allgather_intra_k_bruck
+ *
+ * Function:     allgather using O(logk(N)) steps.
+ * Accepts:      Same arguments as MPI_Allgather
+ * Returns:      MPI_SUCCESS or error code
+ *
+ * Description:  This method extend ompi_coll_base_allgather_intra_bruck to handle any
+ *               radix k; use non-blocking communication to take advantage of multiple ports
+ *               The algorithm detail is described in Bruck et al. (1997),
+ *               "Efficient Algorithms for All-to-all Communications
+ *                in Multiport Message-Passing Systems"
+ * Memory requirements:  The data placed in rbuf after communication phase has a shift due to
+ *                       data wrap around. This buffer with a size of rcount * (size - rank)
+ *                       is used to shift the received data to the proper order
+ *
+ * Example on 10 nodes with k=3:
+ *   Initialization: everyone has its own buffer at location 0 in rbuf
+ *                   This means if user specified MPI_IN_PLACE for sendbuf
+ *                   we must copy our block from recvbuf to beginning!
+ *    #     0      1      2      3      4      5      6      7      8      9
+ *         [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
+ *   Step 0: send message to (rank - k^0 * i), receive message from (rank + k^0 * i)
+ *           message size is k^0 * block size and i is between [1, k-1]
+ *    #     0      1      2      3      4      5      6      7      8      9
+ *         [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
+ *         [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [0]
+ *         [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [0]    [1]
+ *   Step 1: send message to (rank - k^1 * i), receive message from (rank + k^1 * i)
+ *           message size is k^1 * block size
+ *    #     0      1      2      3      4      5      6      7      8      9
+ *         [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
+ *         [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [0]
+ *         [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [0]    [1]
+ *         [3]    [4]    [5]    [6]    [7]    [8]    [9]    [0]    [1]    [2]
+ *         [4]    [5]    [6]    [7]    [8]    [9]    [0]    [1]    [2]    [3]
+ *         [5]    [6]    [7]    [8]    [9]    [0]    [1]    [2]    [3]    [4]
+ *         [6]    [7]    [8]    [9]    [0]    [1]    [2]    [3]    [4]    [5]
+ *         [7]    [8]    [9]    [0]    [1]    [2]    [3]    [4]    [5]    [6]
+ *         [8]    [9]    [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]
+ *   Step 2: send message to (rank - k^2 * i), receive message from (rank + k^2 * i)
+ *           message size is k^2 * block size or "all remaining blocks" for each exchange
+ *    #     0      1      2      3      4      5      6      7      8      9
+ *         [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
+ *         [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [0]
+ *         [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [0]    [1]
+ *         [3]    [4]    [5]    [6]    [7]    [8]    [9]    [0]    [1]    [2]
+ *         [4]    [5]    [6]    [7]    [8]    [9]    [0]    [1]    [2]    [3]
+ *         [5]    [6]    [7]    [8]    [9]    [0]    [1]    [2]    [3]    [4]
+ *         [6]    [7]    [8]    [9]    [0]    [1]    [2]    [3]    [4]    [5]
+ *         [7]    [8]    [9]    [0]    [1]    [2]    [3]    [4]    [5]    [6]
+ *         [8]    [9]    [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]
+ *         [9]    [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]
+ *    Finalization: Do a local shift (except rank 0) to get data in correct place
+ *    #     0      1      2      3      4      5      6      7      8      9
+ *         [0]    [0]    [0]    [0]    [0]    [0]    [0]    [0]    [0]    [0]
+ *         [1]    [1]    [1]    [1]    [1]    [1]    [1]    [1]    [1]    [1]
+ *         [2]    [2]    [2]    [2]    [2]    [2]    [2]    [2]    [2]    [2]
+ *         [3]    [3]    [3]    [3]    [3]    [3]    [3]    [3]    [3]    [3]
+ *         [4]    [4]    [4]    [4]    [4]    [4]    [4]    [4]    [4]    [4]
+ *         [5]    [5]    [5]    [5]    [5]    [5]    [5]    [5]    [5]    [5]
+ *         [6]    [6]    [6]    [6]    [6]    [6]    [6]    [6]    [6]    [6]
+ *         [7]    [7]    [7]    [7]    [7]    [7]    [7]    [7]    [7]    [7]
+ *         [8]    [8]    [8]    [8]    [8]    [8]    [8]    [8]    [8]    [8]
+ *         [9]    [9]    [9]    [9]    [9]    [9]    [9]    [9]    [9]    [9]
+ */
+int ompi_coll_base_allgather_intra_k_bruck(const void *sbuf, size_t scount,
+                                          struct ompi_datatype_t *sdtype,
+                                          void* rbuf, size_t rcount,
+                                          struct ompi_datatype_t *rdtype,
+                                          struct ompi_communicator_t *comm,
+                                          mca_coll_base_module_t *module,
+                                          int radix)
+{
+    int line = -1, rank, size, dst, src, err = MPI_SUCCESS;
+    int recvcount, distance;
+    ptrdiff_t rlb, rextent;
+    ptrdiff_t rsize, rgap = 0;
+    ompi_request_t **reqs = NULL;
+    int num_reqs, max_reqs = 0;
+
+    char *tmpsend = NULL;
+    char *tmprecv = NULL;
+    char *tmp_buf = NULL;
+    char *tmp_buf_start = NULL;
+
+    rank = ompi_comm_rank(comm);
+    size = ompi_comm_size(comm);
+
+    OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
+                 "coll:base:allgather_intra_k_bruck radix %d rank %d", radix, rank));
+    err = ompi_datatype_get_extent (rdtype, &rlb, &rextent);
+    if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
+
+    if (0 != rank) {
+        /* Compute the temporary buffer size, including datatypes empty gaps */
+        rsize = opal_datatype_span(&rdtype->super, (size_t)rcount * (size - rank), &rgap);
+        tmp_buf = (char *) malloc(rsize);
+        tmp_buf_start = tmp_buf - rgap;
+    }
+
+    // tmprecv points to the data initially on this rank, handle mpi_in_place case
+    tmprecv = (char*) rbuf;
+    if (MPI_IN_PLACE != sbuf) {
+        tmpsend = (char*) sbuf;
+        err = ompi_datatype_sndrcv(tmpsend, scount, sdtype, tmprecv, rcount, rdtype);
+        if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
+    } else if (0 != rank) {
+        // root data placement is at the correct poistion
+        tmpsend = ((char*)rbuf) + (ptrdiff_t)rank * (ptrdiff_t)rcount * rextent;
+        err = ompi_datatype_copy_content_same_ddt(rdtype, rcount, tmprecv, tmpsend);
+        if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
+    }
+    /*
+       Maximum number of communication phases logk(n)
+       For each phase i, rank r:
+       - increase the distance and recvcount by k times
+       - sends (k - 1) messages which starts at beginning of rbuf and has size
+       (recvcount) to rank (r - distance * j)
+       - receives (k - 1) messages of size recvcount from rank (r + distance * j)
+       at location (rbuf + distance * j * rcount * rext)
+       - calculate the remaining data for each of the (k - 1) messages in the last
+       phase to complete all transactions
+    */
+    max_reqs = 2 * (radix - 1);
+    reqs = ompi_coll_base_comm_get_reqs(module->base_data, max_reqs);
+    recvcount = 1;
+    tmpsend = (char*) rbuf;
+    for (distance = 1; distance < size; distance *= radix) {
+        num_reqs = 0;
+        for (int j = 1; j < radix; j++)
+        {
+            if (distance * j >= size) {
+                break;
+            }
+            src = (rank + distance * j) % size;
+            dst = (rank - distance * j + size) % size;
+
+            tmprecv = tmpsend + (ptrdiff_t)distance * j * rcount * rextent;
+
+            if (distance <= (size / radix)) {
+                recvcount = distance;
+            } else {
+                recvcount = (distance < (size - distance * j)?
+                            distance:(size - distance * j));
+            }
+
+            err = MCA_PML_CALL(irecv(tmprecv,
+                                     recvcount * rcount,
+                                     rdtype,
+                                     src,
+                                     MCA_COLL_BASE_TAG_ALLGATHER,
+                                     comm,
+                                     &reqs[num_reqs++]));
+            if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
+            err = MCA_PML_CALL(isend(tmpsend,
+                                     recvcount * rcount,
+                                     rdtype,
+                                     dst,
+                                     MCA_COLL_BASE_TAG_ALLGATHER,
+                                     MCA_PML_BASE_SEND_STANDARD,
+                                     comm,
+                                     &reqs[num_reqs++]));
+            if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
+        }
+        err = ompi_request_wait_all(num_reqs, reqs, MPI_STATUSES_IGNORE);
+        if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
+    }
+
+    // Finalization step:        On all ranks except 0, data needs to be shifted locally
+    if (0 != rank) {
+        err = ompi_datatype_copy_content_same_ddt(rdtype,
+                                                  ((ptrdiff_t)(size - rank) * rcount),
+                                                  tmp_buf_start,
+                                                  rbuf);
+        if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
+
+        tmpsend = (char*) rbuf + (ptrdiff_t)(size - rank) * rcount * rextent;
+        err = ompi_datatype_copy_content_same_ddt(rdtype,
+                                                  (ptrdiff_t)rank * rcount,
+                                                  rbuf,
+                                                  tmpsend);
+        if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
+
+        tmprecv = (char*) rbuf + (ptrdiff_t)rank * rcount * rextent;
+        err = ompi_datatype_copy_content_same_ddt(rdtype,
+                                                  (ptrdiff_t)(size - rank) * rcount,
+                                                  tmprecv,
+                                                  tmp_buf_start);
+        if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
+    }
+
+    if(tmp_buf != NULL) free(tmp_buf);
+    return MPI_SUCCESS;
+
+err_hndl:
+    if( NULL != reqs ) {
+        if (MPI_ERR_IN_STATUS == err) {
+            for( num_reqs = 0; num_reqs < max_reqs; num_reqs++ ) {
+                if (MPI_REQUEST_NULL == reqs[num_reqs]) continue;
+                if (MPI_ERR_PENDING == reqs[num_reqs]->req_status.MPI_ERROR) continue;
+                if (reqs[num_reqs]->req_status.MPI_ERROR != MPI_SUCCESS) {
+                    err = reqs[num_reqs]->req_status.MPI_ERROR;
+                    break;
+                }
+            }
+        }
+        ompi_coll_base_free_reqs(reqs, max_reqs);
+    }
+    OPAL_OUTPUT((ompi_coll_base_framework.framework_output,  "%s:%4d\tError occurred %d, rank %2d",
+                 __FILE__, line, err, rank));
+    if(tmp_buf != NULL) {
+        free(tmp_buf);
+        tmp_buf = NULL;
+        tmp_buf_start = NULL;
+    }
+    (void)line;  // silence compiler warning
+    return err;
+}
+
+/**
+ * A greedy algorithm to exchange data among processes in the communicator via
+ * non-blocking direct messaging for allgather. This avoids the round trip in
+ * a rooted communication pattern, e.g. gather on the root and then broadcast to peers.
+ *
+ * This algorithm benefits inter-node allgather latency for small messages.
+ * Caution is needed on larger communicators(n) and data sizes(m), which will
+ * result in m*n^2 total traffic and potential network congestion.
+ */
+int ompi_coll_base_allgather_direct_messaging(const void *sbuf, size_t scount,
+                                              struct ompi_datatype_t *sdtype,
+                                              void* rbuf, size_t rcount,
+                                              struct ompi_datatype_t *rdtype,
+                                              struct ompi_communicator_t *comm,
+                                              mca_coll_base_module_t *module)
+{
+    int line = -1, rank, comm_size, err = MPI_SUCCESS;
+    ptrdiff_t rlb, rextent;
+    ptrdiff_t incr;
+    ompi_request_t **reqs = NULL;
+    int max_reqs = 0, reqs_needed = 0;
+    int peer_rank = 0;
+
+    char *tmpsend = NULL;
+    char *tmprecv = NULL;
+
+    rank = ompi_comm_rank(comm);
+    comm_size = ompi_comm_size(comm);
+
+    OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
+                 "coll:base:ompi_coll_base_allgather_direct_messaging rank %d", rank));
+
+    err = ompi_datatype_get_extent (rdtype, &rlb, &rextent);
+    if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
+
+    if (MPI_IN_PLACE != sbuf) {
+        tmpsend = (char*) sbuf;
+        tmprecv = (char*) rbuf + (ptrdiff_t)rank * (ptrdiff_t)rcount * rextent;
+        err = ompi_datatype_sndrcv(tmpsend, scount, sdtype, tmprecv, rcount, rdtype);
+        if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl;  }
+    }
+    else {
+        tmpsend = (char*) rbuf + (ptrdiff_t)rank * (ptrdiff_t)rcount * rextent;
+    }
+
+    /* Requests for send to AND receive from everyone else */
+    reqs_needed = (comm_size - 1) * 2;
+    reqs = ompi_coll_base_comm_get_reqs(module->base_data, reqs_needed);
+
+    incr = rextent * rcount;
+
+    /* Exchange data with peer processes, excluding self */
+    for (int i = 1; i < comm_size; ++i) {
+        /* Start at the next rank */
+        peer_rank = (rank + i) % comm_size;
+
+        /* Prepare for the next receive buffer */
+        tmprecv = (char*)rbuf + peer_rank * incr;
+
+        err = MCA_PML_CALL(irecv(tmprecv, rcount, rdtype, peer_rank, MCA_COLL_BASE_TAG_ALLGATHER,
+                                 comm, &reqs[max_reqs++]));
+        if (MPI_SUCCESS != err) {
+            goto err_hndl;
+        }
+
+        err = MCA_PML_CALL(isend(tmpsend, scount, sdtype, peer_rank, MCA_COLL_BASE_TAG_ALLGATHER,
+                                 MCA_PML_BASE_SEND_STANDARD, comm, &reqs[max_reqs++]));
+        if (MPI_SUCCESS != err) {
+            goto err_hndl;
+        }
+    }
+
+    err = ompi_request_wait_all(max_reqs, reqs, MPI_STATUSES_IGNORE);
+    if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
+    return MPI_SUCCESS;
+
+err_hndl:
+    if (NULL != reqs) {
+        if (MPI_ERR_IN_STATUS == err) {
+            for (int i = 0; i < reqs_needed; i++) {
+                if (MPI_REQUEST_NULL == reqs[i])
+                    continue;
+                if (MPI_ERR_PENDING == reqs[i]->req_status.MPI_ERROR)
+                    continue;
+                if (MPI_SUCCESS != reqs[i]->req_status.MPI_ERROR) {
+                    err = reqs[i]->req_status.MPI_ERROR;
+                    break;
+                }
+            }
+        }
+        ompi_coll_base_free_reqs(reqs, reqs_needed);
+    }
+    OPAL_OUTPUT((ompi_coll_base_framework.framework_output,  "%s:%4d\tError occurred %d, rank %2d",
+                 __FILE__, line, err, rank));
+    (void)line;  // silence compiler warning
+    return err;
+}
 /* copied function (with appropriate renaming) ends here */
diff --git a/ompi/mca/coll/base/coll_base_allgatherv.c b/ompi/mca/coll/base/coll_base_allgatherv.c
index 3f09d91d6cc..24cd84ec616 100644
--- a/ompi/mca/coll/base/coll_base_allgatherv.c
+++ b/ompi/mca/coll/base/coll_base_allgatherv.c
@@ -25,6 +25,8 @@
 
 #include "ompi_config.h"
 
+#include "math.h"
+
 #include "mpi.h"
 #include "ompi/constants.h"
 #include "ompi/datatype/ompi_datatype.h"
@@ -90,10 +92,10 @@
  *         [5]    [5]    [5]    [5]    [5]    [5]    [5]
  *         [6]    [6]    [6]    [6]    [6]    [6]    [6]
  */
-int ompi_coll_base_allgatherv_intra_bruck(const void *sbuf, int scount,
+int ompi_coll_base_allgatherv_intra_bruck(const void *sbuf, size_t scount,
                                            struct ompi_datatype_t *sdtype,
-                                           void *rbuf, const int *rcounts,
-                                           const int *rdispls,
+                                           void *rbuf, ompi_count_array_t rcounts,
+                                           ompi_disp_array_t rdispls,
                                            struct ompi_datatype_t *rdtype,
                                            struct ompi_communicator_t *comm,
                                            mca_coll_base_module_t *module)
@@ -117,11 +119,11 @@ int ompi_coll_base_allgatherv_intra_bruck(const void *sbuf, int scount,
        - if send buffer is not MPI_IN_PLACE, copy send buffer to block rank of
        the receive buffer.
     */
-    tmprecv = (char*) rbuf + (ptrdiff_t)rdispls[rank] * rext;
+    tmprecv = (char*) rbuf + ompi_disp_array_get(rdispls, rank) * rext;
     if (MPI_IN_PLACE != sbuf) {
         tmpsend = (char*) sbuf;
         err = ompi_datatype_sndrcv(tmpsend, scount, sdtype,
-                                   tmprecv, rcounts[rank], rdtype);
+                                   tmprecv, ompi_count_array_get(rcounts, rank), rdtype);
         if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl;  }
 
     }
@@ -161,10 +163,10 @@ int ompi_coll_base_allgatherv_intra_bruck(const void *sbuf, int scount,
         for (i = 0; i < blockcount; i++) {
             const int tmp_srank = (rank + i) % size;
             const int tmp_rrank = (recvfrom + i) % size;
-            new_scounts[i] = rcounts[tmp_srank];
-            new_sdispls[i] = rdispls[tmp_srank];
-            new_rcounts[i] = rcounts[tmp_rrank];
-            new_rdispls[i] = rdispls[tmp_rrank];
+            new_scounts[i] = ompi_count_array_get(rcounts, tmp_srank);
+            new_sdispls[i] = ompi_disp_array_get(rdispls, tmp_srank);
+            new_rcounts[i] = ompi_count_array_get(rcounts, tmp_rrank);
+            new_rdispls[i] = ompi_disp_array_get(rdispls, tmp_rrank);
         }
         err = ompi_datatype_create_indexed(blockcount, new_scounts, new_sdispls,
                                            rdtype, &new_sdtype);
@@ -202,6 +204,161 @@ int ompi_coll_base_allgatherv_intra_bruck(const void *sbuf, int scount,
     return err;
 }
 
+/*
+ * ompi_coll_base_allgather_intra_sparbit
+ *
+ * Function:     allgather using O(log(N)) steps.
+ * Accepts:      Same arguments as MPI_Allgather
+ * Returns:      MPI_SUCCESS or error code
+ *
+ * Description: Proposal of an allgather algorithm similar to Bruck but with inverted distances
+ *              and non-decreasing exchanged data sizes. Described in "Sparbit: a new
+ *              logarithmic-cost and data locality-aware MPI Allgather algorithm".
+ *
+ * Memory requirements:  
+ *              Additional memory for N requests. 
+ *
+ * Example on 6 nodes, with l representing the highest power of two smaller than N, in this case l =
+ * 4 (more details can be found on the paper):
+ *  Initial state
+ *    #     0      1      2      3      4      5
+ *         [0]    [ ]    [ ]    [ ]    [ ]    [ ]
+ *         [ ]    [1]    [ ]    [ ]    [ ]    [ ]
+ *         [ ]    [ ]    [2]    [ ]    [ ]    [ ]
+ *         [ ]    [ ]    [ ]    [3]    [ ]    [ ]
+ *         [ ]    [ ]    [ ]    [ ]    [4]    [ ]
+ *         [ ]    [ ]    [ ]    [ ]    [ ]    [5]
+ *   Step 0: Each process sends its own block to process r + l and receives another from r - l.
+ *    #     0      1      2      3      4      5
+ *         [0]    [ ]    [ ]    [ ]    [0]    [ ]
+ *         [ ]    [1]    [ ]    [ ]    [ ]    [1]
+ *         [2]    [ ]    [2]    [ ]    [ ]    [ ]
+ *         [ ]    [3]    [ ]    [3]    [ ]    [ ]
+ *         [ ]    [ ]    [4]    [ ]    [4]    [ ]
+ *         [ ]    [ ]    [ ]    [5]    [ ]    [5]
+ *   Step 1: Each process sends its own block to process r + l/2 and receives another from r - l/2.
+ *   The block received on the previous step is ignored to avoid a future double-write.  
+ *    #     0      1      2      3      4      5
+ *         [0]    [ ]    [0]    [ ]    [0]    [ ]
+ *         [ ]    [1]    [ ]    [1]    [ ]    [1]
+ *         [2]    [ ]    [2]    [ ]    [2]    [ ]
+ *         [ ]    [3]    [ ]    [3]    [ ]    [3]
+ *         [4]    [ ]    [4]    [ ]    [4]    [ ]
+ *         [ ]    [5]    [ ]    [5]    [ ]    [5]
+ *   Step 1: Each process sends all the data it has (3 blocks) to process r + l/4 and similarly
+ *   receives all the data from process r - l/4. 
+ *    #     0      1      2      3      4      5
+ *         [0]    [0]    [0]    [0]    [0]    [0]
+ *         [1]    [1]    [1]    [1]    [1]    [1]
+ *         [2]    [2]    [2]    [2]    [2]    [2]
+ *         [3]    [3]    [3]    [3]    [3]    [3]
+ *         [4]    [4]    [4]    [4]    [4]    [4]
+ *         [5]    [5]    [5]    [5]    [5]    [5]
+ */
+
+int ompi_coll_base_allgatherv_intra_sparbit(const void *sbuf, size_t scount,
+                                           struct ompi_datatype_t *sdtype,
+                                           void* rbuf, ompi_count_array_t rcounts,
+                                           ompi_disp_array_t rdispls,
+                                           struct ompi_datatype_t *rdtype,
+                                           struct ompi_communicator_t *comm,
+                                           mca_coll_base_module_t *module)
+{
+    /* ################# VARIABLE DECLARATION, BUFFER CREATION AND PREPARATION FOR THE ALGORITHM ######################## */
+
+    /* list of variable declaration */
+    int rank = 0, comm_size = 0, comm_log = 0, exclusion = 0; 
+    int data_expected = 1, transfer_count = 0, step_requests = 0;
+    int sendto, recvfrom, send_disp, recv_disp;
+    uint32_t last_ignore, ignore_steps, distance = 1;
+
+    int err = 0;
+    int line = -1;
+
+    ptrdiff_t rlb, rext;
+
+    char *tmpsend = NULL, *tmprecv = NULL;
+
+    MPI_Request *requests = NULL;
+
+    /* printf("utilizando o allgatherv novo!!\n"); */
+
+    /* algorithm choice information printing */
+    OPAL_OUTPUT((ompi_coll_base_framework.framework_output, 
+                 "coll:sparbit:allgather_sync_intra rank %d", rank));
+
+    comm_size = ompi_comm_size(comm);
+    rank = ompi_comm_rank(comm);
+
+    err = ompi_datatype_get_extent(rdtype, &rlb, &rext);
+    if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
+
+    /* if the MPI_IN_PLACE condition is not set, copy the send buffer to the receive buffer to perform the sends (all the data is extracted and forwarded from the recv buffer)*/
+    /* tmprecv and tmpsend are used as abstract pointers to simplify send and receive buffer choice */
+    tmprecv = (char *) rbuf;
+    if(MPI_IN_PLACE != sbuf){
+        tmpsend = (char *) sbuf; 
+        err = ompi_datatype_sndrcv(tmpsend, scount, sdtype, tmprecv + ompi_disp_array_get(rdispls, rank) * rext,
+                                   scount, rdtype);
+        if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl;  }
+    }
+    tmpsend = tmprecv;
+
+    requests = (MPI_Request *) malloc(comm_size * sizeof(MPI_Request));
+    
+    /* ################# ALGORITHM LOGIC ######################## */
+
+    /* calculate log2 of the total process count */
+    comm_log = ceil(log(comm_size)/log(2));
+    distance <<= comm_log - 1;
+
+    last_ignore = __builtin_ctz(comm_size);
+    ignore_steps = (~((uint32_t) comm_size >> last_ignore) | 1) << last_ignore;
+
+    /* perform the parallel binomial tree distribution steps */
+    for (int i = 0; i < comm_log; ++i) {
+       sendto = (rank + distance) % comm_size;  
+       recvfrom = (rank - distance + comm_size) % comm_size;  
+       exclusion = (distance & ignore_steps) == distance;
+
+       for (transfer_count = 0; transfer_count < data_expected - exclusion; transfer_count++) {
+           send_disp = (rank - 2 * transfer_count * distance + comm_size) % comm_size;
+           recv_disp = (rank - (2 * transfer_count + 1) * distance + comm_size) % comm_size;
+
+           /* Since each process sends several non-contiguos blocks of data to the same destination,
+            * each block sent (and therefore each send and recv call) needs a different tag. */
+           /* As base OpenMPI only provides one tag for allgather, we are forced to use a tag space
+            * from other components in the send and recv calls */
+           if(ompi_count_array_get(rcounts, send_disp) > 0)
+               MCA_PML_CALL(isend(tmpsend + ompi_disp_array_get(rdispls, send_disp) * rext,
+                                  ompi_count_array_get(rcounts, send_disp), rdtype, sendto,
+                                  MCA_COLL_BASE_TAG_HCOLL_BASE - send_disp,
+                                  MCA_PML_BASE_SEND_STANDARD, comm, requests + step_requests++));
+           if(ompi_count_array_get(rcounts, recv_disp) > 0)
+               MCA_PML_CALL(irecv(tmprecv + ompi_disp_array_get(rdispls, recv_disp) * rext,
+                                  ompi_count_array_get(rcounts, recv_disp), rdtype, recvfrom,
+                                  MCA_COLL_BASE_TAG_HCOLL_BASE - recv_disp, comm,
+                                  requests + step_requests++));
+       }
+       ompi_request_wait_all(step_requests, requests, MPI_STATUSES_IGNORE);
+
+       distance >>= 1; 
+       /* calculates the data expected for the next step, based on the current number of blocks and eventual exclusions */
+       data_expected = (data_expected << 1) - exclusion;
+       exclusion = step_requests = 0;
+    }
+    
+    free(requests);
+
+    return OMPI_SUCCESS;
+
+err_hndl:
+    OPAL_OUTPUT((ompi_coll_base_framework.framework_output,  "%s:%4d\tError occurred %d, rank %2d",
+                 __FILE__, line, err, rank));
+    (void)line;  // silence compiler warning
+    return err;
+
+}
 
 /*
  * ompi_coll_base_allgatherv_intra_ring
@@ -218,9 +375,9 @@ int ompi_coll_base_allgatherv_intra_bruck(const void *sbuf, int scount,
  *               No additional memory requirements.
  *
  */
-int ompi_coll_base_allgatherv_intra_ring(const void *sbuf, int scount,
+int ompi_coll_base_allgatherv_intra_ring(const void *sbuf, size_t scount,
                                           struct ompi_datatype_t *sdtype,
-                                          void* rbuf, const int *rcounts, const int *rdisps,
+                                          void* rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
                                           struct ompi_datatype_t *rdtype,
                                           struct ompi_communicator_t *comm,
                                           mca_coll_base_module_t *module)
@@ -242,11 +399,11 @@ int ompi_coll_base_allgatherv_intra_ring(const void *sbuf, int scount,
        - if send buffer is not MPI_IN_PLACE, copy send buffer to
        the appropriate block of receive buffer
     */
-    tmprecv = (char*) rbuf + (ptrdiff_t)rdisps[rank] * rext;
+    tmprecv = (char*) rbuf + ompi_disp_array_get(rdisps, rank) * rext;
     if (MPI_IN_PLACE != sbuf) {
         tmpsend = (char*) sbuf;
         err = ompi_datatype_sndrcv(tmpsend, scount, sdtype,
-                                   tmprecv, rcounts[rank], rdtype);
+                                   tmprecv, ompi_count_array_get(rcounts, rank), rdtype);
         if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl;  }
     }
 
@@ -256,7 +413,7 @@ int ompi_coll_base_allgatherv_intra_ring(const void *sbuf, int scount,
        [(r - i - 1 + size) % size]
        - sends message to rank [(r + 1) % size] containing data from rank
        [(r - i + size) % size]
-       - sends message which starts at begining of rbuf and has size
+       - sends message which starts at beginning of rbuf and has size
     */
     sendto = (rank + 1) % size;
     recvfrom  = (rank - 1 + size) % size;
@@ -265,13 +422,13 @@ int ompi_coll_base_allgatherv_intra_ring(const void *sbuf, int scount,
         recvdatafrom = (rank - i - 1 + size) % size;
         senddatafrom = (rank - i + size) % size;
 
-        tmprecv = (char*)rbuf + rdisps[recvdatafrom] * rext;
-        tmpsend = (char*)rbuf + rdisps[senddatafrom] * rext;
+        tmprecv = (char*)rbuf + ompi_disp_array_get(rdisps, recvdatafrom) * rext;
+        tmpsend = (char*)rbuf + ompi_disp_array_get(rdisps, senddatafrom) * rext;
 
         /* Sendreceive */
-        err = ompi_coll_base_sendrecv(tmpsend, rcounts[senddatafrom], rdtype,
+        err = ompi_coll_base_sendrecv(tmpsend, ompi_count_array_get(rcounts, senddatafrom), rdtype,
                                        sendto, MCA_COLL_BASE_TAG_ALLGATHERV,
-                                       tmprecv, rcounts[recvdatafrom], rdtype,
+                                       tmprecv, ompi_count_array_get(rcounts, recvdatafrom), rdtype,
                                        recvfrom, MCA_COLL_BASE_TAG_ALLGATHERV,
                                        comm, MPI_STATUS_IGNORE, rank);
         if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
@@ -345,16 +502,18 @@ int ompi_coll_base_allgatherv_intra_ring(const void *sbuf, int scount,
  *         [5]    [5]    [5]    [5]    [5]    [5]
  */
 int
-ompi_coll_base_allgatherv_intra_neighborexchange(const void *sbuf, int scount,
+ompi_coll_base_allgatherv_intra_neighborexchange(const void *sbuf, size_t scount,
                                                   struct ompi_datatype_t *sdtype,
-                                                  void* rbuf, const int *rcounts, const int *rdispls,
+                                                  void* rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdispls,
                                                   struct ompi_datatype_t *rdtype,
                                                   struct ompi_communicator_t *comm,
                                                   mca_coll_base_module_t *module)
 {
     int line = -1, rank, size, i, even_rank, err = 0;
     int neighbor[2], offset_at_step[2], recv_data_from[2], send_data_from;
-    int new_scounts[2], new_sdispls[2], new_rcounts[2], new_rdispls[2];
+    size_t new_scounts[2], new_rcounts[2];
+    ptrdiff_t new_sdispls[2], new_rdispls[2];
+    int tmp_new_scounts[2], tmp_new_rcounts[2], tmp_new_sdispls[2], tmp_new_rdispls[2];
     ptrdiff_t rlb, rext;
     char *tmpsend = NULL, *tmprecv = NULL;
     struct ompi_datatype_t  *new_rdtype, *new_sdtype;
@@ -382,11 +541,11 @@ ompi_coll_base_allgatherv_intra_neighborexchange(const void *sbuf, int scount,
        - if send buffer is not MPI_IN_PLACE, copy send buffer to
        the appropriate block of receive buffer
     */
-    tmprecv = (char*) rbuf + (ptrdiff_t)rdispls[rank] * rext;
+    tmprecv = (char*) rbuf + ompi_disp_array_get(rdispls, rank) * rext;
     if (MPI_IN_PLACE != sbuf) {
         tmpsend = (char*) sbuf;
         err = ompi_datatype_sndrcv(tmpsend, scount, sdtype,
-                                   tmprecv, rcounts[rank], rdtype);
+                                   tmprecv, ompi_count_array_get(rcounts, rank), rdtype);
         if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl;  }
     }
 
@@ -413,15 +572,15 @@ ompi_coll_base_allgatherv_intra_neighborexchange(const void *sbuf, int scount,
        - Rest of the steps:
        update recv_data_from according to offset, and
        exchange two blocks with appropriate neighbor.
-       the send location becomes previous receve location.
+       the send location becomes previous receive location.
        Note, we need to create indexed datatype to send and receive these
        blocks properly.
     */
-    tmprecv = (char*)rbuf + (ptrdiff_t)rdispls[neighbor[0]] * rext;
-    tmpsend = (char*)rbuf + (ptrdiff_t)rdispls[rank] * rext;
-    err = ompi_coll_base_sendrecv(tmpsend, rcounts[rank], rdtype,
+    tmprecv = (char*)rbuf + ompi_disp_array_get(rdispls, neighbor[0]) * rext;
+    tmpsend = (char*)rbuf + ompi_disp_array_get(rdispls, rank) * rext;
+    err = ompi_coll_base_sendrecv(tmpsend, ompi_count_array_get(rcounts, rank), rdtype,
                                    neighbor[0], MCA_COLL_BASE_TAG_ALLGATHERV,
-                                   tmprecv, rcounts[neighbor[0]], rdtype,
+                                   tmprecv, ompi_count_array_get(rcounts, neighbor[0]), rdtype,
                                    neighbor[0], MCA_COLL_BASE_TAG_ALLGATHERV,
                                    comm, MPI_STATUS_IGNORE, rank);
     if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
@@ -443,21 +602,31 @@ ompi_coll_base_allgatherv_intra_neighborexchange(const void *sbuf, int scount,
            We are receiving data from ranks (recv_data_from[i_parity]) and
            (recv_data_from[i_parity]+1).
         */
-        new_scounts[0] = rcounts[send_data_from];
-        new_scounts[1] = rcounts[(send_data_from + 1)];
-        new_sdispls[0] = rdispls[send_data_from];
-        new_sdispls[1] = rdispls[(send_data_from + 1)];
-        err = ompi_datatype_create_indexed(2, new_scounts, new_sdispls, rdtype,
+        new_scounts[0] = ompi_count_array_get(rcounts, send_data_from);
+        new_scounts[1] = ompi_count_array_get(rcounts, send_data_from + 1);
+        new_sdispls[0] = ompi_disp_array_get(rdispls, send_data_from);
+        new_sdispls[1] = ompi_disp_array_get(rdispls, send_data_from + 1);
+        /* TODO:BIGCOUNT: Remove temporaries once ompi_datatype interface is updated */
+        tmp_new_scounts[0] = new_scounts[0];
+        tmp_new_scounts[1] = new_scounts[1];
+        tmp_new_sdispls[0] = new_sdispls[0];
+        tmp_new_sdispls[1] = new_sdispls[1];
+        err = ompi_datatype_create_indexed(2, tmp_new_scounts, tmp_new_sdispls, rdtype,
                                            &new_sdtype);
         if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
         err = ompi_datatype_commit(&new_sdtype);
         if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
 
-        new_rcounts[0] = rcounts[recv_data_from[i_parity]];
-        new_rcounts[1] = rcounts[(recv_data_from[i_parity] + 1)];
-        new_rdispls[0] = rdispls[recv_data_from[i_parity]];
-        new_rdispls[1] = rdispls[(recv_data_from[i_parity] + 1)];
-        err = ompi_datatype_create_indexed(2, new_rcounts, new_rdispls, rdtype,
+        new_rcounts[0] = ompi_count_array_get(rcounts, recv_data_from[i_parity]);
+        new_rcounts[1] = ompi_count_array_get(rcounts, recv_data_from[i_parity] + 1);
+        new_rdispls[0] = ompi_disp_array_get(rdispls, recv_data_from[i_parity]);
+        new_rdispls[1] = ompi_disp_array_get(rdispls, recv_data_from[i_parity] + 1);
+        /* TODO:BIGCOUNT: Remove temporaries once ompi_datatype interface is updated */
+        tmp_new_rcounts[0] = new_rcounts[0];
+        tmp_new_rcounts[1] = new_rcounts[1];
+        tmp_new_rdispls[0] = new_rdispls[0];
+        tmp_new_rdispls[1] = new_rdispls[1];
+        err = ompi_datatype_create_indexed(2, tmp_new_rcounts, tmp_new_rdispls, rdtype,
                                            &new_rdtype);
         if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
         err = ompi_datatype_commit(&new_rdtype);
@@ -490,10 +659,10 @@ ompi_coll_base_allgatherv_intra_neighborexchange(const void *sbuf, int scount,
 }
 
 
-int ompi_coll_base_allgatherv_intra_two_procs(const void *sbuf, int scount,
+int ompi_coll_base_allgatherv_intra_two_procs(const void *sbuf, size_t scount,
                                                struct ompi_datatype_t *sdtype,
-                                               void* rbuf, const int *rcounts,
-                                               const int *rdispls,
+                                               void* rbuf, ompi_count_array_t rcounts,
+                                               ompi_disp_array_t rdispls,
                                                struct ompi_datatype_t *rdtype,
                                                struct ompi_communicator_t *comm,
                                                mca_coll_base_module_t *module)
@@ -522,15 +691,15 @@ int ompi_coll_base_allgatherv_intra_two_procs(const void *sbuf, int scount,
 
     tmpsend = (char*)sbuf;
     if (MPI_IN_PLACE == sbuf) {
-        tmpsend = (char*)rbuf + (ptrdiff_t)rdispls[rank] * rext;
-        scount = rcounts[rank];
+        tmpsend = (char*)rbuf + ompi_disp_array_get(rdispls, rank) * rext;
+        scount = ompi_count_array_get(rcounts, rank);
         sdtype = rdtype;
     }
-    tmprecv = (char*)rbuf + (ptrdiff_t)rdispls[remote] * rext;
+    tmprecv = (char*)rbuf + ompi_disp_array_get(rdispls, remote) * rext;
 
     err = ompi_coll_base_sendrecv(tmpsend, scount, sdtype, remote,
                                    MCA_COLL_BASE_TAG_ALLGATHERV,
-                                   tmprecv, rcounts[remote], rdtype, remote,
+                                   tmprecv, ompi_count_array_get(rcounts, remote), rdtype, remote,
                                    MCA_COLL_BASE_TAG_ALLGATHERV,
                                    comm, MPI_STATUS_IGNORE, rank);
     if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
@@ -538,8 +707,8 @@ int ompi_coll_base_allgatherv_intra_two_procs(const void *sbuf, int scount,
     /* Place your data in correct location if necessary */
     if (MPI_IN_PLACE != sbuf) {
         err = ompi_datatype_sndrcv((char*)sbuf, scount, sdtype,
-                                   (char*)rbuf + (ptrdiff_t)rdispls[rank] * rext,
-                                   rcounts[rank], rdtype);
+                                   (char*)rbuf + ompi_disp_array_get(rdispls, rank) * rext,
+                                   ompi_count_array_get(rcounts, rank), rdtype);
         if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl;  }
     }
 
@@ -576,10 +745,10 @@ int ompi_coll_base_allgatherv_intra_two_procs(const void *sbuf, int scount,
  *	Returns:	- MPI_SUCCESS or error code
  */
 int
-ompi_coll_base_allgatherv_intra_basic_default(const void *sbuf, int scount,
+ompi_coll_base_allgatherv_intra_basic_default(const void *sbuf, size_t scount,
                                               struct ompi_datatype_t *sdtype,
-                                              void *rbuf, const int *rcounts,
-                                              const int *disps,
+                                              void *rbuf, ompi_count_array_t rcounts,
+                                              ompi_disp_array_t disps,
                                               struct ompi_datatype_t *rdtype,
                                               struct ompi_communicator_t *comm,
                                               mca_coll_base_module_t *module)
@@ -588,6 +757,7 @@ ompi_coll_base_allgatherv_intra_basic_default(const void *sbuf, int scount,
     MPI_Aint extent, lb;
     char *send_buf = NULL;
     struct ompi_datatype_t *newtype, *send_type;
+    int *tmp_rcounts, *tmp_disps;
 
     size = ompi_comm_size(comm);
     rank = ompi_comm_rank(comm);
@@ -604,8 +774,8 @@ ompi_coll_base_allgatherv_intra_basic_default(const void *sbuf, int scount,
         ompi_datatype_get_extent(rdtype, &lb, &extent);
         send_type = rdtype;
         send_buf = (char*)rbuf;
-        send_buf += ((ptrdiff_t)disps[rank] * extent);
-        scount = rcounts[rank];
+        send_buf += (ompi_disp_array_get(disps, rank) * extent);
+        scount = ompi_count_array_get(rcounts, rank);
     } else {
         send_buf = (char*)sbuf;
         send_type = sdtype;
@@ -631,7 +801,22 @@ ompi_coll_base_allgatherv_intra_basic_default(const void *sbuf, int scount,
      * datatype.
      */
 
-    err = ompi_datatype_create_indexed(size,rcounts,disps,rdtype,&newtype);
+    /* TODO:BIGCOUNT: Remove temporaries once ompi_datatype interface is updated */
+    tmp_rcounts = malloc(size * sizeof(int));
+    if (NULL == tmp_rcounts) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+    tmp_disps = malloc(size * sizeof(int));
+    if (NULL == tmp_disps) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+    for (int i = 0; i < size; i++) {
+        tmp_rcounts[i] = ompi_count_array_get(rcounts, i);
+        tmp_disps[i] = ompi_disp_array_get(disps, i);
+    }
+    err = ompi_datatype_create_indexed(size,tmp_rcounts,tmp_disps,rdtype,&newtype);
+    free(tmp_rcounts);
+    free(tmp_disps);
     if (MPI_SUCCESS != err) {
         return err;
     }
diff --git a/ompi/mca/coll/base/coll_base_allreduce.c b/ompi/mca/coll/base/coll_base_allreduce.c
index 828b32061a9..82d57328b2e 100644
--- a/ompi/mca/coll/base/coll_base_allreduce.c
+++ b/ompi/mca/coll/base/coll_base_allreduce.c
@@ -17,6 +17,10 @@
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2018      Siberian State University of Telecommunications
  *                         and Information Science. All rights reserved.
+ * Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c)           Amazon.com, Inc. or its affiliates.
+ *                         All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -51,7 +55,7 @@
  *
  */
 int
-ompi_coll_base_allreduce_intra_nonoverlapping(const void *sbuf, void *rbuf, int count,
+ompi_coll_base_allreduce_intra_nonoverlapping(const void *sbuf, void *rbuf, size_t count,
                                                struct ompi_datatype_t *dtype,
                                                struct ompi_op_t *op,
                                                struct ompi_communicator_t *comm,
@@ -128,7 +132,7 @@ ompi_coll_base_allreduce_intra_nonoverlapping(const void *sbuf, void *rbuf, int
  */
 int
 ompi_coll_base_allreduce_intra_recursivedoubling(const void *sbuf, void *rbuf,
-                                                  int count,
+                                                  size_t count,
                                                   struct ompi_datatype_t *dtype,
                                                   struct ompi_op_t *op,
                                                   struct ompi_communicator_t *comm,
@@ -338,7 +342,7 @@ ompi_coll_base_allreduce_intra_recursivedoubling(const void *sbuf, void *rbuf,
  *
  */
 int
-ompi_coll_base_allreduce_intra_ring(const void *sbuf, void *rbuf, int count,
+ompi_coll_base_allreduce_intra_ring(const void *sbuf, void *rbuf, size_t count,
                                      struct ompi_datatype_t *dtype,
                                      struct ompi_op_t *op,
                                      struct ompi_communicator_t *comm,
@@ -356,7 +360,7 @@ ompi_coll_base_allreduce_intra_ring(const void *sbuf, void *rbuf, int count,
     rank = ompi_comm_rank(comm);
 
     OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
-                 "coll:base:allreduce_intra_ring rank %d, count %d", rank, count));
+                 "coll:base:allreduce_intra_ring rank %d, count %zu", rank, count));
 
     /* Special case for size == 1 */
     if (1 == size) {
@@ -368,8 +372,8 @@ ompi_coll_base_allreduce_intra_ring(const void *sbuf, void *rbuf, int count,
     }
 
     /* Special case for count less than size - use recursive doubling */
-    if (count < size) {
-        OPAL_OUTPUT((ompi_coll_base_framework.framework_output, "coll:base:allreduce_ring rank %d/%d, count %d, switching to recursive doubling", rank, size, count));
+    if (count < (size_t) size) {
+        OPAL_OUTPUT((ompi_coll_base_framework.framework_output, "coll:base:allreduce_ring rank %d/%d, count %zu, switching to recursive doubling", rank, size, count));
         return (ompi_coll_base_allreduce_intra_recursivedoubling(sbuf, rbuf,
                                                                   count,
                                                                   dtype, op,
@@ -424,7 +428,7 @@ ompi_coll_base_allreduce_intra_ring(const void *sbuf, void *rbuf, int count,
        - wait on block (r + 1)
        - compute on block (r + 1)
        - send block (r + 1) to rank (r + 1)
-       Note that we must be careful when computing the begining of buffers and
+       Note that we must be careful when computing the beginning of buffers and
        for send operations and computation we must compute the exact block size.
     */
     send_to = (rank + 1) % size;
@@ -615,7 +619,7 @@ ompi_coll_base_allreduce_intra_ring(const void *sbuf, void *rbuf, int count,
  *
  */
 int
-ompi_coll_base_allreduce_intra_ring_segmented(const void *sbuf, void *rbuf, int count,
+ompi_coll_base_allreduce_intra_ring_segmented(const void *sbuf, void *rbuf, size_t count,
                                                struct ompi_datatype_t *dtype,
                                                struct ompi_op_t *op,
                                                struct ompi_communicator_t *comm,
@@ -624,8 +628,8 @@ ompi_coll_base_allreduce_intra_ring_segmented(const void *sbuf, void *rbuf, int
 {
     int ret, line, rank, size, k, recv_from, send_to;
     int early_blockcount, late_blockcount, split_rank;
-    int segcount, max_segcount, num_phases, phase, block_count, inbi;
-    size_t typelng;
+    int num_phases, phase, block_count, inbi;
+    size_t typelng, segcount, max_segcount;
     char *tmpsend = NULL, *tmprecv = NULL, *inbuf[2] = {NULL, NULL};
     ptrdiff_t block_offset, max_real_segsize;
     ompi_request_t *reqs[2] = {MPI_REQUEST_NULL, MPI_REQUEST_NULL};
@@ -635,7 +639,7 @@ ompi_coll_base_allreduce_intra_ring_segmented(const void *sbuf, void *rbuf, int
     rank = ompi_comm_rank(comm);
 
     OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
-                 "coll:base:allreduce_intra_ring_segmented rank %d, count %d", rank, count));
+                 "coll:base:allreduce_intra_ring_segmented rank %d, count %zu", rank, count));
 
     /* Special case for size == 1 */
     if (1 == size) {
@@ -653,16 +657,16 @@ ompi_coll_base_allreduce_intra_ring_segmented(const void *sbuf, void *rbuf, int
     COLL_BASE_COMPUTED_SEGCOUNT(segsize, typelng, segcount)
 
         /* Special case for count less than size * segcount - use regular ring */
-        if (count < (size * segcount)) {
-            OPAL_OUTPUT((ompi_coll_base_framework.framework_output, "coll:base:allreduce_ring_segmented rank %d/%d, count %d, switching to regular ring", rank, size, count));
+        if (count < (size_t) (size * segcount)) {
+            OPAL_OUTPUT((ompi_coll_base_framework.framework_output, "coll:base:allreduce_ring_segmented rank %d/%d, count %zu, switching to regular ring", rank, size, count));
             return (ompi_coll_base_allreduce_intra_ring(sbuf, rbuf, count, dtype, op,
                                                          comm, module));
         }
 
     /* Determine the number of phases of the algorithm */
     num_phases = count / (size * segcount);
-    if ((count % (size * segcount) >= size) &&
-        (count % (size * segcount) > ((size * segcount) / 2))) {
+    if ((count % (size * segcount) >= (size_t) size) &&
+        (count % (size * segcount) > (size_t) ((size * segcount) / 2))) {
         num_phases++;
     }
 
@@ -717,7 +721,7 @@ ompi_coll_base_allreduce_intra_ring_segmented(const void *sbuf, void *rbuf, int
            - wait on block (r + 1)
            - compute on block (r + 1)
            - send block (r + 1) to rank (r + 1)
-           Note that we must be careful when computing the begining of buffers and
+           Note that we must be careful when computing the beginning of buffers and
            for send operations and computation we must compute the exact block size.
         */
         send_to = (rank + 1) % size;
@@ -878,7 +882,7 @@ ompi_coll_base_allreduce_intra_ring_segmented(const void *sbuf, void *rbuf, int
  *	Returns:	- MPI_SUCCESS or error code
  */
 int
-ompi_coll_base_allreduce_intra_basic_linear(const void *sbuf, void *rbuf, int count,
+ompi_coll_base_allreduce_intra_basic_linear(const void *sbuf, void *rbuf, size_t count,
                                              struct ompi_datatype_t *dtype,
                                              struct ompi_op_t *op,
                                              struct ompi_communicator_t *comm,
@@ -968,7 +972,7 @@ ompi_coll_base_allreduce_intra_basic_linear(const void *sbuf, void *rbuf, int co
  *   count * typesize + 4 * \log_2(p) * sizeof(int) = O(count)
  */
 int ompi_coll_base_allreduce_intra_redscat_allgather(
-    const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype,
+    const void *sbuf, void *rbuf, size_t count, struct ompi_datatype_t *dtype,
     struct ompi_op_t *op, struct ompi_communicator_t *comm,
     mca_coll_base_module_t *module)
 {
@@ -980,20 +984,21 @@ int ompi_coll_base_allreduce_intra_redscat_allgather(
                  "coll:base:allreduce_intra_redscat_allgather: rank %d/%d",
                  rank, comm_size));
 
-    /* Find nearest power-of-two less than or equal to comm_size */
-    int nsteps = opal_hibit(comm_size, comm->c_cube_dim + 1);   /* ilog2(comm_size) */
-    assert(nsteps >= 0);
-    int nprocs_pof2 = 1 << nsteps;                              /* flp2(comm_size) */
-
-    if (count < nprocs_pof2 || !ompi_op_is_commute(op)) {
+    if (!ompi_op_is_commute(op)) {
         OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
                      "coll:base:allreduce_intra_redscat_allgather: rank %d/%d "
-                     "count %d switching to basic linear allreduce",
+                     "count %zu switching to basic linear allreduce",
                      rank, comm_size, count));
         return ompi_coll_base_allreduce_intra_basic_linear(sbuf, rbuf, count, dtype,
                                                            op, comm, module);
     }
 
+    /* Find nearest power-of-two less than or equal to comm_size */
+    int nsteps = opal_hibit(comm_size, comm->c_cube_dim + 1);   /* ilog2(comm_size) */
+    if (-1 == nsteps) {
+        return MPI_ERR_ARG;
+    }
+    int nprocs_pof2 = 1 << nsteps;                              /* flp2(comm_size) */
     int err = MPI_SUCCESS;
     ptrdiff_t lb, extent, dsize, gap = 0;
     ompi_datatype_get_extent(dtype, &lb, &extent);
@@ -1122,7 +1127,7 @@ int ompi_coll_base_allreduce_intra_redscat_allgather(
 
         for (int mask = 1; mask < nprocs_pof2; mask <<= 1) {
             /*
-             * On each iteration: rindex[step] = sindex[step] -- begining of the
+             * On each iteration: rindex[step] = sindex[step] -- beginning of the
              * current window. Length of the current window is storded in wsize.
              */
             int vdest = vrank ^ mask;
@@ -1242,4 +1247,98 @@ int ompi_coll_base_allreduce_intra_redscat_allgather(
     return err;
 }
 
+/*
+ *   ompi_coll_base_allreduce_intra_allgather_reduce
+ *
+ *   Function:       use allgather for allreduce operation
+ *   Accepts:        Same as MPI_Allreduce()
+ *   Returns:        MPI_SUCCESS or error code
+ *
+ *   Description:    Implements allgather based allreduce aimed to improve internode 
+ *                   allreduce latency: this method takes advantage of the send and 
+ *                   receive can happen at the same time; first step is allgather
+ *                   operation to allow all ranks to obtain the full dataset; the second
+ *                   step is to do reduction on all ranks to get allreduce result. 
+ *
+ *   Limitations:    This method is designed for small message sizes allreduce because it 
+ *                   is not efficient in terms of network bandwidth comparing
+ *                   to gather/reduce/bcast type of approach.
+ */
+int ompi_coll_base_allreduce_intra_allgather_reduce(const void *sbuf, void *rbuf, size_t count,
+                                                    struct ompi_datatype_t *dtype,
+                                                    struct ompi_op_t *op,
+                                                    struct ompi_communicator_t *comm,
+                                                    mca_coll_base_module_t *module)
+{
+    int line = -1;
+    char *partial_buf = NULL;
+    char *partial_buf_start = NULL;
+    char *sendtmpbuf = NULL;
+    char *tmpsend = NULL;
+    char *tmpsend_start = NULL;
+    int err = OMPI_SUCCESS;
+
+    ptrdiff_t extent, lb;
+    ompi_datatype_get_extent(dtype, &lb, &extent);
+
+    int size = ompi_comm_size(comm);
+
+    sendtmpbuf = (char*) sbuf;
+    if( sbuf == MPI_IN_PLACE ) {
+        sendtmpbuf = (char *)rbuf;
+    }
+    ptrdiff_t buf_size, gap = 0;
+    buf_size = opal_datatype_span(&dtype->super, (int64_t)count * size, &gap);
+    partial_buf = (char *) malloc(buf_size);
+    partial_buf_start = partial_buf - gap;
+    buf_size = opal_datatype_span(&dtype->super, (int64_t)count, &gap);
+    tmpsend = (char *) malloc(buf_size);
+    tmpsend_start = tmpsend - gap;
+
+    err = ompi_datatype_copy_content_same_ddt(dtype, count,
+                                              (char*)tmpsend_start,
+                                              (char*)sendtmpbuf);
+    if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
+
+    // apply allgather data so that each rank has a full copy to do reduce (trade bandwidth for better latency)
+    err = comm->c_coll->coll_allgather(tmpsend_start, count, dtype,
+                                       partial_buf_start, count, dtype,
+                                       comm, comm->c_coll->coll_allgather_module);
+    if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
+
+    for (int target = 1; target < size; target++) {
+        ompi_op_reduce(op,
+                       partial_buf_start + (ptrdiff_t)target * count * extent,
+                       partial_buf_start,
+                       count,
+                       dtype);
+    }
+
+    // move data to rbuf
+    err = ompi_datatype_copy_content_same_ddt(dtype, count,
+                                              (char*)rbuf,
+                                              (char*)partial_buf_start);
+    if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
+
+    if (NULL != partial_buf) free(partial_buf);
+    if (NULL != tmpsend) free(tmpsend);
+    return MPI_SUCCESS;
+
+err_hndl:
+    if (NULL != partial_buf) {
+        free(partial_buf);
+        partial_buf = NULL;
+        partial_buf_start = NULL;
+    }
+     if (NULL != tmpsend) {
+        free(tmpsend);
+        tmpsend = NULL;
+        tmpsend_start = NULL;
+    }
+   OPAL_OUTPUT((ompi_coll_base_framework.framework_output,  "%s:%4d\tError occurred %d, rank %2d",
+                 __FILE__, line, err, ompi_comm_rank(comm)));
+    (void)line;  // silence compiler warning
+    return err;
+
+}
 /* copied function (with appropriate renaming) ends here */
diff --git a/ompi/mca/coll/base/coll_base_alltoall.c b/ompi/mca/coll/base/coll_base_alltoall.c
index a61bf40ca97..3f18f448dcd 100644
--- a/ompi/mca/coll/base/coll_base_alltoall.c
+++ b/ompi/mca/coll/base/coll_base_alltoall.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2023 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -14,7 +14,10 @@
  *                         reserved.
  * Copyright (c) 2014-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2017-2022 IBM Corporation. All rights reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
+ * Copyright (c) 2022      BULL S.A.S. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -27,6 +30,7 @@
 #include "mpi.h"
 #include "ompi/constants.h"
 #include "ompi/datatype/ompi_datatype.h"
+#include "opal/datatype/opal_convertor_internal.h"
 #include "ompi/communicator/communicator.h"
 #include "ompi/mca/coll/coll.h"
 #include "ompi/mca/coll/base/coll_tags.h"
@@ -35,93 +39,137 @@
 #include "coll_base_topo.h"
 #include "coll_base_util.h"
 
-/* MPI_IN_PLACE all to all algorithm. TODO: implement a better one. */
+/*
+ * We want to minimize the amount of temporary memory needed while allowing as many ranks
+ * to exchange data simultaneously. We use a variation of the ring algorithm, where in a
+ * single step a process echange the data with both neighbors at distance k (on the left
+ * and the right on a logical ring topology). With this approach we need to pack the data
+ * for a single of the two neighbors, as we can then use the original buffer (and datatype
+ * and count) to send the data to the other.
+ */
 int
-mca_coll_base_alltoall_intra_basic_inplace(const void *rbuf, int rcount,
+mca_coll_base_alltoall_intra_basic_inplace(const void *rbuf, size_t rcount,
                                            struct ompi_datatype_t *rdtype,
                                            struct ompi_communicator_t *comm,
                                            mca_coll_base_module_t *module)
 {
-    int i, j, size, rank, err = MPI_SUCCESS, line;
-    ptrdiff_t ext, gap = 0;
+    int i, size, rank, left, right, err = MPI_SUCCESS, line;
+    ptrdiff_t extent;
     ompi_request_t *req;
-    char *allocated_buffer = NULL, *tmp_buffer;
-    size_t max_size;
+    char *tmp_buffer;
+    size_t packed_size = 0, max_size;
+    opal_convertor_t convertor;
 
     /* Initialize. */
 
     size = ompi_comm_size(comm);
     rank = ompi_comm_rank(comm);
 
-    /* If only one process, we're done. */
-    if (1 == size) {
+    ompi_datatype_type_size(rdtype, &max_size);
+
+    /* Easy way out */
+    if ((1 == size) || (0 == rcount) || (0 == max_size) ) {
         return MPI_SUCCESS;
     }
 
-    /* Find the largest receive amount */
-    ompi_datatype_type_extent (rdtype, &ext);
-    max_size = opal_datatype_span(&rdtype->super, rcount, &gap);
+    /* Find the largest amount of packed send/recv data among all peers where
+     * we need to pack before the send.
+     */
+#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
+    for (i = 1 ; i <= (size >> 1) ; ++i) {
+        right = (rank + i) % size;
+        ompi_proc_t *ompi_proc = ompi_comm_peer_lookup(comm, right);
+
+        if( OPAL_UNLIKELY(opal_local_arch != ompi_proc->super.proc_convertor->master->remote_arch))  {
+            packed_size = opal_datatype_compute_remote_size(&rdtype->super,
+                                                            ompi_proc->super.proc_convertor->master->remote_sizes);
+            max_size = packed_size > max_size ? packed_size : max_size;
+        }
+    }
+#endif  /* OPAL_ENABLE_HETEROGENEOUS_SUPPORT */
+    max_size *= rcount;
 
-    /* Initiate all send/recv to/from others. */
+    ompi_datatype_type_extent(rdtype, &extent);
 
     /* Allocate a temporary buffer */
-    allocated_buffer = calloc (max_size, 1);
-    if( NULL == allocated_buffer) { err = OMPI_ERR_OUT_OF_RESOURCE; line = __LINE__; goto error_hndl; }
-    tmp_buffer = allocated_buffer - gap;
-    max_size = ext * rcount;
-
-    /* in-place alltoall slow algorithm (but works) */
-    for (i = 0 ; i < size ; ++i) {
-        for (j = i+1 ; j < size ; ++j) {
-            if (i == rank) {
-                /* Copy the data into the temporary buffer */
-                err = ompi_datatype_copy_content_same_ddt (rdtype, rcount, tmp_buffer,
-                                                       (char *) rbuf + j * max_size);
-                if (MPI_SUCCESS != err) { line = __LINE__; goto error_hndl; }
-
-                /* Exchange data with the peer */
-                err = MCA_PML_CALL(irecv ((char *) rbuf + max_size * j, rcount, rdtype,
-                                          j, MCA_COLL_BASE_TAG_ALLTOALL, comm, &req));
-                if (MPI_SUCCESS != err) { line = __LINE__; goto error_hndl; }
-
-                err = MCA_PML_CALL(send ((char *) tmp_buffer,  rcount, rdtype,
-                                          j, MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD,
-                                          comm));
-                if (MPI_SUCCESS != err) { line = __LINE__; goto error_hndl; }
-            } else if (j == rank) {
-                /* Copy the data into the temporary buffer */
-                err = ompi_datatype_copy_content_same_ddt (rdtype, rcount, tmp_buffer,
-                                                       (char *) rbuf + i * max_size);
-                if (MPI_SUCCESS != err) { line = __LINE__; goto error_hndl; }
-
-                /* Exchange data with the peer */
-                err = MCA_PML_CALL(irecv ((char *) rbuf + max_size * i, rcount, rdtype,
-                                          i, MCA_COLL_BASE_TAG_ALLTOALL, comm, &req));
-                if (MPI_SUCCESS != err) { line = __LINE__; goto error_hndl; }
-
-                err = MCA_PML_CALL(send ((char *) tmp_buffer,  rcount, rdtype,
-                                          i, MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD,
-                                          comm));
-                if (MPI_SUCCESS != err) { line = __LINE__; goto error_hndl; }
-            } else {
-                continue;
+    tmp_buffer = calloc (max_size, 1);
+    if( NULL == tmp_buffer) { err = OMPI_ERR_OUT_OF_RESOURCE; line = __LINE__; goto error_hndl; }
+
+    for (i = 1 ; i <= (size >> 1) ; ++i) {
+        struct iovec iov = {.iov_base = tmp_buffer, .iov_len = max_size};
+        uint32_t iov_count = 1;
+
+        right = (rank + i) % size;
+        left  = (rank + size - i) % size;
+
+        ompi_proc_t *right_proc = ompi_comm_peer_lookup(comm, right);
+        opal_convertor_clone(right_proc->super.proc_convertor, &convertor, 0);
+        opal_convertor_prepare_for_send(&convertor, &rdtype->super, rcount,
+                                        (char *) rbuf + (MPI_Aint) right * rcount * extent);
+        packed_size = max_size;
+        err = opal_convertor_pack(&convertor, &iov, &iov_count, &packed_size);
+        if (1 != err) {
+            line = __LINE__;
+            goto error_hndl;
+        }
+
+        /* Receive data from the right */
+        err = MCA_PML_CALL(irecv ((char *) rbuf + (MPI_Aint) right * rcount * extent, rcount, rdtype,
+                                  right, MCA_COLL_BASE_TAG_ALLTOALL, comm, &req));
+        if (MPI_SUCCESS != err) {
+            line = __LINE__;
+            goto error_hndl;
+        }
+
+        if( left != right ) {
+            /* Send data to the left */
+            err = MCA_PML_CALL(send ((char *) rbuf + (MPI_Aint) left * rcount * extent, rcount, rdtype,
+                                     left, MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD,
+                                     comm));
+            if (MPI_SUCCESS != err) {
+                line = __LINE__;
+                goto error_hndl;
+            }
+
+            err = ompi_request_wait (&req, MPI_STATUSES_IGNORE);
+            if (MPI_SUCCESS != err) {
+                line = __LINE__;
+                goto error_hndl;
+            }
+
+            /* Receive data from the left */
+            err = MCA_PML_CALL(irecv ((char *) rbuf + (MPI_Aint) left * rcount * extent, rcount, rdtype,
+                                      left, MCA_COLL_BASE_TAG_ALLTOALL, comm, &req));
+            if (MPI_SUCCESS != err) {
+                line = __LINE__;
+                goto error_hndl;
             }
+        }
 
-            /* Wait for the requests to complete */
-            err = ompi_request_wait ( &req, MPI_STATUSES_IGNORE);
-            if (MPI_SUCCESS != err) { line = __LINE__; goto error_hndl; }
+        /* Send data to the right */
+        err = MCA_PML_CALL(send ((char *) tmp_buffer,  packed_size, MPI_PACKED,
+                                 right, MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD,
+                                 comm));
+        if (MPI_SUCCESS != err) {
+            line = __LINE__;
+            goto error_hndl;
+        }
+
+        err = ompi_request_wait (&req, MPI_STATUSES_IGNORE);
+        if (MPI_SUCCESS != err) {
+            line = __LINE__;
+            goto error_hndl;
         }
     }
 
  error_hndl:
     /* Free the temporary buffer */
-    if( NULL != allocated_buffer )
-        free (allocated_buffer);
+    if( NULL != tmp_buffer )
+        free (tmp_buffer);
 
     if( MPI_SUCCESS != err ) {
         OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
-                     "%s:%4d\tError occurred %d, rank %2d", __FILE__, line, err,
-                     rank));
+                     "%s:%4d\tError occurred %d, rank %2d", __FILE__, line, err, rank));
         (void)line;  // silence compiler warning
     }
 
@@ -129,9 +177,9 @@ mca_coll_base_alltoall_intra_basic_inplace(const void *rbuf, int rcount,
     return err;
 }
 
-int ompi_coll_base_alltoall_intra_pairwise(const void *sbuf, int scount,
+int ompi_coll_base_alltoall_intra_pairwise(const void *sbuf, size_t scount,
                                             struct ompi_datatype_t *sdtype,
-                                            void* rbuf, int rcount,
+                                            void* rbuf, size_t rcount,
                                             struct ompi_datatype_t *rdtype,
                                             struct ompi_communicator_t *comm,
                                             mca_coll_base_module_t *module)
@@ -188,15 +236,15 @@ int ompi_coll_base_alltoall_intra_pairwise(const void *sbuf, int scount,
 }
 
 
-int ompi_coll_base_alltoall_intra_bruck(const void *sbuf, int scount,
+int ompi_coll_base_alltoall_intra_bruck(const void *sbuf, size_t scount,
                                          struct ompi_datatype_t *sdtype,
-                                         void* rbuf, int rcount,
+                                         void* rbuf, size_t rcount,
                                          struct ompi_datatype_t *rdtype,
                                          struct ompi_communicator_t *comm,
                                          mca_coll_base_module_t *module)
 {
-    int i, k, line = -1, rank, size, err = 0;
-    int sendto, recvfrom, distance, *displs = NULL, *blen = NULL;
+    int i, line = -1, rank, size, err = 0;
+    int sendto, recvfrom, distance, *displs = NULL;
     char *tmpbuf = NULL, *tmpbuf_free = NULL;
     ptrdiff_t sext, rext, span, gap = 0;
     struct ompi_datatype_t *new_ddt;
@@ -218,12 +266,7 @@ int ompi_coll_base_alltoall_intra_bruck(const void *sbuf, int scount,
     err = ompi_datatype_type_extent (rdtype, &rext);
     if (err != MPI_SUCCESS) { line = __LINE__; goto err_hndl; }
 
-    span = opal_datatype_span(&sdtype->super, (int64_t)size * scount, &gap);
-
-    displs = (int *) malloc(size * sizeof(int));
-    if (displs == NULL) { line = __LINE__; err = -1; goto err_hndl; }
-    blen = (int *) malloc(size * sizeof(int));
-    if (blen == NULL) { line = __LINE__; err = -1; goto err_hndl; }
+    span = opal_datatype_span(&rdtype->super, (int64_t)size * rcount, &gap);
 
     /* tmp buffer allocation for message data */
     tmpbuf_free = (char *)malloc(span);
@@ -231,18 +274,23 @@ int ompi_coll_base_alltoall_intra_bruck(const void *sbuf, int scount,
     tmpbuf = tmpbuf_free - gap;
 
     /* Step 1 - local rotation - shift up by rank */
-    err = ompi_datatype_copy_content_same_ddt (sdtype,
-                                               (int32_t) ((ptrdiff_t)(size - rank) * (ptrdiff_t)scount),
-                                               tmpbuf,
-                                               ((char*) sbuf) + (ptrdiff_t)rank * (ptrdiff_t)scount * sext);
+    err = ompi_datatype_sndrcv ((char*)sbuf + ((ptrdiff_t) rank * scount * sext),
+                                (int32_t) (size - rank) * scount,
+                                sdtype,
+                                tmpbuf,
+                                (int32_t) (size - rank) * rcount,
+                                rdtype);
     if (err<0) {
         line = __LINE__; err = -1; goto err_hndl;
     }
 
     if (rank != 0) {
-        err = ompi_datatype_copy_content_same_ddt (sdtype, (ptrdiff_t)rank * (ptrdiff_t)scount,
-                                                   tmpbuf + (ptrdiff_t)(size - rank) * (ptrdiff_t)scount* sext,
-                                                   (char*) sbuf);
+        err = ompi_datatype_sndrcv ((char*)sbuf,
+                                    (int32_t) rank * scount,
+                                    sdtype,
+                                    tmpbuf + ((ptrdiff_t) (size - rank) * rcount * rext),
+                                    (int32_t) rank * rcount,
+                                    rdtype);
         if (err<0) {
             line = __LINE__; err = -1; goto err_hndl;
         }
@@ -253,19 +301,19 @@ int ompi_coll_base_alltoall_intra_bruck(const void *sbuf, int scount,
 
         sendto = (rank + distance) % size;
         recvfrom = (rank - distance + size) % size;
-        k = 0;
-
-        /* create indexed datatype */
-        for (i = 1; i < size; i++) {
-            if (( i & distance) == distance) {
-                displs[k] = (ptrdiff_t)i * (ptrdiff_t)scount;
-                blen[k] = scount;
-                k++;
+
+        new_ddt = ompi_datatype_create((1 + size/distance) * (2 + rdtype->super.desc.used));
+
+        /* Create datatype describing data sent/received */
+        for (i = distance; i < size; i += 2*distance) {
+            int nblocks = distance;
+            if (i + distance >= size) {
+                nblocks = size - i;
             }
+            ompi_datatype_add(new_ddt, rdtype, rcount * nblocks,
+                              i * rcount * rext, rext);
         }
-        /* Set indexes and displacements */
-        err = ompi_datatype_create_indexed(k, blen, displs, sdtype, &new_ddt);
-        if (err != MPI_SUCCESS) { line = __LINE__; goto err_hndl;  }
+
         /* Commit the new datatype */
         err = ompi_datatype_commit(&new_ddt);
         if (err != MPI_SUCCESS) { line = __LINE__; goto err_hndl;  }
@@ -297,9 +345,7 @@ int ompi_coll_base_alltoall_intra_bruck(const void *sbuf, int scount,
     }
 
     /* Step 4 - clean up */
-    if (tmpbuf != NULL) free(tmpbuf_free);
-    if (displs != NULL) free(displs);
-    if (blen != NULL) free(blen);
+    if (tmpbuf_free != NULL) free(tmpbuf_free);
     return OMPI_SUCCESS;
 
  err_hndl:
@@ -307,9 +353,8 @@ int ompi_coll_base_alltoall_intra_bruck(const void *sbuf, int scount,
                  "%s:%4d\tError occurred %d, rank %2d", __FILE__, line, err,
                  rank));
     (void)line;  // silence compiler warning
-    if (tmpbuf != NULL) free(tmpbuf_free);
+    if (tmpbuf_free != NULL) free(tmpbuf_free);
     if (displs != NULL) free(displs);
-    if (blen != NULL) free(blen);
     return err;
 }
 
@@ -330,9 +375,9 @@ int ompi_coll_base_alltoall_intra_bruck(const void *sbuf, int scount,
  *                    - wait for any request to complete
  *                    - replace that request by the new one of the same type.
  */
-int ompi_coll_base_alltoall_intra_linear_sync(const void *sbuf, int scount,
+int ompi_coll_base_alltoall_intra_linear_sync(const void *sbuf, size_t scount,
                                                struct ompi_datatype_t *sdtype,
-                                               void* rbuf, int rcount,
+                                               void* rbuf, size_t rcount,
                                                struct ompi_datatype_t *rdtype,
                                                struct ompi_communicator_t *comm,
                                                mca_coll_base_module_t *module,
@@ -474,8 +519,10 @@ int ompi_coll_base_alltoall_intra_linear_sync(const void *sbuf, int scount,
         for( ri = 0; ri < nreqs; ri++ ) {
             if (MPI_REQUEST_NULL == reqs[ri]) continue;
             if (MPI_ERR_PENDING == reqs[ri]->req_status.MPI_ERROR) continue;
-            error = reqs[ri]->req_status.MPI_ERROR;
-            break;
+            if (reqs[ri]->req_status.MPI_ERROR != MPI_SUCCESS) {
+                error = reqs[ri]->req_status.MPI_ERROR;
+                break;
+            }
         }
     }
     OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
@@ -487,9 +534,9 @@ int ompi_coll_base_alltoall_intra_linear_sync(const void *sbuf, int scount,
 }
 
 
-int ompi_coll_base_alltoall_intra_two_procs(const void *sbuf, int scount,
+int ompi_coll_base_alltoall_intra_two_procs(const void *sbuf, size_t scount,
                                              struct ompi_datatype_t *sdtype,
-                                             void* rbuf, int rcount,
+                                             void* rbuf, size_t rcount,
                                              struct ompi_datatype_t *rdtype,
                                              struct ompi_communicator_t *comm,
                                              mca_coll_base_module_t *module)
@@ -566,9 +613,9 @@ int ompi_coll_base_alltoall_intra_two_procs(const void *sbuf, int scount,
 
 /* copied function (with appropriate renaming) starts here */
 
-int ompi_coll_base_alltoall_intra_basic_linear(const void *sbuf, int scount,
+int ompi_coll_base_alltoall_intra_basic_linear(const void *sbuf, size_t scount,
                                                struct ompi_datatype_t *sdtype,
-                                               void* rbuf, int rcount,
+                                               void* rbuf, size_t rcount,
                                                struct ompi_datatype_t *rdtype,
                                                struct ompi_communicator_t *comm,
                                                mca_coll_base_module_t *module)
@@ -677,8 +724,10 @@ int ompi_coll_base_alltoall_intra_basic_linear(const void *sbuf, int scount,
             for( i = 0; i < nreqs; i++ ) {
                 if (MPI_REQUEST_NULL == req[i]) continue;
                 if (MPI_ERR_PENDING == req[i]->req_status.MPI_ERROR) continue;
-                err = req[i]->req_status.MPI_ERROR;
-                break;
+                if (req[i]->req_status.MPI_ERROR != MPI_SUCCESS) {
+                    err = req[i]->req_status.MPI_ERROR;
+                    break;
+                }
             }
         }
         OPAL_OUTPUT( (ompi_coll_base_framework.framework_output,"%s:%4d\tError occurred %d, rank %2d",
diff --git a/ompi/mca/coll/base/coll_base_alltoallv.c b/ompi/mca/coll/base/coll_base_alltoallv.c
index dbe33e8eee4..37bc22acf5c 100644
--- a/ompi/mca/coll/base/coll_base_alltoallv.c
+++ b/ompi/mca/coll/base/coll_base_alltoallv.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -17,6 +17,8 @@
  * Copyright (c) 2014-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -29,6 +31,7 @@
 #include "mpi.h"
 #include "ompi/constants.h"
 #include "ompi/datatype/ompi_datatype.h"
+#include "opal/datatype/opal_convertor_internal.h"
 #include "ompi/communicator/communicator.h"
 #include "ompi/mca/coll/coll.h"
 #include "ompi/mca/coll/base/coll_tags.h"
@@ -36,101 +39,172 @@
 #include "ompi/mca/coll/base/coll_base_functions.h"
 #include "coll_base_topo.h"
 #include "coll_base_util.h"
+#include "opal/util/minmax.h"
 
+/*
+ * We want to minimize the amount of temporary memory needed while allowing as many ranks
+ * to exchange data simultaneously. We use a variation of the ring algorithm, where in a
+ * single step a process exchange the data with both neighbors at distance k (on the left
+ * and the right on a logical ring topology). With this approach we need to pack the data
+ * for a single of the two neighbors, as we can then use the original buffer (and datatype
+ * and count) to send the data to the other.
+ */
 int
-mca_coll_base_alltoallv_intra_basic_inplace(const void *rbuf, const int *rcounts, const int *rdisps,
+mca_coll_base_alltoallv_intra_basic_inplace(const void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
                                             struct ompi_datatype_t *rdtype,
                                             struct ompi_communicator_t *comm,
                                             mca_coll_base_module_t *module)
 {
-    int i, j, size, rank, err=MPI_SUCCESS;
-    char *allocated_buffer, *tmp_buffer;
-    size_t max_size;
-    ptrdiff_t ext, gap = 0;
+    int i, size, rank, left, right, err = MPI_SUCCESS, line;
+    ptrdiff_t extent;
+    ompi_request_t *req = MPI_REQUEST_NULL;
+    char *tmp_buffer;
+    size_t packed_size = 0, max_size, type_size;
+    opal_convertor_t convertor;
 
     /* Initialize. */
 
     size = ompi_comm_size(comm);
     rank = ompi_comm_rank(comm);
+    ompi_datatype_type_size(rdtype, &type_size);
 
-    /* If only one process, we're done. */
-    if (1 == size) {
-        return MPI_SUCCESS;
-    }
-    /* Find the largest receive amount */
-    ompi_datatype_type_extent (rdtype, &ext);
     for (i = 0, max_size = 0 ; i < size ; ++i) {
         if (i == rank) {
             continue;
         }
-        size_t size = opal_datatype_span(&rdtype->super, rcounts[i], &gap);
-        max_size = size > max_size ? size : max_size;
+        packed_size = ompi_count_array_get(rcounts, i) * type_size;
+        max_size = opal_max(packed_size, max_size);
     }
-    /* The gap will always be the same as we are working on the same datatype */
 
-    if (OPAL_UNLIKELY(0 == max_size)) {
+    /* Easy way out */
+    if ((1 == size) || (0 == max_size) ) {
         return MPI_SUCCESS;
     }
 
-    /* Allocate a temporary buffer */
-    allocated_buffer = calloc (max_size, 1);
-    if (NULL == allocated_buffer) {
-        return OMPI_ERR_OUT_OF_RESOURCE;
+    /* Find the largest amount of packed send/recv data among all peers where
+     * we need to pack before the send.
+     */
+#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
+    for (i = 1 ; i <= (size >> 1) ; ++i) {
+        right = (rank + i) % size;
+        ompi_proc_t *ompi_proc = ompi_comm_peer_lookup(comm, right);
+
+        if( OPAL_UNLIKELY(opal_local_arch != ompi_proc->super.proc_convertor->master->remote_arch))  {
+            packed_size = opal_datatype_compute_remote_size(&rdtype->super,
+                                                            ompi_proc->super.proc_convertor->master->remote_sizes);
+            packed_size *= ompi_count_array_get(rcounts, right);
+            max_size = packed_size > max_size ? packed_size : max_size;
+        }
     }
-    tmp_buffer = allocated_buffer - gap;
-
-    /* Initiate all send/recv to/from others. */
-    /* in-place alltoallv slow algorithm (but works) */
-    for (i = 0 ; i < size ; ++i) {
-        for (j = i+1 ; j < size ; ++j) {
-            if (i == rank && 0 != rcounts[j]) {
-                /* Copy the data into the temporary buffer */
-                err = ompi_datatype_copy_content_same_ddt (rdtype, rcounts[j],
-                                                           tmp_buffer, (char *) rbuf + rdisps[j] * ext);
-                if (MPI_SUCCESS != err) { goto error_hndl; }
-
-                /* Exchange data with the peer */
-                err = ompi_coll_base_sendrecv_actual((void *) tmp_buffer,  rcounts[j], rdtype,
-                                                     j, MCA_COLL_BASE_TAG_ALLTOALLV,
-                                                     (char *)rbuf + rdisps[j] * ext, rcounts[j], rdtype,
-                                                     j, MCA_COLL_BASE_TAG_ALLTOALLV,
-                                                     comm, MPI_STATUS_IGNORE);
-                if (MPI_SUCCESS != err) { goto error_hndl; }
-            } else if (j == rank && 0 != rcounts[i]) {
-                /* Copy the data into the temporary buffer */
-                err = ompi_datatype_copy_content_same_ddt (rdtype, rcounts[i],
-                                                           tmp_buffer, (char *) rbuf + rdisps[i] * ext);
-                if (MPI_SUCCESS != err) { goto error_hndl; }
-
-                /* Exchange data with the peer */
-                err = ompi_coll_base_sendrecv_actual((void *) tmp_buffer,  rcounts[i], rdtype,
-                                                     i, MCA_COLL_BASE_TAG_ALLTOALLV,
-                                                     (char *) rbuf + rdisps[i] * ext, rcounts[i], rdtype,
-                                                     i, MCA_COLL_BASE_TAG_ALLTOALLV,
-                                                     comm, MPI_STATUS_IGNORE);
-                if (MPI_SUCCESS != err) { goto error_hndl; }
+#endif  /* OPAL_ENABLE_HETEROGENEOUS_SUPPORT */
+
+    ompi_datatype_type_extent(rdtype, &extent);
+
+    /* Allocate a temporary buffer */
+    tmp_buffer = calloc (max_size, 1);
+    if( NULL == tmp_buffer) { err = OMPI_ERR_OUT_OF_RESOURCE; line = __LINE__; goto error_hndl; }
+
+    for (i = 1 ; i <= (size >> 1) ; ++i) {
+        struct iovec iov = {.iov_base = tmp_buffer, .iov_len = max_size};
+        uint32_t iov_count = 1;
+
+        right = (rank + i) % size;
+        left  = (rank + size - i) % size;
+
+        if( 0 != ompi_count_array_get(rcounts, right) ) {  /* nothing to exchange with the peer on the right */
+            ompi_proc_t *right_proc = ompi_comm_peer_lookup(comm, right);
+            opal_convertor_clone(right_proc->super.proc_convertor, &convertor, 0);
+            opal_convertor_prepare_for_send(&convertor, &rdtype->super, ompi_count_array_get(rcounts, right),
+                                            (char *) rbuf + ompi_disp_array_get(rdisps, right) * extent);
+            packed_size = max_size;
+            err = opal_convertor_pack(&convertor, &iov, &iov_count, &packed_size);
+            if (1 != err) {
+                line = __LINE__;
+                goto error_hndl;
+            }
+
+            /* Receive data from the right */
+            err = MCA_PML_CALL(irecv ((char *) rbuf + ompi_disp_array_get(rdisps, right) * extent,
+                                      ompi_count_array_get(rcounts, right), rdtype,
+                                      right, MCA_COLL_BASE_TAG_ALLTOALLV, comm, &req));
+            if (MPI_SUCCESS != err) {
+                line = __LINE__;
+                goto error_hndl;
+            }
+        }
+
+        if( (left != right) && (0 != ompi_count_array_get(rcounts, left)) ) {
+            /* Send data to the left */
+            err = MCA_PML_CALL(send ((char *) rbuf + ompi_disp_array_get(rdisps, left) * extent,
+                                     ompi_count_array_get(rcounts, left), rdtype,
+                                     left, MCA_COLL_BASE_TAG_ALLTOALLV, MCA_PML_BASE_SEND_STANDARD,
+                                     comm));
+            if (MPI_SUCCESS != err) {
+                line = __LINE__;
+                goto error_hndl;
+             }
+
+            err = ompi_request_wait (&req, MPI_STATUSES_IGNORE);
+            if (MPI_SUCCESS != err) {
+                line = __LINE__;
+                goto error_hndl;
+             }
+
+            /* Receive data from the left */
+            err = MCA_PML_CALL(irecv ((char *) rbuf + ompi_disp_array_get(rdisps, left) * extent,
+                                      ompi_count_array_get(rcounts, left), rdtype,
+                                      left, MCA_COLL_BASE_TAG_ALLTOALLV, comm, &req));
+            if (MPI_SUCCESS != err) {
+                line = __LINE__;
+                goto error_hndl;
+            }
+        }
+
+        if( 0 != ompi_count_array_get(rcounts, right) ) {  /* nothing to exchange with the peer on the right */
+            /* Send data to the right */
+            err = MCA_PML_CALL(send ((char *) tmp_buffer,  packed_size, MPI_PACKED,
+                                     right, MCA_COLL_BASE_TAG_ALLTOALLV, MCA_PML_BASE_SEND_STANDARD,
+                                     comm));
+            if (MPI_SUCCESS != err) {
+                line = __LINE__;
+                goto error_hndl;
             }
         }
+
+        err = ompi_request_wait (&req, MPI_STATUSES_IGNORE);
+        if (MPI_SUCCESS != err) {
+            line = __LINE__;
+            goto error_hndl;
+        }
     }
 
  error_hndl:
     /* Free the temporary buffer */
-    free (allocated_buffer);
+    if( NULL != tmp_buffer )
+        free (tmp_buffer);
+
+    if( MPI_SUCCESS != err ) {
+        OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
+                     "%s:%4d\tError occurred %d, rank %2d", __FILE__, line, err, rank));
+        (void)line;  // silence compiler warning
+    }
 
     /* All done */
     return err;
 }
 
 int
-ompi_coll_base_alltoallv_intra_pairwise(const void *sbuf, const int *scounts, const int *sdisps,
+ompi_coll_base_alltoallv_intra_pairwise(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
                                          struct ompi_datatype_t *sdtype,
-                                         void* rbuf, const int *rcounts, const int *rdisps,
+                                         void* rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
                                          struct ompi_datatype_t *rdtype,
                                          struct ompi_communicator_t *comm,
                                          mca_coll_base_module_t *module)
 {
     int line = -1, err = 0, rank, size, step = 0, sendto, recvfrom;
+    size_t sdtype_size, rdtype_size;
     void *psnd, *prcv;
+    ompi_request_t *req;
     ptrdiff_t sext, rext;
 
     if (MPI_IN_PLACE == sbuf) {
@@ -144,27 +218,50 @@ ompi_coll_base_alltoallv_intra_pairwise(const void *sbuf, const int *scounts, co
     OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
                  "coll:base:alltoallv_intra_pairwise rank %d", rank));
 
+    ompi_datatype_type_size(sdtype, &sdtype_size);
+    ompi_datatype_type_size(rdtype, &rdtype_size);
+
     ompi_datatype_type_extent(sdtype, &sext);
     ompi_datatype_type_extent(rdtype, &rext);
 
-   /* Perform pairwise exchange starting from 1 since local exhange is done */
+   /* Perform pairwise exchange starting from 1 since local exchange is done */
     for (step = 0; step < size; step++) {
+        req = MPI_REQUEST_NULL;
 
         /* Determine sender and receiver for this step. */
         sendto  = (rank + step) % size;
         recvfrom = (rank + size - step) % size;
 
         /* Determine sending and receiving locations */
-        psnd = (char*)sbuf + (ptrdiff_t)sdisps[sendto] * sext;
-        prcv = (char*)rbuf + (ptrdiff_t)rdisps[recvfrom] * rext;
+        psnd = (char*)sbuf + ompi_disp_array_get(sdisps, sendto) * sext;
+        prcv = (char*)rbuf + ompi_disp_array_get(rdisps, recvfrom) * rext;
 
         /* send and receive */
-        err = ompi_coll_base_sendrecv( psnd, scounts[sendto], sdtype, sendto,
-                                        MCA_COLL_BASE_TAG_ALLTOALLV,
-                                        prcv, rcounts[recvfrom], rdtype, recvfrom,
-                                        MCA_COLL_BASE_TAG_ALLTOALLV,
-                                        comm, MPI_STATUS_IGNORE, rank);
-        if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl;  }
+        if (0 < ompi_count_array_get(rcounts, recvfrom) && 0 < rdtype_size) {
+            err = MCA_PML_CALL(irecv(prcv, ompi_count_array_get(rcounts, recvfrom), rdtype, recvfrom,
+                                     MCA_COLL_BASE_TAG_ALLTOALLV, comm, &req));
+            if (MPI_SUCCESS != err) {
+                line = __LINE__;
+                goto err_hndl;
+            }
+        }
+
+        if (0 < ompi_count_array_get(scounts, sendto) && 0 < sdtype_size) {
+            err = MCA_PML_CALL(send(psnd, ompi_count_array_get(scounts, sendto), sdtype, sendto,
+                                    MCA_COLL_BASE_TAG_ALLTOALLV, MCA_PML_BASE_SEND_STANDARD, comm));
+            if (MPI_SUCCESS != err) {
+                line = __LINE__;
+                goto err_hndl;
+            }
+        }
+
+        if (MPI_REQUEST_NULL != req) {
+            err = ompi_request_wait(&req, MPI_STATUS_IGNORE);
+            if (MPI_SUCCESS != err) {
+                line = __LINE__;
+                goto err_hndl;
+            }
+        }
     }
 
     return MPI_SUCCESS;
@@ -186,14 +283,15 @@ ompi_coll_base_alltoallv_intra_pairwise(const void *sbuf, const int *scounts, co
  * differently and so will not have to duplicate code.
  */
 int
-ompi_coll_base_alltoallv_intra_basic_linear(const void *sbuf, const int *scounts, const int *sdisps,
+ompi_coll_base_alltoallv_intra_basic_linear(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
                                             struct ompi_datatype_t *sdtype,
-                                            void *rbuf, const int *rcounts, const int *rdisps,
+                                            void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
                                             struct ompi_datatype_t *rdtype,
                                             struct ompi_communicator_t *comm,
                                             mca_coll_base_module_t *module)
 {
     int i, size, rank, err, nreqs;
+    size_t sdtype_size = 0, rdtype_size = 0;
     char *psnd, *prcv;
     ptrdiff_t sext, rext;
     ompi_request_t **preq, **reqs;
@@ -211,15 +309,18 @@ ompi_coll_base_alltoallv_intra_basic_linear(const void *sbuf, const int *scounts
     OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
                  "coll:base:alltoallv_intra_basic_linear rank %d", rank));
 
+    ompi_datatype_type_size(rdtype, &rdtype_size);
+    ompi_datatype_type_size(sdtype, &sdtype_size);
+
     ompi_datatype_type_extent(sdtype, &sext);
     ompi_datatype_type_extent(rdtype, &rext);
 
     /* Simple optimization - handle send to self first */
-    psnd = ((char *) sbuf) + (ptrdiff_t)sdisps[rank] * sext;
-    prcv = ((char *) rbuf) + (ptrdiff_t)rdisps[rank] * rext;
-    if (0 != scounts[rank]) {
-        err = ompi_datatype_sndrcv(psnd, scounts[rank], sdtype,
-                              prcv, rcounts[rank], rdtype);
+    psnd = ((char *) sbuf) + ompi_disp_array_get(sdisps, rank) * sext;
+    prcv = ((char *) rbuf) + ompi_disp_array_get(rdisps, rank) * rext;
+    if (0 < ompi_count_array_get(scounts, rank) && 0 < sdtype_size) {
+        err = ompi_datatype_sndrcv(psnd, ompi_count_array_get(scounts, rank), sdtype,
+                              prcv, ompi_count_array_get(rcounts, rank), rdtype);
         if (MPI_SUCCESS != err) {
             return err;
         }
@@ -241,12 +342,14 @@ ompi_coll_base_alltoallv_intra_basic_linear(const void *sbuf, const int *scounts
             continue;
         }
 
-        ++nreqs;
-        prcv = ((char *) rbuf) + (ptrdiff_t)rdisps[i] * rext;
-        err = MCA_PML_CALL(irecv_init(prcv, rcounts[i], rdtype,
-                                      i, MCA_COLL_BASE_TAG_ALLTOALLV, comm,
-                                      preq++));
-        if (MPI_SUCCESS != err) { goto err_hndl; }
+        if (0 < ompi_count_array_get(rcounts, i) && 0 < rdtype_size) {
+            ++nreqs;
+            prcv = ((char *) rbuf) + ompi_disp_array_get(rdisps, i) * rext;
+            err = MCA_PML_CALL(irecv_init(prcv, ompi_count_array_get(rcounts, i), rdtype,
+                                          i, MCA_COLL_BASE_TAG_ALLTOALLV, comm,
+                                          preq++));
+            if (MPI_SUCCESS != err) { goto err_hndl; }
+        }
     }
 
     /* Now post all sends */
@@ -255,13 +358,15 @@ ompi_coll_base_alltoallv_intra_basic_linear(const void *sbuf, const int *scounts
             continue;
         }
 
-        ++nreqs;
-        psnd = ((char *) sbuf) + (ptrdiff_t)sdisps[i] * sext;
-        err = MCA_PML_CALL(isend_init(psnd, scounts[i], sdtype,
-                                      i, MCA_COLL_BASE_TAG_ALLTOALLV,
-                                      MCA_PML_BASE_SEND_STANDARD, comm,
-                                      preq++));
-        if (MPI_SUCCESS != err) { goto err_hndl; }
+        if (0 < ompi_count_array_get(scounts, i) && 0 < sdtype_size) {
+            ++nreqs;
+            psnd = ((char *) sbuf) + ompi_disp_array_get(sdisps, i) * sext;
+            err = MCA_PML_CALL(isend_init(psnd, ompi_count_array_get(scounts, i), sdtype,
+                                         i, MCA_COLL_BASE_TAG_ALLTOALLV,
+                                         MCA_PML_BASE_SEND_STANDARD, comm,
+                                         preq++));
+            if (MPI_SUCCESS != err) { goto err_hndl; }
+        }
     }
 
     /* Start your engines.  This will never return an error. */
@@ -281,8 +386,10 @@ ompi_coll_base_alltoallv_intra_basic_linear(const void *sbuf, const int *scounts
         for( i = 0; i < nreqs; i++ ) {
             if (MPI_REQUEST_NULL == reqs[i]) continue;
             if (MPI_ERR_PENDING == reqs[i]->req_status.MPI_ERROR) continue;
-            err = reqs[i]->req_status.MPI_ERROR;
-            break;
+            if (reqs[i]->req_status.MPI_ERROR != MPI_SUCCESS) {
+                err = reqs[i]->req_status.MPI_ERROR;
+                break;
+            }
         }
     }
     /* Free the requests in all cases as they are persistent */
diff --git a/ompi/mca/coll/base/coll_base_barrier.c b/ompi/mca/coll/base/coll_base_barrier.c
index 9975ba74b42..b6206f02666 100644
--- a/ompi/mca/coll/base/coll_base_barrier.c
+++ b/ompi/mca/coll/base/coll_base_barrier.c
@@ -83,7 +83,7 @@ ompi_coll_base_sendrecv_zero( int dest, int stag,
             if( MPI_ERR_PROC_FAILED_PENDING == rc ) {
                 rc = MPI_ERR_PROC_FAILED;
             }
-        } else /* this 'else' intentionaly spills outside the ifdef */
+        } else /* this 'else' intentionally spills outside the ifdef */
 #endif /* OPAL_ENABLE_FT_MPI */
         ompi_request_free(&req);
     }
@@ -95,7 +95,7 @@ ompi_coll_base_sendrecv_zero( int dest, int stag,
 }
 
 /*
- * Barrier is ment to be a synchronous operation, as some BTLs can mark
+ * Barrier is meant to be a synchronous operation, as some BTLs can mark
  * a request done before its passed to the NIC and progress might not be made
  * elsewhere we cannot allow a process to exit the barrier until its last
  * [round of] sends are completed.
@@ -110,7 +110,7 @@ ompi_coll_base_sendrecv_zero( int dest, int stag,
 /*
  * Simple double ring version of barrier
  *
- * synchronous gurantee made by last ring of sends are synchronous
+ * synchronous guarantee made by last ring of sends are synchronous
  *
  */
 int ompi_coll_base_barrier_intra_doublering(struct ompi_communicator_t *comm,
@@ -125,7 +125,7 @@ int ompi_coll_base_barrier_intra_doublering(struct ompi_communicator_t *comm,
 
     OPAL_OUTPUT((ompi_coll_base_framework.framework_output,"ompi_coll_base_barrier_intra_doublering rank %d", rank));
 
-    left = ((rank-1)%size);
+    left = ((size+rank-1)%size);
     right = ((rank+1)%size);
 
     if (rank > 0) { /* receive message from the left */
@@ -400,8 +400,10 @@ int ompi_coll_base_barrier_intra_basic_linear(struct ompi_communicator_t *comm,
             for( i = 0; i < size; i++ ) {
                 if (MPI_REQUEST_NULL == requests[i]) continue;
                 if (MPI_ERR_PENDING == requests[i]->req_status.MPI_ERROR) continue;
-                err = requests[i]->req_status.MPI_ERROR;
-                break;
+                if (requests[i]->req_status.MPI_ERROR != MPI_SUCCESS) {
+                    err = requests[i]->req_status.MPI_ERROR;
+                    break;
+                }
             }
         }
         ompi_coll_base_free_reqs(requests, size);
diff --git a/ompi/mca/coll/base/coll_base_bcast.c b/ompi/mca/coll/base/coll_base_bcast.c
index 7af75353d2d..dece487066f 100644
--- a/ompi/mca/coll/base/coll_base_bcast.c
+++ b/ompi/mca/coll/base/coll_base_bcast.c
@@ -36,7 +36,7 @@
 
 int
 ompi_coll_base_bcast_intra_generic( void* buffer,
-                                     int original_count,
+                                     size_t original_count,
                                      struct ompi_datatype_t* datatype,
                                      int root,
                                      struct ompi_communicator_t* comm,
@@ -46,7 +46,7 @@ ompi_coll_base_bcast_intra_generic( void* buffer,
 {
     int err = 0, line, i, rank, segindex, req_index;
     int num_segments; /* Number of segments */
-    int sendcount;    /* number of elements sent in this segment */
+    size_t sendcount;    /* number of elements sent in this segment */
     size_t realsegsize, type_size;
     char *tmpbuf;
     ptrdiff_t extent, lb;
@@ -218,8 +218,10 @@ ompi_coll_base_bcast_intra_generic( void* buffer,
         for( req_index = 0; req_index < 2; req_index++ ) {
             if (MPI_REQUEST_NULL == recv_reqs[req_index]) continue;
             if (MPI_ERR_PENDING == recv_reqs[req_index]->req_status.MPI_ERROR) continue;
-            err = recv_reqs[req_index]->req_status.MPI_ERROR;
-            break;
+            if (recv_reqs[req_index]->req_status.MPI_ERROR != MPI_SUCCESS) {
+                err = recv_reqs[req_index]->req_status.MPI_ERROR;
+                break;
+            }
         }
     }
     ompi_coll_base_free_reqs( recv_reqs, 2);
@@ -228,8 +230,10 @@ ompi_coll_base_bcast_intra_generic( void* buffer,
             for( req_index = 0; req_index < tree->tree_nextsize; req_index++ ) {
                 if (MPI_REQUEST_NULL == send_reqs[req_index]) continue;
                 if (MPI_ERR_PENDING == send_reqs[req_index]->req_status.MPI_ERROR) continue;
-                err = send_reqs[req_index]->req_status.MPI_ERROR;
-                break;
+                if (send_reqs[req_index]->req_status.MPI_ERROR != MPI_SUCCESS) {
+                    err = send_reqs[req_index]->req_status.MPI_ERROR;
+                    break;
+                }
             }
         }
         ompi_coll_base_free_reqs(send_reqs, tree->tree_nextsize);
@@ -243,14 +247,14 @@ ompi_coll_base_bcast_intra_generic( void* buffer,
 
 int
 ompi_coll_base_bcast_intra_bintree ( void* buffer,
-                                      int count,
+                                      size_t count,
                                       struct ompi_datatype_t* datatype,
                                       int root,
                                       struct ompi_communicator_t* comm,
                                       mca_coll_base_module_t *module,
                                       uint32_t segsize )
 {
-    int segcount = count;
+    size_t segcount = count;
     size_t typelng;
     mca_coll_base_comm_t *data = module->base_data;
 
@@ -262,7 +266,7 @@ ompi_coll_base_bcast_intra_bintree ( void* buffer,
     ompi_datatype_type_size( datatype, &typelng );
     COLL_BASE_COMPUTED_SEGCOUNT( segsize, typelng, segcount );
 
-    OPAL_OUTPUT((ompi_coll_base_framework.framework_output,"coll:base:bcast_intra_binary rank %d ss %5d typelng %lu segcount %d",
+    OPAL_OUTPUT((ompi_coll_base_framework.framework_output,"coll:base:bcast_intra_binary rank %d ss %5d typelng %zu segcount %zu",
                  ompi_comm_rank(comm), segsize, (unsigned long)typelng, segcount));
 
     return ompi_coll_base_bcast_intra_generic( buffer, count, datatype, root, comm, module,
@@ -271,14 +275,14 @@ ompi_coll_base_bcast_intra_bintree ( void* buffer,
 
 int
 ompi_coll_base_bcast_intra_pipeline( void* buffer,
-                                      int count,
+                                      size_t count,
                                       struct ompi_datatype_t* datatype,
                                       int root,
                                       struct ompi_communicator_t* comm,
                                       mca_coll_base_module_t *module,
                                       uint32_t segsize )
 {
-    int segcount = count;
+    size_t segcount = count;
     size_t typelng;
     mca_coll_base_comm_t *data = module->base_data;
 
@@ -290,7 +294,7 @@ ompi_coll_base_bcast_intra_pipeline( void* buffer,
     ompi_datatype_type_size( datatype, &typelng );
     COLL_BASE_COMPUTED_SEGCOUNT( segsize, typelng, segcount );
 
-    OPAL_OUTPUT((ompi_coll_base_framework.framework_output,"coll:base:bcast_intra_pipeline rank %d ss %5d typelng %lu segcount %d",
+    OPAL_OUTPUT((ompi_coll_base_framework.framework_output,"coll:base:bcast_intra_pipeline rank %d ss %5d typelng %zu segcount %zu",
                  ompi_comm_rank(comm), segsize, (unsigned long)typelng, segcount));
 
     return ompi_coll_base_bcast_intra_generic( buffer, count, datatype, root, comm, module,
@@ -299,14 +303,14 @@ ompi_coll_base_bcast_intra_pipeline( void* buffer,
 
 int
 ompi_coll_base_bcast_intra_chain( void* buffer,
-                                   int count,
+                                   size_t count,
                                    struct ompi_datatype_t* datatype,
                                    int root,
                                    struct ompi_communicator_t* comm,
                                    mca_coll_base_module_t *module,
                                    uint32_t segsize, int32_t chains )
 {
-    int segcount = count;
+    size_t segcount = count;
     size_t typelng;
     mca_coll_base_comm_t *data = module->base_data;
 
@@ -318,7 +322,7 @@ ompi_coll_base_bcast_intra_chain( void* buffer,
     ompi_datatype_type_size( datatype, &typelng );
     COLL_BASE_COMPUTED_SEGCOUNT( segsize, typelng, segcount );
 
-    OPAL_OUTPUT((ompi_coll_base_framework.framework_output,"coll:base:bcast_intra_chain rank %d fo %d ss %5d typelng %lu segcount %d",
+    OPAL_OUTPUT((ompi_coll_base_framework.framework_output,"coll:base:bcast_intra_chain rank %d fo %d ss %5d typelng %zu segcount %zu",
                  ompi_comm_rank(comm), chains, segsize, (unsigned long)typelng, segcount));
 
     return ompi_coll_base_bcast_intra_generic( buffer, count, datatype, root, comm, module,
@@ -327,14 +331,14 @@ ompi_coll_base_bcast_intra_chain( void* buffer,
 
 int
 ompi_coll_base_bcast_intra_binomial( void* buffer,
-                                      int count,
+                                      size_t count,
                                       struct ompi_datatype_t* datatype,
                                       int root,
                                       struct ompi_communicator_t* comm,
                                       mca_coll_base_module_t *module,
                                       uint32_t segsize )
 {
-    int segcount = count;
+    size_t segcount = count;
     size_t typelng;
     mca_coll_base_comm_t *data = module->base_data;
 
@@ -346,7 +350,7 @@ ompi_coll_base_bcast_intra_binomial( void* buffer,
     ompi_datatype_type_size( datatype, &typelng );
     COLL_BASE_COMPUTED_SEGCOUNT( segsize, typelng, segcount );
 
-    OPAL_OUTPUT((ompi_coll_base_framework.framework_output,"coll:base:bcast_intra_binomial rank %d ss %5d typelng %lu segcount %d",
+    OPAL_OUTPUT((ompi_coll_base_framework.framework_output,"coll:base:bcast_intra_binomial rank %d ss %5d typelng %zu segcount %zu",
                  ompi_comm_rank(comm), segsize, (unsigned long)typelng, segcount));
 
     return ompi_coll_base_bcast_intra_generic( buffer, count, datatype, root, comm, module,
@@ -355,7 +359,7 @@ ompi_coll_base_bcast_intra_binomial( void* buffer,
 
 int
 ompi_coll_base_bcast_intra_split_bintree ( void* buffer,
-                                            int count,
+                                            size_t count,
                                             struct ompi_datatype_t* datatype,
                                             int root,
                                             struct ompi_communicator_t* comm,
@@ -363,10 +367,10 @@ ompi_coll_base_bcast_intra_split_bintree ( void* buffer,
                                             uint32_t segsize )
 {
     int err=0, line, rank, size, segindex, i, lr, pair;
-    uint32_t counts[2];
-    int segcount[2];       /* Number of elements sent with each segment */
-    int num_segments[2];   /* Number of segmenets */
-    int sendcount[2];      /* the same like segcount, except for the last segment */
+    size_t counts[2];
+    size_t segcount[2];       /* Number of elements sent with each segment */
+    int num_segments[2];      /* Number of segments */
+    size_t sendcount[2];      /* the same like segcount, except for the last segment */
     size_t realsegsize[2], type_size;
     char *tmpbuf[2];
     ptrdiff_t type_extent, lb;
@@ -503,7 +507,7 @@ ompi_coll_base_bcast_intra_split_bintree ( void* buffer,
                 if (err != MPI_SUCCESS) { line = __LINE__; goto error_hndl; }
             } /* end of for each child */
 
-            /* upate the base request */
+            /* update the base request */
             base_req = new_req;
             /* go to the next buffer (ie. the one corresponding to the next recv) */
             tmpbuf[lr] += realsegsize[lr];
@@ -621,7 +625,7 @@ ompi_coll_base_bcast_intra_split_bintree ( void* buffer,
  *  Returns:    - MPI_SUCCESS or error code
  */
 int
-ompi_coll_base_bcast_intra_basic_linear(void *buff, int count,
+ompi_coll_base_bcast_intra_basic_linear(void *buff, size_t count,
                                         struct ompi_datatype_t *datatype, int root,
                                         struct ompi_communicator_t *comm,
                                         mca_coll_base_module_t *module)
@@ -679,8 +683,10 @@ ompi_coll_base_bcast_intra_basic_linear(void *buff, int count,
         for( preq = reqs; preq < reqs+i; preq++ ) {
             if (MPI_REQUEST_NULL == *preq) continue;
             if (MPI_ERR_PENDING == (*preq)->req_status.MPI_ERROR) continue;
-            err = (*preq)->req_status.MPI_ERROR;
-            break;
+            if ((*preq)->req_status.MPI_ERROR != MPI_SUCCESS) {
+                err = (*preq)->req_status.MPI_ERROR;
+                break;
+            }
         }
         ompi_coll_base_free_reqs(reqs, i);
     }
@@ -712,7 +718,7 @@ ompi_coll_base_bcast_intra_basic_linear(void *buff, int count,
  *     7
  */
 int ompi_coll_base_bcast_intra_knomial(
-    void *buf, int count, struct ompi_datatype_t *datatype, int root,
+    void *buf, size_t count, struct ompi_datatype_t *datatype, int root,
     struct ompi_communicator_t *comm, mca_coll_base_module_t *module,
     uint32_t segsize, int radix)
 {
@@ -734,7 +740,7 @@ int ompi_coll_base_bcast_intra_knomial(
     COLL_BASE_COMPUTED_SEGCOUNT(segsize, typesize, segcount);
 
     OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
-                 "coll:base:bcast_intra_knomial rank %d segsize %5d typesize %lu segcount %d",
+                 "coll:base:bcast_intra_knomial rank %d segsize %5d typesize %zu segcount %d",
                  ompi_comm_rank(comm), segsize, (unsigned long)typesize, segcount));
 
     return ompi_coll_base_bcast_intra_generic(buf, count, datatype, root, comm, module,
@@ -766,7 +772,7 @@ int ompi_coll_base_bcast_intra_knomial(
  * 7:           <-+  [*******7]  <-+ [******67]  <--|-+ [****4567] <--------+
  */
 int ompi_coll_base_bcast_intra_scatter_allgather(
-    void *buf, int count, struct ompi_datatype_t *datatype, int root,
+    void *buf, size_t count, struct ompi_datatype_t *datatype, int root,
     struct ompi_communicator_t *comm, mca_coll_base_module_t *module,
     uint32_t segsize)
 {
@@ -785,19 +791,19 @@ int ompi_coll_base_bcast_intra_scatter_allgather(
     if (comm_size < 2 || datatype_size == 0)
         return MPI_SUCCESS;
 
-    if (count < comm_size) {
+    if (count < (size_t)comm_size) {
         OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
                      "coll:base:bcast_intra_scatter_allgather: rank %d/%d "
-                     "count %d switching to basic linear bcast",
+                     "count %zu switching to basic linear bcast",
                      rank, comm_size, count));
         return ompi_coll_base_bcast_intra_basic_linear(buf, count, datatype,
                                                        root, comm, module);
     }
 
     int vrank = (rank - root + comm_size) % comm_size;
-    int recv_count = 0, send_count = 0;
-    int scatter_count = (count + comm_size - 1) / comm_size; /* ceil(count / comm_size) */
-    int curr_count = (rank == root) ? count : 0;
+    size_t recv_count = 0, send_count = 0;
+    size_t scatter_count = (count + comm_size - 1) / comm_size; /* ceil(count / comm_size) */
+    size_t curr_count = (rank == root) ? count : 0;
 
     /* Scatter by binomial tree: receive data from parent */
     int mask = 0x1;
@@ -844,7 +850,7 @@ int ompi_coll_base_bcast_intra_scatter_allgather(
      * Allgather by recursive doubling
      * Each process has the curr_count elems in the buf[vrank * scatter_count, ...]
      */
-    int rem_count = count - vrank * scatter_count;
+    size_t rem_count = count - vrank * scatter_count;
     curr_count = (scatter_count < rem_count) ? scatter_count : rem_count;
     if (curr_count < 0)
         curr_count = 0;
@@ -882,7 +888,7 @@ int ompi_coll_base_bcast_intra_scatter_allgather(
          */
         if (vremote_tree_root + mask > comm_size) {
             int nprocs_alldata = comm_size - vrank_tree_root - mask;
-            int offset = scatter_count * (vrank_tree_root + mask);
+            ptrdiff_t offset = scatter_count * (vrank_tree_root + mask);
             for (int rhalving_mask = mask >> 1; rhalving_mask > 0; rhalving_mask >>= 1) {
                 vremote = vrank ^ rhalving_mask;
                 remote = (vremote + root) % comm_size;
@@ -943,7 +949,7 @@ int ompi_coll_base_bcast_intra_scatter_allgather(
  * 7:           <-+  [*******7]  [******67] [*****567] [****4567] ... [01234567]
  */
 int ompi_coll_base_bcast_intra_scatter_allgather_ring(
-    void *buf, int count, struct ompi_datatype_t *datatype, int root,
+    void *buf, size_t count, struct ompi_datatype_t *datatype, int root,
     struct ompi_communicator_t *comm, mca_coll_base_module_t *module,
     uint32_t segsize)
 {
@@ -962,19 +968,19 @@ int ompi_coll_base_bcast_intra_scatter_allgather_ring(
     if (comm_size < 2 || datatype_size == 0)
         return MPI_SUCCESS;
 
-    if (count < comm_size) {
+    if (count < (size_t)comm_size) {
         OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
                      "coll:base:bcast_intra_scatter_allgather_ring: rank %d/%d "
-                     "count %d switching to basic linear bcast",
+                     "count %zu switching to basic linear bcast",
                      rank, comm_size, count));
         return ompi_coll_base_bcast_intra_basic_linear(buf, count, datatype,
                                                        root, comm, module);
     }
 
     int vrank = (rank - root + comm_size) % comm_size;
-    int recv_count = 0, send_count = 0;
-    int scatter_count = (count + comm_size - 1) / comm_size; /* ceil(count / comm_size) */
-    int curr_count = (rank == root) ? count : 0;
+    size_t recv_count = 0, send_count = 0;
+    size_t scatter_count = (count + comm_size - 1) / comm_size; /* ceil(count / comm_size) */
+    size_t curr_count = (rank == root) ? count : 0;
 
     /* Scatter by binomial tree: receive data from parent */
     int mask = 1;
diff --git a/ompi/mca/coll/base/coll_base_comm_select.c b/ompi/mca/coll/base/coll_base_comm_select.c
index c8932bf4d53..e67aab62c71 100644
--- a/ompi/mca/coll/base/coll_base_comm_select.c
+++ b/ompi/mca/coll/base/coll_base_comm_select.c
@@ -22,6 +22,7 @@
  * Copyright (c) 2016-2017 IBM Corporation.  All rights reserved.
  * Copyright (c) 2017      FUJITSU LIMITED.  All rights reserved.
  * Copyright (c) 2020      BULL S.A.S. All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -41,6 +42,7 @@
 #include "opal/util/argv.h"
 #include "opal/util/show_help.h"
 #include "opal/class/opal_list.h"
+#include "opal/class/opal_hash_table.h"
 #include "opal/class/opal_object.h"
 #include "ompi/mca/mca.h"
 #include "opal/mca/base/base.h"
@@ -66,26 +68,143 @@ static int query(const mca_base_component_t * component,
                  ompi_communicator_t * comm, int *priority,
                  mca_coll_base_module_t ** module);
 
-static int query_2_4_0(const mca_coll_base_component_2_4_0_t *
+static int query_3_0_0(const mca_coll_base_component_3_0_0_t *
                        coll_component, ompi_communicator_t * comm,
                        int *priority,
                        mca_coll_base_module_t ** module);
 
-#define COPY(module, comm, func)                                        \
-    do {                                                                \
-        if (NULL != module->coll_ ## func) {                            \
-            if (NULL != comm->c_coll->coll_ ## func ## _module) {       \
-                OBJ_RELEASE(comm->c_coll->coll_ ## func ## _module);    \
-            }                                                           \
-            comm->c_coll->coll_ ## func = module->coll_ ## func;        \
-            comm->c_coll->coll_ ## func ## _module = module;            \
-            OBJ_RETAIN(module);                                         \
-        }                                                               \
-    } while (0)
-
 #define CHECK_NULL(what, comm, func)                                    \
   ( (what) = # func , NULL == (comm)->c_coll->coll_ ## func)
 
+static void mca_coll_base_get_component_name(ompi_communicator_t *comm, void* module, char** name)
+{
+    mca_coll_base_avail_coll_t *avail;
+
+    *name = NULL;
+    OPAL_LIST_FOREACH(avail, comm->c_coll->module_list, mca_coll_base_avail_coll_t) {
+        if (avail->ac_module == module) {
+            *name = (char*) avail->ac_component_name;
+            break;
+        }
+    }
+}
+
+#define PRINT_NAME(comm, func, func_name)                 \
+    do {                                                  \
+        char *name;                                       \
+        mca_coll_base_get_component_name(comm, (void*)comm->c_coll->coll_ ## func ## _module, &name); \
+        opal_output_verbose(10, ompi_coll_base_framework.framework_output,                                                              \
+                            "coll:base:comm_select: communicator %s rank %d %s -> %s", comm->c_name, comm->c_my_rank, func_name, name); \
+    } while (0);
+
+#define PRINT_ALL_BLOCKING(comm)                     \
+    do {                                                 \
+        PRINT_NAME(comm,  allgather, "allgather");    \
+        PRINT_NAME(comm,  allgatherv, "allgatherv");  \
+        PRINT_NAME(comm,  allreduce, "allreduce");    \
+        PRINT_NAME(comm,  alltoall, "alltoall");      \
+        PRINT_NAME(comm,  alltoallv, "alltoallv");    \
+        PRINT_NAME(comm,  alltoallw, "alltoallw");    \
+        PRINT_NAME(comm,  barrier, "barrier");        \
+        PRINT_NAME(comm,  bcast, "bcast");            \
+        PRINT_NAME(comm,  exscan, "exscan");          \
+        PRINT_NAME(comm,  gather, "gather");          \
+        PRINT_NAME(comm,  gatherv, "gatherv");        \
+        PRINT_NAME(comm,  reduce, "reduce");          \
+        PRINT_NAME(comm,  reduce_scatter_block, "reduce_scatter_block");  \
+        PRINT_NAME(comm,  reduce_scatter, "reduce_scatter");              \
+        PRINT_NAME(comm,  scan, "scan");              \
+        PRINT_NAME(comm,  scatter, "scatter");        \
+        PRINT_NAME(comm,  scatterv, "scatterv");      \
+        PRINT_NAME(comm,  neighbor_allgather, "neighbor_allgather");     \
+        PRINT_NAME(comm,  neighbor_allgatherv, "neighbor_allgatherv");   \
+        PRINT_NAME(comm,  neighbor_alltoall, "neighbor_alltoall");       \
+        PRINT_NAME(comm,  neighbor_alltoallv, "neighbor_alltoallv");     \
+        PRINT_NAME(comm,  neighbor_alltoallw, "neighbor_alltoallw");     \
+        PRINT_NAME(comm,  reduce_local, "reduce_local");                 \
+    } while (0);
+
+#define PRINT_ALL_NB(comm)                            \
+    do {                                              \
+        PRINT_NAME(comm,  iallgather, "iallgather");  \
+        PRINT_NAME(comm,  iallgatherv, "iallgatherv");\
+        PRINT_NAME(comm,  iallreduce, "iallreduce");  \
+        PRINT_NAME(comm,  ialltoall, "ialltoall");    \
+        PRINT_NAME(comm,  ialltoallv, "ialltoallv");  \
+        PRINT_NAME(comm,  ialltoallw, "ialltoallw");  \
+        PRINT_NAME(comm,  ibarrier, "ibarrier");      \
+        PRINT_NAME(comm,  ibcast, "ibcast");          \
+        PRINT_NAME(comm,  iexscan, "iexscan");        \
+        PRINT_NAME(comm,  igather, "igather");        \
+        PRINT_NAME(comm,  igatherv, "igatherv");      \
+        PRINT_NAME(comm,  ireduce, "ireduce");        \
+        PRINT_NAME(comm,  ireduce_scatter_block, "ireduce_scatter_block"); \
+        PRINT_NAME(comm,  ireduce_scatter, "ireduce_scatter");             \
+        PRINT_NAME(comm,  iscan, "iscan");            \
+        PRINT_NAME(comm,  iscatter, "iscatter");      \
+        PRINT_NAME(comm,  iscatterv, "iscatterv");    \
+        PRINT_NAME(comm,  ineighbor_allgather, "ineighbor_allgather");     \
+        PRINT_NAME(comm,  ineighbor_allgatherv, "ineighbor_allgatherv");   \
+        PRINT_NAME(comm,  ineighbor_alltoall, "ineighbor_alltoall");       \
+        PRINT_NAME(comm,  ineighbor_alltoallv, "ineighbor_alltoallv");     \
+        PRINT_NAME(comm,  ineighbor_alltoallw, "ineighbor_alltoallw");     \
+    } while (0);
+
+#define PRINT_ALL_PERSISTENT(comm)                              \
+    do {                                                        \
+        PRINT_NAME(comm,  allgather_init, "allgather_init");    \
+        PRINT_NAME(comm,  allgatherv_init, "allgatherv_init");  \
+        PRINT_NAME(comm,  allreduce_init, "allreduce_init");    \
+        PRINT_NAME(comm,  alltoall_init, "alltoall_init");      \
+        PRINT_NAME(comm,  alltoallv_init, "alltoallv_init");    \
+        PRINT_NAME(comm,  alltoallw_init, "alltoallw_init");    \
+        PRINT_NAME(comm,  barrier_init, "barrier_init");        \
+        PRINT_NAME(comm,  bcast_init, "bcast_init");            \
+        PRINT_NAME(comm,  exscan_init, "exscan_init");          \
+        PRINT_NAME(comm,  gather_init, "gather_init");          \
+        PRINT_NAME(comm,  gatherv_init, "gatherv_init");        \
+        PRINT_NAME(comm,  reduce_init, "reduce_init");          \
+        PRINT_NAME(comm,  reduce_scatter_block_init, "reduce_scatter_block_init"); \
+        PRINT_NAME(comm,  reduce_scatter_init, "reduce_scatter_init");             \
+        PRINT_NAME(comm,  scan_init, "scan_init");              \
+        PRINT_NAME(comm,  scatter_init, "scatter_init");        \
+        PRINT_NAME(comm,  scatterv_init, "scatterv_init");      \
+        PRINT_NAME(comm,  neighbor_allgather_init, "neighbor_allgather_init");     \
+        PRINT_NAME(comm,  neighbor_allgatherv_init, "neighbor_allgatherv_init");   \
+        PRINT_NAME(comm,  neighbor_alltoall_init, "neighbor_alltoall_init");       \
+        PRINT_NAME(comm,  neighbor_alltoallv_init, "neighbor_alltoallv_init");     \
+        PRINT_NAME(comm,  neighbor_alltoallw_init, "neighbor_alltoallw_init");     \
+    } while (0);
+
+#define PRINT_ALL_FT(comm)                    \
+    do {                                      \
+        PRINT_NAME(comm,  agree, "agree");    \
+        PRINT_NAME(comm,  iagree, "iagree");  \
+    } while (0);
+
+static void mca_coll_base_print_component_names(ompi_communicator_t *comm)
+{
+    /*
+    ** Verbosity level 1 - 19 will only print the blocking and non-blocking collectives
+    ** assigned to MPI_COMM_WORLD, but not the persistent and ft ones.
+    **
+    ** Verbosity level 20 will print all blocking and non-blocking collectives for all communicators,
+    ** but not the persistent and ft ones.
+    **
+    ** Verbosity level > 20 will print all collectives for all communicators.
+    */
+    if ( (MPI_COMM_WORLD == comm) || (ompi_coll_base_framework.framework_verbose >= 20)) {
+        PRINT_ALL_BLOCKING (comm);
+        PRINT_ALL_NB (comm);
+        if (ompi_coll_base_framework.framework_verbose > 20) {
+            PRINT_ALL_PERSISTENT (comm);
+#if OPAL_ENABLE_FT_MPI
+            PRINT_ALL_FT (comm);
+#endif
+        }
+    }
+}
+
 /*
  * This function is called at the initialization time of every
  * communicator.  It is used to select which coll component will be
@@ -102,8 +221,8 @@ int mca_coll_base_comm_select(ompi_communicator_t * comm)
 
     /* Announce */
     opal_output_verbose(9, ompi_coll_base_framework.framework_output,
-                        "coll:base:comm_select: new communicator: %s (cid %d)",
-                        comm->c_name, comm->c_contextid);
+                        "coll:base:comm_select: new communicator: %s (cid %s)",
+                        comm->c_name, ompi_comm_print_cid (comm));
 
     /* Initialize all the relevant pointers, since they're used as
      * sentinel values */
@@ -134,7 +253,6 @@ int mca_coll_base_comm_select(ompi_communicator_t * comm)
          NULL != item; item = opal_list_remove_first(selectable)) {
 
         mca_coll_base_avail_coll_t *avail = (mca_coll_base_avail_coll_t *) item;
-
         /* initialize the module */
         ret = avail->ac_module->coll_module_enable(avail->ac_module, comm);
 
@@ -147,95 +265,12 @@ int mca_coll_base_comm_select(ompi_communicator_t * comm)
             /* Save every component that is initialized,
              * queried and enabled successfully */
             opal_list_append(comm->c_coll->module_list, &avail->super);
-
-            /* copy over any of the pointers */
-            COPY(avail->ac_module, comm, allgather);
-            COPY(avail->ac_module, comm, allgatherv);
-            COPY(avail->ac_module, comm, allreduce);
-            COPY(avail->ac_module, comm, alltoall);
-            COPY(avail->ac_module, comm, alltoallv);
-            COPY(avail->ac_module, comm, alltoallw);
-            COPY(avail->ac_module, comm, barrier);
-            COPY(avail->ac_module, comm, bcast);
-            COPY(avail->ac_module, comm, exscan);
-            COPY(avail->ac_module, comm, gather);
-            COPY(avail->ac_module, comm, gatherv);
-            COPY(avail->ac_module, comm, reduce);
-            COPY(avail->ac_module, comm, reduce_scatter_block);
-            COPY(avail->ac_module, comm, reduce_scatter);
-            COPY(avail->ac_module, comm, scan);
-            COPY(avail->ac_module, comm, scatter);
-            COPY(avail->ac_module, comm, scatterv);
-
-            COPY(avail->ac_module, comm, iallgather);
-            COPY(avail->ac_module, comm, iallgatherv);
-            COPY(avail->ac_module, comm, iallreduce);
-            COPY(avail->ac_module, comm, ialltoall);
-            COPY(avail->ac_module, comm, ialltoallv);
-            COPY(avail->ac_module, comm, ialltoallw);
-            COPY(avail->ac_module, comm, ibarrier);
-            COPY(avail->ac_module, comm, ibcast);
-            COPY(avail->ac_module, comm, iexscan);
-            COPY(avail->ac_module, comm, igather);
-            COPY(avail->ac_module, comm, igatherv);
-            COPY(avail->ac_module, comm, ireduce);
-            COPY(avail->ac_module, comm, ireduce_scatter_block);
-            COPY(avail->ac_module, comm, ireduce_scatter);
-            COPY(avail->ac_module, comm, iscan);
-            COPY(avail->ac_module, comm, iscatter);
-            COPY(avail->ac_module, comm, iscatterv);
-
-            COPY(avail->ac_module, comm, allgather_init);
-            COPY(avail->ac_module, comm, allgatherv_init);
-            COPY(avail->ac_module, comm, allreduce_init);
-            COPY(avail->ac_module, comm, alltoall_init);
-            COPY(avail->ac_module, comm, alltoallv_init);
-            COPY(avail->ac_module, comm, alltoallw_init);
-            COPY(avail->ac_module, comm, barrier_init);
-            COPY(avail->ac_module, comm, bcast_init);
-            COPY(avail->ac_module, comm, exscan_init);
-            COPY(avail->ac_module, comm, gather_init);
-            COPY(avail->ac_module, comm, gatherv_init);
-            COPY(avail->ac_module, comm, reduce_init);
-            COPY(avail->ac_module, comm, reduce_scatter_block_init);
-            COPY(avail->ac_module, comm, reduce_scatter_init);
-            COPY(avail->ac_module, comm, scan_init);
-            COPY(avail->ac_module, comm, scatter_init);
-            COPY(avail->ac_module, comm, scatterv_init);
-
-            /* We can not reliably check if this comm has a topology
-             * at this time. The flags are set *after* coll_select */
-            COPY(avail->ac_module, comm, neighbor_allgather);
-            COPY(avail->ac_module, comm, neighbor_allgatherv);
-            COPY(avail->ac_module, comm, neighbor_alltoall);
-            COPY(avail->ac_module, comm, neighbor_alltoallv);
-            COPY(avail->ac_module, comm, neighbor_alltoallw);
-
-            COPY(avail->ac_module, comm, ineighbor_allgather);
-            COPY(avail->ac_module, comm, ineighbor_allgatherv);
-            COPY(avail->ac_module, comm, ineighbor_alltoall);
-            COPY(avail->ac_module, comm, ineighbor_alltoallv);
-            COPY(avail->ac_module, comm, ineighbor_alltoallw);
-
-            COPY(avail->ac_module, comm, neighbor_allgather_init);
-            COPY(avail->ac_module, comm, neighbor_allgatherv_init);
-            COPY(avail->ac_module, comm, neighbor_alltoall_init);
-            COPY(avail->ac_module, comm, neighbor_alltoallv_init);
-            COPY(avail->ac_module, comm, neighbor_alltoallw_init);
-
-            COPY(avail->ac_module, comm, reduce_local);
-
-#if OPAL_ENABLE_FT_MPI
-            COPY(avail->ac_module, comm, agree);
-            COPY(avail->ac_module, comm, iagree);
-#endif
         } else {
             /* release the original module reference and the list item */
             OBJ_RELEASE(avail->ac_module);
             OBJ_RELEASE(avail);
         }
     }
-
     /* Done with the list from the check_components() call so release it. */
     OBJ_RELEASE(selectable);
 
@@ -291,6 +326,10 @@ int mca_coll_base_comm_select(ompi_communicator_t * comm)
         ((OMPI_COMM_IS_INTRA(comm)) && CHECK_NULL(which_func, comm, scan_init)) ||
         CHECK_NULL(which_func, comm, scatter_init) ||
         CHECK_NULL(which_func, comm, scatterv_init) ||
+#if OPAL_ENABLE_FT_MPI
+        CHECK_NULL(which_func, comm, agree) ||
+        CHECK_NULL(which_func, comm, iagree) ||
+#endif  /* OPAL_ENABLE_FT_MPI */
         CHECK_NULL(which_func, comm, reduce_local) ) {
         /* TODO -- Once the topology flags are set before coll_select then
          * check if neighborhood collectives have been set. */
@@ -298,9 +337,14 @@ int mca_coll_base_comm_select(ompi_communicator_t * comm)
         opal_show_help("help-mca-coll-base.txt",
                        "comm-select:no-function-available", true, which_func);
 
-         mca_coll_base_comm_unselect(comm);
+        mca_coll_base_comm_unselect(comm);
         return OMPI_ERR_NOT_FOUND;
     }
+
+    if (ompi_coll_base_framework.framework_verbose > 0) {
+        mca_coll_base_print_component_names(comm);
+    }
+
     return OMPI_SUCCESS;
 }
 
@@ -348,7 +392,6 @@ static opal_list_t *check_components(opal_list_t * components,
     mca_coll_base_module_t *module;
     opal_list_t *selectable;
     mca_coll_base_avail_coll_t *avail;
-    char info_val[OPAL_MAX_INFO_VAL+1];
     char **coll_argv = NULL, **coll_exclude = NULL, **coll_include = NULL;
 
     /* Check if this communicator comes with restrictions on the collective modules
@@ -360,12 +403,14 @@ static opal_list_t *check_components(opal_list_t * components,
      * force a change in the component priority.
      */
     if( NULL != comm->super.s_info) {
+        opal_cstring_t *info_str;
         opal_info_get(comm->super.s_info, "ompi_comm_coll_preference",
-                      sizeof(info_val), info_val, &flag);
+                      &info_str, &flag);
         if( !flag ) {
             goto proceed_to_select;
         }
-        coll_argv = opal_argv_split(info_val, ',');
+        coll_argv = opal_argv_split(info_str->string, ',');
+        OBJ_RELEASE(info_str);
         if(NULL == coll_argv) {
             goto proceed_to_select;
         }
@@ -452,7 +497,7 @@ static opal_list_t *check_components(opal_list_t * components,
     /* For all valid component reorder them not on their provided priorities but on
      * the order requested in the info key. As at this point the coll_include is
      * already ordered backward we can simply append the components.
-     * Note that the last element in selectable will have the highest priorty.
+     * Note that the last element in selectable will have the highest priority.
      */
     for (int idx = count_include-1; idx >= 0; --idx) {
         mca_coll_base_avail_coll_t *item;
@@ -519,11 +564,11 @@ static int query(const mca_base_component_t * component,
                  int *priority, mca_coll_base_module_t ** module)
 {
     *module = NULL;
-    if (2 == component->mca_type_major_version &&
-        4 == component->mca_type_minor_version &&
+    if (3 == component->mca_type_major_version &&
+        0 == component->mca_type_minor_version &&
         0 == component->mca_type_release_version) {
 
-        return query_2_4_0((const mca_coll_base_component_2_4_0_t *)component, comm, priority, module);
+        return query_3_0_0((const mca_coll_base_component_3_0_0_t *)component, comm, priority, module);
     }
 
     /* Unknown coll API version -- return error */
@@ -532,7 +577,7 @@ static int query(const mca_base_component_t * component,
 }
 
 
-static int query_2_4_0(const mca_coll_base_component_2_4_0_t * component,
+static int query_3_0_0(const mca_coll_base_component_3_0_0_t * component,
                        ompi_communicator_t * comm, int *priority,
                        mca_coll_base_module_t ** module)
 {
diff --git a/ompi/mca/coll/base/coll_base_comm_unselect.c b/ompi/mca/coll/base/coll_base_comm_unselect.c
index 9608b498cba..aca7bb23095 100644
--- a/ompi/mca/coll/base/coll_base_comm_unselect.c
+++ b/ompi/mca/coll/base/coll_base_comm_unselect.c
@@ -17,6 +17,7 @@
  * Copyright (c) 2017      IBM Corporation.  All rights reserved.
  * Copyright (c) 2017      FUJITSU LIMITED.  All rights reserved.
  * Copyright (c) 2020      BULL S.A.S. All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -37,113 +38,120 @@
 #include "ompi/mca/coll/base/base.h"
 #include "ompi/mca/coll/base/coll_base_util.h"
 
-#define CLOSE(comm, func)                                        \
+#if OPAL_ENABLE_DEBUG
+#define CHECK_CLEAN_COLL(comm, func)                             \
     do {                                                         \
-        if (NULL != comm->c_coll->coll_ ## func ## _module) {    \
-            if (NULL != comm->c_coll->coll_ ## func ## _module->coll_module_disable) { \
-                comm->c_coll->coll_ ## func ## _module->coll_module_disable(           \
-                              comm->c_coll->coll_ ## func ## _module, comm);           \
-            }                                                    \
-            OBJ_RELEASE(comm->c_coll->coll_ ## func ## _module); \
-            comm->c_coll->coll_## func = NULL;                   \
-            comm->c_coll->coll_## func ## _module = NULL;        \
+        if (NULL != comm->c_coll->coll_ ## func ## _module ||    \
+            NULL != comm->c_coll->coll_ ## func ) {              \
+            opal_output_verbose(10, ompi_coll_base_framework.framework_output,                        \
+                "coll:base:comm_unselect: Comm %p (%s) has a left over %s collective during cleanup", \
+                (void*)comm, comm->c_name, #func);               \
         }                                                        \
     } while (0)
+#else
+#define CHECK_CLEAN_COLL(comm, func)
+#endif  /* OPAL_ENABLE_DEBUG */
 
 int mca_coll_base_comm_unselect(ompi_communicator_t * comm)
 {
     opal_list_item_t *item;
 
-    CLOSE(comm, allgather);
-    CLOSE(comm, allgatherv);
-    CLOSE(comm, allreduce);
-    CLOSE(comm, alltoall);
-    CLOSE(comm, alltoallv);
-    CLOSE(comm, alltoallw);
-    CLOSE(comm, barrier);
-    CLOSE(comm, bcast);
-    CLOSE(comm, exscan);
-    CLOSE(comm, gather);
-    CLOSE(comm, gatherv);
-    CLOSE(comm, reduce);
-    CLOSE(comm, reduce_scatter_block);
-    CLOSE(comm, reduce_scatter);
-    CLOSE(comm, scan);
-    CLOSE(comm, scatter);
-    CLOSE(comm, scatterv);
-
-    CLOSE(comm, iallgather);
-    CLOSE(comm, iallgatherv);
-    CLOSE(comm, iallreduce);
-    CLOSE(comm, ialltoall);
-    CLOSE(comm, ialltoallv);
-    CLOSE(comm, ialltoallw);
-    CLOSE(comm, ibarrier);
-    CLOSE(comm, ibcast);
-    CLOSE(comm, iexscan);
-    CLOSE(comm, igather);
-    CLOSE(comm, igatherv);
-    CLOSE(comm, ireduce);
-    CLOSE(comm, ireduce_scatter_block);
-    CLOSE(comm, ireduce_scatter);
-    CLOSE(comm, iscan);
-    CLOSE(comm, iscatter);
-    CLOSE(comm, iscatterv);
-
-    CLOSE(comm, allgather_init);
-    CLOSE(comm, allgatherv_init);
-    CLOSE(comm, allreduce_init);
-    CLOSE(comm, alltoall_init);
-    CLOSE(comm, alltoallv_init);
-    CLOSE(comm, alltoallw_init);
-    CLOSE(comm, barrier_init);
-    CLOSE(comm, bcast_init);
-    CLOSE(comm, exscan_init);
-    CLOSE(comm, gather_init);
-    CLOSE(comm, gatherv_init);
-    CLOSE(comm, reduce_init);
-    CLOSE(comm, reduce_scatter_block_init);
-    CLOSE(comm, reduce_scatter_init);
-    CLOSE(comm, scan_init);
-    CLOSE(comm, scatter_init);
-    CLOSE(comm, scatterv_init);
-
-    CLOSE(comm, neighbor_allgather);
-    CLOSE(comm, neighbor_allgatherv);
-    CLOSE(comm, neighbor_alltoall);
-    CLOSE(comm, neighbor_alltoallv);
-    CLOSE(comm, neighbor_alltoallw);
-
-    CLOSE(comm, ineighbor_allgather);
-    CLOSE(comm, ineighbor_allgatherv);
-    CLOSE(comm, ineighbor_alltoall);
-    CLOSE(comm, ineighbor_alltoallv);
-    CLOSE(comm, ineighbor_alltoallw);
-
-    CLOSE(comm, neighbor_allgather_init);
-    CLOSE(comm, neighbor_allgatherv_init);
-    CLOSE(comm, neighbor_alltoall_init);
-    CLOSE(comm, neighbor_alltoallv_init);
-    CLOSE(comm, neighbor_alltoallw_init);
-
-    CLOSE(comm, reduce_local);
-
-#if OPAL_ENABLE_FT_MPI
-    CLOSE(comm, agree);
-    CLOSE(comm, iagree);
-#endif
-
-    for (item = opal_list_remove_first(comm->c_coll->module_list);
-         NULL != item; item = opal_list_remove_first(comm->c_coll->module_list)) {
+    /* Call module disable in the reverse order in which enable has been called
+     * in order to allow the modules to properly chain themselves.
+     */
+    for (item = opal_list_remove_last(comm->c_coll->module_list);
+         NULL != item; item = opal_list_remove_last(comm->c_coll->module_list)) {
         mca_coll_base_avail_coll_t *avail = (mca_coll_base_avail_coll_t *) item;
 
         if(avail->ac_module) {
+            if (NULL != avail->ac_module->coll_module_disable ) {
+                avail->ac_module->coll_module_disable(avail->ac_module, comm);
+            }
             OBJ_RELEASE(avail->ac_module);
         }
         OBJ_RELEASE(avail);
     }
     OBJ_RELEASE(comm->c_coll->module_list);
 
+    CHECK_CLEAN_COLL(comm, allgather);
+    CHECK_CLEAN_COLL(comm, allgatherv);
+    CHECK_CLEAN_COLL(comm, allreduce);
+    CHECK_CLEAN_COLL(comm, alltoall);
+    CHECK_CLEAN_COLL(comm, alltoallv);
+    CHECK_CLEAN_COLL(comm, alltoallw);
+    CHECK_CLEAN_COLL(comm, barrier);
+    CHECK_CLEAN_COLL(comm, bcast);
+    CHECK_CLEAN_COLL(comm, exscan);
+    CHECK_CLEAN_COLL(comm, gather);
+    CHECK_CLEAN_COLL(comm, gatherv);
+    CHECK_CLEAN_COLL(comm, reduce);
+    CHECK_CLEAN_COLL(comm, reduce_scatter_block);
+    CHECK_CLEAN_COLL(comm, reduce_scatter);
+    CHECK_CLEAN_COLL(comm, scan);
+    CHECK_CLEAN_COLL(comm, scatter);
+    CHECK_CLEAN_COLL(comm, scatterv);
+
+    CHECK_CLEAN_COLL(comm, iallgather);
+    CHECK_CLEAN_COLL(comm, iallgatherv);
+    CHECK_CLEAN_COLL(comm, iallreduce);
+    CHECK_CLEAN_COLL(comm, ialltoall);
+    CHECK_CLEAN_COLL(comm, ialltoallv);
+    CHECK_CLEAN_COLL(comm, ialltoallw);
+    CHECK_CLEAN_COLL(comm, ibarrier);
+    CHECK_CLEAN_COLL(comm, ibcast);
+    CHECK_CLEAN_COLL(comm, iexscan);
+    CHECK_CLEAN_COLL(comm, igather);
+    CHECK_CLEAN_COLL(comm, igatherv);
+    CHECK_CLEAN_COLL(comm, ireduce);
+    CHECK_CLEAN_COLL(comm, ireduce_scatter_block);
+    CHECK_CLEAN_COLL(comm, ireduce_scatter);
+    CHECK_CLEAN_COLL(comm, iscan);
+    CHECK_CLEAN_COLL(comm, iscatter);
+    CHECK_CLEAN_COLL(comm, iscatterv);
+
+    CHECK_CLEAN_COLL(comm, allgather_init);
+    CHECK_CLEAN_COLL(comm, allgatherv_init);
+    CHECK_CLEAN_COLL(comm, allreduce_init);
+    CHECK_CLEAN_COLL(comm, alltoall_init);
+    CHECK_CLEAN_COLL(comm, alltoallv_init);
+    CHECK_CLEAN_COLL(comm, alltoallw_init);
+    CHECK_CLEAN_COLL(comm, barrier_init);
+    CHECK_CLEAN_COLL(comm, bcast_init);
+    CHECK_CLEAN_COLL(comm, exscan_init);
+    CHECK_CLEAN_COLL(comm, gather_init);
+    CHECK_CLEAN_COLL(comm, gatherv_init);
+    CHECK_CLEAN_COLL(comm, reduce_init);
+    CHECK_CLEAN_COLL(comm, reduce_scatter_block_init);
+    CHECK_CLEAN_COLL(comm, reduce_scatter_init);
+    CHECK_CLEAN_COLL(comm, scan_init);
+    CHECK_CLEAN_COLL(comm, scatter_init);
+    CHECK_CLEAN_COLL(comm, scatterv_init);
+
+    CHECK_CLEAN_COLL(comm, neighbor_allgather);
+    CHECK_CLEAN_COLL(comm, neighbor_allgatherv);
+    CHECK_CLEAN_COLL(comm, neighbor_alltoall);
+    CHECK_CLEAN_COLL(comm, neighbor_alltoallv);
+    CHECK_CLEAN_COLL(comm, neighbor_alltoallw);
+
+    CHECK_CLEAN_COLL(comm, ineighbor_allgather);
+    CHECK_CLEAN_COLL(comm, ineighbor_allgatherv);
+    CHECK_CLEAN_COLL(comm, ineighbor_alltoall);
+    CHECK_CLEAN_COLL(comm, ineighbor_alltoallv);
+    CHECK_CLEAN_COLL(comm, ineighbor_alltoallw);
+
+    CHECK_CLEAN_COLL(comm, neighbor_allgather_init);
+    CHECK_CLEAN_COLL(comm, neighbor_allgatherv_init);
+    CHECK_CLEAN_COLL(comm, neighbor_alltoall_init);
+    CHECK_CLEAN_COLL(comm, neighbor_alltoallv_init);
+    CHECK_CLEAN_COLL(comm, neighbor_alltoallw_init);
+
+    CHECK_CLEAN_COLL(comm, reduce_local);
+
+#if OPAL_ENABLE_FT_MPI
+    CHECK_CLEAN_COLL(comm, agree);
+    CHECK_CLEAN_COLL(comm, iagree);
+#endif
+
     free(comm->c_coll);
     comm->c_coll = NULL;
 
diff --git a/ompi/mca/coll/base/coll_base_exscan.c b/ompi/mca/coll/base/coll_base_exscan.c
index ef984049ae1..d702eb361b9 100644
--- a/ompi/mca/coll/base/coll_base_exscan.c
+++ b/ompi/mca/coll/base/coll_base_exscan.c
@@ -32,7 +32,7 @@
  * Returns:   MPI_SUCCESS or error code
  */
 int
-ompi_coll_base_exscan_intra_linear(const void *sbuf, void *rbuf, int count,
+ompi_coll_base_exscan_intra_linear(const void *sbuf, void *rbuf, size_t count,
                                   struct ompi_datatype_t *dtype,
                                   struct ompi_op_t *op,
                                   struct ompi_communicator_t *comm,
@@ -140,7 +140,7 @@ ompi_coll_base_exscan_intra_linear(const void *sbuf, void *rbuf, int count,
  * Limitations: intra-communicators only
  */
 int ompi_coll_base_exscan_intra_recursivedoubling(
-    const void *sendbuf, void *recvbuf, int count, struct ompi_datatype_t *datatype,
+    const void *sendbuf, void *recvbuf, size_t count, struct ompi_datatype_t *datatype,
     struct ompi_op_t *op, struct ompi_communicator_t *comm,
     mca_coll_base_module_t *module)
 {
diff --git a/ompi/mca/coll/base/coll_base_find_available.c b/ompi/mca/coll/base/coll_base_find_available.c
index 710c5fd2ad1..f45690d4d99 100644
--- a/ompi/mca/coll/base/coll_base_find_available.c
+++ b/ompi/mca/coll/base/coll_base_find_available.c
@@ -107,12 +107,12 @@ int mca_coll_base_find_available(bool enable_progress_threads,
  * Query a specific component, coll v2.4.0
  */
 static inline int
-init_query_2_4_0(const mca_base_component_t * component,
+init_query_3_0_0(const mca_base_component_t * component,
                  bool enable_progress_threads,
                  bool enable_mpi_threads)
 {
-    mca_coll_base_component_2_4_0_t *coll =
-        (mca_coll_base_component_2_4_0_t *) component;
+    mca_coll_base_component_3_0_0_t *coll =
+        (mca_coll_base_component_3_0_0_t *) component;
 
     return coll->collm_init_query(enable_progress_threads,
                                   enable_mpi_threads);
@@ -133,10 +133,10 @@ static int init_query(const mca_base_component_t * component,
     /* This component has already been successfully opened.  So now
        query it. */
 
-    if (2 == component->mca_type_major_version &&
-        4 == component->mca_type_minor_version &&
+    if (3 == component->mca_type_major_version &&
+        0 == component->mca_type_minor_version &&
         0 == component->mca_type_release_version) {
-        ret = init_query_2_4_0(component, enable_progress_threads,
+        ret = init_query_3_0_0(component, enable_progress_threads,
                                enable_mpi_threads);
     } else {
         /* Unrecognized coll API version */
diff --git a/ompi/mca/coll/base/coll_base_frame.c b/ompi/mca/coll/base/coll_base_frame.c
index 5bb6fe38ace..07b7f85cf92 100644
--- a/ompi/mca/coll/base/coll_base_frame.c
+++ b/ompi/mca/coll/base/coll_base_frame.c
@@ -15,6 +15,7 @@
  * Copyright (c) 2014      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -30,8 +31,7 @@
 #include "ompi/mca/mca.h"
 #include "opal/util/output.h"
 #include "opal/mca/base/base.h"
-
-
+#include "opal/mca/base/mca_base_alias.h"
 #include "ompi/mca/coll/coll.h"
 #include "ompi/mca/coll/base/base.h"
 #include "ompi/mca/coll/base/coll_base_functions.h"
@@ -130,5 +130,12 @@ ompi_request_t** ompi_coll_base_comm_get_reqs(mca_coll_base_comm_t* data, int nr
     return data->mcct_reqs;
 }
 
-MCA_BASE_FRAMEWORK_DECLARE(ompi, coll, "Collectives", NULL, NULL, NULL,
+static int mca_coll_base_register(mca_base_register_flag_t flags)
+{
+    (void) mca_base_alias_register("ompi", "coll", "accelerator", "cuda", MCA_BASE_ALIAS_FLAG_DEPRECATED);
+    return OMPI_SUCCESS;
+}
+
+
+MCA_BASE_FRAMEWORK_DECLARE(ompi, coll, "Collectives", mca_coll_base_register, NULL, NULL,
                            mca_coll_base_static_components, 0);
diff --git a/ompi/mca/coll/base/coll_base_functions.h b/ompi/mca/coll/base/coll_base_functions.h
index acaf9dcdd38..eb1486fae77 100644
--- a/ompi/mca/coll/base/coll_base_functions.h
+++ b/ompi/mca/coll/base/coll_base_functions.h
@@ -68,28 +68,28 @@ typedef enum COLLTYPE {
 } COLLTYPE_T;
 
 /* defined arg lists to simply auto inclusion of user overriding decision functions */
-#define ALLGATHER_BASE_ARGS           const void *sendbuf, int sendcount, struct ompi_datatype_t *sendtype, void *recvbuf, int recvcount, struct ompi_datatype_t *recvtype, struct ompi_communicator_t *comm
-#define ALLGATHERV_BASE_ARGS          const void *sendbuf, int sendcount, struct ompi_datatype_t *sendtype, void *recvbuf, const int recvcounts[], const int displs[], struct ompi_datatype_t *recvtype, struct ompi_communicator_t *comm
-#define ALLREDUCE_BASE_ARGS           const void *sendbuf, void *recvbuf, int count, struct ompi_datatype_t *datatype, struct ompi_op_t *op, struct ompi_communicator_t *comm
-#define ALLTOALL_BASE_ARGS            const void *sendbuf, int sendcount, struct ompi_datatype_t *sendtype, void *recvbuf, int recvcount, struct ompi_datatype_t *recvtype, struct ompi_communicator_t *comm
-#define ALLTOALLV_BASE_ARGS           const void *sendbuf, const int sendcounts[], const int sdispls[], struct ompi_datatype_t *sendtype, void *recvbuf, const int recvcounts[], const int rdispls[], struct ompi_datatype_t *recvtype, struct ompi_communicator_t *comm
-#define ALLTOALLW_BASE_ARGS           const void *sendbuf, const int sendcounts[], const int sdispls[], struct ompi_datatype_t * const sendtypes[], void *recvbuf, const int recvcounts[], const int rdispls[], struct ompi_datatype_t * const recvtypes[], struct ompi_communicator_t *comm
+#define ALLGATHER_BASE_ARGS           const void *sendbuf, size_t sendcount, struct ompi_datatype_t *sendtype, void *recvbuf, size_t recvcount, struct ompi_datatype_t *recvtype, struct ompi_communicator_t *comm
+#define ALLGATHERV_BASE_ARGS          const void *sendbuf, size_t sendcount, struct ompi_datatype_t *sendtype, void *recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t displs, struct ompi_datatype_t *recvtype, struct ompi_communicator_t *comm
+#define ALLREDUCE_BASE_ARGS           const void *sendbuf, void *recvbuf, size_t count, struct ompi_datatype_t *datatype, struct ompi_op_t *op, struct ompi_communicator_t *comm
+#define ALLTOALL_BASE_ARGS            const void *sendbuf, size_t sendcount, struct ompi_datatype_t *sendtype, void *recvbuf, size_t recvcount, struct ompi_datatype_t *recvtype, struct ompi_communicator_t *comm
+#define ALLTOALLV_BASE_ARGS           const void *sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t sdispls, struct ompi_datatype_t *sendtype, void *recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t rdispls, struct ompi_datatype_t *recvtype, struct ompi_communicator_t *comm
+#define ALLTOALLW_BASE_ARGS           const void *sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t sdispls, struct ompi_datatype_t * const sendtypes[], void *recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t rdispls, struct ompi_datatype_t * const recvtypes[], struct ompi_communicator_t *comm
 #define BARRIER_BASE_ARGS             struct ompi_communicator_t *comm
-#define BCAST_BASE_ARGS               void *buffer, int count, struct ompi_datatype_t *datatype, int root, struct ompi_communicator_t *comm
-#define EXSCAN_BASE_ARGS              const void *sendbuf, void *recvbuf, int count, struct ompi_datatype_t *datatype, struct ompi_op_t *op, struct ompi_communicator_t *comm
-#define GATHER_BASE_ARGS              const void *sendbuf, int sendcount, struct ompi_datatype_t *sendtype, void *recvbuf, int recvcount, struct ompi_datatype_t *recvtype, int root, struct ompi_communicator_t *comm
-#define GATHERV_BASE_ARGS             const void *sendbuf, int sendcount, struct ompi_datatype_t *sendtype, void *recvbuf, const int recvcounts[], const int displs[], struct ompi_datatype_t *recvtype, int root, struct ompi_communicator_t *comm
-#define REDUCE_BASE_ARGS              const void *sendbuf, void *recvbuf, int count, struct ompi_datatype_t *datatype, struct ompi_op_t *op, int root, struct ompi_communicator_t *comm
-#define REDUCESCATTER_BASE_ARGS       const void *sendbuf, void *recvbuf, const int recvcounts[], struct ompi_datatype_t *datatype, struct ompi_op_t *op, struct ompi_communicator_t *comm
-#define REDUCESCATTERBLOCK_BASE_ARGS  const void *sendbuf, void *recvbuf, int recvcount, struct ompi_datatype_t *datatype, struct ompi_op_t *op, struct ompi_communicator_t *comm
-#define SCAN_BASE_ARGS                const void *sendbuf, void *recvbuf, int count, struct ompi_datatype_t *datatype, struct ompi_op_t *op, struct ompi_communicator_t *comm
-#define SCATTER_BASE_ARGS             const void *sendbuf, int sendcount, struct ompi_datatype_t *sendtype, void *recvbuf, int recvcount, struct ompi_datatype_t *recvtype, int root, struct ompi_communicator_t *comm
-#define SCATTERV_BASE_ARGS            const void *sendbuf, const int sendcounts[], const int displs[], struct ompi_datatype_t *sendtype, void *recvbuf, int recvcount, struct ompi_datatype_t *recvtype, int root, struct ompi_communicator_t *comm
-#define NEIGHBOR_ALLGATHER_BASE_ARGS  const void *sendbuf, int sendcount, struct ompi_datatype_t *sendtype, void *recvbuf, int recvcount, struct ompi_datatype_t *recvtype, struct ompi_communicator_t *comm
-#define NEIGHBOR_ALLGATHERV_BASE_ARGS const void *sendbuf, int sendcount, struct ompi_datatype_t *sendtype, void *recvbuf, const int recvcounts[], const int displs[], struct ompi_datatype_t *recvtype, struct ompi_communicator_t *comm
-#define NEIGHBOR_ALLTOALL_BASE_ARGS   const void *sendbuf, int sendcount, struct ompi_datatype_t *sendtype, void *recvbuf, int recvcount, struct ompi_datatype_t *recvtype, struct ompi_communicator_t *comm
-#define NEIGHBOR_ALLTOALLV_BASE_ARGS  const void *sendbuf, const int sendcounts[], const int sdispls[], struct ompi_datatype_t *sendtype, void *recvbuf, const int recvcounts[], const int rdispls[], struct ompi_datatype_t *recvtype, struct ompi_communicator_t *comm
-#define NEIGHBOR_ALLTOALLW_BASE_ARGS  const void *sendbuf, const int sendcounts[], const MPI_Aint sdispls[], struct ompi_datatype_t * const sendtypes[], void *recvbuf, const int recvcounts[], const MPI_Aint rdispls[], struct ompi_datatype_t * const recvtypes[], struct ompi_communicator_t *comm
+#define BCAST_BASE_ARGS               void *buffer, size_t count, struct ompi_datatype_t *datatype, int root, struct ompi_communicator_t *comm
+#define EXSCAN_BASE_ARGS              const void *sendbuf, void *recvbuf, size_t count, struct ompi_datatype_t *datatype, struct ompi_op_t *op, struct ompi_communicator_t *comm
+#define GATHER_BASE_ARGS              const void *sendbuf, size_t sendcount, struct ompi_datatype_t *sendtype, void *recvbuf, size_t recvcount, struct ompi_datatype_t *recvtype, int root, struct ompi_communicator_t *comm
+#define GATHERV_BASE_ARGS             const void *sendbuf, size_t sendcount, struct ompi_datatype_t *sendtype, void *recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t displs, struct ompi_datatype_t *recvtype, int root, struct ompi_communicator_t *comm
+#define REDUCE_BASE_ARGS              const void *sendbuf, void *recvbuf, size_t count, struct ompi_datatype_t *datatype, struct ompi_op_t *op, int root, struct ompi_communicator_t *comm
+#define REDUCESCATTER_BASE_ARGS       const void *sendbuf, void *recvbuf, ompi_count_array_t recvcounts, struct ompi_datatype_t *datatype, struct ompi_op_t *op, struct ompi_communicator_t *comm
+#define REDUCESCATTERBLOCK_BASE_ARGS  const void *sendbuf, void *recvbuf, size_t recvcount, struct ompi_datatype_t *datatype, struct ompi_op_t *op, struct ompi_communicator_t *comm
+#define SCAN_BASE_ARGS                const void *sendbuf, void *recvbuf, size_t count, struct ompi_datatype_t *datatype, struct ompi_op_t *op, struct ompi_communicator_t *comm
+#define SCATTER_BASE_ARGS             const void *sendbuf, size_t sendcount, struct ompi_datatype_t *sendtype, void *recvbuf, size_t recvcount, struct ompi_datatype_t *recvtype, int root, struct ompi_communicator_t *comm
+#define SCATTERV_BASE_ARGS            const void *sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t displs, struct ompi_datatype_t *sendtype, void *recvbuf, size_t recvcount, struct ompi_datatype_t *recvtype, int root, struct ompi_communicator_t *comm
+#define NEIGHBOR_ALLGATHER_BASE_ARGS  const void *sendbuf, size_t sendcount, struct ompi_datatype_t *sendtype, void *recvbuf, size_t recvcount, struct ompi_datatype_t *recvtype, struct ompi_communicator_t *comm
+#define NEIGHBOR_ALLGATHERV_BASE_ARGS const void *sendbuf, size_t sendcount, struct ompi_datatype_t *sendtype, void *recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t displs, struct ompi_datatype_t *recvtype, struct ompi_communicator_t *comm
+#define NEIGHBOR_ALLTOALL_BASE_ARGS   const void *sendbuf, size_t sendcount, struct ompi_datatype_t *sendtype, void *recvbuf, size_t recvcount, struct ompi_datatype_t *recvtype, struct ompi_communicator_t *comm
+#define NEIGHBOR_ALLTOALLV_BASE_ARGS  const void *sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t sdispls, struct ompi_datatype_t *sendtype, void *recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t rdispls, struct ompi_datatype_t *recvtype, struct ompi_communicator_t *comm
+#define NEIGHBOR_ALLTOALLW_BASE_ARGS  const void *sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t sdispls, struct ompi_datatype_t * const sendtypes[], void *recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t rdispls, struct ompi_datatype_t * const recvtypes[], struct ompi_communicator_t *comm
 
 #define ALLGATHER_ARGS           ALLGATHER_BASE_ARGS,           mca_coll_base_module_t *module
 #define ALLGATHERV_ARGS          ALLGATHERV_BASE_ARGS,          mca_coll_base_module_t *module
@@ -187,15 +187,18 @@ typedef enum COLLTYPE {
 BEGIN_C_DECLS
 
 /* All Gather */
-int ompi_coll_base_allgather_intra_bruck(ALLGATHER_ARGS);
 int ompi_coll_base_allgather_intra_recursivedoubling(ALLGATHER_ARGS);
+int ompi_coll_base_allgather_intra_sparbit(ALLGATHER_ARGS);
 int ompi_coll_base_allgather_intra_ring(ALLGATHER_ARGS);
 int ompi_coll_base_allgather_intra_neighborexchange(ALLGATHER_ARGS);
 int ompi_coll_base_allgather_intra_basic_linear(ALLGATHER_ARGS);
 int ompi_coll_base_allgather_intra_two_procs(ALLGATHER_ARGS);
+int ompi_coll_base_allgather_intra_k_bruck(ALLGATHER_ARGS, int radix);
+int ompi_coll_base_allgather_direct_messaging(ALLGATHER_ARGS);
 
 /* All GatherV */
 int ompi_coll_base_allgatherv_intra_bruck(ALLGATHERV_ARGS);
+int ompi_coll_base_allgatherv_intra_sparbit(ALLGATHERV_ARGS);
 int ompi_coll_base_allgatherv_intra_ring(ALLGATHERV_ARGS);
 int ompi_coll_base_allgatherv_intra_neighborexchange(ALLGATHERV_ARGS);
 int ompi_coll_base_allgatherv_intra_basic_default(ALLGATHERV_ARGS);
@@ -208,6 +211,7 @@ int ompi_coll_base_allreduce_intra_ring(ALLREDUCE_ARGS);
 int ompi_coll_base_allreduce_intra_ring_segmented(ALLREDUCE_ARGS, uint32_t segsize);
 int ompi_coll_base_allreduce_intra_basic_linear(ALLREDUCE_ARGS);
 int ompi_coll_base_allreduce_intra_redscat_allgather(ALLREDUCE_ARGS);
+int ompi_coll_base_allreduce_intra_allgather_reduce(ALLREDUCE_ARGS);
 
 /* AlltoAll */
 int ompi_coll_base_alltoall_intra_pairwise(ALLTOALL_ARGS);
@@ -215,7 +219,7 @@ int ompi_coll_base_alltoall_intra_bruck(ALLTOALL_ARGS);
 int ompi_coll_base_alltoall_intra_basic_linear(ALLTOALL_ARGS);
 int ompi_coll_base_alltoall_intra_linear_sync(ALLTOALL_ARGS, int max_requests);
 int ompi_coll_base_alltoall_intra_two_procs(ALLTOALL_ARGS);
-int mca_coll_base_alltoall_intra_basic_inplace(const void *rbuf, int rcount,
+int mca_coll_base_alltoall_intra_basic_inplace(const void *rbuf, size_t rcount,
                                                struct ompi_datatype_t *rdtype,
                                                struct ompi_communicator_t *comm,
                                                mca_coll_base_module_t *module);  /* special version for INPLACE */
@@ -223,7 +227,7 @@ int mca_coll_base_alltoall_intra_basic_inplace(const void *rbuf, int rcount,
 /* AlltoAllV */
 int ompi_coll_base_alltoallv_intra_pairwise(ALLTOALLV_ARGS);
 int ompi_coll_base_alltoallv_intra_basic_linear(ALLTOALLV_ARGS);
-int mca_coll_base_alltoallv_intra_basic_inplace(const void *rbuf, const int *rcounts, const int *rdisps,
+int mca_coll_base_alltoallv_intra_basic_inplace(const void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
                                                 struct ompi_datatype_t *rdtype,
                                                 struct ompi_communicator_t *comm,
                                                 mca_coll_base_module_t *module);  /* special version for INPLACE */
@@ -263,7 +267,7 @@ int ompi_coll_base_gather_intra_linear_sync(GATHER_ARGS, int first_segment_size)
 /* GatherV */
 
 /* Reduce */
-int ompi_coll_base_reduce_generic(REDUCE_ARGS, ompi_coll_tree_t* tree, int count_by_segment, int max_outstanding_reqs);
+int ompi_coll_base_reduce_generic(REDUCE_ARGS, ompi_coll_tree_t* tree, size_t count_by_segment, int max_outstanding_reqs);
 int ompi_coll_base_reduce_intra_basic_linear(REDUCE_ARGS);
 int ompi_coll_base_reduce_intra_chain(REDUCE_ARGS, uint32_t segsize, int fanout, int max_outstanding_reqs );
 int ompi_coll_base_reduce_intra_pipeline(REDUCE_ARGS, uint32_t segsize, int max_outstanding_reqs );
@@ -271,6 +275,7 @@ int ompi_coll_base_reduce_intra_binary(REDUCE_ARGS, uint32_t segsize, int max_ou
 int ompi_coll_base_reduce_intra_binomial(REDUCE_ARGS, uint32_t segsize, int max_outstanding_reqs );
 int ompi_coll_base_reduce_intra_in_order_binary(REDUCE_ARGS, uint32_t segsize, int max_outstanding_reqs );
 int ompi_coll_base_reduce_intra_redscat_gather(REDUCE_ARGS);
+int ompi_coll_base_reduce_intra_knomial(REDUCE_ARGS, uint32_t segsize, int max_outstanding_reqs, int radix);
 
 /* Reduce_scatter */
 int ompi_coll_base_reduce_scatter_intra_nonoverlapping(REDUCESCATTER_ARGS);
@@ -297,21 +302,21 @@ int ompi_coll_base_scatter_intra_linear_nb(SCATTER_ARGS, int max_reqs);
 /* ScatterV */
 
 /* Reduce_local */
-int mca_coll_base_reduce_local(const void *inbuf, void *inoutbuf, int count,
+int mca_coll_base_reduce_local(const void *inbuf, void *inoutbuf, size_t count,
                                struct ompi_datatype_t * dtype, struct ompi_op_t * op,
                                mca_coll_base_module_t *module);
 
 #if OPAL_ENABLE_FT_MPI
 /* Agreement */
 int ompi_coll_base_agree_noft(void *contrib,
-                              int dt_count,
+                              size_t dt_count,
                               struct ompi_datatype_t *dt,
                               struct ompi_op_t *op,
                               struct ompi_group_t **group, bool update_grp,
                               struct ompi_communicator_t* comm,
                               mca_coll_base_module_t *module);
 int ompi_coll_base_iagree_noft(void *contrib,
-                               int dt_count,
+                               size_t dt_count,
                                struct ompi_datatype_t *dt,
                                struct ompi_op_t *op,
                                struct ompi_group_t **group, bool update_grp,
@@ -517,7 +522,7 @@ OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_coll_base_comm_t);
 
 /**
  * Free all requests in an array. As these requests are usually used during
- * collective communications, and as on a succesful collective they are
+ * collective communications, and as on a successful collective they are
  * expected to be released during the corresponding wait, the array should
  * generally be empty. However, this function might be used on error conditions
  * where it will allow a correct cleanup.
@@ -542,7 +547,7 @@ static inline void ompi_coll_base_free_reqs(ompi_request_t **reqs, int count)
                  * free, as this is the best that can be done in this case. */
                 ompi_request_cancel(reqs[i]);
                 ompi_request_wait(&reqs[i], MPI_STATUS_IGNORE);
-            } else /* this 'else' intentionaly spills outside the ifdef */
+            } else /* this 'else' intentionally spills outside the ifdef */
 #endif /* OPAL_ENABLE_FT_MPI */
             ompi_request_free(&reqs[i]);
         }
diff --git a/ompi/mca/coll/base/coll_base_gather.c b/ompi/mca/coll/base/coll_base_gather.c
index 6fd1e981461..a4486152157 100644
--- a/ompi/mca/coll/base/coll_base_gather.c
+++ b/ompi/mca/coll/base/coll_base_gather.c
@@ -38,15 +38,16 @@
 /* Todo: gather_intra_generic, gather_intra_binary, gather_intra_chain,
  * gather_intra_pipeline, segmentation? */
 int
-ompi_coll_base_gather_intra_binomial(const void *sbuf, int scount,
+ompi_coll_base_gather_intra_binomial(const void *sbuf, size_t scount,
                                       struct ompi_datatype_t *sdtype,
-                                      void *rbuf, int rcount,
+                                      void *rbuf, size_t rcount,
                                       struct ompi_datatype_t *rdtype,
                                       int root,
                                       struct ompi_communicator_t *comm,
                                       mca_coll_base_module_t *module)
 {
-    int line = -1, i, rank, vrank, size, total_recv = 0, err;
+    int line = -1, i, rank, vrank, size, err;
+    size_t total_recv = 0;
     char *ptmp     = NULL, *tempbuf  = NULL;
     ompi_coll_tree_t* bmtree;
     MPI_Status status;
@@ -157,7 +158,7 @@ ompi_coll_base_gather_intra_binomial(const void *sbuf, int scount,
     if (rank != root) {
         /* all nodes except root send to parents */
         OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
-                     "ompi_coll_base_gather_intra_binomial rank %d send %d count %d\n",
+                     "ompi_coll_base_gather_intra_binomial rank %d send %d count %lu\n",
                      rank, bmtree->tree_prev, total_recv));
 
         err = MCA_PML_CALL(send(ptmp, total_recv, sdtype,
@@ -205,9 +206,9 @@ ompi_coll_base_gather_intra_binomial(const void *sbuf, int scount,
  *	Returns:	- MPI_SUCCESS or error code
  */
 int
-ompi_coll_base_gather_intra_linear_sync(const void *sbuf, int scount,
+ompi_coll_base_gather_intra_linear_sync(const void *sbuf, size_t scount,
                                          struct ompi_datatype_t *sdtype,
-                                         void *rbuf, int rcount,
+                                         void *rbuf, size_t rcount,
                                          struct ompi_datatype_t *rdtype,
                                          int root,
                                          struct ompi_communicator_t *comm,
@@ -331,8 +332,10 @@ ompi_coll_base_gather_intra_linear_sync(const void *sbuf, int scount,
             for( i = 0; i < size; i++ ) {
                 if (MPI_REQUEST_NULL == reqs[i]) continue;
                 if (MPI_ERR_PENDING == reqs[i]->req_status.MPI_ERROR) continue;
-                ret = reqs[i]->req_status.MPI_ERROR;
-                break;
+                if (reqs[i]->req_status.MPI_ERROR != MPI_SUCCESS) {
+                    ret = reqs[i]->req_status.MPI_ERROR;
+                    break;
+                }
             }
         }
         ompi_coll_base_free_reqs(reqs, size);
@@ -365,9 +368,9 @@ ompi_coll_base_gather_intra_linear_sync(const void *sbuf, int scount,
  *	Returns:	- MPI_SUCCESS or error code
  */
 int
-ompi_coll_base_gather_intra_basic_linear(const void *sbuf, int scount,
+ompi_coll_base_gather_intra_basic_linear(const void *sbuf, size_t scount,
                                           struct ompi_datatype_t *sdtype,
-                                          void *rbuf, int rcount,
+                                          void *rbuf, size_t rcount,
                                           struct ompi_datatype_t *rdtype,
                                           int root,
                                           struct ompi_communicator_t *comm,
diff --git a/ompi/mca/coll/base/coll_base_reduce.c b/ompi/mca/coll/base/coll_base_reduce.c
index 7950e762982..f127abef311 100644
--- a/ompi/mca/coll/base/coll_base_reduce.c
+++ b/ompi/mca/coll/base/coll_base_reduce.c
@@ -17,6 +17,8 @@
  * Copyright (c) 2016-2017 IBM Corporation.  All rights reserved.
  * Copyright (c) 2018      Siberian State University of Telecommunications
  *                         and Information Science. All rights reserved.
+ * Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2024      Stony Brook University.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -39,7 +41,7 @@
 #include "coll_base_topo.h"
 #include "coll_base_util.h"
 
-int mca_coll_base_reduce_local(const void *inbuf, void *inoutbuf, int count,
+int mca_coll_base_reduce_local(const void *inbuf, void *inoutbuf, size_t count,
                                struct ompi_datatype_t * dtype, struct ompi_op_t * op,
                                mca_coll_base_module_t *module)
 {
@@ -59,11 +61,11 @@ int mca_coll_base_reduce_local(const void *inbuf, void *inoutbuf, int count,
  * for the first block: thus we must copy sendbuf to accumbuf on intermediate
  * to keep the optimized loop happy.
  */
-int ompi_coll_base_reduce_generic( const void* sendbuf, void* recvbuf, int original_count,
+int ompi_coll_base_reduce_generic( const void* sendbuf, void* recvbuf, size_t original_count,
                                     ompi_datatype_t* datatype, ompi_op_t* op,
                                     int root, ompi_communicator_t* comm,
                                     mca_coll_base_module_t *module,
-                                    ompi_coll_tree_t* tree, int count_by_segment,
+                                    ompi_coll_tree_t* tree, size_t count_by_segment,
                                     int max_outstanding_reqs )
 {
     char *inbuf[2] = {NULL, NULL}, *inbuf_free[2] = {NULL, NULL};
@@ -87,7 +89,7 @@ int ompi_coll_base_reduce_generic( const void* sendbuf, void* recvbuf, int origi
         sendtmpbuf = (char *)recvbuf;
     }
 
-    OPAL_OUTPUT((ompi_coll_base_framework.framework_output, "coll:base:reduce_generic count %d, msg size %ld, segsize %ld, max_requests %d",
+    OPAL_OUTPUT((ompi_coll_base_framework.framework_output, "coll:base:reduce_generic count %zu, msg size %ld, segsize %ld, max_requests %d",
                  original_count, (unsigned long)((ptrdiff_t)num_segments * (ptrdiff_t)segment_increment),
                  (unsigned long)segment_increment, max_outstanding_reqs));
 
@@ -98,7 +100,7 @@ int ompi_coll_base_reduce_generic( const void* sendbuf, void* recvbuf, int origi
     if( tree->tree_nextsize > 0 ) {
         ptrdiff_t real_segment_size;
 
-        /* handle non existant recv buffer (i.e. its NULL) and
+        /* handle non existent recv buffer (i.e. its NULL) and
            protect the recv buffer on non-root nodes */
         accumbuf = (char*)recvbuf;
         if( (NULL == accumbuf) || (root != rank) ) {
@@ -112,7 +114,7 @@ int ompi_coll_base_reduce_generic( const void* sendbuf, void* recvbuf, int origi
         }
 
         /* If this is a non-commutative operation we must copy
-           sendbuf to the accumbuf, in order to simplfy the loops */
+           sendbuf to the accumbuf, in order to simplify the loops */
         
         if (!ompi_op_is_commute(op) && MPI_IN_PLACE != sendbuf) {
             ompi_datatype_copy_content_same_ddt(datatype, original_count,
@@ -250,14 +252,14 @@ int ompi_coll_base_reduce_generic( const void* sendbuf, void* recvbuf, int origi
        the number of segments we have two options:
        - send all segments using blocking send to the parent, or
        - avoid overflooding the parent nodes by limiting the number of
-       outstanding requests to max_oustanding_reqs.
+       outstanding requests to max_outstanding_reqs.
        TODO/POSSIBLE IMPROVEMENT: If there is a way to determine the eager size
        for the current communication, synchronization should be used only
        when the message/segment size is smaller than the eager size.
     */
     else {
 
-        /* If the number of segments is less than a maximum number of oustanding
+        /* If the number of segments is less than a maximum number of outstanding
            requests or there is no limit on the maximum number of outstanding
            requests, we send data to the parent using blocking send */
         if ((0 == max_outstanding_reqs) ||
@@ -313,7 +315,7 @@ int ompi_coll_base_reduce_generic( const void* sendbuf, void* recvbuf, int origi
                 ret = ompi_request_wait(&sreq[creq], MPI_STATUS_IGNORE);
                 if (ret != MPI_SUCCESS) { line = __LINE__; goto error_hndl;  }
 
-                if( original_count < count_by_segment ) {
+                if( original_count < (size_t) count_by_segment ) {
                     count_by_segment = original_count;
                 }
                 ret = MCA_PML_CALL( isend((char*)sendbuf +
@@ -343,8 +345,10 @@ int ompi_coll_base_reduce_generic( const void* sendbuf, void* recvbuf, int origi
         for( i = 0; i < 2; i++ ) {
             if (MPI_REQUEST_NULL == reqs[i]) continue;
             if (MPI_ERR_PENDING == reqs[i]->req_status.MPI_ERROR) continue;
-            ret = reqs[i]->req_status.MPI_ERROR;
-            break;
+            if (reqs[i]->req_status.MPI_ERROR != MPI_SUCCESS) {
+                ret = reqs[i]->req_status.MPI_ERROR;
+                break;
+            }
         }
     }
     ompi_coll_base_free_reqs(reqs, 2);
@@ -353,8 +357,10 @@ int ompi_coll_base_reduce_generic( const void* sendbuf, void* recvbuf, int origi
             for( i = 0; i < max_outstanding_reqs; i++ ) {
                 if (MPI_REQUEST_NULL == sreq[i]) continue;
                 if (MPI_ERR_PENDING == sreq[i]->req_status.MPI_ERROR) continue;
-                ret = sreq[i]->req_status.MPI_ERROR;
-                break;
+                if (sreq[i]->req_status.MPI_ERROR != MPI_SUCCESS) {
+                    ret = sreq[i]->req_status.MPI_ERROR;
+                    break;
+                }
             }
         }
         ompi_coll_base_free_reqs(sreq, max_outstanding_reqs);
@@ -376,7 +382,7 @@ int ompi_coll_base_reduce_generic( const void* sendbuf, void* recvbuf, int origi
      meaning that at least one datatype must fit in the segment !
 */
 
-int ompi_coll_base_reduce_intra_chain( const void *sendbuf, void *recvbuf, int count,
+int ompi_coll_base_reduce_intra_chain( const void *sendbuf, void *recvbuf, size_t count,
                                         ompi_datatype_t* datatype,
                                         ompi_op_t* op, int root,
                                         ompi_communicator_t* comm,
@@ -384,7 +390,7 @@ int ompi_coll_base_reduce_intra_chain( const void *sendbuf, void *recvbuf, int c
                                         uint32_t segsize, int fanout,
                                         int max_outstanding_reqs )
 {
-    int segcount = count;
+    size_t segcount = count;
     size_t typelng;
     mca_coll_base_module_t *base_module = (mca_coll_base_module_t*) module;
     mca_coll_base_comm_t *data = base_module->base_data;
@@ -407,14 +413,14 @@ int ompi_coll_base_reduce_intra_chain( const void *sendbuf, void *recvbuf, int c
 
 
 int ompi_coll_base_reduce_intra_pipeline( const void *sendbuf, void *recvbuf,
-                                           int count, ompi_datatype_t* datatype,
+                                           size_t count, ompi_datatype_t* datatype,
                                            ompi_op_t* op, int root,
                                            ompi_communicator_t* comm,
                                            mca_coll_base_module_t *module,
                                            uint32_t segsize,
                                            int max_outstanding_reqs  )
 {
-    int segcount = count;
+    size_t segcount = count;
     size_t typelng;
     mca_coll_base_module_t *base_module = (mca_coll_base_module_t*) module;
     mca_coll_base_comm_t *data = base_module->base_data;
@@ -438,14 +444,14 @@ int ompi_coll_base_reduce_intra_pipeline( const void *sendbuf, void *recvbuf,
 }
 
 int ompi_coll_base_reduce_intra_binary( const void *sendbuf, void *recvbuf,
-                                         int count, ompi_datatype_t* datatype,
+                                         size_t count, ompi_datatype_t* datatype,
                                          ompi_op_t* op, int root,
                                          ompi_communicator_t* comm,
                                          mca_coll_base_module_t *module,
                                          uint32_t segsize,
                                          int max_outstanding_reqs  )
 {
-    int segcount = count;
+    size_t segcount = count;
     size_t typelng;
     mca_coll_base_module_t *base_module = (mca_coll_base_module_t*) module;
     mca_coll_base_comm_t *data = base_module->base_data;
@@ -469,14 +475,14 @@ int ompi_coll_base_reduce_intra_binary( const void *sendbuf, void *recvbuf,
 }
 
 int ompi_coll_base_reduce_intra_binomial( const void *sendbuf, void *recvbuf,
-                                           int count, ompi_datatype_t* datatype,
+                                           size_t count, ompi_datatype_t* datatype,
                                            ompi_op_t* op, int root,
                                            ompi_communicator_t* comm,
                                            mca_coll_base_module_t *module,
                                            uint32_t segsize,
                                            int max_outstanding_reqs  )
 {
-    int segcount = count;
+    size_t segcount = count;
     size_t typelng;
     mca_coll_base_module_t *base_module = (mca_coll_base_module_t*) module;
     mca_coll_base_comm_t *data = base_module->base_data;
@@ -507,7 +513,7 @@ int ompi_coll_base_reduce_intra_binomial( const void *sendbuf, void *recvbuf,
  * Returns:       MPI_SUCCESS or error code
  */
 int ompi_coll_base_reduce_intra_in_order_binary( const void *sendbuf, void *recvbuf,
-                                                  int count,
+                                                  size_t count,
                                                   ompi_datatype_t* datatype,
                                                   ompi_op_t* op, int root,
                                                   ompi_communicator_t* comm,
@@ -578,7 +584,10 @@ int ompi_coll_base_reduce_intra_in_order_binary( const void *sendbuf, void *recv
                                           op, io_root, comm, module,
                                           data->cached_in_order_bintree,
                                           segcount, max_outstanding_reqs );
-    if (MPI_SUCCESS != ret) { return ret; }
+    if (MPI_SUCCESS != ret) {
+        free(tmpbuf_free);
+        return ret;
+    }
 
     /* Clean up */
     if (io_root != root) {
@@ -587,14 +596,20 @@ int ompi_coll_base_reduce_intra_in_order_binary( const void *sendbuf, void *recv
             ret = MCA_PML_CALL(recv(recvbuf, count, datatype, io_root,
                                     MCA_COLL_BASE_TAG_REDUCE, comm,
                                     MPI_STATUS_IGNORE));
-            if (MPI_SUCCESS != ret) { return ret; }
+            if (MPI_SUCCESS != ret) {
+                free(tmpbuf_free);
+                return ret;
+            }
 
         } else if (io_root == rank) {
             /* Send result from use_this_recvbuf to root */
             ret = MCA_PML_CALL(send(use_this_recvbuf, count, datatype, root,
                                     MCA_COLL_BASE_TAG_REDUCE,
                                     MCA_PML_BASE_SEND_STANDARD, comm));
-            if (MPI_SUCCESS != ret) { return ret; }
+            if (MPI_SUCCESS != ret) {
+                free(tmpbuf_free);
+                return ret;
+            }
         }
     }
     if (NULL != tmpbuf_free) {
@@ -624,7 +639,7 @@ int ompi_coll_base_reduce_intra_in_order_binary( const void *sendbuf, void *recv
  *  Returns:    - MPI_SUCCESS or error code
  */
 int
-ompi_coll_base_reduce_intra_basic_linear(const void *sbuf, void *rbuf, int count,
+ompi_coll_base_reduce_intra_basic_linear(const void *sbuf, void *rbuf, size_t count,
                                          struct ompi_datatype_t *dtype,
                                          struct ompi_op_t *op,
                                          int root,
@@ -795,7 +810,7 @@ ompi_coll_base_reduce_intra_basic_linear(const void *sbuf, void *rbuf, int count
  *                  in the root process.
  */
 int ompi_coll_base_reduce_intra_redscat_gather(
-    const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype,
+    const void *sbuf, void *rbuf, size_t count, struct ompi_datatype_t *dtype,
     struct ompi_op_t *op, int root, struct ompi_communicator_t *comm,
     mca_coll_base_module_t *module)
 {
@@ -808,12 +823,14 @@ int ompi_coll_base_reduce_intra_redscat_gather(
 
     /* Find nearest power-of-two less than or equal to comm_size */
     int nsteps = opal_hibit(comm_size, comm->c_cube_dim + 1);   /* ilog2(comm_size) */
-    assert(nsteps >= 0);
+    if (-1 == nsteps) {
+        return MPI_ERR_ARG;
+    }
     int nprocs_pof2 = 1 << nsteps;                              /* flp2(comm_size) */
 
-    if (nprocs_pof2 < 2 || count < nprocs_pof2 || !ompi_op_is_commute(op)) {
+    if (nprocs_pof2 < 2 || count < (size_t) nprocs_pof2 || !ompi_op_is_commute(op)) {
         OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
-                     "coll:base:reduce_intra_redscat_gather: rank %d/%d count %d "
+                     "coll:base:reduce_intra_redscat_gather: rank %d/%d count %zu"
                      "switching to basic linear reduce", rank, comm_size, count));
         return ompi_coll_base_reduce_intra_basic_linear(sbuf, rbuf, count, dtype,
                                                         op, root, comm, module);
@@ -961,7 +978,7 @@ int ompi_coll_base_reduce_intra_redscat_gather(
 
         for (int mask = 1; mask < nprocs_pof2; mask <<= 1) {
             /*
-             * On each iteration: rindex[step] = sindex[step] -- begining of the
+             * On each iteration: rindex[step] = sindex[step] -- beginning of the
              * current window. Length of the current window is storded in wsize.
              */
             int vdest = vrank ^ mask;
@@ -1126,3 +1143,169 @@ int ompi_coll_base_reduce_intra_redscat_gather(
         free(scount);
     return err;
 }
+
+/*
+ * ompi_coll_base_reduce_intra_knomial
+ *
+ * Function:  reduce using k-nomial tree algorithm
+ * Accepts:   Same arguments as MPI_Reduce, plus radix
+ * Returns:   MPI_SUCCESS or error code
+ * Parameters: radix -- k-nomial tree radix (>= 2)
+ *
+ * Time complexity: (radix - 1)O(\log_{radix}(comm_size))
+ *
+ * Example, comm_size=10
+ *    radix=2         radix=3             radix=4
+ *       0               0                   0
+ *    / / \ \       / /  |  \ \         /   / \ \ \
+ *   8 4   2 1     9 3   6   1 2       4   8  1 2 3
+ *   | |\  |         |\  |\           /|\  |
+ *   9 6 5 3         4 5 7 8         5 6 7 9
+ *     |
+ *     7
+ */
+int ompi_coll_base_reduce_intra_knomial( const void *sendbuf, void *recvbuf,
+                                           size_t count, ompi_datatype_t* datatype,
+                                           ompi_op_t* op, int root,
+                                           ompi_communicator_t* comm,
+                                           mca_coll_base_module_t *module,
+                                           uint32_t segsize,
+                                           int max_outstanding_reqs, int radix)
+{
+    int err = OMPI_SUCCESS, rank, line;
+    ptrdiff_t extent, lb;
+    size_t dtype_size;
+    char *child_buf = NULL;
+    char *child_buf_start = NULL;
+    char *reduce_buf = NULL;
+    char *reduce_buf_start = NULL;
+    char *sendtmpbuf = NULL;
+    mca_coll_base_module_t *base_module = (mca_coll_base_module_t*) module;
+    mca_coll_base_comm_t *data = base_module->base_data;
+    ompi_coll_tree_t* tree;
+    int num_children;
+    bool is_leaf;
+    ptrdiff_t buf_size, gap = 0;
+    int max_reqs = 0, num_reqs;
+    ompi_request_t **reqs = NULL;
+
+    OPAL_OUTPUT((ompi_coll_base_framework.framework_output, "coll:base:ompi_coll_base_reduce_intra_knomial msg size %zu, max_requests %d",
+                 count, max_outstanding_reqs));
+
+    rank = ompi_comm_rank(comm);
+
+    // create a k-nomial tree with radix 4
+    COLL_BASE_UPDATE_KMTREE(comm, base_module, root, radix);
+    if (NULL == data->cached_kmtree) {
+        // fail to create knomial tree fallback to previous allreduce method
+        OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
+                     "REDUCE: failed to create knomial tree. \n"));
+        goto err_hndl;
+    }
+
+    tree = data->cached_kmtree;
+    num_children = tree->tree_nextsize;
+    is_leaf = (tree->tree_nextsize == 0) ? true : false;
+
+    ompi_datatype_get_extent(datatype, &lb, &extent);
+    ompi_datatype_type_size(datatype, &dtype_size);
+
+    sendtmpbuf = (char*) sendbuf;
+    if( sendbuf == MPI_IN_PLACE ) {
+        sendtmpbuf = (char *)recvbuf;
+    }
+    buf_size = opal_datatype_span(&datatype->super, (int64_t)count, &gap);
+    reduce_buf = (char *)malloc(buf_size);
+    reduce_buf_start = reduce_buf - gap;
+    err = ompi_datatype_copy_content_same_ddt(datatype, count,
+                                              (char*)reduce_buf_start,
+                                              (char*)sendtmpbuf);
+    if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
+
+    // do transfer in a single transaction instead of segments
+    num_reqs = 0;
+    max_reqs = num_children;
+    if(!is_leaf) {
+        buf_size = opal_datatype_span(&datatype->super, (int64_t)count * num_children, &gap);
+        child_buf = (char *)malloc(buf_size);
+        child_buf_start = child_buf - gap;
+        reqs = ompi_coll_base_comm_get_reqs(data, max_reqs);
+    }
+
+    for (int i = 0; i < num_children; i++) {
+        int child = tree->tree_next[i];
+        err = MCA_PML_CALL(irecv(child_buf_start + (ptrdiff_t)i * count * extent,
+                                 count,
+                                 datatype,
+                                 child,
+                                 MCA_COLL_BASE_TAG_REDUCE,
+                                 comm,
+                                 &reqs[num_reqs++]));
+        if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
+    }
+
+    if (num_reqs > 0) {
+        err = ompi_request_wait_all(num_reqs, reqs, MPI_STATUS_IGNORE);
+        if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
+    }
+
+    for (int i = 0; i < num_children; i++) {
+        ompi_op_reduce(op,
+                       child_buf_start + (ptrdiff_t)i * count * extent,
+                       reduce_buf,
+                       count,
+                       datatype);
+    }
+
+    if (rank != root) {
+        err = MCA_PML_CALL(send(reduce_buf_start,
+                                count,
+                                datatype,
+                                tree->tree_prev,
+                                MCA_COLL_BASE_TAG_REDUCE,
+                                MCA_PML_BASE_SEND_STANDARD,
+                                comm));
+        if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
+    }
+
+    if (rank == root) {
+        err = ompi_datatype_copy_content_same_ddt(datatype, count,
+                                                  (char*)recvbuf,
+                                                  (char*)reduce_buf_start);
+        if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
+    }
+
+    if (NULL != child_buf) free(child_buf);
+    if (NULL != reduce_buf) free(reduce_buf);
+    return MPI_SUCCESS;
+
+ err_hndl:
+    if (NULL != child_buf) {
+        free(child_buf);
+        child_buf = NULL;
+        child_buf_start = NULL;
+    }
+    if (NULL != reduce_buf) {
+        free(reduce_buf);
+        reduce_buf = NULL;
+        reduce_buf_start = NULL;
+    }
+    if( NULL != reqs ) {
+        if (MPI_ERR_IN_STATUS == err) {
+            for( num_reqs = 0; num_reqs < tree->tree_nextsize; num_reqs++ ) {
+                if (MPI_REQUEST_NULL == reqs[num_reqs]) continue;
+                if (MPI_ERR_PENDING == reqs[num_reqs]->req_status.MPI_ERROR) continue;
+                if (reqs[num_reqs]->req_status.MPI_ERROR != MPI_SUCCESS) {
+                    err = reqs[num_reqs]->req_status.MPI_ERROR;
+                    break;
+                }
+            }
+        }
+        ompi_coll_base_free_reqs(reqs, max_reqs);
+    }
+    OPAL_OUTPUT((ompi_coll_base_framework.framework_output,  "%s:%4d\tError occurred %d, rank %2d",
+                 __FILE__, line, err, rank));
+    (void)line;  // silence compiler warning
+    return err;
+
+}
diff --git a/ompi/mca/coll/base/coll_base_reduce_scatter.c b/ompi/mca/coll/base/coll_base_reduce_scatter.c
index 984a91787a0..7a838936378 100644
--- a/ompi/mca/coll/base/coll_base_reduce_scatter.c
+++ b/ompi/mca/coll/base/coll_base_reduce_scatter.c
@@ -45,13 +45,15 @@
  * appropriate scatterv call.
  */
 int ompi_coll_base_reduce_scatter_intra_nonoverlapping(const void *sbuf, void *rbuf,
-                                                        const int *rcounts,
+                                                        ompi_count_array_t rcounts,
                                                         struct ompi_datatype_t *dtype,
                                                         struct ompi_op_t *op,
                                                         struct ompi_communicator_t *comm,
                                                         mca_coll_base_module_t *module)
 {
-    int err, i, rank, size, total_count, *displs = NULL;
+    int err, i, rank, size, total_count;
+    ptrdiff_t *displs = NULL;
+    ompi_disp_array_t displs_arg;
     const int root = 0;
     char *tmprbuf = NULL, *tmprbuf_free = NULL;
 
@@ -60,7 +62,7 @@ int ompi_coll_base_reduce_scatter_intra_nonoverlapping(const void *sbuf, void *r
 
     OPAL_OUTPUT((ompi_coll_base_framework.framework_output,"coll:base:reduce_scatter_intra_nonoverlapping, rank %d", rank));
 
-    for (i = 0, total_count = 0; i < size; i++) { total_count += rcounts[i]; }
+    for (i = 0, total_count = 0; i < size; i++) { total_count += ompi_count_array_get(rcounts, i); }
 
     /* Reduce to rank 0 (root) and scatterv */
     tmprbuf = (char*) rbuf;
@@ -91,18 +93,19 @@ int ompi_coll_base_reduce_scatter_intra_nonoverlapping(const void *sbuf, void *r
         return err;
     }
 
-    displs = (int*) malloc(size * sizeof(int));
+    displs = (ptrdiff_t*) malloc(size * sizeof(ptrdiff_t));
     displs[0] = 0;
     for (i = 1; i < size; i++) {
-        displs[i] = displs[i-1] + rcounts[i-1];
+        displs[i] = displs[i-1] + ompi_count_array_get(rcounts, i-1);
     }
+    OMPI_DISP_ARRAY_INIT(&displs_arg, displs);
     if (MPI_IN_PLACE == sbuf && root == rank) {
-        err =  comm->c_coll->coll_scatterv (tmprbuf, rcounts, displs, dtype,
+        err =  comm->c_coll->coll_scatterv (tmprbuf, rcounts, displs_arg, dtype,
                                            MPI_IN_PLACE, 0, MPI_DATATYPE_NULL,
                                            root, comm, comm->c_coll->coll_scatterv_module);
     } else {
-        err =  comm->c_coll->coll_scatterv (tmprbuf, rcounts, displs, dtype,
-                                           rbuf, rcounts[rank], dtype,
+        err =  comm->c_coll->coll_scatterv (tmprbuf, rcounts, displs_arg, dtype,
+                                           rbuf, ompi_count_array_get(rcounts, rank), dtype,
                                            root, comm, comm->c_coll->coll_scatterv_module);
     }
     free(displs);
@@ -131,14 +134,16 @@ int ompi_coll_base_reduce_scatter_intra_nonoverlapping(const void *sbuf, void *r
 int
 ompi_coll_base_reduce_scatter_intra_basic_recursivehalving( const void *sbuf,
                                                             void *rbuf,
-                                                            const int *rcounts,
+                                                            ompi_count_array_t rcounts,
                                                             struct ompi_datatype_t *dtype,
                                                             struct ompi_op_t *op,
                                                             struct ompi_communicator_t *comm,
                                                             mca_coll_base_module_t *module)
 {
-    int i, rank, size, count, err = OMPI_SUCCESS;
-    int tmp_size, remain = 0, tmp_rank, *disps = NULL;
+    int i, rank, size, err = OMPI_SUCCESS;
+    int tmp_size, remain = 0, tmp_rank;
+    size_t count;
+    ptrdiff_t *disps = NULL;
     ptrdiff_t extent, buf_size, gap = 0;
     char *recv_buf = NULL, *recv_buf_free = NULL;
     char *result_buf = NULL, *result_buf_free = NULL;
@@ -150,14 +155,14 @@ ompi_coll_base_reduce_scatter_intra_basic_recursivehalving( const void *sbuf,
     OPAL_OUTPUT((ompi_coll_base_framework.framework_output,"coll:base:reduce_scatter_intra_basic_recursivehalving, rank %d", rank));
 
     /* Find displacements and the like */
-    disps = (int*) malloc(sizeof(int) * size);
+    disps = (ptrdiff_t*) malloc(sizeof(ptrdiff_t) * size);
     if (NULL == disps) return OMPI_ERR_OUT_OF_RESOURCE;
 
     disps[0] = 0;
     for (i = 0; i < (size - 1); ++i) {
-        disps[i + 1] = disps[i] + rcounts[i];
+        disps[i + 1] = disps[i] + ompi_count_array_get(rcounts, i);
     }
-    count = disps[size - 1] + rcounts[size - 1];
+    count = disps[size - 1] + ompi_count_array_get(rcounts, size - 1);
 
     /* short cut the trivial case */
     if (0 == count) {
@@ -230,18 +235,19 @@ ompi_coll_base_reduce_scatter_intra_basic_recursivehalving( const void *sbuf,
     /* For ranks not kicked out by the above code, perform the
        recursive halving */
     if (tmp_rank >= 0) {
-        int *tmp_disps = NULL, *tmp_rcounts = NULL;
+        size_t *tmp_rcounts = NULL;
+        ptrdiff_t *tmp_disps = NULL;
         int mask, send_index, recv_index, last_index;
 
         /* recalculate disps and rcounts to account for the
            special "remainder" processes that are no longer doing
            anything */
-        tmp_rcounts = (int*) malloc(tmp_size * sizeof(int));
+        tmp_rcounts = (size_t*) malloc(tmp_size * sizeof(size_t));
         if (NULL == tmp_rcounts) {
             err = OMPI_ERR_OUT_OF_RESOURCE;
             goto cleanup;
         }
-        tmp_disps = (int*) malloc(tmp_size * sizeof(int));
+        tmp_disps = (ptrdiff_t*) malloc(tmp_size * sizeof(ptrdiff_t));
         if (NULL == tmp_disps) {
             free(tmp_rcounts);
             err = OMPI_ERR_OUT_OF_RESOURCE;
@@ -251,9 +257,9 @@ ompi_coll_base_reduce_scatter_intra_basic_recursivehalving( const void *sbuf,
         for (i = 0 ; i < tmp_size ; ++i) {
             if (i < remain) {
                 /* need to include old neighbor as well */
-                tmp_rcounts[i] = rcounts[i * 2 + 1] + rcounts[i * 2];
+                tmp_rcounts[i] = ompi_count_array_get(rcounts, i * 2 + 1) + ompi_count_array_get(rcounts, i * 2);
             } else {
-                tmp_rcounts[i] = rcounts[i + remain];
+                tmp_rcounts[i] = ompi_count_array_get(rcounts, i + remain);
             }
         }
 
@@ -270,7 +276,8 @@ ompi_coll_base_reduce_scatter_intra_basic_recursivehalving( const void *sbuf,
         send_index = recv_index = 0;
         last_index = tmp_size;
         while (mask > 0) {
-            int tmp_peer, peer, send_count, recv_count;
+            int tmp_peer, peer;
+            size_t send_count, recv_count;
             struct ompi_request_t *request;
 
             tmp_peer = tmp_rank ^ mask;
@@ -299,7 +306,7 @@ ompi_coll_base_reduce_scatter_intra_basic_recursivehalving( const void *sbuf,
             /* actual data transfer.  Send from result_buf,
                receive into recv_buf */
             if (recv_count > 0) {
-                err = MCA_PML_CALL(irecv(recv_buf + (ptrdiff_t)tmp_disps[recv_index] * extent,
+                err = MCA_PML_CALL(irecv(recv_buf + tmp_disps[recv_index] * extent,
                                          recv_count, dtype, peer,
                                          MCA_COLL_BASE_TAG_REDUCE_SCATTER,
                                          comm, &request));
@@ -310,7 +317,7 @@ ompi_coll_base_reduce_scatter_intra_basic_recursivehalving( const void *sbuf,
                 }
             }
             if (send_count > 0) {
-                err = MCA_PML_CALL(send(result_buf + (ptrdiff_t)tmp_disps[send_index] * extent,
+                err = MCA_PML_CALL(send(result_buf + tmp_disps[send_index] * extent,
                                         send_count, dtype, peer,
                                         MCA_COLL_BASE_TAG_REDUCE_SCATTER,
                                         MCA_PML_BASE_SEND_STANDARD,
@@ -333,8 +340,8 @@ ompi_coll_base_reduce_scatter_intra_basic_recursivehalving( const void *sbuf,
                 }
 
                 ompi_op_reduce(op,
-                               recv_buf + (ptrdiff_t)tmp_disps[recv_index] * extent,
-                               result_buf + (ptrdiff_t)tmp_disps[recv_index] * extent,
+                               recv_buf + tmp_disps[recv_index] * extent,
+                               result_buf + tmp_disps[recv_index] * extent,
                                recv_count, dtype);
             }
 
@@ -345,10 +352,10 @@ ompi_coll_base_reduce_scatter_intra_basic_recursivehalving( const void *sbuf,
         }
 
         /* copy local results from results buffer into real receive buffer */
-        if (0 != rcounts[rank]) {
+        if (0 != ompi_count_array_get(rcounts, rank)) {
             err = ompi_datatype_sndrcv(result_buf + disps[rank] * extent,
-                                       rcounts[rank], dtype,
-                                       rbuf, rcounts[rank], dtype);
+                                       ompi_count_array_get(rcounts, rank), dtype,
+                                       rbuf, ompi_count_array_get(rcounts, rank), dtype);
             if (OMPI_SUCCESS != err) {
                 free(tmp_rcounts);
                 free(tmp_disps);
@@ -364,16 +371,16 @@ ompi_coll_base_reduce_scatter_intra_basic_recursivehalving( const void *sbuf,
        procs send the even procs the proper results */
     if (rank < (2 * remain)) {
         if ((rank & 1) == 0) {
-            if (rcounts[rank]) {
-                err = MCA_PML_CALL(recv(rbuf, rcounts[rank], dtype, rank + 1,
+            if (ompi_count_array_get(rcounts, rank)) {
+                err = MCA_PML_CALL(recv(rbuf, ompi_count_array_get(rcounts, rank), dtype, rank + 1,
                                         MCA_COLL_BASE_TAG_REDUCE_SCATTER,
                                         comm, MPI_STATUS_IGNORE));
                 if (OMPI_SUCCESS != err) goto cleanup;
             }
         } else {
-            if (rcounts[rank - 1]) {
+            if (ompi_count_array_get(rcounts, rank - 1)) {
                 err = MCA_PML_CALL(send(result_buf + disps[rank - 1] * extent,
-                                        rcounts[rank - 1], dtype, rank - 1,
+                                        ompi_count_array_get(rcounts, rank - 1), dtype, rank - 1,
                                         MCA_COLL_BASE_TAG_REDUCE_SCATTER,
                                         MCA_PML_BASE_SEND_STANDARD,
                                         comm));
@@ -453,14 +460,16 @@ ompi_coll_base_reduce_scatter_intra_basic_recursivehalving( const void *sbuf,
  *
  */
 int
-ompi_coll_base_reduce_scatter_intra_ring( const void *sbuf, void *rbuf, const int *rcounts,
+ompi_coll_base_reduce_scatter_intra_ring( const void *sbuf, void *rbuf, ompi_count_array_t rcounts,
                                           struct ompi_datatype_t *dtype,
                                           struct ompi_op_t *op,
                                           struct ompi_communicator_t *comm,
                                           mca_coll_base_module_t *module)
 {
-    int ret, line, rank, size, i, k, recv_from, send_to, total_count, max_block_count;
-    int inbi, *displs = NULL;
+    int ret, line, rank, size, i, k, recv_from, send_to;
+    int inbi;
+    size_t total_count, max_block_count;
+    ptrdiff_t *displs = NULL;
     char *tmpsend = NULL, *tmprecv = NULL, *accumbuf = NULL, *accumbuf_free = NULL;
     char *inbuf_free[2] = {NULL, NULL}, *inbuf[2] = {NULL, NULL};
     ptrdiff_t extent, max_real_segsize, dsize, gap = 0;
@@ -476,15 +485,15 @@ ompi_coll_base_reduce_scatter_intra_ring( const void *sbuf, void *rbuf, const in
     /* Determine the maximum number of elements per node,
        corresponding block size, and displacements array.
     */
-    displs = (int*) malloc(size * sizeof(int));
+    displs = (ptrdiff_t*) malloc(size * sizeof(ptrdiff_t));
     if (NULL == displs) { ret = -1; line = __LINE__; goto error_hndl; }
     displs[0] = 0;
-    total_count = rcounts[0];
-    max_block_count = rcounts[0];
+    total_count = ompi_count_array_get(rcounts, 0);
+    max_block_count = ompi_count_array_get(rcounts, 0);
     for (i = 1; i < size; i++) {
         displs[i] = total_count;
-        total_count += rcounts[i];
-        if (max_block_count < rcounts[i]) max_block_count = rcounts[i];
+        total_count += ompi_count_array_get(rcounts, i);
+        if (max_block_count < ompi_count_array_get(rcounts, i)) max_block_count = ompi_count_array_get(rcounts, i);
     }
 
     /* Special case for size == 1 */
@@ -545,7 +554,7 @@ ompi_coll_base_reduce_scatter_intra_ring( const void *sbuf, void *rbuf, const in
        - wait on block (r)
        - compute on block (r)
        - copy block (r) to rbuf
-       Note that we must be careful when computing the begining of buffers and
+       Note that we must be careful when computing the beginning of buffers and
        for send operations and computation we must compute the exact block size.
     */
     send_to = (rank + 1) % size;
@@ -557,8 +566,8 @@ ompi_coll_base_reduce_scatter_intra_ring( const void *sbuf, void *rbuf, const in
                              MCA_COLL_BASE_TAG_REDUCE_SCATTER, comm,
                              &reqs[inbi]));
     if (MPI_SUCCESS != ret) { line = __LINE__; goto error_hndl; }
-    tmpsend = accumbuf + (ptrdiff_t)displs[recv_from] * extent;
-    ret = MCA_PML_CALL(send(tmpsend, rcounts[recv_from], dtype, send_to,
+    tmpsend = accumbuf + displs[recv_from] * extent;
+    ret = MCA_PML_CALL(send(tmpsend, ompi_count_array_get(rcounts, recv_from), dtype, send_to,
                             MCA_COLL_BASE_TAG_REDUCE_SCATTER,
                             MCA_PML_BASE_SEND_STANDARD, comm));
     if (MPI_SUCCESS != ret) { line = __LINE__; goto error_hndl; }
@@ -581,11 +590,11 @@ ompi_coll_base_reduce_scatter_intra_ring( const void *sbuf, void *rbuf, const in
         /* Apply operation on previous block: result goes to rbuf
            rbuf[prevblock] = inbuf[inbi ^ 0x1] (op) rbuf[prevblock]
         */
-        tmprecv = accumbuf + (ptrdiff_t)displs[prevblock] * extent;
-        ompi_op_reduce(op, inbuf[inbi ^ 0x1], tmprecv, rcounts[prevblock], dtype);
+        tmprecv = accumbuf + displs[prevblock] * extent;
+        ompi_op_reduce(op, inbuf[inbi ^ 0x1], tmprecv, ompi_count_array_get(rcounts, prevblock), dtype);
 
         /* send previous block to send_to */
-        ret = MCA_PML_CALL(send(tmprecv, rcounts[prevblock], dtype, send_to,
+        ret = MCA_PML_CALL(send(tmprecv, ompi_count_array_get(rcounts, prevblock), dtype, send_to,
                                 MCA_COLL_BASE_TAG_REDUCE_SCATTER,
                                 MCA_PML_BASE_SEND_STANDARD, comm));
         if (MPI_SUCCESS != ret) { line = __LINE__; goto error_hndl; }
@@ -597,11 +606,12 @@ ompi_coll_base_reduce_scatter_intra_ring( const void *sbuf, void *rbuf, const in
 
     /* Apply operation on the last block (my block)
        rbuf[rank] = inbuf[inbi] (op) rbuf[rank] */
-    tmprecv = accumbuf + (ptrdiff_t)displs[rank] * extent;
-    ompi_op_reduce(op, inbuf[inbi], tmprecv, rcounts[rank], dtype);
+    tmprecv = accumbuf + displs[rank] * extent;
+    ompi_op_reduce(op, inbuf[inbi], tmprecv, ompi_count_array_get(rcounts, rank), dtype);
 
     /* Copy result from tmprecv to rbuf */
-    ret = ompi_datatype_copy_content_same_ddt(dtype, rcounts[rank], (char *)rbuf, tmprecv);
+    ret = ompi_datatype_copy_content_same_ddt(dtype, ompi_count_array_get(rcounts, rank),
+                                              (char *)rbuf, tmprecv);
     if (ret < 0) { line = __LINE__; goto error_hndl; }
 
     if (NULL != displs) free(displs);
@@ -626,12 +636,12 @@ ompi_coll_base_reduce_scatter_intra_ring( const void *sbuf, void *rbuf, const in
  * ompi_sum_counts: Returns sum of counts [lo, hi]
  *                  lo, hi in {0, 1, ..., nprocs_pof2 - 1}
  */
-static int ompi_sum_counts(const int *counts, int *displs, int nprocs_rem, int lo, int hi)
+static size_t ompi_sum_counts(ompi_count_array_t counts, ptrdiff_t *displs, int nprocs_rem, int lo, int hi)
 {
     /* Adjust lo and hi for taking into account blocks of excluded processes */
     lo = (lo < nprocs_rem) ? lo * 2 : lo + nprocs_rem;
     hi = (hi < nprocs_rem) ? hi * 2 + 1 : hi + nprocs_rem;
-    return displs[hi] + counts[hi] - displs[lo];
+    return displs[hi] + ompi_count_array_get(counts, hi) - displs[lo];
 }
 
 /*
@@ -689,12 +699,12 @@ static int ompi_sum_counts(const int *counts, int *displs, int nprocs_rem, int l
  */
 int
 ompi_coll_base_reduce_scatter_intra_butterfly(
-    const void *sbuf, void *rbuf, const int *rcounts, struct ompi_datatype_t *dtype,
+    const void *sbuf, void *rbuf, ompi_count_array_t rcounts, struct ompi_datatype_t *dtype,
     struct ompi_op_t *op, struct ompi_communicator_t *comm,
     mca_coll_base_module_t *module)
 {
     char *tmpbuf[2] = {NULL, NULL}, *psend, *precv;
-    int *displs = NULL, index;
+    ptrdiff_t *displs = NULL, index;
     ptrdiff_t span, gap, totalcount, extent;
     int err = MPI_SUCCESS;
     int comm_size = ompi_comm_size(comm);
@@ -713,9 +723,9 @@ ompi_coll_base_reduce_scatter_intra_butterfly(
     }
     displs[0] = 0;
     for (int i = 1; i < comm_size; i++) {
-        displs[i] = displs[i - 1] + rcounts[i - 1];
+        displs[i] = displs[i - 1] + ompi_count_array_get(rcounts, i - 1);
     }
-    totalcount = displs[comm_size - 1] + rcounts[comm_size - 1];
+    totalcount = displs[comm_size - 1] + ompi_count_array_get(rcounts, comm_size - 1);
 
     ompi_datatype_type_extent(dtype, &extent);
     span = opal_datatype_span(&dtype->super, totalcount, &gap);
@@ -807,35 +817,35 @@ ompi_coll_base_reduce_scatter_intra_butterfly(
             }
 
             /* Send blocks: [send_index, send_index + nblocks - 1] */
-            int send_count = ompi_sum_counts(rcounts, displs, nprocs_rem,
-                                             send_index, send_index + nblocks - 1);
+            size_t send_count = ompi_sum_counts(rcounts, displs, nprocs_rem,
+                                                send_index, send_index + nblocks - 1);
             index = (send_index < nprocs_rem) ? 2 * send_index : nprocs_rem + send_index;
             ptrdiff_t sdispl = displs[index];
 
             /* Recv blocks: [recv_index, recv_index + nblocks - 1] */
-            int recv_count = ompi_sum_counts(rcounts, displs, nprocs_rem,
-                                             recv_index, recv_index + nblocks - 1);
+            size_t recv_count = ompi_sum_counts(rcounts, displs, nprocs_rem,
+                                                recv_index, recv_index + nblocks - 1);
             index = (recv_index < nprocs_rem) ? 2 * recv_index : nprocs_rem + recv_index;
             ptrdiff_t rdispl = displs[index];
 
-            err = ompi_coll_base_sendrecv(psend + (ptrdiff_t)sdispl * extent, send_count,
+            err = ompi_coll_base_sendrecv(psend + sdispl * extent, send_count,
                                           dtype, peer, MCA_COLL_BASE_TAG_REDUCE_SCATTER,
-                                          precv + (ptrdiff_t)rdispl * extent, recv_count,
+                                          precv + rdispl * extent, recv_count,
                                           dtype, peer, MCA_COLL_BASE_TAG_REDUCE_SCATTER,
                                           comm, MPI_STATUS_IGNORE, rank);
             if (MPI_SUCCESS != err) { goto cleanup_and_return; }
 
             if (vrank < vpeer) {
                 /* precv = psend <op> precv */
-                ompi_op_reduce(op, psend + (ptrdiff_t)rdispl * extent,
-                               precv + (ptrdiff_t)rdispl * extent, recv_count, dtype);
+                ompi_op_reduce(op, psend + rdispl * extent,
+                               precv + rdispl * extent, recv_count, dtype);
                 char *p = psend;
                 psend = precv;
                 precv = p;
             } else {
                 /* psend = precv <op> psend */
-                ompi_op_reduce(op, precv + (ptrdiff_t)rdispl * extent,
-                               psend + (ptrdiff_t)rdispl * extent, recv_count, dtype);
+                ompi_op_reduce(op, precv + rdispl * extent,
+                               psend + rdispl * extent, recv_count, dtype);
             }
             send_index = recv_index;
         }
@@ -852,8 +862,8 @@ ompi_coll_base_reduce_scatter_intra_butterfly(
              * Process has two blocks: for excluded process and own.
              * Send the first block to excluded process.
              */
-            err = MCA_PML_CALL(send(psend + (ptrdiff_t)displs[index] * extent,
-                                    rcounts[index], dtype, peer - 1,
+            err = MCA_PML_CALL(send(psend + displs[index] * extent,
+                                    ompi_count_array_get(rcounts, index), dtype, peer - 1,
                                     MCA_COLL_BASE_TAG_REDUCE_SCATTER,
                                     MCA_PML_BASE_SEND_STANDARD, comm));
             if (MPI_SUCCESS != err) { goto cleanup_and_return; }
@@ -863,16 +873,16 @@ ompi_coll_base_reduce_scatter_intra_butterfly(
         if (vpeer < nprocs_rem)
             index++;
         if (vpeer != vrank) {
-            err = ompi_coll_base_sendrecv(psend + (ptrdiff_t)displs[index] * extent,
-                                          rcounts[index], dtype, peer,
+            err = ompi_coll_base_sendrecv(psend + displs[index] * extent,
+                                          ompi_count_array_get(rcounts, index), dtype, peer,
                                           MCA_COLL_BASE_TAG_REDUCE_SCATTER,
-                                          rbuf, rcounts[rank], dtype, peer,
+                                          rbuf, ompi_count_array_get(rcounts, rank), dtype, peer,
                                           MCA_COLL_BASE_TAG_REDUCE_SCATTER,
                                           comm, MPI_STATUS_IGNORE, rank);
             if (MPI_SUCCESS != err) { goto cleanup_and_return; }
         } else {
-            err = ompi_datatype_copy_content_same_ddt(dtype, rcounts[rank], rbuf,
-                                                      psend + (ptrdiff_t)displs[rank] * extent);
+            err = ompi_datatype_copy_content_same_ddt(dtype, ompi_count_array_get(rcounts, rank), rbuf,
+                                                      psend + displs[rank] * extent);
             if (MPI_SUCCESS != err) { goto cleanup_and_return; }
         }
 
@@ -880,11 +890,11 @@ ompi_coll_base_reduce_scatter_intra_butterfly(
         /* Excluded process: receive result */
         int vpeer = ompi_mirror_perm((rank + 1) / 2, log2_size);
         int peer = (vpeer < nprocs_rem) ? vpeer * 2 + 1 : vpeer + nprocs_rem;
-        err = MCA_PML_CALL(recv(rbuf, rcounts[rank], dtype, peer,
+        err = MCA_PML_CALL(recv(rbuf, ompi_count_array_get(rcounts, rank), dtype, peer,
                                 MCA_COLL_BASE_TAG_REDUCE_SCATTER, comm,
                                 MPI_STATUS_IGNORE));
         if (OMPI_SUCCESS != err) { goto cleanup_and_return; }
-    }
+     }
 
 cleanup_and_return:
     if (displs)
diff --git a/ompi/mca/coll/base/coll_base_reduce_scatter_block.c b/ompi/mca/coll/base/coll_base_reduce_scatter_block.c
index 6dd83daad4f..f72469d1a00 100644
--- a/ompi/mca/coll/base/coll_base_reduce_scatter_block.c
+++ b/ompi/mca/coll/base/coll_base_reduce_scatter_block.c
@@ -17,6 +17,8 @@
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2018      Siberian State University of Telecommunications
  *                         and Information Sciences. All rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -52,13 +54,14 @@
  *     up at some point)
  */
 int
-ompi_coll_base_reduce_scatter_block_basic_linear(const void *sbuf, void *rbuf, int rcount,
+ompi_coll_base_reduce_scatter_block_basic_linear(const void *sbuf, void *rbuf, size_t rcount,
                                                  struct ompi_datatype_t *dtype,
                                                  struct ompi_op_t *op,
                                                  struct ompi_communicator_t *comm,
                                                  mca_coll_base_module_t *module)
 {
-    int rank, size, count, err = OMPI_SUCCESS;
+    int rank, size, err = OMPI_SUCCESS;
+    size_t count;
     ptrdiff_t gap, span;
     char *recv_buf = NULL, *recv_buf_free = NULL;
 
@@ -67,40 +70,106 @@ ompi_coll_base_reduce_scatter_block_basic_linear(const void *sbuf, void *rbuf, i
     size = ompi_comm_size(comm);
 
     /* short cut the trivial case */
-    count = rcount * size;
+    count = rcount * (size_t)size;
     if (0 == count) {
         return OMPI_SUCCESS;
     }
 
-    /* get datatype information */
-    span = opal_datatype_span(&dtype->super, count, &gap);
-
     /* Handle MPI_IN_PLACE */
     if (MPI_IN_PLACE == sbuf) {
         sbuf = rbuf;
     }
 
-    if (0 == rank) {
-        /* temporary receive buffer.  See coll_basic_reduce.c for
-           details on sizing */
-        recv_buf_free = (char*) malloc(span);
-        if (NULL == recv_buf_free) {
-            err = OMPI_ERR_OUT_OF_RESOURCE;
-            goto cleanup;
+    /*
+     * For large payload (defined as a count greater than INT_MAX)
+     * to reduce the memory footprint on the root we segment the
+     * reductions per rank, then send to each rank.
+     *
+     * Additionally, sending the message in the coll_reduce() as
+     * "rcount*size" would exceed the 'int count' parameter in the
+     * coll_reduce() function. So another technique is required
+     * for count values that exceed INT_MAX.
+     */
+    if ( OPAL_UNLIKELY(count > INT_MAX) ) {
+        int i;
+        void *sbuf_ptr;
+
+        /* Get datatype information for an individual block */
+        span = opal_datatype_span(&dtype->super, rcount, &gap);
+
+        if (0 == rank) {
+            /* temporary receive buffer.  See coll_basic_reduce.c for
+               details on sizing */
+            recv_buf_free = (char*) malloc(span);
+            if (NULL == recv_buf_free) {
+                err = OMPI_ERR_OUT_OF_RESOURCE;
+                goto cleanup;
+            }
+            recv_buf = recv_buf_free - gap;
+        }
+
+        for( i = 0; i < size; ++i ) {
+            /* Calculate the portion of the send buffer to reduce over */
+            sbuf_ptr = (char*)sbuf     + span * (size_t)i;
+
+            /* Reduction for this peer */
+            err = comm->c_coll->coll_reduce(sbuf_ptr, recv_buf, rcount,
+                                            dtype, op, 0, comm,
+                                            comm->c_coll->coll_reduce_module);
+            if (MPI_SUCCESS != err) {
+                goto cleanup;
+            }
+
+            /* Send reduce results to this peer */
+            if (0 == rank ) {
+                if( i == rank ) {
+                    err = ompi_datatype_copy_content_same_ddt(dtype, rcount, rbuf, recv_buf);
+                } else {
+                    err = MCA_PML_CALL(send(recv_buf, rcount, dtype, i,
+                                            MCA_COLL_BASE_TAG_REDUCE_SCATTER_BLOCK,
+                                            MCA_PML_BASE_SEND_STANDARD, comm));
+                }
+                if (MPI_SUCCESS != err) {
+                    goto cleanup;
+                }
+            }
+            else if( i == rank ) {
+                err = MCA_PML_CALL(recv(rbuf, rcount, dtype, 0,
+                                        MCA_COLL_BASE_TAG_REDUCE_SCATTER_BLOCK,
+                                        comm, MPI_STATUS_IGNORE));
+                if (MPI_SUCCESS != err) {
+                    goto cleanup;
+                }
+            }
         }
-        recv_buf = recv_buf_free - gap;
     }
+    else {
+        /* get datatype information */
+        span = opal_datatype_span(&dtype->super, count, &gap);
+
+        if (0 == rank) {
+            /* temporary receive buffer.  See coll_basic_reduce.c for
+               details on sizing */
+            recv_buf_free = (char*) malloc(span);
+            if (NULL == recv_buf_free) {
+                err = OMPI_ERR_OUT_OF_RESOURCE;
+                goto cleanup;
+            }
+            recv_buf = recv_buf_free - gap;
+        }
 
-    /* reduction */
-    err =
-        comm->c_coll->coll_reduce(sbuf, recv_buf, count, dtype, op, 0,
-                                 comm, comm->c_coll->coll_reduce_module);
+        /* reduction */
+        err =
+            comm->c_coll->coll_reduce(sbuf, recv_buf, (int)count, dtype, op, 0,
+                                      comm, comm->c_coll->coll_reduce_module);
+        if (MPI_SUCCESS != err) {
+            goto cleanup;
+        }
 
-    /* scatter */
-    if (MPI_SUCCESS == err) {
+        /* scatter */
         err = comm->c_coll->coll_scatter(recv_buf, rcount, dtype,
-                                        rbuf, rcount, dtype, 0,
-                                        comm, comm->c_coll->coll_scatter_module);
+                                         rbuf, rcount, dtype, 0,
+                                         comm, comm->c_coll->coll_scatter_module);
     }
 
  cleanup:
@@ -126,7 +195,7 @@ ompi_coll_base_reduce_scatter_block_basic_linear(const void *sbuf, void *rbuf, i
  */
 int
 ompi_coll_base_reduce_scatter_block_intra_recursivedoubling(
-    const void *sbuf, void *rbuf, int rcount, struct ompi_datatype_t *dtype,
+    const void *sbuf, void *rbuf, size_t rcount, struct ompi_datatype_t *dtype,
     struct ompi_op_t *op, struct ompi_communicator_t *comm,
     mca_coll_base_module_t *module)
 {
@@ -146,7 +215,16 @@ ompi_coll_base_reduce_scatter_block_intra_recursivedoubling(
     if (comm_size < 2)
         return MPI_SUCCESS;
 
-    totalcount = comm_size * rcount;
+    totalcount = comm_size * (size_t)rcount;
+    if( OPAL_UNLIKELY(totalcount > INT_MAX) ) {
+        /*
+         * Large payload collectives are not supported by this algorithm.
+         * The blocklens and displs calculations in the loop below
+         * will overflow an int data type.
+         * Fallback to the linear algorithm.
+         */
+        return ompi_coll_base_reduce_scatter_block_basic_linear(sbuf, rbuf, rcount, dtype, op, comm, module);
+    }
     ompi_datatype_type_extent(dtype, &extent);
     span = opal_datatype_span(&dtype->super, totalcount, &gap);
     tmpbuf_raw = malloc(span);
@@ -168,7 +246,6 @@ ompi_coll_base_reduce_scatter_block_intra_recursivedoubling(
     int is_commutative = ompi_op_is_commute(op);
 
     /* Recursive distance doubling */
-    int rdoubling_step = 0;
     for (int mask = 1; mask < comm_size; mask <<= 1) {
         int remote = rank ^ mask;
         int cur_tree_root = ompi_rounddown(rank, mask);
@@ -273,7 +350,6 @@ ompi_coll_base_reduce_scatter_block_intra_recursivedoubling(
                 if (MPI_SUCCESS != err) { goto cleanup_and_return; }
             }
         }
-        rdoubling_step++;
         err = ompi_datatype_destroy(&dtypesend);
         if (MPI_SUCCESS != err) { goto cleanup_and_return; }
         err = ompi_datatype_destroy(&dtyperecv);
@@ -324,7 +400,7 @@ static int ompi_range_sum(int a, int b, int r)
  */
 int
 ompi_coll_base_reduce_scatter_block_intra_recursivehalving(
-    const void *sbuf, void *rbuf, int rcount, struct ompi_datatype_t *dtype,
+    const void *sbuf, void *rbuf, size_t rcount, struct ompi_datatype_t *dtype,
     struct ompi_op_t *op, struct ompi_communicator_t *comm,
     mca_coll_base_module_t *module)
 {
@@ -347,7 +423,8 @@ ompi_coll_base_reduce_scatter_block_intra_recursivehalving(
         return ompi_coll_base_reduce_scatter_block_basic_linear(sbuf, rbuf, rcount, dtype,
                                                                 op, comm, module);
     }
-    totalcount = comm_size * rcount;
+
+    totalcount = comm_size * (size_t)rcount;
     ompi_datatype_type_extent(dtype, &extent);
     span = opal_datatype_span(&dtype->super, totalcount, &gap);
     tmpbuf_raw = malloc(span);
@@ -431,22 +508,22 @@ ompi_coll_base_reduce_scatter_block_intra_recursivehalving(
              * have their result calculated by the process to their
              * right (rank + 1).
              */
-            int send_count = 0, recv_count = 0;
+            size_t send_count = 0, recv_count = 0;
             if (vrank < vpeer) {
                 /* Send the right half of the buffer, recv the left half */
                 send_index = recv_index + mask;
-                send_count = rcount * ompi_range_sum(send_index, last_index - 1, nprocs_rem - 1);
-                recv_count = rcount * ompi_range_sum(recv_index, send_index - 1, nprocs_rem - 1);
+                send_count = rcount * (size_t)ompi_range_sum(send_index, last_index - 1, nprocs_rem - 1);
+                recv_count = rcount * (size_t)ompi_range_sum(recv_index, send_index - 1, nprocs_rem - 1);
             } else {
                 /* Send the left half of the buffer, recv the right half */
                 recv_index = send_index + mask;
-                send_count = rcount * ompi_range_sum(send_index, recv_index - 1, nprocs_rem - 1);
-                recv_count = rcount * ompi_range_sum(recv_index, last_index - 1, nprocs_rem - 1);
+                send_count = rcount * (size_t)ompi_range_sum(send_index, recv_index - 1, nprocs_rem - 1);
+                recv_count = rcount * (size_t)ompi_range_sum(recv_index, last_index - 1, nprocs_rem - 1);
             }
-            ptrdiff_t rdispl = rcount * ((recv_index <= nprocs_rem - 1) ?
-                                         2 * recv_index : nprocs_rem + recv_index);
-            ptrdiff_t sdispl = rcount * ((send_index <= nprocs_rem - 1) ?
-                                         2 * send_index : nprocs_rem + send_index);
+            ptrdiff_t rdispl = rcount * (size_t)((recv_index <= nprocs_rem - 1) ?
+                                                 2 * recv_index : nprocs_rem + recv_index);
+            ptrdiff_t sdispl = rcount * (size_t)((send_index <= nprocs_rem - 1) ?
+                                                 2 * send_index : nprocs_rem + send_index);
             struct ompi_request_t *request = NULL;
 
             if (recv_count > 0) {
@@ -503,7 +580,7 @@ ompi_coll_base_reduce_scatter_block_intra_recursivehalving(
 }
 
 static int ompi_coll_base_reduce_scatter_block_intra_butterfly_pof2(
-    const void *sbuf, void *rbuf, int rcount, struct ompi_datatype_t *dtype,
+    const void *sbuf, void *rbuf, size_t rcount, struct ompi_datatype_t *dtype,
     struct ompi_op_t *op, struct ompi_communicator_t *comm,
     mca_coll_base_module_t *module);
 
@@ -565,7 +642,7 @@ static int ompi_coll_base_reduce_scatter_block_intra_butterfly_pof2(
  */
 int
 ompi_coll_base_reduce_scatter_block_intra_butterfly(
-    const void *sbuf, void *rbuf, int rcount, struct ompi_datatype_t *dtype,
+    const void *sbuf, void *rbuf, size_t rcount, struct ompi_datatype_t *dtype,
     struct ompi_op_t *op, struct ompi_communicator_t *comm,
     mca_coll_base_module_t *module)
 {
@@ -587,7 +664,7 @@ ompi_coll_base_reduce_scatter_block_intra_butterfly(
                    sbuf, rbuf, rcount, dtype, op, comm, module);
     }
 
-    totalcount = comm_size * rcount;
+    totalcount = comm_size * (size_t)rcount;
     ompi_datatype_type_extent(dtype, &extent);
     span = opal_datatype_span(&dtype->super, totalcount, &gap);
     tmpbuf[0] = malloc(span);
@@ -677,13 +754,17 @@ ompi_coll_base_reduce_scatter_block_intra_butterfly(
                 /* Send the upper half of reduction buffer, recv the lower half */
                 recv_index += nblocks;
             }
-            int send_count = rcount * ompi_range_sum(send_index,
-                                          send_index + nblocks - 1, nprocs_rem - 1);
-            int recv_count = rcount * ompi_range_sum(recv_index,
-                                          recv_index + nblocks - 1, nprocs_rem - 1);
-            ptrdiff_t sdispl = rcount * ((send_index <= nprocs_rem - 1) ?
+            size_t send_count = rcount *
+                                (size_t)ompi_range_sum(send_index,
+                                                       send_index + nblocks - 1,
+                                                       nprocs_rem - 1);
+            size_t recv_count = rcount *
+                                (size_t)ompi_range_sum(recv_index,
+                                                       recv_index + nblocks - 1,
+                                                       nprocs_rem - 1);
+            ptrdiff_t sdispl = rcount * (size_t)((send_index <= nprocs_rem - 1) ?
                                          2 * send_index : nprocs_rem + send_index);
-            ptrdiff_t rdispl = rcount * ((recv_index <= nprocs_rem - 1) ?
+            ptrdiff_t rdispl = rcount * (size_t)((recv_index <= nprocs_rem - 1) ?
                                          2 * recv_index : nprocs_rem + recv_index);
 
             err = ompi_coll_base_sendrecv(psend + (ptrdiff_t)sdispl * extent, send_count,
@@ -719,7 +800,7 @@ ompi_coll_base_reduce_scatter_block_intra_butterfly(
              * Process has two blocks: for excluded process and own.
              * Send result to the excluded process.
              */
-            ptrdiff_t sdispl = rcount * ((send_index <= nprocs_rem - 1) ?
+            ptrdiff_t sdispl = rcount * (size_t)((send_index <= nprocs_rem - 1) ?
                                          2 * send_index : nprocs_rem + send_index);
             err = MCA_PML_CALL(send(psend + (ptrdiff_t)sdispl * extent,
                                     rcount, dtype, peer - 1,
@@ -729,7 +810,7 @@ ompi_coll_base_reduce_scatter_block_intra_butterfly(
         }
 
         /* Send result to a remote process according to a mirror permutation */
-        ptrdiff_t sdispl = rcount * ((send_index <= nprocs_rem - 1) ?
+        ptrdiff_t sdispl = rcount * (size_t)((send_index <= nprocs_rem - 1) ?
                                       2 * send_index : nprocs_rem + send_index);
         /* If process has two blocks, then send the second block (own block) */
         if (vpeer < nprocs_rem)
@@ -808,7 +889,7 @@ ompi_coll_base_reduce_scatter_block_intra_butterfly(
  */
 static int
 ompi_coll_base_reduce_scatter_block_intra_butterfly_pof2(
-    const void *sbuf, void *rbuf, int rcount, struct ompi_datatype_t *dtype,
+    const void *sbuf, void *rbuf, size_t rcount, struct ompi_datatype_t *dtype,
     struct ompi_op_t *op, struct ompi_communicator_t *comm,
     mca_coll_base_module_t *module)
 {
@@ -821,7 +902,7 @@ ompi_coll_base_reduce_scatter_block_intra_butterfly_pof2(
     if (rcount == 0 || comm_size < 2)
         return MPI_SUCCESS;
 
-    totalcount = comm_size * rcount;
+    totalcount = comm_size * (size_t)rcount;
     ompi_datatype_type_extent(dtype, &extent);
     span = opal_datatype_span(&dtype->super, totalcount, &gap);
     tmpbuf[0] = malloc(span);
@@ -843,7 +924,7 @@ ompi_coll_base_reduce_scatter_block_intra_butterfly_pof2(
         if (MPI_SUCCESS != err) { goto cleanup_and_return; }
     }
 
-    int nblocks = totalcount, send_index = 0, recv_index = 0;
+    size_t nblocks = totalcount, send_index = 0, recv_index = 0;
     for (int mask = 1; mask < comm_size; mask <<= 1) {
         int peer = rank ^ mask;
         nblocks /= 2;
diff --git a/ompi/mca/coll/base/coll_base_scan.c b/ompi/mca/coll/base/coll_base_scan.c
index a82e837965f..9ac99ed255e 100644
--- a/ompi/mca/coll/base/coll_base_scan.c
+++ b/ompi/mca/coll/base/coll_base_scan.c
@@ -32,7 +32,7 @@
  * Returns:   MPI_SUCCESS or error code
  */
 int
-ompi_coll_base_scan_intra_linear(const void *sbuf, void *rbuf, int count,
+ompi_coll_base_scan_intra_linear(const void *sbuf, void *rbuf, size_t count,
                                 struct ompi_datatype_t *dtype,
                                 struct ompi_op_t *op,
                                 struct ompi_communicator_t *comm,
@@ -155,7 +155,7 @@ ompi_coll_base_scan_intra_linear(const void *sbuf, void *rbuf, int count,
  * Limitations: intra-communicators only
  */
 int ompi_coll_base_scan_intra_recursivedoubling(
-    const void *sendbuf, void *recvbuf, int count, struct ompi_datatype_t *datatype,
+    const void *sendbuf, void *recvbuf, size_t count, struct ompi_datatype_t *datatype,
     struct ompi_op_t *op, struct ompi_communicator_t *comm,
     mca_coll_base_module_t *module)
 {
diff --git a/ompi/mca/coll/base/coll_base_scatter.c b/ompi/mca/coll/base/coll_base_scatter.c
index 0ca35971532..795f79a5c72 100644
--- a/ompi/mca/coll/base/coll_base_scatter.c
+++ b/ompi/mca/coll/base/coll_base_scatter.c
@@ -61,17 +61,20 @@
  */
 int
 ompi_coll_base_scatter_intra_binomial(
-    const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-    void *rbuf, int rcount, struct ompi_datatype_t *rdtype,
+    const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+    void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
     int root, struct ompi_communicator_t *comm,
     mca_coll_base_module_t *module)
 {
-    int line = -1, rank, vrank, size, err;
-    char *ptmp, *tempbuf = NULL;
-    MPI_Status status;
     mca_coll_base_module_t *base_module = (mca_coll_base_module_t*)module;
     mca_coll_base_comm_t *data = base_module->base_data;
-    ptrdiff_t sextent, rextent, ssize, rsize, sgap = 0, rgap = 0;
+    int line = -1, rank, vrank, size, err, packed_size;
+    size_t curr_count;
+    char *ptmp, *tempbuf = NULL;
+    size_t max_data, packed_sizet;
+    opal_convertor_t convertor;
+    ptrdiff_t sextent;
+    MPI_Status status;
 
     size = ompi_comm_size(comm);
     rank = ompi_comm_rank(comm);
@@ -89,99 +92,101 @@ ompi_coll_base_scatter_intra_binomial(
     vrank = (rank - root + size) % size;
     ptmp = (char *)rbuf;  /* by default suppose leaf nodes, just use rbuf */
 
-    if (rank == root) {
+    if ( vrank % 2 ) {  /* leaves */
+        /* recv from parent on leaf nodes */
+        err = MCA_PML_CALL(recv(rbuf, rcount, rdtype, bmtree->tree_prev,
+                                MCA_COLL_BASE_TAG_SCATTER, comm, &status));
+        if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
+        return MPI_SUCCESS;
+
+    }
+    OBJ_CONSTRUCT( &convertor, opal_convertor_t );
+    if (rank == root) {  /* root and non-leafs */
         ompi_datatype_type_extent(sdtype, &sextent);
-        ssize = opal_datatype_span(&sdtype->super, (int64_t)scount * size, &sgap);
-        if (0 == root) {
-            /* root on 0, just use the send buffer */
-            ptmp = (char *)sbuf;
-            if (rbuf != MPI_IN_PLACE) {
-                /* local copy to rbuf */
-                err = ompi_datatype_sndrcv(sbuf, scount, sdtype,
-                                           rbuf, rcount, rdtype);
-                if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
-            }
-        } else {
-            /* root is not on 0, allocate temp buffer for send */
-            tempbuf = (char *)malloc(ssize);
+        ptmp = (char *)sbuf;  /* if root == 0, just use the send buffer */
+        if (0 != root) {
+            opal_convertor_copy_and_prepare_for_send( ompi_mpi_local_convertor, &(sdtype->super),
+                                                      scount * size, sbuf, 0, &convertor );
+            opal_convertor_get_packed_size( &convertor, &packed_sizet );
+            packed_size = packed_sizet;
+            packed_sizet = packed_sizet / size;
+            ptmp = tempbuf = (char *)malloc(packed_size);
             if (NULL == tempbuf) {
                 err = OMPI_ERR_OUT_OF_RESOURCE; line = __LINE__; goto err_hndl;
             }
-            ptmp = tempbuf - sgap;
-
-            /* and rotate data so they will eventually in the right place */
-            err = ompi_datatype_copy_content_same_ddt(sdtype, (ptrdiff_t)scount * (ptrdiff_t)(size - root),
-                                                      ptmp, (char *) sbuf + sextent * (ptrdiff_t)root * (ptrdiff_t)scount);
-            if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
-
-            err = ompi_datatype_copy_content_same_ddt(sdtype, (ptrdiff_t)scount * (ptrdiff_t)root,
-                                                      ptmp + sextent * (ptrdiff_t)scount * (ptrdiff_t)(size - root), (char *)sbuf);
-            if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
-
-            if (rbuf != MPI_IN_PLACE) {
-                /* local copy to rbuf */
-                err = ompi_datatype_sndrcv(ptmp, scount, sdtype,
-                                           rbuf, rcount, rdtype);
-                if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
-            }
+            /* rotate data so they will eventually be in the right place */
+            struct iovec iov[1];
+            uint32_t iov_size = 1;
+
+            iov[0].iov_base = ptmp + (ptrdiff_t)(size - root) * packed_sizet;
+            iov[0].iov_len = max_data = packed_sizet * (ptrdiff_t)root;
+            opal_convertor_pack(&convertor, iov, &iov_size, &max_data);
+            
+            iov[0].iov_base = ptmp;
+            iov[0].iov_len = max_data = packed_sizet * (ptrdiff_t)(size - root);
+            opal_convertor_pack(&convertor, iov, &iov_size, &max_data);
+            OBJ_DESTRUCT(&convertor);
+
+            sdtype = MPI_PACKED;
+            sextent = 1;  /* bytes */
+            scount = packed_size / size;
         }
-    } else if (!(vrank % 2)) {
-        /* non-root, non-leaf nodes, allocate temp buffer for recv
-         * the most we need is rcount*size/2 */
-        ompi_datatype_type_extent(rdtype, &rextent);
-        rsize = opal_datatype_span(&rdtype->super, (int64_t)rcount * size, &rgap);
-        tempbuf = (char *)malloc(rsize / 2);
+        curr_count = scount * size;
+    } else {  /* (!(vrank % 2)) */
+        opal_convertor_copy_and_prepare_for_send( ompi_mpi_local_convertor, &(rdtype->super),
+                                                  rcount, NULL, 0, &convertor );
+        opal_convertor_get_packed_size( &convertor, &packed_sizet );
+        scount = packed_sizet;
+
+        sdtype = MPI_PACKED;  /* default to MPI_PACKED as the send type */
+
+        /* non-root, non-leaf nodes, allocate temp buffer for recv the most we need is rcount*size/2 (an upper bound) */
+        int vparent = (bmtree->tree_prev - root + size) % size;
+        int subtree_size = vrank - vparent;
+        if (size - vrank < subtree_size)
+            subtree_size = size - vrank;
+        packed_size = scount * subtree_size;
+
+        ptmp = tempbuf = (char *)malloc(packed_size);
         if (NULL == tempbuf) {
             err = OMPI_ERR_OUT_OF_RESOURCE; line = __LINE__; goto err_hndl;
         }
-        ptmp = tempbuf - rgap;
-        sdtype = rdtype;
-        scount = rcount;
-        sextent = rextent;
-    }
 
-    int curr_count = (rank == root) ? scount * size : 0;
-    if (!(vrank % 2)) {
-        if (rank != root) {
-            /* recv from parent on non-root */
-            err = MCA_PML_CALL(recv(ptmp, (ptrdiff_t)rcount * (ptrdiff_t)size, rdtype, bmtree->tree_prev,
-                                    MCA_COLL_BASE_TAG_SCATTER, comm, &status));
-            if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
-
-            /* Get received count */
-            size_t rdtype_size;
-            ompi_datatype_type_size(rdtype, &rdtype_size);
-            curr_count = (int)(status._ucount / rdtype_size);
-
-            /* local copy to rbuf */
-            err = ompi_datatype_sndrcv(ptmp, scount, sdtype,
-                                       rbuf, rcount, rdtype);
-            if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
-        }
-        /* send to children on all non-leaf */
-        for (int i = bmtree->tree_nextsize - 1; i >= 0; i--) {
-            /* figure out how much data I have to send to this child */
-            int vchild = (bmtree->tree_next[i] - root + size) % size;
-            int send_count = vchild - vrank;
-            if (send_count > size - vchild)
-                send_count = size - vchild;
-            send_count *= scount;
-            err = MCA_PML_CALL(send(ptmp + (ptrdiff_t)(curr_count - send_count) * sextent,
-                                    send_count, sdtype, bmtree->tree_next[i],
-                                    MCA_COLL_BASE_TAG_SCATTER,
-                                    MCA_PML_BASE_SEND_STANDARD, comm));
-            if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
-            curr_count -= send_count;
-        }
-        if (NULL != tempbuf)
-            free(tempbuf);
-    } else {
-        /* recv from parent on leaf nodes */
-        err = MCA_PML_CALL(recv(ptmp, rcount, rdtype, bmtree->tree_prev,
+        /* recv from parent on non-root */
+        err = MCA_PML_CALL(recv(ptmp, (ptrdiff_t)packed_size, MPI_PACKED, bmtree->tree_prev,
                                 MCA_COLL_BASE_TAG_SCATTER, comm, &status));
         if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
+
+        /* Get received count */
+        curr_count = status._ucount;  /* no need for conversion, work in bytes */
+        sextent = 1;  /* bytes */
     }
 
+    if (rbuf != MPI_IN_PLACE) {  /* local copy to rbuf */
+        err = ompi_datatype_sndrcv(ptmp, scount, sdtype,
+                                   rbuf, rcount, rdtype);
+        if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
+    }
+
+    /* send to children on all non-leaf */
+    for (int i = bmtree->tree_nextsize - 1; i >= 0; i--) {
+        /* figure out how much data I have to send to this child */
+        int vchild = (bmtree->tree_next[i] - root + size) % size;
+        int send_count = vchild - vrank;
+        if (send_count > size - vchild)
+            send_count = size - vchild;
+        send_count *= scount;
+
+        err = MCA_PML_CALL(send(ptmp + (ptrdiff_t)(curr_count - send_count) * sextent,
+                                send_count, sdtype, bmtree->tree_next[i],
+                                MCA_COLL_BASE_TAG_SCATTER,
+                                MCA_PML_BASE_SEND_STANDARD, comm));
+        if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
+        curr_count -= send_count;
+    }
+    if (NULL != tempbuf)
+        free(tempbuf);
+
     return MPI_SUCCESS;
 
  err_hndl:
@@ -215,9 +220,9 @@ ompi_coll_base_scatter_intra_binomial(
  *	Returns:	- MPI_SUCCESS or error code
  */
 int
-ompi_coll_base_scatter_intra_basic_linear(const void *sbuf, int scount,
+ompi_coll_base_scatter_intra_basic_linear(const void *sbuf, size_t scount,
                                           struct ompi_datatype_t *sdtype,
-                                          void *rbuf, int rcount,
+                                          void *rbuf, size_t rcount,
                                           struct ompi_datatype_t *rdtype,
                                           int root,
                                           struct ompi_communicator_t *comm,
@@ -282,9 +287,9 @@ ompi_coll_base_scatter_intra_basic_linear(const void *sbuf, int scount,
  * progression until the message is sent/(copied to some sort of transmit buffer).
  */
 int
-ompi_coll_base_scatter_intra_linear_nb(const void *sbuf, int scount,
+ompi_coll_base_scatter_intra_linear_nb(const void *sbuf, size_t scount,
                                        struct ompi_datatype_t *sdtype,
-                                       void *rbuf, int rcount,
+                                       void *rbuf, size_t rcount,
                                        struct ompi_datatype_t *rdtype,
                                        int root,
                                        struct ompi_communicator_t *comm,
@@ -373,8 +378,10 @@ ompi_coll_base_scatter_intra_linear_nb(const void *sbuf, int scount,
             for (i = 0; i < nreqs; i++) {
                 if (MPI_REQUEST_NULL == reqs[i]) continue;
                 if (MPI_ERR_PENDING == reqs[i]->req_status.MPI_ERROR) continue;
-                err = reqs[i]->req_status.MPI_ERROR;
-                break;
+                if (reqs[i]->req_status.MPI_ERROR != MPI_SUCCESS) {
+                    err = reqs[i]->req_status.MPI_ERROR;
+                    break;
+                }
             }
         }
         ompi_coll_base_free_reqs(reqs, nreqs);
diff --git a/ompi/mca/coll/base/coll_base_topo.c b/ompi/mca/coll/base/coll_base_topo.c
index 9fcc22fd09f..d3a1f11eab5 100644
--- a/ompi/mca/coll/base/coll_base_topo.c
+++ b/ompi/mca/coll/base/coll_base_topo.c
@@ -59,7 +59,12 @@ static int calculate_num_nodes_up_to_level( int fanout, int level )
 {
     /* just use geometric progression formula for sum:
        a^0+a^1+...a^(n-1) = (a^n-1)/(a-1) */
-    return ((pown(fanout,level) - 1)/(fanout - 1));
+    if (1 == fanout) {
+        return level;
+    }
+    else {
+        return ((pown(fanout,level) - 1)/(fanout - 1));
+    }
 }
 
 /*
diff --git a/ompi/mca/coll/base/coll_base_topo.h b/ompi/mca/coll/base/coll_base_topo.h
index 35ed745a26a..c815554f22b 100644
--- a/ompi/mca/coll/base/coll_base_topo.h
+++ b/ompi/mca/coll/base/coll_base_topo.h
@@ -20,6 +20,7 @@
 #define MCA_COLL_BASE_TOPO_H_HAS_BEEN_INCLUDED
 
 #include "ompi_config.h"
+#include "ompi/communicator/communicator.h"
 #include <stddef.h>
 
 #define MAXTREEFANOUT 32
diff --git a/ompi/mca/coll/base/coll_base_util.c b/ompi/mca/coll/base/coll_base_util.c
index a3e25b27b77..ba74aa01350 100644
--- a/ompi/mca/coll/base/coll_base_util.c
+++ b/ompi/mca/coll/base/coll_base_util.c
@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
+ * Copyright (c) 2004-2022 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -11,6 +11,10 @@
  *                         All rights reserved.
  * Copyright (c) 2014-2020 Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
+ *
+ * Copyright (c) 2024      NVIDIA CORPORATION. All rights reserved.
+ *
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -94,7 +98,7 @@ int ompi_coll_base_sendrecv_actual( const void* sendbuf, size_t scount,
             if( MPI_ERR_PROC_FAILED_PENDING == err ) {
                 err = MPI_ERR_PROC_FAILED;
             }
-        } else /* this 'else' intentionaly spills outside the ifdef */
+        } else /* this 'else' intentionally spills outside the ifdef */
 #endif /* OPAL_ENABLE_FT_MPI */
         ompi_request_free(&req);
     }
@@ -127,14 +131,29 @@ int ompi_rounddown(int num, int factor)
     return num * factor;    /* floor(num / factor) * factor */
 }
 
-static void release_objs_callback(struct ompi_coll_base_nbc_request_t *request) {
-    if (NULL != request->data.objs.objs[0]) {
-        OBJ_RELEASE(request->data.objs.objs[0]);
-        request->data.objs.objs[0] = NULL;
+/**
+ * Release all objects and arrays stored into the nbc_request.
+ * The release_arrays are temporary memory to stored the values
+ * converted from Fortran, and should disappear in same time as the
+ * request itself.
+ */
+static void
+release_objs_callback(struct ompi_coll_base_nbc_request_t *request)
+{
+    if (NULL != request->data.refcounted.objs.objs[0]) {
+        OBJ_RELEASE(request->data.refcounted.objs.objs[0]);
+        request->data.refcounted.objs.objs[0] = NULL;
     }
-    if (NULL != request->data.objs.objs[1]) {
-        OBJ_RELEASE(request->data.objs.objs[1]);
-        request->data.objs.objs[1] = NULL;
+    if (NULL != request->data.refcounted.objs.objs[1]) {
+        OBJ_RELEASE(request->data.refcounted.objs.objs[1]);
+        request->data.refcounted.objs.objs[1] = NULL;
+    }
+    for(int i = 0; i < OMPI_REQ_NB_RELEASE_ARRAYS; i++ ) {
+        if (NULL == request->data.release_arrays[i]) {
+            break;
+        }
+        free(request->data.release_arrays[i]);
+        request->data.release_arrays[i] = NULL;
     }
 }
 
@@ -168,12 +187,12 @@ int ompi_coll_base_retain_op( ompi_request_t *req, ompi_op_t *op,
     }
     if (!ompi_op_is_intrinsic(op)) {
         OBJ_RETAIN(op);
-        request->data.op.op = op;
+        request->data.refcounted.op.op = op;
         retain = true;
     }
     if (!ompi_datatype_is_predefined(type)) {
         OBJ_RETAIN(type);
-        request->data.op.datatype = type;
+        request->data.refcounted.op.datatype = type;
         retain = true;
     }
     if (OPAL_UNLIKELY(retain)) {
@@ -207,12 +226,12 @@ int ompi_coll_base_retain_datatypes( ompi_request_t *req, ompi_datatype_t *stype
     }
     if (NULL != stype && !ompi_datatype_is_predefined(stype)) {
         OBJ_RETAIN(stype);
-        request->data.types.stype = stype;
+        request->data.refcounted.types.stype = stype;
         retain = true;
     }
     if (NULL != rtype && !ompi_datatype_is_predefined(rtype)) {
         OBJ_RETAIN(rtype);
-        request->data.types.rtype = rtype;
+        request->data.refcounted.types.rtype = rtype;
         retain = true;
     }
     if (OPAL_UNLIKELY(retain)) {
@@ -229,29 +248,25 @@ int ompi_coll_base_retain_datatypes( ompi_request_t *req, ompi_datatype_t *stype
     return OMPI_SUCCESS;
 }
 
-static void release_vecs_callback(ompi_coll_base_nbc_request_t *request) {
-    ompi_communicator_t *comm = request->super.req_mpi_object.comm;
-    int scount, rcount;
-    if (OMPI_COMM_IS_TOPO(comm)) {
-        (void)mca_topo_base_neighbor_count (comm, &rcount, &scount);
-    } else {
-        scount = rcount = OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm);
-    }
-    if (NULL != request->data.vecs.stypes) {
-        for (int i=0; i<scount; i++) {
-            if (NULL != request->data.vecs.stypes[i]) {
-                OMPI_DATATYPE_RELEASE_NO_NULLIFY(request->data.vecs.stypes[i]);
+static void release_vecs_callback(ompi_coll_base_nbc_request_t *request)
+{
+    if (NULL != request->data.refcounted.vecs.stypes) {
+        for (int i = 0; i < request->data.refcounted.vecs.scount; i++) {
+            if (NULL != request->data.refcounted.vecs.stypes[i] &&
+                !ompi_datatype_is_predefined(request->data.refcounted.vecs.stypes[i])) {
+                OMPI_DATATYPE_RELEASE_NO_NULLIFY(request->data.refcounted.vecs.stypes[i]);
             }
         }
-        request->data.vecs.stypes = NULL;
+        request->data.refcounted.vecs.stypes = NULL;
     }
-    if (NULL != request->data.vecs.rtypes) {
-        for (int i=0; i<rcount; i++) {
-            if (NULL != request->data.vecs.rtypes[i]) {
-                OMPI_DATATYPE_RELEASE_NO_NULLIFY(request->data.vecs.rtypes[i]);
+    if (NULL != request->data.refcounted.vecs.rtypes) {
+        for (int i = 0; i < request->data.refcounted.vecs.rcount; i++) {
+            if (NULL != request->data.refcounted.vecs.rtypes[i] &&
+                !ompi_datatype_is_predefined(request->data.refcounted.vecs.rtypes[i])) {
+                OMPI_DATATYPE_RELEASE_NO_NULLIFY(request->data.refcounted.vecs.rtypes[i]);
             }
         }
-        request->data.vecs.rtypes = NULL;
+        request->data.refcounted.vecs.rtypes = NULL;
     }
 }
 
@@ -277,35 +292,47 @@ static int free_vecs_callback(struct ompi_request_t **rptr) {
 }
 
 int ompi_coll_base_retain_datatypes_w( ompi_request_t *req,
-                                       ompi_datatype_t * const stypes[], ompi_datatype_t * const rtypes[]) {
+                                       ompi_datatype_t * const stypes[],
+                                       ompi_datatype_t * const rtypes[],
+                                       bool use_topo)
+{
     ompi_coll_base_nbc_request_t *request = (ompi_coll_base_nbc_request_t *)req;
-    bool retain = false;
     ompi_communicator_t *comm = request->super.req_mpi_object.comm;
     int scount, rcount;
+
     if (REQUEST_COMPLETE(req)) {
         return OMPI_SUCCESS;
     }
-    if (OMPI_COMM_IS_TOPO(comm)) {
+
+    if (use_topo && OMPI_COMM_IS_TOPO(comm)) {
         (void)mca_topo_base_neighbor_count (comm, &rcount, &scount);
     } else {
         scount = rcount = OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm);
     }
 
-    for (int i=0; i<scount; i++) {
-        if (NULL != stypes && NULL != stypes[i] && !ompi_datatype_is_predefined(stypes[i])) {
-            OBJ_RETAIN(stypes[i]);
-            retain = true;
+    request->data.refcounted.vecs.scount = 0;  /* default value */
+    if (NULL != stypes) {
+        for (int i = 0; i < scount; i++) {
+            if (NULL != stypes[i] && !ompi_datatype_is_predefined(stypes[i])) {
+                OBJ_RETAIN(stypes[i]);
+                request->data.refcounted.vecs.scount = i;  /* last valid type */
+            }
         }
     }
-    for (int i=0; i<rcount; i++) {
-        if (NULL != rtypes && NULL != rtypes[i] && !ompi_datatype_is_predefined(rtypes[i])) {
-            OBJ_RETAIN(rtypes[i]);
-            retain = true;
+    request->data.refcounted.vecs.rcount = 0;  /* default value */
+    if (NULL != rtypes) {
+        for (int i = 0; i < rcount; i++) {
+            if (NULL != rtypes[i] && !ompi_datatype_is_predefined(rtypes[i])) {
+                OBJ_RETAIN(rtypes[i]);
+                request->data.refcounted.vecs.rcount = i;  /* last valid type */
+            }
         }
     }
-    if (OPAL_UNLIKELY(retain)) {
-        request->data.vecs.stypes = (ompi_datatype_t **) stypes;
-        request->data.vecs.rtypes = (ompi_datatype_t **) rtypes;
+    if (OPAL_LIKELY(request->data.refcounted.vecs.scount | request->data.refcounted.vecs.rcount) ) {
+        request->data.refcounted.vecs.stypes = (ompi_datatype_t **) stypes;
+        request->data.refcounted.vecs.rtypes = (ompi_datatype_t **) rtypes;
+        request->data.refcounted.vecs.scount = scount;
+        request->data.refcounted.vecs.rcount = rcount;
         if (req->req_persistent) {
             request->cb.req_free = req->req_free;
             req->req_free = free_vecs_callback;
@@ -319,15 +346,16 @@ int ompi_coll_base_retain_datatypes_w( ompi_request_t *req,
     return OMPI_SUCCESS;
 }
 
-static void nbc_req_cons(ompi_coll_base_nbc_request_t *req)
+static void nbc_req_constructor(ompi_coll_base_nbc_request_t *req)
 {
     req->cb.req_complete_cb = NULL;
     req->req_complete_cb_data = NULL;
-    req->data.objs.objs[0] = NULL;
-    req->data.objs.objs[1] = NULL;
+    req->data.refcounted.objs.objs[0] = NULL;
+    req->data.refcounted.objs.objs[1] = NULL;
+    req->data.release_arrays[0] = NULL;
 }
 
-OBJ_CLASS_INSTANCE(ompi_coll_base_nbc_request_t, ompi_request_t, nbc_req_cons, NULL);
+OBJ_CLASS_INSTANCE(ompi_coll_base_nbc_request_t, ompi_request_t, nbc_req_constructor, NULL);
 
 /* File reading functions */
 static void skiptonewline (FILE *fptr, int *fileline)
@@ -374,7 +402,7 @@ int ompi_coll_base_file_getnext_long(FILE *fptr, int *fileline, long* val)
 
 int ompi_coll_base_file_getnext_string(FILE *fptr, int *fileline, char** val)
 {
-    char trash, token[32];
+    char trash, token[33];
     int rc;
 
     *val = NULL;  /* security in case we fail */
@@ -457,6 +485,26 @@ int ompi_coll_base_file_peek_next_char_is(FILE *fptr, int *fileline, int expecte
     } while (1);
 }
 
+/**
+ * return non-zero if the next non-space to read on the current line is a digit.
+ * otherwise return 0.
+ */
+int ompi_coll_base_file_peek_next_char_isdigit(FILE *fptr)
+{
+    do {
+        int next = fgetc(fptr);
+
+        if ((' ' == next) || ('\t' == next)) {
+            continue; /* discard space and tab. keep everything else */
+        }
+
+        ungetc(next, fptr); /* put the char back into the stream */
+
+        return isdigit(next); /* report back whether or not next is a digit */
+
+    } while (1);
+}
+
 /**
  * There are certainly simpler implementation for this function when performance
  * is not a critical point. But, as this function is used during the collective
diff --git a/ompi/mca/coll/base/coll_base_util.h b/ompi/mca/coll/base/coll_base_util.h
index 46c95153469..7bceaa7dcc0 100644
--- a/ompi/mca/coll/base/coll_base_util.h
+++ b/ompi/mca/coll/base/coll_base_util.h
@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
+ * Copyright (c) 2004-2022 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2007 High Performance Computing Center Stuttgart,
@@ -11,6 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2014-2020 Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      NVIDIA CORPORATION. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -34,6 +35,13 @@
 
 BEGIN_C_DECLS
 
+/**
+ * The largest array we need to track collective temporary memory. Right now
+ * the record is for ialltoallw, for the array of send and receive types,
+ * count and displacements.
+ */
+#define OMPI_REQ_NB_RELEASE_ARRAYS 7
+
 /**
  * Request structure to be returned by non-blocking
  * collective operations.
@@ -45,22 +53,27 @@ struct ompi_coll_base_nbc_request_t {
         ompi_request_free_fn_t req_free;
     } cb;
     void *req_complete_cb_data;
-    union {
-        struct {
-            ompi_op_t *op;
-            ompi_datatype_t *datatype;
-        } op;
-        struct {
-            ompi_datatype_t *stype;
-            ompi_datatype_t *rtype;
-        } types;
-        struct {
-            opal_object_t *objs[2];
-        } objs;
-        struct {
-            ompi_datatype_t * const *stypes;
-            ompi_datatype_t * const *rtypes;
-        } vecs;
+    struct {
+        union {
+            struct {
+                ompi_op_t *op;
+                ompi_datatype_t *datatype;
+            } op;
+            struct {
+                ompi_datatype_t *stype;
+                ompi_datatype_t *rtype;
+            } types;
+            struct {
+                opal_object_t *objs[2];
+            } objs;
+            struct {
+                ompi_datatype_t * const *stypes;
+                ompi_datatype_t * const *rtypes;
+                int scount;
+                int rcount;
+            } vecs;
+        } refcounted;
+        void* release_arrays[OMPI_REQ_NB_RELEASE_ARRAYS];
     } data;
 };
 
@@ -99,8 +112,7 @@ OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_coll_base_avail_coll_t);
 
 /**
  * A MPI_like function doing a send and a receive simultaneously.
- * If one of the communications results in a zero-byte message the
- * communication is ignored, and no message will cross to the peer.
+ * Posts a irecv, does a send, then gets irecv completion.
  */
 int ompi_coll_base_sendrecv_actual( const void* sendbuf, size_t scount,
                                     ompi_datatype_t* sdatatype,
@@ -113,10 +125,8 @@ int ompi_coll_base_sendrecv_actual( const void* sendbuf, size_t scount,
 
 
 /**
- * Similar to the function above this implementation of send-receive
- * do not generate communications for zero-bytes messages. Thus, it is
- * improper to use in the context of some algorithms for collective
- * communications.
+ * A wrapper around ompi_coll_base_sendrecv_actual, with an optimized
+ * path for self-directed send/recv.
  */
 static inline int
 ompi_coll_base_sendrecv( void* sendbuf, size_t scount, ompi_datatype_t* sdatatype,
@@ -165,7 +175,7 @@ int ompi_coll_base_retain_op( ompi_request_t *request,
  * (will be cast internally).
  */
 int ompi_coll_base_retain_datatypes( ompi_request_t *request,
-                                      ompi_datatype_t *stype,
+                                     ompi_datatype_t *stype,
                                      ompi_datatype_t *rtype);
 
 /**
@@ -175,7 +185,8 @@ int ompi_coll_base_retain_datatypes( ompi_request_t *request,
  */
 int ompi_coll_base_retain_datatypes_w( ompi_request_t *request,
                                        ompi_datatype_t * const stypes[],
-                                       ompi_datatype_t * const rtypes[]);
+                                       ompi_datatype_t * const rtypes[],
+                                       bool use_topo);
 
 /* File reading function */
 int ompi_coll_base_file_getnext_long(FILE *fptr, int *fileline, long* val);
@@ -185,8 +196,9 @@ int ompi_coll_base_file_getnext_string(FILE *fptr, int *fileline, char** val);
  * eat the value, otherwise put it back into the file.
  */
 int ompi_coll_base_file_peek_next_char_is(FILE *fptr, int *fileline, int expected);
+int ompi_coll_base_file_peek_next_char_isdigit(FILE *fptr);
 
-/* Miscelaneous function */
+/* Miscellaneous function */
 const char* mca_coll_base_colltype_to_str(int collid);
 int mca_coll_base_name_to_colltype(const char* name);
 
diff --git a/ompi/mca/coll/base/coll_tags.h b/ompi/mca/coll/base/coll_tags.h
index ec3d86bc5a8..5dc19061b37 100644
--- a/ompi/mca/coll/base/coll_tags.h
+++ b/ompi/mca/coll/base/coll_tags.h
@@ -59,7 +59,9 @@
 /* one extra reserved to avoid revoke for normal reqs, see request/req_ft.c*/
 #define MCA_COLL_BASE_TAG_FT_END                 (MCA_COLL_BASE_TAG_FT_BASE - 3)
 
-#define MCA_COLL_BASE_TAG_STATIC_END             MCA_COLL_BASE_TAG_FT_END
+#define MCA_COLL_BASE_TAG_UCC                    (MCA_COLL_BASE_TAG_FT_END - 1)
+
+#define MCA_COLL_BASE_TAG_STATIC_END             (MCA_COLL_BASE_TAG_UCC - 1)
 
 
 
diff --git a/ompi/mca/coll/base/help-mca-coll-base.txt b/ompi/mca/coll/base/help-mca-coll-base.txt
index d6e0071fa7a..f5afb0ce19b 100644
--- a/ompi/mca/coll/base/help-mca-coll-base.txt
+++ b/ompi/mca/coll/base/help-mca-coll-base.txt
@@ -10,7 +10,8 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2015      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -45,3 +46,10 @@ using it was destroyed.
 
 This is somewhat unusual: the module itself may be at fault, or this
 may be a symptom of another issue (e.g., a memory problem).
+#
+[comm-select:missing collective]
+%s %s collective require support from a prior collective module for the fallback
+case. Such support is missing in this run.
+hostname:   %s
+priority:   %d
+#
diff --git a/ompi/mca/coll/basic/Makefile.am b/ompi/mca/coll/basic/Makefile.am
index 341c5def950..8e1562c12f9 100644
--- a/ompi/mca/coll/basic/Makefile.am
+++ b/ompi/mca/coll/basic/Makefile.am
@@ -14,6 +14,7 @@
 # Copyright (c) 2013      Los Alamos National Security, LLC. All rights
 #                         reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -43,8 +44,6 @@ sources = \
         coll_basic_reduce.c \
         coll_basic_reduce_scatter.c \
         coll_basic_reduce_scatter_block.c \
-        coll_basic_scan.c \
-        coll_basic_exscan.c \
         coll_basic_scatter.c \
         coll_basic_scatterv.c
 
diff --git a/ompi/mca/coll/basic/coll_basic.h b/ompi/mca/coll/basic/coll_basic.h
index fc0b8eedeb9..f1ded3f7e74 100644
--- a/ompi/mca/coll/basic/coll_basic.h
+++ b/ompi/mca/coll/basic/coll_basic.h
@@ -16,6 +16,7 @@
  *                         reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -39,7 +40,7 @@ BEGIN_C_DECLS
 
     /* Globally exported variables */
 
-    OMPI_MODULE_DECLSPEC extern const mca_coll_base_component_2_4_0_t
+    OMPI_DECLSPEC extern const mca_coll_base_component_3_0_0_t
         mca_coll_basic_component;
     extern int mca_coll_basic_priority;
     extern int mca_coll_basic_crossover;
@@ -52,64 +53,61 @@ BEGIN_C_DECLS
         *mca_coll_basic_comm_query(struct ompi_communicator_t *comm,
                                    int *priority);
 
-    int mca_coll_basic_module_enable(mca_coll_base_module_t *module,
-                                     struct ompi_communicator_t *comm);
-
-    int mca_coll_basic_allgather_inter(const void *sbuf, int scount,
+    int mca_coll_basic_allgather_inter(const void *sbuf, size_t scount,
                                        struct ompi_datatype_t *sdtype,
-                                       void *rbuf, int rcount,
+                                       void *rbuf, size_t rcount,
                                        struct ompi_datatype_t *rdtype,
                                        struct ompi_communicator_t *comm,
                                        mca_coll_base_module_t *module);
 
-    int mca_coll_basic_allgatherv_inter(const void *sbuf, int scount,
+    int mca_coll_basic_allgatherv_inter(const void *sbuf, size_t scount,
                                         struct ompi_datatype_t *sdtype,
-                                        void *rbuf, const int *rcounts,
-                                        const int *disps,
+                                        void *rbuf, ompi_count_array_t rcounts,
+                                        ompi_disp_array_t disps,
                                         struct ompi_datatype_t *rdtype,
                                         struct ompi_communicator_t *comm,
                                         mca_coll_base_module_t *module);
 
-    int mca_coll_basic_allreduce_intra(const void *sbuf, void *rbuf, int count,
+    int mca_coll_basic_allreduce_intra(const void *sbuf, void *rbuf, size_t count,
                                        struct ompi_datatype_t *dtype,
                                        struct ompi_op_t *op,
                                        struct ompi_communicator_t *comm,
                                        mca_coll_base_module_t *module);
-    int mca_coll_basic_allreduce_inter(const void *sbuf, void *rbuf, int count,
+    int mca_coll_basic_allreduce_inter(const void *sbuf, void *rbuf, size_t count,
                                        struct ompi_datatype_t *dtype,
                                        struct ompi_op_t *op,
                                        struct ompi_communicator_t *comm,
                                        mca_coll_base_module_t *module);
 
-    int mca_coll_basic_alltoall_inter(const void *sbuf, int scount,
+    int mca_coll_basic_alltoall_inter(const void *sbuf, size_t scount,
                                       struct ompi_datatype_t *sdtype,
-                                      void *rbuf, int rcount,
+                                      void *rbuf, size_t rcount,
                                       struct ompi_datatype_t *rdtype,
                                       struct ompi_communicator_t *comm,
                                       mca_coll_base_module_t *module);
 
-    int mca_coll_basic_alltoallv_inter(const void *sbuf, const int *scounts,
-                                       const int *sdisps,
+    int mca_coll_basic_alltoallv_inter(const void *sbuf, ompi_count_array_t scounts,
+                                       ompi_disp_array_t sdisps,
                                        struct ompi_datatype_t *sdtype,
-                                       void *rbuf, const int *rcounts,
-                                       const int *rdisps,
+                                       void *rbuf, ompi_count_array_t rcounts,
+                                       ompi_disp_array_t rdisps,
                                        struct ompi_datatype_t *rdtype,
                                        struct ompi_communicator_t *comm,
                                        mca_coll_base_module_t *module);
 
-    int mca_coll_basic_alltoallw_intra(const void *sbuf, const int *scounts,
-                                       const int *sdisps,
+    int mca_coll_basic_alltoallw_intra(const void *sbuf, ompi_count_array_t scounts,
+                                       ompi_disp_array_t sdisps,
                                        struct ompi_datatype_t * const *sdtypes,
-                                       void *rbuf, const int *rcounts,
-                                       const int *rdisps,
+                                       void *rbuf, ompi_count_array_t rcounts,
+                                       ompi_disp_array_t rdisps,
                                        struct ompi_datatype_t * const *rdtypes,
                                        struct ompi_communicator_t *comm,
                                        mca_coll_base_module_t *module);
-    int mca_coll_basic_alltoallw_inter(const void *sbuf, const int *scounts,
-                                       const int *sdisps,
+    int mca_coll_basic_alltoallw_inter(const void *sbuf, ompi_count_array_t scounts,
+                                       ompi_disp_array_t sdisps,
                                        struct ompi_datatype_t * const *sdtypes,
-                                       void *rbuf, const int *rcounts,
-                                       const int *rdisps,
+                                       void *rbuf, ompi_count_array_t rcounts,
+                                       ompi_disp_array_t rdisps,
                                        struct ompi_datatype_t * const *rdtypes,
                                        struct ompi_communicator_t *comm,
                                        mca_coll_base_module_t *module);
@@ -120,74 +118,50 @@ BEGIN_C_DECLS
     int mca_coll_basic_barrier_intra_log(struct ompi_communicator_t *comm,
                                          mca_coll_base_module_t *module);
 
-    int mca_coll_basic_bcast_lin_inter(void *buff, int count,
-                                       struct ompi_datatype_t *datatype,
-                                       int root,
-                                       struct ompi_communicator_t *comm,
-                                       mca_coll_base_module_t *module);
-
-    int mca_coll_basic_bcast_log_intra(void *buff, int count,
+    int mca_coll_basic_bcast_lin_inter(void *buff, size_t count,
                                        struct ompi_datatype_t *datatype,
                                        int root,
                                        struct ompi_communicator_t *comm,
                                        mca_coll_base_module_t *module);
 
-    int mca_coll_basic_bcast_log_inter(void *buff, int count,
+    int mca_coll_basic_bcast_log_intra(void *buff, size_t count,
                                        struct ompi_datatype_t *datatype,
                                        int root,
                                        struct ompi_communicator_t *comm,
                                        mca_coll_base_module_t *module);
 
-    int mca_coll_basic_exscan_intra(const void *sbuf, void *rbuf, int count,
-                                    struct ompi_datatype_t *dtype,
-                                    struct ompi_op_t *op,
-                                    struct ompi_communicator_t *comm,
-                                    mca_coll_base_module_t *module);
-
-    int mca_coll_basic_exscan_inter(const void *sbuf, void *rbuf, int count,
-                                    struct ompi_datatype_t *dtype,
-                                    struct ompi_op_t *op,
-                                    struct ompi_communicator_t *comm,
-                                    mca_coll_base_module_t *module);
-
-    int mca_coll_basic_gather_inter(const void *sbuf, int scount,
+    int mca_coll_basic_gather_inter(const void *sbuf, size_t scount,
                                     struct ompi_datatype_t *sdtype,
-                                    void *rbuf, int rcount,
+                                    void *rbuf, size_t rcount,
                                     struct ompi_datatype_t *rdtype,
                                     int root,
                                     struct ompi_communicator_t *comm,
                                     mca_coll_base_module_t *module);
 
-    int mca_coll_basic_gatherv_intra(const void *sbuf, int scount,
+    int mca_coll_basic_gatherv_intra(const void *sbuf, size_t scount,
                                      struct ompi_datatype_t *sdtype,
-                                     void *rbuf, const int *rcounts, const int *disps,
+                                     void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
                                      struct ompi_datatype_t *rdtype,
                                      int root,
                                      struct ompi_communicator_t *comm,
                                      mca_coll_base_module_t *module);
 
-    int mca_coll_basic_gatherv_inter(const void *sbuf, int scount,
+    int mca_coll_basic_gatherv_inter(const void *sbuf, size_t scount,
                                      struct ompi_datatype_t *sdtype,
-                                     void *rbuf, const int *rcounts, const int *disps,
+                                     void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
                                      struct ompi_datatype_t *rdtype,
                                      int root,
                                      struct ompi_communicator_t *comm,
                                      mca_coll_base_module_t *module);
 
-    int mca_coll_basic_reduce_lin_inter(const void *sbuf, void *rbuf, int count,
+    int mca_coll_basic_reduce_lin_inter(const void *sbuf, void *rbuf, size_t count,
                                         struct ompi_datatype_t *dtype,
                                         struct ompi_op_t *op,
                                         int root,
                                         struct ompi_communicator_t *comm,
                                         mca_coll_base_module_t *module);
 
-    int mca_coll_basic_reduce_log_intra(const void *sbuf, void *rbuf, int count,
-                                        struct ompi_datatype_t *dtype,
-                                        struct ompi_op_t *op,
-                                        int root,
-                                        struct ompi_communicator_t *comm,
-                                        mca_coll_base_module_t *module);
-    int mca_coll_basic_reduce_log_inter(const void *sbuf, void *rbuf, int count,
+    int mca_coll_basic_reduce_log_intra(const void *sbuf, void *rbuf, size_t count,
                                         struct ompi_datatype_t *dtype,
                                         struct ompi_op_t *op,
                                         int root,
@@ -195,89 +169,84 @@ BEGIN_C_DECLS
                                         mca_coll_base_module_t *module);
 
     int mca_coll_basic_reduce_scatter_block_intra(const void *sbuf, void *rbuf,
-                                                  int rcount,
+                                                  size_t rcount,
                                                   struct ompi_datatype_t *dtype,
                                                   struct ompi_op_t *op,
                                                   struct ompi_communicator_t *comm,
                                                   mca_coll_base_module_t *module);
 
     int mca_coll_basic_reduce_scatter_block_inter(const void *sbuf, void *rbuf,
-                                                  int rcount,
+                                                  size_t rcount,
                                                   struct ompi_datatype_t *dtype,
                                                   struct ompi_op_t *op,
                                                   struct ompi_communicator_t *comm,
                                                   mca_coll_base_module_t *module);
 
     int mca_coll_basic_reduce_scatter_intra(const void *sbuf, void *rbuf,
-                                            const int *rcounts,
+                                            ompi_count_array_t rcounts,
                                             struct ompi_datatype_t *dtype,
                                             struct ompi_op_t *op,
                                             struct ompi_communicator_t *comm,
                                             mca_coll_base_module_t *module);
 
     int mca_coll_basic_reduce_scatter_inter(const void *sbuf, void *rbuf,
-                                            const int *rcounts,
+                                            ompi_count_array_t rcounts,
                                             struct ompi_datatype_t *dtype,
                                             struct ompi_op_t *op,
                                             struct ompi_communicator_t *comm,
                                             mca_coll_base_module_t *module);
 
-    int mca_coll_basic_scan_intra(const void *sbuf, void *rbuf, int count,
-                                  struct ompi_datatype_t *dtype,
-                                  struct ompi_op_t *op,
-                                  struct ompi_communicator_t *comm,
-                                  mca_coll_base_module_t *module);
-    int mca_coll_basic_scan_inter(const void *sbuf, void *rbuf, int count,
+    int mca_coll_basic_scan_inter(const void *sbuf, void *rbuf, size_t count,
                                   struct ompi_datatype_t *dtype,
                                   struct ompi_op_t *op,
                                   struct ompi_communicator_t *comm,
                                   mca_coll_base_module_t *module);
 
-    int mca_coll_basic_scatter_inter(const void *sbuf, int scount,
+    int mca_coll_basic_scatter_inter(const void *sbuf, size_t scount,
                                      struct ompi_datatype_t *sdtype,
-                                     void *rbuf, int rcount,
+                                     void *rbuf, size_t rcount,
                                      struct ompi_datatype_t *rdtype,
                                      int root,
                                      struct ompi_communicator_t *comm,
                                      mca_coll_base_module_t *module);
 
-    int mca_coll_basic_scatterv_intra(const void *sbuf, const int *scounts, const int *disps,
+    int mca_coll_basic_scatterv_intra(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t disps,
                                       struct ompi_datatype_t *sdtype,
-                                      void *rbuf, int rcount,
+                                      void *rbuf, size_t rcount,
                                       struct ompi_datatype_t *rdtype,
                                       int root,
                                       struct ompi_communicator_t *comm,
                                       mca_coll_base_module_t *module);
-    int mca_coll_basic_scatterv_inter(const void *sbuf, const int *scounts, const int *disps,
+    int mca_coll_basic_scatterv_inter(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t disps,
                                       struct ompi_datatype_t *sdtype,
-                                      void *rbuf, int rcount,
+                                      void *rbuf, size_t rcount,
                                       struct ompi_datatype_t *rdtype,
                                       int root,
                                       struct ompi_communicator_t *comm,
                                       mca_coll_base_module_t *module);
 
-     int mca_coll_basic_neighbor_allgather(const void *sbuf, int scount,
+     int mca_coll_basic_neighbor_allgather(const void *sbuf, size_t scount,
                                            struct ompi_datatype_t *sdtype, void *rbuf,
-                                           int rcount, struct ompi_datatype_t *rdtype,
+                                           size_t rcount, struct ompi_datatype_t *rdtype,
                                            struct ompi_communicator_t *comm,
                                            mca_coll_base_module_t *module);
 
-     int mca_coll_basic_neighbor_allgatherv(const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-                                            void *rbuf, const int rcounts[], const int disps[], struct ompi_datatype_t *rdtype,
+     int mca_coll_basic_neighbor_allgatherv(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                                            void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps, struct ompi_datatype_t *rdtype,
                                             struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
 
-     int mca_coll_basic_neighbor_alltoall(const void *sbuf, int scount, struct ompi_datatype_t *sdtype, void *rbuf,
-                                          int rcount, struct ompi_datatype_t *rdtype, struct ompi_communicator_t *comm,
+     int mca_coll_basic_neighbor_alltoall(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype, void *rbuf,
+                                          size_t rcount, struct ompi_datatype_t *rdtype, struct ompi_communicator_t *comm,
                                           mca_coll_base_module_t *module);
 
-     int mca_coll_basic_neighbor_alltoallv(const void *sbuf, const int scounts[], const int sdisps[],
-                                           struct ompi_datatype_t *sdtype, void *rbuf, const int rcounts[],
-                                           const int rdisps[], struct ompi_datatype_t *rdtype,
+     int mca_coll_basic_neighbor_alltoallv(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
+                                           struct ompi_datatype_t *sdtype, void *rbuf, ompi_count_array_t rcounts,
+                                           ompi_disp_array_t rdisps, struct ompi_datatype_t *rdtype,
                                            struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
 
-     int mca_coll_basic_neighbor_alltoallw(const void *sbuf, const int scounts[], const MPI_Aint sdisps[],
-                                           struct ompi_datatype_t * const *sdtypes, void *rbuf, const int rcounts[],
-                                           const MPI_Aint rdisps[], struct ompi_datatype_t * const *rdtypes,
+     int mca_coll_basic_neighbor_alltoallw(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
+                                           struct ompi_datatype_t * const *sdtypes, void *rbuf, ompi_count_array_t rcounts,
+                                           ompi_disp_array_t rdisps, struct ompi_datatype_t * const *rdtypes,
                                            struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
 
 
diff --git a/ompi/mca/coll/basic/coll_basic_allgather.c b/ompi/mca/coll/basic/coll_basic_allgather.c
index 446a5fe49ad..ea12b66178c 100644
--- a/ompi/mca/coll/basic/coll_basic_allgather.c
+++ b/ompi/mca/coll/basic/coll_basic_allgather.c
@@ -41,9 +41,9 @@
  *	Returns:	- MPI_SUCCESS or error code
  */
 int
-mca_coll_basic_allgather_inter(const void *sbuf, int scount,
+mca_coll_basic_allgather_inter(const void *sbuf, size_t scount,
                                struct ompi_datatype_t *sdtype,
-                               void *rbuf, int rcount,
+                               void *rbuf, size_t rcount,
                                struct ompi_datatype_t *rdtype,
                                struct ompi_communicator_t *comm,
                                mca_coll_base_module_t *module)
@@ -60,10 +60,10 @@ mca_coll_basic_allgather_inter(const void *sbuf, int scount,
     rsize = ompi_comm_remote_size(comm);
 
     /* Algorithm:
-     * - a gather to the root in remote group (simultaniously executed,
-     * thats why we cannot use coll_gather).
+     * - a gather to the root in remote group (simultaneously executed,
+     * that's why we cannot use coll_gather).
      * - exchange the temp-results between two roots
-     * - inter-bcast (again simultanious).
+     * - inter-bcast (again simultaneous).
      */
 
     /* Step one: gather operations: */
@@ -106,7 +106,7 @@ mca_coll_basic_allgather_inter(const void *sbuf, int scount,
         err = ompi_request_wait_all(rsize + 1, reqs, MPI_STATUSES_IGNORE);
         if (OMPI_SUCCESS != err) { line = __LINE__; goto exit; }
 
-        /* Step 2: exchange the resuts between the root processes */
+        /* Step 2: exchange the results between the root processes */
         span = opal_datatype_span(&sdtype->super, (int64_t)scount * (int64_t)size, &gap);
         tmpbuf_free = (char *) malloc(span);
         if (NULL == tmpbuf_free) { line = __LINE__; err = OMPI_ERR_OUT_OF_RESOURCE; goto exit; }
diff --git a/ompi/mca/coll/basic/coll_basic_allgatherv.c b/ompi/mca/coll/basic/coll_basic_allgatherv.c
index 3009bfcae5d..8eae54a295a 100644
--- a/ompi/mca/coll/basic/coll_basic_allgatherv.c
+++ b/ompi/mca/coll/basic/coll_basic_allgatherv.c
@@ -38,20 +38,23 @@
  *	Returns:	- MPI_SUCCESS or error code
  */
 int
-mca_coll_basic_allgatherv_inter(const void *sbuf, int scount,
+mca_coll_basic_allgatherv_inter(const void *sbuf, size_t scount,
                                 struct ompi_datatype_t *sdtype,
-                                void *rbuf, const int *rcounts, const int *disps,
+                                void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
                                 struct ompi_datatype_t *rdtype,
                                 struct ompi_communicator_t *comm,
                                 mca_coll_base_module_t *module)
 {
     int rsize, err, i;
-    int *scounts, *sdisps;
+    size_t *scounts;
+    ptrdiff_t *sdisps;
+    ompi_count_array_t scounts_desc;
+    ompi_disp_array_t sdisps_desc;
 
     rsize = ompi_comm_remote_size(comm);
 
-    scounts = (int *) malloc(2 * rsize * sizeof(int));
-    sdisps = scounts + rsize;
+    scounts = (size_t *) malloc(rsize * sizeof(size_t) + rsize * sizeof(ptrdiff_t));
+    sdisps = (ptrdiff_t *) (scounts + rsize);
     if (NULL == scounts) {
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
@@ -61,7 +64,9 @@ mca_coll_basic_allgatherv_inter(const void *sbuf, int scount,
         sdisps[i] = 0;
     }
 
-    err = comm->c_coll->coll_alltoallv(sbuf, scounts, sdisps, sdtype,
+    OMPI_COUNT_ARRAY_INIT(&scounts_desc, scounts);
+    OMPI_DISP_ARRAY_INIT(&sdisps_desc, sdisps);
+    err = comm->c_coll->coll_alltoallv(sbuf, scounts_desc, sdisps_desc, sdtype,
                                       rbuf, rcounts, disps, rdtype, comm,
                                       comm->c_coll->coll_alltoallv_module);
 
diff --git a/ompi/mca/coll/basic/coll_basic_allreduce.c b/ompi/mca/coll/basic/coll_basic_allreduce.c
index 84f60f2f685..dee7842d175 100644
--- a/ompi/mca/coll/basic/coll_basic_allreduce.c
+++ b/ompi/mca/coll/basic/coll_basic_allreduce.c
@@ -41,7 +41,7 @@
  *	Returns:	- MPI_SUCCESS or error code
  */
 int
-mca_coll_basic_allreduce_intra(const void *sbuf, void *rbuf, int count,
+mca_coll_basic_allreduce_intra(const void *sbuf, void *rbuf, size_t count,
                                struct ompi_datatype_t *dtype,
                                struct ompi_op_t *op,
                                struct ompi_communicator_t *comm,
@@ -76,7 +76,7 @@ mca_coll_basic_allreduce_intra(const void *sbuf, void *rbuf, int count,
  *	Returns:	- MPI_SUCCESS or error code
  */
 int
-mca_coll_basic_allreduce_inter(const void *sbuf, void *rbuf, int count,
+mca_coll_basic_allreduce_inter(const void *sbuf, void *rbuf, size_t count,
                                struct ompi_datatype_t *dtype,
                                struct ompi_op_t *op,
                                struct ompi_communicator_t *comm,
@@ -97,7 +97,7 @@ mca_coll_basic_allreduce_inter(const void *sbuf, void *rbuf, int count,
      * and which one enters coll_reduce with providing
      * MPI_PROC_NULL as root argument etc.) Here,
      * we execute the data exchange for both groups
-     * simultaniously. */
+     * simultaneously. */
     /*****************************************************************/
     if (rank == root) {
         err = ompi_datatype_type_extent(dtype, &extent);
diff --git a/ompi/mca/coll/basic/coll_basic_alltoall.c b/ompi/mca/coll/basic/coll_basic_alltoall.c
index 6d3ff46adcd..5dc43dc3405 100644
--- a/ompi/mca/coll/basic/coll_basic_alltoall.c
+++ b/ompi/mca/coll/basic/coll_basic_alltoall.c
@@ -41,9 +41,9 @@
  *	Returns:	- MPI_SUCCESS or an MPI error code
  */
 int
-mca_coll_basic_alltoall_inter(const void *sbuf, int scount,
+mca_coll_basic_alltoall_inter(const void *sbuf, size_t scount,
                               struct ompi_datatype_t *sdtype,
-                              void *rbuf, int rcount,
+                              void *rbuf, size_t rcount,
                               struct ompi_datatype_t *rdtype,
                               struct ompi_communicator_t *comm,
                               mca_coll_base_module_t *module)
diff --git a/ompi/mca/coll/basic/coll_basic_alltoallv.c b/ompi/mca/coll/basic/coll_basic_alltoallv.c
index 26e585ce2e8..c8dfcd5b5a9 100644
--- a/ompi/mca/coll/basic/coll_basic_alltoallv.c
+++ b/ompi/mca/coll/basic/coll_basic_alltoallv.c
@@ -42,9 +42,9 @@
  *	Returns:	- MPI_SUCCESS or an MPI error code
  */
 int
-mca_coll_basic_alltoallv_inter(const void *sbuf, const int *scounts, const int *sdisps,
+mca_coll_basic_alltoallv_inter(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
                                struct ompi_datatype_t *sdtype, void *rbuf,
-                               const int *rcounts, const int *rdisps,
+                               ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
                                struct ompi_datatype_t *rdtype,
                                struct ompi_communicator_t *comm,
                                mca_coll_base_module_t *module)
@@ -75,9 +75,9 @@ mca_coll_basic_alltoallv_inter(const void *sbuf, const int *scounts, const int *
     /* Post all receives first  */
     /* A simple optimization: do not send and recv msgs of length zero */
     for (i = 0; i < rsize; ++i) {
-        prcv = ((char *) rbuf) + (rdisps[i] * rcvextent);
-        if (rcounts[i] > 0) {
-            err = MCA_PML_CALL(irecv(prcv, rcounts[i], rdtype,
+        prcv = ((char *) rbuf) + (ompi_disp_array_get(rdisps, i) * rcvextent);
+        if (ompi_count_array_get(rcounts, i) > 0) {
+            err = MCA_PML_CALL(irecv(prcv, ompi_count_array_get(rcounts, i), rdtype,
                                      i, MCA_COLL_BASE_TAG_ALLTOALLV, comm,
                                      &preq[i]));
             if (MPI_SUCCESS != err) {
@@ -89,9 +89,9 @@ mca_coll_basic_alltoallv_inter(const void *sbuf, const int *scounts, const int *
 
     /* Now post all sends */
     for (i = 0; i < rsize; ++i) {
-        psnd = ((char *) sbuf) + (sdisps[i] * sndextent);
-        if (scounts[i] > 0) {
-            err = MCA_PML_CALL(isend(psnd, scounts[i], sdtype,
+        psnd = ((char *) sbuf) + (ompi_disp_array_get(sdisps, i) * sndextent);
+        if (ompi_count_array_get(scounts, i) > 0) {
+            err = MCA_PML_CALL(isend(psnd, ompi_count_array_get(scounts, i), sdtype,
                                      i, MCA_COLL_BASE_TAG_ALLTOALLV,
                                      MCA_PML_BASE_SEND_STANDARD, comm,
                                      &preq[rsize + i]));
diff --git a/ompi/mca/coll/basic/coll_basic_alltoallw.c b/ompi/mca/coll/basic/coll_basic_alltoallw.c
index 93fa880fc2d..ca761c053d0 100644
--- a/ompi/mca/coll/basic/coll_basic_alltoallw.c
+++ b/ompi/mca/coll/basic/coll_basic_alltoallw.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2016 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -14,8 +14,8 @@
  * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2013      FUJITSU LIMITED.  All rights reserved.
- * Copyright (c) 2014-2016 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2014-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2014      Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2017      IBM Corporation. All rights reserved.
  * $COPYRIGHT$
@@ -31,100 +31,126 @@
 #include "mpi.h"
 #include "ompi/constants.h"
 #include "ompi/datatype/ompi_datatype.h"
+#include "opal/datatype/opal_convertor_internal.h"
 #include "ompi/mca/coll/coll.h"
 #include "ompi/mca/coll/base/coll_tags.h"
 #include "ompi/mca/pml/pml.h"
 
-
+/*
+ * We want to minimize the amount of temporary memory needed while allowing as many ranks
+ * to exchange data simultaneously. We use a variation of the ring algorithm, where in a
+ * single step a process echange the data with both neighbors at distance k (on the left
+ * and the right on a logical ring topology). With this approach we need to pack the data
+ * for a single of the two neighbors, as we can then use the original buffer (and datatype
+ * and count) to send the data to the other.
+ */
 static int
-mca_coll_basic_alltoallw_intra_inplace(const void *rbuf, const int *rcounts, const int *rdisps,
+mca_coll_basic_alltoallw_intra_inplace(const void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
                                        struct ompi_datatype_t * const *rdtypes,
                                        struct ompi_communicator_t *comm,
                                        mca_coll_base_module_t *module)
 {
-    int i, j, size, rank, err = MPI_SUCCESS, max_size;
-    ompi_request_t *req;
-    char *tmp_buffer, *save_buffer = NULL;
-    ptrdiff_t ext, gap = 0;
-
-    /* Initialize. */
+    int i, size, rank, left, right, err = MPI_SUCCESS;
+    ompi_request_t *req = MPI_REQUEST_NULL;
+    char *tmp_buffer = NULL;
+    size_t max_size = 0, packed_size, msg_size_left, msg_size_right;
+    opal_convertor_t convertor;
 
     size = ompi_comm_size(comm);
-    rank = ompi_comm_rank(comm);
-
-    /* If only one process, we're done. */
-    if (1 == size) {
+    if (1 == size) {  /* If only one process, we're done. */
         return MPI_SUCCESS;
     }
+    rank = ompi_comm_rank(comm);
 
-    /* Find the largest receive amount */
-    for (i = 0, max_size = 0 ; i < size ; ++i) {
-        ext = opal_datatype_span(&rdtypes[i]->super, rcounts[i], &gap);
-
-        max_size = ext > max_size ? ext : max_size;
+    /* Find the largest amount of packed send/recv data among all peers where
+     * we need to pack before the send.
+     */
+    for (i = 1 ; i <= (size >> 1) ; ++i) {
+        right = (rank + i) % size;
+#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
+        ompi_proc_t *ompi_proc = ompi_comm_peer_lookup(comm, right);
+
+        if( OPAL_LIKELY(opal_local_arch == ompi_proc->super.proc_convertor->master->remote_arch))  {
+            opal_datatype_type_size(&rdtypes[right]->super, &packed_size);
+        } else {
+            packed_size = opal_datatype_compute_remote_size(&rdtypes[right]->super,
+                                                            ompi_proc->super.proc_convertor->master->remote_sizes);
+        }
+#else
+        opal_datatype_type_size(&rdtypes[right]->super, &packed_size);
+#endif  /* OPAL_ENABLE_HETEROGENEOUS_SUPPORT */
+        packed_size *= ompi_count_array_get(rcounts, right);
+        max_size = packed_size > max_size ? packed_size : max_size;
     }
 
     /* Allocate a temporary buffer */
-    tmp_buffer = save_buffer = calloc (max_size, 1);
+    tmp_buffer = calloc (max_size, 1);
     if (NULL == tmp_buffer) {
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
-    tmp_buffer -= gap;
-
-    /* in-place alltoallw slow algorithm (but works) */
-    for (i = 0 ; i < size ; ++i) {
-        size_t msg_size_i;
-        ompi_datatype_type_size(rdtypes[i], &msg_size_i);
-        msg_size_i *= rcounts[i];
-        for (j = i+1 ; j < size ; ++j) {
-            size_t msg_size_j;
-            ompi_datatype_type_size(rdtypes[j], &msg_size_j);
-            msg_size_j *= rcounts[j];
-
-            /* Initiate all send/recv to/from others. */
-            if (i == rank && msg_size_j != 0) {
-                /* Copy the data into the temporary buffer */
-                err = ompi_datatype_copy_content_same_ddt (rdtypes[j], rcounts[j],
-                                                           tmp_buffer, (char *) rbuf + rdisps[j]);
-                if (MPI_SUCCESS != err) { goto error_hndl; }
-
-                /* Exchange data with the peer */
-                err = MCA_PML_CALL(irecv ((char *) rbuf + rdisps[j], rcounts[j], rdtypes[j],
-                                          j, MCA_COLL_BASE_TAG_ALLTOALLW, comm, &req));
-                if (MPI_SUCCESS != err) { goto error_hndl; }
-
-                err = MCA_PML_CALL(send ((void *) tmp_buffer,  rcounts[j], rdtypes[j],
-                                          j, MCA_COLL_BASE_TAG_ALLTOALLW, MCA_PML_BASE_SEND_STANDARD,
-                                          comm));
-                if (MPI_SUCCESS != err) { goto error_hndl; }
-            } else if (j == rank && msg_size_i != 0) {
-                /* Copy the data into the temporary buffer */
-                err = ompi_datatype_copy_content_same_ddt (rdtypes[i], rcounts[i],
-                                                           tmp_buffer, (char *) rbuf + rdisps[i]);
-                if (MPI_SUCCESS != err) { goto error_hndl; }
-
-                /* Exchange data with the peer */
-                err = MCA_PML_CALL(irecv ((char *) rbuf + rdisps[i], rcounts[i], rdtypes[i],
-                                          i, MCA_COLL_BASE_TAG_ALLTOALLW, comm, &req));
-                if (MPI_SUCCESS != err) { goto error_hndl; }
-
-                err = MCA_PML_CALL(send ((void *) tmp_buffer,  rcounts[i], rdtypes[i],
-                                          i, MCA_COLL_BASE_TAG_ALLTOALLW, MCA_PML_BASE_SEND_STANDARD,
-                                          comm));
-                if (MPI_SUCCESS != err) { goto error_hndl; }
-            } else {
-                continue;
-            }
-
-            /* Wait for the requests to complete */
+
+    for (i = 1 ; i <= (size >> 1) ; ++i) {
+        struct iovec iov = {.iov_base = tmp_buffer, .iov_len = max_size};
+        uint32_t iov_count = 1;
+
+        right = (rank + i) % size;
+        left  = (rank + size - i) % size;
+
+        ompi_datatype_type_size(rdtypes[right], &msg_size_right);
+        msg_size_right *= ompi_count_array_get(rcounts, right);
+
+        ompi_datatype_type_size(rdtypes[left], &msg_size_left);
+        msg_size_left *= ompi_count_array_get(rcounts, left);
+
+        if( 0 != msg_size_right ) {  /* nothing to exchange with the peer on the right */
+            ompi_proc_t *right_proc = ompi_comm_peer_lookup(comm, right);
+            opal_convertor_clone(right_proc->super.proc_convertor, &convertor, 0);
+            opal_convertor_prepare_for_send(&convertor, &rdtypes[right]->super, ompi_count_array_get(rcounts, right),
+                                            (char *) rbuf + ompi_disp_array_get(rdisps, right));
+            packed_size = max_size;
+            err = opal_convertor_pack(&convertor, &iov, &iov_count, &packed_size);
+            if (1 != err) { goto error_hndl; }
+
+            /* Receive data from the right */
+            err = MCA_PML_CALL(irecv ((char *) rbuf + ompi_disp_array_get(rdisps, right),
+                                      ompi_count_array_get(rcounts, right), rdtypes[right],
+                                      right, MCA_COLL_BASE_TAG_ALLTOALLW, comm, &req));
+            if (MPI_SUCCESS != err) { goto error_hndl; }
+        }
+
+        if( (left != right) && (0 != msg_size_left) ) {
+            /* Send data to the left */
+            err = MCA_PML_CALL(send ((char *) rbuf + ompi_disp_array_get(rdisps, left),
+                                     ompi_count_array_get(rcounts, left), rdtypes[left],
+                                     left, MCA_COLL_BASE_TAG_ALLTOALLW, MCA_PML_BASE_SEND_STANDARD,
+                                     comm));
+            if (MPI_SUCCESS != err) { goto error_hndl; }
+
             err = ompi_request_wait (&req, MPI_STATUSES_IGNORE);
             if (MPI_SUCCESS != err) { goto error_hndl; }
+
+            /* Receive data from the left */
+            err = MCA_PML_CALL(irecv ((char *) rbuf + ompi_disp_array_get(rdisps, left),
+                                      ompi_count_array_get(rcounts, left), rdtypes[left],
+                                      left, MCA_COLL_BASE_TAG_ALLTOALLW, comm, &req));
+            if (MPI_SUCCESS != err) { goto error_hndl; }
         }
+
+        if( 0 != msg_size_right ) {  /* nothing to exchange with the peer on the right */
+            /* Send data to the right */
+            err = MCA_PML_CALL(send ((char *) tmp_buffer,  packed_size, MPI_PACKED,
+                                     right, MCA_COLL_BASE_TAG_ALLTOALLW, MCA_PML_BASE_SEND_STANDARD,
+                                     comm));
+            if (MPI_SUCCESS != err) { goto error_hndl; }
+        }
+
+        err = ompi_request_wait (&req, MPI_STATUSES_IGNORE);
+        if (MPI_SUCCESS != err) { goto error_hndl; }
     }
 
  error_hndl:
     /* Free the temporary buffer */
-    free (save_buffer);
+    free (tmp_buffer);
 
     /* All done */
 
@@ -140,9 +166,9 @@ mca_coll_basic_alltoallw_intra_inplace(const void *rbuf, const int *rcounts, con
  *	Returns:	- MPI_SUCCESS or an MPI error code
  */
 int
-mca_coll_basic_alltoallw_intra(const void *sbuf, const int *scounts, const int *sdisps,
+mca_coll_basic_alltoallw_intra(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
                                struct ompi_datatype_t * const *sdtypes,
-                               void *rbuf, const int *rcounts, const int *rdisps,
+                               void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
                                struct ompi_datatype_t * const *rdtypes,
                                struct ompi_communicator_t *comm,
                                mca_coll_base_module_t *module)
@@ -162,11 +188,11 @@ mca_coll_basic_alltoallw_intra(const void *sbuf, const int *scounts, const int *
 
     /* simple optimization */
 
-    psnd = ((char *) sbuf) + sdisps[rank];
-    prcv = ((char *) rbuf) + rdisps[rank];
+    psnd = ((char *) sbuf) + ompi_disp_array_get(sdisps, rank);
+    prcv = ((char *) rbuf) + ompi_disp_array_get(rdisps, rank);
 
-    err = ompi_datatype_sndrcv(psnd, scounts[rank], sdtypes[rank],
-                               prcv, rcounts[rank], rdtypes[rank]);
+    err = ompi_datatype_sndrcv(psnd, ompi_count_array_get(scounts, rank), sdtypes[rank],
+                               prcv, ompi_count_array_get(rcounts, rank), rdtypes[rank]);
     if (MPI_SUCCESS != err) {
         return err;
     }
@@ -188,13 +214,13 @@ mca_coll_basic_alltoallw_intra(const void *sbuf, const int *scounts, const int *
     for (i = 0; i < size; ++i) {
         size_t msg_size;
         ompi_datatype_type_size(rdtypes[i], &msg_size);
-        msg_size *= rcounts[i];
+        msg_size *= ompi_count_array_get(rcounts, i);
 
         if (i == rank || 0 == msg_size)
             continue;
 
-        prcv = ((char *) rbuf) + rdisps[i];
-        err = MCA_PML_CALL(irecv_init(prcv, rcounts[i], rdtypes[i],
+        prcv = ((char *) rbuf) + ompi_disp_array_get(rdisps, i);
+        err = MCA_PML_CALL(irecv_init(prcv, ompi_count_array_get(rcounts, i), rdtypes[i],
                                       i, MCA_COLL_BASE_TAG_ALLTOALLW, comm,
                                       preq++));
         ++nreqs;
@@ -209,13 +235,13 @@ mca_coll_basic_alltoallw_intra(const void *sbuf, const int *scounts, const int *
     for (i = 0; i < size; ++i) {
         size_t msg_size;
         ompi_datatype_type_size(sdtypes[i], &msg_size);
-        msg_size *= scounts[i];
+        msg_size *= ompi_count_array_get(scounts, i);
 
         if (i == rank || 0 == msg_size)
             continue;
 
-        psnd = ((char *) sbuf) + sdisps[i];
-        err = MCA_PML_CALL(isend_init(psnd, scounts[i], sdtypes[i],
+        psnd = ((char *) sbuf) + ompi_disp_array_get(sdisps, i);
+        err = MCA_PML_CALL(isend_init(psnd, ompi_count_array_get(scounts, i), sdtypes[i],
                                       i, MCA_COLL_BASE_TAG_ALLTOALLW,
                                       MCA_PML_BASE_SEND_STANDARD, comm,
                                       preq++));
@@ -254,9 +280,9 @@ mca_coll_basic_alltoallw_intra(const void *sbuf, const int *scounts, const int *
  *	Returns:	- MPI_SUCCESS or an MPI error code
  */
 int
-mca_coll_basic_alltoallw_inter(const void *sbuf, const int *scounts, const int *sdisps,
+mca_coll_basic_alltoallw_inter(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
                                struct ompi_datatype_t * const *sdtypes,
-                               void *rbuf, const int *rcounts, const int *rdisps,
+                               void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
                                struct ompi_datatype_t * const *rdtypes,
                                struct ompi_communicator_t *comm,
                                mca_coll_base_module_t *module)
@@ -277,13 +303,13 @@ mca_coll_basic_alltoallw_inter(const void *sbuf, const int *scounts, const int *
     for (i = 0; i < size; ++i) {
         size_t msg_size;
         ompi_datatype_type_size(rdtypes[i], &msg_size);
-        msg_size *= rcounts[i];
+        msg_size *= ompi_count_array_get(rcounts, i);
 
         if (0 == msg_size)
             continue;
 
-        prcv = ((char *) rbuf) + rdisps[i];
-        err = MCA_PML_CALL(irecv_init(prcv, rcounts[i], rdtypes[i],
+        prcv = ((char *) rbuf) + ompi_disp_array_get(rdisps, i);
+        err = MCA_PML_CALL(irecv_init(prcv, ompi_count_array_get(rcounts, i), rdtypes[i],
                                       i, MCA_COLL_BASE_TAG_ALLTOALLW,
                                       comm, preq++));
         ++nreqs;
@@ -297,13 +323,13 @@ mca_coll_basic_alltoallw_inter(const void *sbuf, const int *scounts, const int *
     for (i = 0; i < size; ++i) {
         size_t msg_size;
         ompi_datatype_type_size(sdtypes[i], &msg_size);
-        msg_size *= scounts[i];
+        msg_size *= ompi_count_array_get(scounts, i);
 
         if (0 == msg_size)
             continue;
 
-        psnd = ((char *) sbuf) + sdisps[i];
-        err = MCA_PML_CALL(isend_init(psnd, scounts[i], sdtypes[i],
+        psnd = ((char *) sbuf) + ompi_disp_array_get(sdisps, i);
+        err = MCA_PML_CALL(isend_init(psnd, ompi_count_array_get(scounts, i), sdtypes[i],
                                       i, MCA_COLL_BASE_TAG_ALLTOALLW,
                                       MCA_PML_BASE_SEND_STANDARD, comm,
                                       preq++));
diff --git a/ompi/mca/coll/basic/coll_basic_bcast.c b/ompi/mca/coll/basic/coll_basic_bcast.c
index 3003582ded3..b35975bf7b9 100644
--- a/ompi/mca/coll/basic/coll_basic_bcast.c
+++ b/ompi/mca/coll/basic/coll_basic_bcast.c
@@ -41,7 +41,7 @@
  *	Returns:	- MPI_SUCCESS or error code
  */
 int
-mca_coll_basic_bcast_log_intra(void *buff, int count,
+mca_coll_basic_bcast_log_intra(void *buff, size_t count,
                                struct ompi_datatype_t *datatype, int root,
                                struct ompi_communicator_t *comm,
                                mca_coll_base_module_t *module)
@@ -136,7 +136,7 @@ mca_coll_basic_bcast_log_intra(void *buff, int count,
  *	Returns:	- MPI_SUCCESS or error code
  */
 int
-mca_coll_basic_bcast_lin_inter(void *buff, int count,
+mca_coll_basic_bcast_lin_inter(void *buff, size_t count,
                                struct ompi_datatype_t *datatype, int root,
                                struct ompi_communicator_t *comm,
                                mca_coll_base_module_t *module)
@@ -181,20 +181,3 @@ mca_coll_basic_bcast_lin_inter(void *buff, int count,
     /* All done */
     return err;
 }
-
-
-/*
- *	bcast_log_inter
- *
- *	Function:	- broadcast using O(N) algorithm
- *	Accepts:	- same arguments as MPI_Bcast()
- *	Returns:	- MPI_SUCCESS or error code
- */
-int
-mca_coll_basic_bcast_log_inter(void *buff, int count,
-                               struct ompi_datatype_t *datatype, int root,
-                               struct ompi_communicator_t *comm,
-                               mca_coll_base_module_t *module)
-{
-    return OMPI_ERR_NOT_IMPLEMENTED;
-}
diff --git a/ompi/mca/coll/basic/coll_basic_component.c b/ompi/mca/coll/basic/coll_basic_component.c
index a2fd0928c78..d38850744c0 100644
--- a/ompi/mca/coll/basic/coll_basic_component.c
+++ b/ompi/mca/coll/basic/coll_basic_component.c
@@ -13,6 +13,7 @@
  * Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2024      NVIDIA CORPORATION. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -55,13 +56,13 @@ static int basic_register(void);
  * and pointers to our public functions in it
  */
 
-const mca_coll_base_component_2_4_0_t mca_coll_basic_component = {
+const mca_coll_base_component_3_0_0_t mca_coll_basic_component = {
 
     /* First, the mca_component_t struct containing meta information
      * about the component itself */
 
     .collm_version = {
-        MCA_COLL_BASE_VERSION_2_4_0,
+        MCA_COLL_BASE_VERSION_3_0_0,
 
         /* Component name and version */
         .mca_component_name = "basic",
@@ -91,16 +92,16 @@ basic_register(void)
     mca_coll_basic_priority = 10;
     (void) mca_base_component_var_register(&mca_coll_basic_component.collm_version, "priority",
                                            "Priority of the basic coll component",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                            OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           MCA_BASE_VAR_SCOPE_ALL,
                                            &mca_coll_basic_priority);
     mca_coll_basic_crossover = 4;
     (void) mca_base_component_var_register(&mca_coll_basic_component.collm_version, "crossover",
                                            "Minimum number of processes in a communicator before using the logarithmic algorithms",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                            OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           MCA_BASE_VAR_SCOPE_ALL,
                                            &mca_coll_basic_crossover);
 
     return OMPI_SUCCESS;
diff --git a/ompi/mca/coll/basic/coll_basic_exscan.c b/ompi/mca/coll/basic/coll_basic_exscan.c
deleted file mode 100644
index 1c6c23dfee6..00000000000
--- a/ompi/mca/coll/basic/coll_basic_exscan.c
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2014 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2011      NVIDIA Corporation.  All rights reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include "coll_basic.h"
-
-#include <stdio.h>
-
-#include "mpi.h"
-#include "ompi/constants.h"
-#include "ompi/op/op.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/mca/coll/coll.h"
-#include "ompi/mca/coll/base/coll_tags.h"
-#include "coll_basic.h"
-
-
-/*
- *	exscan_intra
- *
- *	Function:	- basic exscan operation
- *	Accepts:	- same arguments as MPI_Exscan()
- *	Returns:	- MPI_SUCCESS or error code
- */
-int
-mca_coll_basic_exscan_intra(const void *sbuf, void *rbuf, int count,
-                            struct ompi_datatype_t *dtype,
-                            struct ompi_op_t *op,
-                            struct ompi_communicator_t *comm,
-                            mca_coll_base_module_t *module)
-{
-    return ompi_coll_base_exscan_intra_linear(sbuf, rbuf, count, dtype, op, comm, module);
-}
-
-
-/*
- *	exscan_inter
- *
- *	Function:	- basic exscan operation
- *	Accepts:	- same arguments as MPI_Exscan()
- *	Returns:	- MPI_SUCCESS or error code
- */
-int
-mca_coll_basic_exscan_inter(const void *sbuf, void *rbuf, int count,
-                            struct ompi_datatype_t *dtype,
-                            struct ompi_op_t *op,
-                            struct ompi_communicator_t *comm,
-                            mca_coll_base_module_t *module)
-{
-    return OMPI_ERR_NOT_IMPLEMENTED;
-}
diff --git a/ompi/mca/coll/basic/coll_basic_gather.c b/ompi/mca/coll/basic/coll_basic_gather.c
index 32753ad1532..1688c9653a1 100644
--- a/ompi/mca/coll/basic/coll_basic_gather.c
+++ b/ompi/mca/coll/basic/coll_basic_gather.c
@@ -38,9 +38,9 @@
  *	Returns:	- MPI_SUCCESS or error code
  */
 int
-mca_coll_basic_gather_inter(const void *sbuf, int scount,
+mca_coll_basic_gather_inter(const void *sbuf, size_t scount,
                             struct ompi_datatype_t *sdtype,
-                            void *rbuf, int rcount,
+                            void *rbuf, size_t rcount,
                             struct ompi_datatype_t *rdtype,
                             int root, struct ompi_communicator_t *comm,
                             mca_coll_base_module_t *module)
diff --git a/ompi/mca/coll/basic/coll_basic_gatherv.c b/ompi/mca/coll/basic/coll_basic_gatherv.c
index 6ea30c49afe..194c483fa64 100644
--- a/ompi/mca/coll/basic/coll_basic_gatherv.c
+++ b/ompi/mca/coll/basic/coll_basic_gatherv.c
@@ -9,8 +9,8 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2017      IBM Corporation. All rights reserved.
  * $COPYRIGHT$
  *
@@ -37,71 +37,107 @@
  *	Returns:	- MPI_SUCCESS or error code
  */
 int
-mca_coll_basic_gatherv_intra(const void *sbuf, int scount,
+mca_coll_basic_gatherv_intra(const void *sbuf, size_t scount,
                              struct ompi_datatype_t *sdtype,
-                             void *rbuf, const int *rcounts, const int *disps,
+                             void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
                              struct ompi_datatype_t *rdtype, int root,
                              struct ompi_communicator_t *comm,
-                            mca_coll_base_module_t *module)
+                             mca_coll_base_module_t *module)
 {
-    int i, rank, size, err;
+    int err, i, peer, rank, size;
     char *ptmp;
     ptrdiff_t lb, extent;
+    size_t rdsize;
 
     size = ompi_comm_size(comm);
     rank = ompi_comm_rank(comm);
 
-    /* Everyone but root sends data and returns.  Don't send anything
-       for sendcounts of 0 (even though MPI_Gatherv has a guard for 0
-       counts, this routine is used elsewhere, like the implementation
-       of allgatherv, so it's possible to get here with a scount of
-       0) */
-
-    if (rank != root) {
-        if (scount > 0) {
-            return MCA_PML_CALL(send(sbuf, scount, sdtype, root,
-                                     MCA_COLL_BASE_TAG_GATHERV,
-                                     MCA_PML_BASE_SEND_STANDARD, comm));
+    if (root == rank) {
+        /* Root receives from everyone else */
+        ompi_datatype_type_size(rdtype, &rdsize);
+        if (OPAL_UNLIKELY(0 == rdsize)) {
+            /* bozzo case */
+            return MPI_SUCCESS;
         }
-        return MPI_SUCCESS;
-    }
 
-    /* I am the root, loop receiving data. */
+        err = ompi_datatype_get_extent(rdtype, &lb, &extent);
+        if (OMPI_SUCCESS != err) {
+            return OMPI_ERROR;
+        }
 
-    err = ompi_datatype_get_extent(rdtype, &lb, &extent);
-    if (OMPI_SUCCESS != err) {
-        return OMPI_ERROR;
-    }
+        if (MPI_IN_PLACE != sbuf && (0 < scount) && (0 < ompi_count_array_get(rcounts, rank))) {
+            /* Directly copy self sbuf to rbuf */
+            err = ompi_datatype_sndrcv(sbuf, scount, sdtype,
+                                       ((char *) rbuf) + (extent * ompi_disp_array_get(disps, rank)),
+                                       ompi_count_array_get(rcounts, rank), rdtype);
+            if (MPI_SUCCESS != err) {
+                return err;
+            }
+        }
 
-    for (i = 0; i < size; ++i) {
-        ptmp = ((char *) rbuf) + (extent * disps[i]);
+        ompi_request_t **reqs;
+        int nrecv = 0, recv_iter = 0;
 
-        if (i == rank) {
-            /* simple optimization */
-            if (MPI_IN_PLACE != sbuf && (0 < scount) && (0 < rcounts[i])) {
-                err = ompi_datatype_sndrcv(sbuf, scount, sdtype,
-                                      ptmp, rcounts[i], rdtype);
+        for (i = 0; i < size; ++i) {
+            /* We directly copied the data from self */
+            if (0 < ompi_count_array_get(rcounts, i) && rank != i) {
+                ++nrecv;
             }
-        } else {
+        }
+
+        if (0 == nrecv) {
+            /* Nothing to receive */
+            return MPI_SUCCESS;
+        }
+
+        reqs = ompi_coll_base_comm_get_reqs(module->base_data, nrecv);
+
+        for (i = 1; i < size; ++i) {
+            peer = (rank + i) % size;
+            ptmp = ((char *) rbuf) + (extent * ompi_disp_array_get(disps, peer));
             /* Only receive if there is something to receive */
-            if (rcounts[i] > 0) {
-                err = MCA_PML_CALL(recv(ptmp, rcounts[i], rdtype, i,
-                                        MCA_COLL_BASE_TAG_GATHERV,
-                                        comm, MPI_STATUS_IGNORE));
+            if (0 < ompi_count_array_get(rcounts, peer)) {
+                err = MCA_PML_CALL(irecv(ptmp, ompi_count_array_get(rcounts, peer), rdtype, peer,
+                                         MCA_COLL_BASE_TAG_GATHERV, comm, &reqs[recv_iter++]));
             }
         }
 
-        if (MPI_SUCCESS != err) {
-            return err;
+        assert(nrecv == recv_iter);
+
+        err = ompi_request_wait_all(nrecv, reqs, MPI_STATUSES_IGNORE);
+
+        if (MPI_ERR_IN_STATUS == err) {
+            for (i = 0; i < nrecv; i++) {
+                if (MPI_REQUEST_NULL == reqs[i])
+                    continue;
+                if (MPI_ERR_PENDING == reqs[i]->req_status.MPI_ERROR)
+                    continue;
+                if (MPI_SUCCESS != reqs[i]->req_status.MPI_ERROR) {
+                    err = reqs[i]->req_status.MPI_ERROR;
+                    break;
+                }
+            }
         }
+
+        ompi_coll_base_free_reqs(reqs, nrecv);
+        return err;
     }
 
-    /* All done */
+    /* Everyone but root sends data and returns.  Don't send anything
+       for sendcounts of 0 (even though MPI_Gatherv has a guard for 0
+       counts, this routine is used elsewhere, like the implementation
+       of allgatherv, so it's possible to get here with a scount of
+       0) */
 
+    size_t sdsize;
+    ompi_datatype_type_size(sdtype, &sdsize);
+    if (scount > 0 && sdsize > 0) {
+        return MCA_PML_CALL(send(sbuf, scount, sdtype, root, MCA_COLL_BASE_TAG_GATHERV,
+                                 MCA_PML_BASE_SEND_STANDARD, comm));
+    }
     return MPI_SUCCESS;
 }
 
-
 /*
  *	gatherv_inter
  *
@@ -110,9 +146,9 @@ mca_coll_basic_gatherv_intra(const void *sbuf, int scount,
  *	Returns:	- MPI_SUCCESS or error code
  */
 int
-mca_coll_basic_gatherv_inter(const void *sbuf, int scount,
+mca_coll_basic_gatherv_inter(const void *sbuf, size_t scount,
                              struct ompi_datatype_t *sdtype,
-                             void *rbuf, const int *rcounts, const int *disps,
+                             void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
                              struct ompi_datatype_t *rdtype, int root,
                              struct ompi_communicator_t *comm,
                              mca_coll_base_module_t *module)
@@ -146,8 +182,8 @@ mca_coll_basic_gatherv_inter(const void *sbuf, int scount,
         if( NULL == reqs ) { return OMPI_ERR_OUT_OF_RESOURCE; }
 
         for (i = 0; i < size; ++i) {
-            ptmp = ((char *) rbuf) + (extent * disps[i]);
-            err = MCA_PML_CALL(irecv(ptmp, rcounts[i], rdtype, i,
+            ptmp = ((char *) rbuf) + (extent * ompi_disp_array_get(disps, i));
+            err = MCA_PML_CALL(irecv(ptmp, ompi_count_array_get(rcounts, i), rdtype, i,
                                      MCA_COLL_BASE_TAG_GATHERV,
                                      comm, &reqs[i]));
             if (OMPI_SUCCESS != err) {
diff --git a/ompi/mca/coll/basic/coll_basic_module.c b/ompi/mca/coll/basic/coll_basic_module.c
index e9f93bd86f4..9913a3cda26 100644
--- a/ompi/mca/coll/basic/coll_basic_module.c
+++ b/ompi/mca/coll/basic/coll_basic_module.c
@@ -17,6 +17,7 @@
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016-2017 IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -35,6 +36,13 @@
 #include "coll_basic.h"
 
 
+static int
+mca_coll_basic_module_enable(mca_coll_base_module_t *module,
+                             struct ompi_communicator_t *comm);
+static int
+mca_coll_basic_module_disable(mca_coll_base_module_t *module,
+                             struct ompi_communicator_t *comm);
+
 /*
  * Initial query function that is invoked during MPI_INIT, allowing
  * this component to disqualify itself if it doesn't support the
@@ -49,7 +57,6 @@ mca_coll_basic_init_query(bool enable_progress_threads,
     return OMPI_SUCCESS;
 }
 
-
 /*
  * Invoked when there's a new communicator that has been created.
  * Look at the communicator and decide which set of functions and
@@ -69,82 +76,19 @@ mca_coll_basic_comm_query(struct ompi_communicator_t *comm,
     /* Choose whether to use [intra|inter], and [linear|log]-based
      * algorithms. */
     basic_module->super.coll_module_enable = mca_coll_basic_module_enable;
-
-    if (OMPI_COMM_IS_INTER(comm)) {
-        basic_module->super.coll_allgather  = mca_coll_basic_allgather_inter;
-        basic_module->super.coll_allgatherv = mca_coll_basic_allgatherv_inter;
-        basic_module->super.coll_allreduce  = mca_coll_basic_allreduce_inter;
-        basic_module->super.coll_alltoall   = mca_coll_basic_alltoall_inter;
-        basic_module->super.coll_alltoallv  = mca_coll_basic_alltoallv_inter;
-        basic_module->super.coll_alltoallw  = mca_coll_basic_alltoallw_inter;
-        basic_module->super.coll_barrier    = mca_coll_basic_barrier_inter_lin;
-        basic_module->super.coll_bcast      = mca_coll_basic_bcast_lin_inter;
-        basic_module->super.coll_exscan     = NULL;
-        basic_module->super.coll_gather     = mca_coll_basic_gather_inter;
-        basic_module->super.coll_gatherv    = mca_coll_basic_gatherv_inter;
-        basic_module->super.coll_reduce     = mca_coll_basic_reduce_lin_inter;
-        basic_module->super.coll_reduce_scatter_block = mca_coll_basic_reduce_scatter_block_inter;
-        basic_module->super.coll_reduce_scatter = mca_coll_basic_reduce_scatter_inter;
-        basic_module->super.coll_scan       = NULL;
-        basic_module->super.coll_scatter    = mca_coll_basic_scatter_inter;
-        basic_module->super.coll_scatterv   = mca_coll_basic_scatterv_inter;
-    } else if (ompi_comm_size(comm) <= mca_coll_basic_crossover) {
-        basic_module->super.coll_allgather  = ompi_coll_base_allgather_intra_basic_linear;
-        basic_module->super.coll_allgatherv = ompi_coll_base_allgatherv_intra_basic_default;
-        basic_module->super.coll_allreduce  = mca_coll_basic_allreduce_intra;
-        basic_module->super.coll_alltoall   = ompi_coll_base_alltoall_intra_basic_linear;
-        basic_module->super.coll_alltoallv  = ompi_coll_base_alltoallv_intra_basic_linear;
-        basic_module->super.coll_alltoallw  = mca_coll_basic_alltoallw_intra;
-        basic_module->super.coll_barrier    = ompi_coll_base_barrier_intra_basic_linear;
-        basic_module->super.coll_bcast      = ompi_coll_base_bcast_intra_basic_linear;
-        basic_module->super.coll_exscan     = mca_coll_basic_exscan_intra;
-        basic_module->super.coll_gather     = ompi_coll_base_gather_intra_basic_linear;
-        basic_module->super.coll_gatherv    = mca_coll_basic_gatherv_intra;
-        basic_module->super.coll_reduce     = ompi_coll_base_reduce_intra_basic_linear;
-        basic_module->super.coll_reduce_scatter_block = mca_coll_basic_reduce_scatter_block_intra;
-        basic_module->super.coll_reduce_scatter = mca_coll_basic_reduce_scatter_intra;
-        basic_module->super.coll_scan       = mca_coll_basic_scan_intra;
-        basic_module->super.coll_scatter    = ompi_coll_base_scatter_intra_basic_linear;
-        basic_module->super.coll_scatterv   = mca_coll_basic_scatterv_intra;
-    } else {
-        basic_module->super.coll_allgather  = ompi_coll_base_allgather_intra_basic_linear;
-        basic_module->super.coll_allgatherv = ompi_coll_base_allgatherv_intra_basic_default;
-        basic_module->super.coll_allreduce  = mca_coll_basic_allreduce_intra;
-        basic_module->super.coll_alltoall   = ompi_coll_base_alltoall_intra_basic_linear;
-        basic_module->super.coll_alltoallv  = ompi_coll_base_alltoallv_intra_basic_linear;
-        basic_module->super.coll_alltoallw  = mca_coll_basic_alltoallw_intra;
-        basic_module->super.coll_barrier    = mca_coll_basic_barrier_intra_log;
-        basic_module->super.coll_bcast      = mca_coll_basic_bcast_log_intra;
-        basic_module->super.coll_exscan     = mca_coll_basic_exscan_intra;
-        basic_module->super.coll_gather     = ompi_coll_base_gather_intra_basic_linear;
-        basic_module->super.coll_gatherv    = mca_coll_basic_gatherv_intra;
-        basic_module->super.coll_reduce     = mca_coll_basic_reduce_log_intra;
-        basic_module->super.coll_reduce_scatter_block = mca_coll_basic_reduce_scatter_block_intra;
-        basic_module->super.coll_reduce_scatter = mca_coll_basic_reduce_scatter_intra;
-        basic_module->super.coll_scan       = mca_coll_basic_scan_intra;
-        basic_module->super.coll_scatter    = ompi_coll_base_scatter_intra_basic_linear;
-        basic_module->super.coll_scatterv   = mca_coll_basic_scatterv_intra;
-    }
-
-    /* These functions will return an error code if comm does not have a virtual topology */
-    basic_module->super.coll_neighbor_allgather = mca_coll_basic_neighbor_allgather;
-    basic_module->super.coll_neighbor_allgatherv = mca_coll_basic_neighbor_allgatherv;
-    basic_module->super.coll_neighbor_alltoall = mca_coll_basic_neighbor_alltoall;
-    basic_module->super.coll_neighbor_alltoallv = mca_coll_basic_neighbor_alltoallv;
-    basic_module->super.coll_neighbor_alltoallw = mca_coll_basic_neighbor_alltoallw;
-
-    basic_module->super.coll_reduce_local = mca_coll_base_reduce_local;
-
-#if OPAL_ENABLE_FT_MPI
-    /* Default to some shim mappings over allreduce */
-    basic_module->super.coll_agree   = ompi_coll_base_agree_noft;
-    basic_module->super.coll_iagree  = ompi_coll_base_iagree_noft;
-#endif
+    basic_module->super.coll_module_disable = mca_coll_basic_module_disable;
 
     return &(basic_module->super);
 }
 
 
+#define BASIC_INSTALL_COLL_API(__comm, __module, __api, __coll)                 \
+    do                                                                          \
+    {                                                                           \
+        (__module)->super.coll_##__api = __coll;                                \
+        MCA_COLL_INSTALL_API(__comm, __api, __coll, &__module->super, "basic"); \
+    } while (0)
+
 /*
  * Init module on the communicator
  */
@@ -152,12 +96,158 @@ int
 mca_coll_basic_module_enable(mca_coll_base_module_t *module,
                              struct ompi_communicator_t *comm)
 {
+    mca_coll_basic_module_t *basic_module = (mca_coll_basic_module_t*)module;
+
     /* prepare the placeholder for the array of request* */
     module->base_data = OBJ_NEW(mca_coll_base_comm_t);
     if (NULL == module->base_data) {
         return OMPI_ERROR;
     }
 
+    if (OMPI_COMM_IS_INTER(comm))
+    {
+        BASIC_INSTALL_COLL_API(comm, basic_module, allgather, mca_coll_basic_allgather_inter);
+        BASIC_INSTALL_COLL_API(comm, basic_module, allgatherv, mca_coll_basic_allgatherv_inter);
+        BASIC_INSTALL_COLL_API(comm, basic_module, allreduce, mca_coll_basic_allreduce_inter);
+        BASIC_INSTALL_COLL_API(comm, basic_module, alltoall, mca_coll_basic_alltoall_inter);
+        BASIC_INSTALL_COLL_API(comm, basic_module, alltoallv, mca_coll_basic_alltoallv_inter);
+        BASIC_INSTALL_COLL_API(comm, basic_module, alltoallw, mca_coll_basic_alltoallw_inter);
+        BASIC_INSTALL_COLL_API(comm, basic_module, barrier, mca_coll_basic_barrier_inter_lin);
+        BASIC_INSTALL_COLL_API(comm, basic_module, bcast, mca_coll_basic_bcast_lin_inter);
+        BASIC_INSTALL_COLL_API(comm, basic_module, gather, mca_coll_basic_gather_inter);
+        BASIC_INSTALL_COLL_API(comm, basic_module, gatherv, mca_coll_basic_gatherv_inter);
+        BASIC_INSTALL_COLL_API(comm, basic_module, reduce, mca_coll_basic_reduce_lin_inter);
+        BASIC_INSTALL_COLL_API(comm, basic_module, reduce_scatter_block, mca_coll_basic_reduce_scatter_block_inter);
+        BASIC_INSTALL_COLL_API(comm, basic_module, reduce_scatter, mca_coll_basic_reduce_scatter_inter);
+        BASIC_INSTALL_COLL_API(comm, basic_module, scatter, mca_coll_basic_scatter_inter);
+        BASIC_INSTALL_COLL_API(comm, basic_module, scatterv, mca_coll_basic_scatterv_inter);
+    }
+    else {
+        if (ompi_comm_size(comm) <= mca_coll_basic_crossover) {
+            BASIC_INSTALL_COLL_API(comm, basic_module, barrier, ompi_coll_base_barrier_intra_basic_linear);
+            BASIC_INSTALL_COLL_API(comm, basic_module, bcast, ompi_coll_base_bcast_intra_basic_linear);
+            BASIC_INSTALL_COLL_API(comm, basic_module, reduce, ompi_coll_base_reduce_intra_basic_linear);
+        } else {
+            BASIC_INSTALL_COLL_API(comm, basic_module, barrier, mca_coll_basic_barrier_intra_log);
+            BASIC_INSTALL_COLL_API(comm, basic_module, bcast, mca_coll_basic_bcast_log_intra);
+            BASIC_INSTALL_COLL_API(comm, basic_module, reduce, mca_coll_basic_reduce_log_intra);
+        }
+        BASIC_INSTALL_COLL_API(comm, basic_module, allgather, ompi_coll_base_allgather_intra_basic_linear);
+        BASIC_INSTALL_COLL_API(comm, basic_module, allgatherv, ompi_coll_base_allgatherv_intra_basic_default);
+        BASIC_INSTALL_COLL_API(comm, basic_module, allreduce, mca_coll_basic_allreduce_intra);
+        BASIC_INSTALL_COLL_API(comm, basic_module, alltoall, ompi_coll_base_alltoall_intra_basic_linear);
+        BASIC_INSTALL_COLL_API(comm, basic_module, alltoallv, ompi_coll_base_alltoallv_intra_basic_linear);
+        BASIC_INSTALL_COLL_API(comm, basic_module, alltoallw, mca_coll_basic_alltoallw_intra);
+        BASIC_INSTALL_COLL_API(comm, basic_module, exscan, ompi_coll_base_exscan_intra_linear);
+        BASIC_INSTALL_COLL_API(comm, basic_module, gather, ompi_coll_base_gather_intra_basic_linear);
+        BASIC_INSTALL_COLL_API(comm, basic_module, gatherv, mca_coll_basic_gatherv_intra);
+        BASIC_INSTALL_COLL_API(comm, basic_module, reduce_scatter_block, mca_coll_basic_reduce_scatter_block_intra);
+        BASIC_INSTALL_COLL_API(comm, basic_module, reduce_scatter, mca_coll_basic_reduce_scatter_intra);
+        BASIC_INSTALL_COLL_API(comm, basic_module, scan, ompi_coll_base_scan_intra_linear);
+        BASIC_INSTALL_COLL_API(comm, basic_module, scatter, ompi_coll_base_scatter_intra_basic_linear);
+        BASIC_INSTALL_COLL_API(comm, basic_module, scatterv, mca_coll_basic_scatterv_intra);
+    }
+    /* These functions will return an error code if comm does not have a virtual topology */
+    BASIC_INSTALL_COLL_API(comm, basic_module, neighbor_allgather, mca_coll_basic_neighbor_allgather);
+    BASIC_INSTALL_COLL_API(comm, basic_module, neighbor_allgatherv, mca_coll_basic_neighbor_allgatherv);
+    BASIC_INSTALL_COLL_API(comm, basic_module, neighbor_alltoall, mca_coll_basic_neighbor_alltoall);
+    BASIC_INSTALL_COLL_API(comm, basic_module, neighbor_alltoallv, mca_coll_basic_neighbor_alltoallv);
+    BASIC_INSTALL_COLL_API(comm, basic_module, neighbor_alltoallw, mca_coll_basic_neighbor_alltoallw);
+
+    /* Default to some shim mappings over allreduce */
+    BASIC_INSTALL_COLL_API(comm, basic_module, agree, ompi_coll_base_agree_noft);
+    BASIC_INSTALL_COLL_API(comm, basic_module, iagree, ompi_coll_base_iagree_noft);
+
+    BASIC_INSTALL_COLL_API(comm, basic_module, reduce_local, mca_coll_base_reduce_local);
+
+    /* All done */
+    return OMPI_SUCCESS;
+}
+
+#define BASIC_UNINSTALL_COLL_API(__comm, __module, __api)                 \
+    do                                                                    \
+    {                                                                     \
+        if (__comm->c_coll->coll_##__api##_module == &__module->super ) { \
+            MCA_COLL_INSTALL_API(__comm, __api, NULL, NULL, "basic");     \
+        }                                                                 \
+    } while (0)
+    
+int
+mca_coll_basic_module_disable(mca_coll_base_module_t *module,
+                              struct ompi_communicator_t *comm)
+{
+    mca_coll_basic_module_t *basic_module = (mca_coll_basic_module_t*)module;
+
+    if (OMPI_COMM_IS_INTER(comm))
+    {
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, allgather);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, allgatherv);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, allreduce);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, alltoall);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, alltoallv);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, alltoallw);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, barrier);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, bcast);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, gather);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, gatherv);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, reduce);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, reduce_scatter_block);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, reduce_scatter);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, scatter);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, scatterv);
+    }
+    else if (ompi_comm_size(comm) <= mca_coll_basic_crossover)
+    {
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, allgather);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, allgatherv);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, allreduce);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, alltoall);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, alltoallv);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, alltoallw);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, barrier);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, bcast);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, exscan);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, gather);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, gatherv);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, reduce);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, reduce_scatter_block);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, reduce_scatter);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, scan);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, scatter);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, scatterv);
+    }
+    else
+    {
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, allgather);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, allgatherv);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, allreduce);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, alltoall);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, alltoallv);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, alltoallw);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, barrier);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, bcast);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, exscan);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, gather);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, gatherv);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, reduce);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, reduce_scatter_block);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, reduce_scatter);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, scan);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, scatter);
+        BASIC_UNINSTALL_COLL_API(comm, basic_module, scatterv);
+    }
+
+    /* These functions will return an error code if comm does not have a virtual topology */
+    BASIC_UNINSTALL_COLL_API(comm, basic_module, neighbor_allgather);
+    BASIC_UNINSTALL_COLL_API(comm, basic_module, neighbor_allgatherv);
+    BASIC_UNINSTALL_COLL_API(comm, basic_module, neighbor_alltoall);
+    BASIC_UNINSTALL_COLL_API(comm, basic_module, neighbor_alltoallv);
+    BASIC_UNINSTALL_COLL_API(comm, basic_module, neighbor_alltoallw);
+
+    BASIC_UNINSTALL_COLL_API(comm, basic_module, agree);
+    BASIC_UNINSTALL_COLL_API(comm, basic_module, iagree);
+
+    BASIC_UNINSTALL_COLL_API(comm, basic_module, reduce_local);
     /* All done */
     return OMPI_SUCCESS;
 }
diff --git a/ompi/mca/coll/basic/coll_basic_neighbor_allgather.c b/ompi/mca/coll/basic/coll_basic_neighbor_allgather.c
index 9bb1e360fe3..e3c164be2e3 100644
--- a/ompi/mca/coll/basic/coll_basic_neighbor_allgather.c
+++ b/ompi/mca/coll/basic/coll_basic_neighbor_allgather.c
@@ -37,9 +37,9 @@
 #include "ompi/mca/topo/base/base.h"
 
 static int
-mca_coll_basic_neighbor_allgather_cart(const void *sbuf, int scount,
+mca_coll_basic_neighbor_allgather_cart(const void *sbuf, size_t scount,
                                        struct ompi_datatype_t *sdtype, void *rbuf,
-                                       int rcount, struct ompi_datatype_t *rdtype,
+                                       size_t rcount, struct ompi_datatype_t *rdtype,
                                        struct ompi_communicator_t *comm,
                                        mca_coll_base_module_t *module)
 {
@@ -117,9 +117,9 @@ mca_coll_basic_neighbor_allgather_cart(const void *sbuf, int scount,
 }
 
 static int
-mca_coll_basic_neighbor_allgather_graph(const void *sbuf, int scount,
+mca_coll_basic_neighbor_allgather_graph(const void *sbuf, size_t scount,
                                         struct ompi_datatype_t *sdtype, void *rbuf,
-                                        int rcount, struct ompi_datatype_t *rdtype,
+                                        size_t rcount, struct ompi_datatype_t *rdtype,
                                         struct ompi_communicator_t *comm,
                                         mca_coll_base_module_t *module)
 {
@@ -170,9 +170,9 @@ mca_coll_basic_neighbor_allgather_graph(const void *sbuf, int scount,
 }
 
 static int
-mca_coll_basic_neighbor_allgather_dist_graph(const void *sbuf, int scount,
+mca_coll_basic_neighbor_allgather_dist_graph(const void *sbuf, size_t scount,
                                              struct ompi_datatype_t *sdtype, void *rbuf,
-                                             int rcount, struct ompi_datatype_t *rdtype,
+                                             size_t rcount, struct ompi_datatype_t *rdtype,
                                              struct ompi_communicator_t *comm,
                                              mca_coll_base_module_t *module)
 {
@@ -229,9 +229,9 @@ mca_coll_basic_neighbor_allgather_dist_graph(const void *sbuf, int scount,
     return rc;
 }
 
-int mca_coll_basic_neighbor_allgather(const void *sbuf, int scount,
+int mca_coll_basic_neighbor_allgather(const void *sbuf, size_t scount,
                                       struct ompi_datatype_t *sdtype, void *rbuf,
-                                      int rcount, struct ompi_datatype_t *rdtype,
+                                      size_t rcount, struct ompi_datatype_t *rdtype,
                                       struct ompi_communicator_t *comm,
                                       mca_coll_base_module_t *module)
 {
diff --git a/ompi/mca/coll/basic/coll_basic_neighbor_allgatherv.c b/ompi/mca/coll/basic/coll_basic_neighbor_allgatherv.c
index 37d97970a30..8261d1de3e8 100644
--- a/ompi/mca/coll/basic/coll_basic_neighbor_allgatherv.c
+++ b/ompi/mca/coll/basic/coll_basic_neighbor_allgatherv.c
@@ -37,8 +37,8 @@
 #include "ompi/mca/topo/base/base.h"
 
 static int
-mca_coll_basic_neighbor_allgatherv_cart(const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-                                        void *rbuf, const int rcounts[], const int disps[],
+mca_coll_basic_neighbor_allgatherv_cart(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                                        void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
                                         struct ompi_datatype_t *rdtype, struct ompi_communicator_t *comm,
                                         mca_coll_base_module_t *module)
 {
@@ -68,7 +68,8 @@ mca_coll_basic_neighbor_allgatherv_cart(const void *sbuf, int scount, struct omp
 
         if (MPI_PROC_NULL != srank) {
             nreqs++;
-            rc = MCA_PML_CALL(irecv((char *) rbuf + disps[i] * extent, rcounts[i], rdtype, srank,
+            rc = MCA_PML_CALL(irecv((char *) rbuf + ompi_disp_array_get(disps, i) * extent,
+                                    ompi_count_array_get(rcounts, i), rdtype, srank,
                                     MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim, comm, preqs++));
             if (OMPI_SUCCESS != rc) break;
 
@@ -83,7 +84,8 @@ mca_coll_basic_neighbor_allgatherv_cart(const void *sbuf, int scount, struct omp
 
         if (MPI_PROC_NULL != drank) {
             nreqs++;
-            rc = MCA_PML_CALL(irecv((char *) rbuf + disps[i+1] * extent, rcounts[i+1], rdtype, drank,
+            rc = MCA_PML_CALL(irecv((char *) rbuf + ompi_disp_array_get(disps, i+1) * extent,
+                                    ompi_count_array_get(rcounts, i+1), rdtype, drank,
                                     MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1, comm, preqs++));
             if (OMPI_SUCCESS != rc) break;
 
@@ -108,8 +110,8 @@ mca_coll_basic_neighbor_allgatherv_cart(const void *sbuf, int scount, struct omp
 }
 
 static int
-mca_coll_basic_neighbor_allgatherv_graph(const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-                                         void *rbuf, const int rcounts[], const int disps[],
+mca_coll_basic_neighbor_allgatherv_graph(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                                         void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
                                          struct ompi_datatype_t *rdtype, struct ompi_communicator_t *comm,
                                          mca_coll_base_module_t *module)
 {
@@ -133,7 +135,7 @@ mca_coll_basic_neighbor_allgatherv_graph(const void *sbuf, int scount, struct om
     if( NULL == reqs ) { return OMPI_ERR_OUT_OF_RESOURCE; }
 
     for (neighbor = 0; neighbor < degree ; ++neighbor) {
-        rc = MCA_PML_CALL(irecv((char *) rbuf + disps[neighbor] * extent, rcounts[neighbor],
+        rc = MCA_PML_CALL(irecv((char *) rbuf + ompi_disp_array_get(disps, neighbor) * extent, ompi_count_array_get(rcounts, neighbor),
                                 rdtype, edges[neighbor], MCA_COLL_BASE_TAG_ALLGATHER, comm, preqs++));
         if (OMPI_SUCCESS != rc) break;
 
@@ -158,8 +160,8 @@ mca_coll_basic_neighbor_allgatherv_graph(const void *sbuf, int scount, struct om
 }
 
 static int
-mca_coll_basic_neighbor_allgatherv_dist_graph(const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-                                              void *rbuf, const int rcounts[], const int disps[],
+mca_coll_basic_neighbor_allgatherv_dist_graph(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                                              void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
                                               struct ompi_datatype_t *rdtype, struct ompi_communicator_t *comm,
                                               mca_coll_base_module_t *module)
 {
@@ -182,7 +184,7 @@ mca_coll_basic_neighbor_allgatherv_dist_graph(const void *sbuf, int scount, stru
     if( NULL == reqs ) { return OMPI_ERR_OUT_OF_RESOURCE; }
 
     for (neighbor = 0; neighbor < indegree ; ++neighbor) {
-        rc = MCA_PML_CALL(irecv((char *) rbuf + disps[neighbor] * extent, rcounts[neighbor], rdtype,
+        rc = MCA_PML_CALL(irecv((char *) rbuf + ompi_disp_array_get(disps, neighbor) * extent, ompi_count_array_get(rcounts, neighbor), rdtype,
                                 inedges[neighbor], MCA_COLL_BASE_TAG_ALLGATHER, comm, preqs++));
         if (OMPI_SUCCESS != rc) break;
     }
@@ -213,8 +215,9 @@ mca_coll_basic_neighbor_allgatherv_dist_graph(const void *sbuf, int scount, stru
     return rc;
 }
 
-int mca_coll_basic_neighbor_allgatherv(const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-                                       void *rbuf, const int rcounts[], const int disps[], struct ompi_datatype_t *rdtype,
+int mca_coll_basic_neighbor_allgatherv(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                                       void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
+                                       struct ompi_datatype_t *rdtype,
                                        struct ompi_communicator_t *comm, mca_coll_base_module_t *module)
 {
     if (OMPI_COMM_IS_INTER(comm)) {
diff --git a/ompi/mca/coll/basic/coll_basic_neighbor_alltoall.c b/ompi/mca/coll/basic/coll_basic_neighbor_alltoall.c
index 6d6468174ff..378d3181820 100644
--- a/ompi/mca/coll/basic/coll_basic_neighbor_alltoall.c
+++ b/ompi/mca/coll/basic/coll_basic_neighbor_alltoall.c
@@ -42,8 +42,8 @@
  * 512 dimensions.
  */
 static int
-mca_coll_basic_neighbor_alltoall_cart(const void *sbuf, int scount, struct ompi_datatype_t *sdtype, void *rbuf,
-                                      int rcount, struct ompi_datatype_t *rdtype, struct ompi_communicator_t *comm,
+mca_coll_basic_neighbor_alltoall_cart(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype, void *rbuf,
+                                      size_t rcount, struct ompi_datatype_t *rdtype, struct ompi_communicator_t *comm,
                                       mca_coll_base_module_t *module)
 {
     const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
@@ -142,8 +142,8 @@ mca_coll_basic_neighbor_alltoall_cart(const void *sbuf, int scount, struct ompi_
 }
 
 static int
-mca_coll_basic_neighbor_alltoall_graph(const void *sbuf, int scount, struct ompi_datatype_t *sdtype, void *rbuf,
-                                       int rcount, struct ompi_datatype_t *rdtype, struct ompi_communicator_t *comm,
+mca_coll_basic_neighbor_alltoall_graph(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype, void *rbuf,
+                                       size_t rcount, struct ompi_datatype_t *rdtype, struct ompi_communicator_t *comm,
                                        mca_coll_base_module_t *module)
 {
     const mca_topo_base_comm_graph_2_2_0_t *graph = comm->c_topo->mtc.graph;
@@ -201,8 +201,8 @@ mca_coll_basic_neighbor_alltoall_graph(const void *sbuf, int scount, struct ompi
 }
 
 static int
-mca_coll_basic_neighbor_alltoall_dist_graph(const void *sbuf, int scount,struct ompi_datatype_t *sdtype, void *rbuf,
-                                            int rcount, struct ompi_datatype_t *rdtype, struct ompi_communicator_t *comm,
+mca_coll_basic_neighbor_alltoall_dist_graph(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype, void *rbuf,
+                                            size_t rcount, struct ompi_datatype_t *rdtype, struct ompi_communicator_t *comm,
                                             mca_coll_base_module_t *module)
 {
     const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
@@ -259,8 +259,8 @@ mca_coll_basic_neighbor_alltoall_dist_graph(const void *sbuf, int scount,struct
     return rc;
 }
 
-int mca_coll_basic_neighbor_alltoall(const void *sbuf, int scount, struct ompi_datatype_t *sdtype, void *rbuf,
-                                     int rcount, struct ompi_datatype_t *rdtype, struct ompi_communicator_t *comm,
+int mca_coll_basic_neighbor_alltoall(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype, void *rbuf,
+                                     size_t rcount, struct ompi_datatype_t *rdtype, struct ompi_communicator_t *comm,
                                      mca_coll_base_module_t *module)
 {
     if (OMPI_COMM_IS_INTER(comm)) {
diff --git a/ompi/mca/coll/basic/coll_basic_neighbor_alltoallv.c b/ompi/mca/coll/basic/coll_basic_neighbor_alltoallv.c
index e963dc25412..9fd155d6393 100644
--- a/ompi/mca/coll/basic/coll_basic_neighbor_alltoallv.c
+++ b/ompi/mca/coll/basic/coll_basic_neighbor_alltoallv.c
@@ -37,9 +37,9 @@
 #include "ompi/mca/topo/base/base.h"
 
 static int
-mca_coll_basic_neighbor_alltoallv_cart(const void *sbuf, const int scounts[], const int sdisps[],
-                                       struct ompi_datatype_t *sdtype, void *rbuf, const int rcounts[],
-                                       const int rdisps[], struct ompi_datatype_t *rdtype,
+mca_coll_basic_neighbor_alltoallv_cart(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
+                                       struct ompi_datatype_t *sdtype, void *rbuf, ompi_count_array_t rcounts,
+                                       ompi_disp_array_t rdisps, struct ompi_datatype_t *rdtype,
                                        struct ompi_communicator_t *comm, mca_coll_base_module_t *module)
 {
     const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
@@ -67,14 +67,16 @@ mca_coll_basic_neighbor_alltoallv_cart(const void *sbuf, const int scounts[], co
 
         if (MPI_PROC_NULL != srank) {
             nreqs++;
-            rc = MCA_PML_CALL(irecv((char *) rbuf + rdisps[i] * rdextent, rcounts[i], rdtype, srank,
+            rc = MCA_PML_CALL(irecv((char *) rbuf + ompi_disp_array_get(rdisps, i) * rdextent,
+                                    ompi_count_array_get(rcounts, i), rdtype, srank,
                                     MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim, comm, preqs++));
             if (OMPI_SUCCESS != rc) break;
         }
 
         if (MPI_PROC_NULL != drank) {
             nreqs++;
-            rc = MCA_PML_CALL(irecv((char *) rbuf + rdisps[i+1] * rdextent, rcounts[i+1], rdtype, drank,
+            rc = MCA_PML_CALL(irecv((char *) rbuf + ompi_disp_array_get(rdisps, i+1) * rdextent,
+                                    ompi_count_array_get(rcounts, i+1), rdtype, drank,
                                     MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1, comm, preqs++));
             if (OMPI_SUCCESS != rc) break;
         }
@@ -97,14 +99,16 @@ mca_coll_basic_neighbor_alltoallv_cart(const void *sbuf, const int scounts[], co
         if (MPI_PROC_NULL != srank) {
             nreqs++;
             /* remove cast from const when the pml layer is updated to take a const for the send buffer */
-            rc = MCA_PML_CALL(isend((char *) sbuf + sdisps[i] * sdextent, scounts[i], sdtype, srank,
+            rc = MCA_PML_CALL(isend((char *) sbuf + ompi_disp_array_get(sdisps, i) * sdextent,
+                                    ompi_count_array_get(scounts, i), sdtype, srank,
                                     MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
             if (OMPI_SUCCESS != rc) break;
         }
 
         if (MPI_PROC_NULL != drank) {
             nreqs++;
-            rc = MCA_PML_CALL(isend((char *) sbuf + sdisps[i+1] * sdextent, scounts[i+1], sdtype, drank,
+            rc = MCA_PML_CALL(isend((char *) sbuf + ompi_disp_array_get(sdisps, i+1) * sdextent,
+                                    ompi_count_array_get(scounts, i+1), sdtype, drank,
                                     MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
             if (OMPI_SUCCESS != rc) break;
         }
@@ -123,9 +127,9 @@ mca_coll_basic_neighbor_alltoallv_cart(const void *sbuf, const int scounts[], co
 }
 
 static int
-mca_coll_basic_neighbor_alltoallv_graph(const void *sbuf, const int scounts[], const int sdisps[],
-                                        struct ompi_datatype_t *sdtype, void *rbuf, const int rcounts[],
-                                        const int rdisps[], struct ompi_datatype_t *rdtype,
+mca_coll_basic_neighbor_alltoallv_graph(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
+                                        struct ompi_datatype_t *sdtype, void *rbuf, ompi_count_array_t rcounts,
+                                        ompi_disp_array_t rdisps, struct ompi_datatype_t *rdtype,
                                         struct ompi_communicator_t *comm, mca_coll_base_module_t *module)
 {
     const mca_topo_base_comm_graph_2_2_0_t *graph = comm->c_topo->mtc.graph;
@@ -150,7 +154,8 @@ mca_coll_basic_neighbor_alltoallv_graph(const void *sbuf, const int scounts[], c
 
     /* post all receives first */
     for (neighbor = 0; neighbor < degree ; ++neighbor) {
-        rc = MCA_PML_CALL(irecv((char *) rbuf + rdisps[neighbor] * rdextent, rcounts[neighbor], rdtype,
+        rc = MCA_PML_CALL(irecv((char *) rbuf + ompi_disp_array_get(rdisps, neighbor) * rdextent,
+                                ompi_count_array_get(rcounts, neighbor), rdtype,
                                 edges[neighbor], MCA_COLL_BASE_TAG_ALLTOALL, comm, preqs++));
         if (OMPI_SUCCESS != rc) break;
     }
@@ -162,7 +167,8 @@ mca_coll_basic_neighbor_alltoallv_graph(const void *sbuf, const int scounts[], c
 
     for (neighbor = 0 ; neighbor < degree ; ++neighbor) {
         /* remove cast from const when the pml layer is updated to take a const for the send buffer */
-        rc = MCA_PML_CALL(isend((char *) sbuf + sdisps[neighbor] * sdextent, scounts[neighbor], sdtype,
+        rc = MCA_PML_CALL(isend((char *) sbuf + ompi_disp_array_get(sdisps, neighbor) * sdextent,
+                                ompi_count_array_get(scounts, neighbor), sdtype,
                                 edges[neighbor], MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD,
                                 comm, preqs++));
         if (OMPI_SUCCESS != rc) break;
@@ -181,9 +187,9 @@ mca_coll_basic_neighbor_alltoallv_graph(const void *sbuf, const int scounts[], c
 }
 
 static int
-mca_coll_basic_neighbor_alltoallv_dist_graph(const void *sbuf, const int scounts[], const int sdisps[],
-                                             struct ompi_datatype_t *sdtype, void *rbuf, const int rcounts[],
-                                             const int rdisps[], struct ompi_datatype_t *rdtype,
+mca_coll_basic_neighbor_alltoallv_dist_graph(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
+                                             struct ompi_datatype_t *sdtype, void *rbuf, ompi_count_array_t rcounts,
+                                             ompi_disp_array_t rdisps, struct ompi_datatype_t *rdtype,
                                              struct ompi_communicator_t *comm, mca_coll_base_module_t *module)
 {
     const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
@@ -207,7 +213,8 @@ mca_coll_basic_neighbor_alltoallv_dist_graph(const void *sbuf, const int scounts
 
     /* post all receives first */
     for (neighbor = 0; neighbor < indegree ; ++neighbor) {
-        rc = MCA_PML_CALL(irecv((char *) rbuf + rdisps[neighbor] * rdextent, rcounts[neighbor], rdtype,
+        rc = MCA_PML_CALL(irecv((char *) rbuf + ompi_disp_array_get(rdisps, neighbor) * rdextent,
+                                ompi_count_array_get(rcounts, neighbor), rdtype,
                                 inedges[neighbor], MCA_COLL_BASE_TAG_ALLTOALL, comm, preqs++));
         if (OMPI_SUCCESS != rc) break;
     }
@@ -219,7 +226,8 @@ mca_coll_basic_neighbor_alltoallv_dist_graph(const void *sbuf, const int scounts
 
     for (neighbor = 0 ; neighbor < outdegree ; ++neighbor) {
         /* remove cast from const when the pml layer is updated to take a const for the send buffer */
-        rc = MCA_PML_CALL(isend((char *) sbuf + sdisps[neighbor] * sdextent, scounts[neighbor], sdtype,
+        rc = MCA_PML_CALL(isend((char *) sbuf + ompi_disp_array_get(sdisps, neighbor) * sdextent,
+                                ompi_count_array_get(scounts, neighbor), sdtype,
                                 outedges[neighbor], MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD,
                                 comm, preqs++));
         if (OMPI_SUCCESS != rc) break;
@@ -237,9 +245,9 @@ mca_coll_basic_neighbor_alltoallv_dist_graph(const void *sbuf, const int scounts
     return rc;
 }
 
-int mca_coll_basic_neighbor_alltoallv(const void *sbuf, const int scounts[], const int sdisps[],
-                                      struct ompi_datatype_t *sdtype, void *rbuf, const int rcounts[],
-                                      const int rdisps[], struct ompi_datatype_t *rdtype,
+int mca_coll_basic_neighbor_alltoallv(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
+                                      struct ompi_datatype_t *sdtype, void *rbuf, ompi_count_array_t rcounts,
+                                      ompi_disp_array_t rdisps, struct ompi_datatype_t *rdtype,
                                       struct ompi_communicator_t *comm, mca_coll_base_module_t *module)
 {
     if (OMPI_COMM_IS_INTER(comm)) {
diff --git a/ompi/mca/coll/basic/coll_basic_neighbor_alltoallw.c b/ompi/mca/coll/basic/coll_basic_neighbor_alltoallw.c
index 29abb456252..d652cd35dd5 100644
--- a/ompi/mca/coll/basic/coll_basic_neighbor_alltoallw.c
+++ b/ompi/mca/coll/basic/coll_basic_neighbor_alltoallw.c
@@ -37,9 +37,9 @@
 #include "ompi/mca/topo/base/base.h"
 
 static int
-mca_coll_basic_neighbor_alltoallw_cart(const void *sbuf, const int scounts[], const MPI_Aint sdisps[],
-                                       struct ompi_datatype_t * const *sdtypes, void *rbuf, const int rcounts[],
-                                       const MPI_Aint rdisps[], struct ompi_datatype_t * const *rdtypes,
+mca_coll_basic_neighbor_alltoallw_cart(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
+                                       struct ompi_datatype_t * const *sdtypes, void *rbuf, ompi_count_array_t rcounts,
+                                       ompi_disp_array_t rdisps, struct ompi_datatype_t * const *rdtypes,
                                        struct ompi_communicator_t *comm, mca_coll_base_module_t *module)
 {
     const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
@@ -64,14 +64,16 @@ mca_coll_basic_neighbor_alltoallw_cart(const void *sbuf, const int scounts[], co
 
         if (MPI_PROC_NULL != srank) {
             nreqs++;
-            rc = MCA_PML_CALL(irecv((char *) rbuf + rdisps[i], rcounts[i], rdtypes[i], srank,
+            rc = MCA_PML_CALL(irecv((char *) rbuf + ompi_disp_array_get(rdisps, i),
+                                    ompi_count_array_get(rcounts, i), rdtypes[i], srank,
                                     MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim, comm, preqs++));
             if (OMPI_SUCCESS != rc) break;
         }
 
         if (MPI_PROC_NULL != drank) {
             nreqs++;
-            rc = MCA_PML_CALL(irecv((char *) rbuf + rdisps[i+1], rcounts[i+1], rdtypes[i+1], drank,
+            rc = MCA_PML_CALL(irecv((char *) rbuf + ompi_disp_array_get(rdisps, i+1),
+                                    ompi_count_array_get(rcounts, i+1), rdtypes[i+1], drank,
                                     MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1, comm, preqs++));
             if (OMPI_SUCCESS != rc) break;
         }
@@ -94,14 +96,16 @@ mca_coll_basic_neighbor_alltoallw_cart(const void *sbuf, const int scounts[], co
         if (MPI_PROC_NULL != srank) {
             nreqs++;
             /* remove cast from const when the pml layer is updated to take a const for the send buffer */
-            rc = MCA_PML_CALL(isend((char *) sbuf + sdisps[i], scounts[i], sdtypes[i], srank,
+            rc = MCA_PML_CALL(isend((char *) sbuf + ompi_disp_array_get(sdisps, i),
+                                    ompi_count_array_get(scounts, i), sdtypes[i], srank,
                                     MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
             if (OMPI_SUCCESS != rc) break;
         }
 
         if (MPI_PROC_NULL != drank) {
             nreqs++;
-            rc = MCA_PML_CALL(isend((char *) sbuf + sdisps[i+1], scounts[i+1], sdtypes[i+1], drank,
+            rc = MCA_PML_CALL(isend((char *) sbuf + ompi_disp_array_get(sdisps, i+1),
+                                    ompi_count_array_get(scounts, i+1), sdtypes[i+1], drank,
                                     MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
             if (OMPI_SUCCESS != rc) break;
         }
@@ -120,9 +124,9 @@ mca_coll_basic_neighbor_alltoallw_cart(const void *sbuf, const int scounts[], co
 }
 
 static int
-mca_coll_basic_neighbor_alltoallw_graph(const void *sbuf, const int scounts[], const MPI_Aint sdisps[],
-                                        struct ompi_datatype_t * const sdtypes[], void *rbuf, const int rcounts[],
-                                        const MPI_Aint rdisps[], struct ompi_datatype_t * const rdtypes[],
+mca_coll_basic_neighbor_alltoallw_graph(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
+                                        struct ompi_datatype_t * const sdtypes[], void *rbuf, ompi_count_array_t rcounts,
+                                        ompi_disp_array_t rdisps, struct ompi_datatype_t * const rdtypes[],
                                         struct ompi_communicator_t *comm, mca_coll_base_module_t *module)
 {
     const mca_topo_base_comm_graph_2_2_0_t *graph = comm->c_topo->mtc.graph;
@@ -144,7 +148,8 @@ mca_coll_basic_neighbor_alltoallw_graph(const void *sbuf, const int scounts[], c
 
     /* post all receives first */
     for (neighbor = 0; neighbor < degree ; ++neighbor) {
-        rc = MCA_PML_CALL(irecv((char *) rbuf + rdisps[neighbor], rcounts[neighbor], rdtypes[neighbor],
+        rc = MCA_PML_CALL(irecv((char *) rbuf + ompi_disp_array_get(rdisps, neighbor),
+                                ompi_count_array_get(rcounts, neighbor), rdtypes[neighbor],
                                 edges[neighbor], MCA_COLL_BASE_TAG_ALLTOALL, comm, preqs++));
         if (OMPI_SUCCESS != rc) break;
     }
@@ -156,7 +161,8 @@ mca_coll_basic_neighbor_alltoallw_graph(const void *sbuf, const int scounts[], c
 
     for (neighbor = 0 ; neighbor < degree ; ++neighbor) {
         /* remove cast from const when the pml layer is updated to take a const for the send buffer */
-        rc = MCA_PML_CALL(isend((char *) sbuf + sdisps[neighbor], scounts[neighbor], sdtypes[neighbor],
+        rc = MCA_PML_CALL(isend((char *) sbuf + ompi_disp_array_get(sdisps, neighbor),
+                                ompi_count_array_get(scounts, neighbor), sdtypes[neighbor],
                                 edges[neighbor], MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD,
                                 comm, preqs++));
         if (OMPI_SUCCESS != rc) break;
@@ -175,9 +181,9 @@ mca_coll_basic_neighbor_alltoallw_graph(const void *sbuf, const int scounts[], c
 }
 
 static int
-mca_coll_basic_neighbor_alltoallw_dist_graph(const void *sbuf, const int scounts[], const MPI_Aint sdisps[],
-                                             struct ompi_datatype_t * const *sdtypes, void *rbuf, const int rcounts[],
-                                             const MPI_Aint rdisps[], struct ompi_datatype_t * const *rdtypes,
+mca_coll_basic_neighbor_alltoallw_dist_graph(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
+                                             struct ompi_datatype_t * const *sdtypes, void *rbuf, ompi_count_array_t rcounts,
+                                             ompi_disp_array_t rdisps, struct ompi_datatype_t * const *rdtypes,
                                              struct ompi_communicator_t *comm, mca_coll_base_module_t *module)
 {
     const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
@@ -200,7 +206,8 @@ mca_coll_basic_neighbor_alltoallw_dist_graph(const void *sbuf, const int scounts
 
     /* post all receives first */
     for (neighbor = 0; neighbor < indegree ; ++neighbor) {
-        rc = MCA_PML_CALL(irecv((char *) rbuf + rdisps[neighbor], rcounts[neighbor], rdtypes[neighbor],
+        rc = MCA_PML_CALL(irecv((char *) rbuf + ompi_disp_array_get(rdisps, neighbor),
+                                ompi_count_array_get(rcounts, neighbor), rdtypes[neighbor],
                                 inedges[neighbor], MCA_COLL_BASE_TAG_ALLTOALL, comm, preqs++));
         if (OMPI_SUCCESS != rc) break;
     }
@@ -212,7 +219,8 @@ mca_coll_basic_neighbor_alltoallw_dist_graph(const void *sbuf, const int scounts
 
     for (neighbor = 0 ; neighbor < outdegree ; ++neighbor) {
         /* remove cast from const when the pml layer is updated to take a const for the send buffer */
-        rc = MCA_PML_CALL(isend((char *) sbuf + sdisps[neighbor], scounts[neighbor], sdtypes[neighbor],
+        rc = MCA_PML_CALL(isend((char *) sbuf + ompi_disp_array_get(sdisps, neighbor),
+                                ompi_count_array_get(scounts, neighbor), sdtypes[neighbor],
                                 outedges[neighbor], MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD,
                                 comm, preqs++));
         if (OMPI_SUCCESS != rc) break;
@@ -230,9 +238,9 @@ mca_coll_basic_neighbor_alltoallw_dist_graph(const void *sbuf, const int scounts
     return rc;
 }
 
-int mca_coll_basic_neighbor_alltoallw(const void *sbuf, const int scounts[], const MPI_Aint sdisps[],
-                                      struct ompi_datatype_t * const *sdtypes, void *rbuf, const int rcounts[],
-                                      const MPI_Aint rdisps[], struct ompi_datatype_t * const *rdtypes,
+int mca_coll_basic_neighbor_alltoallw(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
+                                      struct ompi_datatype_t * const *sdtypes, void *rbuf, ompi_count_array_t rcounts,
+                                      ompi_disp_array_t rdisps, struct ompi_datatype_t * const *rdtypes,
                                       struct ompi_communicator_t *comm, mca_coll_base_module_t *module)
 {
     if (OMPI_COMM_IS_INTER(comm)) {
diff --git a/ompi/mca/coll/basic/coll_basic_reduce.c b/ompi/mca/coll/basic/coll_basic_reduce.c
index ad2fd1e6f3e..c278d5a2a1e 100644
--- a/ompi/mca/coll/basic/coll_basic_reduce.c
+++ b/ompi/mca/coll/basic/coll_basic_reduce.c
@@ -11,6 +11,8 @@
  *                         All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -84,7 +86,7 @@
  *
  */
 int
-mca_coll_basic_reduce_log_intra(const void *sbuf, void *rbuf, int count,
+mca_coll_basic_reduce_log_intra(const void *sbuf, void *rbuf, size_t count,
                                 struct ompi_datatype_t *dtype,
                                 struct ompi_op_t *op,
                                 int root, struct ompi_communicator_t *comm,
@@ -100,9 +102,9 @@ mca_coll_basic_reduce_log_intra(const void *sbuf, void *rbuf, int count,
     char *rcv_buffer = (char*)rbuf;
     char *inplace_temp = NULL;
 
-    /* JMS Codearound for now -- if the operations is not communative,
+    /* JMS Code around for now -- if the operations is not commutative,
      * just call the linear algorithm.  Need to talk to Edgar / George
-     * about fixing this algorithm here to work with non-communative
+     * about fixing this algorithm here to work with non-commutative
      * operations. */
 
     if (!ompi_op_is_commute(op)) {
@@ -145,6 +147,9 @@ mca_coll_basic_reduce_log_intra(const void *sbuf, void *rbuf, int count,
         }
         sbuf = inplace_temp - gap;
         err = ompi_datatype_copy_content_same_ddt(dtype, count, (char*)sbuf, (char*)rbuf);
+        if (MPI_SUCCESS != err) {
+            goto cleanup_and_return;
+        }
     }
     snd_buffer = (char*)sbuf;
 
@@ -281,7 +286,7 @@ mca_coll_basic_reduce_log_intra(const void *sbuf, void *rbuf, int count,
  *	Returns:	- MPI_SUCCESS or error code
  */
 int
-mca_coll_basic_reduce_lin_inter(const void *sbuf, void *rbuf, int count,
+mca_coll_basic_reduce_lin_inter(const void *sbuf, void *rbuf, size_t count,
                                 struct ompi_datatype_t *dtype,
                                 struct ompi_op_t *op,
                                 int root, struct ompi_communicator_t *comm,
@@ -349,21 +354,3 @@ mca_coll_basic_reduce_lin_inter(const void *sbuf, void *rbuf, int count,
     /* All done */
     return err;
 }
-
-
-/*
- *	reduce_log_inter
- *
- *	Function:	- reduction using O(N) algorithm
- *	Accepts:	- same as MPI_Reduce()
- *	Returns:	- MPI_SUCCESS or error code
- */
-int
-mca_coll_basic_reduce_log_inter(const void *sbuf, void *rbuf, int count,
-                                struct ompi_datatype_t *dtype,
-                                struct ompi_op_t *op,
-                                int root, struct ompi_communicator_t *comm,
-                                mca_coll_base_module_t *module)
-{
-    return OMPI_ERR_NOT_IMPLEMENTED;
-}
diff --git a/ompi/mca/coll/basic/coll_basic_reduce_scatter.c b/ompi/mca/coll/basic/coll_basic_reduce_scatter.c
index 8c8b9611af5..ffb22866b04 100644
--- a/ompi/mca/coll/basic/coll_basic_reduce_scatter.c
+++ b/ompi/mca/coll/basic/coll_basic_reduce_scatter.c
@@ -49,7 +49,7 @@
  *	Returns:	- MPI_SUCCESS or error code
  *
  * Algorithm:
- *   Cummutative, reasonable sized messages
+ *   Commutative, reasonable sized messages
  *     recursive halving algorithm
  *   Others:
  *     reduce and scatterv (needs to be cleaned
@@ -57,14 +57,14 @@
  *
  * NOTE: that the recursive halving algorithm should be faster than
  * the reduce/scatter for all message sizes.  However, the memory
- * usage for the recusive halving is msg_size + 2 * comm_size greater
+ * usage for the recursive halving is msg_size + 2 * comm_size greater
  * for the recursive halving, so I've limited where the recursive
  * halving is used to be nice to the app memory wise.  There are much
  * better algorithms for large messages with commutative operations,
  * so this should be investigated further.
  */
 int
-mca_coll_basic_reduce_scatter_intra(const void *sbuf, void *rbuf, const int *rcounts,
+mca_coll_basic_reduce_scatter_intra(const void *sbuf, void *rbuf, ompi_count_array_t rcounts,
                                     struct ompi_datatype_t *dtype,
                                     struct ompi_op_t *op,
                                     struct ompi_communicator_t *comm,
@@ -72,7 +72,8 @@ mca_coll_basic_reduce_scatter_intra(const void *sbuf, void *rbuf, const int *rco
 {
     int i, rank, size, count, err = OMPI_SUCCESS;
     ptrdiff_t extent, buf_size, gap;
-    int *disps = NULL;
+    ptrdiff_t *disps = NULL;
+    ompi_disp_array_t disps_desc;
     char *recv_buf = NULL, *recv_buf_free = NULL;
     char *result_buf = NULL, *result_buf_free = NULL;
     /* Initialize */
@@ -80,14 +81,14 @@ mca_coll_basic_reduce_scatter_intra(const void *sbuf, void *rbuf, const int *rco
     size = ompi_comm_size(comm);
 
     /* Find displacements and the like */
-    disps = (int*) malloc(sizeof(int) * size);
+    disps = (ptrdiff_t*) malloc(sizeof(ptrdiff_t) * size);
     if (NULL == disps) return OMPI_ERR_OUT_OF_RESOURCE;
 
     disps[0] = 0;
     for (i = 0; i < (size - 1); ++i) {
-        disps[i + 1] = disps[i] + rcounts[i];
+        disps[i + 1] = disps[i] + ompi_count_array_get(rcounts, i);
     }
-    count = disps[size - 1] + rcounts[size - 1];
+    count = disps[size - 1] + ompi_count_array_get(rcounts, size - 1);
 
     /* short cut the trivial case */
     if (0 == count) {
@@ -186,9 +187,10 @@ mca_coll_basic_reduce_scatter_intra(const void *sbuf, void *rbuf, const int *rco
             for (i = 0 ; i < tmp_size ; ++i) {
                 if (i < remain) {
                     /* need to include old neighbor as well */
-                    tmp_rcounts[i] = rcounts[i * 2 + 1] + rcounts[i * 2];
+                    tmp_rcounts[i] = ompi_count_array_get(rcounts, i * 2 + 1)
+                                     + ompi_count_array_get(rcounts, i * 2);
                 } else {
-                    tmp_rcounts[i] = rcounts[i + remain];
+                    tmp_rcounts[i] = ompi_count_array_get(rcounts, i + remain);
                 }
             }
 
@@ -280,10 +282,10 @@ mca_coll_basic_reduce_scatter_intra(const void *sbuf, void *rbuf, const int *rco
             }
 
             /* copy local results from results buffer into real receive buffer */
-            if (0 != rcounts[rank]) {
+            if (0 != ompi_count_array_get(rcounts, rank)) {
                 err = ompi_datatype_sndrcv(result_buf + disps[rank] * extent,
-                                      rcounts[rank], dtype,
-                                      rbuf, rcounts[rank], dtype);
+                                      ompi_count_array_get(rcounts, rank), dtype,
+                                      rbuf, ompi_count_array_get(rcounts, rank), dtype);
                 if (OMPI_SUCCESS != err) {
                     free(tmp_rcounts);
                     free(tmp_disps);
@@ -299,16 +301,16 @@ mca_coll_basic_reduce_scatter_intra(const void *sbuf, void *rbuf, const int *rco
            procs send the even procs the proper results */
         if (rank < 2 * remain) {
             if ((rank & 1) == 0) {
-                if (rcounts[rank]) {
-                    err = MCA_PML_CALL(recv(rbuf, rcounts[rank], dtype, rank + 1,
+                if (ompi_count_array_get(rcounts, rank)) {
+                    err = MCA_PML_CALL(recv(rbuf, ompi_count_array_get(rcounts, rank), dtype, rank + 1,
                                             MCA_COLL_BASE_TAG_REDUCE_SCATTER,
                                             comm, MPI_STATUS_IGNORE));
                     if (OMPI_SUCCESS != err) goto cleanup;
                 }
             } else {
-                if (rcounts[rank - 1]) {
+                if (ompi_count_array_get(rcounts, rank - 1)) {
                     err = MCA_PML_CALL(send(result_buf + disps[rank - 1] * extent,
-                                            rcounts[rank - 1], dtype, rank - 1,
+                                            ompi_count_array_get(rcounts, rank - 1), dtype, rank - 1,
                                             MCA_COLL_BASE_TAG_REDUCE_SCATTER,
                                             MCA_PML_BASE_SEND_STANDARD,
                                             comm));
@@ -336,8 +338,9 @@ mca_coll_basic_reduce_scatter_intra(const void *sbuf, void *rbuf, const int *rco
 
         /* scatter */
         if (MPI_SUCCESS == err) {
-            err = comm->c_coll->coll_scatterv(recv_buf, rcounts, disps, dtype,
-                                             rbuf, rcounts[rank], dtype, 0,
+            OMPI_DISP_ARRAY_INIT(&disps_desc, disps);
+            err = comm->c_coll->coll_scatterv(recv_buf, rcounts, disps_desc, dtype,
+                                             rbuf, ompi_count_array_get(rcounts, rank), dtype, 0,
                                              comm, comm->c_coll->coll_scatterv_module);
         }
     }
@@ -359,7 +362,7 @@ mca_coll_basic_reduce_scatter_intra(const void *sbuf, void *rbuf, const int *rco
  *	Returns:	- MPI_SUCCESS or error code
  */
 int
-mca_coll_basic_reduce_scatter_inter(const void *sbuf, void *rbuf, const int *rcounts,
+mca_coll_basic_reduce_scatter_inter(const void *sbuf, void *rbuf, ompi_count_array_t rcounts,
                                     struct ompi_datatype_t *dtype,
                                     struct ompi_op_t *op,
                                     struct ompi_communicator_t *comm,
@@ -369,7 +372,8 @@ mca_coll_basic_reduce_scatter_inter(const void *sbuf, void *rbuf, const int *rco
     char *tmpbuf = NULL, *tmpbuf2 = NULL, *lbuf = NULL, *buf;
     ptrdiff_t gap, span;
     ompi_request_t *req;
-    int *disps = NULL;
+    ptrdiff_t *disps = NULL;
+    ompi_disp_array_t disps_desc;
 
     rank = ompi_comm_rank(comm);
     rsize = ompi_comm_remote_size(comm);
@@ -377,7 +381,7 @@ mca_coll_basic_reduce_scatter_inter(const void *sbuf, void *rbuf, const int *rco
 
     /* Figure out the total amount of data for the reduction. */
     for (totalcounts = 0, i = 0; i < lsize; i++) {
-        totalcounts += rcounts[i];
+        totalcounts += ompi_count_array_get(rcounts, i);
     }
 
     /*
@@ -401,13 +405,13 @@ mca_coll_basic_reduce_scatter_inter(const void *sbuf, void *rbuf, const int *rco
         span = opal_datatype_span(&dtype->super, totalcounts, &gap);
 
         /* Generate displacements for the scatterv part */
-        disps = (int*) malloc(sizeof(int) * lsize);
+        disps = (ptrdiff_t*) malloc(sizeof(ptrdiff_t) * lsize);
         if (NULL == disps) {
             return OMPI_ERR_OUT_OF_RESOURCE;
         }
         disps[0] = 0;
         for (i = 0; i < (lsize - 1); ++i) {
-            disps[i + 1] = disps[i] + rcounts[i];
+            disps[i + 1] = disps[i] + ompi_count_array_get(rcounts, i);
         }
 
         tmpbuf = (char *) malloc(span);
@@ -469,8 +473,9 @@ mca_coll_basic_reduce_scatter_inter(const void *sbuf, void *rbuf, const int *rco
     }
 
     /* Now do a scatterv on the local communicator */
-    err = comm->c_local_comm->c_coll->coll_scatterv(lbuf, rcounts, disps, dtype,
-                                                   rbuf, rcounts[rank], dtype, 0,
+    OMPI_DISP_ARRAY_INIT(&disps_desc, disps);
+    err = comm->c_local_comm->c_coll->coll_scatterv(lbuf, rcounts, disps_desc, dtype,
+                                                   rbuf, ompi_count_array_get(rcounts, rank), dtype, 0,
                                                    comm->c_local_comm,
                                                    comm->c_local_comm->c_coll->coll_scatterv_module);
 
diff --git a/ompi/mca/coll/basic/coll_basic_reduce_scatter_block.c b/ompi/mca/coll/basic/coll_basic_reduce_scatter_block.c
index 044a0804209..5eb7adfda50 100644
--- a/ompi/mca/coll/basic/coll_basic_reduce_scatter_block.c
+++ b/ompi/mca/coll/basic/coll_basic_reduce_scatter_block.c
@@ -14,6 +14,8 @@
  * Copyright (c) 2012      Sandia National Laboratories. All rights reserved.
  * Copyright (c) 2014-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -51,7 +53,7 @@
  *     up at some point)
  */
 int
-mca_coll_basic_reduce_scatter_block_intra(const void *sbuf, void *rbuf, int rcount,
+mca_coll_basic_reduce_scatter_block_intra(const void *sbuf, void *rbuf, size_t rcount,
                                           struct ompi_datatype_t *dtype,
                                           struct ompi_op_t *op,
                                           struct ompi_communicator_t *comm,
@@ -68,7 +70,7 @@ mca_coll_basic_reduce_scatter_block_intra(const void *sbuf, void *rbuf, int rcou
  *	Returns:	- MPI_SUCCESS or error code
  */
 int
-mca_coll_basic_reduce_scatter_block_inter(const void *sbuf, void *rbuf, int rcount,
+mca_coll_basic_reduce_scatter_block_inter(const void *sbuf, void *rbuf, size_t rcount,
                                           struct ompi_datatype_t *dtype,
                                           struct ompi_op_t *op,
                                           struct ompi_communicator_t *comm,
@@ -104,11 +106,12 @@ mca_coll_basic_reduce_scatter_block_inter(const void *sbuf, void *rbuf, int rcou
     if (rank == root) {
         span = opal_datatype_span(&dtype->super, totalcounts, &gap);
 
-        tmpbuf = (char *) malloc(span);
-        tmpbuf2 = (char *) malloc(span);
-        if (NULL == tmpbuf || NULL == tmpbuf2) {
+        tmpbuf = (char *) malloc(2*span);
+        if (NULL == tmpbuf) {
             return OMPI_ERR_OUT_OF_RESOURCE;
         }
+        tmpbuf2 = tmpbuf + span;
+
         lbuf = tmpbuf - gap;
         buf = tmpbuf2 - gap;
 
@@ -172,9 +175,5 @@ mca_coll_basic_reduce_scatter_block_inter(const void *sbuf, void *rbuf, int rcou
         free(tmpbuf);
     }
 
-    if (NULL != tmpbuf2) {
-        free(tmpbuf2);
-    }
-
     return err;
 }
diff --git a/ompi/mca/coll/basic/coll_basic_scan.c b/ompi/mca/coll/basic/coll_basic_scan.c
deleted file mode 100644
index e7399eb91fa..00000000000
--- a/ompi/mca/coll/basic/coll_basic_scan.c
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2014 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include "coll_basic.h"
-
-#include <stdio.h>
-
-#include "mpi.h"
-#include "ompi/constants.h"
-#include "ompi/op/op.h"
-#include "ompi/mca/coll/coll.h"
-#include "ompi/mca/coll/base/coll_tags.h"
-#include "ompi/mca/pml/pml.h"
-#include "coll_basic.h"
-
-
-/*
- *	scan
- *
- *	Function:	- basic scan operation
- *	Accepts:	- same arguments as MPI_Scan()
- *	Returns:	- MPI_SUCCESS or error code
- */
-int
-mca_coll_basic_scan_intra(const void *sbuf, void *rbuf, int count,
-                          struct ompi_datatype_t *dtype,
-                          struct ompi_op_t *op,
-                          struct ompi_communicator_t *comm,
-                          mca_coll_base_module_t *module)
-{
-    return ompi_coll_base_scan_intra_linear(sbuf, rbuf, count, dtype, op, comm, module);
-}
diff --git a/ompi/mca/coll/basic/coll_basic_scatter.c b/ompi/mca/coll/basic/coll_basic_scatter.c
index ea5aa7aecbe..c421ee498bb 100644
--- a/ompi/mca/coll/basic/coll_basic_scatter.c
+++ b/ompi/mca/coll/basic/coll_basic_scatter.c
@@ -12,6 +12,7 @@
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -39,9 +40,9 @@
  *	Returns:	- MPI_SUCCESS or error code
  */
 int
-mca_coll_basic_scatter_inter(const void *sbuf, int scount,
+mca_coll_basic_scatter_inter(const void *sbuf, size_t scount,
                              struct ompi_datatype_t *sdtype,
-                             void *rbuf, int rcount,
+                             void *rbuf, size_t rcount,
                              struct ompi_datatype_t *rdtype,
                              int root, struct ompi_communicator_t *comm,
                              mca_coll_base_module_t *module)
@@ -77,7 +78,7 @@ mca_coll_basic_scatter_inter(const void *sbuf, int scount,
             err = MCA_PML_CALL(isend(ptmp, scount, sdtype, i,
                                      MCA_COLL_BASE_TAG_SCATTER,
                                      MCA_PML_BASE_SEND_STANDARD, comm,
-                                     reqs++));
+                                     &(reqs[i])));
             if (OMPI_SUCCESS != err) {
                 ompi_coll_base_free_reqs(reqs, i + 1);
                 return err;
diff --git a/ompi/mca/coll/basic/coll_basic_scatterv.c b/ompi/mca/coll/basic/coll_basic_scatterv.c
index 16602158b2b..b79d9920467 100644
--- a/ompi/mca/coll/basic/coll_basic_scatterv.c
+++ b/ompi/mca/coll/basic/coll_basic_scatterv.c
@@ -9,9 +9,9 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2017-2022 IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -39,9 +39,9 @@
  *	Returns:	- MPI_SUCCESS or error code
  */
 int
-mca_coll_basic_scatterv_intra(const void *sbuf, const int *scounts,
-                              const int *disps, struct ompi_datatype_t *sdtype,
-                              void *rbuf, int rcount,
+mca_coll_basic_scatterv_intra(const void *sbuf, ompi_count_array_t scounts,
+                              ompi_disp_array_t disps, struct ompi_datatype_t *sdtype,
+                              void *rbuf, size_t rcount,
                               struct ompi_datatype_t *rdtype, int root,
                               struct ompi_communicator_t *comm,
                               mca_coll_base_module_t *module)
@@ -49,6 +49,7 @@ mca_coll_basic_scatterv_intra(const void *sbuf, const int *scounts,
     int i, rank, size, err;
     char *ptmp;
     ptrdiff_t lb, extent;
+    size_t sdsize;
 
     /* Initialize */
 
@@ -58,8 +59,10 @@ mca_coll_basic_scatterv_intra(const void *sbuf, const int *scounts,
     /* If not root, receive data. */
 
     if (rank != root) {
+        size_t rdsize;
+        ompi_datatype_type_size(rdtype, &rdsize);
         /* Only receive if there is something to receive */
-        if (rcount > 0) {
+        if (rcount > 0 && rdsize > 0) {
             return MCA_PML_CALL(recv(rbuf, rcount, rdtype,
                                      root, MCA_COLL_BASE_TAG_SCATTERV,
                                      comm, MPI_STATUS_IGNORE));
@@ -67,6 +70,12 @@ mca_coll_basic_scatterv_intra(const void *sbuf, const int *scounts,
         return MPI_SUCCESS;
     }
 
+    ompi_datatype_type_size(sdtype, &sdsize);
+    if (OPAL_UNLIKELY(0 == sdsize)) {
+        /* bozzo case */
+        return MPI_SUCCESS;
+    }
+
     /* I am the root, loop sending data. */
 
     err = ompi_datatype_get_extent(sdtype, &lb, &extent);
@@ -75,20 +84,23 @@ mca_coll_basic_scatterv_intra(const void *sbuf, const int *scounts,
     }
 
     for (i = 0; i < size; ++i) {
-        ptmp = ((char *) sbuf) + (extent * disps[i]);
+        ptmp = ((char *) sbuf) + (extent * ompi_disp_array_get(disps, i));
 
         /* simple optimization */
 
         if (i == rank) {
             /* simple optimization or a local operation */
-            if (scounts[i] > 0 && MPI_IN_PLACE != rbuf) {
-                err = ompi_datatype_sndrcv(ptmp, scounts[i], sdtype, rbuf, rcount,
+            if (ompi_count_array_get(scounts, i) > 0 && MPI_IN_PLACE != rbuf) {
+                err = ompi_datatype_sndrcv(ptmp, ompi_count_array_get(scounts, i), sdtype, rbuf, rcount,
                                       rdtype);
+                if (MPI_SUCCESS != err) {
+                    return err;
+                }
             }
         } else {
             /* Only send if there is something to send */
-            if (scounts[i] > 0) {
-                err = MCA_PML_CALL(send(ptmp, scounts[i], sdtype, i,
+            if (ompi_count_array_get(scounts, i) > 0) {
+                err = MCA_PML_CALL(send(ptmp, ompi_count_array_get(scounts, i), sdtype, i,
                                         MCA_COLL_BASE_TAG_SCATTERV,
                                         MCA_PML_BASE_SEND_STANDARD, comm));
                 if (MPI_SUCCESS != err) {
@@ -112,9 +124,9 @@ mca_coll_basic_scatterv_intra(const void *sbuf, const int *scounts,
  *	Returns:	- MPI_SUCCESS or error code
  */
 int
-mca_coll_basic_scatterv_inter(const void *sbuf, const int *scounts,
-                              const int *disps, struct ompi_datatype_t *sdtype,
-                              void *rbuf, int rcount,
+mca_coll_basic_scatterv_inter(const void *sbuf, ompi_count_array_t scounts,
+                              ompi_disp_array_t disps, struct ompi_datatype_t *sdtype,
+                              void *rbuf, size_t rcount,
                               struct ompi_datatype_t *rdtype, int root,
                               struct ompi_communicator_t *comm,
                               mca_coll_base_module_t *module)
@@ -149,8 +161,8 @@ mca_coll_basic_scatterv_inter(const void *sbuf, const int *scounts,
         if( NULL == reqs ) { return OMPI_ERR_OUT_OF_RESOURCE; }
 
         for (i = 0; i < size; ++i) {
-            ptmp = ((char *) sbuf) + (extent * disps[i]);
-            err = MCA_PML_CALL(isend(ptmp, scounts[i], sdtype, i,
+            ptmp = ((char *) sbuf) + (extent * ompi_disp_array_get(disps, i));
+            err = MCA_PML_CALL(isend(ptmp, ompi_count_array_get(scounts, i), sdtype, i,
                                      MCA_COLL_BASE_TAG_SCATTERV,
                                      MCA_PML_BASE_SEND_STANDARD, comm,
                                      &(reqs[i])));
diff --git a/ompi/mca/coll/coll.h b/ompi/mca/coll/coll.h
index 1579c542a48..b748b0e8cdd 100644
--- a/ompi/mca/coll/coll.h
+++ b/ompi/mca/coll/coll.h
@@ -20,6 +20,7 @@
  * Copyright (c) 2016-2017 IBM Corporation.  All rights reserved.
  * Copyright (c) 2017      FUJITSU LIMITED.  All rights reserved.
  * Copyright (c) 2020      BULL S.A.S. All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -37,9 +38,9 @@
  * handler invocation, but the collective components provide all other
  * functionality.
  *
- * Component selection is done per commuicator, at Communicator
+ * Component selection is done per communicator, at Communicator
  * construction time.  mca_coll_base_comm_select() is used to
- * create the list of components available to the componenent
+ * create the list of components available to the component
  * collm_comm_query function, instantiating a module for each
  * component that is usable, and sets the module collective function pointers.
  * mca_coll_base_comm_select() then loops through the list of available
@@ -50,7 +51,7 @@
  * is associated with.
  *
  * The module destructors are called for each module used by the
- * communicator, at communicator desctruction time.
+ * communicator, at communicator destruction time.
  *
  * This can result in up to N different components being used for a
  * single communicator, one per needed collective function.
@@ -59,6 +60,15 @@
  * components should be able to handle either style of communicator
  * during initialization (although handling may include indicating the
  * component is not available).
+ *
+ * Unlike the MPI standard, all buffers that are not supposed to be used (and
+ * therefore where the MPI standard does not require the tuple 
+ * (buffer, datatype, count) to be accessible, are replaced with NULL.
+ * As an example, the recvbuf for all non-root ranks in an MPI_Reduce,
+ * will be set to NULL at the MPI API level. The reason behind this is to
+ * allow collective components to indicate when buffers are really valid,
+ * such that collective modules delegating collectives to other modules
+ * can share temporary buffer.
  */
 
 #ifndef OMPI_MCA_COLL_COLL_H
@@ -69,8 +79,10 @@
 #include "mpi.h"
 #include "ompi/mca/mca.h"
 #include "opal/mca/base/base.h"
+#include "ompi/mca/coll/base/base.h"
 
 #include "ompi/request/request.h"
+#include "ompi/util/count_disp_array.h"
 
 BEGIN_C_DECLS
 
@@ -96,7 +108,7 @@ struct ompi_group_t;
  * once during MPI_INIT.
  *
  * @note The component framework is not lazily opened, so attempts
- * should be made to minimze the amount of memory allocated during
+ * should be made to minimize the amount of memory allocated during
  * this function.
  *
  * @param[in] enable_progress_threads True if the component needs to
@@ -110,7 +122,7 @@ struct ompi_group_t;
 typedef int (*mca_coll_base_component_init_query_fn_t)
      (bool enable_progress_threads, bool enable_mpi_threads);
 
-struct mca_coll_base_module_2_4_0_t;
+struct mca_coll_base_module_3_0_0_t;
 
 /**
  * Query whether a component is available for the given communicator
@@ -139,8 +151,8 @@ struct mca_coll_base_module_2_4_0_t;
  * provide a module with the requested functionality or NULL if the
  * component should not be used on the given communicator.
  */
-typedef struct mca_coll_base_module_2_4_0_t *
-  (*mca_coll_base_component_comm_query_2_4_0_fn_t)
+typedef struct mca_coll_base_module_3_0_0_t *
+  (*mca_coll_base_component_comm_query_3_0_0_fn_t)
     (struct ompi_communicator_t *comm, int *priority);
 
 
@@ -179,7 +191,7 @@ typedef struct mca_coll_base_module_2_4_0_t *
  * @param[in]     comm       Communicator being created
  */
 typedef int
-(*mca_coll_base_module_enable_1_1_0_fn_t)(struct mca_coll_base_module_2_4_0_t* module,
+(*mca_coll_base_module_enable_1_1_0_fn_t)(struct mca_coll_base_module_3_0_0_t* module,
                                           struct ompi_communicator_t *comm);
 
 /* not #if conditional on OPAL_ENABLE_FT_MPI for ABI */
@@ -201,15 +213,15 @@ typedef int
  * @param module: the MCA module that defines this agreement.
  */
 typedef int (*mca_coll_base_module_agree_fn_t)
- (void *contrib, int dt_count, struct ompi_datatype_t *dtype,
+ (void *contrib, size_t dt_count, struct ompi_datatype_t *dtype,
    struct ompi_op_t *op, struct ompi_group_t **failedgroup, bool update_failedgroup,
    struct ompi_communicator_t *comm,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_iagree_fn_t)
-  (void *contrib, int dt_count, struct ompi_datatype_t *dtype,
+  (void *contrib, size_t dt_count, struct ompi_datatype_t *dtype,
    struct ompi_op_t *op, struct ompi_group_t **failedgroup, bool update_failedgroup,
    struct ompi_communicator_t *comm, ompi_request_t **request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 
 
 /**
@@ -223,243 +235,243 @@ typedef int (*mca_coll_base_module_iagree_fn_t)
  * @param[in]     comm       Communicator being disabled
  */
 typedef int
-(*mca_coll_base_module_disable_1_2_0_fn_t)(struct mca_coll_base_module_2_4_0_t* module,
+(*mca_coll_base_module_disable_1_2_0_fn_t)(struct mca_coll_base_module_3_0_0_t* module,
                                           struct ompi_communicator_t *comm);
 
 /* blocking collectives */
 typedef int (*mca_coll_base_module_allgather_fn_t)
-  (const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-   void *rbuf, int rcount, struct ompi_datatype_t *rdtype,
-   struct ompi_communicator_t *comm, struct mca_coll_base_module_2_4_0_t *module);
+  (const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+   void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
+   struct ompi_communicator_t *comm, struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_allgatherv_fn_t)
-  (const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-   void * rbuf, const int *rcounts, const int *disps,  struct ompi_datatype_t *rdtype,
-   struct ompi_communicator_t *comm, struct mca_coll_base_module_2_4_0_t *module);
+  (const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+   void * rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,  struct ompi_datatype_t *rdtype,
+   struct ompi_communicator_t *comm, struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_allreduce_fn_t)
-  (const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype,
-   struct ompi_op_t *op, struct ompi_communicator_t *comm, struct mca_coll_base_module_2_4_0_t *module);
+  (const void *sbuf, void *rbuf, size_t count, struct ompi_datatype_t *dtype,
+   struct ompi_op_t *op, struct ompi_communicator_t *comm, struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_alltoall_fn_t)
-  (const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-   void* rbuf, int rcount, struct ompi_datatype_t *rdtype,
-   struct ompi_communicator_t *comm, struct mca_coll_base_module_2_4_0_t *module);
+  (const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+   void* rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
+   struct ompi_communicator_t *comm, struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_alltoallv_fn_t)
-  (const void *sbuf, const int *scounts, const int *sdisps, struct ompi_datatype_t *sdtype,
-   void *rbuf, const int *rcounts, const int *rdisps, struct ompi_datatype_t *rdtype,
-   struct ompi_communicator_t *comm, struct mca_coll_base_module_2_4_0_t *module);
+  (const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps, struct ompi_datatype_t *sdtype,
+   void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps, struct ompi_datatype_t *rdtype,
+   struct ompi_communicator_t *comm, struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_alltoallw_fn_t)
-  (const void *sbuf, const int *scounts, const int *sdisps, struct ompi_datatype_t * const *sdtypes,
-   void *rbuf, const int *rcounts, const int *rdisps, struct ompi_datatype_t * const *rdtypes,
-   struct ompi_communicator_t *comm, struct mca_coll_base_module_2_4_0_t *module);
+  (const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps, struct ompi_datatype_t * const *sdtypes,
+   void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps, struct ompi_datatype_t * const *rdtypes,
+   struct ompi_communicator_t *comm, struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_barrier_fn_t)
-  (struct ompi_communicator_t *comm, struct mca_coll_base_module_2_4_0_t *module);
+  (struct ompi_communicator_t *comm, struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_bcast_fn_t)
-  (void *buff, int count, struct ompi_datatype_t *datatype, int root,
-   struct ompi_communicator_t *comm, struct mca_coll_base_module_2_4_0_t *module);
+  (void *buff, size_t count, struct ompi_datatype_t *datatype, int root,
+   struct ompi_communicator_t *comm, struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_exscan_fn_t)
-  (const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype,
-   struct ompi_op_t *op, struct ompi_communicator_t *comm, struct mca_coll_base_module_2_4_0_t *module);
+  (const void *sbuf, void *rbuf, size_t count, struct ompi_datatype_t *dtype,
+   struct ompi_op_t *op, struct ompi_communicator_t *comm, struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_gather_fn_t)
-  (const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-   void *rbuf, int rcount, struct ompi_datatype_t *rdtype,
-   int root, struct ompi_communicator_t *comm, struct mca_coll_base_module_2_4_0_t *module);
+  (const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+   void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
+   int root, struct ompi_communicator_t *comm, struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_gatherv_fn_t)
-  (const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-   void *rbuf, const int *rcounts, const int *disps, struct ompi_datatype_t *rdtype,
-   int root, struct ompi_communicator_t *comm, struct mca_coll_base_module_2_4_0_t *module);
+  (const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+   void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps, struct ompi_datatype_t *rdtype,
+   int root, struct ompi_communicator_t *comm, struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_reduce_fn_t)
-  (const void *sbuf, void* rbuf, int count, struct ompi_datatype_t *dtype,
-   struct ompi_op_t *op, int root, struct ompi_communicator_t *comm, struct mca_coll_base_module_2_4_0_t *module);
+  (const void *sbuf, void* rbuf, size_t count, struct ompi_datatype_t *dtype,
+   struct ompi_op_t *op, int root, struct ompi_communicator_t *comm, struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_reduce_scatter_fn_t)
-  (const void *sbuf, void *rbuf, const int *rcounts, struct ompi_datatype_t *dtype,
-   struct ompi_op_t *op, struct ompi_communicator_t *comm, struct mca_coll_base_module_2_4_0_t *module);
+  (const void *sbuf, void *rbuf, ompi_count_array_t rcounts, struct ompi_datatype_t *dtype,
+   struct ompi_op_t *op, struct ompi_communicator_t *comm, struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_reduce_scatter_block_fn_t)
-  (const void *sbuf, void *rbuf, int rcount, struct ompi_datatype_t *dtype,
-   struct ompi_op_t *op, struct ompi_communicator_t *comm, struct mca_coll_base_module_2_4_0_t *module);
+  (const void *sbuf, void *rbuf, size_t rcount, struct ompi_datatype_t *dtype,
+   struct ompi_op_t *op, struct ompi_communicator_t *comm, struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_scan_fn_t)
-  (const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype,
-   struct ompi_op_t *op, struct ompi_communicator_t *comm, struct mca_coll_base_module_2_4_0_t *module);
+  (const void *sbuf, void *rbuf, size_t count, struct ompi_datatype_t *dtype,
+   struct ompi_op_t *op, struct ompi_communicator_t *comm, struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_scatter_fn_t)
-  (const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-   void *rbuf, int rcount, struct ompi_datatype_t *rdtype,
-   int root, struct ompi_communicator_t *comm, struct mca_coll_base_module_2_4_0_t *module);
+  (const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+   void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
+   int root, struct ompi_communicator_t *comm, struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_scatterv_fn_t)
-  (const void *sbuf, const int *scounts, const int *disps, struct ompi_datatype_t *sdtype,
-   void* rbuf, int rcount, struct ompi_datatype_t *rdtype,
-   int root, struct ompi_communicator_t *comm, struct mca_coll_base_module_2_4_0_t *module);
+  (const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t disps, struct ompi_datatype_t *sdtype,
+   void* rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
+   int root, struct ompi_communicator_t *comm, struct mca_coll_base_module_3_0_0_t *module);
 
 /* nonblocking collectives */
 typedef int (*mca_coll_base_module_iallgather_fn_t)
-  (const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-   void *rbuf, int rcount, struct ompi_datatype_t *rdtype,
+  (const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+   void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
    struct ompi_communicator_t *comm, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_iallgatherv_fn_t)
-  (const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-   void * rbuf, const int *rcounts, const int *disps,  struct ompi_datatype_t *rdtype,
+  (const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+   void * rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,  struct ompi_datatype_t *rdtype,
    struct ompi_communicator_t *comm, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_iallreduce_fn_t)
-  (const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype,
+  (const void *sbuf, void *rbuf, size_t count, struct ompi_datatype_t *dtype,
    struct ompi_op_t *op, struct ompi_communicator_t *comm,
-   ompi_request_t ** request, struct mca_coll_base_module_2_4_0_t *module);
+   ompi_request_t ** request, struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_ialltoall_fn_t)
-  (const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-   void* rbuf, int rcount, struct ompi_datatype_t *rdtype,
+  (const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+   void* rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
    struct ompi_communicator_t *comm, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_ialltoallv_fn_t)
-  (const void *sbuf, const int *scounts, const int *sdisps, struct ompi_datatype_t *sdtype,
-   void *rbuf, const int *rcounts, const int *rdisps, struct ompi_datatype_t *rdtype,
+  (const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps, struct ompi_datatype_t *sdtype,
+   void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps, struct ompi_datatype_t *rdtype,
    struct ompi_communicator_t *comm, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_ialltoallw_fn_t)
-  (const void *sbuf, const int *scounts, const int *sdisps, struct ompi_datatype_t * const *sdtypes,
-   void *rbuf, const int *rcounts, const int *rdisps, struct ompi_datatype_t * const *rdtypes,
+  (const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps, struct ompi_datatype_t * const *sdtypes,
+   void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps, struct ompi_datatype_t * const *rdtypes,
    struct ompi_communicator_t *comm, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_ibarrier_fn_t)
   (struct ompi_communicator_t *comm, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_ibcast_fn_t)
-  (void *buff, int count, struct ompi_datatype_t *datatype, int root,
+  (void *buff, size_t count, struct ompi_datatype_t *datatype, int root,
    struct ompi_communicator_t *comm, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_iexscan_fn_t)
-  (const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype,
+  (const void *sbuf, void *rbuf, size_t count, struct ompi_datatype_t *dtype,
    struct ompi_op_t *op, struct ompi_communicator_t *comm, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_igather_fn_t)
-  (const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-   void *rbuf, int rcount, struct ompi_datatype_t *rdtype,
+  (const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+   void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
    int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_igatherv_fn_t)
-  (const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-   void *rbuf, const int *rcounts, const int *disps, struct ompi_datatype_t *rdtype,
+  (const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+   void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps, struct ompi_datatype_t *rdtype,
    int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_ireduce_fn_t)
-  (const void *sbuf, void* rbuf, int count, struct ompi_datatype_t *dtype,
+  (const void *sbuf, void* rbuf, size_t count, struct ompi_datatype_t *dtype,
    struct ompi_op_t *op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_ireduce_scatter_fn_t)
-  (const void *sbuf, void *rbuf, const int *rcounts, struct ompi_datatype_t *dtype,
+  (const void *sbuf, void *rbuf, ompi_count_array_t rcounts, struct ompi_datatype_t *dtype,
    struct ompi_op_t *op, struct ompi_communicator_t *comm, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_ireduce_scatter_block_fn_t)
-  (const void *sbuf, void *rbuf, int rcount, struct ompi_datatype_t *dtype,
+  (const void *sbuf, void *rbuf, size_t rcount, struct ompi_datatype_t *dtype,
    struct ompi_op_t *op, struct ompi_communicator_t *comm, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_iscan_fn_t)
-  (const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype,
+  (const void *sbuf, void *rbuf, size_t count, struct ompi_datatype_t *dtype,
    struct ompi_op_t *op, struct ompi_communicator_t *comm, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_iscatter_fn_t)
-  (const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-   void *rbuf, int rcount, struct ompi_datatype_t *rdtype,
+  (const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+   void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
    int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_iscatterv_fn_t)
-  (const void *sbuf, const int *scounts, const int *disps, struct ompi_datatype_t *sdtype,
-   void* rbuf, int rcount, struct ompi_datatype_t *rdtype,
+  (const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t disps, struct ompi_datatype_t *sdtype,
+   void* rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
    int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 
 /* persistent collectives */
 typedef int (*mca_coll_base_module_allgather_init_fn_t)
-  (const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-   void *rbuf, int rcount, struct ompi_datatype_t *rdtype,
+  (const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+   void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
    struct ompi_communicator_t *comm, struct ompi_info_t *info, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_allgatherv_init_fn_t)
-  (const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-   void * rbuf, const int *rcounts, const int *disps,  struct ompi_datatype_t *rdtype,
+  (const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+   void * rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,  struct ompi_datatype_t *rdtype,
    struct ompi_communicator_t *comm, struct ompi_info_t *info, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_allreduce_init_fn_t)
-  (const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype,
+  (const void *sbuf, void *rbuf, size_t count, struct ompi_datatype_t *dtype,
    struct ompi_op_t *op, struct ompi_communicator_t *comm, struct ompi_info_t *info,
-   ompi_request_t ** request, struct mca_coll_base_module_2_4_0_t *module);
+   ompi_request_t ** request, struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_alltoall_init_fn_t)
-  (const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-   void* rbuf, int rcount, struct ompi_datatype_t *rdtype,
+  (const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+   void* rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
    struct ompi_communicator_t *comm, struct ompi_info_t *info, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_alltoallv_init_fn_t)
-  (const void *sbuf, const int *scounts, const int *sdisps, struct ompi_datatype_t *sdtype,
-   void *rbuf, const int *rcounts, const int *rdisps, struct ompi_datatype_t *rdtype,
+  (const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps, struct ompi_datatype_t *sdtype,
+   void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps, struct ompi_datatype_t *rdtype,
    struct ompi_communicator_t *comm, struct ompi_info_t *info, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_alltoallw_init_fn_t)
-  (const void *sbuf, const int *scounts, const int *sdisps, struct ompi_datatype_t * const *sdtypes,
-   void *rbuf, const int *rcounts, const int *rdisps, struct ompi_datatype_t * const *rdtypes,
+  (const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps, struct ompi_datatype_t * const *sdtypes,
+   void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps, struct ompi_datatype_t * const *rdtypes,
    struct ompi_communicator_t *comm, struct ompi_info_t *info, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_barrier_init_fn_t)
   (struct ompi_communicator_t *comm, struct ompi_info_t *info, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_bcast_init_fn_t)
-  (void *buff, int count, struct ompi_datatype_t *datatype, int root,
+  (void *buff, size_t count, struct ompi_datatype_t *datatype, int root,
    struct ompi_communicator_t *comm, struct ompi_info_t *info, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_exscan_init_fn_t)
-  (const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype,
+  (const void *sbuf, void *rbuf, size_t count, struct ompi_datatype_t *dtype,
    struct ompi_op_t *op, struct ompi_communicator_t *comm, struct ompi_info_t *info, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_gather_init_fn_t)
-  (const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-   void *rbuf, int rcount, struct ompi_datatype_t *rdtype,
+  (const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+   void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
    int root, struct ompi_communicator_t *comm, struct ompi_info_t *info, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_gatherv_init_fn_t)
-  (const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-   void *rbuf, const int *rcounts, const int *disps, struct ompi_datatype_t *rdtype,
+  (const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+   void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps, struct ompi_datatype_t *rdtype,
    int root, struct ompi_communicator_t *comm, struct ompi_info_t *info, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_reduce_init_fn_t)
-  (const void *sbuf, void* rbuf, int count, struct ompi_datatype_t *dtype,
+  (const void *sbuf, void* rbuf, size_t count, struct ompi_datatype_t *dtype,
    struct ompi_op_t *op, int root, struct ompi_communicator_t *comm, struct ompi_info_t *info, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_reduce_scatter_init_fn_t)
-  (const void *sbuf, void *rbuf, const int *rcounts, struct ompi_datatype_t *dtype,
+  (const void *sbuf, void *rbuf, ompi_count_array_t rcounts, struct ompi_datatype_t *dtype,
    struct ompi_op_t *op, struct ompi_communicator_t *comm, struct ompi_info_t *info, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_reduce_scatter_block_init_fn_t)
-  (const void *sbuf, void *rbuf, int rcount, struct ompi_datatype_t *dtype,
+  (const void *sbuf, void *rbuf, size_t rcount, struct ompi_datatype_t *dtype,
    struct ompi_op_t *op, struct ompi_communicator_t *comm, struct ompi_info_t *info, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_scan_init_fn_t)
-  (const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype,
+  (const void *sbuf, void *rbuf, size_t count, struct ompi_datatype_t *dtype,
    struct ompi_op_t *op, struct ompi_communicator_t *comm, struct ompi_info_t *info, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_scatter_init_fn_t)
-  (const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-   void *rbuf, int rcount, struct ompi_datatype_t *rdtype,
+  (const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+   void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
    int root, struct ompi_communicator_t *comm, struct ompi_info_t *info, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_scatterv_init_fn_t)
-  (const void *sbuf, const int *scounts, const int *disps, struct ompi_datatype_t *sdtype,
-   void* rbuf, int rcount, struct ompi_datatype_t *rdtype,
+  (const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t disps, struct ompi_datatype_t *sdtype,
+   void* rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
    int root, struct ompi_communicator_t *comm, struct ompi_info_t *info, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 
 /*
  * The signature of the neighborhood alltoallw differs from alltoallw
  */
 typedef int (*mca_coll_base_module_neighbor_alltoallw_fn_t)
-  (const void *sbuf, const int *scounts, const MPI_Aint *sdisps, struct ompi_datatype_t * const *sdtypes,
-   void *rbuf, const int *rcounts, const MPI_Aint *rdisps, struct ompi_datatype_t * const *rdtypes,
-   struct ompi_communicator_t *comm, struct mca_coll_base_module_2_4_0_t *module);
+  (const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps, struct ompi_datatype_t * const *sdtypes,
+   void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps, struct ompi_datatype_t * const *rdtypes,
+   struct ompi_communicator_t *comm, struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_ineighbor_alltoallw_fn_t)
-  (const void *sbuf, const int *scounts, const MPI_Aint *sdisps, struct ompi_datatype_t * const *sdtypes,
-   void *rbuf, const int *rcounts, const MPI_Aint *rdisps, struct ompi_datatype_t * const *rdtypes,
+  (const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps, struct ompi_datatype_t * const *sdtypes,
+   void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps, struct ompi_datatype_t * const *rdtypes,
    struct ompi_communicator_t *comm, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 typedef int (*mca_coll_base_module_neighbor_alltoallw_init_fn_t)
-  (const void *sbuf, const int *scounts, const MPI_Aint *sdisps, struct ompi_datatype_t * const *sdtypes,
-   void *rbuf, const int *rcounts, const MPI_Aint *rdisps, struct ompi_datatype_t * const *rdtypes,
+  (const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps, struct ompi_datatype_t * const *sdtypes,
+   void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps, struct ompi_datatype_t * const *rdtypes,
    struct ompi_communicator_t *comm, struct ompi_info_t *info, ompi_request_t ** request,
-   struct mca_coll_base_module_2_4_0_t *module);
+   struct mca_coll_base_module_3_0_0_t *module);
 
 /*
  * reduce_local
@@ -468,9 +480,9 @@ typedef int (*mca_coll_base_module_neighbor_alltoallw_init_fn_t)
  * option of intercepting it, if desired.
  */
 typedef int (*mca_coll_base_module_reduce_local_fn_t)
-   (const void *inbuf, void *inoutbuf, int count,
+   (const void *inbuf, void *inoutbuf, size_t count,
     struct ompi_datatype_t * dtype, struct ompi_op_t * op,
-    struct mca_coll_base_module_2_4_0_t *module);
+    struct mca_coll_base_module_3_0_0_t *module);
 
 
 /* ******************************************************************** */
@@ -484,7 +496,7 @@ typedef int (*mca_coll_base_module_reduce_local_fn_t)
  * mca_coll_[component_name]_component, must exist in any collective
  * component.
  */
-struct mca_coll_base_component_2_4_0_t {
+struct mca_coll_base_component_3_0_0_t {
     /** Base component description */
     mca_base_component_t collm_version;
     /** Base component data block */
@@ -492,15 +504,15 @@ struct mca_coll_base_component_2_4_0_t {
 
     /** Component initialization function */
     mca_coll_base_component_init_query_fn_t collm_init_query;
-    /** Query whether component is useable for given communicator */
-    mca_coll_base_component_comm_query_2_4_0_fn_t collm_comm_query;
+    /** Query whether component is usable for given communicator */
+    mca_coll_base_component_comm_query_3_0_0_fn_t collm_comm_query;
 };
-typedef struct mca_coll_base_component_2_4_0_t mca_coll_base_component_2_4_0_t;
+typedef struct mca_coll_base_component_3_0_0_t mca_coll_base_component_3_0_0_t;
 
-/** Per guidence in mca.h, use the unversioned struct name if you just
+/** Per guidance in mca.h, use the unversioned struct name if you just
     want to always keep up with the most recent version of the
-    interace. */
-typedef struct mca_coll_base_component_2_4_0_t mca_coll_base_component_t;
+    interface. */
+typedef struct mca_coll_base_component_3_0_0_t mca_coll_base_component_t;
 
 
 /**
@@ -508,7 +520,7 @@ typedef struct mca_coll_base_component_2_4_0_t mca_coll_base_component_t;
  *
  * Module interface to the Collective framework.  Modules are
  * reference counted based on the number of functions from the module
- * used on the commuicator.  There is at most one module per component
+ * used on the communicator.  There is at most one module per component
  * on a given communicator, and there can be many component modules on
  * a given communicator.
  *
@@ -517,7 +529,7 @@ typedef struct mca_coll_base_component_2_4_0_t mca_coll_base_component_t;
  * function, so the component is free to create a structure that
  * inherits from this one for use as the module structure.
  */
-struct mca_coll_base_module_2_4_0_t {
+struct mca_coll_base_module_3_0_0_t {
     /** Collective modules all inherit from opal_object */
     opal_object_t super;
 
@@ -619,12 +631,12 @@ struct mca_coll_base_module_2_4_0_t {
         not be used by other modules */
     struct mca_coll_base_comm_t* base_data;
 };
-typedef struct mca_coll_base_module_2_4_0_t mca_coll_base_module_2_4_0_t;
+typedef struct mca_coll_base_module_3_0_0_t mca_coll_base_module_3_0_0_t;
 
-/** Per guidence in mca.h, use the unversioned struct name if you just
+/** Per guidance in mca.h, use the unversioned struct name if you just
     want to always keep up with the most recent version of the
-    interace. */
-typedef struct mca_coll_base_module_2_4_0_t mca_coll_base_module_t;
+    interface. */
+typedef struct mca_coll_base_module_3_0_0_t mca_coll_base_module_t;
 OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_coll_base_module_t);
 
 /**
@@ -640,155 +652,155 @@ struct mca_coll_base_comm_coll_t {
 
     /* blocking collectives */
     mca_coll_base_module_allgather_fn_t coll_allgather;
-    mca_coll_base_module_2_4_0_t *coll_allgather_module;
+    mca_coll_base_module_3_0_0_t *coll_allgather_module;
     mca_coll_base_module_allgatherv_fn_t coll_allgatherv;
-    mca_coll_base_module_2_4_0_t *coll_allgatherv_module;
+    mca_coll_base_module_3_0_0_t *coll_allgatherv_module;
     mca_coll_base_module_allreduce_fn_t coll_allreduce;
-    mca_coll_base_module_2_4_0_t *coll_allreduce_module;
+    mca_coll_base_module_3_0_0_t *coll_allreduce_module;
     mca_coll_base_module_alltoall_fn_t coll_alltoall;
-    mca_coll_base_module_2_4_0_t *coll_alltoall_module;
+    mca_coll_base_module_3_0_0_t *coll_alltoall_module;
     mca_coll_base_module_alltoallv_fn_t coll_alltoallv;
-    mca_coll_base_module_2_4_0_t *coll_alltoallv_module;
+    mca_coll_base_module_3_0_0_t *coll_alltoallv_module;
     mca_coll_base_module_alltoallw_fn_t coll_alltoallw;
-    mca_coll_base_module_2_4_0_t *coll_alltoallw_module;
+    mca_coll_base_module_3_0_0_t *coll_alltoallw_module;
     mca_coll_base_module_barrier_fn_t coll_barrier;
-    mca_coll_base_module_2_4_0_t *coll_barrier_module;
+    mca_coll_base_module_3_0_0_t *coll_barrier_module;
     mca_coll_base_module_bcast_fn_t coll_bcast;
-    mca_coll_base_module_2_4_0_t *coll_bcast_module;
+    mca_coll_base_module_3_0_0_t *coll_bcast_module;
     mca_coll_base_module_exscan_fn_t coll_exscan;
-    mca_coll_base_module_2_4_0_t *coll_exscan_module;
+    mca_coll_base_module_3_0_0_t *coll_exscan_module;
     mca_coll_base_module_gather_fn_t coll_gather;
-    mca_coll_base_module_2_4_0_t *coll_gather_module;
+    mca_coll_base_module_3_0_0_t *coll_gather_module;
     mca_coll_base_module_gatherv_fn_t coll_gatherv;
-    mca_coll_base_module_2_4_0_t *coll_gatherv_module;
+    mca_coll_base_module_3_0_0_t *coll_gatherv_module;
     mca_coll_base_module_reduce_fn_t coll_reduce;
-    mca_coll_base_module_2_4_0_t *coll_reduce_module;
+    mca_coll_base_module_3_0_0_t *coll_reduce_module;
     mca_coll_base_module_reduce_scatter_fn_t coll_reduce_scatter;
-    mca_coll_base_module_2_4_0_t *coll_reduce_scatter_module;
+    mca_coll_base_module_3_0_0_t *coll_reduce_scatter_module;
     mca_coll_base_module_reduce_scatter_block_fn_t coll_reduce_scatter_block;
-    mca_coll_base_module_2_4_0_t *coll_reduce_scatter_block_module;
+    mca_coll_base_module_3_0_0_t *coll_reduce_scatter_block_module;
     mca_coll_base_module_scan_fn_t coll_scan;
-    mca_coll_base_module_2_4_0_t *coll_scan_module;
+    mca_coll_base_module_3_0_0_t *coll_scan_module;
     mca_coll_base_module_scatter_fn_t coll_scatter;
-    mca_coll_base_module_2_4_0_t *coll_scatter_module;
+    mca_coll_base_module_3_0_0_t *coll_scatter_module;
     mca_coll_base_module_scatterv_fn_t coll_scatterv;
-    mca_coll_base_module_2_4_0_t *coll_scatterv_module;
+    mca_coll_base_module_3_0_0_t *coll_scatterv_module;
 
     /* nonblocking collectives */
     mca_coll_base_module_iallgather_fn_t coll_iallgather;
-    mca_coll_base_module_2_4_0_t *coll_iallgather_module;
+    mca_coll_base_module_3_0_0_t *coll_iallgather_module;
     mca_coll_base_module_iallgatherv_fn_t coll_iallgatherv;
-    mca_coll_base_module_2_4_0_t *coll_iallgatherv_module;
+    mca_coll_base_module_3_0_0_t *coll_iallgatherv_module;
     mca_coll_base_module_iallreduce_fn_t coll_iallreduce;
-    mca_coll_base_module_2_4_0_t *coll_iallreduce_module;
+    mca_coll_base_module_3_0_0_t *coll_iallreduce_module;
     mca_coll_base_module_ialltoall_fn_t coll_ialltoall;
-    mca_coll_base_module_2_4_0_t *coll_ialltoall_module;
+    mca_coll_base_module_3_0_0_t *coll_ialltoall_module;
     mca_coll_base_module_ialltoallv_fn_t coll_ialltoallv;
-    mca_coll_base_module_2_4_0_t *coll_ialltoallv_module;
+    mca_coll_base_module_3_0_0_t *coll_ialltoallv_module;
     mca_coll_base_module_ialltoallw_fn_t coll_ialltoallw;
-    mca_coll_base_module_2_4_0_t *coll_ialltoallw_module;
+    mca_coll_base_module_3_0_0_t *coll_ialltoallw_module;
     mca_coll_base_module_ibarrier_fn_t coll_ibarrier;
-    mca_coll_base_module_2_4_0_t *coll_ibarrier_module;
+    mca_coll_base_module_3_0_0_t *coll_ibarrier_module;
     mca_coll_base_module_ibcast_fn_t coll_ibcast;
-    mca_coll_base_module_2_4_0_t *coll_ibcast_module;
+    mca_coll_base_module_3_0_0_t *coll_ibcast_module;
     mca_coll_base_module_iexscan_fn_t coll_iexscan;
-    mca_coll_base_module_2_4_0_t *coll_iexscan_module;
+    mca_coll_base_module_3_0_0_t *coll_iexscan_module;
     mca_coll_base_module_igather_fn_t coll_igather;
-    mca_coll_base_module_2_4_0_t *coll_igather_module;
+    mca_coll_base_module_3_0_0_t *coll_igather_module;
     mca_coll_base_module_igatherv_fn_t coll_igatherv;
-    mca_coll_base_module_2_4_0_t *coll_igatherv_module;
+    mca_coll_base_module_3_0_0_t *coll_igatherv_module;
     mca_coll_base_module_ireduce_fn_t coll_ireduce;
-    mca_coll_base_module_2_4_0_t *coll_ireduce_module;
+    mca_coll_base_module_3_0_0_t *coll_ireduce_module;
     mca_coll_base_module_ireduce_scatter_fn_t coll_ireduce_scatter;
-    mca_coll_base_module_2_4_0_t *coll_ireduce_scatter_module;
+    mca_coll_base_module_3_0_0_t *coll_ireduce_scatter_module;
     mca_coll_base_module_ireduce_scatter_block_fn_t coll_ireduce_scatter_block;
-    mca_coll_base_module_2_4_0_t *coll_ireduce_scatter_block_module;
+    mca_coll_base_module_3_0_0_t *coll_ireduce_scatter_block_module;
     mca_coll_base_module_iscan_fn_t coll_iscan;
-    mca_coll_base_module_2_4_0_t *coll_iscan_module;
+    mca_coll_base_module_3_0_0_t *coll_iscan_module;
     mca_coll_base_module_iscatter_fn_t coll_iscatter;
-    mca_coll_base_module_2_4_0_t *coll_iscatter_module;
+    mca_coll_base_module_3_0_0_t *coll_iscatter_module;
     mca_coll_base_module_iscatterv_fn_t coll_iscatterv;
-    mca_coll_base_module_2_4_0_t *coll_iscatterv_module;
+    mca_coll_base_module_3_0_0_t *coll_iscatterv_module;
 
     /* persistent collectives */
     mca_coll_base_module_allgather_init_fn_t coll_allgather_init;
-    mca_coll_base_module_2_4_0_t *coll_allgather_init_module;
+    mca_coll_base_module_3_0_0_t *coll_allgather_init_module;
     mca_coll_base_module_allgatherv_init_fn_t coll_allgatherv_init;
-    mca_coll_base_module_2_4_0_t *coll_allgatherv_init_module;
+    mca_coll_base_module_3_0_0_t *coll_allgatherv_init_module;
     mca_coll_base_module_allreduce_init_fn_t coll_allreduce_init;
-    mca_coll_base_module_2_4_0_t *coll_allreduce_init_module;
+    mca_coll_base_module_3_0_0_t *coll_allreduce_init_module;
     mca_coll_base_module_alltoall_init_fn_t coll_alltoall_init;
-    mca_coll_base_module_2_4_0_t *coll_alltoall_init_module;
+    mca_coll_base_module_3_0_0_t *coll_alltoall_init_module;
     mca_coll_base_module_alltoallv_init_fn_t coll_alltoallv_init;
-    mca_coll_base_module_2_4_0_t *coll_alltoallv_init_module;
+    mca_coll_base_module_3_0_0_t *coll_alltoallv_init_module;
     mca_coll_base_module_alltoallw_init_fn_t coll_alltoallw_init;
-    mca_coll_base_module_2_4_0_t *coll_alltoallw_init_module;
+    mca_coll_base_module_3_0_0_t *coll_alltoallw_init_module;
     mca_coll_base_module_barrier_init_fn_t coll_barrier_init;
-    mca_coll_base_module_2_4_0_t *coll_barrier_init_module;
+    mca_coll_base_module_3_0_0_t *coll_barrier_init_module;
     mca_coll_base_module_bcast_init_fn_t coll_bcast_init;
-    mca_coll_base_module_2_4_0_t *coll_bcast_init_module;
+    mca_coll_base_module_3_0_0_t *coll_bcast_init_module;
     mca_coll_base_module_exscan_init_fn_t coll_exscan_init;
-    mca_coll_base_module_2_4_0_t *coll_exscan_init_module;
+    mca_coll_base_module_3_0_0_t *coll_exscan_init_module;
     mca_coll_base_module_gather_init_fn_t coll_gather_init;
-    mca_coll_base_module_2_4_0_t *coll_gather_init_module;
+    mca_coll_base_module_3_0_0_t *coll_gather_init_module;
     mca_coll_base_module_gatherv_init_fn_t coll_gatherv_init;
-    mca_coll_base_module_2_4_0_t *coll_gatherv_init_module;
+    mca_coll_base_module_3_0_0_t *coll_gatherv_init_module;
     mca_coll_base_module_reduce_init_fn_t coll_reduce_init;
-    mca_coll_base_module_2_4_0_t *coll_reduce_init_module;
+    mca_coll_base_module_3_0_0_t *coll_reduce_init_module;
     mca_coll_base_module_reduce_scatter_init_fn_t coll_reduce_scatter_init;
-    mca_coll_base_module_2_4_0_t *coll_reduce_scatter_init_module;
+    mca_coll_base_module_3_0_0_t *coll_reduce_scatter_init_module;
     mca_coll_base_module_reduce_scatter_block_init_fn_t coll_reduce_scatter_block_init;
-    mca_coll_base_module_2_4_0_t *coll_reduce_scatter_block_init_module;
+    mca_coll_base_module_3_0_0_t *coll_reduce_scatter_block_init_module;
     mca_coll_base_module_scan_init_fn_t coll_scan_init;
-    mca_coll_base_module_2_4_0_t *coll_scan_init_module;
+    mca_coll_base_module_3_0_0_t *coll_scan_init_module;
     mca_coll_base_module_scatter_init_fn_t coll_scatter_init;
-    mca_coll_base_module_2_4_0_t *coll_scatter_init_module;
+    mca_coll_base_module_3_0_0_t *coll_scatter_init_module;
     mca_coll_base_module_scatterv_init_fn_t coll_scatterv_init;
-    mca_coll_base_module_2_4_0_t *coll_scatterv_init_module;
+    mca_coll_base_module_3_0_0_t *coll_scatterv_init_module;
 
     /* blocking neighborhood collectives */
     mca_coll_base_module_allgather_fn_t coll_neighbor_allgather;
-    mca_coll_base_module_2_4_0_t *coll_neighbor_allgather_module;
+    mca_coll_base_module_3_0_0_t *coll_neighbor_allgather_module;
     mca_coll_base_module_allgatherv_fn_t coll_neighbor_allgatherv;
-    mca_coll_base_module_2_4_0_t *coll_neighbor_allgatherv_module;
+    mca_coll_base_module_3_0_0_t *coll_neighbor_allgatherv_module;
     mca_coll_base_module_alltoall_fn_t coll_neighbor_alltoall;
-    mca_coll_base_module_2_4_0_t *coll_neighbor_alltoall_module;
+    mca_coll_base_module_3_0_0_t *coll_neighbor_alltoall_module;
     mca_coll_base_module_alltoallv_fn_t coll_neighbor_alltoallv;
-    mca_coll_base_module_2_4_0_t *coll_neighbor_alltoallv_module;
+    mca_coll_base_module_3_0_0_t *coll_neighbor_alltoallv_module;
     mca_coll_base_module_neighbor_alltoallw_fn_t coll_neighbor_alltoallw;
-    mca_coll_base_module_2_4_0_t *coll_neighbor_alltoallw_module;
+    mca_coll_base_module_3_0_0_t *coll_neighbor_alltoallw_module;
 
     /* nonblocking neighborhood collectives */
     mca_coll_base_module_iallgather_fn_t coll_ineighbor_allgather;
-    mca_coll_base_module_2_4_0_t *coll_ineighbor_allgather_module;
+    mca_coll_base_module_3_0_0_t *coll_ineighbor_allgather_module;
     mca_coll_base_module_iallgatherv_fn_t coll_ineighbor_allgatherv;
-    mca_coll_base_module_2_4_0_t *coll_ineighbor_allgatherv_module;
+    mca_coll_base_module_3_0_0_t *coll_ineighbor_allgatherv_module;
     mca_coll_base_module_ialltoall_fn_t coll_ineighbor_alltoall;
-    mca_coll_base_module_2_4_0_t *coll_ineighbor_alltoall_module;
+    mca_coll_base_module_3_0_0_t *coll_ineighbor_alltoall_module;
     mca_coll_base_module_ialltoallv_fn_t coll_ineighbor_alltoallv;
-    mca_coll_base_module_2_4_0_t *coll_ineighbor_alltoallv_module;
+    mca_coll_base_module_3_0_0_t *coll_ineighbor_alltoallv_module;
     mca_coll_base_module_ineighbor_alltoallw_fn_t coll_ineighbor_alltoallw;
-    mca_coll_base_module_2_4_0_t *coll_ineighbor_alltoallw_module;
+    mca_coll_base_module_3_0_0_t *coll_ineighbor_alltoallw_module;
 
     /* persistent neighborhood collectives */
     mca_coll_base_module_allgather_init_fn_t coll_neighbor_allgather_init;
-    mca_coll_base_module_2_4_0_t *coll_neighbor_allgather_init_module;
+    mca_coll_base_module_3_0_0_t *coll_neighbor_allgather_init_module;
     mca_coll_base_module_allgatherv_init_fn_t coll_neighbor_allgatherv_init;
-    mca_coll_base_module_2_4_0_t *coll_neighbor_allgatherv_init_module;
+    mca_coll_base_module_3_0_0_t *coll_neighbor_allgatherv_init_module;
     mca_coll_base_module_alltoall_init_fn_t coll_neighbor_alltoall_init;
-    mca_coll_base_module_2_4_0_t *coll_neighbor_alltoall_init_module;
+    mca_coll_base_module_3_0_0_t *coll_neighbor_alltoall_init_module;
     mca_coll_base_module_alltoallv_init_fn_t coll_neighbor_alltoallv_init;
-    mca_coll_base_module_2_4_0_t *coll_neighbor_alltoallv_init_module;
+    mca_coll_base_module_3_0_0_t *coll_neighbor_alltoallv_init_module;
     mca_coll_base_module_neighbor_alltoallw_init_fn_t coll_neighbor_alltoallw_init;
-    mca_coll_base_module_2_4_0_t *coll_neighbor_alltoallw_init_module;
+    mca_coll_base_module_3_0_0_t *coll_neighbor_alltoallw_init_module;
 
     mca_coll_base_module_reduce_local_fn_t coll_reduce_local;
-    mca_coll_base_module_2_4_0_t *coll_reduce_local_module;
+    mca_coll_base_module_3_0_0_t *coll_reduce_local_module;
 
     mca_coll_base_module_agree_fn_t coll_agree;
-    mca_coll_base_module_2_4_0_t *coll_agree_module;
+    mca_coll_base_module_3_0_0_t *coll_agree_module;
     mca_coll_base_module_iagree_fn_t coll_iagree;
-    mca_coll_base_module_2_4_0_t *coll_iagree_module;
+    mca_coll_base_module_3_0_0_t *coll_iagree_module;
 
     /* List of modules initialized, queried and enabled */
     opal_list_t *module_list;
@@ -802,12 +814,35 @@ typedef struct mca_coll_base_comm_coll_t mca_coll_base_comm_coll_t;
 /*
  * Macro for use in components that are of type coll
  */
-#define MCA_COLL_BASE_VERSION_2_4_0 \
-    OMPI_MCA_BASE_VERSION_2_1_0("coll", 2, 4, 0)
+#define MCA_COLL_BASE_VERSION_3_0_0 \
+    OMPI_MCA_BASE_VERSION_2_1_0("coll", 3, 0, 0)
 
 
 /* ******************************************************************** */
 
+#define MCA_COLL_SAVE_API(__comm, __api, __fct, __bmodule, __c_name)           \
+  do                                                                           \
+  {                                                                            \
+      OPAL_OUTPUT_VERBOSE((50, ompi_coll_base_framework.framework_output,      \
+                           "Save %s collective in comm %p (%s) %p into %s:%s", \
+                           #__api, (void *)__comm, __comm->c_name,             \
+                           (void*)(uintptr_t)__comm->c_coll->coll_##__api,     \
+                           __c_name, #__fct));                                 \
+      __fct = __comm->c_coll->coll_##__api;                                    \
+      __bmodule = __comm->c_coll->coll_##__api##_module;                       \
+  } while (0)
+
+#define MCA_COLL_INSTALL_API(__comm, __api, __fct, __bmodule, __c_name)                \
+  do                                                                                   \
+  {                                                                                    \
+    OPAL_OUTPUT_VERBOSE((50, ompi_coll_base_framework.framework_output,                \
+                         "Replace %s collective in comm %p (%s) from %p to %s:%s(%p)", \
+                         #__api, (void *)__comm, __comm->c_name,                       \
+                         (void*)(uintptr_t)__comm->c_coll->coll_##__api,               \
+                         __c_name, #__fct, (void*)(uintptr_t)__fct));                  \
+    __comm->c_coll->coll_##__api = __fct;                                              \
+    __comm->c_coll->coll_##__api##_module = __bmodule;                                 \
+  } while (0)
 
 END_C_DECLS
 
diff --git a/ompi/mca/coll/cuda/Makefile.am b/ompi/mca/coll/cuda/Makefile.am
deleted file mode 100644
index 74a6ecfd947..00000000000
--- a/ompi/mca/coll/cuda/Makefile.am
+++ /dev/null
@@ -1,40 +0,0 @@
-#
-# Copyright (c) 2014      The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2014      NVIDIA Corporation.  All rights reserved.
-# Copyright (c) 2017      IBM Corporation.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-dist_ompidata_DATA = help-mpi-coll-cuda.txt
-
-sources = coll_cuda_module.c coll_cuda_reduce.c coll_cuda_allreduce.c \
-          coll_cuda_reduce_scatter_block.c coll_cuda_component.c \
-          coll_cuda_scan.c coll_cuda_exscan.c coll_cuda.h
-
-# Make the output library in this directory, and name it either
-# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
-# (for static builds).
-
-if MCA_BUILD_ompi_coll_cuda_DSO
-component_noinst =
-component_install = mca_coll_cuda.la
-else
-component_noinst = libmca_coll_cuda.la
-component_install =
-endif
-
-mcacomponentdir = $(ompilibdir)
-mcacomponent_LTLIBRARIES = $(component_install)
-mca_coll_cuda_la_SOURCES = $(sources)
-mca_coll_cuda_la_LDFLAGS = -module -avoid-version
-mca_coll_cuda_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la
-
-noinst_LTLIBRARIES = $(component_noinst)
-libmca_coll_cuda_la_SOURCES =$(sources)
-libmca_coll_cuda_la_LDFLAGS = -module -avoid-version
-
diff --git a/ompi/mca/coll/cuda/coll_cuda.h b/ompi/mca/coll/cuda/coll_cuda.h
deleted file mode 100644
index 0d0e88501b0..00000000000
--- a/ompi/mca/coll/cuda/coll_cuda.h
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Copyright (c) 2014      The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2014-2015 NVIDIA Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef MCA_COLL_CUDA_EXPORT_H
-#define MCA_COLL_CUDA_EXPORT_H
-
-#include "ompi_config.h"
-
-#include "mpi.h"
-
-#include "opal/class/opal_object.h"
-#include "ompi/mca/mca.h"
-
-#include "ompi/constants.h"
-#include "ompi/mca/coll/coll.h"
-#include "ompi/mca/coll/base/base.h"
-#include "ompi/communicator/communicator.h"
-
-BEGIN_C_DECLS
-
-/* API functions */
-
-int mca_coll_cuda_init_query(bool enable_progress_threads,
-                             bool enable_mpi_threads);
-mca_coll_base_module_t
-*mca_coll_cuda_comm_query(struct ompi_communicator_t *comm,
-                          int *priority);
-
-int mca_coll_cuda_module_enable(mca_coll_base_module_t *module,
-                                struct ompi_communicator_t *comm);
-
-int
-mca_coll_cuda_allreduce(const void *sbuf, void *rbuf, int count,
-                        struct ompi_datatype_t *dtype,
-                        struct ompi_op_t *op,
-                        struct ompi_communicator_t *comm,
-                        mca_coll_base_module_t *module);
-
-int mca_coll_cuda_reduce(const void *sbuf, void *rbuf, int count,
-                         struct ompi_datatype_t *dtype,
-                         struct ompi_op_t *op,
-                         int root,
-                         struct ompi_communicator_t *comm,
-                         mca_coll_base_module_t *module);
-
-int mca_coll_cuda_exscan(const void *sbuf, void *rbuf, int count,
-                         struct ompi_datatype_t *dtype,
-                         struct ompi_op_t *op,
-                         struct ompi_communicator_t *comm,
-                         mca_coll_base_module_t *module);
-
-int mca_coll_cuda_scan(const void *sbuf, void *rbuf, int count,
-                       struct ompi_datatype_t *dtype,
-                       struct ompi_op_t *op,
-                       struct ompi_communicator_t *comm,
-                       mca_coll_base_module_t *module);
-
-int
-mca_coll_cuda_reduce_scatter_block(const void *sbuf, void *rbuf, int rcount,
-                                   struct ompi_datatype_t *dtype,
-                                   struct ompi_op_t *op,
-                                   struct ompi_communicator_t *comm,
-                                   mca_coll_base_module_t *module);
-
-/* Types */
-/* Module */
-
-typedef struct mca_coll_cuda_module_t {
-    mca_coll_base_module_t super;
-
-    /* Pointers to all the "real" collective functions */
-    mca_coll_base_comm_coll_t c_coll;
-} mca_coll_cuda_module_t;
-
-OBJ_CLASS_DECLARATION(mca_coll_cuda_module_t);
-
-/* Component */
-
-typedef struct mca_coll_cuda_component_t {
-    mca_coll_base_component_2_4_0_t super;
-
-    int priority; /* Priority of this component */
-    int disable_cuda_coll;  /* Force disable of the CUDA collective component */
-} mca_coll_cuda_component_t;
-
-/* Globally exported variables */
-
-OMPI_MODULE_DECLSPEC extern mca_coll_cuda_component_t mca_coll_cuda_component;
-
-END_C_DECLS
-
-#endif /* MCA_COLL_CUDA_EXPORT_H */
diff --git a/ompi/mca/coll/cuda/coll_cuda_allreduce.c b/ompi/mca/coll/cuda/coll_cuda_allreduce.c
deleted file mode 100644
index 416c9c7fa8f..00000000000
--- a/ompi/mca/coll/cuda/coll_cuda_allreduce.c
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Copyright (c) 2014-2017 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2014-2015 NVIDIA Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include "coll_cuda.h"
-
-#include <stdio.h>
-
-#include "ompi/op/op.h"
-#include "opal/datatype/opal_convertor.h"
-#include "opal/mca/common/cuda/common_cuda.h"
-
-/*
- *	allreduce_intra
- *
- *	Function:	- allreduce using other MPI collectives
- *	Accepts:	- same as MPI_Allreduce()
- *	Returns:	- MPI_SUCCESS or error code
- */
-int
-mca_coll_cuda_allreduce(const void *sbuf, void *rbuf, int count,
-                        struct ompi_datatype_t *dtype,
-                        struct ompi_op_t *op,
-                        struct ompi_communicator_t *comm,
-                        mca_coll_base_module_t *module)
-{
-    mca_coll_cuda_module_t *s = (mca_coll_cuda_module_t*) module;
-    ptrdiff_t gap;
-    char *rbuf1 = NULL, *sbuf1 = NULL, *rbuf2 = NULL;
-    size_t bufsize;
-    int rc;
-
-    bufsize = opal_datatype_span(&dtype->super, count, &gap);
-
-    if ((MPI_IN_PLACE != sbuf) && (opal_cuda_check_bufs((char *)sbuf, NULL))) {
-        sbuf1 = (char*)malloc(bufsize);
-        if (NULL == sbuf1) {
-            return OMPI_ERR_OUT_OF_RESOURCE;
-        }
-        opal_cuda_memcpy_sync(sbuf1, sbuf, bufsize);
-        sbuf = sbuf1 - gap;
-    }
-
-    if (opal_cuda_check_bufs(rbuf, NULL)) {
-        rbuf1 = (char*)malloc(bufsize);
-        if (NULL == rbuf1) {
-            if (NULL != sbuf1) free(sbuf1);
-            return OMPI_ERR_OUT_OF_RESOURCE;
-        }
-        opal_cuda_memcpy_sync(rbuf1, rbuf, bufsize);
-        rbuf2 = rbuf; /* save away original buffer */
-        rbuf = rbuf1 - gap;
-    }
-    rc = s->c_coll.coll_allreduce(sbuf, rbuf, count, dtype, op, comm, s->c_coll.coll_allreduce_module);
-    if (NULL != sbuf1) {
-        free(sbuf1);
-    }
-    if (NULL != rbuf1) {
-        rbuf = rbuf2;
-        opal_cuda_memcpy_sync(rbuf, rbuf1, bufsize);
-        free(rbuf1);
-    }
-    return rc;
-}
-
diff --git a/ompi/mca/coll/cuda/coll_cuda_component.c b/ompi/mca/coll/cuda/coll_cuda_component.c
deleted file mode 100644
index 4f44d8bafeb..00000000000
--- a/ompi/mca/coll/cuda/coll_cuda_component.c
+++ /dev/null
@@ -1,91 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2014      The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2014-2015 NVIDIA Corporation.  All rights reserved.
- * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <string.h>
-
-#include "mpi.h"
-#include "ompi/constants.h"
-#include "coll_cuda.h"
-
-/*
- * Public string showing the coll ompi_cuda component version number
- */
-const char *mca_coll_cuda_component_version_string =
-    "Open MPI cuda collective MCA component version " OMPI_VERSION;
-
-/*
- * Local function
- */
-static int cuda_register(void);
-
-/*
- * Instantiate the public struct with all of our public information
- * and pointers to our public functions in it
- */
-
-mca_coll_cuda_component_t mca_coll_cuda_component = {
-    {
-        /* First, the mca_component_t struct containing meta information
-         * about the component itself */
-
-        .collm_version = {
-            MCA_COLL_BASE_VERSION_2_4_0,
-
-            /* Component name and version */
-            .mca_component_name = "cuda",
-            MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION,
-                                  OMPI_RELEASE_VERSION),
-
-            /* Component open and close functions */
-            .mca_register_component_params = cuda_register,
-        },
-        .collm_data = {
-            /* The component is checkpoint ready */
-            MCA_BASE_METADATA_PARAM_CHECKPOINT
-        },
-
-        /* Initialization / querying functions */
-
-        .collm_init_query = mca_coll_cuda_init_query,
-        .collm_comm_query = mca_coll_cuda_comm_query,
-    },
-
-    /* cuda-specific component information */
-
-    /* Priority: make it above all point to point collectives including self */
-    78,
-};
-
-
-static int cuda_register(void)
-{
-    (void) mca_base_component_var_register(&mca_coll_cuda_component.super.collm_version,
-                                           "priority", "Priority of the cuda coll component; only relevant if barrier_before or barrier_after is > 0",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_6,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
-                                           &mca_coll_cuda_component.priority);
-
-    (void) mca_base_component_var_register(&mca_coll_cuda_component.super.collm_version,
-                                           "disable_cuda_coll", "Automatically handle the CUDA buffers for the MPI collective.",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_2,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
-                                           &mca_coll_cuda_component.disable_cuda_coll);
-
-    return OMPI_SUCCESS;
-}
diff --git a/ompi/mca/coll/cuda/coll_cuda_exscan.c b/ompi/mca/coll/cuda/coll_cuda_exscan.c
deleted file mode 100644
index 5f736697fe0..00000000000
--- a/ompi/mca/coll/cuda/coll_cuda_exscan.c
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Copyright (c) 2014-2017 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2014-2015 NVIDIA Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include "coll_cuda.h"
-
-#include <stdio.h>
-
-#include "ompi/op/op.h"
-#include "opal/datatype/opal_convertor.h"
-#include "opal/mca/common/cuda/common_cuda.h"
-
-int mca_coll_cuda_exscan(const void *sbuf, void *rbuf, int count,
-                         struct ompi_datatype_t *dtype,
-                         struct ompi_op_t *op,
-                         struct ompi_communicator_t *comm,
-                         mca_coll_base_module_t *module)
-{
-    mca_coll_cuda_module_t *s = (mca_coll_cuda_module_t*) module;
-    ptrdiff_t gap;
-    char *rbuf1 = NULL, *sbuf1 = NULL, *rbuf2 = NULL;
-    size_t bufsize;
-    int rc;
-
-    bufsize = opal_datatype_span(&dtype->super, count, &gap);
-
-    if ((MPI_IN_PLACE != sbuf) && (opal_cuda_check_bufs((char *)sbuf, NULL))) {
-        sbuf1 = (char*)malloc(bufsize);
-        if (NULL == sbuf1) {
-            return OMPI_ERR_OUT_OF_RESOURCE;
-        }
-        opal_cuda_memcpy_sync(sbuf1, sbuf, bufsize);
-        sbuf = sbuf1 - gap;
-    }
-
-    if (opal_cuda_check_bufs(rbuf, NULL)) {
-        rbuf1 = (char*)malloc(bufsize);
-        if (NULL == rbuf1) {
-            if (NULL != sbuf1) free(sbuf1);
-            return OMPI_ERR_OUT_OF_RESOURCE;
-        }
-        opal_cuda_memcpy_sync(rbuf1, rbuf, bufsize);
-        rbuf2 = rbuf; /* save away original buffer */
-        rbuf = rbuf1 - gap;
-    }
-
-    rc = s->c_coll.coll_exscan(sbuf, rbuf, count, dtype, op, comm,
-                               s->c_coll.coll_exscan_module);
-    if (NULL != sbuf1) {
-        free(sbuf1);
-    }
-    if (NULL != rbuf1) {
-        rbuf = rbuf2;
-        opal_cuda_memcpy_sync(rbuf, rbuf1, bufsize);
-        free(rbuf1);
-    }
-    return rc;
-}
diff --git a/ompi/mca/coll/cuda/coll_cuda_module.c b/ompi/mca/coll/cuda/coll_cuda_module.c
deleted file mode 100644
index b8f2ddf5e90..00000000000
--- a/ompi/mca/coll/cuda/coll_cuda_module.c
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Copyright (c) 2014-2017 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2014      NVIDIA Corporation.  All rights reserved.
- * Copyright (c) 2019      Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <string.h>
-#include <stdio.h>
-
-#include "coll_cuda.h"
-
-#include "mpi.h"
-
-
-#include "opal/util/show_help.h"
-
-#include "ompi/constants.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/mca/coll/coll.h"
-#include "ompi/mca/coll/base/base.h"
-#include "coll_cuda.h"
-
-
-static void mca_coll_cuda_module_construct(mca_coll_cuda_module_t *module)
-{
-    memset(&(module->c_coll), 0, sizeof(module->c_coll));
-}
-
-static void mca_coll_cuda_module_destruct(mca_coll_cuda_module_t *module)
-{
-    OBJ_RELEASE(module->c_coll.coll_allreduce_module);
-    OBJ_RELEASE(module->c_coll.coll_reduce_module);
-    OBJ_RELEASE(module->c_coll.coll_reduce_scatter_block_module);
-    OBJ_RELEASE(module->c_coll.coll_scatter_module);
-    /* If the exscan module is not NULL, then this was an
-       intracommunicator, and therefore scan will have a module as
-       well. */
-    if (NULL != module->c_coll.coll_exscan_module) {
-        OBJ_RELEASE(module->c_coll.coll_exscan_module);
-        OBJ_RELEASE(module->c_coll.coll_scan_module);
-    }
-}
-
-OBJ_CLASS_INSTANCE(mca_coll_cuda_module_t, mca_coll_base_module_t,
-                   mca_coll_cuda_module_construct,
-                   mca_coll_cuda_module_destruct);
-
-
-/*
- * Initial query function that is invoked during MPI_INIT, allowing
- * this component to disqualify itself if it doesn't support the
- * required level of thread support.
- */
-int mca_coll_cuda_init_query(bool enable_progress_threads,
-                             bool enable_mpi_threads)
-{
-    /* Nothing to do */
-
-    return OMPI_SUCCESS;
-}
-
-
-/*
- * Invoked when there's a new communicator that has been created.
- * Look at the communicator and decide which set of functions and
- * priority we want to return.
- */
-mca_coll_base_module_t *
-mca_coll_cuda_comm_query(struct ompi_communicator_t *comm,
-                         int *priority)
-{
-    mca_coll_cuda_module_t *cuda_module;
-
-    cuda_module = OBJ_NEW(mca_coll_cuda_module_t);
-    if (NULL == cuda_module) {
-        return NULL;
-    }
-
-    *priority = mca_coll_cuda_component.priority;
-
-    /* Choose whether to use [intra|inter] */
-    cuda_module->super.coll_module_enable = mca_coll_cuda_module_enable;
-
-    cuda_module->super.coll_allgather  = NULL;
-    cuda_module->super.coll_allgatherv = NULL;
-    cuda_module->super.coll_allreduce  = mca_coll_cuda_allreduce;
-    cuda_module->super.coll_alltoall   = NULL;
-    cuda_module->super.coll_alltoallv  = NULL;
-    cuda_module->super.coll_alltoallw  = NULL;
-    cuda_module->super.coll_barrier    = NULL;
-    cuda_module->super.coll_bcast      = NULL;
-    cuda_module->super.coll_exscan     = mca_coll_cuda_exscan;
-    cuda_module->super.coll_gather     = NULL;
-    cuda_module->super.coll_gatherv    = NULL;
-    cuda_module->super.coll_reduce     = mca_coll_cuda_reduce;
-    cuda_module->super.coll_reduce_scatter = NULL;
-    cuda_module->super.coll_reduce_scatter_block = mca_coll_cuda_reduce_scatter_block;
-    cuda_module->super.coll_scan       = mca_coll_cuda_scan;
-    cuda_module->super.coll_scatter    = NULL;
-    cuda_module->super.coll_scatterv   = NULL;
-
-    return &(cuda_module->super);
-}
-
-
-/*
- * Init module on the communicator
- */
-int mca_coll_cuda_module_enable(mca_coll_base_module_t *module,
-                                struct ompi_communicator_t *comm)
-{
-    bool good = true;
-    char *msg = NULL;
-    mca_coll_cuda_module_t *s = (mca_coll_cuda_module_t*) module;
-
-#define CHECK_AND_RETAIN(src, dst, name)                                                   \
-    if (NULL == (src)->c_coll->coll_ ## name ## _module) {                                 \
-        good = false;                                                                      \
-        msg = #name;                                                                       \
-    } else if (good) {                                                                     \
-        (dst)->c_coll.coll_ ## name ## _module = (src)->c_coll->coll_ ## name ## _module;  \
-        (dst)->c_coll.coll_ ## name = (src)->c_coll->coll_ ## name;                        \
-        OBJ_RETAIN((src)->c_coll->coll_ ## name ## _module);                               \
-    }
-
-    CHECK_AND_RETAIN(comm, s, allreduce);
-    CHECK_AND_RETAIN(comm, s, reduce);
-    CHECK_AND_RETAIN(comm, s, reduce_scatter_block);
-    CHECK_AND_RETAIN(comm, s, scatter);
-    if (!OMPI_COMM_IS_INTER(comm)) {
-        /* MPI does not define scan/exscan on intercommunicators */
-        CHECK_AND_RETAIN(comm, s, exscan);
-        CHECK_AND_RETAIN(comm, s, scan);
-    }
-
-    /* All done */
-    if (good) {
-        return OMPI_SUCCESS;
-    }
-    opal_show_help("help-mpi-coll-cuda.txt", "missing collective", true,
-                   ompi_process_info.nodename,
-                   mca_coll_cuda_component.priority, msg);
-    return OMPI_ERR_NOT_FOUND;
-}
-
diff --git a/ompi/mca/coll/cuda/coll_cuda_reduce.c b/ompi/mca/coll/cuda/coll_cuda_reduce.c
deleted file mode 100644
index 5d82667b6bb..00000000000
--- a/ompi/mca/coll/cuda/coll_cuda_reduce.c
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Copyright (c) 2004-2015 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2014-2015 NVIDIA Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include "coll_cuda.h"
-
-#include <stdio.h>
-
-#include "ompi/op/op.h"
-#include "opal/datatype/opal_convertor.h"
-#include "opal/mca/common/cuda/common_cuda.h"
-
-/*
- *	reduce_log_inter
- *
- *	Function:	- reduction using O(N) algorithm
- *	Accepts:	- same as MPI_Reduce()
- *	Returns:	- MPI_SUCCESS or error code
- */
-int
-mca_coll_cuda_reduce(const void *sbuf, void *rbuf, int count,
-                     struct ompi_datatype_t *dtype,
-                     struct ompi_op_t *op,
-                     int root, struct ompi_communicator_t *comm,
-                     mca_coll_base_module_t *module)
-{
-    mca_coll_cuda_module_t *s = (mca_coll_cuda_module_t*) module;
-    ptrdiff_t gap;
-    char *rbuf1 = NULL, *sbuf1 = NULL, *rbuf2 = NULL;
-    const char *sbuf2;
-    size_t bufsize;
-    int rc;
-
-    bufsize = opal_datatype_span(&dtype->super, count, &gap);
-
-
-    if ((MPI_IN_PLACE != sbuf) && (opal_cuda_check_bufs((char *)sbuf, NULL))) {
-        sbuf1 = (char*)malloc(bufsize);
-        if (NULL == sbuf1) {
-            return OMPI_ERR_OUT_OF_RESOURCE;
-        }
-        opal_cuda_memcpy_sync(sbuf1, sbuf, bufsize);
-        sbuf2 = sbuf; /* save away original buffer */
-        sbuf = sbuf1 - gap;
-    }
-
-    if (opal_cuda_check_bufs(rbuf, NULL)) {
-        rbuf1 = (char*)malloc(bufsize);
-        if (NULL == rbuf1) {
-            if (NULL != sbuf1) free(sbuf1);
-            return OMPI_ERR_OUT_OF_RESOURCE;
-        }
-        opal_cuda_memcpy_sync(rbuf1, rbuf, bufsize);
-        rbuf2 = rbuf; /* save away original buffer */
-        rbuf = rbuf1 - gap;
-    }
-    rc = s->c_coll.coll_reduce((void *) sbuf, rbuf, count,
-                               dtype, op, root, comm,
-                               s->c_coll.coll_reduce_module);
-
-    if (NULL != sbuf1) {
-        free(sbuf1);
-    }
-    if (NULL != rbuf1) {
-        rbuf = rbuf2;
-        opal_cuda_memcpy_sync(rbuf, rbuf1, bufsize);
-        free(rbuf1);
-    }
-    return rc;
-}
diff --git a/ompi/mca/coll/cuda/coll_cuda_reduce_scatter_block.c b/ompi/mca/coll/cuda/coll_cuda_reduce_scatter_block.c
deleted file mode 100644
index 907257b0da8..00000000000
--- a/ompi/mca/coll/cuda/coll_cuda_reduce_scatter_block.c
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Copyright (c) 2014-2017 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2014-2015 NVIDIA Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include "coll_cuda.h"
-
-#include <stdio.h>
-
-#include "ompi/op/op.h"
-#include "opal/datatype/opal_convertor.h"
-#include "opal/mca/common/cuda/common_cuda.h"
-
-/*
- *	reduce_scatter_block
- *
- *	Function:	- reduce then scatter
- *	Accepts:	- same as MPI_Reduce_scatter_block()
- *	Returns:	- MPI_SUCCESS or error code
- *
- * Algorithm:
- *     reduce and scatter (needs to be cleaned
- *     up at some point)
- */
-int
-mca_coll_cuda_reduce_scatter_block(const void *sbuf, void *rbuf, int rcount,
-                                   struct ompi_datatype_t *dtype,
-                                   struct ompi_op_t *op,
-                                   struct ompi_communicator_t *comm,
-                                   mca_coll_base_module_t *module)
-{
-    mca_coll_cuda_module_t *s = (mca_coll_cuda_module_t*) module;
-    ptrdiff_t gap;
-    char *rbuf1 = NULL, *sbuf1 = NULL, *rbuf2 = NULL;
-    size_t sbufsize, rbufsize;
-    int rc;
-
-    rbufsize = opal_datatype_span(&dtype->super, rcount, &gap);
-
-    sbufsize = rbufsize * ompi_comm_size(comm);
-
-    if ((MPI_IN_PLACE != sbuf) && (opal_cuda_check_bufs((char *)sbuf, NULL))) {
-        sbuf1 = (char*)malloc(sbufsize);
-        if (NULL == sbuf1) {
-            return OMPI_ERR_OUT_OF_RESOURCE;
-        }
-        opal_cuda_memcpy_sync(sbuf1, sbuf, sbufsize);
-        sbuf = sbuf1 - gap;
-    }
-
-    if (opal_cuda_check_bufs(rbuf, NULL)) {
-        rbuf1 = (char*)malloc(rbufsize);
-        if (NULL == rbuf1) {
-            if (NULL != sbuf1) free(sbuf1);
-            return OMPI_ERR_OUT_OF_RESOURCE;
-        }
-        opal_cuda_memcpy_sync(rbuf1, rbuf, rbufsize);
-        rbuf2 = rbuf; /* save away original buffer */
-        rbuf = rbuf1 - gap;
-    }
-    rc = s->c_coll.coll_reduce_scatter_block(sbuf, rbuf, rcount, dtype, op, comm,
-                                             s->c_coll.coll_reduce_scatter_block_module);
-    if (NULL != sbuf1) {
-        free(sbuf1);
-    }
-    if (NULL != rbuf1) {
-        rbuf = rbuf2;
-        opal_cuda_memcpy_sync(rbuf, rbuf1, rbufsize);
-        free(rbuf1);
-    }
-    return rc;
-}
-
diff --git a/ompi/mca/coll/cuda/coll_cuda_scan.c b/ompi/mca/coll/cuda/coll_cuda_scan.c
deleted file mode 100644
index 4e7300c12f8..00000000000
--- a/ompi/mca/coll/cuda/coll_cuda_scan.c
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Copyright (c) 2014-2017 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2014-2015 NVIDIA Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include "coll_cuda.h"
-
-#include <stdio.h>
-
-#include "ompi/op/op.h"
-#include "opal/datatype/opal_convertor.h"
-#include "opal/mca/common/cuda/common_cuda.h"
-
-/*
- *	scan
- *
- *	Function:	- scan
- *	Accepts:	- same arguments as MPI_Scan()
- *	Returns:	- MPI_SUCCESS or error code
- */
-int mca_coll_cuda_scan(const void *sbuf, void *rbuf, int count,
-                       struct ompi_datatype_t *dtype,
-                       struct ompi_op_t *op,
-                       struct ompi_communicator_t *comm,
-                       mca_coll_base_module_t *module)
-{
-    mca_coll_cuda_module_t *s = (mca_coll_cuda_module_t*) module;
-    ptrdiff_t gap;
-    char *rbuf1 = NULL, *sbuf1 = NULL, *rbuf2 = NULL;
-    size_t bufsize;
-    int rc;
-
-    bufsize = opal_datatype_span(&dtype->super, count, &gap);
-
-    if ((MPI_IN_PLACE != sbuf) && (opal_cuda_check_bufs((char *)sbuf, NULL))) {
-        sbuf1 = (char*)malloc(bufsize);
-        if (NULL == sbuf1) {
-            return OMPI_ERR_OUT_OF_RESOURCE;
-        }
-        opal_cuda_memcpy_sync(sbuf1, sbuf, bufsize);
-        sbuf = sbuf1 - gap;
-    }
-
-    if (opal_cuda_check_bufs(rbuf, NULL)) {
-        rbuf1 = (char*)malloc(bufsize);
-        if (NULL == rbuf1) {
-            if (NULL != sbuf1) free(sbuf1);
-            return OMPI_ERR_OUT_OF_RESOURCE;
-        }
-        opal_cuda_memcpy_sync(rbuf1, rbuf, bufsize);
-        rbuf2 = rbuf; /* save away original buffer */
-        rbuf = rbuf1 - gap;
-    }
-    rc = s->c_coll.coll_scan(sbuf, rbuf, count, dtype, op, comm,
-                             s->c_coll.coll_scan_module);
-    if (NULL != sbuf1) {
-        free(sbuf1);
-    }
-    if (NULL != rbuf1) {
-        rbuf = rbuf2;
-        opal_cuda_memcpy_sync(rbuf, rbuf1, bufsize);
-        free(rbuf1);
-    }
-    return rc;
-}
diff --git a/ompi/mca/coll/cuda/configure.m4 b/ompi/mca/coll/cuda/configure.m4
deleted file mode 100644
index 5f041c74a09..00000000000
--- a/ompi/mca/coll/cuda/configure.m4
+++ /dev/null
@@ -1,29 +0,0 @@
-# -*- shell-script -*-
-#
-# Copyright (c) 2014      The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2014      NVIDIA Corporation.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-# MCA_coll_cuda_CONFIG([action-if-can-compile],
-#                      [action-if-cant-compile])
-# ------------------------------------------------
-AC_DEFUN([MCA_ompi_coll_cuda_CONFIG],[
-    AC_CONFIG_FILES([ompi/mca/coll/cuda/Makefile])
-
-    # make sure that CUDA-aware checks have been done
-    AC_REQUIRE([OPAL_CHECK_CUDA])
-
-    # Only build if CUDA support is available
-    AS_IF([test "x$CUDA_SUPPORT" = "x1"],
-          [$1],
-          [$2])
-
-])dnl
-
diff --git a/ompi/mca/coll/cuda/help-mpi-coll-cuda.txt b/ompi/mca/coll/cuda/help-mpi-coll-cuda.txt
deleted file mode 100644
index 5537420ab4e..00000000000
--- a/ompi/mca/coll/cuda/help-mpi-coll-cuda.txt
+++ /dev/null
@@ -1,28 +0,0 @@
-# -*- text -*-
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2014      NVIDIA Corporation.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-# This is the US/English general help file for Open MPI's CUDA
-# collective reduction component.
-#
-[missing collective]
-There was a problem while initializing support for the CUDA reduction operations.
-hostname:   %s
-priority:   %d
-collective: %s
-#
diff --git a/ompi/mca/coll/cuda/owner.txt b/ompi/mca/coll/cuda/owner.txt
deleted file mode 100644
index 35d26b460ce..00000000000
--- a/ompi/mca/coll/cuda/owner.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-#
-# owner/status file
-# owner: institution that is responsible for this package
-# status: e.g. active, maintenance, unmaintained
-#
-owner: NVIDIA
-status: maintenance
diff --git a/ompi/mca/coll/demo/coll_demo.h b/ompi/mca/coll/demo/coll_demo.h
index 3b3621f7c50..925c41d5e82 100644
--- a/ompi/mca/coll/demo/coll_demo.h
+++ b/ompi/mca/coll/demo/coll_demo.h
@@ -10,6 +10,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -30,7 +31,7 @@ BEGIN_C_DECLS
 
     /* Globally exported variables */
 
-OMPI_MODULE_DECLSPEC extern const mca_coll_base_component_2_4_0_t mca_coll_demo_component;
+    OMPI_DECLSPEC extern const mca_coll_base_component_3_0_0_t mca_coll_demo_component;
     extern int mca_coll_demo_priority;
     extern int mca_coll_demo_verbose;
 
@@ -39,86 +40,83 @@ OMPI_MODULE_DECLSPEC extern const mca_coll_base_component_2_4_0_t mca_coll_demo_
 
     int mca_coll_demo_init_query(bool enable_progress_threads,
                                  bool enable_mpi_threads);
-mca_coll_base_module_t *
-mca_coll_demo_comm_query(struct ompi_communicator_t *comm, int *priority);
+    mca_coll_base_module_t *
+    mca_coll_demo_comm_query(struct ompi_communicator_t *comm, int *priority);
 
     /* Module functions */
 
-int mca_coll_demo_module_enable(mca_coll_base_module_t *module,
-                                struct ompi_communicator_t *comm);
-
-    int mca_coll_demo_allgather_intra(void *sbuf, int scount,
+    int mca_coll_demo_allgather_intra(const void *sbuf, size_t scount,
                                       struct ompi_datatype_t *sdtype,
-                                      void *rbuf, int rcount,
+                                      void *rbuf, size_t rcount,
                                       struct ompi_datatype_t *rdtype,
                                       struct ompi_communicator_t *comm,
                                       mca_coll_base_module_t *module);
-    int mca_coll_demo_allgather_inter(void *sbuf, int scount,
+    int mca_coll_demo_allgather_inter(const void *sbuf, size_t scount,
                                       struct ompi_datatype_t *sdtype,
-                                      void *rbuf, int rcount,
+                                      void *rbuf, size_t rcount,
                                       struct ompi_datatype_t *rdtype,
                                       struct ompi_communicator_t *comm,
                                       mca_coll_base_module_t *module);
 
-    int mca_coll_demo_allgatherv_intra(void *sbuf, int scount,
+    int mca_coll_demo_allgatherv_intra(const void *sbuf, size_t scount,
                                        struct ompi_datatype_t *sdtype,
-                                       void * rbuf, int *rcounts, int *disps,
+                                       void * rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
                                        struct ompi_datatype_t *rdtype,
                                        struct ompi_communicator_t *comm,
                                        mca_coll_base_module_t *module);
-    int mca_coll_demo_allgatherv_inter(void *sbuf, int scount,
+    int mca_coll_demo_allgatherv_inter(const void *sbuf, int scount,
                                        struct ompi_datatype_t *sdtype,
-                                       void * rbuf, int *rcounts, int *disps,
+                                       void * rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
                                        struct ompi_datatype_t *rdtype,
                                        struct ompi_communicator_t *comm,
                                        mca_coll_base_module_t *module);
 
-    int mca_coll_demo_allreduce_intra(void *sbuf, void *rbuf, int count,
+    int mca_coll_demo_allreduce_intra(const void *sbuf, void *rbuf, size_t count,
                                       struct ompi_datatype_t *dtype,
                                       struct ompi_op_t *op,
                                       struct ompi_communicator_t *comm,
                                       mca_coll_base_module_t *module);
-    int mca_coll_demo_allreduce_inter(void *sbuf, void *rbuf, int count,
+    int mca_coll_demo_allreduce_inter(const void *sbuf, void *rbuf, size_t count,
                                       struct ompi_datatype_t *dtype,
                                       struct ompi_op_t *op,
                                       struct ompi_communicator_t *comm,
                                       mca_coll_base_module_t *module);
 
-    int mca_coll_demo_alltoall_intra(void *sbuf, int scount,
+    int mca_coll_demo_alltoall_intra(const void *sbuf, size_t scount,
                                      struct ompi_datatype_t *sdtype,
-                                     void* rbuf, int rcount,
+                                     void* rbuf, size_t rcount,
                                      struct ompi_datatype_t *rdtype,
                                      struct ompi_communicator_t *comm,
                                      mca_coll_base_module_t *module);
-    int mca_coll_demo_alltoall_inter(void *sbuf, int scount,
+    int mca_coll_demo_alltoall_inter(const void *sbuf, size_t scount,
                                      struct ompi_datatype_t *sdtype,
-                                     void* rbuf, int rcount,
+                                     void* rbuf, size_t rcount,
                                      struct ompi_datatype_t *rdtype,
                                      struct ompi_communicator_t *comm,
                                      mca_coll_base_module_t *module);
 
-    int mca_coll_demo_alltoallv_intra(void *sbuf, int *scounts, int *sdisps,
+    int mca_coll_demo_alltoallv_intra(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
                                       struct ompi_datatype_t *sdtype,
-                                      void *rbuf, int *rcounts, int *rdisps,
+                                      void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
                                       struct ompi_datatype_t *rdtype,
                                       struct ompi_communicator_t *comm,
                                       mca_coll_base_module_t *module);
-    int mca_coll_demo_alltoallv_inter(void *sbuf, int *scounts, int *sdisps,
+    int mca_coll_demo_alltoallv_inter(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
                                       struct ompi_datatype_t *sdtype,
-                                      void *rbuf, int *rcounts, int *rdisps,
+                                      void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
                                       struct ompi_datatype_t *rdtype,
                                       struct ompi_communicator_t *comm,
                                       mca_coll_base_module_t *module);
 
-    int mca_coll_demo_alltoallw_intra(void *sbuf, int *scounts, int *sdisps,
+    int mca_coll_demo_alltoallw_intra(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
                                       struct ompi_datatype_t **sdtypes,
-                                      void *rbuf, int *rcounts, int *rdisps,
+                                      void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
                                       struct ompi_datatype_t **rdtypes,
                                       struct ompi_communicator_t *comm,
                                       mca_coll_base_module_t *module);
-    int mca_coll_demo_alltoallw_inter(void *sbuf, int *scounts, int *sdisps,
+    int mca_coll_demo_alltoallw_inter(void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
                                       struct ompi_datatype_t **sdtypes,
-                                      void *rbuf, int *rcounts, int *rdisps,
+                                      void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
                                       struct ompi_datatype_t **rdtypes,
                                       struct ompi_communicator_t *comm,
                                       mca_coll_base_module_t *module);
@@ -128,120 +126,120 @@ int mca_coll_demo_module_enable(mca_coll_base_module_t *module,
     int mca_coll_demo_barrier_inter(struct ompi_communicator_t *comm,
                                     mca_coll_base_module_t *module);
 
-    int mca_coll_demo_bcast_intra(void *buff, int count,
+    int mca_coll_demo_bcast_intra(void *buff, size_t count,
                                   struct ompi_datatype_t *datatype,
                                   int root,
                                   struct ompi_communicator_t *comm,
                                   mca_coll_base_module_t *module);
-    int mca_coll_demo_bcast_inter(void *buff, int count,
+    int mca_coll_demo_bcast_inter(void *buff, size_t count,
                                   struct ompi_datatype_t *datatype,
                                   int root,
                                   struct ompi_communicator_t *comm,
                                   mca_coll_base_module_t *module);
 
-    int mca_coll_demo_exscan_intra(void *sbuf, void *rbuf, int count,
+    int mca_coll_demo_exscan_intra(const void *sbuf, void *rbuf, size_t count,
                                    struct ompi_datatype_t *dtype,
                                    struct ompi_op_t *op,
                                    struct ompi_communicator_t *comm,
                                    mca_coll_base_module_t *module);
-    int mca_coll_demo_exscan_inter(void *sbuf, void *rbuf, int count,
+    int mca_coll_demo_exscan_inter(const void *sbuf, void *rbuf, size_t count,
                                    struct ompi_datatype_t *dtype,
                                    struct ompi_op_t *op,
                                    struct ompi_communicator_t *comm,
                                    mca_coll_base_module_t *module);
 
-    int mca_coll_demo_gather_intra(void *sbuf, int scount,
+    int mca_coll_demo_gather_intra(const void *sbuf, size_t scount,
                                    struct ompi_datatype_t *sdtype, void *rbuf,
-                                   int rcount, struct ompi_datatype_t *rdtype,
+                                   size_t rcount, struct ompi_datatype_t *rdtype,
                                    int root, struct ompi_communicator_t *comm,
                                    mca_coll_base_module_t *module);
-    int mca_coll_demo_gather_inter(void *sbuf, int scount,
+    int mca_coll_demo_gather_inter(const void *sbuf, size_t scount,
                                    struct ompi_datatype_t *sdtype, void *rbuf,
-                                   int rcount, struct ompi_datatype_t *rdtype,
+                                   size_t rcount, struct ompi_datatype_t *rdtype,
                                    int root, struct ompi_communicator_t *comm,
                                    mca_coll_base_module_t *module);
 
-    int mca_coll_demo_gatherv_intra(void *sbuf, int scount,
+    int mca_coll_demo_gatherv_intra(const void *sbuf, int scount,
                                     struct ompi_datatype_t *sdtype, void *rbuf,
-                                    int *rcounts, int *disps,
+                                    ompi_count_array_t rcounts, ompi_disp_array_t disps,
                                     struct ompi_datatype_t *rdtype, int root,
                                     struct ompi_communicator_t *comm,
                                     mca_coll_base_module_t *module);
     int mca_coll_demo_gatherv_inter(void *sbuf, int scount,
                                     struct ompi_datatype_t *sdtype, void *rbuf,
-                                    int *rcounts, int *disps,
+                                    ompi_count_array_t rcounts, ompi_disp_array_t disps,
                                     struct ompi_datatype_t *rdtype, int root,
                                     struct ompi_communicator_t *comm,
                                     mca_coll_base_module_t *module);
 
-    int mca_coll_demo_reduce_intra(void *sbuf, void* rbuf, int count,
+    int mca_coll_demo_reduce_intra(const void *sbuf, void* rbuf, size_t count,
                                    struct ompi_datatype_t *dtype,
                                    struct ompi_op_t *op,
                                    int root,
                                    struct ompi_communicator_t *comm,
                                    mca_coll_base_module_t *module);
-    int mca_coll_demo_reduce_inter(void *sbuf, void* rbuf, int count,
+    int mca_coll_demo_reduce_inter(const void *sbuf, void* rbuf, size_t count,
                                    struct ompi_datatype_t *dtype,
                                    struct ompi_op_t *op,
                                    int root,
                                    struct ompi_communicator_t *comm,
                                    mca_coll_base_module_t *module);
 
-    int mca_coll_demo_reduce_scatter_intra(void *sbuf, void *rbuf,
-                                           int *rcounts,
+    int mca_coll_demo_reduce_scatter_intra(const void *sbuf, void *rbuf,
+                                           ompi_count_array_t rcounts,
                                            struct ompi_datatype_t *dtype,
                                            struct ompi_op_t *op,
                                            struct ompi_communicator_t *comm,
                                            mca_coll_base_module_t *module);
-    int mca_coll_demo_reduce_scatter_inter(void *sbuf, void *rbuf,
-                                           int *rcounts,
+    int mca_coll_demo_reduce_scatter_inter(const void *sbuf, void *rbuf,
+                                           ompi_count_array_t rcounts,
                                            struct ompi_datatype_t *dtype,
                                            struct ompi_op_t *op,
                                            struct ompi_communicator_t *comm,
                                            mca_coll_base_module_t *module);
 
-    int mca_coll_demo_scan_intra(void *sbuf, void *rbuf, int count,
+    int mca_coll_demo_scan_intra(const void *sbuf, void *rbuf, size_t count,
                                  struct ompi_datatype_t *dtype,
                                  struct ompi_op_t *op,
                                  struct ompi_communicator_t *comm,
                                  mca_coll_base_module_t *module);
-    int mca_coll_demo_scan_inter(void *sbuf, void *rbuf, int count,
+    int mca_coll_demo_scan_inter(const void *sbuf, void *rbuf, size_t count,
                                  struct ompi_datatype_t *dtype,
                                  struct ompi_op_t *op,
                                  struct ompi_communicator_t *comm,
                                  mca_coll_base_module_t *module);
 
-    int mca_coll_demo_scatter_intra(void *sbuf, int scount,
+    int mca_coll_demo_scatter_intra(const void *sbuf, size_t scount,
                                     struct ompi_datatype_t *sdtype, void *rbuf,
-                                    int rcount, struct ompi_datatype_t *rdtype,
+                                    size_t rcount, struct ompi_datatype_t *rdtype,
                                     int root, struct ompi_communicator_t *comm,
                                     mca_coll_base_module_t *module);
-    int mca_coll_demo_scatter_inter(void *sbuf, int scount,
+    int mca_coll_demo_scatter_inter(const void *sbuf, size_t scount,
                                     struct ompi_datatype_t *sdtype, void *rbuf,
-                                    int rcount, struct ompi_datatype_t *rdtype,
+                                    size_t rcount, struct ompi_datatype_t *rdtype,
                                     int root, struct ompi_communicator_t *comm,
                                     mca_coll_base_module_t *module);
 
-    int mca_coll_demo_scatterv_intra(void *sbuf, int *scounts, int *disps,
+    int mca_coll_demo_scatterv_intra(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t disps,
                                      struct ompi_datatype_t *sdtype,
-                                     void* rbuf, int rcount,
+                                     void* rbuf, size_t rcount,
                                      struct ompi_datatype_t *rdtype, int root,
                                      struct ompi_communicator_t *comm,
                                      mca_coll_base_module_t *module);
-    int mca_coll_demo_scatterv_inter(void *sbuf, int *scounts, int *disps,
+    int mca_coll_demo_scatterv_inter(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t disps,
                                      struct ompi_datatype_t *sdtype,
-                                     void* rbuf, int rcount,
+                                     void* rbuf, size_t rcount,
                                      struct ompi_datatype_t *rdtype, int root,
                                      struct ompi_communicator_t *comm,
                                      mca_coll_base_module_t *module);
 
-struct mca_coll_demo_module_t {
-    mca_coll_base_module_t super;
+    struct mca_coll_demo_module_t {
+        mca_coll_base_module_t super;
 
-    mca_coll_base_comm_coll_t underlying;
-};
-typedef struct mca_coll_demo_module_t mca_coll_demo_module_t;
-OBJ_CLASS_DECLARATION(mca_coll_demo_module_t);
+        mca_coll_base_comm_coll_t c_coll;
+    };
+    typedef struct mca_coll_demo_module_t mca_coll_demo_module_t;
+    OBJ_CLASS_DECLARATION(mca_coll_demo_module_t);
 
 
 END_C_DECLS
diff --git a/ompi/mca/coll/demo/coll_demo_allgather.c b/ompi/mca/coll/demo/coll_demo_allgather.c
index f33042a71d5..14b6973de68 100644
--- a/ompi/mca/coll/demo/coll_demo_allgather.c
+++ b/ompi/mca/coll/demo/coll_demo_allgather.c
@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -33,17 +34,17 @@
  *	Accepts:	- same as MPI_Allgather()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_demo_allgather_intra(void *sbuf, int scount,
+int mca_coll_demo_allgather_intra(const void *sbuf, size_t scount,
                                   struct ompi_datatype_t *sdtype, void *rbuf,
-                                  int rcount, struct ompi_datatype_t *rdtype,
+                                  size_t rcount, struct ompi_datatype_t *rdtype,
                                   struct ompi_communicator_t *comm,
                                   mca_coll_base_module_t *module)
 {
     mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "In demo allgather_intra");
-    return demo_module->underlying.coll_allgather(sbuf, scount, sdtype, rbuf,
-                                                  rcount, rdtype, comm,
-                                                  demo_module->underlying.coll_allgather_module);
+    return demo_module->c_coll.coll_allgather(sbuf, scount, sdtype, rbuf,
+                                              rcount, rdtype, comm,
+                                              demo_module->c_coll.coll_allgather_module);
 }
 
 
@@ -54,16 +55,16 @@ int mca_coll_demo_allgather_intra(void *sbuf, int scount,
  *	Accepts:	- same as MPI_Allgather()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_demo_allgather_inter(void *sbuf, int scount,
+int mca_coll_demo_allgather_inter(const void *sbuf, size_t scount,
                                   struct ompi_datatype_t *sdtype,
-                                  void *rbuf, int rcount,
+                                  void *rbuf, size_t rcount,
                                   struct ompi_datatype_t *rdtype,
                                   struct ompi_communicator_t *comm,
                                   mca_coll_base_module_t *module)
 {
     mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "In demo allgather_inter");
-    return demo_module->underlying.coll_allgather(sbuf, scount, sdtype, rbuf,
-                                                  rcount, rdtype, comm,
-                                                  demo_module->underlying.coll_allgather_module);
+    return demo_module->c_coll.coll_allgather(sbuf, scount, sdtype, rbuf,
+                                              rcount, rdtype, comm,
+                                              demo_module->c_coll.coll_allgather_module);
 }
diff --git a/ompi/mca/coll/demo/coll_demo_allgatherv.c b/ompi/mca/coll/demo/coll_demo_allgatherv.c
index b6503ec6865..283ccc6522a 100644
--- a/ompi/mca/coll/demo/coll_demo_allgatherv.c
+++ b/ompi/mca/coll/demo/coll_demo_allgatherv.c
@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -33,19 +34,19 @@
  *	Accepts:	- same as MPI_Allgatherv()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_demo_allgatherv_intra(void *sbuf, int scount,
+int mca_coll_demo_allgatherv_intra(const void *sbuf, int scount,
                                    struct ompi_datatype_t *sdtype,
-                                   void * rbuf, int *rcounts, int *disps,
+                                   void * rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
                                    struct ompi_datatype_t *rdtype,
                                    struct ompi_communicator_t *comm,
                                    mca_coll_base_module_t *module)
 {
     mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "In demo allgatherv_intra");
-    return demo_module->underlying.coll_allgatherv(sbuf, scount, sdtype,
-                                                   rbuf, rcounts, disps,
-                                                   rdtype, comm,
-                                                   demo_module->underlying.coll_allgatherv_module);
+    return demo_module->c_coll.coll_allgatherv(sbuf, scount, sdtype,
+                                               rbuf, rcounts, disps,
+                                               rdtype, comm,
+                                               demo_module->c_coll.coll_allgatherv_module);
 }
 
 
@@ -56,17 +57,17 @@ int mca_coll_demo_allgatherv_intra(void *sbuf, int scount,
  *	Accepts:	- same as MPI_Allgatherv()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_demo_allgatherv_inter(void *sbuf, int scount,
-                                    struct ompi_datatype_t *sdtype,
-                                    void * rbuf, int *rcounts, int *disps,
-                                    struct ompi_datatype_t *rdtype,
+int mca_coll_demo_allgatherv_inter(const void *sbuf, int scount,
+                                   struct ompi_datatype_t *sdtype,
+                                   void * rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
+                                   struct ompi_datatype_t *rdtype,
                                    struct ompi_communicator_t *comm,
                                    mca_coll_base_module_t *module)
 {
     mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "In demo allgatherv_inter");
-    return demo_module->underlying.coll_allgatherv(sbuf, scount, sdtype,
-                                                   rbuf, rcounts, disps,
-                                                   rdtype, comm,
-                                                   demo_module->underlying.coll_allgatherv_module);
+    return demo_module->c_coll.coll_allgatherv(sbuf, scount, sdtype,
+                                               rbuf, rcounts, disps,
+                                               rdtype, comm,
+                                               demo_module->c_coll.coll_allgatherv_module);
 }
diff --git a/ompi/mca/coll/demo/coll_demo_allreduce.c b/ompi/mca/coll/demo/coll_demo_allreduce.c
index 15975bacb1c..7f042651156 100644
--- a/ompi/mca/coll/demo/coll_demo_allreduce.c
+++ b/ompi/mca/coll/demo/coll_demo_allreduce.c
@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -33,7 +34,7 @@
  *	Accepts:	- same as MPI_Allreduce()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_demo_allreduce_intra(void *sbuf, void *rbuf, int count,
+int mca_coll_demo_allreduce_intra(const void *sbuf, void *rbuf, size_t count,
                                   struct ompi_datatype_t *dtype,
                                   struct ompi_op_t *op,
                                   struct ompi_communicator_t *comm,
@@ -41,9 +42,9 @@ int mca_coll_demo_allreduce_intra(void *sbuf, void *rbuf, int count,
 {
     mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "In demo allreduce_intra");
-    return demo_module->underlying.coll_allreduce(sbuf, rbuf, count, dtype,
-                                                  op, comm,
-                                                  demo_module->underlying.coll_allreduce_module);
+    return demo_module->c_coll.coll_allreduce(sbuf, rbuf, count, dtype,
+                                              op, comm,
+                                              demo_module->c_coll.coll_allreduce_module);
 }
 
 
@@ -54,7 +55,7 @@ int mca_coll_demo_allreduce_intra(void *sbuf, void *rbuf, int count,
  *	Accepts:	- same as MPI_Allreduce()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_demo_allreduce_inter(void *sbuf, void *rbuf, int count,
+int mca_coll_demo_allreduce_inter(const void *sbuf, void *rbuf, size_t count,
                                   struct ompi_datatype_t *dtype,
                                   struct ompi_op_t *op,
                                   struct ompi_communicator_t *comm,
@@ -62,7 +63,7 @@ int mca_coll_demo_allreduce_inter(void *sbuf, void *rbuf, int count,
 {
     mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "In demo allreduce_inter");
-    return demo_module->underlying.coll_allreduce(sbuf, rbuf, count, dtype,
-                                                  op, comm,
-                                                  demo_module->underlying.coll_allreduce_module);
+    return demo_module->c_coll.coll_allreduce(sbuf, rbuf, count, dtype,
+                                              op, comm,
+                                              demo_module->c_coll.coll_allreduce_module);
 }
diff --git a/ompi/mca/coll/demo/coll_demo_alltoall.c b/ompi/mca/coll/demo/coll_demo_alltoall.c
index d3559970121..3c0bd5bfae7 100644
--- a/ompi/mca/coll/demo/coll_demo_alltoall.c
+++ b/ompi/mca/coll/demo/coll_demo_alltoall.c
@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -33,19 +34,19 @@
  *	Accepts:	- same as MPI_Alltoall()
  *	Returns:	- MPI_SUCCESS or an MPI error code
  */
-int mca_coll_demo_alltoall_intra(void *sbuf, int scount,
+int mca_coll_demo_alltoall_intra(const void *sbuf, size_t scount,
                                  struct ompi_datatype_t *sdtype,
-                                 void *rbuf, int rcount,
+                                 void *rbuf, size_t rcount,
                                  struct ompi_datatype_t *rdtype,
                                  struct ompi_communicator_t *comm,
                                  mca_coll_base_module_t *module)
 {
     mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "In demo alltoall_intra\n");
-    return demo_module->underlying.coll_alltoall(sbuf, scount, sdtype,
-                                                 rbuf, rcount, rdtype,
-                                                 comm,
-                                                 demo_module->underlying.coll_alltoall_module);
+    return demo_module->c_coll.coll_alltoall(sbuf, scount, sdtype,
+                                             rbuf, rcount, rdtype,
+                                             comm,
+                                             demo_module->c_coll.coll_alltoall_module);
 }
 
 
@@ -56,17 +57,17 @@ int mca_coll_demo_alltoall_intra(void *sbuf, int scount,
  *	Accepts:	- same as MPI_Alltoall()
  *	Returns:	- MPI_SUCCESS or an MPI error code
  */
-int mca_coll_demo_alltoall_inter(void *sbuf, int scount,
+int mca_coll_demo_alltoall_inter(const void *sbuf, size_t scount,
                                  struct ompi_datatype_t *sdtype,
-                                 void *rbuf, int rcount,
+                                 void *rbuf, size_t rcount,
                                  struct ompi_datatype_t *rdtype,
                                  struct ompi_communicator_t *comm,
                                  mca_coll_base_module_t *module)
 {
     mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "In demo alltoall_inter\n");
-    return demo_module->underlying.coll_alltoall(sbuf, scount, sdtype,
-                                                 rbuf, rcount, rdtype,
-                                                 comm,
-                                                 demo_module->underlying.coll_alltoall_module);
+    return demo_module->c_coll.coll_alltoall(sbuf, scount, sdtype,
+                                             rbuf, rcount, rdtype,
+                                             comm,
+                                             demo_module->c_coll.coll_alltoall_module);
 }
diff --git a/ompi/mca/coll/demo/coll_demo_alltoallv.c b/ompi/mca/coll/demo/coll_demo_alltoallv.c
index 0e8cf13861b..a1100235b73 100644
--- a/ompi/mca/coll/demo/coll_demo_alltoallv.c
+++ b/ompi/mca/coll/demo/coll_demo_alltoallv.c
@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -34,19 +35,19 @@
  *	Returns:	- MPI_SUCCESS or an MPI error code
  */
 int
-mca_coll_demo_alltoallv_intra(void *sbuf, int *scounts, int *sdisps,
+mca_coll_demo_alltoallv_intra(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
                               struct ompi_datatype_t *sdtype,
-                              void *rbuf, int *rcounts, int *rdisps,
+                              void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
                               struct ompi_datatype_t *rdtype,
                               struct ompi_communicator_t *comm,
                               mca_coll_base_module_t *module)
 {
     mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "In demo alltoallv_intra");
-    return demo_module->underlying.coll_alltoallv(sbuf, scounts, sdisps,
-                                                  sdtype, rbuf, rcounts,
-                                                  rdisps, rdtype, comm,
-                                                  demo_module->underlying.coll_alltoallv_module);
+    return demo_module->c_coll.coll_alltoallv(sbuf, scounts, sdisps,
+                                              sdtype, rbuf, rcounts,
+                                              rdisps, rdtype, comm,
+                                              demo_module->c_coll.coll_alltoallv_module);
 }
 
 
@@ -58,17 +59,17 @@ mca_coll_demo_alltoallv_intra(void *sbuf, int *scounts, int *sdisps,
  *	Returns:	- MPI_SUCCESS or an MPI error code
  */
 int
-mca_coll_demo_alltoallv_inter(void *sbuf, int *scounts, int *sdisps,
+mca_coll_demo_alltoallv_inter(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
                               struct ompi_datatype_t *sdtype, void *rbuf,
-                              int *rcounts, int *rdisps,
+                              ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
                               struct ompi_datatype_t *rdtype,
                               struct ompi_communicator_t *comm,
                               mca_coll_base_module_t *module)
 {
     mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "In demo alltoallv_inter");
-    return demo_module->underlying.coll_alltoallv(sbuf, scounts, sdisps,
-                                                  sdtype, rbuf, rcounts,
-                                                  rdisps, rdtype, comm,
-                                                  demo_module->underlying.coll_alltoallv_module);
+    return demo_module->c_coll.coll_alltoallv(sbuf, scounts, sdisps,
+                                              sdtype, rbuf, rcounts,
+                                              rdisps, rdtype, comm,
+                                              demo_module->c_coll.coll_alltoallv_module);
 }
diff --git a/ompi/mca/coll/demo/coll_demo_alltoallw.c b/ompi/mca/coll/demo/coll_demo_alltoallw.c
index b9c29693178..6c806808be6 100644
--- a/ompi/mca/coll/demo/coll_demo_alltoallw.c
+++ b/ompi/mca/coll/demo/coll_demo_alltoallw.c
@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -33,19 +34,19 @@
  *	Accepts:	- same as MPI_Alltoallw()
  *	Returns:	- MPI_SUCCESS or an MPI error code
  */
-int mca_coll_demo_alltoallw_intra(void *sbuf, int *scounts, int *sdisps,
-                                  struct ompi_datatype_t **sdtypes,
-                                  void *rbuf, int *rcounts, int *rdisps,
-                                  struct ompi_datatype_t **rdtypes,
+int mca_coll_demo_alltoallw_intra(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
+                                  struct ompi_datatype_t *const *sdtypes,
+                                  void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
+                                  struct ompi_datatype_t *const *rdtypes,
                                   struct ompi_communicator_t *comm,
                                   mca_coll_base_module_t *module)
 {
     mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "In demo alltoallw_intra");
-    return demo_module->underlying.coll_alltoallw(sbuf, scounts, sdisps,
-                                                  sdtypes, rbuf, rcounts,
-                                                  rdisps, rdtypes, comm,
-                                                  demo_module->underlying.coll_alltoallw_module);
+    return demo_module->c_coll.coll_alltoallw(sbuf, scounts, sdisps,
+                                              sdtypes, rbuf, rcounts,
+                                              rdisps, rdtypes, comm,
+                                              demo_module->c_coll.coll_alltoallw_module);
 }
 
 
@@ -56,17 +57,17 @@ int mca_coll_demo_alltoallw_intra(void *sbuf, int *scounts, int *sdisps,
  *	Accepts:	- same as MPI_Alltoallw()
  *	Returns:	- MPI_SUCCESS or an MPI error code
  */
-int mca_coll_demo_alltoallw_inter(void *sbuf, int *scounts, int *sdisps,
-                                  struct ompi_datatype_t **sdtypes,
-                                  void *rbuf, int *rcounts, int *rdisps,
-                                  struct ompi_datatype_t **rdtypes,
+int mca_coll_demo_alltoallw_inter(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
+                                  struct ompi_datatype_t *const *sdtypes,
+                                  void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
+                                  struct ompi_datatype_t *const *rdtypes,
                                   struct ompi_communicator_t *comm,
                                   mca_coll_base_module_t *module)
 {
     mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "In demo alltoallw_inter");
-    return demo_module->underlying.coll_alltoallw(sbuf, scounts, sdisps,
-                                                  sdtypes, rbuf, rcounts,
-                                                  rdisps, rdtypes, comm,
-                                                  demo_module->underlying.coll_alltoallw_module);
+    return demo_module->c_coll.coll_alltoallw(sbuf, scounts, sdisps,
+                                              sdtypes, rbuf, rcounts,
+                                              rdisps, rdtypes, comm,
+                                              demo_module->c_coll.coll_alltoallw_module);
 }
diff --git a/ompi/mca/coll/demo/coll_demo_barrier.c b/ompi/mca/coll/demo/coll_demo_barrier.c
index bcede2bf5b5..7965b8856ae 100644
--- a/ompi/mca/coll/demo/coll_demo_barrier.c
+++ b/ompi/mca/coll/demo/coll_demo_barrier.c
@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -38,8 +39,8 @@ int mca_coll_demo_barrier_intra(struct ompi_communicator_t *comm,
 {
     mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "In demo barrier_intra");
-    return demo_module->underlying.coll_barrier(comm,
-                                                demo_module->underlying.coll_barrier_module);
+    return demo_module->c_coll.coll_barrier(comm,
+                                            demo_module->c_coll.coll_barrier_module);
 }
 
 
@@ -55,6 +56,6 @@ int mca_coll_demo_barrier_inter(struct ompi_communicator_t *comm,
 {
     mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "In demo barrier_inter");
-    return demo_module->underlying.coll_barrier(comm,
-                                                demo_module->underlying.coll_barrier_module);
+    return demo_module->c_coll.coll_barrier(comm,
+                                            demo_module->c_coll.coll_barrier_module);
 }
diff --git a/ompi/mca/coll/demo/coll_demo_bcast.c b/ompi/mca/coll/demo/coll_demo_bcast.c
index 645c9e0dd62..2950a274def 100644
--- a/ompi/mca/coll/demo/coll_demo_bcast.c
+++ b/ompi/mca/coll/demo/coll_demo_bcast.c
@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -33,16 +34,16 @@
  *	Accepts:	- same arguments as MPI_Bcast()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_demo_bcast_intra(void *buff, int count,
+int mca_coll_demo_bcast_intra(void *buff, size_t count,
                               struct ompi_datatype_t *datatype, int root,
                               struct ompi_communicator_t *comm,
                               mca_coll_base_module_t *module)
 {
     mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "In demo bcast_intra");
-    return demo_module->underlying.coll_bcast(buff, count, datatype,
-                                              root, comm,
-                                              demo_module->underlying.coll_bcast_module);
+    return demo_module->c_coll.coll_bcast(buff, count, datatype,
+                                          root, comm,
+                                          demo_module->c_coll.coll_bcast_module);
 }
 
 
@@ -53,14 +54,14 @@ int mca_coll_demo_bcast_intra(void *buff, int count,
  *	Accepts:	- same arguments as MPI_Bcast()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_demo_bcast_inter(void *buff, int count,
+int mca_coll_demo_bcast_inter(void *buff, size_t count,
                               struct ompi_datatype_t *datatype, int root,
                               struct ompi_communicator_t *comm,
                               mca_coll_base_module_t *module)
 {
     mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "In demo bcast_inter");
-    return demo_module->underlying.coll_bcast(buff, count, datatype,
-                                              root, comm,
-                                              demo_module->underlying.coll_bcast_module);
+    return demo_module->c_coll.coll_bcast(buff, count, datatype,
+                                          root, comm,
+                                          demo_module->c_coll.coll_bcast_module);
 }
diff --git a/ompi/mca/coll/demo/coll_demo_component.c b/ompi/mca/coll/demo/coll_demo_component.c
index ee23252b2e2..0740d98091c 100644
--- a/ompi/mca/coll/demo/coll_demo_component.c
+++ b/ompi/mca/coll/demo/coll_demo_component.c
@@ -13,6 +13,7 @@
  * Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -55,7 +56,7 @@ static int demo_register(void);
  * and pointers to our public functions in it
  */
 
-const mca_coll_base_component_2_4_0_t mca_coll_demo_component = {
+const mca_coll_base_component_3_0_0_t mca_coll_demo_component = {
 
     /* First, the mca_component_t struct containing meta information
        about the component itself */
@@ -105,39 +106,10 @@ static int demo_register(void)
 static void
 mca_coll_demo_module_construct(mca_coll_demo_module_t *module)
 {
-    memset(&module->underlying, 0, sizeof(mca_coll_base_comm_coll_t));
+    memset(&module->c_coll, 0, sizeof(mca_coll_base_comm_coll_t));
 }
 
-#define RELEASE(module, func)                                           \
-    do {                                                                \
-        if (NULL != module->underlying.coll_ ## func ## _module) { \
-            OBJ_RELEASE(module->underlying.coll_ ## func ## _module); \
-        }                                                               \
-    } while (0)
-
-static void
-mca_coll_demo_module_destruct(mca_coll_demo_module_t *module)
-{
-    RELEASE(module, allgather);
-    RELEASE(module, allgatherv);
-    RELEASE(module, allreduce);
-    RELEASE(module, alltoall);
-    RELEASE(module, alltoallv);
-    RELEASE(module, alltoallw);
-    RELEASE(module, barrier);
-    RELEASE(module, bcast);
-    RELEASE(module, exscan);
-    RELEASE(module, gather);
-    RELEASE(module, gatherv);
-    RELEASE(module, reduce);
-    RELEASE(module, reduce_scatter);
-    RELEASE(module, scan);
-    RELEASE(module, scatter);
-    RELEASE(module, scatterv);
-}
-
-
 OBJ_CLASS_INSTANCE(mca_coll_demo_module_t,
                    mca_coll_base_module_t,
                    mca_coll_demo_module_construct,
-                   mca_coll_demo_module_destruct);
+                   NULL);
diff --git a/ompi/mca/coll/demo/coll_demo_exscan.c b/ompi/mca/coll/demo/coll_demo_exscan.c
index c970369d0dd..f3c3047cb6e 100644
--- a/ompi/mca/coll/demo/coll_demo_exscan.c
+++ b/ompi/mca/coll/demo/coll_demo_exscan.c
@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -33,7 +34,7 @@
  *	Accepts:	- same arguments as MPI_Exscan()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_demo_exscan_intra(void *sbuf, void *rbuf, int count,
+int mca_coll_demo_exscan_intra(const void *sbuf, void *rbuf, size_t count,
                                struct ompi_datatype_t *dtype,
                                struct ompi_op_t *op,
                                struct ompi_communicator_t *comm,
@@ -41,9 +42,9 @@ int mca_coll_demo_exscan_intra(void *sbuf, void *rbuf, int count,
 {
     mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "In demo exscan_intra");
-    return demo_module->underlying.coll_exscan(sbuf, rbuf, count, dtype,
-                                               op, comm,
-                                               demo_module->underlying.coll_exscan_module);
+    return demo_module->c_coll.coll_exscan(sbuf, rbuf, count, dtype,
+                                           op, comm,
+                                           demo_module->c_coll.coll_exscan_module);
 }
 
 /*
diff --git a/ompi/mca/coll/demo/coll_demo_gather.c b/ompi/mca/coll/demo/coll_demo_gather.c
index 9f9840acf8f..e88d71f31f8 100644
--- a/ompi/mca/coll/demo/coll_demo_gather.c
+++ b/ompi/mca/coll/demo/coll_demo_gather.c
@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -32,19 +33,19 @@
  *	Accepts:	- same arguments as MPI_Gather()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_demo_gather_intra(void *sbuf, int scount,
+int mca_coll_demo_gather_intra(const void *sbuf, size_t scount,
                                struct ompi_datatype_t *sdtype,
-                               void *rbuf, int rcount,
+                               void *rbuf, size_t rcount,
                                struct ompi_datatype_t *rdtype,
                                int root, struct ompi_communicator_t *comm,
                                mca_coll_base_module_t *module)
 {
     mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "In demo gather_intra");
-    return demo_module->underlying.coll_gather(sbuf, scount, sdtype,
-                                               rbuf, rcount, rdtype,
-                                               root, comm,
-                                               demo_module->underlying.coll_gather_module);
+    return demo_module->c_coll.coll_gather(sbuf, scount, sdtype,
+                                           rbuf, rcount, rdtype,
+                                           root, comm,
+                                           demo_module->c_coll.coll_gather_module);
 }
 
 
@@ -55,17 +56,17 @@ int mca_coll_demo_gather_intra(void *sbuf, int scount,
  *	Accepts:	- same arguments as MPI_Gather()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_demo_gather_inter(void *sbuf, int scount,
+int mca_coll_demo_gather_inter(const void *sbuf, size_t scount,
                                struct ompi_datatype_t *sdtype,
-                               void *rbuf, int rcount,
+                               void *rbuf, size_t rcount,
                                struct ompi_datatype_t *rdtype,
                                int root, struct ompi_communicator_t *comm,
                                mca_coll_base_module_t *module)
 {
     mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "In demo gather_inter");
-    return demo_module->underlying.coll_gather(sbuf, scount, sdtype,
-                                               rbuf, rcount, rdtype,
-                                               root, comm,
-                                               demo_module->underlying.coll_gather_module);
+    return demo_module->c_coll.coll_gather(sbuf, scount, sdtype,
+                                           rbuf, rcount, rdtype,
+                                           root, comm,
+                                           demo_module->c_coll.coll_gather_module);
 }
diff --git a/ompi/mca/coll/demo/coll_demo_gatherv.c b/ompi/mca/coll/demo/coll_demo_gatherv.c
index f23b37a0d88..cc34d01b15b 100644
--- a/ompi/mca/coll/demo/coll_demo_gatherv.c
+++ b/ompi/mca/coll/demo/coll_demo_gatherv.c
@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -33,19 +34,19 @@
  *	Accepts:	- same arguments as MPI_Gatherv()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_demo_gatherv_intra(void *sbuf, int scount,
+int mca_coll_demo_gatherv_intra(const void *sbuf, int scount,
                                 struct ompi_datatype_t *sdtype,
-                                void *rbuf, int *rcounts, int *disps,
+                                void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
                                 struct ompi_datatype_t *rdtype, int root,
                                 struct ompi_communicator_t *comm,
                                 mca_coll_base_module_t *module)
 {
     mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "In demo gatherv_intra");
-    return demo_module->underlying.coll_gatherv(sbuf, scount, sdtype,
-                                                rbuf, rcounts, disps,
-                                                rdtype, root, comm,
-                                                demo_module->underlying.coll_gatherv_module);
+    return demo_module->c_coll.coll_gatherv(sbuf, scount, sdtype,
+                                            rbuf, rcounts, disps,
+                                            rdtype, root, comm,
+                                            demo_module->c_coll.coll_gatherv_module);
 }
 
 
@@ -56,17 +57,17 @@ int mca_coll_demo_gatherv_intra(void *sbuf, int scount,
  *	Accepts:	- same arguments as MPI_Gatherv()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_demo_gatherv_inter(void *sbuf, int scount,
+int mca_coll_demo_gatherv_inter(const void *sbuf, int scount,
                                 struct ompi_datatype_t *sdtype,
-                                void *rbuf, int *rcounts, int *disps,
+                                void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
                                 struct ompi_datatype_t *rdtype, int root,
                                 struct ompi_communicator_t *comm,
                                 mca_coll_base_module_t *module)
 {
     mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "In demo gatherv_inter");
-    return demo_module->underlying.coll_gatherv(sbuf, scount, sdtype,
-                                                rbuf, rcounts, disps,
-                                                rdtype, root, comm,
-                                                demo_module->underlying.coll_gatherv_module);
+    return demo_module->c_coll.coll_gatherv(sbuf, scount, sdtype,
+                                            rbuf, rcounts, disps,
+                                            rdtype, root, comm,
+                                            demo_module->c_coll.coll_gatherv_module);
 }
diff --git a/ompi/mca/coll/demo/coll_demo_module.c b/ompi/mca/coll/demo/coll_demo_module.c
index 5c66c98059f..0201340d9d9 100644
--- a/ompi/mca/coll/demo/coll_demo_module.c
+++ b/ompi/mca/coll/demo/coll_demo_module.c
@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -27,71 +28,6 @@
 #include "ompi/mca/coll/base/base.h"
 #include "coll_demo.h"
 
-#if 0
-
-/*
- * Linear set of collective algorithms
- */
-static const mca_coll_base_module_1_0_0_t intra = {
-
-    /* Initialization / finalization functions */
-
-    mca_coll_demo_module_init,
-    mca_coll_demo_module_finalize,
-
-    /* Collective function pointers */
-
-    mca_coll_demo_allgather_intra,
-    mca_coll_demo_allgatherv_intra,
-    mca_coll_demo_allreduce_intra,
-    mca_coll_demo_alltoall_intra,
-    mca_coll_demo_alltoallv_intra,
-    mca_coll_demo_alltoallw_intra,
-    mca_coll_demo_barrier_intra,
-    mca_coll_demo_bcast_intra,
-    NULL, /* Leave exscan blank just to force basic to be used */
-    mca_coll_demo_gather_intra,
-    mca_coll_demo_gatherv_intra,
-    mca_coll_demo_reduce_intra,
-    mca_coll_demo_reduce_scatter_intra,
-    mca_coll_demo_scan_intra,
-    mca_coll_demo_scatter_intra,
-    mca_coll_demo_scatterv_intra
-};
-
-
-/*
- * Linear set of collective algorithms for intercommunicators
- */
-static const mca_coll_base_module_1_0_0_t inter = {
-
-    /* Initialization / finalization functions */
-
-    mca_coll_demo_module_init,
-    mca_coll_demo_module_finalize,
-
-    /* Collective function pointers */
-
-    mca_coll_demo_allgather_inter,
-    mca_coll_demo_allgatherv_inter,
-    mca_coll_demo_allreduce_inter,
-    mca_coll_demo_alltoall_inter,
-    mca_coll_demo_alltoallv_inter,
-    mca_coll_demo_alltoallw_inter,
-    mca_coll_demo_barrier_inter,
-    mca_coll_demo_bcast_inter,
-    mca_coll_demo_exscan_inter,
-    mca_coll_demo_gather_inter,
-    mca_coll_demo_gatherv_inter,
-    mca_coll_demo_reduce_inter,
-    mca_coll_demo_reduce_scatter_inter,
-    NULL,
-    mca_coll_demo_scatter_inter,
-    mca_coll_demo_scatterv_inter
-};
-
-#endif
-
 /*
  * Initial query function that is invoked during MPI_INIT, allowing
  * this component to disqualify itself if it doesn't support the
@@ -105,6 +41,13 @@ int mca_coll_demo_init_query(bool enable_progress_threads,
     return OMPI_SUCCESS;
 }
 
+static int
+mca_coll_demo_module_enable(mca_coll_base_module_t *module,
+                            struct ompi_communicator_t *comm);
+static int
+mca_coll_demo_module_disable(mca_coll_base_module_t *module,
+                             struct ompi_communicator_t *comm);
+
 /*
  * Invoked when there's a new communicator that has been created.
  * Look at the communicator and decide which set of functions and
@@ -121,54 +64,38 @@ mca_coll_demo_comm_query(struct ompi_communicator_t *comm, int *priority)
     *priority = mca_coll_demo_priority;
 
     demo_module->super.coll_module_enable = mca_coll_demo_module_enable;
-
-    if (OMPI_COMM_IS_INTRA(comm)) {
-        demo_module->super.coll_allgather  = mca_coll_demo_allgather_intra;
-        demo_module->super.coll_allgatherv = mca_coll_demo_allgatherv_intra;
-        demo_module->super.coll_allreduce  = mca_coll_demo_allreduce_intra;
-        demo_module->super.coll_alltoall   = mca_coll_demo_alltoall_intra;
-        demo_module->super.coll_alltoallv  = mca_coll_demo_alltoallv_intra;
-        demo_module->super.coll_alltoallw  = mca_coll_demo_alltoallw_intra;
-        demo_module->super.coll_barrier    = mca_coll_demo_barrier_intra;
-        demo_module->super.coll_bcast      = mca_coll_demo_bcast_intra;
-        demo_module->super.coll_exscan     = mca_coll_demo_exscan_intra;
-        demo_module->super.coll_gather     = mca_coll_demo_gather_intra;
-        demo_module->super.coll_gatherv    = mca_coll_demo_gatherv_intra;
-        demo_module->super.coll_reduce     = mca_coll_demo_reduce_intra;
-        demo_module->super.coll_reduce_scatter = mca_coll_demo_reduce_scatter_intra;
-        demo_module->super.coll_scan       = mca_coll_demo_scan_intra;
-        demo_module->super.coll_scatter    = mca_coll_demo_scatter_intra;
-        demo_module->super.coll_scatterv   = mca_coll_demo_scatterv_intra;
-    } else {
-        demo_module->super.coll_allgather  = mca_coll_demo_allgather_inter;
-        demo_module->super.coll_allgatherv = mca_coll_demo_allgatherv_inter;
-        demo_module->super.coll_allreduce  = mca_coll_demo_allreduce_inter;
-        demo_module->super.coll_alltoall   = mca_coll_demo_alltoall_inter;
-        demo_module->super.coll_alltoallv  = mca_coll_demo_alltoallv_inter;
-        demo_module->super.coll_alltoallw  = mca_coll_demo_alltoallw_inter;
-        demo_module->super.coll_barrier    = mca_coll_demo_barrier_inter;
-        demo_module->super.coll_bcast      = mca_coll_demo_bcast_inter;
-        demo_module->super.coll_exscan     = NULL;
-        demo_module->super.coll_gather     = mca_coll_demo_gather_inter;
-        demo_module->super.coll_gatherv    = mca_coll_demo_gatherv_inter;
-        demo_module->super.coll_reduce     = mca_coll_demo_reduce_inter;
-        demo_module->super.coll_reduce_scatter = mca_coll_demo_reduce_scatter_inter;
-        demo_module->super.coll_scan       = NULL;
-        demo_module->super.coll_scatter    = mca_coll_demo_scatter_inter;
-        demo_module->super.coll_scatterv   = mca_coll_demo_scatterv_inter;
-    }
+    demo_module->super.coll_module_disable = mca_coll_demo_module_disable;
 
     return &(demo_module->super);
 }
 
-#define COPY(comm, module, func)                                        \
-    do {                                                                \
-        module->underlying.coll_ ## func  = comm->c_coll->coll_ ## func; \
-        module->underlying.coll_ ## func  = comm->c_coll->coll_ ## func; \
-        if (NULL != module->underlying.coll_ ## func ## _module) {      \
-            OBJ_RETAIN(module->underlying.coll_ ## func ## _module);    \
-        }                                                               \
-    } while (0)
+#define DEMO_INSTALL_COLL_API(__comm, __module, __api, __func)                                                                   \
+    do                                                                                                                           \
+    {                                                                                                                            \
+        if (__comm->c_coll->coll_##__api)                                                                                        \
+        {                                                                                                                        \
+            /* save the current selected collective */                                                                           \
+            MCA_COLL_SAVE_API(__comm, __api, (__module)->c_coll.coll_##__api, (__module)->c_coll.coll_##__api##_module, "demo"); \
+            /* install our own */                                                                                                \
+            MCA_COLL_INSTALL_API(__comm, __api, __func, &__module->super, "demo");                                               \
+        }                                                                                                                        \
+        else                                                                                                                     \
+        {                                                                                                                        \
+            opal_show_help("help-mca-coll-base.txt", "comm-select:missing collective", true,                                     \
+                           "demo", #__api, ompi_process_info.nodename,                                                           \
+                           mca_coll_demo_priority);                                                                              \
+        }                                                                                                                        \
+    }    while (0)
+
+#define DEMO_UNINSTALL_COLL_API(__comm, __module, __api)                                                                        \
+    do                                                                                                                          \
+    {                                                                                                                           \
+        if (&(__module)->super == __comm->c_coll->coll_##__api##_module)                                                        \
+        {                                                                                                                       \
+            /* put back the original collective */                                                                              \
+            MCA_COLL_INSTALL_API(__comm, __api, __module->c_coll.coll_##__api, __module->c_coll.coll_##__api##_module, "demo"); \
+        }                                                                                                                       \
+     } while (0)
 
 int
 mca_coll_demo_module_enable(mca_coll_base_module_t *module,
@@ -180,23 +107,70 @@ mca_coll_demo_module_enable(mca_coll_base_module_t *module,
         printf("Hello!  This is the \"demo\" coll component.  I'll be your coll component\ntoday.  Please tip your waitresses well.\n");
     }
 
-    /* save the old pointers */
-    COPY(comm, demo_module, allgather);
-    COPY(comm, demo_module, allgatherv);
-    COPY(comm, demo_module, allreduce);
-    COPY(comm, demo_module, alltoall);
-    COPY(comm, demo_module, alltoallv);
-    COPY(comm, demo_module, alltoallw);
-    COPY(comm, demo_module, barrier);
-    COPY(comm, demo_module, bcast);
-    COPY(comm, demo_module, exscan);
-    COPY(comm, demo_module, gather);
-    COPY(comm, demo_module, gatherv);
-    COPY(comm, demo_module, reduce);
-    COPY(comm, demo_module, reduce_scatter);
-    COPY(comm, demo_module, scan);
-    COPY(comm, demo_module, scatter);
-    COPY(comm, demo_module, scatterv);
+    if (OMPI_COMM_IS_INTRA(comm))
+    {
+        DEMO_INSTALL_COLL_API(comm, demo_module, allgather, mca_coll_demo_allgather_intra);
+        DEMO_INSTALL_COLL_API(comm, demo_module, allgatherv, mca_coll_demo_allgatherv_intra);
+        DEMO_INSTALL_COLL_API(comm, demo_module, allreduce, mca_coll_demo_allreduce_intra);
+        DEMO_INSTALL_COLL_API(comm, demo_module, alltoall, mca_coll_demo_alltoall_intra);
+        DEMO_INSTALL_COLL_API(comm, demo_module, alltoallv, mca_coll_demo_alltoallv_intra);
+        DEMO_INSTALL_COLL_API(comm, demo_module, alltoallw, mca_coll_demo_alltoallw_intra);
+        DEMO_INSTALL_COLL_API(comm, demo_module, barrier, mca_coll_demo_barrier_intra);
+        DEMO_INSTALL_COLL_API(comm, demo_module, bcast, mca_coll_demo_bcast_intra);
+        DEMO_INSTALL_COLL_API(comm, demo_module, exscan, mca_coll_demo_exscan_intra);
+        DEMO_INSTALL_COLL_API(comm, demo_module, gather, mca_coll_demo_gather_intra);
+        DEMO_INSTALL_COLL_API(comm, demo_module, gatherv, mca_coll_demo_gatherv_intra);
+        DEMO_INSTALL_COLL_API(comm, demo_module, reduce, mca_coll_demo_reduce_intra);
+        DEMO_INSTALL_COLL_API(comm, demo_module, reduce_scatter, mca_coll_demo_reduce_scatter_intra);
+        DEMO_INSTALL_COLL_API(comm, demo_module, scan, mca_coll_demo_scan_intra);
+        DEMO_INSTALL_COLL_API(comm, demo_module, scatter, mca_coll_demo_scatter_intra);
+        DEMO_INSTALL_COLL_API(comm, demo_module, scatterv, mca_coll_demo_scatterv_intra);
+    }
+    else
+    {
+        DEMO_INSTALL_COLL_API(comm, demo_module, allgather, mca_coll_demo_allgather_inter);
+        DEMO_INSTALL_COLL_API(comm, demo_module, allgatherv, mca_coll_demo_allgatherv_inter);
+        DEMO_INSTALL_COLL_API(comm, demo_module, allreduce, mca_coll_demo_allreduce_inter);
+        DEMO_INSTALL_COLL_API(comm, demo_module, alltoall, mca_coll_demo_alltoall_inter);
+        DEMO_INSTALL_COLL_API(comm, demo_module, alltoallv, mca_coll_demo_alltoallv_inter);
+        DEMO_INSTALL_COLL_API(comm, demo_module, alltoallw, mca_coll_demo_alltoallw_inter);
+        DEMO_INSTALL_COLL_API(comm, demo_module, barrier, mca_coll_demo_barrier_inter);
+        DEMO_INSTALL_COLL_API(comm, demo_module, bcast, mca_coll_demo_bcast_inter);
+        /* Skip the exscan for inter-comms, it is not supported */
+        DEMO_INSTALL_COLL_API(comm, demo_module, gather, mca_coll_demo_gather_inter);
+        DEMO_INSTALL_COLL_API(comm, demo_module, gatherv, mca_coll_demo_gatherv_inter);
+        DEMO_INSTALL_COLL_API(comm, demo_module, reduce, mca_coll_demo_reduce_inter);
+        DEMO_INSTALL_COLL_API(comm, demo_module, reduce_scatter, mca_coll_demo_reduce_scatter_inter);
+        /* Skip the scan for inter-comms, it is not supported */
+        DEMO_INSTALL_COLL_API(comm, demo_module, scatter, mca_coll_demo_scatter_inter);
+        DEMO_INSTALL_COLL_API(comm, demo_module, scatterv, mca_coll_demo_scatterv_inter);
+    }
+    return OMPI_SUCCESS;
+}
+
+static int
+mca_coll_demo_module_disable(mca_coll_base_module_t *module,
+                             struct ompi_communicator_t *comm)
+{
+    mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
+
+    /* put back the old pointers */
+    DEMO_UNINSTALL_COLL_API(comm, demo_module, allgather);
+    DEMO_UNINSTALL_COLL_API(comm, demo_module, allgatherv);
+    DEMO_UNINSTALL_COLL_API(comm, demo_module, allreduce);
+    DEMO_UNINSTALL_COLL_API(comm, demo_module, alltoall);
+    DEMO_UNINSTALL_COLL_API(comm, demo_module, alltoallv);
+    DEMO_UNINSTALL_COLL_API(comm, demo_module, alltoallw);
+    DEMO_UNINSTALL_COLL_API(comm, demo_module, barrier);
+    DEMO_UNINSTALL_COLL_API(comm, demo_module, bcast);
+    DEMO_UNINSTALL_COLL_API(comm, demo_module, exscan);
+    DEMO_UNINSTALL_COLL_API(comm, demo_module, gather);
+    DEMO_UNINSTALL_COLL_API(comm, demo_module, gatherv);
+    DEMO_UNINSTALL_COLL_API(comm, demo_module, reduce);
+    DEMO_UNINSTALL_COLL_API(comm, demo_module, reduce_scatter);
+    DEMO_UNINSTALL_COLL_API(comm, demo_module, scan);
+    DEMO_UNINSTALL_COLL_API(comm, demo_module, scatter);
+    DEMO_UNINSTALL_COLL_API(comm, demo_module, scatterv);
 
     return OMPI_SUCCESS;
 }
diff --git a/ompi/mca/coll/demo/coll_demo_reduce.c b/ompi/mca/coll/demo/coll_demo_reduce.c
index 6df413902b6..653df7c44f7 100644
--- a/ompi/mca/coll/demo/coll_demo_reduce.c
+++ b/ompi/mca/coll/demo/coll_demo_reduce.c
@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -33,7 +34,7 @@
  *	Accepts:	- same as MPI_Reduce()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_demo_reduce_intra(void *sbuf, void *rbuf, int count,
+int mca_coll_demo_reduce_intra(const void *sbuf, void *rbuf, size_t count,
                                struct ompi_datatype_t *dtype,
                                struct ompi_op_t *op,
                                int root, struct ompi_communicator_t *comm,
@@ -41,9 +42,9 @@ int mca_coll_demo_reduce_intra(void *sbuf, void *rbuf, int count,
 {
     mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "In demo reduce_intra");
-    return demo_module->underlying.coll_reduce(sbuf, rbuf, count, dtype,
-                                               op, root, comm,
-                                               demo_module->underlying.coll_reduce_module);
+    return demo_module->c_coll.coll_reduce(sbuf, rbuf, count, dtype,
+                                           op, root, comm,
+                                           demo_module->c_coll.coll_reduce_module);
 }
 
 
@@ -54,7 +55,7 @@ int mca_coll_demo_reduce_intra(void *sbuf, void *rbuf, int count,
  *	Accepts:	- same as MPI_Reduce()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_demo_reduce_inter(void *sbuf, void *rbuf, int count,
+int mca_coll_demo_reduce_inter(const void *sbuf, void *rbuf, size_t count,
                                struct ompi_datatype_t *dtype,
                                struct ompi_op_t *op,
                                int root, struct ompi_communicator_t *comm,
@@ -62,7 +63,7 @@ int mca_coll_demo_reduce_inter(void *sbuf, void *rbuf, int count,
 {
     mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "In demo reduce_inter");
-    return demo_module->underlying.coll_reduce(sbuf, rbuf, count, dtype,
-                                               op, root, comm,
-                                               demo_module->underlying.coll_reduce_module);
+    return demo_module->c_coll.coll_reduce(sbuf, rbuf, count, dtype,
+                                           op, root, comm,
+                                           demo_module->c_coll.coll_reduce_module);
 }
diff --git a/ompi/mca/coll/demo/coll_demo_reduce_scatter.c b/ompi/mca/coll/demo/coll_demo_reduce_scatter.c
index 438f1008b3a..7ff7b2c9ab1 100644
--- a/ompi/mca/coll/demo/coll_demo_reduce_scatter.c
+++ b/ompi/mca/coll/demo/coll_demo_reduce_scatter.c
@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -33,7 +34,7 @@
  *	Accepts:	- same as MPI_Reduce_scatter()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_demo_reduce_scatter_intra(void *sbuf, void *rbuf, int *rcounts,
+int mca_coll_demo_reduce_scatter_intra(const void *sbuf, void *rbuf, ompi_count_array_t rcounts,
                                        struct ompi_datatype_t *dtype,
                                        struct ompi_op_t *op,
                                        struct ompi_communicator_t *comm,
@@ -41,9 +42,9 @@ int mca_coll_demo_reduce_scatter_intra(void *sbuf, void *rbuf, int *rcounts,
 {
     mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "In demo scatter_intra");
-    return demo_module->underlying.coll_reduce_scatter(sbuf, rbuf, rcounts,
-                                                       dtype, op, comm,
-                                                       demo_module->underlying.coll_reduce_scatter_module);
+    return demo_module->c_coll.coll_reduce_scatter(sbuf, rbuf, rcounts,
+                                                   dtype, op, comm,
+                                                   demo_module->c_coll.coll_reduce_scatter_module);
 }
 
 
@@ -54,7 +55,7 @@ int mca_coll_demo_reduce_scatter_intra(void *sbuf, void *rbuf, int *rcounts,
  *	Accepts:	- same arguments as MPI_Reduce_scatter()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_demo_reduce_scatter_inter(void *sbuf, void *rbuf, int *rcounts,
+int mca_coll_demo_reduce_scatter_inter(const void *sbuf, void *rbuf, ompi_count_array_t rcounts,
                                        struct ompi_datatype_t *dtype,
                                        struct ompi_op_t *op,
                                        struct ompi_communicator_t *comm,
@@ -62,7 +63,7 @@ int mca_coll_demo_reduce_scatter_inter(void *sbuf, void *rbuf, int *rcounts,
 {
     mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "In demo scatter_inter");
-    return demo_module->underlying.coll_reduce_scatter(sbuf, rbuf, rcounts,
-                                                       dtype, op, comm,
-                                                       demo_module->underlying.coll_reduce_scatter_module);
+    return demo_module->c_coll.coll_reduce_scatter(sbuf, rbuf, rcounts,
+                                                   dtype, op, comm,
+                                                   demo_module->c_coll.coll_reduce_scatter_module);
 }
diff --git a/ompi/mca/coll/demo/coll_demo_scan.c b/ompi/mca/coll/demo/coll_demo_scan.c
index 90d3cb343b1..030b8827ced 100644
--- a/ompi/mca/coll/demo/coll_demo_scan.c
+++ b/ompi/mca/coll/demo/coll_demo_scan.c
@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -33,7 +34,7 @@
  *	Accepts:	- same arguments as MPI_Scan()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_demo_scan_intra(void *sbuf, void *rbuf, int count,
+int mca_coll_demo_scan_intra(void *sbuf, void *rbuf, size_t count,
                              struct ompi_datatype_t *dtype,
                              struct ompi_op_t *op,
                              struct ompi_communicator_t *comm,
@@ -41,9 +42,9 @@ int mca_coll_demo_scan_intra(void *sbuf, void *rbuf, int count,
 {
     mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "In demo scan_intra");
-    return demo_module->underlying.coll_scan(sbuf, rbuf, count,
-                                             dtype, op, comm,
-                                             demo_module->underlying.coll_scan_module);
+    return demo_module->c_coll.coll_scan(sbuf, rbuf, count,
+                                         dtype, op, comm,
+                                         demo_module->c_coll.coll_scan_module);
 }
 
 
diff --git a/ompi/mca/coll/demo/coll_demo_scatter.c b/ompi/mca/coll/demo/coll_demo_scatter.c
index ccc2e401df6..b285e026e39 100644
--- a/ompi/mca/coll/demo/coll_demo_scatter.c
+++ b/ompi/mca/coll/demo/coll_demo_scatter.c
@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -33,9 +34,9 @@
  *	Accepts:	- same arguments as MPI_Scatter()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_demo_scatter_intra(void *sbuf, int scount,
+int mca_coll_demo_scatter_intra(const void *sbuf, size_t scount,
                                 struct ompi_datatype_t *sdtype,
-                                void *rbuf, int rcount,
+                                void *rbuf, size_t rcount,
                                 struct ompi_datatype_t *rdtype,
                                 int root,
                                 struct ompi_communicator_t *comm,
@@ -43,10 +44,10 @@ int mca_coll_demo_scatter_intra(void *sbuf, int scount,
 {
     mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "In demo scatter_intra");
-    return demo_module->underlying.coll_scatter(sbuf, scount, sdtype,
-                                                rbuf, rcount, rdtype,
-                                                root, comm,
-                                                demo_module->underlying.coll_scatter_module);
+    return demo_module->c_coll.coll_scatter(sbuf, scount, sdtype,
+                                            rbuf, rcount, rdtype,
+                                            root, comm,
+                                            demo_module->c_coll.coll_scatter_module);
 }
 
 
@@ -57,9 +58,9 @@ int mca_coll_demo_scatter_intra(void *sbuf, int scount,
  *	Accepts:	- same arguments as MPI_Scatter()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_demo_scatter_inter(void *sbuf, int scount,
+int mca_coll_demo_scatter_inter(const void *sbuf, size_t scount,
                                 struct ompi_datatype_t *sdtype,
-                                void *rbuf, int rcount,
+                                void *rbuf, size_t rcount,
                                 struct ompi_datatype_t *rdtype,
                                 int root,
                                 struct ompi_communicator_t *comm,
@@ -67,8 +68,8 @@ int mca_coll_demo_scatter_inter(void *sbuf, int scount,
 {
     mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "In demo scatter_inter");
-    return demo_module->underlying.coll_scatter(sbuf, scount, sdtype,
-                                                rbuf, rcount, rdtype,
-                                                root, comm,
-                                                demo_module->underlying.coll_scatter_module);
+    return demo_module->c_coll.coll_scatter(sbuf, scount, sdtype,
+                                            rbuf, rcount, rdtype,
+                                            root, comm,
+                                            demo_module->c_coll.coll_scatter_module);
 }
diff --git a/ompi/mca/coll/demo/coll_demo_scatterv.c b/ompi/mca/coll/demo/coll_demo_scatterv.c
index 3084efc0de5..11e42905df5 100644
--- a/ompi/mca/coll/demo/coll_demo_scatterv.c
+++ b/ompi/mca/coll/demo/coll_demo_scatterv.c
@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -33,8 +34,8 @@
  *	Accepts:	- same arguments as MPI_Scatterv()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_demo_scatterv_intra(void *sbuf, int *scounts,
-                                 int *disps, struct ompi_datatype_t *sdtype,
+int mca_coll_demo_scatterv_intra(const void *sbuf, ompi_count_array_t scounts,
+                                 ompi_disp_array_t disps, struct ompi_datatype_t *sdtype,
                                  void *rbuf, int rcount,
                                  struct ompi_datatype_t *rdtype, int root,
                                  struct ompi_communicator_t *comm,
@@ -42,10 +43,10 @@ int mca_coll_demo_scatterv_intra(void *sbuf, int *scounts,
 {
     mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "In demo scatterv_intra");
-    return demo_module->underlying.coll_scatterv(sbuf, scounts, disps,
-                                                 sdtype, rbuf, rcount,
-                                                 rdtype, root, comm,
-                                                 demo_module->underlying.coll_scatterv_module);
+    return demo_module->c_coll.coll_scatterv(sbuf, scounts, disps,
+                                             sdtype, rbuf, rcount,
+                                             rdtype, root, comm,
+                                             demo_module->c_coll.coll_scatterv_module);
 }
 
 
@@ -56,8 +57,8 @@ int mca_coll_demo_scatterv_intra(void *sbuf, int *scounts,
  *	Accepts:	- same arguments as MPI_Scatterv()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_demo_scatterv_inter(void *sbuf, int *scounts,
-                                 int *disps, struct ompi_datatype_t *sdtype,
+int mca_coll_demo_scatterv_inter(const void *sbuf, ompi_count_array_t scounts,
+                                 ompi_disp_array_t disps, struct ompi_datatype_t *sdtype,
                                  void *rbuf, int rcount,
                                  struct ompi_datatype_t *rdtype, int root,
                                  struct ompi_communicator_t *comm,
@@ -65,8 +66,8 @@ int mca_coll_demo_scatterv_inter(void *sbuf, int *scounts,
 {
     mca_coll_demo_module_t *demo_module = (mca_coll_demo_module_t*) module;
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "In demo scatterv_inter");
-    return demo_module->underlying.coll_scatterv(sbuf, scounts, disps,
-                                                 sdtype, rbuf, rcount,
-                                                 rdtype, root, comm,
-                                                 demo_module->underlying.coll_scatterv_module);
+    return demo_module->c_coll.coll_scatterv(sbuf, scounts, disps,
+                                             sdtype, rbuf, rcount,
+                                             rdtype, root, comm,
+                                             demo_module->c_coll.coll_scatterv_module);
 }
diff --git a/ompi/mca/coll/ftagree/coll_ftagree.h b/ompi/mca/coll/ftagree/coll_ftagree.h
index 8a3bd8e2a7d..ae9d12e58c1 100644
--- a/ompi/mca/coll/ftagree/coll_ftagree.h
+++ b/ompi/mca/coll/ftagree/coll_ftagree.h
@@ -3,6 +3,7 @@
  * Copyright (c) 2012-2020 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -32,7 +33,7 @@ BEGIN_C_DECLS
 
 /* Globally exported variables */
 
-OMPI_MODULE_DECLSPEC extern const mca_coll_base_component_2_4_0_t
+OMPI_DECLSPEC extern const mca_coll_base_component_3_0_0_t
 mca_coll_ftagree_component;
 extern int mca_coll_ftagree_priority;
 
@@ -98,9 +99,6 @@ mca_coll_base_module_t
 *mca_coll_ftagree_comm_query(struct ompi_communicator_t *comm,
                              int *priority);
 
-int mca_coll_ftagree_module_enable(mca_coll_base_module_t *module,
-                                   struct ompi_communicator_t *comm);
-
 /*
  * Agreement algorithms
  */
@@ -108,7 +106,7 @@ int mca_coll_ftagree_module_enable(mca_coll_base_module_t *module,
 /* Early termination algorithm */
 int
 mca_coll_ftagree_eta_intra(     void* contrib,
-                                int dt_count,
+                                size_t dt_count,
                                 ompi_datatype_t *dt,
                                 ompi_op_t *op,
                                 ompi_group_t **group, bool grp_update,
@@ -117,14 +115,14 @@ mca_coll_ftagree_eta_intra(     void* contrib,
 /* Early returning algorithm */
 int
 mca_coll_ftagree_era_intra(     void* contrib,
-                                int dt_count,
+                                size_t dt_count,
                                 ompi_datatype_t *dt,
                                 ompi_op_t *op,
                                 ompi_group_t **group, bool grp_update,
                                 ompi_communicator_t* comm,
                                 mca_coll_base_module_t *module);
 int mca_coll_ftagree_iera_intra(void* contrib,
-                                int dt_count,
+                                size_t dt_count,
                                 ompi_datatype_t *dt,
                                 ompi_op_t *op,
                                 ompi_group_t **group, bool grp_update,
@@ -132,7 +130,7 @@ int mca_coll_ftagree_iera_intra(void* contrib,
                                 ompi_request_t **request,
                                 mca_coll_base_module_t *module);
 int mca_coll_ftagree_era_inter( void* contrib,
-                                int dt_count,
+                                size_t dt_count,
                                 ompi_datatype_t *dt,
                                 ompi_op_t *op,
                                 ompi_group_t **group, bool grp_update,
diff --git a/ompi/mca/coll/ftagree/coll_ftagree_component.c b/ompi/mca/coll/ftagree/coll_ftagree_component.c
index 45b161e4395..7414833d3fc 100644
--- a/ompi/mca/coll/ftagree/coll_ftagree_component.c
+++ b/ompi/mca/coll/ftagree/coll_ftagree_component.c
@@ -48,13 +48,13 @@ static int ftagree_close(void);
  * and pointers to our public functions in it
  */
 
-const mca_coll_base_component_2_4_0_t mca_coll_ftagree_component = {
+const mca_coll_base_component_3_0_0_t mca_coll_ftagree_component = {
 
     /* First, the mca_component_t struct containing meta information
      * about the component itself */
 
     {
-     MCA_COLL_BASE_VERSION_2_4_0,
+     MCA_COLL_BASE_VERSION_3_0_0,
 
      /* Component name and version */
      "ftagree",
@@ -105,7 +105,7 @@ ftagree_register(void)
     if( ompi_ftmpi_enabled ) value = 1;
     else value = 0; /* NOFT: do not initialize ERA */
     (void) mca_base_component_var_register(&mca_coll_ftagree_component.collm_version,
-                                           "agreement", "Agreement algorithm 0: Allreduce (NOT FAULT TOLERANT); 1: Early Returning Concensus (era); 2: Early Terminating Concensus (eta)",
+                                           "agreement", "Agreement algorithm 0: Allreduce (NOT FAULT TOLERANT); 1: Early Returning Consensus (era); 2: Early Terminating Consensus (eta)",
                                            MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
                                            OPAL_INFO_LVL_6,
                                            MCA_BASE_VAR_SCOPE_READONLY,
diff --git a/ompi/mca/coll/ftagree/coll_ftagree_earlyreturning.c b/ompi/mca/coll/ftagree/coll_ftagree_earlyreturning.c
index 99d8b5fa230..73ef2f06374 100644
--- a/ompi/mca/coll/ftagree/coll_ftagree_earlyreturning.c
+++ b/ompi/mca/coll/ftagree/coll_ftagree_earlyreturning.c
@@ -1,8 +1,12 @@
 /* -*- Mode: C; c-basic-offset:4 ; -*- */
 /*
- * Copyright (c) 2014-2020 The University of Tennessee and The University
+ * Copyright (c) 2014-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2021      Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
+ *
  *
  * $COPYRIGHT$
  *
@@ -115,7 +119,7 @@ typedef struct {
                                               *   min over all ranks. */
     int      operand;                        /**< operand applied on bytes.
                                               *   One of OMPI_OP_BASE_FORTRAN_* values in mca/op/op.h */
-    int      dt_count;                       /**< The number of datatypes in bytes */
+    size_t   dt_count;                       /**< The number of datatypes in bytes */
     int      datatype;                       /**< Fortran index of predefined basic datatype in bytes */
     int      nb_new_dead;                    /**< Number of newly discovered dead */
 } era_value_header_t;
@@ -129,9 +133,9 @@ typedef struct {
                                               *   containing the value on which the agreement is done */
     int                 *new_dead_array;     /**< array of header.nb_new_dead integers with the newly discovered
                                               *   processes. */
-} era_value_t;
+} ompi_coll_ftagree_era_value_t;
 
-static void  era_value_constructor (era_value_t *value)
+static void  era_value_constructor (ompi_coll_ftagree_era_value_t *value)
 {
     value->header.ret = 0;
     value->header.operand = -1;
@@ -144,7 +148,7 @@ static void  era_value_constructor (era_value_t *value)
     value->new_dead_array = NULL;
 }
 
-static void  era_value_destructor (era_value_t *value)
+static void  era_value_destructor (ompi_coll_ftagree_era_value_t *value)
 {
     if( NULL != value->bytes )
         free(value->bytes);
@@ -152,7 +156,7 @@ static void  era_value_destructor (era_value_t *value)
         free(value->new_dead_array);
 }
 
-OBJ_CLASS_INSTANCE(era_value_t, opal_object_t, era_value_constructor, era_value_destructor);
+OBJ_CLASS_INSTANCE(ompi_coll_ftagree_era_value_t, opal_object_t, era_value_constructor, era_value_destructor);
 
 typedef struct {
     era_msg_type_t      msg_type;
@@ -194,9 +198,9 @@ typedef struct {
 typedef struct {
     opal_list_item_t super;
     int32_t          rank;        /**< The rank of the descendent that provided information */
-} era_rank_item_t;
+} ompi_coll_ftagree_era_rank_item_t;
 
-OBJ_CLASS_INSTANCE(era_rank_item_t, opal_list_item_t, NULL, NULL /*print_destructor*/);
+OBJ_CLASS_INSTANCE(ompi_coll_ftagree_era_rank_item_t, opal_list_item_t, NULL, NULL /*print_destructor*/);
 
 /**
  * topology-dependent tree structure
@@ -224,9 +228,9 @@ typedef struct era_comm_agreement_specific_s {
     int           acked_ra_size;
     int          *acked_ra;
     int           ags_status;
-} era_comm_agreement_specific_t;
+} ompi_coll_ftagree_era_comm_agreement_specific_t;
 
-static void  era_agreement_comm_specific_constructor(era_comm_agreement_specific_t *comm_specific)
+static void  era_agreement_comm_specific_constructor(ompi_coll_ftagree_era_comm_agreement_specific_t *comm_specific)
 {
     comm_specific->agreed_failed_ranks = NULL;
     comm_specific->afr_size            = 0;
@@ -237,7 +241,7 @@ static void  era_agreement_comm_specific_constructor(era_comm_agreement_specific
     comm_specific->ags_status          = AGS_TREE_DIRTY | AGS_AFR_DIRTY; /**< the AGS does not exist, so it needs to be created */
 }
 
-static void  era_agreement_comm_specific_destructor(era_comm_agreement_specific_t *comm_specific)
+static void  era_agreement_comm_specific_destructor(ompi_coll_ftagree_era_comm_agreement_specific_t *comm_specific)
 {
     if( NULL != comm_specific->agreed_failed_ranks ) {
         free(comm_specific->agreed_failed_ranks );
@@ -250,12 +254,12 @@ static void  era_agreement_comm_specific_destructor(era_comm_agreement_specific_
     }
 }
 
-OBJ_CLASS_INSTANCE(era_comm_agreement_specific_t,
+OBJ_CLASS_INSTANCE(ompi_coll_ftagree_era_comm_agreement_specific_t,
                    opal_object_t,
                    era_agreement_comm_specific_constructor,
                    era_agreement_comm_specific_destructor);
 
-#define AGS(comm)  ( (era_comm_agreement_specific_t*)(comm)->agreement_specific )
+#define AGS(comm)  ( (ompi_coll_ftagree_era_comm_agreement_specific_t*)(comm)->agreement_specific )
 
 /* This mutex is used for both thread safety and to prevent recursive
  * invocation of ERA callbacks; ERA callbacks are not reentrant. Hence,
@@ -276,7 +280,7 @@ static opal_mutex_t era_mutex;
 static opal_mutex_t era_incomplete_msg_mutex;
 
 
-typedef struct era_iagree_request_s era_iagree_request_t;
+typedef struct era_iagree_request_s ompi_coll_ftagree_era_iagree_request_t;
 
 /**
  * Main structure to remember the current status of an agreement that
@@ -285,13 +289,15 @@ typedef struct era_iagree_request_s era_iagree_request_t;
 typedef struct {
     opal_object_t         super;
     era_identifier_t      agreement_id;
-    era_value_t          *current_value;
+    ompi_coll_ftagree_era_value_t          *current_value;
     era_proc_status_t     status;            /**< status of this process in that agreement. */
     uint16_t              min_passed_aid;
     ompi_communicator_t  *comm;              /**< Communicator related to that agreement. Might be NULL
                                               *   if this process has not entered the agreement yet.*/
-    era_iagree_request_t *req;               /**< Request, if this is called through an iagree */
-    era_comm_agreement_specific_t *ags;      /**< Communicator-specific agreement information, at
+    ompi_coll_ftagree_era_iagree_request_t *req;
+                                             /**< Request, if this is called through an iagree */
+    ompi_coll_ftagree_era_comm_agreement_specific_t *ags;
+                                             /**< Communicator-specific agreement information, at
                                               *   the time this agreement was started (might be different
                                               *   from comm->ags during the agreement) */
     int                   nb_acked;          /**< size of acked */
@@ -306,13 +312,13 @@ typedef struct {
                                               *   and passing over the passed agreements after the decision is taken
                                               *   is too slow */
     int                   waiting_down_from; /**< If in BROADCAST state: who is supposed to send me the info */
-} era_agreement_info_t;
+} ompi_coll_ftagree_era_agreement_info_t;
 
-static void era_build_tree_structure(era_agreement_info_t *ci);
-static int era_next_child(era_agreement_info_t *ci, int prev_child_in_comm);
-static int era_parent(era_agreement_info_t *ci);
+static void era_build_tree_structure(ompi_coll_ftagree_era_agreement_info_t *ci);
+static int era_next_child(ompi_coll_ftagree_era_agreement_info_t *ci, int prev_child_in_comm);
+static int era_parent(ompi_coll_ftagree_era_agreement_info_t *ci);
 
-static void  era_agreement_info_constructor (era_agreement_info_t *agreement_info)
+static void  era_agreement_info_constructor (ompi_coll_ftagree_era_agreement_info_t *agreement_info)
 {
     agreement_info->agreement_id.ERAID_KEY = 0;
     agreement_info->status = NOT_CONTRIBUTED | OP_NOT_DEFINED;
@@ -332,7 +338,7 @@ static void  era_agreement_info_constructor (era_agreement_info_t *agreement_inf
                          (void*)agreement_info));
 }
 
-static void  era_agreement_info_destructor (era_agreement_info_t *agreement_info)
+static void  era_agreement_info_destructor (ompi_coll_ftagree_era_agreement_info_t *agreement_info)
 {
     opal_list_item_t *li;
     while( NULL != (li = opal_list_remove_first(&agreement_info->early_requesters)) ) {
@@ -362,7 +368,7 @@ static void  era_agreement_info_destructor (era_agreement_info_t *agreement_info
     agreement_info->req = NULL;
 }
 
-OBJ_CLASS_INSTANCE(era_agreement_info_t,
+OBJ_CLASS_INSTANCE(ompi_coll_ftagree_era_agreement_info_t,
                    opal_object_t,
                    era_agreement_info_constructor,
                    era_agreement_info_destructor);
@@ -372,10 +378,10 @@ struct era_iagree_request_s {
     era_identifier_t      agreement_id;
     void                 *contrib;
     ompi_group_t        **outgroup;
-    era_agreement_info_t *ci;
+    ompi_coll_ftagree_era_agreement_info_t *ci;
 };
 
-OBJ_CLASS_INSTANCE(era_iagree_request_t,
+OBJ_CLASS_INSTANCE(ompi_coll_ftagree_era_iagree_request_t,
                    ompi_request_t,
                    NULL,
                    NULL);
@@ -414,6 +420,7 @@ static const char *era_status_to_string(era_proc_status_t s) {
     }
     return "UNDEFINED STATUS";
 }
+#endif /* OPAL_ENABLE_DEBUG */
 
 static const char *era_msg_type_to_string(int type) {
     switch(type) {
@@ -426,24 +433,23 @@ static const char *era_msg_type_to_string(int type) {
     }
     return "UNDEFINED MESSAGE TYPE";
 }
-#endif /* OPAL_ENABLE_DEBUG */
 
-static era_agreement_info_t *era_lookup_agreement_info(era_identifier_t agreement_id)
+static ompi_coll_ftagree_era_agreement_info_t *era_lookup_agreement_info(era_identifier_t agreement_id)
 {
     void *value;
 
     if( opal_hash_table_get_value_uint64(&era_ongoing_agreements,
                                          agreement_id.ERAID_KEY,
                                          &value) == OPAL_SUCCESS ) {
-        return (era_agreement_info_t *)value;
+        return (ompi_coll_ftagree_era_agreement_info_t *)value;
     } else {
         return NULL;
     }
 }
 
-static era_agreement_info_t *era_create_agreement_info(era_identifier_t agreement_id, era_value_header_t *header)
+static ompi_coll_ftagree_era_agreement_info_t *era_create_agreement_info(era_identifier_t agreement_id, era_value_header_t *header)
 {
-    era_agreement_info_t *ci;
+    ompi_coll_ftagree_era_agreement_info_t *ci;
     int r;
     size_t value_bytes;
 #if OPAL_ENABLE_DEBUG
@@ -452,9 +458,9 @@ static era_agreement_info_t *era_create_agreement_info(era_identifier_t agreemen
                                              agreement_id.ERAID_KEY,
                                              &value) != OPAL_SUCCESS );
 #endif /* OPAL_ENABLE_DEBUG */
-    ci = OBJ_NEW(era_agreement_info_t);
+    ci = OBJ_NEW(ompi_coll_ftagree_era_agreement_info_t);
     ci->agreement_id.ERAID_KEY = agreement_id.ERAID_KEY;
-    ci->current_value = OBJ_NEW(era_value_t);
+    ci->current_value = OBJ_NEW(ompi_coll_ftagree_era_value_t);
     memcpy(&ci->current_value->header, header, sizeof(era_value_header_t));
 
     assert( header->datatype >= 0 && header->datatype < OMPI_DATATYPE_MPI_MAX_PREDEFINED );
@@ -491,7 +497,7 @@ static void era_debug_print_group(int lvl, ompi_group_t *group, ompi_communicato
     }
     s = 128 + n * 16;
     str = (char*)malloc(s);
-    sprintf(str, "Group of size %d. Ranks in %d.%d: (", n, comm->c_contextid, comm->c_epoch);
+    sprintf(str, "Group of size %d. Ranks in %d.%d: (", n, comm->c_index, comm->c_epoch);
     p = strlen(str);
     for(i = 0; i < n; i++) {
         snprintf(str + p, s - p, "%d%s", gra[i], i==n-1 ? "" : ", ");
@@ -513,7 +519,7 @@ static void era_debug_print_group(int lvl, ompi_group_t *group, ompi_communicato
 #define era_debug_print_group(g, c, i, h) do {} while(0)
 #endif /* OPAL_ENABLE_DEBUG */
 
-static void era_update_return_value(era_agreement_info_t *ci, int nb_acked, int *acked) {
+static void era_update_return_value(ompi_coll_ftagree_era_agreement_info_t *ci, int nb_acked, int *acked) {
     ompi_group_t *ack_after_agreement_group, *tmp_sub_group;
     int r, abag_array[3];
 
@@ -664,7 +670,7 @@ static int compare_uint16_ts(const void *a, const void *b)
     return (*ia > *ib) - (*ia < *ib);
 }
 
-static void era_merge_new_dead_list(era_agreement_info_t *ci, int nb_src, int *src)
+static void era_merge_new_dead_list(ompi_coll_ftagree_era_agreement_info_t *ci, int nb_src, int *src)
 {
     int  s;
     int *dst, d, nb_dst;
@@ -725,7 +731,7 @@ static void era_merge_new_dead_list(era_agreement_info_t *ci, int nb_src, int *s
     free(merge);
 }
 
-static void era_agreement_value_set_gcrange(era_identifier_t eid, era_value_t *era_value)
+static void era_agreement_value_set_gcrange(era_identifier_t eid, ompi_coll_ftagree_era_value_t *era_value)
 {
     uint16_t *aid_list = NULL;
     uint64_t key64;
@@ -776,10 +782,10 @@ static void era_agreement_value_set_gcrange(era_identifier_t eid, era_value_t *e
 
 }
 
-static void era_update_new_dead_list(era_agreement_info_t *ci)
+static void era_update_new_dead_list(ompi_coll_ftagree_era_agreement_info_t *ci)
 {
     int *ra, s, r, t;
-    era_comm_agreement_specific_t *ags;
+    ompi_coll_ftagree_era_comm_agreement_specific_t *ags;
     ompi_communicator_t *comm;
 
     comm = ci->comm;
@@ -802,12 +808,12 @@ static void era_update_new_dead_list(era_agreement_info_t *ci)
     }
 
     OPAL_OUTPUT_VERBOSE((30, ompi_ftmpi_output_handle,
-                         "%s ftagree:agreement (ERA) agreement (%d.%d).%d -- adding %d procs to the list of newly dead processes",
+                         "%s ftagree:agreement (ERA) agreement (%d.%d).%d -- adding %d procs to the list of newly dead processes (%d currently; AFR size is %d)",
                          OMPI_NAME_PRINT(OMPI_PROC_MY_NAME),
                          ci->agreement_id.ERAID_FIELDS.contextid,
                          ci->agreement_id.ERAID_FIELDS.epoch,
                          ci->agreement_id.ERAID_FIELDS.agreementid,
-                         r));
+                         r, t, ags->afr_size));
 
 #if OPAL_ENABLE_DEBUG
     {
@@ -830,9 +836,9 @@ static void era_update_new_dead_list(era_agreement_info_t *ci)
     free(ra);
 }
 
-static void era_ci_get_clean_ags_copy(era_agreement_info_t *ci)
+static void era_ci_get_clean_ags_copy(ompi_coll_ftagree_era_agreement_info_t *ci)
 {
-    era_comm_agreement_specific_t *old, *new;
+    ompi_coll_ftagree_era_comm_agreement_specific_t *old, *new;
 
     if( AGS(ci->comm)->ags_status & (AGS_TREE_DIRTY | AGS_AFR_DIRTY) ) {
         old = AGS(ci->comm);
@@ -843,7 +849,7 @@ static void era_ci_get_clean_ags_copy(era_agreement_info_t *ci)
              *  reuse it: create a copy for the other cis that point to it,
              *  and attach that new copy to the communicator, then update it.
              */
-            new = OBJ_NEW(era_comm_agreement_specific_t);
+            new = OBJ_NEW(ompi_coll_ftagree_era_comm_agreement_specific_t);
             if( old->afr_size > 0 ) {
                 /** old does not need to remember the agreed_failed ranks, avoid allocating memory */
                 new->agreed_failed_ranks = old->agreed_failed_ranks;
@@ -891,27 +897,27 @@ static void era_ci_get_clean_ags_copy(era_agreement_info_t *ci)
     }
 }
 
-static void era_agreement_info_set_comm(era_agreement_info_t *ci, ompi_communicator_t *comm, ompi_group_t *acked_group)
+static void era_agreement_info_set_comm(ompi_coll_ftagree_era_agreement_info_t *ci, ompi_communicator_t *comm, ompi_group_t *acked_group)
 {
     int *src_ra;
     int r, grp_size;
 
-    assert( comm->c_contextid == ci->agreement_id.ERAID_FIELDS.contextid );
+    assert( comm->c_index     == ci->agreement_id.ERAID_FIELDS.contextid );
     assert( comm->c_epoch     == ci->agreement_id.ERAID_FIELDS.epoch     );
     assert( ci->comm          == NULL                                    );
     ci->comm = comm;
     OBJ_RETAIN(comm);
 
     OPAL_OUTPUT_VERBOSE((30, ompi_ftmpi_output_handle,
-                         "%s ftagree:agreement (ERA) Agreement (%d.%d).%d: assigning to communicator %d\n",
+                         "%s ftagree:agreement (ERA) Agreement (%d.%d).%d: assigning to communicator %s\n",
                          OMPI_NAME_PRINT(OMPI_PROC_MY_NAME),
                          ci->agreement_id.ERAID_FIELDS.contextid,
                          ci->agreement_id.ERAID_FIELDS.epoch,
                          ci->agreement_id.ERAID_FIELDS.agreementid,
-                         comm->c_contextid));
+                         ompi_comm_print_cid(comm)));
 
     if( AGS(comm) == NULL ) {
-        era_comm_agreement_specific_t *ags = OBJ_NEW(era_comm_agreement_specific_t);
+        ompi_coll_ftagree_era_comm_agreement_specific_t *ags = OBJ_NEW(ompi_coll_ftagree_era_comm_agreement_specific_t);
         comm->agreement_specific = &ags->parent;
     }
 
@@ -961,13 +967,13 @@ static void send_msg(ompi_communicator_t *comm,
                      opal_process_name_t *proc_name,
                      era_identifier_t agreement_id,
                      era_msg_type_t type,
-                     era_value_t *value,
+                     ompi_coll_ftagree_era_value_t *value,
                      int          nb_up_msg,
                      int         *ack_failed);
 
 #define ERA_TAG_AGREEMENT MCA_COLL_BASE_TAG_AGREEMENT
 
-static void era_combine_agreement_values(era_agreement_info_t *ni, era_value_t *value)
+static void era_combine_agreement_values(ompi_coll_ftagree_era_agreement_info_t *ni, ompi_coll_ftagree_era_value_t *value)
 {
     ompi_op_t *op;
     ompi_datatype_t *dt;
@@ -1006,6 +1012,7 @@ static void era_combine_agreement_values(era_agreement_info_t *ni, era_value_t *
             ni->current_value->header.max_aid = value->header.max_aid;
     }
 
+    assert(NULL != value->new_dead_array || 0 == value->header.nb_new_dead);
     era_merge_new_dead_list(ni, value->header.nb_new_dead, value->new_dead_array);
 }
 
@@ -1040,7 +1047,7 @@ static int era_tree_check_node(era_tree_t *tree, int tree_size, int r, int displ
     if( tree[r].rank_in_comm <
         tree[ tree[r].parent ].rank_in_comm ) {
         tree_errors++;
-        fprintf(stderr, "TC %s -- %d/%d(%d): broken hiearchy as my parent is %d in the communicator\n",
+        fprintf(stderr, "TC %s -- %d/%d(%d): broken hierarchy as my parent is %d in the communicator\n",
                 OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), r, tree_size, tree[r].rank_in_comm, tree[tree[r].parent].rank_in_comm);
     }
 
@@ -1153,7 +1160,7 @@ typedef struct {
     era_tree_t *tree;
 } hierarch_tree_info_t;
 
-static void era_call_tree_fn(era_agreement_info_t *ci)
+static void era_call_tree_fn(ompi_coll_ftagree_era_agreement_info_t *ci)
 {
 #if 0
     hierarch_tree_info_t *reps, *rep_p;
@@ -1296,7 +1303,7 @@ static void era_call_tree_fn(era_agreement_info_t *ci)
     }
 #else
     /* Hierarchical tree disabled for now. ESS does not give daemon names
-     * anymore. TODO: ENABLE_FT_MPI: restore hierarchical tree capabilitiy. */
+     * anymore. TODO: ENABLE_FT_MPI: restore hierarchical tree capability. */
     era_tree_fn(AGS(ci->comm)->tree, AGS(ci->comm)->tree_size);
 #endif
 }
@@ -1338,7 +1345,7 @@ static char *era_debug_tree(era_tree_t *tree, int tree_size)
 }
 #endif /* OPAL_ENABLE_DEBUG */
 
-static void era_build_tree_structure(era_agreement_info_t *ci)
+static void era_build_tree_structure(ompi_coll_ftagree_era_agreement_info_t *ci)
 {
     int m, n;
     int offset, next_dead_idx;
@@ -1365,23 +1372,21 @@ static void era_build_tree_structure(era_agreement_info_t *ci)
 
     era_call_tree_fn(ci);
 
-    if( ompi_comm_rank(ci->comm) == 0 ) {
-        OPAL_OUTPUT_VERBOSE((4, ompi_ftmpi_output_handle,
-                             "%s ftagree:agreement (ERA) Agreement (%d.%d).%d: re-built the tree structure with size %d: %s\n",
-                             OMPI_NAME_PRINT(OMPI_PROC_MY_NAME),
-                             ci->agreement_id.ERAID_FIELDS.contextid,
-                             ci->agreement_id.ERAID_FIELDS.epoch,
-                             ci->agreement_id.ERAID_FIELDS.agreementid,
-                             AGS(ci->comm)->tree_size,
-                             era_debug_tree(ci->ags->tree, ci->ags->tree_size)));
-    }
+    OPAL_OUTPUT_VERBOSE(((ompi_comm_rank(ci->comm) == 0)? 4: 50, ompi_ftmpi_output_handle,
+                        "%s ftagree:agreement (ERA) Agreement (%d.%d).%d: re-built the tree structure with size %d: %s\n",
+                        OMPI_NAME_PRINT(OMPI_PROC_MY_NAME),
+                        ci->agreement_id.ERAID_FIELDS.contextid,
+                        ci->agreement_id.ERAID_FIELDS.epoch,
+                        ci->agreement_id.ERAID_FIELDS.agreementid,
+                        AGS(ci->comm)->tree_size,
+                        era_debug_tree(ci->ags->tree, ci->ags->tree_size)));
 
 #if OPAL_ENABLE_DEBUG
     era_tree_check(ci->ags->tree, ci->ags->tree_size, 0);
 #endif /* OPAL_ENABLE_DEBUG */
 }
 
-static int era_tree_rank_from_comm_rank(era_agreement_info_t *ci, int r_in_comm)
+static int era_tree_rank_from_comm_rank(ompi_coll_ftagree_era_agreement_info_t *ci, int r_in_comm)
 {
     int r_in_tree = r_in_comm >= ci->ags->tree_size ? ci->ags->tree_size-1 : r_in_comm;
 
@@ -1396,7 +1401,7 @@ static int era_tree_rank_from_comm_rank(era_agreement_info_t *ci, int r_in_comm)
     return r_in_tree;
 }
 
-static void era_tree_remove_node(era_agreement_info_t *ci, int r_in_tree)
+static void era_tree_remove_node(ompi_coll_ftagree_era_agreement_info_t *ci, int r_in_tree)
 {
     /** All indices are in tree */
     int p, c, s, t;
@@ -1516,7 +1521,7 @@ static void era_tree_remove_node(era_agreement_info_t *ci, int r_in_tree)
 #endif /* OPAL_ENABLE_DEBUG */
 }
 
-static int era_parent(era_agreement_info_t *ci)
+static int era_parent(ompi_coll_ftagree_era_agreement_info_t *ci)
 {
     int r_in_comm = ompi_comm_rank(ci->comm);
     int r_in_tree = era_tree_rank_from_comm_rank(ci, r_in_comm);
@@ -1534,7 +1539,7 @@ static int era_parent(era_agreement_info_t *ci)
     }
 }
 
-static int era_next_child(era_agreement_info_t *ci, int prev_child_in_comm)
+static int era_next_child(ompi_coll_ftagree_era_agreement_info_t *ci, int prev_child_in_comm)
 {
     ompi_communicator_t *comm = ci->comm;
     int prev_child_in_tree;
@@ -1599,7 +1604,7 @@ static void era_collect_passed_agreements(era_identifier_t agreement_id, uint16_
         for(r = min_aid; r <= max_aid; r++) {
             pid.ERAID_FIELDS.agreementid = r;
             if( opal_hash_table_get_value_uint64(&era_passed_agreements, pid.ERAID_KEY, &value) == OMPI_SUCCESS ) {
-                era_value_t *av = (era_value_t*)value;
+                ompi_coll_ftagree_era_value_t *av = (ompi_coll_ftagree_era_value_t*)value;
                 opal_hash_table_remove_value_uint64(&era_passed_agreements, pid.ERAID_KEY);
                 OBJ_RELEASE(av);
                 OPAL_OUTPUT_VERBOSE((17, ompi_ftmpi_output_handle,
@@ -1617,25 +1622,25 @@ static void era_collect_passed_agreements(era_identifier_t agreement_id, uint16_
     }
 }
 
-static void era_decide(era_value_t *decided_value, era_agreement_info_t *ci)
+static void era_decide(ompi_coll_ftagree_era_value_t *decided_value, ompi_coll_ftagree_era_agreement_info_t *ci)
 {
     ompi_communicator_t *comm;
-    era_rank_item_t *rl;
+    ompi_coll_ftagree_era_rank_item_t *rl;
     int r, s, dead_size;
-    void *value;
 
     assert( 0 != ci->agreement_id.ERAID_FIELDS.agreementid );
 
 #if OPAL_ENABLE_DEBUG
+    void *value;
     r = era_parent(ci);
     if( opal_hash_table_get_value_uint64(&era_passed_agreements,
                                          ci->agreement_id.ERAID_KEY, &value) == OMPI_SUCCESS ) {
         /**
          * If the value was already decided, then this DOWN message
-         * *must* provide the same decision: it can only be a dupplicate.
+         * *must* provide the same decision: it can only be a duplicate.
          */
-        era_value_t *old_agreement_value;
-        old_agreement_value = (era_value_t*)value;
+        ompi_coll_ftagree_era_value_t *old_agreement_value;
+        old_agreement_value = (ompi_coll_ftagree_era_value_t*)value;
         assert( old_agreement_value->header.ret == decided_value->header.ret &&
                 old_agreement_value->header.nb_new_dead == decided_value->header.nb_new_dead );
     }
@@ -1655,7 +1660,7 @@ static void era_decide(era_value_t *decided_value, era_agreement_info_t *ci)
     OPAL_OUTPUT_VERBOSE(((ci->comm->c_my_rank == r)? 2: 10, ompi_ftmpi_output_handle,
                          "%s ftagree:agreement (ERA) decide %08x.%d.%d.. on agreement (%d.%d).%d\n",
                          OMPI_NAME_PRINT(OMPI_PROC_MY_NAME),
-                         (0!=ERA_VALUE_BYTES_COUNT(&decided_value->header))? *(int*)decided_value->bytes: 0,
+                         (NULL != decided_value->bytes)? *(int*)decided_value->bytes: 0,
                          decided_value->header.ret,
                          decided_value->header.nb_new_dead,
                          ci->agreement_id.ERAID_FIELDS.contextid,
@@ -1673,7 +1678,7 @@ static void era_decide(era_value_t *decided_value, era_agreement_info_t *ci)
         OPAL_OUTPUT_VERBOSE((30, ompi_ftmpi_output_handle,
                              "%s ftagree:agreement (ERA) decide %08x.%d.%d on agreement (%d.%d).%d: adding up to %d processes to the list of agreed deaths\n",
                              OMPI_NAME_PRINT(OMPI_PROC_MY_NAME),
-                             (0!=ERA_VALUE_BYTES_COUNT(&decided_value->header))? *(int*)decided_value->bytes: 0,
+                             (NULL != decided_value->bytes)? *(int*)decided_value->bytes: 0,
                              decided_value->header.ret,
                              decided_value->header.nb_new_dead,
                              ci->agreement_id.ERAID_FIELDS.contextid,
@@ -1734,7 +1739,7 @@ static void era_decide(era_value_t *decided_value, era_agreement_info_t *ci)
     OPAL_OUTPUT_VERBOSE((10, ompi_ftmpi_output_handle,
                          "%s ftagree:agreement (ERA) decide %08x.%d.%d.. on agreement (%d.%d).%d: group of agreed deaths is of size %d\n",
                          OMPI_NAME_PRINT(OMPI_PROC_MY_NAME),
-                         (0!=ERA_VALUE_BYTES_COUNT(&decided_value->header))? *(int*)decided_value->bytes: 0,
+                         (NULL != decided_value->bytes)? *(int*)decided_value->bytes: 0,
                          decided_value->header.ret,
                          decided_value->header.nb_new_dead,
                          ci->agreement_id.ERAID_FIELDS.contextid,
@@ -1745,11 +1750,11 @@ static void era_decide(era_value_t *decided_value, era_agreement_info_t *ci)
     r = -1;
     while( (r = era_next_child(ci, r)) < ompi_comm_size(comm) ) {
 
-        /** Cleanup the early_requesters list, to avoid sending unecessary dupplicate messages */
+        /** Cleanup the early_requesters list, to avoid sending unnecessary duplicate messages */
         if( opal_list_get_size(&ci->early_requesters) > 0 ) {
-            for(rl = (era_rank_item_t*)opal_list_get_first(&ci->early_requesters);
-                rl != (era_rank_item_t*)opal_list_get_end(&ci->early_requesters);
-                rl = (era_rank_item_t*)opal_list_get_next(&rl->super)) {
+            for(rl = (ompi_coll_ftagree_era_rank_item_t*)opal_list_get_first(&ci->early_requesters);
+                rl != (ompi_coll_ftagree_era_rank_item_t*)opal_list_get_end(&ci->early_requesters);
+                rl = (ompi_coll_ftagree_era_rank_item_t*)opal_list_get_next(&rl->super)) {
                 if( rl->rank == r ) {
                     opal_list_remove_item(&ci->early_requesters, &rl->super);
                     break;
@@ -1762,9 +1767,9 @@ static void era_decide(era_value_t *decided_value, era_agreement_info_t *ci)
 
     /* In case we have some child we haven't detected yet */
     if( opal_list_get_size(&ci->early_requesters) > 0 ) {
-        for(rl = (era_rank_item_t*)opal_list_get_first(&ci->early_requesters);
-            rl != (era_rank_item_t*)opal_list_get_end(&ci->early_requesters);
-            rl = (era_rank_item_t*)opal_list_get_next(&rl->super)) {
+        for(rl = (ompi_coll_ftagree_era_rank_item_t*)opal_list_get_first(&ci->early_requesters);
+            rl != (ompi_coll_ftagree_era_rank_item_t*)opal_list_get_end(&ci->early_requesters);
+            rl = (ompi_coll_ftagree_era_rank_item_t*)opal_list_get_next(&rl->super)) {
             send_msg(comm, rl->rank, NULL, ci->agreement_id, MSG_DOWN, decided_value, 0, NULL);
         }
     }
@@ -1785,10 +1790,10 @@ static void era_decide(era_value_t *decided_value, era_agreement_info_t *ci)
     }
 }
 
-static void era_check_status(era_agreement_info_t *ci)
+static void era_check_status(ompi_coll_ftagree_era_agreement_info_t *ci)
 {
     int r;
-    era_rank_item_t *rl;
+    ompi_coll_ftagree_era_rank_item_t *rl;
 
     OPAL_OUTPUT_VERBOSE((10, ompi_ftmpi_output_handle,
                          "%s ftagree:agreement (ERA) check_status for Agreement ID = (%d.%d).%d, current status = %s\n",
@@ -1819,9 +1824,9 @@ static void era_check_status(era_agreement_info_t *ci)
                                  ci->agreement_id.ERAID_FIELDS.epoch,
                                  ci->agreement_id.ERAID_FIELDS.agreementid,
                                  r));
-            for( rl =  (era_rank_item_t*)opal_list_get_first(&ci->gathered_info);
-                 rl != (era_rank_item_t*)opal_list_get_end(&ci->gathered_info);
-                 rl =  (era_rank_item_t*)opal_list_get_next(&rl->super) ) {
+            for( rl =  (ompi_coll_ftagree_era_rank_item_t*)opal_list_get_first(&ci->gathered_info);
+                 rl != (ompi_coll_ftagree_era_rank_item_t*)opal_list_get_end(&ci->gathered_info);
+                 rl =  (ompi_coll_ftagree_era_rank_item_t*)opal_list_get_next(&rl->super) ) {
                 if( rl->rank == r ) {
                     OPAL_OUTPUT_VERBOSE((20, ompi_ftmpi_output_handle,
                                          "%s ftagree:agreement (ERA) check_status for Agreement ID = (%d.%d).%d, child %d has sent its message\n",
@@ -1833,7 +1838,7 @@ static void era_check_status(era_agreement_info_t *ci)
                     break;
                 }
             }
-            if( rl == (era_rank_item_t*)opal_list_get_end(&ci->gathered_info) ) {
+            if( rl == (ompi_coll_ftagree_era_rank_item_t*)opal_list_get_end(&ci->gathered_info) ) {
                 OPAL_OUTPUT_VERBOSE((20, ompi_ftmpi_output_handle,
                                      "%s ftagree:agreement (ERA) check_status for Agreement ID = (%d.%d).%d, some children have not contributed\n",
                                      OMPI_NAME_PRINT(OMPI_PROC_MY_NAME),
@@ -1876,10 +1881,10 @@ static void era_check_status(era_agreement_info_t *ci)
     }
 }
 
-static void restart_agreement_from_me(era_agreement_info_t *ci)
+static void restart_agreement_from_me(ompi_coll_ftagree_era_agreement_info_t *ci)
 {
     int r;
-    era_rank_item_t *rc;
+    ompi_coll_ftagree_era_rank_item_t *rc;
     assert( NULL != ci->comm );
     assert( 0 == opal_list_get_size(&ci->waiting_res_from) );
 
@@ -1896,7 +1901,7 @@ static void restart_agreement_from_me(era_agreement_info_t *ci)
      */
     r = -1;
     while( (r = era_next_child(ci, r)) != ompi_comm_size(ci->comm) ) {
-        rc = OBJ_NEW(era_rank_item_t);
+        rc = OBJ_NEW(ompi_coll_ftagree_era_rank_item_t);
         rc->rank = r;
         opal_list_append(&ci->waiting_res_from, &rc->super);
         send_msg(ci->comm, r, NULL, ci->agreement_id, MSG_RESULT_REQUEST, ci->current_value,
@@ -1913,25 +1918,25 @@ static void restart_agreement_from_me(era_agreement_info_t *ci)
 /* helper to bounce recursive errors back from to the main thread opal_progress */
 typedef struct era_error_event_s {
     opal_event_t ev;
-    era_agreement_info_t* ci;
+    ompi_coll_ftagree_era_agreement_info_t* ci;
     int rank;
 } era_error_event_t;
 
-static void era_mark_process_failed(era_agreement_info_t *ci, int rank);
+static void era_mark_process_failed(ompi_coll_ftagree_era_agreement_info_t *ci, int rank);
 
 static void *era_error_event_cb(int fd, int flags, void *context) {
     era_error_event_t *event = (era_error_event_t*) context;
     int r = event->rank;
-    era_agreement_info_t* ci = event->ci;
+    ompi_coll_ftagree_era_agreement_info_t* ci = event->ci;
     free(event);
     era_mark_process_failed(ci, r);
     return NULL;
 }
 
-static void era_mark_process_failed(era_agreement_info_t *ci, int rank)
+static void era_mark_process_failed(ompi_coll_ftagree_era_agreement_info_t *ci, int rank)
 {
     int r;
-    era_rank_item_t *rl;
+    ompi_coll_ftagree_era_rank_item_t *rl;
 
     assert( ci->comm == NULL || (ci->comm->c_local_group->grp_my_rank != rank) );
 
@@ -2018,9 +2023,9 @@ static void era_mark_process_failed(era_agreement_info_t *ci, int rank)
                              ci->agreement_id.ERAID_FIELDS.agreementid));
         OBJ_RETAIN(ci);
         /* It could be one of the guys that we contacted about a restarting agreement. */
-        for(rl = (era_rank_item_t*)opal_list_get_first(&ci->waiting_res_from);
-            rl != (era_rank_item_t*)opal_list_get_end(&ci->waiting_res_from);
-            rl = (era_rank_item_t*)opal_list_get_next(&rl->super)) {
+        for(rl = (ompi_coll_ftagree_era_rank_item_t*)opal_list_get_first(&ci->waiting_res_from);
+            rl != (ompi_coll_ftagree_era_rank_item_t*)opal_list_get_end(&ci->waiting_res_from);
+            rl = (ompi_coll_ftagree_era_rank_item_t*)opal_list_get_next(&rl->super)) {
             if(rl->rank == rank) {
                 OPAL_OUTPUT_VERBOSE((20,  ompi_ftmpi_output_handle,
                                      "%s ftagree:agreement (ERA) Handling failure of process %d: I was waiting for the contribution of that process for Agreement (%d.%d).%d.\n",
@@ -2041,15 +2046,15 @@ static void era_mark_process_failed(era_agreement_info_t *ci, int rank)
                  */
                 r = -1;
                 while( (r = era_next_child(ci, r)) != ompi_comm_size(ci->comm) ) {
-                    for(rl = (era_rank_item_t*)opal_list_get_first(&ci->waiting_res_from);
-                        rl != (era_rank_item_t*)opal_list_get_end(&ci->waiting_res_from);
-                        rl = (era_rank_item_t*)opal_list_get_next(&rl->super)) {
+                    for(rl = (ompi_coll_ftagree_era_rank_item_t*)opal_list_get_first(&ci->waiting_res_from);
+                        rl != (ompi_coll_ftagree_era_rank_item_t*)opal_list_get_end(&ci->waiting_res_from);
+                        rl = (ompi_coll_ftagree_era_rank_item_t*)opal_list_get_next(&rl->super)) {
                         if( rl->rank == r ) {
                             break;
                         }
                     }
 
-                    if( rl == (era_rank_item_t*)opal_list_get_end(&ci->waiting_res_from) ) {
+                    if( rl == (ompi_coll_ftagree_era_rank_item_t*)opal_list_get_end(&ci->waiting_res_from) ) {
                         OPAL_OUTPUT_VERBOSE((20,  ompi_ftmpi_output_handle,
                                              "%s ftagree:agreement (ERA) Handling failure of process %d: Requesting contribution of process %d for Agreement (%d.%d).%d.\n",
                                              OMPI_NAME_PRINT(OMPI_PROC_MY_NAME),
@@ -2057,7 +2062,7 @@ static void era_mark_process_failed(era_agreement_info_t *ci, int rank)
                                              ci->agreement_id.ERAID_FIELDS.contextid,
                                              ci->agreement_id.ERAID_FIELDS.epoch,
                                              ci->agreement_id.ERAID_FIELDS.agreementid));
-                        rl = OBJ_NEW(era_rank_item_t);
+                        rl = OBJ_NEW(ompi_coll_ftagree_era_rank_item_t);
                         rl->rank = r;
                         opal_list_append(&ci->waiting_res_from, &rl->super);
                         send_msg(ci->comm, r, NULL, ci->agreement_id, MSG_RESULT_REQUEST, ci->current_value,
@@ -2092,7 +2097,7 @@ static void send_msg(ompi_communicator_t *comm,
                      opal_process_name_t *proc_name,
                      era_identifier_t agreement_id,
                      era_msg_type_t type,
-                     era_value_t *value,
+                     ompi_coll_ftagree_era_value_t *value,
                      int          nb_ack_failed,
                      int         *ack_failed)
 {
@@ -2119,7 +2124,7 @@ static void send_msg(ompi_communicator_t *comm,
                              agreement_id.ERAID_FIELDS.epoch,
                              agreement_id.ERAID_FIELDS.agreementid,
                              era_msg_type_to_string(type),
-                             (0!=ERA_VALUE_BYTES_COUNT(&value->header))? *(int*)value->bytes: 0,
+                             (NULL != value->bytes)? *(int*)value->bytes: 0,
                              value->header.ret,
                              value->header.nb_new_dead,
                              dst,
@@ -2136,7 +2141,7 @@ static void send_msg(ompi_communicator_t *comm,
                              agreement_id.ERAID_FIELDS.epoch,
                              agreement_id.ERAID_FIELDS.agreementid,
                              era_msg_type_to_string(type),
-                             (0!=ERA_VALUE_BYTES_COUNT(&value->header))? *(int*)value->bytes: 0,
+                             (NULL != value->bytes)? *(int*)value->bytes: 0,
                              value->header.ret,
                              value->header.nb_new_dead,
                              nb_ack_failed,
@@ -2150,7 +2155,7 @@ static void send_msg(ompi_communicator_t *comm,
                              agreement_id.ERAID_FIELDS.epoch,
                              agreement_id.ERAID_FIELDS.agreementid,
                              era_msg_type_to_string(type),
-                             (0!=ERA_VALUE_BYTES_COUNT(&value->header))? *(int*)value->bytes: 0,
+                             (NULL != value->bytes)? *(int*)value->bytes: 0,
                              value->header.ret,
                              value->header.nb_new_dead,
                              dst,
@@ -2166,7 +2171,7 @@ static void send_msg(ompi_communicator_t *comm,
     }
 #endif /* OPAL_ENABLE_DEBUG */
 
-    assert( NULL == comm || agreement_id.ERAID_FIELDS.contextid == ompi_comm_get_cid(comm) );
+    assert( NULL == comm || agreement_id.ERAID_FIELDS.contextid == ompi_comm_get_local_cid(comm) );
     assert( NULL == comm || agreement_id.ERAID_FIELDS.epoch == comm->c_epoch );
 
     if( NULL == comm ) {
@@ -2177,7 +2182,21 @@ static void send_msg(ompi_communicator_t *comm,
     }
     assert(NULL != peer);
     endpoint = mca_bml_base_get_endpoint(peer);
-    assert(NULL != endpoint);
+    if(NULL == endpoint) {
+      opal_output_verbose(5, ompi_ftmpi_output_handle,
+                             "%s ftagree:agreement (ERA) CANNOT send message [(%d.%d).%d, %s, %08x.%d.%d..] to %d/%s (no endpoint)\n",
+                             OMPI_NAME_PRINT(OMPI_PROC_MY_NAME),
+                             agreement_id.ERAID_FIELDS.contextid,
+                             agreement_id.ERAID_FIELDS.epoch,
+                             agreement_id.ERAID_FIELDS.agreementid,
+                             era_msg_type_to_string(type),
+                             (NULL != value->bytes)? *(int*)value->bytes: 0,
+                             value->header.ret,
+                             value->header.nb_new_dead,
+                             dst,
+                             NULL != proc_name ? OMPI_NAME_PRINT(proc_name) : "(null)");
+      return; /* bail out: the algorithm should reconnect when the failed proc is detected */
+    }
     bml_btl = mca_bml_base_btl_array_get_index(&endpoint->btl_eager, 0);
     assert(NULL != bml_btl);
     btl_endpoint = bml_btl->btl_endpoint;
@@ -2230,7 +2249,7 @@ static void send_msg(ompi_communicator_t *comm,
                          agreement_id.ERAID_FIELDS.epoch,
                          agreement_id.ERAID_FIELDS.agreementid,
                          era_msg_type_to_string(type),
-                         (0!=ERA_VALUE_BYTES_COUNT(&value->header))? *(int*)value->bytes: 0,
+                         (NULL != value->bytes)? *(int*)value->bytes: 0,
                          value->header.ret,
                          value->header.nb_new_dead,
                          msg_header.nb_ack,
@@ -2274,7 +2293,7 @@ static void send_msg(ompi_communicator_t *comm,
                                      agreement_id.ERAID_FIELDS.epoch,
                                      agreement_id.ERAID_FIELDS.agreementid,
                                      era_msg_type_to_string(type),
-                                     (0!=ERA_VALUE_BYTES_COUNT(&value->header))? *(int*)value->bytes: 0,
+                                     (NULL != value->bytes)? *(int*)value->bytes: 0,
                                      value->header.ret,
                                      value->header.nb_new_dead,
                                      msg_header.nb_ack,
@@ -2329,8 +2348,8 @@ static void send_msg(ompi_communicator_t *comm,
 static void result_request(era_msg_header_t *msg_header)
 {
     void *value;
-    era_value_t *old_agreement_value;
-    era_agreement_info_t *ci;
+    ompi_coll_ftagree_era_value_t *old_agreement_value;
+    ompi_coll_ftagree_era_agreement_info_t *ci;
     int r;
 
     OPAL_OUTPUT_VERBOSE((5, ompi_ftmpi_output_handle,
@@ -2345,7 +2364,7 @@ static void result_request(era_msg_header_t *msg_header)
     if( opal_hash_table_get_value_uint64(&era_passed_agreements,
                                          msg_header->agreement_id.ERAID_KEY,
                                          &value) == OPAL_SUCCESS ) {
-        old_agreement_value = (era_value_t*)value;
+        old_agreement_value = (ompi_coll_ftagree_era_value_t*)value;
         send_msg(NULL, msg_header->src_comm_rank, &msg_header->src_proc_name, msg_header->agreement_id, MSG_DOWN, old_agreement_value, 0, NULL);
         return;
     }
@@ -2374,8 +2393,8 @@ static void result_request(era_msg_header_t *msg_header)
         send_msg(ci->comm, r, NULL, ci->agreement_id, MSG_UP, ci->current_value,
                  ci->nb_acked, ci->acked);
     } else {
-        era_value_t success_value;
-        OBJ_CONSTRUCT(&success_value, era_value_t);
+        ompi_coll_ftagree_era_value_t success_value;
+        OBJ_CONSTRUCT(&success_value, ompi_coll_ftagree_era_value_t);
         success_value.header.ret = MPI_SUCCESS;
         success_value.header.dt_count = 0;
         success_value.header.operand  = ompi_mpi_op_band.op.o_f_to_c_index;
@@ -2408,10 +2427,10 @@ static void result_request(era_msg_header_t *msg_header)
 
 static void msg_up(era_msg_header_t *msg_header, uint8_t *bytes, int *new_dead, int *ack_failed)
 {
-    era_agreement_info_t *ci;
-    era_rank_item_t *rank_item;
+    ompi_coll_ftagree_era_agreement_info_t *ci;
+    ompi_coll_ftagree_era_rank_item_t *rank_item;
     void *value;
-    era_value_t *av;
+    ompi_coll_ftagree_era_value_t *av;
 
     OPAL_OUTPUT_VERBOSE((5, ompi_ftmpi_output_handle,
                          "%s ftagree:agreement (ERA) Received UP Message: Agreement ID = (%d.%d).%d, sender: %d/%s, msg value: %08x.%d.%d/%d\n",
@@ -2421,7 +2440,7 @@ static void msg_up(era_msg_header_t *msg_header, uint8_t *bytes, int *new_dead,
                          msg_header->agreement_id.ERAID_FIELDS.agreementid,
                          msg_header->src_comm_rank,
                          OMPI_NAME_PRINT(&msg_header->src_proc_name),
-                         (0!=ERA_VALUE_BYTES_COUNT(&msg_header->agreement_value_header))? *(int*)bytes: 0,
+                         (NULL != bytes)? *(int*)bytes: 0,
                          msg_header->agreement_value_header.ret,
                          msg_header->agreement_value_header.nb_new_dead,
                          msg_header->nb_ack));
@@ -2435,7 +2454,7 @@ static void msg_up(era_msg_header_t *msg_header, uint8_t *bytes, int *new_dead,
     if( opal_hash_table_get_value_uint64(&era_passed_agreements,
                                          msg_header->agreement_id.ERAID_KEY,
                                          &value) == OPAL_SUCCESS ) {
-        av = (era_value_t*)value;
+        av = (ompi_coll_ftagree_era_value_t*)value;
         send_msg(NULL, msg_header->src_comm_rank, &msg_header->src_proc_name, msg_header->agreement_id, MSG_DOWN, av,
                  0, NULL);
         return;
@@ -2475,16 +2494,16 @@ static void msg_up(era_msg_header_t *msg_header, uint8_t *bytes, int *new_dead,
                              msg_header->src_comm_rank));
 
         /** We could receive multiple messages from msg_header->src_comm_rank, because the messages are split */
-        for(rank_item = (era_rank_item_t*)opal_list_get_first(&ci->early_requesters);
-            rank_item != (era_rank_item_t*)opal_list_get_end(&ci->early_requesters);
-            rank_item = (era_rank_item_t*)opal_list_get_next(&rank_item->super)) {
+        for(rank_item = (ompi_coll_ftagree_era_rank_item_t*)opal_list_get_first(&ci->early_requesters);
+            rank_item != (ompi_coll_ftagree_era_rank_item_t*)opal_list_get_end(&ci->early_requesters);
+            rank_item = (ompi_coll_ftagree_era_rank_item_t*)opal_list_get_next(&rank_item->super)) {
             if( rank_item->rank == msg_header->src_comm_rank ) {
                 return;
             }
         }
 
         /** If not, add it */
-        rank_item = OBJ_NEW(era_rank_item_t);
+        rank_item = OBJ_NEW(ompi_coll_ftagree_era_rank_item_t);
         rank_item->rank = msg_header->src_comm_rank;
         opal_list_append(&ci->early_requesters, &rank_item->super);
         return;
@@ -2496,9 +2515,9 @@ static void msg_up(era_msg_header_t *msg_header, uint8_t *bytes, int *new_dead,
      *  of another failure), but it discovered the first failure at the
      *  same time, and started sending without me requesting.
      */
-    for( rank_item = (era_rank_item_t *)opal_list_get_first( &ci->gathered_info );
-         rank_item != (era_rank_item_t *)opal_list_get_end( &ci->gathered_info );
-         rank_item = (era_rank_item_t *)opal_list_get_next( &rank_item->super ) ) {
+    for( rank_item = (ompi_coll_ftagree_era_rank_item_t *)opal_list_get_first( &ci->gathered_info );
+         rank_item != (ompi_coll_ftagree_era_rank_item_t *)opal_list_get_end( &ci->gathered_info );
+         rank_item = (ompi_coll_ftagree_era_rank_item_t *)opal_list_get_next( &rank_item->super ) ) {
         if( rank_item->rank == msg_header->src_comm_rank ) {
             /* We are not waiting from more messages, thank you */
             /* Maybe you're telling me again you want me to send? Let's check if I can't */
@@ -2507,7 +2526,7 @@ static void msg_up(era_msg_header_t *msg_header, uint8_t *bytes, int *new_dead,
         }
     }
 
-    av = OBJ_NEW(era_value_t);
+    av = OBJ_NEW(ompi_coll_ftagree_era_value_t);
     memcpy(&av->header, &msg_header->agreement_value_header, sizeof(era_value_header_t));
     /* We don't allocate the arrays of bytes and new_dead ranks: we point in the message.
      * These pointers will need to be set to NULL *before* calling RELEASE.
@@ -2527,7 +2546,7 @@ static void msg_up(era_msg_header_t *msg_header, uint8_t *bytes, int *new_dead,
     OBJ_RELEASE(av);
 
     /* We already checked above that this process did not contribute yet */
-    rank_item = OBJ_NEW(era_rank_item_t);
+    rank_item = OBJ_NEW(ompi_coll_ftagree_era_rank_item_t);
     rank_item->rank = msg_header->src_comm_rank;
     opal_list_append(&ci->gathered_info, &rank_item->super);
     OPAL_OUTPUT_VERBOSE((20, ompi_ftmpi_output_handle,
@@ -2540,8 +2559,8 @@ static void msg_up(era_msg_header_t *msg_header, uint8_t *bytes, int *new_dead,
 
 static void msg_down(era_msg_header_t *msg_header, uint8_t *bytes, int *new_dead)
 {
-    era_agreement_info_t *ci;
-    era_value_t *av;
+    ompi_coll_ftagree_era_agreement_info_t *ci;
+    ompi_coll_ftagree_era_value_t *av;
     size_t value_bytes;
 
     OPAL_OUTPUT_VERBOSE((5, ompi_ftmpi_output_handle,
@@ -2552,7 +2571,7 @@ static void msg_down(era_msg_header_t *msg_header, uint8_t *bytes, int *new_dead
                          msg_header->agreement_id.ERAID_FIELDS.agreementid,
                          msg_header->src_comm_rank,
                          OMPI_NAME_PRINT(&msg_header->src_proc_name),
-                         (0!=ERA_VALUE_BYTES_COUNT(&msg_header->agreement_value_header))? *(int*)bytes: 0,
+                         (NULL != bytes)? *(int*)bytes: 0,
                          msg_header->agreement_value_header.nb_new_dead));
 
     ci = era_lookup_agreement_info( msg_header->agreement_id );
@@ -2563,7 +2582,7 @@ static void msg_down(era_msg_header_t *msg_header, uint8_t *bytes, int *new_dead
          */
         return;
     }
-    /** if I receive a down message on an agreement I know about, I already participated. 
+    /** if I receive a down message on an agreement I know about, I already participated.
      * There is a non-erroneous code; erroneous execution that may also trigger this assert:
      * consider the following case with false detection:
      *   1. some ancestor A has detected the current process C as failed
@@ -2576,7 +2595,7 @@ static void msg_down(era_msg_header_t *msg_header, uint8_t *bytes, int *new_dead
      */
     assert( NULL != ci->comm );
 
-    av = OBJ_NEW(era_value_t);
+    av = OBJ_NEW(ompi_coll_ftagree_era_value_t);
     memcpy(&av->header, &msg_header->agreement_value_header, sizeof(era_value_header_t));
     /* We must allocate the arrays of bytes and new_dead ranks, because era_decide is going
      * to keep that era_value_t */
@@ -2635,7 +2654,6 @@ static void era_cb_fn(struct mca_btl_base_module_t* btl,
                       const mca_btl_base_receive_descriptor_t* descriptor)
 {
     era_incomplete_msg_t *incomplete_msg = NULL;
-    mca_btl_base_tag_t tag = descriptor->tag;
     era_msg_header_t *msg_header;
     era_frag_t *frag;
     uint64_t src_hash;
@@ -2645,7 +2663,7 @@ static void era_cb_fn(struct mca_btl_base_module_t* btl,
     int *new_dead;
     int *ack_failed;
 
-    assert(MCA_BTL_TAG_FT_AGREE == tag);
+    assert(MCA_BTL_TAG_FT_AGREE == descriptor->tag);
     assert(1 == descriptor->des_segment_count);
 
     frag = (era_frag_t*)descriptor->des_segments->seg_addr.pval;
@@ -2723,7 +2741,7 @@ static void era_cb_fn(struct mca_btl_base_module_t* btl,
                              msg_header->agreement_id.ERAID_FIELDS.epoch,
                              msg_header->agreement_id.ERAID_FIELDS.agreementid,
                              msg_header->msg_type,
-                             (0!=ERA_VALUE_BYTES_COUNT(&msg_header->agreement_value_header))? *(int*)value_bytes: 0,
+                             (NULL != value_bytes)? *(int*)value_bytes: 0,
                              msg_header->agreement_value_header.ret,
                              msg_header->agreement_value_header.nb_new_dead,
                              msg_header->src_comm_rank,
@@ -2765,7 +2783,7 @@ static void era_cb_fn(struct mca_btl_base_module_t* btl,
 static void era_on_comm_rank_failure(ompi_communicator_t *comm, int rank, bool remote)
 {
     void *value, *next_value;
-    era_agreement_info_t *ci;
+    ompi_coll_ftagree_era_agreement_info_t *ci;
     void *node;
     uint64_t key64, key64_2;
     int rc;
@@ -2774,10 +2792,10 @@ static void era_on_comm_rank_failure(ompi_communicator_t *comm, int rank, bool r
     opal_hash_table_t *msg_table;
 
     OPAL_OUTPUT_VERBOSE((4, ompi_ftmpi_output_handle,
-                         "%s ftagree:agreement (ERA) %d in communicator (%d.%d) died\n",
+                         "%s ftagree:agreement (ERA) %d in communicator (%s.%d) died\n",
                          OMPI_NAME_PRINT(OMPI_PROC_MY_NAME),
                          rank,
-                         comm->c_contextid,
+                         ompi_comm_print_cid(comm),
                          comm->c_epoch));
 
     if( AGS(comm) != NULL ) {
@@ -2810,9 +2828,9 @@ static void era_on_comm_rank_failure(ompi_communicator_t *comm, int rank, bool r
                                                      &key64, &next_value,
                                                      node, &node);
 
-            if( cid.ERAID_FIELDS.contextid == comm->c_contextid &&
+            if( cid.ERAID_FIELDS.contextid == comm->c_contextid.cid_sub.u64 &&
                 cid.ERAID_FIELDS.epoch     == comm->c_epoch ) {
-                ci = (era_agreement_info_t *)value;
+                ci = (ompi_coll_ftagree_era_agreement_info_t *)value;
                 OPAL_OUTPUT_VERBOSE((6, ompi_ftmpi_output_handle,
                                      "%s ftagree:agreement (ERA) Agreement ID (%d.%d).%d, rank %d died while doing the agreement\n",
                                      OMPI_NAME_PRINT(OMPI_PROC_MY_NAME),
@@ -2872,9 +2890,9 @@ int mca_coll_ftagree_era_init(void)
 
     OBJ_CONSTRUCT( &era_iagree_requests, opal_free_list_t);
     opal_free_list_init( &era_iagree_requests,
-                         sizeof(era_iagree_request_t),
+                         sizeof(ompi_coll_ftagree_era_iagree_request_t),
                          opal_cache_line_size,
-                         OBJ_CLASS(era_iagree_request_t),
+                         OBJ_CLASS(ompi_coll_ftagree_era_iagree_request_t),
                          0, opal_cache_line_size,
                          /* initial number of elements to allocate */ 0,
                          /* maximum number of elements */ INT_MAX,
@@ -2917,8 +2935,8 @@ int mca_coll_ftagree_era_finalize(void)
     void                 *node;
     void                 *value;
     uint64_t              key64;
-    era_value_t          *av;
-    era_agreement_info_t *un_agreement;
+    ompi_coll_ftagree_era_value_t          *av;
+    ompi_coll_ftagree_era_agreement_info_t *un_agreement;
     opal_hash_table_t    *msg_table;
     era_incomplete_msg_t *inc_msg;
     int rc;
@@ -2950,7 +2968,7 @@ int mca_coll_ftagree_era_finalize(void)
                              pid.ERAID_FIELDS.epoch,
                              pid.ERAID_FIELDS.agreementid));
 #endif /* OPAL_ENABLE_DEBUG */
-        av = (era_value_t *)value;
+        av = (ompi_coll_ftagree_era_value_t *)value;
         OBJ_RELEASE(av);
     }
     OBJ_DESTRUCT( &era_passed_agreements );
@@ -2959,7 +2977,7 @@ int mca_coll_ftagree_era_finalize(void)
     for( rc = opal_hash_table_get_first_key_uint64(&era_ongoing_agreements, &key64, &value, &node);
          OPAL_SUCCESS == rc;
          rc = opal_hash_table_get_next_key_uint64(&era_ongoing_agreements, &key64, &value, node, &node) ) {
-        un_agreement = (era_agreement_info_t *)value;
+        un_agreement = (ompi_coll_ftagree_era_agreement_info_t *)value;
         opal_output(0, "%s ftagree:agreement (ERA) ERRONEOUS: Agreement ID (%d.%d).%d was started by some processor, but I never completed to it\n",
                     OMPI_NAME_PRINT(OMPI_PROC_MY_NAME),
                     un_agreement->agreement_id.ERAID_FIELDS.contextid,
@@ -3000,17 +3018,17 @@ static int mca_coll_ftagree_era_prepare_agreement(ompi_communicator_t* comm,
                                                             ompi_group_t *group,
                                                             ompi_op_t *op,
                                                             ompi_datatype_t *dt,
-                                                            int dt_count,
+                                                            size_t dt_count,
                                                             void *contrib,
                                                             mca_coll_base_module_t *module,
                                                             era_identifier_t *paid,
-                                                            era_agreement_info_t **pci)
+                                                            ompi_coll_ftagree_era_agreement_info_t **pci)
 {
-    era_agreement_info_t *ci;
+    ompi_coll_ftagree_era_agreement_info_t *ci;
     era_identifier_t agreement_id;
     void *value;
-    era_value_t agreement_value;
-    era_value_t *pa;
+    ompi_coll_ftagree_era_value_t agreement_value;
+    ompi_coll_ftagree_era_value_t *pa;
     mca_coll_ftagree_t *ag_info;
 
     ag_info = ( (mca_coll_ftagree_module_t *)module )->agreement_info;
@@ -3026,7 +3044,7 @@ static int mca_coll_ftagree_era_prepare_agreement(ompi_communicator_t* comm,
     }
 
     /* Let's find the id of the new agreement */
-    agreement_id.ERAID_FIELDS.contextid   = comm->c_contextid;
+    agreement_id.ERAID_FIELDS.contextid   = comm->c_contextid.cid_sub.u64;
     agreement_id.ERAID_FIELDS.epoch       = comm->c_epoch;
     agreement_id.ERAID_FIELDS.agreementid = (uint16_t)ag_info->agreement_seq_num;
 
@@ -3050,7 +3068,7 @@ static int mca_coll_ftagree_era_prepare_agreement(ompi_communicator_t* comm,
     }
 #endif
 
-    OBJ_CONSTRUCT(&agreement_value, era_value_t);
+    OBJ_CONSTRUCT(&agreement_value, ompi_coll_ftagree_era_value_t);
 
     agreement_value.header.ret         = 0;
     agreement_value.header.operand     = op->o_f_to_c_index;
@@ -3075,7 +3093,7 @@ static int mca_coll_ftagree_era_prepare_agreement(ompi_communicator_t* comm,
                     agreement_id.ERAID_FIELDS.epoch,
                     agreement_id.ERAID_FIELDS.agreementid);
         assert(0 != agreement_id.ERAID_FIELDS.agreementid);
-        pa = (era_value_t*)value;
+        pa = (ompi_coll_ftagree_era_value_t*)value;
         opal_hash_table_remove_value_uint64(&era_passed_agreements, agreement_id.ERAID_KEY);
         OBJ_RELEASE(pa);
     }
@@ -3106,28 +3124,28 @@ static int mca_coll_ftagree_era_complete_agreement(era_identifier_t agreement_id
                                                              void *contrib,
                                                              ompi_group_t **group)
 {
-    era_value_t *av;
+    ompi_coll_ftagree_era_value_t *av;
     int ret;
     int i;
-    era_agreement_info_t *ci;
+    ompi_coll_ftagree_era_agreement_info_t *ci;
     ompi_communicator_t *comm;
     void *value;
 
     assert(0 != agreement_id.ERAID_FIELDS.agreementid);
     ci = era_lookup_agreement_info(agreement_id);
+    assert(NULL != ci);
+    comm = ci->comm;
 
     /** Now, it's time to remove that guy from the ongoing agreements */
     opal_hash_table_remove_value_uint64(&era_ongoing_agreements, agreement_id.ERAID_KEY);
 
-    comm = ci->comm;
-
     OBJ_RELEASE(ci); /* This will take care of the content of ci too */
 
     ret = opal_hash_table_get_value_uint64(&era_passed_agreements,
                                            agreement_id.ERAID_KEY,
                                            &value);
     assert( OPAL_SUCCESS == ret);
-    av = (era_value_t *)value;
+    av = (ompi_coll_ftagree_era_value_t *)value;
 
     memcpy(contrib, av->bytes, ERA_VALUE_BYTES_COUNT(&av->header));
     ret = av->header.ret;
@@ -3156,7 +3174,7 @@ static int mca_coll_ftagree_era_complete_agreement(era_identifier_t agreement_id
                          agreement_id.ERAID_FIELDS.epoch,
                          agreement_id.ERAID_FIELDS.agreementid,
                          ret,
-                         (0!=ERA_VALUE_BYTES_COUNT(&av->header))? *(int*)contrib: 0));
+                         (NULL != contrib)? *(int*)contrib: 0));
 
     return ret;
 }
@@ -3169,7 +3187,7 @@ static int mca_coll_ftagree_era_complete_agreement(era_identifier_t agreement_id
  * Returns:	- MPI_SUCCESS or an MPI error code
  */
 int mca_coll_ftagree_era_intra(void *contrib,
-                                         int dt_count,
+                                         size_t dt_count,
                                          ompi_datatype_t *dt,
                                          ompi_op_t *op,
                                          ompi_group_t **group, bool grp_update,
@@ -3196,7 +3214,7 @@ int mca_coll_ftagree_era_intra(void *contrib,
  * Returns:	- MPI_SUCCESS or an MPI error code
  */
 int mca_coll_ftagree_era_inter(void *contrib,
-                                         int dt_count,
+                                         size_t dt_count,
                                          ompi_datatype_t *dt,
                                          ompi_op_t *op,
                                          ompi_group_t **group, bool grp_update,
@@ -3226,13 +3244,29 @@ int mca_coll_ftagree_era_inter(void *contrib,
         contriblh[0] = ~0;
         contriblh[1] = *(int*)contrib;
     }
-    ompi_comm_set(&shadowcomm, comm,
-                  ompi_group_size(uniongrp), NULL, 0, NULL,
-                  NULL, comm->error_handler, NULL,
-                  uniongrp, NULL);
+
+    /* The 'shadowcomm' is used to perform the agreement on the union of the
+     * local and remote groups. We create a 'fake' new communicator that shares
+     * the cid/c_index with the original. This is possible because ERA does not
+     * use normal MPI messages, but only uses c_index to match agreements
+     * from its own BML callbacks.
+     */
+    ompi_comm_set(&shadowcomm,                     /* new comm */
+                  comm,                            /* old comm */
+                  ompi_group_size(uniongrp),       /* local_size */
+                  NULL,                            /* local_procs */
+                  0,                               /* remote_size */
+                  NULL,                            /* remote procs */
+                  NULL,                            /* attrs */
+                  comm->error_handler,             /* error handler */
+                  uniongrp,                        /* local group */
+                  NULL,                            /* remote group */
+                  0);                              /* flags */
+
     ompi_group_free(&uniongrp);
     shadowcomm->c_contextid = comm->c_contextid;
     shadowcomm->c_epoch = comm->c_epoch;
+    shadowcomm->c_index = comm->c_index;
     snprintf(shadowcomm->c_name, MPI_MAX_OBJECT_NAME, "SHADOW OF %s", &comm->c_name[0]);
     shadowcomm->any_source_offset = comm->any_source_offset;
     shadowcomm->agreement_specific = comm->agreement_specific;
@@ -3249,7 +3283,7 @@ int mca_coll_ftagree_era_inter(void *contrib,
 
 static int era_iagree_req_free(struct ompi_request_t** rptr)
 {
-    era_iagree_request_t *req = (era_iagree_request_t *)*rptr;
+    ompi_coll_ftagree_era_iagree_request_t *req = (ompi_coll_ftagree_era_iagree_request_t *)*rptr;
 
     if( NULL != req->ci )
         req->ci->req = NULL;
@@ -3263,7 +3297,7 @@ static int era_iagree_req_free(struct ompi_request_t** rptr)
 
 static int era_iagree_req_complete_cb(struct ompi_request_t* request)
 {
-    era_iagree_request_t *req = (era_iagree_request_t *)request;
+    ompi_coll_ftagree_era_iagree_request_t *req = (ompi_coll_ftagree_era_iagree_request_t *)request;
     int rc;
 
     assert( req->ci != NULL );
@@ -3276,7 +3310,7 @@ static int era_iagree_req_complete_cb(struct ompi_request_t* request)
 }
 
 int mca_coll_ftagree_iera_intra(void *contrib,
-                                          int dt_count,
+                                          size_t dt_count,
                                           ompi_datatype_t *dt,
                                           ompi_op_t *op,
                                           ompi_group_t **group, bool grp_update,
@@ -3285,13 +3319,13 @@ int mca_coll_ftagree_iera_intra(void *contrib,
                                           mca_coll_base_module_t *module)
 {
     opal_free_list_item_t* item;
-    era_iagree_request_t *req;
+    ompi_coll_ftagree_era_iagree_request_t *req;
     era_identifier_t agreement_id;
-    era_agreement_info_t *ci;
+    ompi_coll_ftagree_era_agreement_info_t *ci;
 
     item = opal_free_list_get(&era_iagree_requests);
     if( NULL == item ) return OMPI_ERR_OUT_OF_RESOURCE;
-    req = (era_iagree_request_t*)item;
+    req = (ompi_coll_ftagree_era_iagree_request_t*)item;
 
     OMPI_REQUEST_INIT(&req->super, false);
     assert(MPI_UNDEFINED == req->super.req_f_to_c_index);
@@ -3332,6 +3366,11 @@ int mca_coll_ftagree_iera_intra(void *contrib,
     return OMPI_SUCCESS;
 }
 
+#if 0
+// Per @bosilca and @jsquyres discussion 29 Apr 2021: there is
+// probably a memory leak in MPI_FINALIZE right now, because this
+// function does not appear to be being called from anywhere.
+// @bosilca's team is looking into it.
 int mca_coll_ftagree_era_free_comm(ompi_communicator_t* comm,
                                    mca_coll_base_module_t *module)
 {
@@ -3359,7 +3398,7 @@ int mca_coll_ftagree_era_free_comm(ompi_communicator_t* comm,
     } while(rc != MPI_SUCCESS);
     OBJ_RELEASE(acked);
 
-    aid.ERAID_FIELDS.contextid = comm->c_contextid;
+    aid.ERAID_FIELDS.contextid = comm->c_contextid.cid_sub.u64;
     aid.ERAID_FIELDS.epoch     = comm->c_epoch;
 
     opal_mutex_lock(&era_mutex);
@@ -3369,3 +3408,4 @@ int mca_coll_ftagree_era_free_comm(ompi_communicator_t* comm,
 
     return OMPI_SUCCESS;
 }
+#endif
diff --git a/ompi/mca/coll/ftagree/coll_ftagree_earlyterminating.c b/ompi/mca/coll/ftagree/coll_ftagree_earlyterminating.c
index eeee1195eaa..9ea18207a5f 100644
--- a/ompi/mca/coll/ftagree/coll_ftagree_earlyterminating.c
+++ b/ompi/mca/coll/ftagree/coll_ftagree_earlyterminating.c
@@ -62,7 +62,7 @@ typedef struct {
 
 int
 mca_coll_ftagree_eta_intra(void *contrib,
-                           int dt_count,
+                           size_t dt_count,
                            ompi_datatype_t *dt,
                            ompi_op_t *op,
                            ompi_group_t **group, bool update_grp,
@@ -101,7 +101,7 @@ mca_coll_ftagree_eta_intra(void *contrib,
     { /* ignore acked failures (add them later to the result) */
         ompi_group_t* ackedgrp = NULL; int npa; int *aranks, *cranks;
         ackedgrp = *group;
-        if( 0 != (npa = ompi_group_size(ackedgrp)) ) {
+        if( 0 != (npa = (NULL == ackedgrp? 0: ompi_group_size(ackedgrp))) ) {
             aranks = calloc( npa, sizeof(int) );
             for( i = 0; i < npa; i++ ) aranks[i] = i;
             cranks = calloc( npa, sizeof(int) );
@@ -336,7 +336,7 @@ mca_coll_ftagree_eta_intra(void *contrib,
     free(statuses);
     free(in);
     /* Let's build the group of failed processes */
-    if( NULL != group ) {
+    if( update_grp ) {
         int pos;
         /* We overwrite proc_status because it is not used anymore */
         int *failed = proc_status;
@@ -346,10 +346,10 @@ mca_coll_ftagree_eta_intra(void *contrib,
                 failed[pos++] = i;
             }
         }
-        if( update_grp ) {
+        if( NULL != *group ) {
             OBJ_RELEASE(*group);
-            ompi_group_incl(comm->c_remote_group, pos, failed, group);
         }
+        ompi_group_incl(comm->c_remote_group, pos, failed, group);
     }
     free(proc_status);
 
@@ -363,7 +363,7 @@ mca_coll_ftagree_eta_intra(void *contrib,
     OPAL_OUTPUT_VERBOSE((5, ompi_ftmpi_output_handle,
                          "%s ftagree:agreement (ETA) return %d with 4 first bytes of result 0x%08x and dead group with %d processes",
                          OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), ret, *(int*)contrib,
-                         (NULL == group) ? 0 : (*group)->grp_proc_count));
+                         (NULL == *group) ? 0 : (*group)->grp_proc_count));
     return ret;
 }
 
diff --git a/ompi/mca/coll/ftagree/coll_ftagree_module.c b/ompi/mca/coll/ftagree/coll_ftagree_module.c
index 6947139a7f5..755fca1b9bc 100644
--- a/ompi/mca/coll/ftagree/coll_ftagree_module.c
+++ b/ompi/mca/coll/ftagree/coll_ftagree_module.c
@@ -2,6 +2,7 @@
  * Copyright (c) 2012-2020 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -33,6 +34,30 @@ mca_coll_ftagree_init_query(bool enable_progress_threads,
     return OMPI_SUCCESS;
 }
 
+/*
+ * Init/Fini module on the communicator
+ */
+static int
+mca_coll_ftagree_module_enable(mca_coll_base_module_t *module,
+                             struct ompi_communicator_t *comm)
+{
+    MCA_COLL_INSTALL_API(comm, agree, module->coll_agree, module, "ftagree");
+    MCA_COLL_INSTALL_API(comm, iagree, module->coll_iagree, module, "ftagree");
+
+    /* All done */
+    return OMPI_SUCCESS;
+}
+
+static int
+mca_coll_ftagree_module_disable(mca_coll_base_module_t *module,
+                                struct ompi_communicator_t *comm)
+{
+    MCA_COLL_INSTALL_API(comm, agree, NULL, NULL, "ftagree");
+    MCA_COLL_INSTALL_API(comm, iagree, NULL, NULL, "ftagree");
+
+    /* All done */
+    return OMPI_SUCCESS;
+}
 
 /*
  * Invoked when there's a new communicator that has been created.
@@ -76,6 +101,7 @@ mca_coll_ftagree_comm_query(struct ompi_communicator_t *comm,
      * algorithms.
      */
     ftagree_module->super.coll_module_enable = mca_coll_ftagree_module_enable;
+    ftagree_module->super.coll_module_disable = mca_coll_ftagree_module_disable;
 
     /* This component does not provide any base collectives,
      * just the FT collectives.
@@ -112,15 +138,3 @@ mca_coll_ftagree_comm_query(struct ompi_communicator_t *comm,
     return &(ftagree_module->super);
 }
 
-
-/*
- * Init module on the communicator
- */
-int
-mca_coll_ftagree_module_enable(mca_coll_base_module_t *module,
-                             struct ompi_communicator_t *comm)
-{
-    /* All done */
-    return OMPI_SUCCESS;
-}
-
diff --git a/ompi/mca/coll/han/Makefile.am b/ompi/mca/coll/han/Makefile.am
index 78136d73090..978ef1805cb 100644
--- a/ompi/mca/coll/han/Makefile.am
+++ b/ompi/mca/coll/han/Makefile.am
@@ -2,6 +2,8 @@
 # Copyright (c) 2018-2020 The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
+# Copyright (c) 2024      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+# Copyright (c) 2022      BULL S.A.S. All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -12,18 +14,24 @@
 sources = \
 coll_han.h \
 coll_han_trigger.h \
+coll_han_algorithms.h \
+coll_han_alltoall.c \
+coll_han_alltoallv.c \
 coll_han_dynamic.h \
 coll_han_dynamic_file.h \
 coll_han_barrier.c \
 coll_han_bcast.c \
 coll_han_reduce.c \
 coll_han_scatter.c \
+coll_han_scatterv.c \
 coll_han_gather.c \
+coll_han_gatherv.c \
 coll_han_allreduce.c \
 coll_han_allgather.c \
 coll_han_component.c \
 coll_han_module.c \
 coll_han_trigger.c \
+coll_han_algorithms.c \
 coll_han_dynamic.c \
 coll_han_dynamic_file.c \
 coll_han_topo.c \
@@ -41,9 +49,6 @@ else
 component_noinst += libmca_coll_han.la
 endif
 
-# See ompi/mca/btl/sm/Makefile.am for an explanation of
-# libmca_common_sm.la.
-
 mcacomponentdir = $(ompilibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_coll_han_la_SOURCES = $(sources)
diff --git a/ompi/mca/coll/han/coll_han.h b/ompi/mca/coll/han/coll_han.h
index 025c2c32edc..c6bf543c7d9 100644
--- a/ompi/mca/coll/han/coll_han.h
+++ b/ompi/mca/coll/han/coll_han.h
@@ -2,7 +2,13 @@
  * Copyright (c) 2018-2020 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
- * Copyright (c) 2020      Bull S.A.S. All rights reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
+ * Copyright (c) 2020-2022 Bull S.A.S. All rights reserved.
+ * Copyright (c)           Amazon.com, Inc. or its affiliates.
+ *                         All rights reserved.
+ * Copyright (c) 2023      Computer Architecture and VLSI Systems (CARV)
+ *                         Laboratory, ICS Forth. All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -36,17 +42,19 @@
 #include "mpi.h"
 #include "ompi/mca/mca.h"
 #include "opal/util/output.h"
+#include "opal/mca/smsc/smsc.h"
 #include "ompi/mca/coll/base/coll_base_functions.h"
 #include "coll_han_trigger.h"
 #include "ompi/mca/coll/han/coll_han_dynamic.h"
+#include "coll_han_algorithms.h"
 
 /*
  * Today;
- * . only 2 modules available for intranode (low) level
+ * . 3 modules available for intranode (low) level
  * . only 2 modules available for internode (up) level
  */
 
-#define COLL_HAN_LOW_MODULES 2
+#define COLL_HAN_LOW_MODULES 3
 #define COLL_HAN_UP_MODULES 2
 
 struct mca_coll_han_bcast_args_s {
@@ -118,8 +126,8 @@ struct mca_coll_han_scatter_args_s {
     void *rbuf;
     ompi_datatype_t *sdtype;
     ompi_datatype_t *rdtype;
-    int scount;
-    int rcount;
+    size_t scount;
+    size_t rcount;
     int root;
     int root_up_rank;
     int root_low_rank;
@@ -159,8 +167,8 @@ struct mca_coll_han_allgather_s {
     void *rbuf;
     ompi_datatype_t *sdtype;
     ompi_datatype_t *rdtype;
-    int scount;
-    int rcount;
+    size_t scount;
+    size_t rcount;
     int root_low_rank;
     int w_rank;
     bool noop;
@@ -169,6 +177,31 @@ struct mca_coll_han_allgather_s {
 };
 typedef struct mca_coll_han_allgather_s mca_coll_han_allgather_t;
 
+typedef struct mca_coll_han_op_up_low_module_name_t {
+    char* han_op_up_module_name;
+    char* han_op_low_module_name;
+} mca_coll_han_op_up_low_module_name_t;
+
+/**
+ * The only reason we need to keep these around is because our MCA system does
+ * not support MCA variables that do not point to existing variables (aka. where
+ * mbv_storage does not exists until the completion of the application). Thus,
+ * we need to keep track of the storage for all variables, even the ones we
+ * only use to translated into a string.
+ */
+typedef struct mca_coll_han_op_module_name_t {
+    mca_coll_han_op_up_low_module_name_t bcast;
+    mca_coll_han_op_up_low_module_name_t reduce;
+    mca_coll_han_op_up_low_module_name_t allreduce;
+    mca_coll_han_op_up_low_module_name_t allgather;
+    mca_coll_han_op_up_low_module_name_t gather;
+    mca_coll_han_op_up_low_module_name_t gatherv;
+    mca_coll_han_op_up_low_module_name_t scatter;
+    mca_coll_han_op_up_low_module_name_t scatterv;
+    mca_coll_han_op_up_low_module_name_t alltoall;
+    mca_coll_han_op_up_low_module_name_t alltoallv;
+} mca_coll_han_op_module_name_t;
+
 /**
  * Structure to hold the han coll component.  First it holds the
  * base coll component, and then holds a bunch of
@@ -177,12 +210,13 @@ typedef struct mca_coll_han_allgather_s mca_coll_han_allgather_t;
  */
 typedef struct mca_coll_han_component_t {
     /** Base coll component */
-    mca_coll_base_component_2_4_0_t super;
+    mca_coll_base_component_3_0_0_t super;
 
     /** MCA parameter: Priority of this component */
     int han_priority;
     /* whether output the log message */
     int han_output;
+    int han_output_verbose; /* activation level of coll han verbosity */
     /* segment size for bcast */
     uint32_t han_bcast_segsize;
     /* up level module for bcast */
@@ -209,15 +243,39 @@ typedef struct mca_coll_han_component_t {
     uint32_t han_gather_up_module;
     /* low level module for gather */
     uint32_t han_gather_low_module;
+    /* up level module for gatherv */
+    uint32_t han_gatherv_up_module;
+    /* low level module for gatherv */
+    uint32_t han_gatherv_low_module;
     /* up level module for scatter */
     uint32_t han_scatter_up_module;
     /* low level module for scatter */
     uint32_t han_scatter_low_module;
+    /* up level module for scatterv */
+    uint32_t han_scatterv_up_module;
+    /* low level module for scatterv */
+    uint32_t han_scatterv_low_module;
+
+    /* low level module for alltoall */
+    uint32_t han_alltoall_low_module;
+    /* alltoall: parallel stages */
+    int32_t han_alltoall_pstages;
+
+    /* low level module for alltoallv */
+    uint32_t han_alltoallv_low_module;
+    int64_t han_alltoallv_smsc_avg_send_limit;
+    double han_alltoallv_smsc_noncontig_activation_limit;
+
+
+    /* name of the modules */
+    mca_coll_han_op_module_name_t han_op_module_name;
     /* whether we need reproducible results
      * (but disables topological optimisations)
      */
     bool han_reproducible;
     bool use_simple_algorithm[COLLCOUNT];
+    int use_algorithm[COLLCOUNT];
+    int use_algorithm_param[COLLCOUNT]; // MCA parmeter id for algo, to know if user provided
 
     /* Dynamic configuration rules */
     bool use_dynamic_file_rules;
@@ -226,28 +284,41 @@ typedef struct mca_coll_han_component_t {
     /* Dynamic rules from file */
     mca_coll_han_dynamic_rules_t dynamic_rules;
     /* Dynamic rules from mca parameter */
-    COMPONENT_T mca_rules[COLLCOUNT][NB_TOPO_LVL];
+    COMPONENT_T mca_sub_components[COLLCOUNT][NB_TOPO_LVL];
+
+    int num_available_algorithms[COLLCOUNT]; // not counting "default" behaviour
+    /* to show algorithms in ompi_info */
+    mca_base_var_enum_value_t* algorithm_enumerator[COLLCOUNT];
 
     /* Define maximum dynamic errors printed by rank 0 with a 0 verbosity level */
     int max_dynamic_errors;
-} mca_coll_han_component_t;
 
+    opal_free_list_t pack_buffers;
+    int64_t han_packbuf_max_count;
+    int64_t han_packbuf_bytes;
+} mca_coll_han_component_t;
 
 /*
  * Structure used to store what is necessary for the collective operations
  * routines in case of fallback.
  */
-typedef struct mca_coll_han_single_collective_fallback_s {
-    union {
+typedef struct mca_coll_han_single_collective_fallback_s
+{
+    union
+    {
+        mca_coll_base_module_alltoall_fn_t alltoall;
+        mca_coll_base_module_alltoallv_fn_t alltoallv;
         mca_coll_base_module_allgather_fn_t allgather;
         mca_coll_base_module_allgatherv_fn_t allgatherv;
         mca_coll_base_module_allreduce_fn_t allreduce;
         mca_coll_base_module_barrier_fn_t barrier;
         mca_coll_base_module_bcast_fn_t bcast;
         mca_coll_base_module_gather_fn_t gather;
+        mca_coll_base_module_gatherv_fn_t gatherv;
         mca_coll_base_module_reduce_fn_t reduce;
         mca_coll_base_module_scatter_fn_t scatter;
-    } module_fn;
+        mca_coll_base_module_scatterv_fn_t scatterv;
+    };
     mca_coll_base_module_t* module;
 } mca_coll_han_single_collective_fallback_t;
 
@@ -256,7 +327,10 @@ typedef struct mca_coll_han_single_collective_fallback_s {
  * by HAN. This structure is used as a fallback during subcommunicator
  * creation.
  */
-typedef struct mca_coll_han_collectives_fallback_s {
+typedef struct mca_coll_han_collectives_fallback_s
+{
+    mca_coll_han_single_collective_fallback_t alltoall;
+    mca_coll_han_single_collective_fallback_t alltoallv;
     mca_coll_han_single_collective_fallback_t allgather;
     mca_coll_han_single_collective_fallback_t allgatherv;
     mca_coll_han_single_collective_fallback_t allreduce;
@@ -264,7 +338,9 @@ typedef struct mca_coll_han_collectives_fallback_s {
     mca_coll_han_single_collective_fallback_t bcast;
     mca_coll_han_single_collective_fallback_t reduce;
     mca_coll_han_single_collective_fallback_t gather;
+    mca_coll_han_single_collective_fallback_t gatherv;
     mca_coll_han_single_collective_fallback_t scatter;
+    mca_coll_han_single_collective_fallback_t scatterv;
 } mca_coll_han_collectives_fallback_t;
 
 /** Coll han module */
@@ -283,9 +359,10 @@ typedef struct mca_coll_han_module_t {
     int *cached_topo;
     bool is_mapbycore;
     bool are_ppn_imbalanced;
+    bool is_heterogeneous;
 
     /* To be able to fallback when the cases are not supported */
-    struct mca_coll_han_collectives_fallback_s fallback;
+    mca_coll_han_collectives_fallback_t fallback;
 
     /* To be able to fallback on reproducible algorithm */
     mca_coll_base_module_reduce_fn_t reproducible_reduce;
@@ -316,54 +393,69 @@ OBJ_CLASS_DECLARATION(mca_coll_han_module_t);
  * Some defines to stick to the naming used in the other components in terms of
  * fallback routines
  */
-#define previous_allgather          fallback.allgather.module_fn.allgather
+#define previous_alltoall           fallback.alltoall.alltoall
+#define previous_alltoall_module    fallback.alltoall.module
+
+#define previous_alltoallv           fallback.alltoallv.alltoallv
+#define previous_alltoallv_module    fallback.alltoallv.module
+
+#define previous_allgather          fallback.allgather.allgather
 #define previous_allgather_module   fallback.allgather.module
 
-#define previous_allgatherv         fallback.allgatherv.module_fn.allgatherv
+#define previous_allgatherv         fallback.allgatherv.allgatherv
 #define previous_allgatherv_module  fallback.allgatherv.module
 
-#define previous_allreduce          fallback.allreduce.module_fn.allreduce
+#define previous_allreduce          fallback.allreduce.allreduce
 #define previous_allreduce_module   fallback.allreduce.module
 
-#define previous_barrier            fallback.barrier.module_fn.barrier
+#define previous_barrier            fallback.barrier.barrier
 #define previous_barrier_module     fallback.barrier.module
 
-#define previous_bcast              fallback.bcast.module_fn.bcast
+#define previous_bcast              fallback.bcast.bcast
 #define previous_bcast_module       fallback.bcast.module
 
-#define previous_reduce             fallback.reduce.module_fn.reduce
+#define previous_reduce             fallback.reduce.reduce
 #define previous_reduce_module      fallback.reduce.module
 
-#define previous_gather             fallback.gather.module_fn.gather
+#define previous_gather             fallback.gather.gather
 #define previous_gather_module      fallback.gather.module
 
-#define previous_scatter            fallback.scatter.module_fn.scatter
+#define previous_gatherv            fallback.gatherv.gatherv
+#define previous_gatherv_module     fallback.gatherv.module
+
+#define previous_scatter            fallback.scatter.scatter
 #define previous_scatter_module     fallback.scatter.module
 
+#define previous_scatterv           fallback.scatterv.scatterv
+#define previous_scatterv_module    fallback.scatterv.module
 
 /* macro to correctly load a fallback collective module */
-#define HAN_LOAD_FALLBACK_COLLECTIVE(HANM, COMM, COLL)                            \
-    do {                                                                          \
-        if ( ((COMM)->c_coll->coll_ ## COLL ## _module) == (mca_coll_base_module_t*)(HANM) ) { \
-            (COMM)->c_coll->coll_ ## COLL = (HANM)->fallback.COLL.module_fn.COLL;               \
-            mca_coll_base_module_t *coll_module = (COMM)->c_coll->coll_ ## COLL ## _module; \
-            (COMM)->c_coll->coll_ ## COLL ## _module = (HANM)->fallback.COLL.module;  \
-            OBJ_RETAIN((COMM)->c_coll->coll_ ## COLL ## _module);                     \
-            OBJ_RELEASE(coll_module);                                                 \
-        }                                                                             \
-    } while(0)
-
-/* macro to correctly load /all/ fallback collectives */
-#define HAN_LOAD_FALLBACK_COLLECTIVES(HANM, COMM)                            \
-    do {                                                                     \
-        HAN_LOAD_FALLBACK_COLLECTIVE(HANM, COMM, barrier);                   \
-        HAN_LOAD_FALLBACK_COLLECTIVE(HANM, COMM, bcast);                     \
-        HAN_LOAD_FALLBACK_COLLECTIVE(HANM, COMM, scatter);                   \
-        HAN_LOAD_FALLBACK_COLLECTIVE(HANM, COMM, gather);                    \
-        HAN_LOAD_FALLBACK_COLLECTIVE(HANM, COMM, reduce);                    \
-        HAN_LOAD_FALLBACK_COLLECTIVE(HANM, COMM, allreduce);                 \
-        HAN_LOAD_FALLBACK_COLLECTIVE(HANM, COMM, allgather);                 \
-        HAN_LOAD_FALLBACK_COLLECTIVE(HANM, COMM, allgatherv);                \
+#define HAN_UNINSTALL_COLL_API(__comm, __module, __api)                                  \
+    do                                                                                   \
+    {                                                                                    \
+        if (__comm->c_coll->coll_##__api##_module == &__module->super)                   \
+        {                                                                                \
+            MCA_COLL_INSTALL_API(__comm, __api, __module->previous_##__api,              \
+                                 __module->previous_##__api##_module, "han");            \
+            /* Do not reset the fallback to NULL it will be needed */                    \
+        }                                                                                \
+    } while (0)
+
+ /* macro to correctly load /all/ fallback collectives */
+#define HAN_LOAD_FALLBACK_COLLECTIVES(COMM, HANM)                      \
+    do {                                                               \
+        HAN_UNINSTALL_COLL_API(COMM, HANM, barrier);                   \
+        HAN_UNINSTALL_COLL_API(COMM, HANM, bcast);                     \
+        HAN_UNINSTALL_COLL_API(COMM, HANM, scatter);                   \
+        HAN_UNINSTALL_COLL_API(COMM, HANM, scatterv);                  \
+        HAN_UNINSTALL_COLL_API(COMM, HANM, gather);                    \
+        HAN_UNINSTALL_COLL_API(COMM, HANM, gatherv);                   \
+        HAN_UNINSTALL_COLL_API(COMM, HANM, reduce);                    \
+        HAN_UNINSTALL_COLL_API(COMM, HANM, allreduce);                 \
+        HAN_UNINSTALL_COLL_API(COMM, HANM, allgather);                 \
+        HAN_UNINSTALL_COLL_API(COMM, HANM, allgatherv);                \
+        HAN_UNINSTALL_COLL_API(COMM, HANM, alltoall);                  \
+        HAN_UNINSTALL_COLL_API(COMM, HANM, alltoallv);                 \
         han_module->enabled = false;  /* entire module set to pass-through from now on */ \
     } while(0)
 
@@ -371,7 +463,7 @@ OBJ_CLASS_DECLARATION(mca_coll_han_module_t);
 /**
  * Global component instance
  */
-OMPI_MODULE_DECLSPEC extern mca_coll_han_component_t mca_coll_han_component;
+OMPI_DECLSPEC extern mca_coll_han_component_t mca_coll_han_component;
 
 /*
  * coll module functions
@@ -380,7 +472,7 @@ int mca_coll_han_init_query(bool enable_progress_threads, bool enable_mpi_thread
 
 mca_coll_base_module_t *mca_coll_han_comm_query(struct ompi_communicator_t *comm, int *priority);
 
-int han_request_free(ompi_request_t ** request);
+int ompi_coll_han_request_free(ompi_request_t ** request);
 
 /* Subcommunicator creation */
 int mca_coll_han_comm_create(struct ompi_communicator_t *comm, mca_coll_han_module_t * han_module);
@@ -400,11 +492,16 @@ int *mca_coll_han_topo_init(struct ompi_communicator_t *comm, mca_coll_han_modul
 
 /* Utils */
 static inline void
-mca_coll_han_get_ranks(int *vranks, int root, int low_size,
-                       int *root_low_rank, int *root_up_rank)
+mca_coll_han_get_ranks(int *vranks, int w_rank, int low_size,
+                       int *low_rank, int *up_rank)
 {
-    *root_up_rank = vranks[root] / low_size;
-    *root_low_rank = vranks[root] % low_size;
+    if (up_rank) {
+        *up_rank = vranks[w_rank] / low_size;
+    }
+
+    if (low_rank) {
+        *low_rank = vranks[w_rank] % low_size;
+    }
 }
 
 const char* mca_coll_han_topo_lvl_to_str(TOPO_LVL_T topo_lvl);
@@ -419,6 +516,12 @@ mca_coll_han_get_all_coll_modules(struct ompi_communicator_t *comm,
                                   mca_coll_han_module_t *han_module);
 
 int
+mca_coll_han_alltoall_intra_dynamic(ALLTOALL_BASE_ARGS,
+                                    mca_coll_base_module_t *module);
+int
+mca_coll_han_alltoallv_intra_dynamic(ALLTOALLV_BASE_ARGS,
+                                    mca_coll_base_module_t *module);
+int
 mca_coll_han_allgather_intra_dynamic(ALLGATHER_BASE_ARGS,
                                      mca_coll_base_module_t *module);
 int
@@ -437,140 +540,42 @@ int
 mca_coll_han_gather_intra_dynamic(GATHER_BASE_ARGS,
                                   mca_coll_base_module_t *module);
 int
+mca_coll_han_gatherv_intra_dynamic(GATHERV_BASE_ARGS,
+                                   mca_coll_base_module_t *module);
+int
 mca_coll_han_reduce_intra_dynamic(REDUCE_BASE_ARGS,
                                   mca_coll_base_module_t *module);
 int
 mca_coll_han_scatter_intra_dynamic(SCATTER_BASE_ARGS,
                                    mca_coll_base_module_t *module);
-
-int mca_coll_han_barrier_intra_simple(struct ompi_communicator_t *comm,
-                                      mca_coll_base_module_t *module);
-/* Bcast */
-int mca_coll_han_bcast_intra_simple(void *buff,
-                                    int count,
-                                    struct ompi_datatype_t *dtype,
-                                    int root,
-                                    struct ompi_communicator_t *comm,
-                                    mca_coll_base_module_t *module);
-int mca_coll_han_bcast_intra(void *buff, int count, struct ompi_datatype_t *dtype, int root,
-                             struct ompi_communicator_t *comm, mca_coll_base_module_t * module);
-
-/* Reduce */
-int
-mca_coll_han_reduce_intra_simple(const void *sbuf,
-                                 void* rbuf,
-                                 int count,
-                                 struct ompi_datatype_t *dtype,
-                                 ompi_op_t *op,
-                                 int root,
-                                 struct ompi_communicator_t *comm,
-                                 mca_coll_base_module_t *module);
-int
-mca_coll_han_reduce_reproducible_decision(struct ompi_communicator_t *comm,
-                                          mca_coll_base_module_t *module);
-int
-mca_coll_han_reduce_reproducible(const void *sbuf,
-                                 void *rbuf,
-                                 int count,
-                                 struct ompi_datatype_t *dtype,
-                                 struct ompi_op_t *op,
-                                 int root,
-                                 struct ompi_communicator_t *comm,
-                                 mca_coll_base_module_t *module);
-
-int mca_coll_han_reduce_intra(const void *sbuf,
-                              void *rbuf,
-                              int count,
-                              struct ompi_datatype_t *dtype,
-                              ompi_op_t* op,
-                              int root,
-                              struct ompi_communicator_t *comm,
-                              mca_coll_base_module_t * module);
-
-/* Allreduce */
-int
-mca_coll_han_allreduce_intra_simple(const void *sbuf,
-                                    void *rbuf,
-                                    int count,
-                                    struct ompi_datatype_t *dtype,
-                                    struct ompi_op_t *op,
-                                    struct ompi_communicator_t *comm,
-                                    mca_coll_base_module_t *module);
-int
-mca_coll_han_allreduce_reproducible_decision(struct ompi_communicator_t *comm,
-                                             mca_coll_base_module_t *module);
 int
-mca_coll_han_allreduce_reproducible(const void *sbuf,
-                                    void *rbuf,
-                                    int count,
-                                    struct ompi_datatype_t *dtype,
-                                    struct ompi_op_t *op,
-                                    struct ompi_communicator_t *comm,
+mca_coll_han_scatterv_intra_dynamic(SCATTERV_BASE_ARGS,
                                     mca_coll_base_module_t *module);
 
-int mca_coll_han_allreduce_intra(const void *sbuf,
-                                 void *rbuf,
-                                 int count,
-                                 struct ompi_datatype_t *dtype,
-                                 struct ompi_op_t *op,
-                                 struct ompi_communicator_t *comm, mca_coll_base_module_t * module);
+int mca_coll_han_barrier_intra_simple(struct ompi_communicator_t *comm,
+                                      mca_coll_base_module_t *module);
 
-/* Scatter */
-int
-mca_coll_han_scatter_intra(const void *sbuf, int scount,
-                           struct ompi_datatype_t *sdtype,
-                           void *rbuf, int rcount,
-                           struct ompi_datatype_t *rdtype,
-                           int root,
-                           struct ompi_communicator_t *comm, mca_coll_base_module_t * module);
-int
-mca_coll_han_scatter_intra_simple(const void *sbuf, int scount,
-                                  struct ompi_datatype_t *sdtype,
-                                  void *rbuf, int rcount,
-                                  struct ompi_datatype_t *rdtype,
-                                  int root,
-                                  struct ompi_communicator_t *comm,
-                                  mca_coll_base_module_t * module);
-
-/* Gather */
-int
-mca_coll_han_gather_intra(const void *sbuf, int scount,
-                          struct ompi_datatype_t *sdtype,
-                          void *rbuf, int rcount,
-                          struct ompi_datatype_t *rdtype,
-                          int root,
-                          struct ompi_communicator_t *comm, mca_coll_base_module_t * module);
-int
-mca_coll_han_gather_intra_simple(const void *sbuf, int scount,
-                                 struct ompi_datatype_t *sdtype,
-                                 void *rbuf, int rcount,
-                                 struct ompi_datatype_t *rdtype,
-                                 int root,
-                                 struct ompi_communicator_t *comm,
-                                 mca_coll_base_module_t *module);
 /* reordering after gather, for unordered ranks */
 void
 ompi_coll_han_reorder_gather(const void *sbuf,
-                             void *rbuf, int rcount,
+                             void *rbuf, size_t rcount,
                              struct ompi_datatype_t *rdtype,
                              struct ompi_communicator_t *comm,
                              int * topo);
 
-
-
-/* Allgather */
-int
-mca_coll_han_allgather_intra(const void *sbuf, int scount,
-                             struct ompi_datatype_t *sdtype,
-                             void *rbuf, int rcount,
-                             struct ompi_datatype_t *rdtype,
-                             struct ompi_communicator_t *comm, mca_coll_base_module_t * module);
-int
-mca_coll_han_allgather_intra_simple(const void *sbuf, int scount,
-                                    struct ompi_datatype_t *sdtype,
-                                    void* rbuf, int rcount,
-                                    struct ompi_datatype_t *rdtype,
-                                    struct ompi_communicator_t *comm,
-                                    mca_coll_base_module_t *module);
+static inline struct mca_smsc_endpoint_t *mca_coll_han_get_smsc_endpoint (struct ompi_proc_t *proc) {
+    extern opal_mutex_t mca_coll_han_lock;
+    if (NULL == proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_SMSC]) {
+        if (NULL == proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_SMSC]) {
+            OPAL_THREAD_LOCK(&mca_coll_han_lock);
+            if (NULL == proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_SMSC]) {
+                proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_SMSC] = mca_smsc->get_endpoint(&proc->super);
+            }
+            OPAL_THREAD_UNLOCK(&mca_coll_han_lock);
+        }
+    }
+
+    return (struct mca_smsc_endpoint_t *) proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_SMSC];
+}
 
 #endif                          /* MCA_COLL_HAN_EXPORT_H */
diff --git a/ompi/mca/coll/han/coll_han_algorithms.c b/ompi/mca/coll/han/coll_han_algorithms.c
new file mode 100644
index 00000000000..1c821a3abfe
--- /dev/null
+++ b/ompi/mca/coll/han/coll_han_algorithms.c
@@ -0,0 +1,244 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2020-2022 Bull S.A.S. All rights reserved.
+ * Copyright (c) 2024      Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/*
+ * @file
+ * This files contains helps for algorithm selection
+ *
+ */
+
+#include "coll_han.h"
+#include "coll_han_algorithms.h"
+
+/* default algo names, formatted for MCA var_enum_create */
+mca_base_var_enum_value_t han_default_algorithms_enum[] = {
+    { 0, "default" }, // algo #0 is called "defaut"
+    { 0, NULL }
+};
+
+/**
+ * all available algorithms must be added here to be selectable in
+ * the component.
+ *
+ * note: algorithm 'default' / #0 is automatically added to each
+ * implemented collective, it is the default behaviour in
+ * coll_han_dynamic.c
+ */
+mca_coll_han_algorithm_value_t*  mca_coll_han_available_algorithms[COLLCOUNT] = {
+    [BARRIER] = (mca_coll_han_algorithm_value_t[]) {
+        {"simple", (fnptr_t) &mca_coll_han_barrier_intra_simple}, // 2-level
+        { 0 }
+    },
+    [BCAST] = (mca_coll_han_algorithm_value_t[]){
+        {"intra", (fnptr_t) &mca_coll_han_bcast_intra}, // 2-level
+        {"simple", (fnptr_t) &mca_coll_han_bcast_intra_simple}, // 2-level
+        { 0 }
+    },
+    [REDUCE] = (mca_coll_han_algorithm_value_t[]){
+        {"intra", (fnptr_t) &mca_coll_han_reduce_intra}, // 2-level
+        {"simple", (fnptr_t) &mca_coll_han_reduce_intra_simple}, // 2-level
+        {"reproducible", (fnptr_t) &mca_coll_han_reduce_reproducible}, // fallback
+        { 0 }
+    },
+    [ALLREDUCE] = (mca_coll_han_algorithm_value_t[]){
+        {"intra", (fnptr_t) &mca_coll_han_allreduce_intra}, // 2-level
+        {"simple", (fnptr_t) &mca_coll_han_allreduce_intra_simple}, // 2-level
+        {"reproducible", (fnptr_t) &mca_coll_han_allreduce_reproducible}, // fallback
+        { 0 }
+    },
+    [SCATTER] = (mca_coll_han_algorithm_value_t[]){
+        {"intra", (fnptr_t) &mca_coll_han_scatter_intra}, // 2-level
+        {"simple", (fnptr_t) &mca_coll_han_scatter_intra_simple}, // 2-level
+        { 0 }
+    },
+    [SCATTERV] = (mca_coll_han_algorithm_value_t[]){
+        {"intra", (fnptr_t) &mca_coll_han_scatterv_intra}, // 2-level
+        { 0 }
+    },
+    [GATHER] = (mca_coll_han_algorithm_value_t[]){
+        {"intra", (fnptr_t) &mca_coll_han_gather_intra}, // 2-level
+        {"simple", (fnptr_t) &mca_coll_han_gather_intra_simple}, // 2-level
+        { 0 }
+    },
+    [GATHERV] = (mca_coll_han_algorithm_value_t[]){
+        {"intra", (fnptr_t) &mca_coll_han_gatherv_intra}, // 2-level
+        { 0 }
+    },
+    [ALLGATHER] = (mca_coll_han_algorithm_value_t[]){
+        {"intra", (fnptr_t)&mca_coll_han_allgather_intra}, // 2-level
+        {"simple", (fnptr_t)&mca_coll_han_allgather_intra_simple}, // 2-level
+        { 0 }
+    },
+    [ALLTOALL] = (mca_coll_han_algorithm_value_t[]){
+        {"smsc", (fnptr_t)&mca_coll_han_alltoall_using_smsc}, // 2-level
+        { 0 }
+    },
+    [ALLTOALLV] = (mca_coll_han_algorithm_value_t[]){
+        {"smsc", (fnptr_t)&mca_coll_han_alltoallv_using_smsc}, // 2-level
+        { 0 }
+    },
+};
+
+int
+mca_coll_han_algorithm_id_is_valid(int coll_id, int algorithm_id)
+{
+    if (!mca_coll_han_is_coll_dynamic_implemented(coll_id)) {
+        return false;
+    }
+    if (0 > algorithm_id) {
+        return false;
+    }
+
+    /*
+     * user provided algorithms + 'default' algorithm #0
+     */
+    return algorithm_id < mca_coll_han_component.num_available_algorithms[coll_id] + 1;
+}
+
+fnptr_t
+mca_coll_han_algorithm_id_to_fn(int coll_id, int algorithm_id)
+{
+    if (algorithm_id == 0 || !mca_coll_han_algorithm_id_is_valid(coll_id, algorithm_id)) {
+        return NULL;
+    }
+    /* algorithm 0 is not included here */
+    return mca_coll_han_available_algorithms[coll_id][algorithm_id - 1].fn;
+}
+
+char*
+mca_coll_han_algorithm_id_to_name(int coll_id, int algorithm_id)
+{
+    if (!mca_coll_han_algorithm_id_is_valid(coll_id, algorithm_id)) {
+        return NULL;
+    }
+    if (0 == algorithm_id) {
+        return "default";
+    }
+    /* algorithm 0 is not included here */
+    return mca_coll_han_available_algorithms[coll_id][algorithm_id - 1].name;
+}
+
+int
+mca_coll_han_algorithm_name_to_id(COLLTYPE_T coll_id, const char* algorithm_name)
+{
+    // shortcut for default
+    if (0 == strcmp(algorithm_name, "default")) {
+        return 0;
+    }
+    if (0 > mca_coll_han_component.num_available_algorithms[coll_id]) {
+        return -1;
+    }
+
+    const mca_base_var_enum_value_t* algorithms_values
+                = mca_coll_han_component.algorithm_enumerator[coll_id];
+    for (int i = 0; algorithms_values[i].string != NULL; i++) {
+        if (0 == strcmp(algorithm_name, algorithms_values[i].string))
+            return i;
+    }
+    return -1;
+}
+
+/**
+ * count algorithms for this collective (other than default)
+ */
+static int
+mca_han_algorithm_count(mca_coll_han_algorithm_value_t* algorithm_values)
+{
+    int n = 0;
+    if (NULL != algorithm_values) {
+        while (algorithm_values[n].name != NULL) {
+            n++;
+        }
+    }
+    return n;
+}
+
+/**
+ * Initializes algorithm_enumerator, used to show algorithm id/name in ompi_info
+ */
+static int
+mca_han_algorithm_enumerator_create(mca_base_var_enum_value_t** algorithm_enumerator,
+                                    mca_coll_han_algorithm_value_t* algorithm_values)
+{
+    *algorithm_enumerator = NULL;
+
+    int n_algorithm_values = mca_han_algorithm_count(algorithm_values);
+    if (0 == n_algorithm_values) {
+        return OMPI_SUCCESS;
+    }
+
+    // n_algorithm_values+2 because of 'default' and termination by 0
+    mca_base_var_enum_value_t* enum_values =
+        malloc((n_algorithm_values + 2) * sizeof(mca_base_var_enum_value_t));
+    if (NULL == enum_values) {
+        goto alloc_error;
+    }
+
+    // always add "default" algorithm
+    enum_values[0].value = 0;
+    enum_values[0].string = "default";
+    // fill data for other algorithms
+    for (int i = 0; i < n_algorithm_values; i++) {
+        enum_values[i + 1].value = i + 1;
+        enum_values[i + 1].string = algorithm_values[i].name;
+    }
+    // last value of enum must be zeroed
+    enum_values[n_algorithm_values + 1] = (mca_base_var_enum_value_t){ 0 };
+    *algorithm_enumerator = enum_values;
+    return OMPI_SUCCESS;
+
+alloc_error:
+    opal_output(0, "coll/han failed to initialize available algorithms "
+                   "(allocation error)");
+    return OMPI_ERROR;
+}
+
+
+/**
+ * initializes componnent algorithm_info
+ */
+int
+mca_coll_han_init_algorithms(void)
+{
+    memset(mca_coll_han_component.num_available_algorithms, 0,
+           COLLCOUNT * sizeof(int));
+    memset(mca_coll_han_component.algorithm_enumerator, 0,
+           COLLCOUNT * sizeof(mca_base_var_enum_value_t*));
+    for (int coll=0; coll<COLLCOUNT; coll++) {
+        mca_coll_han_component.num_available_algorithms[coll] =
+            mca_han_algorithm_count(mca_coll_han_available_algorithms[coll]);
+        if (0 == mca_coll_han_component.num_available_algorithms[coll]
+            || !mca_coll_han_is_coll_dynamic_implemented(coll)) {
+            continue;
+        }
+        if (OMPI_SUCCESS !=
+            mca_han_algorithm_enumerator_create(&mca_coll_han_component.algorithm_enumerator[coll],
+                                                mca_coll_han_available_algorithms[coll])) {
+            goto init_failed;
+        }
+    }
+    return OMPI_SUCCESS;
+ init_failed:
+    mca_coll_han_free_algorithms();
+    return OMPI_ERROR;
+}
+
+int
+mca_coll_han_free_algorithms(void)
+{
+    for (int coll_id = 0; coll_id < COLLCOUNT; coll_id++) {
+        if (NULL != mca_coll_han_component.algorithm_enumerator[coll_id]) {
+            free(mca_coll_han_component.algorithm_enumerator[coll_id]);
+        }
+    }
+    return OPAL_SUCCESS;
+}
diff --git a/ompi/mca/coll/han/coll_han_algorithms.h b/ompi/mca/coll/han/coll_han_algorithms.h
new file mode 100644
index 00000000000..af486669f60
--- /dev/null
+++ b/ompi/mca/coll/han/coll_han_algorithms.h
@@ -0,0 +1,223 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2020-2022 Bull S.A.S. All rights reserved.
+ * Copyright (c) 2024      Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/** Han algorithms, for han algorithm selection. This contains the
+ * helper functions for the selection mechanism and declarations of
+ * collective algorithm.
+ *
+ * All algorithms need to be added to the
+ * 'mca_coll_han_available_algorithms' array in in
+ * coll_han_algorithms.c. After mca_han_algorithm_info_init() various info on
+ * id/name/functions are filled in component.algorithm_info[], 'default'
+ * algorithm with #0 is added, and they are selectable in the
+ * component.
+ *
+ * Algorithms which are selectable can be queried in ompi_info, see for
+ * 'coll_han_use_<COLLNAME>_algorithm:enumerator'
+ *
+ */
+
+#ifndef MCA_COLL_HAN_ALGORITHMS_H
+#define MCA_COLL_HAN_ALGORITHMS_H
+
+/* use this pointer type instead of void* to avoid warnings as it is
+ * not legal to convert function pointers to void*
+ *
+ * note: alternatively we could use a union of function types, but
+ * then it is heavy to declare available algorithms as nested
+ * datastructure
+ */
+typedef void (*fnptr_t)(void);
+// Han algorithms, data declarations per collective
+// structure used to declare an array of algorithms: {name, fn}
+typedef struct mca_coll_han_algorithm_value_s {
+    char* name;
+    fnptr_t fn;
+} mca_coll_han_algorithm_value_t;
+
+// datastructure generated from previous by mca_han_init_algorithm_info()
+typedef struct mca_coll_han_collective_algorithm_info_s {
+    mca_base_var_enum_value_t* enum_values;
+} mca_coll_han_collective_algorithm_info_t;
+
+// initialise before using algorithms id name fn var_enum_value
+int
+mca_coll_han_init_algorithms(void);
+int
+mca_coll_han_free_algorithms(void);
+
+int
+mca_coll_han_algorithm_name_to_id(COLLTYPE_T coll_id, const char* algorithm_name);
+int
+mca_coll_han_algorithm_id_is_valid(int coll_id, int algorithm_id);
+fnptr_t
+mca_coll_han_algorithm_id_to_fn(int coll_id, int algorithm_id);
+char*
+mca_coll_han_algorithm_id_to_name(int coll_id, int algorithm_id);
+
+/**
+ * Available han algorithms
+ *
+ * They must be added to 'mca_coll_han_available_algorithms'
+ * in coll_han_algorithms.c
+ */
+
+int mca_coll_han_barrier_intra_simple(struct ompi_communicator_t *comm,
+                                      mca_coll_base_module_t *module);
+/* Bcast */
+int mca_coll_han_bcast_intra_simple(void *buff,
+                                    size_t count,
+                                    struct ompi_datatype_t *dtype,
+                                    int root,
+                                    struct ompi_communicator_t *comm,
+                                    mca_coll_base_module_t *module);
+int mca_coll_han_bcast_intra(void *buff, size_t count, struct ompi_datatype_t *dtype, int root,
+                             struct ompi_communicator_t *comm, mca_coll_base_module_t * module);
+
+/* Reduce */
+int
+mca_coll_han_reduce_intra_simple(const void *sbuf,
+                                 void* rbuf,
+                                 size_t count,
+                                 struct ompi_datatype_t *dtype,
+                                 ompi_op_t *op,
+                                 int root,
+                                 struct ompi_communicator_t *comm,
+                                 mca_coll_base_module_t *module);
+int
+mca_coll_han_reduce_reproducible_decision(struct ompi_communicator_t *comm,
+                                          mca_coll_base_module_t *module);
+int
+mca_coll_han_reduce_reproducible(const void *sbuf,
+                                 void *rbuf,
+                                 size_t count,
+                                 struct ompi_datatype_t *dtype,
+                                 struct ompi_op_t *op,
+                                 int root,
+                                 struct ompi_communicator_t *comm,
+                                 mca_coll_base_module_t *module);
+
+int mca_coll_han_reduce_intra(const void *sbuf,
+                              void *rbuf,
+                              size_t count,
+                              struct ompi_datatype_t *dtype,
+                              ompi_op_t* op,
+                              int root,
+                              struct ompi_communicator_t *comm,
+                              mca_coll_base_module_t * module);
+
+/* Allreduce */
+int
+mca_coll_han_allreduce_intra_simple(const void *sbuf,
+                                    void *rbuf,
+                                    size_t count,
+                                    struct ompi_datatype_t *dtype,
+                                    struct ompi_op_t *op,
+                                    struct ompi_communicator_t *comm,
+                                    mca_coll_base_module_t *module);
+int
+mca_coll_han_allreduce_reproducible_decision(struct ompi_communicator_t *comm,
+                                             mca_coll_base_module_t *module);
+int
+mca_coll_han_allreduce_reproducible(const void *sbuf,
+                                    void *rbuf,
+                                    size_t count,
+                                    struct ompi_datatype_t *dtype,
+                                    struct ompi_op_t *op,
+                                    struct ompi_communicator_t *comm,
+                                    mca_coll_base_module_t *module);
+
+int mca_coll_han_allreduce_intra(const void *sbuf,
+                                 void *rbuf,
+                                 size_t count,
+                                 struct ompi_datatype_t *dtype,
+                                 struct ompi_op_t *op,
+                                 struct ompi_communicator_t *comm, mca_coll_base_module_t * module);
+
+/* Scatter */
+int
+mca_coll_han_scatter_intra(const void *sbuf, size_t scount,
+                           struct ompi_datatype_t *sdtype,
+                           void *rbuf, size_t rcount,
+                           struct ompi_datatype_t *rdtype,
+                           int root,
+                           struct ompi_communicator_t *comm, mca_coll_base_module_t * module);
+int
+mca_coll_han_scatter_intra_simple(const void *sbuf, size_t scount,
+                                  struct ompi_datatype_t *sdtype,
+                                  void *rbuf, size_t rcount,
+                                  struct ompi_datatype_t *rdtype,
+                                  int root,
+                                  struct ompi_communicator_t *comm,
+                                  mca_coll_base_module_t * module);
+
+/* Scatterv */
+int
+mca_coll_han_scatterv_intra(const void *sbuf, ompi_count_array_t scounts,
+                            ompi_disp_array_t displs, struct ompi_datatype_t *sdtype,
+                            void *rbuf, size_t rcount,
+                            struct ompi_datatype_t *rdtype, 
+                            int root,
+                            struct ompi_communicator_t *comm,
+                            mca_coll_base_module_t *module);
+
+/* Gather */
+int
+mca_coll_han_gather_intra(const void *sbuf, size_t scount,
+                          struct ompi_datatype_t *sdtype,
+                          void *rbuf, size_t rcount,
+                          struct ompi_datatype_t *rdtype,
+                          int root,
+                          struct ompi_communicator_t *comm, mca_coll_base_module_t * module);
+int
+mca_coll_han_gather_intra_simple(const void *sbuf, size_t scount,
+                                 struct ompi_datatype_t *sdtype,
+                                 void *rbuf, size_t rcount,
+                                 struct ompi_datatype_t *rdtype,
+                                 int root,
+                                 struct ompi_communicator_t *comm,
+                                 mca_coll_base_module_t *module);
+
+/* Gatherv */
+int
+mca_coll_han_gatherv_intra(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                           void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t displs,
+                           struct ompi_datatype_t *rdtype, int root,
+                           struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+/* Allgather */
+int
+mca_coll_han_allgather_intra(const void *sbuf, size_t scount,
+                             struct ompi_datatype_t *sdtype,
+                             void *rbuf, size_t rcount,
+                             struct ompi_datatype_t *rdtype,
+                             struct ompi_communicator_t *comm, mca_coll_base_module_t * module);
+int
+mca_coll_han_allgather_intra_simple(const void *sbuf, size_t scount,
+                                    struct ompi_datatype_t *sdtype,
+                                    void* rbuf, size_t rcount,
+                                    struct ompi_datatype_t *rdtype,
+                                    struct ompi_communicator_t *comm,
+                                    mca_coll_base_module_t *module);
+
+/* Alltoall */
+int
+mca_coll_han_alltoall_using_smsc(ALLTOALL_BASE_ARGS,
+                                    mca_coll_base_module_t *module);
+
+/* Alltoallv */
+int
+mca_coll_han_alltoallv_using_smsc(ALLTOALLV_BASE_ARGS,
+                                    mca_coll_base_module_t *module);
+
+
+#endif
diff --git a/ompi/mca/coll/han/coll_han_allgather.c b/ompi/mca/coll/han/coll_han_allgather.c
index 4255df23faa..9d3a0825f83 100644
--- a/ompi/mca/coll/han/coll_han_allgather.c
+++ b/ompi/mca/coll/han/coll_han_allgather.c
@@ -1,8 +1,10 @@
 /*
- * Copyright (c) 2018-2020 The University of Tennessee and The University
+ * Copyright (c) 2018-2023 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2020      Bull S.A.S. All rights reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -31,10 +33,10 @@ mca_coll_han_set_allgather_args(mca_coll_han_allgather_t * args,
                                 mca_coll_task_t * cur_task,
                                 void *sbuf,
                                 void *sbuf_inter_free,
-                                int scount,
+                                size_t scount,
                                 struct ompi_datatype_t *sdtype,
                                 void *rbuf,
-                                int rcount,
+                                size_t rcount,
                                 struct ompi_datatype_t *rdtype,
                                 int root_low_rank,
                                 struct ompi_communicator_t *up_comm,
@@ -68,9 +70,9 @@ mca_coll_han_set_allgather_args(mca_coll_han_allgather_t * args,
  * Main function for taskified allgather: calls lg task, a gather on low comm
  */
 int
-mca_coll_han_allgather_intra(const void *sbuf, int scount,
+mca_coll_han_allgather_intra(const void *sbuf, size_t scount,
                              struct ompi_datatype_t *sdtype,
-                             void *rbuf, int rcount,
+                             void *rbuf, size_t rcount,
                              struct ompi_datatype_t *rdtype,
                              struct ompi_communicator_t *comm,
                              mca_coll_base_module_t * module)
@@ -81,9 +83,9 @@ mca_coll_han_allgather_intra(const void *sbuf, int scount,
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "han cannot handle allgather within this communicator. Fall back on another component\n"));
         /* HAN cannot work with this communicator so fallback on all collectives */
-        HAN_LOAD_FALLBACK_COLLECTIVES(han_module, comm);
-        return comm->c_coll->coll_allgather(sbuf, scount, sdtype, rbuf, rcount, rdtype,
-                                            comm, comm->c_coll->coll_allgather_module);
+        HAN_LOAD_FALLBACK_COLLECTIVES(comm, han_module);
+        return han_module->previous_allgather(sbuf, scount, sdtype, rbuf, rcount, rdtype,
+                                              comm, han_module->previous_allgather_module);
     }
     ompi_communicator_t *low_comm = han_module->sub_comm[INTRA_NODE];
     ompi_communicator_t *up_comm = han_module->sub_comm[INTER_NODE];
@@ -96,9 +98,9 @@ mca_coll_han_allgather_intra(const void *sbuf, int scount,
     if (han_module->are_ppn_imbalanced) {
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "han cannot handle allgather with this communicator (imbalance). Fall back on another component\n"));
-        HAN_LOAD_FALLBACK_COLLECTIVE(han_module, comm, allgather);
-        return comm->c_coll->coll_allgather(sbuf, scount, sdtype, rbuf, rcount, rdtype,
-                                            comm, comm->c_coll->coll_allgather_module);
+        HAN_UNINSTALL_COLL_API(comm, han_module, allgather);
+        return han_module->previous_allgather(sbuf, scount, sdtype, rbuf, rcount, rdtype,
+                                              comm, han_module->previous_allgather_module);
     }
 
     ompi_request_t *temp_request;
@@ -106,7 +108,7 @@ mca_coll_han_allgather_intra(const void *sbuf, int scount,
     temp_request = OBJ_NEW(ompi_request_t);
     temp_request->req_state = OMPI_REQUEST_ACTIVE;
     temp_request->req_type = OMPI_REQUEST_COLL;
-    temp_request->req_free = han_request_free;
+    temp_request->req_free = ompi_coll_han_request_free;
     temp_request->req_status = (ompi_status_public_t){0};
     temp_request->req_complete = REQUEST_PENDING;
 
@@ -291,9 +293,9 @@ int mca_coll_han_allgather_lb_task(void *task_args)
  * communications without tasks.
  */
 int
-mca_coll_han_allgather_intra_simple(const void *sbuf, int scount,
+mca_coll_han_allgather_intra_simple(const void *sbuf, size_t scount,
                                     struct ompi_datatype_t *sdtype,
-                                    void* rbuf, int rcount,
+                                    void* rbuf, size_t rcount,
                                     struct ompi_datatype_t *rdtype,
                                     struct ompi_communicator_t *comm,
                                     mca_coll_base_module_t *module){
@@ -305,9 +307,9 @@ mca_coll_han_allgather_intra_simple(const void *sbuf, int scount,
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "han cannot handle allgather within this communicator. Fall back on another component\n"));
         /* HAN cannot work with this communicator so fallback on all collectives */
-        HAN_LOAD_FALLBACK_COLLECTIVES(han_module, comm);
-        return comm->c_coll->coll_allgather(sbuf, scount, sdtype, rbuf, rcount, rdtype,
-                                            comm, comm->c_coll->coll_allgather_module);
+        HAN_LOAD_FALLBACK_COLLECTIVES(comm, han_module);
+        return han_module->previous_allgather(sbuf, scount, sdtype, rbuf, rcount, rdtype,
+                                              comm, han_module->previous_allgather_module);
     }
     /* discovery topology */
     int *topo = mca_coll_han_topo_init(comm, han_module, 2);
@@ -319,9 +321,9 @@ mca_coll_han_allgather_intra_simple(const void *sbuf, int scount,
         /* Put back the fallback collective support and call it once. All
          * future calls will then be automatically redirected.
          */
-        HAN_LOAD_FALLBACK_COLLECTIVE(han_module, comm, allgather);
-        return comm->c_coll->coll_allgather(sbuf, scount, sdtype, rbuf, rcount, rdtype,
-                                            comm, comm->c_coll->coll_allgather_module);
+        HAN_UNINSTALL_COLL_API(comm, han_module, allgather);
+        return han_module->previous_allgather(sbuf, scount, sdtype, rbuf, rcount, rdtype,
+                                              comm, han_module->previous_allgather_module);
     }
 
     ompi_communicator_t *low_comm = han_module->sub_comm[INTRA_NODE];
diff --git a/ompi/mca/coll/han/coll_han_allreduce.c b/ompi/mca/coll/han/coll_han_allreduce.c
index 5bc1f2f8bfe..26ed44daf89 100644
--- a/ompi/mca/coll/han/coll_han_allreduce.c
+++ b/ompi/mca/coll/han/coll_han_allreduce.c
@@ -1,10 +1,12 @@
 /*
- * Copyright (c) 2018-2020 The University of Tennessee and The University
+ * Copyright (c) 2018-2023 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2020      Bull S.A.S. All rights reserved.
  *
  * Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -70,7 +72,7 @@ mca_coll_han_set_allreduce_args(mca_coll_han_allreduce_args_t * args,
 }
 
 /*
- * Each segment of the messsage needs to go though 4 steps to perform MPI_Allreduce:
+ * Each segment of the message needs to go though 4 steps to perform MPI_Allreduce:
  *     lr: lower level (shared-memory or intra-node) reduce,
  *     ur: upper level (inter-node) reduce,
  *     ub: upper level (inter-node) bcast,
@@ -89,7 +91,7 @@ mca_coll_han_set_allreduce_args(mca_coll_han_allreduce_args_t * args,
 int
 mca_coll_han_allreduce_intra(const void *sbuf,
                              void *rbuf,
-                             int count,
+                             size_t count,
                              struct ompi_datatype_t *dtype,
                              struct ompi_op_t *op,
                              struct ompi_communicator_t *comm, mca_coll_base_module_t * module)
@@ -108,9 +110,9 @@ mca_coll_han_allreduce_intra(const void *sbuf,
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "han cannot handle allreduce with this communicator. Drop HAN support in this communicator and fall back on another component\n"));
         /* HAN cannot work with this communicator so fallback on all collectives */
-        HAN_LOAD_FALLBACK_COLLECTIVES(han_module, comm);
-        return comm->c_coll->coll_allreduce(sbuf, rbuf, count, dtype, op,
-                                            comm, comm->c_coll->coll_reduce_module);
+        HAN_LOAD_FALLBACK_COLLECTIVES(comm, han_module);
+        return han_module->previous_allreduce(sbuf, rbuf, count, dtype, op,
+                                              comm, han_module->previous_allreduce_module);
     }
 
     ptrdiff_t extent, lb;
@@ -130,8 +132,8 @@ mca_coll_han_allreduce_intra(const void *sbuf,
                                 seg_count);
 
     /* Determine number of elements sent per task. */
-    OPAL_OUTPUT_VERBOSE((10, mca_coll_han_component.han_output,
-                         "In HAN Allreduce seg_size %d seg_count %d count %d\n",
+    OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
+                         "In HAN Allreduce seg_size %d seg_count %d count %zu\n",
                          mca_coll_han_component.han_allreduce_segsize, seg_count, count));
     int num_segments = (count + seg_count - 1) / seg_count;
 
@@ -183,7 +185,7 @@ mca_coll_han_allreduce_intra(const void *sbuf,
         mca_coll_task_t *t_next_seg = OBJ_NEW(mca_coll_task_t);
         /* Setup up t_next_seg task arguments */
         t->cur_task = t_next_seg;
-        t->sbuf = (char *) t->sbuf + extent * t->seg_count;
+        t->sbuf = (t->sbuf == MPI_IN_PLACE) ? MPI_IN_PLACE : (char *) t->sbuf + extent * t->seg_count;
         t->rbuf = (char *) t->rbuf + extent * t->seg_count;
         t->cur_seg = t->cur_seg + 1;
         /* Init t_next_seg task */
@@ -261,11 +263,26 @@ int mca_coll_han_allreduce_t1_task(void *task_args)
         if (t->cur_seg == t->num_segments - 2 && t->last_seg_count != t->seg_count) {
             tmp_count = t->last_seg_count;
         }
-        t->low_comm->c_coll->coll_reduce((char *) t->sbuf + extent * t->seg_count,
-                                         (char *) t->rbuf + extent * t->seg_count, tmp_count,
-                                         t->dtype, t->op, t->root_low_rank, t->low_comm,
-                                         t->low_comm->c_coll->coll_reduce_module);
 
+        if (t->sbuf == MPI_IN_PLACE) {
+            if (!t->noop) {
+                t->low_comm->c_coll->coll_reduce(MPI_IN_PLACE,
+                                                 (char *) t->rbuf + extent * t->seg_count, tmp_count,
+                                                 t->dtype, t->op, t->root_low_rank, t->low_comm,
+                                                 t->low_comm->c_coll->coll_reduce_module);
+            } else {
+                t->low_comm->c_coll->coll_reduce((char *) t->rbuf + extent * t->seg_count,
+                                                 NULL, tmp_count,
+                                                 t->dtype, t->op, t->root_low_rank, t->low_comm,
+                                                 t->low_comm->c_coll->coll_reduce_module);
+
+            }
+        } else {
+            t->low_comm->c_coll->coll_reduce((char *) t->sbuf + extent * t->seg_count,
+                                             (char *) t->rbuf + extent * t->seg_count, tmp_count,
+                                             t->dtype, t->op, t->root_low_rank, t->low_comm,
+                                             t->low_comm->c_coll->coll_reduce_module);
+	}
     }
     if (!t->noop) {
         ompi_request_wait(&ireduce_req, MPI_STATUS_IGNORE);
@@ -320,10 +337,26 @@ int mca_coll_han_allreduce_t2_task(void *task_args)
         if (t->cur_seg == t->num_segments - 3 && t->last_seg_count != t->seg_count) {
             tmp_count = t->last_seg_count;
         }
-        t->low_comm->c_coll->coll_reduce((char *) t->sbuf + 2 * extent * t->seg_count,
-                                         (char *) t->rbuf + 2 * extent * t->seg_count, tmp_count,
-                                         t->dtype, t->op, t->root_low_rank, t->low_comm,
-                                         t->low_comm->c_coll->coll_reduce_module);
+
+	if (t->sbuf == MPI_IN_PLACE) {
+	    if (!t->noop) {
+                t->low_comm->c_coll->coll_reduce(MPI_IN_PLACE,
+                                                 (char *) t->rbuf + 2 * extent * t->seg_count, tmp_count,
+                                                 t->dtype, t->op, t->root_low_rank, t->low_comm,
+                                                 t->low_comm->c_coll->coll_reduce_module);
+	    } else {
+                t->low_comm->c_coll->coll_reduce((char *) t->rbuf + 2 * extent * t->seg_count,
+                                                 NULL, tmp_count,
+                                                 t->dtype, t->op, t->root_low_rank, t->low_comm,
+                                                 t->low_comm->c_coll->coll_reduce_module);
+
+	    }
+	} else {
+            t->low_comm->c_coll->coll_reduce((char *) t->sbuf + 2 * extent * t->seg_count,
+                                             (char *) t->rbuf + 2 * extent * t->seg_count, tmp_count,
+                                             t->dtype, t->op, t->root_low_rank, t->low_comm,
+                                             t->low_comm->c_coll->coll_reduce_module);
+	}
     }
     if (!t->noop && req_count > 0) {
         ompi_request_wait_all(req_count, reqs, MPI_STATUSES_IGNORE);
@@ -353,7 +386,7 @@ int mca_coll_han_allreduce_t3_task(void *task_args)
             if (t->cur_seg == t->num_segments - 2 && t->last_seg_count != t->seg_count) {
                 tmp_count = t->last_seg_count;
             }
-            t->up_comm->c_coll->coll_ibcast((char *) t->rbuf + extent * t->seg_count, t->seg_count,
+            t->up_comm->c_coll->coll_ibcast((char *) t->rbuf + extent * t->seg_count, tmp_count,
                                             t->dtype, t->root_up_rank, t->up_comm, &(reqs[0]),
                                             t->up_comm->c_coll->coll_ibcast_module);
             req_count++;
@@ -384,13 +417,34 @@ int mca_coll_han_allreduce_t3_task(void *task_args)
         if (t->cur_seg == t->num_segments - 4 && t->last_seg_count != t->seg_count) {
             tmp_count = t->last_seg_count;
         }
-        t->low_comm->c_coll->coll_reduce((char *) t->sbuf + 3 * extent * t->seg_count,
-                                         (char *) t->rbuf + 3 * extent * t->seg_count, tmp_count,
-                                         t->dtype, t->op, t->root_low_rank, t->low_comm,
-                                         t->low_comm->c_coll->coll_reduce_module);
+
+        if (t->sbuf == MPI_IN_PLACE) {
+            if (!t->noop) {
+                t->low_comm->c_coll->coll_reduce(MPI_IN_PLACE,
+                                                 (char *) t->rbuf + 3 * extent * t->seg_count, tmp_count,
+                                                 t->dtype, t->op, t->root_low_rank, t->low_comm,
+                                                 t->low_comm->c_coll->coll_reduce_module);
+	    } else {
+                t->low_comm->c_coll->coll_reduce((char *) t->rbuf + 3 * extent * t->seg_count,
+                                                 NULL, tmp_count,
+                                                 t->dtype, t->op, t->root_low_rank, t->low_comm,
+                                                 t->low_comm->c_coll->coll_reduce_module);
+            }
+        } else {
+            t->low_comm->c_coll->coll_reduce((char *) t->sbuf + 3 * extent * t->seg_count,
+                                             (char *) t->rbuf + 3 * extent * t->seg_count, tmp_count,
+                                             t->dtype, t->op, t->root_low_rank, t->low_comm,
+                                             t->low_comm->c_coll->coll_reduce_module);
+        }
     }
     /* lb of cur_seg */
-    t->low_comm->c_coll->coll_bcast((char *) t->rbuf, t->seg_count, t->dtype, t->root_low_rank,
+    if (t->cur_seg == t->num_segments - 1 && t->last_seg_count != t->seg_count) {
+        tmp_count = t->last_seg_count;
+    } else {
+        tmp_count = t->seg_count;
+    }
+
+    t->low_comm->c_coll->coll_bcast((char *) t->rbuf, tmp_count, t->dtype, t->root_low_rank,
                                     t->low_comm, t->low_comm->c_coll->coll_bcast_module);
     if (!t->noop && req_count > 0) {
         ompi_request_wait_all(req_count, reqs, MPI_STATUSES_IGNORE);
@@ -411,7 +465,7 @@ int mca_coll_han_allreduce_t3_task(void *task_args)
 int
 mca_coll_han_allreduce_intra_simple(const void *sbuf,
                                     void *rbuf,
-                                    int count,
+                                    size_t count,
                                     struct ompi_datatype_t *dtype,
                                     struct ompi_op_t *op,
                                     struct ompi_communicator_t *comm,
@@ -442,9 +496,9 @@ mca_coll_han_allreduce_intra_simple(const void *sbuf,
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "han cannot handle allreduce with this communicator. Drop HAN support in this communicator and fall back on another component\n"));
         /* HAN cannot work with this communicator so fallback on all collectives */
-        HAN_LOAD_FALLBACK_COLLECTIVES(han_module, comm);
-        return comm->c_coll->coll_allreduce(sbuf, rbuf, count, dtype, op,
-                                            comm, comm->c_coll->coll_reduce_module);
+        HAN_LOAD_FALLBACK_COLLECTIVES(comm, han_module);
+        return han_module->previous_allreduce(sbuf, rbuf, count, dtype, op,
+                                              comm, han_module->previous_allreduce_module);
     }
 
     low_comm = han_module->sub_comm[INTRA_NODE];
@@ -536,7 +590,7 @@ mca_coll_han_allreduce_reproducible_decision(struct ompi_communicator_t *comm,
                 opal_output_verbose(30, mca_coll_han_component.han_output,
                                     "coll:han:allreduce_reproducible: "
                                     "fallback on %s\n",
-                                    available_components[fallback].component_name);
+                                    ompi_coll_han_available_components[fallback].component_name);
             }
             han_module->reproducible_allreduce_module = fallback_module;
             han_module->reproducible_allreduce = fallback_module->coll_allreduce;
@@ -558,7 +612,7 @@ mca_coll_han_allreduce_reproducible_decision(struct ompi_communicator_t *comm,
 int
 mca_coll_han_allreduce_reproducible(const void *sbuf,
                                     void *rbuf,
-                                     int count,
+                                     size_t count,
                                      struct ompi_datatype_t *dtype,
                                      struct ompi_op_t *op,
                                      struct ompi_communicator_t *comm,
diff --git a/ompi/mca/coll/han/coll_han_alltoall.c b/ompi/mca/coll/han/coll_han_alltoall.c
new file mode 100644
index 00000000000..6437ea7a6af
--- /dev/null
+++ b/ompi/mca/coll/han/coll_han_alltoall.c
@@ -0,0 +1,397 @@
+/*
+ * Copyright (c) 2024      Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Copyright (c) 2025      Barcelona Supercomputing Center (BSC-CNS). All Rights Reserved.
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/**
+ * @file
+ *
+ * This file contains the hierarchical implementations of alltoall.
+ *
+ * mca_coll_han_alltoall_using_smsc:
+ * This algorithm relies on SMSC and specifically XPMEM because of
+ * the need to direct-map the memory.
+ *
+ * Each rank on one host is assigned a single
+ * partner on a remote host and vice versa.  Then the rank collects all
+ * the data its partner will need to receive from its host, and sends it
+ * in one large send, and likewise receives its data in one large recv,
+ * then cycles to the next host.
+ */
+
+#include "coll_han.h"
+#include "ompi/mca/coll/base/coll_base_functions.h"
+#include "ompi/mca/coll/base/coll_tags.h"
+#include "ompi/mca/pml/pml.h"
+#include "coll_han_trigger.h"
+#include "opal/mca/smsc/smsc.h"
+#include "opal/mca/rcache/rcache.h"
+#include "ompi/mca/osc/base/base.h"
+
+
+
+/* Who is the given ranks partner during the exchange?
+   This function will require rounds comm_size-many rounds, and your partner
+   will select you in the same round which you select that partner. */
+static inline int ring_partner_no_skip(int rank, int round, int comm_size) {
+    /* make sure ring_partner is positive: make argument to modulo > 0 with +comm_size.*/
+    return (comm_size + round - rank) % comm_size;
+}
+
+/* Who is the given ranks partner during the exchange?
+   This function will require rounds comm_size-many rounds, and does
+   self-exchange last. */
+static inline int ring_partner(int rank, int round, int comm_size) {
+    round = round % comm_size;
+    if (round == comm_size - 1) {
+        /* last round: self-exchange */
+        return rank;
+    }
+    int self_round = (2*rank) % comm_size;
+    if ( round < self_round )
+        return ring_partner_no_skip(rank, round, comm_size);
+    else
+        return ring_partner_no_skip(rank, round+1, comm_size);
+}
+
+int mca_coll_han_alltoall_using_smsc(
+        const void *sbuf, size_t scount,
+        struct ompi_datatype_t *sdtype,
+        void* rbuf, size_t rcount,
+        struct ompi_datatype_t *rdtype,
+        struct ompi_communicator_t *comm,
+        mca_coll_base_module_t *module)
+{
+
+    mca_coll_han_module_t *han_module = (mca_coll_han_module_t *)module;
+
+    OPAL_OUTPUT_VERBOSE((90, mca_coll_han_component.han_output,
+                            "Entering mca_coll_han_alltoall_using_smsc\n"));
+
+    if (!mca_smsc || !mca_smsc_base_has_feature(MCA_SMSC_FEATURE_CAN_MAP)) {
+        /* Assume all hosts take this path together :-\ */
+        opal_output_verbose(1, mca_coll_han_component.han_output, "in mca_coll_han_alltoall_using_smsc, "
+            "but MCA_SMSC_FEATURE_CAN_MAP not available.  Disqualifying this alg!\n");
+        HAN_UNINSTALL_COLL_API(comm, han_module, alltoall);
+        return han_module->previous_alltoall(sbuf, scount, sdtype, rbuf, rcount, rdtype,
+                                             comm, han_module->previous_alltoall_module);
+    }
+
+    /* Create the subcommunicators */
+    if( OMPI_SUCCESS != mca_coll_han_comm_create_new(comm, han_module) ) {
+        opal_output_verbose(1, mca_coll_han_component.han_output,
+                             "han cannot handle alltoall with this communicator. Fall back on another component\n");
+        /* HAN cannot work with this communicator so fallback on all collectives */
+        HAN_LOAD_FALLBACK_COLLECTIVES(comm, han_module);
+        return han_module->previous_alltoall(sbuf, scount, sdtype, rbuf, rcount, rdtype,
+                                          comm, han_module->previous_alltoall_module);
+    }
+
+    /* Topo must be initialized to know rank distribution which then is used to
+     * determine if han can be used */
+    mca_coll_han_topo_init(comm, han_module, 2);
+    if (han_module->are_ppn_imbalanced || !han_module->is_mapbycore){
+        opal_output_verbose(1, mca_coll_han_component.han_output,
+                             "han cannot handle alltoall with this communicator (imbalance/!mapbycore).  "
+                             "Fall back on another component\n");
+        /* Put back the fallback collective support and call it once. All
+         * future calls will then be automatically redirected.
+         */
+        HAN_UNINSTALL_COLL_API(comm, han_module, alltoall);
+        return han_module->previous_alltoall(sbuf, scount, sdtype, rbuf, rcount, rdtype,
+                                             comm, han_module->previous_alltoall_module);
+    }
+
+    int rc, send_needs_bounce, ii_push_data;
+    size_t sndsize;
+    MPI_Aint sextent, rextent, lb;
+    char *send_bounce;
+    opal_convertor_t convertor;
+    size_t packed_size = 0, packed_size_tmp;
+    int use_isend;
+    void *gather_buf_in[4];
+    int up_rank;
+
+    ompi_communicator_t *low_comm = han_module->sub_comm[INTRA_NODE];
+    ompi_communicator_t *up_comm = han_module->sub_comm[INTER_NODE];
+    ompi_request_t **inter_recv_reqs;
+    ompi_request_t **inter_send_reqs;
+
+    rc = ompi_datatype_get_extent( sdtype, &lb, &sextent);
+    rc = ompi_datatype_get_extent( rdtype, &lb, &rextent);
+    opal_datatype_type_size( &sdtype->super, &sndsize);
+
+    int w_rank = ompi_comm_rank(comm);
+    int w_size = ompi_comm_size(comm);
+
+    /* information about sub-communicators */
+    int low_rank = ompi_comm_rank(low_comm);
+    int low_size = ompi_comm_size(low_comm);
+    int up_size = ompi_comm_size(up_comm);
+
+    int fanout = mca_coll_han_component.han_alltoall_pstages;
+    if (!fanout) {
+        fanout = 1;
+    }
+    if (fanout > up_size) { fanout = up_size; }
+
+    OBJ_CONSTRUCT( &convertor, opal_convertor_t );
+
+
+    send_needs_bounce = 0;
+    /* get converter for copying to one of the leader ranks, and get packed size: */
+    opal_convertor_copy_and_prepare_for_send(ompi_mpi_local_convertor, &sdtype->super, scount, sbuf, 0, &convertor);
+    send_needs_bounce |= 0 != opal_convertor_on_device(&convertor);
+    send_needs_bounce |= opal_convertor_need_buffers(&convertor);
+    opal_convertor_cleanup(&convertor);
+
+    opal_convertor_copy_and_prepare_for_recv(ompi_mpi_local_convertor, &rdtype->super, rcount, rbuf, 0, &convertor);
+    send_needs_bounce |= 0 != opal_convertor_on_device(&convertor);
+    send_needs_bounce |= opal_convertor_need_buffers(&convertor);
+    opal_convertor_get_packed_size( &convertor, &packed_size );
+    opal_convertor_cleanup(&convertor);
+
+    /*
+      Because push-mode needs extra synchronizations, we'd like to avoid it,
+      however it might be necessary:
+
+      If application buffer is non-contigious or non-homogenous, then we'll
+      need to "push" the data so that the packing process properly knows the
+      memory layout and types.
+
+      If the application buffer is device memory, we'll also need to exchange
+      in push mode so that the process which has device registrations can
+      perform the reads.
+
+      In both of these cases, we'll need to use the bounce buffer too.
+    */
+    ii_push_data = send_needs_bounce;
+
+    /*
+      If we have a fanout > 1, we'll need somewhere to put data for the next
+      send while the previous send is still in-flight.  We'll need a dedicated
+      bounce buffer for this, but it doesn't mean we have to use "push" mode.
+    */
+    send_needs_bounce |= fanout != 1;
+
+    up_rank = w_rank / low_size;
+    assert( w_rank % low_size == low_rank );
+    int64_t send_bytes_per_fan = low_size * packed_size;
+    inter_send_reqs = malloc(sizeof(*inter_send_reqs) * fanout);
+    inter_recv_reqs = malloc(sizeof(*inter_recv_reqs) * up_size );
+    char **low_bufs = malloc(low_size * sizeof(*low_bufs));
+    void **sbuf_map_ctx = malloc(low_size * sizeof(&sbuf_map_ctx));
+
+    const int nptrs_gather = 3;
+    void **gather_buf_out = calloc(low_size*nptrs_gather, sizeof(void*));
+    bool send_bounce_is_allocated = false;
+
+    do {
+start_allgather:
+        if ( 0 == send_needs_bounce ) {
+            send_bounce = (char*)rbuf + up_rank*send_bytes_per_fan;
+        } else {
+            if (!send_bounce_is_allocated) {
+                send_bounce = malloc(send_bytes_per_fan * fanout);
+                send_bounce_is_allocated = true;
+            }
+        }
+
+        if (ii_push_data) {
+            /* all ranks will push to the other ranks' bounce buffer */
+            gather_buf_in[0] = send_bounce;
+        } else {
+            /* all ranks will pull from the other ranks' sbuf */
+            gather_buf_in[0] = (void*)sbuf;
+        }
+        gather_buf_in[1] = (void*)(intptr_t)send_needs_bounce;
+        gather_buf_in[2] = (void*)(intptr_t)ii_push_data;
+
+        rc = low_comm->c_coll->coll_allgather(gather_buf_in, nptrs_gather, MPI_AINT,
+                    gather_buf_out, nptrs_gather, MPI_AINT, low_comm,
+                    low_comm->c_coll->coll_allgather_module);
+
+        if (rc != 0) {
+            OPAL_OUTPUT_VERBOSE((40, mca_coll_han_component.han_output,
+            "Allgather failed with %d\n",rc));
+            goto cleanup;
+        }
+
+        for (int jother=0; jother<low_size; jother++) {
+            int other_push_data = (uintptr_t)gather_buf_out[nptrs_gather*jother + 2] & 0x1;
+            if (ii_push_data != other_push_data) {
+                ii_push_data = 1;
+                goto start_allgather;
+            }
+            send_needs_bounce  |= (uintptr_t)gather_buf_out[nptrs_gather*jother + 1] & 0x1;
+            ii_push_data       |= other_push_data;
+        }
+    } while (0);
+
+    use_isend = fanout > 1 || ii_push_data;
+
+    for (int jother=0; jother<low_size; jother++) {
+        low_bufs[jother] = gather_buf_out[nptrs_gather*jother];
+        if (jother == low_rank) {
+            sbuf_map_ctx[low_rank] = NULL;
+        } else {
+            struct ompi_proc_t* ompi_proc = ompi_comm_peer_lookup(low_comm, jother);
+            mca_smsc_endpoint_t *smsc_ep;
+            smsc_ep = mca_coll_han_get_smsc_endpoint(ompi_proc);
+
+            sbuf_map_ctx[jother] = mca_smsc->map_peer_region(
+                smsc_ep,
+                MCA_RCACHE_FLAGS_PERSIST,
+                low_bufs[jother],
+                sextent*w_size*scount,
+                (void**) &low_bufs[jother] );
+        }
+    }
+
+    for (int jslot=0; jslot < fanout; jslot++) {
+        inter_send_reqs[jslot] = MPI_REQUEST_NULL;
+    }
+
+
+    /* pre-post all our receives.  We will be ready to receive all data regardless of fan-out.
+       (This is not an in-place algorithm)*/
+    int inter_recv_count = 0;
+    for (int jround=0; jround<up_size; jround++) {
+            int up_partner = ring_partner(up_rank, jround, up_size);
+            int first_remote_wrank = up_partner*low_size;
+            /* pre-post the receive for remote.  Receive directly into application buffer */
+            char *recv_chunk = ((char*)rbuf) + rextent*rcount*first_remote_wrank;
+
+            MCA_PML_CALL(irecv
+                        (recv_chunk, rcount*low_size, rdtype, first_remote_wrank+low_rank,
+                        MCA_COLL_BASE_TAG_ALLTOALL,
+                        comm, &inter_recv_reqs[inter_recv_count++]));
+    }
+
+    /* outer loop: in typical "fanout=1 case" do 1 iter per rank in the upper comm (ie, 1 per host)*/
+    int nloops = up_size + fanout*use_isend;
+    for (int jloop=0; jloop<nloops; jloop++) {
+
+        int up_partner = ring_partner(up_rank, jloop, up_size);
+        int jfan_slot = jloop % fanout;
+
+        /* complete previous inter-node send */
+        int prev_slot = jloop - fanout;
+        if (use_isend && prev_slot >= 0 ) {
+            /* we cannot fill for our next send until the previous send using this buffer is completed. */
+            ompi_request_wait(&inter_send_reqs[jfan_slot], MPI_STATUS_IGNORE);
+            if (ii_push_data && jloop < up_size) {
+                /*  barrier here so followers know all leaders have completed
+                    previous isend for this slot, and may begin overwriting bounce slot. */
+                low_comm->c_coll->coll_barrier(low_comm, low_comm->c_coll->coll_barrier_module);
+            }
+        }
+
+        if (jloop < up_size) {
+            /* For this upper-comm partner, we must provide our data to all the leaders. */
+
+            int first_remote_wrank;
+            first_remote_wrank = up_partner*low_size;
+
+            assert(up_partner >= 0);
+            assert(up_partner < up_size);
+            assert(first_remote_wrank <= w_size - low_size );
+
+            /* pack data into each of the leaders' buffers */
+            for (int jlow=0; jlow<low_size; jlow++) {
+
+                int remote_wrank = first_remote_wrank + jlow;
+                struct iovec iov;
+                uint32_t iov_count = 1;
+
+                ptrdiff_t fan_slot_offset = send_bytes_per_fan*jfan_slot;
+                ptrdiff_t rank_offset = packed_size*low_rank;
+                void *from_addr, *to_addr;
+
+                if (ii_push_data) {
+                    /* push our own data for remote into local leader's send_buf  */
+                    from_addr = (char*)sbuf + sextent*scount*remote_wrank;
+                    to_addr = low_bufs[jlow] + fan_slot_offset + rank_offset;
+                } else {
+                    /* pull other ranks' data for remote into our own send_buf */
+                    to_addr = send_bounce + fan_slot_offset + packed_size*jlow;
+                    from_addr = low_bufs[jlow] + packed_size*(low_size*up_partner+low_rank);
+                }
+                /* prepare: set the source for the copy */
+                opal_convertor_copy_and_prepare_for_send(ompi_mpi_local_convertor, &sdtype->super, scount,
+                                    from_addr,
+                                    0, &convertor);
+                /* iovec: set the destination of the copy */
+                iov.iov_base = to_addr;
+                iov.iov_len = sextent*scount;
+
+                /* pack the data directly into local leader's sendbuf */
+                packed_size_tmp = packed_size;
+                rc = opal_convertor_pack(&convertor, &iov, &iov_count, &packed_size_tmp);
+                opal_convertor_cleanup(&convertor);
+
+                if (1 != rc) {
+                    opal_output_verbose(1, mca_coll_han_component.han_output,
+                            "opal_convert_pack failed with %d\n",rc);
+                    rc = MPI_ERR_TRUNCATE;
+                    goto cleanup;
+                }
+                rc = MPI_SUCCESS;
+            }
+
+            if (ii_push_data) {
+                /* barrier here so leaders know all followers have filled data,
+                   and can issue send. */
+                low_comm->c_coll->coll_barrier(low_comm, low_comm->c_coll->coll_barrier_module);
+            }
+
+            if (use_isend == 0) {
+                MCA_PML_CALL(send
+                        (send_bounce,
+                        send_bytes_per_fan, MPI_PACKED, first_remote_wrank+low_rank,
+                        MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD,
+                        comm) );
+            } else {
+                /* send the data to our remote partner */
+                MCA_PML_CALL(isend
+                        (&send_bounce[send_bytes_per_fan*jfan_slot],
+                        send_bytes_per_fan, MPI_PACKED, first_remote_wrank+low_rank,
+                        MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD,
+                        comm, &inter_send_reqs[jfan_slot]));
+            }
+        }
+    }
+
+    /* wait for all irecv to complete */
+    ompi_request_wait_all(inter_recv_count, inter_recv_reqs, MPI_STATUS_IGNORE);
+
+cleanup:
+
+    /* we may still have neighbors reading directly from our buffer, so we must ensure it is not modified */
+    if (!ii_push_data)
+    {
+        low_comm->c_coll->coll_barrier(low_comm, low_comm->c_coll->coll_barrier_module);
+    }
+
+    for (int jlow=0; jlow<low_size; jlow++) {
+        if (jlow != low_rank ) {
+            mca_smsc->unmap_peer_region(sbuf_map_ctx[jlow]);
+        }
+    }
+    OBJ_DESTRUCT(&convertor);
+    if (send_bounce_is_allocated) free(send_bounce);
+    free(inter_send_reqs);
+    free(inter_recv_reqs);
+    free(sbuf_map_ctx);
+    free(low_bufs);
+    free(gather_buf_out);
+
+    OPAL_OUTPUT_VERBOSE((40, mca_coll_han_component.han_output,
+                "Alltoall Complete with %d\n",rc));
+    return rc;
+}
diff --git a/ompi/mca/coll/han/coll_han_alltoallv.c b/ompi/mca/coll/han/coll_han_alltoallv.c
new file mode 100644
index 00000000000..b3c25014b2f
--- /dev/null
+++ b/ompi/mca/coll/han/coll_han_alltoallv.c
@@ -0,0 +1,991 @@
+/*
+ * Copyright (c) 2024      Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/**
+ * @file
+ *
+ * This file contains the hierarchical implementations of alltoallv.
+ *
+ * mca_coll_han_alltoallv_using_smsc:
+ * This algorithm relies on SMSC and specifically XPMEM because of
+ * the need to direct-map the memory.
+ *
+ * Each rank on one host is assigned a single
+ * partner on a remote host and vice versa.  Then the rank collects all
+ * the data its partner will need to receive from its host, and sends it
+ * in one large send, and likewise receives its data in one large recv,
+ * then cycles to the next host.
+ */
+
+#include "coll_han.h"
+#include "ompi/mca/coll/base/coll_base_functions.h"
+#include "ompi/mca/coll/base/coll_tags.h"
+#include "ompi/mca/pml/pml.h"
+#include "coll_han_trigger.h"
+#include "opal/mca/smsc/smsc.h"
+#include "opal/mca/rcache/rcache.h"
+#include "opal/datatype/opal_datatype.h"
+#include "opal/datatype/opal_datatype_internal.h"
+#include "ompi/mca/osc/base/base.h"
+
+/* Who is the given ranks partner during the exchange?
+   This function will require rounds comm_size-many rounds, and your partner
+   will select you in the same round which you select that partner. */
+static inline int ring_partner_no_skip(int rank, int round, int comm_size) {
+    /* make sure ring_partner is positive: make argument to modulo > 0 with +comm_size.*/
+    return (comm_size + round - rank) % comm_size;
+}
+
+/* Who is the given ranks partner during the exchange?
+   This function will require rounds comm_size-many rounds, and does
+   self-exchange last. */
+static inline int ring_partner(int rank, int round, int comm_size) {
+    round = round % comm_size;
+    if (round == comm_size - 1) {
+        /* last round: self-exchange */
+        return rank;
+    }
+    int self_round = (2*rank) % comm_size;
+    if ( round < self_round )
+        return ring_partner_no_skip(rank, round, comm_size);
+    else
+        return ring_partner_no_skip(rank, round+1, comm_size);
+}
+
+struct peer_data {
+    const void *sbuf;           /* mmapped: ctx in map_ctx[1] */
+    struct peer_counts *counts; /* mmapped: ctx in map_ctx[0] */
+    opal_datatype_t *sendtype;  /* local. deserialized from tailer of map_ctx[0] */
+    void *map_ctx[2];
+};
+
+struct peer_counts {
+    size_t scount;
+    size_t sdispl;
+    size_t rcount;
+    size_t rdispl;
+};
+
+struct gathered_data {
+    size_t stype_serialized_length;
+    ssize_t sbuf_span;
+    ssize_t sbuf_gap;
+    void *sbuf;
+    void *serialization_buffer;
+};
+
+/** calculate the minumum true lower bound and maximum true upper bound for all
+ * peers accessing a buffer.
+ * Outputs: min_lb and max_lb contain signed offsets in bytes from buffer pointer.
+ */
+static void coll_han_alltoallv_calc_all_span(
+                    int nranks,
+                    opal_datatype_t *dtype,
+                    ompi_count_array_t counts,
+                    ompi_disp_array_t displs,
+                    ssize_t *gap,
+                    ssize_t *span ) {
+    ptrdiff_t min_lb = PTRDIFF_MAX;
+    ptrdiff_t max_ub = PTRDIFF_MIN;
+    *gap = 0;
+    *span = 0;
+
+    ptrdiff_t dummy_lb;
+    ptrdiff_t dtype_extent;
+    opal_datatype_get_extent(dtype, &dummy_lb, &dtype_extent);
+
+    for (int jrankw=0; jrankw<nranks; jrankw++) {
+        ptrdiff_t displ;
+        ptrdiff_t count;
+        ssize_t lb, ub;
+
+        count = ompi_count_array_get(counts,jrankw);
+        displ = ompi_disp_array_get(displs,jrankw);
+        if (count > 0) {
+            ptrdiff_t one_gap;
+            ptrdiff_t one_span = opal_datatype_span(dtype, count, &one_gap);
+            lb = dtype_extent*displ + one_gap;
+            ub = lb + one_span;
+            min_lb = MIN( min_lb, lb );
+            max_ub = MAX( max_ub, ub );
+        }
+    }
+    if (max_ub > min_lb) {
+        *gap = min_lb;
+        *span = max_ub - min_lb;
+    }
+}
+
+/* Serialize the datatype into the buffer and return buffer length.
+   If buf is NULL, just return the length of the required buffer. */
+static size_t ddt_pack_datatype(opal_datatype_t* type, uint8_t* buf)
+{
+    size_t length = sizeof(opal_datatype_t) - offsetof(opal_datatype_t, flags);
+    size_t n_copy = length;
+    bool count_only = buf == NULL;
+    if (!count_only) {
+        memcpy(buf, &type->flags, length);
+    }
+    buf += length;
+
+    if( type->opt_desc.used ) {
+        /* we are losing the non optimized description of the datatype,
+         * but it is only useful for dumping the datatype description.
+         */
+        n_copy = (1 + type->opt_desc.used) * sizeof(dt_elem_desc_t);
+        if (!count_only) {
+            memcpy(buf, type->opt_desc.desc, n_copy);
+            buf += n_copy;
+        }
+        length += n_copy;
+    } else {
+        n_copy = (1 + type->desc.used) * sizeof(dt_elem_desc_t);
+        if (!count_only) {
+            memcpy(buf, type->desc.desc, n_copy);
+            buf += n_copy;
+        }
+        length += n_copy;
+    }
+    /* The following is not necessary and in fact, non-function in homogenous configurations.*/
+    // if (type->ptypes) {
+    //     n_copy = OPAL_DATATYPE_MAX_SUPPORTED * sizeof(size_t);
+    // } else {
+    //     n_copy = 0;
+    // }
+    // if (!count_only) {
+    //     memcpy(buf, type->ptypes, n_copy);
+    // }
+    // length += n_copy;
+    return length;
+}
+
+static size_t ddt_unpack_datatype(opal_datatype_t* type, uint8_t* buf)
+{
+    OBJ_CONSTRUCT(type, opal_datatype_t);
+    size_t length = sizeof(opal_datatype_t) - offsetof(opal_datatype_t, flags);
+    memcpy(&type->flags, buf, length);
+    buf += length;
+    size_t nbytes_copy = (1+type->opt_desc.used) * sizeof(dt_elem_desc_t);
+    type->opt_desc.desc = (dt_elem_desc_t*)malloc(nbytes_copy);
+    memcpy(type->opt_desc.desc, buf, nbytes_copy);
+    length += nbytes_copy;
+    type->desc = type->opt_desc;
+    buf += nbytes_copy;
+    type->ptypes = NULL;
+    return length;
+}
+
+/* Simple implementation: send all buffers without packing, but still keeping a
+   limited number in flight.
+
+      Note: CMA on XPMEM-mapped buffers does not work.  If the low-level network
+      provider attempts to use CMA to implement send/recv, then errors will
+      occur!
+*/
+static inline int alltoallv_sendrecv_w_direct_for_debugging(
+            void **send_from_addrs,
+            size_t *send_counts,
+            opal_datatype_t **send_types,
+            int jrank_sendto,
+            int ntypes_send,
+            void **recv_to_addrs,
+            size_t *recv_counts,
+            opal_datatype_t **recv_types,
+            int jrank_recvfrom,
+            int ntypes_recv,
+            struct ompi_communicator_t *comm) {
+
+
+    const int MAX_BUF_COUNT=8;
+    int nreqs = MAX_BUF_COUNT;
+    ompi_request_t *requests[MAX_BUF_COUNT];
+    const char* problem_hint;
+
+    int jfirst_sendreq = nreqs/2 + nreqs%2;
+
+    int jreq;
+    ompi_datatype_t *ompi_dtype_from_opal;
+    int rc = 0;
+    int jloop;
+
+    int jrecvs_posted = 0;
+    int jrecvs_completed = 0;
+    int jsends_posted = 0;
+    int jsends_completed = 0;
+
+    for (jloop = 0; ; jloop++) {
+        int have_completion;
+
+        if (jsends_completed == ntypes_send && jrecvs_completed == ntypes_recv )
+            break;
+
+        if (jloop < nreqs){
+            jreq = jloop;
+            have_completion = 0;
+            requests[jreq] = MPI_REQUEST_NULL;
+        } else {
+            have_completion = 1;
+            jreq = jloop%nreqs;
+            if (requests[jreq] == MPI_REQUEST_NULL) {
+                continue;
+            }
+            problem_hint = "waiting for request completion";
+            rc = ompi_request_wait(&requests[jreq], MPI_STATUS_IGNORE);
+            if (rc) break;
+            requests[jreq] = MPI_REQUEST_NULL;
+        }
+        int ii_send_req = jreq >= jfirst_sendreq;
+        if (have_completion) {
+            if (ii_send_req)
+                jsends_completed++;
+            else
+                jrecvs_completed++;
+        }
+
+        if (ii_send_req && jsends_posted < ntypes_send) {
+            problem_hint = "creating or committing temporary datatype";
+            rc = ompi_datatype_create_contiguous( 1, (ompi_datatype_t*)send_types[jsends_posted], &ompi_dtype_from_opal );
+            if (rc) break;
+            rc = ompi_datatype_commit(&ompi_dtype_from_opal);
+            if (rc) break;
+            problem_hint = "posting isend";
+            rc = MCA_PML_CALL(isend
+                (send_from_addrs[jsends_posted], (int)send_counts[jsends_posted], ompi_dtype_from_opal, jrank_sendto,
+                MCA_COLL_BASE_TAG_ALLTOALLV, MCA_PML_BASE_SEND_STANDARD,
+                comm, &requests[jreq]));
+            if (rc) break;
+            problem_hint = "destroying temporary datatype";
+            rc = ompi_datatype_destroy( &ompi_dtype_from_opal );
+            if (rc) break;
+
+            jsends_posted++;
+        }
+        if (!ii_send_req && jrecvs_posted < ntypes_recv ) {
+            problem_hint = "creating or committing temporary datatype";
+            rc = ompi_datatype_create_contiguous( 1, (ompi_datatype_t*)recv_types[jrecvs_posted], &ompi_dtype_from_opal );
+            if (rc) break;
+            rc = ompi_datatype_commit(&ompi_dtype_from_opal);
+            if (rc) break;
+            problem_hint = "posting irecv";
+            rc = MCA_PML_CALL(irecv
+                (recv_to_addrs[jrecvs_posted], (int)recv_counts[jrecvs_posted], ompi_dtype_from_opal, jrank_recvfrom,
+                MCA_COLL_BASE_TAG_ALLTOALLV,
+                comm, &requests[jreq]));
+            if (rc) break;
+            problem_hint = "destroying temporary datatype";
+            rc = ompi_datatype_destroy( &ompi_dtype_from_opal );
+            if (rc) break;
+            jrecvs_posted++;
+        }
+    }
+    if (rc) {
+        opal_output_verbose(1, mca_coll_han_component.han_output,
+            "Failed in alltoallv_sendrecv_w_direct_for_debugging while %s: jloop=%d, rc=%d\n",
+            problem_hint, jloop,rc);
+    }
+    return rc;
+}
+
+static int alltoallv_sendrecv_w(
+            void **send_from_addrs,
+            size_t *send_counts,
+            opal_datatype_t **send_types,
+            int jrank_sendto,
+            int ntypes_send,
+            void **recv_to_addrs,
+            size_t *recv_counts,
+            opal_datatype_t **recv_types,
+            int jrank_recvfrom,
+            int ntypes_recv,
+            struct ompi_communicator_t *comm) {
+
+    uint32_t iov_count = 1;
+    struct iovec iov;
+
+
+
+    const int MAX_BUF_COUNT=8;
+    ompi_request_t *requests[MAX_BUF_COUNT];
+    opal_free_list_item_t *buf_items[MAX_BUF_COUNT];
+
+    size_t buf_len = mca_coll_han_component.han_packbuf_bytes;
+    int nbufs = MAX_BUF_COUNT;
+    for (int jbuf=0; jbuf<nbufs; jbuf++) {
+        buf_items[jbuf] = opal_free_list_get(&mca_coll_han_component.pack_buffers);
+        if (buf_items[jbuf] == NULL) {
+            nbufs = jbuf;
+            opal_output_verbose(30, mca_coll_han_component.han_output,
+                "alltoallv_sendrecv_w: Number of buffers reduced to %d instead of %d.  "
+                "Check mca parameter coll_han_packbuf_max_count (currently %ld).\n",
+                nbufs, MAX_BUF_COUNT, mca_coll_han_component.han_packbuf_max_count);
+            break;
+        }
+    }
+
+    /* although we could feasibly disqualify the higher-level alltoallv
+    algorithm for these conditions, they represent invalid user parameters, so
+    let's fail and tell the user why.  */
+    if (nbufs < 2) {
+        opal_output_verbose(1, mca_coll_han_component.han_output,
+                    "ERROR: Need at least 2 buffers from HAN pack buffers!  "
+                    "Check mca parameter coll_han_packbuf_max_count (currently %ld)\n",
+                    mca_coll_han_component.han_packbuf_max_count);
+        return MPI_ERR_NO_MEM;
+    }
+    if (buf_len < 16) {
+        opal_output_verbose(1, mca_coll_han_component.han_output,
+                    "ERROR: Need a buffer that can hold at least 16 bytes!  "
+                    "Check mca parameter coll_han_packbuf_bytes (currently %ld)\n",
+                    mca_coll_han_component.han_packbuf_bytes);
+        return MPI_ERR_NO_MEM;
+    }
+
+    size_t nreqs = nbufs;
+    int jreq;
+    int jfirst_sendreq = nbufs/2 + nbufs%2;
+
+    size_t jloop = 0;
+    size_t send_pack_bytes_remaining = 0;
+    size_t recv_convertor_bytes_remaining = 0;
+    int have_completion = 0;
+    int jtype_send;
+    int jtype_recv;
+    size_t nbytes_pack;
+
+    const int buf_header_len = sizeof(int);
+    int nsend_req_pending = 0;
+    opal_convertor_t send_convertor;
+    opal_convertor_t recv_convertor;
+    OBJ_CONSTRUCT( &send_convertor, opal_convertor_t );
+    OBJ_CONSTRUCT( &recv_convertor, opal_convertor_t );
+
+    /* In the main loop, we will be posting recvs ahead of our recv convertors.
+       This gives us two options when it comes to the end of the transfer:
+        1) We could always pre-post data, then once our last convertor finds
+           there is no more data coming, we could cancel the recvs we posted
+        2) We could make a first-pass and count total bytes to recv, and be
+           careful not to post more than we know is coming.
+
+        After realizing that the convertor may decide to not fully pack the
+        buffer, path 1 was the only clear way forward (since it was no longer
+        clear how many bytes would be arriving.)
+
+        Note this is not necessary for Sends, because our convertor leads
+        posting the sends rather than trails it.
+    */
+   int ii_more_recvs_to_post = 0;
+   for (jtype_recv=0; jtype_recv<ntypes_recv; jtype_recv++) {
+        opal_convertor_copy_and_prepare_for_recv(ompi_mpi_local_convertor,
+            recv_types[jtype_recv],
+            recv_counts[jtype_recv],
+            recv_to_addrs[jtype_recv],
+            0,
+            &recv_convertor);
+        opal_convertor_get_packed_size( &recv_convertor, &recv_convertor_bytes_remaining );
+        if (recv_convertor_bytes_remaining) {
+            ii_more_recvs_to_post = 1;
+            break;
+        }
+   }
+
+   recv_convertor_bytes_remaining = 0;
+   send_pack_bytes_remaining = 0;
+
+    /*
+        Main loop.
+
+        First make a pass through all the request buffers, posting sends and
+        recvs.
+
+        After the first pass, we reach steady state: we call MPI_Waitany for
+        either a send or recv to complete, and then re-use that buffer and
+        that request to re-post the next send/recv.
+
+        We may exit the loop only after both sending and recving are done, which
+        means we must handle cases were there is no more send work or no more
+        recv work to do.
+    */
+    jtype_send = -1;
+    jtype_recv = -1;
+    int sequential_continues = 0;
+    for (jloop=0; ; jloop++) {
+        int ii_more_sends_to_post = jtype_send < ntypes_send || send_pack_bytes_remaining > 0;
+        int ii_more_sends_to_complete = nsend_req_pending > 0;
+
+        if ( !( ii_more_sends_to_post || ii_more_sends_to_complete ||
+                ii_more_recvs_to_post ) ) {
+            /* exit.  All done! */
+            break;
+        }
+
+        if (jloop >= nreqs) {
+            /* Common Case: */
+            /* wait for the send or recv to complete */
+            jreq = jloop%nreqs;
+            if (requests[jreq] == MPI_REQUEST_NULL) {
+                if (++sequential_continues > nbufs) {
+                    opal_output_verbose(1, mca_coll_han_component.han_output,
+                        "ERROR: no active requests to wait on! Loop=%ld: %d %d %d\n",
+                        jloop,
+                        ii_more_sends_to_post, ii_more_sends_to_complete,
+                        ii_more_recvs_to_post );
+                    return MPI_ERR_INTERN;
+                }
+                continue;
+            }
+            sequential_continues = 0;
+            ompi_request_wait( &requests[jreq], MPI_STATUS_IGNORE );
+            have_completion = 1;
+            requests[jreq] = MPI_REQUEST_NULL;
+        } else {
+            /* priming the loop: post sends or recvs while have_completion=0.
+
+            note: it is possible we have more buffers than data, so logic below
+            makes sure we don't infinite loop or send empty messages */
+            jreq = jloop;
+            have_completion = 0;
+        }
+        int ii_send_req = jreq >= jfirst_sendreq;
+        char *req_buf = buf_items[jreq]->ptr;
+
+        if (ii_send_req) {
+
+            if (have_completion) {
+                /* send request has completed */
+                nsend_req_pending--;
+            }
+
+            int buf_remain = buf_len - buf_header_len;
+            while (buf_remain > 0 && (jtype_send < ntypes_send || send_pack_bytes_remaining > 0) ) {
+                if (jtype_send < ntypes_send && send_pack_bytes_remaining==0) {
+                    /* switch to next datatype and prepare convertor: */
+                    jtype_send++;
+                    if (jtype_send < ntypes_send) {
+                        opal_convertor_copy_and_prepare_for_send(ompi_mpi_local_convertor,
+                            send_types[jtype_send],
+                            send_counts[jtype_send],
+                            send_from_addrs[jtype_send],
+                            0,
+                            &send_convertor);
+                        opal_convertor_get_packed_size( &send_convertor, &send_pack_bytes_remaining );
+                    }
+                }
+
+                /* pack more data */
+                if (send_pack_bytes_remaining > 0 && buf_remain > 0) {
+                    /* pack into the buffer described by the iov */
+                    /* iovec: set the destination of the copy */
+                    size_t start_offset = buf_len - buf_remain;
+                    iov.iov_base = (char*)(req_buf) + start_offset;
+                    iov.iov_len = buf_remain;
+                    iov_count = 1;
+                    opal_convertor_pack(&send_convertor, &iov, &iov_count, &nbytes_pack);
+                    if (!nbytes_pack) {
+                        /* can happen when buffer is only a few bytes from full,
+                        and convertor doesn't want to split a primitive data
+                        type. */
+                        break;
+                    }
+                    buf_remain -= nbytes_pack;
+                    send_pack_bytes_remaining -= nbytes_pack;
+                }
+            }
+            /* start the buffer with an integer describing how many bytes we
+            packed, *including* the integer's size */
+            int pack_bytes_len = buf_len - buf_remain;
+
+            /* post send */
+            if (pack_bytes_len > buf_header_len) {
+                *((int*)req_buf) = pack_bytes_len;
+                MCA_PML_CALL(isend
+                    (req_buf, pack_bytes_len, MPI_PACKED, jrank_sendto,
+                    MCA_COLL_BASE_TAG_ALLTOALLV, MCA_PML_BASE_SEND_STANDARD,
+                    comm, &requests[jreq]));
+                nsend_req_pending++;
+            } else {
+                requests[jreq] = MPI_REQUEST_NULL;
+            }
+
+        } else { /* recv request */
+            if (have_completion) {
+                /* unpack data */
+                ssize_t buf_remain = *((int*)req_buf) - buf_header_len;
+                size_t buf_converted = buf_header_len;
+                while (true) {
+                    if (jtype_recv < ntypes_recv && recv_convertor_bytes_remaining==0) {
+                        /* switch to next datatype and prepare convertor: */
+                        jtype_recv++;
+                        if (jtype_recv < ntypes_recv) {
+                            opal_convertor_copy_and_prepare_for_recv(ompi_mpi_local_convertor,
+                                recv_types[jtype_recv],
+                                recv_counts[jtype_recv],
+                                recv_to_addrs[jtype_recv],
+                                0,
+                                &recv_convertor);
+                            opal_convertor_get_packed_size( &recv_convertor, &recv_convertor_bytes_remaining );
+                            if (recv_convertor_bytes_remaining == 0) {
+                                continue;
+                            }
+                        }
+                    }
+                    if (jtype_recv == ntypes_recv && recv_convertor_bytes_remaining==0 ) {
+                        /* _all_ recving work is done! */
+                        ii_more_recvs_to_post = false;
+                        buf_remain = 0;
+                        for (int kreq=0; kreq<jfirst_sendreq; kreq++) {
+                            if (requests[kreq] != MPI_REQUEST_NULL) {
+                                ompi_request_cancel(requests[kreq]);
+                                requests[kreq] = MPI_REQUEST_NULL;
+                            }
+                        }
+                    }
+                    if (buf_remain == 0) { break; }
+
+                    /* unpack more data */
+                    if (recv_convertor_bytes_remaining > 0) {
+                        /* unpack from the buffer described by the iov */
+                        iov.iov_base = (char*)(req_buf) + buf_converted;
+                        iov.iov_len = buf_remain;
+                        iov_count = 1;
+                        opal_convertor_unpack(&recv_convertor, &iov, &iov_count, &nbytes_pack);
+
+                        buf_remain -= nbytes_pack;
+                        buf_converted += nbytes_pack;
+                        recv_convertor_bytes_remaining -= nbytes_pack;
+                    }
+                }
+            }
+
+            if (ii_more_recvs_to_post) {
+                /* post a new recv */
+                MCA_PML_CALL(irecv
+                    (req_buf, buf_len, MPI_PACKED, jrank_recvfrom,
+                    MCA_COLL_BASE_TAG_ALLTOALLV,
+                    comm, &requests[jreq]));
+            } else {
+                requests[jreq] = MPI_REQUEST_NULL;
+            }
+        }
+    }
+
+    OBJ_DESTRUCT(&send_convertor);
+    OBJ_DESTRUCT(&recv_convertor);
+
+    for (int jbuf=0; jbuf<nbufs; jbuf++) {
+        opal_free_list_return(&mca_coll_han_component.pack_buffers, buf_items[jbuf]);
+    }
+    return 0;
+}
+
+static int decide_to_use_smsc_alg(
+    int *use_smsc,
+    const void *sbuf,
+    ompi_count_array_t scounts,
+    ompi_disp_array_t sdispls,
+    struct ompi_datatype_t *sdtype,
+    void* rbuf,
+    ompi_count_array_t rcounts,
+    ompi_disp_array_t rdispls,
+    struct ompi_datatype_t *rdtype,
+    struct ompi_communicator_t *comm)
+{
+    opal_convertor_t convertor;
+    long long reduce_buf_input[3];
+    long long reduce_buf_output[3];
+    int bufs_on_device = 0;
+    size_t avg_send_bytes = 0;
+    size_t packed_size_bytes = 0;
+    int need_bufs = 0;
+    int rc;
+    int comm_size = ompi_comm_size(comm);
+    int comm_rank = ompi_comm_rank(comm);
+
+    /*
+    Perform an allreduce on all ranks to decide if this algorithm is worth
+    using. There are four important things:
+
+     1. sbuf == MPI_IN_PLACE.  This algorithm does not currently support the
+        in-place operation.  (Future developers may note that the inter-node
+        communications are ordered such that in-place could be supported, but
+        additional ordering and/or buffering would be required to ensure local
+        ranks do not overwrite buffers before sends are posted. However, for now
+        we just bail out.)
+     2. Device buffers.  XPMEM doesn't support GPU buffers, so we cannot proceed
+        with this algorithm.
+     3. Send size per rank.  This algorithm can pack small messages together,
+        but this typically isn't helpful for large messages, and XPMEM-mapped
+        memory cannot be registered with ibv_reg_mr.
+     4. Contiguous buffers.  The exception to #2 above is if we can't post our
+        sends/recvs in one large block to the NIC.  For these non-contiguous
+        datatypes, our packing algorithm is better because (a) we re-use our
+        buffers from a free-list which can remain registered to the NIC, and (b)
+        we pipeline multiple sends at once (c) we can pack more data into each
+        send than the network layer because we have mmap'ed all local ranks and
+        understand their datatypes.
+
+    The allreduce adds about 50 usec on 256 ranks across 4 nodes, but for large
+    message sizes this is insignificant, and for small message sizes it is <1/4
+    our execution time, which is <1/10 of the "basic" algorithm.
+    */
+
+   if (sbuf == MPI_IN_PLACE) {
+        if (comm_rank == 0) {
+            opal_output_verbose(10, mca_coll_han_component.han_output, "alltoallv: decide_to_use_smsc_alg: "
+                "MPI_IN_PLACE operation prevents smsc_alg from being used.  "
+                "Continue with SMSC? ==> no.\n");
+        }
+        *use_smsc = 0;
+   }
+
+    /* some magic in the count: if we pick 1, need_buffers() might not be
+    accurate.  We could be precisely correct and compute need_buffers for every
+    rank's count, but that could be a lot of iteration.  Just use 2 and assume
+    some rank will be sending more than 1 item, and adjust packed size
+    accordingly. */
+    const int count_for_convertor = 2;
+
+    OBJ_CONSTRUCT( &convertor, opal_convertor_t );
+    rc = opal_convertor_copy_and_prepare_for_recv(ompi_mpi_local_convertor,
+        &rdtype->super, count_for_convertor, rbuf, 0, &convertor);
+    if (rc) goto cleanup1;
+    bufs_on_device = opal_convertor_on_device(&convertor);
+    need_bufs = opal_convertor_need_buffers(&convertor);
+    rc = opal_convertor_cleanup(&convertor);
+    if (rc) goto cleanup1;
+    rc = opal_convertor_copy_and_prepare_for_send(ompi_mpi_local_convertor,
+        &sdtype->super, count_for_convertor, sbuf, 0, &convertor);
+    if (rc) goto cleanup1;
+    bufs_on_device |= opal_convertor_on_device(&convertor);
+    opal_convertor_get_packed_size(&convertor, &packed_size_bytes);
+    for (int jrank=0; jrank<comm_size; jrank++) {
+        avg_send_bytes += packed_size_bytes/count_for_convertor * ompi_count_array_get(scounts,jrank);
+    }
+    need_bufs |= opal_convertor_need_buffers(&convertor);
+
+    rc = opal_convertor_cleanup(&convertor);
+cleanup1:
+    OBJ_DESTRUCT( &convertor );
+    if (rc != OMPI_SUCCESS) { return rc;}
+
+    avg_send_bytes = avg_send_bytes / comm_size;
+    reduce_buf_input[0] = !!(bufs_on_device);
+    reduce_buf_input[1] = avg_send_bytes;
+    reduce_buf_input[2] = !!(need_bufs);
+    rc =comm->c_coll->coll_allreduce(
+        &reduce_buf_input, &reduce_buf_output, 3, MPI_LONG_LONG, &ompi_mpi_op_sum.op,
+        comm, comm->c_coll->coll_allreduce_module );
+    if (rc != OMPI_SUCCESS) {return rc;}
+
+    if (reduce_buf_output[0] > 0) {
+        *use_smsc = 0;
+        /* can't proceed: at least one rank using GPU buffers. */
+    } else if (reduce_buf_output[2] >= comm_size * mca_coll_han_component.han_alltoallv_smsc_noncontig_activation_limit) {
+        /* always proceed: enough ranks have non-contiguous data that the pack-and-send method is fastest. */
+        *use_smsc = 1;
+    } else if (reduce_buf_output[1] >= comm_size * mca_coll_han_component.han_alltoallv_smsc_avg_send_limit) {
+        *use_smsc = 0;
+        /* don't proceed: messages are large and contiguous.  It is faster to fall back to basic alg. */
+    } else {
+        *use_smsc = 1;
+    }
+
+    if (comm_rank == 0) {
+        opal_output_verbose(10, mca_coll_han_component.han_output, "alltoallv: decide_to_use_smsc_alg: "
+            "Ranks with GPU buffers: %lld (limit is 0).  "
+            "Average send_size: %.1f bytes (limit is %ld).  "
+            "Fraction with non-contiguous buffers: %.3f (activation limit: %.3f).  "
+            "Continue with SMSC? ==>%s.\n",
+            reduce_buf_output[0],
+            ((double)reduce_buf_output[1])/comm_size, mca_coll_han_component.han_alltoallv_smsc_avg_send_limit,
+            ((double)reduce_buf_output[2])/comm_size, mca_coll_han_component.han_alltoallv_smsc_noncontig_activation_limit,
+            (*use_smsc) ? "Yes" : "No");
+    }
+    return rc;
+}
+
+int mca_coll_han_alltoallv_using_smsc(
+        const void *sbuf,
+        ompi_count_array_t scounts,
+        ompi_disp_array_t sdispls,
+        struct ompi_datatype_t *sdtype,
+        void* rbuf,
+        ompi_count_array_t rcounts,
+        ompi_disp_array_t rdispls,
+        struct ompi_datatype_t *rdtype,
+        struct ompi_communicator_t *comm,
+        mca_coll_base_module_t *module)
+{
+    int rc;
+    void **send_from_addrs = NULL;
+    void **recv_to_addrs = NULL;
+    size_t *send_counts = NULL;
+    size_t *recv_counts = NULL;
+    opal_datatype_t **send_types = NULL;
+    opal_datatype_t **recv_types = NULL;
+    mca_coll_han_module_t *han_module = (mca_coll_han_module_t *)module;
+
+    OPAL_OUTPUT_VERBOSE((90, mca_coll_han_component.han_output,
+                            "Entering mca_coll_han_alltoall_using_smsc\n"));
+
+    if (!mca_smsc || !mca_smsc_base_has_feature(MCA_SMSC_FEATURE_CAN_MAP)) {
+        /* Assume all hosts take this path together :-\ */
+        opal_output_verbose(1, mca_coll_han_component.han_output, "in mca_coll_han_alltoallv_using_smsc, "
+            "but MCA_SMSC_FEATURE_CAN_MAP not available.  Disqualifying this alg!\n");
+        HAN_UNINSTALL_COLL_API(comm, han_module, alltoallv);
+        return han_module->previous_alltoallv(sbuf, scounts, sdispls, sdtype, rbuf, rcounts, rdispls, rdtype,
+                                             comm, han_module->previous_alltoallv_module);
+    }
+
+    /* Create the subcommunicators */
+    if( OMPI_SUCCESS != mca_coll_han_comm_create_new(comm, han_module) ) {
+        opal_output_verbose(1, mca_coll_han_component.han_output,
+                             "han cannot handle alltoallv with this communicator. Fall back on another component\n");
+        /* HAN cannot work with this communicator so fallback on all collectives */
+        HAN_LOAD_FALLBACK_COLLECTIVES(comm, han_module);
+        return han_module->previous_alltoallv(sbuf, scounts, sdispls, sdtype, rbuf, rcounts, rdispls, rdtype,
+                                          comm, han_module->previous_alltoallv_module);
+    }
+
+    /* Topo must be initialized to know rank distribution which then is used to
+     * determine if han can be used */
+    mca_coll_han_topo_init(comm, han_module, 2);
+    if (han_module->are_ppn_imbalanced || !han_module->is_mapbycore){
+        opal_output_verbose(1, mca_coll_han_component.han_output,
+                             "han cannot handle alltoallv with this communicator (imbalance/!mapbycore).  "
+                             "Fall back on another component\n");
+        /* Put back the fallback collective support and call it once. All
+         * future calls will then be automatically redirected.
+         */
+        HAN_UNINSTALL_COLL_API(comm, han_module, alltoallv);
+        return han_module->previous_alltoallv(sbuf, scounts, sdispls, sdtype, rbuf, rcounts, rdispls, rdtype,
+                                             comm, han_module->previous_alltoallv_module);
+    }
+
+    int w_size = ompi_comm_size(comm);
+
+    int use_smsc;
+    rc = decide_to_use_smsc_alg(&use_smsc,
+        sbuf, scounts, sdispls, sdtype, rbuf, rcounts, rdispls, rdtype, comm);
+    if (rc != 0) {
+        opal_output_verbose(1, mca_coll_han_component.han_output,
+            "decide_to_use_smsc_alg failed during execution! rc=%d\n", rc);
+    }
+    if (!use_smsc) {
+        return han_module->previous_alltoallv(sbuf, scounts, sdispls, sdtype, rbuf, rcounts, rdispls, rdtype,
+                                             comm, han_module->previous_alltoallv_module);
+    }
+
+    ompi_communicator_t *low_comm = han_module->sub_comm[INTRA_NODE];
+    ompi_communicator_t *up_comm = han_module->sub_comm[INTER_NODE];
+
+    /* information about sub-communicators */
+    int low_rank = ompi_comm_rank(low_comm);
+    int low_size = ompi_comm_size(low_comm);
+    int up_size = ompi_comm_size(up_comm);
+    int up_rank = ompi_comm_rank(up_comm);
+
+    struct gathered_data low_gather_in;
+    struct gathered_data *low_gather_out;
+
+
+    low_gather_in.stype_serialized_length = ddt_pack_datatype(&sdtype->super, NULL);
+    uint8_t *serialization_buf;
+
+    size_t serialization_buf_length = low_gather_in.stype_serialized_length
+        + sizeof(struct peer_counts)*w_size;
+
+    /* allocate data */
+    serialization_buf = malloc(serialization_buf_length);
+    low_gather_out = malloc(sizeof(*low_gather_out) * low_size);
+    struct peer_data *peers = malloc(sizeof(*peers) * low_size);
+    opal_datatype_t *peer_send_types = malloc(sizeof(*peer_send_types) * low_size);
+    bool have_bufs_and_types = false;
+
+    low_gather_in.serialization_buffer = serialization_buf;
+    low_gather_in.sbuf = (void*)sbuf; // cast to discard the const
+
+    ptrdiff_t r_extent, r_lb;
+    rc = ompi_datatype_get_extent( rdtype, &r_lb, &r_extent);
+
+    /* calculate the full gap and span of all accesses to our buffer: */
+    coll_han_alltoallv_calc_all_span( w_size, &sdtype->super, scounts, sdispls,
+                &low_gather_in.sbuf_gap, &low_gather_in.sbuf_span );
+
+    /* pack the serialization buffer: first the array of counts */
+    size_t buf_packed = 0;
+    for (int jrankw=0; jrankw<w_size; jrankw++) {
+        struct peer_counts *counts = (void*)(serialization_buf + buf_packed);
+        counts->scount = ompi_count_array_get(scounts,jrankw);
+        counts->rcount = ompi_count_array_get(rcounts,jrankw);
+
+        counts->sdispl = ompi_disp_array_get(sdispls,jrankw);
+        counts->rdispl = ompi_disp_array_get(rdispls,jrankw);
+        buf_packed += sizeof(*counts);
+    }
+    /* pack the serialization buffer: next the send and recv dtypes */
+    buf_packed += ddt_pack_datatype(&sdtype->super, serialization_buf + buf_packed);
+    assert(buf_packed == serialization_buf_length);
+
+    rc = low_comm->c_coll->coll_allgather(&low_gather_in, sizeof(low_gather_in), MPI_BYTE,
+            low_gather_out, sizeof(low_gather_in), MPI_BYTE, low_comm,
+            low_comm->c_coll->coll_allgather_module);
+    if (rc != 0) {
+        opal_output_verbose(1, mca_coll_han_component.han_output,
+            "During mca_coll_han_alltoallv_using_smsc: Allgather failed with rc=%d\n",rc);
+        goto cleanup;
+    }
+
+    /*
+    In this loop we unpack the data received in allgather:
+      - use SMSC to memory-map the serialization buffer
+        - construct dtype objects from the serialization buffer
+        - set pointers to read counts and displacements directly from serialization buffer
+      - Memory-map the send-bufs.
+
+    */
+    for (int jrank=0; jrank<low_size; jrank++) {
+        void *tmp_ptr;
+        peers[jrank].map_ctx[0] = NULL;
+        peers[jrank].map_ctx[1] = NULL;
+
+        if (jrank == low_rank) {
+            /* special case for ourself */
+            peers[jrank].counts = (struct peer_counts *)serialization_buf;
+            peers[jrank].sbuf   = sbuf;
+            peers[jrank].sendtype = &sdtype->super;
+            continue;
+        }
+
+        struct gathered_data *gathered = &low_gather_out[jrank];
+        struct ompi_proc_t* ompi_proc = ompi_comm_peer_lookup(low_comm, jrank);
+        mca_smsc_endpoint_t *smsc_ep;
+        smsc_ep = mca_coll_han_get_smsc_endpoint(ompi_proc);
+
+        uint8_t *peer_serialization_buf;
+        size_t peer_serialization_buf_length;
+        peer_serialization_buf_length = w_size * sizeof(struct peer_counts);
+        peer_serialization_buf_length += gathered->stype_serialized_length;
+
+        /* mmap the buffers */
+        peers[jrank].map_ctx[0] = mca_smsc->map_peer_region(
+                smsc_ep,
+                MCA_RCACHE_FLAGS_PERSIST,
+                gathered->serialization_buffer,
+                peer_serialization_buf_length,
+                (void**) &peer_serialization_buf );
+        if (gathered->sbuf_span > 0) {
+            peers[jrank].map_ctx[1] = mca_smsc->map_peer_region(
+                    smsc_ep,
+                    MCA_RCACHE_FLAGS_PERSIST,
+                    (char*)gathered->sbuf + gathered->sbuf_gap,
+                    gathered->sbuf_span,
+                    &tmp_ptr );
+            peers[jrank].sbuf = (char*)tmp_ptr - gathered->sbuf_gap;
+        }
+
+        /* point the counts pointer into the mmapped serialization buffer */
+        peers[jrank].counts = (struct peer_counts*)peer_serialization_buf;
+        peer_serialization_buf += w_size * sizeof(struct peer_counts);
+
+        /* unpack the dtypes */
+        peer_serialization_buf += ddt_unpack_datatype( &peer_send_types[jrank], peer_serialization_buf);
+        peers[jrank].sendtype = &peer_send_types[jrank];
+    }
+
+    have_bufs_and_types = true;
+    send_from_addrs = malloc(sizeof(*send_from_addrs)*low_size);
+    recv_to_addrs = malloc(sizeof(*recv_to_addrs)*low_size);
+    send_counts = malloc(sizeof(*send_counts)*low_size);
+    recv_counts = malloc(sizeof(*recv_counts)*low_size);
+    send_types = malloc(sizeof(*send_types)*low_size);
+    recv_types = malloc(sizeof(*recv_types)*low_size);
+
+    /****
+     *  Main exchange loop
+     ****/
+    int nloops = up_size;
+    for (int jloop=0; jloop<nloops; jloop++) {
+
+        int up_partner = ring_partner(up_rank, jloop, up_size);
+        int jrank_sendto = up_partner*low_size + low_rank;
+        int jrank_recvfrom = jrank_sendto;
+
+        for (int jlow=0; jlow<low_size; jlow++) {
+            int remote_wrank = up_partner*low_size + jlow;
+            ptrdiff_t peer_sextent;
+
+            rc = opal_datatype_type_extent( peers[jlow].sendtype, &peer_sextent);
+            if (rc != 0) {
+                opal_output_verbose(1, mca_coll_han_component.han_output,
+                    "opal_datatype_type_extent returned error code = %d during mca_coll_han_alltoallv_using_smsc!\n",rc);
+                goto cleanup;
+            }
+            void *from_addr = (uint8_t*)peers[jlow].sbuf + peers[jlow].counts[jrank_sendto].sdispl*peer_sextent;
+
+            send_from_addrs[jlow] = from_addr;
+            send_counts[jlow] = peers[jlow].counts[jrank_sendto].scount;
+            send_types[jlow] = peers[jlow].sendtype;
+
+
+            recv_to_addrs[jlow] = (uint8_t*)rbuf + ompi_disp_array_get(rdispls,remote_wrank)*r_extent;
+            recv_counts[jlow] = ompi_count_array_get(rcounts,remote_wrank);
+            recv_types[jlow] = &(rdtype->super);
+        }
+
+        int ntypes_send = low_size;
+        int ntypes_recv = low_size;
+
+        rc = alltoallv_sendrecv_w(
+            send_from_addrs, send_counts, send_types, jrank_sendto, ntypes_send,
+            recv_to_addrs, recv_counts, recv_types, jrank_recvfrom, ntypes_recv,
+            comm);
+        if (rc != 0) {
+            opal_output_verbose(1, mca_coll_han_component.han_output,
+                "alltoallv_sendrecv_w returned error code = %d!\n",rc);
+            goto cleanup;
+        }
+
+
+    }
+    rc=0;
+
+    cleanup:
+    low_comm->c_coll->coll_barrier(low_comm, low_comm->c_coll->coll_barrier_module);
+
+    if (send_from_addrs) {
+        free(send_from_addrs);
+        free(recv_to_addrs);
+        free(send_counts);
+        free(recv_counts);
+        free(send_types);
+        free(recv_types);
+    }
+
+    if (have_bufs_and_types) {
+        for (int jlow=0; jlow<low_size; jlow++) {
+            if (jlow != low_rank) {
+                OBJ_DESTRUCT(&peer_send_types[jlow]);
+            }
+
+            for (int jbuf=0; jbuf<2; jbuf++) {
+                if (peers[jlow].map_ctx[jbuf]) {
+                    mca_smsc->unmap_peer_region(peers[jlow].map_ctx[jbuf]);
+                }
+            }
+        }
+    }
+    free(serialization_buf);
+    free(low_gather_out);
+    free(peers);
+    free(peer_send_types);
+
+    OPAL_OUTPUT_VERBOSE((40, mca_coll_han_component.han_output,
+                "Alltoall Complete with %d\n",rc));
+    return rc;
+
+}
diff --git a/ompi/mca/coll/han/coll_han_barrier.c b/ompi/mca/coll/han/coll_han_barrier.c
index 01a930692fe..19b0e38c9a7 100644
--- a/ompi/mca/coll/han/coll_han_barrier.c
+++ b/ompi/mca/coll/han/coll_han_barrier.c
@@ -1,8 +1,9 @@
 /*
- * Copyright (c) 2018-2020 The University of Tennessee and The University
+ * Copyright (c) 2018-2023 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2020      Bull S.A.S. All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -36,11 +37,9 @@ mca_coll_han_barrier_intra_simple(struct ompi_communicator_t *comm,
     if( OMPI_SUCCESS != mca_coll_han_comm_create_new(comm, han_module) ) {
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "han cannot handle barrier with this communicator. Fall back on another component\n"));
-        /* Put back the fallback collective support and call it once. All
-         * future calls will then be automatically redirected.
-         */
-        HAN_LOAD_FALLBACK_COLLECTIVES(han_module, comm);
-        return comm->c_coll->coll_barrier(comm, comm->c_coll->coll_bcast_module);
+        /* Try to put back the fallback collective support and call it once.  */
+        HAN_LOAD_FALLBACK_COLLECTIVES(comm, han_module);
+        return han_module->previous_barrier(comm, han_module->previous_barrier_module);
     }
 
     low_comm = han_module->sub_comm[INTRA_NODE];
diff --git a/ompi/mca/coll/han/coll_han_bcast.c b/ompi/mca/coll/han/coll_han_bcast.c
index 36bd42cb378..f75cab4fc69 100644
--- a/ompi/mca/coll/han/coll_han_bcast.c
+++ b/ompi/mca/coll/han/coll_han_bcast.c
@@ -1,9 +1,10 @@
 /*
- * Copyright (c) 2018-2020 The University of Tennessee and The University
+ * Copyright (c) 2018-2023 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2020      Bull S.A.S. All rights reserved.
  * Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -51,7 +52,7 @@ mca_coll_han_set_bcast_args(mca_coll_han_bcast_args_t * args, mca_coll_task_t *
 }
 
 /*
- * Each segment of the messsage needs to go though 2 steps to perform MPI_Bcast:
+ * Each segment of the message needs to go though 2 steps to perform MPI_Bcast:
  *     ub: upper level (inter-node) bcast
  *     lb: low level (shared-memory or intra-node) bcast.
  * Hence, in each iteration, there is a combination of collective operations which is called a task.
@@ -63,8 +64,8 @@ mca_coll_han_set_bcast_args(mca_coll_han_bcast_args_t * args, mca_coll_task_t *
  * iter 4 |       |       |       |  lb   | task: t1, contains lb
  */
 int
-mca_coll_han_bcast_intra(void *buff,
-                         int count,
+mca_coll_han_bcast_intra(void *buf,
+                         size_t count,
                          struct ompi_datatype_t *dtype,
                          int root,
                          struct ompi_communicator_t *comm, mca_coll_base_module_t * module)
@@ -83,9 +84,9 @@ mca_coll_han_bcast_intra(void *buff,
         /* Put back the fallback collective support and call it once. All
          * future calls will then be automatically redirected.
          */
-        HAN_LOAD_FALLBACK_COLLECTIVES(han_module, comm);
-        return comm->c_coll->coll_bcast(buff, count, dtype, root,
-                                        comm, comm->c_coll->coll_bcast_module);
+        HAN_LOAD_FALLBACK_COLLECTIVES(comm, han_module);
+        return han_module->previous_bcast(buf, count, dtype, root,
+                                          comm, han_module->previous_bcast_module);
     }
     /* Topo must be initialized to know rank distribution which then is used to
      * determine if han can be used */
@@ -96,9 +97,9 @@ mca_coll_han_bcast_intra(void *buff,
         /* Put back the fallback collective support and call it once. All
          * future calls will then be automatically redirected.
          */
-        HAN_LOAD_FALLBACK_COLLECTIVE(han_module, comm, bcast);
-        return comm->c_coll->coll_bcast(buff, count, dtype, root,
-                                        comm, comm->c_coll->coll_bcast_module);
+        HAN_UNINSTALL_COLL_API(comm, han_module, bcast);
+        return han_module->previous_bcast(buf, count, dtype, root,
+                                          comm, han_module->previous_bcast_module);
     }
 
     ompi_datatype_get_extent(dtype, &lb, &extent);
@@ -112,7 +113,7 @@ mca_coll_han_bcast_intra(void *buff,
 
     int num_segments = (count + seg_count - 1) / seg_count;
     OPAL_OUTPUT_VERBOSE((20, mca_coll_han_component.han_output,
-                         "In HAN seg_count %d count %d num_seg %d\n",
+                         "In HAN seg_count %d count %zu num_seg %d\n",
                          seg_count, count, num_segments));
 
     int *vranks = han_module->cached_vranks;
@@ -129,7 +130,7 @@ mca_coll_han_bcast_intra(void *buff,
     mca_coll_task_t *t0 = OBJ_NEW(mca_coll_task_t);
     /* Setup up t0 task arguments */
     mca_coll_han_bcast_args_t *t = malloc(sizeof(mca_coll_han_bcast_args_t));
-    mca_coll_han_set_bcast_args(t, t0, (char *) buff, seg_count, dtype,
+    mca_coll_han_set_bcast_args(t, t0, (char *)buf, seg_count, dtype,
                                 root_up_rank, root_low_rank, up_comm, low_comm,
                                 num_segments, 0, w_rank, count - (num_segments - 1) * seg_count,
                                 low_rank != root_low_rank);
@@ -222,8 +223,8 @@ int mca_coll_han_bcast_t1_task(void *task_args)
  * communications without tasks.
  */
 int
-mca_coll_han_bcast_intra_simple(void *buff,
-                                int count,
+mca_coll_han_bcast_intra_simple(void *buf,
+                                size_t count,
                                 struct ompi_datatype_t *dtype,
                                 int root,
                                 struct ompi_communicator_t *comm,
@@ -245,9 +246,9 @@ mca_coll_han_bcast_intra_simple(void *buff,
         /* Put back the fallback collective support and call it once. All
          * future calls will then be automatically redirected.
          */
-        HAN_LOAD_FALLBACK_COLLECTIVES(han_module, comm);
-        return comm->c_coll->coll_bcast(buff, count, dtype, root,
-                                        comm, comm->c_coll->coll_bcast_module);
+        HAN_LOAD_FALLBACK_COLLECTIVES(comm, han_module);
+        return han_module->previous_bcast(buf, count, dtype, root,
+                                          comm, han_module->previous_bcast_module);
     }
     /* Topo must be initialized to know rank distribution which then is used to
      * determine if han can be used */
@@ -258,9 +259,9 @@ mca_coll_han_bcast_intra_simple(void *buff,
         /* Put back the fallback collective support and call it once. All
          * future calls will then be automatically redirected.
          */
-        HAN_LOAD_FALLBACK_COLLECTIVE(han_module, comm, bcast);
-        return comm->c_coll->coll_bcast(buff, count, dtype, root,
-                                        comm, comm->c_coll->coll_bcast_module);
+        HAN_UNINSTALL_COLL_API(comm, han_module, bcast);
+        return han_module->previous_bcast(buf, count, dtype, root,
+                                          comm, han_module->previous_bcast_module);
     }
 
     low_comm = han_module->sub_comm[INTRA_NODE];
@@ -277,18 +278,18 @@ mca_coll_han_bcast_intra_simple(void *buff,
                          w_rank, root_low_rank, root_up_rank));
 
     if (low_rank == root_low_rank) {
-        up_comm->c_coll->coll_bcast(buff, count, dtype, root_up_rank,
+        up_comm->c_coll->coll_bcast(buf, count, dtype, root_up_rank,
                                     up_comm, up_comm->c_coll->coll_bcast_module);
 
         /* To remove when han has better sub-module selection.
            For now switching to ibcast enables to make runs with libnbc. */
         //ompi_request_t req;
-        //up_comm->c_coll->coll_ibcast(buff, count, dtype, root_up_rank,
+        //up_comm->c_coll->coll_ibcast(buf, count, dtype, root_up_rank,
         //                             up_comm, &req, up_comm->c_coll->coll_ibcast_module);
         //ompi_request_wait(&req, MPI_STATUS_IGNORE);
 
     }
-    low_comm->c_coll->coll_bcast(buff, count, dtype, root_low_rank,
+    low_comm->c_coll->coll_bcast(buf, count, dtype, root_low_rank,
                                  low_comm, low_comm->c_coll->coll_bcast_module);
 
     return OMPI_SUCCESS;
diff --git a/ompi/mca/coll/han/coll_han_component.c b/ompi/mca/coll/han/coll_han_component.c
index b38891eef10..a8a649dd080 100644
--- a/ompi/mca/coll/han/coll_han_component.c
+++ b/ompi/mca/coll/han/coll_han_component.c
@@ -2,7 +2,12 @@
  * Copyright (c) 2018-2020 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
- * Copyright (c) 2020      Bull S.A.S. All rights reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
+ * Copyright (c) 2020-2022 Bull S.A.S. All rights reserved.
+ * Copyright (c) 2023      Computer Architecture and VLSI Systems (CARV)
+ *                         Laboratory, ICS Forth. All rights reserved.
+ * Copyright (c) 2024      Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * Copyright (c) 2024      NVIDIA CORPORATION. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -20,11 +25,13 @@
 #include "ompi_config.h"
 
 #include "opal/util/show_help.h"
+#include "opal/util/argv.h"
 #include "ompi/constants.h"
 #include "ompi/mca/coll/coll.h"
 #include "coll_han.h"
 #include "coll_han_dynamic.h"
 #include "coll_han_dynamic_file.h"
+#include "coll_han_algorithms.h"
 #include "ompi/mca/coll/base/coll_base_util.h"
 
 /*
@@ -33,16 +40,22 @@
 const char *mca_coll_han_component_version_string =
     "Open MPI HAN collective MCA component version " OMPI_VERSION;
 
-ompi_coll_han_components available_components[COMPONENTS_COUNT] = {
-    { SELF, "self",  NULL },
-    { BASIC, "basic", NULL },
-    { LIBNBC, "libnbc", NULL },
-    { TUNED, "tuned", NULL },
-    { SM, "sm", NULL },
-    { ADAPT, "adapt", NULL },
-    { HAN, "han", NULL }
+ompi_coll_han_components ompi_coll_han_available_components[COMPONENTS_COUNT] = {
+    { SELF, "self" },
+    { BASIC, "basic" },
+    { LIBNBC, "libnbc" },
+    { TUNED, "tuned" },
+    { SM, "sm" },  /* this should not be used, the collective component is gone */
+    { ADAPT, "adapt" },
+    { HAN, "han" },
+    { XHC, "xhc" }
 };
 
+/*
+ * Thread lock for han
+ */
+opal_mutex_t mca_coll_han_lock = OPAL_MUTEX_STATIC_INIT;
+
 /*
  * Local functions
  */
@@ -62,7 +75,7 @@ mca_coll_han_component_t mca_coll_han_component = {
            information about the component itself */
 
         .collm_version = {
-            MCA_COLL_BASE_VERSION_2_4_0,
+            MCA_COLL_BASE_VERSION_3_0_0,
 
             /* Component name and version */
             .mca_component_name = "han",
@@ -84,10 +97,12 @@ mca_coll_han_component_t mca_coll_han_component = {
         .collm_comm_query = mca_coll_han_comm_query,
     },
 
-    /* han-component specifc information */
+    /* han-component specific information */
 
     /* (default) priority */
-    20,
+    .han_priority = 35,
+    /* workaround for nvcc compiler */
+    .dynamic_rules_filename = NULL,
 };
 
 /*
@@ -96,18 +111,84 @@ mca_coll_han_component_t mca_coll_han_component = {
 static int han_open(void)
 {
     /* Get the global coll verbosity: it will be ours */
-    mca_coll_han_component.han_output = ompi_coll_base_framework.framework_output;
+    if (mca_coll_han_component.han_output_verbose) {
+        mca_coll_han_component.han_output = opal_output_open(NULL);
+        opal_output_set_verbosity(mca_coll_han_component.han_output,
+                                  mca_coll_han_component.han_output_verbose);
+    } else {
+        mca_coll_han_component.han_output = ompi_coll_base_framework.framework_output;
+    }
+
+    OBJ_CONSTRUCT(&mca_coll_han_component.pack_buffers, opal_free_list_t);
+
+    int ret = opal_free_list_init(
+        /* *flist,frag_size,frag_alignment */
+        &mca_coll_han_component.pack_buffers, sizeof(opal_free_list_item_t), 8,
+        /* opal_class_t *frag_class */
+        OBJ_CLASS(opal_free_list_item_t),
+        /* payload_buffer_size, payload_buffer_alignment */
+        mca_coll_han_component.han_packbuf_bytes, 8,
+        /* num_elements_to_alloc, max_elements_to_alloc, num_elements_per_alloc */
+        0, mca_coll_han_component.han_packbuf_max_count, 8,
+        /* *mpool, rcache_reg_flags, *rcache, */
+        NULL, 0, NULL,
+        /* fn_t item_init, void *ctx */
+        NULL, NULL);
+    if (ret != 0) {
+        printf("han: initializing free list got %d\n",ret);
+    }
 
     return mca_coll_han_init_dynamic_rules();
 }
 
-
 /*
  * Shut down the component
  */
 static int han_close(void)
 {
     mca_coll_han_free_dynamic_rules();
+    mca_coll_han_free_algorithms();
+
+    free(mca_coll_han_component.han_op_module_name.bcast.han_op_up_module_name);
+    mca_coll_han_component.han_op_module_name.bcast.han_op_up_module_name = NULL;
+    free(mca_coll_han_component.han_op_module_name.bcast.han_op_low_module_name);
+    mca_coll_han_component.han_op_module_name.bcast.han_op_low_module_name = NULL;
+
+    free(mca_coll_han_component.han_op_module_name.reduce.han_op_up_module_name);
+    mca_coll_han_component.han_op_module_name.reduce.han_op_up_module_name = NULL;
+    free(mca_coll_han_component.han_op_module_name.reduce.han_op_low_module_name);
+    mca_coll_han_component.han_op_module_name.reduce.han_op_low_module_name = NULL;
+
+    free(mca_coll_han_component.han_op_module_name.allreduce.han_op_up_module_name);
+    mca_coll_han_component.han_op_module_name.allreduce.han_op_up_module_name = NULL;
+    free(mca_coll_han_component.han_op_module_name.allreduce.han_op_low_module_name);
+    mca_coll_han_component.han_op_module_name.allreduce.han_op_low_module_name = NULL;
+
+    free(mca_coll_han_component.han_op_module_name.allgather.han_op_up_module_name);
+    mca_coll_han_component.han_op_module_name.allgather.han_op_up_module_name = NULL;
+    free(mca_coll_han_component.han_op_module_name.allgather.han_op_low_module_name);
+    mca_coll_han_component.han_op_module_name.allgather.han_op_low_module_name = NULL;
+
+    free(mca_coll_han_component.han_op_module_name.gather.han_op_up_module_name);
+    mca_coll_han_component.han_op_module_name.gather.han_op_up_module_name = NULL;
+    free(mca_coll_han_component.han_op_module_name.gather.han_op_low_module_name);
+    mca_coll_han_component.han_op_module_name.gather.han_op_low_module_name = NULL;
+
+    free(mca_coll_han_component.han_op_module_name.gatherv.han_op_up_module_name);
+    mca_coll_han_component.han_op_module_name.gatherv.han_op_up_module_name = NULL;
+    free(mca_coll_han_component.han_op_module_name.gatherv.han_op_low_module_name);
+    mca_coll_han_component.han_op_module_name.gatherv.han_op_low_module_name = NULL;
+
+    free(mca_coll_han_component.han_op_module_name.scatter.han_op_up_module_name);
+    mca_coll_han_component.han_op_module_name.scatter.han_op_up_module_name = NULL;
+    free(mca_coll_han_component.han_op_module_name.scatter.han_op_low_module_name);
+    mca_coll_han_component.han_op_module_name.scatter.han_op_low_module_name = NULL;
+
+    free(mca_coll_han_component.han_op_module_name.scatterv.han_op_up_module_name);
+    mca_coll_han_component.han_op_module_name.scatterv.han_op_up_module_name = NULL;
+    free(mca_coll_han_component.han_op_module_name.scatterv.han_op_low_module_name);
+    mca_coll_han_component.han_op_module_name.scatterv.han_op_low_module_name = NULL;
+
     return OMPI_SUCCESS;
 }
 
@@ -129,24 +210,69 @@ static bool is_simple_implemented(COLLTYPE_T coll)
     }
 }
 
+/**
+ * topo level conversions both ways; str <-> id
+ * An enum is used for conversions.
+ */
+static mca_base_var_enum_value_t level_enumerator[] = {
+    { INTRA_NODE, "intra_node" },
+    { INTER_NODE, "inter_node" },
+    { GLOBAL_COMMUNICATOR, "global_communicator" },
+    { 0 }
+};
+
 /*
  * Stringifier for topological level
  */
-const char* mca_coll_han_topo_lvl_to_str(TOPO_LVL_T topo_lvl)
+const char* mca_coll_han_topo_lvl_to_str(TOPO_LVL_T topo_lvl_id)
 {
-    switch(topo_lvl) {
-        case INTRA_NODE:
-            return "intra_node";
-        case INTER_NODE:
-            return "inter_node";
-        case GLOBAL_COMMUNICATOR:
-            return "global_communicator";
-        case NB_TOPO_LVL:
-        default:
-            return "invalid topologic level";
+    for (int i = 0; level_enumerator[i].string != NULL; i++) {
+        if (topo_lvl_id == (TOPO_LVL_T) level_enumerator[i].value) {
+            return level_enumerator[i].string;
+        }
+    }
+    return "invalid topologic level";
+}
+int mca_coll_han_topo_lvl_name_to_id(const char *topo_level_name)
+{
+    for (int i = 0; level_enumerator[i].string != NULL; i++) {
+        if (0 == strcmp(topo_level_name, level_enumerator[i].string)) {
+            return i;
+        }
     }
+    return -1;
 }
 
+static int
+mca_coll_han_query_module_from_mca(mca_base_component_t* c,
+                                   const char* param_name,
+                                   const char* param_doc,
+                                   int info_level,
+                                   uint32_t* module_id,
+                                   char** storage)
+{
+    char *module_name, *endptr = NULL;
+
+    int mod_id = COMPONENTS_COUNT-1;
+    mod_id = (*module_id > (uint32_t)mod_id) ? mod_id : (int)*module_id; /* stay in range */
+
+    *storage = ompi_coll_han_available_components[mod_id].component_name;
+
+    (void) mca_base_component_var_register(c, param_name, param_doc,
+                                           MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
+                                           info_level,
+                                           MCA_BASE_VAR_SCOPE_ALL, storage);
+    module_name = *storage;
+    mod_id = strtol(module_name, &endptr, 10);
+    if( module_name == endptr ) {  /* no conversion, maybe we got a module name instead */
+        /* Convert module name to id */
+        mod_id = mca_coll_han_component_name_to_id(module_name);
+    }
+    /* Keep the module in the range */
+    *module_id = (mod_id < 0) ? 0 : mod_id;
+
+    return OMPI_SUCCESS;
+}
 
 /*
  * Register MCA params
@@ -163,141 +289,245 @@ static int han_register(void)
     TOPO_LVL_T topo_lvl;
     COMPONENT_T component;
 
-    cs->han_priority = 0;
     (void) mca_base_component_var_register(c, "priority", "Priority of the HAN coll component",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
+                                           OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_ALL, &cs->han_priority);
+
+    cs->han_output_verbose = 0;
+    (void) mca_base_component_var_register(c, "verbose", "Verbosity of the HAN coll component (use coll base verbosity if not set)",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                            OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY, &cs->han_priority);
+                                           MCA_BASE_VAR_SCOPE_ALL, &cs->han_output_verbose);
 
     cs->han_bcast_segsize = 65536;
     (void) mca_base_component_var_register(c, "bcast_segsize",
                                            "segment size for bcast",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                            OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY, &cs->han_bcast_segsize);
+                                           MCA_BASE_VAR_SCOPE_ALL, &cs->han_bcast_segsize);
 
     cs->han_bcast_up_module = 0;
-    (void) mca_base_component_var_register(c, "bcast_up_module",
-                                           "up level module for bcast, 0 libnbc, 1 adapt",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY, &cs->han_bcast_up_module);
+    (void) mca_coll_han_query_module_from_mca(c, "bcast_up_module",
+                                              "up level module for bcast, 0 libnbc, 1 adapt",
+                                              OPAL_INFO_LVL_9, &cs->han_bcast_up_module,
+                                              &cs->han_op_module_name.bcast.han_op_up_module_name);
 
     cs->han_bcast_low_module = 0;
-    (void) mca_base_component_var_register(c, "bcast_low_module",
-                                           "low level module for bcast, 0 tuned, 1 sm",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY, &cs->han_bcast_low_module);
+    (void) mca_coll_han_query_module_from_mca(c, "bcast_low_module",
+                                              "low level module for bcast, 0 tuned, 2 xhc",
+                                              OPAL_INFO_LVL_9,
+                                              &cs->han_bcast_low_module,
+                                              &cs->han_op_module_name.bcast.han_op_low_module_name);
 
     cs->han_reduce_segsize = 65536;
     (void) mca_base_component_var_register(c, "reduce_segsize",
                                            "segment size for reduce",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                            OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY, &cs->han_reduce_segsize);
+                                           MCA_BASE_VAR_SCOPE_ALL, &cs->han_reduce_segsize);
 
     cs->han_reduce_up_module = 0;
-    (void) mca_base_component_var_register(c, "reduce_up_module",
-                                           "up level module for allreduce, 0 libnbc, 1 adapt",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY, &cs->han_reduce_up_module);
+    (void) mca_coll_han_query_module_from_mca(c, "reduce_up_module",
+                                              "up level module for allreduce, 0 libnbc, 1 adapt",
+                                              OPAL_INFO_LVL_9, &cs->han_reduce_up_module,
+                                              &cs->han_op_module_name.reduce.han_op_up_module_name);
 
     cs->han_reduce_low_module = 0;
-    (void) mca_base_component_var_register(c, "reduce_low_module",
-                                           "low level module for allreduce, 0 tuned, 1 sm",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY, &cs->han_reduce_low_module);
+    (void) mca_coll_han_query_module_from_mca(c, "reduce_low_module",
+                                              "low level module for allreduce, 0 tuned, 2 xhc",
+                                              OPAL_INFO_LVL_9, &cs->han_reduce_low_module,
+                                              &cs->han_op_module_name.reduce.han_op_low_module_name);
+
     cs->han_allreduce_segsize = 65536;
     (void) mca_base_component_var_register(c, "allreduce_segsize",
                                            "segment size for allreduce",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                            OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY, &cs->han_allreduce_segsize);
+                                           MCA_BASE_VAR_SCOPE_ALL, &cs->han_allreduce_segsize);
 
     cs->han_allreduce_up_module = 0;
-    (void) mca_base_component_var_register(c, "allreduce_up_module",
-                                           "up level module for allreduce, 0 libnbc, 1 adapt",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY, &cs->han_allreduce_up_module);
+    (void) mca_coll_han_query_module_from_mca(c, "allreduce_up_module",
+                                              "up level module for allreduce, 0 libnbc, 1 adapt",
+                                              OPAL_INFO_LVL_9, &cs->han_allreduce_up_module,
+                                              &cs->han_op_module_name.allreduce.han_op_up_module_name);
 
     cs->han_allreduce_low_module = 0;
-    (void) mca_base_component_var_register(c, "allreduce_low_module",
-                                           "low level module for allreduce, 0 tuned, 1 sm",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY, &cs->han_allreduce_low_module);
+    (void) mca_coll_han_query_module_from_mca(c, "allreduce_low_module",
+                                              "low level module for allreduce, 0 tuned, 2 xhc",
+                                              OPAL_INFO_LVL_9, &cs->han_allreduce_low_module,
+                                              &cs->han_op_module_name.allreduce.han_op_low_module_name);
 
     cs->han_allgather_up_module = 0;
-    (void) mca_base_component_var_register(c, "allgather_up_module",
-                                           "up level module for allgather, 0 libnbc, 1 adapt",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY, &cs->han_allgather_up_module);
+    (void) mca_coll_han_query_module_from_mca(c, "allgather_up_module",
+                                              "up level module for allgather, 0 libnbc, 1 adapt",
+                                              OPAL_INFO_LVL_9, &cs->han_allgather_up_module,
+                                              &cs->han_op_module_name.allgather.han_op_up_module_name);
 
     cs->han_allgather_low_module = 0;
-    (void) mca_base_component_var_register(c, "allgather_low_module",
-                                           "low level module for allgather, 0 tuned, 1 sm",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY, &cs->han_allgather_low_module);
+    (void) mca_coll_han_query_module_from_mca(c, "allgather_low_module",
+                                              "low level module for allgather, 0 tuned, 2 xhc",
+                                              OPAL_INFO_LVL_9, &cs->han_allgather_low_module,
+                                              &cs->han_op_module_name.allgather.han_op_low_module_name);
 
     cs->han_gather_up_module = 0;
-    (void) mca_base_component_var_register(c, "gather_up_module",
-                                           "up level module for gather, 0 libnbc, 1 adapt",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY, &cs->han_gather_up_module);
+    (void) mca_coll_han_query_module_from_mca(c, "gather_up_module",
+                                              "up level module for gather, 0 libnbc, 1 adapt",
+                                              OPAL_INFO_LVL_9, &cs->han_gather_up_module,
+                                              &cs->han_op_module_name.gather.han_op_up_module_name);
 
     cs->han_gather_low_module = 0;
-    (void) mca_base_component_var_register(c, "gather_low_module",
-                                           "low level module for gather, 0 tuned, 1 sm",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY, &cs->han_gather_low_module);
+    (void) mca_coll_han_query_module_from_mca(c, "gather_low_module",
+                                              "low level module for gather, 0 tuned, 2 xhc",
+                                              OPAL_INFO_LVL_9, &cs->han_gather_low_module,
+                                              &cs->han_op_module_name.gather.han_op_low_module_name);
+
+    cs->han_gatherv_up_module = 0;
+    (void) mca_coll_han_query_module_from_mca(c, "gatherv_up_module",
+                                              "up level module for gatherv, 0 basic",
+                                              OPAL_INFO_LVL_9, &cs->han_gatherv_up_module,
+                                              &cs->han_op_module_name.gatherv.han_op_up_module_name);
+
+    cs->han_gatherv_low_module = 0;
+    (void) mca_coll_han_query_module_from_mca(c, "gatherv_low_module",
+                                              "low level module for gatherv, 0 basic",
+                                              OPAL_INFO_LVL_9, &cs->han_gatherv_low_module,
+                                              &cs->han_op_module_name.gatherv.han_op_low_module_name);
 
     cs->han_scatter_up_module = 0;
-    (void) mca_base_component_var_register(c, "scatter_up_module",
-                                           "up level module for scatter, 0 libnbc, 1 adapt",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY, &cs->han_scatter_up_module);
+    (void) mca_coll_han_query_module_from_mca(c, "scatter_up_module",
+                                              "up level module for scatter, 0 libnbc, 1 adapt",
+                                              OPAL_INFO_LVL_9, &cs->han_scatter_up_module,
+                                              &cs->han_op_module_name.scatter.han_op_up_module_name);
 
     cs->han_scatter_low_module = 0;
-    (void) mca_base_component_var_register(c, "scatter_low_module",
-                                           "low level module for scatter, 0 tuned, 1 sm",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY, &cs->han_scatter_low_module);
+    (void) mca_coll_han_query_module_from_mca(c, "scatter_low_module",
+                                              "low level module for scatter, 0 tuned, 2 xhc",
+                                              OPAL_INFO_LVL_9, &cs->han_scatter_low_module,
+                                              &cs->han_op_module_name.scatter.han_op_low_module_name);
+
+    cs->han_scatterv_up_module = 0;
+    (void) mca_coll_han_query_module_from_mca(c, "scatterv_up_module",
+                                              "up level module for scatterv, 0 basic",
+                                              OPAL_INFO_LVL_9, &cs->han_scatterv_up_module,
+                                              &cs->han_op_module_name.scatterv.han_op_up_module_name);
+
+    cs->han_scatterv_low_module = 0;
+    (void) mca_coll_han_query_module_from_mca(c, "scatterv_low_module",
+                                              "low level module for scatterv, 0 basic",
+                                              OPAL_INFO_LVL_9, &cs->han_scatterv_low_module,
+                                              &cs->han_op_module_name.scatterv.han_op_low_module_name);
+    cs->han_alltoall_low_module = 0;
+    (void) mca_coll_han_query_module_from_mca(c, "alltoall_lower_module",
+                                              "low level module for alltoall, 0 tuned, 1 sm ",
+                                              OPAL_INFO_LVL_9, &cs->han_alltoall_low_module,
+                                              &cs->han_op_module_name.alltoall.han_op_low_module_name);
+    cs->han_alltoall_pstages = 0;
+    (void) mca_base_component_var_register(c, "alltoall_pstages",
+                                              "Parallel Stages for alltoall.  Higher numbers require more memory, "
+                                              "and performs more communication in parallel.  0 chooses pstages based on message size.",
+                                              MCA_BASE_VAR_TYPE_INT32_T, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
+                                              OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_ALL,
+                                              &cs->han_alltoall_pstages);
+
+    cs->han_alltoallv_low_module = 0;
+    (void) mca_coll_han_query_module_from_mca(c, "alltoallv_lower_module",
+                                              "low level module for alltoallv, 0 tuned, 1 sm ",
+                                              OPAL_INFO_LVL_9, &cs->han_alltoallv_low_module,
+                                              &cs->han_op_module_name.alltoallv.han_op_low_module_name);
+    cs->han_alltoallv_smsc_avg_send_limit = 8192;
+    (void) mca_base_component_var_register(c, "alltoallv_smsc_avg_send_limit",
+                                              "The per-rank averaged send bytes limit above which smsc-based alltoallv will disqualify itself.",
+                                              MCA_BASE_VAR_TYPE_INT64_T, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
+                                              OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_ALL,
+                                              &cs->han_alltoallv_smsc_avg_send_limit);
+    cs->han_alltoallv_smsc_noncontig_activation_limit = 0.10;
+    (void) mca_base_component_var_register(c, "alltoallv_smsc_noncontig_limit",
+                                              "The fractional (0.00-1.00) limit of peers in the communicator which have "
+                                              "strided or otherwise non-contiguous data buffers.  Above this limit "
+                                              "smsc-based alltoallv will ignore the avg_send_limit, and always remain active.",
+                                              MCA_BASE_VAR_TYPE_DOUBLE, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
+                                              OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_ALL,
+                                              &cs->han_alltoallv_smsc_noncontig_activation_limit);
 
     cs->han_reproducible = 0;
     (void) mca_base_component_var_register(c, "reproducible",
                                            "whether we need reproducible results "
                                            "(enabling this disables optimisations using topology)"
                                            "0 disable 1 enable, default 0",
-                                           MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
+                                           MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                            OPAL_INFO_LVL_3,
-                                           MCA_BASE_VAR_SCOPE_READONLY, &cs->han_reproducible);
+                                           MCA_BASE_VAR_SCOPE_ALL, &cs->han_reproducible);
+
+    cs->han_packbuf_bytes = 128*1024;
+    (void) mca_base_component_var_register(c, "packbuf_bytes",
+                                           "The number of bytes in each HAN packbuf.",
+                                           MCA_BASE_VAR_TYPE_INT64_T, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
+                                           OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_ALL,
+                                           &cs->han_packbuf_bytes);
+    cs->han_packbuf_max_count = 32;
+    (void) mca_base_component_var_register(c, "packbuf_max_count",
+                                           "The maximum number of packbufs that are allowed to be allocated.",
+                                           MCA_BASE_VAR_TYPE_INT64_T, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
+                                           OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_ALL,
+                                           &cs->han_packbuf_max_count);
+
+    /*
+     * Han algorithms MCA parameters for each collective.
+     * Shows algorithms thanks to enumerator
+     */
+    if (OMPI_ERROR == mca_coll_han_init_algorithms()) { // needs to be initialised here to show available algorithms
+       return OMPI_ERROR;
+    }
+
+    mca_base_var_enum_t *new_enum;
+    for(coll = 0 ; coll < COLLCOUNT ; coll++) {
+        if (!mca_coll_han_is_coll_dynamic_implemented(coll)
+            || (0 == mca_coll_han_component.num_available_algorithms[coll])) {
+          continue;
+        }
+        cs->use_algorithm[coll] = 0; // default algorithm is 0
+        snprintf(param_name, sizeof(param_name), "use_%s_algorithm",
+                 mca_coll_base_colltype_to_str(coll));
+        snprintf(param_desc, sizeof(param_desc), "which han algorithm is used for %s",
+                 mca_coll_base_colltype_to_str(coll));
+        // note: the enumerator is create in mca_coll_han_init_algorithms()
+        (void) mca_base_var_enum_create(param_name,
+                                        mca_coll_han_component.algorithm_enumerator[coll],
+                                        &new_enum);
+        cs->use_algorithm_param[coll] = mca_base_component_var_register(c,
+                                        param_name,
+                                        param_desc,
+                                        MCA_BASE_VAR_TYPE_INT, new_enum, 0, MCA_BASE_VAR_FLAG_SETTABLE,
+                                        OPAL_INFO_LVL_5,
+                                        MCA_BASE_VAR_SCOPE_ALL,
+                                        &(cs->use_algorithm[coll]));
+        OBJ_RELEASE(new_enum);
+    }
 
     /*
      * Simple algorithms MCA parameters :
      * using simple algorithms will just perform hierarchical communications.
-     * By default communications are also splitted into tasks
+     * By default communications are also split into tasks
      * to handle thread noise
      */
     for(coll = 0 ; coll < COLLCOUNT ; coll++) {
-        cs->use_simple_algorithm[coll] = false;
+        if (coll != GATHER) {
+            cs->use_simple_algorithm[coll] = false;
+        } else {
+            cs->use_simple_algorithm[coll] = true;
+        }
         if(is_simple_implemented(coll)) {
+            const char *collstr = mca_coll_base_colltype_to_str(coll);
             snprintf(param_name, sizeof(param_name), "use_simple_%s",
-                     mca_coll_base_colltype_to_str(coll));
-            snprintf(param_desc, sizeof(param_desc), "whether to enable simple algo for %s",
-                     mca_coll_base_colltype_to_str(coll));
+                     collstr);
+            snprintf(param_desc, sizeof(param_desc), "whether to enable simple algorithm for %s. "
+                     "Prefer use_%s_algorithm=simple or configuration file instead.",
+                     collstr, collstr);
             mca_base_component_var_register(c, param_name,
                                             param_desc,
-                                            MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
+                                            MCA_BASE_VAR_TYPE_BOOL, NULL, 0,
+                                            MCA_BASE_VAR_FLAG_DEPRECATED,
                                             OPAL_INFO_LVL_5,
                                             MCA_BASE_VAR_SCOPE_READONLY,
                                             &(cs->use_simple_algorithm[coll]));
@@ -305,7 +535,7 @@ static int han_register(void)
     }
 
     /* Dynamic rules MCA parameters */
-    memset(cs->mca_rules, 0,
+    memset(cs->mca_sub_components, 0,
            COLLCOUNT * (GLOBAL_COMMUNICATOR+1) * sizeof(COMPONENT_T));
 
     for(coll = 0; coll < COLLCOUNT; coll++) {
@@ -315,12 +545,12 @@ static int han_register(void)
         /*
          * Default values
          */
-        cs->mca_rules[coll][INTRA_NODE] = TUNED;
-        cs->mca_rules[coll][INTER_NODE] = BASIC;
-        cs->mca_rules[coll][GLOBAL_COMMUNICATOR] = HAN;
+        for (topo_lvl = 0 ; topo_lvl < GLOBAL_COMMUNICATOR ; topo_lvl++) {
+            cs->mca_sub_components[coll][topo_lvl] = TUNED;
+        }
+        cs->mca_sub_components[coll][GLOBAL_COMMUNICATOR] = HAN;
     }
     /* Specific default values */
-    cs->mca_rules[BARRIER][INTER_NODE] = TUNED;
 
     /* Dynamic rule MCA var registration */
     for(coll = 0; coll < COLLCOUNT; coll++) {
@@ -350,14 +580,14 @@ static int han_register(void)
                 param_desc_size += snprintf(param_desc+param_desc_size, sizeof(param_desc) - param_desc_size,
                                             "%d = %s; ",
                                             component,
-                                            available_components[component].component_name);
+                                            ompi_coll_han_available_components[component].component_name);
             }
 
             mca_base_component_var_register(c, param_name, param_desc,
-                                            MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                            MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                             OPAL_INFO_LVL_9,
-                                            MCA_BASE_VAR_SCOPE_READONLY,
-                                            &(cs->mca_rules[coll][topo_lvl]));
+                                            MCA_BASE_VAR_SCOPE_ALL,
+                                            &(cs->mca_sub_components[coll][topo_lvl]));
         }
     }
 
@@ -366,27 +596,27 @@ static int han_register(void)
     (void) mca_base_component_var_register(&mca_coll_han_component.super.collm_version,
                                            "use_dynamic_file_rules",
                                            "Enable the dynamic selection provided via the dynamic_rules_filename MCA",
-                                           MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
+                                           MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                            OPAL_INFO_LVL_6,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           MCA_BASE_VAR_SCOPE_ALL,
                                            &(cs->use_dynamic_file_rules));
 
     cs->dynamic_rules_filename = NULL;
     (void) mca_base_component_var_register(&mca_coll_han_component.super.collm_version,
                                            "dynamic_rules_filename",
                                            "Configuration file containing the dynamic selection rules",
-                                           MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
+                                           MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                            OPAL_INFO_LVL_6,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           MCA_BASE_VAR_SCOPE_ALL,
                                            &(cs->dynamic_rules_filename));
 
     cs->dump_dynamic_rules = false;
     (void) mca_base_component_var_register(&mca_coll_han_component.super.collm_version,
                                            "dump_dynamic_rules",
                                            "Switch used to decide if we dump  dynamic rules provided by configuration file",
-                                           MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
+                                           MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                            OPAL_INFO_LVL_6,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           MCA_BASE_VAR_SCOPE_ALL,
                                            &(cs->dump_dynamic_rules));
 
     if((cs->dump_dynamic_rules || NULL != cs->dynamic_rules_filename)
@@ -403,9 +633,9 @@ static int han_register(void)
                                            "errors printed on rank 0 "
                                            "with a 0 verbosity."
                                            "Useless if coll_base_verbose is 30 or more.",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                            OPAL_INFO_LVL_6,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           MCA_BASE_VAR_SCOPE_ALL,
                                            &(cs->max_dynamic_errors));
 
 
diff --git a/ompi/mca/coll/han/coll_han_dynamic.c b/ompi/mca/coll/han/coll_han_dynamic.c
index 9e3469b0160..3d483b35aa9 100644
--- a/ompi/mca/coll/han/coll_han_dynamic.c
+++ b/ompi/mca/coll/han/coll_han_dynamic.c
@@ -1,6 +1,10 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2020      Bull S.A.S. All rights reserved.
+ * Copyright (c) 2020-2022 Bull S.A.S. All rights reserved.
+ * Copyright (c) 2021      Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
+ * Copyright (c) 2024      Amazon.com, Inc. or its affiliates. All Rights Reserved.
  *
  * $COPYRIGHT$
  *
@@ -20,11 +24,14 @@
 #include "opal/class/opal_list.h"
 #include "ompi/mca/coll/han/coll_han.h"
 #include "ompi/mca/coll/han/coll_han_dynamic.h"
+#include "ompi/mca/coll/han/coll_han_algorithms.h"
 #include "ompi/mca/coll/base/coll_base_util.h"
 
+#define MCA_COLL_HAN_ANY_MESSAGE_SIZE 0
+
 /*
  * Tests if a dynamic collective is implemented
- * Usefull for file reading warnings and MCA parameter generation
+ * Useful for file reading warnings and MCA parameter generation
  * When a new dynamic collective is implemented, this function must
  * return true for it
  */
@@ -34,11 +41,15 @@ bool mca_coll_han_is_coll_dynamic_implemented(COLLTYPE_T coll_id)
     case ALLGATHER:
     case ALLGATHERV:
     case ALLREDUCE:
+    case ALLTOALL:
+    case ALLTOALLV:
     case BARRIER:
     case BCAST:
     case GATHER:
+    case GATHERV:
     case REDUCE:
     case SCATTER:
+    case SCATTERV:
         return true;
     default:
         return false;
@@ -53,7 +64,7 @@ mca_coll_han_component_name_to_id(const char* name)
     }
 
     for( int i = SELF; i < COMPONENTS_COUNT ; i++ ) {
-        if (0 == strcmp(name, available_components[i].component_name)) {
+        if (0 == strcmp(name, ompi_coll_han_available_components[i].component_name)) {
             return i;
         }
     }
@@ -94,9 +105,9 @@ mca_coll_han_get_all_coll_modules(struct ompi_communicator_t *comm,
             han_module->modules_storage.modules[id].module_handler = module;
             opal_output_verbose(80, mca_coll_han_component.han_output,
                                 "coll:han:get_all_coll_modules HAN found module %s with id %d "
-                                "for topological level %d (%s) for communicator (%d/%s)\n",
+                                "for topological level %d (%s) for communicator (%s/%s)\n",
                                 name, id, topo_lvl, mca_coll_han_topo_lvl_to_str(topo_lvl),
-                                comm->c_contextid, comm->c_name);
+                                ompi_comm_print_cid(comm), comm->c_name);
             nb_modules++;
         }
     }
@@ -113,9 +124,9 @@ mca_coll_han_get_all_coll_modules(struct ompi_communicator_t *comm,
     opal_output_verbose(60, mca_coll_han_component.han_output,
                         "coll:han:get_all_coll_modules HAN sub-communicator modules storage "
                         "for topological level %d (%s) gets %d modules "
-                        "for communicator (%d/%s)\n",
+                        "for communicator (%s/%s)\n",
                         topo_lvl, mca_coll_han_topo_lvl_to_str(topo_lvl),
-                        nb_modules, comm->c_contextid, comm->c_name);
+                        nb_modules, ompi_comm_print_cid(comm), comm->c_name);
 
     assert(0 != nb_modules);
 
@@ -248,7 +259,7 @@ get_dynamic_rule(COLLTYPE_T collective,
                         msg_size_rule->topologic_level,
                         mca_coll_han_topo_lvl_to_str(msg_size_rule->topologic_level),
                         msg_size_rule->configuration_size,
-                        msg_size_rule->msg_size, component, available_components[component].component_name);
+                        msg_size_rule->msg_size, component, ompi_coll_han_available_components[component].component_name);
 
     return msg_size_rule;
 }
@@ -269,7 +280,7 @@ get_module(COLLTYPE_T coll_id,
     COMPONENT_T mca_rule_component;
 
     topo_lvl = han_module->topologic_level;
-    mca_rule_component = mca_coll_han_component.mca_rules[coll_id][topo_lvl];
+    mca_rule_component = mca_coll_han_component.mca_sub_components[coll_id][topo_lvl];
 
     mca_coll_han_get_all_coll_modules(comm, han_module);
 
@@ -302,6 +313,60 @@ get_module(COLLTYPE_T coll_id,
     return han_module->modules_storage.modules[mca_rule_component].module_handler;
 }
 
+/*
+ * whether OMPI_MCA_coll_han_use_xxx_algorithm was set by user
+ */
+static bool
+han_algorithm_is_user_provided(COLLTYPE_T coll_id) {
+    const int *value = NULL;
+    mca_base_var_source_t source;
+    mca_base_var_get_value(mca_coll_han_component.use_algorithm_param[coll_id],
+                           &value, &source, NULL);
+    return (MCA_BASE_VAR_SOURCE_DEFAULT != source);
+}
+
+/*
+ * Return the algorithm to use for the collective coll_id
+ * for a msg_size sized message on the comm communicator
+ * following the dynamic rules
+ */
+static int
+get_algorithm(COLLTYPE_T coll_id,
+              size_t msg_size,
+              struct ompi_communicator_t *comm,
+              mca_coll_han_module_t *han_module)
+{
+    int algorithm_id = -1;
+    int rank = ompi_comm_rank(comm);
+    algorithm_id = mca_coll_han_component.use_algorithm[coll_id];
+    if (!han_algorithm_is_user_provided(coll_id)) {
+        /* find the correct dynamic rule to check */
+        const msg_size_rule_t *dynamic_rule = get_dynamic_rule(coll_id,
+                                                               msg_size,
+                                                               comm,
+                                                               han_module);
+        if(NULL != dynamic_rule && dynamic_rule->algorithm_id >= 0) {
+            /* Use dynamic rule from file */
+            algorithm_id = dynamic_rule->algorithm_id;
+        } else {
+            /*
+             * No dynamic rule from file
+             * Use default behaviour
+             */
+            algorithm_id = 0;
+        }
+    }
+    if ( 0 == rank ) {
+        opal_output_verbose(30, mca_coll_han_component.han_output,
+                            "coll:han:get_algorithm %s size:%ld algorithm:%d %s",
+                            mca_coll_base_colltype_to_str(coll_id),
+                            msg_size,
+                            algorithm_id,
+                            mca_coll_han_algorithm_id_to_name(coll_id, algorithm_id));
+    }
+    return algorithm_id;
+}
+
 
 /*
  * Allgather selector:
@@ -310,9 +375,9 @@ get_module(COLLTYPE_T coll_id,
  * calls the correct module if fallback mechanism is activated
  */
 int
-mca_coll_han_allgather_intra_dynamic(const void *sbuf, int scount,
+mca_coll_han_allgather_intra_dynamic(const void *sbuf, size_t scount,
                                      struct ompi_datatype_t *sdtype,
-                                     void *rbuf, int rcount,
+                                     void *rbuf, size_t rcount,
                                      struct ompi_datatype_t *rdtype,
                                      struct ompi_communicator_t *comm,
                                      mca_coll_base_module_t *module)
@@ -352,11 +417,11 @@ mca_coll_han_allgather_intra_dynamic(const void *sbuf, int scount,
         opal_output_verbose(verbosity, mca_coll_han_component.han_output,
                             "coll:han:mca_coll_han_allgather_intra_dynamic "
                             "HAN did not find any valid module for collective %d (%s) "
-                            "with topological level %d (%s) on communicator (%d/%s). "
+                            "with topological level %d (%s) on communicator (%s/%s). "
                             "Please check dynamic file/mca parameters\n",
                             ALLGATHER, mca_coll_base_colltype_to_str(ALLGATHER),
                             topo_lvl, mca_coll_han_topo_lvl_to_str(topo_lvl),
-                            comm->c_contextid, comm->c_name);
+                            ompi_comm_print_cid(comm), comm->c_name);
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "HAN/ALLGATHER: No module found for the sub-communicator. "
                              "Falling back to another component\n"));
@@ -370,11 +435,11 @@ mca_coll_han_allgather_intra_dynamic(const void *sbuf, int scount,
         han_module->dynamic_errors++;
         opal_output_verbose(verbosity, mca_coll_han_component.han_output,
                             "coll:han:mca_coll_han_allgather_intra_dynamic HAN found valid module for collective %d (%s) "
-                            "with topological level %d (%s) on communicator (%d/%s) but this module cannot handle this collective. "
+                            "with topological level %d (%s) on communicator (%s/%s) but this module cannot handle this collective. "
                             "Please check dynamic file/mca parameters\n",
                             ALLGATHER, mca_coll_base_colltype_to_str(ALLGATHER),
                             topo_lvl, mca_coll_han_topo_lvl_to_str(topo_lvl),
-                            comm->c_contextid, comm->c_name);
+                            ompi_comm_print_cid(comm), comm->c_name);
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "HAN/ALLGATHER: the module found for the sub-communicator"
                              " cannot handle the ALLGATHER operation. Falling back to another component\n"));
@@ -387,10 +452,17 @@ mca_coll_han_allgather_intra_dynamic(const void *sbuf, int scount,
          * sub_module->coll_allgather is valid and point to this function
          * Call han topological collective algorithm
          */
-        if(mca_coll_han_component.use_simple_algorithm[ALLGATHER]) {
-            allgather = mca_coll_han_allgather_intra_simple;
-        } else {
-            allgather = mca_coll_han_allgather_intra;
+        int algorithm_id = get_algorithm(ALLGATHER,
+                                         dtype_size,
+                                         comm,
+                                         han_module);
+        allgather = (mca_coll_base_module_allgather_fn_t)mca_coll_han_algorithm_id_to_fn(ALLGATHER, algorithm_id);
+        if (NULL == allgather) { /* default behaviour */
+            if(mca_coll_han_component.use_simple_algorithm[ALLGATHER]) {
+                allgather = mca_coll_han_allgather_intra_simple;
+            } else {
+                allgather = mca_coll_han_allgather_intra;
+            }
         }
     } else {
         /*
@@ -417,10 +489,10 @@ mca_coll_han_allgather_intra_dynamic(const void *sbuf, int scount,
  * The allgatherv size is the size of the biggest segment
  */
 int
-mca_coll_han_allgatherv_intra_dynamic(const void *sbuf, int scount,
+mca_coll_han_allgatherv_intra_dynamic(const void *sbuf, size_t scount,
                                       struct ompi_datatype_t *sdtype,
-                                      void *rbuf, const int *rcounts,
-                                      const int *displs,
+                                      void *rbuf, ompi_count_array_t rcounts,
+                                      ompi_disp_array_t displs,
                                       struct ompi_datatype_t *rdtype,
                                       struct ompi_communicator_t *comm,
                                       mca_coll_base_module_t *module)
@@ -437,8 +509,8 @@ mca_coll_han_allgatherv_intra_dynamic(const void *sbuf, int scount,
     ompi_datatype_type_size(rdtype, &dtype_size);
 
     for(i = 0; i < comm_size; i++) {
-        if(dtype_size * rcounts[i] > msg_size) {
-            msg_size = dtype_size * rcounts[i];
+        if(dtype_size * ompi_count_array_get(rcounts, i) > msg_size) {
+            msg_size = dtype_size * ompi_count_array_get(rcounts, i);
         }
     }
 
@@ -462,11 +534,11 @@ mca_coll_han_allgatherv_intra_dynamic(const void *sbuf, int scount,
         opal_output_verbose(verbosity, mca_coll_han_component.han_output,
                             "coll:han:mca_coll_han_allgatherv_intra_dynamic "
                             "HAN did not find any valid module for collective %d (%s) "
-                            "with topological level %d (%s) on communicator (%d/%s). "
+                            "with topological level %d (%s) on communicator (%s/%s). "
                             "Please check dynamic file/mca parameters\n",
                             ALLGATHERV, mca_coll_base_colltype_to_str(ALLGATHERV),
                             topo_lvl, mca_coll_han_topo_lvl_to_str(topo_lvl),
-                            comm->c_contextid, comm->c_name);
+                            ompi_comm_print_cid(comm), comm->c_name);
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "HAN/ALLGATHERV: No module found for the sub-communicator. "
                              "Falling back to another component\n"));
@@ -481,12 +553,12 @@ mca_coll_han_allgatherv_intra_dynamic(const void *sbuf, int scount,
         opal_output_verbose(verbosity, mca_coll_han_component.han_output,
                             "coll:han:mca_coll_han_allgatherv_intra_dynamic "
                             "HAN found valid module for collective %d (%s) "
-                            "with topological level %d (%s) on communicator (%d/%s) "
+                            "with topological level %d (%s) on communicator (%s/%s) "
                             "but this module cannot handle this collective. "
                             "Please check dynamic file/mca parameters\n",
                             ALLGATHERV, mca_coll_base_colltype_to_str(ALLGATHERV),
                             topo_lvl, mca_coll_han_topo_lvl_to_str(topo_lvl),
-                            comm->c_contextid, comm->c_name);
+                            ompi_comm_print_cid(comm), comm->c_name);
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "HAN/ALLGATHERV: the module found for the sub-"
                              "communicator cannot handle the ALLGATHERV operation. "
@@ -503,11 +575,11 @@ mca_coll_han_allgatherv_intra_dynamic(const void *sbuf, int scount,
         opal_output_verbose(30, mca_coll_han_component.han_output,
                             "coll:han:mca_coll_han_allgatherv_intra_dynamic "
                             "HAN used for collective %d (%s) with topological level %d (%s) "
-                            "on communicator (%d/%s) but this module cannot handle "
+                            "on communicator (%s/%s) but this module cannot handle "
                             "this collective on this topologic level\n",
                             ALLGATHERV, mca_coll_base_colltype_to_str(ALLGATHERV),
                             topo_lvl, mca_coll_han_topo_lvl_to_str(topo_lvl),
-                            comm->c_contextid, comm->c_name);
+                            ompi_comm_print_cid(comm), comm->c_name);
         allgatherv = han_module->previous_allgatherv;
         sub_module = han_module->previous_allgatherv_module;
     } else {
@@ -536,7 +608,7 @@ mca_coll_han_allgatherv_intra_dynamic(const void *sbuf, int scount,
 int
 mca_coll_han_allreduce_intra_dynamic(const void *sbuf,
                                      void *rbuf,
-                                     int count,
+                                     size_t count,
                                      struct ompi_datatype_t *dtype,
                                      struct ompi_op_t *op,
                                      struct ompi_communicator_t *comm,
@@ -549,6 +621,11 @@ mca_coll_han_allreduce_intra_dynamic(const void *sbuf,
     size_t dtype_size;
     int rank, verbosity = 0;
 
+    if (!han_module->enabled) {
+        return han_module->previous_allreduce(sbuf, rbuf, count, dtype, op, comm,
+                                              han_module->previous_allreduce_module);
+    }
+
     /* Compute configuration information for dynamic rules */
     ompi_datatype_type_size(dtype, &dtype_size);
     dtype_size = dtype_size * count;
@@ -573,11 +650,11 @@ mca_coll_han_allreduce_intra_dynamic(const void *sbuf,
         opal_output_verbose(verbosity, mca_coll_han_component.han_output,
                             "coll:han:mca_coll_han_allreduce_intra_dynamic "
                             "HAN did not find any valid module for collective %d (%s) "
-                            "with topological level %d (%s) on communicator (%d/%s). "
+                            "with topological level %d (%s) on communicator (%s/%s). "
                             "Please check dynamic file/mca parameters\n",
                             ALLREDUCE, mca_coll_base_colltype_to_str(ALLREDUCE),
                             topo_lvl, mca_coll_han_topo_lvl_to_str(topo_lvl),
-                            comm->c_contextid, comm->c_name);
+                            ompi_comm_print_cid(comm), comm->c_name);
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "HAN/ALLREDUCE: No module found for the sub-communicator. "
                              "Falling back to another component\n"));
@@ -592,12 +669,12 @@ mca_coll_han_allreduce_intra_dynamic(const void *sbuf,
         opal_output_verbose(verbosity, mca_coll_han_component.han_output,
                             "coll:han:mca_coll_han_allreduce_intra_dynamic "
                             "HAN found valid module for collective %d (%s) "
-                            "with topological level %d (%s) on communicator (%d/%s) "
+                            "with topological level %d (%s) on communicator (%s/%s) "
                             "but this module cannot handle this collective. "
                             "Please check dynamic file/mca parameters\n",
                             ALLREDUCE, mca_coll_base_colltype_to_str(ALLREDUCE),
                             topo_lvl, mca_coll_han_topo_lvl_to_str(topo_lvl),
-                            comm->c_contextid, comm->c_name);
+                            ompi_comm_print_cid(comm), comm->c_name);
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "HAN/ALLREDUCE: the module found for the sub-"
                              "communicator cannot handle the ALLREDUCE operation. "
@@ -605,7 +682,7 @@ mca_coll_han_allreduce_intra_dynamic(const void *sbuf,
         allreduce = han_module->previous_allreduce;
         sub_module = han_module->previous_allreduce_module;
     } else if (GLOBAL_COMMUNICATOR == topo_lvl && sub_module == module) {
-        /* Reproducibility: fallback on reproducible algo */
+        /* Reproducibility: fallback on reproducible algorithm */
         if (mca_coll_han_component.han_reproducible) {
             allreduce = mca_coll_han_allreduce_reproducible;
         } else {
@@ -615,13 +692,17 @@ mca_coll_han_allreduce_intra_dynamic(const void *sbuf,
              * sub_module->coll_allreduce is valid and point to this function
              * Call han topological collective algorithm
              */
-            if(mca_coll_han_component.use_simple_algorithm[ALLREDUCE]) {
-                allreduce = mca_coll_han_allreduce_intra_simple;
-            } else {
-                allreduce = mca_coll_han_allreduce_intra;
+            int algorithm_id = get_algorithm(ALLREDUCE, dtype_size, comm, han_module);
+            allreduce = (mca_coll_base_module_allreduce_fn_t) mca_coll_han_algorithm_id_to_fn(ALLREDUCE, algorithm_id);
+
+            if (NULL == allreduce) { /* default behaviour */
+                if(mca_coll_han_component.use_simple_algorithm[ALLREDUCE]) {
+                    allreduce = mca_coll_han_allreduce_intra_simple;
+                } else {
+                    allreduce = mca_coll_han_allreduce_intra;
+                }
             }
         }
-        sub_module = module;
     } else {
         /*
          * If we get here:
@@ -630,7 +711,7 @@ mca_coll_han_allreduce_intra_dynamic(const void *sbuf,
          * They points to the collective to use, according to the dynamic rules
          * Selector's job is done, call the collective
          */
-        allreduce = mca_coll_han_allreduce_intra;
+        allreduce = sub_module->coll_allreduce;
     }
     return allreduce(sbuf, rbuf, count, dtype,
                      op, comm, sub_module);
@@ -653,6 +734,9 @@ mca_coll_han_barrier_intra_dynamic(struct ompi_communicator_t *comm,
     mca_coll_base_module_t *sub_module;
     int rank, verbosity = 0;
 
+    if (!han_module->enabled) {
+        return han_module->previous_barrier(comm, han_module->previous_barrier_module);
+    }
 
     /* Compute configuration information for dynamic rules */
     sub_module = get_module(BARRIER,
@@ -675,11 +759,11 @@ mca_coll_han_barrier_intra_dynamic(struct ompi_communicator_t *comm,
         opal_output_verbose(verbosity, mca_coll_han_component.han_output,
                             "coll:han:mca_coll_han_barrier_intra_dynamic "
                             "Han did not find any valid module for collective %d (%s) "
-                            "with topological level %d (%s) on communicator (%d/%s). "
+                            "with topological level %d (%s) on communicator (%s/%s). "
                             "Please check dynamic file/mca parameters\n",
                             BARRIER, mca_coll_base_colltype_to_str(BARRIER),
                             topo_lvl, mca_coll_han_topo_lvl_to_str(topo_lvl),
-                            comm->c_contextid, comm->c_name);
+                            ompi_comm_print_cid(comm), comm->c_name);
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "HAN/BARRIER: No module found for the sub-communicator. "
                              "Falling back to another component\n"));
@@ -694,12 +778,12 @@ mca_coll_han_barrier_intra_dynamic(struct ompi_communicator_t *comm,
         opal_output_verbose(verbosity, mca_coll_han_component.han_output,
                             "coll:han:mca_coll_han_barrier_intra_dynamic "
                             "Han found valid module for collective %d (%s) "
-                            "with topological level %d (%s) on communicator (%d/%s) "
+                            "with topological level %d (%s) on communicator (%s/%s) "
                             "but this module cannot handle this collective. "
                             "Please check dynamic file/mca parameters\n",
                             BARRIER, mca_coll_base_colltype_to_str(BARRIER),
                             topo_lvl, mca_coll_han_topo_lvl_to_str(topo_lvl),
-                            comm->c_contextid, comm->c_name);
+                            ompi_comm_print_cid(comm), comm->c_name);
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "HAN/BARRIER: the module found for the sub-"
                              "communicator cannot handle the BARRIER operation. "
@@ -713,7 +797,11 @@ mca_coll_han_barrier_intra_dynamic(struct ompi_communicator_t *comm,
          * sub_module->coll_barrier is valid and point to this function
          * Call han topological collective algorithm
          */
-        barrier = mca_coll_han_barrier_intra_simple;
+        int algorithm_id = get_algorithm(BARRIER, 0, comm, han_module);
+        barrier = (mca_coll_base_module_barrier_fn_t) mca_coll_han_algorithm_id_to_fn(BARRIER, algorithm_id);
+        if (NULL == barrier) { /* default behaviour*/
+            barrier = mca_coll_han_barrier_intra_simple;
+        }
     } else {
         /*
          * If we get here:
@@ -735,7 +823,7 @@ mca_coll_han_barrier_intra_dynamic(struct ompi_communicator_t *comm,
  */
 int
 mca_coll_han_bcast_intra_dynamic(void *buff,
-                                 int count,
+                                 size_t count,
                                  struct ompi_datatype_t *dtype,
                                  int root,
                                  struct ompi_communicator_t *comm,
@@ -748,6 +836,11 @@ mca_coll_han_bcast_intra_dynamic(void *buff,
     size_t dtype_size;
     int rank, verbosity = 0;
 
+    if (!han_module->enabled) {
+        return han_module->previous_bcast(buff, count, dtype, root, comm,
+                                          han_module->previous_bcast_module);
+    }
+
     /* Compute configuration information for dynamic rules */
     ompi_datatype_type_size(dtype, &dtype_size);
     dtype_size = dtype_size * count;
@@ -772,11 +865,11 @@ mca_coll_han_bcast_intra_dynamic(void *buff,
         opal_output_verbose(verbosity, mca_coll_han_component.han_output,
                             "coll:han:mca_coll_han_bcast_intra_dynamic "
                             "HAN did not find any valid module for collective %d (%s) "
-                            "with topological level %d (%s) on communicator (%d/%s). "
+                            "with topological level %d (%s) on communicator (%s/%s). "
                             "Please check dynamic file/mca parameters\n",
                             BCAST, mca_coll_base_colltype_to_str(BCAST),
                             topo_lvl, mca_coll_han_topo_lvl_to_str(topo_lvl),
-                            comm->c_contextid, comm->c_name);
+                            ompi_comm_print_cid(comm), comm->c_name);
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "HAN/BCAST: No module found for the sub-communicator. "
                              "Falling back to another component\n"));
@@ -791,12 +884,12 @@ mca_coll_han_bcast_intra_dynamic(void *buff,
         opal_output_verbose(verbosity, mca_coll_han_component.han_output,
                             "coll:han:mca_coll_han_bcast_intra_dynamic "
                             "HAN found valid module for collective %d (%s) "
-                            "with topological level %d (%s) on communicator (%d/%s) "
+                            "with topological level %d (%s) on communicator (%s/%s) "
                             "but this module cannot handle this collective. "
                             "Please check dynamic file/mca parameters\n",
                             BCAST, mca_coll_base_colltype_to_str(BCAST),
                             topo_lvl, mca_coll_han_topo_lvl_to_str(topo_lvl),
-                            comm->c_contextid, comm->c_name);
+                            ompi_comm_print_cid(comm), comm->c_name);
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "HAN/BCAST: the module found for the sub-"
                              "communicator cannot handle the BCAST operation. "
@@ -810,12 +903,18 @@ mca_coll_han_bcast_intra_dynamic(void *buff,
          * sub_module->coll_bcast is valid and point to this function
          * Call han topological collective algorithm
          */
-        if(mca_coll_han_component.use_simple_algorithm[BCAST]) {
-            bcast = mca_coll_han_bcast_intra_simple;
-        } else {
-            bcast = mca_coll_han_bcast_intra;
+        int algorithm_id = get_algorithm(BCAST,
+                                         dtype_size,
+                                         comm,
+                                         han_module);
+        bcast = (mca_coll_base_module_bcast_fn_t)mca_coll_han_algorithm_id_to_fn(BCAST, algorithm_id);
+        if (NULL == bcast) { /* default behaviour */
+             if(mca_coll_han_component.use_simple_algorithm[BCAST]) {
+                bcast = mca_coll_han_bcast_intra_simple;
+            } else {
+                bcast = mca_coll_han_bcast_intra;
+            }
         }
-        sub_module = module;
     } else {
         /*
          * If we get here:
@@ -838,9 +937,9 @@ mca_coll_han_bcast_intra_dynamic(void *buff,
  * calls the correct module if fallback mechanism is activated
  */
 int
-mca_coll_han_gather_intra_dynamic(const void *sbuf, int scount,
+mca_coll_han_gather_intra_dynamic(const void *sbuf, size_t scount,
                                   struct ompi_datatype_t *sdtype,
-                                  void *rbuf, int rcount,
+                                  void *rbuf, size_t rcount,
                                   struct ompi_datatype_t *rdtype,
                                   int root,
                                   struct ompi_communicator_t *comm,
@@ -853,6 +952,11 @@ mca_coll_han_gather_intra_dynamic(const void *sbuf, int scount,
     size_t dtype_size;
     int rank, verbosity = 0;
 
+    if (!han_module->enabled) {
+        return han_module->previous_gather(sbuf, scount, sdtype, rbuf, rcount, rdtype, root, comm,
+                                           han_module->previous_gather_module);
+    }
+
     /* Compute configuration information for dynamic rules */
     if( MPI_IN_PLACE != sbuf ) {
         ompi_datatype_type_size(sdtype, &dtype_size);
@@ -882,11 +986,11 @@ mca_coll_han_gather_intra_dynamic(const void *sbuf, int scount,
         opal_output_verbose(verbosity, mca_coll_han_component.han_output,
                             "coll:han:mca_coll_han_gather_intra_dynamic "
                             "HAN did not find any valid module for collective %d (%s) "
-                            "with topological level %d (%s) on communicator (%d/%s). "
+                            "with topological level %d (%s) on communicator (%s/%s). "
                             "Please check dynamic file/mca parameters\n",
                             GATHER, mca_coll_base_colltype_to_str(GATHER),
                             topo_lvl, mca_coll_han_topo_lvl_to_str(topo_lvl),
-                            comm->c_contextid, comm->c_name);
+                            ompi_comm_print_cid(comm), comm->c_name);
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "HAN/GATHER: No module found for the sub-communicator. "
                              "Falling back to another component\n"));
@@ -901,12 +1005,12 @@ mca_coll_han_gather_intra_dynamic(const void *sbuf, int scount,
         opal_output_verbose(verbosity, mca_coll_han_component.han_output,
                             "coll:han:mca_coll_han_gather_intra_dynamic "
                             "HAN found valid module for collective %d (%s) "
-                            "with topological level %d (%s) on communicator (%d/%s) "
+                            "with topological level %d (%s) on communicator (%s/%s) "
                             "but this module cannot handle this collective. "
                             "Please check dynamic file/mca parameters\n",
                             GATHER, mca_coll_base_colltype_to_str(GATHER),
                             topo_lvl, mca_coll_han_topo_lvl_to_str(topo_lvl),
-                            comm->c_contextid, comm->c_name);
+                            ompi_comm_print_cid(comm), comm->c_name);
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "HAN/GATHER: the module found for the sub-"
                              "communicator cannot handle the GATHER operation. "
@@ -920,10 +1024,17 @@ mca_coll_han_gather_intra_dynamic(const void *sbuf, int scount,
          * sub_module->coll_gather is valid and point to this function
          * Call han topological collective algorithm
          */
-        if(mca_coll_han_component.use_simple_algorithm[GATHER]) {
-            gather = mca_coll_han_gather_intra_simple;
-        } else {
-            gather = mca_coll_han_gather_intra;
+        int algorithm_id = get_algorithm(GATHER,
+                                         dtype_size,
+                                         comm,
+                                         han_module);
+        gather = (mca_coll_base_module_gather_fn_t) mca_coll_han_algorithm_id_to_fn(GATHER, algorithm_id);
+        if (NULL == gather) { /* default behaviour */
+            if(mca_coll_han_component.use_simple_algorithm[GATHER]) {
+                gather = mca_coll_han_gather_intra_simple;
+            } else {
+                gather = mca_coll_han_gather_intra;
+            }
         }
     } else {
         /*
@@ -941,6 +1052,103 @@ mca_coll_han_gather_intra_dynamic(const void *sbuf, int scount,
                   sub_module);
 }
 
+/*
+ * Gatherv selector:
+ * On a sub-communicator, checks the stored rules to find the module to use
+ * On the global communicator, calls the han collective implementation, or
+ * calls the correct module if fallback mechanism is activated
+ */
+int mca_coll_han_gatherv_intra_dynamic(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                                       void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t displs,
+                                       struct ompi_datatype_t *rdtype, int root,
+                                       struct ompi_communicator_t *comm,
+                                       mca_coll_base_module_t *module)
+{
+    mca_coll_han_module_t *han_module = (mca_coll_han_module_t*) module;
+    TOPO_LVL_T topo_lvl = han_module->topologic_level;
+    mca_coll_base_module_gatherv_fn_t gatherv;
+    mca_coll_base_module_t *sub_module;
+    int rank, verbosity = 0;
+
+    if (!han_module->enabled) {
+        return han_module->previous_gatherv(sbuf, scount, sdtype, rbuf, rcounts, displs, rdtype,
+                                            root, comm, han_module->previous_gatherv_module);
+    }
+
+    /* v collectives do not support message-size based dynamic rules */
+    sub_module = get_module(GATHERV, MCA_COLL_HAN_ANY_MESSAGE_SIZE, comm, han_module);
+
+    /* First errors are always printed by rank 0 */
+    rank = ompi_comm_rank(comm);
+    if( (0 == rank) && (han_module->dynamic_errors < mca_coll_han_component.max_dynamic_errors) ) {
+        verbosity = 30;
+    }
+
+    if(NULL == sub_module) {
+        /*
+         * No valid collective module from dynamic rules
+         * nor from mca parameter
+         */
+        han_module->dynamic_errors++;
+        opal_output_verbose(verbosity, mca_coll_han_component.han_output,
+                            "coll:han:mca_coll_han_gatherv_intra_dynamic "
+                            "HAN did not find any valid module for collective %d (%s) "
+                            "with topological level %d (%s) on communicator (%s/%s). "
+                            "Please check dynamic file/mca parameters\n",
+                            GATHERV, mca_coll_base_colltype_to_str(GATHERV),
+                            topo_lvl, mca_coll_han_topo_lvl_to_str(topo_lvl),
+                            ompi_comm_print_cid(comm), comm->c_name);
+        OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
+                             "HAN/GATHERV: No module found for the sub-communicator. "
+                             "Falling back to another component\n"));
+        gatherv = han_module->previous_gatherv;
+        sub_module = han_module->previous_gatherv_module;
+    } else if (NULL == sub_module->coll_gatherv) {
+        /*
+         * No valid collective from dynamic rules
+         * nor from mca parameter
+         */
+        han_module->dynamic_errors++;
+        opal_output_verbose(verbosity, mca_coll_han_component.han_output,
+                            "coll:han:mca_coll_han_gatherv_intra_dynamic "
+                            "HAN found valid module for collective %d (%s) "
+                            "with topological level %d (%s) on communicator (%s/%s) "
+                            "but this module cannot handle this collective. "
+                            "Please check dynamic file/mca parameters\n",
+                            GATHERV, mca_coll_base_colltype_to_str(GATHERV),
+                            topo_lvl, mca_coll_han_topo_lvl_to_str(topo_lvl),
+                            ompi_comm_print_cid(comm), comm->c_name);
+        OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
+                             "HAN/GATHERV: the module found for the sub-"
+                             "communicator cannot handle the GATHERV operation. "
+                             "Falling back to another component\n"));
+        gatherv = han_module->previous_gatherv;
+        sub_module = han_module->previous_gatherv_module;
+    } else if (GLOBAL_COMMUNICATOR == topo_lvl && sub_module == module) {
+        /*
+         * No fallback mechanism activated for this configuration
+         * sub_module is valid
+         * sub_module->coll_gatherv is valid and point to this function
+         * Call han topological collective algorithm
+         */
+        int algorithm_id = get_algorithm(GATHERV, MCA_COLL_HAN_ANY_MESSAGE_SIZE, comm, han_module);
+        gatherv = (mca_coll_base_module_gatherv_fn_t) mca_coll_han_algorithm_id_to_fn(GATHERV, algorithm_id);
+        if (NULL == gatherv) { /* default behaviour */
+            gatherv = mca_coll_han_gatherv_intra;
+        }
+    } else {
+        /*
+         * If we get here:
+         * sub_module is valid
+         * sub_module->coll_gatherv is valid
+         * They points to the collective to use, according to the dynamic rules
+         * Selector's job is done, call the collective
+         */
+        gatherv = sub_module->coll_gatherv;
+    }
+    return gatherv(sbuf, scount, sdtype, rbuf, rcounts, displs, rdtype, root, comm, sub_module);
+}
+
 
 /*
  * Reduce selector:
@@ -951,7 +1159,7 @@ mca_coll_han_gather_intra_dynamic(const void *sbuf, int scount,
 int
 mca_coll_han_reduce_intra_dynamic(const void *sbuf,
                                   void *rbuf,
-                                  int count,
+                                  size_t count,
                                   struct ompi_datatype_t *dtype,
                                   struct ompi_op_t *op,
                                   int root,
@@ -965,6 +1173,11 @@ mca_coll_han_reduce_intra_dynamic(const void *sbuf,
     size_t dtype_size;
     int rank, verbosity = 0;
 
+    if (!han_module->enabled) {
+        return han_module->previous_reduce(sbuf, rbuf, count, dtype, op, root, comm,
+                                           han_module->previous_reduce_module);
+    }
+
     /* Compute configuration information for dynamic rules */
     ompi_datatype_type_size(dtype, &dtype_size);
     dtype_size = dtype_size * count;
@@ -989,11 +1202,11 @@ mca_coll_han_reduce_intra_dynamic(const void *sbuf,
         opal_output_verbose(verbosity, mca_coll_han_component.han_output,
                             "coll:han:mca_coll_han_reduce_intra_dynamic "
                             "HAN did not find any valid module for collective %d (%s) "
-                            "with topological level %d (%s) on communicator (%d/%s). "
+                            "with topological level %d (%s) on communicator (%s/%s). "
                             "Please check dynamic file/mca parameters\n",
                             REDUCE, mca_coll_base_colltype_to_str(REDUCE),
                             topo_lvl, mca_coll_han_topo_lvl_to_str(topo_lvl),
-                            comm->c_contextid, comm->c_name);
+                            ompi_comm_print_cid(comm), comm->c_name);
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "HAN/REDUCE: No module found for the sub-communicator. "
                              "Falling back to another component\n"));
@@ -1008,12 +1221,12 @@ mca_coll_han_reduce_intra_dynamic(const void *sbuf,
         opal_output_verbose(verbosity, mca_coll_han_component.han_output,
                             "coll:han:mca_coll_han_reduce_intra_dynamic "
                             "HAN found valid module for collective %d (%s) "
-                            "with topological level %d (%s) on communicator (%d/%s) "
+                            "with topological level %d (%s) on communicator (%s/%s) "
                             "but this module cannot handle this collective. "
                             "Please check dynamic file/mca parameters\n",
                             REDUCE, mca_coll_base_colltype_to_str(REDUCE),
                             topo_lvl, mca_coll_han_topo_lvl_to_str(topo_lvl),
-                            comm->c_contextid, comm->c_name);
+                            ompi_comm_print_cid(comm), comm->c_name);
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "HAN/REDUCE: the module found for the sub-"
                              "communicator cannot handle the REDUCE operation. "
@@ -1021,7 +1234,7 @@ mca_coll_han_reduce_intra_dynamic(const void *sbuf,
         reduce = han_module->previous_reduce;
         sub_module = han_module->previous_reduce_module;
     } else if (GLOBAL_COMMUNICATOR == topo_lvl && sub_module == module) {
-        /* Reproducibility: fallback on reproducible algo */
+        /* Reproducibility: fallback on reproducible algorithm */
         if (mca_coll_han_component.han_reproducible) {
             reduce = mca_coll_han_reduce_reproducible;
         } else {
@@ -1031,13 +1244,19 @@ mca_coll_han_reduce_intra_dynamic(const void *sbuf,
              * sub_module->coll_reduce is valid and point to this function
              * Call han topological collective algorithm
              */
-            if(mca_coll_han_component.use_simple_algorithm[REDUCE]) {
-                reduce = mca_coll_han_reduce_intra_simple;
-            } else {
-                reduce = mca_coll_han_reduce_intra;
+            int algorithm_id = get_algorithm(REDUCE,
+                                             dtype_size,
+                                             comm,
+                                             han_module);
+            reduce = (mca_coll_base_module_reduce_fn_t)mca_coll_han_algorithm_id_to_fn(REDUCE, algorithm_id);
+            if (NULL == reduce) { /* default behaviour */
+                if(mca_coll_han_component.use_simple_algorithm[REDUCE]) {
+                    reduce = mca_coll_han_reduce_intra_simple;
+                } else {
+                    reduce = mca_coll_han_reduce_intra;
+                }
             }
         }
-        sub_module = module;
     } else {
         /*
          * If we get here:
@@ -1060,9 +1279,9 @@ mca_coll_han_reduce_intra_dynamic(const void *sbuf,
  * calls the correct module if fallback mechanism is activated
  */
 int
-mca_coll_han_scatter_intra_dynamic(const void *sbuf, int scount,
+mca_coll_han_scatter_intra_dynamic(const void *sbuf, size_t scount,
                                    struct ompi_datatype_t *sdtype,
-                                   void *rbuf, int rcount,
+                                   void *rbuf, size_t rcount,
                                    struct ompi_datatype_t *rdtype,
                                    int root,
                                    struct ompi_communicator_t *comm,
@@ -1075,6 +1294,11 @@ mca_coll_han_scatter_intra_dynamic(const void *sbuf, int scount,
     size_t dtype_size;
     int rank, verbosity = 0;
 
+    if (!han_module->enabled) {
+        return han_module->previous_scatter(sbuf, scount, sdtype, rbuf, rcount, rdtype, root, comm,
+                                            han_module->previous_scatter_module);
+    }
+
     /* Compute configuration information for dynamic rules */
     if( MPI_IN_PLACE != rbuf ) {
         ompi_datatype_type_size(rdtype, &dtype_size);
@@ -1104,11 +1328,11 @@ mca_coll_han_scatter_intra_dynamic(const void *sbuf, int scount,
         opal_output_verbose(verbosity, mca_coll_han_component.han_output,
                             "coll:han:mca_coll_han_scatter_intra_dynamic "
                             "HAN did not find any valid module for collective %d (%s) "
-                            "with topological level %d (%s) on communicator (%d/%s). "
+                            "with topological level %d (%s) on communicator (%s/%s). "
                             "Please check dynamic file/mca parameters\n",
                             SCATTER, mca_coll_base_colltype_to_str(SCATTER),
                             topo_lvl, mca_coll_han_topo_lvl_to_str(topo_lvl),
-                            comm->c_contextid, comm->c_name);
+                            ompi_comm_print_cid(comm), comm->c_name);
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "HAN/SCATTER: No module found for the sub-communicator. "
                              "Falling back to another component\n"));
@@ -1123,12 +1347,12 @@ mca_coll_han_scatter_intra_dynamic(const void *sbuf, int scount,
         opal_output_verbose(verbosity, mca_coll_han_component.han_output,
                             "coll:han:mca_coll_han_scatter_intra_dynamic "
                             "HAN found valid module for collective %d (%s) "
-                            "with topological level %d (%s) on communicator (%d/%s) "
+                            "with topological level %d (%s) on communicator (%s/%s) "
                             "but this module cannot handle this collective. "
                             "Please check dynamic file/mca parameters\n",
                             SCATTER, mca_coll_base_colltype_to_str(SCATTER),
                             topo_lvl, mca_coll_han_topo_lvl_to_str(topo_lvl),
-                            comm->c_contextid, comm->c_name);
+                            ompi_comm_print_cid(comm), comm->c_name);
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "HAN/SCATTER: the module found for the sub-"
                              "communicator cannot handle the SCATTER operation. "
@@ -1142,10 +1366,17 @@ mca_coll_han_scatter_intra_dynamic(const void *sbuf, int scount,
          * sub_module->coll_scatter is valid and point to this function
          * Call han topological collective algorithm
          */
-        if(mca_coll_han_component.use_simple_algorithm[SCATTER]) {
-            scatter = mca_coll_han_scatter_intra_simple;
-        } else {
-            scatter = mca_coll_han_scatter_intra;
+        int algorithm_id = get_algorithm(SCATTER,
+                                         dtype_size,
+                                         comm,
+                                         han_module);
+        scatter = (mca_coll_base_module_scatter_fn_t)mca_coll_han_algorithm_id_to_fn(SCATTER, algorithm_id);
+        if (NULL == scatter) { /* default behaviour */
+            if(mca_coll_han_component.use_simple_algorithm[SCATTER]) {
+                scatter = mca_coll_han_scatter_intra_simple;
+            } else {
+                scatter = mca_coll_han_scatter_intra;
+            }
         }
     } else {
         /*
@@ -1170,3 +1401,349 @@ mca_coll_han_scatter_intra_dynamic(const void *sbuf, int scount,
                    root, comm,
                    sub_module);
 }
+
+
+/*
+ * Scatterv selector:
+ * On a sub-communicator, checks the stored rules to find the module to use
+ * On the global communicator, calls the han collective implementation, or
+ * calls the correct module if fallback mechanism is activated
+ */
+int
+mca_coll_han_scatterv_intra_dynamic(const void *sbuf, ompi_count_array_t scounts,
+                                    ompi_disp_array_t displs, struct ompi_datatype_t *sdtype,
+                                    void *rbuf, size_t rcount,
+                                    struct ompi_datatype_t *rdtype, 
+                                    int root,
+                                    struct ompi_communicator_t *comm,
+                                    mca_coll_base_module_t *module)
+{
+    mca_coll_han_module_t *han_module = (mca_coll_han_module_t*) module;
+    TOPO_LVL_T topo_lvl = han_module->topologic_level;
+    mca_coll_base_module_scatterv_fn_t scatterv;
+    mca_coll_base_module_t *sub_module;
+    int rank, verbosity = 0;
+
+    if (!han_module->enabled) {
+        return han_module->previous_scatterv(sbuf, scounts, displs, sdtype, rbuf, rcount, rdtype, 
+                                             root, comm, han_module->previous_scatterv_module);
+    }
+
+    /* v collectives do not support message-size based dynamic rules */
+    sub_module = get_module(SCATTERV,
+                            MCA_COLL_HAN_ANY_MESSAGE_SIZE,
+                            comm,
+                            han_module);
+
+    /* First errors are always printed by rank 0 */
+    rank = ompi_comm_rank(comm);
+    if( (0 == rank) && (han_module->dynamic_errors < mca_coll_han_component.max_dynamic_errors) ) {
+        verbosity = 30;
+    }
+
+    if(NULL == sub_module) {
+        /*
+         * No valid collective module from dynamic rules
+         * nor from mca parameter
+         */
+        han_module->dynamic_errors++;
+        opal_output_verbose(verbosity, mca_coll_han_component.han_output,
+                            "coll:han:mca_coll_han_scatterv_intra_dynamic "
+                            "HAN did not find any valid module for collective %d (%s) "
+                            "with topological level %d (%s) on communicator (%s/%s). "
+                            "Please check dynamic file/mca parameters\n",
+                            SCATTERV, mca_coll_base_colltype_to_str(SCATTERV),
+                            topo_lvl, mca_coll_han_topo_lvl_to_str(topo_lvl),
+                            ompi_comm_print_cid(comm), comm->c_name);
+        OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
+                             "HAN/SCATTERV: No module found for the sub-communicator. "
+                             "Falling back to another component\n"));
+        scatterv = han_module->previous_scatterv;
+        sub_module = han_module->previous_scatterv_module;
+    } else if (NULL == sub_module->coll_scatterv) {
+        /*
+         * No valid collective from dynamic rules
+         * nor from mca parameter
+         */
+        han_module->dynamic_errors++;
+        opal_output_verbose(verbosity, mca_coll_han_component.han_output,
+                            "coll:han:mca_coll_han_scatterv_intra_dynamic "
+                            "HAN found valid module for collective %d (%s) "
+                            "with topological level %d (%s) on communicator (%s/%s) "
+                            "but this module cannot handle this collective. "
+                            "Please check dynamic file/mca parameters\n",
+                            SCATTERV, mca_coll_base_colltype_to_str(SCATTERV),
+                            topo_lvl, mca_coll_han_topo_lvl_to_str(topo_lvl),
+                            ompi_comm_print_cid(comm), comm->c_name);
+        OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
+                             "HAN/SCATTERV: the module found for the sub-"
+                             "communicator cannot handle the SCATTERV operation. "
+                             "Falling back to another component\n"));
+        scatterv = han_module->previous_scatterv;
+        sub_module = han_module->previous_scatterv_module;
+    } else if (GLOBAL_COMMUNICATOR == topo_lvl && sub_module == module) {
+        /*
+         * No fallback mechanism activated for this configuration
+         * sub_module is valid
+         * sub_module->coll_scatterv is valid and point to this function
+         * Call han topological collective algorithm
+         */
+        int algorithm_id = get_algorithm(SCATTERV,
+                                         MCA_COLL_HAN_ANY_MESSAGE_SIZE,
+                                         comm,
+                                         han_module);
+        scatterv = (mca_coll_base_module_scatterv_fn_t)mca_coll_han_algorithm_id_to_fn(SCATTERV, algorithm_id);
+        if (NULL == scatterv) { /* default behaviour */
+            scatterv = mca_coll_han_scatterv_intra;
+        }
+    } else {
+        /*
+         * If we get here:
+         * sub_module is valid
+         * sub_module->coll_scatterv is valid
+         * They point to the collective to use, according to the dynamic rules
+         * Selector's job is done, call the collective
+         */
+        scatterv = sub_module->coll_scatterv;
+    }
+
+    return scatterv(sbuf, scounts, displs, sdtype, 
+                    rbuf, rcount, rdtype, 
+                    root, comm, sub_module);
+}
+
+/*
+ * alltoall selector:
+ * On a sub-communicator, checks the stored rules to find the module to use
+ * On the global communicator, calls the han collective implementation, or
+ * calls the correct module if fallback mechanism is activated
+ */
+int
+mca_coll_han_alltoall_intra_dynamic(const void *sbuf, size_t scount,
+                                    struct ompi_datatype_t *sdtype,
+                                    void* rbuf, size_t rcount,
+                                    struct ompi_datatype_t *rdtype,
+                                    struct ompi_communicator_t *comm,
+                                    mca_coll_base_module_t *module)
+{
+    mca_coll_han_module_t *han_module = (mca_coll_han_module_t*) module;
+    TOPO_LVL_T topo_lvl = han_module->topologic_level;
+    mca_coll_base_module_alltoall_fn_t alltoall;
+    mca_coll_base_module_t *sub_module;
+    size_t dtype_size;
+    int rank, verbosity = 0;
+
+    if (!han_module->enabled) {
+        return han_module->previous_alltoall(sbuf, scount, sdtype, rbuf, rcount, rdtype, comm,
+                                            han_module->previous_alltoall_module);
+    }
+
+    /* Compute configuration information for dynamic rules */
+    if( MPI_IN_PLACE != rbuf ) {
+        ompi_datatype_type_size(rdtype, &dtype_size);
+        dtype_size = dtype_size * rcount;
+    } else {
+        ompi_datatype_type_size(sdtype, &dtype_size);
+        dtype_size = dtype_size * scount;
+    }
+
+    sub_module = get_module(ALLTOALL,
+                            dtype_size,
+                            comm,
+                            han_module);
+
+    /* First errors are always printed by rank 0 */
+    rank = ompi_comm_rank(comm);
+    if( (0 == rank) && (han_module->dynamic_errors < mca_coll_han_component.max_dynamic_errors) ) {
+        verbosity = 30;
+    }
+
+    if(NULL == sub_module) {
+        /*
+         * No valid collective module from dynamic rules
+         * nor from mca parameter
+         */
+        han_module->dynamic_errors++;
+        opal_output_verbose(verbosity, mca_coll_han_component.han_output,
+                            "coll:han:mca_coll_han_alltoall_intra_dynamic "
+                            "HAN did not find any valid module for collective %d (%s) "
+                            "with topological level %d (%s) on communicator (%s/%s). "
+                            "Please check dynamic file/mca parameters\n",
+                            ALLTOALL, mca_coll_base_colltype_to_str(ALLTOALL),
+                            topo_lvl, mca_coll_han_topo_lvl_to_str(topo_lvl),
+                            ompi_comm_print_cid(comm), comm->c_name);
+        OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
+                             "HAN/ALLTOALL: No module found for the sub-communicator. "
+                             "Falling back to another component\n"));
+        alltoall = han_module->previous_alltoall;
+        sub_module = han_module->previous_alltoall_module;
+    } else if (NULL == sub_module->coll_alltoall) {
+        /*
+         * No valid collective from dynamic rules
+         * nor from mca parameter
+         */
+        han_module->dynamic_errors++;
+        opal_output_verbose(verbosity, mca_coll_han_component.han_output,
+                            "coll:han:mca_coll_han_alltoall_intra_dynamic "
+                            "HAN found valid module for collective %d (%s) "
+                            "with topological level %d (%s) on communicator (%s/%s) "
+                            "but this module cannot handle this collective. "
+                            "Please check dynamic file/mca parameters\n",
+                            ALLTOALL, mca_coll_base_colltype_to_str(ALLTOALL),
+                            topo_lvl, mca_coll_han_topo_lvl_to_str(topo_lvl),
+                            ompi_comm_print_cid(comm), comm->c_name);
+        OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
+                             "HAN/ALLTOALL: the module found for the sub-"
+                             "communicator cannot handle the ALLTOALL operation. "
+                             "Falling back to another component\n"));
+        alltoall = han_module->previous_alltoall;
+        sub_module = han_module->previous_alltoall_module;
+    } else if (GLOBAL_COMMUNICATOR == topo_lvl && sub_module == module) {
+        /*
+         * No fallback mechanism activated for this configuration
+         * sub_module is valid
+         * sub_module->coll_alltoall is valid and point to this function
+         * Call han topological collective algorithm
+         */
+        int algorithm_id = get_algorithm(ALLTOALL,
+                                         dtype_size,
+                                         comm,
+                                         han_module);
+        alltoall = (mca_coll_base_module_alltoall_fn_t)mca_coll_han_algorithm_id_to_fn(ALLTOALL, algorithm_id);
+        if (NULL == alltoall) { /* default behaviour */
+            alltoall = mca_coll_han_alltoall_using_smsc;
+        }
+    } else {
+        /*
+         * If we get here:
+         * sub_module is valid
+         * sub_module->coll_alltoall is valid
+         * They points to the collective to use, according to the dynamic rules
+         * Selector's job is done, call the collective
+         */
+        alltoall = sub_module->coll_alltoall;
+    }
+
+    /*
+     * If we get here:
+     * sub_module is valid
+     * sub_module->coll_alltoall is valid
+     * They points to the collective to use, according to the dynamic rules
+     * Selector's job is done, call the collective
+     */
+    return alltoall(sbuf, scount, sdtype,
+                   rbuf, rcount, rdtype,
+                   comm,
+                   sub_module);
+}
+
+
+/*
+ * alltoallv selector:
+ * On a sub-communicator, checks the stored rules to find the module to use
+ * On the global communicator, calls the han collective implementation, or
+ * calls the correct module if fallback mechanism is activated
+ */
+int
+mca_coll_han_alltoallv_intra_dynamic(
+        ALLTOALLV_BASE_ARGS,
+        mca_coll_base_module_t *module)
+{
+    mca_coll_han_module_t *han_module = (mca_coll_han_module_t*) module;
+    TOPO_LVL_T topo_lvl = han_module->topologic_level;
+    mca_coll_base_module_alltoallv_fn_t alltoallv;
+    mca_coll_base_module_t *sub_module;
+    int rank, verbosity = 0;
+
+    if (!han_module->enabled) {
+        return han_module->previous_alltoallv(ALLTOALLV_BASE_ARG_NAMES,
+                                              han_module->previous_alltoallv_module);
+    }
+
+    /* v collectives do not support message-size based dynamic rules */
+    sub_module = get_module(ALLTOALLV,
+                            MCA_COLL_HAN_ANY_MESSAGE_SIZE,
+                            comm,
+                            han_module);
+
+    /* First errors are always printed by rank 0 */
+    rank = ompi_comm_rank(comm);
+    if( (0 == rank) && (han_module->dynamic_errors < mca_coll_han_component.max_dynamic_errors) ) {
+        verbosity = 30;
+    }
+
+    if(NULL == sub_module) {
+        /*
+         * No valid collective module from dynamic rules
+         * nor from mca parameter
+         */
+        han_module->dynamic_errors++;
+        opal_output_verbose(verbosity, mca_coll_han_component.han_output,
+                            "coll:han:mca_coll_han_alltoallv_intra_dynamic "
+                            "HAN did not find any valid module for collective %d (%s) "
+                            "with topological level %d (%s) on communicator (%s/%s). "
+                            "Please check dynamic file/mca parameters\n",
+                            ALLTOALLV, mca_coll_base_colltype_to_str(ALLTOALLV),
+                            topo_lvl, mca_coll_han_topo_lvl_to_str(topo_lvl),
+                            ompi_comm_print_cid(comm), comm->c_name);
+        OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
+                             "HAN/ALLTOALLV: No module found for the sub-communicator. "
+                             "Falling back to another component\n"));
+        alltoallv = han_module->previous_alltoallv;
+        sub_module = han_module->previous_alltoallv_module;
+    } else if (NULL == sub_module->coll_alltoallv) {
+        /*
+         * No valid collective from dynamic rules
+         * nor from mca parameter
+         */
+        han_module->dynamic_errors++;
+        opal_output_verbose(verbosity, mca_coll_han_component.han_output,
+                            "coll:han:mca_coll_han_alltoallv_intra_dynamic "
+                            "HAN found valid module for collective %d (%s) "
+                            "with topological level %d (%s) on communicator (%s/%s) "
+                            "but this module cannot handle this collective. "
+                            "Please check dynamic file/mca parameters\n",
+                            ALLTOALLV, mca_coll_base_colltype_to_str(ALLTOALLV),
+                            topo_lvl, mca_coll_han_topo_lvl_to_str(topo_lvl),
+                            ompi_comm_print_cid(comm), comm->c_name);
+        OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
+                             "HAN/ALLTOALLV: the module found for the sub-"
+                             "communicator cannot handle the ALLTOALLV operation. "
+                             "Falling back to another component\n"));
+        alltoallv = han_module->previous_alltoallv;
+        sub_module = han_module->previous_alltoallv_module;
+    } else if (GLOBAL_COMMUNICATOR == topo_lvl && sub_module == module) {
+        /*
+         * No fallback mechanism activated for this configuration
+         * sub_module is valid
+         * sub_module->coll_alltoallv is valid and point to this function
+         * Call han topological collective algorithm
+         */
+        int algorithm_id = get_algorithm(ALLTOALLV,
+                                         MCA_COLL_HAN_ANY_MESSAGE_SIZE,
+                                         comm,
+                                         han_module);
+        alltoallv = (mca_coll_base_module_alltoallv_fn_t)mca_coll_han_algorithm_id_to_fn(ALLTOALLV, algorithm_id);
+        if (NULL == alltoallv) { /* default behaviour */
+            alltoallv = mca_coll_han_alltoallv_using_smsc;
+        }
+    } else {
+        /*
+         * If we get here:
+         * sub_module is valid
+         * sub_module->coll_alltoallv is valid
+         * They points to the collective to use, according to the dynamic rules
+         * Selector's job is done, call the collective
+         */
+        alltoallv = sub_module->coll_alltoallv;
+    }
+
+    /*
+     * If we get here:
+     * sub_module is valid
+     * sub_module->coll_alltoallv is valid
+     * They points to the collective to use, according to the dynamic rules
+     * Selector's job is done, call the collective
+     */
+    return alltoallv(ALLTOALLV_BASE_ARG_NAMES, sub_module);
+}
diff --git a/ompi/mca/coll/han/coll_han_dynamic.h b/ompi/mca/coll/han/coll_han_dynamic.h
index 88e512f1d2c..82114227308 100644
--- a/ompi/mca/coll/han/coll_han_dynamic.h
+++ b/ompi/mca/coll/han/coll_han_dynamic.h
@@ -4,6 +4,9 @@
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2020      Bull S.A.S. All rights reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
+ * Copyright (c) 2023      Computer Architecture and VLSI Systems (CARV)
+ *                         Laboratory, ICS Forth. All rights reserved.
  *
  * $COPYRIGHT$
  *
@@ -23,7 +26,6 @@
 #include "ompi/mca/coll/base/coll_base_functions.h"
 #include "ompi/mca/coll/han/coll_han.h"
 
-
 /*
  * @file
  * #################################################
@@ -54,14 +56,14 @@
  *     - MCA parameter defined rules
  *     - File defined rules
  *
- * MCA parameter defined rules are stored in mca_coll_han_component.mca_rules.
- * This is a double indexed table. The first index is the coresponding collective
+ * MCA parameter defined rules are stored in mca_coll_han_component.mca_sub_components.
+ * This is a double indexed table. The first index is the corresponding collective
  * communication and the second index is the topological level aimed by the rule.
  * These parameters define the collective component to use for a specific
  * collective communication on a specific topologic level.
  *
  * File defined rules are stored in mca_coll_han_component.dynamic_rules.
- * These structures are defined bellow. The rule storage is directy deduced
+ * These structures are defined below. The rule storage is directly deduced
  * from the rule file format.
  *
  * File defined rules precede MCA parameter defined rules.
@@ -105,16 +107,16 @@ typedef enum COMPONENTS {
     SM,
     ADAPT,
     HAN,
+    XHC,
     COMPONENTS_COUNT
 } COMPONENT_T;
 
 typedef struct {
     COMPONENT_T id;
     char* component_name;
-    mca_coll_base_component_t* component;
 } ompi_coll_han_components;
 
-extern ompi_coll_han_components available_components[COMPONENTS_COUNT];
+extern ompi_coll_han_components ompi_coll_han_available_components[COMPONENTS_COUNT];
 
 /* Topologic levels */
 typedef enum TOPO_LVL {
@@ -140,6 +142,7 @@ typedef struct msg_size_rule_s {
     /* Component to use on this specific configuration
      * and message size */
     COMPONENT_T component;
+    int algorithm_id;
 } msg_size_rule_t;
 
 /* Rule for a specific configuration
@@ -210,5 +213,6 @@ typedef struct mca_coll_han_collective_modules_storage_s {
 /* Tests if a dynamic collective is implemented */
 bool mca_coll_han_is_coll_dynamic_implemented(COLLTYPE_T coll_id);
 COMPONENT_T mca_coll_han_component_name_to_id(const char* name);
+int mca_coll_han_topo_lvl_name_to_id(const char *topo_level_str);
 
 #endif
diff --git a/ompi/mca/coll/han/coll_han_dynamic_file.c b/ompi/mca/coll/han/coll_han_dynamic_file.c
index 850462f102b..0500cb99a90 100644
--- a/ompi/mca/coll/han/coll_han_dynamic_file.c
+++ b/ompi/mca/coll/han/coll_han_dynamic_file.c
@@ -1,9 +1,10 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2018-2020 The University of Tennessee and The University
+ * Copyright (c) 2018-2022 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
- * Copyright (c) 2020      Bull S.A.S. All rights reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
+ * Copyright (c) 2020-2022 Bull S.A.S. All rights reserved.
  *
  * $COPYRIGHT$
  *
@@ -28,6 +29,7 @@
 #include "coll_han.h"
 #include "coll_han_dynamic.h"
 #include "coll_han_dynamic_file.h"
+#include "coll_han_algorithms.h"
 
 #include "ompi/mca/coll/base/coll_base_util.h"
 
@@ -42,7 +44,7 @@ static int fileline = 1;
 
 /*
  * File parsing function. Allocated memory depending on the number of rules.
- * This functions expects a file formatted as describbed in coll_han_dynamic_file.h.
+ * This functions expects a file formatted as described in coll_han_dynamic_file.h.
  * The configuration is then used by coll/han component to determine which module to
  * use at each topological level.
  */
@@ -57,25 +59,29 @@ mca_coll_han_init_dynamic_rules(void)
     /* Loop counters */
     int i, j, k, l;
 
-    /* Collective informations */
-    long nb_coll, coll_id;
+    /* Collective information */
+    long nb_coll;
+    COLLTYPE_T coll_id;
+    int algorithm_id;
     char * coll_name = NULL;
+    char * algorithm_name = NULL;
+    char * target_comp_name = NULL;
     collective_rule_t *coll_rules;
 
-    /* Topo informations */
+    /* Topo information */
     long nb_topo, topo_lvl;
     topologic_rule_t *topo_rules;
 
-    /* Configuration informations */
+    /* Configuration information */
     long nb_rules, conf_size;
     configuration_rule_t *conf_rules;
 
-    /* Message size informations */
+    /* Message size information */
     long nb_msg_size;
     size_t msg_size;
     msg_size_rule_t *msg_size_rules;
 
-    /* Component informations */
+    /* Component information */
     long component;
 
     /* If the dynamic rules are not used, do not even read the file */
@@ -91,7 +97,8 @@ mca_coll_han_init_dynamic_rules(void)
         return OMPI_SUCCESS;
     }
 
-    if( NULL == (fptr = fopen(fname, "r")) ) {
+    fptr = fopen(fname, "r");
+    if( NULL == fptr ) {
         opal_output_verbose(5, mca_coll_han_component.han_output,
                             "coll:han:mca_coll_han_init_dynamic_rules cannot open dynamic file provided by "
                             "coll_han_dynamic_rules_filename=%s. Make sure it provides the  full path and "
@@ -125,9 +132,11 @@ mca_coll_han_init_dynamic_rules(void)
     /* Iterates on collective rules */
     for( i = 0 ; i < nb_coll ; i++ ) {
         coll_rules[i].nb_topologic_levels = 0;
+        coll_rules[i].topologic_rules = NULL;
         mca_coll_han_component.dynamic_rules.nb_collectives = i+1;
 
         /* Get the collective identifier */
+        free(coll_name);
         if( getnext_string(fptr, &coll_name) < 0 ) {
             opal_output_verbose(5, mca_coll_han_component.han_output,
                                 "coll:han:mca_coll_han_init_dynamic_rules invalid collective at line %d."
@@ -148,16 +157,14 @@ mca_coll_han_init_dynamic_rules(void)
                                     coll_name, fileline, ALLGATHER, COLLCOUNT);
                 goto file_reading_error;
             }
-            if( NULL != coll_name ) {
-                free(coll_name);
-            }
+            free(coll_name);
             coll_name = strdup(mca_coll_base_colltype_to_str(coll_id));
         }
 
         if(!mca_coll_han_is_coll_dynamic_implemented(coll_id)) {
             opal_output_verbose(5, mca_coll_han_component.han_output,
                                 "coll:han:mca_coll_han_init_dynamic_rules found an error on dynamic rules file %s "
-                                "read collective id %ld at line %d but this collective is not implemented yet. "
+                                "read collective id %d at line %d but this collective is not implemented yet. "
                                 "This is not an error but this set of rules will not be used\n",
                                 fname, coll_id, fileline);
         }
@@ -175,7 +182,7 @@ mca_coll_han_init_dynamic_rules(void)
             goto file_reading_error;
         }
 
-        /* Store the collective rule informations */
+        /* Store the collective rule information */
         coll_rules[i].nb_topologic_levels = nb_topo;
         coll_rules[i].collective_id = (COLLTYPE_T)coll_id;
 
@@ -198,17 +205,33 @@ mca_coll_han_init_dynamic_rules(void)
         /* Iterates on topologic rules */
         for( j = 0 ; j < nb_topo ; j++ ) {
             topo_rules[j].nb_rules = 0;
+            topo_rules[j].configuration_rules = NULL;
             coll_rules[i].nb_topologic_levels = j+1;
 
             /* Get the topologic level identifier */
-            if( (getnext_long(fptr, &topo_lvl) < 0) || (topo_lvl < INTRA_NODE) || (topo_lvl >= NB_TOPO_LVL) ) {
+            char *topo_lvl_name = NULL;
+            if( getnext_string(fptr, &topo_lvl_name) < 0 ) {
                 opal_output_verbose(5, mca_coll_han_component.han_output,
                                     "coll:han:mca_coll_han_init_dynamic_rules found an error on dynamic rules file %s "
-                                    "at line %d: an invalid topo level %ld is given or the reader encountered an unexpected EOF. "
-                                    "Topologic level must be at least %d and less than %d\n",
-                                    fname, fileline, topo_lvl, INTRA_NODE, NB_TOPO_LVL);
+                                    "at line %d: cannot read the name/id of a topo level\n",
+                                    fname, fileline);
                 goto file_reading_error;
             }
+            topo_lvl = mca_coll_han_topo_lvl_name_to_id(topo_lvl_name);
+            if (topo_lvl < 0) {
+                char *endp;
+                topo_lvl = (int)strtol(topo_lvl_name, &endp, 10);
+                if (('\0' != *endp ) || (topo_lvl < INTRA_NODE) || (topo_lvl >= NB_TOPO_LVL)) {
+                    opal_output_verbose(5, mca_coll_han_component.han_output,
+                                        "coll:han:mca_coll_han_init_dynamic_rules found an error on dynamic rules file %s "
+                                        "at line %d: unknown topo level '%s'\n",
+                                        fname, fileline, topo_lvl_name);
+                    free(topo_lvl_name);
+                    topo_lvl_name = NULL;
+                    goto file_reading_error;
+                }
+            }
+            free (topo_lvl_name);
 
             /*
              * The first information of a topologic rule
@@ -224,7 +247,7 @@ mca_coll_han_init_dynamic_rules(void)
                 goto file_reading_error;
             }
 
-            /* Store the topologic rule informations */
+            /* Store the topologic rule information */
             topo_rules[j].collective_id = coll_id;
             topo_rules[j].topologic_level = (TOPO_LVL_T)topo_lvl;
             topo_rules[j].nb_rules = nb_rules;
@@ -248,6 +271,7 @@ mca_coll_han_init_dynamic_rules(void)
             /* Iterate on configuration rules */
             for( k = 0; k < nb_rules; k++ ) {
                 conf_rules[k].nb_msg_size = 0;
+                conf_rules[k].msg_size_rules = NULL;
                 topo_rules[j].nb_rules = k+1;
 
                 /* Get the configuration size */
@@ -297,7 +321,6 @@ mca_coll_han_init_dynamic_rules(void)
 
                 /* Iterate on message size rules */
                 for( l = 0; l < nb_msg_size; l++ ) {
-                    char* target_comp_name = NULL;
                     conf_rules[k].nb_msg_size = l+1;
 
                     /* Get the message size */
@@ -314,6 +337,7 @@ mca_coll_han_init_dynamic_rules(void)
                     }
 
                     /* Get the component identifier for this message size rule */
+                    free(target_comp_name);
                     if( getnext_string(fptr, &target_comp_name) < 0 ) {
                         opal_output_verbose(5, mca_coll_han_component.han_output,
                                             "coll:han:mca_coll_han_init_dynamic_rules found an error on dynamic rules file %s "
@@ -329,16 +353,48 @@ mca_coll_han_init_dynamic_rules(void)
                                             "reader encountered an unexpected EOF. Collective component id must be at "
                                             "least %d and less than %d\n",
                                             fname, fileline, target_comp_name, SELF, COMPONENTS_COUNT);
-                        free(target_comp_name);
                         goto file_reading_error;
                     }
 
+                    /* Get the optionnal algorithm for han  */
+                    algorithm_id = 0; // default for all collectives
+                    if ((component == HAN) && (1 == ompi_coll_base_file_peek_next_char_is(fptr, &fileline, '@')) ) {
+
+                        free(algorithm_name);
+                        algorithm_name = NULL;
+                        if( getnext_string(fptr, &algorithm_name) < 0 ) {
+                            opal_output_verbose(5, mca_coll_han_component.han_output,
+                                                "coll:han:mca_coll_han_init_dynamic_rules found an error on dynamic rules file %s "
+                                                "at line %d: cannot read the name/id of an algorithm\n",
+                                                fname, fileline);
+                            goto file_reading_error;
+                        }
+                        algorithm_id = mca_coll_han_algorithm_name_to_id(coll_id, algorithm_name);
+                        if (algorithm_id < 0) {
+                            char *endp;
+                            algorithm_id = (int)strtol(algorithm_name, &endp, 10);
+                            char endc = *endp;
+                            if (('\0' != endc ) || !mca_coll_han_algorithm_id_is_valid(coll_id, algorithm_id)) {
+                                opal_output_verbose(5, mca_coll_han_component.han_output,
+                                                    "coll:han:mca_coll_han_init_dynamic_rules found an error on dynamic rules file %s "
+                                                    "at line %d: unknown algorithm '%s' for %s\n",
+                                                    fname, fileline, algorithm_name, coll_name);
+                                goto file_reading_error;
+                            }
+                        }
+                        opal_output_verbose(5, mca_coll_han_component.han_output,
+                                            "coll:han:mca_coll_han_init_dynamic_rules found for coll=%s msg_size=%ld : algorithm '%s' %d\n",
+                                            coll_name, msg_size, algorithm_name, algorithm_id);
+                    }
+
+
                     /* Store message size rule information */
                     msg_size_rules[l].collective_id = coll_id;
                     msg_size_rules[l].topologic_level = topo_lvl;
                     msg_size_rules[l].configuration_size = conf_size;
                     msg_size_rules[l].msg_size = msg_size;
                     msg_size_rules[l].component = (COMPONENT_T)component;
+                    msg_size_rules[l].algorithm_id = algorithm_id;
 
                     nb_entries++;
                     /* do we have the optional segment length */
@@ -360,19 +416,13 @@ mca_coll_han_init_dynamic_rules(void)
                                                     "file %s line %d found end of file while reading the optional list "
                                                     "of segment lengths for collective %s component %s\n",
                                                     fname, fileline, coll_name, target_comp_name);
-                                free(target_comp_name);
                                 goto file_reading_error;
                             }
                         }
                     }
-                    free(target_comp_name);
                 }
             }
         }
-        if( NULL != coll_name ) {
-            free(coll_name);
-            coll_name = NULL;
-        }
     }
 
     if( getnext_long(fptr, &nb_coll) > 0 ) {
@@ -393,6 +443,9 @@ mca_coll_han_init_dynamic_rules(void)
     fclose(fptr);
 
     check_dynamic_rules();
+    free(coll_name);
+    free(algorithm_name);
+    free(target_comp_name);
     return OMPI_SUCCESS;
 
 cannot_allocate:
@@ -402,9 +455,9 @@ mca_coll_han_init_dynamic_rules(void)
     opal_output_verbose(0, mca_coll_han_component.han_output,
                         "coll:han:mca_coll_han_init_dynamic_rules "
                         "cannot allocate dynamic rules\n");
-    if( NULL != coll_name ) {
-        free(coll_name);
-    }
+    free(coll_name);
+    free(algorithm_name);
+    free(target_comp_name);
     fclose (fptr);
     /* We disable the module, we don't need to keep the rules */
     mca_coll_han_free_dynamic_rules();
@@ -417,9 +470,9 @@ mca_coll_han_init_dynamic_rules(void)
                         "Will use mca parameters defined rules. "
                         "To see error detail, please set "
                         "collective verbosity level over 5\n");
-    if( NULL != coll_name ) {
-        free(coll_name);
-    }
+    free(coll_name);
+    free(algorithm_name);
+    free(target_comp_name);
     fclose (fptr);
     /* We disable the module, we don't need to keep the rules */
     mca_coll_han_free_dynamic_rules();
@@ -485,25 +538,25 @@ static void check_dynamic_rules(void)
     /* Loop counters */
     int i, j, k, l;
 
-    /* Collective informations */
+    /* Collective information */
     int nb_coll;
     COLLTYPE_T coll_id;
     collective_rule_t *coll_rules;
 
-    /* Topo informations */
+    /* Topo information */
     TOPO_LVL_T topo_lvl;
     topologic_rule_t *topo_rules;
 
-    /* Configuration informations */
+    /* Configuration information */
     int nb_rules, conf_size;
     configuration_rule_t *conf_rules;
 
-    /* Message size informations */
+    /* Message size information */
     int nb_msg_size;
     size_t msg_size;
     msg_size_rule_t *msg_size_rules;
 
-    /* Component informations */
+    /* Component information */
     COMPONENT_T component;
 
     nb_coll = mca_coll_han_component.dynamic_rules.nb_collectives;
@@ -564,24 +617,24 @@ void mca_coll_han_dump_dynamic_rules(void)
 {
     int nb_entries = 0;
 
-    /* Collective informations */
+    /* Collective information */
     int nb_coll;
     COLLTYPE_T coll_id;
     collective_rule_t *coll_rules;
 
-    /* Topo informations */
+    /* Topo information */
     TOPO_LVL_T topo_lvl;
     topologic_rule_t *topo_rules;
 
-    /* Configuration informations */
+    /* Configuration information */
     int nb_rules, conf_size;
     configuration_rule_t *conf_rules;
 
-    /* Message size informations */
+    /* Message size information */
     int nb_msg_size, msg_size;
     msg_size_rule_t *msg_size_rules;
 
-    /* Component informations */
+    /* Component information */
     COMPONENT_T component;
 
     nb_coll = mca_coll_han_component.dynamic_rules.nb_collectives;
@@ -609,10 +662,10 @@ void mca_coll_han_dump_dynamic_rules(void)
                     opal_output(mca_coll_han_component.han_output,
                                 "coll:han:dump_dynamic_rules %d collective %d (%s) "
                                 "topology level %d (%s) configuration size %d "
-                                "mesage size %d -> collective component %d (%s)\n",
+                                "message size %d -> collective component %d (%s)\n",
                                 nb_entries, coll_id, mca_coll_base_colltype_to_str(coll_id),
                                 topo_lvl, mca_coll_han_topo_lvl_to_str(topo_lvl), conf_size,
-                                msg_size, component, available_components[component].component_name);
+                                msg_size, component, ompi_coll_han_available_components[component].component_name);
 
                     nb_entries++;
                 }
diff --git a/ompi/mca/coll/han/coll_han_gather.c b/ompi/mca/coll/han/coll_han_gather.c
index 722697331f2..e3259267b9a 100644
--- a/ompi/mca/coll/han/coll_han_gather.c
+++ b/ompi/mca/coll/han/coll_han_gather.c
@@ -1,9 +1,11 @@
 /*
- * Copyright (c) 2018-2020 The University of Tennessee and The University
+ * Copyright (c) 2018-2023 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2020      Bull S.A.S. All rights reserved.
  * Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -34,10 +36,10 @@ mca_coll_han_set_gather_args(mca_coll_han_gather_args_t * args,
                              mca_coll_task_t * cur_task,
                              void *sbuf,
                              void *sbuf_inter_free,
-                             int scount,
+                             size_t scount,
                              struct ompi_datatype_t *sdtype,
                              void *rbuf,
-                             int rcount,
+                             size_t rcount,
                              struct ompi_datatype_t *rdtype,
                              int root,
                              int root_up_rank,
@@ -70,9 +72,9 @@ mca_coll_han_set_gather_args(mca_coll_han_gather_args_t * args,
  * Main function for taskified gather: calls lg task, a gather on low comm
  */
 int
-mca_coll_han_gather_intra(const void *sbuf, int scount,
+mca_coll_han_gather_intra(const void *sbuf, size_t scount,
                           struct ompi_datatype_t *sdtype,
-                          void *rbuf, int rcount,
+                          void *rbuf, size_t rcount,
                           struct ompi_datatype_t *rdtype,
                           int root,
                           struct ompi_communicator_t *comm,
@@ -91,10 +93,9 @@ mca_coll_han_gather_intra(const void *sbuf, int scount,
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "han cannot handle gather with this communicator. Fall back on another component\n"));
         /* HAN cannot work with this communicator so fallback on all collectives */
-        HAN_LOAD_FALLBACK_COLLECTIVES(han_module, comm);
-        return comm->c_coll->coll_gather(sbuf, scount, sdtype, rbuf,
-                                         rcount, rdtype, root,
-                                         comm, comm->c_coll->coll_gather_module);
+        HAN_LOAD_FALLBACK_COLLECTIVES(comm, han_module);
+        return han_module->previous_gather(sbuf, scount, sdtype, rbuf, rcount, rdtype, root,
+                                          comm, han_module->previous_gather_module);
     }
 
     /* Topo must be initialized to know rank distribution which then is used to
@@ -103,13 +104,10 @@ mca_coll_han_gather_intra(const void *sbuf, int scount,
     if (han_module->are_ppn_imbalanced) {
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "han cannot handle gather with this communicator (imbalance). Fall back on another component\n"));
-        /* Put back the fallback collective support and call it once. All
-         * future calls will then be automatically redirected.
-         */
-        HAN_LOAD_FALLBACK_COLLECTIVE(han_module, comm, gather);
-        return comm->c_coll->coll_gather(sbuf, scount, sdtype, rbuf,
-                                         rcount, rdtype, root,
-                                         comm, comm->c_coll->coll_gather_module);
+        /* Unregister HAN gather if possible, and execute the fallback gather */
+        HAN_UNINSTALL_COLL_API(comm, han_module, gather);
+        return han_module->previous_gather(sbuf, scount, sdtype, rbuf, rcount, rdtype, root,
+                                          comm, han_module->previous_gather_module);
     }
 
     w_rank = ompi_comm_rank(comm);
@@ -119,7 +117,7 @@ mca_coll_han_gather_intra(const void *sbuf, int scount,
     temp_request = OBJ_NEW(ompi_request_t);
     temp_request->req_state = OMPI_REQUEST_ACTIVE;
     temp_request->req_type = OMPI_REQUEST_COLL;
-    temp_request->req_free = han_request_free;
+    temp_request->req_free = ompi_coll_han_request_free;
     temp_request->req_status = (ompi_status_public_t){0};
     temp_request->req_complete = REQUEST_PENDING;
 
@@ -190,32 +188,11 @@ mca_coll_han_gather_intra(const void *sbuf, int scount,
 
     ompi_request_wait(&temp_request, MPI_STATUS_IGNORE);
 
-    /* Suppose, the expected message is 0 1 2 3 4 5 6 7 but the processes are
-     * mapped on 2 nodes, for example |0 2 4 6| |1 3 5 7|. The messages from
-     * low gather will be 0 2 4 6 and 1 3 5 7.
-     * So the upper gather result is 0 2 4 6 1 3 5 7 which must be reordered.
-     * The 3rd element (4) must be recopied at the 4th place. In general, the
-     * i-th element must be recopied at the place given by the i-th entry of the
-     * topology, which is topo[i*topolevel +1]
-     */
     /* reorder rbuf based on rank */
     if (w_rank == root && !han_module->is_mapbycore) {
-        ptrdiff_t rextent;
-        ompi_datatype_type_extent(rdtype, &rextent);
-        for (int i = 0 ; i < w_size ; i++) {
-            OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
-                                 "[%d]: Han Gather copy from %d to %d\n",
-                                 w_rank,
-                                 i * 2 + 1,
-                                 topo[i * 2 + 1]));
-            ptrdiff_t block_size = rextent * (ptrdiff_t)rcount;
-            ptrdiff_t src_shift = block_size * i;
-            ptrdiff_t dest_shift = block_size * (ptrdiff_t)topo[i * 2 + 1];
-            ompi_datatype_copy_content_same_ddt(rdtype,
-                                                (ptrdiff_t)rcount,
-                                                reorder_rbuf + src_shift,
-                                                (char *)rbuf + dest_shift);
-        }
+        ompi_coll_han_reorder_gather(reorder_buf,
+                                     rbuf, rcount, rdtype,
+                                     comm, topo);
         free(reorder_buf);
     }
 
@@ -228,15 +205,8 @@ int mca_coll_han_gather_lg_task(void *task_args)
     mca_coll_han_gather_args_t *t = (mca_coll_han_gather_args_t *) task_args;
     OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output, "[%d] Han Gather:  lg\n",
                          t->w_rank));
-    ompi_datatype_t *dtype;
-    size_t count;
-    if (t->w_rank == t->root) {
-        dtype = t->rdtype;
-        count = t->rcount;
-    } else {
-        dtype = t->sdtype;
-        count = t->scount;
-    }
+    ompi_datatype_t* dtype = (t->w_rank == t->root) ? t->rdtype : t->sdtype;
+    size_t count = (t->w_rank == t->root) ? t->rcount : t->scount;
 
     /* If the process is one of the node leader */
     char *tmp_buf = NULL;
@@ -252,18 +222,16 @@ int mca_coll_han_gather_lg_task(void *task_args)
                                    &rgap);
         tmp_buf = (char *) malloc(rsize);
         tmp_rbuf = tmp_buf - rgap;
-        if (t->w_rank == t->root) {
-            if (MPI_IN_PLACE == t->sbuf) {
-                ptrdiff_t rextent;
-                ompi_datatype_type_extent(dtype, &rextent);
-                ptrdiff_t block_size = rextent * (ptrdiff_t)count;
-                ptrdiff_t src_shift = block_size * t->w_rank;
-                ptrdiff_t dest_shift = block_size * low_rank;
-                ompi_datatype_copy_content_same_ddt(dtype,
-                                                    (ptrdiff_t)count,
-                                                    tmp_rbuf + dest_shift,
-                                                    (char *)t->rbuf + src_shift);
-            }
+        if (t->w_rank == t->root && MPI_IN_PLACE == t->sbuf) {
+            ptrdiff_t rextent;
+            ompi_datatype_type_extent(dtype, &rextent);
+            ptrdiff_t block_size = rextent * (ptrdiff_t)count;
+            ptrdiff_t src_shift = block_size * t->w_rank;
+            ptrdiff_t dest_shift = block_size * low_rank;
+            ompi_datatype_copy_content_same_ddt(dtype,
+                                                (ptrdiff_t)count,
+                                                tmp_rbuf + dest_shift,
+                                                (char *)t->rbuf + src_shift);
         }
     }
 
@@ -302,15 +270,8 @@ int mca_coll_han_gather_ug_task(void *task_args)
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "[%d] Han Gather:  ug noop\n", t->w_rank));
     } else {
-        ompi_datatype_t *dtype;
-        size_t count;
-        if (t->w_rank == t->root) {
-            dtype = t->rdtype;
-            count = t->rcount;
-        } else {
-            dtype = t->sdtype;
-            count = t->scount;
-        }
+        ompi_datatype_t* dtype = (t->w_rank == t->root) ? t->rdtype : t->sdtype;
+        size_t count = (t->w_rank == t->root) ? t->rcount : t->scount;
 
 
         int low_size = ompi_comm_size(t->low_comm);
@@ -340,9 +301,9 @@ int mca_coll_han_gather_ug_task(void *task_args)
 
 /* only work with regular situation (each node has equal number of processes) */
 int
-mca_coll_han_gather_intra_simple(const void *sbuf, int scount,
+mca_coll_han_gather_intra_simple(const void *sbuf, size_t scount,
                                  struct ompi_datatype_t *sdtype,
-                                 void *rbuf, int rcount,
+                                 void *rbuf, size_t rcount,
                                  struct ompi_datatype_t *rdtype,
                                  int root,
                                  struct ompi_communicator_t *comm,
@@ -357,10 +318,9 @@ mca_coll_han_gather_intra_simple(const void *sbuf, int scount,
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "han cannot handle gather with this communicator. Fall back on another component\n"));
         /* HAN cannot work with this communicator so fallback on all collectives */
-        HAN_LOAD_FALLBACK_COLLECTIVES(han_module, comm);
-        return comm->c_coll->coll_gather(sbuf, scount, sdtype, rbuf,
-                                         rcount, rdtype, root,
-                                         comm, comm->c_coll->coll_gather_module);
+        HAN_LOAD_FALLBACK_COLLECTIVES(comm, han_module);
+        return han_module->previous_gather(sbuf, scount, sdtype, rbuf, rcount, rdtype, root,
+                                          comm, han_module->previous_gather_module);
     }
 
     /* Topo must be initialized to know rank distribution which then is used to
@@ -369,28 +329,17 @@ mca_coll_han_gather_intra_simple(const void *sbuf, int scount,
     if (han_module->are_ppn_imbalanced){
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "han cannot handle gather with this communicator (imbalance). Fall back on another component\n"));
-        /* Put back the fallback collective support and call it once. All
-         * future calls will then be automatically redirected.
-         */
-        HAN_LOAD_FALLBACK_COLLECTIVE(han_module, comm, gather);
-        return comm->c_coll->coll_gather(sbuf, scount, sdtype, rbuf,
-                                         rcount, rdtype, root,
-                                         comm, comm->c_coll->coll_gather_module);
+        /* Unregister HAN gather if possible, and execute the fallback gather */
+        HAN_UNINSTALL_COLL_API(comm, han_module, gather);
+        return han_module->previous_gather(sbuf, scount, sdtype, rbuf, rcount, rdtype, root,
+                                          comm, han_module->previous_gather_module);
     }
 
     ompi_communicator_t *low_comm = han_module->sub_comm[INTRA_NODE];
     ompi_communicator_t *up_comm = han_module->sub_comm[INTER_NODE];
-    ompi_datatype_t *dtype;
-    size_t count;
-
-    if (w_rank == root) {
-        dtype = rdtype;
-        count = rcount;
-    } else {
-        dtype = sdtype;
-        count = scount;
-    }
 
+    ompi_datatype_t* dtype = (w_rank == root) ? rdtype : sdtype;
+    size_t count = (w_rank == root) ? rcount : scount;
 
     /* Get the 'virtual ranks' mapping corresponding to the communicators */
     int *vranks = han_module->cached_vranks;
@@ -408,6 +357,11 @@ mca_coll_han_gather_intra_simple(const void *sbuf, int scount,
     char *reorder_buf = NULL;  // allocated memory
     char *reorder_buf_start = NULL; // start of the data
     if (w_rank == root) {
+        if (MPI_IN_PLACE == sbuf) {
+            ptrdiff_t rextent;
+            ompi_datatype_type_extent(rdtype, &rextent);
+            sbuf = (char*)rbuf + rextent * (ptrdiff_t)rcount * w_rank;
+        }
         if (han_module->is_mapbycore) {
             reorder_buf_start = (char *)rbuf;
         } else {
@@ -495,7 +449,7 @@ mca_coll_han_gather_intra_simple(const void *sbuf, int scount,
  */
 void
 ompi_coll_han_reorder_gather(const void *sbuf,
-                             void *rbuf, int count,
+                             void *rbuf, size_t count,
                              struct ompi_datatype_t *dtype,
                              struct ompi_communicator_t *comm,
                              int * topo)
@@ -507,13 +461,11 @@ ompi_coll_han_reorder_gather(const void *sbuf,
     int w_size = ompi_comm_size(comm);
     ptrdiff_t rextent;
     ompi_datatype_type_extent(dtype, &rextent);
+    const ptrdiff_t block_size = rextent * (ptrdiff_t)count;
     for ( i = 0; i < w_size; i++ ) {
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
-                             "[%d]: Future reorder from %d to %d\n",
-                             w_rank,
-                             i * topolevel + 1,
-                             topo[i * topolevel + 1]));
-        ptrdiff_t block_size = rextent * (ptrdiff_t)count;
+                             "[%d]: HAN Gather reorder from %d to %d\n",
+                             w_rank, i, topo[i * topolevel + 1]));
         ptrdiff_t src_shift = block_size * i;
         ptrdiff_t dest_shift = block_size * (ptrdiff_t)topo[i * topolevel + 1];
         ompi_datatype_copy_content_same_ddt(dtype,
diff --git a/ompi/mca/coll/han/coll_han_gatherv.c b/ompi/mca/coll/han/coll_han_gatherv.c
new file mode 100644
index 00000000000..fd600b9d1cd
--- /dev/null
+++ b/ompi/mca/coll/han/coll_han_gatherv.c
@@ -0,0 +1,371 @@
+/*
+ * Copyright (c) 2018-2023 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2020      Bull S.A.S. All rights reserved.
+ * Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
+ * Copyright (c)           Amazon.com, Inc. or its affiliates.
+ *                         All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "coll_han.h"
+#include "coll_han_trigger.h"
+#include "ompi/mca/coll/base/coll_base_functions.h"
+#include "ompi/mca/coll/base/coll_tags.h"
+#include "ompi/mca/pml/pml.h"
+
+/*
+ * @file
+ *
+ * This files contains the hierarchical implementations of gatherv.
+ * Only work with regular situation (each node has equal number of processes).
+ */
+
+/*
+ * Implement hierarchical Gatherv to optimize large-scale communications where multiple nodes and
+ * multiple processes per node send non-zero sized messages to the root, i.e. high incast.
+ *
+ * In Gatherv, only the root(receiver) process has the information of the amount of data, i.e.
+ * datatype and count, from each sender process. Therefore node leaders need an additional step to
+ * collect the expected data from its local peers. In summary, the steps are:
+ * 1. Root
+ *      a. Receive data from local peers (Low Gatherv)
+ *      b. Receive data from other node leaders (Up Gatherv)
+ *      c. If necessary reorder data from node leaders(see discussion below)
+ * 2. Root's local peers
+ *      a. Send data to root. (Low Gatherv)
+ * 3. Node leaders:
+ *      a. Collect the data transfer sizes(in bytes) from local peers (Low Gather)
+ *      b. Receive data from local peers (Low Gatherv)
+ *      c. Send data to the root (Up Gatherv)
+ * 4. Node followers:
+ *      a. Send the data transfer size(in bytes) to the node leader (Low Gather)
+ *      b. Send data to the node leader (Low Gatherv)
+ *
+ * Note on reodering:
+ * In Up Gatherv, data from each node is stored in a contiguous buffer sorted by the sender's
+ * local rank, and MUST be reordered according to the root's displacement requirement on the output
+ * buffer. Concretely, reordering can avoided if and only if both of following conditions are met:
+ * 1. Data from processes on each node, other than the root's node, are placed in the output buffer
+ *    in the same **order** as their local ranks. Note, it is possible to receive the data in the
+ *    correct order even if the process are NOT mapped by core.
+ * 2. No **gap** exists between data from the same node, other than the root's node, in the output
+ *    buffer - it is ok if data from different nodes has gap.
+ */
+int mca_coll_han_gatherv_intra(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                               void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t displs,
+                               struct ompi_datatype_t *rdtype, int root,
+                               struct ompi_communicator_t *comm, mca_coll_base_module_t *module)
+{
+    mca_coll_han_module_t *han_module = (mca_coll_han_module_t *) module;
+    int w_rank, w_size;              /* information about the global communicator */
+    int root_low_rank, root_up_rank; /* root ranks for both sub-communicators */
+    int err, *vranks, low_rank, low_size, up_rank, up_size, *topo;
+    size_t *low_rcounts = NULL;
+    ptrdiff_t *low_displs = NULL;
+    ompi_count_array_t low_rcounts_desc;
+    ompi_disp_array_t low_displs_desc;
+
+    /* Create the subcommunicators */
+    err = mca_coll_han_comm_create(comm, han_module);
+    if (OMPI_SUCCESS != err) {
+        OPAL_OUTPUT_VERBOSE(
+            (30, mca_coll_han_component.han_output,
+             "han cannot handle gatherv with this communicator. Fall back on another component\n"));
+        /* HAN cannot work with this communicator so fallback on all collectives */
+        HAN_LOAD_FALLBACK_COLLECTIVES(comm, han_module);
+        return han_module->previous_gatherv(sbuf, scount, sdtype, rbuf, rcounts, displs, rdtype,
+                                            root, comm, han_module->previous_gatherv_module);
+    }
+
+    /* Topo must be initialized to know rank distribution which then is used to determine if han can
+     * be used */
+    topo = mca_coll_han_topo_init(comm, han_module, 2);
+    if (han_module->are_ppn_imbalanced) {
+        OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
+                             "han cannot handle gatherv with this communicator (imbalance). Fall "
+                             "back on another component\n"));
+        /* Put back the fallback collective support and call it once. All
+         * future calls will then be automatically redirected.
+         */
+        HAN_UNINSTALL_COLL_API(comm, han_module, gatherv);
+        return han_module->previous_gatherv(sbuf, scount, sdtype, rbuf, rcounts, displs, rdtype,
+                                            root, comm, han_module->previous_gatherv_module);
+    }
+
+    w_rank = ompi_comm_rank(comm);
+    w_size = ompi_comm_size(comm);
+
+    /* create the subcommunicators */
+    ompi_communicator_t *low_comm
+        = han_module->cached_low_comms[mca_coll_han_component.han_gatherv_low_module];
+    ompi_communicator_t *up_comm
+        = han_module->cached_up_comms[mca_coll_han_component.han_gatherv_up_module];
+
+    /* Get the 'virtual ranks' mapping corresponding to the communicators */
+    vranks = han_module->cached_vranks;
+    /* information about sub-communicators */
+    low_rank = ompi_comm_rank(low_comm);
+    low_size = ompi_comm_size(low_comm);
+    up_rank = ompi_comm_rank(up_comm);
+    up_size = ompi_comm_size(up_comm);
+    /* Get root ranks for low and up comms */
+    mca_coll_han_get_ranks(vranks, root, low_size, &root_low_rank, &root_up_rank);
+
+    OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
+                         "[%d]: Han Gatherv root %d root_low_rank %d root_up_rank %d\n", w_rank,
+                         root, root_low_rank, root_up_rank));
+
+    err = OMPI_SUCCESS;
+    /* #################### Root ########################### */
+    if (root == w_rank) {
+        int need_bounce_buf = 0;
+        size_t total_up_rcounts = 0;
+        ptrdiff_t *up_displs = NULL;
+        size_t *up_rcounts = NULL, *up_peer_ub = NULL;
+        ompi_count_array_t up_rcounts_desc;
+        ompi_disp_array_t up_displs_desc;
+        char *bounce_buf = NULL;
+
+        low_rcounts = malloc(low_size * sizeof(size_t));
+        low_displs = malloc(low_size * sizeof(ptrdiff_t));
+        if (!low_rcounts || !low_displs) {
+            err = OMPI_ERR_OUT_OF_RESOURCE;
+            goto root_out;
+        }
+
+        int low_peer, up_peer, w_peer;
+        for (w_peer = 0; w_peer < w_size; ++w_peer) {
+            mca_coll_han_get_ranks(vranks, w_peer, low_size, &low_peer, &up_peer);
+            if (root_up_rank != up_peer) {
+                /* Not a local peer */
+                continue;
+            }
+            low_displs[low_peer] = ompi_disp_array_get(displs, w_peer);
+            low_rcounts[low_peer] = ompi_count_array_get(rcounts, w_peer);
+        }
+
+        /* Low Gatherv */
+        OMPI_COUNT_ARRAY_INIT(&low_rcounts_desc, low_rcounts);
+        OMPI_DISP_ARRAY_INIT(&low_displs_desc, low_displs);
+        low_comm->c_coll->coll_gatherv(sbuf, scount, sdtype, rbuf, low_rcounts_desc,
+                                       low_displs_desc, rdtype, root_low_rank, low_comm,
+                                       low_comm->c_coll->coll_gatherv_module);
+
+        char *tmp_rbuf = rbuf;
+
+        up_rcounts = calloc(up_size, sizeof(size_t));
+        up_displs = malloc(up_size * sizeof(ptrdiff_t));
+        up_peer_ub = calloc(up_size, sizeof(size_t));
+        if (!up_rcounts || !up_displs || !up_peer_ub) {
+            err = OMPI_ERR_OUT_OF_RESOURCE;
+            goto root_out;
+        }
+
+        for (up_peer = 0; up_peer < up_size; ++up_peer) {
+            up_displs[up_peer] = PTRDIFF_MAX;
+        }
+
+        /* Calculate recv counts for the inter-node gatherv - no need to gather
+         * from self again because the data is already in place */
+        for (w_peer = 0; w_peer < w_size; ++w_peer) {
+            mca_coll_han_get_ranks(vranks, w_peer, low_size, NULL, &up_peer);
+
+            if (!need_bounce_buf && root_up_rank != up_peer && 0 < ompi_count_array_get(rcounts, w_peer) && 0 < w_peer
+                && ompi_disp_array_get(displs, w_peer) < ompi_disp_array_get(displs, w_peer - 1)) {
+                /* Data is not placed in the rank order so reordering is needed */
+                need_bounce_buf = 1;
+            }
+
+            if (root_up_rank == up_peer) {
+                /* No need to gather data on the same node again */
+                continue;
+            }
+
+            up_peer_ub[up_peer] = 0 < ompi_count_array_get(rcounts, w_peer)
+                                          && ompi_disp_array_get(displs, w_peer) + ompi_count_array_get(rcounts, w_peer) > up_peer_ub[up_peer]
+                                      ? ompi_disp_array_get(displs, w_peer) + ompi_count_array_get(rcounts, w_peer)
+                                      : up_peer_ub[up_peer];
+
+            up_rcounts[up_peer] += ompi_count_array_get(rcounts, w_peer);
+            total_up_rcounts += ompi_count_array_get(rcounts, w_peer);
+
+            /* Optimize for the happy path */
+            up_displs[up_peer] = 0 < ompi_count_array_get(rcounts, w_peer) && ompi_disp_array_get(displs, w_peer) < up_displs[up_peer]
+                                     ? ompi_disp_array_get(displs, w_peer)
+                                     : up_displs[up_peer];
+        }
+
+        /* If the data is not placed contiguously on recv buf, then we will need temp buf to store
+         * the gap data and recover it later */
+        for (up_peer = 0; up_peer < up_size; ++up_peer) {
+            if (root_up_rank == up_peer) {
+                continue;
+            }
+            if (!need_bounce_buf && 0 < up_rcounts[up_peer]
+                && up_rcounts[up_peer] < up_peer_ub[up_peer] - up_displs[up_peer]) {
+                need_bounce_buf = 1;
+                break;
+            }
+        }
+
+        if (need_bounce_buf) {
+            ptrdiff_t rsize, rgap;
+            rsize = opal_datatype_span(&rdtype->super, total_up_rcounts, &rgap);
+            bounce_buf = malloc(rsize);
+            if (!bounce_buf) {
+                err = OMPI_ERR_OUT_OF_RESOURCE;
+                goto root_out;
+            }
+
+            /* Calculate displacements for the inter-node gatherv */
+            for (up_peer = 0; up_peer < up_size; ++up_peer) {
+                up_displs[up_peer] = 0 < up_peer ? up_displs[up_peer - 1] + up_rcounts[up_peer - 1]
+                                                 : 0;
+            }
+
+            tmp_rbuf = bounce_buf - rgap;
+        }
+
+        /* Up Gatherv */
+        OMPI_COUNT_ARRAY_INIT(&up_rcounts_desc, up_rcounts);
+        OMPI_DISP_ARRAY_INIT(&up_displs_desc, up_displs);
+        up_comm->c_coll->coll_gatherv(sbuf, 0, sdtype, tmp_rbuf, up_rcounts_desc, up_displs_desc, rdtype,
+                                      root_up_rank, up_comm, up_comm->c_coll->coll_gatherv_module);
+
+        /* Use a temp buffer to reorder the output buffer if needed */
+        if (need_bounce_buf) {
+            ptrdiff_t offset = 0, rdext;
+            ompi_datatype_type_extent(rdtype, &rdext);
+
+            for (int i = 0; i < w_size; ++i) {
+                up_peer = topo[2 * i];
+                if (root_up_rank == up_peer) {
+                    continue;
+                }
+
+                w_peer = topo[2 * i + 1];
+
+                ompi_datatype_copy_content_same_ddt(rdtype, ompi_count_array_get(rcounts, w_peer),
+                                                    (char *) rbuf + ompi_disp_array_get(displs, w_peer) * rdext,
+                                                    bounce_buf + offset);
+                offset += rdext * ompi_count_array_get(rcounts, w_peer);
+            }
+        }
+
+    root_out:
+        if (low_displs) {
+            free(low_displs);
+        }
+        if (low_rcounts) {
+            free(low_rcounts);
+        }
+        if (up_displs) {
+            free(up_displs);
+        }
+        if (up_rcounts) {
+            free(up_rcounts);
+        }
+        if (up_peer_ub) {
+            free(up_peer_ub);
+        }
+        if (bounce_buf) {
+            free(bounce_buf);
+        }
+
+        return err;
+    }
+
+    /* #################### Root's local peers ########################### */
+    if (root_up_rank == up_rank) {
+        /* Low Gatherv */
+        low_comm->c_coll->coll_gatherv(sbuf, scount, sdtype, NULL, 0, 0, NULL, root_low_rank,
+                                       low_comm, low_comm->c_coll->coll_gatherv_module);
+        return OMPI_SUCCESS;
+    }
+
+    size_t sdsize = 0;
+    size_t send_size = 0;
+
+    ompi_datatype_type_size(sdtype, &sdsize);
+    send_size = sdsize * scount;
+
+    /* #################### Other node followers ########################### */
+    if (root_low_rank != low_rank) {
+        /* Low Gather - Gather each local peer's send data size */
+        low_comm->c_coll->coll_gather((const void *) &send_size, sizeof(size_t), MPI_BYTE, NULL,
+                                      sizeof(size_t), MPI_BYTE, root_low_rank, low_comm,
+                                      low_comm->c_coll->coll_gather_module);
+        /* Low Gatherv */
+        low_comm->c_coll->coll_gatherv(sbuf, scount, sdtype, NULL, 0, 0, NULL, root_low_rank,
+                                       low_comm, low_comm->c_coll->coll_gatherv_module);
+        return OMPI_SUCCESS;
+    }
+
+    /* #################### Node leaders ########################### */
+
+    char *tmp_buf = NULL;
+
+    /* Allocate a temporary array to gather the data size, i.e. data type size x count,
+     * in bytes from local peers */
+    low_rcounts = malloc(low_size * sizeof(size_t));
+    if (!low_rcounts) {
+        err = OMPI_ERR_OUT_OF_RESOURCE;
+        goto node_leader_out;
+    }
+
+    /* Low Gather -  Gather local peers' send data sizes */
+    low_comm->c_coll->coll_gather((const void *) &send_size, sizeof(size_t), MPI_BYTE,
+                                  (void *) low_rcounts, sizeof(size_t), MPI_BYTE, root_low_rank, low_comm,
+                                  low_comm->c_coll->coll_gather_module);
+
+    low_displs = malloc(low_size * sizeof(ptrdiff_t));
+    if (!low_displs) {
+        err = OMPI_ERR_OUT_OF_RESOURCE;
+        goto node_leader_out;
+    }
+
+    size_t total_rsize = 0;
+    for (int i = 0; i < low_size; ++i) {
+        low_displs[i] = i > 0 ? low_displs[i - 1] + low_rcounts[i - 1] : 0;
+        total_rsize += low_rcounts[i];
+    }
+
+    tmp_buf = (char *) malloc(total_rsize); /* tmp_buf is still valid if total_rsize is 0 */
+    if (!tmp_buf) {
+        err = OMPI_ERR_OUT_OF_RESOURCE;
+        goto node_leader_out;
+    }
+
+    /* Low Gatherv */
+    OMPI_COUNT_ARRAY_INIT(&low_rcounts_desc, low_rcounts);
+    OMPI_DISP_ARRAY_INIT(&low_displs_desc, low_displs);
+    low_comm->c_coll->coll_gatherv(sbuf, scount, sdtype, (void *) tmp_buf,
+                                   low_rcounts_desc, low_displs_desc,
+                                   MPI_BYTE, root_low_rank, low_comm,
+                                   low_comm->c_coll->coll_gatherv_module);
+
+    /* Up Gatherv */
+    up_comm->c_coll->coll_gatherv(tmp_buf, total_rsize, MPI_BYTE, NULL, 0, 0, NULL,
+                                  root_up_rank, up_comm, up_comm->c_coll->coll_gatherv_module);
+
+node_leader_out:
+    if (low_rcounts) {
+        free(low_rcounts);
+    }
+    if (low_displs) {
+        free(low_displs);
+    }
+    if (tmp_buf) {
+        free(tmp_buf);
+    }
+
+    return err;
+}
diff --git a/ompi/mca/coll/han/coll_han_module.c b/ompi/mca/coll/han/coll_han_module.c
index f926e4de1a3..df6f0d08c2e 100644
--- a/ompi/mca/coll/han/coll_han_module.c
+++ b/ompi/mca/coll/han/coll_han_module.c
@@ -3,6 +3,12 @@
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2020      Bull S.A.S. All rights reserved.
+ * Copyright (c) 2021      Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2021      Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2024      Amazon.com, Inc. or its affiliates. All Rights Reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -19,21 +25,22 @@
 
 /*
  *@file
- * Coll han module managment file. Used for each new communicator.
+ * Coll han module management file. Used for each new communicator.
  */
 
 /*
  * Local functions
  */
-static int han_module_enable(mca_coll_base_module_t * module,
-                             struct ompi_communicator_t *comm);
+static int mca_coll_han_module_enable(mca_coll_base_module_t * module,
+                                      struct ompi_communicator_t *comm);
 static int mca_coll_han_module_disable(mca_coll_base_module_t * module,
                                        struct ompi_communicator_t *comm);
 
-#define CLEAN_PREV_COLL(HANDLE, NAME)                    \
-    do {                                                 \
-        (HANDLE)->fallback.NAME.module_fn.NAME = NULL;   \
-        (HANDLE)->fallback.NAME.module = NULL;           \
+#define CLEAN_PREV_COLL(__module, __api)              \
+    do                                                \
+    {                                                 \
+        (__module)->previous_##__api          = NULL; \
+        (__module)->previous_##__api##_module = NULL; \
     } while (0)
 
 /*
@@ -41,6 +48,8 @@ static int mca_coll_han_module_disable(mca_coll_base_module_t * module,
  */
 static void han_module_clear(mca_coll_han_module_t *han_module)
 {
+    CLEAN_PREV_COLL(han_module, alltoall);
+    CLEAN_PREV_COLL(han_module, alltoallv);
     CLEAN_PREV_COLL(han_module, allgather);
     CLEAN_PREV_COLL(han_module, allgatherv);
     CLEAN_PREV_COLL(han_module, allreduce);
@@ -48,7 +57,9 @@ static void han_module_clear(mca_coll_han_module_t *han_module)
     CLEAN_PREV_COLL(han_module, bcast);
     CLEAN_PREV_COLL(han_module, reduce);
     CLEAN_PREV_COLL(han_module, gather);
+    CLEAN_PREV_COLL(han_module, gatherv);
     CLEAN_PREV_COLL(han_module, scatter);
+    CLEAN_PREV_COLL(han_module, scatterv);
 
     han_module->reproducible_reduce = NULL;
     han_module->reproducible_reduce_module = NULL;
@@ -65,7 +76,6 @@ static void mca_coll_han_module_construct(mca_coll_han_module_t * module)
 
     module->enabled = true;
     module->recursive_free_depth = 0;
-    module->super.coll_module_disable = mca_coll_han_module_disable;
     module->cached_low_comms = NULL;
     module->cached_up_comms = NULL;
     module->cached_vranks = NULL;
@@ -84,14 +94,6 @@ static void mca_coll_han_module_construct(mca_coll_han_module_t * module)
     han_module_clear(module);
 }
 
-
-#define OBJ_RELEASE_IF_NOT_NULL(obj)            \
-    do {                                        \
-        if (NULL != (obj)) {                    \
-            OBJ_RELEASE(obj);                   \
-        }                                       \
-    } while (0)
-
 /*
  * Module destructor
  */
@@ -140,13 +142,6 @@ mca_coll_han_module_destruct(mca_coll_han_module_t * module)
         }
     }
 
-    OBJ_RELEASE_IF_NOT_NULL(module->previous_allgather_module);
-    OBJ_RELEASE_IF_NOT_NULL(module->previous_allreduce_module);
-    OBJ_RELEASE_IF_NOT_NULL(module->previous_bcast_module);
-    OBJ_RELEASE_IF_NOT_NULL(module->previous_gather_module);
-    OBJ_RELEASE_IF_NOT_NULL(module->previous_reduce_module);
-    OBJ_RELEASE_IF_NOT_NULL(module->previous_scatter_module);
-
     han_module_clear(module);
 }
 
@@ -186,21 +181,14 @@ mca_coll_han_comm_query(struct ompi_communicator_t * comm, int *priority)
      */
     if (OMPI_COMM_IS_INTER(comm)) {
         opal_output_verbose(10, ompi_coll_base_framework.framework_output,
-                            "coll:han:comm_query (%d/%s): intercomm; disqualifying myself",
-                            comm->c_contextid, comm->c_name);
+                            "coll:han:comm_query (%s/%s): intercomm; disqualifying myself",
+                            ompi_comm_print_cid(comm), comm->c_name);
         return NULL;
     }
     if (1 == ompi_comm_size(comm)) {
         opal_output_verbose(10, ompi_coll_base_framework.framework_output,
-                            "coll:han:comm_query (%d/%s): comm is too small; disqualifying myself",
-                            comm->c_contextid, comm->c_name);
-        return NULL;
-    }
-    if( !ompi_group_have_remote_peers(comm->c_local_group) ) {
-        /* The group only contains local processes. Disable HAN for now */
-        opal_output_verbose(10, ompi_coll_base_framework.framework_output,
-                            "coll:han:comm_query (%d/%s): comm has only local processes; disqualifying myself",
-                            comm->c_contextid, comm->c_name);
+                            "coll:han:comm_query (%s/%s): comm is too small; disqualifying myself",
+                            ompi_comm_print_cid(comm), comm->c_name);
         return NULL;
     }
     /* Get the priority level attached to this module. If priority is less
@@ -208,52 +196,60 @@ mca_coll_han_comm_query(struct ompi_communicator_t * comm, int *priority)
     *priority = mca_coll_han_component.han_priority;
     if (mca_coll_han_component.han_priority < 0) {
         opal_output_verbose(10, ompi_coll_base_framework.framework_output,
-                            "coll:han:comm_query (%d/%s): priority too low; disqualifying myself",
-                            comm->c_contextid, comm->c_name);
+                            "coll:han:comm_query (%s/%s): priority too low; disqualifying myself",
+                            ompi_comm_print_cid(comm), comm->c_name);
         return NULL;
     }
 
-    han_module = OBJ_NEW(mca_coll_han_module_t);
-    if (NULL == han_module) {
-        return NULL;
-    }
-
-    /* All is good -- return a module */
-    han_module->topologic_level = GLOBAL_COMMUNICATOR;
+    int topologic_level = GLOBAL_COMMUNICATOR;
 
     if (NULL != comm->super.s_info) {
-        char info_val[OPAL_MAX_INFO_VAL+1];
-
         /* Get the info value disaqualifying coll components */
+        opal_cstring_t *info_str;
         opal_info_get(comm->super.s_info, "ompi_comm_coll_han_topo_level",
-                      sizeof(info_val), info_val, &flag);
+                      &info_str, &flag);
 
         if (flag) {
-            if (0 == strcmp(info_val, "INTER_NODE")) {
-                han_module->topologic_level = INTER_NODE;
-            } else {
-                han_module->topologic_level = INTRA_NODE;
-            }
+            topologic_level = strcmp(info_str->string, "INTER_NODE") ? INTRA_NODE : INTER_NODE;
+            OBJ_RELEASE(info_str);
         }
     }
 
-    han_module->super.coll_module_enable = han_module_enable;
-    han_module->super.coll_alltoall   = NULL;
-    han_module->super.coll_alltoallv  = NULL;
+    if( !ompi_group_have_remote_peers(comm->c_local_group)
+            && INTRA_NODE != topologic_level ) {
+        /* The group only contains local processes, and this is not a
+         * intra-node subcomm we created. Disable HAN for now */
+        opal_output_verbose(10, ompi_coll_base_framework.framework_output,
+                            "coll:han:comm_query (%s/%s): comm has only local processes; disqualifying myself",
+                            ompi_comm_print_cid(comm), comm->c_name);
+        return NULL;
+    }
+
+    /* All is good -- return a module */
+    han_module = OBJ_NEW(mca_coll_han_module_t);
+    if (NULL == han_module) {
+        return NULL;
+    }
+    han_module->topologic_level = topologic_level;
+
+    han_module->super.coll_module_enable = mca_coll_han_module_enable;
+    han_module->super.coll_module_disable = mca_coll_han_module_disable;
+
+    han_module->super.coll_alltoall   = mca_coll_han_alltoall_intra_dynamic;
+    han_module->super.coll_alltoallv  = mca_coll_han_alltoallv_intra_dynamic;
     han_module->super.coll_alltoallw  = NULL;
     han_module->super.coll_exscan     = NULL;
-    han_module->super.coll_gatherv    = NULL;
     han_module->super.coll_reduce_scatter = NULL;
     han_module->super.coll_scan       = NULL;
-    han_module->super.coll_scatterv   = NULL;
+    han_module->super.coll_scatterv   = mca_coll_han_scatterv_intra_dynamic;
     han_module->super.coll_barrier    = mca_coll_han_barrier_intra_dynamic;
     han_module->super.coll_scatter    = mca_coll_han_scatter_intra_dynamic;
     han_module->super.coll_reduce     = mca_coll_han_reduce_intra_dynamic;
     han_module->super.coll_gather     = mca_coll_han_gather_intra_dynamic;
+    han_module->super.coll_gatherv    = mca_coll_han_gatherv_intra_dynamic;
     han_module->super.coll_bcast      = mca_coll_han_bcast_intra_dynamic;
     han_module->super.coll_allreduce  = mca_coll_han_allreduce_intra_dynamic;
     han_module->super.coll_allgather  = mca_coll_han_allgather_intra_dynamic;
-
     if (GLOBAL_COMMUNICATOR == han_module->topologic_level) {
         /* We are on the global communicator, return topological algorithms */
         han_module->super.coll_allgatherv = NULL;
@@ -263,8 +259,8 @@ mca_coll_han_comm_query(struct ompi_communicator_t * comm, int *priority)
     }
 
     opal_output_verbose(10, ompi_coll_base_framework.framework_output,
-                        "coll:han:comm_query (%d/%s): pick me! pick me!",
-                        comm->c_contextid, comm->c_name);
+                        "coll:han:comm_query (%s/%s): pick me! pick me!",
+                        ompi_comm_print_cid(comm), comm->c_name);
     return &(han_module->super);
 }
 
@@ -275,53 +271,50 @@ mca_coll_han_comm_query(struct ompi_communicator_t * comm, int *priority)
  * . ompi_communicator_t *comm
  * . mca_coll_han_module_t *han_module
  */
-#define HAN_SAVE_PREV_COLL_API(__api)                                   \
+#define HAN_INSTALL_COLL_API(__comm, __module, __api)                   \
     do {                                                                \
-        if (!comm->c_coll->coll_ ## __api || !comm->c_coll->coll_ ## __api ## _module) { \
-            opal_output_verbose(1, ompi_coll_base_framework.framework_output, \
-                                "(%d/%s): no underlying " # __api"; disqualifying myself", \
-                                comm->c_contextid, comm->c_name); \
-            goto handle_error;                                  \
-        }                                                       \
-        han_module->previous_ ## __api            = comm->c_coll->coll_ ## __api; \
-        han_module->previous_ ## __api ## _module = comm->c_coll->coll_ ## __api ## _module; \
-        OBJ_RETAIN(han_module->previous_ ## __api ## _module);  \
-    } while(0)
+        if( NULL != __module->super.coll_ ## __api ) {                  \
+            if (!__comm->c_coll->coll_##__api || !__comm->c_coll->coll_##__api##_module) {  \
+                    opal_output_verbose(10, ompi_coll_base_framework.framework_output, \
+                                        "(%d/%s): no underlying " #__api " ; disqualifying myself", \
+                                        __comm->c_index, __comm->c_name); \
+            } else {                                                    \
+                MCA_COLL_SAVE_API(__comm, __api, __module->previous_##__api, \
+                                  __module->previous_##__api##_module, "han"); \
+                MCA_COLL_INSTALL_API(__comm, __api, __module->super.coll_##__api, &__module->super, "han"); \
+            }                                                           \
+        }                                                               \
+    } while (0)
+
+/* The HAN_UNINSTALL_COLL_API is in coll_han.h header as it is needed in several places. */
 
 /*
  * Init module on the communicator
  */
 static int
-han_module_enable(mca_coll_base_module_t * module,
-                  struct ompi_communicator_t *comm)
+mca_coll_han_module_enable(mca_coll_base_module_t * module,
+                           struct ompi_communicator_t *comm)
 {
     mca_coll_han_module_t * han_module = (mca_coll_han_module_t*) module;
 
-    HAN_SAVE_PREV_COLL_API(allgather);
-    HAN_SAVE_PREV_COLL_API(allgatherv);
-    HAN_SAVE_PREV_COLL_API(allreduce);
-    HAN_SAVE_PREV_COLL_API(barrier);
-    HAN_SAVE_PREV_COLL_API(bcast);
-    HAN_SAVE_PREV_COLL_API(gather);
-    HAN_SAVE_PREV_COLL_API(reduce);
-    HAN_SAVE_PREV_COLL_API(scatter);
+    HAN_INSTALL_COLL_API(comm, han_module, alltoall);
+    HAN_INSTALL_COLL_API(comm, han_module, alltoallv);
+    HAN_INSTALL_COLL_API(comm, han_module, allgather);
+    HAN_INSTALL_COLL_API(comm, han_module, allgatherv);
+    HAN_INSTALL_COLL_API(comm, han_module, allreduce);
+    HAN_INSTALL_COLL_API(comm, han_module, barrier);
+    HAN_INSTALL_COLL_API(comm, han_module, bcast);
+    HAN_INSTALL_COLL_API(comm, han_module, gather);
+    HAN_INSTALL_COLL_API(comm, han_module, gatherv);
+    HAN_INSTALL_COLL_API(comm, han_module, reduce);
+    HAN_INSTALL_COLL_API(comm, han_module, scatter);
+    HAN_INSTALL_COLL_API(comm, han_module, scatterv);
 
     /* set reproducible algos */
     mca_coll_han_reduce_reproducible_decision(comm, module);
     mca_coll_han_allreduce_reproducible_decision(comm, module);
 
     return OMPI_SUCCESS;
-
-handle_error:
-    OBJ_RELEASE_IF_NOT_NULL(han_module->previous_allgather_module);
-    OBJ_RELEASE_IF_NOT_NULL(han_module->previous_allgatherv_module);
-    OBJ_RELEASE_IF_NOT_NULL(han_module->previous_allreduce_module);
-    OBJ_RELEASE_IF_NOT_NULL(han_module->previous_bcast_module);
-    OBJ_RELEASE_IF_NOT_NULL(han_module->previous_gather_module);
-    OBJ_RELEASE_IF_NOT_NULL(han_module->previous_reduce_module);
-    OBJ_RELEASE_IF_NOT_NULL(han_module->previous_scatter_module);
-
-    return OMPI_ERROR;
 }
 
 /*
@@ -333,14 +326,18 @@ mca_coll_han_module_disable(mca_coll_base_module_t * module,
 {
     mca_coll_han_module_t * han_module = (mca_coll_han_module_t *) module;
 
-    OBJ_RELEASE_IF_NOT_NULL(han_module->previous_allgather_module);
-    OBJ_RELEASE_IF_NOT_NULL(han_module->previous_allgatherv_module);
-    OBJ_RELEASE_IF_NOT_NULL(han_module->previous_allreduce_module);
-    OBJ_RELEASE_IF_NOT_NULL(han_module->previous_barrier_module);
-    OBJ_RELEASE_IF_NOT_NULL(han_module->previous_bcast_module);
-    OBJ_RELEASE_IF_NOT_NULL(han_module->previous_gather_module);
-    OBJ_RELEASE_IF_NOT_NULL(han_module->previous_reduce_module);
-    OBJ_RELEASE_IF_NOT_NULL(han_module->previous_scatter_module);
+    HAN_UNINSTALL_COLL_API(comm, han_module, alltoall);
+    HAN_UNINSTALL_COLL_API(comm, han_module, alltoallv);
+    HAN_UNINSTALL_COLL_API(comm, han_module, allgather);
+    HAN_UNINSTALL_COLL_API(comm, han_module, allgatherv);
+    HAN_UNINSTALL_COLL_API(comm, han_module, allreduce);
+    HAN_UNINSTALL_COLL_API(comm, han_module, barrier);
+    HAN_UNINSTALL_COLL_API(comm, han_module, bcast);
+    HAN_UNINSTALL_COLL_API(comm, han_module, gather);
+    HAN_UNINSTALL_COLL_API(comm, han_module, gatherv);
+    HAN_UNINSTALL_COLL_API(comm, han_module, reduce);
+    HAN_UNINSTALL_COLL_API(comm, han_module, scatter);
+    HAN_UNINSTALL_COLL_API(comm, han_module, scatterv);
 
     han_module_clear(han_module);
 
@@ -351,7 +348,7 @@ mca_coll_han_module_disable(mca_coll_base_module_t * module,
 /*
  * Free the han request
  */
-int han_request_free(ompi_request_t ** request)
+int ompi_coll_han_request_free(ompi_request_t ** request)
 {
     (*request)->req_state = OMPI_REQUEST_INVALID;
     OBJ_RELEASE(*request);
diff --git a/ompi/mca/coll/han/coll_han_reduce.c b/ompi/mca/coll/han/coll_han_reduce.c
index e47b9d29466..097da6e7662 100644
--- a/ompi/mca/coll/han/coll_han_reduce.c
+++ b/ompi/mca/coll/han/coll_han_reduce.c
@@ -1,8 +1,12 @@
 /*
- * Copyright (c) 2018-2020 The University of Tennessee and The University
+ * Copyright (c) 2018-2023 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2020      Bull S.A.S. All rights reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
+ * Copyright (c) 2024      Computer Architecture and VLSI Systems (CARV)
+ *                         Laboratory, ICS Forth. All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -51,7 +55,7 @@ mca_coll_han_set_reduce_args(mca_coll_han_reduce_args_t * args, mca_coll_task_t
 }
 
 /*
- * Each segment of the messsage needs to go though 2 steps to perform MPI_Reduce:
+ * Each segment of the message needs to go though 2 steps to perform MPI_Reduce:
  *     lb: low level (shared-memory or intra-node) reduce.
  *     ub: upper level (inter-node) reduce
  * Hence, in each iteration, there is a combination of collective operations which is called a task.
@@ -65,7 +69,7 @@ mca_coll_han_set_reduce_args(mca_coll_han_reduce_args_t * args, mca_coll_task_t
 int
 mca_coll_han_reduce_intra(const void *sbuf,
                           void *rbuf,
-                          int count,
+                          size_t count,
                           struct ompi_datatype_t *dtype,
                           ompi_op_t* op,
                           int root,
@@ -89,9 +93,9 @@ mca_coll_han_reduce_intra(const void *sbuf,
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "han cannot handle reduce with this communicator. Drop HAN support in this communicator and fall back on another component\n"));
         /* HAN cannot work with this communicator so fallback on all modules */
-        HAN_LOAD_FALLBACK_COLLECTIVES(han_module, comm);
-        return comm->c_coll->coll_reduce(sbuf, rbuf, count, dtype, op, root,
-                                         comm, comm->c_coll->coll_reduce_module);
+        HAN_LOAD_FALLBACK_COLLECTIVES(comm, han_module);
+        return han_module->previous_reduce(sbuf, rbuf, count, dtype, op, root,
+                                          comm, han_module->previous_reduce_module);
     }
 
     /* Topo must be initialized to know rank distribution which then is used to
@@ -103,9 +107,9 @@ mca_coll_han_reduce_intra(const void *sbuf,
         /* Put back the fallback collective support and call it once. All
          * future calls will then be automatically redirected.
          */
-        HAN_LOAD_FALLBACK_COLLECTIVE(han_module, comm, reduce);
-        return comm->c_coll->coll_reduce(sbuf, rbuf, count, dtype, op, root,
-                                         comm, comm->c_coll->coll_reduce_module);
+        HAN_UNINSTALL_COLL_API(comm, han_module, reduce);
+        return han_module->previous_reduce(sbuf, rbuf, count, dtype, op, root,
+                                          comm, han_module->previous_reduce_module);
     }
 
     ompi_datatype_get_extent(dtype, &lb, &extent);
@@ -123,13 +127,12 @@ mca_coll_han_reduce_intra(const void *sbuf,
 
     int num_segments = (count + seg_count - 1) / seg_count;
     OPAL_OUTPUT_VERBOSE((20, mca_coll_han_component.han_output,
-                         "In HAN seg_count %d count %d num_seg %d\n",
+                         "In HAN seg_count %d count %zu num_seg %d\n",
                          seg_count, count, num_segments));
 
     int *vranks = han_module->cached_vranks;
     int low_rank = ompi_comm_rank(low_comm);
     int low_size = ompi_comm_size(low_comm);
-    int up_rank  = ompi_comm_rank(up_comm);
 
     int root_low_rank;
     int root_up_rank;
@@ -138,11 +141,18 @@ mca_coll_han_reduce_intra(const void *sbuf,
                          "[%d]: root_low_rank %d root_up_rank %d\n", w_rank, root_low_rank,
                          root_up_rank));
 
-    void *tmp_rbuf = rbuf;
+    /* node leaders require a buffer to store intermediate results */
+    void *tmp_rbuf = NULL;
     void *tmp_rbuf_to_free = NULL;
-    if (low_rank == root_low_rank && root_up_rank != up_rank) {
-        /* allocate 2 segments on node leaders that are not the global root */
+    if (w_rank == root) {
+        /* the global root already has one */
+        tmp_rbuf = rbuf;
+    } else if (low_rank == root_low_rank) {
+        /* allocate 2 temporary segments on node leaders that are not the global root */
         tmp_rbuf = malloc(2*extent*seg_count);
+        if (NULL == tmp_rbuf) {
+            return OMPI_ERR_OUT_OF_RESOURCE;
+        }
         tmp_rbuf_to_free = tmp_rbuf;
     }
 
@@ -171,8 +181,11 @@ mca_coll_han_reduce_intra(const void *sbuf,
         mca_coll_task_t *t_next_seg = OBJ_NEW(mca_coll_task_t);
         /* Setup up t_next_seg task arguments */
         t->cur_task = t_next_seg;
-        t->sbuf = (char *) t->sbuf + extent * t->seg_count;
-        if (up_rank == root_up_rank) {
+        if (t->sbuf != MPI_IN_PLACE) {
+            t->sbuf = (char *) t->sbuf + extent * t->seg_count;
+        }
+
+        if (w_rank == root) {
             t->rbuf = (char *) t->rbuf + extent * t->seg_count;
         }
         t->cur_seg = t->cur_seg + 1;
@@ -241,6 +254,7 @@ int mca_coll_han_reduce_t1_task(void *task_args) {
     if (next_seg <= t->num_segments - 1) {
         int tmp_count = t->seg_count;
         char *tmp_rbuf = NULL;
+        char *tmp_sbuf = NULL;
         if (next_seg == t->num_segments - 1 && t->last_seg_count != t->seg_count) {
             tmp_count = t->last_seg_count;
         }
@@ -249,7 +263,10 @@ int mca_coll_han_reduce_t1_task(void *task_args) {
         } else if (NULL != t->rbuf) {
             tmp_rbuf = (char*)t->rbuf + extent * t->seg_count;
         }
-        t->low_comm->c_coll->coll_reduce((char *) t->sbuf + extent * t->seg_count,
+
+        tmp_sbuf = (t->sbuf == MPI_IN_PLACE) ? MPI_IN_PLACE : (char *)t->sbuf + extent * t->seg_count;
+
+        t->low_comm->c_coll->coll_reduce((char *) tmp_sbuf,
                                          (char *) tmp_rbuf, tmp_count,
                                          t->dtype, t->op, t->root_low_rank, t->low_comm,
                                          t->low_comm->c_coll->coll_reduce_module);
@@ -267,7 +284,7 @@ int mca_coll_han_reduce_t1_task(void *task_args) {
 int
 mca_coll_han_reduce_intra_simple(const void *sbuf,
                                  void* rbuf,
-                                 int count,
+                                 size_t count,
                                  struct ompi_datatype_t *dtype,
                                  ompi_op_t *op,
                                  int root,
@@ -295,9 +312,9 @@ mca_coll_han_reduce_intra_simple(const void *sbuf,
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "han cannot handle reduce with this communicator. Drop HAN support in this communicator and fall back on another component\n"));
         /* HAN cannot work with this communicator so fallback on all collectives */
-        HAN_LOAD_FALLBACK_COLLECTIVES(han_module, comm);
-        return comm->c_coll->coll_reduce(sbuf, rbuf, count, dtype, op, root,
-                                         comm, comm->c_coll->coll_reduce_module);
+        HAN_LOAD_FALLBACK_COLLECTIVES(comm, han_module);
+        return han_module->previous_reduce(sbuf, rbuf, count, dtype, op, root,
+                                          comm, han_module->previous_reduce_module);
     }
 
     /* Topo must be initialized to know rank distribution which then is used to
@@ -309,9 +326,9 @@ mca_coll_han_reduce_intra_simple(const void *sbuf,
         /* Put back the fallback collective support and call it once. All
          * future calls will then be automatically redirected.
          */
-        HAN_LOAD_FALLBACK_COLLECTIVE(han_module, comm, reduce);
-        return comm->c_coll->coll_reduce(sbuf, rbuf, count, dtype, op, root,
-                                         comm, comm->c_coll->coll_reduce_module);
+        HAN_UNINSTALL_COLL_API(comm, han_module, reduce);
+        return han_module->previous_reduce(sbuf, rbuf, count, dtype, op, root,
+                                          comm, han_module->previous_reduce_module);
     }
 
     ompi_communicator_t *low_comm =
@@ -364,7 +381,7 @@ mca_coll_han_reduce_intra_simple(const void *sbuf,
             free(tmp_buf);
         } else {
             /* Take advantage of any optimisation made for IN_PLACE
-             * communcations */
+             * communications */
             ret = up_comm->c_coll->coll_reduce(MPI_IN_PLACE, (char *)tmp_buf,
                         count, dtype, op, root_up_rank,
                         up_comm, up_comm->c_coll->coll_reduce_module);
@@ -410,7 +427,7 @@ mca_coll_han_reduce_reproducible_decision(struct ompi_communicator_t *comm,
                 opal_output_verbose(30, mca_coll_han_component.han_output,
                                     "coll:han:reduce_reproducible: "
                                     "fallback on %s\n",
-                                    available_components[fallback].component_name);
+                                    ompi_coll_han_available_components[fallback].component_name);
             }
             han_module->reproducible_reduce_module = fallback_module;
             han_module->reproducible_reduce = fallback_module->coll_reduce;
@@ -434,7 +451,7 @@ mca_coll_han_reduce_reproducible_decision(struct ompi_communicator_t *comm,
 int
 mca_coll_han_reduce_reproducible(const void *sbuf,
                                  void *rbuf,
-                                  int count,
+                                  size_t count,
                                   struct ompi_datatype_t *dtype,
                                   struct ompi_op_t *op,
                                   int root,
diff --git a/ompi/mca/coll/han/coll_han_scatter.c b/ompi/mca/coll/han/coll_han_scatter.c
index 31ec78f5a6b..2122103a5dd 100644
--- a/ompi/mca/coll/han/coll_han_scatter.c
+++ b/ompi/mca/coll/han/coll_han_scatter.c
@@ -1,7 +1,9 @@
 /*
- * Copyright (c) 2018-2020 The University of Tennessee and The University
+ * Copyright (c) 2018-2023 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -32,10 +34,10 @@ mca_coll_han_set_scatter_args(mca_coll_han_scatter_args_t * args,
                               void *sbuf,
                               void *sbuf_inter_free,
                               void *sbuf_reorder_free,
-                              int scount,
+                              size_t scount,
                               struct ompi_datatype_t *sdtype,
                               void *rbuf,
-                              int rcount,
+                              size_t rcount,
                               struct ompi_datatype_t *rdtype,
                               int root,
                               int root_up_rank,
@@ -68,9 +70,9 @@ mca_coll_han_set_scatter_args(mca_coll_han_scatter_args_t * args,
  * after data reordring, calls us task, a scatter on up communicator
  */
 int
-mca_coll_han_scatter_intra(const void *sbuf, int scount,
+mca_coll_han_scatter_intra(const void *sbuf, size_t scount,
                            struct ompi_datatype_t *sdtype,
-                           void *rbuf, int rcount,
+                           void *rbuf, size_t rcount,
                            struct ompi_datatype_t *rdtype,
                            int root,
                            struct ompi_communicator_t *comm, mca_coll_base_module_t * module)
@@ -85,9 +87,9 @@ mca_coll_han_scatter_intra(const void *sbuf, int scount,
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "han cannot handle scatter with this communicator. Fall back on another component\n"));
         /* HAN cannot work with this communicator so fallback on all collectives */
-        HAN_LOAD_FALLBACK_COLLECTIVES(han_module, comm);
-        return comm->c_coll->coll_scatter(sbuf, scount, sdtype, rbuf, rcount, rdtype, root,
-                                          comm, comm->c_coll->coll_scatter_module);
+        HAN_LOAD_FALLBACK_COLLECTIVES(comm, han_module);
+        return han_module->previous_scatter(sbuf, scount, sdtype, rbuf, rcount, rdtype, root,
+                                            comm, han_module->previous_scatter_module);
     }
 
     /* Topo must be initialized to know rank distribution which then is used to
@@ -99,9 +101,9 @@ mca_coll_han_scatter_intra(const void *sbuf, int scount,
         /* Put back the fallback collective support and call it once. All
          * future calls will then be automatically redirected.
          */
-        HAN_LOAD_FALLBACK_COLLECTIVE(han_module, comm, scatter);
-        return comm->c_coll->coll_scatter(sbuf, scount, sdtype, rbuf, rcount, rdtype, root,
-                                          comm, comm->c_coll->coll_scatter_module);
+        HAN_UNINSTALL_COLL_API(comm, han_module, scatter);
+        return han_module->previous_scatter(sbuf, scount, sdtype, rbuf, rcount, rdtype, root,
+                                            comm, han_module->previous_scatter_module);
     }
 
     ompi_communicator_t *low_comm =
@@ -117,7 +119,7 @@ mca_coll_han_scatter_intra(const void *sbuf, int scount,
     ompi_request_t *temp_request = OBJ_NEW(ompi_request_t);
     temp_request->req_state = OMPI_REQUEST_ACTIVE;
     temp_request->req_type = OMPI_REQUEST_COLL;
-    temp_request->req_free = han_request_free;
+    temp_request->req_free = ompi_coll_han_request_free;
     temp_request->req_status = (ompi_status_public_t){0};
     temp_request->req_complete = REQUEST_PENDING;
 
@@ -186,7 +188,7 @@ mca_coll_han_scatter_intra(const void *sbuf, int scount,
 
 }
 
-/* us: upper level (intra-node) scatter task */
+/* us: upper level (inter-node) scatter task */
 int mca_coll_han_scatter_us_task(void *task_args)
 {
     mca_coll_han_scatter_args_t *t = (mca_coll_han_scatter_args_t *) task_args;
@@ -217,6 +219,8 @@ int mca_coll_han_scatter_us_task(void *task_args)
                                          t->up_comm, t->up_comm->c_coll->coll_scatter_module);
         t->sbuf = tmp_rbuf;
         t->sbuf_inter_free = tmp_buf;
+        t->sdtype = dtype;
+        t->scount = count;
     }
 
     if (t->sbuf_reorder_free != NULL && t->root == t->w_rank) {
@@ -259,9 +263,9 @@ int mca_coll_han_scatter_ls_task(void *task_args)
 
 
 int
-mca_coll_han_scatter_intra_simple(const void *sbuf, int scount,
+mca_coll_han_scatter_intra_simple(const void *sbuf, size_t scount,
                                   struct ompi_datatype_t *sdtype,
-                                  void *rbuf, int rcount,
+                                  void *rbuf, size_t rcount,
                                   struct ompi_datatype_t *rdtype,
                                   int root,
                                   struct ompi_communicator_t *comm,
@@ -281,8 +285,8 @@ mca_coll_han_scatter_intra_simple(const void *sbuf, int scount,
                              "han cannot handle allgather within this communicator."
                              " Fall back on another component\n"));
         /* HAN cannot work with this communicator so fallback on all collectives */
-        HAN_LOAD_FALLBACK_COLLECTIVES(han_module, comm);
-        return comm->c_coll->coll_scatter(sbuf, scount, sdtype, rbuf, rcount, rdtype, root,
+        HAN_LOAD_FALLBACK_COLLECTIVES(comm, han_module);
+        return han_module->previous_scatter(sbuf, scount, sdtype, rbuf, rcount, rdtype, root,
                                             comm, han_module->previous_scatter_module);
     }
     /* Topo must be initialized to know rank distribution which then is used to
@@ -291,8 +295,8 @@ mca_coll_han_scatter_intra_simple(const void *sbuf, int scount,
     if (han_module->are_ppn_imbalanced){
         OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
                              "han cannot handle scatter with this communicator. It needs to fall back on another component\n"));
-        HAN_LOAD_FALLBACK_COLLECTIVES(han_module, comm);
-        return comm->c_coll->coll_scatter(sbuf, scount, sdtype, rbuf, rcount, rdtype, root,
+        HAN_UNINSTALL_COLL_API(comm, han_module, scatter);
+        return han_module->previous_scatter(sbuf, scount, sdtype, rbuf, rcount, rdtype, root,
                                             comm, han_module->previous_scatter_module);
     }
     ompi_communicator_t *low_comm = han_module->sub_comm[INTRA_NODE];
@@ -379,10 +383,6 @@ mca_coll_han_scatter_intra_simple(const void *sbuf, int scount,
                     root_up_rank,
                     up_comm,
                     up_comm->c_coll->coll_scatter_module);
-        if(reorder_buf != sbuf){
-            free(reorder_buf);
-            reorder_buf = NULL;
-        }
     }
 
     /* 2. low scatter on nodes leaders */
@@ -400,6 +400,9 @@ mca_coll_han_scatter_intra_simple(const void *sbuf, int scount,
         free(tmp_buf);
         tmp_buf = NULL;
     }
+    if (reorder_buf != sbuf) {
+        free(reorder_buf);
+    }
 
     return OMPI_SUCCESS;
 
diff --git a/ompi/mca/coll/han/coll_han_scatterv.c b/ompi/mca/coll/han/coll_han_scatterv.c
new file mode 100644
index 00000000000..18c67baff91
--- /dev/null
+++ b/ompi/mca/coll/han/coll_han_scatterv.c
@@ -0,0 +1,390 @@
+/*
+ * Copyright (c) 2018-2023 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2020      Bull S.A.S. All rights reserved.
+ * Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
+ * Copyright (c)           Amazon.com, Inc. or its affiliates.
+ *                         All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "coll_han.h"
+#include "coll_han_trigger.h"
+#include "ompi/mca/coll/base/coll_base_functions.h"
+#include "ompi/mca/coll/base/coll_tags.h"
+#include "ompi/mca/pml/pml.h"
+
+/*
+ * @file
+ *
+ * This files contains the hierarchical implementations of scatterv.
+ * Only work with regular situation (each node has equal number of processes).
+ */
+
+/*
+ * Implement hierarchical Scatterv to optimize large-scale communications where root sends
+ * non-zero sized messages to multiple nodes and multiple processes per node, i.e. high incast.
+ *
+ * In Scatterv, only the root(sender) process has the information of the amount of data, i.e.
+ * datatype and count, to each receiver process. Therefore node leaders need an additional step to
+ * collect the expected data from its local peers. In summary, the steps are:
+ * 1. Root:
+ *      a. If necessary, reorder and sort data (See discussion below)
+ *      b. Send data to other node leaders (Up Iscatterv)
+ *      c. Send data to local peers (Low Scatterv)
+ * 2. Root's local peers:
+ *      a. Receive data from root. (Low Scatterv)
+ * 3. Node leaders:
+ *      a. Collect the data transfer sizes(in bytes) from local peers (Low Gather)
+ *      b. Receive data from the root (Up Iscatterv)
+ *      c. Send data to local peers (Low Scatterv)
+ * 4. Node followers:
+ *      a. Send the data transfer size(in bytes) to the node leader (Low Gather)
+ *      b. Receive data from the node leader (Low Scatterv)
+ *
+ * Note on reordering:
+ * In Up Iscatterv, reordering the send buffer can be avoided if and only if both of following
+ * conditions are met:
+ * 1. The data for each node is sorted in the same order as peer local ranks. Note, it is possible
+ *    to send the data in the correct order even if the process are NOT mapped by core.
+ * 2. In the send buffer, other than the root's node, data destined to the same node are continuous
+ *    - it is ok if data to different nodes has gap.
+ *
+ * Limitation:
+ * The node leader acts as a broker between the Root and node followers, but it cannot match the
+ * exact type signature of the followers; instead it forwards the intermediate data from Root in its
+ * packed form of MPI_BYTE type. This works for Gatherv but NOT for Scatterv provided that the Root
+ * has a different architecture, e.g. endianness, integer representation, etc.
+ */
+int mca_coll_han_scatterv_intra(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t displs,
+                                struct ompi_datatype_t *sdtype, void *rbuf, size_t rcount,
+                                struct ompi_datatype_t *rdtype, int root,
+                                struct ompi_communicator_t *comm, mca_coll_base_module_t *module)
+{
+    mca_coll_han_module_t *han_module = (mca_coll_han_module_t *) module;
+    int w_rank, w_size;              /* information about the global communicator */
+    int root_low_rank, root_up_rank; /* root ranks for both sub-communicators */
+    int err, *vranks, low_rank, low_size, up_rank, up_size, *topo;
+    size_t *low_scounts = NULL;
+    ptrdiff_t *low_displs = NULL;
+    ompi_count_array_t low_scounts_desc;
+    ompi_disp_array_t low_displs_desc;
+    ompi_request_t *iscatterv_req = NULL;
+
+    /* Create the subcommunicators */
+    err = mca_coll_han_comm_create(comm, han_module);
+    if (OMPI_SUCCESS != err) {
+        OPAL_OUTPUT_VERBOSE((
+            30, mca_coll_han_component.han_output,
+            "han cannot handle scatterv with this communicator. Fall back on another component\n"));
+        /* HAN cannot work with this communicator so fallback on all collectives */
+        HAN_LOAD_FALLBACK_COLLECTIVES(comm, han_module);
+        return han_module->previous_scatterv(sbuf, scounts, displs, sdtype, rbuf, rcount, rdtype,
+                                             root, comm, han_module->previous_scatterv_module);
+    }
+
+    /* Topo must be initialized to know rank distribution which then is used to determine if han can
+     * be used */
+    topo = mca_coll_han_topo_init(comm, han_module, 2);
+    if (han_module->are_ppn_imbalanced) {
+        OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
+                             "han cannot handle scatterv with this communicator (imbalance). Fall "
+                             "back on another component\n"));
+        /* Put back the fallback collective support and call it once. All
+         * future calls will then be automatically redirected.
+         */
+        HAN_UNINSTALL_COLL_API(comm, han_module, scatterv);
+        return han_module->previous_scatterv(sbuf, scounts, displs, sdtype, rbuf, rcount, rdtype,
+                                             root, comm, han_module->previous_scatterv_module);
+    }
+    if (han_module->is_heterogeneous) {
+        OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
+                             "han cannot handle scatterv with this communicator (heterogeneous). Fall "
+                             "back on another component\n"));
+        HAN_UNINSTALL_COLL_API(comm, han_module, scatterv);
+        return han_module->previous_scatterv(sbuf, scounts, displs, sdtype, rbuf, rcount, rdtype,
+                                             root, comm, han_module->previous_scatterv_module);
+    }
+
+    w_rank = ompi_comm_rank(comm);
+    w_size = ompi_comm_size(comm);
+
+    /* create the subcommunicators */
+    ompi_communicator_t *low_comm
+        = han_module->cached_low_comms[mca_coll_han_component.han_scatterv_low_module];
+    ompi_communicator_t *up_comm
+        = han_module->cached_up_comms[mca_coll_han_component.han_scatterv_up_module];
+
+    /* Get the 'virtual ranks' mapping corresponding to the communicators */
+    vranks = han_module->cached_vranks;
+    /* information about sub-communicators */
+    low_rank = ompi_comm_rank(low_comm);
+    low_size = ompi_comm_size(low_comm);
+    up_rank = ompi_comm_rank(up_comm);
+    up_size = ompi_comm_size(up_comm);
+    /* Get root ranks for low and up comms */
+    mca_coll_han_get_ranks(vranks, root, low_size, &root_low_rank, &root_up_rank);
+
+    OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
+                         "[%d]: Han scatterv root %d root_low_rank %d root_up_rank %d\n", w_rank,
+                         root, root_low_rank, root_up_rank));
+
+    err = OMPI_SUCCESS;
+    /* #################### Root ########################### */
+    if (root == w_rank) {
+        int low_peer, up_peer, w_peer;
+        int need_bounce_buf = 0;
+        size_t total_up_scounts = 0;
+        ptrdiff_t *up_displs = NULL;
+        size_t *up_scounts = NULL, *up_peer_ub = NULL;
+        ompi_count_array_t up_scounts_desc;
+        ompi_disp_array_t up_displs_desc;
+        char *reorder_sbuf = (char *) sbuf, *bounce_buf = NULL;
+
+        low_scounts = malloc(low_size * sizeof(size_t));
+        low_displs = malloc(low_size * sizeof(ptrdiff_t));
+        if (!low_scounts || !low_displs) {
+            err = OMPI_ERR_OUT_OF_RESOURCE;
+            goto root_out;
+        }
+
+        for (w_peer = 0; w_peer < w_size; ++w_peer) {
+            mca_coll_han_get_ranks(vranks, w_peer, low_size, &low_peer, &up_peer);
+            if (root_up_rank != up_peer) {
+                /* Not a local peer */
+                continue;
+            }
+            low_displs[low_peer] = ompi_disp_array_get(displs, w_peer);
+            low_scounts[low_peer] = ompi_count_array_get(scounts, w_peer);
+        }
+
+        up_scounts = calloc(up_size, sizeof(size_t));
+        up_displs = malloc(up_size * sizeof(ptrdiff_t));
+        up_peer_ub = calloc(up_size, sizeof(size_t));
+        if (!up_scounts || !up_displs || !up_peer_ub) {
+            err = OMPI_ERR_OUT_OF_RESOURCE;
+            goto root_out;
+        }
+
+        for (up_peer = 0; up_peer < up_size; ++up_peer) {
+            up_displs[up_peer] = PTRDIFF_MAX;
+        }
+
+        /* Calculate send counts for the inter-node scatterv */
+        for (w_peer = 0; w_peer < w_size; ++w_peer) {
+            mca_coll_han_get_ranks(vranks, w_peer, low_size, NULL, &up_peer);
+
+            if (!need_bounce_buf && root_up_rank != up_peer
+                && 0 < ompi_count_array_get(scounts, w_peer) && 0 < w_peer
+                && ompi_disp_array_get(displs, w_peer) < ompi_disp_array_get(displs, w_peer - 1)) {
+                /* Data is not placed in the rank order so reordering is needed */
+                need_bounce_buf = 1;
+            }
+
+            if (root_up_rank == up_peer) {
+                /* No need to scatter data on the same node again */
+                continue;
+            }
+
+            up_peer_ub[up_peer] = 0 < ompi_count_array_get(scounts, w_peer)
+                                          && ompi_disp_array_get(displs, w_peer)
+                                             + ompi_count_array_get(scounts, w_peer) > up_peer_ub[up_peer]
+                                      ? ompi_disp_array_get(displs, w_peer)
+                                        + ompi_count_array_get(scounts, w_peer)
+                                      : up_peer_ub[up_peer];
+
+            up_scounts[up_peer] += ompi_count_array_get(scounts, w_peer);
+            total_up_scounts += ompi_count_array_get(scounts, w_peer);
+
+            /* Optimize for the happy path */
+            up_displs[up_peer] = 0 < ompi_count_array_get(scounts, w_peer)
+                                 && ompi_disp_array_get(displs, w_peer) < up_displs[up_peer]
+                                     ? ompi_disp_array_get(displs, w_peer)
+                                     : up_displs[up_peer];
+        }
+
+        /* If the data is not placed contiguously on send buf without overlaping, then we need a
+         * temp buf without gaps */
+        for (up_peer = 0; up_peer < up_size; ++up_peer) {
+            if (root_up_rank == up_peer) {
+                continue;
+            }
+            if (!need_bounce_buf && 0 < up_scounts[up_peer]
+                && up_scounts[up_peer] != up_peer_ub[up_peer] - up_displs[up_peer]) {
+                need_bounce_buf = 1;
+                break;
+            }
+        }
+
+        if (need_bounce_buf) {
+            ptrdiff_t ssize, sgap;
+            ssize = opal_datatype_span(&rdtype->super, total_up_scounts, &sgap);
+            bounce_buf = malloc(ssize);
+            if (!bounce_buf) {
+                err = OMPI_ERR_OUT_OF_RESOURCE;
+                goto root_out;
+            }
+            reorder_sbuf = bounce_buf - sgap;
+
+            /* Calculate displacements for the inter-node scatterv */
+            for (up_peer = 0; up_peer < up_size; ++up_peer) {
+                up_displs[up_peer] = 0 < up_peer ? up_displs[up_peer - 1] + up_scounts[up_peer - 1]
+                                                 : 0;
+            }
+
+            /* Use a temp buffer to reorder the send buffer if needed */
+            ptrdiff_t offset = 0, sdext;
+            ompi_datatype_type_extent(sdtype, &sdext);
+
+            for (int i = 0; i < w_size; ++i) {
+                up_peer = topo[2 * i];
+                if (root_up_rank == up_peer) {
+                    continue;
+                }
+
+                w_peer = topo[2 * i + 1];
+
+                ompi_datatype_copy_content_same_ddt(sdtype, ompi_count_array_get(scounts, w_peer),
+                                                    reorder_sbuf + offset,
+                                                    (char *) sbuf
+                                                        + ompi_disp_array_get(displs, w_peer) * sdext);
+                offset += sdext * ompi_count_array_get(scounts, w_peer);
+            }
+        }
+
+        /* Up Iscatterv */
+        OMPI_COUNT_ARRAY_INIT(&up_scounts_desc, up_scounts);
+        OMPI_DISP_ARRAY_INIT(&up_displs_desc, up_displs);
+        up_comm->c_coll->coll_iscatterv((const char *) reorder_sbuf,
+                                        up_scounts_desc, up_displs_desc, sdtype,
+                                        rbuf, rcount, rdtype, root_up_rank, up_comm, &iscatterv_req,
+                                        up_comm->c_coll->coll_iscatterv_module);
+
+        /* Low Scatterv */
+        OMPI_COUNT_ARRAY_INIT(&low_scounts_desc, low_scounts);
+        OMPI_DISP_ARRAY_INIT(&low_displs_desc, low_displs);
+        low_comm->c_coll->coll_scatterv(sbuf, low_scounts_desc, low_displs_desc,
+                                        sdtype, rbuf, rcount, rdtype,
+                                        root_low_rank, low_comm,
+                                        low_comm->c_coll->coll_scatterv_module);
+
+        ompi_request_wait(&iscatterv_req, MPI_STATUS_IGNORE);
+
+    root_out:
+        if (low_displs) {
+            free(low_displs);
+        }
+        if (low_scounts) {
+            free(low_scounts);
+        }
+        if (up_displs) {
+            free(up_displs);
+        }
+        if (up_scounts) {
+            free(up_scounts);
+        }
+        if (up_peer_ub) {
+            free(up_peer_ub);
+        }
+        if (bounce_buf) {
+            free(bounce_buf);
+        }
+
+        return err;
+    }
+
+    /* #################### Root's local peers ########################### */
+    if (root_up_rank == up_rank) {
+        /* Low Scatterv */
+        low_comm->c_coll->coll_scatterv(NULL, 0, 0, NULL, rbuf, rcount, rdtype, root_low_rank,
+                                        low_comm, low_comm->c_coll->coll_scatterv_module);
+        return OMPI_SUCCESS;
+    }
+
+    size_t rdsize = 0;
+    size_t receive_size = 0;
+
+    ompi_datatype_type_size(rdtype, &rdsize);
+    receive_size = rdsize * rcount;
+
+    /* #################### Other node followers ########################### */
+    if (root_low_rank != low_rank) {
+        /* Low Gather - Gather each local peer's receive data size */
+        low_comm->c_coll->coll_gather((const void *) &receive_size, sizeof(size_t), MPI_BYTE, NULL,
+                                      sizeof(size_t), MPI_BYTE, root_low_rank, low_comm,
+                                      low_comm->c_coll->coll_gather_module);
+        /* Low Scatterv */
+        low_comm->c_coll->coll_scatterv(NULL, 0, 0, NULL, rbuf, rcount, rdtype, root_low_rank,
+                                        low_comm, low_comm->c_coll->coll_scatterv_module);
+        return OMPI_SUCCESS;
+    }
+
+    /* #################### Node leaders ########################### */
+
+    char *tmp_buf = NULL;
+
+    /* Allocate a temporary array to gather the data size, i.e. data type size x count,
+     * in bytes from local peers */
+    low_scounts = malloc(low_size * sizeof(size_t));
+    if (!low_scounts) {
+        err = OMPI_ERR_OUT_OF_RESOURCE;
+        goto node_leader_out;
+    }
+
+    /* Low Gather -  Gather local peers' receive data sizes */
+    low_comm->c_coll->coll_gather((const void *) &receive_size, sizeof(size_t), MPI_BYTE,
+                                  (void *) low_scounts, sizeof(size_t), MPI_BYTE, root_low_rank, low_comm,
+                                  low_comm->c_coll->coll_gather_module);
+
+    low_displs = malloc(low_size * sizeof(ptrdiff_t));
+    if (!low_displs) {
+        err = OMPI_ERR_OUT_OF_RESOURCE;
+        goto node_leader_out;
+    }
+
+    size_t total_rsize = 0;
+    for (int i = 0; i < low_size; ++i) {
+        low_displs[i] = i > 0 ? low_displs[i - 1] + low_scounts[i - 1] : 0;
+        total_rsize += low_scounts[i];
+    }
+
+    tmp_buf = (char *) malloc(total_rsize); /* tmp_buf is still valid if total_rsize is 0 */
+    if (!tmp_buf) {
+        err = OMPI_ERR_OUT_OF_RESOURCE;
+        goto node_leader_out;
+    }
+
+    /* Up Iscatterv */
+    up_comm->c_coll->coll_iscatterv(NULL, 0, 0, NULL, (void *) tmp_buf, total_rsize,
+                                    MPI_BYTE, root_up_rank, up_comm, &iscatterv_req,
+                                    up_comm->c_coll->coll_iscatterv_module);
+
+    ompi_request_wait(&iscatterv_req, MPI_STATUS_IGNORE);
+
+    /* Low Scatterv */
+    OMPI_COUNT_ARRAY_INIT(&low_scounts_desc, low_scounts);
+    OMPI_DISP_ARRAY_INIT(&low_displs_desc, low_displs);
+    low_comm->c_coll->coll_scatterv((void *) tmp_buf, low_scounts_desc, low_displs_desc, MPI_BYTE, rbuf,
+                                    rcount, rdtype, root_low_rank, low_comm,
+                                    low_comm->c_coll->coll_scatterv_module);
+
+node_leader_out:
+    if (low_scounts) {
+        free(low_scounts);
+    }
+    if (low_displs) {
+        free(low_displs);
+    }
+    if (tmp_buf) {
+        free(tmp_buf);
+    }
+
+    return err;
+}
diff --git a/ompi/mca/coll/han/coll_han_subcomms.c b/ompi/mca/coll/han/coll_han_subcomms.c
index bb9397c008a..57344074a55 100644
--- a/ompi/mca/coll/han/coll_han_subcomms.c
+++ b/ompi/mca/coll/han/coll_han_subcomms.c
@@ -3,7 +3,11 @@
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2020      Bull S.A.S. All rights reserved.
+ * Copyright (c) 2023      Computer Architecture and VLSI Systems (CARV)
+ *                         Laboratory, ICS Forth. All rights reserved.
+ * Copyright (c) 2024      Amazon.com, Inc. or its affiliates. All Rights Reserved.
  *
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -26,19 +30,21 @@
 #include "coll_han.h"
 #include "coll_han_dynamic.h"
 
-#define HAN_SUBCOM_SAVE_COLLECTIVE(FALLBACKS, COMM, HANM, COLL)                  \
-    do {                                                                         \
-        (FALLBACKS).COLL.module_fn.COLL = (COMM)->c_coll->coll_ ## COLL;         \
-        (FALLBACKS).COLL.module = (COMM)->c_coll->coll_ ## COLL ## _module;      \
-        (COMM)->c_coll->coll_ ## COLL = (HANM)->fallback.COLL.module_fn.COLL;    \
-        (COMM)->c_coll->coll_ ## COLL ## _module = (HANM)->fallback.COLL.module; \
-    } while(0)
-
-#define HAN_SUBCOM_LOAD_COLLECTIVE(FALLBACKS, COMM, HANM, COLL)                  \
-    do {                                                                         \
-        (COMM)->c_coll->coll_ ## COLL = (FALLBACKS).COLL.module_fn.COLL;         \
-        (COMM)->c_coll->coll_ ## COLL ## _module = (FALLBACKS).COLL.module;      \
-    } while(0)
+#define HAN_SUBCOM_SAVE_COLLECTIVE(FALLBACKS, COMM, HANM, COLL)              \
+    do                                                                       \
+    {                                                                        \
+        (FALLBACKS).COLL.COLL = (COMM)->c_coll->coll_##COLL;                 \
+        (FALLBACKS).COLL.module = (COMM)->c_coll->coll_##COLL##_module;      \
+        (COMM)->c_coll->coll_##COLL = (HANM)->fallback.COLL.COLL;            \
+        (COMM)->c_coll->coll_##COLL##_module = (HANM)->fallback.COLL.module; \
+    } while (0)
+
+#define HAN_SUBCOM_RESTORE_COLLECTIVE(FALLBACKS, COMM, HANM, COLL)           \
+    do                                                                       \
+    {                                                                        \
+        (COMM)->c_coll->coll_##COLL = (FALLBACKS).COLL.COLL;                 \
+        (COMM)->c_coll->coll_##COLL##_module = (FALLBACKS).COLL.module;      \
+    } while (0)
 
 /*
  * Routine that creates the local hierarchical sub-communicators
@@ -52,7 +58,7 @@ int mca_coll_han_comm_create_new(struct ompi_communicator_t *comm,
     ompi_communicator_t **low_comm = &(han_module->sub_comm[INTRA_NODE]);
     ompi_communicator_t **up_comm = &(han_module->sub_comm[INTER_NODE]);
     mca_coll_han_collectives_fallback_t fallbacks;
-    int vrank, *vranks;
+    int rc = OMPI_SUCCESS, vrank, *vranks;
     opal_info_t comm_info;
 
     /* The sub communicators have already been created */
@@ -64,21 +70,25 @@ int mca_coll_han_comm_create_new(struct ompi_communicator_t *comm,
 
     /*
      * We cannot use han allreduce and allgather without sub-communicators,
-     * but we are in the creation of the data structures for the HAN, and
-     * temporarily need to save back the old collective.
+     * but we are in the creation of the data structures for HAN, and
+     * temporarily need to use the old collective.
      *
      * Allgather is used to compute vranks
      * Allreduce is used by ompi_comm_split_type in create_intranode_comm_new
      * Reduce + Bcast may be called by the allreduce implementation
      * Gather + Bcast may be called by the allgather implementation
      */
+    HAN_SUBCOM_SAVE_COLLECTIVE(fallbacks, comm, han_module, alltoall);
+    HAN_SUBCOM_SAVE_COLLECTIVE(fallbacks, comm, han_module, alltoallv);
     HAN_SUBCOM_SAVE_COLLECTIVE(fallbacks, comm, han_module, allgatherv);
     HAN_SUBCOM_SAVE_COLLECTIVE(fallbacks, comm, han_module, allgather);
     HAN_SUBCOM_SAVE_COLLECTIVE(fallbacks, comm, han_module, allreduce);
     HAN_SUBCOM_SAVE_COLLECTIVE(fallbacks, comm, han_module, bcast);
     HAN_SUBCOM_SAVE_COLLECTIVE(fallbacks, comm, han_module, reduce);
     HAN_SUBCOM_SAVE_COLLECTIVE(fallbacks, comm, han_module, gather);
+    HAN_SUBCOM_SAVE_COLLECTIVE(fallbacks, comm, han_module, gatherv);
     HAN_SUBCOM_SAVE_COLLECTIVE(fallbacks, comm, han_module, scatter);
+    HAN_SUBCOM_SAVE_COLLECTIVE(fallbacks, comm, han_module, scatterv);
 
     /**
      * HAN is not yet optimized for a single process per node case, we should
@@ -88,23 +98,31 @@ int mca_coll_han_comm_create_new(struct ompi_communicator_t *comm,
      * outside the MPI support (with PRRTE the info will be eventually available,
      * but we don't want to delay anything until then). We can achieve the same
      * goal by using a reduction over the maximum number of peers per node among
-     * all participants.
+     * all participants, but we need to call the fallback allreduce (or we will
+     * call HAN's allreduce again).
      */
     int local_procs = ompi_group_count_local_peers(comm->c_local_group);
-    comm->c_coll->coll_allreduce(MPI_IN_PLACE, &local_procs, 1, MPI_INT,
-                                 MPI_MAX, comm,
-                                 comm->c_coll->coll_allreduce_module);
+    rc = comm->c_coll->coll_allreduce(MPI_IN_PLACE, &local_procs, 1, MPI_INT,
+                                      MPI_MAX, comm,
+                                      comm->c_coll->coll_allreduce_module);
+    if( OMPI_SUCCESS != rc ) {
+        goto return_with_error;
+    }
     if( local_procs == 1 ) {
-        /* restore saved collectives */
-        HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, allgatherv);
-        HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, allgather);
-        HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, allreduce);
-        HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, bcast);
-        HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, reduce);
-        HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, gather);
-        HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, scatter);
         han_module->enabled = false;  /* entire module set to pass-through from now on */
-        return OMPI_ERR_NOT_SUPPORTED;
+        /* restore saved collectives */
+        HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, alltoall);
+        HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, alltoallv);
+        HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, allgatherv);
+        HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, allgather);
+        HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, allreduce);
+        HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, bcast);
+        HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, reduce);
+        HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, gather);
+        HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, gatherv);
+        HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, scatter);
+        HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, scatterv);
+         return OMPI_ERR_NOT_SUPPORTED;
     }
 
     OBJ_CONSTRUCT(&comm_info, opal_info_t);
@@ -116,10 +134,14 @@ int mca_coll_han_comm_create_new(struct ompi_communicator_t *comm,
     /*
      * This sub-communicator contains the ranks that share my node.
      */
-    opal_info_set(&comm_info, "ompi_comm_coll_preference", "^han");
+    opal_info_set(&comm_info, "ompi_comm_coll_preference", "han");
     opal_info_set(&comm_info, "ompi_comm_coll_han_topo_level", "INTRA_NODE");
-    ompi_comm_split_type(comm, MPI_COMM_TYPE_SHARED, 0,
-                         &comm_info, low_comm);
+    rc = ompi_comm_split_type(comm, MPI_COMM_TYPE_SHARED, 0,
+                              &comm_info, low_comm);
+    if( OMPI_SUCCESS != rc ) {
+        /* cannot create subcommunicators. Return the error upstream */
+        goto return_with_error;
+    }
 
     /*
      * Get my local rank and the local size
@@ -132,7 +154,11 @@ int mca_coll_han_comm_create_new(struct ompi_communicator_t *comm,
      * same intra-node rank id share such a sub-communicator
      */
     opal_info_set(&comm_info, "ompi_comm_coll_han_topo_level", "INTER_NODE");
-    ompi_comm_split_with_info(comm, low_rank, w_rank, &comm_info, up_comm, false);
+    rc = ompi_comm_split_with_info(comm, low_rank, w_rank, &comm_info, up_comm, false);
+    if( OMPI_SUCCESS != rc ) {
+        /* cannot create subcommunicators. Return the error upstream */
+        goto return_with_error;
+    }
 
     up_rank = ompi_comm_rank(*up_comm);
 
@@ -150,31 +176,49 @@ int mca_coll_han_comm_create_new(struct ompi_communicator_t *comm,
      * gather vrank from each process so every process will know other processes
      * vrank
      */
-    comm->c_coll->coll_allgather(&vrank,
-                                 1,
-                                 MPI_INT,
-                                 vranks,
-                                 1,
-                                 MPI_INT,
-                                 comm,
-                                 comm->c_coll->coll_allgather_module);
+    rc = comm->c_coll->coll_allgather(&vrank, 1, MPI_INT,
+                                 vranks, 1, MPI_INT,
+                                 comm, comm->c_coll->coll_allgather_module);
+    if( OMPI_SUCCESS != rc ) {
+        /* cannot create subcommunicators. Return the error upstream */
+        goto return_with_error;
+    }
 
     /*
      * Set the cached info
      */
+    if (NULL != han_module->cached_vranks) {
+        free(han_module->cached_vranks);
+    }
     han_module->cached_vranks = vranks;
 
-    /* Reset the saved collectives to point back to HAN */
-    HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, allgatherv);
-    HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, allgather);
-    HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, allreduce);
-    HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, bcast);
-    HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, reduce);
-    HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, gather);
-    HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, scatter);
+    /* Restore the saved collectives */
+    HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, alltoall);
+    HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, alltoallv);
+    HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, allgatherv);
+    HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, allgather);
+    HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, allreduce);
+    HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, bcast);
+    HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, reduce);
+    HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, gather);
+    HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, gatherv);
+    HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, scatter);
+    HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, scatterv);
 
     OBJ_DESTRUCT(&comm_info);
     return OMPI_SUCCESS;
+
+return_with_error:
+    han_module->enabled = false;  /* entire module set to pass-through from now on */
+    if( NULL != *low_comm ) {
+        ompi_comm_free(low_comm);
+        *low_comm = NULL;  /* don't leave the MPI_COMM_NULL set by ompi_comm_free */
+    }
+    if( NULL != *up_comm ) {
+        ompi_comm_free(up_comm);
+        *up_comm = NULL;  /* don't leave the MPI_COMM_NULL set by ompi_comm_free */
+    }
+    return rc;
 }
 
 /*
@@ -202,20 +246,24 @@ int mca_coll_han_comm_create(struct ompi_communicator_t *comm,
     /*
      * We cannot use han allreduce and allgather without sub-communicators,
      * but we are in the creation of the data structures for the HAN, and
-     * temporarily need to save back the old collective.
+     * temporarily need to use the old collective.
      *
      * Allgather is used to compute vranks
      * Allreduce is used by ompi_comm_split_type in create_intranode_comm_new
      * Reduce + Bcast may be called by the allreduce implementation
      * Gather + Bcast may be called by the allgather implementation
      */
+    HAN_SUBCOM_SAVE_COLLECTIVE(fallbacks, comm, han_module, alltoall);
+    HAN_SUBCOM_SAVE_COLLECTIVE(fallbacks, comm, han_module, alltoallv);
     HAN_SUBCOM_SAVE_COLLECTIVE(fallbacks, comm, han_module, allgatherv);
     HAN_SUBCOM_SAVE_COLLECTIVE(fallbacks, comm, han_module, allgather);
     HAN_SUBCOM_SAVE_COLLECTIVE(fallbacks, comm, han_module, allreduce);
     HAN_SUBCOM_SAVE_COLLECTIVE(fallbacks, comm, han_module, bcast);
     HAN_SUBCOM_SAVE_COLLECTIVE(fallbacks, comm, han_module, reduce);
     HAN_SUBCOM_SAVE_COLLECTIVE(fallbacks, comm, han_module, gather);
+    HAN_SUBCOM_SAVE_COLLECTIVE(fallbacks, comm, han_module, gatherv);
     HAN_SUBCOM_SAVE_COLLECTIVE(fallbacks, comm, han_module, scatter);
+    HAN_SUBCOM_SAVE_COLLECTIVE(fallbacks, comm, han_module, scatterv);
 
     /**
      * HAN is not yet optimized for a single process per node case, we should
@@ -233,13 +281,17 @@ int mca_coll_han_comm_create(struct ompi_communicator_t *comm,
                                  comm->c_coll->coll_allreduce_module);
     if( local_procs == 1 ) {
         /* restore saved collectives */
-        HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, allgatherv);
-        HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, allgather);
-        HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, allreduce);
-        HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, bcast);
-        HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, reduce);
-        HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, gather);
-        HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, scatter);
+        HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, alltoall);
+        HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, alltoallv);
+        HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, allgatherv);
+        HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, allgather);
+        HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, allreduce);
+        HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, bcast);
+        HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, reduce);
+        HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, gather);
+        HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, gatherv);
+        HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, scatter);
+        HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, scatterv);
         han_module->enabled = false;  /* entire module set to pass-through from now on */
         return OMPI_ERR_NOT_SUPPORTED;
     }
@@ -261,6 +313,7 @@ int mca_coll_han_comm_create(struct ompi_communicator_t *comm,
     opal_info_set(&comm_info, "ompi_comm_coll_preference", "tuned,^han");
     ompi_comm_split_type(comm, MPI_COMM_TYPE_SHARED, 0,
                          &comm_info, &(low_comms[0]));
+    assert(OMPI_COMM_IS_DISJOINT_SET(low_comms[0]) && !OMPI_COMM_IS_DISJOINT(low_comms[0]));
 
     /*
      * Get my local rank and the local size
@@ -275,6 +328,11 @@ int mca_coll_han_comm_create(struct ompi_communicator_t *comm,
     opal_info_set(&comm_info, "ompi_comm_coll_preference", "sm,^han");
     ompi_comm_split_type(comm, MPI_COMM_TYPE_SHARED, 0,
                          &comm_info, &(low_comms[1]));
+    assert(OMPI_COMM_IS_DISJOINT_SET(low_comms[1]) && !OMPI_COMM_IS_DISJOINT(low_comms[1]));
+
+    opal_info_set(&comm_info, "ompi_comm_coll_preference", "xhc,^han");
+    ompi_comm_split_type(comm, MPI_COMM_TYPE_SHARED, 0,
+                         &comm_info, &(low_comms[2]));
 
     /*
      * Upgrade libnbc module priority to set up up_comms[0] with libnbc module
@@ -283,8 +341,8 @@ int mca_coll_han_comm_create(struct ompi_communicator_t *comm,
      */
     opal_info_set(&comm_info, "ompi_comm_coll_preference", "libnbc,^han");
     ompi_comm_split_with_info(comm, low_rank, w_rank, &comm_info, &(up_comms[0]), false);
-
     up_rank = ompi_comm_rank(up_comms[0]);
+    assert(OMPI_COMM_IS_DISJOINT_SET(up_comms[0]) && OMPI_COMM_IS_DISJOINT(up_comms[0]));
 
     /*
      * Upgrade adapt module priority to set up up_comms[0] with adapt module
@@ -292,6 +350,7 @@ int mca_coll_han_comm_create(struct ompi_communicator_t *comm,
      */
     opal_info_set(&comm_info, "ompi_comm_coll_preference", "adapt,^han");
     ompi_comm_split_with_info(comm, low_rank, w_rank, &comm_info, &(up_comms[1]), false);
+    assert(OMPI_COMM_IS_DISJOINT_SET(up_comms[1]) && OMPI_COMM_IS_DISJOINT(up_comms[1]));
 
     /*
      * Set my virtual rank number.
@@ -318,16 +377,18 @@ int mca_coll_han_comm_create(struct ompi_communicator_t *comm,
     han_module->cached_vranks = vranks;
 
     /* Reset the saved collectives to point back to HAN */
-    HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, allgatherv);
-    HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, allgather);
-    HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, allreduce);
-    HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, bcast);
-    HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, reduce);
-    HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, gather);
-    HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, scatter);
+    HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, alltoall);
+    HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, alltoallv);
+    HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, allgatherv);
+    HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, allgather);
+    HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, allreduce);
+    HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, bcast);
+    HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, reduce);
+    HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, gather);
+    HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, gatherv);
+    HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, scatter);
+    HAN_SUBCOM_RESTORE_COLLECTIVE(fallbacks, comm, han_module, scatterv);
 
     OBJ_DESTRUCT(&comm_info);
     return OMPI_SUCCESS;
 }
-
-
diff --git a/ompi/mca/coll/han/coll_han_topo.c b/ompi/mca/coll/han/coll_han_topo.c
index e60d8fd819d..58597853c16 100644
--- a/ompi/mca/coll/han/coll_han_topo.c
+++ b/ompi/mca/coll/han/coll_han_topo.c
@@ -92,12 +92,19 @@ mca_coll_han_topo_init(struct ompi_communicator_t *comm,
     }
     assert(up_comm != NULL && low_comm != NULL);
 
+    int up_rank = ompi_comm_rank(up_comm);
     int low_rank = ompi_comm_rank(low_comm);
     int low_size = ompi_comm_size(low_comm);
 
+    ompi_proc_t *up_proc = NULL;
+
     int *topo = (int *)malloc(sizeof(int) * size * num_topo_level);
-    int is_imbalanced = 1;
-    int ranks_non_consecutive = 0;
+    int is_imbalanced = 1, ranks_non_consecutive = 0, is_heterogeneous = 0;
+
+    if (0 != up_rank) {
+        up_proc = ompi_comm_peer_lookup(up_comm, 0);
+        is_heterogeneous = up_proc->super.proc_convertor->remoteArch != opal_local_arch;
+    }
 
     /* node leaders translate the node-local ranks to global ranks and check whether they are placed consecutively */
     if (0 == low_rank) {
@@ -116,15 +123,16 @@ mca_coll_han_topo_init(struct ompi_communicator_t *comm,
             }
         }
 
-        int reduce_vals[] = {ranks_non_consecutive, low_size, -low_size};
+        int reduce_vals[] = {ranks_non_consecutive, low_size, -low_size, is_heterogeneous};
 
-        up_comm->c_coll->coll_allreduce(MPI_IN_PLACE, &reduce_vals, 3,
+        up_comm->c_coll->coll_allreduce(MPI_IN_PLACE, &reduce_vals, 4,
                                         MPI_INT, MPI_MAX, up_comm,
                                         up_comm->c_coll->coll_allreduce_module);
 
         /* is the distribution of processes balanced per node? */
         is_imbalanced = (reduce_vals[1] == -reduce_vals[2]) ? 0 : 1;
         ranks_non_consecutive = reduce_vals[0];
+        is_heterogeneous = reduce_vals[3];
 
         if ( ranks_non_consecutive && !is_imbalanced ) {
             /* kick off up_comm allgather to collect non-consecutive rank information at node leaders */
@@ -136,12 +144,13 @@ mca_coll_han_topo_init(struct ompi_communicator_t *comm,
     }
 
 
-    /* broadcast balanced and consecutive properties from node leaders to remaining ranks */
-    int bcast_vals[] = {is_imbalanced, ranks_non_consecutive};
-    low_comm->c_coll->coll_bcast(bcast_vals, 2, MPI_INT, 0,
+    /* broadcast balanced, consecutive and homogeneity properties from node leaders to remaining ranks */
+    int bcast_vals[] = {is_imbalanced, ranks_non_consecutive, is_heterogeneous};
+    low_comm->c_coll->coll_bcast(bcast_vals, 3, MPI_INT, 0,
                                  low_comm, low_comm->c_coll->coll_bcast_module);
     is_imbalanced = bcast_vals[0];
     ranks_non_consecutive = bcast_vals[1];
+    han_module->is_heterogeneous = bcast_vals[2];
 
     /* error out if the rank distribution is not balanced */
     if (is_imbalanced) {
diff --git a/ompi/mca/coll/han/coll_han_trigger.h b/ompi/mca/coll/han/coll_han_trigger.h
index ac4a017b98f..928da6a18d1 100644
--- a/ompi/mca/coll/han/coll_han_trigger.h
+++ b/ompi/mca/coll/han/coll_han_trigger.h
@@ -14,7 +14,7 @@
  * @file
  *
  * This file defines the API for tasks: a collective operation may be
- * splitted in tasks to balance compute load on all the resources.
+ * split in tasks to balance compute load on all the resources.
  * This solution provide some noise resiliency.
  */
 
diff --git a/ompi/mca/coll/han/configure.m4 b/ompi/mca/coll/han/configure.m4
new file mode 100644
index 00000000000..e84205eaa13
--- /dev/null
+++ b/ompi/mca/coll/han/configure.m4
@@ -0,0 +1,18 @@
+# -*- autoconf -*-
+#
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+AC_DEFUN([MCA_ompi_coll_han_CONFIG],[
+    AC_CONFIG_FILES([ompi/mca/coll/han/Makefile])
+    $1
+])dnl
+
+AC_DEFUN([MCA_ompi_coll_han_POST_CONFIG], [
+    OMPI_REQUIRE_ENDPOINT_TAG([SMSC])
+])dnl
diff --git a/ompi/mca/coll/hcoll/coll_hcoll.h b/ompi/mca/coll/hcoll/coll_hcoll.h
index 951c32b1bd5..07460aeb3f8 100644
--- a/ompi/mca/coll/hcoll/coll_hcoll.h
+++ b/ompi/mca/coll/hcoll/coll_hcoll.h
@@ -58,7 +58,7 @@ OBJ_CLASS_DECLARATION(mca_coll_hcoll_dtype_t);
 extern mca_coll_hcoll_dtype_t zero_dte_mapping;
 struct mca_coll_hcoll_component_t {
     /** Base coll component */
-    mca_coll_base_component_2_4_0_t super;
+    mca_coll_base_component_3_0_0_t super;
 
     /** MCA parameter: Priority of this component */
     int hcoll_priority;
@@ -100,7 +100,7 @@ struct mca_coll_hcoll_component_t {
 };
 typedef struct mca_coll_hcoll_component_t mca_coll_hcoll_component_t;
 
-OMPI_MODULE_DECLSPEC extern mca_coll_hcoll_component_t mca_coll_hcoll_component;
+OMPI_DECLSPEC extern mca_coll_hcoll_component_t mca_coll_hcoll_component;
 
 
 
@@ -179,87 +179,87 @@ void hcoll_rte_fns_setup(void);
 int mca_coll_hcoll_barrier(struct ompi_communicator_t *comm,
                          mca_coll_base_module_t *module);
 
-int mca_coll_hcoll_bcast(void *buff, int count,
+int mca_coll_hcoll_bcast(void *buff, size_t count,
                         struct ompi_datatype_t *datatype, int root,
                         struct ompi_communicator_t *comm,
                         mca_coll_base_module_t *module);
 
-int mca_coll_hcoll_allgather(const void *sbuf, int scount,
+int mca_coll_hcoll_allgather(const void *sbuf, size_t scount,
                             struct ompi_datatype_t *sdtype,
-                            void *rbuf, int rcount,
+                            void *rbuf, size_t rcount,
                             struct ompi_datatype_t *rdtype,
                             struct ompi_communicator_t *comm,
                             mca_coll_base_module_t *module);
 
 int mca_coll_hcoll_allgatherv(const void *sbuf, int scount,
                             struct ompi_datatype_t *sdtype,
-                            void *rbuf, const int *rcount,
-                            const int *displs,
+                            void *rbuf, ompi_count_array_t rcount,
+                            ompi_disp_array_t displs,
                             struct ompi_datatype_t *rdtype,
                             struct ompi_communicator_t *comm,
                             mca_coll_base_module_t *module);
 
-int mca_coll_hcoll_gather(const void *sbuf, int scount,
+int mca_coll_hcoll_gather(const void *sbuf, size_t scount,
                           struct ompi_datatype_t *sdtype,
-                          void *rbuf, int rcount,
+                          void *rbuf, size_t rcount,
                           struct ompi_datatype_t *rdtype,
                           int root,
                           struct ompi_communicator_t *comm,
                           mca_coll_base_module_t *module);
 
-int mca_coll_hcoll_allreduce(const void *sbuf, void *rbuf, int count,
+int mca_coll_hcoll_allreduce(const void *sbuf, void *rbuf, size_t count,
                             struct ompi_datatype_t *dtype,
                             struct ompi_op_t *op,
                             struct ompi_communicator_t *comm,
                             mca_coll_base_module_t *module);
 
 #if HCOLL_API > HCOLL_VERSION(4,5)
-int mca_coll_hcoll_reduce_scatter_block(const void *sbuf, void *rbuf, int rcount,
+int mca_coll_hcoll_reduce_scatter_block(const void *sbuf, void *rbuf, size_t rcount,
                                         struct ompi_datatype_t *dtype,
                                         struct ompi_op_t *op,
                                         struct ompi_communicator_t *comm,
                                         mca_coll_base_module_t *module);
-int mca_coll_hcoll_reduce_scatter(const void *sbuf, void *rbuf, const int* rcounts,
+int mca_coll_hcoll_reduce_scatter(const void *sbuf, void *rbuf, ompi_count_array_t rcounts,
                                   struct ompi_datatype_t *dtype,
                                   struct ompi_op_t *op,
                                   struct ompi_communicator_t *comm,
                                   mca_coll_base_module_t *module);
 #endif
-int mca_coll_hcoll_reduce(const void *sbuf, void *rbuf, int count,
+int mca_coll_hcoll_reduce(const void *sbuf, void *rbuf, size_t count,
                             struct ompi_datatype_t *dtype,
                             struct ompi_op_t *op,
                             int root,
                             struct ompi_communicator_t *comm,
                             mca_coll_base_module_t *module);
 
-int mca_coll_hcoll_alltoall(const void *sbuf, int scount,
+int mca_coll_hcoll_alltoall(const void *sbuf, size_t scount,
                            struct ompi_datatype_t *sdtype,
-                           void* rbuf, int rcount,
+                           void* rbuf, size_t rcount,
                            struct ompi_datatype_t *rdtype,
                            struct ompi_communicator_t *comm,
                            mca_coll_base_module_t *module);
 
-int mca_coll_hcoll_alltoallv(const void *sbuf, const int *scounts,
-                            const int *sdisps,
+int mca_coll_hcoll_alltoallv(const void *sbuf, ompi_count_array_t scounts,
+                            ompi_disp_array_t sdisps,
                             struct ompi_datatype_t *sdtype,
-                            void *rbuf, const int *rcounts,
-                            const int *rdisps,
+                            void *rbuf, ompi_count_array_t rcounts,
+                            ompi_disp_array_t rdisps,
                             struct ompi_datatype_t *rdtype,
                             struct ompi_communicator_t *comm,
                             mca_coll_base_module_t *module);
 
-int mca_coll_hcoll_gatherv(const void* sbuf, int scount,
+int mca_coll_hcoll_gatherv(const void* sbuf, size_t scount,
                             struct ompi_datatype_t *sdtype,
-                            void* rbuf, const int *rcounts, const int *displs,
+                            void* rbuf, ompi_count_array_t rcounts, ompi_disp_array_t displs,
                             struct ompi_datatype_t *rdtype,
                             int root,
                             struct ompi_communicator_t *comm,
                             mca_coll_base_module_t *module);
 
 
-int mca_coll_hcoll_scatterv(const void* sbuf, const int *scounts, const int *displs,
+int mca_coll_hcoll_scatterv(const void* sbuf, ompi_count_array_t scounts, ompi_disp_array_t displs,
                             struct ompi_datatype_t *sdtype,
-                            void* rbuf, int rcount,
+                            void* rbuf, size_t rcount,
                             struct ompi_datatype_t *rdtype,
                             int root,
                             struct ompi_communicator_t *comm,
@@ -269,37 +269,37 @@ int mca_coll_hcoll_ibarrier(struct ompi_communicator_t *comm,
                             ompi_request_t** request,
                             mca_coll_base_module_t *module);
 
-int mca_coll_hcoll_ibcast(void *buff, int count,
+int mca_coll_hcoll_ibcast(void *buff, size_t count,
                             struct ompi_datatype_t *datatype, int root,
                             struct ompi_communicator_t *comm,
                             ompi_request_t** request,
                             mca_coll_base_module_t *module);
 
-int mca_coll_hcoll_iallgather(const void *sbuf, int scount,
+int mca_coll_hcoll_iallgather(const void *sbuf, size_t scount,
                             struct ompi_datatype_t *sdtype,
-                            void *rbuf, int rcount,
+                            void *rbuf, size_t rcount,
                             struct ompi_datatype_t *rdtype,
                             struct ompi_communicator_t *comm,
                             ompi_request_t** request,
                             mca_coll_base_module_t *module);
 
-int mca_coll_hcoll_iallgatherv(const void *sbuf, int scount,
+int mca_coll_hcoll_iallgatherv(const void *sbuf, size_t scount,
                             struct ompi_datatype_t *sdtype,
-                            void *rbuf, const int *rcount,
-                            const int *displs,
+                            void *rbuf, ompi_count_array_t rcount,
+                            ompi_disp_array_t displs,
                             struct ompi_datatype_t *rdtype,
                             struct ompi_communicator_t *comm,
                             ompi_request_t** request,
                             mca_coll_base_module_t *module);
 
-int mca_coll_hcoll_iallreduce(const void *sbuf, void *rbuf, int count,
+int mca_coll_hcoll_iallreduce(const void *sbuf, void *rbuf, size_t count,
                             struct ompi_datatype_t *dtype,
                             struct ompi_op_t *op,
                             struct ompi_communicator_t *comm,
                             ompi_request_t** request,
                             mca_coll_base_module_t *module);
 
-int mca_coll_hcoll_ireduce(const void *sbuf, void *rbuf, int count,
+int mca_coll_hcoll_ireduce(const void *sbuf, void *rbuf, size_t count,
                             struct ompi_datatype_t *dtype,
                             struct ompi_op_t *op,
                             int root,
@@ -307,29 +307,29 @@ int mca_coll_hcoll_ireduce(const void *sbuf, void *rbuf, int count,
                             ompi_request_t** request,
                             mca_coll_base_module_t *module);
 
-int mca_coll_hcoll_ialltoall(const void *sbuf, int scount,
+int mca_coll_hcoll_ialltoall(const void *sbuf, size_t scount,
                             struct ompi_datatype_t *sdtype,
-                            void* rbuf, int rcount,
+                            void* rbuf, size_t rcount,
                             struct ompi_datatype_t *rdtype,
                             struct ompi_communicator_t *comm,
                             ompi_request_t **req,
                             mca_coll_base_module_t *module);
 
 #if HCOLL_API >= HCOLL_VERSION(3,7)
-int mca_coll_hcoll_ialltoallv(const void *sbuf, const int *scounts,
-                            const int *sdisps,
+int mca_coll_hcoll_ialltoallv(const void *sbuf, ompi_count_array_t scounts,
+                            ompi_disp_array_t sdisps,
                             struct ompi_datatype_t *sdtype,
-                            void *rbuf, const int *rcounts,
-                            const int *rdisps,
+                            void *rbuf, ompi_count_array_t rcounts,
+                            ompi_disp_array_t rdisps,
                             struct ompi_datatype_t *rdtype,
                             struct ompi_communicator_t *comm,
                             ompi_request_t **req,
                             mca_coll_base_module_t *module);
 #endif
 
-int mca_coll_hcoll_igatherv(const void* sbuf, int scount,
+int mca_coll_hcoll_igatherv(const void* sbuf, size_t scount,
                             struct ompi_datatype_t *sdtype,
-                            void* rbuf, const int *rcounts, const int *displs,
+                            void* rbuf, ompi_count_array_t rcounts, ompi_disp_array_t displs,
                             struct ompi_datatype_t *rdtype,
                             int root,
                             struct ompi_communicator_t *comm,
diff --git a/ompi/mca/coll/hcoll/coll_hcoll_component.c b/ompi/mca/coll/hcoll/coll_hcoll_component.c
index cee65bc87ce..e34169a0781 100644
--- a/ompi/mca/coll/hcoll/coll_hcoll_component.c
+++ b/ompi/mca/coll/hcoll/coll_hcoll_component.c
@@ -3,6 +3,7 @@
  * Copyright (c) 2011 Mellanox Technologies. All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2024 NVIDIA CORPORATION. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -36,7 +37,7 @@ mca_coll_hcoll_component_t mca_coll_hcoll_component = {
        about the component itfca */
     {
         .collm_version = {
-            MCA_COLL_BASE_VERSION_2_4_0,
+            MCA_COLL_BASE_VERSION_3_0_0,
 
             /* Component name and version */
             .mca_component_name = "hcoll",
@@ -107,8 +108,8 @@ static int reg_int(const char* param_name,
     index = mca_base_component_var_register(
             &mca_coll_hcoll_component.super.collm_version,
             param_name, param_desc, MCA_BASE_VAR_TYPE_INT,
-            NULL, 0, 0,OPAL_INFO_LVL_9,
-            MCA_BASE_VAR_SCOPE_READONLY, storage);
+            NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,OPAL_INFO_LVL_9,
+            MCA_BASE_VAR_SCOPE_ALL, storage);
     if (NULL != deprecated_param_name) {
         (void) mca_base_var_register_synonym(index,
                 "ompi", "coll", "hcoll", deprecated_param_name,
diff --git a/ompi/mca/coll/hcoll/coll_hcoll_dtypes.h b/ompi/mca/coll/hcoll/coll_hcoll_dtypes.h
index bf1126ee823..481ff855e58 100644
--- a/ompi/mca/coll/hcoll/coll_hcoll_dtypes.h
+++ b/ompi/mca/coll/hcoll/coll_hcoll_dtypes.h
@@ -34,7 +34,7 @@ extern mca_coll_hcoll_dtype_t zero_dte_mapping;
 total 15 types
 */
 
-static dte_data_representation_t* ompi_datatype_2_dte_data_rep[OMPI_DATATYPE_MAX_PREDEFINED] = {
+static dte_data_representation_t* ompi_datatype_2_dte_data_rep[OPAL_DATATYPE_MAX_PREDEFINED] = {
     &DTE_ZERO,                  /*OPAL_DATATYPE_LOOP           0 */
     &DTE_ZERO,                  /*OPAL_DATATYPE_END_LOOP       1 */
 #if defined(DTE_LB)
@@ -92,7 +92,17 @@ static dte_data_representation_t* ompi_datatype_2_dte_data_rep[OMPI_DATATYPE_MAX
 #else
     &DTE_ZERO,
 #endif
-    &DTE_ZERO                   /*OPAL_DATATYPE_UNAVAILABLE    25 */
+#if SIZEOF_LONG == 4
+    &DTE_INT32,
+#else
+    &DTE_INT64,                /*OPAL_DATATYPE_LONG           25 */
+#endif
+#if SIZEOF_UNSIGNED_LONG == 4
+    &DTE_UINT32,
+#else
+    &DTE_UINT64,                /*OPAL_DATATYPE_UNSIGNED_LONG  26 */
+#endif
+    &DTE_ZERO                   /*OPAL_DATATYPE_UNAVAILABLE    27 */
 };
 
 enum {
diff --git a/ompi/mca/coll/hcoll/coll_hcoll_module.c b/ompi/mca/coll/hcoll/coll_hcoll_module.c
index d09607d8d02..5ca588a8154 100644
--- a/ompi/mca/coll/hcoll/coll_hcoll_module.c
+++ b/ompi/mca/coll/hcoll/coll_hcoll_module.c
@@ -1,10 +1,13 @@
 /**
  * Copyright (c) 2011 Mellanox Technologies. All rights reserved.
- * Copyright (c) 2016      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2016-2022 IBM Corporation.  All rights reserved.
  * Copyright (c) 2017      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2018      Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -27,7 +30,7 @@ int mca_coll_hcoll_init_query(bool enable_progress_threads, bool enable_mpi_thre
 {
 #if HCOLL_API < HCOLL_VERSION(3,2)
     if (enable_mpi_threads) {
-        HCOL_VERBOSE(1, "MPI_THREAD_MULTIPLE not suppported; skipping hcoll component");
+        HCOL_VERBOSE(1, "MPI_THREAD_MULTIPLE not supported; skipping hcoll component");
         return OMPI_ERROR;
     }
 #endif
@@ -84,8 +87,6 @@ static void mca_coll_hcoll_module_clear(mca_coll_hcoll_module_t *hcoll_module)
     hcoll_module->previous_igatherv_module    = NULL;
     hcoll_module->previous_ialltoall_module   = NULL;
     hcoll_module->previous_ialltoallv_module  = NULL;
-
-
 }
 
 static void mca_coll_hcoll_module_construct(mca_coll_hcoll_module_t *hcoll_module)
@@ -99,8 +100,6 @@ void mca_coll_hcoll_mem_release_cb(void *buf, size_t length,
     hcoll_mem_unmap(buf, length, cbdata, from_alloc);
 }
 
-#define OBJ_RELEASE_IF_NOT_NULL( obj ) if( NULL != (obj) ) OBJ_RELEASE( obj );
-
 static void mca_coll_hcoll_module_destruct(mca_coll_hcoll_module_t *hcoll_module)
 {
     int context_destroyed;
@@ -117,37 +116,7 @@ static void mca_coll_hcoll_module_destruct(mca_coll_hcoll_module_t *hcoll_module
        destroy hcoll context*/
 
     if (hcoll_module->hcoll_context != NULL){
-        OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_barrier_module);
-        OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_bcast_module);
-        OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_allreduce_module);
-        OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_reduce_scatter_block_module);
-        OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_reduce_scatter_module);
-        OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_allgather_module);
-        OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_allgatherv_module);
-        OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_gatherv_module);
-        OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_scatterv_module);
-        OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_alltoall_module);
-        OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_alltoallv_module);
-        OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_reduce_module);
-
-        OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_ibarrier_module);
-        OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_ibcast_module);
-        OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_iallreduce_module);
-        OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_iallgather_module);
-        OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_iallgatherv_module);
-        OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_igatherv_module);
-        OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_ialltoall_module);
-        OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_ialltoallv_module);
-        OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_ireduce_module);
-
-        /*
-        OBJ_RELEASE(hcoll_module->previous_allgatherv_module);
-        OBJ_RELEASE(hcoll_module->previous_gather_module);
-        OBJ_RELEASE(hcoll_module->previous_gatherv_module);
-        OBJ_RELEASE(hcoll_module->previous_alltoallw_module);
-        OBJ_RELEASE(hcoll_module->previous_reduce_scatter_module);
-        OBJ_RELEASE(hcoll_module->previous_reduce_module);
-        */
+
 #if !defined(HAVE_HCOLL_CONTEXT_FREE)
         context_destroyed = 0;
         hcoll_destroy_context(hcoll_module->hcoll_context,
@@ -158,52 +127,105 @@ static void mca_coll_hcoll_module_destruct(mca_coll_hcoll_module_t *hcoll_module
     mca_coll_hcoll_module_clear(hcoll_module);
 }
 
-#define HCOL_SAVE_PREV_COLL_API(__api) do {\
-    hcoll_module->previous_ ## __api            = comm->c_coll->coll_ ## __api;\
-    hcoll_module->previous_ ## __api ## _module = comm->c_coll->coll_ ## __api ## _module;\
-    if (!comm->c_coll->coll_ ## __api || !comm->c_coll->coll_ ## __api ## _module) {\
-        return OMPI_ERROR;\
-    }\
-    OBJ_RETAIN(hcoll_module->previous_ ## __api ## _module);\
-} while(0)
-
+#define HCOL_INSTALL_COLL_API(__comm, __module, __api)                                                                              \
+    do                                                                                                                              \
+    {                                                                                                                               \
+        if (NULL != __module->super.coll_##__api)                                                                                   \
+        {                                                                                                                           \
+            if (comm->c_coll->coll_##__api && !comm->c_coll->coll_##__api##_module)                                                 \
+            {                                                                                                                       \
+                /* save the current selected collective */                                                                          \
+                MCA_COLL_SAVE_API(__comm, __api, hcoll_module->previous_##__api, hcoll_module->previous_##__api##_module, "hcoll"); \
+                /* install our own */                                                                                               \
+                MCA_COLL_INSTALL_API(__comm, __api, __module->super.coll_##__api, &__module->super, "hcoll");                       \
+            }                                                                                                                       \
+        }                                                                                                                           \
+    } while (0)
+
+#define HCOL_UNINSTALL_COLL_API(__comm, __module, __api)                                                                   \
+    do                                                                                                                     \
+    {                                                                                                                      \
+        if (&__module->super == comm->c_coll->coll_##__api##_module)                                                       \
+        {                                                                                                                  \
+            MCA_COLL_INSTALL_API(__comm, __api, __module->previous_##__api, __module->previous_##__api##_module, "hcoll"); \
+            hcoll_module->previous_##__api = NULL;                                                                         \
+            hcoll_module->previous_##__api##_module = NULL;                                                                \
+        }                                                                                                                  \
+    } while (0)
 
 static int mca_coll_hcoll_save_coll_handlers(mca_coll_hcoll_module_t *hcoll_module)
 {
     ompi_communicator_t *comm;
     comm = hcoll_module->comm;
 
-    HCOL_SAVE_PREV_COLL_API(barrier);
-    HCOL_SAVE_PREV_COLL_API(bcast);
-    HCOL_SAVE_PREV_COLL_API(allreduce);
-    HCOL_SAVE_PREV_COLL_API(reduce_scatter_block);
-    HCOL_SAVE_PREV_COLL_API(reduce_scatter);
-    HCOL_SAVE_PREV_COLL_API(reduce);
-    HCOL_SAVE_PREV_COLL_API(allgather);
-    HCOL_SAVE_PREV_COLL_API(allgatherv);
-    HCOL_SAVE_PREV_COLL_API(gatherv);
-    HCOL_SAVE_PREV_COLL_API(scatterv);
-    HCOL_SAVE_PREV_COLL_API(alltoall);
-    HCOL_SAVE_PREV_COLL_API(alltoallv);
-
-    HCOL_SAVE_PREV_COLL_API(ibarrier);
-    HCOL_SAVE_PREV_COLL_API(ibcast);
-    HCOL_SAVE_PREV_COLL_API(iallreduce);
-    HCOL_SAVE_PREV_COLL_API(ireduce);
-    HCOL_SAVE_PREV_COLL_API(iallgather);
-    HCOL_SAVE_PREV_COLL_API(iallgatherv);
-    HCOL_SAVE_PREV_COLL_API(igatherv);
-    HCOL_SAVE_PREV_COLL_API(ialltoall);
-    HCOL_SAVE_PREV_COLL_API(ialltoallv);
+    hcoll_module->super.coll_barrier = hcoll_collectives.coll_barrier ? mca_coll_hcoll_barrier : NULL;
+    hcoll_module->super.coll_bcast = hcoll_collectives.coll_bcast ? mca_coll_hcoll_bcast : NULL;
+    hcoll_module->super.coll_allgather = hcoll_collectives.coll_allgather ? mca_coll_hcoll_allgather : NULL;
+    hcoll_module->super.coll_allgatherv = hcoll_collectives.coll_allgatherv ? mca_coll_hcoll_allgatherv : NULL;
+    hcoll_module->super.coll_allreduce = hcoll_collectives.coll_allreduce ? mca_coll_hcoll_allreduce : NULL;
+    hcoll_module->super.coll_alltoall = hcoll_collectives.coll_alltoall ? mca_coll_hcoll_alltoall : NULL;
+    hcoll_module->super.coll_alltoallv = hcoll_collectives.coll_alltoallv ? mca_coll_hcoll_alltoallv : NULL;
+    hcoll_module->super.coll_gatherv = hcoll_collectives.coll_gatherv ? mca_coll_hcoll_gatherv : NULL;
+    hcoll_module->super.coll_scatterv = hcoll_collectives.coll_scatterv ? mca_coll_hcoll_scatterv : NULL;
+    hcoll_module->super.coll_reduce = hcoll_collectives.coll_reduce ? mca_coll_hcoll_reduce : NULL;
+    hcoll_module->super.coll_ibarrier = hcoll_collectives.coll_ibarrier ? mca_coll_hcoll_ibarrier : NULL;
+    hcoll_module->super.coll_ibcast = hcoll_collectives.coll_ibcast ? mca_coll_hcoll_ibcast : NULL;
+    hcoll_module->super.coll_iallgather = hcoll_collectives.coll_iallgather ? mca_coll_hcoll_iallgather : NULL;
+#if HCOLL_API >= HCOLL_VERSION(3, 5)
+    hcoll_module->super.coll_iallgatherv = hcoll_collectives.coll_iallgatherv ? mca_coll_hcoll_iallgatherv : NULL;
+#else
+    hcoll_module->super.coll_iallgatherv = NULL;
+#endif
+    hcoll_module->super.coll_iallreduce = hcoll_collectives.coll_iallreduce ? mca_coll_hcoll_iallreduce : NULL;
+#if HCOLL_API >= HCOLL_VERSION(3, 5)
+    hcoll_module->super.coll_ireduce = hcoll_collectives.coll_ireduce ? mca_coll_hcoll_ireduce : NULL;
+#else
+    hcoll_module->super.coll_ireduce = NULL;
+#endif
+    hcoll_module->super.coll_gather = /*hcoll_collectives.coll_gather ? mca_coll_hcoll_gather :*/ NULL;
+    hcoll_module->super.coll_igatherv = hcoll_collectives.coll_igatherv ? mca_coll_hcoll_igatherv : NULL;
+    hcoll_module->super.coll_ialltoall = /*hcoll_collectives.coll_ialltoall ? mca_coll_hcoll_ialltoall : */ NULL;
+#if HCOLL_API >= HCOLL_VERSION(3, 7)
+    hcoll_module->super.coll_ialltoallv = hcoll_collectives.coll_ialltoallv ? mca_coll_hcoll_ialltoallv : NULL;
+#else
+    hcoll_module->super.coll_ialltoallv = NULL;
+#endif
+#if HCOLL_API > HCOLL_VERSION(4, 5)
+    hcoll_module->super.coll_reduce_scatter_block = hcoll_collectives.coll_reduce_scatter_block ? mca_coll_hcoll_reduce_scatter_block : NULL;
+    hcoll_module->super.coll_reduce_scatter = hcoll_collectives.coll_reduce_scatter ? mca_coll_hcoll_reduce_scatter : NULL;
+#endif
+
+    HCOL_INSTALL_COLL_API(comm, hcoll_module, barrier);
+    HCOL_INSTALL_COLL_API(comm, hcoll_module, bcast);
+    HCOL_INSTALL_COLL_API(comm, hcoll_module, allreduce);
+    HCOL_INSTALL_COLL_API(comm, hcoll_module, reduce_scatter_block);
+    HCOL_INSTALL_COLL_API(comm, hcoll_module, reduce_scatter);
+    HCOL_INSTALL_COLL_API(comm, hcoll_module, reduce);
+    HCOL_INSTALL_COLL_API(comm, hcoll_module, allgather);
+    HCOL_INSTALL_COLL_API(comm, hcoll_module, allgatherv);
+    HCOL_INSTALL_COLL_API(comm, hcoll_module, gatherv);
+    HCOL_INSTALL_COLL_API(comm, hcoll_module, scatterv);
+    HCOL_INSTALL_COLL_API(comm, hcoll_module, alltoall);
+    HCOL_INSTALL_COLL_API(comm, hcoll_module, alltoallv);
+
+    HCOL_INSTALL_COLL_API(comm, hcoll_module, ibarrier);
+    HCOL_INSTALL_COLL_API(comm, hcoll_module, ibcast);
+    HCOL_INSTALL_COLL_API(comm, hcoll_module, iallreduce);
+    HCOL_INSTALL_COLL_API(comm, hcoll_module, ireduce);
+    HCOL_INSTALL_COLL_API(comm, hcoll_module, iallgather);
+    HCOL_INSTALL_COLL_API(comm, hcoll_module, iallgatherv);
+    HCOL_INSTALL_COLL_API(comm, hcoll_module, igatherv);
+    HCOL_INSTALL_COLL_API(comm, hcoll_module, ialltoall);
+    HCOL_INSTALL_COLL_API(comm, hcoll_module, ialltoallv);
 
     /*
       These collectives are not yet part of hcoll, so
       don't retain them on hcoll module
-    HCOL_SAVE_PREV_COLL_API(reduce_scatter);
-    HCOL_SAVE_PREV_COLL_API(gather);
-    HCOL_SAVE_PREV_COLL_API(reduce);
-    HCOL_SAVE_PREV_COLL_API(allgatherv);
-    HCOL_SAVE_PREV_COLL_API(alltoallw);
+    HCOL_INSTALL_COLL_API(comm, hcoll_module, reduce_scatter);
+    HCOL_INSTALL_COLL_API(comm, hcoll_module, gather);
+    HCOL_INSTALL_COLL_API(comm, hcoll_module, reduce);
+    HCOL_INSTALL_COLL_API(comm, hcoll_module, allgatherv);
+    HCOL_INSTALL_COLL_API(comm, hcoll_module, alltoallw);
     */
     return OMPI_SUCCESS;
 }
@@ -249,6 +271,45 @@ static int mca_coll_hcoll_module_enable(mca_coll_base_module_t *module,
     return OMPI_SUCCESS;
 }
 
+static int mca_coll_hcoll_module_disable(mca_coll_base_module_t *module,
+                                         struct ompi_communicator_t *comm)
+{
+    mca_coll_hcoll_module_t *hcoll_module = (mca_coll_hcoll_module_t *)module;
+
+    HCOL_UNINSTALL_COLL_API(comm, hcoll_module, barrier);
+    HCOL_UNINSTALL_COLL_API(comm, hcoll_module, bcast);
+    HCOL_UNINSTALL_COLL_API(comm, hcoll_module, allreduce);
+    HCOL_UNINSTALL_COLL_API(comm, hcoll_module, reduce_scatter_block);
+    HCOL_UNINSTALL_COLL_API(comm, hcoll_module, reduce_scatter);
+    HCOL_UNINSTALL_COLL_API(comm, hcoll_module, reduce);
+    HCOL_UNINSTALL_COLL_API(comm, hcoll_module, allgather);
+    HCOL_UNINSTALL_COLL_API(comm, hcoll_module, allgatherv);
+    HCOL_UNINSTALL_COLL_API(comm, hcoll_module, gatherv);
+    HCOL_UNINSTALL_COLL_API(comm, hcoll_module, scatterv);
+    HCOL_UNINSTALL_COLL_API(comm, hcoll_module, alltoall);
+    HCOL_UNINSTALL_COLL_API(comm, hcoll_module, alltoallv);
+
+    HCOL_UNINSTALL_COLL_API(comm, hcoll_module, ibarrier);
+    HCOL_UNINSTALL_COLL_API(comm, hcoll_module, ibcast);
+    HCOL_UNINSTALL_COLL_API(comm, hcoll_module, iallreduce);
+    HCOL_UNINSTALL_COLL_API(comm, hcoll_module, ireduce);
+    HCOL_UNINSTALL_COLL_API(comm, hcoll_module, iallgather);
+    HCOL_UNINSTALL_COLL_API(comm, hcoll_module, iallgatherv);
+    HCOL_UNINSTALL_COLL_API(comm, hcoll_module, igatherv);
+    HCOL_UNINSTALL_COLL_API(comm, hcoll_module, ialltoall);
+    HCOL_UNINSTALL_COLL_API(comm, hcoll_module, ialltoallv);
+
+    /*
+      These collectives are not yet part of hcoll, so
+      don't retain them on hcoll module
+    HCOL_UNINSTALL_COLL_API(comm, hcoll_module, reduce_scatter);
+    HCOL_UNINSTALL_COLL_API(comm, hcoll_module, gather);
+    HCOL_UNINSTALL_COLL_API(comm, hcoll_module, reduce);
+    HCOL_UNINSTALL_COLL_API(comm, hcoll_module, allgatherv);
+    HCOL_UNINSTALL_COLL_API(comm, hcoll_module, alltoallw);
+    */
+    return OMPI_SUCCESS;
+}
 
 OBJ_CLASS_INSTANCE(mca_coll_hcoll_dtype_t,
                    opal_free_list_item_t,
@@ -285,7 +346,7 @@ mca_coll_hcoll_comm_query(struct ompi_communicator_t *comm, int *priority)
 
     if (!cm->libhcoll_initialized)
     {
-        /* libhcoll should be initialized here since current implmentation of
+        /* libhcoll should be initialized here since current implementation of
            mxm bcol in libhcoll needs world_group fully functional during init
            world_group, i.e. ompi_comm_world, is not ready at hcoll component open
            call */
@@ -332,7 +393,7 @@ mca_coll_hcoll_comm_query(struct ompi_communicator_t *comm, int *priority)
         } else {
             cm->using_mem_hooks = 0;
         }
-        copy_fn.attr_communicator_copy_fn = (MPI_Comm_internal_copy_attr_function*) MPI_COMM_NULL_COPY_FN;
+        copy_fn.attr_communicator_copy_fn = MPI_COMM_NULL_COPY_FN;
         del_fn.attr_communicator_delete_fn = hcoll_comm_attr_del_fn;
         err = ompi_attr_create_keyval(COMM_ATTR, copy_fn, del_fn, &hcoll_comm_attr_keyval, NULL ,0, NULL);
         if (OMPI_SUCCESS != err) {
@@ -345,7 +406,7 @@ mca_coll_hcoll_comm_query(struct ompi_communicator_t *comm, int *priority)
 
         if (mca_coll_hcoll_component.derived_types_support_enabled) {
             zero_dte_mapping.type = DTE_ZERO;
-            copy_fn.attr_datatype_copy_fn = (MPI_Type_internal_copy_attr_function *) MPI_TYPE_NULL_COPY_FN;
+            copy_fn.attr_datatype_copy_fn = MPI_TYPE_NULL_COPY_FN;
             del_fn.attr_datatype_delete_fn = hcoll_type_attr_del_fn;
             err = ompi_attr_create_keyval(TYPE_ATTR, copy_fn, del_fn, &hcoll_type_attr_keyval, NULL ,0, NULL);
             if (OMPI_SUCCESS != err) {
@@ -376,7 +437,7 @@ mca_coll_hcoll_comm_query(struct ompi_communicator_t *comm, int *priority)
     hcoll_module->comm = comm;
 
     HCOL_VERBOSE(10,"Creating hcoll_context for comm %p, comm_id %d, comm_size %d",
-                 (void*)comm,comm->c_contextid,ompi_comm_size(comm));
+                 (void*)comm,comm->c_index,ompi_comm_size(comm));
 
     hcoll_module->hcoll_context =
         hcoll_create_context((rte_grp_handle_t)comm);
@@ -393,44 +454,8 @@ mca_coll_hcoll_comm_query(struct ompi_communicator_t *comm, int *priority)
     }
 
     hcoll_module->super.coll_module_enable = mca_coll_hcoll_module_enable;
-    hcoll_module->super.coll_barrier = hcoll_collectives.coll_barrier ? mca_coll_hcoll_barrier : NULL;
-    hcoll_module->super.coll_bcast = hcoll_collectives.coll_bcast ? mca_coll_hcoll_bcast : NULL;
-    hcoll_module->super.coll_allgather = hcoll_collectives.coll_allgather ? mca_coll_hcoll_allgather : NULL;
-    hcoll_module->super.coll_allgatherv = hcoll_collectives.coll_allgatherv ? mca_coll_hcoll_allgatherv : NULL;
-    hcoll_module->super.coll_allreduce = hcoll_collectives.coll_allreduce ? mca_coll_hcoll_allreduce : NULL;
-    hcoll_module->super.coll_alltoall = hcoll_collectives.coll_alltoall ? mca_coll_hcoll_alltoall : NULL;
-    hcoll_module->super.coll_alltoallv = hcoll_collectives.coll_alltoallv ? mca_coll_hcoll_alltoallv : NULL;
-    hcoll_module->super.coll_gatherv = hcoll_collectives.coll_gatherv ? mca_coll_hcoll_gatherv : NULL;
-    hcoll_module->super.coll_scatterv = hcoll_collectives.coll_scatterv ? mca_coll_hcoll_scatterv : NULL;
-    hcoll_module->super.coll_reduce = hcoll_collectives.coll_reduce ? mca_coll_hcoll_reduce : NULL;
-    hcoll_module->super.coll_ibarrier = hcoll_collectives.coll_ibarrier ? mca_coll_hcoll_ibarrier : NULL;
-    hcoll_module->super.coll_ibcast = hcoll_collectives.coll_ibcast ? mca_coll_hcoll_ibcast : NULL;
-    hcoll_module->super.coll_iallgather = hcoll_collectives.coll_iallgather ? mca_coll_hcoll_iallgather : NULL;
-#if HCOLL_API >= HCOLL_VERSION(3,5)
-    hcoll_module->super.coll_iallgatherv = hcoll_collectives.coll_iallgatherv ? mca_coll_hcoll_iallgatherv : NULL;
-#else
-    hcoll_module->super.coll_iallgatherv = NULL;
-#endif
-    hcoll_module->super.coll_iallreduce = hcoll_collectives.coll_iallreduce ? mca_coll_hcoll_iallreduce : NULL;
-#if HCOLL_API >= HCOLL_VERSION(3,5)
-    hcoll_module->super.coll_ireduce = hcoll_collectives.coll_ireduce ? mca_coll_hcoll_ireduce : NULL;
-#else
-    hcoll_module->super.coll_ireduce = NULL;
-#endif
-    hcoll_module->super.coll_gather = /*hcoll_collectives.coll_gather ? mca_coll_hcoll_gather :*/ NULL;
-    hcoll_module->super.coll_igatherv = hcoll_collectives.coll_igatherv ? mca_coll_hcoll_igatherv : NULL;
-    hcoll_module->super.coll_ialltoall = /*hcoll_collectives.coll_ialltoall ? mca_coll_hcoll_ialltoall : */ NULL;
-#if HCOLL_API >= HCOLL_VERSION(3,7)
-    hcoll_module->super.coll_ialltoallv = hcoll_collectives.coll_ialltoallv ? mca_coll_hcoll_ialltoallv : NULL;
-#else
-    hcoll_module->super.coll_ialltoallv = NULL;
-#endif
-#if HCOLL_API > HCOLL_VERSION(4,5)
-    hcoll_module->super.coll_reduce_scatter_block = hcoll_collectives.coll_reduce_scatter_block ?
-        mca_coll_hcoll_reduce_scatter_block : NULL;
-    hcoll_module->super.coll_reduce_scatter = hcoll_collectives.coll_reduce_scatter ?
-        mca_coll_hcoll_reduce_scatter : NULL;
-#endif
+    hcoll_module->super.coll_module_disable = mca_coll_hcoll_module_disable;
+
     *priority = cm->hcoll_priority;
     module = &hcoll_module->super;
 
@@ -447,5 +472,7 @@ OBJ_CLASS_INSTANCE(mca_coll_hcoll_module_t,
         mca_coll_hcoll_module_construct,
         mca_coll_hcoll_module_destruct);
 
-
-
+static void safety_valve(void) __opal_attribute_destructor__;
+void safety_valve(void) {
+    opal_mem_hooks_unregister_release(mca_coll_hcoll_mem_release_cb);
+}
diff --git a/ompi/mca/coll/hcoll/coll_hcoll_ops.c b/ompi/mca/coll/hcoll/coll_hcoll_ops.c
index 0f52a25b752..e491899d2dd 100644
--- a/ompi/mca/coll/hcoll/coll_hcoll_ops.c
+++ b/ompi/mca/coll/hcoll/coll_hcoll_ops.c
@@ -36,7 +36,7 @@ int mca_coll_hcoll_barrier(struct ompi_communicator_t *comm,
     return hcoll_module->previous_barrier(comm,hcoll_module->previous_barrier_module);
 }
 
-int mca_coll_hcoll_bcast(void *buff, int count,
+int mca_coll_hcoll_bcast(void *buff, size_t count,
                         struct ompi_datatype_t *datatype, int root,
                         struct ompi_communicator_t *comm,
                         mca_coll_base_module_t *module)
@@ -65,9 +65,9 @@ int mca_coll_hcoll_bcast(void *buff, int count,
     return rc;
 }
 
-int mca_coll_hcoll_allgather(const void *sbuf, int scount,
+int mca_coll_hcoll_allgather(const void *sbuf, size_t scount,
                             struct ompi_datatype_t *sdtype,
-                            void *rbuf, int rcount,
+                            void *rbuf, size_t rcount,
                             struct ompi_datatype_t *rdtype,
                             struct ompi_communicator_t *comm,
                             mca_coll_base_module_t *module)
@@ -108,8 +108,8 @@ int mca_coll_hcoll_allgather(const void *sbuf, int scount,
 
 int mca_coll_hcoll_allgatherv(const void *sbuf, int scount,
                             struct ompi_datatype_t *sdtype,
-                            void *rbuf, const int *rcount,
-                            const int *displs,
+                            void *rbuf, ompi_count_array_t rcount,
+                            ompi_disp_array_t displs,
                             struct ompi_datatype_t *rdtype,
                             struct ompi_communicator_t *comm,
                             mca_coll_base_module_t *module)
@@ -121,7 +121,8 @@ int mca_coll_hcoll_allgatherv(const void *sbuf, int scount,
     mca_coll_hcoll_module_t *hcoll_module = (mca_coll_hcoll_module_t*)module;
     stype = ompi_dtype_2_hcoll_dtype(sdtype, NO_DERIVED);
     rtype = ompi_dtype_2_hcoll_dtype(rdtype, NO_DERIVED);
-    if (OPAL_UNLIKELY(HCOL_DTE_IS_ZERO(stype) || HCOL_DTE_IS_ZERO(rtype))) {
+    if (OPAL_UNLIKELY(HCOL_DTE_IS_ZERO(stype) || HCOL_DTE_IS_ZERO(rtype)
+				              || ompi_count_array_is_64bit(rcount))) {
         /*If we are here then datatype is not simple predefined datatype */
         /*In future we need to add more complex mapping to the dte_data_representation_t */
         /* Now use fallback */
@@ -136,8 +137,14 @@ int mca_coll_hcoll_allgatherv(const void *sbuf, int scount,
                                              hcoll_module->previous_allgatherv_module);
         return rc;
     }
-    rc = hcoll_collectives.coll_allgatherv((void *)sbuf,scount,stype,rbuf,(int*)rcount,
-                                           (int*)displs,rtype,hcoll_module->hcoll_context);
+    rc = hcoll_collectives.coll_allgatherv((void *)sbuf,
+                                           scount,
+                                           stype,
+                                           rbuf,
+                                           (void *)ompi_count_array_ptr(rcount),
+                                           (void *)ompi_disp_array_ptr(displs),
+                                           rtype,
+                                           hcoll_module->hcoll_context);
     if (HCOLL_SUCCESS != rc){
         HCOL_VERBOSE(20,"RUNNING FALLBACK ALLGATHERV");
         rc = hcoll_module->previous_allgatherv(sbuf,scount,sdtype,
@@ -150,18 +157,25 @@ int mca_coll_hcoll_allgatherv(const void *sbuf, int scount,
     return rc;
 }
 
-int mca_coll_hcoll_gather(const void *sbuf, int scount,
+int mca_coll_hcoll_gather(const void *sbuf, size_t scount,
                           struct ompi_datatype_t *sdtype,
-                          void *rbuf, int rcount,
+                          void *rbuf, size_t rcount,
                           struct ompi_datatype_t *rdtype,
                           int root,
                           struct ompi_communicator_t *comm,
-                          mca_coll_base_module_t *module){
+                          mca_coll_base_module_t *module)
+{
+    mca_coll_hcoll_module_t  *hcoll_module = (mca_coll_hcoll_module_t*)module;
     dte_data_representation_t stype;
     dte_data_representation_t rtype;
     int rc;
+
     HCOL_VERBOSE(20,"RUNNING HCOL GATHER");
-    mca_coll_hcoll_module_t *hcoll_module = (mca_coll_hcoll_module_t*)module;
+
+    if (root != comm->c_my_rank) {
+        rdtype = sdtype;
+    }
+
     stype = ompi_dtype_2_hcoll_dtype(sdtype, NO_DERIVED);
     rtype = ompi_dtype_2_hcoll_dtype(rdtype, NO_DERIVED);
     if (OPAL_UNLIKELY(HCOL_DTE_IS_ZERO(stype) || HCOL_DTE_IS_ZERO(rtype))) {
@@ -189,7 +203,7 @@ int mca_coll_hcoll_gather(const void *sbuf, int scount,
 
 }
 
-int mca_coll_hcoll_allreduce(const void *sbuf, void *rbuf, int count,
+int mca_coll_hcoll_allreduce(const void *sbuf, void *rbuf, size_t count,
                             struct ompi_datatype_t *dtype,
                             struct ompi_op_t *op,
                             struct ompi_communicator_t *comm,
@@ -236,7 +250,7 @@ int mca_coll_hcoll_allreduce(const void *sbuf, void *rbuf, int count,
     return rc;
 }
 
-int mca_coll_hcoll_reduce(const void *sbuf, void *rbuf, int count,
+int mca_coll_hcoll_reduce(const void *sbuf, void *rbuf, size_t count,
                             struct ompi_datatype_t *dtype,
                             struct ompi_op_t *op,
                             int root,
@@ -287,9 +301,9 @@ int mca_coll_hcoll_reduce(const void *sbuf, void *rbuf, int count,
     return rc;
 }
 
-int mca_coll_hcoll_alltoall(const void *sbuf, int scount,
+int mca_coll_hcoll_alltoall(const void *sbuf, size_t scount,
                            struct ompi_datatype_t *sdtype,
-                           void* rbuf, int rcount,
+                           void* rbuf, size_t rcount,
                            struct ompi_datatype_t *rdtype,
                            struct ompi_communicator_t *comm,
                            mca_coll_base_module_t *module)
@@ -325,9 +339,9 @@ int mca_coll_hcoll_alltoall(const void *sbuf, int scount,
     return rc;
 }
 
-int mca_coll_hcoll_alltoallv(const void *sbuf, const int *scounts, const int *sdisps,
+int mca_coll_hcoll_alltoallv(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
                             struct ompi_datatype_t *sdtype,
-                            void *rbuf, const int *rcounts, const int *rdisps,
+                            void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
                             struct ompi_datatype_t *rdtype,
                             struct ompi_communicator_t *comm,
                             mca_coll_base_module_t *module)
@@ -339,7 +353,8 @@ int mca_coll_hcoll_alltoallv(const void *sbuf, const int *scounts, const int *sd
     mca_coll_hcoll_module_t *hcoll_module = (mca_coll_hcoll_module_t*)module;
     stype = ompi_dtype_2_hcoll_dtype(sdtype, NO_DERIVED);
     rtype = ompi_dtype_2_hcoll_dtype(rdtype, NO_DERIVED);
-    if (OPAL_UNLIKELY(HCOL_DTE_IS_ZERO(stype) || HCOL_DTE_IS_ZERO(rtype))) {
+    if (OPAL_UNLIKELY(HCOL_DTE_IS_ZERO(stype) || HCOL_DTE_IS_ZERO(rtype) 
+			                      || ompi_count_array_is_64bit(scounts))) {
         HCOL_VERBOSE(20,"Ompi_datatype is not supported: sdtype = %s, rdtype = %s; calling fallback alltoallv;",
                      sdtype->super.name,
                      rdtype->super.name);
@@ -348,9 +363,15 @@ int mca_coll_hcoll_alltoallv(const void *sbuf, const int *scounts, const int *sd
                                             comm, hcoll_module->previous_alltoallv_module);
         return rc;
     }
-    rc = hcoll_collectives.coll_alltoallv((void *)sbuf, (int *)scounts, (int *)sdisps, stype,
-                                            rbuf, (int *)rcounts, (int *)rdisps, rtype,
-                                                hcoll_module->hcoll_context);
+    rc = hcoll_collectives.coll_alltoallv((void *)sbuf,
+                                          (void *)ompi_count_array_ptr(scounts),
+                                          (void *)ompi_disp_array_ptr(sdisps),
+                                          stype,
+                                          rbuf,
+                                          (void *)ompi_count_array_ptr(rcounts),
+                                          (void *)ompi_disp_array_ptr(rdisps),
+                                          rtype,
+                                          hcoll_module->hcoll_context);
     if (HCOLL_SUCCESS != rc){
         HCOL_VERBOSE(20,"RUNNING FALLBACK ALLTOALLV");
         rc = hcoll_module->previous_alltoallv(sbuf, scounts, sdisps, sdtype,
@@ -360,22 +381,29 @@ int mca_coll_hcoll_alltoallv(const void *sbuf, const int *scounts, const int *sd
     return rc;
 }
 
-int mca_coll_hcoll_gatherv(const void* sbuf, int scount,
+int mca_coll_hcoll_gatherv(const void* sbuf, size_t scount,
                             struct ompi_datatype_t *sdtype,
-                            void* rbuf, const int *rcounts, const int *displs,
+                            void* rbuf, ompi_count_array_t rcounts, ompi_disp_array_t displs,
                             struct ompi_datatype_t *rdtype,
                             int root,
                             struct ompi_communicator_t *comm,
                             mca_coll_base_module_t *module)
 {
+    mca_coll_hcoll_module_t  *hcoll_module = (mca_coll_hcoll_module_t*)module;
     dte_data_representation_t stype;
     dte_data_representation_t rtype;
     int rc;
     HCOL_VERBOSE(20,"RUNNING HCOL GATHERV");
-    mca_coll_hcoll_module_t *hcoll_module = (mca_coll_hcoll_module_t*)module;
+
+    if (root != comm->c_my_rank) {
+        rdtype = sdtype;
+    }
+
     stype = ompi_dtype_2_hcoll_dtype(sdtype, NO_DERIVED);
     rtype = ompi_dtype_2_hcoll_dtype(rdtype, NO_DERIVED);
-    if (OPAL_UNLIKELY(HCOL_DTE_IS_ZERO(stype) || HCOL_DTE_IS_ZERO(rtype))) {
+
+    if (OPAL_UNLIKELY(HCOL_DTE_IS_ZERO(stype) || HCOL_DTE_IS_ZERO(rtype)
+			                      || ompi_count_array_is_64bit(rcounts))) {
         /*If we are here then datatype is not simple predefined datatype */
         /*In future we need to add more complex mapping to the dte_data_representation_t */
         /* Now use fallback */
@@ -387,7 +415,15 @@ int mca_coll_hcoll_gatherv(const void* sbuf, int scount,
                                            comm, hcoll_module->previous_gatherv_module);
         return rc;
     }
-    rc = hcoll_collectives.coll_gatherv((void *)sbuf, scount, stype, rbuf, (int *)rcounts, (int *)displs, rtype, root, hcoll_module->hcoll_context);
+    rc = hcoll_collectives.coll_gatherv((void *)sbuf,
+                                        scount,
+                                        stype,
+                                        rbuf,
+                                        (void *)ompi_count_array_ptr(rcounts),
+                                        (void *)ompi_disp_array_ptr(displs),
+                                        rtype,
+                                        root,
+                                        hcoll_module->hcoll_context);
     if (HCOLL_SUCCESS != rc){
         HCOL_VERBOSE(20,"RUNNING FALLBACK GATHERV");
         rc = hcoll_module->previous_gatherv(sbuf,scount,sdtype,
@@ -398,26 +434,34 @@ int mca_coll_hcoll_gatherv(const void* sbuf, int scount,
 
 }
 
-int mca_coll_hcoll_scatterv(const void* sbuf, const int *scounts, const int *displs,
+int mca_coll_hcoll_scatterv(const void* sbuf, ompi_count_array_t scounts, ompi_disp_array_t displs,
                             struct ompi_datatype_t *sdtype,
-                            void* rbuf, int rcount,
+                            void* rbuf, size_t rcount,
                             struct ompi_datatype_t *rdtype,
                             int root,
                             struct ompi_communicator_t *comm,
                             mca_coll_base_module_t *module)
 {
+    mca_coll_hcoll_module_t  *hcoll_module = (mca_coll_hcoll_module_t*)module;
     dte_data_representation_t stype;
     dte_data_representation_t rtype;
     int rc;
+
     HCOL_VERBOSE(20,"RUNNING HCOL SCATTERV");
-    mca_coll_hcoll_module_t *hcoll_module = (mca_coll_hcoll_module_t*)module;
+
+    if (root != comm->c_my_rank) {
+        sdtype = rdtype;
+    }
+
     stype = ompi_dtype_2_hcoll_dtype(sdtype, NO_DERIVED);
     rtype = ompi_dtype_2_hcoll_dtype(rdtype, NO_DERIVED);
+
     if (rbuf == MPI_IN_PLACE) {
         assert(root == comm->c_my_rank);
         rtype = stype;
     }
-    if (OPAL_UNLIKELY(HCOL_DTE_IS_ZERO(stype) || HCOL_DTE_IS_ZERO(rtype))) {
+    if (OPAL_UNLIKELY(HCOL_DTE_IS_ZERO(stype) || HCOL_DTE_IS_ZERO(rtype)
+			                      || ompi_count_array_is_64bit(scounts))) {
         /*If we are here then datatype is not simple predefined datatype */
         /*In future we need to add more complex mapping to the dte_data_representation_t */
         /* Now use fallback */
@@ -429,12 +473,25 @@ int mca_coll_hcoll_scatterv(const void* sbuf, const int *scounts, const int *dis
                                            comm, hcoll_module->previous_scatterv_module);
         return rc;
     }
-    rc = hcoll_collectives.coll_scatterv((void *)sbuf, (int *)scounts, (int *)displs, stype, rbuf, rcount, rtype, root, hcoll_module->hcoll_context);
+    rc = hcoll_collectives.coll_scatterv((void *)sbuf,
+                                         (void *)ompi_count_array_ptr(scounts),
+                                         (void *)ompi_disp_array_ptr(displs),
+                                         stype,
+                                         rbuf,
+                                         rcount,
+                                         rtype,
+                                         root,
+                                         hcoll_module->hcoll_context);
     if (HCOLL_SUCCESS != rc){
         HCOL_VERBOSE(20,"RUNNING FALLBACK SCATTERV");
-        rc = hcoll_module->previous_scatterv(sbuf, scounts, displs, sdtype,
-                                           rbuf, rcount, rdtype, root,
-                                           comm, hcoll_module->previous_scatterv_module);
+        rc = hcoll_module->previous_scatterv(sbuf, scounts, displs,
+                                             sdtype,
+                                             rbuf,
+                                             rcount,
+                                             rdtype,
+                                             root,
+                                             comm,
+                                             hcoll_module->previous_scatterv_module);
     }
     return rc;
 }
@@ -456,7 +513,7 @@ int mca_coll_hcoll_ibarrier(struct ompi_communicator_t *comm,
     return rc;
 }
 
-int mca_coll_hcoll_ibcast(void *buff, int count,
+int mca_coll_hcoll_ibcast(void *buff, size_t count,
                         struct ompi_datatype_t *datatype, int root,
                         struct ompi_communicator_t *comm,
                         ompi_request_t ** request,
@@ -487,9 +544,9 @@ int mca_coll_hcoll_ibcast(void *buff, int count,
     return rc;
 }
 
-int mca_coll_hcoll_iallgather(const void *sbuf, int scount,
+int mca_coll_hcoll_iallgather(const void *sbuf, size_t scount,
                             struct ompi_datatype_t *sdtype,
-                            void *rbuf, int rcount,
+                            void *rbuf, size_t rcount,
                             struct ompi_datatype_t *rdtype,
                             struct ompi_communicator_t *comm,
                             ompi_request_t ** request,
@@ -530,10 +587,10 @@ int mca_coll_hcoll_iallgather(const void *sbuf, int scount,
     return rc;
 }
 #if HCOLL_API >= HCOLL_VERSION(3,5)
-int mca_coll_hcoll_iallgatherv(const void *sbuf, int scount,
+int mca_coll_hcoll_iallgatherv(const void *sbuf, size_t scount,
                             struct ompi_datatype_t *sdtype,
-                            void *rbuf, const int *rcount,
-                            const int *displs,
+                            void *rbuf, ompi_count_array_t rcount,
+                            ompi_disp_array_t displs,
                             struct ompi_datatype_t *rdtype,
                             struct ompi_communicator_t *comm,
                             ompi_request_t ** request,
@@ -547,7 +604,8 @@ int mca_coll_hcoll_iallgatherv(const void *sbuf, int scount,
     stype = ompi_dtype_2_hcoll_dtype(sdtype, NO_DERIVED);
     rtype = ompi_dtype_2_hcoll_dtype(rdtype, NO_DERIVED);
     void **rt_handle = (void **) request;
-    if (OPAL_UNLIKELY(HCOL_DTE_IS_ZERO(stype) || HCOL_DTE_IS_ZERO(rtype))) {
+    if (OPAL_UNLIKELY(HCOL_DTE_IS_ZERO(stype) || HCOL_DTE_IS_ZERO(rtype)
+			                      || ompi_count_array_is_64bit(rcount))) {
         /*If we are here then datatype is not simple predefined datatype */
         /*In future we need to add more complex mapping to the dte_data_representation_t */
         /* Now use fallback */
@@ -563,7 +621,13 @@ int mca_coll_hcoll_iallgatherv(const void *sbuf, int scount,
                                              hcoll_module->previous_iallgatherv_module);
         return rc;
     }
-    rc = hcoll_collectives.coll_iallgatherv((void *)sbuf,scount,stype,rbuf,(int*)rcount,(int*)displs,rtype,
+    rc = hcoll_collectives.coll_iallgatherv((void *)sbuf,
+                                            scount,
+                                            stype,
+                                            rbuf,
+                                            (void *)ompi_count_array_ptr(rcount),
+                                            (void *)ompi_disp_array_ptr(displs),
+                                            rtype,
             hcoll_module->hcoll_context, rt_handle);
     if (HCOLL_SUCCESS != rc){
        HCOL_VERBOSE(20,"RUNNING FALLBACK NON-BLOCKING ALLGATHER");
@@ -578,7 +642,7 @@ int mca_coll_hcoll_iallgatherv(const void *sbuf, int scount,
     return rc;
 }
 #endif
-int mca_coll_hcoll_iallreduce(const void *sbuf, void *rbuf, int count,
+int mca_coll_hcoll_iallreduce(const void *sbuf, void *rbuf, size_t count,
                             struct ompi_datatype_t *dtype,
                             struct ompi_op_t *op,
                             struct ompi_communicator_t *comm,
@@ -628,7 +692,7 @@ int mca_coll_hcoll_iallreduce(const void *sbuf, void *rbuf, int count,
     return rc;
 }
 #if HCOLL_API >= HCOLL_VERSION(3,5)
-int mca_coll_hcoll_ireduce(const void *sbuf, void *rbuf, int count,
+int mca_coll_hcoll_ireduce(const void *sbuf, void *rbuf, size_t count,
                             struct ompi_datatype_t *dtype,
                             struct ompi_op_t *op,
                             int root,
@@ -684,25 +748,33 @@ int mca_coll_hcoll_ireduce(const void *sbuf, void *rbuf, int count,
     return rc;
 }
 #endif
-int mca_coll_hcoll_igatherv(const void* sbuf, int scount,
+int mca_coll_hcoll_igatherv(const void* sbuf, size_t scount,
                             struct ompi_datatype_t *sdtype,
-                            void* rbuf, const int *rcounts, const int *displs,
+                            void* rbuf, ompi_count_array_t rcounts, ompi_disp_array_t displs,
                             struct ompi_datatype_t *rdtype,
                             int root,
                             struct ompi_communicator_t *comm,
                             ompi_request_t ** request,
                             mca_coll_base_module_t *module)
 {
+    mca_coll_hcoll_module_t  *hcoll_module = (mca_coll_hcoll_module_t*)module;
     dte_data_representation_t stype;
     dte_data_representation_t rtype;
     int rc;
     void** rt_handle;
+
     HCOL_VERBOSE(20,"RUNNING HCOL IGATHERV");
-    mca_coll_hcoll_module_t *hcoll_module = (mca_coll_hcoll_module_t*)module;
+
     rt_handle = (void**) request;
+
+    if (root != comm->c_my_rank) {
+        rdtype = sdtype;
+    }
+
     stype = ompi_dtype_2_hcoll_dtype(sdtype, NO_DERIVED);
     rtype = ompi_dtype_2_hcoll_dtype(rdtype, NO_DERIVED);
-    if (OPAL_UNLIKELY(HCOL_DTE_IS_ZERO(stype) || HCOL_DTE_IS_ZERO(rtype))) {
+    if (OPAL_UNLIKELY(HCOL_DTE_IS_ZERO(stype) || HCOL_DTE_IS_ZERO(rtype)
+			                      || ompi_count_array_is_64bit(rcounts))) {
         /*If we are here then datatype is not simple predefined datatype */
         /*In future we need to add more complex mapping to the dte_data_representation_t */
         /* Now use fallback */
@@ -715,7 +787,16 @@ int mca_coll_hcoll_igatherv(const void* sbuf, int scount,
                                            hcoll_module->previous_igatherv_module);
         return rc;
     }
-    rc = hcoll_collectives.coll_igatherv((void *)sbuf, scount, stype, rbuf, (int *)rcounts, (int *)displs, rtype, root, hcoll_module->hcoll_context, rt_handle);
+    rc = hcoll_collectives.coll_igatherv((void *)sbuf,
+                                         scount,
+                                         stype,
+                                         rbuf,
+                                         (void *)ompi_count_array_ptr(rcounts),
+                                         (void *)ompi_disp_array_ptr(displs),
+                                         rtype,
+                                         root,
+                                         hcoll_module->hcoll_context,
+                                         rt_handle);
     if (HCOLL_SUCCESS != rc){
         HCOL_VERBOSE(20,"RUNNING FALLBACK IGATHERV");
         rc = hcoll_module->previous_igatherv(sbuf,scount,sdtype,
@@ -729,9 +810,9 @@ int mca_coll_hcoll_igatherv(const void* sbuf, int scount,
 
 
 #if HCOLL_API >= HCOLL_VERSION(3,7)
-int mca_coll_hcoll_ialltoallv(const void *sbuf, const int *scounts, const int *sdisps,
+int mca_coll_hcoll_ialltoallv(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
                               struct ompi_datatype_t *sdtype,
-                              void *rbuf, const int *rcounts, const int *rdisps,
+                              void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
                               struct ompi_datatype_t *rdtype,
                               struct ompi_communicator_t *comm,
                               ompi_request_t ** request,
@@ -744,7 +825,8 @@ int mca_coll_hcoll_ialltoallv(const void *sbuf, const int *scounts, const int *s
     mca_coll_hcoll_module_t *hcoll_module = (mca_coll_hcoll_module_t*)module;
     stype = ompi_dtype_2_hcoll_dtype(sdtype, NO_DERIVED);
     rtype = ompi_dtype_2_hcoll_dtype(rdtype, NO_DERIVED);
-    if (OPAL_UNLIKELY(HCOL_DTE_IS_ZERO(stype) || HCOL_DTE_IS_ZERO(rtype))) {
+    if (OPAL_UNLIKELY(HCOL_DTE_IS_ZERO(stype) || HCOL_DTE_IS_ZERO(rtype)
+			                      || ompi_count_array_is_64bit(scounts))) {
         HCOL_VERBOSE(20,"Ompi_datatype is not supported: sdtype = %s, rdtype = %s; calling fallback ialltoallv;",
                      sdtype->super.name,
                      rdtype->super.name);
@@ -753,9 +835,16 @@ int mca_coll_hcoll_ialltoallv(const void *sbuf, const int *scounts, const int *s
                                                comm, request, hcoll_module->previous_alltoallv_module);
         return rc;
     }
-    rc = hcoll_collectives.coll_ialltoallv((void *)sbuf, (int *)scounts, (int *)sdisps, stype,
-                                           rbuf, (int *)rcounts, (int *)rdisps, rtype,
-                                           hcoll_module->hcoll_context, (void**)request);
+    rc = hcoll_collectives.coll_ialltoallv((void *)sbuf,
+                                           (void *)ompi_count_array_ptr(scounts),
+                                           (void *)ompi_disp_array_ptr(sdisps),
+                                           stype,
+                                           rbuf,
+                                           (void *)ompi_count_array_ptr(rcounts),
+                                           (void *)ompi_disp_array_ptr(rdisps),
+                                           rtype,
+                                           hcoll_module->hcoll_context,
+                                           (void**)request);
     if (HCOLL_SUCCESS != rc){
         HCOL_VERBOSE(20,"RUNNING FALLBACK IALLTOALLV");
         rc = hcoll_module->previous_ialltoallv(sbuf, scounts, sdisps, sdtype,
@@ -767,7 +856,7 @@ int mca_coll_hcoll_ialltoallv(const void *sbuf, const int *scounts, const int *s
 #endif
 
 #if HCOLL_API > HCOLL_VERSION(4,5)
-int mca_coll_hcoll_reduce_scatter_block(const void *sbuf, void *rbuf, int rcount,
+int mca_coll_hcoll_reduce_scatter_block(const void *sbuf, void *rbuf, size_t rcount,
                                         struct ompi_datatype_t *dtype,
                                         struct ompi_op_t *op,
                                         struct ompi_communicator_t *comm,
@@ -808,7 +897,7 @@ int mca_coll_hcoll_reduce_scatter_block(const void *sbuf, void *rbuf, int rcount
     return rc;
 }
 
-int mca_coll_hcoll_reduce_scatter(const void *sbuf, void *rbuf, const int* rcounts,
+int mca_coll_hcoll_reduce_scatter(const void *sbuf, void *rbuf, ompi_count_array_t rcounts,
                                   struct ompi_datatype_t *dtype,
                                   struct ompi_op_t *op,
                                   struct ompi_communicator_t *comm,
@@ -819,7 +908,7 @@ int mca_coll_hcoll_reduce_scatter(const void *sbuf, void *rbuf, const int* rcoun
     HCOL_VERBOSE(20,"RUNNING HCOL REDUCE SCATTER");
     mca_coll_hcoll_module_t *hcoll_module = (mca_coll_hcoll_module_t*)module;
     Dtype = ompi_dtype_2_hcoll_dtype(dtype, NO_DERIVED);
-    if (OPAL_UNLIKELY(HCOL_DTE_IS_ZERO(Dtype))){
+    if (OPAL_UNLIKELY(HCOL_DTE_IS_ZERO(Dtype) || ompi_count_array_is_64bit(rcounts))){
         /*If we are here then datatype is not simple predefined datatype */
         /*In future we need to add more complex mapping to the dte_data_representation_t */
         /* Now use fallback */
@@ -838,8 +927,12 @@ int mca_coll_hcoll_reduce_scatter(const void *sbuf, void *rbuf, const int* rcoun
         goto fallback;
     }
 
-    rc = hcoll_collectives.coll_reduce_scatter((void*)sbuf, rbuf, (int*)rcounts,
-                                               Dtype, Op, hcoll_module->hcoll_context);
+    rc = hcoll_collectives.coll_reduce_scatter((void *)sbuf,
+                                               rbuf,
+                                               (void *)ompi_count_array_ptr(rcounts),
+                                               Dtype,
+                                               Op,
+                                               hcoll_module->hcoll_context);
     if (HCOLL_SUCCESS != rc){
     fallback:        
         HCOL_VERBOSE(20,"RUNNING FALLBACK ALLREDUCE");
diff --git a/ompi/mca/coll/hcoll/coll_hcoll_rte.c b/ompi/mca/coll/hcoll/coll_hcoll_rte.c
index e2a1ea9dabf..882880f645e 100644
--- a/ompi/mca/coll/hcoll/coll_hcoll_rte.c
+++ b/ompi/mca/coll/hcoll/coll_hcoll_rte.c
@@ -51,7 +51,7 @@
 
 
 static int recv_nb(dte_data_representation_t data ,
-                   uint32_t count ,
+                   size_t count ,
                    void *buffer,
                    rte_ec_handle_t ,
                    rte_grp_handle_t ,
@@ -59,7 +59,7 @@ static int recv_nb(dte_data_representation_t data ,
                    rte_request_handle_t * req);
 
 static int send_nb(dte_data_representation_t data,
-                   uint32_t count,
+                   size_t count,
                    void *buffer,
                    rte_ec_handle_t ec_h,
                    rte_grp_handle_t grp_h,
@@ -161,7 +161,7 @@ void hcoll_rte_fns_setup(void)
 }
 
 static int recv_nb(struct dte_data_representation_t data,
-                   uint32_t count ,
+                   size_t count,
                    void *buffer,
                    rte_ec_handle_t ec_h,
                    rte_grp_handle_t grp_h,
@@ -187,8 +187,8 @@ static int recv_nb(struct dte_data_representation_t data,
     }
     size = (size_t)data.rep.in_line_rep.data_handle.in_line.packed_size*count/8;
 
-    HCOL_VERBOSE(30,"PML_IRECV: dest = %d: buf = %p: size = %u: comm = %p",
-                 ec_h.rank, buffer, (unsigned int)size, (void *)comm);
+    HCOL_VERBOSE(30,"PML_IRECV: dest = %d: buf = %p: size = %zu: comm = %p",
+                 ec_h.rank, buffer, size, (void *)comm);
     if (MCA_PML_CALL(irecv(buffer,size,&(ompi_mpi_unsigned_char.dt),ec_h.rank,
                            tag,comm,&ompi_req)))
     {
@@ -202,7 +202,7 @@ static int recv_nb(struct dte_data_representation_t data,
 
 
 static int send_nb( dte_data_representation_t data,
-                    uint32_t count,
+                    size_t count,
                     void *buffer,
                     rte_ec_handle_t ec_h,
                     rte_grp_handle_t grp_h,
@@ -226,8 +226,8 @@ static int send_nb( dte_data_representation_t data,
         return HCOLL_ERROR;
     }
     size = (size_t)data.rep.in_line_rep.data_handle.in_line.packed_size*count/8;
-    HCOL_VERBOSE(30,"PML_ISEND: dest = %d: buf = %p: size = %u: comm = %p",
-                 ec_h.rank, buffer, (unsigned int)size, (void *)comm);
+    HCOL_VERBOSE(30,"PML_ISEND: dest = %d: buf = %p: size = %zu: comm = %p",
+                 ec_h.rank, buffer, size, (void *)comm);
     if (MCA_PML_CALL(isend(buffer,size,&(ompi_mpi_unsigned_char.dt),ec_h.rank,
                            tag,MCA_PML_BASE_SEND_STANDARD,comm,&ompi_req)))
     {
@@ -320,7 +320,7 @@ static uint32_t jobid(void){
 }
 
 static int group_id(rte_grp_handle_t group){
-    return ((ompi_communicator_t *)group)->c_contextid;
+    return ((ompi_communicator_t *)group)->c_index;
 }
 
 static int
@@ -331,7 +331,7 @@ request_free(struct ompi_request_t **ompi_req)
         return OMPI_ERROR;
     }
     coll_handle_free(req);
-    *ompi_req = &ompi_request_empty;
+    *ompi_req = MPI_REQUEST_NULL;
     return OMPI_SUCCESS;
 }
 
@@ -352,8 +352,8 @@ static void* get_coll_handle(void)
     ompi_req->super.req_state            = OMPI_REQUEST_ACTIVE;
     ompi_req->super.req_free             = request_free;
     ompi_req->super.req_type             = OMPI_REQUEST_COLL;
-    ompi_req->data.objs.objs[0]          = NULL;
-    ompi_req->data.objs.objs[1]          = NULL;
+    ompi_req->data.refcounted.objs.objs[0]          = NULL;
+    ompi_req->data.refcounted.objs.objs[1]          = NULL;
     return (void *)ompi_req;
 }
 
diff --git a/ompi/mca/coll/hcoll/configure.m4 b/ompi/mca/coll/hcoll/configure.m4
index f3f57532e22..3d2c2b3a581 100644
--- a/ompi/mca/coll/hcoll/configure.m4
+++ b/ompi/mca/coll/hcoll/configure.m4
@@ -4,6 +4,7 @@
 # Copyright (c) 2011      Mellanox Technologies. All rights reserved.
 # Copyright (c) 2015      Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -30,7 +31,6 @@ AC_DEFUN([MCA_ompi_coll_hcoll_CONFIG],[
           [$2])
 
     # substitute in the things needed to build hcoll
-    AC_SUBST([coll_hcoll_CFLAGS])
     AC_SUBST([coll_hcoll_CPPFLAGS])
     AC_SUBST([coll_hcoll_LDFLAGS])
     AC_SUBST([coll_hcoll_LIBS])
diff --git a/ompi/mca/coll/inter/Makefile.am b/ompi/mca/coll/inter/Makefile.am
index d9c691cf458..ea9188cffd4 100644
--- a/ompi/mca/coll/inter/Makefile.am
+++ b/ompi/mca/coll/inter/Makefile.am
@@ -11,6 +11,7 @@
 #                         All rights reserved.
 # Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -43,7 +44,7 @@ libmca_coll_inter_la_LDFLAGS = -module -avoid-version
 
 sources = \
         coll_inter.h \
-        coll_inter.c \
+        coll_inter_module.c \
         coll_inter_allreduce.c \
 	coll_inter_allgather.c \
 	coll_inter_allgatherv.c \
diff --git a/ompi/mca/coll/inter/coll_inter.c b/ompi/mca/coll/inter/coll_inter.c
deleted file mode 100644
index d75994e396a..00000000000
--- a/ompi/mca/coll/inter/coll_inter.c
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2006 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2007 University of Houston. All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2013 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2017      IBM Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include "coll_inter.h"
-
-#include <stdio.h>
-
-#include "mpi.h"
-#include "ompi/communicator/communicator.h"
-
-#include "ompi/mca/coll/coll.h"
-#include "ompi/mca/coll/base/base.h"
-#include "ompi/mca/coll/base/coll_base_functions.h"
-#include "ompi/mca/coll/base/coll_tags.h"
-
-#include "ompi/mca/bml/base/base.h"
-
-
-#if 0
-static void mca_coll_inter_dump_struct ( struct mca_coll_base_comm_t *c);
-
-static const mca_coll_base_module_1_0_0_t inter = {
-
-  /* Initialization / finalization functions */
-
-  mca_coll_inter_module_init,
-  mca_coll_inter_module_finalize,
-
-  /* Collective function pointers */
-  /* function pointers marked with NULL are not yet implemented
-     and will use the functions provided in the basic module */
-  mca_coll_inter_allgather_inter,
-  mca_coll_inter_allgatherv_inter,
-  mca_coll_inter_allreduce_inter,
-  NULL, /* alltoall */
-  NULL, /* alltoallv */
-  NULL, /* alltoallw */
-  NULL, /* barrier */
-  mca_coll_inter_bcast_inter,
-  NULL,  /* exscan */
-  mca_coll_inter_gather_inter,
-  mca_coll_inter_gatherv_inter,
-  mca_coll_inter_reduce_inter,
-  NULL,  /* reduce_scatter */
-  NULL,  /* scan */
-  mca_coll_inter_scatter_inter,
-  mca_coll_inter_scatterv_inter
-};
-#endif
-
-
-/*
- * Initial query function that is invoked during MPI_INIT, allowing
- * this module to indicate what level of thread support it provides.
- */
-int mca_coll_inter_init_query(bool allow_inter_user_threads,
-                             bool have_hidden_user_threads)
-{
-    /* Don't ask. All done */
-    return OMPI_SUCCESS;
-}
-
-
-/*
- * Invoked when there's a new communicator that has been created.
- * Look at the communicator and decide which set of functions and
- * priority we want to return.
- */
-mca_coll_base_module_t *
-mca_coll_inter_comm_query(struct ompi_communicator_t *comm, int *priority)
-{
-    int size, rsize;
-    mca_coll_inter_module_t *inter_module;
-
-    /* This module only works for inter-communicators */
-    if (!OMPI_COMM_IS_INTER(comm)) {
-        return NULL;
-    }
-
-    /* Get the priority level attached to this module. If priority is less
-     * than or equal to 0, then the module is unavailable. */
-    *priority = mca_coll_inter_priority_param;
-    if (0 >= mca_coll_inter_priority_param) {
-	return NULL;
-    }
-
-    size = ompi_comm_size(comm);
-    rsize = ompi_comm_remote_size(comm);
-
-    if ( size < mca_coll_inter_crossover && rsize < mca_coll_inter_crossover) {
-	return NULL;
-    }
-
-    inter_module = OBJ_NEW(mca_coll_inter_module_t);
-    if (NULL == inter_module) {
-	return NULL;
-    }
-
-    inter_module->super.coll_module_enable = mca_coll_inter_module_enable;
-
-    inter_module->super.coll_allgather  = mca_coll_inter_allgather_inter;
-    inter_module->super.coll_allgatherv = mca_coll_inter_allgatherv_inter;
-    inter_module->super.coll_allreduce  = mca_coll_inter_allreduce_inter;
-    inter_module->super.coll_alltoall   = NULL;
-    inter_module->super.coll_alltoallv  = NULL;
-    inter_module->super.coll_alltoallw  = NULL;
-    inter_module->super.coll_barrier    = NULL;
-    inter_module->super.coll_bcast      = mca_coll_inter_bcast_inter;
-    inter_module->super.coll_exscan     = NULL;
-    inter_module->super.coll_gather     = mca_coll_inter_gather_inter;
-    inter_module->super.coll_gatherv    = mca_coll_inter_gatherv_inter;
-    inter_module->super.coll_reduce     = mca_coll_inter_reduce_inter;
-    inter_module->super.coll_reduce_scatter = NULL;
-    inter_module->super.coll_scan       = NULL;
-    inter_module->super.coll_scatter    = mca_coll_inter_scatter_inter;
-    inter_module->super.coll_scatterv   = mca_coll_inter_scatterv_inter;
-    inter_module->super.coll_reduce_local = mca_coll_base_reduce_local;
-
-    return &(inter_module->super);
-}
-
-
-/*
- * Init module on the communicator
- */
-int
-mca_coll_inter_module_enable(mca_coll_base_module_t *module,
-                             struct ompi_communicator_t *comm)
-{
-    mca_coll_inter_module_t *inter_module = (mca_coll_inter_module_t*) module;
-
-    inter_module->inter_comm = comm;
-
-#if 0
-    if ( mca_coll_inter_verbose_param ) {
-      mca_coll_inter_dump_struct (data);
-    }
-#endif
-
-    return OMPI_SUCCESS;
-}
-
-
-#if 0
-static void mca_coll_inter_dump_struct ( struct mca_coll_base_comm_t *c)
-{
-    int rank;
-
-    rank = ompi_comm_rank ( c->inter_comm );
-
-    printf("%d: Dump of inter-struct for  comm %s cid %u\n",
-           rank, c->inter_comm->c_name, c->inter_comm->c_contextid);
-
-
-    return;
-}
-#endif
diff --git a/ompi/mca/coll/inter/coll_inter.h b/ompi/mca/coll/inter/coll_inter.h
index ac0710c7d04..eb6bd7563f1 100644
--- a/ompi/mca/coll/inter/coll_inter.h
+++ b/ompi/mca/coll/inter/coll_inter.h
@@ -13,6 +13,7 @@
  * Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -37,7 +38,7 @@ BEGIN_C_DECLS
  * Globally exported variable
  */
 
-OMPI_MODULE_DECLSPEC extern const mca_coll_base_component_2_4_0_t mca_coll_inter_component;
+OMPI_DECLSPEC extern const mca_coll_base_component_3_0_0_t mca_coll_inter_component;
 extern int mca_coll_inter_priority_param;
 extern int mca_coll_inter_verbose_param;
 
@@ -50,59 +51,56 @@ int mca_coll_inter_init_query(bool allow_inter_user_threads,
 mca_coll_base_module_t *
 mca_coll_inter_comm_query(struct ompi_communicator_t *comm, int *priority);
 
-int mca_coll_inter_module_enable(mca_coll_base_module_t *module,
-                                 struct ompi_communicator_t *comm);
-
-int mca_coll_inter_allgather_inter(const void *sbuf, int scount,
+int mca_coll_inter_allgather_inter(const void *sbuf, size_t scount,
 				   struct ompi_datatype_t *sdtype,
-				   void *rbuf, int rcount,
+				   void *rbuf, size_t rcount,
 				   struct ompi_datatype_t *rdtype,
 				   struct ompi_communicator_t *comm,
                                    mca_coll_base_module_t *module);
-int mca_coll_inter_allgatherv_inter(const void *sbuf, int scount,
+int mca_coll_inter_allgatherv_inter(const void *sbuf, size_t scount,
 				    struct ompi_datatype_t *sdtype,
-				    void *rbuf, const int *rcounts, const int *disps,
+				    void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
 				    struct ompi_datatype_t *rdtype,
 				    struct ompi_communicator_t *comm,
                                     mca_coll_base_module_t *module);
-int mca_coll_inter_allreduce_inter(const void *sbuf, void *rbuf, int count,
+int mca_coll_inter_allreduce_inter(const void *sbuf, void *rbuf, size_t count,
 				   struct ompi_datatype_t *dtype,
 				   struct ompi_op_t *op,
 				   struct ompi_communicator_t *comm,
                                    mca_coll_base_module_t *module);
-int mca_coll_inter_bcast_inter(void *buff, int count,
+int mca_coll_inter_bcast_inter(void *buff, size_t count,
 			       struct ompi_datatype_t *datatype,
 			       int root,
 			       struct ompi_communicator_t *comm,
                                mca_coll_base_module_t *module);
-int mca_coll_inter_gather_inter(const void *sbuf, int scount,
+int mca_coll_inter_gather_inter(const void *sbuf, size_t scount,
 				struct ompi_datatype_t *sdtype,
-				void *rbuf, int rcount,
+				void *rbuf, size_t rcount,
 				struct ompi_datatype_t *rdtype,
 				int root,
 				struct ompi_communicator_t *comm,
                                 mca_coll_base_module_t *module);
-int mca_coll_inter_gatherv_inter(const void *sbuf, int scount,
+int mca_coll_inter_gatherv_inter(const void *sbuf, size_t scount,
 				 struct ompi_datatype_t *sdtype,
-				 void *rbuf, const int *rcounts, const int *disps,
+				 void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
 				 struct ompi_datatype_t *rdtype,
 				 int root,
 				 struct ompi_communicator_t *comm,
                                  mca_coll_base_module_t *module);
-int mca_coll_inter_reduce_inter(const void *sbuf, void* rbuf, int count,
+int mca_coll_inter_reduce_inter(const void *sbuf, void* rbuf, size_t count,
 				struct ompi_datatype_t *dtype,
 				struct ompi_op_t *op,
 				int root,
 				struct ompi_communicator_t *comm,
                                 mca_coll_base_module_t *module);
-int mca_coll_inter_scatter_inter(const void *sbuf, int scount,
+int mca_coll_inter_scatter_inter(const void *sbuf, size_t scount,
 				 struct ompi_datatype_t *sdtype, void *rbuf,
-				 int rcount, struct ompi_datatype_t *rdtype,
+				 size_t rcount, struct ompi_datatype_t *rdtype,
 				 int root, struct ompi_communicator_t *comm,
                                  mca_coll_base_module_t *module);
-int mca_coll_inter_scatterv_inter(const void *sbuf, const int *scounts, const int *disps,
+int mca_coll_inter_scatterv_inter(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t disps,
 				  struct ompi_datatype_t *sdtype,
-				  void* rbuf, int rcount,
+				  void* rbuf, size_t rcount,
 				  struct ompi_datatype_t *rdtype, int root,
 				  struct ompi_communicator_t *comm,
                                   mca_coll_base_module_t *module);
diff --git a/ompi/mca/coll/inter/coll_inter_allgather.c b/ompi/mca/coll/inter/coll_inter_allgather.c
index 6bd0e91b58d..87b902c94d1 100644
--- a/ompi/mca/coll/inter/coll_inter_allgather.c
+++ b/ompi/mca/coll/inter/coll_inter_allgather.c
@@ -12,6 +12,8 @@
  * Copyright (c) 2006-2010 University of Houston. All rights reserved.
  * Copyright (c) 2015-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -41,16 +43,17 @@
  *	Returns:	- MPI_SUCCESS or error code
  */
 int
-mca_coll_inter_allgather_inter(const void *sbuf, int scount,
+mca_coll_inter_allgather_inter(const void *sbuf, size_t scount,
                                struct ompi_datatype_t *sdtype,
-                               void *rbuf, int rcount,
+                               void *rbuf, size_t rcount,
                                struct ompi_datatype_t *rdtype,
                                struct ompi_communicator_t *comm,
                                mca_coll_base_module_t *module)
 {
-    int rank, root = 0, size, rsize, err = OMPI_SUCCESS;
+    int rank, root = 0, size, rsize, err = OMPI_SUCCESS, i;
     char *ptmp_free = NULL, *ptmp = NULL;
     ptrdiff_t gap, span;
+    void *rbuf_ptr;
 
     rank = ompi_comm_rank(comm);
     size = ompi_comm_size(comm->c_local_comm);
@@ -59,26 +62,26 @@ mca_coll_inter_allgather_inter(const void *sbuf, int scount,
     /* Perform the gather locally at the root */
     if ( scount > 0 ) {
         span = opal_datatype_span(&sdtype->super, (int64_t)scount*(int64_t)size, &gap);
-	ptmp_free = (char*)malloc(span);
-	if (NULL == ptmp_free) {
-	    return OMPI_ERR_OUT_OF_RESOURCE;
-	}
+	    ptmp_free = (char*)malloc(span);
+	    if (NULL == ptmp_free) {
+	        return OMPI_ERR_OUT_OF_RESOURCE;
+	    }
         ptmp = ptmp_free - gap;
 
-	err = comm->c_local_comm->c_coll->coll_gather(sbuf, scount, sdtype,
+	    err = comm->c_local_comm->c_coll->coll_gather(sbuf, scount, sdtype,
 						     ptmp, scount, sdtype,
 						     0, comm->c_local_comm,
 						     comm->c_local_comm->c_coll->coll_gather_module);
-	if (OMPI_SUCCESS != err) {
-	    goto exit;
-	}
+	    if (OMPI_SUCCESS != err) {
+	        goto exit;
+	    }
     }
 
     if (rank == root) {
-	/* Do a send-recv between the two root procs. to avoid deadlock */
-        err = ompi_coll_base_sendrecv_actual(ptmp, scount*size, sdtype, 0,
+	    /* Do a send-recv between the two root procs. to avoid deadlock */
+        err = ompi_coll_base_sendrecv_actual(ptmp, scount*(size_t)size, sdtype, 0,
                                              MCA_COLL_BASE_TAG_ALLGATHER,
-                                             rbuf, rcount*rsize, rdtype, 0,
+                                             rbuf, rcount*(size_t)rsize, rdtype, 0,
                                              MCA_COLL_BASE_TAG_ALLGATHER,
                                              comm, MPI_STATUS_IGNORE);
         if (OMPI_SUCCESS != err) {
@@ -87,12 +90,28 @@ mca_coll_inter_allgather_inter(const void *sbuf, int scount,
     }
     /* bcast the message to all the local processes */
     if ( rcount > 0 ) {
-	err = comm->c_local_comm->c_coll->coll_bcast(rbuf, rcount*rsize, rdtype,
-						    root, comm->c_local_comm,
-						    comm->c_local_comm->c_coll->coll_bcast_module);
-	if (OMPI_SUCCESS != err) {
-	    goto exit;
-	}
+        if ( OPAL_UNLIKELY(rcount*(size_t)rsize > INT_MAX) ) {
+            // Sending the message in the coll_bcast as "rcount*rsize" would exceed
+            // the 'int count' parameter in the coll_bcast() function. Instead broadcast
+            // the result in "rcount" chunks to the local group.
+            span = opal_datatype_span(&rdtype->super, rcount, &gap);
+            for( i = 0; i < rsize; ++i) {
+                rbuf_ptr = (char*)rbuf + span * (size_t)i;
+                err = comm->c_local_comm->c_coll->coll_bcast(rbuf_ptr, rcount, rdtype,
+                                                             root, comm->c_local_comm,
+                                                             comm->c_local_comm->c_coll->coll_bcast_module);
+                if (OMPI_SUCCESS != err) {
+                    goto exit;
+                }
+            }
+        } else {
+            err = comm->c_local_comm->c_coll->coll_bcast(rbuf, rcount*rsize, rdtype,
+                                                         root, comm->c_local_comm,
+                                                         comm->c_local_comm->c_coll->coll_bcast_module);
+            if (OMPI_SUCCESS != err) {
+                goto exit;
+            }
+        }
     }
 
  exit:
diff --git a/ompi/mca/coll/inter/coll_inter_allgatherv.c b/ompi/mca/coll/inter/coll_inter_allgatherv.c
index 0728fd28072..fa7c9e14301 100644
--- a/ompi/mca/coll/inter/coll_inter_allgatherv.c
+++ b/ompi/mca/coll/inter/coll_inter_allgatherv.c
@@ -12,6 +12,7 @@
  * Copyright (c) 2006-2010 University of Houston. All rights reserved.
  * Copyright (c) 2015-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -40,15 +41,19 @@
  *	Returns:	- MPI_SUCCESS or error code
  */
 int
-mca_coll_inter_allgatherv_inter(const void *sbuf, int scount,
+mca_coll_inter_allgatherv_inter(const void *sbuf, size_t scount,
                                 struct ompi_datatype_t *sdtype,
-                                void *rbuf, const int *rcounts, const int *disps,
+                                void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
                                 struct ompi_datatype_t *rdtype,
                                 struct ompi_communicator_t *comm,
                                mca_coll_base_module_t *module)
 {
-    int i, rank, size, size_local, total=0, err;
-    int *count=NULL,*displace=NULL;
+    int i, rank, size, size_local, err;
+    size_t total = 0;
+    size_t *count=NULL;
+    ptrdiff_t *displace=NULL;
+    ompi_count_array_t count_arg;
+    ompi_disp_array_t displace_arg;
     char *ptmp_free=NULL, *ptmp=NULL;
     ompi_datatype_t *ndtype = NULL;
 
@@ -57,18 +62,18 @@ mca_coll_inter_allgatherv_inter(const void *sbuf, int scount,
     size = ompi_comm_remote_size(comm);
 
     if (0 == rank) {
-	count = (int *)malloc(sizeof(int) * size_local);
-	displace = (int *)malloc(sizeof(int) * size_local);
+	count = (size_t *)malloc(sizeof(size_t) * size_local);
+	displace = (ptrdiff_t *)malloc(sizeof(ptrdiff_t) * size_local);
 	if ((NULL == count) || (NULL == displace)) {
             err = OMPI_ERR_OUT_OF_RESOURCE;
             goto exit;
 	}
     }
     /* Local gather to get the scount of each process */
-    err = comm->c_local_comm->c_coll->coll_gather(&scount, 1, MPI_INT,
-						 count, 1, MPI_INT,
-						 0, comm->c_local_comm,
-                                                 comm->c_local_comm->c_coll->coll_gather_module);
+    err = comm->c_local_comm->c_coll->coll_gather(&scount, sizeof(size_t), MPI_BYTE,
+                                                  count, sizeof(size_t), MPI_BYTE,
+                                                  0, comm->c_local_comm,
+                                                  comm->c_local_comm->c_coll->coll_gather_module);
     if (OMPI_SUCCESS != err) {
         goto exit;
     }
@@ -92,16 +97,31 @@ mca_coll_inter_allgatherv_inter(const void *sbuf, int scount,
             ptmp = ptmp_free - gap;
 	}
     }
+    OMPI_COUNT_ARRAY_INIT(&count_arg, count);
+    OMPI_DISP_ARRAY_INIT(&displace_arg, displace);
     err = comm->c_local_comm->c_coll->coll_gatherv(sbuf, scount, sdtype,
-						  ptmp, count, displace,
-						  sdtype,0, comm->c_local_comm,
-                                                  comm->c_local_comm->c_coll->coll_gatherv_module);
+                                                   ptmp, count_arg, displace_arg,
+                                                   sdtype,0, comm->c_local_comm,
+                                                   comm->c_local_comm->c_coll->coll_gatherv_module);
     if (OMPI_SUCCESS != err) {
         goto exit;
     }
 
-    ompi_datatype_create_indexed(size,rcounts,disps,rdtype,&ndtype);
+    /* TODO:BIGCOUNT: Remove tehese temporaries once ompi_datatype is updated for bigcount */
+    int *tmp_rcounts = malloc(sizeof(int) * size);
+    int *tmp_disps = malloc(sizeof(int) * size);
+    if (NULL == tmp_rcounts || NULL == tmp_disps) {
+        err = OMPI_ERR_OUT_OF_RESOURCE;
+        goto exit;
+    }
+    for (i = 0; i < size; ++i) {
+        tmp_rcounts[i] = (int) ompi_count_array_get(rcounts, i);
+        tmp_disps[i] = (int) ompi_disp_array_get(disps, i);
+    }
+    ompi_datatype_create_indexed(size,tmp_rcounts,tmp_disps,rdtype,&ndtype);
     ompi_datatype_commit(&ndtype);
+    free(tmp_rcounts);
+    free(tmp_disps);
 
     if (0 == rank) {
 	/* Exchange data between roots */
diff --git a/ompi/mca/coll/inter/coll_inter_allreduce.c b/ompi/mca/coll/inter/coll_inter_allreduce.c
index 91ca00ff858..3c999435ddf 100644
--- a/ompi/mca/coll/inter/coll_inter_allreduce.c
+++ b/ompi/mca/coll/inter/coll_inter_allreduce.c
@@ -13,6 +13,7 @@
  * Copyright (c) 2013      Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2015-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -41,7 +42,7 @@
  *	Returns:	- MPI_SUCCESS or error code
  */
 int
-mca_coll_inter_allreduce_inter(const void *sbuf, void *rbuf, int count,
+mca_coll_inter_allreduce_inter(const void *sbuf, void *rbuf, size_t count,
                                struct ompi_datatype_t *dtype,
                                struct ompi_op_t *op,
                                struct ompi_communicator_t *comm,
@@ -49,6 +50,7 @@ mca_coll_inter_allreduce_inter(const void *sbuf, void *rbuf, int count,
 {
     int err, rank, root = 0;
     char *tmpbuf = NULL, *pml_buffer = NULL;
+    const char *source;
     ptrdiff_t gap, span;
 
     rank = ompi_comm_rank(comm);
@@ -58,20 +60,21 @@ mca_coll_inter_allreduce_inter(const void *sbuf, void *rbuf, int count,
 
     tmpbuf = (char *) malloc(span);
     if (NULL == tmpbuf) {
-	return OMPI_ERR_OUT_OF_RESOURCE;
+        return OMPI_ERR_OUT_OF_RESOURCE;
     }
     pml_buffer = tmpbuf - gap;
+    source = (MPI_IN_PLACE == sbuf) ? rbuf : sbuf;
 
-    err = comm->c_local_comm->c_coll->coll_reduce(sbuf, pml_buffer, count,
-						 dtype, op, root,
-						 comm->c_local_comm,
-                                                 comm->c_local_comm->c_coll->coll_reduce_module);
+    err = comm->c_local_comm->c_coll->coll_reduce(source, pml_buffer, count,
+                                                  dtype, op, root,
+                                                  comm->c_local_comm,
+                                                  comm->c_local_comm->c_coll->coll_reduce_module);
     if (OMPI_SUCCESS != err) {
-	goto exit;
+        goto exit;
     }
 
     if (rank == root) {
-	/* Do a send-recv between the two root procs. to avoid deadlock */
+        /* Do a send-recv between the two root procs. to avoid deadlock */
         err = ompi_coll_base_sendrecv_actual(pml_buffer, count, dtype, 0,
                                              MCA_COLL_BASE_TAG_ALLREDUCE,
                                              rbuf, count, dtype, 0,
@@ -84,8 +87,8 @@ mca_coll_inter_allreduce_inter(const void *sbuf, void *rbuf, int count,
 
     /* bcast the message to all the local processes */
     err = comm->c_local_comm->c_coll->coll_bcast(rbuf, count, dtype,
-						root, comm->c_local_comm,
-                                                comm->c_local_comm->c_coll->coll_bcast_module);
+                                                 root, comm->c_local_comm,
+                                                 comm->c_local_comm->c_coll->coll_bcast_module);
     if (OMPI_SUCCESS != err) {
             goto exit;
     }
diff --git a/ompi/mca/coll/inter/coll_inter_bcast.c b/ompi/mca/coll/inter/coll_inter_bcast.c
index a58dfedc8fa..266d2df0745 100644
--- a/ompi/mca/coll/inter/coll_inter_bcast.c
+++ b/ompi/mca/coll/inter/coll_inter_bcast.c
@@ -37,7 +37,7 @@
  *	Returns:	- MPI_SUCCESS or error code
  */
 int
-mca_coll_inter_bcast_inter(void *buff, int count,
+mca_coll_inter_bcast_inter(void *buff, size_t count,
                            struct ompi_datatype_t *datatype, int root,
                            struct ompi_communicator_t *comm,
                            mca_coll_base_module_t *module)
@@ -51,7 +51,7 @@ mca_coll_inter_bcast_inter(void *buff, int count,
         /* do nothing */
         err = OMPI_SUCCESS;
     } else if (MPI_ROOT != root) {
-        /* Non-root, first process recieves the data and bcast to others */
+        /* Non-root, first process receives the data and bcast to others */
 	if ( 0 == rank ) {
 	    err = MCA_PML_CALL(recv(buff, count, datatype, root,
 				    MCA_COLL_BASE_TAG_BCAST, comm,
diff --git a/ompi/mca/coll/inter/coll_inter_component.c b/ompi/mca/coll/inter/coll_inter_component.c
index 39ff40c9cfb..4b3d28d1b74 100644
--- a/ompi/mca/coll/inter/coll_inter_component.c
+++ b/ompi/mca/coll/inter/coll_inter_component.c
@@ -14,6 +14,7 @@
  * Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -56,13 +57,13 @@ static int inter_register(void);
  * and pointers to our public functions in it
  */
 
-const mca_coll_base_component_2_4_0_t mca_coll_inter_component = {
+const mca_coll_base_component_3_0_0_t mca_coll_inter_component = {
 
     /* First, the mca_component_t struct containing meta information
        about the component itself */
 
     .collm_version = {
-        MCA_COLL_BASE_VERSION_2_4_0,
+        MCA_COLL_BASE_VERSION_3_0_0,
 
         /* Component name and version */
         .mca_component_name = "inter",
@@ -113,14 +114,7 @@ mca_coll_inter_module_construct(mca_coll_inter_module_t *module)
     module->inter_comm = NULL;
 }
 
-static void
-mca_coll_inter_module_destruct(mca_coll_inter_module_t *module)
-{
-
-}
-
-
 OBJ_CLASS_INSTANCE(mca_coll_inter_module_t,
                    mca_coll_base_module_t,
                    mca_coll_inter_module_construct,
-                   mca_coll_inter_module_destruct);
+                   NULL);
diff --git a/ompi/mca/coll/inter/coll_inter_gather.c b/ompi/mca/coll/inter/coll_inter_gather.c
index f1a7356224d..c314ad0851f 100644
--- a/ompi/mca/coll/inter/coll_inter_gather.c
+++ b/ompi/mca/coll/inter/coll_inter_gather.c
@@ -12,6 +12,7 @@
  * Copyright (c) 2006-2007 University of Houston. All rights reserved.
  * Copyright (c) 2015-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -38,9 +39,9 @@
  *	Returns:	- MPI_SUCCESS or error code
  */
 int
-mca_coll_inter_gather_inter(const void *sbuf, int scount,
+mca_coll_inter_gather_inter(const void *sbuf, size_t scount,
                             struct ompi_datatype_t *sdtype,
-                            void *rbuf, int rcount,
+                            void *rbuf, size_t rcount,
                             struct ompi_datatype_t *rdtype,
                             int root, struct ompi_communicator_t *comm,
                             mca_coll_base_module_t *module)
@@ -76,7 +77,7 @@ mca_coll_inter_gather_inter(const void *sbuf, int scount,
                                                      comm->c_local_comm->c_coll->coll_gather_module);
 	if (0 == rank) {
 	    /* First process sends data to the root */
-	    err = MCA_PML_CALL(send(ptmp, scount*size_local, sdtype, root,
+	    err = MCA_PML_CALL(send(ptmp, scount*(size_t)size_local, sdtype, root,
 				    MCA_COLL_BASE_TAG_GATHER,
 				    MCA_PML_BASE_SEND_STANDARD, comm));
 	    if (OMPI_SUCCESS != err) {
@@ -86,7 +87,7 @@ mca_coll_inter_gather_inter(const void *sbuf, int scount,
         free(ptmp_free);
     } else {
         /* I am the root, loop receiving the data. */
-	err = MCA_PML_CALL(recv(rbuf, rcount*size, rdtype, 0,
+	err = MCA_PML_CALL(recv(rbuf, rcount*(size_t)size, rdtype, 0,
 				MCA_COLL_BASE_TAG_GATHER,
 				comm, MPI_STATUS_IGNORE));
 	if (OMPI_SUCCESS != err) {
diff --git a/ompi/mca/coll/inter/coll_inter_gatherv.c b/ompi/mca/coll/inter/coll_inter_gatherv.c
index 5dd9f7b4b68..1e1d8840a44 100644
--- a/ompi/mca/coll/inter/coll_inter_gatherv.c
+++ b/ompi/mca/coll/inter/coll_inter_gatherv.c
@@ -12,6 +12,7 @@
  * Copyright (c) 2006-2010 University of Houston. All rights reserved.
  * Copyright (c) 2015-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -37,15 +38,19 @@
  *	Returns:	- MPI_SUCCESS or error code
  */
 int
-mca_coll_inter_gatherv_inter(const void *sbuf, int scount,
+mca_coll_inter_gatherv_inter(const void *sbuf, size_t scount,
                              struct ompi_datatype_t *sdtype,
-                             void *rbuf, const int *rcounts, const int *disps,
+                             void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
                              struct ompi_datatype_t *rdtype, int root,
                              struct ompi_communicator_t *comm,
                              mca_coll_base_module_t *module)
 {
-    int i, rank, size, size_local, total=0, err;
-    int *count=NULL, *displace=NULL;
+    int i, rank, size, size_local, err;
+    size_t total = 0;
+    size_t *count=NULL;
+    ptrdiff_t *displace=NULL;
+    ompi_count_array_t count_arg;
+    ompi_disp_array_t displace_arg;
     char *ptmp_free=NULL, *ptmp=NULL;
     ompi_datatype_t *ndtype;
 
@@ -57,8 +62,20 @@ mca_coll_inter_gatherv_inter(const void *sbuf, int scount,
     size_local = ompi_comm_size(comm);
 
     if (MPI_ROOT == root) { /* I am the root, receiving the data from zero. */
-        ompi_datatype_create_indexed(size, rcounts, disps, rdtype, &ndtype);
+        /* TODO:BIGCOUNT: Remove these temporaries once ompi_datatype is updated for bigcount */
+        int *tmp_rcounts = malloc(sizeof(int) * size);
+        int *tmp_disps = malloc(sizeof(int) * size);
+        if (NULL == tmp_rcounts || NULL == tmp_disps) {
+            return OMPI_ERR_OUT_OF_RESOURCE;
+        }
+        for (i = 0; i < size; ++i) {
+            tmp_rcounts[i] = ompi_count_array_get(rcounts, i);
+            tmp_disps[i] = ompi_disp_array_get(disps, i);
+        }
+        ompi_datatype_create_indexed(size, tmp_rcounts, tmp_disps, rdtype, &ndtype);
         ompi_datatype_commit(&ndtype);
+        free(tmp_rcounts);
+        free(tmp_disps);
 
         err = MCA_PML_CALL(recv(rbuf, 1, ndtype, 0,
                                 MCA_COLL_BASE_TAG_GATHERV,
@@ -68,18 +85,18 @@ mca_coll_inter_gatherv_inter(const void *sbuf, int scount,
     }
 
     if (0 == rank) {
-        count = (int *)malloc(sizeof(int) * size_local);
-        displace = (int *)malloc(sizeof(int) * size_local);
+        count = (size_t *)malloc(sizeof(size_t) * size_local);
+        displace = (ptrdiff_t *)malloc(sizeof(ptrdiff_t) * size_local);
         if ((NULL == displace) || (NULL == count)) {
             err = OMPI_ERR_OUT_OF_RESOURCE;
             goto exit;
         }
     }
 
-    err = comm->c_local_comm->c_coll->coll_gather(&scount, 1, MPI_INT,
-                                                 count, 1, MPI_INT,
-                                                 0, comm->c_local_comm,
-                                                 comm->c_local_comm->c_coll->coll_gather_module);
+    err = comm->c_local_comm->c_coll->coll_gather(&scount, sizeof(size_t), MPI_BYTE,
+                                                  count, sizeof(size_t), MPI_BYTE,
+                                                  0, comm->c_local_comm,
+                                                  comm->c_local_comm->c_coll->coll_gather_module);
     if (OMPI_SUCCESS != err) {
         goto exit;
     }
@@ -103,10 +120,12 @@ mca_coll_inter_gatherv_inter(const void *sbuf, int scount,
             ptmp = ptmp_free - gap;
         }
     }
+    OMPI_COUNT_ARRAY_INIT(&count_arg, count);
+    OMPI_DISP_ARRAY_INIT(&displace_arg, displace);
     err = comm->c_local_comm->c_coll->coll_gatherv(sbuf, scount, sdtype,
-                                                  ptmp, count, displace,
-                                                  sdtype,0, comm->c_local_comm,
-                                                  comm->c_local_comm->c_coll->coll_gatherv_module);
+                                                   ptmp, count_arg, displace_arg,
+                                                   sdtype,0, comm->c_local_comm,
+                                                   comm->c_local_comm->c_coll->coll_gatherv_module);
     if (OMPI_SUCCESS != err) {
         goto exit;
     }
diff --git a/ompi/mca/coll/inter/coll_inter_module.c b/ompi/mca/coll/inter/coll_inter_module.c
new file mode 100644
index 00000000000..2dd35c10fa2
--- /dev/null
+++ b/ompi/mca/coll/inter/coll_inter_module.c
@@ -0,0 +1,205 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 University of Houston. All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2013 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "coll_inter.h"
+
+#include <stdio.h>
+
+#include "mpi.h"
+#include "ompi/communicator/communicator.h"
+
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/coll/base/base.h"
+#include "ompi/mca/coll/base/coll_base_functions.h"
+#include "ompi/mca/coll/base/coll_tags.h"
+
+#include "ompi/mca/bml/base/base.h"
+
+
+#if 0
+
+static const mca_coll_base_module_1_0_0_t inter = {
+
+  /* Initialization / finalization functions */
+
+  mca_coll_inter_module_init,
+  mca_coll_inter_module_finalize,
+
+  /* Collective function pointers */
+  /* function pointers marked with NULL are not yet implemented
+     and will use the functions provided in the basic module */
+  mca_coll_inter_allgather_inter,
+  mca_coll_inter_allgatherv_inter,
+  mca_coll_inter_allreduce_inter,
+  NULL, /* alltoall */
+  NULL, /* alltoallv */
+  NULL, /* alltoallw */
+  NULL, /* barrier */
+  mca_coll_inter_bcast_inter,
+  NULL,  /* exscan */
+  mca_coll_inter_gather_inter,
+  mca_coll_inter_gatherv_inter,
+  mca_coll_inter_reduce_inter,
+  NULL,  /* reduce_scatter */
+  NULL,  /* scan */
+  mca_coll_inter_scatter_inter,
+  mca_coll_inter_scatterv_inter
+};
+#endif
+
+static int
+mca_coll_inter_module_enable(mca_coll_base_module_t *module,
+                             struct ompi_communicator_t *comm);
+static int
+mca_coll_inter_module_disable(mca_coll_base_module_t *module,
+                              struct ompi_communicator_t *comm);
+
+/*
+ * Initial query function that is invoked during MPI_INIT, allowing
+ * this module to indicate what level of thread support it provides.
+ */
+int mca_coll_inter_init_query(bool allow_inter_user_threads,
+                             bool have_hidden_user_threads)
+{
+    /* Don't ask. All done */
+    return OMPI_SUCCESS;
+}
+
+
+/*
+ * Invoked when there's a new communicator that has been created.
+ * Look at the communicator and decide which set of functions and
+ * priority we want to return.
+ */
+mca_coll_base_module_t *
+mca_coll_inter_comm_query(struct ompi_communicator_t *comm, int *priority)
+{
+    int size, rsize;
+    mca_coll_inter_module_t *inter_module;
+
+    /* This module only works for inter-communicators */
+    if (!OMPI_COMM_IS_INTER(comm)) {
+        return NULL;
+    }
+
+    /* Get the priority level attached to this module. If priority is less
+     * than or equal to 0, then the module is unavailable. */
+    *priority = mca_coll_inter_priority_param;
+    if (0 >= mca_coll_inter_priority_param) {
+        return NULL;
+    }
+
+    size = ompi_comm_size(comm);
+    rsize = ompi_comm_remote_size(comm);
+
+    if ( size < mca_coll_inter_crossover && rsize < mca_coll_inter_crossover) {
+        return NULL;
+    }
+
+    inter_module = OBJ_NEW(mca_coll_inter_module_t);
+    if (NULL == inter_module) {
+        return NULL;
+    }
+
+    inter_module->super.coll_module_enable = mca_coll_inter_module_enable;
+    inter_module->super.coll_module_disable = mca_coll_inter_module_disable;
+
+    inter_module->super.coll_allgather  = mca_coll_inter_allgather_inter;
+    inter_module->super.coll_allgatherv = mca_coll_inter_allgatherv_inter;
+    inter_module->super.coll_allreduce  = mca_coll_inter_allreduce_inter;
+    inter_module->super.coll_alltoall   = NULL;
+    inter_module->super.coll_alltoallv  = NULL;
+    inter_module->super.coll_alltoallw  = NULL;
+    inter_module->super.coll_barrier    = NULL;
+    inter_module->super.coll_bcast      = mca_coll_inter_bcast_inter;
+    inter_module->super.coll_exscan     = NULL;
+    inter_module->super.coll_gather     = mca_coll_inter_gather_inter;
+    inter_module->super.coll_gatherv    = mca_coll_inter_gatherv_inter;
+    inter_module->super.coll_reduce     = mca_coll_inter_reduce_inter;
+    inter_module->super.coll_reduce_scatter = NULL;
+    inter_module->super.coll_scan       = NULL;
+    inter_module->super.coll_scatter    = mca_coll_inter_scatter_inter;
+    inter_module->super.coll_scatterv   = mca_coll_inter_scatterv_inter;
+    inter_module->super.coll_reduce_local = mca_coll_base_reduce_local;
+
+    return &(inter_module->super);
+}
+
+#define INTER_INSTALL_COLL_API(__comm, __module, __api)                                               \
+    do                                                                                                \
+    {                                                                                                 \
+        MCA_COLL_INSTALL_API(__comm, __api, __module->super.coll_##__api, &__module->super, "inter"); \
+    } while (0)
+
+#define INTER_UNINSTALL_COLL_API(__comm, __module, __api)               \
+    do                                                                  \
+    {                                                                   \
+        if (__comm->c_coll->coll_##__api##_module == &__module->super)  \
+        {                                                               \
+            MCA_COLL_INSTALL_API(__comm, __api, NULL, NULL, "inter");   \
+        }                                                               \
+    } while (0)
+/*
+ * Init module on the communicator
+ */
+static int
+mca_coll_inter_module_enable(mca_coll_base_module_t *module,
+                             struct ompi_communicator_t *comm)
+{
+    mca_coll_inter_module_t *inter_module = (mca_coll_inter_module_t*) module;
+
+    inter_module->inter_comm = comm;
+
+    INTER_INSTALL_COLL_API(comm, inter_module, allgather);
+    INTER_INSTALL_COLL_API(comm, inter_module, allgatherv);
+    INTER_INSTALL_COLL_API(comm, inter_module, allreduce);
+    INTER_INSTALL_COLL_API(comm, inter_module, bcast);
+    INTER_INSTALL_COLL_API(comm, inter_module, gather);
+    INTER_INSTALL_COLL_API(comm, inter_module, gatherv);
+    INTER_INSTALL_COLL_API(comm, inter_module, reduce);
+    INTER_INSTALL_COLL_API(comm, inter_module, scatter);
+    INTER_INSTALL_COLL_API(comm, inter_module, scatterv);
+    return OMPI_SUCCESS;
+}
+
+static int
+mca_coll_inter_module_disable(mca_coll_base_module_t *module,
+                              struct ompi_communicator_t *comm)
+{
+    mca_coll_inter_module_t *inter_module = (mca_coll_inter_module_t*) module;
+
+    INTER_UNINSTALL_COLL_API(comm, inter_module, allgather);
+    INTER_UNINSTALL_COLL_API(comm, inter_module, allgatherv);
+    INTER_UNINSTALL_COLL_API(comm, inter_module, allreduce);
+    INTER_UNINSTALL_COLL_API(comm, inter_module, bcast);
+    INTER_UNINSTALL_COLL_API(comm, inter_module, gather);
+    INTER_UNINSTALL_COLL_API(comm, inter_module, gatherv);
+    INTER_UNINSTALL_COLL_API(comm, inter_module, reduce);
+    INTER_UNINSTALL_COLL_API(comm, inter_module, scatter);
+    INTER_UNINSTALL_COLL_API(comm, inter_module, scatterv);
+
+    inter_module->inter_comm = NULL;
+
+    return OMPI_SUCCESS;
+}
diff --git a/ompi/mca/coll/inter/coll_inter_reduce.c b/ompi/mca/coll/inter/coll_inter_reduce.c
index 1894d2b9c16..bbdb4180f69 100644
--- a/ompi/mca/coll/inter/coll_inter_reduce.c
+++ b/ompi/mca/coll/inter/coll_inter_reduce.c
@@ -40,7 +40,7 @@
  *	Returns:	- MPI_SUCCESS or error code
  */
 int
-mca_coll_inter_reduce_inter(const void *sbuf, void *rbuf, int count,
+mca_coll_inter_reduce_inter(const void *sbuf, void *rbuf, size_t count,
                             struct ompi_datatype_t *dtype,
                             struct ompi_op_t *op,
                             int root, struct ompi_communicator_t *comm,
diff --git a/ompi/mca/coll/inter/coll_inter_scatter.c b/ompi/mca/coll/inter/coll_inter_scatter.c
index 94871f0be93..bc1ad336a85 100644
--- a/ompi/mca/coll/inter/coll_inter_scatter.c
+++ b/ompi/mca/coll/inter/coll_inter_scatter.c
@@ -12,6 +12,7 @@
  * Copyright (c) 2006-2008 University of Houston. All rights reserved.
  * Copyright (c) 2015-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -37,9 +38,9 @@
  *	Returns:	- MPI_SUCCESS or error code
  */
 int
-mca_coll_inter_scatter_inter(const void *sbuf, int scount,
+mca_coll_inter_scatter_inter(const void *sbuf, size_t scount,
                              struct ompi_datatype_t *sdtype,
-                             void *rbuf, int rcount,
+                             void *rbuf, size_t rcount,
                              struct ompi_datatype_t *rdtype,
                              int root, struct ompi_communicator_t *comm,
                              mca_coll_base_module_t *module)
@@ -69,7 +70,7 @@ mca_coll_inter_scatter_inter(const void *sbuf, int scount,
 	    }
             ptmp = ptmp_free - gap;
 
-	    err = MCA_PML_CALL(recv(ptmp, rcount*size_local, rdtype,
+	    err = MCA_PML_CALL(recv(ptmp, rcount*(size_t)size_local, rdtype,
 				    root, MCA_COLL_BASE_TAG_SCATTER,
 				    comm, MPI_STATUS_IGNORE));
 	    if (OMPI_SUCCESS != err) {
@@ -86,7 +87,7 @@ mca_coll_inter_scatter_inter(const void *sbuf, int scount,
 	}
     } else {
 	/* Root sends data to the first process in the remote group */
-	err = MCA_PML_CALL(send(sbuf, scount*size, sdtype, 0,
+	err = MCA_PML_CALL(send(sbuf, scount*(size_t)size, sdtype, 0,
 				MCA_COLL_BASE_TAG_SCATTER,
 				MCA_PML_BASE_SEND_STANDARD, comm));
 	if (OMPI_SUCCESS != err) {
diff --git a/ompi/mca/coll/inter/coll_inter_scatterv.c b/ompi/mca/coll/inter/coll_inter_scatterv.c
index e0ccaedd1e6..5d98e1ea099 100644
--- a/ompi/mca/coll/inter/coll_inter_scatterv.c
+++ b/ompi/mca/coll/inter/coll_inter_scatterv.c
@@ -12,6 +12,7 @@
  * Copyright (c) 2006-2010 University of Houston. All rights reserved.
  * Copyright (c) 2015-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -38,15 +39,19 @@
  *	Returns:	- MPI_SUCCESS or error code
  */
 int
-mca_coll_inter_scatterv_inter(const void *sbuf, const int *scounts,
-                              const int *disps, struct ompi_datatype_t *sdtype,
-                              void *rbuf, int rcount,
+mca_coll_inter_scatterv_inter(const void *sbuf, ompi_count_array_t scounts,
+                              ompi_disp_array_t disps, struct ompi_datatype_t *sdtype,
+                              void *rbuf, size_t rcount,
                               struct ompi_datatype_t *rdtype, int root,
                               struct ompi_communicator_t *comm,
                               mca_coll_base_module_t *module)
 {
-    int i, rank, size, err, total=0, size_local;
-    int *counts=NULL,*displace=NULL;
+    int i, rank, size, err, size_local;
+    size_t total = 0;
+    size_t *counts=NULL;
+    ptrdiff_t *displace=NULL;
+    ompi_count_array_t counts_arg;
+    ompi_disp_array_t displace_arg;
     char *ptmp_free=NULL, *ptmp=NULL;
     ompi_datatype_t *ndtype;
 
@@ -61,9 +66,9 @@ mca_coll_inter_scatterv_inter(const void *sbuf, const int *scounts,
         err = OMPI_SUCCESS;
     } else if (MPI_ROOT != root) {
 	if(0 == rank) {
-	    /* local root recieves the counts from the root */
-	    counts = (int *)malloc(sizeof(int) * size_local);
-	    err = MCA_PML_CALL(recv(counts, size_local, MPI_INT,
+	    /* local root receives the counts from the root */
+	    counts = (size_t *)malloc(sizeof(size_t) * size_local);
+	    err = MCA_PML_CALL(recv(counts, sizeof(size_t) * size_local, MPI_BYTE,
 				    root, MCA_COLL_BASE_TAG_SCATTERV,
 				    comm, MPI_STATUS_IGNORE));
 	    if (OMPI_SUCCESS != err) {
@@ -89,14 +94,16 @@ mca_coll_inter_scatterv_inter(const void *sbuf, const int *scounts,
 		return err;
 	    }
 	    /* set the local displacement i.e. no displacements here */
-	    displace = (int *)malloc(sizeof(int) * size_local);
+	    displace = (ptrdiff_t *)malloc(sizeof(ptrdiff_t) * size_local);
 	    displace[0] = 0;
 	    for (i = 1; i < size_local; i++) {
 		displace[i] = displace[i-1] + counts[i-1];
 	    }
 	}
 	/* perform the scatterv locally */
-	err = comm->c_local_comm->c_coll->coll_scatterv(ptmp, counts, displace,
+	OMPI_COUNT_ARRAY_INIT(&counts_arg, counts);
+	OMPI_DISP_ARRAY_INIT(&displace_arg, displace);
+	err = comm->c_local_comm->c_coll->coll_scatterv(ptmp, counts_arg, displace_arg,
 						       rdtype, rbuf, rcount,
 						       rdtype, 0, comm->c_local_comm,
                                                        comm->c_local_comm->c_coll->coll_scatterv_module);
@@ -115,15 +122,37 @@ mca_coll_inter_scatterv_inter(const void *sbuf, const int *scounts,
 	}
 
     } else {
-	err = MCA_PML_CALL(send(scounts, size, MPI_INT, 0,
+	/* We must ensure that rank 0 receives a size_t array */
+	size_t *tmp_scounts_root = malloc(sizeof(size_t) * size);
+	if (NULL == tmp_scounts_root) {
+	    return OMPI_ERR_OUT_OF_RESOURCE;
+	}
+	for (i = 0; i < size; ++i) {
+	    tmp_scounts_root[i] = ompi_count_array_get(scounts, i);
+	}
+	err = MCA_PML_CALL(send(tmp_scounts_root, sizeof(size_t) * size, MPI_BYTE, 0,
 				MCA_COLL_BASE_TAG_SCATTERV,
 				MCA_PML_BASE_SEND_STANDARD, comm));
+	free(tmp_scounts_root);
+
 	if (OMPI_SUCCESS != err) {
 	    return err;
 	}
 
-	ompi_datatype_create_indexed(size,scounts,disps,sdtype,&ndtype);
+	/* TODO:BIGCOUNT: Remove these temporaries once ompi_datatype is updated for bigcount */
+	int *tmp_scounts = malloc(sizeof(int) * size);
+	int *tmp_disps = malloc(sizeof(int) * size);
+	if (NULL == tmp_scounts || NULL == tmp_disps) {
+	    return OMPI_ERR_OUT_OF_RESOURCE;
+	}
+	for (i = 0; i < size; ++i) {
+	    tmp_scounts[i] = (int) ompi_count_array_get(scounts, i);
+	    tmp_disps[i] = (int) ompi_disp_array_get(disps, i);
+	}
+	ompi_datatype_create_indexed(size,tmp_scounts,tmp_disps,sdtype,&ndtype);
 	ompi_datatype_commit(&ndtype);
+	free(tmp_scounts);
+	free(tmp_disps);
 
 	err = MCA_PML_CALL(send(sbuf, 1, ndtype, 0,
 				MCA_COLL_BASE_TAG_SCATTERV,
diff --git a/ompi/mca/coll/libnbc/coll_libnbc.h b/ompi/mca/coll/libnbc/coll_libnbc.h
index 8a1fdde4f1a..b68688002ef 100644
--- a/ompi/mca/coll/libnbc/coll_libnbc.h
+++ b/ompi/mca/coll/libnbc/coll_libnbc.h
@@ -50,7 +50,7 @@ BEGIN_C_DECLS
 #undef NBC_CACHE_SCHEDULE
 #endif
 #define NBC_SCHED_DICT_UPPER 1024 /* max. number of dict entries */
-#define NBC_SCHED_DICT_LOWER 512  /* nuber of dict entries after wipe, if SCHED_DICT_UPPER is reached */
+#define NBC_SCHED_DICT_LOWER 512  /* number of dict entries after wipe, if SCHED_DICT_UPPER is reached */
 
 /********************* end of LibNBC tuning parameters ************************/
 
@@ -79,7 +79,7 @@ extern int libnbc_ireduce_algorithm;
 extern int libnbc_iscan_algorithm;
 
 struct ompi_coll_libnbc_component_t {
-    mca_coll_base_component_2_4_0_t super;
+    mca_coll_base_component_3_0_0_t super;
     opal_free_list_t requests;
     opal_list_t active_requests;
     opal_atomic_int32_t active_comms;
@@ -88,7 +88,7 @@ struct ompi_coll_libnbc_component_t {
 typedef struct ompi_coll_libnbc_component_t ompi_coll_libnbc_component_t;
 
 /* Globally exported variables */
-OMPI_MODULE_DECLSPEC extern ompi_coll_libnbc_component_t mca_coll_libnbc_component;
+OMPI_DECLSPEC extern ompi_coll_libnbc_component_t mca_coll_libnbc_component;
 
 struct ompi_coll_libnbc_module_t {
     mca_coll_base_module_t super;
@@ -161,258 +161,262 @@ int NBC_Init_comm(MPI_Comm comm, ompi_coll_libnbc_module_t *module);
 int NBC_Progress(NBC_Handle *handle);
 
 
-int ompi_coll_libnbc_iallgather(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+int ompi_coll_libnbc_iallgather(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf, size_t recvcount,
                                 MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                 mca_coll_base_module_t *module);
-int ompi_coll_libnbc_iallgatherv(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs,
+int ompi_coll_libnbc_iallgatherv(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf,
+                                 ompi_count_array_t recvcounts, ompi_disp_array_t displs,
                                  MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                  mca_coll_base_module_t *module);
-int ompi_coll_libnbc_iallreduce(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
+int ompi_coll_libnbc_iallreduce(const void* sendbuf, void* recvbuf, size_t count, MPI_Datatype datatype, MPI_Op op,
                                 struct ompi_communicator_t *comm, ompi_request_t ** request,
                                 mca_coll_base_module_t *module);
-int ompi_coll_libnbc_ialltoall(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+int ompi_coll_libnbc_ialltoall(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf, size_t recvcount,
                                MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                mca_coll_base_module_t *module);
-int ompi_coll_libnbc_ialltoallv(const void* sendbuf, const int *sendcounts, const int *sdispls,
-                                MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls,
+int ompi_coll_libnbc_ialltoallv(const void* sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t sdispls,
+                                MPI_Datatype sendtype, void* recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t rdispls,
                                 MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                 mca_coll_base_module_t *module);
-int ompi_coll_libnbc_ialltoallw(const void *sbuf, const int *scounts, const int *sdisps, struct ompi_datatype_t * const *sdtypes,
-                                void *rbuf, const int *rcounts, const int *rdisps, struct ompi_datatype_t * const *rdtypes,
+int ompi_coll_libnbc_ialltoallw(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps, struct ompi_datatype_t * const *sdtypes,
+                                void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps, struct ompi_datatype_t * const *rdtypes,
                                 struct ompi_communicator_t *comm, ompi_request_t **request,
                                 mca_coll_base_module_t *module);
 int ompi_coll_libnbc_ibarrier(struct ompi_communicator_t *comm, ompi_request_t ** request,
                               mca_coll_base_module_t *module);
-int ompi_coll_libnbc_ibcast(void *buffer, int count, MPI_Datatype datatype, int root,
+int ompi_coll_libnbc_ibcast(void *buffer, size_t count, MPI_Datatype datatype, int root,
                             struct ompi_communicator_t *comm, ompi_request_t ** request,
                             mca_coll_base_module_t *module);
-int ompi_coll_libnbc_iexscan(const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype,
+int ompi_coll_libnbc_iexscan(const void *sbuf, void *rbuf, size_t count, struct ompi_datatype_t *dtype,
                              struct ompi_op_t *op, struct ompi_communicator_t *comm, ompi_request_t **request,
                              mca_coll_base_module_t *module);
-int ompi_coll_libnbc_igather(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+int ompi_coll_libnbc_igather(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf, size_t recvcount,
                              MPI_Datatype recvtype, int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
                              mca_coll_base_module_t *module);
-int ompi_coll_libnbc_igatherv(const void* sendbuf, int sendcount, MPI_Datatype sendtype,
-                              void* recvbuf, const int *recvcounts, const int *displs, MPI_Datatype recvtype,
+int ompi_coll_libnbc_igatherv(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype,
+                              void* recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t displs, MPI_Datatype recvtype,
                               int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
                               mca_coll_base_module_t *module);
-int ompi_coll_libnbc_ireduce(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype,
+int ompi_coll_libnbc_ireduce(const void* sendbuf, void* recvbuf, size_t count, MPI_Datatype datatype,
                              MPI_Op op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
                              mca_coll_base_module_t *module);
-int ompi_coll_libnbc_ireduce_scatter(const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype,
+int ompi_coll_libnbc_ireduce_scatter(const void* sendbuf, void* recvbuf, ompi_count_array_t recvcounts, MPI_Datatype datatype,
                                      MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                      mca_coll_base_module_t *module);
-int ompi_coll_libnbc_ireduce_scatter_block(const void *sbuf, void *rbuf, int rcount, struct ompi_datatype_t *dtype,
+int ompi_coll_libnbc_ireduce_scatter_block(const void *sbuf, void *rbuf, size_t rcount, struct ompi_datatype_t *dtype,
                                            struct ompi_op_t *op, struct ompi_communicator_t *comm,
                                            ompi_request_t **request,
                                            mca_coll_base_module_t *module);
-int ompi_coll_libnbc_iscan(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
+int ompi_coll_libnbc_iscan(const void* sendbuf, void* recvbuf, size_t count, MPI_Datatype datatype, MPI_Op op,
                            struct ompi_communicator_t *comm, ompi_request_t ** request,
                            mca_coll_base_module_t *module);
-int ompi_coll_libnbc_iscatter(const void* sendbuf, int sendcount, MPI_Datatype sendtype,
-                              void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
+int ompi_coll_libnbc_iscatter(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype,
+                              void* recvbuf, size_t recvcount, MPI_Datatype recvtype, int root,
                               struct ompi_communicator_t *comm, ompi_request_t ** request,
                               mca_coll_base_module_t *module);
-int ompi_coll_libnbc_iscatterv(const void* sendbuf, const int *sendcounts, const int *displs, MPI_Datatype sendtype,
-                               void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
+int ompi_coll_libnbc_iscatterv(const void* sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t displs, MPI_Datatype sendtype,
+                               void* recvbuf, size_t recvcount, MPI_Datatype recvtype, int root,
                                struct ompi_communicator_t *comm, ompi_request_t ** request,
                                mca_coll_base_module_t *module);
 
 
-int ompi_coll_libnbc_iallgather_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+int ompi_coll_libnbc_iallgather_inter(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf, size_t recvcount,
                                 MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                 mca_coll_base_module_t *module);
-int ompi_coll_libnbc_iallgatherv_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs,
-                                 MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
-                                 mca_coll_base_module_t *module);
-int ompi_coll_libnbc_iallreduce_inter(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
+int ompi_coll_libnbc_iallgatherv_inter(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf,
+                                       ompi_count_array_t recvcounts, ompi_disp_array_t displs,
+                                       MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
+                                       mca_coll_base_module_t *module);
+int ompi_coll_libnbc_iallreduce_inter(const void* sendbuf, void* recvbuf, size_t count, MPI_Datatype datatype, MPI_Op op,
                                 struct ompi_communicator_t *comm, ompi_request_t ** request,
                                 mca_coll_base_module_t *module);
-int ompi_coll_libnbc_ialltoall_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+int ompi_coll_libnbc_ialltoall_inter(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf, size_t recvcount,
                                MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                mca_coll_base_module_t *module);
-int ompi_coll_libnbc_ialltoallv_inter(const void* sendbuf, const int *sendcounts, const int *sdispls,
-                                MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls,
+int ompi_coll_libnbc_ialltoallv_inter(const void* sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t sdispls,
+                                MPI_Datatype sendtype, void* recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t rdispls,
                                 MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                 mca_coll_base_module_t *module);
-int ompi_coll_libnbc_ialltoallw_inter(const void *sbuf, const int *scounts, const int *sdisps, struct ompi_datatype_t * const *sdtypes,
-                                      void *rbuf, const int *rcounts, const int *rdisps, struct ompi_datatype_t * const *rdtypes,
+int ompi_coll_libnbc_ialltoallw_inter(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps, struct ompi_datatype_t * const *sdtypes,
+                                      void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps, struct ompi_datatype_t * const *rdtypes,
                                       struct ompi_communicator_t *comm, ompi_request_t **request,
                                       mca_coll_base_module_t *module);
 int ompi_coll_libnbc_ibarrier_inter(struct ompi_communicator_t *comm, ompi_request_t ** request,
                               mca_coll_base_module_t *module);
-int ompi_coll_libnbc_ibcast_inter(void *buffer, int count, MPI_Datatype datatype, int root,
+int ompi_coll_libnbc_ibcast_inter(void *buffer, size_t count, MPI_Datatype datatype, int root,
                             struct ompi_communicator_t *comm, ompi_request_t ** request,
                             mca_coll_base_module_t *module);
-int ompi_coll_libnbc_igather_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+int ompi_coll_libnbc_igather_inter(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf, size_t recvcount,
                              MPI_Datatype recvtype, int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
                              mca_coll_base_module_t *module);
-int ompi_coll_libnbc_igatherv_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype,
-                              void* recvbuf, const int *recvcounts, const int *displs, MPI_Datatype recvtype,
+int ompi_coll_libnbc_igatherv_inter(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype,
+                              void* recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t displs, MPI_Datatype recvtype,
                               int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
                               mca_coll_base_module_t *module);
-int ompi_coll_libnbc_ireduce_inter(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype,
+int ompi_coll_libnbc_ireduce_inter(const void* sendbuf, void* recvbuf, size_t count, MPI_Datatype datatype,
                              MPI_Op op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
                              mca_coll_base_module_t *module);
-int ompi_coll_libnbc_ireduce_scatter_inter(const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype,
+int ompi_coll_libnbc_ireduce_scatter_inter(const void* sendbuf, void* recvbuf, ompi_count_array_t recvcounts, MPI_Datatype datatype,
                                      MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                      mca_coll_base_module_t *module);
-int ompi_coll_libnbc_ireduce_scatter_block_inter(const void *sbuf, void *rbuf, int rcount, struct ompi_datatype_t *dtype,
+int ompi_coll_libnbc_ireduce_scatter_block_inter(const void *sbuf, void *rbuf, size_t rcount, struct ompi_datatype_t *dtype,
                                                  struct ompi_op_t *op, struct ompi_communicator_t *comm,
                                                  ompi_request_t **request,
                                                  mca_coll_base_module_t *module);
-int ompi_coll_libnbc_iscatter_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype,
-                              void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
+int ompi_coll_libnbc_iscatter_inter(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype,
+                              void* recvbuf, size_t recvcount, MPI_Datatype recvtype, int root,
                               struct ompi_communicator_t *comm, ompi_request_t ** request,
                               mca_coll_base_module_t *module);
-int ompi_coll_libnbc_iscatterv_inter(const void* sendbuf, const int *sendcounts, const int *displs, MPI_Datatype sendtype,
-                               void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
+int ompi_coll_libnbc_iscatterv_inter(const void* sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t displs, MPI_Datatype sendtype,
+                               void* recvbuf, size_t recvcount, MPI_Datatype recvtype, int root,
                                struct ompi_communicator_t *comm, ompi_request_t ** request,
                                mca_coll_base_module_t *module);
 
 
-int ompi_coll_libnbc_ineighbor_allgather(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
-                                         int rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm,
+int ompi_coll_libnbc_ineighbor_allgather(const void *sbuf, size_t scount, MPI_Datatype stype, void *rbuf,
+                                         size_t rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm,
                                          ompi_request_t ** request, mca_coll_base_module_t *module);
-int ompi_coll_libnbc_ineighbor_allgatherv(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
-                                          const int *rcounts, const int *displs, MPI_Datatype rtype,
+int ompi_coll_libnbc_ineighbor_allgatherv(const void *sbuf, size_t scount, MPI_Datatype stype, void *rbuf,
+                                          ompi_count_array_t rcounts, ompi_disp_array_t displs, MPI_Datatype rtype,
                                           struct ompi_communicator_t *comm, ompi_request_t ** request,
                                           mca_coll_base_module_t *module);
-int ompi_coll_libnbc_ineighbor_alltoall(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
-                                        int rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm,
+int ompi_coll_libnbc_ineighbor_alltoall(const void *sbuf, size_t scount, MPI_Datatype stype, void *rbuf,
+                                        size_t rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm,
                                         ompi_request_t ** request, mca_coll_base_module_t *module);
-int ompi_coll_libnbc_ineighbor_alltoallv(const void *sbuf, const int *scounts, const int *sdispls, MPI_Datatype stype,
-                                         void *rbuf, const int *rcounts, const int *rdispls, MPI_Datatype rtype,
+int ompi_coll_libnbc_ineighbor_alltoallv(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdispls, MPI_Datatype stype,
+                                         void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdispls, MPI_Datatype rtype,
                                          struct ompi_communicator_t *comm, ompi_request_t ** request,
                                          mca_coll_base_module_t *module);
-int ompi_coll_libnbc_ineighbor_alltoallw(const void *sbuf, const int *scounts, const MPI_Aint *sdisps, struct ompi_datatype_t * const *stypes,
-                                         void *rbuf, const int *rcounts, const MPI_Aint *rdisps, struct ompi_datatype_t * const *rtypes,
+int ompi_coll_libnbc_ineighbor_alltoallw(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps, struct ompi_datatype_t * const *stypes,
+                                         void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps, struct ompi_datatype_t * const *rtypes,
                                          struct ompi_communicator_t *comm, ompi_request_t ** request,
                                          mca_coll_base_module_t *module);
 
-int ompi_coll_libnbc_allgather_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+int ompi_coll_libnbc_allgather_init(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf, size_t recvcount,
                                     MPI_Datatype recvtype, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                     mca_coll_base_module_t *module);
-int ompi_coll_libnbc_allgatherv_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs,
+int ompi_coll_libnbc_allgatherv_init(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf,
+                                     ompi_count_array_t recvcounts, ompi_disp_array_t displs,
                                      MPI_Datatype recvtype, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                      mca_coll_base_module_t *module);
-int ompi_coll_libnbc_allreduce_init(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
+int ompi_coll_libnbc_allreduce_init(const void* sendbuf, void* recvbuf, size_t count, MPI_Datatype datatype, MPI_Op op,
                                     struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                     mca_coll_base_module_t *module);
-int ompi_coll_libnbc_alltoall_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+int ompi_coll_libnbc_alltoall_init(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf, size_t recvcount,
                                    MPI_Datatype recvtype, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                    mca_coll_base_module_t *module);
-int ompi_coll_libnbc_alltoallv_init(const void* sendbuf, const int *sendcounts, const int *sdispls,
-                                    MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls,
+int ompi_coll_libnbc_alltoallv_init(const void* sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t sdispls,
+                                    MPI_Datatype sendtype, void* recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t rdispls,
                                     MPI_Datatype recvtype, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                     mca_coll_base_module_t *module);
-int ompi_coll_libnbc_alltoallw_init(const void *sbuf, const int *scounts, const int *sdisps, struct ompi_datatype_t * const *sdtypes,
-                                    void *rbuf, const int *rcounts, const int *rdisps, struct ompi_datatype_t * const *rdtypes,
+int ompi_coll_libnbc_alltoallw_init(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps, struct ompi_datatype_t * const *sdtypes,
+                                    void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps, struct ompi_datatype_t * const *rdtypes,
                                     struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t **request,
                                     mca_coll_base_module_t *module);
 int ompi_coll_libnbc_barrier_init(struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                   mca_coll_base_module_t *module);
-int ompi_coll_libnbc_bcast_init(void *buffer, int count, MPI_Datatype datatype, int root,
+int ompi_coll_libnbc_bcast_init(void *buffer, size_t count, MPI_Datatype datatype, int root,
                                 struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                 mca_coll_base_module_t *module);
-int ompi_coll_libnbc_exscan_init(const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype,
+int ompi_coll_libnbc_exscan_init(const void *sbuf, void *rbuf, size_t count, struct ompi_datatype_t *dtype,
                                  struct ompi_op_t *op, struct ompi_communicator_t *comm, MPI_Info info,  ompi_request_t **request,
                                  mca_coll_base_module_t *module);
-int ompi_coll_libnbc_gather_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+int ompi_coll_libnbc_gather_init(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf, size_t recvcount,
                                  MPI_Datatype recvtype, int root, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                  mca_coll_base_module_t *module);
-int ompi_coll_libnbc_gatherv_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype,
-                                  void* recvbuf, const int *recvcounts, const int *displs, MPI_Datatype recvtype,
+int ompi_coll_libnbc_gatherv_init(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype,
+                                  void* recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t displs, MPI_Datatype recvtype,
                                   int root, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                   mca_coll_base_module_t *module);
-int ompi_coll_libnbc_reduce_init(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype,
+int ompi_coll_libnbc_reduce_init(const void* sendbuf, void* recvbuf, size_t count, MPI_Datatype datatype,
                                  MPI_Op op, int root, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                  mca_coll_base_module_t *module);
-int ompi_coll_libnbc_reduce_scatter_init(const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype,
+int ompi_coll_libnbc_reduce_scatter_init(const void* sendbuf, void* recvbuf, ompi_count_array_t recvcounts, MPI_Datatype datatype,
                                          MPI_Op op, struct ompi_communicator_t *comm, MPI_Info info,  ompi_request_t ** request,
                                          mca_coll_base_module_t *module);
-int ompi_coll_libnbc_reduce_scatter_block_init(const void *sbuf, void *rbuf, int rcount, struct ompi_datatype_t *dtype,
+int ompi_coll_libnbc_reduce_scatter_block_init(const void *sbuf, void *rbuf, size_t rcount, struct ompi_datatype_t *dtype,
                                                struct ompi_op_t *op, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t **request,
                                                mca_coll_base_module_t *module);
-int ompi_coll_libnbc_scan_init(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
+int ompi_coll_libnbc_scan_init(const void* sendbuf, void* recvbuf, size_t count, MPI_Datatype datatype, MPI_Op op,
                                struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                mca_coll_base_module_t *module);
-int ompi_coll_libnbc_scatter_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype,
-                                  void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
+int ompi_coll_libnbc_scatter_init(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype,
+                                  void* recvbuf, size_t recvcount, MPI_Datatype recvtype, int root,
                                   struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                   mca_coll_base_module_t *module);
-int ompi_coll_libnbc_scatterv_init(const void* sendbuf, const int *sendcounts, const int *displs, MPI_Datatype sendtype,
-                                   void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
+int ompi_coll_libnbc_scatterv_init(const void* sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t displs, MPI_Datatype sendtype,
+                                   void* recvbuf, size_t recvcount, MPI_Datatype recvtype, int root,
                                    struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                    mca_coll_base_module_t *module);
 
-int ompi_coll_libnbc_allgather_inter_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+int ompi_coll_libnbc_allgather_inter_init(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf, size_t recvcount,
                                           MPI_Datatype recvtype, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                           mca_coll_base_module_t *module);
-int ompi_coll_libnbc_allgatherv_inter_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs,
+int ompi_coll_libnbc_allgatherv_inter_init(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf,
+                                           ompi_count_array_t recvcounts, ompi_disp_array_t displs,
                                            MPI_Datatype recvtype, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                            mca_coll_base_module_t *module);
-int ompi_coll_libnbc_allreduce_inter_init(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
+int ompi_coll_libnbc_allreduce_inter_init(const void* sendbuf, void* recvbuf, size_t count, MPI_Datatype datatype, MPI_Op op,
                                           struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                           mca_coll_base_module_t *module);
-int ompi_coll_libnbc_alltoall_inter_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+int ompi_coll_libnbc_alltoall_inter_init(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf, size_t recvcount,
                                          MPI_Datatype recvtype, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                          mca_coll_base_module_t *module);
-int ompi_coll_libnbc_alltoallv_inter_init(const void* sendbuf, const int *sendcounts, const int *sdispls,
-                                          MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls,
+int ompi_coll_libnbc_alltoallv_inter_init(const void* sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t sdispls,
+                                          MPI_Datatype sendtype, void* recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t rdispls,
                                           MPI_Datatype recvtype, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                           mca_coll_base_module_t *module);
-int ompi_coll_libnbc_alltoallw_inter_init(const void *sbuf, const int *scounts, const int *sdisps, struct ompi_datatype_t * const *sdtypes,
-                                          void *rbuf, const int *rcounts, const int *rdisps, struct ompi_datatype_t * const *rdtypes,
+int ompi_coll_libnbc_alltoallw_inter_init(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps, struct ompi_datatype_t * const *sdtypes,
+                                          void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps, struct ompi_datatype_t * const *rdtypes,
                                           struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t **request,
                                           mca_coll_base_module_t *module);
 int ompi_coll_libnbc_barrier_inter_init(struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                         mca_coll_base_module_t *module);
-int ompi_coll_libnbc_bcast_inter_init(void *buffer, int count, MPI_Datatype datatype, int root,
+int ompi_coll_libnbc_bcast_inter_init(void *buffer, size_t count, MPI_Datatype datatype, int root,
                                       struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                       mca_coll_base_module_t *module);
-int ompi_coll_libnbc_gather_inter_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+int ompi_coll_libnbc_gather_inter_init(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf, size_t recvcount,
                                        MPI_Datatype recvtype, int root, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                        mca_coll_base_module_t *module);
-int ompi_coll_libnbc_gatherv_inter_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype,
-                                        void* recvbuf, const int *recvcounts, const int *displs, MPI_Datatype recvtype,
+int ompi_coll_libnbc_gatherv_inter_init(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype,
+                                        void* recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t displs, MPI_Datatype recvtype,
                                         int root, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                         mca_coll_base_module_t *module);
-int ompi_coll_libnbc_reduce_inter_init(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype,
+int ompi_coll_libnbc_reduce_inter_init(const void* sendbuf, void* recvbuf, size_t count, MPI_Datatype datatype,
                                        MPI_Op op, int root, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                        mca_coll_base_module_t *module);
-int ompi_coll_libnbc_reduce_scatter_inter_init(const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype,
+int ompi_coll_libnbc_reduce_scatter_inter_init(const void* sendbuf, void* recvbuf, ompi_count_array_t recvcounts, MPI_Datatype datatype,
                                                MPI_Op op, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                                mca_coll_base_module_t *module);
-int ompi_coll_libnbc_reduce_scatter_block_inter_init(const void *sbuf, void *rbuf, int rcount, struct ompi_datatype_t *dtype,
+int ompi_coll_libnbc_reduce_scatter_block_inter_init(const void *sbuf, void *rbuf, size_t rcount, struct ompi_datatype_t *dtype,
                                                      struct ompi_op_t *op, struct ompi_communicator_t *comm,
                                                      MPI_Info info, ompi_request_t **request,
                                                      mca_coll_base_module_t *module);
-int ompi_coll_libnbc_scatter_inter_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype,
-                                        void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
+int ompi_coll_libnbc_scatter_inter_init(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype,
+                                        void* recvbuf, size_t recvcount, MPI_Datatype recvtype, int root,
                                         struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                         mca_coll_base_module_t *module);
-int ompi_coll_libnbc_scatterv_inter_init(const void* sendbuf, const int *sendcounts, const int *displs, MPI_Datatype sendtype,
-                                         void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
+int ompi_coll_libnbc_scatterv_inter_init(const void* sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t displs, MPI_Datatype sendtype,
+                                         void* recvbuf, size_t recvcount, MPI_Datatype recvtype, int root,
                                          struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                          mca_coll_base_module_t *module);
 
-int ompi_coll_libnbc_neighbor_allgather_init(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
-                                             int rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm,
+int ompi_coll_libnbc_neighbor_allgather_init(const void *sbuf, size_t scount, MPI_Datatype stype, void *rbuf,
+                                             size_t rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm,
                                              MPI_Info info, ompi_request_t ** request, mca_coll_base_module_t *module);
-int ompi_coll_libnbc_neighbor_allgatherv_init(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
-                                              const int *rcounts, const int *displs, MPI_Datatype rtype,
+int ompi_coll_libnbc_neighbor_allgatherv_init(const void *sbuf, size_t scount, MPI_Datatype stype, void *rbuf,
+                                              ompi_count_array_t rcounts, ompi_disp_array_t displs, MPI_Datatype rtype,
                                               struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                               mca_coll_base_module_t *module);
-int ompi_coll_libnbc_neighbor_alltoall_init(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
-                                            int rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm, MPI_Info info,
+int ompi_coll_libnbc_neighbor_alltoall_init(const void *sbuf, size_t scount, MPI_Datatype stype, void *rbuf,
+                                            size_t rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm, MPI_Info info,
                                             ompi_request_t ** request, mca_coll_base_module_t *module);
-int ompi_coll_libnbc_neighbor_alltoallv_init(const void *sbuf, const int *scounts, const int *sdispls, MPI_Datatype stype,
-                                             void *rbuf, const int *rcounts, const int *rdispls, MPI_Datatype rtype,
+int ompi_coll_libnbc_neighbor_alltoallv_init(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdispls, MPI_Datatype stype,
+                                             void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdispls, MPI_Datatype rtype,
                                              struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                              mca_coll_base_module_t *module);
-int ompi_coll_libnbc_neighbor_alltoallw_init(const void *sbuf, const int *scounts, const MPI_Aint *sdisps, struct ompi_datatype_t * const *stypes,
-                                             void *rbuf, const int *rcounts, const MPI_Aint *rdisps, struct ompi_datatype_t * const *rtypes,
+int ompi_coll_libnbc_neighbor_alltoallw_init(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps, struct ompi_datatype_t * const *stypes,
+                                             void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps, struct ompi_datatype_t * const *rtypes,
                                              struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                              mca_coll_base_module_t *module);
 
diff --git a/ompi/mca/coll/libnbc/coll_libnbc_component.c b/ompi/mca/coll/libnbc/coll_libnbc_component.c
index f39048ce901..4d0f6b74265 100644
--- a/ompi/mca/coll/libnbc/coll_libnbc_component.c
+++ b/ompi/mca/coll/libnbc/coll_libnbc_component.c
@@ -19,6 +19,7 @@
  * Copyright (c) 2017      Ian Bradley Morgan and Anthony Skjellum. All
  *                         rights reserved.
  * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -30,6 +31,7 @@
 
 #include "coll_libnbc.h"
 #include "nbc_internal.h"
+#include "ompi/mca/coll/base/base.h"
 
 #include "mpi.h"
 #include "ompi/mca/coll/coll.h"
@@ -106,6 +108,7 @@ static int libnbc_register(void);
 static int libnbc_init_query(bool, bool);
 static mca_coll_base_module_t *libnbc_comm_query(struct ompi_communicator_t *, int *);
 static int libnbc_module_enable(mca_coll_base_module_t *, struct ompi_communicator_t *);
+static int libnbc_module_disable(mca_coll_base_module_t *, struct ompi_communicator_t *);
 
 /*
  * Instantiate the public struct with all of our public information
@@ -117,7 +120,7 @@ ompi_coll_libnbc_component_t mca_coll_libnbc_component = {
         /* First, the mca_component_t struct containing meta information
          * about the component itself */
         .collm_version = {
-            MCA_COLL_BASE_VERSION_2_4_0,
+            MCA_COLL_BASE_VERSION_3_0_0,
 
             /* Component name and version */
             .mca_component_name = "libnbc",
@@ -200,7 +203,7 @@ libnbc_register(void)
      * As a result we are adding an MCA parameter to make a conservative
      * decision to avoid this issue. If the user knows that their application
      * does not use data types in this way, then they can set this parameter
-     * to get the old behavior. Once the issue is truely fixed, then this
+     * to get the old behavior. Once the issue is truly fixed, then this
      * parameter can be removed.
      */
     libnbc_ibcast_skip_dt_decision = true;
@@ -313,6 +316,7 @@ libnbc_comm_query(struct ompi_communicator_t *comm,
     *priority = libnbc_priority;
 
     module->super.coll_module_enable = libnbc_module_enable;
+    module->super.coll_module_disable = libnbc_module_disable;
     if (OMPI_COMM_IS_INTER(comm)) {
         module->super.coll_iallgather = ompi_coll_libnbc_iallgather_inter;
         module->super.coll_iallgatherv = ompi_coll_libnbc_iallgatherv_inter;
@@ -407,7 +411,23 @@ libnbc_comm_query(struct ompi_communicator_t *comm,
     return &(module->super);
 }
 
-
+#define NBC_INSTALL_COLL_API(__comm, __module, __api)                                                     \
+    do                                                                                                    \
+    {                                                                                                     \
+        if (__module->super.coll_##__api)                                                                 \
+        {                                                                                                 \
+            MCA_COLL_INSTALL_API(__comm, __api, __module->super.coll_##__api, &__module->super, "libnbc"); \
+        }                                                                                                 \
+    } while (0)
+
+#define NBC_UNINSTALL_COLL_API(__comm, __module, __api)                \
+    do                                                                 \
+    {                                                                  \
+        if (__comm->c_coll->coll_##__api##_module == &__module->super) \
+        {                                                              \
+            MCA_COLL_INSTALL_API(__comm, __api, NULL, NULL, "libnbc"); \
+        }                                                              \
+    } while (0)
 /*
  * Init module on the communicator
  */
@@ -415,10 +435,120 @@ static int
 libnbc_module_enable(mca_coll_base_module_t *module,
                      struct ompi_communicator_t *comm)
 {
-    /* All done */
+    ompi_coll_libnbc_module_t *nbc_module = (ompi_coll_libnbc_module_t*)module;
+
+    NBC_INSTALL_COLL_API(comm, nbc_module, iallgather);
+    NBC_INSTALL_COLL_API(comm, nbc_module, iallgatherv);
+    NBC_INSTALL_COLL_API(comm, nbc_module, iallreduce);
+    NBC_INSTALL_COLL_API(comm, nbc_module, ialltoall);
+    NBC_INSTALL_COLL_API(comm, nbc_module, ialltoallv);
+    NBC_INSTALL_COLL_API(comm, nbc_module, ialltoallw);
+    NBC_INSTALL_COLL_API(comm, nbc_module, ibarrier);
+    NBC_INSTALL_COLL_API(comm, nbc_module, ibcast);
+    NBC_INSTALL_COLL_API(comm, nbc_module, igather);
+    NBC_INSTALL_COLL_API(comm, nbc_module, igatherv);
+    NBC_INSTALL_COLL_API(comm, nbc_module, ireduce);
+    NBC_INSTALL_COLL_API(comm, nbc_module, ireduce_scatter);
+    NBC_INSTALL_COLL_API(comm, nbc_module, ireduce_scatter_block);
+    NBC_INSTALL_COLL_API(comm, nbc_module, iscatter);
+    NBC_INSTALL_COLL_API(comm, nbc_module, iscatterv);
+
+    NBC_INSTALL_COLL_API(comm, nbc_module, allgather_init);
+    NBC_INSTALL_COLL_API(comm, nbc_module, allgatherv_init);
+    NBC_INSTALL_COLL_API(comm, nbc_module, allreduce_init);
+    NBC_INSTALL_COLL_API(comm, nbc_module, alltoall_init);
+    NBC_INSTALL_COLL_API(comm, nbc_module, alltoallv_init);
+    NBC_INSTALL_COLL_API(comm, nbc_module, alltoallw_init);
+    NBC_INSTALL_COLL_API(comm, nbc_module, barrier_init);
+    NBC_INSTALL_COLL_API(comm, nbc_module, bcast_init);
+    NBC_INSTALL_COLL_API(comm, nbc_module, gather_init);
+    NBC_INSTALL_COLL_API(comm, nbc_module, gatherv_init);
+    NBC_INSTALL_COLL_API(comm, nbc_module, reduce_init);
+    NBC_INSTALL_COLL_API(comm, nbc_module, reduce_scatter_init);
+    NBC_INSTALL_COLL_API(comm, nbc_module, reduce_scatter_block_init);
+    NBC_INSTALL_COLL_API(comm, nbc_module, scatter_init);
+    NBC_INSTALL_COLL_API(comm, nbc_module, scatterv_init);
+
+    if (!OMPI_COMM_IS_INTER(comm)) {
+        NBC_INSTALL_COLL_API(comm, nbc_module, exscan_init);
+        NBC_INSTALL_COLL_API(comm, nbc_module, iexscan);
+        NBC_INSTALL_COLL_API(comm, nbc_module, scan_init);
+        NBC_INSTALL_COLL_API(comm, nbc_module, iscan);
+
+        NBC_INSTALL_COLL_API(comm, nbc_module, ineighbor_allgather);
+        NBC_INSTALL_COLL_API(comm, nbc_module, ineighbor_allgatherv);
+        NBC_INSTALL_COLL_API(comm, nbc_module, ineighbor_alltoall);
+        NBC_INSTALL_COLL_API(comm, nbc_module, ineighbor_alltoallv);
+        NBC_INSTALL_COLL_API(comm, nbc_module, ineighbor_alltoallw);
+
+        NBC_INSTALL_COLL_API(comm, nbc_module, neighbor_allgather_init);
+        NBC_INSTALL_COLL_API(comm, nbc_module, neighbor_allgatherv_init);
+        NBC_INSTALL_COLL_API(comm, nbc_module, neighbor_alltoall_init);
+        NBC_INSTALL_COLL_API(comm, nbc_module, neighbor_alltoallv_init);
+        NBC_INSTALL_COLL_API(comm, nbc_module, neighbor_alltoallw_init);
+    } /* All done */
+
     return OMPI_SUCCESS;
 }
 
+static int
+libnbc_module_disable(mca_coll_base_module_t *module,
+                      struct ompi_communicator_t *comm)
+{
+    ompi_coll_libnbc_module_t *nbc_module = (ompi_coll_libnbc_module_t*)module;
+
+    NBC_UNINSTALL_COLL_API(comm, nbc_module, iallgather);
+    NBC_UNINSTALL_COLL_API(comm, nbc_module, iallgatherv);
+    NBC_UNINSTALL_COLL_API(comm, nbc_module, iallreduce);
+    NBC_UNINSTALL_COLL_API(comm, nbc_module, ialltoall);
+    NBC_UNINSTALL_COLL_API(comm, nbc_module, ialltoallv);
+    NBC_UNINSTALL_COLL_API(comm, nbc_module, ialltoallw);
+    NBC_UNINSTALL_COLL_API(comm, nbc_module, ibarrier);
+    NBC_UNINSTALL_COLL_API(comm, nbc_module, ibcast);
+    NBC_UNINSTALL_COLL_API(comm, nbc_module, igather);
+    NBC_UNINSTALL_COLL_API(comm, nbc_module, igatherv);
+    NBC_UNINSTALL_COLL_API(comm, nbc_module, ireduce);
+    NBC_UNINSTALL_COLL_API(comm, nbc_module, ireduce_scatter);
+    NBC_UNINSTALL_COLL_API(comm, nbc_module, ireduce_scatter_block);
+    NBC_UNINSTALL_COLL_API(comm, nbc_module, iscatter);
+    NBC_UNINSTALL_COLL_API(comm, nbc_module, iscatterv);
+
+    NBC_UNINSTALL_COLL_API(comm, nbc_module, allgather_init);
+    NBC_UNINSTALL_COLL_API(comm, nbc_module, allgatherv_init);
+    NBC_UNINSTALL_COLL_API(comm, nbc_module, allreduce_init);
+    NBC_UNINSTALL_COLL_API(comm, nbc_module, alltoall_init);
+    NBC_UNINSTALL_COLL_API(comm, nbc_module, alltoallv_init);
+    NBC_UNINSTALL_COLL_API(comm, nbc_module, alltoallw_init);
+    NBC_UNINSTALL_COLL_API(comm, nbc_module, barrier_init);
+    NBC_UNINSTALL_COLL_API(comm, nbc_module, bcast_init);
+    NBC_UNINSTALL_COLL_API(comm, nbc_module, gather_init);
+    NBC_UNINSTALL_COLL_API(comm, nbc_module, gatherv_init);
+    NBC_UNINSTALL_COLL_API(comm, nbc_module, reduce_init);
+    NBC_UNINSTALL_COLL_API(comm, nbc_module, reduce_scatter_init);
+    NBC_UNINSTALL_COLL_API(comm, nbc_module, reduce_scatter_block_init);
+    NBC_UNINSTALL_COLL_API(comm, nbc_module, scatter_init);
+    NBC_UNINSTALL_COLL_API(comm, nbc_module, scatterv_init);
+
+    if (!OMPI_COMM_IS_INTER(comm)) {
+        NBC_UNINSTALL_COLL_API(comm, nbc_module, exscan_init);
+        NBC_UNINSTALL_COLL_API(comm, nbc_module, iexscan);
+        NBC_UNINSTALL_COLL_API(comm, nbc_module, scan_init);
+        NBC_UNINSTALL_COLL_API(comm, nbc_module, iscan);
+
+        NBC_UNINSTALL_COLL_API(comm, nbc_module, ineighbor_allgather);
+        NBC_UNINSTALL_COLL_API(comm, nbc_module, ineighbor_allgatherv);
+        NBC_UNINSTALL_COLL_API(comm, nbc_module, ineighbor_alltoall);
+        NBC_UNINSTALL_COLL_API(comm, nbc_module, ineighbor_alltoallv);
+        NBC_UNINSTALL_COLL_API(comm, nbc_module, ineighbor_alltoallw);
+
+        NBC_UNINSTALL_COLL_API(comm, nbc_module, neighbor_allgather_init);
+        NBC_UNINSTALL_COLL_API(comm, nbc_module, neighbor_allgatherv_init);
+        NBC_UNINSTALL_COLL_API(comm, nbc_module, neighbor_alltoall_init);
+        NBC_UNINSTALL_COLL_API(comm, nbc_module, neighbor_alltoallv_init);
+        NBC_UNINSTALL_COLL_API(comm, nbc_module, neighbor_alltoallw_init);
+    } /* All done */
+    return OMPI_SUCCESS;
+}
 
 int
 ompi_coll_libnbc_progress(void)
diff --git a/ompi/mca/coll/libnbc/libdict/dict.c b/ompi/mca/coll/libnbc/libdict/dict.c
index 64caf9c9adf..73f91b66f26 100644
--- a/ompi/mca/coll/libnbc/libdict/dict.c
+++ b/ompi/mca/coll/libnbc/libdict/dict.c
@@ -3,6 +3,7 @@
  *
  * Implementation of generic dictionary routines.
  * Copyright (C) 2001-2004 Farooq Mela.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  *
  * $Id: dict.c,v 1.7 2001/11/25 06:00:49 farooq Exp farooq $
  */
@@ -12,39 +13,12 @@
 #include "dict.h"
 #include "dict_private.h"
 
-dict_malloc_func _dict_malloc = malloc;
-dict_free_func _dict_free = free;
+dict_malloc_func ompi_coll_libnbc_dict_malloc = malloc;
+dict_free_func ompi_coll_libnbc_dict_free = free;
 
-dict_malloc_func
-dict_set_malloc(dict_malloc_func func)
-{
-	dict_malloc_func old = _dict_malloc;
-	_dict_malloc = func ? func : malloc;
-	return old;
-}
-
-dict_free_func
-dict_set_free(dict_free_func func)
-{
-	dict_free_func old = _dict_free;
-	_dict_free = func ? func : free;
-	return old;
-}
-
-/*
- * In comparing, we cannot simply subtract because that might result in signed
- * overflow.
- */
-int
-dict_int_cmp(const void *k1, const void *k2)
-{
-	const int *a = (int*)k1, *b = (int*)k2;
-
-	return (*a < *b) ? -1 : (*a > *b) ? +1 : 0;
-}
 
 int
-dict_uint_cmp(const void *k1, const void *k2)
+ompi_coll_libnbc_dict_uint_cmp(const void *k1, const void *k2)
 {
 	const unsigned int *a = (unsigned int*)k1, *b = (unsigned int*)k2;
 
@@ -52,7 +26,7 @@ dict_uint_cmp(const void *k1, const void *k2)
 }
 
 int
-dict_long_cmp(const void *k1, const void *k2)
+ompi_coll_libnbc_dict_long_cmp(const void *k1, const void *k2)
 {
 	const long *a = (long*)k1, *b = (long*)k2;
 
@@ -60,7 +34,7 @@ dict_long_cmp(const void *k1, const void *k2)
 }
 
 int
-dict_ulong_cmp(const void *k1, const void *k2)
+ompi_coll_libnbc_dict_ulong_cmp(const void *k1, const void *k2)
 {
 	const unsigned long *a = (unsigned long*)k1, *b = (unsigned long*)k2;
 
@@ -68,13 +42,13 @@ dict_ulong_cmp(const void *k1, const void *k2)
 }
 
 int
-dict_ptr_cmp(const void *k1, const void *k2)
+ompi_coll_libnbc_dict_ptr_cmp(const void *k1, const void *k2)
 {
 	return (k1 > k2) - (k1 < k2);
 }
 
 int
-dict_str_cmp(const void *k1, const void *k2)
+ompi_coll_libnbc_dict_str_cmp(const void *k1, const void *k2)
 {
 	const char *a = (char*)k1, *b = (char*)k2;
 	char p, q;
@@ -88,7 +62,7 @@ dict_str_cmp(const void *k1, const void *k2)
 }
 
 void
-dict_destroy(dict *dct, int del)
+ompi_coll_libnbc_dict_destroy(dict *dct, int del)
 {
 	ASSERT(dct != NULL);
 
@@ -97,7 +71,7 @@ dict_destroy(dict *dct, int del)
 }
 
 void
-dict_itor_destroy(dict_itor *itor)
+ompi_coll_libnbc_dict_itor_destroy(dict_itor *itor)
 {
 	ASSERT(itor != NULL);
 
diff --git a/ompi/mca/coll/libnbc/libdict/dict.h b/ompi/mca/coll/libnbc/libdict/dict.h
index 61f213ebaf8..7b667bb61dd 100644
--- a/ompi/mca/coll/libnbc/libdict/dict.h
+++ b/ompi/mca/coll/libnbc/libdict/dict.h
@@ -3,6 +3,7 @@
  *
  * Interface for generic access to dictionary library.
  * Copyright (C) 2001-2004 Farooq Mela.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  *
  * $Id: dict.h,v 1.6 2001/11/14 05:21:10 farooq Exp farooq $
  */
@@ -46,9 +47,6 @@ BEGIN_DECL
 typedef void *(*dict_malloc_func)(size_t);
 typedef void  (*dict_free_func)(void *);
 
-dict_malloc_func	dict_set_malloc		__P((dict_malloc_func func));
-dict_free_func		dict_set_free		__P((dict_free_func func));
-
 typedef int			(*dict_cmp_func)	__P((const void *, const void *));
 typedef void		(*dict_del_func)	__P((void *));
 typedef int			(*dict_vis_func)	__P((const void *, void *));
@@ -78,7 +76,6 @@ struct dict {
 #define dict_walk(dct,f)		(dct)->_walk((dct)->_object, (f))
 #define dict_count(dct)			(dct)->_count((dct)->_object)
 #define dict_empty(dct,d)		(dct)->_empty((dct)->_object, (d))
-void dict_destroy __P((dict *dct, int del));
 #define dict_itor_new(dct)		(dct)->_inew((dct)->_object)
 
 struct dict_itor {
@@ -116,14 +113,14 @@ struct dict_itor {
 #define dict_itor_cdata(i)			(i)->_cdata((i)->_itor)
 #define dict_itor_set_data(i,dat,d)	(i)->_setdata((i)->_itor, (dat), (d))
 #define dict_itor_remove(i)			(i)->_remove((i)->_itor)
-void dict_itor_destroy __P((dict_itor *itor));
-
-int		dict_int_cmp __P((const void *k1, const void *k2));
-int		dict_uint_cmp __P((const void *k1, const void *k2));
-int		dict_long_cmp __P((const void *k1, const void *k2));
-int		dict_ulong_cmp __P((const void *k1, const void *k2));
-int		dict_ptr_cmp __P((const void *k1, const void *k2));
-int		dict_str_cmp __P((const void *k1, const void *k2));
+
+int ompi_coll_libnbc_dict_ptr_cmp __P((const void *k1, const void *k2));
+void ompi_coll_libnbc_dict_destroy __P((dict *dct, int del));
+void ompi_coll_libnbc_dict_itor_destroy __P((dict_itor *itor));
+int  ompi_coll_libnbc_dict_uint_cmp __P((const void *k1, const void *k2));
+int  ompi_coll_libnbc_dict_long_cmp __P((const void *k1, const void *k2));
+int  ompi_coll_libnbc_dict_ulong_cmp __P((const void *k1, const void *k2));
+int  ompi_coll_libnbc_dict_str_cmp __P((const void *k1, const void *k2));
 
 END_DECL
 
diff --git a/ompi/mca/coll/libnbc/libdict/dict_private.h b/ompi/mca/coll/libnbc/libdict/dict_private.h
index 3635f58700b..60d714d50a4 100644
--- a/ompi/mca/coll/libnbc/libdict/dict_private.h
+++ b/ompi/mca/coll/libnbc/libdict/dict_private.h
@@ -59,10 +59,10 @@ typedef int			 (*icompare_func)	__P((void *, void *itor2));
 # define ASSERT(expr)
 #endif
 
-extern dict_malloc_func _dict_malloc;
-extern dict_free_func _dict_free;
-#define MALLOC(n)	(*_dict_malloc)(n)
-#define FREE(p)		(*_dict_free)(p)
+extern dict_malloc_func ompi_coll_libnbc_dict_malloc;
+extern dict_free_func ompi_coll_libnbc_dict_free;
+#define MALLOC(n)	(*ompi_coll_libnbc_dict_malloc)(n)
+#define FREE(p)		(*ompi_coll_libnbc_dict_free)(p)
 
 #define ABS(a)		((a) < 0 ? -(a) : +(a))
 #define MIN(a,b)	((a) < (b) ? (a) : (b))
diff --git a/ompi/mca/coll/libnbc/libdict/hb_tree.c b/ompi/mca/coll/libnbc/libdict/hb_tree.c
index 4b454c44574..5c794427332 100644
--- a/ompi/mca/coll/libnbc/libdict/hb_tree.c
+++ b/ompi/mca/coll/libnbc/libdict/hb_tree.c
@@ -3,6 +3,7 @@
  *
  * Implementation of height balanced tree.
  * Copyright (C) 2001-2004 Farooq Mela.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  *
  * $Id: hb_tree.c,v 1.10 2001/11/25 08:30:21 farooq Exp farooq $
  *
@@ -42,6 +43,7 @@ struct hb_itor {
 
 static int rot_left __P((hb_tree *tree, hb_node *node));
 static int rot_right __P((hb_tree *tree, hb_node *node));
+static int hb_itor_search __P((hb_itor *itor, const void *key));
 static unsigned node_height __P((const hb_node *node));
 static unsigned node_mheight __P((const hb_node *node));
 static unsigned node_pathlen __P((const hb_node *node, unsigned level));
@@ -52,7 +54,7 @@ static hb_node *node_next __P((hb_node *node));
 static hb_node *node_prev __P((hb_node *node));
 
 hb_tree *
-hb_tree_new(dict_cmp_func key_cmp, dict_del_func key_del,
+ompi_coll_libnbc_hb_tree_new(dict_cmp_func key_cmp, dict_del_func key_del,
 			dict_del_func dat_del)
 {
 	hb_tree *tree;
@@ -62,7 +64,7 @@ hb_tree_new(dict_cmp_func key_cmp, dict_del_func key_del,
 
 	tree->root = NULL;
 	tree->count = 0;
-	tree->key_cmp = key_cmp ? key_cmp : dict_ptr_cmp;
+	tree->key_cmp = key_cmp ? key_cmp : ompi_coll_libnbc_dict_ptr_cmp;
 	tree->key_del = key_del;
 	tree->dat_del = dat_del;
 
@@ -70,7 +72,7 @@ hb_tree_new(dict_cmp_func key_cmp, dict_del_func key_del,
 }
 
 dict *
-hb_dict_new(dict_cmp_func key_cmp, dict_del_func key_del,
+ompi_coll_libnbc_hb_dict_new(dict_cmp_func key_cmp, dict_del_func key_del,
 			dict_del_func dat_del)
 {
 	dict *dct;
@@ -79,38 +81,38 @@ hb_dict_new(dict_cmp_func key_cmp, dict_del_func key_del,
 	if ((dct = (dict*)MALLOC(sizeof(*dct))) == NULL)
 		return NULL;
 
-	if ((tree = hb_tree_new(key_cmp, key_del, dat_del)) == NULL) {
+	if ((tree = ompi_coll_libnbc_hb_tree_new(key_cmp, key_del, dat_del)) == NULL) {
 		FREE(dct);
 		return NULL;
 	}
 
 	dct->_object = tree;
-	dct->_inew = (inew_func)hb_dict_itor_new;
-	dct->_destroy = (destroy_func)hb_tree_destroy;
-	dct->_insert = (insert_func)hb_tree_insert;
-	dct->_probe = (probe_func)hb_tree_probe;
-	dct->_search = (search_func)hb_tree_search;
-	dct->_remove = (remove_func)hb_tree_remove;
-	dct->_empty = (empty_func)hb_tree_empty;
-	dct->_walk = (walk_func)hb_tree_walk;
-	dct->_count = (count_func)hb_tree_count;
+	dct->_inew = (inew_func)ompi_coll_libnbc_hb_dict_itor_new;
+	dct->_destroy = (destroy_func)ompi_coll_libnbc_hb_tree_destroy;
+	dct->_insert = (insert_func)ompi_coll_libnbc_hb_tree_insert;
+	dct->_probe = (probe_func)ompi_coll_libnbc_hb_tree_probe;
+	dct->_search = (search_func)ompi_coll_libnbc_hb_tree_search;
+	dct->_remove = (remove_func)ompi_coll_libnbc_hb_tree_remove;
+	dct->_empty = (empty_func)ompi_coll_libnbc_hb_tree_empty;
+	dct->_walk = (walk_func)ompi_coll_libnbc_hb_tree_walk;
+	dct->_count = (count_func)ompi_coll_libnbc_hb_tree_count;
 
 	return dct;
 }
 
 void
-hb_tree_destroy(hb_tree *tree, int del)
+ompi_coll_libnbc_hb_tree_destroy(hb_tree *tree, int del)
 {
 	ASSERT(tree != NULL);
 
 	if (tree->root)
-		hb_tree_empty(tree, del);
+		ompi_coll_libnbc_hb_tree_empty(tree, del);
 
 	FREE(tree);
 }
 
 void
-hb_tree_empty(hb_tree *tree, int del)
+ompi_coll_libnbc_hb_tree_empty(hb_tree *tree, int del)
 {
 	hb_node *node, *parent;
 
@@ -148,7 +150,7 @@ hb_tree_empty(hb_tree *tree, int del)
 }
 
 void *
-hb_tree_search(hb_tree *tree, const void *key)
+ompi_coll_libnbc_hb_tree_search(hb_tree *tree, const void *key)
 {
 	int rv;
 	hb_node *node;
@@ -170,7 +172,7 @@ hb_tree_search(hb_tree *tree, const void *key)
 }
 
 int
-hb_tree_insert(hb_tree *tree, void *key, void *dat, int overwrite)
+ompi_coll_libnbc_hb_tree_insert(hb_tree *tree, void *key, void *dat, int overwrite)
 {
 	int rv = 0;
 	hb_node *node, *parent = NULL, *q = NULL;
@@ -237,7 +239,7 @@ hb_tree_insert(hb_tree *tree, void *key, void *dat, int overwrite)
 }
 
 int
-hb_tree_probe(hb_tree *tree, void *key, void **dat)
+ompi_coll_libnbc_hb_tree_probe(hb_tree *tree, void *key, void **dat)
 {
 	int rv = 0;
 	hb_node *node, *parent = NULL, *q = NULL;
@@ -306,7 +308,7 @@ hb_tree_probe(hb_tree *tree, void *key, void **dat)
 	FREE(n)
 
 int
-hb_tree_remove(hb_tree *tree, const void *key, int del)
+ompi_coll_libnbc_hb_tree_remove(hb_tree *tree, const void *key, int del)
 {
 	int rv, left;
 	hb_node *node, *out, *parent = NULL;
@@ -403,7 +405,7 @@ hb_tree_remove(hb_tree *tree, const void *key, int del)
 }
 
 const void *
-hb_tree_min(const hb_tree *tree)
+ompi_coll_libnbc_hb_tree_min(const hb_tree *tree)
 {
 	const hb_node *node;
 
@@ -418,7 +420,7 @@ hb_tree_min(const hb_tree *tree)
 }
 
 const void *
-hb_tree_max(const hb_tree *tree)
+ompi_coll_libnbc_hb_tree_max(const hb_tree *tree)
 {
 	const hb_node *node;
 
@@ -433,7 +435,7 @@ hb_tree_max(const hb_tree *tree)
 }
 
 void
-hb_tree_walk(hb_tree *tree, dict_vis_func visit)
+ompi_coll_libnbc_hb_tree_walk(hb_tree *tree, dict_vis_func visit)
 {
 	hb_node *node;
 
@@ -447,7 +449,7 @@ hb_tree_walk(hb_tree *tree, dict_vis_func visit)
 }
 
 unsigned
-hb_tree_count(const hb_tree *tree)
+ompi_coll_libnbc_hb_tree_count(const hb_tree *tree)
 {
 	ASSERT(tree != NULL);
 
@@ -455,7 +457,7 @@ hb_tree_count(const hb_tree *tree)
 }
 
 unsigned
-hb_tree_height(const hb_tree *tree)
+ompi_coll_libnbc_hb_tree_height(const hb_tree *tree)
 {
 	ASSERT(tree != NULL);
 
@@ -463,7 +465,7 @@ hb_tree_height(const hb_tree *tree)
 }
 
 unsigned
-hb_tree_mheight(const hb_tree *tree)
+ompi_coll_libnbc_hb_tree_mheight(const hb_tree *tree)
 {
 	ASSERT(tree != NULL);
 
@@ -471,7 +473,7 @@ hb_tree_mheight(const hb_tree *tree)
 }
 
 unsigned
-hb_tree_pathlen(const hb_tree *tree)
+ompi_coll_libnbc_hb_tree_pathlen(const hb_tree *tree)
 {
 	ASSERT(tree != NULL);
 
@@ -683,7 +685,7 @@ rot_right(hb_tree *tree, hb_node *node)
 }
 
 hb_itor *
-hb_itor_new(hb_tree *tree)
+ompi_coll_libnbc_hb_itor_new(hb_tree *tree)
 {
 	hb_itor *itor;
 
@@ -693,12 +695,12 @@ hb_itor_new(hb_tree *tree)
 		return NULL;
 
 	itor->tree = tree;
-	hb_itor_first(itor);
+	ompi_coll_libnbc_hb_itor_first(itor);
 	return itor;
 }
 
 dict_itor *
-hb_dict_itor_new(hb_tree *tree)
+ompi_coll_libnbc_hb_dict_itor_new(hb_tree *tree)
 {
 	dict_itor *itor;
 
@@ -707,31 +709,31 @@ hb_dict_itor_new(hb_tree *tree)
 	if ((itor = (dict_itor*)MALLOC(sizeof(*itor))) == NULL)
 		return NULL;
 
-	if ((itor->_itor = hb_itor_new(tree)) == NULL) {
+	if ((itor->_itor = ompi_coll_libnbc_hb_itor_new(tree)) == NULL) {
 		FREE(itor);
 		return NULL;
 	}
 
-	itor->_destroy = (idestroy_func)hb_itor_destroy;
-	itor->_valid = (valid_func)hb_itor_valid;
-	itor->_invalid = (invalidate_func)hb_itor_invalidate;
-	itor->_next = (next_func)hb_itor_next;
-	itor->_prev = (prev_func)hb_itor_prev;
-	itor->_nextn = (nextn_func)hb_itor_nextn;
-	itor->_prevn = (prevn_func)hb_itor_prevn;
-	itor->_first = (first_func)hb_itor_first;
-	itor->_last = (last_func)hb_itor_last;
+	itor->_destroy = (idestroy_func)ompi_coll_libnbc_hb_itor_destroy;
+	itor->_valid = (valid_func)ompi_coll_libnbc_hb_itor_valid;
+	itor->_invalid = (invalidate_func)ompi_coll_libnbc_hb_itor_invalidate;
+	itor->_next = (next_func)ompi_coll_libnbc_hb_itor_next;
+	itor->_prev = (prev_func)ompi_coll_libnbc_hb_itor_prev;
+	itor->_nextn = (nextn_func)ompi_coll_libnbc_hb_itor_nextn;
+	itor->_prevn = (prevn_func)ompi_coll_libnbc_hb_itor_prevn;
+	itor->_first = (first_func)ompi_coll_libnbc_hb_itor_first;
+	itor->_last = (last_func)ompi_coll_libnbc_hb_itor_last;
 	itor->_search = (isearch_func)hb_itor_search;
-	itor->_key = (key_func)hb_itor_key;
-	itor->_data = (data_func)hb_itor_data;
-	itor->_cdata = (cdata_func)hb_itor_cdata;
-	itor->_setdata = (dataset_func)hb_itor_set_data;
+	itor->_key = (key_func)ompi_coll_libnbc_hb_itor_key;
+	itor->_data = (data_func)ompi_coll_libnbc_hb_itor_data;
+	itor->_cdata = (cdata_func)ompi_coll_libnbc_hb_itor_cdata;
+	itor->_setdata = (dataset_func)ompi_coll_libnbc_hb_itor_set_data;
 
 	return itor;
 }
 
 void
-hb_itor_destroy(hb_itor *itor)
+ompi_coll_libnbc_hb_itor_destroy(hb_itor *itor)
 {
 	ASSERT(itor != NULL);
 
@@ -741,7 +743,7 @@ hb_itor_destroy(hb_itor *itor)
 #define RETVALID(itor)		return itor->node != NULL
 
 int
-hb_itor_valid(const hb_itor *itor)
+ompi_coll_libnbc_hb_itor_valid(const hb_itor *itor)
 {
 	ASSERT(itor != NULL);
 
@@ -749,7 +751,7 @@ hb_itor_valid(const hb_itor *itor)
 }
 
 void
-hb_itor_invalidate(hb_itor *itor)
+ompi_coll_libnbc_hb_itor_invalidate(hb_itor *itor)
 {
 	ASSERT(itor != NULL);
 
@@ -757,37 +759,37 @@ hb_itor_invalidate(hb_itor *itor)
 }
 
 int
-hb_itor_next(hb_itor *itor)
+ompi_coll_libnbc_hb_itor_next(hb_itor *itor)
 {
 	ASSERT(itor != NULL);
 
 	if (itor->node == NULL)
-		hb_itor_first(itor);
+		ompi_coll_libnbc_hb_itor_first(itor);
 	else
 		itor->node = node_next(itor->node);
 	RETVALID(itor);
 }
 
 int
-hb_itor_prev(hb_itor *itor)
+ompi_coll_libnbc_hb_itor_prev(hb_itor *itor)
 {
 	ASSERT(itor != NULL);
 
 	if (itor->node == NULL)
-		hb_itor_last(itor);
+		ompi_coll_libnbc_hb_itor_last(itor);
 	else
 		itor->node = node_prev(itor->node);
 	RETVALID(itor);
 }
 
 int
-hb_itor_nextn(hb_itor *itor, unsigned count)
+ompi_coll_libnbc_hb_itor_nextn(hb_itor *itor, unsigned count)
 {
 	ASSERT(itor != NULL);
 
 	if (count) {
 		if (itor->node == NULL) {
-			hb_itor_first(itor);
+			ompi_coll_libnbc_hb_itor_first(itor);
 			count--;
 		}
 
@@ -799,13 +801,13 @@ hb_itor_nextn(hb_itor *itor, unsigned count)
 }
 
 int
-hb_itor_prevn(hb_itor *itor, unsigned count)
+ompi_coll_libnbc_hb_itor_prevn(hb_itor *itor, unsigned count)
 {
 	ASSERT(itor != NULL);
 
 	if (count) {
 		if (itor->node == NULL) {
-			hb_itor_last(itor);
+			ompi_coll_libnbc_hb_itor_last(itor);
 			count--;
 		}
 
@@ -817,7 +819,7 @@ hb_itor_prevn(hb_itor *itor, unsigned count)
 }
 
 int
-hb_itor_first(hb_itor *itor)
+ompi_coll_libnbc_hb_itor_first(hb_itor *itor)
 {
 	hb_tree *t;
 
@@ -829,7 +831,7 @@ hb_itor_first(hb_itor *itor)
 }
 
 int
-hb_itor_last(hb_itor *itor)
+ompi_coll_libnbc_hb_itor_last(hb_itor *itor)
 {
 	hb_tree *t;
 
@@ -840,7 +842,7 @@ hb_itor_last(hb_itor *itor)
 	RETVALID(itor);
 }
 
-int
+static int
 hb_itor_search(hb_itor *itor, const void *key)
 {
 	int rv;
@@ -861,7 +863,7 @@ hb_itor_search(hb_itor *itor, const void *key)
 }
 
 const void *
-hb_itor_key(const hb_itor *itor)
+ompi_coll_libnbc_hb_itor_key(const hb_itor *itor)
 {
 	ASSERT(itor != NULL);
 
@@ -869,7 +871,7 @@ hb_itor_key(const hb_itor *itor)
 }
 
 void *
-hb_itor_data(hb_itor *itor)
+ompi_coll_libnbc_hb_itor_data(hb_itor *itor)
 {
 	ASSERT(itor != NULL);
 
@@ -877,7 +879,7 @@ hb_itor_data(hb_itor *itor)
 }
 
 const void *
-hb_itor_cdata(const hb_itor *itor)
+ompi_coll_libnbc_hb_itor_cdata(const hb_itor *itor)
 {
 	ASSERT(itor != NULL);
 
@@ -885,7 +887,7 @@ hb_itor_cdata(const hb_itor *itor)
 }
 
 int
-hb_itor_set_data(hb_itor *itor, void *dat, int del)
+ompi_coll_libnbc_hb_itor_set_data(hb_itor *itor, void *dat, int del)
 {
 	ASSERT(itor != NULL);
 
diff --git a/ompi/mca/coll/libnbc/libdict/hb_tree.h b/ompi/mca/coll/libnbc/libdict/hb_tree.h
index 8be94e14538..c9c3689a14c 100644
--- a/ompi/mca/coll/libnbc/libdict/hb_tree.h
+++ b/ompi/mca/coll/libnbc/libdict/hb_tree.h
@@ -3,6 +3,7 @@
  *
  * Interface for height balanced tree.
  * Copyright (C) 2001 Farooq Mela.
+ * Copyright (c) 2022 IBM Corporation.  All rights reserved.
  *
  * $Id: hb_tree.h,v 1.2 2001/09/10 06:46:41 farooq Exp $
  */
@@ -17,46 +18,45 @@ BEGIN_DECL
 struct hb_tree;
 typedef struct hb_tree hb_tree;
 
-hb_tree *hb_tree_new __P((dict_cmp_func key_cmp, dict_del_func key_del,
+hb_tree *ompi_coll_libnbc_hb_tree_new __P((dict_cmp_func key_cmp, dict_del_func key_del,
 						  dict_del_func dat_del));
-dict	*hb_dict_new __P((dict_cmp_func key_cmp, dict_del_func key_del,
-						  dict_del_func dat_del));
-void	 hb_tree_destroy __P((hb_tree *tree, int del));
-
-int hb_tree_insert __P((hb_tree *tree, void *key, void *dat, int overwrite));
-int hb_tree_probe __P((hb_tree *tree, void *key, void **dat));
-void *hb_tree_search __P((hb_tree *tree, const void *key));
-int hb_tree_remove __P((hb_tree *tree, const void *key, int del));
-void hb_tree_empty __P((hb_tree *tree, int del));
-void hb_tree_walk __P((hb_tree *tree, dict_vis_func visit));
-unsigned hb_tree_count __P((const hb_tree *tree));
-unsigned hb_tree_height __P((const hb_tree *tree));
-unsigned hb_tree_mheight __P((const hb_tree *tree));
-unsigned hb_tree_pathlen __P((const hb_tree *tree));
-const void *hb_tree_min __P((const hb_tree *tree));
-const void *hb_tree_max __P((const hb_tree *tree));
+
+int ompi_coll_libnbc_hb_tree_insert __P((hb_tree *tree, void *key, void *dat, int overwrite));
+void *ompi_coll_libnbc_hb_tree_search __P((hb_tree *tree, const void *key));
+int ompi_coll_libnbc_hb_tree_remove __P((hb_tree *tree, const void *key, int del));
 
 struct hb_itor;
 typedef struct hb_itor hb_itor;
 
-hb_itor *hb_itor_new __P((hb_tree *tree));
-dict_itor *hb_dict_itor_new __P((hb_tree *tree));
-void hb_itor_destroy __P((hb_itor *tree));
-
-int hb_itor_valid __P((const hb_itor *itor));
-void hb_itor_invalidate __P((hb_itor *itor));
-int hb_itor_next __P((hb_itor *itor));
-int hb_itor_prev __P((hb_itor *itor));
-int hb_itor_nextn __P((hb_itor *itor, unsigned count));
-int hb_itor_prevn __P((hb_itor *itor, unsigned count));
-int hb_itor_first __P((hb_itor *itor));
-int hb_itor_last __P((hb_itor *itor));
-int hb_itor_search __P((hb_itor *itor, const void *key));
-const void *hb_itor_key __P((const hb_itor *itor));
-void *hb_itor_data __P((hb_itor *itor));
-const void *hb_itor_cdata __P((const hb_itor *itor));
-int hb_itor_set_data __P((hb_itor *itor, void *dat, int del));
-int hb_itor_remove __P((hb_itor *itor, int del));
+hb_itor *ompi_coll_libnbc_hb_itor_new __P((hb_tree *tree));
+void ompi_coll_libnbc_hb_itor_destroy __P((hb_itor *tree));
+
+int ompi_coll_libnbc_hb_itor_valid __P((const hb_itor *itor));
+int ompi_coll_libnbc_hb_itor_next __P((hb_itor *itor));
+const void *ompi_coll_libnbc_hb_itor_key __P((const hb_itor *itor));
+int ompi_coll_libnbc_hb_itor_remove __P((hb_itor *itor, int del));
+dict *ompi_coll_libnbc_hb_dict_new __P((dict_cmp_func key_cmp, dict_del_func key_del,
+                                                  dict_del_func dat_del));
+dict_itor *ompi_coll_libnbc_hb_dict_itor_new __P((hb_tree *tree));
+void ompi_coll_libnbc_hb_itor_invalidate __P((hb_itor *itor));
+int ompi_coll_libnbc_hb_itor_first __P((hb_itor *itor));
+void *ompi_coll_libnbc_hb_itor_data __P((hb_itor *itor));
+const void *ompi_coll_libnbc_hb_itor_cdata __P((const hb_itor *itor));
+int ompi_coll_libnbc_hb_itor_last __P((hb_itor *itor));
+int ompi_coll_libnbc_hb_itor_nextn __P((hb_itor *itor, unsigned count));
+int ompi_coll_libnbc_hb_itor_prev __P((hb_itor *itor));
+int ompi_coll_libnbc_hb_itor_prevn __P((hb_itor *itor, unsigned count));
+int ompi_coll_libnbc_hb_itor_set_data __P((hb_itor *itor, void *dat, int del));
+unsigned ompi_coll_libnbc_hb_tree_count __P((const hb_tree *tree));
+void ompi_coll_libnbc_hb_tree_destroy __P((hb_tree *tree, int del));
+void ompi_coll_libnbc_hb_tree_empty __P((hb_tree *tree, int del));
+unsigned ompi_coll_libnbc_hb_tree_height __P((const hb_tree *tree));
+const void *ompi_coll_libnbc_hb_tree_max __P((const hb_tree *tree));
+unsigned ompi_coll_libnbc_hb_tree_mheight __P((const hb_tree *tree));
+const void *ompi_coll_libnbc_hb_tree_min __P((const hb_tree *tree));
+unsigned ompi_coll_libnbc_hb_tree_pathlen __P((const hb_tree *tree));
+int ompi_coll_libnbc_hb_tree_probe __P((hb_tree *tree, void *key, void **dat));
+void ompi_coll_libnbc_hb_tree_walk __P((hb_tree *tree, dict_vis_func visit));
 
 END_DECL
 
diff --git a/ompi/mca/coll/libnbc/nbc.c b/ompi/mca/coll/libnbc/nbc.c
index 2054ecf7735..1c380c682de 100644
--- a/ompi/mca/coll/libnbc/nbc.c
+++ b/ompi/mca/coll/libnbc/nbc.c
@@ -16,7 +16,7 @@
  * Author(s): Torsten Hoefler <htor@cs.indiana.edu>
  *
  * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
- * Copyright (c) 2016      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2016-2021 IBM Corporation.  All rights reserved.
  * Copyright (c) 2017      Ian Bradley Morgan and Anthony Skjellum. All
  *                         rights reserved.
  * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
@@ -119,7 +119,7 @@ static int nbc_schedule_round_append (NBC_Schedule *schedule, void *data, int da
 }
 
 /* this function puts a send into the schedule */
-static int NBC_Sched_send_internal (const void* buf, char tmpbuf, int count, MPI_Datatype datatype, int dest, bool local, NBC_Schedule *schedule, bool barrier) {
+static int NBC_Sched_send_internal (const void* buf, char tmpbuf, size_t count, MPI_Datatype datatype, int dest, bool local, NBC_Schedule *schedule, bool barrier) {
   NBC_Args_send send_args;
   int ret;
 
@@ -143,16 +143,16 @@ static int NBC_Sched_send_internal (const void* buf, char tmpbuf, int count, MPI
   return OMPI_SUCCESS;
 }
 
-int NBC_Sched_send (const void* buf, char tmpbuf, int count, MPI_Datatype datatype, int dest, NBC_Schedule *schedule, bool barrier) {
+int NBC_Sched_send (const void* buf, char tmpbuf, size_t count, MPI_Datatype datatype, int dest, NBC_Schedule *schedule, bool barrier) {
   return NBC_Sched_send_internal (buf, tmpbuf, count, datatype, dest, false, schedule, barrier);
 }
 
-int NBC_Sched_local_send (const void* buf, char tmpbuf, int count, MPI_Datatype datatype, int dest, NBC_Schedule *schedule, bool barrier) {
+int NBC_Sched_local_send (const void* buf, char tmpbuf, size_t count, MPI_Datatype datatype, int dest, NBC_Schedule *schedule, bool barrier) {
   return NBC_Sched_send_internal (buf, tmpbuf, count, datatype, dest, true, schedule, barrier);
 }
 
 /* this function puts a receive into the schedule */
-static int NBC_Sched_recv_internal (void* buf, char tmpbuf, int count, MPI_Datatype datatype, int source, bool local, NBC_Schedule *schedule, bool barrier) {
+static int NBC_Sched_recv_internal (void* buf, char tmpbuf, size_t count, MPI_Datatype datatype, int source, bool local, NBC_Schedule *schedule, bool barrier) {
   NBC_Args_recv recv_args;
   int ret;
 
@@ -176,16 +176,16 @@ static int NBC_Sched_recv_internal (void* buf, char tmpbuf, int count, MPI_Datat
   return OMPI_SUCCESS;
 }
 
-int NBC_Sched_recv (void* buf, char tmpbuf, int count, MPI_Datatype datatype, int source, NBC_Schedule *schedule, bool barrier) {
+int NBC_Sched_recv (void* buf, char tmpbuf, size_t count, MPI_Datatype datatype, int source, NBC_Schedule *schedule, bool barrier) {
   return NBC_Sched_recv_internal(buf, tmpbuf, count, datatype, source, false, schedule, barrier);
 }
 
-int NBC_Sched_local_recv (void* buf, char tmpbuf, int count, MPI_Datatype datatype, int source, NBC_Schedule *schedule, bool barrier) {
+int NBC_Sched_local_recv (void* buf, char tmpbuf, size_t count, MPI_Datatype datatype, int source, NBC_Schedule *schedule, bool barrier) {
   return NBC_Sched_recv_internal(buf, tmpbuf, count, datatype, source, true, schedule, barrier);
 }
 
 /* this function puts an operation into the schedule */
-int NBC_Sched_op (const void* buf1, char tmpbuf1, void* buf2, char tmpbuf2, int count, MPI_Datatype datatype,
+int NBC_Sched_op (const void* buf1, char tmpbuf1, void* buf2, char tmpbuf2, size_t count, MPI_Datatype datatype,
                   MPI_Op op, NBC_Schedule *schedule, bool barrier) {
   NBC_Args_op op_args;
   int ret;
@@ -212,7 +212,8 @@ int NBC_Sched_op (const void* buf1, char tmpbuf1, void* buf2, char tmpbuf2, int
 }
 
 /* this function puts a copy into the schedule */
-int NBC_Sched_copy (void *src, char tmpsrc, int srccount, MPI_Datatype srctype, void *tgt, char tmptgt, int tgtcount,
+int NBC_Sched_copy (void *src, char tmpsrc, size_t srccount, MPI_Datatype srctype,
+                    void *tgt, char tmptgt, size_t tgtcount,
                     MPI_Datatype tgttype, NBC_Schedule *schedule, bool barrier) {
   NBC_Args_copy copy_args;
   int ret;
@@ -240,7 +241,7 @@ int NBC_Sched_copy (void *src, char tmpsrc, int srccount, MPI_Datatype srctype,
 }
 
 /* this function puts a unpack into the schedule */
-int NBC_Sched_unpack (void *inbuf, char tmpinbuf, int count, MPI_Datatype datatype, void *outbuf, char tmpoutbuf,
+int NBC_Sched_unpack (void *inbuf, char tmpinbuf, size_t count, MPI_Datatype datatype, void *outbuf, char tmpoutbuf,
                       NBC_Schedule *schedule, bool barrier) {
   NBC_Args_unpack unpack_args;
   int ret;
@@ -534,6 +535,7 @@ static inline int NBC_Start_round(NBC_Handle *handle) {
         } else {
           buf2=opargs.buf2;
         }
+
         ompi_op_reduce(opargs.op, buf1, buf2, opargs.count, opargs.datatype);
         break;
       case COPY:
diff --git a/ompi/mca/coll/libnbc/nbc_iallgather.c b/ompi/mca/coll/libnbc/nbc_iallgather.c
index ffed75958bb..3a585749074 100644
--- a/ompi/mca/coll/libnbc/nbc_iallgather.c
+++ b/ompi/mca/coll/libnbc/nbc_iallgather.c
@@ -22,11 +22,11 @@
 
 static inline int allgather_sched_linear(
     int rank, int comm_size, NBC_Schedule *schedule, const void *sendbuf,
-    int scount, struct ompi_datatype_t *sdtype, void *recvbuf, int rcount,
+    size_t scount, struct ompi_datatype_t *sdtype, void *recvbuf, size_t rcount,
     struct ompi_datatype_t *rdtype);
 static inline int allgather_sched_recursivedoubling(
     int rank, int comm_size, NBC_Schedule *schedule, const void *sbuf,
-    int scount, struct ompi_datatype_t *sdtype, void *rbuf, int rcount,
+    size_t scount, struct ompi_datatype_t *sdtype, void *rbuf, size_t rcount,
     struct ompi_datatype_t *rdtype);
 
 #ifdef NBC_CACHE_SCHEDULE
@@ -49,7 +49,7 @@ int NBC_Allgather_args_compare(NBC_Allgather_args *a, NBC_Allgather_args *b, voi
 }
 #endif
 
-static int nbc_allgather_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+static int nbc_allgather_init(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf, size_t recvcount,
                               MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
                               mca_coll_base_module_t *module, bool persistent)
 {
@@ -92,7 +92,7 @@ static int nbc_allgather_init(const void* sendbuf, int sendcount, MPI_Datatype s
     sendcount = recvcount;
   } else if (!persistent) { /* for persistent, the copy must be scheduled */
     /* copy my data to receive buffer */
-    rbuf = (char *) recvbuf + rank * recvcount * rcvext;
+    rbuf = (char *) recvbuf + (MPI_Aint)rcvext * rank * recvcount;
     res = NBC_Copy (sendbuf, sendcount, sendtype, rbuf, recvcount, recvtype, comm);
     if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
       return res;
@@ -121,7 +121,7 @@ static int nbc_allgather_init(const void* sendbuf, int sendcount, MPI_Datatype s
     if (persistent && !inplace) {
       /* for nonblocking, data has been copied already */
       /* copy my data to receive buffer (= send buffer of NBC_Sched_send) */
-      rbuf = (char *)recvbuf + rank * recvcount * rcvext;
+      rbuf = (char *)recvbuf + (MPI_Aint) rcvext * rank * recvcount;
       res = NBC_Sched_copy((void *)sendbuf, false, sendcount, sendtype,
                             rbuf, false, recvcount, recvtype, schedule, true);
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
@@ -190,7 +190,7 @@ static int nbc_allgather_init(const void* sendbuf, int sendcount, MPI_Datatype s
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_iallgather(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+int ompi_coll_libnbc_iallgather(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf, size_t recvcount,
                                 MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                 mca_coll_base_module_t *module)
 {
@@ -210,7 +210,7 @@ int ompi_coll_libnbc_iallgather(const void* sendbuf, int sendcount, MPI_Datatype
     return OMPI_SUCCESS;
 }
 
-static int nbc_allgather_inter_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+static int nbc_allgather_inter_init(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf, size_t recvcount,
                                     MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                     mca_coll_base_module_t *module, bool persistent)
 {
@@ -237,7 +237,7 @@ static int nbc_allgather_inter_init(const void* sendbuf, int sendcount, MPI_Data
   /* do rsize - 1 rounds */
   for (int r = 0 ; r < rsize ; ++r) {
     /* recv from rank r */
-    rbuf = (char *) recvbuf + r * recvcount * rcvext;
+    rbuf = (char *) recvbuf + (MPI_Aint) rcvext * r * recvcount;
     res = NBC_Sched_recv (rbuf, false, recvcount, recvtype, r, schedule, false);
     if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
       OBJ_RELEASE(schedule);
@@ -267,7 +267,7 @@ static int nbc_allgather_inter_init(const void* sendbuf, int sendcount, MPI_Data
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_iallgather_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+int ompi_coll_libnbc_iallgather_inter(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf, size_t recvcount,
 				      MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
 				      mca_coll_base_module_t *module) {
     int res = nbc_allgather_inter_init(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype,
@@ -296,19 +296,19 @@ int ompi_coll_libnbc_iallgather_inter(const void* sendbuf, int sendcount, MPI_Da
  */
 static inline int allgather_sched_linear(
     int rank, int comm_size, NBC_Schedule *schedule, const void *sendbuf,
-    int scount, struct ompi_datatype_t *sdtype, void *recvbuf, int rcount,
+    size_t scount, struct ompi_datatype_t *sdtype, void *recvbuf, size_t rcount,
     struct ompi_datatype_t *rdtype)
 {
     int res = OMPI_SUCCESS;
     ptrdiff_t rlb, rext;
 
     res = ompi_datatype_get_extent(rdtype, &rlb, &rext);
-    char *sbuf = (char *)recvbuf + rank * rcount * rext;
+    char *sbuf = (char *)recvbuf + (MPI_Aint) rext * rank * rcount;
 
     for (int remote = 0; remote < comm_size ; ++remote) {
         if (remote != rank) {
             /* Recv from rank remote */
-            char *rbuf = (char *)recvbuf + remote * rcount * rext;
+            char *rbuf = (char *)recvbuf + (MPI_Aint) rext * remote * rcount;
             res = NBC_Sched_recv(rbuf, false, rcount, rdtype, remote, schedule, false);
             if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { goto cleanup_and_return; }
 
@@ -354,7 +354,7 @@ static inline int allgather_sched_linear(
  */
 static inline int allgather_sched_recursivedoubling(
     int rank, int comm_size, NBC_Schedule *schedule, const void *sbuf,
-    int scount, struct ompi_datatype_t *sdtype, void *rbuf, int rcount,
+    size_t scount, struct ompi_datatype_t *sdtype, void *rbuf, size_t rcount,
     struct ompi_datatype_t *rdtype)
 {
     int res = OMPI_SUCCESS;
@@ -389,7 +389,7 @@ static inline int allgather_sched_recursivedoubling(
     return res;
 }
 
-int ompi_coll_libnbc_allgather_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+int ompi_coll_libnbc_allgather_init(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf, size_t recvcount,
                                     MPI_Datatype recvtype, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                     mca_coll_base_module_t *module) {
     int res = nbc_allgather_init(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype,
@@ -401,7 +401,7 @@ int ompi_coll_libnbc_allgather_init(const void* sendbuf, int sendcount, MPI_Data
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_allgather_inter_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+int ompi_coll_libnbc_allgather_inter_init(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf, size_t recvcount,
                                           MPI_Datatype recvtype, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                           mca_coll_base_module_t *module) {
     int res = nbc_allgather_inter_init(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype,
diff --git a/ompi/mca/coll/libnbc/nbc_iallgatherv.c b/ompi/mca/coll/libnbc/nbc_iallgatherv.c
index 91fa7df3981..4bb42a5d3a8 100644
--- a/ompi/mca/coll/libnbc/nbc_iallgatherv.c
+++ b/ompi/mca/coll/libnbc/nbc_iallgatherv.c
@@ -23,7 +23,7 @@
 #include "nbc_internal.h"
 
 /* an allgatherv schedule can not be cached easily because the contents
- * ot the recvcounts array may change, so a comparison of the address
+ * of the recvcounts array may change, so a comparison of the address
  * would not be sufficient ... we simply do not cache it */
 
 /* simple linear MPI_Iallgatherv
@@ -34,7 +34,8 @@
  * second round:
  *   each node sends to node (rank+2)%p sendcount elements
  *   each node receives from node (rank-2)%p recvcounts[(rank+2)%p] elements */
-static int nbc_allgatherv_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs,
+static int nbc_allgatherv_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf,
+                               ompi_count_array_t recvcounts, ompi_disp_array_t displs,
                                MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                mca_coll_base_module_t *module, bool persistent)
 {
@@ -57,11 +58,11 @@ static int nbc_allgatherv_init(const void* sendbuf, int sendcount, MPI_Datatype
 
   if (inplace) {
       sendtype = recvtype;
-      sendcount = recvcounts[rank];
+      sendcount = ompi_count_array_get(recvcounts, rank);
   } else if (!persistent) { /* for persistent, the copy must be scheduled */
     /* copy my data to receive buffer */
-    rbuf = (char *) recvbuf + displs[rank] * rcvext;
-    res = NBC_Copy (sendbuf, sendcount, sendtype, rbuf, recvcounts[rank], recvtype, comm);
+    rbuf = (char *) recvbuf + ompi_disp_array_get(displs, rank) * rcvext;
+    res = NBC_Copy (sendbuf, sendcount, sendtype, rbuf, ompi_count_array_get(recvcounts, rank), recvtype, comm);
     if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
       return res;
     }
@@ -72,12 +73,12 @@ static int nbc_allgatherv_init(const void* sendbuf, int sendcount, MPI_Datatype
     return OMPI_ERR_OUT_OF_RESOURCE;
   }
 
-  sbuf = (char *) recvbuf + displs[rank] * rcvext;
+  sbuf = (char *) recvbuf + ompi_disp_array_get(displs, rank) * rcvext;
 
   if (persistent && !inplace) { /* for nonblocking, data has been copied already */
     /* copy my data to receive buffer (= send buffer of NBC_Sched_send) */
     res = NBC_Sched_copy ((void *)sendbuf, false, sendcount, sendtype,
-                          sbuf, false, recvcounts[rank], recvtype, schedule, true);
+                          sbuf, false, ompi_count_array_get(recvcounts, rank), recvtype, schedule, true);
     if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
       OBJ_RELEASE(schedule);
       return res;
@@ -88,16 +89,16 @@ static int nbc_allgatherv_init(const void* sendbuf, int sendcount, MPI_Datatype
   for (int r = 1 ; r < p ; ++r) {
     speer = (rank + r) % p;
     rpeer = (rank - r + p) % p;
-    rbuf = (char *)recvbuf + displs[rpeer] * rcvext;
+    rbuf = (char *)recvbuf + ompi_disp_array_get(displs, rpeer) * rcvext;
 
-    res = NBC_Sched_recv (rbuf, false, recvcounts[rpeer], recvtype, rpeer, schedule, false);
+    res = NBC_Sched_recv (rbuf, false, ompi_count_array_get(recvcounts, rpeer), recvtype, rpeer, schedule, false);
     if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
       OBJ_RELEASE(schedule);
       return res;
     }
 
     /* send to rank r - not from the sendbuf to optimize MPI_IN_PLACE */
-    res = NBC_Sched_send (sbuf, false, recvcounts[rank], recvtype, speer, schedule, false);
+    res = NBC_Sched_send (sbuf, false, ompi_count_array_get(recvcounts, rank), recvtype, speer, schedule, false);
     if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
       OBJ_RELEASE(schedule);
       return res;
@@ -119,7 +120,8 @@ static int nbc_allgatherv_init(const void* sendbuf, int sendcount, MPI_Datatype
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_iallgatherv(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs,
+int ompi_coll_libnbc_iallgatherv(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf,
+                                 ompi_count_array_t recvcounts, ompi_disp_array_t displs,
                                  MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                  mca_coll_base_module_t *module) {
     int res = nbc_allgatherv_init(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype,
@@ -138,7 +140,8 @@ int ompi_coll_libnbc_iallgatherv(const void* sendbuf, int sendcount, MPI_Datatyp
     return OMPI_SUCCESS;
 }
 
-static int nbc_allgatherv_inter_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs,
+static int nbc_allgatherv_inter_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf,
+                                     ompi_count_array_t recvcounts, ompi_disp_array_t displs,
                                      MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                      mca_coll_base_module_t *module, bool persistent)
 {
@@ -162,10 +165,10 @@ static int nbc_allgatherv_inter_init(const void* sendbuf, int sendcount, MPI_Dat
 
   /* do rsize  rounds */
   for (int r = 0 ; r < rsize ; ++r) {
-    char *rbuf = (char *) recvbuf + displs[r] * rcvext;
+    char *rbuf = (char *) recvbuf + ompi_disp_array_get(displs, r) * rcvext;
 
-    if (recvcounts[r]) {
-      res = NBC_Sched_recv (rbuf, false, recvcounts[r], recvtype, r, schedule, false);
+    if (ompi_count_array_get(recvcounts, r)) {
+      res = NBC_Sched_recv (rbuf, false, ompi_count_array_get(recvcounts, r), recvtype, r, schedule, false);
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
         OBJ_RELEASE(schedule);
         return res;
@@ -198,7 +201,8 @@ static int nbc_allgatherv_inter_init(const void* sendbuf, int sendcount, MPI_Dat
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_iallgatherv_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs,
+int ompi_coll_libnbc_iallgatherv_inter(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf,
+                                       ompi_count_array_t recvcounts, ompi_disp_array_t displs,
                                        MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                        mca_coll_base_module_t *module) {
     int res = nbc_allgatherv_inter_init(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype,
@@ -217,7 +221,8 @@ int ompi_coll_libnbc_iallgatherv_inter(const void* sendbuf, int sendcount, MPI_D
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_allgatherv_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs,
+int ompi_coll_libnbc_allgatherv_init(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf,
+                                     ompi_count_array_t recvcounts, ompi_disp_array_t displs,
                                      MPI_Datatype recvtype, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                      mca_coll_base_module_t *module) {
     int res = nbc_allgatherv_init(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype,
@@ -229,7 +234,8 @@ int ompi_coll_libnbc_allgatherv_init(const void* sendbuf, int sendcount, MPI_Dat
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_allgatherv_inter_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs,
+int ompi_coll_libnbc_allgatherv_inter_init(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf,
+                                           ompi_count_array_t recvcounts, ompi_disp_array_t displs,
                                            MPI_Datatype recvtype, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                            mca_coll_base_module_t *module) {
     int res = nbc_allgatherv_inter_init(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype,
diff --git a/ompi/mca/coll/libnbc/nbc_iallreduce.c b/ompi/mca/coll/libnbc/nbc_iallreduce.c
index ff0212b49c6..24d52614b9b 100644
--- a/ompi/mca/coll/libnbc/nbc_iallreduce.c
+++ b/ompi/mca/coll/libnbc/nbc_iallreduce.c
@@ -9,7 +9,7 @@
  *                         reserved.
  * Copyright (c) 2014-2018 Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2017-2022 IBM Corporation.  All rights reserved.
  * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
  * $COPYRIGHT$
  *
@@ -26,19 +26,19 @@
 
 #include <assert.h>
 
-static inline int allred_sched_diss(int rank, int p, int count, MPI_Datatype datatype, ptrdiff_t gap, const void *sendbuf,
+static inline int allred_sched_diss(int rank, int p, size_t count, MPI_Datatype datatype, ptrdiff_t gap, const void *sendbuf,
                                     void *recvbuf, MPI_Op op, char inplace, NBC_Schedule *schedule, void *tmpbuf);
 static inline int allred_sched_recursivedoubling(int rank, int p, const void *sendbuf, void *recvbuf,
-                                                 int count, MPI_Datatype datatype, ptrdiff_t gap, MPI_Op op,
+                                                 size_t count, MPI_Datatype datatype, ptrdiff_t gap, MPI_Op op,
                                                  char inplace, NBC_Schedule *schedule, void *tmpbuf);
-static inline int allred_sched_ring(int rank, int p, int count, MPI_Datatype datatype, const void *sendbuf,
+static inline int allred_sched_ring(int rank, int p, size_t count, MPI_Datatype datatype, const void *sendbuf,
                                     void *recvbuf, MPI_Op op, int size, int ext, NBC_Schedule *schedule,
                                     void *tmpbuf);
-static inline int allred_sched_linear(int rank, int p, const void *sendbuf, void *recvbuf, int count,
+static inline int allred_sched_linear(int rank, int p, const void *sendbuf, void *recvbuf, size_t count,
                                       MPI_Datatype datatype, ptrdiff_t gap, MPI_Op op, int ext, int size,
                                       NBC_Schedule *schedule, void *tmpbuf);
 static inline int allred_sched_redscat_allgather(
-    int rank, int comm_size, int count, MPI_Datatype datatype, ptrdiff_t gap,
+    int rank, int comm_size, size_t count, MPI_Datatype datatype, ptrdiff_t gap,
     const void *sbuf, void *rbuf, MPI_Op op, char inplace,
     NBC_Schedule *schedule, void *tmpbuf, struct ompi_communicator_t *comm);
 
@@ -61,7 +61,7 @@ int NBC_Allreduce_args_compare(NBC_Allreduce_args *a, NBC_Allreduce_args *b, voi
 }
 #endif
 
-static int nbc_allreduce_init(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
+static int nbc_allreduce_init(const void* sendbuf, void* recvbuf, size_t count, MPI_Datatype datatype, MPI_Op op,
                               struct ompi_communicator_t *comm, ompi_request_t ** request,
                               mca_coll_base_module_t *module, bool persistent)
 {
@@ -112,27 +112,24 @@ static int nbc_allreduce_init(const void* sendbuf, void* recvbuf, int count, MPI
     return OMPI_ERR_OUT_OF_RESOURCE;
   }
 
+  alg = NBC_ARED_RING;  /* default generic selection */
   /* algorithm selection */
   int nprocs_pof2 = opal_next_poweroftwo(p) >> 1;
   if (libnbc_iallreduce_algorithm == 0) {
     if(p < 4 || size*count < 65536 || !ompi_op_is_commute(op) || inplace) {
       alg = NBC_ARED_BINOMIAL;
-    } else if (count >= nprocs_pof2 && ompi_op_is_commute(op)) {
+    } else if (count >= (size_t) nprocs_pof2 && ompi_op_is_commute(op)) {
       alg = NBC_ARED_REDSCAT_ALLGATHER;
-    } else {
-      alg = NBC_ARED_RING;
     }
   } else {
     if (libnbc_iallreduce_algorithm == 1)
       alg = NBC_ARED_RING;
     else if (libnbc_iallreduce_algorithm == 2)
       alg = NBC_ARED_BINOMIAL;
-    else if (libnbc_iallreduce_algorithm == 3 && count >= nprocs_pof2 && ompi_op_is_commute(op))
+    else if (libnbc_iallreduce_algorithm == 3 && count >= (size_t) nprocs_pof2 && ompi_op_is_commute(op))
       alg = NBC_ARED_REDSCAT_ALLGATHER;
     else if (libnbc_iallreduce_algorithm == 4)
       alg = NBC_ARED_RDBL;
-    else
-      alg = NBC_ARED_RING;
   }
 #ifdef NBC_CACHE_SCHEDULE
   /* search schedule in communicator specific tree */
@@ -224,7 +221,7 @@ static int nbc_allreduce_init(const void* sendbuf, void* recvbuf, int count, MPI
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_iallreduce(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
+int ompi_coll_libnbc_iallreduce(const void* sendbuf, void* recvbuf, size_t count, MPI_Datatype datatype, MPI_Op op,
                                 struct ompi_communicator_t *comm, ompi_request_t ** request,
                                 mca_coll_base_module_t *module) {
     int res = nbc_allreduce_init(sendbuf, recvbuf, count, datatype, op,
@@ -243,7 +240,7 @@ int ompi_coll_libnbc_iallreduce(const void* sendbuf, void* recvbuf, int count, M
     return OMPI_SUCCESS;
 }
 
-static int nbc_allreduce_inter_init(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
+static int nbc_allreduce_inter_init(const void* sendbuf, void* recvbuf, size_t count, MPI_Datatype datatype, MPI_Op op,
                                     struct ompi_communicator_t *comm, ompi_request_t ** request,
                                     mca_coll_base_module_t *module, bool persistent)
 {
@@ -307,7 +304,7 @@ static int nbc_allreduce_inter_init(const void* sendbuf, void* recvbuf, int coun
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_iallreduce_inter(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
+int ompi_coll_libnbc_iallreduce_inter(const void* sendbuf, void* recvbuf, size_t count, MPI_Datatype datatype, MPI_Op op,
                                       struct ompi_communicator_t *comm, ompi_request_t ** request,
                                       mca_coll_base_module_t *module) {
     int res = nbc_allreduce_inter_init(sendbuf, recvbuf, count, datatype, op,
@@ -361,7 +358,7 @@ int ompi_coll_libnbc_iallreduce_inter(const void* sendbuf, void* recvbuf, int co
   if (vrank == 0) rank = root; \
   if (vrank == root) rank = 0; \
 }
-static inline int allred_sched_diss(int rank, int p, int count, MPI_Datatype datatype, ptrdiff_t gap, const void *sendbuf, void *recvbuf,
+static inline int allred_sched_diss(int rank, int p, size_t count, MPI_Datatype datatype, ptrdiff_t gap, const void *sendbuf, void *recvbuf,
                                     MPI_Op op, char inplace, NBC_Schedule *schedule, void *tmpbuf) {
   int root, vrank, maxr, vpeer, peer, res;
   char *rbuf, *lbuf, *buf;
@@ -369,7 +366,7 @@ static inline int allred_sched_diss(int rank, int p, int count, MPI_Datatype dat
 
   root = 0; /* this makes the code for ireduce and iallreduce nearly identical - could be changed to improve performance */
   RANK2VRANK(rank, vrank, root);
-  maxr = (int)ceil((log((double)p)/LOG2));
+  maxr = ceil_of_log2(p);
   /* ensure the result ends up in recvbuf on vrank 0 */
   if (0 == (maxr%2)) {
     rbuf = (void *)(-gap);
@@ -519,7 +516,7 @@ static inline int allred_sched_diss(int rank, int p, int count, MPI_Datatype dat
  *
  */
 static inline int allred_sched_recursivedoubling(int rank, int p, const void *sendbuf, void *recvbuf,
-                                                 int count, MPI_Datatype datatype, ptrdiff_t gap, MPI_Op op,
+                                                 size_t count, MPI_Datatype datatype, ptrdiff_t gap, MPI_Op op,
                                                  char inplace, NBC_Schedule *schedule, void *tmpbuf)
 {
   int res, pof2, nprocs_rem, vrank;
@@ -633,38 +630,37 @@ static inline int allred_sched_recursivedoubling(int rank, int p, const void *se
   return OMPI_SUCCESS;
 }
 
-static inline int allred_sched_ring (int r, int p, int count, MPI_Datatype datatype, const void *sendbuf, void *recvbuf, MPI_Op op,
-                                     int size, int ext, NBC_Schedule *schedule, void *tmpbuf) {
+static inline int
+allred_sched_ring(int r, int p,
+                  size_t count, MPI_Datatype datatype, const void *sendbuf, void *recvbuf,
+                  MPI_Op op, int size, int ext, NBC_Schedule *schedule, void *tmpbuf)
+{
   int segsize, *segsizes, *segoffsets; /* segment sizes and offsets per segment (number of segments == number of nodes */
-  int speer, rpeer; /* send and recvpeer */
+  int speer, rpeer; /* send and recv peers */
   int res = OMPI_SUCCESS;
 
-  if (count == 0) {
+  if (0 == count) {
     return OMPI_SUCCESS;
   }
 
-  segsizes = (int *) malloc (sizeof (int) * p);
-  segoffsets = (int *) malloc (sizeof (int) * p);
-  if (NULL == segsizes || NULL == segoffsets) {
-    free (segsizes);
-    free (segoffsets);
+  segsizes = (int *) malloc((2 * p + 1 ) *sizeof (int));
+  if (NULL == segsizes) {
     return OMPI_ERR_OUT_OF_RESOURCE;
   }
+  segoffsets = segsizes + p;
 
-  segsize = (count + p - 1) / p; /* size of the segments */
+  segsize = count / p; /* size of the segments across the last ranks.
+                          The remainder will be evenly distributed across the smaller ranks */
 
   segoffsets[0] = 0;
-  for (int i = 0, mycount = count ; i < p ; ++i) {
-    mycount -= segsize;
+  for (int i = 0, mycount = count % p; i < p ; ++i) {
     segsizes[i] = segsize;
-    if (mycount < 0) {
-      segsizes[i] = segsize + mycount;
-      mycount = 0;
+    if( mycount > 0 ) {  /* We have extra segments to distribute */
+        segsizes[i]++;
+        mycount--;
     }
 
-    if (i) {
-      segoffsets[i] = segoffsets[i-1] + segsizes[i-1];
-    }
+    segoffsets[i+1] = segoffsets[i] + segsizes[i];
   }
 
   /* reduce peers */
@@ -772,9 +768,9 @@ static inline int allred_sched_ring (int r, int p, int count, MPI_Datatype datat
   /* first p-1 rounds are reductions */
   for (int round = 0 ; round < p - 1 ; ++round) {
     int selement = (r+1-round + 2*p /*2*p avoids negative mod*/)%p; /* the element I am sending */
-    int soffset = segoffsets[selement]*ext;
+    size_t soffset = segoffsets[selement]*(size_t)ext;
     int relement = (r-round + 2*p /*2*p avoids negative mod*/)%p; /* the element that I receive from my neighbor */
-    int roffset = segoffsets[relement]*ext;
+    size_t roffset = segoffsets[relement]*(size_t)ext;
 
     /* first message come out of sendbuf */
     if (round == 0) {
@@ -786,54 +782,53 @@ static inline int allred_sched_ring (int r, int p, int count, MPI_Datatype datat
     }
 
     if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
-      break;
+      goto free_and_return;
     }
-
-    res = NBC_Sched_recv ((char *) recvbuf + roffset, false, segsizes[relement], datatype, rpeer,
-                          schedule, true);
-    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
-      break;
+    if( recvbuf != sendbuf ) {  /* check for MPI_IN_PLACE */
+        res = NBC_Sched_recv ((char *) recvbuf + roffset, false, segsizes[relement], datatype, rpeer,
+                              schedule, true);
+        if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+          goto free_and_return;
+        }
+        res = NBC_Sched_op ((char *) sendbuf + roffset, false, (char *) recvbuf + roffset, false,
+                             segsizes[relement], datatype, op, schedule, true);
+    } else {
+        res = NBC_Sched_recv ((char *) tmpbuf, false, segsizes[relement], datatype, rpeer,
+                              schedule, true);
+        if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+          goto free_and_return;
+        }
+        res = NBC_Sched_op ((char *) tmpbuf, false, (char *) recvbuf + roffset, false,
+                             segsizes[relement], datatype, op, schedule, true);
     }
-
-    res = NBC_Sched_op ((char *) sendbuf + roffset, false, (char *) recvbuf + roffset, false,
-                         segsizes[relement], datatype, op, schedule, true);
     if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
-      break;
+      goto free_and_return;
     }
   }
-
-  if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
-    free (segsizes);
-    free (segoffsets);
-    return res;
-  }
-
   for (int round = p - 1 ; round < 2 * p - 2 ; ++round) {
     int selement = (r+1-round + 2*p /*2*p avoids negative mod*/)%p; /* the element I am sending */
-    int soffset = segoffsets[selement]*ext;
+    size_t soffset = segoffsets[selement]*(size_t)ext;
     int relement = (r-round + 2*p /*2*p avoids negative mod*/)%p; /* the element that I receive from my neighbor */
-    int roffset = segoffsets[relement]*ext;
+    size_t roffset = segoffsets[relement]*(size_t)ext;
 
     res = NBC_Sched_send ((char *) recvbuf + soffset, false, segsizes[selement], datatype, speer,
                           schedule, false);
     if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
       break;
     }
-
     res = NBC_Sched_recv ((char *) recvbuf + roffset, false, segsizes[relement], datatype, rpeer,
                           schedule, true);
     if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
       break;
     }
   }
-
+free_and_return:
   free (segsizes);
-  free (segoffsets);
 
   return res;
 }
 
-static inline int allred_sched_linear(int rank, int rsize, const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
+static inline int allred_sched_linear(int rank, int rsize, const void *sendbuf, void *recvbuf, size_t count, MPI_Datatype datatype,
 				      ptrdiff_t gap, MPI_Op op, int ext, int size, NBC_Schedule *schedule, void *tmpbuf) {
   int res;
 
@@ -974,7 +969,7 @@ static inline int allred_sched_linear(int rank, int rsize, const void *sendbuf,
  * Schedule length (rounds): O(\log(p))
  */
 static inline int allred_sched_redscat_allgather(
-    int rank, int comm_size, int count, MPI_Datatype datatype, ptrdiff_t gap,
+    int rank, int comm_size, size_t count, MPI_Datatype datatype, ptrdiff_t gap,
     const void *sbuf, void *rbuf, MPI_Op op, char inplace,
     NBC_Schedule *schedule, void *tmpbuf, struct ompi_communicator_t *comm)
 {
@@ -1084,7 +1079,7 @@ static inline int allred_sched_redscat_allgather(
         sindex[0] = rindex[0] = 0;
          for (int mask = 1; mask < nprocs_pof2; mask <<= 1) {
             /*
-             * On each iteration: rindex[step] = sindex[step] -- begining of the
+             * On each iteration: rindex[step] = sindex[step] -- beginning of the
              * current window. Length of the current window is storded in wsize.
              */
             int vdest = vrank ^ mask;
@@ -1184,7 +1179,7 @@ static inline int allred_sched_redscat_allgather(
     return res;
 }
 
-int ompi_coll_libnbc_allreduce_init(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
+int ompi_coll_libnbc_allreduce_init(const void* sendbuf, void* recvbuf, size_t count, MPI_Datatype datatype, MPI_Op op,
                                     struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                     mca_coll_base_module_t *module) {
     int res = nbc_allreduce_init(sendbuf, recvbuf, count, datatype, op,
@@ -1196,7 +1191,7 @@ int ompi_coll_libnbc_allreduce_init(const void* sendbuf, void* recvbuf, int coun
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_allreduce_inter_init(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
+int ompi_coll_libnbc_allreduce_inter_init(const void* sendbuf, void* recvbuf, size_t count, MPI_Datatype datatype, MPI_Op op,
                                           struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                           mca_coll_base_module_t *module) {
     int res = nbc_allreduce_inter_init(sendbuf, recvbuf, count, datatype, op,
diff --git a/ompi/mca/coll/libnbc/nbc_ialltoall.c b/ompi/mca/coll/libnbc/nbc_ialltoall.c
index ebed3c4c3b2..a95e6eb4786 100644
--- a/ompi/mca/coll/libnbc/nbc_ialltoall.c
+++ b/ompi/mca/coll/libnbc/nbc_ialltoall.c
@@ -22,15 +22,15 @@
 #include "nbc_internal.h"
 
 static inline int a2a_sched_linear(int rank, int p, MPI_Aint sndext, MPI_Aint rcvext, NBC_Schedule *schedule,
-                                   const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf,
-                                   int recvcount, MPI_Datatype recvtype, MPI_Comm comm);
+                                   const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf,
+                                   size_t recvcount, MPI_Datatype recvtype, MPI_Comm comm);
 static inline int a2a_sched_pairwise(int rank, int p, MPI_Aint sndext, MPI_Aint rcvext, NBC_Schedule *schedule,
-                                     const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf,
-                                     int recvcount, MPI_Datatype recvtype, MPI_Comm comm);
+                                     const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf,
+                                     size_t recvcount, MPI_Datatype recvtype, MPI_Comm comm);
 static inline int a2a_sched_diss(int rank, int p, MPI_Aint sndext, MPI_Aint rcvext, NBC_Schedule* schedule,
-                                 const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf,
-                                 int recvcount, MPI_Datatype recvtype, MPI_Comm comm, void* tmpbuf);
-static inline int a2a_sched_inplace(int rank, int p, NBC_Schedule* schedule, void* buf, int count,
+                                 const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf,
+                                 size_t recvcount, MPI_Datatype recvtype, MPI_Comm comm, void* tmpbuf);
+static inline int a2a_sched_inplace(int rank, int p, NBC_Schedule* schedule, void* buf, size_t count,
                                    MPI_Datatype type, MPI_Aint ext, ptrdiff_t gap, MPI_Comm comm);
 
 #ifdef NBC_CACHE_SCHEDULE
@@ -54,7 +54,7 @@ int NBC_Alltoall_args_compare(NBC_Alltoall_args *a, NBC_Alltoall_args *b, void *
 #endif
 
 /* simple linear MPI_Ialltoall the (simple) algorithm just sends to all nodes */
-static int nbc_alltoall_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+static int nbc_alltoall_init(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf, size_t recvcount,
                              MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
                              mca_coll_base_module_t *module, bool persistent)
 {
@@ -71,6 +71,10 @@ static int nbc_alltoall_init(const void* sendbuf, int sendcount, MPI_Datatype se
   void *tmpbuf = NULL;
   ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
   ptrdiff_t span, gap = 0;
+  uint64_t flags;
+  int is_accel_buf1;
+  int is_accel_buf2;
+  int dev_id;
 
   NBC_IN_PLACE(sendbuf, recvbuf, inplace);
 
@@ -145,11 +149,15 @@ static int nbc_alltoall_init(const void* sendbuf, int sendcount, MPI_Datatype se
     }
 
     /* phase 1 - rotate n data blocks upwards into the tmpbuffer */
-#if OPAL_CUDA_SUPPORT
-    if (NBC_Type_intrinsic(sendtype) && !(opal_cuda_check_bufs((char *)sendbuf, (char *)recvbuf))) {
-#else
-    if (NBC_Type_intrinsic(sendtype)) {
-#endif /* OPAL_CUDA_SUPPORT */
+    is_accel_buf1 = opal_accelerator.check_addr(sendbuf, &dev_id, &flags);
+    is_accel_buf2 = opal_accelerator.check_addr(recvbuf, &dev_id, &flags);
+    if (is_accel_buf1 < 0) {
+        return is_accel_buf1;
+    } else if (is_accel_buf2 < 0) {
+        return is_accel_buf2;
+    }
+    if (NBC_Type_intrinsic(sendtype) &&
+        is_accel_buf1 == 0 && is_accel_buf2 == 0) {
       /* contiguous - just copy (1st copy) */
       memcpy (tmpbuf, (char *) sendbuf + datasize * rank, datasize * (p - rank));
       if (rank != 0) {
@@ -281,7 +289,7 @@ static int nbc_alltoall_init(const void* sendbuf, int sendcount, MPI_Datatype se
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_ialltoall(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+int ompi_coll_libnbc_ialltoall(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf, size_t recvcount,
                                MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                mca_coll_base_module_t *module) {
     int res = nbc_alltoall_init(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype,
@@ -300,7 +308,7 @@ int ompi_coll_libnbc_ialltoall(const void* sendbuf, int sendcount, MPI_Datatype
     return OMPI_SUCCESS;
 }
 
-static int nbc_alltoall_inter_init (const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+static int nbc_alltoall_inter_init (const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf, size_t recvcount,
                                     MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                     mca_coll_base_module_t *module, bool persistent)
 {
@@ -331,14 +339,14 @@ static int nbc_alltoall_inter_init (const void* sendbuf, int sendcount, MPI_Data
 
   for (int i = 0; i < rsize; i++) {
     /* post all sends */
-    sbuf = (char *) sendbuf + i * sendcount * sndext;
+    sbuf = (char *) sendbuf + (MPI_Aint) sndext * i * sendcount;
     res = NBC_Sched_send (sbuf, false, sendcount, sendtype, i, schedule, false);
     if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
       break;
     }
 
     /* post all receives */
-    rbuf = (char *) recvbuf + i * recvcount * rcvext;
+    rbuf = (char *) recvbuf + (MPI_Aint) rcvext * i * recvcount;
     res = NBC_Sched_recv (rbuf, false, recvcount, recvtype, i, schedule, false);
     if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
       break;
@@ -365,7 +373,7 @@ static int nbc_alltoall_inter_init (const void* sendbuf, int sendcount, MPI_Data
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_ialltoall_inter (const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+int ompi_coll_libnbc_ialltoall_inter (const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf, size_t recvcount,
 				      MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
 				      mca_coll_base_module_t *module) {
     int res = nbc_alltoall_inter_init(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype,
@@ -385,7 +393,7 @@ int ompi_coll_libnbc_ialltoall_inter (const void* sendbuf, int sendcount, MPI_Da
 }
 
 static inline int a2a_sched_pairwise(int rank, int p, MPI_Aint sndext, MPI_Aint rcvext, NBC_Schedule* schedule,
-                                     const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+                                     const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf, size_t recvcount,
                                      MPI_Datatype recvtype, MPI_Comm comm) {
   int res;
 
@@ -397,13 +405,13 @@ static inline int a2a_sched_pairwise(int rank, int p, MPI_Aint sndext, MPI_Aint
     int sndpeer = (rank + r) % p;
     int rcvpeer = (rank - r + p) % p;
 
-    char *rbuf = (char *) recvbuf + rcvpeer * recvcount * rcvext;
+    char *rbuf = (char *) recvbuf + (MPI_Aint) rcvext * rcvpeer * recvcount;
     res = NBC_Sched_recv (rbuf, false, recvcount, recvtype, rcvpeer, schedule, false);
     if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
       return res;
     }
 
-    char *sbuf = (char *) sendbuf + sndpeer * sendcount * sndext;
+    char *sbuf = (char *) sendbuf + (MPI_Aint) sndext * sndpeer * sendcount;
     res = NBC_Sched_send (sbuf, false, sendcount, sendtype, sndpeer, schedule, true);
     if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
       return res;
@@ -414,7 +422,7 @@ static inline int a2a_sched_pairwise(int rank, int p, MPI_Aint sndext, MPI_Aint
 }
 
 static inline int a2a_sched_linear(int rank, int p, MPI_Aint sndext, MPI_Aint rcvext, NBC_Schedule* schedule,
-                                   const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+                                   const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf, size_t recvcount,
                                    MPI_Datatype recvtype, MPI_Comm comm) {
   int res;
 
@@ -441,7 +449,7 @@ static inline int a2a_sched_linear(int rank, int p, MPI_Aint sndext, MPI_Aint rc
 }
 
 static inline int a2a_sched_diss(int rank, int p, MPI_Aint sndext, MPI_Aint rcvext, NBC_Schedule* schedule,
-                                 const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+                                 const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf, size_t recvcount,
                                  MPI_Datatype recvtype, MPI_Comm comm, void* tmpbuf) {
   int res, speer, rpeer, virtp;
   MPI_Aint datasize, offset;
@@ -523,7 +531,7 @@ static inline int a2a_sched_diss(int rank, int p, MPI_Aint sndext, MPI_Aint rcve
 
   /* phase 3 - reorder - data is now in wrong order in tmpbuf - reorder it into recvbuf */
   for (int i = 0 ; i < p; ++i) {
-    rbuf = (char *) recvbuf + ((rank - i + p) % p) * recvcount * rcvext;
+    rbuf = (char *) recvbuf + (MPI_Aint) rcvext * ((rank - i + p) % p) * recvcount;
     res = NBC_Sched_unpack ((void *)(intptr_t) (i * datasize), true, recvcount, recvtype, rbuf, false, schedule,
                             false);
     if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
@@ -534,7 +542,7 @@ static inline int a2a_sched_diss(int rank, int p, MPI_Aint sndext, MPI_Aint rcve
   return OMPI_SUCCESS;
 }
 
-static inline int a2a_sched_inplace(int rank, int p, NBC_Schedule* schedule, void* buf, int count,
+static inline int a2a_sched_inplace(int rank, int p, NBC_Schedule* schedule, void* buf, size_t count,
                                    MPI_Datatype type, MPI_Aint ext, ptrdiff_t gap, MPI_Comm comm) {
   int res;
 
@@ -591,7 +599,7 @@ static inline int a2a_sched_inplace(int rank, int p, NBC_Schedule* schedule, voi
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_alltoall_init (const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+int ompi_coll_libnbc_alltoall_init (const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf, size_t recvcount,
                                     MPI_Datatype recvtype, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                     mca_coll_base_module_t *module) {
     int res = nbc_alltoall_init(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype,
@@ -603,7 +611,7 @@ int ompi_coll_libnbc_alltoall_init (const void* sendbuf, int sendcount, MPI_Data
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_alltoall_inter_init (const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+int ompi_coll_libnbc_alltoall_inter_init (const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf, size_t recvcount,
                                           MPI_Datatype recvtype, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                           mca_coll_base_module_t *module) {
     int res = nbc_alltoall_inter_init(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype,
diff --git a/ompi/mca/coll/libnbc/nbc_ialltoallv.c b/ompi/mca/coll/libnbc/nbc_ialltoallv.c
index 5b9db1119d8..99b7baf1a0e 100644
--- a/ompi/mca/coll/libnbc/nbc_ialltoallv.c
+++ b/ompi/mca/coll/libnbc/nbc_ialltoallv.c
@@ -11,6 +11,7 @@
  *                         reserved.
  * Copyright (c) 2017      IBM Corporation.  All rights reserved.
  * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2021      Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -21,32 +22,33 @@
 #include "nbc_internal.h"
 
 static inline int a2av_sched_linear(int rank, int p, NBC_Schedule *schedule,
-                                    const void *sendbuf, const int *sendcounts,
-                                    const int *sdispls, MPI_Aint sndext, MPI_Datatype sendtype,
-                                    void *recvbuf, const int *recvcounts,
-                                    const int *rdispls, MPI_Aint rcvext, MPI_Datatype recvtype);
+                                    const void *sendbuf, ompi_count_array_t sendcounts,
+                                    ompi_disp_array_t sdispls, MPI_Aint sndext, MPI_Datatype sendtype, const size_t sdtype_size,
+                                    void *recvbuf, ompi_count_array_t recvcounts,
+                                    ompi_disp_array_t rdispls, MPI_Aint rcvext, MPI_Datatype recvtype, const size_t rdtype_size);
 
 static inline int a2av_sched_pairwise(int rank, int p, NBC_Schedule *schedule,
-                                      const void *sendbuf, const int *sendcounts, const int *sdispls,
-                                      MPI_Aint sndext, MPI_Datatype sendtype,
-                                      void *recvbuf, const int *recvcounts, const int *rdispls,
-                                      MPI_Aint rcvext, MPI_Datatype recvtype);
+                                      const void *sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t sdispls,
+                                      MPI_Aint sndext, MPI_Datatype sendtype, const size_t sdtype_size,
+                                      void *recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t rdispls,
+                                      MPI_Aint rcvext, MPI_Datatype recvtype, const size_t rdtype_size);
 
 static inline int a2av_sched_inplace(int rank, int p, NBC_Schedule *schedule,
-                                    void *buf, const int *counts, const int *displs,
-                                    MPI_Aint ext, MPI_Datatype type, ptrdiff_t gap);
+                                    void *buf, ompi_count_array_t counts, ompi_disp_array_t displs,
+                                    MPI_Aint ext, MPI_Datatype type, const size_t dtype_size, ptrdiff_t gap);
 
 /* an alltoallv schedule can not be cached easily because the contents
- * ot the recvcounts array may change, so a comparison of the address
+ * of the recvcounts array may change, so a comparison of the address
  * would not be sufficient ... we simply do not cache it */
 
 /* simple linear Alltoallv */
-static int nbc_alltoallv_init(const void* sendbuf, const int *sendcounts, const int *sdispls,
-                              MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls,
+static int nbc_alltoallv_init(const void* sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t sdispls,
+                              MPI_Datatype sendtype, void* recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t rdispls,
                               MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
                               mca_coll_base_module_t *module, bool persistent)
 {
   int rank, p, res;
+  size_t sdtype_size, rdtype_size;
   MPI_Aint sndext, rcvext;
   NBC_Schedule *schedule;
   char *rbuf, *sbuf, inplace;
@@ -59,6 +61,7 @@ static int nbc_alltoallv_init(const void* sendbuf, const int *sendcounts, const
   rank = ompi_comm_rank (comm);
   p = ompi_comm_size (comm);
 
+  ompi_datatype_type_size(recvtype, &rdtype_size);
   res = ompi_datatype_type_extent (recvtype, &rcvext);
   if (MPI_SUCCESS != res) {
     NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res);
@@ -67,17 +70,17 @@ static int nbc_alltoallv_init(const void* sendbuf, const int *sendcounts, const
 
   /* copy data to receivbuffer */
   if (inplace) {
-    int count = 0;
+    size_t count = 0;
     for (int i = 0; i < p; i++) {
-      if (recvcounts[i] > count) {
-        count = recvcounts[i];
+      if (ompi_count_array_get(recvcounts, i) > count) {
+        count = ompi_count_array_get(recvcounts, i);
       }
     }
     span = opal_datatype_span(&recvtype->super, count, &gap);
     /**
      * If this process has no data to send or receive it can bail out early,
-     * but it needs to increase the nonblocking tag to stay in sycn with the
-     * rest of the processses.
+     * but it needs to increase the nonblocking tag to stay in sync with the
+     * rest of the processes.
      */
     if (OPAL_UNLIKELY(0 == span)) {
       ompi_coll_base_nbc_reserve_tags(comm, 1);
@@ -90,7 +93,10 @@ static int nbc_alltoallv_init(const void* sendbuf, const int *sendcounts, const
 
     sendcounts = recvcounts;
     sdispls = rdispls;
+    sndext = rcvext;
+    sdtype_size = rdtype_size;
   } else {
+    ompi_datatype_type_size(sendtype, &sdtype_size);
     res = ompi_datatype_type_extent (sendtype, &sndext);
     if (MPI_SUCCESS != res) {
       NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res);
@@ -104,12 +110,11 @@ static int nbc_alltoallv_init(const void* sendbuf, const int *sendcounts, const
     return OMPI_ERR_OUT_OF_RESOURCE;
   }
 
-
-  if (!inplace && sendcounts[rank] != 0) {
-    rbuf = (char *) recvbuf + rdispls[rank] * rcvext;
-    sbuf = (char *) sendbuf + sdispls[rank] * sndext;
-    res = NBC_Sched_copy (sbuf, false, sendcounts[rank], sendtype,
-                          rbuf, false, recvcounts[rank], recvtype, schedule, false);
+  if (!inplace && 0 < ompi_count_array_get(sendcounts, rank) && 0 < sdtype_size) {
+    rbuf = (char *) recvbuf + ompi_disp_array_get(rdispls, rank) * rcvext;
+    sbuf = (char *) sendbuf + ompi_disp_array_get(sdispls, rank) * sndext;
+    res = NBC_Sched_copy (sbuf, false, ompi_count_array_get(sendcounts, rank), sendtype,
+                          rbuf, false, ompi_count_array_get(recvcounts, rank), recvtype, schedule, false);
     if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
       OBJ_RELEASE(schedule);
       return res;
@@ -117,12 +122,12 @@ static int nbc_alltoallv_init(const void* sendbuf, const int *sendcounts, const
   }
 
   if (inplace) {
-    res = a2av_sched_inplace(rank, p, schedule, recvbuf, recvcounts,
-                                 rdispls, rcvext, recvtype, gap);
+    res = a2av_sched_inplace(rank, p, schedule, recvbuf, recvcounts, rdispls, rcvext, recvtype,
+                             rdtype_size, gap);
   } else {
     res = a2av_sched_linear(rank, p, schedule,
-                            sendbuf, sendcounts, sdispls, sndext, sendtype,
-                            recvbuf, recvcounts, rdispls, rcvext, recvtype);
+                            sendbuf, sendcounts, sdispls, sndext, sendtype, sdtype_size,
+                            recvbuf, recvcounts, rdispls, rcvext, recvtype, rdtype_size);
   }
   if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
     OBJ_RELEASE(schedule);
@@ -147,8 +152,8 @@ static int nbc_alltoallv_init(const void* sendbuf, const int *sendcounts, const
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_ialltoallv(const void* sendbuf, const int *sendcounts, const int *sdispls,
-                                MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls,
+int ompi_coll_libnbc_ialltoallv(const void* sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t sdispls,
+                                MPI_Datatype sendtype, void* recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t rdispls,
                                 MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                 mca_coll_base_module_t *module) {
     int res = nbc_alltoallv_init(sendbuf, sendcounts, sdispls, sendtype,
@@ -169,16 +174,19 @@ int ompi_coll_libnbc_ialltoallv(const void* sendbuf, const int *sendcounts, cons
 }
 
 /* simple linear Alltoallv */
-static int nbc_alltoallv_inter_init (const void* sendbuf, const int *sendcounts, const int *sdispls,
-                                     MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls,
+static int nbc_alltoallv_inter_init (const void* sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t sdispls,
+                                     MPI_Datatype sendtype, void* recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t rdispls,
                                      MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                      mca_coll_base_module_t *module, bool persistent)
 {
   int res, rsize;
+  size_t sdtype_size, rdtype_size;
   MPI_Aint sndext, rcvext;
   NBC_Schedule *schedule;
   ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
 
+  ompi_datatype_type_size(sendtype, &sdtype_size);
+  ompi_datatype_type_size(recvtype, &rdtype_size);
 
   res = ompi_datatype_type_extent(sendtype, &sndext);
   if (MPI_SUCCESS != res) {
@@ -201,18 +209,18 @@ static int nbc_alltoallv_inter_init (const void* sendbuf, const int *sendcounts,
 
   for (int i = 0; i < rsize; i++) {
     /* post all sends */
-    if (sendcounts[i] != 0) {
-      char *sbuf = (char *) sendbuf + sdispls[i] * sndext;
-      res = NBC_Sched_send (sbuf, false, sendcounts[i], sendtype, i, schedule, false);
+    if (0 < ompi_count_array_get(sendcounts, i) && 0 < sdtype_size) {
+      char *sbuf = (char *) sendbuf + ompi_disp_array_get(sdispls, i) * sndext;
+      res = NBC_Sched_send (sbuf, false, ompi_count_array_get(sendcounts, i), sendtype, i, schedule, false);
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
         OBJ_RELEASE(schedule);
         return res;
       }
     }
     /* post all receives */
-    if (recvcounts[i] != 0) {
-      char *rbuf = (char *) recvbuf + rdispls[i] * rcvext;
-      res = NBC_Sched_recv (rbuf, false, recvcounts[i], recvtype, i, schedule, false);
+    if (0 < ompi_count_array_get(recvcounts, i) && 0 < rdtype_size) {
+      char *rbuf = (char *) recvbuf + ompi_disp_array_get(rdispls, i) * rcvext;
+      res = NBC_Sched_recv (rbuf, false, ompi_count_array_get(recvcounts, i), recvtype, i, schedule, false);
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
         OBJ_RELEASE(schedule);
         return res;
@@ -235,8 +243,8 @@ static int nbc_alltoallv_inter_init (const void* sendbuf, const int *sendcounts,
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_ialltoallv_inter (const void* sendbuf, const int *sendcounts, const int *sdispls,
-				       MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls,
+int ompi_coll_libnbc_ialltoallv_inter (const void* sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t sdispls,
+				       MPI_Datatype sendtype, void* recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t rdispls,
 				       MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
 				       mca_coll_base_module_t *module) {
     int res = nbc_alltoallv_inter_init(sendbuf, sendcounts, sdispls, sendtype,
@@ -258,10 +266,10 @@ int ompi_coll_libnbc_ialltoallv_inter (const void* sendbuf, const int *sendcount
 
 __opal_attribute_unused__
 static inline int a2av_sched_linear(int rank, int p, NBC_Schedule *schedule,
-                                    const void *sendbuf, const int *sendcounts, const int *sdispls,
-                                    MPI_Aint sndext, MPI_Datatype sendtype,
-                                    void *recvbuf, const int *recvcounts, const int *rdispls,
-                                    MPI_Aint rcvext, MPI_Datatype recvtype) {
+                                    const void *sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t sdispls,
+                                    MPI_Aint sndext, MPI_Datatype sendtype, const size_t sdtype_size,
+                                    void *recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t rdispls,
+                                    MPI_Aint rcvext, MPI_Datatype recvtype, const size_t rdtype_size) {
   int res;
 
   for (int i = 0 ; i < p ; ++i) {
@@ -270,18 +278,18 @@ static inline int a2av_sched_linear(int rank, int p, NBC_Schedule *schedule,
     }
 
     /* post send */
-    if (sendcounts[i] != 0) {
-      char *sbuf = ((char *) sendbuf) + (sdispls[i] * sndext);
-      res = NBC_Sched_send(sbuf, false, sendcounts[i], sendtype, i, schedule, false);
+    if (0 < ompi_count_array_get(sendcounts, i) && 0 < sdtype_size) {
+      char *sbuf = ((char *) sendbuf) + (ompi_disp_array_get(sdispls, i) * sndext);
+      res = NBC_Sched_send(sbuf, false, ompi_count_array_get(sendcounts, i), sendtype, i, schedule, false);
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
         return res;
       }
     }
 
     /* post receive */
-    if (recvcounts[i] != 0) {
-      char *rbuf = ((char *) recvbuf) + (rdispls[i] * rcvext);
-      res = NBC_Sched_recv(rbuf, false, recvcounts[i], recvtype, i, schedule, false);
+    if (0 < ompi_count_array_get(recvcounts, i) && 0 < rdtype_size) {
+      char *rbuf = ((char *) recvbuf) + (ompi_disp_array_get(rdispls, i) * rcvext);
+      res = NBC_Sched_recv(rbuf, false, ompi_count_array_get(recvcounts, i), recvtype, i, schedule, false);
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
         return res;
       }
@@ -293,10 +301,10 @@ static inline int a2av_sched_linear(int rank, int p, NBC_Schedule *schedule,
 
 __opal_attribute_unused__
 static inline int a2av_sched_pairwise(int rank, int p, NBC_Schedule *schedule,
-                                      const void *sendbuf, const int *sendcounts, const int *sdispls,
-                                      MPI_Aint sndext, MPI_Datatype sendtype,
-                                      void *recvbuf, const int *recvcounts, const int *rdispls,
-                                      MPI_Aint rcvext, MPI_Datatype recvtype) {
+                                      const void *sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t sdispls,
+                                      MPI_Aint sndext, MPI_Datatype sendtype, const size_t sdtype_size,
+                                      void *recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t rdispls,
+                                      MPI_Aint rcvext, MPI_Datatype recvtype, const size_t rdtype_size) {
   int res;
 
   for (int i = 1 ; i < p ; ++i) {
@@ -304,18 +312,18 @@ static inline int a2av_sched_pairwise(int rank, int p, NBC_Schedule *schedule,
     int rcvpeer = (rank + p - i) %p;
 
     /* post send */
-    if (sendcounts[sndpeer] != 0) {
-      char *sbuf = ((char *) sendbuf) + (sdispls[sndpeer] * sndext);
-      res = NBC_Sched_send(sbuf, false, sendcounts[sndpeer], sendtype, sndpeer, schedule, false);
+    if (0 < ompi_count_array_get(sendcounts, sndpeer) && 0 < sdtype_size) {
+      char *sbuf = ((char *) sendbuf) + (ompi_disp_array_get(sdispls, sndpeer) * sndext);
+      res = NBC_Sched_send(sbuf, false, ompi_count_array_get(sendcounts, sndpeer), sendtype, sndpeer, schedule, false);
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
         return res;
       }
     }
 
     /* post receive */
-    if (recvcounts[rcvpeer] != 0) {
-      char *rbuf = ((char *) recvbuf) + (rdispls[rcvpeer] * rcvext);
-      res = NBC_Sched_recv(rbuf, false, recvcounts[rcvpeer], recvtype, rcvpeer, schedule, true);
+    if (0 < ompi_count_array_get(recvcounts, rcvpeer) && 0 < rdtype_size) {
+      char *rbuf = ((char *) recvbuf) + (ompi_disp_array_get(rdispls, rcvpeer) * rcvext);
+      res = NBC_Sched_recv(rbuf, false, ompi_count_array_get(recvcounts, rcvpeer), recvtype, rcvpeer, schedule, true);
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
         return res;
       }
@@ -326,45 +334,50 @@ static inline int a2av_sched_pairwise(int rank, int p, NBC_Schedule *schedule,
 }
 
 static inline int a2av_sched_inplace(int rank, int p, NBC_Schedule *schedule,
-                                    void *buf, const int *counts, const int *displs,
-                                    MPI_Aint ext, MPI_Datatype type, ptrdiff_t gap) {
+                                    void *buf, ompi_count_array_t counts, ompi_disp_array_t displs,
+                                    MPI_Aint ext, MPI_Datatype type, const size_t dtype_size, ptrdiff_t gap) {
   int res;
 
   for (int i = 1; i < (p+1)/2; i++) {
     int speer = (rank + i) % p;
     int rpeer = (rank + p - i) % p;
-    char *sbuf = (char *) buf + displs[speer] * ext;
-    char *rbuf = (char *) buf + displs[rpeer] * ext;
+    char *sbuf = (char *) buf + ompi_disp_array_get(displs, speer) * ext;
+    char *rbuf = (char *) buf + ompi_disp_array_get(displs, rpeer) * ext;
+
+    if (0 == dtype_size) {
+      /* Nothing to exchange */
+      return OMPI_SUCCESS;
+    }
 
-    if (0 != counts[rpeer]) {
-      res = NBC_Sched_copy (rbuf, false, counts[rpeer], type,
-                            (void *)(-gap), true, counts[rpeer], type,
+    if (0 < ompi_count_array_get(counts, rpeer)) {
+      res = NBC_Sched_copy (rbuf, false, ompi_count_array_get(counts, rpeer), type,
+                            (void *)(-gap), true, ompi_count_array_get(counts, rpeer), type,
                             schedule, true);
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
         return res;
       }
     }
-    if (0 != counts[speer]) {
-      res = NBC_Sched_send (sbuf, false , counts[speer], type, speer, schedule, false);
+    if (0 < ompi_count_array_get(counts, speer)) {
+      res = NBC_Sched_send (sbuf, false , ompi_count_array_get(counts, speer), type, speer, schedule, false);
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
         return res;
       }
     }
-    if (0 != counts[rpeer]) {
-      res = NBC_Sched_recv (rbuf, false , counts[rpeer], type, rpeer, schedule, true);
+    if (0 < ompi_count_array_get(counts, rpeer)) {
+      res = NBC_Sched_recv (rbuf, false , ompi_count_array_get(counts, rpeer), type, rpeer, schedule, true);
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
         return res;
       }
     }
 
-    if (0 != counts[rpeer]) {
-      res = NBC_Sched_send ((void *)(-gap), true, counts[rpeer], type, rpeer, schedule, false);
+    if (0 < ompi_count_array_get(counts, rpeer)) {
+      res = NBC_Sched_send ((void *)(-gap), true, ompi_count_array_get(counts, rpeer), type, rpeer, schedule, false);
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
         return res;
       }
     }
-    if (0 != counts[speer]) {
-      res = NBC_Sched_recv (sbuf, false, counts[speer], type, speer, schedule, true);
+    if (0 < ompi_count_array_get(counts, speer)) {
+      res = NBC_Sched_recv (sbuf, false, ompi_count_array_get(counts, speer), type, speer, schedule, true);
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
         return res;
       }
@@ -372,20 +385,23 @@ static inline int a2av_sched_inplace(int rank, int p, NBC_Schedule *schedule,
   }
   if (0 == (p%2)) {
     int peer = (rank + p/2) % p;
-
-    char *tbuf = (char *) buf + displs[peer] * ext;
-    res = NBC_Sched_copy (tbuf, false, counts[peer], type,
-                          (void *)(-gap), true, counts[peer], type,
-                          schedule, true);
-    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
-      return res;
+    char *tbuf = (char *) buf + ompi_disp_array_get(displs, peer) * ext;
+
+    if (0 < ompi_count_array_get(counts, peer)) {
+        res = NBC_Sched_copy(tbuf, false, ompi_count_array_get(counts, peer), type,
+                             (void *) (-gap), true, ompi_count_array_get(counts, peer),
+                             type, schedule, true);
+        if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+            return res;
+        }
     }
-    if (0 != counts[peer]) {
-      res = NBC_Sched_send ((void *)(-gap), true , counts[peer], type, peer, schedule, false);
+
+    if (0 < ompi_count_array_get(counts, peer)) {
+      res = NBC_Sched_send ((void *)(-gap), true, ompi_count_array_get(counts, peer), type, peer, schedule, false);
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
         return res;
       }
-      res = NBC_Sched_recv (tbuf, false , counts[peer], type, peer, schedule, true);
+      res = NBC_Sched_recv (tbuf, false , ompi_count_array_get(counts, peer), type, peer, schedule, true);
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
         return res;
       }
@@ -395,8 +411,8 @@ static inline int a2av_sched_inplace(int rank, int p, NBC_Schedule *schedule,
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_alltoallv_init(const void* sendbuf, const int *sendcounts, const int *sdispls,
-                                    MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls,
+int ompi_coll_libnbc_alltoallv_init(const void* sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t sdispls,
+                                    MPI_Datatype sendtype, void* recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t rdispls,
                                     MPI_Datatype recvtype, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                     mca_coll_base_module_t *module) {
     int res = nbc_alltoallv_init(sendbuf, sendcounts, sdispls, sendtype, recvbuf, recvcounts, rdispls, recvtype,
@@ -408,8 +424,8 @@ int ompi_coll_libnbc_alltoallv_init(const void* sendbuf, const int *sendcounts,
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_alltoallv_inter_init(const void* sendbuf, const int *sendcounts, const int *sdispls,
-                                          MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls,
+int ompi_coll_libnbc_alltoallv_inter_init(const void* sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t sdispls,
+                                          MPI_Datatype sendtype, void* recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t rdispls,
                                           MPI_Datatype recvtype, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                           mca_coll_base_module_t *module) {
     int res = nbc_alltoallv_inter_init(sendbuf, sendcounts, sdispls, sendtype, recvbuf, recvcounts, rdispls, recvtype,
diff --git a/ompi/mca/coll/libnbc/nbc_ialltoallw.c b/ompi/mca/coll/libnbc/nbc_ialltoallw.c
index cf000db9a4a..606b39fea15 100644
--- a/ompi/mca/coll/libnbc/nbc_ialltoallw.c
+++ b/ompi/mca/coll/libnbc/nbc_ialltoallw.c
@@ -21,28 +21,28 @@
 #include "nbc_internal.h"
 
 static inline int a2aw_sched_linear(int rank, int p, NBC_Schedule *schedule,
-                                    const void *sendbuf, const int *sendcounts, const int *sdispls,
+                                    const void *sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t sdispls,
                                     struct ompi_datatype_t * const * sendtypes,
-                                    void *recvbuf, const int *recvcounts, const int *rdispls,
+                                    void *recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t rdispls,
                                     struct ompi_datatype_t * const * recvtypes);
 
 static inline int a2aw_sched_pairwise(int rank, int p, NBC_Schedule *schedule,
-                                      const void *sendbuf, const int *sendcounts, const int *sdispls,
+                                      const void *sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t sdispls,
                                       struct ompi_datatype_t * const * sendtypes,
-                                      void *recvbuf, const int *recvcounts, const int *rdispls,
+                                      void *recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t rdispls,
                                       struct ompi_datatype_t * const * recvtypes);
 
 static inline int a2aw_sched_inplace(int rank, int p, NBC_Schedule *schedule,
-                                    void *buf, const int *counts, const int *displs,
+                                    void *buf, ompi_count_array_t counts, ompi_disp_array_t displs,
                                     struct ompi_datatype_t * const * types);
 
 /* an alltoallw schedule can not be cached easily because the contents
- * ot the recvcounts array may change, so a comparison of the address
+ * of the recvcounts array may change, so a comparison of the address
  * would not be sufficient ... we simply do not cache it */
 
 /* simple linear Alltoallw */
-static int nbc_alltoallw_init(const void* sendbuf, const int *sendcounts, const int *sdispls,
-                              struct ompi_datatype_t * const *sendtypes, void* recvbuf, const int *recvcounts, const int *rdispls,
+static int nbc_alltoallw_init(const void* sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t sdispls,
+                              struct ompi_datatype_t * const *sendtypes, void* recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t rdispls,
                               struct ompi_datatype_t * const *recvtypes, struct ompi_communicator_t *comm, ompi_request_t ** request,
                               mca_coll_base_module_t *module, bool persistent)
 {
@@ -62,15 +62,15 @@ static int nbc_alltoallw_init(const void* sendbuf, const int *sendcounts, const
   if (inplace) {
     ptrdiff_t lgap, lspan;
     for (int i = 0; i < p; i++) {
-      lspan = opal_datatype_span(&recvtypes[i]->super, recvcounts[i], &lgap);
+      lspan = opal_datatype_span(&recvtypes[i]->super, ompi_count_array_get(recvcounts, i), &lgap);
       if (lspan > span) {
         span = lspan;
       }
     }
     /**
      * If this process has no data to send or receive it can bail out early,
-     * but it needs to increase the nonblocking tag to stay in sycn with the
-     * rest of the processses.
+     * but it needs to increase the nonblocking tag to stay in sync with the
+     * rest of the processes.
      */
     if (OPAL_UNLIKELY(0 == span)) {
       ompi_coll_base_nbc_reserve_tags(comm, 1);
@@ -91,11 +91,11 @@ static int nbc_alltoallw_init(const void* sendbuf, const int *sendcounts, const
     return OMPI_ERR_OUT_OF_RESOURCE;
   }
 
-  if (!inplace && sendcounts[rank] != 0) {
-    rbuf = (char *) recvbuf + rdispls[rank];
-    sbuf = (char *) sendbuf + sdispls[rank];
-    res = NBC_Sched_copy(sbuf, false, sendcounts[rank], sendtypes[rank],
-                         rbuf, false, recvcounts[rank], recvtypes[rank], schedule, false);
+  if (!inplace && ompi_count_array_get(sendcounts, rank) != 0) {
+    rbuf = (char *) recvbuf + ompi_disp_array_get(rdispls, rank);
+    sbuf = (char *) sendbuf + ompi_disp_array_get(sdispls, rank);
+    res = NBC_Sched_copy(sbuf, false, ompi_count_array_get(sendcounts, rank), sendtypes[rank],
+                         rbuf, false, ompi_count_array_get(recvcounts, rank), recvtypes[rank], schedule, false);
     if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
       return res;
     }
@@ -132,8 +132,8 @@ static int nbc_alltoallw_init(const void* sendbuf, const int *sendcounts, const
   return OMPI_SUCCESS;
 } 
 
-int ompi_coll_libnbc_ialltoallw(const void* sendbuf, const int *sendcounts, const int *sdispls,
-                                struct ompi_datatype_t * const *sendtypes, void* recvbuf, const int *recvcounts, const int *rdispls,
+int ompi_coll_libnbc_ialltoallw(const void* sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t sdispls,
+                                struct ompi_datatype_t * const *sendtypes, void* recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t rdispls,
                                 struct ompi_datatype_t * const *recvtypes, struct ompi_communicator_t *comm, ompi_request_t ** request,
 				mca_coll_base_module_t *module) {
     int res = nbc_alltoallw_init(sendbuf, sendcounts, sdispls, sendtypes,
@@ -154,8 +154,8 @@ int ompi_coll_libnbc_ialltoallw(const void* sendbuf, const int *sendcounts, cons
 }
 
 /* simple linear Alltoallw */
-static int nbc_alltoallw_inter_init (const void* sendbuf, const int *sendcounts, const int *sdispls,
-                                     struct ompi_datatype_t * const *sendtypes, void* recvbuf, const int *recvcounts, const int *rdispls,
+static int nbc_alltoallw_inter_init (const void* sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t sdispls,
+                                     struct ompi_datatype_t * const *sendtypes, void* recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t rdispls,
                                      struct ompi_datatype_t * const *recvtypes, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                      mca_coll_base_module_t *module, bool persistent)
 {
@@ -173,18 +173,18 @@ static int nbc_alltoallw_inter_init (const void* sendbuf, const int *sendcounts,
 
   for (int i = 0 ; i < rsize ; ++i) {
     /* post all sends */
-    if (sendcounts[i] != 0) {
-      sbuf = (char *) sendbuf + sdispls[i];
-      res = NBC_Sched_send (sbuf, false, sendcounts[i], sendtypes[i], i, schedule, false);
+    if (ompi_count_array_get(sendcounts, i) != 0) {
+      sbuf = (char *) sendbuf + ompi_disp_array_get(sdispls, i);
+      res = NBC_Sched_send (sbuf, false, ompi_count_array_get(sendcounts, i), sendtypes[i], i, schedule, false);
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
         OBJ_RELEASE(schedule);
         return res;
       }
     }
     /* post all receives */
-    if (recvcounts[i] != 0) {
-      rbuf = (char *) recvbuf + rdispls[i];
-      res = NBC_Sched_recv (rbuf, false, recvcounts[i], recvtypes[i], i, schedule, false);
+    if (ompi_count_array_get(recvcounts, i) != 0) {
+      rbuf = (char *) recvbuf + ompi_disp_array_get(rdispls, i);
+      res = NBC_Sched_recv (rbuf, false, ompi_count_array_get(recvcounts, i), recvtypes[i], i, schedule, false);
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
         OBJ_RELEASE(schedule);
         return res;
@@ -207,8 +207,8 @@ static int nbc_alltoallw_inter_init (const void* sendbuf, const int *sendcounts,
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_ialltoallw_inter(const void* sendbuf, const int *sendcounts, const int *sdispls,
-                                      struct ompi_datatype_t * const *sendtypes, void* recvbuf, const int *recvcounts, const int *rdispls,
+int ompi_coll_libnbc_ialltoallw_inter(const void* sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t sdispls,
+                                      struct ompi_datatype_t * const *sendtypes, void* recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t rdispls,
                                       struct ompi_datatype_t * const *recvtypes, struct ompi_communicator_t *comm, ompi_request_t ** request,
 				      mca_coll_base_module_t *module) {
     int res = nbc_alltoallw_inter_init(sendbuf, sendcounts, sdispls, sendtypes,
@@ -229,9 +229,9 @@ int ompi_coll_libnbc_ialltoallw_inter(const void* sendbuf, const int *sendcounts
 }
 
 static inline int a2aw_sched_linear(int rank, int p, NBC_Schedule *schedule,
-                                    const void *sendbuf, const int *sendcounts, const int *sdispls,
+                                    const void *sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t sdispls,
                                     struct ompi_datatype_t * const * sendtypes,
-                                    void *recvbuf, const int *recvcounts, const int *rdispls,
+                                    void *recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t rdispls,
                                     struct ompi_datatype_t * const * recvtypes) {
   int res;
 
@@ -242,19 +242,19 @@ static inline int a2aw_sched_linear(int rank, int p, NBC_Schedule *schedule,
     }
 
     /* post send */
-    span = opal_datatype_span(&sendtypes[i]->super, sendcounts[i], &gap);
+    span = opal_datatype_span(&sendtypes[i]->super, ompi_count_array_get(sendcounts, i), &gap);
     if (OPAL_LIKELY(0 < span)) {
-      char *sbuf = (char *) sendbuf + sdispls[i];
-      res = NBC_Sched_send (sbuf, false, sendcounts[i], sendtypes[i], i, schedule, false);
+      char *sbuf = (char *) sendbuf + ompi_disp_array_get(sdispls, i);
+      res = NBC_Sched_send (sbuf, false, ompi_count_array_get(sendcounts, i), sendtypes[i], i, schedule, false);
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
         return res;
       }
     }
     /* post receive */
-    span = opal_datatype_span(&recvtypes[i]->super, recvcounts[i], &gap);
+    span = opal_datatype_span(&recvtypes[i]->super, ompi_count_array_get(recvcounts, i), &gap);
     if (OPAL_LIKELY(0 < span)) {
-      char *rbuf = (char *) recvbuf + rdispls[i];
-      res = NBC_Sched_recv (rbuf, false, recvcounts[i], recvtypes[i], i, schedule, false);
+      char *rbuf = (char *) recvbuf + ompi_disp_array_get(rdispls, i);
+      res = NBC_Sched_recv (rbuf, false, ompi_count_array_get(recvcounts, i), recvtypes[i], i, schedule, false);
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
         return res;
       }
@@ -266,9 +266,9 @@ static inline int a2aw_sched_linear(int rank, int p, NBC_Schedule *schedule,
 
 __opal_attribute_unused__
 static inline int a2aw_sched_pairwise(int rank, int p, NBC_Schedule *schedule,
-                                      const void *sendbuf, const int *sendcounts, const int *sdispls,
+                                      const void *sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t sdispls,
                                       struct ompi_datatype_t * const * sendtypes,
-                                      void *recvbuf, const int *recvcounts, const int *rdispls,
+                                      void *recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t rdispls,
                                       struct ompi_datatype_t * const * recvtypes) {
   int res;
 
@@ -277,17 +277,17 @@ static inline int a2aw_sched_pairwise(int rank, int p, NBC_Schedule *schedule,
     int rcvpeer = (rank + p - i) % p;
 
     /* post send */
-    if (sendcounts[sndpeer] != 0) {
-      char *sbuf = (char *) sendbuf + sdispls[sndpeer];
-      res = NBC_Sched_send (sbuf, false, sendcounts[sndpeer], sendtypes[sndpeer], sndpeer, schedule, false);
+    if (ompi_count_array_get(sendcounts, sndpeer) != 0) {
+      char *sbuf = (char *) sendbuf + ompi_disp_array_get(sdispls, sndpeer);
+      res = NBC_Sched_send (sbuf, false, ompi_count_array_get(sendcounts, sndpeer), sendtypes[sndpeer], sndpeer, schedule, false);
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
         return res;
       }
     }
     /* post receive */
-    if (recvcounts[rcvpeer] != 0) {
-      char *rbuf = (char *) recvbuf + rdispls[rcvpeer];
-      res = NBC_Sched_recv (rbuf, false, recvcounts[rcvpeer], recvtypes[rcvpeer], rcvpeer, schedule, true);
+    if (ompi_count_array_get(recvcounts, rcvpeer) != 0) {
+      char *rbuf = (char *) recvbuf + ompi_disp_array_get(rdispls, rcvpeer);
+      res = NBC_Sched_recv (rbuf, false, ompi_count_array_get(recvcounts, rcvpeer), recvtypes[rcvpeer], rcvpeer, schedule, true);
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
         return res;
       }
@@ -298,7 +298,7 @@ static inline int a2aw_sched_pairwise(int rank, int p, NBC_Schedule *schedule,
 }
 
 static inline int a2aw_sched_inplace(int rank, int p, NBC_Schedule *schedule,
-                                     void *buf, const int *counts, const int *displs,
+                                     void *buf, ompi_count_array_t counts, ompi_disp_array_t displs,
                                      struct ompi_datatype_t * const * types) {
   ptrdiff_t gap = 0;
   int res;
@@ -306,39 +306,39 @@ static inline int a2aw_sched_inplace(int rank, int p, NBC_Schedule *schedule,
   for (int i = 1; i < (p+1)/2; i++) {
     int speer = (rank + i) % p;
     int rpeer = (rank + p - i) % p;
-    char *sbuf = (char *) buf + displs[speer];
-    char *rbuf = (char *) buf + displs[rpeer];
+    char *sbuf = (char *) buf + ompi_disp_array_get(displs, speer);
+    char *rbuf = (char *) buf + ompi_disp_array_get(displs, rpeer);
 
-    if (0 != counts[rpeer]) {
-      (void)opal_datatype_span(&types[rpeer]->super, counts[rpeer], &gap);
-      res = NBC_Sched_copy (rbuf, false, counts[rpeer], types[rpeer],
-                            (void *)(-gap), true, counts[rpeer], types[rpeer],
+    if (0 != ompi_count_array_get(counts, rpeer)) {
+      (void)opal_datatype_span(&types[rpeer]->super, ompi_count_array_get(counts, rpeer), &gap);
+      res = NBC_Sched_copy (rbuf, false, ompi_count_array_get(counts, rpeer), types[rpeer],
+                            (void *)(-gap), true, ompi_count_array_get(counts, rpeer), types[rpeer],
                             schedule, true);
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
         return res;
       }
     }
-    if (0 != counts[speer]) {
-      res = NBC_Sched_send (sbuf, false , counts[speer], types[speer], speer, schedule, false);
+    if (0 != ompi_count_array_get(counts, speer)) {
+      res = NBC_Sched_send (sbuf, false, ompi_count_array_get(counts, speer), types[speer], speer, schedule, false);
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
         return res;
       }
     }
-    if (0 != counts[rpeer]) {
-      res = NBC_Sched_recv (rbuf, false , counts[rpeer], types[rpeer], rpeer, schedule, true);
+    if (0 != ompi_count_array_get(counts, rpeer)) {
+      res = NBC_Sched_recv (rbuf, false, ompi_count_array_get(counts, rpeer), types[rpeer], rpeer, schedule, true);
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
         return res;
       }
     }
 
-    if (0 != counts[rpeer]) {
-      res = NBC_Sched_send ((void *)(-gap), true, counts[rpeer], types[rpeer], rpeer, schedule, false);
+    if (0 != ompi_count_array_get(counts, rpeer)) {
+      res = NBC_Sched_send ((void *)(-gap), true, ompi_count_array_get(counts, rpeer), types[rpeer], rpeer, schedule, false);
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
         return res;
       }
     }
-    if (0 != counts[speer]) {
-      res = NBC_Sched_recv (sbuf, false, counts[speer], types[speer], speer, schedule, true);
+    if (0 != ompi_count_array_get(counts, speer)) {
+      res = NBC_Sched_recv (sbuf, false, ompi_count_array_get(counts, speer), types[speer], speer, schedule, true);
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
         return res;
       }
@@ -347,19 +347,19 @@ static inline int a2aw_sched_inplace(int rank, int p, NBC_Schedule *schedule,
   if (0 == (p%2)) {
     int peer = (rank + p/2) % p;
 
-    char *tbuf = (char *) buf + displs[peer];
-    (void)opal_datatype_span(&types[peer]->super, counts[peer], &gap);
-    res = NBC_Sched_copy (tbuf, false, counts[peer], types[peer],
-                          (void *)(-gap), true, counts[peer], types[peer],
+    char *tbuf = (char *) buf + ompi_disp_array_get(displs, peer);
+    (void)opal_datatype_span(&types[peer]->super, ompi_count_array_get(counts, peer), &gap);
+    res = NBC_Sched_copy (tbuf, false, ompi_count_array_get(counts, peer), types[peer],
+                          (void *)(-gap), true, ompi_count_array_get(counts, peer), types[peer],
                           schedule, true);
     if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
       return res;
     }
-    res = NBC_Sched_send ((void *)(-gap), true , counts[peer], types[peer], peer, schedule, false);
+    res = NBC_Sched_send ((void *)(-gap), true, ompi_count_array_get(counts, peer), types[peer], peer, schedule, false);
     if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
       return res;
     }
-    res = NBC_Sched_recv (tbuf, false , counts[peer], types[peer], peer, schedule, true);
+    res = NBC_Sched_recv (tbuf, false, ompi_count_array_get(counts, peer), types[peer], peer, schedule, true);
     if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
       return res;
     }
@@ -368,8 +368,8 @@ static inline int a2aw_sched_inplace(int rank, int p, NBC_Schedule *schedule,
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_alltoallw_init(const void* sendbuf, const int *sendcounts, const int *sdispls,
-                                    struct ompi_datatype_t * const *sendtypes, void* recvbuf, const int *recvcounts, const int *rdispls,
+int ompi_coll_libnbc_alltoallw_init(const void* sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t sdispls,
+                                    struct ompi_datatype_t * const *sendtypes, void* recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t rdispls,
                                     struct ompi_datatype_t * const *recvtypes, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                     mca_coll_base_module_t *module) {
     int res = nbc_alltoallw_init(sendbuf, sendcounts, sdispls, sendtypes, recvbuf, recvcounts, rdispls, recvtypes,
@@ -381,8 +381,9 @@ int ompi_coll_libnbc_alltoallw_init(const void* sendbuf, const int *sendcounts,
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_alltoallw_inter_init(const void* sendbuf, const int *sendcounts, const int *sdispls,
-                                          struct ompi_datatype_t * const *sendtypes, void* recvbuf, const int *recvcounts, const int *rdispls,
+int ompi_coll_libnbc_alltoallw_inter_init(const void* sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t sdispls,
+                                          struct ompi_datatype_t * const *sendtypes, void* recvbuf,
+                                          ompi_count_array_t recvcounts, ompi_disp_array_t rdispls,
                                           struct ompi_datatype_t * const *recvtypes, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                           mca_coll_base_module_t *module) {
     int res = nbc_alltoallw_inter_init(sendbuf, sendcounts, sdispls, sendtypes, recvbuf, recvcounts, rdispls, recvtypes,
diff --git a/ompi/mca/coll/libnbc/nbc_ibarrier.c b/ompi/mca/coll/libnbc/nbc_ibarrier.c
index 42891eaba11..5c1ff994050 100644
--- a/ompi/mca/coll/libnbc/nbc_ibarrier.c
+++ b/ompi/mca/coll/libnbc/nbc_ibarrier.c
@@ -45,7 +45,7 @@ static int nbc_barrier_init(struct ompi_communicator_t *comm, ompi_request_t **
       return OMPI_ERR_OUT_OF_RESOURCE;
     }
 
-    maxround = (int)ceil((log((double)p)/LOG2)-1);
+    maxround = ceil_of_log2(p) -1;
 
     for (int round = 0 ; round <= maxround ; ++round) {
       sendpeer = (rank + (1 << round)) % p;
diff --git a/ompi/mca/coll/libnbc/nbc_ibcast.c b/ompi/mca/coll/libnbc/nbc_ibcast.c
index a9069361a9f..6e978382c4a 100644
--- a/ompi/mca/coll/libnbc/nbc_ibcast.c
+++ b/ompi/mca/coll/libnbc/nbc_ibcast.c
@@ -20,14 +20,14 @@
  */
 #include "nbc_internal.h"
 
-static inline int bcast_sched_binomial(int rank, int p, int root, NBC_Schedule *schedule, void *buffer, int count,
+static inline int bcast_sched_binomial(int rank, int p, int root, NBC_Schedule *schedule, void *buffer, size_t count,
                                        MPI_Datatype datatype);
-static inline int bcast_sched_linear(int rank, int p, int root, NBC_Schedule *schedule, void *buffer, int count,
+static inline int bcast_sched_linear(int rank, int p, int root, NBC_Schedule *schedule, void *buffer, size_t count,
                                      MPI_Datatype datatype);
-static inline int bcast_sched_chain(int rank, int p, int root, NBC_Schedule *schedule, void *buffer, int count,
-                                    MPI_Datatype datatype, int fragsize, size_t size);
+static inline int bcast_sched_chain(int rank, int p, int root, NBC_Schedule *schedule, void *buffer, size_t count,
+                                    MPI_Datatype datatype, size_t fragsize, size_t size);
 static inline int bcast_sched_knomial(int rank, int comm_size, int root, NBC_Schedule *schedule, void *buf,
-                                      int count, MPI_Datatype datatype, int knomial_radix);
+                                      size_t count, MPI_Datatype datatype, int knomial_radix);
 
 #ifdef NBC_CACHE_SCHEDULE
 /* tree comparison function for schedule cache */
@@ -47,7 +47,7 @@ int NBC_Bcast_args_compare(NBC_Bcast_args *a, NBC_Bcast_args *b, void *param) {
 }
 #endif
 
-static int nbc_bcast_init(void *buffer, int count, MPI_Datatype datatype, int root,
+static int nbc_bcast_init(void *buffer, size_t count, MPI_Datatype datatype, int root,
                           struct ompi_communicator_t *comm, ompi_request_t ** request,
                           mca_coll_base_module_t *module, bool persistent)
 {
@@ -191,7 +191,7 @@ static int nbc_bcast_init(void *buffer, int count, MPI_Datatype datatype, int ro
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_ibcast(void *buffer, int count, MPI_Datatype datatype, int root,
+int ompi_coll_libnbc_ibcast(void *buffer, size_t count, MPI_Datatype datatype, int root,
                             struct ompi_communicator_t *comm, ompi_request_t ** request,
                             mca_coll_base_module_t *module)
 {
@@ -235,10 +235,10 @@ int ompi_coll_libnbc_ibcast(void *buffer, int count, MPI_Datatype datatype, int
   if (vrank == 0) rank = root; \
   if (vrank == root) rank = 0; \
 }
-static inline int bcast_sched_binomial(int rank, int p, int root, NBC_Schedule *schedule, void *buffer, int count, MPI_Datatype datatype) {
+static inline int bcast_sched_binomial(int rank, int p, int root, NBC_Schedule *schedule, void *buffer, size_t count, MPI_Datatype datatype) {
   int maxr, vrank, peer, res;
 
-  maxr = (int)ceil((log((double)p)/LOG2));
+  maxr = ceil_of_log2(p);
 
   RANK2VRANK(rank, vrank, root);
 
@@ -275,7 +275,7 @@ static inline int bcast_sched_binomial(int rank, int p, int root, NBC_Schedule *
 }
 
 /* simple linear MPI_Ibcast */
-static inline int bcast_sched_linear(int rank, int p, int root, NBC_Schedule *schedule, void *buffer, int count, MPI_Datatype datatype) {
+static inline int bcast_sched_linear(int rank, int p, int root, NBC_Schedule *schedule, void *buffer, size_t count, MPI_Datatype datatype) {
   int res;
 
   /* send to all others */
@@ -301,7 +301,7 @@ static inline int bcast_sched_linear(int rank, int p, int root, NBC_Schedule *sc
 }
 
 /* simple chained MPI_Ibcast */
-static inline int bcast_sched_chain(int rank, int p, int root, NBC_Schedule *schedule, void *buffer, int count, MPI_Datatype datatype, int fragsize, size_t size) {
+static inline int bcast_sched_chain(int rank, int p, int root, NBC_Schedule *schedule, void *buffer, size_t count, MPI_Datatype datatype, size_t fragsize, size_t size) {
   int res, vrank, rpeer, speer, numfrag, fragcount, thiscount;
   MPI_Aint ext;
   char *buf;
@@ -327,11 +327,11 @@ static inline int bcast_sched_chain(int rank, int p, int root, NBC_Schedule *sch
   fragcount = count/numfrag;
 
   for (int fragnum = 0 ; fragnum < numfrag ; ++fragnum) {
-    buf = (char *) buffer + fragnum * fragcount * ext;
+    buf = (char *) buffer + (MPI_Aint)ext * fragnum * fragcount;
     thiscount = fragcount;
     if (fragnum == numfrag-1) {
       /* last fragment may not be full */
-      thiscount = count - fragcount * fragnum;
+      thiscount = count - (size_t)fragcount * fragnum;
     }
 
     /* root does not receive */
@@ -372,7 +372,7 @@ static inline int bcast_sched_chain(int rank, int p, int root, NBC_Schedule *sch
  */
 static inline int bcast_sched_knomial(
     int rank, int comm_size, int root, NBC_Schedule *schedule, void *buf,
-    int count, MPI_Datatype datatype, int knomial_radix)
+    size_t count, MPI_Datatype datatype, int knomial_radix)
 {
     int res = OMPI_SUCCESS;
 
@@ -408,7 +408,7 @@ static inline int bcast_sched_knomial(
     return res;
 }
 
-static int nbc_bcast_inter_init(void *buffer, int count, MPI_Datatype datatype, int root,
+static int nbc_bcast_inter_init(void *buffer, size_t count, MPI_Datatype datatype, int root,
                                 struct ompi_communicator_t *comm, ompi_request_t ** request,
                                 mca_coll_base_module_t *module, bool persistent) {
   int res;
@@ -460,7 +460,7 @@ static int nbc_bcast_inter_init(void *buffer, int count, MPI_Datatype datatype,
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_ibcast_inter(void *buffer, int count, MPI_Datatype datatype, int root,
+int ompi_coll_libnbc_ibcast_inter(void *buffer, size_t count, MPI_Datatype datatype, int root,
                                   struct ompi_communicator_t *comm, ompi_request_t ** request,
                                   mca_coll_base_module_t *module) {
     int res = nbc_bcast_inter_init(buffer, count, datatype, root,
@@ -479,7 +479,7 @@ int ompi_coll_libnbc_ibcast_inter(void *buffer, int count, MPI_Datatype datatype
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_bcast_init(void *buffer, int count, MPI_Datatype datatype, int root,
+int ompi_coll_libnbc_bcast_init(void *buffer, size_t count, MPI_Datatype datatype, int root,
                                 struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                 mca_coll_base_module_t *module) {
     int res = nbc_bcast_init(buffer, count, datatype, root,
@@ -491,7 +491,7 @@ int ompi_coll_libnbc_bcast_init(void *buffer, int count, MPI_Datatype datatype,
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_bcast_inter_init(void *buffer, int count, MPI_Datatype datatype, int root,
+int ompi_coll_libnbc_bcast_inter_init(void *buffer, size_t count, MPI_Datatype datatype, int root,
                                       struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                       mca_coll_base_module_t *module) {
     int res = nbc_bcast_inter_init(buffer, count, datatype, root,
diff --git a/ompi/mca/coll/libnbc/nbc_iexscan.c b/ompi/mca/coll/libnbc/nbc_iexscan.c
index f8a1bff4674..918993e8a63 100644
--- a/ompi/mca/coll/libnbc/nbc_iexscan.c
+++ b/ompi/mca/coll/libnbc/nbc_iexscan.c
@@ -24,12 +24,12 @@
 #include "nbc_internal.h"
 
 static inline int exscan_sched_linear(
-    int rank, int comm_size, const void *sendbuf, void *recvbuf, int count,
+    int rank, int comm_size, const void *sendbuf, void *recvbuf, size_t count,
     MPI_Datatype datatype,  MPI_Op op, char inplace, NBC_Schedule *schedule,
     void *tmpbuf);
 static inline int exscan_sched_recursivedoubling(
     int rank, int comm_size, const void *sendbuf, void *recvbuf,
-    int count, MPI_Datatype datatype,  MPI_Op op, char inplace,
+    size_t count, MPI_Datatype datatype,  MPI_Op op, char inplace,
     NBC_Schedule *schedule, void *tmpbuf1, void *tmpbuf2);
 
 #ifdef NBC_CACHE_SCHEDULE
@@ -51,7 +51,7 @@ int NBC_Scan_args_compare(NBC_Scan_args *a, NBC_Scan_args *b, void *param) {
 }
 #endif
 
-static int nbc_exscan_init(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
+static int nbc_exscan_init(const void* sendbuf, void* recvbuf, size_t count, MPI_Datatype datatype, MPI_Op op,
                            struct ompi_communicator_t *comm, ompi_request_t ** request,
                            mca_coll_base_module_t *module, bool persistent) {
     int rank, p, res;
@@ -165,7 +165,7 @@ static int nbc_exscan_init(const void* sendbuf, void* recvbuf, int count, MPI_Da
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_iexscan(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
+int ompi_coll_libnbc_iexscan(const void* sendbuf, void* recvbuf, size_t count, MPI_Datatype datatype, MPI_Op op,
                              struct ompi_communicator_t *comm, ompi_request_t ** request,
                              mca_coll_base_module_t *module) {
     int res = nbc_exscan_init(sendbuf, recvbuf, count, datatype, op,
@@ -184,7 +184,7 @@ int ompi_coll_libnbc_iexscan(const void* sendbuf, void* recvbuf, int count, MPI_
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_exscan_init(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
+int ompi_coll_libnbc_exscan_init(const void* sendbuf, void* recvbuf, size_t count, MPI_Datatype datatype, MPI_Op op,
                                  struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                  mca_coll_base_module_t *module) {
     int res = nbc_exscan_init(sendbuf, recvbuf, count, datatype, op,
@@ -211,7 +211,7 @@ int ompi_coll_libnbc_exscan_init(const void* sendbuf, void* recvbuf, int count,
  * Schedule length: O(1)
  */
 static inline int exscan_sched_linear(
-    int rank, int comm_size, const void *sendbuf, void *recvbuf, int count,
+    int rank, int comm_size, const void *sendbuf, void *recvbuf, size_t count,
     MPI_Datatype datatype,  MPI_Op op, char inplace, NBC_Schedule *schedule,
     void *tmpbuf)
 {
@@ -293,7 +293,7 @@ static inline int exscan_sched_linear(
  * Schedule length: O(log(p))
  */
 static inline int exscan_sched_recursivedoubling(
-    int rank, int comm_size, const void *sendbuf, void *recvbuf, int count,
+    int rank, int comm_size, const void *sendbuf, void *recvbuf, size_t count,
     MPI_Datatype datatype, MPI_Op op, char inplace,
     NBC_Schedule *schedule, void *tmpbuf1, void *tmpbuf2)
 {
diff --git a/ompi/mca/coll/libnbc/nbc_igather.c b/ompi/mca/coll/libnbc/nbc_igather.c
index 72ce6c51006..dca34eea384 100644
--- a/ompi/mca/coll/libnbc/nbc_igather.c
+++ b/ompi/mca/coll/libnbc/nbc_igather.c
@@ -44,8 +44,8 @@ int NBC_Gather_args_compare(NBC_Gather_args *a, NBC_Gather_args *b, void *param)
 }
 #endif
 
-static int nbc_gather_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf,
-                           int recvcount, MPI_Datatype recvtype, int root,
+static int nbc_gather_init(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf,
+                           size_t recvcount, MPI_Datatype recvtype, int root,
                            struct ompi_communicator_t *comm, ompi_request_t ** request,
                            mca_coll_base_module_t *module, bool persistent) {
   int rank, p, res;
@@ -103,7 +103,7 @@ static int nbc_gather_init(const void* sendbuf, int sendcount, MPI_Datatype send
       }
     } else {
       for (int i = 0 ; i < p ; ++i) {
-        rbuf = (char *)recvbuf + i * recvcount * rcvext;
+        rbuf = (char *)recvbuf + (MPI_Aint) rcvext * i * recvcount;
         if (i == root) {
           if (!inplace) {
             /* if I am the root - just copy the message */
@@ -173,8 +173,8 @@ static int nbc_gather_init(const void* sendbuf, int sendcount, MPI_Datatype send
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_igather(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf,
-                             int recvcount, MPI_Datatype recvtype, int root,
+int ompi_coll_libnbc_igather(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf,
+                             size_t recvcount, MPI_Datatype recvtype, int root,
                              struct ompi_communicator_t *comm, ompi_request_t ** request,
                              mca_coll_base_module_t *module) {
     int res = nbc_gather_init(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root,
@@ -193,8 +193,8 @@ int ompi_coll_libnbc_igather(const void* sendbuf, int sendcount, MPI_Datatype se
     return OMPI_SUCCESS;
 }
 
-static int nbc_gather_inter_init (const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf,
-                                  int recvcount, MPI_Datatype recvtype, int root,
+static int nbc_gather_inter_init (const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf,
+                                  size_t recvcount, MPI_Datatype recvtype, int root,
                                   struct ompi_communicator_t *comm, ompi_request_t ** request,
                                   mca_coll_base_module_t *module, bool persistent) {
     int res, rsize;
@@ -228,7 +228,7 @@ static int nbc_gather_inter_init (const void* sendbuf, int sendcount, MPI_Dataty
         }
     } else if (MPI_ROOT == root) {
         for (int i = 0 ; i < rsize ; ++i) {
-            rbuf = ((char *)recvbuf) + (i * recvcount * rcvext);
+            rbuf = ((char *)recvbuf) + ((MPI_Aint) rcvext * i * recvcount);
             /* root receives message to the right buffer */
             res = NBC_Sched_recv (rbuf, false, recvcount, recvtype, i, schedule, false);
             if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
@@ -253,8 +253,8 @@ static int nbc_gather_inter_init (const void* sendbuf, int sendcount, MPI_Dataty
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_igather_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf,
-                                   int recvcount, MPI_Datatype recvtype, int root,
+int ompi_coll_libnbc_igather_inter(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf,
+                                   size_t recvcount, MPI_Datatype recvtype, int root,
                                    struct ompi_communicator_t *comm, ompi_request_t ** request,
                                    mca_coll_base_module_t *module) {
     int res = nbc_gather_inter_init(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root,
@@ -273,8 +273,8 @@ int ompi_coll_libnbc_igather_inter(const void* sendbuf, int sendcount, MPI_Datat
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_gather_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf,
-                                 int recvcount, MPI_Datatype recvtype, int root,
+int ompi_coll_libnbc_gather_init(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf,
+                                 size_t recvcount, MPI_Datatype recvtype, int root,
                                  struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                  mca_coll_base_module_t *module) {
     int res = nbc_gather_init(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root,
@@ -286,8 +286,8 @@ int ompi_coll_libnbc_gather_init(const void* sendbuf, int sendcount, MPI_Datatyp
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_gather_inter_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf,
-                                       int recvcount, MPI_Datatype recvtype, int root,
+int ompi_coll_libnbc_gather_inter_init(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype, void* recvbuf,
+                                       size_t recvcount, MPI_Datatype recvtype, int root,
                                        struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                        mca_coll_base_module_t *module) {
     int res = nbc_gather_inter_init(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root,
diff --git a/ompi/mca/coll/libnbc/nbc_igatherv.c b/ompi/mca/coll/libnbc/nbc_igatherv.c
index 066817a4e82..1cc1187d868 100644
--- a/ompi/mca/coll/libnbc/nbc_igatherv.c
+++ b/ompi/mca/coll/libnbc/nbc_igatherv.c
@@ -25,12 +25,12 @@
 #include "nbc_internal.h"
 
 /* an gatherv schedule can not be cached easily because the contents
- * ot the recvcounts array may change, so a comparison of the address
+ * of the recvcounts array may change, so a comparison of the address
  * would not be sufficient ... we simply do not cache it */
 
 
 static int nbc_gatherv_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype,
-                            void* recvbuf, const int *recvcounts, const int *displs, MPI_Datatype recvtype,
+                            void* recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t displs, MPI_Datatype recvtype,
                             int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
                             mca_coll_base_module_t *module, bool persistent) {
   int rank, p, res;
@@ -68,12 +68,12 @@ static int nbc_gatherv_init(const void* sendbuf, int sendcount, MPI_Datatype sen
     }
   } else {
     for (int i = 0 ; i < p ; ++i) {
-      rbuf = (char *) recvbuf + displs[i] * rcvext;
+      rbuf = (char *) recvbuf + ompi_disp_array_get(displs, i) * rcvext;
       if (i == root) {
         if (!inplace) {
           /* if I am the root - just copy the message */
           res = NBC_Sched_copy ((void *)sendbuf, false, sendcount, sendtype,
-                                rbuf, false, recvcounts[i], recvtype, schedule, false);
+                                rbuf, false, ompi_count_array_get(recvcounts, i), recvtype, schedule, false);
           if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
             OBJ_RELEASE(schedule);
             return res;
@@ -81,7 +81,7 @@ static int nbc_gatherv_init(const void* sendbuf, int sendcount, MPI_Datatype sen
         }
       } else {
         /* root receives message to the right buffer */
-        res = NBC_Sched_recv (rbuf, false, recvcounts[i], recvtype, i, schedule, false);
+        res = NBC_Sched_recv (rbuf, false, ompi_count_array_get(recvcounts, i), recvtype, i, schedule, false);
         if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
           OBJ_RELEASE(schedule);
           return res;
@@ -105,8 +105,8 @@ static int nbc_gatherv_init(const void* sendbuf, int sendcount, MPI_Datatype sen
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_igatherv(const void* sendbuf, int sendcount, MPI_Datatype sendtype,
-                              void* recvbuf, const int *recvcounts, const int *displs, MPI_Datatype recvtype,
+int ompi_coll_libnbc_igatherv(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype,
+                              void* recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t displs, MPI_Datatype recvtype,
                               int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
                               mca_coll_base_module_t *module) {
     int res = nbc_gatherv_init(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, root,
@@ -125,8 +125,8 @@ int ompi_coll_libnbc_igatherv(const void* sendbuf, int sendcount, MPI_Datatype s
     return OMPI_SUCCESS;
 }
 
-static int nbc_gatherv_inter_init (const void* sendbuf, int sendcount, MPI_Datatype sendtype,
-                                   void* recvbuf, const int *recvcounts, const int *displs, MPI_Datatype recvtype,
+static int nbc_gatherv_inter_init (const void* sendbuf, size_t sendcount, MPI_Datatype sendtype,
+                                   void* recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t displs, MPI_Datatype recvtype,
                                    int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                    mca_coll_base_module_t *module, bool persistent) {
   int res, rsize;
@@ -160,9 +160,9 @@ static int nbc_gatherv_inter_init (const void* sendbuf, int sendcount, MPI_Datat
     }
   } else if (MPI_ROOT == root) {
     for (int i = 0 ; i < rsize ; ++i) {
-      rbuf = (char *) recvbuf + displs[i] * rcvext;
+      rbuf = (char *) recvbuf + ompi_disp_array_get(displs, i) * rcvext;
       /* root receives message to the right buffer */
-      res = NBC_Sched_recv (rbuf, false, recvcounts[i], recvtype, i, schedule, false);
+      res = NBC_Sched_recv (rbuf, false, ompi_count_array_get(recvcounts, i), recvtype, i, schedule, false);
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
         OBJ_RELEASE(schedule);
         return res;
@@ -185,8 +185,8 @@ static int nbc_gatherv_inter_init (const void* sendbuf, int sendcount, MPI_Datat
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_igatherv_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype,
-                                    void* recvbuf, const int *recvcounts, const int *displs, MPI_Datatype recvtype,
+int ompi_coll_libnbc_igatherv_inter(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype,
+                                    void* recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t displs, MPI_Datatype recvtype,
                                     int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                     mca_coll_base_module_t *module) {
     int res = nbc_gatherv_inter_init(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, root,
@@ -205,8 +205,8 @@ int ompi_coll_libnbc_igatherv_inter(const void* sendbuf, int sendcount, MPI_Data
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_gatherv_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype,
-                                  void* recvbuf, const int *recvcounts, const int *displs, MPI_Datatype recvtype,
+int ompi_coll_libnbc_gatherv_init(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype,
+                                  void* recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t displs, MPI_Datatype recvtype,
                                   int root, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                   mca_coll_base_module_t *module) {
     int res = nbc_gatherv_init(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, root,
@@ -218,8 +218,8 @@ int ompi_coll_libnbc_gatherv_init(const void* sendbuf, int sendcount, MPI_Dataty
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_gatherv_inter_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype,
-                                        void* recvbuf, const int *recvcounts, const int *displs, MPI_Datatype recvtype,
+int ompi_coll_libnbc_gatherv_inter_init(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype,
+                                        void* recvbuf, ompi_count_array_t recvcounts, ompi_disp_array_t displs, MPI_Datatype recvtype,
                                         int root, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                         mca_coll_base_module_t *module) {
     int res = nbc_gatherv_inter_init(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, root,
diff --git a/ompi/mca/coll/libnbc/nbc_ineighbor_allgather.c b/ompi/mca/coll/libnbc/nbc_ineighbor_allgather.c
index b4e2915ddf6..54ad66484d5 100644
--- a/ompi/mca/coll/libnbc/nbc_ineighbor_allgather.c
+++ b/ompi/mca/coll/libnbc/nbc_ineighbor_allgather.c
@@ -44,8 +44,8 @@ int NBC_Ineighbor_allgather_args_compare(NBC_Ineighbor_allgather_args *a, NBC_In
 #endif
 
 
-static int nbc_neighbor_allgather_init(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
-                                       int rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm,
+static int nbc_neighbor_allgather_init(const void *sbuf, size_t scount, MPI_Datatype stype, void *rbuf,
+                                       size_t rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm,
                                        ompi_request_t ** request,
                                        mca_coll_base_module_t *module, bool persistent) {
   int res, indegree, outdegree, *srcs, *dsts;
@@ -86,7 +86,7 @@ static int nbc_neighbor_allgather_init(const void *sbuf, int scount, MPI_Datatyp
 
     for (int i = 0 ; i < indegree ; ++i) {
       if (MPI_PROC_NULL != srcs[i]) {
-        res = NBC_Sched_recv ((char *) rbuf + i * rcount * rcvext, true, rcount, rtype, srcs[i], schedule, false);
+        res = NBC_Sched_recv ((char *) rbuf + (MPI_Aint) rcvext * i * rcount, true, rcount, rtype, srcs[i], schedule, false);
         if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
           break;
         }
@@ -163,8 +163,8 @@ static int nbc_neighbor_allgather_init(const void *sbuf, int scount, MPI_Datatyp
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_ineighbor_allgather(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
-                                         int rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm,
+int ompi_coll_libnbc_ineighbor_allgather(const void *sbuf, size_t scount, MPI_Datatype stype, void *rbuf,
+                                         size_t rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm,
                                          ompi_request_t ** request, mca_coll_base_module_t *module) {
     int res = nbc_neighbor_allgather_init(sbuf, scount, stype, rbuf, rcount, rtype,
                                           comm, request, module, false);
@@ -182,8 +182,8 @@ int ompi_coll_libnbc_ineighbor_allgather(const void *sbuf, int scount, MPI_Datat
 }
 
 
-int ompi_coll_libnbc_neighbor_allgather_init(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
-                                             int rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm,
+int ompi_coll_libnbc_neighbor_allgather_init(const void *sbuf, size_t scount, MPI_Datatype stype, void *rbuf,
+                                             size_t rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm,
                                              MPI_Info info, ompi_request_t ** request, mca_coll_base_module_t *module) {
     int res = nbc_neighbor_allgather_init(sbuf, scount, stype, rbuf, rcount, rtype,
                                           comm, request, module, true);
diff --git a/ompi/mca/coll/libnbc/nbc_ineighbor_allgatherv.c b/ompi/mca/coll/libnbc/nbc_ineighbor_allgatherv.c
index 3994c578657..c3a978c0dd6 100644
--- a/ompi/mca/coll/libnbc/nbc_ineighbor_allgatherv.c
+++ b/ompi/mca/coll/libnbc/nbc_ineighbor_allgatherv.c
@@ -45,7 +45,7 @@ int NBC_Ineighbor_allgatherv_args_compare(NBC_Ineighbor_allgatherv_args *a, NBC_
 
 
 static int nbc_neighbor_allgatherv_init(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
-                                        const int *rcounts, const int *displs, MPI_Datatype rtype,
+                                        ompi_count_array_t rcounts, ompi_disp_array_t displs, MPI_Datatype rtype,
                                         struct ompi_communicator_t *comm, ompi_request_t ** request,
                                         mca_coll_base_module_t *module, bool persistent) {
   int res, indegree, outdegree, *srcs, *dsts;
@@ -87,7 +87,8 @@ static int nbc_neighbor_allgatherv_init(const void *sbuf, int scount, MPI_Dataty
     /* simply loop over neighbors and post send/recv operations */
     for (int i = 0 ; i < indegree ; ++i) {
       if (srcs[i] != MPI_PROC_NULL) {
-        res = NBC_Sched_recv ((char *) rbuf + displs[i] * rcvext, false, rcounts[i], rtype, srcs[i], schedule, false);
+        res = NBC_Sched_recv ((char *) rbuf + ompi_disp_array_get(displs, i) * rcvext,
+                              false, ompi_count_array_get(rcounts, i), rtype, srcs[i], schedule, false);
         if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
           break;
         }
@@ -164,8 +165,8 @@ static int nbc_neighbor_allgatherv_init(const void *sbuf, int scount, MPI_Dataty
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_ineighbor_allgatherv(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
-					  const int *rcounts, const int *displs, MPI_Datatype rtype,
+int ompi_coll_libnbc_ineighbor_allgatherv(const void *sbuf, size_t scount, MPI_Datatype stype, void *rbuf,
+					  ompi_count_array_t rcounts, ompi_disp_array_t displs, MPI_Datatype rtype,
 					  struct ompi_communicator_t *comm, ompi_request_t ** request,
 					  mca_coll_base_module_t *module) {
     int res = nbc_neighbor_allgatherv_init(sbuf, scount, stype, rbuf, rcounts, displs, rtype,
@@ -183,8 +184,8 @@ int ompi_coll_libnbc_ineighbor_allgatherv(const void *sbuf, int scount, MPI_Data
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_neighbor_allgatherv_init(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
-                                              const int *rcounts, const int *displs, MPI_Datatype rtype,
+int ompi_coll_libnbc_neighbor_allgatherv_init(const void *sbuf, size_t scount, MPI_Datatype stype, void *rbuf,
+                                              ompi_count_array_t rcounts, ompi_disp_array_t displs, MPI_Datatype rtype,
                                               struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                               mca_coll_base_module_t *module) {
     int res = nbc_neighbor_allgatherv_init(sbuf, scount, stype, rbuf, rcounts, displs, rtype,
diff --git a/ompi/mca/coll/libnbc/nbc_ineighbor_alltoall.c b/ompi/mca/coll/libnbc/nbc_ineighbor_alltoall.c
index b830105d90c..aa2907147dd 100644
--- a/ompi/mca/coll/libnbc/nbc_ineighbor_alltoall.c
+++ b/ompi/mca/coll/libnbc/nbc_ineighbor_alltoall.c
@@ -41,8 +41,8 @@ int NBC_Ineighbor_alltoall_args_compare(NBC_Ineighbor_alltoall_args *a, NBC_Inei
 }
 #endif
 
-static int nbc_neighbor_alltoall_init(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
-                                      int rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm,
+static int nbc_neighbor_alltoall_init(const void *sbuf, size_t scount, MPI_Datatype stype, void *rbuf,
+                                      size_t rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm,
                                       ompi_request_t ** request,
                                       mca_coll_base_module_t *module, bool persistent) {
   int res, indegree, outdegree, *srcs, *dsts;
@@ -89,7 +89,7 @@ static int nbc_neighbor_alltoall_init(const void *sbuf, int scount, MPI_Datatype
 
     for (int i = 0 ; i < indegree ; ++i) {
       if (MPI_PROC_NULL != srcs[i]) {
-        res = NBC_Sched_recv ((char *) rbuf + i * rcount * rcvext, true, rcount, rtype, srcs[i], schedule, false);
+        res = NBC_Sched_recv ((char *) rbuf + (MPI_Aint) rcvext * i * rcount, true, rcount, rtype, srcs[i], schedule, false);
         if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
           break;
         }
@@ -106,7 +106,7 @@ static int nbc_neighbor_alltoall_init(const void *sbuf, int scount, MPI_Datatype
 
     for (int i = 0 ; i < outdegree ; ++i) {
       if (MPI_PROC_NULL != dsts[i]) {
-        res = NBC_Sched_send ((char *) sbuf + i * scount * sndext, false, scount, stype, dsts[i], schedule, false);
+        res = NBC_Sched_send ((char *) sbuf + (MPI_Aint) sndext * i * scount, false, scount, stype, dsts[i], schedule, false);
         if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
           break;
         }
@@ -167,8 +167,8 @@ static int nbc_neighbor_alltoall_init(const void *sbuf, int scount, MPI_Datatype
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_ineighbor_alltoall(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
-                                        int rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm,
+int ompi_coll_libnbc_ineighbor_alltoall(const void *sbuf, size_t scount, MPI_Datatype stype, void *rbuf,
+                                        size_t rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm,
                                         ompi_request_t ** request, mca_coll_base_module_t *module) {
     int res = nbc_neighbor_alltoall_init(sbuf, scount, stype, rbuf, rcount, rtype,
                                          comm, request, module, false);
@@ -185,8 +185,8 @@ int ompi_coll_libnbc_ineighbor_alltoall(const void *sbuf, int scount, MPI_Dataty
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_neighbor_alltoall_init(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
-                                            int rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm, MPI_Info info,
+int ompi_coll_libnbc_neighbor_alltoall_init(const void *sbuf, size_t scount, MPI_Datatype stype, void *rbuf,
+                                            size_t rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm, MPI_Info info,
                                             ompi_request_t ** request, mca_coll_base_module_t *module) {
     int res = nbc_neighbor_alltoall_init(sbuf, scount, stype, rbuf, rcount, rtype,
                                          comm, request, module, true);
diff --git a/ompi/mca/coll/libnbc/nbc_ineighbor_alltoallv.c b/ompi/mca/coll/libnbc/nbc_ineighbor_alltoallv.c
index 763c017692f..d711961f45e 100644
--- a/ompi/mca/coll/libnbc/nbc_ineighbor_alltoallv.c
+++ b/ompi/mca/coll/libnbc/nbc_ineighbor_alltoallv.c
@@ -44,8 +44,8 @@ int NBC_Ineighbor_alltoallv_args_compare(NBC_Ineighbor_alltoallv_args *a, NBC_In
 #endif
 
 
-static int nbc_neighbor_alltoallv_init(const void *sbuf, const int *scounts, const int *sdispls, MPI_Datatype stype,
-                                       void *rbuf, const int *rcounts, const int *rdispls, MPI_Datatype rtype,
+static int nbc_neighbor_alltoallv_init(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdispls, MPI_Datatype stype,
+                                       void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdispls, MPI_Datatype rtype,
                                        struct ompi_communicator_t *comm, ompi_request_t ** request,
                                        mca_coll_base_module_t *module, bool persistent) {
   int res, indegree, outdegree, *srcs, *dsts;
@@ -93,7 +93,8 @@ static int nbc_neighbor_alltoallv_init(const void *sbuf, const int *scounts, con
     /* simply loop over neighbors and post send/recv operations */
     for (int i = 0 ; i < indegree ; ++i) {
       if (srcs[i] != MPI_PROC_NULL) {
-        res = NBC_Sched_recv ((char *) rbuf + rdispls[i] * rcvext, false, rcounts[i], rtype, srcs[i], schedule, false);
+        res = NBC_Sched_recv ((char *) rbuf + ompi_disp_array_get(rdispls, i) * rcvext, false,
+                              ompi_count_array_get(rcounts, i), rtype, srcs[i], schedule, false);
         if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
           break;
         }
@@ -110,7 +111,8 @@ static int nbc_neighbor_alltoallv_init(const void *sbuf, const int *scounts, con
 
     for (int i = 0 ; i < outdegree ; ++i) {
       if (dsts[i] != MPI_PROC_NULL) {
-        res = NBC_Sched_send ((char *) sbuf + sdispls[i] * sndext, false, scounts[i], stype, dsts[i], schedule, false);
+        res = NBC_Sched_send ((char *) sbuf + ompi_disp_array_get(sdispls, i) * sndext, false,
+                              ompi_count_array_get(scounts, i), stype, dsts[i], schedule, false);
         if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
           break;
         }
@@ -171,8 +173,8 @@ static int nbc_neighbor_alltoallv_init(const void *sbuf, const int *scounts, con
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_ineighbor_alltoallv(const void *sbuf, const int *scounts, const int *sdispls, MPI_Datatype stype,
-                                         void *rbuf, const int *rcounts, const int *rdispls, MPI_Datatype rtype,
+int ompi_coll_libnbc_ineighbor_alltoallv(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdispls, MPI_Datatype stype,
+                                         void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdispls, MPI_Datatype rtype,
                                          struct ompi_communicator_t *comm, ompi_request_t ** request,
                                          mca_coll_base_module_t *module) {
     int res = nbc_neighbor_alltoallv_init(sbuf, scounts, sdispls, stype, rbuf, rcounts, rdispls, rtype,
@@ -190,8 +192,8 @@ int ompi_coll_libnbc_ineighbor_alltoallv(const void *sbuf, const int *scounts, c
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_neighbor_alltoallv_init(const void *sbuf, const int *scounts, const int *sdispls, MPI_Datatype stype,
-                                             void *rbuf, const int *rcounts, const int *rdispls, MPI_Datatype rtype,
+int ompi_coll_libnbc_neighbor_alltoallv_init(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdispls, MPI_Datatype stype,
+                                             void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdispls, MPI_Datatype rtype,
                                              struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                              mca_coll_base_module_t *module) {
     int res = nbc_neighbor_alltoallv_init(sbuf, scounts, sdispls, stype, rbuf, rcounts, rdispls, rtype,
diff --git a/ompi/mca/coll/libnbc/nbc_ineighbor_alltoallw.c b/ompi/mca/coll/libnbc/nbc_ineighbor_alltoallw.c
index 021f71a5dd7..cd6d2bb0502 100644
--- a/ompi/mca/coll/libnbc/nbc_ineighbor_alltoallw.c
+++ b/ompi/mca/coll/libnbc/nbc_ineighbor_alltoallw.c
@@ -43,8 +43,8 @@ int NBC_Ineighbor_alltoallw_args_compare(NBC_Ineighbor_alltoallw_args *a, NBC_In
 }
 #endif
 
-static int nbc_neighbor_alltoallw_init(const void *sbuf, const int *scounts, const MPI_Aint *sdisps, struct ompi_datatype_t * const *stypes,
-                                       void *rbuf, const int *rcounts, const MPI_Aint *rdisps, struct ompi_datatype_t * const *rtypes,
+static int nbc_neighbor_alltoallw_init(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps, struct ompi_datatype_t * const *stypes,
+                                       void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps, struct ompi_datatype_t * const *rtypes,
                                        struct ompi_communicator_t *comm, ompi_request_t ** request,
                                        mca_coll_base_module_t *module, bool persistent) {
   int res, indegree, outdegree, *srcs, *dsts;
@@ -79,7 +79,8 @@ static int nbc_neighbor_alltoallw_init(const void *sbuf, const int *scounts, con
     /* simply loop over neighbors and post send/recv operations */
     for (int i = 0 ; i < indegree ; ++i) {
       if (srcs[i] != MPI_PROC_NULL) {
-        res = NBC_Sched_recv ((char *) rbuf + rdisps[i], false, rcounts[i], rtypes[i], srcs[i], schedule, false);
+        res = NBC_Sched_recv ((char *) rbuf + ompi_disp_array_get(rdisps, i), false,
+                              ompi_count_array_get(rcounts, i), rtypes[i], srcs[i], schedule, false);
         if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
           break;
         }
@@ -96,7 +97,8 @@ static int nbc_neighbor_alltoallw_init(const void *sbuf, const int *scounts, con
 
     for (int i = 0 ; i < outdegree ; ++i) {
       if (dsts[i] != MPI_PROC_NULL) {
-        res = NBC_Sched_send ((char *) sbuf + sdisps[i], false, scounts[i], stypes[i], dsts[i], schedule, false);
+        res = NBC_Sched_send ((char *) sbuf + ompi_disp_array_get(sdisps, i), false,
+                              ompi_count_array_get(scounts, i), stypes[i], dsts[i], schedule, false);
         if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
           break;
         }
@@ -156,8 +158,8 @@ static int nbc_neighbor_alltoallw_init(const void *sbuf, const int *scounts, con
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_ineighbor_alltoallw(const void *sbuf, const int *scounts, const MPI_Aint *sdisps, struct ompi_datatype_t * const *stypes,
-                                         void *rbuf, const int *rcounts, const MPI_Aint *rdisps, struct ompi_datatype_t * const *rtypes,
+int ompi_coll_libnbc_ineighbor_alltoallw(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps, struct ompi_datatype_t * const *stypes,
+                                         void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps, struct ompi_datatype_t * const *rtypes,
                                          struct ompi_communicator_t *comm, ompi_request_t ** request,
                                          mca_coll_base_module_t *module) {
     int res = nbc_neighbor_alltoallw_init(sbuf, scounts, sdisps, stypes, rbuf, rcounts, rdisps, rtypes,
@@ -175,8 +177,8 @@ int ompi_coll_libnbc_ineighbor_alltoallw(const void *sbuf, const int *scounts, c
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_neighbor_alltoallw_init(const void *sbuf, const int *scounts, const MPI_Aint *sdisps, struct ompi_datatype_t * const *stypes,
-                                             void *rbuf, const int *rcounts, const MPI_Aint *rdisps, struct ompi_datatype_t * const *rtypes,
+int ompi_coll_libnbc_neighbor_alltoallw_init(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps, struct ompi_datatype_t * const *stypes,
+                                             void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps, struct ompi_datatype_t * const *rtypes,
                                              struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                              mca_coll_base_module_t *module) {
     int res = nbc_neighbor_alltoallw_init(sbuf, scounts, sdisps, stypes, rbuf, rcounts, rdisps, rtypes,
diff --git a/ompi/mca/coll/libnbc/nbc_internal.h b/ompi/mca/coll/libnbc/nbc_internal.h
index 4d5f42bc060..f18dd90e776 100644
--- a/ompi/mca/coll/libnbc/nbc_internal.h
+++ b/ompi/mca/coll/libnbc/nbc_internal.h
@@ -5,16 +5,19 @@
  *                    Corporation.  All rights reserved.
  * Copyright (c) 2006 The Technical University of Chemnitz. All
  *                    rights reserved.
+ * Copyright (c) 2022 IBM Corporation. All rights reserved
  *
  * Author(s): Torsten Hoefler <htor@cs.indiana.edu>
  *
  * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
  * Copyright (c) 2014      NVIDIA Corporation.  All rights reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
+ * Copyright (c) 2015-2021 Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2021      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -29,10 +32,7 @@
 #include "mpi.h"
 
 #include "coll_libnbc.h"
-#if OPAL_CUDA_SUPPORT
-#include "opal/datatype/opal_convertor.h"
-#include "opal/mca/common/cuda/common_cuda.h"
-#endif /* OPAL_CUDA_SUPPORT */
+#include "opal/mca/accelerator/accelerator.h"
 #include "ompi/include/ompi/constants.h"
 #include "ompi/request/request.h"
 #include "ompi/datatype/ompi_datatype.h"
@@ -50,8 +50,15 @@
 extern "C" {
 #endif
 
-/* log(2) */
-#define LOG2 0.69314718055994530941
+/* Dividing very close floats may lead to unexpected roundings */
+static inline int
+ceil_of_log2 (int val) {
+    int ret = 0;
+    while (1 << ret < val) {
+        ret ++;
+    }
+    return ret;
+}
 
 /* true/false */
 #define true 1
@@ -90,7 +97,7 @@ typedef enum {
 /* the send argument struct */
 typedef struct {
   NBC_Fn_type type;
-  int count;
+  size_t count;
   const void *buf;
   MPI_Datatype datatype;
   int dest;
@@ -101,7 +108,7 @@ typedef struct {
 /* the receive argument struct */
 typedef struct {
   NBC_Fn_type type;
-  int count;
+  size_t count;
   void *buf;
   MPI_Datatype datatype;
   char tmpbuf;
@@ -118,18 +125,18 @@ typedef struct {
   void *buf2;
   MPI_Op op;
   MPI_Datatype datatype;
-  int count;
+  size_t count;
 } NBC_Args_op;
 
 /* the copy argument struct */
 typedef struct {
   NBC_Fn_type type;
-  int srccount;
+  size_t srccount;
   void *src;
   void *tgt;
   MPI_Datatype srctype;
   MPI_Datatype tgttype;
-  int tgtcount;
+  size_t tgtcount;
   char tmpsrc;
   char tmptgt;
 } NBC_Args_copy;
@@ -137,7 +144,7 @@ typedef struct {
 /* unpack operation arguments */
 typedef struct {
   NBC_Fn_type type;
-  int count;
+  size_t count;
   void *inbuf;
   void *outbuf;
   MPI_Datatype datatype;
@@ -146,15 +153,15 @@ typedef struct {
 } NBC_Args_unpack;
 
 /* internal function prototypes */
-int NBC_Sched_send (const void* buf, char tmpbuf, int count, MPI_Datatype datatype, int dest, NBC_Schedule *schedule, bool barrier);
-int NBC_Sched_local_send (const void* buf, char tmpbuf, int count, MPI_Datatype datatype, int dest,NBC_Schedule *schedule, bool barrier);
-int NBC_Sched_recv (void* buf, char tmpbuf, int count, MPI_Datatype datatype, int source, NBC_Schedule *schedule, bool barrier);
-int NBC_Sched_local_recv (void* buf, char tmpbuf, int count, MPI_Datatype datatype, int source, NBC_Schedule *schedule, bool barrier);
-int NBC_Sched_op (const void* buf1, char tmpbuf1, void* buf2, char tmpbuf2, int count, MPI_Datatype datatype,
+int NBC_Sched_send (const void* buf, char tmpbuf, size_t count, MPI_Datatype datatype, int dest, NBC_Schedule *schedule, bool barrier);
+int NBC_Sched_local_send (const void* buf, char tmpbuf, size_t count, MPI_Datatype datatype, int dest,NBC_Schedule *schedule, bool barrier);
+int NBC_Sched_recv (void* buf, char tmpbuf, size_t count, MPI_Datatype datatype, int source, NBC_Schedule *schedule, bool barrier);
+int NBC_Sched_local_recv (void* buf, char tmpbuf, size_t count, MPI_Datatype datatype, int source, NBC_Schedule *schedule, bool barrier);
+int NBC_Sched_op (const void* buf1, char tmpbuf1, void* buf2, char tmpbuf2, size_t count, MPI_Datatype datatype,
                   MPI_Op op, NBC_Schedule *schedule, bool barrier);
-int NBC_Sched_copy (void *src, char tmpsrc, int srccount, MPI_Datatype srctype, void *tgt, char tmptgt, int tgtcount,
+int NBC_Sched_copy (void *src, char tmpsrc, size_t srccount, MPI_Datatype srctype, void *tgt, char tmptgt, size_t tgtcount,
                     MPI_Datatype tgttype, NBC_Schedule *schedule, bool barrier);
-int NBC_Sched_unpack (void *inbuf, char tmpinbuf, int count, MPI_Datatype datatype, void *outbuf, char tmpoutbuf,
+int NBC_Sched_unpack (void *inbuf, char tmpinbuf, size_t count, MPI_Datatype datatype, void *outbuf, char tmpoutbuf,
                       NBC_Schedule *schedule, bool barrier);
 
 int NBC_Sched_barrier (NBC_Schedule *schedule);
@@ -172,10 +179,10 @@ struct NBC_dummyarg {
 typedef struct {
   NBC_Schedule *schedule;
   void *sendbuf;
-  int sendcount;
+  size_t sendcount;
   MPI_Datatype sendtype;
   void* recvbuf;
-  int recvcount;
+  size_t recvcount;
   MPI_Datatype recvtype;
 } NBC_Alltoall_args;
 int NBC_Alltoall_args_compare(NBC_Alltoall_args *a, NBC_Alltoall_args *b, void *param);
@@ -183,10 +190,10 @@ int NBC_Alltoall_args_compare(NBC_Alltoall_args *a, NBC_Alltoall_args *b, void *
 typedef struct {
   NBC_Schedule *schedule;
   void *sendbuf;
-  int sendcount;
+  size_t sendcount;
   MPI_Datatype sendtype;
   void* recvbuf;
-  int recvcount;
+  size_t recvcount;
   MPI_Datatype recvtype;
 } NBC_Allgather_args;
 int NBC_Allgather_args_compare(NBC_Allgather_args *a, NBC_Allgather_args *b, void *param);
@@ -195,7 +202,7 @@ typedef struct {
   NBC_Schedule *schedule;
   void *sendbuf;
   void* recvbuf;
-  int count;
+  size_t count;
   MPI_Datatype datatype;
   MPI_Op op;
 } NBC_Allreduce_args;
@@ -204,7 +211,7 @@ int NBC_Allreduce_args_compare(NBC_Allreduce_args *a, NBC_Allreduce_args *b, voi
 typedef struct {
   NBC_Schedule *schedule;
   void *buffer;
-  int count;
+  size_t count;
   MPI_Datatype datatype;
   int root;
 } NBC_Bcast_args;
@@ -213,10 +220,10 @@ int NBC_Bcast_args_compare(NBC_Bcast_args *a, NBC_Bcast_args *b, void *param);
 typedef struct {
   NBC_Schedule *schedule;
   void *sendbuf;
-  int sendcount;
+  size_t sendcount;
   MPI_Datatype sendtype;
   void* recvbuf;
-  int recvcount;
+  size_t recvcount;
   MPI_Datatype recvtype;
   int root;
 } NBC_Gather_args;
@@ -226,7 +233,7 @@ typedef struct {
   NBC_Schedule *schedule;
   void *sendbuf;
   void* recvbuf;
-  int count;
+  size_t count;
   MPI_Datatype datatype;
   MPI_Op op;
   int root;
@@ -237,7 +244,7 @@ typedef struct {
   NBC_Schedule *schedule;
   void *sendbuf;
   void* recvbuf;
-  int count;
+  size_t count;
   MPI_Datatype datatype;
   MPI_Op op;
 } NBC_Scan_args;
@@ -246,10 +253,10 @@ int NBC_Scan_args_compare(NBC_Scan_args *a, NBC_Scan_args *b, void *param);
 typedef struct {
   NBC_Schedule *schedule;
   void *sendbuf;
-  int sendcount;
+  size_t sendcount;
   MPI_Datatype sendtype;
   void* recvbuf;
-  int recvcount;
+  size_t recvcount;
   MPI_Datatype recvtype;
   int root;
 } NBC_Scatter_args;
@@ -499,7 +506,7 @@ static inline int NBC_Type_intrinsic(MPI_Datatype type) {
 }
 
 /* let's give a try to inline functions */
-static inline int NBC_Copy(const void *src, int srccount, MPI_Datatype srctype, void *tgt, int tgtcount, MPI_Datatype tgttype, MPI_Comm comm) {
+static inline int NBC_Copy(const void *src, size_t srccount, MPI_Datatype srctype, void *tgt, size_t tgtcount, MPI_Datatype tgttype, MPI_Comm comm) {
   int res;
 
   res = ompi_datatype_sndrcv(src, srccount, srctype, tgt, tgtcount, tgttype);
@@ -511,21 +518,30 @@ static inline int NBC_Copy(const void *src, int srccount, MPI_Datatype srctype,
   return OMPI_SUCCESS;
 }
 
-static inline int NBC_Unpack(void *src, int srccount, MPI_Datatype srctype, void *tgt, MPI_Comm comm) {
+static inline int NBC_Unpack(void *src, size_t srccount, MPI_Datatype srctype, void *tgt, MPI_Comm comm) {
   MPI_Aint size, pos;
   int res;
   ptrdiff_t ext, lb;
+  uint64_t flags;
+  int is_accel_buf1, is_accel_buf2;
+  int dev_id;
 
   res = ompi_datatype_pack_external_size("external32", srccount, srctype, &size);
   if (OMPI_SUCCESS != res) {
     NBC_Error ("MPI Error in ompi_datatype_pack_external_size() (%i)", res);
     return res;
   }
-#if OPAL_CUDA_SUPPORT
-  if(NBC_Type_intrinsic(srctype) && !(opal_cuda_check_bufs((char *)tgt, (char *)src))) {
-#else
-  if(NBC_Type_intrinsic(srctype)) {
-#endif /* OPAL_CUDA_SUPPORT */
+
+  is_accel_buf1 = opal_accelerator.check_addr(tgt, &dev_id, &flags);
+  is_accel_buf2 = opal_accelerator.check_addr(src, &dev_id, &flags);
+  if (is_accel_buf1 < 0) {
+    return is_accel_buf1;
+  } else if (is_accel_buf2 < 0) {
+    return is_accel_buf2;
+  }
+  if(NBC_Type_intrinsic(srctype) &&
+     is_accel_buf1 == 0 &&
+     is_accel_buf2 == 0) {
     /* if we have the same types and they are contiguous (intrinsic
      * types are contiguous), we can just use a single memcpy */
     res = ompi_datatype_get_extent (srctype, &lb, &ext);
@@ -549,22 +565,22 @@ static inline int NBC_Unpack(void *src, int srccount, MPI_Datatype srctype, void
   return OMPI_SUCCESS;
 }
 
-/* deletes elements from dict until low watermark is reached */
-static inline void NBC_SchedCache_dictwipe(hb_tree *dict, int *size) {
+/* deletes elements from dict_in until low watermark is reached */
+static inline void NBC_SchedCache_dictwipe(hb_tree *dict_in, int *size) {
   hb_itor *itor;
 
-  itor = hb_itor_new(dict);
-  for (; hb_itor_valid(itor) && (*size>NBC_SCHED_DICT_LOWER); hb_itor_next(itor)) {
-    hb_tree_remove(dict, hb_itor_key(itor), 0);
+  itor = ompi_coll_libnbc_hb_itor_new(dict_in);
+  for (; ompi_coll_libnbc_hb_itor_valid(itor) && (*size>NBC_SCHED_DICT_LOWER); ompi_coll_libnbc_hb_itor_next(itor)) {
+    ompi_coll_libnbc_hb_tree_remove(dict_in, ompi_coll_libnbc_hb_itor_key(itor), 0);
     *size = *size-1;
   }
-  hb_itor_destroy(itor);
+  ompi_coll_libnbc_hb_itor_destroy(itor);
 }
 
 #define NBC_IN_PLACE(sendbuf, recvbuf, inplace) \
 { \
   inplace = 0; \
-  if(recvbuf == sendbuf) { \
+  if(recvbuf == sendbuf && MPI_BOTTOM != sendbuf) { \
     inplace = 1; \
   } else \
   if(sendbuf == MPI_IN_PLACE) { \
diff --git a/ompi/mca/coll/libnbc/nbc_ireduce.c b/ompi/mca/coll/libnbc/nbc_ireduce.c
index 819c92880b9..2049386e9c5 100644
--- a/ompi/mca/coll/libnbc/nbc_ireduce.c
+++ b/ompi/mca/coll/libnbc/nbc_ireduce.c
@@ -26,16 +26,16 @@
 
 #include "nbc_internal.h"
 
-static inline int red_sched_binomial (int rank, int p, int root, const void *sendbuf, void *redbuf, char tmpredbuf, int count, MPI_Datatype datatype,
+static inline int red_sched_binomial (int rank, int p, int root, const void *sendbuf, void *redbuf, char tmpredbuf, size_t count, MPI_Datatype datatype,
                                       MPI_Op op, char inplace, NBC_Schedule *schedule, void *tmpbuf);
-static inline int red_sched_chain (int rank, int p, int root, const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
-                                   MPI_Op op, int ext, size_t size, NBC_Schedule *schedule, void *tmpbuf, int fragsize);
+static inline int red_sched_chain (int rank, int p, int root, const void *sendbuf, void *recvbuf, size_t count, MPI_Datatype datatype,
+                                   MPI_Op op, MPI_Aint ext, size_t size, NBC_Schedule *schedule, void *tmpbuf, size_t fragsize);
 
-static inline int red_sched_linear (int rank, int rsize, int root, const void *sendbuf, void *recvbuf, void *tmpbuf, int count, MPI_Datatype datatype,
+static inline int red_sched_linear (int rank, int rsize, int root, const void *sendbuf, void *recvbuf, void *tmpbuf, size_t count, MPI_Datatype datatype,
                                     MPI_Op op, NBC_Schedule *schedule);
 static inline int red_sched_redscat_gather(
     int rank, int comm_size, int root, const void *sbuf, void *rbuf,
-    char tmpredbuf, int count, MPI_Datatype datatype, MPI_Op op, char inplace,
+    char tmpredbuf, size_t count, MPI_Datatype datatype, MPI_Op op, char inplace,
     NBC_Schedule *schedule, void *tmp_buf, struct ompi_communicator_t *comm);
 
 #ifdef NBC_CACHE_SCHEDULE
@@ -59,7 +59,7 @@ int NBC_Reduce_args_compare(NBC_Reduce_args *a, NBC_Reduce_args *b, void *param)
 #endif
 
 /* the non-blocking reduce */
-static int nbc_reduce_init(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype,
+static int nbc_reduce_init(const void* sendbuf, void* recvbuf, size_t count, MPI_Datatype datatype,
                            MPI_Op op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
                            mca_coll_base_module_t *module, bool persistent) {
   int rank, p, res, segsize;
@@ -106,7 +106,7 @@ static int nbc_reduce_init(const void* sendbuf, void* recvbuf, int count, MPI_Da
   /* algorithm selection */
   int nprocs_pof2 = opal_next_poweroftwo(p) >> 1;
   if (libnbc_ireduce_algorithm == 0) {
-    if (ompi_op_is_commute(op) && p > 2 && count >= nprocs_pof2) {
+    if (ompi_op_is_commute(op) && p > 2 && count >= (size_t) nprocs_pof2) {
       alg = NBC_RED_REDSCAT_GATHER;
     } else if (p > 4 || size * count < 65536 || !ompi_op_is_commute(op)) {
       alg = NBC_RED_BINOMIAL;
@@ -118,7 +118,7 @@ static int nbc_reduce_init(const void* sendbuf, void* recvbuf, int count, MPI_Da
       alg = NBC_RED_CHAIN;
     } else if (libnbc_ireduce_algorithm == 2) {
       alg = NBC_RED_BINOMIAL;
-    } else if (libnbc_ireduce_algorithm == 3 && ompi_op_is_commute(op) && p > 2 && count >= nprocs_pof2) {
+    } else if (libnbc_ireduce_algorithm == 3 && ompi_op_is_commute(op) && p > 2 && count >= (size_t) nprocs_pof2) {
       alg = NBC_RED_REDSCAT_GATHER;
     } else {
       alg = NBC_RED_CHAIN;
@@ -237,7 +237,7 @@ static int nbc_reduce_init(const void* sendbuf, void* recvbuf, int count, MPI_Da
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_ireduce(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype,
+int ompi_coll_libnbc_ireduce(const void* sendbuf, void* recvbuf, size_t count, MPI_Datatype datatype,
                              MPI_Op op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
                              mca_coll_base_module_t *module) {
     int res = nbc_reduce_init(sendbuf, recvbuf, count, datatype, op, root,
@@ -255,7 +255,7 @@ int ompi_coll_libnbc_ireduce(const void* sendbuf, void* recvbuf, int count, MPI_
     return OMPI_SUCCESS;
 }
 
-static int nbc_reduce_inter_init(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype,
+static int nbc_reduce_inter_init(const void* sendbuf, void* recvbuf, size_t count, MPI_Datatype datatype,
                                  MPI_Op op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                  mca_coll_base_module_t *module, bool persistent) {
   int rank, res, rsize;
@@ -303,7 +303,7 @@ static int nbc_reduce_inter_init(const void* sendbuf, void* recvbuf, int count,
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_ireduce_inter(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype,
+int ompi_coll_libnbc_ireduce_inter(const void* sendbuf, void* recvbuf, size_t count, MPI_Datatype datatype,
                                    MPI_Op op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                    mca_coll_base_module_t *module) {
     int res = nbc_reduce_inter_init(sendbuf, recvbuf, count, datatype, op, root,
@@ -353,7 +353,7 @@ int ompi_coll_libnbc_ireduce_inter(const void* sendbuf, void* recvbuf, int count
   if (vrank == 0) rank = root; \
   if (vrank == root) rank = 0; \
 }
-static inline int red_sched_binomial (int rank, int p, int root, const void *sendbuf, void *redbuf, char tmpredbuf, int count, MPI_Datatype datatype,
+static inline int red_sched_binomial (int rank, int p, int root, const void *sendbuf, void *redbuf, char tmpredbuf, size_t count, MPI_Datatype datatype,
                                       MPI_Op op, char inplace, NBC_Schedule *schedule, void *tmpbuf) {
   int vroot, vrank, vpeer, peer, res, maxr;
   char *rbuf, *lbuf, *buf;
@@ -367,7 +367,7 @@ static inline int red_sched_binomial (int rank, int p, int root, const void *sen
     vroot = 0;
   }
   RANK2VRANK(rank, vrank, vroot);
-  maxr = (int)ceil((log((double)p)/LOG2));
+  maxr = ceil_of_log2(p);
 
   if (rank != root) {
     inplace = 0;
@@ -458,8 +458,8 @@ static inline int red_sched_binomial (int rank, int p, int root, const void *sen
 }
 
 /* chain send ... */
-static inline int red_sched_chain (int rank, int p, int root, const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
-                                   MPI_Op op, int ext, size_t size, NBC_Schedule *schedule, void *tmpbuf, int fragsize) {
+static inline int red_sched_chain (int rank, int p, int root, const void *sendbuf, void *recvbuf, size_t count, MPI_Datatype datatype,
+                                   MPI_Op op, MPI_Aint ext, size_t size, NBC_Schedule *schedule, void *tmpbuf, size_t fragsize) {
   int res, vrank, rpeer, speer, numfrag, fragcount, thiscount;
   long offset;
 
@@ -479,11 +479,11 @@ static inline int red_sched_chain (int rank, int p, int root, const void *sendbu
   fragcount = count / numfrag;
 
   for (int fragnum = 0 ; fragnum < numfrag ; ++fragnum) {
-    offset = fragnum * fragcount * ext;
+    offset = (MPI_Aint) ext * fragnum * fragcount;
     thiscount = fragcount;
     if(fragnum == numfrag - 1) {
       /* last fragment may not be full */
-      thiscount = count - fragcount * fragnum;
+      thiscount = count - (size_t)fragcount * fragnum;
     }
 
     /* last node does not recv */
@@ -536,7 +536,7 @@ static inline int red_sched_chain (int rank, int p, int root, const void *sendbu
 }
 
 /* simple linear algorithm for intercommunicators */
-static inline int red_sched_linear (int rank, int rsize, int root, const void *sendbuf, void *recvbuf, void *tmpbuf, int count, MPI_Datatype datatype,
+static inline int red_sched_linear (int rank, int rsize, int root, const void *sendbuf, void *recvbuf, void *tmpbuf, size_t count, MPI_Datatype datatype,
                                     MPI_Op op, NBC_Schedule *schedule) {
   int res;
   char *rbuf, *lbuf, *buf;
@@ -648,7 +648,7 @@ static inline int red_sched_linear (int rank, int rsize, int root, const void *s
  */
 static inline int red_sched_redscat_gather(
     int rank, int comm_size, int root, const void *sbuf, void *rbuf,
-    char tmpredbuf, int count, MPI_Datatype datatype, MPI_Op op, char inplace,
+    char tmpredbuf, size_t count, MPI_Datatype datatype, MPI_Op op, char inplace,
     NBC_Schedule *schedule, void *tmp_buf, struct ompi_communicator_t *comm)
 {
     int res = OMPI_SUCCESS;
@@ -781,7 +781,7 @@ static inline int red_sched_redscat_gather(
 
         for (int mask = 1; mask < nprocs_pof2; mask <<= 1) {
             /*
-             * On each iteration: rindex[step] = sindex[step] -- begining of the
+             * On each iteration: rindex[step] = sindex[step] -- beginning of the
              * current window. Length of the current window is storded in wsize.
              */
             int vdest = vrank ^ mask;
@@ -937,7 +937,7 @@ static inline int red_sched_redscat_gather(
     return res;
 }
 
-int ompi_coll_libnbc_reduce_init(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype,
+int ompi_coll_libnbc_reduce_init(const void* sendbuf, void* recvbuf, size_t count, MPI_Datatype datatype,
                                  MPI_Op op, int root, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                  mca_coll_base_module_t *module) {
     int res = nbc_reduce_init(sendbuf, recvbuf, count, datatype, op, root,
@@ -949,7 +949,7 @@ int ompi_coll_libnbc_reduce_init(const void* sendbuf, void* recvbuf, int count,
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_reduce_inter_init(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype,
+int ompi_coll_libnbc_reduce_inter_init(const void* sendbuf, void* recvbuf, size_t count, MPI_Datatype datatype,
                                        MPI_Op op, int root, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                        mca_coll_base_module_t *module) {
     int res = nbc_reduce_inter_init(sendbuf, recvbuf, count, datatype, op, root,
diff --git a/ompi/mca/coll/libnbc/nbc_ireduce_scatter.c b/ompi/mca/coll/libnbc/nbc_ireduce_scatter.c
index fbe9b3b3f4b..cd4270a4a41 100644
--- a/ompi/mca/coll/libnbc/nbc_ireduce_scatter.c
+++ b/ompi/mca/coll/libnbc/nbc_ireduce_scatter.c
@@ -12,7 +12,7 @@
  * Copyright (c) 2015      The University of Tennessee and The University
  *                         of Tennessee Research Foundation. All rights
  *                         reserved.
- * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2017-2021 IBM Corporation.  All rights reserved.
  * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
  * $COPYRIGHT$
  *
@@ -26,7 +26,7 @@
 #include "nbc_internal.h"
 
 /* an reduce_csttare schedule can not be cached easily because the contents
- * ot the recvcounts array may change, so a comparison of the address
+ * of the recvcounts array may change, so a comparison of the address
  * would not be sufficient ... we simply do not cache it */
 
 /* binomial reduce to rank 0 followed by a linear scatter ...
@@ -42,10 +42,11 @@
  *
  */
 
-static int nbc_reduce_scatter_init(const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype,
+static int nbc_reduce_scatter_init(const void* sendbuf, void* recvbuf, ompi_count_array_t recvcounts, MPI_Datatype datatype,
                                    MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                    mca_coll_base_module_t *module, bool persistent) {
-  int peer, rank, maxr, p, res, count;
+  int peer, rank, maxr, p, res;
+  size_t count;
   MPI_Aint ext;
   ptrdiff_t gap, span, span_align;
   char *sbuf, inplace;
@@ -67,13 +68,13 @@ static int nbc_reduce_scatter_init(const void* sendbuf, void* recvbuf, const int
 
   count = 0;
   for (int r = 0 ; r < p ; ++r) {
-    count += recvcounts[r];
+    count += ompi_count_array_get(recvcounts, r);
   }
 
   if ((1 == p && (!persistent || inplace)) || 0 == count) {
     if (!inplace) {
       /* single node not in_place: copy data to recvbuf */
-      res = NBC_Copy(sendbuf, recvcounts[0], datatype, recvbuf, recvcounts[0], datatype, comm);
+      res = NBC_Copy(sendbuf, ompi_count_array_get(recvcounts, 0), datatype, recvbuf, ompi_count_array_get(recvcounts, 0), datatype, comm);
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
         return res;
       }
@@ -82,7 +83,7 @@ static int nbc_reduce_scatter_init(const void* sendbuf, void* recvbuf, const int
     return nbc_get_noop_request(persistent, request);
   }
 
-  maxr = (int) ceil ((log((double) p) / LOG2));
+  maxr = ceil_of_log2(p);
 
   span = opal_datatype_span(&datatype->super, count, &gap);
   span_align = OPAL_ALIGN(span, datatype->super.align, ptrdiff_t);
@@ -161,11 +162,12 @@ static int nbc_reduce_scatter_init(const void* sendbuf, void* recvbuf, const int
 
   /* rank 0 is root and sends - all others receive */
   if (rank == 0) {
-    for (long int r = 1, offset = 0 ; r < p ; ++r) {
-      offset += recvcounts[r-1];
+    size_t offset = 0;
+    for (long int r = 1 ; r < p ; ++r) {
+      offset += ompi_count_array_get(recvcounts, r-1);
       sbuf = lbuf + (offset*ext);
       /* root sends the right buffer to the right receiver */
-      res = NBC_Sched_send (sbuf, true, recvcounts[r], datatype, r, schedule,
+      res = NBC_Sched_send (sbuf, true, ompi_count_array_get(recvcounts, r), datatype, r, schedule,
                             false);
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
         OBJ_RELEASE(schedule);
@@ -176,14 +178,14 @@ static int nbc_reduce_scatter_init(const void* sendbuf, void* recvbuf, const int
 
     if (p == 1) {
       /* single node not in_place: copy data to recvbuf */
-      res = NBC_Sched_copy ((void *)sendbuf, false, recvcounts[0], datatype,
-                            recvbuf, false, recvcounts[0], datatype, schedule, false);
+      res = NBC_Sched_copy ((void *)sendbuf, false, ompi_count_array_get(recvcounts, 0), datatype,
+                            recvbuf, false, ompi_count_array_get(recvcounts, 0), datatype, schedule, false);
     } else {
-      res = NBC_Sched_copy (lbuf, true, recvcounts[0], datatype, recvbuf, false,
-                            recvcounts[0], datatype, schedule, false);
+      res = NBC_Sched_copy (lbuf, true, ompi_count_array_get(recvcounts, 0), datatype, recvbuf, false,
+                            ompi_count_array_get(recvcounts, 0), datatype, schedule, false);
     }
   } else {
-    res = NBC_Sched_recv (recvbuf, false, recvcounts[rank], datatype, 0, schedule, false);
+    res = NBC_Sched_recv (recvbuf, false, ompi_count_array_get(recvcounts, rank), datatype, 0, schedule, false);
   }
 
   if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
@@ -209,7 +211,7 @@ static int nbc_reduce_scatter_init(const void* sendbuf, void* recvbuf, const int
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_ireduce_scatter (const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype,
+int ompi_coll_libnbc_ireduce_scatter (const void* sendbuf, void* recvbuf, ompi_count_array_t recvcounts, MPI_Datatype datatype,
                                       MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                       mca_coll_base_module_t *module) {
     int res = nbc_reduce_scatter_init(sendbuf, recvbuf, recvcounts, datatype, op,
@@ -226,10 +228,11 @@ int ompi_coll_libnbc_ireduce_scatter (const void* sendbuf, void* recvbuf, const
 
     return OMPI_SUCCESS;
 }
-static int nbc_reduce_scatter_inter_init (const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype,
+static int nbc_reduce_scatter_inter_init (const void* sendbuf, void* recvbuf, ompi_count_array_t recvcounts, MPI_Datatype datatype,
                                           MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                           mca_coll_base_module_t *module, bool persistent) {
-  int rank, res, count, lsize, rsize;
+  int rank, res, lsize, rsize;
+  size_t count;
   MPI_Aint ext;
   ptrdiff_t gap, span, span_align;
   NBC_Schedule *schedule;
@@ -248,7 +251,7 @@ static int nbc_reduce_scatter_inter_init (const void* sendbuf, void* recvbuf, co
 
   count = 0;
   for (int r = 0 ; r < lsize ; ++r) {
-    count += recvcounts[r];
+    count += ompi_count_array_get(recvcounts, r);
   }
 
   span = opal_datatype_span(&datatype->super, count, &gap);
@@ -306,15 +309,16 @@ static int nbc_reduce_scatter_inter_init (const void* sendbuf, void* recvbuf, co
     }
 
     /* do the local scatterv with the local communicator */
-    res = NBC_Sched_copy (lbuf, true, recvcounts[0], datatype, recvbuf, false,
-                          recvcounts[0], datatype, schedule, false);
+    res = NBC_Sched_copy (lbuf, true, ompi_count_array_get(recvcounts, 0), datatype, recvbuf, false,
+                          ompi_count_array_get(recvcounts, 0), datatype, schedule, false);
     if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
       OBJ_RELEASE(schedule);
       free(tmpbuf);
       return res;
     }
-    for (int peer = 1, offset = recvcounts[0] * ext; peer < lsize ; ++peer) {
-      res = NBC_Sched_local_send (lbuf + offset, true, recvcounts[peer], datatype, peer, schedule,
+    size_t offset = ompi_count_array_get(recvcounts, 0) * ext;
+    for (int peer = 1; peer < lsize ; ++peer) {
+      res = NBC_Sched_local_send (lbuf + offset, true, ompi_count_array_get(recvcounts, peer), datatype, peer, schedule,
                                   false);
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
         OBJ_RELEASE(schedule);
@@ -322,11 +326,11 @@ static int nbc_reduce_scatter_inter_init (const void* sendbuf, void* recvbuf, co
         return res;
       }
 
-      offset += recvcounts[peer] * ext;
+      offset += ompi_count_array_get(recvcounts, peer) * ext;
     }
   } else {
     /* receive my block */
-    res = NBC_Sched_local_recv (recvbuf, false, recvcounts[rank], datatype, 0, schedule, false);
+    res = NBC_Sched_local_recv (recvbuf, false, ompi_count_array_get(recvcounts, rank), datatype, 0, schedule, false);
     if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
       OBJ_RELEASE(schedule);
       free(tmpbuf);
@@ -351,7 +355,7 @@ static int nbc_reduce_scatter_inter_init (const void* sendbuf, void* recvbuf, co
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_ireduce_scatter_inter (const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype,
+int ompi_coll_libnbc_ireduce_scatter_inter (const void* sendbuf, void* recvbuf, ompi_count_array_t recvcounts, MPI_Datatype datatype,
                                             MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                             mca_coll_base_module_t *module) {
     int res = nbc_reduce_scatter_inter_init(sendbuf, recvbuf, recvcounts, datatype, op,
@@ -369,7 +373,7 @@ int ompi_coll_libnbc_ireduce_scatter_inter (const void* sendbuf, void* recvbuf,
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_reduce_scatter_init(const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype,
+int ompi_coll_libnbc_reduce_scatter_init(const void* sendbuf, void* recvbuf, ompi_count_array_t recvcounts, MPI_Datatype datatype,
                                          MPI_Op op, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                          mca_coll_base_module_t *module) {
     int res = nbc_reduce_scatter_init(sendbuf, recvbuf, recvcounts, datatype, op,
@@ -381,7 +385,7 @@ int ompi_coll_libnbc_reduce_scatter_init(const void* sendbuf, void* recvbuf, con
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_reduce_scatter_inter_init(const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype,
+int ompi_coll_libnbc_reduce_scatter_inter_init(const void* sendbuf, void* recvbuf, ompi_count_array_t recvcounts, MPI_Datatype datatype,
                                                MPI_Op op, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                                mca_coll_base_module_t *module) {
     int res = nbc_reduce_scatter_inter_init(sendbuf, recvbuf, recvcounts, datatype, op,
diff --git a/ompi/mca/coll/libnbc/nbc_ireduce_scatter_block.c b/ompi/mca/coll/libnbc/nbc_ireduce_scatter_block.c
index 4cc99a3545e..89c9c861b61 100644
--- a/ompi/mca/coll/libnbc/nbc_ireduce_scatter_block.c
+++ b/ompi/mca/coll/libnbc/nbc_ireduce_scatter_block.c
@@ -10,7 +10,7 @@
  *                         reserved.
  * Copyright (c) 2014-2018 Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2017-2021 IBM Corporation.  All rights reserved.
  * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
  * $COPYRIGHT$
  *
@@ -24,7 +24,7 @@
 #include "nbc_internal.h"
 
 /* an reduce_csttare schedule can not be cached easily because the contents
- * ot the recvcount value may change, so a comparison of the address
+ * of the recvcount value may change, so a comparison of the address
  * would not be sufficient ... we simply do not cache it */
 
 /* binomial reduce to rank 0 followed by a linear scatter ...
@@ -40,10 +40,11 @@
  *
  */
 
-static int nbc_reduce_scatter_block_init(const void* sendbuf, void* recvbuf, int recvcount, MPI_Datatype datatype,
+static int nbc_reduce_scatter_block_init(const void* sendbuf, void* recvbuf, size_t recvcount, MPI_Datatype datatype,
                                          MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                          mca_coll_base_module_t *module, bool persistent) {
-  int peer, rank, maxr, p, res, count;
+  int peer, rank, maxr, p, res;
+  size_t count;
   MPI_Aint ext;
   ptrdiff_t gap, span;
   char *redbuf, *sbuf, inplace;
@@ -67,9 +68,9 @@ static int nbc_reduce_scatter_block_init(const void* sendbuf, void* recvbuf, int
     return OMPI_ERR_OUT_OF_RESOURCE;
   }
 
-  maxr = (int)ceil((log((double)p)/LOG2));
+  maxr = ceil_of_log2(p);
 
-  count = p * recvcount;
+  count = (size_t) p * recvcount;
 
   if (0 < count) {
     char *rbuf, *lbuf, *buf;
@@ -166,7 +167,8 @@ static int nbc_reduce_scatter_block_init(const void* sendbuf, void* recvbuf, int
         return res;
       }
     } else {
-      for (int r = 1, offset = 0 ; r < p ; ++r) {
+      size_t offset = 0;
+      for (int r = 1 ; r < p ; ++r) {
         offset += recvcount;
         sbuf = lbuf + (offset*ext);
         /* root sends the right buffer to the right receiver */
@@ -207,7 +209,7 @@ static int nbc_reduce_scatter_block_init(const void* sendbuf, void* recvbuf, int
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_ireduce_scatter_block(const void* sendbuf, void* recvbuf, int recvcount, MPI_Datatype datatype,
+int ompi_coll_libnbc_ireduce_scatter_block(const void* sendbuf, void* recvbuf, size_t recvcount, MPI_Datatype datatype,
                                            MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                            mca_coll_base_module_t *module) {
     int res = nbc_reduce_scatter_block_init(sendbuf, recvbuf, recvcount, datatype, op,
@@ -225,10 +227,11 @@ int ompi_coll_libnbc_ireduce_scatter_block(const void* sendbuf, void* recvbuf, i
     return OMPI_SUCCESS;
 }
 
-static int nbc_reduce_scatter_block_inter_init(const void *sendbuf, void *recvbuf, int rcount, struct ompi_datatype_t *dtype,
+static int nbc_reduce_scatter_block_inter_init(const void *sendbuf, void *recvbuf, size_t rcount, struct ompi_datatype_t *dtype,
                                                struct ompi_op_t *op, struct ompi_communicator_t *comm, ompi_request_t **request,
                                                mca_coll_base_module_t *module, bool persistent) {
-  int rank, res, count, lsize, rsize;
+  int rank, res, lsize, rsize;
+  size_t count;
   MPI_Aint ext;
   ptrdiff_t gap, span, span_align;
   NBC_Schedule *schedule;
@@ -245,7 +248,7 @@ static int nbc_reduce_scatter_block_inter_init(const void *sendbuf, void *recvbu
     return res;
   }
 
-  count = rcount * lsize;
+  count = (size_t)rcount * lsize;
 
   span = opal_datatype_span(&dtype->super, count, &gap);
   span_align = OPAL_ALIGN(span, dtype->super.align, ptrdiff_t);
@@ -346,7 +349,7 @@ static int nbc_reduce_scatter_block_inter_init(const void *sendbuf, void *recvbu
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_ireduce_scatter_block_inter(const void* sendbuf, void* recvbuf, int recvcount, MPI_Datatype datatype,
+int ompi_coll_libnbc_ireduce_scatter_block_inter(const void* sendbuf, void* recvbuf, size_t recvcount, MPI_Datatype datatype,
                                                  MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                                  mca_coll_base_module_t *module) {
     int res = nbc_reduce_scatter_block_inter_init(sendbuf, recvbuf, recvcount, datatype, op,
@@ -364,7 +367,7 @@ int ompi_coll_libnbc_ireduce_scatter_block_inter(const void* sendbuf, void* recv
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_reduce_scatter_block_init(const void* sendbuf, void* recvbuf, int recvcount, MPI_Datatype datatype,
+int ompi_coll_libnbc_reduce_scatter_block_init(const void* sendbuf, void* recvbuf, size_t recvcount, MPI_Datatype datatype,
                                                MPI_Op op, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                                mca_coll_base_module_t *module) {
     int res = nbc_reduce_scatter_block_init(sendbuf, recvbuf, recvcount, datatype, op,
@@ -376,7 +379,7 @@ int ompi_coll_libnbc_reduce_scatter_block_init(const void* sendbuf, void* recvbu
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_reduce_scatter_block_inter_init(const void* sendbuf, void* recvbuf, int recvcount, MPI_Datatype datatype,
+int ompi_coll_libnbc_reduce_scatter_block_inter_init(const void* sendbuf, void* recvbuf, size_t recvcount, MPI_Datatype datatype,
                                                      MPI_Op op, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                                      mca_coll_base_module_t *module) {
     int res = nbc_reduce_scatter_block_inter_init(sendbuf, recvbuf, recvcount, datatype, op,
diff --git a/ompi/mca/coll/libnbc/nbc_iscan.c b/ompi/mca/coll/libnbc/nbc_iscan.c
index 1d628973985..788147103b4 100644
--- a/ompi/mca/coll/libnbc/nbc_iscan.c
+++ b/ompi/mca/coll/libnbc/nbc_iscan.c
@@ -24,12 +24,12 @@
 #include "nbc_internal.h"
 
 static inline int scan_sched_linear(
-    int rank, int comm_size, const void *sendbuf, void *recvbuf, int count,
+    int rank, int comm_size, const void *sendbuf, void *recvbuf, size_t count,
     MPI_Datatype datatype,  MPI_Op op, char inplace, NBC_Schedule *schedule,
     void *tmpbuf);
 static inline int scan_sched_recursivedoubling(
     int rank, int comm_size, const void *sendbuf, void *recvbuf,
-    int count, MPI_Datatype datatype,  MPI_Op op, char inplace,
+    size_t count, MPI_Datatype datatype,  MPI_Op op, char inplace,
     NBC_Schedule *schedule, void *tmpbuf1, void *tmpbuf2);
 
 #ifdef NBC_CACHE_SCHEDULE
@@ -51,7 +51,7 @@ int NBC_Scan_args_compare(NBC_Scan_args *a, NBC_Scan_args *b, void *param) {
 }
 #endif
 
-static int nbc_scan_init(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
+static int nbc_scan_init(const void* sendbuf, void* recvbuf, size_t count, MPI_Datatype datatype, MPI_Op op,
                          struct ompi_communicator_t *comm, ompi_request_t ** request,
                          mca_coll_base_module_t *module, bool persistent) {
     int rank, p, res;
@@ -181,7 +181,7 @@ static int nbc_scan_init(const void* sendbuf, void* recvbuf, int count, MPI_Data
  * Schedule length: O(1)
  */
 static inline int scan_sched_linear(
-    int rank, int comm_size, const void *sendbuf, void *recvbuf, int count,
+    int rank, int comm_size, const void *sendbuf, void *recvbuf, size_t count,
     MPI_Datatype datatype,  MPI_Op op, char inplace, NBC_Schedule *schedule,
     void *tmpbuf)
 {
@@ -251,7 +251,7 @@ static inline int scan_sched_linear(
  * Schedule length: O(log(p))
  */
 static inline int scan_sched_recursivedoubling(
-    int rank, int comm_size, const void *sendbuf, void *recvbuf, int count,
+    int rank, int comm_size, const void *sendbuf, void *recvbuf, size_t count,
     MPI_Datatype datatype, MPI_Op op, char inplace,
     NBC_Schedule *schedule, void *tmpbuf1, void *tmpbuf2)
 {
@@ -312,7 +312,7 @@ static inline int scan_sched_recursivedoubling(
     return res;
 }
 
-int ompi_coll_libnbc_iscan(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
+int ompi_coll_libnbc_iscan(const void* sendbuf, void* recvbuf, size_t count, MPI_Datatype datatype, MPI_Op op,
                            struct ompi_communicator_t *comm, ompi_request_t ** request,
                            mca_coll_base_module_t *module) {
     int res = nbc_scan_init(sendbuf, recvbuf, count, datatype, op,
@@ -330,7 +330,7 @@ int ompi_coll_libnbc_iscan(const void* sendbuf, void* recvbuf, int count, MPI_Da
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_scan_init(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
+int ompi_coll_libnbc_scan_init(const void* sendbuf, void* recvbuf, size_t count, MPI_Datatype datatype, MPI_Op op,
                                struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                mca_coll_base_module_t *module) {
     int res = nbc_scan_init(sendbuf, recvbuf, count, datatype, op,
diff --git a/ompi/mca/coll/libnbc/nbc_iscatter.c b/ompi/mca/coll/libnbc/nbc_iscatter.c
index 01de0a4d359..99e2bbf9a42 100644
--- a/ompi/mca/coll/libnbc/nbc_iscatter.c
+++ b/ompi/mca/coll/libnbc/nbc_iscatter.c
@@ -45,8 +45,8 @@ int NBC_Scatter_args_compare(NBC_Scatter_args *a, NBC_Scatter_args *b, void *par
 #endif
 
 /* simple linear MPI_Iscatter */
-static int nbc_scatter_init (const void* sendbuf, int sendcount, MPI_Datatype sendtype,
-                             void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
+static int nbc_scatter_init (const void* sendbuf, size_t sendcount, MPI_Datatype sendtype,
+                             void* recvbuf, size_t recvcount, MPI_Datatype recvtype, int root,
                              struct ompi_communicator_t *comm, ompi_request_t ** request,
                              mca_coll_base_module_t *module, bool persistent) {
   int rank, p, res;
@@ -99,7 +99,7 @@ static int nbc_scatter_init (const void* sendbuf, int sendcount, MPI_Datatype se
       }
     } else {
       for (int i = 0 ; i < p ; ++i) {
-        sbuf = (char *) sendbuf + i * sendcount * sndext;
+        sbuf = (char *) sendbuf + (MPI_Aint) sndext * i * sendcount;
         if (i == root) {
           if (!inplace) {
             /* if I am the root - just copy the message */
@@ -168,8 +168,8 @@ static int nbc_scatter_init (const void* sendbuf, int sendcount, MPI_Datatype se
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_iscatter (const void* sendbuf, int sendcount, MPI_Datatype sendtype,
-                               void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
+int ompi_coll_libnbc_iscatter (const void* sendbuf, size_t sendcount, MPI_Datatype sendtype,
+                               void* recvbuf, size_t recvcount, MPI_Datatype recvtype, int root,
                                struct ompi_communicator_t *comm, ompi_request_t ** request,
                                mca_coll_base_module_t *module) {
     int res = nbc_scatter_init(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root,
@@ -187,8 +187,8 @@ int ompi_coll_libnbc_iscatter (const void* sendbuf, int sendcount, MPI_Datatype
     return OMPI_SUCCESS;
 }
 
-static int nbc_scatter_inter_init (const void* sendbuf, int sendcount, MPI_Datatype sendtype,
-                                   void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
+static int nbc_scatter_inter_init (const void* sendbuf, size_t sendcount, MPI_Datatype sendtype,
+                                   void* recvbuf, size_t recvcount, MPI_Datatype recvtype, int root,
                                    struct ompi_communicator_t *comm, ompi_request_t ** request,
                                    mca_coll_base_module_t *module, bool persistent) {
     int res, rsize;
@@ -222,7 +222,7 @@ static int nbc_scatter_inter_init (const void* sendbuf, int sendcount, MPI_Datat
         }
     } else if (MPI_ROOT == root) {
         for (int i = 0 ; i < rsize ; ++i) {
-            sbuf = ((char *)sendbuf) + (i * sendcount * sndext);
+            sbuf = ((char *)sendbuf) + ((MPI_Aint) sndext * i * sendcount);
             /* root sends the right buffer to the right receiver */
             res = NBC_Sched_send(sbuf, false, sendcount, sendtype, i, schedule, false);
             if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
@@ -247,8 +247,8 @@ static int nbc_scatter_inter_init (const void* sendbuf, int sendcount, MPI_Datat
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_iscatter_inter (const void* sendbuf, int sendcount, MPI_Datatype sendtype,
-                                     void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
+int ompi_coll_libnbc_iscatter_inter (const void* sendbuf, size_t sendcount, MPI_Datatype sendtype,
+                                     void* recvbuf, size_t recvcount, MPI_Datatype recvtype, int root,
                                      struct ompi_communicator_t *comm, ompi_request_t ** request,
                                      mca_coll_base_module_t *module) {
     int res = nbc_scatter_inter_init(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root,
@@ -266,8 +266,8 @@ int ompi_coll_libnbc_iscatter_inter (const void* sendbuf, int sendcount, MPI_Dat
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_scatter_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype,
-                                  void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
+int ompi_coll_libnbc_scatter_init(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype,
+                                  void* recvbuf, size_t recvcount, MPI_Datatype recvtype, int root,
                                   struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                   mca_coll_base_module_t *module) {
     int res = nbc_scatter_init(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root,
@@ -279,8 +279,8 @@ int ompi_coll_libnbc_scatter_init(const void* sendbuf, int sendcount, MPI_Dataty
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_scatter_inter_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype,
-                                        void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
+int ompi_coll_libnbc_scatter_inter_init(const void* sendbuf, size_t sendcount, MPI_Datatype sendtype,
+                                        void* recvbuf, size_t recvcount, MPI_Datatype recvtype, int root,
                                         struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                         mca_coll_base_module_t *module) {
     int res = nbc_scatter_inter_init(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root,
diff --git a/ompi/mca/coll/libnbc/nbc_iscatterv.c b/ompi/mca/coll/libnbc/nbc_iscatterv.c
index 2e305e6ec61..a42ace566da 100644
--- a/ompi/mca/coll/libnbc/nbc_iscatterv.c
+++ b/ompi/mca/coll/libnbc/nbc_iscatterv.c
@@ -24,12 +24,12 @@
 #include "nbc_internal.h"
 
 /* a scatterv schedule can not be cached easily because the contents
- * ot the recvcounts array may change, so a comparison of the address
+ * of the recvcounts array may change, so a comparison of the address
  * would not be sufficient ... we simply do not cache it */
 
 /* simple linear MPI_Iscatterv */
-static int nbc_scatterv_init(const void* sendbuf, const int *sendcounts, const int *displs, MPI_Datatype sendtype,
-                             void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
+static int nbc_scatterv_init(const void* sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t displs, MPI_Datatype sendtype,
+                             void* recvbuf, size_t recvcount, MPI_Datatype recvtype, int root,
                              struct ompi_communicator_t *comm, ompi_request_t ** request,
                              mca_coll_base_module_t *module, bool persistent) {
   int rank, p, res;
@@ -60,18 +60,18 @@ static int nbc_scatterv_init(const void* sendbuf, const int *sendcounts, const i
     }
 
     for (int i = 0 ; i < p ; ++i) {
-      sbuf = (char *) sendbuf + displs[i] * sndext;
+      sbuf = (char *) sendbuf + ompi_disp_array_get(displs, i) * sndext;
       if (i == root) {
         if (!inplace) {
           /* if I am the root - just copy the message */
-          res = NBC_Sched_copy (sbuf, false, sendcounts[i], sendtype,
+          res = NBC_Sched_copy (sbuf, false, ompi_count_array_get(sendcounts, i), sendtype,
                                 recvbuf, false, recvcount, recvtype, schedule, false);
         } else {
           res = OMPI_SUCCESS;
         }
       } else {
         /* root sends the right buffer to the right receiver */
-        res = NBC_Sched_send (sbuf, false, sendcounts[i], sendtype, i, schedule, false);
+        res = NBC_Sched_send (sbuf, false, ompi_count_array_get(sendcounts, i), sendtype, i, schedule, false);
       }
 
       if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
@@ -103,8 +103,8 @@ static int nbc_scatterv_init(const void* sendbuf, const int *sendcounts, const i
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_iscatterv(const void* sendbuf, const int *sendcounts, const int *displs, MPI_Datatype sendtype,
-                               void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
+int ompi_coll_libnbc_iscatterv(const void* sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t displs, MPI_Datatype sendtype,
+                               void* recvbuf, size_t recvcount, MPI_Datatype recvtype, int root,
                                struct ompi_communicator_t *comm, ompi_request_t ** request,
                                mca_coll_base_module_t *module) {
     int res = nbc_scatterv_init(sendbuf, sendcounts, displs, sendtype, recvbuf, recvcount, recvtype, root,
@@ -122,8 +122,8 @@ int ompi_coll_libnbc_iscatterv(const void* sendbuf, const int *sendcounts, const
     return OMPI_SUCCESS;
 }
 
-static int nbc_scatterv_inter_init (const void* sendbuf, const int *sendcounts, const int *displs, MPI_Datatype sendtype,
-                                    void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
+static int nbc_scatterv_inter_init (const void* sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t displs, MPI_Datatype sendtype,
+                                    void* recvbuf, size_t recvcount, MPI_Datatype recvtype, int root,
                                     struct ompi_communicator_t *comm, ompi_request_t ** request,
                                     mca_coll_base_module_t *module, bool persistent) {
     int res, rsize;
@@ -156,9 +156,9 @@ static int nbc_scatterv_inter_init (const void* sendbuf, const int *sendcounts,
         }
 
         for (int i = 0 ; i < rsize ; ++i) {
-            sbuf = (char *)sendbuf + displs[i] * sndext;
+            sbuf = (char *)sendbuf + ompi_disp_array_get(displs, i) * sndext;
             /* root sends the right buffer to the right receiver */
-            res = NBC_Sched_send (sbuf, false, sendcounts[i], sendtype, i, schedule, false);
+            res = NBC_Sched_send (sbuf, false, ompi_count_array_get(sendcounts, i), sendtype, i, schedule, false);
             if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
                 OBJ_RELEASE(schedule);
                 return res;
@@ -181,8 +181,8 @@ static int nbc_scatterv_inter_init (const void* sendbuf, const int *sendcounts,
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_iscatterv_inter(const void* sendbuf, const int *sendcounts, const int *displs, MPI_Datatype sendtype,
-                                     void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
+int ompi_coll_libnbc_iscatterv_inter(const void* sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t displs, MPI_Datatype sendtype,
+                                     void* recvbuf, size_t recvcount, MPI_Datatype recvtype, int root,
                                      struct ompi_communicator_t *comm, ompi_request_t ** request,
                                      mca_coll_base_module_t *module) {
     int res = nbc_scatterv_inter_init(sendbuf, sendcounts, displs, sendtype, recvbuf, recvcount, recvtype, root,
@@ -200,8 +200,8 @@ int ompi_coll_libnbc_iscatterv_inter(const void* sendbuf, const int *sendcounts,
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_scatterv_init(const void* sendbuf, const int *sendcounts, const int *displs, MPI_Datatype sendtype,
-                                   void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
+int ompi_coll_libnbc_scatterv_init(const void* sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t displs, MPI_Datatype sendtype,
+                                   void* recvbuf, size_t recvcount, MPI_Datatype recvtype, int root,
                                    struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                    mca_coll_base_module_t *module) {
     int res = nbc_scatterv_init(sendbuf, sendcounts, displs, sendtype, recvbuf, recvcount, recvtype, root,
@@ -213,8 +213,8 @@ int ompi_coll_libnbc_scatterv_init(const void* sendbuf, const int *sendcounts, c
     return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_scatterv_inter_init(const void* sendbuf, const int *sendcounts, const int *displs, MPI_Datatype sendtype,
-                                         void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
+int ompi_coll_libnbc_scatterv_inter_init(const void* sendbuf, ompi_count_array_t sendcounts, ompi_disp_array_t displs, MPI_Datatype sendtype,
+                                         void* recvbuf, size_t recvcount, MPI_Datatype recvtype, int root,
                                          struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
                                          mca_coll_base_module_t *module) {
     int res = nbc_scatterv_inter_init(sendbuf, sendcounts, displs, sendtype, recvbuf, recvcount, recvtype, root,
diff --git a/ompi/mca/coll/monitoring/coll_monitoring.h b/ompi/mca/coll/monitoring/coll_monitoring.h
index 3f741970907..de8b4e5303b 100644
--- a/ompi/mca/coll/monitoring/coll_monitoring.h
+++ b/ompi/mca/coll/monitoring/coll_monitoring.h
@@ -4,6 +4,7 @@
  *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2017      Amazon.com, Inc. or its affiliates.  All Rights
  *                         reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -36,7 +37,6 @@ struct mca_coll_monitoring_module_t {
     mca_coll_base_module_t super;
     mca_coll_base_comm_coll_t real;
     mca_monitoring_coll_data_t*data;
-    opal_atomic_int32_t is_initialized;
 };
 typedef struct mca_coll_monitoring_module_t mca_coll_monitoring_module_t;
 OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_coll_monitoring_module_t);
@@ -46,48 +46,48 @@ OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_coll_monitoring_module_t);
  */
 
 /* Blocking */
-extern int mca_coll_monitoring_allgather(const void *sbuf, int scount,
+extern int mca_coll_monitoring_allgather(const void *sbuf, size_t scount,
                                          struct ompi_datatype_t *sdtype,
-                                         void *rbuf, int rcount,
+                                         void *rbuf, size_t rcount,
                                          struct ompi_datatype_t *rdtype,
                                          struct ompi_communicator_t *comm,
                                          mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_allgatherv(const void *sbuf, int scount,
+extern int mca_coll_monitoring_allgatherv(const void *sbuf, size_t scount,
                                           struct ompi_datatype_t *sdtype,
-                                          void *rbuf, const int *rcounts,
-                                          const int *disps,
+                                          void *rbuf, ompi_count_array_t rcounts,
+                                          ompi_disp_array_t disps,
                                           struct ompi_datatype_t *rdtype,
                                           struct ompi_communicator_t *comm,
                                           mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_allreduce(const void *sbuf, void *rbuf, int count,
+extern int mca_coll_monitoring_allreduce(const void *sbuf, void *rbuf, size_t count,
                                          struct ompi_datatype_t *dtype,
                                          struct ompi_op_t *op,
                                          struct ompi_communicator_t *comm,
                                          mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_alltoall(const void *sbuf, int scount,
+extern int mca_coll_monitoring_alltoall(const void *sbuf, size_t scount,
                                         struct ompi_datatype_t *sdtype,
-                                        void *rbuf, int rcount,
+                                        void *rbuf, size_t rcount,
                                         struct ompi_datatype_t *rdtype,
                                         struct ompi_communicator_t *comm,
                                         mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_alltoallv(const void *sbuf, const int *scounts,
-                                         const int *sdisps,
+extern int mca_coll_monitoring_alltoallv(const void *sbuf, ompi_count_array_t scounts,
+                                         ompi_disp_array_t sdisps,
                                          struct ompi_datatype_t *sdtype,
-                                         void *rbuf, const int *rcounts,
-                                         const int *rdisps,
+                                         void *rbuf, ompi_count_array_t rcounts,
+                                         ompi_disp_array_t rdisps,
                                          struct ompi_datatype_t *rdtype,
                                          struct ompi_communicator_t *comm,
                                          mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_alltoallw(const void *sbuf, const int *scounts,
-                                         const int *sdisps,
+extern int mca_coll_monitoring_alltoallw(const void *sbuf, ompi_count_array_t scounts,
+                                         ompi_disp_array_t sdisps,
                                          struct ompi_datatype_t * const *sdtypes,
-                                         void *rbuf, const int *rcounts,
-                                         const int *rdisps,
+                                         void *rbuf, ompi_count_array_t rcounts,
+                                         ompi_disp_array_t rdisps,
                                          struct ompi_datatype_t * const *rdtypes,
                                          struct ompi_communicator_t *comm,
                                          mca_coll_base_module_t *module);
@@ -95,33 +95,33 @@ extern int mca_coll_monitoring_alltoallw(const void *sbuf, const int *scounts,
 extern int mca_coll_monitoring_barrier(struct ompi_communicator_t *comm,
                                        mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_bcast(void *buff, int count,
+extern int mca_coll_monitoring_bcast(void *buff, size_t count,
                                      struct ompi_datatype_t *datatype,
                                      int root,
                                      struct ompi_communicator_t *comm,
                                      mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_exscan(const void *sbuf, void *rbuf, int count,
+extern int mca_coll_monitoring_exscan(const void *sbuf, void *rbuf, size_t count,
                                       struct ompi_datatype_t *dtype,
                                       struct ompi_op_t *op,
                                       struct ompi_communicator_t *comm,
                                       mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_gather(const void *sbuf, int scount,
+extern int mca_coll_monitoring_gather(const void *sbuf, size_t scount,
                                       struct ompi_datatype_t *sdtype,
-                                      void *rbuf, int rcount, struct ompi_datatype_t *rdtype,
+                                      void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
                                       int root, struct ompi_communicator_t *comm,
                                       mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_gatherv(const void *sbuf, int scount,
+extern int mca_coll_monitoring_gatherv(const void *sbuf, size_t scount,
                                        struct ompi_datatype_t *sdtype,
-                                       void *rbuf, const int *rcounts, const int *disps,
+                                       void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
                                        struct ompi_datatype_t *rdtype,
                                        int root,
                                        struct ompi_communicator_t *comm,
                                        mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_reduce(const void *sbuf, void *rbuf, int count,
+extern int mca_coll_monitoring_reduce(const void *sbuf, void *rbuf, size_t count,
                                       struct ompi_datatype_t *dtype,
                                       struct ompi_op_t *op,
                                       int root,
@@ -129,89 +129,89 @@ extern int mca_coll_monitoring_reduce(const void *sbuf, void *rbuf, int count,
                                       mca_coll_base_module_t *module);
 
 extern int mca_coll_monitoring_reduce_scatter(const void *sbuf, void *rbuf,
-                                              const int *rcounts,
+                                              ompi_count_array_t rcounts,
                                               struct ompi_datatype_t *dtype,
                                               struct ompi_op_t *op,
                                               struct ompi_communicator_t *comm,
                                               mca_coll_base_module_t *module);
 
 extern int mca_coll_monitoring_reduce_scatter_block(const void *sbuf, void *rbuf,
-                                                    int rcount,
+                                                    size_t rcount,
                                                     struct ompi_datatype_t *dtype,
                                                     struct ompi_op_t *op,
                                                     struct ompi_communicator_t *comm,
                                                     mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_scan(const void *sbuf, void *rbuf, int count,
+extern int mca_coll_monitoring_scan(const void *sbuf, void *rbuf, size_t count,
                                     struct ompi_datatype_t *dtype,
                                     struct ompi_op_t *op,
                                     struct ompi_communicator_t *comm,
                                     mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_scatter(const void *sbuf, int scount,
+extern int mca_coll_monitoring_scatter(const void *sbuf, size_t scount,
                                        struct ompi_datatype_t *sdtype,
-                                       void *rbuf, int rcount,
+                                       void *rbuf, size_t rcount,
                                        struct ompi_datatype_t *rdtype,
                                        int root,
                                        struct ompi_communicator_t *comm,
                                        mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_scatterv(const void *sbuf, const int *scounts, const int *disps,
+extern int mca_coll_monitoring_scatterv(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t disps,
                                         struct ompi_datatype_t *sdtype,
-                                        void *rbuf, int rcount,
+                                        void *rbuf, size_t rcount,
                                         struct ompi_datatype_t *rdtype,
                                         int root,
                                         struct ompi_communicator_t *comm,
                                         mca_coll_base_module_t *module);
 
 /* Nonblocking */
-extern int mca_coll_monitoring_iallgather(const void *sbuf, int scount,
+extern int mca_coll_monitoring_iallgather(const void *sbuf, size_t scount,
                                           struct ompi_datatype_t *sdtype,
-                                          void *rbuf, int rcount,
+                                          void *rbuf, size_t rcount,
                                           struct ompi_datatype_t *rdtype,
                                           struct ompi_communicator_t *comm,
                                           ompi_request_t ** request,
                                           mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_iallgatherv(const void *sbuf, int scount,
+extern int mca_coll_monitoring_iallgatherv(const void *sbuf, size_t scount,
                                            struct ompi_datatype_t *sdtype,
-                                           void *rbuf, const int *rcounts,
-                                           const int *disps,
+                                           void *rbuf, ompi_count_array_t rcounts,
+                                           ompi_disp_array_t disps,
                                            struct ompi_datatype_t *rdtype,
                                            struct ompi_communicator_t *comm,
                                            ompi_request_t ** request,
                                            mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_iallreduce(const void *sbuf, void *rbuf, int count,
+extern int mca_coll_monitoring_iallreduce(const void *sbuf, void *rbuf, size_t count,
                                           struct ompi_datatype_t *dtype,
                                           struct ompi_op_t *op,
                                           struct ompi_communicator_t *comm,
                                           ompi_request_t ** request,
                                           mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_ialltoall(const void *sbuf, int scount,
+extern int mca_coll_monitoring_ialltoall(const void *sbuf, size_t scount,
                                          struct ompi_datatype_t *sdtype,
-                                         void *rbuf, int rcount,
+                                         void *rbuf, size_t rcount,
                                          struct ompi_datatype_t *rdtype,
                                          struct ompi_communicator_t *comm,
                                          ompi_request_t ** request,
                                          mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_ialltoallv(const void *sbuf, const int *scounts,
-                                          const int *sdisps,
+extern int mca_coll_monitoring_ialltoallv(const void *sbuf, ompi_count_array_t scounts,
+                                          ompi_disp_array_t sdisps,
                                           struct ompi_datatype_t *sdtype,
-                                          void *rbuf, const int *rcounts,
-                                          const int *rdisps,
+                                          void *rbuf, ompi_count_array_t rcounts,
+                                          ompi_disp_array_t rdisps,
                                           struct ompi_datatype_t *rdtype,
                                           struct ompi_communicator_t *comm,
                                           ompi_request_t ** request,
                                           mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_ialltoallw(const void *sbuf, const int *scounts,
-                                          const int *sdisps,
+extern int mca_coll_monitoring_ialltoallw(const void *sbuf, ompi_count_array_t scounts,
+                                          ompi_disp_array_t sdisps,
                                           struct ompi_datatype_t * const *sdtypes,
-                                          void *rbuf, const int *rcounts,
-                                          const int *rdisps,
+                                          void *rbuf, ompi_count_array_t rcounts,
+                                          ompi_disp_array_t rdisps,
                                           struct ompi_datatype_t * const *rdtypes,
                                           struct ompi_communicator_t *comm,
                                           ompi_request_t ** request,
@@ -221,37 +221,37 @@ extern int mca_coll_monitoring_ibarrier(struct ompi_communicator_t *comm,
                                         ompi_request_t ** request,
                                         mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_ibcast(void *buff, int count,
+extern int mca_coll_monitoring_ibcast(void *buff, size_t count,
                                       struct ompi_datatype_t *datatype,
                                       int root,
                                       struct ompi_communicator_t *comm,
                                       ompi_request_t ** request,
                                       mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_iexscan(const void *sbuf, void *rbuf, int count,
+extern int mca_coll_monitoring_iexscan(const void *sbuf, void *rbuf, size_t count,
                                        struct ompi_datatype_t *dtype,
                                        struct ompi_op_t *op,
                                        struct ompi_communicator_t *comm,
                                        ompi_request_t ** request,
                                        mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_igather(const void *sbuf, int scount,
+extern int mca_coll_monitoring_igather(const void *sbuf, size_t scount,
                                        struct ompi_datatype_t *sdtype,
-                                       void *rbuf, int rcount, struct ompi_datatype_t *rdtype,
+                                       void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
                                        int root, struct ompi_communicator_t *comm,
                                        ompi_request_t ** request,
                                        mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_igatherv(const void *sbuf, int scount,
+extern int mca_coll_monitoring_igatherv(const void *sbuf, size_t scount,
                                         struct ompi_datatype_t *sdtype,
-                                        void *rbuf, const int *rcounts, const int *disps,
+                                        void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
                                         struct ompi_datatype_t *rdtype,
                                         int root,
                                         struct ompi_communicator_t *comm,
                                         ompi_request_t ** request,
                                         mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_ireduce(const void *sbuf, void *rbuf, int count,
+extern int mca_coll_monitoring_ireduce(const void *sbuf, void *rbuf, size_t count,
                                        struct ompi_datatype_t *dtype,
                                        struct ompi_op_t *op,
                                        int root,
@@ -260,7 +260,7 @@ extern int mca_coll_monitoring_ireduce(const void *sbuf, void *rbuf, int count,
                                        mca_coll_base_module_t *module);
 
 extern int mca_coll_monitoring_ireduce_scatter(const void *sbuf, void *rbuf,
-                                               const int *rcounts,
+                                               ompi_count_array_t rcounts,
                                                struct ompi_datatype_t *dtype,
                                                struct ompi_op_t *op,
                                                struct ompi_communicator_t *comm,
@@ -268,32 +268,32 @@ extern int mca_coll_monitoring_ireduce_scatter(const void *sbuf, void *rbuf,
                                                mca_coll_base_module_t *module);
 
 extern int mca_coll_monitoring_ireduce_scatter_block(const void *sbuf, void *rbuf,
-                                                     int rcount,
+                                                     size_t rcount,
                                                      struct ompi_datatype_t *dtype,
                                                      struct ompi_op_t *op,
                                                      struct ompi_communicator_t *comm,
                                                      ompi_request_t ** request,
                                                      mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_iscan(const void *sbuf, void *rbuf, int count,
+extern int mca_coll_monitoring_iscan(const void *sbuf, void *rbuf, size_t count,
                                      struct ompi_datatype_t *dtype,
                                      struct ompi_op_t *op,
                                      struct ompi_communicator_t *comm,
                                      ompi_request_t ** request,
                                      mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_iscatter(const void *sbuf, int scount,
+extern int mca_coll_monitoring_iscatter(const void *sbuf, size_t scount,
                                         struct ompi_datatype_t *sdtype,
-                                        void *rbuf, int rcount,
+                                        void *rbuf, size_t rcount,
                                         struct ompi_datatype_t *rdtype,
                                         int root,
                                         struct ompi_communicator_t *comm,
                                         ompi_request_t ** request,
                                         mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_iscatterv(const void *sbuf, const int *scounts, const int *disps,
+extern int mca_coll_monitoring_iscatterv(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t disps,
                                          struct ompi_datatype_t *sdtype,
-                                         void *rbuf, int rcount,
+                                         void *rbuf, size_t rcount,
                                          struct ompi_datatype_t *rdtype,
                                          int root,
                                          struct ompi_communicator_t *comm,
@@ -301,82 +301,82 @@ extern int mca_coll_monitoring_iscatterv(const void *sbuf, const int *scounts, c
                                          mca_coll_base_module_t *module);
 
 /* Neighbor */
-extern int mca_coll_monitoring_neighbor_allgather(const void *sbuf, int scount,
+extern int mca_coll_monitoring_neighbor_allgather(const void *sbuf, size_t scount,
                                                   struct ompi_datatype_t *sdtype, void *rbuf,
-                                                  int rcount, struct ompi_datatype_t *rdtype,
+                                                  size_t rcount, struct ompi_datatype_t *rdtype,
                                                   struct ompi_communicator_t *comm,
                                                   mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_neighbor_allgatherv(const void *sbuf, int scount,
+extern int mca_coll_monitoring_neighbor_allgatherv(const void *sbuf, size_t scount,
                                                    struct ompi_datatype_t *sdtype, void * rbuf,
-                                                   const int *rcounts, const int *disps,
+                                                   ompi_count_array_t rcounts, ompi_disp_array_t disps,
                                                    struct ompi_datatype_t *rdtype,
                                                    struct ompi_communicator_t *comm,
                                                    mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_neighbor_alltoall(const void *sbuf, int scount,
+extern int mca_coll_monitoring_neighbor_alltoall(const void *sbuf, size_t scount,
                                                  struct ompi_datatype_t *sdtype,
-                                                 void *rbuf, int rcount,
+                                                 void *rbuf, size_t rcount,
                                                  struct ompi_datatype_t *rdtype,
                                                  struct ompi_communicator_t *comm,
                                                  mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_neighbor_alltoallv(const void *sbuf, const int *scounts,
-                                                  const int *sdisps,
+extern int mca_coll_monitoring_neighbor_alltoallv(const void *sbuf, ompi_count_array_t scounts,
+                                                  ompi_disp_array_t sdisps,
                                                   struct ompi_datatype_t *sdtype,
-                                                  void *rbuf, const int *rcounts,
-                                                  const int *rdisps,
+                                                  void *rbuf, ompi_count_array_t rcounts,
+                                                  ompi_disp_array_t rdisps,
                                                   struct ompi_datatype_t *rdtype,
                                                   struct ompi_communicator_t *comm,
                                                   mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_neighbor_alltoallw(const void *sbuf, const int *scounts,
-                                                  const MPI_Aint *sdisps,
+extern int mca_coll_monitoring_neighbor_alltoallw(const void *sbuf, ompi_count_array_t scounts,
+                                                  ompi_disp_array_t sdisps,
                                                   struct ompi_datatype_t * const *sdtypes,
-                                                  void *rbuf, const int *rcounts,
-                                                  const MPI_Aint *rdisps,
+                                                  void *rbuf, ompi_count_array_t rcounts,
+                                                  ompi_disp_array_t rdisps,
                                                   struct ompi_datatype_t * const *rdtypes,
                                                   struct ompi_communicator_t *comm,
                                                   mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_ineighbor_allgather(const void *sbuf, int scount,
+extern int mca_coll_monitoring_ineighbor_allgather(const void *sbuf, size_t scount,
                                                    struct ompi_datatype_t *sdtype, void *rbuf,
-                                                   int rcount, struct ompi_datatype_t *rdtype,
+                                                   size_t rcount, struct ompi_datatype_t *rdtype,
                                                    struct ompi_communicator_t *comm,
                                                    ompi_request_t ** request,
                                                    mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_ineighbor_allgatherv(const void *sbuf, int scount,
+extern int mca_coll_monitoring_ineighbor_allgatherv(const void *sbuf, size_t scount,
                                                     struct ompi_datatype_t *sdtype,
-                                                    void * rbuf, const int *rcounts,
-                                                    const int *disps,
+                                                    void * rbuf, ompi_count_array_t rcounts,
+                                                    ompi_disp_array_t disps,
                                                     struct ompi_datatype_t *rdtype,
                                                     struct ompi_communicator_t *comm,
                                                     ompi_request_t ** request,
                                                     mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_ineighbor_alltoall(const void *sbuf, int scount,
+extern int mca_coll_monitoring_ineighbor_alltoall(const void *sbuf, size_t scount,
                                                   struct ompi_datatype_t *sdtype, void *rbuf,
-                                                  int rcount, struct ompi_datatype_t *rdtype,
+                                                  size_t rcount, struct ompi_datatype_t *rdtype,
                                                   struct ompi_communicator_t *comm,
                                                   ompi_request_t ** request,
                                                   mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_ineighbor_alltoallv(const void *sbuf, const int *scounts,
-                                                   const int *sdisps,
+extern int mca_coll_monitoring_ineighbor_alltoallv(const void *sbuf, ompi_count_array_t scounts,
+                                                   ompi_disp_array_t sdisps,
                                                    struct ompi_datatype_t *sdtype,
-                                                   void *rbuf, const int *rcounts,
-                                                   const int *rdisps,
+                                                   void *rbuf, ompi_count_array_t rcounts,
+                                                   ompi_disp_array_t rdisps,
                                                    struct ompi_datatype_t *rdtype,
                                                    struct ompi_communicator_t *comm,
                                                    ompi_request_t ** request,
                                                    mca_coll_base_module_t *module);
 
-extern int mca_coll_monitoring_ineighbor_alltoallw(const void *sbuf, const int *scounts,
-                                                   const MPI_Aint *sdisps,
+extern int mca_coll_monitoring_ineighbor_alltoallw(const void *sbuf, ompi_count_array_t scounts,
+                                                   ompi_disp_array_t sdisps,
                                                    struct ompi_datatype_t * const *sdtypes,
-                                                   void *rbuf, const int *rcounts,
-                                                   const MPI_Aint *rdisps,
+                                                   void *rbuf, ompi_count_array_t rcounts,
+                                                   ompi_disp_array_t rdisps,
                                                    struct ompi_datatype_t * const *rdtypes,
                                                    struct ompi_communicator_t *comm,
                                                    ompi_request_t ** request,
diff --git a/ompi/mca/coll/monitoring/coll_monitoring_allgather.c b/ompi/mca/coll/monitoring/coll_monitoring_allgather.c
index 10115e2d946..25f3ff6666e 100644
--- a/ompi/mca/coll/monitoring/coll_monitoring_allgather.c
+++ b/ompi/mca/coll/monitoring/coll_monitoring_allgather.c
@@ -15,9 +15,9 @@
 #include "ompi/communicator/communicator.h"
 #include "coll_monitoring.h"
 
-int mca_coll_monitoring_allgather(const void *sbuf, int scount,
+int mca_coll_monitoring_allgather(const void *sbuf, size_t scount,
                                   struct ompi_datatype_t *sdtype,
-                                  void *rbuf, int rcount,
+                                  void *rbuf, size_t rcount,
                                   struct ompi_datatype_t *rdtype,
                                   struct ompi_communicator_t *comm,
                                   mca_coll_base_module_t *module)
@@ -34,7 +34,7 @@ int mca_coll_monitoring_allgather(const void *sbuf, int scount,
         if( i == my_rank ) continue; /* No communication for self */
         /**
          * If this fails the destination is not part of my MPI_COM_WORLD
-         * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+         * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
          */
         if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
             mca_common_monitoring_record_coll(rank, data_size);
@@ -43,9 +43,9 @@ int mca_coll_monitoring_allgather(const void *sbuf, int scount,
     return monitoring_module->real.coll_allgather(sbuf, scount, sdtype, rbuf, rcount, rdtype, comm, monitoring_module->real.coll_allgather_module);
 }
 
-int mca_coll_monitoring_iallgather(const void *sbuf, int scount,
+int mca_coll_monitoring_iallgather(const void *sbuf, size_t scount,
                                    struct ompi_datatype_t *sdtype,
-                                   void *rbuf, int rcount,
+                                   void *rbuf, size_t rcount,
                                    struct ompi_datatype_t *rdtype,
                                    struct ompi_communicator_t *comm,
                                    ompi_request_t ** request,
@@ -63,7 +63,7 @@ int mca_coll_monitoring_iallgather(const void *sbuf, int scount,
         if( my_rank == i ) continue; /* No communication for self */
         /**
          * If this fails the destination is not part of my MPI_COM_WORLD
-         * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+         * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
          */
         if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
             mca_common_monitoring_record_coll(rank, data_size);
diff --git a/ompi/mca/coll/monitoring/coll_monitoring_allgatherv.c b/ompi/mca/coll/monitoring/coll_monitoring_allgatherv.c
index 40468e73045..746d32a6738 100644
--- a/ompi/mca/coll/monitoring/coll_monitoring_allgatherv.c
+++ b/ompi/mca/coll/monitoring/coll_monitoring_allgatherv.c
@@ -15,9 +15,9 @@
 #include "ompi/communicator/communicator.h"
 #include "coll_monitoring.h"
 
-int mca_coll_monitoring_allgatherv(const void *sbuf, int scount,
+int mca_coll_monitoring_allgatherv(const void *sbuf, size_t scount,
                                    struct ompi_datatype_t *sdtype,
-                                   void * rbuf, const int *rcounts, const int *disps,
+                                   void * rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
                                    struct ompi_datatype_t *rdtype,
                                    struct ompi_communicator_t *comm,
                                    mca_coll_base_module_t *module)
@@ -34,7 +34,7 @@ int mca_coll_monitoring_allgatherv(const void *sbuf, int scount,
         if( my_rank == i ) continue; /* No communication for self */
         /**
          * If this fails the destination is not part of my MPI_COM_WORLD
-         * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+         * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
          */
         if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
             mca_common_monitoring_record_coll(rank, data_size);
@@ -43,9 +43,9 @@ int mca_coll_monitoring_allgatherv(const void *sbuf, int scount,
     return monitoring_module->real.coll_allgatherv(sbuf, scount, sdtype, rbuf, rcounts, disps, rdtype, comm, monitoring_module->real.coll_allgatherv_module);
 }
 
-int mca_coll_monitoring_iallgatherv(const void *sbuf, int scount,
+int mca_coll_monitoring_iallgatherv(const void *sbuf, size_t scount,
                                     struct ompi_datatype_t *sdtype,
-                                    void * rbuf, const int *rcounts, const int *disps,
+                                    void * rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
                                     struct ompi_datatype_t *rdtype,
                                     struct ompi_communicator_t *comm,
                                     ompi_request_t ** request,
@@ -63,7 +63,7 @@ int mca_coll_monitoring_iallgatherv(const void *sbuf, int scount,
         if( my_rank == i ) continue; /* No communication for self */
         /**
          * If this fails the destination is not part of my MPI_COM_WORLD
-         * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+         * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
          */
         if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
             mca_common_monitoring_record_coll(rank, data_size);
diff --git a/ompi/mca/coll/monitoring/coll_monitoring_allreduce.c b/ompi/mca/coll/monitoring/coll_monitoring_allreduce.c
index a1ec61ade82..afa3387c229 100644
--- a/ompi/mca/coll/monitoring/coll_monitoring_allreduce.c
+++ b/ompi/mca/coll/monitoring/coll_monitoring_allreduce.c
@@ -16,7 +16,7 @@
 #include "ompi/communicator/communicator.h"
 #include "coll_monitoring.h"
 
-int mca_coll_monitoring_allreduce(const void *sbuf, void *rbuf, int count,
+int mca_coll_monitoring_allreduce(const void *sbuf, void *rbuf, size_t count,
                                           struct ompi_datatype_t *dtype,
                                           struct ompi_op_t *op,
                                           struct ompi_communicator_t *comm,
@@ -34,7 +34,7 @@ int mca_coll_monitoring_allreduce(const void *sbuf, void *rbuf, int count,
         if( my_rank == i ) continue; /* No communication for self */
         /**
          * If this fails the destination is not part of my MPI_COM_WORLD
-         * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+         * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
          */
         if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
             mca_common_monitoring_record_coll(rank, data_size);
@@ -43,7 +43,7 @@ int mca_coll_monitoring_allreduce(const void *sbuf, void *rbuf, int count,
     return monitoring_module->real.coll_allreduce(sbuf, rbuf, count, dtype, op, comm, monitoring_module->real.coll_allreduce_module);
 }
 
-int mca_coll_monitoring_iallreduce(const void *sbuf, void *rbuf, int count,
+int mca_coll_monitoring_iallreduce(const void *sbuf, void *rbuf, size_t count,
                                           struct ompi_datatype_t *dtype,
                                           struct ompi_op_t *op,
                                           struct ompi_communicator_t *comm,
@@ -62,7 +62,7 @@ int mca_coll_monitoring_iallreduce(const void *sbuf, void *rbuf, int count,
         if( my_rank == i ) continue; /* No communication for self */
         /**
          * If this fails the destination is not part of my MPI_COM_WORLD
-         * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+         * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
          */
         if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
             mca_common_monitoring_record_coll(rank, data_size);
diff --git a/ompi/mca/coll/monitoring/coll_monitoring_alltoall.c b/ompi/mca/coll/monitoring/coll_monitoring_alltoall.c
index a40ce6a4981..05ef7ba3e9c 100644
--- a/ompi/mca/coll/monitoring/coll_monitoring_alltoall.c
+++ b/ompi/mca/coll/monitoring/coll_monitoring_alltoall.c
@@ -15,8 +15,8 @@
 #include "ompi/communicator/communicator.h"
 #include "coll_monitoring.h"
 
-int mca_coll_monitoring_alltoall(const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-                                 void* rbuf, int rcount, struct ompi_datatype_t *rdtype,
+int mca_coll_monitoring_alltoall(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                                 void* rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
                                  struct ompi_communicator_t *comm,
                                  mca_coll_base_module_t *module)
 {
@@ -32,7 +32,7 @@ int mca_coll_monitoring_alltoall(const void *sbuf, int scount, struct ompi_datat
         if( my_rank == i ) continue; /* No communication for self */
         /**
          * If this fails the destination is not part of my MPI_COM_WORLD
-         * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+         * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
          */
         if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
             mca_common_monitoring_record_coll(rank, data_size);
@@ -41,9 +41,9 @@ int mca_coll_monitoring_alltoall(const void *sbuf, int scount, struct ompi_datat
     return monitoring_module->real.coll_alltoall(sbuf, scount, sdtype, rbuf, rcount, rdtype, comm, monitoring_module->real.coll_alltoall_module);
 }
 
-int mca_coll_monitoring_ialltoall(const void *sbuf, int scount,
+int mca_coll_monitoring_ialltoall(const void *sbuf, size_t scount,
                                   struct ompi_datatype_t *sdtype,
-                                  void *rbuf, int rcount,
+                                  void *rbuf, size_t rcount,
                                   struct ompi_datatype_t *rdtype,
                                   struct ompi_communicator_t *comm,
                                   ompi_request_t ** request,
@@ -61,7 +61,7 @@ int mca_coll_monitoring_ialltoall(const void *sbuf, int scount,
         if( my_rank == i ) continue; /* No communication for self */
         /**
          * If this fails the destination is not part of my MPI_COM_WORLD
-         * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+         * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
          */
         if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
             mca_common_monitoring_record_coll(rank, data_size);
diff --git a/ompi/mca/coll/monitoring/coll_monitoring_alltoallv.c b/ompi/mca/coll/monitoring/coll_monitoring_alltoallv.c
index 2b5f9d1b703..6714a29cc9c 100644
--- a/ompi/mca/coll/monitoring/coll_monitoring_alltoallv.c
+++ b/ompi/mca/coll/monitoring/coll_monitoring_alltoallv.c
@@ -15,9 +15,9 @@
 #include "ompi/communicator/communicator.h"
 #include "coll_monitoring.h"
 
-int mca_coll_monitoring_alltoallv(const void *sbuf, const int *scounts, const int *sdisps,
+int mca_coll_monitoring_alltoallv(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
                                   struct ompi_datatype_t *sdtype,
-                                  void *rbuf, const int *rcounts, const int *rdisps,
+                                  void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
                                   struct ompi_datatype_t *rdtype,
                                   struct ompi_communicator_t *comm,
                                   mca_coll_base_module_t *module)
@@ -30,10 +30,10 @@ int mca_coll_monitoring_alltoallv(const void *sbuf, const int *scounts, const in
     ompi_datatype_type_size(sdtype, &type_size);
     for( i = 0; i < comm_size; ++i ) {
         if( my_rank == i ) continue; /* No communication for self */
-        data_size = scounts[i] * type_size;
+        data_size = ompi_count_array_get(scounts, i) * type_size;
         /**
          * If this fails the destination is not part of my MPI_COM_WORLD
-         * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+         * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
          */
         if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
             mca_common_monitoring_record_coll(rank, data_size);
@@ -44,11 +44,11 @@ int mca_coll_monitoring_alltoallv(const void *sbuf, const int *scounts, const in
     return monitoring_module->real.coll_alltoallv(sbuf, scounts, sdisps, sdtype, rbuf, rcounts, rdisps, rdtype, comm, monitoring_module->real.coll_alltoallv_module);
 }
 
-int mca_coll_monitoring_ialltoallv(const void *sbuf, const int *scounts,
-                                   const int *sdisps,
+int mca_coll_monitoring_ialltoallv(const void *sbuf, ompi_count_array_t scounts,
+                                   ompi_disp_array_t sdisps,
                                    struct ompi_datatype_t *sdtype,
-                                   void *rbuf, const int *rcounts,
-                                   const int *rdisps,
+                                   void *rbuf, ompi_count_array_t rcounts,
+                                   ompi_disp_array_t rdisps,
                                    struct ompi_datatype_t *rdtype,
                                    struct ompi_communicator_t *comm,
                                    ompi_request_t ** request,
@@ -62,10 +62,10 @@ int mca_coll_monitoring_ialltoallv(const void *sbuf, const int *scounts,
     ompi_datatype_type_size(sdtype, &type_size);
     for( i = 0; i < comm_size; ++i ) {
         if( my_rank == i ) continue; /* No communication for self */
-        data_size = scounts[i] * type_size;
+        data_size = ompi_count_array_get(scounts, i) * type_size;
         /**
          * If this fails the destination is not part of my MPI_COM_WORLD
-         * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+         * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
          */
         if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
             mca_common_monitoring_record_coll(rank, data_size);
diff --git a/ompi/mca/coll/monitoring/coll_monitoring_alltoallw.c b/ompi/mca/coll/monitoring/coll_monitoring_alltoallw.c
index 383d25a5593..83d14b02d07 100644
--- a/ompi/mca/coll/monitoring/coll_monitoring_alltoallw.c
+++ b/ompi/mca/coll/monitoring/coll_monitoring_alltoallw.c
@@ -15,11 +15,11 @@
 #include "ompi/communicator/communicator.h"
 #include "coll_monitoring.h"
 
-int mca_coll_monitoring_alltoallw(const void *sbuf, const int *scounts,
-                                  const int *sdisps,
+int mca_coll_monitoring_alltoallw(const void *sbuf, ompi_count_array_t scounts,
+                                  ompi_disp_array_t sdisps,
                                   struct ompi_datatype_t * const *sdtypes,
-                                  void *rbuf, const int *rcounts,
-                                  const int *rdisps,
+                                  void *rbuf, ompi_count_array_t rcounts,
+                                  ompi_disp_array_t rdisps,
                                   struct ompi_datatype_t * const *rdtypes,
                                   struct ompi_communicator_t *comm,
                                   mca_coll_base_module_t *module)
@@ -32,10 +32,10 @@ int mca_coll_monitoring_alltoallw(const void *sbuf, const int *scounts,
     for( i = 0; i < comm_size; ++i ) {
         if( my_rank == i ) continue; /* No communication for self */
         ompi_datatype_type_size(sdtypes[i], &type_size);
-        data_size = scounts[i] * type_size;
+        data_size = ompi_count_array_get(scounts, i) * type_size;
         /**
          * If this fails the destination is not part of my MPI_COM_WORLD
-         * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+         * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
          */
         if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
             mca_common_monitoring_record_coll(rank, data_size);
@@ -46,11 +46,11 @@ int mca_coll_monitoring_alltoallw(const void *sbuf, const int *scounts,
     return monitoring_module->real.coll_alltoallw(sbuf, scounts, sdisps, sdtypes, rbuf, rcounts, rdisps, rdtypes, comm, monitoring_module->real.coll_alltoallw_module);
 }
 
-int mca_coll_monitoring_ialltoallw(const void *sbuf, const int *scounts,
-                                   const int *sdisps,
+int mca_coll_monitoring_ialltoallw(const void *sbuf, ompi_count_array_t scounts,
+                                   ompi_disp_array_t sdisps,
                                    struct ompi_datatype_t * const *sdtypes,
-                                   void *rbuf, const int *rcounts,
-                                   const int *rdisps,
+                                   void *rbuf, ompi_count_array_t rcounts,
+                                   ompi_disp_array_t rdisps,
                                    struct ompi_datatype_t * const *rdtypes,
                                    struct ompi_communicator_t *comm,
                                    ompi_request_t ** request,
@@ -64,10 +64,10 @@ int mca_coll_monitoring_ialltoallw(const void *sbuf, const int *scounts,
     for( i = 0; i < comm_size; ++i ) {
         if( my_rank == i ) continue; /* No communication for self */
         ompi_datatype_type_size(sdtypes[i], &type_size);
-        data_size = scounts[i] * type_size;
+        data_size = ompi_count_array_get(scounts, i) * type_size;
         /**
          * If this fails the destination is not part of my MPI_COM_WORLD
-         * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+         * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
          */
         if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
             mca_common_monitoring_record_coll(rank, data_size);
diff --git a/ompi/mca/coll/monitoring/coll_monitoring_barrier.c b/ompi/mca/coll/monitoring/coll_monitoring_barrier.c
index c127afa07c0..169158397c8 100644
--- a/ompi/mca/coll/monitoring/coll_monitoring_barrier.c
+++ b/ompi/mca/coll/monitoring/coll_monitoring_barrier.c
@@ -25,7 +25,7 @@ int mca_coll_monitoring_barrier(struct ompi_communicator_t *comm,
 	if( my_rank == i ) continue; /* No communication for self */
 	/**
 	 * If this fails the destination is not part of my MPI_COM_WORLD
-	 * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+	 * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
 	 */
 	if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
 	    mca_common_monitoring_record_coll(rank, 0);
@@ -47,7 +47,7 @@ int mca_coll_monitoring_ibarrier(struct ompi_communicator_t *comm,
 	if( my_rank == i ) continue; /* No communication for self */
 	/**
 	 * If this fails the destination is not part of my MPI_COM_WORLD
-	 * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+	 * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
 	 */
 	if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
 	    mca_common_monitoring_record_coll(rank, 0);
diff --git a/ompi/mca/coll/monitoring/coll_monitoring_bcast.c b/ompi/mca/coll/monitoring/coll_monitoring_bcast.c
index ab34c4d5041..9e547996bd9 100644
--- a/ompi/mca/coll/monitoring/coll_monitoring_bcast.c
+++ b/ompi/mca/coll/monitoring/coll_monitoring_bcast.c
@@ -15,7 +15,7 @@
 #include "ompi/communicator/communicator.h"
 #include "coll_monitoring.h"
 
-int mca_coll_monitoring_bcast(void *buff, int count,
+int mca_coll_monitoring_bcast(void *buff, size_t count,
                               struct ompi_datatype_t *datatype,
                               int root,
                               struct ompi_communicator_t *comm,
@@ -33,7 +33,7 @@ int mca_coll_monitoring_bcast(void *buff, int count,
             if( i == root ) continue; /* No self sending */
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
                 mca_common_monitoring_record_coll(rank, data_size);
@@ -43,7 +43,7 @@ int mca_coll_monitoring_bcast(void *buff, int count,
     return monitoring_module->real.coll_bcast(buff, count, datatype, root, comm, monitoring_module->real.coll_bcast_module);
 }
 
-int mca_coll_monitoring_ibcast(void *buff, int count,
+int mca_coll_monitoring_ibcast(void *buff, size_t count,
                                struct ompi_datatype_t *datatype,
                                int root,
                                struct ompi_communicator_t *comm,
@@ -62,7 +62,7 @@ int mca_coll_monitoring_ibcast(void *buff, int count,
             if( i == root ) continue; /* No self sending */
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
                 mca_common_monitoring_record_coll(rank, data_size);
diff --git a/ompi/mca/coll/monitoring/coll_monitoring_component.c b/ompi/mca/coll/monitoring/coll_monitoring_component.c
index b23c3308ca2..670081fb70c 100644
--- a/ompi/mca/coll/monitoring/coll_monitoring_component.c
+++ b/ompi/mca/coll/monitoring/coll_monitoring_component.c
@@ -7,6 +7,7 @@
  *                         reserved.
  * Copyright (c) 2019      Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -19,51 +20,46 @@
 #include "ompi/constants.h"
 #include "ompi/communicator/communicator.h"
 #include "ompi/mca/coll/coll.h"
+#include "ompi/mca/coll/base/base.h"
 #include "opal/mca/base/mca_base_component_repository.h"
 
-#define MONITORING_SAVE_PREV_COLL_API(__module, __comm, __api)          \
-    do {                                                                \
-        if( NULL != __comm->c_coll->coll_ ## __api ## _module ) {        \
-            __module->real.coll_ ## __api = __comm->c_coll->coll_ ## __api; \
-            __module->real.coll_ ## __api ## _module = __comm->c_coll->coll_ ## __api ## _module; \
-            OBJ_RETAIN(__module->real.coll_ ## __api ## _module);       \
-        } else {                                                        \
-            /* If no function previously provided, do not monitor */    \
-            __module->super.coll_ ## __api = NULL;                      \
-            OPAL_MONITORING_PRINT_WARN("COMM \"%s\": No monitoring available for " \
-                                       "coll_" # __api, __comm->c_name); \
-        }                                                               \
-        if( NULL != __comm->c_coll->coll_i ## __api ## _module ) {       \
-            __module->real.coll_i ## __api = __comm->c_coll->coll_i ## __api; \
-            __module->real.coll_i ## __api ## _module = __comm->c_coll->coll_i ## __api ## _module; \
-            OBJ_RETAIN(__module->real.coll_i ## __api ## _module);      \
-        } else {                                                        \
-            /* If no function previously provided, do not monitor */    \
-            __module->super.coll_i ## __api = NULL;                     \
-            OPAL_MONITORING_PRINT_WARN("COMM \"%s\": No monitoring available for " \
-                                       "coll_i" # __api, __comm->c_name); \
-        }                                                               \
-    } while(0)
-
-#define MONITORING_RELEASE_PREV_COLL_API(__module, __comm, __api)       \
-    do {                                                                \
-        if( NULL != __module->real.coll_ ## __api ## _module ) {        \
-            if( NULL != __module->real.coll_ ## __api ## _module->coll_module_disable ) { \
-                __module->real.coll_ ## __api ## _module->coll_module_disable(__module->real.coll_ ## __api ## _module, __comm); \
-            }                                                           \
-            OBJ_RELEASE(__module->real.coll_ ## __api ## _module);      \
-            __module->real.coll_ ## __api = NULL;                       \
-            __module->real.coll_ ## __api ## _module = NULL;            \
-        }                                                               \
-        if( NULL != __module->real.coll_i ## __api ## _module ) {       \
-            if( NULL != __module->real.coll_i ## __api ## _module->coll_module_disable ) { \
-                __module->real.coll_i ## __api ## _module->coll_module_disable(__module->real.coll_i ## __api ## _module, __comm); \
-            }                                                           \
-            OBJ_RELEASE(__module->real.coll_i ## __api ## _module);     \
-            __module->real.coll_i ## __api = NULL;                      \
-            __module->real.coll_i ## __api ## _module = NULL;           \
-        }                                                               \
-    } while(0)
+#define MONITORING_INSTALL_COLL_API(__module, __comm, __api)                                                                        \
+    do                                                                                                                              \
+    {                                                                                                                               \
+        if ((NULL != __comm->c_coll->coll_##__api##_module) &&                                                                      \
+            (NULL != __module->super.coll_##__api))                                                                                 \
+        {                                                                                                                           \
+            /* save the current selected collective */                                                                              \
+            MCA_COLL_SAVE_API(__comm, __api, __module->real.coll_##__api, __module->real.coll_##__api##_module, "monitoring");      \
+            /* install our own */                                                                                                   \
+            MCA_COLL_INSTALL_API(__comm, __api, __module->super.coll_##__api, &__module->super, "monitoring");                      \
+        }                                                                                                                           \
+        if ((NULL != __comm->c_coll->coll_i##__api##_module) &&                                                                     \
+            (NULL != __module->super.coll_i##__api))                                                                                \
+        {                                                                                                                           \
+            /* save the current selected collective */                                                                              \
+            MCA_COLL_SAVE_API(__comm, i##__api, __module->real.coll_i##__api, __module->real.coll_i##__api##_module, "monitoring"); \
+            /* install our own */                                                                                                   \
+            MCA_COLL_INSTALL_API(__comm, i##__api, __module->super.coll_i##__api, &__module->super, "monitoring");                  \
+        }                                                                                                                           \
+    } while (0)
+
+#define MONITORING_UNINSTALL_COLL_API(__module, __comm, __api)                                                                         \
+    do                                                                                                                                 \
+    {                                                                                                                                  \
+        if (&__module->super == __comm->c_coll->coll_##__api##_module)                                                                 \
+        {                                                                                                                              \
+            MCA_COLL_INSTALL_API(__comm, __api, __module->real.coll_##__api, __module->real.coll_##__api##_module, "monitoring");      \
+            __module->real.coll_##__api = NULL;                                                                                        \
+            __module->real.coll_##__api##_module = NULL;                                                                               \
+        }                                                                                                                              \
+        if (&__module->super == __comm->c_coll->coll_i##__api##_module)                                                                \
+        {                                                                                                                              \
+            MCA_COLL_INSTALL_API(__comm, i##__api, __module->real.coll_i##__api, __module->real.coll_i##__api##_module, "monitoring"); \
+            __module->real.coll_i##__api = NULL;                                                                                       \
+            __module->real.coll_i##__api##_module = NULL;                                                                              \
+        }                                                                                                                              \
+    } while (0)
 
 #define MONITORING_SET_FULL_PREV_COLL_API(m, c, operation)      \
     do {                                                        \
@@ -91,11 +87,11 @@
         operation(m, c, neighbor_alltoallw);                    \
     } while(0)
 
-#define MONITORING_SAVE_FULL_PREV_COLL_API(m, c)                        \
-    MONITORING_SET_FULL_PREV_COLL_API((m), (c), MONITORING_SAVE_PREV_COLL_API)
+#define MONITORING_SAVE_FULL_PREV_COLL_API(m, c) \
+    MONITORING_SET_FULL_PREV_COLL_API((m), (c), MONITORING_INSTALL_COLL_API)
 
-#define MONITORING_RELEASE_FULL_PREV_COLL_API(m, c)                     \
-    MONITORING_SET_FULL_PREV_COLL_API((m), (c), MONITORING_RELEASE_PREV_COLL_API)
+#define MONITORING_RELEASE_FULL_PREV_COLL_API(m, c) \
+    MONITORING_SET_FULL_PREV_COLL_API((m), (c), MONITORING_UNINSTALL_COLL_API)
 
 static int mca_coll_monitoring_component_open(void)
 {
@@ -125,11 +121,11 @@ static int
 mca_coll_monitoring_module_enable(mca_coll_base_module_t*module, struct ompi_communicator_t*comm)
 {
     mca_coll_monitoring_module_t*monitoring_module = (mca_coll_monitoring_module_t*) module;
-    if( 1 == opal_atomic_add_fetch_32(&monitoring_module->is_initialized, 1) ) {
-        MONITORING_SAVE_FULL_PREV_COLL_API(monitoring_module, comm);
-        monitoring_module->data = mca_common_monitoring_coll_new(comm);
-        OPAL_MONITORING_PRINT_INFO("coll_module_enabled");    
-    }
+
+    MONITORING_SAVE_FULL_PREV_COLL_API(monitoring_module, comm);
+    monitoring_module->data = mca_common_monitoring_coll_new(comm);
+    OPAL_MONITORING_PRINT_INFO("coll_module_enabled");    
+
     return OMPI_SUCCESS;
 }
 
@@ -137,12 +133,12 @@ static int
 mca_coll_monitoring_module_disable(mca_coll_base_module_t*module, struct ompi_communicator_t*comm)
 {
     mca_coll_monitoring_module_t*monitoring_module = (mca_coll_monitoring_module_t*) module;
-    if( 0 == opal_atomic_sub_fetch_32(&monitoring_module->is_initialized, 1) ) {
-        MONITORING_RELEASE_FULL_PREV_COLL_API(monitoring_module, comm);
-        mca_common_monitoring_coll_release(monitoring_module->data);
-        monitoring_module->data = NULL;
-        OPAL_MONITORING_PRINT_INFO("coll_module_disabled");    
-    }
+
+    MONITORING_RELEASE_FULL_PREV_COLL_API(monitoring_module, comm);
+    mca_common_monitoring_coll_release(monitoring_module->data);
+    monitoring_module->data = NULL;
+    OPAL_MONITORING_PRINT_INFO("coll_module_disabled"); 
+
     return OMPI_SUCCESS;
 }
 
@@ -208,9 +204,6 @@ mca_coll_monitoring_component_query(struct ompi_communicator_t*comm, int*priorit
     monitoring_module->super.coll_ineighbor_alltoallv  = mca_coll_monitoring_ineighbor_alltoallv;
     monitoring_module->super.coll_ineighbor_alltoallw  = mca_coll_monitoring_ineighbor_alltoallw;
 
-    /* Initialization flag */
-    monitoring_module->is_initialized = 0;
-
     *priority = mca_coll_monitoring_component.priority;
 
     return &(monitoring_module->super);
@@ -221,7 +214,7 @@ mca_coll_monitoring_component_t mca_coll_monitoring_component = {
         /* First, the mca_base_component_t struct containing meta
            information about the component itself */
         .collm_version = {
-            MCA_COLL_BASE_VERSION_2_4_0,
+            MCA_COLL_BASE_VERSION_3_0_0,
 
             .mca_component_name = "monitoring", /* MCA component name */
             MCA_MONITORING_MAKE_VERSION,
diff --git a/ompi/mca/coll/monitoring/coll_monitoring_exscan.c b/ompi/mca/coll/monitoring/coll_monitoring_exscan.c
index acd2a0c9975..57846cb1b94 100644
--- a/ompi/mca/coll/monitoring/coll_monitoring_exscan.c
+++ b/ompi/mca/coll/monitoring/coll_monitoring_exscan.c
@@ -16,7 +16,7 @@
 #include "ompi/communicator/communicator.h"
 #include "coll_monitoring.h"
 
-int mca_coll_monitoring_exscan(const void *sbuf, void *rbuf, int count,
+int mca_coll_monitoring_exscan(const void *sbuf, void *rbuf, size_t count,
                                struct ompi_datatype_t *dtype,
                                struct ompi_op_t *op,
                                struct ompi_communicator_t *comm,
@@ -33,7 +33,7 @@ int mca_coll_monitoring_exscan(const void *sbuf, void *rbuf, int count,
     for( i = my_rank + 1; i < comm_size; ++i ) {
         /**
          * If this fails the destination is not part of my MPI_COM_WORLD
-         * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+         * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
          */
         if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
             mca_common_monitoring_record_coll(rank, data_size);
@@ -42,7 +42,7 @@ int mca_coll_monitoring_exscan(const void *sbuf, void *rbuf, int count,
     return monitoring_module->real.coll_exscan(sbuf, rbuf, count, dtype, op, comm, monitoring_module->real.coll_exscan_module);
 }
 
-int mca_coll_monitoring_iexscan(const void *sbuf, void *rbuf, int count,
+int mca_coll_monitoring_iexscan(const void *sbuf, void *rbuf, size_t count,
                                 struct ompi_datatype_t *dtype,
                                 struct ompi_op_t *op,
                                 struct ompi_communicator_t *comm,
@@ -60,7 +60,7 @@ int mca_coll_monitoring_iexscan(const void *sbuf, void *rbuf, int count,
     for( i = my_rank + 1; i < comm_size; ++i ) {
         /**
          * If this fails the destination is not part of my MPI_COM_WORLD
-         * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+         * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
          */
         if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
             mca_common_monitoring_record_coll(rank, data_size);
diff --git a/ompi/mca/coll/monitoring/coll_monitoring_gather.c b/ompi/mca/coll/monitoring/coll_monitoring_gather.c
index 628e19302d0..80fa5d6b296 100644
--- a/ompi/mca/coll/monitoring/coll_monitoring_gather.c
+++ b/ompi/mca/coll/monitoring/coll_monitoring_gather.c
@@ -15,9 +15,9 @@
 #include "ompi/communicator/communicator.h"
 #include "coll_monitoring.h"
 
-int mca_coll_monitoring_gather(const void *sbuf, int scount,
+int mca_coll_monitoring_gather(const void *sbuf, size_t scount,
                                struct ompi_datatype_t *sdtype,
-                               void *rbuf, int rcount, struct ompi_datatype_t *rdtype,
+                               void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
                                int root, struct ompi_communicator_t *comm,
                                mca_coll_base_module_t *module)
 {
@@ -32,7 +32,7 @@ int mca_coll_monitoring_gather(const void *sbuf, int scount,
             if( root == i ) continue; /* No communication for self */
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
                 mca_common_monitoring_record_coll(rank, data_size);
@@ -43,9 +43,9 @@ int mca_coll_monitoring_gather(const void *sbuf, int scount,
     return monitoring_module->real.coll_gather(sbuf, scount, sdtype, rbuf, rcount, rdtype, root, comm, monitoring_module->real.coll_gather_module);
 }
 
-int mca_coll_monitoring_igather(const void *sbuf, int scount,
+int mca_coll_monitoring_igather(const void *sbuf, size_t scount,
                                 struct ompi_datatype_t *sdtype,
-                                void *rbuf, int rcount, struct ompi_datatype_t *rdtype,
+                                void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
                                 int root, struct ompi_communicator_t *comm,
                                 ompi_request_t ** request,
                                 mca_coll_base_module_t *module)
@@ -61,7 +61,7 @@ int mca_coll_monitoring_igather(const void *sbuf, int scount,
             if( root == i ) continue; /* No communication for self */
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
                 mca_common_monitoring_record_coll(rank, data_size);
diff --git a/ompi/mca/coll/monitoring/coll_monitoring_gatherv.c b/ompi/mca/coll/monitoring/coll_monitoring_gatherv.c
index 0a285ceddda..a0ebd45a94d 100644
--- a/ompi/mca/coll/monitoring/coll_monitoring_gatherv.c
+++ b/ompi/mca/coll/monitoring/coll_monitoring_gatherv.c
@@ -15,9 +15,9 @@
 #include "ompi/communicator/communicator.h"
 #include "coll_monitoring.h"
 
-int mca_coll_monitoring_gatherv(const void *sbuf, int scount,
+int mca_coll_monitoring_gatherv(const void *sbuf, size_t scount,
                                 struct ompi_datatype_t *sdtype,
-                                void *rbuf, const int *rcounts, const int *disps,
+                                void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
                                 struct ompi_datatype_t *rdtype,
                                 int root,
                                 struct ompi_communicator_t *comm,
@@ -31,10 +31,10 @@ int mca_coll_monitoring_gatherv(const void *sbuf, int scount,
         ompi_datatype_type_size(rdtype, &type_size);
         for( i = 0; i < comm_size; ++i ) {
             if( root == i ) continue; /* No communication for self */
-            data_size = rcounts[i] * type_size;
+            data_size = ompi_count_array_get(rcounts, i) * type_size;
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
                 mca_common_monitoring_record_coll(rank, data_size);
@@ -46,9 +46,9 @@ int mca_coll_monitoring_gatherv(const void *sbuf, int scount,
     return monitoring_module->real.coll_gatherv(sbuf, scount, sdtype, rbuf, rcounts, disps, rdtype, root, comm, monitoring_module->real.coll_gatherv_module);
 }
 
-int mca_coll_monitoring_igatherv(const void *sbuf, int scount,
+int mca_coll_monitoring_igatherv(const void *sbuf, size_t scount,
                                  struct ompi_datatype_t *sdtype,
-                                 void *rbuf, const int *rcounts, const int *disps,
+                                 void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
                                  struct ompi_datatype_t *rdtype,
                                  int root,
                                  struct ompi_communicator_t *comm,
@@ -63,10 +63,10 @@ int mca_coll_monitoring_igatherv(const void *sbuf, int scount,
         ompi_datatype_type_size(rdtype, &type_size);
         for( i = 0; i < comm_size; ++i ) {
             if( root == i ) continue; /* No communication for self */
-            data_size = rcounts[i] * type_size;
+            data_size = ompi_count_array_get(rcounts, i) * type_size;
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
                 mca_common_monitoring_record_coll(rank, data_size);
diff --git a/ompi/mca/coll/monitoring/coll_monitoring_neighbor_allgather.c b/ompi/mca/coll/monitoring/coll_monitoring_neighbor_allgather.c
index fdc96da965e..b3b5fcdcae0 100644
--- a/ompi/mca/coll/monitoring/coll_monitoring_neighbor_allgather.c
+++ b/ompi/mca/coll/monitoring/coll_monitoring_neighbor_allgather.c
@@ -16,9 +16,9 @@
 #include "ompi/mca/topo/base/base.h"
 #include "coll_monitoring.h"
 
-int mca_coll_monitoring_neighbor_allgather(const void *sbuf, int scount,
+int mca_coll_monitoring_neighbor_allgather(const void *sbuf, size_t scount,
                                            struct ompi_datatype_t *sdtype, void *rbuf,
-                                           int rcount, struct ompi_datatype_t *rdtype,
+                                           size_t rcount, struct ompi_datatype_t *rdtype,
                                            struct ompi_communicator_t *comm,
                                            mca_coll_base_module_t *module)
 {
@@ -43,7 +43,7 @@ int mca_coll_monitoring_neighbor_allgather(const void *sbuf, int scount,
         if (MPI_PROC_NULL != srank) {
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(srank, comm->c_remote_group, &world_rank) ) {
                 mca_common_monitoring_record_coll(world_rank, data_size);
@@ -54,7 +54,7 @@ int mca_coll_monitoring_neighbor_allgather(const void *sbuf, int scount,
         if (MPI_PROC_NULL != drank) {
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(drank, comm->c_remote_group, &world_rank) ) {
                 mca_common_monitoring_record_coll(world_rank, data_size);
@@ -68,9 +68,9 @@ int mca_coll_monitoring_neighbor_allgather(const void *sbuf, int scount,
     return monitoring_module->real.coll_neighbor_allgather(sbuf, scount, sdtype, rbuf, rcount, rdtype, comm, monitoring_module->real.coll_neighbor_allgather_module);
 }
 
-int mca_coll_monitoring_ineighbor_allgather(const void *sbuf, int scount,
+int mca_coll_monitoring_ineighbor_allgather(const void *sbuf, size_t scount,
                                             struct ompi_datatype_t *sdtype, void *rbuf,
-                                            int rcount, struct ompi_datatype_t *rdtype,
+                                            size_t rcount, struct ompi_datatype_t *rdtype,
                                             struct ompi_communicator_t *comm,
                                             ompi_request_t ** request,
                                             mca_coll_base_module_t *module)
@@ -96,7 +96,7 @@ int mca_coll_monitoring_ineighbor_allgather(const void *sbuf, int scount,
         if (MPI_PROC_NULL != srank) {
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(srank, comm->c_remote_group, &world_rank) ) {
                 mca_common_monitoring_record_coll(world_rank, data_size);
@@ -107,7 +107,7 @@ int mca_coll_monitoring_ineighbor_allgather(const void *sbuf, int scount,
         if (MPI_PROC_NULL != drank) {
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(drank, comm->c_remote_group, &world_rank) ) {
                 mca_common_monitoring_record_coll(world_rank, data_size);
diff --git a/ompi/mca/coll/monitoring/coll_monitoring_neighbor_allgatherv.c b/ompi/mca/coll/monitoring/coll_monitoring_neighbor_allgatherv.c
index db9fb7e4f57..8b00623b30a 100644
--- a/ompi/mca/coll/monitoring/coll_monitoring_neighbor_allgatherv.c
+++ b/ompi/mca/coll/monitoring/coll_monitoring_neighbor_allgatherv.c
@@ -16,9 +16,9 @@
 #include "ompi/mca/topo/base/base.h"
 #include "coll_monitoring.h"
 
-int mca_coll_monitoring_neighbor_allgatherv(const void *sbuf, int scount,
+int mca_coll_monitoring_neighbor_allgatherv(const void *sbuf, size_t scount,
                                             struct ompi_datatype_t *sdtype,
-                                            void * rbuf, const int *rcounts, const int *disps,
+                                            void * rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
                                             struct ompi_datatype_t *rdtype,
                                             struct ompi_communicator_t *comm,
                                             mca_coll_base_module_t *module)
@@ -44,7 +44,7 @@ int mca_coll_monitoring_neighbor_allgatherv(const void *sbuf, int scount,
         if (MPI_PROC_NULL != srank) {
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(srank, comm->c_remote_group, &world_rank) ) {
                 mca_common_monitoring_record_coll(world_rank, data_size);
@@ -55,7 +55,7 @@ int mca_coll_monitoring_neighbor_allgatherv(const void *sbuf, int scount,
         if (MPI_PROC_NULL != drank) {
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(drank, comm->c_remote_group, &world_rank) ) {
                 mca_common_monitoring_record_coll(world_rank, data_size);
@@ -69,9 +69,9 @@ int mca_coll_monitoring_neighbor_allgatherv(const void *sbuf, int scount,
     return monitoring_module->real.coll_neighbor_allgatherv(sbuf, scount, sdtype, rbuf, rcounts, disps, rdtype, comm, monitoring_module->real.coll_neighbor_allgatherv_module);
 }
 
-int mca_coll_monitoring_ineighbor_allgatherv(const void *sbuf, int scount,
+int mca_coll_monitoring_ineighbor_allgatherv(const void *sbuf, size_t scount,
                                              struct ompi_datatype_t *sdtype,
-                                             void * rbuf, const int *rcounts, const int *disps,
+                                             void * rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
                                              struct ompi_datatype_t *rdtype,
                                              struct ompi_communicator_t *comm,
                                              ompi_request_t ** request,
@@ -98,7 +98,7 @@ int mca_coll_monitoring_ineighbor_allgatherv(const void *sbuf, int scount,
         if (MPI_PROC_NULL != srank) {
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(srank, comm->c_remote_group, &world_rank) ) {
                 mca_common_monitoring_record_coll(world_rank, data_size);
@@ -109,7 +109,7 @@ int mca_coll_monitoring_ineighbor_allgatherv(const void *sbuf, int scount,
         if (MPI_PROC_NULL != drank) {
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(drank, comm->c_remote_group, &world_rank) ) {
                 mca_common_monitoring_record_coll(world_rank, data_size);
diff --git a/ompi/mca/coll/monitoring/coll_monitoring_neighbor_alltoall.c b/ompi/mca/coll/monitoring/coll_monitoring_neighbor_alltoall.c
index 51e1b4eb385..cfa95954ed4 100644
--- a/ompi/mca/coll/monitoring/coll_monitoring_neighbor_alltoall.c
+++ b/ompi/mca/coll/monitoring/coll_monitoring_neighbor_alltoall.c
@@ -16,9 +16,9 @@
 #include "ompi/mca/topo/base/base.h"
 #include "coll_monitoring.h"
 
-int mca_coll_monitoring_neighbor_alltoall(const void *sbuf, int scount,
+int mca_coll_monitoring_neighbor_alltoall(const void *sbuf, size_t scount,
                                           struct ompi_datatype_t *sdtype,
-                                          void* rbuf, int rcount,
+                                          void* rbuf, size_t rcount,
                                           struct ompi_datatype_t *rdtype,
                                           struct ompi_communicator_t *comm,
                                           mca_coll_base_module_t *module)
@@ -44,7 +44,7 @@ int mca_coll_monitoring_neighbor_alltoall(const void *sbuf, int scount,
         if (MPI_PROC_NULL != srank) {
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(srank, comm->c_remote_group, &world_rank) ) {
                 mca_common_monitoring_record_coll(world_rank, data_size);
@@ -55,7 +55,7 @@ int mca_coll_monitoring_neighbor_alltoall(const void *sbuf, int scount,
         if (MPI_PROC_NULL != drank) {
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(drank, comm->c_remote_group, &world_rank) ) {
                 mca_common_monitoring_record_coll(world_rank, data_size);
@@ -69,9 +69,9 @@ int mca_coll_monitoring_neighbor_alltoall(const void *sbuf, int scount,
     return monitoring_module->real.coll_neighbor_alltoall(sbuf, scount, sdtype, rbuf, rcount, rdtype, comm, monitoring_module->real.coll_neighbor_alltoall_module);
 }
 
-int mca_coll_monitoring_ineighbor_alltoall(const void *sbuf, int scount,
+int mca_coll_monitoring_ineighbor_alltoall(const void *sbuf, size_t scount,
                                            struct ompi_datatype_t *sdtype,
-                                           void *rbuf, int rcount,
+                                           void *rbuf, size_t rcount,
                                            struct ompi_datatype_t *rdtype,
                                            struct ompi_communicator_t *comm,
                                            ompi_request_t ** request,
@@ -98,7 +98,7 @@ int mca_coll_monitoring_ineighbor_alltoall(const void *sbuf, int scount,
         if (MPI_PROC_NULL != srank) {
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(srank, comm->c_remote_group, &world_rank) ) {
                 mca_common_monitoring_record_coll(world_rank, data_size);
@@ -109,7 +109,7 @@ int mca_coll_monitoring_ineighbor_alltoall(const void *sbuf, int scount,
         if (MPI_PROC_NULL != drank) {
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(drank, comm->c_remote_group, &world_rank) ) {
                 mca_common_monitoring_record_coll(world_rank, data_size);
diff --git a/ompi/mca/coll/monitoring/coll_monitoring_neighbor_alltoallv.c b/ompi/mca/coll/monitoring/coll_monitoring_neighbor_alltoallv.c
index b979c32c771..57cf99dd4eb 100644
--- a/ompi/mca/coll/monitoring/coll_monitoring_neighbor_alltoallv.c
+++ b/ompi/mca/coll/monitoring/coll_monitoring_neighbor_alltoallv.c
@@ -16,9 +16,9 @@
 #include "ompi/mca/topo/base/base.h"
 #include "coll_monitoring.h"
 
-int mca_coll_monitoring_neighbor_alltoallv(const void *sbuf, const int *scounts,
-                                           const int *sdisps, struct ompi_datatype_t *sdtype,
-                                           void *rbuf, const int *rcounts, const int *rdisps,
+int mca_coll_monitoring_neighbor_alltoallv(const void *sbuf, ompi_count_array_t scounts,
+                                           ompi_disp_array_t sdisps, struct ompi_datatype_t *sdtype,
+                                           void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
                                            struct ompi_datatype_t *rdtype,
                                            struct ompi_communicator_t *comm,
                                            mca_coll_base_module_t *module)
@@ -41,10 +41,10 @@ int mca_coll_monitoring_neighbor_alltoallv(const void *sbuf, const int *scounts,
         }
 
         if (MPI_PROC_NULL != srank) {
-            data_size = scounts[i] * type_size;
+            data_size = ompi_count_array_get(scounts, i) * type_size;
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(srank, comm->c_remote_group, &world_rank) ) {
                 mca_common_monitoring_record_coll(world_rank, data_size);
@@ -54,10 +54,10 @@ int mca_coll_monitoring_neighbor_alltoallv(const void *sbuf, const int *scounts,
         }
 
         if (MPI_PROC_NULL != drank) {
-            data_size = scounts[i] * type_size;
+            data_size = ompi_count_array_get(scounts, i) * type_size;
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(drank, comm->c_remote_group, &world_rank) ) {
                 mca_common_monitoring_record_coll(world_rank, data_size);
@@ -72,11 +72,11 @@ int mca_coll_monitoring_neighbor_alltoallv(const void *sbuf, const int *scounts,
     return monitoring_module->real.coll_neighbor_alltoallv(sbuf, scounts, sdisps, sdtype, rbuf, rcounts, rdisps, rdtype, comm, monitoring_module->real.coll_neighbor_alltoallv_module);
 }
 
-int mca_coll_monitoring_ineighbor_alltoallv(const void *sbuf, const int *scounts,
-                                            const int *sdisps,
+int mca_coll_monitoring_ineighbor_alltoallv(const void *sbuf, ompi_count_array_t scounts,
+                                            ompi_disp_array_t sdisps,
                                             struct ompi_datatype_t *sdtype,
-                                            void *rbuf, const int *rcounts,
-                                            const int *rdisps,
+                                            void *rbuf, ompi_count_array_t rcounts,
+                                            ompi_disp_array_t rdisps,
                                             struct ompi_datatype_t *rdtype,
                                             struct ompi_communicator_t *comm,
                                             ompi_request_t ** request,
@@ -100,10 +100,10 @@ int mca_coll_monitoring_ineighbor_alltoallv(const void *sbuf, const int *scounts
         }
 
         if (MPI_PROC_NULL != srank) {
-            data_size = scounts[i] * type_size;
+            data_size = ompi_count_array_get(scounts, i) * type_size;
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(srank, comm->c_remote_group, &world_rank) ) {
                 mca_common_monitoring_record_coll(world_rank, data_size);
@@ -113,10 +113,10 @@ int mca_coll_monitoring_ineighbor_alltoallv(const void *sbuf, const int *scounts
         }
 
         if (MPI_PROC_NULL != drank) {
-            data_size = scounts[i] * type_size;
+            data_size = ompi_count_array_get(scounts, i) * type_size;
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(drank, comm->c_remote_group, &world_rank) ) {
                 mca_common_monitoring_record_coll(world_rank, data_size);
diff --git a/ompi/mca/coll/monitoring/coll_monitoring_neighbor_alltoallw.c b/ompi/mca/coll/monitoring/coll_monitoring_neighbor_alltoallw.c
index 9bbb122933f..c1c5944609b 100644
--- a/ompi/mca/coll/monitoring/coll_monitoring_neighbor_alltoallw.c
+++ b/ompi/mca/coll/monitoring/coll_monitoring_neighbor_alltoallw.c
@@ -16,11 +16,11 @@
 #include "ompi/mca/topo/base/base.h"
 #include "coll_monitoring.h"
 
-int mca_coll_monitoring_neighbor_alltoallw(const void *sbuf, const int *scounts,
-                                           const MPI_Aint *sdisps,
+int mca_coll_monitoring_neighbor_alltoallw(const void *sbuf, ompi_count_array_t scounts,
+                                           ompi_disp_array_t sdisps,
                                            struct ompi_datatype_t * const *sdtypes,
-                                           void *rbuf, const int *rcounts,
-                                           const MPI_Aint *rdisps,
+                                           void *rbuf, ompi_count_array_t rcounts,
+                                           ompi_disp_array_t rdisps,
                                            struct ompi_datatype_t * const *rdtypes,
                                            struct ompi_communicator_t *comm,
                                            mca_coll_base_module_t *module)
@@ -42,10 +42,10 @@ int mca_coll_monitoring_neighbor_alltoallw(const void *sbuf, const int *scounts,
 
         if (MPI_PROC_NULL != srank) {
             ompi_datatype_type_size(sdtypes[i], &type_size);
-            data_size = scounts[i] * type_size;
+            data_size = ompi_count_array_get(scounts, i) * type_size;
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(srank, comm->c_remote_group, &world_rank) ) {
                 mca_common_monitoring_record_coll(world_rank, data_size);
@@ -56,10 +56,10 @@ int mca_coll_monitoring_neighbor_alltoallw(const void *sbuf, const int *scounts,
 
         if (MPI_PROC_NULL != drank) {
             ompi_datatype_type_size(sdtypes[i], &type_size);
-            data_size = scounts[i] * type_size;
+            data_size = ompi_count_array_get(scounts, i) * type_size;
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(drank, comm->c_remote_group, &world_rank) ) {
                 mca_common_monitoring_record_coll(world_rank, data_size);
@@ -74,11 +74,11 @@ int mca_coll_monitoring_neighbor_alltoallw(const void *sbuf, const int *scounts,
     return monitoring_module->real.coll_neighbor_alltoallw(sbuf, scounts, sdisps, sdtypes, rbuf, rcounts, rdisps, rdtypes, comm, monitoring_module->real.coll_neighbor_alltoallw_module);
 }
 
-int mca_coll_monitoring_ineighbor_alltoallw(const void *sbuf, const int *scounts,
-                                            const MPI_Aint *sdisps,
+int mca_coll_monitoring_ineighbor_alltoallw(const void *sbuf, ompi_count_array_t scounts,
+                                            ompi_disp_array_t sdisps,
                                             struct ompi_datatype_t * const *sdtypes,
-                                            void *rbuf, const int *rcounts,
-                                            const MPI_Aint *rdisps,
+                                            void *rbuf, ompi_count_array_t rcounts,
+                                            ompi_disp_array_t rdisps,
                                             struct ompi_datatype_t * const *rdtypes,
                                             struct ompi_communicator_t *comm,
                                             ompi_request_t ** request,
@@ -101,10 +101,10 @@ int mca_coll_monitoring_ineighbor_alltoallw(const void *sbuf, const int *scounts
 
         if (MPI_PROC_NULL != srank) {
             ompi_datatype_type_size(sdtypes[i], &type_size);
-            data_size = scounts[i] * type_size;
+            data_size = ompi_count_array_get(scounts, i) * type_size;
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(srank, comm->c_remote_group, &world_rank) ) {
                 mca_common_monitoring_record_coll(world_rank, data_size);
@@ -115,10 +115,10 @@ int mca_coll_monitoring_ineighbor_alltoallw(const void *sbuf, const int *scounts
 
         if (MPI_PROC_NULL != drank) {
             ompi_datatype_type_size(sdtypes[i], &type_size);
-            data_size = scounts[i] * type_size;
+            data_size = ompi_count_array_get(scounts, i) * type_size;
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(drank, comm->c_remote_group, &world_rank) ) {
                 mca_common_monitoring_record_coll(world_rank, data_size);
diff --git a/ompi/mca/coll/monitoring/coll_monitoring_reduce.c b/ompi/mca/coll/monitoring/coll_monitoring_reduce.c
index 94778b732bc..a99fb304855 100644
--- a/ompi/mca/coll/monitoring/coll_monitoring_reduce.c
+++ b/ompi/mca/coll/monitoring/coll_monitoring_reduce.c
@@ -16,7 +16,7 @@
 #include "ompi/communicator/communicator.h"
 #include "coll_monitoring.h"
 
-int mca_coll_monitoring_reduce(const void *sbuf, void *rbuf, int count,
+int mca_coll_monitoring_reduce(const void *sbuf, void *rbuf, size_t count,
                                struct ompi_datatype_t *dtype,
                                struct ompi_op_t *op,
                                int root,
@@ -34,7 +34,7 @@ int mca_coll_monitoring_reduce(const void *sbuf, void *rbuf, int count,
             if( root == i ) continue; /* No communication for self */
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
                 mca_common_monitoring_record_coll(rank, data_size);
@@ -45,7 +45,7 @@ int mca_coll_monitoring_reduce(const void *sbuf, void *rbuf, int count,
     return monitoring_module->real.coll_reduce(sbuf, rbuf, count, dtype, op, root, comm, monitoring_module->real.coll_reduce_module);
 }
 
-int mca_coll_monitoring_ireduce(const void *sbuf, void *rbuf, int count,
+int mca_coll_monitoring_ireduce(const void *sbuf, void *rbuf, size_t count,
                                 struct ompi_datatype_t *dtype,
                                 struct ompi_op_t *op,
                                 int root,
@@ -64,7 +64,7 @@ int mca_coll_monitoring_ireduce(const void *sbuf, void *rbuf, int count,
             if( root == i ) continue; /* No communication for self */
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
                 mca_common_monitoring_record_coll(rank, data_size);
diff --git a/ompi/mca/coll/monitoring/coll_monitoring_reduce_scatter.c b/ompi/mca/coll/monitoring/coll_monitoring_reduce_scatter.c
index 0bd312c7545..137da504721 100644
--- a/ompi/mca/coll/monitoring/coll_monitoring_reduce_scatter.c
+++ b/ompi/mca/coll/monitoring/coll_monitoring_reduce_scatter.c
@@ -17,7 +17,7 @@
 #include "coll_monitoring.h"
 
 int mca_coll_monitoring_reduce_scatter(const void *sbuf, void *rbuf,
-                                       const int *rcounts,
+                                       ompi_count_array_t rcounts,
                                        struct ompi_datatype_t *dtype,
                                        struct ompi_op_t *op,
                                        struct ompi_communicator_t *comm,
@@ -31,10 +31,10 @@ int mca_coll_monitoring_reduce_scatter(const void *sbuf, void *rbuf,
     ompi_datatype_type_size(dtype, &type_size);
     for( i = 0; i < comm_size; ++i ) {
         if( my_rank == i ) continue; /* No communication for self */
-        data_size = rcounts[i] * type_size;
+        data_size = ompi_count_array_get(rcounts, i) * type_size;
         /**
          * If this fails the destination is not part of my MPI_COM_WORLD
-         * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+         * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
          */
         if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
             mca_common_monitoring_record_coll(rank, data_size);
@@ -46,7 +46,7 @@ int mca_coll_monitoring_reduce_scatter(const void *sbuf, void *rbuf,
 }
 
 int mca_coll_monitoring_ireduce_scatter(const void *sbuf, void *rbuf,
-                                        const int *rcounts,
+                                        ompi_count_array_t rcounts,
                                         struct ompi_datatype_t *dtype,
                                         struct ompi_op_t *op,
                                         struct ompi_communicator_t *comm,
@@ -61,10 +61,10 @@ int mca_coll_monitoring_ireduce_scatter(const void *sbuf, void *rbuf,
     ompi_datatype_type_size(dtype, &type_size);
     for( i = 0; i < comm_size; ++i ) {
         if( my_rank == i ) continue; /* No communication for self */
-        data_size = rcounts[i] * type_size;
+        data_size = ompi_count_array_get(rcounts, i) * type_size;
         /**
          * If this fails the destination is not part of my MPI_COM_WORLD
-         * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+         * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
          */
         if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
             mca_common_monitoring_record_coll(rank, data_size);
diff --git a/ompi/mca/coll/monitoring/coll_monitoring_reduce_scatter_block.c b/ompi/mca/coll/monitoring/coll_monitoring_reduce_scatter_block.c
index b8ddb89abd9..62c23d00406 100644
--- a/ompi/mca/coll/monitoring/coll_monitoring_reduce_scatter_block.c
+++ b/ompi/mca/coll/monitoring/coll_monitoring_reduce_scatter_block.c
@@ -17,7 +17,7 @@
 #include "coll_monitoring.h"
 
 int mca_coll_monitoring_reduce_scatter_block(const void *sbuf, void *rbuf,
-                                             int rcount,
+                                             size_t rcount,
                                              struct ompi_datatype_t *dtype,
                                              struct ompi_op_t *op,
                                              struct ompi_communicator_t *comm,
@@ -34,7 +34,7 @@ int mca_coll_monitoring_reduce_scatter_block(const void *sbuf, void *rbuf,
         if( my_rank == i ) continue; /* No communication for self */
         /**
          * If this fails the destination is not part of my MPI_COM_WORLD
-         * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+         * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
          */
         if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
             mca_common_monitoring_record_coll(rank, data_size);
@@ -45,7 +45,7 @@ int mca_coll_monitoring_reduce_scatter_block(const void *sbuf, void *rbuf,
 }
 
 int mca_coll_monitoring_ireduce_scatter_block(const void *sbuf, void *rbuf,
-                                              int rcount,
+                                              size_t rcount,
                                               struct ompi_datatype_t *dtype,
                                               struct ompi_op_t *op,
                                               struct ompi_communicator_t *comm,
@@ -63,7 +63,7 @@ int mca_coll_monitoring_ireduce_scatter_block(const void *sbuf, void *rbuf,
         if( my_rank == i ) continue; /* No communication for self */
         /**
          * If this fails the destination is not part of my MPI_COM_WORLD
-         * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+         * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
          */
         if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
             mca_common_monitoring_record_coll(rank, data_size);
diff --git a/ompi/mca/coll/monitoring/coll_monitoring_scan.c b/ompi/mca/coll/monitoring/coll_monitoring_scan.c
index 7fe8dcd2d26..c312afb968c 100644
--- a/ompi/mca/coll/monitoring/coll_monitoring_scan.c
+++ b/ompi/mca/coll/monitoring/coll_monitoring_scan.c
@@ -16,7 +16,7 @@
 #include "ompi/communicator/communicator.h"
 #include "coll_monitoring.h"
 
-int mca_coll_monitoring_scan(const void *sbuf, void *rbuf, int count,
+int mca_coll_monitoring_scan(const void *sbuf, void *rbuf, size_t count,
                              struct ompi_datatype_t *dtype,
                              struct ompi_op_t *op,
                              struct ompi_communicator_t *comm,
@@ -33,7 +33,7 @@ int mca_coll_monitoring_scan(const void *sbuf, void *rbuf, int count,
     for( i = my_rank + 1; i < comm_size; ++i ) {
         /**
          * If this fails the destination is not part of my MPI_COM_WORLD
-         * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+         * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
          */
         if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
             mca_common_monitoring_record_coll(rank, data_size);
@@ -42,7 +42,7 @@ int mca_coll_monitoring_scan(const void *sbuf, void *rbuf, int count,
     return monitoring_module->real.coll_scan(sbuf, rbuf, count, dtype, op, comm, monitoring_module->real.coll_scan_module);
 }
 
-int mca_coll_monitoring_iscan(const void *sbuf, void *rbuf, int count,
+int mca_coll_monitoring_iscan(const void *sbuf, void *rbuf, size_t count,
                               struct ompi_datatype_t *dtype,
                               struct ompi_op_t *op,
                               struct ompi_communicator_t *comm,
@@ -60,7 +60,7 @@ int mca_coll_monitoring_iscan(const void *sbuf, void *rbuf, int count,
     for( i = my_rank + 1; i < comm_size; ++i ) {
         /**
          * If this fails the destination is not part of my MPI_COM_WORLD
-         * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+         * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
          */
         if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
             mca_common_monitoring_record_coll(rank, data_size);
diff --git a/ompi/mca/coll/monitoring/coll_monitoring_scatter.c b/ompi/mca/coll/monitoring/coll_monitoring_scatter.c
index 833ec27ae6d..0cff5448141 100644
--- a/ompi/mca/coll/monitoring/coll_monitoring_scatter.c
+++ b/ompi/mca/coll/monitoring/coll_monitoring_scatter.c
@@ -15,9 +15,9 @@
 #include "ompi/communicator/communicator.h"
 #include "coll_monitoring.h"
 
-int mca_coll_monitoring_scatter(const void *sbuf, int scount,
+int mca_coll_monitoring_scatter(const void *sbuf, size_t scount,
                                 struct ompi_datatype_t *sdtype,
-                                void *rbuf, int rcount,
+                                void *rbuf, size_t rcount,
                                 struct ompi_datatype_t *rdtype,
                                 int root,
                                 struct ompi_communicator_t *comm,
@@ -35,7 +35,7 @@ int mca_coll_monitoring_scatter(const void *sbuf, int scount,
             if( my_rank == i ) continue; /* No communication for self */
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
                 mca_common_monitoring_record_coll(rank, data_size);
@@ -47,9 +47,9 @@ int mca_coll_monitoring_scatter(const void *sbuf, int scount,
 }
 
 
-int mca_coll_monitoring_iscatter(const void *sbuf, int scount,
+int mca_coll_monitoring_iscatter(const void *sbuf, size_t scount,
                                  struct ompi_datatype_t *sdtype,
-                                 void *rbuf, int rcount,
+                                 void *rbuf, size_t rcount,
                                  struct ompi_datatype_t *rdtype,
                                  int root,
                                  struct ompi_communicator_t *comm,
@@ -68,7 +68,7 @@ int mca_coll_monitoring_iscatter(const void *sbuf, int scount,
             if( my_rank == i ) continue; /* No communication for self */
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
                 mca_common_monitoring_record_coll(rank, data_size);
diff --git a/ompi/mca/coll/monitoring/coll_monitoring_scatterv.c b/ompi/mca/coll/monitoring/coll_monitoring_scatterv.c
index 458d22c8f1e..acf2c041e61 100644
--- a/ompi/mca/coll/monitoring/coll_monitoring_scatterv.c
+++ b/ompi/mca/coll/monitoring/coll_monitoring_scatterv.c
@@ -15,9 +15,9 @@
 #include "ompi/communicator/communicator.h"
 #include "coll_monitoring.h"
 
-int mca_coll_monitoring_scatterv(const void *sbuf, const int *scounts, const int *disps,
+int mca_coll_monitoring_scatterv(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t disps,
                                  struct ompi_datatype_t *sdtype,
-                                 void* rbuf, int rcount, struct ompi_datatype_t *rdtype,
+                                 void* rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
                                  int root, struct ompi_communicator_t *comm,
                                  mca_coll_base_module_t *module)
 {
@@ -29,10 +29,10 @@ int mca_coll_monitoring_scatterv(const void *sbuf, const int *scounts, const int
         int i, rank;
         ompi_datatype_type_size(sdtype, &type_size);
         for( i = 0; i < comm_size; ++i ) {
-            data_size = scounts[i] * type_size;
+            data_size = ompi_count_array_get(scounts, i) * type_size;
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
                 mca_common_monitoring_record_coll(rank, data_size);
@@ -44,9 +44,9 @@ int mca_coll_monitoring_scatterv(const void *sbuf, const int *scounts, const int
     return monitoring_module->real.coll_scatterv(sbuf, scounts, disps, sdtype, rbuf, rcount, rdtype, root, comm, monitoring_module->real.coll_scatterv_module);
 }
 
-int mca_coll_monitoring_iscatterv(const void *sbuf, const int *scounts, const int *disps,
+int mca_coll_monitoring_iscatterv(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t disps,
                                   struct ompi_datatype_t *sdtype,
-                                  void *rbuf, int rcount, struct ompi_datatype_t *rdtype,
+                                  void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
                                   int root, struct ompi_communicator_t *comm,
                                   ompi_request_t ** request,
                                   mca_coll_base_module_t *module)
@@ -59,10 +59,10 @@ int mca_coll_monitoring_iscatterv(const void *sbuf, const int *scounts, const in
         int i, rank;
         ompi_datatype_type_size(sdtype, &type_size);
         for( i = 0; i < comm_size; ++i ) {
-            data_size = scounts[i] * type_size;
+            data_size = ompi_count_array_get(scounts, i) * type_size;
             /**
              * If this fails the destination is not part of my MPI_COM_WORLD
-             * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+             * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
              */
             if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(i, comm->c_remote_group, &rank) ) {
                 mca_common_monitoring_record_coll(rank, data_size);
diff --git a/ompi/mca/coll/monitoring/configure.m4 b/ompi/mca/coll/monitoring/configure.m4
index 008bff46994..1d0b88fcb26 100644
--- a/ompi/mca/coll/monitoring/configure.m4
+++ b/ompi/mca/coll/monitoring/configure.m4
@@ -15,9 +15,6 @@
 # ------------------------------------------------
 AC_DEFUN([MCA_ompi_coll_monitoring_CONFIG],[
     AC_CONFIG_FILES([ompi/mca/coll/monitoring/Makefile])
-
-    AS_IF([test "$MCA_BUILD_ompi_common_monitoring_DSO_TRUE" = ''],
-          [$1],
-          [$2])
+    [$1]
 ])dnl
 
diff --git a/ompi/mca/coll/portals4/coll_portals4.h b/ompi/mca/coll/portals4/coll_portals4.h
index 2671466367a..6dd36c88e23 100644
--- a/ompi/mca/coll/portals4/coll_portals4.h
+++ b/ompi/mca/coll/portals4/coll_portals4.h
@@ -19,22 +19,22 @@
 
 #include "ompi_config.h"
 
-#include <portals4.h>
-#include "mpi.h"
+#include "opal/datatype/opal_convertor.h"
+
+#include "ompi/communicator/communicator.h"
 #include "ompi/constants.h"
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/datatype/ompi_datatype_internal.h"
-#include "ompi/op/op.h"
 #include "ompi/mca/mca.h"
-#include "opal/datatype/opal_convertor.h"
+#include "ompi/mca/coll/base/base.h"
 #include "ompi/mca/coll/coll.h"
+#include "ompi/op/op.h"
 #include "ompi/request/request.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/mca/coll/base/base.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/mtl/portals4/mtl_portals4_endpoint.h"
 
 #include "ompi/mca/mtl/portals4/mtl_portals4.h"
+#include "ompi/mca/mtl/portals4/mtl_portals4_endpoint.h"
+
+#include <portals4.h>
 
 #define MAXTREEFANOUT 32
 
@@ -73,7 +73,7 @@ struct mca_coll_portals4_component_t {
 
 };
 typedef struct mca_coll_portals4_component_t mca_coll_portals4_component_t;
-OMPI_MODULE_DECLSPEC extern mca_coll_portals4_component_t mca_coll_portals4_component;
+OMPI_DECLSPEC extern mca_coll_portals4_component_t mca_coll_portals4_component;
 
 
 /*
@@ -95,11 +95,31 @@ struct mca_coll_portals4_module_t {
     opal_atomic_size_t coll_count;
 
     /* record handlers dedicated to fallback if offloaded operations are not supported */
+    mca_coll_base_module_barrier_fn_t previous_barrier;
+    mca_coll_base_module_t *previous_barrier_module;
+    mca_coll_base_module_ibarrier_fn_t previous_ibarrier;
+    mca_coll_base_module_t *previous_ibarrier_module;
+
+    mca_coll_base_module_bcast_fn_t previous_bcast;
+    mca_coll_base_module_t *previous_bcast_module;
+    mca_coll_base_module_ibcast_fn_t previous_ibcast;
+    mca_coll_base_module_t *previous_ibcast_module;
+
+    mca_coll_base_module_gather_fn_t previous_gather;
+    mca_coll_base_module_t *previous_gather_module;
+    mca_coll_base_module_igather_fn_t previous_igather;
+    mca_coll_base_module_t *previous_igather_module;
+
     mca_coll_base_module_reduce_fn_t previous_reduce;
     mca_coll_base_module_t *previous_reduce_module;
     mca_coll_base_module_ireduce_fn_t previous_ireduce;
     mca_coll_base_module_t *previous_ireduce_module;
 
+    mca_coll_base_module_scatter_fn_t previous_scatter;
+    mca_coll_base_module_t *previous_scatter_module;
+    mca_coll_base_module_iscatter_fn_t previous_iscatter;
+    mca_coll_base_module_t *previous_iscatter_module;
+
     mca_coll_base_module_allreduce_fn_t previous_allreduce;
     mca_coll_base_module_t *previous_allreduce_module;
     mca_coll_base_module_iallreduce_fn_t previous_iallreduce;
@@ -187,22 +207,22 @@ int ompi_coll_portals4_ibarrier_intra(struct ompi_communicator_t *comm,
         mca_coll_base_module_t *module);
 int ompi_coll_portals4_ibarrier_intra_fini(struct ompi_coll_portals4_request_t *request);
 
-int ompi_coll_portals4_bcast_intra(void *buff, int count,
+int ompi_coll_portals4_bcast_intra(void *buff, size_t count,
         struct ompi_datatype_t *datatype, int root,
         struct ompi_communicator_t *comm,mca_coll_base_module_t *module);
-int ompi_coll_portals4_ibcast_intra(void *buff, int count,
+int ompi_coll_portals4_ibcast_intra(void *buff, size_t count,
         struct ompi_datatype_t *datatype, int root,
         struct ompi_communicator_t *comm,
         ompi_request_t **request,
         mca_coll_base_module_t *module);
 int ompi_coll_portals4_ibcast_intra_fini(struct ompi_coll_portals4_request_t *request);
 
-int ompi_coll_portals4_reduce_intra(const void *sbuf, void *rbuf, int count,
+int ompi_coll_portals4_reduce_intra(const void *sbuf, void *rbuf, size_t count,
         MPI_Datatype dtype, MPI_Op op,
         int root,
         struct ompi_communicator_t *comm,
         mca_coll_base_module_t *module);
-int ompi_coll_portals4_ireduce_intra(const void* sendbuf, void* recvbuf, int count,
+int ompi_coll_portals4_ireduce_intra(const void* sendbuf, void* recvbuf, size_t count,
         MPI_Datatype dype, MPI_Op op,
         int root,
         struct ompi_communicator_t *comm,
@@ -210,11 +230,11 @@ int ompi_coll_portals4_ireduce_intra(const void* sendbuf, void* recvbuf, int cou
         mca_coll_base_module_t *module);
 int ompi_coll_portals4_ireduce_intra_fini(struct ompi_coll_portals4_request_t *request);
 
-int ompi_coll_portals4_allreduce_intra(const void* sendbuf, void* recvbuf, int count,
+int ompi_coll_portals4_allreduce_intra(const void* sendbuf, void* recvbuf, size_t count,
         MPI_Datatype dtype, MPI_Op op,
         struct ompi_communicator_t *comm,
         mca_coll_base_module_t *module);
-int ompi_coll_portals4_iallreduce_intra(const void* sendbuf, void* recvbuf, int count,
+int ompi_coll_portals4_iallreduce_intra(const void* sendbuf, void* recvbuf, size_t count,
         MPI_Datatype dtype, MPI_Op op,
         struct ompi_communicator_t *comm,
         ompi_request_t ** ompi_request,
@@ -222,26 +242,26 @@ int ompi_coll_portals4_iallreduce_intra(const void* sendbuf, void* recvbuf, int
 int
 ompi_coll_portals4_iallreduce_intra_fini(struct ompi_coll_portals4_request_t *request);
 
-int ompi_coll_portals4_gather_intra(const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-                                    void *rbuf, int rcount, struct ompi_datatype_t *rdtype,
+int ompi_coll_portals4_gather_intra(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                                    void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
                                     int root,
                                     struct ompi_communicator_t *comm,
                                     mca_coll_base_module_t *module);
-int ompi_coll_portals4_igather_intra(const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-                                     void *rbuf, int rcount, struct ompi_datatype_t *rdtype,
+int ompi_coll_portals4_igather_intra(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                                     void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
                                      int root,
                                      struct ompi_communicator_t *comm,
                                      ompi_request_t **request,
                                      mca_coll_base_module_t *module);
 int ompi_coll_portals4_igather_intra_fini(struct ompi_coll_portals4_request_t *request);
 
-int ompi_coll_portals4_scatter_intra(const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-                                     void *rbuf, int rcount, struct ompi_datatype_t *rdtype,
+int ompi_coll_portals4_scatter_intra(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                                     void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
                                      int root,
                                      struct ompi_communicator_t *comm,
                                      mca_coll_base_module_t *module);
-int ompi_coll_portals4_iscatter_intra(const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-                                      void *rbuf, int rcount, struct ompi_datatype_t *rdtype,
+int ompi_coll_portals4_iscatter_intra(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                                      void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
                                       int root,
                                       struct ompi_communicator_t *comm,
                                       ompi_request_t **request,
diff --git a/ompi/mca/coll/portals4/coll_portals4_allreduce.c b/ompi/mca/coll/portals4/coll_portals4_allreduce.c
index 513457d7296..666180b832a 100644
--- a/ompi/mca/coll/portals4/coll_portals4_allreduce.c
+++ b/ompi/mca/coll/portals4/coll_portals4_allreduce.c
@@ -12,27 +12,27 @@
  */
 
 #include "ompi_config.h"
-#include "coll_portals4.h"
-#include "coll_portals4_request.h"
 
-#include <stdio.h>
+#include "opal/util/bit_ops.h"
 
-#include "mpi.h"
 #include "ompi/constants.h"
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/datatype/ompi_datatype_internal.h"
-#include "ompi/op/op.h"
-#include "opal/util/bit_ops.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/mca/coll/coll.h"
 #include "ompi/mca/coll/base/base.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/op/op.h"
 
+#include "coll_portals4.h"
+#include "coll_portals4_request.h"
+
+#include <stdio.h>
 
 #define COLL_PORTALS4_ALLREDUCE_MAX_SEGMENT	128
 #define COLL_PORTALS4_ALLREDUCE_MAX_CHILDREN	2
 
 static int
-allreduce_kary_tree_top(const void *sendbuf, void *recvbuf, int count,
+allreduce_kary_tree_top(const void *sendbuf, void *recvbuf, size_t count,
         MPI_Datatype dtype, MPI_Op op,
         struct ompi_communicator_t *comm,
         ompi_coll_portals4_request_t *request,
@@ -96,13 +96,13 @@ allreduce_kary_tree_top(const void *sendbuf, void *recvbuf, int count,
          */
 
         /* Compute match bits */
-        COLL_PORTALS4_SET_BITS(match_bits_ack, ompi_comm_get_cid(comm), 1, 0,
+        COLL_PORTALS4_SET_BITS(match_bits_ack, ompi_comm_get_local_cid(comm), 1, 0,
                 COLL_PORTALS4_ALLREDUCE, 0, internal_count);
 
-        COLL_PORTALS4_SET_BITS(match_bits_rtr, ompi_comm_get_cid(comm), 0, 1,
+        COLL_PORTALS4_SET_BITS(match_bits_rtr, ompi_comm_get_local_cid(comm), 0, 1,
                 COLL_PORTALS4_ALLREDUCE, 0, internal_count);
 
-        COLL_PORTALS4_SET_BITS(match_bits, ompi_comm_get_cid(comm), 0, 0,
+        COLL_PORTALS4_SET_BITS(match_bits, ompi_comm_get_local_cid(comm), 0, 0,
                 COLL_PORTALS4_ALLREDUCE, 0, internal_count);
 
         if ((ret = PtlCTAlloc(mca_coll_portals4_component.ni_h, &request->u.allreduce.trig_ct_h)) != 0) {
@@ -219,7 +219,7 @@ allreduce_kary_tree_top(const void *sendbuf, void *recvbuf, int count,
             }
 
             /*
-             ** Prepare ME for receving RTR
+             ** Prepare ME for receiving RTR
              ** Priority List, match also with "Overflow list Me" in coll_portals4_component
              */
 
@@ -380,7 +380,7 @@ allreduce_kary_tree_bottom(ompi_coll_portals4_request_t *request)
     return (OMPI_SUCCESS);
 }
 
-int ompi_coll_portals4_allreduce_intra(const void* sendbuf, void* recvbuf, int count,
+int ompi_coll_portals4_allreduce_intra(const void* sendbuf, void* recvbuf, size_t count,
         MPI_Datatype dtype, MPI_Op op,
         struct ompi_communicator_t *comm,
         mca_coll_base_module_t *module)
@@ -409,7 +409,7 @@ int ompi_coll_portals4_allreduce_intra(const void* sendbuf, void* recvbuf, int c
 }
 
 
-int ompi_coll_portals4_iallreduce_intra(const void* sendbuf, void* recvbuf, int count,
+int ompi_coll_portals4_iallreduce_intra(const void* sendbuf, void* recvbuf, size_t count,
         MPI_Datatype dtype, MPI_Op op,
         struct ompi_communicator_t *comm,
         ompi_request_t ** ompi_request,
diff --git a/ompi/mca/coll/portals4/coll_portals4_barrier.c b/ompi/mca/coll/portals4/coll_portals4_barrier.c
index e510ae88d18..e3232f4158e 100644
--- a/ompi/mca/coll/portals4/coll_portals4_barrier.c
+++ b/ompi/mca/coll/portals4/coll_portals4_barrier.c
@@ -14,16 +14,15 @@
 
 #include "ompi_config.h"
 
-#include "coll_portals4.h"
-#include "coll_portals4_request.h"
+#include "opal/util/bit_ops.h"
 
-#include "mpi.h"
 #include "ompi/constants.h"
-#include "opal/util/bit_ops.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/mca/coll/coll.h"
 #include "ompi/mca/coll/base/base.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/pml/pml.h"
 
+#include "coll_portals4.h"
+#include "coll_portals4_request.h"
 
 static int
 barrier_hypercube_top(struct ompi_communicator_t *comm,
@@ -55,10 +54,10 @@ barrier_hypercube_top(struct ompi_communicator_t *comm,
         return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
     }
 
-    COLL_PORTALS4_SET_BITS(match_bits_rtr, ompi_comm_get_cid(comm),
+    COLL_PORTALS4_SET_BITS(match_bits_rtr, ompi_comm_get_local_cid(comm),
             0, 1, COLL_PORTALS4_BARRIER, 0, count);
 
-    COLL_PORTALS4_SET_BITS(match_bits, ompi_comm_get_cid(comm),
+    COLL_PORTALS4_SET_BITS(match_bits, ompi_comm_get_local_cid(comm),
             0, 0, COLL_PORTALS4_BARRIER, 0, count);
 
     /* Build "tree" out of hypercube */
diff --git a/ompi/mca/coll/portals4/coll_portals4_bcast.c b/ompi/mca/coll/portals4/coll_portals4_bcast.c
index 8432d5823cd..9cde263ca75 100644
--- a/ompi/mca/coll/portals4/coll_portals4_bcast.c
+++ b/ompi/mca/coll/portals4/coll_portals4_bcast.c
@@ -10,16 +10,16 @@
 
 #include "ompi_config.h"
 
-#include "coll_portals4.h"
-#include "coll_portals4_request.h"
+#include "opal/util/bit_ops.h"
 
-#include "mpi.h"
 #include "ompi/constants.h"
-#include "opal/util/bit_ops.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/mca/coll/coll.h"
-#include "ompi/mca/coll/base/base.h"
 #include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/base.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/pml/pml.h"
+
+#include "coll_portals4.h"
+#include "coll_portals4_request.h"
 
 /*
  * the bcast communication is based on 1 to N scheme
@@ -200,13 +200,13 @@ bcast_kary_tree_top(void *buff, int count,
     }
 
     /* Compute match bits */
-    COLL_PORTALS4_SET_BITS(match_bits_ack, ompi_comm_get_cid(comm), 1, 0,
+    COLL_PORTALS4_SET_BITS(match_bits_ack, ompi_comm_get_local_cid(comm), 1, 0,
             COLL_PORTALS4_BCAST, 0, internal_count);
 
-    COLL_PORTALS4_SET_BITS(match_bits_rtr, ompi_comm_get_cid(comm), 0, 1,
+    COLL_PORTALS4_SET_BITS(match_bits_rtr, ompi_comm_get_local_cid(comm), 0, 1,
             COLL_PORTALS4_BCAST, 0, internal_count);
 
-    COLL_PORTALS4_SET_BITS(match_bits, ompi_comm_get_cid(comm), 0, 0,
+    COLL_PORTALS4_SET_BITS(match_bits, ompi_comm_get_local_cid(comm), 0, 0,
             COLL_PORTALS4_BCAST, 0, internal_count);
 
     /* The data will be cut in segment_nb segments.
@@ -531,13 +531,13 @@ bcast_pipeline_top(void *buff, int count,
     }
 
     /* Compute match bits */
-    COLL_PORTALS4_SET_BITS(match_bits_ack, ompi_comm_get_cid(comm), 1, 0,
+    COLL_PORTALS4_SET_BITS(match_bits_ack, ompi_comm_get_local_cid(comm), 1, 0,
             COLL_PORTALS4_BCAST, 0, internal_count);
 
-    COLL_PORTALS4_SET_BITS(match_bits_rtr, ompi_comm_get_cid(comm), 0, 1,
+    COLL_PORTALS4_SET_BITS(match_bits_rtr, ompi_comm_get_local_cid(comm), 0, 1,
             COLL_PORTALS4_BCAST, 0, internal_count);
 
-    COLL_PORTALS4_SET_BITS(match_bits, ompi_comm_get_cid(comm), 0, 0,
+    COLL_PORTALS4_SET_BITS(match_bits, ompi_comm_get_local_cid(comm), 0, 0,
             COLL_PORTALS4_BCAST, 0, internal_count);
     /* The data will be cut in segment_nb segments.
      * nb_long segments will have a size of (seg_size + 1)
@@ -834,7 +834,7 @@ bcast_pipeline_bottom(ompi_coll_portals4_request_t *request)
 
 
 int
-ompi_coll_portals4_bcast_intra(void *buff, int count,
+ompi_coll_portals4_bcast_intra(void *buff, size_t count,
         struct ompi_datatype_t *datatype, int root,
         struct ompi_communicator_t *comm,
         mca_coll_base_module_t *module)
@@ -878,7 +878,7 @@ ompi_coll_portals4_bcast_intra(void *buff, int count,
 
 
 int
-ompi_coll_portals4_ibcast_intra(void *buff, int count,
+ompi_coll_portals4_ibcast_intra(void *buff, size_t count,
         struct ompi_datatype_t *datatype, int root,
         struct ompi_communicator_t *comm,
         ompi_request_t **ompi_request,
diff --git a/ompi/mca/coll/portals4/coll_portals4_component.c b/ompi/mca/coll/portals4/coll_portals4_component.c
index 61266376e88..5c95f129703 100644
--- a/ompi/mca/coll/portals4/coll_portals4_component.c
+++ b/ompi/mca/coll/portals4/coll_portals4_component.c
@@ -10,10 +10,11 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013-2015 Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2013-2022 Sandia National Laboratories. All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2015      Bull SAS.  All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -24,14 +25,14 @@
 
 #include "ompi_config.h"
 
-#include "coll_portals4.h"
-#include "coll_portals4_request.h"
-
-#include "mpi.h"
-#include "ompi/op/op.h"
+#include "ompi/datatype/ompi_datatype.h"
 #include "ompi/datatype/ompi_datatype_internal.h"
-#include "ompi/mca/coll/coll.h"
 #include "ompi/mca/coll/base/base.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/op/op.h"
+
+#include "coll_portals4.h"
+#include "coll_portals4_request.h"
 
 #define REQ_COLL_TABLE_ID           15
 #define REQ_COLL_FINISH_TABLE_ID    16
@@ -127,20 +128,37 @@ ptl_datatype_t ompi_coll_portals4_atomic_datatype [OMPI_DATATYPE_MPI_MAX_PREDEFI
 
 };
 
-
-#define PORTALS4_SAVE_PREV_COLL_API(__module, __comm, __api)                                \
-    do {                                                                                    \
-        __module->previous_ ## __api            = __comm->c_coll->coll_ ## __api;            \
-        __module->previous_ ## __api ## _module = __comm->c_coll->coll_ ## __api ## _module; \
-        if (!comm->c_coll->coll_ ## __api || !comm->c_coll->coll_ ## __api ## _module) {      \
-            opal_output_verbose(1, ompi_coll_base_framework.framework_output,               \
-                    "(%d/%s): no underlying " # __api"; disqualifying myself",              \
-                    __comm->c_contextid, __comm->c_name);                                   \
-                    return OMPI_ERROR;                                                      \
-        }                                                                                   \
-        OBJ_RETAIN(__module->previous_ ## __api ## _module);                                \
-    } while(0)
-
+#define PORTALS4_INSTALL_COLL_API(__module, __comm, __api)                                                                \
+    do                                                                                                                    \
+    {                                                                                                                     \
+        if (!comm->c_coll->coll_##__api || !comm->c_coll->coll_##__api##_module)                                          \
+        {                                                                                                                 \
+            opal_output_verbose(1, ompi_coll_base_framework.framework_output,                                             \
+                                "(%d/%s): no underlying " #__api "; disqualifying myself",                                \
+                                ompi_comm_get_local_cid(__comm), __comm->c_name); \
+            __module->previous_##__api = NULL;                                                                            \
+            __module->previous_##__api##_module = NULL;                                                                   \
+        }                                                                                                                 \
+        else                                                                                                              \
+        {                                                                                                                 \
+            /* save the current selected collective */                                                                    \
+            MCA_COLL_SAVE_API(__comm, __api, __module->previous_##__api, __module->previous_##__api##_module, "portals"); \
+            /* install our own */                                                                                         \
+            MCA_COLL_INSTALL_API(__comm, __api, __module->super.coll_##__api, &__module->super, "portals");                \
+        }                                                                                                                 \
+    } while (0)
+
+#define PORTALS4_UNINSTALL_COLL_API(__module, __comm, __api)                                                                 \
+    do                                                                                                                       \
+    {                                                                                                                        \
+        if ((&__module->super == comm->c_coll->coll_##__api##_module) &&                                                     \
+            (NULL != __module->previous_##__api##_module))                                                                   \
+        {                                                                                                                    \
+            MCA_COLL_INSTALL_API(__comm, __api, __module->previous_##__api, __module->previous_##__api##_module, "portals"); \
+            __module->previous_##__api = NULL;                                                                               \
+            __module->previous_##__api##_module = NULL;                                                                      \
+        }                                                                                                                    \
+    } while (0)
 
 const char *mca_coll_portals4_component_version_string =
         "Open MPI Portals 4 collective MCA component version " OMPI_VERSION;
@@ -158,6 +176,8 @@ static mca_coll_base_module_t* portals4_comm_query(struct ompi_communicator_t *c
         int *priority);
 static int portals4_module_enable(mca_coll_base_module_t *module,
         struct ompi_communicator_t *comm);
+static int portals4_module_disable(mca_coll_base_module_t *module,
+        struct ompi_communicator_t *comm);
 static int portals4_progress(void);
 
 
@@ -167,7 +187,7 @@ mca_coll_portals4_component_t mca_coll_portals4_component = {
          * about the component itself */
 
         .collm_version = {
-            MCA_COLL_BASE_VERSION_2_4_0,
+            MCA_COLL_BASE_VERSION_3_0_0,
 
             /* Component name and version */
             .mca_component_name = "portals4",
@@ -618,6 +638,7 @@ portals4_comm_query(struct ompi_communicator_t *comm,
     *priority = mca_coll_portals4_priority;
     portals4_module->coll_count = 0;
     portals4_module->super.coll_module_enable = portals4_module_enable;
+    portals4_module->super.coll_module_disable = portals4_module_disable;
 
     portals4_module->super.coll_barrier = ompi_coll_portals4_barrier_intra;
     portals4_module->super.coll_ibarrier = ompi_coll_portals4_ibarrier_intra;
@@ -653,15 +674,50 @@ portals4_module_enable(mca_coll_base_module_t *module,
 {
     mca_coll_portals4_module_t *portals4_module = (mca_coll_portals4_module_t*) module;
 
-    PORTALS4_SAVE_PREV_COLL_API(portals4_module, comm, allreduce);
-    PORTALS4_SAVE_PREV_COLL_API(portals4_module, comm, iallreduce);
-    PORTALS4_SAVE_PREV_COLL_API(portals4_module, comm, reduce);
-    PORTALS4_SAVE_PREV_COLL_API(portals4_module, comm, ireduce);
+    PORTALS4_INSTALL_COLL_API(portals4_module, comm, iallreduce);
+    PORTALS4_INSTALL_COLL_API(portals4_module, comm, allreduce);
+    PORTALS4_INSTALL_COLL_API(portals4_module, comm, ireduce);
+    PORTALS4_INSTALL_COLL_API(portals4_module, comm, reduce);
+
+    PORTALS4_INSTALL_COLL_API(portals4_module, comm, barrier);
+    PORTALS4_INSTALL_COLL_API(portals4_module, comm, ibarrier);
+    PORTALS4_INSTALL_COLL_API(portals4_module, comm, gather);
+    PORTALS4_INSTALL_COLL_API(portals4_module, comm, igather);
+    PORTALS4_INSTALL_COLL_API(portals4_module, comm, scatter);
+    PORTALS4_INSTALL_COLL_API(portals4_module, comm, iscatter);
+    PORTALS4_INSTALL_COLL_API(portals4_module, comm, bcast);
+    PORTALS4_INSTALL_COLL_API(portals4_module, comm, ibcast);
 
     return OMPI_SUCCESS;
 }
 
+static int
+portals4_module_disable(mca_coll_base_module_t *module,
+                        struct ompi_communicator_t *comm)
+{
+    mca_coll_portals4_module_t *portals4_module = (mca_coll_portals4_module_t *)module;
+
+    PORTALS4_UNINSTALL_COLL_API(portals4_module, comm, allreduce);
+    PORTALS4_UNINSTALL_COLL_API(portals4_module, comm, iallreduce);
+    PORTALS4_UNINSTALL_COLL_API(portals4_module, comm, reduce);
+    PORTALS4_UNINSTALL_COLL_API(portals4_module, comm, ireduce);
+
+    PORTALS4_UNINSTALL_COLL_API(portals4_module, comm, barrier);
+    PORTALS4_UNINSTALL_COLL_API(portals4_module, comm, ibarrier);
+    PORTALS4_UNINSTALL_COLL_API(portals4_module, comm, gather);
+    PORTALS4_UNINSTALL_COLL_API(portals4_module, comm, igather);
+    PORTALS4_UNINSTALL_COLL_API(portals4_module, comm, scatter);
+    PORTALS4_UNINSTALL_COLL_API(portals4_module, comm, iscatter);
+    PORTALS4_UNINSTALL_COLL_API(portals4_module, comm, bcast);
+    PORTALS4_UNINSTALL_COLL_API(portals4_module, comm, ibcast);
 
+    return OMPI_SUCCESS;
+}
+#if OPAL_ENABLE_DEBUG
+/* These string maps are only used for debugging output.
+ * They will be compiled-out when OPAL is configured
+ * without --enable-debug.
+ */
 static char *failtype[] = {
         "PTL_NI_OK",
         "PTL_NI_PERM_VIOLATION",
@@ -695,6 +751,7 @@ static char *evname[] = {
         "PTL_EVENT_SEARCH",
         "PTL_EVENT_LINK"
 };
+#endif
 
 /* Target EQ */
 static int
diff --git a/ompi/mca/coll/portals4/coll_portals4_gather.c b/ompi/mca/coll/portals4/coll_portals4_gather.c
index 274e9d4ee89..d78234ccf95 100644
--- a/ompi/mca/coll/portals4/coll_portals4_gather.c
+++ b/ompi/mca/coll/portals4/coll_portals4_gather.c
@@ -10,13 +10,13 @@
 
 #include "ompi_config.h"
 
-#include "mpi.h"
+#include "opal/util/bit_ops.h"
+
 #include "ompi/constants.h"
 #include "ompi/datatype/ompi_datatype.h"
-#include "opal/util/bit_ops.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/mca/coll/coll.h"
 #include "ompi/mca/coll/base/base.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/pml/pml.h"
 
 #include "coll_portals4.h"
 #include "coll_portals4_request.h"
@@ -76,7 +76,7 @@ static ompi_coll_portals4_tree_t*
 ompi_coll_portals4_build_in_order_bmtree( struct ompi_communicator_t* comm,
                                             int root )
 {
-    int childs = 0, rank, vrank, vparent, size, mask = 1, remote, i;
+    int childs = 0, rank, vrank, vparent, size, mask = 1, remote;
     ompi_coll_portals4_tree_t *bmtree;
 
     /*
@@ -97,7 +97,7 @@ ompi_coll_portals4_build_in_order_bmtree( struct ompi_communicator_t* comm,
     bmtree->tree_bmtree   = 1;
     bmtree->tree_root     = MPI_UNDEFINED;
     bmtree->tree_nextsize = MPI_UNDEFINED;
-    for(i=0;i<MAXTREEFANOUT;i++) {
+    for(int i = 0; i < MAXTREEFANOUT; i++) {
         bmtree->tree_next[i] = -1;
     }
 
@@ -215,7 +215,7 @@ setup_gather_buffers_binomial(struct ompi_communicator_t   *comm,
         opal_output_verbose(30, ompi_coll_base_framework.framework_output,
                             "%s:%d:vrank(%d): root - gather_buf(%p) - gather_bytes(%lu)=packed_size(%ld) * size(%d)",
                             __FILE__, __LINE__, vrank,
-                            request->u.gather.gather_buf, request->u.gather.gather_bytes,
+                            (void*)request->u.gather.gather_buf, request->u.gather.gather_bytes,
                             request->u.gather.packed_size, request->u.gather.size);
     } else if (bmtree->tree_nextsize) {
         /*
@@ -238,7 +238,7 @@ setup_gather_buffers_binomial(struct ompi_communicator_t   *comm,
         opal_output_verbose(30, ompi_coll_base_framework.framework_output,
                             "%s:%d:vrank(%d): nonleaf - gather_buf(%p) - gather_bytes(%lu)=packed_size(%ld) * (bmtree->tree_numdescendants(%d) + 1)",
                             __FILE__, __LINE__, vrank,
-                            request->u.gather.gather_buf, request->u.gather.gather_bytes,
+                            (void*)request->u.gather.gather_buf, request->u.gather.gather_bytes,
                             request->u.gather.packed_size, bmtree->tree_numdescendants);
     } else {
         /* leaf nodes, allocate space to pack into and put from */
@@ -257,7 +257,7 @@ setup_gather_buffers_binomial(struct ompi_communicator_t   *comm,
         opal_output_verbose(30, ompi_coll_base_framework.framework_output,
                             "%s:%d:vrank(%d): leaf - gather_buf(%p) - gather_bytes(%lu)=packed_size(%ld)",
                             __FILE__, __LINE__, vrank,
-                            request->u.gather.gather_buf, request->u.gather.gather_bytes,
+                            (void*)request->u.gather.gather_buf, request->u.gather.gather_bytes,
                             request->u.gather.packed_size);
     }
 
@@ -316,7 +316,7 @@ setup_gather_buffers_linear(struct ompi_communicator_t   *comm,
         opal_output_verbose(30, ompi_coll_base_framework.framework_output,
                             "%s:%d:rank(%d): root - gather_buf(%p) - gather_bytes(%lu)=packed_size(%ld) * size(%d)",
                             __FILE__, __LINE__, request->u.gather.my_rank,
-                            request->u.gather.gather_buf, request->u.gather.gather_bytes,
+                            (void*)request->u.gather.gather_buf, request->u.gather.gather_bytes,
                             request->u.gather.packed_size, request->u.gather.size);
     } else {
         /* non-root nodes, allocate space to pack into and put from */
@@ -334,7 +334,7 @@ setup_gather_buffers_linear(struct ompi_communicator_t   *comm,
         opal_output_verbose(30, ompi_coll_base_framework.framework_output,
                             "%s:%d:rank(%d): leaf - gather_buf(%p) - gather_bytes(%lu)=packed_size(%ld)",
                             __FILE__, __LINE__, request->u.gather.my_rank,
-                            request->u.gather.gather_buf, request->u.gather.gather_bytes,
+                            (void*)request->u.gather.gather_buf, request->u.gather.gather_bytes,
                             request->u.gather.packed_size);
     }
 
@@ -360,7 +360,7 @@ setup_gather_handles(struct ompi_communicator_t   *comm,
     /**********************************/
     /* Setup Gather Handles           */
     /**********************************/
-    COLL_PORTALS4_SET_BITS(request->u.gather.gather_match_bits, ompi_comm_get_cid(comm),
+    COLL_PORTALS4_SET_BITS(request->u.gather.gather_match_bits, ompi_comm_get_local_cid(comm),
             0, 0, COLL_PORTALS4_GATHER, 0, request->u.gather.coll_count);
 
     ret = PtlCTAlloc(mca_coll_portals4_component.ni_h,
@@ -413,7 +413,7 @@ setup_sync_handles(struct ompi_communicator_t   *comm,
     /**********************************/
     /* Setup Sync Handles             */
     /**********************************/
-    COLL_PORTALS4_SET_BITS(request->u.gather.sync_match_bits, ompi_comm_get_cid(comm),
+    COLL_PORTALS4_SET_BITS(request->u.gather.sync_match_bits, ompi_comm_get_local_cid(comm),
             0, 1, COLL_PORTALS4_GATHER, 0, request->u.gather.coll_count);
 
     ret = PtlCTAlloc(mca_coll_portals4_component.ni_h,
@@ -505,8 +505,8 @@ cleanup_sync_handles(ompi_coll_portals4_request_t *request)
 }
 
 static int
-ompi_coll_portals4_gather_intra_binomial_top(const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-                                             void *rbuf, int rcount, struct ompi_datatype_t *rdtype,
+ompi_coll_portals4_gather_intra_binomial_top(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                                             void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
                                              int root,
                                              struct ompi_communicator_t *comm,
                                              ompi_coll_portals4_request_t *request,
@@ -520,8 +520,6 @@ ompi_coll_portals4_gather_intra_binomial_top(const void *sbuf, int scount, struc
 
     int vrank=-1;
 
-    int32_t i=0;
-
     ompi_coll_portals4_tree_t* bmtree;
 
     int32_t expected_ops =0;
@@ -606,7 +604,7 @@ ompi_coll_portals4_gather_intra_binomial_top(const void *sbuf, int scount, struc
                          "%s:%d: packed_size=%lu, fragment_size=%lu",
                          __FILE__, __LINE__, request->u.gather.packed_size, mca_coll_portals4_component.ni_limits.max_msg_size));
 
-    for (int i =0; i < bmtree->tree_nextsize; i++) {
+    for (int i = 0; i < bmtree->tree_nextsize; i++) {
         int child_vrank = VRANK(bmtree->tree_next[i], request->u.gather.root_rank, request->u.gather.size);
         int sub_tree_size = get_tree_numdescendants_of(comm, child_vrank) + 1;
         ptl_size_t local_number_of_fragment = ((sub_tree_size * request->u.gather.packed_size) + mca_coll_portals4_component.ni_limits.max_msg_size -1) / mca_coll_portals4_component.ni_limits.max_msg_size;
@@ -688,7 +686,7 @@ ompi_coll_portals4_gather_intra_binomial_top(const void *sbuf, int scount, struc
     /************************************/
     /* put Recv-ACK to each child       */
     /************************************/
-    for (i=0;i<bmtree->tree_nextsize;i++) {
+    for (int i = 0; i < bmtree->tree_nextsize; i++) {
         int32_t child=bmtree->tree_next[i];
         ret = PtlTriggeredPut(request->u.gather.sync_mdh,
                               0,
@@ -730,7 +728,7 @@ ompi_coll_portals4_gather_intra_binomial_top(const void *sbuf, int scount, struc
     /**********************************/
     /* put RTR to each child          */
     /**********************************/
-    for (i=0;i<bmtree->tree_nextsize;i++) {
+    for (int i = 0; i < bmtree->tree_nextsize; i++) {
         int32_t child=bmtree->tree_next[i];
         ret = PtlTriggeredPut(request->u.gather.sync_mdh,
                               0,
@@ -750,7 +748,7 @@ ompi_coll_portals4_gather_intra_binomial_top(const void *sbuf, int scount, struc
     /**********************************/
     /* put RTR to each child          */
     /**********************************/
-    for (i=0;i<bmtree->tree_nextsize;i++) {
+    for (int i = 0; i < bmtree->tree_nextsize; i++) {
         int32_t child=bmtree->tree_next[i];
         ret = PtlPut(request->u.gather.sync_mdh,
                      0,
@@ -803,8 +801,8 @@ ompi_coll_portals4_gather_intra_binomial_top(const void *sbuf, int scount, struc
 }
 
 static int
-ompi_coll_portals4_gather_intra_linear_top(const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-                                           void *rbuf, int rcount, struct ompi_datatype_t *rdtype,
+ompi_coll_portals4_gather_intra_linear_top(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                                           void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
                                            int root,
                                            struct ompi_communicator_t *comm,
                                            ompi_coll_portals4_request_t *request,
@@ -818,8 +816,6 @@ ompi_coll_portals4_gather_intra_linear_top(const void *sbuf, int scount, struct
 
     int8_t i_am_root;
 
-    int32_t i=0;
-
     int32_t expected_ops =0;
     int32_t expected_acks=0;
 
@@ -975,7 +971,7 @@ ompi_coll_portals4_gather_intra_linear_top(const void *sbuf, int scount, struct
     /* root puts Recv-ACK to all other ranks */
     /*****************************************/
     if (i_am_root) {
-        for (i=0;i<request->u.gather.size;i++) {
+        for (int i = 0; i < request->u.gather.size; i++) {
             if (i == request->u.gather.root_rank) { continue; }
             ret = PtlTriggeredPut(request->u.gather.sync_mdh,
                                   0,
@@ -1019,7 +1015,7 @@ ompi_coll_portals4_gather_intra_linear_top(const void *sbuf, int scount, struct
     /* root puts RTR to all other ranks */
     /************************************/
     if (i_am_root) {
-        for (i=0;i<request->u.gather.size;i++) {
+        for (int i = 0; i < request->u.gather.size; i++) {
             if (i == request->u.gather.root_rank) { continue; }
             ret = PtlTriggeredPut(request->u.gather.sync_mdh,
                                   0,
@@ -1041,7 +1037,7 @@ ompi_coll_portals4_gather_intra_linear_top(const void *sbuf, int scount, struct
     /* root puts RTR to all other ranks */
     /************************************/
     if (i_am_root) {
-        for (i=0;i<request->u.gather.size;i++) {
+        for (int i = 0; i < request->u.gather.size; i++) {
             if (i == request->u.gather.root_rank) { continue; }
             ret = PtlPut(request->u.gather.sync_mdh,
                          0,
@@ -1093,7 +1089,6 @@ ompi_coll_portals4_gather_intra_binomial_bottom(struct ompi_communicator_t *comm
                                                 ompi_coll_portals4_request_t *request)
 {
     int ret, line;
-    int i;
 
     OPAL_OUTPUT_VERBOSE((10, ompi_coll_base_framework.framework_output,
                  "coll:portals4:gather_intra_binomial_bottom enter rank %d", request->u.gather.my_rank));
@@ -1109,7 +1104,7 @@ ompi_coll_portals4_gather_intra_binomial_bottom(struct ompi_communicator_t *comm
         struct iovec iov;
         size_t max_data;
 
-        for (i=0;i<request->u.gather.size;i++) {
+        for (int i = 0; i < request->u.gather.size; i++) {
             uint64_t offset = request->u.gather.unpack_dst_extent * request->u.gather.unpack_dst_count * ((request->u.gather.my_rank + i) % request->u.gather.size);
 
             opal_output_verbose(30, ompi_coll_base_framework.framework_output,
@@ -1161,7 +1156,6 @@ ompi_coll_portals4_gather_intra_linear_bottom(struct ompi_communicator_t *comm,
                                               ompi_coll_portals4_request_t *request)
 {
     int ret, line;
-    int i;
 
     OPAL_OUTPUT_VERBOSE((10, ompi_coll_base_framework.framework_output,
                  "coll:portals4:gather_intra_linear_bottom enter rank %d", request->u.gather.my_rank));
@@ -1177,7 +1171,7 @@ ompi_coll_portals4_gather_intra_linear_bottom(struct ompi_communicator_t *comm,
         struct iovec iov;
         size_t max_data;
 
-        for (i=0;i<request->u.gather.size;i++) {
+        for (int i = 0; i < request->u.gather.size; i++) {
             ompi_coll_portals4_create_recv_converter (&request->u.gather.recv_converter,
                                                       request->u.gather.unpack_dst_buf + (request->u.gather.unpack_dst_extent*request->u.gather.unpack_dst_count*i),
                                                       ompi_comm_peer_lookup(comm, request->u.gather.my_rank),
@@ -1218,8 +1212,8 @@ ompi_coll_portals4_gather_intra_linear_bottom(struct ompi_communicator_t *comm,
 }
 
 int
-ompi_coll_portals4_gather_intra(const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-                                void *rbuf, int rcount, struct ompi_datatype_t *rdtype,
+ompi_coll_portals4_gather_intra(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                                void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
                                 int root,
                                 struct ompi_communicator_t *comm,
                                 mca_coll_base_module_t *module)
@@ -1290,8 +1284,8 @@ ompi_coll_portals4_gather_intra(const void *sbuf, int scount, struct ompi_dataty
 
 
 int
-ompi_coll_portals4_igather_intra(const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-                                 void *rbuf, int rcount, struct ompi_datatype_t *rdtype,
+ompi_coll_portals4_igather_intra(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                                 void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
                                  int root,
                                  struct ompi_communicator_t *comm,
                                  ompi_request_t **ompi_request,
diff --git a/ompi/mca/coll/portals4/coll_portals4_reduce.c b/ompi/mca/coll/portals4/coll_portals4_reduce.c
index a96db5088a9..e5f8888d529 100644
--- a/ompi/mca/coll/portals4/coll_portals4_reduce.c
+++ b/ompi/mca/coll/portals4/coll_portals4_reduce.c
@@ -12,26 +12,27 @@
  */
 
 #include "ompi_config.h"
-#include "coll_portals4.h"
-#include "coll_portals4_request.h"
 
-#include <stdio.h>
+#include "opal/util/bit_ops.h"
 
-#include "mpi.h"
 #include "ompi/constants.h"
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/datatype/ompi_datatype_internal.h"
-#include "ompi/op/op.h"
-#include "opal/util/bit_ops.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/mca/coll/coll.h"
 #include "ompi/mca/coll/base/base.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/op/op.h"
+
+#include "coll_portals4.h"
+#include "coll_portals4_request.h"
+
+#include <stdio.h>
 
 #define COLL_PORTALS4_REDUCE_MAX_CHILDREN	2
 
 
 static int
-reduce_kary_tree_top(const void *sendbuf, void *recvbuf, int count,
+reduce_kary_tree_top(const void *sendbuf, void *recvbuf, size_t count,
         MPI_Datatype dtype, MPI_Op op,
         int root,
         struct ompi_communicator_t *comm,
@@ -97,13 +98,13 @@ reduce_kary_tree_top(const void *sendbuf, void *recvbuf, int count,
          */
 
         /* Compute match bits */
-        COLL_PORTALS4_SET_BITS(match_bits_ack, ompi_comm_get_cid(comm), 1, 0,
+        COLL_PORTALS4_SET_BITS(match_bits_ack, ompi_comm_get_local_cid(comm), 1, 0,
                 COLL_PORTALS4_REDUCE, 0, internal_count);
 
-        COLL_PORTALS4_SET_BITS(match_bits_rtr, ompi_comm_get_cid(comm), 0, 1,
+        COLL_PORTALS4_SET_BITS(match_bits_rtr, ompi_comm_get_local_cid(comm), 0, 1,
                 COLL_PORTALS4_REDUCE, 0, internal_count);
 
-        COLL_PORTALS4_SET_BITS(match_bits, ompi_comm_get_cid(comm), 0, 0,
+        COLL_PORTALS4_SET_BITS(match_bits, ompi_comm_get_local_cid(comm), 0, 0,
                 COLL_PORTALS4_REDUCE, 0, internal_count);
 
         if ((ret = PtlCTAlloc(mca_coll_portals4_component.ni_h, &request->u.reduce.trig_ct_h)) != 0) {
@@ -376,7 +377,7 @@ reduce_kary_tree_bottom(ompi_coll_portals4_request_t *request)
 
 
 int
-ompi_coll_portals4_reduce_intra(const void *sendbuf, void *recvbuf, int count,
+ompi_coll_portals4_reduce_intra(const void *sendbuf, void *recvbuf, size_t count,
         MPI_Datatype dtype, MPI_Op op,
         int root,
         struct ompi_communicator_t *comm,
@@ -411,7 +412,7 @@ ompi_coll_portals4_reduce_intra(const void *sendbuf, void *recvbuf, int count,
 
 
 int
-ompi_coll_portals4_ireduce_intra(const void* sendbuf, void* recvbuf, int count,
+ompi_coll_portals4_ireduce_intra(const void* sendbuf, void* recvbuf, size_t count,
         MPI_Datatype dtype, MPI_Op op,
         int root,
         struct ompi_communicator_t *comm,
diff --git a/ompi/mca/coll/portals4/coll_portals4_request.c b/ompi/mca/coll/portals4/coll_portals4_request.c
index 001594f5d5c..9bebe56825c 100644
--- a/ompi/mca/coll/portals4/coll_portals4_request.c
+++ b/ompi/mca/coll/portals4/coll_portals4_request.c
@@ -27,7 +27,7 @@ request_free(struct ompi_request_t **ompi_req)
     ompi_coll_portals4_request_t *request =
         (ompi_coll_portals4_request_t*) *ompi_req;
 
-    if (true != request->super.req_complete) {
+    if (!REQUEST_COMPLETE(&request->super)) {
         return MPI_ERR_REQUEST;
     }
 
diff --git a/ompi/mca/coll/portals4/coll_portals4_request.h b/ompi/mca/coll/portals4/coll_portals4_request.h
index f78c2dee631..e420e73203c 100644
--- a/ompi/mca/coll/portals4/coll_portals4_request.h
+++ b/ompi/mca/coll/portals4/coll_portals4_request.h
@@ -13,7 +13,10 @@
 #ifndef COLL_PORTALS4_REQUEST_H
 #define COLL_PORTALS4_REQUEST_H
 
+#include "ompi_config.h"
+
 #include "ompi/request/request.h"
+
 #include "coll_portals4.h"
 
 
@@ -176,7 +179,6 @@ OBJ_CLASS_DECLARATION(ompi_coll_portals4_request_t);
                 req = (ompi_coll_portals4_request_t*) item;               \
                 OMPI_REQUEST_INIT(&req->super, false);                    \
                 req->super.req_mpi_object.comm = comm;                    \
-                req->super.req_complete = false;                          \
                 req->super.req_state = OMPI_REQUEST_ACTIVE;               \
     } while (0)
 
diff --git a/ompi/mca/coll/portals4/coll_portals4_scatter.c b/ompi/mca/coll/portals4/coll_portals4_scatter.c
index 0049a61d001..d62764fcaf9 100644
--- a/ompi/mca/coll/portals4/coll_portals4_scatter.c
+++ b/ompi/mca/coll/portals4/coll_portals4_scatter.c
@@ -10,13 +10,13 @@
 
 #include "ompi_config.h"
 
-#include "mpi.h"
+#include "opal/util/bit_ops.h"
+
 #include "ompi/constants.h"
 #include "ompi/datatype/ompi_datatype.h"
-#include "opal/util/bit_ops.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/mca/coll/coll.h"
 #include "ompi/mca/coll/base/base.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/pml/pml.h"
 
 #include "coll_portals4.h"
 #include "coll_portals4_request.h"
@@ -91,7 +91,7 @@ setup_scatter_buffers_linear(struct ompi_communicator_t   *comm,
         opal_output_verbose(30, ompi_coll_base_framework.framework_output,
                             "%s:%d:rank(%d): root - scatter_buf(%p) - scatter_bytes(%lu)=packed_size(%ld) * size(%d)",
                             __FILE__, __LINE__, request->u.scatter.my_rank,
-                            request->u.scatter.scatter_buf, request->u.scatter.scatter_bytes,
+                            (void*)request->u.scatter.scatter_buf, request->u.scatter.scatter_bytes,
                             request->u.scatter.packed_size, request->u.scatter.size);
     } else {
         request->u.scatter.scatter_bytes=request->u.scatter.packed_size;
@@ -104,7 +104,7 @@ setup_scatter_buffers_linear(struct ompi_communicator_t   *comm,
         opal_output_verbose(30, ompi_coll_base_framework.framework_output,
                             "%s:%d:rank(%d): leaf - scatter_buf(%p) - scatter_bytes(%lu)=packed_size(%ld)",
                             __FILE__, __LINE__, request->u.scatter.my_rank,
-                            request->u.scatter.scatter_buf, request->u.scatter.scatter_bytes,
+                            (void*)request->u.scatter.scatter_buf, request->u.scatter.scatter_bytes,
                             request->u.scatter.packed_size);
     }
 
@@ -133,7 +133,7 @@ setup_scatter_handles(struct ompi_communicator_t   *comm,
     /**********************************/
     /* Setup Scatter Handles           */
     /**********************************/
-    COLL_PORTALS4_SET_BITS(request->u.scatter.scatter_match_bits, ompi_comm_get_cid(comm),
+    COLL_PORTALS4_SET_BITS(request->u.scatter.scatter_match_bits, ompi_comm_get_local_cid(comm),
             0, 0, COLL_PORTALS4_SCATTER, 0, request->u.scatter.coll_count);
 
     OPAL_OUTPUT_VERBOSE((10, ompi_coll_base_framework.framework_output,
@@ -194,7 +194,7 @@ setup_sync_handles(struct ompi_communicator_t   *comm,
     /**********************************/
     /* Setup Sync Handles             */
     /**********************************/
-    COLL_PORTALS4_SET_BITS(request->u.scatter.sync_match_bits, ompi_comm_get_cid(comm),
+    COLL_PORTALS4_SET_BITS(request->u.scatter.sync_match_bits, ompi_comm_get_local_cid(comm),
             0, 1, COLL_PORTALS4_SCATTER, 0, request->u.scatter.coll_count);
 
     OPAL_OUTPUT_VERBOSE((10, ompi_coll_base_framework.framework_output,
@@ -306,8 +306,8 @@ cleanup_sync_handles(ompi_coll_portals4_request_t *request)
 }
 
 static int
-ompi_coll_portals4_scatter_intra_linear_top(const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-                                            void *rbuf, int rcount, struct ompi_datatype_t *rdtype,
+ompi_coll_portals4_scatter_intra_linear_top(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                                            void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
                                             int root,
                                             struct ompi_communicator_t *comm,
                                             ompi_coll_portals4_request_t *request,
@@ -646,8 +646,8 @@ ompi_coll_portals4_scatter_intra_linear_bottom(struct ompi_communicator_t *comm,
 }
 
 int
-ompi_coll_portals4_scatter_intra(const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-                                 void *rbuf, int rcount, struct ompi_datatype_t *rdtype,
+ompi_coll_portals4_scatter_intra(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                                 void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
                                  int root,
                                  struct ompi_communicator_t *comm,
                                  mca_coll_base_module_t *module)
@@ -705,8 +705,8 @@ ompi_coll_portals4_scatter_intra(const void *sbuf, int scount, struct ompi_datat
 
 
 int
-ompi_coll_portals4_iscatter_intra(const void *sbuf, int scount, struct ompi_datatype_t *sdtype,
-                                 void *rbuf, int rcount, struct ompi_datatype_t *rdtype,
+ompi_coll_portals4_iscatter_intra(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                                 void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
                                  int root,
                                  struct ompi_communicator_t *comm,
                                  ompi_request_t **ompi_request,
diff --git a/ompi/mca/coll/self/coll_self.h b/ompi/mca/coll/self/coll_self.h
index 02cf94d1c37..1c38e94fe85 100644
--- a/ompi/mca/coll/self/coll_self.h
+++ b/ompi/mca/coll/self/coll_self.h
@@ -12,6 +12,7 @@
  * Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -35,7 +36,7 @@ BEGIN_C_DECLS
  * Globally exported variable
  */
 
-OMPI_MODULE_DECLSPEC extern const mca_coll_base_component_2_4_0_t mca_coll_self_component;
+OMPI_DECLSPEC extern const mca_coll_base_component_3_0_0_t mca_coll_self_component;
 extern int ompi_coll_self_priority;
 
 /*
@@ -50,92 +51,89 @@ int mca_coll_self_init_query(bool enable_progress_threads,
 mca_coll_base_module_t *
 mca_coll_self_comm_query(struct ompi_communicator_t *comm, int *priority);
 
-int mca_coll_self_module_enable(mca_coll_base_module_t *module,
-                                struct ompi_communicator_t *comm);
-
-int mca_coll_self_allgather_intra(const void *sbuf, int scount,
+int mca_coll_self_allgather_intra(const void *sbuf, size_t scount,
                                   struct ompi_datatype_t *sdtype,
-                                  void *rbuf, int rcount,
+                                  void *rbuf, size_t rcount,
                                   struct ompi_datatype_t *rdtype,
                                   struct ompi_communicator_t *comm,
                                   mca_coll_base_module_t *module);
-int mca_coll_self_allgatherv_intra(const void *sbuf, int scount,
+int mca_coll_self_allgatherv_intra(const void *sbuf, size_t scount,
                                    struct ompi_datatype_t *sdtype,
-                                   void * rbuf, const int *rcounts, const int *disps,
+                                   void * rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
                                    struct ompi_datatype_t *rdtype,
                                    struct ompi_communicator_t *comm,
                                    mca_coll_base_module_t *module);
-int mca_coll_self_allreduce_intra(const void *sbuf, void *rbuf, int count,
+int mca_coll_self_allreduce_intra(const void *sbuf, void *rbuf, size_t count,
                                   struct ompi_datatype_t *dtype,
                                   struct ompi_op_t *op,
                                   struct ompi_communicator_t *comm,
                                   mca_coll_base_module_t *module);
-int mca_coll_self_alltoall_intra(const void *sbuf, int scount,
+int mca_coll_self_alltoall_intra(const void *sbuf, size_t scount,
                                  struct ompi_datatype_t *sdtype,
-                                 void* rbuf, int rcount,
+                                 void* rbuf, size_t rcount,
                                  struct ompi_datatype_t *rdtype,
                                  struct ompi_communicator_t *comm,
                                  mca_coll_base_module_t *module);
-int mca_coll_self_alltoallv_intra(const void *sbuf, const int *scounts, const int *sdisps,
+int mca_coll_self_alltoallv_intra(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
                                   struct ompi_datatype_t *sdtype,
-                                  void *rbuf, const int *rcounts, const int *rdisps,
+                                  void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
                                   struct ompi_datatype_t *rdtype,
                                   struct ompi_communicator_t *comm,
                                   mca_coll_base_module_t *module);
-int mca_coll_self_alltoallw_intra(const void *sbuf, const int *scounts, const int *sdisps,
+int mca_coll_self_alltoallw_intra(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
                                   struct ompi_datatype_t * const *sdtypes,
-                                  void *rbuf, const int *rcounts, const int *rdisps,
+                                  void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
                                   struct ompi_datatype_t * const *rdtypes,
                                   struct ompi_communicator_t *comm,
                                   mca_coll_base_module_t *module);
 int mca_coll_self_barrier_intra(struct ompi_communicator_t *comm,
                                 mca_coll_base_module_t *module);
-int mca_coll_self_bcast_intra(void *buff, int count,
+int mca_coll_self_bcast_intra(void *buff, size_t count,
                               struct ompi_datatype_t *datatype,
                               int root,
                               struct ompi_communicator_t *comm,
                               mca_coll_base_module_t *module);
-int mca_coll_self_exscan_intra(const void *sbuf, void *rbuf, int count,
+int mca_coll_self_exscan_intra(const void *sbuf, void *rbuf, size_t count,
                                struct ompi_datatype_t *dtype,
                                struct ompi_op_t *op,
                                struct ompi_communicator_t *comm,
                                mca_coll_base_module_t *module);
-int mca_coll_self_gather_intra(const void *sbuf, int scount,
+int mca_coll_self_gather_intra(const void *sbuf, size_t scount,
                                struct ompi_datatype_t *sdtype, void *rbuf,
-                               int rcount, struct ompi_datatype_t *rdtype,
+                               size_t rcount, struct ompi_datatype_t *rdtype,
                                int root, struct ompi_communicator_t *comm,
                                mca_coll_base_module_t *module);
-int mca_coll_self_gatherv_intra(const void *sbuf, int scount,
+int mca_coll_self_gatherv_intra(const void *sbuf, size_t scount,
                                 struct ompi_datatype_t *sdtype, void *rbuf,
-                                const int *rcounts, const int *disps,
+                                ompi_count_array_t rcounts, ompi_disp_array_t disps,
                                 struct ompi_datatype_t *rdtype, int root,
                                 struct ompi_communicator_t *comm,
                                 mca_coll_base_module_t *module);
-int mca_coll_self_reduce_intra(const void *sbuf, void* rbuf, int count,
+int mca_coll_self_reduce_intra(const void *sbuf, void* rbuf, size_t count,
                                struct ompi_datatype_t *dtype,
                                struct ompi_op_t *op,
                                int root,
                                struct ompi_communicator_t *comm,
                                mca_coll_base_module_t *module);
 int mca_coll_self_reduce_scatter_intra(const void *sbuf, void *rbuf,
-                                       const int *rcounts,
+                                       ompi_count_array_t rcounts,
                                        struct ompi_datatype_t *dtype,
                                        struct ompi_op_t *op,
                                        struct ompi_communicator_t *comm,
                                        mca_coll_base_module_t *module);
-int mca_coll_self_scan_intra(const void *sbuf, void *rbuf, int count,
+int mca_coll_self_scan_intra(const void *sbuf, void *rbuf, size_t count,
                              struct ompi_datatype_t *dtype,
                              struct ompi_op_t *op,
                              struct ompi_communicator_t *comm,
                              mca_coll_base_module_t *module);
-int mca_coll_self_scatter_intra(const void *sbuf, int scount,
+int mca_coll_self_scatter_intra(const void *sbuf, size_t scount,
                                 struct ompi_datatype_t *sdtype, void *rbuf,
-                                int rcount, struct ompi_datatype_t *rdtype,
+                                size_t rcount, struct ompi_datatype_t *rdtype,
                                 int root, struct ompi_communicator_t *comm,
                                 mca_coll_base_module_t *module);
-int mca_coll_self_scatterv_intra(const void *sbuf, const int *scounts, const int *disps,
+int mca_coll_self_scatterv_intra(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t disps,
                                  struct ompi_datatype_t *sdtype,
-                                 void* rbuf, int rcount,
+                                 void* rbuf, size_t rcount,
                                  struct ompi_datatype_t *rdtype, int root,
                                  struct ompi_communicator_t *comm,
                                  mca_coll_base_module_t *module);
diff --git a/ompi/mca/coll/self/coll_self_allgather.c b/ompi/mca/coll/self/coll_self_allgather.c
index 25028bc6817..ed30a2a8ed0 100644
--- a/ompi/mca/coll/self/coll_self_allgather.c
+++ b/ompi/mca/coll/self/coll_self_allgather.c
@@ -32,9 +32,9 @@
  *	Accepts:	- same as MPI_Allgather()
  *	Returns:	- MPI_SUCCESS, or error code
  */
-int mca_coll_self_allgather_intra(const void *sbuf, int scount,
+int mca_coll_self_allgather_intra(const void *sbuf, size_t scount,
                                   struct ompi_datatype_t *sdtype, void *rbuf,
-                                  int rcount, struct ompi_datatype_t *rdtype,
+                                  size_t rcount, struct ompi_datatype_t *rdtype,
                                   struct ompi_communicator_t *comm,
                                   mca_coll_base_module_t *module)
 {
diff --git a/ompi/mca/coll/self/coll_self_allgatherv.c b/ompi/mca/coll/self/coll_self_allgatherv.c
index 89c5403bdec..cca0bd7eea7 100644
--- a/ompi/mca/coll/self/coll_self_allgatherv.c
+++ b/ompi/mca/coll/self/coll_self_allgatherv.c
@@ -32,9 +32,9 @@
  *	Accepts:	- same as MPI_Allgatherv()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_self_allgatherv_intra(const void *sbuf, int scount,
+int mca_coll_self_allgatherv_intra(const void *sbuf, size_t scount,
                                    struct ompi_datatype_t *sdtype,
-                                   void * rbuf, const int *rcounts, const int *disps,
+                                   void * rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
                                    struct ompi_datatype_t *rdtype,
                                    struct ompi_communicator_t *comm,
                                    mca_coll_base_module_t *module)
@@ -49,6 +49,6 @@ int mca_coll_self_allgatherv_intra(const void *sbuf, int scount,
             return OMPI_ERROR;
         }
         return ompi_datatype_sndrcv(sbuf, scount, sdtype,
-                               ((char *) rbuf) + disps[0] * extent, rcounts[0], rdtype);
+                               ((char *) rbuf) + ompi_disp_array_get(disps, 0) * extent, ompi_count_array_get(rcounts, 0), rdtype);
     }
 }
diff --git a/ompi/mca/coll/self/coll_self_allreduce.c b/ompi/mca/coll/self/coll_self_allreduce.c
index cfcde182953..f76d128f95b 100644
--- a/ompi/mca/coll/self/coll_self_allreduce.c
+++ b/ompi/mca/coll/self/coll_self_allreduce.c
@@ -32,7 +32,7 @@
  *	Accepts:	- same as MPI_Allreduce()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_self_allreduce_intra(const void *sbuf, void *rbuf, int count,
+int mca_coll_self_allreduce_intra(const void *sbuf, void *rbuf, size_t count,
                                   struct ompi_datatype_t *dtype,
                                   struct ompi_op_t *op,
                                   struct ompi_communicator_t *comm,
diff --git a/ompi/mca/coll/self/coll_self_alltoall.c b/ompi/mca/coll/self/coll_self_alltoall.c
index 56a24a5c72d..609c9c60dfa 100644
--- a/ompi/mca/coll/self/coll_self_alltoall.c
+++ b/ompi/mca/coll/self/coll_self_alltoall.c
@@ -33,9 +33,9 @@
  *	Accepts:	- same as MPI_Alltoall()
  *	Returns:	- MPI_SUCCESS or an MPI error code
  */
-int mca_coll_self_alltoall_intra(const void *sbuf, int scount,
+int mca_coll_self_alltoall_intra(const void *sbuf, size_t scount,
                                  struct ompi_datatype_t *sdtype,
-                                 void *rbuf, int rcount,
+                                 void *rbuf, size_t rcount,
                                  struct ompi_datatype_t *rdtype,
                                  struct ompi_communicator_t *comm,
                                  mca_coll_base_module_t *module)
diff --git a/ompi/mca/coll/self/coll_self_alltoallv.c b/ompi/mca/coll/self/coll_self_alltoallv.c
index 7d34daa1645..e13460434da 100644
--- a/ompi/mca/coll/self/coll_self_alltoallv.c
+++ b/ompi/mca/coll/self/coll_self_alltoallv.c
@@ -34,9 +34,9 @@
  *	Returns:	- MPI_SUCCESS or an MPI error code
  */
 int
-mca_coll_self_alltoallv_intra(const void *sbuf, const int *scounts, const int *sdisps,
+mca_coll_self_alltoallv_intra(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
                               struct ompi_datatype_t *sdtype,
-                              void *rbuf, const int *rcounts, const int *rdisps,
+                              void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
                               struct ompi_datatype_t *rdtype,
                               struct ompi_communicator_t *comm,
                               mca_coll_base_module_t *module)
@@ -56,8 +56,8 @@ mca_coll_self_alltoallv_intra(const void *sbuf, const int *scounts, const int *s
     if (OMPI_SUCCESS != err) {
         return OMPI_ERROR;
     }
-    return ompi_datatype_sndrcv(((char *) sbuf) + sdisps[0] * sextent,
-                           scounts[0], sdtype,
-                           ((char *) rbuf) + rdisps[0] * rextent,
-                           rcounts[0], rdtype);
+    return ompi_datatype_sndrcv(((char *) sbuf) + ompi_disp_array_get(sdisps, 0) * sextent,
+                           ompi_count_array_get(scounts, 0), sdtype,
+                           ((char *) rbuf) + ompi_disp_array_get(rdisps, 0) * rextent,
+                           ompi_count_array_get(rcounts, 0), rdtype);
 }
diff --git a/ompi/mca/coll/self/coll_self_alltoallw.c b/ompi/mca/coll/self/coll_self_alltoallw.c
index 1cf6dc1217e..edc625494da 100644
--- a/ompi/mca/coll/self/coll_self_alltoallw.c
+++ b/ompi/mca/coll/self/coll_self_alltoallw.c
@@ -33,9 +33,9 @@
  *	Accepts:	- same as MPI_Alltoallw()
  *	Returns:	- MPI_SUCCESS or an MPI error code
  */
-int mca_coll_self_alltoallw_intra(const void *sbuf, const int *scounts, const int *sdisps,
+int mca_coll_self_alltoallw_intra(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
                                   struct ompi_datatype_t * const *sdtypes,
-                                  void *rbuf, const int *rcounts, const int *rdisps,
+                                  void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
                                   struct ompi_datatype_t * const *rdtypes,
                                   struct ompi_communicator_t *comm,
                                   mca_coll_base_module_t *module)
@@ -56,8 +56,8 @@ int mca_coll_self_alltoallw_intra(const void *sbuf, const int *scounts, const in
         return OMPI_ERROR;
     }
 
-    return ompi_datatype_sndrcv(((char *) sbuf) + sdisps[0] * sextent,
-                           scounts[0], sdtypes[0],
-                           ((char *) rbuf) + rdisps[0] * rextent,
-                           rcounts[0], rdtypes[0]);
+    return ompi_datatype_sndrcv(((char *) sbuf) + ompi_disp_array_get(sdisps, 0) * sextent,
+                           ompi_count_array_get(scounts, 0), sdtypes[0],
+                           ((char *) rbuf) + ompi_disp_array_get(rdisps, 0) * rextent,
+                           ompi_count_array_get(rcounts, 0), rdtypes[0]);
 }
diff --git a/ompi/mca/coll/self/coll_self_bcast.c b/ompi/mca/coll/self/coll_self_bcast.c
index d6813ee3fc5..4f4d0b82413 100644
--- a/ompi/mca/coll/self/coll_self_bcast.c
+++ b/ompi/mca/coll/self/coll_self_bcast.c
@@ -30,7 +30,7 @@
  *	Accepts:	- same arguments as MPI_Bcast()
  *	Returns:	- MPI_SUCCESS
  */
-int mca_coll_self_bcast_intra(void *buff, int count,
+int mca_coll_self_bcast_intra(void *buff, size_t count,
                               struct ompi_datatype_t *datatype, int root,
                               struct ompi_communicator_t *comm,
                               mca_coll_base_module_t *module)
diff --git a/ompi/mca/coll/self/coll_self_component.c b/ompi/mca/coll/self/coll_self_component.c
index e18dca1df61..9afb69f6a4b 100644
--- a/ompi/mca/coll/self/coll_self_component.c
+++ b/ompi/mca/coll/self/coll_self_component.c
@@ -55,13 +55,13 @@ static int self_register(void);
  * and pointers to our public functions in it
  */
 
-const mca_coll_base_component_2_4_0_t mca_coll_self_component = {
+const mca_coll_base_component_3_0_0_t mca_coll_self_component = {
 
     /* First, the mca_component_t struct containing meta information
        about the component itself */
 
     .collm_version = {
-        MCA_COLL_BASE_VERSION_2_4_0,
+        MCA_COLL_BASE_VERSION_3_0_0,
 
         /* Component name and version */
         .mca_component_name = "self",
diff --git a/ompi/mca/coll/self/coll_self_exscan.c b/ompi/mca/coll/self/coll_self_exscan.c
index 5b0b2fac134..b9cd5e8857c 100644
--- a/ompi/mca/coll/self/coll_self_exscan.c
+++ b/ompi/mca/coll/self/coll_self_exscan.c
@@ -31,7 +31,7 @@
  *	Accepts:	- same arguments as MPI_Exccan()
  *	Returns:	- MPI_SUCCESS
  */
-int mca_coll_self_exscan_intra(const void *sbuf, void *rbuf, int count,
+int mca_coll_self_exscan_intra(const void *sbuf, void *rbuf, size_t count,
                                struct ompi_datatype_t *dtype,
                                struct ompi_op_t *op,
                                struct ompi_communicator_t *comm,
diff --git a/ompi/mca/coll/self/coll_self_gather.c b/ompi/mca/coll/self/coll_self_gather.c
index 57cf69dd6f6..dc46cae2343 100644
--- a/ompi/mca/coll/self/coll_self_gather.c
+++ b/ompi/mca/coll/self/coll_self_gather.c
@@ -32,9 +32,9 @@
  *	Accepts:	- same arguments as MPI_Gather()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_self_gather_intra(const void *sbuf, int scount,
+int mca_coll_self_gather_intra(const void *sbuf, size_t scount,
                                struct ompi_datatype_t *sdtype,
-                               void *rbuf, int rcount,
+                               void *rbuf, size_t rcount,
                                struct ompi_datatype_t *rdtype,
                                int root, struct ompi_communicator_t *comm,
                                mca_coll_base_module_t *module)
diff --git a/ompi/mca/coll/self/coll_self_gatherv.c b/ompi/mca/coll/self/coll_self_gatherv.c
index 136b4652ae3..5691ada77f6 100644
--- a/ompi/mca/coll/self/coll_self_gatherv.c
+++ b/ompi/mca/coll/self/coll_self_gatherv.c
@@ -32,9 +32,9 @@
  *	Accepts:	- same arguments as MPI_Gatherv()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_self_gatherv_intra(const void *sbuf, int scount,
+int mca_coll_self_gatherv_intra(const void *sbuf, size_t scount,
                                 struct ompi_datatype_t *sdtype,
-                                void *rbuf, const int *rcounts, const int *disps,
+                                void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
                                 struct ompi_datatype_t *rdtype, int root,
                                 struct ompi_communicator_t *comm,
                                 mca_coll_base_module_t *module)
@@ -49,7 +49,7 @@ int mca_coll_self_gatherv_intra(const void *sbuf, int scount,
             return OMPI_ERROR;
         }
         return ompi_datatype_sndrcv(sbuf, scount, sdtype,
-                               ((char *) rbuf) + disps[0]*extent,
-                               rcounts[0], rdtype);
+                               ((char *) rbuf) + ompi_disp_array_get(disps, 0)*extent,
+                               ompi_count_array_get(rcounts, 0), rdtype);
     }
 }
diff --git a/ompi/mca/coll/self/coll_self_module.c b/ompi/mca/coll/self/coll_self_module.c
index d782b998165..ec706b52613 100644
--- a/ompi/mca/coll/self/coll_self_module.c
+++ b/ompi/mca/coll/self/coll_self_module.c
@@ -10,6 +10,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -29,6 +30,12 @@
 #include "ompi/mca/coll/base/coll_base_functions.h"
 #include "coll_self.h"
 
+static int
+mca_coll_self_module_enable(mca_coll_base_module_t *module,
+                            struct ompi_communicator_t *comm);
+static int
+mca_coll_self_module_disable(mca_coll_base_module_t *module,
+                             struct ompi_communicator_t *comm);
 
 /*
  * Initial query function that is invoked during MPI_INIT, allowing
@@ -63,22 +70,7 @@ mca_coll_self_comm_query(struct ompi_communicator_t *comm,
         if (NULL == module) return NULL;
 
         module->super.coll_module_enable = mca_coll_self_module_enable;
-        module->super.coll_allgather  = mca_coll_self_allgather_intra;
-        module->super.coll_allgatherv = mca_coll_self_allgatherv_intra;
-        module->super.coll_allreduce  = mca_coll_self_allreduce_intra;
-        module->super.coll_alltoall   = mca_coll_self_alltoall_intra;
-        module->super.coll_alltoallv  = mca_coll_self_alltoallv_intra;
-        module->super.coll_alltoallw  = mca_coll_self_alltoallw_intra;
-        module->super.coll_barrier    = mca_coll_self_barrier_intra;
-        module->super.coll_bcast      = mca_coll_self_bcast_intra;
-        module->super.coll_exscan     = mca_coll_self_exscan_intra;
-        module->super.coll_gather     = mca_coll_self_gather_intra;
-        module->super.coll_gatherv    = mca_coll_self_gatherv_intra;
-        module->super.coll_reduce     = mca_coll_self_reduce_intra;
-        module->super.coll_reduce_scatter = mca_coll_self_reduce_scatter_intra;
-        module->super.coll_scan       = mca_coll_self_scan_intra;
-        module->super.coll_scatter    = mca_coll_self_scatter_intra;
-        module->super.coll_scatterv   = mca_coll_self_scatterv_intra;
+        module->super.coll_module_disable = mca_coll_self_module_disable;
 
         module->super.coll_reduce_local = mca_coll_base_reduce_local;
 
@@ -88,13 +80,73 @@ mca_coll_self_comm_query(struct ompi_communicator_t *comm,
     return NULL;
 }
 
+#define SELF_INSTALL_COLL_API(__comm, __module, __api)                                                 \
+    do                                                                                                 \
+    {                                                                                                  \
+        MCA_COLL_INSTALL_API(__comm, __api, mca_coll_self_##__api##_intra, &__module->super, "self");  \
+    } while (0)
+
+#define SELF_UNINSTALL_COLL_API(__comm, __module, __api)                \
+    do                                                                  \
+    {                                                                   \
+        if (__comm->c_coll->coll_##__api##_module == &__module->super)  \
+        {                                                               \
+            MCA_COLL_INSTALL_API(__comm, __api, NULL, NULL, "self");    \
+        }                                                               \
+    } while (0)
 
 /*
  * Init module on the communicator
  */
-int
+static int
 mca_coll_self_module_enable(mca_coll_base_module_t *module,
                             struct ompi_communicator_t *comm)
 {
+    mca_coll_self_module_t *sm_module = (mca_coll_self_module_t*)module;
+
+    SELF_INSTALL_COLL_API(comm, sm_module, allgather);
+    SELF_INSTALL_COLL_API(comm, sm_module, allgatherv);
+    SELF_INSTALL_COLL_API(comm, sm_module, allreduce);
+    SELF_INSTALL_COLL_API(comm, sm_module, alltoall);
+    SELF_INSTALL_COLL_API(comm, sm_module, alltoallv);
+    SELF_INSTALL_COLL_API(comm, sm_module, alltoallw);
+    SELF_INSTALL_COLL_API(comm, sm_module, barrier);
+    SELF_INSTALL_COLL_API(comm, sm_module, bcast);
+    SELF_INSTALL_COLL_API(comm, sm_module, exscan);
+    SELF_INSTALL_COLL_API(comm, sm_module, gather);
+    SELF_INSTALL_COLL_API(comm, sm_module, gatherv);
+    SELF_INSTALL_COLL_API(comm, sm_module, reduce);
+    SELF_INSTALL_COLL_API(comm, sm_module, reduce_scatter);
+    SELF_INSTALL_COLL_API(comm, sm_module, scan);
+    SELF_INSTALL_COLL_API(comm, sm_module, scatter);
+    SELF_INSTALL_COLL_API(comm, sm_module, scatterv);
+
+    MCA_COLL_INSTALL_API(comm, reduce_local, mca_coll_base_reduce_local, module, "self");
+    return OMPI_SUCCESS;
+}
+
+static int
+mca_coll_self_module_disable(mca_coll_base_module_t *module,
+                             struct ompi_communicator_t *comm)
+{
+    mca_coll_self_module_t *sm_module = (mca_coll_self_module_t *)module;
+
+    SELF_UNINSTALL_COLL_API(comm, sm_module, allgather);
+    SELF_UNINSTALL_COLL_API(comm, sm_module, allgatherv);
+    SELF_UNINSTALL_COLL_API(comm, sm_module, allreduce);
+    SELF_UNINSTALL_COLL_API(comm, sm_module, alltoall);
+    SELF_UNINSTALL_COLL_API(comm, sm_module, alltoallv);
+    SELF_UNINSTALL_COLL_API(comm, sm_module, alltoallw);
+    SELF_UNINSTALL_COLL_API(comm, sm_module, barrier);
+    SELF_UNINSTALL_COLL_API(comm, sm_module, bcast);
+    SELF_UNINSTALL_COLL_API(comm, sm_module, exscan);
+    SELF_UNINSTALL_COLL_API(comm, sm_module, gather);
+    SELF_UNINSTALL_COLL_API(comm, sm_module, gatherv);
+    SELF_UNINSTALL_COLL_API(comm, sm_module, reduce);
+    SELF_UNINSTALL_COLL_API(comm, sm_module, reduce_scatter);
+    SELF_UNINSTALL_COLL_API(comm, sm_module, scan);
+    SELF_UNINSTALL_COLL_API(comm, sm_module, scatter);
+    SELF_UNINSTALL_COLL_API(comm, sm_module, scatterv);
+
     return OMPI_SUCCESS;
 }
diff --git a/ompi/mca/coll/self/coll_self_reduce.c b/ompi/mca/coll/self/coll_self_reduce.c
index 5fd3f4b6f60..6c4ba66ff0d 100644
--- a/ompi/mca/coll/self/coll_self_reduce.c
+++ b/ompi/mca/coll/self/coll_self_reduce.c
@@ -32,7 +32,7 @@
  *	Accepts:	- same as MPI_Reduce()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_self_reduce_intra(const void *sbuf, void *rbuf, int count,
+int mca_coll_self_reduce_intra(const void *sbuf, void *rbuf, size_t count,
                                struct ompi_datatype_t *dtype,
                                struct ompi_op_t *op,
                                int root, struct ompi_communicator_t *comm,
diff --git a/ompi/mca/coll/self/coll_self_reduce_scatter.c b/ompi/mca/coll/self/coll_self_reduce_scatter.c
index 92006cbf463..61411b0c874 100644
--- a/ompi/mca/coll/self/coll_self_reduce_scatter.c
+++ b/ompi/mca/coll/self/coll_self_reduce_scatter.c
@@ -32,7 +32,7 @@
  *	Accepts:	- same as MPI_Reduce_scatter()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_self_reduce_scatter_intra(const void *sbuf, void *rbuf, const int *rcounts,
+int mca_coll_self_reduce_scatter_intra(const void *sbuf, void *rbuf, ompi_count_array_t rcounts,
                                        struct ompi_datatype_t *dtype,
                                        struct ompi_op_t *op,
                                        struct ompi_communicator_t *comm,
@@ -41,6 +41,6 @@ int mca_coll_self_reduce_scatter_intra(const void *sbuf, void *rbuf, const int *
     if (MPI_IN_PLACE == sbuf) {
         return MPI_SUCCESS;
     } else {
-        return ompi_datatype_copy_content_same_ddt(dtype, rcounts[0], (char*)rbuf, (char*)sbuf);
+        return ompi_datatype_copy_content_same_ddt(dtype, ompi_count_array_get(rcounts, 0), (char*)rbuf, (char*)sbuf);
     }
 }
diff --git a/ompi/mca/coll/self/coll_self_scan.c b/ompi/mca/coll/self/coll_self_scan.c
index 14bd1cbf3f2..f373de2ad10 100644
--- a/ompi/mca/coll/self/coll_self_scan.c
+++ b/ompi/mca/coll/self/coll_self_scan.c
@@ -32,7 +32,7 @@
  *	Accepts:	- same arguments as MPI_Scan()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_self_scan_intra(const void *sbuf, void *rbuf, int count,
+int mca_coll_self_scan_intra(const void *sbuf, void *rbuf, size_t count,
                              struct ompi_datatype_t *dtype,
                              struct ompi_op_t *op,
                              struct ompi_communicator_t *comm,
diff --git a/ompi/mca/coll/self/coll_self_scatter.c b/ompi/mca/coll/self/coll_self_scatter.c
index 179a7062f76..78d13b91dd8 100644
--- a/ompi/mca/coll/self/coll_self_scatter.c
+++ b/ompi/mca/coll/self/coll_self_scatter.c
@@ -32,9 +32,9 @@
  *	Accepts:	- same arguments as MPI_Scatter()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_self_scatter_intra(const void *sbuf, int scount,
+int mca_coll_self_scatter_intra(const void *sbuf, size_t scount,
                                 struct ompi_datatype_t *sdtype,
-                                void *rbuf, int rcount,
+                                void *rbuf, size_t rcount,
                                 struct ompi_datatype_t *rdtype,
                                 int root,
                                 struct ompi_communicator_t *comm,
diff --git a/ompi/mca/coll/self/coll_self_scatterv.c b/ompi/mca/coll/self/coll_self_scatterv.c
index e7b0b5badea..2a1250ac26c 100644
--- a/ompi/mca/coll/self/coll_self_scatterv.c
+++ b/ompi/mca/coll/self/coll_self_scatterv.c
@@ -32,9 +32,9 @@
  *	Accepts:	- same arguments as MPI_Scatter()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_self_scatterv_intra(const void *sbuf, const int *scounts,
-                                 const int *disps, struct ompi_datatype_t *sdtype,
-                                 void *rbuf, int rcount,
+int mca_coll_self_scatterv_intra(const void *sbuf, ompi_count_array_t scounts,
+                                 ompi_disp_array_t disps, struct ompi_datatype_t *sdtype,
+                                 void *rbuf, size_t rcount,
                                  struct ompi_datatype_t *rdtype, int root,
                                  struct ompi_communicator_t *comm,
                                  mca_coll_base_module_t *module)
@@ -48,7 +48,7 @@ int mca_coll_self_scatterv_intra(const void *sbuf, const int *scounts,
         if (OMPI_SUCCESS != err) {
             return OMPI_ERROR;
         }
-        return ompi_datatype_sndrcv(((char *) sbuf) + disps[0]*extent, scounts[0],
+        return ompi_datatype_sndrcv(((char *) sbuf) + ompi_disp_array_get(disps, 0)*extent, ompi_count_array_get(scounts, 0),
                                sdtype, rbuf, rcount, rdtype);
     }
 }
diff --git a/ompi/mca/coll/sm/Makefile.am b/ompi/mca/coll/sm/Makefile.am
deleted file mode 100644
index fafcbd7e473..00000000000
--- a/ompi/mca/coll/sm/Makefile.am
+++ /dev/null
@@ -1,70 +0,0 @@
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2009-2014 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2017      IBM Corporation.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-dist_ompidata_DATA = help-mpi-coll-sm.txt
-
-not_used_yet = \
-        coll_sm_allgather.c \
-        coll_sm_allgatherv.c \
-        coll_sm_alltoall.c \
-        coll_sm_alltoallv.c \
-        coll_sm_alltoallw.c \
-        coll_sm_gather.c \
-        coll_sm_gatherv.c \
-        coll_sm_reduce_scatter.c \
-        coll_sm_scan.c \
-        coll_sm_exscan.c \
-        coll_sm_scatter.c \
-        coll_sm_scatterv.c
-
-sources = \
-        coll_sm.h \
-        coll_sm_allreduce.c \
-        coll_sm_barrier.c \
-        coll_sm_bcast.c \
-        coll_sm_component.c \
-        coll_sm_module.c \
-        coll_sm_reduce.c
-
-# Make the output library in this directory, and name it either
-# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
-# (for static builds).
-
-component_noinst =
-component_install =
-if MCA_BUILD_ompi_coll_sm_DSO
-component_install += mca_coll_sm.la
-else
-component_noinst += libmca_coll_sm.la
-endif
-
-# See ompi/mca/btl/sm/Makefile.am for an explanation of
-# libmca_common_sm.la.
-
-mcacomponentdir = $(ompilibdir)
-mcacomponent_LTLIBRARIES = $(component_install)
-mca_coll_sm_la_SOURCES = $(sources)
-mca_coll_sm_la_LDFLAGS = -module -avoid-version
-mca_coll_sm_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
-        $(OMPI_TOP_BUILDDIR)/opal/mca/common/sm/lib@OPAL_LIB_PREFIX@mca_common_sm.la
-
-noinst_LTLIBRARIES = $(component_noinst)
-libmca_coll_sm_la_SOURCES =$(sources)
-libmca_coll_sm_la_LDFLAGS = -module -avoid-version
diff --git a/ompi/mca/coll/sm/coll_sm.h b/ompi/mca/coll/sm/coll_sm.h
deleted file mode 100644
index b5e2a3f1c22..00000000000
--- a/ompi/mca/coll/sm/coll_sm.h
+++ /dev/null
@@ -1,453 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2014 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008-2009 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-/** @file */
-
-#ifndef MCA_COLL_SM_EXPORT_H
-#define MCA_COLL_SM_EXPORT_H
-
-#include "ompi_config.h"
-
-#include "mpi.h"
-#include "ompi/mca/mca.h"
-#include "opal/datatype/opal_convertor.h"
-#include "opal/mca/common/sm/common_sm.h"
-#include "ompi/mca/coll/coll.h"
-
-BEGIN_C_DECLS
-
-/* Attempt to give some sort of progress / fairness if we're blocked
-   in an sm collective for a long time: call opal_progress once in a
-   great while.  Use a "goto" label for expdiency to exit loops. */
-#define SPIN_CONDITION_MAX 100000
-#define SPIN_CONDITION(cond, exit_label) \
-  do { int i; \
-       if (cond) goto exit_label; \
-       for (i = 0; i < SPIN_CONDITION_MAX; ++i) { \
-           if (cond) { goto exit_label; } \
-       } \
-       opal_progress(); \
-  } while (1); \
-  exit_label:
-
-    /**
-     * Structure to hold the sm coll component.  First it holds the
-     * base coll component, and then holds a bunch of
-     * sm-coll-component-specific stuff (e.g., current MCA param
-     * values).
-     */
-    typedef struct mca_coll_sm_component_t {
-        /** Base coll component */
-        mca_coll_base_component_2_4_0_t super;
-
-        /** MCA parameter: Priority of this component */
-        int sm_priority;
-
-        /** MCA parameter: Length of a cache line or page (in bytes) */
-        int sm_control_size;
-
-        /** MCA parameter: Number of "in use" flags in each
-            communicator's area in the data mpool */
-        int sm_comm_num_in_use_flags;
-
-        /** MCA parameter: Number of segments for each communicator in
-            the data mpool */
-        int sm_comm_num_segments;
-
-        /** MCA parameter: Fragment size for data */
-        int sm_fragment_size;
-
-        /** MCA parameter: Degree of tree for tree-based collectives */
-        int sm_tree_degree;
-
-        /** MCA parameter: Number of processes to use in the
-            calculation of the "info" MCA parameter */
-        int sm_info_comm_size;
-
-        /******* end of MCA params ********/
-
-        /** How many fragment segments are protected by a single
-            in-use flags.  This is solely so that we can only perform
-            the division once and then just use the value without
-            having to re-calculate. */
-        int sm_segs_per_inuse_flag;
-    } mca_coll_sm_component_t;
-
-    /**
-     * Structure for representing a node in the tree
-     */
-    typedef struct mca_coll_sm_tree_node_t {
-        /** Arbitrary ID number, starting from 0 */
-        int mcstn_id;
-        /** Pointer to parent, or NULL if root */
-        struct mca_coll_sm_tree_node_t *mcstn_parent;
-        /** Number of children, or 0 if a leaf */
-        int mcstn_num_children;
-        /** Pointer to an array of children, or NULL if 0 ==
-            mcstn_num_children */
-        struct mca_coll_sm_tree_node_t **mcstn_children;
-    } mca_coll_sm_tree_node_t;
-
-    /**
-     * Simple structure comprising the "in use" flags.  Contains two
-     * members: the number of processes that are currently using this
-     * set of segments and the operation number of the current
-     * operation.
-     */
-    typedef struct mca_coll_sm_in_use_flag_t {
-        /** Number of processes currently using this set of
-            segments */
-        opal_atomic_uint32_t mcsiuf_num_procs_using;
-        /** Must match data->mcb_count */
-        volatile uint32_t mcsiuf_operation_count;
-    } mca_coll_sm_in_use_flag_t;
-
-    /**
-     * Structure containing pointers to various arrays of data in the
-     * per-communicator shmem data segment (one of these indexes a
-     * single segment in the per-communicator shmem data segment).
-     * Nothing is hard-coded because all the array lengths and
-     * displacements of the pointers all depend on how many processes
-     * are in the communicator.
-     */
-    typedef struct mca_coll_sm_data_index_t {
-        /** Pointer to beginning of control data */
-        uint32_t volatile *mcbmi_control;
-        /** Pointer to beginning of message fragment data */
-        char *mcbmi_data;
-    } mca_coll_sm_data_index_t;
-
-    /**
-     * Structure for the sm coll module to hang off the communicator.
-     * Contains communicator-specific information, including pointers
-     * into the per-communicator shmem data data segment for this
-     * comm's sm collective operations area.
-     */
-    typedef struct mca_coll_sm_comm_t {
-        /* Meta data that we get back from the common mmap allocation
-           function */
-        mca_common_sm_module_t *sm_bootstrap_meta;
-
-        /** Pointer to my barrier control pages (odd index pages are
-            "in", even index pages are "out") */
-        uint32_t *mcb_barrier_control_me;
-
-        /** Pointer to my parent's barrier control pages (will be NULL
-            for communicator rank 0; odd index pages are "in", even
-            index pages are "out") */
-        opal_atomic_uint32_t *mcb_barrier_control_parent;
-
-        /** Pointers to my childrens' barrier control pages (they're
-            contiguous in memory, so we only point to the base -- the
-            number of children is in my entry in the mcb_tree); will
-            be NULL if this process has no children (odd index pages
-            are "in", even index pages are "out") */
-        uint32_t *mcb_barrier_control_children;
-
-        /** Number of barriers that we have executed (i.e., which set
-            of barrier buffers to use). */
-        int mcb_barrier_count;
-
-        /** "In use" flags indicating which segments are available */
-        mca_coll_sm_in_use_flag_t *mcb_in_use_flags;
-
-        /** Array of indexes into the per-communicator shmem data
-            segment for control and data fragment passing (containing
-            pointers to each segments control and data areas). */
-        mca_coll_sm_data_index_t *mcb_data_index;
-
-        /** Array of graph nodes representing the tree used for
-            communications */
-        mca_coll_sm_tree_node_t *mcb_tree;
-
-        /** Operation number (i.e., which segment number to use) */
-        uint32_t mcb_operation_count;
-    } mca_coll_sm_comm_t;
-
-    /** Coll sm module */
-    typedef struct mca_coll_sm_module_t {
-        /** Base module */
-	mca_coll_base_module_t super;
-
-        /* Whether this module has been lazily initialized or not yet */
-        bool enabled;
-
-        /* Data that hangs off the communicator */
-	mca_coll_sm_comm_t *sm_comm_data;
-
-        /* Underlying reduce function and module */
-	mca_coll_base_module_reduce_fn_t previous_reduce;
-	mca_coll_base_module_t *previous_reduce_module;
-    } mca_coll_sm_module_t;
-    OBJ_CLASS_DECLARATION(mca_coll_sm_module_t);
-
-    /**
-     * Global component instance
-     */
-    OMPI_MODULE_DECLSPEC extern mca_coll_sm_component_t mca_coll_sm_component;
-
-    /*
-     * coll module functions
-     */
-    int mca_coll_sm_init_query(bool enable_progress_threads,
-			       bool enable_mpi_threads);
-
-    mca_coll_base_module_t *
-    mca_coll_sm_comm_query(struct ompi_communicator_t *comm, int *priority);
-
-    /* Lazily enable a module (since it involves expensive/slow mmap
-       allocation, etc.) */
-    int ompi_coll_sm_lazy_enable(mca_coll_base_module_t *module,
-                                 struct ompi_communicator_t *comm);
-
-    int mca_coll_sm_allgather_intra(const void *sbuf, int scount,
-				    struct ompi_datatype_t *sdtype,
-				    void *rbuf, int rcount,
-				    struct ompi_datatype_t *rdtype,
-				    struct ompi_communicator_t *comm,
-				    mca_coll_base_module_t *module);
-
-    int mca_coll_sm_allgatherv_intra(const void *sbuf, int scount,
-				     struct ompi_datatype_t *sdtype,
-				     void * rbuf, const int *rcounts, const int *disps,
-				     struct ompi_datatype_t *rdtype,
-				     struct ompi_communicator_t *comm,
-				     mca_coll_base_module_t *module);
-    int mca_coll_sm_allreduce_intra(const void *sbuf, void *rbuf, int count,
-				    struct ompi_datatype_t *dtype,
-				    struct ompi_op_t *op,
-				    struct ompi_communicator_t *comm,
-				    mca_coll_base_module_t *module);
-    int mca_coll_sm_alltoall_intra(const void *sbuf, int scount,
-				   struct ompi_datatype_t *sdtype,
-				   void* rbuf, int rcount,
-				   struct ompi_datatype_t *rdtype,
-				   struct ompi_communicator_t *comm,
-				   mca_coll_base_module_t *module);
-    int mca_coll_sm_alltoallv_intra(const void *sbuf, const int *scounts, const int *sdisps,
-				    struct ompi_datatype_t *sdtype,
-				    void *rbuf, const int *rcounts, const int *rdisps,
-				    struct ompi_datatype_t *rdtype,
-				    struct ompi_communicator_t *comm,
-				    mca_coll_base_module_t *module);
-    int mca_coll_sm_alltoallw_intra(const void *sbuf, const int *scounts, const int *sdisps,
-				    struct ompi_datatype_t * const *sdtypes,
-				    void *rbuf, const int *rcounts, const int *rdisps,
-				    struct ompi_datatype_t * const *rdtypes,
-				    struct ompi_communicator_t *comm,
-				    mca_coll_base_module_t *module);
-    int mca_coll_sm_barrier_intra(struct ompi_communicator_t *comm,
-				  mca_coll_base_module_t *module);
-    int mca_coll_sm_bcast_intra(void *buff, int count,
-				struct ompi_datatype_t *datatype,
-				int root,
-				struct ompi_communicator_t *comm,
-				mca_coll_base_module_t *module);
-    int mca_coll_sm_bcast_log_intra(void *buff, int count,
-				    struct ompi_datatype_t *datatype,
-				    int root,
-				    struct ompi_communicator_t *comm,
-				    mca_coll_base_module_t *module);
-    int mca_coll_sm_exscan_intra(const void *sbuf, void *rbuf, int count,
-				 struct ompi_datatype_t *dtype,
-				 struct ompi_op_t *op,
-				 struct ompi_communicator_t *comm,
-				 mca_coll_base_module_t *module);
-    int mca_coll_sm_gather_intra(void *sbuf, int scount,
-				 struct ompi_datatype_t *sdtype, void *rbuf,
-				 int rcount, struct ompi_datatype_t *rdtype,
-				 int root, struct ompi_communicator_t *comm,
-				 mca_coll_base_module_t *module);
-    int mca_coll_sm_gatherv_intra(void *sbuf, int scount,
-				  struct ompi_datatype_t *sdtype, void *rbuf,
-				  int *rcounts, int *disps,
-				  struct ompi_datatype_t *rdtype, int root,
-				  struct ompi_communicator_t *comm,
-				  mca_coll_base_module_t *module);
-    int mca_coll_sm_reduce_intra(const void *sbuf, void* rbuf, int count,
-				 struct ompi_datatype_t *dtype,
-				 struct ompi_op_t *op,
-				 int root,
-				 struct ompi_communicator_t *comm,
-				 mca_coll_base_module_t *module);
-    int mca_coll_sm_reduce_log_intra(const void *sbuf, void* rbuf, int count,
-				     struct ompi_datatype_t *dtype,
-				     struct ompi_op_t *op,
-				     int root,
-				     struct ompi_communicator_t *comm,
-				     mca_coll_base_module_t *module);
-    int mca_coll_sm_reduce_scatter_intra(const void *sbuf, void *rbuf,
-					 int *rcounts,
-					 struct ompi_datatype_t *dtype,
-					 struct ompi_op_t *op,
-					 struct ompi_communicator_t *comm,
-					 mca_coll_base_module_t *module);
-    int mca_coll_sm_scan_intra(const void *sbuf, void *rbuf, int count,
-			       struct ompi_datatype_t *dtype,
-			       struct ompi_op_t *op,
-			       struct ompi_communicator_t *comm,
-			       mca_coll_base_module_t *module);
-    int mca_coll_sm_scatter_intra(const void *sbuf, int scount,
-				  struct ompi_datatype_t *sdtype, void *rbuf,
-				  int rcount, struct ompi_datatype_t *rdtype,
-				  int root, struct ompi_communicator_t *comm,
-				  mca_coll_base_module_t *module);
-    int mca_coll_sm_scatterv_intra(const void *sbuf, const int *scounts, const int *disps,
-				   struct ompi_datatype_t *sdtype,
-				   void* rbuf, int rcount,
-				   struct ompi_datatype_t *rdtype, int root,
-				   struct ompi_communicator_t *comm,
-				   mca_coll_base_module_t *module);
-
-/**
- * Global variables used in the macros (essentially constants, so
- * these are thread safe)
- */
-extern uint32_t mca_coll_sm_one;
-
-
-/**
- * Macro to setup flag usage
- */
-#define FLAG_SETUP(flag_num, flag, data) \
-    (flag) = (mca_coll_sm_in_use_flag_t*) \
-        (((char *) (data)->mcb_in_use_flags) + \
-        ((flag_num) * mca_coll_sm_component.sm_control_size))
-
-/**
- * Macro to wait for the in-use flag to become idle (used by the root)
- */
-#define FLAG_WAIT_FOR_IDLE(flag, label) \
-    SPIN_CONDITION(0 == (flag)->mcsiuf_num_procs_using, label)
-
-/**
- * Macro to wait for a flag to indicate that it's ready for this
- * operation (used by non-root processes to know when FLAG_SET() has
- * been called)
- */
-#define FLAG_WAIT_FOR_OP(flag, op, label) \
-    SPIN_CONDITION((op) == flag->mcsiuf_operation_count, label)
-
-/**
- * Macro to set an in-use flag with relevant data to claim it
- */
-#define FLAG_RETAIN(flag, num_procs, op_count) \
-    (flag)->mcsiuf_num_procs_using = (num_procs); \
-    (flag)->mcsiuf_operation_count = (op_count)
-
-/**
- * Macro to release an in-use flag from this process
- */
-#define FLAG_RELEASE(flag) \
-    opal_atomic_add(&(flag)->mcsiuf_num_procs_using, -1)
-
-/**
- * Macro to copy a single segment in from a user buffer to a shared
- * segment
- */
-#define COPY_FRAGMENT_IN(convertor, index, rank, iov, max_data) \
-    (iov).iov_base = \
-        (index)->mcbmi_data + \
-        ((rank) * mca_coll_sm_component.sm_fragment_size); \
-    (iov).iov_len = (max_data); \
-    opal_convertor_pack(&(convertor), &(iov), &mca_coll_sm_one, \
-                        &(max_data) )
-
-/**
- * Macro to copy a single segment out from a shared segment to a user
- * buffer
- */
-#define COPY_FRAGMENT_OUT(convertor, src_rank, index, iov, max_data) \
-    (iov).iov_base = (((char*) (index)->mcbmi_data) + \
-                       ((src_rank) * (mca_coll_sm_component.sm_fragment_size))); \
-    (iov).iov_len = (max_data); \
-    opal_convertor_unpack(&(convertor), &(iov), &mca_coll_sm_one, \
-                          &(max_data) )
-
-/**
- * Macro to memcpy a fragment between one shared segment and another
- */
-#define COPY_FRAGMENT_BETWEEN(src_rank, dest_rank, index, len) \
-    memcpy(((index)->mcbmi_data + \
-            ((dest_rank) * mca_coll_sm_component.sm_fragment_size)), \
-           ((index)->mcbmi_data + \
-            ((src_rank) * \
-             mca_coll_sm_component.sm_fragment_size)), \
-           (len))
-
-/**
- * Macro to tell children that a segment is ready (normalize
- * the child's ID based on the shift used to calculate the "me" node
- * in the tree).  Used in fan out opertations.
- */
-#define PARENT_NOTIFY_CHILDREN(children, num_children, index, value) \
-    do { \
-        for (i = 0; i < (num_children); ++i) { \
-            *((size_t*) \
-              (((char*) index->mcbmi_control) + \
-               (mca_coll_sm_component.sm_control_size * \
-                (((children)[i]->mcstn_id + root) % size)))) = (value); \
-        } \
-    } while (0)
-
-/**
- * Macro for childen to wait for parent notification (use real rank).
- * Save the value passed and then reset it when done.  Used in fan out
- * operations.
- */
-#define CHILD_WAIT_FOR_NOTIFY(rank, index, value, label) \
-    do { \
-        uint32_t volatile *ptr = ((uint32_t*) \
-                                  (((char*) index->mcbmi_control) + \
-                                   ((rank) * mca_coll_sm_component.sm_control_size))); \
-        SPIN_CONDITION(0 != *ptr, label); \
-        (value) = *ptr; \
-        *ptr = 0; \
-    } while (0)
-
-/**
- * Macro for children to tell parent that the data is ready in their
- * segment.  Used for fan in operations.
- */
-#define CHILD_NOTIFY_PARENT(child_rank, parent_rank, index, value) \
-    ((size_t volatile *) \
-     (((char*) (index)->mcbmi_control) + \
-      (mca_coll_sm_component.sm_control_size * \
-       (parent_rank))))[(child_rank)] = (value)
-
-/**
- * Macro for parent to wait for a specific child to tell it that the
- * data is in the child's segment.  Save the value when done.  Used
- * for fan in operations.
- */
-#define PARENT_WAIT_FOR_NOTIFY_SPECIFIC(child_rank, parent_rank, index, value, label) \
-    do { \
-        size_t volatile *ptr = ((size_t volatile *) \
-                                (((char*) index->mcbmi_control) + \
-                                 (mca_coll_sm_component.sm_control_size * \
-                                  (parent_rank)))) + child_rank; \
-        SPIN_CONDITION(0 != *ptr, label); \
-        (value) = *ptr; \
-        *ptr = 0; \
-    } while (0)
-
-END_C_DECLS
-
-#endif /* MCA_COLL_SM_EXPORT_H */
diff --git a/ompi/mca/coll/sm/coll_sm_allgather.c b/ompi/mca/coll/sm/coll_sm_allgather.c
deleted file mode 100644
index 56d0a2ed8e2..00000000000
--- a/ompi/mca/coll/sm/coll_sm_allgather.c
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/constants.h"
-#include "coll_sm.h"
-
-
-/*
- *	allgather
- *
- *	Function:	- allgather
- *	Accepts:	- same as MPI_Allgather()
- *	Returns:	- MPI_SUCCESS or error code
- */
-int mca_coll_sm_allgather_intra(const void *sbuf, int scount,
-                                struct ompi_datatype_t *sdtype, void *rbuf,
-                                int rcount, struct ompi_datatype_t *rdtype,
-                                struct ompi_communicator_t *comm,
-                                mca_coll_base_module_t *module)
-{
-    return OMPI_ERR_NOT_IMPLEMENTED;
-}
diff --git a/ompi/mca/coll/sm/coll_sm_allgatherv.c b/ompi/mca/coll/sm/coll_sm_allgatherv.c
deleted file mode 100644
index 4f14374a87a..00000000000
--- a/ompi/mca/coll/sm/coll_sm_allgatherv.c
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/constants.h"
-#include "coll_sm.h"
-
-
-/*
- *	allgatherv_intra
- *
- *	Function:	- allgatherv
- *	Accepts:	- same as MPI_Allgatherv()
- *	Returns:	- MPI_SUCCESS or error code
- */
-int mca_coll_sm_allgatherv_intra(const void *sbuf, int scount,
-                                 struct ompi_datatype_t *sdtype,
-                                 void * rbuf, const int *rcounts, const int *disps,
-                                 struct ompi_datatype_t *rdtype,
-                                 struct ompi_communicator_t *comm,
-                                mca_coll_base_module_t *module)
-{
-    return OMPI_ERR_NOT_IMPLEMENTED;
-}
diff --git a/ompi/mca/coll/sm/coll_sm_allreduce.c b/ompi/mca/coll/sm/coll_sm_allreduce.c
deleted file mode 100644
index 190e16849c5..00000000000
--- a/ompi/mca/coll/sm/coll_sm_allreduce.c
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-/** @file */
-
-#include "ompi_config.h"
-
-#include "ompi/constants.h"
-#include "ompi/communicator/communicator.h"
-#include "coll_sm.h"
-
-
-/**
- * Shared memory allreduce.
- *
- * For the moment, all we're doing is a reduce to root==0 and then a
- * broadcast.  It is possible that we'll do something better someday.
- */
-int mca_coll_sm_allreduce_intra(const void *sbuf, void *rbuf, int count,
-                                struct ompi_datatype_t *dtype,
-                                struct ompi_op_t *op,
-                                struct ompi_communicator_t *comm,
-                                mca_coll_base_module_t *module)
-{
-    int ret;
-
-    /* Note that only the root can pass MPI_IN_PLACE to MPI_REDUCE, so
-       have slightly different logic for that case. */
-
-    if (MPI_IN_PLACE == sbuf) {
-        int rank = ompi_comm_rank(comm);
-        if (0 == rank) {
-            ret = mca_coll_sm_reduce_intra(sbuf, rbuf, count, dtype, op, 0,
-                                           comm, module);
-        } else {
-            ret = mca_coll_sm_reduce_intra(rbuf, NULL, count, dtype, op, 0,
-                                           comm, module);
-        }
-    } else {
-        ret = mca_coll_sm_reduce_intra(sbuf, rbuf, count, dtype, op, 0,
-                                       comm, module);
-    }
-    return (ret == OMPI_SUCCESS) ?
-        mca_coll_sm_bcast_intra(rbuf, count, dtype, 0, comm, module) : ret;
-}
diff --git a/ompi/mca/coll/sm/coll_sm_alltoall.c b/ompi/mca/coll/sm/coll_sm_alltoall.c
deleted file mode 100644
index 1fe9d38efc5..00000000000
--- a/ompi/mca/coll/sm/coll_sm_alltoall.c
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/constants.h"
-#include "coll_sm.h"
-
-
-/*
- *	alltoall_intra
- *
- *	Function:	- MPI_Alltoall
- *	Accepts:	- same as MPI_Alltoall()
- *	Returns:	- MPI_SUCCESS or an MPI error code
- */
-int mca_coll_sm_alltoall_intra(const void *sbuf, int scount,
-                               struct ompi_datatype_t *sdtype, void *rbuf,
-                               int rcount, struct ompi_datatype_t *rdtype,
-                               struct ompi_communicator_t *comm,
-                                mca_coll_base_module_t *module)
-{
-    return OMPI_ERR_NOT_IMPLEMENTED;
-}
diff --git a/ompi/mca/coll/sm/coll_sm_alltoallv.c b/ompi/mca/coll/sm/coll_sm_alltoallv.c
deleted file mode 100644
index d5ae55b7ed5..00000000000
--- a/ompi/mca/coll/sm/coll_sm_alltoallv.c
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/constants.h"
-#include "coll_sm.h"
-
-
-/*
- *	alltoallv_intra
- *
- *	Function:	- MPI_Alltoallv
- *	Accepts:	- same as MPI_Alltoallv()
- *	Returns:	- MPI_SUCCESS or an MPI error code
- */
-int mca_coll_sm_alltoallv_intra(const void *sbuf, const int *scounts, const int *sdisps,
-                                struct ompi_datatype_t *sdtype,
-                                void *rbuf, const int *rcounts, const int *rdisps,
-                                struct ompi_datatype_t *rdtype,
-                                struct ompi_communicator_t *comm,
-                                mca_coll_base_module_t *module)
-{
-    return OMPI_ERR_NOT_IMPLEMENTED;
-}
diff --git a/ompi/mca/coll/sm/coll_sm_alltoallw.c b/ompi/mca/coll/sm/coll_sm_alltoallw.c
deleted file mode 100644
index 977f2b869c0..00000000000
--- a/ompi/mca/coll/sm/coll_sm_alltoallw.c
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/constants.h"
-#include "coll_sm.h"
-
-
-/*
- *	alltoallw_intra
- *
- *	Function:	- MPI_Alltoallw
- *	Accepts:	- same as MPI_Alltoallw()
- *	Returns:	- MPI_SUCCESS or an MPI error code
- */
-int mca_coll_sm_alltoallw_intra(const void *sbuf, const int *scounts, const int *sdisps,
-                                struct ompi_datatype_t * const *sdtypes,
-                                void *rbuf, const int *rcounts, const int *rdisps,
-                                struct ompi_datatype_t * const *rdtypes,
-                                struct ompi_communicator_t *comm,
-                                mca_coll_base_module_t *module)
-{
-    return OMPI_ERR_NOT_IMPLEMENTED;
-}
diff --git a/ompi/mca/coll/sm/coll_sm_barrier.c b/ompi/mca/coll/sm/coll_sm_barrier.c
deleted file mode 100644
index e8e0fb5b786..00000000000
--- a/ompi/mca/coll/sm/coll_sm_barrier.c
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2014 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-/** @file */
-
-#include "ompi_config.h"
-
-#include "ompi/constants.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/mca/coll/coll.h"
-#include "opal/sys/atomic.h"
-#include "coll_sm.h"
-
-/**
- * Shared memory barrier.
- *
- * Tree-based algorithm for a barrier: a fan in to rank 0 followed by
- * a fan out using the barrier segments in the shared memory area.
- *
- * There are 2 sets of barrier buffers -- since there can only be, at
- * most, 2 outstanding barriers at any time, there is no need for more
- * than this.  The generalized in-use flags, control, and data
- * segments are not used.
- *
- * The general algorithm is for a given process to wait for its N
- * children to fan in by monitoring a uint32_t in its barrier "in"
- * buffer.  When this value reaches N (i.e., each of the children have
- * atomically incremented the value), then the process atomically
- * increases the uint32_t in its parent's "in" buffer.  Then the
- * process waits for the parent to set a "1" in the process' "out"
- * buffer.  Once this happens, the process writes a "1" in each of its
- * children's "out" buffers, and returns.
- *
- * There's corner cases, of course, such as the root that has no
- * parent, and the leaves that have no children.  But that's the
- * general idea.
- */
-int mca_coll_sm_barrier_intra(struct ompi_communicator_t *comm,
-                              mca_coll_base_module_t *module)
-{
-    int rank, buffer_set;
-    mca_coll_sm_comm_t *data;
-    uint32_t i, num_children;
-    volatile uint32_t *me_in, *me_out, *children = NULL;
-    opal_atomic_uint32_t *parent;
-    int uint_control_size;
-    mca_coll_sm_module_t *sm_module = (mca_coll_sm_module_t*) module;
-
-    /* Lazily enable the module the first time we invoke a collective
-       on it */
-    if (!sm_module->enabled) {
-        int ret;
-        if (OMPI_SUCCESS != (ret = ompi_coll_sm_lazy_enable(module, comm))) {
-            return ret;
-        }
-    }
-
-    uint_control_size =
-        mca_coll_sm_component.sm_control_size / sizeof(uint32_t);
-    data = sm_module->sm_comm_data;
-    rank = ompi_comm_rank(comm);
-    num_children = data->mcb_tree[rank].mcstn_num_children;
-    buffer_set = ((data->mcb_barrier_count++) % 2) * 2;
-    me_in = &data->mcb_barrier_control_me[buffer_set];
-    me_out = (uint32_t*)
-        (((char*) me_in) + mca_coll_sm_component.sm_control_size);
-
-    /* Wait for my children to write to my *in* buffer */
-
-    if (0 != num_children) {
-        /* Get children *out* buffer */
-        children = data->mcb_barrier_control_children + buffer_set +
-            uint_control_size;
-        SPIN_CONDITION(*me_in == num_children, exit_label1);
-        *me_in = 0;
-    }
-
-    /* Send to my parent and wait for a response (don't poll on
-       parent's out buffer -- that would cause a lot of network
-       traffic / contention / faults / etc.  Instead, children poll on
-       local memory and therefore only num_children messages are sent
-       across the network [vs. num_children *each* time all the
-       children poll] -- i.e., the memory is only being polled by one
-       process, and it is only changed *once* by an external
-       process) */
-
-    if (0 != rank) {
-        /* Get parent *in* buffer */
-        parent = &data->mcb_barrier_control_parent[buffer_set];
-        opal_atomic_add (parent, 1);
-
-        SPIN_CONDITION(0 != *me_out, exit_label2);
-        *me_out = 0;
-    }
-
-    /* Send to my children */
-
-    for (i = 0; i < num_children; ++i) {
-        children[i * uint_control_size * 4] = 1;
-    }
-
-    /* All done!  End state of the control segment:
-
-       me_in: 0
-       me_out: 0
-    */
-
-    return OMPI_SUCCESS;
-}
diff --git a/ompi/mca/coll/sm/coll_sm_bcast.c b/ompi/mca/coll/sm/coll_sm_bcast.c
deleted file mode 100644
index 754581ea160..00000000000
--- a/ompi/mca/coll/sm/coll_sm_bcast.c
+++ /dev/null
@@ -1,260 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-/** @file */
-
-#include "ompi_config.h"
-
-#include <string.h>
-
-#include "opal/datatype/opal_convertor.h"
-#include "ompi/constants.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/coll/coll.h"
-#include "opal/sys/atomic.h"
-#include "coll_sm.h"
-
-/**
- * Shared memory broadcast.
- *
- * For the root, the general algorithm is to wait for a set of
- * segments to become available.  Once it is, the root claims the set
- * by writing the current operation number and the number of processes
- * using the set to the flag.  The root then loops over the set of
- * segments; for each segment, it copies a fragment of the user's
- * buffer into the shared data segment and then writes the data size
- * into its childrens' control buffers.  The process is repeated until
- * all fragments have been written.
- *
- * For non-roots, for each set of buffers, they wait until the current
- * operation number appears in the in-use flag (i.e., written by the
- * root).  Then for each segment, they wait for a nonzero to appear
- * into their control buffers.  If they have children, they copy the
- * data from their parent's shared data segment into their shared data
- * segment, and write the data size into each of their childrens'
- * control buffers.  They then copy the data from their shared [local]
- * data segment into the user's output buffer.  The process is
- * repeated until all fragments have been received.  If they do not
- * have children, they copy the data directly from the parent's shared
- * data segment into the user's output buffer.
- */
-int mca_coll_sm_bcast_intra(void *buff, int count,
-                            struct ompi_datatype_t *datatype, int root,
-                            struct ompi_communicator_t *comm,
-                            mca_coll_base_module_t *module)
-{
-    struct iovec iov;
-    mca_coll_sm_module_t *sm_module = (mca_coll_sm_module_t*) module;
-    mca_coll_sm_comm_t *data;
-    int i, ret, rank, size, num_children, src_rank;
-    int flag_num, segment_num, max_segment_num;
-    int parent_rank;
-    size_t total_size, max_data, bytes;
-    mca_coll_sm_in_use_flag_t *flag;
-    opal_convertor_t convertor;
-    mca_coll_sm_tree_node_t *me, *parent, **children;
-    mca_coll_sm_data_index_t *index;
-
-    /* Lazily enable the module the first time we invoke a collective
-       on it */
-    if (!sm_module->enabled) {
-        if (OMPI_SUCCESS != (ret = ompi_coll_sm_lazy_enable(module, comm))) {
-            return ret;
-        }
-    }
-    data = sm_module->sm_comm_data;
-
-    /* Setup some identities */
-
-    rank = ompi_comm_rank(comm);
-    size = ompi_comm_size(comm);
-
-    OBJ_CONSTRUCT(&convertor, opal_convertor_t);
-    iov.iov_len = mca_coll_sm_component.sm_fragment_size;
-    bytes = 0;
-
-    me = &data->mcb_tree[(rank + size - root) % size];
-    parent = me->mcstn_parent;
-    children = me->mcstn_children;
-    num_children = me->mcstn_num_children;
-
-    /* Only have one top-level decision as to whether I'm the root or
-       not.  Do this at the slight expense of repeating a little logic
-       -- but it's better than a conditional branch in every loop
-       iteration. */
-
-    /*********************************************************************
-     * Root
-     *********************************************************************/
-
-    if (root == rank) {
-
-        /* The root needs a send convertor to pack from the user's
-           buffer to shared memory */
-
-        if (OMPI_SUCCESS !=
-            (ret =
-             opal_convertor_copy_and_prepare_for_send(ompi_mpi_local_convertor,
-                                                      &(datatype->super),
-                                                      count,
-                                                      buff,
-                                                      0,
-                                                      &convertor))) {
-            return ret;
-        }
-        opal_convertor_get_packed_size(&convertor, &total_size);
-
-        /* Main loop over sending fragments */
-
-        do {
-            flag_num = (data->mcb_operation_count++ %
-                        mca_coll_sm_component.sm_comm_num_in_use_flags);
-
-            FLAG_SETUP(flag_num, flag, data);
-            FLAG_WAIT_FOR_IDLE(flag, bcast_root_label);
-            FLAG_RETAIN(flag, size - 1, data->mcb_operation_count - 1);
-
-            /* Loop over all the segments in this set */
-
-            segment_num =
-                flag_num * mca_coll_sm_component.sm_segs_per_inuse_flag;
-            max_segment_num =
-                (flag_num + 1) * mca_coll_sm_component.sm_segs_per_inuse_flag;
-            do {
-                index = &(data->mcb_data_index[segment_num]);
-
-                /* Copy the fragment from the user buffer to my fragment
-                   in the current segment */
-                max_data = mca_coll_sm_component.sm_fragment_size;
-                COPY_FRAGMENT_IN(convertor, index, rank, iov, max_data);
-                bytes += max_data;
-
-                /* Wait for the write to absolutely complete */
-                opal_atomic_wmb();
-
-                /* Tell my children that this fragment is ready */
-                PARENT_NOTIFY_CHILDREN(children, num_children, index,
-                                       max_data);
-
-                ++segment_num;
-            } while (bytes < total_size && segment_num < max_segment_num);
-        } while (bytes < total_size);
-    }
-
-    /*********************************************************************
-     * Non-root
-     *********************************************************************/
-
-    else {
-
-        /* Non-root processes need a receive convertor to unpack from
-           shared mmory to the user's buffer */
-
-        if (OMPI_SUCCESS !=
-            (ret =
-             opal_convertor_copy_and_prepare_for_recv(ompi_mpi_local_convertor,
-                                                      &(datatype->super),
-                                                      count,
-                                                      buff,
-                                                      0,
-                                                      &convertor))) {
-            return ret;
-        }
-        opal_convertor_get_packed_size(&convertor, &total_size);
-
-        /* Loop over receiving (and possibly re-sending) the
-           fragments */
-
-        do {
-            flag_num = (data->mcb_operation_count %
-                        mca_coll_sm_component.sm_comm_num_in_use_flags);
-
-            /* Wait for the root to mark this set of segments as
-               ours */
-            FLAG_SETUP(flag_num, flag, data);
-            FLAG_WAIT_FOR_OP(flag, data->mcb_operation_count, bcast_nonroot_label1);
-            ++data->mcb_operation_count;
-
-            /* Loop over all the segments in this set */
-
-            segment_num =
-                flag_num * mca_coll_sm_component.sm_segs_per_inuse_flag;
-            max_segment_num =
-                (flag_num + 1) * mca_coll_sm_component.sm_segs_per_inuse_flag;
-            do {
-
-                /* Pre-calculate some values */
-                parent_rank = (parent->mcstn_id + root) % size;
-                index = &(data->mcb_data_index[segment_num]);
-
-                /* Wait for my parent to tell me that the segment is ready */
-                CHILD_WAIT_FOR_NOTIFY(rank, index, max_data, bcast_nonroot_label2);
-
-                /* If I have children, send the data to them */
-                if (num_children > 0) {
-                    /* Copy the fragment from the parent's portion in
-                       the segment to my portion in the segment. */
-                    COPY_FRAGMENT_BETWEEN(parent_rank, rank, index, max_data);
-
-                    /* Wait for the write to absolutely complete */
-                    opal_atomic_wmb();
-
-                    /* Tell my children that this fragment is ready */
-                    PARENT_NOTIFY_CHILDREN(children, num_children, index,
-                                           max_data);
-
-                    /* Set the "copy from buffer" to be my local
-                       segment buffer so that we don't potentially
-                       incur a non-local memory copy from the parent's
-                       fan out data segment [again] when copying to
-                       the user's buffer */
-                    src_rank = rank;
-                }
-
-                /* If I don't have any children, set the "copy from
-                   buffer" to be my parent's fan out segment to copy
-                   directly from my parent */
-
-                else {
-                    src_rank = parent_rank;
-                }
-
-                /* Copy to my output buffer */
-                COPY_FRAGMENT_OUT(convertor, src_rank, index, iov, max_data);
-
-                bytes += max_data;
-                ++segment_num;
-            } while (bytes < total_size && segment_num < max_segment_num);
-
-            /* Wait for all copy-out writes to complete before I say
-               I'm done with the segments */
-            opal_atomic_wmb();
-
-            /* We're finished with this set of segments */
-            FLAG_RELEASE(flag);
-        } while (bytes < total_size);
-    }
-
-    /* Kill the convertor */
-
-    OBJ_DESTRUCT(&convertor);
-
-    /* All done */
-
-    return OMPI_SUCCESS;
-}
diff --git a/ompi/mca/coll/sm/coll_sm_component.c b/ompi/mca/coll/sm/coll_sm_component.c
deleted file mode 100644
index 8a752806be2..00000000000
--- a/ompi/mca/coll/sm/coll_sm_component.c
+++ /dev/null
@@ -1,259 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008-2009 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2011-2015 Los Alamos National Security, LLC.
- *                         All rights reserved.
- * Copyright (c) 2015      Intel, Inc. All rights reserved.
- * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-/**
- * @file
- *
- * Most of the description of the data layout is in the
- * coll_sm_module.c file.
- */
-
-#include "ompi_config.h"
-
-#include "opal/util/show_help.h"
-#include "opal/util/printf.h"
-#include "ompi/constants.h"
-#include "ompi/mca/coll/coll.h"
-#include "coll_sm.h"
-
-
-/*
- * Public string showing the coll ompi_sm component version number
- */
-const char *mca_coll_sm_component_version_string =
-    "Open MPI sm collective MCA component version " OMPI_VERSION;
-
-
-/*
- * Local functions
- */
-static int sm_close(void);
-static int sm_register(void);
-
-static int coll_sm_shared_mem_used_data;
-
-/*
- * Instantiate the public struct with all of our public information
- * and pointers to our public functions in it
- */
-
-mca_coll_sm_component_t mca_coll_sm_component = {
-
-    /* First, fill in the super */
-
-    {
-        /* First, the mca_component_t struct containing meta
-           information about the component itself */
-
-        .collm_version = {
-            MCA_COLL_BASE_VERSION_2_4_0,
-
-            /* Component name and version */
-            .mca_component_name = "sm",
-            MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION,
-                                  OMPI_RELEASE_VERSION),
-
-            /* Component functions */
-            .mca_close_component = sm_close,
-            .mca_register_component_params = sm_register,
-        },
-        .collm_data = {
-            /* The component is not checkpoint ready */
-            MCA_BASE_METADATA_PARAM_NONE
-        },
-
-        /* Initialization / querying functions */
-
-        .collm_init_query = mca_coll_sm_init_query,
-        .collm_comm_query = mca_coll_sm_comm_query,
-    },
-
-    /* sm-component specifc information */
-
-    /* (default) priority */
-    /* JMS temporarily lowered until we can get more testing */
-    0,
-
-    /* (default) control size (bytes) */
-    4096,
-
-    /* (default) number of "in use" flags for each communicator's area
-       in the per-communicator shmem segment */
-    2,
-
-    /* (default) number of segments for each communicator in the
-       per-communicator shmem segment */
-    8,
-
-    /* (default) fragment size */
-    8192,
-
-    /* (default) degree of tree for tree-based operations (must be <=
-       control unit size) */
-    4,
-
-    /* (default) number of processes in coll_sm_shared_mem_size
-       information variable */
-    4,
-
-    /* default values for non-MCA parameters */
-    /* Not specifying values here gives us all 0's */
-};
-
-
-/*
- * Shut down the component
- */
-static int sm_close(void)
-{
-    return OMPI_SUCCESS;
-}
-
-static int sm_verify_mca_variables(void)
-{
-    mca_coll_sm_component_t *cs = &mca_coll_sm_component;
-
-    if (0 != (cs->sm_fragment_size % cs->sm_control_size)) {
-        cs->sm_fragment_size += cs->sm_control_size -
-            (cs->sm_fragment_size % cs->sm_control_size);
-    }
-
-    if (cs->sm_comm_num_in_use_flags < 2) {
-        cs->sm_comm_num_in_use_flags = 2;
-    }
-
-    if (cs->sm_comm_num_segments < cs->sm_comm_num_in_use_flags) {
-        cs->sm_comm_num_segments = cs->sm_comm_num_in_use_flags;
-    }
-    if (0 != (cs->sm_comm_num_segments % cs->sm_comm_num_in_use_flags)) {
-        cs->sm_comm_num_segments += cs->sm_comm_num_in_use_flags -
-            (cs->sm_comm_num_segments % cs->sm_comm_num_in_use_flags);
-    }
-    cs->sm_segs_per_inuse_flag =
-        cs->sm_comm_num_segments / cs->sm_comm_num_in_use_flags;
-
-    if (cs->sm_tree_degree > cs->sm_control_size) {
-        opal_show_help("help-mpi-coll-sm.txt",
-                       "tree-degree-larger-than-control", true,
-                       cs->sm_tree_degree, cs->sm_control_size);
-        cs->sm_tree_degree = cs->sm_control_size;
-    }
-    if (cs->sm_tree_degree > 255) {
-        opal_show_help("help-mpi-coll-sm.txt",
-                       "tree-degree-larger-than-255", true,
-                       cs->sm_tree_degree);
-        cs->sm_tree_degree = 255;
-    }
-
-    coll_sm_shared_mem_used_data = (int)(4 * cs->sm_control_size +
-        (cs->sm_comm_num_in_use_flags * cs->sm_control_size) +
-        (cs->sm_comm_num_segments * (cs->sm_info_comm_size * cs->sm_control_size * 2)) +
-        (cs->sm_comm_num_segments * (cs->sm_info_comm_size * cs->sm_fragment_size)));
-
-    return OMPI_SUCCESS;
-}
-
-/*
- * Register MCA params
- */
-static int sm_register(void)
-{
-    mca_base_component_t *c = &mca_coll_sm_component.super.collm_version;
-    mca_coll_sm_component_t *cs = &mca_coll_sm_component;
-
-    /* If we want to be selected (i.e., all procs on one node), then
-       we should have a high priority */
-
-    cs->sm_priority = 0;
-    (void) mca_base_component_var_register(c, "priority", "Priority of the sm coll component",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
-                                           &cs->sm_priority);
-
-    cs->sm_control_size = 4096;
-    (void) mca_base_component_var_register(c, "control_size",
-                                           "Length of the control data -- should usually be either the length of a cache line on most SMPs, or the size of a page on machines that support direct memory affinity page placement (in bytes)",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
-                                           &cs->sm_control_size);
-
-    cs->sm_fragment_size = 8192;
-    (void) mca_base_component_var_register(c, "fragment_size",
-                                           "Fragment size (in bytes) used for passing data through shared memory (will be rounded up to the nearest control_size size)",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
-                                           &cs->sm_fragment_size);
-
-    cs->sm_comm_num_in_use_flags = 2;
-    (void) mca_base_component_var_register(c, "comm_in_use_flags",
-                                           "Number of \"in use\" flags, used to mark a message passing area segment as currently being used or not (must be >= 2 and <= comm_num_segments)",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
-                                           &cs->sm_comm_num_in_use_flags);
-
-    cs->sm_comm_num_segments = 8;
-    (void) mca_base_component_var_register(c, "comm_num_segments",
-                                           "Number of segments in each communicator's shared memory message passing area (must be >= 2, and must be a multiple of comm_in_use_flags)",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
-                                           &cs->sm_comm_num_segments);
-
-    cs->sm_tree_degree = 4;
-    (void) mca_base_component_var_register(c, "tree_degree",
-                                           "Degree of the tree for tree-based operations (must be => 1 and <= min(control_size, 255))",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
-                                           &cs->sm_tree_degree);
-
-    /* INFO: Calculate how much space we need in the per-communicator
-       shmem data segment.  This formula taken directly from
-       coll_sm_module.c. */
-    cs->sm_info_comm_size = 4;
-    (void) mca_base_component_var_register(c, "info_num_procs",
-                                           "Number of processes to use for the calculation of the shared_mem_size MCA information parameter (must be => 2)",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
-                                           &cs->sm_info_comm_size);
-
-    coll_sm_shared_mem_used_data = (int)(4 * cs->sm_control_size +
-        (cs->sm_comm_num_in_use_flags * cs->sm_control_size) +
-        (cs->sm_comm_num_segments * (cs->sm_info_comm_size * cs->sm_control_size * 2)) +
-        (cs->sm_comm_num_segments * (cs->sm_info_comm_size * cs->sm_fragment_size)));
-
-    (void) mca_base_component_var_register(c, "shared_mem_used_data",
-                                           "Amount of shared memory used, per communicator, in the shared memory data area for info_num_procs processes (in bytes)",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0,
-                                           MCA_BASE_VAR_FLAG_DEFAULT_ONLY,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
-                                           &coll_sm_shared_mem_used_data);
-
-    return sm_verify_mca_variables();
-}
diff --git a/ompi/mca/coll/sm/coll_sm_exscan.c b/ompi/mca/coll/sm/coll_sm_exscan.c
deleted file mode 100644
index be008f226f4..00000000000
--- a/ompi/mca/coll/sm/coll_sm_exscan.c
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/constants.h"
-#include "coll_sm.h"
-
-
-/*
- *	exscan_intra
- *
- *	Function:	- basic exscan operation
- *	Accepts:	- same arguments as MPI_Exscan()
- *	Returns:	- MPI_SUCCESS or error code
- */
-int mca_coll_sm_exscan_intra(const void *sbuf, void *rbuf, int count,
-                             struct ompi_datatype_t *dtype,
-                             struct ompi_op_t *op,
-                             struct ompi_communicator_t *comm,
-                             mca_coll_base_module_t *module)
-{
-  return OMPI_ERR_NOT_IMPLEMENTED;
-}
diff --git a/ompi/mca/coll/sm/coll_sm_gather.c b/ompi/mca/coll/sm/coll_sm_gather.c
deleted file mode 100644
index 51cd944f25d..00000000000
--- a/ompi/mca/coll/sm/coll_sm_gather.c
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/constants.h"
-#include "coll_sm.h"
-
-
-/*
- *      gather
- *
- *      Function:       - shared memory gather
- *      Accepts:        - same as MPI_Gather()
- *      Returns:        - MPI_SUCCESS or error code
- */
-int mca_coll_sm_gather_intra(const void *sbuf, int scount,
-                             struct ompi_datatype_t *sdtype, void *rbuf,
-                             int rcount, struct ompi_datatype_t *rdtype,
-                             int root, struct ompi_communicator_t *comm,
-                             mca_coll_base_module_t *module)
-{
-    return OMPI_ERR_NOT_IMPLEMENTED;
-}
diff --git a/ompi/mca/coll/sm/coll_sm_gatherv.c b/ompi/mca/coll/sm/coll_sm_gatherv.c
deleted file mode 100644
index 601b3b18234..00000000000
--- a/ompi/mca/coll/sm/coll_sm_gatherv.c
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/constants.h"
-#include "coll_sm.h"
-
-
-/*
- *	gatherv_intra
- *
- *	Function:	- basic gatherv operation
- *	Accepts:	- same arguments as MPI_Gatherb()
- *	Returns:	- MPI_SUCCESS or error code
- */
-int mca_coll_sm_gatherv_intra(const void *sbuf, int scount,
-                              struct ompi_datatype_t *sdtype,
-                              void *rbuf, const int *rcounts, const int *disps,
-                              struct ompi_datatype_t *rdtype, int root,
-                              struct ompi_communicator_t *comm,
-                              mca_coll_base_module_t *module)
-{
-    return OMPI_ERR_NOT_IMPLEMENTED;
-}
diff --git a/ompi/mca/coll/sm/coll_sm_module.c b/ompi/mca/coll/sm/coll_sm_module.c
deleted file mode 100644
index 005cd431d7b..00000000000
--- a/ompi/mca/coll/sm/coll_sm_module.c
+++ /dev/null
@@ -1,591 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2009-2013 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Los Alamos National Security, LLC.
- *                         All rights reserved.
- * Copyright (c) 2014-2015 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2015-2019 Intel, Inc.  All rights reserved.
- * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-/**
- * @file
- *
- * Warning: this is not for the faint of heart -- don't even bother
- * reading this source code if you don't have a strong understanding
- * of nested data structures and pointer math (remember that
- * associativity and order of C operations is *critical* in terms of
- * pointer math!).
- */
-
-#include "ompi_config.h"
-
-#include <stdio.h>
-#include <string.h>
-#ifdef HAVE_SCHED_H
-#include <sched.h>
-#endif
-#include <sys/types.h>
-#ifdef HAVE_SYS_MMAN_H
-#include <sys/mman.h>
-#endif  /* HAVE_SYS_MMAN_H */
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif  /* HAVE_UNISTD_H */
-
-#include "mpi.h"
-#include "opal_stdint.h"
-#include "opal/mca/hwloc/base/base.h"
-#include "opal/util/os_path.h"
-#include "opal/util/printf.h"
-
-#include "ompi/communicator/communicator.h"
-#include "ompi/group/group.h"
-#include "ompi/mca/coll/coll.h"
-#include "ompi/mca/coll/base/base.h"
-#include "ompi/runtime/ompi_rte.h"
-#include "ompi/proc/proc.h"
-#include "coll_sm.h"
-
-#include "ompi/mca/coll/base/coll_tags.h"
-#include "ompi/mca/pml/pml.h"
-
-/*
- * Global variables
- */
-uint32_t mca_coll_sm_one = 1;
-
-
-/*
- * Local functions
- */
-static int sm_module_enable(mca_coll_base_module_t *module,
-                          struct ompi_communicator_t *comm);
-static int bootstrap_comm(ompi_communicator_t *comm,
-                          mca_coll_sm_module_t *module);
-static int mca_coll_sm_module_disable(mca_coll_base_module_t *module,
-                          struct ompi_communicator_t *comm);
-
-/*
- * Module constructor
- */
-static void mca_coll_sm_module_construct(mca_coll_sm_module_t *module)
-{
-    module->enabled = false;
-    module->sm_comm_data = NULL;
-    module->previous_reduce = NULL;
-    module->previous_reduce_module = NULL;
-    module->super.coll_module_disable = mca_coll_sm_module_disable;
-}
-
-/*
- * Module destructor
- */
-static void mca_coll_sm_module_destruct(mca_coll_sm_module_t *module)
-{
-    mca_coll_sm_comm_t *c = module->sm_comm_data;
-
-    if (NULL != c) {
-        /* Munmap the per-communicator shmem data segment */
-        if (NULL != c->sm_bootstrap_meta) {
-            /* Ignore any errors -- what are we going to do about
-               them? */
-            mca_common_sm_fini(c->sm_bootstrap_meta);
-            OBJ_RELEASE(c->sm_bootstrap_meta);
-        }
-        free(c);
-    }
-
-    /* It should always be non-NULL, but just in case */
-    if (NULL != module->previous_reduce_module) {
-        OBJ_RELEASE(module->previous_reduce_module);
-    }
-
-    module->enabled = false;
-}
-
-/*
- * Module disable
- */
-static int mca_coll_sm_module_disable(mca_coll_base_module_t *module, struct ompi_communicator_t *comm)
-{
-    mca_coll_sm_module_t *sm_module = (mca_coll_sm_module_t*) module;
-    if (NULL != sm_module->previous_reduce_module) {
-	sm_module->previous_reduce = NULL;
-        OBJ_RELEASE(sm_module->previous_reduce_module);
-	sm_module->previous_reduce_module = NULL;
-    }
-    return OMPI_SUCCESS;
-}
-
-
-OBJ_CLASS_INSTANCE(mca_coll_sm_module_t,
-                   mca_coll_base_module_t,
-                   mca_coll_sm_module_construct,
-                   mca_coll_sm_module_destruct);
-
-/*
- * Initial query function that is invoked during MPI_INIT, allowing
- * this component to disqualify itself if it doesn't support the
- * required level of thread support.  This function is invoked exactly
- * once.
- */
-int mca_coll_sm_init_query(bool enable_progress_threads,
-                           bool enable_mpi_threads)
-{
-    /* if no session directory was created, then we cannot be used */
-    if (NULL == ompi_process_info.job_session_dir) {
-        return OMPI_ERR_OUT_OF_RESOURCE;
-    }
-    /* Don't do much here because we don't really want to allocate any
-       shared memory until this component is selected to be used. */
-    opal_output_verbose(10, ompi_coll_base_framework.framework_output,
-                        "coll:sm:init_query: pick me! pick me!");
-    return OMPI_SUCCESS;
-}
-
-
-/*
- * Invoked when there's a new communicator that has been created.
- * Look at the communicator and decide which set of functions and
- * priority we want to return.
- */
-mca_coll_base_module_t *
-mca_coll_sm_comm_query(struct ompi_communicator_t *comm, int *priority)
-{
-    mca_coll_sm_module_t *sm_module;
-
-    /* If we're intercomm, or if there's only one process in the
-       communicator, or if not all the processes in the communicator
-       are not on this node, then we don't want to run */
-    if (OMPI_COMM_IS_INTER(comm) || 1 == ompi_comm_size(comm) || ompi_group_have_remote_peers (comm->c_local_group)) {
-        opal_output_verbose(10, ompi_coll_base_framework.framework_output,
-                            "coll:sm:comm_query (%d/%s): intercomm, comm is too small, or not all peers local; disqualifying myself", comm->c_contextid, comm->c_name);
-        return NULL;
-    }
-
-    /* Get the priority level attached to this module. If priority is less
-     * than or equal to 0, then the module is unavailable. */
-    *priority = mca_coll_sm_component.sm_priority;
-    if (mca_coll_sm_component.sm_priority < 0) {
-        opal_output_verbose(10, ompi_coll_base_framework.framework_output,
-                            "coll:sm:comm_query (%d/%s): priority too low; disqualifying myself", comm->c_contextid, comm->c_name);
-        return NULL;
-    }
-
-    sm_module = OBJ_NEW(mca_coll_sm_module_t);
-    if (NULL == sm_module) {
-        return NULL;
-    }
-
-    /* All is good -- return a module */
-    sm_module->super.coll_module_enable = sm_module_enable;
-    sm_module->super.coll_allgather  = NULL;
-    sm_module->super.coll_allgatherv = NULL;
-    sm_module->super.coll_allreduce  = mca_coll_sm_allreduce_intra;
-    sm_module->super.coll_alltoall   = NULL;
-    sm_module->super.coll_alltoallv  = NULL;
-    sm_module->super.coll_alltoallw  = NULL;
-    sm_module->super.coll_barrier    = mca_coll_sm_barrier_intra;
-    sm_module->super.coll_bcast      = mca_coll_sm_bcast_intra;
-    sm_module->super.coll_exscan     = NULL;
-    sm_module->super.coll_gather     = NULL;
-    sm_module->super.coll_gatherv    = NULL;
-    sm_module->super.coll_reduce     = mca_coll_sm_reduce_intra;
-    sm_module->super.coll_reduce_scatter = NULL;
-    sm_module->super.coll_scan       = NULL;
-    sm_module->super.coll_scatter    = NULL;
-    sm_module->super.coll_scatterv   = NULL;
-
-    opal_output_verbose(10, ompi_coll_base_framework.framework_output,
-                        "coll:sm:comm_query (%d/%s): pick me! pick me!",
-                        comm->c_contextid, comm->c_name);
-    return &(sm_module->super);
-}
-
-
-/*
- * Init module on the communicator
- */
-static int sm_module_enable(mca_coll_base_module_t *module,
-                            struct ompi_communicator_t *comm)
-{
-    if (NULL == comm->c_coll->coll_reduce ||
-        NULL == comm->c_coll->coll_reduce_module) {
-        opal_output_verbose(10, ompi_coll_base_framework.framework_output,
-                            "coll:sm:enable (%d/%s): no underlying reduce; disqualifying myself",
-                            comm->c_contextid, comm->c_name);
-        return OMPI_ERROR;
-    }
-
-    /* We do everything lazily in ompi_coll_sm_enable() */
-    return OMPI_SUCCESS;
-}
-
-int ompi_coll_sm_lazy_enable(mca_coll_base_module_t *module,
-                             struct ompi_communicator_t *comm)
-{
-    int i, j, root, ret;
-    int rank = ompi_comm_rank(comm);
-    int size = ompi_comm_size(comm);
-    mca_coll_sm_module_t *sm_module = (mca_coll_sm_module_t*) module;
-    mca_coll_sm_comm_t *data = NULL;
-    size_t control_size, frag_size;
-    mca_coll_sm_component_t *c = &mca_coll_sm_component;
-    opal_hwloc_base_memory_segment_t *maffinity;
-    int parent, min_child, num_children;
-    unsigned char *base = NULL;
-    const int num_barrier_buffers = 2;
-
-    /* Just make sure we haven't been here already */
-    if (sm_module->enabled) {
-        return OMPI_SUCCESS;
-    }
-    sm_module->enabled = true;
-
-    /* Get some space to setup memory affinity (just easier to try to
-       alloc here to handle the error case) */
-    maffinity = (opal_hwloc_base_memory_segment_t*)
-        malloc(sizeof(opal_hwloc_base_memory_segment_t) *
-               c->sm_comm_num_segments * 3);
-    if (NULL == maffinity) {
-        opal_output_verbose(10, ompi_coll_base_framework.framework_output,
-                            "coll:sm:enable (%d/%s): malloc failed (1)",
-                            comm->c_contextid, comm->c_name);
-        return OMPI_ERR_OUT_OF_RESOURCE;
-    }
-
-    /* Allocate data to hang off the communicator.  The memory we
-       alloc will be laid out as follows:
-
-       1. mca_coll_base_comm_t
-       2. array of num_segments mca_coll_base_mpool_index_t instances
-          (pointed to by the array in 2)
-       3. array of ompi_comm_size(comm) mca_coll_sm_tree_node_t
-          instances
-       4. array of sm_tree_degree pointers to other tree nodes (i.e.,
-          this nodes' children) for each instance of
-          mca_coll_sm_tree_node_t
-    */
-    sm_module->sm_comm_data = data = (mca_coll_sm_comm_t*)
-        malloc(sizeof(mca_coll_sm_comm_t) +
-               (c->sm_comm_num_segments *
-                sizeof(mca_coll_sm_data_index_t)) +
-               (size *
-                (sizeof(mca_coll_sm_tree_node_t) +
-                 (sizeof(mca_coll_sm_tree_node_t*) * c->sm_tree_degree))));
-    if (NULL == data) {
-        free(maffinity);
-        opal_output_verbose(10, ompi_coll_base_framework.framework_output,
-                            "coll:sm:enable (%d/%s): malloc failed (2)",
-                            comm->c_contextid, comm->c_name);
-        return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
-    }
-    data->mcb_operation_count = 0;
-
-    /* Setup #2: set the array to point immediately beyond the
-       mca_coll_base_comm_t */
-    data->mcb_data_index = (mca_coll_sm_data_index_t*) (data + 1);
-    /* Setup array of pointers for #3 */
-    data->mcb_tree = (mca_coll_sm_tree_node_t*)
-        (data->mcb_data_index + c->sm_comm_num_segments);
-    /* Finally, setup the array of children pointers in the instances
-       in #5 to point to their corresponding arrays in #6 */
-    data->mcb_tree[0].mcstn_children = (mca_coll_sm_tree_node_t**)
-        (data->mcb_tree + size);
-    for (i = 1; i < size; ++i) {
-        data->mcb_tree[i].mcstn_children =
-            data->mcb_tree[i - 1].mcstn_children + c->sm_tree_degree;
-    }
-
-    /* Pre-compute a tree for a given number of processes and degree.
-       We'll re-use this tree for all possible values of root (i.e.,
-       shift everyone's process to be the "0"/root in this tree. */
-    for (root = 0; root < size; ++root) {
-        parent = (root - 1) / mca_coll_sm_component.sm_tree_degree;
-        num_children = mca_coll_sm_component.sm_tree_degree;
-
-        /* Do we have children?  If so, how many? */
-
-        if ((root * num_children) + 1 >= size) {
-            /* Leaves */
-            min_child = -1;
-            num_children = 0;
-        } else {
-            /* Interior nodes */
-            int max_child;
-            min_child = root * num_children + 1;
-            max_child = root * num_children + num_children;
-            if (max_child >= size) {
-                max_child = size - 1;
-            }
-            num_children = max_child - min_child + 1;
-        }
-
-        /* Save the values */
-        data->mcb_tree[root].mcstn_id = root;
-        if (root == 0 && parent == 0) {
-            data->mcb_tree[root].mcstn_parent = NULL;
-        } else {
-            data->mcb_tree[root].mcstn_parent = &data->mcb_tree[parent];
-        }
-        data->mcb_tree[root].mcstn_num_children = num_children;
-        for (i = 0; i < c->sm_tree_degree; ++i) {
-            data->mcb_tree[root].mcstn_children[i] =
-                (i < num_children) ?
-                &data->mcb_tree[min_child + i] : NULL;
-        }
-    }
-
-    /* Attach to this communicator's shmem data segment */
-    if (OMPI_SUCCESS != (ret = bootstrap_comm(comm, sm_module))) {
-        free(data);
-        free(maffinity);
-        sm_module->sm_comm_data = NULL;
-        return ret;
-    }
-
-    /* Once the communicator is bootstrapped, setup the pointers into
-       the per-communicator shmem data segment.  First, setup the
-       barrier buffers.  There are 2 sets of barrier buffers (because
-       there can never be more than one outstanding barrier occuring
-       at any timie).  Setup pointers to my control buffers, my
-       parents, and [the beginning of] my children (note that the
-       children are contiguous, so having the first pointer and the
-       num_children from the mcb_tree data is sufficient). */
-    control_size = c->sm_control_size;
-    base = data->sm_bootstrap_meta->module_data_addr;
-    data->mcb_barrier_control_me = (uint32_t*)
-        (base + (rank * control_size * num_barrier_buffers * 2));
-    if (data->mcb_tree[rank].mcstn_parent) {
-        data->mcb_barrier_control_parent = (opal_atomic_uint32_t*)
-            (base +
-             (data->mcb_tree[rank].mcstn_parent->mcstn_id * control_size *
-              num_barrier_buffers * 2));
-    } else {
-        data->mcb_barrier_control_parent = NULL;
-    }
-    if (data->mcb_tree[rank].mcstn_num_children > 0) {
-        data->mcb_barrier_control_children = (uint32_t*)
-            (base +
-             (data->mcb_tree[rank].mcstn_children[0]->mcstn_id * control_size *
-              num_barrier_buffers * 2));
-    } else {
-        data->mcb_barrier_control_children = NULL;
-    }
-    data->mcb_barrier_count = 0;
-
-    /* Next, setup the pointer to the in-use flags.  The number of
-       segments will be an even multiple of the number of in-use
-       flags. */
-    base += (c->sm_control_size * size * num_barrier_buffers * 2);
-    data->mcb_in_use_flags = (mca_coll_sm_in_use_flag_t*) base;
-
-    /* All things being equal, if we're rank 0, then make the in-use
-       flags be local (memory affinity).  Then zero them all out so
-       that they're marked as unused. */
-    j = 0;
-    if (0 == rank) {
-        maffinity[j].mbs_start_addr = base;
-        maffinity[j].mbs_len = c->sm_control_size *
-            c->sm_comm_num_in_use_flags;
-        /* Set the op counts to 1 (actually any nonzero value will do)
-           so that the first time children/leaf processes come
-           through, they don't see a value of 0 and think that the
-           root/parent has already set the count to their op number
-           (i.e., 0 is the first op count value). */
-        for (i = 0; i < mca_coll_sm_component.sm_comm_num_in_use_flags; ++i) {
-            ((mca_coll_sm_in_use_flag_t *)base)[i].mcsiuf_operation_count = 1;
-            ((mca_coll_sm_in_use_flag_t *)base)[i].mcsiuf_num_procs_using = 0;
-        }
-        ++j;
-    }
-
-    /* Next, setup pointers to the control and data portions of the
-       segments, as well as to the relevant in-use flags. */
-    base += (c->sm_comm_num_in_use_flags * c->sm_control_size);
-    control_size = size * c->sm_control_size;
-    frag_size = size * c->sm_fragment_size;
-    for (i = 0; i < c->sm_comm_num_segments; ++i) {
-        data->mcb_data_index[i].mcbmi_control = (uint32_t*)
-            (base + (i * (control_size + frag_size)));
-        data->mcb_data_index[i].mcbmi_data =
-            (((char*) data->mcb_data_index[i].mcbmi_control) +
-             control_size);
-
-        /* Memory affinity: control */
-
-        maffinity[j].mbs_len = c->sm_control_size;
-        maffinity[j].mbs_start_addr = (void *)
-            (((char*) data->mcb_data_index[i].mcbmi_control) +
-             (rank * c->sm_control_size));
-        ++j;
-
-        /* Memory affinity: data */
-
-        maffinity[j].mbs_len = c->sm_fragment_size;
-        maffinity[j].mbs_start_addr =
-            ((char*) data->mcb_data_index[i].mcbmi_data) +
-            (rank * c->sm_control_size);
-        ++j;
-    }
-
-    /* Setup memory affinity so that the pages that belong to this
-       process are local to this process */
-    opal_hwloc_base_memory_set(maffinity, j);
-    free(maffinity);
-
-    /* Zero out the control structures that belong to this process */
-    memset(data->mcb_barrier_control_me, 0,
-           num_barrier_buffers * 2 * c->sm_control_size);
-    for (i = 0; i < c->sm_comm_num_segments; ++i) {
-        memset((void *) data->mcb_data_index[i].mcbmi_control, 0,
-               c->sm_control_size);
-    }
-
-    /* Save previous component's reduce information */
-    sm_module->previous_reduce = comm->c_coll->coll_reduce;
-    sm_module->previous_reduce_module = comm->c_coll->coll_reduce_module;
-    OBJ_RETAIN(sm_module->previous_reduce_module);
-
-    /* Indicate that we have successfully attached and setup */
-    opal_atomic_add (&(data->sm_bootstrap_meta->module_seg->seg_inited), 1);
-
-    /* Wait for everyone in this communicator to attach and setup */
-    opal_output_verbose(10, ompi_coll_base_framework.framework_output,
-                        "coll:sm:enable (%d/%s): waiting for peers to attach",
-                        comm->c_contextid, comm->c_name);
-    SPIN_CONDITION(size == data->sm_bootstrap_meta->module_seg->seg_inited, seg_init_exit);
-
-    /* Once we're all here, remove the mmap file; it's not needed anymore */
-    if (0 == rank) {
-        unlink(data->sm_bootstrap_meta->shmem_ds.seg_name);
-        opal_output_verbose(10, ompi_coll_base_framework.framework_output,
-                            "coll:sm:enable (%d/%s): removed mmap file %s",
-                            comm->c_contextid, comm->c_name,
-                            data->sm_bootstrap_meta->shmem_ds.seg_name);
-    }
-
-    /* All done */
-
-    opal_output_verbose(10, ompi_coll_base_framework.framework_output,
-                        "coll:sm:enable (%d/%s): success!",
-                        comm->c_contextid, comm->c_name);
-    return OMPI_SUCCESS;
-}
-
-static int bootstrap_comm(ompi_communicator_t *comm,
-                          mca_coll_sm_module_t *module)
-{
-    int i;
-    char *shortpath, *fullpath;
-    mca_coll_sm_component_t *c = &mca_coll_sm_component;
-    mca_coll_sm_comm_t *data = module->sm_comm_data;
-    int comm_size = ompi_comm_size(comm);
-    int num_segments = c->sm_comm_num_segments;
-    int num_in_use = c->sm_comm_num_in_use_flags;
-    int frag_size = c->sm_fragment_size;
-    int control_size = c->sm_control_size;
-    ompi_process_name_t *lowest_name = NULL;
-    size_t size;
-    ompi_proc_t *proc;
-
-    /* Make the rendezvous filename for this communicators shmem data
-       segment.  The CID is not guaranteed to be unique among all
-       procs on this node, so also pair it with the PID of the proc
-       with the lowest ORTE name to form a unique filename. */
-    proc = ompi_group_peer_lookup(comm->c_local_group, 0);
-    lowest_name = OMPI_CAST_RTE_NAME(&proc->super.proc_name);
-    for (i = 1; i < comm_size; ++i) {
-        proc = ompi_group_peer_lookup(comm->c_local_group, i);
-        if (ompi_rte_compare_name_fields(OMPI_RTE_CMP_ALL,
-                                          OMPI_CAST_RTE_NAME(&proc->super.proc_name),
-                                          lowest_name) < 0) {
-            lowest_name = OMPI_CAST_RTE_NAME(&proc->super.proc_name);
-        }
-    }
-    opal_asprintf(&shortpath, "coll-sm-cid-%d-name-%s.mmap", comm->c_contextid,
-             OMPI_NAME_PRINT(lowest_name));
-    if (NULL == shortpath) {
-        opal_output_verbose(10, ompi_coll_base_framework.framework_output,
-                            "coll:sm:enable:bootstrap comm (%d/%s): asprintf failed",
-                            comm->c_contextid, comm->c_name);
-        return OMPI_ERR_OUT_OF_RESOURCE;
-    }
-    fullpath = opal_os_path(false, ompi_process_info.job_session_dir,
-                            shortpath, NULL);
-    free(shortpath);
-    if (NULL == fullpath) {
-        opal_output_verbose(10, ompi_coll_base_framework.framework_output,
-                            "coll:sm:enable:bootstrap comm (%d/%s): opal_os_path failed",
-                            comm->c_contextid, comm->c_name);
-        return OMPI_ERR_OUT_OF_RESOURCE;
-    }
-
-    /* Calculate how much space we need in the per-communicator shmem
-       data segment.  There are several values to add:
-
-       - size of the barrier data (2 of these):
-           - fan-in data (num_procs * control_size)
-           - fan-out data (num_procs * control_size)
-       - size of the "in use" buffers:
-           - num_in_use_buffers * control_size
-       - size of the message fragment area (one for each segment):
-           - control (num_procs * control_size)
-           - fragment data (num_procs * (frag_size))
-
-       So it's:
-
-           barrier: 2 * control_size + 2 * control_size
-           in use:  num_in_use * control_size
-           control: num_segments * (num_procs * control_size * 2 +
-                                    num_procs * control_size)
-           message: num_segments * (num_procs * frag_size)
-     */
-
-    size = 4 * control_size +
-        (num_in_use * control_size) +
-        (num_segments * (comm_size * control_size * 2)) +
-        (num_segments * (comm_size * frag_size));
-    opal_output_verbose(10, ompi_coll_base_framework.framework_output,
-                        "coll:sm:enable:bootstrap comm (%d/%s): attaching to %" PRIsize_t " byte mmap: %s",
-                        comm->c_contextid, comm->c_name, size, fullpath);
-    if (0 == ompi_comm_rank (comm)) {
-        data->sm_bootstrap_meta = mca_common_sm_module_create_and_attach (size, fullpath, sizeof(mca_common_sm_seg_header_t), 8);
-        if (NULL == data->sm_bootstrap_meta) {
-            opal_output_verbose(10, ompi_coll_base_framework.framework_output,
-                "coll:sm:enable:bootstrap comm (%d/%s): mca_common_sm_init_group failed",
-                comm->c_contextid, comm->c_name);
-            free(fullpath);
-            return OMPI_ERR_OUT_OF_RESOURCE;
-        }
-
-        for (int i = 1 ; i < ompi_comm_size (comm) ; ++i) {
-            MCA_PML_CALL(send(&data->sm_bootstrap_meta->shmem_ds, sizeof (data->sm_bootstrap_meta->shmem_ds), MPI_BYTE,
-                         i, MCA_COLL_BASE_TAG_BCAST, MCA_PML_BASE_SEND_STANDARD, comm));
-        }
-    } else {
-        opal_shmem_ds_t shmem_ds;
-        MCA_PML_CALL(recv(&shmem_ds, sizeof (shmem_ds), MPI_BYTE, 0, MCA_COLL_BASE_TAG_BCAST, comm, MPI_STATUS_IGNORE));
-        data->sm_bootstrap_meta = mca_common_sm_module_attach (&shmem_ds, sizeof(mca_common_sm_seg_header_t), 8);
-    }
-
-    /* All done */
-    free(fullpath);
-    return OMPI_SUCCESS;
-}
diff --git a/ompi/mca/coll/sm/coll_sm_reduce.c b/ompi/mca/coll/sm/coll_sm_reduce.c
deleted file mode 100644
index c731b87d2b8..00000000000
--- a/ompi/mca/coll/sm/coll_sm_reduce.c
+++ /dev/null
@@ -1,568 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2015 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2009-2013 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <string.h>
-
-#include "opal/datatype/opal_convertor.h"
-#include "opal/sys/atomic.h"
-#include "ompi/constants.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/mca/coll/coll.h"
-#include "ompi/op/op.h"
-#include "coll_sm.h"
-
-
-/*
- * Local functions
- */
-static int reduce_inorder(const void *sbuf, void* rbuf, int count,
-                          struct ompi_datatype_t *dtype,
-                          struct ompi_op_t *op,
-                          int root, struct ompi_communicator_t *comm,
-                          mca_coll_base_module_t *module);
-#define WANT_REDUCE_NO_ORDER 0
-#if WANT_REDUCE_NO_ORDER
-static int reduce_no_order(const void *sbuf, void* rbuf, int count,
-                           struct ompi_datatype_t *dtype,
-                           struct ompi_op_t *op,
-                           int root, struct ompi_communicator_t *comm,
-                           mca_coll_base_module_t *module);
-#endif
-
-/*
- * Useful utility routine
- */
-#if !defined(min)
-static inline int min(int a, int b)
-{
-    return (a < b) ? a : b;
-}
-#endif
-
-/**
- * Shared memory reduction.
- *
- * Simply farms out to the associative or non-associative functions.
- */
-int mca_coll_sm_reduce_intra(const void *sbuf, void* rbuf, int count,
-                             struct ompi_datatype_t *dtype,
-                             struct ompi_op_t *op,
-                             int root, struct ompi_communicator_t *comm,
-                             mca_coll_base_module_t *module)
-{
-    size_t size;
-    mca_coll_sm_module_t *sm_module = (mca_coll_sm_module_t*) module;
-
-    /* There are several possibilities:
-     *
-     * 0. If the datatype is larger than a segment, fall back to
-     *    underlying module
-     * 1. If the op is user-defined, use the strict order
-     * 2. If the op is intrinsic:
-     *    a. If the op is float-associative, use the unordered
-     *    b. If the op is not float-associative:
-     *       i. if the data is floating point, use the strict order
-     *       ii. if the data is not floating point, use the unordered
-     */
-
-    ompi_datatype_type_size(dtype, &size);
-    if ((int)size > mca_coll_sm_component.sm_control_size) {
-        return sm_module->previous_reduce(sbuf, rbuf, count,
-                                          dtype, op, root, comm,
-                                          sm_module->previous_reduce_module);
-    }
-#if WANT_REDUCE_NO_ORDER
-    else {
-        /* Lazily enable the module the first time we invoke a
-           collective on it */
-        if (!sm_module->enabled) {
-            if (OMPI_SUCCESS !=
-                (ret = ompi_coll_sm_lazy_enable(module, comm))) {
-                return ret;
-            }
-        }
-
-        if (!ompi_op_is_intrinsic(op) ||
-            (ompi_op_is_intrinsic(op) && !ompi_op_is_float_assoc(op) &&
-             0 != (dtype->flags & OMPI_DATATYPE_FLAG_DATA_FLOAT))) {
-            return reduce_inorder(sbuf, rbuf, count, dtype, op,
-                                  root, comm, module);
-        } else {
-            return reduce_no_order(sbuf, rbuf, count, dtype, op,
-                                   root, comm, module);
-        }
-    }
-#else
-    else {
-        /* Lazily enable the module the first time we invoke a
-           collective on it */
-        if (!sm_module->enabled) {
-            int ret;
-
-            if (OMPI_SUCCESS !=
-                (ret = ompi_coll_sm_lazy_enable(module, comm))) {
-                return ret;
-            }
-        }
-
-        return reduce_inorder(sbuf, rbuf, count, dtype, op, root, comm, module);
-    }
-#endif
-}
-
-
-/**
- * In-order shared memory reduction.
- *
- * This function performs the reduction in order -- combining elements
- * starting with (0 operation 1), then (result operation 2), then
- * (result operation 3), etc.
- *
- * Root's algorithm:
- *
- * If our datatype is "friendly" (i.e., the representation of the
- * buffer is the same packed as it is unpacked), then the root doesn't
- * need a temporary buffer -- we can combine the operands directly
- * from the shared memory segments to the root's rbuf.  Otherwise, we
- * need a receive convertor and receive each fragment into a temporary
- * buffer where we can combine that operan with the root's rbuf.
- *
- * In general, there are two loops:
- *
- * 1. loop over all fragments (which must be done in units of an
- * integer number of datatypes -- remember that if this function is
- * called, we know that the datattype is smaller than the max size of
- * a fragment, so this is definitely possible)
- *
- * 2. loop over all the processes -- 0 to (comm_size-1).
- * For process 0:
- * - if the root==0, copy the *entire* buffer (i.e., don't copy
- *   fragment by fragment -- might as well copy the entire thing) the
- *   first time through the algorithm, and no-op every other time
- * - else, copy from the shmem fragment to the out buffer
- * For all other proceses:
- * - if root==i, combine the relevant fragment from the sbuf to the
- *   relevant fragment on the rbuf
- * - else, if the datatype is friendly, combine relevant fragment from
- *   the shmem segment to the relevant fragment in the rbuf.  Otherwise,
- *   use the convertor to copy the fragment out of shmem into a temp
- *   buffer and do the combination from there to the rbuf.
- *
- * If we don't have a friendly datatype, then free the temporary
- * buffer at the end.
- */
-
-
-static int reduce_inorder(const void *sbuf, void* rbuf, int count,
-                          struct ompi_datatype_t *dtype,
-                          struct ompi_op_t *op,
-                          int root, struct ompi_communicator_t *comm,
-                          mca_coll_base_module_t *module)
-{
-    struct iovec iov;
-    mca_coll_sm_module_t *sm_module = (mca_coll_sm_module_t*) module;
-    mca_coll_sm_comm_t *data = sm_module->sm_comm_data;
-    int ret, rank, size;
-    int flag_num, segment_num, max_segment_num;
-    size_t total_size, max_data, bytes;
-    mca_coll_sm_in_use_flag_t *flag;
-    mca_coll_sm_data_index_t *index;
-    size_t ddt_size, segsize;
-    size_t segment_ddt_count, segment_ddt_bytes, zero = 0;
-    ptrdiff_t extent, gap;
-
-    /* Setup some identities */
-
-    rank = ompi_comm_rank(comm);
-    size = ompi_comm_size(comm);
-
-    /* Figure out how much we should have the convertor copy.  We need
-       to have it be in units of a datatype -- i.e., we only want to
-       copy a whole datatype worth of data or none at all (we've
-       already guaranteed above that the datatype is not larger than a
-       segment, so we'll at least get 1). */
-
-    /* ddt_size is the packed size (e.g., MPI_SHORT_INT is 6) */
-    ompi_datatype_type_size(dtype, &ddt_size);
-    /* extent is from lb to ub (e.g., MPI_SHORT_INT is 8) */
-    ompi_datatype_type_extent(dtype, &extent);
-    segment_ddt_count = mca_coll_sm_component.sm_fragment_size / ddt_size;
-    iov.iov_len = segment_ddt_bytes = segment_ddt_count * ddt_size;
-    total_size = ddt_size * count;
-
-    bytes = 0;
-
-    /* Only have one top-level decision as to whether I'm the root or
-       not.  Do this at the slight expense of repeating a little logic
-       -- but it's better than a conditional branch in every loop
-       iteration. */
-
-    /*********************************************************************
-     * Root
-     *********************************************************************/
-
-    if (root == rank) {
-        opal_convertor_t rtb_convertor, rbuf_convertor;
-        char *reduce_temp_buffer, *free_buffer, *reduce_target;
-        char *inplace_temp;
-        int peer;
-        size_t count_left = (size_t)count;
-        int frag_num = 0;
-        bool first_operation = true;
-
-        /* If the datatype is the same packed as it is unpacked, we
-           can save a memory copy and just do the reduction operation
-           directly from the shared memory segment.  However, if the
-           representation is not the same, then we need to get a
-           receive convertor and a temporary buffer to receive
-           into. */
-
-        if (ompi_datatype_is_contiguous_memory_layout(dtype, count)) {
-            reduce_temp_buffer = free_buffer = NULL;
-        } else {
-            /* When we have a non-contiguous datatype, we need one or
-             * two convertors:
-             *
-             * rtb_convertor: unpacking from the shmem to the
-             * reduce_temp_buffer (where we can then apply the
-             * reduction).
-             *
-             * rbuf_convertor: unpacking from the shmem directly to the
-             * rbuf (no need to go to the reduce_temp_buffer first and
-             * then apply the reduction -- just copy straight to the
-             * target buffer).
-             */
-            OBJ_CONSTRUCT(&rtb_convertor, opal_convertor_t);
-            OBJ_CONSTRUCT(&rbuf_convertor, opal_convertor_t);
-
-            /* See lengthy comment in coll basic reduce about
-               explanation for how to malloc the extra buffer.  Note
-               that we do not need a buffer big enough to hold "count"
-               instances of the datatype (i.e., big enough to hold the
-               entire user buffer) -- we only need to be able to hold
-               "segment_ddt_count" instances (i.e., the number of
-               instances that can be held in a single fragment) */
-
-            segsize = opal_datatype_span(&dtype->super, segment_ddt_count, &gap);
-
-            free_buffer = (char*)malloc(segsize);
-            if (NULL == free_buffer) {
-                return OMPI_ERR_OUT_OF_RESOURCE;
-            }
-            reduce_temp_buffer = free_buffer - gap;
-
-            /* Trickery here: we use a potentially smaller count than
-               the user count -- use the largest count that is <=
-               user's count that will fit within a single segment. */
-
-            if (OMPI_SUCCESS !=
-                (ret = opal_convertor_copy_and_prepare_for_recv(
-                                       ompi_mpi_local_convertor,
-                                       &(dtype->super),
-                                       segment_ddt_count,
-                                       reduce_temp_buffer,
-                                       0,
-                                       &rtb_convertor))) {
-                free(free_buffer);
-                return ret;
-            }
-
-            /* See if we need the rbuf_convertor */
-            if (size - 1 != rank) {
-                if (OMPI_SUCCESS !=
-                    (ret = opal_convertor_copy_and_prepare_for_recv(
-                                       ompi_mpi_local_convertor,
-                                       &(dtype->super),
-                                       count,
-                                       rbuf,
-                                       0,
-                                       &rbuf_convertor))) {
-                    free(free_buffer);
-                    return ret;
-                }
-            }
-        }
-
-        /* If we're a) doing MPI_IN_PLACE (which means we're the root
-           -- wouldn't have gotten down here with MPI_IN_PLACE if we
-           weren't the root), and b) we're not rank (size-1), then we
-           need to copy the rbuf into a temporary buffer and use that
-           as the sbuf */
-
-        if (MPI_IN_PLACE == sbuf && (size - 1) != rank) {
-            segsize = opal_datatype_span(&dtype->super, count, &gap);
-            inplace_temp = (char*)malloc(segsize);
-            if (NULL == inplace_temp) {
-                if (NULL != free_buffer) {
-                    free(free_buffer);
-                }
-                return OMPI_ERR_OUT_OF_RESOURCE;
-            }
-            sbuf = inplace_temp - gap;
-            ompi_datatype_copy_content_same_ddt(dtype, count, (char *)sbuf, (char *)rbuf);
-        } else {
-            inplace_temp = NULL;
-        }
-
-        /* Main loop over receiving / reducing fragments */
-
-        do {
-            flag_num = (data->mcb_operation_count %
-                        mca_coll_sm_component.sm_comm_num_in_use_flags);
-            FLAG_SETUP(flag_num, flag, data);
-            FLAG_WAIT_FOR_IDLE(flag, reduce_root_flag_label);
-            FLAG_RETAIN(flag, size, data->mcb_operation_count);
-            ++data->mcb_operation_count;
-
-            /* Loop over all the segments in this set */
-
-            segment_num =
-                flag_num * mca_coll_sm_component.sm_segs_per_inuse_flag;
-            max_segment_num =
-                (flag_num + 1) * mca_coll_sm_component.sm_segs_per_inuse_flag;
-            reduce_target = (((char*) rbuf) + (frag_num * extent * segment_ddt_count));
-            do {
-
-                /* Note that all the other coll modules reduce from
-                   process (size-1) to 0, so that's the order we'll do
-                   it here. */
-                /* Process (size-1) is the root (special case) */
-                if (size - 1 == rank) {
-                    /* If we're the root *and* the first process to be
-                       combined *and* this is the first segment in the
-                       entire algorithm, then just copy the whole sbuf
-                       to rbuf.  That way, we never need to copy from
-                       my sbuf again (i.e., do the copy all at once
-                       since all the data is local, and then don't
-                       worry about it for the rest of the
-                       algorithm) */
-                    if (first_operation) {
-                        first_operation = false;
-                        if (MPI_IN_PLACE != sbuf) {
-                            ompi_datatype_copy_content_same_ddt(dtype, count,
-                                               reduce_target, (char*)sbuf);
-                        }
-                    }
-                }
-
-                /* Process (size-1) is not the root */
-                else {
-                    /* Wait for the data to be copied into shmem, just
-                       like any other non-root process */
-                    index = &(data->mcb_data_index[segment_num]);
-                    PARENT_WAIT_FOR_NOTIFY_SPECIFIC(size - 1, rank, index, max_data, reduce_root_parent_label1);
-
-                    /* If the datatype is contiguous, just copy it
-                       straight to the reduce_target */
-                    if (NULL == free_buffer) {
-                        memcpy(reduce_target, ((char*)index->mcbmi_data) +
-                               (size - 1) * mca_coll_sm_component.sm_fragment_size, max_data);
-                    }
-                    /* If the datatype is noncontiguous, use the
-                       rbuf_convertor to unpack it straight to the
-                       rbuf */
-                    else {
-                        max_data = segment_ddt_bytes;
-                        COPY_FRAGMENT_OUT(rbuf_convertor, size - 1, index,
-                                          iov, max_data);
-                    }
-                }
-
-                /* Loop over all the remaining processes, receiving
-                   and reducing them in order */
-
-                for (peer = size - 2; peer >= 0; --peer) {
-
-                    /* Handle the case where the source is this
-                       process (which, by definition, excludes the
-                       sbuf_copied_to_rbuf case because that can
-                       *only* happen when root==0).  In this case, we
-                       don't need to wait for the peer (i.e., me) to
-                       copy into shmem -- just reduce directly from my
-                       sbuf. */
-                    if (rank == peer) {
-                        ompi_op_reduce(op,
-                                       ((char *) sbuf) +
-                                       frag_num * extent * segment_ddt_count,
-                                       reduce_target,
-                                       min(count_left, segment_ddt_count),
-                                       dtype);
-                    }
-
-                    /* Now handle the case where the source is not
-                       this process.  Wait for the process to copy to
-                       the segment into shmem. */
-                    else {
-                        index = &(data->mcb_data_index[segment_num]);
-                        PARENT_WAIT_FOR_NOTIFY_SPECIFIC(peer, rank,
-                                                        index, max_data, reduce_root_parent_label2);
-
-                        /* If we don't need an extra buffer, then do the
-                           reduction operation on the fragment straight
-                           from the shmem. */
-
-                        if (NULL == free_buffer) {
-                            ompi_op_reduce(op,
-                                           (index->mcbmi_data +
-                                            (peer * mca_coll_sm_component.sm_fragment_size)),
-                                           reduce_target,
-                                           min(count_left, segment_ddt_count),
-                                           dtype);
-                        }
-
-                        /* Otherwise, unpack the fragment to the temporary
-                           buffer and then do the reduction from there */
-
-                        else {
-                            /* Unpack the fragment into my temporary
-                               buffer */
-                            max_data = segment_ddt_bytes;
-                            COPY_FRAGMENT_OUT(rtb_convertor, peer, index,
-                                              iov, max_data);
-                            opal_convertor_set_position(&rtb_convertor, &zero);
-
-                            /* Do the reduction on this fragment */
-                            ompi_op_reduce(op, reduce_temp_buffer,
-                                           reduce_target,
-                                           min(count_left, segment_ddt_count),
-                                           dtype);
-                        }
-                    } /* whether this process was me or not */
-                } /* loop over all proceses */
-
-                /* We've iterated through all the processes -- now we
-                   move on to the next segment */
-
-                count_left -= segment_ddt_count;
-                bytes += segment_ddt_bytes;
-                ++segment_num;
-                ++frag_num;
-                reduce_target += extent * segment_ddt_count;
-            } while (bytes < total_size && segment_num < max_segment_num);
-
-            /* Root is now done with this set of segments */
-            FLAG_RELEASE(flag);
-        } while (bytes < total_size);
-
-        /* Kill the convertor, if we had one */
-
-        if (NULL != free_buffer) {
-            OBJ_DESTRUCT(&rtb_convertor);
-            OBJ_DESTRUCT(&rbuf_convertor);
-            free(free_buffer);
-        }
-        if (NULL != inplace_temp) {
-            free(inplace_temp);
-        }
-    }
-
-    /*********************************************************************
-     * Non-root
-     *********************************************************************/
-
-    else {
-        /* Here we get a convertor for the full count that the user
-           provided (as opposed to the convertor that the root got) */
-
-        opal_convertor_t sbuf_convertor;
-        OBJ_CONSTRUCT(&sbuf_convertor, opal_convertor_t);
-        if (OMPI_SUCCESS !=
-            (ret =
-             opal_convertor_copy_and_prepare_for_send(ompi_mpi_local_convertor,
-                                                      &(dtype->super),
-                                                      count,
-                                                      sbuf,
-                                                      0,
-                                                      &sbuf_convertor))) {
-            return ret;
-        }
-
-        /* Loop over sending fragments to the root */
-
-        do {
-            flag_num = (data->mcb_operation_count %
-                        mca_coll_sm_component.sm_comm_num_in_use_flags);
-
-            /* Wait for the root to mark this set of segments as
-               ours */
-            FLAG_SETUP(flag_num, flag, data);
-            FLAG_WAIT_FOR_OP(flag, data->mcb_operation_count, reduce_nonroot_flag_label);
-            ++data->mcb_operation_count;
-
-            /* Loop over all the segments in this set */
-
-            segment_num =
-                flag_num * mca_coll_sm_component.sm_segs_per_inuse_flag;
-            max_segment_num =
-                (flag_num + 1) * mca_coll_sm_component.sm_segs_per_inuse_flag;
-            do {
-                index = &(data->mcb_data_index[segment_num]);
-
-                /* Copy from the user's buffer to my shared mem
-                   segment */
-                max_data = segment_ddt_bytes;
-                COPY_FRAGMENT_IN(sbuf_convertor, index, rank, iov, max_data);
-                bytes += max_data;
-
-                /* Wait for the write to absolutely complete */
-                opal_atomic_wmb();
-
-                /* Tell my parent (always the reduction root -- we're
-                   ignoring the mcb_tree parent/child relationships
-                   here) that this fragment is ready */
-                CHILD_NOTIFY_PARENT(rank, root, index, max_data);
-
-                ++segment_num;
-            } while (bytes < total_size && segment_num < max_segment_num);
-
-            /* We're finished with this set of segments */
-            FLAG_RELEASE(flag);
-        } while (bytes < total_size);
-
-        /* Kill the convertor */
-
-        OBJ_DESTRUCT(&sbuf_convertor);
-    }
-
-    /* All done */
-
-    return OMPI_SUCCESS;
-}
-
-
-#if WANT_REDUCE_NO_ORDER
-/**
- * Unordered shared memory reduction.
- *
- * This function performs the reduction in whatever order the operands
- * arrive.
- */
-static int reduce_no_order(const void *sbuf, void* rbuf, int count,
-                           struct ompi_datatype_t *dtype,
-                           struct ompi_op_t *op,
-                           int root, struct ompi_communicator_t *comm,
-                           mca_coll_base_module_t *module)
-{
-    return OMPI_ERR_NOT_IMPLEMENTED;
-}
-#endif
diff --git a/ompi/mca/coll/sm/coll_sm_reduce_scatter.c b/ompi/mca/coll/sm/coll_sm_reduce_scatter.c
deleted file mode 100644
index 0ef35a0d896..00000000000
--- a/ompi/mca/coll/sm/coll_sm_reduce_scatter.c
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/constants.h"
-#include "coll_sm.h"
-
-
-/*
- *	reduce_scatter
- *
- *	Function:	- reduce then scatter
- *	Accepts:	- same as MPI_Reduce_scatter()
- *	Returns:	- MPI_SUCCESS or error code
- */
-int mca_coll_sm_reduce_scatter_intra(const void *sbuf, void *rbuf, const int *rcounts,
-                                     struct ompi_datatype_t *dtype,
-                                     struct ompi_op_t *op,
-                                     struct ompi_communicator_t *comm,
-                                     mca_coll_base_module_t *module)
-{
-    return OMPI_ERR_NOT_IMPLEMENTED;
-}
diff --git a/ompi/mca/coll/sm/coll_sm_scan.c b/ompi/mca/coll/sm/coll_sm_scan.c
deleted file mode 100644
index 4a2ce3e6001..00000000000
--- a/ompi/mca/coll/sm/coll_sm_scan.c
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/constants.h"
-#include "coll_sm.h"
-
-
-/*
- *	scan
- *
- *	Function:	- basic scan operation
- *	Accepts:	- same arguments as MPI_Scan()
- *	Returns:	- MPI_SUCCESS or error code
- */
-int mca_coll_sm_scan_intra(const void *sbuf, void *rbuf, int count,
-                           struct ompi_datatype_t *dtype,
-                           struct ompi_op_t *op,
-                           struct ompi_communicator_t *comm,
-                           mca_coll_base_module_t *module)
-{
-    return OMPI_ERR_NOT_IMPLEMENTED;
-}
diff --git a/ompi/mca/coll/sm/coll_sm_scatter.c b/ompi/mca/coll/sm/coll_sm_scatter.c
deleted file mode 100644
index 3b7da5550ad..00000000000
--- a/ompi/mca/coll/sm/coll_sm_scatter.c
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/constants.h"
-#include "coll_sm.h"
-
-
-/*
- *      scatter
- *
- *      Function:       - shared memory reduce
- *      Accepts:        - same as MPI_Scatter()
- *      Returns:        - MPI_SUCCESS or error code
- */
-int mca_coll_sm_scatter_intra(const void *sbuf, int scount,
-                              struct ompi_datatype_t *sdtype, void *rbuf,
-                              int rcount, struct ompi_datatype_t *rdtype,
-                              int root, struct ompi_communicator_t *comm,
-                              mca_coll_base_module_t *module)
-{
-    return OMPI_ERR_NOT_IMPLEMENTED;
-}
diff --git a/ompi/mca/coll/sm/coll_sm_scatterv.c b/ompi/mca/coll/sm/coll_sm_scatterv.c
deleted file mode 100644
index ae6e08a687c..00000000000
--- a/ompi/mca/coll/sm/coll_sm_scatterv.c
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/constants.h"
-#include "coll_sm.h"
-
-
-/*
- *	scatterv_intra
- *
- *	Function:	- scatterv operation
- *	Accepts:	- same arguments as MPI_Scatterv()
- *	Returns:	- MPI_SUCCESS or error code
- */
-int mca_coll_sm_scatterv_intra(const void *sbuf, const int *scounts,
-                               const int *disps, struct ompi_datatype_t *sdtype,
-                               void *rbuf, int rcount,
-                               struct ompi_datatype_t *rdtype, int root,
-                               struct ompi_communicator_t *comm,
-                               mca_coll_base_module_t *module)
-{
-    return OMPI_ERR_NOT_IMPLEMENTED;
-}
diff --git a/ompi/mca/coll/sm/help-mpi-coll-sm.txt b/ompi/mca/coll/sm/help-mpi-coll-sm.txt
deleted file mode 100644
index ce42bb65dfb..00000000000
--- a/ompi/mca/coll/sm/help-mpi-coll-sm.txt
+++ /dev/null
@@ -1,36 +0,0 @@
-# -*- text -*-
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-# This is the US/English general help file for Open MPI's Shared memory
-# collective component.
-#
-[tree-degree-larger-than-control]
-The specified shared memory collective tree degree
-(coll_sm_tree_degree = %d) is too large.  It must be less than or
-equal to the control size (coll_sm_control_size = %d).
-
-Automatically adjusting the tree degree to be equal to the control
-size and continuing...
-#
-[tree-degree-larger-than-255]
-The specified shared memory collective tree degree
-(coll_sm_tree_degree = %d) is too large.  It must be less than or
-equal to 255.
-
-Automatically adjusting the tree degree to be 255 and continuing...
-#
diff --git a/ompi/mca/coll/sm/memory-layout.ppt b/ompi/mca/coll/sm/memory-layout.ppt
deleted file mode 100644
index 6151678ac27..00000000000
Binary files a/ompi/mca/coll/sm/memory-layout.ppt and /dev/null differ
diff --git a/ompi/mca/coll/sm/owner.txt b/ompi/mca/coll/sm/owner.txt
deleted file mode 100644
index 29b7476009c..00000000000
--- a/ompi/mca/coll/sm/owner.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-#
-# owner/status file
-# owner: institution that is responsible for this package
-# status: e.g. active, maintenance, unmaintained
-#
-owner: nobody
-status: unmaintained
diff --git a/ompi/mca/coll/sync/Makefile.am b/ompi/mca/coll/sync/Makefile.am
index 2f75cd2dfa5..dad4f8f7138 100644
--- a/ompi/mca/coll/sync/Makefile.am
+++ b/ompi/mca/coll/sync/Makefile.am
@@ -12,6 +12,7 @@
 # Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2016      Intel, Inc. All rights reserved
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -19,8 +20,6 @@
 # $HEADER$
 #
 
-dist_ompidata_DATA = help-coll-sync.txt
-
 sources = \
         coll_sync.h \
         coll_sync_component.c \
diff --git a/ompi/mca/coll/sync/coll_sync.h b/ompi/mca/coll/sync/coll_sync.h
index ae333903c7e..9aed3f103ac 100644
--- a/ompi/mca/coll/sync/coll_sync.h
+++ b/ompi/mca/coll/sync/coll_sync.h
@@ -10,6 +10,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2008-2009 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -43,41 +44,38 @@ mca_coll_base_module_t
 *mca_coll_sync_comm_query(struct ompi_communicator_t *comm,
                           int *priority);
 
-int mca_coll_sync_module_enable(mca_coll_base_module_t *module,
-                                struct ompi_communicator_t *comm);
-
 int mca_coll_sync_barrier(struct ompi_communicator_t *comm,
                           mca_coll_base_module_t *module);
 
-int mca_coll_sync_bcast(void *buff, int count,
+int mca_coll_sync_bcast(void *buff, size_t count,
                         struct ompi_datatype_t *datatype,
                         int root,
                         struct ompi_communicator_t *comm,
                         mca_coll_base_module_t *module);
 
-int mca_coll_sync_exscan(const void *sbuf, void *rbuf, int count,
+int mca_coll_sync_exscan(const void *sbuf, void *rbuf, size_t count,
                          struct ompi_datatype_t *dtype,
                          struct ompi_op_t *op,
                          struct ompi_communicator_t *comm,
                          mca_coll_base_module_t *module);
 
-int mca_coll_sync_gather(const void *sbuf, int scount,
+int mca_coll_sync_gather(const void *sbuf, size_t scount,
                          struct ompi_datatype_t *sdtype,
-                         void *rbuf, int rcount,
+                         void *rbuf, size_t rcount,
                          struct ompi_datatype_t *rdtype,
                          int root,
                          struct ompi_communicator_t *comm,
                          mca_coll_base_module_t *module);
 
-int mca_coll_sync_gatherv(const void *sbuf, int scount,
+int mca_coll_sync_gatherv(const void *sbuf, size_t scount,
                           struct ompi_datatype_t *sdtype,
-                          void *rbuf, const int *rcounts, const int *disps,
+                          void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
                           struct ompi_datatype_t *rdtype,
                           int root,
                           struct ompi_communicator_t *comm,
                           mca_coll_base_module_t *module);
 
-int mca_coll_sync_reduce(const void *sbuf, void *rbuf, int count,
+int mca_coll_sync_reduce(const void *sbuf, void *rbuf, size_t count,
                          struct ompi_datatype_t *dtype,
                          struct ompi_op_t *op,
                          int root,
@@ -85,29 +83,29 @@ int mca_coll_sync_reduce(const void *sbuf, void *rbuf, int count,
                          mca_coll_base_module_t *module);
 
 int mca_coll_sync_reduce_scatter(const void *sbuf, void *rbuf,
-                                 const int *rcounts,
+                                 ompi_count_array_t rcounts,
                                  struct ompi_datatype_t *dtype,
                                  struct ompi_op_t *op,
                                  struct ompi_communicator_t *comm,
                                  mca_coll_base_module_t *module);
 
-int mca_coll_sync_scan(const void *sbuf, void *rbuf, int count,
+int mca_coll_sync_scan(const void *sbuf, void *rbuf, size_t count,
                        struct ompi_datatype_t *dtype,
                        struct ompi_op_t *op,
                        struct ompi_communicator_t *comm,
                        mca_coll_base_module_t *module);
 
-int mca_coll_sync_scatter(const void *sbuf, int scount,
+int mca_coll_sync_scatter(const void *sbuf, size_t scount,
                           struct ompi_datatype_t *sdtype,
-                          void *rbuf, int rcount,
+                          void *rbuf, size_t rcount,
                           struct ompi_datatype_t *rdtype,
                           int root,
                           struct ompi_communicator_t *comm,
                           mca_coll_base_module_t *module);
 
-int mca_coll_sync_scatterv(const void *sbuf, const int *scounts, const int *disps,
+int mca_coll_sync_scatterv(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t disps,
                            struct ompi_datatype_t *sdtype,
-                           void *rbuf, int rcount,
+                           void *rbuf, size_t rcount,
                            struct ompi_datatype_t *rdtype,
                            int root,
                            struct ompi_communicator_t *comm,
@@ -137,7 +135,7 @@ OBJ_CLASS_DECLARATION(mca_coll_sync_module_t);
 /* Component */
 
 typedef struct mca_coll_sync_component_t {
-    mca_coll_base_component_2_4_0_t super;
+    mca_coll_base_component_3_0_0_t super;
 
     /* Priority of this component */
     int priority;
@@ -151,7 +149,7 @@ typedef struct mca_coll_sync_component_t {
 
 /* Globally exported variables */
 
-OMPI_MODULE_DECLSPEC extern mca_coll_sync_component_t mca_coll_sync_component;
+OMPI_DECLSPEC extern mca_coll_sync_component_t mca_coll_sync_component;
 
 /* Macro used in most of the collectives */
 
diff --git a/ompi/mca/coll/sync/coll_sync_bcast.c b/ompi/mca/coll/sync/coll_sync_bcast.c
index 5bca949a128..73435361870 100644
--- a/ompi/mca/coll/sync/coll_sync_bcast.c
+++ b/ompi/mca/coll/sync/coll_sync_bcast.c
@@ -30,7 +30,7 @@
  *	Accepts:	- same arguments as MPI_Bcast()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_sync_bcast(void *buff, int count,
+int mca_coll_sync_bcast(void *buff, size_t count,
                         struct ompi_datatype_t *datatype, int root,
                         struct ompi_communicator_t *comm,
                         mca_coll_base_module_t *module)
diff --git a/ompi/mca/coll/sync/coll_sync_component.c b/ompi/mca/coll/sync/coll_sync_component.c
index 9263a0451e3..92d9fcd25c4 100644
--- a/ompi/mca/coll/sync/coll_sync_component.c
+++ b/ompi/mca/coll/sync/coll_sync_component.c
@@ -49,7 +49,7 @@ mca_coll_sync_component_t mca_coll_sync_component = {
          * about the component itself */
 
        .collm_version = {
-            MCA_COLL_BASE_VERSION_2_4_0,
+            MCA_COLL_BASE_VERSION_3_0_0,
 
             /* Component name and version */
             .mca_component_name = "sync",
diff --git a/ompi/mca/coll/sync/coll_sync_exscan.c b/ompi/mca/coll/sync/coll_sync_exscan.c
index b2a3648b07a..6bc7bdb778a 100644
--- a/ompi/mca/coll/sync/coll_sync_exscan.c
+++ b/ompi/mca/coll/sync/coll_sync_exscan.c
@@ -29,7 +29,7 @@
  *	Accepts:	- same arguments as MPI_Exscan()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_sync_exscan(const void *sbuf, void *rbuf, int count,
+int mca_coll_sync_exscan(const void *sbuf, void *rbuf, size_t count,
                          struct ompi_datatype_t *dtype,
                          struct ompi_op_t *op,
                          struct ompi_communicator_t *comm,
diff --git a/ompi/mca/coll/sync/coll_sync_gather.c b/ompi/mca/coll/sync/coll_sync_gather.c
index 05817bd0234..1a6d6b76caa 100644
--- a/ompi/mca/coll/sync/coll_sync_gather.c
+++ b/ompi/mca/coll/sync/coll_sync_gather.c
@@ -29,9 +29,9 @@
  *	Accepts:	- same arguments as MPI_Gather()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_sync_gather(const void *sbuf, int scount,
+int mca_coll_sync_gather(const void *sbuf, size_t scount,
                          struct ompi_datatype_t *sdtype,
-                         void *rbuf, int rcount,
+                         void *rbuf, size_t rcount,
                          struct ompi_datatype_t *rdtype,
                          int root, struct ompi_communicator_t *comm,
                          mca_coll_base_module_t *module)
diff --git a/ompi/mca/coll/sync/coll_sync_gatherv.c b/ompi/mca/coll/sync/coll_sync_gatherv.c
index 8271c92fb0d..88dacd7415a 100644
--- a/ompi/mca/coll/sync/coll_sync_gatherv.c
+++ b/ompi/mca/coll/sync/coll_sync_gatherv.c
@@ -29,9 +29,9 @@
  *	Accepts:	- same arguments as MPI_Gatherv()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_sync_gatherv(const void *sbuf, int scount,
+int mca_coll_sync_gatherv(const void *sbuf, size_t scount,
                           struct ompi_datatype_t *sdtype,
-                          void *rbuf, const int *rcounts, const int *disps,
+                          void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
                           struct ompi_datatype_t *rdtype, int root,
                           struct ompi_communicator_t *comm,
                           mca_coll_base_module_t *module)
diff --git a/ompi/mca/coll/sync/coll_sync_module.c b/ompi/mca/coll/sync/coll_sync_module.c
index 128cb8c3fa9..65b32f1ba16 100644
--- a/ompi/mca/coll/sync/coll_sync_module.c
+++ b/ompi/mca/coll/sync/coll_sync_module.c
@@ -13,6 +13,7 @@
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2018-2019 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -40,6 +41,12 @@
 #include "ompi/mca/coll/base/base.h"
 #include "coll_sync.h"
 
+static int
+mca_coll_sync_module_enable(mca_coll_base_module_t *module,
+                            struct ompi_communicator_t *comm);
+static int
+mca_coll_sync_module_disable(mca_coll_base_module_t *module,
+                             struct ompi_communicator_t *comm);
 
 static void mca_coll_sync_module_construct(mca_coll_sync_module_t *module)
 {
@@ -111,16 +118,10 @@ mca_coll_sync_comm_query(struct ompi_communicator_t *comm,
 
     /* Choose whether to use [intra|inter] */
     sync_module->super.coll_module_enable = mca_coll_sync_module_enable;
+    sync_module->super.coll_module_disable = mca_coll_sync_module_disable;
 
     /* The "all" versions are already synchronous.  So no need for an
        additional barrier there. */
-    sync_module->super.coll_allgather  = NULL;
-    sync_module->super.coll_allgatherv = NULL;
-    sync_module->super.coll_allreduce  = NULL;
-    sync_module->super.coll_alltoall   = NULL;
-    sync_module->super.coll_alltoallv  = NULL;
-    sync_module->super.coll_alltoallw  = NULL;
-    sync_module->super.coll_barrier    = NULL;
     sync_module->super.coll_bcast      = mca_coll_sync_bcast;
     sync_module->super.coll_exscan     = mca_coll_sync_exscan;
     sync_module->super.coll_gather     = mca_coll_sync_gather;
@@ -134,47 +135,78 @@ mca_coll_sync_comm_query(struct ompi_communicator_t *comm,
     return &(sync_module->super);
 }
 
+#define SYNC_INSTALL_COLL_API(__comm, __module, __api)                                                                   \
+    do                                                                                                                   \
+    {                                                                                                                    \
+        /* save the current selected collective */                                                                       \
+        MCA_COLL_SAVE_API(__comm, __api, __module->c_coll.coll_##__api, __module->c_coll.coll_##__api##_module, "sync"); \
+        /* install our own */                                                                                            \
+        MCA_COLL_INSTALL_API(__comm, __api, mca_coll_sync_##__api, &__module->super, "sync");                            \
+    } while (0)
+
+#define SYNC_UNINSTALL_COLL_API(__comm, __module, __api)                                                                \
+    do                                                                                                                  \
+    {                                                                                                                   \
+        if (__comm->c_coll->coll_##__api##_module == &__module->super)                                                  \
+        {                                                                                                               \
+            MCA_COLL_INSTALL_API(__comm, __api, mca_coll_sync_##__api, __module->c_coll.coll_##__api##_module, "sync"); \
+            __module->c_coll.coll_##__api = NULL;                                                                       \
+            __module->c_coll.coll_##__api##_module = NULL;                                                              \
+        }                                                                                                               \
+    } while (0)
 
 /*
  * Init module on the communicator
  */
-int mca_coll_sync_module_enable(mca_coll_base_module_t *module,
-                                struct ompi_communicator_t *comm)
+static int
+mca_coll_sync_module_enable(mca_coll_base_module_t *module,
+                            struct ompi_communicator_t *comm)
 {
-    bool good = true;
-    char *msg = NULL;
-    mca_coll_sync_module_t *s = (mca_coll_sync_module_t*) module;
+    mca_coll_sync_module_t *sync_module = (mca_coll_sync_module_t*) module;
 
-    /* Save the prior layer of coll functions */
-    s->c_coll = *comm->c_coll;
-
-#define CHECK_AND_RETAIN(name)                           \
-    if (NULL == s->c_coll.coll_ ## name ## _module) {    \
-        good = false;                                    \
-        msg = #name;                                     \
-    } else if (good) {                                   \
-        OBJ_RETAIN(s->c_coll.coll_ ## name ## _module);  \
-    }
+    /* The "all" versions are already synchronous.  So no need for an
+       additional barrier there. */
+    SYNC_INSTALL_COLL_API(comm, sync_module, bcast);
+    SYNC_INSTALL_COLL_API(comm, sync_module, gather);
+    SYNC_INSTALL_COLL_API(comm, sync_module, gatherv);
+    SYNC_INSTALL_COLL_API(comm, sync_module, reduce);
+    SYNC_INSTALL_COLL_API(comm, sync_module, reduce_scatter);
+    SYNC_INSTALL_COLL_API(comm, sync_module, scatter);
+    SYNC_INSTALL_COLL_API(comm, sync_module, scatterv);
 
-    CHECK_AND_RETAIN(bcast);
-    CHECK_AND_RETAIN(gather);
-    CHECK_AND_RETAIN(gatherv);
-    CHECK_AND_RETAIN(reduce);
-    CHECK_AND_RETAIN(reduce_scatter);
-    CHECK_AND_RETAIN(scatter);
-    CHECK_AND_RETAIN(scatterv);
     if (!OMPI_COMM_IS_INTER(comm)) {
         /* MPI does not define scan/exscan on intercommunicators */
-        CHECK_AND_RETAIN(exscan);
-        CHECK_AND_RETAIN(scan);
+        SYNC_INSTALL_COLL_API(comm, sync_module, scan);
+        SYNC_INSTALL_COLL_API(comm, sync_module, exscan);
     }
 
-    /* All done */
-    if (good) {
-        return OMPI_SUCCESS;
+    return OMPI_SUCCESS;
+}
+
+static int
+mca_coll_sync_module_disable(mca_coll_base_module_t *module,
+                             struct ompi_communicator_t *comm)
+{
+    mca_coll_sync_module_t *sync_module = (mca_coll_sync_module_t*) module;
+
+    /* Save the prior layer of coll functions */
+    sync_module->c_coll = *comm->c_coll;
+
+    /* The "all" versions are already synchronous.  So no need for an
+       additional barrier there. */
+    SYNC_UNINSTALL_COLL_API(comm, sync_module, bcast);
+    SYNC_UNINSTALL_COLL_API(comm, sync_module, gather);
+    SYNC_UNINSTALL_COLL_API(comm, sync_module, gatherv);
+    SYNC_UNINSTALL_COLL_API(comm, sync_module, reduce);
+    SYNC_UNINSTALL_COLL_API(comm, sync_module, reduce_scatter);
+    SYNC_UNINSTALL_COLL_API(comm, sync_module, scatter);
+    SYNC_UNINSTALL_COLL_API(comm, sync_module, scatterv);
+
+    if (!OMPI_COMM_IS_INTER(comm)) {
+        /* MPI does not define scan/exscan on intercommunicators */
+        SYNC_INSTALL_COLL_API(comm, sync_module, scan);
+        SYNC_INSTALL_COLL_API(comm, sync_module, exscan);
     }
-    opal_show_help("help-coll-sync.txt", "missing collective", true,
-                   ompi_process_info.nodename,
-                   mca_coll_sync_component.priority, msg);
-    return OMPI_ERR_NOT_FOUND;
+
+    return OMPI_SUCCESS;
 }
diff --git a/ompi/mca/coll/sync/coll_sync_reduce.c b/ompi/mca/coll/sync/coll_sync_reduce.c
index 82d701d2395..17d99bc8711 100644
--- a/ompi/mca/coll/sync/coll_sync_reduce.c
+++ b/ompi/mca/coll/sync/coll_sync_reduce.c
@@ -28,7 +28,7 @@
  *	Accepts:	- same as MPI_Reduce()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_sync_reduce(const void *sbuf, void *rbuf, int count,
+int mca_coll_sync_reduce(const void *sbuf, void *rbuf, size_t count,
                          struct ompi_datatype_t *dtype,
                          struct ompi_op_t *op,
                          int root, struct ompi_communicator_t *comm,
diff --git a/ompi/mca/coll/sync/coll_sync_reduce_scatter.c b/ompi/mca/coll/sync/coll_sync_reduce_scatter.c
index 459daac74a8..51a05b60387 100644
--- a/ompi/mca/coll/sync/coll_sync_reduce_scatter.c
+++ b/ompi/mca/coll/sync/coll_sync_reduce_scatter.c
@@ -30,7 +30,7 @@
  *	Accepts:	- same as MPI_Reduce_scatter()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_sync_reduce_scatter(const void *sbuf, void *rbuf, const int *rcounts,
+int mca_coll_sync_reduce_scatter(const void *sbuf, void *rbuf, ompi_count_array_t rcounts,
                                  struct ompi_datatype_t *dtype,
                                  struct ompi_op_t *op,
                                  struct ompi_communicator_t *comm,
diff --git a/ompi/mca/coll/sync/coll_sync_scan.c b/ompi/mca/coll/sync/coll_sync_scan.c
index f0983a9e6aa..8e185c6a469 100644
--- a/ompi/mca/coll/sync/coll_sync_scan.c
+++ b/ompi/mca/coll/sync/coll_sync_scan.c
@@ -28,7 +28,7 @@
  *	Accepts:	- same arguments as MPI_Scan()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_sync_scan(const void *sbuf, void *rbuf, int count,
+int mca_coll_sync_scan(const void *sbuf, void *rbuf, size_t count,
                        struct ompi_datatype_t *dtype,
                        struct ompi_op_t *op,
                        struct ompi_communicator_t *comm,
diff --git a/ompi/mca/coll/sync/coll_sync_scatter.c b/ompi/mca/coll/sync/coll_sync_scatter.c
index 19a3d80441a..89619abb14f 100644
--- a/ompi/mca/coll/sync/coll_sync_scatter.c
+++ b/ompi/mca/coll/sync/coll_sync_scatter.c
@@ -29,9 +29,9 @@
  *	Accepts:	- same arguments as MPI_Scatter()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_sync_scatter(const void *sbuf, int scount,
+int mca_coll_sync_scatter(const void *sbuf, size_t scount,
                           struct ompi_datatype_t *sdtype,
-                          void *rbuf, int rcount,
+                          void *rbuf, size_t rcount,
                           struct ompi_datatype_t *rdtype,
                           int root, struct ompi_communicator_t *comm,
                           mca_coll_base_module_t *module)
diff --git a/ompi/mca/coll/sync/coll_sync_scatterv.c b/ompi/mca/coll/sync/coll_sync_scatterv.c
index 5aba9831766..b0e182bf42c 100644
--- a/ompi/mca/coll/sync/coll_sync_scatterv.c
+++ b/ompi/mca/coll/sync/coll_sync_scatterv.c
@@ -29,9 +29,9 @@
  *	Accepts:	- same arguments as MPI_Scatterv()
  *	Returns:	- MPI_SUCCESS or error code
  */
-int mca_coll_sync_scatterv(const void *sbuf, const int *scounts,
-                           const int *disps, struct ompi_datatype_t *sdtype,
-                           void *rbuf, int rcount,
+int mca_coll_sync_scatterv(const void *sbuf, ompi_count_array_t scounts,
+                           ompi_disp_array_t disps, struct ompi_datatype_t *sdtype,
+                           void *rbuf, size_t rcount,
                            struct ompi_datatype_t *rdtype, int root,
                            struct ompi_communicator_t *comm,
                            mca_coll_base_module_t *module)
diff --git a/ompi/mca/coll/sync/help-coll-sync.txt b/ompi/mca/coll/sync/help-coll-sync.txt
deleted file mode 100644
index 4a5c871207e..00000000000
--- a/ompi/mca/coll/sync/help-coll-sync.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-# -*- text -*-
-#
-# Copyright (c) 2009 Cisco Systems, Inc.  All rights reserved.
-# $COPYRIGHT$
-# 
-# Additional copyrights may follow
-# 
-# $HEADER$
-#
-# This is the US/English general help file for Open MPI's sync
-# collective component.
-#
-[missing collective]
-The sync collective component in Open MPI was activated on a
-communicator where it did not find an underlying collective operation
-defined.  This usually means that the sync collective module's
-priority was not set high enough.  Please try increasing sync's
-priority.
-
-  Local host: %s
-  Sync coll module priority: %d
-  First discovered missing collective: %s
diff --git a/ompi/mca/coll/tuned/coll_tuned.h b/ompi/mca/coll/tuned/coll_tuned.h
index 892a3319a32..bb5ed3f762a 100644
--- a/ompi/mca/coll/tuned/coll_tuned.h
+++ b/ompi/mca/coll/tuned/coll_tuned.h
@@ -6,6 +6,9 @@
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2019      Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2025      Amazon.com, Inc. or its affiliates.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -29,6 +32,8 @@
 
 BEGIN_C_DECLS
 
+#define COLL_TUNED_TRACING_VERBOSE 50
+
 /* these are the same across all modules and are loaded at component query time */
 extern int   ompi_coll_tuned_stream;
 extern int   ompi_coll_tuned_priority;
@@ -37,10 +42,6 @@ extern char* ompi_coll_tuned_dynamic_rules_filename;
 extern int   ompi_coll_tuned_init_tree_fanout;
 extern int   ompi_coll_tuned_init_chain_fanout;
 extern int   ompi_coll_tuned_init_max_requests;
-extern int   ompi_coll_tuned_alltoall_small_msg;
-extern int   ompi_coll_tuned_alltoall_intermediate_msg;
-extern int   ompi_coll_tuned_alltoall_large_msg;
-extern int   ompi_coll_tuned_alltoall_min_procs;
 extern int   ompi_coll_tuned_alltoall_max_requests;
 extern int   ompi_coll_tuned_scatter_intermediate_msg;
 extern int   ompi_coll_tuned_scatter_large_msg;
@@ -184,7 +185,7 @@ int ompi_coll_tuned_scan_intra_check_forced_init (coll_tuned_force_algorithm_mca
 
 struct mca_coll_tuned_component_t {
 	/** Base coll component */
-	mca_coll_base_component_2_4_0_t super;
+	mca_coll_base_component_3_0_0_t super;
 
 	/** MCA parameter: Priority of this component */
 	int tuned_priority;
@@ -204,7 +205,7 @@ typedef struct mca_coll_tuned_component_t mca_coll_tuned_component_t;
 /**
  * Global component instance
  */
-OMPI_MODULE_DECLSPEC extern mca_coll_tuned_component_t mca_coll_tuned_component;
+OMPI_DECLSPEC extern mca_coll_tuned_component_t mca_coll_tuned_component;
 
 struct mca_coll_tuned_module_t {
     mca_coll_base_module_t super;
@@ -219,4 +220,9 @@ struct mca_coll_tuned_module_t {
 typedef struct mca_coll_tuned_module_t mca_coll_tuned_module_t;
 OBJ_CLASS_DECLARATION(mca_coll_tuned_module_t);
 
+int coll_tuned_alg_from_str(int collective_id, const char *alg_name, int *alg_index);
+int coll_tuned_alg_to_str(int collective_id, int alg_value, char **alg_string);
+int coll_tuned_alg_register_options(int collective_id, mca_base_var_enum_t *options);
+
+
 #endif  /* MCA_COLL_TUNED_EXPORT_H */
diff --git a/ompi/mca/coll/tuned/coll_tuned_allgather_decision.c b/ompi/mca/coll/tuned/coll_tuned_allgather_decision.c
index 74ba5201ca7..052c1d5f9e4 100644
--- a/ompi/mca/coll/tuned/coll_tuned_allgather_decision.c
+++ b/ompi/mca/coll/tuned/coll_tuned_allgather_decision.c
@@ -4,6 +4,9 @@
  *                         reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2021      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2025      Amazon.com, Inc. or its affiliates.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -34,11 +37,13 @@ static int coll_tuned_allgather_chain_fanout;
 static const mca_base_var_enum_value_t allgather_algorithms[] = {
     {0, "ignore"},
     {1, "linear"},
-    {2, "bruck"},
+    {2, "bruck-k-fanout"},
     {3, "recursive_doubling"},
     {4, "ring"},
     {5, "neighbor"},
     {6, "two_proc"},
+    {7, "sparbit"},
+    {8, "direct-messaging"},
     {0, NULL}
 };
 
@@ -77,12 +82,13 @@ ompi_coll_tuned_allgather_intra_check_forced_init(coll_tuned_force_algorithm_mca
     mca_param_indices->algorithm_param_index =
         mca_base_component_var_register(&mca_coll_tuned_component.super.collm_version,
                                         "allgather_algorithm",
-                                        "Which allallgather algorithm is used. Can be locked down to choice of: 0 ignore, 1 basic linear, 2 bruck, 3 recursive doubling, 4 ring, 5 neighbor exchange, 6: two proc only. "
+                                        "Which allgather algorithm is used. Can be locked down to choice of: 0 ignore, 1 basic linear, 2 bruck with radix k, 3 recursive doubling, 4 ring, 5 neighbor exchange, 6: two proc only, 7: sparbit, 8: direct messaging. "
                                         "Only relevant if coll_tuned_use_dynamic_rules is true.",
                                         MCA_BASE_VAR_TYPE_INT, new_enum, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                         OPAL_INFO_LVL_5,
                                         MCA_BASE_VAR_SCOPE_ALL,
                                         &coll_tuned_allgather_forced_algorithm);
+    coll_tuned_alg_register_options( ALLGATHER, new_enum );
     OBJ_RELEASE(new_enum);
     if (mca_param_indices->algorithm_param_index < 0) {
         return mca_param_indices->algorithm_param_index;
@@ -121,18 +127,17 @@ ompi_coll_tuned_allgather_intra_check_forced_init(coll_tuned_force_algorithm_mca
     return (MPI_SUCCESS);
 }
 
-int ompi_coll_tuned_allgather_intra_do_this(const void *sbuf, int scount,
+int ompi_coll_tuned_allgather_intra_do_this(const void *sbuf, size_t scount,
                                             struct ompi_datatype_t *sdtype,
-                                            void* rbuf, int rcount,
+                                            void* rbuf, size_t rcount,
                                             struct ompi_datatype_t *rdtype,
                                             struct ompi_communicator_t *comm,
                                             mca_coll_base_module_t *module,
                                             int algorithm, int faninout, int segsize)
 {
-    OPAL_OUTPUT((ompi_coll_tuned_stream,
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
                  "coll:tuned:allgather_intra_do_this selected algorithm %d topo faninout %d segsize %d",
                  algorithm, faninout, segsize));
-
     switch (algorithm) {
     case (0):
         return ompi_coll_tuned_allgather_intra_dec_fixed(sbuf, scount, sdtype,
@@ -143,9 +148,9 @@ int ompi_coll_tuned_allgather_intra_do_this(const void *sbuf, int scount,
                                                            rbuf, rcount, rdtype,
                                                            comm, module);
     case (2):
-        return ompi_coll_base_allgather_intra_bruck(sbuf, scount, sdtype,
-                                                    rbuf, rcount, rdtype,
-                                                    comm, module);
+        return ompi_coll_base_allgather_intra_k_bruck(sbuf, scount, sdtype,
+                                                      rbuf, rcount, rdtype,
+                                                      comm, module, faninout);
     case (3):
         return ompi_coll_base_allgather_intra_recursivedoubling(sbuf, scount, sdtype,
                                                                 rbuf, rcount, rdtype,
@@ -162,8 +167,16 @@ int ompi_coll_tuned_allgather_intra_do_this(const void *sbuf, int scount,
         return ompi_coll_base_allgather_intra_two_procs(sbuf, scount, sdtype,
                                                         rbuf, rcount, rdtype,
                                                         comm, module);
+    case (7):
+        return ompi_coll_base_allgather_intra_sparbit(sbuf, scount, sdtype,
+                                                        rbuf, rcount, rdtype,
+                                                        comm, module);
+    case (8):
+        return ompi_coll_base_allgather_direct_messaging(sbuf, scount, sdtype,
+                                                         rbuf, rcount, rdtype,
+                                                         comm, module);
     } /* switch */
-    OPAL_OUTPUT((ompi_coll_tuned_stream,
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
                  "coll:tuned:allgather_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
                  algorithm, ompi_coll_tuned_forced_max_algorithms[ALLGATHER]));
     return (MPI_ERR_ARG);
diff --git a/ompi/mca/coll/tuned/coll_tuned_allgatherv_decision.c b/ompi/mca/coll/tuned/coll_tuned_allgatherv_decision.c
index e2828786ae5..5eb9801222c 100644
--- a/ompi/mca/coll/tuned/coll_tuned_allgatherv_decision.c
+++ b/ompi/mca/coll/tuned/coll_tuned_allgatherv_decision.c
@@ -5,6 +5,9 @@
  *                         reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2021      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2025      Amazon.com, Inc. or its affiliates.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -38,6 +41,7 @@ static const mca_base_var_enum_value_t allgatherv_algorithms[] = {
     {3, "ring"},
     {4, "neighbor"},
     {5, "two_proc"},
+    {6, "sparbit"},
     {0, NULL}
 };
 
@@ -76,12 +80,13 @@ ompi_coll_tuned_allgatherv_intra_check_forced_init(coll_tuned_force_algorithm_mc
     mca_param_indices->algorithm_param_index =
         mca_base_component_var_register(&mca_coll_tuned_component.super.collm_version,
                                         "allgatherv_algorithm",
-                                        "Which allallgatherv algorithm is used. Can be locked down to choice of: 0 ignore, 1 default (allgathervv + bcast), 2 bruck, 3 ring, 4 neighbor exchange, 5: two proc only. "
+                                        "Which allgatherv algorithm is used. Can be locked down to choice of: 0 ignore, 1 default (allgathervv + bcast), 2 bruck, 3 ring, 4 neighbor exchange, 5: two proc only, 6: sparbit. "
                                         "Only relevant if coll_tuned_use_dynamic_rules is true.",
                                         MCA_BASE_VAR_TYPE_INT, new_enum, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                         OPAL_INFO_LVL_5,
                                         MCA_BASE_VAR_SCOPE_CONSTANT,
                                         &coll_tuned_allgatherv_forced_algorithm);
+    coll_tuned_alg_register_options( ALLGATHERV, new_enum );
     OBJ_RELEASE(new_enum);
     if (mca_param_indices->algorithm_param_index < 0) {
         return mca_param_indices->algorithm_param_index;
@@ -120,17 +125,17 @@ ompi_coll_tuned_allgatherv_intra_check_forced_init(coll_tuned_force_algorithm_mc
     return (MPI_SUCCESS);
 }
 
-int ompi_coll_tuned_allgatherv_intra_do_this(const void *sbuf, int scount,
+int ompi_coll_tuned_allgatherv_intra_do_this(const void *sbuf, size_t scount,
                                              struct ompi_datatype_t *sdtype,
-                                             void *rbuf, const int *rcounts,
-                                             const int *rdispls,
+                                             void *rbuf, ompi_count_array_t rcounts,
+                                             ompi_disp_array_t rdispls,
                                              struct ompi_datatype_t *rdtype,
                                              struct ompi_communicator_t *comm,
                                              mca_coll_base_module_t *module,
                                              int algorithm, int faninout,
                                              int segsize)
 {
-    OPAL_OUTPUT((ompi_coll_tuned_stream,
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
                  "coll:tuned:allgatherv_intra_do_this selected algorithm %d topo faninout %d segsize %d",
                  algorithm, faninout, segsize));
 
@@ -159,8 +164,12 @@ int ompi_coll_tuned_allgatherv_intra_do_this(const void *sbuf, int scount,
         return ompi_coll_base_allgatherv_intra_two_procs(sbuf, scount, sdtype,
                                                          rbuf, rcounts, rdispls, rdtype,
                                                          comm, module);
+    case (6):
+        return ompi_coll_base_allgatherv_intra_sparbit(sbuf, scount, sdtype,
+                                                         rbuf, rcounts, rdispls, rdtype,
+                                                         comm, module);
     } /* switch */
-    OPAL_OUTPUT((ompi_coll_tuned_stream,
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
                  "coll:tuned:allgatherv_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
                  algorithm, ompi_coll_tuned_forced_max_algorithms[ALLGATHERV]));
     return (MPI_ERR_ARG);
diff --git a/ompi/mca/coll/tuned/coll_tuned_allreduce_decision.c b/ompi/mca/coll/tuned/coll_tuned_allreduce_decision.c
index eabe6f17378..9a63d8c5abb 100644
--- a/ompi/mca/coll/tuned/coll_tuned_allreduce_decision.c
+++ b/ompi/mca/coll/tuned/coll_tuned_allreduce_decision.c
@@ -5,6 +5,8 @@
  *                         reserved.
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2025      Amazon.com, Inc. or its affiliates.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -42,6 +44,7 @@ static const mca_base_var_enum_value_t allreduce_algorithms[] = {
     {4, "ring"},
     {5, "segmented_ring"},
     {6, "rabenseifner"},
+    {7, "allgather_reduce"},
     {0, NULL}
 };
 
@@ -77,12 +80,13 @@ int ompi_coll_tuned_allreduce_intra_check_forced_init (coll_tuned_force_algorith
     mca_param_indices->algorithm_param_index =
         mca_base_component_var_register(&mca_coll_tuned_component.super.collm_version,
                                         "allreduce_algorithm",
-                                        "Which allreduce algorithm is used. Can be locked down to any of: 0 ignore, 1 basic linear, 2 nonoverlapping (tuned reduce + tuned bcast), 3 recursive doubling, 4 ring, 5 segmented ring. "
+                                        "Which allreduce algorithm is used. Can be locked down to any of: 0 ignore, 1 basic linear, 2 nonoverlapping (tuned reduce + tuned bcast), 3 recursive doubling, 4 ring, 5 segmented ring, 6 rabenseifner, 7 allgather_reduce. "
                                         "Only relevant if coll_tuned_use_dynamic_rules is true.",
                                         MCA_BASE_VAR_TYPE_INT, new_enum, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                         OPAL_INFO_LVL_5,
                                         MCA_BASE_VAR_SCOPE_ALL,
                                         &coll_tuned_allreduce_forced_algorithm);
+    coll_tuned_alg_register_options( ALLREDUCE, new_enum );
     OBJ_RELEASE(new_enum);
     if (mca_param_indices->algorithm_param_index < 0) {
         return mca_param_indices->algorithm_param_index;
@@ -121,15 +125,16 @@ int ompi_coll_tuned_allreduce_intra_check_forced_init (coll_tuned_force_algorith
     return (MPI_SUCCESS);
 }
 
-int ompi_coll_tuned_allreduce_intra_do_this(const void *sbuf, void *rbuf, int count,
+int ompi_coll_tuned_allreduce_intra_do_this(const void *sbuf, void *rbuf, size_t count,
                                             struct ompi_datatype_t *dtype,
                                             struct ompi_op_t *op,
                                             struct ompi_communicator_t *comm,
                                             mca_coll_base_module_t *module,
                                             int algorithm, int faninout, int segsize)
 {
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:allreduce_intra_do_this algorithm %d topo fan in/out %d segsize %d",
-                 algorithm, faninout, segsize));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "coll:tuned:allreduce_intra_do_this algorithm %d topo fan in/out %d segsize %d",
+        algorithm, faninout, segsize));
 
     switch (algorithm) {
     case (0):
@@ -146,8 +151,11 @@ int ompi_coll_tuned_allreduce_intra_do_this(const void *sbuf, void *rbuf, int co
         return ompi_coll_base_allreduce_intra_ring_segmented(sbuf, rbuf, count, dtype, op, comm, module, segsize);
     case (6):
         return ompi_coll_base_allreduce_intra_redscat_allgather(sbuf, rbuf, count, dtype, op, comm, module);
+    case (7):
+        return ompi_coll_base_allreduce_intra_allgather_reduce(sbuf, rbuf, count, dtype, op, comm, module);
     } /* switch */
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:allreduce_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
-                 algorithm, ompi_coll_tuned_forced_max_algorithms[ALLREDUCE]));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "coll:tuned:allreduce_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
+        algorithm, ompi_coll_tuned_forced_max_algorithms[ALLREDUCE]));
     return (MPI_ERR_ARG);
 }
diff --git a/ompi/mca/coll/tuned/coll_tuned_alltoall_decision.c b/ompi/mca/coll/tuned/coll_tuned_alltoall_decision.c
index 487f9da4fde..e3482116c84 100644
--- a/ompi/mca/coll/tuned/coll_tuned_alltoall_decision.c
+++ b/ompi/mca/coll/tuned/coll_tuned_alltoall_decision.c
@@ -5,6 +5,9 @@
  *                         reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2025      Amazon.com, Inc. or its affiliates.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -74,12 +77,13 @@ int ompi_coll_tuned_alltoall_intra_check_forced_init (coll_tuned_force_algorithm
     mca_param_indices->algorithm_param_index =
         mca_base_component_var_register(&mca_coll_tuned_component.super.collm_version,
                                         "alltoall_algorithm",
-                                        "Which alltoall algorithm is used. Can be locked down to choice of: 0 ignore, 1 basic linear, 2 pairwise, 3: modified bruck, 4: linear with sync, 5:two proc only. "
+                                        "Which alltoall algorithm is used. Can be locked down to choice of: 0 ignore, 1 linear, 2 pairwise, 3: modified_bruck, 4: linear_sync, 5:two_proc. "
                                         "Only relevant if coll_tuned_use_dynamic_rules is true.",
                                         MCA_BASE_VAR_TYPE_INT, new_enum, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                         OPAL_INFO_LVL_5,
                                         MCA_BASE_VAR_SCOPE_ALL,
                                         &coll_tuned_alltoall_forced_algorithm);
+    coll_tuned_alg_register_options( ALLTOALL, new_enum );
     OBJ_RELEASE(new_enum);
     if (mca_param_indices->algorithm_param_index < 0) {
         return mca_param_indices->algorithm_param_index;
@@ -115,21 +119,22 @@ int ompi_coll_tuned_alltoall_intra_check_forced_init (coll_tuned_force_algorithm
                                       MCA_BASE_VAR_SCOPE_ALL,
                                       &coll_tuned_alltoall_chain_fanout);
 
+    int deprecated_mca_params = -1;
     (void) mca_base_component_var_register(&mca_coll_tuned_component.super.collm_version,
                                            "alltoall_large_msg",
                                            "use pairwise exchange algorithm for messages larger than this value",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_DEPRECATED,
                                            OPAL_INFO_LVL_6,
                                            MCA_BASE_VAR_SCOPE_READONLY,
-                                           &ompi_coll_tuned_alltoall_large_msg);
+                                           &deprecated_mca_params);
 
     (void) mca_base_component_var_register(&mca_coll_tuned_component.super.collm_version,
                                            "alltoall_min_procs",
                                            "use pairwise exchange algorithm for communicators larger than this value",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_DEPRECATED,
                                            OPAL_INFO_LVL_6,
                                            MCA_BASE_VAR_SCOPE_READONLY,
-                                           &ompi_coll_tuned_alltoall_min_procs);
+                                           &deprecated_mca_params);
 
     mca_param_indices->max_requests_param_index =
       mca_base_component_var_register(&mca_coll_tuned_component.super.collm_version,
@@ -153,17 +158,18 @@ int ompi_coll_tuned_alltoall_intra_check_forced_init (coll_tuned_force_algorithm
     return (MPI_SUCCESS);
 }
 
-int ompi_coll_tuned_alltoall_intra_do_this(const void *sbuf, int scount,
+int ompi_coll_tuned_alltoall_intra_do_this(const void *sbuf, size_t scount,
                                            struct ompi_datatype_t *sdtype,
-                                           void* rbuf, int rcount,
+                                           void* rbuf, size_t rcount,
                                            struct ompi_datatype_t *rdtype,
                                            struct ompi_communicator_t *comm,
                                            mca_coll_base_module_t *module,
                                            int algorithm, int faninout, int segsize,
                                            int max_requests)
 {
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:alltoall_intra_do_this selected algorithm %d topo faninout %d segsize %d",
-                 algorithm, faninout, segsize));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "coll:tuned:alltoall_intra_do_this selected algorithm %d topo faninout %d segsize %d",
+        algorithm, faninout, segsize));
 
     switch (algorithm) {
     case (0):
@@ -179,7 +185,8 @@ int ompi_coll_tuned_alltoall_intra_do_this(const void *sbuf, int scount,
     case (5):
         return ompi_coll_base_alltoall_intra_two_procs(sbuf, scount, sdtype, rbuf, rcount, rdtype, comm, module);
     } /* switch */
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:alltoall_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
-                 algorithm, ompi_coll_tuned_forced_max_algorithms[ALLTOALL]));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "coll:tuned:alltoall_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
+        algorithm, ompi_coll_tuned_forced_max_algorithms[ALLTOALL]));
     return (MPI_ERR_ARG);
 }
diff --git a/ompi/mca/coll/tuned/coll_tuned_alltoallv_decision.c b/ompi/mca/coll/tuned/coll_tuned_alltoallv_decision.c
index c5d72095c91..60a8fb10c85 100644
--- a/ompi/mca/coll/tuned/coll_tuned_alltoallv_decision.c
+++ b/ompi/mca/coll/tuned/coll_tuned_alltoallv_decision.c
@@ -5,6 +5,8 @@
  *                         reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2025      Amazon.com, Inc. or its affiliates.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -77,7 +79,7 @@ int ompi_coll_tuned_alltoallv_intra_check_forced_init(coll_tuned_force_algorithm
                                         OPAL_INFO_LVL_5,
                                         MCA_BASE_VAR_SCOPE_ALL,
                                         &coll_tuned_alltoallv_forced_algorithm);
-
+    coll_tuned_alg_register_options( ALLTOALLV, new_enum );
     OBJ_RELEASE(new_enum);
     if (mca_param_indices->algorithm_param_index < 0) {
         return mca_param_indices->algorithm_param_index;
@@ -88,15 +90,15 @@ int ompi_coll_tuned_alltoallv_intra_check_forced_init(coll_tuned_force_algorithm
 
 /* If the user selects dynamic rules and specifies the algorithm to
  * use, then this function is called.  */
-int ompi_coll_tuned_alltoallv_intra_do_this(const void *sbuf, const int *scounts, const int *sdisps,
+int ompi_coll_tuned_alltoallv_intra_do_this(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
                                             struct ompi_datatype_t *sdtype,
-                                            void* rbuf, const int *rcounts, const int *rdisps,
+                                            void* rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
                                             struct ompi_datatype_t *rdtype,
                                             struct ompi_communicator_t *comm,
                                             mca_coll_base_module_t *module,
                                             int algorithm)
 {
-    OPAL_OUTPUT((ompi_coll_tuned_stream,
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
                  "coll:tuned:alltoallv_intra_do_this selected algorithm %d ",
                  algorithm));
 
@@ -114,7 +116,7 @@ int ompi_coll_tuned_alltoallv_intra_do_this(const void *sbuf, const int *scounts
                                                        rbuf, rcounts, rdisps, rdtype,
                                                        comm, module);
     }  /* switch */
-    OPAL_OUTPUT((ompi_coll_tuned_stream,
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
                  "coll:tuned:alltoall_intra_do_this attempt to select "
                  "algorithm %d when only 0-%d is valid.",
                  algorithm, ompi_coll_tuned_forced_max_algorithms[ALLTOALLV]));
diff --git a/ompi/mca/coll/tuned/coll_tuned_barrier_decision.c b/ompi/mca/coll/tuned/coll_tuned_barrier_decision.c
index dca24ad27d2..fa2c3f2bf29 100644
--- a/ompi/mca/coll/tuned/coll_tuned_barrier_decision.c
+++ b/ompi/mca/coll/tuned/coll_tuned_barrier_decision.c
@@ -3,6 +3,8 @@
  * Copyright (c) 2004-2020 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2025      Amazon.com, Inc. or its affiliates.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -78,6 +80,7 @@ int ompi_coll_tuned_barrier_intra_check_forced_init (coll_tuned_force_algorithm_
                                         OPAL_INFO_LVL_5,
                                         MCA_BASE_VAR_SCOPE_ALL,
                                         &coll_tuned_barrier_forced_algorithm);
+    coll_tuned_alg_register_options( BARRIER, new_enum );
     OBJ_RELEASE(new_enum);
     if (mca_param_indices->algorithm_param_index < 0) {
         return mca_param_indices->algorithm_param_index;
@@ -90,7 +93,7 @@ int ompi_coll_tuned_barrier_intra_do_this (struct ompi_communicator_t *comm,
                                            mca_coll_base_module_t *module,
                                            int algorithm, int faninout, int segsize)
 {
-    OPAL_OUTPUT((ompi_coll_tuned_stream,
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
                  "coll:tuned:barrier_intra_do_this selected algorithm %d topo fanin/out%d",
                  algorithm, faninout));
 
@@ -103,7 +106,8 @@ int ompi_coll_tuned_barrier_intra_do_this (struct ompi_communicator_t *comm,
     case (5):   return ompi_coll_base_barrier_intra_two_procs(comm, module);
     case (6):   return ompi_coll_base_barrier_intra_tree(comm, module);
     } /* switch */
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:barrier_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
-                 algorithm, ompi_coll_tuned_forced_max_algorithms[BARRIER]));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "coll:tuned:barrier_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
+        algorithm, ompi_coll_tuned_forced_max_algorithms[BARRIER]));
     return (MPI_ERR_ARG);
 }
diff --git a/ompi/mca/coll/tuned/coll_tuned_bcast_decision.c b/ompi/mca/coll/tuned/coll_tuned_bcast_decision.c
index b949b3ab744..90ecab634bf 100644
--- a/ompi/mca/coll/tuned/coll_tuned_bcast_decision.c
+++ b/ompi/mca/coll/tuned/coll_tuned_bcast_decision.c
@@ -5,6 +5,8 @@
  *                         reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2025      Amazon.com, Inc. or its affiliates.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -86,6 +88,7 @@ int ompi_coll_tuned_bcast_intra_check_forced_init (coll_tuned_force_algorithm_mc
                                         OPAL_INFO_LVL_5,
                                         MCA_BASE_VAR_SCOPE_ALL,
                                         &coll_tuned_bcast_forced_algorithm);
+    coll_tuned_alg_register_options( BCAST, new_enum );
     OBJ_RELEASE(new_enum);
     if (mca_param_indices->algorithm_param_index < 0) {
         return mca_param_indices->algorithm_param_index;
@@ -132,15 +135,16 @@ int ompi_coll_tuned_bcast_intra_check_forced_init (coll_tuned_force_algorithm_mc
     return (MPI_SUCCESS);
 }
 
-int ompi_coll_tuned_bcast_intra_do_this(void *buf, int count,
+int ompi_coll_tuned_bcast_intra_do_this(void *buf, size_t count,
                                         struct ompi_datatype_t *dtype,
                                         int root,
                                         struct ompi_communicator_t *comm,
                                         mca_coll_base_module_t *module,
                                         int algorithm, int faninout, int segsize)
 {
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:bcast_intra_do_this algorithm %d topo faninout %d segsize %d",
-                 algorithm, faninout, segsize));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "coll:tuned:bcast_intra_do_this algorithm %d topo faninout %d segsize %d",
+        algorithm, faninout, segsize));
 
     switch (algorithm) {
     case (0):
@@ -165,7 +169,8 @@ int ompi_coll_tuned_bcast_intra_do_this(void *buf, int count,
     case (9):
         return ompi_coll_base_bcast_intra_scatter_allgather_ring(buf, count, dtype, root, comm, module, segsize);
     } /* switch */
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:bcast_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
-                 algorithm, ompi_coll_tuned_forced_max_algorithms[BCAST]));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "coll:tuned:bcast_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
+        algorithm, ompi_coll_tuned_forced_max_algorithms[BCAST]));
     return (MPI_ERR_ARG);
 }
diff --git a/ompi/mca/coll/tuned/coll_tuned_component.c b/ompi/mca/coll/tuned/coll_tuned_component.c
index a34c4ba907f..39894bb1cf1 100644
--- a/ompi/mca/coll/tuned/coll_tuned_component.c
+++ b/ompi/mca/coll/tuned/coll_tuned_component.c
@@ -17,6 +17,9 @@
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2019      Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2025      Amazon.com, Inc. or its affiliates.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -55,14 +58,11 @@ char* ompi_coll_tuned_dynamic_rules_filename = (char*) NULL;
 int   ompi_coll_tuned_init_tree_fanout = 4;
 int   ompi_coll_tuned_init_chain_fanout = 4;
 int   ompi_coll_tuned_init_max_requests = 128;
-int   ompi_coll_tuned_alltoall_small_msg = 200;
-int   ompi_coll_tuned_alltoall_intermediate_msg = 3000;
+int   ompi_coll_tuned_verbose = 0;
 
 /* Set it to the same value as intermediate msg by default, so it does not affect
  * default algorithm selection. Changing this value will force using linear with
  * sync algorithm on certain message sizes. */
-int   ompi_coll_tuned_alltoall_large_msg = 3000;
-int   ompi_coll_tuned_alltoall_min_procs = 0; /* disable by default */
 int   ompi_coll_tuned_alltoall_max_requests  = 0; /* no limit for alltoall by default */
 
 /* Disable by default */
@@ -71,12 +71,16 @@ int   ompi_coll_tuned_scatter_large_msg = 0;
 int   ompi_coll_tuned_scatter_min_procs = 0;
 int   ompi_coll_tuned_scatter_blocking_send_ratio = 0;
 
-/* forced alogrithm variables */
+/* forced algorithm variables */
 /* indices for the MCA parameters */
 coll_tuned_force_algorithm_mca_param_indices_t ompi_coll_tuned_forced_params[COLLCOUNT] = {{0}};
 /* max algorithm values */
 int ompi_coll_tuned_forced_max_algorithms[COLLCOUNT] = {0};
 
+/* names of each algorithm for each collective */
+mca_base_var_enum_t *coll_tuned_algorithm_enums[COLLCOUNT] = {0};
+
+
 /*
  * Local function
  */
@@ -95,7 +99,7 @@ mca_coll_tuned_component_t mca_coll_tuned_component = {
         /* First, the mca_component_t struct containing meta information
            about the component itself */
         .collm_version = {
-            MCA_COLL_BASE_VERSION_2_4_0,
+            MCA_COLL_BASE_VERSION_3_0_0,
 
             /* Component name and version */
             .mca_component_name = "tuned",
@@ -132,65 +136,73 @@ static int tuned_register(void)
     ompi_coll_tuned_priority = 30;
     (void) mca_base_component_var_register(&mca_coll_tuned_component.super.collm_version,
                                            "priority", "Priority of the tuned coll component",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                            OPAL_INFO_LVL_6,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           MCA_BASE_VAR_SCOPE_ALL,
                                            &ompi_coll_tuned_priority);
 
     /* some initial guesses at topology parameters */
     ompi_coll_tuned_init_tree_fanout = 4;
     (void) mca_base_component_var_register(&mca_coll_tuned_component.super.collm_version,
                                            "init_tree_fanout",
-                                           "Inital fanout used in the tree topologies for each communicator. This is only an initial guess, if a tuned collective needs a different fanout for an operation, it build it dynamically. This parameter is only for the first guess and might save a little time",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           "Initial fanout used in the tree topologies for each communicator. This is only an initial guess, if a tuned collective needs a different fanout for an operation, it build it dynamically. This parameter is only for the first guess and might save a little time",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                            OPAL_INFO_LVL_6,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           MCA_BASE_VAR_SCOPE_ALL,
                                            &ompi_coll_tuned_init_tree_fanout);
 
     ompi_coll_tuned_init_chain_fanout = 4;
     (void) mca_base_component_var_register(&mca_coll_tuned_component.super.collm_version,
                                            "init_chain_fanout",
-                                           "Inital fanout used in the chain (fanout followed by pipeline) topologies for each communicator. This is only an initial guess, if a tuned collective needs a different fanout for an operation, it build it dynamically. This parameter is only for the first guess and might save a little time",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           "Initial fanout used in the chain (fanout followed by pipeline) topologies for each communicator. This is only an initial guess, if a tuned collective needs a different fanout for an operation, it build it dynamically. This parameter is only for the first guess and might save a little time",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                            OPAL_INFO_LVL_6,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           MCA_BASE_VAR_SCOPE_ALL,
                                            &ompi_coll_tuned_init_chain_fanout);
 
-    ompi_coll_tuned_alltoall_small_msg = 200;
+    int deprecated_mca_params = -1;
     (void) mca_base_component_var_register(&mca_coll_tuned_component.super.collm_version,
                                            "alltoall_small_msg",
                                            "threshold (if supported) to decide if small MSGs alltoall algorithm will be used",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_DEPRECATED,
                                            OPAL_INFO_LVL_6,
                                            MCA_BASE_VAR_SCOPE_READONLY,
-                                           &ompi_coll_tuned_alltoall_small_msg);
+                                           &deprecated_mca_params);
 
-    ompi_coll_tuned_alltoall_intermediate_msg = 3000;
     (void) mca_base_component_var_register(&mca_coll_tuned_component.super.collm_version,
                                            "alltoall_intermediate_msg",
                                            "threshold (if supported) to decide if intermediate MSGs alltoall algorithm will be used",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_DEPRECATED,
                                            OPAL_INFO_LVL_6,
                                            MCA_BASE_VAR_SCOPE_READONLY,
-                                           &ompi_coll_tuned_alltoall_intermediate_msg);
+                                           &deprecated_mca_params);
 
     (void) mca_base_component_var_register(&mca_coll_tuned_component.super.collm_version,
                                            "use_dynamic_rules",
                                            "Switch used to decide if we use static (compiled/if statements) or dynamic (built at runtime) decision function rules",
-                                           MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
+                                           MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                            OPAL_INFO_LVL_6,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           MCA_BASE_VAR_SCOPE_ALL,
                                            &ompi_coll_tuned_use_dynamic_rules);
 
     ompi_coll_tuned_dynamic_rules_filename = NULL;
     (void) mca_base_component_var_register(&mca_coll_tuned_component.super.collm_version,
                                            "dynamic_rules_filename",
                                            "Filename of configuration file that contains the dynamic (@runtime) decision function rules",
-                                           MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
+                                           MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                            OPAL_INFO_LVL_6,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           MCA_BASE_VAR_SCOPE_ALL,
                                            &ompi_coll_tuned_dynamic_rules_filename);
 
+    ompi_coll_tuned_verbose = 0;
+    (void) mca_base_component_var_register(&mca_coll_tuned_component.super.collm_version,
+                                           "verbose",
+                                           "Verbosity of the tuned coll component",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
+                                           OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_ALL,
+                                           &ompi_coll_tuned_verbose);
+
     /* register forced params */
     ompi_coll_tuned_allreduce_intra_check_forced_init(&ompi_coll_tuned_forced_params[ALLREDUCE]);
     ompi_coll_tuned_alltoall_intra_check_forced_init(&ompi_coll_tuned_forced_params[ALLTOALL]);
@@ -214,16 +226,15 @@ static int tuned_open(void)
 {
     int rc;
 
-#if OPAL_ENABLE_DEBUG
-    if (ompi_coll_base_framework.framework_verbose) {
+    if (ompi_coll_tuned_verbose) {
         ompi_coll_tuned_stream = opal_output_open(NULL);
+        opal_output_set_verbosity(ompi_coll_tuned_stream, ompi_coll_tuned_verbose);
     }
-#endif  /* OPAL_ENABLE_DEBUG */
 
     /* now check that the user hasn't overrode any of the decision functions if dynamic rules are enabled */
     /* the user can redo this before every comm dup/create if they like */
     /* this is useful for benchmarking and user knows best tuning */
-    /* as this is the component we only lookup the indicies of the mca params */
+    /* as this is the component we only lookup the indices of the mca params */
     /* the actual values are looked up during comm create via module init */
 
     /* intra functions first */
@@ -231,40 +242,52 @@ static int tuned_open(void)
     /* by default DISABLE dynamic rules and instead use fixed [if based] rules */
     if (ompi_coll_tuned_use_dynamic_rules) {
         if( ompi_coll_tuned_dynamic_rules_filename ) {
-            OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:component_open Reading collective rules file [%s]",
-                         ompi_coll_tuned_dynamic_rules_filename));
+            OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+                "coll:tuned:component_open Reading collective rules file [%s]",
+                ompi_coll_tuned_dynamic_rules_filename));
             rc = ompi_coll_tuned_read_rules_config_file( ompi_coll_tuned_dynamic_rules_filename,
-                                                         &(mca_coll_tuned_component.all_base_rules), COLLCOUNT);
-            if( rc >= 0 ) {
-                OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:module_open Read %d valid rules\n", rc));
+                                                         &(mca_coll_tuned_component.all_base_rules));
+            if( rc == OPAL_SUCCESS ) {
+                OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+                    "coll:tuned:module_open Read a valid rules file"));
             } else {
-                OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:module_open Reading collective rules file failed\n"));
+                OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+                    "coll:tuned:module_open Reading collective rules file failed\n"));
                 mca_coll_tuned_component.all_base_rules = NULL;
             }
         }
     }
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream, "coll:tuned:component_open: done!"));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "coll:tuned:component_open: done!"));
 
     return OMPI_SUCCESS;
 }
 
 /* here we should clean up state stored on the component */
-/* i.e. alg table and dynamic changable rules if allocated etc */
+/* i.e. alg table and dynamic changeable rules if allocated etc */
 static int tuned_close(void)
 {
-    OPAL_OUTPUT((ompi_coll_tuned_stream, "coll:tuned:component_close: called"));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "coll:tuned:component_close: called"));
 
     /* dealloc alg table if allocated */
-    /* dealloc dynamic changable rules if allocated */
+    /* dealloc dynamic changeable rules if allocated */
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream, "coll:tuned:component_close: done!"));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "coll:tuned:component_close: done!"));
 
     if( NULL != mca_coll_tuned_component.all_base_rules ) {
-        ompi_coll_tuned_free_all_rules(mca_coll_tuned_component.all_base_rules, COLLCOUNT);
+        ompi_coll_tuned_free_all_rules(mca_coll_tuned_component.all_base_rules);
         mca_coll_tuned_component.all_base_rules = NULL;
     }
 
+    for (int i=0; i<COLLCOUNT; i++) {
+        if (coll_tuned_algorithm_enums[i] != NULL) {
+            OBJ_RELEASE(coll_tuned_algorithm_enums[i]);
+        }
+    }
+
     return OMPI_SUCCESS;
 }
 
@@ -279,5 +302,40 @@ mca_coll_tuned_module_construct(mca_coll_tuned_module_t *module)
     }
 }
 
+int coll_tuned_alg_from_str(int collective_id, const char *alg_name, int *alg_value) {
+    int rc;
+    if (collective_id >= COLLCOUNT || collective_id < 0) { return OPAL_ERROR; };
+    rc = coll_tuned_algorithm_enums[collective_id]->value_from_string(
+        coll_tuned_algorithm_enums[collective_id],
+        alg_name, alg_value );
+    return rc;
+}
+
+/* return the enum's value and string.  caller's responsibility to free alg_string if NULL was not provided. */
+int coll_tuned_alg_to_str(int collective_id, int alg_value, char **alg_string) {
+    int rc;
+    if (collective_id >= COLLCOUNT || collective_id < 0) { return OPAL_ERROR; };
+    rc = coll_tuned_algorithm_enums[collective_id]->string_from_value(
+        coll_tuned_algorithm_enums[collective_id],
+        alg_value, alg_string );
+    return rc;
+}
+
+
+int coll_tuned_alg_register_options(int collective_id, mca_base_var_enum_t *options) {
+    /* use the same enum used for mca parameters to allow tuning files to use
+    algorithm names rather than just numbers.*/
+    if (!options) { return OPAL_ERROR; }
+    if (collective_id >= COLLCOUNT || collective_id < 0) {
+        return OPAL_ERROR;
+    }
+
+    /* retain the enum until tuned_close() */
+    OBJ_RETAIN(options);
+    coll_tuned_algorithm_enums[collective_id] = options;
+    return OPAL_SUCCESS;
+}
+
+
 OBJ_CLASS_INSTANCE(mca_coll_tuned_module_t, mca_coll_base_module_t,
                    mca_coll_tuned_module_construct, NULL);
diff --git a/ompi/mca/coll/tuned/coll_tuned_decision_dynamic.c b/ompi/mca/coll/tuned/coll_tuned_decision_dynamic.c
index 4133a64eefc..5d6a699b301 100644
--- a/ompi/mca/coll/tuned/coll_tuned_decision_dynamic.c
+++ b/ompi/mca/coll/tuned/coll_tuned_decision_dynamic.c
@@ -12,7 +12,7 @@
  * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2020      Amazon.com, Inc. or its affiliates.
+ * Copyright (c) 2020-2025 Amazon.com, Inc. or its affiliates.
  *                         All Rights reserved.
  * $COPYRIGHT$
  *
@@ -52,7 +52,7 @@
  *  Returns:    - MPI_SUCCESS or error code
  */
 int
-ompi_coll_tuned_allreduce_intra_dec_dynamic (const void *sbuf, void *rbuf, int count,
+ompi_coll_tuned_allreduce_intra_dec_dynamic (const void *sbuf, void *rbuf, size_t count,
                                              struct ompi_datatype_t *dtype,
                                              struct ompi_op_t *op,
                                              struct ompi_communicator_t *comm,
@@ -60,7 +60,8 @@ ompi_coll_tuned_allreduce_intra_dec_dynamic (const void *sbuf, void *rbuf, int c
 {
     mca_coll_tuned_module_t *tuned_module = (mca_coll_tuned_module_t*) module;
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream, "ompi_coll_tuned_allreduce_intra_dec_dynamic"));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "ompi_coll_tuned_allreduce_intra_dec_dynamic"));
 
     /* Check first if an algorithm is set explicitly for this collective */
     if (tuned_module->user_forced[ALLREDUCE].algorithm) {
@@ -102,16 +103,17 @@ ompi_coll_tuned_allreduce_intra_dec_dynamic (const void *sbuf, void *rbuf, int c
  *    Returns:    - MPI_SUCCESS or error code (passed from the alltoall implementation)
  */
 
-int ompi_coll_tuned_alltoall_intra_dec_dynamic(const void *sbuf, int scount,
+int ompi_coll_tuned_alltoall_intra_dec_dynamic(const void *sbuf, size_t scount,
                                                struct ompi_datatype_t *sdtype,
-                                               void* rbuf, int rcount,
+                                               void* rbuf, size_t rcount,
                                                struct ompi_datatype_t *rdtype,
                                                struct ompi_communicator_t *comm,
                                                mca_coll_base_module_t *module)
 {
     mca_coll_tuned_module_t *tuned_module = (mca_coll_tuned_module_t*) module;
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream, "ompi_coll_tuned_alltoall_intra_dec_dynamic"));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "ompi_coll_tuned_alltoall_intra_dec_dynamic"));
 
     /* Check first if an algorithm is set explicitly for this collective */
     if (tuned_module->user_forced[ALLTOALL].algorithm) {
@@ -158,16 +160,17 @@ int ompi_coll_tuned_alltoall_intra_dec_dynamic(const void *sbuf, int scount,
  *    Returns:    - MPI_SUCCESS or error code
  */
 
-int ompi_coll_tuned_alltoallv_intra_dec_dynamic(const void *sbuf, const int *scounts, const int *sdisps,
+int ompi_coll_tuned_alltoallv_intra_dec_dynamic(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
                                                 struct ompi_datatype_t *sdtype,
-                                                void* rbuf, const int *rcounts, const int *rdisps,
+                                                void* rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
                                                 struct ompi_datatype_t *rdtype,
                                                 struct ompi_communicator_t *comm,
                                                 mca_coll_base_module_t *module)
 {
     mca_coll_tuned_module_t *tuned_module = (mca_coll_tuned_module_t*) module;
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream, "ompi_coll_tuned_alltoallv_intra_dec_dynamic"));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "ompi_coll_tuned_alltoallv_intra_dec_dynamic"));
 
     /* Check first if an algorithm is set explicitly for this collective */
     if (tuned_module->user_forced[ALLTOALLV].algorithm) {
@@ -215,7 +218,8 @@ int ompi_coll_tuned_barrier_intra_dec_dynamic(struct ompi_communicator_t *comm,
 {
     mca_coll_tuned_module_t *tuned_module = (mca_coll_tuned_module_t*) module;
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"ompi_coll_tuned_barrier_intra_dec_dynamic"));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "ompi_coll_tuned_barrier_intra_dec_dynamic"));
 
     /* Check first if an algorithm is set explicitly for this collective */
     if (tuned_module->user_forced[BARRIER].algorithm) {
@@ -250,14 +254,15 @@ int ompi_coll_tuned_barrier_intra_dec_dynamic(struct ompi_communicator_t *comm,
  *   Accepts:   - same arguments as MPI_Bcast()
  *   Returns:   - MPI_SUCCESS or error code (passed from the bcast implementation)
  */
-int ompi_coll_tuned_bcast_intra_dec_dynamic(void *buf, int count,
+int ompi_coll_tuned_bcast_intra_dec_dynamic(void *buf, size_t count,
                                             struct ompi_datatype_t *dtype, int root,
                                             struct ompi_communicator_t *comm,
                                             mca_coll_base_module_t *module)
 {
     mca_coll_tuned_module_t *tuned_module = (mca_coll_tuned_module_t*) module;
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream, "coll:tuned:bcast_intra_dec_dynamic"));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "coll:tuned:bcast_intra_dec_dynamic"));
 
     /* Check first if an algorithm is set explicitly for this collective */
     if (tuned_module->user_forced[BCAST].algorithm) {
@@ -302,14 +307,15 @@ int ompi_coll_tuned_bcast_intra_dec_dynamic(void *buf, int count,
  *
  */
 int ompi_coll_tuned_reduce_intra_dec_dynamic( const void *sbuf, void *rbuf,
-                                              int count, struct ompi_datatype_t* dtype,
+                                              size_t count, struct ompi_datatype_t* dtype,
                                               struct ompi_op_t* op, int root,
                                               struct ompi_communicator_t* comm,
                                               mca_coll_base_module_t *module)
 {
     mca_coll_tuned_module_t *tuned_module = (mca_coll_tuned_module_t*) module;
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream, "coll:tuned:reduce_intra_dec_dynamic"));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "coll:tuned:reduce_intra_dec_dynamic"));
 
     /* Check first if an algorithm is set explicitly for this collective */
     if (tuned_module->user_forced[REDUCE].algorithm) {
@@ -357,7 +363,7 @@ int ompi_coll_tuned_reduce_intra_dec_dynamic( const void *sbuf, void *rbuf,
  *
  */
 int ompi_coll_tuned_reduce_scatter_intra_dec_dynamic(const void *sbuf, void *rbuf,
-                                                     const int *rcounts,
+                                                     ompi_count_array_t rcounts,
                                                      struct ompi_datatype_t *dtype,
                                                      struct ompi_op_t *op,
                                                      struct ompi_communicator_t *comm,
@@ -365,7 +371,8 @@ int ompi_coll_tuned_reduce_scatter_intra_dec_dynamic(const void *sbuf, void *rbu
 {
     mca_coll_tuned_module_t *tuned_module = (mca_coll_tuned_module_t*) module;
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream, "coll:tuned:reduce_scatter_intra_dec_dynamic"));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "coll:tuned:reduce_scatter_intra_dec_dynamic"));
 
     /* Check first if an algorithm is set explicitly for this collective */
     if (tuned_module->user_forced[REDUCESCATTER].algorithm) {
@@ -383,7 +390,7 @@ int ompi_coll_tuned_reduce_scatter_intra_dec_dynamic(const void *sbuf, void *rbu
         int alg, faninout, segsize, ignoreme, i, count, size;
         size_t dsize;
         size = ompi_comm_size(comm);
-        for (i = 0, count = 0; i < size; i++) { count += rcounts[i];}
+        for (i = 0, count = 0; i < size; i++) { count += ompi_count_array_get(rcounts, i);}
         ompi_datatype_type_size (dtype, &dsize);
         dsize *= count;
 
@@ -412,7 +419,7 @@ int ompi_coll_tuned_reduce_scatter_intra_dec_dynamic(const void *sbuf, void *rbu
  *
  */
 int ompi_coll_tuned_reduce_scatter_block_intra_dec_dynamic(const void *sbuf, void *rbuf,
-                                                           int rcount,
+                                                           size_t rcount,
                                                            struct ompi_datatype_t *dtype,
                                                            struct ompi_op_t *op,
                                                            struct ompi_communicator_t *comm,
@@ -420,7 +427,8 @@ int ompi_coll_tuned_reduce_scatter_block_intra_dec_dynamic(const void *sbuf, voi
 {
     mca_coll_tuned_module_t *tuned_module = (mca_coll_tuned_module_t*) module;
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream, "coll:tuned:reduce_scatter_block_intra_dec_dynamic"));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "coll:tuned:reduce_scatter_block_intra_dec_dynamic"));
 
     /* Check first if an algorithm is set explicitly for this collective */
     if (tuned_module->user_forced[REDUCESCATTERBLOCK].algorithm) {
@@ -465,16 +473,16 @@ int ompi_coll_tuned_reduce_scatter_block_intra_dec_dynamic(const void *sbuf, voi
  *                        allgather function).
  */
 
-int ompi_coll_tuned_allgather_intra_dec_dynamic(const void *sbuf, int scount,
+int ompi_coll_tuned_allgather_intra_dec_dynamic(const void *sbuf, size_t scount,
                                                 struct ompi_datatype_t *sdtype,
-                                                void* rbuf, int rcount,
+                                                void* rbuf, size_t rcount,
                                                 struct ompi_datatype_t *rdtype,
                                                 struct ompi_communicator_t *comm,
                                                 mca_coll_base_module_t *module)
 {
     mca_coll_tuned_module_t *tuned_module = (mca_coll_tuned_module_t*) module;
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream,
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
                  "ompi_coll_tuned_allgather_intra_dec_dynamic"));
 
     /* Check first if an algorithm is set explicitly for this collective */
@@ -526,17 +534,17 @@ int ompi_coll_tuned_allgather_intra_dec_dynamic(const void *sbuf, int scount,
  *                        allgatherv function).
  */
 
-int ompi_coll_tuned_allgatherv_intra_dec_dynamic(const void *sbuf, int scount,
+int ompi_coll_tuned_allgatherv_intra_dec_dynamic(const void *sbuf, size_t scount,
                                                  struct ompi_datatype_t *sdtype,
-                                                 void* rbuf, const int *rcounts,
-                                                 const int *rdispls,
+                                                 void* rbuf, ompi_count_array_t rcounts,
+                                                 ompi_disp_array_t rdispls,
                                                  struct ompi_datatype_t *rdtype,
                                                  struct ompi_communicator_t *comm,
                                                  mca_coll_base_module_t *module)
 {
     mca_coll_tuned_module_t *tuned_module = (mca_coll_tuned_module_t*) module;
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream,
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
                  "ompi_coll_tuned_allgatherv_intra_dec_dynamic"));
 
     /* Check first if an algorithm is set explicitly for this collective */
@@ -560,7 +568,7 @@ int ompi_coll_tuned_allgatherv_intra_dec_dynamic(const void *sbuf, int scount,
         comsize = ompi_comm_size(comm);
         ompi_datatype_type_size (sdtype, &dsize);
         total_size = 0;
-        for (i = 0; i < comsize; i++) { total_size += dsize * rcounts[i]; }
+        for (i = 0; i < comsize; i++) { total_size += dsize * ompi_count_array_get(rcounts, i); }
 
         per_rank_size = total_size / comsize;
 
@@ -583,9 +591,9 @@ int ompi_coll_tuned_allgatherv_intra_dec_dynamic(const void *sbuf, int scount,
                                                        comm, module);
 }
 
-int ompi_coll_tuned_gather_intra_dec_dynamic(const void *sbuf, int scount,
+int ompi_coll_tuned_gather_intra_dec_dynamic(const void *sbuf, size_t scount,
                                              struct ompi_datatype_t *sdtype,
-                                             void* rbuf, int rcount,
+                                             void* rbuf, size_t rcount,
                                              struct ompi_datatype_t *rdtype,
                                              int root,
                                              struct ompi_communicator_t *comm,
@@ -593,7 +601,7 @@ int ompi_coll_tuned_gather_intra_dec_dynamic(const void *sbuf, int scount,
 {
     mca_coll_tuned_module_t *tuned_module = (mca_coll_tuned_module_t*) module;
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream,
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
                  "ompi_coll_tuned_gather_intra_dec_dynamic"));
 
     /* Check first if an algorithm is set explicitly for this collective */
@@ -634,16 +642,16 @@ int ompi_coll_tuned_gather_intra_dec_dynamic(const void *sbuf, int scount,
                                                    root, comm, module);
 }
 
-int ompi_coll_tuned_scatter_intra_dec_dynamic(const void *sbuf, int scount,
+int ompi_coll_tuned_scatter_intra_dec_dynamic(const void *sbuf, size_t scount,
                                               struct ompi_datatype_t *sdtype,
-                                              void* rbuf, int rcount,
+                                              void* rbuf, size_t rcount,
                                               struct ompi_datatype_t *rdtype,
                                               int root, struct ompi_communicator_t *comm,
                                               mca_coll_base_module_t *module)
 {
     mca_coll_tuned_module_t *tuned_module = (mca_coll_tuned_module_t*) module;
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream,
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
                  "ompi_coll_tuned_scatter_intra_dec_dynamic"));
 
     /* Check first if an algorithm is set explicitly for this collective */
@@ -684,7 +692,7 @@ int ompi_coll_tuned_scatter_intra_dec_dynamic(const void *sbuf, int scount,
                                                     root, comm, module);
 }
 
-int ompi_coll_tuned_exscan_intra_dec_dynamic(const void *sbuf, void* rbuf, int count,
+int ompi_coll_tuned_exscan_intra_dec_dynamic(const void *sbuf, void* rbuf, size_t count,
                                               struct ompi_datatype_t *dtype,
                                               struct ompi_op_t *op,
                                               struct ompi_communicator_t *comm,
@@ -692,7 +700,7 @@ int ompi_coll_tuned_exscan_intra_dec_dynamic(const void *sbuf, void* rbuf, int c
 {
     mca_coll_tuned_module_t *tuned_module = (mca_coll_tuned_module_t*) module;
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream,
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
                  "ompi_coll_tuned_exscan_intra_dec_dynamic"));
 
     /* Check first if an algorithm is set explicitly for this collective */
@@ -728,7 +736,7 @@ int ompi_coll_tuned_exscan_intra_dec_dynamic(const void *sbuf, void* rbuf, int c
                                               op, comm, module);
 }
 
-int ompi_coll_tuned_scan_intra_dec_dynamic(const void *sbuf, void* rbuf, int count,
+int ompi_coll_tuned_scan_intra_dec_dynamic(const void *sbuf, void* rbuf, size_t count,
                                            struct ompi_datatype_t *dtype,
                                            struct ompi_op_t *op,
                                            struct ompi_communicator_t *comm,
@@ -736,7 +744,7 @@ int ompi_coll_tuned_scan_intra_dec_dynamic(const void *sbuf, void* rbuf, int cou
 {
     mca_coll_tuned_module_t *tuned_module = (mca_coll_tuned_module_t*) module;
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream,
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
                  "ompi_coll_tuned_scan_intra_dec_dynamic"));
 
     /* Check first if an algorithm is set explicitly for this collective */
diff --git a/ompi/mca/coll/tuned/coll_tuned_decision_fixed.c b/ompi/mca/coll/tuned/coll_tuned_decision_fixed.c
index 1c445c0c889..fa31aef1860 100644
--- a/ompi/mca/coll/tuned/coll_tuned_decision_fixed.c
+++ b/ompi/mca/coll/tuned/coll_tuned_decision_fixed.c
@@ -16,7 +16,7 @@
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2019      Mellanox Technologies. All rights reserved.
- * Copyright (c) 2020      Amazon.com, Inc. or its affiliates.
+ * Copyright (c) 2020-2025 Amazon.com, Inc. or its affiliates.
  *                         All Rights reserved.
  * $COPYRIGHT$
  *
@@ -52,7 +52,7 @@
  *  Returns:    - MPI_SUCCESS or error code
  */
 int
-ompi_coll_tuned_allreduce_intra_dec_fixed(const void *sbuf, void *rbuf, int count,
+ompi_coll_tuned_allreduce_intra_dec_fixed(const void *sbuf, void *rbuf, size_t count,
                                           struct ompi_datatype_t *dtype,
                                           struct ompi_op_t *op,
                                           struct ompi_communicator_t *comm,
@@ -61,7 +61,8 @@ ompi_coll_tuned_allreduce_intra_dec_fixed(const void *sbuf, void *rbuf, int coun
     size_t dsize, total_dsize;
     int communicator_size, alg;
     communicator_size = ompi_comm_size(comm);
-    OPAL_OUTPUT((ompi_coll_tuned_stream, "ompi_coll_tuned_allreduce_intra_dec_fixed"));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "ompi_coll_tuned_allreduce_intra_dec_fixed"));
 
     ompi_datatype_type_size(dtype, &dsize);
     total_dsize = dsize * (ptrdiff_t)count;
@@ -225,9 +226,9 @@ ompi_coll_tuned_allreduce_intra_dec_fixed(const void *sbuf, void *rbuf, int coun
  *	Returns:	- MPI_SUCCESS or error code
  */
 
-int ompi_coll_tuned_alltoall_intra_dec_fixed(const void *sbuf, int scount,
+int ompi_coll_tuned_alltoall_intra_dec_fixed(const void *sbuf, size_t scount,
                                              struct ompi_datatype_t *sdtype,
-                                             void* rbuf, int rcount,
+                                             void* rbuf, size_t rcount,
                                              struct ompi_datatype_t *rdtype,
                                              struct ompi_communicator_t *comm,
                                              mca_coll_base_module_t *module)
@@ -415,9 +416,9 @@ int ompi_coll_tuned_alltoall_intra_dec_fixed(const void *sbuf, int scount,
  *      Accepts:        - same arguments as MPI_Alltoallv()
  *      Returns:        - MPI_SUCCESS or error code
  */
-int ompi_coll_tuned_alltoallv_intra_dec_fixed(const void *sbuf, const int *scounts, const int *sdisps,
+int ompi_coll_tuned_alltoallv_intra_dec_fixed(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdisps,
                                               struct ompi_datatype_t *sdtype,
-                                              void *rbuf, const int *rcounts, const int *rdisps,
+                                              void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
                                               struct ompi_datatype_t *rdtype,
                                               struct ompi_communicator_t *comm,
                                               mca_coll_base_module_t *module)
@@ -425,7 +426,8 @@ int ompi_coll_tuned_alltoallv_intra_dec_fixed(const void *sbuf, const int *scoun
     int communicator_size, alg;
     communicator_size = ompi_comm_size(comm);
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream, "ompi_coll_tuned_alltoallv_intra_dec_fixed com_size %d",
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "ompi_coll_tuned_alltoallv_intra_dec_fixed com_size %d",
                  communicator_size));
     /** Algorithms:
      *  {1, "basic_linear"},
@@ -467,8 +469,9 @@ int ompi_coll_tuned_barrier_intra_dec_fixed(struct ompi_communicator_t *comm,
     int communicator_size, alg;
     communicator_size = ompi_comm_size(comm);
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream, "ompi_coll_tuned_barrier_intra_dec_fixed com_size %d",
-                 communicator_size));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "ompi_coll_tuned_barrier_intra_dec_fixed com_size %d",
+        communicator_size));
     /** Algorithms:
      *  {1, "linear"},
      *  {2, "double_ring"},
@@ -509,7 +512,7 @@ int ompi_coll_tuned_barrier_intra_dec_fixed(struct ompi_communicator_t *comm,
  *	Accepts:	- same arguments as MPI_Bcast()
  *	Returns:	- MPI_SUCCESS or error code (passed from the bcast implementation)
  */
-int ompi_coll_tuned_bcast_intra_dec_fixed(void *buff, int count,
+int ompi_coll_tuned_bcast_intra_dec_fixed(void *buff, size_t count,
                                           struct ompi_datatype_t *datatype, int root,
                                           struct ompi_communicator_t *comm,
                                           mca_coll_base_module_t *module)
@@ -521,9 +524,9 @@ int ompi_coll_tuned_bcast_intra_dec_fixed(void *buff, int count,
     ompi_datatype_type_size(datatype, &dsize);
     total_dsize = dsize * (unsigned long)count;
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream, "ompi_coll_tuned_bcast_intra_dec_fixed"
-                 " root %d rank %d com_size %d",
-                 root, ompi_comm_rank(comm), communicator_size));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "ompi_coll_tuned_bcast_intra_dec_fixed root %d rank %d com_size %d",
+        root, ompi_comm_rank(comm), communicator_size));
 
     /** Algorithms:
      *  {1, "basic_linear"},
@@ -660,7 +663,7 @@ int ompi_coll_tuned_bcast_intra_dec_fixed(void *buff, int count,
  *
  */
 int ompi_coll_tuned_reduce_intra_dec_fixed( const void *sendbuf, void *recvbuf,
-                                            int count, struct ompi_datatype_t* datatype,
+                                            size_t count, struct ompi_datatype_t* datatype,
                                             struct ompi_op_t* op, int root,
                                             struct ompi_communicator_t* comm,
                                             mca_coll_base_module_t *module)
@@ -670,8 +673,9 @@ int ompi_coll_tuned_reduce_intra_dec_fixed( const void *sendbuf, void *recvbuf,
 
     communicator_size = ompi_comm_size(comm);
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream, "ompi_coll_tuned_reduce_intra_dec_fixed "
-                 "root %d rank %d com_size %d", root, ompi_comm_rank(comm), communicator_size));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "ompi_coll_tuned_reduce_intra_dec_fixed root %d rank %d com_size %d",
+        root, ompi_comm_rank(comm), communicator_size));
 
     ompi_datatype_type_size(datatype, &dsize);
     total_dsize = dsize * (ptrdiff_t)count;   /* needed for decision */
@@ -808,9 +812,10 @@ int ompi_coll_tuned_reduce_intra_dec_fixed( const void *sendbuf, void *recvbuf,
         }
     }
 
+    int faninout = 2;
     return  ompi_coll_tuned_reduce_intra_do_this (sendbuf, recvbuf, count, datatype,
                                                   op, root, comm, module,
-                                                  alg, 0, 0, 0);
+                                                  alg, faninout, 0, 0);
 }
 
 /*
@@ -822,7 +827,7 @@ int ompi_coll_tuned_reduce_intra_dec_fixed( const void *sendbuf, void *recvbuf,
  *                        the reduce scatter implementation)
  */
 int ompi_coll_tuned_reduce_scatter_intra_dec_fixed( const void *sbuf, void *rbuf,
-                                                    const int *rcounts,
+                                                    ompi_count_array_t rcounts,
                                                     struct ompi_datatype_t *dtype,
                                                     struct ompi_op_t *op,
                                                     struct ompi_communicator_t *comm,
@@ -831,13 +836,14 @@ int ompi_coll_tuned_reduce_scatter_intra_dec_fixed( const void *sbuf, void *rbuf
     int communicator_size, i, alg;
     size_t total_dsize, dsize;
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream, "ompi_coll_tuned_reduce_scatter_intra_dec_fixed"));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "ompi_coll_tuned_reduce_scatter_intra_dec_fixed"));
 
     communicator_size = ompi_comm_size(comm);
     ompi_datatype_type_size(dtype, &dsize);
     total_dsize = 0;
     for (i = 0; i < communicator_size; i++) {
-        total_dsize += rcounts[i];
+        total_dsize += ompi_count_array_get(rcounts, i);
     }
     total_dsize *= dsize;
 
@@ -970,7 +976,7 @@ int ompi_coll_tuned_reduce_scatter_intra_dec_fixed( const void *sbuf, void *rbuf
  *                        the reduce scatter implementation)
  */
 int ompi_coll_tuned_reduce_scatter_block_intra_dec_fixed(const void *sbuf, void *rbuf,
-                                                         int rcount,
+                                                         size_t rcount,
                                                          struct ompi_datatype_t *dtype,
                                                          struct ompi_op_t *op,
                                                          struct ompi_communicator_t *comm,
@@ -979,7 +985,8 @@ int ompi_coll_tuned_reduce_scatter_block_intra_dec_fixed(const void *sbuf, void
     int communicator_size, alg;
     size_t dsize, total_dsize;
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream, "ompi_coll_tuned_reduce_scatter_block_intra_dec_fixed"));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "ompi_coll_tuned_reduce_scatter_block_intra_dec_fixed"));
 
 
     ompi_datatype_type_size(dtype, &dsize);
@@ -1090,9 +1097,9 @@ int ompi_coll_tuned_reduce_scatter_block_intra_dec_fixed(const void *sbuf, void
  *                        internal allgather function.
  */
 
-int ompi_coll_tuned_allgather_intra_dec_fixed(const void *sbuf, int scount,
+int ompi_coll_tuned_allgather_intra_dec_fixed(const void *sbuf, size_t scount,
                                               struct ompi_datatype_t *sdtype,
-                                              void* rbuf, int rcount,
+                                              void* rbuf, size_t rcount,
                                               struct ompi_datatype_t *rdtype,
                                               struct ompi_communicator_t *comm,
                                               mca_coll_base_module_t *module)
@@ -1219,12 +1226,14 @@ int ompi_coll_tuned_allgather_intra_dec_fixed(const void *sbuf, int scount,
         }
     }
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream, "ompi_coll_tuned_allgather_intra_dec_fixed"
-                 " rank %d com_size %d", ompi_comm_rank(comm), communicator_size));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "ompi_coll_tuned_allgather_intra_dec_fixed rank %d com_size %d",
+        ompi_comm_rank(comm), communicator_size));
 
+    int faninout = 2;
     return ompi_coll_tuned_allgather_intra_do_this(sbuf, scount, sdtype,
                                                    rbuf, rcount, rdtype,
-                                                   comm, module, alg, 0, 0);
+                                                   comm, module, alg, faninout, 0);
 }
 
 /*
@@ -1236,10 +1245,10 @@ int ompi_coll_tuned_allgather_intra_dec_fixed(const void *sbuf, int scount,
  *                        internal allgatherv function.
  */
 
-int ompi_coll_tuned_allgatherv_intra_dec_fixed(const void *sbuf, int scount,
+int ompi_coll_tuned_allgatherv_intra_dec_fixed(const void *sbuf, size_t scount,
                                                struct ompi_datatype_t *sdtype,
-                                               void* rbuf, const int *rcounts,
-                                               const int *rdispls,
+                                               void* rbuf, ompi_count_array_t rcounts,
+                                               ompi_disp_array_t rdispls,
                                                struct ompi_datatype_t *rdtype,
                                                struct ompi_communicator_t *comm,
                                                mca_coll_base_module_t *module)
@@ -1256,7 +1265,7 @@ int ompi_coll_tuned_allgatherv_intra_dec_fixed(const void *sbuf, int scount,
     }
 
     total_dsize = 0;
-    for (i = 0; i < communicator_size; i++) { total_dsize += dsize * rcounts[i]; }
+    for (i = 0; i < communicator_size; i++) { total_dsize += dsize * ompi_count_array_get(rcounts, i); }
 
     /* use the per-rank data size as basis, similar to allgather */
     per_rank_dsize = total_dsize / communicator_size;
@@ -1356,7 +1365,7 @@ int ompi_coll_tuned_allgatherv_intra_dec_fixed(const void *sbuf, int scount,
         }
     }
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream,
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
                  "ompi_coll_tuned_allgatherv_intra_dec_fixed"
                  " rank %d com_size %d", ompi_comm_rank(comm), communicator_size));
 
@@ -1376,9 +1385,9 @@ int ompi_coll_tuned_allgatherv_intra_dec_fixed(const void *sbuf, int scount,
  *                        internal allgather function.
  */
 
-int ompi_coll_tuned_gather_intra_dec_fixed(const void *sbuf, int scount,
+int ompi_coll_tuned_gather_intra_dec_fixed(const void *sbuf, size_t scount,
                                            struct ompi_datatype_t *sdtype,
-                                           void* rbuf, int rcount,
+                                           void* rbuf, size_t rcount,
                                            struct ompi_datatype_t *rdtype,
                                            int root,
                                            struct ompi_communicator_t *comm,
@@ -1387,7 +1396,7 @@ int ompi_coll_tuned_gather_intra_dec_fixed(const void *sbuf, int scount,
     int communicator_size, alg, rank;
     size_t dsize, total_dsize;
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream,
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
                  "ompi_coll_tuned_gather_intra_dec_fixed"));
 
     communicator_size = ompi_comm_size(comm);
@@ -1465,9 +1474,9 @@ int ompi_coll_tuned_gather_intra_dec_fixed(const void *sbuf, int scount,
  *                        internal allgather function.
  */
 
-int ompi_coll_tuned_scatter_intra_dec_fixed(const void *sbuf, int scount,
+int ompi_coll_tuned_scatter_intra_dec_fixed(const void *sbuf, size_t scount,
                                             struct ompi_datatype_t *sdtype,
-                                            void* rbuf, int rcount,
+                                            void* rbuf, size_t rcount,
                                             struct ompi_datatype_t *rdtype,
                                             int root, struct ompi_communicator_t *comm,
                                             mca_coll_base_module_t *module)
@@ -1475,7 +1484,7 @@ int ompi_coll_tuned_scatter_intra_dec_fixed(const void *sbuf, int scount,
     int communicator_size, alg, rank;
     size_t dsize, total_dsize;
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream,
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
                  "ompi_coll_tuned_scatter_intra_dec_fixed"));
 
     communicator_size = ompi_comm_size(comm);
diff --git a/ompi/mca/coll/tuned/coll_tuned_dynamic_file.c b/ompi/mca/coll/tuned/coll_tuned_dynamic_file.c
index a259c789ac2..7439c67951e 100644
--- a/ompi/mca/coll/tuned/coll_tuned_dynamic_file.c
+++ b/ompi/mca/coll/tuned/coll_tuned_dynamic_file.c
@@ -11,6 +11,10 @@
  *                         All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
+ * Copyright (c) 2024      NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2025      Amazon.com, Inc. or its affiliates.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -25,6 +29,9 @@
 #include "mpi.h"
 #include "ompi/mca/mca.h"
 #include "coll_tuned.h"
+#include "ompi/mca/coll/base/coll_base_functions.h"
+#include "opal/util/json/opal_json.h"
+
 
 /* need to include our own topo prototypes so we can malloc data on the comm correctly */
 #include "ompi/mca/coll/base/coll_base_topo.h"
@@ -41,24 +48,359 @@
 static int fileline=0; /* used for verbose error messages */
 
 #define getnext(fptr, pval)        ompi_coll_base_file_getnext_long(fptr, &fileline, pval)
+#define isnext_digit(fptr)         ompi_coll_base_file_peek_next_char_isdigit(fptr)
+
+
+static int coll_tuned_read_alg(const opal_json_t *msg_rule, ompi_coll_msg_rule_t *msg_p, int coll_id) {
+    const char *RESULT_ALG_FIELD = "alg";
+    const opal_json_t *alg_prop = NULL;
+    int rc, rc_as_str, rc_as_int, rc_validation;
+    const char* string_buf;
+    size_t string_len;
+    char int_as_str[24];
+    int64_t int_val;
+
+    rc = opal_json_get_key( msg_rule, RESULT_ALG_FIELD, &alg_prop);
+    if (rc != OPAL_SUCCESS) {
+        opal_output_verbose(1, ompi_coll_tuned_stream,
+                "The \"alg\" field is required, but was not found." );
+        return OPAL_ERROR;
+    }
+    rc_as_str = opal_json_read_string(alg_prop, &string_buf, &string_len);
+    rc_as_int = opal_json_read_integer(alg_prop, &int_val);
+    opal_json_free(&alg_prop);
+    if (rc_as_str == OPAL_SUCCESS) {
+        rc_validation = coll_tuned_alg_from_str( coll_id, string_buf, &msg_p->result_alg );
+    } else if (rc_as_int == OPAL_SUCCESS) {
+        rc_validation = coll_tuned_alg_to_str( coll_id, int_val, NULL );
+        if (rc_validation != OPAL_SUCCESS) {
+            snprintf(int_as_str, 23, "%ld", int_val);
+            int_as_str[23] = '\0';
+            string_buf = int_as_str;
+        } else {
+            msg_p->result_alg = int_val;
+        }
+    } else {
+        opal_output_verbose(1, ompi_coll_tuned_stream,
+                "The \"alg\" field must be either a string or an integer, but it is something else." );
+        return OPAL_ERROR;
+    }
+    if (rc_validation != OPAL_SUCCESS) {
+        opal_output_verbose(1, ompi_coll_tuned_stream,
+        "Algorithm (%s) provided for collective \"%s\" is not valid.  "
+        "Check documentation for valid configurations of coll_tuned_%s_algorithm.  Ignoring this rule.",
+        string_buf, mca_coll_base_colltype_to_str(coll_id), mca_coll_base_colltype_to_str(coll_id) );
+        /* the rationale for disabling the rule is to allow two ompi versions to use the same file. */
+        /* disable the rule by making an impossible condition: */
+        msg_p->msg_size_min = COLL_RULES_MESSAGE_SIZE_INF;
+        msg_p->msg_size_max = 0;
+        rc = OPAL_SUCCESS;
+    }
+    return OPAL_SUCCESS;
+}
+
+/* returns the value in the field.
+   *rc is set as follows:
+    OPAL_SUCCESS (value found and parsed)
+    OPAL_ERROR   (value found, but not an integer)
+    OPAL_ERR_DATA_VALUE_NOT_FOUND (value not found)
+
+   When optional==0, and the field is not found, or when the field is not
+   an integer, then a warning message is printed.
+ */
+static int coll_tuned_get_json_integer_field(const opal_json_t *parent,
+                                                 const char* fieldname,
+                                                 int optional,
+                                                 int64_t *val ) {
+    const opal_json_t *json_val;
+    int rc;
+    const char *string_buf;
+    size_t str_len;
+    int rc_as_int, rc_as_str;
+    rc = opal_json_get_key( parent, fieldname, &json_val);
+    if (rc == OPAL_SUCCESS) {
+        rc_as_int = opal_json_read_integer(json_val, val);
+        if ( rc_as_int != OPAL_SUCCESS ) {
+            rc_as_str = opal_json_read_string(json_val, &string_buf, &str_len);
+        }
+        opal_json_free(&json_val);
+        if (rc_as_int == OPAL_SUCCESS) {
+            return rc_as_int;
+        } else if ( rc_as_str == OPAL_SUCCESS) {
+            if (strncmp("inf",string_buf, 3)==0 ||
+                strncmp("Inf",string_buf, 3)==0 ||
+                strncmp("INF",string_buf, 3)==0 ) {
+                    *val = COLL_RULES_MESSAGE_SIZE_INF;
+                    return rc_as_str;
+            }
+        }
+        opal_output_verbose(1, ompi_coll_tuned_stream,
+            "Found field \"%s\" as expected, but could not interpret it as an integer or \"inf\".",
+            fieldname);
+        return OPAL_ERROR;
+    }
+    if (optional == 0) {
+        opal_output_verbose(1, ompi_coll_tuned_stream,
+            "Missing the required field \"%s\".",
+            fieldname);
+    }
+    return OPAL_ERR_DATA_VALUE_NOT_FOUND;
+}
+
+static int coll_tuned_read_message_size_rule(   ompi_coll_msg_rule_t *msg_p,
+                                                const opal_json_t *msg_rule, int coll_id ) {
+    int rc;
+    const char *MSG_SIZE_FIELD = "msg_size_min";
+    const char *MAX_MSG_SIZE_FIELD = "msg_size_max";
+    const char *TOPO_FANINOUT_FIELD = "faninout";
+    const char *SEGSIZE_FIELD = "seg_size";
+    const char *MAX_REQUESTS_FIELD = "reqs";
+    const int OPTIONAL = 1;
+    // const int REQUIRED = 0;
+    int64_t int_val;
+
+
+#define OPTIONAL_READ(field, target) \
+    rc = coll_tuned_get_json_integer_field(msg_rule, field, OPTIONAL, &int_val); \
+    if (rc == OPAL_ERROR) {return rc;} \
+    if (rc == OPAL_SUCCESS) {target = int_val;}
+#define REQUIRED_READ(field, target) \
+    rc = coll_tuned_get_json_integer_field(msg_rule, field, REQUIRED, &int_val); \
+    if (rc != OPAL_SUCCESS) {return rc;} \
+    if (rc == OPAL_SUCCESS) {target = int_val;}
+
+    msg_p->msg_size_min = 0;
+    OPTIONAL_READ( MSG_SIZE_FIELD, msg_p->msg_size_min )
+
+    msg_p->msg_size_max = COLL_RULES_MESSAGE_SIZE_INF;
+    OPTIONAL_READ( MAX_MSG_SIZE_FIELD, msg_p->msg_size_max )
+
+    msg_p->result_topo_faninout = 0;
+    OPTIONAL_READ( TOPO_FANINOUT_FIELD, msg_p->result_topo_faninout )
+
+    msg_p->result_segsize = 0;
+    OPTIONAL_READ( SEGSIZE_FIELD, msg_p->result_segsize )
+
+    msg_p->result_max_requests = 0;
+    OPTIONAL_READ( MAX_REQUESTS_FIELD, msg_p->result_max_requests )
+
+    rc = coll_tuned_read_alg( msg_rule, msg_p, coll_id );
+
+    return OPAL_SUCCESS;
+
+#undef REQUIRED_READ
+#undef OPTIONAL_READ
+}
+
+static int coll_tuned_get_comm_distribution(const opal_json_t *parent, enum comm_rank_distro_t *distro) {
+    int rc;
+    const opal_json_t *json_val;
+    const char *string_buf;
+    size_t string_len;
+    const char *comm_rank_distro_name = "comm_rank_distribution";
+
+    rc = opal_json_get_key( parent, comm_rank_distro_name, &json_val);
+    if (rc == OPAL_ERROR) {
+        *distro = COLL_RULES_DISTRO_ANY;
+        return OPAL_SUCCESS;
+    }
+    rc = opal_json_read_string(json_val, &string_buf, &string_len);
+    opal_json_free(&json_val);
+    if ( rc == OPAL_SUCCESS) {
+        rc = coll_rules_comm_rank_distro_from_str(string_buf, distro);
+
+        if (rc == OPAL_ERROR) {
+            opal_output_verbose(1, ompi_coll_tuned_stream,
+                    "Unrecognized value for field \"%s\".  Got \"%s\", assuming \"%s\".",
+                        comm_rank_distro_name, string_buf,
+                        coll_rules_comm_rank_distro_to_str(COLL_RULES_DISTRO_ANY) );
+            *distro = COLL_RULES_DISTRO_ANY;
+            rc = OPAL_SUCCESS;
+        }
+    }
+    return rc;
+}
+
+
+static int ompi_coll_tuned_read_rules_json (const opal_json_t *json_root, ompi_coll_alg_rule_t** rules) {
+
+    int rc = OPAL_ERROR;
+
+    /* complete table of rules */
+    ompi_coll_alg_rule_t *alg_rules = (ompi_coll_alg_rule_t*) NULL;
+
+    /* individual pointers to sections of rules */
+    ompi_coll_alg_rule_t *alg_p = (ompi_coll_alg_rule_t*) NULL;
+    ompi_coll_com_rule_t *com_p = (ompi_coll_com_rule_t*) NULL;
+    ompi_coll_msg_rule_t *msg_p = (ompi_coll_msg_rule_t*) NULL;
+
+    size_t jmsg_rule = 0;
+    size_t jcomm_rule = 0;
+    size_t jcol = 0;
+    int64_t int_val;
+    const char* coll_name = "";
+
+    alg_rules = ompi_coll_tuned_mk_alg_rules(COLLCOUNT);
+
+    const opal_json_t *collectives_obj = NULL;
+    const opal_json_t *comm_rule_array = NULL;
+    const opal_json_t *comm_rule = NULL;
+    const opal_json_t *msg_size_array = NULL;
+
+    size_t num_collectives = 0;
+    size_t num_comm_rules = 0;
+    rc = opal_json_get_key(json_root, "collectives", &collectives_obj);
+    if (rc != OPAL_SUCCESS) {
+        opal_output_verbose(1, ompi_coll_tuned_stream,
+            "Required top-level field \"collectives\" was not found.");
+        return OPAL_ERROR;
+    }
+
+    rc = opal_json_get_container_size(collectives_obj, &num_collectives);
+    if (rc != OPAL_SUCCESS || OPAL_JSON_OBJECT != collectives_obj->type) {
+        opal_output_verbose(1, ompi_coll_tuned_stream,
+            "The \"collectives\" field must be a dictionary of collectives.");
+        opal_json_free(&collectives_obj);
+        return OPAL_ERROR;
+    }
+
+    for(jcol = 0; jcol < num_collectives; jcol++ ) {
+        int coll_id;
+        rc = opal_json_get_key_by_index( collectives_obj, jcol, &coll_name, &comm_rule_array);
+        if (rc) {
+            opal_output_verbose(1, ompi_coll_tuned_stream,
+                "Internal json error when attempting to parse the collective at index %ld\n",
+                jcol);
+            goto error_cleanup;
+        }
+        coll_id = mca_coll_base_name_to_colltype(coll_name);
+        if (coll_id < 0) {
+            opal_output_verbose(1, ompi_coll_tuned_stream,
+                "Unrecognized collective: \"%s\". Use all lowercase such as \"allgather\"",
+                coll_name);
+            goto error_cleanup;
+        }
+
+        alg_p = &alg_rules[coll_id];
+        alg_p->coll_id = coll_id;
+        rc = opal_json_get_container_size(comm_rule_array, &num_comm_rules);
+        if (rc != OPAL_SUCCESS || OPAL_JSON_ARRAY != comm_rule_array->type) {
+            opal_output_verbose(1, ompi_coll_tuned_stream,
+                "Problem parsing the collective at index %ld (for %s).  Expected an array of comm-related rules.",
+                jcol, mca_coll_base_colltype_to_str(coll_id) );
+            goto error_cleanup;
+        }
+        alg_p->n_com_sizes = (int)num_comm_rules;
+        alg_p->com_rules = ompi_coll_tuned_mk_com_rules (num_comm_rules, coll_id);
+
+        for (jcomm_rule=0; jcomm_rule < num_comm_rules; jcomm_rule++) {
+            size_t num_msg_rules;
+            rc = opal_json_get_index(comm_rule_array, jcomm_rule, &comm_rule);
+            com_p = &(alg_p->com_rules[jcomm_rule]);
+
+            com_p->mpi_comsize_min = 0;
+            rc = coll_tuned_get_json_integer_field(comm_rule, "comm_size_min", 1, &int_val);
+            if (rc == OPAL_ERROR) { goto error_bad_comm_rule; }
+            if (rc == OPAL_SUCCESS) { com_p->mpi_comsize_min = int_val; }
+
+
+            com_p->mpi_comsize_max = INT_MAX;
+            rc = coll_tuned_get_json_integer_field(comm_rule, "comm_size_max", 1, &int_val);
+            if (rc == OPAL_ERROR) { goto error_bad_comm_rule; }
+            if (rc == OPAL_SUCCESS) { com_p->mpi_comsize_max = int_val; }
+
+            rc = coll_tuned_get_comm_distribution(comm_rule, &com_p->comm_rank_distribution);
+            if (rc == OPAL_ERROR) { goto error_bad_comm_rule; }
+
+            rc = opal_json_get_key( comm_rule, "rules", &msg_size_array);
+            if (rc != OPAL_SUCCESS) {
+                opal_output_verbose(1, ompi_coll_tuned_stream,
+                    "Expected a set of message rules in this communicator rule");
+                goto error_bad_comm_rule;
+            }
+            rc = opal_json_get_container_size(msg_size_array, &num_msg_rules);
+            if (rc != OPAL_SUCCESS) { goto error_bad_comm_rule; }
+
+            com_p->n_rules = num_msg_rules;
+            com_p->msg_rules = ompi_coll_tuned_mk_msg_rules (num_msg_rules, coll_id, com_p->mpi_comsize_min);
+
+
+            for (jmsg_rule=0; jmsg_rule < num_msg_rules; jmsg_rule++) {
+                const opal_json_t *msg_rule;
+                msg_p = &(com_p->msg_rules[jmsg_rule]);
+                rc = opal_json_get_index(msg_size_array, jmsg_rule, &msg_rule);
+
+                rc = coll_tuned_read_message_size_rule( msg_p, msg_rule, coll_id);
+                opal_json_free(&msg_rule);
+                if (rc != OPAL_SUCCESS) {
+                    goto error_bad_message_rule;
+                }
+            }
+
+            opal_json_free(&msg_size_array);
+            opal_json_free(&comm_rule);
+        }
+
+        opal_json_free(&comm_rule_array);
+    }
+    *rules = alg_rules;
+
+    opal_json_free(&collectives_obj);
+    return rc;
+error_bad_message_rule:
+    opal_output_verbose(1, ompi_coll_tuned_stream,
+        "Problem occurred within collective %s, "
+        "comm_size_min=%d, comm_size_max=%d (entry number %ld in the comm_size array), "
+        "message size entry number %ld.", coll_name, com_p->mpi_comsize_min, com_p->mpi_comsize_max, 1+jcomm_rule, 1+jmsg_rule);
+    goto error_cleanup;
+error_bad_comm_rule:
+    opal_output_verbose(1, ompi_coll_tuned_stream,
+        "Problem occurred within collective %s, "
+        "in entry number %ld of the comm_size array", coll_name, 1+jcomm_rule);
+    goto error_cleanup;
+error_cleanup:
+    opal_json_free(&msg_size_array);
+    opal_json_free(&comm_rule);
+    opal_json_free(&comm_rule_array);
+    opal_json_free(&collectives_obj);
+    return OMPI_ERROR;
+}
 
 /*
  * Reads a rule file called fname
- * Builds the algorithm rule table for a max of n_collectives
+ * The rule file defines a set of sets of rules. The outer set is keyed on
+ * communicator size while the inner set is keyed on message size.  When a
+ * communicator is constructed its size is used to look up the nested set of
+ * message size keyed rules.  When a collective is called the message size
+ * determined from its call arguments are used to lookup a specific rule in the
+ * inner set.
+ *
+ * Rules for communicator and message sizes 0 and N (where N is the larger than
+ * largest key you provide) can be specified to fall back to the fixed decision
+ * framework above and below the communicator and message size ranges of
+ * interest.
  *
  * If an error occurs it removes rule table and then exits with a very verbose
- * error message (this stops the user using a half baked rule table
+ * error message. this stops the user using a half baked rule table.
  *
- * Returns the number of actual collectives that a rule exists for
- * (note 0 is NOT an error)
+ * Returns OPAL_ERROR or OPAL_SUCCESS
  *
  */
 
-int ompi_coll_tuned_read_rules_config_file (char *fname, ompi_coll_alg_rule_t** rules, int n_collectives)
+static int ompi_coll_tuned_read_rules_config_file_classic (char *fname, ompi_coll_alg_rule_t** rules)
 {
-    long CI, NCS, CS, ALG, NMS, FANINOUT, X, MS, SS;
+    long NCOL = 0,      /* number of collectives for which rules are provided  */
+         COLID = 0,     /* identifies the collective type to associate the rules with */
+         NCOMSIZES = 0, /* number of sets of message size rules. the key is communicator size */
+         COMSIZE = 0,   /* communicator size, the key identifying a specific set of message size rules. */
+         NMSGSIZES = 0, /* number of message size rules in the set. */
+         MSGSIZE = 0,   /* message size, the key identifying a specific rule in the set. */
+         ALG = 0,       /* the collective specific algorithm to use */
+         FANINOUT = 0,  /* algorithm specific tuning parameter */
+         SEGSIZE = 0,   /* algorithm specific tuning parameter */
+         MAXREQ = 0;    /* algorithm specific tuning parameter */
     FILE *fptr = (FILE*) NULL;
-    int x, ncs, nms;
+    int x, ncs, nms, version;
 
     ompi_coll_alg_rule_t *alg_rules = (ompi_coll_alg_rule_t*) NULL;   /* complete table of rules */
 
@@ -73,161 +415,228 @@ int ompi_coll_tuned_read_rules_config_file (char *fname, ompi_coll_alg_rule_t**
     int total_msg_count = 0;
 
     if (!fname) {
-        OPAL_OUTPUT((ompi_coll_tuned_stream,"Gave NULL as rule table configuration file for tuned collectives... ignoring!\n"));
-        return (-1);
+        opal_output_verbose(1, ompi_coll_tuned_stream,
+            "Gave NULL as rule table configuration file for tuned collectives... ignoring!\n");
+        return OPAL_ERROR;
     }
 
     if (!rules) {
-        OPAL_OUTPUT((ompi_coll_tuned_stream,"Gave NULL as rule table result ptr!... ignoring!\n"));
-        return (-2);
-    }
-
-    if (n_collectives<1) {
-        OPAL_OUTPUT((ompi_coll_tuned_stream,"Gave %d as max number of collectives in the rule table configuration file for tuned collectives!... ignoring!\n", n_collectives));
-        return (-3);
+        opal_output_verbose(1, ompi_coll_tuned_stream,
+            "Gave NULL as rule table result ptr!... ignoring!\n");
+        return OPAL_ERROR;
     }
 
     fptr = fopen (fname, "r");
     if (!fptr) {
-        OPAL_OUTPUT((ompi_coll_tuned_stream,"Cannot read rules file [%s]\n", fname));
+        opal_output_verbose(1, ompi_coll_tuned_stream,
+            "Cannot read rules file [%s]\n", fname);
         goto on_file_error;
     }
 
     /* make space and init the algorithm rules for each of the n_collectives MPI collectives */
-    alg_rules = ompi_coll_tuned_mk_alg_rules (n_collectives);
+    alg_rules = ompi_coll_tuned_mk_alg_rules(COLLCOUNT);
     if (NULL == alg_rules) {
-        OPAL_OUTPUT((ompi_coll_tuned_stream,"Cannot allocate rules for file [%s]\n", fname));
+        opal_output_verbose(1, ompi_coll_tuned_stream,
+            "Cannot allocate rules for file [%s]\n", fname);
         goto on_file_error;
     }
 
-    if( (getnext(fptr, &X) < 0) || (X < 0) ) {
-        OPAL_OUTPUT((ompi_coll_tuned_stream,"Could not read number of collectives in configuration file around line %d\n", fileline));
+    /* consume the optional version identifier */
+    if (0 == fscanf(fptr, "rule-file-version-%d", &version)) {
+        version = 1;
+    }
+
+    /* get the number of collectives for which rules are provided in the file */
+    if( (getnext(fptr, &NCOL) < 0) || (NCOL < 0) ) {
+        opal_output_verbose(1, ompi_coll_tuned_stream,
+            "Could not read number of collectives in configuration file around line %d\n", fileline);
         goto on_file_error;
     }
-    if (X>n_collectives) {
-        OPAL_OUTPUT((ompi_coll_tuned_stream,"Number of collectives in configuration file %ld is greater than number of MPI collectives possible %d ??? error around line %d\n", X, n_collectives, fileline));
+    if (NCOL>COLLCOUNT) {
+        opal_output_verbose(1, ompi_coll_tuned_stream,
+            "Number of collectives in configuration file %ld is greater than number of MPI collectives possible %d ??? error around line %d\n",
+            NCOL, COLLCOUNT, fileline);
         goto on_file_error;
     }
 
-    for (x=0;x<X;x++) { /* for each collective */
+    for (x=0;x<NCOL;x++) { /* for each collective */
 
-        if( (getnext(fptr, &CI) < 0) || (CI < 0) ) {
-            OPAL_OUTPUT((ompi_coll_tuned_stream,"Could not read next Collective id in configuration file around line %d\n", fileline));
+        /* get the collective for which rules are being provided */
+        if( (getnext(fptr, &COLID) < 0) || (COLID < 0) ) {
+            opal_output_verbose(1, ompi_coll_tuned_stream,
+                "Could not read next Collective id in configuration file around line %d\n", fileline);
             goto on_file_error;
         }
-        if (CI>=n_collectives) {
-            OPAL_OUTPUT((ompi_coll_tuned_stream,"Collective id in configuration file %ld is greater than MPI collectives possible %d. Error around line %d\n", CI, n_collectives, fileline));
+        if (COLID>=COLLCOUNT) {
+            opal_output_verbose(1, ompi_coll_tuned_stream,
+                "Collective id in configuration file %ld is greater than MPI collectives possible %d. Error around line %d\n", COLID, COLLCOUNT, fileline);
             goto on_file_error;
         }
 
-        if (alg_rules[CI].alg_rule_id != CI) {
-            OPAL_OUTPUT((ompi_coll_tuned_stream, "Internal error in handling collective ID %ld\n", CI));
+        if (alg_rules[COLID].coll_id != COLID) {
+            opal_output_verbose(1, ompi_coll_tuned_stream,
+                "Internal error in handling collective ID %ld\n", COLID);
             goto on_file_error;
         }
-        OPAL_OUTPUT((ompi_coll_tuned_stream, "Reading dynamic rule for collective ID %ld\n", CI));
-        alg_p = &alg_rules[CI];
+        opal_output_verbose(25, ompi_coll_tuned_stream,
+            "Reading dynamic rule for collective ID %ld\n", COLID);
+        alg_p = &alg_rules[COLID];
 
-        alg_p->alg_rule_id = CI;
+        alg_p->coll_id = COLID;
         alg_p->n_com_sizes = 0;
         alg_p->com_rules = (ompi_coll_com_rule_t *) NULL;
 
-        if( (getnext (fptr, &NCS) < 0) || (NCS < 0) ) {
-            OPAL_OUTPUT((ompi_coll_tuned_stream,"Could not read count of communicators for collective ID %ld at around line %d\n", CI, fileline));
+        /* get the number of communicator sizes for which a set of rules are to be provided */
+        if( (getnext (fptr, &NCOMSIZES) < 0) || (NCOMSIZES < 0) ) {
+            opal_output_verbose(1, ompi_coll_tuned_stream,
+                "Could not read count of communicators for collective ID %ld at around line %d\n", COLID, fileline);
+            goto on_file_error;
+        }
+        opal_output_verbose(25, ompi_coll_tuned_stream,
+            "Read communicator count %ld for dynamic rule for collective ID %ld\n", NCOMSIZES, COLID);
+        if( NCOMSIZES > INT_MAX) {
+            opal_output_verbose(25, ompi_coll_tuned_stream,
+                "Refusing to proceed: suspiciously large number found for the number of communicator-based rules: %ld\n", NCOMSIZES);
             goto on_file_error;
         }
-        OPAL_OUTPUT((ompi_coll_tuned_stream, "Read communicator count %ld for dynamic rule for collective ID %ld\n", NCS, CI));
-        alg_p->n_com_sizes = NCS;
-        alg_p->com_rules = ompi_coll_tuned_mk_com_rules (NCS, CI);
+        alg_p->n_com_sizes = NCOMSIZES;
+        alg_p->com_rules = ompi_coll_tuned_mk_com_rules (NCOMSIZES, COLID);
         if (NULL == alg_p->com_rules) {
-            OPAL_OUTPUT((ompi_coll_tuned_stream,"Cannot allocate com rules for file [%s]\n", fname));
+            opal_output_verbose(1, ompi_coll_tuned_stream,
+                "Cannot allocate com rules for file [%s]\n", fname);
             goto on_file_error;
         }
 
-        for (ncs=0;ncs<NCS;ncs++) {	/* for each comm size */
+        for (ncs=0;ncs<NCOMSIZES;ncs++) {	/* for each comm size */
 
             com_p = &(alg_p->com_rules[ncs]);
 
-            if( (getnext (fptr, &CS) < 0) || (CS < 0) ) {
-                OPAL_OUTPUT((ompi_coll_tuned_stream,"Could not read communicator size for collective ID %ld com rule %d at around line %d\n", CI, ncs, fileline));
+            /* get the communicator size to associate the set of rules with */
+            if( (getnext (fptr, &COMSIZE) < 0) || (COMSIZE < 0) ) {
+                opal_output_verbose(1, ompi_coll_tuned_stream,
+                    "Could not read communicator size for collective ID %ld com rule %d at around line %d\n", COLID, ncs, fileline);
                 goto on_file_error;
             }
 
-            com_p->mpi_comsize = CS;
+            if (ncs > 0) {
+                com_p->mpi_comsize_min = alg_p->com_rules[ncs-1].mpi_comsize_max + 1;
+            } else {
+                com_p->mpi_comsize_min = 0;
+            }
+            if (ncs == NCOMSIZES-1) {
+                com_p->mpi_comsize_max = INT_MAX;
+            } else {
+                com_p->mpi_comsize_max = COMSIZE - 1;
+            }
 
-            if( (getnext (fptr, &NMS) < 0) || (NMS < 0) ) {
-                OPAL_OUTPUT((ompi_coll_tuned_stream,"Could not read number of message sizes for collective ID %ld com rule %d at around line %d\n", CI, ncs, fileline));
+            /* get the number of message sizes to specify rules for. inner set size */
+            if( (getnext (fptr, &NMSGSIZES) < 0) || (NMSGSIZES < 0) ) {
+                opal_output_verbose(1, ompi_coll_tuned_stream,
+                    "Could not read number of message sizes for collective ID %ld com rule %d at around line %d\n", COLID, ncs, fileline);
                 goto on_file_error;
             }
-            OPAL_OUTPUT((ompi_coll_tuned_stream, "Read message count %ld for dynamic rule for collective ID %ld and comm size %ld\n",
-                         NMS, CI, CS));
-            com_p->n_msg_sizes = NMS;
-            com_p->msg_rules = ompi_coll_tuned_mk_msg_rules (NMS, CI, ncs, CS);
+            opal_output_verbose(25, ompi_coll_tuned_stream,
+                "Read message count %ld for dynamic rule for collective ID %ld and comm size %ld\n",
+                         NMSGSIZES, COLID, COMSIZE);
+            com_p->n_rules = NMSGSIZES;
+            com_p->msg_rules = ompi_coll_tuned_mk_msg_rules (NMSGSIZES, COLID, COMSIZE);
             if (NULL == com_p->msg_rules) {
-                OPAL_OUTPUT((ompi_coll_tuned_stream,"Cannot allocate msg rules for file [%s]\n", fname));
+                opal_output_verbose(1, ompi_coll_tuned_stream,
+                    "Cannot allocate msg rules for file [%s]\n", fname);
                 goto on_file_error;
             }
 
             msg_p = com_p->msg_rules;
 
-            for (nms=0;nms<NMS;nms++) {	/* for each msg size */
+            for (nms=0;nms<NMSGSIZES;nms++) {	/* for each msg size */
 
                 msg_p = &(com_p->msg_rules[nms]);
 
-                if( (getnext (fptr, &MS) < 0) || (MS < 0) ) {
-                    OPAL_OUTPUT((ompi_coll_tuned_stream,"Could not read message size for collective ID %ld com rule %d msg rule %d at around line %d\n", CI, ncs, nms, fileline));
+                /* read the message size to associate the rule with */
+                if( (getnext (fptr, &MSGSIZE) < 0) || (MSGSIZE < 0) ) {
+                    opal_output_verbose(1, ompi_coll_tuned_stream,
+                        "Could not read message size for collective ID %ld com rule %d msg rule %d at around line %d\n", COLID, ncs, nms, fileline);
                     goto on_file_error;
                 }
-                msg_p->msg_size = (size_t)MS;
+                msg_p->msg_size_min = (size_t)MSGSIZE;
+                msg_p->msg_size_max = COLL_RULES_MESSAGE_SIZE_INF;
+                if (nms > 0) {
+                    com_p->msg_rules[nms-1].msg_size_max = msg_p->msg_size_min - 1;
+                }
 
+                /* read the collective specific algorithm identifier */
                 if( (getnext (fptr, &ALG) < 0) || (ALG < 0) ) {
-                    OPAL_OUTPUT((ompi_coll_tuned_stream,"Could not read target algorithm method for collective ID %ld com rule %d msg rule %d at around line %d\n", CI, ncs, nms, fileline));
+                    opal_output_verbose(1, ompi_coll_tuned_stream,
+                        "Could not read target algorithm method for collective ID %ld com rule %d msg rule %d at around line %d\n", COLID, ncs, nms, fileline);
                     goto on_file_error;
                 }
                 msg_p->result_alg = ALG;
 
+                /* read faninout tuning parameter. required */
                 if( (getnext (fptr, &FANINOUT) < 0) || (FANINOUT < 0) ) {
-                    OPAL_OUTPUT((ompi_coll_tuned_stream,"Could not read fan in/out topo for collective ID %ld com rule %d msg rule %d at around line %d\n", CI, ncs, nms, fileline));
+                    opal_output_verbose(1, ompi_coll_tuned_stream,
+                        "Could not read fan in/out topo for collective ID %ld com rule %d msg rule %d at around line %d\n", COLID, ncs, nms, fileline);
                     goto on_file_error;
                 }
                 msg_p->result_topo_faninout = FANINOUT;
 
-                if( (getnext (fptr, &SS) < 0) || (SS < 0) ) {
-                    OPAL_OUTPUT((ompi_coll_tuned_stream,"Could not read target segment size for collective ID %ld com rule %d msg rule %d at around line %d\n", CI, ncs, nms, fileline));
+                /* read segsize tuning parameter. required */
+                if( (getnext (fptr, &SEGSIZE) < 0) || (SEGSIZE < 0) ) {
+                    opal_output_verbose(1, ompi_coll_tuned_stream,
+                        "Could not read target segment size for collective ID %ld com rule %d msg rule %d at around line %d\n", COLID, ncs, nms, fileline);
                     goto on_file_error;
                 }
-                msg_p->result_segsize = SS;
+                msg_p->result_segsize = SEGSIZE;
+
+                /* read the max requests tuning parameter. optional */
+                msg_p->result_max_requests = ompi_coll_tuned_alltoall_max_requests;
+                if( (version > 1) && isnext_digit(fptr) ) {
+                    if( (getnext (fptr, &MAXREQ) < 0) || (MAXREQ < 0) ) {
+                        opal_output_verbose(1, ompi_coll_tuned_stream,
+                            "Could not read max requests for collective ID %ld com rule %d msg rule %d at around line %d\n", COLID, ncs, nms, fileline);
+                        goto on_file_error;
+                    }
+                    msg_p->result_max_requests = MAXREQ;
+                }
 
-                if (!nms && MS) {
-                    OPAL_OUTPUT((ompi_coll_tuned_stream,"All algorithms must specify a rule for message size of zero upwards always first!\n"));
-                    OPAL_OUTPUT((ompi_coll_tuned_stream,"Message size was %lu for collective ID %ld com rule %d msg rule %d at around line %d\n", MS, CI, ncs, nms, fileline));
+                /* check the first rule is for 0 size. look-up depends on this */
+                if (!nms && MSGSIZE) {
+                    opal_output_verbose(1, ompi_coll_tuned_stream,
+                        "All algorithms must specify a rule for message size of zero upwards always first!\n");
+                    opal_output_verbose(1, ompi_coll_tuned_stream,
+                        "Message size was %lu for collective ID %ld com rule %d msg rule %d at around line %d\n", MSGSIZE, COLID, ncs, nms, fileline);
                     goto on_file_error;
                 }
-
                 total_msg_count++;
-
             } /* msg size */
-
             total_com_count++;
-
         } /* comm size */
 
         total_alg_count++;
-        OPAL_OUTPUT((ompi_coll_tuned_stream, "Done reading dynamic rule for collective ID %ld\n", CI));
+        opal_output_verbose(25, ompi_coll_tuned_stream,
+            "Done reading dynamic rule for collective ID %ld\n", COLID);
 
     } /* per collective */
 
     fclose (fptr);
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"\nConfigure file Stats\n"));
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"Collectives with rules\t\t\t: %5d\n", total_alg_count));
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"Communicator sizes with rules\t\t: %5d\n", total_com_count));
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"Message sizes with rules\t\t: %5d\n", total_msg_count));
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"Lines in configuration file read\t\t: %5d\n", fileline));
+    opal_output_verbose(25, ompi_coll_tuned_stream,
+        "\nConfigure file Stats\n");
+    opal_output_verbose(25, ompi_coll_tuned_stream,
+        "Version\t\t\t\t\t: %5u\n", version);
+    opal_output_verbose(25, ompi_coll_tuned_stream,
+        "Collectives with rules\t\t\t: %5d\n", total_alg_count);
+    opal_output_verbose(25, ompi_coll_tuned_stream,
+        "Communicator sizes with rules\t\t: %5d\n", total_com_count);
+    opal_output_verbose(25, ompi_coll_tuned_stream,
+        "Message sizes with rules\t\t: %5d\n", total_msg_count);
+    opal_output_verbose(25, ompi_coll_tuned_stream,
+        "Lines in configuration file read\t\t: %5d\n", fileline);
 
     /* return the rules to the caller */
     *rules = alg_rules;
 
-    return (total_alg_count);
+    return OPAL_SUCCESS;
 
 
  on_file_error:
@@ -236,18 +645,78 @@ int ompi_coll_tuned_read_rules_config_file (char *fname, ompi_coll_alg_rule_t**
     /* we return back a verbose message and a count of -1 algorithms read */
     /* draconian but its better than having a bad collective decision table */
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"read_rules_config_file: bad configure file [%s]. Read afar as line %d\n", fname, fileline));
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"Ignoring user supplied tuned collectives configuration decision file.\n"));
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"Switching back to [compiled in] fixed decision table.\n"));
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"Fix errors as listed above and try again.\n"));
+    opal_output_verbose(1, ompi_coll_tuned_stream,
+        "read_rules_config_file: bad configure file [%s]. Read afar as line %d\n", fname, fileline);
+    opal_output_verbose(1, ompi_coll_tuned_stream,
+        "Ignoring user supplied tuned collectives configuration decision file.\n");
+    opal_output_verbose(1, ompi_coll_tuned_stream,
+        "Switching back to [compiled in] fixed decision table.\n");
+    opal_output_verbose(1, ompi_coll_tuned_stream,
+        "Fix errors as listed above and try again.\n");
 
     /* deallocate memory if allocated */
-    if (alg_rules) ompi_coll_tuned_free_all_rules (alg_rules, n_collectives);
+    if (alg_rules) ompi_coll_tuned_free_all_rules (alg_rules);
 
     /* close file */
     if (fptr) fclose (fptr);
 
     *rules = (ompi_coll_alg_rule_t*) NULL;
-    return (-1);
+    return OPAL_ERROR;
 }
 
+/**
+ * Reads a rule file called fname
+ *
+ * In Open MPI 6.0 we introduced json-based rule file, but we continue to read
+ * the original format for now.
+ *
+ * This funtion attempts a json read, and if it fails, falls back to classic
+ * read.  If both fail, we fall back to fixed rules.
+ *
+ * Errors will be entirely hidden from the user unless coll_base_verbose is set
+ * to at least 1.
+ *
+ * With coll_base_verbose set to 25, rules file is dumped to output stream.
+ *
+ * Returns OPAL_ERROR or OPAL_SUCCESS
+ */
+
+int ompi_coll_tuned_read_rules_config_file (char *fname, ompi_coll_alg_rule_t** rules) {
+    if (!fname) {
+        opal_output_verbose(1, ompi_coll_tuned_stream,
+            "Gave NULL as rule table configuration file for tuned collectives... ignoring!\n");
+        return OPAL_ERROR;
+    }
+
+    if (!rules) {
+        opal_output_verbose(1, ompi_coll_tuned_stream,
+            "Gave NULL as rule table result ptr!... ignoring!\n");
+        return OPAL_ERROR;
+    }
+
+    const opal_json_t *json;
+    opal_output_verbose(1, ompi_coll_tuned_stream, "Attempting to read tuned rules as JSON...\n");
+    int ret = opal_json_load_file(fname, &json, 0);
+    if (ret == OPAL_SUCCESS) {
+        ret = ompi_coll_tuned_read_rules_json(json, rules);
+        opal_json_free(&json);
+        if (ret != OPAL_SUCCESS) {
+            opal_output_verbose(1, ompi_coll_tuned_stream,
+                "ERROR: %s is valid json, but there were errors reading the rules from the file. "
+                "Falling back to default tuning and ignoring the file.\n", fname);
+        }
+    } else {
+        opal_output_verbose(1, ompi_coll_tuned_stream, "Failed to parse %s as valid json.  Assuming classic format.\n",fname);
+        ret = ompi_coll_tuned_read_rules_config_file_classic(fname, rules);
+        if (ret != OPAL_SUCCESS) {
+            opal_output_verbose(1, ompi_coll_tuned_stream, "Failed to load %s in either json or classic readers.  Check format.\n",fname);
+        }
+    }
+
+    if (ret == OPAL_SUCCESS && opal_output_check_verbosity( 2, ompi_coll_tuned_stream)) {
+        opal_output_verbose( 2, ompi_coll_tuned_stream, "Dumping rules:\n");
+        ompi_coll_tuned_dump_all_rules(*rules);
+    }
+    return ret;
+
+}
\ No newline at end of file
diff --git a/ompi/mca/coll/tuned/coll_tuned_dynamic_file.h b/ompi/mca/coll/tuned/coll_tuned_dynamic_file.h
index 595e436fa49..9efc1839dcf 100644
--- a/ompi/mca/coll/tuned/coll_tuned_dynamic_file.h
+++ b/ompi/mca/coll/tuned/coll_tuned_dynamic_file.h
@@ -9,6 +9,8 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2025      Amazon.com, Inc. or its affiliates.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -27,7 +29,7 @@
 
 BEGIN_C_DECLS
 
-int ompi_coll_tuned_read_rules_config_file (char *fname, ompi_coll_alg_rule_t** rules, int n_collectives);
+int ompi_coll_tuned_read_rules_config_file (char *fname, ompi_coll_alg_rule_t** rules);
 
 
 END_C_DECLS
diff --git a/ompi/mca/coll/tuned/coll_tuned_dynamic_rules.c b/ompi/mca/coll/tuned/coll_tuned_dynamic_rules.c
index 2c2b4469635..8a681a6bafa 100644
--- a/ompi/mca/coll/tuned/coll_tuned_dynamic_rules.c
+++ b/ompi/mca/coll/tuned/coll_tuned_dynamic_rules.c
@@ -12,6 +12,8 @@
  * Copyright (c) 2011-2012 FUJITSU LIMITED.  All rights reserved.
  * Copyright (c) 2017      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2025      Amazon.com, Inc. or its affiliates.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -24,6 +26,7 @@
 #include "mpi.h"
 #include "ompi/mca/mca.h"
 #include "ompi/constants.h"
+#include "ompi/communicator/communicator.h"
 #include "coll_tuned.h"
 
 /* need to include our own topo prototypes so we can malloc data on the comm correctly */
@@ -48,13 +51,13 @@ ompi_coll_alg_rule_t* ompi_coll_tuned_mk_alg_rules (int n_alg)
 
     /* set all we can at this point */
     for (i=0;i<n_alg;i++) {
-        alg_rules[i].alg_rule_id = i;
+        alg_rules[i].coll_id = i;
     }
     return (alg_rules);
 }
 
 
-ompi_coll_com_rule_t* ompi_coll_tuned_mk_com_rules (int n_com_rules, int alg_rule_id)
+ompi_coll_com_rule_t* ompi_coll_tuned_mk_com_rules (int n_com_rules, int coll_id)
 {
     int i;
     ompi_coll_com_rule_t * com_rules;
@@ -63,17 +66,18 @@ ompi_coll_com_rule_t* ompi_coll_tuned_mk_com_rules (int n_com_rules, int alg_rul
     if (!com_rules) return (com_rules);
 
     for (i=0;i<n_com_rules;i++) {
-        com_rules[i].mpi_comsize = 0;   /* unknown */
-        com_rules[i].alg_rule_id = alg_rule_id;
-        com_rules[i].com_rule_id = i;
-        com_rules[i].n_msg_sizes = 0;   /* unknown */
+        com_rules[i].mpi_comsize_min = 0;   /* unknown */
+        com_rules[i].mpi_comsize_max = INT_MAX;
+        com_rules[i].comm_rank_distribution = COLL_RULES_DISTRO_ANY;
+        com_rules[i].coll_id = coll_id;
+        com_rules[i].n_rules = 0;   /* unknown */
         com_rules[i].msg_rules = (ompi_coll_msg_rule_t *) NULL;
     }
     return (com_rules);
 }
 
 
-ompi_coll_msg_rule_t* ompi_coll_tuned_mk_msg_rules (int n_msg_rules, int alg_rule_id, int com_rule_id, int mpi_comsize)
+ompi_coll_msg_rule_t* ompi_coll_tuned_mk_msg_rules (int n_msg_rules, int coll_id, int mpi_comsize)
 {
     int i;
     ompi_coll_msg_rule_t *msg_rules;
@@ -82,11 +86,9 @@ ompi_coll_msg_rule_t* ompi_coll_tuned_mk_msg_rules (int n_msg_rules, int alg_rul
     if (!msg_rules) return (msg_rules);
 
     for( i = 0; i < n_msg_rules; i++ ) {
-        msg_rules[i].mpi_comsize = mpi_comsize;
-        msg_rules[i].alg_rule_id = alg_rule_id;
-        msg_rules[i].com_rule_id = com_rule_id;
-        msg_rules[i].msg_rule_id = i;
-        msg_rules[i].msg_size = 0;               /* unknown */
+        msg_rules[i].coll_id = coll_id;
+        msg_rules[i].msg_size_min = 0;           /* unknown */
+        msg_rules[i].msg_size_max = COLL_RULES_MESSAGE_SIZE_INF;
         msg_rules[i].result_alg = 0;             /* unknown */
         msg_rules[i].result_topo_faninout = 0;   /* unknown */
         msg_rules[i].result_segsize = 0;         /* unknown */
@@ -102,17 +104,35 @@ ompi_coll_msg_rule_t* ompi_coll_tuned_mk_msg_rules (int n_msg_rules, int alg_rul
  */
 int ompi_coll_tuned_dump_msg_rule (ompi_coll_msg_rule_t* msg_p)
 {
+    char max_msg_size[24]; // should only need 19+1 chars.
+    char *alg_name;
+    int rc;
+
     if (!msg_p) {
-        OPAL_OUTPUT((ompi_coll_tuned_stream,"Message rule was a NULL ptr?!\n"));
+        opal_output(ompi_coll_tuned_stream, "ERROR: Message rule was a NULL ptr?!\n");
         return (-1);
     }
-
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"alg_id %3d\tcom_id %3d\tcom_size %3d\tmsg_id %3d\t", msg_p->alg_rule_id,
-                 msg_p->com_rule_id, msg_p->mpi_comsize, msg_p->msg_rule_id));
-
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"msg_size %10lu -> algorithm %2d\ttopo in/out %2d\tsegsize %5ld\tmax_requests %4d\n",
-                 msg_p->msg_size, msg_p->result_alg, msg_p->result_topo_faninout, msg_p->result_segsize,
-                 msg_p->result_max_requests));
+    if (msg_p->msg_size_max == COLL_RULES_MESSAGE_SIZE_INF) {
+        snprintf(max_msg_size, 23, "%s", "Inf");
+    } else {
+        snprintf(max_msg_size, 23, "%lu", msg_p->msg_size_max);
+    }
+    max_msg_size[23] = '\0';
+
+    if (msg_p->msg_size_max < msg_p->msg_size_min) {
+        opal_output(ompi_coll_tuned_stream,"\t\t\tIGNORED RULE (check log)\n");
+    } else {
+        rc = coll_tuned_alg_to_str(msg_p->coll_id, msg_p->result_alg, &alg_name);
+        if (rc != OPAL_SUCCESS) {
+            alg_name = "ERROR_BAD_ALG_ID";
+        }
+        opal_output(ompi_coll_tuned_stream,"\t\t\tmsg_size %lu:%s -> %s (%2d).  Params: topo in/out %2d, segsize %5ld, max_requests %4d\n",
+                    msg_p->msg_size_min, max_msg_size, alg_name, msg_p->result_alg, msg_p->result_topo_faninout, msg_p->result_segsize,
+                    msg_p->result_max_requests);
+        if (rc == OPAL_SUCCESS) {
+            free(alg_name);
+        }
+    }
 
     return (0);
 }
@@ -123,20 +143,20 @@ int ompi_coll_tuned_dump_com_rule (ompi_coll_com_rule_t* com_p)
     int i;
 
     if (!com_p) {
-        OPAL_OUTPUT((ompi_coll_tuned_stream,"Com rule was a NULL ptr?!\n"));
+        opal_output(ompi_coll_tuned_stream,"\t\tERROR: Com rule was a NULL ptr?!\n");
         return (-1);
     }
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream, "alg_id %3d\tcom_id %3d\tcom_size %3d\t", com_p->alg_rule_id, com_p->com_rule_id, com_p->mpi_comsize));
-
-    if (!com_p->n_msg_sizes) {
-        OPAL_OUTPUT((ompi_coll_tuned_stream,"no msgsizes defined\n"));
+    if (!com_p->n_rules) {
+        opal_output(ompi_coll_tuned_stream,"\t\tno message size rules defined\n");
         return (0);
     }
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"number of message sizes %3d\n", com_p->n_msg_sizes));
+    opal_output(ompi_coll_tuned_stream, "\t\t[%s] comm_size: %d:%d, distro:%s -> %d message-size rules",
+        mca_coll_base_colltype_to_str(com_p->coll_id), com_p->mpi_comsize_min, com_p->mpi_comsize_max,
+        coll_rules_comm_rank_distro_to_str(com_p->comm_rank_distribution), com_p->n_rules);
 
-    for (i=0;i<com_p->n_msg_sizes;i++) {
+    for (i=0;i<com_p->n_rules;i++) {
         ompi_coll_tuned_dump_msg_rule (&(com_p->msg_rules[i]));
     }
 
@@ -149,18 +169,16 @@ int ompi_coll_tuned_dump_alg_rule (ompi_coll_alg_rule_t* alg_p)
     int i;
 
     if (!alg_p) {
-        OPAL_OUTPUT((ompi_coll_tuned_stream,"Algorithm rule was a NULL ptr?!\n"));
+        opal_output(ompi_coll_tuned_stream,"ERROR: Algorithm rule was a NULL ptr?!\n");
         return (-1);
     }
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"alg_id %3d\t", alg_p->alg_rule_id));
-
     if (!alg_p->n_com_sizes) {
-        OPAL_OUTPUT((ompi_coll_tuned_stream,"no coms defined\n"));
+        opal_output(ompi_coll_tuned_stream,"\tno rules defined\n");
         return (0);
     }
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"number of com sizes %3d\n", alg_p->n_com_sizes));
+    opal_output(ompi_coll_tuned_stream,"\tnumber of communicator rules: %d\n", alg_p->n_com_sizes);
 
     for (i=0;i<alg_p->n_com_sizes;i++) {
         ompi_coll_tuned_dump_com_rule (&(alg_p->com_rules[i]));
@@ -170,19 +188,18 @@ int ompi_coll_tuned_dump_alg_rule (ompi_coll_alg_rule_t* alg_p)
 }
 
 
-int ompi_coll_tuned_dump_all_rules (ompi_coll_alg_rule_t* alg_p, int n_rules)
+int ompi_coll_tuned_dump_all_rules (ompi_coll_alg_rule_t* alg_p)
 {
     int i;
 
     if (!alg_p) {
-        OPAL_OUTPUT((ompi_coll_tuned_stream,"Algorithm rule was a NULL ptr?!\n"));
+        opal_output(ompi_coll_tuned_stream,"Algorithm rule was a NULL ptr?!\n");
         return (-1);
     }
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"Number of algorithm rules %3d\n", n_rules));
-
-    for (i=0;i<n_rules;i++) {
-        ompi_coll_tuned_dump_alg_rule (&(alg_p[i]));
+    for (i=0;i<COLLCOUNT;i++) {
+        opal_output(ompi_coll_tuned_stream, "Rules for %s (id=%3d)\n", mca_coll_base_colltype_to_str(i), i);
+        ompi_coll_tuned_dump_alg_rule( &alg_p[i] );
     }
 
     return (0);
@@ -199,15 +216,17 @@ int ompi_coll_tuned_free_msg_rules_in_com_rule (ompi_coll_com_rule_t* com_p)
     ompi_coll_msg_rule_t* msg_p;
 
     if (!com_p) {
-        OPAL_OUTPUT((ompi_coll_tuned_stream,"attempt to free NULL com_rule ptr\n"));
+        opal_output_verbose(1, ompi_coll_tuned_stream,
+            "attempt to free NULL com_rule ptr\n");
         return (-1);
     }
 
-    if (com_p->n_msg_sizes) {
+    if (com_p->n_rules) {
         msg_p = com_p->msg_rules;
 
         if (!msg_p) {
-            OPAL_OUTPUT((ompi_coll_tuned_stream,"attempt to free NULL msg_rules when msg count was %d\n", com_p->n_msg_sizes));
+            opal_output_verbose(1, ompi_coll_tuned_stream,
+                "attempt to free NULL n_rules when msg count was %d\n", com_p->n_rules);
             rc = -1; /* some error */
         }
         else {
@@ -230,7 +249,7 @@ int ompi_coll_tuned_free_coms_in_alg_rule (ompi_coll_alg_rule_t* alg_p)
     ompi_coll_com_rule_t* com_p;
 
     if (!alg_p) {
-        OPAL_OUTPUT((ompi_coll_tuned_stream,"attempt to free NULL alg_rule ptr\n"));
+        opal_output_verbose(1, ompi_coll_tuned_stream,"attempt to free NULL alg_rule ptr\n");
         return (-1);
     }
 
@@ -238,7 +257,7 @@ int ompi_coll_tuned_free_coms_in_alg_rule (ompi_coll_alg_rule_t* alg_p)
         com_p = alg_p->com_rules;
 
         if (!com_p) {
-            OPAL_OUTPUT((ompi_coll_tuned_stream,"attempt to free NULL com_rules when com count was %d\n", alg_p->n_com_sizes));
+            opal_output_verbose(1, ompi_coll_tuned_stream,"attempt to free NULL com_rules when com count was %d\n", alg_p->n_com_sizes);
         } else {
             /* ok, memory exists for the com rules so free their message rules first */
             for( i = 0; i < alg_p->n_com_sizes; i++ ) {
@@ -256,12 +275,12 @@ int ompi_coll_tuned_free_coms_in_alg_rule (ompi_coll_alg_rule_t* alg_p)
 }
 
 
-int ompi_coll_tuned_free_all_rules (ompi_coll_alg_rule_t* alg_p, int n_algs)
+int ompi_coll_tuned_free_all_rules (ompi_coll_alg_rule_t* alg_p)
 {
     int i;
     int rc = 0;
 
-    for( i = 0; i < n_algs; i++ ) {
+    for( i = 0; i < COLLCOUNT; i++ ) {
         rc += ompi_coll_tuned_free_coms_in_alg_rule (&(alg_p[i]));
     }
 
@@ -279,50 +298,74 @@ int ompi_coll_tuned_free_all_rules (ompi_coll_alg_rule_t* alg_p, int n_algs)
 
 /*
  * This function is used to get the pointer to the nearest (less than or equal)
- * com rule for this MPI collective (alg_id) for a given
+ * com rule for this MPI collective (coll_id) for a given
  * MPI communicator size. The complete rule base must be presented.
  *
  * If no rule exits returns NULL, else the com rule ptr
  * (which can be used in the coll_tuned_get_target_method_params() call)
  *
  */
-ompi_coll_com_rule_t* ompi_coll_tuned_get_com_rule_ptr (ompi_coll_alg_rule_t* rules, int alg_id, int mpi_comsize)
+ompi_coll_com_rule_t* ompi_coll_tuned_get_com_rule_ptr (ompi_coll_alg_rule_t* rules, int coll_id, struct ompi_communicator_t *comm)
 {
     ompi_coll_alg_rule_t*  alg_p = (ompi_coll_alg_rule_t*) NULL;
     ompi_coll_com_rule_t*  com_p = (ompi_coll_com_rule_t*) NULL;
     ompi_coll_com_rule_t*  best_com_p = (ompi_coll_com_rule_t*) NULL;
     int i;
+    int mpi_comsize;
 
     if (!rules) {                    /* no rule base no resulting com rule */
         return ((ompi_coll_com_rule_t*)NULL);
     }
 
-    alg_p = &(rules[alg_id]); /* get the algorithm rule pointer */
+    alg_p = &(rules[coll_id]); /* get the algorithm rule pointer */
 
     if (!alg_p->n_com_sizes) {   /* check for count of communicator sizes */
         return ((ompi_coll_com_rule_t*)NULL);    /* no com sizes so no rule */
     }
 
-    /* ok have some com sizes, now to find the one closest to my mpi_comsize */
+    if (OMPI_COMM_IS_INTER(comm)) {
+        mpi_comsize = ompi_comm_remote_size(comm);
+    } else {
+        mpi_comsize = ompi_comm_size(comm);
+    }
 
-    /* make a copy of the first com rule */
-    best_com_p = com_p = alg_p->com_rules;
-    i = 0;
+    for (i=0; i<alg_p->n_com_sizes; i++) {
+        com_p = &alg_p->com_rules[i];
+        if (com_p->comm_rank_distribution == COLL_RULES_DISTRO_DISJOINT) {
+            if (!OMPI_COMM_IS_DISJOINT_SET(comm) || !OMPI_COMM_IS_DISJOINT(comm) ) {
+                /* rule says disjoint, and comm is not disjoint, skip this rule */
+                continue;
+            }
+        }
+        if (com_p->comm_rank_distribution == COLL_RULES_DISTRO_SINGLENODE) {
+            int local_peers = ompi_group_count_local_peers(comm->c_local_group);
+            int all_peers = ompi_comm_size(comm);
+            if (local_peers != all_peers) {
+                /* rule says single-node, but we have non-local peers, skip this rule */
+                continue;
+            }
+        }
 
-    while( i < alg_p->n_com_sizes ) {
-        if (com_p->mpi_comsize > mpi_comsize) {
+        if (com_p->mpi_comsize_min <= mpi_comsize &&
+                                      mpi_comsize <= com_p->mpi_comsize_max ) {
+            best_com_p = com_p;
             break;
         }
-        best_com_p = com_p;
-        /* go to the next entry */
-        com_p++;
-        i++;
     }
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"Selected the following com rule id %d\n", best_com_p->com_rule_id));
-    ompi_coll_tuned_dump_com_rule (best_com_p);
+    if ( opal_output_check_verbosity(25, ompi_coll_tuned_stream) ) {
+        if (!best_com_p) {
+            opal_output( ompi_coll_tuned_stream, "coll:tuned:dynamic: For %s with communicator size %d "
+                "no matching rule found.  Using fixed defaults.",
+                mca_coll_base_colltype_to_str(coll_id), mpi_comsize);
+        } else {
+            opal_output(ompi_coll_tuned_stream,"coll:tuned:dynamic: For %s with communicator size %d selected rule number %d\n",
+                mca_coll_base_colltype_to_str(coll_id), mpi_comsize, i+1);
+            ompi_coll_tuned_dump_com_rule(best_com_p);
+        }
+    }
 
-    return (best_com_p);
+    return best_com_p;
 }
 
 /*
@@ -348,34 +391,38 @@ int ompi_coll_tuned_get_target_method_params (ompi_coll_com_rule_t* base_com_rul
     int i;
 
     /* No rule or zero rules */
-    if( (NULL == base_com_rule) || (0 == base_com_rule->n_msg_sizes)) {
+    if( (NULL == base_com_rule) || (0 == base_com_rule->n_rules)) {
         return (0);
     }
 
-    /* ok have some msg sizes, now to find the one closest to my mpi_msgsize */
-
-    /* make a copy of the first msg rule */
-    best_msg_p = msg_p = base_com_rule->msg_rules;
-    i = 0;
-
-    while (i<base_com_rule->n_msg_sizes) {
-        /*       OPAL_OUTPUT((ompi_coll_tuned_stream,"checking mpi_msgsize %d against com_id %d msg_id %d index %d msg_size %d",  */
-        /*             mpi_msgsize, msg_p->com_rule_id, msg_p->msg_rule_id, i, msg_p->msg_size)); */
-        if (msg_p->msg_size <= mpi_msgsize) {
+    /* search for the first comm rule that matches */
+    for(i=0; i<base_com_rule->n_rules; i++) {
+        msg_p = &base_com_rule->msg_rules[i];
+        if (msg_p->msg_size_min <= mpi_msgsize &&
+                                   mpi_msgsize <= msg_p->msg_size_max ) {
             best_msg_p = msg_p;
-            /*          OPAL_OUTPUT((ompi_coll_tuned_stream(":ok\n")); */
-        }
-        else {
-            /*          OPAL_OUTPUT((ompi_coll_tuned_stream(":nop\n")); */
             break;
         }
-        /* go to the next entry */
-        msg_p++;
-        i++;
     }
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"Selected the following msg rule id %d\n", best_msg_p->msg_rule_id));
-    ompi_coll_tuned_dump_msg_rule (best_msg_p);
+    if ( opal_output_check_verbosity(25, ompi_coll_tuned_stream) ) {
+        if (!best_msg_p) {
+            opal_output(ompi_coll_tuned_stream,
+                "coll:tuned:dynamic For %s with msg_size=%lu: no matching rule, using fixed defaults.",
+                mca_coll_base_colltype_to_str(msg_p->coll_id), mpi_msgsize);
+        } else {
+            opal_output(ompi_coll_tuned_stream,
+                "coll:tuned:dynamic For %s with msg_size=%ld: selected rule number %d:",
+                mca_coll_base_colltype_to_str(msg_p->coll_id), mpi_msgsize, i+1);
+            ompi_coll_tuned_dump_msg_rule (best_msg_p);
+        }
+    }
+
+    if (!best_msg_p) {
+        /* no match, use defaults */
+        return 0;
+    }
+
 
     /* return the segment size */
     *result_topo_faninout = best_msg_p->result_topo_faninout;
@@ -389,3 +436,28 @@ int ompi_coll_tuned_get_target_method_params (ompi_coll_com_rule_t* base_com_rul
     /* return the algorithm/method to use */
     return (best_msg_p->result_alg);
 }
+
+static const char* coll_rules_comm_rank_distro_table[] = {
+    [COLL_RULES_DISTRO_ANY] = "any",
+    [COLL_RULES_DISTRO_DISJOINT] = "one-per-node",
+    [COLL_RULES_DISTRO_SINGLENODE] = "single-node",
+};
+
+const char* coll_rules_comm_rank_distro_to_str(enum comm_rank_distro_t distro)
+{
+    if( (distro < 0) || (distro >= COLL_RULES_DISTRO_COUNT) ) {
+        return NULL;
+    }
+    return coll_rules_comm_rank_distro_table[distro];
+}
+
+int coll_rules_comm_rank_distro_from_str(const char *name, enum comm_rank_distro_t *distro)
+{
+    for (int i=0; i<COLL_RULES_DISTRO_COUNT; i++) {
+        if (0 == strcmp(name, coll_rules_comm_rank_distro_table[i])) {
+            *distro = i;
+            return OPAL_SUCCESS;
+        }
+    }
+    return OPAL_ERROR;
+}
diff --git a/ompi/mca/coll/tuned/coll_tuned_dynamic_rules.h b/ompi/mca/coll/tuned/coll_tuned_dynamic_rules.h
index 7e8f672d21d..85abe211a1a 100644
--- a/ompi/mca/coll/tuned/coll_tuned_dynamic_rules.h
+++ b/ompi/mca/coll/tuned/coll_tuned_dynamic_rules.h
@@ -11,6 +11,8 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2011-2012 FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2025      Amazon.com, Inc. or its affiliates.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -25,18 +27,32 @@
 
 BEGIN_C_DECLS
 
+enum coll_rules_magic_message_sizes {
+    /* our message sizes are unsigned, but our json reader library returns only
+    signed integers. Hopefully no one is sending 9 exabytes in a single message. */
+    COLL_RULES_MESSAGE_SIZE_INF = SSIZE_MAX,
+    COLL_RULES_VALUE_NOT_PROVIDED = -1,
+};
+
+enum comm_rank_distro_t {
+    COLL_RULES_DISTRO_ANY=0,
+    COLL_RULES_DISTRO_DISJOINT,
+    COLL_RULES_DISTRO_SINGLENODE,
+    COLL_RULES_DISTRO_COUNT, /* keep as last entry, count of valid rules */
+};
+
+/* translate the comm_rank_distro_t values from string */
+int coll_rules_comm_rank_distro_from_str(const char *name, enum comm_rank_distro_t *distro);
+/* translate the comm_rank_distro_t values to string */
+const char* coll_rules_comm_rank_distro_to_str(enum comm_rank_distro_t distro);
 
 typedef struct msg_rule_s {
-    /* paranoid / debug */
-    int mpi_comsize;  /* which MPI comm size this is is for */
-
-    /* paranoid / debug */
-    int alg_rule_id; /* unique alg rule id */
-    int com_rule_id; /* unique com rule id */
-    int msg_rule_id; /* unique msg rule id */
+    /* keep track of which collective this rule is for */
+    int coll_id;     /* the collective ID for this message rule */
 
     /* RULE */
-    size_t msg_size; /* message size */
+    size_t msg_size_min;     /* message size */
+    size_t msg_size_max; /* message size max (defaults to SSIZE_MAX)*/
 
     /* RESULT */
     int result_alg;              /* result algorithm to use */
@@ -48,14 +64,14 @@ typedef struct msg_rule_s {
 
 typedef struct com_rule_s {
     /* paranoid / debug */
-    int mpi_comsize;  /* which MPI comm size this is is for */
+    int mpi_comsize_min;  /* which MPI comm size is this for */
+    int mpi_comsize_max;
+    enum comm_rank_distro_t comm_rank_distribution;
 
-    /* paranoid / debug */
-    int alg_rule_id; /* unique alg rule id */
-    int com_rule_id; /* unique com rule id */
+    int coll_id; /* unique alg rule id */
 
     /* RULE */
-    int n_msg_sizes;
+    int n_rules;
     ompi_coll_msg_rule_t *msg_rules;
 
 }  ompi_coll_com_rule_t;
@@ -63,7 +79,7 @@ typedef struct com_rule_s {
 
 typedef struct alg_rule_s {
     /* paranoid / debug */
-    int alg_rule_id; /* unique alg rule id */
+    int coll_id; /* unique alg rule id */
 
     /* RULE */
     int n_com_sizes;
@@ -75,24 +91,24 @@ typedef struct alg_rule_s {
 
 /* these are used to build the rule tables (by the read file routines) */
 ompi_coll_alg_rule_t* ompi_coll_tuned_mk_alg_rules (int n_alg);
-ompi_coll_com_rule_t* ompi_coll_tuned_mk_com_rules (int n_com_rules, int alg_rule_id);
-ompi_coll_msg_rule_t* ompi_coll_tuned_mk_msg_rules (int n_msg_rules, int alg_rule_id, int com_rule_id, int mpi_comsize);
+ompi_coll_com_rule_t* ompi_coll_tuned_mk_com_rules (int n_com_rules, int coll_id);
+ompi_coll_msg_rule_t* ompi_coll_tuned_mk_msg_rules (int n_msg_rules, int coll_id, int mpi_comsize);
 
 /* debugging support */
 int ompi_coll_tuned_dump_msg_rule (ompi_coll_msg_rule_t* msg_p);
 int ompi_coll_tuned_dump_com_rule (ompi_coll_com_rule_t* com_p);
 int ompi_coll_tuned_dump_alg_rule (ompi_coll_alg_rule_t* alg_p);
-int ompi_coll_tuned_dump_all_rules (ompi_coll_alg_rule_t* alg_p, int n_rules);
+int ompi_coll_tuned_dump_all_rules (ompi_coll_alg_rule_t* alg_p);
 
 /* free alloced memory routines, used by file and tuned component/module */
 int ompi_coll_tuned_free_msg_rules_in_com_rule (ompi_coll_com_rule_t* com_p);
 int ompi_coll_tuned_free_coms_in_alg_rule (ompi_coll_alg_rule_t* alg_p);
-int ompi_coll_tuned_free_all_rules (ompi_coll_alg_rule_t* alg_p, int n_algs);
+int ompi_coll_tuned_free_all_rules (ompi_coll_alg_rule_t* alg_p);
 
 
 /* the IMPORTANT routines, i.e. the ones that do stuff for everyday communicators and collective calls */
 
-ompi_coll_com_rule_t* ompi_coll_tuned_get_com_rule_ptr (ompi_coll_alg_rule_t* rules, int alg_id, int mpi_comsize);
+ompi_coll_com_rule_t* ompi_coll_tuned_get_com_rule_ptr (ompi_coll_alg_rule_t* rules, int alg_id, struct ompi_communicator_t *comm);
 
 int ompi_coll_tuned_get_target_method_params (ompi_coll_com_rule_t* base_com_rule, size_t mpi_msgsize,
                                               int* result_topo_faninout, int* result_segsize,
diff --git a/ompi/mca/coll/tuned/coll_tuned_exscan_decision.c b/ompi/mca/coll/tuned/coll_tuned_exscan_decision.c
index 42668d2e18f..48288c5d7d7 100644
--- a/ompi/mca/coll/tuned/coll_tuned_exscan_decision.c
+++ b/ompi/mca/coll/tuned/coll_tuned_exscan_decision.c
@@ -5,6 +5,8 @@
  * Copyright (c) 2020      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2025      Amazon.com, Inc. or its affiliates.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -77,6 +79,7 @@ int ompi_coll_tuned_exscan_intra_check_forced_init (coll_tuned_force_algorithm_m
                                         OPAL_INFO_LVL_5,
                                         MCA_BASE_VAR_SCOPE_ALL,
                                         &coll_tuned_exscan_forced_algorithm);
+    coll_tuned_alg_register_options( EXSCAN, new_enum );
     OBJ_RELEASE(new_enum);
     if (mca_param_indices->algorithm_param_index < 0) {
         return mca_param_indices->algorithm_param_index;
@@ -85,15 +88,16 @@ int ompi_coll_tuned_exscan_intra_check_forced_init (coll_tuned_force_algorithm_m
     return (MPI_SUCCESS);
 }
 
-int ompi_coll_tuned_exscan_intra_do_this(const void *sbuf, void* rbuf, int count,
+int ompi_coll_tuned_exscan_intra_do_this(const void *sbuf, void* rbuf, size_t count,
                                          struct ompi_datatype_t *dtype,
                                          struct ompi_op_t *op,
                                          struct ompi_communicator_t *comm,
                                          mca_coll_base_module_t *module,
                                          int algorithm)
 {
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:exscan_intra_do_this selected algorithm %d",
-                 algorithm));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "coll:tuned:exscan_intra_do_this selected algorithm %d",
+        algorithm));
 
     switch (algorithm) {
     case (0):
@@ -102,7 +106,8 @@ int ompi_coll_tuned_exscan_intra_do_this(const void *sbuf, void* rbuf, int count
     case (2):  return ompi_coll_base_exscan_intra_recursivedoubling(sbuf, rbuf, count, dtype,
                                                                     op, comm, module);
     } /* switch */
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:exscan_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
-                 algorithm, ompi_coll_tuned_forced_max_algorithms[EXSCAN]));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "coll:tuned:exscan_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
+        algorithm, ompi_coll_tuned_forced_max_algorithms[EXSCAN]));
     return (MPI_ERR_ARG);
 }
diff --git a/ompi/mca/coll/tuned/coll_tuned_gather_decision.c b/ompi/mca/coll/tuned/coll_tuned_gather_decision.c
index 1b09687234a..d356202a3bf 100644
--- a/ompi/mca/coll/tuned/coll_tuned_gather_decision.c
+++ b/ompi/mca/coll/tuned/coll_tuned_gather_decision.c
@@ -5,6 +5,8 @@
  *                         reserved.
  * Copyright (c) 2015-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2025      Amazon.com, Inc. or its affiliates.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -81,6 +83,7 @@ ompi_coll_tuned_gather_intra_check_forced_init(coll_tuned_force_algorithm_mca_pa
                                         OPAL_INFO_LVL_5,
                                         MCA_BASE_VAR_SCOPE_ALL,
                                         &coll_tuned_gather_forced_algorithm);
+    coll_tuned_alg_register_options( GATHER, new_enum );
     OBJ_RELEASE(new_enum);
     if (mca_param_indices->algorithm_param_index < 0) {
         return mca_param_indices->algorithm_param_index;
@@ -120,16 +123,16 @@ ompi_coll_tuned_gather_intra_check_forced_init(coll_tuned_force_algorithm_mca_pa
 }
 
 int
-ompi_coll_tuned_gather_intra_do_this(const void *sbuf, int scount,
+ompi_coll_tuned_gather_intra_do_this(const void *sbuf, size_t scount,
                                      struct ompi_datatype_t *sdtype,
-                                     void* rbuf, int rcount,
+                                     void* rbuf, size_t rcount,
                                      struct ompi_datatype_t *rdtype,
                                      int root,
                                      struct ompi_communicator_t *comm,
                                      mca_coll_base_module_t *module,
                                      int algorithm, int faninout, int segsize)
 {
-    OPAL_OUTPUT((ompi_coll_tuned_stream,
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
                  "coll:tuned:gather_intra_do_this selected algorithm %d topo faninout %d segsize %d",
                  algorithm, faninout, segsize));
 
@@ -152,7 +155,7 @@ ompi_coll_tuned_gather_intra_do_this(const void *sbuf, int scount,
                                                        root, comm, module,
                                                        segsize);
     } /* switch */
-    OPAL_OUTPUT((ompi_coll_tuned_stream,
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
                  "coll:tuned:gather_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
                  algorithm, ompi_coll_tuned_forced_max_algorithms[GATHER]));
     return (MPI_ERR_ARG);
diff --git a/ompi/mca/coll/tuned/coll_tuned_module.c b/ompi/mca/coll/tuned/coll_tuned_module.c
index 1c408e92f97..eb4fb125380 100644
--- a/ompi/mca/coll/tuned/coll_tuned_module.c
+++ b/ompi/mca/coll/tuned/coll_tuned_module.c
@@ -13,6 +13,9 @@
  * Copyright (c) 2016      Intel, Inc.  All rights reserved.
  * Copyright (c) 2018      Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2025      Amazon.com, Inc. or its affiliates.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -36,6 +39,8 @@
 
 static int tuned_module_enable(mca_coll_base_module_t *module,
                    struct ompi_communicator_t *comm);
+static int tuned_module_disable(mca_coll_base_module_t *module,
+                   struct ompi_communicator_t *comm);
 /*
  * Initial query function that is invoked during MPI_INIT, allowing
  * this component to disqualify itself if it doesn't support the
@@ -58,7 +63,8 @@ ompi_coll_tuned_comm_query(struct ompi_communicator_t *comm, int *priority)
 {
     mca_coll_tuned_module_t *tuned_module;
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream, "coll:tuned:module_tuned query called"));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "coll:tuned:module_tuned query called"));
 
     /**
      * No support for inter-communicator yet.
@@ -89,6 +95,7 @@ ompi_coll_tuned_comm_query(struct ompi_communicator_t *comm, int *priority)
      * but this would probably add an extra if and funct call to the path
      */
     tuned_module->super.coll_module_enable = tuned_module_enable;
+    tuned_module->super.coll_module_disable = tuned_module_disable;
 
     /* By default stick with the fixed version of the tuned collectives. Later on,
      * when the module get enabled, set the correct version based on the availability
@@ -99,18 +106,13 @@ ompi_coll_tuned_comm_query(struct ompi_communicator_t *comm, int *priority)
     tuned_module->super.coll_allreduce  = ompi_coll_tuned_allreduce_intra_dec_fixed;
     tuned_module->super.coll_alltoall   = ompi_coll_tuned_alltoall_intra_dec_fixed;
     tuned_module->super.coll_alltoallv  = ompi_coll_tuned_alltoallv_intra_dec_fixed;
-    tuned_module->super.coll_alltoallw  = NULL;
     tuned_module->super.coll_barrier    = ompi_coll_tuned_barrier_intra_dec_fixed;
     tuned_module->super.coll_bcast      = ompi_coll_tuned_bcast_intra_dec_fixed;
-    tuned_module->super.coll_exscan     = NULL;
     tuned_module->super.coll_gather     = ompi_coll_tuned_gather_intra_dec_fixed;
-    tuned_module->super.coll_gatherv    = NULL;
     tuned_module->super.coll_reduce     = ompi_coll_tuned_reduce_intra_dec_fixed;
     tuned_module->super.coll_reduce_scatter = ompi_coll_tuned_reduce_scatter_intra_dec_fixed;
     tuned_module->super.coll_reduce_scatter_block = ompi_coll_tuned_reduce_scatter_block_intra_dec_fixed;
-    tuned_module->super.coll_scan       = NULL;
     tuned_module->super.coll_scatter    = ompi_coll_tuned_scatter_intra_dec_fixed;
-    tuned_module->super.coll_scatterv   = NULL;
 
     return &(tuned_module->super);
 }
@@ -129,7 +131,7 @@ ompi_coll_tuned_forced_getvalues( enum COLLTYPE type,
 
     /**
      * Set the selected algorithm to 0 by default. Later on we can check this against 0
-     * to see if it was setted explicitly (if we suppose that setting it to 0 enable the
+     * to see if it was set explicitly (if we suppose that setting it to 0 enable the
      * default behavior) or not.
      */
     mca_base_var_get_value(mca_params->algorithm_param_index, &tmp, NULL, NULL);
@@ -148,8 +150,26 @@ ompi_coll_tuned_forced_getvalues( enum COLLTYPE type,
     return (MPI_SUCCESS);
 }
 
-#define COLL_TUNED_EXECUTE_IF_DYNAMIC(TMOD, TYPE, EXECUTE)              \
+#define TUNED_INSTALL_COLL_API(__comm, __module, __api)                                                     \
+    do                                                                                                      \
+    {                                                                                                       \
+        if (__module->super.coll_##__api)                                                                   \
+        {                                                                                                   \
+            MCA_COLL_INSTALL_API(__comm, __api, __module->super.coll_##__api, &__module->super, "tuned");  \
+        }                                                                                                   \
+    } while (0)
+
+#define TUNED_UNINSTALL_COLL_API(__comm, __module, __api)               \
+    do                                                                  \
     {                                                                   \
+        if (__comm->c_coll->coll_##__api##_module == &__module->super)  \
+        {                                                               \
+            MCA_COLL_INSTALL_API(__comm, __api, NULL, NULL, "tuned");   \
+        }                                                               \
+    } while (0)
+    
+#define COLL_TUNED_EXECUTE_IF_DYNAMIC(TMOD, TYPE, EXECUTE)              \
+    do {                                                                \
         int need_dynamic_decision = 0;                                  \
         ompi_coll_tuned_forced_getvalues( (TYPE), &((TMOD)->user_forced[(TYPE)]) ); \
         (TMOD)->com_rules[(TYPE)] = NULL;                               \
@@ -159,16 +179,17 @@ ompi_coll_tuned_forced_getvalues( enum COLLTYPE type,
         if( NULL != mca_coll_tuned_component.all_base_rules ) {         \
             (TMOD)->com_rules[(TYPE)]                                   \
                 = ompi_coll_tuned_get_com_rule_ptr( mca_coll_tuned_component.all_base_rules, \
-                                                    (TYPE), size );     \
+                                                    (TYPE), comm );     \
             if( NULL != (TMOD)->com_rules[(TYPE)] ) {                   \
                 need_dynamic_decision = 1;                              \
             }                                                           \
         }                                                               \
         if( 1 == need_dynamic_decision ) {                              \
-            OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned: enable dynamic selection for "#TYPE)); \
+            OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream, \
+                "coll:tuned: enable dynamic selection for "#TYPE));     \
             EXECUTE;                                                    \
         }                                                               \
-    }
+    } while(0)
 
 /*
  * Init module on the communicator
@@ -177,18 +198,10 @@ static int
 tuned_module_enable( mca_coll_base_module_t *module,
                      struct ompi_communicator_t *comm )
 {
-    int size;
     mca_coll_tuned_module_t *tuned_module = (mca_coll_tuned_module_t *) module;
     mca_coll_base_comm_t *data = NULL;
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:module_init called."));
-
-    /* Allocate the data that hangs off the communicator */
-    if (OMPI_COMM_IS_INTER(comm)) {
-        size = ompi_comm_remote_size(comm);
-    } else {
-        size = ompi_comm_size(comm);
-    }
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream, "coll:tuned:module_init called."));
 
     /**
      * we still malloc data as it is used by the TUNED modules
@@ -208,7 +221,7 @@ tuned_module_enable( mca_coll_base_module_t *module,
     }
 
     if (ompi_coll_tuned_use_dynamic_rules) {
-        OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:module_init MCW & Dynamic"));
+        OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream, "coll:tuned:module_init MCW & Dynamic"));
 
         /**
          * next dynamic state, recheck all forced rules as well
@@ -249,6 +262,23 @@ tuned_module_enable( mca_coll_base_module_t *module,
         COLL_TUNED_EXECUTE_IF_DYNAMIC(tuned_module, SCATTERV,
                                       tuned_module->super.coll_scatterv   = NULL);
     }
+    TUNED_INSTALL_COLL_API(comm, tuned_module, allgather);
+    TUNED_INSTALL_COLL_API(comm, tuned_module, allgatherv);
+    TUNED_INSTALL_COLL_API(comm, tuned_module, allreduce);
+    TUNED_INSTALL_COLL_API(comm, tuned_module, alltoall);
+    TUNED_INSTALL_COLL_API(comm, tuned_module, alltoallv);
+    TUNED_INSTALL_COLL_API(comm, tuned_module, alltoallw);
+    TUNED_INSTALL_COLL_API(comm, tuned_module, barrier);
+    TUNED_INSTALL_COLL_API(comm, tuned_module, bcast);
+    TUNED_INSTALL_COLL_API(comm, tuned_module, exscan);
+    TUNED_INSTALL_COLL_API(comm, tuned_module, gather);
+    TUNED_INSTALL_COLL_API(comm, tuned_module, gatherv);
+    TUNED_INSTALL_COLL_API(comm, tuned_module, reduce);
+    TUNED_INSTALL_COLL_API(comm, tuned_module, reduce_scatter);
+    TUNED_INSTALL_COLL_API(comm, tuned_module, reduce_scatter_block);
+    TUNED_INSTALL_COLL_API(comm, tuned_module, scan);
+    TUNED_INSTALL_COLL_API(comm, tuned_module, scatter);
+    TUNED_INSTALL_COLL_API(comm, tuned_module, scatterv);
 
     /* general n fan out tree */
     data->cached_ntree = NULL;
@@ -270,6 +300,33 @@ tuned_module_enable( mca_coll_base_module_t *module,
     /* All done */
     tuned_module->super.base_data = data;
 
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:module_init Tuned is in use"));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,"coll:tuned:module_init Tuned is in use"));
+    return OMPI_SUCCESS;
+}
+
+static int
+tuned_module_disable(mca_coll_base_module_t *module,
+                     struct ompi_communicator_t *comm)
+{
+    mca_coll_tuned_module_t *tuned_module = (mca_coll_tuned_module_t *) module;
+
+    TUNED_UNINSTALL_COLL_API(comm, tuned_module, allgather);
+    TUNED_UNINSTALL_COLL_API(comm, tuned_module, allgatherv);
+    TUNED_UNINSTALL_COLL_API(comm, tuned_module, allreduce);
+    TUNED_UNINSTALL_COLL_API(comm, tuned_module, alltoall);
+    TUNED_UNINSTALL_COLL_API(comm, tuned_module, alltoallv);
+    TUNED_UNINSTALL_COLL_API(comm, tuned_module, alltoallw);
+    TUNED_UNINSTALL_COLL_API(comm, tuned_module, barrier);
+    TUNED_UNINSTALL_COLL_API(comm, tuned_module, bcast);
+    TUNED_UNINSTALL_COLL_API(comm, tuned_module, exscan);
+    TUNED_UNINSTALL_COLL_API(comm, tuned_module, gather);
+    TUNED_UNINSTALL_COLL_API(comm, tuned_module, gatherv);
+    TUNED_UNINSTALL_COLL_API(comm, tuned_module, reduce);
+    TUNED_UNINSTALL_COLL_API(comm, tuned_module, reduce_scatter);
+    TUNED_UNINSTALL_COLL_API(comm, tuned_module, reduce_scatter_block);
+    TUNED_UNINSTALL_COLL_API(comm, tuned_module, scan);
+    TUNED_UNINSTALL_COLL_API(comm, tuned_module, scatter);
+    TUNED_UNINSTALL_COLL_API(comm, tuned_module, scatterv);
+
     return OMPI_SUCCESS;
 }
diff --git a/ompi/mca/coll/tuned/coll_tuned_reduce_decision.c b/ompi/mca/coll/tuned/coll_tuned_reduce_decision.c
index 40e500d1c04..6ae3c00f7d9 100644
--- a/ompi/mca/coll/tuned/coll_tuned_reduce_decision.c
+++ b/ompi/mca/coll/tuned/coll_tuned_reduce_decision.c
@@ -5,6 +5,8 @@
  *                         reserved.
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2025      Amazon.com, Inc. or its affiliates.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -42,6 +44,7 @@ static const mca_base_var_enum_value_t reduce_algorithms[] = {
     {5, "binomial"},
     {6, "in-order_binary"},
     {7, "rabenseifner"},
+    {8, "knomial"},
     {0, NULL}
 };
 
@@ -80,12 +83,13 @@ int ompi_coll_tuned_reduce_intra_check_forced_init (coll_tuned_force_algorithm_m
     mca_param_indices->algorithm_param_index =
         mca_base_component_var_register(&mca_coll_tuned_component.super.collm_version,
                                         "reduce_algorithm",
-                                        "Which reduce algorithm is used. Can be locked down to choice of: 0 ignore, 1 linear, 2 chain, 3 pipeline, 4 binary, 5 binomial, 6 in-order binary, 7 rabenseifner. "
+                                        "Which reduce algorithm is used. Can be locked down to choice of: 0 ignore, 1 linear, 2 chain, 3 pipeline, 4 binary, 5 binomial, 6 in-order binary, 7 rabenseifner, 8 knomial. "
                                         "Only relevant if coll_tuned_use_dynamic_rules is true.",
                                         MCA_BASE_VAR_TYPE_INT, new_enum, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                                         OPAL_INFO_LVL_5,
                                         MCA_BASE_VAR_SCOPE_ALL,
                                         &coll_tuned_reduce_forced_algorithm);
+    coll_tuned_alg_register_options( REDUCE, new_enum );
     OBJ_RELEASE(new_enum);
     if (mca_param_indices->algorithm_param_index < 0) {
         return mca_param_indices->algorithm_param_index;
@@ -144,7 +148,7 @@ int ompi_coll_tuned_reduce_intra_check_forced_init (coll_tuned_force_algorithm_m
     return (MPI_SUCCESS);
 }
 
-int ompi_coll_tuned_reduce_intra_do_this(const void *sbuf, void* rbuf, int count,
+int ompi_coll_tuned_reduce_intra_do_this(const void *sbuf, void* rbuf, size_t count,
                                          struct ompi_datatype_t *dtype,
                                          struct ompi_op_t *op, int root,
                                          struct ompi_communicator_t *comm,
@@ -152,8 +156,9 @@ int ompi_coll_tuned_reduce_intra_do_this(const void *sbuf, void* rbuf, int count
                                          int algorithm, int faninout,
                                          int segsize, int max_requests )
 {
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:reduce_intra_do_this selected algorithm %d topo faninout %d segsize %d",
-                 algorithm, faninout, segsize));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "coll:tuned:reduce_intra_do_this selected algorithm %d topo faninout %d segsize %d",
+        algorithm, faninout, segsize));
 
     switch (algorithm) {
     case (0):  return ompi_coll_tuned_reduce_intra_dec_fixed(sbuf, rbuf, count, dtype,
@@ -177,8 +182,13 @@ int ompi_coll_tuned_reduce_intra_do_this(const void *sbuf, void* rbuf, int count
                                                                   segsize, max_requests);
     case (7):  return ompi_coll_base_reduce_intra_redscat_gather(sbuf, rbuf, count, dtype,
                                                                   op, root, comm, module);
+    case (8):  return ompi_coll_base_reduce_intra_knomial(sbuf, rbuf, count, dtype,
+                                                          op, root, comm, module,
+                                                          segsize, max_requests,
+                                                          faninout);
     } /* switch */
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:reduce_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
-                 algorithm, ompi_coll_tuned_forced_max_algorithms[REDUCE]));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "coll:tuned:reduce_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
+        algorithm, ompi_coll_tuned_forced_max_algorithms[REDUCE]));
     return (MPI_ERR_ARG);
 }
diff --git a/ompi/mca/coll/tuned/coll_tuned_reduce_scatter_block_decision.c b/ompi/mca/coll/tuned/coll_tuned_reduce_scatter_block_decision.c
index 40144b68af7..f4f6bdb7590 100644
--- a/ompi/mca/coll/tuned/coll_tuned_reduce_scatter_block_decision.c
+++ b/ompi/mca/coll/tuned/coll_tuned_reduce_scatter_block_decision.c
@@ -7,6 +7,8 @@
  * Copyright (c) 2020      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2025      Amazon.com, Inc. or its affiliates.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -86,6 +88,7 @@ int ompi_coll_tuned_reduce_scatter_block_intra_check_forced_init (coll_tuned_for
                                         OPAL_INFO_LVL_5,
                                         MCA_BASE_VAR_SCOPE_ALL,
                                         &coll_tuned_reduce_scatter_block_forced_algorithm);
+    coll_tuned_alg_register_options( REDUCESCATTERBLOCK, new_enum );
     OBJ_RELEASE(new_enum);
     if (mca_param_indices->algorithm_param_index < 0) {
         return mca_param_indices->algorithm_param_index;
@@ -115,15 +118,16 @@ int ompi_coll_tuned_reduce_scatter_block_intra_check_forced_init (coll_tuned_for
 }
 
 int ompi_coll_tuned_reduce_scatter_block_intra_do_this(const void *sbuf, void *rbuf,
-                                                       int rcount,
+                                                       size_t rcount,
                                                        struct ompi_datatype_t *dtype,
                                                        struct ompi_op_t *op,
                                                        struct ompi_communicator_t *comm,
                                                        mca_coll_base_module_t *module,
                                                        int algorithm, int faninout, int segsize)
 {
-    OPAL_OUTPUT((ompi_coll_tuned_stream, "coll:tuned:reduce_scatter_block_intra_do_this selected algorithm %d topo faninout %d segsize %d",
-                 algorithm, faninout, segsize));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "coll:tuned:reduce_scatter_block_intra_do_this selected algorithm %d topo faninout %d segsize %d",
+        algorithm, faninout, segsize));
 
     switch (algorithm) {
     case (0): return ompi_coll_tuned_reduce_scatter_block_intra_dec_fixed(sbuf, rbuf, rcount,
@@ -137,7 +141,8 @@ int ompi_coll_tuned_reduce_scatter_block_intra_do_this(const void *sbuf, void *r
     case (4): return ompi_coll_base_reduce_scatter_block_intra_butterfly(sbuf, rbuf, rcount, dtype, op, comm,
                                                                          module);
     } /* switch */
-    OPAL_OUTPUT((ompi_coll_tuned_stream, "coll:tuned:reduce_scatter_block_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
-                 algorithm, ompi_coll_tuned_forced_max_algorithms[REDUCESCATTERBLOCK]));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "coll:tuned:reduce_scatter_block_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
+        algorithm, ompi_coll_tuned_forced_max_algorithms[REDUCESCATTERBLOCK]));
     return (MPI_ERR_ARG);
 }
diff --git a/ompi/mca/coll/tuned/coll_tuned_reduce_scatter_decision.c b/ompi/mca/coll/tuned/coll_tuned_reduce_scatter_decision.c
index b4c1b434863..16747598b6e 100644
--- a/ompi/mca/coll/tuned/coll_tuned_reduce_scatter_decision.c
+++ b/ompi/mca/coll/tuned/coll_tuned_reduce_scatter_decision.c
@@ -5,6 +5,8 @@
  *                         reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2025      Amazon.com, Inc. or its affiliates.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -83,6 +85,7 @@ int ompi_coll_tuned_reduce_scatter_intra_check_forced_init (coll_tuned_force_alg
                                         OPAL_INFO_LVL_5,
                                         MCA_BASE_VAR_SCOPE_ALL,
                                         &coll_tuned_reduce_scatter_forced_algorithm);
+    coll_tuned_alg_register_options( REDUCESCATTER, new_enum );
     OBJ_RELEASE(new_enum);
     if (mca_param_indices->algorithm_param_index < 0) {
         return mca_param_indices->algorithm_param_index;
@@ -122,15 +125,16 @@ int ompi_coll_tuned_reduce_scatter_intra_check_forced_init (coll_tuned_force_alg
 }
 
 int ompi_coll_tuned_reduce_scatter_intra_do_this(const void *sbuf, void* rbuf,
-                                                 const int *rcounts,
+                                                 ompi_count_array_t rcounts,
                                                  struct ompi_datatype_t *dtype,
                                                  struct ompi_op_t *op,
                                                  struct ompi_communicator_t *comm,
                                                  mca_coll_base_module_t *module,
                                                  int algorithm, int faninout, int segsize)
 {
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:reduce_scatter_intra_do_this selected algorithm %d topo faninout %d segsize %d",
-                 algorithm, faninout, segsize));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "coll:tuned:reduce_scatter_intra_do_this selected algorithm %d topo faninout %d segsize %d",
+        algorithm, faninout, segsize));
 
     switch (algorithm) {
     case (0): return ompi_coll_tuned_reduce_scatter_intra_dec_fixed(sbuf, rbuf, rcounts,
@@ -144,7 +148,8 @@ int ompi_coll_tuned_reduce_scatter_intra_do_this(const void *sbuf, void* rbuf,
     case (4): return ompi_coll_base_reduce_scatter_intra_butterfly(sbuf, rbuf, rcounts,
                                                                    dtype, op, comm, module);
     } /* switch */
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:reduce_scatter_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
-                 algorithm, ompi_coll_tuned_forced_max_algorithms[REDUCESCATTER]));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "coll:tuned:reduce_scatter_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
+        algorithm, ompi_coll_tuned_forced_max_algorithms[REDUCESCATTER]));
     return (MPI_ERR_ARG);
 }
diff --git a/ompi/mca/coll/tuned/coll_tuned_scan_decision.c b/ompi/mca/coll/tuned/coll_tuned_scan_decision.c
index f35872023f0..903e76c4694 100644
--- a/ompi/mca/coll/tuned/coll_tuned_scan_decision.c
+++ b/ompi/mca/coll/tuned/coll_tuned_scan_decision.c
@@ -5,6 +5,8 @@
  * Copyright (c) 2020      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2025      Amazon.com, Inc. or its affiliates.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -77,6 +79,7 @@ int ompi_coll_tuned_scan_intra_check_forced_init (coll_tuned_force_algorithm_mca
                                         OPAL_INFO_LVL_5,
                                         MCA_BASE_VAR_SCOPE_ALL,
                                         &coll_tuned_scan_forced_algorithm);
+    coll_tuned_alg_register_options( SCAN, new_enum );
     OBJ_RELEASE(new_enum);
     if (mca_param_indices->algorithm_param_index < 0) {
         return mca_param_indices->algorithm_param_index;
@@ -85,15 +88,16 @@ int ompi_coll_tuned_scan_intra_check_forced_init (coll_tuned_force_algorithm_mca
     return (MPI_SUCCESS);
 }
 
-int ompi_coll_tuned_scan_intra_do_this(const void *sbuf, void* rbuf, int count,
+int ompi_coll_tuned_scan_intra_do_this(const void *sbuf, void* rbuf, size_t count,
                                          struct ompi_datatype_t *dtype,
                                          struct ompi_op_t *op,
                                          struct ompi_communicator_t *comm,
                                          mca_coll_base_module_t *module,
                                          int algorithm)
 {
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:scan_intra_do_this selected algorithm %d",
-                 algorithm));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "coll:tuned:scan_intra_do_this selected algorithm %d",
+        algorithm));
 
     switch (algorithm) {
     case (0):
@@ -102,7 +106,8 @@ int ompi_coll_tuned_scan_intra_do_this(const void *sbuf, void* rbuf, int count,
     case (2):  return ompi_coll_base_scan_intra_recursivedoubling(sbuf, rbuf, count, dtype,
                                                                   op, comm, module);
     } /* switch */
-    OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:scan_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
-                 algorithm, ompi_coll_tuned_forced_max_algorithms[SCAN]));
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
+        "coll:tuned:scan_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
+        algorithm, ompi_coll_tuned_forced_max_algorithms[SCAN]));
     return (MPI_ERR_ARG);
 }
diff --git a/ompi/mca/coll/tuned/coll_tuned_scatter_decision.c b/ompi/mca/coll/tuned/coll_tuned_scatter_decision.c
index 75a8d64d207..b1449b2955c 100644
--- a/ompi/mca/coll/tuned/coll_tuned_scatter_decision.c
+++ b/ompi/mca/coll/tuned/coll_tuned_scatter_decision.c
@@ -6,6 +6,8 @@
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2019      Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2025      Amazon.com, Inc. or its affiliates.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -82,6 +84,7 @@ ompi_coll_tuned_scatter_intra_check_forced_init(coll_tuned_force_algorithm_mca_p
                                         OPAL_INFO_LVL_5,
                                         MCA_BASE_VAR_SCOPE_ALL,
                                         &coll_tuned_scatter_forced_algorithm);
+    coll_tuned_alg_register_options( SCATTER, new_enum );
     OBJ_RELEASE(new_enum);
     if (mca_param_indices->algorithm_param_index < 0) {
         return mca_param_indices->algorithm_param_index;
@@ -153,16 +156,16 @@ ompi_coll_tuned_scatter_intra_check_forced_init(coll_tuned_force_algorithm_mca_p
 }
 
 int
-ompi_coll_tuned_scatter_intra_do_this(const void *sbuf, int scount,
+ompi_coll_tuned_scatter_intra_do_this(const void *sbuf, size_t scount,
                                       struct ompi_datatype_t *sdtype,
-                                      void* rbuf, int rcount,
+                                      void* rbuf, size_t rcount,
                                       struct ompi_datatype_t *rdtype,
                                       int root,
                                       struct ompi_communicator_t *comm,
                                       mca_coll_base_module_t *module,
                                       int algorithm, int faninout, int segsize)
 {
-    OPAL_OUTPUT((ompi_coll_tuned_stream,
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
                  "coll:tuned:scatter_intra_do_this selected algorithm %d topo faninout %d segsize %d",
                  algorithm, faninout, segsize));
 
@@ -185,7 +188,7 @@ ompi_coll_tuned_scatter_intra_do_this(const void *sbuf, int scount,
                                                       root, comm, module,
                                                       ompi_coll_tuned_scatter_blocking_send_ratio);
     } /* switch */
-    OPAL_OUTPUT((ompi_coll_tuned_stream,
+    OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
                  "coll:tuned:scatter_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
                  algorithm, ompi_coll_tuned_forced_max_algorithms[SCATTER]));
     return MPI_ERR_ARG;
diff --git a/ompi/mca/coll/ucc/Makefile.am b/ompi/mca/coll/ucc/Makefile.am
new file mode 100644
index 00000000000..8d2f4c7c3eb
--- /dev/null
+++ b/ompi/mca/coll/ucc/Makefile.am
@@ -0,0 +1,60 @@
+# -*- shell-script -*-
+#
+#
+# Copyright (c) 2021 Mellanox Technologies. All rights reserved.
+# Copyright (c) 2022 NVIDIA Corporation.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+#
+
+AM_CPPFLAGS = $(coll_ucc_CPPFLAGS)
+
+coll_ucc_sources =                       \
+		coll_ucc.h                       \
+		coll_ucc_debug.h                 \
+		coll_ucc_dtypes.h                \
+		coll_ucc_common.h                \
+		coll_ucc_module.c                \
+		coll_ucc_component.c             \
+		coll_ucc_allgather.c             \
+		coll_ucc_allgatherv.c            \
+		coll_ucc_allreduce.c             \
+		coll_ucc_alltoall.c              \
+		coll_ucc_alltoallv.c             \
+		coll_ucc_barrier.c               \
+		coll_ucc_bcast.c                 \
+		coll_ucc_gather.c                \
+		coll_ucc_gatherv.c               \
+		coll_ucc_reduce.c                \
+		coll_ucc_reduce_scatter_block.c  \
+		coll_ucc_reduce_scatter.c        \
+		coll_ucc_scatter.c               \
+		coll_ucc_scatterv.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_ompi_coll_ucc_DSO
+component_noinst =
+component_install = mca_coll_ucc.la
+else
+component_noinst = libmca_coll_ucc.la
+component_install =
+endif
+
+mcacomponentdir          = $(ompilibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_coll_ucc_la_SOURCES  = $(coll_ucc_sources)
+mca_coll_ucc_la_LIBADD   = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
+	                     $(coll_ucc_LIBS)
+mca_coll_ucc_la_LDFLAGS  = -module -avoid-version $(coll_ucc_LDFLAGS)
+
+noinst_LTLIBRARIES         = $(component_noinst)
+libmca_coll_ucc_la_SOURCES = $(coll_ucc_sources)
+libmca_coll_ucc_la_LIBADD  = $(coll_ucc_LIBS)
+libmca_coll_ucc_la_LDFLAGS = -module -avoid-version $(coll_ucc_LDFLAGS)
diff --git a/ompi/mca/coll/ucc/coll_ucc.h b/ompi/mca/coll/ucc/coll_ucc.h
new file mode 100644
index 00000000000..510e4796448
--- /dev/null
+++ b/ompi/mca/coll/ucc/coll_ucc.h
@@ -0,0 +1,309 @@
+/**
+  Copyright (c) 2021      Mellanox Technologies. All rights reserved.
+  Copyright (c) 2022 NVIDIA Corporation.  All rights reserved.
+  $COPYRIGHT$
+
+  Additional copyrights may follow
+
+  $HEADER$
+ */
+
+#ifndef MCA_COLL_UCC_H
+#define MCA_COLL_UCC_H
+
+#include "ompi_config.h"
+#include "mpi.h"
+#include "ompi/mca/mca.h"
+#include "opal/memoryhooks/memory.h"
+#include "opal/mca/memory/base/base.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/attribute/attribute.h"
+#include "ompi/op/op.h"
+#include "coll_ucc_debug.h"
+#include <ucc/api/ucc.h>
+
+BEGIN_C_DECLS
+
+#define COLL_UCC_CTS (UCC_COLL_TYPE_BARRIER        | UCC_COLL_TYPE_BCAST    | \
+                      UCC_COLL_TYPE_ALLREDUCE      | UCC_COLL_TYPE_ALLTOALL | \
+                      UCC_COLL_TYPE_ALLTOALLV      | UCC_COLL_TYPE_ALLGATHER | \
+                      UCC_COLL_TYPE_REDUCE         | UCC_COLL_TYPE_ALLGATHERV | \
+                      UCC_COLL_TYPE_GATHER         | UCC_COLL_TYPE_GATHERV | \
+                      UCC_COLL_TYPE_REDUCE_SCATTER | UCC_COLL_TYPE_REDUCE_SCATTERV | \
+                      UCC_COLL_TYPE_SCATTERV       | UCC_COLL_TYPE_SCATTER)
+
+#define COLL_UCC_CTS_STR "barrier,bcast,allreduce,alltoall,alltoallv,allgather," \
+                         "allgatherv,reduce,gather,gatherv,reduce_scatter_block,"\
+                         "reduce_scatter,scatterv,scatter," \
+                         "ibarrier,ibcast,iallreduce,ialltoall,ialltoallv,iallgather,"\
+                         "iallgatherv,ireduce,igather,igatherv,ireduce_scatter_block,"\
+                         "ireduce_scatter,iscatterv,iscatter"
+
+typedef struct mca_coll_ucc_req {
+    ompi_request_t super;
+    ucc_coll_req_h ucc_req;
+} mca_coll_ucc_req_t;
+OBJ_CLASS_DECLARATION(mca_coll_ucc_req_t);
+
+struct mca_coll_ucc_component_t {
+    mca_coll_base_component_3_0_0_t super;
+    int                             ucc_priority;
+    int                             ucc_verbose;
+    int                             ucc_enable;
+    int                             ucc_np;
+    char                           *cls;
+    char                           *cts;
+    const char                     *compiletime_version;
+    const char                     *runtime_version;
+    bool                            libucc_initialized;
+    ucc_lib_h                       ucc_lib;
+    ucc_lib_attr_t                  ucc_lib_attr;
+    ucc_coll_type_t                 cts_requested;
+    ucc_coll_type_t                 nb_cts_requested;
+    ucc_context_h                   ucc_context;
+    opal_free_list_t                requests;
+};
+typedef struct mca_coll_ucc_component_t mca_coll_ucc_component_t;
+
+OMPI_DECLSPEC extern mca_coll_ucc_component_t mca_coll_ucc_component;
+
+/**
+ * UCC enabled communicator
+ */
+struct mca_coll_ucc_module_t {
+    mca_coll_base_module_t                          super;
+    ompi_communicator_t*                            comm;
+    int                                             rank;
+    ucc_team_h                                      ucc_team;
+    mca_coll_base_module_allreduce_fn_t             previous_allreduce;
+    mca_coll_base_module_t*                         previous_allreduce_module;
+    mca_coll_base_module_iallreduce_fn_t            previous_iallreduce;
+    mca_coll_base_module_t*                         previous_iallreduce_module;
+    mca_coll_base_module_reduce_fn_t                previous_reduce;
+    mca_coll_base_module_t*                         previous_reduce_module;
+    mca_coll_base_module_ireduce_fn_t               previous_ireduce;
+    mca_coll_base_module_t*                         previous_ireduce_module;
+    mca_coll_base_module_barrier_fn_t               previous_barrier;
+    mca_coll_base_module_t*                         previous_barrier_module;
+    mca_coll_base_module_ibarrier_fn_t              previous_ibarrier;
+    mca_coll_base_module_t*                         previous_ibarrier_module;
+    mca_coll_base_module_bcast_fn_t                 previous_bcast;
+    mca_coll_base_module_t*                         previous_bcast_module;
+    mca_coll_base_module_ibcast_fn_t                previous_ibcast;
+    mca_coll_base_module_t*                         previous_ibcast_module;
+    mca_coll_base_module_alltoall_fn_t              previous_alltoall;
+    mca_coll_base_module_t*                         previous_alltoall_module;
+    mca_coll_base_module_ialltoall_fn_t             previous_ialltoall;
+    mca_coll_base_module_t*                         previous_ialltoall_module;
+    mca_coll_base_module_alltoallv_fn_t             previous_alltoallv;
+    mca_coll_base_module_t*                         previous_alltoallv_module;
+    mca_coll_base_module_ialltoallv_fn_t            previous_ialltoallv;
+    mca_coll_base_module_t*                         previous_ialltoallv_module;
+    mca_coll_base_module_allgather_fn_t             previous_allgather;
+    mca_coll_base_module_t*                         previous_allgather_module;
+    mca_coll_base_module_iallgather_fn_t            previous_iallgather;
+    mca_coll_base_module_t*                         previous_iallgather_module;
+    mca_coll_base_module_allgatherv_fn_t            previous_allgatherv;
+    mca_coll_base_module_t*                         previous_allgatherv_module;
+    mca_coll_base_module_iallgatherv_fn_t           previous_iallgatherv;
+    mca_coll_base_module_t*                         previous_iallgatherv_module;
+    mca_coll_base_module_gather_fn_t                previous_gather;
+    mca_coll_base_module_t*                         previous_gather_module;
+    mca_coll_base_module_igather_fn_t               previous_igather;
+    mca_coll_base_module_t*                         previous_igather_module;
+    mca_coll_base_module_gatherv_fn_t               previous_gatherv;
+    mca_coll_base_module_t*                         previous_gatherv_module;
+    mca_coll_base_module_igatherv_fn_t              previous_igatherv;
+    mca_coll_base_module_t*                         previous_igatherv_module;
+    mca_coll_base_module_reduce_scatter_block_fn_t  previous_reduce_scatter_block;
+    mca_coll_base_module_t*                         previous_reduce_scatter_block_module;
+    mca_coll_base_module_ireduce_scatter_block_fn_t previous_ireduce_scatter_block;
+    mca_coll_base_module_t*                         previous_ireduce_scatter_block_module;
+    mca_coll_base_module_reduce_scatter_fn_t        previous_reduce_scatter;
+    mca_coll_base_module_t*                         previous_reduce_scatter_module;
+    mca_coll_base_module_ireduce_scatter_fn_t       previous_ireduce_scatter;
+    mca_coll_base_module_t*                         previous_ireduce_scatter_module;
+    mca_coll_base_module_scatterv_fn_t              previous_scatterv;
+    mca_coll_base_module_t*                         previous_scatterv_module;
+    mca_coll_base_module_iscatterv_fn_t             previous_iscatterv;
+    mca_coll_base_module_t*                         previous_iscatterv_module;
+    mca_coll_base_module_scatter_fn_t               previous_scatter;
+    mca_coll_base_module_t*                         previous_scatter_module;
+    mca_coll_base_module_iscatter_fn_t              previous_iscatter;
+    mca_coll_base_module_t*                         previous_iscatter_module;
+};
+typedef struct mca_coll_ucc_module_t mca_coll_ucc_module_t;
+OBJ_CLASS_DECLARATION(mca_coll_ucc_module_t);
+
+int mca_coll_ucc_init_query(bool enable_progress_threads, bool enable_mpi_threads);
+mca_coll_base_module_t *mca_coll_ucc_comm_query(struct ompi_communicator_t *comm, int *priority);
+
+int mca_coll_ucc_allreduce(const void *sbuf, void *rbuf, size_t count,
+                           struct ompi_datatype_t *dtype, struct ompi_op_t *op,
+                           struct ompi_communicator_t *comm,
+                           mca_coll_base_module_t *module);
+
+int mca_coll_ucc_iallreduce(const void *sbuf, void *rbuf, size_t count,
+                            struct ompi_datatype_t *dtype, struct ompi_op_t *op,
+                            struct ompi_communicator_t *comm,
+                            ompi_request_t** request,
+                            mca_coll_base_module_t *module);
+
+int mca_coll_ucc_reduce(const void *sbuf, void* rbuf, size_t count,
+                        struct ompi_datatype_t *dtype, struct ompi_op_t *op,
+                        int root, struct ompi_communicator_t *comm,
+                        struct mca_coll_base_module_3_0_0_t *module);
+
+int mca_coll_ucc_ireduce(const void *sbuf, void* rbuf, size_t count,
+                         struct ompi_datatype_t *dtype, struct ompi_op_t *op,
+                         int root, struct ompi_communicator_t *comm,
+                         ompi_request_t** request,
+                         struct mca_coll_base_module_3_0_0_t *module);
+
+int mca_coll_ucc_barrier(struct ompi_communicator_t *comm,
+                         mca_coll_base_module_t *module);
+
+int mca_coll_ucc_ibarrier(struct ompi_communicator_t *comm,
+                          ompi_request_t** request,
+                          mca_coll_base_module_t *module);
+
+int mca_coll_ucc_bcast(void *buf, size_t count, struct ompi_datatype_t *dtype,
+                       int root, struct ompi_communicator_t *comm,
+                       mca_coll_base_module_t *module);
+
+int mca_coll_ucc_ibcast(void *buf, size_t count, struct ompi_datatype_t *dtype,
+                        int root, struct ompi_communicator_t *comm,
+                        ompi_request_t** request,
+                        mca_coll_base_module_t *module);
+
+int mca_coll_ucc_alltoall(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                          void* rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
+                          struct ompi_communicator_t *comm,
+                          mca_coll_base_module_t *module);
+
+int mca_coll_ucc_ialltoall(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                           void* rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
+                           struct ompi_communicator_t *comm,
+                           ompi_request_t** request,
+                           mca_coll_base_module_t *module);
+
+int mca_coll_ucc_alltoallv(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdips,
+                           struct ompi_datatype_t *sdtype,
+                           void* rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
+                           struct ompi_datatype_t *rdtype,
+                           struct ompi_communicator_t *comm,
+                           mca_coll_base_module_t *module);
+
+int mca_coll_ucc_ialltoallv(const void *sbuf, ompi_count_array_t scounts, ompi_disp_array_t sdips,
+                            struct ompi_datatype_t *sdtype,
+                            void* rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
+                            struct ompi_datatype_t *rdtype,
+                            struct ompi_communicator_t *comm,
+                            ompi_request_t** request,
+                            mca_coll_base_module_t *module);
+
+int mca_coll_ucc_allgather(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                           void* rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
+                           struct ompi_communicator_t *comm,
+                           mca_coll_base_module_t *module);
+
+int mca_coll_ucc_iallgather(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                            void* rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
+                            struct ompi_communicator_t *comm,
+                            ompi_request_t** request,
+                            mca_coll_base_module_t *module);
+
+int mca_coll_ucc_allgatherv(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                            void* rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
+                            struct ompi_datatype_t *rdtype,
+                            struct ompi_communicator_t *comm,
+                            mca_coll_base_module_t *module);
+
+int mca_coll_ucc_iallgatherv(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                             void* rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
+                             struct ompi_datatype_t *rdtype,
+                             struct ompi_communicator_t *comm,
+                             ompi_request_t** request,
+                             mca_coll_base_module_t *module);
+
+int mca_coll_ucc_gather(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                        void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
+                        int root, struct ompi_communicator_t *comm,
+                        mca_coll_base_module_t *module);
+
+int mca_coll_ucc_igather(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                         void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
+                         int root, struct ompi_communicator_t *comm,
+                         ompi_request_t** request,
+                         mca_coll_base_module_t *module);
+
+int mca_coll_ucc_gatherv(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                         void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
+                         struct ompi_datatype_t *rdtype, int root,
+                         struct ompi_communicator_t *comm,
+                         mca_coll_base_module_t *module);
+
+int mca_coll_ucc_igatherv(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                          void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
+                          struct ompi_datatype_t *rdtype, int root,
+                          struct ompi_communicator_t *comm,
+                          ompi_request_t** request,
+                          mca_coll_base_module_t *module);
+
+int mca_coll_ucc_reduce_scatter_block(const void *sbuf, void *rbuf, size_t rcount,
+                                      struct ompi_datatype_t *dtype,
+                                      struct ompi_op_t *op,
+                                      struct ompi_communicator_t *comm,
+                                      mca_coll_base_module_t *module);
+
+int mca_coll_ucc_ireduce_scatter_block(const void *sbuf, void *rbuf, size_t rcount,
+                                       struct ompi_datatype_t *dtype,
+                                       struct ompi_op_t *op,
+                                       struct ompi_communicator_t *comm,
+                                       ompi_request_t** request,
+                                       mca_coll_base_module_t *module);
+
+int mca_coll_ucc_reduce_scatter(const void *sbuf, void *rbuf, ompi_count_array_t rcounts,
+                                struct ompi_datatype_t *dtype,
+                                struct ompi_op_t *op,
+                                struct ompi_communicator_t *comm,
+                                mca_coll_base_module_t *module);
+
+int mca_coll_ucc_ireduce_scatter(const void *sbuf, void *rbuf, ompi_count_array_t rcounts,
+                                struct ompi_datatype_t *dtype,
+                                struct ompi_op_t *op,
+                                struct ompi_communicator_t *comm,
+                                ompi_request_t** request,
+                                mca_coll_base_module_t *module);
+
+int mca_coll_ucc_scatterv(const void *sbuf, ompi_count_array_t scounts,
+                          ompi_disp_array_t disps, struct ompi_datatype_t *sdtype,
+                          void *rbuf, size_t rcount,
+                          struct ompi_datatype_t *rdtype, int root,
+                          struct ompi_communicator_t *comm,
+                          mca_coll_base_module_t *module);
+
+int mca_coll_ucc_iscatterv(const void *sbuf, ompi_count_array_t scounts,
+                           ompi_disp_array_t disps, struct ompi_datatype_t *sdtype,
+                           void *rbuf, size_t rcount,
+                           struct ompi_datatype_t *rdtype, int root,
+                           struct ompi_communicator_t *comm,
+                           ompi_request_t** request,
+                           mca_coll_base_module_t *module);
+
+int mca_coll_ucc_scatter(const void *sbuf, size_t scount,
+                         struct ompi_datatype_t *sdtype, void *rbuf, size_t rcount,
+                         struct ompi_datatype_t *rdtype, int root,
+                         struct ompi_communicator_t *comm,
+                         mca_coll_base_module_t *module);
+
+int mca_coll_ucc_iscatter(const void *sbuf, size_t scount,
+                         struct ompi_datatype_t *sdtype, void *rbuf, size_t rcount,
+                         struct ompi_datatype_t *rdtype, int root,
+                         struct ompi_communicator_t *comm,
+                         ompi_request_t** request,
+                         mca_coll_base_module_t *module);
+
+END_C_DECLS
+#endif
diff --git a/ompi/mca/coll/ucc/coll_ucc_allgather.c b/ompi/mca/coll/ucc/coll_ucc_allgather.c
new file mode 100644
index 00000000000..2dd3ac68a55
--- /dev/null
+++ b/ompi/mca/coll/ucc/coll_ucc_allgather.c
@@ -0,0 +1,114 @@
+
+/**
+ * Copyright (c) 2021 Mellanox Technologies. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ */
+
+#include "coll_ucc_common.h"
+
+static inline ucc_status_t mca_coll_ucc_allgather_init(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                                                       void* rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
+                                                       mca_coll_ucc_module_t *ucc_module,
+                                                       ucc_coll_req_h *req,
+                                                       mca_coll_ucc_req_t *coll_req)
+{
+    ucc_datatype_t ucc_sdt = UCC_DT_INT8, ucc_rdt = UCC_DT_INT8;
+    bool is_inplace = (MPI_IN_PLACE == sbuf);
+    int comm_size = ompi_comm_size(ucc_module->comm);
+
+    if (!(is_inplace || ompi_datatype_is_contiguous_memory_layout(sdtype, scount)) ||
+        !ompi_datatype_is_contiguous_memory_layout(rdtype, rcount * comm_size)) {
+        goto fallback;
+    }
+
+    ucc_rdt = ompi_dtype_to_ucc_dtype(rdtype);
+    if (!is_inplace) {
+        ucc_sdt = ompi_dtype_to_ucc_dtype(sdtype);
+    }
+
+    if (COLL_UCC_DT_UNSUPPORTED == ucc_sdt ||
+        COLL_UCC_DT_UNSUPPORTED == ucc_rdt) {
+        UCC_VERBOSE(5, "ompi_datatype is not supported: dtype = %s",
+                    (COLL_UCC_DT_UNSUPPORTED == ucc_sdt) ?
+                    sdtype->super.name : rdtype->super.name);
+        goto fallback;
+    }
+
+    ucc_coll_args_t coll = {
+        .mask      = 0,
+        .flags     = 0,
+        .coll_type = UCC_COLL_TYPE_ALLGATHER,
+        .src.info = {
+            .buffer   = (void*)sbuf,
+            .count    = scount,
+            .datatype = ucc_sdt,
+            .mem_type = UCC_MEMORY_TYPE_UNKNOWN
+        },
+        .dst.info = {
+            .buffer   = (void*)rbuf,
+            .count    = rcount * comm_size,
+            .datatype = ucc_rdt,
+            .mem_type = UCC_MEMORY_TYPE_UNKNOWN
+        }
+    };
+
+    if (is_inplace) {
+        coll.mask  = UCC_COLL_ARGS_FIELD_FLAGS;
+        coll.flags = UCC_COLL_ARGS_FLAG_IN_PLACE;
+    }
+    COLL_UCC_REQ_INIT(coll_req, req, coll, ucc_module);
+    return UCC_OK;
+fallback:
+    return UCC_ERR_NOT_SUPPORTED;
+}
+
+int mca_coll_ucc_allgather(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                           void* rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
+                           struct ompi_communicator_t *comm,
+                           mca_coll_base_module_t *module)
+{
+    mca_coll_ucc_module_t *ucc_module = (mca_coll_ucc_module_t*)module;
+    ucc_coll_req_h         req;
+
+    UCC_VERBOSE(3, "running ucc allgather");
+    COLL_UCC_CHECK(mca_coll_ucc_allgather_init(sbuf, scount, sdtype,
+                                               rbuf, rcount, rdtype,
+                                               ucc_module, &req, NULL));
+    COLL_UCC_POST_AND_CHECK(req);
+    COLL_UCC_CHECK(coll_ucc_req_wait(req));
+    return OMPI_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback allgather");
+    return ucc_module->previous_allgather(sbuf, scount, sdtype, rbuf, rcount, rdtype,
+                                          comm, ucc_module->previous_allgather_module);
+}
+
+int mca_coll_ucc_iallgather(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                            void* rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
+                            struct ompi_communicator_t *comm,
+                            ompi_request_t** request,
+                            mca_coll_base_module_t *module)
+{
+    mca_coll_ucc_module_t *ucc_module = (mca_coll_ucc_module_t*)module;
+    ucc_coll_req_h         req;
+    mca_coll_ucc_req_t    *coll_req = NULL;
+
+    UCC_VERBOSE(3, "running ucc iallgather");
+    COLL_UCC_GET_REQ(coll_req);
+    COLL_UCC_CHECK(mca_coll_ucc_allgather_init(sbuf, scount, sdtype,
+                                               rbuf, rcount, rdtype,
+                                               ucc_module, &req, coll_req));
+    COLL_UCC_POST_AND_CHECK(req);
+    *request = &coll_req->super;
+    return OMPI_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback iallgather");
+    if (coll_req) {
+        mca_coll_ucc_req_free((ompi_request_t **)&coll_req);
+    }
+    return ucc_module->previous_iallgather(sbuf, scount, sdtype, rbuf, rcount, rdtype,
+                                           comm, request, ucc_module->previous_iallgather_module);
+}
diff --git a/ompi/mca/coll/ucc/coll_ucc_allgatherv.c b/ompi/mca/coll/ucc/coll_ucc_allgatherv.c
new file mode 100644
index 00000000000..68e786e0c2a
--- /dev/null
+++ b/ompi/mca/coll/ucc/coll_ucc_allgatherv.c
@@ -0,0 +1,119 @@
+
+/**
+ * Copyright (c) 2021 Mellanox Technologies. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ */
+
+#include "coll_ucc_common.h"
+
+static inline ucc_status_t mca_coll_ucc_allgatherv_init(const void *sbuf, size_t scount,
+                                                        struct ompi_datatype_t *sdtype,
+                                                        void* rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
+                                                        struct ompi_datatype_t *rdtype,
+                                                        mca_coll_ucc_module_t *ucc_module,
+                                                        ucc_coll_req_h *req,
+                                                        mca_coll_ucc_req_t *coll_req)
+{
+    ucc_datatype_t ucc_sdt = UCC_DT_INT8, ucc_rdt = UCC_DT_INT8;
+    bool is_inplace = (MPI_IN_PLACE == sbuf);
+    uint64_t flags = 0;
+
+    ucc_rdt = ompi_dtype_to_ucc_dtype(rdtype);
+    if (!is_inplace) {
+        ucc_sdt = ompi_dtype_to_ucc_dtype(sdtype);
+    }
+
+    if (COLL_UCC_DT_UNSUPPORTED == ucc_sdt ||
+        COLL_UCC_DT_UNSUPPORTED == ucc_rdt) {
+        UCC_VERBOSE(5, "ompi_datatype is not supported: dtype = %s",
+                    (COLL_UCC_DT_UNSUPPORTED == ucc_sdt) ?
+                    sdtype->super.name : rdtype->super.name);
+        goto fallback;
+    }
+
+    flags = (ompi_count_array_is_64bit(rcounts) ? UCC_COLL_ARGS_FLAG_COUNT_64BIT : 0) |
+            (ompi_disp_array_is_64bit(rdisps) ? UCC_COLL_ARGS_FLAG_DISPLACEMENTS_64BIT : 0) |
+            (is_inplace ? UCC_COLL_ARGS_FLAG_IN_PLACE : 0);
+
+    ucc_coll_args_t coll = {
+        .mask      = flags ? UCC_COLL_ARGS_FIELD_FLAGS : 0,
+        .flags     = flags,
+        .coll_type = UCC_COLL_TYPE_ALLGATHERV,
+        .src.info = {
+            .buffer        = (void*)sbuf,
+            .count         = scount,
+            .datatype      = ucc_sdt,
+            .mem_type      = UCC_MEMORY_TYPE_UNKNOWN
+        },
+        .dst.info_v = {
+            .buffer        = (void*)rbuf,
+            .counts        = (ucc_count_t*)ompi_count_array_ptr(rcounts),
+            .displacements = (ucc_aint_t*)ompi_disp_array_ptr(rdisps),
+            .datatype      = ucc_rdt,
+            .mem_type      = UCC_MEMORY_TYPE_UNKNOWN
+        }
+    };
+
+    COLL_UCC_REQ_INIT(coll_req, req, coll, ucc_module);
+    return UCC_OK;
+fallback:
+    return UCC_ERR_NOT_SUPPORTED;
+}
+
+int mca_coll_ucc_allgatherv(const void *sbuf, size_t scount,
+                            struct ompi_datatype_t *sdtype,
+                            void* rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
+                            struct ompi_datatype_t *rdtype,
+                            struct ompi_communicator_t *comm,
+                            mca_coll_base_module_t *module)
+{
+    mca_coll_ucc_module_t *ucc_module = (mca_coll_ucc_module_t*)module;
+    ucc_coll_req_h         req;
+
+    UCC_VERBOSE(3, "running ucc allgatherv");
+
+    COLL_UCC_CHECK(mca_coll_ucc_allgatherv_init(sbuf, scount, sdtype,
+                                                rbuf, rcounts, rdisps, rdtype,
+                                                ucc_module, &req, NULL));
+    COLL_UCC_POST_AND_CHECK(req);
+    COLL_UCC_CHECK(coll_ucc_req_wait(req));
+    return OMPI_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback allgatherv");
+    return ucc_module->previous_allgatherv(sbuf, scount, sdtype,
+                                           rbuf, rcounts, rdisps, rdtype,
+                                           comm, ucc_module->previous_allgatherv_module);
+}
+
+int mca_coll_ucc_iallgatherv(const void *sbuf, size_t scount,
+                             struct ompi_datatype_t *sdtype,
+                             void* rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
+                             struct ompi_datatype_t *rdtype,
+                             struct ompi_communicator_t *comm,
+                             ompi_request_t** request,
+                             mca_coll_base_module_t *module)
+{
+    mca_coll_ucc_module_t *ucc_module = (mca_coll_ucc_module_t*)module;
+    ucc_coll_req_h         req;
+    mca_coll_ucc_req_t    *coll_req = NULL;
+
+    UCC_VERBOSE(3, "running ucc iallgatherv");
+    COLL_UCC_GET_REQ(coll_req);
+    COLL_UCC_CHECK(mca_coll_ucc_allgatherv_init(sbuf, scount, sdtype,
+                                                rbuf, rcounts, rdisps, rdtype,
+                                                ucc_module, &req, coll_req));
+    COLL_UCC_POST_AND_CHECK(req);
+    *request = &coll_req->super;
+    return OMPI_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback iallgatherv");
+    if (coll_req) {
+        mca_coll_ucc_req_free((ompi_request_t **)&coll_req);
+    }
+    return ucc_module->previous_iallgatherv(sbuf, scount, sdtype,
+                                            rbuf, rcounts, rdisps, rdtype,
+                                            comm, request, ucc_module->previous_iallgatherv_module);
+}
diff --git a/ompi/mca/coll/ucc/coll_ucc_allreduce.c b/ompi/mca/coll/ucc/coll_ucc_allreduce.c
new file mode 100644
index 00000000000..d44b93df07e
--- /dev/null
+++ b/ompi/mca/coll/ucc/coll_ucc_allreduce.c
@@ -0,0 +1,105 @@
+
+/**
+ * Copyright (c) 2021 Mellanox Technologies. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ */
+
+#include "coll_ucc_common.h"
+
+static inline ucc_status_t mca_coll_ucc_allreduce_init(const void *sbuf, void *rbuf, size_t count,
+                                                       struct ompi_datatype_t *dtype,
+                                                       struct ompi_op_t *op, mca_coll_ucc_module_t *ucc_module,
+                                                       ucc_coll_req_h *req,
+                                                       mca_coll_ucc_req_t *coll_req)
+{
+    ucc_datatype_t         ucc_dt;
+    ucc_reduction_op_t     ucc_op;
+
+    ucc_dt = ompi_dtype_to_ucc_dtype(dtype);
+    ucc_op = ompi_op_to_ucc_op(op);
+    if (OPAL_UNLIKELY(COLL_UCC_DT_UNSUPPORTED == ucc_dt)) {
+        UCC_VERBOSE(5, "ompi_datatype is not supported: dtype = %s",
+                    dtype->super.name);
+        goto fallback;
+    }
+    if (OPAL_UNLIKELY(COLL_UCC_OP_UNSUPPORTED == ucc_op)) {
+        UCC_VERBOSE(5, "ompi_op is not supported: op = %s",
+                    op->o_name);
+        goto fallback;
+    }
+    ucc_coll_args_t coll = {
+        .mask      = 0,
+        .flags     = 0,
+        .coll_type = UCC_COLL_TYPE_ALLREDUCE,
+        .src.info = {
+            .buffer   = (void*)sbuf,
+            .count    = count,
+            .datatype = ucc_dt,
+            .mem_type = UCC_MEMORY_TYPE_UNKNOWN
+        },
+        .dst.info = {
+            .buffer   = rbuf,
+            .count    = count,
+            .datatype = ucc_dt,
+            .mem_type = UCC_MEMORY_TYPE_UNKNOWN
+        },
+        .op = ucc_op,
+    };
+    if (MPI_IN_PLACE == sbuf) {
+        coll.mask |= UCC_COLL_ARGS_FIELD_FLAGS;
+        coll.flags = UCC_COLL_ARGS_FLAG_IN_PLACE;
+    }
+    COLL_UCC_REQ_INIT(coll_req, req, coll, ucc_module);
+    return UCC_OK;
+fallback:
+    return UCC_ERR_NOT_SUPPORTED;
+}
+
+int mca_coll_ucc_allreduce(const void *sbuf, void *rbuf, size_t count,
+                           struct ompi_datatype_t *dtype,
+                           struct ompi_op_t *op, struct ompi_communicator_t *comm,
+                           mca_coll_base_module_t *module)
+{
+    mca_coll_ucc_module_t *ucc_module = (mca_coll_ucc_module_t*)module;
+    ucc_coll_req_h         req;
+
+    UCC_VERBOSE(3, "running ucc allreduce");
+    COLL_UCC_CHECK(mca_coll_ucc_allreduce_init(sbuf, rbuf, count, dtype, op,
+                                               ucc_module, &req, NULL));
+    COLL_UCC_POST_AND_CHECK(req);
+    COLL_UCC_CHECK(coll_ucc_req_wait(req));
+    return OMPI_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback allreduce");
+    return ucc_module->previous_allreduce(sbuf, rbuf, count, dtype, op,
+                                          comm, ucc_module->previous_allreduce_module);
+}
+
+int mca_coll_ucc_iallreduce(const void *sbuf, void *rbuf, size_t count,
+                            struct ompi_datatype_t *dtype,
+                            struct ompi_op_t *op, struct ompi_communicator_t *comm,
+                            ompi_request_t** request,
+                            mca_coll_base_module_t *module)
+{
+    mca_coll_ucc_module_t *ucc_module = (mca_coll_ucc_module_t*)module;
+    ucc_coll_req_h         req;
+    mca_coll_ucc_req_t    *coll_req = NULL;
+
+    UCC_VERBOSE(3, "running ucc iallreduce");
+    COLL_UCC_GET_REQ(coll_req);
+    COLL_UCC_CHECK(mca_coll_ucc_allreduce_init(sbuf, rbuf, count, dtype, op,
+                                               ucc_module, &req, coll_req));
+    COLL_UCC_POST_AND_CHECK(req);
+    *request = &coll_req->super;
+    return OMPI_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback iallreduce");
+    if (coll_req) {
+        mca_coll_ucc_req_free((ompi_request_t **)&coll_req);
+    }
+    return ucc_module->previous_iallreduce(sbuf, rbuf, count, dtype, op,
+                                           comm, request, ucc_module->previous_iallreduce_module);
+}
diff --git a/ompi/mca/coll/ucc/coll_ucc_alltoall.c b/ompi/mca/coll/ucc/coll_ucc_alltoall.c
new file mode 100644
index 00000000000..cfb56f47418
--- /dev/null
+++ b/ompi/mca/coll/ucc/coll_ucc_alltoall.c
@@ -0,0 +1,114 @@
+
+/**
+ * Copyright (c) 2021 Mellanox Technologies. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ */
+
+#include "coll_ucc_common.h"
+
+static inline ucc_status_t mca_coll_ucc_alltoall_init(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                                                      void* rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
+                                                      mca_coll_ucc_module_t *ucc_module,
+                                                      ucc_coll_req_h *req,
+                                                      mca_coll_ucc_req_t *coll_req)
+{
+    ucc_datatype_t ucc_sdt = UCC_DT_INT8, ucc_rdt = UCC_DT_INT8;
+    bool is_inplace = (MPI_IN_PLACE == sbuf);
+    int comm_size = ompi_comm_size(ucc_module->comm);
+
+    if (!(is_inplace || ompi_datatype_is_contiguous_memory_layout(sdtype, scount * comm_size)) ||
+        !ompi_datatype_is_contiguous_memory_layout(rdtype, rcount * comm_size)) {
+        goto fallback;
+    }
+
+    ucc_rdt = ompi_dtype_to_ucc_dtype(rdtype);
+    if (!is_inplace) {
+        ucc_sdt = ompi_dtype_to_ucc_dtype(sdtype);
+    }
+
+    if (COLL_UCC_DT_UNSUPPORTED == ucc_sdt ||
+        COLL_UCC_DT_UNSUPPORTED == ucc_rdt) {
+        UCC_VERBOSE(5, "ompi_datatype is not supported: dtype = %s",
+                    (COLL_UCC_DT_UNSUPPORTED == ucc_sdt) ?
+                    sdtype->super.name : rdtype->super.name);
+        goto fallback;
+    }
+
+    ucc_coll_args_t coll = {
+        .mask      = 0,
+        .flags     = 0,
+        .coll_type = UCC_COLL_TYPE_ALLTOALL,
+        .src.info = {
+            .buffer   = (void*)sbuf,
+            .count    = scount * comm_size,
+            .datatype = ucc_sdt,
+            .mem_type = UCC_MEMORY_TYPE_UNKNOWN
+        },
+        .dst.info = {
+            .buffer   = (void*)rbuf,
+            .count    = rcount * comm_size,
+            .datatype = ucc_rdt,
+            .mem_type = UCC_MEMORY_TYPE_UNKNOWN
+        }
+    };
+
+    if (is_inplace) {
+        coll.mask  = UCC_COLL_ARGS_FIELD_FLAGS;
+        coll.flags = UCC_COLL_ARGS_FLAG_IN_PLACE;
+    }
+    COLL_UCC_REQ_INIT(coll_req, req, coll, ucc_module);
+    return UCC_OK;
+fallback:
+    return UCC_ERR_NOT_SUPPORTED;
+}
+
+int mca_coll_ucc_alltoall(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                          void* rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
+                          struct ompi_communicator_t *comm,
+                          mca_coll_base_module_t *module)
+{
+    mca_coll_ucc_module_t *ucc_module = (mca_coll_ucc_module_t*)module;
+    ucc_coll_req_h         req;
+
+    UCC_VERBOSE(3, "running ucc alltoall");
+    COLL_UCC_CHECK(mca_coll_ucc_alltoall_init(sbuf, scount, sdtype,
+                                              rbuf, rcount, rdtype,
+                                              ucc_module, &req, NULL));
+    COLL_UCC_POST_AND_CHECK(req);
+    COLL_UCC_CHECK(coll_ucc_req_wait(req));
+    return OMPI_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback alltoall");
+    return ucc_module->previous_alltoall(sbuf, scount, sdtype, rbuf, rcount, rdtype,
+                                          comm, ucc_module->previous_alltoall_module);
+}
+
+int mca_coll_ucc_ialltoall(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                           void* rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
+                           struct ompi_communicator_t *comm,
+                           ompi_request_t** request,
+                           mca_coll_base_module_t *module)
+{
+    mca_coll_ucc_module_t *ucc_module = (mca_coll_ucc_module_t*)module;
+    ucc_coll_req_h         req;
+    mca_coll_ucc_req_t    *coll_req = NULL;
+
+    UCC_VERBOSE(3, "running ucc ialltoall");
+    COLL_UCC_GET_REQ(coll_req);
+    COLL_UCC_CHECK(mca_coll_ucc_alltoall_init(sbuf, scount, sdtype,
+                                              rbuf, rcount, rdtype,
+                                              ucc_module, &req, coll_req));
+    COLL_UCC_POST_AND_CHECK(req);
+    *request = &coll_req->super;
+    return OMPI_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback ialltoall");
+    if (coll_req) {
+        mca_coll_ucc_req_free((ompi_request_t **)&coll_req);
+    }
+    return ucc_module->previous_ialltoall(sbuf, scount, sdtype, rbuf, rcount, rdtype,
+                                          comm, request, ucc_module->previous_ialltoall_module);
+}
diff --git a/ompi/mca/coll/ucc/coll_ucc_alltoallv.c b/ompi/mca/coll/ucc/coll_ucc_alltoallv.c
new file mode 100644
index 00000000000..1e9e311cf94
--- /dev/null
+++ b/ompi/mca/coll/ucc/coll_ucc_alltoallv.c
@@ -0,0 +1,121 @@
+
+/**
+ * Copyright (c) 2021 Mellanox Technologies. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ */
+
+#include "coll_ucc_common.h"
+
+static inline ucc_status_t mca_coll_ucc_alltoallv_init(const void *sbuf, ompi_count_array_t scounts,
+                                                       ompi_disp_array_t sdisps, struct ompi_datatype_t *sdtype,
+                                                       void* rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
+                                                       struct ompi_datatype_t *rdtype,
+                                                       mca_coll_ucc_module_t *ucc_module,
+                                                       ucc_coll_req_h *req,
+                                                       mca_coll_ucc_req_t *coll_req)
+{
+    ucc_datatype_t ucc_sdt = UCC_DT_INT8, ucc_rdt = UCC_DT_INT8;
+    bool is_inplace = (MPI_IN_PLACE == sbuf);
+    uint64_t flags = 0;
+
+    ucc_rdt = ompi_dtype_to_ucc_dtype(rdtype);
+    if (!is_inplace) {
+        ucc_sdt = ompi_dtype_to_ucc_dtype(sdtype);
+    }
+
+    if (COLL_UCC_DT_UNSUPPORTED == ucc_sdt ||
+        COLL_UCC_DT_UNSUPPORTED == ucc_rdt) {
+        UCC_VERBOSE(5, "ompi_datatype is not supported: dtype = %s",
+                    (COLL_UCC_DT_UNSUPPORTED == ucc_sdt) ?
+                    sdtype->super.name : rdtype->super.name);
+        goto fallback;
+    }
+
+    /* Assumes that send counts/displs and recv counts/displs are both 32-bit or both 64-bit */
+    flags = (ompi_count_array_is_64bit(scounts) ? UCC_COLL_ARGS_FLAG_COUNT_64BIT : 0) |
+            (ompi_disp_array_is_64bit(sdisps) ? UCC_COLL_ARGS_FLAG_DISPLACEMENTS_64BIT : 0) |
+            (is_inplace ? UCC_COLL_ARGS_FLAG_IN_PLACE : 0);
+
+    ucc_coll_args_t coll = {
+        .mask      = flags ? UCC_COLL_ARGS_FIELD_FLAGS : 0,
+        .flags     = flags,
+        .coll_type = UCC_COLL_TYPE_ALLTOALLV,
+        .src.info_v = {
+            .buffer        = (void*)sbuf,
+            .counts        = (ucc_count_t*)ompi_count_array_ptr(scounts),
+            .displacements = (ucc_aint_t*)ompi_disp_array_ptr(sdisps),
+            .datatype      = ucc_sdt,
+            .mem_type      = UCC_MEMORY_TYPE_UNKNOWN
+        },
+        .dst.info_v = {
+            .buffer        = (void*)rbuf,
+            .counts        = (ucc_count_t*)ompi_count_array_ptr(rcounts),
+            .displacements = (ucc_aint_t*)ompi_disp_array_ptr(rdisps),
+            .datatype      = ucc_rdt,
+            .mem_type      = UCC_MEMORY_TYPE_UNKNOWN
+        }
+    };
+
+    COLL_UCC_REQ_INIT(coll_req, req, coll, ucc_module);
+    return UCC_OK;
+fallback:
+    return UCC_ERR_NOT_SUPPORTED;
+}
+
+int mca_coll_ucc_alltoallv(const void *sbuf, ompi_count_array_t scounts,
+                           ompi_disp_array_t sdisps, struct ompi_datatype_t *sdtype,
+                           void* rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
+                           struct ompi_datatype_t *rdtype,
+                           struct ompi_communicator_t *comm,
+                           mca_coll_base_module_t *module)
+{
+    mca_coll_ucc_module_t *ucc_module = (mca_coll_ucc_module_t*)module;
+    ucc_coll_req_h         req;
+
+    UCC_VERBOSE(3, "running ucc alltoallv");
+
+    COLL_UCC_CHECK(mca_coll_ucc_alltoallv_init(sbuf, scounts, sdisps, sdtype,
+                                               rbuf, rcounts, rdisps, rdtype,
+                                               ucc_module, &req, NULL));
+    COLL_UCC_POST_AND_CHECK(req);
+    COLL_UCC_CHECK(coll_ucc_req_wait(req));
+    return OMPI_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback alltoallv");
+    return ucc_module->previous_alltoallv(sbuf, scounts, sdisps, sdtype,
+                                          rbuf, rcounts, rdisps, rdtype,
+                                          comm, ucc_module->previous_alltoallv_module);
+}
+
+int mca_coll_ucc_ialltoallv(const void *sbuf, ompi_count_array_t scounts,
+                            ompi_disp_array_t sdisps, struct ompi_datatype_t *sdtype,
+                            void* rbuf, ompi_count_array_t rcounts, ompi_disp_array_t rdisps,
+                            struct ompi_datatype_t *rdtype,
+                            struct ompi_communicator_t *comm,
+                            ompi_request_t** request,
+                            mca_coll_base_module_t *module)
+{
+    mca_coll_ucc_module_t *ucc_module = (mca_coll_ucc_module_t*)module;
+    ucc_coll_req_h         req;
+    mca_coll_ucc_req_t    *coll_req = NULL;
+
+    UCC_VERBOSE(3, "running ucc ialltoallv");
+    COLL_UCC_GET_REQ(coll_req);
+    COLL_UCC_CHECK(mca_coll_ucc_alltoallv_init(sbuf, scounts, sdisps, sdtype,
+                                               rbuf, rcounts, rdisps, rdtype,
+                                               ucc_module, &req, coll_req));
+    COLL_UCC_POST_AND_CHECK(req);
+    *request = &coll_req->super;
+    return OMPI_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback ialltoallv");
+    if (coll_req) {
+        mca_coll_ucc_req_free((ompi_request_t **)&coll_req);
+    }
+    return ucc_module->previous_ialltoallv(sbuf, scounts, sdisps, sdtype,
+                                          rbuf, rcounts, rdisps, rdtype,
+                                           comm, request, ucc_module->previous_ialltoallv_module);
+}
diff --git a/ompi/mca/coll/ucc/coll_ucc_barrier.c b/ompi/mca/coll/ucc/coll_ucc_barrier.c
new file mode 100644
index 00000000000..010ca177fdf
--- /dev/null
+++ b/ompi/mca/coll/ucc/coll_ucc_barrier.c
@@ -0,0 +1,63 @@
+/**
+ * Copyright (c) 2021 Mellanox Technologies. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ */
+
+#include "coll_ucc_common.h"
+
+static inline ucc_status_t mca_coll_ucc_barrier_init(mca_coll_ucc_module_t *ucc_module,
+                                                     ucc_coll_req_h *req,
+                                                     mca_coll_ucc_req_t *coll_req)
+{
+    ucc_coll_args_t coll = {
+        .mask      = 0,
+        .flags     = 0,
+        .coll_type = UCC_COLL_TYPE_BARRIER
+    };
+    COLL_UCC_REQ_INIT(coll_req, req, coll, ucc_module);
+    return UCC_OK;
+fallback:
+    return UCC_ERR_NOT_SUPPORTED;
+}
+
+int mca_coll_ucc_barrier(struct ompi_communicator_t *comm,
+                         mca_coll_base_module_t *module)
+{
+    mca_coll_ucc_module_t *ucc_module = (mca_coll_ucc_module_t*)module;
+    ucc_coll_req_h         req;
+
+    UCC_VERBOSE(3, "running ucc barrier");
+    COLL_UCC_CHECK(mca_coll_ucc_barrier_init(ucc_module, &req, NULL));
+    COLL_UCC_POST_AND_CHECK(req);
+    COLL_UCC_CHECK(coll_ucc_req_wait(req));
+    return OMPI_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback barrier");
+    return ucc_module->previous_barrier(comm, ucc_module->previous_barrier_module);
+}
+
+int mca_coll_ucc_ibarrier(struct ompi_communicator_t *comm,
+                          ompi_request_t** request,
+                          mca_coll_base_module_t *module)
+{
+    mca_coll_ucc_module_t *ucc_module = (mca_coll_ucc_module_t*)module;
+    ucc_coll_req_h         req;
+    mca_coll_ucc_req_t    *coll_req = NULL;
+
+    UCC_VERBOSE(3, "running ucc ibarrier");
+    COLL_UCC_GET_REQ(coll_req);
+    COLL_UCC_CHECK(mca_coll_ucc_barrier_init(ucc_module, &req, coll_req));
+    COLL_UCC_POST_AND_CHECK(req);
+    *request = &coll_req->super;
+    return OMPI_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback ibarrier");
+    if (coll_req) {
+        mca_coll_ucc_req_free((ompi_request_t **)&coll_req);
+    }
+    return ucc_module->previous_ibarrier(comm, request,
+                                         ucc_module->previous_ibarrier_module);
+}
diff --git a/ompi/mca/coll/ucc/coll_ucc_bcast.c b/ompi/mca/coll/ucc/coll_ucc_bcast.c
new file mode 100644
index 00000000000..34296d74b97
--- /dev/null
+++ b/ompi/mca/coll/ucc/coll_ucc_bcast.c
@@ -0,0 +1,81 @@
+/**
+ * Copyright (c) 2021 Mellanox Technologies. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ */
+
+#include "coll_ucc_common.h"
+
+static inline ucc_status_t mca_coll_ucc_bcast_init(void *buf, size_t count, struct ompi_datatype_t *dtype,
+                                                   int root, mca_coll_ucc_module_t *ucc_module,
+                                                   ucc_coll_req_h *req,
+                                                   mca_coll_ucc_req_t *coll_req)
+{
+    ucc_datatype_t         ucc_dt     = ompi_dtype_to_ucc_dtype(dtype);
+    if (COLL_UCC_DT_UNSUPPORTED == ucc_dt) {
+        UCC_VERBOSE(5, "ompi_datatype is not supported: dtype = %s", dtype->super.name);
+        goto fallback;
+    }
+
+    ucc_coll_args_t coll = {
+        .mask      = 0,
+        .flags     = 0,
+        .coll_type = UCC_COLL_TYPE_BCAST,
+        .root = root,
+        .src.info = {
+            .buffer   = buf,
+            .count    = count,
+            .datatype = ucc_dt,
+            .mem_type = UCC_MEMORY_TYPE_UNKNOWN
+        }
+    };
+    COLL_UCC_REQ_INIT(coll_req, req, coll, ucc_module);
+    return UCC_OK;
+fallback:
+    return UCC_ERR_NOT_SUPPORTED;
+}
+
+int mca_coll_ucc_bcast(void *buf, size_t count, struct ompi_datatype_t *dtype,
+                       int root, struct ompi_communicator_t *comm,
+                       mca_coll_base_module_t *module)
+{
+    mca_coll_ucc_module_t *ucc_module = (mca_coll_ucc_module_t*)module;
+    ucc_coll_req_h         req;
+    UCC_VERBOSE(3, "running ucc bcast");
+    COLL_UCC_CHECK(mca_coll_ucc_bcast_init(buf, count, dtype, root,
+                                           ucc_module, &req, NULL));
+    COLL_UCC_POST_AND_CHECK(req);
+    COLL_UCC_CHECK(coll_ucc_req_wait(req));
+    return OMPI_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback bcast");
+    return ucc_module->previous_bcast(buf, count, dtype, root,
+                                       comm, ucc_module->previous_bcast_module);
+}
+
+int mca_coll_ucc_ibcast(void *buf, size_t count, struct ompi_datatype_t *dtype,
+                        int root, struct ompi_communicator_t *comm,
+                        ompi_request_t** request,
+                        mca_coll_base_module_t *module)
+{
+    mca_coll_ucc_module_t *ucc_module = (mca_coll_ucc_module_t*)module;
+    ucc_coll_req_h         req;
+    mca_coll_ucc_req_t    *coll_req = NULL;
+
+    UCC_VERBOSE(3, "running ucc ibcast");
+    COLL_UCC_GET_REQ(coll_req);
+    COLL_UCC_CHECK(mca_coll_ucc_bcast_init(buf, count, dtype, root,
+                                           ucc_module, &req, coll_req));
+    COLL_UCC_POST_AND_CHECK(req);
+    *request = &coll_req->super;
+    return OMPI_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback ibcast");
+    if (coll_req) {
+        mca_coll_ucc_req_free((ompi_request_t **)&coll_req);
+    }
+    return ucc_module->previous_ibcast(buf, count, dtype, root,
+                                       comm, request, ucc_module->previous_ibcast_module);
+}
diff --git a/ompi/mca/coll/ucc/coll_ucc_common.h b/ompi/mca/coll/ucc/coll_ucc_common.h
new file mode 100644
index 00000000000..9d9163aa46d
--- /dev/null
+++ b/ompi/mca/coll/ucc/coll_ucc_common.h
@@ -0,0 +1,80 @@
+/**
+  Copyright (c) 2021      Mellanox Technologies. All rights reserved.
+  $COPYRIGHT$
+  Additional copyrights may follow
+  $HEADER$
+ */
+
+#ifndef MCA_COLL_UCC_COMMON_H
+#define MCA_COLL_UCC_COMMON_H
+
+#include "ompi/constants.h"
+#include "coll_ucc.h"
+#include "coll_ucc_dtypes.h"
+
+#define COLL_UCC_CHECK(_call) do {              \
+        if (UCC_OK != (_call)) {                \
+            goto fallback;                      \
+        }                                       \
+    } while(0)
+
+#define COLL_UCC_POST_AND_CHECK(_req) do {           \
+        if (UCC_OK != ucc_collective_post(_req)) {   \
+            ucc_collective_finalize(_req);           \
+            goto fallback;                           \
+        }                                            \
+    } while(0)
+
+#define COLL_UCC_GET_REQ(_coll_req) do {                                \
+        opal_free_list_item_t *item;                                    \
+        item = opal_free_list_wait (&mca_coll_ucc_component.requests);  \
+        if (OPAL_UNLIKELY(NULL == item)) {                              \
+            UCC_ERROR("failed to get mca_coll_ucc_req from free_list"); \
+            goto fallback;                                              \
+        }                                                               \
+        _coll_req = (mca_coll_ucc_req_t*)item;                          \
+        OMPI_REQUEST_INIT(&_coll_req->super, false);                    \
+        _coll_req->super.req_complete_cb      = NULL;                   \
+        _coll_req->super.req_complete_cb_data = NULL;                   \
+        _coll_req->super.req_status.MPI_ERROR = MPI_SUCCESS;            \
+        _coll_req->super.req_state            = OMPI_REQUEST_ACTIVE;    \
+        _coll_req->super.req_free             = mca_coll_ucc_req_free;  \
+        _coll_req->super.req_type             = OMPI_REQUEST_COLL;      \
+    } while(0)
+
+#define COLL_UCC_REQ_INIT(_coll_req, _req, _coll, _module) do{          \
+        if (_coll_req) {                                                \
+            _coll.mask   |= UCC_COLL_ARGS_FIELD_CB;                     \
+            _coll.cb.cb   = mca_coll_ucc_completion;                    \
+            _coll.cb.data = (void*)_coll_req;                           \
+        } else {                                                        \
+            _coll.mask  |= UCC_COLL_ARGS_FIELD_FLAGS;                   \
+            _coll.flags |= UCC_COLL_ARGS_HINT_OPTIMIZE_LATENCY;         \
+        }                                                               \
+        COLL_UCC_CHECK(ucc_collective_init(&_coll, _req,                \
+                                           _module->ucc_team));         \
+        if (_coll_req) {                                                \
+            _coll_req->ucc_req = *(_req);                               \
+        }                                                               \
+    } while(0)
+
+static inline ucc_status_t coll_ucc_req_wait(ucc_coll_req_h req)
+{
+    ucc_status_t status;
+    while (UCC_OK != (status = ucc_collective_test(req))) {
+        if (status < 0) {
+            UCC_ERROR("ucc_collective_test failed: %s",
+                      ucc_status_string(status));
+            ucc_collective_finalize(req);
+            return status;
+        }
+        ucc_context_progress(mca_coll_ucc_component.ucc_context);
+        opal_progress();
+    }
+    return ucc_collective_finalize(req);
+}
+
+int mca_coll_ucc_req_free(struct ompi_request_t **ompi_req);
+void mca_coll_ucc_completion(void *data, ucc_status_t status);
+
+#endif
diff --git a/ompi/mca/coll/ucc/coll_ucc_component.c b/ompi/mca/coll/ucc/coll_ucc_component.c
new file mode 100644
index 00000000000..b697ab787c2
--- /dev/null
+++ b/ompi/mca/coll/ucc/coll_ucc_component.c
@@ -0,0 +1,195 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2021 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2022 NVIDIA Corporation. All rights reserved.
+ * Copyright (c) 2024 NVIDIA CORPORATION. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include "coll_ucc.h"
+#include "coll_ucc_dtypes.h"
+#include "opal/util/argv.h"
+
+static int mca_coll_ucc_open(void);
+static int mca_coll_ucc_close(void);
+static int mca_coll_ucc_register(void);
+
+int mca_coll_ucc_output = -1;
+
+mca_coll_ucc_component_t mca_coll_ucc_component = {
+    /* First, the mca_component_t struct containing meta information
+       about the component  */
+    {
+        .collm_version = {
+            MCA_COLL_BASE_VERSION_3_0_0,
+
+            /* Component name and version */
+            .mca_component_name = "ucc",
+            MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION,
+                                  OMPI_RELEASE_VERSION),
+
+            /* Component open and close functions */
+            .mca_open_component            = mca_coll_ucc_open,
+            .mca_close_component           = mca_coll_ucc_close,
+            .mca_register_component_params = mca_coll_ucc_register,
+            .mca_query_component           = NULL,
+        },
+        .collm_data = {
+            /* The component is not checkpoint ready */
+            MCA_BASE_METADATA_PARAM_NONE
+        },
+
+        /* Initialization / querying functions */
+        .collm_init_query = mca_coll_ucc_init_query,
+        .collm_comm_query = mca_coll_ucc_comm_query,
+    },
+    10,                /* ucc_priority                */
+    0,                 /* ucc_verbose                 */
+    0,                 /* ucc_enable                  */
+    2,                 /* ucc_np                      */
+    "",                /* cls                         */
+    COLL_UCC_CTS_STR,  /* requested coll_types string */
+    UCC_VERSION_STRING /* ucc version                 */
+};
+
+static int mca_coll_ucc_register(void)
+{
+    mca_coll_ucc_component_t *cm = &mca_coll_ucc_component;
+    mca_base_component_t     *c  = &cm->super.collm_version;
+    mca_base_component_var_register(c, "priority", "Priority of the UCC coll component",
+                                    MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
+                                    OPAL_INFO_LVL_9,
+                                    MCA_BASE_VAR_SCOPE_ALL, &cm->ucc_priority);
+
+    mca_base_component_var_register(c, "verbose", "Verbose level of the UCC coll component",
+                                    MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
+                                    OPAL_INFO_LVL_9,
+                                    MCA_BASE_VAR_SCOPE_ALL, &cm->ucc_verbose);
+
+    mca_base_component_var_register(c, "enable", "[0|1] Enable/Disable the UCC coll component",
+                                    MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
+                                    OPAL_INFO_LVL_9,
+                                    MCA_BASE_VAR_SCOPE_ALL, &cm->ucc_enable);
+
+    mca_base_component_var_register(c, "np", "Minimal communicator size for the UCC coll component",
+                                    MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
+                                    OPAL_INFO_LVL_9,
+                                    MCA_BASE_VAR_SCOPE_ALL, &cm->ucc_np);
+
+    mca_base_component_var_register(c, MCA_COMPILETIME_VER,
+                                    "Version of the libucc library with which Open MPI was compiled",
+                                    MCA_BASE_VAR_TYPE_VERSION_STRING, NULL, 0, 0,
+                                    OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_READONLY,
+                                    &cm->compiletime_version);
+
+    mca_base_component_var_register(c, MCA_RUNTIME_VER,
+                                    "Version of the libucc library with which Open MPI is running",
+                                    MCA_BASE_VAR_TYPE_VERSION_STRING, NULL, 0, 0,
+                                    OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_READONLY,
+                                    &cm->runtime_version);
+
+    cm->cls = "";
+    mca_base_component_var_register(c, "cls",
+                                    "Comma separated list of UCC CLS to be used for team creation",
+                                    MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
+                                    OPAL_INFO_LVL_6, MCA_BASE_VAR_SCOPE_ALL, &cm->cls);
+
+    cm->cts = COLL_UCC_CTS_STR;
+    mca_base_component_var_register(c, "cts",
+                                    "Comma separated list of UCC coll types to be enabled",
+                                    MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
+                                    OPAL_INFO_LVL_6, MCA_BASE_VAR_SCOPE_ALL, &cm->cts);
+    return OMPI_SUCCESS;
+}
+
+static ucc_coll_type_t mca_coll_ucc_str_to_type(const char *str)
+{
+    if (0 == strcasecmp(str, "barrier")) {
+        return UCC_COLL_TYPE_BARRIER;
+    } else if (0 == strcasecmp(str, "bcast")) {
+        return UCC_COLL_TYPE_BCAST;
+    } else if (0 == strcasecmp(str, "allreduce")) {
+        return UCC_COLL_TYPE_ALLREDUCE;
+    } else if (0 == strcasecmp(str, "alltoall")) {
+        return UCC_COLL_TYPE_ALLTOALL;
+    } else if (0 == strcasecmp(str, "alltoallv")) {
+        return UCC_COLL_TYPE_ALLTOALLV;
+    } else if (0 == strcasecmp(str, "allgather")) {
+        return UCC_COLL_TYPE_ALLGATHER;
+    } else if (0 == strcasecmp(str, "allgatherv")) {
+        return UCC_COLL_TYPE_ALLGATHERV;
+    } else if (0 == strcasecmp(str, "reduce")) {
+        return UCC_COLL_TYPE_REDUCE;
+    } else if (0 == strcasecmp(str, "gather")) {
+        return UCC_COLL_TYPE_GATHER;
+    } else if (0 == strcasecmp(str, "gatherv")) {
+        return UCC_COLL_TYPE_GATHERV;
+    } else if (0 == strcasecmp(str, "reduce_scatter_block")) {
+        return UCC_COLL_TYPE_REDUCE_SCATTER;
+    } else if (0 == strcasecmp(str, "reduce_scatter")) {
+        return UCC_COLL_TYPE_REDUCE_SCATTERV;
+    } else if (0 == strcasecmp(str, "scatterv")) {
+        return UCC_COLL_TYPE_SCATTERV;
+    } else if (0 == strcasecmp(str, "scatter")) {
+        return UCC_COLL_TYPE_SCATTER;
+    }
+    UCC_ERROR("incorrect value for cts: %s, allowed: %s",
+              str, COLL_UCC_CTS_STR);
+    return UCC_COLL_TYPE_LAST;
+}
+
+static void mca_coll_ucc_init_default_cts(void)
+{
+    mca_coll_ucc_component_t *cm = &mca_coll_ucc_component;
+    bool                      disable;
+    char**                    cts;
+    int                       n_cts, i;
+    char*                     str;
+    ucc_coll_type_t           *ct, c;
+
+    disable              = (cm->cts[0] == '^') ? true : false;
+    cts                  = opal_argv_split(disable ? (cm->cts + 1) : cm->cts, ',');
+    n_cts                = opal_argv_count(cts);
+    cm->cts_requested    = disable ? COLL_UCC_CTS : 0;
+    cm->nb_cts_requested = disable ? COLL_UCC_CTS : 0;
+    for (i = 0; i < n_cts; i++) {
+        if (('i' == cts[i][0]) || ('I' == cts[i][0])) {
+            /* non blocking collective setting */
+            str = cts[i] + 1;
+            ct  = &cm->nb_cts_requested;
+        } else {
+            str = cts[i];
+            ct  = &cm->cts_requested;
+        }
+        c = mca_coll_ucc_str_to_type(str);
+        if (UCC_COLL_TYPE_LAST == c) {
+            *ct = COLL_UCC_CTS;
+            break;
+        }
+        if (disable) {
+            (*ct) &= ~c;
+        } else {
+            (*ct) |= c;
+        }
+    }
+    opal_argv_free(cts);
+}
+
+static int mca_coll_ucc_open(void)
+{
+    mca_coll_ucc_component_t *cm = &mca_coll_ucc_component;
+    mca_coll_ucc_output          = opal_output_open(NULL);
+    cm->libucc_initialized       = false;
+    opal_output_set_verbosity(mca_coll_ucc_output, cm->ucc_verbose);
+    mca_coll_ucc_init_default_cts();
+    return OMPI_SUCCESS;
+}
+
+static int mca_coll_ucc_close(void)
+{
+    return OMPI_SUCCESS;
+}
diff --git a/ompi/mca/coll/ucc/coll_ucc_debug.h b/ompi/mca/coll/ucc/coll_ucc_debug.h
new file mode 100644
index 00000000000..f50ae30f12e
--- /dev/null
+++ b/ompi/mca/coll/ucc/coll_ucc_debug.h
@@ -0,0 +1,30 @@
+/**
+  Copyright (c) 2021 Mellanox Technologies. All rights reserved.
+  $COPYRIGHT$
+
+  Additional copyrights may follow
+
+  $HEADER$
+ */
+
+#ifndef COLL_UCC_DEBUG_H
+#define COLL_UCC_DEBUG_H
+#include "ompi_config.h"
+#pragma GCC system_header
+
+#ifdef __BASE_FILE__
+#define __UCC_FILE__ __BASE_FILE__
+#else
+#define __UCC_FILE__ __FILE__
+#endif
+
+#define UCC_VERBOSE(level, format, ...) \
+    opal_output_verbose(level, mca_coll_ucc_output, "%s:%d - %s() " format, \
+                        __UCC_FILE__, __LINE__, __FUNCTION__, ## __VA_ARGS__)
+
+#define UCC_ERROR(format, ... ) \
+    opal_output_verbose(0, mca_coll_ucc_output, "Error: %s:%d - %s() " format, \
+                        __UCC_FILE__, __LINE__, __FUNCTION__, ## __VA_ARGS__)
+
+extern int mca_coll_ucc_output;
+#endif
diff --git a/ompi/mca/coll/ucc/coll_ucc_dtypes.h b/ompi/mca/coll/ucc/coll_ucc_dtypes.h
new file mode 100644
index 00000000000..ad16a00ad26
--- /dev/null
+++ b/ompi/mca/coll/ucc/coll_ucc_dtypes.h
@@ -0,0 +1,114 @@
+/*
+ * Copyright (c) 2021 Mellanox Technologies. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#ifndef COLL_UCC_DTYPES_H
+#define COLL_UCC_DTYPES_H
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/datatype/ompi_datatype_internal.h"
+#include "ompi/mca/op/op.h"
+#include <ucc/api/ucc.h>
+
+#define COLL_UCC_DT_UNSUPPORTED ((ucc_datatype_t)-1)
+#define COLL_UCC_OP_UNSUPPORTED ((ucc_reduction_op_t)-1)
+
+static ucc_datatype_t ompi_datatype_2_ucc_dt[OPAL_DATATYPE_MAX_PREDEFINED] = {
+    [OPAL_DATATYPE_LOOP]                      = COLL_UCC_DT_UNSUPPORTED,
+    [OPAL_DATATYPE_END_LOOP]                  = COLL_UCC_DT_UNSUPPORTED,
+    [OPAL_DATATYPE_LB]                        = COLL_UCC_DT_UNSUPPORTED,
+    [OPAL_DATATYPE_UB]                        = COLL_UCC_DT_UNSUPPORTED,
+    [OPAL_DATATYPE_INT1]                      = UCC_DT_INT8,
+    [OPAL_DATATYPE_INT2]                      = UCC_DT_INT16,
+    [OPAL_DATATYPE_INT4]                      = UCC_DT_INT32,
+    [OPAL_DATATYPE_INT8]                      = UCC_DT_INT64,
+    [OPAL_DATATYPE_INT16]                     = UCC_DT_INT128,
+    [OPAL_DATATYPE_UINT1]                     = UCC_DT_UINT8,
+    [OPAL_DATATYPE_UINT2]                     = UCC_DT_UINT16,
+    [OPAL_DATATYPE_UINT4]                     = UCC_DT_UINT32,
+    [OPAL_DATATYPE_UINT8]                     = UCC_DT_UINT64,
+    [OPAL_DATATYPE_UINT16]                    = UCC_DT_UINT128,
+    [OPAL_DATATYPE_FLOAT2]                    = UCC_DT_FLOAT16,
+    [OPAL_DATATYPE_FLOAT4]                    = UCC_DT_FLOAT32,
+    [OPAL_DATATYPE_FLOAT8]                    = UCC_DT_FLOAT64,
+    [OPAL_DATATYPE_FLOAT12]                   = COLL_UCC_DT_UNSUPPORTED,
+    [OPAL_DATATYPE_BOOL]                      = COLL_UCC_DT_UNSUPPORTED,
+    [OPAL_DATATYPE_WCHAR]                     = COLL_UCC_DT_UNSUPPORTED,
+    [OPAL_DATATYPE_SHORT_FLOAT_COMPLEX]       = COLL_UCC_DT_UNSUPPORTED,
+#if SIZEOF_LONG == 4
+    [OPAL_DATATYPE_LONG]                      = UCC_DT_INT32,
+    [OPAL_DATATYPE_UNSIGNED_LONG]             = UCC_DT_UINT32,
+#elif SIZEOF_LONG == 8
+    [OPAL_DATATYPE_LONG]                      = UCC_DT_INT64,
+    [OPAL_DATATYPE_UNSIGNED_LONG]             = UCC_DT_UINT64,
+#endif
+#if UCC_HAVE_COMPLEX_AND_FLOAT128_DT
+    [OPAL_DATATYPE_FLOAT16]                   = UCC_DT_FLOAT128,
+    #if SIZEOF_FLOAT__COMPLEX == 8
+        [OPAL_DATATYPE_FLOAT_COMPLEX]         = UCC_DT_FLOAT32_COMPLEX,
+    #else
+        [OPAL_DATATYPE_FLOAT_COMPLEX]         = COLL_UCC_DT_UNSUPPORTED,
+    #endif
+    #if SIZEOF_DOUBLE__COMPLEX == 16
+        [OPAL_DATATYPE_DOUBLE_COMPLEX]        = UCC_DT_FLOAT64_COMPLEX,
+    #else
+        [OPAL_DATATYPE_DOUBLE_COMPLEX]        = COLL_UCC_DT_UNSUPPORTED,
+    #endif
+    #if SIZEOF_LONG_DOUBLE__COMPLEX == 32
+        [OPAL_DATATYPE_LONG_DOUBLE_COMPLEX]   = UCC_DT_FLOAT128_COMPLEX,
+    #else
+        [OPAL_DATATYPE_LONG_DOUBLE_COMPLEX]   = COLL_UCC_DT_UNSUPPORTED,
+    #endif
+#else
+    [OPAL_DATATYPE_FLOAT16]                   = COLL_UCC_DT_UNSUPPORTED,
+    [OPAL_DATATYPE_FLOAT_COMPLEX]             = COLL_UCC_DT_UNSUPPORTED,
+    [OPAL_DATATYPE_DOUBLE_COMPLEX]            = COLL_UCC_DT_UNSUPPORTED,
+    [OPAL_DATATYPE_LONG_DOUBLE_COMPLEX]       = COLL_UCC_DT_UNSUPPORTED,
+#endif
+    [OPAL_DATATYPE_UNAVAILABLE]               = COLL_UCC_DT_UNSUPPORTED
+};
+
+static inline ucc_datatype_t ompi_dtype_to_ucc_dtype(ompi_datatype_t *dtype)
+{
+    int ompi_type_id = dtype->id;
+    int opal_type_id = dtype->super.id;
+
+    if (ompi_type_id < OMPI_DATATYPE_MPI_MAX_PREDEFINED &&
+        dtype->super.flags & OMPI_DATATYPE_FLAG_PREDEFINED) {
+        if (opal_type_id > 0 && opal_type_id < OPAL_DATATYPE_MAX_PREDEFINED) {
+            return  ompi_datatype_2_ucc_dt[opal_type_id];
+        }
+    }
+    return COLL_UCC_DT_UNSUPPORTED;
+}
+
+static ucc_reduction_op_t ompi_op_to_ucc_op_map[OMPI_OP_BASE_FORTRAN_OP_MAX + 1] = {
+   COLL_UCC_OP_UNSUPPORTED,     /* OMPI_OP_BASE_FORTRAN_NULL = 0 */
+   UCC_OP_MAX,                  /* OMPI_OP_BASE_FORTRAN_MAX */
+   UCC_OP_MIN,                  /* OMPI_OP_BASE_FORTRAN_MIN */
+   UCC_OP_SUM,                  /* OMPI_OP_BASE_FORTRAN_SUM */
+   UCC_OP_PROD,                 /* OMPI_OP_BASE_FORTRAN_PROD */
+   UCC_OP_LAND,                 /* OMPI_OP_BASE_FORTRAN_LAND */
+   UCC_OP_BAND,                 /* OMPI_OP_BASE_FORTRAN_BAND */
+   UCC_OP_LOR,                  /* OMPI_OP_BASE_FORTRAN_LOR */
+   UCC_OP_BOR,                  /* OMPI_OP_BASE_FORTRAN_BOR */
+   UCC_OP_LXOR,                 /* OMPI_OP_BASE_FORTRAN_LXOR */
+   UCC_OP_BXOR,                 /* OMPI_OP_BASE_FORTRAN_BXOR */
+   COLL_UCC_OP_UNSUPPORTED,     /* OMPI_OP_BASE_FORTRAN_MAXLOC */
+   COLL_UCC_OP_UNSUPPORTED,     /* OMPI_OP_BASE_FORTRAN_MINLOC */
+   COLL_UCC_OP_UNSUPPORTED,     /* OMPI_OP_BASE_FORTRAN_REPLACE */
+   COLL_UCC_OP_UNSUPPORTED,     /* OMPI_OP_BASE_FORTRAN_NO_OP */
+   COLL_UCC_OP_UNSUPPORTED      /* OMPI_OP_BASE_FORTRAN_OP_MAX */
+};
+
+static inline ucc_reduction_op_t ompi_op_to_ucc_op(ompi_op_t *op) {
+    if (op->o_f_to_c_index > OMPI_OP_BASE_FORTRAN_OP_MAX) {
+        return COLL_UCC_OP_UNSUPPORTED;
+    }
+    return ompi_op_to_ucc_op_map[op->o_f_to_c_index];
+}
+
+#endif /* COLL_UCC_DTYPES_H */
diff --git a/ompi/mca/coll/ucc/coll_ucc_gather.c b/ompi/mca/coll/ucc/coll_ucc_gather.c
new file mode 100644
index 00000000000..ba91b40b189
--- /dev/null
+++ b/ompi/mca/coll/ucc/coll_ucc_gather.c
@@ -0,0 +1,133 @@
+
+/**
+ * Copyright (c) 2021 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2022 NVIDIA Corporation. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ */
+
+#include "coll_ucc_common.h"
+
+static inline
+ucc_status_t mca_coll_ucc_gather_init(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                                      void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
+                                      int root, mca_coll_ucc_module_t *ucc_module,
+                                      ucc_coll_req_h *req,
+                                      mca_coll_ucc_req_t *coll_req)
+{
+    ucc_datatype_t ucc_sdt = UCC_DT_INT8, ucc_rdt = UCC_DT_INT8;
+    bool is_inplace = (MPI_IN_PLACE == sbuf);
+    int comm_rank = ompi_comm_rank(ucc_module->comm);
+    int comm_size = ompi_comm_size(ucc_module->comm);
+
+    if (comm_rank == root) {
+        if (!(is_inplace || ompi_datatype_is_contiguous_memory_layout(sdtype, scount)) ||
+            !ompi_datatype_is_contiguous_memory_layout(rdtype, rcount * comm_size)) {
+            goto fallback;
+        }
+
+        ucc_rdt = ompi_dtype_to_ucc_dtype(rdtype);
+        if (!is_inplace) {
+            ucc_sdt = ompi_dtype_to_ucc_dtype(sdtype);
+        }
+
+        if ((COLL_UCC_DT_UNSUPPORTED == ucc_sdt) ||
+            (COLL_UCC_DT_UNSUPPORTED == ucc_rdt)) {
+            UCC_VERBOSE(5, "ompi_datatype is not supported: dtype = %s",
+                        (COLL_UCC_DT_UNSUPPORTED == ucc_sdt) ?
+                        sdtype->super.name : rdtype->super.name);
+            goto fallback;
+        }
+    } else {
+        if (!ompi_datatype_is_contiguous_memory_layout(sdtype, scount)) {
+            goto fallback;
+        }
+
+        ucc_sdt = ompi_dtype_to_ucc_dtype(sdtype);
+        if (COLL_UCC_DT_UNSUPPORTED == ucc_sdt) {
+            UCC_VERBOSE(5, "ompi_datatype is not supported: dtype = %s",
+                        sdtype->super.name);
+            goto fallback;
+        }
+    }
+
+    ucc_coll_args_t coll = {
+        .mask      = 0,
+        .flags     = 0,
+        .coll_type = UCC_COLL_TYPE_GATHER,
+        .root      = root,
+        .src.info = {
+            .buffer   = (void*)sbuf,
+            .count    = scount,
+            .datatype = ucc_sdt,
+            .mem_type = UCC_MEMORY_TYPE_UNKNOWN
+        },
+        .dst.info = {
+            .buffer   = (void*)rbuf,
+            .count    = rcount * comm_size,
+            .datatype = ucc_rdt,
+            .mem_type = UCC_MEMORY_TYPE_UNKNOWN
+        },
+    };
+
+    if (is_inplace) {
+        coll.mask |= UCC_COLL_ARGS_FIELD_FLAGS;
+        coll.flags = UCC_COLL_ARGS_FLAG_IN_PLACE;
+    }
+    COLL_UCC_REQ_INIT(coll_req, req, coll, ucc_module);
+    return UCC_OK;
+fallback:
+    return UCC_ERR_NOT_SUPPORTED;
+}
+
+int mca_coll_ucc_gather(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                        void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
+                        int root, struct ompi_communicator_t *comm,
+                        mca_coll_base_module_t *module)
+{
+    mca_coll_ucc_module_t *ucc_module = (mca_coll_ucc_module_t*)module;
+    ucc_coll_req_h         req;
+
+    UCC_VERBOSE(3, "running ucc gather");
+    COLL_UCC_CHECK(mca_coll_ucc_gather_init(sbuf, scount, sdtype, rbuf, rcount,
+                                            rdtype, root, ucc_module,
+                                            &req, NULL));
+    COLL_UCC_POST_AND_CHECK(req);
+    COLL_UCC_CHECK(coll_ucc_req_wait(req));
+    return OMPI_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback gather");
+    return ucc_module->previous_gather(sbuf, scount, sdtype, rbuf, rcount,
+                                       rdtype, root, comm,
+                                       ucc_module->previous_gather_module);
+}
+
+int mca_coll_ucc_igather(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                         void *rbuf, size_t rcount, struct ompi_datatype_t *rdtype,
+                         int root, struct ompi_communicator_t *comm,
+                         ompi_request_t** request,
+                         mca_coll_base_module_t *module)
+{
+    mca_coll_ucc_module_t *ucc_module = (mca_coll_ucc_module_t*)module;
+    ucc_coll_req_h         req;
+    mca_coll_ucc_req_t    *coll_req = NULL;
+
+    UCC_VERBOSE(3, "running ucc igather");
+    COLL_UCC_GET_REQ(coll_req);
+    COLL_UCC_CHECK(mca_coll_ucc_gather_init(sbuf, scount, sdtype, rbuf, rcount,
+                                            rdtype, root, ucc_module,
+                                            &req, coll_req));
+    COLL_UCC_POST_AND_CHECK(req);
+    *request = &coll_req->super;
+    return OMPI_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback igather");
+    if (coll_req) {
+        mca_coll_ucc_req_free((ompi_request_t **)&coll_req);
+    }
+    return ucc_module->previous_igather(sbuf, scount, sdtype, rbuf, rcount,
+                                        rdtype, root, comm, request,
+                                        ucc_module->previous_igather_module);
+}
diff --git a/ompi/mca/coll/ucc/coll_ucc_gatherv.c b/ompi/mca/coll/ucc/coll_ucc_gatherv.c
new file mode 100644
index 00000000000..5a1da52356c
--- /dev/null
+++ b/ompi/mca/coll/ucc/coll_ucc_gatherv.c
@@ -0,0 +1,126 @@
+
+/**
+ * Copyright (c) 2021 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2022 NVIDIA Corporation. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ */
+
+#include "coll_ucc_common.h"
+
+static inline ucc_status_t mca_coll_ucc_gatherv_init(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                                                     void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
+                                                     struct ompi_datatype_t *rdtype, int root,
+                                                     mca_coll_ucc_module_t *ucc_module,
+                                                     ucc_coll_req_h *req,
+                                                     mca_coll_ucc_req_t *coll_req)
+{
+    ucc_datatype_t ucc_sdt = UCC_DT_INT8, ucc_rdt = UCC_DT_INT8;
+    bool is_inplace = (MPI_IN_PLACE == sbuf);
+    int comm_rank = ompi_comm_rank(ucc_module->comm);
+    uint64_t flags = 0;
+
+    if (comm_rank == root) {
+        ucc_rdt = ompi_dtype_to_ucc_dtype(rdtype);
+        if (!is_inplace) {
+            ucc_sdt = ompi_dtype_to_ucc_dtype(sdtype);
+        }
+        if ((COLL_UCC_DT_UNSUPPORTED == ucc_sdt) ||
+            (COLL_UCC_DT_UNSUPPORTED == ucc_rdt)) {
+            UCC_VERBOSE(5, "ompi_datatype is not supported: dtype = %s",
+                        (COLL_UCC_DT_UNSUPPORTED == ucc_sdt) ?
+                        sdtype->super.name : rdtype->super.name);
+            goto fallback;
+        }
+    } else {
+        ucc_sdt = ompi_dtype_to_ucc_dtype(sdtype);
+        if (COLL_UCC_DT_UNSUPPORTED == ucc_sdt) {
+            UCC_VERBOSE(5, "ompi_datatype is not supported: dtype = %s",
+                        sdtype->super.name);
+            goto fallback;
+        }
+    }
+
+    flags = (ompi_count_array_is_64bit(rcounts) ? UCC_COLL_ARGS_FLAG_COUNT_64BIT : 0) |
+            (ompi_disp_array_is_64bit(disps) ? UCC_COLL_ARGS_FLAG_DISPLACEMENTS_64BIT : 0) |
+            (is_inplace ? UCC_COLL_ARGS_FLAG_IN_PLACE : 0);
+
+    ucc_coll_args_t coll = {
+        .mask      = flags ? UCC_COLL_ARGS_FIELD_FLAGS : 0,
+        .flags     = flags,
+        .coll_type = UCC_COLL_TYPE_GATHERV,
+        .root      = root,
+        .src.info = {
+            .buffer   = (void*)sbuf,
+            .count    = scount,
+            .datatype = ucc_sdt,
+            .mem_type = UCC_MEMORY_TYPE_UNKNOWN
+        },
+        .dst.info_v = {
+            .buffer        = (void*)rbuf,
+            .counts        = (ucc_count_t*)ompi_count_array_ptr(rcounts),
+            .displacements = (ucc_aint_t*)ompi_disp_array_ptr(disps),
+            .datatype      = ucc_rdt,
+            .mem_type      = UCC_MEMORY_TYPE_UNKNOWN
+        },
+    };
+
+    COLL_UCC_REQ_INIT(coll_req, req, coll, ucc_module);
+    return UCC_OK;
+fallback:
+    return UCC_ERR_NOT_SUPPORTED;
+}
+
+int mca_coll_ucc_gatherv(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                         void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
+                         struct ompi_datatype_t *rdtype, int root,
+                         struct ompi_communicator_t *comm,
+                         mca_coll_base_module_t *module)
+{
+    mca_coll_ucc_module_t *ucc_module = (mca_coll_ucc_module_t*)module;
+    ucc_coll_req_h         req;
+
+    UCC_VERBOSE(3, "running ucc gatherv");
+    COLL_UCC_CHECK(mca_coll_ucc_gatherv_init(sbuf, scount, sdtype, rbuf, rcounts,
+                                             disps, rdtype, root, ucc_module,
+                                             &req, NULL));
+    COLL_UCC_POST_AND_CHECK(req);
+    COLL_UCC_CHECK(coll_ucc_req_wait(req));
+    return OMPI_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback gatherv");
+    return ucc_module->previous_gatherv(sbuf, scount, sdtype, rbuf, rcounts,
+                                        disps, rdtype, root, comm,
+                                        ucc_module->previous_gatherv_module);
+}
+
+int mca_coll_ucc_igatherv(const void *sbuf, size_t scount, struct ompi_datatype_t *sdtype,
+                          void *rbuf, ompi_count_array_t rcounts, ompi_disp_array_t disps,
+                          struct ompi_datatype_t *rdtype, int root,
+                          struct ompi_communicator_t *comm,
+                          ompi_request_t** request,
+                          mca_coll_base_module_t *module)
+{
+    mca_coll_ucc_module_t *ucc_module = (mca_coll_ucc_module_t*)module;
+    ucc_coll_req_h         req;
+    mca_coll_ucc_req_t    *coll_req = NULL;
+
+    UCC_VERBOSE(3, "running ucc igatherv");
+    COLL_UCC_GET_REQ(coll_req);
+    COLL_UCC_CHECK(mca_coll_ucc_gatherv_init(sbuf, scount, sdtype, rbuf, rcounts,
+                                             disps, rdtype, root, ucc_module,
+                                             &req, coll_req));
+    COLL_UCC_POST_AND_CHECK(req);
+    *request = &coll_req->super;
+    return OMPI_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback igatherv");
+    if (coll_req) {
+        mca_coll_ucc_req_free((ompi_request_t **)&coll_req);
+    }
+    return ucc_module->previous_igatherv(sbuf, scount, sdtype, rbuf, rcounts,
+                                         disps, rdtype, root, comm, request,
+                                         ucc_module->previous_igatherv_module);
+}
diff --git a/ompi/mca/coll/ucc/coll_ucc_module.c b/ompi/mca/coll/ucc/coll_ucc_module.c
new file mode 100644
index 00000000000..3916650b949
--- /dev/null
+++ b/ompi/mca/coll/ucc/coll_ucc_module.c
@@ -0,0 +1,592 @@
+/**
+ * Copyright (c) 2021      Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * Copyright (c) 2022-2024 NVIDIA Corporation. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "coll_ucc.h"
+#include "coll_ucc_dtypes.h"
+#include "ompi/mca/coll/base/coll_tags.h"
+#include "ompi/mca/pml/pml.h"
+
+static int ucc_comm_attr_keyval;
+/*
+ * Initial query function that is invoked during MPI_INIT, allowing
+ * this module to indicate what level of thread support it provides.
+ */
+int mca_coll_ucc_init_query(bool enable_progress_threads, bool enable_mpi_threads)
+{
+    return OMPI_SUCCESS;
+}
+
+static void mca_coll_ucc_module_clear(mca_coll_ucc_module_t *ucc_module)
+{
+    ucc_module->ucc_team                              = NULL;
+    ucc_module->previous_allreduce                    = NULL;
+    ucc_module->previous_allreduce_module             = NULL;
+    ucc_module->previous_iallreduce                   = NULL;
+    ucc_module->previous_iallreduce_module            = NULL;
+    ucc_module->previous_barrier                      = NULL;
+    ucc_module->previous_barrier_module               = NULL;
+    ucc_module->previous_ibarrier                     = NULL;
+    ucc_module->previous_ibarrier_module              = NULL;
+    ucc_module->previous_bcast                        = NULL;
+    ucc_module->previous_bcast_module                 = NULL;
+    ucc_module->previous_ibcast                       = NULL;
+    ucc_module->previous_ibcast_module                = NULL;
+    ucc_module->previous_alltoall                     = NULL;
+    ucc_module->previous_alltoall_module              = NULL;
+    ucc_module->previous_ialltoall                    = NULL;
+    ucc_module->previous_ialltoall_module             = NULL;
+    ucc_module->previous_alltoallv                    = NULL;
+    ucc_module->previous_alltoallv_module             = NULL;
+    ucc_module->previous_ialltoallv                   = NULL;
+    ucc_module->previous_ialltoallv_module            = NULL;
+    ucc_module->previous_allgather                    = NULL;
+    ucc_module->previous_allgather_module             = NULL;
+    ucc_module->previous_iallgather                   = NULL;
+    ucc_module->previous_iallgather_module            = NULL;
+    ucc_module->previous_allgatherv                   = NULL;
+    ucc_module->previous_allgatherv_module            = NULL;
+    ucc_module->previous_iallgatherv                  = NULL;
+    ucc_module->previous_iallgatherv_module           = NULL;
+    ucc_module->previous_reduce                       = NULL;
+    ucc_module->previous_reduce_module                = NULL;
+    ucc_module->previous_ireduce                      = NULL;
+    ucc_module->previous_ireduce_module               = NULL;
+    ucc_module->previous_gather                       = NULL;
+    ucc_module->previous_gather_module                = NULL;
+    ucc_module->previous_igather                      = NULL;
+    ucc_module->previous_igather_module               = NULL;
+    ucc_module->previous_gatherv                      = NULL;
+    ucc_module->previous_gatherv_module               = NULL;
+    ucc_module->previous_igatherv                     = NULL;
+    ucc_module->previous_igatherv_module              = NULL;
+    ucc_module->previous_reduce_scatter_block         = NULL;
+    ucc_module->previous_reduce_scatter_block_module  = NULL;
+    ucc_module->previous_ireduce_scatter_block        = NULL;
+    ucc_module->previous_ireduce_scatter_block_module = NULL;
+    ucc_module->previous_reduce_scatter               = NULL;
+    ucc_module->previous_reduce_scatter_module        = NULL;
+    ucc_module->previous_ireduce_scatter              = NULL;
+    ucc_module->previous_ireduce_scatter_module       = NULL;
+    ucc_module->previous_scatterv                     = NULL;
+    ucc_module->previous_scatterv_module              = NULL;
+    ucc_module->previous_iscatterv                    = NULL;
+    ucc_module->previous_iscatterv_module             = NULL;
+    ucc_module->previous_scatter                      = NULL;
+    ucc_module->previous_scatter_module               = NULL;
+    ucc_module->previous_iscatter                     = NULL;
+    ucc_module->previous_iscatter_module              = NULL;
+}
+
+static void mca_coll_ucc_module_construct(mca_coll_ucc_module_t *ucc_module)
+{
+    mca_coll_ucc_module_clear(ucc_module);
+}
+
+static int mca_coll_ucc_progress(void)
+{
+    ucc_context_progress(mca_coll_ucc_component.ucc_context);
+    return OPAL_SUCCESS;
+}
+
+static void mca_coll_ucc_module_destruct(mca_coll_ucc_module_t *ucc_module)
+{
+    if (ucc_module->comm == &ompi_mpi_comm_world.comm){
+        if (OMPI_SUCCESS != ompi_attr_free_keyval(COMM_ATTR, &ucc_comm_attr_keyval, 0)) {
+            UCC_ERROR("ucc ompi_attr_free_keyval failed");
+        }
+    }
+    mca_coll_ucc_module_clear(ucc_module);
+}
+
+/*
+** Communicator free callback
+*/
+static int ucc_comm_attr_del_fn(MPI_Comm comm, int keyval, void *attr_val, void *extra)
+{
+    mca_coll_ucc_module_t *ucc_module = (mca_coll_ucc_module_t*) attr_val;
+    ucc_status_t status;
+    while(UCC_INPROGRESS == (status = ucc_team_destroy(ucc_module->ucc_team))) {}
+    if (ucc_module->comm == &ompi_mpi_comm_world.comm) {
+        if (mca_coll_ucc_component.libucc_initialized) {
+            UCC_VERBOSE(1,"finalizing ucc library");
+            opal_progress_unregister(mca_coll_ucc_progress);
+            ucc_context_destroy(mca_coll_ucc_component.ucc_context);
+            ucc_finalize(mca_coll_ucc_component.ucc_lib);
+        }
+    }
+    if (UCC_OK != status) {
+        UCC_ERROR("UCC team destroy failed");
+        return OMPI_ERROR;
+    }
+    return OMPI_SUCCESS;
+}
+
+typedef struct oob_allgather_req{
+    void           *sbuf;
+    void           *rbuf;
+    void           *oob_coll_ctx;
+    size_t          msglen;
+    int             iter;
+    ompi_request_t *reqs[2];
+} oob_allgather_req_t;
+
+static ucc_status_t oob_allgather_test(void *req)
+{
+    oob_allgather_req_t *oob_req = (oob_allgather_req_t*)req;
+    ompi_communicator_t *comm    = (ompi_communicator_t *)oob_req->oob_coll_ctx;
+    char                *tmpsend = NULL;
+    char                *tmprecv = NULL;
+    size_t               msglen  = oob_req->msglen;
+    int                  probe_count = 5;
+    int rank, size, sendto, recvfrom, recvdatafrom,
+        senddatafrom, completed, probe;
+
+    size = ompi_comm_size(comm);
+    rank = ompi_comm_rank(comm);
+    if (oob_req->iter == 0) {
+        tmprecv = (char*) oob_req->rbuf + (ptrdiff_t)rank * (ptrdiff_t)msglen;
+        memcpy(tmprecv, oob_req->sbuf, msglen);
+    }
+    sendto   = (rank + 1) % size;
+    recvfrom = (rank - 1 + size) % size;
+    for (; oob_req->iter < size - 1; oob_req->iter++) {
+        if (oob_req->iter > 0) {
+            probe = 0;
+            do {
+                ompi_request_test_all(2, oob_req->reqs, &completed, MPI_STATUS_IGNORE);
+                probe++;
+            } while (!completed && probe < probe_count);
+            if (!completed) {
+                return UCC_INPROGRESS;
+            }
+        }
+        recvdatafrom = (rank - oob_req->iter - 1 + size) % size;
+        senddatafrom = (rank - oob_req->iter + size) % size;
+        tmprecv = (char*)oob_req->rbuf + (ptrdiff_t)recvdatafrom * (ptrdiff_t)msglen;
+        tmpsend = (char*)oob_req->rbuf + (ptrdiff_t)senddatafrom * (ptrdiff_t)msglen;
+        MCA_PML_CALL(isend(tmpsend, msglen, MPI_BYTE, sendto, MCA_COLL_BASE_TAG_UCC,
+                           MCA_PML_BASE_SEND_STANDARD, comm, &oob_req->reqs[0]));
+        MCA_PML_CALL(irecv(tmprecv, msglen, MPI_BYTE, recvfrom,
+                           MCA_COLL_BASE_TAG_UCC, comm, &oob_req->reqs[1]));
+    }
+    probe = 0;
+    do {
+        ompi_request_test_all(2, oob_req->reqs, &completed, MPI_STATUS_IGNORE);
+        probe++;
+    } while (!completed && probe < probe_count);
+    if (!completed) {
+        return UCC_INPROGRESS;
+    }
+    return UCC_OK;
+}
+
+static ucc_status_t oob_allgather_free(void *req)
+{
+    free(req);
+    return UCC_OK;
+}
+
+static ucc_status_t oob_allgather(void *sbuf, void *rbuf, size_t msglen,
+                                  void *oob_coll_ctx, void **req)
+{
+    oob_allgather_req_t *oob_req = malloc(sizeof(*oob_req));
+    oob_req->sbuf                = sbuf;
+    oob_req->rbuf                = rbuf;
+    oob_req->msglen              = msglen;
+    oob_req->oob_coll_ctx        = oob_coll_ctx;
+    oob_req->iter                = 0;
+    *req                         = oob_req;
+    return UCC_OK;
+}
+
+
+static int mca_coll_ucc_init_ctx() {
+    mca_coll_ucc_component_t     *cm = &mca_coll_ucc_component;
+    char                          str_buf[256];
+    ompi_attribute_fn_ptr_union_t del_fn;
+    ompi_attribute_fn_ptr_union_t copy_fn;
+    ucc_lib_config_h              lib_config;
+    ucc_context_config_h          ctx_config;
+    ucc_thread_mode_t             tm_requested;
+    ucc_lib_params_t              lib_params;
+    ucc_context_params_t          ctx_params;
+
+    tm_requested           = ompi_mpi_thread_multiple ? UCC_THREAD_MULTIPLE :
+                                                        UCC_THREAD_SINGLE;
+    lib_params.mask        = UCC_LIB_PARAM_FIELD_THREAD_MODE;
+    lib_params.thread_mode = tm_requested;
+
+    if (UCC_OK != ucc_lib_config_read("OMPI", NULL, &lib_config)) {
+        UCC_ERROR("UCC lib config read failed");
+        return OMPI_ERROR;
+    }
+    if (strlen(cm->cls) > 0) {
+        if (UCC_OK != ucc_lib_config_modify(lib_config, "CLS", cm->cls)) {
+            ucc_lib_config_release(lib_config);
+            UCC_ERROR("failed to modify UCC lib config to set CLS");
+            return OMPI_ERROR;
+        }
+    }
+
+    if (UCC_OK != ucc_init(&lib_params, lib_config, &cm->ucc_lib)) {
+        UCC_ERROR("UCC lib init failed");
+        ucc_lib_config_release(lib_config);
+        cm->ucc_enable = 0;
+        return OMPI_ERROR;
+    }
+    ucc_lib_config_release(lib_config);
+
+    cm->ucc_lib_attr.mask = UCC_LIB_ATTR_FIELD_THREAD_MODE |
+                            UCC_LIB_ATTR_FIELD_COLL_TYPES;
+    if (UCC_OK != ucc_lib_get_attr(cm->ucc_lib, &cm->ucc_lib_attr)) {
+        UCC_ERROR("UCC get lib attr failed");
+        goto cleanup_lib;
+    }
+
+    if (cm->ucc_lib_attr.thread_mode < tm_requested) {
+        UCC_ERROR("UCC library doesn't support MPI_THREAD_MULTIPLE");
+        goto cleanup_lib;
+    }
+    ctx_params.mask             = UCC_CONTEXT_PARAM_FIELD_OOB;
+    ctx_params.oob.allgather    = oob_allgather;
+    ctx_params.oob.req_test     = oob_allgather_test;
+    ctx_params.oob.req_free     = oob_allgather_free;
+    ctx_params.oob.coll_info    = (void*)MPI_COMM_WORLD;
+    ctx_params.oob.n_oob_eps    = ompi_comm_size(&ompi_mpi_comm_world.comm);
+    ctx_params.oob.oob_ep       = ompi_comm_rank(&ompi_mpi_comm_world.comm);
+    if (UCC_OK != ucc_context_config_read(cm->ucc_lib, NULL, &ctx_config)) {
+        UCC_ERROR("UCC context config read failed");
+        goto cleanup_lib;
+    }
+
+    sprintf(str_buf, "%u", ompi_proc_world_size());
+    if (UCC_OK != ucc_context_config_modify(ctx_config, NULL, "ESTIMATED_NUM_EPS",
+                                            str_buf)) {
+        UCC_ERROR("UCC context config modify failed for estimated_num_eps");
+        goto cleanup_lib;
+    }
+
+    sprintf(str_buf, "%u", opal_process_info.num_local_peers + 1);
+    if (UCC_OK != ucc_context_config_modify(ctx_config, NULL, "ESTIMATED_NUM_PPN",
+                                            str_buf)) {
+        UCC_ERROR("UCC context config modify failed for estimated_num_eps");
+        goto cleanup_lib;
+    }
+
+    if (UCC_OK != ucc_context_create(cm->ucc_lib, &ctx_params,
+                                     ctx_config, &cm->ucc_context)) {
+        UCC_ERROR("UCC context create failed");
+        ucc_context_config_release(ctx_config);
+        goto cleanup_lib;
+    }
+    ucc_context_config_release(ctx_config);
+
+    copy_fn.attr_communicator_copy_fn  = MPI_COMM_NULL_COPY_FN;
+    del_fn.attr_communicator_delete_fn = ucc_comm_attr_del_fn;
+    if (OMPI_SUCCESS != ompi_attr_create_keyval(COMM_ATTR, copy_fn, del_fn,
+                                                &ucc_comm_attr_keyval, NULL ,0, NULL)) {
+        UCC_ERROR("UCC comm keyval create failed");
+        goto cleanup_ctx;
+    }
+
+    OBJ_CONSTRUCT(&cm->requests, opal_free_list_t);
+    opal_free_list_init(&cm->requests, sizeof(mca_coll_ucc_req_t),
+                        opal_cache_line_size, OBJ_CLASS(mca_coll_ucc_req_t),
+                        0, 0,                     /* no payload data */
+                        8, -1, 8,                 /* num_to_alloc, max, per alloc */
+                        NULL, 0, NULL, NULL, NULL /* no Mpool or init function */);
+
+    opal_progress_register(mca_coll_ucc_progress);
+    UCC_VERBOSE(1, "initialized ucc context");
+    cm->libucc_initialized = true;
+    return OMPI_SUCCESS;
+cleanup_ctx:
+    ucc_context_destroy(cm->ucc_context);
+
+cleanup_lib:
+    ucc_finalize(cm->ucc_lib);
+    cm->ucc_enable         = 0;
+    cm->libucc_initialized = false;
+    return OMPI_ERROR;
+}
+
+uint64_t rank_map_cb(uint64_t ep, void *cb_ctx)
+{
+    struct ompi_communicator_t *comm = cb_ctx;
+
+    return ((ompi_process_name_t*)&ompi_comm_peer_lookup(comm, ep)->super.
+            proc_name)->vpid;
+}
+
+static inline ucc_ep_map_t get_rank_map(struct ompi_communicator_t *comm)
+{
+    ucc_ep_map_t map;
+    int64_t      r1, r2, stride;
+    uint64_t     i;
+    int          is_strided;
+
+    map.ep_num = ompi_comm_size(comm);
+    if (comm == &ompi_mpi_comm_world.comm) {
+        map.type = UCC_EP_MAP_FULL;
+        return map;
+    }
+
+    /* try to detect strided pattern */
+    is_strided = 1;
+    r1         = rank_map_cb(0, comm);
+    r2         = rank_map_cb(1, comm);
+    stride     = r2 - r1;
+    for (i = 2; i < map.ep_num; i++) {
+        r1 = r2;
+        r2 = rank_map_cb(i, comm);
+        if (r2 - r1 != stride) {
+            is_strided = 0;
+            break;
+        }
+    }
+
+    if (is_strided) {
+        map.type           = UCC_EP_MAP_STRIDED;
+        map.strided.start  = r1;
+        map.strided.stride = stride;
+    } else {
+        map.type      = UCC_EP_MAP_CB;
+        map.cb.cb     = rank_map_cb;
+        map.cb.cb_ctx = (void*)comm;
+    }
+
+    return map;
+}
+
+#define UCC_INSTALL_COLL_API(__comm, __ucc_module, __COLL, __api)                                                                          \
+    do                                                                                                                                     \
+    {                                                                                                                                      \
+        if ((mca_coll_ucc_component.ucc_lib_attr.coll_types & UCC_COLL_TYPE_##__COLL))                                                     \
+        {                                                                                                                                  \
+            if (mca_coll_ucc_component.cts_requested & UCC_COLL_TYPE_##__COLL)                                                             \
+            {                                                                                                                              \
+                MCA_COLL_SAVE_API(__comm, __api, (__ucc_module)->previous_##__api, (__ucc_module)->previous_##__api##_module, "ucc");      \
+                MCA_COLL_INSTALL_API(__comm, __api, mca_coll_ucc_##__api, &__ucc_module->super, "ucc");                                    \
+                (__ucc_module)->super.coll_##__api = mca_coll_ucc_##__api;                                                                 \
+            }                                                                                                                              \
+            if (mca_coll_ucc_component.nb_cts_requested & UCC_COLL_TYPE_##__COLL)                                                          \
+            {                                                                                                                              \
+                MCA_COLL_SAVE_API(__comm, i##__api, (__ucc_module)->previous_i##__api, (__ucc_module)->previous_i##__api##_module, "ucc"); \
+                MCA_COLL_INSTALL_API(__comm, i##__api, mca_coll_ucc_i##__api, &__ucc_module->super, "ucc");                                \
+                (__ucc_module)->super.coll_i##__api = mca_coll_ucc_i##__api;                                                               \
+            }                                                                                                                              \
+        }                                                                                                                                  \
+    } while (0)
+
+static int mca_coll_ucc_replace_coll_handlers(mca_coll_ucc_module_t *ucc_module)
+{
+    ompi_communicator_t *comm = ucc_module->comm;
+
+    UCC_INSTALL_COLL_API(comm, ucc_module, ALLREDUCE, allreduce);
+    UCC_INSTALL_COLL_API(comm, ucc_module, BARRIER, barrier);
+    UCC_INSTALL_COLL_API(comm, ucc_module, BCAST, bcast);
+    UCC_INSTALL_COLL_API(comm, ucc_module, ALLTOALL, alltoall);
+    UCC_INSTALL_COLL_API(comm, ucc_module, ALLTOALLV, alltoallv);
+    UCC_INSTALL_COLL_API(comm, ucc_module, ALLGATHER, allgather);
+    UCC_INSTALL_COLL_API(comm, ucc_module, ALLGATHERV, allgatherv);
+    UCC_INSTALL_COLL_API(comm, ucc_module, REDUCE, reduce);
+
+    UCC_INSTALL_COLL_API(comm, ucc_module, GATHER, gather);
+    UCC_INSTALL_COLL_API(comm, ucc_module, GATHERV, gatherv);
+    UCC_INSTALL_COLL_API(comm, ucc_module, REDUCE_SCATTER, reduce_scatter_block);
+    UCC_INSTALL_COLL_API(comm, ucc_module, REDUCE_SCATTERV, reduce_scatter);
+    UCC_INSTALL_COLL_API(comm, ucc_module, SCATTER, scatter);
+    UCC_INSTALL_COLL_API(comm, ucc_module, SCATTERV, scatterv);
+
+    return OMPI_SUCCESS;
+}
+
+/*
+ * Initialize module on the communicator
+ */
+static int mca_coll_ucc_module_enable(mca_coll_base_module_t *module,
+                                      struct ompi_communicator_t *comm)
+{
+    mca_coll_ucc_component_t *cm         = &mca_coll_ucc_component;
+    mca_coll_ucc_module_t    *ucc_module = (mca_coll_ucc_module_t *)module;
+    ucc_status_t              status;
+    int rc;
+    ucc_team_params_t team_params = {
+        .mask   = UCC_TEAM_PARAM_FIELD_EP_MAP   |
+                  UCC_TEAM_PARAM_FIELD_EP       |
+                  UCC_TEAM_PARAM_FIELD_EP_RANGE |
+                  UCC_TEAM_PARAM_FIELD_ID,
+        .ep_map = {
+            .type      = (comm == &ompi_mpi_comm_world.comm) ?
+                          UCC_EP_MAP_FULL : UCC_EP_MAP_CB,
+            .ep_num    = ompi_comm_size(comm),
+            .cb.cb     = rank_map_cb,
+            .cb.cb_ctx = (void*)comm
+        },
+        .ep       = ompi_comm_rank(comm),
+        .ep_range = UCC_COLLECTIVE_EP_RANGE_CONTIG,
+        .id       = ompi_comm_get_local_cid(comm)
+    };
+    UCC_VERBOSE(2, "creating ucc_team for comm %p, comm_id %llu, comm_size %d",
+                (void*)comm, (long long unsigned)team_params.id,
+                ompi_comm_size(comm));
+
+    if (UCC_OK != ucc_team_create_post(&cm->ucc_context, 1,
+                                       &team_params, &ucc_module->ucc_team)) {
+        UCC_ERROR("ucc_team_create_post failed");
+        goto err;
+    }
+    while (UCC_INPROGRESS == (status = ucc_team_create_test(
+                                  ucc_module->ucc_team))) {
+        opal_progress();
+    }
+    if (UCC_OK != status) {
+        UCC_ERROR("ucc_team_create_test failed");
+        goto err;
+    }
+
+    if (OMPI_SUCCESS != mca_coll_ucc_replace_coll_handlers(ucc_module)) {
+        UCC_ERROR("mca_coll_ucc_replace_coll_handlers failed");
+        goto err;
+    }
+
+    rc = ompi_attr_set_c(COMM_ATTR, comm, &comm->c_keyhash,
+                         ucc_comm_attr_keyval, (void *)module, false);
+    if (OMPI_SUCCESS != rc) {
+        UCC_ERROR("ucc ompi_attr_set_c failed");
+        goto err;
+    }
+
+    return OMPI_SUCCESS;
+
+err:
+    ucc_module->ucc_team = NULL;
+    cm->ucc_enable       = 0;
+    opal_progress_unregister(mca_coll_ucc_progress);
+    return OMPI_ERROR;
+}
+
+#define UCC_UNINSTALL_COLL_API(__comm, __ucc_module, __api)                                                                          \
+    do                                                                                                                               \
+    {                                                                                                                                \
+        if (&(__ucc_module)->super == (__comm)->c_coll->coll_##__api##_module)                                                       \
+        {                                                                                                                            \
+            MCA_COLL_INSTALL_API(__comm, __api, (__ucc_module)->previous_##__api, (__ucc_module)->previous_##__api##_module, "ucc"); \
+            (__ucc_module)->previous_##__api = NULL;                                                                                 \
+            (__ucc_module)->previous_##__api##_module = NULL;                                                                        \
+        }                                                                                                                            \
+    } while (0)
+
+/**
+ * The disable will be called once per collective module, in the reverse order
+ * in which enable has been called. This reverse order allows the module to properly
+ * unregister the collective function pointers they provide for the communicator.
+ */
+static int
+mca_coll_ucc_module_disable(mca_coll_base_module_t *module,
+                            struct ompi_communicator_t *comm)
+{
+    mca_coll_ucc_module_t *ucc_module = (mca_coll_ucc_module_t*)module;
+    UCC_UNINSTALL_COLL_API(comm, ucc_module, allreduce);
+    UCC_UNINSTALL_COLL_API(comm, ucc_module, iallreduce);
+    UCC_UNINSTALL_COLL_API(comm, ucc_module, barrier);
+    UCC_UNINSTALL_COLL_API(comm, ucc_module, ibarrier);
+    UCC_UNINSTALL_COLL_API(comm, ucc_module, bcast);
+    UCC_UNINSTALL_COLL_API(comm, ucc_module, ibcast);
+    UCC_UNINSTALL_COLL_API(comm, ucc_module, alltoall);
+    UCC_UNINSTALL_COLL_API(comm, ucc_module, ialltoall);
+    UCC_UNINSTALL_COLL_API(comm, ucc_module, alltoallv);
+    UCC_UNINSTALL_COLL_API(comm, ucc_module, ialltoallv);
+    UCC_UNINSTALL_COLL_API(comm, ucc_module, allgather);
+    UCC_UNINSTALL_COLL_API(comm, ucc_module, iallgather);
+    UCC_UNINSTALL_COLL_API(comm, ucc_module, allgatherv);
+    UCC_UNINSTALL_COLL_API(comm, ucc_module, iallgatherv);
+    UCC_UNINSTALL_COLL_API(comm, ucc_module, reduce);
+    UCC_UNINSTALL_COLL_API(comm, ucc_module, ireduce);
+    UCC_UNINSTALL_COLL_API(comm, ucc_module, gather);
+    UCC_UNINSTALL_COLL_API(comm, ucc_module, gatherv);
+    UCC_UNINSTALL_COLL_API(comm, ucc_module, reduce_scatter_block);
+    UCC_UNINSTALL_COLL_API(comm, ucc_module, reduce_scatter);
+    UCC_UNINSTALL_COLL_API(comm, ucc_module, scatter);
+    UCC_UNINSTALL_COLL_API(comm, ucc_module, scatterv);
+
+    return OMPI_SUCCESS;
+}
+
+
+/*
+ * Invoked when there's a new communicator that has been created.
+ * Look at the communicator and decide which set of functions and
+ * priority we want to return.
+ */
+mca_coll_base_module_t *
+mca_coll_ucc_comm_query(struct ompi_communicator_t *comm, int *priority)
+{
+    mca_coll_ucc_component_t *cm = &mca_coll_ucc_component;
+    mca_coll_ucc_module_t    *ucc_module;
+    *priority = 0;
+
+    if (!cm->ucc_enable){
+        return NULL;
+    }
+
+    if (OMPI_COMM_IS_INTER(comm) || ompi_comm_size(comm) < cm->ucc_np
+        || ompi_comm_size(comm) < 2){
+        return NULL;
+    }
+
+    if (!cm->libucc_initialized) {
+        if (OMPI_SUCCESS != mca_coll_ucc_init_ctx()) {
+            cm->ucc_enable = 0;
+            return NULL;
+        }
+    }
+
+    ucc_module = OBJ_NEW(mca_coll_ucc_module_t);
+    if (!ucc_module) {
+        cm->ucc_enable = 0;
+        return NULL;
+    }
+    ucc_module->comm                      = comm;
+    ucc_module->super.coll_module_enable  = mca_coll_ucc_module_enable;
+    ucc_module->super.coll_module_disable = mca_coll_ucc_module_disable;
+    *priority                             = cm->ucc_priority;
+
+    return &ucc_module->super;
+}
+
+
+OBJ_CLASS_INSTANCE(mca_coll_ucc_module_t,
+                   mca_coll_base_module_t,
+                   mca_coll_ucc_module_construct,
+                   mca_coll_ucc_module_destruct);
+
+OBJ_CLASS_INSTANCE(mca_coll_ucc_req_t, ompi_request_t,
+                   NULL, NULL);
+
+int mca_coll_ucc_req_free(struct ompi_request_t **ompi_req)
+{
+    opal_free_list_return (&mca_coll_ucc_component.requests,
+                           (opal_free_list_item_t *)(*ompi_req));
+    *ompi_req = MPI_REQUEST_NULL;
+    return OMPI_SUCCESS;
+}
+
+
+void mca_coll_ucc_completion(void *data, ucc_status_t status)
+{
+    mca_coll_ucc_req_t *coll_req = (mca_coll_ucc_req_t*)data;
+    ucc_collective_finalize(coll_req->ucc_req);
+    ompi_request_complete(&coll_req->super, true);
+}
diff --git a/ompi/mca/coll/ucc/coll_ucc_reduce.c b/ompi/mca/coll/ucc/coll_ucc_reduce.c
new file mode 100644
index 00000000000..97b5d424ccf
--- /dev/null
+++ b/ompi/mca/coll/ucc/coll_ucc_reduce.c
@@ -0,0 +1,108 @@
+/**
+ * Copyright (c) 2021 Mellanox Technologies. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ */
+
+#include "coll_ucc_common.h"
+
+static inline ucc_status_t mca_coll_ucc_reduce_init(const void *sbuf, void *rbuf, size_t count,
+                                                    struct ompi_datatype_t *dtype,
+                                                    struct ompi_op_t *op, int root,
+                                                    mca_coll_ucc_module_t *ucc_module,
+                                                    ucc_coll_req_h *req,
+                                                    mca_coll_ucc_req_t *coll_req)
+{
+    ucc_datatype_t         ucc_dt;
+    ucc_reduction_op_t     ucc_op;
+
+    ucc_dt = ompi_dtype_to_ucc_dtype(dtype);
+    ucc_op = ompi_op_to_ucc_op(op);
+    if (OPAL_UNLIKELY(COLL_UCC_DT_UNSUPPORTED == ucc_dt)) {
+        UCC_VERBOSE(5, "ompi_datatype is not supported: dtype = %s",
+                    dtype->super.name);
+        goto fallback;
+    }
+    if (OPAL_UNLIKELY(COLL_UCC_OP_UNSUPPORTED == ucc_op)) {
+        UCC_VERBOSE(5, "ompi_op is not supported: op = %s",
+                    op->o_name);
+        goto fallback;
+    }
+    ucc_coll_args_t coll = {
+        .mask      = 0,
+        .flags     = 0,
+        .coll_type = UCC_COLL_TYPE_REDUCE,
+        .root = root,
+        .src.info = {
+            .buffer   = (void*)sbuf,
+            .count    = count,
+            .datatype = ucc_dt,
+            .mem_type = UCC_MEMORY_TYPE_UNKNOWN
+        },
+        .dst.info = {
+            .buffer   = rbuf,
+            .count    = count,
+            .datatype = ucc_dt,
+            .mem_type = UCC_MEMORY_TYPE_UNKNOWN
+        },
+        .op = ucc_op,
+    };
+    if (MPI_IN_PLACE == sbuf) {
+        coll.mask |= UCC_COLL_ARGS_FIELD_FLAGS;
+        coll.flags = UCC_COLL_ARGS_FLAG_IN_PLACE;
+    }
+    COLL_UCC_REQ_INIT(coll_req, req, coll, ucc_module);
+    return UCC_OK;
+fallback:
+    return UCC_ERR_NOT_SUPPORTED;
+}
+
+int mca_coll_ucc_reduce(const void *sbuf, void* rbuf, size_t count,
+                        struct ompi_datatype_t *dtype,
+                        struct ompi_op_t *op, int root,
+                        struct ompi_communicator_t *comm,
+                        struct mca_coll_base_module_3_0_0_t *module)
+{
+    mca_coll_ucc_module_t *ucc_module = (mca_coll_ucc_module_t*)module;
+    ucc_coll_req_h         req;
+
+    UCC_VERBOSE(3, "running ucc reduce");
+    COLL_UCC_CHECK(mca_coll_ucc_reduce_init(sbuf, rbuf, count, dtype, op,
+                                            root, ucc_module, &req, NULL));
+    COLL_UCC_POST_AND_CHECK(req);
+    COLL_UCC_CHECK(coll_ucc_req_wait(req));
+    return OMPI_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback reduce");
+    return ucc_module->previous_reduce(sbuf, rbuf, count, dtype, op, root,
+                                       comm, ucc_module->previous_reduce_module);
+}
+
+int mca_coll_ucc_ireduce(const void *sbuf, void* rbuf, size_t count,
+                         struct ompi_datatype_t *dtype,
+                         struct ompi_op_t *op, int root,
+                         struct ompi_communicator_t *comm,
+                         ompi_request_t** request,
+                         struct mca_coll_base_module_3_0_0_t *module)
+{
+    mca_coll_ucc_module_t *ucc_module = (mca_coll_ucc_module_t*)module;
+    ucc_coll_req_h         req;
+    mca_coll_ucc_req_t    *coll_req = NULL;
+
+    UCC_VERBOSE(3, "running ucc ireduce");
+    COLL_UCC_GET_REQ(coll_req);
+    COLL_UCC_CHECK(mca_coll_ucc_reduce_init(sbuf, rbuf, count, dtype, op, root,
+                                            ucc_module, &req, coll_req));
+    COLL_UCC_POST_AND_CHECK(req);
+    *request = &coll_req->super;
+    return OMPI_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback ireduce");
+    if (coll_req) {
+        mca_coll_ucc_req_free((ompi_request_t **)&coll_req);
+    }
+    return ucc_module->previous_ireduce(sbuf, rbuf, count, dtype, op, root,
+                                        comm, request, ucc_module->previous_ireduce_module);
+}
diff --git a/ompi/mca/coll/ucc/coll_ucc_reduce_scatter.c b/ompi/mca/coll/ucc/coll_ucc_reduce_scatter.c
new file mode 100644
index 00000000000..dabc8f11d03
--- /dev/null
+++ b/ompi/mca/coll/ucc/coll_ucc_reduce_scatter.c
@@ -0,0 +1,124 @@
+/**
+ * Copyright (c) 2021 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2022 NVIDIA Corporation. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ */
+
+#include "coll_ucc_common.h"
+
+static inline
+ucc_status_t mca_coll_ucc_reduce_scatter_init(const void *sbuf, void *rbuf, ompi_count_array_t rcounts,
+                                              struct ompi_datatype_t *dtype,
+                                              struct ompi_op_t *op, mca_coll_ucc_module_t *ucc_module,
+                                              ucc_coll_req_h *req,
+                                              mca_coll_ucc_req_t *coll_req)
+{
+    ucc_datatype_t ucc_dt;
+    ucc_reduction_op_t ucc_op;
+    size_t total_count;
+    int i;
+    int comm_size = ompi_comm_size(ucc_module->comm);
+    uint64_t flags = 0;
+
+    if (MPI_IN_PLACE == sbuf) {
+        /* TODO: UCC defines inplace differently:
+           data in rbuf of rank R is shifted by R * rcount */
+        UCC_VERBOSE(5, "inplace reduce_scatter is not supported");
+        return UCC_ERR_NOT_SUPPORTED;
+    }
+
+    ucc_dt = ompi_dtype_to_ucc_dtype(dtype);
+    ucc_op = ompi_op_to_ucc_op(op);
+    if (OPAL_UNLIKELY(COLL_UCC_DT_UNSUPPORTED == ucc_dt)) {
+        UCC_VERBOSE(5, "ompi_datatype is not supported: dtype = %s",
+                    dtype->super.name);
+        goto fallback;
+    }
+    if (OPAL_UNLIKELY(COLL_UCC_OP_UNSUPPORTED == ucc_op)) {
+        UCC_VERBOSE(5, "ompi_op is not supported: op = %s",
+                    op->o_name);
+        goto fallback;
+    }
+    total_count = 0;
+    for (i = 0; i < comm_size; i++) {
+        total_count += ompi_count_array_get(rcounts, i);
+    }
+
+    flags = (ompi_count_array_is_64bit(rcounts) ? UCC_COLL_ARGS_FLAG_COUNT_64BIT : 0);
+
+    ucc_coll_args_t coll = {
+        .mask      = flags ? UCC_COLL_ARGS_FIELD_FLAGS : 0,
+        .flags     = flags,
+        .coll_type = UCC_COLL_TYPE_REDUCE_SCATTERV,
+        .src.info = {
+            .buffer   = (void*)sbuf,
+            .count    = total_count,
+            .datatype = ucc_dt,
+            .mem_type = UCC_MEMORY_TYPE_UNKNOWN
+        },
+        .dst.info_v = {
+            .buffer   = rbuf,
+            .counts   = (ucc_count_t*)ompi_count_array_ptr(rcounts),
+            .datatype = ucc_dt,
+            .mem_type = UCC_MEMORY_TYPE_UNKNOWN
+        },
+        .op = ucc_op,
+    };
+    COLL_UCC_REQ_INIT(coll_req, req, coll, ucc_module);
+    return UCC_OK;
+fallback:
+    return UCC_ERR_NOT_SUPPORTED;
+}
+
+int mca_coll_ucc_reduce_scatter(const void *sbuf, void *rbuf, ompi_count_array_t rcounts,
+                                struct ompi_datatype_t *dtype,
+                                struct ompi_op_t *op,
+                                struct ompi_communicator_t *comm,
+                                mca_coll_base_module_t *module)
+{
+    mca_coll_ucc_module_t *ucc_module = (mca_coll_ucc_module_t*)module;
+    ucc_coll_req_h         req;
+
+    UCC_VERBOSE(3, "running ucc reduce_scatter");
+    COLL_UCC_CHECK(mca_coll_ucc_reduce_scatter_init(sbuf, rbuf, rcounts, dtype,
+                                                    op, ucc_module, &req, NULL));
+    COLL_UCC_POST_AND_CHECK(req);
+    COLL_UCC_CHECK(coll_ucc_req_wait(req));
+    return OMPI_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback reduce_scatter");
+    return ucc_module->previous_reduce_scatter(sbuf, rbuf, rcounts, dtype, op,
+                                               comm,
+                                               ucc_module->previous_reduce_scatter_module);
+}
+
+int mca_coll_ucc_ireduce_scatter(const void *sbuf, void *rbuf, ompi_count_array_t rcounts,
+                                 struct ompi_datatype_t *dtype,
+                                 struct ompi_op_t *op,
+                                 struct ompi_communicator_t *comm,
+                                 ompi_request_t** request,
+                                 mca_coll_base_module_t *module)
+{
+    mca_coll_ucc_module_t *ucc_module = (mca_coll_ucc_module_t*)module;
+    ucc_coll_req_h         req;
+    mca_coll_ucc_req_t    *coll_req = NULL;
+
+    UCC_VERBOSE(3, "running ucc ireduce_scatter");
+    COLL_UCC_GET_REQ(coll_req);
+    COLL_UCC_CHECK(mca_coll_ucc_reduce_scatter_init(sbuf, rbuf, rcounts, dtype,
+                                                    op, ucc_module, &req, coll_req));
+    COLL_UCC_POST_AND_CHECK(req);
+    *request = &coll_req->super;
+    return OMPI_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback ireduce_scatter");
+    if (coll_req) {
+        mca_coll_ucc_req_free((ompi_request_t **)&coll_req);
+    }
+    return ucc_module->previous_ireduce_scatter(sbuf, rbuf, rcounts, dtype, op,
+                                                comm, request,
+                                                ucc_module->previous_ireduce_scatter_module);
+}
diff --git a/ompi/mca/coll/ucc/coll_ucc_reduce_scatter_block.c b/ompi/mca/coll/ucc/coll_ucc_reduce_scatter_block.c
new file mode 100644
index 00000000000..781776e42ca
--- /dev/null
+++ b/ompi/mca/coll/ucc/coll_ucc_reduce_scatter_block.c
@@ -0,0 +1,117 @@
+/**
+ * Copyright (c) 2021 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2022 NVIDIA Corporation. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ */
+
+#include "coll_ucc_common.h"
+
+static inline
+ucc_status_t mca_coll_ucc_reduce_scatter_block_init(const void *sbuf, void *rbuf,
+                                                    size_t rcount,
+                                                    struct ompi_datatype_t *dtype,
+                                                    struct ompi_op_t *op,
+                                                    mca_coll_ucc_module_t *ucc_module,
+                                                    ucc_coll_req_h *req,
+                                                    mca_coll_ucc_req_t *coll_req)
+{
+    ucc_datatype_t ucc_dt;
+    ucc_reduction_op_t ucc_op;
+    int comm_size = ompi_comm_size(ucc_module->comm);
+
+    if (MPI_IN_PLACE == sbuf) {
+        /* TODO: UCC defines inplace differently:
+           data in rbuf of rank R is shifted by R * rcount */
+        UCC_VERBOSE(5, "inplace reduce_scatter_block is not supported");
+        return UCC_ERR_NOT_SUPPORTED;
+    }
+    ucc_dt = ompi_dtype_to_ucc_dtype(dtype);
+    ucc_op = ompi_op_to_ucc_op(op);
+    if (OPAL_UNLIKELY(COLL_UCC_DT_UNSUPPORTED == ucc_dt)) {
+        UCC_VERBOSE(5, "ompi_datatype is not supported: dtype = %s",
+                    dtype->super.name);
+        goto fallback;
+    }
+    if (OPAL_UNLIKELY(COLL_UCC_OP_UNSUPPORTED == ucc_op)) {
+        UCC_VERBOSE(5, "ompi_op is not supported: op = %s",
+                    op->o_name);
+        goto fallback;
+    }
+    ucc_coll_args_t coll = {
+        .mask      = 0,
+        .flags     = 0,
+        .coll_type = UCC_COLL_TYPE_REDUCE_SCATTER,
+        .src.info = {
+            .buffer   = (void*)sbuf,
+            .count    = ((size_t)rcount) * comm_size,
+            .datatype = ucc_dt,
+            .mem_type = UCC_MEMORY_TYPE_UNKNOWN
+        },
+        .dst.info = {
+            .buffer   = rbuf,
+            .count    = rcount,
+            .datatype = ucc_dt,
+            .mem_type = UCC_MEMORY_TYPE_UNKNOWN
+        },
+        .op = ucc_op,
+    };
+    COLL_UCC_REQ_INIT(coll_req, req, coll, ucc_module);
+    return UCC_OK;
+fallback:
+    return UCC_ERR_NOT_SUPPORTED;
+}
+
+int mca_coll_ucc_reduce_scatter_block(const void *sbuf, void *rbuf, size_t rcount,
+                                      struct ompi_datatype_t *dtype,
+                                      struct ompi_op_t *op,
+                                      struct ompi_communicator_t *comm,
+                                      mca_coll_base_module_t *module)
+{
+    mca_coll_ucc_module_t *ucc_module = (mca_coll_ucc_module_t*)module;
+    ucc_coll_req_h         req;
+
+    UCC_VERBOSE(3, "running ucc reduce scatter block");
+    COLL_UCC_CHECK(mca_coll_ucc_reduce_scatter_block_init(sbuf, rbuf, rcount,
+                                                          dtype, op, ucc_module,
+                                                          &req, NULL));
+    COLL_UCC_POST_AND_CHECK(req);
+    COLL_UCC_CHECK(coll_ucc_req_wait(req));
+    return OMPI_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback reduce_scatter_block");
+    return ucc_module->previous_reduce_scatter_block(sbuf, rbuf, rcount, dtype,
+                                                     op, comm,
+                                                     ucc_module->previous_reduce_scatter_block_module);
+}
+
+int mca_coll_ucc_ireduce_scatter_block(const void *sbuf, void *rbuf, size_t rcount,
+                                       struct ompi_datatype_t *dtype,
+                                       struct ompi_op_t *op,
+                                       struct ompi_communicator_t *comm,
+                                       ompi_request_t** request,
+                                       mca_coll_base_module_t *module)
+{
+    mca_coll_ucc_module_t *ucc_module = (mca_coll_ucc_module_t*)module;
+    ucc_coll_req_h         req;
+    mca_coll_ucc_req_t    *coll_req = NULL;
+
+    UCC_VERBOSE(3, "running ucc ireduce_scatter_block");
+    COLL_UCC_GET_REQ(coll_req);
+    COLL_UCC_CHECK(mca_coll_ucc_reduce_scatter_block_init(sbuf, rbuf, rcount,
+                                                          dtype, op, ucc_module,
+                                                          &req, coll_req));
+    COLL_UCC_POST_AND_CHECK(req);
+    *request = &coll_req->super;
+    return OMPI_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback ireduce_scatter_block");
+    if (coll_req) {
+        mca_coll_ucc_req_free((ompi_request_t **)&coll_req);
+    }
+    return ucc_module->previous_ireduce_scatter_block(sbuf, rbuf, rcount, dtype,
+                                                      op, comm, request,
+                                                      ucc_module->previous_ireduce_scatter_block_module);
+}
diff --git a/ompi/mca/coll/ucc/coll_ucc_scatter.c b/ompi/mca/coll/ucc/coll_ucc_scatter.c
new file mode 100644
index 00000000000..481365f22bd
--- /dev/null
+++ b/ompi/mca/coll/ucc/coll_ucc_scatter.c
@@ -0,0 +1,137 @@
+/**
+ * Copyright (c) 2021 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2022 NVIDIA Corporation. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ */
+
+#include "coll_ucc_common.h"
+
+static inline
+ucc_status_t mca_coll_ucc_scatter_init(const void *sbuf, size_t scount,
+                                       struct ompi_datatype_t *sdtype,
+                                       void *rbuf, size_t rcount,
+                                       struct ompi_datatype_t *rdtype, int root,
+                                       mca_coll_ucc_module_t *ucc_module,
+                                       ucc_coll_req_h *req,
+                                       mca_coll_ucc_req_t *coll_req)
+{
+    ucc_datatype_t ucc_sdt = UCC_DT_INT8, ucc_rdt = UCC_DT_INT8;
+    bool is_inplace = (MPI_IN_PLACE == rbuf);
+    int comm_rank = ompi_comm_rank(ucc_module->comm);
+    int comm_size = ompi_comm_size(ucc_module->comm);
+
+    if (comm_rank == root) {
+        if (!(is_inplace || ompi_datatype_is_contiguous_memory_layout(rdtype, rcount)) ||
+            !ompi_datatype_is_contiguous_memory_layout(sdtype, scount * comm_size)) {
+            goto fallback;
+        }
+
+        ucc_sdt = ompi_dtype_to_ucc_dtype(sdtype);
+        if (!is_inplace) {
+            ucc_rdt = ompi_dtype_to_ucc_dtype(rdtype);
+        }
+
+        if ((COLL_UCC_DT_UNSUPPORTED == ucc_sdt) ||
+            (COLL_UCC_DT_UNSUPPORTED == ucc_rdt)) {
+            UCC_VERBOSE(5, "ompi_datatype is not supported: dtype = %s",
+                        (COLL_UCC_DT_UNSUPPORTED == ucc_sdt) ?
+                        sdtype->super.name : rdtype->super.name);
+            goto fallback;
+        }
+    } else {
+        if (!ompi_datatype_is_contiguous_memory_layout(rdtype, rcount)) {
+            goto fallback;
+        }
+
+        ucc_rdt = ompi_dtype_to_ucc_dtype(rdtype);
+        if (COLL_UCC_DT_UNSUPPORTED == ucc_rdt) {
+            UCC_VERBOSE(5, "ompi_datatype is not supported: dtype = %s",
+                        rdtype->super.name);
+            goto fallback;
+        }
+    }
+
+    ucc_coll_args_t coll = {
+        .mask      = 0,
+        .flags     = 0,
+        .coll_type = UCC_COLL_TYPE_SCATTER,
+        .root      = root,
+        .src.info  = {
+            .buffer   = (void*)sbuf,
+            .count    = scount * comm_size,
+            .datatype = ucc_sdt,
+            .mem_type = UCC_MEMORY_TYPE_UNKNOWN
+        },
+        .dst.info = {
+            .buffer   = (void*)rbuf,
+            .count    = rcount,
+            .datatype = ucc_rdt,
+            .mem_type = UCC_MEMORY_TYPE_UNKNOWN
+        },
+    };
+
+    if (is_inplace) {
+        coll.mask |= UCC_COLL_ARGS_FIELD_FLAGS;
+        coll.flags = UCC_COLL_ARGS_FLAG_IN_PLACE;
+    }
+    COLL_UCC_REQ_INIT(coll_req, req, coll, ucc_module);
+    return UCC_OK;
+fallback:
+    return UCC_ERR_NOT_SUPPORTED;
+}
+
+int mca_coll_ucc_scatter(const void *sbuf, size_t scount,
+                         struct ompi_datatype_t *sdtype, void *rbuf, size_t rcount,
+                         struct ompi_datatype_t *rdtype, int root,
+                         struct ompi_communicator_t *comm,
+                         mca_coll_base_module_t *module)
+{
+    mca_coll_ucc_module_t *ucc_module = (mca_coll_ucc_module_t*)module;
+    ucc_coll_req_h         req;
+
+    UCC_VERBOSE(3, "running ucc scatter");
+    COLL_UCC_CHECK(mca_coll_ucc_scatter_init(sbuf, scount, sdtype, rbuf, rcount,
+                                             rdtype, root, ucc_module, &req,
+                                             NULL));
+    COLL_UCC_POST_AND_CHECK(req);
+    COLL_UCC_CHECK(coll_ucc_req_wait(req));
+    return OMPI_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback scatter");
+    return ucc_module->previous_scatter(sbuf, scount, sdtype, rbuf, rcount,
+                                        rdtype, root, comm,
+                                        ucc_module->previous_scatter_module);
+
+}
+
+int mca_coll_ucc_iscatter(const void *sbuf, size_t scount,
+                         struct ompi_datatype_t *sdtype, void *rbuf, size_t rcount,
+                         struct ompi_datatype_t *rdtype, int root,
+                         struct ompi_communicator_t *comm,
+                         ompi_request_t** request,
+                         mca_coll_base_module_t *module)
+{
+    mca_coll_ucc_module_t *ucc_module = (mca_coll_ucc_module_t*)module;
+    ucc_coll_req_h         req;
+    mca_coll_ucc_req_t    *coll_req = NULL;
+
+    UCC_VERBOSE(3, "running ucc iscatter");
+    COLL_UCC_GET_REQ(coll_req);
+    COLL_UCC_CHECK(mca_coll_ucc_scatter_init(sbuf, scount, sdtype, rbuf, rcount,
+                                             rdtype, root, ucc_module, &req,
+                                             coll_req));
+    COLL_UCC_POST_AND_CHECK(req);
+    *request = &coll_req->super;
+    return OMPI_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback iscatter");
+    if (coll_req) {
+        mca_coll_ucc_req_free((ompi_request_t **)&coll_req);
+    }
+    return ucc_module->previous_iscatter(sbuf, scount, sdtype, rbuf, rcount,
+                                         rdtype, root, comm, request,
+                                         ucc_module->previous_iscatter_module);
+}
diff --git a/ompi/mca/coll/ucc/coll_ucc_scatterv.c b/ompi/mca/coll/ucc/coll_ucc_scatterv.c
new file mode 100644
index 00000000000..36d4086a113
--- /dev/null
+++ b/ompi/mca/coll/ucc/coll_ucc_scatterv.c
@@ -0,0 +1,129 @@
+/**
+ * Copyright (c) 2021 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2022 NVIDIA Corporation. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ */
+
+#include "coll_ucc_common.h"
+
+static inline
+ucc_status_t mca_coll_ucc_scatterv_init(const void *sbuf, ompi_count_array_t scounts,
+                                        ompi_disp_array_t disps, struct ompi_datatype_t *sdtype,
+                                        void *rbuf, size_t rcount,
+                                        struct ompi_datatype_t *rdtype, int root,
+                                        mca_coll_ucc_module_t *ucc_module,
+                                        ucc_coll_req_h *req,
+                                        mca_coll_ucc_req_t *coll_req)
+{
+    ucc_datatype_t ucc_sdt = UCC_DT_INT8, ucc_rdt = UCC_DT_INT8;
+    bool is_inplace = (MPI_IN_PLACE == rbuf);
+    int comm_rank = ompi_comm_rank(ucc_module->comm);
+    uint64_t flags = 0;
+    if (comm_rank == root) {
+        ucc_sdt = ompi_dtype_to_ucc_dtype(sdtype);
+        if (!is_inplace) {
+            ucc_rdt = ompi_dtype_to_ucc_dtype(rdtype);
+        }
+
+        if ((COLL_UCC_DT_UNSUPPORTED == ucc_sdt) ||
+            (COLL_UCC_DT_UNSUPPORTED == ucc_rdt)) {
+            UCC_VERBOSE(5, "ompi_datatype is not supported: dtype = %s",
+                        (COLL_UCC_DT_UNSUPPORTED == ucc_sdt) ?
+                        sdtype->super.name : rdtype->super.name);
+            goto fallback;
+        }
+    } else {
+        ucc_rdt = ompi_dtype_to_ucc_dtype(rdtype);
+        if (COLL_UCC_DT_UNSUPPORTED == ucc_rdt) {
+            UCC_VERBOSE(5, "ompi_datatype is not supported: dtype = %s",
+                        rdtype->super.name);
+            goto fallback;
+        }
+    }
+
+    flags = (ompi_count_array_is_64bit(scounts) ? UCC_COLL_ARGS_FLAG_COUNT_64BIT : 0) |
+            (ompi_disp_array_is_64bit(disps) ? UCC_COLL_ARGS_FLAG_DISPLACEMENTS_64BIT : 0) |
+            (is_inplace ? UCC_COLL_ARGS_FLAG_IN_PLACE : 0);
+
+    ucc_coll_args_t coll = {
+        .mask      = flags ? UCC_COLL_ARGS_FIELD_FLAGS : 0,
+        .flags     = flags,
+        .coll_type = UCC_COLL_TYPE_SCATTERV,
+        .root      = root,
+        .src.info_v = {
+            .buffer        = (void*)sbuf,
+            .counts        = (ucc_count_t*)ompi_count_array_ptr(scounts),
+            .displacements = (ucc_aint_t*)ompi_disp_array_ptr(disps),
+            .datatype      = ucc_sdt,
+            .mem_type      = UCC_MEMORY_TYPE_UNKNOWN
+        },
+        .dst.info = {
+            .buffer   = (void*)rbuf,
+            .count    = rcount,
+            .datatype = ucc_rdt,
+            .mem_type = UCC_MEMORY_TYPE_UNKNOWN
+        },
+    };
+
+    COLL_UCC_REQ_INIT(coll_req, req, coll, ucc_module);
+    return UCC_OK;
+fallback:
+    return UCC_ERR_NOT_SUPPORTED;
+}
+
+int mca_coll_ucc_scatterv(const void *sbuf, ompi_count_array_t scounts,
+                          ompi_disp_array_t disps, struct ompi_datatype_t *sdtype,
+                          void *rbuf, size_t rcount,
+                          struct ompi_datatype_t *rdtype, int root,
+                          struct ompi_communicator_t *comm,
+                          mca_coll_base_module_t *module)
+{
+    mca_coll_ucc_module_t *ucc_module = (mca_coll_ucc_module_t*)module;
+    ucc_coll_req_h         req;
+
+    UCC_VERBOSE(3, "running ucc scatterv");
+    COLL_UCC_CHECK(mca_coll_ucc_scatterv_init(sbuf, scounts, disps, sdtype,
+                                              rbuf, rcount, rdtype, root,
+                                              ucc_module, &req, NULL));
+    COLL_UCC_POST_AND_CHECK(req);
+    COLL_UCC_CHECK(coll_ucc_req_wait(req));
+    return OMPI_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback scatterv");
+    return ucc_module->previous_scatterv(sbuf, scounts, disps, sdtype, rbuf,
+                                         rcount, rdtype, root, comm,
+                                         ucc_module->previous_scatterv_module);
+}
+
+int mca_coll_ucc_iscatterv(const void *sbuf, ompi_count_array_t scounts,
+                           ompi_disp_array_t disps, struct ompi_datatype_t *sdtype,
+                           void *rbuf, size_t rcount,
+                           struct ompi_datatype_t *rdtype, int root,
+                           struct ompi_communicator_t *comm,
+                           ompi_request_t** request,
+                           mca_coll_base_module_t *module)
+{
+    mca_coll_ucc_module_t *ucc_module = (mca_coll_ucc_module_t*)module;
+    ucc_coll_req_h         req;
+    mca_coll_ucc_req_t    *coll_req = NULL;
+
+    UCC_VERBOSE(3, "running ucc iscatterv");
+    COLL_UCC_GET_REQ(coll_req);
+    COLL_UCC_CHECK(mca_coll_ucc_scatterv_init(sbuf, scounts, disps, sdtype,
+                                              rbuf, rcount, rdtype, root,
+                                              ucc_module, &req, coll_req));
+    COLL_UCC_POST_AND_CHECK(req);
+    *request = &coll_req->super;
+    return OMPI_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback iscatterv");
+    if (coll_req) {
+        mca_coll_ucc_req_free((ompi_request_t **)&coll_req);
+    }
+    return ucc_module->previous_iscatterv(sbuf, scounts, disps, sdtype, rbuf,
+                                          rcount, rdtype, root, comm, request,
+                                          ucc_module->previous_iscatterv_module);
+}
diff --git a/ompi/mca/coll/ucc/configure.m4 b/ompi/mca/coll/ucc/configure.m4
new file mode 100644
index 00000000000..46c39f6ec7d
--- /dev/null
+++ b/ompi/mca/coll/ucc/configure.m4
@@ -0,0 +1,39 @@
+# -*- shell-script -*-
+#
+#
+# Copyright (c) 2021      Mellanox Technologies. All rights reserved.
+# Copyright (c) 2015      Research Organization for Information Science
+#                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+
+# MCA_coll_ucc_CONFIG([action-if-can-compile],
+#                      [action-if-cant-compile])
+# ------------------------------------------------
+AC_DEFUN([MCA_ompi_coll_ucc_CONFIG],[
+    AC_CONFIG_FILES([ompi/mca/coll/ucc/Makefile])
+
+    OMPI_CHECK_UCC([coll_ucc],
+                     [coll_ucc_happy="yes"],
+                     [coll_ucc_happy="no"])
+
+    AS_IF([test "$coll_ucc_happy" = "yes"],
+          [coll_ucc_WRAPPER_EXTRA_LDFLAGS="$coll_ucc_LDFLAGS"
+           coll_ucc_CPPFLAGS="$coll_ucc_CPPFLAGS"
+           coll_ucc_WRAPPER_EXTRA_LIBS="$coll_ucc_LIBS"
+           $1],
+          [$2])
+    
+    OPAL_SUMMARY_ADD([Miscellaneous], [Open UCC], [], [$coll_ucc_happy])
+
+    # substitute in the things needed to build ucc
+    AC_SUBST([coll_ucc_CPPFLAGS])
+    AC_SUBST([coll_ucc_LDFLAGS])
+    AC_SUBST([coll_ucc_LIBS])
+])dnl
diff --git a/ompi/mca/coll/xhc/Makefile.am b/ompi/mca/coll/xhc/Makefile.am
new file mode 100644
index 00000000000..9351e214200
--- /dev/null
+++ b/ompi/mca/coll/xhc/Makefile.am
@@ -0,0 +1,47 @@
+#
+# Copyright (c) 2021-2024 Computer Architecture and VLSI Systems (CARV)
+#                         Laboratory, ICS Forth. All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+EXTRA_DIST = help-coll-xhc.txt
+
+sources = \
+    coll_xhc.h \
+    coll_xhc_intrinsic.h \
+    coll_xhc.c \
+    coll_xhc_comm.c \
+    coll_xhc_hierarchy.c \
+    coll_xhc_component.c \
+    coll_xhc_module.c \
+    coll_xhc_bcast.c \
+    coll_xhc_barrier.c \
+    coll_xhc_reduce.c \
+    coll_xhc_allreduce.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+component_noinst =
+component_install =
+if MCA_BUILD_ompi_coll_xhc_DSO
+component_install += mca_coll_xhc.la
+else
+component_noinst += libmca_coll_xhc.la
+endif
+
+mcacomponentdir = $(ompilibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_coll_xhc_la_SOURCES = $(sources)
+mca_coll_xhc_la_LDFLAGS = -module -avoid-version
+mca_coll_xhc_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_coll_xhc_la_SOURCES = $(sources)
+libmca_coll_xhc_la_LDFLAGS = -module -avoid-version
diff --git a/ompi/mca/coll/xhc/README.md b/ompi/mca/coll/xhc/README.md
new file mode 100644
index 00000000000..213062a5edc
--- /dev/null
+++ b/ompi/mca/coll/xhc/README.md
@@ -0,0 +1,371 @@
+# XHC: XPMEM-based Hierarchical Collectives
+
+XHC implements hierarchical & topology-aware intra-node MPI collectives,
+utilizing XPMEM for efficient shared address space memory access between
+processes.
+
+## Main features
+
+* XHC constructs an **n-level hierarchy** (i.e. no algorithmic limitation on
+level count), based on intra-node topological features. Rank/process locality
+information is known thanks to Hwloc, and is obtained from Open MPI's
+integrated book-keeping.
+	
+	Topological features that can currently be defined:
+	
+	- NUMA node
+	- CPU Socket
+	- L1/L2/L3 cache
+	- Hwthread/core
+	- Node (all ranks *are* in same node --> flat, no hierarchy at all)
+	
+	Example of a 3-level XHC hierarchy (numa+socket+node configuration):
+	
+	![Example of 3-level XHC hierarchy](resources/xhc-hierarchy.svg)
+	
+	Furthermore, support for custom virtual user-defined hierarchies is
+	available, to allow fine-grained control over the communication pattern.
+
+* **Single-copy** transportation
+	
+	- Supported through integration with Open MPI's `opal/smsc`
+	(shared-memory-single-copy) framework. Selecting `smsc/xpmem` is highly
+	recommended.
+	
+		- Bcast support: XPMEM, CMA, KNEM
+		- Allreduce/Reduce support: XPMEM
+		- Barrier support: *(irrelevant)*
+	
+	- Application buffers are attached on the fly the first time they appear,
+	saved on and recovered from the registration cache in subsequent
+	appearances. (assuming smsc/xpmem)
+
+* **Copy-in-copy-out (CICO)** transportation
+	
+	- Through shared memory buffers that remain active throughout the
+	component's lifetime.
+	
+	- Switchover with single-copy at configurable message size.
+	
+	- Supported in all ops, regardless of smsc support or XPMEM presence (up to
+	maximum allowed message size).
+
+* **Inline** transportation
+	
+	- For especially small messages, payload data is inlined in the same cache
+	line as the control data.
+	
+	- Supported in all ops, regardless of smsc support or XPMEM presence (up to
+	maximum allowed message size).
+
+* Data-wise **pipelining** across all levels of the hierarchy. Allows for
+lowering hierarchy-induced start-up overheads, and interleaving of operations
+in applicable operations (e.g. reduce+bcast in allreduce).
+
+* **Lock-free** single-writer synchronization, with appropriate cache-line
+separation where necessary. Consistency ensured via lightweight *read* or
+*write* memory barriers.
+
+## Configuration options -- MCA params
+
+XHC can be customized via a number of standard Open MPI MCA parameters, though
+defaults that should satisfy a wide number of systems are in place.
+
+The available parameters (also found in `coll_xhc_component.c`):
+
+#### *(prepend with "coll_xhc_")*
+
+* **priority** (default `0`): The priority of the coll/xhc component, used
+during the component selection process.
+
+* **print_info** (default `false`): Print information about XHC's generated
+hierarchy and its configuration.
+
+* **shmem_backing** (default `/dev/shm`): Backing directory for shmem files
+used for XHC's synchronization fields and CICO buffers.
+
+* **dynamic_leader** (default `false`): Enables the feature that dynamically
+elects an XHC-communicator leader at each collective (currently only applicable
+for bcast).
+
+* **dynamic_reduce** (default `1`=`non-float`): Enables support for
+out-of-order reduction. Ranks fetch data to reduce from multiple peers;
+out-of-order reduction allows them to temporarily skip a peer when the expected
+data is not yet prepared, instead of stalling. The default value auto-enables
+it when the data is of non-float type; setting to `2`=`enabled for all types`,
+might/will harm reproducibility of reductions with float types.
+
+* **reduce_load_balance** (default `0`=`non-leader`): Controls the
+leader-to-member load balancing mode in reductions. Under `non-leader`, the
+members, and not the leaders, perform reductions. With `top-level`, all members
+as well as the leader of the top-most level perform reductions. With
+`first-chunk`, leaders perform a single reduction on each level for a single
+chunk at the beginning of the operation. `top+first` combines `top-level` and
+`first-chunk`. Finally, with `all`, all ranks perform reductions equally.
+
+* **hierarchy** (default `"numa,socket"`): A comma separated list of
+topological feature to which XHC's hierarchy-building algorithm should be
+sensitive. `ompi_info` reports the possible values for the parameter.
+	
+	- In some ways, this is "just" a suggestion. The resulting hierarchy may
+	not exactly match the requested one. Reasons that this will occur:
+		
+		- A requested topological feature does not effectively segment the set
+		of ranks. (eg. `numa` was specified, but all ranks reside in the same
+		NUMA node)
+		
+		- No feature that all ranks have in common was provided. This a more
+		intrinsic detail, that you probably don't need to be aware of, but you
+		might come across if eg. you investigate the output of `print_info`. An
+		additional level will automatically be added in this case, no need to
+		worry about it.
+			
+			For all intents and purposes, a hierarchy of `numa,socket` is
+			interpreted as "segment the ranks according to NUMA node locality,
+			and then further segment them according to CPU socket locality".
+		
+		- The provided features will automatically be re-ordered when their
+		order does not match their order in the physical system. (unless a
+		virtual hierarchy is present in the list)
+	
+	- *Virtual Hierarchies*: The string may alternatively also contain "rank
+	lists" which specify exactly which ranks to group together, as well as some
+	other special modifiers. See
+	`coll_xhc_component.c:xhc_component_parse_hierarchy()` for further
+	explanation as well as syntax information.
+
+* **chunk_size** (default `16K`): The chunk size for the pipelining process.
+Data is processed (eg broadcast, reduced) in this-much sized pieces at once.
+	
+	- It's possible to have a different chunk size for each level of the
+	hierarchy, achieved via providing a comma-separated list of sizes (eg.
+	`"16K,16K,128K"`) instead of single one. The sizes in this list's *DO NOT*
+	correspond to the items on hierarchy list; the hierarchy keys might be
+	re-ordered or reduced to match the system, but the chunk sizes will be
+	consumed in the order they are given, left-to-right -> bottom-to-top.
+
+* **uniform_chunks** (default `true`): Automatically optimize the chunk size
+in reduction collectives, according to the message size, so that all members
+will perform equal work.
+
+* **uniform_chunks_min** (default `1K`): The lowest allowed value for the chunk
+size when uniform chunks are enabled.
+
+* **cico_max** (default `1K`): Copy-in-copy-out, instead of single-copy, will
+be used for messages of *cico_max* or less bytes.
+
+*(Removed Parameters)*
+
+* **rcache_max**, **rcache_max_global** *(REMOVED with shift to opal/smsc)*:
+Limit to number of attachments that the registration cache should hold.
+	
+	- A case can be made about their usefulness. If desired, shall be
+	re-implemented at smsc-level.
+
+## Limitations
+
+- *Intra-node support only*
+	- Define XHC as `coll/HAN`'s intra-node component to reap its benefits in
+	multi-node runs.
+
+- **Heterogeneity**: XHC does not support nodes with non-uniform (rank-wise)
+datatype representations. (determined according to Open MPI's `proc_arch`)
+
+- **Non-commutative** operators are not supported by XHC's reduction
+collectives. In past versions, they were, but only with a flat hierarchy; this
+could make a return at some point.
+
+- **Derived Datatypes** are currently not supported.
+
+- XHC's Reduce currently only supports rank 0 as the root, and will
+automatically fall back to another component for other cases.
+
+## Building
+
+This section describes how to compile the XHC component.
+
+XPMEM support in Open MPI is required to reap the full benefits of XHC.
+	
+- The XHC component will build and work without XPMEM support, but for large
+messages (i.e. ones above the CICO threshold) Allreduce/Reduce will be
+disabled, and Broadcast will fall-back to less efficient mechanisms.
+
+- XPMEM can be obtained from <https://github.com/hpc/xpmem>, and then
+compiled like a common kernel module. You might need to manually point Open
+MPI's configure script to XPMEM's installation location, via the
+`--with-xpmem=` parameter.
+
+- At run-time, you will need to insert the kernel module and obtain proper
+access rights to `/dev/xpmem`.
+
+Apart from instructing Open MPI to include XPMEM support, the rest of the build
+process is standard. General information on building Open MPI can be found in
+its documentation.
+
+<https://www.open-mpi.org/doc/>  
+<https://www.open-mpi.org/faq/?category=building>  
+<https://github.com/open-mpi/ompi/blob/master/README.md>
+
+## Running
+
+General information on running Open MPI jobs can be found here:  
+<https://www.open-mpi.org/faq/?category=running>  
+<https://docs.open-mpi.org/en/v5.0.x/launching-apps/index.html>
+
+`mpirun`'s man page will also be useful:  
+<https://docs.open-mpi.org/en/v5.0.x/man-openmpi/man1/mpirun.1.html>
+
+In order for the XHC component to be chosen, its priority must be manually set
+higher than other collectives components that implement the same primitives,
+via the `coll_xhc_priority` MCA param.
+
+	- Example: `--mca coll_xhc_priority 100`
+
+* Most likely, you will also want the `--bind-to core` param. Otherwise, the
+reported process localities might be too general, preventing XHC from correctly
+segmenting the system. (MCA `coll_xhc_print_info` will report the generated
+hierarchy if you wish to experiment)
+
+### Tuning
+
+* Optional: You might wish to manually specify the topological features that
+XHC's hierarchy should conform to. The default is `numa,socket`, which will
+group the processes according to NUMA locality and then further group them
+according to socket locality. See the `coll_xhc_hierarchy` param.
+	
+	- Example: `--mca coll_xhc_hierarchy numa,socket`
+	- Example: `--mca coll_xhc_hierarchy numa`
+	- Example: `--mca coll_xhc_hierarchy flat`
+	
+	In some systems, small-message Broadcast or the Barrier operation might
+	perform better with a flat tree instead of a hierarchical one. Currently,
+	manual benchmarking is required to accurately determine this.
+
+* Optional: You might wish to tune XHC's chunk size (default `16K`). Use the
+`coll_xhc_chunk_size` param, and try values close to the default and see if
+improvements are observed.
+
+	- Example: `--mca coll_xhc_chunk_size 16K`
+
+* Optional: If you wish to focus on latencies of small/medium size messages,
+you can try altering the cico-to-zcopy switchover point (MCA
+`coll_xhc_cico_max`, default `1K`).
+	
+	- Example: `--mca coll_xhc_cico_max 1K`
+
+* Optional: If your application is heavy in Broadcast calls and you suspect
+that specific ranks might be joining the collective with delay and causing
+others to stall waiting for them, try enabling dynamic leadership (MCA
+`coll_xhc_dynamic_leader`), and seeing if it makes an improvement. Please let
+us know if it does :-).
+	
+	- Example: `--mca coll_xhc_dynamic_leader 1`
+
+### Example command lines
+
+*Assuming `PATH` and `LD_LIBRARY_PATH` have been set appropriately.*
+
+Default XHC configuration:  
+`$ mpirun --mca coll_xhc_priority 100 --bind-to core <application>`
+
+XHC w/ numa-sensitive hierarchy, chunk size @ 16K:  
+`$ mpirun --mca coll_xhc_priority 100 --mca coll_xhc_hierarchy numa --mca coll_xhc_chunk_size 16K --bind-to core <application>`
+
+XHC with flat hierarchy (ie. none at all):  
+`$ mpirun --mca coll_xhc_priority 100 --mca coll_xhc_hierarchy node [--bind-to core] <application>`
+
+## Benchmarking
+
+This section outlines some tips for benchmarking XHC and intra-node MPI
+collectives in general.
+
+### Micro-Benchmarks
+
+For our micro-benchmarking purposes, we have been using [OSU's microbenchmark
+suite](https://mvapich.cse.ohio-state.edu/benchmarks/). However, when
+micro-benchmarking intra-node collectives, there are some important details
+that one needs to look out for.
+
+**CPU Cache** An issue with the OSU micro-benchmarks is that they use the same
+buffer for each iteration without altering it. Since modern processors
+implicitly cache data, this can lead to false/unrealistic/unrepresentative
+results, given that actual real-world applications do not (usually/optimally!)
+perform duplicate operations.
+
+Availability of collective operation source data on a processor's local cache
+hierarchy will cause certain phenomenons (e.g. slow path memory transactions)
+and their effects to remain hidden and undetected in the micro-benchmarking
+process, even though they *will* negatively impact performance in actual
+applications,
+
+We have created "data-varying" (`_dv` suffix) benchmarks to counter this
+problem, which will alter the data before each iteration.
+
+**Microbenchmark's pre-op Barrier** One also needs to be aware how the barrier
+that appears before each iteration in the OSU micro-benchmarks affects the
+result, especially so when latencies of small messages are concerned. The
+underlying implementation of this barrier and the speed/efficiency of its
+"release stage" will affect how fast and how synchronized ranks will exit the
+barrier, and therefore how fast/synchronized they will enter the benchmarked
+collective operation.
+
+For as accurate/clean performance reporting as possible, use a barrier
+implementation that has as low a latency as possible. Furthermore, ideally,
+all ranks should exit the barrier at the exact same time -- this is more
+complex to measure, but can make a difference. In order to have a common
+baseline when benchmarking and comparing multiple collectives implementation,
+use this same barrier implementation for all benchmark scenarios.
+
+In the environments we tested, XHC's barrier was the best performing one. To
+make using this barrier easier, we have put together a small new collective
+component, `XB` (= xhc barrier).
+
+XB creates a new nested (duplicate) communicator with a hint to prioritize XHC,
+and delegates barrier operations to it. A slightly inconvenient side-effect is
+that XHC needs to be on the coll list (MCA `--mca coll`); it doesn't need to
+have a high priority, though it can't be less than 0.
+
+* To benchmark Open MPI's `coll/tuned` with XB: `--mca coll basic,libnbc,tuned,xb,xhc --mca coll_xhc_priority 0 --mca coll_xb_priority 95 --mca coll_tuned_priority 90`
+
+* Or XHC itself, with XB: `--mca coll basic,libnbc,xb,xhc --mca coll_xhc_priority 90 --mca coll_xb_priority 95`
+
+It is also possible to specify the hierarchy to be used for XB's barrier (the
+request will be passed in string form to XHC, only for the nested communicator)
+via the `coll_xb_hierarchy` MCA parameter.
+
+In our fork of the OSU micro-benchmarks, you will also find
+"integrity-checking" variants (`_integrity` suffix). These can help verify that
+collective operations complete successfully without data corruption.
+
+Our OSU micro-benchmarks fork:  
+<https://github.com/CARV-ICS-FORTH/XHC-OpenMPI/tree/xhc-fresh/osu-micro-benchmarks>
+
+The XB component:  
+<https://github.com/CARV-ICS-FORTH/XHC-OpenMPI/tree/xhc-fresh/xb>
+
+### Applications
+
+We expect to see any meaningful performance improvement with XHC in actual
+applications, only if they spend a non-insignificant percentage of their
+runtime in the collective operations that XHC implements: Broadcast, Barrier,
+Allreduce, Reduce.
+
+One known such application is [miniAMR](https://github.com/Mantevo/miniAMR).
+The application parameters (e.g. the refine count and frequency) will affect
+the amount of time spent in the Allreduce primitive.
+
+Another one is Microsoft's [CNTK](https://github.com/microsoft/CNTK), also
+heavy in Allreduce, though it actually makes use of the non-blocking
+`Iallreduce` variant. However, it can easily be converted to use the blocking
+variant instead (contact for patch). Comparing the performance of the
+unmodified CNTK with OpenMPI's `coll/libnbc`, versus that of the patched CNTK
+with XHC reveals that this modification is sensible and beneficial.
+
+Finally, while we have not yet rigorously evaluated it,
+[PiSvM](http://pisvm.sourceforge.net/) is another candidate, with intense use
+of MPI Broadcast.
+
+---
+
+Contact: George Katevenis (gkatev@ics.forth.gr), Manolis Ploumidis (ploumid@ics.forth.gr)  
+Computer Architecture and VLSI Systems (CARV) Laboratory, ICS Forth
diff --git a/ompi/mca/coll/xhc/coll_xhc.c b/ompi/mca/coll/xhc/coll_xhc.c
new file mode 100644
index 00000000000..1358da7de69
--- /dev/null
+++ b/ompi/mca/coll/xhc/coll_xhc.c
@@ -0,0 +1,826 @@
+/*
+ * Copyright (c) 2021-2024 Computer Architecture and VLSI Systems (CARV)
+ *                         Laboratory, ICS Forth. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "mpi.h"
+
+#include "ompi/constants.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/mca/coll/coll.h"
+
+#include "opal/class/opal_hash_table.h"
+#include "opal/mca/rcache/rcache.h"
+#include "opal/mca/shmem/base/base.h"
+#include "opal/mca/smsc/smsc.h"
+
+#include "opal/util/show_help.h"
+#include "opal/util/minmax.h"
+
+#include "coll_xhc.h"
+
+// ------------------------------------------------
+
+static int xhc_alloc_bcast_cico(xhc_module_t *module,
+    ompi_communicator_t *comm);
+
+static int xhc_print_config_info(xhc_module_t *module,
+    ompi_communicator_t *comm);
+static int xhc_print_op_info(xhc_module_t *module,
+    ompi_communicator_t *comm, XHC_COLLTYPE_T colltype);
+static int xhc_print_op_hierarchy_dot(xhc_module_t *module,
+    ompi_communicator_t *comm, XHC_COLLTYPE_T colltype);
+
+static mca_smsc_endpoint_t *xhc_smsc_ep(xhc_peer_info_t *peer_info);
+
+// ------------------------------------------------
+
+int mca_coll_xhc_lazy_init(xhc_module_t *module, ompi_communicator_t *comm) {
+    xhc_peer_info_t *peer_info = NULL;
+
+    int comm_size = ompi_comm_size(comm);
+    int rank = ompi_comm_rank(comm);
+
+    int err, return_code = OMPI_SUCCESS;
+
+    errno = 0;
+
+    if(module->init) {
+        return OMPI_SUCCESS;
+    } else if(module->error) {
+        return OMPI_ERROR;
+    }
+
+    // ---
+
+    peer_info = calloc(comm_size, sizeof(xhc_peer_info_t));
+    if(!peer_info) {RETURN_WITH_ERROR(return_code, OMPI_ERR_OUT_OF_RESOURCE, end);}
+
+    for(int r = 0; r < comm_size; r++) {
+        peer_info[r].proc = ompi_comm_peer_lookup(comm, r);
+        peer_info[r].locality = peer_info[r].proc->super.proc_flags;
+    }
+
+    peer_info[rank].locality |= ((1 << XHC_LOC_EXT_BITS) - 1) << XHC_LOC_EXT_START;
+
+    // ---
+
+    OBJ_CONSTRUCT(&module->hierarchy_cache, opal_hash_table_t);
+    err = opal_hash_table_init(&module->hierarchy_cache, XHC_COLLCOUNT);
+    if(OPAL_SUCCESS != err) {RETURN_WITH_ERROR(return_code, err, end);}
+
+    // ---
+
+    module->comm_size = comm_size;
+    module->rank = rank;
+
+    module->peer_info = peer_info;
+    module->init = true;
+
+    if((mca_coll_xhc_component.print_info & XHC_PRINT_INFO_CONFIG) && 0 == rank) {
+        err = xhc_print_config_info(module, comm);
+
+        if(OMPI_SUCCESS != err) {
+            opal_output_verbose(MCA_BASE_VERBOSE_WARN,
+                ompi_coll_base_framework.framework_output,
+                "coll:xhc: Warning: xhc_print_config_info() "
+                "failed with error code %d", err);
+        }
+    }
+
+    // ---
+
+    end:
+
+    if(OMPI_SUCCESS != return_code) {
+        free(peer_info);
+
+        opal_show_help("help-coll-xhc.txt", "xhc-init-failed", true,
+            return_code, errno, strerror(errno));
+
+        module->error = true;
+    }
+
+    return return_code;
+}
+
+int mca_coll_xhc_init_op(xhc_module_t *module,
+        ompi_communicator_t *comm, XHC_COLLTYPE_T colltype) {
+
+    int err, return_code = OMPI_SUCCESS;
+
+    if(!module->init && !module->error) {
+        err = xhc_lazy_init(module, comm);
+        if(OMPI_SUCCESS != err) {return err;}
+    }
+
+    if(module->op_data[colltype].init) {
+        return OMPI_SUCCESS;
+    }
+
+    // ---
+
+    xhc_op_config_t *config = &module->op_config[colltype];
+    xhc_op_data_t *data = &module->op_data[colltype];
+
+    data->colltype = colltype;
+    data->seq = 0;
+
+    if(XHC_BCAST == colltype) {
+        err = xhc_alloc_bcast_cico(module, comm);
+        if(OMPI_SUCCESS != err) {RETURN_WITH_ERROR(return_code, err, end);}
+    }
+
+    // ---
+
+    /* Check if a hierarchy for this hierarchy string has already been
+     * prepared, and use (borrow) it instead of re-creating it. */
+
+    xhc_op_config_t *lender;
+
+    err = opal_hash_table_get_value_ptr(&module->hierarchy_cache,
+        config->hierarchy_string, strlen(config->hierarchy_string),
+        (void **) &lender);
+
+    if(OPAL_SUCCESS == err) {
+        config->hierarchy = lender->hierarchy;
+        config->hierarchy_len = lender->hierarchy_len;
+    } else {
+        if(OPAL_ERR_NOT_FOUND != err) {
+            opal_output_verbose(MCA_BASE_VERBOSE_WARN,
+                ompi_coll_base_framework.framework_output,
+                "coll:xhc: Warning: opal_hash_table_get_value_ptr() "
+                "returned %d", err);
+        }
+
+        err = xhc_hierarchy_make(module, comm, config->hierarchy_string,
+            &config->hierarchy, &config->hierarchy_len);
+        if(OMPI_SUCCESS != err) {RETURN_WITH_ERROR(return_code, err, end);}
+
+        err = opal_hash_table_set_value_ptr(&module->hierarchy_cache,
+            config->hierarchy_string, strlen(config->hierarchy_string),
+            (void *) config);
+
+        if(OPAL_SUCCESS != err) {
+            opal_output_verbose(MCA_BASE_VERBOSE_WARN,
+                ompi_coll_base_framework.framework_output,
+                "coll:xhc: Warning: opal_hash_table_set_value_ptr() "
+                "returned %d", err);
+        }
+    }
+
+    // ---
+
+    err = xhc_comms_make(comm, module, config, data);
+    if(OMPI_SUCCESS != err) {RETURN_WITH_ERROR(return_code, err, end);}
+
+    data->init = true;
+
+    if(mca_coll_xhc_component.print_info & (1 << colltype)) {
+        err = xhc_print_op_info(module, comm, colltype);
+
+        if(OMPI_SUCCESS != err) {
+            opal_output_verbose(MCA_BASE_VERBOSE_WARN,
+                ompi_coll_base_framework.framework_output,
+                "coll:xhc: Warning: xhc_print_op_info() "
+                "failed with error code %d", err);
+        }
+
+        if(mca_coll_xhc_component.print_info & XHC_PRINT_INFO_HIER_DOT) {
+            err = xhc_print_op_hierarchy_dot(module, comm, colltype);
+
+            if(OMPI_SUCCESS != err) {
+                opal_output_verbose(MCA_BASE_VERBOSE_WARN,
+                    ompi_coll_base_framework.framework_output,
+                    "coll:xhc: Warning: xhc_print_op_hierarchy_dot() "
+                    "failed with error code %d", err);
+            }
+        }
+    }
+
+    // ---
+
+    end:
+
+    if(OMPI_SUCCESS != return_code) {
+        opal_show_help("help-coll-xhc.txt", "xhc-op-init-failed", true,
+            xhc_colltype_to_str(colltype), return_code, errno, strerror(errno));
+    }
+
+    return return_code;
+}
+
+static int xhc_alloc_bcast_cico(xhc_module_t *module, ompi_communicator_t *comm) {
+    opal_shmem_ds_t *ds_list = NULL;
+    opal_shmem_ds_t cico_ds;
+    void *cico_buffer = NULL;
+    xhc_coll_fns_t xhc_fns;
+
+    int err, return_code = OMPI_SUCCESS;
+
+    int comm_size = ompi_comm_size(comm);
+    int rank = ompi_comm_rank(comm);
+
+    size_t cico_size = module->op_config[XHC_BCAST].cico_max;
+
+    if(0 == cico_size) {
+        return OMPI_SUCCESS;
+    }
+
+    xhc_module_set_coll_fns(comm, &module->prev_colls, &xhc_fns);
+
+    // --
+
+    ds_list = malloc(comm_size * sizeof(opal_shmem_ds_t));
+    if(!ds_list) {return OMPI_ERR_OUT_OF_RESOURCE;}
+
+    cico_buffer = xhc_shmem_create(&cico_ds, cico_size, comm, "cico", 0, 0);
+    if(!cico_buffer) {RETURN_WITH_ERROR(return_code, OMPI_ERR_OUT_OF_RESOURCE, end);}
+
+    /* Manually "touch" to assert allocation in local NUMA node
+     * (assuming linux's default first-touch-alloc policy). */
+    memset(cico_buffer, 0, cico_size);
+
+    err = comm->c_coll->coll_allgather(&cico_ds, sizeof(opal_shmem_ds_t),
+        MPI_BYTE, ds_list, sizeof(opal_shmem_ds_t), MPI_BYTE, comm,
+        comm->c_coll->coll_allgather_module);
+    if(OMPI_SUCCESS != err) {RETURN_WITH_ERROR(return_code, err, end);}
+
+    module->peer_info[rank].cico_ds = cico_ds;
+    module->peer_info[rank].cico_buffer = cico_buffer;
+
+    for(int r = 0; r < comm_size; r++) {
+        if(r == rank) {continue;}
+        module->peer_info[r].cico_ds = ds_list[r];
+    }
+
+    // --
+
+    end:
+
+    free(ds_list);
+    xhc_module_set_coll_fns(comm, &xhc_fns, NULL);
+
+    if(OMPI_SUCCESS != return_code) {
+        if(cico_buffer) {
+            opal_shmem_unlink(&cico_ds);
+            opal_shmem_segment_detach(&cico_ds);
+        }
+    }
+
+    return return_code;
+}
+
+void mca_coll_xhc_fini(mca_coll_xhc_module_t *module) {
+    if(module->peer_info) {
+        for(int r = 0; r < module->comm_size; r++) {
+            if(module->peer_info[r].smsc_ep) {
+                MCA_SMSC_CALL(return_endpoint, module->peer_info[r].smsc_ep);
+            }
+
+            if(module->peer_info[r].cico_buffer) {
+                // OMPI #11123
+                /* if(r == module->rank) {
+                    opal_shmem_unlink(&module->peer_info[r].cico_ds);
+                } */
+
+                opal_shmem_segment_detach(&module->peer_info[r].cico_ds);
+            }
+        }
+
+        free(module->peer_info);
+    }
+
+    xhc_op_config_t *ht_config_value;
+    void *ht_key;
+
+    OPAL_HASH_TABLE_FOREACH_PTR(ht_key, ht_config_value,
+            &module->hierarchy_cache, {
+        free(ht_config_value->hierarchy);
+    });
+
+    OBJ_DESTRUCT(&module->hierarchy_cache);
+
+    for(int t = 0; t < XHC_COLLCOUNT; t++) {
+        if(module->op_data[t].init) {
+            xhc_comms_destroy(module->op_data[t].comms,
+                module->op_data[t].comm_count);
+
+            free(module->op_data[t].comms);
+        }
+    }
+
+    free(module->rbuf);
+}
+
+// ------------------------------------------------
+
+int mca_coll_xhc_read_op_config(xhc_module_t *module,
+        ompi_communicator_t *comm, XHC_COLLTYPE_T colltype) {
+
+    /* Assimilate the tuning parameters here, based on the various MCA params
+     * and any provided info keys. We want to determine the hierarchy, chunk
+     * size, and cico threshold, for each supported primitive. The info keys
+     * take precedence over the MCA params, with op-specific info keys having
+     * higher priority than global ones. Note that the op-specific and global
+     * MCA params are mutually exclusive; this is implemented in xhc_register().
+     */
+
+    xhc_op_config_t *config = &module->op_config[colltype];
+
+    const char *param[] = {"hierarchy", "chunk_size", "cico_max"};
+    for(size_t p = 0; p < sizeof(param)/sizeof(param[0]); p++) {
+        xhc_op_mca_t op_mca = mca_coll_xhc_component.op_mca[colltype];
+        xhc_config_source_t config_source;
+
+        opal_cstring_t *info_val;
+        int info_flag = 0;
+
+        int err;
+
+        if(comm->super.s_info) {
+            char *key;
+
+            // Op-specific info key
+            err = opal_asprintf(&key, "ompi_comm_coll_xhc_%s_%s",
+                xhc_colltype_to_str(colltype), param[p]);
+            if(err < 0) {return OMPI_ERR_OUT_OF_RESOURCE;}
+
+            opal_info_get(comm->super.s_info,
+                key, &info_val, &info_flag);
+            free(key);
+
+            if(info_flag) {
+                config_source = XHC_CONFIG_SOURCE_INFO_OP;
+            } else {
+                // Non-specific info key
+                err = opal_asprintf(&key, "ompi_comm_coll_xhc_%s",
+                    param[p]);
+                if(err < 0) {return OMPI_ERR_OUT_OF_RESOURCE;}
+
+                opal_info_get(comm->super.s_info, key,
+                    &info_val, &info_flag);
+                free(key);
+
+                if(info_flag) {
+                    config_source = XHC_CONFIG_SOURCE_INFO_GLOBAL;
+                }
+            }
+        }
+
+        if(!info_flag) {
+            const mca_base_var_t *var_global;
+
+            err = mca_base_var_get(mca_base_var_find("ompi",
+                "coll", "xhc", param[p]), &var_global);
+            if(OPAL_SUCCESS != err) {return err;}
+
+            config_source = (MCA_BASE_VAR_SOURCE_DEFAULT != var_global->mbv_source
+                ? XHC_CONFIG_SOURCE_MCA_GLOBAL : XHC_CONFIG_SOURCE_MCA_OP);
+        }
+
+        switch(p) {
+            case 0:
+                config->hierarchy_string = strdup(info_flag ?
+                    info_val->string : op_mca.hierarchy);
+
+                if(info_flag) {
+                    OBJ_RELEASE(info_val);
+                }
+
+                if(!config->hierarchy_string) {
+                    return OMPI_ERR_OUT_OF_RESOURCE;
+                }
+
+                config->hierarchy_source = config_source;
+
+                break;
+            case 1:
+                /* We don't *need* to keep record of the chunk string, since
+                 * we parse it here. But it's good to have; we currently use
+                 * it when showing information (print_info). */
+                config->chunk_string = strdup(info_flag ?
+                    info_val->string : op_mca.chunk_size);
+
+                if(info_flag) {
+                    OBJ_RELEASE(info_val);
+                }
+
+                if(!config->chunk_string) {
+                    return OMPI_ERR_OUT_OF_RESOURCE;
+                }
+
+                err = xhc_component_parse_chunk_sizes(config->chunk_string,
+                    &config->chunks, &config->chunks_len);
+
+                if(OMPI_SUCCESS != err) {
+                    return err;
+                }
+
+                config->chunk_source = config_source;
+
+                break;
+            case 2:
+                if(info_flag) {
+                    err = xhc_component_parse_cico_max(
+                        info_val->string, &config->cico_max);
+                    OBJ_RELEASE(info_val);
+
+                    if(OMPI_SUCCESS != err) {
+                        return err;
+                    }
+                } else {
+                    config->cico_max = op_mca.cico_max;
+                }
+
+                config->cico_max_source = config_source;
+
+                break;
+        }
+    }
+
+    // ---
+
+    /* Enforce a resonable minimum chunk size */
+
+    if(XHC_BARRIER != colltype) {
+        bool altered_chunks = false;
+        for(int i = 0; i < config->chunks_len; i++) {
+            if(config->chunks[i] < XHC_MIN_CHUNK_SIZE) {
+                config->chunks[i] = XHC_MIN_CHUNK_SIZE;
+                altered_chunks = true;
+            }
+        }
+
+        if(altered_chunks) {
+            char *tmp, *str = strdup("");
+            for(int i = 0; i < config->chunks_len; i++) {
+                int err = opal_asprintf(&tmp, "%s%s%zu", str,
+                    (i > 0 ? "," : ""), config->chunks[i]);
+                free(str); str = tmp;
+                if(err < 0) {return OMPI_ERR_OUT_OF_RESOURCE;}
+            }
+
+            free(config->chunk_string);
+            config->chunk_string = str;
+        }
+    }
+
+    // ---
+
+    return OMPI_SUCCESS;
+}
+
+// ------------------------------------------------
+
+static int xhc_print_config_info(xhc_module_t *module, ompi_communicator_t *comm) {
+    char *drval_str, *lb_policy_str, *un_min_str;
+    int err;
+
+    switch(mca_coll_xhc_component.dynamic_reduce) {
+        case XHC_DYNAMIC_REDUCE_DISABLED:
+            drval_str = "OFF"; break;
+        case XHC_DYNAMIC_REDUCE_NON_FLOAT:
+            drval_str = "ON (non-float)"; break;
+        case XHC_DYNAMIC_REDUCE_ALL:
+            drval_str = "ON (all)"; break;
+        default:
+            drval_str = "???";
+    }
+
+    switch((int) mca_coll_xhc_component.reduce_load_balance) {
+        case 0:
+            lb_policy_str = "none"; break;
+        case XHC_REDUCE_LB_LEADER_ASSIST_TOP_LEVEL:
+            lb_policy_str = "top level"; break;
+        case XHC_REDUCE_LB_LEADER_ASSIST_FIRST_CHUNK:
+            lb_policy_str = "first chunk"; break;
+        case XHC_REDUCE_LB_LEADER_ASSIST_TOP_LEVEL
+                | XHC_REDUCE_LB_LEADER_ASSIST_FIRST_CHUNK:
+            lb_policy_str = "top level + first chunk"; break;
+        case XHC_REDUCE_LB_LEADER_ASSIST_ALL:
+            lb_policy_str = "all"; break;
+        default:
+            lb_policy_str = "???";
+    }
+
+    err = opal_asprintf(&un_min_str, " (min %zu bytes)",
+        mca_coll_xhc_component.uniform_chunks_min);
+    if(err < 0) {return OMPI_ERR_OUT_OF_RESOURCE;}
+
+    printf("------------------------------------------------\n"
+        "OMPI coll/xhc @ %s, priority %d\n"
+        "  dynamic leader '%s', dynamic reduce '%s'\n"
+        "  reduce load balance '%s'\n"
+        "  allreduce uniform chunks '%s'%s\n",
+        comm->c_name, mca_coll_xhc_component.priority,
+        (mca_coll_xhc_component.dynamic_leader ? "ON" : "OFF"),
+        drval_str, lb_policy_str,
+        (mca_coll_xhc_component.uniform_chunks ? "ON" : "OFF"),
+        (mca_coll_xhc_component.uniform_chunks ? un_min_str : ""));
+
+    free(un_min_str);
+
+    for(int t = 0; t < XHC_COLLCOUNT; t++) {
+        xhc_op_config_t *config = &module->op_config[t];
+
+        if(XHC_BARRIER == t) {
+            printf("\n"
+                "  [%s]\n"
+                "    Hierarchy: %s (source: %s)\n",
+                xhc_colltype_to_str(t),
+                config->hierarchy_string, xhc_config_source_to_str(config->hierarchy_source));
+        } else {
+            printf("\n"
+                "  [%s]\n"
+                "    Hierarchy: %s (source: %s)\n"
+                "    Chunk size(s): %s (source: %s)\n"
+                "    CICO: Up to %zu bytes (source: %s)\n",
+                xhc_colltype_to_str(t),
+                config->hierarchy_string, xhc_config_source_to_str(config->hierarchy_source),
+                config->chunk_string, xhc_config_source_to_str(config->chunk_source),
+                config->cico_max, xhc_config_source_to_str(config->cico_max_source));
+        }
+    }
+
+    printf("------------------------------------------------\n");
+
+    return OMPI_SUCCESS;
+}
+
+static int xhc_print_op_info(xhc_module_t *module,
+        ompi_communicator_t *comm, XHC_COLLTYPE_T colltype) {
+
+    xhc_comm_t *comms = module->op_data[colltype].comms;
+    int comm_count = module->op_data[colltype].comm_count;
+
+    int rank = ompi_comm_rank(comm);
+
+    for(int i = 0; i < comm_count; i++) {
+        char *memb_list = NULL, *tmp;
+        int err;
+
+        err = opal_asprintf(&memb_list, "%d", comms[i].owner_rank);
+        if(err < 0) {return OMPI_ERR_OUT_OF_RESOURCE;}
+
+        for(int m = 1; m < comms[i].size; m++) {
+            if(m == comms[i].my_id) {
+                if(0 == i || comms[i-1].owner_rank == rank) {
+                    err = opal_asprintf(&tmp, "%s %d", memb_list, rank);
+                } else {
+                    err = opal_asprintf(&tmp, "%s _", memb_list);
+                }
+            } else {
+                err = opal_asprintf(&tmp, "%s x", memb_list);
+            }
+
+            free(memb_list);
+            memb_list = tmp;
+
+            if(err < 0) {
+                return OMPI_ERR_OUT_OF_RESOURCE;
+            }
+        }
+
+        if(XHC_BARRIER == colltype) {
+            printf("XHC_COMM ompi_comm=%s rank=%d op=%s loc=0x%08x members=%d [%s]\n",
+                comm->c_name, rank, xhc_colltype_to_str(colltype), comms[i].locality,
+                comms[i].size, memb_list);
+        } else {
+            printf("XHC_COMM ompi_comm=%s rank=%d op=%s loc=0x%08x chunk_size=%zu "
+                "cico_size=%zu members=%d [%s]\n", comm->c_name, rank,
+                xhc_colltype_to_str(colltype), comms[i].locality, comms[i].chunk_size,
+                comms[i].cico_size, comms[i].size, memb_list);
+        }
+
+        free(memb_list);
+    }
+
+    return OMPI_SUCCESS;
+}
+
+static int xhc_print_op_hierarchy_dot(xhc_module_t *module,
+        ompi_communicator_t *comm, XHC_COLLTYPE_T colltype) {
+
+    xhc_coll_fns_t xhc_fns;
+    xhc_comm_t *comms = module->op_data[colltype].comms;
+
+    int *src_rank = NULL;
+    int my_src = -1;
+
+    FILE *outfile = stdout;
+
+    if(0 == module->rank) {
+        src_rank = malloc(module->comm_size * sizeof(int));
+        if(!src_rank) {return OMPI_ERR_OUT_OF_RESOURCE;}
+    }
+
+    for(xhc_comm_t *xc = comms; xc; xc = xc->up) {
+        if(0 != xc->my_id) {
+            my_src = xc->owner_rank;
+            break;
+        }
+    }
+
+    xhc_module_set_coll_fns(comm, &module->prev_colls, &xhc_fns);
+
+    int err = comm->c_coll->coll_gather(&my_src,
+        1, MPI_INT, src_rank, 1, MPI_INT, 0,
+        comm, comm->c_coll->coll_gather_module);
+
+    xhc_module_set_coll_fns(comm, &xhc_fns, NULL);
+
+    if(OMPI_SUCCESS != err) {
+        if(0 == module->rank) {
+            free(src_rank);
+        }
+        return err;
+    }
+
+    if(0 == module->rank) {
+        char *dir;
+
+        switch(colltype) {
+            case XHC_BCAST:
+                dir = "forward"; break;
+            case XHC_REDUCE: case XHC_ALLREDUCE:
+                dir = "back"; break;
+            case XHC_BARRIER:
+                dir = "both"; break;
+            default:
+                dir = "none";
+        }
+
+        fprintf(outfile, "digraph xhc_%s_hierarchy {\n", xhc_colltype_to_str(colltype));
+
+        for(int r = 1; r < module->comm_size; r++) {
+            fprintf(outfile, "\t%d -> %d [dir=%s];\n", src_rank[r], r, dir);
+        }
+
+        fprintf(outfile, "}\n");
+
+        free(src_rank);
+    }
+
+    return OMPI_SUCCESS;
+}
+
+// ------------------------------------------------
+
+void *mca_coll_xhc_shmem_create(opal_shmem_ds_t *seg_ds, size_t size,
+        ompi_communicator_t *ompi_comm, const char *name, int id1, int id2) {
+
+    char *shmem_file;
+    int err;
+
+    // xhc_shmem_seg.<UID>@<HOST>.<JOBID>.<RANK@COMM_WORLD>:<CID>_<NAME>:<ID1>:<ID2>
+
+    err = opal_asprintf(&shmem_file, "%s" OPAL_PATH_SEP
+        "xhc_shmem_seg.%u@%s.%x.%d:%d_%s:%d:%d", mca_coll_xhc_component.shmem_backing,
+        geteuid(), opal_process_info.nodename, OPAL_PROC_MY_NAME.jobid,
+        ompi_comm_rank(MPI_COMM_WORLD), ompi_comm_get_local_cid(ompi_comm),
+        name, id1, id2);
+
+    if(err < 0) {
+        return NULL;
+    }
+
+    // Not 100% sure what this does!, copied from btl/sm
+    opal_pmix_register_cleanup(shmem_file, false, false, false);
+
+    err = opal_shmem_segment_create(seg_ds, shmem_file, size);
+
+    free(shmem_file);
+
+    if(OPAL_SUCCESS != err) {
+        opal_output_verbose(MCA_BASE_VERBOSE_ERROR,
+            ompi_coll_base_framework.framework_output,
+            "coll:xhc: Error: Could not create shared memory segment");
+
+        return NULL;
+    }
+
+    void *addr = xhc_shmem_attach(seg_ds);
+
+    if(NULL == addr) {
+        opal_shmem_unlink(seg_ds);
+    }
+
+    return addr;
+}
+
+void *mca_coll_xhc_shmem_attach(opal_shmem_ds_t *seg_ds) {
+    void *addr = opal_shmem_segment_attach(seg_ds);
+
+    if(NULL == addr) {
+        opal_output_verbose(MCA_BASE_VERBOSE_ERROR,
+            ompi_coll_base_framework.framework_output,
+            "coll:xhc: Error: Could not attach to shared memory segment");
+    }
+
+    return addr;
+}
+
+// ------------------------------------------------
+
+void *mca_coll_xhc_get_cico(xhc_peer_info_t *peer_info, int rank) {
+    if(NULL == peer_info[rank].cico_buffer) {
+        peer_info[rank].cico_buffer = xhc_shmem_attach(&peer_info[rank].cico_ds);
+    }
+
+    return peer_info[rank].cico_buffer;
+}
+
+static mca_smsc_endpoint_t *xhc_smsc_ep(xhc_peer_info_t *peer_info) {
+    if(!peer_info->smsc_ep) {
+        peer_info->smsc_ep = MCA_SMSC_CALL(get_endpoint, &peer_info->proc->super);
+
+        if(!peer_info->smsc_ep) {
+            opal_output_verbose(MCA_BASE_VERBOSE_ERROR,
+                ompi_coll_base_framework.framework_output,
+                "coll:xhc: Error: Failed to initialize smsc endpoint");
+
+            return NULL;
+        }
+    }
+
+    return peer_info->smsc_ep;
+}
+
+int mca_coll_xhc_copy_expose_region(void *base,
+        size_t len, xhc_copy_data_t **region_data) {
+
+    if(mca_smsc_base_has_feature(MCA_SMSC_FEATURE_REQUIRE_REGISTRATION)) {
+        void *data = MCA_SMSC_CALL(register_region, base, len);
+
+        if(NULL == data) {
+            opal_output_verbose(MCA_BASE_VERBOSE_ERROR,
+                ompi_coll_base_framework.framework_output,
+                "coll:xhc: Error: Failed to register memory region with smsc");
+
+            return -1;
+        }
+
+        *region_data = data;
+    }
+
+    return 0;
+}
+
+void mca_coll_xhc_copy_region_post(void *dst, xhc_copy_data_t *region_data) {
+    memcpy(dst, region_data, mca_smsc_base_registration_data_size());
+}
+
+int mca_coll_xhc_copy_from(xhc_peer_info_t *peer_info,
+        void *dst, void *src, size_t size, void *access_token) {
+
+    mca_smsc_endpoint_t *smsc_ep = xhc_smsc_ep(peer_info);
+
+    if(NULL == smsc_ep) {
+        return -1;
+    }
+
+    int status = MCA_SMSC_CALL(copy_from, smsc_ep,
+        dst, src, size, access_token);
+
+    return (OPAL_SUCCESS == status ? 0 : -1);
+}
+
+void mca_coll_xhc_copy_close_region(xhc_copy_data_t *region_data) {
+    if(mca_smsc_base_has_feature(MCA_SMSC_FEATURE_REQUIRE_REGISTRATION)) {
+        MCA_SMSC_CALL(deregister_region, region_data);
+    }
+}
+
+void *mca_coll_xhc_get_registration(xhc_peer_info_t *peer_info,
+        void *peer_vaddr, size_t size, xhc_reg_t **reg) {
+
+    mca_smsc_endpoint_t *smsc_ep = xhc_smsc_ep(peer_info);
+
+    if(NULL == smsc_ep) {
+        return NULL;
+    }
+
+    void *local_ptr;
+
+    *reg = MCA_SMSC_CALL(map_peer_region, smsc_ep,
+        MCA_RCACHE_FLAGS_PERSIST, peer_vaddr, size, &local_ptr);
+
+    if(NULL == *reg) {
+        return NULL;
+    }
+
+    return local_ptr;
+}
+
+void mca_coll_xhc_return_registration(xhc_reg_t *reg) {
+    /* Won't actually unmap/detach, since we've set the
+     * MCA_RCACHE_FLAGS_PERSIST flag to map_peer_region */
+    MCA_SMSC_CALL(unmap_peer_region, reg);
+}
diff --git a/ompi/mca/coll/xhc/coll_xhc.h b/ompi/mca/coll/xhc/coll_xhc.h
new file mode 100644
index 00000000000..1a5b12d259a
--- /dev/null
+++ b/ompi/mca/coll/xhc/coll_xhc.h
@@ -0,0 +1,739 @@
+/*
+ * Copyright (c) 2021-2024 Computer Architecture and VLSI Systems (CARV)
+ *                         Laboratory, ICS Forth. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef MCA_COLL_XHC_EXPORT_H
+#define MCA_COLL_XHC_EXPORT_H
+
+#include "ompi_config.h"
+
+#include <stdint.h>
+#include <limits.h>
+
+#include "mpi.h"
+
+#include "ompi/mca/mca.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/coll/base/base.h"
+#include "ompi/mca/coll/base/coll_base_functions.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+
+#include "ompi/communicator/communicator.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/op/op.h"
+
+#include "opal/class/opal_hash_table.h"
+#include "opal/mca/shmem/shmem.h"
+#include "opal/mca/smsc/smsc.h"
+#include "opal/util/minmax.h"
+
+#include "coll_xhc_intrinsic.h"
+
+#define RETURN_WITH_ERROR(var, err, label) do {(var) = (err); goto label;} \
+    while(0)
+
+#define OBJ_RELEASE_IF_NOT_NULL(obj) do {if(NULL != (obj)) {OBJ_RELEASE(obj);}} while(0)
+
+#define REALLOC(p, s, t) do {void *_tmp = realloc(p, (s)*sizeof(t)); \
+    if(_tmp) {(p) = _tmp;}} while(0)
+
+#define PEER_IS_LOCAL(peer_info, rank, loc) \
+    (((peer_info)[(rank)].locality & (loc)) == (loc))
+
+#define XHC_LOC_EXT_BITS (8*(sizeof(xhc_loc_t) - sizeof(opal_hwloc_locality_t)))
+#define XHC_LOC_EXT_START (8*sizeof(opal_hwloc_locality_t))
+
+#define XHC_CALL_FALLBACK(fns, colltype, op, ...) \
+    ((mca_coll_base_module_ ## op ## _fn_t) (fns).coll_fn[colltype]) \
+    (__VA_ARGS__, (mca_coll_base_module_t *) (fns).coll_module[colltype])
+
+// Change the pointers so that the fallback is always called in the future
+#define XHC_INSTALL_FALLBACK(module, comm, colltype, op) do { \
+    (comm)->c_coll->coll_ ## op = (mca_coll_base_module_ ## op ## _fn_t) \
+        (module)->prev_colls.coll_fn[colltype]; \
+    (comm)->c_coll->coll_ ## op ## _module = (mca_coll_base_module_t *) \
+        (module)->prev_colls.coll_module[colltype]; \
+} while(0)
+
+#define WARN_ONCE(...) do { \
+    static bool warn_shown = false; \
+    if(!warn_shown) { \
+        opal_output_verbose(MCA_BASE_VERBOSE_WARN, \
+            ompi_coll_base_framework.framework_output, __VA_ARGS__); \
+        warn_shown = true; \
+    } \
+} while(0)
+
+// ---
+
+// Set according to CPU cache line (safe bet)
+#define XHC_ALIGN 128
+
+// Chunk size can't be set lower than this
+#define XHC_MIN_CHUNK_SIZE 64
+
+// Call opal_progress every this many ticks when busy-waiting
+#define XHC_OPAL_PROGRESS_CYCLE 10000
+
+#define XHC_PRINT_INFO_HIER_DOT (1 << (sizeof(mca_coll_xhc_component.print_info) * 8 - 4))
+#define XHC_PRINT_INFO_CONFIG (1 << (sizeof(mca_coll_xhc_component.print_info) * 8 - 3))
+#define XHC_PRINT_INFO_ALL (1 << (sizeof(mca_coll_xhc_component.print_info) * 8 - 2))
+
+/* While we treat the cache line as 128 bytes (XHC_ALIGN), we calculate
+ * IMM_SIZE according to conservative 64 bytes, to ensure the immediate data
+ * is always inside one cache line. Though, it might also be okay if it also
+ * occupied the next cache line, thanks to the L2 adjacent line prefetcher.
+ * The results were mixed and thus we chose the simplest of the two options. */
+#define XHC_BCAST_IMM_SIZE (64 - (offsetof(xhc_comm_ctrl_t, \
+    imm_data) - offsetof(xhc_comm_ctrl_t, seq)))
+#define XHC_REDUCE_IMM_SIZE (64 - (offsetof(xhc_member_ctrl_t, \
+    imm_data) - offsetof(xhc_member_ctrl_t, seq)))
+
+// ---
+
+BEGIN_C_DECLS
+
+// ----------------------------------------
+
+typedef uint32_t xhc_loc_t;
+typedef void xhc_reg_t;
+typedef void xhc_copy_data_t;
+
+typedef struct xhc_hierarchy_t xhc_hierarchy_t;
+
+typedef struct mca_coll_xhc_component_t mca_coll_xhc_component_t;
+typedef struct mca_coll_xhc_module_t mca_coll_xhc_module_t;
+typedef struct mca_coll_xhc_module_t xhc_module_t;
+
+typedef struct xhc_coll_fns_t xhc_coll_fns_t;
+typedef struct xhc_peer_info_t xhc_peer_info_t;
+
+typedef struct xhc_op_mca_t xhc_op_mca_t;
+typedef struct xhc_op_config_t xhc_op_config_t;
+typedef struct xhc_op_data_t xhc_op_data_t;
+
+typedef struct xhc_comm_t xhc_comm_t;
+typedef struct xhc_comm_ctrl_t xhc_comm_ctrl_t;
+typedef struct xhc_member_ctrl_t xhc_member_ctrl_t;
+typedef struct xhc_member_info_t xhc_member_info_t;
+
+typedef struct xhc_sh_slice_t xhc_sh_slice_t;
+typedef struct xhc_reduce_queue_item_t xhc_rq_item_t;
+typedef struct xhc_reduce_area_t xhc_reduce_area_t;
+
+typedef struct xhc_rank_range_t xhc_rank_range_t;
+typedef struct xhc_loc_def_t xhc_loc_def_t;
+
+OMPI_DECLSPEC extern mca_coll_xhc_component_t mca_coll_xhc_component;
+OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_coll_xhc_module_t);
+OMPI_DECLSPEC OBJ_CLASS_DECLARATION(xhc_rq_item_t);
+OMPI_DECLSPEC OBJ_CLASS_DECLARATION(xhc_loc_def_item_t);
+
+// ----------------------------------------
+
+typedef enum xhc_dynamic_reduce_enum_t {
+    XHC_DYNAMIC_REDUCE_DISABLED = 0,
+    XHC_DYNAMIC_REDUCE_NON_FLOAT,
+    XHC_DYNAMIC_REDUCE_ALL
+} xhc_dynamic_reduce_enum_t;
+
+/* Reduce load balancing, AKA what should leaders do? Leaders propagate the
+ * group's results, and perform reductions on upper level(s). Thus we generally
+ * don't want them to not do reductions inside the group, so they can focus on
+ * the upper ones. However, in certain instances, like at the start of the op
+ * (when no chunks are ready to be reduced on the upper levels), or on the top
+ * level, the leaders do not actually have any other work to do, so might as
+ * well enlist the for the intra-group reductions.
+ *
+ * - ASSIST_TOP_LEVEL: The top level's leader performs reductions
+ *   (on the top level), as if a common member.
+ * - ASSIST_FIRST_CHUNK: Leaders reduce only a single chunk,
+ *   on each level, at the beginning of the operation.
+ * - ASSIST_ALL: All leaders performs reductions exactly as if common members
+ *
+ * (ASSIST_TOP_LEVEL and ASSIST_FIRST_CHUNK are combinable) */
+typedef enum xhc_reduce_load_balance_enum_t {
+    XHC_REDUCE_LB_LEADER_ASSIST_TOP_LEVEL = 0x01,
+    XHC_REDUCE_LB_LEADER_ASSIST_FIRST_CHUNK = 0x02,
+    XHC_REDUCE_LB_LEADER_ASSIST_ALL = 0x80
+} xhc_reduce_load_balance_enum_t;
+
+/* Changes to XHC_COLLTYPE_T have to be reflected in:
+ * 1. xhc_colltype_to_universal_map[]
+ * 2. xhc_colltype_to_c_coll_fn_offset_map[]
+ * 3. xhc_colltype_to_c_coll_module_offset_map[]
+ * 4. xhc_colltype_to_coll_base_fn_offset_map[] */
+typedef enum XHC_COLLTYPE_T {
+    XHC_BCAST = 0,
+    XHC_BARRIER,
+    XHC_REDUCE,
+    XHC_ALLREDUCE,
+
+    XHC_COLLCOUNT
+} XHC_COLLTYPE_T;
+
+typedef enum xhc_config_source_t {
+    XHC_CONFIG_SOURCE_INFO_GLOBAL = 0,
+    XHC_CONFIG_SOURCE_INFO_OP,
+    XHC_CONFIG_SOURCE_MCA_GLOBAL,
+    XHC_CONFIG_SOURCE_MCA_OP,
+
+    XHC_CONFIG_SOURCE_COUNT
+} xhc_config_source_t;
+
+typedef enum xhc_copy_method_t {
+    /* 0 is reserved */
+    XHC_COPY_IMM = 1,
+    XHC_COPY_CICO,
+    XHC_COPY_SMSC_NO_MAP,
+    XHC_COPY_SMSC_MAP,
+    XHC_COPY_SMSC, /* kind of a catch-all value for ops that don't
+                    * make a distinction between map/no_map */
+} xhc_copy_method_t;
+
+struct mca_coll_xhc_component_t {
+    mca_coll_base_component_t super;
+
+    int priority;
+    uint print_info;
+
+    char *shmem_backing;
+
+    size_t memcpy_chunk_size;
+
+    bool dynamic_leader;
+
+    int barrier_root;
+    int allreduce_root;
+
+    xhc_dynamic_reduce_enum_t dynamic_reduce;
+    xhc_reduce_load_balance_enum_t reduce_load_balance;
+
+    bool uniform_chunks;
+    size_t uniform_chunks_min;
+
+    struct xhc_op_mca_t {
+        char *hierarchy;
+        char *chunk_size;
+        size_t cico_max;
+    } op_mca[XHC_COLLCOUNT];
+
+    xhc_op_mca_t op_mca_global;
+};
+
+struct mca_coll_xhc_module_t {
+    mca_coll_base_module_t super;
+
+    /* pointers to functions/modules of
+     * previous coll components for fallback */
+    struct xhc_coll_fns_t {
+        void (*coll_fn[XHC_COLLCOUNT])(void);
+        void *coll_module[XHC_COLLCOUNT];
+    } prev_colls;
+
+    // copied from OMPI comm
+    int comm_size;
+    int rank;
+
+    // ---
+
+    bool zcopy_support;
+    bool zcopy_map_support;
+
+    // temporary (private) internal buffer, for methods like Reduce
+    void *rbuf;
+    size_t rbuf_size;
+
+    // book-keeping for info on other ranks
+    struct xhc_peer_info_t {
+        xhc_loc_t locality;
+        ompi_proc_t *proc;
+
+        mca_smsc_endpoint_t *smsc_ep;
+
+        opal_shmem_ds_t cico_ds;
+        void *cico_buffer;
+    } *peer_info;
+
+    // ---
+
+    opal_hash_table_t hierarchy_cache;
+
+    struct xhc_op_config_t {
+        char *hierarchy_string;
+        xhc_loc_t *hierarchy;
+        int hierarchy_len;
+
+        char *chunk_string;
+        size_t *chunks;
+        int chunks_len;
+
+        size_t cico_max;
+
+        xhc_config_source_t hierarchy_source;
+        xhc_config_source_t chunk_source;
+        xhc_config_source_t cico_max_source;
+    } op_config[XHC_COLLCOUNT];
+
+    struct xhc_op_data_t {
+        XHC_COLLTYPE_T colltype;
+        xf_sig_t seq;
+
+        xhc_comm_t *comms;
+        int comm_count;
+
+        bool init;
+    } op_data[XHC_COLLCOUNT];
+
+    bool init;
+    bool error;
+};
+
+// -----
+
+struct xhc_comm_t {
+    xhc_loc_t locality;
+
+    size_t chunk_size;
+    size_t cico_size;
+
+    int my_id;
+    int size;
+
+    int owner_rank;
+
+    // ---
+
+    /* Op state (any) */
+
+    // Am I a leader in the current collective?
+    bool is_leader;
+
+    // Op-specific state tracking
+    int op_state;
+
+    // Book-keeping for multi-sliced shared areas
+    struct xhc_sh_slice_t {
+        bool in_use; // slice currently in use
+
+        xf_sig_t seq; // if in use, the seq number of the op
+        size_t len; // if in use, the message size of the op
+        bool is_cico; // if in use, whether the op is CICO type
+    } *slices;
+    int n_slices;
+
+    int slice_id; // designated slice ID for this op
+    bool slice_ready; // is the designed slice ready to use?
+
+    // ---
+
+    /* Reduce op state */
+
+    int leader_id;
+    bool do_all_work;
+
+    // see init_reduce_areas()
+    struct xhc_reduce_area_t {
+        size_t start; // where the area begins
+        size_t len; // the size of the area
+        int workers; // how many processes perform reductions in the area
+        size_t stride; /* how much to advance inside the area after
+                        * each reduction, unused for non-combo areas */
+
+        // local process settings
+        size_t work_begin; // where to begin the first reduction from
+        size_t work_end; // up to where to reduce
+        size_t work_chunk; // how much to reduce each time
+        size_t work_leftover; /* assigned leftover elements to include as
+                               * part of the last reduction in the area */
+    } reduce_areas[3];
+    int n_reduce_areas;
+
+    // To keep track of reduce progress for different peers
+    opal_list_t *reduce_queue;
+
+    /* These are cached copies of the respective fields in my_ctrl.
+     * We want to minimize accesses to the shared variables. */
+    xf_size_t reduce_ready;
+    xf_size_t reduce_done;
+
+    struct xhc_member_info_t {
+        xhc_reg_t *sbuf_reg, *rbuf_reg;
+        void *sbuf, *rbuf;
+        bool attach, join;
+    } *member_info;
+
+    xhc_member_info_t *my_info; // = &member_info[my_id]
+
+    // ---
+
+    /* Shared structs */
+
+    xhc_comm_ctrl_t *comm_ctrl;
+
+    xhc_member_ctrl_t *member_ctrl;
+    xhc_member_ctrl_t *my_ctrl; // = &member_ctrl[my_id]
+
+    // Shared CICO buffer for sharing data to be reduced
+    volatile char *reduce_buffer;
+
+    /* Original addresses of these potentially multi-sliced fields, as
+     * the respective pointers above may be modified in per-op basis */
+    void *comm_ctrl_base;
+    void *member_ctrl_base;
+    void *reduce_buffer_base;
+
+    opal_shmem_ds_t comm_ds;
+
+    // ---
+
+    xhc_comm_t *up, *down;
+    xhc_comm_t *top, *bottom;
+    bool is_top, is_bottom;
+};
+
+// -----
+
+struct xhc_comm_ctrl_t {
+    // leader_seq, ack, seq in different cache lines
+    volatile xf_sig_t leader_seq;
+    volatile xf_sig_t ack __attribute__((aligned(XHC_ALIGN)));
+    volatile xf_sig_t seq __attribute__((aligned(XHC_ALIGN)));
+
+    // below fields in same cache line as seq
+
+    union {
+        struct {
+            volatile int leader_rank;
+
+            volatile xf_size_t bytes_ready;
+            void* volatile data_vaddr;
+        };
+
+        /* sized 4 here, but will actually use all that's left
+         * of the cache line. Keep it last in the struct. */
+        volatile char imm_data[4];
+    };
+
+    /* imm_data can overwrite access_token. That's
+     * okay, the two aren't used at the same time. */
+    volatile char access_token[];
+} __attribute__((aligned(XHC_ALIGN)));
+
+struct xhc_member_ctrl_t {
+    volatile xf_sig_t ack;
+    volatile xf_sig_t seq __attribute__((aligned(XHC_ALIGN)));
+
+    // below fields in same cache line as seq
+
+    volatile xf_size_t reduce_ready;
+    volatile xf_size_t reduce_done;
+
+    union {
+        struct {
+            void* volatile sbuf_vaddr;
+            void* volatile rbuf_vaddr;
+
+            volatile int rank;
+            volatile bool is_leader;
+        };
+
+        /* sized 4 here, but will actually use all that's left
+         * of the cache line. Keep it last in the struct. */
+        volatile char imm_data[4];
+    };
+} __attribute__((aligned(XHC_ALIGN)));
+
+// -----
+
+struct xhc_reduce_queue_item_t {
+    opal_list_item_t super;
+    int member; // ID of member
+    size_t count; // current reduction progress for member
+    int area_id; // current reduce area
+};
+
+struct xhc_loc_def_t {
+    opal_list_item_t super;
+
+    opal_hwloc_locality_t named_loc;
+
+    struct xhc_rank_range_t {
+        int start_rank, end_rank;
+    } *rank_list;
+
+    int rank_list_len;
+
+    int split;
+    int max_ranks;
+
+    bool repeat;
+};
+
+// ----------------------------------------
+
+typedef struct xhc_bcast_ctx_t {
+    void *buf;
+    size_t datacount;
+    ompi_datatype_t *datatype;
+    int root;
+    ompi_communicator_t *ompi_comm;
+    xhc_module_t *module;
+
+    int rank;
+
+    xhc_op_data_t *data;
+
+    xf_sig_t seq;
+    xhc_comm_t *comms;
+
+    xhc_comm_t *src_comm;
+    xhc_copy_method_t method;
+
+    void *self_cico;
+    void *src_buffer;
+
+    xhc_copy_data_t *region_data;
+    xhc_reg_t *reg;
+
+    size_t bytes_total;
+    size_t bytes_avail;
+    size_t bytes_done;
+} xhc_bcast_ctx_t;
+
+// ----------------------------------------
+
+// coll_xhc_component.c
+// --------------------
+
+#define xhc_colltype_to_universal(...) \
+    mca_coll_xhc_colltype_to_universal(__VA_ARGS__)
+#define xhc_colltype_to_str(...) \
+    mca_coll_xhc_colltype_to_str(__VA_ARGS__)
+#define xhc_config_source_to_str(...) \
+    mca_coll_xhc_config_source_to_str(__VA_ARGS__)
+
+#define xhc_component_parse_hierarchy(...) \
+    mca_coll_xhc_component_parse_hierarchy(__VA_ARGS__)
+#define xhc_component_parse_chunk_sizes(...) \
+    mca_coll_xhc_component_parse_chunk_sizes(__VA_ARGS__)
+#define xhc_component_parse_cico_max(...) \
+    mca_coll_xhc_component_parse_cico_max(__VA_ARGS__)
+
+int mca_coll_xhc_component_init_query(bool enable_progress_threads,
+    bool enable_mpi_threads);
+COLLTYPE_T mca_coll_xhc_colltype_to_universal(XHC_COLLTYPE_T xhc_colltype);
+const char *mca_coll_xhc_colltype_to_str(XHC_COLLTYPE_T colltype);
+const char *mca_coll_xhc_config_source_to_str(xhc_config_source_t source);
+
+int mca_coll_xhc_component_parse_hierarchy(const char *val_str,
+    opal_list_t **level_defs_dst, int *nlevel_defs_dst);
+int mca_coll_xhc_component_parse_chunk_sizes(const char *val_str,
+    size_t **vals_dst, int *len_dst);
+int mca_coll_xhc_component_parse_cico_max(const char *val_str,
+    size_t *cico_max_dst);
+
+// coll_xhc_module.c
+// -----------------
+
+#define xhc_module_set_coll_fns(...) \
+    mca_coll_xhc_module_set_coll_fns(__VA_ARGS__)
+#define xhc_module_prepare_hierarchy(...) \
+    mca_coll_xhc_module_prepare_hierarchy(__VA_ARGS__)
+
+mca_coll_base_module_t *mca_coll_xhc_module_comm_query(
+    ompi_communicator_t *comm, int *priority);
+
+int mca_coll_xhc_module_enable(mca_coll_base_module_t *module,
+    ompi_communicator_t *comm);
+int mca_coll_xhc_module_disable(mca_coll_base_module_t *module,
+    ompi_communicator_t *comm);
+
+void mca_coll_xhc_module_set_coll_fns(ompi_communicator_t *comm,
+    xhc_coll_fns_t *new_fns, xhc_coll_fns_t *save_fns);
+
+// coll_xhc.c
+// ----------
+
+#define xhc_lazy_init(...) mca_coll_xhc_lazy_init(__VA_ARGS__)
+#define xhc_init_op(...) mca_coll_xhc_init_op(__VA_ARGS__)
+#define xhc_fini(...) mca_coll_xhc_fini(__VA_ARGS__)
+#define xhc_read_op_config(...) mca_coll_xhc_read_op_config(__VA_ARGS__)
+
+#define xhc_get_cico(...) mca_coll_xhc_get_cico(__VA_ARGS__)
+
+#define xhc_shmem_create(...) mca_coll_xhc_shmem_create(__VA_ARGS__)
+#define xhc_shmem_attach(...) mca_coll_xhc_shmem_attach(__VA_ARGS__)
+
+#define xhc_copy_expose_region(...) mca_coll_xhc_copy_expose_region(__VA_ARGS__)
+#define xhc_copy_region_post(...) mca_coll_xhc_copy_region_post(__VA_ARGS__)
+#define xhc_copy_from(...) mca_coll_xhc_copy_from(__VA_ARGS__)
+#define xhc_copy_close_region(...) mca_coll_xhc_copy_close_region(__VA_ARGS__)
+
+#define xhc_get_registration(...) mca_coll_xhc_get_registration(__VA_ARGS__)
+#define xhc_return_registration(...) mca_coll_xhc_return_registration(__VA_ARGS__)
+
+int mca_coll_xhc_lazy_init(mca_coll_xhc_module_t *module, ompi_communicator_t *comm);
+int mca_coll_xhc_init_op(xhc_module_t *module, ompi_communicator_t *comm,
+    XHC_COLLTYPE_T colltype);
+void mca_coll_xhc_fini(mca_coll_xhc_module_t *module);
+
+int mca_coll_xhc_read_op_config(xhc_module_t *module,
+    ompi_communicator_t *comm, XHC_COLLTYPE_T colltype);
+
+void *mca_coll_xhc_shmem_create(opal_shmem_ds_t *seg_ds, size_t size,
+    ompi_communicator_t *ompi_comm, const char *name, int id1, int id2);
+void *mca_coll_xhc_shmem_attach(opal_shmem_ds_t *seg_ds);
+
+void *mca_coll_xhc_get_cico(xhc_peer_info_t *peer_info, int rank);
+
+int mca_coll_xhc_copy_expose_region(void *base, size_t len,
+    xhc_copy_data_t **region_data);
+void mca_coll_xhc_copy_region_post(void *dst, xhc_copy_data_t *region_data);
+int mca_coll_xhc_copy_from(xhc_peer_info_t *peer_info, void *dst,
+    void *src, size_t size, void *access_token);
+void mca_coll_xhc_copy_close_region(xhc_copy_data_t *region_data);
+
+void *mca_coll_xhc_get_registration(xhc_peer_info_t *peer_info,
+    void *peer_vaddr, size_t size, xhc_reg_t **reg);
+void mca_coll_xhc_return_registration(xhc_reg_t *reg);
+
+// coll_xhc_comm.c
+// --------------------
+
+#define xhc_comms_make(...) mca_coll_xhc_comms_make(__VA_ARGS__)
+#define xhc_comms_destroy(...) mca_coll_xhc_comms_destroy(__VA_ARGS__)
+
+int mca_coll_xhc_comms_make(ompi_communicator_t *ompi_comm,
+    xhc_module_t *module, xhc_op_config_t *config, xhc_op_data_t *data);
+
+void mca_coll_xhc_comms_destroy(xhc_comm_t *comms, int comm_count);
+
+// coll_xhc_hierarchy.c
+// --------------------
+
+#define xhc_hierarchy_make(...) mca_coll_xhc_hierarchy_make(__VA_ARGS__)
+
+int mca_coll_xhc_hierarchy_make(xhc_module_t *module,
+    ompi_communicator_t *comm, const char *hierarchy_string,
+    xhc_loc_t **hierarchy_dst, int *hierarchy_len_dst);
+
+// Primitives (respective file)
+// ----------------------------
+
+int mca_coll_xhc_bcast(void *buf, size_t count, ompi_datatype_t *datatype,
+    int root, ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+int mca_coll_xhc_barrier(ompi_communicator_t *ompi_comm,
+    mca_coll_base_module_t *module);
+
+int mca_coll_xhc_reduce(const void *sbuf, void *rbuf,
+    size_t count, ompi_datatype_t *datatype, ompi_op_t *op, int root,
+    ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+int mca_coll_xhc_allreduce(const void *sbuf, void *rbuf,
+    size_t count, ompi_datatype_t *datatype, ompi_op_t *op,
+    ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+// coll_xhc_bcast.c
+// ----------------
+
+#define xhc_bcast_notify(...) mca_coll_xhc_bcast_notify(__VA_ARGS__)
+#define xhc_bcast_init(...) mca_coll_xhc_bcast_init(__VA_ARGS__)
+#define xhc_bcast_start(...) mca_coll_xhc_bcast_start(__VA_ARGS__)
+#define xhc_bcast_work(...) mca_coll_xhc_bcast_work(__VA_ARGS__)
+#define xhc_bcast_ack(...) mca_coll_xhc_bcast_ack(__VA_ARGS__)
+#define xhc_bcast_fini(...) mca_coll_xhc_bcast_fini(__VA_ARGS__)
+
+void mca_coll_xhc_bcast_notify(xhc_bcast_ctx_t *ctx,
+    xhc_comm_t *xc, size_t bytes_ready);
+
+int mca_coll_xhc_bcast_init(void *buf, size_t count, ompi_datatype_t *datatype,
+    int root, ompi_communicator_t *ompi_comm, xhc_module_t *module,
+    xhc_bcast_ctx_t *ctx_dst);
+
+int mca_coll_xhc_bcast_start(xhc_bcast_ctx_t *ctx);
+int mca_coll_xhc_bcast_work(xhc_bcast_ctx_t *ctx);
+void mca_coll_xhc_bcast_ack(xhc_bcast_ctx_t *ctx);
+void mca_coll_xhc_bcast_fini(xhc_bcast_ctx_t *ctx);
+
+// coll_xhc_allreduce.c
+// --------------------
+
+#define xhc_allreduce_internal(...) mca_coll_xhc_allreduce_internal(__VA_ARGS__)
+
+int mca_coll_xhc_allreduce_internal(const void *sbuf, void *rbuf, size_t count,
+    ompi_datatype_t *datatype, ompi_op_t *op, ompi_communicator_t *ompi_comm,
+    mca_coll_base_module_t *module, bool require_bcast);
+
+// ----------------------------------------
+
+/* Rollover-safe check that _flag_ has reached _thresh_,
+ * without having exceeded it by more than _win_. */
+static inline bool CHECK_FLAG(volatile xf_sig_t *flag,
+        xf_sig_t thresh, xf_sig_t win) {
+
+    // This is okay because xf_sig_t is unsigned. Take care.
+    // The cast's necessity is dependent on the size of xf_sig_t
+    return ((xf_sig_t) (*flag - thresh) <= win);
+}
+
+static inline void WAIT_FLAG(volatile xf_sig_t *flag,
+        xf_sig_t thresh, xf_sig_t win) {
+    bool ready = false;
+
+    do {
+        for(int i = 0; i < XHC_OPAL_PROGRESS_CYCLE; i++) {
+            if(CHECK_FLAG(flag, thresh, win)) {
+                ready = true;
+                break;
+            }
+
+            /* xf_sig_t f = *flag;
+            if(CHECK_FLAG(&f, thresh, win)) {
+                ready = true;
+                break;
+            } else if(CHECK_FLAG(&f, thresh, 1000)) {
+                printf("Debug: Flag check with window %d failed, "
+                    "but succeeded with window 1000. flag = %d, "
+                    "thresh = %d\n", win, f, thresh);
+            } */
+        }
+
+        if(!ready) {
+            opal_progress();
+        }
+    } while(!ready);
+}
+
+/* Modeled after smsc/xpmem. We don't currently want to deviate from its
+ * behaviour. But this adds a second parameter (additionally to smsc/xpmem's)
+ * to set if one wants to adjust the limit. Ideally there would be a singular
+ * opal-level MCA parameter for this, and/or an opal_memcpy that would do the
+ * job that xhc_mempy and smsc/xpmem's respective function do. */
+static inline void xhc_memcpy(void *dst, const void *src, size_t size) {
+    for(size_t copied = 0; copied < size; ) {
+        size_t chunk = opal_min(size - copied,
+            mca_coll_xhc_component.memcpy_chunk_size);
+
+        memcpy((char *) dst + copied, (char *) src + copied, chunk);
+        copied += chunk;
+    }
+}
+
+static inline void xhc_memcpy_offset(void *dst, const void *src,
+        size_t offset, size_t size) {
+    xhc_memcpy((char *) dst + offset, (char *) src + offset, size);
+}
+
+// ----------------------------------------
+
+END_C_DECLS
+
+#endif
diff --git a/ompi/mca/coll/xhc/coll_xhc_allreduce.c b/ompi/mca/coll/xhc/coll_xhc_allreduce.c
new file mode 100644
index 00000000000..d8553441e97
--- /dev/null
+++ b/ompi/mca/coll/xhc/coll_xhc_allreduce.c
@@ -0,0 +1,1417 @@
+/*
+ * Copyright (c) 2021-2024 Computer Architecture and VLSI Systems (CARV)
+ *                         Laboratory, ICS Forth. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "mpi.h"
+
+#include "ompi/constants.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/op/op.h"
+
+#include "opal/mca/rcache/base/base.h"
+#include "opal/util/show_help.h"
+#include "opal/util/minmax.h"
+
+#include "coll_xhc.h"
+
+// -----------------------------
+
+#define COMM_ALL_JOINED 0x01
+#define COMM_REDUCE_FINI 0x02
+
+OBJ_CLASS_INSTANCE(xhc_rq_item_t, opal_list_item_t, NULL, NULL);
+
+static inline char *CICO_BUFFER(xhc_comm_t *xc, int member) {
+    return (char *) xc->reduce_buffer + member * xc->cico_size;
+}
+
+// -----------------------------
+
+/* Calculate/generate the reduce sets/areas. A reduce set defines a
+ * range/area of data to be reduced, and its settings. We require
+ * multiple areas, because there might be different circumstances:
+ *
+ * 1. Under certain load balancing policies, leaders perform reductions
+ *    for just one chunk, and then they don't. Thus, the worker count
+ *    changes, and the settings have to recomputed for the next areas.
+ *
+ * 2. During the "middle" of the operation, all members continuously
+ *    reduce data in maximum-sized pieces (according to the configured
+ *    chunk size). But, towards the end of the operation, the remaining
+ *    elements are less than ((workers * elem_chunk)), we have to
+ *    recalculate `elem_chunk`, so that all workers will perform
+ *    equal work.
+ *
+ * See also the comments in the reduce area struct definition for the different
+ * fields, and the comments in the defintion of xhc_reduce_load_balance_enum_t. */
+static void init_reduce_areas(xhc_comm_t *comms, size_t allreduce_count,
+        size_t dtype_size, xhc_reduce_load_balance_enum_t lb_policy) {
+
+    bool uniform_chunks = mca_coll_xhc_component.uniform_chunks;
+
+    for(xhc_comm_t *xc = comms; xc; xc = xc->up) {
+        int max_reduce_areas = sizeof(xc->reduce_areas)/sizeof(xc->reduce_areas[0]);
+        int avail_workers[max_reduce_areas];
+
+        for(int area_id = 0; area_id < max_reduce_areas; area_id++) {
+            int workers = xc->size - 1;
+
+            if(lb_policy & XHC_REDUCE_LB_LEADER_ASSIST_TOP_LEVEL) {
+                if(xc->is_top && workers < xc->size) {
+                    workers++;
+                }
+            }
+
+            if(lb_policy & XHC_REDUCE_LB_LEADER_ASSIST_FIRST_CHUNK) {
+                if(0 == area_id && workers < xc->size) {
+                    workers++;
+                }
+            }
+
+            if(lb_policy & XHC_REDUCE_LB_LEADER_ASSIST_ALL) {
+                workers = xc->size;
+            }
+
+            avail_workers[area_id] = workers;
+        }
+
+        // Min/max work that a worker may perform (one step)
+        size_t min_elems = mca_coll_xhc_component.uniform_chunks_min / dtype_size;
+        size_t max_elems = xc->chunk_size / dtype_size;
+
+        int area_id = 0;
+        size_t el_idx = 0;
+
+        while(area_id < max_reduce_areas && el_idx < allreduce_count) {
+            xhc_reduce_area_t *area = &xc->reduce_areas[area_id];
+
+            *area = (xhc_reduce_area_t) {0};
+
+            size_t remaining = allreduce_count - el_idx;
+            int workers = avail_workers[area_id];
+
+            size_t elems_per_member;
+            size_t repeat = 0;
+
+            size_t area_elems = opal_min(max_elems * workers, remaining);
+
+            /* We should consider the future size of the next area. If it's
+             * too small in relation to the minimum chunk (min_elems), some
+             * workers of the next area won't perform work, leading to load
+             * imbalance. In this case, we elect to either shrink the current
+             * area so that we will be able to better balance the load in the
+             * next one, or if the elements that remain for the next area are
+             * especially few, we make this area absorb the next one.
+             * Specifically, we absorb it if the increase of each worker's
+             * load is no more than 10% of the maximum load set. */
+            if(uniform_chunks && area_id < max_reduce_areas - 1) {
+                int next_workers = avail_workers[area_id+1];
+                size_t next_remaining = allreduce_count - (el_idx + area_elems);
+
+                if(next_remaining < next_workers * min_elems) {
+                    if(next_remaining/workers <= max_elems/10){
+                        area_elems += next_remaining;
+                    } else {
+                        size_t ideal_donate =
+                            next_workers * min_elems - next_remaining;
+
+                        /* Don't donate so much elements that this area
+                         * won't cover its own min reduction chunk size */
+                        ssize_t max_donate = area_elems - workers * min_elems;
+                        max_donate = opal_max(max_donate, 0);
+
+                        area_elems -= opal_min(ideal_donate, max_donate);
+                    }
+                }
+            }
+
+            if(uniform_chunks) {
+                /* The elements might not be enough for every worker to do
+                 * work. We calculate how many workers we need so that no
+                 * one of them does less than min_elems work, and use the
+                 * result to calculate the final elements per member. */
+                workers = opal_min(area_elems/min_elems, workers);
+                workers = opal_max(workers, 1);
+
+                elems_per_member = area_elems / workers;
+            } else {
+                elems_per_member = max_elems;
+                workers = area_elems/max_elems;
+            }
+
+            // If this is the middle area, try to maximize its size
+            if(1 == area_id && workers > 0) {
+                size_t set = workers * elems_per_member;
+                repeat = (size_t)((remaining-area_elems)/set);
+                area_elems += repeat * set;
+            }
+
+            area->start = el_idx;
+            area->len = area_elems;
+            area->workers = workers;
+            area->stride = workers * elems_per_member;
+
+            /* My ID, assuming that if some member is not reducing, it is
+             * the one with ID=0, because currently only member 0 becomes
+             * the leader, and the leader is the only one that might not
+             * be reducing. */
+            int worker_id = xc->my_id - (xc->size - avail_workers[area_id]);
+
+            area->work_begin = el_idx + worker_id * elems_per_member;
+            area->work_chunk = (worker_id >= 0 && worker_id < workers ?
+                elems_per_member : 0);
+
+            area->work_leftover = 0;
+
+            size_t leftover_elems = (workers > 0 ?
+                (area_elems % (workers * elems_per_member)) : area_elems);
+            if(leftover_elems) {
+                if(worker_id == (uniform_chunks ? workers - 1 : workers)) {
+                    area->work_leftover = leftover_elems;
+                }
+            }
+
+            area->work_end = area->work_begin + (repeat * area->stride)
+                + area->work_chunk + area->work_leftover;
+
+            el_idx += area_elems;
+            area_id++;
+        }
+
+        assert(el_idx == allreduce_count);
+
+        xc->n_reduce_areas = area_id;
+
+        // Erase zero-work areas
+        while(xc->n_reduce_areas > 0
+                && 0 == xc->reduce_areas[xc->n_reduce_areas - 1].work_chunk
+                && 0 == xc->reduce_areas[xc->n_reduce_areas - 1].work_leftover) {
+            xc->n_reduce_areas--;
+        }
+
+        /* If not a leader on this comm, nothing
+         * to do on next ones whatsoever */
+        if(!xc->is_leader) {
+            break;
+        }
+    }
+}
+
+static void xhc_allreduce_init_local(xhc_comm_t *comms,
+        size_t allreduce_count, size_t dtype_size, XHC_COLLTYPE_T colltype,
+        xhc_reduce_load_balance_enum_t lb_policy, xf_sig_t seq) {
+
+    for(xhc_comm_t *xc = comms; xc; xc = xc->up) {
+        // Non-leader by default
+        xc->is_leader = false;
+        xc->leader_id = -1;
+
+        xc->op_state = 0;
+
+        /* Reduce is multi-sliced. In Allreduce, the slice's
+         * readiness is guaranteed with other mechanisms. */
+        xc->slice_ready = (XHC_REDUCE != colltype);
+        xc->slice_id = seq % xc->n_slices;
+
+        /* Set pointers to the current slice's shared sync structs */
+
+        xc->member_ctrl = (void *) ((char *) xc->member_ctrl_base
+            + xc->size * sizeof(xhc_member_ctrl_t) * xc->slice_id);
+        xc->my_ctrl = &xc->member_ctrl[xc->my_id];
+
+        xc->reduce_buffer = (char *) xc->reduce_buffer_base
+            + xc->size * xc->cico_size * xc->slice_id;
+
+        for(int m = 0; m < xc->size; m++) {
+            xc->member_info[m] = (xhc_member_info_t) {0};
+        }
+    }
+
+    for(xhc_comm_t *xc = comms; xc; xc = xc->up) {
+        /* The owner is the leader. Even in the dynamic reduce case,
+         * there (currently) shouldn't be any real benefit from the
+         * leader being dynamic in allreduce. */
+        if(0 != xc->my_id) {
+            break;
+        }
+
+        xc->is_leader = true;
+        xc->leader_id = xc->my_id;
+        xc->do_all_work = false;
+    }
+
+    init_reduce_areas(comms, allreduce_count, dtype_size, lb_policy);
+
+    for(xhc_comm_t *xc = comms; xc; xc = xc->up) {
+        size_t init_count = (xc->n_reduce_areas > 0 ?
+            xc->reduce_areas[0].work_begin : allreduce_count);
+
+        int m = 0;
+        OPAL_LIST_FOREACH_DECL(item, xc->reduce_queue, xhc_rq_item_t) {
+            if(m == xc->my_id) {m++;}
+
+            *item = (xhc_rq_item_t) {.super = item->super, .member = m++,
+                .count = init_count, .area_id = 0};
+        }
+
+        /* Check if this member is the only one that does any reductions
+         * on this comm. Useful on the top level to know if the results
+         * can be placed directly on the root's rbuf. */
+        xc->do_all_work = true;
+        for(int a = 0; a < xc->n_reduce_areas; a++) {
+            xhc_reduce_area_t *area = &xc->reduce_areas[a];
+
+            if(area->work_begin != area->start
+                    || area->work_end != area->len) {
+                xc->do_all_work = false;
+                break;
+            }
+        }
+
+        if(!xc->is_leader) {
+            break;
+        }
+    }
+}
+
+static void xhc_allreduce_init_comm(xhc_comm_t *comms,
+    int rank, xf_sig_t seq) {
+
+    for(xhc_comm_t *xc = comms; xc; xc = xc->up) {
+        if(!xc->is_leader) {break;}
+
+        WAIT_FLAG(&xc->comm_ctrl->ack, seq - 1, 0);
+
+        /* The rest of the fields in comm_ctrl ended up not being necessary.
+         * And since other operations have their own comm ctrl, there's no
+         * problem not keeping fields like comm seq up to date. */
+    }
+}
+
+static int xhc_allreduce_init_member(xhc_comm_t *xc, xhc_peer_info_t *peer_info,
+        void *sbuf, void *rbuf, size_t allreduce_count, size_t dtype_size,
+        xhc_copy_method_t method, xhc_copy_method_t bcast_method, int rank,
+        xf_sig_t seq, bool should_block) {
+
+    /* Only makes sense to init a comm on which a member is participating,
+     * i.e. either any comm that it's a leader on, or the lowest comm on
+     * which it's not a leader (or comm 0, if not a leader on any level). */
+    assert(xc->is_bottom || xc->down->is_leader);
+
+    /* Make sure that the previous owner of my member ctrl is not still
+     * using it (tip: can occur with dynamic leadership (or non-zero
+     * root?!), when it is implemented ^^) . Note that thanks to the
+     * broadcast step in Allreduce, which has some synchronizing
+     * properties, by the time that bcast ends and member ack is set,
+     * it's guaranteed that no other member in a previous collective
+     * is accessing the member's flags or data. In reduce, where there
+     * is no broadcast step, this guarantee is kept by only setting
+     * member ack *after* comm ack. */
+
+    /* In the multi-slice approach, the seq number of the last operation
+     * that made use of the current slice depends on the number of slices. */
+    while(!CHECK_FLAG(&xc->my_ctrl->ack, seq - xc->n_slices, 0)) {
+        if(!should_block) {return OMPI_ERR_WOULD_BLOCK;}
+    }
+
+    xhc_rq_item_t *rq_first = (xhc_rq_item_t *)
+        opal_list_get_first(xc->reduce_queue);
+
+    // ---
+
+    if(XHC_COPY_IMM == method) {
+        memcpy((void *) xc->my_ctrl->imm_data, sbuf, allreduce_count * dtype_size);
+    } else {
+        xc->my_ctrl->rank = rank;
+        xc->my_ctrl->is_leader = (xc->is_leader);
+
+        if(XHC_COPY_SMSC == method) {
+            xc->my_ctrl->sbuf_vaddr = (xc->is_bottom ? sbuf : rbuf);
+            xc->my_ctrl->rbuf_vaddr = (xc->is_leader ? rbuf : NULL);
+        }
+    }
+
+    /* In single or imm copy, the data on the bottom comm is
+     * already in place and ready to be read by other members. */
+    xc->my_ctrl->reduce_ready = (xc->reduce_ready = (xc->is_bottom
+        && XHC_COPY_CICO != method ? allreduce_count : 0));
+
+    xc->my_ctrl->reduce_done = (xc->reduce_done = rq_first->count);
+
+    xhc_atomic_wmb();
+    xc->my_ctrl->seq = seq;
+
+    // ---
+
+    switch(method) {
+        case XHC_COPY_IMM:
+            xc->my_info->sbuf = (void *) xc->my_ctrl->imm_data;
+            xc->my_info->rbuf = rbuf;
+
+            break;
+
+        case XHC_COPY_CICO:
+            xc->my_info->sbuf = CICO_BUFFER(xc, xc->my_id);
+
+            if(xc->up) {
+                xc->my_info->rbuf = CICO_BUFFER(xc->up, xc->up->my_id);
+            } else {
+                xc->my_info->rbuf = (xc->do_all_work ? rbuf
+                    : CICO_BUFFER(xc, xc->my_id));
+            }
+
+            break;
+
+        case XHC_COPY_SMSC:
+            xc->my_info->sbuf = (xc->is_bottom ? sbuf : rbuf);
+            xc->my_info->rbuf = rbuf;
+
+            break;
+
+        default:
+            assert(0);
+    }
+
+    /* If we're doing CICO broadcast, arrange for the top-comm
+     * results to be directly placed on the root's CICO buffer. */
+    if(XHC_COPY_CICO == bcast_method && xc->is_top && xc->is_leader) {
+        xc->my_info->rbuf = xhc_get_cico(peer_info, rank);
+    }
+
+    xc->my_info->attach = true;
+    xc->my_info->join = true;
+
+    return OMPI_SUCCESS;
+}
+
+// -----------------------------
+
+static int xhc_allreduce_attach_member(xhc_comm_t *xc, int member,
+        xhc_peer_info_t *peer_info, size_t bytes, xhc_copy_method_t method,
+        xhc_copy_method_t bcast_method, xf_sig_t seq) {
+
+    xhc_member_info_t *m_info = &xc->member_info[member];
+    xhc_member_ctrl_t *m_ctrl = &xc->member_ctrl[member];
+
+    if(m_info->attach) {
+        return OMPI_SUCCESS;
+    }
+
+    /* If we're doing CICO broadcast, arrange for the top-comm
+     * results to be directly placed on the root's CICO buffer. */
+    if(XHC_COPY_CICO == bcast_method && xc->is_top && m_ctrl->is_leader) {
+        m_info->rbuf = xhc_get_cico(peer_info, m_ctrl->rank);
+    }
+
+    switch(method) {
+        case XHC_COPY_IMM:
+            m_info->sbuf = (void *) m_ctrl->imm_data;
+            break;
+
+        case XHC_COPY_CICO:
+            m_info->sbuf = CICO_BUFFER(xc, member);
+
+            if(m_ctrl->is_leader && !m_info->rbuf) {
+                if(xc->up) {
+                    /* On the comm on the next level, all members
+                     * of this one appear under the same ID. */
+                    m_info->rbuf = CICO_BUFFER(xc->up, xc->up->my_id);
+                } else {
+                    m_info->rbuf = CICO_BUFFER(xc, member);
+                }
+            }
+
+            break;
+
+        case XHC_COPY_SMSC: {
+            void *sbuf_vaddr = m_ctrl->sbuf_vaddr;
+            void *rbuf_vaddr = m_ctrl->rbuf_vaddr;
+
+            m_info->sbuf = xhc_get_registration(&peer_info[m_ctrl->rank],
+                sbuf_vaddr, bytes, &m_info->sbuf_reg);
+
+            if(NULL == m_info->sbuf) {
+                return OMPI_ERR_UNREACH;
+            }
+
+            if(m_ctrl->is_leader && !m_info->rbuf) {
+                if(rbuf_vaddr != sbuf_vaddr) {
+                    m_info->rbuf = xhc_get_registration(&peer_info[m_ctrl->rank],
+                        rbuf_vaddr, bytes, &m_info->rbuf_reg);
+
+                    if(NULL == m_info->rbuf) {
+                        return OMPI_ERR_UNREACH;
+                    }
+                } else {
+                    m_info->rbuf = m_info->sbuf;
+                }
+            }
+
+            break;
+        }
+
+        default:
+            assert(0);
+    }
+
+    // In imm, is_leader is repurposed for payload storage
+    if(XHC_COPY_IMM != method && m_ctrl->is_leader) {
+        xc->leader_id = member;
+    }
+
+    m_info->attach = true;
+
+    return OMPI_SUCCESS;
+}
+
+static bool xhc_allreduce_check_all_joined(xhc_comm_t *xc, xf_sig_t seq) {
+    bool status = true;
+
+    for(int m = 0; m < xc->size; m++) {
+        if(xc->member_info[m].join) {
+            continue;
+        }
+
+        if(CHECK_FLAG(&xc->member_ctrl[m].seq, seq, 0)) {
+            xc->member_info[m].join = true;
+        } else {
+            status = false;
+        }
+    }
+
+    return status;
+}
+
+static void xhc_allreduce_slice_gc(xhc_comm_t *xc) {
+    xf_sig_t ack = xc->comm_ctrl->ack;
+
+    for(int s = 0; s < xc->n_slices; s++) {
+        xhc_sh_slice_t *slice = &xc->slices[s];
+
+        if(slice->in_use && ack >= slice->seq) {
+            if(slice->is_cico) {
+                char *buffer = (char *) xc->reduce_buffer_base
+                    + xc->size * xc->cico_size * (slice->seq % xc->n_slices)
+                    + xc->my_id * xc->cico_size;
+
+                /* Prefetch RFO the reclaimed slice; the 1st
+                 * and 2nd slices into L2, the rest into L3. */
+                xhc_prefetchw(buffer, slice->len, (s < 2 ? 2 : 3));
+            }
+
+            *slice = (xhc_sh_slice_t) {.in_use = false};
+        }
+    }
+}
+
+static bool xhc_allreduce_slice_claim(xhc_comm_t *xc, xhc_peer_info_t *peer_info,
+        void *sbuf, void *rbuf, size_t allreduce_count, size_t dtype_size,
+        xhc_copy_method_t method, xhc_copy_method_t bcast_method, int rank,
+        xf_sig_t seq) {
+
+    xhc_sh_slice_t *slice = &xc->slices[xc->slice_id];
+
+    /* The slice to use has already been determined in init_local() according
+     * to the sequence number. We don't just use any one available slice,
+     * because we want to be predictable, so that (1) we know the slice early,
+     * and (2) so that others know it for certain, without communication. */
+
+    /* If not already recorded as available, check
+     * the ack number and try to reclaim slices. */
+    if(slice->in_use) {
+        xhc_allreduce_slice_gc(xc);
+    }
+
+    if(!slice->in_use) {
+        int err = OMPI_SUCCESS;
+
+        /* slice_claim() might be called for a comm on which this process is
+         * not a participant; recall that this happens because the reduction
+         * result is placed directly on the CICO reduce buffer on the next
+         * level. For this type of call, we don't call init_member(). */
+
+        if(xc->is_bottom || xc->down->is_leader) {
+            err = xhc_allreduce_init_member(xc, peer_info, sbuf, rbuf,
+                allreduce_count, dtype_size, method, bcast_method,
+                rank, seq, false);
+        }
+
+        if(OMPI_SUCCESS == err) {
+            *slice = (xhc_sh_slice_t) {
+                .in_use = true, .seq = seq,
+                .len = allreduce_count * dtype_size,
+                .is_cico = (XHC_COPY_CICO == method)
+            };
+
+            xc->slice_ready = true;
+        }
+    }
+
+    return xc->slice_ready;
+}
+
+static void xhc_allreduce_disconnect_peers(xhc_comm_t *comms) {
+    for(xhc_comm_t *xc = comms; xc; xc = xc->up) {
+        xhc_reg_t *reg;
+
+        for(int m = 0; m < xc->size; m++) {
+            if(!xc->member_info[m].attach) {
+                continue;
+            }
+
+            if((reg = xc->member_info[m].sbuf_reg)) {
+                xhc_return_registration(reg);
+            }
+
+            if((reg = xc->member_info[m].rbuf_reg)) {
+                xhc_return_registration(reg);
+            }
+        }
+    }
+}
+
+// -----------------------------
+
+static void xhc_allreduce_ack(xhc_comm_t *comms,
+        xf_sig_t seq, xhc_bcast_ctx_t *bcast_ctx) {
+
+    // Set personal ack(s), in the (all)reduce hierarchy
+    for(xhc_comm_t *xc = comms; xc; xc = xc->up) {
+        xc->my_ctrl->ack = seq;
+
+        if(!xc->is_leader) {
+            break;
+        }
+
+        xhc_prefetchw((void *) &xc->comm_ctrl->ack,
+            sizeof(xc->comm_ctrl->ack), 1);
+    }
+
+    /* Do the ACK process for the broadcast operation. This is necessary
+     * in order to appropriately set of the fields in the bcast hierarchy.
+     * Furthermore, we also leverage it for synchronizing the end of the
+     * allreduce operation; the broadcast being completed for all ranks,
+     * means that the allreduce is also completed for all. */
+
+    xhc_bcast_ack(bcast_ctx);
+
+    /* Once xhc_bcast_ack completed, all ranks have acknowledged to have
+     * finished the collective. No need to check their member ack fields.
+     * Set ack appropriately. */
+
+    for(xhc_comm_t *xc = comms; xc; xc = xc->up) {
+        if(!xc->is_leader) {break;}
+        xc->comm_ctrl->ack = seq;
+    }
+
+    /* Note that relying on bcast's ack procedure like above, means that
+     * comm ack will be set only after the prodedure has finished on ALL
+     * levels of the bcast hierarchy. Theoretically, this might slighly
+     * delay the assignment of comm ack on some levels. One alternative
+     * could be to monitor acknowledgements in the broadcast ACK process
+     * as they happen, and when there's an xhc comm in the allreduce
+     * hierarchy whose all members have acknowledged, set comm ack. This
+     * is quite complex due to all the necessary translation, and might
+     * involve taking into account multiple pieces of information (e.g.
+     * dynamic leadership). Another alternative would be to do both the
+     * broadcast and the allreduce ack phases in parallel in non-blocking
+     * manner. But this would have us doing 2 similar ack phases, while
+     * only of them is really necessary, ultimately leading to increased
+     * coherency traffic and allegedly higher latency for the ack phase.
+     * I don't currently believe that delaying comm acks ever so slightly
+     * is too big a problem to warrant these more drastic alternatives. */
+}
+
+static void xhc_reduce_ack(xhc_comm_t *comms,
+        xhc_copy_method_t method, xf_sig_t seq) {
+
+    for(xhc_comm_t *xc = comms; xc; xc = xc->up) {
+        /* In single-copy mode we must absolutely wait for the leader to let
+         * us know that reductions are over before exiting. Otherwise, a peer
+         * might get stale data when accessing our app-level buffer (since the
+         * application is free to modify it). In CICO mode, we have increased
+         * control over our internal buffer(s), and members are allowed to exit
+         * even before all reductions have been completed. */
+        if(!xc->is_leader && XHC_COPY_SMSC == method) {
+            WAIT_FLAG(&xc->comm_ctrl->ack, seq, 0);
+        }
+
+        /* Check the ack number of the comm, to reclaim previously thought
+         * in-use slices. We do this opportunistically at the finish of this
+         * operation, with the hope that we won't have to do it before the
+         * next one and delay its start. */
+        xhc_allreduce_slice_gc(xc);
+
+        /* Recall that in CICO mode, even if not a leader, we claim a slice
+         * on the next level, to directly place the reduction results there. */
+        if(!xc->is_leader && xc->up && xc->up->slice_ready) {
+            xhc_allreduce_slice_gc(xc->up);
+        }
+
+        xc->my_ctrl->ack = seq;
+
+        if(!xc->is_leader) {
+            break;
+        }
+    }
+}
+
+// -----------------------------
+
+static void xhc_allreduce_cico_publish(xhc_comm_t *xc, void *data_src,
+        xhc_peer_info_t *peer_info, int rank, size_t allreduce_count,
+        size_t dtype_size) {
+
+    size_t ready = xc->reduce_ready;
+
+    /* The chunk size here is just a means of pipelining the CICO publishing,
+     * for whichever case this might be necessary in. There isn't really any
+     * reason to consult reduce areas and their chunk sizes here.*/
+    size_t elements = opal_min(xc->chunk_size/dtype_size, allreduce_count - ready);
+
+    void *src = (char *) data_src + ready * dtype_size;
+    void *dst = (char *) CICO_BUFFER(xc, xc->my_id) + ready * dtype_size;
+
+    xhc_memcpy(dst, src, elements * dtype_size);
+    xhc_atomic_wmb();
+
+    xhc_atomic_store_size_t(&xc->my_ctrl->reduce_ready,
+        (xc->reduce_ready = ready + elements));
+}
+
+static int xhc_allreduce_reduce_get_next(xhc_comm_t *xc, xhc_peer_info_t *peer_info,
+        size_t allreduce_count, size_t dtype_size, xhc_copy_method_t method,
+        xhc_copy_method_t bcast_method, bool out_of_order_reduce, xf_sig_t seq,
+        xhc_rq_item_t **item_dst) {
+
+    /* Iterate the reduce queue, to determine which member's data to reduce,
+     * and from what index. The reduction queue aids in the implementation of
+     * the rationale that members that are not ready at some point should be
+     * temporarily skipped, to prevent stalling in the collective. Reasons
+     * that a member may not be "ready" are (1) it has not yet joined the
+     * collective, (2) the necessary data has not yet been produced (eg.
+     * because the member's children have not finished their reduction on the
+     * previous communicator) or has not been copied to the CICO buffer.
+     * However, when floating point data is concerned, skipping members and
+     * therefore doing certain reductions in non-deterministic order results
+     * to reproducibility concerns. Hence the existence of the "dynamic reduce"
+     * switch; when enabled, members are skipped when not ready. When disabled,
+     * members are skipped, but only the data of members with a lower ID that
+     * the one that has stalled can be reduced (eg. member 2 has stalled, but
+     * reduction for future chunks of members 0 and 1 (only, not of member 3,
+     * even if it is ready) will begin instead of completely stalling). The
+     * reduction queue is sorted according to the reduction progress counter in
+     * each entry. This helps ensure fully reduced chunks are generated as soon
+     * as possible, so that leaders can quickly propagate them upwards. */
+
+    xhc_rq_item_t *member_item = NULL;
+    int stalled_member = xc->size;
+
+    OPAL_LIST_FOREACH_DECL(item, xc->reduce_queue, xhc_rq_item_t) {
+        int member = item->member;
+
+        if(!xc->member_info[member].attach
+                && CHECK_FLAG(&xc->member_ctrl[member].seq, seq, 0)) {
+
+            xhc_atomic_rmb();
+
+            int err = xhc_allreduce_attach_member(xc, member, peer_info,
+                allreduce_count * dtype_size, method, bcast_method, seq);
+
+            if(OMPI_SUCCESS != err) {
+                return err;
+            }
+        }
+
+        if(xc->member_info[member].attach && item->count < allreduce_count) {
+            xhc_reduce_area_t *area = &xc->reduce_areas[item->area_id];
+            size_t elements = area->work_chunk;
+
+            if(item->count + elements + area->work_leftover == area->work_end) {
+                elements += area->work_leftover;
+            }
+
+            size_t self_ready = xc->reduce_ready;
+            size_t member_ready = xhc_atomic_load_size_t(
+                &xc->member_ctrl[member].reduce_ready);
+
+            if(self_ready >= item->count + elements
+                    && member_ready >= item->count + elements
+                    && member < stalled_member) {
+
+                member_item = item;
+                break;
+            }
+        }
+
+        if(!out_of_order_reduce) {
+            stalled_member = opal_min(stalled_member, member);
+        }
+    }
+
+    if(member_item) {
+        opal_list_remove_item(xc->reduce_queue, (opal_list_item_t *) member_item);
+        *item_dst = member_item;
+    }
+
+    return OMPI_SUCCESS;
+}
+
+static void xhc_allreduce_do_reduce(xhc_comm_t *xc, xhc_rq_item_t *member_item,
+        void *tmp_rbuf, size_t allreduce_count, ompi_datatype_t *dtype,
+        size_t dtype_size, ompi_op_t *op, xhc_copy_method_t method) {
+
+    xhc_reduce_area_t *area = &xc->reduce_areas[member_item->area_id];
+
+    size_t elements = area->work_chunk;
+
+    if(member_item->count + elements + area->work_leftover == area->work_end) {
+        elements += area->work_leftover;
+    }
+
+    // ---
+
+    bool first_reduction = false;
+    bool last_reduction = false;
+
+    /* Remember that member_item is not currently in
+     * the queue, as we removed it in get_next(). */
+
+    if(0 == opal_list_get_size(xc->reduce_queue)) {
+        first_reduction = true;
+        last_reduction = true;
+    } else {
+        xhc_rq_item_t *rq_first = (xhc_rq_item_t *)
+            opal_list_get_first(xc->reduce_queue);
+        xhc_rq_item_t *rq_last = (xhc_rq_item_t *)
+            opal_list_get_last(xc->reduce_queue);
+
+        /* If this count is equal or larger than the last one, it means that
+         * no other count in the queue is larger than it. Therefore, this is the
+         * first reduction taking place for the 'member_item->count' chunk idx. */
+        if(member_item->count >= rq_last->count) {
+            first_reduction = true;
+        }
+
+        /* If this count is uniquely minimum in the queue, this is the
+         * last reduction taking place for this specific chunk index. */
+        if(member_item->count < rq_first->count) {
+            last_reduction = true;
+        }
+    }
+
+    // ---
+
+    char *src, *dst, *src2 = NULL;
+    size_t offset = member_item->count * dtype_size;
+
+    src = (char *) xc->member_info[member_item->member].sbuf + offset;
+
+    /* In cico & zcopy, the leader is discovered during attach() (one of
+     * the members has member_ctrl->is_leader=true). In imm, is_leader is
+     * repurposed for the payload, but in that case we force the leader to
+     * do all reductions, so really he's the only that needs to know. */
+    assert(XHC_COPY_IMM != method || xc->is_leader);
+    assert(!last_reduction || xc->leader_id >= 0);
+
+    if(last_reduction) {
+        dst = (char *) xc->member_info[xc->leader_id].rbuf + offset;
+    } else {
+        dst = (char *) tmp_rbuf + offset;
+    }
+
+    if(first_reduction) {
+        src2 = (char *) xc->my_info->sbuf + offset;
+    } else if(last_reduction) {
+        src2 = (char *) tmp_rbuf + offset;
+    }
+
+    // Might happen under MPI_IN_PLACE or CICO
+    if(src2 == dst) {
+        src2 = NULL;
+    } else if(src == dst) {
+        src = src2;
+        src2 = NULL;
+    }
+
+    xhc_atomic_rmb();
+
+    if(src2) {ompi_3buff_op_reduce(op, src2, src, dst, elements, dtype);}
+    else {ompi_op_reduce(op, src, dst, elements, dtype);}
+
+    // ---
+
+    /* If we reached the end of the area after this reduction, switch
+     * to the next one, or mark completion if it was the last one.
+     * Otherwise, adjust the count according to the area's parameters. */
+
+    if(member_item->count + elements == area->work_end) {
+        if(member_item->area_id < xc->n_reduce_areas - 1) {
+            member_item->area_id++;
+            member_item->count = xc->reduce_areas[member_item->area_id].work_begin;
+        } else {
+            member_item->count = allreduce_count;
+        }
+    } else {
+        member_item->count += area->stride;
+    }
+
+    // ---
+
+    /* Place the item back in the queue, keeping
+     * it incrementally sorted by item->count. */
+
+    bool placed = false;
+
+    xhc_rq_item_t *item;
+    OPAL_LIST_FOREACH_REV(item, xc->reduce_queue, xhc_rq_item_t) {
+        if(member_item->count >= item->count) {
+            opal_list_insert_pos(xc->reduce_queue,
+                (opal_list_item_t *) item->super.opal_list_next,
+                (opal_list_item_t *) member_item);
+
+            placed = true;
+            break;
+        }
+    }
+
+    if(!placed) {
+        opal_list_prepend(xc->reduce_queue, (opal_list_item_t *) member_item);
+    }
+
+    /* Find the new min count in the queue after this reduction,
+     * and appropriately adjust reduce_done if/as necessary. */
+
+    xhc_rq_item_t *rq_first = (xhc_rq_item_t *)
+        opal_list_get_first(xc->reduce_queue);
+
+    if(rq_first->count > xc->reduce_done) {
+        xhc_atomic_wmb();
+
+        xhc_atomic_store_size_t(&xc->my_ctrl->reduce_done,
+            (xc->reduce_done = rq_first->count));
+    }
+}
+
+// -----------------------------
+
+int mca_coll_xhc_allreduce_internal(const void *sbuf, void *rbuf, size_t count,
+        ompi_datatype_t *datatype, ompi_op_t *op, ompi_communicator_t *ompi_comm,
+        mca_coll_base_module_t *ompi_module, bool require_bcast) {
+
+    XHC_COLLTYPE_T colltype = (require_bcast ? XHC_ALLREDUCE : XHC_REDUCE);
+    xhc_module_t *module = (xhc_module_t *) ompi_module;
+
+    int err;
+
+    // ---
+
+    if(!ompi_datatype_is_predefined(datatype)) {
+        WARN_ONCE("coll:xhc: Warning: XHC does not currently support "
+            "derived datatypes; utilizing fallback component");
+        goto _fallback;
+    }
+
+    if(!ompi_op_is_commute(op)) {
+        WARN_ONCE("coll:xhc: Warning: (all)reduce does not support "
+            "non-commutative operators; utilizing fallback component");
+        goto _fallback;
+    }
+
+    if(!module->zcopy_support) {
+        size_t dtype_size; ompi_datatype_type_size(datatype, &dtype_size);
+        size_t cico_size = module->op_config[colltype].cico_max;
+        if(count * dtype_size > cico_size) {
+            WARN_ONCE("coll:xhc: Warning: No smsc support; utilizing fallback "
+                "component for %s greater than %zu bytes", (require_bcast ?
+                "allreduce" : "reduce"), cico_size);
+            goto _fallback;
+        }
+    }
+
+    if(!module->op_data[colltype].init) {
+        err = xhc_init_op(module, ompi_comm, colltype);
+        if(OMPI_SUCCESS != err) {goto _fallback_permanent;}
+    }
+
+    if(require_bcast && !module->op_data[XHC_BCAST].init) {
+        err = xhc_init_op(module, ompi_comm, XHC_BCAST);
+        if(OMPI_SUCCESS != err) {goto _fallback_permanent_bcast;}
+    }
+
+    // ---
+
+    xhc_peer_info_t *peer_info = module->peer_info;
+    xhc_op_data_t *op_data = &module->op_data[colltype];
+
+    xhc_comm_t *comms = op_data->comms;
+
+    size_t dtype_size, bytes_total;
+    ompi_datatype_type_size(datatype, &dtype_size);
+    bytes_total = count * dtype_size;
+
+    xhc_copy_method_t method;
+
+    bool out_of_order_reduce = false;
+    xhc_reduce_load_balance_enum_t lb_policy;
+
+    int rank = ompi_comm_rank(ompi_comm);
+
+    /* Currently hard-coded. Okay for Allreduce. For reduce, it's
+     * because we don't yet support non-zero root... (TODO) */
+    int root = comms->top->owner_rank;
+
+    // ---
+
+    switch(mca_coll_xhc_component.dynamic_reduce) {
+        case XHC_DYNAMIC_REDUCE_DISABLED:
+            out_of_order_reduce = false;
+            break;
+
+        case XHC_DYNAMIC_REDUCE_NON_FLOAT:
+            out_of_order_reduce = !(datatype->super.flags
+                & OMPI_DATATYPE_FLAG_DATA_FLOAT);
+            break;
+
+        case XHC_DYNAMIC_REDUCE_ALL:
+            out_of_order_reduce = true;
+            break;
+    }
+
+    if(bytes_total <= XHC_REDUCE_IMM_SIZE) {
+        method = XHC_COPY_IMM;
+    } else if(bytes_total <= comms[0].cico_size) {
+        method = XHC_COPY_CICO;
+    } else {
+        method = XHC_COPY_SMSC;
+    }
+
+    /* In XHC_COPY_IMM, we force the leaders to perform the reductions,
+     * to greatly simplify imm buffer management. Don't see any reason
+     * for any other member to do them anyway... */
+    if(XHC_COPY_IMM == method) {lb_policy = XHC_REDUCE_LB_LEADER_ASSIST_ALL;}
+    else {lb_policy = mca_coll_xhc_component.reduce_load_balance;}
+
+    /* We require a buffer to store intermediate data. In cases like MPI_Ruduce,
+     * non-root ranks don't normally have an rbuf, so allocate an internal one.
+     * TODO: Strictly speaking, the members that won't do reductions, shouldn't
+     * require an rbuf; consult this and don't allocate one for them?? */
+    if(NULL == rbuf) {
+        if(module->rbuf_size < bytes_total) {
+            void *new_rbuf = realloc(module->rbuf, bytes_total);
+            if(!new_rbuf) {return OPAL_ERR_OUT_OF_RESOURCE;}
+
+            module->rbuf = new_rbuf;
+            module->rbuf_size = bytes_total;
+        }
+
+        rbuf = module->rbuf;
+    }
+
+    if(MPI_IN_PLACE == sbuf) {
+        sbuf = rbuf;
+    }
+
+    // ---
+
+    xf_sig_t seq = ++op_data->seq;
+
+    xhc_allreduce_init_local(comms, count, dtype_size, colltype, lb_policy, seq);
+    xhc_allreduce_init_comm(comms, rank, seq);
+
+    // My conscience is clear!
+    if(require_bcast) {goto _allreduce;}
+    else {goto _reduce;}
+
+// =============================================================================
+
+_allreduce: {
+
+    xhc_bcast_ctx_t bcast_ctx;
+    bool bcast_started = false;
+
+    void *bcast_buf = rbuf;
+
+    /* In CICO, the reduced data is placed on the root's cico reduce buffer,
+     * unless the root does all the reduction, in which case just have him
+     * place the final data directly on his rbuf instead. Determine which
+     * case we're in, and supply the appropriate buffer to bcast. Even if
+     * the final data is in the reduce buffer and we have to copy it to rbuf,
+     * better do it after the bcast has started, rather than delay it. FYI,
+     * if bcast is also CICO, the data will end up getting placed directly
+     * on the root's bcast cico buffer (not a problem for setting bcast_buf,
+     * just so you know there's a bit more to it!). */
+    if(XHC_COPY_CICO == method && rank == root && !comms->top->do_all_work) {
+        bcast_buf = CICO_BUFFER(comms->top, comms->top->my_id);
+    }
+
+    xhc_bcast_init(bcast_buf, count, datatype, root,
+        ompi_comm, module, &bcast_ctx);
+
+    /* Allreduce is not multi-sliced (no perf benefit from multi-slicing?),
+     * so init the member struct on all comms here, in blocking manner. */
+    for(xhc_comm_t *xc = comms; xc; xc = xc->up) {
+        xhc_allreduce_init_member(xc, peer_info, (void *) sbuf, rbuf, count,
+            dtype_size, method, bcast_ctx.method, rank, seq, true);
+        if(!xc->is_leader) {break;}
+    }
+
+    for(size_t bytes_done = 0; bytes_done < bytes_total; ) {
+
+        // CICO mode, copy-in phase
+        if(XHC_COPY_CICO == method && comms->bottom->reduce_ready < count) {
+            xhc_allreduce_cico_publish(comms->bottom, (void *) sbuf,
+                peer_info, rank, count, dtype_size);
+        }
+
+        for(xhc_comm_t *xc = comms; xc; xc = xc->up) {
+
+            if(xc->reduce_done < count) {
+                xhc_rq_item_t *member_item = NULL;
+
+                err = xhc_allreduce_reduce_get_next(xc, peer_info, count,
+                    dtype_size, method, bcast_ctx.method, out_of_order_reduce,
+                    seq, &member_item);
+                if(OMPI_SUCCESS != err) {return err;}
+
+                if(member_item) {
+                    xhc_allreduce_do_reduce(xc, member_item, rbuf,
+                        count, datatype, dtype_size, op, method);
+                }
+            }
+
+            /* If not a leader in this comm, no propagation to
+             * do, and not participating in higher-up comms. */
+            if(!xc->is_leader) {break;}
+
+            /* Leaders monitor children's progress and appropriately
+             * propagate towards upper levels. The top level leader
+             * initiates the broadcast of fully reduced chunks. */
+
+            // ---
+
+            if(xc->op_state & COMM_REDUCE_FINI) {
+                continue;
+            }
+
+            /* Check if all have joined the collective, so that we may safely
+             * access their information in member_ctrl, plus we don't waste our
+             * time attempting propagation; reductions can't have finished if
+             * not all members have joined. */
+            if(!(xc->op_state & COMM_ALL_JOINED)) {
+                if(xhc_allreduce_check_all_joined(xc, seq)) {
+                    xc->op_state |= COMM_ALL_JOINED;
+                } else {continue;}
+            }
+
+            /* Don't bother checking the members' shared counters (potentially
+             * expensive), if mine is about to hold the process back any way. */
+            if(!((xc->up && xc->reduce_done > xc->up->reduce_ready)
+            || (xc->is_top && xc->reduce_done * dtype_size > bytes_done)
+            || (xc->reduce_done >= count))) {
+                continue;
+            }
+
+            // ---
+
+            size_t completed = count;
+            size_t completed_bytes;
+
+            for(int m = 0; m < xc->size; m++) {
+                size_t member_done = (m == xc->my_id ? xc->reduce_done :
+                    xhc_atomic_load_size_t(&xc->member_ctrl[m].reduce_done));
+
+                /* Watch out for double evaluation here, don't perform
+                 * sensitive loads inside opal_min()'s parameter list. */
+                completed = opal_min(completed, member_done);
+            }
+
+            completed_bytes = completed * dtype_size;
+
+            if(xc->up && completed > xc->up->reduce_ready) {
+                /* In imm mode: copy the data into my ctrl on
+                 * the next level as part of the propagation. */
+                if(XHC_COPY_IMM == method) {
+                    size_t up_bytes_ready = xc->up->reduce_ready * dtype_size;
+                    xhc_memcpy_offset((void *) xc->up->my_ctrl->imm_data,
+                        xc->my_info->rbuf, up_bytes_ready,
+                        completed_bytes - up_bytes_ready);
+                }
+
+                xhc_atomic_store_size_t(&xc->up->my_ctrl->reduce_ready,
+                    (xc->up->reduce_ready = completed));
+            } else if(xc->is_top && completed_bytes > bytes_done) {
+                for(xhc_comm_t *bxc = bcast_ctx.comms->top; bxc; bxc = bxc->down) {
+                    xhc_bcast_notify(&bcast_ctx, bxc, completed_bytes);
+                }
+
+                if(xc->my_info->rbuf != rbuf) {
+                    xhc_memcpy_offset(rbuf, xc->my_info->rbuf,
+                        bytes_done, completed_bytes - bytes_done);
+                }
+
+                bytes_done = completed_bytes;
+                bcast_ctx.bytes_done = completed_bytes;
+            }
+
+            /* As soon as reduction and propagation is fully finished on
+             * this comm, no reason to 'visit' it anymore. Furthermore,
+             * once all reductions are done, it's possible that members
+             * will receive (through bcast) all final data and exit the
+             * collective. And they may well enter a new collective and
+             * start initializing their member_ctrl, so it's not
+             * appropriate to keep reading their ctrl data. */
+            if(completed >= count) {
+                xc->op_state |= COMM_REDUCE_FINI;
+            }
+        }
+
+        // Broadcast
+        // ---------
+
+        if(!bcast_started) {
+            err = xhc_bcast_start(&bcast_ctx);
+            if(OMPI_SUCCESS == err) {bcast_started = true;}
+            else if(OMPI_ERR_WOULD_BLOCK != err) {return err;}
+        }
+
+        if(bcast_ctx.src_comm && bcast_ctx.bytes_done < bcast_ctx.bytes_total) {
+            /* Currently, in single-copy mode, even though we already have
+             * some established xpmem attachments, these might need to be
+             * re-established in bcast. Some form of small/quick caching
+             * could be implemented in get_registration to avoid this. Or
+             * the allreduce implementation could hint to broadcast that
+             * registrations are available. */
+
+            err = xhc_bcast_work(&bcast_ctx);
+
+            if(OMPI_SUCCESS != err && OMPI_ERR_WOULD_BLOCK != err) {
+                return err;
+            }
+
+            bytes_done = bcast_ctx.bytes_done;
+        }
+    }
+
+    // ---
+
+    /* This is theoretically necessary, for the case that a leader has copied
+     * all chunks and thus exited the loop, but hasn't yet notified some of
+     * its children (e.g. because they were still active in a previous op). */
+    while(!bcast_started) {
+        err = xhc_bcast_start(&bcast_ctx);
+        if(OMPI_SUCCESS == err) {bcast_started = true;}
+        else if(OMPI_ERR_WOULD_BLOCK != err) {return err;}
+    }
+
+    xhc_allreduce_ack(comms, seq, &bcast_ctx);
+
+    xhc_bcast_fini(&bcast_ctx);
+
+    /* See respective comment in xhc_bcast_fini(). Note that prefetchw is also
+     * used in Reduce, but that happens inside xhc_allreduce_slice_gc(). */
+    if(XHC_COPY_CICO == method) {
+        for(xhc_comm_t *xc = comms; xc; xc = xc->up) {
+            xhc_prefetchw(CICO_BUFFER(xc, xc->my_id), bytes_total, 2);
+            if(!xc->is_leader) {break;}
+        }
+    }
+
+    goto _finish;
+}
+
+// =============================================================================
+
+_reduce: {
+
+    for(size_t bytes_done = 0; bytes_done < bytes_total; ) {
+        for(xhc_comm_t *xc = comms; xc; xc = xc->up) {
+
+            /* We can't really proceed if the slice is not reserved.
+             * Recall that a number of counters (like reduce_ready) are
+             * initialized inside init_member(), which is asynchronously
+             * called from slice_claim() when possible. */
+            if(!xc->slice_ready) {
+                xhc_allreduce_slice_claim(xc, peer_info, (void *) sbuf, rbuf,
+                    count, dtype_size, method, 0, rank, seq);
+
+                if(!xc->slice_ready) {
+                    break;
+                }
+            }
+
+            /* CICO mode, copy-in phase. Watch out, unlike Allreduce,
+             * we want this here, *after* we've claimed the slice. */
+            if(XHC_COPY_CICO == method && xc->is_bottom && xc->reduce_ready < count) {
+                xhc_allreduce_cico_publish(xc, (void *) sbuf,
+                    peer_info, rank, count, dtype_size);
+            }
+
+            if(xc->reduce_done < count) {
+                /* In CICO mode, reductions results are stored directly
+                 * on the buffer on the next level; make sure we are
+                 * allowed to do this (by reserving a slice). */
+                if(XHC_COPY_CICO == method && xc->up && !xc->up->slice_ready) {
+                    xhc_allreduce_slice_claim(xc->up, peer_info, (void *) sbuf,
+                        rbuf, count, dtype_size, method, 0, rank, seq);
+
+                    if(!xc->up->slice_ready) {
+                        break;
+                    }
+                }
+
+                xhc_rq_item_t *member_item = NULL;
+
+                err = xhc_allreduce_reduce_get_next(xc, peer_info, count,
+                    dtype_size, method, 0, out_of_order_reduce, seq, &member_item);
+                if(OMPI_SUCCESS != err) {return err;}
+
+                if(member_item) {
+                    xhc_allreduce_do_reduce(xc, member_item, rbuf,
+                        count, datatype, dtype_size, op, method);
+                }
+            }
+
+            if(!xc->is_leader) {
+                /* When both of these reach `count`, all my tasks
+                 * are done, and am free to exit the loop. */
+                bytes_done = opal_min(xc->reduce_ready,
+                    xc->reduce_done) * dtype_size;
+
+                /* Not a leader in this comm, so no propagation,
+                 * and not participating in higher-up ones. */
+                break;
+            }
+
+            // ---
+
+            if(xc->op_state & COMM_REDUCE_FINI) {
+                continue;
+            }
+
+            /* Check if all have joined the collective, so that we may safely
+             * access their information in member_ctrl, plus we don't waste our
+             * time attempting propagation; reductions can't have finished if
+             * not all members have joined. */
+            if(!(xc->op_state & COMM_ALL_JOINED)) {
+                if(xhc_allreduce_check_all_joined(xc, seq)) {
+                    xc->op_state |= COMM_ALL_JOINED;
+                } else {continue;}
+            }
+
+            /* Don't bother checking the members' shared counters (potentially
+             * expensive), if mine is about to hold the process back any way. */
+            if(!((xc->up && xc->reduce_done > xc->up->reduce_ready)
+            || (xc->is_top && xc->reduce_done * dtype_size > bytes_done)
+            || (xc->reduce_done >= count))) {
+                continue;
+            }
+
+            if(xc->up && !xc->up->slice_ready) {
+                xhc_allreduce_slice_claim(xc->up, peer_info, (void *) sbuf,
+                    rbuf, count, dtype_size, method, 0, rank, seq);
+
+                if(!xc->up->slice_ready) {
+                    break;
+                }
+            }
+
+            // ---
+
+            size_t completed = count;
+            size_t completed_bytes;
+
+            for(int m = 0; m < xc->size; m++) {
+                size_t member_done = (m == xc->my_id ? xc->reduce_done :
+                    xhc_atomic_load_size_t(&xc->member_ctrl[m].reduce_done));
+
+                /* Watch out for double evaluation here, don't perform
+                 * sensitive loads inside opal_min()'s parameter list. */
+                completed = opal_min(completed, member_done);
+            }
+
+            completed_bytes = completed * dtype_size;
+
+            if(xc->up && completed > xc->up->reduce_ready) {
+                /* In imm mode: copy the data into my ctrl on
+                 * the next level as part of the propagation. */
+                if(XHC_COPY_IMM == method) {
+                    size_t up_bytes_ready = xc->up->reduce_ready * dtype_size;
+                    xhc_memcpy_offset((void *) xc->up->my_ctrl->imm_data,
+                        xc->my_info->rbuf, up_bytes_ready,
+                        completed_bytes - up_bytes_ready);
+                }
+
+                xhc_atomic_store_size_t(&xc->up->my_ctrl->reduce_ready,
+                    (xc->up->reduce_ready = completed));
+            } else if(xc->is_top && completed_bytes > bytes_done) {
+                if(xc->my_info->rbuf != rbuf) {
+                    xhc_memcpy_offset(rbuf, xc->my_info->rbuf,
+                        bytes_done, completed_bytes - bytes_done);
+                }
+
+                bytes_done = completed_bytes;
+            }
+
+            if(completed >= count) {
+                xc->comm_ctrl->ack = seq;
+                xc->op_state |= COMM_REDUCE_FINI;
+            }
+        }
+    }
+
+    xhc_reduce_ack(comms, method, seq);
+
+    goto _finish;
+}
+
+// =============================================================================
+
+_finish:
+
+    if(XHC_COPY_SMSC == method) {
+        xhc_allreduce_disconnect_peers(comms);
+    }
+
+    return OMPI_SUCCESS;
+
+_fallback_permanent_bcast:
+
+    XHC_INSTALL_FALLBACK(module, ompi_comm, XHC_BCAST, bcast);
+
+_fallback_permanent:
+
+    if(require_bcast) {
+        XHC_INSTALL_FALLBACK(module,
+            ompi_comm, XHC_ALLREDUCE, allreduce);
+    } else {
+        XHC_INSTALL_FALLBACK(module,
+            ompi_comm, XHC_REDUCE, reduce);
+    }
+
+_fallback:
+
+    if(require_bcast) {
+        return XHC_CALL_FALLBACK(module->prev_colls, XHC_ALLREDUCE,
+            allreduce, sbuf, rbuf, count, datatype, op, ompi_comm);
+    } else {
+        return XHC_CALL_FALLBACK(module->prev_colls, XHC_REDUCE,
+            reduce, sbuf, rbuf, count, datatype, op, 0, ompi_comm);
+    }
+}
+
+int mca_coll_xhc_allreduce(const void *sbuf, void *rbuf,
+        size_t count, ompi_datatype_t *datatype, ompi_op_t *op,
+        ompi_communicator_t *ompi_comm, mca_coll_base_module_t *ompi_module) {
+
+    return xhc_allreduce_internal(sbuf, rbuf, count,
+        datatype, op, ompi_comm, ompi_module, true);
+}        
diff --git a/ompi/mca/coll/xhc/coll_xhc_barrier.c b/ompi/mca/coll/xhc/coll_xhc_barrier.c
new file mode 100644
index 00000000000..d2779d23477
--- /dev/null
+++ b/ompi/mca/coll/xhc/coll_xhc_barrier.c
@@ -0,0 +1,140 @@
+/*
+ * Copyright (c) 2021-2024 Computer Architecture and VLSI Systems (CARV)
+ *                         Laboratory, ICS Forth. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "mpi.h"
+
+#include "ompi/constants.h"
+#include "ompi/communicator/communicator.h"
+
+#include "coll_xhc.h"
+
+static void xhc_barrier_leader(xhc_comm_t *comms, int comm_count,
+        xhc_peer_info_t *peer_info, int rank, int root, xf_sig_t seq) {
+
+    for(xhc_comm_t *xc = comms; xc; xc = xc->up) {
+        // Non-leader by default
+        xc->is_leader = false;
+    }
+
+    for(xhc_comm_t *xc = comms; xc; xc = xc->up) {
+        // I'm the root and therefore always a leader
+        if(rank == root) {
+            xc->comm_ctrl->leader_seq = seq;
+            xc->is_leader = true;
+
+            continue;
+        }
+
+        // The root takes leadership precedence when local
+        if(PEER_IS_LOCAL(peer_info, root, xc->locality)) {
+            break;
+        }
+
+        // The member with the lowest ID (ie. the owner) becomes the leader
+        if(0 == xc->my_id) {
+            xc->comm_ctrl->leader_seq = seq;
+            xc->is_leader = true;
+        }
+
+        // Non-leaders exit; they can't become leaders on higher levels
+        if(false == xc->is_leader) {
+            break;
+        }
+    }
+}
+
+/* Hierarchical Barrier with seq/ack flags
+ * -----------------------------------------------------------------
+ * 1. Ranks write their member seq field to signal they have joined
+ *    the collective. Leaders propagate this information towards
+ *    the top-most comm's leader using the same method.
+ *
+ * 2. The top-most comm's leader (root) sets the comm's comm ack
+ *    field to signal, that all ranks have joined the barrier.
+ *
+ * 3. Leaders propagate the info towards the bottom-most comm, using
+ *    the same method. Ranks wait on their comm ack flag, set their
+ *    own ack, and exit the collective.
+ * ----------------------------------------------------------------- */
+int mca_coll_xhc_barrier(ompi_communicator_t *ompi_comm,
+        mca_coll_base_module_t *ompi_module) {
+
+    xhc_module_t *module = (xhc_module_t *) ompi_module;
+
+    if(!module->op_data[XHC_BARRIER].init) {
+        int err = xhc_init_op(module, ompi_comm, XHC_BARRIER);
+        if(OMPI_SUCCESS != err) {goto _fallback_permanent;}
+    }
+
+    xhc_peer_info_t *peer_info = module->peer_info;
+    xhc_op_data_t *data = &module->op_data[XHC_BARRIER];
+
+    xhc_comm_t *comms = data->comms;
+    int comm_count = data->comm_count;
+
+    int rank = ompi_comm_rank(ompi_comm);
+
+    xf_sig_t seq = ++data->seq;
+
+    xhc_barrier_leader(comms, comm_count, peer_info, rank,
+        mca_coll_xhc_component.barrier_root, seq);
+
+    // 1. Upwards SEQ Wave
+    for(xhc_comm_t *xc = comms; xc; xc = xc->up) {
+        xc->my_ctrl->seq = seq;
+
+        if(!xc->is_leader) {
+            break;
+        }
+
+        for(int m = 0; m < xc->size; m++) {
+            if(m == xc->my_id) {
+                continue;
+            }
+
+            /* Poll comm members and wait for them to join the barrier.
+             * No need for windowed comparison here; Ranks won't exit the
+             * barrier before the leader has set the comm ack flag. */
+            WAIT_FLAG(&xc->member_ctrl[m].seq, seq, 0);
+        }
+    }
+
+    // 2. Wait for ACK (root won't wait!)
+    for(xhc_comm_t *xc = comms; xc; xc = xc->up) {
+        if(false == xc->is_leader) {
+            WAIT_FLAG(&xc->comm_ctrl->ack, seq, 0);
+            break;
+        }
+    }
+
+    // 3. Trigger ACK Wave
+    for(xhc_comm_t *xc = comms; xc; xc = xc->up) {
+        if(!xc->is_leader) {
+            break;
+        }
+
+        xc->comm_ctrl->ack = seq;
+    }
+
+    return OMPI_SUCCESS;
+
+    // ---
+
+_fallback_permanent:
+
+    XHC_INSTALL_FALLBACK(module,
+        ompi_comm, XHC_BARRIER, barrier);
+
+// _fallback:
+
+    return XHC_CALL_FALLBACK(module->prev_colls,
+        XHC_BARRIER, barrier, ompi_comm);
+}
diff --git a/ompi/mca/coll/xhc/coll_xhc_bcast.c b/ompi/mca/coll/xhc/coll_xhc_bcast.c
new file mode 100644
index 00000000000..ebfc3145b6a
--- /dev/null
+++ b/ompi/mca/coll/xhc/coll_xhc_bcast.c
@@ -0,0 +1,468 @@
+/*
+ * Copyright (c) 2021-2024 Computer Architecture and VLSI Systems (CARV)
+ *                         Laboratory, ICS Forth. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "mpi.h"
+
+#include "ompi/constants.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/communicator/communicator.h"
+#include "opal/util/show_help.h"
+#include "opal/util/minmax.h"
+
+#include "coll_xhc.h"
+
+// ------------------------------------------------
+
+#define COMM_PREV_FINI 0x01
+#define COMM_CTRL_INIT 0x02
+
+// ------------------------------------------------
+
+/* When dynamic leadership is enabled, the first rank of
+ * each xhc comm to join the collective becomes leader */
+static void xhc_bcast_init_local(xhc_comm_t *comms, xhc_peer_info_t *peer_info,
+        int rank, int root, xf_sig_t seq) {
+
+    for(xhc_comm_t *xc = comms; xc; xc = xc->up) {
+        // Non-leader by default
+        xc->is_leader = false;
+
+        xc->op_state = 0;
+    }
+
+    for(xhc_comm_t *xc = comms; xc; xc = xc->up) {
+        // I'm the root and therefore always a leader
+        if(rank == root) {
+            xc->comm_ctrl->leader_seq = seq;
+            xc->is_leader = true;
+
+            continue;
+        }
+
+        // The root takes leadership precedence when local
+        if(PEER_IS_LOCAL(peer_info, root, xc->locality)) {
+            break;
+        }
+
+        if(false == mca_coll_xhc_component.dynamic_leader) {
+            /* If dynamic leadership is disabled, the member with
+             * the lowest ID (ie. the owner) becomes the leader */
+            if(0 == xc->my_id) {
+                xc->comm_ctrl->leader_seq = seq;
+                xc->is_leader = true;
+            }
+        } else {
+            // An opportunity exists to become the leader
+            if(xc->comm_ctrl->leader_seq != seq) {
+                xf_sig_t oldval = seq - 1;
+
+                xc->is_leader = xhc_atomic_cmpxchg_strong_relaxed(
+                    &xc->comm_ctrl->leader_seq, &oldval, seq);
+            }
+        }
+
+        // Non-leaders exit; they can't become leaders on higher levels
+        if(false == xc->is_leader) {
+            break;
+        }
+    }
+
+    /* The writes and the cmpxchg to comm_ctrl->leader_seq, are relaxed.
+     * They do not synchronize access to any other data, and it's not a
+     * problem if some closeby loads/stores are reordered with it. The
+     * only purpose of leader_seq is to determine if a rank will be leader
+     * or not. Only the result of the cmp operation is utilized. */
+}
+
+// ------------------------------------------------
+
+void mca_coll_xhc_bcast_notify(xhc_bcast_ctx_t *ctx,
+        xhc_comm_t *xc, size_t bytes_ready) {
+
+    if(!(xc->op_state & COMM_PREV_FINI)) {
+        return;
+    }
+
+    if(xc->op_state & COMM_CTRL_INIT) {
+        assert(XHC_COPY_IMM != ctx->method);
+        xhc_atomic_store_size_t(&xc->comm_ctrl->bytes_ready, bytes_ready);
+    } else {
+        if(XHC_COPY_IMM == ctx->method) {
+            xhc_memcpy((void *) xc->comm_ctrl->imm_data, ctx->buf, bytes_ready);
+        } else {
+            xc->comm_ctrl->leader_rank = ctx->rank;
+            xc->comm_ctrl->bytes_ready = bytes_ready;
+
+            if(XHC_COPY_SMSC_MAP == ctx->method
+                    || XHC_COPY_SMSC_NO_MAP == ctx->method) {
+                xc->comm_ctrl->data_vaddr = ctx->buf;
+
+                if(NULL != ctx->region_data) {
+                    xhc_copy_region_post((void *) xc->comm_ctrl->access_token,
+                        ctx->region_data);
+                }
+            }
+        }
+
+        /* Make sure the above stores complete
+         * before the one to the control flag */
+        xhc_atomic_wmb();
+
+        xc->comm_ctrl->seq = ctx->seq;
+        xc->op_state |= COMM_CTRL_INIT;
+    }
+}
+
+// ------------------------------------------------
+
+int mca_coll_xhc_bcast_init(void *buf, size_t count, ompi_datatype_t *datatype,
+        int root, ompi_communicator_t *ompi_comm, xhc_module_t *module,
+        xhc_bcast_ctx_t *ctx) {
+
+    ctx->buf = buf;
+    ctx->datacount = count;
+    ctx->root = root;
+    ctx->datatype = datatype;
+    ctx->ompi_comm = ompi_comm;
+    ctx->module = module;
+
+    ctx->rank = ompi_comm_rank(ompi_comm);
+
+    ctx->src_comm = NULL;
+
+    ctx->region_data = NULL;
+    ctx->reg = NULL;
+
+    ctx->bytes_done = 0;
+    ctx->bytes_avail = 0;
+
+    // --
+
+    size_t dtype_size;
+    ompi_datatype_type_size(datatype, &dtype_size);
+    ctx->bytes_total = count * dtype_size;
+
+    ctx->data = &module->op_data[XHC_BCAST];
+
+    ctx->seq = ++(ctx->data->seq);
+    ctx->comms = ctx->data->comms;
+
+    if(ctx->bytes_total <= XHC_BCAST_IMM_SIZE) {
+        ctx->method = XHC_COPY_IMM;
+        ctx->bytes_avail = ctx->bytes_total;
+    } else if(ctx->bytes_total <= ctx->comms[0].cico_size) {
+        ctx->method = XHC_COPY_CICO;
+        ctx->self_cico = xhc_get_cico(module->peer_info, ctx->rank);
+    } else {
+        ctx->method = (module->zcopy_map_support ?
+            XHC_COPY_SMSC_MAP : XHC_COPY_SMSC_NO_MAP);
+
+        int err = xhc_copy_expose_region(ctx->buf,
+            ctx->bytes_total, &ctx->region_data);
+        if(0 != err) {return OMPI_ERROR;}
+    }
+
+    // --
+
+    xhc_bcast_init_local(ctx->comms, ctx->module->peer_info,
+        ctx->rank, ctx->root, ctx->seq);
+
+    for(xhc_comm_t *xc = ctx->comms; xc; xc = xc->up) {
+        if(!xc->is_leader) {
+            ctx->src_comm = xc;
+            break;
+        }
+    }
+
+    return OMPI_SUCCESS;
+}
+
+int mca_coll_xhc_bcast_start(xhc_bcast_ctx_t *ctx) {
+    int err = OMPI_SUCCESS;
+
+    for(xhc_comm_t *xc = ctx->comms->top; xc; xc = xc->down) {
+        if(!xc->is_leader || (xc->op_state & COMM_PREV_FINI)) {
+            continue;
+        }
+
+        if(!CHECK_FLAG(&xc->comm_ctrl->ack, ctx->seq - 1, 0)) {
+            err = OMPI_ERR_WOULD_BLOCK;
+            continue;
+        }
+
+        /* Load-store control dependency between the load for comm ack
+         * and the stores in xhc_bcast_notify(); no barrier required */
+
+        xc->op_state |= COMM_PREV_FINI;
+
+        /* If comm ctrl is not initialized here/now through xhc_bcast_notify(),
+         * it will be later, lazily. With XHC_COPY_SMSC_MAP, children attach to
+         * their parents' buffers while waiting for the bytes_ready signal. In
+         * other cases, they don't do anything after receiving comm seq and
+         * before polling bytes_ready; avoid writing the cache line and thus
+         * triggering the cache coherency protocol twice. */
+
+        if(ctx->bytes_done > 0 || XHC_COPY_SMSC_MAP == ctx->method) {
+            xhc_bcast_notify(ctx, xc, ctx->bytes_done);
+        }
+    }
+
+    return err;
+}
+
+int mca_coll_xhc_bcast_work(xhc_bcast_ctx_t *ctx) {
+    xhc_comm_t *src_comm = ctx->src_comm;
+    xhc_comm_ctrl_t *src_ctrl = src_comm->comm_ctrl;
+
+    xhc_peer_info_t *peer_info = ctx->module->peer_info;
+
+    int err;
+
+    // ---
+
+    // Check if leader has joined (without blocking)
+    if(!(src_comm->op_state & COMM_CTRL_INIT)) {
+        if(CHECK_FLAG(&src_ctrl->seq, ctx->seq, 0)) {
+            switch(ctx->method) {
+                case XHC_COPY_IMM:
+                    ctx->src_buffer = (void *) src_ctrl->imm_data;
+                    break;
+
+                case XHC_COPY_CICO:
+                    ctx->src_buffer = xhc_get_cico(peer_info, src_ctrl->leader_rank);
+                    if(NULL == ctx->src_buffer) {return OMPI_ERR_OUT_OF_RESOURCE;}
+                    break;
+
+                case XHC_COPY_SMSC_MAP:
+                    ctx->src_buffer = xhc_get_registration(
+                        &ctx->module->peer_info[src_ctrl->leader_rank],
+                        src_ctrl->data_vaddr, ctx->bytes_total, &ctx->reg);
+                    if(NULL == ctx->src_buffer) {return OMPI_ERROR;}
+                    break;
+
+                case XHC_COPY_SMSC_NO_MAP:
+                    ctx->src_buffer = src_ctrl->data_vaddr;
+                    break;
+
+                default:
+                    assert(0);
+            }
+
+            src_comm->op_state |= COMM_CTRL_INIT;
+        } else {
+            return OPAL_ERR_WOULD_BLOCK;
+        }
+
+        xhc_atomic_rmb();
+    }
+
+    // ---
+
+    size_t copy_size = opal_min(src_comm->chunk_size,
+        ctx->bytes_total - ctx->bytes_done);
+
+    // Check if data's ready to be copied (without blocking)
+    if(ctx->bytes_avail < copy_size) {
+        ctx->bytes_avail = xhc_atomic_load_size_t(
+            &src_ctrl->bytes_ready) - ctx->bytes_done;
+
+        if(ctx->bytes_avail < copy_size) {
+            return OMPI_ERR_WOULD_BLOCK;
+        }
+
+        xhc_atomic_rmb();
+    }
+
+    void *data_src = (char *) ctx->src_buffer + ctx->bytes_done;
+    void *data_dst = (char *) ctx->buf + ctx->bytes_done;
+    void *cico_dst = (char *) ctx->self_cico + ctx->bytes_done;
+
+    bool is_leader = (ctx->comms[0].is_leader);
+
+    /* Pipelined copy not necessary when
+     * non-leader; copy all at once. */
+    if(!is_leader) {
+        copy_size = ctx->bytes_avail;
+    }
+
+    switch(ctx->method) {
+        case XHC_COPY_IMM:
+        case XHC_COPY_SMSC_MAP:
+            xhc_memcpy(data_dst, data_src, copy_size);
+            break;
+
+        case XHC_COPY_CICO:
+            xhc_memcpy((is_leader ? cico_dst : data_dst), data_src, copy_size);
+            break;
+
+        case XHC_COPY_SMSC_NO_MAP:
+            err = xhc_copy_from(&peer_info[src_ctrl->leader_rank], data_dst,
+                data_src, copy_size, (void *) src_ctrl->access_token);
+            if(0 != err) {return OMPI_ERROR;}
+
+            break;
+
+        default:
+            assert(0);
+    }
+
+    ctx->bytes_done += copy_size;
+    ctx->bytes_avail -= copy_size;
+
+    /* Make sure the memcpy has completed before
+     * writing to bytes_ready (in xhc_bcast_notify) */
+    xhc_atomic_wmb();
+
+    // Notify any children
+    for(xhc_comm_t *xc = src_comm->down; xc; xc = xc->down) {
+        xhc_bcast_notify(ctx, xc, ctx->bytes_done);
+    }
+
+    if(XHC_COPY_CICO == ctx->method && is_leader) {
+        xhc_memcpy(data_dst, cico_dst, copy_size);
+    }
+
+    return OMPI_SUCCESS;
+}
+
+void mca_coll_xhc_bcast_ack(xhc_bcast_ctx_t *ctx) {
+
+    // Set personal ack(s)
+    for(xhc_comm_t *xc = ctx->comms; xc; xc = xc->up) {
+        xc->my_ctrl->ack = ctx->seq;
+
+        if(!xc->is_leader) {
+            break;
+        }
+    }
+
+    // Gather members' acks and set comm ack
+    for(xhc_comm_t *xc = ctx->comms; xc; xc = xc->up) {
+        if(!xc->is_leader) {
+            break;
+        }
+
+        /* We are about to write this after gathering
+         * the members' acks, let's prefetch it now! */
+        xhc_prefetchw((void *) &xc->comm_ctrl->ack,
+            sizeof(xc->comm_ctrl->ack), 1);
+
+        for(int m = 0; m < xc->size; m++) {
+            if(m == xc->my_id) {
+                continue;
+            }
+
+            WAIT_FLAG(&xc->member_ctrl[m].ack, ctx->seq, 0);
+        }
+
+        xc->comm_ctrl->ack = ctx->seq;
+    }
+}
+
+void mca_coll_xhc_bcast_fini(xhc_bcast_ctx_t *ctx) {
+    if(ctx->reg) {
+        xhc_return_registration(ctx->reg);
+    }
+
+    if(ctx->region_data) {
+        xhc_copy_close_region(ctx->region_data);
+    }
+
+    /* The operation is done, all children have copied from the leader's CICO
+     * buffer, and thus the cache lines comprising it are in various caches,
+     * probably in shared state. Do an RFO-prefetch, to claim them back in the
+     * local cache in exclusive state, to speed up writing to the CICO buffer
+     * in the next op. Specify prefetch to L2, to try to avoid cache pollution */
+    if(XHC_COPY_CICO == ctx->method && ctx->comms[0].is_leader) {
+        xhc_prefetchw(ctx->self_cico, ctx->bytes_total, 2);
+    }
+}
+
+// ------------------------------------------------
+
+int mca_coll_xhc_bcast(void *buf, size_t count, ompi_datatype_t *datatype, int root,
+        ompi_communicator_t *ompi_comm, mca_coll_base_module_t *ompi_module) {
+
+    xhc_module_t *module = (xhc_module_t *) ompi_module;
+
+    // ---
+
+    if(!ompi_datatype_is_predefined(datatype)) {
+        WARN_ONCE("coll:xhc: Warning: XHC does not currently support "
+            "derived datatypes; utilizing fallback component");
+        goto _fallback;
+    }
+
+    if(!module->zcopy_support) {
+        size_t dtype_size; ompi_datatype_type_size(datatype, &dtype_size);
+        size_t cico_size = module->op_config[XHC_BCAST].cico_max;
+        if(count * dtype_size > cico_size) {
+            WARN_ONCE("coll:xhc: Warning: No smsc support; utilizing fallback "
+                "component for bcast greater than %zu bytes", cico_size);
+            goto _fallback;
+        }
+    }
+
+    // ---
+
+    xhc_bcast_ctx_t ctx;
+    int err;
+
+    if(!module->op_data[XHC_BCAST].init) {
+        err = xhc_init_op(module, ompi_comm, XHC_BCAST);
+        if(OMPI_SUCCESS != err) {goto _fallback_permanent;}
+    }
+
+    err = xhc_bcast_init(buf, count, datatype,
+        root, ompi_comm, module, &ctx);
+    if(OMPI_SUCCESS != err) {return err;}
+
+    /* Safe to alter the CICO buffer without checking any flags,
+     * because this is this rank's personal buffer. In any past
+     * ops where others copied from it, the rank has gathered
+     * acks that these copies have completed. */
+    if(ctx.rank == root && XHC_COPY_CICO == ctx.method) {
+        xhc_memcpy(ctx.self_cico, ctx.buf, ctx.bytes_total);
+    }
+
+    if(ctx.rank == root) {
+        ctx.bytes_done = ctx.bytes_total;
+    }
+
+    while((err = xhc_bcast_start(&ctx)) != OMPI_SUCCESS) {
+        if(OMPI_ERR_WOULD_BLOCK != err) {return err;}
+    }
+
+    while(ctx.bytes_done < ctx.bytes_total) {
+        err = xhc_bcast_work(&ctx);
+
+        if(OMPI_SUCCESS != err && OMPI_ERR_WOULD_BLOCK != err) {
+            return err;
+        }
+    }
+
+    xhc_bcast_ack(&ctx);
+    xhc_bcast_fini(&ctx);
+
+    return OMPI_SUCCESS;
+
+    // ---
+
+_fallback_permanent:
+
+    XHC_INSTALL_FALLBACK(module,
+        ompi_comm, XHC_BCAST, bcast);
+
+_fallback:
+
+    return XHC_CALL_FALLBACK(module->prev_colls, XHC_BCAST,
+        bcast, buf, count, datatype, root, ompi_comm);
+}
diff --git a/ompi/mca/coll/xhc/coll_xhc_comm.c b/ompi/mca/coll/xhc/coll_xhc_comm.c
new file mode 100644
index 00000000000..e5948bceb03
--- /dev/null
+++ b/ompi/mca/coll/xhc/coll_xhc_comm.c
@@ -0,0 +1,399 @@
+/*
+ * Copyright (c) 2021-2024 Computer Architecture and VLSI Systems (CARV)
+ *                         Laboratory, ICS Forth. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "mpi.h"
+
+#include "ompi/constants.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/coll/base/base.h"
+#include "opal/mca/shmem/base/base.h"
+#include "opal/util/minmax.h"
+
+#include "coll_xhc.h"
+
+// ------------------------------------------------
+
+/* This is the method that constructs XHC's hierarchies. It receives a
+ * config object that contains the specs (a list of localities) for the
+ * new hierarchiy. The algorithm groups ranks together according to it,
+ * and outputs the resulting list of xhc communicators. See also the
+ * inline comments for more specific implementation details. */
+int mca_coll_xhc_comms_make(ompi_communicator_t *ompi_comm,
+        xhc_module_t *module, xhc_op_config_t *config, xhc_op_data_t *data) {
+
+    xhc_peer_info_t *peer_info = module->peer_info;
+    xhc_coll_fns_t xhc_fns;
+
+    int rank = ompi_comm_rank(ompi_comm);
+    int n_ranks = ompi_comm_size(ompi_comm);
+
+    xhc_comm_t *comms = NULL;
+    int comms_size = 0;
+    int comm_count = 0;
+
+    opal_shmem_ds_t *ds_list;
+    bool *candidate_list;
+
+    size_t smsc_reg_size = 0;
+
+    int n_slices = 1;
+
+    int return_code = OMPI_SUCCESS;
+    int err;
+
+    xhc_module_set_coll_fns(ompi_comm, &module->prev_colls, &xhc_fns);
+
+    comms = malloc((comms_size = 5) * sizeof(xhc_comm_t));
+    ds_list = malloc(n_ranks * sizeof(opal_shmem_ds_t));
+    candidate_list = malloc(n_ranks * sizeof(bool));
+
+    if(!comms || !ds_list || !candidate_list) {
+        RETURN_WITH_ERROR(return_code, OMPI_ERR_OUT_OF_RESOURCE, end);
+    }
+
+    if(mca_smsc_base_has_feature(MCA_SMSC_FEATURE_REQUIRE_REGISTRATION)) {
+        smsc_reg_size = mca_smsc_base_registration_data_size();
+    }
+
+    // MPI_Reduce implementation is 'multi-sliced'!
+    if(XHC_REDUCE == data->colltype) {
+        n_slices = 2;
+    }
+
+    // Initialize seq in a way that will have the first op use slice 0
+    data->seq = n_slices - 1;
+
+    /* We process each locality in the spec in order, and place all ranks
+     * that share it in the same group. The one amongst them with the lowest
+     * rank number becomes the 'owner' of the group, i.e. the one that will
+     * allocate the shared resources. The owner is also the de facto leader,
+     * only stepping aside in cases of dynamic leadership, or when another
+     * rank in the xhc comm is the root of the collective.
+     *
+     * For each locality, only the ranks that where owners in the comms
+     * resulting from the preceding locality are considered on this one. */
+
+    for(int h = 0; h < config->hierarchy_len; h++) {
+        xhc_comm_t *xc = &comms[comm_count];
+
+        if(comm_count == comms_size) {
+            void *tmp = realloc(comms, (comms_size *= 2) * sizeof(xhc_comm_t));
+            if(!tmp) {RETURN_WITH_ERROR(return_code, OMPI_ERR_OUT_OF_RESOURCE, end);}
+            comms = tmp;
+        }
+
+        *xc = (xhc_comm_t) {
+            .locality = config->hierarchy[h],
+
+            .chunk_size = (comm_count < config->chunks_len ?
+                config->chunks[comm_count] : comms[comm_count-1].chunk_size),
+
+            .cico_size = config->cico_max,
+
+            .my_id = -1,
+            .size = 0,
+
+            .owner_rank = -1,
+
+            .slices = NULL,
+            .n_slices = n_slices,
+
+            .reduce_queue = NULL,
+            .reduce_buffer = NULL
+        };
+
+        xc->slices = calloc(xc->n_slices, sizeof(xhc_sh_slice_t));
+        if(!xc->slices) {RETURN_WITH_ERROR(return_code, OMPI_ERR_OUT_OF_RESOURCE, end);}
+
+        // ----
+
+        /* Only ranks that were leaders in the previous level are candidates
+         * for this one. Every rank advertises whether others may consider
+         * it for inclusion on this one via an Allgather. */
+
+        bool is_candidate = (0 == comm_count
+            || rank == comms[comm_count - 1].owner_rank);
+
+        err = ompi_comm->c_coll->coll_allgather(&is_candidate, 1,
+            MPI_C_BOOL, candidate_list, 1, MPI_C_BOOL,
+            ompi_comm, ompi_comm->c_coll->coll_allgather_module);
+        if(OMPI_SUCCESS != err) {
+            RETURN_WITH_ERROR(return_code, err, comm_error);
+        }
+
+        for(int r = 0; r < n_ranks; r++) {
+            /* Only consider ranks that were leaders on the previous comm.
+             * Don't get tempted to omit this check for the bottom comm; even
+             * if this is the local's rank's bottom comm, it may not be for a
+             * peer of his (e.g. with some non-symmetric hierarchies). */
+            if(false == candidate_list[r]) {
+                continue;
+            }
+
+            // Non-local --> not part of the comm :/
+            if(!PEER_IS_LOCAL(peer_info, r, xc->locality)) {
+                continue;
+            }
+
+            /* The member ID will mean slightly different things whether on
+             * the bottom comm or on higher up ones. On the bottom comm,
+             * each member ID corresponds to a single rank. On higher-up comms,
+             * each member ID represents not a single process, but a whole
+             * comm for the preceding level. */
+            if(r == rank || (comm_count > 0 && r == comms[comm_count - 1].owner_rank)) {
+                xc->my_id = xc->size;
+            }
+
+            // First rank to join the comm becomes the owner
+            if(-1 == xc->owner_rank) {
+                xc->owner_rank = r;
+            }
+
+            xc->size++;
+        }
+
+        /* If there are no local peers in regards to this locality, no
+         * XHC comm is created for this process on this level. */
+        if(xc->size <= 1) {
+            opal_output_verbose(MCA_BASE_VERBOSE_WARN,
+                ompi_coll_base_framework.framework_output,
+                "coll:xhc: Warning: Locality 0x%04x does not result "
+                "in any new groupings; skipping it", xc->locality);
+
+            /* Even though there was no other rank local to this one for
+             * this locality, and thus no XHC comm was created, this might
+             * not be the case for foreign ranks. We are obligated to
+             * participate in the Allgather they'll do on the ompi comm in
+             * order to share control structurs, even if it's useless to us. */
+
+            err = ompi_comm->c_coll->coll_allgather(&xc->comm_ds,
+                sizeof(opal_shmem_ds_t), MPI_BYTE, ds_list,
+                sizeof(opal_shmem_ds_t), MPI_BYTE, ompi_comm,
+                ompi_comm->c_coll->coll_allgather_module);
+            if(OMPI_SUCCESS != err) {
+                RETURN_WITH_ERROR(return_code, err, comm_error);
+            }
+
+            xhc_comms_destroy(xc, 1);
+            continue;
+        }
+
+        // ----
+
+        /* Init comm stuff */
+
+        xc->member_info = calloc(xc->size, sizeof(xhc_member_info_t));
+        if(NULL == xc->member_info) {
+            RETURN_WITH_ERROR(return_code, OMPI_ERR_OUT_OF_RESOURCE, comm_error);
+        }
+
+        xc->my_info = &xc->member_info[xc->my_id];
+
+        if(XHC_REDUCE == data->colltype || XHC_ALLREDUCE == data->colltype) {
+            xc->reduce_queue = OBJ_NEW(opal_list_t);
+            if(!xc->reduce_queue) {RETURN_WITH_ERROR(return_code,
+                OMPI_ERR_OUT_OF_RESOURCE, comm_error);}
+
+            for(int m = 0; m < xc->size - 1; m++) {
+                xhc_rq_item_t *item = OBJ_NEW(xhc_rq_item_t);
+                if(!item) {RETURN_WITH_ERROR(return_code,
+                    OMPI_ERR_OUT_OF_RESOURCE, comm_error);}
+
+                opal_list_append(xc->reduce_queue, (opal_list_item_t *) item);
+            }
+        }
+
+        // ----
+
+        char *ds_base = NULL;
+
+        // Create shared structs
+        if(rank == xc->owner_rank) {
+            size_t ds_len = sizeof(xhc_comm_ctrl_t) + smsc_reg_size
+                + xc->size * sizeof(xhc_member_ctrl_t) * xc->n_slices;
+
+            if(XHC_REDUCE == data->colltype || XHC_ALLREDUCE == data->colltype) {
+                ds_len += xc->size * xc->cico_size * xc->n_slices;
+            }
+
+            ds_base = xhc_shmem_create(&xc->comm_ds, ds_len,
+                ompi_comm, "ctrl", data->colltype, comm_count);
+            if(NULL == ds_base) {
+                RETURN_WITH_ERROR(return_code, OMPI_ERROR, comm_error);
+            }
+
+            /* Manually 'touch' to assert allocation in local NUMA node
+            * (assuming linux's default first-touch-alloc NUMA policy) */
+            memset(ds_base, 0, ds_len);
+
+            /* Initialize comm/member ctrl */
+
+            xhc_comm_ctrl_t *c_ctrl = (void *) ds_base;
+
+            for(int s = 0; s < n_slices; s++) {
+                xhc_member_ctrl_t *m_ctrl = (void *) (ds_base
+                    + sizeof(xhc_comm_ctrl_t) + smsc_reg_size
+                    + xc->size * sizeof(xhc_member_ctrl_t) * s);
+
+                for(int m = 0; m < xc->size; m++) {
+                    /* At the beggining of an OP with seq 'x', a member
+                     * will check for completion of previous ops using
+                     * this slice by looking for an ack of 'x - n_slices'.
+                     * The first time ever that each slice _s_ will be
+                     * used, the seq number will be 'n_slices + s'
+                     * (data->seq is initialized to n_slices - 1). */
+                    m_ctrl[m] = (xhc_member_ctrl_t) {
+                        .seq = (xf_sig_t) s,
+                        .ack = (xf_sig_t) s,
+                    };
+                }
+            }
+
+            *c_ctrl = (xhc_comm_ctrl_t) {
+                .seq = data->seq,
+                .ack = data->seq,
+                .leader_seq = data->seq
+            };
+        }
+
+        /* The comm's owners share the details of the communication structs
+         * with their children, so that they may attach to them. There's no
+         * MPI communicator that only includes the member of the XHC comm,
+         * so a single Allgather on the original MPI comm is preformed. */
+
+        err = ompi_comm->c_coll->coll_allgather(&xc->comm_ds,
+            sizeof(opal_shmem_ds_t), MPI_BYTE, ds_list,
+            sizeof(opal_shmem_ds_t), MPI_BYTE, ompi_comm,
+            ompi_comm->c_coll->coll_allgather_module);
+        if(OMPI_SUCCESS != err) {
+            RETURN_WITH_ERROR(return_code, err, comm_error);
+        }
+
+        // Attach to owner's shared structs
+        if(rank != xc->owner_rank) {
+            xc->comm_ds = ds_list[xc->owner_rank];
+
+            ds_base = xhc_shmem_attach(&xc->comm_ds);
+            if(NULL == ds_base) {
+                RETURN_WITH_ERROR(return_code, OMPI_ERROR, comm_error);
+            }
+        }
+
+        // ----
+
+        xc->comm_ctrl_base = (void *) ds_base;
+        xc->member_ctrl_base = (void *) (ds_base
+            + sizeof(xhc_comm_ctrl_t) + smsc_reg_size);
+
+        if(XHC_REDUCE == data->colltype || XHC_ALLREDUCE == data->colltype) {
+            xc->reduce_buffer_base = (void *) (ds_base
+                + sizeof(xhc_comm_ctrl_t) + smsc_reg_size
+                + xc->size * sizeof(xhc_member_ctrl_t) * xc->n_slices);
+        }
+
+        // Assign to first slice by default
+        xc->comm_ctrl = xc->comm_ctrl_base;
+        xc->member_ctrl = xc->member_ctrl_base;
+        xc->my_ctrl = &xc->member_ctrl[xc->my_id];
+        xc->reduce_buffer = xc->reduce_buffer_base;
+
+        // ----
+
+        comm_count++;
+        continue;
+
+        comm_error: {
+            xhc_comms_destroy(comms, comm_count+1);
+            comm_count = -1;
+
+            goto end;
+        }
+    }
+
+    for(int i = 0; i < comm_count; i++) {
+        comms[i].up = (i < comm_count - 1 ? &comms[i + 1] : NULL);
+        comms[i].down = (i > 0 ? &comms[i - 1] : NULL);
+
+        comms[i].top = &comms[comm_count - 1];
+        comms[i].bottom = &comms[0];
+
+        comms[i].is_top = (NULL == comms[i].up);
+        comms[i].is_bottom = (NULL == comms[i].down);
+    }
+
+    if(config->chunks_len > 1 && config->chunks_len < comm_count) {
+        opal_output_verbose(MCA_BASE_VERBOSE_WARN,
+            ompi_coll_base_framework.framework_output,
+            "coll:xhc: Warning: The chunk sizes count (%d) is shorter than the "
+            "hierarchy size (%d); filling in with the last entry provided",
+            config->chunks_len, comm_count);
+    } else if(config->chunks_len > comm_count) {
+        opal_output_verbose(MCA_BASE_VERBOSE_WARN,
+            ompi_coll_base_framework.framework_output,
+            "coll:xhc: Warning: The chunk size count (%d) is larger than the "
+            "hierarchy size (%d); omitting last entries",
+            config->chunks_len, comm_count);
+    }
+
+    // --
+
+    REALLOC(comms, comm_count, xhc_comm_t);
+
+    data->comms = comms;
+    data->comm_count = comm_count;
+
+    // --
+
+    end:
+
+    xhc_module_set_coll_fns(ompi_comm, &xhc_fns, NULL);
+
+    free(candidate_list);
+    free(ds_list);
+
+    if(OMPI_SUCCESS != return_code) {
+        free(comms);
+    }
+
+    return return_code;
+}
+
+void mca_coll_xhc_comms_destroy(xhc_comm_t *comms, int comm_count) {
+    bool is_owner = true;
+
+    for(int i = 0; i < comm_count; i++) {
+        xhc_comm_t *xc = &comms[i];
+
+        if(0 != xc->my_id) {
+            is_owner = false;
+        }
+
+        free(xc->slices);
+        free(xc->member_info);
+
+        if(xc->reduce_queue) {
+            OPAL_LIST_RELEASE(xc->reduce_queue);
+        }
+
+        if(xc->comm_ctrl) {
+            if(is_owner) {
+                // OMPI issue #11123
+                // opal_shmem_unlink(&xc->comm_ds);
+                (void) is_owner;
+            }
+
+            opal_shmem_segment_detach(&xc->comm_ds);
+        }
+
+        *xc = (xhc_comm_t) {0};
+    }
+}
diff --git a/ompi/mca/coll/xhc/coll_xhc_component.c b/ompi/mca/coll/xhc/coll_xhc_component.c
new file mode 100644
index 00000000000..bfd03259de4
--- /dev/null
+++ b/ompi/mca/coll/xhc/coll_xhc_component.c
@@ -0,0 +1,954 @@
+/*
+ * Copyright (c) 2021-2024 Computer Architecture and VLSI Systems (CARV)
+ *                         Laboratory, ICS Forth. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "mpi.h"
+
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/coll/base/base.h"
+#include "ompi/mca/coll/base/coll_base_functions.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+
+#include "opal/include/opal/align.h"
+#include "opal/mca/shmem/base/base.h"
+#include "opal/util/show_help.h"
+
+#include "coll_xhc.h"
+
+// -----------------------------
+
+typedef int (*csv_parse_conv_fn_t)(char *str, void *dst);
+typedef void (*csv_parse_destruct_fn_t)(void *data);
+
+static int xhc_register(void);
+static int xhc_var_check_exclusive(const char *param_a, const char *param_b);
+
+// -----------------------------
+
+static const char *xhc_topo_str[] = {
+    "node", "flat",
+    "socket",
+    "numa",
+    "l3", "l3cache",
+    "l2", "l2cache",
+    "l1", "l1cache",
+    "core",
+    "hwthread", "thread"
+};
+
+static const xhc_loc_t xhc_topo_val[] = {
+    OPAL_PROC_ON_NODE, OPAL_PROC_ON_NODE,
+    OPAL_PROC_ON_SOCKET,
+    OPAL_PROC_ON_NUMA,
+    OPAL_PROC_ON_L3CACHE, OPAL_PROC_ON_L3CACHE,
+    OPAL_PROC_ON_L2CACHE, OPAL_PROC_ON_L2CACHE,
+    OPAL_PROC_ON_L1CACHE, OPAL_PROC_ON_L1CACHE,
+    OPAL_PROC_ON_CORE,
+    OPAL_PROC_ON_HWTHREAD, OPAL_PROC_ON_HWTHREAD
+};
+
+static const COLLTYPE_T xhc_colltype_to_universal_map[XHC_COLLCOUNT] = {
+    [XHC_BCAST] = BCAST,
+    [XHC_BARRIER] = BARRIER,
+    [XHC_REDUCE] = REDUCE,
+    [XHC_ALLREDUCE] = ALLREDUCE
+};
+
+static const char *xhc_config_source_to_str_map[XHC_CONFIG_SOURCE_COUNT] = {
+    [XHC_CONFIG_SOURCE_INFO_GLOBAL] = "info/global",
+    [XHC_CONFIG_SOURCE_INFO_OP] = "info/op",
+    [XHC_CONFIG_SOURCE_MCA_GLOBAL] = "mca/global",
+    [XHC_CONFIG_SOURCE_MCA_OP] = "mca/op"
+};
+
+const char *mca_coll_xhc_component_version_string =
+    "Open MPI xhc collective MCA component version " OMPI_VERSION;
+
+mca_coll_xhc_component_t mca_coll_xhc_component = {
+    .super = {
+        .collm_version = {
+            MCA_COLL_BASE_VERSION_3_0_0,
+
+            .mca_component_name = "xhc",
+            MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION,
+                OMPI_MINOR_VERSION, OMPI_RELEASE_VERSION),
+
+            .mca_register_component_params = xhc_register,
+        },
+
+        .collm_data = {
+            MCA_BASE_METADATA_PARAM_CHECKPOINT
+        },
+
+        .collm_init_query = mca_coll_xhc_component_init_query,
+        .collm_comm_query = mca_coll_xhc_module_comm_query,
+    },
+
+    .priority = 0,
+    .print_info = 0,
+
+    .shmem_backing = NULL,
+
+    .memcpy_chunk_size = 256 << 10,
+
+    .dynamic_leader = false,
+
+    .barrier_root = 0,
+    .allreduce_root = 0,
+
+    .dynamic_reduce = XHC_DYNAMIC_REDUCE_NON_FLOAT,
+    .reduce_load_balance = (XHC_REDUCE_LB_LEADER_ASSIST_TOP_LEVEL
+        | XHC_REDUCE_LB_LEADER_ASSIST_FIRST_CHUNK),
+
+    .uniform_chunks = true,
+    .uniform_chunks_min = 4096,
+
+    .op_mca = {{0}},
+    .op_mca_global = {0}
+};
+
+/* Rather than having the defaults directly inside the component, we keep
+ * them in a separate structure and copy them over (in xhc_register()). The
+ * structs in the component are used as storage for the MCA variables, and
+ * the MCA system will nullify the storage of string variables when it is
+ * teared down during Finalize. This is a problem if we have multiple MPI
+ * Sessions, as we'll have lost our defaults the next time we attempt to
+ * initialize our MCA variables at the second Init. */
+static xhc_op_mca_t op_mca_default[XHC_COLLCOUNT] = {
+     [XHC_BCAST] = {
+        .hierarchy = "numa,socket",
+        .chunk_size = "16K",
+        .cico_max = 256
+    },
+
+    [XHC_BARRIER] = {
+        .hierarchy = "numa,socket",
+        .chunk_size = "1",
+        .cico_max = 0
+    },
+
+    [XHC_REDUCE] = {
+        .hierarchy = "l3,numa,socket",
+        .chunk_size = "16K",
+        .cico_max = 4096
+    },
+
+    [XHC_ALLREDUCE] = {
+        .hierarchy = "l3,numa,socket",
+        .chunk_size = "16K",
+        .cico_max = 4096
+    }
+};
+static xhc_op_mca_t op_mca_global_default = {0};
+
+// -----------------------------
+
+/* Initial query function that is invoked during MPI_INIT, allowing
+ * this component to disqualify itself if it doesn't support the
+ * required level of thread support. */
+int mca_coll_xhc_component_init_query(bool enable_progress_threads,
+        bool enable_mpi_threads) {
+    return OMPI_SUCCESS;
+}
+
+COLLTYPE_T mca_coll_xhc_colltype_to_universal(XHC_COLLTYPE_T xhc_colltype) {
+    if(xhc_colltype < 0 || xhc_colltype >= XHC_COLLCOUNT) {
+        return -1;
+    }
+
+    return xhc_colltype_to_universal_map[xhc_colltype];
+}
+
+const char *mca_coll_xhc_colltype_to_str(XHC_COLLTYPE_T colltype) {
+    return mca_coll_base_colltype_to_str(xhc_colltype_to_universal(colltype));
+}
+
+const char *mca_coll_xhc_config_source_to_str(xhc_config_source_t source) {
+    return (source >= 0 && source < XHC_CONFIG_SOURCE_COUNT ?
+        xhc_config_source_to_str_map[source] : NULL);
+}
+
+static int xhc_register(void) {
+    mca_base_var_enum_t *var_enum;
+    mca_base_var_enum_flag_t *var_enum_flag;
+    const mca_base_var_t *var = NULL;
+    int vari;
+
+    char *tmp, *name, *desc;
+    int err;
+
+    /* Priority */
+    // -----------
+
+    mca_base_component_var_register(&mca_coll_xhc_component.super.collm_version,
+        "priority", "Priority of the xhc component.",
+        MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_2,
+        MCA_BASE_VAR_SCOPE_READONLY, &mca_coll_xhc_component.priority);
+
+    /* SHM Backing dir */
+    // ------------------
+
+    mca_coll_xhc_component.shmem_backing = (0 == access("/dev/shm", W_OK) ?
+        "/dev/shm" : opal_process_info.job_session_dir);
+
+    mca_base_component_var_register(&mca_coll_xhc_component.super.collm_version,
+        "shmem_backing", "Directory to place backing files for shared-memory"
+        " control-data communication.", MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
+        OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_READONLY,
+        &mca_coll_xhc_component.shmem_backing);
+
+    /* Memcpy limit (see smsc_xpmem_memcpy_chunk_size) */
+    // --------------------------------------------------
+
+    int var_index = mca_base_var_find("opal", "smsc", "xpmem", "memcpy_chunk_size");
+
+    if(var_index >= 0) {
+        size_t *var_value_ptr = NULL;
+
+        err = mca_base_var_get_value(var_index, &var_value_ptr, NULL, NULL);
+
+        if(OPAL_SUCCESS == err) {
+            mca_coll_xhc_component.memcpy_chunk_size = *var_value_ptr;
+        } else {
+            opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT,
+                ompi_coll_base_framework.framework_output, "coll:xhc:component: "
+                "Can't get smsc_xpmem_memcpy_chunk_size MCA param value "
+                "from var index %d (%d); using xhc default", var_index, err);
+        }
+    } else {
+        opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT,
+            ompi_coll_base_framework.framework_output, "coll:xhc:component: "
+            "Can't find smsc_xpmem_memcpy_chunk_size MCA param (%d); "
+            "using xhc default", var_index);
+    }
+
+    mca_base_component_var_register(&mca_coll_xhc_component.super.collm_version,
+        "memcpy_chunk_size", "Maximum size to copy with a single call to memcpy, "
+        "following smsc/xpmem's paradigm. A smaller/larger value may provide "
+        "better performance on some systems.", MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0,
+        MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_READONLY,
+        &mca_coll_xhc_component.memcpy_chunk_size);
+
+    /* Print Info */
+    // -------------
+
+    mca_base_var_enum_value_flag_t print_info_options[XHC_COLLCOUNT + 4] = {
+        [XHC_COLLCOUNT + 0] = {XHC_PRINT_INFO_HIER_DOT, "dot", 0},
+        [XHC_COLLCOUNT + 1] = {XHC_PRINT_INFO_CONFIG, "config", 0},
+        [XHC_COLLCOUNT + 2] = {XHC_PRINT_INFO_ALL, "all", 0},
+        [XHC_COLLCOUNT + 3] = {0, NULL, 0}
+    };
+
+    for(int t = 0; t < XHC_COLLCOUNT; t++) {
+        print_info_options[t] = (mca_base_var_enum_value_flag_t) {
+            1 << t, xhc_colltype_to_str(t), 0};
+    }
+
+    err = mca_base_var_enum_create_flag("coll_xhc_print_info",
+        print_info_options, &var_enum_flag);
+    if(OPAL_SUCCESS != err) {return err;}
+
+    mca_base_component_var_register(&mca_coll_xhc_component.super.collm_version,
+        "print_info", "Print information during initialization.",
+        MCA_BASE_VAR_TYPE_UNSIGNED_INT, &var_enum_flag->super, 0, 0, OPAL_INFO_LVL_3,
+        MCA_BASE_VAR_SCOPE_READONLY, &mca_coll_xhc_component.print_info);
+
+    if(mca_coll_xhc_component.print_info & XHC_PRINT_INFO_ALL) {
+        for(mca_base_var_enum_value_flag_t *flag = print_info_options;
+                flag->string; flag++) {
+            mca_coll_xhc_component.print_info |= flag->flag;
+        }
+    }
+
+    OBJ_RELEASE(var_enum_flag);
+
+    /* Root ranks for unrooted collectives */
+    // --------------------------------------
+
+    mca_base_component_var_register(&mca_coll_xhc_component.super.collm_version,
+        "barrier_root", "Internal root for the barrier operation (rank ID).",
+        MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
+        MCA_BASE_VAR_SCOPE_READONLY, &mca_coll_xhc_component.barrier_root);
+
+    // Currently, only rank 0 is supported in allreduce.
+    mca_base_component_var_register(&mca_coll_xhc_component.super.collm_version,
+        "allreduce_root", "Internal root for the allreduce operation (rank ID).",
+        MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_DEFAULT_ONLY, OPAL_INFO_LVL_5,
+        MCA_BASE_VAR_SCOPE_READONLY, &mca_coll_xhc_component.allreduce_root);
+
+    /* Dynamic leader */
+    // -----------------
+
+    mca_base_component_var_register(&mca_coll_xhc_component.super.collm_version,
+        "dynamic_leader", "Enable dynamic operation-wise group-leader selection.",
+        MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_5,
+        MCA_BASE_VAR_SCOPE_READONLY, &mca_coll_xhc_component.dynamic_leader);
+
+    /* Dynamic reduce */
+    // -----------------
+
+    static mca_base_var_enum_value_t dynamic_reduce_options[] = {
+        {XHC_DYNAMIC_REDUCE_DISABLED, "disabled"},
+        {XHC_DYNAMIC_REDUCE_NON_FLOAT, "non-float"},
+        {XHC_DYNAMIC_REDUCE_ALL, "all"},
+        {0, NULL}
+    };
+
+    err = mca_base_var_enum_create("coll_xhc_dynamic_reduce_options",
+        dynamic_reduce_options, &var_enum);
+    if(OPAL_SUCCESS != err) {return err;}
+
+    mca_base_component_var_register(&mca_coll_xhc_component.super.collm_version,
+        "dynamic_reduce", "Dynamic/out-of-order intra-group reduction.",
+        MCA_BASE_VAR_TYPE_INT, var_enum, 0, 0, OPAL_INFO_LVL_6,
+        MCA_BASE_VAR_SCOPE_READONLY, &mca_coll_xhc_component.dynamic_reduce);
+
+    OBJ_RELEASE(var_enum);
+
+    /* Load balancing: Reduction leader assistance */
+    // ----------------------------------------------
+
+    mca_base_var_enum_value_flag_t reduce_load_balance_options[] = {
+        {XHC_REDUCE_LB_LEADER_ASSIST_TOP_LEVEL, "top", XHC_REDUCE_LB_LEADER_ASSIST_ALL},
+        {XHC_REDUCE_LB_LEADER_ASSIST_FIRST_CHUNK, "first", XHC_REDUCE_LB_LEADER_ASSIST_ALL},
+        {XHC_REDUCE_LB_LEADER_ASSIST_ALL, "all", (XHC_REDUCE_LB_LEADER_ASSIST_TOP_LEVEL |
+            XHC_REDUCE_LB_LEADER_ASSIST_FIRST_CHUNK)},
+        {0, NULL, 0}
+    };
+
+    err = mca_base_var_enum_create_flag("coll_xhc_reduce_load_balance",
+        reduce_load_balance_options, &var_enum_flag);
+    if(OPAL_SUCCESS != err) {return err;}
+
+    mca_base_component_var_register(&mca_coll_xhc_component.super.collm_version,
+        "reduce_load_balance", "Reduction leader assistance modes for load balancing.",
+        MCA_BASE_VAR_TYPE_INT, &var_enum_flag->super, 0, 0, OPAL_INFO_LVL_6,
+        MCA_BASE_VAR_SCOPE_READONLY, &mca_coll_xhc_component.reduce_load_balance);
+
+    OBJ_RELEASE(var_enum_flag);
+
+    /* (All)reduce uniform chunks */
+    // -----------------------------
+
+    mca_base_component_var_register(&mca_coll_xhc_component.super.collm_version,
+        "uniform_chunks", "Automatically optimize chunk size in reduction "
+        "collectives according to message size, for load balancing.",
+        MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_5,
+        MCA_BASE_VAR_SCOPE_READONLY, &mca_coll_xhc_component.uniform_chunks);
+
+    mca_base_component_var_register(&mca_coll_xhc_component.super.collm_version,
+        "uniform_chunks_min", "Minimum chunk size for reduction collectives, "
+        "when \"uniform chunks\" are enabled.", MCA_BASE_VAR_TYPE_SIZE_T,
+        NULL, 0, 0, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_READONLY,
+        &mca_coll_xhc_component.uniform_chunks_min);
+
+    /* Apply the op mca defaults. Gotta do it here rather than in-line in
+     * the registration loops below, as some iterations are skipped, for the
+     * variables that are not applicable (e.g. chunk size in Barrier). */
+
+    for(int t = 0; t < XHC_COLLCOUNT; t++) {
+        mca_coll_xhc_component.op_mca[t] = op_mca_default[t];
+    }
+    mca_coll_xhc_component.op_mca_global = op_mca_global_default;
+
+    /* Hierarchy */
+    // ------------
+
+    char *topo_list = NULL;
+
+    for(size_t i = 0; i < sizeof(xhc_topo_str)/sizeof(char *); i++) {
+        err = opal_asprintf(&tmp, "%s%s%s", (i > 0 ? topo_list : ""),
+            (i > 0 ? ", " : ""), xhc_topo_str[i]);
+        free(topo_list); topo_list = tmp;
+        if(err < 0) {return OPAL_ERR_OUT_OF_RESOURCE;}
+    }
+
+    err = opal_asprintf(&desc, "Comma-separated list of topology features to "
+        "consider for the hierarchy (%s), for all collectives. Mutually "
+        "exclusive with respective op-specific params.", topo_list);
+    if(err < 0) {free(topo_list); return OMPI_ERR_OUT_OF_RESOURCE;}
+
+    vari = mca_base_component_var_register(&mca_coll_xhc_component.super.collm_version,
+        "hierarchy", desc, MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_DEF_UNSET,
+        OPAL_INFO_LVL_4, MCA_BASE_VAR_SCOPE_READONLY,
+        &mca_coll_xhc_component.op_mca_global.hierarchy);
+
+    free(desc);
+
+    mca_base_var_get(vari, &var);
+
+    for(int t = 0; t < XHC_COLLCOUNT; t++) {
+        err = opal_asprintf(&name, "%s_hierarchy", xhc_colltype_to_str(t));
+        if(err < 0) {free(topo_list); return OPAL_ERR_OUT_OF_RESOURCE;}
+
+        err = opal_asprintf(&desc, "Comma-separated list of topology features to "
+            "consider for the hierarchy (%s), for %s.", topo_list, xhc_colltype_to_str(t));
+        if(err < 0) {free(topo_list); free(name); return OMPI_ERR_OUT_OF_RESOURCE;}
+
+        mca_base_component_var_register(&mca_coll_xhc_component.super.collm_version,
+            name, desc, MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_7,
+            MCA_BASE_VAR_SCOPE_READONLY, &mca_coll_xhc_component.op_mca[t].hierarchy);
+
+        /* The op-specific MCA vars are mutually exclusive with
+         * the global one. Otherwise things get complicated. */
+        err = xhc_var_check_exclusive("hierarchy", name);
+
+        free(name);
+        free(desc);
+
+        if(OPAL_SUCCESS != err) {
+            return err;
+        }
+
+        /* If the global MCA var was set (because the user set it, it's unset
+         * by default), it overrides the op-specific ones. (and both of them
+         * can't be set at the same time, as they are mutually exclusive. */
+        if(var && MCA_BASE_VAR_SOURCE_DEFAULT != var->mbv_source) {
+            mca_coll_xhc_component.op_mca[t].hierarchy =
+                strdup(mca_coll_xhc_component.op_mca_global.hierarchy);
+        }
+    }
+
+    free(topo_list);
+
+    /* Chunk size */
+    // ---------------
+
+    vari = mca_base_component_var_register(&mca_coll_xhc_component.super.collm_version,
+        "chunk_size", "Chunk size(s) for the pipeline (single value, or comma-separated "
+        "list for different hierarchy levels (bottom to top)), for all collectives. "
+        "Mutually exclusive with respective op-specific params.",
+        MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_DEF_UNSET, OPAL_INFO_LVL_5,
+        MCA_BASE_VAR_SCOPE_READONLY, &mca_coll_xhc_component.op_mca_global.chunk_size);
+
+    mca_base_var_get(vari, &var);
+
+    for(int t = 0; t < XHC_COLLCOUNT; t++) {
+        if(XHC_BARRIER == t) {
+            continue;
+        }
+
+        err = opal_asprintf(&name, "%s_chunk_size", xhc_colltype_to_str(t));
+        if(err < 0) {return OPAL_ERR_OUT_OF_RESOURCE;}
+
+        err = opal_asprintf(&desc, "Chunk size(s) for the pipeline (single "
+            "value, or comma-separated list for different hierarchy levels "
+            "(bottom to top)), for %s.", xhc_colltype_to_str(t));
+        if(err < 0) {free(name); return OMPI_ERR_OUT_OF_RESOURCE;}
+
+        mca_base_component_var_register(&mca_coll_xhc_component.super.collm_version,
+            name, desc, MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_8,
+            MCA_BASE_VAR_SCOPE_READONLY, &mca_coll_xhc_component.op_mca[t].chunk_size);
+
+        err = xhc_var_check_exclusive("chunk_size", name);
+
+        free(name);
+        free(desc);
+
+        if(OPAL_SUCCESS != err) {
+            return err;
+        }
+
+        if(var && MCA_BASE_VAR_SOURCE_DEFAULT != var->mbv_source) {
+            mca_coll_xhc_component.op_mca[t].chunk_size =
+                strdup(mca_coll_xhc_component.op_mca_global.chunk_size);
+        }
+    }
+
+    /* CICO threshold */
+    // -----------------
+
+    vari = mca_base_component_var_register(&mca_coll_xhc_component.super.collm_version,
+        "cico_max", "Maximum message size up to which to use CICO, for all collectives. "
+        "Mutually exclusive with respective op-specific params.",
+        MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, MCA_BASE_VAR_FLAG_DEF_UNSET, OPAL_INFO_LVL_5,
+        MCA_BASE_VAR_SCOPE_READONLY, &mca_coll_xhc_component.op_mca_global.cico_max);
+
+    mca_coll_xhc_component.op_mca_global.cico_max = OPAL_ALIGN(
+        mca_coll_xhc_component.op_mca_global.cico_max, XHC_ALIGN, size_t);
+
+    mca_base_var_get(vari, &var);
+
+    for(int t = 0; t < XHC_COLLCOUNT; t++) {
+        if(XHC_BARRIER == t) {
+            continue;
+        }
+
+        err = opal_asprintf(&name, "%s_cico_max", xhc_colltype_to_str(t));
+        if(err < 0) {return OPAL_ERR_OUT_OF_RESOURCE;}
+
+        err = opal_asprintf(&desc, "Maximum message size up to which "
+            "to use CICO, for %s.", xhc_colltype_to_str(t));
+        if(err < 0) {free(name); return OMPI_ERR_OUT_OF_RESOURCE;}
+
+        mca_base_component_var_register(&mca_coll_xhc_component.super.collm_version,
+            name, desc, MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0,
+            OPAL_INFO_LVL_8, MCA_BASE_VAR_SCOPE_READONLY,
+            &mca_coll_xhc_component.op_mca[t].cico_max);
+
+        mca_coll_xhc_component.op_mca[t].cico_max = OPAL_ALIGN(
+            mca_coll_xhc_component.op_mca[t].cico_max, XHC_ALIGN, size_t);
+
+        err = xhc_var_check_exclusive("cico_max", name);
+
+        free(name);
+        free(desc);
+
+        if(OPAL_SUCCESS != err) {
+            return err;
+        }
+
+        if(var && MCA_BASE_VAR_SOURCE_DEFAULT != var->mbv_source) {
+            mca_coll_xhc_component.op_mca[t].cico_max =
+                mca_coll_xhc_component.op_mca_global.cico_max;
+        }
+    }
+
+    return OMPI_SUCCESS;
+}
+
+static int xhc_var_check_exclusive(const char *param_a, const char *param_b) {
+    return mca_base_var_check_exclusive("ompi",
+        mca_coll_xhc_component.super.collm_version.mca_type_name,
+        mca_coll_xhc_component.super.collm_version.mca_component_name, param_a,
+        mca_coll_xhc_component.super.collm_version.mca_type_name,
+        mca_coll_xhc_component.super.collm_version.mca_component_name, param_b);
+}
+
+// -----------------------------
+
+static int parse_csv(const char *csv_orig, char sep, char ignore_start,
+        char ignore_end, void **vals_dst, int *len_dst, size_t type_size,
+        csv_parse_conv_fn_t conv_fn, csv_parse_destruct_fn_t destructor_fn,
+        char *err_help_header) {
+
+    if(NULL == csv_orig || 0 == strlen(csv_orig)) {
+        *vals_dst = NULL;
+        *len_dst = 0;
+        return OMPI_SUCCESS;
+    }
+
+    char *csv = NULL;
+    void *vals = NULL;
+
+    int vals_size = 0;
+    int ntokens = 0;
+
+    int return_code = OMPI_SUCCESS;
+
+    if(!(csv = strdup(csv_orig))) {
+        RETURN_WITH_ERROR(return_code, OMPI_ERR_OUT_OF_RESOURCE, end);
+    }
+
+    if(!(vals = malloc((vals_size = 5) * type_size))) {
+        RETURN_WITH_ERROR(return_code, OMPI_ERR_OUT_OF_RESOURCE, end);
+    }
+
+    int ignore_cnt = 0;
+    char *token = csv;
+
+    int csv_len = strlen(csv);
+
+    for(int i = 0; i < csv_len + 1; i++) {
+        char *c = csv+i;
+
+        if(ntokens == vals_size) {
+            void *tmp = realloc(vals, (vals_size *= 2) * sizeof(type_size));
+            if(!tmp) {RETURN_WITH_ERROR(return_code, OMPI_ERR_OUT_OF_RESOURCE, end);}
+            vals = tmp;
+        }
+
+        if(0 != ignore_start) {
+            if(*c == ignore_start) {
+                ignore_cnt++;
+            } else if(*c == ignore_end) {
+                ignore_cnt--;
+            }
+
+            if(ignore_cnt < 0) {
+                RETURN_WITH_ERROR(return_code, OMPI_ERR_BAD_PARAM, end);
+            }
+        }
+
+        if(0 == ignore_cnt && (*c == sep || '\0' == *c)) {
+            char oldc = *c;
+            *c = '\0';
+
+            int status = conv_fn(token, (char *) vals + ntokens*type_size);
+
+            if(OMPI_SUCCESS != status) {
+                if(err_help_header) {
+                    opal_show_help("help-coll-xhc.txt",
+                        err_help_header, true, token, csv_orig);
+                }
+
+                RETURN_WITH_ERROR(return_code, status, end);
+            }
+
+            ntokens++;
+
+            *c = oldc;
+            token = c + 1;
+        }
+    }
+
+    *vals_dst = vals;
+    *len_dst = ntokens;
+
+    end:
+
+    free(csv);
+
+    if(OMPI_SUCCESS != return_code) {
+        if(vals && destructor_fn) {
+            for(int i = 0; i < ntokens; i++) {
+                destructor_fn((char *) vals + i*type_size);
+            }
+        }
+
+        free(vals);
+    }
+
+    return return_code;
+}
+
+// -----------------------------
+
+static int conv_xhc_loc_def_rank_list(char *str, void *result) {
+    char *strs[2] = {str, NULL};
+    int nums[2] = {-1, -1};
+
+    char *range_op_pos = NULL;
+
+    int return_code = OMPI_SUCCESS;
+
+    if((range_op_pos = strstr(str, ".."))) {
+        strs[1] = range_op_pos + 2;
+        *range_op_pos = '\0';
+    }
+
+    for(int i = 0; i < 2 && strs[i]; i++) {
+        char *endptr;
+
+        nums[i] = strtol(strs[i], &endptr, 10);
+
+        if('\0' != endptr[0] || nums[i] < 0) {
+            RETURN_WITH_ERROR(return_code, OMPI_ERR_BAD_PARAM, end);
+        }
+    }
+
+    ((xhc_rank_range_t *) result)->start_rank = nums[0];
+    ((xhc_rank_range_t *) result)->end_rank = (-1 != nums[1] ? nums[1] : nums[0]);
+
+    end:
+
+    if(range_op_pos) {
+        *range_op_pos = '.';
+    }
+
+    return return_code;
+}
+
+static void mca_coll_xhc_loc_def_construct(xhc_loc_def_t *def) {
+    def->named_loc = 0;
+    def->rank_list = NULL;
+    def->rank_list_len = 0;
+    def->split = 0;
+    def->max_ranks = 0;
+    def->repeat = false;
+}
+
+static void mca_coll_xhc_loc_def_destruct(xhc_loc_def_t *def) {
+    free(def->rank_list);
+}
+
+OBJ_CLASS_INSTANCE(xhc_loc_def_t, opal_list_item_t,
+    mca_coll_xhc_loc_def_construct, mca_coll_xhc_loc_def_destruct);
+
+static int conv_xhc_loc_def(char *str, void *result) {
+    int return_code = OMPI_SUCCESS;
+
+    char *s = strdup(str);
+    xhc_loc_def_t *def = OBJ_NEW(xhc_loc_def_t);
+
+    if(!s || !def) {
+        RETURN_WITH_ERROR(return_code, OMPI_ERR_OUT_OF_RESOURCE, end);
+    }
+
+    /* Parse modifiers and remove them from string */
+
+    if('*' == s[strlen(s) - 1]) {
+        def->repeat = true;
+        s[strlen(s) - 1] = '\0';
+    }
+
+    char *colon_pos = strrchr(s, ':');
+    char *qmark_pos = strrchr(s, '?');
+
+    if(colon_pos && qmark_pos) {
+        RETURN_WITH_ERROR(return_code, OMPI_ERR_BAD_PARAM, end);
+    } else if(colon_pos || qmark_pos) {
+        char *numstr = (colon_pos ? colon_pos : qmark_pos);
+        char *endptr;
+
+        int num = strtol(numstr + 1, &endptr, 10);
+
+        if('\0' != endptr[0] || num <= 0) {
+            RETURN_WITH_ERROR(return_code, OMPI_ERR_BAD_PARAM, end);
+        }
+
+        if(colon_pos) {def->split = num;}
+        else {def->max_ranks = num;}
+
+        *numstr = '\0';
+    }
+
+    /* Parse locality definition */
+
+    if('[' == s[0]) {
+        if(def->repeat) { // repeat only makes sense with named localities
+            RETURN_WITH_ERROR(return_code, OMPI_ERR_BAD_PARAM, end);
+        }
+
+        s[strlen(s) - 1] = '\0';
+
+        int status = parse_csv(s+1, ',', 0, 0, (void **) &def->rank_list,
+            &def->rank_list_len, sizeof(xhc_rank_range_t),
+            conv_xhc_loc_def_rank_list, NULL, NULL);
+
+        if(OMPI_SUCCESS != status) {
+            RETURN_WITH_ERROR(return_code, status, end);
+        }
+    } else {
+        bool found = false;
+
+        for(size_t i = 0; i < sizeof(xhc_topo_str)/sizeof(char *); i++) {
+            if(0 == strcasecmp(s, xhc_topo_str[i])) {
+                def->named_loc = xhc_topo_val[i];
+                found = true;
+                break;
+            }
+        }
+
+        if(!found) {
+            RETURN_WITH_ERROR(return_code, OMPI_ERR_BAD_PARAM, end);
+        }
+    }
+
+    * (xhc_loc_def_t **) result = def;
+
+    end:
+
+    free(s);
+
+    if(OMPI_SUCCESS != return_code) {
+        OBJ_RELEASE_IF_NOT_NULL(def);
+    }
+
+    return return_code;
+}
+
+static void destruct_xhc_loc_def(void *data) {
+    OBJ_RELEASE(* (xhc_loc_def_t **) data);
+}
+
+static int conv_xhc_loc_def_combination(char *str, void *result) {
+    xhc_loc_def_t **defs;
+    int ndefs;
+
+    int status = parse_csv(str, '+', 0, 0, (void **) &defs,
+        &ndefs, sizeof(xhc_loc_def_t *), conv_xhc_loc_def,
+        destruct_xhc_loc_def, NULL);
+    if(OMPI_SUCCESS != status) {return status;}
+
+    opal_list_t *def_list = (opal_list_t *) result;
+    OBJ_CONSTRUCT(def_list, opal_list_t);
+
+    for(int i = 0; i < ndefs; i++) {
+        opal_list_append(def_list, (opal_list_item_t *) defs[i]);
+    }
+
+    free(defs);
+
+    return OMPI_SUCCESS;
+}
+
+static void destruct_xhc_loc_def_combination(void *data) {
+    OPAL_LIST_DESTRUCT((opal_list_t *) data);
+}
+
+int mca_coll_xhc_component_parse_hierarchy(const char *val_str,
+        opal_list_t **level_defs_dst, int *nlevel_defs_dst) {
+
+    /* The hierarchy is in a comma-separated list format. Each item in the
+     * list specifies how to group ranks, and each different item entails
+     * a grouping step.
+     *
+     * Each item in this list is a '+'-separated list. Of course, this can
+     * be just one item, without any delimiter, specifying the locality to
+     * follow for the grouping (e.g. numa, socket, etc).
+     *
+     * But, it can also be more complex (multiple '+'-separated items), used
+     * to describe virtual hierarchies. This allows to group different ranks
+     * in different ways, e.g. some ranks according to numa, then others by
+     * something else, etc.
+     *
+     * Each item in this '+'-separated list, can be of the following types:
+     * 1. A "named locality", e.g. hwloc's localities (only ones currently
+     *    available), see xhc_topo_str[].
+     * 2. A list of ranks that should be grouped together. This is a comma-
+     *    separated list of integers, enclosed in [] (I know, list-ception!).
+     *    It may also contain range operators (..), to select multiple ranks
+     *    at once (e.g. 0..3 expands to 0,1,2,3). Example: [0..15,20,22].
+     *    The order of the ranks does not matter.
+     *
+     * Finally, each such item may be suffixed by a special modifier:
+     * 1. The split modifier (:<n>) specifies to group according to the
+     *    locality it refers to, but to split each such group into multiple
+     *    parts. E.g. the locality 'numa:2' will group ranks into half-numa
+     *    groups, such that for each NUMA node, half the ranks are in one
+     *    group, and the rest are in another.
+     * 2. The max-ranks modifier (?<n>) works similarly to the split modifier,
+     *    only that it specifies that at most _n_ ranks should be placed in
+     *    each group. If more than _n_ ranks share the locality the modifier
+     *    refers to, multiple groups will be created for these ranks, each one
+     *    not more than _n_ ranks in size.
+     * 3. The repeat modifier (*), which can be specified along with the two
+     *    previous modifiers, allows manual control over the repetition of
+     *    named localities. See below, under 'repetition'.
+     *
+     * Repetition:
+     *   Named localities are repeated for all distinct rank clusters. For
+     *   example, "numa", even though it is a single key, means to group
+     *   all ranks that are in the same NUMA together, which will lead to
+     *   multiple groups if multiple NUMA nodes are present. This is in
+     *   contrast to rank lists, which only create a single group, containing
+     *   the ranks specified in it. The different items in the '+'-separated
+     *   list are consumed in-order left-to-right, and any named localities
+     *   are automatically repeated to apply to all ranks that are not included
+     *   in other items. When multiple named localities are present one after
+     *   the other, the last one is repeated, unless another repetition was
+     *   explicitly requested via the repeat modifier.
+     *
+     * Examples:
+     *   "numa": Group according to numa locality
+     *   "numa,socket": Group according to numa and then socket locality
+     *   "node"/"flat": Group according to node locality -> all ranks in
+     *     same node -> flat hierarchy i.e. none at all
+     *
+     *   "numa:2,socket": Group according to numa locality but with two
+     *     groups per NUMA, and then according to socket.
+     *   "numa:2,numa,socket": Similar to the previous one, but this case
+     *     will result in one of the two half-numa-leaders further becoming
+     *     the leader of the NUMA node.
+     *   "numa?10,socket": Group according to numa, but no more than 10 ranks
+     *     per NUMA; create multiple groups if necessary. Then group according
+     *     to socket.
+     *
+     *   "[0..9]+[10..24]": Create 2 groups: one for the first 10 ranks,
+     *     and another for the next 15 ones.
+     *   "[0..39]+numa,socket": Group the first 40 ranks, and the rest
+     *     according to numa locality. Then group according to socket.
+     *
+     *   "socket+socket:2": Create at least two groups: one for all ranks
+     *     in the first socket, and all the other ranks group them according
+     *     to socket locality, but with two groups for each socket.
+     *   "socket*+socket:2": Similar to the previous one, but only the last
+     *     socket is split into two groups, all the other ranks are grouped
+     *     according to socket locality.
+     *
+     * If the top-most locality specified does not cover all ranks, one such
+     * locality will automatically be added (in the hierarchy sort method).
+     *
+     * (Oh god what have I done! -Frankenstein, probably) */
+
+    int status = parse_csv(val_str, ',', '[', ']', (void **) level_defs_dst,
+        nlevel_defs_dst, sizeof(opal_list_t), conv_xhc_loc_def_combination,
+        destruct_xhc_loc_def_combination, "bad-hierarchy-item");
+
+    return status;
+}
+
+// -----------------------------
+
+static int conv_num_size(char *str, void *result) {
+    size_t last_idx = strlen(str) - 1;
+    char saved_char = str[last_idx];
+
+    size_t mult = 1;
+
+    switch(str[last_idx]) {
+        case 'g': case 'G':
+            mult *= 1024;
+        case 'm': case 'M':
+            mult *= 1024;
+        case 'k': case 'K':
+            mult *= 1024;
+
+        str[last_idx] = '\0';
+    }
+
+    bool legal = ('\0' != str[0]);
+
+    for(char *c = str; *c; c++) {
+        if((*c < '0' || *c > '9') && '-' != *c) {
+            legal = false;
+            break;
+        }
+    }
+
+    if(legal) {
+        long long num = atoll(str) * mult;
+        * (size_t *) result = (size_t) (num > 0 ? num : -1);
+    }
+
+    str[last_idx] = saved_char;
+
+    return (legal ? OMPI_SUCCESS : OMPI_ERR_BAD_PARAM);
+}
+
+int mca_coll_xhc_component_parse_chunk_sizes(const char *val_str,
+        size_t **chunks_dst, int *len_dst) {
+
+    if(NULL == val_str) {
+        *chunks_dst = malloc(sizeof(size_t));
+        if(NULL == *chunks_dst) {
+            return OMPI_ERR_OUT_OF_RESOURCE;
+        }
+
+        (*chunks_dst)[0] = (size_t) -1;
+        *len_dst = 1;
+
+        return OMPI_SUCCESS;
+    }
+
+    int status = parse_csv(val_str, ',', 0, 0, (void **) chunks_dst, len_dst,
+        sizeof(size_t), conv_num_size, NULL, "bad-chunk-size-item");
+
+    return status;
+}
+
+int mca_coll_xhc_component_parse_cico_max(const char *val_str,
+        size_t *cico_max_dst) {
+
+    int status;
+
+    if(val_str) {
+        status = conv_num_size((char *) val_str, cico_max_dst);
+    } else {
+        status = OMPI_ERR_BAD_PARAM;
+    }
+
+    if(OMPI_ERR_BAD_PARAM == status) {
+        opal_show_help("help-coll-xhc.txt", "bad_cico_max", true, val_str);
+    }
+
+    return status;
+}
diff --git a/ompi/mca/coll/xhc/coll_xhc_hierarchy.c b/ompi/mca/coll/xhc/coll_xhc_hierarchy.c
new file mode 100644
index 00000000000..8a6eca8cfe7
--- /dev/null
+++ b/ompi/mca/coll/xhc/coll_xhc_hierarchy.c
@@ -0,0 +1,420 @@
+/*
+ * Copyright (c) 2021-2024 Computer Architecture and VLSI Systems (CARV)
+ *                         Laboratory, ICS Forth. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "mpi.h"
+
+#include "ompi/constants.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/coll/base/base.h"
+#include "opal/util/show_help.h"
+#include "opal/util/minmax.h"
+
+#include "coll_xhc.h"
+
+// ------------------------------------------------
+
+static int xhc_hierarchy_create(xhc_module_t *module, ompi_communicator_t *comm,
+    opal_list_t *level_defs, int nlevel_defs, xhc_loc_t **hierarchy_dst,
+    int *hierarchy_len_dst);
+
+static int xhc_hierarchy_sort(mca_coll_xhc_module_t *module,
+    ompi_communicator_t *comm, xhc_loc_t **hierarchy_dst,
+    int *hierarchy_len_dst);
+
+// ------------------------------------------------
+
+int mca_coll_xhc_hierarchy_make(xhc_module_t *module,
+        ompi_communicator_t *comm, const char *hierarchy_string,
+        xhc_loc_t **hierarchy_dst, int *hierarchy_len_dst) {
+
+    xhc_loc_t *hierarchy = NULL;
+    int hierarchy_len;
+
+    opal_list_t *level_defs = NULL;
+    int nlevel_defs = 0;
+
+    int err, return_code = OMPI_SUCCESS;
+
+    // ---
+
+    err = xhc_component_parse_hierarchy(hierarchy_string,
+        &level_defs, &nlevel_defs);
+    if(OMPI_SUCCESS != err) {RETURN_WITH_ERROR(return_code, err, end);}
+
+    err = xhc_hierarchy_create(module, comm, level_defs,
+        nlevel_defs, &hierarchy, &hierarchy_len);
+    if(OMPI_SUCCESS != err) {RETURN_WITH_ERROR(return_code, err, end);}
+
+    err = xhc_hierarchy_sort(module, comm, &hierarchy, &hierarchy_len);
+    if(OMPI_SUCCESS != err) {RETURN_WITH_ERROR(return_code, err, end);}
+
+    // ---
+
+    *hierarchy_dst = hierarchy;
+    *hierarchy_len_dst = hierarchy_len;
+
+    end:
+
+    for(int i = 0; i < nlevel_defs; i++) {
+        OPAL_LIST_DESTRUCT(&level_defs[i]);
+    }
+    free(level_defs);
+
+    if(OMPI_SUCCESS != err) {
+        free(hierarchy);
+    }
+
+    return return_code;
+}
+
+// ------------------------------------------------
+
+static int xhc_hierarchy_create(xhc_module_t *module, ompi_communicator_t *comm,
+        opal_list_t *level_defs, int nlevel_defs, xhc_loc_t **hierarchy_dst,
+        int *hierarchy_len_dst) {
+
+    xhc_peer_info_t *peer_info = module->peer_info;
+    xhc_coll_fns_t xhc_fns;
+
+    int comm_size = ompi_comm_size(comm);
+    int rank = ompi_comm_rank(comm);
+
+    xhc_loc_t *hierarchy = NULL;
+    int nvirt_hiers = 0;
+
+    int *rank_list;
+
+    opal_hwloc_locality_t *loc_list;
+    ompi_datatype_t *hwloc_locality_type = NULL;
+
+    int err, return_code = OMPI_SUCCESS;
+
+    xhc_module_set_coll_fns(comm, &module->prev_colls, &xhc_fns);
+
+    hierarchy = malloc(nlevel_defs * sizeof(xhc_loc_t));
+    rank_list = malloc(comm_size * sizeof(int));
+    loc_list = malloc(comm_size * sizeof(opal_hwloc_locality_t));
+
+    if(!hierarchy || !rank_list || !loc_list) {
+        RETURN_WITH_ERROR(return_code, OMPI_ERR_OUT_OF_RESOURCE, end);
+    }
+
+    switch(sizeof(opal_hwloc_locality_t)) {
+        case 1: hwloc_locality_type = MPI_UINT8_T; break;
+        case 2: hwloc_locality_type = MPI_UINT16_T; break;
+        case 4: hwloc_locality_type = MPI_UINT32_T; break;
+        case 8: hwloc_locality_type = MPI_UINT64_T; break;
+    }
+    assert(hwloc_locality_type);
+
+    for(int h = 0; h < nlevel_defs; h++) {
+        opal_list_t *defs = &level_defs[h];
+
+        xhc_loc_def_t *my_def = NULL;
+        xhc_loc_t locality;
+
+        xhc_loc_def_t *def_0 = (xhc_loc_def_t *) opal_list_get_first(defs);
+
+        bool is_virtual = (opal_list_get_size(defs) > 1 || def_0->rank_list
+            || def_0->split > 1 || def_0->max_ranks > 0);
+
+        if(is_virtual) {
+            if(XHC_LOC_EXT_BITS == nvirt_hiers) {
+                opal_show_help("help-coll-xhc.txt", "too-many-virt-hiers", true);
+                RETURN_WITH_ERROR(return_code, OMPI_ERR_NOT_SUPPORTED, end);
+            }
+
+            locality = 1 << (XHC_LOC_EXT_START + nvirt_hiers);
+            nvirt_hiers++;
+        } else {
+            locality = def_0->named_loc;
+        }
+
+        hierarchy[h] = locality;
+        def_0 = NULL;
+
+        xhc_loc_def_t *def, *def_next;
+
+        /* Handle rank lists; take note if I belong
+         * in one, and remove them from the mix */
+        OPAL_LIST_FOREACH_SAFE(def, def_next, defs, xhc_loc_def_t) {
+            if(def->rank_list) {
+                if(!my_def) {
+                    for(int rl = 0; rl < def->rank_list_len; rl++) {
+                        if(rank >= def->rank_list[rl].start_rank
+                                && rank <= def->rank_list[rl].end_rank) {
+                            my_def = def;
+                            break;
+                        }
+                    }
+                }
+
+                opal_list_remove_item(defs, (opal_list_item_t *) def);
+                if(def != my_def) {OBJ_RELEASE(def);}
+            }
+        }
+
+        bool dir_fwd = true;
+
+        /* When multiple locality defitions are present, they are assigned
+         * to groups in a left-to-right fashion. At every turn, the first
+         * rank (determined by the minimum ID) that's still not part of
+         * a locality, as well as the other ranks that are local with it,
+         * claim/consume the next locality from the list. The direction
+         * serves to implement the repeat modifier. When it is located,
+         * the process starts taking place right-to-left following the max
+         * ID. At the end and after the loop, the repeated locality will
+         * be the only one left and all remaining ranks will follow it. */
+        while(opal_list_get_size(defs) > 1) {
+            def = (xhc_loc_def_t *) (dir_fwd ? opal_list_get_first(defs)
+                : opal_list_get_last(defs));
+
+            if(dir_fwd && def->repeat) {
+                dir_fwd = false;
+                continue;
+            }
+
+            int ticket = (NULL == my_def ? rank : (dir_fwd ? comm_size : -1));
+            int chosen;
+
+            err = comm->c_coll->coll_allreduce(&ticket, &chosen, 1,
+                MPI_INT, (dir_fwd ? MPI_MIN : MPI_MAX), comm,
+                comm->c_coll->coll_allreduce_module);
+            if(OMPI_SUCCESS != err) {
+                RETURN_WITH_ERROR(return_code, err, end);
+            }
+
+            if(chosen >= 0 && chosen < comm_size
+                    && PEER_IS_LOCAL(peer_info, chosen, def->named_loc)) {
+                my_def = def;
+            }
+
+            opal_list_remove_item(defs, (opal_list_item_t *) def);
+            if(def != my_def) {OBJ_RELEASE(def);}
+        }
+
+        if(opal_list_get_size(defs) > 0 && !my_def) {
+            my_def = (xhc_loc_def_t *) opal_list_get_first(defs);
+            opal_list_remove_item(defs, (opal_list_item_t *) my_def);
+        }
+
+        /* Share which named locality each rank follows; ranks that
+         * follow different localities shouldn't be grouped together */ 
+        opal_hwloc_locality_t follow_loc = (my_def ? my_def->named_loc : 0);
+        err = comm->c_coll->coll_allgather(&follow_loc, 1,
+            hwloc_locality_type, loc_list, 1, hwloc_locality_type,
+            comm, comm->c_coll->coll_allgather_module);
+        if(OMPI_SUCCESS != err) {
+            RETURN_WITH_ERROR(return_code, err, end);
+        }
+
+        if(NULL == my_def) {
+            continue;
+        }
+
+        int my_id = -1;
+        int members = 0;
+
+        // If working with rank list, set the ranks from the list as "local"
+        if(my_def->rank_list) {
+            for(int i = 0; i < my_def->rank_list_len; i++) {
+                for(int r = my_def->rank_list[i].start_rank;
+                        r <= my_def->rank_list[i].end_rank && r < comm_size; r++) {
+                    if(r == rank) {
+                        my_id = members;
+                    }
+
+                    peer_info[r].locality |= locality;
+                    rank_list[members++] = r;
+                }
+            }
+        } else if(is_virtual) {
+            /* We might have a named locality instead of a rank list, but if
+             * we still needed to create a virtual one, we need to apply it. */
+            for(int r = 0; r < comm_size; r++) {
+                if(loc_list[r] != my_def->named_loc) {
+                    continue;
+                }
+
+                if(!PEER_IS_LOCAL(peer_info, r, my_def->named_loc)) {
+                    continue;
+                }
+
+                if(r == rank) {
+                    my_id = members;
+                }
+
+                peer_info[r].locality |= locality;
+                rank_list[members++] = r;
+            }
+        }
+
+        /* If split or max ranks was specified, must partition the locality
+         * and remove the previously added locality mapping to some ranks. */
+        if(my_def->split > 1) {
+            int piece_size = members / my_def->split;
+            int leftover = members % my_def->split;
+
+            for(int m = 0, next_border = 0; m < members; m++) {
+                if(m == next_border) {
+                    next_border += piece_size + (leftover > 0 ? 1 : 0);
+                    if(leftover > 0) {leftover--;}
+
+                    if(my_id >= m && my_id < next_border) {
+                        m = next_border - 1;
+                        continue;
+                    }
+                }
+
+                peer_info[rank_list[m]].locality &= ~locality;
+            }
+        } else if(my_def->max_ranks > 1) {
+            for(int m = 0; m < members; m++) {
+                if(m % my_def->max_ranks == 0) {
+                    if(my_id >= m && my_id - m < my_def->max_ranks) {
+                        m += my_def->max_ranks - 1;
+                        continue;
+                    }
+                }
+
+                peer_info[rank_list[m]].locality &= ~locality;
+            }
+        }
+
+        OBJ_RELEASE_IF_NOT_NULL(my_def);
+    }
+
+    *hierarchy_dst = hierarchy;
+    *hierarchy_len_dst = nlevel_defs;
+
+end:
+
+    xhc_module_set_coll_fns(comm, &xhc_fns, NULL);
+
+    free(rank_list);
+    free(loc_list);
+
+    if(OMPI_SUCCESS != return_code) {
+        free(hierarchy);
+    }
+
+    return return_code;
+}
+
+static int xhc_hierarchy_sort(mca_coll_xhc_module_t *module,
+        ompi_communicator_t *comm, xhc_loc_t **hierarchy_dst,
+        int *hierarchy_len_dst) {
+
+    xhc_peer_info_t *peer_info = module->peer_info;
+    int comm_size = ompi_comm_size(comm);
+
+    xhc_loc_t *old_hier = *hierarchy_dst;
+    int hier_len = *hierarchy_len_dst;
+
+    xhc_loc_t *new_hier = NULL;
+    bool *hier_done = NULL;
+
+    int return_code = OMPI_SUCCESS;
+
+    new_hier = malloc((hier_len + 1) * sizeof(xhc_loc_t));
+    hier_done = calloc(hier_len, sizeof(bool));
+
+    if(NULL == new_hier || NULL == hier_done) {
+        RETURN_WITH_ERROR(return_code, OMPI_ERR_OUT_OF_RESOURCE, end);
+    }
+
+    bool has_virtual = false;
+    for(int i = 0; i < hier_len; i++) {
+        if(old_hier[i] >= (1 << XHC_LOC_EXT_START)) {
+            has_virtual = true;
+            break;
+        }
+    }
+
+    /* If any virtual hierarchy is involved, attempting to sort it is likely
+     * asking for trouble. Skip the sorting, and only consider adding a top
+     * common locality. There is a chance it wasn't actually necessary, but
+     * it never hurts. */
+
+    if(has_virtual) {
+        memcpy(new_hier, old_hier, hier_len * sizeof(xhc_loc_t));
+    } else {
+        for(int new_idx = hier_len - 1; new_idx >= 0; new_idx--) {
+            int max_matches_count = -1;
+            int max_matches_hier_idx = -1;
+
+            for(int i = 0; i < hier_len; i++) {
+                if(hier_done[i]) {
+                    continue;
+                }
+
+                int matches = 0;
+
+                for(int r = 0; r < comm_size; r++) {
+                    if(PEER_IS_LOCAL(peer_info, r, old_hier[i])) {
+                        matches++;
+                    }
+                }
+
+                if(matches > max_matches_count) {
+                    max_matches_count = matches;
+                    max_matches_hier_idx = i;
+                }
+            }
+
+            assert(-1 != max_matches_count);
+
+            new_hier[new_idx] = old_hier[max_matches_hier_idx];
+            hier_done[max_matches_hier_idx] = true;
+        }
+    }
+
+    xhc_loc_t common_locality = (xhc_loc_t) -1;
+
+    for(int r = 0; r < comm_size; r++) {
+        ompi_proc_t *proc = ompi_comm_peer_lookup(comm, r);
+        common_locality &= proc->super.proc_flags;
+    }
+
+    if(0 == common_locality) {
+        opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT,
+            ompi_coll_base_framework.framework_output,
+            "coll:xhc: Error: There is no locality common "
+            "to all ranks in the communicator");
+
+        RETURN_WITH_ERROR(return_code, OMPI_ERR_NOT_SUPPORTED, end);
+    }
+
+    if(0 == hier_len || (common_locality & new_hier[hier_len - 1])
+            != new_hier[hier_len - 1]) {
+
+        new_hier[hier_len] = common_locality;
+        hier_len++;
+    }
+
+    REALLOC(new_hier, hier_len, xhc_loc_t);
+
+    free(old_hier);
+
+    *hierarchy_dst = new_hier;
+    *hierarchy_len_dst = hier_len;
+
+end:
+
+    free(hier_done);
+
+    if(OMPI_SUCCESS != return_code) {
+        free(new_hier);
+    }
+
+    return return_code;
+}
diff --git a/ompi/mca/coll/xhc/coll_xhc_intrinsic.h b/ompi/mca/coll/xhc/coll_xhc_intrinsic.h
new file mode 100644
index 00000000000..ee61b8d7db3
--- /dev/null
+++ b/ompi/mca/coll/xhc/coll_xhc_intrinsic.h
@@ -0,0 +1,151 @@
+/*
+ * Copyright (c) 2021-2024 Computer Architecture and VLSI Systems (CARV)
+ *                         Laboratory, ICS Forth. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef MCA_COLL_XHC_ATOMIC_EXPORT_H
+#define MCA_COLL_XHC_ATOMIC_EXPORT_H
+
+#include <stdint.h>
+#include "opal/sys/atomic.h"
+
+// ----------------------------------------
+
+#define IS_SIG_ATOMIC_X_BITS(x) \
+    (SIG_ATOMIC_MAX == INT ## x ## _MAX) || (SIG_ATOMIC_MAX == UINT ## x ## _MAX)
+
+// If xf_sig_t is ever re-defined to be signed,
+ // CHECK_FLAGS()'s comparisons must be adjusted
+#if IS_SIG_ATOMIC_X_BITS(64)
+    typedef uint64_t xf_sig_t;
+#elif IS_SIG_ATOMIC_X_BITS(32)
+    typedef uint32_t xf_sig_t;
+#elif IS_SIG_ATOMIC_X_BITS(16)
+    typedef uint16_t xf_sig_t;
+#elif IS_SIG_ATOMIC_X_BITS(8)
+    typedef uint8_t xf_sig_t;
+#endif
+
+typedef int __attribute__((aligned(SIZEOF_INT))) xf_int_t;
+typedef size_t __attribute__((aligned(SIZEOF_SIZE_T))) xf_size_t;
+
+// ----------------------------------------
+
+#define xhc_atomic_rmb opal_atomic_rmb
+#define xhc_atomic_wmb opal_atomic_wmb
+#define xhc_atomic_fmb opal_atomic_mb
+
+// https://github.com/open-mpi/ompi/issues/9722
+
+#if OPAL_USE_GCC_BUILTIN_ATOMICS || OPAL_USE_C11_ATOMICS
+    #define xhc_atomic_load_int(addr) __atomic_load_n(addr, __ATOMIC_RELAXED)
+    #define xhc_atomic_store_int(addr, val) __atomic_store_n(addr, val, __ATOMIC_RELAXED)
+
+    #define xhc_atomic_load_size_t(addr) __atomic_load_n(addr, __ATOMIC_RELAXED)
+    #define xhc_atomic_store_size_t(addr, val) __atomic_store_n(addr, val, __ATOMIC_RELAXED)
+#else
+    #define xhc_atomic_load_int(addr) (*(addr))
+    #define xhc_atomic_store_int(addr, val) (*(addr) = (val))
+
+    #define xhc_atomic_load_size_t(addr) (*(addr))
+    #define xhc_atomic_store_size_t(addr, val) (*(addr) = (val))
+
+    #warning "GCC or the C11 atomics backend was not found. XHC might not function correctly"
+/* #else
+    #error "XHC atomics do not yet work without the GCC or the C11 backend" */
+#endif
+
+
+// If/when opal atomic load/store size_t is added
+
+/* #define xhc_atomic_load_size_t(addr) \
+    opal_atomic_load_size_t ((opal_atomic_size_t *) addr)
+#define xhc_atomic_store_size_t(addr, val) \
+    opal_atomic_store_size_t ((opal_atomic_size_t *) addr, val) */
+
+
+// If/when opal atomic load/store is added, and if opal atomic load/store int is not
+
+/* #if 4 == SIZEOF_INT
+    #define xhc_atomic_load_int(addr) opal_atomic_load_32 ((opal_atomic_int32_t *) addr)
+    #define xhc_atomic_store_int(addr, val) opal_atomic_store_32 ((opal_atomic_int32_t *) addr, val)
+#elif 8 == SIZEOF_INT
+    #define xhc_atomic_load_int(addr) opal_atomic_load_64 ((opal_atomic_int64_t *) addr)
+    #define xhc_atomic_store_int(addr, val) opal_atomic_store_64 ((opal_atomic_int64_t *) addr, val)
+#else
+    #error "Unsupported int size"
+#endif */
+
+
+// If/when opal atomic load/store is added, and if opal atomic load/store size_t is not
+
+/* #if 4 == SIZEOF_SIZE_T
+    #define xhc_atomic_load_size_t(addr) opal_atomic_load_32 ((opal_atomic_int32_t *) addr)
+    #define xhc_atomic_store_size_t(addr, val) opal_atomic_store_32 ((opal_atomic_int32_t *) addr, val)
+#elif 8 == SIZEOF_SIZE_T
+    #define xhc_atomic_load_size_t(addr) opal_atomic_load_64 ((opal_atomic_int64_t *) addr)
+    #define xhc_atomic_store_size_t(addr, val) opal_atomic_store_64 ((opal_atomic_int64_t *) addr, val)
+#else
+    #error "Unsupported size_t size"
+#endif */
+
+static inline bool xhc_atomic_cmpxchg_strong_relaxed(volatile xf_sig_t *addr,
+        xf_sig_t *oldval, xf_sig_t newval) {
+
+    #if OPAL_USE_GCC_BUILTIN_ATOMICS || OPAL_USE_C11_ATOMICS
+        return __atomic_compare_exchange_n(addr, oldval, newval,
+            false, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+    #else
+        #if IS_SIG_ATOMIC_X_BITS(32)
+            return opal_atomic_compare_exchange_strong_32(addr, oldval, newval);
+        #elif IS_SIG_ATOMIC_X_BITS(64)
+            return opal_atomic_compare_exchange_strong_64(addr, oldval, newval);
+        #else
+            #error "Unsupported sig_atomic_t size"
+        #endif
+    #endif
+}
+
+// ----------------------------------------
+
+static inline void xhc_prefetchw(void *addr, size_t len, int target) {
+    for(char *p = (char *) ((uintptr_t) addr & ~63);
+            p < (char *) addr + len; p += 64) {
+
+        switch(target) {
+            case 0:
+            case 1: // L1 cache
+            #if defined(PLATFORM_ARCH_X86) || defined(PLATFORM_ARCH_X86_64)
+                __asm__ __volatile__("prefetchw (%1)" : : "i" (0), "r" (p));
+            #elif defined(PLATFORM_ARCH_AARCH64)
+                __asm__ __volatile__("prfm pstl1keep, %a0\n" : : "p" (p));
+            #endif
+                break;
+
+            case 2: // L2 cache
+            #if defined(PLATFORM_ARCH_X86) || defined(PLATFORM_ARCH_X86_64)
+                __asm__ __volatile__("prefetchwt1 (%1)" : : "i" (0), "r" (p));
+            #elif defined(PLATFORM_ARCH_AARCH64)
+                __asm__ __volatile__("prfm pstl2keep, %a0\n" : : "p" (p));
+            #endif
+                break;
+
+            case 3: // L3 cache
+            default:
+            #if defined(PLATFORM_ARCH_X86) || defined(PLATFORM_ARCH_X86_64)
+                // no such thing as a 'prefetchwt2'
+                __asm__ __volatile__("prefetchwt1 (%1)" : : "i" (0), "r" (p));
+            #elif defined(PLATFORM_ARCH_AARCH64)
+                __asm__ __volatile__("prfm pstl3keep, %a0\n" : : "p" (p));
+            #endif
+                break;
+        }
+    }
+}
+
+#endif
diff --git a/ompi/mca/coll/xhc/coll_xhc_module.c b/ompi/mca/coll/xhc/coll_xhc_module.c
new file mode 100644
index 00000000000..a75232c5157
--- /dev/null
+++ b/ompi/mca/coll/xhc/coll_xhc_module.c
@@ -0,0 +1,284 @@
+/*
+ * Copyright (c) 2021-2024 Computer Architecture and VLSI Systems (CARV)
+ *                         Laboratory, ICS Forth. All rights reserved.
+ * Copyright (c) 2024      Jeffrey M. Squyres.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include <stdio.h>
+#include <string.h>
+
+#include "mpi.h"
+
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/coll/base/base.h"
+#include "opal/mca/smsc/smsc.h"
+
+#include "opal/util/arch.h"
+#include "opal/util/show_help.h"
+#include "opal/util/minmax.h"
+
+#include "coll_xhc.h"
+
+// -----------------------------
+
+static void mca_coll_xhc_module_construct(mca_coll_xhc_module_t *module);
+static void mca_coll_xhc_module_destruct(mca_coll_xhc_module_t *module);
+
+OBJ_CLASS_INSTANCE(mca_coll_xhc_module_t, mca_coll_base_module_t,
+    mca_coll_xhc_module_construct, mca_coll_xhc_module_destruct);
+
+// -----------------------------
+
+static size_t xhc_colltype_to_c_coll_fn_offset_map[XHC_COLLCOUNT] = {
+    [XHC_BCAST] = offsetof(mca_coll_base_comm_coll_t, coll_bcast),
+    [XHC_BARRIER] = offsetof(mca_coll_base_comm_coll_t, coll_barrier),
+    [XHC_REDUCE] = offsetof(mca_coll_base_comm_coll_t, coll_reduce),
+    [XHC_ALLREDUCE] = offsetof(mca_coll_base_comm_coll_t, coll_allreduce)
+};
+
+static size_t xhc_colltype_to_c_coll_module_offset_map[XHC_COLLCOUNT] = {
+    [XHC_BCAST] = offsetof(mca_coll_base_comm_coll_t, coll_bcast_module),
+    [XHC_BARRIER] = offsetof(mca_coll_base_comm_coll_t, coll_barrier_module),
+    [XHC_REDUCE] = offsetof(mca_coll_base_comm_coll_t, coll_reduce_module),
+    [XHC_ALLREDUCE] = offsetof(mca_coll_base_comm_coll_t, coll_allreduce_module)
+};
+
+static size_t xhc_colltype_to_base_module_fn_offset_map[XHC_COLLCOUNT] = {
+    [XHC_BCAST] = offsetof(mca_coll_base_module_t, coll_bcast),
+    [XHC_BARRIER] = offsetof(mca_coll_base_module_t, coll_barrier),
+    [XHC_REDUCE] = offsetof(mca_coll_base_module_t, coll_reduce),
+    [XHC_ALLREDUCE] = offsetof(mca_coll_base_module_t, coll_allreduce)
+};
+
+static inline void (*MODULE_COLL_FN(xhc_module_t *module,
+        XHC_COLLTYPE_T colltype))(void) {
+
+    return * (void (**)(void)) ((uintptr_t) &module->super
+        + xhc_colltype_to_base_module_fn_offset_map[colltype]);
+}
+
+static inline void INSTALL_COLL_API(ompi_communicator_t *comm,
+        XHC_COLLTYPE_T colltype, void (*coll_fn)(void), void *coll_module) {
+
+    * (void (**)(void)) ((uintptr_t) comm->c_coll
+        + xhc_colltype_to_c_coll_fn_offset_map[colltype]) = coll_fn;
+    * (void **) ((uintptr_t) comm->c_coll
+        + xhc_colltype_to_c_coll_module_offset_map[colltype]) = coll_module;
+}
+
+static inline void GET_COLL_API(ompi_communicator_t *comm, XHC_COLLTYPE_T colltype,
+        void (**coll_fn_dst)(void), void **coll_module_dst) {
+
+    *coll_fn_dst = * (void (**)(void)) ((uintptr_t) comm->c_coll
+        + xhc_colltype_to_c_coll_fn_offset_map[colltype]);
+
+    *coll_module_dst = * (void **) ((uintptr_t) comm->c_coll
+        + xhc_colltype_to_c_coll_module_offset_map[colltype]);
+}
+
+// -----------------------------
+
+static void xhc_module_clear(xhc_module_t *module) {
+    module->comm_size = 0;
+    module->rank = -1;
+
+    module->zcopy_support = false;
+    module->zcopy_map_support = false;
+
+    module->rbuf = NULL;
+    module->rbuf_size = 0;
+
+    module->peer_info = NULL;
+
+    memset(&module->prev_colls, 0, sizeof(module->prev_colls));
+    memset(&module->op_config, 0, sizeof(module->op_config));
+    memset(&module->op_data, 0, sizeof(module->op_data));
+
+    module->init = false;
+    module->error = false;
+}
+
+static void mca_coll_xhc_module_construct(mca_coll_xhc_module_t *module) {
+    xhc_module_clear(module);
+}
+
+static void mca_coll_xhc_module_destruct(mca_coll_xhc_module_t *module) {
+    /* Anything that's allocated during the module's creation/enable, is
+     * deallocated here. The stuff that's allocated lazily inside/under
+     * xhc_lazy_init and xhc_init_op, is deallocated inside xhc_fini. */
+
+    if(module->init) {
+        xhc_fini(module);
+    }
+
+    for(int t = 0; t < XHC_COLLCOUNT; t++) {
+        free(module->op_config[t].hierarchy_string);
+        free(module->op_config[t].chunk_string);
+        free(module->op_config[t].chunks);
+    }
+
+    xhc_module_clear(module);
+}
+
+// -----------------------------
+
+mca_coll_base_module_t *mca_coll_xhc_module_comm_query(ompi_communicator_t *comm,
+        int *priority) {
+
+    if((*priority = mca_coll_xhc_component.priority) < 0) {
+        return NULL;
+    }
+
+    int comm_size = ompi_comm_size(comm);
+
+    if(OMPI_COMM_IS_INTER(comm) || 1 == comm_size
+            || ompi_group_have_remote_peers (comm->c_local_group)) {
+
+        opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT,
+            ompi_coll_base_framework.framework_output,
+            "coll:xhc:comm_query (%s/%s): intercomm, self-comm, "
+            "or not all ranks local; disqualifying myself",
+            ompi_comm_print_cid(comm), comm->c_name);
+
+        return NULL;
+    }
+
+    for(int r = 0; r < comm_size; r++) {
+        ompi_proc_t *proc = ompi_comm_peer_lookup(comm, r);
+
+        if(proc->super.proc_arch != opal_local_arch) {
+            opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT,
+                ompi_coll_base_framework.framework_output,
+                "coll:xhc:comm_query (%s/%s): All ranks not of the same arch; "
+                "disabling myself", ompi_comm_print_cid(comm), comm->c_name);
+
+            return NULL;
+        }
+    }
+
+    mca_coll_xhc_module_t *module = OBJ_NEW(mca_coll_xhc_module_t);
+
+    if(NULL == module) {
+        return NULL;
+    }
+
+    module->zcopy_support = (NULL != mca_smsc);
+    module->zcopy_map_support = mca_smsc_base_has_feature(MCA_SMSC_FEATURE_CAN_MAP);
+
+    if(!module->zcopy_support) {
+        opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT,
+            ompi_coll_base_framework.framework_output,
+            "coll:xhc: Warning: No opal/smsc support found; "
+            "xhc will only work in CICO mode");
+    } else if(!module->zcopy_map_support) {
+        opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT,
+            ompi_coll_base_framework.framework_output,
+            "coll:xhc: Warning: opal/smsc module isn't CAN_MAP "
+            "capable; reduced performance is to be expected");
+    }
+
+    module->super.coll_module_enable = mca_coll_xhc_module_enable;
+    module->super.coll_module_disable = mca_coll_xhc_module_disable;
+
+    module->super.coll_bcast = mca_coll_xhc_bcast;
+    module->super.coll_barrier = mca_coll_xhc_barrier;
+    module->super.coll_allreduce = mca_coll_xhc_allreduce;
+    module->super.coll_reduce = mca_coll_xhc_reduce;
+
+    return &module->super;
+}
+
+int mca_coll_xhc_module_enable(mca_coll_base_module_t *ompi_module,
+        ompi_communicator_t *comm) {
+
+    xhc_module_t *module = (xhc_module_t *) ompi_module;
+
+    // ---
+
+    /* Assimilate the various MCA parameters. We do this inside module_enable
+     * rather than lazy_init, so we may use the values as early as possible
+     * (e.g. checking cico and single-copy support at the beginning of ops). */
+    for(int t = 0; t < XHC_COLLCOUNT; t++) {
+        int err = xhc_read_op_config(module, comm, t);
+        if(OMPI_SUCCESS != err) {
+            return err;
+        }
+    }
+
+    // ---
+
+    for(int t = 0; t < XHC_COLLCOUNT; t++) {
+        /* Don't want to save a fallback for
+         * any op that we won't support */
+        if(NULL == MODULE_COLL_FN(module, t)) {
+            continue;
+        }
+
+        void (*fallback_fn)(void), *fallback_module;
+        GET_COLL_API(comm, t, &fallback_fn, &fallback_module);
+
+        if(NULL == fallback_fn || NULL == fallback_module) {
+            opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT,
+                ompi_coll_base_framework.framework_output,
+                "coll:xhc:module_enable (%s/%s): No previous fallback component "
+                "found; disabling myself", ompi_comm_print_cid(comm), comm->c_name);
+
+            return OMPI_ERR_NOT_FOUND;
+        }
+
+        module->prev_colls.coll_fn[t] = fallback_fn;
+        module->prev_colls.coll_module[t] = fallback_module;
+    }
+
+    /* We perform the pointer installation last, after we've
+     * successfully captured all the fallback pointers we need,
+     * and we know xhc_module_enable can no longer fail. */
+    for(int t = 0; t < XHC_COLLCOUNT; t++) {
+        void (*fn)(void) = MODULE_COLL_FN(module, t);
+        if(fn) {INSTALL_COLL_API(comm, t, fn, module);}
+    }
+
+    // ---
+
+    return OMPI_SUCCESS;
+}
+
+int mca_coll_xhc_module_disable(mca_coll_base_module_t *ompi_module,
+        ompi_communicator_t *comm) {
+
+    mca_coll_xhc_module_destruct((xhc_module_t *) ompi_module);
+    return OMPI_SUCCESS;
+}
+
+// -----------------------------
+
+void mca_coll_xhc_module_set_coll_fns(ompi_communicator_t *comm,
+        xhc_coll_fns_t *new_fns, xhc_coll_fns_t *saved_fns_dst) {
+
+    xhc_coll_fns_t saved = {0};
+
+    for(int t = 0; t < XHC_COLLCOUNT; t++) {
+        if(!new_fns->coll_fn[t]) {
+            continue;
+        }
+
+        if(saved_fns_dst) {
+            GET_COLL_API(comm, t, &saved.coll_fn[t],
+                &saved.coll_module[t]);
+        }
+
+        INSTALL_COLL_API(comm, t, new_fns->coll_fn[t],
+            new_fns->coll_module[t]);
+    }
+
+    if(saved_fns_dst) {
+        *saved_fns_dst = saved;
+    }
+}
diff --git a/ompi/mca/coll/xhc/coll_xhc_reduce.c b/ompi/mca/coll/xhc/coll_xhc_reduce.c
new file mode 100644
index 00000000000..87c20117ee0
--- /dev/null
+++ b/ompi/mca/coll/xhc/coll_xhc_reduce.c
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2021-2024 Computer Architecture and VLSI Systems (CARV)
+ *                         Laboratory, ICS Forth. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "mpi.h"
+
+#include "ompi/constants.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/op/op.h"
+
+#include "opal/mca/rcache/base/base.h"
+#include "opal/util/show_help.h"
+#include "opal/util/minmax.h"
+
+#include "coll_xhc.h"
+
+int mca_coll_xhc_reduce(const void *sbuf, void *rbuf,
+        size_t count, ompi_datatype_t *datatype, ompi_op_t *op, int root,
+        ompi_communicator_t *ompi_comm, mca_coll_base_module_t *ompi_module) {
+
+    xhc_module_t *module = (xhc_module_t *) ompi_module;
+
+    /* Currently, XHC's reduce only supports the top-level
+     * owner as the root (typically rank 0). */
+    if(0 == root) {
+        return mca_coll_xhc_allreduce_internal(sbuf, rbuf, count,
+            datatype, op, ompi_comm, ompi_module, false);
+    } else {
+        WARN_ONCE("coll:xhc: Warning: XHC does not currently support "
+            "non-zero-root reduce; utilizing fallback component");
+        return XHC_CALL_FALLBACK(module->prev_colls, XHC_REDUCE,
+            reduce, sbuf, rbuf, count, datatype, op, root, ompi_comm);
+    }
+}
diff --git a/ompi/mca/coll/xhc/help-coll-xhc.txt b/ompi/mca/coll/xhc/help-coll-xhc.txt
new file mode 100644
index 00000000000..c367f62d699
--- /dev/null
+++ b/ompi/mca/coll/xhc/help-coll-xhc.txt
@@ -0,0 +1,43 @@
+#
+# Copyright (c) 2021-2024 Computer Architecture and VLSI Systems (CARV)
+#                         Laboratory, ICS Forth. All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+[bad-hierarchy-item]
+ERROR (coll/xhc)
+Unrecognized locality definition: '%s'
+In hierarchy parameter string: '%s'
+Falling back to another component
+#
+[bad-chunk-size-item]
+ERROR (coll/xhc)
+Malformed chunk size item: '%s'
+In chunk size parameter string: '%s'
+Falling back to another component
+#
+[bad-cico-max]
+ERROR (coll/xhc)
+Malformed cico_max parameter '%s'
+Falling back to another component
+#
+[xhc-init-failed]
+ERROR (coll/xhc)
+Initialization failed, error code %d
+Falling back to another component
+Errno: %d (%s)
+#
+[xhc-op-init-failed]
+ERROR (coll/xhc)
+Initialization for %s failed, error code %d
+Falling back to another component
+Errno: %d (%s)
+#
+[too-many-virt-hiers]
+ERROR (coll/xhc)
+Too many virtual hierarchies requested
+Falling back to another component
diff --git a/ompi/mca/coll/xhc/resources/xhc-hierarchy.svg b/ompi/mca/coll/xhc/resources/xhc-hierarchy.svg
new file mode 100644
index 00000000000..c8f6d8a2da3
--- /dev/null
+++ b/ompi/mca/coll/xhc/resources/xhc-hierarchy.svg
@@ -0,0 +1,1176 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   width="169.571mm"
+   height="119.894mm"
+   viewBox="0 0 169.571 119.89402"
+   version="1.1"
+   id="svg5"
+   inkscape:version="1.2.1 (9c6d41e410, 2022-07-14, custom)"
+   sodipodi:docname="xhc-hierarchy.svg"
+   inkscape:export-filename="../xhc-hierarchy.png"
+   inkscape:export-xdpi="300"
+   inkscape:export-ydpi="300"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <sodipodi:namedview
+     id="namedview7"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     inkscape:document-units="mm"
+     showgrid="false"
+     inkscape:zoom="0.75290071"
+     inkscape:cx="286.22632"
+     inkscape:cy="274.93665"
+     inkscape:window-width="1920"
+     inkscape:window-height="1018"
+     inkscape:window-x="1920"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="layer1"
+     inkscape:snap-global="false"
+     inkscape:showpageshadow="2"
+     inkscape:deskcolor="#d1d1d1" />
+  <defs
+     id="defs2">
+    <linearGradient
+       id="linearGradient12307"
+       inkscape:swatch="solid">
+      <stop
+         style="stop-color:#006d00;stop-opacity:1;"
+         offset="0"
+         id="stop12305" />
+    </linearGradient>
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect576"
+       is_visible="true"
+       lpeversion="1" />
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect572"
+       is_visible="true"
+       lpeversion="1" />
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect568"
+       is_visible="true"
+       lpeversion="1" />
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect564"
+       is_visible="true"
+       lpeversion="1" />
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect560"
+       is_visible="true"
+       lpeversion="1" />
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect556"
+       is_visible="true"
+       lpeversion="1" />
+    <marker
+       style="overflow:visible"
+       id="TriangleStart"
+       refX="0"
+       refY="0"
+       orient="auto-start-reverse"
+       inkscape:stockid="TriangleStart"
+       markerWidth="4.2600002"
+       markerHeight="4.9248557"
+       viewBox="0 0 5.3244081 6.1553851"
+       inkscape:isstock="true"
+       inkscape:collect="always"
+       preserveAspectRatio="xMidYMid">
+      <path
+         transform="scale(0.5)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 5.77,0 -2.88,5 V -5 Z"
+         id="path135" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="TriangleStart_Fnone_S-000000"
+       refX="0"
+       refY="0"
+       orient="auto-start-reverse"
+       inkscape:stockid="TriangleStart"
+       markerWidth="4.2600002"
+       markerHeight="4.9248557"
+       viewBox="0 0 5.3244081 6.1553851"
+       inkscape:isstock="true"
+       inkscape:collect="always"
+       preserveAspectRatio="xMidYMid">
+      <path
+         transform="scale(0.5)"
+         style="fill:#000000;fill-rule:evenodd;stroke:#000000;stroke-width:1pt"
+         d="M 5.77,0 -2.88,5 V -5 Z"
+         id="path5624" />
+    </marker>
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect3490"
+       is_visible="true"
+       lpeversion="1" />
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect3486"
+       is_visible="true"
+       lpeversion="1" />
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect1360"
+       is_visible="true"
+       lpeversion="1" />
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect1448"
+       is_visible="true"
+       lpeversion="1" />
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect1390"
+       is_visible="true"
+       lpeversion="1" />
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect1328"
+       is_visible="true"
+       lpeversion="1" />
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect441"
+       is_visible="true"
+       lpeversion="1" />
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect2806"
+       is_visible="true"
+       lpeversion="1" />
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect2682"
+       is_visible="true"
+       lpeversion="1" />
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect2678"
+       is_visible="true"
+       lpeversion="1" />
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect2379"
+       is_visible="true"
+       lpeversion="1" />
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect108773"
+       is_visible="true"
+       lpeversion="1" />
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect108767"
+       is_visible="true"
+       lpeversion="1" />
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect108562"
+       is_visible="true"
+       lpeversion="1" />
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect108388"
+       is_visible="true"
+       lpeversion="1" />
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect108136"
+       is_visible="true"
+       lpeversion="1" />
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect108050"
+       is_visible="true"
+       lpeversion="1" />
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect107847"
+       is_visible="true"
+       lpeversion="1" />
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect107761"
+       is_visible="true"
+       lpeversion="1" />
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect107484"
+       is_visible="true"
+       lpeversion="1" />
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect95429"
+       is_visible="true"
+       lpeversion="1" />
+    <marker
+       style="overflow:visible"
+       id="TriangleOutM"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="TriangleOutM"
+       inkscape:isstock="true"
+       viewBox="0 0 4.2595265 4.9243081"
+       markerWidth="4.2600002"
+       markerHeight="4.9248552"
+       preserveAspectRatio="xMidYMid">
+      <path
+         transform="scale(0.4)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 5.77,0 -2.88,5 V -5 Z"
+         id="path62344" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="Arrow1Lstart"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Lstart"
+       inkscape:isstock="true">
+      <path
+         transform="matrix(0.8,0,0,0.8,10,0)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path62199" />
+    </marker>
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect62117"
+       is_visible="true"
+       lpeversion="1" />
+    <marker
+       style="overflow:visible"
+       id="TriangleOutM-3"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="TriangleOutM"
+       inkscape:isstock="true">
+      <path
+         transform="scale(0.4)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 5.77,0 -2.88,5 V -5 Z"
+         id="path62344-6" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="TriangleOutM-3_Fnone_S-000000"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="TriangleOutM"
+       inkscape:isstock="true">
+      <path
+         transform="scale(0.4)"
+         style="fill:#000000;fill-rule:evenodd;stroke:#000000;stroke-width:1pt"
+         d="M 5.77,0 -2.88,5 V -5 Z"
+         id="path5627" />
+    </marker>
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect108050-7"
+       is_visible="true"
+       lpeversion="1" />
+    <marker
+       style="overflow:visible"
+       id="marker1227"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="TriangleOutM"
+       inkscape:isstock="true"
+       viewBox="0 0 4.2595265 4.9243081"
+       markerWidth="4.2595263"
+       markerHeight="4.9243083"
+       preserveAspectRatio="xMidYMid">
+      <path
+         transform="scale(0.4)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 5.77,0 -2.88,5 V -5 Z"
+         id="path1225" />
+    </marker>
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect108136-5"
+       is_visible="true"
+       lpeversion="1" />
+    <marker
+       style="overflow:visible"
+       id="marker1232"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="TriangleOutM"
+       inkscape:isstock="true">
+      <path
+         transform="scale(0.4)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 5.77,0 -2.88,5 V -5 Z"
+         id="path1230" />
+    </marker>
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect108388-3"
+       is_visible="true"
+       lpeversion="1" />
+    <marker
+       style="overflow:visible"
+       id="marker1237"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="TriangleOutM"
+       inkscape:isstock="true">
+      <path
+         transform="scale(0.4)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 5.77,0 -2.88,5 V -5 Z"
+         id="path1235" />
+    </marker>
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect108562-5"
+       is_visible="true"
+       lpeversion="1" />
+    <marker
+       style="overflow:visible"
+       id="marker1242"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="TriangleOutM"
+       inkscape:isstock="true">
+      <path
+         transform="scale(0.4)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 5.77,0 -2.88,5 V -5 Z"
+         id="path1240" />
+    </marker>
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect108767-6"
+       is_visible="true"
+       lpeversion="1" />
+    <marker
+       style="overflow:visible"
+       id="marker1247"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="TriangleOutM"
+       inkscape:isstock="true"
+       viewBox="0 0 4.2595265 4.9243081"
+       markerWidth="4.2595263"
+       markerHeight="4.9243083"
+       preserveAspectRatio="xMidYMid">
+      <path
+         transform="scale(0.4)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 5.77,0 -2.88,5 V -5 Z"
+         id="path1245" />
+    </marker>
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect108773-2"
+       is_visible="true"
+       lpeversion="1" />
+    <marker
+       style="overflow:visible"
+       id="TriangleOutM-3-7"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="TriangleOutM"
+       inkscape:isstock="true"
+       viewBox="0 0 4.2595265 4.9243081"
+       markerWidth="4.2595263"
+       markerHeight="4.9243083"
+       preserveAspectRatio="xMidYMid">
+      <path
+         transform="scale(0.4)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 5.77,0 -2.88,5 V -5 Z"
+         id="path62344-6-5" />
+    </marker>
+    <inkscape:path-effect
+       effect="spiro"
+       id="path-effect108767-6-3"
+       is_visible="true"
+       lpeversion="1" />
+  </defs>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(-430.99854,-194.24567)">
+    <rect
+       style="fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:0.992327"
+       id="rect351"
+       width="169.571"
+       height="119.894"
+       x="430.99854"
+       y="194.24567" />
+    <text
+       xml:space="preserve"
+       style="font-size:7.05556px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="523.56769"
+       y="295.47681"
+       id="text7266-3"><tspan
+         sodipodi:role="line"
+         id="tspan7264-6"
+         style="font-size:7.05556px;text-align:center;text-anchor:middle;stroke-width:0.264583"
+         x="523.56769"
+         y="295.47681">NUMA Level</tspan></text>
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.500001;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:2, 1;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke"
+       id="rect29192-1"
+       width="117.44209"
+       height="33.103409"
+       x="465.46393"
+       y="232.30515"
+       ry="2"
+       rx="1.9999999" />
+    <text
+       xml:space="preserve"
+       style="font-size:7.05556px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="523.71375"
+       y="260.33276"
+       id="text37653-1"><tspan
+         sodipodi:role="line"
+         id="tspan37651-0"
+         style="font-size:7.05556px;text-align:center;text-anchor:middle;stroke-width:0.264583"
+         x="523.71375"
+         y="260.33276">Socket Level</tspan></text>
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.4808;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:1.9232, 0.961599;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke"
+       id="rect1048-9"
+       width="147.28586"
+       height="30.605822"
+       x="450.54153"
+       y="267.75421"
+       ry="1.8491039"
+       rx="2.0002611" />
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.500001;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:2, 0.999982;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke"
+       id="rect48770-3"
+       width="56.600784"
+       height="33.103409"
+       x="495.88324"
+       y="196.99588"
+       ry="2"
+       rx="1.9999996" />
+    <rect
+       style="font-variation-settings:normal;fill:#afafe9;fill-opacity:1;stroke:#000000;stroke-width:0.753;stroke-linecap:butt;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke;stop-color:#000000"
+       id="rect1928-1"
+       width="29.11949"
+       height="12.373198"
+       x="509.6239"
+       y="210.07121"
+       ry="2.4999995"
+       rx="2.5" />
+    <g
+       id="g742"
+       transform="translate(28.708569,28.979003)">
+      <rect
+         style="fill:#ffffff;stroke:#000001;stroke-width:0.264999;stroke-linejoin:round;stroke-opacity:1;paint-order:markers fill stroke;stop-color:#000000"
+         id="rect1930-5"
+         width="5.7350011"
+         height="5.7350011"
+         x="486.96146"
+         y="184.41132" />
+      <rect
+         style="fill:#ffffff;stroke:#000001;stroke-width:0.264999;stroke-linejoin:round;stroke-opacity:1;paint-order:markers fill stroke;stop-color:#000000"
+         id="rect1932-4"
+         width="5.7350011"
+         height="5.7350011"
+         x="498.25366"
+         y="184.41132" />
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-size:7.05556px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="524.31555"
+       y="205.65865"
+       id="text53597-7"><tspan
+         sodipodi:role="line"
+         id="tspan53595-6"
+         style="font-size:7.05556px;text-align:center;text-anchor:middle;stroke-width:0.264583"
+         x="524.31555"
+         y="205.65865">System Level</tspan></text>
+    <path
+       style="fill:none;stroke:#000000;stroke-width:0.494089;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#TriangleStart_Fnone_S-000000);marker-end:url(#TriangleOutM-3_Fnone_S-000000)"
+       d="M 561.29231,236.42783 530.28249,217.01021"
+       id="path108384-9"
+       inkscape:path-effect="#path-effect108388-3"
+       inkscape:original-d="m 561.29231,236.42783 c -10.38789,-6.52565 -20.67275,-12.94489 -31.00982,-19.41762" />
+    <rect
+       style="font-variation-settings:normal;fill:#87cdde;fill-opacity:1;stroke:#000000;stroke-width:0.753;stroke-linecap:butt;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke;stop-color:#000000"
+       id="rect1248-1"
+       width="29.11949"
+       height="12.373198"
+       x="472.74966"
+       y="238.28702"
+       ry="2.4999995"
+       rx="2.5" />
+    <rect
+       style="fill:#ffffff;stroke:#000001;stroke-width:0.264999;stroke-linejoin:round;stroke-opacity:1;paint-order:markers fill stroke;stop-color:#000000"
+       id="rect1541-4"
+       width="5.7350011"
+       height="5.7350011"
+       x="478.79581"
+       y="241.60611" />
+    <rect
+       style="fill:#ffffff;stroke:#000001;stroke-width:0.264999;stroke-linejoin:round;stroke-opacity:1;paint-order:markers fill stroke;stop-color:#000000"
+       id="rect1857-7"
+       width="5.7350011"
+       height="5.7350011"
+       x="490.08801"
+       y="241.60611" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:0.489502;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#TriangleStart_Fnone_S-000000);marker-end:url(#TriangleOutM-3_Fnone_S-000000)"
+       d="M 471.54671,271.46782 481.70372,244.9286"
+       id="path108763-7"
+       inkscape:path-effect="#path-effect108767-6"
+       inkscape:original-d="m 471.54671,271.46782 c 3.40244,-8.91899 6.77119,-17.69255 10.15701,-26.53922" />
+    <rect
+       style="font-variation-settings:normal;fill:#ffeeaa;fill-opacity:1;stroke:#000000;stroke-width:0.753;stroke-linecap:butt;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke;stop-color:#000000"
+       id="rect12758-0"
+       width="29.118999"
+       height="12.373198"
+       x="456.87491"
+       y="273.43073"
+       ry="2.4999995"
+       rx="2.5" />
+    <rect
+       style="font-variation-settings:normal;fill:#ffeeaa;fill-opacity:1;stroke:#000000;stroke-width:0.753;stroke-linecap:butt;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke;stop-color:#000000"
+       id="rect2265"
+       width="29.118999"
+       height="12.373198"
+       x="488.62491"
+       y="273.43073"
+       ry="2.4999995"
+       rx="2.5" />
+    <g
+       id="g2275"
+       transform="translate(76.684113,22.629088)">
+      <ellipse
+         style="fill:#afe9dd;fill-opacity:1;stroke:#000000;stroke-width:1;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke"
+         id="ellipse2267"
+         cy="258.04657"
+         cx="423.40134"
+         rx="0.5"
+         ry="0.50640655" />
+      <ellipse
+         style="fill:#afe9dd;fill-opacity:1;stroke:#000000;stroke-width:1;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke"
+         id="ellipse2269"
+         cy="258.04657"
+         cx="429.5993"
+         rx="0.5"
+         ry="0.50640655" />
+      <ellipse
+         style="fill:none;fill-opacity:1;stroke:#000000;stroke-width:1;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke"
+         id="ellipse2271"
+         cy="258.04657"
+         cx="417.20334"
+         rx="0.5"
+         ry="0.50640655" />
+      <ellipse
+         style="fill:#afe9dd;fill-opacity:1;stroke:#000000;stroke-width:1;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke"
+         id="ellipse2273"
+         cy="258.04657"
+         cx="435.79727"
+         rx="0.5"
+         ry="0.50640655" />
+    </g>
+    <path
+       style="fill:none;stroke:#000000;stroke-width:0.490156;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#TriangleStart_Fnone_S-000000);marker-end:url(#TriangleOutM-3_Fnone_S-000000)"
+       d="M 502.77673,271.53882 492.86064,244.688"
+       id="path2375"
+       inkscape:path-effect="#path-effect2379"
+       inkscape:original-d="m 502.77673,271.53882 c -3.32175,-9.0237 -6.61058,-17.90027 -9.91609,-26.85082" />
+    <rect
+       style="font-variation-settings:normal;fill:#87cdde;fill-opacity:1;stroke:#000000;stroke-width:0.753;stroke-linecap:butt;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke;stop-color:#000000"
+       id="rect1916-5"
+       width="29.11949"
+       height="12.373198"
+       x="546.49811"
+       y="237.65749"
+       ry="2.4999995"
+       rx="2.5" />
+    <rect
+       style="fill:#ffffff;stroke:#000001;stroke-width:0.264999;stroke-linejoin:round;stroke-opacity:1;paint-order:markers fill stroke;stop-color:#000000"
+       id="rect1918-3"
+       width="5.7350011"
+       height="5.7350011"
+       x="552.54425"
+       y="240.97658" />
+    <rect
+       style="fill:#ffffff;stroke:#000001;stroke-width:0.264999;stroke-linejoin:round;stroke-opacity:1;paint-order:markers fill stroke;stop-color:#000000"
+       id="rect1920-6"
+       width="5.7350011"
+       height="5.7350011"
+       x="563.83643"
+       y="240.97658" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:0.490799;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#TriangleStart_Fnone_S-000000);marker-end:url(#TriangleOutM-3_Fnone_S-000000)"
+       d="M 545.30179,271.61213 555.401,244.23858"
+       id="path2640"
+       inkscape:path-effect="#path-effect2678"
+       inkscape:original-d="m 545.30179,271.61213 c 3.38308,-9.19938 6.73266,-18.24875 10.09921,-27.37355" />
+    <rect
+       style="font-variation-settings:normal;fill:#ffeeaa;fill-opacity:1;stroke:#000000;stroke-width:0.753;stroke-linecap:butt;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke;stop-color:#000000"
+       id="rect2642"
+       width="29.118999"
+       height="12.373198"
+       x="530.62335"
+       y="273.43073"
+       ry="2.4999995"
+       rx="2.5" />
+    <g
+       id="g2652"
+       transform="translate(118.68254,22.629088)">
+      <ellipse
+         style="fill:#afe9dd;fill-opacity:1;stroke:#000000;stroke-width:1;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke"
+         id="ellipse2644"
+         cy="258.04657"
+         cx="423.40134"
+         rx="0.5"
+         ry="0.50640655" />
+      <ellipse
+         style="fill:#afe9dd;fill-opacity:1;stroke:#000000;stroke-width:1;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke"
+         id="ellipse2646"
+         cy="258.04657"
+         cx="429.5993"
+         rx="0.5"
+         ry="0.50640655" />
+      <ellipse
+         style="fill:none;fill-opacity:1;stroke:#000000;stroke-width:1;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke"
+         id="ellipse2648"
+         cy="258.04657"
+         cx="417.20334"
+         rx="0.5"
+         ry="0.50640655" />
+      <ellipse
+         style="fill:#afe9dd;fill-opacity:1;stroke:#000000;stroke-width:1;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke"
+         id="ellipse2650"
+         cy="258.04657"
+         cx="435.79727"
+         rx="0.5"
+         ry="0.50640655" />
+    </g>
+    <rect
+       style="font-variation-settings:normal;fill:#ffeeaa;fill-opacity:1;stroke:#000000;stroke-width:0.753;stroke-linecap:butt;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke;stop-color:#000000"
+       id="rect2656"
+       width="29.118999"
+       height="12.373198"
+       x="562.37335"
+       y="273.43073"
+       ry="2.4999995"
+       rx="2.5" />
+    <g
+       id="g2666"
+       transform="translate(150.43255,22.629088)">
+      <ellipse
+         style="fill:#afe9dd;fill-opacity:1;stroke:#000000;stroke-width:1;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke"
+         id="ellipse2658"
+         cy="258.04657"
+         cx="423.40134"
+         rx="0.5"
+         ry="0.50640655" />
+      <ellipse
+         style="fill:#afe9dd;fill-opacity:1;stroke:#000000;stroke-width:1;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke"
+         id="ellipse2660"
+         cy="258.04657"
+         cx="429.5993"
+         rx="0.5"
+         ry="0.50640655" />
+      <ellipse
+         style="fill:none;fill-opacity:1;stroke:#000000;stroke-width:1;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke"
+         id="ellipse2662"
+         cy="258.04657"
+         cx="417.20334"
+         rx="0.5"
+         ry="0.50640655" />
+      <ellipse
+         style="fill:#afe9dd;fill-opacity:1;stroke:#000000;stroke-width:1;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke"
+         id="ellipse2664"
+         cy="258.04657"
+         cx="435.79727"
+         rx="0.5"
+         ry="0.50640655" />
+    </g>
+    <path
+       style="fill:none;stroke:#000000;stroke-width:0.490838;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#TriangleStart_Fnone_S-000000);marker-end:url(#TriangleOutM-3_Fnone_S-000000)"
+       d="m 576.51676,271.60662 -9.73113,-27.59994"
+       id="path2672"
+       inkscape:path-effect="#path-effect2682"
+       inkscape:original-d="m 576.51676,271.60662 c -3.25979,-9.27547 -6.48726,-18.39969 -9.73113,-27.59994" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:0.490715;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#TriangleStart_Fnone_S-000000);marker-end:url(#TriangleOutM-3_Fnone_S-000000)"
+       d="m 487.56018,236.95873 30.3709,-19.72977"
+       id="path2802"
+       inkscape:path-effect="#path-effect2806"
+       inkscape:original-d="m 487.56018,236.95873 c 10.17386,-6.63057 20.2468,-13.15301 30.3709,-19.72977" />
+    <text
+       xml:space="preserve"
+       style="font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="-165.23523"
+       y="497.55908"
+       id="text270"
+       transform="rotate(-75)"><tspan
+         sodipodi:role="line"
+         id="tspan268"
+         style="font-size:6.35px;stroke-width:0.264583"
+         x="-165.23523"
+         y="497.55908">Cores</tspan></text>
+    <g
+       id="g9766"
+       transform="matrix(0,1.4312737,-0.77511095,0,565.75587,22.73314)"
+       style="stroke-width:0.474708;stroke-dasharray:none">
+      <g
+         id="g3745">
+        <g
+           id="g3749"
+           transform="matrix(0,1.8465404,-0.3519523,0,241.33077,-180.72934)"
+           style="stroke-width:0.588851">
+          <path
+             style="fill:none;stroke:#000000;stroke-width:0.588851;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1"
+             d="m 179.75303,159.69653 c 4.27425,0 -0.12961,15.95005 4.29826,16.04131"
+             id="path9169"
+             sodipodi:nodetypes="cc" />
+          <path
+             style="fill:none;stroke:#000000;stroke-width:0.588851;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1"
+             d="m 179.75303,191.77915 c 4.27425,0 -0.12961,-15.95005 4.29826,-16.04131"
+             id="path9169-7"
+             sodipodi:nodetypes="cc" />
+        </g>
+      </g>
+    </g>
+    <g
+       id="g14573"
+       transform="matrix(-1,0,0,1,924.11737,0)">
+      <text
+         xml:space="preserve"
+         style="font-size:5.64444px;line-height:1;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+         x="-447.51492"
+         y="305.93857"
+         id="text656"
+         transform="scale(-1,1)"><tspan
+           sodipodi:role="line"
+           style="font-size:5.64444px;text-align:center;text-anchor:middle;stroke-width:0.264583"
+           x="-447.51492"
+           y="305.93857"
+           id="tspan1416">NUMA 0</tspan><tspan
+           sodipodi:role="line"
+           style="font-size:5.64444px;text-align:center;text-anchor:middle;stroke-width:0.264583"
+           x="-447.51492"
+           y="311.58301"
+           id="tspan2490">Leader</tspan></text>
+      <path
+         style="fill:none;stroke:#000000;stroke-width:0.5;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#TriangleStart_Fnone_S-000000)"
+         d="m 462.02562,283.92066 v 20.70002 l -2.80093,2.32144"
+         id="path3546"
+         sodipodi:nodetypes="ccc" />
+    </g>
+    <g
+       id="g2167"
+       transform="translate(44.916471,22.629088)">
+      <ellipse
+         style="fill:#afe9dd;fill-opacity:1;stroke:#000000;stroke-width:1;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke"
+         id="circle12748-9"
+         cy="258.04657"
+         cx="423.40134"
+         rx="0.5"
+         ry="0.50640655" />
+      <ellipse
+         style="fill:#afe9dd;fill-opacity:1;stroke:#000000;stroke-width:1;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke"
+         id="circle12750-3"
+         cy="258.04657"
+         cx="429.5993"
+         rx="0.5"
+         ry="0.50640655" />
+      <ellipse
+         style="fill:none;fill-opacity:1;stroke:#000000;stroke-width:1;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke"
+         id="circle12752-6"
+         cy="258.04657"
+         cx="417.20334"
+         rx="0.5"
+         ry="0.50640655" />
+      <ellipse
+         style="fill:#afe9dd;fill-opacity:1;stroke:#000000;stroke-width:1;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke"
+         id="circle12754-0"
+         cy="258.04657"
+         cx="435.79727"
+         rx="0.5"
+         ry="0.50640655" />
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.52778px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+       x="459.90964"
+       y="278.60001"
+       id="text1273"><tspan
+         sodipodi:role="line"
+         id="tspan1271"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:3.52778px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';stroke-width:0.264583"
+         x="459.90964"
+         y="278.60001">P0</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.52778px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+       x="466.32285"
+       y="278.59824"
+       id="text1327"><tspan
+         sodipodi:role="line"
+         id="tspan1325"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:3.52778px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';stroke-width:0.264583"
+         x="466.32285"
+         y="278.59824">P1</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.52778px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+       x="472.31091"
+       y="278.59824"
+       id="text1331"><tspan
+         sodipodi:role="line"
+         id="tspan1329"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:3.52778px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';stroke-width:0.264583"
+         x="472.31091"
+         y="278.59824">P2</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.52778px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+       x="478.46829"
+       y="278.59824"
+       id="text1335"><tspan
+         sodipodi:role="line"
+         id="tspan1333"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:3.52778px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';stroke-width:0.264583"
+         x="478.46829"
+         y="278.59824">P3</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.52778px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+       x="491.64203"
+       y="278.60001"
+       id="text1504"><tspan
+         sodipodi:role="line"
+         id="tspan1502"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:3.52778px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';stroke-width:0.264583"
+         x="491.64203"
+         y="278.60001">P4</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.52778px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+       x="497.89117"
+       y="278.59824"
+       id="text1508"><tspan
+         sodipodi:role="line"
+         id="tspan1506"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:3.52778px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';stroke-width:0.264583"
+         x="497.89117"
+         y="278.59824">P5</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.52778px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+       x="504.06619"
+       y="278.59824"
+       id="text1512"><tspan
+         sodipodi:role="line"
+         id="tspan1510"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:3.52778px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';stroke-width:0.264583"
+         x="504.06619"
+         y="278.59824">P6</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.52778px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+       x="510.26416"
+       y="278.59824"
+       id="text1516"><tspan
+         sodipodi:role="line"
+         id="tspan1514"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:3.52778px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';stroke-width:0.264583"
+         x="510.26416"
+         y="278.59824">P7</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.52778px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+       x="533.67395"
+       y="278.60004"
+       id="text1522"><tspan
+         sodipodi:role="line"
+         id="tspan1520"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:3.52778px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';stroke-width:0.264583"
+         x="533.67395"
+         y="278.60004">P8</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.52778px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+       x="539.87195"
+       y="278.59827"
+       id="text1526"><tspan
+         sodipodi:role="line"
+         id="tspan1524"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:3.52778px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';stroke-width:0.264583"
+         x="539.87195"
+         y="278.59827">P9</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.52778px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+       x="546.22516"
+       y="278.59827"
+       id="text1530"><tspan
+         sodipodi:role="line"
+         id="tspan1528"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:3.52778px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';stroke-width:0.264583"
+         x="546.22516"
+         y="278.59827">10</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.52778px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+       x="552.63831"
+       y="278.59827"
+       id="text1534"><tspan
+         sodipodi:role="line"
+         id="tspan1532"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:3.52778px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';stroke-width:0.264583"
+         x="552.63831"
+         y="278.59827">11</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.52778px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+       x="565.57214"
+       y="278.60004"
+       id="text1548"><tspan
+         sodipodi:role="line"
+         id="tspan1546"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:3.52778px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';stroke-width:0.264583"
+         x="565.57214"
+         y="278.60004">12</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.52778px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+       x="571.78247"
+       y="278.59827"
+       id="text1552"><tspan
+         sodipodi:role="line"
+         id="tspan1550"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:3.52778px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';stroke-width:0.264583"
+         x="571.78247"
+         y="278.59827">13</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.52778px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+       x="577.93988"
+       y="278.59827"
+       id="text1556"><tspan
+         sodipodi:role="line"
+         id="tspan1554"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:3.52778px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';stroke-width:0.264583"
+         x="577.93988"
+         y="278.59827">14</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.52778px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+       x="584.18903"
+       y="278.59827"
+       id="text1560"><tspan
+         sodipodi:role="line"
+         id="tspan1558"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:3.52778px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';stroke-width:0.264583"
+         x="584.18903"
+         y="278.59827">15</tspan></text>
+    <g
+       id="g8037"
+       transform="matrix(-1,0,0,1,944.89717,-0.52916667)">
+      <g
+         id="g7602">
+        <text
+           xml:space="preserve"
+           style="font-style:normal;font-weight:normal;font-size:3.52778px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+           x="-475.6575"
+           y="257.66113"
+           id="text7420"
+           transform="scale(-1,1)"><tspan
+             sodipodi:role="line"
+             id="tspan7418"
+             style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:3.52778px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';stroke-width:0.264583"
+             x="-475.6575"
+             y="257.66113">P0</tspan></text>
+        <path
+           style="fill:none;stroke:#000000;stroke-width:0.365;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#TriangleStart_Fnone_S-000000);marker-end:url(#TriangleStart_Fnone_S-000000)"
+           d="m 470.86331,258.62414 -1.74113,-4.07931"
+           id="path7434" />
+      </g>
+    </g>
+    <g
+       id="g7632"
+       transform="translate(84.666671,0.52917333)">
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:3.52778px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+         x="471.76636"
+         y="257.13196"
+         id="text7628"><tspan
+           sodipodi:role="line"
+           id="tspan7626"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:3.52778px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';stroke-width:0.264583"
+           x="471.76636"
+           y="257.13196">P8</tspan></text>
+      <path
+         style="fill:none;stroke:#000000;stroke-width:0.365;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#TriangleStart_Fnone_S-000000);marker-end:url(#TriangleStart_Fnone_S-000000)"
+         d="m 469.19586,257.5658 1.74113,-4.07931"
+         id="path7630" />
+    </g>
+    <g
+       id="g8800"
+       transform="translate(0,0.52917333)">
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:3.52778px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+         x="574.71814"
+         y="256.47675"
+         id="text7636"><tspan
+           sodipodi:role="line"
+           id="tspan7634"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:3.52778px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';stroke-width:0.264583"
+           x="574.71814"
+           y="256.47675">P12</tspan></text>
+      <path
+         style="fill:none;stroke:#000000;stroke-width:0.365;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#TriangleStart_Fnone_S-000000);marker-end:url(#TriangleStart_Fnone_S-000000)"
+         d="m 574.78426,257.5658 -1.74113,-4.07931"
+         id="path7638" />
+    </g>
+    <g
+       id="g7666"
+       transform="translate(-83.60834)">
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:3.52778px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+         x="571.57312"
+         y="257.65936"
+         id="text7662"><tspan
+           sodipodi:role="line"
+           id="tspan7660"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:3.52778px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';stroke-width:0.264583"
+           x="571.57312"
+           y="257.65936">P4</tspan></text>
+      <path
+         style="fill:none;stroke:#000000;stroke-width:0.365;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#TriangleStart_Fnone_S-000000);marker-end:url(#TriangleStart_Fnone_S-000000)"
+         d="m 578.30495,258.09497 -1.74113,-4.07931"
+         id="path7664" />
+    </g>
+    <g
+       id="g8059"
+       transform="rotate(180,501.4769,223.23715)">
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:3.52778px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+         x="-503.70346"
+         y="-224.43806"
+         id="text8041"
+         transform="scale(-1)"><tspan
+           sodipodi:role="line"
+           id="tspan8039"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:3.52778px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';stroke-width:0.264583"
+           x="-503.70346"
+           y="-224.43806">P0</tspan></text>
+      <path
+         style="fill:none;stroke:#000000;stroke-width:0.365;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#TriangleStart_Fnone_S-000000);marker-end:url(#TriangleStart_Fnone_S-000000)"
+         d="m 497.88046,224.80497 3.61184,-2.57427"
+         id="path8043" />
+    </g>
+    <g
+       id="g13977"
+       transform="matrix(1,0,0,-1,45.394312,446.4743)">
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:3.52778px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+         x="499.28314"
+         y="-224.43806"
+         id="text13973"
+         transform="scale(1,-1)"><tspan
+           sodipodi:role="line"
+           id="tspan13971"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:3.52778px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';stroke-width:0.264583"
+           x="499.28314"
+           y="-224.43806">P8</tspan></text>
+      <path
+         style="fill:none;stroke:#000000;stroke-width:0.365;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#TriangleStart_Fnone_S-000000);marker-end:url(#TriangleStart_Fnone_S-000000)"
+         d="m 497.88046,224.80497 3.61184,-2.57427"
+         id="path13975" />
+    </g>
+    <g
+       id="g554"
+       transform="matrix(-1,0,0,1,955.86739,0)">
+      <text
+         xml:space="preserve"
+         style="font-size:5.64444px;line-height:1;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+         x="-447.51492"
+         y="305.93857"
+         id="text550"
+         transform="scale(-1,1)"><tspan
+           sodipodi:role="line"
+           style="font-size:5.64444px;text-align:center;text-anchor:middle;stroke-width:0.264583"
+           x="-447.51492"
+           y="305.93857"
+           id="tspan546">NUMA 1</tspan><tspan
+           sodipodi:role="line"
+           style="font-size:5.64444px;text-align:center;text-anchor:middle;stroke-width:0.264583"
+           x="-447.51492"
+           y="311.58301"
+           id="tspan548">Leader</tspan></text>
+      <path
+         style="fill:none;stroke:#000000;stroke-width:0.5;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#TriangleStart_Fnone_S-000000)"
+         d="m 462.02562,283.92066 v 20.70002 l -2.80093,2.32144"
+         id="path552"
+         sodipodi:nodetypes="ccc" />
+    </g>
+    <g
+       id="g666"
+       transform="matrix(-1,0,0,1,1029.9509,0)">
+      <text
+         xml:space="preserve"
+         style="font-size:5.64444px;line-height:1;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+         x="-447.51492"
+         y="305.93857"
+         id="text662"
+         transform="scale(-1,1)"><tspan
+           sodipodi:role="line"
+           style="font-size:5.64444px;text-align:center;text-anchor:middle;stroke-width:0.264583"
+           x="-447.51492"
+           y="305.93857"
+           id="tspan658">NUMA 3</tspan><tspan
+           sodipodi:role="line"
+           style="font-size:5.64444px;text-align:center;text-anchor:middle;stroke-width:0.264583"
+           x="-447.51492"
+           y="311.58301"
+           id="tspan660">Leader</tspan></text>
+      <path
+         style="fill:none;stroke:#000000;stroke-width:0.5;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#TriangleStart_Fnone_S-000000)"
+         d="m 462.02562,283.92066 v 20.70002 l -2.80093,2.32144"
+         id="path664"
+         sodipodi:nodetypes="ccc" />
+    </g>
+  </g>
+</svg>
diff --git a/ompi/mca/common/monitoring/HowTo_pml_monitoring.tex b/ompi/mca/common/monitoring/HowTo_pml_monitoring.tex
index 0a837e9b1df..279a375da46 100644
--- a/ompi/mca/common/monitoring/HowTo_pml_monitoring.tex
+++ b/ompi/mca/common/monitoring/HowTo_pml_monitoring.tex
@@ -356,7 +356,7 @@ \subsection{How to use the performance variables}
 \texttt{MPI\brkunds{}T\brkunds{}pvar\brkunds{}read} but cannot be
 written. Stopping the \textit{flush} performance variable, with a call
 to \texttt{MPI\brkunds{}T\brkunds{}pvar\brkunds{}stop}, force the
-counters to be flushed into the given file, reseting to 0 the counters
+counters to be flushed into the given file, resetting to 0 the counters
 at the same time. Also, binding a new handle to the \textit{flush}
 variable will reset the counters. Finally, please note that the size
 and counter performance variables may overflow for multiple large
@@ -377,7 +377,7 @@ \subsection{How to use the performance variables}
 
 \subsection{Overview of the calls}
 
-To summarize the previous informations, here is the list of available
+To summarize the previous information, here is the list of available
 performance variables, and the outline of the different calls to be
 used to properly access monitored data through the \mpit{} interface.
 \begin{itemize}
@@ -631,7 +631,7 @@ \subsubsection{Monitoring Phases}
 
 As it show on the sample profiling, for each kind of communication
 (point-to-point, one-sided and collective), you find all the related
-informations. There is one line per peers communicating. Each line
+information. There is one line per peers communicating. Each line
 start with a lettre describing the kind of communication, such as
 follows:
 
diff --git a/ompi/mca/common/monitoring/Makefile.am b/ompi/mca/common/monitoring/Makefile.am
index 08088818508..28f4d442d27 100644
--- a/ompi/mca/common/monitoring/Makefile.am
+++ b/ompi/mca/common/monitoring/Makefile.am
@@ -35,7 +35,7 @@ ompi_monitoring_prof_la_LDFLAGS= \
     -module -avoid-version -shared $(WRAPPER_EXTRA_LDFLAGS)
 ompi_monitoring_prof_la_LIBADD = \
     $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
-    $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+    $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 
 if OPAL_INSTALL_BINARIES
 bin_SCRIPTS = profile2mat.pl aggregate_profile.pl
diff --git a/ompi/mca/common/monitoring/README.md b/ompi/mca/common/monitoring/README.md
index 4f46523e6c6..74ac044988a 100644
--- a/ompi/mca/common/monitoring/README.md
+++ b/ompi/mca/common/monitoring/README.md
@@ -144,7 +144,7 @@ Two perl scripts are provided in test/monitoring:
 For instance, the provided examples store phases output in `./prof`:
 
 ```
-shell$ mpirun -np 4 --mca pml_monitoring_enable 2 ./monitoring_test
+shell$ mpirun -n 4 --mca pml_monitoring_enable 2 ./monitoring_test
 ```
 
 Should provide the following output:
diff --git a/ompi/mca/common/monitoring/common_monitoring.c b/ompi/mca/common/monitoring/common_monitoring.c
index 9c82a7bba8c..14e69fb4326 100644
--- a/ompi/mca/common/monitoring/common_monitoring.c
+++ b/ompi/mca/common/monitoring/common_monitoring.c
@@ -11,6 +11,7 @@
  *                         reserved.
  * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * Copyright (c) 2019      Triad National Security, LLC. All rights reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -55,7 +56,7 @@ static opal_output_stream_t mca_common_monitoring_output_stream_obj = {
 };
 
 /*** MCA params to mark the monitoring as enabled. ***/
-/* This signals that the monitoring will highjack the PML, OSC and COLL */
+/* This signals that the monitoring will hijack the PML, OSC and COLL */
 int mca_common_monitoring_enabled = 0;
 int mca_common_monitoring_current_state = 0;
 /* Signals there will be an output of the monitored data at component close */
@@ -64,7 +65,7 @@ static int mca_common_monitoring_output_enabled = 0;
 static char* mca_common_monitoring_initial_filename = "";
 static char* mca_common_monitoring_current_filename = NULL;
 
-/* array for stroring monitoring data*/
+/* array for storing monitoring data*/
 static opal_atomic_size_t* pml_data = NULL;
 static opal_atomic_size_t* pml_count = NULL;
 static opal_atomic_size_t* filtered_pml_data = NULL;
@@ -83,7 +84,7 @@ static double log10_2 = 0.;
 static int rank_world = -1;
 static int nprocs_world = 0;
 
-opal_hash_table_t *common_monitoring_translation_ht = NULL;
+opal_hash_table_t *ompi_common_monitoring_translation_ht = NULL;
 
 /* Reset all the monitoring arrays */
 static void mca_common_monitoring_reset ( void );
@@ -91,35 +92,35 @@ static void mca_common_monitoring_reset ( void );
 /* Flushes the monitored data and reset the values */
 static int mca_common_monitoring_flush (int fd, char* filename);
 
-/* Retreive the PML recorded count of messages sent */
+/* Retrieve the PML recorded count of messages sent */
 static int mca_common_monitoring_get_pml_count (const struct mca_base_pvar_t *pvar,
                                                 void *value, void *obj_handle);
 
-/* Retreive the PML recorded amount of data sent */
+/* Retrieve the PML recorded amount of data sent */
 static int mca_common_monitoring_get_pml_size (const struct mca_base_pvar_t *pvar,
                                                void *value, void *obj_handle);
 
-/* Retreive the OSC recorded count of messages sent */
+/* Retrieve the OSC recorded count of messages sent */
 static int mca_common_monitoring_get_osc_sent_count (const struct mca_base_pvar_t *pvar,
                                                      void *value, void *obj_handle);
 
-/* Retreive the OSC recorded amount of data sent */
+/* Retrieve the OSC recorded amount of data sent */
 static int mca_common_monitoring_get_osc_sent_size (const struct mca_base_pvar_t *pvar,
                                                     void *value, void *obj_handle);
 
-/* Retreive the OSC recorded count of messages received */
+/* Retrieve the OSC recorded count of messages received */
 static int mca_common_monitoring_get_osc_recv_count (const struct mca_base_pvar_t *pvar,
                                                      void *value, void *obj_handle);
 
-/* Retreive the OSC recorded amount of data received */
+/* Retrieve the OSC recorded amount of data received */
 static int mca_common_monitoring_get_osc_recv_size (const struct mca_base_pvar_t *pvar,
                                                     void *value, void *obj_handle);
 
-/* Retreive the COLL recorded count of messages sent */
+/* Retrieve the COLL recorded count of messages sent */
 static int mca_common_monitoring_get_coll_count (const struct mca_base_pvar_t *pvar,
                                                  void *value, void *obj_handle);
 
-/* Retreive the COLL recorded amount of data sent */
+/* Retrieve the COLL recorded amount of data sent */
 static int mca_common_monitoring_get_coll_size (const struct mca_base_pvar_t *pvar,
                                                 void *value, void *obj_handle);
 
@@ -225,8 +226,8 @@ int mca_common_monitoring_init( void )
     mca_common_monitoring_output_stream_id =
         opal_output_open(&mca_common_monitoring_output_stream_obj);
     /* Initialize proc translation hashtable */
-    common_monitoring_translation_ht = OBJ_NEW(opal_hash_table_t);
-    opal_hash_table_init(common_monitoring_translation_ht, 2048);
+    ompi_common_monitoring_translation_ht = OBJ_NEW(opal_hash_table_t);
+    opal_hash_table_init(ompi_common_monitoring_translation_ht, 2048);
     return OMPI_SUCCESS;
 }
 
@@ -236,7 +237,7 @@ void mca_common_monitoring_finalize( void )
         0 < opal_atomic_sub_fetch_32(&mca_common_monitoring_hold, 1) ) return;
 
     OPAL_MONITORING_PRINT_INFO("common_component_finish");
-    /* Dump monitoring informations */
+    /* Dump monitoring information */
     mca_common_monitoring_flush(mca_common_monitoring_output_enabled,
                                 mca_common_monitoring_current_filename);
     /* Disable all monitoring */
@@ -246,8 +247,8 @@ void mca_common_monitoring_finalize( void )
     free(mca_common_monitoring_output_stream_obj.lds_prefix);
     /* Free internal data structure */
     free((void *) pml_data);  /* a single allocation */
-    opal_hash_table_remove_all( common_monitoring_translation_ht );
-    OBJ_RELEASE(common_monitoring_translation_ht);
+    opal_hash_table_remove_all( ompi_common_monitoring_translation_ht );
+    OBJ_RELEASE(ompi_common_monitoring_translation_ht);
     mca_common_monitoring_coll_finalize();
     if( NULL != mca_common_monitoring_current_filename ) {
         free(mca_common_monitoring_current_filename);
@@ -484,7 +485,7 @@ int mca_common_monitoring_add_procs(struct ompi_proc_t **procs,
 
             key = *((uint64_t*)&tmp);
             /* save the rank of the process in MPI_COMM_WORLD in the hash using the proc_name as the key */
-            if( OPAL_SUCCESS != opal_hash_table_set_value_uint64(common_monitoring_translation_ht,
+            if( OPAL_SUCCESS != opal_hash_table_set_value_uint64(ompi_common_monitoring_translation_ht,
                                                                  key, (void*)(uintptr_t)peer_rank) ) {
                 return OMPI_ERR_OUT_OF_RESOURCE;  /* failed to allocate memory or growing the hash table */
             }
diff --git a/ompi/mca/common/monitoring/common_monitoring.h b/ompi/mca/common/monitoring/common_monitoring.h
index 8a2323effc1..2127717f933 100644
--- a/ompi/mca/common/monitoring/common_monitoring.h
+++ b/ompi/mca/common/monitoring/common_monitoring.h
@@ -2,6 +2,7 @@
  * Copyright (c) 2016-2018 Inria.  All rights reserved.
  * Copyright (c) 2019      Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -45,7 +46,7 @@ BEGIN_C_DECLS
 extern int mca_common_monitoring_output_stream_id;
 extern int mca_common_monitoring_enabled;
 extern int mca_common_monitoring_current_state;
-extern opal_hash_table_t *common_monitoring_translation_ht;
+extern opal_hash_table_t *ompi_common_monitoring_translation_ht;
 
 OMPI_DECLSPEC int mca_common_monitoring_init( void );
 OMPI_DECLSPEC void mca_common_monitoring_finalize( void );
@@ -86,9 +87,9 @@ static inline int mca_common_monitoring_get_world_rank(int dest, ompi_group_t *g
     uint64_t rank, key = *((uint64_t*)&tmp);
     /**
      * If this fails the destination is not part of my MPI_COM_WORLD
-     * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+     * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
      */
-    int ret = opal_hash_table_get_value_uint64(common_monitoring_translation_ht,
+    int ret = opal_hash_table_get_value_uint64(ompi_common_monitoring_translation_ht,
                                                key, (void *)&rank);
 
     /* Use intermediate variable to avoid overwriting while looking up in the hashtbale. */
@@ -97,7 +98,7 @@ static inline int mca_common_monitoring_get_world_rank(int dest, ompi_group_t *g
 }
 
 /* Return the current status of the monitoring system 0 if off or the
- * seperation between internal tags and external tags is disabled. Any
+ * separation between internal tags and external tags is disabled. Any
  * other positive value if the segregation between point-to-point and
  * collective is enabled.
  */
diff --git a/ompi/mca/common/monitoring/common_monitoring_coll.c b/ompi/mca/common/monitoring/common_monitoring_coll.c
index b15db151b92..a7fa7bdb0df 100644
--- a/ompi/mca/common/monitoring/common_monitoring_coll.c
+++ b/ompi/mca/common/monitoring/common_monitoring_coll.c
@@ -123,7 +123,7 @@ void mca_common_monitoring_coll_release(mca_monitoring_coll_data_t*data)
 {
 #if OPAL_ENABLE_DEBUG
     if( NULL == data ) {
-        OPAL_MONITORING_PRINT_ERR("coll: release: data structure empty or already desallocated");
+        OPAL_MONITORING_PRINT_ERR("coll: release: data structure empty or already deallocated");
         return;
     }
 #endif /* OPAL_ENABLE_DEBUG */
@@ -137,7 +137,7 @@ static void mca_common_monitoring_coll_cond_release(mca_monitoring_coll_data_t*d
 {
 #if OPAL_ENABLE_DEBUG
     if( NULL == data ) {
-        OPAL_MONITORING_PRINT_ERR("coll: release: data structure empty or already desallocated");
+        OPAL_MONITORING_PRINT_ERR("coll: release: data structure empty or already deallocated");
         return;
     }
 #endif /* OPAL_ENABLE_DEBUG */
diff --git a/ompi/mca/common/monitoring/common_monitoring_coll.h b/ompi/mca/common/monitoring/common_monitoring_coll.h
index ab6d3c5fc41..6d887da7dfb 100644
--- a/ompi/mca/common/monitoring/common_monitoring_coll.h
+++ b/ompi/mca/common/monitoring/common_monitoring_coll.h
@@ -19,6 +19,7 @@ BEGIN_C_DECLS
 
 #include "ompi_config.h"
 #include "opal/mca/base/mca_base_pvar.h"
+#include "ompi/mca/common/monitoring/common_monitoring.h"
 
 OMPI_DECLSPEC void mca_common_monitoring_coll_flush(FILE *pf, mca_monitoring_coll_data_t*data);
 
diff --git a/ompi/mca/common/monitoring/configure.m4 b/ompi/mca/common/monitoring/configure.m4
index b7632bd4b8d..11cd33b3e8c 100644
--- a/ompi/mca/common/monitoring/configure.m4
+++ b/ompi/mca/common/monitoring/configure.m4
@@ -16,13 +16,10 @@
 AC_DEFUN([MCA_ompi_common_monitoring_CONFIG],[
     AC_CONFIG_FILES([ompi/mca/common/monitoring/Makefile])
 
-    m4_ifdef([project_ompi], [
-          m4_ifdef([MCA_BUILD_ompi_common_monitoring_DSO_TRUE],
-                   [AC_CONFIG_LINKS(profile2mat.pl:test/monitoring/profile2mat.pl
-                                    aggregate_profile.pl:test/monitoring/aggregate_profile.pl)])])
+    m4_ifdef([project_ompi],
+             [AC_CONFIG_LINKS(test/monitoring/profile2mat.pl:ompi/mca/common/monitoring/profile2mat.pl
+                              test/monitoring/aggregate_profile.pl:ompi/mca/common/monitoring/aggregate_profile.pl)])
 
 
-    AS_IF([test "$MCA_BUILD_ompi_common_monitoring_DSO_TRUE" = ''],
-          [$1],
-          [$2])
+    [$1]
 ])dnl
diff --git a/ompi/mca/common/monitoring/monitoring_prof.c b/ompi/mca/common/monitoring/monitoring_prof.c
index 2c23e7fce81..43fa73c957b 100644
--- a/ompi/mca/common/monitoring/monitoring_prof.c
+++ b/ompi/mca/common/monitoring/monitoring_prof.c
@@ -27,7 +27,7 @@ Contact the authors for questions.
 
 To be run as:
 
-mpirun -np 4 \
+mpirun -n 4 \
     --mca pml_monitoring_enable 1 \
     -x LD_PRELOAD=ompi_install_dir/lib/ompi_monitoring_prof.so \
     ./my_app
@@ -112,7 +112,7 @@ int MPI_Init(int* argc, char*** argv)
 
     MPIT_result = MPI_T_init_thread(MPI_THREAD_SINGLE, &provided);
     if (MPIT_result != MPI_SUCCESS) {
-        fprintf(stderr, "ERROR : failed to intialize MPI_T interface, preventing to get monitoring results: check your OpenMPI installation\n");
+        fprintf(stderr, "ERROR : failed to initialize MPI_T interface, preventing to get monitoring results: check your OpenMPI installation\n");
         PMPI_Abort(MPI_COMM_WORLD, MPIT_result);
     }
 
diff --git a/ompi/mca/common/monitoring/profile2mat.pl b/ompi/mca/common/monitoring/profile2mat.pl
index 69275a24ff5..5dbf90de6da 100755
--- a/ompi/mca/common/monitoring/profile2mat.pl
+++ b/ompi/mca/common/monitoring/profile2mat.pl
@@ -15,15 +15,15 @@
 #
 # Author Emmanuel Jeannot <emmanuel.jeannot@inria.fr>
 #
-# Take a profile file and aggregates all the recorded communicaton into matrices.
-# It generated a matrices for teh number of messages, (msg),
+# Take a profile file and aggregates all the recorded communication into matrices.
+# It generated a matrices for the number of messages, (msg),
 # for the total bytes transmitted (size) and
 # the average nulber of bytes per messages (avg)
 #
-# The output matix is symetric
+# The output matrix is symmetric
 #
 # If possible it creates file with "internal" tags (collexctive and eta data),
-# "external" tags (point to point messages)  and "all" (every messgaes).
+# "external" tags (point to point messages)  and "all" (every messages).
 #
 # ensure that this script as the executable right: chmod +x ...
 #
diff --git a/ompi/mca/common/ompio/Makefile.am b/ompi/mca/common/ompio/Makefile.am
index de1ec7d2550..ae74f6399fe 100644
--- a/ompi/mca/common/ompio/Makefile.am
+++ b/ompi/mca/common/ompio/Makefile.am
@@ -35,6 +35,7 @@ sources = \
 	common_ompio_file_open.c   \
 	common_ompio_file_view.c   \
 	common_ompio_file_read.c   \
+	common_ompio_file_read_all.c \
 	common_ompio_buffer.c      \
 	common_ompio_file_write.c
 
diff --git a/ompi/mca/common/ompio/common_ompio.h b/ompi/mca/common/ompio/common_ompio.h
index 1f2bbc585d7..8b12f7b8cc5 100644
--- a/ompi/mca/common/ompio/common_ompio.h
+++ b/ompi/mca/common/ompio/common_ompio.h
@@ -14,6 +14,9 @@
  * Copyright (c) 2018      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2018      DataDirect Networks. All rights reserved.
+ * Copyright (c) 2022-2023 Advanced Micro Devices, Inc. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -88,7 +91,7 @@
 #define OMPIO_LOCK_SELECTIVE      11
 
 #define OMPIO_FCOLL_WANT_TIME_BREAKDOWN 0
-#define MCA_IO_DEFAULT_FILE_VIEW_SIZE 4*1024*1024
+#define MCA_IO_DEFAULT_FILE_VIEW_SIZE 6*1024*1024
 
 #define OMPIO_UNIFORM_DIST_THRESHOLD     0.5
 #define OMPIO_CONTG_THRESHOLD        1048576
@@ -140,9 +143,24 @@ typedef int (*mca_common_ompio_generate_current_file_view_fn_t) (struct ompio_fi
    temporary buffer on aggregators from the fcoll modules */
 typedef int (*mca_common_ompio_get_mca_parameter_value_fn_t) ( char *mca_parameter_name, int name_length );
 
-
 struct mca_common_ompio_print_queue;
 
+/* File View parameters and position in file*/
+struct ompio_fview_t {
+    uint32_t               f_flags;
+    OMPI_MPI_OFFSET_TYPE   f_offset; /* byte offset of current position */
+    OMPI_MPI_OFFSET_TYPE   f_disp;   /* file_view displacement */
+    struct iovec          *f_decoded_iov;
+    uint32_t               f_iov_count;
+    size_t                 f_position_in_file_view; /* in bytes */
+    size_t                 f_total_bytes; /* total bytes read/written within 1 Fview*/
+    int                    f_index_in_file_view;
+    ptrdiff_t              f_view_extent;
+    size_t                 f_view_size;
+    size_t                 f_etype_size;
+};
+typedef struct ompio_fview_t ompio_fview_t;
+
 /**
  * Back-end structure for MPI_File
  */
@@ -150,19 +168,18 @@ struct ompio_file_t {
     /* General parameters */
     int                    fd;
     struct ompi_file_t    *f_fh;     /* pointer back to the file_t structure */
-    OMPI_MPI_OFFSET_TYPE   f_offset; /* byte offset of current position */
-    OMPI_MPI_OFFSET_TYPE   f_disp;   /* file_view displacement */
+    uint32_t               f_flags;
     int                    f_rank;
     int                    f_size;
     int                    f_amode;
     int                    f_perm;
     ompi_communicator_t   *f_comm;
     const char            *f_filename;
+    char                  *f_fullfilename;
     char                  *f_datarep;
     opal_convertor_t      *f_mem_convertor;
     opal_convertor_t      *f_file_convertor;
     opal_info_t           *f_info;
-    int32_t                f_flags;
     void                  *f_fs_ptr;
     int                    f_fs_block_size;
     int                    f_atomicity;
@@ -179,24 +196,16 @@ struct ompio_file_t {
     */
     void                  *f_sharedfp_data;
 
-
     /* File View parameters */
-    struct iovec     *f_decoded_iov;
-    uint32_t          f_iov_count;
-    ompi_datatype_t  *f_iov_type;
-    size_t            f_position_in_file_view; /* in bytes */
-    size_t            f_total_bytes; /* total bytes read/written within 1 Fview*/
-    int               f_index_in_file_view;
-    ptrdiff_t         f_view_extent;
-    size_t            f_view_size;
-    ompi_datatype_t  *f_etype;
-    ompi_datatype_t  *f_filetype;
-    ompi_datatype_t  *f_orig_filetype; /* the fileview passed by the user to us */
-    size_t            f_etype_size;
+    struct ompio_fview_t   f_fview;
+    ompi_datatype_t       *f_iov_type;
+    ompi_datatype_t       *f_etype;
+    ompi_datatype_t       *f_filetype;
+    ompi_datatype_t       *f_orig_filetype; /* the fileview passed by the user to us */
 
     /* contains IO requests that needs to be read/written */
     mca_common_ompio_io_array_t *f_io_array;
-    int                      f_num_of_io_entries;
+    int                    f_num_of_io_entries;
 
     /* Hooks for modules to hang things */
     mca_base_component_t *f_fs_component;
@@ -242,68 +251,72 @@ typedef struct mca_common_ompio_data_t mca_common_ompio_data_t;
 #include "common_ompio_print_queue.h"
 #include "common_ompio_aggregators.h"
 
-OMPI_DECLSPEC int mca_common_ompio_file_write (ompio_file_t *fh, const void *buf,  int count,
+OMPI_DECLSPEC int mca_common_ompio_file_write (ompio_file_t *fh, const void *buf, size_t count,
                                                struct ompi_datatype_t *datatype, 
                                                ompi_status_public_t *status);
 
 OMPI_DECLSPEC int mca_common_ompio_file_write_at (ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE offset,  const void *buf,
-                                                  int count,  struct ompi_datatype_t *datatype, 
+                                                  size_t count, struct ompi_datatype_t *datatype,
                                                   ompi_status_public_t *status);
 
-OMPI_DECLSPEC int mca_common_ompio_file_iwrite (ompio_file_t *fh, const void *buf, int count,
+OMPI_DECLSPEC int mca_common_ompio_file_iwrite (ompio_file_t *fh, const void *buf, size_t count,
                                                 struct ompi_datatype_t *datatype, ompi_request_t **request);
 
+OMPI_DECLSPEC int mca_common_ompio_file_iwrite_pregen (ompio_file_t *fh, ompi_request_t *request);
+
 OMPI_DECLSPEC int mca_common_ompio_file_iwrite_at (ompio_file_t *fh,  OMPI_MPI_OFFSET_TYPE offset,
-                                                   const void *buf,  int count,  struct ompi_datatype_t *datatype,
+                                                   const void *buf,  size_t count,  struct ompi_datatype_t *datatype,
                                                    ompi_request_t **request);
 
 OMPI_DECLSPEC int mca_common_ompio_file_write_all (ompio_file_t *fh, const void *buf,
-                                                   int count, struct ompi_datatype_t *datatype, 
+                                                   size_t count, struct ompi_datatype_t *datatype,
                                                    ompi_status_public_t *status);
 
 OMPI_DECLSPEC int mca_common_ompio_file_write_at_all (ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE offset, const void *buf,
-                                                      int count, struct ompi_datatype_t *datatype, 
+                                                      size_t count, struct ompi_datatype_t *datatype,
                                                       ompi_status_public_t *status);
 
 OMPI_DECLSPEC int mca_common_ompio_file_iwrite_all (ompio_file_t *fp, const void *buf,
-                                                    int count, struct ompi_datatype_t *datatype, ompi_request_t **request);
+                                                    size_t count, struct ompi_datatype_t *datatype, ompi_request_t **request);
 
 OMPI_DECLSPEC int mca_common_ompio_file_iwrite_at_all (ompio_file_t *fp, OMPI_MPI_OFFSET_TYPE offset, const void *buf,
-                                                       int count, struct ompi_datatype_t *datatype, ompi_request_t **request);
+                                                       size_t count, struct ompi_datatype_t *datatype, ompi_request_t **request);
 
-OMPI_DECLSPEC int mca_common_ompio_build_io_array ( ompio_file_t *fh, int index, int cycles,
+OMPI_DECLSPEC int mca_common_ompio_build_io_array ( ompio_fview_t *fview, int index, int cycles,
                                                     size_t bytes_per_cycle, size_t max_data, uint32_t iov_count,
-                                                    struct iovec *decoded_iov, int *ii, int *jj, size_t *tbw,
+                                                    struct iovec *decoded_iov, int *ii, size_t *tbw,
                                                     size_t *spc, mca_common_ompio_io_array_t **io_array,
                                                     int *num_io_entries );
 
 
-OMPI_DECLSPEC int mca_common_ompio_file_read (ompio_file_t *fh,  void *buf,  int count,
+OMPI_DECLSPEC int mca_common_ompio_file_read (ompio_file_t *fh,  void *buf,  size_t count,
                                               struct ompi_datatype_t *datatype, ompi_status_public_t *status);
 
 OMPI_DECLSPEC int mca_common_ompio_file_read_at (ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE offset,  void *buf,
-                                                 int count, struct ompi_datatype_t *datatype, 
+                                                 size_t count, struct ompi_datatype_t *datatype,
                                                  ompi_status_public_t * status);
 
-OMPI_DECLSPEC int mca_common_ompio_file_iread (ompio_file_t *fh, void *buf, int count,
+OMPI_DECLSPEC int mca_common_ompio_file_iread (ompio_file_t *fh, void *buf, size_t count,
                                                struct ompi_datatype_t *datatype, ompi_request_t **request);
 
+OMPI_DECLSPEC int mca_common_ompio_file_iread_pregen (ompio_file_t *fh, ompi_request_t *request);
+
 OMPI_DECLSPEC int mca_common_ompio_file_iread_at (ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE offset,
-                                                  void *buf, int count, struct ompi_datatype_t *datatype,
+                                                  void *buf, size_t count, struct ompi_datatype_t *datatype,
                                                   ompi_request_t **request);
 
-OMPI_DECLSPEC int mca_common_ompio_file_read_all (ompio_file_t *fh, void *buf, int count, struct ompi_datatype_t *datatype,
+OMPI_DECLSPEC int mca_common_ompio_file_read_all (ompio_file_t *fh, void *buf, size_t count, struct ompi_datatype_t *datatype,
                                                   ompi_status_public_t * status);
 
 OMPI_DECLSPEC int mca_common_ompio_file_read_at_all (ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE offset,
-                                                     void *buf, int count, struct ompi_datatype_t *datatype,
+                                                     void *buf, size_t count, struct ompi_datatype_t *datatype,
                                                      ompi_status_public_t * status);
 
-OMPI_DECLSPEC int mca_common_ompio_file_iread_all (ompio_file_t *fp, void *buf, int count, struct ompi_datatype_t *datatype,
+OMPI_DECLSPEC int mca_common_ompio_file_iread_all (ompio_file_t *fp, void *buf, size_t count, struct ompi_datatype_t *datatype,
                                                    ompi_request_t **request);
 
 OMPI_DECLSPEC int mca_common_ompio_file_iread_at_all (ompio_file_t *fp, OMPI_MPI_OFFSET_TYPE offset,
-                                                      void *buf, int count, struct ompi_datatype_t *datatype,
+                                                      void *buf, size_t count, struct ompi_datatype_t *datatype,
                                                       ompi_request_t **request);
 
 OMPI_DECLSPEC int mca_common_ompio_file_open (ompi_communicator_t *comm, const char *filename,
@@ -323,6 +336,9 @@ OMPI_DECLSPEC int mca_common_ompio_set_file_defaults (ompio_file_t *fh);
 OMPI_DECLSPEC int mca_common_ompio_set_view (ompio_file_t *fh,  OMPI_MPI_OFFSET_TYPE disp,
                                              ompi_datatype_t *etype,  ompi_datatype_t *filetype, const char *datarep,
                                              opal_info_t *info);
+OMPI_DECLSPEC int mca_common_ompio_base_file_read_all (struct ompio_file_t *fh, void *buf, size_t count,
+                                                       struct ompi_datatype_t *datatype, ompi_status_public_t *status);
+
  
 
 /*
@@ -331,13 +347,15 @@ OMPI_DECLSPEC int mca_common_ompio_set_view (ompio_file_t *fh,  OMPI_MPI_OFFSET_
  */
 OMPI_DECLSPEC int mca_common_ompio_decode_datatype (struct ompio_file_t *fh,
                                                     struct ompi_datatype_t *datatype,
-                                                    int count,
+                                                    size_t count,
                                                     const void *buf,
                                                     size_t *max_data,
                                                     opal_convertor_t *convertor,
                                                     struct iovec **iov,
                                                     uint32_t *iov_count);
 
+OMPI_DECLSPEC int mca_common_ompio_fview_duplicate (struct ompio_fview_t *outfv, struct ompio_fview_t *infv);
+
 OMPI_DECLSPEC int mca_common_ompio_set_callbacks(mca_common_ompio_generate_current_file_view_fn_t generate_current_file_view,
                                                  mca_common_ompio_get_mca_parameter_value_fn_t get_mca_parameter_value);
 #endif /* MCA_COMMON_OMPIO_H */
diff --git a/ompi/mca/common/ompio/common_ompio_aggregators.c b/ompi/mca/common/ompio/common_ompio_aggregators.c
index 088e88db3fd..050e35dc146 100644
--- a/ompi/mca/common/ompio/common_ompio_aggregators.c
+++ b/ompi/mca/common/ompio/common_ompio_aggregators.c
@@ -16,6 +16,9 @@
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -42,7 +45,7 @@
 #include "common_ompio.h"
 
 /*
-** This file contains all the functionality related to determing the number of aggregators
+** This file contains all the functionality related to determining the number of aggregators
 ** and the list of aggregators.
 **
 ** The first group functions determines the number of aggregators based on various characteristics
@@ -195,16 +198,13 @@ int  mca_common_ompio_forced_grouping ( ompio_file_t *fh,
     int rest = fh->f_size % num_groups;
     int flag = OMPI_COMM_IS_MAPBY_NODE (&ompi_mpi_comm_world.comm);
     int k=0, p=0, g=0;
-    int total_procs = 0; 
 
     for ( k=0, p=0; p<num_groups; p++ ) {
         if ( p < rest ) {
             contg_groups[p].procs_per_contg_group = group_size+1;
-            total_procs +=(group_size+1);
         }
         else {
             contg_groups[p].procs_per_contg_group = group_size;
-            total_procs +=group_size;
         }
 
         if ( flag ) {
@@ -239,13 +239,13 @@ int mca_common_ompio_fview_based_grouping(ompio_file_t *fh,
     OMPI_MPI_OFFSET_TYPE *start_offsets_lens = NULL;
 
     //Store start offset,length and corresponding rank in an array
-    if(NULL == fh->f_decoded_iov){
+    if(NULL == fh->f_fview.f_decoded_iov){
       start_offset_len[0] = 0;
       start_offset_len[1] = 0;
     }
     else{
-       start_offset_len[0] = (OMPI_MPI_OFFSET_TYPE) fh->f_decoded_iov[0].iov_base;
-       start_offset_len[1] = fh->f_decoded_iov[0].iov_len;
+       start_offset_len[0] = (OMPI_MPI_OFFSET_TYPE) fh->f_fview.f_decoded_iov[0].iov_base;
+       start_offset_len[1] = fh->f_fview.f_decoded_iov[0].iov_len;
     }
     start_offset_len[2] = fh->f_rank;
 
@@ -1050,11 +1050,11 @@ int mca_common_ompio_merge_groups(ompio_file_t *fh,
 			          int num_merge_aggrs)
 {
     int i = 0;
-    int *sizes_old_group;
+    size_t *sizes_old_group;
     int ret;
-    int *displs = NULL;
+    ptrdiff_t *displs = NULL;
 
-    sizes_old_group = (int*)malloc(num_merge_aggrs * sizeof(int));
+    sizes_old_group = (size_t *)malloc(num_merge_aggrs * sizeof(size_t));
     if (NULL == sizes_old_group) {
         opal_output (1, "OUT OF MEMORY\n");
         ret = OMPI_ERR_OUT_OF_RESOURCE;
@@ -1062,7 +1062,7 @@ int mca_common_ompio_merge_groups(ompio_file_t *fh,
     }
 
 
-    displs = (int*)malloc(num_merge_aggrs * sizeof(int));
+    displs = (ptrdiff_t *)malloc(num_merge_aggrs * sizeof(ptrdiff_t));
     if (NULL == displs) {
         opal_output (1, "OUT OF MEMORY\n");
         ret = OMPI_ERR_OUT_OF_RESOURCE;
@@ -1269,19 +1269,16 @@ int mca_common_ompio_prepare_to_group(ompio_file_t *fh,
     int i = 0;
     int j = 0;
     int k = 0;
-    int merge_count = 0;
-    int split_count = 0; //not req?
-    int retain_as_is_count = 0; //not req?
     int ret=OMPI_SUCCESS;
 
     //Store start offset and length in an array //also add bytes per process
-    if(NULL == fh->f_decoded_iov){
+    if(NULL == fh->f_fview.f_decoded_iov){
          start_offset_len[0] = 0;
          start_offset_len[1] = 0;
     }
     else{
-         start_offset_len[0] = (OMPI_MPI_OFFSET_TYPE) fh->f_decoded_iov[0].iov_base;
-         start_offset_len[1] = fh->f_decoded_iov[0].iov_len;
+         start_offset_len[0] = (OMPI_MPI_OFFSET_TYPE) fh->f_fview.f_decoded_iov[0].iov_base;
+         start_offset_len[1] = fh->f_fview.f_decoded_iov[0].iov_len;
     }
     start_offset_len[2] = bytes_per_proc;
     start_offsets_lens_tmp = (OMPI_MPI_OFFSET_TYPE* )malloc (3 * fh->f_init_procs_per_group * sizeof(OMPI_MPI_OFFSET_TYPE));
@@ -1367,16 +1364,13 @@ int mca_common_ompio_prepare_to_group(ompio_file_t *fh,
        if((size_t)(aggr_bytes_per_group_tmp[i])>
           (size_t)OMPIO_MCA_GET(fh, bytes_per_agg)){
           decision_list_tmp[i] = OMPIO_SPLIT;
-          split_count++;
        }
        else if((size_t)(aggr_bytes_per_group_tmp[i])<
                (size_t)OMPIO_MCA_GET(fh, bytes_per_agg)){
             decision_list_tmp[i] = OMPIO_MERGE;
-            merge_count++;
        }
        else{
 	   decision_list_tmp[i] = OMPIO_RETAIN;
-	   retain_as_is_count++;
 	   }
     }
 
@@ -1418,7 +1412,7 @@ int mca_common_ompio_prepare_to_group(ompio_file_t *fh,
     }
 
     //print decision list of aggregators
-    /*printf("RANK%d  : Printing decsion list   : \n",fh->f_rank);
+    /*printf("RANK%d  : Printing decision list   : \n",fh->f_rank);
     for( i = 0; i < fh->f_init_num_aggrs; i++){
         if(decision_list_tmp[i] == OMPIO_MERGE)
             printf("MERGE,");
@@ -1459,8 +1453,8 @@ int mca_common_ompio_prepare_to_group(ompio_file_t *fh,
 */
 static double cost_calc (int P, int P_a, size_t d_p, size_t b_c, int dim )
 {
-    float  n_as=1.0, m_s=1.0, n_s=1.0;
-    float  n_ar=1.0;
+    double  n_as=1.0, m_s=1.0, n_s=1.0;
+    double  n_ar=1.0;
     double t_send, t_recv, t_tot;
 
     /* LogGP parameters based on DDR InfiniBand values */
@@ -1470,7 +1464,7 @@ static double cost_calc (int P, int P_a, size_t d_p, size_t b_c, int dim )
     double G=.00000000067;
     
     long file_domain = (P * d_p) / P_a;
-    float n_r = (float)file_domain/(float) b_c;
+    double n_r = (double)file_domain/(double) b_c;
     
     switch (dim) {
 	case DIM1:
@@ -1480,10 +1474,10 @@ static double cost_calc (int P, int P_a, size_t d_p, size_t b_c, int dim )
 		n_ar = 1;
 		n_as = 1;
 		m_s = b_c;
-		n_s = (float)d_p/(float)b_c;
+		n_s = (double)d_p/(double)b_c;
 	    }
 	    else {
-		n_ar = (float)b_c/(float)d_p;
+		n_ar = (double)b_c/(double)d_p;
 		n_as = 1;
 		m_s = d_p;
 		n_s = 1;
@@ -1495,14 +1489,14 @@ static double cost_calc (int P, int P_a, size_t d_p, size_t b_c, int dim )
 	    int P_x, P_y;
 	    
 	    P_x = P_y = (int) sqrt(P);
-	    n_as = (float) P_a / (float)P_x;
+	    n_as = (double) P_a / (double)P_x;
 	    
-	    n_ar = (float) P_y;
+	    n_ar = (double) P_y;
 	    if ( d_p > (P_a*b_c/P )) {
-		m_s = fmin(b_c / P_y, d_p);
+		m_s = fmin((double) b_c / (double)P_y, (double)d_p);
 	    }
 	    else {
-		m_s = fmin(d_p * P_x / P_a, d_p);
+		m_s = fmin((double) (d_p * P_x) / (double)P_a, (double)d_p);
 	    }
 	    break;	  
 	}
@@ -1511,7 +1505,7 @@ static double cost_calc (int P, int P_a, size_t d_p, size_t b_c, int dim )
 	    break;
     } 
     
-    n_s = (float) d_p / (float)(n_as * m_s);
+    n_s = (double) d_p / (double)(n_as * m_s);
     
     if( m_s < 33554432) {
 	g = .00000108;
diff --git a/ompi/mca/common/ompio/common_ompio_buffer.c b/ompi/mca/common/ompio/common_ompio_buffer.c
index dbd7e30e6b4..c8ce40c0561 100644
--- a/ompi/mca/common/ompio/common_ompio_buffer.c
+++ b/ompi/mca/common/ompio/common_ompio_buffer.c
@@ -10,6 +10,8 @@
  *  Copyright (c) 2004-2005 The Regents of the University of California.
  *                          All rights reserved.
  *  Copyright (c) 2008-2019 University of Houston. All rights reserved.
+ *  Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ *  Copyright (c) 2025      Advanced Micro Devices, Inc. All rights reserved.
  *  $COPYRIGHT$
  *
  *  Additional copyrights may follow
@@ -20,7 +22,7 @@
 #include "ompi_config.h"
 
 #include "opal/datatype/opal_convertor.h"
-#include "opal/mca/common/cuda/common_cuda.h"
+#include "opal/mca/accelerator/accelerator.h"
 #include "opal/util/sys_limits.h"
 
 #include "opal/mca/allocator/allocator.h"
@@ -38,26 +40,28 @@ static int32_t  mca_common_ompio_pagesize=4096;
 static void* mca_common_ompio_buffer_alloc_seg ( void *ctx, size_t *size );
 static void mca_common_ompio_buffer_free_seg ( void *ctx, void *buf );
 
-#if OPAL_CUDA_SUPPORT
 void mca_common_ompio_check_gpu_buf ( ompio_file_t *fh, const void *buf, int *is_gpu, 
 				      int *is_managed)
 {
-    opal_convertor_t    convertor;  
-    
+    uint64_t flags = 0;
+    int dev_id;
+
     *is_gpu=0;
     *is_managed=0;
-    
-    convertor.flags=0;
-    if ( opal_cuda_check_one_buf ( (char *)buf, &convertor ) ) {
+
+    if (fh->f_fh->f_flags & OMPI_FILE_ASSERT_NO_ACCEL_BUF) {
+        return;
+    }
+
+    if (0 < opal_accelerator.check_addr(buf, &dev_id, &flags)) {
         *is_gpu = 1;
-        if ( convertor.flags & CONVERTOR_CUDA_UNIFIED ){
-            *is_managed =1;
+        if (flags & MCA_ACCELERATOR_FLAGS_UNIFIED_MEMORY) {
+            *is_managed = 1;
         }
-    } 
-    
+    }
+
     return;
 }
-#endif
 
 static void* mca_common_ompio_buffer_alloc_seg ( void*ctx, size_t *size )
 {
@@ -67,22 +71,25 @@ static void* mca_common_ompio_buffer_alloc_seg ( void*ctx, size_t *size )
     numpages = (*size + mca_common_ompio_pagesize -1 )/mca_common_ompio_pagesize;
     realsize = numpages * mca_common_ompio_pagesize;
 
-    buf = malloc ( realsize);
-#if OPAL_CUDA_SUPPORT
-    if ( NULL != buf ) {
-        mca_common_cuda_register ( ( char *)buf, realsize, NULL  );
+    buf = malloc (realsize);
+
+    if (NULL != buf) {
+        opal_accelerator.host_register(MCA_ACCELERATOR_NO_DEVICE_ID, (void *)buf, realsize);
     }
-#endif
+
     *size = realsize;
     return buf;
 }
 
 static void mca_common_ompio_buffer_free_seg ( void *ctx, void *buf )
 {
+    uint64_t flags = 0;
+    int dev_id;
+
     if ( NULL != buf ) {
-#if OPAL_CUDA_SUPPORT
-        mca_common_cuda_unregister ( (char *) buf, NULL );
-#endif
+        if (0 == opal_accelerator.check_addr(buf, &dev_id, &flags)) {
+            opal_accelerator.host_unregister(dev_id, (void *)buf);
+        }
         free ( buf );
     }
     return;
diff --git a/ompi/mca/common/ompio/common_ompio_buffer.h b/ompi/mca/common/ompio/common_ompio_buffer.h
index 2e5e7fcbb4c..3a968ded346 100644
--- a/ompi/mca/common/ompio/common_ompio_buffer.h
+++ b/ompi/mca/common/ompio/common_ompio_buffer.h
@@ -22,46 +22,38 @@
 #define MCA_COMMON_OMPIO_CUDA_H
 
 
-#define OMPIO_PREPARE_BUF(_fh,_buf,_count,_datatype,_tbuf,_convertor,_max_data,_decoded_iov,_iov_count){ \
+#define OMPIO_PREPARE_BUF(_fh,_buf,_count,_datatype,_tbuf,_convertor,_max_data,_tmp_buf_size,_decoded_iov,_iov_count){ \
         OBJ_CONSTRUCT( _convertor, opal_convertor_t);                                    \
     opal_convertor_copy_and_prepare_for_send ( _fh->f_file_convertor,  &(_datatype->super), _count, _buf, CONVERTOR_SEND_CONVERSION, _convertor ); \
     opal_convertor_get_packed_size( _convertor, &_max_data );           \
-    _tbuf = mca_common_ompio_alloc_buf (_fh, _max_data);                \
+    _tbuf = mca_common_ompio_alloc_buf (_fh, _tmp_buf_size==0 ? _max_data : _tmp_buf_size); \
     if ( NULL == _tbuf ) {                                              \
         opal_output(1, "common_ompio: error allocating memory\n");      \
+        opal_convertor_cleanup (_convertor);                            \
         return OMPI_ERR_OUT_OF_RESOURCE;                                \
     }                                                                   \
-    _decoded_iov = (struct iovec *) malloc ( sizeof ( struct iovec ));  \
-    if ( NULL == _decoded_iov ) {                                       \
-        opal_output(1, "common_ompio: could not allocate memory.\n");   \
-        return OMPI_ERR_OUT_OF_RESOURCE;                                \
-    }                                                                   \
-    _decoded_iov->iov_base = _tbuf;                                     \
-    _decoded_iov->iov_len  = _max_data;                                 \
-    _iov_count=1;}
+    if (NULL != _decoded_iov) {                                         \
+        ((struct iovec*)_decoded_iov)->iov_base = _tbuf;                \
+        ((struct iovec*)_decoded_iov)->iov_len  = _max_data;            \
+        _iov_count=1;}}
 
-#define OMPIO_PREPARE_READ_BUF(_fh,_buf,_count,_datatype,_tbuf,_convertor,_max_data,_decoded_iov,_iov_count){ \
+#define OMPIO_PREPARE_READ_BUF(_fh,_buf,_count,_datatype,_tbuf,_convertor,_max_data,_tmp_buf_size,_decoded_iov,_iov_count){ \
         OBJ_CONSTRUCT( _convertor, opal_convertor_t);                                    \
     opal_convertor_copy_and_prepare_for_recv ( _fh->f_file_convertor,  &(_datatype->super), _count, _buf, 0, _convertor ); \
     opal_convertor_get_packed_size( _convertor, &_max_data );           \
-    _tbuf = mca_common_ompio_alloc_buf (_fh, _max_data);                \
+    _tbuf = mca_common_ompio_alloc_buf (_fh, _tmp_buf_size==0 ? _max_data : _tmp_buf_size); \
     if ( NULL == _tbuf ) {                                              \
         opal_output(1, "common_ompio: error allocating memory\n");      \
+        opal_convertor_cleanup (_convertor);                            \
         return OMPI_ERR_OUT_OF_RESOURCE;                                \
     }                                                                   \
-    _decoded_iov = (struct iovec *) malloc ( sizeof ( struct iovec ));  \
-    if ( NULL == _decoded_iov ) {                                       \
-        opal_output(1, "common_ompio: could not allocate memory.\n");   \
-        return OMPI_ERR_OUT_OF_RESOURCE;                                \
-    }                                                                   \
-    _decoded_iov->iov_base = _tbuf;                                     \
-    _decoded_iov->iov_len  = _max_data;                                 \
-    _iov_count=1;}
+    if (NULL != _decoded_iov) {                                         \
+        ((struct iovec*)_decoded_iov)->iov_base = _tbuf;                \
+        ((struct iovec*)_decoded_iov)->iov_len  = _max_data;            \
+	_iov_count=1;}}
 
-#if OPAL_CUDA_SUPPORT
 void mca_common_ompio_check_gpu_buf ( ompio_file_t *fh, const void *buf, 
 				      int *is_gpu, int *is_managed);
-#endif
 int mca_common_ompio_buffer_alloc_init ( void );
 int mca_common_ompio_buffer_alloc_fini ( void );
 
diff --git a/ompi/mca/common/ompio/common_ompio_file_open.c b/ompi/mca/common/ompio/common_ompio_file_open.c
index 778bd0365f8..9104b175e70 100644
--- a/ompi/mca/common/ompio/common_ompio_file_open.c
+++ b/ompi/mca/common/ompio/common_ompio_file_open.c
@@ -15,6 +15,8 @@
  * Copyright (c) 2016      Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2017      IBM Corporation. All rights reserved.
  * Copyright (c) 2018      DataDirect Networks. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -41,6 +43,9 @@
 #include <math.h>
 #include "common_ompio.h"
 #include "ompi/mca/topo/topo.h"
+#include "opal/util/opal_getcwd.h"
+#include "opal/util/path.h"
+#include "opal/util/os_path.h"
 
 static mca_common_ompio_generate_current_file_view_fn_t generate_current_file_view_fn;
 static mca_common_ompio_get_mca_parameter_value_fn_t get_mca_parameter_value_fn;
@@ -96,10 +101,26 @@ int mca_common_ompio_file_open (ompi_communicator_t *comm,
     ompio_fh->f_info   = info;
 
     /* set some function pointers required for fcoll, fbtls and sharedfp modules*/
-    ompio_fh->f_generate_current_file_view=generate_current_file_view_fn;
-    ompio_fh->f_get_mca_parameter_value=get_mca_parameter_value_fn;
+    ompio_fh->f_generate_current_file_view = generate_current_file_view_fn;
+    ompio_fh->f_get_mca_parameter_value    = get_mca_parameter_value_fn;
 
     ompio_fh->f_filename = filename;
+    if (opal_path_is_absolute(filename) ) {
+        ompio_fh->f_fullfilename = strdup(filename);
+    }
+    else {
+        char path[OPAL_PATH_MAX];
+        ret = opal_getcwd(path, OPAL_PATH_MAX);
+        if (OPAL_SUCCESS != ret) {
+            goto fn_fail;
+        }
+        ompio_fh->f_fullfilename = opal_os_path(0, path, filename, NULL);
+        if (NULL == ompio_fh->f_fullfilename){
+            ret = OMPI_ERROR;
+            goto fn_fail;
+        }
+    }
+    
     mca_common_ompio_set_file_defaults (ompio_fh);
 
     ompio_fh->f_split_coll_req    = NULL;
@@ -235,6 +256,11 @@ int mca_common_ompio_file_close (ompio_file_t *ompio_fh)
     int delete_flag = 0;
     char name[256];
 
+    /* Call coll_barrier only if collectives are set (same reasoning as below for f_fs) */
+    if (NULL == ompio_fh->f_comm || NULL == ompio_fh->f_comm->c_coll) {
+        return OMPI_SUCCESS;
+    }
+
     ret = ompio_fh->f_comm->c_coll->coll_barrier ( ompio_fh->f_comm, ompio_fh->f_comm->c_coll->coll_barrier_module);
     if ( OMPI_SUCCESS != ret ) {
         /* Not sure what to do */
@@ -280,7 +306,7 @@ int mca_common_ompio_file_close (ompio_file_t *ompio_fh)
 	ret = ompio_fh->f_fs->fs_file_close (ompio_fh);
     }
     if ( delete_flag ) {
-        ret = mca_common_ompio_file_delete ( ompio_fh->f_filename, &(MPI_INFO_NULL->super) );
+        ret = mca_common_ompio_file_delete ( ompio_fh->f_fullfilename, &(MPI_INFO_NULL->super) );
     }
 
     if ( NULL != ompio_fh->f_fs ) {
@@ -319,9 +345,9 @@ int mca_common_ompio_file_close (ompio_file_t *ompio_fh)
         ompio_fh->f_procs_in_group = NULL;
     }
 
-    if (NULL != ompio_fh->f_decoded_iov) {
-        free (ompio_fh->f_decoded_iov);
-        ompio_fh->f_decoded_iov = NULL;
+    if (NULL != ompio_fh->f_fview.f_decoded_iov) {
+        free (ompio_fh->f_fview.f_decoded_iov);
+        ompio_fh->f_fview.f_decoded_iov = NULL;
     }
 
     if (NULL != ompio_fh->f_mem_convertor) {
@@ -345,7 +371,8 @@ int mca_common_ompio_file_close (ompio_file_t *ompio_fh)
         free ( ompio_fh->f_coll_write_time );
         ompio_fh->f_coll_write_time = NULL;
     }
-
+    free (ompio_fh->f_fullfilename);
+    
     if ( NULL != ompio_fh->f_coll_read_time ) {
         free ( ompio_fh->f_coll_read_time );
         ompio_fh->f_coll_read_time = NULL;
@@ -366,8 +393,7 @@ int mca_common_ompio_file_close (ompio_file_t *ompio_fh)
     if ( MPI_DATATYPE_NULL != ompio_fh->f_orig_filetype ){
 	ompi_datatype_destroy (&ompio_fh->f_orig_filetype);
     }
-
-
+    
     if (MPI_COMM_NULL != ompio_fh->f_comm && !(ompio_fh->f_flags & OMPIO_SHAREDFP_IS_SET) )  {
         ompi_comm_free (&ompio_fh->f_comm);
     }
@@ -391,21 +417,21 @@ int mca_common_ompio_file_get_position (ompio_file_t *fh,
 {
     OMPI_MPI_OFFSET_TYPE off;
 
-    if ( 0 == fh->f_view_extent ||
-         0 == fh->f_view_size   ||
-         0 == fh->f_etype_size ) {
+    if ( 0 == fh->f_fview.f_view_extent ||
+         0 == fh->f_fview.f_view_size   ||
+         0 == fh->f_fview.f_etype_size ) {
         /* not sure whether we should raise an error here */
         *offset = 0;
         return OMPI_SUCCESS;
     }
     /* No. of copies of the entire file view */
-    off = (fh->f_offset - fh->f_disp)/fh->f_view_extent;
+    off = (fh->f_fview.f_offset - fh->f_fview.f_disp)/fh->f_fview.f_view_extent;
 
     /* No. of elements per view */
-    off *= (fh->f_view_size / fh->f_etype_size);
+    off *= (fh->f_fview.f_view_size / fh->f_fview.f_etype_size);
 
     /* No of elements used in the current copy of the view */
-    off += fh->f_total_bytes / fh->f_etype_size;
+    off += fh->f_fview.f_total_bytes / fh->f_fview.f_etype_size;
 
     *offset = off;
     return OMPI_SUCCESS;
@@ -415,61 +441,60 @@ int mca_common_ompio_set_file_defaults (ompio_file_t *fh)
 {
 
    if (NULL != fh) {
-       char char_stripe[MPI_MAX_INFO_VAL];
+       opal_cstring_t *stripe_str;
        ompi_datatype_t *types[2];
        int blocklen[2] = {1, 1};
        ptrdiff_t d[2], base;
        int i, flag;
        
-       fh->f_io_array = NULL;
-       fh->f_perm = OMPIO_PERM_NULL;
-       fh->f_flags = 0;
+       fh->f_flags         = 0;
+       fh->f_perm          = OMPIO_PERM_NULL;
+       fh->f_io_array      = NULL;
        
        fh->f_bytes_per_agg = OMPIO_MCA_GET(fh, bytes_per_agg);
-       opal_info_get (fh->f_info, "cb_buffer_size", MPI_MAX_INFO_VAL, char_stripe, &flag);
+       opal_info_get (fh->f_info, "cb_buffer_size", &stripe_str, &flag);
        if ( flag ) {
            /* Info object trumps mca parameter value */
-           sscanf ( char_stripe, "%d", &fh->f_bytes_per_agg  );
-           OMPIO_MCA_PRINT_INFO(fh, "cb_buffer_size", char_stripe, "");
+           sscanf ( stripe_str->string, "%d", &fh->f_bytes_per_agg  );
+           OMPIO_MCA_PRINT_INFO(fh, "cb_buffer_size", stripe_str->string, "");
+           OBJ_RELEASE(stripe_str);
        }
 
-       fh->f_atomicity = 0;
        fh->f_fs_block_size = 4096;
+       fh->f_atomicity     = 0;
+       fh->f_stripe_size   = 0;
+       fh->f_stripe_count  = 0;
        
-       fh->f_offset = 0;
-       fh->f_disp = 0;
-       fh->f_position_in_file_view = 0;
-       fh->f_index_in_file_view = 0;
-       fh->f_total_bytes = 0;
+       /* File View */
+       fh->f_fview.f_flags                 = 0;
+       fh->f_fview.f_offset                = 0;
+       fh->f_fview.f_disp                  = 0;
+       fh->f_fview.f_position_in_file_view = 0;
+       fh->f_fview.f_index_in_file_view    = 0;
+       fh->f_fview.f_total_bytes           = 0;
+       fh->f_fview.f_decoded_iov           = NULL;
+
+       fh->f_iov_type      = MPI_DATATYPE_NULL;
+       fh->f_etype         = MPI_DATATYPE_NULL;
+       fh->f_filetype      = MPI_DATATYPE_NULL;
+       fh->f_orig_filetype = MPI_DATATYPE_NULL;
        
        fh->f_init_procs_per_group = -1;
-       fh->f_init_procs_in_group = NULL;
-       
-       fh->f_procs_per_group = -1;
-       fh->f_procs_in_group = NULL;
-       
-       fh->f_init_num_aggrs = -1;
-       fh->f_init_aggr_list = NULL;
-       
-       fh->f_num_aggrs = -1;
-       fh->f_aggr_list = NULL;
-       
-       /* Default file View */
-       fh->f_iov_type = MPI_DATATYPE_NULL;
-       fh->f_stripe_size = 0;
-       /*Decoded iovec of the file-view*/
-       fh->f_decoded_iov = NULL;
-       fh->f_etype = MPI_DATATYPE_NULL;
-       fh->f_filetype = MPI_DATATYPE_NULL;
-       fh->f_orig_filetype = MPI_DATATYPE_NULL;
-       fh->f_datarep = NULL;
+       fh->f_init_procs_in_group  = NULL;
+       fh->f_procs_per_group      = -1;
+       fh->f_procs_in_group       = NULL;
+       fh->f_init_num_aggrs       = -1;
+       fh->f_init_aggr_list       = NULL;
+       fh->f_num_aggrs            = -1;
+       fh->f_aggr_list            = NULL;
+       fh->f_datarep              = NULL;
        
        /*Create a derived datatype for the created iovec */
        types[0] = &ompi_mpi_long.dt;
        types[1] = &ompi_mpi_long.dt;
        
-       d[0] = (ptrdiff_t) fh->f_decoded_iov;
-       d[1] = (ptrdiff_t) &fh->f_decoded_iov[0].iov_len;
+       d[0] = (ptrdiff_t) fh->f_fview.f_decoded_iov;
+       d[1] = (ptrdiff_t) &fh->f_fview.f_decoded_iov[0].iov_len;
        
        base = d[0];
        for (i=0 ; i<2 ; i++) {
@@ -568,7 +593,7 @@ int mca_common_ompio_create_incomplete_file_handle (const char *filename,
 
 int mca_common_ompio_decode_datatype (struct ompio_file_t *fh,
                                       ompi_datatype_t *datatype,
-                                      int count,
+                                      size_t count,
                                       const void *buf,
                                       size_t *max_data,
                                       opal_convertor_t *conv,
diff --git a/ompi/mca/common/ompio/common_ompio_file_read.c b/ompi/mca/common/ompio/common_ompio_file_read.c
index 695b291fe76..7dd180d2df5 100644
--- a/ompi/mca/common/ompio/common_ompio_file_read.c
+++ b/ompi/mca/common/ompio/common_ompio_file_read.c
@@ -12,6 +12,9 @@
  *  Copyright (c) 2008-2019 University of Houston. All rights reserved.
  *  Copyright (c) 2018      Research Organization for Information Science
  *                          and Technology (RIST). All rights reserved.
+ *  Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved.
+ *  Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                          reserved.
  *  $COPYRIGHT$
  *
  *  Additional copyrights may follow
@@ -43,168 +46,286 @@
 **   mca_io_ompio_file_read/write[_at]
 **
 ** routines are the ones registered with the ompio modules.
-** The
+** They call the
 **
 ** mca_common_ompio_file_read/write[_at]
 **
-** routesin are used e.g. from the shared file pointer modules.
+** routines, which are however also used from the shared file pointer modules.
 ** The main difference is, that the first one takes an ompi_file_t
 ** as a file pointer argument, while the second uses the ompio internal
 ** ompio_file_t structure.
 */
 
-int mca_common_ompio_file_read (ompio_file_t *fh,
-			      void *buf,
-			      int count,
-			      struct ompi_datatype_t *datatype,
-			      ompi_status_public_t *status)
-{
-    int ret = OMPI_SUCCESS;
+static int mca_common_ompio_file_read_default (ompio_file_t *fh, void *buf,
+                                               size_t count, struct ompi_datatype_t *datatype,
+                                               ompi_status_public_t *status);
 
-    size_t total_bytes_read = 0;       /* total bytes that have been read*/
-    size_t bytes_per_cycle = 0;        /* total read in each cycle by each process*/
-    int index = 0;
-    int cycles = 0;
+static int mca_common_ompio_file_read_pipelined (ompio_file_t *fh, void *buf,
+                                                 size_t count, struct ompi_datatype_t *datatype,
+                                                 ompi_status_public_t *status);
 
-    uint32_t iov_count = 0;
-    struct iovec *decoded_iov = NULL;
-
-    size_t max_data=0, real_bytes_read=0;
-    size_t spc=0;
-    ssize_t ret_code=0;
-    int i = 0; /* index into the decoded iovec of the buffer */
-    int j = 0; /* index into the file vie iovec */
+int mca_common_ompio_file_read (ompio_file_t *fh,
+                              void *buf,
+                              size_t count,
+                              struct ompi_datatype_t *datatype,
+                              ompi_status_public_t *status)
+{
+    bool need_to_copy = false;
+    int is_gpu, is_managed;
 
-    if (fh->f_amode & MPI_MODE_WRONLY){
-//      opal_output(10, "Improper use of FILE Mode, Using WRONLY for Read!\n");
-        ret = MPI_ERR_ACCESS;
-      return ret;
+    if (fh->f_amode & MPI_MODE_WRONLY) {
+        return MPI_ERR_ACCESS;
     }
 
-    if ( 0 == count ) {
-        if ( MPI_STATUS_IGNORE != status ) {
+    if (0 == count || 0 == fh->f_fview.f_iov_count) {
+        if (MPI_STATUS_IGNORE != status) {
             status->_ucount = 0;
         }
-        return ret;
+        return OMPI_SUCCESS;
     }
 
-    bool need_to_copy = false;    
-    opal_convertor_t convertor;
-#if OPAL_CUDA_SUPPORT
-    int is_gpu, is_managed;
     mca_common_ompio_check_gpu_buf ( fh, buf, &is_gpu, &is_managed);
-    if ( is_gpu && !is_managed ) {
+    if (is_gpu && !is_managed) {
         need_to_copy = true;
     }
-#endif
 
     if ( !( fh->f_flags & OMPIO_DATAREP_NATIVE ) &&
          !(datatype == &ompi_mpi_byte.dt  ||
            datatype == &ompi_mpi_char.dt   )) {
         /* only need to copy if any of these conditions are given:
-           1. buffer is an unmanaged CUDA buffer (checked above).
+           1. buffer is an unmanaged device buffer (checked above).
            2. Datarepresentation is anything other than 'native' and
            3. datatype is not byte or char (i.e it does require some actual
               work to be done e.g. for external32.
         */
         need_to_copy = true;
     }         
-    
-    if ( need_to_copy ) {
-        char *tbuf=NULL;
 
-        OMPIO_PREPARE_READ_BUF(fh,buf,count,datatype,tbuf,&convertor,max_data,decoded_iov,iov_count);        
-    }
-    else {
-        mca_common_ompio_decode_datatype (fh,
-                                          datatype,
-                                          count,
-                                          buf,
-                                          &max_data,
-                                          fh->f_mem_convertor,
-                                          &decoded_iov,
-                                          &iov_count);
+    if (need_to_copy) {
+        return mca_common_ompio_file_read_pipelined (fh, buf, count, datatype, status);
+    } else {
+        return mca_common_ompio_file_read_default (fh, buf, count, datatype, status);
     }
 
-    if ( 0 < max_data && 0 == fh->f_iov_count  ) {
-        if ( MPI_STATUS_IGNORE != status ) {
-            status->_ucount = 0;
-        }
-        if (NULL != decoded_iov) {
-            free (decoded_iov);
-            decoded_iov = NULL;
-        }
-        return OMPI_SUCCESS;
-    }
+    return OMPI_SUCCESS; //silence compiler
+}
 
-    if ( -1 == OMPIO_MCA_GET(fh, cycle_buffer_size )) {
-        bytes_per_cycle = max_data;
-    }
-    else {
-	bytes_per_cycle = OMPIO_MCA_GET(fh, cycle_buffer_size);
-    }
+int mca_common_ompio_file_read_default (ompio_file_t *fh, void *buf,
+                                        size_t count, struct ompi_datatype_t *datatype,
+                                        ompi_status_public_t *status)
+{
+    size_t total_bytes_read = 0;       /* total bytes that have been read*/
+    size_t bytes_per_cycle = 0;        /* total read in each cycle by each process*/
+    int index = 0;
+    int cycles = 0;
+
+    uint32_t iov_count = 0;
+    struct iovec *decoded_iov = NULL;
+
+    size_t max_data=0, real_bytes_read=0;
+    size_t spc=0;
+    ssize_t ret_code=0;
+    int i = 0; /* index into the decoded iovec of the buffer */
+
+    mca_common_ompio_decode_datatype (fh, datatype, count, buf,
+                                      &max_data, fh->f_mem_convertor,
+                                      &decoded_iov, &iov_count);
+
+    bytes_per_cycle = OMPIO_MCA_GET(fh, cycle_buffer_size);
     cycles = ceil((double)max_data/bytes_per_cycle);
     
 #if 0
-	printf ("Bytes per Cycle: %d   Cycles: %d max_data:%d \n",bytes_per_cycle, cycles, max_data);
+    printf ("Bytes per Cycle: %d   Cycles: %d max_data:%d \n", bytes_per_cycle,
+            cycles, max_data);
 #endif
 
-    j = fh->f_index_in_file_view;
-
     for (index = 0; index < cycles; index++) {
+        mca_common_ompio_build_io_array (&(fh->f_fview), index, cycles, bytes_per_cycle,
+                                         max_data, iov_count, decoded_iov,
+                                         &i, &total_bytes_read, &spc,
+                                         &fh->f_io_array, &fh->f_num_of_io_entries);
+        if (fh->f_num_of_io_entries == 0) {
+	    ret_code = 0;
+	    goto exit;
+	}
 
-	mca_common_ompio_build_io_array ( fh,
-                                          index,
-                                          cycles,
-                                          bytes_per_cycle,
-                                          max_data,
-                                          iov_count,
-                                          decoded_iov,
-                                          &i,
-                                          &j,
-                                          &total_bytes_read, 
-                                          &spc,
-                                          &fh->f_io_array,
-                                          &fh->f_num_of_io_entries);
-
-        if (fh->f_num_of_io_entries) {
-            ret_code = fh->f_fbtl->fbtl_preadv (fh);
-            if ( 0<= ret_code ) {
-                real_bytes_read+=(size_t)ret_code;
-            }
-        }
+	ret_code = fh->f_fbtl->fbtl_preadv (fh);
+	if (0 <= ret_code) {
+	    real_bytes_read += (size_t)ret_code;
+	    // Reset ret_code since it is also used to return an error
+	    ret_code = 0;
+	} else {
+	    goto exit;
+	}
 
         fh->f_num_of_io_entries = 0;
-        if (NULL != fh->f_io_array) {
-            free (fh->f_io_array);
-            fh->f_io_array = NULL;
+        free (fh->f_io_array);
+        fh->f_io_array = NULL;
+    }
+
+ exit:
+    free (decoded_iov);
+    if (MPI_STATUS_IGNORE != status) {
+        status->_ucount = real_bytes_read;
+    }
+
+    return ret_code;
+}
+
+int mca_common_ompio_file_read_pipelined (ompio_file_t *fh, void *buf,
+                                          size_t count, struct ompi_datatype_t *datatype,
+                                          ompi_status_public_t *status)
+{
+    size_t tbr = 0;               /* total bytes that have been read*/
+    size_t bytes_per_cycle = 0;   /* total read in each cycle by each process*/
+    size_t bytes_this_cycle = 0, bytes_prev_cycle = 0;
+    int index = 0;
+    int cycles = 0;
+    uint32_t iov_count = 0;
+    struct iovec decoded_iov;
+
+    size_t max_data=0, real_bytes_read=0;
+    size_t spc=0;
+    ssize_t ret_code=0;
+    int i = 0; /* index into the decoded iovec of the buffer */
+
+    char *tbuf1=NULL, *tbuf2=NULL;
+    char *unpackbuf=NULL, *readbuf=NULL;
+    mca_ompio_request_t *ompio_req=NULL, *prev_ompio_req=NULL;
+    opal_convertor_t convertor;
+    bool can_overlap = (NULL != fh->f_fbtl->fbtl_ipreadv);
+
+    bytes_per_cycle = OMPIO_MCA_GET(fh, pipeline_buffer_size);
+    OMPIO_PREPARE_READ_BUF (fh, buf, count, datatype, tbuf1, &convertor,
+                            max_data, bytes_per_cycle, &decoded_iov, iov_count);
+    cycles = ceil((double)max_data/bytes_per_cycle);
+
+    readbuf = unpackbuf = tbuf1;
+    if (can_overlap) {
+        tbuf2 = mca_common_ompio_alloc_buf (fh, bytes_per_cycle);
+        if (NULL == tbuf2) {
+            opal_output(1, "common_ompio: error allocating memory\n");
+            opal_convertor_cleanup (&convertor);
+            return OMPI_ERR_OUT_OF_RESOURCE;
         }
+        unpackbuf = tbuf2;
     }
 
-    if ( need_to_copy ) {
-        size_t pos=0;
+#if 0
+    printf ("Bytes per Cycle: %d Cycles: %d max_data:%d \n", bytes_per_cycle,
+            cycles, max_data);
+#endif
 
-        opal_convertor_unpack (&convertor, decoded_iov, &iov_count, &pos );
-        opal_convertor_cleanup (&convertor);
-        mca_common_ompio_release_buf (fh, decoded_iov->iov_base);
+    /*
+    ** The code combines two scenarios:
+    ** 1. having async read (i.e. ipreadv) which allows to overlap two 
+    **    iterations.
+    ** 2. not having async read, which doesn't allow for overlap.
+    ** 
+    ** In the first case we use a double buffering technique, the sequence is
+    **    - construct io-array for iter i 
+    **    - post ipreadv for iter i
+    **    - wait for iter i-1
+    **    - unpack buffer i-1
+    **    - swap buffers
+    **
+    ** In the second case, the sequence is
+    **    - construct io-array for iter i
+    **    - post preadv for iter i
+    **    - unpack buffer i
+    */
+    
+    if (can_overlap) {
+	mca_common_ompio_register_progress ();	    
+    }
+
+    for (index = 0; index < cycles+1; index++) {
+        if (index < cycles) {
+            decoded_iov.iov_base  = readbuf;
+            decoded_iov.iov_len   = bytes_per_cycle;
+            bytes_this_cycle      = (index == cycles-1) ?
+                (max_data - (index * bytes_per_cycle)) :
+                bytes_per_cycle;
+
+	    i   = 0;
+            spc = 0;
+            tbr = 0;
+
+            mca_common_ompio_build_io_array (&(fh->f_fview), index, cycles, bytes_per_cycle,
+                                             bytes_this_cycle, iov_count,
+                                             &decoded_iov, &i, &tbr, &spc,
+                                             &fh->f_io_array, &fh->f_num_of_io_entries);
+	    if (fh->f_num_of_io_entries == 0) {
+		ret_code = 0;
+		goto exit;
+	    }
+
+            if (can_overlap) {
+                mca_common_ompio_request_alloc (&ompio_req, MCA_OMPIO_REQUEST_READ);
+		fh->f_fbtl->fbtl_ipreadv (fh, (ompi_request_t *)ompio_req);
+            } else {
+		ret_code = fh->f_fbtl->fbtl_preadv (fh);
+		if (0 <= ret_code) {
+		    real_bytes_read+=(size_t)ret_code;
+		    // Reset ret_code since it is also used to return an error
+		    ret_code = 0;
+		}
+		else {
+		    goto exit;
+		}
+	    }
+	}
+
+	if (can_overlap) {
+	    if (index != 0) {
+		ompi_status_public_t stat;
+		ret_code = ompi_request_wait ((ompi_request_t **)&prev_ompio_req, &stat);
+		if (OMPI_SUCCESS != ret_code) {
+		    goto exit;
+		}
+		real_bytes_read += stat._ucount;
+	    }
+	    prev_ompio_req = ompio_req;
+	}
+
+	if ((can_overlap && index != 0) ||
+	    (!can_overlap && index < cycles)) {
+	    size_t pos = 0;
+	    decoded_iov.iov_base = unpackbuf;
+	    decoded_iov.iov_len  = can_overlap ? bytes_prev_cycle : bytes_this_cycle;
+	    opal_convertor_unpack (&convertor, &decoded_iov, &iov_count, &pos);
+	}
+
+	fh->f_num_of_io_entries = 0;
+	free (fh->f_io_array);
+	fh->f_io_array = NULL;
+        bytes_prev_cycle = bytes_this_cycle;
+
+        if (can_overlap) {
+            char *tmp = unpackbuf;
+            unpackbuf = readbuf;
+            readbuf   = tmp;
+        }
     }
 
-    if (NULL != decoded_iov) {
-        free (decoded_iov);
-        decoded_iov = NULL;
+ exit:
+    opal_convertor_cleanup (&convertor);
+    mca_common_ompio_release_buf (fh, tbuf1);
+    if (can_overlap) {
+        mca_common_ompio_release_buf (fh, tbuf2);
     }
 
     if ( MPI_STATUS_IGNORE != status ) {
         status->_ucount = real_bytes_read;
     }
 
-    return ret;
+    return ret_code;
 }
 
 int mca_common_ompio_file_read_at (ompio_file_t *fh,
 				 OMPI_MPI_OFFSET_TYPE offset,
 				 void *buf,
-				 int count,
+				 size_t count,
 				 struct ompi_datatype_t *datatype,
 				 ompi_status_public_t * status)
 {
@@ -228,60 +349,106 @@ int mca_common_ompio_file_read_at (ompio_file_t *fh,
     return ret;
 }
 
+static void mca_common_ompio_post_next_read_subreq(struct mca_ompio_request_t *req, int index)
+{
+    uint32_t iov_count = 1;
+    size_t pos = 0, spc = 0, tbw = 0;
+    int i = 0;
+    mca_ompio_request_t *ompio_subreq=NULL;
+    size_t bytes_per_cycle = OMPIO_MCA_GET(req->req_fh, pipeline_buffer_size);
+    struct iovec decoded_iov;
+
+    /* Step 1: finish index-1 unpack operation */
+    if (index - 1 >= 0) {
+        size_t num_bytes = bytes_per_cycle;
+        /**
+         *  should really be 'req_num_subreqs -1 == index -1'
+         *  which is the same as below.
+         */
+        if (req->req_num_subreqs == index) {
+            num_bytes = req->req_max_data - (index-1)* bytes_per_cycle;
+        }
+        decoded_iov.iov_base = req->req_tbuf;
+        decoded_iov.iov_len  = num_bytes;
+        opal_convertor_unpack (&req->req_convertor, &decoded_iov, &iov_count, &pos);
+    }
+
+    /* Step 2: post next iread subrequest */
+    if (req->req_num_subreqs == index) {
+        /* all done */
+        return;
+    }
+
+    decoded_iov.iov_base = req->req_tbuf;
+    decoded_iov.iov_len  = (req->req_num_subreqs-1 == index) ?
+	req->req_max_data - (index* bytes_per_cycle) : req->req_size;
+    mca_common_ompio_build_io_array (req->req_fview, index, req->req_num_subreqs,
+                                     bytes_per_cycle, decoded_iov.iov_len,
+                                     iov_count, &decoded_iov,
+                                     &i, &tbw, &spc,
+                                     &req->req_fh->f_io_array,
+                                     &req->req_fh->f_num_of_io_entries);
+#if 0
+    printf("Next subreq: memory address %p offset %lu length %ld first element %ld\n",
+	   req->req_fh->f_io_array[0].memory_address,
+	   (long unsigned)req->req_fh->f_io_array[0].offset,
+	   req->req_fh->f_io_array[0].length,
+	   ((long*)req->req_fh->f_io_array[0].memory_address)[0]);
+#endif
+    mca_common_ompio_request_alloc ( &ompio_subreq, MCA_OMPIO_REQUEST_READ);
+    ompio_subreq->req_parent = req;
+    req->req_fh->f_fbtl->fbtl_ipreadv (req->req_fh, (ompi_request_t *)ompio_subreq);
+
+    free(req->req_fh->f_io_array);
+    req->req_fh->f_io_array = NULL;
+    req->req_fh->f_num_of_io_entries = 0;
+}
 
 int mca_common_ompio_file_iread (ompio_file_t *fh,
-			       void *buf,
-			       int count,
-			       struct ompi_datatype_t *datatype,
-			       ompi_request_t **request)
+                               void *buf,
+                               size_t count,
+                               struct ompi_datatype_t *datatype,
+                               ompi_request_t **request)
 {
     int ret = OMPI_SUCCESS;
     mca_ompio_request_t *ompio_req=NULL;
-    size_t spc=0;
+    struct iovec *decoded_iov = NULL;
 
     if (fh->f_amode & MPI_MODE_WRONLY){
-//      opal_output(10, "Improper use of FILE Mode, Using WRONLY for Read!\n");
         ret = MPI_ERR_ACCESS;
-      return ret;
+        return ret;
     }
 
-    mca_common_ompio_request_alloc ( &ompio_req, MCA_OMPIO_REQUEST_READ);
+    mca_common_ompio_request_alloc (&ompio_req, MCA_OMPIO_REQUEST_READ);
 
-    if ( 0 == count ) {
+    if (0 == count || 0 == fh->f_fview.f_iov_count) {
         ompio_req->req_ompi.req_status.MPI_ERROR = OMPI_SUCCESS;
         ompio_req->req_ompi.req_status._ucount = 0;
         ompi_request_complete (&ompio_req->req_ompi, false);
         *request = (ompi_request_t *) ompio_req;
-        
+
         return OMPI_SUCCESS;
     }
 
-    if ( NULL != fh->f_fbtl->fbtl_ipreadv ) {
+    if (NULL != fh->f_fbtl->fbtl_ipreadv) {
         // This fbtl has support for non-blocking operations
-
-        size_t total_bytes_read = 0;       /* total bytes that have been read*/
         uint32_t iov_count = 0;
-        struct iovec *decoded_iov = NULL;
-        
         size_t max_data = 0;
-        int i = 0; /* index into the decoded iovec of the buffer */
-        int j = 0; /* index into the file vie iovec */
-        
         bool need_to_copy = false;    
-    
-#if OPAL_CUDA_SUPPORT
         int is_gpu, is_managed;
-        mca_common_ompio_check_gpu_buf ( fh, buf, &is_gpu, &is_managed);
-        if ( is_gpu && !is_managed ) {
+
+        mca_common_ompio_check_gpu_buf (fh, buf, &is_gpu, &is_managed);
+        if (is_gpu && !is_managed) {
             need_to_copy = true;
         }
-#endif
+
+        mca_common_ompio_register_progress ();
 
         if ( !( fh->f_flags & OMPIO_DATAREP_NATIVE ) &&
              !(datatype == &ompi_mpi_byte.dt  ||
                datatype == &ompi_mpi_char.dt   )) {
             /* only need to copy if any of these conditions are given:
-               1. buffer is an unmanaged CUDA buffer (checked above).
+               1. buffer is an unmanaged device buffer (checked above).
                2. Datarepresentation is anything other than 'native' and
                3. datatype is not byte or char (i.e it does require some actual
                work to be done e.g. for external32.
@@ -289,91 +456,147 @@ int mca_common_ompio_file_iread (ompio_file_t *fh,
             need_to_copy = true;
         }         
         
-        if ( need_to_copy ) {
-            char *tbuf=NULL;
-            
-            OMPIO_PREPARE_READ_BUF(fh,buf,count,datatype,tbuf,&ompio_req->req_convertor,max_data,decoded_iov,iov_count); 
+        if (need_to_copy) {
+            OMPI_MPI_OFFSET_TYPE prev_offset;
+            size_t pipeline_buf_size = OMPIO_MCA_GET(fh, pipeline_buffer_size);
             
-            ompio_req->req_tbuf = tbuf;
-            ompio_req->req_size = max_data;
+            OMPIO_PREPARE_READ_BUF(fh, buf, count, datatype, ompio_req->req_tbuf,
+                                   &ompio_req->req_convertor, max_data,
+                                   pipeline_buf_size, NULL, iov_count);
+
+            ompio_req->req_num_subreqs = ceil((double)max_data/pipeline_buf_size);
+            ompio_req->req_size        = pipeline_buf_size;
+            ompio_req->req_max_data    = max_data;
+            ompio_req->req_post_next_subreq = mca_common_ompio_post_next_read_subreq;
+            ompio_req->req_fh          = fh;
+            ompio_req->req_ompi.req_status.MPI_ERROR = MPI_SUCCESS;
+
+	    ompio_req->req_fview = (struct ompio_fview_t *) malloc(sizeof(struct ompio_fview_t));
+	    if (NULL == ompio_req->req_fview) {
+		opal_output(1, "common_ompio: error allocating memory\n");
+		return OMPI_ERR_OUT_OF_RESOURCE;
+	    }
+            ret = mca_common_ompio_fview_duplicate(ompio_req->req_fview, &fh->f_fview);
+            if (OMPI_SUCCESS != ret) {
+                return ret;
+            }
+            mca_common_ompio_file_get_position (fh, &prev_offset );
+            mca_common_ompio_post_next_read_subreq (ompio_req, 0);
+
+            /* Move file pointer to the end of the operation.
+             * Otherwise posting another I/O operation will start of
+             * from the wrong file position. The request will update
+             * the position where to write the next chunk of data
+             * using its internal copy of the file view and file pointer
+             * position.
+             */
+            mca_common_ompio_set_explicit_offset (fh, prev_offset+max_data);
         }
         else {
-            mca_common_ompio_decode_datatype (fh,
-                                              datatype,
-                                              count,
-                                              buf,
-                                              &max_data,
-                                              fh->f_mem_convertor,
-                                              &decoded_iov,
-                                              &iov_count);
+	    int i = 0;
+	    size_t spc = 0, tbr = 0;
+	    mca_common_ompio_decode_datatype (fh, datatype, count, buf,
+                                              &max_data, fh->f_mem_convertor,
+                                              &decoded_iov, &iov_count);
+
+            /**
+             * Non-blocking operations have to occur in a single cycle
+             * If the f_io_array is too long, the fbtl will chunk it up
+             * internally.
+             */
+            mca_common_ompio_build_io_array (&(fh->f_fview), 0, 1, max_data, max_data,
+                                             iov_count, decoded_iov, &i,
+                                             &tbr, &spc,
+                                             &fh->f_io_array, &fh->f_num_of_io_entries);
+
+            fh->f_fbtl->fbtl_ipreadv (fh, (ompi_request_t *) ompio_req);
         }
-    
-        if ( 0 < max_data && 0 == fh->f_iov_count  ) {
-            ompio_req->req_ompi.req_status.MPI_ERROR = OMPI_SUCCESS;
-            ompio_req->req_ompi.req_status._ucount = 0;
-            ompi_request_complete (&ompio_req->req_ompi, false);
-            *request = (ompi_request_t *) ompio_req;
-            if (NULL != decoded_iov) {
-                free (decoded_iov);
-                decoded_iov = NULL;
-            }
+    }
+    else {
+        // This fbtl does not  support non-blocking operations
+        ompi_status_public_t status;
+        ret = mca_common_ompio_file_read (fh, buf, count, datatype, &status);
 
-            return OMPI_SUCCESS;
-        }
+        ompio_req->req_ompi.req_status.MPI_ERROR = ret;
+        ompio_req->req_ompi.req_status._ucount = status._ucount;
+        ompi_request_complete (&ompio_req->req_ompi, false);
+    }
 
-        // Non-blocking operations have to occur in a single cycle
-        j = fh->f_index_in_file_view;
-        
-        mca_common_ompio_build_io_array ( fh,
-                                          0,         // index
-                                          1,         // no. of cyces
-                                          max_data,  // setting bytes per cycle to match data
-                                          max_data,
-                                          iov_count,
-                                          decoded_iov,
-                                          &i,
-                                          &j,
-                                          &total_bytes_read, 
-                                          &spc,
-                                          &fh->f_io_array,
-                                          &fh->f_num_of_io_entries);
-
-	if (fh->f_num_of_io_entries) {
-	  fh->f_fbtl->fbtl_ipreadv (fh, (ompi_request_t *) ompio_req);
-	}
+    fh->f_num_of_io_entries = 0;
+    free (fh->f_io_array);
+    fh->f_io_array = NULL;
+    free (decoded_iov);
 
-        mca_common_ompio_register_progress ();
+    *request = (ompi_request_t *) ompio_req;
+    return ret;
+}
 
-	fh->f_num_of_io_entries = 0;
-	if (NULL != fh->f_io_array) {
-	    free (fh->f_io_array);
-	    fh->f_io_array = NULL;
-	}
+/*
+** This routine is invoked from file_read_all.
+** It is only used if the temporary buffer is a gpu buffer,
+** and the fbtl supports the ipreadv operation.
+**
+** The io-array has already been generated in file_read_all,
+** and we use the pre-computed offsets to created a pseudo fview.
+** The position of the file pointer is updated in the file_read_all
+** operation, not here.
+*/
 
-	if (NULL != decoded_iov) {
-	    free (decoded_iov);
-	    decoded_iov = NULL;
-	}
+int mca_common_ompio_file_iread_pregen (ompio_file_t *fh,
+                                        ompi_request_t *request)
+{
+    uint32_t i;
+    size_t max_data;
+    size_t pipeline_buf_size;
+    mca_ompio_request_t *ompio_req = (mca_ompio_request_t *) request;
+
+    max_data = fh->f_io_array[0].length;
+    pipeline_buf_size = OMPIO_MCA_GET(fh, pipeline_buffer_size);
+
+    mca_common_ompio_register_progress ();
+
+    OMPIO_PREPARE_READ_BUF (fh, fh->f_io_array[0].memory_address, max_data, MPI_BYTE,
+                            ompio_req->req_tbuf, &ompio_req->req_convertor, max_data,
+                            pipeline_buf_size, NULL, i);
+
+    ompio_req->req_num_subreqs = ceil((double)max_data/pipeline_buf_size);
+    ompio_req->req_size        = pipeline_buf_size;
+    ompio_req->req_max_data    = max_data;
+    ompio_req->req_post_next_subreq = mca_common_ompio_post_next_read_subreq;
+    ompio_req->req_fh          = fh;
+    ompio_req->req_ompi.req_status.MPI_ERROR = MPI_SUCCESS;
+
+    ompio_req->req_fview = (struct ompio_fview_t *) calloc(1, sizeof(struct ompio_fview_t));
+    if (NULL == ompio_req->req_fview) {
+        opal_output(1, "common_ompio: error allocating memory\n");
+        return OMPI_ERR_OUT_OF_RESOURCE;
     }
-    else {
-	// This fbtl does not  support non-blocking operations
-	ompi_status_public_t status;
-	ret = mca_common_ompio_file_read (fh, buf, count, datatype, &status);
 
-	ompio_req->req_ompi.req_status.MPI_ERROR = ret;
-	ompio_req->req_ompi.req_status._ucount = status._ucount;
-	ompi_request_complete (&ompio_req->req_ompi, false);
+   ompio_req->req_fview->f_decoded_iov = (struct iovec*) malloc (fh->f_num_of_io_entries *
+                                                                 sizeof(struct iovec));
+    if (NULL == ompio_req->req_fview->f_decoded_iov) {
+        opal_output(1, "common_ompio_file_iread_pregen: could not allocate memory\n");
+        return OMPI_ERR_OUT_OF_RESOURCE;
     }
 
-    *request = (ompi_request_t *) ompio_req;
-    return ret;
-}
+    ompio_req->req_fview->f_iov_count = fh->f_num_of_io_entries;
+    for (i=0; i < ompio_req->req_fview->f_iov_count; i++) {
+        ompio_req->req_fview->f_decoded_iov[i].iov_base = fh->f_io_array[i].offset;
+        ompio_req->req_fview->f_decoded_iov[i].iov_len  = fh->f_io_array[i].length ;
+    }
 
+    fh->f_num_of_io_entries = 0;
+    free (fh->f_io_array);
+    fh->f_io_array = NULL;
+
+    mca_common_ompio_post_next_read_subreq(ompio_req, 0);
+    return OMPI_SUCCESS;
+}
 
 int mca_common_ompio_file_iread_at (ompio_file_t *fh,
 				  OMPI_MPI_OFFSET_TYPE offset,
 				  void *buf,
-				  int count,
+				  size_t count,
 				  struct ompi_datatype_t *datatype,
 				  ompi_request_t **request)
 {
@@ -405,7 +628,7 @@ int mca_common_ompio_file_iread_at (ompio_file_t *fh,
 /* Infrastructure for collective operations  */
 int mca_common_ompio_file_read_all (ompio_file_t *fh,
                                     void *buf,
-                                    int count,
+                                    size_t count,
                                     struct ompi_datatype_t *datatype,
                                     ompi_status_public_t * status)
 {
@@ -421,7 +644,7 @@ int mca_common_ompio_file_read_all (ompio_file_t *fh,
            
            If the individual fcoll component is used: there are no aggregators 
            in that concept. However, since they call common_ompio_file_write, 
-           CUDA buffers are handled by that routine.
+           device buffers are handled by that routine.
 
            Thus, we only check for
            1. Datarepresentation is anything other than 'native' and
@@ -431,23 +654,20 @@ int mca_common_ompio_file_read_all (ompio_file_t *fh,
         size_t pos=0, max_data=0;
         char *tbuf=NULL;
         opal_convertor_t convertor;
-        struct iovec *decoded_iov = NULL;
+        struct iovec decoded_iov;
         uint32_t iov_count = 0;
 
-        OMPIO_PREPARE_READ_BUF(fh,buf,count,datatype,tbuf,&convertor,max_data,decoded_iov,iov_count);   
+        OMPIO_PREPARE_READ_BUF(fh, buf, count, datatype, tbuf, &convertor,
+                               max_data, 0, &decoded_iov, iov_count);
         ret = fh->f_fcoll->fcoll_file_read_all (fh,
-                                                decoded_iov->iov_base,
-                                                decoded_iov->iov_len,
+                                                decoded_iov.iov_base,
+                                                decoded_iov.iov_len,
                                                 MPI_BYTE,
                                                 status);
-        opal_convertor_unpack (&convertor, decoded_iov, &iov_count, &pos );
+        opal_convertor_unpack (&convertor, &decoded_iov, &iov_count, &pos );
 
         opal_convertor_cleanup (&convertor);
-        mca_common_ompio_release_buf (fh, decoded_iov->iov_base);
-        if (NULL != decoded_iov) {
-            free (decoded_iov);
-            decoded_iov = NULL;
-        }
+        mca_common_ompio_release_buf (fh, decoded_iov.iov_base);
     }
     else {
         ret = fh->f_fcoll->fcoll_file_read_all (fh,
@@ -462,7 +682,7 @@ int mca_common_ompio_file_read_all (ompio_file_t *fh,
 int mca_common_ompio_file_read_at_all (ompio_file_t *fh,
 				     OMPI_MPI_OFFSET_TYPE offset,
 				     void *buf,
-				     int count,
+				     size_t count,
 				     struct ompi_datatype_t *datatype,
 				     ompi_status_public_t * status)
 {
@@ -483,7 +703,7 @@ int mca_common_ompio_file_read_at_all (ompio_file_t *fh,
 
 int mca_common_ompio_file_iread_all (ompio_file_t *fp,
                                      void *buf,
-                                     int count,
+                                     size_t count,
                                      struct ompi_datatype_t *datatype,
                                      ompi_request_t **request)
 {
@@ -509,7 +729,7 @@ int mca_common_ompio_file_iread_all (ompio_file_t *fp,
 int mca_common_ompio_file_iread_at_all (ompio_file_t *fp,
 				      OMPI_MPI_OFFSET_TYPE offset,
 				      void *buf,
-				      int count,
+				      size_t count,
 				      struct ompi_datatype_t *datatype,
 				      ompi_request_t **request)
 {
@@ -536,29 +756,29 @@ int mca_common_ompio_set_explicit_offset (ompio_file_t *fh,
     size_t i = 0;
     size_t k = 0;
 
-    if ( fh->f_view_size  > 0 ) {
+    if ( fh->f_fview.f_view_size  > 0 ) {
 	/* starting offset of the current copy of the filew view */
-	fh->f_offset = (fh->f_view_extent *
-			((offset*fh->f_etype_size) / fh->f_view_size)) + fh->f_disp;
+	fh->f_fview.f_offset = (fh->f_fview.f_view_extent *
+			((offset*fh->f_fview.f_etype_size) / fh->f_fview.f_view_size)) + fh->f_fview.f_disp;
 
 
 	/* number of bytes used within the current copy of the file view */
-	fh->f_total_bytes = (offset*fh->f_etype_size) % fh->f_view_size;
-	i = fh->f_total_bytes;
+	fh->f_fview.f_total_bytes = (offset*fh->f_fview.f_etype_size) % fh->f_fview.f_view_size;
+	i = fh->f_fview.f_total_bytes;
 
 
 	/* Initialize the block id and the starting offset of the current block
 	   within the current copy of the file view to zero */
-	fh->f_index_in_file_view = 0;
-	fh->f_position_in_file_view = 0;
+	fh->f_fview.f_index_in_file_view = 0;
+	fh->f_fview.f_position_in_file_view = 0;
 
 	/* determine block id that the offset is located in and
 	   the starting offset of that block */
-	k = fh->f_decoded_iov[fh->f_index_in_file_view].iov_len;
+	k = fh->f_fview.f_decoded_iov[fh->f_fview.f_index_in_file_view].iov_len;
 	while (i >= k) {
-	    fh->f_position_in_file_view = k;
-	    fh->f_index_in_file_view++;
-	    k += fh->f_decoded_iov[fh->f_index_in_file_view].iov_len;
+	    fh->f_fview.f_position_in_file_view = k;
+	    fh->f_fview.f_index_in_file_view++;
+	    k += fh->f_fview.f_decoded_iov[fh->f_fview.f_index_in_file_view].iov_len;
 	}
     }
 
diff --git a/ompi/mca/common/ompio/common_ompio_file_read_all.c b/ompi/mca/common/ompio/common_ompio_file_read_all.c
new file mode 100644
index 00000000000..1b2f8d6c474
--- /dev/null
+++ b/ompi/mca/common/ompio/common_ompio_file_read_all.c
@@ -0,0 +1,1131 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008-2022 University of Houston. All rights reserved.
+ * Copyright (c) 2017-2018 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2024      Advanced Micro Devices, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "mpi.h"
+#include "ompi/constants.h"
+#include "ompi/mca/fcoll/fcoll.h"
+#include "ompi/mca/fcoll/base/fcoll_base_coll_array.h"
+#include "ompi/mca/fcoll/base/base.h"
+#include "ompi/mca/common/ompio/common_ompio.h"
+#include "ompi/mca/common/ompio/common_ompio_request.h"
+#include "ompi/mca/common/ompio/common_ompio_buffer.h"
+#include "ompi/mca/io/io.h"
+#include "math.h"
+#include "ompi/mca/pml/pml.h"
+#include "opal/mca/accelerator/accelerator.h"
+#include <unistd.h>
+
+#define DEBUG_ON 0
+#define COMMON_OMPIO_SHUFFLE_TAG 123
+#define INIT_LEN 10
+
+/*Used for loading file-offsets per aggregator*/
+typedef struct mca_io_ompio_local_io_array{
+    OMPI_MPI_OFFSET_TYPE offset;
+    MPI_Aint             length;
+    int                  process_id;
+}mca_io_ompio_local_io_array;
+
+
+static int read_heap_sort (mca_io_ompio_local_io_array *io_array,
+                           int num_entries,
+                           int *sorted);
+
+
+
+int
+mca_common_ompio_base_file_read_all (struct ompio_file_t *fh,
+                                     void *buf,
+                                     size_t count,
+                                     struct ompi_datatype_t *datatype,
+                                     ompi_status_public_t *status)
+{
+    MPI_Aint total_bytes = 0;          /* total bytes to be read */
+    MPI_Aint bytes_to_read_in_cycle = 0; /* left to be read in a cycle*/
+    MPI_Aint bytes_per_cycle = 0;      /* total read in each cycle by each process*/
+    int index = 0, ret=OMPI_SUCCESS;
+    int cycles = 0;
+    int i=0, j=0, l=0;
+    int n=0; /* current position in total_bytes_per_process array */
+    MPI_Aint bytes_remaining = 0; /* how many bytes have been read from the current
+                                     value from total_bytes_per_process */
+    int *sorted_file_offsets=NULL, entries_per_aggregator=0;
+    int bytes_received = 0;
+    int blocks = 0;
+    /* iovec structure and count of the buffer passed in */
+    uint32_t iov_count = 0;
+    struct iovec *decoded_iov = NULL;
+    int iov_index = 0;
+    size_t current_position = 0;
+    struct iovec *local_iov_array=NULL, *global_iov_array=NULL;
+    MPI_Aint *memory_displacements=NULL;
+    /* global iovec at the readers that contain the iovecs created from
+       file_set_view */
+    uint32_t total_fview_count = 0;
+    size_t local_count = 0;
+    int temp_local_count = 0;
+    size_t *fview_count = NULL;
+    ptrdiff_t *disp_index=NULL, *temp_disp_index=NULL;
+    size_t current_index=0, temp_index=0;
+    int **blocklen_per_process=NULL;
+    MPI_Aint **displs_per_process=NULL;
+    char *global_buf = NULL;
+    mca_io_ompio_local_io_array *file_offsets_for_agg=NULL;
+
+    /* array that contains the sorted indices of the global_iov */
+    int *sorted = NULL;
+    ptrdiff_t *displs = NULL;
+    int base_num_io_procs;
+    size_t max_data = 0;
+    MPI_Aint *total_bytes_per_process = NULL;
+    ompi_datatype_t **sendtype = NULL;
+    MPI_Request *send_req = NULL;
+    MPI_Request recv_req = MPI_REQUEST_NULL;
+    int my_aggregator =-1;
+
+    int* blocklength_proc       = NULL;
+    ptrdiff_t* displs_proc      = NULL;
+
+    int is_gpu, is_managed;
+    bool use_accelerator_buffer = false;
+
+#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
+    double read_time = 0.0, start_read_time = 0.0, end_read_time = 0.0;
+    double rcomm_time = 0.0, start_rcomm_time = 0.0, end_rcomm_time = 0.0;
+    double read_exch = 0.0, start_rexch = 0.0, end_rexch = 0.0;
+    mca_common_ompio_print_entry nentry;
+#endif
+
+    /**************************************************************************
+     ** 1. In case the data is not contiguous in memory, decode it into an iovec
+     **************************************************************************/
+    ret = mca_common_ompio_decode_datatype ((struct ompio_file_t *)fh,
+                                            datatype,
+                                            count,
+                                            buf,
+                                            &max_data,
+                                            fh->f_mem_convertor,
+                                            &decoded_iov,
+                                            &iov_count);
+    if (OMPI_SUCCESS != ret){
+        goto exit;
+    }
+
+    if ( MPI_STATUS_IGNORE != status ) {
+        status->_ucount = max_data;
+    }
+
+    base_num_io_procs = fh->f_get_mca_parameter_value ( "num_aggregators", strlen ("num_aggregators"));
+    if ( OMPI_ERR_MAX == base_num_io_procs ) {
+        ret = OMPI_ERROR;
+        goto exit;
+    }
+
+    mca_common_ompio_check_gpu_buf (fh, buf, &is_gpu, &is_managed);
+    if (is_gpu && !is_managed && NULL != fh->f_fbtl->fbtl_ipreadv &&
+	fh->f_get_mca_parameter_value ("use_accelerator_buffers", strlen("use_accelerator_buffers"))) {
+	use_accelerator_buffer = true;
+    }
+
+    ret = mca_common_ompio_set_aggregator_props ((struct ompio_file_t *) fh,
+                                                 base_num_io_procs,
+                                                 max_data);
+    if (OMPI_SUCCESS != ret){
+        goto exit;
+    }
+    my_aggregator = fh->f_procs_in_group[0];
+
+    /**************************************************************************
+     ** 2. Determine the total amount of data to be written
+     **************************************************************************/
+    total_bytes_per_process = (MPI_Aint*)malloc(fh->f_procs_per_group*sizeof(MPI_Aint));
+    if (NULL == total_bytes_per_process) {
+        opal_output (1, "OUT OF MEMORY\n");
+        ret = OMPI_ERR_OUT_OF_RESOURCE;
+        goto exit;
+    }
+#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
+    start_rcomm_time = MPI_Wtime();
+#endif
+    ret = ompi_fcoll_base_coll_allgather_array (&max_data,
+						1,
+						MPI_LONG,
+						total_bytes_per_process,
+						1,
+						MPI_LONG,
+						0,
+						fh->f_procs_in_group,
+						fh->f_procs_per_group,
+						fh->f_comm);
+    if (OMPI_SUCCESS != ret){
+        goto exit;
+    }
+#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
+    end_rcomm_time = MPI_Wtime();
+    rcomm_time += end_rcomm_time - start_rcomm_time;
+#endif
+
+    for (i=0 ; i<fh->f_procs_per_group ; i++) {
+        total_bytes += total_bytes_per_process[i];
+    }
+
+    if (NULL != total_bytes_per_process) {
+        free (total_bytes_per_process);
+        total_bytes_per_process = NULL;
+    }
+
+    /*********************************************************************
+     *** 3. Generate the File offsets/lengths corresponding to this write
+     ********************************************************************/
+    ret = fh->f_generate_current_file_view ((struct ompio_file_t *) fh,
+                                            max_data,
+                                            &local_iov_array,
+                                            &temp_local_count);
+    local_count = temp_local_count;
+
+    if (ret != OMPI_SUCCESS){
+        goto exit;
+    }
+
+    /*************************************************************
+     *** 4. Allgather the File View information at all processes
+     *************************************************************/
+
+    fview_count = (size_t *)malloc (fh->f_procs_per_group * sizeof (size_t));
+    if (NULL == fview_count) {
+        opal_output (1, "OUT OF MEMORY\n");
+        ret = OMPI_ERR_OUT_OF_RESOURCE;
+        goto exit;
+    }
+#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
+    start_rcomm_time = MPI_Wtime();
+#endif
+    ret = ompi_fcoll_base_coll_allgather_array (&local_count,
+						sizeof(size_t),
+						MPI_BYTE,
+						fview_count,
+						sizeof(size_t),
+						MPI_BYTE,
+						0,
+						fh->f_procs_in_group,
+						fh->f_procs_per_group,
+						fh->f_comm);
+    
+    if (OMPI_SUCCESS != ret){
+        goto exit;
+    }
+#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
+    end_rcomm_time = MPI_Wtime();
+    rcomm_time += end_rcomm_time - start_rcomm_time;
+#endif
+
+    displs = (ptrdiff_t *)malloc (fh->f_procs_per_group*sizeof(ptrdiff_t));
+    if (NULL == displs) {
+        opal_output (1, "OUT OF MEMORY\n");
+        ret = OMPI_ERR_OUT_OF_RESOURCE;
+        goto exit;
+    }
+
+    displs[0] = 0;
+    total_fview_count = fview_count[0];
+    for (i=1 ; i<fh->f_procs_per_group ; i++) {
+        total_fview_count += fview_count[i];
+        displs[i] = displs[i-1] + fview_count[i-1];
+    }
+
+#if DEBUG_ON
+    if (my_aggregator == fh->f_rank) {
+    for (i=0 ; i<fh->f_procs_per_group ; i++) {
+    printf ("%d: PROCESS: %d  ELEMENTS: %d  DISPLS: %d\n",
+        fh->f_rank,
+        i,
+        fview_count[i],
+        displs[i]);
+}
+}
+#endif
+
+    /* allocate the global iovec  */
+    if (0 != total_fview_count) {
+        global_iov_array = (struct iovec*)malloc (total_fview_count *
+                                                  sizeof(struct iovec));
+        if (NULL == global_iov_array) {
+            opal_output (1, "OUT OF MEMORY\n");
+            ret = OMPI_ERR_OUT_OF_RESOURCE;
+            goto exit;
+        }
+    }
+#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
+    start_rcomm_time = MPI_Wtime();
+#endif
+    ret =  ompi_fcoll_base_coll_allgatherv_array (local_iov_array,
+						  local_count,
+						  fh->f_iov_type,
+						  global_iov_array,
+						  fview_count,
+						  displs,
+						  fh->f_iov_type,
+						  0,
+						  fh->f_procs_in_group,
+						  fh->f_procs_per_group,
+						  fh->f_comm);
+    
+    if (OMPI_SUCCESS != ret){
+        goto exit;
+    }
+#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
+    end_rcomm_time = MPI_Wtime();
+    rcomm_time += end_rcomm_time - start_rcomm_time;
+#endif
+
+    /****************************************************************************************
+     *** 5. Sort the global offset/lengths list based on the offsets.
+     *** The result of the sort operation is the 'sorted', an integer array,
+     *** which contains the indexes of the global_iov_array based on the offset.
+     *** For example, if global_iov_array[x].offset is followed by global_iov_array[y].offset
+     *** in the file, and that one is followed by global_iov_array[z].offset, than
+     *** sorted[0] = x, sorted[1]=y and sorted[2]=z;
+     ******************************************************************************************/
+    if (0 != total_fview_count) {
+       sorted = (int *)malloc (total_fview_count * sizeof(int));
+      if (NULL == sorted) {
+            opal_output (1, "OUT OF MEMORY\n");
+            ret = OMPI_ERR_OUT_OF_RESOURCE;
+            goto exit;
+        }
+        ompi_fcoll_base_sort_iovec (global_iov_array, total_fview_count, sorted);
+    }
+
+    if (NULL != local_iov_array) {
+        free (local_iov_array);
+        local_iov_array = NULL;
+    }
+
+#if DEBUG_ON
+    if (my_aggregator == fh->f_rank) {
+        for (i=0 ; i<total_fview_count ; i++) {
+            printf("%d: OFFSET: %p   LENGTH: %d\n",
+                   fh->f_rank,
+                   global_iov_array[sorted[i]].iov_base,
+                   global_iov_array[sorted[i]].iov_len);
+        }
+    }
+#endif
+
+    /*************************************************************
+     *** 6. Determine the number of cycles required to execute this
+     ***    operation
+     *************************************************************/
+    bytes_per_cycle = fh->f_bytes_per_agg;
+    cycles = ceil((double)total_bytes/bytes_per_cycle);
+
+    if ( my_aggregator == fh->f_rank) {
+      disp_index = (ptrdiff_t *)malloc (fh->f_procs_per_group * sizeof (ptrdiff_t));
+      if (NULL == disp_index) {
+            opal_output (1, "OUT OF MEMORY\n");
+            ret = OMPI_ERR_OUT_OF_RESOURCE;
+            goto exit;
+        }
+
+        blocklen_per_process = (int **)malloc (fh->f_procs_per_group * sizeof (int*));
+        if (NULL == blocklen_per_process) {
+            opal_output (1, "OUT OF MEMORY\n");
+            ret = OMPI_ERR_OUT_OF_RESOURCE;
+            goto exit;
+        }
+
+        displs_per_process = (MPI_Aint **)malloc (fh->f_procs_per_group * sizeof (MPI_Aint*));
+        if (NULL == displs_per_process){
+            opal_output (1, "OUT OF MEMORY\n");
+            ret = OMPI_ERR_OUT_OF_RESOURCE;
+            goto exit;
+        }
+
+        for (i=0;i<fh->f_procs_per_group;i++){
+            blocklen_per_process[i] = NULL;
+            displs_per_process[i] = NULL;
+        }
+
+	send_req = (MPI_Request *) malloc (fh->f_procs_per_group * sizeof(MPI_Request));
+	if (NULL == send_req){
+	    opal_output ( 1, "OUT OF MEMORY\n");
+	    ret = OMPI_ERR_OUT_OF_RESOURCE;
+	    goto exit;
+	}
+
+        if (use_accelerator_buffer) {
+            opal_output_verbose(10, ompi_fcoll_base_framework.framework_output,
+                                "Allocating GPU device buffer for aggregation\n");
+            ret = opal_accelerator.mem_alloc(MCA_ACCELERATOR_NO_DEVICE_ID, (void**)&global_buf,
+                                             bytes_per_cycle);
+            if (OPAL_SUCCESS != ret) {
+                opal_output(1, "Could not allocate accelerator memory");
+                ret = OMPI_ERR_OUT_OF_RESOURCE;
+                goto exit;
+            }
+        } else {global_buf = (char *) malloc (bytes_per_cycle);
+            if (NULL == global_buf){
+                opal_output(1, "OUT OF MEMORY\n");
+                ret = OMPI_ERR_OUT_OF_RESOURCE;
+                goto exit;
+            }
+	}
+
+	sendtype = (ompi_datatype_t **) malloc (fh->f_procs_per_group * sizeof(ompi_datatype_t *));
+	if (NULL == sendtype) {
+            opal_output (1, "OUT OF MEMORY\n");
+	    ret = OMPI_ERR_OUT_OF_RESOURCE;
+	    goto exit;
+	}
+
+	for(l=0;l<fh->f_procs_per_group;l++){
+            sendtype[l] = MPI_DATATYPE_NULL;
+	}
+    }
+
+
+
+
+#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
+    start_rexch = MPI_Wtime();
+#endif
+    n = 0;
+    bytes_remaining = 0;
+    current_index = 0;
+
+    for (index = 0; index < cycles; index++) {
+        /**********************************************************************
+         ***  7a. Getting ready for next cycle: initializing and freeing buffers
+	 **********************************************************************/
+        if (my_aggregator == fh->f_rank) {
+             if (NULL != fh->f_io_array) {
+                free (fh->f_io_array);
+                fh->f_io_array = NULL;
+            }
+            fh->f_num_of_io_entries = 0;
+
+            if (NULL != sendtype){
+                for (i =0; i< fh->f_procs_per_group; i++) {
+		    if ( MPI_DATATYPE_NULL != sendtype[i] ) {
+                        ompi_datatype_destroy(&sendtype[i]);
+                        sendtype[i] = MPI_DATATYPE_NULL;
+                    }
+		}
+            }
+
+            for(l=0;l<fh->f_procs_per_group;l++){
+                disp_index[l] =  1;
+
+                if (NULL != blocklen_per_process[l]){
+                    free(blocklen_per_process[l]);
+                    blocklen_per_process[l] = NULL;
+                }
+                if (NULL != displs_per_process[l]){
+                    free(displs_per_process[l]);
+                    displs_per_process[l] = NULL;
+                }
+                blocklen_per_process[l] = (int *) calloc (1, sizeof(int));
+                if (NULL == blocklen_per_process[l]) {
+                    opal_output (1, "OUT OF MEMORY for blocklen\n");
+                    ret = OMPI_ERR_OUT_OF_RESOURCE;
+                    goto exit;
+                }
+                displs_per_process[l] = (MPI_Aint *) calloc (1, sizeof(MPI_Aint));
+                if (NULL == displs_per_process[l]){
+                    opal_output (1, "OUT OF MEMORY for displs\n");
+                    ret = OMPI_ERR_OUT_OF_RESOURCE;
+                    goto exit;
+                }
+            }
+
+            if (NULL != sorted_file_offsets){
+                free(sorted_file_offsets);
+                sorted_file_offsets = NULL;
+            }
+
+            if(NULL != file_offsets_for_agg){
+                free(file_offsets_for_agg);
+                file_offsets_for_agg = NULL;
+            }
+            if (NULL != memory_displacements){
+                free(memory_displacements);
+                memory_displacements = NULL;
+            }
+        }  /* (my_aggregator == fh->f_rank */
+
+        /**************************************************************************
+         ***  7b. Determine the number of bytes to be actually read in this cycle
+	 **************************************************************************/
+        if (cycles-1 == index) {
+            bytes_to_read_in_cycle = total_bytes - bytes_per_cycle*index;
+        }
+        else {
+            bytes_to_read_in_cycle = bytes_per_cycle;
+        }
+
+#if DEBUG_ON
+        if (my_aggregator == fh->f_rank) {
+            printf ("****%d: CYCLE %d   Bytes %d**********\n",
+                    fh->f_rank,
+                    index,
+                    bytes_to_write_in_cycle);
+        }
+#endif
+
+        /*****************************************************************
+         *** 7c. Calculate how much data will be contributed in this cycle
+	 ***     by each process
+         *****************************************************************/
+        bytes_received = 0;
+
+        while (bytes_to_read_in_cycle) {
+            /* This next block identifies which process is the holder
+            ** of the sorted[current_index] element;
+            */
+            blocks = fview_count[0];
+            for (j=0 ; j<fh->f_procs_per_group ; j++) {
+                if (sorted[current_index] < blocks) {
+                    n = j;
+                    break;
+                }
+                else {
+                    blocks += fview_count[j+1];
+                }
+            }
+
+            if (bytes_remaining) {
+                /* Finish up a partially used buffer from the previous  cycle */
+                if (bytes_remaining <= bytes_to_read_in_cycle) {
+                    /* Data fits completely into the block */
+                    if (my_aggregator == fh->f_rank) {
+                        blocklen_per_process[n][disp_index[n] - 1] = bytes_remaining;
+                        displs_per_process[n][disp_index[n] - 1] =
+                            (ptrdiff_t)global_iov_array[sorted[current_index]].iov_base +
+                            (global_iov_array[sorted[current_index]].iov_len - bytes_remaining);
+
+                        blocklen_per_process[n] = (int *) realloc
+                            ((void *)blocklen_per_process[n], (disp_index[n]+1)*sizeof(int));
+                        displs_per_process[n] = (MPI_Aint *) realloc
+                            ((void *)displs_per_process[n], (disp_index[n]+1)*sizeof(MPI_Aint));
+                        blocklen_per_process[n][disp_index[n]] = 0;
+                        displs_per_process[n][disp_index[n]] = 0;
+                        disp_index[n] += 1;
+                    }
+                    if (fh->f_procs_in_group[n] == fh->f_rank) {
+                        bytes_received += bytes_remaining;
+                    }
+                    current_index ++;
+                    bytes_to_read_in_cycle -= bytes_remaining;
+                    bytes_remaining = 0;
+                    continue;
+                }
+                else {
+                     /* the remaining data from the previous cycle is larger than the
+                        bytes_to_write_in_cycle, so we have to segment again */
+                    if (my_aggregator == fh->f_rank) {
+                        blocklen_per_process[n][disp_index[n] - 1] = bytes_to_read_in_cycle;
+                        displs_per_process[n][disp_index[n] - 1] =
+                            (ptrdiff_t)global_iov_array[sorted[current_index]].iov_base +
+                            (global_iov_array[sorted[current_index]].iov_len
+                             - bytes_remaining);
+                    }
+                    if (fh->f_procs_in_group[n] == fh->f_rank) {
+                        bytes_received += bytes_to_read_in_cycle;
+                    }
+                    bytes_remaining -= bytes_to_read_in_cycle;
+                    bytes_to_read_in_cycle = 0;
+                    break;
+                }
+            }
+            else {
+                /* No partially used entry available, have to start a new one */
+                if (bytes_to_read_in_cycle <
+                    (MPI_Aint) global_iov_array[sorted[current_index]].iov_len) {
+                    /* This entry has more data than we can sendin one cycle */
+                    if (my_aggregator == fh->f_rank) {
+                        blocklen_per_process[n][disp_index[n] - 1] = bytes_to_read_in_cycle;
+                        displs_per_process[n][disp_index[n] - 1] =
+                            (ptrdiff_t)global_iov_array[sorted[current_index]].iov_base ;
+                    }
+
+                    if (fh->f_procs_in_group[n] == fh->f_rank) {
+                        bytes_received += bytes_to_read_in_cycle;
+                    }
+                    bytes_remaining = global_iov_array[sorted[current_index]].iov_len -
+                        bytes_to_read_in_cycle;
+                    bytes_to_read_in_cycle = 0;
+                    break;
+                }
+                else {
+                    /* Next data entry is less than bytes_to_write_in_cycle */
+                    if (my_aggregator ==  fh->f_rank) {
+                        blocklen_per_process[n][disp_index[n] - 1] =
+                            global_iov_array[sorted[current_index]].iov_len;
+                        displs_per_process[n][disp_index[n] - 1] = (ptrdiff_t)
+                            global_iov_array[sorted[current_index]].iov_base;
+                        blocklen_per_process[n] =
+                            (int *) realloc ((void *)blocklen_per_process[n], (disp_index[n]+1)*sizeof(int));
+                        displs_per_process[n] = (MPI_Aint *)realloc
+                            ((void *)displs_per_process[n], (disp_index[n]+1)*sizeof(MPI_Aint));
+                        blocklen_per_process[n][disp_index[n]] = 0;
+                        displs_per_process[n][disp_index[n]] = 0;
+                        disp_index[n] += 1;
+                    }
+                    if (fh->f_procs_in_group[n] == fh->f_rank) {
+                        bytes_received +=
+                            global_iov_array[sorted[current_index]].iov_len;
+                    }
+                    bytes_to_read_in_cycle -=
+                        global_iov_array[sorted[current_index]].iov_len;
+                    current_index ++;
+                    continue;
+                }
+            }
+        } /* end while (bytes_to_read_in_cycle) */
+
+        /*************************************************************************
+	 *** 7d. Calculate the displacement on where to put the data and allocate
+         ***     the receive buffer (global_buf)
+	 *************************************************************************/
+        if (my_aggregator == fh->f_rank) {
+            entries_per_aggregator=0;
+            for (i=0;i<fh->f_procs_per_group; i++){
+                for (j=0;j<disp_index[i];j++){
+                    if (blocklen_per_process[i][j] > 0)
+                        entries_per_aggregator++ ;
+                }
+            }
+            if (entries_per_aggregator > 0){
+                file_offsets_for_agg = (mca_io_ompio_local_io_array *)
+                    malloc(entries_per_aggregator*sizeof(mca_io_ompio_local_io_array));
+                if (NULL == file_offsets_for_agg) {
+                    opal_output (1, "OUT OF MEMORY\n");
+                    ret = OMPI_ERR_OUT_OF_RESOURCE;
+                    goto exit;
+                }
+                sorted_file_offsets = (int *)
+                    malloc (entries_per_aggregator*sizeof(int));
+                if (NULL == sorted_file_offsets){
+                    opal_output (1, "OUT OF MEMORY\n");
+                    ret =  OMPI_ERR_OUT_OF_RESOURCE;
+                    goto exit;
+                }
+                /*Moving file offsets to an IO array!*/
+                temp_index = 0;
+                for (i=0;i<fh->f_procs_per_group; i++){
+                    for(j=0;j<disp_index[i];j++){
+                        if (blocklen_per_process[i][j] > 0){
+                            file_offsets_for_agg[temp_index].length =
+                                blocklen_per_process[i][j];
+                            file_offsets_for_agg[temp_index].process_id = i;
+                            file_offsets_for_agg[temp_index].offset =
+                                displs_per_process[i][j];
+                            temp_index++;
+                        }
+                    }
+                }
+            }
+            else{
+                continue;
+            }
+
+             /* Sort the displacements for each aggregator */
+            read_heap_sort (file_offsets_for_agg,
+                            entries_per_aggregator,
+                            sorted_file_offsets);
+
+            memory_displacements = (MPI_Aint *) malloc
+                (entries_per_aggregator * sizeof(MPI_Aint));
+            memory_displacements[sorted_file_offsets[0]] = 0;
+            for (i=1; i<entries_per_aggregator; i++){
+                memory_displacements[sorted_file_offsets[i]] =
+                    memory_displacements[sorted_file_offsets[i-1]] +
+                    file_offsets_for_agg[sorted_file_offsets[i-1]].length;
+            }
+
+             /**********************************************************
+	     *** 7e. Create the io array, and pass it to fbtl
+	     *********************************************************/
+            fh->f_io_array = (mca_common_ompio_io_array_t *) malloc
+                (entries_per_aggregator * sizeof (mca_common_ompio_io_array_t));
+            if (NULL == fh->f_io_array) {
+                opal_output(1, "OUT OF MEMORY\n");
+                ret = OMPI_ERR_OUT_OF_RESOURCE;
+                goto exit;
+            }
+
+            fh->f_num_of_io_entries = 0;
+            fh->f_io_array[0].offset =
+                (IOVBASE_TYPE *)(intptr_t)file_offsets_for_agg[sorted_file_offsets[0]].offset;
+            fh->f_io_array[0].length =
+                file_offsets_for_agg[sorted_file_offsets[0]].length;
+            fh->f_io_array[0].memory_address =
+                global_buf+memory_displacements[sorted_file_offsets[0]];
+            fh->f_num_of_io_entries++;
+            for (i=1;i<entries_per_aggregator;i++){
+                if (file_offsets_for_agg[sorted_file_offsets[i-1]].offset +
+                    file_offsets_for_agg[sorted_file_offsets[i-1]].length ==
+                    file_offsets_for_agg[sorted_file_offsets[i]].offset){
+                    fh->f_io_array[fh->f_num_of_io_entries - 1].length +=
+                        file_offsets_for_agg[sorted_file_offsets[i]].length;
+                }
+                else{
+                    fh->f_io_array[fh->f_num_of_io_entries].offset =
+                        (IOVBASE_TYPE *)(intptr_t)file_offsets_for_agg[sorted_file_offsets[i]].offset;
+                    fh->f_io_array[fh->f_num_of_io_entries].length =
+                        file_offsets_for_agg[sorted_file_offsets[i]].length;
+                    fh->f_io_array[fh->f_num_of_io_entries].memory_address =
+                        global_buf+memory_displacements[sorted_file_offsets[i]];
+                    fh->f_num_of_io_entries++;
+                }
+            }
+
+
+#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
+            start_read_time = MPI_Wtime();
+#endif
+
+            if (fh->f_num_of_io_entries) {
+                if (use_accelerator_buffer) {
+		    mca_ompio_request_t *ompio_req = NULL;
+		    mca_common_ompio_request_alloc (&ompio_req, MCA_OMPIO_REQUEST_READ);
+
+                    ret = mca_common_ompio_file_iread_pregen(fh, (ompi_request_t *) ompio_req);
+                    if(0 > ret) {
+                        opal_output (1, "common_ompio_file_read_all: mca_common_ompio_iread_pregen failed\n");
+                        ompio_req->req_ompi.req_status.MPI_ERROR = ret;
+                        ompio_req->req_ompi.req_status._ucount = 0;
+                    }
+		    ret = ompi_request_wait ((ompi_request_t**)&ompio_req, MPI_STATUS_IGNORE);
+		    if (OMPI_SUCCESS != ret){
+			goto exit;
+		    }
+                } else {
+                    if ( 0 >  fh->f_fbtl->fbtl_preadv (fh)) {
+                        opal_output (1, "READ FAILED\n");
+                        ret = OMPI_ERROR;
+                        goto exit;
+                    }
+                }
+            }
+
+#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
+            end_read_time = MPI_Wtime();
+            read_time += end_read_time - start_read_time;
+#endif
+            /**********************************************************
+             ******************** DONE READING ************************
+             *********************************************************/
+
+            temp_disp_index = (ptrdiff_t *)calloc (1, fh->f_procs_per_group * sizeof (ptrdiff_t));
+            if (NULL == temp_disp_index) {
+                opal_output (1, "OUT OF MEMORY\n");
+                ret = OMPI_ERR_OUT_OF_RESOURCE;
+                goto exit;
+            }
+            for (i=0; i<entries_per_aggregator; i++){
+                temp_index =
+                    file_offsets_for_agg[sorted_file_offsets[i]].process_id;
+                displs_per_process[temp_index][temp_disp_index[temp_index]] =
+                    memory_displacements[sorted_file_offsets[i]];
+                if (temp_disp_index[temp_index] < disp_index[temp_index]){
+                    temp_disp_index[temp_index] += 1;
+                }
+                else{
+                    printf("temp_disp_index[%zu]: %ld is greater than disp_index[%zu]: %ld\n",
+                           temp_index, (long) temp_disp_index[temp_index],
+                           temp_index, (long) disp_index[temp_index]);
+                }
+            }
+            if (NULL != temp_disp_index){
+                free(temp_disp_index);
+                temp_disp_index = NULL;
+            }
+
+#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
+            start_rcomm_time = MPI_Wtime();
+#endif
+            for (i=0;i<fh->f_procs_per_group;i++){
+                size_t datatype_size;
+                send_req[i] = MPI_REQUEST_NULL;
+                if ( 0 < disp_index[i] ) {
+                    ompi_datatype_create_hindexed(disp_index[i],
+                                                  blocklen_per_process[i],
+                                                  displs_per_process[i],
+                                                  MPI_BYTE,
+                                                  &sendtype[i]);
+                    ompi_datatype_commit(&sendtype[i]);
+                    opal_datatype_type_size(&sendtype[i]->super, &datatype_size);
+
+                    if(datatype_size) {
+                        ret = MCA_PML_CALL (isend(global_buf,
+                                                  1,
+                                                  sendtype[i],
+                                                  fh->f_procs_in_group[i],
+                                                  COMMON_OMPIO_SHUFFLE_TAG,
+                                                  MCA_PML_BASE_SEND_STANDARD,
+                                                  fh->f_comm,
+                                                  &send_req[i]));
+                        if(OMPI_SUCCESS != ret){
+                            goto exit;
+                        }
+                    }
+                }
+            }
+#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
+            end_rcomm_time = MPI_Wtime();
+            rcomm_time += end_rcomm_time - start_rcomm_time;
+#endif
+        }
+
+        /**********************************************************
+         *** 7f.  Scatter the Data from the readers
+         *********************************************************/
+        if(bytes_received) {
+            size_t remaining            = bytes_received;
+            int block_index             = -1;
+            int blocklength_size        = INIT_LEN;
+
+            ptrdiff_t recv_mem_address  = 0;
+            ompi_datatype_t *newType    = MPI_DATATYPE_NULL;
+
+            blocklength_proc            = (int *)       calloc (blocklength_size, sizeof (int));
+            displs_proc                 = (ptrdiff_t *) calloc (blocklength_size, sizeof (ptrdiff_t));
+
+            if (NULL == blocklength_proc || NULL == displs_proc ) {
+                opal_output (1, "OUT OF MEMORY\n");
+                ret = OMPI_ERR_OUT_OF_RESOURCE;
+                goto exit;
+            }
+
+            while (remaining) {
+                block_index++;
+
+                if(0 == block_index) {
+                    recv_mem_address = (ptrdiff_t) (decoded_iov[iov_index].iov_base) + current_position;
+                }
+                else {
+                    // Reallocate more memory if blocklength_size is not enough
+                    if(0 == block_index % INIT_LEN) {
+                        blocklength_size += INIT_LEN;
+                        blocklength_proc = (int *)       realloc(blocklength_proc, blocklength_size * sizeof(int));
+                        displs_proc      = (ptrdiff_t *) realloc(displs_proc, blocklength_size * sizeof(ptrdiff_t));
+                    }
+                    displs_proc[block_index] = (ptrdiff_t) (decoded_iov[iov_index].iov_base) +
+                                                            current_position - recv_mem_address;
+                }
+
+                if (remaining >= (decoded_iov[iov_index].iov_len - current_position)) {
+                    blocklength_proc[block_index] = decoded_iov[iov_index].iov_len - current_position;
+
+                    remaining = remaining - blocklength_proc[block_index];
+                    iov_index = iov_index + 1;
+                    current_position = 0;
+                }
+                else {
+                    blocklength_proc[block_index] = remaining;
+                    current_position += remaining;
+                    remaining = 0;
+                }
+            }
+
+            ompi_datatype_create_hindexed(block_index+1,
+                                          blocklength_proc,
+                                          displs_proc,
+                                          MPI_BYTE,
+                                          &newType);
+            ompi_datatype_commit(&newType);
+
+#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
+            start_rcomm_time = MPI_Wtime();
+#endif
+            ret = MCA_PML_CALL(irecv((char *)recv_mem_address,
+                                     1,
+                                     newType,
+                                     my_aggregator,
+                                     COMMON_OMPIO_SHUFFLE_TAG,
+                                     fh->f_comm,
+                                     &recv_req));
+
+            if ( MPI_DATATYPE_NULL != newType ) {
+                ompi_datatype_destroy(&newType);
+            }
+
+            if (OMPI_SUCCESS != ret){
+                goto exit;
+            }
+        }
+
+        if (my_aggregator == fh->f_rank){
+            ret = ompi_request_wait_all (fh->f_procs_per_group,
+                                         send_req,
+                                         MPI_STATUS_IGNORE);
+            if (OMPI_SUCCESS != ret){
+                goto exit;
+            }
+        }
+
+        ret = ompi_request_wait (&recv_req, MPI_STATUS_IGNORE);
+        if (OMPI_SUCCESS != ret){
+            goto exit;
+        }
+
+#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
+        if(bytes_received) {
+            end_rcomm_time = MPI_Wtime();
+            rcomm_time += end_rcomm_time - start_rcomm_time;
+        }
+#endif
+    } /* end for (index=0; index < cycles; index ++) */
+
+#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
+    end_rexch = MPI_Wtime();
+    read_exch += end_rexch - start_rexch;
+    nentry.time[0] = read_time;
+    nentry.time[1] = rcomm_time;
+    nentry.time[2] = read_exch;
+    if (my_aggregator == fh->f_rank)
+        nentry.aggregator = 1;
+    else
+        nentry.aggregator = 0;
+    nentry.nprocs_for_coll = base_num_io_procs;
+    if (!mca_common_ompio_full_print_queue(fh->f_coll_read_time)){
+        mca_common_ompio_register_print_entry(fh->f_coll_read_time,
+                                              nentry);
+    }
+#endif
+
+exit:
+    if (NULL != global_buf) {
+	if (use_accelerator_buffer) {
+	    opal_accelerator.mem_release(MCA_ACCELERATOR_NO_DEVICE_ID, global_buf);
+	} else {
+	    free (global_buf);
+	}
+        global_buf = NULL;
+    }
+    if (NULL != sorted) {
+        free (sorted);
+        sorted = NULL;
+    }
+    if (NULL != global_iov_array) {
+        free (global_iov_array);
+        global_iov_array = NULL;
+    }
+    if (NULL != fview_count) {
+        free (fview_count);
+        fview_count = NULL;
+    }
+    if (NULL != decoded_iov) {
+        free (decoded_iov);
+        decoded_iov = NULL;
+    }
+    if (NULL != local_iov_array){
+        free(local_iov_array);
+        local_iov_array=NULL;
+    }
+
+    if (NULL != displs) {
+        free (displs);
+        displs = NULL;
+    }
+
+    if (NULL != blocklength_proc) {
+        free (blocklength_proc);
+        blocklength_proc = NULL;
+    }
+
+    if (NULL != displs_proc) {
+        free (displs_proc);
+        displs_proc = NULL;
+    }
+
+    if (my_aggregator == fh->f_rank) {
+
+        if (NULL != sorted_file_offsets){
+            free(sorted_file_offsets);
+            sorted_file_offsets = NULL;
+        }
+        if (NULL != file_offsets_for_agg){
+            free(file_offsets_for_agg);
+            file_offsets_for_agg = NULL;
+        }
+        if (NULL != memory_displacements){
+            free(memory_displacements);
+            memory_displacements= NULL;
+        }
+        if (NULL != sendtype){
+            for (i = 0; i < fh->f_procs_per_group; i++) {
+                if ( MPI_DATATYPE_NULL != sendtype[i] ) {
+                    ompi_datatype_destroy(&sendtype[i]);
+                }
+            }
+            free(sendtype);
+            sendtype=NULL;
+        }
+
+        if (NULL != disp_index){
+            free(disp_index);
+            disp_index = NULL;
+        }
+
+        if ( NULL != blocklen_per_process){
+            for(l=0;l<fh->f_procs_per_group;l++){
+                if (NULL != blocklen_per_process[l]){
+                    free(blocklen_per_process[l]);
+                    blocklen_per_process[l] = NULL;
+                }
+            }
+
+            free(blocklen_per_process);
+            blocklen_per_process = NULL;
+        }
+
+        if (NULL != displs_per_process){
+            for (l=0; i<fh->f_procs_per_group; l++){
+                if (NULL != displs_per_process[l]){
+                    free(displs_per_process[l]);
+                    displs_per_process[l] = NULL;
+                }
+            }
+            free(displs_per_process);
+            displs_per_process = NULL;
+        }
+        if ( NULL != send_req ) {
+            free ( send_req );
+            send_req = NULL;
+        }
+    }
+    return ret;
+}
+
+
+static int read_heap_sort (mca_io_ompio_local_io_array *io_array,
+                           int num_entries,
+                           int *sorted)
+{
+    int i = 0;
+    int j = 0;
+    int left = 0;
+    int right = 0;
+    int largest = 0;
+    int heap_size = num_entries - 1;
+    int temp = 0;
+    unsigned char done = 0;
+    int* temp_arr = NULL;
+
+    temp_arr = (int*)malloc(num_entries*sizeof(int));
+    if (NULL == temp_arr) {
+        opal_output (1, "OUT OF MEMORY\n");
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+    temp_arr[0] = 0;
+    for (i = 1; i < num_entries; ++i) {
+        temp_arr[i] = i;
+    }
+    /* num_entries can be a large no. so NO RECURSION */
+    for (i = num_entries/2-1 ; i>=0 ; i--) {
+        done = 0;
+        j = i;
+        largest = j;
+
+        while (!done) {
+            left = j*2+1;
+            right = j*2+2;
+            if ((left <= heap_size) &&
+                (io_array[temp_arr[left]].offset > io_array[temp_arr[j]].offset)) {
+                largest = left;
+            }
+            else {
+                largest = j;
+            }
+            if ((right <= heap_size) &&
+                (io_array[temp_arr[right]].offset >
+                 io_array[temp_arr[largest]].offset)) {
+                largest = right;
+            }
+            if (largest != j) {
+                temp = temp_arr[largest];
+                temp_arr[largest] = temp_arr[j];
+                temp_arr[j] = temp;
+                j = largest;
+            }
+            else {
+                done = 1;
+            }
+        }
+    }
+
+    for (i = num_entries-1; i >=1; --i) {
+        temp = temp_arr[0];
+        temp_arr[0] = temp_arr[i];
+        temp_arr[i] = temp;
+        heap_size--;
+        done = 0;
+        j = 0;
+        largest = j;
+
+        while (!done) {
+            left =  j*2+1;
+            right = j*2+2;
+
+            if ((left <= heap_size) &&
+                (io_array[temp_arr[left]].offset >
+                 io_array[temp_arr[j]].offset)) {
+                largest = left;
+            }
+            else {
+                largest = j;
+            }
+            if ((right <= heap_size) &&
+                (io_array[temp_arr[right]].offset >
+                 io_array[temp_arr[largest]].offset)) {
+                largest = right;
+            }
+            if (largest != j) {
+                temp = temp_arr[largest];
+                temp_arr[largest] = temp_arr[j];
+                temp_arr[j] = temp;
+                j = largest;
+            }
+            else {
+                done = 1;
+            }
+        }
+        sorted[i] = temp_arr[i];
+    }
+    sorted[0] = temp_arr[0];
+
+    if (NULL != temp_arr) {
+        free(temp_arr);
+        temp_arr = NULL;
+    }
+    return OMPI_SUCCESS;
+}
+
+
+
diff --git a/ompi/mca/common/ompio/common_ompio_file_view.c b/ompi/mca/common/ompio/common_ompio_file_view.c
index a1800192e92..ad569c56f7a 100644
--- a/ompi/mca/common/ompio/common_ompio_file_view.c
+++ b/ompi/mca/common/ompio/common_ompio_file_view.c
@@ -13,6 +13,7 @@
  * Copyright (c) 2017-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2023      Advanced Micro Devices, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -53,6 +54,16 @@ static int datatype_duplicate  (ompi_datatype_t *oldtype, ompi_datatype_t **newt
     return OMPI_SUCCESS;
 }
 
+static void fview_clear (struct ompio_fview_t *fview) {
+    if (NULL != fview->f_decoded_iov) {
+        free (fview->f_decoded_iov);
+        fview->f_decoded_iov = NULL;
+    }
+
+    /* Reset the flags */
+    fview->f_flags &= ~OMPIO_CONTIGUOUS_FVIEW;
+    fview->f_flags &= ~OMPIO_UNIFORM_FVIEW;
+}
 
 int mca_common_ompio_set_view (ompio_file_t *fh,
                                OMPI_MPI_OFFSET_TYPE disp,
@@ -82,19 +93,16 @@ int mca_common_ompio_set_view (ompio_file_t *fh,
         shared_fp_base_module->sharedfp_get_position(fh, &disp);
     }
     
-    if ( NULL != fh->f_etype ) {
+    fview_clear(&(fh->f_fview));
+    if (NULL != fh->f_etype) {
         ompi_datatype_destroy (&fh->f_etype);
     }
-    if ( NULL != fh->f_filetype ) {
+    if (NULL != fh->f_filetype) {
         ompi_datatype_destroy (&fh->f_filetype);
     }
-    if ( NULL != fh->f_orig_filetype ) {
+    if (NULL != fh->f_orig_filetype) {
         ompi_datatype_destroy (&fh->f_orig_filetype);
     }
-    if (NULL != fh->f_decoded_iov) {
-        free (fh->f_decoded_iov);
-        fh->f_decoded_iov = NULL;
-    }
 
     if (NULL != fh->f_datarep) {
         free (fh->f_datarep);
@@ -106,17 +114,8 @@ int mca_common_ompio_set_view (ompio_file_t *fh,
         free (fh->f_file_convertor);
         fh->f_file_convertor = NULL;
     }
-    
-    /* Reset the flags first */
-    if ( fh->f_flags & OMPIO_CONTIGUOUS_FVIEW ) {
-        fh->f_flags &= ~OMPIO_CONTIGUOUS_FVIEW;
-    }
-    if ( fh->f_flags & OMPIO_UNIFORM_FVIEW ) {
-        fh->f_flags &= ~OMPIO_UNIFORM_FVIEW;
-    }
-    if ( fh->f_flags & OMPIO_DATAREP_NATIVE ) {
-        fh->f_flags &= ~OMPIO_DATAREP_NATIVE;
-    }
+
+    fh->f_flags  &= ~OMPIO_DATAREP_NATIVE;
     fh->f_datarep = strdup (datarep);
 
     if ( !(strcmp(datarep, "external32") && strcmp(datarep, "EXTERNAL32"))) {
@@ -131,9 +130,9 @@ int mca_common_ompio_set_view (ompio_file_t *fh,
         fh->f_flags |= OMPIO_DATAREP_NATIVE;
     }
     
-    datatype_duplicate (filetype, &fh->f_orig_filetype );
-    opal_datatype_get_extent(&filetype->super, &lb, &ftype_extent);
-    opal_datatype_type_size (&filetype->super, &ftype_size);
+    datatype_duplicate       (filetype, &(fh->f_orig_filetype));
+    opal_datatype_get_extent (&filetype->super, &lb, &ftype_extent);
+    opal_datatype_type_size  (&filetype->super, &ftype_size);
 
     if ( etype == filetype                             &&
 	 ompi_datatype_is_predefined (filetype )       &&
@@ -144,16 +143,16 @@ int mca_common_ompio_set_view (ompio_file_t *fh,
 	ompi_datatype_commit (&newfiletype);
     }
     else {
-        newfiletype = filetype;
-	fh->f_flags |= OMPIO_FILE_VIEW_IS_SET;
+        newfiletype          = filetype;
+        fh->f_fview.f_flags |= OMPIO_FILE_VIEW_IS_SET;
     }
 
-    fh->f_iov_count   = 0;
-    fh->f_disp        = disp;
-    fh->f_offset      = disp;
-    fh->f_total_bytes = 0;
-    fh->f_index_in_file_view=0;
-    fh->f_position_in_file_view=0;
+    fh->f_fview.f_iov_count             = 0;
+    fh->f_fview.f_disp                  = disp;
+    fh->f_fview.f_offset                = disp;
+    fh->f_fview.f_total_bytes           = 0;
+    fh->f_fview.f_index_in_file_view    = 0;
+    fh->f_fview.f_position_in_file_view = 0;
 
     mca_common_ompio_decode_datatype (fh,
                                       newfiletype,
@@ -161,26 +160,26 @@ int mca_common_ompio_set_view (ompio_file_t *fh,
                                       NULL,
                                       &max_data,
                                       fh->f_file_convertor,
-                                      &fh->f_decoded_iov,
-                                      &fh->f_iov_count);
+                                      &fh->f_fview.f_decoded_iov,
+                                      &fh->f_fview.f_iov_count);
 
-    opal_datatype_get_extent(&newfiletype->super, &lb, &fh->f_view_extent);
+    opal_datatype_get_extent(&newfiletype->super, &lb, &fh->f_fview.f_view_extent);
     opal_datatype_type_ub   (&newfiletype->super, &ub);
-    opal_datatype_type_size (&etype->super, &fh->f_etype_size);
-    opal_datatype_type_size (&newfiletype->super, &fh->f_view_size);
+    opal_datatype_type_size (&etype->super, &fh->f_fview.f_etype_size);
+    opal_datatype_type_size (&newfiletype->super, &fh->f_fview.f_view_size);
     datatype_duplicate (etype, &fh->f_etype);
     // This file type is our own representation. The original is stored
     // in orig_file type, No need to set args on this one.
     ompi_datatype_duplicate (newfiletype, &fh->f_filetype);
 
-    if ( (fh->f_view_size % fh->f_etype_size) ) {
+    if ( (fh->f_fview.f_view_size % fh->f_fview.f_etype_size) ) {
         // File view is not a multiple of the etype.
         return MPI_ERR_ARG;
     }
     
     // make sure that displacement is not negative, which could
     // lead to an illegal access.
-    if ( 0 < fh->f_iov_count && 0 > (off_t)fh->f_decoded_iov[0].iov_base ) {
+    if ( 0 < fh->f_fview.f_iov_count && 0 > (off_t)fh->f_fview.f_decoded_iov[0].iov_base ) {
         // I think MPI_ERR_TYPE would be more appropriate, but
         // this is the error code expected in a testsuite, so I just
         // go with this.
@@ -197,8 +196,8 @@ int mca_common_ompio_set_view (ompio_file_t *fh,
 
     if (opal_datatype_is_contiguous_memory_layout(&etype->super,1)) {
         if (opal_datatype_is_contiguous_memory_layout(&filetype->super,1) &&
-	    fh->f_view_extent == (ptrdiff_t)fh->f_view_size ) {
-            fh->f_flags |= OMPIO_CONTIGUOUS_FVIEW;
+	    fh->f_fview.f_view_extent == (ptrdiff_t)fh->f_fview.f_view_size ) {
+            fh->f_fview.f_flags |= OMPIO_CONTIGUOUS_FVIEW;
         }
     }
 
@@ -220,19 +219,23 @@ int mca_common_ompio_set_view (ompio_file_t *fh,
        }
     }
 
-    char char_stripe[MPI_MAX_INFO_VAL];
+    opal_cstring_t *stripe_str;
     /* Check the info object set during File_open */
-    opal_info_get (fh->f_info, "cb_nodes", MPI_MAX_INFO_VAL, char_stripe, &flag);
+    opal_info_get (info, "cb_nodes", &stripe_str, &flag);
     if ( flag ) {
-        sscanf ( char_stripe, "%d", &num_cb_nodes );
-        OMPIO_MCA_PRINT_INFO(fh, "cb_nodes", char_stripe, "");
+        sscanf ( stripe_str->string, "%d", &num_cb_nodes );
+        OMPIO_MCA_PRINT_INFO(fh, "cb_nodes", stripe_str->string, "");
+        /* add the key/value to the file's info object */
+        opal_info_set_cstring(fh->f_info, "cb_nodes", stripe_str);
+        OBJ_RELEASE(stripe_str);
     }
     else {
         /* Check the info object set during file_set_view */
-        opal_info_get (info, "cb_nodes", MPI_MAX_INFO_VAL, char_stripe, &flag);
+        opal_info_get (fh->f_info, "cb_nodes", &stripe_str, &flag);
         if ( flag ) {
-            sscanf ( char_stripe, "%d", &num_cb_nodes );
-            OMPIO_MCA_PRINT_INFO(fh, "cb_nodes", char_stripe, "");
+            sscanf ( stripe_str->string, "%d", &num_cb_nodes );
+            OMPIO_MCA_PRINT_INFO(fh, "cb_nodes", stripe_str->string, "");
+            OBJ_RELEASE(stripe_str);
         }
     }
         
@@ -323,23 +326,27 @@ int mca_common_ompio_set_view (ompio_file_t *fh,
     }
 
     bool info_is_set=false;
-    opal_info_get (fh->f_info, "collective_buffering", MPI_MAX_INFO_VAL, char_stripe, &flag);
+    opal_info_get (info, "collective_buffering", &stripe_str, &flag);
     if ( flag ) {
-        if ( strncmp ( char_stripe, "false", sizeof("true") )){
+        if ( strncmp ( stripe_str->string, "false", sizeof("true") )){
             info_is_set = true;
-            OMPIO_MCA_PRINT_INFO(fh, "collective_buffering", char_stripe, "enforcing using individual fcoll component");
+            OMPIO_MCA_PRINT_INFO(fh, "collective_buffering", stripe_str->string, "enforcing using individual fcoll component");
         } else {
-            OMPIO_MCA_PRINT_INFO(fh, "collective_buffering", char_stripe, "");
+            OMPIO_MCA_PRINT_INFO(fh, "collective_buffering", stripe_str->string, "");
         }
+        /* add the key/value to the file's info object */
+        opal_info_set_cstring(fh->f_info, "collective_buffering", stripe_str);
+        OBJ_RELEASE(stripe_str);
     } else {
-        opal_info_get (info, "collective_buffering", MPI_MAX_INFO_VAL, char_stripe, &flag);
+        opal_info_get (fh->f_info, "collective_buffering", &stripe_str, &flag);
         if ( flag ) {
-            if ( strncmp ( char_stripe, "false", sizeof("true") )){
+            if ( strncmp ( stripe_str->string, "false", sizeof("true") )){
                 info_is_set = true;
-                OMPIO_MCA_PRINT_INFO(fh, "collective_buffering", char_stripe, "enforcing using individual fcoll component");
+                OMPIO_MCA_PRINT_INFO(fh, "collective_buffering", stripe_str->string, "enforcing using individual fcoll component");
             } else {
-                OMPIO_MCA_PRINT_INFO(fh, "collective_buffering", char_stripe, "");
+                OMPIO_MCA_PRINT_INFO(fh, "collective_buffering", stripe_str->string, "");
             }
+            OBJ_RELEASE(stripe_str);
         }
     }
 
@@ -381,17 +388,17 @@ OMPI_MPI_OFFSET_TYPE get_contiguous_chunk_size (ompio_file_t *fh, int flag)
 
     if ( flag  ) {
         global_avg[0] = MCA_IO_DEFAULT_FILE_VIEW_SIZE;
-        fh->f_avg_view_size = fh->f_view_size;
+        fh->f_avg_view_size = fh->f_fview.f_view_size;
     }
     else {
-        for (i=0 ; i<(int)fh->f_iov_count ; i++) {
-            avg[0] += fh->f_decoded_iov[i].iov_len;
+        for (i=0 ; i<(int)fh->f_fview.f_iov_count ; i++) {
+            avg[0] += fh->f_fview.f_decoded_iov[i].iov_len;
         }
-        if ( 0 != fh->f_iov_count ) {
-            avg[0] = avg[0]/fh->f_iov_count;
+        if ( 0 != fh->f_fview.f_iov_count ) {
+            avg[0] = avg[0]/fh->f_fview.f_iov_count;
         }
-        avg[1] = (OMPI_MPI_OFFSET_TYPE) fh->f_iov_count;
-        avg[2] = (OMPI_MPI_OFFSET_TYPE) fh->f_view_size;
+        avg[1] = (OMPI_MPI_OFFSET_TYPE) fh->f_fview.f_iov_count;
+        avg[2] = (OMPI_MPI_OFFSET_TYPE) fh->f_fview.f_view_size;
         
         fh->f_comm->c_coll->coll_allreduce (avg,
                                             global_avg,
@@ -408,4 +415,32 @@ OMPI_MPI_OFFSET_TYPE get_contiguous_chunk_size (ompio_file_t *fh, int flag)
     return global_avg[0];
 }
 
+int mca_common_ompio_fview_duplicate (struct ompio_fview_t *outfv, struct ompio_fview_t *infv)
+{
+    uint32_t i;
+
+    memset(outfv, 0, sizeof(struct ompio_fview_t));
+    outfv->f_flags     = infv->f_flags;
+    outfv->f_offset    = infv->f_offset;
+    outfv->f_disp      = infv->f_disp;
+    outfv->f_iov_count = infv->f_iov_count;
 
+    outfv->f_decoded_iov = (struct iovec*) malloc (outfv->f_iov_count * sizeof(struct iovec));
+    if (NULL == outfv->f_decoded_iov) {
+        opal_output(1, "common_ompio_duplicate_fview: could not allocate memory\n");
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+    for (i=0; i < outfv->f_iov_count; i++) {
+        outfv->f_decoded_iov[i].iov_base = infv->f_decoded_iov[i].iov_base;
+        outfv->f_decoded_iov[i].iov_len  = infv->f_decoded_iov[i].iov_len ;
+    }
+
+    outfv->f_position_in_file_view = infv->f_position_in_file_view;
+    outfv->f_total_bytes           = infv->f_total_bytes;
+    outfv->f_index_in_file_view    = infv->f_index_in_file_view;
+    outfv->f_view_extent           = infv->f_view_extent;
+    outfv->f_view_size             = infv->f_view_size;
+    outfv->f_etype_size            = infv->f_etype_size;
+
+    return OMPI_SUCCESS;
+}
diff --git a/ompi/mca/common/ompio/common_ompio_file_write.c b/ompi/mca/common/ompio/common_ompio_file_write.c
index 066afb8844a..b5d3900e7a3 100644
--- a/ompi/mca/common/ompio/common_ompio_file_write.c
+++ b/ompi/mca/common/ompio/common_ompio_file_write.c
@@ -12,6 +12,9 @@
  * Copyright (c) 2008-2019 University of Houston. All rights reserved.
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -35,156 +38,260 @@
 #include <unistd.h>
 #include <math.h>
 
+static int mca_common_ompio_file_write_pipelined (ompio_file_t *fh, const void *buf,
+                                                  size_t count, struct ompi_datatype_t *datatype,
+                                                  ompi_status_public_t *status);
+
+static int mca_common_ompio_file_write_default (ompio_file_t *fh, const void *buf,
+                                                size_t count, struct ompi_datatype_t *datatype,
+                                                ompi_status_public_t *status);
+
 int mca_common_ompio_file_write (ompio_file_t *fh,
-			       const void *buf,
-			       int count,
-			       struct ompi_datatype_t *datatype,
-			       ompi_status_public_t *status)
+                               const void *buf,
+                               size_t count,
+                               struct ompi_datatype_t *datatype,
+                               ompi_status_public_t *status)
 {
     int ret = OMPI_SUCCESS;
-    int index = 0;
-    int cycles = 0;
-
-    uint32_t iov_count = 0;
-    struct iovec *decoded_iov = NULL;
-    size_t bytes_per_cycle=0;
-    size_t total_bytes_written = 0;
-    size_t max_data=0, real_bytes_written=0;
-    ssize_t ret_code=0;
-    size_t spc=0;
-    int i = 0; /* index into the decoded iovec of the buffer */
-    int j = 0; /* index into the file view iovec */
-
     if (fh->f_amode & MPI_MODE_RDONLY){
-//      opal_output(10, "Improper use of FILE Mode, Using RDONLY for write!\n");
         ret = MPI_ERR_READ_ONLY;
-      return ret;
+        return ret;
     }
 
-    
-    if ( 0 == count ) {
-        if ( MPI_STATUS_IGNORE != status ) {
+    if (0 == count || 0 == fh->f_fview.f_iov_count) {
+        if (MPI_STATUS_IGNORE != status) {
             status->_ucount = 0;
         }
         return ret;
     }
 
     bool need_to_copy = false;
-
-#if OPAL_CUDA_SUPPORT
     int is_gpu, is_managed;
-    mca_common_ompio_check_gpu_buf ( fh, buf, &is_gpu, &is_managed);
-    if ( is_gpu && !is_managed ) {
+    mca_common_ompio_check_gpu_buf (fh, buf, &is_gpu, &is_managed);
+    if (is_gpu && !is_managed) {
         need_to_copy = true;
     }
-#endif
 
-    if ( !( fh->f_flags & OMPIO_DATAREP_NATIVE ) &&
+    if ( !(fh->f_flags & OMPIO_DATAREP_NATIVE ) &&
          !(datatype == &ompi_mpi_byte.dt  ||
            datatype == &ompi_mpi_char.dt   )) {
         /* only need to copy if any of these conditions are given:
-           1. buffer is an unmanaged CUDA buffer (checked above).
+           1. buffer is an unmanaged device buffer (checked above).
            2. Datarepresentation is anything other than 'native' and
            3. datatype is not byte or char (i.e it does require some actual
               work to be done e.g. for external32.
         */
         need_to_copy = true;
     }         
-    
-    if ( need_to_copy ) {
-        size_t pos=0;
-        char *tbuf=NULL;
-        opal_convertor_t convertor;
-        
-        OMPIO_PREPARE_BUF(fh,buf,count,datatype,tbuf,&convertor,max_data,decoded_iov,iov_count);     
-        opal_convertor_pack (&convertor, decoded_iov, &iov_count, &pos );
-        opal_convertor_cleanup ( &convertor);
-    }
-    else {
-        mca_common_ompio_decode_datatype (fh,
-                                          datatype,
-                                          count,
-                                          buf,
-                                          &max_data,
-                                          fh->f_mem_convertor,
-                                          &decoded_iov,
-                                          &iov_count);
-    }
 
-    if ( 0 < max_data && 0 == fh->f_iov_count  ) {
-        if ( MPI_STATUS_IGNORE != status ) {
-            status->_ucount = 0;
-        }
-        if (NULL != decoded_iov) {
-            free (decoded_iov);
-            decoded_iov = NULL;
-        }
-        return OMPI_SUCCESS;
+    if (need_to_copy) {
+        return mca_common_ompio_file_write_pipelined (fh, buf, count, datatype, status);
+    } else {
+        return mca_common_ompio_file_write_default (fh, buf, count, datatype, status);
     }
 
-    if ( -1 == OMPIO_MCA_GET(fh, cycle_buffer_size )) {
-        bytes_per_cycle = max_data;
-    }
-    else {
-	bytes_per_cycle = OMPIO_MCA_GET(fh, cycle_buffer_size);
-    }
-    cycles = ceil((double)max_data/bytes_per_cycle);
+    return OMPI_SUCCESS; //silence compiler
+}
 
-#if 0
-    printf ("Bytes per Cycle: %d   Cycles: %d\n", bytes_per_cycle, cycles);
-#endif
+int mca_common_ompio_file_write_default (ompio_file_t *fh,
+                                         const void *buf,
+                                         size_t count,
+                                         struct ompi_datatype_t *datatype,
+                                         ompi_status_public_t *status)
+{
+    int index = 0;
+    int cycles = 0;
+    uint32_t iov_count = 0;
+    struct iovec *decoded_iov = NULL;
+    size_t bytes_per_cycle = 0;
+    size_t total_bytes_written = 0;
+    size_t max_data = 0, real_bytes_written = 0;
+    ssize_t ret_code = 0;
+    size_t spc = 0;
+    int i = 0; /* index into the decoded iovec of the buffer */
+
+    mca_common_ompio_decode_datatype (fh, datatype, count,
+                                      buf, &max_data,
+                                      fh->f_mem_convertor,
+                                      &decoded_iov, &iov_count);
+
+    bytes_per_cycle = OMPIO_MCA_GET(fh, cycle_buffer_size);
+    cycles = ceil((double)max_data/bytes_per_cycle);
 
-    j = fh->f_index_in_file_view;
     for (index = 0; index < cycles; index++) {
-        mca_common_ompio_build_io_array ( fh,
-                                          index,
-                                          cycles,
-                                          bytes_per_cycle,
-                                          max_data,
-                                          iov_count,
-                                          decoded_iov,
-                                          &i,
-                                          &j,
-                                          &total_bytes_written, 
-                                          &spc,
-                                          &fh->f_io_array,
-                                          &fh->f_num_of_io_entries);
-
-        if (fh->f_num_of_io_entries) {
-            ret_code =fh->f_fbtl->fbtl_pwritev (fh);
-            if ( 0<= ret_code ) {
-                real_bytes_written+= (size_t)ret_code;
-            }
-        }
+        mca_common_ompio_build_io_array ( &(fh->f_fview), index, cycles,
+                                          bytes_per_cycle, max_data,
+                                          iov_count, decoded_iov,
+                                          &i, &total_bytes_written, &spc,
+                                          &fh->f_io_array, &fh->f_num_of_io_entries);
+        if (fh->f_num_of_io_entries == 0) {
+	    ret_code = 0;
+	    goto exit;
+	}
+
+	ret_code = fh->f_fbtl->fbtl_pwritev (fh);
+	if (0 <= ret_code) {
+	    real_bytes_written+= (size_t)ret_code;
+	    // Reset ret_code since it is also used to return an error
+	    ret_code = 0;
+	} else {
+	    goto exit;
+	}
 
         fh->f_num_of_io_entries = 0;
-        if (NULL != fh->f_io_array) {
-            free (fh->f_io_array);
-            fh->f_io_array = NULL;
+        free (fh->f_io_array);
+        fh->f_io_array = NULL;
+    }
+
+ exit:
+    free (decoded_iov);
+    if ( MPI_STATUS_IGNORE != status ) {
+        status->_ucount = real_bytes_written;
+    }
+
+    return ret_code;
+}
+
+int mca_common_ompio_file_write_pipelined (ompio_file_t *fh,
+                                           const void *buf,
+                                           size_t count,
+                                           struct ompi_datatype_t *datatype,
+                                           ompi_status_public_t *status)
+{
+    int index = 0;
+    int cycles = 0;
+
+    uint32_t iov_count = 0;
+    struct iovec decoded_iov;
+    size_t bytes_per_cycle=0, tbw = 0;
+    size_t max_data=0, real_bytes_written=0;
+    ssize_t ret_code=0;
+    size_t spc=0;
+    int i = 0; /* index into the decoded iovec of the buffer */
+
+    size_t pos=0;
+    char *tbuf1=NULL, *tbuf2=NULL;
+    char *packbuf=NULL, *writebuf=NULL;
+    mca_ompio_request_t *ompio_req=NULL, *prev_ompio_req=NULL;
+    opal_convertor_t convertor;
+    bool can_overlap = (NULL != fh->f_fbtl->fbtl_ipwritev);
+
+    bytes_per_cycle = OMPIO_MCA_GET(fh, pipeline_buffer_size);
+    OMPIO_PREPARE_BUF (fh, buf, count, datatype, tbuf1, &convertor,
+		       max_data, bytes_per_cycle, &decoded_iov, iov_count);
+    cycles = ceil((double)max_data/bytes_per_cycle);
+
+    packbuf = tbuf1;
+    if (can_overlap) {
+        //Allocate second buffer to alternate packing and writing
+        tbuf2 = mca_common_ompio_alloc_buf (fh, bytes_per_cycle);
+        if (NULL == tbuf2) {
+            opal_output(1, "common_ompio: error allocating memory\n");
+            return OMPI_ERR_OUT_OF_RESOURCE;
         }
+        writebuf = tbuf2;
     }
 
-    if ( need_to_copy ) {
-        mca_common_ompio_release_buf (fh, decoded_iov->iov_base);
+    /*
+    ** The code combines two scenarios:
+    ** 1. having async write (i.e. ipwritev) which allows to overlap two 
+    **    iterations.
+    ** 2. not having async write, which doesn't allow for overlap.
+    ** 
+    ** In the first case we use a double buffering technique, the sequence is
+    **    - construct io-array for iter i 
+    **    - pack buffer for iter i
+    **    - post ipwritev for iter i
+    **    - wait for iter i-1
+    **    - swap buffers
+    **
+    ** In the second case, the sequence is
+    **    - construct io-array for iter i
+    **    - pack buffer i
+    **    - post pwrite for iter i
+    */
+    
+    if (can_overlap) {
+	mca_common_ompio_register_progress ();	    
     }
 
+    for (index = 0; index <= cycles; index++) {
+        if (index < cycles) {
+            decoded_iov.iov_base = packbuf;
+            decoded_iov.iov_len  = bytes_per_cycle;
+            iov_count             = 1;
+
+            opal_convertor_pack (&convertor, &decoded_iov, &iov_count, &pos);
+            spc = 0;
+            tbw = 0;
+            i   = 0;
+            mca_common_ompio_build_io_array (&(fh->f_fview), index, cycles,
+                                             bytes_per_cycle, pos,
+                                             iov_count, &decoded_iov,
+                                             &i, &tbw, &spc,
+                                             &fh->f_io_array, &fh->f_num_of_io_entries);
+	    if (fh->f_num_of_io_entries== 0) {
+		ret_code = 0;
+		goto exit;
+	    }
+
+	    if (can_overlap) {
+		mca_common_ompio_request_alloc ( &ompio_req, MCA_OMPIO_REQUEST_WRITE);
+		fh->f_fbtl->fbtl_ipwritev (fh, (ompi_request_t *)ompio_req);
+	    } else {
+		ret_code = fh->f_fbtl->fbtl_pwritev (fh);
+		if (0 <= ret_code) {
+		    real_bytes_written += (size_t)ret_code;
+		    // Reset ret_code since it is also used to return an error
+		    ret_code = 0;
+		} else {
+		    goto exit;
+		}
+            }
+	}
+	
+        if (can_overlap) {
+            if (index != 0) {
+                ompi_status_public_t stat;
+                ret_code = ompi_request_wait ((ompi_request_t **)&prev_ompio_req, &stat);
+                if (OMPI_SUCCESS != ret_code) {
+                    goto exit;
+                }
+                real_bytes_written += stat._ucount;
+            }
+	    prev_ompio_req = ompio_req;
+        }
+	    
+	fh->f_num_of_io_entries = 0;
+	free (fh->f_io_array);
+	fh->f_io_array = NULL;
+	
+	if (can_overlap) {
+	    char *tmp = packbuf;
+	    packbuf   = writebuf;
+	    writebuf  = tmp;
+	}
+    }
 
-    if (NULL != decoded_iov) {
-        free (decoded_iov);
-        decoded_iov = NULL;
+ exit:
+    mca_common_ompio_release_buf (fh, tbuf1);
+    if (can_overlap) {
+        mca_common_ompio_release_buf (fh, tbuf2);
     }
 
+    opal_convertor_cleanup (&convertor);
+
     if ( MPI_STATUS_IGNORE != status ) {
         status->_ucount = real_bytes_written;
     }
 
-    return ret;
+    return ret_code;
 }
 
 int mca_common_ompio_file_write_at (ompio_file_t *fh,
 				  OMPI_MPI_OFFSET_TYPE offset,
 				  const void *buf,
-				  int count,
+				  size_t count,
 				  struct ompi_datatype_t *datatype,
 				  ompi_status_public_t *status)
 {
@@ -205,58 +312,87 @@ int mca_common_ompio_file_write_at (ompio_file_t *fh,
     return ret;
 }
 
+static void mca_common_ompio_post_next_write_subreq(struct mca_ompio_request_t *req, int index)
+{
+    uint32_t iov_count = 1;
+    size_t bytes_per_cycle = OMPIO_MCA_GET(req->req_fh, pipeline_buffer_size);
+    size_t pos=0, spc = 0, tbw = 0;
+    int i = 0;
+    mca_ompio_request_t *ompio_subreq=NULL;
+    struct iovec decoded_iov;
+
+    if (req->req_num_subreqs == index) {
+        /* all done */
+        return;
+    }
+
+    decoded_iov.iov_base = req->req_tbuf;
+    decoded_iov.iov_len  = req->req_size;
+    opal_convertor_pack (&req->req_convertor, &decoded_iov, &iov_count, &pos);
+
+    mca_common_ompio_build_io_array (req->req_fview, index, req->req_num_subreqs,
+                                     bytes_per_cycle, pos,
+                                     iov_count, &decoded_iov,
+                                     &i, &tbw, &spc,
+                                     &req->req_fh->f_io_array,
+                                     &req->req_fh->f_num_of_io_entries);
+
+    mca_common_ompio_request_alloc (&ompio_subreq, MCA_OMPIO_REQUEST_WRITE);
+    ompio_subreq->req_parent = req;
+    req->req_fh->f_fbtl->fbtl_ipwritev (req->req_fh, (ompi_request_t *)ompio_subreq);
+
+    free(req->req_fh->f_io_array);
+    req->req_fh->f_io_array = NULL;
+    req->req_fh->f_num_of_io_entries = 0;
+}
+
 int mca_common_ompio_file_iwrite (ompio_file_t *fh,
-				const void *buf,
-				int count,
-				struct ompi_datatype_t *datatype,
-				ompi_request_t **request)
+                                const void *buf,
+                                size_t count,
+                                struct ompi_datatype_t *datatype,
+                                ompi_request_t **request)
 {
     int ret = OMPI_SUCCESS;
     mca_ompio_request_t *ompio_req=NULL;
+    struct iovec *decoded_iov = NULL;
     size_t spc=0;
 
     if (fh->f_amode & MPI_MODE_RDONLY){
-//      opal_output(10, "Improper use of FILE Mode, Using RDONLY for write!\n");
         ret = MPI_ERR_READ_ONLY;
-      return ret;
+        return ret;
     }
-    
-    mca_common_ompio_request_alloc ( &ompio_req, MCA_OMPIO_REQUEST_WRITE);
 
-    if ( 0 == count ) {
+    mca_common_ompio_request_alloc (&ompio_req, MCA_OMPIO_REQUEST_WRITE);
+
+    if (0 == count || 0 == fh->f_fview.f_iov_count) {
         ompio_req->req_ompi.req_status.MPI_ERROR = OMPI_SUCCESS;
         ompio_req->req_ompi.req_status._ucount = 0;
         ompi_request_complete (&ompio_req->req_ompi, false);
         *request = (ompi_request_t *) ompio_req;
-        
+
         return OMPI_SUCCESS;
     }
 
-    if ( NULL != fh->f_fbtl->fbtl_ipwritev ) {
+    if (NULL != fh->f_fbtl->fbtl_ipwritev) {
         /* This fbtl has support for non-blocking operations */
-        
         uint32_t iov_count = 0;
-        struct iovec *decoded_iov = NULL;
         size_t max_data = 0;
         size_t total_bytes_written =0;
         int i = 0; /* index into the decoded iovec of the buffer */
-        int j = 0; /* index into the file vie iovec */
-
         bool need_to_copy = false;
 
-#if OPAL_CUDA_SUPPORT
         int is_gpu, is_managed;
         mca_common_ompio_check_gpu_buf ( fh, buf, &is_gpu, &is_managed);
         if ( is_gpu && !is_managed ) {
             need_to_copy = true;
         }
-#endif
+        mca_common_ompio_register_progress ();
 
-        if ( !( fh->f_flags & OMPIO_DATAREP_NATIVE ) &&
+        if ( !(fh->f_flags & OMPIO_DATAREP_NATIVE) &&
              !(datatype == &ompi_mpi_byte.dt  ||
-               datatype == &ompi_mpi_char.dt   )) {
+               datatype == &ompi_mpi_char.dt) ) {
             /* only need to copy if any of these conditions are given:
-               1. buffer is an unmanaged CUDA buffer (checked above).
+               1. buffer is an unmanaged device buffer (checked above).
                2. Datarepresentation is anything other than 'native' and
                3. datatype is not byte or char (i.e it does require some actual
                work to be done e.g. for external32.
@@ -264,76 +400,61 @@ int mca_common_ompio_file_iwrite (ompio_file_t *fh,
             need_to_copy = true;
         }         
         
-        if ( need_to_copy ) {
-            size_t pos=0;
-            char *tbuf=NULL;
-            opal_convertor_t convertor;
-            
-            OMPIO_PREPARE_BUF(fh,buf,count,datatype,tbuf,&convertor,max_data,decoded_iov,iov_count);                    
-            opal_convertor_pack (&convertor, decoded_iov, &iov_count, &pos );
-            opal_convertor_cleanup (&convertor);
-
-            ompio_req->req_tbuf = tbuf;
-            ompio_req->req_size = max_data;
+        if (need_to_copy) {
+            OMPI_MPI_OFFSET_TYPE prev_offset;
+            size_t pipeline_buf_size = OMPIO_MCA_GET(fh, pipeline_buffer_size);
+
+            OMPIO_PREPARE_BUF (fh, buf, count, datatype, ompio_req->req_tbuf,
+                               &ompio_req->req_convertor, max_data,
+                               pipeline_buf_size, NULL, iov_count);
+
+            ompio_req->req_num_subreqs = ceil((double)max_data/pipeline_buf_size);
+            ompio_req->req_size        = pipeline_buf_size;
+            ompio_req->req_max_data    = max_data;
+            ompio_req->req_post_next_subreq = mca_common_ompio_post_next_write_subreq;
+            ompio_req->req_fh          = fh;
+            ompio_req->req_ompi.req_status.MPI_ERROR = MPI_SUCCESS;
+
+	    ompio_req->req_fview = (struct ompio_fview_t *) malloc(sizeof(struct ompio_fview_t));
+	    if (NULL == ompio_req->req_fview) {
+		opal_output(1, "common_ompio: error allocating memory\n");
+		return OMPI_ERR_OUT_OF_RESOURCE;
+	    }
+            ret = mca_common_ompio_fview_duplicate(ompio_req->req_fview, &fh->f_fview);
+            if (OMPI_SUCCESS != ret) {
+                return ret;
+            }
+            mca_common_ompio_file_get_position (fh, &prev_offset );
+            mca_common_ompio_post_next_write_subreq (ompio_req, 0);
+
+            /* Move file pointer to the end of the operation.
+             * Otherwise posting another I/O operation will start of
+             * from the wrong file position. The request will update
+             * the position where to write the next chunk of data
+             * using its internal copy of the file view and file pointer
+             * position.
+             */
+            mca_common_ompio_set_explicit_offset (fh, prev_offset+max_data);
         }
         else {
-            mca_common_ompio_decode_datatype (fh,
-                                              datatype,
-                                              count,
-                                              buf,
-                                              &max_data,
+            mca_common_ompio_decode_datatype (fh, datatype, count,
+                                              buf, &max_data,
                                               fh->f_mem_convertor,
-                                              &decoded_iov,
-                                              &iov_count);
-        }
-
-        if ( 0 < max_data && 0 == fh->f_iov_count  ) {
-            ompio_req->req_ompi.req_status.MPI_ERROR = OMPI_SUCCESS;
-            ompio_req->req_ompi.req_status._ucount = 0;
-            ompi_request_complete (&ompio_req->req_ompi, false);
-            *request = (ompi_request_t *) ompio_req;
-            if (NULL != decoded_iov) {
-                free (decoded_iov);
-                decoded_iov = NULL;
-            }
+                                              &decoded_iov, &iov_count);
+
+            /**
+             * Non blocking operations have to occur in a single cycle
+             * If the f_io_array is too long, the fbtl will chunk it up
+             * internally.
+             */
+            mca_common_ompio_build_io_array (&(fh->f_fview), 0, 1, max_data, max_data,
+					     iov_count, decoded_iov,
+					     &i, &total_bytes_written, &spc,
+					     &fh->f_io_array, &fh->f_num_of_io_entries);
 
-            return OMPI_SUCCESS;
-        }
-
-        j = fh->f_index_in_file_view;
-
-        /* Non blocking operations have to occur in a single cycle */
-        mca_common_ompio_build_io_array ( fh,
-                                          0,         // index of current cycle iteration
-                                          1,         // number of cycles
-                                          max_data,  // setting bytes_per_cycle to max_data
-                                          max_data,
-                                          iov_count,
-                                          decoded_iov,
-                                          &i,
-                                          &j,
-                                          &total_bytes_written, 
-                                          &spc,
-                                          &fh->f_io_array,
-                                          &fh->f_num_of_io_entries);
-        
-        if (fh->f_num_of_io_entries) {
             fh->f_fbtl->fbtl_ipwritev (fh, (ompi_request_t *) ompio_req);
         }
-        
-        mca_common_ompio_register_progress ();
-
-        fh->f_num_of_io_entries = 0;
-        if (NULL != fh->f_io_array) {
-            free (fh->f_io_array);
-            fh->f_io_array = NULL;
-        }
-        if (NULL != decoded_iov) {
-            free (decoded_iov);
-            decoded_iov = NULL;
-        }
-    }
-    else {
+    } else {
         // This fbtl does not support non-blocking write operations
         ompi_status_public_t status;
         ret = mca_common_ompio_file_write(fh,buf,count,datatype, &status);
@@ -343,14 +464,85 @@ int mca_common_ompio_file_iwrite (ompio_file_t *fh,
         ompi_request_complete (&ompio_req->req_ompi, false);
     }
 
+    fh->f_num_of_io_entries = 0;
+    free (fh->f_io_array);
+    fh->f_io_array = NULL;
+    free (decoded_iov);
+
     *request = (ompi_request_t *) ompio_req;
     return ret;
 }
 
+/*
+** This routine is invoked from the fcoll component.
+** It is only used if the temporary buffer is a gpu buffer,
+** and the fbtl supports the ipwritev operation.
+**
+** The io-array has already been generated in fcoll/xxx/file_write_all,
+** and we use the pre-computed offsets to created a pseudo fview.
+** The position of the file pointer is updated in the fcoll
+** component, not here.
+*/
+
+int mca_common_ompio_file_iwrite_pregen (ompio_file_t *fh,
+					 ompi_request_t *request)
+{
+    uint32_t i;
+    size_t max_data;
+    size_t pipeline_buf_size;
+    mca_ompio_request_t *ompio_req = (mca_ompio_request_t *) request;
+
+    if (NULL == fh->f_fbtl->fbtl_ipwritev) {
+	return MPI_ERR_INTERN;
+    }
+
+    max_data = fh->f_io_array[0].length;
+    pipeline_buf_size = OMPIO_MCA_GET(fh, pipeline_buffer_size);
+
+    mca_common_ompio_register_progress ();
+
+    OMPIO_PREPARE_BUF (fh, fh->f_io_array[0].memory_address, max_data, MPI_BYTE,
+		       ompio_req->req_tbuf, &ompio_req->req_convertor, max_data,
+		       pipeline_buf_size, NULL, i);
+
+    ompio_req->req_num_subreqs = ceil((double)max_data/pipeline_buf_size);
+    ompio_req->req_size        = pipeline_buf_size;
+    ompio_req->req_max_data    = max_data;
+    ompio_req->req_post_next_subreq = mca_common_ompio_post_next_write_subreq;
+    ompio_req->req_fh          = fh;
+    ompio_req->req_ompi.req_status.MPI_ERROR = MPI_SUCCESS;
+
+    ompio_req->req_fview = (struct ompio_fview_t *) calloc(1, sizeof(struct ompio_fview_t));
+    if (NULL == ompio_req->req_fview) {
+	opal_output(1, "common_ompio: error allocating memory\n");
+	return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+   ompio_req->req_fview->f_decoded_iov = (struct iovec*) malloc ( fh->f_num_of_io_entries *
+								  sizeof(struct iovec));
+    if (NULL == ompio_req->req_fview->f_decoded_iov) {
+        opal_output(1, "common_ompio_file_iwrite_pregen: could not allocate memory\n");
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    ompio_req->req_fview->f_iov_count = fh->f_num_of_io_entries;
+    for (i=0; i < ompio_req->req_fview->f_iov_count; i++) {
+        ompio_req->req_fview->f_decoded_iov[i].iov_base = fh->f_io_array[i].offset;
+        ompio_req->req_fview->f_decoded_iov[i].iov_len  = fh->f_io_array[i].length ;
+    }
+
+    fh->f_num_of_io_entries = 0;
+    free (fh->f_io_array);
+    fh->f_io_array = NULL;
+
+    mca_common_ompio_post_next_write_subreq(ompio_req, 0);
+    return OMPI_SUCCESS;
+}
+
 int mca_common_ompio_file_iwrite_at (ompio_file_t *fh,
 				   OMPI_MPI_OFFSET_TYPE offset,
 				   const void *buf,
-				   int count,
+				   size_t count,
 				   struct ompi_datatype_t *datatype,
 				   ompi_request_t **request)
 {
@@ -382,7 +574,7 @@ int mca_common_ompio_file_iwrite_at (ompio_file_t *fh,
 /******************************************************************/
 int mca_common_ompio_file_write_all (ompio_file_t *fh,
                                      const void *buf,
-                                     int count,
+                                     size_t count,
                                      struct ompi_datatype_t *datatype,
                                      ompi_status_public_t *status)
 {
@@ -397,7 +589,7 @@ int mca_common_ompio_file_write_all (ompio_file_t *fh,
            
            If the individual fcoll component is used: there are no aggregators 
            in that concept. However, since they call common_ompio_file_write, 
-           CUDA buffers are handled by that routine.
+           device buffers are handled by that routine.
 
            Thus, we only check for
            1. Datarepresentation is anything other than 'native' and
@@ -407,25 +599,22 @@ int mca_common_ompio_file_write_all (ompio_file_t *fh,
         size_t pos=0, max_data=0;
         char *tbuf=NULL;
         opal_convertor_t convertor;
-        struct iovec *decoded_iov = NULL;
+        struct iovec decoded_iov;
         uint32_t iov_count = 0;
         
-        OMPIO_PREPARE_BUF(fh,buf,count,datatype,tbuf,&convertor,max_data,decoded_iov,iov_count);     
-        opal_convertor_pack (&convertor, decoded_iov, &iov_count, &pos );
-        opal_convertor_cleanup ( &convertor);
+        OMPIO_PREPARE_BUF (fh, buf, count, datatype, tbuf, &convertor,
+			   max_data, 0, &decoded_iov, iov_count);
+        opal_convertor_pack (&convertor, &decoded_iov, &iov_count, &pos );
+        opal_convertor_cleanup (&convertor);
 
         ret = fh->f_fcoll->fcoll_file_write_all (fh,
-                                                 decoded_iov->iov_base,
-                                                 decoded_iov->iov_len,
+                                                 decoded_iov.iov_base,
+                                                 decoded_iov.iov_len,
                                                  MPI_BYTE,
                                                  status);
 
 
-        mca_common_ompio_release_buf (fh, decoded_iov->iov_base);
-        if (NULL != decoded_iov) {
-            free (decoded_iov);
-            decoded_iov = NULL;
-        }
+        mca_common_ompio_release_buf (fh, decoded_iov.iov_base);
     }
     else {
         ret = fh->f_fcoll->fcoll_file_write_all (fh,
@@ -440,7 +629,7 @@ int mca_common_ompio_file_write_all (ompio_file_t *fh,
 int mca_common_ompio_file_write_at_all (ompio_file_t *fh,
 				      OMPI_MPI_OFFSET_TYPE offset,
 				      const void *buf,
-				      int count,
+				      size_t count,
 				      struct ompi_datatype_t *datatype,
 				      ompi_status_public_t *status)
 {
@@ -461,7 +650,7 @@ int mca_common_ompio_file_write_at_all (ompio_file_t *fh,
 
 int mca_common_ompio_file_iwrite_all (ompio_file_t *fp,
                                       const void *buf,
-                                      int count,
+                                      size_t count,
                                       struct ompi_datatype_t *datatype,
                                       ompi_request_t **request)
 {
@@ -488,7 +677,7 @@ int mca_common_ompio_file_iwrite_all (ompio_file_t *fp,
 int mca_common_ompio_file_iwrite_at_all (ompio_file_t *fp,
 				       OMPI_MPI_OFFSET_TYPE offset,
 				       const void *buf,
-				       int count,
+				       size_t count,
 				       struct ompi_datatype_t *datatype,
 				       ompi_request_t **request)
 {
@@ -510,9 +699,9 @@ int mca_common_ompio_file_iwrite_at_all (ompio_file_t *fp,
 /* Helper function used by both read and write operations     */
 /**************************************************************/
 
-int mca_common_ompio_build_io_array ( ompio_file_t *fh, int index, int cycles,
+int mca_common_ompio_build_io_array ( ompio_fview_t *fview, int index, int cycles,
                                       size_t bytes_per_cycle, size_t  max_data, uint32_t iov_count,
-                                      struct iovec *decoded_iov, int *ii, int *jj, size_t *tbw, 
+                                      struct iovec *decoded_iov, int *ii, size_t *tbw,
                                       size_t *spc, mca_common_ompio_io_array_t **io_array,
                                       int *num_io_entries)
 {
@@ -526,11 +715,11 @@ int mca_common_ompio_build_io_array ( ompio_file_t *fh, int index, int cycles,
     size_t sum_previous_length = 0;
     int k = 0; /* index into the io_array */
     int i = *ii;
-    int j = *jj;
+    int j = fview->f_index_in_file_view;
     mca_common_ompio_io_array_t *f_io_array=NULL;
     int f_num_io_entries=0;
     
-    sum_previous_length = fh->f_position_in_file_view;
+    sum_previous_length = fview->f_position_in_file_view;
 
     if ((index == cycles-1) && (max_data % bytes_per_cycle)) {
 	bytes_to_write_in_cycle = max_data % bytes_per_cycle;
@@ -579,60 +768,45 @@ int mca_common_ompio_build_io_array ( ompio_file_t *fh, int index, int cycles,
 		(total_bytes_written - sum_previous_counts);
 	}
 
-	if (! (fh->f_flags & OMPIO_CONTIGUOUS_FVIEW)) {
-	    if (fh->f_decoded_iov[j].iov_len -
-		(fh->f_total_bytes - sum_previous_length) <= 0) {
-		sum_previous_length += fh->f_decoded_iov[j].iov_len;
+	if (! (fview->f_flags & OMPIO_CONTIGUOUS_FVIEW)) {
+	    if (fview->f_decoded_iov[j].iov_len -
+		(fview->f_total_bytes - sum_previous_length) <= 0) {
+		sum_previous_length += fview->f_decoded_iov[j].iov_len;
 		j = j + 1;
-		if (j == (int)fh->f_iov_count) {
+		if (j == (int)fview->f_iov_count) {
 		    j = 0;
 		    sum_previous_length = 0;
-		    fh->f_offset += fh->f_view_extent;
-		    fh->f_position_in_file_view = sum_previous_length;
-		    fh->f_index_in_file_view = j;
-		    fh->f_total_bytes = 0;
+		    fview->f_offset += fview->f_view_extent;
+		    fview->f_position_in_file_view = sum_previous_length;
+		    fview->f_index_in_file_view = j;
+		    fview->f_total_bytes = 0;
 		}
 	    }
 	}
 
-	disp = (ptrdiff_t)fh->f_decoded_iov[j].iov_base +
-	    (fh->f_total_bytes - sum_previous_length);
-	f_io_array[k].offset = (IOVBASE_TYPE *)(intptr_t)(disp + fh->f_offset);
+	disp = (ptrdiff_t)fview->f_decoded_iov[j].iov_base +
+	    (fview->f_total_bytes - sum_previous_length);
+	f_io_array[k].offset = (IOVBASE_TYPE *)(intptr_t)(disp + fview->f_offset);
 
-	if (! (fh->f_flags & OMPIO_CONTIGUOUS_FVIEW)) {
-	    if (fh->f_decoded_iov[j].iov_len -
-		(fh->f_total_bytes - sum_previous_length)
+	if (! (fview->f_flags & OMPIO_CONTIGUOUS_FVIEW)) {
+	    if (fview->f_decoded_iov[j].iov_len -
+		(fview->f_total_bytes - sum_previous_length)
 		< f_io_array[k].length) {
-		f_io_array[k].length = fh->f_decoded_iov[j].iov_len -
-		    (fh->f_total_bytes - sum_previous_length);
+		f_io_array[k].length = fview->f_decoded_iov[j].iov_len -
+		    (fview->f_total_bytes - sum_previous_length);
 	    }
 	}
 
 	total_bytes_written += f_io_array[k].length;
-	fh->f_total_bytes += f_io_array[k].length;
+	fview->f_total_bytes += f_io_array[k].length;
 	bytes_to_write_in_cycle -= f_io_array[k].length;
 	k = k + 1;
     }
-    fh->f_position_in_file_view = sum_previous_length;
-    fh->f_index_in_file_view = j;
+    fview->f_position_in_file_view = sum_previous_length;
+    fview->f_index_in_file_view = j;
     f_num_io_entries = k;
 
-#if 0
-    if (fh->f_rank == 0) {
-	int d;
-	printf("*************************** %d\n", f_num_io_entries);
-
-	for (d=0 ; d<f_num_of_entries ; d++) {
-	    printf(" ADDRESS: %p  OFFSET: %p   LENGTH: %d prev_count=%ld prev_length=%ld\n",
-		   f_io_array[d].memory_address,
-		   f_io_array[d].offset,
-		   f_io_array[d].length, 
-                   sum_previous_counts, sum_previous_length);
-	}
-    }
-#endif
     *ii = i;
-    *jj = j;
     *tbw = total_bytes_written;
     *spc = sum_previous_counts;
     *io_array = f_io_array;
diff --git a/ompi/mca/common/ompio/common_ompio_request.c b/ompi/mca/common/ompio/common_ompio_request.c
index f03da7abdd1..b53b067b8d0 100644
--- a/ompi/mca/common/ompio/common_ompio_request.c
+++ b/ompi/mca/common/ompio/common_ompio_request.c
@@ -11,6 +11,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2008-2019 University of Houston. All rights reserved.
+ * Copyright (c) 2022-2023 Advanced Micro Devices, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -18,6 +19,8 @@
  * $HEADER$
  */
 
+#include "opal_config.h"
+#include "opal/class/opal_list.h"
 #include "common_ompio_request.h"
 #include "common_ompio_buffer.h"
 
@@ -30,7 +33,7 @@ bool mca_common_ompio_progress_is_registered=false;
  */
 opal_list_t mca_common_ompio_pending_requests = {{0}};
 
-
+static opal_mutex_t mca_common_ompio_mutex = OPAL_MUTEX_STATIC_INIT;
 
 static int mca_common_ompio_request_free ( struct ompi_request_t **req)
 {
@@ -69,19 +72,28 @@ OBJ_CLASS_INSTANCE(mca_ompio_request_t, ompi_request_t,
 void mca_common_ompio_request_construct(mca_ompio_request_t* req)
 {
     OMPI_REQUEST_INIT (&(req->req_ompi), false );
-    req->req_ompi.req_free   = mca_common_ompio_request_free;
-    req->req_ompi.req_cancel = mca_common_ompio_request_cancel;
-    req->req_ompi.req_type   = OMPI_REQUEST_IO;
-    req->req_data            = NULL;
-    req->req_tbuf            = NULL;
-    req->req_size            = 0;
-    req->req_progress_fn     = NULL;
-    req->req_free_fn         = NULL;
+    req->req_ompi.req_free     = mca_common_ompio_request_free;
+    req->req_ompi.req_cancel   = mca_common_ompio_request_cancel;
+    req->req_ompi.req_type     = OMPI_REQUEST_IO;
+    req->req_data              = NULL;
+    req->req_tbuf              = NULL;
+    req->req_size              = 0;
+    req->req_max_data          = 0;
+    req->req_progress_fn       = NULL;
+    req->req_free_fn           = NULL;
+    req->req_parent            = NULL;
+    req->req_post_next_subreq  = NULL;
+    req->req_num_subreqs       = 0;
+    req->req_subreqs_completed = 0;
+    req->req_fh                = NULL;
+    req->req_fview             = NULL;
+    req->req_post_followup     = false;
 
     OBJ_CONSTRUCT(&req->req_item, opal_list_item_t);
     opal_list_append (&mca_common_ompio_pending_requests, &req->req_item);
     return;
 }
+
 void mca_common_ompio_request_destruct(mca_ompio_request_t* req)
 {
     OMPI_REQUEST_FINI ( &(req->req_ompi));
@@ -89,7 +101,10 @@ void mca_common_ompio_request_destruct(mca_ompio_request_t* req)
     if ( NULL != req->req_data ) {
         free (req->req_data);
     }
-
+    if (NULL != req->req_fview) {
+	free (req->req_fview->f_decoded_iov);
+	free (req->req_fview);
+    }
     return;
 }
 
@@ -103,10 +118,19 @@ void mca_common_ompio_request_init ( void )
 void mca_common_ompio_request_fini ( void ) 
 {
     /* Destroy the list of pending requests */
-    /* JMS: Good opprotunity here to list out all the IO requests that
+    /* JMS: Good opportunity here to list out all the IO requests that
        were not destroyed / completed upon MPI_FINALIZE */
 
     OBJ_DESTRUCT(&mca_common_ompio_pending_requests);
+    if (mca_common_ompio_progress_is_registered) {
+        OPAL_THREAD_LOCK (&mca_common_ompio_mutex);
+        if (mca_common_ompio_progress_is_registered) {
+            opal_progress_unregister(mca_common_ompio_progress);
+            mca_common_ompio_progress_is_registered=false;
+        }
+        OPAL_THREAD_UNLOCK (&mca_common_ompio_mutex);
+    }
+
     return;
 }
 
@@ -125,33 +149,105 @@ void mca_common_ompio_request_alloc ( mca_ompio_request_t **req, mca_ompio_reque
 
 void mca_common_ompio_register_progress ( void ) 
 {
-    if ( false == mca_common_ompio_progress_is_registered) {
+    if (false == mca_common_ompio_progress_is_registered) {
+        OPAL_THREAD_LOCK (&mca_common_ompio_mutex);
+        if (mca_common_ompio_progress_is_registered) {
+            OPAL_THREAD_UNLOCK (&mca_common_ompio_mutex);
+            return;
+        }
         opal_progress_register (mca_common_ompio_progress);
         mca_common_ompio_progress_is_registered=true;
+        OPAL_THREAD_UNLOCK (&mca_common_ompio_mutex);
     }
     return;
 }
+
 int mca_common_ompio_progress ( void )
 {
     mca_ompio_request_t *req=NULL;
     opal_list_item_t *litem=NULL;
+    opal_list_item_t *lnext=NULL;
     int completed=0;
 
-    OPAL_LIST_FOREACH(litem, &mca_common_ompio_pending_requests, opal_list_item_t) {
-        req = GET_OMPIO_REQ_FROM_ITEM(litem);
-        if( REQUEST_COMPLETE(&req->req_ompi) ) {
-            continue;
-        }
-        if ( NULL != req->req_progress_fn ) {
-            if ( req->req_progress_fn(req) ) {
-                completed++;
-                ompi_request_complete (&req->req_ompi, true);
-                /* The fbtl progress function is expected to set the
-                 * status elements
+    /* The mca_common_ompio_mutex is used to
+    ** avoid multiple progress threads potentially interfering
+    ** with each other
+    */
+    if (!OPAL_THREAD_TRYLOCK(&mca_common_ompio_mutex)) {
+        OPAL_LIST_FOREACH(litem, &mca_common_ompio_pending_requests, opal_list_item_t) {
+            req = GET_OMPIO_REQ_FROM_ITEM(litem);
+            if (REQUEST_COMPLETE(&req->req_ompi) ) {
+                continue;
+            }
+            if (NULL != req->req_progress_fn) {
+                if (req->req_progress_fn(req)) {
+                    /**
+                     * To support pipelined read/write operations, a user level request
+                     * can contain multiple internal requests. These sub-requests
+                     * contain a pointer to the parent request.
+                     */
+                    mca_ompio_request_t *parent = req->req_parent;
+                    if (NULL != parent) {
+                        /* This is a subrequest */
+                        if (OMPI_SUCCESS != req->req_ompi.req_status.MPI_ERROR) {
+                            parent->req_ompi.req_status.MPI_ERROR = req->req_ompi.req_status.MPI_ERROR;
+                            ompi_request_complete (&parent->req_ompi, true);
+                            continue;
+                        }
+                        parent->req_subreqs_completed++;
+                        parent->req_ompi.req_status._ucount += req->req_ompi.req_status._ucount;
+                        req->req_post_followup = true;
+                    } else {
+                        /* This is a request without subrequests */
+                        completed++;
+                        ompi_request_complete (&req->req_ompi, true);
+                    }
+                    /* The fbtl progress function is expected to set the
+                     * status elements
+                     */
+                }
+            } else {
+                /* This is a request without a lower level progress function, .e.g
+                 * a parent request
                  */
+                if (req->req_num_subreqs == req->req_subreqs_completed) {
+                    completed++;
+                    ompi_request_complete (&req->req_ompi, true);
+                }
             }
         }
 
+        /**
+         * Splitting the ompio progress loop is necessary to avoid that a pending operation
+         * consisting of multiple subrequests is executed in a single invokation of the progress
+         * function.
+         *
+         * Otherwise it can happen that the next subrequest is posted, which ends up at the tail
+         * of the ompio_pending_requests_list, and would be processed in the same loop execution;
+         * which then posts the next subrequest, which is also processed potentially right away
+         * etc. This would make the ompio_progress function block for a long time, and prevent
+         * overlapping operations.
+         *
+         * Splitting the loop into two parts, one checking for completion and one posting
+         * the next subrequest if necessary avoids the problem.
+         */
+        OPAL_LIST_FOREACH_SAFE(litem, lnext, &mca_common_ompio_pending_requests, opal_list_item_t) {
+            req = GET_OMPIO_REQ_FROM_ITEM(litem);
+            if (true == req->req_post_followup) {
+                /* This lock is used to serialize access to the
+                ** file across multiple threads.
+                */
+                if (OPAL_THREAD_TRYLOCK(&req->req_fh->f_fh->f_lock)) {
+                    continue;
+                }
+                mca_ompio_request_t *parent = req->req_parent;
+                parent->req_post_next_subreq(parent, parent->req_subreqs_completed);
+                OPAL_THREAD_UNLOCK(&req->req_fh->f_fh->f_lock);
+                ompi_request_complete (&req->req_ompi, false);
+                ompi_request_free ((ompi_request_t**)&req);
+            }
+        }
+        OPAL_THREAD_UNLOCK(&mca_common_ompio_mutex);
     }
 
     return completed;
diff --git a/ompi/mca/common/ompio/common_ompio_request.h b/ompi/mca/common/ompio/common_ompio_request.h
index 18083862df9..e6d90e09276 100644
--- a/ompi/mca/common/ompio/common_ompio_request.h
+++ b/ompi/mca/common/ompio/common_ompio_request.h
@@ -13,6 +13,7 @@
  * Copyright (c) 2008-2019 University of Houston. All rights reserved.
  * Copyright (c) 2018      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      Advanced Micro Devices, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -43,6 +44,8 @@ typedef enum {
     MCA_OMPIO_REQUEST_READ_ALL,
 } mca_ompio_request_type_t;
 
+struct mca_ompio_request_t;
+typedef void(*mca_ompio_post_next_subreq_t)(struct mca_ompio_request_t *req, int val);
 
 /**
  * Main structure for OMPIO requests
@@ -54,9 +57,17 @@ struct mca_ompio_request_t {
     opal_list_item_t                               req_item;
     void                                          *req_tbuf;
     size_t                                         req_size;
+    size_t                                     req_max_data;
     opal_convertor_t                          req_convertor;
     mca_fbtl_base_module_progress_fn_t      req_progress_fn;
     mca_fbtl_base_module_request_free_fn_t      req_free_fn;
+    mca_ompio_post_next_subreq_t       req_post_next_subreq;
+    struct mca_ompio_request_t                  *req_parent;
+    int                                     req_num_subreqs;
+    int                               req_subreqs_completed;
+    ompio_file_t                                    *req_fh;
+    ompio_fview_t                                *req_fview;
+    bool                                  req_post_followup;
 };
 typedef struct mca_ompio_request_t mca_ompio_request_t;
 OBJ_CLASS_DECLARATION(mca_ompio_request_t);
diff --git a/ompi/mca/fbtl/base/Makefile.am b/ompi/mca/fbtl/base/Makefile.am
index 7758c02f745..3c8ca757290 100644
--- a/ompi/mca/fbtl/base/Makefile.am
+++ b/ompi/mca/fbtl/base/Makefile.am
@@ -22,6 +22,10 @@ headers += \
 
 libmca_fbtl_la_SOURCES += \
         base/fbtl_base_frame.c \
-        base/fbtl_base_file_select.c \
-        base/fbtl_base_file_unselect.c \
         base/fbtl_base_find_available.c
+
+if OMPI_OMPIO_SUPPORT
+libmca_fbtl_la_SOURCES += \
+        base/fbtl_base_file_select.c \
+        base/fbtl_base_file_unselect.c
+endif
diff --git a/ompi/mca/fbtl/base/fbtl_base_file_select.c b/ompi/mca/fbtl/base/fbtl_base_file_select.c
index 8fa5f7d31bf..f8ddc4c3204 100644
--- a/ompi/mca/fbtl/base/fbtl_base_file_select.c
+++ b/ompi/mca/fbtl/base/fbtl_base_file_select.c
@@ -49,7 +49,7 @@ static OBJ_CLASS_INSTANCE(queried_module_t, opal_list_item_t, NULL, NULL);
 /*
  * Only one fbtl module can be attached to each file.
  *
- * This module calls the query funtion on all the components that were
+ * This module calls the query function on all the components that were
  * detected by fbtl_base_open. This function is called on a
  * per-file basis. This function has the following function.
  *
@@ -218,14 +218,14 @@ int mca_fbtl_base_file_select (struct ompio_file_t *file,
             * module of this component.
             *
             * ANJU: a component might not have all the functions
-            * defined.  Whereever a function pointer is null in the
+            * defined.  Wherever a function pointer is null in the
             * module structure we need to fill it in with the base
             * structure function pointers. This is yet to be done
             */
 
             /*
-             * We don return here coz we still need to go through and
-             * elease the other objects
+             * We don't return here coz we still need to go through and
+             * release the other objects
              */
 
             /*fill_null_pointers (om->om_module);*/
@@ -234,7 +234,7 @@ int mca_fbtl_base_file_select (struct ompio_file_t *file,
             file->f_fbtl_component = (mca_base_component_t *)best_component;
          } else {
             /*
-             * this is not the "choosen one", finalize
+             * this is not the "chosen one", finalize
              */
              if (NULL != om->om_component->fbtlm_file_unquery) {
                 /* unquery the component only if they have some clean
diff --git a/ompi/mca/fbtl/ime/fbtl_ime.c b/ompi/mca/fbtl/ime/fbtl_ime.c
index da6fb2b1250..1aab8f44620 100644
--- a/ompi/mca/fbtl/ime/fbtl_ime.c
+++ b/ompi/mca/fbtl/ime/fbtl_ime.c
@@ -20,7 +20,7 @@
  * *******************************************************************
  */
 static mca_fbtl_base_module_1_0_0_t ime =  {
-    mca_fbtl_ime_module_init,     /* initalise after being selected */
+    mca_fbtl_ime_module_init,     /* initialise after being selected */
     mca_fbtl_ime_module_finalize, /* close a module on a communicator */
     mca_fbtl_ime_preadv,          /* blocking read */
     mca_fbtl_ime_ipreadv,         /* non-blocking read*/
@@ -99,7 +99,7 @@ bool mca_fbtl_ime_progress ( mca_ompio_request_t *req)
             data->aio_req_status[i] = FBTL_IME_REQ_CLOSED;
         }
         else if ( data->aio_req_status[i] == FBTL_IME_REQ_ERROR ) {
-            /* an error occured. */
+            /* an error occurred. */
             data->aio_open_reqs--;
             lcount++;
             data->aio_req_fail_count++;
diff --git a/ompi/mca/fbtl/ime/fbtl_ime.h b/ompi/mca/fbtl/ime/fbtl_ime.h
index fe0a555a58b..5c98991f73a 100644
--- a/ompi/mca/fbtl/ime/fbtl_ime.h
+++ b/ompi/mca/fbtl/ime/fbtl_ime.h
@@ -44,7 +44,7 @@ int mca_fbtl_ime_component_file_unquery (ompio_file_t *file);
 int mca_fbtl_ime_module_init (ompio_file_t *file);
 int mca_fbtl_ime_module_finalize (ompio_file_t *file);
 
-OMPI_MODULE_DECLSPEC extern mca_fbtl_base_component_2_0_0_t mca_fbtl_ime_component;
+OMPI_DECLSPEC extern mca_fbtl_base_component_2_0_0_t mca_fbtl_ime_component;
 /*
  * ******************************************************************
  * ********* functions which are implemented in this module *********
diff --git a/ompi/mca/fbtl/ime/fbtl_ime_blocking_op.c b/ompi/mca/fbtl/ime/fbtl_ime_blocking_op.c
index 571a31773ad..c2ea5c292d2 100644
--- a/ompi/mca/fbtl/ime/fbtl_ime_blocking_op.c
+++ b/ompi/mca/fbtl/ime/fbtl_ime_blocking_op.c
@@ -74,7 +74,7 @@ static ssize_t mca_fbtl_ime_blocking_op(ompio_file_t *fh, int io_op)
            - OR the next iovec is not "contiguous"
            - OR we exceeded the advised number of iovecs for IME
            Then: pwritev/preadv shall be called,
-                 and the iovec array resetted */
+                 and the iovec array reset */
         if (i+1 == fh->f_num_of_io_entries ||
             ((OMPI_MPI_OFFSET_TYPE)(intptr_t)fh->f_io_array[i].offset +
              (ptrdiff_t)fh->f_io_array[i].length) !=
diff --git a/ompi/mca/fbtl/posix/Makefile.am b/ompi/mca/fbtl/posix/Makefile.am
index a7b0624d3ec..1ce19cb09b7 100644
--- a/ompi/mca/fbtl/posix/Makefile.am
+++ b/ompi/mca/fbtl/posix/Makefile.am
@@ -34,7 +34,8 @@ mcacomponentdir = $(ompilibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_fbtl_posix_la_SOURCES = $(sources)
 mca_fbtl_posix_la_LDFLAGS = -module -avoid-version
-mca_fbtl_posix_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la
+mca_fbtl_posix_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
+    $(OMPI_TOP_BUILDDIR)/ompi/mca/common/ompio/libmca_common_ompio.la
 
 noinst_LTLIBRARIES = $(component_noinst)
 libmca_fbtl_posix_la_SOURCES = $(sources)
diff --git a/ompi/mca/fbtl/posix/fbtl_posix.c b/ompi/mca/fbtl/posix/fbtl_posix.c
index e776f6da323..111bcff840b 100644
--- a/ompi/mca/fbtl/posix/fbtl_posix.c
+++ b/ompi/mca/fbtl/posix/fbtl_posix.c
@@ -13,6 +13,7 @@
  * Copyright (c) 2018      Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2018      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -35,7 +36,7 @@
 #include <aio.h>
 #endif
 
-int fbtl_posix_max_aio_active_reqs=2048;
+int ompi_fbtl_posix_max_prd_active_reqs=2048;
 
 #include "ompi/mca/fbtl/fbtl.h"
 #include "ompi/mca/fbtl/posix/fbtl_posix.h"
@@ -46,7 +47,7 @@ int fbtl_posix_max_aio_active_reqs=2048;
  * *******************************************************************
  */
 static mca_fbtl_base_module_1_0_0_t posix =  {
-    mca_fbtl_posix_module_init,     /* initalise after being selected */
+    mca_fbtl_posix_module_init,     /* initialise after being selected */
     mca_fbtl_posix_module_finalize, /* close a module on a communicator */
     mca_fbtl_posix_preadv,          /* blocking read */
 #if defined (FBTL_POSIX_HAVE_AIO)
@@ -105,7 +106,7 @@ int mca_fbtl_posix_module_init (ompio_file_t *file) {
 #if defined (FBTL_POSIX_HAVE_AIO)
     long val = sysconf(_SC_AIO_MAX);
     if ( -1 != val ) {
-	fbtl_posix_max_aio_active_reqs = (int)val;
+        ompi_fbtl_posix_max_prd_active_reqs = (int)val;
     }
 #endif
     return OMPI_SUCCESS;
@@ -124,147 +125,148 @@ bool mca_fbtl_posix_progress ( mca_ompio_request_t *req)
     mca_fbtl_posix_request_data_t *data=(mca_fbtl_posix_request_data_t *)req->req_data;
     off_t start_offset, end_offset, total_length;
 
-    for (i=data->aio_first_active_req; i < data->aio_last_active_req; i++ ) {
-	if ( EINPROGRESS == data->aio_req_status[i] ) {
-	    data->aio_req_status[i] = aio_error ( &data->aio_reqs[i]);
-	    if ( 0 == data->aio_req_status[i]){
-		/* assuming right now that aio_return will return
-		** the number of bytes written/read and not an error code,
-		** since aio_error should have returned an error in that
-		** case and not 0 ( which means request is complete)
-		*/
-                ssize_t ret2 = aio_return (&data->aio_reqs[i]);
-		data->aio_total_len += ret2;
-                if ( data->aio_reqs[i].aio_nbytes != (size_t)ret2 ) {
+    for (i=data->prd_first_active_req; i < data->prd_last_active_req; i++ ) {
+        if ( EINPROGRESS == data->prd_aio.aio_req_status[i] ) {
+            data->prd_aio.aio_req_status[i] = aio_error ( &data->prd_aio.aio_reqs[i]);
+            if ( 0 == data->prd_aio.aio_req_status[i]){
+                /* assuming right now that aio_return will return
+                ** the number of bytes written/read and not an error code,
+                ** since aio_error should have returned an error in that
+                ** case and not 0 ( which means request is complete)
+                */
+                ssize_t ret2 = aio_return (&data->prd_aio.aio_reqs[i]);
+                data->prd_total_len += ret2;
+                if ( data->prd_aio.aio_reqs[i].aio_nbytes != (size_t)ret2 ) {
                     /* Partial completion */
-                    data->aio_reqs[i].aio_offset += ret2;
-                    data->aio_reqs[i].aio_buf    = (char*)data->aio_reqs[i].aio_buf + ret2;
-                    data->aio_reqs[i].aio_nbytes -= ret2;
-                    data->aio_reqs[i].aio_reqprio = 0;
-                    data->aio_reqs[i].aio_sigevent.sigev_notify = SIGEV_NONE;
-                    data->aio_req_status[i]        = EINPROGRESS;
-                    start_offset = data->aio_reqs[i].aio_offset;
-                    total_length = data->aio_reqs[i].aio_nbytes;
+                    data->prd_aio.aio_reqs[i].aio_offset += ret2;
+                    data->prd_aio.aio_reqs[i].aio_buf    = (char*)data->prd_aio.aio_reqs[i].aio_buf + ret2;
+                    data->prd_aio.aio_reqs[i].aio_nbytes -= ret2;
+                    data->prd_aio.aio_reqs[i].aio_reqprio = 0;
+                    data->prd_aio.aio_reqs[i].aio_sigevent.sigev_notify = SIGEV_NONE;
+                    data->prd_aio.aio_req_status[i]        = EINPROGRESS;
+                    start_offset = data->prd_aio.aio_reqs[i].aio_offset;
+                    total_length = data->prd_aio.aio_reqs[i].aio_nbytes;
                     /* release previous lock */
-                    mca_fbtl_posix_unlock ( &data->aio_lock, data->aio_fh, &data->aio_lock_counter );
+                    mca_fbtl_posix_unlock ( &data->prd_lock, data->prd_fh, &data->prd_lock_counter );
                     
-                    if ( data->aio_req_type == FBTL_POSIX_WRITE ) {
-                        ret_code = mca_fbtl_posix_lock( &data->aio_lock, data->aio_fh, F_WRLCK, start_offset, total_length,
-                                                        OMPIO_LOCK_ENTIRE_REGION, &data->aio_lock_counter );
+                    if ( data->prd_req_type == FBTL_POSIX_AIO_WRITE ) {
+                        ret_code = mca_fbtl_posix_lock( &data->prd_lock, data->prd_fh, F_WRLCK, start_offset, total_length,
+                                                        OMPIO_LOCK_ENTIRE_REGION, &data->prd_lock_counter );
                         if ( 0 < ret_code ) {
                             opal_output(1, "mca_fbtl_posix_progress: error in mca_fbtl_posix_lock() %d", ret_code);
                             /* Just in case some part of the lock actually succeeded. */
-                            mca_fbtl_posix_unlock ( &data->aio_lock, data->aio_fh, &data->aio_lock_counter );
-                            return OMPI_ERROR;
+                            mca_fbtl_posix_unlock ( &data->prd_lock, data->prd_fh, &data->prd_lock_counter );
+                            return false;
                         }
-                        if (-1 == aio_write(&data->aio_reqs[i])) {
+                        if (-1 == aio_write(&data->prd_aio.aio_reqs[i])) {
                             opal_output(1, "mca_fbtl_posix_progress: error in aio_write()");
-                            mca_fbtl_posix_unlock ( &data->aio_lock, data->aio_fh, &data->aio_lock_counter );
-                            return OMPI_ERROR;
-                        }                        
+                            mca_fbtl_posix_unlock ( &data->prd_lock, data->prd_fh, &data->prd_lock_counter );
+                            return false;
+                        }
                     }
-                    else if (  data->aio_req_type == FBTL_POSIX_READ ) {
-                        ret_code = mca_fbtl_posix_lock( &data->aio_lock, data->aio_fh, F_RDLCK, start_offset, total_length,
-                                                        OMPIO_LOCK_ENTIRE_REGION, &data->aio_lock_counter );
+                    else if (  data->prd_req_type == FBTL_POSIX_AIO_READ ) {
+                        ret_code = mca_fbtl_posix_lock( &data->prd_lock, data->prd_fh, F_RDLCK, start_offset, total_length,
+                                                        OMPIO_LOCK_ENTIRE_REGION, &data->prd_lock_counter );
                         if ( 0 < ret_code ) {
                             opal_output(1, "mca_fbtl_posix_progress: error in mca_fbtl_posix_lock() %d", ret_code);
                             /* Just in case some part of the lock actually succeeded. */
-                            mca_fbtl_posix_unlock ( &data->aio_lock, data->aio_fh, &data->aio_lock_counter );
-                            return OMPI_ERROR;
+                            mca_fbtl_posix_unlock ( &data->prd_lock, data->prd_fh, &data->prd_lock_counter );
+                            return false;
                         }
-                        if (-1 == aio_read(&data->aio_reqs[i])) {
+                        if (-1 == aio_read(&data->prd_aio.aio_reqs[i])) {
                             opal_output(1, "mca_fbtl_posix_progress: error in aio_read()");
-                            mca_fbtl_posix_unlock ( &data->aio_lock, data->aio_fh, &data->aio_lock_counter );
-                            return OMPI_ERROR;
+                            mca_fbtl_posix_unlock ( &data->prd_lock, data->prd_fh, &data->prd_lock_counter );
+                            return false;
                         }
                     }
                 }
-		else {
-                    data->aio_open_reqs--;
+                else {
+                    data->prd_open_reqs--;
                     lcount++;
                 }
-	    }
-	    else if ( EINPROGRESS == data->aio_req_status[i]){
-		/* not yet done */
-		continue;
-	    }
-	    else {
-		/* an error occured. Mark the request done, but
-		   set an error code in the status */
-		req->req_ompi.req_status.MPI_ERROR = OMPI_ERROR;
-		req->req_ompi.req_status._ucount = data->aio_total_len;
-		ret = true;
-		break;
-	    }
-	}
-	else {
-	    lcount++;
-	}
+            }
+            else if ( EINPROGRESS == data->prd_aio.aio_req_status[i]){
+                /* not yet done */
+                continue;
+            }
+            else {
+                /* an error occurred. Mark the request done, but
+                   set an error code in the status */
+                req->req_ompi.req_status.MPI_ERROR = OMPI_ERROR;
+                req->req_ompi.req_status._ucount = data->prd_total_len;
+                ret = true;
+                break;
+            }
+        }
+        else {
+            lcount++;
+        }
     }
 #if 0
-    printf("lcount=%d open_reqs=%d\n", lcount, data->aio_open_reqs );
+    printf("lcount=%d open_reqs=%d\n", lcount, data->prd_open_reqs );
 #endif
-    if ( (lcount == data->aio_req_chunks) && (0 != data->aio_open_reqs )) {
+    if ( (lcount == data->prd_req_chunks) && (0 != data->prd_open_reqs )) {
         /* release the lock of the previous operations */
-        mca_fbtl_posix_unlock ( &data->aio_lock, data->aio_fh, &data->aio_lock_counter );
+        mca_fbtl_posix_unlock ( &data->prd_lock, data->prd_fh, &data->prd_lock_counter );
         
-	/* post the next batch of operations */
-	data->aio_first_active_req = data->aio_last_active_req;
-	if ( (data->aio_req_count-data->aio_last_active_req) > data->aio_req_chunks ) {
-	    data->aio_last_active_req += data->aio_req_chunks;
-	}
-	else {
-	    data->aio_last_active_req = data->aio_req_count;
-	}
+        /* post the next batch of operations */
+        data->prd_first_active_req = data->prd_last_active_req;
+        if ( (data->prd_req_count-data->prd_last_active_req) > data->prd_req_chunks ) {
+            data->prd_last_active_req += data->prd_req_chunks;
+        }
+        else {
+            data->prd_last_active_req = data->prd_req_count;
+        }
 
-        start_offset = data->aio_reqs[data->aio_first_active_req].aio_offset;
-        end_offset   = data->aio_reqs[data->aio_last_active_req-1].aio_offset + data->aio_reqs[data->aio_last_active_req-1].aio_nbytes;
+        start_offset = data->prd_aio.aio_reqs[data->prd_first_active_req].aio_offset;
+        end_offset   = data->prd_aio.aio_reqs[data->prd_last_active_req-1].aio_offset +
+                       data->prd_aio.aio_reqs[data->prd_last_active_req-1].aio_nbytes;
         total_length = (end_offset - start_offset);
 
-        if ( FBTL_POSIX_READ == data->aio_req_type ) {
-            ret_code = mca_fbtl_posix_lock( &data->aio_lock, data->aio_fh, F_RDLCK, start_offset, total_length,
-                                            OMPIO_LOCK_ENTIRE_REGION, &data->aio_lock_counter );
+        if ( FBTL_POSIX_AIO_READ == data->prd_req_type ) {
+            ret_code = mca_fbtl_posix_lock( &data->prd_lock, data->prd_fh, F_RDLCK, start_offset, total_length,
+                                            OMPIO_LOCK_ENTIRE_REGION, &data->prd_lock_counter );
         }
-        else if ( FBTL_POSIX_WRITE == data->aio_req_type ) {
-            ret_code = mca_fbtl_posix_lock( &data->aio_lock, data->aio_fh, F_WRLCK, start_offset, total_length,
-                                            OMPIO_LOCK_ENTIRE_REGION, &data->aio_lock_counter );
+        else if ( FBTL_POSIX_AIO_WRITE == data->prd_req_type ) {
+            ret_code = mca_fbtl_posix_lock( &data->prd_lock, data->prd_fh, F_WRLCK, start_offset, total_length,
+                                            OMPIO_LOCK_ENTIRE_REGION, &data->prd_lock_counter );
         }
         if ( 0 < ret_code ) {
             opal_output(1, "mca_fbtl_posix_progress: error in mca_fbtl_posix_lock() %d", ret_code);
             /* Just in case some part of the lock actually succeeded. */
-            mca_fbtl_posix_unlock ( &data->aio_lock, data->aio_fh, &data->aio_lock_counter );
-            return OMPI_ERROR;
+            mca_fbtl_posix_unlock ( &data->prd_lock, data->prd_fh, &data->prd_lock_counter );
+            return false;
         }
         
-	for ( i=data->aio_first_active_req; i< data->aio_last_active_req; i++ ) {
-	    if ( FBTL_POSIX_READ == data->aio_req_type ) {
-		if (-1 == aio_read(&data->aio_reqs[i])) {
-		    opal_output(1, "mca_fbtl_posix_progress: error in aio_read()");
-                    mca_fbtl_posix_unlock ( &data->aio_lock, data->aio_fh, &data->aio_lock_counter );
-		    return OMPI_ERROR;
-		}
-	    }
-	    else if ( FBTL_POSIX_WRITE == data->aio_req_type ) {
-		if (-1 == aio_write(&data->aio_reqs[i])) {
-		    opal_output(1, "mca_fbtl_posix_progress: error in aio_write()");
-                    mca_fbtl_posix_unlock ( &data->aio_lock, data->aio_fh, &data->aio_lock_counter );
-		    return OMPI_ERROR;
-		}
-	    }
-	}
+        for ( i=data->prd_first_active_req; i< data->prd_last_active_req; i++ ) {
+            if ( FBTL_POSIX_AIO_READ == data->prd_req_type ) {
+                if (-1 == aio_read(&data->prd_aio.aio_reqs[i])) {
+                    opal_output(1, "mca_fbtl_posix_progress: error in aio_read()");
+                    mca_fbtl_posix_unlock ( &data->prd_lock, data->prd_fh, &data->prd_lock_counter );
+                    return false;
+                }
+            }
+            else if ( FBTL_POSIX_AIO_WRITE == data->prd_req_type ) {
+                if (-1 == aio_write(&data->prd_aio.aio_reqs[i])) {
+                    opal_output(1, "mca_fbtl_posix_progress: error in aio_write()");
+                    mca_fbtl_posix_unlock ( &data->prd_lock, data->prd_fh, &data->prd_lock_counter );
+                    return false;
+                }
+            }
+        }
 #if 0
-	printf("posting new batch: first=%d last=%d\n", data->aio_first_active_req, data->aio_last_active_req );
+        printf("posting new batch: first=%d last=%d\n", data->prd_first_active_req, data->prd_last_active_req );
 #endif
     }
 
-    if ( 0 == data->aio_open_reqs ) {
-	/* all pending operations are finished for this request */
-	req->req_ompi.req_status.MPI_ERROR = OMPI_SUCCESS;
-	req->req_ompi.req_status._ucount = data->aio_total_len;
-        mca_fbtl_posix_unlock ( &data->aio_lock, data->aio_fh, &data->aio_lock_counter );
+    if ( 0 == data->prd_open_reqs ) {
+        /* all pending operations are finished for this request */
+        req->req_ompi.req_status.MPI_ERROR = OMPI_SUCCESS;
+        req->req_ompi.req_status._ucount = data->prd_total_len;
+        mca_fbtl_posix_unlock ( &data->prd_lock, data->prd_fh, &data->prd_lock_counter );
 
-        if ( data->aio_fh->f_atomicity ) {
-            mca_fbtl_posix_unlock ( &data->aio_lock, data->aio_fh, &data->aio_lock_counter );
+        if ( data->prd_fh->f_atomicity ) {
+            mca_fbtl_posix_unlock ( &data->prd_lock, data->prd_fh, &data->prd_lock_counter );
         }
 
         ret = true;
@@ -280,14 +282,14 @@ void mca_fbtl_posix_request_free ( mca_ompio_request_t *req)
     mca_fbtl_posix_request_data_t *data=(mca_fbtl_posix_request_data_t *)req->req_data;
     if (NULL != data ) {
             
-        if ( NULL != data->aio_reqs ) {
-	    free ( data->aio_reqs);
-	}
-	if ( NULL != data->aio_req_status ) {
-	    free ( data->aio_req_status );
-	}
-	free ( data );
-	req->req_data = NULL;
+        if ( NULL != data->prd_aio.aio_reqs ) {
+            free ( data->prd_aio.aio_reqs);
+        }
+        if ( NULL != data->prd_aio.aio_req_status ) {
+            free ( data->prd_aio.aio_req_status );
+        }
+        free (data);
+        req->req_data = NULL;
     }
 #endif
   return;
diff --git a/ompi/mca/fbtl/posix/fbtl_posix.h b/ompi/mca/fbtl/posix/fbtl_posix.h
index d61e80054c8..3eae2da68e2 100644
--- a/ompi/mca/fbtl/posix/fbtl_posix.h
+++ b/ompi/mca/fbtl/posix/fbtl_posix.h
@@ -12,6 +12,7 @@
  * Copyright (c) 2008-2021 University of Houston. All rights reserved.
  * Copyright (c) 2018      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -46,9 +47,9 @@ int mca_fbtl_posix_component_file_unquery (ompio_file_t *file);
 int mca_fbtl_posix_module_init (ompio_file_t *file);
 int mca_fbtl_posix_module_finalize (ompio_file_t *file);
 
-extern int fbtl_posix_max_aio_active_reqs;
+extern int ompi_fbtl_posix_max_prd_active_reqs;
 
-OMPI_MODULE_DECLSPEC extern mca_fbtl_base_component_2_0_0_t mca_fbtl_posix_component;
+OMPI_DECLSPEC extern mca_fbtl_base_component_2_0_0_t mca_fbtl_posix_component;
 /*
  * ******************************************************************
  * ********* functions which are implemented in this module *********
@@ -71,29 +72,36 @@ int mca_fbtl_posix_lock ( struct flock *lock, ompio_file_t *fh, int op,
                           int *lock_counter);
 void  mca_fbtl_posix_unlock ( struct flock *lock, ompio_file_t *fh, int *lock_counter );
 
+/* Right now statically defined, will become a configure check */
+#define FBTL_POSIX_HAVE_AIO 1
 
 struct mca_fbtl_posix_request_data_t {
-    int            aio_req_count;       /* total number of aio reqs */
-    int            aio_open_reqs;       /* number of unfinished reqs */
-    int            aio_req_type;        /* read or write */
-    int            aio_req_chunks;      /* max. no. of aio reqs that can be posted at once*/
-    int            aio_first_active_req; /* first active posted req */
-    int            aio_last_active_req;  /* last currently active poted req */
-    struct aiocb       *aio_reqs;       /* pointer array of req structures */
-    int           *aio_req_status;       /* array of statuses */
-    ssize_t        aio_total_len;       /* total amount of data written */
-    struct flock   aio_lock;            /* lock used for certain file systems */
-    int            aio_lock_counter;    /* to keep track of no. of lock calls */
-    ompio_file_t  *aio_fh;       /* pointer back to the mca_io_ompio_fh structure */
+    int            prd_req_count;        /* total number of sub reqs */
+    int            prd_open_reqs;        /* number of unfinished reqs */
+    int            prd_req_type;         /* read or write */
+    int            prd_req_chunks;       /* max. no. of sub reqs that can be posted at once*/
+    int            prd_first_active_req; /* first active posted req */
+    int            prd_last_active_req;  /* last currently active poted req */
+    ssize_t        prd_total_len;        /* total amount of data written */
+    struct flock   prd_lock;             /* lock used for certain file systems */
+    int            prd_lock_counter;     /* to keep track of no. of lock calls */
+    ompio_file_t  *prd_fh;               /* pointer to the ompio_fh structure */
+    union {
+#if defined (FBTL_POSIX_HAVE_AIO)
+        struct {
+          struct aiocb  *aio_reqs;            /* pointer array of req structures */
+          int           *aio_req_status;      /* array of statuses */
+      } prd_aio;
+#endif
+    };
+
 };
 typedef struct mca_fbtl_posix_request_data_t mca_fbtl_posix_request_data_t;
 
-/* Right now statically defined, will become a configure check */
-#define FBTL_POSIX_HAVE_AIO 1
 
 /* define constants for AIO requests */
-#define FBTL_POSIX_READ 1
-#define FBTL_POSIX_WRITE 2
+#define FBTL_POSIX_AIO_READ   1
+#define FBTL_POSIX_AIO_WRITE  2
 
 #define OMPIO_SET_ATOMICITY_LOCK(_fh, _lock, _lock_counter, _op) {     \
      int32_t _orig_flags = _fh->f_flags;                               \
diff --git a/ompi/mca/fbtl/posix/fbtl_posix_ipreadv.c b/ompi/mca/fbtl/posix/fbtl_posix_ipreadv.c
index ba72c3b07ca..3a31d33fbe4 100644
--- a/ompi/mca/fbtl/posix/fbtl_posix_ipreadv.c
+++ b/ompi/mca/fbtl/posix/fbtl_posix_ipreadv.c
@@ -12,6 +12,7 @@
  * Copyright (c) 2008-2021 University of Houston. All rights reserved.
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -36,7 +37,7 @@
 #define MAX_ATTEMPTS 10
 
 ssize_t mca_fbtl_posix_ipreadv (ompio_file_t *fh,
-			       ompi_request_t *request)
+                               ompi_request_t *request)
 {
 #if defined (FBTL_POSIX_HAVE_AIO)
     mca_fbtl_posix_request_data_t *data;
@@ -50,83 +51,84 @@ ssize_t mca_fbtl_posix_ipreadv (ompio_file_t *fh,
         return 0;
     }
 
-    data->aio_req_count = fh->f_num_of_io_entries;
-    data->aio_open_reqs = fh->f_num_of_io_entries;
-    data->aio_req_type  = FBTL_POSIX_READ;
-    data->aio_req_chunks = fbtl_posix_max_aio_active_reqs;
-    data->aio_total_len = 0;
-    data->aio_reqs = (struct aiocb *) malloc (sizeof(struct aiocb) *
-                                              fh->f_num_of_io_entries);
-    if (NULL == data->aio_reqs) {
+    data->prd_req_count = fh->f_num_of_io_entries;
+    data->prd_open_reqs = fh->f_num_of_io_entries;
+    data->prd_req_type  = FBTL_POSIX_AIO_READ;
+    data->prd_req_chunks = ompi_fbtl_posix_max_prd_active_reqs;
+    data->prd_total_len = 0;
+    data->prd_aio.aio_reqs = (struct aiocb *) calloc (fh->f_num_of_io_entries,
+                                                      sizeof(struct aiocb));
+    if (NULL == data->prd_aio.aio_reqs) {
         opal_output(1, "mca_fbtl_posix_ipreadv: could not allocate memory\n");
         free(data);
         return 0;
     }
 
-    data->aio_req_status = (int *) malloc (sizeof(int) * fh->f_num_of_io_entries);
-    if (NULL == data->aio_req_status) {
+    data->prd_aio.aio_req_status = (int *) malloc (sizeof(int) * fh->f_num_of_io_entries);
+    if (NULL == data->prd_aio.aio_req_status) {
         opal_output(1, "mca_fbtl_posix_ipreadv: could not allocate memory\n");
-        free(data->aio_reqs);
+        free(data->prd_aio.aio_reqs);
         free(data);
         return 0;
     }
-    data->aio_lock_counter = 0;
-    data->aio_fh = fh;
+    data->prd_lock_counter = 0;
+    data->prd_fh = fh;
 
     if ( fh->f_atomicity ) {
-        OMPIO_SET_ATOMICITY_LOCK(fh, data->aio_lock, data->aio_lock_counter, F_RDLCK);
+        OMPIO_SET_ATOMICITY_LOCK(fh, data->prd_lock, data->prd_lock_counter, F_RDLCK);
     }
     
     for ( i=0; i<fh->f_num_of_io_entries; i++ ) {
-        data->aio_reqs[i].aio_offset  = (OMPI_MPI_OFFSET_TYPE)(intptr_t)
+        data->prd_aio.aio_reqs[i].aio_offset  = (OMPI_MPI_OFFSET_TYPE)(intptr_t)
             fh->f_io_array[i].offset;
-        data->aio_reqs[i].aio_buf     = fh->f_io_array[i].memory_address;
-        data->aio_reqs[i].aio_nbytes  = fh->f_io_array[i].length;
-        data->aio_reqs[i].aio_fildes  = fh->fd;
-        data->aio_reqs[i].aio_reqprio = 0;
-        data->aio_reqs[i].aio_sigevent.sigev_notify = SIGEV_NONE;
-	data->aio_req_status[i]        = EINPROGRESS;
+        data->prd_aio.aio_reqs[i].aio_buf     = fh->f_io_array[i].memory_address;
+        data->prd_aio.aio_reqs[i].aio_nbytes  = fh->f_io_array[i].length;
+        data->prd_aio.aio_reqs[i].aio_fildes  = fh->fd;
+        data->prd_aio.aio_reqs[i].aio_reqprio = 0;
+        data->prd_aio.aio_reqs[i].aio_sigevent.sigev_notify = SIGEV_NONE;
+        data->prd_aio.aio_req_status[i]        = EINPROGRESS;
     }
 
-    data->aio_first_active_req = 0;
-    if ( data->aio_req_count > data->aio_req_chunks ) {
-	data->aio_last_active_req = data->aio_req_chunks;
+    data->prd_first_active_req = 0;
+    if ( data->prd_req_count > data->prd_req_chunks ) {
+        data->prd_last_active_req = data->prd_req_chunks;
     }
     else {
-	data->aio_last_active_req = data->aio_req_count;
+        data->prd_last_active_req = data->prd_req_count;
     }
 
-    start_offset = data->aio_reqs[data->aio_first_active_req].aio_offset;
-    end_offset   = data->aio_reqs[data->aio_last_active_req-1].aio_offset + data->aio_reqs[data->aio_last_active_req-1].aio_nbytes;
+    start_offset = data->prd_aio.aio_reqs[data->prd_first_active_req].aio_offset;
+    end_offset   = data->prd_aio.aio_reqs[data->prd_last_active_req-1].aio_offset +
+                   data->prd_aio.aio_reqs[data->prd_last_active_req-1].aio_nbytes;
     total_length = (end_offset - start_offset);
-    ret = mca_fbtl_posix_lock( &data->aio_lock, data->aio_fh, F_RDLCK, start_offset, total_length,
-                               OMPIO_LOCK_ENTIRE_REGION, &data->aio_lock_counter );
+    ret = mca_fbtl_posix_lock( &data->prd_lock, data->prd_fh, F_RDLCK, start_offset, total_length,
+                               OMPIO_LOCK_ENTIRE_REGION, &data->prd_lock_counter );
     if ( 0 < ret ) {
         opal_output(1, "mca_fbtl_posix_ipreadv: error in mca_fbtl_posix_lock() error ret=%d  %s", ret, strerror(errno));
-        mca_fbtl_posix_unlock ( &data->aio_lock, data->aio_fh, &data->aio_lock_counter );            
-        free(data->aio_reqs);
-        free(data->aio_req_status);
+        mca_fbtl_posix_unlock ( &data->prd_lock, data->prd_fh, &data->prd_lock_counter);
+        free(data->prd_aio.aio_reqs);
+        free(data->prd_aio.aio_req_status);
         free(data);
         return OMPI_ERROR;
     }
 
-    for (i=0; i < data->aio_last_active_req; i++) {
+    for (i=0; i < data->prd_last_active_req; i++) {
         int counter=0;
         while ( MAX_ATTEMPTS > counter ) { 
-	    if  ( -1 != aio_read(&data->aio_reqs[i]) ) {
-	        break;
-	    }
-	    counter++;
-	    mca_common_ompio_progress();
-	}
-	if ( MAX_ATTEMPTS == counter ) {
+            if  ( -1 != aio_read(&data->prd_aio.aio_reqs[i]) ) {
+                break;
+            }
+            counter++;
+            mca_common_ompio_progress();
+        }
+        if ( MAX_ATTEMPTS == counter ) {
            opal_output(1, "mca_fbtl_posix_ipreadv: error in aio_read(): errno %d %s", errno, strerror(errno));
-	   mca_fbtl_posix_unlock ( &data->aio_lock, data->aio_fh, &data->aio_lock_counter );            
-	   free(data->aio_reqs);
-	   free(data->aio_req_status);
-	   free(data);
-	   return OMPI_ERROR;
-	}
+           mca_fbtl_posix_unlock ( &data->prd_lock, data->prd_fh, &data->prd_lock_counter);
+           free(data->prd_aio.aio_reqs);
+           free(data->prd_aio.aio_req_status);
+           free(data);
+           return OMPI_ERROR;
+        }
     }
 
     req->req_data = data;
diff --git a/ompi/mca/fbtl/posix/fbtl_posix_ipwritev.c b/ompi/mca/fbtl/posix/fbtl_posix_ipwritev.c
index f4394bf620b..272c717ed00 100644
--- a/ompi/mca/fbtl/posix/fbtl_posix_ipwritev.c
+++ b/ompi/mca/fbtl/posix/fbtl_posix_ipwritev.c
@@ -12,6 +12,7 @@
  * Copyright (c) 2008-2021 University of Houston. All rights reserved.
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -35,7 +36,7 @@
 #define MAX_ATTEMPTS 10
 
 ssize_t  mca_fbtl_posix_ipwritev (ompio_file_t *fh,
-				 ompi_request_t *request)
+                                 ompi_request_t *request)
 {
 #if defined(FBTL_POSIX_HAVE_AIO)
     mca_fbtl_posix_request_data_t *data;
@@ -49,80 +50,80 @@ ssize_t  mca_fbtl_posix_ipwritev (ompio_file_t *fh,
         return 0;
     }
 
-    data->aio_req_count = fh->f_num_of_io_entries;
-    data->aio_open_reqs = fh->f_num_of_io_entries;
-    data->aio_req_type  = FBTL_POSIX_WRITE;
-    data->aio_req_chunks = fbtl_posix_max_aio_active_reqs;
-    data->aio_total_len = 0;
-    data->aio_reqs = (struct aiocb *) malloc (sizeof(struct aiocb) *
-                                              fh->f_num_of_io_entries);
-    if (NULL == data->aio_reqs) {
+    data->prd_req_count = fh->f_num_of_io_entries;
+    data->prd_open_reqs = fh->f_num_of_io_entries;
+    data->prd_req_type  = FBTL_POSIX_AIO_WRITE;
+    data->prd_req_chunks = ompi_fbtl_posix_max_prd_active_reqs;
+    data->prd_total_len = 0;
+    data->prd_aio.aio_reqs = (struct aiocb *) calloc (fh->f_num_of_io_entries,
+                                                      sizeof(struct aiocb));
+    if (NULL == data->prd_aio.aio_reqs) {
         opal_output (1,"mca_fbtl_posix_ipwritev: could not allocate memory\n");
         free(data);
         return 0;
     }
 
-    data->aio_req_status = (int *) malloc (sizeof(int) * fh->f_num_of_io_entries);
-    if (NULL == data->aio_req_status) {
+    data->prd_aio.aio_req_status = (int *) malloc (sizeof(int) * fh->f_num_of_io_entries);
+    if (NULL == data->prd_aio.aio_req_status) {
         opal_output (1,"mca_fbtl_posix_ipwritev: could not allocate memory\n");
-        free(data->aio_reqs);
+        free(data->prd_aio.aio_reqs);
         free(data);
         return 0;
     }
-    data->aio_lock_counter = 0;
-    data->aio_fh = fh;
+    data->prd_lock_counter = 0;
+    data->prd_fh = fh;
     if ( fh->f_atomicity ) {
-        OMPIO_SET_ATOMICITY_LOCK(fh, data->aio_lock, data->aio_lock_counter, F_WRLCK);
+        OMPIO_SET_ATOMICITY_LOCK(fh, data->prd_lock, data->prd_lock_counter, F_WRLCK);
     }
-
     
     for ( i=0; i<fh->f_num_of_io_entries; i++ ) {
-        data->aio_reqs[i].aio_offset  = (OMPI_MPI_OFFSET_TYPE)(intptr_t)
+        data->prd_aio.aio_reqs[i].aio_offset  = (OMPI_MPI_OFFSET_TYPE)(intptr_t)
             fh->f_io_array[i].offset;
-        data->aio_reqs[i].aio_buf     = fh->f_io_array[i].memory_address;
-        data->aio_reqs[i].aio_nbytes  = fh->f_io_array[i].length;
-        data->aio_reqs[i].aio_fildes  = fh->fd;
-        data->aio_reqs[i].aio_reqprio = 0;
-        data->aio_reqs[i].aio_sigevent.sigev_notify = SIGEV_NONE;
-	data->aio_req_status[i]        = EINPROGRESS;
+        data->prd_aio.aio_reqs[i].aio_buf     = fh->f_io_array[i].memory_address;
+        data->prd_aio.aio_reqs[i].aio_nbytes  = fh->f_io_array[i].length;
+        data->prd_aio.aio_reqs[i].aio_fildes  = fh->fd;
+        data->prd_aio.aio_reqs[i].aio_reqprio = 0;
+        data->prd_aio.aio_reqs[i].aio_sigevent.sigev_notify = SIGEV_NONE;
+        data->prd_aio.aio_req_status[i]        = EINPROGRESS;
     }
 
-    data->aio_first_active_req = 0;
-    if ( data->aio_req_count > data->aio_req_chunks ) {
-	data->aio_last_active_req = data->aio_req_chunks;
+    data->prd_first_active_req = 0;
+    if ( data->prd_req_count > data->prd_req_chunks ) {
+        data->prd_last_active_req = data->prd_req_chunks;
     }
     else {
-	data->aio_last_active_req = data->aio_req_count;
+        data->prd_last_active_req = data->prd_req_count;
     }
     
-    start_offset = data->aio_reqs[data->aio_first_active_req].aio_offset;
-    end_offset   = data->aio_reqs[data->aio_last_active_req-1].aio_offset + data->aio_reqs[data->aio_last_active_req-1].aio_nbytes;
+    start_offset = data->prd_aio.aio_reqs[data->prd_first_active_req].aio_offset;
+    end_offset   = data->prd_aio.aio_reqs[data->prd_last_active_req-1].aio_offset +
+                   data->prd_aio.aio_reqs[data->prd_last_active_req-1].aio_nbytes;
     total_length = (end_offset - start_offset);
-    ret = mca_fbtl_posix_lock( &data->aio_lock, data->aio_fh, F_WRLCK, start_offset, total_length,
-                               OMPIO_LOCK_ENTIRE_REGION, &data->aio_lock_counter );
+    ret = mca_fbtl_posix_lock( &data->prd_lock, data->prd_fh, F_WRLCK, start_offset, total_length,
+                               OMPIO_LOCK_ENTIRE_REGION, &data->prd_lock_counter );
     if ( 0 < ret ) {
         opal_output(1, "mca_fbtl_posix_ipwritev: error in mca_fbtl_posix_lock() error ret=%d %s", ret, strerror(errno));
-        mca_fbtl_posix_unlock ( &data->aio_lock, data->aio_fh, &data->aio_lock_counter );            
-        free(data->aio_reqs);
-        free(data->aio_req_status);
+        mca_fbtl_posix_unlock ( &data->prd_lock, data->prd_fh, &data->prd_lock_counter);
+        free(data->prd_aio.aio_reqs);
+        free(data->prd_aio.aio_req_status);
         free(data);
         return OMPI_ERROR;
     }
 
-    for (i=0; i < data->aio_last_active_req; i++) {
+    for (i=0; i < data->prd_last_active_req; i++) {
         int counter=0;
-	while ( MAX_ATTEMPTS > counter ) {
-	    if (-1 != aio_write(&data->aio_reqs[i])) {
-   	        break;
-	    }
-	    counter++;
-	    mca_common_ompio_progress();
-	}
-	if ( MAX_ATTEMPTS == counter ) {
+        while ( MAX_ATTEMPTS > counter ) {
+            if (-1 != aio_write(&data->prd_aio.aio_reqs[i])) {
+                break;
+            }
+            counter++;
+            mca_common_ompio_progress();
+        }
+        if ( MAX_ATTEMPTS == counter ) {
             opal_output(1, "mca_fbtl_posix_ipwritev: error in aio_write():  %s", strerror(errno));
-            mca_fbtl_posix_unlock ( &data->aio_lock, data->aio_fh, &data->aio_lock_counter );                    
-            free(data->aio_req_status);
-            free(data->aio_reqs);
+            mca_fbtl_posix_unlock ( &data->prd_lock, data->prd_fh, &data->prd_lock_counter);
+            free(data->prd_aio.aio_req_status);
+            free(data->prd_aio.aio_reqs);
             free(data);
             return OMPI_ERROR;
         }
diff --git a/ompi/mca/fbtl/posix/fbtl_posix_preadv.c b/ompi/mca/fbtl/posix/fbtl_posix_preadv.c
index c3c09acbc95..1b66ae1331a 100644
--- a/ompi/mca/fbtl/posix/fbtl_posix_preadv.c
+++ b/ompi/mca/fbtl/posix/fbtl_posix_preadv.c
@@ -12,6 +12,8 @@
  * Copyright (c) 2008-2021 University of Houston. All rights reserved.
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
+ * Copyright (c) 2023      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -25,15 +27,20 @@
 #include "mpi.h"
 #include <unistd.h>
 #include <limits.h>
+#ifdef HAVE_SYSLIMITS_H
+#include <syslimits.h>
+#endif  /* HAVE_SYSLIMITS_H */
 #include "ompi/constants.h"
 #include "ompi/mca/fbtl/fbtl.h"
 
+#ifndef IOV_MAX
+#define IOV_MAX 1024
+#endif
 
 static ssize_t mca_fbtl_posix_preadv_datasieving (ompio_file_t *fh, struct flock *lock, int *lock_counter);
 static ssize_t mca_fbtl_posix_preadv_generic (ompio_file_t *fh, struct flock *lock, int *lock_counter);
 static ssize_t mca_fbtl_posix_preadv_single (ompio_file_t *fh, struct flock *lock, int *lock_counter);
 
-#define MAX_RETRIES 10
 
 ssize_t mca_fbtl_posix_preadv (ompio_file_t *fh )
 {
@@ -108,7 +115,6 @@ ssize_t mca_fbtl_posix_preadv_single (ompio_file_t *fh, struct flock *lock, int
         return OMPI_ERROR;
     }
     
-    int retries = 0;
     size_t len = fh->f_io_array[0].length;
     while ( total_bytes < len ) {
         ret_code = pread(fh->fd, (char*)fh->f_io_array[0].memory_address+total_bytes,
@@ -121,13 +127,7 @@ ssize_t mca_fbtl_posix_preadv_single (ompio_file_t *fh, struct flock *lock, int
         }
         if ( ret_code == 0 ) {
             // end of file
-            retries++;
-            if ( retries == MAX_RETRIES ) {
-                break;
-            }
-            else {
-                continue;
-            }
+	    break;
         }
         total_bytes += ret_code;
     }   
@@ -160,7 +160,8 @@ ssize_t mca_fbtl_posix_preadv_datasieving (ompio_file_t *fh, struct flock *lock,
         }
         
         size_t sstart = (size_t)fh->f_io_array[startindex].offset;
-        size_t slen=0;
+        size_t slen=0, maxlen=0;
+        int maxindex = startindex;
 
         for ( j = startindex; j < fh->f_num_of_io_entries; j++ ) {
             endindex = j;
@@ -169,15 +170,19 @@ ssize_t mca_fbtl_posix_preadv_datasieving (ompio_file_t *fh, struct flock *lock,
                 endindex = j-1;
                 break;
             }
+            if (slen > maxlen) {
+                maxlen   = slen;
+                maxindex = endindex;
+            }
         }
         // Need to increment the value of endindex
         // by one for the loop syntax to work correctly.
         endindex++;
         
         start = (size_t)fh->f_io_array[startindex].offset;
-        end   = (size_t)fh->f_io_array[endindex-1].offset + fh->f_io_array[endindex-1].length;
+        end   = (size_t)fh->f_io_array[maxindex].offset + fh->f_io_array[maxindex].length;
         len   = end - start;
-        
+
         if ( len > bufsize ) {
             if ( NULL != temp_buf ) {
                 free ( temp_buf);
@@ -201,7 +206,6 @@ ssize_t mca_fbtl_posix_preadv_datasieving (ompio_file_t *fh, struct flock *lock,
             return OMPI_ERROR;
         }
         size_t total_bytes = 0;
-        int retries = 0;
         
         while ( total_bytes < len ) {
             ret_code = pread (fh->fd, temp_buf+total_bytes, len-total_bytes, start+total_bytes);
@@ -213,13 +217,7 @@ ssize_t mca_fbtl_posix_preadv_datasieving (ompio_file_t *fh, struct flock *lock,
             }
             if ( ret_code == 0 ) {
                 // end of file
-                retries++;
-                if ( retries == MAX_RETRIES ) {
-                    break;
-                }
-                else {
-                    continue;
-                }
+		break;
             }
             total_bytes += ret_code;
         }
@@ -231,12 +229,12 @@ ssize_t mca_fbtl_posix_preadv_datasieving (ompio_file_t *fh, struct flock *lock,
         size_t start_offset = (size_t) fh->f_io_array[startindex].offset;
         for ( i = startindex ; i < endindex ; i++) {
             pos = (size_t) fh->f_io_array[i].offset - start_offset;
-            if ( (ssize_t) pos > ret_code ) {
+            if ( pos > total_bytes ) {
                 break;
             }
             num_bytes = fh->f_io_array[i].length;
-            if ( ((ssize_t) pos + (ssize_t)num_bytes) > ret_code ) {
-                num_bytes = ret_code - (ssize_t)pos;
+            if ( (pos + (size_t)num_bytes) > total_bytes ) {
+                num_bytes = total_bytes - (ssize_t)pos;
             }
             
             memcpy (fh->f_io_array[i].memory_address, temp_buf + pos, num_bytes);
diff --git a/ompi/mca/fbtl/posix/fbtl_posix_pwritev.c b/ompi/mca/fbtl/posix/fbtl_posix_pwritev.c
index b96bddcb894..cb20d7cc9aa 100644
--- a/ompi/mca/fbtl/posix/fbtl_posix_pwritev.c
+++ b/ompi/mca/fbtl/posix/fbtl_posix_pwritev.c
@@ -12,6 +12,7 @@
  * Copyright (c) 2008-2021 University of Houston. All rights reserved.
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2023      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -25,17 +26,21 @@
 
 #include "mpi.h"
 #include <unistd.h>
-#include <sys/uio.h>
 #include <limits.h>
+#ifdef HAVE_SYSLIMITS_H
+#include <syslimits.h>
+#endif  /* HAVE_SYSLIMITS_H */
 #include "ompi/constants.h"
 #include "ompi/mca/fbtl/fbtl.h"
 
+#ifndef IOV_MAX
+#define IOV_MAX 1024
+#endif
+
 static ssize_t mca_fbtl_posix_pwritev_datasieving (ompio_file_t *fh, struct flock *lock, int *lock_counter );
 static ssize_t mca_fbtl_posix_pwritev_generic (ompio_file_t *fh, struct flock *lock, int *lock_counter );
 static ssize_t mca_fbtl_posix_pwritev_single (ompio_file_t *fh, struct flock *lock, int *lock_counter );
 
-#define MAX_RETRIES 10
-
 ssize_t  mca_fbtl_posix_pwritev(ompio_file_t *fh )
 {
     ssize_t bytes_written=0;
@@ -192,7 +197,6 @@ ssize_t mca_fbtl_posix_pwritev_datasieving (ompio_file_t *fh, struct flock *lock
             return OMPI_ERROR;
         }
         
-        int retries=0;
         while ( total_bytes < len ) {
             ret_code = pread (fh->fd, temp_buf, len, start);
             if ( ret_code == -1 ) {
@@ -203,13 +207,7 @@ ssize_t mca_fbtl_posix_pwritev_datasieving (ompio_file_t *fh, struct flock *lock
             }
             if ( ret_code == 0 ) {
                 // end of file
-                retries++;
-                if ( retries == MAX_RETRIES ) {
-                    break;
-                }
-                else {
-                    continue;
-                }
+		break;
             }
             total_bytes += ret_code;
         }
diff --git a/ompi/mca/fbtl/pvfs2/Makefile.am b/ompi/mca/fbtl/pvfs2/Makefile.am
deleted file mode 100644
index 66582947d93..00000000000
--- a/ompi/mca/fbtl/pvfs2/Makefile.am
+++ /dev/null
@@ -1,56 +0,0 @@
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2008-2011 University of Houston. All rights reserved.
-# Copyright (c) 2017      IBM Corporation.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-# Make the output library in this directory, and name it either
-# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
-# (for static builds).
-
-if MCA_BUILD_ompi_fbtl_pvfs2_DSO
-component_noinst =
-component_install = mca_fbtl_pvfs2.la
-else
-component_noinst = libmca_fbtl_pvfs2.la
-component_install =
-endif
-
-# Source files
-
-fbtl_pvfs2_sources = \
-        fbtl_pvfs2.h \
-        fbtl_pvfs2.c \
-        fbtl_pvfs2_component.c \
-        fbtl_pvfs2_preadv.c \
-        fbtl_pvfs2_ipreadv.c \
-        fbtl_pvfs2_pwritev.c \
-        fbtl_pvfs2_ipwritev.c
-
-AM_CPPFLAGS = $(fbtl_pvfs2_CPPFLAGS)
-
-mcacomponentdir = $(ompilibdir)
-mcacomponent_LTLIBRARIES = $(component_install)
-mca_fbtl_pvfs2_la_SOURCES = $(fbtl_pvfs2_sources)
-mca_fbtl_pvfs2_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
-	$(fbtl_pvfs2_LIBS)
-mca_fbtl_pvfs2_la_LDFLAGS = -module -avoid-version $(fbtl_pvfs2_LDFLAGS)
-
-noinst_LTLIBRARIES = $(component_noinst)
-libmca_fbtl_pvfs2_la_SOURCES = $(fbtl_pvfs2_sources)
-libmca_fbtl_pvfs2_la_LIBADD = $(fbtl_pvfs2_LIBS)
-libmca_fbtl_pvfs2_la_LDFLAGS = -module -avoid-version $(fbtl_pvfs2_LDFLAGS)
diff --git a/ompi/mca/fbtl/pvfs2/configure.m4 b/ompi/mca/fbtl/pvfs2/configure.m4
deleted file mode 100644
index 6fea62d4d07..00000000000
--- a/ompi/mca/fbtl/pvfs2/configure.m4
+++ /dev/null
@@ -1,43 +0,0 @@
-# -*- shell-script -*-
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2008-2012 University of Houston. All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-
-# MCA_fbtl_pvfs2_CONFIG(action-if-can-compile,
-#                        [action-if-cant-compile])
-# ------------------------------------------------
-AC_DEFUN([MCA_ompi_fbtl_pvfs2_CONFIG],[
-    AC_CONFIG_FILES([ompi/mca/fbtl/pvfs2/Makefile])
-
-    OMPI_CHECK_PVFS2([fbtl_pvfs2],
-                     [fbtl_pvfs2_happy="yes"],
-                     [fbtl_pvfs2_happy="no"])
-
-    AS_IF([test "$fbtl_pvfs2_happy" = "yes"],
-          [$1],
-          [$2])
-
-
-
-    # substitute in the things needed to build pvfs2
-    AC_SUBST([fbtl_pvfs2_CPPFLAGS])
-    AC_SUBST([fbtl_pvfs2_LDFLAGS])
-    AC_SUBST([fbtl_pvfs2_LIBS])
-])dnl
diff --git a/ompi/mca/fbtl/pvfs2/fbtl_pvfs2.c b/ompi/mca/fbtl/pvfs2/fbtl_pvfs2.c
deleted file mode 100644
index 4376d6668af..00000000000
--- a/ompi/mca/fbtl/pvfs2/fbtl_pvfs2.c
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2006 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008-2014 University of Houston. All rights reserved.
- * Copyright (c) 2018      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- *
- * These symbols are in a file by themselves to provide nice linker
- * semantics. Since linkers generally pull in symbols by object fules,
- * keeping these symbols as the only symbols in this file prevents
- * utility programs such as "ompi_info" from having to import entire
- * modules just to query their version and parameters
- */
-
-#include "ompi_config.h"
-#include "mpi.h"
-#include "ompi/mca/fbtl/fbtl.h"
-#include "ompi/mca/fbtl/base/base.h"
-#include "ompi/mca/fbtl/pvfs2/fbtl_pvfs2.h"
-
-/*
- * *******************************************************************
- * ************************ actions structure ************************
- * *******************************************************************
- */
-static mca_fbtl_base_module_1_0_0_t pvfs2 =  {
-    mca_fbtl_pvfs2_module_init,     /* initalise after being selected */
-    mca_fbtl_pvfs2_module_finalize, /* close a module on a communicator */
-    mca_fbtl_pvfs2_preadv,          /* blocking read */
-    NULL,                           /* non-blocking read */
-    mca_fbtl_pvfs2_pwritev,         /* blocking write */
-    NULL,                           /* non-blocking write */
-    NULL,                           /* module specific progress */
-    NULL,                           /* free module specific data items on the request */
-    mca_fbtl_base_check_atomicity   /* check whether atomicity is supported on this fs */
-};
-/*
- * *******************************************************************
- * ************************* structure ends **************************
- * *******************************************************************
- */
-
-int mca_fbtl_pvfs2_component_init_query(bool enable_progress_threads,
-                                      bool enable_mpi_threads) {
-    /* Nothing to do */
-
-   return OMPI_SUCCESS;
-}
-
-struct mca_fbtl_base_module_1_0_0_t *
-mca_fbtl_pvfs2_component_file_query (ompio_file_t *fh, int *priority) {
-   *priority = mca_fbtl_pvfs2_priority;
-
-   if (PVFS2 == fh->f_fstype) {
-       if (*priority < 50) {
-           *priority = 50;
-       }
-   }
-
-   return &pvfs2;
-}
-
-int mca_fbtl_pvfs2_component_file_unquery (ompio_file_t *file) {
-   /* This function might be needed for some purposes later. for now it
-    * does not have anything to do since there are no steps which need
-    * to be undone if this module is not selected */
-
-   return OMPI_SUCCESS;
-}
-
-int mca_fbtl_pvfs2_module_init (ompio_file_t *file) {
-    return OMPI_SUCCESS;
-}
-
-
-int mca_fbtl_pvfs2_module_finalize (ompio_file_t *file) {
-    return OMPI_SUCCESS;
-}
diff --git a/ompi/mca/fbtl/pvfs2/fbtl_pvfs2.h b/ompi/mca/fbtl/pvfs2/fbtl_pvfs2.h
deleted file mode 100644
index 63fcd274e70..00000000000
--- a/ompi/mca/fbtl/pvfs2/fbtl_pvfs2.h
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2006 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008-2014 University of Houston. All rights reserved.
- * Copyright (c) 2018      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef MCA_FBTL_PVFS2_H
-#define MCA_FBTL_PVFS2_H
-
-#include "ompi_config.h"
-#include "ompi/mca/mca.h"
-#include "ompi/mca/fbtl/fbtl.h"
-#include "ompi/mca/common/ompio/common_ompio.h"
-#include "ompi/mca/fs/pvfs2/fs_pvfs2.h"
-#include "pvfs2.h"
-#include "pvfs2-compat.h"
-
-/*
-#ifdef HAVE_PVFS2_H
-#include "pvfs2.h"
-#endif
-
-#ifdef PVFS2_VERSION_MAJOR
-#include "pvfs2-compat.h"
-#endif
-*/
-extern int mca_fbtl_pvfs2_priority;
-
-BEGIN_C_DECLS
-
-int mca_fbtl_pvfs2_component_init_query(bool enable_progress_threads,
-                                        bool enable_mpi_threads);
-struct mca_fbtl_base_module_1_0_0_t *
-mca_fbtl_pvfs2_component_file_query (ompio_file_t *file, int *priority);
-int mca_fbtl_pvfs2_component_file_unquery (ompio_file_t *file);
-
-int mca_fbtl_pvfs2_module_init (ompio_file_t *file);
-int mca_fbtl_pvfs2_module_finalize (ompio_file_t *file);
-
-OMPI_MODULE_DECLSPEC extern mca_fbtl_base_component_2_0_0_t mca_fbtl_pvfs2_component;
-/*
- * ******************************************************************
- * ********* functions which are implemented in this module *********
- * ******************************************************************
- */
-
-ssize_t mca_fbtl_pvfs2_preadv (ompio_file_t *file);
-ssize_t mca_fbtl_pvfs2_pwritev (ompio_file_t *file);
-ssize_t mca_fbtl_pvfs2_ipreadv (ompio_file_t *file,
-                               ompi_request_t **request);
-ssize_t mca_fbtl_pvfs2_ipwritev (ompio_file_t *file,
-                               ompi_request_t **request);
-
-/*
- * ******************************************************************
- * ************ functions implemented in this module end ************
- * ******************************************************************
- */
-
-END_C_DECLS
-
-#endif /* MCA_FBTL_PVFS2_H */
diff --git a/ompi/mca/fbtl/pvfs2/fbtl_pvfs2_component.c b/ompi/mca/fbtl/pvfs2/fbtl_pvfs2_component.c
deleted file mode 100644
index 45e0a58b73b..00000000000
--- a/ompi/mca/fbtl/pvfs2/fbtl_pvfs2_component.c
+++ /dev/null
@@ -1,65 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008-2011 University of Houston. All rights reserved.
- * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- *
- * These symbols are in a file by themselves to provide nice linker
- * semantics.  Since linkers generally pull in symbols by object
- * files, keeping these symbols as the only symbols in this file
- * prevents utility programs such as "ompi_info" from having to import
- * entire components just to query their version and parameters.
- */
-
-#include "ompi_config.h"
-#include "fbtl_pvfs2.h"
-#include "mpi.h"
-
-/*
- * Public string showing the fbtl pvfs2 component version number
- */
-const char *mca_fbtl_pvfs2_component_version_string =
-  "OMPI/MPI pvfs2 FBTL MCA component version " OMPI_VERSION;
-
-int mca_fbtl_pvfs2_priority = 0;
-
-/*
- * Instantiate the public struct with all of our public information
- * and pointers to our public functions in it
- */
-mca_fbtl_base_component_2_0_0_t mca_fbtl_pvfs2_component = {
-
-    /* First, the mca_component_t struct containing meta information
-       about the component itself */
-
-    .fbtlm_version = {
-        MCA_FBTL_BASE_VERSION_2_0_0,
-
-        /* Component name and version */
-        .mca_component_name = "pvfs2",
-        MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION,
-                              OMPI_RELEASE_VERSION),
-    },
-    .fbtlm_data = {
-        /* This component is checkpointable */
-      MCA_BASE_METADATA_PARAM_CHECKPOINT
-    },
-    .fbtlm_init_query = mca_fbtl_pvfs2_component_init_query,      /* get thread level */
-    .fbtlm_file_query = mca_fbtl_pvfs2_component_file_query,      /* get priority and actions */
-    .fbtlm_file_unquery = mca_fbtl_pvfs2_component_file_unquery,  /* undo what was done by previous function */
-};
diff --git a/ompi/mca/fbtl/pvfs2/fbtl_pvfs2_ipreadv.c b/ompi/mca/fbtl/pvfs2/fbtl_pvfs2_ipreadv.c
deleted file mode 100644
index eaf99c6f282..00000000000
--- a/ompi/mca/fbtl/pvfs2/fbtl_pvfs2_ipreadv.c
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2011 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008-2011 University of Houston. All rights reserved.
- * Copyright (c) 2018      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-
-#include "ompi_config.h"
-#include "fbtl_pvfs2.h"
-
-#include "mpi.h"
-#include "ompi/constants.h"
-#include "ompi/mca/fbtl/fbtl.h"
-
-ssize_t  mca_fbtl_pvfs2_ipreadv (ompio_file_t *file,
-				ompi_request_t **request)
-{
-    printf ("PVFS2 IPREADV\n");
-    return OMPI_SUCCESS;
-}
diff --git a/ompi/mca/fbtl/pvfs2/fbtl_pvfs2_ipwritev.c b/ompi/mca/fbtl/pvfs2/fbtl_pvfs2_ipwritev.c
deleted file mode 100644
index b6e46c26818..00000000000
--- a/ompi/mca/fbtl/pvfs2/fbtl_pvfs2_ipwritev.c
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2011 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008-2011 University of Houston. All rights reserved.
- * Copyright (c) 2018      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-
-#include "ompi_config.h"
-#include "fbtl_pvfs2.h"
-
-#include "mpi.h"
-#include "ompi/constants.h"
-#include "ompi/mca/fbtl/fbtl.h"
-
-ssize_t  mca_fbtl_pvfs2_ipwritev (ompio_file_t *file,
-				 ompi_request_t **request)
-{
-    printf ("PVFS2 IPWRITEV\n");
-    return OMPI_SUCCESS;
-}
diff --git a/ompi/mca/fbtl/pvfs2/fbtl_pvfs2_preadv.c b/ompi/mca/fbtl/pvfs2/fbtl_pvfs2_preadv.c
deleted file mode 100644
index 898e4d8d0ce..00000000000
--- a/ompi/mca/fbtl/pvfs2/fbtl_pvfs2_preadv.c
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2011 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008-2014 University of Houston. All rights reserved.
- * Copyright (c) 2017-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-/* This code is based on the PVFS2 ADIO module in ROMIO
- *   Copyright (C) 1997 University of Chicago.
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ompi_config.h"
-#include "fbtl_pvfs2.h"
-
-#include "mpi.h"
-#include <unistd.h>
-#include "ompi/constants.h"
-#include "ompi/mca/fbtl/fbtl.h"
-
-ssize_t  mca_fbtl_pvfs2_preadv (ompio_file_t *fh)
-{
-    int i;
-    int ret;
-    size_t k;
-    int merge = 0;
-    char *merge_buf = NULL;
-    size_t merge_length = 0;
-    OMPI_MPI_OFFSET_TYPE merge_offset = 0;
-    PVFS_sysresp_io resp_io;
-    PVFS_Request file_req;
-    PVFS_Request mem_req;
-    mca_fs_pvfs2 *pvfs2_fs;
-    ssize_t total_bytes_read=0;
-
-    pvfs2_fs = (mca_fs_pvfs2 *)fh->f_fs_ptr;
-
-    if (NULL == fh->f_io_array) {
-        return OMPI_ERROR;
-    }
-
-    for (i=0 ; i<fh->f_num_of_io_entries ; i++) {
-	if (fh->f_num_of_io_entries != i+1) {
-	    if (((OMPI_MPI_OFFSET_TYPE)fh->f_io_array[i].offset +
-		 (ptrdiff_t)fh->f_io_array[i].length) ==
-		(OMPI_MPI_OFFSET_TYPE)fh->f_io_array[i+1].offset) {
-		if (!merge) {
-		    merge_offset = (OMPI_MPI_OFFSET_TYPE)
-			fh->f_io_array[i].offset;
-		    merge_length = fh->f_io_array[i].length;
-		}
-		merge_length += fh->f_io_array[i+1].length;
-		merge++;
-		continue;
-	    }
-	}
-	if (merge) {
-	    merge_buf = malloc (merge_length);
-
-	    ret = PVFS_Request_contiguous (merge_length,
-					   PVFS_BYTE,
-					   &mem_req);
-	    if (ret != 0) {
-		perror("PVFS_Request_contiguous() error");
-		return OMPI_ERROR;
-	    }
-	    ret = PVFS_Request_contiguous (merge_length,
-					   PVFS_BYTE,
-					   &file_req);
-	    if (ret != 0) {
-		perror("PVFS_Request_contiguous() error");
-		return OMPI_ERROR;
-	    }
-	    ret = PVFS_sys_read (pvfs2_fs->object_ref,
-				 file_req,
-				 merge_offset,
-				 merge_buf,
-				 mem_req,
-				 &(pvfs2_fs->credentials),
-				 &resp_io);
-	    if (ret != 0) {
-		perror("PVFS_sys_write() error");
-		return OMPI_ERROR;
-	    }
-	    total_bytes_read += (ssize_t)resp_io.total_completed;
-
-	    k = 0;
-	    while (merge >= 0) {
-		memcpy (fh->f_io_array[i-merge].memory_address,
-			merge_buf + k,
-			fh->f_io_array[i-merge].length);
-		k += fh->f_io_array[i-merge].length;
-		merge --;
-	    }
-	    merge = 0;
-	    merge_offset = 0;
-	    merge_length = 0;
-	    if (NULL != merge_buf) {
-		free (merge_buf);
-		merge_buf = NULL;
-	    }
-	}
-	else {
-	    ret = PVFS_Request_contiguous (fh->f_io_array[i].length,
-					   PVFS_BYTE,
-					   &mem_req);
-	    if (ret != 0) {
-		perror("PVFS_Request_contiguous() error");
-		return OMPI_ERROR;
-	    }
-	    ret = PVFS_Request_contiguous (fh->f_io_array[i].length,
-					   PVFS_BYTE,
-					   &file_req);
-	    if (ret != 0) {
-		perror("PVFS_Request_contiguous() error");
-		return OMPI_ERROR;
-	    }
-	    ret = PVFS_sys_read (pvfs2_fs->object_ref,
-				 file_req,
-				 (OMPI_MPI_OFFSET_TYPE)
-				 fh ->f_io_array[i].offset,
-				 fh->f_io_array[i].memory_address,
-				 mem_req,
-				 &(pvfs2_fs->credentials),
-				 &resp_io);
-	    if (ret != 0) {
-		perror("PVFS_sys_write() error");
-		return OMPI_ERROR;
-	    }
-	    total_bytes_read += (ssize_t)resp_io.total_completed;
-	}
-    }
-
-    return total_bytes_read;
-}
diff --git a/ompi/mca/fbtl/pvfs2/fbtl_pvfs2_pwritev.c b/ompi/mca/fbtl/pvfs2/fbtl_pvfs2_pwritev.c
deleted file mode 100644
index 496b405ca67..00000000000
--- a/ompi/mca/fbtl/pvfs2/fbtl_pvfs2_pwritev.c
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2011 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008-2014 University of Houston. All rights reserved.
- * Copyright (c) 2017-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-/* This code is based on the PVFS2 ADIO module in ROMIO
- *   Copyright (C) 1997 University of Chicago.
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ompi_config.h"
-#include "fbtl_pvfs2.h"
-
-#include "mpi.h"
-#include <unistd.h>
-#include <sys/uio.h>
-#include "ompi/constants.h"
-#include "ompi/mca/fbtl/fbtl.h"
-
-ssize_t  mca_fbtl_pvfs2_pwritev (ompio_file_t *fh )
-{
-    int i;
-    int merge = 0;
-    int ret;
-    size_t k;
-    char *merge_buf = NULL;
-    size_t merge_length = 0;
-    OMPI_MPI_OFFSET_TYPE merge_offset = 0;
-    PVFS_sysresp_io resp_io;
-    PVFS_Request file_req;
-    PVFS_Request mem_req;
-    mca_fs_pvfs2 *pvfs2_fs;
-    ssize_t total_bytes_written=0;
-
-    pvfs2_fs = (mca_fs_pvfs2 *)fh->f_fs_ptr;
-
-    if (NULL == fh->f_io_array) {
-        return OMPI_ERROR;
-    }
-
-    for (i=0 ; i<fh->f_num_of_io_entries ; i++) {
-	if (fh->f_num_of_io_entries != i+1) {
-	    if (((OMPI_MPI_OFFSET_TYPE)fh->f_io_array[i].offset +
-		 (ptrdiff_t)fh->f_io_array[i].length) ==
-		(OMPI_MPI_OFFSET_TYPE)fh->f_io_array[i+1].offset) {
-		if (!merge) {
-		    merge_offset = (OMPI_MPI_OFFSET_TYPE)
-			fh->f_io_array[i].offset;
-		    merge_length = fh->f_io_array[i].length;
-		}
-		merge_length += fh->f_io_array[i+1].length;
-		merge++;
-		continue;
-	    }
-	}
-	/*
-	  printf ("RANK: %d Entries: %d MERGE: %d \n", fh->f_rank,fh->f_num_of_io_entries,merge);
-	  if (merge)
-	  printf ("Merge size: %lld\n",merge_length);
-	  else
-	  printf ("entry size: %lld\n",fh->f_io_array[i].length);
-	  sleep(2);
-	*/
-	if (merge) {
-	    merge_buf = malloc (merge_length);
-	    k = 0;
-	    while (merge >= 0) {
-		memcpy (merge_buf + k,
-			fh->f_io_array[i-merge].memory_address,
-			fh->f_io_array[i-merge].length);
-		k += fh->f_io_array[i-merge].length;
-		merge --;
-	    }
-
-	    ret = PVFS_Request_contiguous (merge_length,
-					   PVFS_BYTE,
-					   &mem_req);
-	    if (ret != 0) {
-		perror("PVFS_Request_contiguous() error");
-		return OMPI_ERROR;
-	    }
-	    ret = PVFS_Request_contiguous (merge_length,
-					   PVFS_BYTE,
-					   &file_req);
-	    if (ret != 0) {
-		perror("PVFS_Request_contiguous() error");
-		return OMPI_ERROR;
-	    }
-	    ret = PVFS_sys_write (pvfs2_fs->object_ref,
-				  file_req,
-				  merge_offset,
-				  merge_buf,
-				  mem_req,
-				  &(pvfs2_fs->credentials),
-				  &resp_io);
-	    if (ret != 0) {
-		perror("PVFS_sys_write() error");
-		return OMPI_ERROR;
-	    }
-	    total_bytes_written += (ssize_t)resp_io.total_completed;
-
-	    merge = 0;
-	    merge_offset = 0;
-	    merge_length = 0;
-	    if (NULL != merge_buf) {
-		free (merge_buf);
-		merge_buf = NULL;
-	    }
-	}
-	else {
-	    ret = PVFS_Request_contiguous (fh->f_io_array[i].length,
-					   PVFS_BYTE,
-					   &mem_req);
-	    if (ret != 0) {
-		perror("PVFS_Request_contiguous() error");
-		return OMPI_ERROR;
-	    }
-	    ret = PVFS_Request_contiguous (fh->f_io_array[i].length,
-					   PVFS_BYTE,
-					   &file_req);
-	    if (ret != 0) {
-		perror("PVFS_Request_contiguous() error");
-		return OMPI_ERROR;
-	    }
-	    ret = PVFS_sys_write (pvfs2_fs->object_ref,
-				  file_req,
-				  (OMPI_MPI_OFFSET_TYPE)
-				  fh ->f_io_array[i].offset,
-				  fh->f_io_array[i].memory_address,
-				  mem_req,
-				  &(pvfs2_fs->credentials),
-				  &resp_io);
-	    if (ret != 0) {
-		perror("PVFS_sys_write() error");
-		return OMPI_ERROR;
-	    }
-	    total_bytes_written += (ssize_t)resp_io.total_completed;
-
-	}
-    }
-
-    return total_bytes_written;
-}
diff --git a/ompi/mca/fbtl/pvfs2/owner.txt b/ompi/mca/fbtl/pvfs2/owner.txt
deleted file mode 100644
index 2e9726c28a4..00000000000
--- a/ompi/mca/fbtl/pvfs2/owner.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-#
-# owner/status file
-# owner: institution that is responsible for this package
-# status: e.g. active, maintenance, unmaintained
-#
-owner: UH
-status: active
diff --git a/ompi/mca/fcoll/base/Makefile.am b/ompi/mca/fcoll/base/Makefile.am
index c91d548bac2..1c1d0df73e9 100644
--- a/ompi/mca/fcoll/base/Makefile.am
+++ b/ompi/mca/fcoll/base/Makefile.am
@@ -9,7 +9,7 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2008-2011 University of Houston. All rights reserved.
+# Copyright (c) 2008-2022 University of Houston. All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -23,8 +23,12 @@ headers += \
 
 libmca_fcoll_la_SOURCES += \
         base/fcoll_base_frame.c \
+        base/fcoll_base_find_available.c
+
+if OMPI_OMPIO_SUPPORT
+libmca_fcoll_la_SOURCES += \
         base/fcoll_base_file_select.c \
         base/fcoll_base_file_unselect.c \
-        base/fcoll_base_find_available.c \
         base/fcoll_base_sort.c \
         base/fcoll_base_coll_array.c
+endif
diff --git a/ompi/mca/fcoll/base/base.h b/ompi/mca/fcoll/base/base.h
index 353c062a599..4a22d3f0c8e 100644
--- a/ompi/mca/fcoll/base/base.h
+++ b/ompi/mca/fcoll/base/base.h
@@ -10,10 +10,12 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2008-2011 University of Houston. All rights reserved.
+ * Copyright (c) 2008-2022 University of Houston. All rights reserved.
  * Copyright (c) 2017      IBM Corporation. All rights reserved.
  * Copyright (c) 2018      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -55,6 +57,10 @@ OMPI_DECLSPEC int ompi_fcoll_base_sort_iovec (struct iovec *iov, int num_entries
 
 OMPI_DECLSPEC mca_fcoll_base_component_t* mca_fcoll_base_component_lookup(const char* name);
 
+OMPI_DECLSPEC int mca_fcoll_base_file_read_all (ompio_file_t *fh, void *buf, size_t count,
+                                                struct ompi_datatype_t *datatype,
+                                                ompi_status_public_t *status);
+
 /*
  * Globals
  */
diff --git a/ompi/mca/fcoll/base/fcoll_base_coll_array.c b/ompi/mca/fcoll/base/fcoll_base_coll_array.c
index 4812426f560..68f25ace6fb 100644
--- a/ompi/mca/fcoll/base/fcoll_base_coll_array.c
+++ b/ompi/mca/fcoll/base/fcoll_base_coll_array.c
@@ -36,11 +36,11 @@
 
 
 int ompi_fcoll_base_coll_allgatherv_array (void *sbuf,
-                                      int scount,
+                                      size_t scount,
                                       ompi_datatype_t *sdtype,
                                       void *rbuf,
-                                      int *rcounts,
-                                      int *disps,
+                                      size_t *rcounts,
+                                      ptrdiff_t *disps,
                                       ompi_datatype_t *rdtype,
                                       int root_index,
                                       int *procs_in_group,
@@ -50,6 +50,7 @@ int ompi_fcoll_base_coll_allgatherv_array (void *sbuf,
     int err = OMPI_SUCCESS;
     ptrdiff_t extent, lb;
     int i, rank, j;
+    int *tmp_rcounts = NULL, *tmp_disps = NULL;
     char *send_buf = NULL;
     struct ompi_datatype_t *newtype, *send_type;
 
@@ -92,11 +93,24 @@ int ompi_fcoll_base_coll_allgatherv_array (void *sbuf,
         return err;
     }
 
+    /* TODO:BIGCOUNT: remove tmp_rcounts and tmp_disps once the ompi_datatype
+     * interface is udpated to use size_t/ptrdiff_t
+     */
+    tmp_rcounts = (int *)malloc(2 * procs_per_group * sizeof(int));
+    if (NULL == tmp_rcounts) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+    tmp_disps = tmp_rcounts + procs_per_group;
+    for (i = 0; i < procs_per_group; i++) {
+        tmp_rcounts[i] = (int) rcounts[i];
+        tmp_disps[i] = (int) disps[i];
+    }
     err = ompi_datatype_create_indexed (procs_per_group,
-                                        rcounts,
-                                        disps,
+                                        tmp_rcounts,
+                                        tmp_disps,
                                         rdtype,
                                         &newtype);
+    free(tmp_rcounts);
     if (MPI_SUCCESS != err) {
         return err;
     }
@@ -119,11 +133,11 @@ int ompi_fcoll_base_coll_allgatherv_array (void *sbuf,
 }
 
 int ompi_fcoll_base_coll_gatherv_array (void *sbuf,
-                                   int scount,
+                                   size_t scount,
                                    ompi_datatype_t *sdtype,
                                    void *rbuf,
-                                   int *rcounts,
-                                   int *disps,
+                                   size_t *rcounts,
+                                   ptrdiff_t *disps,
                                    ompi_datatype_t *rdtype,
                                    int root_index,
                                    int *procs_in_group,
@@ -151,7 +165,7 @@ int ompi_fcoll_base_coll_gatherv_array (void *sbuf,
         return err;
     }
 
-    /* writer processes, loop receiving data from proceses
+    /* writer processes, loop receiving data from processes
        belonging to each corresponding root */
 
     err = opal_datatype_get_extent (&rdtype->super, &lb, &extent);
@@ -208,11 +222,11 @@ int ompi_fcoll_base_coll_gatherv_array (void *sbuf,
 }
 
 int ompi_fcoll_base_coll_scatterv_array (void *sbuf,
-                                    int *scounts,
-                                    int *disps,
+                                    size_t *scounts,
+                                    ptrdiff_t *disps,
                                     ompi_datatype_t *sdtype,
                                     void *rbuf,
-                                    int rcount,
+                                    size_t rcount,
                                     ompi_datatype_t *rdtype,
                                     int root_index,
                                     int *procs_in_group,
@@ -240,7 +254,7 @@ int ompi_fcoll_base_coll_scatterv_array (void *sbuf,
         return err;
     }
 
-    /* writer processes, loop sending data to proceses
+    /* writer processes, loop sending data to processes
        belonging to each corresponding root */
 
     err = opal_datatype_get_extent (&sdtype->super, &lb, &extent);
@@ -298,10 +312,10 @@ int ompi_fcoll_base_coll_scatterv_array (void *sbuf,
 }
 
 int ompi_fcoll_base_coll_allgather_array (void *sbuf,
-                                     int scount,
+                                     size_t scount,
                                      ompi_datatype_t *sdtype,
                                      void *rbuf,
-                                     int rcount,
+                                     size_t rcount,
                                      ompi_datatype_t *rdtype,
                                      int root_index,
                                      int *procs_in_group,
@@ -351,10 +365,10 @@ int ompi_fcoll_base_coll_allgather_array (void *sbuf,
 }
 
 int ompi_fcoll_base_coll_gather_array (void *sbuf,
-                                  int scount,
+                                  size_t scount,
                                   ompi_datatype_t *sdtype,
                                   void *rbuf,
-                                  int rcount,
+                                  size_t rcount,
                                   ompi_datatype_t *rdtype,
                                   int root_index,
                                   int *procs_in_group,
@@ -441,7 +455,7 @@ int ompi_fcoll_base_coll_gather_array (void *sbuf,
 }
 
 int ompi_fcoll_base_coll_bcast_array (void *buff,
-                                 int count,
+                                 size_t count,
                                  ompi_datatype_t *datatype,
                                  int root_index,
                                  int *procs_in_group,
diff --git a/ompi/mca/fcoll/base/fcoll_base_coll_array.h b/ompi/mca/fcoll/base/fcoll_base_coll_array.h
index 7f6c21ca488..f31f18c6172 100644
--- a/ompi/mca/fcoll/base/fcoll_base_coll_array.h
+++ b/ompi/mca/fcoll/base/fcoll_base_coll_array.h
@@ -14,6 +14,8 @@
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -43,32 +45,32 @@
  * Based on an array of procs in group
  */
 OMPI_DECLSPEC int ompi_fcoll_base_coll_gatherv_array (void *sbuf,
-                                                 int scount,
+                                                 size_t scount,
                                                  ompi_datatype_t *sdtype,
                                                  void *rbuf,
-                                                 int *rcounts,
-                                                 int *disps,
+                                                 size_t *rcounts,
+                                                 ptrdiff_t *disps,
                                                  ompi_datatype_t *rdtype,
                                                  int root_index,
                                                  int *procs_in_group,
                                                  int procs_per_group,
                                                  ompi_communicator_t *comm);
 OMPI_DECLSPEC int ompi_fcoll_base_coll_scatterv_array (void *sbuf,
-                                                  int *scounts,
-                                                  int *disps,
+                                                  size_t *scounts,
+                                                  ptrdiff_t *disps,
                                                   ompi_datatype_t *sdtype,
                                                   void *rbuf,
-                                                  int rcount,
+                                                  size_t rcount,
                                                   ompi_datatype_t *rdtype,
                                                   int root_index,
                                                   int *procs_in_group,
                                                   int procs_per_group,
                                                   ompi_communicator_t *comm);
 OMPI_DECLSPEC int ompi_fcoll_base_coll_allgather_array (void *sbuf,
-                                                   int scount,
+                                                   size_t scount,
                                                    ompi_datatype_t *sdtype,
                                                    void *rbuf,
-                                                   int rcount,
+                                                   size_t rcount,
                                                    ompi_datatype_t *rdtype,
                                                    int root_index,
                                                    int *procs_in_group,
@@ -76,28 +78,28 @@ OMPI_DECLSPEC int ompi_fcoll_base_coll_allgather_array (void *sbuf,
                                                    ompi_communicator_t *comm);
 
 OMPI_DECLSPEC int ompi_fcoll_base_coll_allgatherv_array (void *sbuf,
-                                                    int scount,
+                                                    size_t scount,
                                                     ompi_datatype_t *sdtype,
                                                     void *rbuf,
-                                                    int *rcounts,
-                                                    int *disps,
+                                                    size_t *rcounts,
+                                                    ptrdiff_t *disps,
                                                     ompi_datatype_t *rdtype,
                                                     int root_index,
                                                     int *procs_in_group,
                                                     int procs_per_group,
                                                     ompi_communicator_t *comm);
 OMPI_DECLSPEC int ompi_fcoll_base_coll_gather_array (void *sbuf,
-                                                int scount,
+                                                size_t scount,
                                                 ompi_datatype_t *sdtype,
                                                 void *rbuf,
-                                                int rcount,
+                                                size_t rcount,
                                                 ompi_datatype_t *rdtype,
                                                 int root_index,
                                                 int *procs_in_group,
                                                 int procs_per_group,
                                                 ompi_communicator_t *comm);
 OMPI_DECLSPEC int ompi_fcoll_base_coll_bcast_array (void *buff,
-                                               int count,
+                                               size_t count,
                                                ompi_datatype_t *datatype,
                                                int root_index,
                                                int *procs_in_group,
diff --git a/ompi/mca/fcoll/base/fcoll_base_file_select.c b/ompi/mca/fcoll/base/fcoll_base_file_select.c
index 043f776d603..4989a0e34bd 100644
--- a/ompi/mca/fcoll/base/fcoll_base_file_select.c
+++ b/ompi/mca/fcoll/base/fcoll_base_file_select.c
@@ -49,7 +49,7 @@ static OBJ_CLASS_INSTANCE(queried_module_t, opal_list_item_t, NULL, NULL);
 /*
  * Only one fcoll module can be attached to each file.
  *
- * This module calls the query funtion on all the components that were
+ * This module calls the query function on all the components that were
  * detected by fcoll_base_open. This function is called on a
  * per-file basis. This function has the following function.
  *
@@ -212,14 +212,14 @@ int mca_fcoll_base_file_select (struct ompio_file_t *file,
             * module of this component.
             *
             * ANJU: a component might not have all the functions
-            * defined.  Whereever a function pointer is null in the
+            * defined.  Wherever a function pointer is null in the
             * module structure we need to fill it in with the base
             * structure function pointers. This is yet to be done
             */
 
             /*
-             * We don return here coz we still need to go through and
-             * elease the other objects
+             * We don't return here coz we still need to go through and
+             * release the other objects
              */
 
             /*fill_null_pointers (om->om_module);*/
@@ -231,7 +231,7 @@ int mca_fcoll_base_file_select (struct ompio_file_t *file,
             */
          } else {
             /*
-             * this is not the "choosen one", finalize
+             * this is not the "chosen one", finalize
              */
              if (NULL != om->om_component->fcollm_file_unquery) {
                 /* unquery the component only if they have some clean
diff --git a/ompi/mca/fcoll/base/fcoll_base_find_available.c b/ompi/mca/fcoll/base/fcoll_base_find_available.c
index 47303eb5c12..5680c8d4d16 100644
--- a/ompi/mca/fcoll/base/fcoll_base_find_available.c
+++ b/ompi/mca/fcoll/base/fcoll_base_find_available.c
@@ -10,6 +10,8 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2008-2011 University of Houston. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -31,7 +33,7 @@
 static int init_query(const mca_base_component_t *m,
                       bool enable_progress_threads,
                       bool enable_mpi_threads);
-static int init_query_2_0_0(const mca_base_component_t *component,
+static int init_query_3_0_0(const mca_base_component_t *component,
                             bool enable_progress_threads,
                             bool enable_mpi_threads);
 
@@ -83,10 +85,10 @@ static int init_query(const mca_base_component_t *m,
                         m->mca_component_name);
 
     /* This component has been successfully opened, now try to query it */
-    if (2 == m->mca_type_major_version &&
+    if (3 == m->mca_type_major_version &&
         0 == m->mca_type_minor_version &&
         0 == m->mca_type_release_version) {
-        ret = init_query_2_0_0(m, enable_progress_threads,
+        ret = init_query_3_0_0(m, enable_progress_threads,
                                enable_mpi_threads);
     } else {
         /* unrecognised API version */
@@ -114,12 +116,12 @@ static int init_query(const mca_base_component_t *m,
 }
 
 
-static int init_query_2_0_0(const mca_base_component_t *component,
+static int init_query_3_0_0(const mca_base_component_t *component,
                             bool enable_progress_threads,
                             bool enable_mpi_threads)
 {
-    mca_fcoll_base_component_2_0_0_t *fcoll =
-        (mca_fcoll_base_component_2_0_0_t *) component;
+    mca_fcoll_base_component_3_0_0_t *fcoll =
+        (mca_fcoll_base_component_3_0_0_t *) component;
 
     return fcoll->fcollm_init_query(enable_progress_threads,
                               enable_mpi_threads);
diff --git a/ompi/mca/fcoll/dynamic/Makefile.am b/ompi/mca/fcoll/dynamic/Makefile.am
index 603fa55df72..1a0aed29947 100644
--- a/ompi/mca/fcoll/dynamic/Makefile.am
+++ b/ompi/mca/fcoll/dynamic/Makefile.am
@@ -9,7 +9,7 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2008-2015 University of Houston. All rights reserved.
+# Copyright (c) 2008-2022 University of Houston. All rights reserved.
 # Copyright (c) 2012 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
 # Copyright (c) 2018      Research Organization for Information Science
diff --git a/ompi/mca/fcoll/dynamic/fcoll_dynamic.h b/ompi/mca/fcoll/dynamic/fcoll_dynamic.h
index d929def61ae..b338713f7bd 100644
--- a/ompi/mca/fcoll/dynamic/fcoll_dynamic.h
+++ b/ompi/mca/fcoll/dynamic/fcoll_dynamic.h
@@ -9,9 +9,11 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2008-2016 University of Houston. All rights reserved.
+ * Copyright (c) 2008-2022 University of Houston. All rights reserved.
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -36,13 +38,13 @@ BEGIN_C_DECLS
 
 extern int mca_fcoll_dynamic_priority;
 
-OMPI_MODULE_DECLSPEC extern mca_fcoll_base_component_2_0_0_t mca_fcoll_dynamic_component;
+OMPI_DECLSPEC extern mca_fcoll_base_component_3_0_0_t mca_fcoll_dynamic_component;
 
 /* API functions */
 
 int mca_fcoll_dynamic_component_init_query(bool enable_progress_threads,
                                            bool enable_mpi_threads);
-struct mca_fcoll_base_module_1_0_0_t *
+struct mca_fcoll_base_module_2_0_0_t *
 mca_fcoll_dynamic_component_file_query (ompio_file_t *fh, int *priority);
 
 int mca_fcoll_dynamic_component_file_unquery (ompio_file_t *file);
@@ -50,19 +52,18 @@ int mca_fcoll_dynamic_component_file_unquery (ompio_file_t *file);
 int mca_fcoll_dynamic_module_init (ompio_file_t *file);
 int mca_fcoll_dynamic_module_finalize (ompio_file_t *file);
 
-int mca_fcoll_dynamic_file_read_all (ompio_file_t *fh,
+int mca_fcoll_dynamic_file_write_all (struct ompio_file_t *fh,
+                                      const void *buf,
+                                      size_t count,
+                                      struct ompi_datatype_t *datatype,
+                                      ompi_status_public_t * status);
+int mca_fcoll_dynamic_file_read_all (struct ompio_file_t *fh,
                                      void *buf,
-                                     int count,
+                                     size_t count,
                                      struct ompi_datatype_t *datatype,
                                      ompi_status_public_t * status);
 
 
-int mca_fcoll_dynamic_file_write_all (ompio_file_t *fh,
-                                      const void *buf,
-                                      int count,
-                                      struct ompi_datatype_t *datatype,
-                                      ompi_status_public_t * status);
-
 
 END_C_DECLS
 
diff --git a/ompi/mca/fcoll/dynamic/fcoll_dynamic_component.c b/ompi/mca/fcoll/dynamic/fcoll_dynamic_component.c
index e4aaa46a81a..711d4745bc1 100644
--- a/ompi/mca/fcoll/dynamic/fcoll_dynamic_component.c
+++ b/ompi/mca/fcoll/dynamic/fcoll_dynamic_component.c
@@ -14,6 +14,8 @@
  * Copyright (c) 2008-2014 University of Houston. All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -51,13 +53,13 @@ static int dynamic_register(void);
  * Instantiate the public struct with all of our public information
  * and pointers to our public functions in it
  */
-mca_fcoll_base_component_2_0_0_t mca_fcoll_dynamic_component = {
+mca_fcoll_base_component_3_0_0_t mca_fcoll_dynamic_component = {
 
     /* First, the mca_component_t struct containing meta information
      * about the component itself */
 
     .fcollm_version = {
-        MCA_FCOLL_BASE_VERSION_2_0_0,
+        MCA_FCOLL_BASE_VERSION_3_0_0,
 
         /* Component name and version */
         .mca_component_name = "dynamic",
diff --git a/ompi/mca/fcoll/dynamic/fcoll_dynamic_file_read_all.c b/ompi/mca/fcoll/dynamic/fcoll_dynamic_file_read_all.c
index b743d08a0ea..0a40d11f9a8 100644
--- a/ompi/mca/fcoll/dynamic/fcoll_dynamic_file_read_all.c
+++ b/ompi/mca/fcoll/dynamic/fcoll_dynamic_file_read_all.c
@@ -10,9 +10,11 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2008-2015 University of Houston. All rights reserved.
- * Copyright (c) 2017-2018 Research Organization for Information Science
+ * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -24,1060 +26,18 @@
 #include "fcoll_dynamic.h"
 
 #include "mpi.h"
-#include "ompi/constants.h"
-#include "ompi/mca/fcoll/fcoll.h"
-#include "ompi/mca/fcoll/base/fcoll_base_coll_array.h"
 #include "ompi/mca/common/ompio/common_ompio.h"
-#include "ompi/mca/io/io.h"
-#include "math.h"
-#include "ompi/mca/pml/pml.h"
-#include <unistd.h>
-
-#define DEBUG_ON 0
-
-/*Used for loading file-offsets per aggregator*/
-typedef struct mca_io_ompio_local_io_array{
-    OMPI_MPI_OFFSET_TYPE offset;
-    MPI_Aint             length;
-    int                  process_id;
-}mca_io_ompio_local_io_array;
-
-
-static int read_heap_sort (mca_io_ompio_local_io_array *io_array,
-                           int num_entries,
-                           int *sorted);
 
 
 
 int
-mca_fcoll_dynamic_file_read_all (ompio_file_t *fh,
+mca_fcoll_dynamic_file_read_all (struct ompio_file_t *fh,
                                  void *buf,
-                                 int count,
+                                 size_t count,
                                  struct ompi_datatype_t *datatype,
                                  ompi_status_public_t *status)
 {
-    MPI_Aint position = 0;
-    MPI_Aint total_bytes = 0;          /* total bytes to be read */
-    MPI_Aint bytes_to_read_in_cycle = 0; /* left to be read in a cycle*/
-    MPI_Aint bytes_per_cycle = 0;      /* total read in each cycle by each process*/
-    int index = 0, ret=OMPI_SUCCESS;
-    int cycles = 0;
-    int i=0, j=0, l=0;
-    int n=0; /* current position in total_bytes_per_process array */
-    MPI_Aint bytes_remaining = 0; /* how many bytes have been read from the current
-                                     value from total_bytes_per_process */
-    int *sorted_file_offsets=NULL, entries_per_aggregator=0;
-    int bytes_received = 0;
-    int blocks = 0;
-    /* iovec structure and count of the buffer passed in */
-    uint32_t iov_count = 0;
-    struct iovec *decoded_iov = NULL;
-    int iov_index = 0;
-    size_t current_position = 0;
-    struct iovec *local_iov_array=NULL, *global_iov_array=NULL;
-    char *receive_buf = NULL;
-    MPI_Aint *memory_displacements=NULL;
-    /* global iovec at the readers that contain the iovecs created from
-       file_set_view */
-    uint32_t total_fview_count = 0;
-    int local_count = 0;
-    int *fview_count = NULL, *disp_index=NULL, *temp_disp_index=NULL;
-    int current_index=0, temp_index=0;
-    int **blocklen_per_process=NULL;
-    MPI_Aint **displs_per_process=NULL;
-    char *global_buf = NULL;
-    MPI_Aint global_count = 0;
-    mca_io_ompio_local_io_array *file_offsets_for_agg=NULL;
-
-    /* array that contains the sorted indices of the global_iov */
-    int *sorted = NULL;
-    int *displs = NULL;
-    int dynamic_num_io_procs;
-    size_t max_data = 0;
-    MPI_Aint *total_bytes_per_process = NULL;
-    ompi_datatype_t **sendtype = NULL;
-    MPI_Request *send_req=NULL, recv_req=NULL;
-    int my_aggregator =-1;
-    bool recvbuf_is_contiguous=false;
-    size_t ftype_size;
-    ptrdiff_t ftype_extent, lb;
-
-
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-    double read_time = 0.0, start_read_time = 0.0, end_read_time = 0.0;
-    double rcomm_time = 0.0, start_rcomm_time = 0.0, end_rcomm_time = 0.0;
-    double read_exch = 0.0, start_rexch = 0.0, end_rexch = 0.0;
-    mca_common_ompio_print_entry nentry;
-#endif
-
-    /**************************************************************************
-     ** 1. In case the data is not contigous in memory, decode it into an iovec
-     **************************************************************************/
-
-    opal_datatype_type_size ( &datatype->super, &ftype_size );
-    opal_datatype_get_extent ( &datatype->super, &lb, &ftype_extent );
-
-    if ( (ftype_extent == (ptrdiff_t) ftype_size)             &&
-        opal_datatype_is_contiguous_memory_layout(&datatype->super,1) &&
-        0 == lb ) {
-        recvbuf_is_contiguous = true;
-    }
-
-
-    if (! recvbuf_is_contiguous ) {
-        ret = mca_common_ompio_decode_datatype ((struct ompio_file_t *)fh,
-                                                datatype,
-                                                count,
-                                                buf,
-                                                &max_data,
-                                                fh->f_mem_convertor,
-                                                &decoded_iov,
-                                                &iov_count);
-        if (OMPI_SUCCESS != ret){
-            goto exit;
-        }
-    }
-    else {
-        max_data = count * datatype->super.size;
-    }
-
-    if ( MPI_STATUS_IGNORE != status ) {
-        status->_ucount = max_data;
-    }
-
-    dynamic_num_io_procs = fh->f_get_mca_parameter_value ( "num_aggregators", strlen ("num_aggregators"));
-    if ( OMPI_ERR_MAX == dynamic_num_io_procs ) {
-        ret = OMPI_ERROR;
-        goto exit;
-    }
-    ret = mca_common_ompio_set_aggregator_props ((struct ompio_file_t *) fh,
-                                                 dynamic_num_io_procs,
-                                                 max_data);
-    if (OMPI_SUCCESS != ret){
-        goto exit;
-    }
-    my_aggregator = fh->f_procs_in_group[0];
-
-    /**************************************************************************
-     ** 2. Determine the total amount of data to be written
-     **************************************************************************/
-    total_bytes_per_process = (MPI_Aint*)malloc(fh->f_procs_per_group*sizeof(MPI_Aint));
-    if (NULL == total_bytes_per_process) {
-        opal_output (1, "OUT OF MEMORY\n");
-        ret = OMPI_ERR_OUT_OF_RESOURCE;
-        goto exit;
-    }
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-    start_rcomm_time = MPI_Wtime();
-#endif
-    ret = ompi_fcoll_base_coll_allgather_array (&max_data,
-                                           1,
-                                           MPI_LONG,
-                                           total_bytes_per_process,
-                                           1,
-                                           MPI_LONG,
-                                           0,
-                                           fh->f_procs_in_group,
-                                           fh->f_procs_per_group,
-                                           fh->f_comm);
-    if (OMPI_SUCCESS != ret){
-        goto exit;
-    }
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-    end_rcomm_time = MPI_Wtime();
-    rcomm_time += end_rcomm_time - start_rcomm_time;
-#endif
-
-    for (i=0 ; i<fh->f_procs_per_group ; i++) {
-        total_bytes += total_bytes_per_process[i];
-    }
-
-    if (NULL != total_bytes_per_process) {
-        free (total_bytes_per_process);
-        total_bytes_per_process = NULL;
-    }
-
-    /*********************************************************************
-     *** 3. Generate the File offsets/lengths corresponding to this write
-     ********************************************************************/
-    ret = fh->f_generate_current_file_view ((struct ompio_file_t *) fh,
-                                            max_data,
-                                            &local_iov_array,
-                                            &local_count);
-
-    if (ret != OMPI_SUCCESS){
-        goto exit;
-    }
-
-    /*************************************************************
-     *** 4. Allgather the File View information at all processes
-     *************************************************************/
-
-    fview_count = (int *) malloc (fh->f_procs_per_group * sizeof (int));
-    if (NULL == fview_count) {
-        opal_output (1, "OUT OF MEMORY\n");
-        ret = OMPI_ERR_OUT_OF_RESOURCE;
-        goto exit;
-    }
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-    start_rcomm_time = MPI_Wtime();
-#endif
-    ret = ompi_fcoll_base_coll_allgather_array (&local_count,
-                                           1,
-                                           MPI_INT,
-                                           fview_count,
-                                           1,
-                                           MPI_INT,
-                                           0,
-                                           fh->f_procs_in_group,
-                                           fh->f_procs_per_group,
-                                           fh->f_comm);
-    
-    if (OMPI_SUCCESS != ret){
-        goto exit;
-    }
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-    end_rcomm_time = MPI_Wtime();
-    rcomm_time += end_rcomm_time - start_rcomm_time;
-#endif
-
-    displs = (int*)malloc (fh->f_procs_per_group*sizeof(int));
-    if (NULL == displs) {
-        opal_output (1, "OUT OF MEMORY\n");
-        ret = OMPI_ERR_OUT_OF_RESOURCE;
-        goto exit;
-    }
-
-    displs[0] = 0;
-    total_fview_count = fview_count[0];
-    for (i=1 ; i<fh->f_procs_per_group ; i++) {
-        total_fview_count += fview_count[i];
-        displs[i] = displs[i-1] + fview_count[i-1];
-    }
-
-#if DEBUG_ON
-    if (my_aggregator == fh->f_rank) {
-    for (i=0 ; i<fh->f_procs_per_group ; i++) {
-    printf ("%d: PROCESS: %d  ELEMENTS: %d  DISPLS: %d\n",
-        fh->f_rank,
-        i,
-        fview_count[i],
-        displs[i]);
+    return mca_common_ompio_base_file_read_all (fh, buf, count, datatype, status);
 }
-}
-#endif
-
-    /* allocate the global iovec  */
-    if (0 != total_fview_count) {
-        global_iov_array = (struct iovec*)malloc (total_fview_count *
-                                                  sizeof(struct iovec));
-        if (NULL == global_iov_array) {
-            opal_output (1, "OUT OF MEMORY\n");
-            ret = OMPI_ERR_OUT_OF_RESOURCE;
-            goto exit;
-        }
-    }
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-    start_rcomm_time = MPI_Wtime();
-#endif
-    ret =  ompi_fcoll_base_coll_allgatherv_array (local_iov_array,
-                                             local_count,
-                                             fh->f_iov_type,
-                                             global_iov_array,
-                                             fview_count,
-                                             displs,
-                                             fh->f_iov_type,
-                                             0,
-                                             fh->f_procs_in_group,
-                                             fh->f_procs_per_group,
-                                             fh->f_comm);
-    
-    if (OMPI_SUCCESS != ret){
-        goto exit;
-    }
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-    end_rcomm_time = MPI_Wtime();
-    rcomm_time += end_rcomm_time - start_rcomm_time;
-#endif
-
-    /****************************************************************************************
-     *** 5. Sort the global offset/lengths list based on the offsets.
-     *** The result of the sort operation is the 'sorted', an integer array,
-     *** which contains the indexes of the global_iov_array based on the offset.
-     *** For example, if global_iov_array[x].offset is followed by global_iov_array[y].offset
-     *** in the file, and that one is followed by global_iov_array[z].offset, than
-     *** sorted[0] = x, sorted[1]=y and sorted[2]=z;
-     ******************************************************************************************/
-    if (0 != total_fview_count) {
-       sorted = (int *)malloc (total_fview_count * sizeof(int));
-      if (NULL == sorted) {
-            opal_output (1, "OUT OF MEMORY\n");
-            ret = OMPI_ERR_OUT_OF_RESOURCE;
-            goto exit;
-        }
-        ompi_fcoll_base_sort_iovec (global_iov_array, total_fview_count, sorted);
-    }
-
-    if (NULL != local_iov_array) {
-        free (local_iov_array);
-        local_iov_array = NULL;
-    }
-
-#if DEBUG_ON
-    if (my_aggregator == fh->f_rank) {
-        for (i=0 ; i<total_fview_count ; i++) {
-            printf("%d: OFFSET: %p   LENGTH: %d\n",
-                   fh->f_rank,
-                   global_iov_array[sorted[i]].iov_base,
-                   global_iov_array[sorted[i]].iov_len);
-        }
-    }
-#endif
-
-    /*************************************************************
-     *** 6. Determine the number of cycles required to execute this
-     ***    operation
-     *************************************************************/
-    bytes_per_cycle = fh->f_bytes_per_agg;
-    cycles = ceil((double)total_bytes/bytes_per_cycle);
-
-    if ( my_aggregator == fh->f_rank) {
-      disp_index = (int *)malloc (fh->f_procs_per_group * sizeof (int));
-      if (NULL == disp_index) {
-            opal_output (1, "OUT OF MEMORY\n");
-            ret = OMPI_ERR_OUT_OF_RESOURCE;
-            goto exit;
-        }
-
-        blocklen_per_process = (int **)malloc (fh->f_procs_per_group * sizeof (int*));
-        if (NULL == blocklen_per_process) {
-            opal_output (1, "OUT OF MEMORY\n");
-            ret = OMPI_ERR_OUT_OF_RESOURCE;
-            goto exit;
-        }
-
-        displs_per_process = (MPI_Aint **)malloc (fh->f_procs_per_group * sizeof (MPI_Aint*));
-        if (NULL == displs_per_process){
-            opal_output (1, "OUT OF MEMORY\n");
-            ret = OMPI_ERR_OUT_OF_RESOURCE;
-            goto exit;
-        }
-
-        for (i=0;i<fh->f_procs_per_group;i++){
-            blocklen_per_process[i] = NULL;
-            displs_per_process[i] = NULL;
-        }
-
-	send_req = (MPI_Request *) malloc (fh->f_procs_per_group * sizeof(MPI_Request));
-	if (NULL == send_req){
-	    opal_output ( 1, "OUT OF MEMORY\n");
-	    ret = OMPI_ERR_OUT_OF_RESOURCE;
-	    goto exit;
-	}
-
-	global_buf = (char *) malloc (bytes_per_cycle);
-	if (NULL == global_buf){
-	    opal_output(1, "OUT OF MEMORY\n");
-	    ret = OMPI_ERR_OUT_OF_RESOURCE;
-	    goto exit;
-	}
-
-	sendtype = (ompi_datatype_t **) malloc (fh->f_procs_per_group * sizeof(ompi_datatype_t *));
-	if (NULL == sendtype) {
-            opal_output (1, "OUT OF MEMORY\n");
-	    ret = OMPI_ERR_OUT_OF_RESOURCE;
-	    goto exit;
-	}
-
-	for(l=0;l<fh->f_procs_per_group;l++){
-            sendtype[l] = MPI_DATATYPE_NULL;
-	}
-    }
-
-
-
-
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-    start_rexch = MPI_Wtime();
-#endif
-    n = 0;
-    bytes_remaining = 0;
-    current_index = 0;
-
-    for (index = 0; index < cycles; index++) {
-        /**********************************************************************
-         ***  7a. Getting ready for next cycle: initializing and freeing buffers
-	 **********************************************************************/
-        if (my_aggregator == fh->f_rank) {
-             if (NULL != fh->f_io_array) {
-                free (fh->f_io_array);
-                fh->f_io_array = NULL;
-            }
-            fh->f_num_of_io_entries = 0;
-
-            if (NULL != sendtype){
-                for (i =0; i< fh->f_procs_per_group; i++) {
-		    if ( MPI_DATATYPE_NULL != sendtype[i] ) {
-                        ompi_datatype_destroy(&sendtype[i]);
-                        sendtype[i] = MPI_DATATYPE_NULL;
-                    }
-		}
-            }
-
-            for(l=0;l<fh->f_procs_per_group;l++){
-                disp_index[l] =  1;
-
-                if (NULL != blocklen_per_process[l]){
-                    free(blocklen_per_process[l]);
-                    blocklen_per_process[l] = NULL;
-                }
-                if (NULL != displs_per_process[l]){
-                    free(displs_per_process[l]);
-                    displs_per_process[l] = NULL;
-                }
-                blocklen_per_process[l] = (int *) calloc (1, sizeof(int));
-                if (NULL == blocklen_per_process[l]) {
-                    opal_output (1, "OUT OF MEMORY for blocklen\n");
-                    ret = OMPI_ERR_OUT_OF_RESOURCE;
-                    goto exit;
-                }
-                displs_per_process[l] = (MPI_Aint *) calloc (1, sizeof(MPI_Aint));
-                if (NULL == displs_per_process[l]){
-                    opal_output (1, "OUT OF MEMORY for displs\n");
-                    ret = OMPI_ERR_OUT_OF_RESOURCE;
-                    goto exit;
-                }
-            }
-
-            if (NULL != sorted_file_offsets){
-                free(sorted_file_offsets);
-                sorted_file_offsets = NULL;
-            }
-
-            if(NULL != file_offsets_for_agg){
-                free(file_offsets_for_agg);
-                file_offsets_for_agg = NULL;
-            }
-            if (NULL != memory_displacements){
-                free(memory_displacements);
-                memory_displacements = NULL;
-            }
-        }  /* (my_aggregator == fh->f_rank */
-
-        /**************************************************************************
-         ***  7b. Determine the number of bytes to be actually read in this cycle
-	 **************************************************************************/
-        if (cycles-1 == index) {
-            bytes_to_read_in_cycle = total_bytes - bytes_per_cycle*index;
-        }
-        else {
-            bytes_to_read_in_cycle = bytes_per_cycle;
-        }
-
-#if DEBUG_ON
-        if (my_aggregator == fh->f_rank) {
-            printf ("****%d: CYCLE %d   Bytes %d**********\n",
-                    fh->f_rank,
-                    index,
-                    bytes_to_write_in_cycle);
-        }
-#endif
-
-        /*****************************************************************
-         *** 7c. Calculate how much data will be contributed in this cycle
-	 ***     by each process
-         *****************************************************************/
-        bytes_received = 0;
-
-        while (bytes_to_read_in_cycle) {
-            /* This next block identifies which process is the holder
-            ** of the sorted[current_index] element;
-            */
-            blocks = fview_count[0];
-            for (j=0 ; j<fh->f_procs_per_group ; j++) {
-                if (sorted[current_index] < blocks) {
-                    n = j;
-                    break;
-                }
-                else {
-                    blocks += fview_count[j+1];
-                }
-            }
-
-            if (bytes_remaining) {
-                /* Finish up a partially used buffer from the previous  cycle */
-                if (bytes_remaining <= bytes_to_read_in_cycle) {
-                    /* Data fits completely into the block */
-                    if (my_aggregator == fh->f_rank) {
-                        blocklen_per_process[n][disp_index[n] - 1] = bytes_remaining;
-                        displs_per_process[n][disp_index[n] - 1] =
-                            (ptrdiff_t)global_iov_array[sorted[current_index]].iov_base +
-                            (global_iov_array[sorted[current_index]].iov_len - bytes_remaining);
-
-                        blocklen_per_process[n] = (int *) realloc
-                            ((void *)blocklen_per_process[n], (disp_index[n]+1)*sizeof(int));
-                        displs_per_process[n] = (MPI_Aint *) realloc
-                            ((void *)displs_per_process[n], (disp_index[n]+1)*sizeof(MPI_Aint));
-                        blocklen_per_process[n][disp_index[n]] = 0;
-                        displs_per_process[n][disp_index[n]] = 0;
-                        disp_index[n] += 1;
-                    }
-                    if (fh->f_procs_in_group[n] == fh->f_rank) {
-                        bytes_received += bytes_remaining;
-                    }
-                    current_index ++;
-                    bytes_to_read_in_cycle -= bytes_remaining;
-                    bytes_remaining = 0;
-                    continue;
-                }
-                else {
-                     /* the remaining data from the previous cycle is larger than the
-                        bytes_to_write_in_cycle, so we have to segment again */
-                    if (my_aggregator == fh->f_rank) {
-                        blocklen_per_process[n][disp_index[n] - 1] = bytes_to_read_in_cycle;
-                        displs_per_process[n][disp_index[n] - 1] =
-                            (ptrdiff_t)global_iov_array[sorted[current_index]].iov_base +
-                            (global_iov_array[sorted[current_index]].iov_len
-                             - bytes_remaining);
-                    }
-                    if (fh->f_procs_in_group[n] == fh->f_rank) {
-                        bytes_received += bytes_to_read_in_cycle;
-                    }
-                    bytes_remaining -= bytes_to_read_in_cycle;
-                    bytes_to_read_in_cycle = 0;
-                    break;
-                }
-            }
-            else {
-                /* No partially used entry available, have to start a new one */
-                if (bytes_to_read_in_cycle <
-                    (MPI_Aint) global_iov_array[sorted[current_index]].iov_len) {
-                    /* This entry has more data than we can sendin one cycle */
-                    if (my_aggregator == fh->f_rank) {
-                        blocklen_per_process[n][disp_index[n] - 1] = bytes_to_read_in_cycle;
-                        displs_per_process[n][disp_index[n] - 1] =
-                            (ptrdiff_t)global_iov_array[sorted[current_index]].iov_base ;
-                    }
-
-                    if (fh->f_procs_in_group[n] == fh->f_rank) {
-                        bytes_received += bytes_to_read_in_cycle;
-                    }
-                    bytes_remaining = global_iov_array[sorted[current_index]].iov_len -
-                        bytes_to_read_in_cycle;
-                    bytes_to_read_in_cycle = 0;
-                    break;
-                }
-                else {
-                    /* Next data entry is less than bytes_to_write_in_cycle */
-                    if (my_aggregator ==  fh->f_rank) {
-                        blocklen_per_process[n][disp_index[n] - 1] =
-                            global_iov_array[sorted[current_index]].iov_len;
-                        displs_per_process[n][disp_index[n] - 1] = (ptrdiff_t)
-                            global_iov_array[sorted[current_index]].iov_base;
-                        blocklen_per_process[n] =
-                            (int *) realloc ((void *)blocklen_per_process[n], (disp_index[n]+1)*sizeof(int));
-                        displs_per_process[n] = (MPI_Aint *)realloc
-                            ((void *)displs_per_process[n], (disp_index[n]+1)*sizeof(MPI_Aint));
-                        blocklen_per_process[n][disp_index[n]] = 0;
-                        displs_per_process[n][disp_index[n]] = 0;
-                        disp_index[n] += 1;
-                    }
-                    if (fh->f_procs_in_group[n] == fh->f_rank) {
-                        bytes_received +=
-                            global_iov_array[sorted[current_index]].iov_len;
-                    }
-                    bytes_to_read_in_cycle -=
-                        global_iov_array[sorted[current_index]].iov_len;
-                    current_index ++;
-                    continue;
-                }
-            }
-        } /* end while (bytes_to_read_in_cycle) */
-
-        /*************************************************************************
-	 *** 7d. Calculate the displacement on where to put the data and allocate
-         ***     the recieve buffer (global_buf)
-	 *************************************************************************/
-        if (my_aggregator == fh->f_rank) {
-            entries_per_aggregator=0;
-            for (i=0;i<fh->f_procs_per_group; i++){
-                for (j=0;j<disp_index[i];j++){
-                    if (blocklen_per_process[i][j] > 0)
-                        entries_per_aggregator++ ;
-                }
-            }
-            if (entries_per_aggregator > 0){
-                file_offsets_for_agg = (mca_io_ompio_local_io_array *)
-                    malloc(entries_per_aggregator*sizeof(mca_io_ompio_local_io_array));
-                if (NULL == file_offsets_for_agg) {
-                    opal_output (1, "OUT OF MEMORY\n");
-                    ret = OMPI_ERR_OUT_OF_RESOURCE;
-                    goto exit;
-                }
-                sorted_file_offsets = (int *)
-                    malloc (entries_per_aggregator*sizeof(int));
-                if (NULL == sorted_file_offsets){
-                    opal_output (1, "OUT OF MEMORY\n");
-                    ret =  OMPI_ERR_OUT_OF_RESOURCE;
-                    goto exit;
-                }
-                /*Moving file offsets to an IO array!*/
-                temp_index = 0;
-                global_count = 0;
-                for (i=0;i<fh->f_procs_per_group; i++){
-                    for(j=0;j<disp_index[i];j++){
-                        if (blocklen_per_process[i][j] > 0){
-                            file_offsets_for_agg[temp_index].length =
-                                blocklen_per_process[i][j];
-                            global_count += blocklen_per_process[i][j];
-                            file_offsets_for_agg[temp_index].process_id = i;
-                            file_offsets_for_agg[temp_index].offset =
-                                displs_per_process[i][j];
-                            temp_index++;
-                        }
-                    }
-                }
-            }
-            else{
-                continue;
-            }
-
-             /* Sort the displacements for each aggregator */
-            read_heap_sort (file_offsets_for_agg,
-                            entries_per_aggregator,
-                            sorted_file_offsets);
-
-            memory_displacements = (MPI_Aint *) malloc
-                (entries_per_aggregator * sizeof(MPI_Aint));
-            memory_displacements[sorted_file_offsets[0]] = 0;
-            for (i=1; i<entries_per_aggregator; i++){
-                memory_displacements[sorted_file_offsets[i]] =
-                    memory_displacements[sorted_file_offsets[i-1]] +
-                    file_offsets_for_agg[sorted_file_offsets[i-1]].length;
-            }
-
-             /**********************************************************
-	     *** 7e. Create the io array, and pass it to fbtl
-	     *********************************************************/
-            fh->f_io_array = (mca_common_ompio_io_array_t *) malloc
-                (entries_per_aggregator * sizeof (mca_common_ompio_io_array_t));
-            if (NULL == fh->f_io_array) {
-                opal_output(1, "OUT OF MEMORY\n");
-                ret = OMPI_ERR_OUT_OF_RESOURCE;
-                goto exit;
-            }
-
-            fh->f_num_of_io_entries = 0;
-            fh->f_io_array[0].offset =
-                (IOVBASE_TYPE *)(intptr_t)file_offsets_for_agg[sorted_file_offsets[0]].offset;
-            fh->f_io_array[0].length =
-                file_offsets_for_agg[sorted_file_offsets[0]].length;
-            fh->f_io_array[0].memory_address =
-                global_buf+memory_displacements[sorted_file_offsets[0]];
-            fh->f_num_of_io_entries++;
-            for (i=1;i<entries_per_aggregator;i++){
-                if (file_offsets_for_agg[sorted_file_offsets[i-1]].offset +
-                    file_offsets_for_agg[sorted_file_offsets[i-1]].length ==
-                    file_offsets_for_agg[sorted_file_offsets[i]].offset){
-                    fh->f_io_array[fh->f_num_of_io_entries - 1].length +=
-                        file_offsets_for_agg[sorted_file_offsets[i]].length;
-                }
-                else{
-                    fh->f_io_array[fh->f_num_of_io_entries].offset =
-                        (IOVBASE_TYPE *)(intptr_t)file_offsets_for_agg[sorted_file_offsets[i]].offset;
-                    fh->f_io_array[fh->f_num_of_io_entries].length =
-                        file_offsets_for_agg[sorted_file_offsets[i]].length;
-                    fh->f_io_array[fh->f_num_of_io_entries].memory_address =
-                        global_buf+memory_displacements[sorted_file_offsets[i]];
-                    fh->f_num_of_io_entries++;
-                }
-            }
-
-
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-            start_read_time = MPI_Wtime();
-#endif
-
-            if (fh->f_num_of_io_entries) {
-                if ( 0 >  fh->f_fbtl->fbtl_preadv (fh)) {
-                    opal_output (1, "READ FAILED\n");
-                    ret = OMPI_ERROR;
-                    goto exit;
-                }
-            }
-
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-            end_read_time = MPI_Wtime();
-            read_time += end_read_time - start_read_time;
-#endif
-            /**********************************************************
-             ******************** DONE READING ************************
-             *********************************************************/
-
-            temp_disp_index = (int *)calloc (1, fh->f_procs_per_group * sizeof (int));
-            if (NULL == temp_disp_index) {
-                opal_output (1, "OUT OF MEMORY\n");
-                ret = OMPI_ERR_OUT_OF_RESOURCE;
-                goto exit;
-            }
-            for (i=0; i<entries_per_aggregator; i++){
-                temp_index =
-                    file_offsets_for_agg[sorted_file_offsets[i]].process_id;
-                displs_per_process[temp_index][temp_disp_index[temp_index]] =
-                    memory_displacements[sorted_file_offsets[i]];
-                if (temp_disp_index[temp_index] < disp_index[temp_index]){
-                    temp_disp_index[temp_index] += 1;
-                }
-                else{
-                    printf("temp_disp_index[%d]: %d is greater than disp_index[%d]: %d\n",
-                           temp_index, temp_disp_index[temp_index],
-                           temp_index, disp_index[temp_index]);
-                }
-            }
-            if (NULL != temp_disp_index){
-                free(temp_disp_index);
-                temp_disp_index = NULL;
-            }
-
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-            start_rcomm_time = MPI_Wtime();
-#endif
-            for (i=0;i<fh->f_procs_per_group;i++){
-                send_req[i] = MPI_REQUEST_NULL;
-                if ( 0 < disp_index[i] ) {
-                    ompi_datatype_create_hindexed(disp_index[i],
-                                                  blocklen_per_process[i],
-                                                  displs_per_process[i],
-                                                  MPI_BYTE,
-                                                  &sendtype[i]);
-                    ompi_datatype_commit(&sendtype[i]);
-                    ret = MCA_PML_CALL (isend(global_buf,
-                                              1,
-                                              sendtype[i],
-                                              fh->f_procs_in_group[i],
-                                              123,
-                                              MCA_PML_BASE_SEND_STANDARD,
-                                              fh->f_comm,
-                                              &send_req[i]));
-                    if(OMPI_SUCCESS != ret){
-                        goto exit;
-                    }
-                }
-            }
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-            end_rcomm_time = MPI_Wtime();
-            rcomm_time += end_rcomm_time - start_rcomm_time;
-#endif
-        }
-
-        /**********************************************************
-         *** 7f.  Scatter the Data from the readers
-         *********************************************************/
-        if ( recvbuf_is_contiguous ) {
-            receive_buf = &((char*)buf)[position];
-        }
-        else if (bytes_received) {
-            /* allocate a receive buffer and copy the data that needs
-               to be received into it in case the data is non-contigous
-               in memory */
-            receive_buf = malloc (bytes_received);
-            if (NULL == receive_buf) {
-                opal_output (1, "OUT OF MEMORY\n");
-                ret = OMPI_ERR_OUT_OF_RESOURCE;
-                goto exit;
-            }
-        }
-
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-        start_rcomm_time = MPI_Wtime();
-#endif
-        ret = MCA_PML_CALL(irecv(receive_buf,
-                                 bytes_received,
-                                 MPI_BYTE,
-                                 my_aggregator,
-                                 123,
-                                 fh->f_comm,
-                                 &recv_req));
-        if (OMPI_SUCCESS != ret){
-            goto exit;
-        }
-
-
-        if (my_aggregator == fh->f_rank){
-            ret = ompi_request_wait_all (fh->f_procs_per_group,
-                                         send_req,
-                                         MPI_STATUS_IGNORE);
-            if (OMPI_SUCCESS != ret){
-                goto exit;
-            }
-        }
-
-        ret = ompi_request_wait (&recv_req, MPI_STATUS_IGNORE);
-        if (OMPI_SUCCESS != ret){
-            goto exit;
-        }
-        position += bytes_received;
-
-        /* If data is not contigous in memory, copy the data from the
-           receive buffer into the buffer passed in */
-        if (!recvbuf_is_contiguous ) {
-            ptrdiff_t mem_address;
-            size_t remaining = 0;
-            size_t temp_position = 0;
-
-            remaining = bytes_received;
-
-            while (remaining) {
-                mem_address = (ptrdiff_t)
-                    (decoded_iov[iov_index].iov_base) + current_position;
-
-                if (remaining >=
-                    (decoded_iov[iov_index].iov_len - current_position)) {
-                    memcpy ((IOVBASE_TYPE *) mem_address,
-                            receive_buf+temp_position,
-                            decoded_iov[iov_index].iov_len - current_position);
-                    remaining = remaining -
-                        (decoded_iov[iov_index].iov_len - current_position);
-                    temp_position = temp_position +
-                        (decoded_iov[iov_index].iov_len - current_position);
-                    iov_index = iov_index + 1;
-                    current_position = 0;
-                }
-                else {
-                    memcpy ((IOVBASE_TYPE *) mem_address,
-                            receive_buf+temp_position,
-                            remaining);
-                    current_position = current_position + remaining;
-                    remaining = 0;
-                }
-            }
-
-            if (NULL != receive_buf) {
-                free (receive_buf);
-                receive_buf = NULL;
-            }
-        }
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-        end_rcomm_time = MPI_Wtime();
-        rcomm_time += end_rcomm_time - start_rcomm_time;
-#endif
-    } /* end for (index=0; index < cycles; index ++) */
-
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-    end_rexch = MPI_Wtime();
-    read_exch += end_rexch - start_rexch;
-    nentry.time[0] = read_time;
-    nentry.time[1] = rcomm_time;
-    nentry.time[2] = read_exch;
-    if (my_aggregator == fh->f_rank)
-        nentry.aggregator = 1;
-    else
-        nentry.aggregator = 0;
-    nentry.nprocs_for_coll = dynamic_num_io_procs;
-    if (!mca_common_ompio_full_print_queue(fh->f_coll_read_time)){
-        mca_common_ompio_register_print_entry(fh->f_coll_read_time,
-                                              nentry);
-    }
-#endif
-
-exit:
-    if (!recvbuf_is_contiguous) {
-        if (NULL != receive_buf) {
-            free (receive_buf);
-            receive_buf = NULL;
-        }
-    }
-    if (NULL != global_buf) {
-        free (global_buf);
-        global_buf = NULL;
-    }
-    if (NULL != sorted) {
-        free (sorted);
-        sorted = NULL;
-    }
-    if (NULL != global_iov_array) {
-        free (global_iov_array);
-        global_iov_array = NULL;
-    }
-    if (NULL != fview_count) {
-        free (fview_count);
-        fview_count = NULL;
-    }
-    if (NULL != decoded_iov) {
-        free (decoded_iov);
-        decoded_iov = NULL;
-    }
-    if (NULL != local_iov_array){
-        free(local_iov_array);
-        local_iov_array=NULL;
-    }
-
-    if (NULL != displs) {
-        free (displs);
-        displs = NULL;
-    }
-    if (my_aggregator == fh->f_rank) {
-
-        if (NULL != sorted_file_offsets){
-            free(sorted_file_offsets);
-            sorted_file_offsets = NULL;
-        }
-        if (NULL != file_offsets_for_agg){
-            free(file_offsets_for_agg);
-            file_offsets_for_agg = NULL;
-        }
-        if (NULL != memory_displacements){
-            free(memory_displacements);
-            memory_displacements= NULL;
-        }
-        if (NULL != sendtype){
-            for (i = 0; i < fh->f_procs_per_group; i++) {
-                if ( MPI_DATATYPE_NULL != sendtype[i] ) {
-                    ompi_datatype_destroy(&sendtype[i]);
-                }
-            }
-            free(sendtype);
-            sendtype=NULL;
-        }
-
-        if (NULL != disp_index){
-            free(disp_index);
-            disp_index = NULL;
-        }
-
-        if ( NULL != blocklen_per_process){
-            for(l=0;l<fh->f_procs_per_group;l++){
-                if (NULL != blocklen_per_process[l]){
-                    free(blocklen_per_process[l]);
-                    blocklen_per_process[l] = NULL;
-                }
-            }
-
-            free(blocklen_per_process);
-            blocklen_per_process = NULL;
-        }
-
-        if (NULL != displs_per_process){
-            for (l=0; i<fh->f_procs_per_group; l++){
-                if (NULL != displs_per_process[l]){
-                    free(displs_per_process[l]);
-                    displs_per_process[l] = NULL;
-                }
-            }
-            free(displs_per_process);
-            displs_per_process = NULL;
-        }
-        if ( NULL != send_req ) {
-            free ( send_req );
-            send_req = NULL;
-        }
-    }
-    return ret;
-}
-
-
-static int read_heap_sort (mca_io_ompio_local_io_array *io_array,
-                           int num_entries,
-                           int *sorted)
-{
-    int i = 0;
-    int j = 0;
-    int left = 0;
-    int right = 0;
-    int largest = 0;
-    int heap_size = num_entries - 1;
-    int temp = 0;
-    unsigned char done = 0;
-    int* temp_arr = NULL;
-
-    temp_arr = (int*)malloc(num_entries*sizeof(int));
-    if (NULL == temp_arr) {
-        opal_output (1, "OUT OF MEMORY\n");
-        return OMPI_ERR_OUT_OF_RESOURCE;
-    }
-    temp_arr[0] = 0;
-    for (i = 1; i < num_entries; ++i) {
-        temp_arr[i] = i;
-    }
-    /* num_entries can be a large no. so NO RECURSION */
-    for (i = num_entries/2-1 ; i>=0 ; i--) {
-        done = 0;
-        j = i;
-        largest = j;
-
-        while (!done) {
-            left = j*2+1;
-            right = j*2+2;
-            if ((left <= heap_size) &&
-                (io_array[temp_arr[left]].offset > io_array[temp_arr[j]].offset)) {
-                largest = left;
-            }
-            else {
-                largest = j;
-            }
-            if ((right <= heap_size) &&
-                (io_array[temp_arr[right]].offset >
-                 io_array[temp_arr[largest]].offset)) {
-                largest = right;
-            }
-            if (largest != j) {
-                temp = temp_arr[largest];
-                temp_arr[largest] = temp_arr[j];
-                temp_arr[j] = temp;
-                j = largest;
-            }
-            else {
-                done = 1;
-            }
-        }
-    }
-
-    for (i = num_entries-1; i >=1; --i) {
-        temp = temp_arr[0];
-        temp_arr[0] = temp_arr[i];
-        temp_arr[i] = temp;
-        heap_size--;
-        done = 0;
-        j = 0;
-        largest = j;
-
-        while (!done) {
-            left =  j*2+1;
-            right = j*2+2;
-
-            if ((left <= heap_size) &&
-                (io_array[temp_arr[left]].offset >
-                 io_array[temp_arr[j]].offset)) {
-                largest = left;
-            }
-            else {
-                largest = j;
-            }
-            if ((right <= heap_size) &&
-                (io_array[temp_arr[right]].offset >
-                 io_array[temp_arr[largest]].offset)) {
-                largest = right;
-            }
-            if (largest != j) {
-                temp = temp_arr[largest];
-                temp_arr[largest] = temp_arr[j];
-                temp_arr[j] = temp;
-                j = largest;
-            }
-            else {
-                done = 1;
-            }
-        }
-        sorted[i] = temp_arr[i];
-    }
-    sorted[0] = temp_arr[0];
-
-    if (NULL != temp_arr) {
-        free(temp_arr);
-        temp_arr = NULL;
-    }
-    return OMPI_SUCCESS;
-}
-
 
 
diff --git a/ompi/mca/fcoll/dynamic/fcoll_dynamic_file_write_all.c b/ompi/mca/fcoll/dynamic/fcoll_dynamic_file_write_all.c
index 5920694b4e7..2ce3ef5d27f 100644
--- a/ompi/mca/fcoll/dynamic/fcoll_dynamic_file_write_all.c
+++ b/ompi/mca/fcoll/dynamic/fcoll_dynamic_file_write_all.c
@@ -13,6 +13,9 @@
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -51,9 +54,9 @@ static int local_heap_sort (mca_io_ompio_local_io_array *io_array,
 
 
 int
-mca_fcoll_dynamic_file_write_all (ompio_file_t *fh,
+mca_fcoll_dynamic_file_write_all (struct ompio_file_t *fh,
                                   const void *buf,
-                                  int count,
+                                  size_t count,
                                   struct ompi_datatype_t *datatype,
                                   ompi_status_public_t *status)
 {
@@ -81,17 +84,20 @@ mca_fcoll_dynamic_file_write_all (ompio_file_t *fh,
     /* global iovec at the writers that contain the iovecs created from
        file_set_view */
     uint32_t total_fview_count = 0;
-    int local_count = 0, temp_pindex;
-    int *fview_count = NULL, *disp_index=NULL, *temp_disp_index=NULL;
+    size_t local_count = 0, temp_pindex;
+    int temp_local_count;
+    size_t *fview_count = NULL;
+    ptrdiff_t *disp_index=NULL, *temp_disp_index=NULL;
     int current_index = 0, temp_index=0;
 
     char *global_buf = NULL;
+#if DEBUG_ON
     MPI_Aint global_count = 0;
-
+#endif
 
     /* array that contains the sorted indices of the global_iov */
     int *sorted = NULL, *sorted_file_offsets=NULL;
-    int *displs = NULL;
+    ptrdiff_t *displs = NULL;
     int dynamic_num_io_procs;
     size_t max_data = 0, datatype_size = 0;
     int **blocklen_per_process=NULL;
@@ -116,7 +122,7 @@ mca_fcoll_dynamic_file_write_all (ompio_file_t *fh,
     opal_datatype_get_extent ( &datatype->super, &lb, &ftype_extent );
 
     /**************************************************************************
-     ** 1.  In case the data is not contigous in memory, decode it into an iovec
+     ** 1.  In case the data is not contiguous in memory, decode it into an iovec
      **************************************************************************/
     if ( ( ftype_extent == (ptrdiff_t) ftype_size)             &&
          opal_datatype_is_contiguous_memory_layout(&datatype->super,1) &&
@@ -209,10 +215,11 @@ mca_fcoll_dynamic_file_write_all (ompio_file_t *fh,
     ret = fh->f_generate_current_file_view( (struct ompio_file_t *) fh,
 					    max_data,
 					    &local_iov_array,
-					    &local_count);
+					    &temp_local_count);
     if (ret != OMPI_SUCCESS){
 	goto exit;
     }
+    local_count = temp_local_count;
 
 #if DEBUG_ON
     for (i=0 ; i<local_count ; i++) {
@@ -228,7 +235,7 @@ mca_fcoll_dynamic_file_write_all (ompio_file_t *fh,
     /*************************************************************
      *** 4. Allgather the offset/lengths array from all processes
      *************************************************************/
-    fview_count = (int *) malloc (fh->f_procs_per_group * sizeof (int));
+    fview_count = (size_t *)malloc (fh->f_procs_per_group * sizeof (size_t));
     if (NULL == fview_count) {
         opal_output (1, "OUT OF MEMORY\n");
         ret = OMPI_ERR_OUT_OF_RESOURCE;
@@ -238,11 +245,11 @@ mca_fcoll_dynamic_file_write_all (ompio_file_t *fh,
     start_comm_time = MPI_Wtime();
 #endif
     ret = ompi_fcoll_base_coll_allgather_array (&local_count,
-                                           1,
-                                           MPI_INT,
+                                           sizeof(size_t),
+                                           MPI_BYTE,
                                            fview_count,
-                                           1,
-                                           MPI_INT,
+                                           sizeof(size_t),
+                                           MPI_BYTE,
                                            0,
                                            fh->f_procs_in_group,
                                            fh->f_procs_per_group,
@@ -256,7 +263,7 @@ mca_fcoll_dynamic_file_write_all (ompio_file_t *fh,
     comm_time += (end_comm_time - start_comm_time);
 #endif
 
-    displs = (int*) malloc (fh->f_procs_per_group * sizeof (int));
+    displs = (ptrdiff_t *)malloc (fh->f_procs_per_group * sizeof (ptrdiff_t));
     if (NULL == displs) {
         opal_output (1, "OUT OF MEMORY\n");
         ret = OMPI_ERR_OUT_OF_RESOURCE;
@@ -274,11 +281,11 @@ mca_fcoll_dynamic_file_write_all (ompio_file_t *fh,
     printf("total_fview_count : %d\n", total_fview_count);
     if (my_aggregator == fh->f_rank) {
         for (i=0 ; i<fh->f_procs_per_group ; i++) {
-            printf ("%d: PROCESS: %d  ELEMENTS: %d  DISPLS: %d\n",
+            printf ("%d: PROCESS: %d  ELEMENTS: %zu  DISPLS: %ld\n",
                     fh->f_rank,
                     i,
                     fview_count[i],
-                    displs[i]);
+                    (long) displs[i]);
         }
     }
 #endif
@@ -366,7 +373,7 @@ mca_fcoll_dynamic_file_write_all (ompio_file_t *fh,
     cycles = ceil((double)total_bytes/bytes_per_cycle);
 
     if (my_aggregator == fh->f_rank) {
-        disp_index = (int *)malloc (fh->f_procs_per_group * sizeof (int));
+        disp_index = (ptrdiff_t *)malloc (fh->f_procs_per_group * sizeof (ptrdiff_t));
         if (NULL == disp_index) {
             opal_output (1, "OUT OF MEMORY\n");
             ret = OMPI_ERR_OUT_OF_RESOURCE;
@@ -622,7 +629,7 @@ mca_fcoll_dynamic_file_write_all (ompio_file_t *fh,
 
         /*************************************************************************
 	 *** 7d. Calculate the displacement on where to put the data and allocate
-         ***     the recieve buffer (global_buf)
+         ***     the receive buffer (global_buf)
 	 *************************************************************************/
         if (my_aggregator == fh->f_rank) {
             entries_per_aggregator=0;
@@ -705,7 +712,7 @@ mca_fcoll_dynamic_file_write_all (ompio_file_t *fh,
                     file_offsets_for_agg[sorted_file_offsets[i-1]].length;
             }
 
-            temp_disp_index = (int *)calloc (1, fh->f_procs_per_group * sizeof (int));
+            temp_disp_index = (ptrdiff_t *)calloc (1, fh->f_procs_per_group * sizeof (ptrdiff_t));
             if (NULL == temp_disp_index) {
                 opal_output (1, "OUT OF MEMORY\n");
                 ret = OMPI_ERR_OUT_OF_RESOURCE;
@@ -713,7 +720,9 @@ mca_fcoll_dynamic_file_write_all (ompio_file_t *fh,
             }
 
             /*Now update the displacements array  with memory offsets*/
+#if DEBUG_ON
             global_count = 0;
+#endif
             for (i=0;i<entries_per_aggregator;i++){
                 temp_pindex =
                     file_offsets_for_agg[sorted_file_offsets[i]].process_id;
@@ -722,12 +731,14 @@ mca_fcoll_dynamic_file_write_all (ompio_file_t *fh,
                 if (temp_disp_index[temp_pindex] < disp_index[temp_pindex])
                     temp_disp_index[temp_pindex] += 1;
                 else{
-                    printf("temp_disp_index[%d]: %d is greater than disp_index[%d]: %d\n",
-                           temp_pindex, temp_disp_index[temp_pindex],
-                           temp_pindex, disp_index[temp_pindex]);
+                    printf("temp_disp_index[%zu]: %ld is greater than disp_index[%zu]: %ld\n",
+                           temp_pindex, (long) temp_disp_index[temp_pindex],
+                           temp_pindex, (long) disp_index[temp_pindex]);
                 }
+#if DEBUG_ON
                 global_count +=
                     file_offsets_for_agg[sorted_file_offsets[i]].length;
+#endif
             }
 
             if (NULL != temp_disp_index){
@@ -802,7 +813,7 @@ mca_fcoll_dynamic_file_write_all (ompio_file_t *fh,
         }
         else if (bytes_sent) {
             /* allocate a send buffer and copy the data that needs
-               to be sent into it in case the data is non-contigous
+               to be sent into it in case the data is non-contiguous
                in memory */
             ptrdiff_t mem_address;
             size_t remaining = 0;
@@ -844,7 +855,7 @@ mca_fcoll_dynamic_file_write_all (ompio_file_t *fh,
 	}
 	total_bytes_written += bytes_sent;
 
-	/* Gather the sendbuf from each process in appropritate locations in
+	/* Gather the sendbuf from each process in appropriate locations in
            aggregators*/
 
 	if (bytes_sent){
@@ -927,7 +938,7 @@ mca_fcoll_dynamic_file_write_all (ompio_file_t *fh,
             fh->f_num_of_io_entries++;
 
             for (i=1;i<entries_per_aggregator;i++){
-                /* If the enrties are contiguous merge them,
+                /* If the entries are contiguous merge them,
                    else make a new entry */
                 if (file_offsets_for_agg[sorted_file_offsets[i-1]].offset +
                     file_offsets_for_agg[sorted_file_offsets[i-1]].length ==
diff --git a/ompi/mca/fcoll/dynamic/fcoll_dynamic_module.c b/ompi/mca/fcoll/dynamic/fcoll_dynamic_module.c
index 96b88a3f849..48b94e0ffd9 100644
--- a/ompi/mca/fcoll/dynamic/fcoll_dynamic_module.c
+++ b/ompi/mca/fcoll/dynamic/fcoll_dynamic_module.c
@@ -9,9 +9,11 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2008-2015 University of Houston. All rights reserved.
+ * Copyright (c) 2008-2022 University of Houston. All rights reserved.
  * Copyright (c) 2018      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -34,7 +36,7 @@
  * ************************ actions structure ************************
  * *******************************************************************
  */
-static mca_fcoll_base_module_1_0_0_t dynamic =  {
+static mca_fcoll_base_module_2_0_0_t dynamic =  {
     mca_fcoll_dynamic_module_init,
     mca_fcoll_dynamic_module_finalize,
     mca_fcoll_dynamic_file_read_all,
@@ -54,7 +56,7 @@ mca_fcoll_dynamic_component_init_query(bool enable_progress_threads,
     return OMPI_SUCCESS;
 }
 
-mca_fcoll_base_module_1_0_0_t *
+mca_fcoll_base_module_2_0_0_t *
 mca_fcoll_dynamic_component_file_query (ompio_file_t *fh, int *priority)
 {
     *priority = mca_fcoll_dynamic_priority;
diff --git a/ompi/mca/fcoll/dynamic_gen2/Makefile.am b/ompi/mca/fcoll/dynamic_gen2/Makefile.am
index a4e91e5cd7e..5328f3b5a83 100644
--- a/ompi/mca/fcoll/dynamic_gen2/Makefile.am
+++ b/ompi/mca/fcoll/dynamic_gen2/Makefile.am
@@ -9,7 +9,7 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2008-2015 University of Houston. All rights reserved.
+# Copyright (c) 2008-2022 University of Houston. All rights reserved.
 # Copyright (c) 2012      Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
 # Copyright (c) 2018      Research Organization for Information Science
diff --git a/ompi/mca/fcoll/dynamic_gen2/fcoll_dynamic_gen2.h b/ompi/mca/fcoll/dynamic_gen2/fcoll_dynamic_gen2.h
index d0767a3046f..2221845fcd5 100644
--- a/ompi/mca/fcoll/dynamic_gen2/fcoll_dynamic_gen2.h
+++ b/ompi/mca/fcoll/dynamic_gen2/fcoll_dynamic_gen2.h
@@ -9,9 +9,11 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2008-2021 University of Houston. All rights reserved.
+ * Copyright (c) 2008-2022 University of Houston. All rights reserved.
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -40,13 +42,13 @@ BEGIN_C_DECLS
 extern int mca_fcoll_dynamic_gen2_priority;
 extern int mca_fcoll_dynamic_gen2_num_groups;
 
-OMPI_MODULE_DECLSPEC extern mca_fcoll_base_component_2_0_0_t mca_fcoll_dynamic_gen2_component;
+OMPI_DECLSPEC extern mca_fcoll_base_component_3_0_0_t mca_fcoll_dynamic_gen2_component;
 
 /* API functions */
 
 int mca_fcoll_dynamic_gen2_component_init_query(bool enable_progress_threads,
                                            bool enable_mpi_threads);
-struct mca_fcoll_base_module_1_0_0_t *
+struct mca_fcoll_base_module_2_0_0_t *
 mca_fcoll_dynamic_gen2_component_file_query (ompio_file_t *fh, int *priority);
 
 int mca_fcoll_dynamic_gen2_component_file_unquery (ompio_file_t *file);
@@ -54,18 +56,17 @@ int mca_fcoll_dynamic_gen2_component_file_unquery (ompio_file_t *file);
 int mca_fcoll_dynamic_gen2_module_init (ompio_file_t *file);
 int mca_fcoll_dynamic_gen2_module_finalize (ompio_file_t *file);
 
-int mca_fcoll_dynamic_gen2_file_read_all (ompio_file_t *fh,
-                                     void *buf,
-                                     int count,
-                                     struct ompi_datatype_t *datatype,
-                                     ompi_status_public_t * status);
-
-
-int mca_fcoll_dynamic_gen2_file_write_all (ompio_file_t *fh,
+int mca_fcoll_dynamic_gen2_file_write_all (struct ompio_file_t *fh,
                                       const void *buf,
-                                      int count,
+                                      size_t count,
                                       struct ompi_datatype_t *datatype,
                                       ompi_status_public_t * status);
+int mca_fcoll_dynamic_gen2_file_read_all (struct ompio_file_t *fh,
+                                          void *buf,
+                                          size_t count,
+                                          struct ompi_datatype_t *datatype,
+                                          ompi_status_public_t * status);
+
 
 
 END_C_DECLS
diff --git a/ompi/mca/fcoll/dynamic_gen2/fcoll_dynamic_gen2_component.c b/ompi/mca/fcoll/dynamic_gen2/fcoll_dynamic_gen2_component.c
index 688a70138a1..0b51382b2c1 100644
--- a/ompi/mca/fcoll/dynamic_gen2/fcoll_dynamic_gen2_component.c
+++ b/ompi/mca/fcoll/dynamic_gen2/fcoll_dynamic_gen2_component.c
@@ -14,6 +14,8 @@
  * Copyright (c) 2008-2020 University of Houston. All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -52,13 +54,13 @@ static int dynamic_gen2_register(void);
  * Instantiate the public struct with all of our public information
  * and pointers to our public functions in it
  */
-mca_fcoll_base_component_2_0_0_t mca_fcoll_dynamic_gen2_component = {
+mca_fcoll_base_component_3_0_0_t mca_fcoll_dynamic_gen2_component = {
 
     /* First, the mca_component_t struct containing meta information
      * about the component itself */
 
     .fcollm_version = {
-        MCA_FCOLL_BASE_VERSION_2_0_0,
+        MCA_FCOLL_BASE_VERSION_3_0_0,
 
         /* Component name and version */
         .mca_component_name = "dynamic_gen2",
diff --git a/ompi/mca/fcoll/dynamic_gen2/fcoll_dynamic_gen2_file_read_all.c b/ompi/mca/fcoll/dynamic_gen2/fcoll_dynamic_gen2_file_read_all.c
index d64f3127a94..ea21fbf3f7e 100644
--- a/ompi/mca/fcoll/dynamic_gen2/fcoll_dynamic_gen2_file_read_all.c
+++ b/ompi/mca/fcoll/dynamic_gen2/fcoll_dynamic_gen2_file_read_all.c
@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2017 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -10,9 +10,12 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2008-2021 University of Houston. All rights reserved.
- * Copyright (c) 2017-2018 Research Organization for Information Science
+ * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2018      Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -24,1059 +27,16 @@
 #include "fcoll_dynamic_gen2.h"
 
 #include "mpi.h"
-#include "ompi/constants.h"
-#include "ompi/mca/fcoll/fcoll.h"
-#include "ompi/mca/fcoll/base/fcoll_base_coll_array.h"
 #include "ompi/mca/common/ompio/common_ompio.h"
-#include "ompi/mca/io/io.h"
-#include "math.h"
-#include "ompi/mca/pml/pml.h"
-#include <unistd.h>
-
-#define DEBUG_ON 0
-
-/*Used for loading file-offsets per aggregator*/
-typedef struct mca_io_ompio_local_io_array{
-    OMPI_MPI_OFFSET_TYPE offset;
-    MPI_Aint             length;
-    int                  process_id;
-}mca_io_ompio_local_io_array;
-
-
-static int read_heap_sort (mca_io_ompio_local_io_array *io_array,
-                           int num_entries,
-                           int *sorted);
-
-
-
-int
-mca_fcoll_dynamic_gen2_file_read_all (ompio_file_t *fh,
-                                 void *buf,
-                                 int count,
-                                 struct ompi_datatype_t *datatype,
-                                 ompi_status_public_t *status)
-{
-    MPI_Aint total_bytes = 0;          /* total bytes to be read */
-    MPI_Aint bytes_to_read_in_cycle = 0; /* left to be read in a cycle*/
-    MPI_Aint bytes_per_cycle = 0;      /* total read in each cycle by each process*/
-    int index = 0, ret=OMPI_SUCCESS;
-    int cycles = 0;
-    int i=0, j=0, l=0;
-    int n=0; /* current position in total_bytes_per_process array */
-    MPI_Aint bytes_remaining = 0; /* how many bytes have been read from the current
-                                     value from total_bytes_per_process */
-    int *sorted_file_offsets=NULL, entries_per_aggregator=0;
-    int bytes_received = 0;
-    int blocks = 0;
-    /* iovec structure and count of the buffer passed in */
-    uint32_t iov_count = 0;
-    struct iovec *decoded_iov = NULL;
-    int iov_index = 0;
-    size_t current_position = 0;
-    struct iovec *local_iov_array=NULL, *global_iov_array=NULL;
-    MPI_Aint *memory_displacements=NULL;
-    /* global iovec at the readers that contain the iovecs created from
-       file_set_view */
-    uint32_t total_fview_count = 0;
-    int local_count = 0;
-    int *fview_count = NULL, *disp_index=NULL, *temp_disp_index=NULL;
-    int current_index=0, temp_index=0;
-    int **blocklen_per_process=NULL;
-    MPI_Aint **displs_per_process=NULL;
-    char *global_buf = NULL;
-    MPI_Aint global_count = 0;
-    mca_io_ompio_local_io_array *file_offsets_for_agg=NULL;
-
-    /* array that contains the sorted indices of the global_iov */
-    int *sorted = NULL;
-    int *displs = NULL;
-    int dynamic_gen2_num_io_procs;
-    size_t max_data = 0;
-    MPI_Aint *total_bytes_per_process = NULL;
-    ompi_datatype_t **sendtype = NULL;
-    MPI_Request *send_req = NULL;
-    MPI_Request recv_req = MPI_REQUEST_NULL;
-    int my_aggregator =-1;
-
-    int* blocklength_proc       = NULL;
-    ptrdiff_t* displs_proc      = NULL;
-
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-    double read_time = 0.0, start_read_time = 0.0, end_read_time = 0.0;
-    double rcomm_time = 0.0, start_rcomm_time = 0.0, end_rcomm_time = 0.0;
-    double read_exch = 0.0, start_rexch = 0.0, end_rexch = 0.0;
-    mca_common_ompio_print_entry nentry;
-#endif
-
-    /**************************************************************************
-     ** 1. In case the data is not contigous in memory, decode it into an iovec
-     **************************************************************************/
-    ret = mca_common_ompio_decode_datatype ((struct ompio_file_t *)fh,
-                                            datatype,
-                                            count,
-                                            buf,
-                                            &max_data,
-                                            fh->f_mem_convertor,
-                                            &decoded_iov,
-                                            &iov_count);
-    if (OMPI_SUCCESS != ret){
-        goto exit;
-    }
-
-    if ( MPI_STATUS_IGNORE != status ) {
-        status->_ucount = max_data;
-    }
-
-    dynamic_gen2_num_io_procs = fh->f_get_mca_parameter_value ( "num_aggregators", strlen ("num_aggregators"));
-    if ( OMPI_ERR_MAX == dynamic_gen2_num_io_procs ) {
-        ret = OMPI_ERROR;
-        goto exit;
-    }
-    ret = mca_common_ompio_set_aggregator_props ((struct ompio_file_t *) fh,
-                                                 dynamic_gen2_num_io_procs,
-                                                 max_data);
-    if (OMPI_SUCCESS != ret){
-        goto exit;
-    }
-    my_aggregator = fh->f_procs_in_group[0];
-
-    /**************************************************************************
-     ** 2. Determine the total amount of data to be written
-     **************************************************************************/
-    total_bytes_per_process = (MPI_Aint*)malloc(fh->f_procs_per_group*sizeof(MPI_Aint));
-    if (NULL == total_bytes_per_process) {
-        opal_output (1, "OUT OF MEMORY\n");
-        ret = OMPI_ERR_OUT_OF_RESOURCE;
-        goto exit;
-    }
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-    start_rcomm_time = MPI_Wtime();
-#endif
-    ret = ompi_fcoll_base_coll_allgather_array (&max_data,
-                                           1,
-                                           MPI_LONG,
-                                           total_bytes_per_process,
-                                           1,
-                                           MPI_LONG,
-                                           0,
-                                           fh->f_procs_in_group,
-                                           fh->f_procs_per_group,
-                                           fh->f_comm);
-    if (OMPI_SUCCESS != ret){
-        goto exit;
-    }
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-    end_rcomm_time = MPI_Wtime();
-    rcomm_time += end_rcomm_time - start_rcomm_time;
-#endif
-
-    for (i=0 ; i<fh->f_procs_per_group ; i++) {
-        total_bytes += total_bytes_per_process[i];
-    }
-
-    if (NULL != total_bytes_per_process) {
-        free (total_bytes_per_process);
-        total_bytes_per_process = NULL;
-    }
-
-    /*********************************************************************
-     *** 3. Generate the File offsets/lengths corresponding to this write
-     ********************************************************************/
-    ret = fh->f_generate_current_file_view ((struct ompio_file_t *) fh,
-                                            max_data,
-                                            &local_iov_array,
-                                            &local_count);
-
-    if (ret != OMPI_SUCCESS){
-        goto exit;
-    }
-
-    /*************************************************************
-     *** 4. Allgather the File View information at all processes
-     *************************************************************/
-
-    fview_count = (int *) malloc (fh->f_procs_per_group * sizeof (int));
-    if (NULL == fview_count) {
-        opal_output (1, "OUT OF MEMORY\n");
-        ret = OMPI_ERR_OUT_OF_RESOURCE;
-        goto exit;
-    }
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-    start_rcomm_time = MPI_Wtime();
-#endif
-    ret = ompi_fcoll_base_coll_allgather_array (&local_count,
-                                           1,
-                                           MPI_INT,
-                                           fview_count,
-                                           1,
-                                           MPI_INT,
-                                           0,
-                                           fh->f_procs_in_group,
-                                           fh->f_procs_per_group,
-                                           fh->f_comm);
-    
-    if (OMPI_SUCCESS != ret){
-        goto exit;
-    }
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-    end_rcomm_time = MPI_Wtime();
-    rcomm_time += end_rcomm_time - start_rcomm_time;
-#endif
-
-    displs = (int*)malloc (fh->f_procs_per_group*sizeof(int));
-    if (NULL == displs) {
-        opal_output (1, "OUT OF MEMORY\n");
-        ret = OMPI_ERR_OUT_OF_RESOURCE;
-        goto exit;
-    }
-
-    displs[0] = 0;
-    total_fview_count = fview_count[0];
-    for (i=1 ; i<fh->f_procs_per_group ; i++) {
-        total_fview_count += fview_count[i];
-        displs[i] = displs[i-1] + fview_count[i-1];
-    }
-
-#if DEBUG_ON
-    if (my_aggregator == fh->f_rank) {
-    for (i=0 ; i<fh->f_procs_per_group ; i++) {
-    printf ("%d: PROCESS: %d  ELEMENTS: %d  DISPLS: %d\n",
-        fh->f_rank,
-        i,
-        fview_count[i],
-        displs[i]);
-}
-}
-#endif
-
-    /* allocate the global iovec  */
-    if (0 != total_fview_count) {
-        global_iov_array = (struct iovec*)malloc (total_fview_count *
-                                                  sizeof(struct iovec));
-        if (NULL == global_iov_array) {
-            opal_output (1, "OUT OF MEMORY\n");
-            ret = OMPI_ERR_OUT_OF_RESOURCE;
-            goto exit;
-        }
-    }
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-    start_rcomm_time = MPI_Wtime();
-#endif
-    ret =  ompi_fcoll_base_coll_allgatherv_array (local_iov_array,
-                                             local_count,
-                                             fh->f_iov_type,
-                                             global_iov_array,
-                                             fview_count,
-                                             displs,
-                                             fh->f_iov_type,
-                                             0,
-                                             fh->f_procs_in_group,
-                                             fh->f_procs_per_group,
-                                             fh->f_comm);
-
-    if (OMPI_SUCCESS != ret){
-        goto exit;
-    }
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-    end_rcomm_time = MPI_Wtime();
-    rcomm_time += end_rcomm_time - start_rcomm_time;
-#endif
-
-    /****************************************************************************************
-     *** 5. Sort the global offset/lengths list based on the offsets.
-     *** The result of the sort operation is the 'sorted', an integer array,
-     *** which contains the indexes of the global_iov_array based on the offset.
-     *** For example, if global_iov_array[x].offset is followed by global_iov_array[y].offset
-     *** in the file, and that one is followed by global_iov_array[z].offset, than
-     *** sorted[0] = x, sorted[1]=y and sorted[2]=z;
-     ******************************************************************************************/
-    if (0 != total_fview_count) {
-       sorted = (int *)malloc (total_fview_count * sizeof(int));
-      if (NULL == sorted) {
-            opal_output (1, "OUT OF MEMORY\n");
-            ret = OMPI_ERR_OUT_OF_RESOURCE;
-            goto exit;
-        }
-        ompi_fcoll_base_sort_iovec (global_iov_array, total_fview_count, sorted);
-    }
-
-    if (NULL != local_iov_array) {
-        free (local_iov_array);
-        local_iov_array = NULL;
-    }
-
-#if DEBUG_ON
-    if (my_aggregator == fh->f_rank) {
-        for (i=0 ; i<total_fview_count ; i++) {
-            printf("%d: OFFSET: %p   LENGTH: %d\n",
-                   fh->f_rank,
-                   global_iov_array[sorted[i]].iov_base,
-                   global_iov_array[sorted[i]].iov_len);
-        }
-    }
-#endif
-
-    /*************************************************************
-     *** 6. Determine the number of cycles required to execute this
-     ***    operation
-     *************************************************************/
-    bytes_per_cycle = fh->f_bytes_per_agg;
-    cycles = ceil((double)total_bytes/bytes_per_cycle);
-
-    if ( my_aggregator == fh->f_rank) {
-      disp_index = (int *)malloc (fh->f_procs_per_group * sizeof (int));
-      if (NULL == disp_index) {
-            opal_output (1, "OUT OF MEMORY\n");
-            ret = OMPI_ERR_OUT_OF_RESOURCE;
-            goto exit;
-        }
-
-        blocklen_per_process = (int **)malloc (fh->f_procs_per_group * sizeof (int*));
-        if (NULL == blocklen_per_process) {
-            opal_output (1, "OUT OF MEMORY\n");
-            ret = OMPI_ERR_OUT_OF_RESOURCE;
-            goto exit;
-        }
-
-        displs_per_process = (MPI_Aint **)malloc (fh->f_procs_per_group * sizeof (MPI_Aint*));
-        if (NULL == displs_per_process){
-            opal_output (1, "OUT OF MEMORY\n");
-            ret = OMPI_ERR_OUT_OF_RESOURCE;
-            goto exit;
-        }
-
-        for (i=0;i<fh->f_procs_per_group;i++){
-            blocklen_per_process[i] = NULL;
-            displs_per_process[i] = NULL;
-        }
-
-	send_req = (MPI_Request *) malloc (fh->f_procs_per_group * sizeof(MPI_Request));
-	if (NULL == send_req){
-	    opal_output ( 1, "OUT OF MEMORY\n");
-	    ret = OMPI_ERR_OUT_OF_RESOURCE;
-	    goto exit;
-	}
-
-	global_buf = (char *) malloc (bytes_per_cycle);
-	if (NULL == global_buf){
-	    opal_output(1, "OUT OF MEMORY\n");
-	    ret = OMPI_ERR_OUT_OF_RESOURCE;
-	    goto exit;
-	}
-
-	sendtype = (ompi_datatype_t **) malloc (fh->f_procs_per_group * sizeof(ompi_datatype_t *));
-	if (NULL == sendtype) {
-            opal_output (1, "OUT OF MEMORY\n");
-	    ret = OMPI_ERR_OUT_OF_RESOURCE;
-	    goto exit;
-	}
-
-	for(l=0;l<fh->f_procs_per_group;l++){
-            sendtype[l] = MPI_DATATYPE_NULL;
-	}
-    }
-
-
-
-
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-    start_rexch = MPI_Wtime();
-#endif
-    n = 0;
-    bytes_remaining = 0;
-    current_index = 0;
-
-    for (index = 0; index < cycles; index++) {
-        /**********************************************************************
-         ***  7a. Getting ready for next cycle: initializing and freeing buffers
-	 **********************************************************************/
-        if (my_aggregator == fh->f_rank) {
-             if (NULL != fh->f_io_array) {
-                free (fh->f_io_array);
-                fh->f_io_array = NULL;
-            }
-            fh->f_num_of_io_entries = 0;
-
-            if (NULL != sendtype){
-                for (i =0; i< fh->f_procs_per_group; i++) {
-		    if ( MPI_DATATYPE_NULL != sendtype[i] ) {
-                        ompi_datatype_destroy(&sendtype[i]);
-                        sendtype[i] = MPI_DATATYPE_NULL;
-                    }
-		}
-            }
-
-            for(l=0;l<fh->f_procs_per_group;l++){
-                disp_index[l] =  1;
-
-                if (NULL != blocklen_per_process[l]){
-                    free(blocklen_per_process[l]);
-                    blocklen_per_process[l] = NULL;
-                }
-                if (NULL != displs_per_process[l]){
-                    free(displs_per_process[l]);
-                    displs_per_process[l] = NULL;
-                }
-                blocklen_per_process[l] = (int *) calloc (1, sizeof(int));
-                if (NULL == blocklen_per_process[l]) {
-                    opal_output (1, "OUT OF MEMORY for blocklen\n");
-                    ret = OMPI_ERR_OUT_OF_RESOURCE;
-                    goto exit;
-                }
-                displs_per_process[l] = (MPI_Aint *) calloc (1, sizeof(MPI_Aint));
-                if (NULL == displs_per_process[l]){
-                    opal_output (1, "OUT OF MEMORY for displs\n");
-                    ret = OMPI_ERR_OUT_OF_RESOURCE;
-                    goto exit;
-                }
-            }
-
-            if (NULL != sorted_file_offsets){
-                free(sorted_file_offsets);
-                sorted_file_offsets = NULL;
-            }
-
-            if(NULL != file_offsets_for_agg){
-                free(file_offsets_for_agg);
-                file_offsets_for_agg = NULL;
-            }
-            if (NULL != memory_displacements){
-                free(memory_displacements);
-                memory_displacements = NULL;
-            }
-        }  /* (my_aggregator == fh->f_rank */
-
-        /**************************************************************************
-         ***  7b. Determine the number of bytes to be actually read in this cycle
-	 **************************************************************************/
-        if (cycles-1 == index) {
-            bytes_to_read_in_cycle = total_bytes - bytes_per_cycle*index;
-        }
-        else {
-            bytes_to_read_in_cycle = bytes_per_cycle;
-        }
 
-#if DEBUG_ON
-        if (my_aggregator == fh->f_rank) {
-            printf ("****%d: CYCLE %d   Bytes %d**********\n",
-                    fh->f_rank,
-                    index,
-                    bytes_to_write_in_cycle);
-        }
-#endif
 
-        /*****************************************************************
-         *** 7c. Calculate how much data will be contributed in this cycle
-	 ***     by each process
-         *****************************************************************/
-        bytes_received = 0;
 
-        while (bytes_to_read_in_cycle) {
-            /* This next block identifies which process is the holder
-            ** of the sorted[current_index] element;
-            */
-            blocks = fview_count[0];
-            for (j=0 ; j<fh->f_procs_per_group ; j++) {
-                if (sorted[current_index] < blocks) {
-                    n = j;
-                    break;
-                }
-                else {
-                    blocks += fview_count[j+1];
-                }
-            }
-
-            if (bytes_remaining) {
-                /* Finish up a partially used buffer from the previous  cycle */
-                if (bytes_remaining <= bytes_to_read_in_cycle) {
-                    /* Data fits completely into the block */
-                    if (my_aggregator == fh->f_rank) {
-                        blocklen_per_process[n][disp_index[n] - 1] = bytes_remaining;
-                        displs_per_process[n][disp_index[n] - 1] =
-                            (ptrdiff_t)global_iov_array[sorted[current_index]].iov_base +
-                            (global_iov_array[sorted[current_index]].iov_len - bytes_remaining);
-
-                        blocklen_per_process[n] = (int *) realloc
-                            ((void *)blocklen_per_process[n], (disp_index[n]+1)*sizeof(int));
-                        displs_per_process[n] = (MPI_Aint *) realloc
-                            ((void *)displs_per_process[n], (disp_index[n]+1)*sizeof(MPI_Aint));
-                        blocklen_per_process[n][disp_index[n]] = 0;
-                        displs_per_process[n][disp_index[n]] = 0;
-                        disp_index[n] += 1;
-                    }
-                    if (fh->f_procs_in_group[n] == fh->f_rank) {
-                        bytes_received += bytes_remaining;
-                    }
-                    current_index ++;
-                    bytes_to_read_in_cycle -= bytes_remaining;
-                    bytes_remaining = 0;
-                    continue;
-                }
-                else {
-                     /* the remaining data from the previous cycle is larger than the
-                        bytes_to_write_in_cycle, so we have to segment again */
-                    if (my_aggregator == fh->f_rank) {
-                        blocklen_per_process[n][disp_index[n] - 1] = bytes_to_read_in_cycle;
-                        displs_per_process[n][disp_index[n] - 1] =
-                            (ptrdiff_t)global_iov_array[sorted[current_index]].iov_base +
-                            (global_iov_array[sorted[current_index]].iov_len
-                             - bytes_remaining);
-                    }
-                    if (fh->f_procs_in_group[n] == fh->f_rank) {
-                        bytes_received += bytes_to_read_in_cycle;
-                    }
-                    bytes_remaining -= bytes_to_read_in_cycle;
-                    bytes_to_read_in_cycle = 0;
-                    break;
-                }
-            }
-            else {
-                /* No partially used entry available, have to start a new one */
-                if (bytes_to_read_in_cycle <
-                    (MPI_Aint) global_iov_array[sorted[current_index]].iov_len) {
-                    /* This entry has more data than we can sendin one cycle */
-                    if (my_aggregator == fh->f_rank) {
-                        blocklen_per_process[n][disp_index[n] - 1] = bytes_to_read_in_cycle;
-                        displs_per_process[n][disp_index[n] - 1] =
-                            (ptrdiff_t)global_iov_array[sorted[current_index]].iov_base ;
-                    }
-
-                    if (fh->f_procs_in_group[n] == fh->f_rank) {
-                        bytes_received += bytes_to_read_in_cycle;
-                    }
-                    bytes_remaining = global_iov_array[sorted[current_index]].iov_len -
-                        bytes_to_read_in_cycle;
-                    bytes_to_read_in_cycle = 0;
-                    break;
-                }
-                else {
-                    /* Next data entry is less than bytes_to_write_in_cycle */
-                    if (my_aggregator ==  fh->f_rank) {
-                        blocklen_per_process[n][disp_index[n] - 1] =
-                            global_iov_array[sorted[current_index]].iov_len;
-                        displs_per_process[n][disp_index[n] - 1] = (ptrdiff_t)
-                            global_iov_array[sorted[current_index]].iov_base;
-                        blocklen_per_process[n] =
-                            (int *) realloc ((void *)blocklen_per_process[n], (disp_index[n]+1)*sizeof(int));
-                        displs_per_process[n] = (MPI_Aint *)realloc
-                            ((void *)displs_per_process[n], (disp_index[n]+1)*sizeof(MPI_Aint));
-                        blocklen_per_process[n][disp_index[n]] = 0;
-                        displs_per_process[n][disp_index[n]] = 0;
-                        disp_index[n] += 1;
-                    }
-                    if (fh->f_procs_in_group[n] == fh->f_rank) {
-                        bytes_received +=
-                            global_iov_array[sorted[current_index]].iov_len;
-                    }
-                    bytes_to_read_in_cycle -=
-                        global_iov_array[sorted[current_index]].iov_len;
-                    current_index ++;
-                    continue;
-                }
-            }
-        } /* end while (bytes_to_read_in_cycle) */
-
-        /*************************************************************************
-	 *** 7d. Calculate the displacement on where to put the data and allocate
-         ***     the recieve buffer (global_buf)
-	 *************************************************************************/
-        if (my_aggregator == fh->f_rank) {
-            entries_per_aggregator=0;
-            for (i=0;i<fh->f_procs_per_group; i++){
-                for (j=0;j<disp_index[i];j++){
-                    if (blocklen_per_process[i][j] > 0)
-                        entries_per_aggregator++ ;
-                }
-            }
-            if (entries_per_aggregator > 0){
-                file_offsets_for_agg = (mca_io_ompio_local_io_array *)
-                    malloc(entries_per_aggregator*sizeof(mca_io_ompio_local_io_array));
-                if (NULL == file_offsets_for_agg) {
-                    opal_output (1, "OUT OF MEMORY\n");
-                    ret = OMPI_ERR_OUT_OF_RESOURCE;
-                    goto exit;
-                }
-                sorted_file_offsets = (int *)
-                    malloc (entries_per_aggregator*sizeof(int));
-                if (NULL == sorted_file_offsets){
-                    opal_output (1, "OUT OF MEMORY\n");
-                    ret =  OMPI_ERR_OUT_OF_RESOURCE;
-                    goto exit;
-                }
-                /*Moving file offsets to an IO array!*/
-                temp_index = 0;
-                global_count = 0;
-                for (i=0;i<fh->f_procs_per_group; i++){
-                    for(j=0;j<disp_index[i];j++){
-                        if (blocklen_per_process[i][j] > 0){
-                            file_offsets_for_agg[temp_index].length =
-                                blocklen_per_process[i][j];
-                            global_count += blocklen_per_process[i][j];
-                            file_offsets_for_agg[temp_index].process_id = i;
-                            file_offsets_for_agg[temp_index].offset =
-                                displs_per_process[i][j];
-                            temp_index++;
-                        }
-                    }
-                }
-            }
-            else{
-                continue;
-            }
-
-             /* Sort the displacements for each aggregator */
-            read_heap_sort (file_offsets_for_agg,
-                            entries_per_aggregator,
-                            sorted_file_offsets);
-
-            memory_displacements = (MPI_Aint *) malloc
-                (entries_per_aggregator * sizeof(MPI_Aint));
-            memory_displacements[sorted_file_offsets[0]] = 0;
-            for (i=1; i<entries_per_aggregator; i++){
-                memory_displacements[sorted_file_offsets[i]] =
-                    memory_displacements[sorted_file_offsets[i-1]] +
-                    file_offsets_for_agg[sorted_file_offsets[i-1]].length;
-            }
-
-             /**********************************************************
-	     *** 7e. Create the io array, and pass it to fbtl
-	     *********************************************************/
-            fh->f_io_array = (mca_common_ompio_io_array_t *) malloc
-                (entries_per_aggregator * sizeof (mca_common_ompio_io_array_t));
-            if (NULL == fh->f_io_array) {
-                opal_output(1, "OUT OF MEMORY\n");
-                ret = OMPI_ERR_OUT_OF_RESOURCE;
-                goto exit;
-            }
-
-            fh->f_num_of_io_entries = 0;
-            fh->f_io_array[0].offset =
-                (IOVBASE_TYPE *)(intptr_t)file_offsets_for_agg[sorted_file_offsets[0]].offset;
-            fh->f_io_array[0].length =
-                file_offsets_for_agg[sorted_file_offsets[0]].length;
-            fh->f_io_array[0].memory_address =
-                global_buf+memory_displacements[sorted_file_offsets[0]];
-            fh->f_num_of_io_entries++;
-            for (i=1;i<entries_per_aggregator;i++){
-                if (file_offsets_for_agg[sorted_file_offsets[i-1]].offset +
-                    file_offsets_for_agg[sorted_file_offsets[i-1]].length ==
-                    file_offsets_for_agg[sorted_file_offsets[i]].offset){
-                    fh->f_io_array[fh->f_num_of_io_entries - 1].length +=
-                        file_offsets_for_agg[sorted_file_offsets[i]].length;
-                }
-                else{
-                    fh->f_io_array[fh->f_num_of_io_entries].offset =
-                        (IOVBASE_TYPE *)(intptr_t)file_offsets_for_agg[sorted_file_offsets[i]].offset;
-                    fh->f_io_array[fh->f_num_of_io_entries].length =
-                        file_offsets_for_agg[sorted_file_offsets[i]].length;
-                    fh->f_io_array[fh->f_num_of_io_entries].memory_address =
-                        global_buf+memory_displacements[sorted_file_offsets[i]];
-                    fh->f_num_of_io_entries++;
-                }
-            }
-
-
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-            start_read_time = MPI_Wtime();
-#endif
-
-            if (fh->f_num_of_io_entries) {
-                if ( 0 >  fh->f_fbtl->fbtl_preadv (fh)) {
-                    opal_output (1, "READ FAILED\n");
-                    ret = OMPI_ERROR;
-                    goto exit;
-                }
-            }
-
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-            end_read_time = MPI_Wtime();
-            read_time += end_read_time - start_read_time;
-#endif
-            /**********************************************************
-             ******************** DONE READING ************************
-             *********************************************************/
-
-            temp_disp_index = (int *)calloc (1, fh->f_procs_per_group * sizeof (int));
-            if (NULL == temp_disp_index) {
-                opal_output (1, "OUT OF MEMORY\n");
-                ret = OMPI_ERR_OUT_OF_RESOURCE;
-                goto exit;
-            }
-            for (i=0; i<entries_per_aggregator; i++){
-                temp_index =
-                    file_offsets_for_agg[sorted_file_offsets[i]].process_id;
-                displs_per_process[temp_index][temp_disp_index[temp_index]] =
-                    memory_displacements[sorted_file_offsets[i]];
-                if (temp_disp_index[temp_index] < disp_index[temp_index]){
-                    temp_disp_index[temp_index] += 1;
-                }
-                else{
-                    printf("temp_disp_index[%d]: %d is greater than disp_index[%d]: %d\n",
-                           temp_index, temp_disp_index[temp_index],
-                           temp_index, disp_index[temp_index]);
-                }
-            }
-            if (NULL != temp_disp_index){
-                free(temp_disp_index);
-                temp_disp_index = NULL;
-            }
-
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-            start_rcomm_time = MPI_Wtime();
-#endif
-            for (i=0;i<fh->f_procs_per_group;i++){
-                size_t datatype_size;
-                send_req[i] = MPI_REQUEST_NULL;
-                if ( 0 < disp_index[i] ) {
-                    ompi_datatype_create_hindexed(disp_index[i],
-                                                  blocklen_per_process[i],
-                                                  displs_per_process[i],
-                                                  MPI_BYTE,
-                                                  &sendtype[i]);
-                    ompi_datatype_commit(&sendtype[i]);
-                    opal_datatype_type_size(&sendtype[i]->super, &datatype_size);
-
-                    if(datatype_size) {
-                        ret = MCA_PML_CALL (isend(global_buf,
-                                                  1,
-                                                  sendtype[i],
-                                                  fh->f_procs_in_group[i],
-                                                  FCOLL_DYNAMIC_GEN2_SHUFFLE_TAG,
-                                                  MCA_PML_BASE_SEND_STANDARD,
-                                                  fh->f_comm,
-                                                  &send_req[i]));
-                        if(OMPI_SUCCESS != ret){
-                            goto exit;
-                        }
-                    }
-                }
-            }
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-            end_rcomm_time = MPI_Wtime();
-            rcomm_time += end_rcomm_time - start_rcomm_time;
-#endif
-        }
-
-        /**********************************************************
-         *** 7f.  Scatter the Data from the readers
-         *********************************************************/
-        if(bytes_received) {
-            size_t remaining            = bytes_received;
-            int block_index             = -1;
-            int blocklength_size        = INIT_LEN;
-
-            ptrdiff_t recv_mem_address  = 0;
-            ompi_datatype_t *newType    = MPI_DATATYPE_NULL;
-
-            blocklength_proc            = (int *)       calloc (blocklength_size, sizeof (int));
-            displs_proc                 = (ptrdiff_t *) calloc (blocklength_size, sizeof (ptrdiff_t));
-
-            if (NULL == blocklength_proc || NULL == displs_proc ) {
-                opal_output (1, "OUT OF MEMORY\n");
-                ret = OMPI_ERR_OUT_OF_RESOURCE;
-                goto exit;
-            }
-
-            while (remaining) {
-                block_index++;
-
-                if(0 == block_index) {
-                    recv_mem_address = (ptrdiff_t) (decoded_iov[iov_index].iov_base) + current_position;
-                }
-                else {
-                    // Reallocate more memory if blocklength_size is not enough
-                    if(0 == block_index % INIT_LEN) {
-                        blocklength_size += INIT_LEN;
-                        blocklength_proc = (int *)       realloc(blocklength_proc, blocklength_size * sizeof(int));
-                        displs_proc      = (ptrdiff_t *) realloc(displs_proc, blocklength_size * sizeof(ptrdiff_t));
-                    }
-                    displs_proc[block_index] = (ptrdiff_t) (decoded_iov[iov_index].iov_base) +
-                                                            current_position - recv_mem_address;
-                }
-
-                if (remaining >= (decoded_iov[iov_index].iov_len - current_position)) {
-                    blocklength_proc[block_index] = decoded_iov[iov_index].iov_len - current_position;
-
-                    remaining = remaining - blocklength_proc[block_index];
-                    iov_index = iov_index + 1;
-                    current_position = 0;
-                }
-                else {
-                    blocklength_proc[block_index] = remaining;
-                    current_position += remaining;
-                    remaining = 0;
-                }
-            }
-
-            ompi_datatype_create_hindexed(block_index+1,
-                                          blocklength_proc,
-                                          displs_proc,
-                                          MPI_BYTE,
-                                          &newType);
-            ompi_datatype_commit(&newType);
-
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-            start_rcomm_time = MPI_Wtime();
-#endif
-            ret = MCA_PML_CALL(irecv((char *)recv_mem_address,
-                                     1,
-                                     newType,
-                                     my_aggregator,
-                                     FCOLL_DYNAMIC_GEN2_SHUFFLE_TAG,
-                                     fh->f_comm,
-                                     &recv_req));
-
-            if ( MPI_DATATYPE_NULL != newType ) {
-                ompi_datatype_destroy(&newType);
-            }
-
-            if (OMPI_SUCCESS != ret){
-                goto exit;
-            }
-        }
-
-        if (my_aggregator == fh->f_rank){
-            ret = ompi_request_wait_all (fh->f_procs_per_group,
-                                         send_req,
-                                         MPI_STATUS_IGNORE);
-            if (OMPI_SUCCESS != ret){
-                goto exit;
-            }
-        }
-
-        ret = ompi_request_wait (&recv_req, MPI_STATUS_IGNORE);
-        if (OMPI_SUCCESS != ret){
-            goto exit;
-        }
-
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-        if(bytes_received) {
-            end_rcomm_time = MPI_Wtime();
-            rcomm_time += end_rcomm_time - start_rcomm_time;
-        }
-#endif
-    } /* end for (index=0; index < cycles; index ++) */
-
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-    end_rexch = MPI_Wtime();
-    read_exch += end_rexch - start_rexch;
-    nentry.time[0] = read_time;
-    nentry.time[1] = rcomm_time;
-    nentry.time[2] = read_exch;
-    if (my_aggregator == fh->f_rank)
-        nentry.aggregator = 1;
-    else
-        nentry.aggregator = 0;
-    nentry.nprocs_for_coll = dynamic_gen2_num_io_procs;
-    if (!mca_common_ompio_full_print_queue(fh->f_coll_read_time)){
-        mca_common_ompio_register_print_entry(fh->f_coll_read_time,
-                                              nentry);
-    }
-#endif
-
-exit:
-    if (NULL != global_buf) {
-        free (global_buf);
-        global_buf = NULL;
-    }
-    if (NULL != sorted) {
-        free (sorted);
-        sorted = NULL;
-    }
-    if (NULL != global_iov_array) {
-        free (global_iov_array);
-        global_iov_array = NULL;
-    }
-    if (NULL != fview_count) {
-        free (fview_count);
-        fview_count = NULL;
-    }
-    if (NULL != decoded_iov) {
-        free (decoded_iov);
-        decoded_iov = NULL;
-    }
-    if (NULL != local_iov_array){
-        free(local_iov_array);
-        local_iov_array=NULL;
-    }
-
-    if (NULL != displs) {
-        free (displs);
-        displs = NULL;
-    }
-
-    if (NULL != blocklength_proc) {
-        free (blocklength_proc);
-        blocklength_proc = NULL;
-    }
-
-    if (NULL != displs_proc) {
-        free (displs_proc);
-        displs_proc = NULL;
-    }
-
-    if (my_aggregator == fh->f_rank) {
-
-        if (NULL != sorted_file_offsets){
-            free(sorted_file_offsets);
-            sorted_file_offsets = NULL;
-        }
-        if (NULL != file_offsets_for_agg){
-            free(file_offsets_for_agg);
-            file_offsets_for_agg = NULL;
-        }
-        if (NULL != memory_displacements){
-            free(memory_displacements);
-            memory_displacements= NULL;
-        }
-        if (NULL != sendtype){
-            for (i = 0; i < fh->f_procs_per_group; i++) {
-                if ( MPI_DATATYPE_NULL != sendtype[i] ) {
-                    ompi_datatype_destroy(&sendtype[i]);
-                }
-            }
-            free(sendtype);
-            sendtype=NULL;
-        }
-
-        if (NULL != disp_index){
-            free(disp_index);
-            disp_index = NULL;
-        }
-
-        if ( NULL != blocklen_per_process){
-            for(l=0;l<fh->f_procs_per_group;l++){
-                if (NULL != blocklen_per_process[l]){
-                    free(blocklen_per_process[l]);
-                    blocklen_per_process[l] = NULL;
-                }
-            }
-
-            free(blocklen_per_process);
-            blocklen_per_process = NULL;
-        }
-
-        if (NULL != displs_per_process){
-            for (l=0; i<fh->f_procs_per_group; l++){
-                if (NULL != displs_per_process[l]){
-                    free(displs_per_process[l]);
-                    displs_per_process[l] = NULL;
-                }
-            }
-            free(displs_per_process);
-            displs_per_process = NULL;
-        }
-        if ( NULL != send_req ) {
-            free ( send_req );
-            send_req = NULL;
-        }
-    }
-    return ret;
-}
-
-
-static int read_heap_sort (mca_io_ompio_local_io_array *io_array,
-                           int num_entries,
-                           int *sorted)
+int mca_fcoll_dynamic_gen2_file_read_all (struct ompio_file_t *fh,
+                                          void *buf,
+                                          size_t count,
+                                          struct ompi_datatype_t *datatype,
+                                          ompi_status_public_t *status)
 {
-    int i = 0;
-    int j = 0;
-    int left = 0;
-    int right = 0;
-    int largest = 0;
-    int heap_size = num_entries - 1;
-    int temp = 0;
-    unsigned char done = 0;
-    int* temp_arr = NULL;
-
-    temp_arr = (int*)malloc(num_entries*sizeof(int));
-    if (NULL == temp_arr) {
-        opal_output (1, "OUT OF MEMORY\n");
-        return OMPI_ERR_OUT_OF_RESOURCE;
-    }
-    temp_arr[0] = 0;
-    for (i = 1; i < num_entries; ++i) {
-        temp_arr[i] = i;
-    }
-    /* num_entries can be a large no. so NO RECURSION */
-    for (i = num_entries/2-1 ; i>=0 ; i--) {
-        done = 0;
-        j = i;
-        largest = j;
-
-        while (!done) {
-            left = j*2+1;
-            right = j*2+2;
-            if ((left <= heap_size) &&
-                (io_array[temp_arr[left]].offset > io_array[temp_arr[j]].offset)) {
-                largest = left;
-            }
-            else {
-                largest = j;
-            }
-            if ((right <= heap_size) &&
-                (io_array[temp_arr[right]].offset >
-                 io_array[temp_arr[largest]].offset)) {
-                largest = right;
-            }
-            if (largest != j) {
-                temp = temp_arr[largest];
-                temp_arr[largest] = temp_arr[j];
-                temp_arr[j] = temp;
-                j = largest;
-            }
-            else {
-                done = 1;
-            }
-        }
-    }
-
-    for (i = num_entries-1; i >=1; --i) {
-        temp = temp_arr[0];
-        temp_arr[0] = temp_arr[i];
-        temp_arr[i] = temp;
-        heap_size--;
-        done = 0;
-        j = 0;
-        largest = j;
-
-        while (!done) {
-            left =  j*2+1;
-            right = j*2+2;
-
-            if ((left <= heap_size) &&
-                (io_array[temp_arr[left]].offset >
-                 io_array[temp_arr[j]].offset)) {
-                largest = left;
-            }
-            else {
-                largest = j;
-            }
-            if ((right <= heap_size) &&
-                (io_array[temp_arr[right]].offset >
-                 io_array[temp_arr[largest]].offset)) {
-                largest = right;
-            }
-            if (largest != j) {
-                temp = temp_arr[largest];
-                temp_arr[largest] = temp_arr[j];
-                temp_arr[j] = temp;
-                j = largest;
-            }
-            else {
-                done = 1;
-            }
-        }
-        sorted[i] = temp_arr[i];
-    }
-    sorted[0] = temp_arr[0];
-
-    if (NULL != temp_arr) {
-        free(temp_arr);
-        temp_arr = NULL;
-    }
-    return OMPI_SUCCESS;
+    return mca_common_ompio_base_file_read_all (fh, buf, count, datatype, status);
 }
 
-
-
diff --git a/ompi/mca/fcoll/dynamic_gen2/fcoll_dynamic_gen2_file_write_all.c b/ompi/mca/fcoll/dynamic_gen2/fcoll_dynamic_gen2_file_write_all.c
index 129d47c8a42..1f9b5f8bef7 100644
--- a/ompi/mca/fcoll/dynamic_gen2/fcoll_dynamic_gen2_file_write_all.c
+++ b/ompi/mca/fcoll/dynamic_gen2/fcoll_dynamic_gen2_file_write_all.c
@@ -14,6 +14,9 @@
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2017      IBM Corporation. All rights reserved.
  * Copyright (c) 2018      Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -45,7 +48,8 @@ typedef struct mca_io_ompio_local_io_array{
 }mca_io_ompio_local_io_array;
 
 typedef struct mca_io_ompio_aggregator_data {
-    int *disp_index, *sorted, *fview_count, n;
+    int *disp_index, *sorted, n;
+    size_t *fview_count;
     int *max_disp_index;
     int **blocklen_per_process;
     MPI_Aint **displs_per_process, total_bytes, bytes_per_cycle, total_bytes_written;
@@ -112,9 +116,9 @@ int mca_fcoll_dynamic_gen2_split_iov_array ( ompio_file_t *fh, mca_common_ompio_
                                              int num_entries, int *last_array_pos, int *last_pos_in_field );
 
 
-int mca_fcoll_dynamic_gen2_file_write_all (ompio_file_t *fh,
+int mca_fcoll_dynamic_gen2_file_write_all (struct ompio_file_t *fh,
                                       const void *buf,
-                                      int count,
+                                      size_t count,
                                       struct ompi_datatype_t *datatype,
                                       ompi_status_public_t *status)
 {
@@ -130,7 +134,7 @@ int mca_fcoll_dynamic_gen2_file_write_all (ompio_file_t *fh,
     ompi_request_t **curr_reqs=NULL,**prev_reqs=NULL;
     mca_io_ompio_aggregator_data **aggr_data=NULL;
     
-    int *displs = NULL;
+    ptrdiff_t *displs = NULL;
     int dynamic_gen2_num_io_procs;
     size_t max_data = 0;
     MPI_Aint *total_bytes_per_process = NULL;
@@ -143,8 +147,10 @@ int mca_fcoll_dynamic_gen2_file_write_all (ompio_file_t *fh,
 
     int *aggregators=NULL;
     int *result_counts=NULL;
-    
-    
+
+    ompi_count_array_t fview_count_desc;
+    ompi_disp_array_t displs_desc;
+
 #if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
     double write_time = 0.0, start_write_time = 0.0, end_write_time = 0.0;
     double comm_time = 0.0, start_comm_time = 0.0, end_comm_time = 0.0;
@@ -154,7 +160,7 @@ int mca_fcoll_dynamic_gen2_file_write_all (ompio_file_t *fh,
     
     
     /**************************************************************************
-     ** 1.  In case the data is not contigous in memory, decode it into an iovec
+     ** 1.  In case the data is not contiguous in memory, decode it into an iovec
      **************************************************************************/
     bytes_per_cycle = fh->f_bytes_per_agg;
 
@@ -356,7 +362,7 @@ int mca_fcoll_dynamic_gen2_file_write_all (ompio_file_t *fh,
     for ( i=0; i< dynamic_gen2_num_io_procs; i++ ) {
         aggr_data[i]->total_bytes = broken_total_lengths[i];
         aggr_data[i]->decoded_iov = broken_decoded_iovs[i];
-        aggr_data[i]->fview_count = (int *) malloc (fh->f_procs_per_group * sizeof (int));
+        aggr_data[i]->fview_count = (size_t *)malloc (fh->f_procs_per_group * sizeof (size_t));
         if (NULL == aggr_data[i]->fview_count) {
             opal_output (1, "OUT OF MEMORY\n");
             ret = OMPI_ERR_OUT_OF_RESOURCE;
@@ -365,7 +371,7 @@ int mca_fcoll_dynamic_gen2_file_write_all (ompio_file_t *fh,
         for ( j=0; j <fh->f_procs_per_group; j++ ) {
             aggr_data[i]->fview_count[j] = result_counts[dynamic_gen2_num_io_procs*j+i];
         }
-        displs = (int*) malloc (fh->f_procs_per_group * sizeof (int));
+        displs = (ptrdiff_t *)malloc (fh->f_procs_per_group * sizeof (ptrdiff_t));
         if (NULL == displs) {
             opal_output (1, "OUT OF MEMORY\n");
             ret = OMPI_ERR_OUT_OF_RESOURCE;
@@ -373,7 +379,7 @@ int mca_fcoll_dynamic_gen2_file_write_all (ompio_file_t *fh,
         }
         
         displs[0] = 0;
-        total_fview_count = aggr_data[i]->fview_count[0];
+        total_fview_count = (int) aggr_data[i]->fview_count[0];
         for (j=1 ; j<fh->f_procs_per_group ; j++) {
             total_fview_count += aggr_data[i]->fview_count[j];
             displs[j] = displs[j-1] + aggr_data[i]->fview_count[j-1];
@@ -407,12 +413,14 @@ int mca_fcoll_dynamic_gen2_file_write_all (ompio_file_t *fh,
         start_comm_time = MPI_Wtime();
 #endif
         if ( 1 == mca_fcoll_dynamic_gen2_num_groups ) {
+            OMPI_COUNT_ARRAY_INIT(&fview_count_desc, aggr_data[i]->fview_count);
+            OMPI_DISP_ARRAY_INIT(&displs_desc, displs);
             ret = fh->f_comm->c_coll->coll_allgatherv (broken_iov_arrays[i],
                                                       broken_counts[i],
                                                       fh->f_iov_type,
                                                       aggr_data[i]->global_iov_array,
-                                                      aggr_data[i]->fview_count,
-                                                      displs,
+                                                      fview_count_desc,
+                                                      displs_desc,
                                                       fh->f_iov_type,
                                                       fh->f_comm,
                                                       fh->f_comm->c_coll->coll_allgatherv_module );
@@ -788,7 +796,9 @@ static int shuffle_init ( int index, int cycles, int aggregator, int rank, mca_i
     int temp_index=0;
     MPI_Aint *memory_displacements=NULL;
     int *temp_disp_index=NULL;
+#if DEBUG_ON
     MPI_Aint global_count = 0;
+#endif
     int* blocklength_proc=NULL;
     ptrdiff_t* displs_proc=NULL;
 
@@ -997,7 +1007,7 @@ static int shuffle_init ( int index, int cycles, int aggregator, int rank, mca_i
     
     /*************************************************************************
      *** 7d. Calculate the displacement on where to put the data and allocate
-     ***     the recieve buffer (global_buf)
+     ***     the receive buffer (global_buf)
      *************************************************************************/
     if (aggregator == rank) {
         entries_per_aggregator=0;
@@ -1085,7 +1095,9 @@ static int shuffle_init ( int index, int cycles, int aggregator, int rank, mca_i
             }
             
             /*Now update the displacements array  with memory offsets*/
+#if DEBUG_ON
             global_count = 0;
+#endif
             for (i=0;i<entries_per_aggregator;i++){
                 temp_pindex =
                     file_offsets_for_agg[sorted_file_offsets[i]].process_id;
@@ -1098,8 +1110,10 @@ static int shuffle_init ( int index, int cycles, int aggregator, int rank, mca_i
                            temp_pindex, temp_disp_index[temp_pindex],
                            temp_pindex, data->disp_index[temp_pindex]);
                 }
+#if DEBUG_ON
                 global_count +=
                     file_offsets_for_agg[sorted_file_offsets[i]].length;
+#endif
             }
             
             if (NULL != temp_disp_index){
@@ -1289,7 +1303,7 @@ static int shuffle_init ( int index, int cycles, int aggregator, int rank, mca_i
         data->num_io_entries++;
         
         for (i=1;i<entries_per_aggregator;i++){
-            /* If the enrties are contiguous merge them,
+            /* If the entries are contiguous merge them,
                else make a new entry */
             if (file_offsets_for_agg[sorted_file_offsets[i-1]].offset +
                 file_offsets_for_agg[sorted_file_offsets[i-1]].length ==
diff --git a/ompi/mca/fcoll/dynamic_gen2/fcoll_dynamic_gen2_module.c b/ompi/mca/fcoll/dynamic_gen2/fcoll_dynamic_gen2_module.c
index c3786cf3dfd..6690dc24c3c 100644
--- a/ompi/mca/fcoll/dynamic_gen2/fcoll_dynamic_gen2_module.c
+++ b/ompi/mca/fcoll/dynamic_gen2/fcoll_dynamic_gen2_module.c
@@ -9,9 +9,11 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2008-2015 University of Houston. All rights reserved.
+ * Copyright (c) 2008-2022 University of Houston. All rights reserved.
  * Copyright (c) 2018      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -34,7 +36,7 @@
  * ************************ actions structure ************************
  * *******************************************************************
  */
-static mca_fcoll_base_module_1_0_0_t dynamic_gen2 =  {
+static mca_fcoll_base_module_2_0_0_t dynamic_gen2 =  {
     mca_fcoll_dynamic_gen2_module_init,
     mca_fcoll_dynamic_gen2_module_finalize,
     mca_fcoll_dynamic_gen2_file_read_all,
@@ -54,7 +56,7 @@ mca_fcoll_dynamic_gen2_component_init_query(bool enable_progress_threads,
     return OMPI_SUCCESS;
 }
 
-mca_fcoll_base_module_1_0_0_t *
+mca_fcoll_base_module_2_0_0_t *
 mca_fcoll_dynamic_gen2_component_file_query (ompio_file_t *fh, int *priority)
 {
     *priority = mca_fcoll_dynamic_gen2_priority;
diff --git a/ompi/mca/fcoll/fcoll.h b/ompi/mca/fcoll/fcoll.h
index 4e4f2fc694c..14a5967b530 100644
--- a/ompi/mca/fcoll/fcoll.h
+++ b/ompi/mca/fcoll/fcoll.h
@@ -15,6 +15,8 @@
  *                         reserved.
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -39,8 +41,8 @@ struct mca_fcoll_request_t;
 /*
  * Macro for use in components that are of type coll
  */
-#define MCA_FCOLL_BASE_VERSION_2_0_0 \
-    OMPI_MCA_BASE_VERSION_2_1_0("fcoll", 2, 0, 0)
+#define MCA_FCOLL_BASE_VERSION_3_0_0 \
+    OMPI_MCA_BASE_VERSION_2_1_0("fcoll", 3, 0, 0)
 
 /*
  * This framework provides the abstraction for the collective file
@@ -75,7 +77,7 @@ typedef int (*mca_fcoll_base_component_init_query_1_0_0_fn_t)
     (bool enable_progress_threads,
      bool enable_mpi_threads);
 
-typedef struct mca_fcoll_base_module_1_0_0_t *
+typedef struct mca_fcoll_base_module_2_0_0_t *
 (*mca_fcoll_base_component_file_query_1_0_0_fn_t) (struct ompio_file_t *file,
                                                    int *priority);
 
@@ -84,10 +86,10 @@ typedef int (*mca_fcoll_base_component_file_unquery_1_0_0_fn_t)
 
 /*
  * ****************** component struct ******************************
- * Structure for fcoll v2.0.0 components.This is chained to MCA v2.0.0
+ * Structure for fcoll v3.0.0 components.This is chained to MCA v2.0.0
  * ****************** component struct ******************************
  */
-struct mca_fcoll_base_component_2_0_0_t {
+struct mca_fcoll_base_component_3_0_0_t {
     mca_base_component_t fcollm_version;
     mca_base_component_data_t fcollm_data;
 
@@ -95,8 +97,8 @@ struct mca_fcoll_base_component_2_0_0_t {
     mca_fcoll_base_component_file_query_1_0_0_fn_t fcollm_file_query;
     mca_fcoll_base_component_file_unquery_1_0_0_fn_t fcollm_file_unquery;
 };
-typedef struct mca_fcoll_base_component_2_0_0_t mca_fcoll_base_component_2_0_0_t;
-typedef struct mca_fcoll_base_component_2_0_0_t mca_fcoll_base_component_t;
+typedef struct mca_fcoll_base_component_3_0_0_t mca_fcoll_base_component_3_0_0_t;
+typedef struct mca_fcoll_base_component_3_0_0_t mca_fcoll_base_component_t;
 
 /*
  * ***********************************************************************
@@ -115,28 +117,28 @@ typedef int (*mca_fcoll_base_module_finalize_1_0_0_fn_t)
 typedef int (*mca_fcoll_base_module_file_read_all_fn_t)
 (struct ompio_file_t *fh,
  void *buf,
- int count,
+ size_t count,
  struct ompi_datatype_t *datatype,
  ompi_status_public_t *status);
 
 typedef int (*mca_fcoll_base_module_file_iread_all_fn_t)
 (struct ompio_file_t *fh,
  void *buf,
- int count,
+ size_t count,
  struct ompi_datatype_t *datatype,
  ompi_request_t **request);
 
 typedef int (*mca_fcoll_base_module_file_write_all_fn_t)
 (struct ompio_file_t *fh,
  const void *buf,
- int count,
+ size_t count,
  struct ompi_datatype_t *datatype,
  ompi_status_public_t *status);
 
 typedef int (*mca_fcoll_base_module_file_iwrite_all_fn_t)
 (struct ompio_file_t *fh,
  const void *buf,
- int count,
+ size_t count,
  struct ompi_datatype_t *datatype,
  ompi_request_t **request);
 
@@ -151,7 +153,7 @@ typedef void (*mca_fcoll_base_module_request_free_fn_t)
  * ***************************  module structure *************************
  * ***********************************************************************
  */
-struct mca_fcoll_base_module_1_0_0_t {
+struct mca_fcoll_base_module_2_0_0_t {
     /*
      * Per-file initialization function. This is called only
      * on the module which is selected. The finalize corresponding to
@@ -169,8 +171,8 @@ struct mca_fcoll_base_module_1_0_0_t {
     mca_fcoll_base_module_request_free_fn_t            fcoll_request_free;
 
 };
-typedef struct mca_fcoll_base_module_1_0_0_t mca_fcoll_base_module_1_0_0_t;
-typedef mca_fcoll_base_module_1_0_0_t mca_fcoll_base_module_t;
+typedef struct mca_fcoll_base_module_2_0_0_t mca_fcoll_base_module_2_0_0_t;
+typedef mca_fcoll_base_module_2_0_0_t mca_fcoll_base_module_t;
 
 END_C_DECLS
 
diff --git a/ompi/mca/fcoll/individual/fcoll_individual.h b/ompi/mca/fcoll/individual/fcoll_individual.h
index 911a0aa6888..63d0e126ce4 100644
--- a/ompi/mca/fcoll/individual/fcoll_individual.h
+++ b/ompi/mca/fcoll/individual/fcoll_individual.h
@@ -12,6 +12,8 @@
  * Copyright (c) 2008-2016 University of Houston. All rights reserved.
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -36,13 +38,13 @@ BEGIN_C_DECLS
 
 extern int mca_fcoll_individual_priority;
 
-OMPI_MODULE_DECLSPEC extern mca_fcoll_base_component_2_0_0_t mca_fcoll_individual_component;
+OMPI_DECLSPEC extern mca_fcoll_base_component_3_0_0_t mca_fcoll_individual_component;
 
 /* API functions */
 
 int mca_fcoll_individual_component_init_query(bool enable_progress_threads,
                                            bool enable_mpi_threads);
-struct mca_fcoll_base_module_1_0_0_t *
+struct mca_fcoll_base_module_2_0_0_t *
 mca_fcoll_individual_component_file_query (ompio_file_t *fh, int *priority);
 
 int mca_fcoll_individual_component_file_unquery (ompio_file_t *file);
@@ -52,14 +54,14 @@ int mca_fcoll_individual_module_finalize (ompio_file_t *file);
 
 int mca_fcoll_individual_file_read_all (ompio_file_t *fh,
                                      void *buf,
-                                     int count,
+                                     size_t count,
                                      struct ompi_datatype_t *datatype,
                                      ompi_status_public_t * status);
 
 
 int mca_fcoll_individual_file_write_all (ompio_file_t *fh,
                                       const void *buf,
-                                      int count,
+                                      size_t count,
                                       struct ompi_datatype_t *datatype,
                                       ompi_status_public_t * status);
 
diff --git a/ompi/mca/fcoll/individual/fcoll_individual_component.c b/ompi/mca/fcoll/individual/fcoll_individual_component.c
index 722b80c42f4..681bfbc3363 100644
--- a/ompi/mca/fcoll/individual/fcoll_individual_component.c
+++ b/ompi/mca/fcoll/individual/fcoll_individual_component.c
@@ -14,6 +14,8 @@
  * Copyright (c) 2008-2014 University of Houston. All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -54,13 +56,13 @@ static int individual_register(void);
  * Instantiate the public struct with all of our public information
  * and pointers to our public functions in it
  */
-mca_fcoll_base_component_2_0_0_t mca_fcoll_individual_component = {
+mca_fcoll_base_component_3_0_0_t mca_fcoll_individual_component = {
 
     /* First, the mca_component_t struct containing meta information
      * about the component itself */
 
     .fcollm_version = {
-        MCA_FCOLL_BASE_VERSION_2_0_0,
+        MCA_FCOLL_BASE_VERSION_3_0_0,
 
         /* Component name and version */
         .mca_component_name = "individual",
diff --git a/ompi/mca/fcoll/individual/fcoll_individual_file_read_all.c b/ompi/mca/fcoll/individual/fcoll_individual_file_read_all.c
index 77966d0a2d0..408e3e8cef6 100644
--- a/ompi/mca/fcoll/individual/fcoll_individual_file_read_all.c
+++ b/ompi/mca/fcoll/individual/fcoll_individual_file_read_all.c
@@ -12,6 +12,8 @@
  * Copyright (c) 2008-2015 University of Houston. All rights reserved.
  * Copyright (c) 2018      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -35,7 +37,7 @@
 int
 mca_fcoll_individual_file_read_all (ompio_file_t *fh,
                                     void *buf,
-                                    int count,
+                                    size_t count,
                                     struct ompi_datatype_t *datatype,
                                     ompi_status_public_t *status)
 {
diff --git a/ompi/mca/fcoll/individual/fcoll_individual_file_write_all.c b/ompi/mca/fcoll/individual/fcoll_individual_file_write_all.c
index 148b5e7edc7..6aaab630740 100644
--- a/ompi/mca/fcoll/individual/fcoll_individual_file_write_all.c
+++ b/ompi/mca/fcoll/individual/fcoll_individual_file_write_all.c
@@ -12,6 +12,8 @@
  * Copyright (c) 2008-2015 University of Houston. All rights reserved.
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -32,7 +34,7 @@
 
 int mca_fcoll_individual_file_write_all (ompio_file_t *fh,
                                          const void *buf,
-                                         int count,
+                                         size_t count,
                                          struct ompi_datatype_t *datatype,
                                          ompi_status_public_t *status)
 {
diff --git a/ompi/mca/fcoll/individual/fcoll_individual_module.c b/ompi/mca/fcoll/individual/fcoll_individual_module.c
index 4151f1acb72..33ad6eac28d 100644
--- a/ompi/mca/fcoll/individual/fcoll_individual_module.c
+++ b/ompi/mca/fcoll/individual/fcoll_individual_module.c
@@ -12,6 +12,8 @@
  * Copyright (c) 2008-2015 University of Houston. All rights reserved.
  * Copyright (c) 2018      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -34,7 +36,7 @@
  * ************************ actions structure ************************
  * *******************************************************************
  */
-static mca_fcoll_base_module_1_0_0_t individual =  {
+static mca_fcoll_base_module_2_0_0_t individual =  {
     mca_fcoll_individual_module_init,
     mca_fcoll_individual_module_finalize,
     mca_fcoll_individual_file_read_all,
@@ -54,7 +56,7 @@ mca_fcoll_individual_component_init_query(bool enable_progress_threads,
     return OMPI_SUCCESS;
 }
 
-mca_fcoll_base_module_1_0_0_t *
+mca_fcoll_base_module_2_0_0_t *
 mca_fcoll_individual_component_file_query (ompio_file_t *fh, int *priority)
 {
     *priority = mca_fcoll_individual_priority;
diff --git a/ompi/mca/fcoll/vulcan/Makefile.am b/ompi/mca/fcoll/vulcan/Makefile.am
index 63f1ad9da23..c4680544abb 100644
--- a/ompi/mca/fcoll/vulcan/Makefile.am
+++ b/ompi/mca/fcoll/vulcan/Makefile.am
@@ -9,10 +9,11 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2008-2018 University of Houston. All rights reserved.
+# Copyright (c) 2008-2022 University of Houston. All rights reserved.
 # Copyright (c) 2012 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2018      Research Organization for Information Science
 #                         and Technology (RIST).  All rights reserved.
+# Copyright (c) 2024      Advanced Micro Devices, Inc. All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -22,6 +23,7 @@
 
 sources = \
         fcoll_vulcan.h \
+        fcoll_vulcan_internal.h \
         fcoll_vulcan_module.c \
         fcoll_vulcan_component.c \
         fcoll_vulcan_file_read_all.c \
diff --git a/ompi/mca/fcoll/vulcan/fcoll_vulcan.h b/ompi/mca/fcoll/vulcan/fcoll_vulcan.h
index 46c1f471634..3165a0b0797 100644
--- a/ompi/mca/fcoll/vulcan/fcoll_vulcan.h
+++ b/ompi/mca/fcoll/vulcan/fcoll_vulcan.h
@@ -12,6 +12,9 @@
  * Copyright (c) 2008-2021 University of Houston. All rights reserved.
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2024      Advanced Micro Devices, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -38,17 +41,16 @@ BEGIN_C_DECLS
 /* Globally exported variables */
 
 extern int mca_fcoll_vulcan_priority;
-extern int mca_fcoll_vulcan_num_groups;
-extern int mca_fcoll_vulcan_write_chunksize;
 extern int mca_fcoll_vulcan_async_io;
+extern int mca_fcoll_vulcan_use_accelerator_buffers;
 
-OMPI_MODULE_DECLSPEC extern mca_fcoll_base_component_2_0_0_t mca_fcoll_vulcan_component;
+OMPI_DECLSPEC extern mca_fcoll_base_component_3_0_0_t mca_fcoll_vulcan_component;
 
 /* API functions */
 
 int mca_fcoll_vulcan_component_init_query(bool enable_progress_threads,
                                            bool enable_mpi_threads);
-struct mca_fcoll_base_module_1_0_0_t *
+struct mca_fcoll_base_module_2_0_0_t *
 mca_fcoll_vulcan_component_file_query (ompio_file_t *fh, int *priority);
 
 int mca_fcoll_vulcan_component_file_unquery (ompio_file_t *file);
@@ -56,18 +58,16 @@ int mca_fcoll_vulcan_component_file_unquery (ompio_file_t *file);
 int mca_fcoll_vulcan_module_init (ompio_file_t *file);
 int mca_fcoll_vulcan_module_finalize (ompio_file_t *file);
 
-int mca_fcoll_vulcan_file_read_all (ompio_file_t *fh,
-                                    void *buf,
-                                    int count,
-                                    struct ompi_datatype_t *datatype,
-                                    ompi_status_public_t * status);
-
-
-int mca_fcoll_vulcan_file_write_all (ompio_file_t *fh,
+int mca_fcoll_vulcan_file_write_all (struct ompio_file_t *fh,
                                      const void *buf,
-                                     int count,
+                                     size_t count,
                                      struct ompi_datatype_t *datatype,
                                      ompi_status_public_t * status);
+int mca_fcoll_vulcan_file_read_all (struct ompio_file_t *fh,
+                                    void *buf,
+                                    size_t count,
+                                    struct ompi_datatype_t *datatype,
+                                    ompi_status_public_t * status);
 
 
 END_C_DECLS
diff --git a/ompi/mca/fcoll/vulcan/fcoll_vulcan_component.c b/ompi/mca/fcoll/vulcan/fcoll_vulcan_component.c
index 09a80810d00..5fc8254f164 100644
--- a/ompi/mca/fcoll/vulcan/fcoll_vulcan_component.c
+++ b/ompi/mca/fcoll/vulcan/fcoll_vulcan_component.c
@@ -14,6 +14,9 @@
  * Copyright (c) 2008-2017 University of Houston. All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2024      Advanced Micro Devices, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -41,8 +44,6 @@ const char *mca_fcoll_vulcan_component_version_string =
  * Global variables
  */
 int mca_fcoll_vulcan_priority = 10;
-int mca_fcoll_vulcan_num_groups = 1;
-int mca_fcoll_vulcan_write_chunksize = -1;
 int mca_fcoll_vulcan_async_io = 0;
 
 /*
@@ -54,13 +55,13 @@ static int vulcan_register(void);
  * Instantiate the public struct with all of our public information
  * and pointers to our public functions in it
  */
-mca_fcoll_base_component_2_0_0_t mca_fcoll_vulcan_component = {
+mca_fcoll_base_component_3_0_0_t mca_fcoll_vulcan_component = {
 
     /* First, the mca_component_t struct containing meta information
      * about the component itself */
 
     .fcollm_version = {
-        MCA_FCOLL_BASE_VERSION_2_0_0,
+        MCA_FCOLL_BASE_VERSION_3_0_0,
 
         /* Component name and version */
         .mca_component_name = "vulcan",
@@ -89,20 +90,6 @@ vulcan_register(void)
                                            OPAL_INFO_LVL_9,
                                            MCA_BASE_VAR_SCOPE_READONLY, &mca_fcoll_vulcan_priority);
 
-    mca_fcoll_vulcan_num_groups = 1;
-    (void) mca_base_component_var_register(&mca_fcoll_vulcan_component.fcollm_version,
-                                           "num_groups", "Number of subgroups created by the vulcan component",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY, &mca_fcoll_vulcan_num_groups);
-
-    mca_fcoll_vulcan_write_chunksize = -1;
-    (void) mca_base_component_var_register(&mca_fcoll_vulcan_component.fcollm_version,
-                                           "write_chunksize", "Chunk size written at once. Default: stripe_size of the file system",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY, &mca_fcoll_vulcan_write_chunksize);
-
     mca_fcoll_vulcan_async_io = 0;
     (void) mca_base_component_var_register(&mca_fcoll_vulcan_component.fcollm_version,
                                            "async_io", "Asynchronous I/O support options. 0: Automatic choice (default) "
diff --git a/ompi/mca/fcoll/vulcan/fcoll_vulcan_file_read_all.c b/ompi/mca/fcoll/vulcan/fcoll_vulcan_file_read_all.c
index a00bf8e340a..f6a492e621c 100644
--- a/ompi/mca/fcoll/vulcan/fcoll_vulcan_file_read_all.c
+++ b/ompi/mca/fcoll/vulcan/fcoll_vulcan_file_read_all.c
@@ -1,8 +1,9 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2017 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -10,8 +11,11 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2008-2021 University of Houston. All rights reserved.
- * Copyright (c) 2017-2018 Research Organization for Information Science
+ * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2024      Advanced Micro Devices, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -21,1062 +25,920 @@
 
 #include "ompi_config.h"
 #include "fcoll_vulcan.h"
+#include "fcoll_vulcan_internal.h"
 
 #include "mpi.h"
 #include "ompi/constants.h"
 #include "ompi/mca/fcoll/fcoll.h"
 #include "ompi/mca/fcoll/base/fcoll_base_coll_array.h"
 #include "ompi/mca/common/ompio/common_ompio.h"
+#include "ompi/mca/common/ompio/common_ompio_buffer.h"
 #include "ompi/mca/io/io.h"
+#include "ompi/mca/common/ompio/common_ompio_request.h"
 #include "math.h"
 #include "ompi/mca/pml/pml.h"
+#include "opal/mca/accelerator/accelerator.h"
 #include <unistd.h>
 
 #define DEBUG_ON 0
+#define NOT_AGGR_INDEX -1
 
-/*Used for loading file-offsets per aggregator*/
-typedef struct mca_io_ompio_local_io_array{
-    OMPI_MPI_OFFSET_TYPE offset;
-    MPI_Aint             length;
-    int                  process_id;
-}mca_io_ompio_local_io_array;
-
-
-static int read_heap_sort (mca_io_ompio_local_io_array *io_array,
-                           int num_entries,
-                           int *sorted);
+static int shuffle_init (int index, int cycles, int aggregator, int rank,
+                         mca_io_ompio_aggregator_data *data, ompi_request_t **reqs);
 
+static int read_init (ompio_file_t *fh, int index, int cycles, int aggregator, int rank,
+		      mca_io_ompio_aggregator_data *aggr_data,
+                      int read_syncType, ompi_request_t **request,
+                      bool is_accelerator_buffer);
 
-
-int
-mca_fcoll_vulcan_file_read_all (ompio_file_t *fh,
-                                 void *buf,
-                                 int count,
-                                 struct ompi_datatype_t *datatype,
-                                 ompi_status_public_t *status)
+int mca_fcoll_vulcan_file_read_all (struct ompio_file_t *fh,
+                                    void *buf,
+                                    size_t count,
+                                    struct ompi_datatype_t *datatype,
+                                    ompi_status_public_t *status)
 {
-    MPI_Aint total_bytes = 0;          /* total bytes to be read */
-    MPI_Aint bytes_to_read_in_cycle = 0; /* left to be read in a cycle*/
-    MPI_Aint bytes_per_cycle = 0;      /* total read in each cycle by each process*/
-    int index = 0, ret=OMPI_SUCCESS;
+    int index = 0;
     int cycles = 0;
-    int i=0, j=0, l=0;
-    int n=0; /* current position in total_bytes_per_process array */
-    MPI_Aint bytes_remaining = 0; /* how many bytes have been read from the current
-                                     value from total_bytes_per_process */
-    int *sorted_file_offsets=NULL, entries_per_aggregator=0;
-    int bytes_received = 0;
-    int blocks = 0;
-    /* iovec structure and count of the buffer passed in */
+    int ret =0, l, i, j, bytes_per_cycle;
     uint32_t iov_count = 0;
     struct iovec *decoded_iov = NULL;
-    int iov_index = 0;
-    size_t current_position = 0;
-    struct iovec *local_iov_array=NULL, *global_iov_array=NULL;
-    MPI_Aint *memory_displacements=NULL;
-    /* global iovec at the readers that contain the iovecs created from
-       file_set_view */
+    struct iovec *local_iov_array=NULL;
     uint32_t total_fview_count = 0;
     int local_count = 0;
-    int *fview_count = NULL, *disp_index=NULL, *temp_disp_index=NULL;
-    int current_index=0, temp_index=0;
-    int **blocklen_per_process=NULL;
-    MPI_Aint **displs_per_process=NULL;
-    char *global_buf = NULL;
-    MPI_Aint global_count = 0;
-    mca_io_ompio_local_io_array *file_offsets_for_agg=NULL;
+    ompi_request_t **reqs = NULL;
+    ompi_request_t *req_iread = MPI_REQUEST_NULL;
+    ompi_request_t *req_tmp = MPI_REQUEST_NULL;
+    mca_io_ompio_aggregator_data **aggr_data=NULL;
 
-    /* array that contains the sorted indices of the global_iov */
-    int *sorted = NULL;
-    int *displs = NULL;
+    ptrdiff_t *displs = NULL;
     int vulcan_num_io_procs;
     size_t max_data = 0;
-    MPI_Aint *total_bytes_per_process = NULL;
-    ompi_datatype_t **sendtype = NULL;
-    MPI_Request *send_req = NULL;
-    MPI_Request recv_req = MPI_REQUEST_NULL;
-    int my_aggregator =-1;
 
-    int* blocklength_proc       = NULL;
-    ptrdiff_t* displs_proc      = NULL;
+    struct iovec **broken_iov_arrays=NULL;
+    struct iovec **broken_decoded_iovs=NULL;
+    int *broken_counts=NULL;
+    int *broken_iov_counts=NULL;
+    MPI_Aint *broken_total_lengths=NULL;
+
+    int aggr_index = NOT_AGGR_INDEX;
+    int read_sync_type = 2;
+    int *result_counts=NULL;
+
+    ompi_count_array_t fview_count_desc;
+    ompi_disp_array_t displs_desc;
+    int is_gpu, is_managed;
+    bool use_accelerator_buffer = false;
 
 #if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
     double read_time = 0.0, start_read_time = 0.0, end_read_time = 0.0;
-    double rcomm_time = 0.0, start_rcomm_time = 0.0, end_rcomm_time = 0.0;
-    double read_exch = 0.0, start_rexch = 0.0, end_rexch = 0.0;
+    double comm_time = 0.0, start_comm_time = 0.0, end_comm_time = 0.0;
+    double exch_read = 0.0, start_exch = 0.0, end_exch = 0.0;
     mca_common_ompio_print_entry nentry;
 #endif
 
-    /**************************************************************************
-     ** 1. In case the data is not contigous in memory, decode it into an iovec
-     **************************************************************************/
-    ret = mca_common_ompio_decode_datatype ((struct ompio_file_t *)fh,
-                                            datatype,
-                                            count,
-                                            buf,
-                                            &max_data,
-                                            fh->f_mem_convertor,
-                                            &decoded_iov,
-                                            &iov_count);
-    if (OMPI_SUCCESS != ret){
-        goto exit;
-    }
-
-    if ( MPI_STATUS_IGNORE != status ) {
-        status->_ucount = max_data;
-    }
-
     vulcan_num_io_procs = fh->f_get_mca_parameter_value ( "num_aggregators", strlen ("num_aggregators"));
-    if ( OMPI_ERR_MAX == vulcan_num_io_procs ) {
+    if (OMPI_ERR_MAX == vulcan_num_io_procs) {
         ret = OMPI_ERROR;
         goto exit;
     }
+    bytes_per_cycle = fh->f_bytes_per_agg;
 
-    ret = mca_common_ompio_set_aggregator_props ((struct ompio_file_t *) fh,
-                                                 vulcan_num_io_procs,
-                                                 max_data);
-    if (OMPI_SUCCESS != ret){
+    if ((1 == mca_fcoll_vulcan_async_io) && (NULL == fh->f_fbtl->fbtl_ipreadv)) {
+        opal_output (1, "vulcan_read_all: fbtl Does NOT support ipreadv() (asynchronous read) \n");
+        ret = MPI_ERR_UNSUPPORTED_OPERATION;
         goto exit;
     }
-    my_aggregator = fh->f_procs_in_group[0];
+
+    mca_common_ompio_check_gpu_buf (fh, buf, &is_gpu, &is_managed);
+    if (is_gpu && !is_managed &&
+        fh->f_get_mca_parameter_value ("use_accelerator_buffers", strlen("use_accelerator_buffers"))) {
+        use_accelerator_buffer = true;
+    }
+    /* since we want to overlap 2 iterations, define the bytes_per_cycle to be half of what
+       the user requested */
+    bytes_per_cycle = bytes_per_cycle/2;
 
     /**************************************************************************
-     ** 2. Determine the total amount of data to be written
+     ** 1. Decode user buffer into an iovec
      **************************************************************************/
-    total_bytes_per_process = (MPI_Aint*)malloc(fh->f_procs_per_group*sizeof(MPI_Aint));
-    if (NULL == total_bytes_per_process) {
-        opal_output (1, "OUT OF MEMORY\n");
-        ret = OMPI_ERR_OUT_OF_RESOURCE;
+    ret = mca_common_ompio_decode_datatype ((struct ompio_file_t *) fh,
+                                            datatype, count, buf, &max_data,
+                                            fh->f_mem_convertor, &decoded_iov,
+                                            &iov_count);
+    if (OMPI_SUCCESS != ret){
         goto exit;
     }
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-    start_rcomm_time = MPI_Wtime();
-#endif
-    ret = ompi_fcoll_base_coll_allgather_array (&max_data,
-						1,
-						MPI_LONG,
-						total_bytes_per_process,
-						1,
-						MPI_LONG,
-						0,
-						fh->f_procs_in_group,
-						fh->f_procs_per_group,
-						fh->f_comm);
+
+    if (MPI_STATUS_IGNORE != status) {
+        status->_ucount = max_data;
+    }
+
+    ret = mca_fcoll_vulcan_get_configuration (fh, vulcan_num_io_procs, max_data);
     if (OMPI_SUCCESS != ret){
         goto exit;
     }
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-    end_rcomm_time = MPI_Wtime();
-    rcomm_time += end_rcomm_time - start_rcomm_time;
-#endif
+    opal_output_verbose(10, ompi_fcoll_base_framework.framework_output,
+                        "Using %d aggregators for the read_all operation \n", fh->f_num_aggrs);
 
-    for (i=0 ; i<fh->f_procs_per_group ; i++) {
-        total_bytes += total_bytes_per_process[i];
+    aggr_data = (mca_io_ompio_aggregator_data **) malloc (fh->f_num_aggrs *
+                                            sizeof(mca_io_ompio_aggregator_data*));
+    if (NULL == aggr_data) {
+        ret = OMPI_ERR_OUT_OF_RESOURCE;
+        goto exit;
     }
 
-    if (NULL != total_bytes_per_process) {
-        free (total_bytes_per_process);
-        total_bytes_per_process = NULL;
+    for (i = 0; i < fh->f_num_aggrs; i++) {
+        // At this point we know the number of aggregators. If there is a correlation between
+        // number of aggregators and number of IO nodes, we know how many aggr_data arrays we need
+        // to allocate.
+        aggr_data[i] = (mca_io_ompio_aggregator_data *) calloc (1, sizeof(mca_io_ompio_aggregator_data));
+        aggr_data[i]->procs_per_group = fh->f_procs_per_group;
+        aggr_data[i]->procs_in_group  = fh->f_procs_in_group;
+        aggr_data[i]->comm = fh->f_comm;
+        // Identify if the process is an aggregator.
+        // If so, aggr_index would be its index in "aggr_data" and "aggregators" arrays.
+        if (fh->f_aggr_list[i] == fh->f_rank) {
+            aggr_index = i;
+        }
     }
 
     /*********************************************************************
-     *** 3. Generate the File offsets/lengths corresponding to this write
+     *** 2. Generate the local offsets/lengths array corresponding to
+     ***    this read operation
      ********************************************************************/
     ret = fh->f_generate_current_file_view ((struct ompio_file_t *) fh,
-                                            max_data,
-                                            &local_iov_array,
+                                            max_data, &local_iov_array,
                                             &local_count);
-
-    if (ret != OMPI_SUCCESS){
+    if (ret != OMPI_SUCCESS) {
         goto exit;
     }
 
-    /*************************************************************
-     *** 4. Allgather the File View information at all processes
-     *************************************************************/
+    /*************************************************************************
+     ** 2b. Separate the local_iov_array entries based on the number of aggregators
+     *************************************************************************/
+    // Modifications for the even distribution:
+    long domain_size;
+    ret = mca_fcoll_vulcan_minmax (fh, local_iov_array, local_count, fh->f_num_aggrs, &domain_size);
 
-    fview_count = (int *) malloc (fh->f_procs_per_group * sizeof (int));
-    if (NULL == fview_count) {
-        opal_output (1, "OUT OF MEMORY\n");
-        ret = OMPI_ERR_OUT_OF_RESOURCE;
-        goto exit;
-    }
+    // broken_iov_arrays[0] contains broken_counts[0] entries to aggregator 0,
+    // broken_iov_arrays[1] contains broken_counts[1] entries to aggregator 1, etc.
+    ret = mca_fcoll_vulcan_break_file_view (decoded_iov, iov_count,
+                                            local_iov_array, local_count,
+                                            &broken_decoded_iovs, &broken_iov_counts,
+                                            &broken_iov_arrays, &broken_counts,
+                                            &broken_total_lengths,
+                                            fh->f_num_aggrs, domain_size);
+
+    /**************************************************************************
+     ** 3. Determine the total amount of data to be read and no. of cycles
+     **************************************************************************/
 #if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-    start_rcomm_time = MPI_Wtime();
+    start_comm_time = MPI_Wtime();
 #endif
-    ret = ompi_fcoll_base_coll_allgather_array (&local_count,
-						1,
-						MPI_INT,
-						fview_count,
-						1,
-						MPI_INT,
-						0,
-						fh->f_procs_in_group,
-						fh->f_procs_per_group,
-						fh->f_comm);
-    
-    if (OMPI_SUCCESS != ret){
+    ret = fh->f_comm->c_coll->coll_allreduce (MPI_IN_PLACE, broken_total_lengths,
+                                              fh->f_num_aggrs, MPI_LONG, MPI_SUM,
+                                              fh->f_comm,
+                                              fh->f_comm->c_coll->coll_allreduce_module);
+    if (OMPI_SUCCESS != ret) {
         goto exit;
     }
+
 #if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-    end_rcomm_time = MPI_Wtime();
-    rcomm_time += end_rcomm_time - start_rcomm_time;
+    end_comm_time = MPI_Wtime();
+    comm_time += (end_comm_time - start_comm_time);
 #endif
 
-    displs = (int*)malloc (fh->f_procs_per_group*sizeof(int));
-    if (NULL == displs) {
-        opal_output (1, "OUT OF MEMORY\n");
-        ret = OMPI_ERR_OUT_OF_RESOURCE;
-        goto exit;
-    }
-
-    displs[0] = 0;
-    total_fview_count = fview_count[0];
-    for (i=1 ; i<fh->f_procs_per_group ; i++) {
-        total_fview_count += fview_count[i];
-        displs[i] = displs[i-1] + fview_count[i-1];
-    }
-
+    cycles=0;
+    for (i = 0; i < fh->f_num_aggrs; i++) {
 #if DEBUG_ON
-    if (my_aggregator == fh->f_rank) {
-    for (i=0 ; i<fh->f_procs_per_group ; i++) {
-    printf ("%d: PROCESS: %d  ELEMENTS: %d  DISPLS: %d\n",
-        fh->f_rank,
-        i,
-        fview_count[i],
-        displs[i]);
-}
-}
+        printf("%d: Overall broken_total_lengths[%d] = %ld\n", fh->f_rank, i, broken_total_lengths[i]);
 #endif
-
-    /* allocate the global iovec  */
-    if (0 != total_fview_count) {
-        global_iov_array = (struct iovec*)malloc (total_fview_count *
-                                                  sizeof(struct iovec));
-        if (NULL == global_iov_array) {
-            opal_output (1, "OUT OF MEMORY\n");
-            ret = OMPI_ERR_OUT_OF_RESOURCE;
-            goto exit;
+        if (ceil((double)broken_total_lengths[i]/bytes_per_cycle) > cycles) {
+            cycles = ceil((double)broken_total_lengths[i]/bytes_per_cycle);
         }
     }
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-    start_rcomm_time = MPI_Wtime();
-#endif
-    ret =  ompi_fcoll_base_coll_allgatherv_array (local_iov_array,
-						  local_count,
-						  fh->f_iov_type,
-						  global_iov_array,
-						  fview_count,
-						  displs,
-						  fh->f_iov_type,
-						  0,
-						  fh->f_procs_in_group,
-						  fh->f_procs_per_group,
-						  fh->f_comm);
-    
-    if (OMPI_SUCCESS != ret){
+
+    result_counts = (int *) malloc (fh->f_num_aggrs * fh->f_procs_per_group * sizeof(int));
+    if (NULL == result_counts) {
+        ret = OMPI_ERR_OUT_OF_RESOURCE;
         goto exit;
     }
+
 #if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-    end_rcomm_time = MPI_Wtime();
-    rcomm_time += end_rcomm_time - start_rcomm_time;
+    start_comm_time = MPI_Wtime();
 #endif
-
-    /****************************************************************************************
-     *** 5. Sort the global offset/lengths list based on the offsets.
-     *** The result of the sort operation is the 'sorted', an integer array,
-     *** which contains the indexes of the global_iov_array based on the offset.
-     *** For example, if global_iov_array[x].offset is followed by global_iov_array[y].offset
-     *** in the file, and that one is followed by global_iov_array[z].offset, than
-     *** sorted[0] = x, sorted[1]=y and sorted[2]=z;
-     ******************************************************************************************/
-    if (0 != total_fview_count) {
-       sorted = (int *)malloc (total_fview_count * sizeof(int));
-      if (NULL == sorted) {
-            opal_output (1, "OUT OF MEMORY\n");
-            ret = OMPI_ERR_OUT_OF_RESOURCE;
-            goto exit;
-        }
-        ompi_fcoll_base_sort_iovec (global_iov_array, total_fview_count, sorted);
-    }
-
-    if (NULL != local_iov_array) {
-        free (local_iov_array);
-        local_iov_array = NULL;
-    }
-
-#if DEBUG_ON
-    if (my_aggregator == fh->f_rank) {
-        for (i=0 ; i<total_fview_count ; i++) {
-            printf("%d: OFFSET: %p   LENGTH: %d\n",
-                   fh->f_rank,
-                   global_iov_array[sorted[i]].iov_base,
-                   global_iov_array[sorted[i]].iov_len);
-        }
+    ret = fh->f_comm->c_coll->coll_allgather (broken_counts, fh->f_num_aggrs, MPI_INT,
+                                              result_counts, fh->f_num_aggrs, MPI_INT,
+                                              fh->f_comm,
+					      fh->f_comm->c_coll->coll_allgather_module);
+    if (OMPI_SUCCESS != ret) {
+        goto exit;
     }
+#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
+    end_comm_time = MPI_Wtime();
+    comm_time += (end_comm_time - start_comm_time);
 #endif
 
     /*************************************************************
-     *** 6. Determine the number of cycles required to execute this
-     ***    operation
+     *** 4. Allgather the offset/lengths array from all processes
      *************************************************************/
-    bytes_per_cycle = fh->f_bytes_per_agg;
-    cycles = ceil((double)total_bytes/bytes_per_cycle);
-
-    if ( my_aggregator == fh->f_rank) {
-      disp_index = (int *)malloc (fh->f_procs_per_group * sizeof (int));
-      if (NULL == disp_index) {
+    for (i = 0; i < fh->f_num_aggrs; i++) {
+        aggr_data[i]->total_bytes = broken_total_lengths[i];
+        aggr_data[i]->decoded_iov = broken_decoded_iovs[i];
+        aggr_data[i]->fview_count = (size_t *)malloc (fh->f_procs_per_group * sizeof (size_t));
+        if (NULL == aggr_data[i]->fview_count) {
             opal_output (1, "OUT OF MEMORY\n");
             ret = OMPI_ERR_OUT_OF_RESOURCE;
             goto exit;
         }
 
-        blocklen_per_process = (int **)malloc (fh->f_procs_per_group * sizeof (int*));
-        if (NULL == blocklen_per_process) {
-            opal_output (1, "OUT OF MEMORY\n");
-            ret = OMPI_ERR_OUT_OF_RESOURCE;
-            goto exit;
+        for (j = 0; j < fh->f_procs_per_group; j++) {
+            aggr_data[i]->fview_count[j] = result_counts[fh->f_num_aggrs*j+i];
         }
 
-        displs_per_process = (MPI_Aint **)malloc (fh->f_procs_per_group * sizeof (MPI_Aint*));
-        if (NULL == displs_per_process){
+        displs = (ptrdiff_t *)malloc (fh->f_procs_per_group * sizeof (ptrdiff_t));
+        if (NULL == displs) {
             opal_output (1, "OUT OF MEMORY\n");
             ret = OMPI_ERR_OUT_OF_RESOURCE;
             goto exit;
         }
 
-        for (i=0;i<fh->f_procs_per_group;i++){
-            blocklen_per_process[i] = NULL;
-            displs_per_process[i] = NULL;
+        displs[0] = 0;
+        total_fview_count = (uint32_t) aggr_data[i]->fview_count[0];
+        for (j = 1 ; j < fh->f_procs_per_group ; j++) {
+            total_fview_count += aggr_data[i]->fview_count[j];
+            displs[j] = displs[j-1] + aggr_data[i]->fview_count[j-1];
         }
 
-	send_req = (MPI_Request *) malloc (fh->f_procs_per_group * sizeof(MPI_Request));
-	if (NULL == send_req){
-	    opal_output ( 1, "OUT OF MEMORY\n");
-	    ret = OMPI_ERR_OUT_OF_RESOURCE;
-	    goto exit;
-	}
-
-	global_buf = (char *) malloc (bytes_per_cycle);
-	if (NULL == global_buf){
-	    opal_output(1, "OUT OF MEMORY\n");
-	    ret = OMPI_ERR_OUT_OF_RESOURCE;
-	    goto exit;
-	}
-
-	sendtype = (ompi_datatype_t **) malloc (fh->f_procs_per_group * sizeof(ompi_datatype_t *));
-	if (NULL == sendtype) {
-            opal_output (1, "OUT OF MEMORY\n");
-	    ret = OMPI_ERR_OUT_OF_RESOURCE;
-	    goto exit;
-	}
-
-	for(l=0;l<fh->f_procs_per_group;l++){
-            sendtype[l] = MPI_DATATYPE_NULL;
-	}
-    }
-
-
-
-
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-    start_rexch = MPI_Wtime();
-#endif
-    n = 0;
-    bytes_remaining = 0;
-    current_index = 0;
-
-    for (index = 0; index < cycles; index++) {
-        /**********************************************************************
-         ***  7a. Getting ready for next cycle: initializing and freeing buffers
-	 **********************************************************************/
-        if (my_aggregator == fh->f_rank) {
-             if (NULL != fh->f_io_array) {
-                free (fh->f_io_array);
-                fh->f_io_array = NULL;
+#if DEBUG_ON
+        printf("total_fview_count : %d\n", total_fview_count);
+        if (fh->f_aggr_list[i] == fh->f_rank) {
+            for (j=0 ; j<fh->f_procs_per_group ; i++) {
+                printf ("%d: PROCESS: %d  ELEMENTS: %ld  DISPLS: %ld\n",
+                        fh->f_rank, j,
+                        aggr_data[i]->fview_count[j],
+                        displs[j]);
             }
-            fh->f_num_of_io_entries = 0;
+        }
+#endif
 
-            if (NULL != sendtype){
-                for (i =0; i< fh->f_procs_per_group; i++) {
-		    if ( MPI_DATATYPE_NULL != sendtype[i] ) {
-                        ompi_datatype_destroy(&sendtype[i]);
-                        sendtype[i] = MPI_DATATYPE_NULL;
-                    }
-		}
+        /* allocate the global iovec  */
+        if (0 != total_fview_count) {
+            aggr_data[i]->global_iov_array = (struct iovec*) malloc (total_fview_count *
+                                                                     sizeof(struct iovec));
+            if (NULL == aggr_data[i]->global_iov_array) {
+                opal_output(1, "OUT OF MEMORY\n");
+                ret = OMPI_ERR_OUT_OF_RESOURCE;
+                goto exit;
             }
+        }
 
-            for(l=0;l<fh->f_procs_per_group;l++){
-                disp_index[l] =  1;
-
-                if (NULL != blocklen_per_process[l]){
-                    free(blocklen_per_process[l]);
-                    blocklen_per_process[l] = NULL;
-                }
-                if (NULL != displs_per_process[l]){
-                    free(displs_per_process[l]);
-                    displs_per_process[l] = NULL;
-                }
-                blocklen_per_process[l] = (int *) calloc (1, sizeof(int));
-                if (NULL == blocklen_per_process[l]) {
-                    opal_output (1, "OUT OF MEMORY for blocklen\n");
-                    ret = OMPI_ERR_OUT_OF_RESOURCE;
-                    goto exit;
-                }
-                displs_per_process[l] = (MPI_Aint *) calloc (1, sizeof(MPI_Aint));
-                if (NULL == displs_per_process[l]){
-                    opal_output (1, "OUT OF MEMORY for displs\n");
-                    ret = OMPI_ERR_OUT_OF_RESOURCE;
-                    goto exit;
-                }
-            }
+#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
+        start_comm_time = MPI_Wtime();
+#endif
+        OMPI_COUNT_ARRAY_INIT(&fview_count_desc, aggr_data[i]->fview_count);
+        OMPI_DISP_ARRAY_INIT(&displs_desc, displs);
+        ret = fh->f_comm->c_coll->coll_allgatherv (broken_iov_arrays[i],
+                                                   broken_counts[i],
+                                                   fh->f_iov_type,
+                                                   aggr_data[i]->global_iov_array,
+                                                   fview_count_desc,
+                                                   displs_desc,
+                                                   fh->f_iov_type,
+                                                   fh->f_comm,
+                                                   fh->f_comm->c_coll->coll_allgatherv_module );
+        if (OMPI_SUCCESS != ret) {
+            goto exit;
+        }
 
-            if (NULL != sorted_file_offsets){
-                free(sorted_file_offsets);
-                sorted_file_offsets = NULL;
-            }
+#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
+        end_comm_time = MPI_Wtime();
+        comm_time += (end_comm_time - start_comm_time);
+#endif
 
-            if(NULL != file_offsets_for_agg){
-                free(file_offsets_for_agg);
-                file_offsets_for_agg = NULL;
-            }
-            if (NULL != memory_displacements){
-                free(memory_displacements);
-                memory_displacements = NULL;
+        /****************************************************************************************
+         *** 5. Sort the global offset/lengths list based on the offsets.
+         *** The result of the sort operation is the 'sorted', an integer array,
+         *** which contains the indexes of the global_iov_array based on the offset.
+         *** For example, if global_iov_array[x].offset is followed by global_iov_array[y].offset
+         *** in the file, and that one is followed by global_iov_array[z].offset, than
+         *** sorted[0] = x, sorted[1]=y and sorted[2]=z;
+         ******************************************************************************************/
+        if (0 != total_fview_count) {
+            aggr_data[i]->sorted = (int *)malloc (total_fview_count * sizeof(int));
+            if (NULL == aggr_data[i]->sorted) {
+                opal_output (1, "OUT OF MEMORY\n");
+                ret = OMPI_ERR_OUT_OF_RESOURCE;
+                goto exit;
             }
-        }  /* (my_aggregator == fh->f_rank */
+            ompi_fcoll_base_sort_iovec (aggr_data[i]->global_iov_array, total_fview_count,
+					aggr_data[i]->sorted);
+        }
 
-        /**************************************************************************
-         ***  7b. Determine the number of bytes to be actually read in this cycle
-	 **************************************************************************/
-        if (cycles-1 == index) {
-            bytes_to_read_in_cycle = total_bytes - bytes_per_cycle*index;
+        if (NULL != local_iov_array) {
+            free(local_iov_array);
+            local_iov_array = NULL;
         }
-        else {
-            bytes_to_read_in_cycle = bytes_per_cycle;
+
+        if (NULL != displs) {
+            free(displs);
+            displs=NULL;
         }
 
 #if DEBUG_ON
-        if (my_aggregator == fh->f_rank) {
-            printf ("****%d: CYCLE %d   Bytes %d**********\n",
-                    fh->f_rank,
-                    index,
-                    bytes_to_write_in_cycle);
+        if (fh->f_aggr_list[i] == fh->f_rank) {
+            uint32_t tv=0;
+            for (tv = 0 ; tv < total_fview_count ; tv++) {
+                printf("%d: OFFSET: %lu   LENGTH: %ld\n",
+                       fh->f_rank,
+                       (uint64_t)aggr_data[i]->global_iov_array[aggr_data[i]->sorted[tv]].iov_base,
+                       aggr_data[i]->global_iov_array[aggr_data[i]->sorted[tv]].iov_len);
+            }
         }
 #endif
-
-        /*****************************************************************
-         *** 7c. Calculate how much data will be contributed in this cycle
-	 ***     by each process
-         *****************************************************************/
-        bytes_received = 0;
-
-        while (bytes_to_read_in_cycle) {
-            /* This next block identifies which process is the holder
-            ** of the sorted[current_index] element;
-            */
-            blocks = fview_count[0];
-            for (j=0 ; j<fh->f_procs_per_group ; j++) {
-                if (sorted[current_index] < blocks) {
-                    n = j;
-                    break;
-                }
-                else {
-                    blocks += fview_count[j+1];
-                }
+        /*************************************************************
+         *** 6. Determine the number of cycles required to execute this
+         ***    operation
+         *************************************************************/
+        aggr_data[i]->bytes_per_cycle = bytes_per_cycle;
+
+        if (fh->f_aggr_list[i] == fh->f_rank) {
+            aggr_data[i]->disp_index = (int *)malloc (fh->f_procs_per_group * sizeof (int));
+            if (NULL == aggr_data[i]->disp_index) {
+                opal_output (1, "OUT OF MEMORY\n");
+                ret = OMPI_ERR_OUT_OF_RESOURCE;
+                goto exit;
             }
 
-            if (bytes_remaining) {
-                /* Finish up a partially used buffer from the previous  cycle */
-                if (bytes_remaining <= bytes_to_read_in_cycle) {
-                    /* Data fits completely into the block */
-                    if (my_aggregator == fh->f_rank) {
-                        blocklen_per_process[n][disp_index[n] - 1] = bytes_remaining;
-                        displs_per_process[n][disp_index[n] - 1] =
-                            (ptrdiff_t)global_iov_array[sorted[current_index]].iov_base +
-                            (global_iov_array[sorted[current_index]].iov_len - bytes_remaining);
-
-                        blocklen_per_process[n] = (int *) realloc
-                            ((void *)blocklen_per_process[n], (disp_index[n]+1)*sizeof(int));
-                        displs_per_process[n] = (MPI_Aint *) realloc
-                            ((void *)displs_per_process[n], (disp_index[n]+1)*sizeof(MPI_Aint));
-                        blocklen_per_process[n][disp_index[n]] = 0;
-                        displs_per_process[n][disp_index[n]] = 0;
-                        disp_index[n] += 1;
-                    }
-                    if (fh->f_procs_in_group[n] == fh->f_rank) {
-                        bytes_received += bytes_remaining;
-                    }
-                    current_index ++;
-                    bytes_to_read_in_cycle -= bytes_remaining;
-                    bytes_remaining = 0;
-                    continue;
-                }
-                else {
-                     /* the remaining data from the previous cycle is larger than the
-                        bytes_to_write_in_cycle, so we have to segment again */
-                    if (my_aggregator == fh->f_rank) {
-                        blocklen_per_process[n][disp_index[n] - 1] = bytes_to_read_in_cycle;
-                        displs_per_process[n][disp_index[n] - 1] =
-                            (ptrdiff_t)global_iov_array[sorted[current_index]].iov_base +
-                            (global_iov_array[sorted[current_index]].iov_len
-                             - bytes_remaining);
-                    }
-                    if (fh->f_procs_in_group[n] == fh->f_rank) {
-                        bytes_received += bytes_to_read_in_cycle;
-                    }
-                    bytes_remaining -= bytes_to_read_in_cycle;
-                    bytes_to_read_in_cycle = 0;
-                    break;
-                }
+            aggr_data[i]->max_disp_index = (int *)calloc (fh->f_procs_per_group,  sizeof (int));
+            if (NULL == aggr_data[i]->max_disp_index) {
+                opal_output (1, "OUT OF MEMORY\n");
+                ret = OMPI_ERR_OUT_OF_RESOURCE;
+                goto exit;
             }
-            else {
-                /* No partially used entry available, have to start a new one */
-                if (bytes_to_read_in_cycle <
-                    (MPI_Aint) global_iov_array[sorted[current_index]].iov_len) {
-                    /* This entry has more data than we can sendin one cycle */
-                    if (my_aggregator == fh->f_rank) {
-                        blocklen_per_process[n][disp_index[n] - 1] = bytes_to_read_in_cycle;
-                        displs_per_process[n][disp_index[n] - 1] =
-                            (ptrdiff_t)global_iov_array[sorted[current_index]].iov_base ;
-                    }
 
-                    if (fh->f_procs_in_group[n] == fh->f_rank) {
-                        bytes_received += bytes_to_read_in_cycle;
-                    }
-                    bytes_remaining = global_iov_array[sorted[current_index]].iov_len -
-                        bytes_to_read_in_cycle;
-                    bytes_to_read_in_cycle = 0;
-                    break;
-                }
-                else {
-                    /* Next data entry is less than bytes_to_write_in_cycle */
-                    if (my_aggregator ==  fh->f_rank) {
-                        blocklen_per_process[n][disp_index[n] - 1] =
-                            global_iov_array[sorted[current_index]].iov_len;
-                        displs_per_process[n][disp_index[n] - 1] = (ptrdiff_t)
-                            global_iov_array[sorted[current_index]].iov_base;
-                        blocklen_per_process[n] =
-                            (int *) realloc ((void *)blocklen_per_process[n], (disp_index[n]+1)*sizeof(int));
-                        displs_per_process[n] = (MPI_Aint *)realloc
-                            ((void *)displs_per_process[n], (disp_index[n]+1)*sizeof(MPI_Aint));
-                        blocklen_per_process[n][disp_index[n]] = 0;
-                        displs_per_process[n][disp_index[n]] = 0;
-                        disp_index[n] += 1;
-                    }
-                    if (fh->f_procs_in_group[n] == fh->f_rank) {
-                        bytes_received +=
-                            global_iov_array[sorted[current_index]].iov_len;
-                    }
-                    bytes_to_read_in_cycle -=
-                        global_iov_array[sorted[current_index]].iov_len;
-                    current_index ++;
-                    continue;
-                }
+            aggr_data[i]->blocklen_per_process = (int **)calloc (fh->f_procs_per_group, sizeof (int*));
+            if (NULL == aggr_data[i]->blocklen_per_process) {
+                opal_output (1, "OUT OF MEMORY\n");
+                ret = OMPI_ERR_OUT_OF_RESOURCE;
+                goto exit;
             }
-        } /* end while (bytes_to_read_in_cycle) */
-
-        /*************************************************************************
-	 *** 7d. Calculate the displacement on where to put the data and allocate
-         ***     the recieve buffer (global_buf)
-	 *************************************************************************/
-        if (my_aggregator == fh->f_rank) {
-            entries_per_aggregator=0;
-            for (i=0;i<fh->f_procs_per_group; i++){
-                for (j=0;j<disp_index[i];j++){
-                    if (blocklen_per_process[i][j] > 0)
-                        entries_per_aggregator++ ;
-                }
+
+            aggr_data[i]->displs_per_process = (MPI_Aint **)calloc (fh->f_procs_per_group, sizeof (MPI_Aint*));
+            if (NULL == aggr_data[i]->displs_per_process) {
+                opal_output (1, "OUT OF MEMORY\n");
+                ret = OMPI_ERR_OUT_OF_RESOURCE;
+                goto exit;
             }
-            if (entries_per_aggregator > 0){
-                file_offsets_for_agg = (mca_io_ompio_local_io_array *)
-                    malloc(entries_per_aggregator*sizeof(mca_io_ompio_local_io_array));
-                if (NULL == file_offsets_for_agg) {
-                    opal_output (1, "OUT OF MEMORY\n");
+
+            if (use_accelerator_buffer) {
+                opal_output_verbose(10, ompi_fcoll_base_framework.framework_output,
+                                    "Allocating GPU device buffer for aggregation\n");
+                ret = opal_accelerator.mem_alloc(MCA_ACCELERATOR_NO_DEVICE_ID, (void**)&aggr_data[i]->global_buf,
+                                                 bytes_per_cycle);
+                if (OPAL_SUCCESS != ret) {
+                    opal_output(1, "Could not allocate accelerator memory");
                     ret = OMPI_ERR_OUT_OF_RESOURCE;
                     goto exit;
                 }
-                sorted_file_offsets = (int *)
-                    malloc (entries_per_aggregator*sizeof(int));
-                if (NULL == sorted_file_offsets){
-                    opal_output (1, "OUT OF MEMORY\n");
-                    ret =  OMPI_ERR_OUT_OF_RESOURCE;
+                ret = opal_accelerator.mem_alloc(MCA_ACCELERATOR_NO_DEVICE_ID, (void**)&aggr_data[i]->prev_global_buf,
+                                                 bytes_per_cycle);
+                if (OPAL_SUCCESS != ret) {
+                    opal_output(1, "Could not allocate accelerator memory");
+                    ret = OMPI_ERR_OUT_OF_RESOURCE;
                     goto exit;
                 }
-                /*Moving file offsets to an IO array!*/
-                temp_index = 0;
-                global_count = 0;
-                for (i=0;i<fh->f_procs_per_group; i++){
-                    for(j=0;j<disp_index[i];j++){
-                        if (blocklen_per_process[i][j] > 0){
-                            file_offsets_for_agg[temp_index].length =
-                                blocklen_per_process[i][j];
-                            global_count += blocklen_per_process[i][j];
-                            file_offsets_for_agg[temp_index].process_id = i;
-                            file_offsets_for_agg[temp_index].offset =
-                                displs_per_process[i][j];
-                            temp_index++;
-                        }
-                    }
+            } else {
+                aggr_data[i]->global_buf       = (char *) malloc (bytes_per_cycle);
+                aggr_data[i]->prev_global_buf  = (char *) malloc (bytes_per_cycle);
+                if (NULL == aggr_data[i]->global_buf || NULL == aggr_data[i]->prev_global_buf){
+                    opal_output(1, "OUT OF MEMORY");
+                    ret = OMPI_ERR_OUT_OF_RESOURCE;
+                    goto exit;
                 }
             }
-            else{
-                continue;
-            }
 
-             /* Sort the displacements for each aggregator */
-            read_heap_sort (file_offsets_for_agg,
-                            entries_per_aggregator,
-                            sorted_file_offsets);
-
-            memory_displacements = (MPI_Aint *) malloc
-                (entries_per_aggregator * sizeof(MPI_Aint));
-            memory_displacements[sorted_file_offsets[0]] = 0;
-            for (i=1; i<entries_per_aggregator; i++){
-                memory_displacements[sorted_file_offsets[i]] =
-                    memory_displacements[sorted_file_offsets[i-1]] +
-                    file_offsets_for_agg[sorted_file_offsets[i-1]].length;
-            }
-
-             /**********************************************************
-	     *** 7e. Create the io array, and pass it to fbtl
-	     *********************************************************/
-            fh->f_io_array = (mca_common_ompio_io_array_t *) malloc
-                (entries_per_aggregator * sizeof (mca_common_ompio_io_array_t));
-            if (NULL == fh->f_io_array) {
-                opal_output(1, "OUT OF MEMORY\n");
+            aggr_data[i]->recvtype = (ompi_datatype_t **) malloc (fh->f_procs_per_group  *
+                                                                  sizeof(ompi_datatype_t *));
+            aggr_data[i]->prev_recvtype = (ompi_datatype_t **) malloc (fh->f_procs_per_group  *
+                                                                       sizeof(ompi_datatype_t *));
+            if (NULL == aggr_data[i]->recvtype || NULL == aggr_data[i]->prev_recvtype) {
+                opal_output (1, "OUT OF MEMORY\n");
                 ret = OMPI_ERR_OUT_OF_RESOURCE;
                 goto exit;
             }
-
-            fh->f_num_of_io_entries = 0;
-            fh->f_io_array[0].offset =
-                (IOVBASE_TYPE *)(intptr_t)file_offsets_for_agg[sorted_file_offsets[0]].offset;
-            fh->f_io_array[0].length =
-                file_offsets_for_agg[sorted_file_offsets[0]].length;
-            fh->f_io_array[0].memory_address =
-                global_buf+memory_displacements[sorted_file_offsets[0]];
-            fh->f_num_of_io_entries++;
-            for (i=1;i<entries_per_aggregator;i++){
-                if (file_offsets_for_agg[sorted_file_offsets[i-1]].offset +
-                    file_offsets_for_agg[sorted_file_offsets[i-1]].length ==
-                    file_offsets_for_agg[sorted_file_offsets[i]].offset){
-                    fh->f_io_array[fh->f_num_of_io_entries - 1].length +=
-                        file_offsets_for_agg[sorted_file_offsets[i]].length;
-                }
-                else{
-                    fh->f_io_array[fh->f_num_of_io_entries].offset =
-                        (IOVBASE_TYPE *)(intptr_t)file_offsets_for_agg[sorted_file_offsets[i]].offset;
-                    fh->f_io_array[fh->f_num_of_io_entries].length =
-                        file_offsets_for_agg[sorted_file_offsets[i]].length;
-                    fh->f_io_array[fh->f_num_of_io_entries].memory_address =
-                        global_buf+memory_displacements[sorted_file_offsets[i]];
-                    fh->f_num_of_io_entries++;
-                }
+            for(l=0;l<fh->f_procs_per_group;l++){
+                aggr_data[i]->recvtype[l]      = MPI_DATATYPE_NULL;
+                aggr_data[i]->prev_recvtype[l] = MPI_DATATYPE_NULL;
             }
-
+        }
 
 #if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-            start_read_time = MPI_Wtime();
+        start_exch = MPI_Wtime();
 #endif
+    }
 
-            if (fh->f_num_of_io_entries) {
-                if ( 0 >  fh->f_fbtl->fbtl_preadv (fh)) {
-                    opal_output (1, "READ FAILED\n");
-                    ret = OMPI_ERROR;
-                    goto exit;
-                }
-            }
+    reqs = (ompi_request_t **)malloc ((fh->f_procs_per_group + 1 )*fh->f_num_aggrs *sizeof(ompi_request_t *));
+    if (NULL == reqs) {
+        opal_output (1, "OUT OF MEMORY\n");
+        ret = OMPI_ERR_OUT_OF_RESOURCE;
+        goto exit;
+    }
+
+    for (l = 0, i = 0; i < fh->f_num_aggrs; i++) {
+        for (j=0; j< (fh->f_procs_per_group+1); j++) {
+            reqs[l] = MPI_REQUEST_NULL;
+            l++;
+        }
+    }
+
+    if( (1 == mca_fcoll_vulcan_async_io) ||
+        ( (0 == mca_fcoll_vulcan_async_io) && (NULL != fh->f_fbtl->fbtl_ipreadv) && (2 < cycles))) {
+        read_sync_type = 1;
+    }
 
+    if (cycles > 0) {
+        if (NOT_AGGR_INDEX != aggr_index) {
+	    // Register progress function that should be used by ompi_request_wait
+	    mca_common_ompio_register_progress ();
+	}
 #if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-            end_read_time = MPI_Wtime();
-            read_time += end_read_time - start_read_time;
+	start_read_time = MPI_Wtime();
 #endif
-            /**********************************************************
-             ******************** DONE READING ************************
-             *********************************************************/
-
-            temp_disp_index = (int *)calloc (1, fh->f_procs_per_group * sizeof (int));
-            if (NULL == temp_disp_index) {
-                opal_output (1, "OUT OF MEMORY\n");
-                ret = OMPI_ERR_OUT_OF_RESOURCE;
+        for (i = 0; i < fh->f_num_aggrs; i++) {            
+            ret = read_init (fh, 0, cycles, fh->f_aggr_list[i], fh->f_rank,
+                             aggr_data[i], read_sync_type, &req_tmp,
+                             use_accelerator_buffer);
+            if (OMPI_SUCCESS != ret) {
                 goto exit;
             }
-            for (i=0; i<entries_per_aggregator; i++){
-                temp_index =
-                    file_offsets_for_agg[sorted_file_offsets[i]].process_id;
-                displs_per_process[temp_index][temp_disp_index[temp_index]] =
-                    memory_displacements[sorted_file_offsets[i]];
-                if (temp_disp_index[temp_index] < disp_index[temp_index]){
-                    temp_disp_index[temp_index] += 1;
-                }
-                else{
-                    printf("temp_disp_index[%d]: %d is greater than disp_index[%d]: %d\n",
-                           temp_index, temp_disp_index[temp_index],
-                           temp_index, disp_index[temp_index]);
-                }
-            }
-            if (NULL != temp_disp_index){
-                free(temp_disp_index);
-                temp_disp_index = NULL;
+            if (fh->f_aggr_list[i] == fh->f_rank) {
+                req_iread = req_tmp;
             }
+	}
 
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-            start_rcomm_time = MPI_Wtime();
-#endif
-            for (i=0;i<fh->f_procs_per_group;i++){
-                size_t datatype_size;
-                send_req[i] = MPI_REQUEST_NULL;
-                if ( 0 < disp_index[i] ) {
-                    ompi_datatype_create_hindexed(disp_index[i],
-                                                  blocklen_per_process[i],
-                                                  displs_per_process[i],
-                                                  MPI_BYTE,
-                                                  &sendtype[i]);
-                    ompi_datatype_commit(&sendtype[i]);
-                    opal_datatype_type_size(&sendtype[i]->super, &datatype_size);
-
-                    if(datatype_size) {
-                        ret = MCA_PML_CALL (isend(global_buf,
-                                                  1,
-                                                  sendtype[i],
-                                                  fh->f_procs_in_group[i],
-                                                  FCOLL_VULCAN_SHUFFLE_TAG,
-                                                  MCA_PML_BASE_SEND_STANDARD,
-                                                  fh->f_comm,
-                                                  &send_req[i]));
-                        if(OMPI_SUCCESS != ret){
-                            goto exit;
-                        }
-                    }
-                }
+        if (NOT_AGGR_INDEX != aggr_index) {
+            ret = ompi_request_wait(&req_iread, MPI_STATUS_IGNORE);
+            if (OMPI_SUCCESS != ret){
+                goto exit;
             }
+        }
 #if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-            end_rcomm_time = MPI_Wtime();
-            rcomm_time += end_rcomm_time - start_rcomm_time;
+	end_read_time = MPI_Wtime();
+	read_time += end_read_time - start_read_time;
 #endif
-        }
-
-        /**********************************************************
-         *** 7f.  Scatter the Data from the readers
-         *********************************************************/
-        if(bytes_received) {
-            size_t remaining            = bytes_received;
-            int block_index             = -1;
-            int blocklength_size        = INIT_LEN;
-
-            ptrdiff_t recv_mem_address  = 0;
-            ompi_datatype_t *newType    = MPI_DATATYPE_NULL;
-
-            blocklength_proc            = (int *)       calloc (blocklength_size, sizeof (int));
-            displs_proc                 = (ptrdiff_t *) calloc (blocklength_size, sizeof (ptrdiff_t));
+    }
 
-            if (NULL == blocklength_proc || NULL == displs_proc ) {
-                opal_output (1, "OUT OF MEMORY\n");
-                ret = OMPI_ERR_OUT_OF_RESOURCE;
+    for (index = 1; index < cycles; index++) {
+        for (i = 0; i < fh->f_num_aggrs; i++) {
+            ret = shuffle_init (index-1, cycles, fh->f_aggr_list[i], fh->f_rank, aggr_data[i],
+                                &reqs[i*(fh->f_procs_per_group + 1)] );
+            if (OMPI_SUCCESS != ret) {
                 goto exit;
             }
+        }
 
-            while (remaining) {
-                block_index++;
-
-                if(0 == block_index) {
-                    recv_mem_address = (ptrdiff_t) (decoded_iov[iov_index].iov_base) + current_position;
-                }
-                else {
-                    // Reallocate more memory if blocklength_size is not enough
-                    if(0 == block_index % INIT_LEN) {
-                        blocklength_size += INIT_LEN;
-                        blocklength_proc = (int *)       realloc(blocklength_proc, blocklength_size * sizeof(int));
-                        displs_proc      = (ptrdiff_t *) realloc(displs_proc, blocklength_size * sizeof(ptrdiff_t));
-                    }
-                    displs_proc[block_index] = (ptrdiff_t) (decoded_iov[iov_index].iov_base) +
-                                                            current_position - recv_mem_address;
-                }
-
-                if (remaining >= (decoded_iov[iov_index].iov_len - current_position)) {
-                    blocklength_proc[block_index] = decoded_iov[iov_index].iov_len - current_position;
-
-                    remaining = remaining - blocklength_proc[block_index];
-                    iov_index = iov_index + 1;
-                    current_position = 0;
-                }
-                else {
-                    blocklength_proc[block_index] = remaining;
-                    current_position += remaining;
-                    remaining = 0;
-                }
-            }
-
-            ompi_datatype_create_hindexed(block_index+1,
-                                          blocklength_proc,
-                                          displs_proc,
-                                          MPI_BYTE,
-                                          &newType);
-            ompi_datatype_commit(&newType);
-
+        SWAP_AGGR_POINTERS(aggr_data, fh->f_num_aggrs);
 #if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-            start_rcomm_time = MPI_Wtime();
+        start_read_time = MPI_Wtime();
 #endif
-            ret = MCA_PML_CALL(irecv((char *)recv_mem_address,
-                                     1,
-                                     newType,
-                                     my_aggregator,
-                                     FCOLL_VULCAN_SHUFFLE_TAG,
-                                     fh->f_comm,
-                                     &recv_req));
-
-            if ( MPI_DATATYPE_NULL != newType ) {
-                ompi_datatype_destroy(&newType);
-            }
-
+        for (i = 0; i < fh->f_num_aggrs; i++) {
+            ret = read_init (fh, index, cycles, fh->f_aggr_list[i], fh->f_rank,
+			     aggr_data[i], read_sync_type,
+			     &req_tmp, use_accelerator_buffer);
             if (OMPI_SUCCESS != ret){
                 goto exit;
             }
+            if (fh->f_aggr_list[i] == fh->f_rank) {
+                req_iread = req_tmp;
+            }
+        }
+#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
+	end_read_time = MPI_Wtime();
+	read_time += end_read_time - start_read_time;
+#endif
+	ret = ompi_request_wait_all ((fh->f_procs_per_group + 1 )*fh->f_num_aggrs,
+                                     reqs, MPI_STATUS_IGNORE);
+        if (OMPI_SUCCESS != ret){
+            goto exit;
         }
 
-        if (my_aggregator == fh->f_rank){
-            ret = ompi_request_wait_all (fh->f_procs_per_group,
-                                         send_req,
-                                         MPI_STATUS_IGNORE);
+        if (NOT_AGGR_INDEX != aggr_index) {
+            ret = ompi_request_wait (&req_iread, MPI_STATUS_IGNORE);
             if (OMPI_SUCCESS != ret){
                 goto exit;
             }
         }
+    } /* end  for (index = 1; index < cycles; index++) */
 
-        ret = ompi_request_wait (&recv_req, MPI_STATUS_IGNORE);
+    if (cycles > 0) {
+        for (i = 0; i < fh->f_num_aggrs; i++) {
+            ret = shuffle_init (index-1, cycles, fh->f_aggr_list[i], fh->f_rank, aggr_data[i],
+                                &reqs[i*(fh->f_procs_per_group + 1)] );
+            if (OMPI_SUCCESS != ret) {
+                goto exit;
+            }
+        }
+	ret = ompi_request_wait_all ((fh->f_procs_per_group + 1 )*fh->f_num_aggrs,
+                                     reqs, MPI_STATUS_IGNORE);
         if (OMPI_SUCCESS != ret){
             goto exit;
         }
+    }
 
 #if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-        if(bytes_received) {
-            end_rcomm_time = MPI_Wtime();
-            rcomm_time += end_rcomm_time - start_rcomm_time;
-        }
-#endif
-    } /* end for (index=0; index < cycles; index ++) */
-
-#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-    end_rexch = MPI_Wtime();
-    read_exch += end_rexch - start_rexch;
+    end_exch = MPI_Wtime();
+    exch_read += end_exch - start_exch;
     nentry.time[0] = read_time;
-    nentry.time[1] = rcomm_time;
-    nentry.time[2] = read_exch;
-    if (my_aggregator == fh->f_rank)
+    nentry.time[1] = comm_time;
+    nentry.time[2] = exch_read;
+    nentry.aggregator = 0;
+    for ( i=0; i<fh->f_num_aggrs; i++ ) {
+        if (fh->f_aggr_list[i] == fh->f_rank)
         nentry.aggregator = 1;
-    else
-        nentry.aggregator = 0;
-    nentry.nprocs_for_coll = vulcan_num_io_procs;
+    }
+    nentry.nprocs_for_coll = fh->f_num_aggrs;
     if (!mca_common_ompio_full_print_queue(fh->f_coll_read_time)){
         mca_common_ompio_register_print_entry(fh->f_coll_read_time,
-                                              nentry);
+                                               nentry);
     }
 #endif
 
-exit:
-    if (NULL != global_buf) {
-        free (global_buf);
-        global_buf = NULL;
-    }
-    if (NULL != sorted) {
-        free (sorted);
-        sorted = NULL;
-    }
-    if (NULL != global_iov_array) {
-        free (global_iov_array);
-        global_iov_array = NULL;
-    }
-    if (NULL != fview_count) {
-        free (fview_count);
-        fview_count = NULL;
-    }
-    if (NULL != decoded_iov) {
-        free (decoded_iov);
-        decoded_iov = NULL;
-    }
-    if (NULL != local_iov_array){
-        free(local_iov_array);
-        local_iov_array=NULL;
-    }
+exit :
+    if (NULL != aggr_data) {
 
-    if (NULL != displs) {
-        free (displs);
-        displs = NULL;
-    }
+        for (i = 0; i < fh->f_num_aggrs; i++) {
+            if (fh->f_aggr_list[i] == fh->f_rank) {
+                if (NULL != aggr_data[i]->recvtype){
+                    for (j = 0; j < aggr_data[i]->procs_per_group; j++) {
+                        if (MPI_DATATYPE_NULL != aggr_data[i]->recvtype[j]) {
+                            ompi_datatype_destroy(&aggr_data[i]->recvtype[j]);
+                        }
+                        if (MPI_DATATYPE_NULL != aggr_data[i]->prev_recvtype[j]) {
+                            ompi_datatype_destroy(&aggr_data[i]->prev_recvtype[j]);
+                        }
+                    }
+                    free(aggr_data[i]->recvtype);
+                    free(aggr_data[i]->prev_recvtype);
+                }
+
+                free (aggr_data[i]->disp_index);
+                free (aggr_data[i]->max_disp_index);
+                if (use_accelerator_buffer) {
+                    opal_accelerator.mem_release(MCA_ACCELERATOR_NO_DEVICE_ID, aggr_data[i]->global_buf);
+                    opal_accelerator.mem_release(MCA_ACCELERATOR_NO_DEVICE_ID, aggr_data[i]->prev_global_buf);
+                } else {
+                    free (aggr_data[i]->global_buf);
+                    free (aggr_data[i]->prev_global_buf);
+                }
+                for (l = 0;l < aggr_data[i]->procs_per_group; l++) {
+                    free (aggr_data[i]->blocklen_per_process[l]);
+                    free (aggr_data[i]->displs_per_process[l]);
+                }
 
-    if (NULL != blocklength_proc) {
-        free (blocklength_proc);
-        blocklength_proc = NULL;
+                free (aggr_data[i]->blocklen_per_process);
+                free (aggr_data[i]->displs_per_process);
+            }
+            free (aggr_data[i]->sorted);
+            free (aggr_data[i]->global_iov_array);
+            free (aggr_data[i]->fview_count);
+            free (aggr_data[i]->decoded_iov);
+
+            free (aggr_data[i]);
+        }
+        free (aggr_data);
+    }
+    free(displs);
+    free(decoded_iov);
+    free(broken_counts);
+    free(broken_total_lengths);
+    free(broken_iov_counts);
+    free(broken_decoded_iovs); // decoded_iov arrays[i] were freed as aggr_data[i]->decoded_iov;
+    if (NULL != broken_iov_arrays) {
+        for (i = 0; i < fh->f_num_aggrs; i++) {
+            free(broken_iov_arrays[i]);
+        }
     }
+    free(broken_iov_arrays);
+    free(fh->f_procs_in_group);
+    free(fh->f_aggr_list);
+    fh->f_procs_in_group=NULL;
+    fh->f_procs_per_group=0;
+    fh->f_aggr_list=NULL;
+    free(result_counts);
+    free(reqs);
+
+    return ret;
+}
+
+static int read_init (ompio_file_t *fh, int index, int cycles, int aggregator, int rank,
+		      mca_io_ompio_aggregator_data *data,
+                      int read_syncType, ompi_request_t **request,
+                      bool is_accelerator_buffer)
+{
+    int ret = OMPI_SUCCESS;
+    ssize_t ret_temp = 0;
+    mca_ompio_request_t *ompio_req = NULL;
+    int i, j, l;
+    int entries_per_aggregator=0;
+    mca_io_ompio_local_io_array *file_offsets_for_agg=NULL;
+    MPI_Aint *memory_displacements=NULL;
+    int* blocklength_proc=NULL;
+    ptrdiff_t* displs_proc=NULL;
+    int *sorted_file_offsets=NULL;
+
+    /**********************************************************************
+     ***  7a. Getting ready for next cycle: initializing and freeing buffers
+     **********************************************************************/
+    data->bytes_sent = 0;
+
+    if (aggregator == rank) {
+	if (NULL != data->recvtype){
+	    for (i = 0; i < data->procs_per_group; i++) {
+		if (MPI_DATATYPE_NULL != data->recvtype[i]) {
+		    ompi_datatype_destroy(&data->recvtype[i]);
+		    data->recvtype[i] = MPI_DATATYPE_NULL;
+		}
+	    }
+	}
+
+	for (l = 0; l < data->procs_per_group; l++) {
+	    data->disp_index[l] = 0;
+
+	    if (data->max_disp_index[l] == 0) {
+		data->blocklen_per_process[l] = (int *) calloc (INIT_LEN, sizeof(int));
+		data->displs_per_process[l] = (MPI_Aint *) calloc (INIT_LEN, sizeof(MPI_Aint));
+		if (NULL == data->displs_per_process[l] || NULL == data->blocklen_per_process[l]){
+		    opal_output (1, "OUT OF MEMORY for displs\n");
+		    ret = OMPI_ERR_OUT_OF_RESOURCE;
+		    goto exit;
+		}
+		data->max_disp_index[l] = INIT_LEN;
+	    } else {
+		memset (data->blocklen_per_process[l], 0, data->max_disp_index[l]*sizeof(int));
+		memset (data->displs_per_process[l], 0, data->max_disp_index[l]*sizeof(MPI_Aint));
+	    }
+	}
+    } /* rank == aggregator */
 
-    if (NULL != displs_proc) {
-        free (displs_proc);
-        displs_proc = NULL;
+    /**************************************************************************
+     ***  7b. Determine the number of bytes to be actually read in this cycle
+     **************************************************************************/
+    int local_cycles= ceil((double)data->total_bytes / data->bytes_per_cycle);
+    if (index  < (local_cycles -1)) {
+        data->bytes_to_write_in_cycle = data->bytes_per_cycle;
+    } else if ( index == (local_cycles -1)) {
+        data->bytes_to_write_in_cycle = data->total_bytes - data->bytes_per_cycle*index;
+    } else {
+        data->bytes_to_write_in_cycle = 0;
     }
+    data->bytes_to_write = data->bytes_to_write_in_cycle;
 
-    if (my_aggregator == fh->f_rank) {
+#if DEBUG_ON
+    if (aggregator == rank) {
+        printf ("****%d: CYCLE %d   Bytes %d**********\n",
+                rank, index, data->bytes_to_write_in_cycle);
+    }
+#endif
 
-        if (NULL != sorted_file_offsets){
-            free(sorted_file_offsets);
-            sorted_file_offsets = NULL;
-        }
-        if (NULL != file_offsets_for_agg){
-            free(file_offsets_for_agg);
-            file_offsets_for_agg = NULL;
-        }
-        if (NULL != memory_displacements){
-            free(memory_displacements);
-            memory_displacements= NULL;
-        }
-        if (NULL != sendtype){
-            for (i = 0; i < fh->f_procs_per_group; i++) {
-                if ( MPI_DATATYPE_NULL != sendtype[i] ) {
-                    ompi_datatype_destroy(&sendtype[i]);
-                }
+    /*****************************************************************
+     *** 7c. Calculate how much data will be sent to each process in
+     *** this cycle
+     *****************************************************************/
+    mca_fcoll_vulcan_calc_blocklen_disps(data, aggregator, rank, &data->bytes_sent);
+
+    /*************************************************************************
+     *** 7d. Calculate the displacement
+     *************************************************************************/
+    if (rank == aggregator) {
+        for (i = 0; i < data->procs_per_group; i++){
+            for (j = 0; j < data->disp_index[i]; j++){
+                if (data->blocklen_per_process[i][j] > 0)
+                    entries_per_aggregator++ ;
             }
-            free(sendtype);
-            sendtype=NULL;
         }
+    }
+#if DEBUG_ON
+    if (aggregator == rank) {
+        printf("%d : Entries per aggregator : %d\n", rank, entries_per_aggregator);
+    }
+#endif
 
-        if (NULL != disp_index){
-            free(disp_index);
-            disp_index = NULL;
-        }
+    if (entries_per_aggregator > 0) {
+	file_offsets_for_agg = (mca_io_ompio_local_io_array *) malloc (entries_per_aggregator
+							* sizeof(mca_io_ompio_local_io_array));
+	memory_displacements = (MPI_Aint *) malloc (entries_per_aggregator * sizeof(MPI_Aint));
+	sorted_file_offsets = (int *) malloc (entries_per_aggregator*sizeof(int));
+	if (NULL == file_offsets_for_agg || NULL == memory_displacements ||
+	    NULL == sorted_file_offsets) {
+	    opal_output (1, "OUT OF MEMORY\n");
+	    ret = OMPI_ERR_OUT_OF_RESOURCE;
+	    goto exit;
+	}
 
-        if ( NULL != blocklen_per_process){
-            for(l=0;l<fh->f_procs_per_group;l++){
-                if (NULL != blocklen_per_process[l]){
-                    free(blocklen_per_process[l]);
-                    blocklen_per_process[l] = NULL;
-                }
-            }
+	ret = mca_fcoll_vulcan_calc_file_offsets(data, file_offsets_for_agg, sorted_file_offsets,
+						 memory_displacements, entries_per_aggregator,
+						 rank, index);
+	if (OMPI_SUCCESS != ret) {
+	  goto exit;
+	}
 
-            free(blocklen_per_process);
-            blocklen_per_process = NULL;
+	/**********************************************************
+	 *** 7f. Create the io array
+	 *********************************************************/
+        fh->f_io_array = (mca_common_ompio_io_array_t *) malloc (entries_per_aggregator
+						* sizeof (mca_common_ompio_io_array_t));
+        if (NULL == fh->f_io_array) {
+            opal_output(1, "OUT OF MEMORY\n");
+            ret = OMPI_ERR_OUT_OF_RESOURCE;
+            goto exit;
         }
 
-        if (NULL != displs_per_process){
-            for (l=0; i<fh->f_procs_per_group; l++){
-                if (NULL != displs_per_process[l]){
-                    free(displs_per_process[l]);
-                    displs_per_process[l] = NULL;
+        mca_fcoll_vulcan_calc_io_array(fh->f_io_array, &fh->f_num_of_io_entries, entries_per_aggregator,
+                                       (char*)data->global_buf, file_offsets_for_agg, sorted_file_offsets,
+                                       memory_displacements, rank);
+    }
+
+    if (rank == aggregator && fh->f_num_of_io_entries) {
+        mca_common_ompio_request_alloc (&ompio_req, MCA_OMPIO_REQUEST_READ);
+
+        if (1 == read_syncType) {
+            if (is_accelerator_buffer) {
+                ret = mca_common_ompio_file_iread_pregen(fh, (ompi_request_t *) ompio_req);
+                if (0 > ret) {
+                    opal_output (1, "vulcan_read_all: mca_common_ompio_iread_pregen failed\n");
+                    ompio_req->req_ompi.req_status.MPI_ERROR = ret;
+                    ompio_req->req_ompi.req_status._ucount = 0;
+                }
+            } else {
+                ret = fh->f_fbtl->fbtl_ipreadv(fh, (ompi_request_t *) ompio_req);
+                if (0 > ret) {
+                    opal_output (1, "vulcan_read_all: fbtl_ipreadv failed\n");
+                    ompio_req->req_ompi.req_status.MPI_ERROR = ret;
+                    ompio_req->req_ompi.req_status._ucount = 0;
                 }
             }
-            free(displs_per_process);
-            displs_per_process = NULL;
         }
-        if ( NULL != send_req ) {
-            free ( send_req );
-            send_req = NULL;
+        else {
+            ret_temp = fh->f_fbtl->fbtl_preadv(fh);
+            if (0 > ret_temp) {
+                opal_output (1, "vulcan_read_all: fbtl_preadv failed\n");
+                ret = ret_temp;
+                ret_temp = 0;
+            }
+
+            ompio_req->req_ompi.req_status.MPI_ERROR = ret;
+            ompio_req->req_ompi.req_status._ucount = ret_temp;
+            ompi_request_complete (&ompio_req->req_ompi, false);
         }
+
+        free(fh->f_io_array);
     }
+
+#if DEBUG_ON
+    printf("************Cycle: %d,  Aggregator: %d ***************\n",
+	   index, rank);
+    for (i = 0; i < data->procs_per_group; i++) {
+	for (j = 0; j < data->disp_index[i]; j++) {
+	    if (data->blocklen_per_process[i][j] > 0) {
+		printf("%d sends blocklen[%d]: %d, disp[%d]: %ld to %d\n",
+		       data->procs_in_group[i],j,
+		       data->blocklen_per_process[i][j],j,
+		       data->displs_per_process[i][j], rank);
+	    }
+	}
+    }
+#endif
+
+exit:
+    free(sorted_file_offsets);
+    free(file_offsets_for_agg);
+    free(memory_displacements);
+    free(blocklength_proc);
+    free(displs_proc);
+
+    fh->f_io_array = NULL;
+    fh->f_num_of_io_entries = 0;
+
+    *request = (ompi_request_t *) ompio_req;
     return ret;
 }
 
-
-static int read_heap_sort (mca_io_ompio_local_io_array *io_array,
-                           int num_entries,
-                           int *sorted)
+static int shuffle_init (int index, int cycles, int aggregator, int rank, mca_io_ompio_aggregator_data *data,
+                         ompi_request_t **reqs)
 {
-    int i = 0;
-    int j = 0;
-    int left = 0;
-    int right = 0;
-    int largest = 0;
-    int heap_size = num_entries - 1;
-    int temp = 0;
-    unsigned char done = 0;
-    int* temp_arr = NULL;
-
-    temp_arr = (int*)malloc(num_entries*sizeof(int));
-    if (NULL == temp_arr) {
-        opal_output (1, "OUT OF MEMORY\n");
-        return OMPI_ERR_OUT_OF_RESOURCE;
-    }
-    temp_arr[0] = 0;
-    for (i = 1; i < num_entries; ++i) {
-        temp_arr[i] = i;
-    }
-    /* num_entries can be a large no. so NO RECURSION */
-    for (i = num_entries/2-1 ; i>=0 ; i--) {
-        done = 0;
-        j = i;
-        largest = j;
-
-        while (!done) {
-            left = j*2+1;
-            right = j*2+2;
-            if ((left <= heap_size) &&
-                (io_array[temp_arr[left]].offset > io_array[temp_arr[j]].offset)) {
-                largest = left;
-            }
-            else {
-                largest = j;
-            }
-            if ((right <= heap_size) &&
-                (io_array[temp_arr[right]].offset >
-                 io_array[temp_arr[largest]].offset)) {
-                largest = right;
-            }
-            if (largest != j) {
-                temp = temp_arr[largest];
-                temp_arr[largest] = temp_arr[j];
-                temp_arr[j] = temp;
-                j = largest;
-            }
-            else {
-                done = 1;
-            }
+    int i, ret = OMPI_SUCCESS;
+    int* blocklength_proc=NULL;
+    ptrdiff_t* displs_proc=NULL;
+
+    /*************************************************************************
+     *** 7e. Perform the actual communication
+     *************************************************************************/
+    if (aggregator == rank ) {
+	for (i = 0; i < data->procs_per_group; i++) {
+	    size_t datatype_size;
+            reqs[i] = MPI_REQUEST_NULL;
+	    if (0 < data->disp_index[i]) {
+		ompi_datatype_create_hindexed (data->disp_index[i],
+					       data->blocklen_per_process[i],
+					       data->displs_per_process[i],
+					       MPI_BYTE,
+					       &data->recvtype[i]);
+		ompi_datatype_commit (&data->recvtype[i]);
+		opal_datatype_type_size (&data->recvtype[i]->super, &datatype_size);
+
+		if (datatype_size){
+		    ret = MCA_PML_CALL(isend(data->global_buf,
+					     1, data->recvtype[i],
+					     data->procs_in_group[i],
+					     FCOLL_VULCAN_SHUFFLE_TAG+index,
+					     MCA_PML_BASE_SEND_STANDARD,
+					     data->comm, &reqs[i]));
+		    if (OMPI_SUCCESS != ret){
+			goto exit;
+		    }
+		}
+	    }
+	}
+	// }  /* end if (entries_per_aggr > 0 ) */
+    }/* end if (aggregator == rank ) */
+
+    reqs[data->procs_per_group] = MPI_REQUEST_NULL;
+    if (data->bytes_sent) {
+        size_t remaining      = data->bytes_sent;
+        int block_index       = -1;
+        int blocklength_size  = INIT_LEN;
+
+        ptrdiff_t recv_mem_address  = 0;
+        ompi_datatype_t *newType    = MPI_DATATYPE_NULL;
+        blocklength_proc            = (int *)       calloc (blocklength_size, sizeof (int));
+        displs_proc                 = (ptrdiff_t *) calloc (blocklength_size, sizeof (ptrdiff_t));
+
+        if (NULL == blocklength_proc || NULL == displs_proc ) {
+            opal_output (1, "OUT OF MEMORY\n");
+            ret = OMPI_ERR_OUT_OF_RESOURCE;
+            goto exit;
         }
-    }
 
-    for (i = num_entries-1; i >=1; --i) {
-        temp = temp_arr[0];
-        temp_arr[0] = temp_arr[i];
-        temp_arr[i] = temp;
-        heap_size--;
-        done = 0;
-        j = 0;
-        largest = j;
-
-        while (!done) {
-            left =  j*2+1;
-            right = j*2+2;
-
-            if ((left <= heap_size) &&
-                (io_array[temp_arr[left]].offset >
-                 io_array[temp_arr[j]].offset)) {
-                largest = left;
+        while (remaining) {
+            block_index++;
+
+            if(0 == block_index) {
+                recv_mem_address = (ptrdiff_t) (data->decoded_iov[data->iov_index].iov_base) +
+                                                data->current_position;
             }
             else {
-                largest = j;
+                // Reallocate more memory if blocklength_size is not enough
+                if(0 == block_index % INIT_LEN) {
+                    blocklength_size += INIT_LEN;
+                    blocklength_proc = (int *)       realloc(blocklength_proc, blocklength_size * sizeof(int));
+                    displs_proc      = (ptrdiff_t *) realloc(displs_proc, blocklength_size * sizeof(ptrdiff_t));
+                }
+                displs_proc[block_index] = (ptrdiff_t) (data->decoded_iov[data->iov_index].iov_base) +
+                                                        data->current_position - recv_mem_address;
             }
-            if ((right <= heap_size) &&
-                (io_array[temp_arr[right]].offset >
-                 io_array[temp_arr[largest]].offset)) {
-                largest = right;
+
+            if (remaining >=
+                (data->decoded_iov[data->iov_index].iov_len - data->current_position)) {
+
+                blocklength_proc[block_index] = data->decoded_iov[data->iov_index].iov_len -
+                                                data->current_position;
+                remaining = remaining - (data->decoded_iov[data->iov_index].iov_len -
+					 data->current_position);
+                data->iov_index = data->iov_index + 1;
+                data->current_position = 0;
+            } else {
+                blocklength_proc[block_index] = remaining;
+                data->current_position += remaining;
+                remaining = 0;
             }
-            if (largest != j) {
-                temp = temp_arr[largest];
-                temp_arr[largest] = temp_arr[j];
-                temp_arr[j] = temp;
-                j = largest;
+        }
+
+        data->total_bytes_written += data->bytes_sent;
+
+        if (0 <= block_index) {
+            ompi_datatype_create_hindexed (block_index+1,
+                                           blocklength_proc,
+                                           displs_proc,
+                                           MPI_BYTE,
+                                           &newType);
+            ompi_datatype_commit (&newType);
+
+            ret = MCA_PML_CALL(irecv((char *)recv_mem_address,
+                                     1,
+                                     newType,
+                                     aggregator,
+                                     FCOLL_VULCAN_SHUFFLE_TAG+index,
+                                     data->comm,
+                                     &reqs[data->procs_per_group]));
+            if (MPI_DATATYPE_NULL != newType) {
+                ompi_datatype_destroy(&newType);
             }
-            else {
-                done = 1;
+            if (OMPI_SUCCESS != ret){
+                goto exit;
             }
         }
-        sorted[i] = temp_arr[i];
-    }
-    sorted[0] = temp_arr[0];
-
-    if (NULL != temp_arr) {
-        free(temp_arr);
-        temp_arr = NULL;
     }
-    return OMPI_SUCCESS;
+exit:
+    return ret;
 }
-
-
-
diff --git a/ompi/mca/fcoll/vulcan/fcoll_vulcan_file_write_all.c b/ompi/mca/fcoll/vulcan/fcoll_vulcan_file_write_all.c
index 9de463fc879..b6e9be6d2ca 100644
--- a/ompi/mca/fcoll/vulcan/fcoll_vulcan_file_write_all.c
+++ b/ompi/mca/fcoll/vulcan/fcoll_vulcan_file_write_all.c
@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -12,6 +13,10 @@
  * Copyright (c) 2008-2021 University of Houston. All rights reserved.
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2024      Advanced Micro Devices, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -21,104 +26,38 @@
 
 #include "ompi_config.h"
 #include "fcoll_vulcan.h"
+#include "fcoll_vulcan_internal.h"
 
 #include "mpi.h"
 #include "ompi/constants.h"
 #include "ompi/mca/fcoll/fcoll.h"
 #include "ompi/mca/fcoll/base/fcoll_base_coll_array.h"
 #include "ompi/mca/common/ompio/common_ompio.h"
+#include "ompi/mca/common/ompio/common_ompio_buffer.h"
 #include "ompi/mca/io/io.h"
 #include "ompi/mca/common/ompio/common_ompio_request.h"
 #include "math.h"
 #include "ompi/mca/pml/pml.h"
+#include "opal/mca/accelerator/accelerator.h"
 #include <unistd.h>
 
 #define DEBUG_ON 0
 #define NOT_AGGR_INDEX -1
 
-/*Used for loading file-offsets per aggregator*/
-typedef struct mca_io_ompio_local_io_array{
-    OMPI_MPI_OFFSET_TYPE offset;
-    MPI_Aint             length;
-    int                  process_id;
-}mca_io_ompio_local_io_array;
-
-typedef struct mca_io_ompio_aggregator_data {
-    int *disp_index, *sorted, *fview_count, n;
-    int *max_disp_index;
-    int **blocklen_per_process;
-    MPI_Aint **displs_per_process, total_bytes, bytes_per_cycle, total_bytes_written;
-    MPI_Comm comm;
-    char *buf, *global_buf, *prev_global_buf;
-    ompi_datatype_t **recvtype, **prev_recvtype;
-    struct iovec *global_iov_array;
-    int current_index, current_position;
-    int bytes_to_write_in_cycle, bytes_remaining, procs_per_group;    
-    int *procs_in_group, iov_index;
-    int bytes_sent, prev_bytes_sent;
-    struct iovec *decoded_iov;
-    int bytes_to_write, prev_bytes_to_write;
-    mca_common_ompio_io_array_t *io_array, *prev_io_array;
-    int num_io_entries, prev_num_io_entries;
-} mca_io_ompio_aggregator_data;
-
-
-#define SWAP_REQUESTS(_r1,_r2) { \
-    ompi_request_t **_t=_r1;     \
-    _r1=_r2;                     \
-    _r2=_t;}
-
-#define SWAP_AGGR_POINTERS(_aggr,_num) {                        \
-    int _i;                                                     \
-    char *_t;                                                   \
-    for (_i=0; _i<_num; _i++ ) {                                \
-        _aggr[_i]->prev_io_array=_aggr[_i]->io_array;             \
-        _aggr[_i]->prev_num_io_entries=_aggr[_i]->num_io_entries; \
-        _aggr[_i]->prev_bytes_sent=_aggr[_i]->bytes_sent;         \
-        _aggr[_i]->prev_bytes_to_write=_aggr[_i]->bytes_to_write; \
-        _t=_aggr[_i]->prev_global_buf;                            \
-        _aggr[_i]->prev_global_buf=_aggr[_i]->global_buf;         \
-        _aggr[_i]->global_buf=_t;                                 \
-        _t=(char *)_aggr[_i]->recvtype;                           \
-        _aggr[_i]->recvtype=_aggr[_i]->prev_recvtype;             \
-        _aggr[_i]->prev_recvtype=(ompi_datatype_t **)_t;          }                                                             \
-}
-
 
+static int shuffle_init (int index, int num_cycles, int aggregator, int rank,
+                         mca_io_ompio_aggregator_data *data, ompi_request_t **reqs);
 
-static int shuffle_init ( int index, int cycles, int aggregator, int rank, 
-                          mca_io_ompio_aggregator_data *data, 
-                          ompi_request_t **reqs );
 static int write_init (ompio_file_t *fh, int aggregator, mca_io_ompio_aggregator_data *aggr_data,
-                        int write_chunksize, int write_synchType, ompi_request_t **request);
-int mca_fcoll_vulcan_break_file_view ( struct iovec *decoded_iov, int iov_count, 
-                                        struct iovec *local_iov_array, int local_count, 
-                                        struct iovec ***broken_decoded_iovs, int **broken_iov_counts,
-                                        struct iovec ***broken_iov_arrays, int **broken_counts, 
-                                        MPI_Aint **broken_total_lengths,
-                                        int stripe_count, size_t stripe_size); 
-
-
-int mca_fcoll_vulcan_get_configuration (ompio_file_t *fh, int num_io_procs, 
-                                        int num_groups, size_t max_data);
-
+                       int write_syncType, ompi_request_t **request,
+                       bool is_accelerator_buffer);
 
 static int local_heap_sort (mca_io_ompio_local_io_array *io_array,
-			    int num_entries,
-			    int *sorted);
-
-int mca_fcoll_vulcan_split_iov_array ( ompio_file_t *fh, mca_common_ompio_io_array_t *work_array,
-                                             int num_entries, int *last_array_pos, int *last_pos_in_field,
-                                             int chunk_size );
+			    int num_entries, int *sorted);
 
-
-static int mca_fcoll_vulcan_minmax ( ompio_file_t *fh, struct iovec *iov, int iov_count,  int num_aggregators, 
-                                     long *new_stripe_size);
-
-
-int mca_fcoll_vulcan_file_write_all (ompio_file_t *fh,
+int mca_fcoll_vulcan_file_write_all (struct ompio_file_t *fh,
                                       const void *buf,
-                                      int count,
+                                      size_t count,
                                       struct ompi_datatype_t *datatype,
                                       ompi_status_public_t *status)
 {
@@ -134,10 +73,9 @@ int mca_fcoll_vulcan_file_write_all (ompio_file_t *fh,
     ompi_request_t *req_iwrite = MPI_REQUEST_NULL;
     mca_io_ompio_aggregator_data **aggr_data=NULL;
     
-    int *displs = NULL;
+    ptrdiff_t *displs = NULL;
     int vulcan_num_io_procs;
     size_t max_data = 0;
-    MPI_Aint *total_bytes_per_process = NULL;
     
     struct iovec **broken_iov_arrays=NULL;
     struct iovec **broken_decoded_iovs=NULL;
@@ -147,8 +85,13 @@ int mca_fcoll_vulcan_file_write_all (ompio_file_t *fh,
 
     int aggr_index = NOT_AGGR_INDEX;
     int write_synch_type = 2;
-    int write_chunksize, *result_counts=NULL;
-    
+    int *result_counts=NULL;
+
+    ompi_count_array_t fview_count_desc;
+    ompi_disp_array_t displs_desc;
+    int is_gpu, is_managed;
+    bool use_accelerator_buffer = false;
+
 #if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
     double write_time = 0.0, start_write_time = 0.0, end_write_time = 0.0;
     double comm_time = 0.0, start_comm_time = 0.0, end_comm_time = 0.0;
@@ -158,7 +101,7 @@ int mca_fcoll_vulcan_file_write_all (ompio_file_t *fh,
     
     
     /**************************************************************************
-     ** 1.  In case the data is not contigous in memory, decode it into an iovec
+     ** 1.  In case the data is not contiguous in memory, decode it into an iovec
      **************************************************************************/
     vulcan_num_io_procs = fh->f_get_mca_parameter_value ( "num_aggregators", strlen ("num_aggregators"));
     if ( OMPI_ERR_MAX == vulcan_num_io_procs ) {
@@ -168,15 +111,19 @@ int mca_fcoll_vulcan_file_write_all (ompio_file_t *fh,
     bytes_per_cycle = fh->f_bytes_per_agg;
 
     if( (1 == mca_fcoll_vulcan_async_io) && (NULL == fh->f_fbtl->fbtl_ipwritev) ) {
-        opal_output (1, "vulcan_write_all: fbtl Does NOT support ipwritev() (asynchrounous write) \n");
+        opal_output (1, "vulcan_write_all: fbtl Does NOT support ipwritev() (asynchronous write) \n");
         ret = MPI_ERR_UNSUPPORTED_OPERATION;
         goto exit;
     }
 
+    mca_common_ompio_check_gpu_buf (fh, buf, &is_gpu, &is_managed);
+    if (is_gpu && !is_managed &&
+        fh->f_get_mca_parameter_value ("use_accelerator_buffers", strlen("use_accelerator_buffers"))) {
+        use_accelerator_buffer = true;
+    }
     /* since we want to overlap 2 iterations, define the bytes_per_cycle to be half of what
        the user requested */
     bytes_per_cycle =bytes_per_cycle/2;
-    write_chunksize = bytes_per_cycle;
     
     ret =   mca_common_ompio_decode_datatype ((struct ompio_file_t *) fh,
                                               datatype,
@@ -191,14 +138,15 @@ int mca_fcoll_vulcan_file_write_all (ompio_file_t *fh,
     }
     
     if ( MPI_STATUS_IGNORE != status ) {
-	status->_ucount = max_data;
+        status->_ucount = max_data;
     }
     
-    
-    ret = mca_fcoll_vulcan_get_configuration (fh, vulcan_num_io_procs, mca_fcoll_vulcan_num_groups, max_data);
+    ret = mca_fcoll_vulcan_get_configuration (fh, vulcan_num_io_procs, max_data);
     if (OMPI_SUCCESS != ret){
-	goto exit;
+        goto exit;
     }
+    opal_output_verbose(10, ompi_fcoll_base_framework.framework_output,
+        "Using %d aggregators for the write_all operation \n", fh->f_num_aggrs);
 
     aggr_data = (mca_io_ompio_aggregator_data **) malloc ( fh->f_num_aggrs * 
                                                            sizeof(mca_io_ompio_aggregator_data*));
@@ -211,7 +159,6 @@ int mca_fcoll_vulcan_file_write_all (ompio_file_t *fh,
         aggr_data[i]->procs_per_group = fh->f_procs_per_group;
         aggr_data[i]->procs_in_group  = fh->f_procs_in_group;
         aggr_data[i]->comm = fh->f_comm;
-        aggr_data[i]->buf  = (char *)buf;             // should not be used in the new version.
         // Identify if the process is an aggregator.
         // If so, aggr_index would be its index in "aggr_data" and "aggregators" arrays.
         if(fh->f_aggr_list[i] == fh->f_rank) {
@@ -224,11 +171,11 @@ int mca_fcoll_vulcan_file_write_all (ompio_file_t *fh,
      ***    this write operation
      ********************************************************************/
     ret = fh->f_generate_current_file_view( (struct ompio_file_t *) fh,
-					    max_data,
-					    &local_iov_array,
-					    &local_count);
+                                            max_data,
+                                            &local_iov_array,
+                                            &local_count);
     if (ret != OMPI_SUCCESS){
-	goto exit;
+        goto exit;
     }
     
     /*************************************************************************
@@ -254,52 +201,15 @@ int mca_fcoll_vulcan_file_write_all (ompio_file_t *fh,
 #if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
     start_comm_time = MPI_Wtime();
 #endif
-    if ( 1 == mca_fcoll_vulcan_num_groups ) {
-        ret = fh->f_comm->c_coll->coll_allreduce (MPI_IN_PLACE,
-                                                  broken_total_lengths,
-						  fh->f_num_aggrs,
-						  MPI_LONG,
-                                                  MPI_SUM,
-						  fh->f_comm,
-						  fh->f_comm->c_coll->coll_allreduce_module);
-        if( OMPI_SUCCESS != ret){
-            goto exit;
-        }
-
-    }
-    else {
-        total_bytes_per_process = (MPI_Aint*)malloc
-            (fh->f_num_aggrs * fh->f_procs_per_group*sizeof(MPI_Aint));
-        if (NULL == total_bytes_per_process) {
-            opal_output (1, "OUT OF MEMORY\n");
-            ret = OMPI_ERR_OUT_OF_RESOURCE;
-            goto exit;
-        }
-    
-        ret = ompi_fcoll_base_coll_allgather_array (broken_total_lengths,
-						    fh->f_num_aggrs,
-						    MPI_LONG,
-						    total_bytes_per_process,
-						    fh->f_num_aggrs,
-						    MPI_LONG,
-						    0,
-						    fh->f_procs_in_group,
-						    fh->f_procs_per_group,
-						    fh->f_comm);
-        if( OMPI_SUCCESS != ret){
-            goto exit;
-        }
-
-        for ( i=0; i<fh->f_num_aggrs; i++ ) {
-            broken_total_lengths[i] = 0;
-            for (j=0 ; j<fh->f_procs_per_group ; j++) {
-                broken_total_lengths[i] += total_bytes_per_process[j*fh->f_num_aggrs + i];
-            }
-        }
-        if (NULL != total_bytes_per_process) {
-            free (total_bytes_per_process);
-            total_bytes_per_process = NULL;
-        }    
+    ret = fh->f_comm->c_coll->coll_allreduce (MPI_IN_PLACE,
+                                              broken_total_lengths,
+                                              fh->f_num_aggrs,
+                                              MPI_LONG,
+                                              MPI_SUM,
+                                              fh->f_comm,
+                                              fh->f_comm->c_coll->coll_allreduce_module);
+    if( OMPI_SUCCESS != ret){
+      goto exit;
     }
     
 #if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
@@ -326,28 +236,14 @@ int mca_fcoll_vulcan_file_write_all (ompio_file_t *fh,
 #if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
     start_comm_time = MPI_Wtime();
 #endif
-    if ( 1 == mca_fcoll_vulcan_num_groups ) {
-        ret = fh->f_comm->c_coll->coll_allgather(broken_counts,
-						 fh->f_num_aggrs,
-						 MPI_INT,
-						 result_counts,
-						 fh->f_num_aggrs,
-						 MPI_INT,
-						 fh->f_comm,
-						 fh->f_comm->c_coll->coll_allgather_module);            
-    }
-    else {
-        ret = ompi_fcoll_base_coll_allgather_array (broken_counts,
-						    fh->f_num_aggrs,
-						    MPI_INT,
-						    result_counts,
-						    fh->f_num_aggrs,
-						    MPI_INT,
-						    0,
-						    fh->f_procs_in_group,
-						    fh->f_procs_per_group,
-						    fh->f_comm);
-    }
+    ret = fh->f_comm->c_coll->coll_allgather(broken_counts,
+                                                 fh->f_num_aggrs,
+                                                 MPI_INT,
+                                                 result_counts,
+                                                 fh->f_num_aggrs,
+                                                 MPI_INT,
+                                                 fh->f_comm,
+                                                 fh->f_comm->c_coll->coll_allgather_module);
     if( OMPI_SUCCESS != ret){
         goto exit;
     }
@@ -362,7 +258,7 @@ int mca_fcoll_vulcan_file_write_all (ompio_file_t *fh,
     for ( i=0; i< fh->f_num_aggrs; i++ ) {
         aggr_data[i]->total_bytes = broken_total_lengths[i];
         aggr_data[i]->decoded_iov = broken_decoded_iovs[i];
-        aggr_data[i]->fview_count = (int *) malloc (fh->f_procs_per_group * sizeof (int));
+        aggr_data[i]->fview_count = (size_t *)malloc (fh->f_procs_per_group * sizeof (size_t));
         if (NULL == aggr_data[i]->fview_count) {
             opal_output (1, "OUT OF MEMORY\n");
             ret = OMPI_ERR_OUT_OF_RESOURCE;
@@ -371,7 +267,7 @@ int mca_fcoll_vulcan_file_write_all (ompio_file_t *fh,
         for ( j=0; j <fh->f_procs_per_group; j++ ) {
             aggr_data[i]->fview_count[j] = result_counts[fh->f_num_aggrs*j+i];
         }
-        displs = (int*) malloc (fh->f_procs_per_group * sizeof (int));
+        displs = (ptrdiff_t *)malloc (fh->f_procs_per_group * sizeof (ptrdiff_t));
         if (NULL == displs) {
             opal_output (1, "OUT OF MEMORY\n");
             ret = OMPI_ERR_OUT_OF_RESOURCE;
@@ -379,7 +275,7 @@ int mca_fcoll_vulcan_file_write_all (ompio_file_t *fh,
         }
         
         displs[0] = 0;
-        total_fview_count = aggr_data[i]->fview_count[0];
+        total_fview_count = (uint32_t) aggr_data[i]->fview_count[0];
         for (j=1 ; j<fh->f_procs_per_group ; j++) {
             total_fview_count += aggr_data[i]->fview_count[j];
             displs[j] = displs[j-1] + aggr_data[i]->fview_count[j-1];
@@ -412,30 +308,17 @@ int mca_fcoll_vulcan_file_write_all (ompio_file_t *fh,
 #if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
         start_comm_time = MPI_Wtime();
 #endif
-        if ( 1 == mca_fcoll_vulcan_num_groups ) {
-            ret = fh->f_comm->c_coll->coll_allgatherv (broken_iov_arrays[i],
-                                                      broken_counts[i],
-                                                      fh->f_iov_type,
-                                                      aggr_data[i]->global_iov_array,
-                                                      aggr_data[i]->fview_count,
-                                                      displs,
-                                                      fh->f_iov_type,
-                                                      fh->f_comm,
-                                                      fh->f_comm->c_coll->coll_allgatherv_module );
-        }
-        else {
-            ret = ompi_fcoll_base_coll_allgatherv_array (broken_iov_arrays[i],
-							 broken_counts[i],
-							 fh->f_iov_type,
-							 aggr_data[i]->global_iov_array,
-							 aggr_data[i]->fview_count,
-							 displs,
-							 fh->f_iov_type,
-							 fh->f_aggr_list[i],
-							 fh->f_procs_in_group,
-							 fh->f_procs_per_group,
-							 fh->f_comm);
-        }
+        OMPI_COUNT_ARRAY_INIT(&fview_count_desc, aggr_data[i]->fview_count);
+        OMPI_DISP_ARRAY_INIT(&displs_desc, displs);
+        ret = fh->f_comm->c_coll->coll_allgatherv (broken_iov_arrays[i],
+                                                   broken_counts[i],
+                                                   fh->f_iov_type,
+                                                   aggr_data[i]->global_iov_array,
+                                                   fview_count_desc,
+                                                   displs_desc,
+                                                   fh->f_iov_type,
+                                                   fh->f_comm,
+                                                   fh->f_comm->c_coll->coll_allgatherv_module );
         if (OMPI_SUCCESS != ret){
             goto exit;
         }
@@ -520,13 +403,31 @@ int mca_fcoll_vulcan_file_write_all (ompio_file_t *fh,
                 goto exit;
             }
         
-            
-            aggr_data[i]->global_buf       = (char *) malloc (bytes_per_cycle);
-            aggr_data[i]->prev_global_buf  = (char *) malloc (bytes_per_cycle);
-            if (NULL == aggr_data[i]->global_buf || NULL == aggr_data[i]->prev_global_buf){
-                opal_output(1, "OUT OF MEMORY");
-                ret = OMPI_ERR_OUT_OF_RESOURCE;
-                goto exit;
+            if (use_accelerator_buffer) {
+                opal_output_verbose(10, ompi_fcoll_base_framework.framework_output,
+                                    "Allocating GPU device buffer for aggregation\n");
+                ret = opal_accelerator.mem_alloc(MCA_ACCELERATOR_NO_DEVICE_ID, (void**)&aggr_data[i]->global_buf,
+                                                 bytes_per_cycle);
+                if (OPAL_SUCCESS != ret) {
+                    opal_output(1, "Could not allocate accelerator memory");
+                    ret = OMPI_ERR_OUT_OF_RESOURCE;
+                    goto exit;
+                }
+                ret = opal_accelerator.mem_alloc(MCA_ACCELERATOR_NO_DEVICE_ID, (void**)&aggr_data[i]->prev_global_buf,
+                                                 bytes_per_cycle);
+                if (OPAL_SUCCESS != ret) {
+                    opal_output(1, "Could not allocate accelerator memory");
+                    ret = OMPI_ERR_OUT_OF_RESOURCE;
+                    goto exit;
+                }
+            } else {
+                aggr_data[i]->global_buf       = (char *) malloc (bytes_per_cycle);
+                aggr_data[i]->prev_global_buf  = (char *) malloc (bytes_per_cycle);
+                if (NULL == aggr_data[i]->global_buf || NULL == aggr_data[i]->prev_global_buf){
+                    opal_output(1, "OUT OF MEMORY");
+                    ret = OMPI_ERR_OUT_OF_RESOURCE;
+                    goto exit;
+                }
             }
         
             aggr_data[i]->recvtype = (ompi_datatype_t **) malloc (fh->f_procs_per_group  * 
@@ -547,10 +448,9 @@ int mca_fcoll_vulcan_file_write_all (ompio_file_t *fh,
 #if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
         start_exch = MPI_Wtime();
 #endif
-    }    
+    }
 
     reqs = (ompi_request_t **)malloc ((fh->f_procs_per_group + 1 )*fh->f_num_aggrs *sizeof(ompi_request_t *));
-
     if ( NULL == reqs ) {
         opal_output (1, "OUT OF MEMORY\n");
         ret = OMPI_ERR_OUT_OF_RESOURCE;
@@ -596,7 +496,7 @@ int mca_fcoll_vulcan_file_write_all (ompio_file_t *fh,
             start_write_time = MPI_Wtime();
 #endif
             ret = write_init (fh, fh->f_aggr_list[aggr_index], aggr_data[aggr_index],
-                              write_chunksize, write_synch_type, &req_iwrite);
+                              write_synch_type, &req_iwrite, use_accelerator_buffer);
             if (OMPI_SUCCESS != ret){
                 goto exit;
             }
@@ -636,7 +536,7 @@ int mca_fcoll_vulcan_file_write_all (ompio_file_t *fh,
             start_write_time = MPI_Wtime();
 #endif
             ret = write_init (fh, fh->f_aggr_list[aggr_index], aggr_data[aggr_index],
-                              write_chunksize, write_synch_type, &req_iwrite);
+                              write_synch_type, &req_iwrite, use_accelerator_buffer);
             if (OMPI_SUCCESS != ret){
                 goto exit;
             }
@@ -663,7 +563,7 @@ int mca_fcoll_vulcan_file_write_all (ompio_file_t *fh,
     nentry.aggregator = 0;
     for ( i=0; i<fh->f_num_aggrs; i++ ) {
         if (fh->f_aggr_list[i] == fh->f_rank)
-	nentry.aggregator = 1;
+        nentry.aggregator = 1;
     }
     nentry.nprocs_for_coll = fh->f_num_aggrs;
     if (!mca_common_ompio_full_print_queue(fh->f_coll_write_time)){
@@ -671,15 +571,13 @@ int mca_fcoll_vulcan_file_write_all (ompio_file_t *fh,
                                                nentry);
     }
 #endif
-    
-    
+
 exit :
-    
+
     if ( NULL != aggr_data ) {
-        
-        for ( i=0; i< fh->f_num_aggrs; i++ ) {            
+        for ( i=0; i< fh->f_num_aggrs; i++ ) {
             if (fh->f_aggr_list[i] == fh->f_rank) {
-                if (NULL != aggr_data[i]->recvtype){
+                if (NULL != aggr_data[i]->recvtype) {
                     for (j =0; j< aggr_data[i]->procs_per_group; j++) {
                         if ( MPI_DATATYPE_NULL != aggr_data[i]->recvtype[j] ) {
                             ompi_datatype_destroy(&aggr_data[i]->recvtype[j]);
@@ -687,21 +585,25 @@ exit :
                         if ( MPI_DATATYPE_NULL != aggr_data[i]->prev_recvtype[j] ) {
                             ompi_datatype_destroy(&aggr_data[i]->prev_recvtype[j]);
                         }
-			
                     }
                     free(aggr_data[i]->recvtype);
                     free(aggr_data[i]->prev_recvtype);
                 }
-                
+
                 free (aggr_data[i]->disp_index);
                 free (aggr_data[i]->max_disp_index);
-                free (aggr_data[i]->global_buf);
-                free (aggr_data[i]->prev_global_buf);
+                if (use_accelerator_buffer) {
+                    opal_accelerator.mem_release(MCA_ACCELERATOR_NO_DEVICE_ID, aggr_data[i]->global_buf);
+                    opal_accelerator.mem_release(MCA_ACCELERATOR_NO_DEVICE_ID, aggr_data[i]->prev_global_buf);
+                } else {
+                    free (aggr_data[i]->global_buf);
+                    free (aggr_data[i]->prev_global_buf);
+                }
                 for(l=0;l<aggr_data[i]->procs_per_group;l++){
                     free (aggr_data[i]->blocklen_per_process[l]);
                     free (aggr_data[i]->displs_per_process[l]);
                 }
-                
+
                 free (aggr_data[i]->blocklen_per_process);
                 free (aggr_data[i]->displs_per_process);
             }
@@ -709,7 +611,6 @@ exit :
             free (aggr_data[i]->global_iov_array);
             free (aggr_data[i]->fview_count);
             free (aggr_data[i]->decoded_iov);
-            
             free (aggr_data[i]);
         }
         free (aggr_data);
@@ -727,45 +628,60 @@ exit :
     }
     free(broken_iov_arrays);
     free(fh->f_procs_in_group);
+    free(fh->f_aggr_list);
     fh->f_procs_in_group=NULL;
     fh->f_procs_per_group=0;
+    fh->f_aggr_list=NULL;
     free(result_counts);
     free(reqs);
-     
+
     return OMPI_SUCCESS;
 }
 
 static int write_init (ompio_file_t *fh,
                        int aggregator,
                        mca_io_ompio_aggregator_data *aggr_data,
-                       int write_chunksize,
-                       int write_synchType,
-                       ompi_request_t **request )
+                       int write_syncType,
+                       ompi_request_t **request,
+                       bool is_accelerator_buffer)
 {
     int ret = OMPI_SUCCESS;
     ssize_t ret_temp = 0;
-    int last_array_pos = 0;
-    int last_pos = 0;
+    int i;
     mca_ompio_request_t *ompio_req = NULL;
 
     mca_common_ompio_request_alloc ( &ompio_req, MCA_OMPIO_REQUEST_WRITE );
 
     if (aggr_data->prev_num_io_entries) {
-        /*  In this case, aggr_data->prev_num_io_entries is always == 1.
-            Therefore we can write the data of size aggr_data->prev_bytes_to_write in one iteration.
-            In fact, aggr_data->prev_bytes_to_write <= write_chunksize.
-        */
-        mca_fcoll_vulcan_split_iov_array (fh, aggr_data->prev_io_array,
-                                          aggr_data->prev_num_io_entries,
-                                          &last_array_pos, &last_pos,
-                                          write_chunksize);
-
-        if (1 == write_synchType) {
-            ret = fh->f_fbtl->fbtl_ipwritev(fh, (ompi_request_t *) ompio_req);
-            if(0 > ret) {
-                opal_output (1, "vulcan_write_all: fbtl_ipwritev failed\n");
-                ompio_req->req_ompi.req_status.MPI_ERROR = ret;
-                ompio_req->req_ompi.req_status._ucount = 0;
+        fh->f_num_of_io_entries = aggr_data->prev_num_io_entries;
+        fh->f_io_array = (mca_common_ompio_io_array_t *) malloc (fh->f_num_of_io_entries *
+                                                                 sizeof(mca_common_ompio_io_array_t));
+        if ( NULL == fh->f_io_array ){
+            opal_output (1,"Could not allocate memory\n");
+            return -1;
+        }
+
+        for (i = 0; i < fh->f_num_of_io_entries; i++) {
+            fh->f_io_array[i].memory_address = aggr_data->prev_io_array[i].memory_address;
+            fh->f_io_array[i].offset = aggr_data->prev_io_array[i].offset;
+            fh->f_io_array[i].length = aggr_data->prev_io_array[i].length;
+        }
+
+        if (1 == write_syncType) {
+            if (is_accelerator_buffer) {
+                ret = mca_common_ompio_file_iwrite_pregen(fh, (ompi_request_t *) ompio_req);
+                if(0 > ret) {
+                    opal_output (1, "vulcan_write_all: mca_common_ompio_iwrite_pregen failed\n");
+                    ompio_req->req_ompi.req_status.MPI_ERROR = ret;
+                    ompio_req->req_ompi.req_status._ucount = 0;
+                }
+            } else {
+                ret = fh->f_fbtl->fbtl_ipwritev(fh, (ompi_request_t *) ompio_req);
+                if(0 > ret) {
+                    opal_output (1, "vulcan_write_all: fbtl_ipwritev failed\n");
+                    ompio_req->req_ompi.req_status.MPI_ERROR = ret;
+                    ompio_req->req_ompi.req_status._ucount = 0;
+                }
             }
         }
         else {
@@ -800,19 +716,16 @@ static int write_init (ompio_file_t *fh,
     return ret;
 }
 
-static int shuffle_init ( int index, int cycles, int aggregator, int rank, mca_io_ompio_aggregator_data *data, 
-                          ompi_request_t **reqs )
+static int shuffle_init (int index, int num_cycles, int aggregator, int rank,
+                         mca_io_ompio_aggregator_data *data, ompi_request_t **reqs)
 {
-    int bytes_sent = 0;
-    int blocks=0, temp_pindex;
-    int i, j, l, ret;
-    int  entries_per_aggregator=0;
+    size_t bytes_sent = 0;
+    int i, j, l;
+    int ret = OMPI_SUCCESS;
+    int  entries_per_aggregator = 0;
     mca_io_ompio_local_io_array *file_offsets_for_agg=NULL;
     int *sorted_file_offsets=NULL;
-    int temp_index=0;
     MPI_Aint *memory_displacements=NULL;
-    int *temp_disp_index=NULL;
-    MPI_Aint global_count = 0;
     int* blocklength_proc=NULL;
     ptrdiff_t* displs_proc=NULL;
 
@@ -824,21 +737,19 @@ static int shuffle_init ( int index, int cycles, int aggregator, int rank, mca_i
      ***  7a. Getting ready for next cycle: initializing and freeing buffers
      **********************************************************************/
     if (aggregator == rank) {
-        
         if (NULL != data->recvtype){
             for (i =0; i< data->procs_per_group; i++) {
-                if ( MPI_DATATYPE_NULL != data->recvtype[i] ) {
+                if (MPI_DATATYPE_NULL != data->recvtype[i]) {
                     ompi_datatype_destroy(&data->recvtype[i]);
                     data->recvtype[i] = MPI_DATATYPE_NULL;
                 }
             }
         }
 
-        
-        for(l=0;l<data->procs_per_group;l++){
+        for(l = 0; l < data->procs_per_group; l++){
             data->disp_index[l] = 0;
-            
-            if ( data->max_disp_index[l] == 0 ) {
+
+            if (data->max_disp_index[l] == 0) {
                 data->blocklen_per_process[l] = (int *) calloc (INIT_LEN, sizeof(int));
                 data->displs_per_process[l] = (MPI_Aint *) calloc (INIT_LEN, sizeof(MPI_Aint));
                 if (NULL == data->displs_per_process[l] || NULL == data->blocklen_per_process[l]){
@@ -847,25 +758,22 @@ static int shuffle_init ( int index, int cycles, int aggregator, int rank, mca_i
                     goto exit;
                 }
                 data->max_disp_index[l] = INIT_LEN;
-            }
-            else {
-                memset ( data->blocklen_per_process[l], 0, data->max_disp_index[l]*sizeof(int) );
-                memset ( data->displs_per_process[l], 0, data->max_disp_index[l]*sizeof(MPI_Aint) );
+            } else {
+                memset (data->blocklen_per_process[l], 0, data->max_disp_index[l]*sizeof(int));
+                memset (data->displs_per_process[l], 0, data->max_disp_index[l]*sizeof(MPI_Aint));
             }
         }
     } /* (aggregator == rank */
-    
+
     /**************************************************************************
      ***  7b. Determine the number of bytes to be actually written in this cycle
      **************************************************************************/
     int local_cycles= ceil((double)data->total_bytes / data->bytes_per_cycle);
-    if ( index  < (local_cycles -1) ) {
+    if (index  < (local_cycles -1)) {
         data->bytes_to_write_in_cycle = data->bytes_per_cycle;
-    }
-    else if ( index == (local_cycles -1)) {
+    } else if (index == (local_cycles -1)) {
         data->bytes_to_write_in_cycle = data->total_bytes - data->bytes_per_cycle*index ;
-    }
-    else {
+    } else {
         data->bytes_to_write_in_cycle = 0;
     }
     data->bytes_to_write = data->bytes_to_write_in_cycle;
@@ -873,305 +781,57 @@ static int shuffle_init ( int index, int cycles, int aggregator, int rank, mca_i
 #if DEBUG_ON
     if (aggregator == rank) {
         printf ("****%d: CYCLE %d   Bytes %lld**********\n",
-                rank,
-                index,
-                data->bytes_to_write_in_cycle);
+                rank, index, data->bytes_to_write_in_cycle);
     }
 #endif
-    /**********************************************************
-     **Gather the Data from all the processes at the writers **
-     *********************************************************/
-    
-#if DEBUG_ON
-    printf("bytes_to_write_in_cycle: %ld, cycle : %d\n", data->bytes_to_write_in_cycle,
-           index);
-#endif
-    
+
     /*****************************************************************
      *** 7c. Calculate how much data will be contributed in this cycle
      ***     by each process
      *****************************************************************/
-    
-    /* The blocklen and displs calculation only done at aggregators!*/
-    while (data->bytes_to_write_in_cycle) {
-        
-        /* This next block identifies which process is the holder
-        ** of the sorted[current_index] element;
-        */
-        blocks = data->fview_count[0];
-        for (j=0 ; j<data->procs_per_group ; j++) {
-            if (data->sorted[data->current_index] < blocks) {
-                data->n = j;
-                break;
-            }
-            else {
-                blocks += data->fview_count[j+1];
-            }
-        }
+    mca_fcoll_vulcan_calc_blocklen_disps (data, aggregator, rank, &bytes_sent);
         
-        if (data->bytes_remaining) {
-            /* Finish up a partially used buffer from the previous  cycle */
-            
-            if (data->bytes_remaining <= data->bytes_to_write_in_cycle) {
-                /* The data fits completely into the block */
-                if (aggregator == rank) {
-                    data->blocklen_per_process[data->n][data->disp_index[data->n]] = data->bytes_remaining;
-                    data->displs_per_process[data->n][data->disp_index[data->n]] =
-                        (ptrdiff_t)data->global_iov_array[data->sorted[data->current_index]].iov_base +
-                        (data->global_iov_array[data->sorted[data->current_index]].iov_len
-                         - data->bytes_remaining);
-
-                    data->disp_index[data->n] += 1;
-                    
-                    /* In this cases the length is consumed so allocating for
-                       next displacement and blocklength*/
-                    if ( data->disp_index[data->n] == data->max_disp_index[data->n] ) {
-                        data->max_disp_index[data->n] *= 2;
-
-                        data->blocklen_per_process[data->n] = (int *) realloc
-                            ((void *)data->blocklen_per_process[data->n], 
-                             (data->max_disp_index[data->n])*sizeof(int));
-                        data->displs_per_process[data->n] = (MPI_Aint *) realloc
-                            ((void *)data->displs_per_process[data->n], 
-                             (data->max_disp_index[data->n])*sizeof(MPI_Aint));
-                    }
-                    data->blocklen_per_process[data->n][data->disp_index[data->n]] = 0;
-                    data->displs_per_process[data->n][data->disp_index[data->n]] = 0;
-
-                }
-                if (data->procs_in_group[data->n] == rank) {
-                    bytes_sent += data->bytes_remaining;
-                }
-                data->current_index ++;
-                data->bytes_to_write_in_cycle -= data->bytes_remaining;
-                data->bytes_remaining = 0;
-            }
-            else {
-                /* the remaining data from the previous cycle is larger than the
-                   data->bytes_to_write_in_cycle, so we have to segment again */
-                if (aggregator == rank) {
-                    data->blocklen_per_process[data->n][data->disp_index[data->n]] = data->bytes_to_write_in_cycle;
-                    data->displs_per_process[data->n][data->disp_index[data->n]] =
-                        (ptrdiff_t)data->global_iov_array[data->sorted[data->current_index]].iov_base +
-                        (data->global_iov_array[data->sorted[data->current_index]].iov_len
-                         - data->bytes_remaining);
-                    data->disp_index[data->n] += 1;
-                }
-                
-                if (data->procs_in_group[data->n] == rank) {
-                    bytes_sent += data->bytes_to_write_in_cycle;
-                }
-                data->bytes_remaining -= data->bytes_to_write_in_cycle;
-                data->bytes_to_write_in_cycle = 0;
-                break;
-            }
-        }
-        else {
-            /* No partially used entry available, have to start a new one */
-            if (data->bytes_to_write_in_cycle <
-                (MPI_Aint) data->global_iov_array[data->sorted[data->current_index]].iov_len) {
-                /* This entry has more data than we can sendin one cycle */
-                if (aggregator == rank) {
-                    data->blocklen_per_process[data->n][data->disp_index[data->n]] = data->bytes_to_write_in_cycle;
-                    data->displs_per_process[data->n][data->disp_index[data->n]] =
-                        (ptrdiff_t)data->global_iov_array[data->sorted[data->current_index]].iov_base ;
-                    data->disp_index[data->n] += 1;
-                }
-                if (data->procs_in_group[data->n] == rank) {
-                    bytes_sent += data->bytes_to_write_in_cycle;
-                    
-                }
-                data->bytes_remaining = data->global_iov_array[data->sorted[data->current_index]].iov_len -
-                    data->bytes_to_write_in_cycle;
-                data->bytes_to_write_in_cycle = 0;
-                break;
-            }
-            else {
-                /* Next data entry is less than data->bytes_to_write_in_cycle */
-                if (aggregator == rank) {
-                    data->blocklen_per_process[data->n][data->disp_index[data->n]] =
-                        data->global_iov_array[data->sorted[data->current_index]].iov_len;
-                    data->displs_per_process[data->n][data->disp_index[data->n]] = (ptrdiff_t)
-                        data->global_iov_array[data->sorted[data->current_index]].iov_base;
-
-                    data->disp_index[data->n] += 1;
-                    
-                    /*realloc for next blocklength
-                      and assign this displacement and check for next displs as
-                      the total length of this entry has been consumed!*/
-                    if ( data->disp_index[data->n] == data->max_disp_index[data->n] ) {
-                        data->max_disp_index[data->n] *=2 ;
-                        data->blocklen_per_process[data->n] = (int *) realloc (
-                            (void *)data->blocklen_per_process[data->n], 
-                            (data->max_disp_index[data->n]*sizeof(int)));
-                        data->displs_per_process[data->n] = (MPI_Aint *)realloc (
-                            (void *)data->displs_per_process[data->n], 
-                            (data->max_disp_index[data->n]*sizeof(MPI_Aint)));
-                    }
-                    data->blocklen_per_process[data->n][data->disp_index[data->n]] = 0;
-                    data->displs_per_process[data->n][data->disp_index[data->n]] = 0;
-                }
-                if (data->procs_in_group[data->n] == rank) {
-                    bytes_sent += data->global_iov_array[data->sorted[data->current_index]].iov_len;
-                }
-                data->bytes_to_write_in_cycle -=
-                    data->global_iov_array[data->sorted[data->current_index]].iov_len;
-                data->current_index ++;
-            }
-        }
-    }
-    
-    
     /*************************************************************************
-     *** 7d. Calculate the displacement on where to put the data and allocate
-     ***     the recieve buffer (global_buf)
+     *** 7d. Calculate the displacement on where to put the data
      *************************************************************************/
     if (aggregator == rank) {
         entries_per_aggregator=0;
-        for (i=0;i<data->procs_per_group; i++){
-            for (j=0;j<data->disp_index[i];j++){
-                if (data->blocklen_per_process[i][j] > 0)
+        for (i = 0; i < data->procs_per_group; i++){
+            for (j = 0; j < data->disp_index[i];j++){
+                if (data->blocklen_per_process[i][j] > 0) {
                     entries_per_aggregator++ ;
+                }
             }
         }
-        
 #if DEBUG_ON
-        printf("%d: cycle: %d, bytes_sent: %d\n ",rank,index,
-               bytes_sent);
-        printf("%d : Entries per aggregator : %d\n",rank,entries_per_aggregator);
+        printf("%d : Entries per aggregator : %d\n", rank, entries_per_aggregator);
 #endif
-        
-        if (entries_per_aggregator > 0){
-            file_offsets_for_agg = (mca_io_ompio_local_io_array *)
-                malloc(entries_per_aggregator*sizeof(mca_io_ompio_local_io_array));
-            if (NULL == file_offsets_for_agg) {
+
+        if (entries_per_aggregator > 0) {
+            file_offsets_for_agg = (mca_io_ompio_local_io_array *) malloc(entries_per_aggregator *
+                                                              sizeof(mca_io_ompio_local_io_array));
+            memory_displacements = (MPI_Aint *) malloc (entries_per_aggregator * sizeof(MPI_Aint));
+            sorted_file_offsets = (int *) malloc (entries_per_aggregator*sizeof(int));
+            if (NULL == memory_displacements || NULL == file_offsets_for_agg ||
+                NULL == sorted_file_offsets) {
                 opal_output (1, "OUT OF MEMORY\n");
                 ret = OMPI_ERR_OUT_OF_RESOURCE;
                 goto exit;
             }
-            
-            sorted_file_offsets = (int *)
-                malloc (entries_per_aggregator*sizeof(int));
-            if (NULL == sorted_file_offsets){
-                opal_output (1, "OUT OF MEMORY\n");
-                ret =  OMPI_ERR_OUT_OF_RESOURCE;
-                goto exit;
-            }
-            
-            /*Moving file offsets to an IO array!*/
-            temp_index = 0;
-            
-            for (i=0;i<data->procs_per_group; i++){
-                for(j=0;j<data->disp_index[i];j++){
-                    if (data->blocklen_per_process[i][j] > 0){
-                        file_offsets_for_agg[temp_index].length =
-                            data->blocklen_per_process[i][j];
-                        file_offsets_for_agg[temp_index].process_id = i;
-                        file_offsets_for_agg[temp_index].offset =
-                            data->displs_per_process[i][j];
-                        temp_index++;
-                        
-#if DEBUG_ON
-                        printf("************Cycle: %d,  Aggregator: %d ***************\n",
-                               index+1,rank);
-                        
-                        printf("%d sends blocklen[%d]: %d, disp[%d]: %ld to %d\n",
-                               data->procs_in_group[i],j,
-                               data->blocklen_per_process[i][j],j,
-                               data->displs_per_process[i][j],
-                               rank);
-#endif
-                    }
-                }
-            }
-                
-            /* Sort the displacements for each aggregator*/
-            local_heap_sort (file_offsets_for_agg,
-                             entries_per_aggregator,
-                             sorted_file_offsets);
-            
-            /*create contiguous memory displacements
-              based on blocklens on the same displs array
-              and map it to this aggregator's actual
-              file-displacements (this is in the io-array created above)*/
-            memory_displacements = (MPI_Aint *) malloc
-                (entries_per_aggregator * sizeof(MPI_Aint));
-            
-            memory_displacements[sorted_file_offsets[0]] = 0;
-            for (i=1; i<entries_per_aggregator; i++){
-                memory_displacements[sorted_file_offsets[i]] =
-                    memory_displacements[sorted_file_offsets[i-1]] +
-                    file_offsets_for_agg[sorted_file_offsets[i-1]].length;
-            }
-            
-            temp_disp_index = (int *)calloc (1, data->procs_per_group * sizeof (int));
-            if (NULL == temp_disp_index) {
-                opal_output (1, "OUT OF MEMORY\n");
-                ret = OMPI_ERR_OUT_OF_RESOURCE;
+
+            ret = mca_fcoll_vulcan_calc_file_offsets(data, file_offsets_for_agg, sorted_file_offsets,
+                                                     memory_displacements, entries_per_aggregator,
+                                                     rank, index);
+            if (OMPI_SUCCESS != ret) {
                 goto exit;
             }
-            
-            /*Now update the displacements array  with memory offsets*/
-            global_count = 0;
-            for (i=0;i<entries_per_aggregator;i++){
-                temp_pindex =
-                    file_offsets_for_agg[sorted_file_offsets[i]].process_id;
-                data->displs_per_process[temp_pindex][temp_disp_index[temp_pindex]] =
-                    memory_displacements[sorted_file_offsets[i]];
-                if (temp_disp_index[temp_pindex] < data->disp_index[temp_pindex])
-                    temp_disp_index[temp_pindex] += 1;
-                else{
-                    printf("temp_disp_index[%d]: %d is greater than disp_index[%d]: %d\n",
-                           temp_pindex, temp_disp_index[temp_pindex],
-                           temp_pindex, data->disp_index[temp_pindex]);
-                }
-                global_count +=
-                    file_offsets_for_agg[sorted_file_offsets[i]].length;
-            }
-            
-            if (NULL != temp_disp_index){
-                free(temp_disp_index);
-                temp_disp_index = NULL;
-            }
-            
-#if DEBUG_ON
-            
-            printf("************Cycle: %d,  Aggregator: %d ***************\n",
-                   index+1,rank);
-            for (i=0;i<data->procs_per_group; i++){
-                for(j=0;j<data->disp_index[i];j++){
-                    if (data->blocklen_per_process[i][j] > 0){
-                        printf("%d sends blocklen[%d]: %d, disp[%d]: %ld to %d\n",
-                               data->procs_in_group[i],j,
-                               data->blocklen_per_process[i][j],j,
-                               data->displs_per_process[i][j],
-                               rank);
-                        
-                    }
-                }
-            }
-            printf("************Cycle: %d,  Aggregator: %d ***************\n",
-                   index+1,rank);
-            for (i=0; i<entries_per_aggregator;i++){
-                printf("%d: OFFSET: %lld   LENGTH: %ld, Mem-offset: %ld\n",
-                       file_offsets_for_agg[sorted_file_offsets[i]].process_id,
-                       file_offsets_for_agg[sorted_file_offsets[i]].offset,
-                       file_offsets_for_agg[sorted_file_offsets[i]].length,
-                       memory_displacements[sorted_file_offsets[i]]);
-            }
-            printf("%d : global_count : %ld, bytes_sent : %d\n",
-                   rank,global_count, bytes_sent);
-#endif
-//#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-//            start_comm_time = MPI_Wtime();
-//#endif
             /*************************************************************************
              *** 7e. Perform the actual communication
              *************************************************************************/
-            for (i=0;i<data->procs_per_group; i++) {
+            for (i = 0; i < data->procs_per_group; i++) {
                 size_t datatype_size;
                 reqs[i] = MPI_REQUEST_NULL;
-                if ( 0 < data->disp_index[i] ) {
+                if (0 < data->disp_index[i]) {
                     ompi_datatype_create_hindexed(data->disp_index[i],
                                                   data->blocklen_per_process[i],
                                                   data->displs_per_process[i],
@@ -1180,7 +840,7 @@ static int shuffle_init ( int index, int cycles, int aggregator, int rank, mca_i
                     ompi_datatype_commit(&data->recvtype[i]);
                     opal_datatype_type_size(&data->recvtype[i]->super, &datatype_size);
                     
-                    if (datatype_size){
+                    if (datatype_size) {
                         ret = MCA_PML_CALL(irecv(data->global_buf,
                                                  1,
                                                  data->recvtype[i],
@@ -1219,8 +879,7 @@ static int shuffle_init ( int index, int cycles, int aggregator, int rank, mca_i
             if(0 == block_index) {
                 send_mem_address = (ptrdiff_t) (data->decoded_iov[data->iov_index].iov_base) +
                                                 data->current_position;
-            }
-            else {
+            } else {
                 // Reallocate more memory if blocklength_size is not enough
                 if(0 == block_index % INIT_LEN) {
                     blocklength_size += INIT_LEN;
@@ -1231,17 +890,14 @@ static int shuffle_init ( int index, int cycles, int aggregator, int rank, mca_i
                                                         data->current_position - send_mem_address;
             }
 
-            if (remaining >=
-                (data->decoded_iov[data->iov_index].iov_len - data->current_position)) {
-
+            if (remaining >= (data->decoded_iov[data->iov_index].iov_len - data->current_position)) {
                 blocklength_proc[block_index] = data->decoded_iov[data->iov_index].iov_len -
                                                 data->current_position;
                 remaining = remaining -
                             (data->decoded_iov[data->iov_index].iov_len - data->current_position);
                 data->iov_index = data->iov_index + 1;
                 data->current_position = 0;
-            }
-            else {
+            } else {
                 blocklength_proc[block_index] = remaining;
                 data->current_position += remaining;
                 remaining = 0;
@@ -1276,77 +932,23 @@ static int shuffle_init ( int index, int cycles, int aggregator, int rank, mca_i
         }
     }
 
-#if DEBUG_ON
-    if (aggregator == rank){
-        printf("************Cycle: %d,  Aggregator: %d ***************\n",
-               index+1,rank);
-        for (i=0 ; i<global_count/4 ; i++)
-            printf (" RECV %d \n",((int *)data->global_buf)[i]);
-    }
-#endif
-    
-//#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
-//    end_comm_time = MPI_Wtime();
-//    comm_time += (end_comm_time - start_comm_time);
-//#endif
     /**********************************************************
      *** 7f. Create the io array, and pass it to fbtl
      *********************************************************/
-    
-    if (aggregator == rank && entries_per_aggregator>0) {
-        
-        
-        data->io_array = (mca_common_ompio_io_array_t *) malloc
-            (entries_per_aggregator * sizeof (mca_common_ompio_io_array_t));
+    if (aggregator == rank && entries_per_aggregator > 0) {
+        data->io_array = (mca_common_ompio_io_array_t *) malloc (entries_per_aggregator *
+                                                         sizeof (mca_common_ompio_io_array_t));
         if (NULL == data->io_array) {
             opal_output(1, "OUT OF MEMORY\n");
             ret = OMPI_ERR_OUT_OF_RESOURCE;
             goto exit;
         }
-        
-        data->num_io_entries = 0;
-        /*First entry for every aggregator*/
-        data->io_array[0].offset =
-            (IOVBASE_TYPE *)(intptr_t)file_offsets_for_agg[sorted_file_offsets[0]].offset;
-        data->io_array[0].length =
-            file_offsets_for_agg[sorted_file_offsets[0]].length;
-        data->io_array[0].memory_address =
-            data->global_buf+memory_displacements[sorted_file_offsets[0]];
-        data->num_io_entries++;
-        
-        for (i=1;i<entries_per_aggregator;i++){
-            /* If the enrties are contiguous merge them,
-               else make a new entry */
-            if (file_offsets_for_agg[sorted_file_offsets[i-1]].offset +
-                file_offsets_for_agg[sorted_file_offsets[i-1]].length ==
-                file_offsets_for_agg[sorted_file_offsets[i]].offset){
-                data->io_array[data->num_io_entries - 1].length +=
-                    file_offsets_for_agg[sorted_file_offsets[i]].length;
-            }
-            else {
-                data->io_array[data->num_io_entries].offset =
-                    (IOVBASE_TYPE *)(intptr_t)file_offsets_for_agg[sorted_file_offsets[i]].offset;
-                data->io_array[data->num_io_entries].length =
-                    file_offsets_for_agg[sorted_file_offsets[i]].length;
-                data->io_array[data->num_io_entries].memory_address =
-                    data->global_buf+memory_displacements[sorted_file_offsets[i]];
-                data->num_io_entries++;
-            }
-            
-        }
-        
-#if DEBUG_ON
-        printf("*************************** %d\n", num_of_io_entries);
-        for (i=0 ; i<num_of_io_entries ; i++) {
-            printf(" ADDRESS: %p  OFFSET: %ld   LENGTH: %ld\n",
-                   io_array[i].memory_address,
-                   (ptrdiff_t)io_array[i].offset,
-                   io_array[i].length);
-        }
-        
-#endif
+
+        mca_fcoll_vulcan_calc_io_array(data->io_array, &data->num_io_entries, entries_per_aggregator,
+                                       (char*)data->global_buf, file_offsets_for_agg, sorted_file_offsets,
+                                       memory_displacements, rank);
     }
-        
+
 exit:
     free(sorted_file_offsets);
     free(file_offsets_for_agg);
@@ -1354,10 +956,10 @@ static int shuffle_init ( int index, int cycles, int aggregator, int rank, mca_i
     free(blocklength_proc);
     free(displs_proc);
 
-    return OMPI_SUCCESS;
+    return ret;
 }
     
-static int mca_fcoll_vulcan_minmax ( ompio_file_t *fh, struct iovec *iov, int iov_count,  int num_aggregators, long *new_stripe_size)
+int mca_fcoll_vulcan_minmax (ompio_file_t *fh, struct iovec *iov, int iov_count,  int num_aggregators, long *new_stripe_size)
 {
     long min, max, globalmin, globalmax;
     long stripe_size;
@@ -1371,12 +973,10 @@ static int mca_fcoll_vulcan_minmax ( ompio_file_t *fh, struct iovec *iov, int io
         max = 0;
     }
     fh->f_comm->c_coll->coll_allreduce ( &min, &globalmin, 1, MPI_LONG, MPI_MIN,
-					 fh->f_comm, fh->f_comm->c_coll->coll_allreduce_module);
+                                         fh->f_comm, fh->f_comm->c_coll->coll_allreduce_module);
     
     fh->f_comm->c_coll->coll_allreduce ( &max, &globalmax, 1, MPI_LONG, MPI_MAX,
-					 fh->f_comm, fh->f_comm->c_coll->coll_allreduce_module);
-
-    //    if ( fh->f_rank < 10 ) printf("[%d]: min=%ld max=%ld globalmin=%ld, globalmax=%ld num_aggregators=%d\n", fh->f_rank, min, max, globalmin, globalmax, num_aggregators);
+                                         fh->f_comm, fh->f_comm->c_coll->coll_allreduce_module);
 
     stripe_size = (globalmax - globalmin)/num_aggregators;
     if ( (globalmax - globalmin) % num_aggregators ) {
@@ -1384,13 +984,9 @@ static int mca_fcoll_vulcan_minmax ( ompio_file_t *fh, struct iovec *iov, int io
     }
 
     *new_stripe_size  = stripe_size;
-    //    if ( fh->f_rank == 0 ) 
-    //    printf(" partition size is %ld\n", stripe_size);
 
     return OMPI_SUCCESS;
 }
-    
-    
 
 int mca_fcoll_vulcan_break_file_view ( struct iovec *mem_iov, int mem_count, 
                                         struct iovec *file_iov, int file_count, 
@@ -1619,21 +1215,11 @@ int mca_fcoll_vulcan_break_file_view ( struct iovec *mem_iov, int mem_count,
     return ret;
 }
 
-
-int mca_fcoll_vulcan_get_configuration (ompio_file_t *fh, int num_io_procs, int num_groups, 
-                                        size_t max_data)
+int mca_fcoll_vulcan_get_configuration (ompio_file_t *fh, int num_io_procs, size_t max_data)
 {
     int i, ret;
-    ret = mca_common_ompio_set_aggregator_props (fh, num_io_procs, max_data);
 
-    /* Note: as of this version of the vulcan component, we are not using yet
-       the num_groups parameter to split the aggregators (and processes) into
-       distinct subgroups. This will however hopefullty be done in a second step
-       as well, allowing to keep communication just to individual subgroups of processes,
-       each subgroup using however the classic two-phase collective I/O algorithm
-       with multiple aggregators and even partitioning internally. 
-    
-       For now, logically all processes are in a single group. */
+    ret = mca_common_ompio_set_aggregator_props (fh, num_io_procs, max_data);
 
     fh->f_procs_per_group = fh->f_size;
     if ( NULL != fh->f_procs_in_group ) {
@@ -1649,63 +1235,269 @@ int mca_fcoll_vulcan_get_configuration (ompio_file_t *fh, int num_io_procs, int
     
     return ret;
 }    
-    
 
-int mca_fcoll_vulcan_split_iov_array ( ompio_file_t *fh, mca_common_ompio_io_array_t *io_array, int num_entries,
-                                             int *ret_array_pos, int *ret_pos,  int chunk_size )
+void mca_fcoll_vulcan_calc_blocklen_disps (mca_io_ompio_aggregator_data *data, int aggregator,
+                                           int rank, size_t *bytes_comm)
 {
+    size_t bytes_tmp = *bytes_comm;
+    int blocks = 0;
+    int j;
 
-    int array_pos = *ret_array_pos;
-    int pos       = *ret_pos;
-    size_t bytes_written = 0;
-    size_t bytes_to_write = chunk_size;
+    /* The blocklen and displs calculation only done at aggregators */
+    while (data->bytes_to_write_in_cycle) {
 
-    if ( 0 == array_pos && 0 == pos ) {
-        fh->f_io_array = (mca_common_ompio_io_array_t *) malloc ( num_entries * sizeof(mca_common_ompio_io_array_t));
-        if ( NULL == fh->f_io_array ){
-            opal_output (1,"Could not allocate memory\n");
-            return -1;
+        /* This next block identifies which process is the holder
+        ** of the sorted[current_index] element;
+        */
+        blocks = data->fview_count[0];
+        for (j = 0 ; j < data->procs_per_group ; j++) {
+            if (data->sorted[data->current_index] < blocks) {
+                data->n = j;
+                break;
+            } else {
+                blocks += data->fview_count[j+1];
+            }
+        }
+
+        if (data->bytes_remaining) {
+            /* Finish up a partially used buffer from the previous  cycle */
+
+            if (data->bytes_remaining <= data->bytes_to_write_in_cycle) {
+                /* The data fits completely into the block */
+                if (aggregator == rank) {
+                    data->blocklen_per_process[data->n][data->disp_index[data->n]] = data->bytes_remaining;
+                    data->displs_per_process[data->n][data->disp_index[data->n]] =
+                        (ptrdiff_t)data->global_iov_array[data->sorted[data->current_index]].iov_base +
+                        (data->global_iov_array[data->sorted[data->current_index]].iov_len
+                         - data->bytes_remaining);
+
+                    data->disp_index[data->n] += 1;
+
+                    /* In this cases the length is consumed so allocating for
+                       next displacement and blocklength*/
+                    if (data->disp_index[data->n] == data->max_disp_index[data->n]) {
+                        data->max_disp_index[data->n] *= 2;
+
+                        data->blocklen_per_process[data->n] = (int *) realloc
+                            ((void *)data->blocklen_per_process[data->n],
+                             (data->max_disp_index[data->n])*sizeof(int));
+                        data->displs_per_process[data->n] = (MPI_Aint *) realloc
+                            ((void *)data->displs_per_process[data->n],
+                             (data->max_disp_index[data->n])*sizeof(MPI_Aint));
+                    }
+                    data->blocklen_per_process[data->n][data->disp_index[data->n]] = 0;
+                    data->displs_per_process[data->n][data->disp_index[data->n]] = 0;
+                }
+                if (data->procs_in_group[data->n] == rank) {
+                    bytes_tmp += data->bytes_remaining;
+                }
+                data->current_index ++;
+                data->bytes_to_write_in_cycle -= data->bytes_remaining;
+                data->bytes_remaining = 0;
+            } else {
+                /* the remaining data from the previous cycle is larger than the
+                   data->bytes_to_write_in_cycle, so we have to segment again */
+                if (aggregator == rank) {
+                    data->blocklen_per_process[data->n][data->disp_index[data->n]] = data->bytes_to_write_in_cycle;
+                    data->displs_per_process[data->n][data->disp_index[data->n]] =
+                        (ptrdiff_t)data->global_iov_array[data->sorted[data->current_index]].iov_base +
+                        (data->global_iov_array[data->sorted[data->current_index]].iov_len
+                         - data->bytes_remaining);
+                    data->disp_index[data->n] += 1;
+                }
+
+                if (data->procs_in_group[data->n] == rank) {
+                    bytes_tmp += data->bytes_to_write_in_cycle;
+                }
+                data->bytes_remaining -= data->bytes_to_write_in_cycle;
+                data->bytes_to_write_in_cycle = 0;
+                break;
+            }
+        } else {
+            /* No partially used entry available, have to start a new one */
+            if (data->bytes_to_write_in_cycle <
+                (MPI_Aint) data->global_iov_array[data->sorted[data->current_index]].iov_len) {
+                /* This entry has more data than we can sendin one cycle */
+                if (aggregator == rank) {
+                    data->blocklen_per_process[data->n][data->disp_index[data->n]] = data->bytes_to_write_in_cycle;
+                    data->displs_per_process[data->n][data->disp_index[data->n]] =
+                        (ptrdiff_t)data->global_iov_array[data->sorted[data->current_index]].iov_base ;
+                    data->disp_index[data->n] += 1;
+                }
+                if (data->procs_in_group[data->n] == rank) {
+                    bytes_tmp += data->bytes_to_write_in_cycle;
+                }
+                data->bytes_remaining = data->global_iov_array[data->sorted[data->current_index]].iov_len -
+                    data->bytes_to_write_in_cycle;
+                data->bytes_to_write_in_cycle = 0;
+                break;
+            } else {
+                /* Next data entry is less than data->bytes_to_write_in_cycle */
+                if (aggregator == rank) {
+                    data->blocklen_per_process[data->n][data->disp_index[data->n]] =
+                        data->global_iov_array[data->sorted[data->current_index]].iov_len;
+                    data->displs_per_process[data->n][data->disp_index[data->n]] = (ptrdiff_t)
+                        data->global_iov_array[data->sorted[data->current_index]].iov_base;
+
+                    data->disp_index[data->n] += 1;
+
+                    /* realloc for next blocklength and assign this displacement
+                    ** and check for next displs as the total length of this entry
+                    ** has been consumed */
+                    if (data->disp_index[data->n] == data->max_disp_index[data->n] ) {
+                        data->max_disp_index[data->n] *=2 ;
+                        data->blocklen_per_process[data->n] = (int *) realloc (
+                            (void *)data->blocklen_per_process[data->n],
+                            (data->max_disp_index[data->n]*sizeof(int)));
+                        data->displs_per_process[data->n] = (MPI_Aint *)realloc (
+                            (void *)data->displs_per_process[data->n],
+                            (data->max_disp_index[data->n]*sizeof(MPI_Aint)));
+                    }
+                    data->blocklen_per_process[data->n][data->disp_index[data->n]] = 0;
+                    data->displs_per_process[data->n][data->disp_index[data->n]] = 0;
+                }
+                if (data->procs_in_group[data->n] == rank) {
+                    bytes_tmp += data->global_iov_array[data->sorted[data->current_index]].iov_len;
+                }
+                data->bytes_to_write_in_cycle -=
+                    data->global_iov_array[data->sorted[data->current_index]].iov_len;
+                data->current_index ++;
+            }
         }
     }
-        
-    int i=0;
-    while (bytes_to_write > 0 ) {
-        fh->f_io_array[i].memory_address = &(((char *)io_array[array_pos].memory_address)[pos]);
-        fh->f_io_array[i].offset = &(((char *)io_array[array_pos].offset)[pos]);
 
-        if ( (io_array[array_pos].length - pos ) >= bytes_to_write ) {
-            fh->f_io_array[i].length = bytes_to_write;
+    *bytes_comm = bytes_tmp;
+}
+
+int mca_fcoll_vulcan_calc_file_offsets(mca_io_ompio_aggregator_data *data, mca_io_ompio_local_io_array *file_offsets_for_agg,
+                                       int *sorted_file_offsets, MPI_Aint *memory_displacements, int entries_per_aggregator,
+                                       int rank, int index)
+{
+    int *temp_disp_index;
+    int temp_index = 0;
+    int temp_pindex;
+    int i, j;
+
+    /* Moving file offsets to an IO array */
+    for (i = 0; i < data->procs_per_group; i++){
+        for(j = 0; j < data->disp_index[i];j++){
+            if (data->blocklen_per_process[i][j] > 0){
+                file_offsets_for_agg[temp_index].length =
+                    data->blocklen_per_process[i][j];
+                file_offsets_for_agg[temp_index].process_id = i;
+                file_offsets_for_agg[temp_index].offset =
+                    data->displs_per_process[i][j];
+                temp_index++;
+            }
+        }
+    }
+
+    /* Sort the displacements for each aggregator */
+    local_heap_sort (file_offsets_for_agg, entries_per_aggregator,
+                     sorted_file_offsets);
+
+    /* create contiguous memory displacements based on blocklens
+    ** on the same displs array and map it to this aggregator's actual
+    ** file-displacements */
+    memory_displacements[sorted_file_offsets[0]] = 0;
+    for (i = 1; i < entries_per_aggregator; i++){
+        memory_displacements[sorted_file_offsets[i]] =
+            memory_displacements[sorted_file_offsets[i-1]] +
+            file_offsets_for_agg[sorted_file_offsets[i-1]].length;
+    }
+
+    temp_disp_index = (int *)calloc (1, data->procs_per_group * sizeof (int));
+    if (NULL == temp_disp_index) {
+        opal_output (1, "OUT OF MEMORY\n");
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    /* Now update the displacements array  with memory offsets */
+    for (i = 0; i < entries_per_aggregator;i++) {
+        temp_pindex = file_offsets_for_agg[sorted_file_offsets[i]].process_id;
+        data->displs_per_process[temp_pindex][temp_disp_index[temp_pindex]] =
+            memory_displacements[sorted_file_offsets[i]];
+        if (temp_disp_index[temp_pindex] < data->disp_index[temp_pindex]) {
+            temp_disp_index[temp_pindex] += 1;
         }
         else {
-            fh->f_io_array[i].length = io_array[array_pos].length - pos;
+            printf("temp_disp_index[%d]: %d is greater than disp_index[%d]: %d\n",
+                   temp_pindex, temp_disp_index[temp_pindex],
+                   temp_pindex, data->disp_index[temp_pindex]);
         }
+    }
 
-        pos           += fh->f_io_array[i].length;
-        bytes_written += fh->f_io_array[i].length;
-        bytes_to_write-= fh->f_io_array[i].length;
-        i++;
+    free(temp_disp_index);
 
-        if ( pos == (int)io_array[array_pos].length ) {
-            pos = 0;
-            if ((array_pos + 1) < num_entries) {
-                array_pos++;
-            }
-            else {
-                break;
+#if DEBUG_ON
+    printf("************Cycle: %d,  Aggregator: %d ***************\n",
+           index+1, rank);
+    for (i = 0; i < data->procs_per_group; i++){
+        for(j = 0; j < data->disp_index[i]; j++){
+            if (data->blocklen_per_process[i][j] > 0){
+                printf("%d communicate blocklen[%d]: %d, disp[%d]: %ld to %d\n",
+                       data->procs_in_group[i],j,
+                       data->blocklen_per_process[i][j],j,
+                       data->displs_per_process[i][j],
+                       rank);
             }
         }
     }
-    
-    fh->f_num_of_io_entries = i;
-    *ret_array_pos   = array_pos;
-    *ret_pos         = pos;
-    return bytes_written;
+    printf("************Cycle: %d,  Aggregator: %d ***************\n",
+           index+1, rank);
+    for (i = 0; i < entries_per_aggregator;i++){
+        printf("%d: OFFSET: %lld   LENGTH: %ld, Mem-offset: %ld\n",
+               file_offsets_for_agg[sorted_file_offsets[i]].process_id,
+               file_offsets_for_agg[sorted_file_offsets[i]].offset,
+               file_offsets_for_agg[sorted_file_offsets[i]].length,
+               memory_displacements[sorted_file_offsets[i]]);
+    }
+#endif
+
+    return OMPI_SUCCESS;
+}
+
+void mca_fcoll_vulcan_calc_io_array(mca_common_ompio_io_array_t *io_array, int *num_io_entries, int max_io_entries,
+                                    char *global_buf, mca_io_ompio_local_io_array *file_offsets_for_agg,
+                                    int *sorted_offsets, MPI_Aint *memory_displacements, int rank)
+{
+    int i;
+    int num_entries;
+
+    /* First entry for every aggregator */
+    io_array[0].offset = (IOVBASE_TYPE *)(intptr_t)file_offsets_for_agg[sorted_offsets[0]].offset;
+    io_array[0].length = file_offsets_for_agg[sorted_offsets[0]].length;
+    io_array[0].memory_address = global_buf + memory_displacements[sorted_offsets[0]];
+    num_entries = 1;
+
+    /* If the entries are contiguous merge them, else add a new entry */
+    for (i = 1; i < max_io_entries; i++) {
+        if (file_offsets_for_agg[sorted_offsets[i-1]].offset +
+            file_offsets_for_agg[sorted_offsets[i-1]].length ==
+            file_offsets_for_agg[sorted_offsets[i]].offset) {
+            io_array[num_entries - 1].length += file_offsets_for_agg[sorted_offsets[i]].length;
+        } else {
+            io_array[num_entries].offset = (IOVBASE_TYPE *)(intptr_t)file_offsets_for_agg[sorted_offsets[i]].offset;
+            io_array[num_entries].length = file_offsets_for_agg[sorted_offsets[i]].length;
+            io_array[num_entries].memory_address = global_buf + memory_displacements[sorted_offsets[i]];
+            num_entries++;
+        }
+    }
+
+    *num_io_entries = num_entries;
+#if DEBUG_ON
+    printf("*************************** %d\n", num_entries);
+    for (i = 0; i < num_entries; i++) {
+        printf(" AGGREGATOR %d ADDRESS: %p  OFFSET: %ld   LENGTH: %ld\n",
+               rank, io_array[i].memory_address,
+               (ptrdiff_t)io_array[i].offset,
+               io_array[i].length);
+    }
+#endif
 }
 
-    
 static int local_heap_sort (mca_io_ompio_local_io_array *io_array,
-			    int num_entries,
-			    int *sorted)
+			    int num_entries, int *sorted)
 {
     int i = 0;
     int j = 0;
@@ -1805,5 +1597,3 @@ static int local_heap_sort (mca_io_ompio_local_io_array *io_array,
     }
     return OMPI_SUCCESS;
 }
-
-
diff --git a/ompi/mca/fcoll/vulcan/fcoll_vulcan_internal.h b/ompi/mca/fcoll/vulcan/fcoll_vulcan_internal.h
new file mode 100644
index 00000000000..76402297044
--- /dev/null
+++ b/ompi/mca/fcoll/vulcan/fcoll_vulcan_internal.h
@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2024      Advanced Micro Devices, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef MCA_FCOLL_VULCAN_INTERNAL_H
+#define MCA_FCOLL_VULCAN_INTERNAL_H
+
+#include "ompi_config.h"
+
+
+BEGIN_C_DECLS
+/* Used for loading file-offsets per aggregator*/
+typedef struct mca_io_ompio_local_io_array{
+    OMPI_MPI_OFFSET_TYPE offset;
+    MPI_Aint             length;
+    int                  process_id;
+}mca_io_ompio_local_io_array;
+
+typedef struct mca_io_ompio_aggregator_data {
+    int *disp_index, *sorted, n;
+    size_t *fview_count;
+    int *max_disp_index;
+    int **blocklen_per_process;
+    MPI_Aint **displs_per_process, total_bytes, bytes_per_cycle, total_bytes_written;
+    MPI_Comm comm;
+    char *global_buf, *prev_global_buf;
+    ompi_datatype_t **recvtype, **prev_recvtype;
+    struct iovec *global_iov_array;
+    int current_index, current_position;
+    int bytes_to_write_in_cycle, bytes_remaining, procs_per_group;    
+    int *procs_in_group, iov_index;
+    size_t bytes_sent, prev_bytes_sent;
+    struct iovec *decoded_iov;
+    int bytes_to_write, prev_bytes_to_write;
+    mca_common_ompio_io_array_t *io_array, *prev_io_array;
+    int num_io_entries, prev_num_io_entries;
+} mca_io_ompio_aggregator_data;
+
+
+#define SWAP_REQUESTS(_r1,_r2) { \
+    ompi_request_t **_t=_r1;     \
+    _r1=_r2;                     \
+    _r2=_t;}
+
+#define SWAP_AGGR_POINTERS(_aggr,_num) {                        \
+    int _i;                                                     \
+    char *_t;                                                   \
+    for (_i=0; _i<_num; _i++ ) {                                \
+        _aggr[_i]->prev_io_array=_aggr[_i]->io_array;             \
+        _aggr[_i]->prev_num_io_entries=_aggr[_i]->num_io_entries; \
+        _aggr[_i]->prev_bytes_sent=_aggr[_i]->bytes_sent;         \
+        _aggr[_i]->prev_bytes_to_write=_aggr[_i]->bytes_to_write; \
+        _t=_aggr[_i]->prev_global_buf;                            \
+        _aggr[_i]->prev_global_buf=_aggr[_i]->global_buf;         \
+        _aggr[_i]->global_buf=_t;                                 \
+        _t=(char *)_aggr[_i]->recvtype;                           \
+        _aggr[_i]->recvtype=_aggr[_i]->prev_recvtype;             \
+        _aggr[_i]->prev_recvtype=(ompi_datatype_t **)_t;          }                                                             \
+}
+
+int mca_fcoll_vulcan_break_file_view (struct iovec *decoded_iov, int iov_count,
+                                      struct iovec *local_iov_array, int local_count,
+                                      struct iovec ***broken_decoded_iovs, int **broken_iov_counts,
+                                      struct iovec ***broken_iov_arrays, int **broken_counts,
+                                      MPI_Aint **broken_total_lengths,
+                                      int stripe_count, size_t stripe_size);
+
+int mca_fcoll_vulcan_get_configuration (ompio_file_t *fh, int num_io_procs,
+                                        size_t max_data);
+
+int mca_fcoll_vulcan_minmax (ompio_file_t *fh, struct iovec *iov, int iov_count,
+			     int num_aggregators, long *new_stripe_size);
+
+void mca_fcoll_vulcan_calc_blocklen_disps (mca_io_ompio_aggregator_data *data, int aggregator,
+                                           int rank, size_t *bytes_comm);
+
+int mca_fcoll_vulcan_calc_file_offsets(mca_io_ompio_aggregator_data *data,
+                                       mca_io_ompio_local_io_array *file_offsets_for_agg,
+                                       int *sorted_file_offsets, MPI_Aint *memory_displacements,
+                                       int entries_per_aggregator, int rank, int index);
+
+void mca_fcoll_vulcan_calc_io_array(mca_common_ompio_io_array_t *io_array, int *num_io_entries, int max_io_arrays,
+                                    char *global_buf, mca_io_ompio_local_io_array *file_offsets_for_agg,
+                                    int *sorted_offsets, MPI_Aint *memory_displacements, int rank);
+
+END_C_DECLS
+
+#endif /* MCA_FCOLL_VULCAN_INTERNAL_H */
diff --git a/ompi/mca/fcoll/vulcan/fcoll_vulcan_module.c b/ompi/mca/fcoll/vulcan/fcoll_vulcan_module.c
index f3f847b9fcf..6f4230d919c 100644
--- a/ompi/mca/fcoll/vulcan/fcoll_vulcan_module.c
+++ b/ompi/mca/fcoll/vulcan/fcoll_vulcan_module.c
@@ -12,6 +12,8 @@
  * Copyright (c) 2008-2017 University of Houston. All rights reserved.
  * Copyright (c) 2018      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -34,7 +36,7 @@
  * ************************ actions structure ************************
  * *******************************************************************
  */
-static mca_fcoll_base_module_1_0_0_t vulcan =  {
+static mca_fcoll_base_module_2_0_0_t vulcan =  {
     mca_fcoll_vulcan_module_init,
     mca_fcoll_vulcan_module_finalize,
     mca_fcoll_vulcan_file_read_all,
@@ -54,7 +56,7 @@ mca_fcoll_vulcan_component_init_query(bool enable_progress_threads,
     return OMPI_SUCCESS;
 }
 
-mca_fcoll_base_module_1_0_0_t *
+mca_fcoll_base_module_2_0_0_t *
 mca_fcoll_vulcan_component_file_query (ompio_file_t *fh, int *priority)
 {
     *priority = mca_fcoll_vulcan_priority;
diff --git a/ompi/mca/fs/base/Makefile.am b/ompi/mca/fs/base/Makefile.am
index 215c8cdc28d..9ed1e563456 100644
--- a/ompi/mca/fs/base/Makefile.am
+++ b/ompi/mca/fs/base/Makefile.am
@@ -22,12 +22,16 @@ headers += \
 
 libmca_fs_la_SOURCES += \
         base/fs_base_frame.c \
+        base/fs_base_find_available.c
+
+if OMPI_OMPIO_SUPPORT
+libmca_fs_la_SOURCES += \
         base/fs_base_file_select.c \
         base/fs_base_file_unselect.c \
-        base/fs_base_find_available.c \
         base/fs_base_get_parent_dir.c \
         base/fs_base_file_close.c \
         base/fs_base_file_sync.c \
         base/fs_base_file_delete.c \
         base/fs_base_file_set_size.c \
         base/fs_base_file_get_size.c
+endif
diff --git a/ompi/mca/fs/base/base.h b/ompi/mca/fs/base/base.h
index de57221cf83..3cd193c62a9 100644
--- a/ompi/mca/fs/base/base.h
+++ b/ompi/mca/fs/base/base.h
@@ -37,6 +37,19 @@
 #include "ompi/mca/fs/fs.h"
 
 
+#ifdef HAVE_SYS_STATFS_H
+#include <sys/statfs.h> /* or <sys/vfs.h> */
+#endif
+#ifdef HAVE_SYS_PARAM_H
+#include <sys/param.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
 BEGIN_C_DECLS
 
 OMPI_DECLSPEC int mca_fs_base_file_select(struct ompio_file_t *file,
@@ -62,6 +75,42 @@ OMPI_DECLSPEC int mca_fs_base_file_get_size (ompio_file_t *fh, OMPI_MPI_OFFSET_T
 OMPI_DECLSPEC int mca_fs_base_file_set_size (ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE size);
 OMPI_DECLSPEC int mca_fs_base_file_close (ompio_file_t *fh);
 
+
+static inline bool mca_fs_base_is_link (const char *filename)
+{
+    int err;
+    bool ret = true;
+    struct stat statbuf;
+
+    err = lstat(filename, &statbuf);
+
+    if (err || (!S_ISLNK(statbuf.st_mode))) {
+        ret = false;
+    }
+
+    return ret;
+}
+
+static inline void mca_fs_base_get_real_filename (const char *filename, char **rfilename)
+{
+    int namelen;
+    char linkbuf[OPAL_PATH_MAX+1];
+
+    namelen = readlink(filename, linkbuf, OPAL_PATH_MAX);
+    if (namelen == -1) {
+        /* something strange has happened between the time that
+         * we determined that this was a link and the time that
+         * we attempted to read it; punt and use the old name.
+         */
+        *rfilename = strdup(filename);
+    }
+    else {
+        /* successfully read the link */
+        linkbuf[namelen] = '\0'; /* readlink doesn't null terminate */
+        *rfilename = strdup(linkbuf);
+    }
+}
+
 /*
  * Globals
  */
diff --git a/ompi/mca/fs/base/fs_base_file_get_size.c b/ompi/mca/fs/base/fs_base_file_get_size.c
index 95438b2a2d1..5548ce30d04 100644
--- a/ompi/mca/fs/base/fs_base_file_get_size.c
+++ b/ompi/mca/fs/base/fs_base_file_get_size.c
@@ -44,7 +44,7 @@ int mca_fs_base_file_get_size (ompio_file_t *fh,
         return OMPI_ERROR;
     }
 
-    if (-1 == (lseek(fh->fd, fh->f_offset, SEEK_SET))) {
+    if (-1 == (lseek(fh->fd, fh->f_fview.f_offset, SEEK_SET))) {
         perror ("lseek");
         return OMPI_ERROR;
     }
diff --git a/ompi/mca/fs/base/fs_base_file_select.c b/ompi/mca/fs/base/fs_base_file_select.c
index a91209741d9..3ac05d3980b 100644
--- a/ompi/mca/fs/base/fs_base_file_select.c
+++ b/ompi/mca/fs/base/fs_base_file_select.c
@@ -49,7 +49,7 @@ static OBJ_CLASS_INSTANCE(queried_module_t, opal_list_item_t, NULL, NULL);
 /*
  * Only one fs module can be attached to each file.
  *
- * This module calls the query funtion on all the components that were
+ * This module calls the query function on all the components that were
  * detected by fs_base_open. This function is called on a
  * per-file basis. This function has the following function.
  *
@@ -213,14 +213,14 @@ int mca_fs_base_file_select (struct ompio_file_t *file,
             * module of this component.
             *
             * ANJU: a component might not have all the functions
-            * defined.  Whereever a function pointer is null in the
+            * defined.  Wherever a function pointer is null in the
             * module structure we need to fill it in with the base
             * structure function pointers. This is yet to be done
             */
 
             /*
-             * We don return here coz we still need to go through and
-             * elease the other objects
+             * We don't return here coz we still need to go through and
+             * release the other objects
              */
 
             /*fill_null_pointers (om->om_module);*/
@@ -229,7 +229,7 @@ int mca_fs_base_file_select (struct ompio_file_t *file,
             file->f_fs_component = (mca_base_component_t *)best_component;
          } else {
             /*
-             * this is not the "choosen one", finalize
+             * this is not the "chosen one", finalize
              */
              if (NULL != om->om_component->fsm_file_unquery) {
                 /* unquery the component only if they have some clean
diff --git a/ompi/mca/fs/base/fs_base_get_parent_dir.c b/ompi/mca/fs/base/fs_base_get_parent_dir.c
index cd784c0caac..c1ac050eb2b 100644
--- a/ompi/mca/fs/base/fs_base_get_parent_dir.c
+++ b/ompi/mca/fs/base/fs_base_get_parent_dir.c
@@ -35,36 +35,25 @@
 #include "ompi/mca/fs/base/base.h"
 #include "ompi/mca/common/ompio/common_ompio.h"
 
-#ifdef HAVE_SYS_STATFS_H
-#include <sys/statfs.h> /* or <sys/vfs.h> */
-#endif
-#ifdef HAVE_SYS_PARAM_H
-#include <sys/param.h>
-#endif
+/*
+ * Be careful moving this include.
+ * It's easy to hit problems similar to that reported in
+ * https://github.com/systemd/systemd/issues/8507
+ */
 #ifdef HAVE_SYS_MOUNT_H
 #include <sys/mount.h>
 #endif
-#ifdef HAVE_SYS_STAT_H
-#include <sys/stat.h>
-#endif
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
 
 void mca_fs_base_get_parent_dir ( char *filename, char **dirnamep)
 {
-    int err;
     char *dir = NULL, *slash;
-    struct stat statbuf;
 
     if (strlen(filename) < 1) {
         opal_asprintf(dirnamep, ".%s", OPAL_PATH_SEP);
         return;
     }
 
-    err = lstat(filename, &statbuf);
-
-    if (err || (!S_ISLNK(statbuf.st_mode))) {
+    if (!mca_fs_base_is_link(filename)) {
 	/* no such file, or file is not a link; these are the "normal"
 	 * cases where we can just return the parent directory.
 	 */
@@ -76,22 +65,7 @@ void mca_fs_base_get_parent_dir ( char *filename, char **dirnamep)
 	 * but this code doesn't care if the target is really there
 	 * or not.
 	 */
-	int namelen;
-	char linkbuf[PATH_MAX+1];
-
-	namelen = readlink(filename, linkbuf, PATH_MAX);
-	if (namelen == -1) {
-	    /* something strange has happened between the time that
-	     * we determined that this was a link and the time that
-	     * we attempted to read it; punt and use the old name.
-	     */
-	    dir = strdup(filename);
-	}
-	else {
-	    /* successfully read the link */
-	    linkbuf[namelen] = '\0'; /* readlink doesn't null terminate */
-	    dir = strdup(linkbuf);
-	}
+        mca_fs_base_get_real_filename(filename, &dir);
     }
 
     slash = strrchr(dir, '/');
@@ -129,9 +103,6 @@ int  mca_fs_base_get_fstype(char *fname )
     if ( 0 == strncasecmp(fstype, "lustre", sizeof("lustre")) ) {
         ompio_type = LUSTRE;
     }
-    else if ( 0 == strncasecmp(fstype, "pvfs2", sizeof("pvfs2"))) {
-        ompio_type = PVFS2;
-    }
     else if ( 0 == strncasecmp(fstype, "ime", sizeof("ime"))) {
         ompio_type = IME;
     }
diff --git a/ompi/mca/fs/gpfs/configure.m4 b/ompi/mca/fs/gpfs/configure.m4
index a51ae2aee07..980c6700c39 100644
--- a/ompi/mca/fs/gpfs/configure.m4
+++ b/ompi/mca/fs/gpfs/configure.m4
@@ -12,6 +12,7 @@
 #                         All rights reserved.
 # Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2008-2012 University of Houston. All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # $COPYRIGHT$
 # 
 # Additional copyrights may follow
@@ -24,22 +25,22 @@
 #                        [action-if-cant-compile])
 # ------------------------------------------------
 AC_DEFUN([MCA_ompi_fs_gpfs_CONFIG],[
+    OPAL_VAR_SCOPE_PUSH([fs_gpfs_happy])
+
     AC_CONFIG_FILES([ompi/mca/fs/gpfs/Makefile])
 
     OMPI_CHECK_GPFS([fs_gpfs],
-                     [fs_gpfs_happy="yes"],
-                     [fs_gpfs_happy="no"])
+                    [fs_gpfs_happy="yes"],
+                    [fs_gpfs_happy="no"])
 
     AS_IF([test "$fs_gpfs_happy" = "yes"],
-           [fs_gpfs_WRAPPER_EXTRA_LDFLAGS="$fs_gpfs_LDFLAGS"
-           fs_gpfs_WRAPPER_EXTRA_LIBS="$fs_gpfs_LIBS"
-           $1],
-          [$2])
-
-    OPAL_SUMMARY_ADD([[OMPIO File Systems]],[[IBM Spectrum Scale/GPFS]],[$1],[$fs_gpfs_happy])
+           [$1],
+           [$2])
 
     # substitute in the things needed to build gpfs
     AC_SUBST([fs_gpfs_CPPFLAGS])
     AC_SUBST([fs_gpfs_LDFLAGS])
     AC_SUBST([fs_gpfs_LIBS])
+
+    OPAL_VAR_SCOPE_POP
 ])dnl
diff --git a/ompi/mca/fs/gpfs/fs_gpfs.c b/ompi/mca/fs/gpfs/fs_gpfs.c
index 99752c6617a..a1bf194f3e3 100644
--- a/ompi/mca/fs/gpfs/fs_gpfs.c
+++ b/ompi/mca/fs/gpfs/fs_gpfs.c
@@ -52,7 +52,7 @@
  * *******************************************************************
  */
 static mca_fs_base_module_1_0_0_t gpfs =  {
-    mca_fs_gpfs_module_init, /* initalise after being selected */
+    mca_fs_gpfs_module_init, /* initialise after being selected */
     mca_fs_gpfs_module_finalize, /* close a module on a communicator */
     mca_fs_gpfs_file_open,
     mca_fs_base_file_close,
diff --git a/ompi/mca/fs/gpfs/fs_gpfs.h b/ompi/mca/fs/gpfs/fs_gpfs.h
index ede11621962..f34de540a7b 100644
--- a/ompi/mca/fs/gpfs/fs_gpfs.h
+++ b/ompi/mca/fs/gpfs/fs_gpfs.h
@@ -40,7 +40,7 @@ int mca_fs_gpfs_component_file_unquery(ompio_file_t *file);
 
 int mca_fs_gpfs_module_init(ompio_file_t *file);
 int mca_fs_gpfs_module_finalize(ompio_file_t *file);
-OMPI_MODULE_DECLSPEC extern mca_fs_base_component_2_0_0_t mca_fs_gpfs_component;
+OMPI_DECLSPEC extern mca_fs_base_component_2_0_0_t mca_fs_gpfs_component;
 
 /*
  * ******************************************************************
diff --git a/ompi/mca/fs/gpfs/fs_gpfs_file_set_info.c b/ompi/mca/fs/gpfs/fs_gpfs_file_set_info.c
index f9e04fdc8c3..e810bcdc2ed 100644
--- a/ompi/mca/fs/gpfs/fs_gpfs_file_set_info.c
+++ b/ompi/mca/fs/gpfs/fs_gpfs_file_set_info.c
@@ -44,9 +44,9 @@ int mca_fs_gpfs_file_set_info(ompio_file_t *fh, struct ompi_info_t *info)
 {
     int rc = 0;
     int flag;
-    int valueLen = MPI_MAX_INFO_VAL;
-    char value[MPI_MAX_INFO_VAL + 1];
+    opal_cstring_t *info_str;
     char gpfsHintsKey[50];
+    bool info_bool;
     const char* split = ",";
     char* token;
     int ret = OMPI_SUCCESS;
@@ -163,9 +163,9 @@ int mca_fs_gpfs_file_set_info(ompio_file_t *fh, struct ompi_info_t *info)
     */
 
     strcpy(gpfsHintsKey, "useSIOXLib");
-    ompi_info_get(info_selected, gpfsHintsKey, valueLen, value, &flag);
+    ompi_info_get_bool(info_selected, gpfsHintsKey, &info_bool, &flag);
     if (flag) {
-        if(strcmp(value, "true") == 0) {
+        if(info_bool) {
             //using the SIOX lib and the I/O pattern selection
             ret = mca_fs_gpfs_io_selection(fh, info, info_selected);
             if (ret != OMPI_SUCCESS)
@@ -179,10 +179,10 @@ int mca_fs_gpfs_file_set_info(ompio_file_t *fh, struct ompi_info_t *info)
 
     //Setting GPFS Hint - gpfsAccessRange
     strcpy(gpfsHintsKey, "gpfsAccessRange");
-    ompi_info_get(info_selected, gpfsHintsKey, valueLen, value, &flag);
+    ompi_info_get(info_selected, gpfsHintsKey, &info_str, &flag);
     if (flag) {
         opal_output(ompi_fs_base_framework.framework_output,
-                    "GPFS Access Range is set: %s: %s\n", gpfsHintsKey, value);
+                    "GPFS Access Range is set: %s: %s\n", gpfsHintsKey, info_str->string);
         gpfs_hint_AccessRange.gpfsFcntlHeader.totalLength = sizeof(gpfs_hint_AccessRange);
         gpfs_hint_AccessRange.gpfsFcntlHeader.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION;
         gpfs_hint_AccessRange.gpfsFcntlHeader.fcntlReserved = 0;
@@ -190,12 +190,15 @@ int mca_fs_gpfs_file_set_info(ompio_file_t *fh, struct ompi_info_t *info)
         gpfs_hint_AccessRange.gpfsAccessRange.structLen =
                 sizeof(gpfs_hint_AccessRange.gpfsAccessRange);
         gpfs_hint_AccessRange.gpfsAccessRange.structType = GPFS_ACCESS_RANGE;
-        token = strtok(value, split);
+        char *info_str_dup = strdup(info_str->string);
+        OBJ_RELEASE(info_str);
+        token = strtok(info_str_dup, split);
         gpfs_hint_AccessRange.gpfsAccessRange.start = atol(token);
         token = strtok(NULL, split);
         gpfs_hint_AccessRange.gpfsAccessRange.length = atol(token);
         token = strtok(NULL, split);
         gpfs_hint_AccessRange.gpfsAccessRange.isWrite = atoi(token);
+        free(info_str_dup);
 
         rc = gpfs_fcntl(gpfs_file_handle, &gpfs_hint_AccessRange);
         if (rc != 0) {
@@ -209,10 +212,10 @@ int mca_fs_gpfs_file_set_info(ompio_file_t *fh, struct ompi_info_t *info)
 
     //Setting GPFS Hint - gpfsFreeRange
     strcpy(gpfsHintsKey, "gpfsFreeRange");
-    ompi_info_get(info_selected, gpfsHintsKey, valueLen, value, &flag);
+    ompi_info_get(info_selected, gpfsHintsKey, &info_str, &flag);
     if (flag) {
         opal_output(ompi_fs_base_framework.framework_output,
-                    "GPFS Free Range is set: %s: %s\n", gpfsHintsKey, value);
+                    "GPFS Free Range is set: %s: %s\n", gpfsHintsKey, info_str->string);
         gpfs_hint_FreeRange.gpfsFcntlHeader.totalLength = sizeof(gpfs_hint_FreeRange);
         gpfs_hint_FreeRange.gpfsFcntlHeader.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION;
         gpfs_hint_FreeRange.gpfsFcntlHeader.fcntlReserved = 0;
@@ -220,10 +223,13 @@ int mca_fs_gpfs_file_set_info(ompio_file_t *fh, struct ompi_info_t *info)
         gpfs_hint_FreeRange.gpfsFreeRange.structLen =
                 sizeof(gpfs_hint_FreeRange.gpfsFreeRange);
         gpfs_hint_FreeRange.gpfsFreeRange.structType = GPFS_FREE_RANGE;
-        token = strtok(value, split);
+        char *info_str_dup = strdup(info_str->string);
+        OBJ_RELEASE(info_str);
+        token = strtok(info_str_dup, split);
         gpfs_hint_FreeRange.gpfsFreeRange.start = atol(token);
         token = strtok(NULL, split);
         gpfs_hint_FreeRange.gpfsFreeRange.length = atol(token);
+        free(info_str_dup);
 
         rc = gpfs_fcntl(gpfs_file_handle, &gpfs_hint_FreeRange);
         if (rc != 0) {
@@ -241,10 +247,10 @@ int mca_fs_gpfs_file_set_info(ompio_file_t *fh, struct ompi_info_t *info)
 
     //Setting GPFS Hint - gpfsClearFileCache
     strcpy(gpfsHintsKey, "gpfsClearFileCache");
-    ompi_info_get(info_selected, gpfsHintsKey, valueLen, value, &flag);
-    if (flag & (strcmp(value, "true") == 0)) {
+    ompi_info_get_bool(info_selected, gpfsHintsKey, &info_bool, &flag);
+    if (flag && info_bool) {
         opal_output(ompi_fs_base_framework.framework_output,
-                    "GPFS Clear File Cache is set: %s: %s\n", gpfsHintsKey, value);
+                    "GPFS Clear File Cache is set: %s\n", gpfsHintsKey);
         gpfs_hint_ClearFileCache.gpfsFcntlHeader.totalLength = sizeof(gpfs_hint_ClearFileCache);
         gpfs_hint_ClearFileCache.gpfsFcntlHeader.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION;
         gpfs_hint_ClearFileCache.gpfsFcntlHeader.fcntlReserved = 0;
@@ -265,10 +271,10 @@ int mca_fs_gpfs_file_set_info(ompio_file_t *fh, struct ompi_info_t *info)
 
     //Setting GPFS Hint - gpfsCancelHints
     strcpy(gpfsHintsKey, "gpfsCancelHints");
-    ompi_info_get(info_selected, gpfsHintsKey, valueLen, value, &flag);
-    if (flag & (strcmp(value, "true") == 0)) {
+    ompi_info_get_bool(info_selected, gpfsHintsKey, &info_bool, &flag);
+    if (flag && info_bool) {
         opal_output(ompi_fs_base_framework.framework_output,
-                    "GPFS Cancel Hints is set: %s: %s\n", gpfsHintsKey, value);
+                    "GPFS Cancel Hints is set: %s\n", gpfsHintsKey);
         gpfs_hint_CancelHints.gpfsFcntlHeader.totalLength = sizeof(gpfs_hint_CancelHints);
         gpfs_hint_CancelHints.gpfsFcntlHeader.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION;
         gpfs_hint_CancelHints.gpfsFcntlHeader.fcntlReserved = 0;
@@ -289,10 +295,10 @@ int mca_fs_gpfs_file_set_info(ompio_file_t *fh, struct ompi_info_t *info)
 
     //Setting GPFS Hint - gpfsSetReplication
     strcpy(gpfsHintsKey, "gpfsSetReplication");
-    ompi_info_get(info_selected, gpfsHintsKey, valueLen, value, &flag);
+    ompi_info_get(info_selected, gpfsHintsKey, &info_str, &flag);
     if (flag) {
         opal_output(ompi_fs_base_framework.framework_output,
-                    "GPFS Set Replication is set: %s: %s\n", gpfsHintsKey, value);
+                    "GPFS Set Replication is set: %s: %s\n", gpfsHintsKey, info_str->string);
         gpfs_hint_SetReplication.gpfsFcntlHeader.totalLength = sizeof(gpfs_hint_SetReplication);
         gpfs_hint_SetReplication.gpfsFcntlHeader.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION;
         gpfs_hint_SetReplication.gpfsFcntlHeader.fcntlReserved = 0;
@@ -300,12 +306,15 @@ int mca_fs_gpfs_file_set_info(ompio_file_t *fh, struct ompi_info_t *info)
         gpfs_hint_SetReplication.gpfsSetReplication.structLen =
                 sizeof(gpfs_hint_SetReplication.gpfsSetReplication);
         gpfs_hint_SetReplication.gpfsSetReplication.structType = GPFS_FCNTL_SET_REPLICATION;
-        token = strtok(value, split);
+        char *info_str_dup = strdup(info_str->string);
+        OBJ_RELEASE(info_str);
+        token = strtok(info_str_dup, split);
         gpfs_hint_SetReplication.gpfsSetReplication.metadataReplicas = atoi(token);
         gpfs_hint_SetReplication.gpfsSetReplication.maxMetadataReplicas = atoi(token);
         gpfs_hint_SetReplication.gpfsSetReplication.dataReplicas = atoi(token);
         gpfs_hint_SetReplication.gpfsSetReplication.maxDataReplicas = atoi(token);
         gpfs_hint_SetReplication.gpfsSetReplication.reserved = 0;
+        free(info_str_dup);
 
         rc = gpfs_fcntl(gpfs_file_handle, &gpfs_hint_SetReplication);
         if (rc != 0) {
@@ -322,18 +331,21 @@ int mca_fs_gpfs_file_set_info(ompio_file_t *fh, struct ompi_info_t *info)
 
     //Setting GPFS Hint - gpfsByteRange
     strcpy(gpfsHintsKey, "gpfsByteRange");
-    ompi_info_get(info_selected, gpfsHintsKey, valueLen, value, &flag);
+    ompi_info_get(info_selected, gpfsHintsKey, &info_str, &flag);
     if (flag) {
         opal_output(ompi_fs_base_framework.framework_output,
-                    "GPFS Byte Range is set: %s: %s\n", gpfsHintsKey, value);
+                    "GPFS Byte Range is set: %s: %s\n", gpfsHintsKey, info_str->string);
         gpfs_hint_ByteRange.gpfsFcntlHeader.totalLength = sizeof(gpfs_hint_ByteRange);
         gpfs_hint_ByteRange.gpfsFcntlHeader.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION;
         gpfs_hint_ByteRange.gpfsFcntlHeader.fcntlReserved = 0;
 
-        token = strtok(value, split);
+        char *info_str_dup = strdup(info_str->string);
+        OBJ_RELEASE(info_str);
+        token = strtok(info_str_dup, split);
         gpfs_hint_ByteRange.gpfsByteRange.startOffset = atol(token);
-        token = strtok(value, split);
+        token = strtok(NULL, split);
         gpfs_hint_ByteRange.gpfsByteRange.numOfBlks = atol(token);
+        free(info_str_dup);
 
         rc = gpfs_fcntl(gpfs_file_handle, &gpfs_hint_ByteRange);
         if (rc != 0) {
@@ -347,10 +359,10 @@ int mca_fs_gpfs_file_set_info(ompio_file_t *fh, struct ompi_info_t *info)
 
     //Setting GPFS Hint - gpfsRestripeData
     strcpy(gpfsHintsKey, "gpfsRestripeData");
-    ompi_info_get(info_selected, gpfsHintsKey, valueLen, value, &flag);
+    ompi_info_get(info_selected, gpfsHintsKey, &info_str, &flag);
     if (flag) {
         opal_output(ompi_fs_base_framework.framework_output,
-                    "GPFS Restripe Data is set: %s: %s\n", gpfsHintsKey, value);
+                    "GPFS Restripe Data is set: %s: %s\n", gpfsHintsKey, info_str->string);
         gpfs_hint_RestripeData.gpfsFcntlHeader.totalLength = sizeof(gpfs_hint_RestripeData);
         gpfs_hint_RestripeData.gpfsFcntlHeader.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION;
         gpfs_hint_RestripeData.gpfsFcntlHeader.fcntlReserved = 0;
@@ -358,10 +370,13 @@ int mca_fs_gpfs_file_set_info(ompio_file_t *fh, struct ompi_info_t *info)
         gpfs_hint_RestripeData.gpfsRestripeData.structLen =
                 sizeof(gpfs_hint_RestripeData.gpfsRestripeData);
         gpfs_hint_RestripeData.gpfsRestripeData.structType = GPFS_FCNTL_RESTRIPE_DATA;
-        token = strtok(value, split);
+        char *info_str_dup = strdup(info_str->string);
+        OBJ_RELEASE(info_str);
+        token = strtok(info_str_dup, split);
         gpfs_hint_RestripeData.gpfsRestripeData.options = atoi(token);
         gpfs_hint_RestripeData.gpfsRestripeData.reserved1 = 0;
         gpfs_hint_RestripeData.gpfsRestripeData.reserved2 = 0;
+        free(info_str_dup);
 
         rc = gpfs_fcntl(gpfs_file_handle, &gpfs_hint_RestripeData);
         if (rc != 0) {
@@ -446,7 +461,7 @@ int mca_fs_gpfs_io_selection(ompio_file_t *fh,
                 siox_attribute_array[i], fh->f_filename);
     i++;
 
-//CN: Code duplication en mass (9 times same code block wich changing key!)
+//CN: Code duplication en mass (9 times same code block with changing key!)
 //CN: do this with a loop over a list of sioxHintsKeys
     //START setting the siox activity attributes
     strcpy(sioxHintsKey, "sioxAccessRange");
diff --git a/ompi/mca/fs/ime/configure.m4 b/ompi/mca/fs/ime/configure.m4
index 44bdce422ba..c28744ab954 100644
--- a/ompi/mca/fs/ime/configure.m4
+++ b/ompi/mca/fs/ime/configure.m4
@@ -1,6 +1,7 @@
 # -*- shell-script -*-
 #
 # Copyright (c) 2018      DataDirect Networks. All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -18,7 +19,7 @@ AC_DEFUN([MCA_ompi_fs_ime_CONFIG],[
                     [fs_ime_happy="yes"],
                     [fs_ime_happy="no"])
 
-    OPAL_SUMMARY_ADD([[OMPIO File Systems]],[[DDN Infinite Memory Engine]],[$1],[$fs_ime_happy])
+    OPAL_SUMMARY_ADD([OMPIO File Systems], [DDN Infinite Memory Engine], [], [$fs_ime_happy])
     AS_IF([test "$fs_ime_happy" = "yes"],
           [$1],
           [$2])
diff --git a/ompi/mca/fs/ime/fs_ime.c b/ompi/mca/fs/ime/fs_ime.c
index 1308652b144..df21237ef6e 100644
--- a/ompi/mca/fs/ime/fs_ime.c
+++ b/ompi/mca/fs/ime/fs_ime.c
@@ -21,7 +21,7 @@
  * *******************************************************************
  */
 static mca_fs_base_module_1_0_0_t ime =  {
-    mca_fs_ime_module_init, /* initalise after being selected */
+    mca_fs_ime_module_init, /* initialise after being selected */
     mca_fs_ime_module_finalize, /* close a module on a communicator */
     mca_fs_ime_file_open,
     mca_fs_ime_file_close,
diff --git a/ompi/mca/fs/ime/fs_ime.h b/ompi/mca/fs/ime/fs_ime.h
index 170f03a06e2..77a94baa9ed 100644
--- a/ompi/mca/fs/ime/fs_ime.h
+++ b/ompi/mca/fs/ime/fs_ime.h
@@ -1,5 +1,7 @@
 /*
  * Copyright (c) 2018      DataDirect Networks. All rights reserved.
+ * Copyright (c) 2021      Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -33,9 +35,9 @@ int mca_fs_ime_component_file_unquery (ompio_file_t *file);
 int mca_fs_ime_module_init (ompio_file_t *file);
 int mca_fs_ime_module_finalize (ompio_file_t *file);
 
-int mca_fs_ime_native_fini();
+int mca_fs_ime_native_fini(void);
 
-OMPI_MODULE_DECLSPEC extern mca_fs_base_component_2_0_0_t mca_fs_ime_component;
+OMPI_DECLSPEC extern mca_fs_base_component_2_0_0_t mca_fs_ime_component;
 /*
  * ******************************************************************
  * ********* functions which are implemented in this module *********
diff --git a/ompi/mca/fs/ime/fs_ime_component.c b/ompi/mca/fs/ime/fs_ime_component.c
index ea4e06de96a..5b6febff964 100644
--- a/ompi/mca/fs/ime/fs_ime_component.c
+++ b/ompi/mca/fs/ime/fs_ime_component.c
@@ -1,6 +1,8 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2018      DataDirect Networks. All rights reserved.
+ * Copyright (c) 2021      Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -44,6 +46,7 @@ mca_fs_base_component_2_0_0_t mca_fs_ime_component = {
         MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION,
                               OMPI_RELEASE_VERSION),
         .mca_register_component_params = register_component,
+        .mca_close_component = mca_fs_ime_native_fini,
     },
     .fsm_data = {
         /* This component is checkpointable */
diff --git a/ompi/mca/fs/lustre/configure.m4 b/ompi/mca/fs/lustre/configure.m4
index 9e64c6b5351..7b4c647820d 100644
--- a/ompi/mca/fs/lustre/configure.m4
+++ b/ompi/mca/fs/lustre/configure.m4
@@ -12,6 +12,7 @@
 #                         All rights reserved.
 # Copyright (c) 2010-2017 Cisco Systems, Inc.  All rights reserved
 # Copyright (c) 2008-2012 University of Houston. All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -24,6 +25,8 @@
 #                        [action-if-cant-compile])
 # ------------------------------------------------
 AC_DEFUN([MCA_ompi_fs_lustre_CONFIG],[
+    OPAL_VAR_SCOPE_PUSH([fs_lustre_happy])
+
     AC_CONFIG_FILES([ompi/mca/fs/lustre/Makefile])
 
     OMPI_CHECK_LUSTRE([fs_lustre],
@@ -34,10 +37,10 @@ AC_DEFUN([MCA_ompi_fs_lustre_CONFIG],[
           [$1],
           [$2])
 
-    OPAL_SUMMARY_ADD([[OMPIO File Systems]],[[Lustre]],[$1],[$fs_lustre_happy])
-
     # substitute in the things needed to build lustre
     AC_SUBST([fs_lustre_CPPFLAGS])
     AC_SUBST([fs_lustre_LDFLAGS])
     AC_SUBST([fs_lustre_LIBS])
+
+    OPAL_VAR_SCOPE_POP
 ])dnl
diff --git a/ompi/mca/fs/lustre/fs_lustre.c b/ompi/mca/fs/lustre/fs_lustre.c
index 20fdfc76991..494114416b4 100644
--- a/ompi/mca/fs/lustre/fs_lustre.c
+++ b/ompi/mca/fs/lustre/fs_lustre.c
@@ -38,7 +38,7 @@
  * *******************************************************************
  */
 static mca_fs_base_module_1_0_0_t lustre =  {
-    mca_fs_lustre_module_init, /* initalise after being selected */
+    mca_fs_lustre_module_init, /* initialise after being selected */
     mca_fs_lustre_module_finalize, /* close a module on a communicator */
     mca_fs_lustre_file_open,
     mca_fs_base_file_close,
diff --git a/ompi/mca/fs/lustre/fs_lustre.h b/ompi/mca/fs/lustre/fs_lustre.h
index 6ff2fac47f0..ebd9da168b7 100644
--- a/ompi/mca/fs/lustre/fs_lustre.h
+++ b/ompi/mca/fs/lustre/fs_lustre.h
@@ -13,6 +13,7 @@
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Advanced Micro Devices, Inc. All rights reserverd.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -31,6 +32,12 @@
 extern int mca_fs_lustre_priority;
 extern int mca_fs_lustre_stripe_size;
 extern int mca_fs_lustre_stripe_width;
+extern int mca_fs_lustre_lock_algorithm;
+
+#define FS_LUSTRE_LOCK_AUTO        0
+#define FS_LUSTRE_LOCK_NEVER       1
+#define FS_LUSTRE_LOCK_ENTIRE_FILE 2
+#define FS_LUSTRE_LOCK_RANGES      3
 
 BEGIN_C_DECLS
 
@@ -51,7 +58,7 @@ int mca_fs_lustre_component_file_unquery (ompio_file_t *file);
 int mca_fs_lustre_module_init (ompio_file_t *file);
 int mca_fs_lustre_module_finalize (ompio_file_t *file);
 
-OMPI_MODULE_DECLSPEC extern mca_fs_base_component_2_0_0_t mca_fs_lustre_component;
+OMPI_DECLSPEC extern mca_fs_base_component_2_0_0_t mca_fs_lustre_component;
 /*
  * ******************************************************************
  * ********* functions which are implemented in this module *********
diff --git a/ompi/mca/fs/lustre/fs_lustre_component.c b/ompi/mca/fs/lustre/fs_lustre_component.c
index d8392af482d..c3316654e4e 100644
--- a/ompi/mca/fs/lustre/fs_lustre_component.c
+++ b/ompi/mca/fs/lustre/fs_lustre_component.c
@@ -13,6 +13,7 @@
  * Copyright (c) 2008-2011 University of Houston. All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2024      Advanced Micro Devices, Inc. All rights reserverd.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -45,6 +46,16 @@ int mca_fs_lustre_priority = 20;
    runtime also*/
 int mca_fs_lustre_stripe_size = 0;
 int mca_fs_lustre_stripe_width = 0;
+int mca_fs_lustre_lock_algorithm = 0; /* auto */
+
+static const mca_base_var_enum_value_t ompi_fs_lustre_lock_algorithm_modes[] = {
+    {.value = 0, .string = "auto"},
+    {.value = 1, .string = "skip locking"},
+    {.value = 2, .string = "always lock entire file"},
+    {.value = 3, .string = "lock specific ranges"},
+    {.string = NULL},
+};
+
 /*
  * Instantiate the public struct with all of our public information
  * and pointers to our public functions in it
@@ -75,6 +86,8 @@ mca_fs_base_component_2_0_0_t mca_fs_lustre_component = {
 static int
 lustre_register(void)
 {
+    mca_base_var_enum_t *new_enum;
+
     mca_fs_lustre_priority = 20;
     (void) mca_base_component_var_register(&mca_fs_lustre_component.fsm_version,
                                            "priority", "Priority of the lustre fs component",
@@ -94,5 +107,17 @@ lustre_register(void)
                                            OPAL_INFO_LVL_9,
                                            MCA_BASE_VAR_SCOPE_READONLY, &mca_fs_lustre_stripe_width);
 
+    (void) mca_base_var_enum_create("mca_fs_lustre_lock_algorithm", ompi_fs_lustre_lock_algorithm_modes, &new_enum);
+
+    mca_fs_lustre_lock_algorithm = 0;
+    (void) mca_base_component_var_register(&mca_fs_lustre_component.fsm_version,
+                                           "lock_algorithm", "Locking algorithm used by the fs lustre component. "
+                                           "(default: auto)",
+                                           MCA_BASE_VAR_TYPE_INT, new_enum, 0, 0,
+                                           OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           &mca_fs_lustre_lock_algorithm);
+    OBJ_RELEASE(new_enum);
+
     return OMPI_SUCCESS;
 }
diff --git a/ompi/mca/fs/lustre/fs_lustre_file_open.c b/ompi/mca/fs/lustre/fs_lustre_file_open.c
index 3a0e1c049b7..9318825987e 100644
--- a/ompi/mca/fs/lustre/fs_lustre_file_open.c
+++ b/ompi/mca/fs/lustre/fs_lustre_file_open.c
@@ -13,6 +13,7 @@
  * Copyright (c) 2015-2020 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Advanced Micro Devices, Inc. All rights reserverd.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -69,23 +70,41 @@ mca_fs_lustre_file_open (struct ompi_communicator_t *comm,
     int flag;
     int fs_lustre_stripe_size = -1;
     int fs_lustre_stripe_width = -1;
-    char char_stripe[MPI_MAX_INFO_KEY];
-
+    opal_cstring_t *stripe_str;
+    char *rfilename = (char *)filename;
     struct lov_user_md *lump=NULL;
 
     perm = mca_fs_base_get_file_perm(fh);
     amode = mca_fs_base_get_file_amode(fh->f_rank, access_mode);
 
-    opal_info_get (info, "stripe_size", MPI_MAX_INFO_VAL, char_stripe, &flag);
+    opal_info_get (info, "striping_unit", &stripe_str, &flag);
     if ( flag ) {
-        sscanf ( char_stripe, "%d", &fs_lustre_stripe_size );
+        sscanf ( stripe_str->string, "%d", &fs_lustre_stripe_size );
+        OBJ_RELEASE(stripe_str);
     }
-
-    opal_info_get (info, "stripe_width", MPI_MAX_INFO_VAL, char_stripe, &flag);
+    else {
+        //internal info object name used earlier. Kept for backwards compatibility.
+        opal_info_get (info, "stripe_size", &stripe_str, &flag);
+        if ( flag ) {
+            sscanf ( stripe_str->string, "%d", &fs_lustre_stripe_size );
+            OBJ_RELEASE(stripe_str);
+        }
+    }
+    
+    opal_info_get (info, "striping_factor", &stripe_str, &flag);
     if ( flag ) {
-        sscanf ( char_stripe, "%d", &fs_lustre_stripe_width );
+        sscanf ( stripe_str->string, "%d", &fs_lustre_stripe_width );
+        OBJ_RELEASE(stripe_str);
     }
-
+    else {
+        //internal info object name used earlier. Kept for backwards compatibility.        
+        opal_info_get (info, "stripe_width", &stripe_str, &flag);
+        if ( flag ) {
+            sscanf ( stripe_str->string, "%d", &fs_lustre_stripe_width );
+            OBJ_RELEASE(stripe_str);
+        }
+    }
+    
     if (fs_lustre_stripe_size < 0) {
         fs_lustre_stripe_size = mca_fs_lustre_stripe_size;
     }   
@@ -94,6 +113,16 @@ mca_fs_lustre_file_open (struct ompi_communicator_t *comm,
         fs_lustre_stripe_width = mca_fs_lustre_stripe_width;
     }
 
+    /* Check for soft links and replace filename by the actual
+       file used in case it is a soft link */
+    if (mca_fs_base_is_link(filename)) {
+        mca_fs_base_get_real_filename(filename, &rfilename);
+        /* make sure the real file is also on a Lustre file system */
+        if (LUSTRE != mca_fs_base_get_fstype(rfilename)) {
+            opal_output(1, "cannot use a soft-link between a LUSTRE and non-LUSTRE file system\n");
+            return OPAL_ERROR;
+        }
+    }
     
     /* Reset errno */
     errno = 0;
@@ -101,6 +130,8 @@ mca_fs_lustre_file_open (struct ompi_communicator_t *comm,
         if ( (fs_lustre_stripe_size>0 || fs_lustre_stripe_width>0) &&
              ( amode&O_CREAT)                                      && 
              ( (amode&O_RDWR)|| amode&O_WRONLY) ) {
+            /* this cannot be a soft-link since we are creating the file.
+               Not using rfilename here */
             llapi_file_create(filename,
                               fs_lustre_stripe_size,
                               -1, /* MSC need to change that */
@@ -118,7 +149,7 @@ mca_fs_lustre_file_open (struct ompi_communicator_t *comm,
         }
     }
 
-   comm->c_coll->coll_bcast ( &ret, 1, MPI_INT, 0, comm, comm->c_coll->coll_bcast_module);
+    comm->c_coll->coll_bcast ( &ret, 1, MPI_INT, 0, comm, comm->c_coll->coll_bcast_module);
     if ( OMPI_SUCCESS != ret ) {
         fh->fd = -1;
         return ret;
@@ -132,19 +163,35 @@ mca_fs_lustre_file_open (struct ompi_communicator_t *comm,
     }
 
     lump = alloc_lum();
-    if (NULL == lump ){
+    if (NULL == lump) {
         fprintf(stderr,"Cannot allocate memory for extracting stripe size\n");
         return OMPI_ERROR;
     }
-    rc = llapi_file_get_stripe(filename, lump);
+    rc = llapi_file_get_stripe(rfilename, lump);
     if (rc != 0) {
         opal_output(1, "get_stripe failed: %d (%s)\n", errno, strerror(errno));
+        free(lump);
         return OMPI_ERROR;
     }
     fh->f_stripe_size   = lump->lmm_stripe_size;
     fh->f_stripe_count  = lump->lmm_stripe_count;
     fh->f_fs_block_size = lump->lmm_stripe_size;
-    fh->f_flags |= OMPIO_LOCK_NEVER;
-    
+    free(lump);
+
+    if (FS_LUSTRE_LOCK_AUTO == mca_fs_lustre_lock_algorithm ||
+        FS_LUSTRE_LOCK_NEVER == mca_fs_lustre_lock_algorithm ) {
+        fh->f_flags |= OMPIO_LOCK_NEVER;
+    }
+    else if (FS_LUSTRE_LOCK_ENTIRE_FILE == mca_fs_lustre_lock_algorithm) {
+        fh->f_flags |= OMPIO_LOCK_ENTIRE_FILE;
+    }
+    else if (FS_LUSTRE_LOCK_RANGES == mca_fs_lustre_lock_algorithm) {
+        /* Nothing to be done. This is what the posix fbtl component would do
+           anyway without additional information . */
+    }
+    else {
+        opal_output ( 1, "Invalid value for mca_fs_lustre_lock_algorithm %d", mca_fs_lustre_lock_algorithm );
+    }
+
     return OMPI_SUCCESS;
 }
diff --git a/ompi/mca/fs/pvfs2/Makefile.am b/ompi/mca/fs/pvfs2/Makefile.am
deleted file mode 100644
index 64c147493ac..00000000000
--- a/ompi/mca/fs/pvfs2/Makefile.am
+++ /dev/null
@@ -1,58 +0,0 @@
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2008-2011 University of Houston. All rights reserved.
-# Copyright (c) 2017      IBM Corporation.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-# Make the output library in this directory, and name it either
-# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
-# (for static builds).
-
-if MCA_BUILD_ompi_fs_pvfs2_DSO
-component_noinst =
-component_install = mca_fs_pvfs2.la
-else
-component_noinst = libmca_fs_pvfs2.la
-component_install =
-endif
-
-# Source files
-
-fs_pvfs2_sources = \
-        fs_pvfs2.h \
-        fs_pvfs2.c \
-        fs_pvfs2_component.c \
-        fs_pvfs2_file_open.c \
-	fs_pvfs2_file_close.c \
-	fs_pvfs2_file_delete.c \
-	fs_pvfs2_file_sync.c \
-	fs_pvfs2_file_set_size.c \
-        fs_pvfs2_file_get_size.c
-
-AM_CPPFLAGS = $(fs_pvfs2_CPPFLAGS)
-
-mcacomponentdir = $(ompilibdir)
-mcacomponent_LTLIBRARIES = $(component_install)
-mca_fs_pvfs2_la_SOURCES = $(fs_pvfs2_sources)
-mca_fs_pvfs2_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
-	$(fs_pvfs2_LIBS)
-mca_fs_pvfs2_la_LDFLAGS = -module -avoid-version $(fs_pvfs2_LDFLAGS)
-
-noinst_LTLIBRARIES = $(component_noinst)
-libmca_fs_pvfs2_la_SOURCES = $(fs_pvfs2_sources)
-libmca_fs_pvfs2_la_LIBADD =  $(fs_pvfs2_LIBS)
-libmca_fs_pvfs2_la_LDFLAGS = -module -avoid-version $(fs_pvfs2_LDFLAGS)
diff --git a/ompi/mca/fs/pvfs2/configure.m4 b/ompi/mca/fs/pvfs2/configure.m4
deleted file mode 100644
index 0f404ea0319..00000000000
--- a/ompi/mca/fs/pvfs2/configure.m4
+++ /dev/null
@@ -1,48 +0,0 @@
-# -*- shell-script -*-
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2008-2012 University of Houston. All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-
-# MCA_fs_pvfs2_CONFIG(action-if-can-compile,
-#                        [action-if-cant-compile])
-# ------------------------------------------------
-AC_DEFUN([MCA_ompi_fs_pvfs2_CONFIG],[
-    AC_CONFIG_FILES([ompi/mca/fs/pvfs2/Makefile])
-
-    OMPI_CHECK_PVFS2([fs_pvfs2],
-                     [fs_pvfs2_happy="yes"],
-                     [fs_pvfs2_happy="no"])
-
-    OPAL_SUMMARY_ADD([[OMPIO File Systems]],[[PVFS2/OrangeFS]],[$1],[$fs_pvfs2_happy])
-    AS_IF([test "$fs_pvfs2_happy" = "yes"],
-          [$1],
-          [$2])
-
-#    AC_CHECK_HEADERS([pvfs2.h], [],
-#                      [AC_CHECK_HEADERS([pvfs2.h], [], [$2],
-#                          [AC_INCLUDES_DEFAULT])],
-#                      [AC_INCLUDES_DEFAULT])
-
-
-    # substitute in the things needed to build pvfs2
-    AC_SUBST([fs_pvfs2_CPPFLAGS])
-    AC_SUBST([fs_pvfs2_LDFLAGS])
-    AC_SUBST([fs_pvfs2_LIBS])
-])dnl
diff --git a/ompi/mca/fs/pvfs2/fs_pvfs2.c b/ompi/mca/fs/pvfs2/fs_pvfs2.c
deleted file mode 100644
index 89712cf0bc4..00000000000
--- a/ompi/mca/fs/pvfs2/fs_pvfs2.c
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008-2016 University of Houston. All rights reserved.
- * Copyright (c) 2018      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2018      DataDirect Networks. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- *
- * These symbols are in a file by themselves to provide nice linker
- * semantics. Since linkers generally pull in symbols by object fules,
- * keeping these symbols as the only symbols in this file prevents
- * utility programs such as "ompi_info" from having to import entire
- * modules just to query their version and parameters
- */
-
-
-#include "ompi_config.h"
-#include "mpi.h"
-#include "ompi/mca/fs/fs.h"
-#include "ompi/mca/fs/base/base.h"
-#include "ompi/mca/fs/pvfs2/fs_pvfs2.h"
-
-#ifdef HAVE_SYS_STATFS_H
-#include <sys/statfs.h> /* or <sys/vfs.h> */
-#endif
-#ifdef HAVE_SYS_PARAM_H
-#include <sys/param.h>
-#endif
-#ifdef HAVE_SYS_MOUNT_H
-#include <sys/mount.h>
-#endif
-#ifdef HAVE_SYS_STAT_H
-#include <sys/stat.h>
-#endif
-
-/*
- * *******************************************************************
- * ************************ actions structure ************************
- * *******************************************************************
- */
-static mca_fs_base_module_1_0_0_t pvfs2 =  {
-    mca_fs_pvfs2_module_init, /* initalise after being selected */
-    mca_fs_pvfs2_module_finalize, /* close a module on a communicator */
-    mca_fs_pvfs2_file_open,
-    mca_fs_pvfs2_file_close,
-    mca_fs_pvfs2_file_delete,
-    mca_fs_pvfs2_file_set_size,
-    mca_fs_pvfs2_file_get_size,
-    mca_fs_pvfs2_file_sync
-};
-/*
- * *******************************************************************
- * ************************* structure ends **************************
- * *******************************************************************
- */
-
-int mca_fs_pvfs2_component_init_query(bool enable_progress_threads,
-                                      bool enable_mpi_threads)
-{
-    /* Nothing to do */
-
-   return OMPI_SUCCESS;
-}
-
-struct mca_fs_base_module_1_0_0_t *
-mca_fs_pvfs2_component_file_query (ompio_file_t *fh, int *priority)
-{
-
-    /* The code in this function is based on the ADIO FS selection in ROMIO
-     *   Copyright (C) 1997 University of Chicago.
-     *   See COPYRIGHT notice in top-level directory.
-     */
-
-    char *tmp;
-
-    /* The code in this function is based on the ADIO FS selection in ROMIO
-     *   Copyright (C) 1997 University of Chicago.
-     *   See COPYRIGHT notice in top-level directory.
-     */
-
-   *priority = mca_fs_pvfs2_priority;
-
-    tmp = strchr (fh->f_filename, ':');
-    if (!tmp) {
-        /* The communicator might be NULL if we only want to delete the file */
-        if (OMPIO_ROOT == fh->f_rank || MPI_COMM_NULL == fh->f_comm) {
-            fh->f_fstype = mca_fs_base_get_fstype ( fh->f_filename );
-	    }
-        if (fh->f_comm != MPI_COMM_NULL) {
-            fh->f_comm->c_coll->coll_bcast (&(fh->f_fstype),
-                        1,
-                        MPI_INT,
-                        OMPIO_ROOT,
-                        fh->f_comm,
-                        fh->f_comm->c_coll->coll_bcast_module);
-        }
-    }
-    else {
-        if (!strncmp(fh->f_filename, "pvfs2:", 6) ||
-            !strncmp(fh->f_filename, "PVFS2:", 6)) {
-            fh->f_fstype = PVFS2;
-        }
-    }
-
-   if (PVFS2 == fh->f_fstype) {
-       if (*priority < 50) {
-           *priority = 50;
-	   return &pvfs2;
-       }
-   }
-
-   return NULL;
-}
-
-int mca_fs_pvfs2_component_file_unquery (ompio_file_t *file)
-{
-   /* This function might be needed for some purposes later. for now it
-    * does not have anything to do since there are no steps which need
-    * to be undone if this module is not selected */
-
-   return OMPI_SUCCESS;
-}
-
-int mca_fs_pvfs2_module_init (ompio_file_t *file)
-{
-    /* Make sure the file type is not overwritten by the last queried
-	 * component */
-    file->f_fstype = PVFS2;
-    return OMPI_SUCCESS;
-}
-
-
-int mca_fs_pvfs2_module_finalize (ompio_file_t *file)
-{
-    return OMPI_SUCCESS;
-}
diff --git a/ompi/mca/fs/pvfs2/fs_pvfs2.h b/ompi/mca/fs/pvfs2/fs_pvfs2.h
deleted file mode 100644
index f879d7f992d..00000000000
--- a/ompi/mca/fs/pvfs2/fs_pvfs2.h
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2006 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008-2016 University of Houston. All rights reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef MCA_FS_PVFS2_H
-#define MCA_FS_PVFS2_H
-/*
-#ifdef HAVE_PVFS2_H
-#include "pvfs2.h"
-#endif
-
-#ifdef PVFS2_VERSION_MAJOR
-#include "pvfs2-compat.h"
-#endif
-*/
-#include "pvfs2.h"
-#include "pvfs2-compat.h"
-
-#include "ompi_config.h"
-#include "ompi/mca/mca.h"
-#include "ompi/mca/fs/fs.h"
-#include "ompi/mca/common/ompio/common_ompio.h"
-
-extern int mca_fs_pvfs2_priority;
-extern int mca_fs_pvfs2_stripe_size;
-extern int mca_fs_pvfs2_stripe_width;
-extern int mca_fs_pvfs2_IS_INITIALIZED;
-
-BEGIN_C_DECLS
-
-#define OMPIO_PVFS2_MAX_NAME 100
-
-struct mca_fs_pvfs2_s {
-    PVFS_credentials credentials;
-    PVFS_object_ref object_ref;
-};
-typedef struct mca_fs_pvfs2_s mca_fs_pvfs2;
-
-int mca_fs_pvfs2_component_init_query(bool enable_progress_threads,
-                                        bool enable_mpi_threads);
-struct mca_fs_base_module_1_0_0_t *
-mca_fs_pvfs2_component_file_query (ompio_file_t *fh, int *priority);
-int mca_fs_pvfs2_component_file_unquery (ompio_file_t *file);
-
-int mca_fs_pvfs2_module_init (ompio_file_t *file);
-int mca_fs_pvfs2_module_finalize (ompio_file_t *file);
-
-OMPI_MODULE_DECLSPEC extern mca_fs_base_component_2_0_0_t mca_fs_pvfs2_component;
-/*
- * ******************************************************************
- * ********* functions which are implemented in this module *********
- * ******************************************************************
- */
-
-int mca_fs_pvfs2_file_open (struct ompi_communicator_t *comm,
-                            const char *filename,
-                            int amode,
-                            struct opal_info_t *info,
-                            ompio_file_t *fh);
-
-int mca_fs_pvfs2_file_close (ompio_file_t *fh);
-
-int mca_fs_pvfs2_file_delete (char *filename,
-                              struct opal_info_t *info);
-
-int mca_fs_pvfs2_file_set_size (ompio_file_t *fh,
-                                OMPI_MPI_OFFSET_TYPE size);
-
-int mca_fs_pvfs2_file_get_size (ompio_file_t *fh,
-                                OMPI_MPI_OFFSET_TYPE *size);
-
-int mca_fs_pvfs2_file_sync (ompio_file_t *fh);
-
-/*
- * ******************************************************************
- * ************ functions implemented in this module end ************
- * ******************************************************************
- */
-
-END_C_DECLS
-
-#endif /* MCA_FS_PVFS2_H */
diff --git a/ompi/mca/fs/pvfs2/fs_pvfs2_component.c b/ompi/mca/fs/pvfs2/fs_pvfs2_component.c
deleted file mode 100644
index ef9bf933cc2..00000000000
--- a/ompi/mca/fs/pvfs2/fs_pvfs2_component.c
+++ /dev/null
@@ -1,98 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008-2013 University of Houston. All rights reserved.
- * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2016      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- *
- * These symbols are in a file by themselves to provide nice linker
- * semantics.  Since linkers generally pull in symbols by object
- * files, keeping these symbols as the only symbols in this file
- * prevents utility programs such as "ompi_info" from having to import
- * entire components just to query their version and parameters.
- */
-
-#include "ompi_config.h"
-#include "fs_pvfs2.h"
-#include "mpi.h"
-
-/*
- * Public string showing the fs pvfs2 component version number
- */
-const char *mca_fs_pvfs2_component_version_string =
-  "OMPI/MPI pvfs2 FS MCA component version " OMPI_VERSION;
-
-static int pvfs2_register(void);
-
-int mca_fs_pvfs2_priority = 20;
-int mca_fs_pvfs2_stripe_size = -1;
-int mca_fs_pvfs2_stripe_width = -1;
-int mca_fs_pvfs2_IS_INITIALIZED = 0;
-
-/*
- * Instantiate the public struct with all of our public information
- * and pointers to our public functions in it
- */
-mca_fs_base_component_2_0_0_t mca_fs_pvfs2_component = {
-
-    /* First, the mca_component_t struct containing meta information
-       about the component itself */
-
-    .fsm_version = {
-        MCA_FS_BASE_VERSION_2_0_0,
-
-        /* Component name and version */
-        .mca_component_name = "pvfs2",
-        MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION,
-                              OMPI_RELEASE_VERSION),
-        .mca_register_component_params = pvfs2_register,
-    },
-    .fsm_data = {
-        /* This component is checkpointable */
-      MCA_BASE_METADATA_PARAM_CHECKPOINT
-    },
-    .fsm_init_query = mca_fs_pvfs2_component_init_query,      /* get thread level */
-    .fsm_file_query = mca_fs_pvfs2_component_file_query,      /* get priority and actions */
-    .fsm_file_unquery =  mca_fs_pvfs2_component_file_unquery, /* undo what was done by previous function */
-};
-
-static int
-pvfs2_register(void)
-{
-    mca_fs_pvfs2_priority = 20;
-    (void) mca_base_component_var_register (&mca_fs_pvfs2_component.fsm_version,
-                                            "priority", "Priority of the pvfs2 fs component",
-                                            MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                            OPAL_INFO_LVL_9,
-                                            MCA_BASE_VAR_SCOPE_READONLY, &mca_fs_pvfs2_priority);
-    mca_fs_pvfs2_stripe_size = -1;
-    (void) mca_base_component_var_register (&mca_fs_pvfs2_component.fsm_version,
-                                            "stripe_size", "stripe size of a file over pvfs2",
-                                            MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                            OPAL_INFO_LVL_9,
-                                            MCA_BASE_VAR_SCOPE_READONLY, &mca_fs_pvfs2_stripe_size);
-    mca_fs_pvfs2_stripe_width = -1;
-    (void) mca_base_component_var_register (&mca_fs_pvfs2_component.fsm_version,
-                                            "stripe_width", "stripe width of a file over pvfs2",
-                                            MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                            OPAL_INFO_LVL_9,
-                                            MCA_BASE_VAR_SCOPE_READONLY, &mca_fs_pvfs2_stripe_width);
-
-    return OMPI_SUCCESS;
-}
diff --git a/ompi/mca/fs/pvfs2/fs_pvfs2_file_close.c b/ompi/mca/fs/pvfs2/fs_pvfs2_file_close.c
deleted file mode 100644
index 80f96b3ec48..00000000000
--- a/ompi/mca/fs/pvfs2/fs_pvfs2_file_close.c
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008-2011 University of Houston. All rights reserved.
- * Copyright (c) 2018      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-/* This code is based on the PVFS2 ADIO module in ROMIO
- *   Copyright (C) 1997 University of Chicago.
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ompi_config.h"
-#include "fs_pvfs2.h"
-
-#include <fcntl.h>
-#include <unistd.h>
-#include "mpi.h"
-#include "ompi/constants.h"
-#include "ompi/mca/fs/fs.h"
-
-/*
- *	file_close_pvfs2
- *
- *	Function:	- closes a new file
- *	Accepts:	- file handle
- *	Returns:	- Success if file closed
- */
-int
-mca_fs_pvfs2_file_close (ompio_file_t *fh)
-{
-    if (NULL != fh->f_fs_ptr) {
-        free (fh->f_fs_ptr);
-        fh->f_fs_ptr = NULL;
-    }
-    /*
-    fh->f_comm->c_coll->coll_barrier (fh->f_comm,
-                                     fh->f_comm->c_coll->coll_barrier_module);
-    close (fh->fd);
-    */
-    return OMPI_SUCCESS;
-}
diff --git a/ompi/mca/fs/pvfs2/fs_pvfs2_file_delete.c b/ompi/mca/fs/pvfs2/fs_pvfs2_file_delete.c
deleted file mode 100644
index ca60a8dcce8..00000000000
--- a/ompi/mca/fs/pvfs2/fs_pvfs2_file_delete.c
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2011 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008-2011 University of Houston. All rights reserved.
- * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-/* This code is based on the PVFS2 ADIO module in ROMIO
- *   Copyright (C) 1997 University of Chicago.
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ompi_config.h"
-#include "fs_pvfs2.h"
-
-#include "mpi.h"
-#include "ompi/constants.h"
-#include "ompi/mca/fs/fs.h"
-
-/*
- *	file_delete_pvfs2
- *
- *	Function:	- deletes a file
- *	Accepts:	- file name & info
- *	Returns:	- Success if file closed
- */
-int
-mca_fs_pvfs2_file_delete (char* file_name,
-                          struct opal_info_t *info)
-{
-    PVFS_credentials credentials;
-    PVFS_sysresp_getparent resp_getparent;
-    int ret;
-    PVFS_fs_id pvfs2_id;
-    char pvfs2_path[OMPIO_PVFS2_MAX_NAME] = {0};
-    char * ncache_timeout;
-
-    if (!mca_fs_pvfs2_IS_INITIALIZED) {
-        /* disable the pvfs2 ncache */
-        ncache_timeout = getenv("PVFS2_NCACHE_TIMEOUT");
-        if (ncache_timeout == NULL )
-            setenv("PVFS2_NCACHE_TIMEOUT", "0", 1);
-
-        ret = PVFS_util_init_defaults();
-        if (ret < 0) {
-            return OMPI_ERROR;
-        }
-        mca_fs_pvfs2_IS_INITIALIZED = 1;
-    }
-
-    memset (&credentials, 0, sizeof(PVFS_credentials));
-    PVFS_util_gen_credentials (&credentials);
-
-    ret = PVFS_util_resolve(file_name, &pvfs2_id, pvfs2_path, OMPIO_PVFS2_MAX_NAME);
-    if (ret != 0) {
-        return OMPI_ERROR;
-    }
-
-    ret = PVFS_sys_getparent(pvfs2_id, pvfs2_path, &credentials, &resp_getparent);
-
-    ret = PVFS_sys_remove(resp_getparent.basename,
-                          resp_getparent.parent_ref, &credentials);
-    if (ret != 0) {
-        return OMPI_ERROR;
-    }
-    return OMPI_SUCCESS;
-}
diff --git a/ompi/mca/fs/pvfs2/fs_pvfs2_file_get_size.c b/ompi/mca/fs/pvfs2/fs_pvfs2_file_get_size.c
deleted file mode 100644
index cc71622e70b..00000000000
--- a/ompi/mca/fs/pvfs2/fs_pvfs2_file_get_size.c
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2011 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008-2011 University of Houston. All rights reserved.
- * Copyright (c) 2018      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-/* This code is based on the PVFS2 ADIO module in ROMIO
- *   Copyright (C) 1997 University of Chicago.
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ompi_config.h"
-#include "fs_pvfs2.h"
-
-#include "mpi.h"
-#include "ompi/constants.h"
-#include "ompi/mca/fs/fs.h"
-
-/*
- *	file_get_size_pvfs2
- *
- *	Function:	- get_size of a file
- *	Accepts:	- same arguments as MPI_File_get_size()
- *	Returns:	- Success if size is get
- */
-int
-mca_fs_pvfs2_file_get_size (ompio_file_t *fh,
-                            OMPI_MPI_OFFSET_TYPE *size)
-{
-    int ret;
-    mca_fs_pvfs2 *pvfs2_fs;
-    PVFS_sysresp_getattr resp_getattr;
-
-    pvfs2_fs = (mca_fs_pvfs2 *)fh->f_fs_ptr;
-
-    ret = PVFS_sys_getattr (pvfs2_fs->object_ref, PVFS_ATTR_SYS_SIZE,
-                            &(pvfs2_fs->credentials), &resp_getattr);
-    if (ret != 0 ) {
-        return OMPI_ERROR;
-    }
-
-    *size = resp_getattr.attr.size;
-
-    return OMPI_SUCCESS;
-}
diff --git a/ompi/mca/fs/pvfs2/fs_pvfs2_file_open.c b/ompi/mca/fs/pvfs2/fs_pvfs2_file_open.c
deleted file mode 100644
index cfc9acb5913..00000000000
--- a/ompi/mca/fs/pvfs2/fs_pvfs2_file_open.c
+++ /dev/null
@@ -1,321 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008-2014 University of Houston. All rights reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-/* This code is based on the PVFS2 ADIO module in ROMIO
- *   Copyright (C) 1997 University of Chicago.
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ompi_config.h"
-#include "fs_pvfs2.h"
-
-#include "mpi.h"
-#include "ompi/constants.h"
-#include "ompi/mca/fs/fs.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/info/info.h"
-#include "opal/datatype/opal_convertor.h"
-#include "opal/datatype/opal_datatype.h"
-#include "ompi/datatype/ompi_datatype.h"
-
-struct open_status_s {
-    int error;
-    PVFS_object_ref object_ref;
-};
-typedef struct open_status_s open_status;
-
-static void fake_an_open(PVFS_fs_id id,
-                         const char *pvfs2_name,
-                         int access_mode,
-	                 int stripe_width,
-                         PVFS_size stripe_size,
-                         mca_fs_pvfs2 *pvfs2_fs,
-			 open_status *o_status);
-/*
- *	file_open_pvfs2: This is the same strategy as ROMIO's pvfs2 open
- *
- *	Function:	- opens a new file
- *	Accepts:	- same arguments as MPI_File_open()
- *	Returns:	- Success if new file handle
- */
-int
-mca_fs_pvfs2_file_open (struct ompi_communicator_t *comm,
-                        const char* filename,
-                        int access_mode,
-                        struct opal_info_t *info,
-                        ompio_file_t *fh)
-{
-    int ret;
-    mca_fs_pvfs2 *pvfs2_fs;
-    PVFS_fs_id pvfs2_id;
-    char pvfs2_path[OMPIO_PVFS2_MAX_NAME] = {0};
-    char * ncache_timeout;
-    open_status o_status = {0, {0, 0}};
-    struct ompi_datatype_t *open_status_type;
-    struct ompi_datatype_t *types[2] = {&ompi_mpi_int.dt, &ompi_mpi_byte.dt};
-    int lens[2] = {1, sizeof(PVFS_object_ref)};
-    ptrdiff_t offsets[2];
-    char char_stripe[MPI_MAX_INFO_KEY];
-    int flag;
-    int fs_pvfs2_stripe_size = -1;
-    int fs_pvfs2_stripe_width = -1;
-
-    /* We are going to do what ROMIO does with one process resolving
-     * the name and broadcasting to others */
-
-    pvfs2_fs = (mca_fs_pvfs2 *) malloc(sizeof(mca_fs_pvfs2));
-    if (NULL == pvfs2_fs) {
-        opal_output (1, "OUT OF MEMORY\n");
-        return OMPI_ERR_OUT_OF_RESOURCE;
-    }
-
-    if (!mca_fs_pvfs2_IS_INITIALIZED) {
-        /* disable the pvfs2 ncache */
-        ncache_timeout = getenv("PVFS2_NCACHE_TIMEOUT");
-        if (ncache_timeout == NULL ) {
-            setenv("PVFS2_NCACHE_TIMEOUT", "0", 1);
-        }
-        ret = PVFS_util_init_defaults();
-        if (ret < 0) {
-            PVFS_perror("PVFS_util_init_defaults", ret);
-            return OMPI_ERROR;
-        }
-        mca_fs_pvfs2_IS_INITIALIZED = 1;
-    }
-
-    memset(&(pvfs2_fs->credentials), 0, sizeof(PVFS_credentials));
-    PVFS_util_gen_credentials(&(pvfs2_fs->credentials));
-
-    /* check for stripe size and stripe depth in the info object and
-       update mca_fs_pvfs2_stripe_width and mca_fs_pvfs2_stripe_size
-       before calling fake_an_open() */
-
-    opal_info_get (info, "stripe_size", MPI_MAX_INFO_VAL, char_stripe, &flag);
-    if ( flag ) {
-        sscanf ( char_stripe, "%d", &fs_pvfs2_stripe_size );
-    }
-
-    opal_info_get (info, "stripe_width", MPI_MAX_INFO_VAL, char_stripe, &flag);
-    if ( flag ) {
-        sscanf ( char_stripe, "%d", &fs_pvfs2_stripe_width );
-    }
-
-    if (fs_pvfs2_stripe_size < 0) {
-        fs_pvfs2_stripe_size = mca_fs_pvfs2_stripe_size;
-    }
-
-    if (fs_pvfs2_stripe_width < 0) {
-        fs_pvfs2_stripe_width = mca_fs_pvfs2_stripe_width;
-    }
-
-
-    if (OMPIO_ROOT == fh->f_rank) {
-        ret = PVFS_util_resolve(filename, &pvfs2_id, pvfs2_path, OMPIO_PVFS2_MAX_NAME);
-        if (ret < 0 ) {
-            PVFS_perror("PVFS_util_resolve", ret);
-	    o_status.error = -1;
-        }
-        else {
-            fake_an_open (pvfs2_id,
-                          pvfs2_path,
-                          access_mode,
-                          fs_pvfs2_stripe_width,
-                          (PVFS_size)fs_pvfs2_stripe_size,
-                          pvfs2_fs,
-                          &o_status);
-        }
-        pvfs2_fs->object_ref = o_status.object_ref;
-	fh->f_fs_ptr = pvfs2_fs;
-    }
-
-    /* broadcast status and (possibly valid) object reference */
-    offsets[0] = (MPI_Aint)(&o_status.error);
-    offsets[1] = (MPI_Aint)(&o_status.object_ref);
-
-    ompi_datatype_create_struct (2, lens, offsets, types, &open_status_type);
-    ompi_datatype_commit (&open_status_type);
-
-    fh->f_comm->c_coll->coll_bcast (MPI_BOTTOM,
-                                   1,
-                                   open_status_type,
-                                   OMPIO_ROOT,
-                                   fh->f_comm,
-                                   fh->f_comm->c_coll->coll_bcast_module);
-
-    ompi_datatype_destroy (&open_status_type);
-
-    if (o_status.error != 0) {
-	/* No need to free the pvfs2_fs structure, since it will
-	   be deallocated in file_close in case of an error */
-	fh->f_fs_ptr = NULL;
-	return OMPI_ERROR;
-    }
-
-    pvfs2_fs->object_ref = o_status.object_ref;
-    fh->f_fs_ptr = pvfs2_fs;
-
-    /* update the internal ompio structure to store stripe
-       size and stripe depth correctly.
-       Hadi(to be done): For this read the stripe size and stripe depth from
-       the file itself
-    */
-
-    if (fs_pvfs2_stripe_size > 0 && fs_pvfs2_stripe_width > 0) {
-        fh->f_stripe_size = fs_pvfs2_stripe_size;
-        fh->f_stripe_count = fs_pvfs2_stripe_width;
-    }
-
-    return OMPI_SUCCESS;
-}
-
-static void fake_an_open(PVFS_fs_id id,
-                         const char *pvfs2_name,
-                         int access_mode,
-	                 int stripe_width,
-                         PVFS_size stripe_size,
-                         mca_fs_pvfs2 *pvfs2_fs,
-			 open_status *o_status)
-{
-    int ret;
-    PVFS_sysresp_lookup resp_lookup;
-    PVFS_sysresp_getparent resp_getparent;
-    PVFS_sysresp_create resp_create;
-    PVFS_sys_attr attribs;
-    PVFS_sys_dist *dist;
-
-    memset(&attribs, 0, sizeof(PVFS_sys_attr));
-
-    attribs.owner = geteuid();
-    attribs.group = getegid();
-    attribs.perms = 0644;
-    attribs.mask =  PVFS_ATTR_SYS_ALL_SETABLE;
-    attribs.atime = time(NULL);
-    attribs.mtime = attribs.atime;
-    attribs.ctime = attribs.atime;
-
-    if (stripe_width > 0 ) {
-	attribs.dfile_count = stripe_width;
-	attribs.mask |= PVFS_ATTR_SYS_DFILE_COUNT;
-    }
-
-    dist = NULL;
-
-    memset(&resp_lookup, 0, sizeof(resp_lookup));
-    memset(&resp_getparent, 0, sizeof(resp_getparent));
-    memset(&resp_create, 0, sizeof(resp_create));
-
-    ret = PVFS_sys_lookup(id,
-                          pvfs2_name,
-                          &(pvfs2_fs->credentials),
-                          &resp_lookup,
-                          PVFS2_LOOKUP_LINK_FOLLOW);
-
-    if ( ret == (-PVFS_ENOENT)) {
-	if (access_mode & MPI_MODE_CREATE)  {
-	    ret = PVFS_sys_getparent(id,
-                                     pvfs2_name,
-                                     &(pvfs2_fs->credentials),
-                                     &resp_getparent);
-	    if (ret < 0) {
-                opal_output (1, "pvfs_sys_getparent returns with %d\n", ret);
-		o_status->error = ret;
-		return;
-	    }
-
-            /* Set the distribution stripe size if specified */
-            if (0 < stripe_size) {
-                /* Note that the distribution is hardcoded here */
-                dist = PVFS_sys_dist_lookup ("simple_stripe");
-                ret = PVFS_sys_dist_setparam (dist,
-                                              "strip_size",
-                                              &stripe_size);
-                if (ret < 0)  {
-                    opal_output (1,
-                            "pvfs_sys_dist_setparam returns with %d\n", ret);
-                    o_status->error = ret;
-                }
-            }
-
-            /* Perform file creation */
-            ret = PVFS_sys_create(resp_getparent.basename,
-                                  resp_getparent.parent_ref,
-                                  attribs,
-                                  &(pvfs2_fs->credentials),
-                                  dist,
-                                  &resp_create);
-            /*
-#ifdef HAVE_PVFS2_CREATE_WITHOUT_LAYOUT
-            ret = PVFS_sys_create(resp_getparent.basename,
-                                  resp_getparent.parent_ref,
-                                  attribs,
-                                  &(pvfs2_fs->credentials),
-                                  dist,
-                                  &resp_create);
-				  #else
-            ret = PVFS_sys_create(resp_getparent.basename,
-                                  resp_getparent.parent_ref,
-                                  attribs,
-                                  &(pvfs2_fs->credentials),
-                                  dist,
-                                  NULL,
-                                  &resp_create);
-            #endif
-            */
-
-	    /* if many creates are happening in this directory, the earlier
-	     * sys_lookup may have returned ENOENT, but the sys_create could
-	     * return EEXISTS.  That means the file has been created anyway, so
-	     * less work for us and we can just open it up and return the
-	     * handle */
-	    if (ret == (-PVFS_EEXIST)) {
-		ret = PVFS_sys_lookup(id,
-                                      pvfs2_name,
-                                      &(pvfs2_fs->credentials),
-                                      &resp_lookup,
-                                      PVFS2_LOOKUP_LINK_FOLLOW);
-		if ( ret < 0 ) {
-		    o_status->error = ret;
-		    return;
-		}
-		o_status->error = ret;
-		o_status->object_ref = resp_lookup.ref;
-		return;
-	    }
-	    o_status->object_ref = resp_create.ref;
-	}
-        else {
-	    opal_output (10, "cannot create file without MPI_MODE_CREATE\n");
-	    o_status->error = ret;
-	    return;
-	}
-    }
-    else if (access_mode & MPI_MODE_EXCL) {
-	/* lookup should not succeed if opened with EXCL */
-	o_status->error = -PVFS_EEXIST;
-	return;
-    }
-    else {
-	o_status->object_ref = resp_lookup.ref;
-    }
-    o_status->error = ret;
-    return;
-}
diff --git a/ompi/mca/fs/pvfs2/fs_pvfs2_file_set_size.c b/ompi/mca/fs/pvfs2/fs_pvfs2_file_set_size.c
deleted file mode 100644
index 65239a0ee5b..00000000000
--- a/ompi/mca/fs/pvfs2/fs_pvfs2_file_set_size.c
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008-2011 University of Houston. All rights reserved.
- * Copyright (c) 2018      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-/* This code is based on the PVFS2 ADIO module in ROMIO
- *   Copyright (C) 1997 University of Chicago.
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ompi_config.h"
-#include "fs_pvfs2.h"
-
-#include "mpi.h"
-#include "ompi/constants.h"
-#include "ompi/mca/fs/fs.h"
-
-/*
- *	file_set_size_pvfs2
- *
- *	Function:	- set_size of a file
- *	Accepts:	- same arguments as MPI_File_set_size()
- *	Returns:	- Success if size is set
- */
-int
-mca_fs_pvfs2_file_set_size (ompio_file_t *fh,
-                            OMPI_MPI_OFFSET_TYPE size)
-{
-    int ret;
-    mca_fs_pvfs2 *pvfs2_fs;
-
-    pvfs2_fs = (mca_fs_pvfs2 *)fh->f_fs_ptr;
-
-    if (OMPIO_ROOT == fh->f_rank) {
-        ret = PVFS_sys_truncate(pvfs2_fs->object_ref,
-                                size, &(pvfs2_fs->credentials));
-        fh->f_comm->c_coll->coll_bcast (&ret,
-                                       1,
-                                       MPI_INT,
-                                       OMPIO_ROOT,
-                                       fh->f_comm,
-                                       fh->f_comm->c_coll->coll_bcast_module);
-    }
-    else {
-        fh->f_comm->c_coll->coll_bcast (&ret,
-                                       1,
-                                       MPI_INT,
-                                       OMPIO_ROOT,
-                                       fh->f_comm,
-                                       fh->f_comm->c_coll->coll_bcast_module);
-    }
-
-    if (ret != 0) {
-        return OMPI_ERROR;
-    }
-
-    return OMPI_SUCCESS;
-}
diff --git a/ompi/mca/fs/pvfs2/fs_pvfs2_file_sync.c b/ompi/mca/fs/pvfs2/fs_pvfs2_file_sync.c
deleted file mode 100644
index 9f2bd46d6ec..00000000000
--- a/ompi/mca/fs/pvfs2/fs_pvfs2_file_sync.c
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008-2011 University of Houston. All rights reserved.
- * Copyright (c) 2018      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-/* This code is based on the PVFS2 ADIO module in ROMIO
- *   Copyright (C) 1997 University of Chicago.
- *   See COPYRIGHT notice in top-level directory.
- */
-
-#include "ompi_config.h"
-#include "fs_pvfs2.h"
-
-#include "mpi.h"
-#include "ompi/constants.h"
-#include "ompi/mca/fs/fs.h"
-
-/*
- *	file_sync_pvfs2
- *
- *	Function:	- closes a new file
- *	Accepts:	- file handle
- *	Returns:	- Success if file closed
- */
-int
-mca_fs_pvfs2_file_sync (ompio_file_t *fh)
-{
-    int ret;
-    mca_fs_pvfs2 *pvfs2_fs;
-
-    ret = OMPI_SUCCESS;
-
-    pvfs2_fs = (mca_fs_pvfs2 *)fh->f_fs_ptr;
-
-    if (OMPIO_ROOT == fh->f_rank) {
-        ret = PVFS_sys_flush(pvfs2_fs->object_ref, &(pvfs2_fs->credentials));
-    }
-
-    fh->f_comm->c_coll->coll_bcast (&ret,
-                                   1,
-                                   MPI_INT,
-                                   OMPIO_ROOT,
-                                   fh->f_comm,
-                                   fh->f_comm->c_coll->coll_bcast_module);
-
-    if (0 != ret) {
-        return OMPI_ERROR;
-    }
-    return OMPI_SUCCESS;
-}
diff --git a/ompi/mca/fs/pvfs2/owner.txt b/ompi/mca/fs/pvfs2/owner.txt
deleted file mode 100644
index 2e9726c28a4..00000000000
--- a/ompi/mca/fs/pvfs2/owner.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-#
-# owner/status file
-# owner: institution that is responsible for this package
-# status: e.g. active, maintenance, unmaintained
-#
-owner: UH
-status: active
diff --git a/ompi/mca/fs/ufs/configure.m4 b/ompi/mca/fs/ufs/configure.m4
index dcc617b55d3..b07600c139f 100644
--- a/ompi/mca/fs/ufs/configure.m4
+++ b/ompi/mca/fs/ufs/configure.m4
@@ -12,6 +12,7 @@
 #                         All rights reserved.
 # Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2008-2018 University of Houston. All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -25,5 +26,5 @@
 AC_DEFUN([MCA_ompi_fs_ufs_CONFIG],[
     AC_CONFIG_FILES([ompi/mca/fs/ufs/Makefile])
 
-    OPAL_SUMMARY_ADD([[OMPIO File Systems]],[[Generic Unix FS]],[$1],[yes])
+    OPAL_SUMMARY_ADD([OMPIO File Systems], [Generic Unix FS], [], [yes])
 ])dnl
diff --git a/ompi/mca/fs/ufs/fs_ufs.c b/ompi/mca/fs/ufs/fs_ufs.c
index eacfc745504..dd4070b0c12 100644
--- a/ompi/mca/fs/ufs/fs_ufs.c
+++ b/ompi/mca/fs/ufs/fs_ufs.c
@@ -38,7 +38,7 @@
  * *******************************************************************
  */
 static mca_fs_base_module_1_0_0_t ufs =  {
-    mca_fs_ufs_module_init, /* initalise after being selected */
+    mca_fs_ufs_module_init, /* initialise after being selected */
     mca_fs_ufs_module_finalize, /* close a module on a communicator */
     mca_fs_ufs_file_open,
     mca_fs_base_file_close,
diff --git a/ompi/mca/fs/ufs/fs_ufs.h b/ompi/mca/fs/ufs/fs_ufs.h
index 0ea10df68db..1d8bd527a93 100644
--- a/ompi/mca/fs/ufs/fs_ufs.h
+++ b/ompi/mca/fs/ufs/fs_ufs.h
@@ -47,7 +47,7 @@ int mca_fs_ufs_component_file_unquery (ompio_file_t *file);
 int mca_fs_ufs_module_init (ompio_file_t *file);
 int mca_fs_ufs_module_finalize (ompio_file_t *file);
 
-OMPI_MODULE_DECLSPEC extern mca_fs_base_component_2_0_0_t mca_fs_ufs_component;
+OMPI_DECLSPEC extern mca_fs_base_component_2_0_0_t mca_fs_ufs_component;
 /*
  * ******************************************************************
  * ********* functions which are implemented in this module *********
diff --git a/ompi/mca/fs/ufs/fs_ufs_component.c b/ompi/mca/fs/ufs/fs_ufs_component.c
index 7ecaf9e0fd3..dd6e4bcb51b 100644
--- a/ompi/mca/fs/ufs/fs_ufs_component.c
+++ b/ompi/mca/fs/ufs/fs_ufs_component.c
@@ -32,6 +32,15 @@
 
 int mca_fs_ufs_priority = 10;
 int mca_fs_ufs_lock_algorithm=0; /* auto */
+
+static const mca_base_var_enum_value_t ompi_fs_ufs_lock_algorithm_modes[] = {
+    {.value = 0, .string = "auto"},
+    {.value = 1, .string = "skip locking"},
+    {.value = 2, .string = "always lock entire file"},
+    {.value = 3, .string = "lock specific ranges"},
+    {.string = NULL},
+};
+
 /*
  * Private functions
  */
@@ -73,6 +82,8 @@ mca_fs_base_component_2_0_0_t mca_fs_ufs_component = {
 
 static int register_component(void)
 {
+    mca_base_var_enum_t *new_enum;
+
     mca_fs_ufs_priority = 10;
     (void) mca_base_component_var_register(&mca_fs_ufs_component.fsm_version,
                                            "priority", "Priority of the fs ufs component",
@@ -81,15 +92,17 @@ static int register_component(void)
                                            MCA_BASE_VAR_SCOPE_READONLY,
                                            &mca_fs_ufs_priority);
 
+    (void) mca_base_var_enum_create("mca_fs_ufs_lock_algorithm", ompi_fs_ufs_lock_algorithm_modes, &new_enum);
+
     mca_fs_ufs_lock_algorithm = 0;
     (void) mca_base_component_var_register(&mca_fs_ufs_component.fsm_version,
                                            "lock_algorithm", "Locking algorithm used by the fs ufs component. "
-                                           " 0: auto (default), 1: skip locking, 2: always lock entire file, "
-                                           "3: lock only specific ranges",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           "(default: auto)",
+                                           MCA_BASE_VAR_TYPE_INT, new_enum, 0, 0,
                                            OPAL_INFO_LVL_9,
                                            MCA_BASE_VAR_SCOPE_READONLY,
-                                           &mca_fs_ufs_lock_algorithm );
+                                           &mca_fs_ufs_lock_algorithm);
+    OBJ_RELEASE(new_enum);
 
     return OMPI_SUCCESS;
 }
diff --git a/ompi/mca/fs/ufs/fs_ufs_file_open.c b/ompi/mca/fs/ufs/fs_ufs_file_open.c
index 1159de993bb..273934b9748 100644
--- a/ompi/mca/fs/ufs/fs_ufs_file_open.c
+++ b/ompi/mca/fs/ufs/fs_ufs_file_open.c
@@ -13,6 +13,7 @@
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Advanced Micro Devices, Inc. All rights reserverd.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -105,12 +106,6 @@ mca_fs_ufs_file_open (struct ompi_communicator_t *comm,
                    component. */
                 fh->f_flags |= OMPIO_LOCK_ENTIRE_FILE;
             }
-            else {
-                fh->f_flags |= OMPIO_LOCK_NEVER;
-            }
-        }
-        else {
-            fh->f_flags |= OMPIO_LOCK_NEVER;
         }
         free (fstype);
     }
diff --git a/ompi/mca/hook/base/Makefile.am b/ompi/mca/hook/base/Makefile.am
index 29ac5246be5..c89bdfeec55 100644
--- a/ompi/mca/hook/base/Makefile.am
+++ b/ompi/mca/hook/base/Makefile.am
@@ -1,5 +1,6 @@
 #
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -7,7 +8,7 @@
 # $HEADER$
 #
 
-dist_ompidata_DATA = base/help-mca-hook-base.txt
+EXTRA_DIST = base/help-mca-hook-base.txt
 
 headers += \
         base/base.h
diff --git a/ompi/mca/hook/base/hook_base.c b/ompi/mca/hook/base/hook_base.c
index 8017d749415..ae1366fa6fc 100644
--- a/ompi/mca/hook/base/hook_base.c
+++ b/ompi/mca/hook/base/hook_base.c
@@ -196,7 +196,7 @@ MCA_BASE_FRAMEWORK_DECLARE(ompi, hook, "hook hooks",
 
 /*
  * Once the framework is open then call all available components with
- * the approprate function pointer. Call order:
+ * the appropriate function pointer. Call order:
  * 1) static components
  * 2) dynamic components
  * 3) 'registered' components (those registered by ompi_hook_base_register_callbacks)
diff --git a/ompi/mca/hook/comm_method/hook_comm_method.h b/ompi/mca/hook/comm_method/hook_comm_method.h
index 5f93360d370..9f1e58978be 100644
--- a/ompi/mca/hook/comm_method/hook_comm_method.h
+++ b/ompi/mca/hook/comm_method/hook_comm_method.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020 IBM Corporation. All rights reserved.
+ * Copyright (c) 2016-2022 IBM Corporation. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -18,14 +18,14 @@
 
 BEGIN_C_DECLS
 
-OMPI_MODULE_DECLSPEC extern const ompi_hook_base_component_1_0_0_t mca_hook_comm_method_component;
+OMPI_DECLSPEC extern ompi_hook_base_component_1_0_0_t mca_hook_comm_method_component;
 
 extern int mca_hook_comm_method_verbose;
 extern int mca_hook_comm_method_output;
 extern bool mca_hook_comm_method_enable_mpi_init;
 extern bool mca_hook_comm_method_enable_mpi_finalize;
 extern int mca_hook_comm_method_max;
-extern int mca_hook_comm_method_brief;
+extern bool mca_hook_comm_method_brief;
 extern char *mca_hook_comm_method_fakefile;
 
 void ompi_hook_comm_method_mpi_init_bottom(int argc, char **argv, int requested, int *provided);
diff --git a/ompi/mca/hook/comm_method/hook_comm_method_component.c b/ompi/mca/hook/comm_method/hook_comm_method_component.c
index ca27c8703be..0adbfccea18 100644
--- a/ompi/mca/hook/comm_method/hook_comm_method_component.c
+++ b/ompi/mca/hook/comm_method/hook_comm_method_component.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020 IBM Corporation. All rights reserved.
+ * Copyright (c) 2016-2022 IBM Corporation. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -25,7 +25,7 @@ const char *mca_hook_comm_method_component_version_string =
  * Instantiate the public struct with all of our public information
  * and pointers to our public functions in it
  */
-const ompi_hook_base_component_1_0_0_t mca_hook_comm_method_component = {
+ompi_hook_base_component_1_0_0_t mca_hook_comm_method_component = {
 
     /* First, the mca_component_t struct containing meta information
      * about the component itself */
@@ -79,7 +79,7 @@ bool mca_hook_comm_method_enable_mpi_init = false;
 bool mca_hook_comm_method_enable_mpi_finalize = false;
 uint32_t mca_hook_comm_method_enabled_flags = 0x00;
 int mca_hook_comm_method_max = 12;
-int mca_hook_comm_method_brief = 0;
+bool mca_hook_comm_method_brief = false;
 char *mca_hook_comm_method_fakefile = NULL;
 
 static mca_base_var_enum_value_flag_t mca_hook_comm_method_modes[] = {
@@ -174,7 +174,7 @@ static int ompi_hook_comm_method_component_register(void)
     // hook_comm_method_brief
     (void) mca_base_component_var_register(&mca_hook_comm_method_component.hookm_version, "brief",
                                  "Only print the comm method summary, skip the 2d table.",
-                                 MCA_BASE_VAR_TYPE_INT, NULL,
+                                 MCA_BASE_VAR_TYPE_BOOL, NULL,
                                  0, 0,
                                  OPAL_INFO_LVL_3,
                                  MCA_BASE_VAR_SCOPE_READONLY,
diff --git a/ompi/mca/hook/comm_method/hook_comm_method_fns.c b/ompi/mca/hook/comm_method/hook_comm_method_fns.c
index 42762884dce..b1ab8c200b3 100644
--- a/ompi/mca/hook/comm_method/hook_comm_method_fns.c
+++ b/ompi/mca/hook/comm_method/hook_comm_method_fns.c
@@ -1,5 +1,6 @@
 /*
- * Copyright (c) 2016-2020 IBM Corporation. All rights reserved.
+ * Copyright (c) 2016-2022 IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -15,6 +16,7 @@
 #include <dlfcn.h>
 #endif
 
+#include "opal/util/string_copy.h"
 #include "ompi/communicator/communicator.h"
 #include "ompi/mca/pml/pml.h"
 #include "opal/mca/btl/btl.h"
@@ -22,23 +24,13 @@
 #include "ompi/mca/bml/base/base.h"
 #include "ompi/mca/mtl/base/base.h"
 
-// In regular strncpy up to n bytes are copied, so if the 'to' buffer
-// was char string[16] and you called strncpy(string, , 16) you could
-// get 16 bytes of chars without a null.  My preferred API is to let
-// n be the size of the buffer, and to let n-1 chars be copied, and
-// to guarantee null termination.
-static void
-mystrncpy(char *to, const char *from, int n) {
-    strncpy(to, from, n-1);
-    to[n-1] = 0;
-}
-
 // For converting comm_method strings to comm_method id# and back.
 // This starts as our local set of strings, but gets Allreduced into
 // a global mapping so all the strings at all the ranks are represented.
-// If an MCA's name is more than 15 chars it gets truncated.
-#define COMM_METHOD_STRING_SIZE 16
-#define MAX_COMM_METHODS 50
+#define COMM_METHOD_STRING_SIZE 200
+#define MAX_COMM_METHODS 1000
+#define UCX_TAG "ucx="
+
 typedef struct {
     int n;
     char str[MAX_COMM_METHODS][COMM_METHOD_STRING_SIZE];
@@ -50,6 +42,9 @@ static comm_method_string_conversion_t comm_method_string_conversion;
 #define MODE_IS_MTL 2
 #define MODE_IS_BTL 3
 
+#define CALLED_FROM_MPI_INIT 1
+#define CALLED_FROM_MPI_FINALIZE 2
+
 // ----------------------------------------------------------------------------
 
 // return the pml's module:component:name function pointer in fp
@@ -70,6 +65,10 @@ static char*
 lookup_btl_name_for_send(ompi_communicator_t* comm, int rank) {
     ompi_proc_t *dst_proc = ompi_group_peer_lookup_existing(comm->c_remote_group, rank);
 
+    if (NULL == dst_proc) {
+        return NULL;
+    }
+
     mca_bml_base_endpoint_t* endpoint = mca_bml_base_get_endpoint(dst_proc);
     if (endpoint &&
         endpoint->btl_send.bml_btls &&
@@ -90,22 +89,71 @@ lookup_btl_name_for_send(ompi_communicator_t* comm, int rank) {
 // that the caller has to free.
 static char *
 comm_method_string(MPI_Comm comm, int rank, int *comm_mode) {
-    char *p;
-    char *string = malloc(COMM_METHOD_STRING_SIZE);
-
-    if (!string) { return NULL; }
-
-    p = lookup_pml_name();
-    if (p && 0==strncmp("ob1", p, 4)) {      // BTL
-        if (comm_mode) { *comm_mode = MODE_IS_BTL; }
-        mystrncpy(string, lookup_btl_name_for_send(comm, rank), COMM_METHOD_STRING_SIZE);
+    char *p, *btl;
+    char *string, *comma_delim = "";
+    mca_pml_transports_t *transports = NULL;
+    int name_length;
+    unsigned int i;
+    if (NULL != mca_pml.pml_get_transports) {
+        transports = mca_pml.pml_get_transports(comm, rank);
+    }
+    if (NULL == transports) {
+        string = malloc(COMM_METHOD_STRING_SIZE);
+        if (!string) {
+            return NULL;
+        }
+        p = lookup_pml_name();
+        if (p && 0==strncmp("ob1", p, 4)) {      // BTL
+            if (comm_mode) { *comm_mode = MODE_IS_BTL; }
+            btl = lookup_btl_name_for_send(comm, rank);
+            if (NULL == btl) {
+                opal_string_copy(string, "n/a", COMM_METHOD_STRING_SIZE);
+            } else {
+                opal_string_copy(string, btl, COMM_METHOD_STRING_SIZE);
+            }
+        }
+        else if (p && 0==strncmp("cm", p, 3)) {  // MTL
+            if (comm_mode) { *comm_mode = MODE_IS_MTL; }
+            opal_string_copy(string, lookup_mtl_name(), COMM_METHOD_STRING_SIZE);
+        } else {                        // PML
+            if (comm_mode) { *comm_mode = MODE_IS_PML; }
+            if (p) {
+                opal_string_copy(string, p, COMM_METHOD_STRING_SIZE);
+            }
+            else {
+                opal_string_copy(string, "n/a", COMM_METHOD_STRING_SIZE);
+            }
+        }
+    }
+    else {
+        /* Determine how much memory is needed to store UCX transport info */
+        char *s = UCX_TAG;
+        /* Allocate storage to store UCX transport info, accounting for
+         * trailing '\0' in UCX_TAG and ',' and ';' delimiting each transport string
+         * then build the info string */
+        name_length = strlen(s) + 1;
+        for (i = 0; i < transports->count; i++) {
+            name_length = name_length + strlen(transports->entries[i].transport_name) +
+                                        strlen(transports->entries[i].device_name) + 2;
+        }
+        string = malloc(name_length);
+        if (!string) {
+            return NULL;
+        }
+        strcpy(string, s);
+        for (i = 0; i < transports->count; i++) {
+            strcat(string, comma_delim);
+            comma_delim = ",";
+            strcat(string, transports->entries[i].transport_name);
+            strcat(string, ";");
+            strcat(string, transports->entries[i].device_name);
+        }
+        free(transports->entries);
+        free(transports);
     }
-    else if (p && 0==strncmp("cm", p, 3)) {  // MTL
-        if (comm_mode) { *comm_mode = MODE_IS_MTL; }
-        mystrncpy(string, lookup_mtl_name(), COMM_METHOD_STRING_SIZE);
-    } else {                        // PML
-        if (comm_mode) { *comm_mode = MODE_IS_PML; }
-        mystrncpy(string, p, COMM_METHOD_STRING_SIZE);
+    if (comm_mode) {
+        // UCX is used for PML mode only
+        *comm_mode = MODE_IS_PML;
     }
     return string;
 }
@@ -134,7 +182,7 @@ lookup_string_in_conversion_struct(comm_method_string_conversion_t *data, char *
 {
     int i;
     for (i=0; i<data->n; ++i) {
-        if (0==strncmp(data->str[i], string, COMM_METHOD_STRING_SIZE)) {
+        if (0==strcmp(data->str[i], string)) {
             return i;
         }
     }
@@ -150,16 +198,16 @@ static void
 add_string_to_conversion_struct(comm_method_string_conversion_t *data, char *string)
 {
     int i;
-    if (0 == strcmp(string, "n/a")) { return; }
+    if (NULL == string || 0 == strcmp(string, "n/a")) { return; }
 
     i = lookup_string_in_conversion_struct(data, string);
     if (i == 0) { // didn't find string in list, so add it
         if (data->n < MAX_COMM_METHODS) {
-            mystrncpy(data->str[data->n], string, COMM_METHOD_STRING_SIZE);
+            strncpy(data->str[data->n], string, COMM_METHOD_STRING_SIZE);
+            data->str[data->n][COMM_METHOD_STRING_SIZE - 1] = '\0';
             ++(data->n);
         }
     }
-    qsort(&data->str[1], data->n - 1, COMM_METHOD_STRING_SIZE, &mycompar);
 }
 
 // For MPI_Allreduce of a comm_method_string_conversion_t
@@ -173,7 +221,6 @@ static void myfn(void* invec, void* inoutvec, int *len, MPI_Datatype *dt) {
         for (j=0; j<b->n; ++j) { // for each entry j in 'b', add it to 'a'
             add_string_to_conversion_struct(a, b->str[j]);
         }
-        qsort(&a->str[1], a->n - 1, COMM_METHOD_STRING_SIZE, &mycompar);
     }
 }
 
@@ -220,14 +267,14 @@ static void ompi_report_comm_methods(int called_from_location);
 void ompi_hook_comm_method_mpi_init_bottom(int argc, char **argv, int requested, int *provided)
 {
     if( mca_hook_comm_method_enable_mpi_init ) {
-        ompi_report_comm_methods( 1 );
+        ompi_report_comm_methods( CALLED_FROM_MPI_INIT );
     }
 }
 
 void ompi_hook_comm_method_mpi_finalize_top(void)
 {
     if( mca_hook_comm_method_enable_mpi_finalize ) {
-        ompi_report_comm_methods( 2 );
+        ompi_report_comm_methods( CALLED_FROM_MPI_FINALIZE );
     }
 }
 
@@ -318,9 +365,9 @@ abbreviate_list_into_string(char *str, int max, int *list, int nlist)
 // When activated from init: we establish connections before printing.
 // When activated from finalize: we just print whatever info is available.
 static void
-ompi_report_comm_methods(int called_from_location) // 1 = from init, 2 = from finalize
+ompi_report_comm_methods(int called_from_location)
 {
-    int numhosts, i, j, k;
+    int numhosts, i, j, k, n;
     int max2Dprottable = 12;
     int max2D1Cprottable = 36;
     int hpmp_myrank;
@@ -328,6 +375,7 @@ ompi_report_comm_methods(int called_from_location) // 1 = from init, 2 = from fi
     int ret;
     ompi_communicator_t *local_comm, *leader_comm;
     int *method;
+    unsigned char *methods_used;
     char *hoststring;
     char **allhoststrings;
     int comm_mode; // MODE_IS_BTL / MTL / PML
@@ -423,6 +471,8 @@ ompi_report_comm_methods(int called_from_location) // 1 = from init, 2 = from fi
 // If we're running during init, establish connections between all peers
 // (in leader_comm, which is all the ranks that are here at this point)
     if (called_from_location == 1) {
+        int speer = (myleaderrank + 1) % nleaderranks;
+        int rpeer = (myleaderrank - 1 + nleaderranks) % nleaderranks;
         for (i=0; i<=nleaderranks/2; ++i) {
 // (Examples to show why the loop is i<=nleaderranks/2)
 // np4 : 0 1 2 3    i=0 0c0  i=1 0c0&1&3  i=2 0c0&1&3&2
@@ -430,9 +480,6 @@ ompi_report_comm_methods(int called_from_location) // 1 = from init, 2 = from fi
             MPI_Request sreq, rreq;
             MPI_Status status;
             int sbuf, rbuf;
-            int speer = (myleaderrank + 1) % nleaderranks;
-            int rpeer = (myleaderrank - 1 + nleaderranks) % nleaderranks;
-
             sbuf = rbuf = 0;
             MCA_PML_CALL(isend(&sbuf, 1, MPI_INT, speer, 99,
                     MCA_PML_BASE_SEND_STANDARD,
@@ -441,6 +488,11 @@ ompi_report_comm_methods(int called_from_location) // 1 = from init, 2 = from fi
                     leader_comm, &rreq));
             ompi_request_wait(&sreq, &status);
             ompi_request_wait(&rreq, &status);
+            speer = (speer + 1) % nleaderranks;
+            rpeer = (rpeer - 1) % nleaderranks;
+            if (rpeer < 0) {
+                rpeer = nleaderranks - 1;
+            }
         }
     }
 
@@ -470,6 +522,10 @@ ompi_report_comm_methods(int called_from_location) // 1 = from init, 2 = from fi
     MPI_Op_free(&myop);
     MPI_Type_free(&mydt);
 
+    // Sort communication method string arrays after reduction
+    qsort(&comm_method_string_conversion.str[1],
+          comm_method_string_conversion.n - 1, COMM_METHOD_STRING_SIZE, &mycompar);
+
 // Each host leader fills in a "numhosts" sized array method[] of
 // how it communicates with each peer.
     for (i=0; i<nleaderranks; ++i) {
@@ -479,10 +535,8 @@ ompi_report_comm_methods(int called_from_location) // 1 = from init, 2 = from fi
 // unless there's only one rank and "self" is the best answer. So if
 // there's more than one rank on our host, we get our local-host's
 // communication method for a neighbor on this host.
-        if (i == myleaderrank) {
-            if (nlocalranks > 1) {
-                method[i] = comm_method(local_comm, 1);
-            }
+        if ((i == myleaderrank) && (nlocalranks > 1)) {
+            method[i] = comm_method(local_comm, 1);
         }
     }
 
@@ -491,7 +545,11 @@ ompi_report_comm_methods(int called_from_location) // 1 = from init, 2 = from fi
 // the sizes first and allocate the receiving string.
     {
         int len, *lens, *disps;
+        ompi_count_array_t lens_desc;
+        ompi_disp_array_t disps_desc;
 
+        // First get the array of host strings (host names and task lists) 
+        // for all nodes.
         len = strlen(hoststring) + 1;
         if (myleaderrank == 0) {
             lens = malloc(nleaderranks * sizeof(int));
@@ -516,23 +574,27 @@ ompi_report_comm_methods(int called_from_location) // 1 = from init, 2 = from fi
                 allhoststrings[i] = p;
                 p += lens[i];
             }
+            OMPI_COUNT_ARRAY_INIT(&lens_desc, lens);
+            OMPI_DISP_ARRAY_INIT(&disps_desc, disps);
             leader_comm->c_coll->coll_gatherv(
                 hoststring, strlen(hoststring) + 1, MPI_CHAR,
-                &allhoststrings[0][0], lens, disps, MPI_CHAR,
+                &allhoststrings[0][0], lens_desc, disps_desc, MPI_CHAR,
                 0, leader_comm, leader_comm->c_coll->coll_gatherv_module);
         } else {
             // matching above call from rank 0, just &allhoststrings[0][0]
             // isn't legal here, and those args aren't used at non-root anyway
             leader_comm->c_coll->coll_gatherv(
                 hoststring, strlen(hoststring) + 1, MPI_CHAR,
-                NULL, NULL, NULL, MPI_CHAR,
+                NULL, 0, 0, MPI_CHAR,
                 0, leader_comm, leader_comm->c_coll->coll_gatherv_module);
         }
         if (myleaderrank == 0) {
             free(lens);
             free(disps);
         }
-// and a simpler gather for the methods
+
+        // and a simpler gather for the arrays of communication method indices
+        // for all nodes.
         leader_comm->c_coll->coll_gather(
             method, nleaderranks, MPI_INT,
             method, nleaderranks, MPI_INT,
@@ -551,13 +613,17 @@ ompi_report_comm_methods(int called_from_location) // 1 = from init, 2 = from fi
             fp = fopen(mca_hook_comm_method_fakefile, "r");
             for (i=0; i<nleaderranks; ++i) {
                 for (k=0; k<nleaderranks; ++k) {
-                    fscanf(fp, "%d", &setting);
+                    if (fscanf(fp, "%d", &setting) != 1) {
+                        break;
+                    }
                     // let -1 mean "use existing (real) setting"
                     if (setting != -1) {
                         method[i * nleaderranks + k] = setting;
                     }
                 }
-                fscanf(fp, "\n");
+                if (fscanf(fp, "\n") != 0) {
+                    break;
+                }
             }
             fclose(fp);
         }
@@ -576,14 +642,22 @@ ompi_report_comm_methods(int called_from_location) // 1 = from init, 2 = from fi
 // 2: 2d table
         if (nleaderranks <= max2Dprottable) {
             char *str, *p;
-            int tmp, per;
+            int tmp, per, has_ucx_transport;
             int strlens[NUM_COMM_METHODS];
 
             // characters per entry in the 2d table, must be large enough
             // for the digits needed for host numbers, and for whatever is
             // the longest string used in the table, plus a space.
             for (i=0; i<NUM_COMM_METHODS; ++i) {
-                strlens[i] = strlen(comm_method_to_string(i));
+                p = comm_method_to_string(i);
+                if (0 == strncmp(p, UCX_TAG, strlen(UCX_TAG))) {
+                    // Assume no more than 1000 UCX transport strings
+                    // See PML_UCX_MAX_TRANSPORT_ENTRIES in pml_ucx.c
+                    strlens[i] = strlen("ucx[000]");
+                }
+                else {
+                    strlens[i] = strlen(p);
+                }
             }
             per = 2;
             tmp = nleaderranks;
@@ -605,6 +679,10 @@ ompi_report_comm_methods(int called_from_location) // 1 = from init, 2 = from fi
                 p[j] = 0;
                 p += j;
             }
+            // Use a bitmap to trace which UCX transport strings are used.
+            n = (nleaderranks + 7) / 8;
+            methods_used = malloc(n * sizeof(unsigned char));
+            memset(methods_used, 0, n);
             tmp = (int)strlen(str);
             --p;
             while (p>=str && ((*p)==' ')) { *(p--)=0; }
@@ -612,12 +690,27 @@ ompi_report_comm_methods(int called_from_location) // 1 = from init, 2 = from fi
             memset(str, (int)'=', tmp);
             str[tmp] = 0;
             printf("======|=%s\n", str);
+            has_ucx_transport = 0;
 
             for (i=0; i<nleaderranks; ++i) {
                 str[0] = 0;
                 p = str;
                 for (k=0; k<nleaderranks; ++k) {
-                    strcat(p, comm_method_to_string(method[i * nleaderranks + k]));
+                    char *method_string;
+                    char ucx_label[20];
+                    
+                    method_string = comm_method_to_string(method[i * nleaderranks + k]);
+                    if (0 == strncmp(method_string, UCX_TAG, strlen(UCX_TAG))) {
+                        n = lookup_string_in_conversion_struct(&comm_method_string_conversion,
+                                                               method_string);
+                        sprintf(ucx_label, "ucx[%3d]", n);
+                        strcat(p, ucx_label);
+                        methods_used[n / 8] |= (1 << (n % 8));
+                        has_ucx_transport = 1;
+                    }
+                    else {
+                        strcat(p, method_string);
+                    }
                     for (j=(int)strlen(p); j<per; ++j) {
                         p[j] = ' ';
                     }
@@ -630,6 +723,35 @@ ompi_report_comm_methods(int called_from_location) // 1 = from init, 2 = from fi
             }
             printf("\n");
             free(str);
+            if (has_ucx_transport) {
+                printf("UCX Transport/Device\n");
+                for (i = 0; i < comm_method_string_conversion.n; i++) {
+                    // Check bitmap to check if method was used
+                    if (methods_used[i / 8] & (1 << (i % 8))) {
+                        p = comm_method_to_string(i);
+                        if (0 == strncmp(p, UCX_TAG, strlen(UCX_TAG))) {
+                            char *temp_str, *token;
+                            n = lookup_string_in_conversion_struct(&comm_method_string_conversion, p);
+                            printf("ucx[%3d]:\n", n);
+                            temp_str = strdup(p + 4);
+                            token = strtok(temp_str, ",");
+                            while (NULL != token) {
+                                p = strchr(token, ';');
+                                if (NULL == p) {
+                                    printf("    %-16s\n", token);
+                                }
+                                else {
+                                    *p = '\0';
+                                    printf("    %-16s %-16s\n", token, p + 1);
+                                }
+                                token = strtok(NULL, ",");
+                            }
+                            free(temp_str);
+                        }
+                    }
+                }
+            }
+            free(methods_used);
         }
         else if (nleaderranks <= max2D1Cprottable) {
             char *str, *p;
diff --git a/ompi/mca/hook/demo/hook_demo.h b/ompi/mca/hook/demo/hook_demo.h
index 2022df4938c..17f627803f7 100644
--- a/ompi/mca/hook/demo/hook_demo.h
+++ b/ompi/mca/hook/demo/hook_demo.h
@@ -20,7 +20,7 @@
 
 BEGIN_C_DECLS
 
-OMPI_MODULE_DECLSPEC extern const ompi_hook_base_component_1_0_0_t mca_hook_demo_component;
+OMPI_DECLSPEC extern const ompi_hook_base_component_1_0_0_t mca_hook_demo_component;
 
 void ompi_hook_demo_mpi_initialized_top(int *flag);
 void ompi_hook_demo_mpi_initialized_bottom(int *flag);
diff --git a/ompi/mca/hook/hook.h b/ompi/mca/hook/hook.h
index 4625b17f43f..61febe5c88c 100644
--- a/ompi/mca/hook/hook.h
+++ b/ompi/mca/hook/hook.h
@@ -18,7 +18,7 @@
  *
  * Why not a PMPI library?
  *  - The desire is to allow a component to be built statically into the library
- *    for licensing and disgnostic purposes. As such we need the ability for
+ *    for licensing and diagnostic purposes. As such we need the ability for
  *    the component to be statically linked into the libmpi library.
  *
  * Why is there not a hook for location XYZ?
@@ -202,7 +202,7 @@ typedef struct ompi_hook_base_component_1_0_0_t ompi_hook_base_component_1_0_0_t
 typedef ompi_hook_base_component_1_0_0_t ompi_hook_base_component_t;
 /* 
  * Note: We do -not- expose a component object for this framework.
- * All interation with the component should go through the base/base.h interfaces.
+ * All interaction with the component should go through the base/base.h interfaces.
  * See that header for more information on calling functions.
  */
 
diff --git a/ompi/mca/io/base/io_base_delete.c b/ompi/mca/io/base/io_base_delete.c
index b5926aad5b8..f59517f95d8 100644
--- a/ompi/mca/io/base/io_base_delete.c
+++ b/ompi/mca/io/base/io_base_delete.c
@@ -14,6 +14,8 @@
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
  * Copyright (c) 2008-2018 University of Houston. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -64,7 +66,7 @@ static avail_io_t *check_one_component(const mca_base_component_t *component,
 
 static avail_io_t *query(const mca_base_component_t *component,
                          const char *filename, struct opal_info_t *info);
-static avail_io_t *query_2_0_0(const mca_io_base_component_2_0_0_t *io_component,
+static avail_io_t *query_3_0_0(const mca_io_base_component_3_0_0_t *io_component,
                                const char *filename, struct opal_info_t *info);
 
 static void unquery(avail_io_t *avail, const char *filename, struct opal_info_t *info);
@@ -125,7 +127,7 @@ int mca_io_base_delete(const char *filename, struct opal_info_t *info)
        everyone has available */
 #if 1
     /* For the moment, just take the top module off the list */
-    /* MSC actually take the buttom */
+    /* MSC actually take the bottom */
     item = opal_list_remove_last(selectable);
     avail = (avail_io_t *) item;
     selected = *avail;
@@ -147,7 +149,7 @@ int mca_io_base_delete(const char *filename, struct opal_info_t *info)
     }
     OBJ_RELEASE(selectable);
 
-    if (!strcmp (selected.ai_component.v2_0_0.io_version.mca_component_name,
+    if (!strcmp (selected.ai_component.v3_0_0.io_version.mca_component_name,
                  "ompio")) {
         int ret;
 
@@ -174,7 +176,7 @@ int mca_io_base_delete(const char *filename, struct opal_info_t *info)
 
     opal_output_verbose(10, ompi_io_base_framework.framework_output,
                         "io:base:delete: Selected io component %s",
-                        selected.ai_component.v2_0_0.io_version.mca_component_name);
+                        selected.ai_component.v3_0_0.io_version.mca_component_name);
 
     return OMPI_SUCCESS;
 }
@@ -308,16 +310,16 @@ static avail_io_t *check_one_component(const mca_base_component_t *component,
 static avail_io_t *query(const mca_base_component_t *component,
                          const char *filename, struct opal_info_t *info)
 {
-    const mca_io_base_component_2_0_0_t *ioc_200;
+    const mca_io_base_component_3_0_0_t *ioc_300;
 
-    /* io v2.0.0 */
+    /* io v3.0.0 */
 
     if (MCA_BASE_VERSION_MAJOR == component->mca_major_version &&
         MCA_BASE_VERSION_MINOR == component->mca_minor_version &&
         MCA_BASE_VERSION_RELEASE == component->mca_release_version) {
-        ioc_200 = (mca_io_base_component_2_0_0_t *) component;
+        ioc_300 = (mca_io_base_component_3_0_0_t *) component;
 
-        return query_2_0_0(ioc_200, filename, info);
+        return query_3_0_0(ioc_300, filename, info);
     }
 
     /* Unknown io API version -- return error */
@@ -326,7 +328,7 @@ static avail_io_t *query(const mca_base_component_t *component,
 }
 
 
-static avail_io_t *query_2_0_0(const mca_io_base_component_2_0_0_t *component,
+static avail_io_t *query_3_0_0(const mca_io_base_component_3_0_0_t *component,
                                const char *filename, struct opal_info_t *info)
 {
     bool usable;
@@ -334,7 +336,7 @@ static avail_io_t *query_2_0_0(const mca_io_base_component_2_0_0_t *component,
     avail_io_t *avail;
     struct mca_io_base_delete_t *private_data;
 
-    /* Query v2.0.0 */
+    /* Query v3.0.0 */
 
     avail = NULL;
     private_data = NULL;
@@ -343,9 +345,9 @@ static avail_io_t *query_2_0_0(const mca_io_base_component_2_0_0_t *component,
                                      &priority);
     if (OMPI_SUCCESS == ret && usable) {
         avail = OBJ_NEW(avail_io_t);
-        avail->ai_version = MCA_IO_BASE_V_2_0_0;
+        avail->ai_version = MCA_IO_BASE_V_3_0_0;
         avail->ai_priority = priority;
-        avail->ai_component.v2_0_0 = *component;
+        avail->ai_component.v3_0_0 = *component;
         avail->ai_private_data = private_data;
     }
 
@@ -359,13 +361,13 @@ static avail_io_t *query_2_0_0(const mca_io_base_component_2_0_0_t *component,
 
 static void unquery(avail_io_t *avail, const char *filename, struct opal_info_t *info)
 {
-    const mca_io_base_component_2_0_0_t *ioc_200;
+    const mca_io_base_component_3_0_0_t *ioc_300;
 
     switch(avail->ai_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        ioc_200 = &(avail->ai_component.v2_0_0);
-        if (NULL != ioc_200->io_delete_unquery) {
-            ioc_200->io_delete_unquery(filename, info, avail->ai_private_data);
+    case MCA_IO_BASE_V_3_0_0:
+        ioc_300 = &(avail->ai_component.v3_0_0);
+        if (NULL != ioc_300->io_delete_unquery) {
+            ioc_300->io_delete_unquery(filename, info, avail->ai_private_data);
         }
         break;
 
@@ -384,12 +386,12 @@ static void unquery(avail_io_t *avail, const char *filename, struct opal_info_t
  */
 static int delete_file(avail_io_t *avail, const char *filename, struct opal_info_t *info)
 {
-    const mca_io_base_component_2_0_0_t *ioc_200;
+    const mca_io_base_component_3_0_0_t *ioc_300;
 
     switch(avail->ai_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        ioc_200 = &(avail->ai_component.v2_0_0);
-        return ioc_200->io_delete_select(filename, info,
+    case MCA_IO_BASE_V_3_0_0:
+        ioc_300 = &(avail->ai_component.v3_0_0);
+        return ioc_300->io_delete_select(filename, info,
                                          avail->ai_private_data);
         break;
 
diff --git a/ompi/mca/io/base/io_base_file_select.c b/ompi/mca/io/base/io_base_file_select.c
index 507f23c9950..b14ea2049d8 100644
--- a/ompi/mca/io/base/io_base_file_select.c
+++ b/ompi/mca/io/base/io_base_file_select.c
@@ -14,6 +14,8 @@
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -76,7 +78,7 @@ static avail_io_t *check_one_component(ompi_file_t *file,
 
 static avail_io_t *query(const mca_base_component_t *component,
                          ompi_file_t *file);
-static avail_io_t *query_2_0_0(const mca_io_base_component_2_0_0_t *io_component,
+static avail_io_t *query_2_0_0(const mca_io_base_component_3_0_0_t *io_component,
                                ompi_file_t *file);
 
 static void unquery(avail_io_t *avail, ompi_file_t *file);
@@ -172,7 +174,7 @@ int mca_io_base_file_select(ompi_file_t *file,
 
 #if 1
     /* For the moment, just take the top module off the list */
-    /* MSC actually take the buttom */
+    /* MSC actually take the bottom */
     item = opal_list_remove_last(selectable);
     avail = (avail_io_t *) item;
     selected = *avail;
@@ -192,16 +194,10 @@ int mca_io_base_file_select(ompi_file_t *file,
         unquery(avail, file);
         OBJ_RELEASE(item);
     }
-    OBJ_RELEASE(selectable);
 
-    /* Save the pointers of the selected module on the ompi_file_t */
-
-    file->f_io_version = selected.ai_version;
-    file->f_io_selected_component = selected.ai_component;
-    file->f_io_selected_module = selected.ai_module;
-    file->f_io_selected_data = selected.ai_module_data;
+    OBJ_RELEASE(selectable);
 
-    if (!strcmp (selected.ai_component.v2_0_0.io_version.mca_component_name,
+    if (!strcmp (selected.ai_component.v3_0_0.io_version.mca_component_name,
                  "ompio")) {
         int ret;
 
@@ -242,7 +238,15 @@ int mca_io_base_file_select(ompi_file_t *file,
         }
 
     }
-    /* Finally -- intialize the selected module. */
+
+    /* Save the pointers of the selected module on the ompi_file_t */
+
+    file->f_io_version = selected.ai_version;
+    file->f_io_selected_component = selected.ai_component;
+    file->f_io_selected_module = selected.ai_module;
+    file->f_io_selected_data = selected.ai_module_data;
+
+    /* Finally -- initialize the selected module. */
 
     if (OMPI_SUCCESS != (err = module_init(file))) {
         return err;
@@ -252,15 +256,15 @@ int mca_io_base_file_select(ompi_file_t *file,
 
     opal_output_verbose(10, ompi_io_base_framework.framework_output,
                         "io:base:file_select: Selected io module %s",
-                        selected.ai_component.v2_0_0.io_version.mca_component_name);
+                        selected.ai_component.v3_0_0.io_version.mca_component_name);
 
 #if OPAL_ENABLE_FT_MPI
     if(ompi_ftmpi_enabled) {
         /* check if module is tested for FT, warn if not. */
         const char* ft_whitelist="";
         opal_show_help("help-mpi-ft.txt", "module:untested:failundef", true,
-            selected.ai_component.v2_0_0.io_version.mca_type_name,
-            selected.ai_component.v2_0_0.io_version.mca_component_name,
+            selected.ai_component.v3_0_0.io_version.mca_type_name,
+            selected.ai_component.v3_0_0.io_version.mca_component_name,
             ft_whitelist);
     }
 #endif /* OPAL_ENABLE_FT_MPI */
@@ -394,16 +398,16 @@ static avail_io_t *check_one_component(ompi_file_t *file,
 static avail_io_t *query(const mca_base_component_t *component,
                          ompi_file_t *file)
 {
-    const mca_io_base_component_2_0_0_t *ioc_200;
+    const mca_io_base_component_3_0_0_t *ioc_300;
 
     /* MCA version check */
 
     if (MCA_BASE_VERSION_MAJOR == component->mca_major_version &&
         MCA_BASE_VERSION_MINOR == component->mca_minor_version &&
         MCA_BASE_VERSION_RELEASE == component->mca_release_version) {
-        ioc_200 = (mca_io_base_component_2_0_0_t *) component;
+        ioc_300 = (mca_io_base_component_3_0_0_t *) component;
 
-        return query_2_0_0(ioc_200, file);
+        return query_2_0_0(ioc_300, file);
     }
 
     /* Unknown io API version -- return error */
@@ -412,12 +416,12 @@ static avail_io_t *query(const mca_base_component_t *component,
 }
 
 
-static avail_io_t *query_2_0_0(const mca_io_base_component_2_0_0_t *component,
+static avail_io_t *query_2_0_0(const mca_io_base_component_3_0_0_t *component,
                                ompi_file_t *file)
 {
     int priority;
     avail_io_t *avail;
-    const mca_io_base_module_2_0_0_t *module;
+    const mca_io_base_module_3_0_0_t *module;
     struct mca_io_base_file_t *module_data;
 
     /* Query v2.0.0 */
@@ -427,10 +431,10 @@ static avail_io_t *query_2_0_0(const mca_io_base_component_2_0_0_t *component,
     module = component->io_file_query(file, &module_data, &priority);
     if (NULL != module) {
         avail = OBJ_NEW(avail_io_t);
-        avail->ai_version = MCA_IO_BASE_V_2_0_0;
+        avail->ai_version = MCA_IO_BASE_V_3_0_0;
         avail->ai_priority = priority;
-        avail->ai_component.v2_0_0 = *component;
-        avail->ai_module.v2_0_0 = *module;
+        avail->ai_component.v3_0_0 = *component;
+        avail->ai_module.v3_0_0 = *module;
         avail->ai_module_data = module_data;
     }
 
@@ -444,12 +448,12 @@ static avail_io_t *query_2_0_0(const mca_io_base_component_2_0_0_t *component,
 
 static void unquery(avail_io_t *avail, ompi_file_t *file)
 {
-    const mca_io_base_component_2_0_0_t *ioc_200;
+    const mca_io_base_component_3_0_0_t *ioc_300;
 
     switch(avail->ai_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        ioc_200 = &(avail->ai_component.v2_0_0);
-        ioc_200->io_file_unquery(file, avail->ai_module_data);
+    case MCA_IO_BASE_V_3_0_0:
+        ioc_300 = &(avail->ai_component.v3_0_0);
+        ioc_300->io_file_unquery(file, avail->ai_module_data);
         break;
 
     default:
@@ -467,12 +471,12 @@ static void unquery(avail_io_t *avail, ompi_file_t *file)
  */
 static int module_init(ompi_file_t *file)
 {
-    const mca_io_base_module_2_0_0_t *iom_200;
+    const mca_io_base_module_3_0_0_t *iom_300;
 
     switch(file->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        iom_200 = &(file->f_io_selected_module.v2_0_0);
-        return iom_200->io_module_file_open(file->f_comm, file->f_filename,
+    case MCA_IO_BASE_V_3_0_0:
+        iom_300 = &(file->f_io_selected_module.v3_0_0);
+        return iom_300->io_module_file_open(file->f_comm, file->f_filename,
                                             file->f_amode, file->super.s_info,
                                             file);
         break;
diff --git a/ompi/mca/io/base/io_base_find_available.c b/ompi/mca/io/base/io_base_find_available.c
index 502140f2dee..bdf71a9a326 100644
--- a/ompi/mca/io/base/io_base_find_available.c
+++ b/ompi/mca/io/base/io_base_find_available.c
@@ -10,6 +10,8 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -39,7 +41,7 @@
 static int init_query(const mca_base_component_t *ls,
                       bool enable_progress_threads,
                       bool enable_mpi_threads);
-static int init_query_2_0_0(const mca_base_component_t *ls,
+static int init_query_3_0_0(const mca_base_component_t *ls,
                             bool enable_progress_threads,
                             bool enable_mpi_threads);
 
@@ -106,10 +108,10 @@ static int init_query(const mca_base_component_t *m,
     /* This component has already been successfully opened.  So now
        query it. */
 
-    if (2 == m->mca_type_major_version &&
+    if (3 == m->mca_type_major_version &&
         0 == m->mca_type_minor_version &&
         0 == m->mca_type_release_version) {
-        ret = init_query_2_0_0(m, enable_progress_threads,
+        ret = init_query_3_0_0(m, enable_progress_threads,
                                enable_mpi_threads);
     } else {
         /* Unrecognized io API version */
@@ -142,14 +144,14 @@ static int init_query(const mca_base_component_t *m,
 
 
 /*
- * Query a specific component, io v2.0.0
+ * Query a specific component, io v3.0.0
  */
-static int init_query_2_0_0(const mca_base_component_t *component,
+static int init_query_3_0_0(const mca_base_component_t *component,
                             bool enable_progress_threads,
                             bool enable_mpi_threads)
 {
-    mca_io_base_component_2_0_0_t *io =
-	(mca_io_base_component_2_0_0_t *) component;
+    mca_io_base_component_3_0_0_t *io =
+	(mca_io_base_component_3_0_0_t *) component;
 
     return io->io_init_query(enable_progress_threads,
                              enable_mpi_threads);
diff --git a/ompi/mca/io/base/io_base_register_datarep.c b/ompi/mca/io/base/io_base_register_datarep.c
index 8890e06b989..806c23bc09f 100644
--- a/ompi/mca/io/base/io_base_register_datarep.c
+++ b/ompi/mca/io/base/io_base_register_datarep.c
@@ -12,6 +12,8 @@
  * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -37,7 +39,7 @@ mca_io_base_register_datarep(const char *datarep,
 {
     mca_base_component_list_item_t *cli;
     const mca_base_component_t *component;
-    const mca_io_base_component_2_0_0_t *v200;
+    const mca_io_base_component_3_0_0_t *v200;
     int tmp, ret = OMPI_SUCCESS;
 
     /* Find the maximum additional number of bytes required by all io
@@ -50,7 +52,7 @@ mca_io_base_register_datarep(const char *datarep,
         if (component->mca_type_major_version == 2 &&
             component->mca_type_minor_version == 0 &&
             component->mca_type_release_version == 0) {
-            v200 = (mca_io_base_component_2_0_0_t *) component;
+            v200 = (mca_io_base_component_3_0_0_t *) component;
 
             /* return first non-good error-code */
             tmp = v200->io_register_datarep(datarep, read_fn, write_fn,
diff --git a/ompi/mca/io/io.h b/ompi/mca/io/io.h
index bd7762c798a..1900f1c40c1 100644
--- a/ompi/mca/io/io.h
+++ b/ompi/mca/io/io.h
@@ -17,6 +17,8 @@
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016-2019 IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -43,7 +45,7 @@ struct mca_io_base_delete_t;
 /*
  * Forward declarations of things declared in this file
  */
-struct mca_io_base_module_2_0_0_t;
+struct mca_io_base_module_3_0_0_t;
 union mca_io_base_modules_t;
 
 
@@ -59,7 +61,7 @@ union mca_io_base_modules_t;
  */
 enum mca_io_base_version_t {
     MCA_IO_BASE_V_NONE,
-    MCA_IO_BASE_V_2_0_0,
+    MCA_IO_BASE_V_3_0_0,
 
     MCA_IO_BASE_V_MAX
 };
@@ -73,18 +75,18 @@ typedef enum mca_io_base_version_t mca_io_base_version_t;
  * Macro for use in components that are of type io
  * 1-1 mapping of all MPI-IO functions
  */
-#define MCA_IO_BASE_VERSION_2_0_0 \
-    OMPI_MCA_BASE_VERSION_2_1_0("io", 2, 0, 0)
+#define MCA_IO_BASE_VERSION_3_0_0 \
+    OMPI_MCA_BASE_VERSION_2_1_0("io", 3, 0, 0)
 
 /*
  * Component
  */
 
-struct mca_io_base_module_2_0_0_t;
+struct mca_io_base_module_3_0_0_t;
 typedef int (*mca_io_base_component_init_query_fn_t)
     (bool enable_progress_threads, bool enable_mpi_threads);
-typedef const struct mca_io_base_module_2_0_0_t *
-    (*mca_io_base_component_file_query_2_0_0_fn_t)
+typedef const struct mca_io_base_module_3_0_0_t *
+    (*mca_io_base_component_file_query_3_0_0_fn_t)
     (struct ompi_file_t *file, struct mca_io_base_file_t **private_data,
      int *priority);
 typedef int (*mca_io_base_component_file_unquery_fn_t)
@@ -110,12 +112,12 @@ typedef int (*mca_io_base_component_register_datarep_fn_t)(
 
 
 /* IO component version and interface functions. */
-struct mca_io_base_component_2_0_0_t {
+struct mca_io_base_component_3_0_0_t {
     mca_base_component_t io_version;
     mca_base_component_data_t io_data;
 
     mca_io_base_component_init_query_fn_t io_init_query;
-    mca_io_base_component_file_query_2_0_0_fn_t io_file_query;
+    mca_io_base_component_file_query_3_0_0_fn_t io_file_query;
     mca_io_base_component_file_unquery_fn_t io_file_unquery;
 
     mca_io_base_component_file_delete_query_fn_t io_delete_query;
@@ -124,14 +126,14 @@ struct mca_io_base_component_2_0_0_t {
 
     mca_io_base_component_register_datarep_fn_t io_register_datarep;
 };
-typedef struct mca_io_base_component_2_0_0_t mca_io_base_component_2_0_0_t;
+typedef struct mca_io_base_component_3_0_0_t mca_io_base_component_3_0_0_t;
 
 
 /*
  * All component versions
  */
 union mca_io_base_components_t {
-    mca_io_base_component_2_0_0_t v2_0_0;
+    mca_io_base_component_3_0_0_t v3_0_0;
 };
 typedef union mca_io_base_components_t mca_io_base_components_t;
 
@@ -169,64 +171,64 @@ typedef int (*mca_io_base_module_file_get_view_fn_t)
 
 typedef int (*mca_io_base_module_file_read_at_fn_t)
     (struct ompi_file_t *fh, MPI_Offset offset, void *buf,
-     int count, struct ompi_datatype_t *datatype,
+     size_t count, struct ompi_datatype_t *datatype,
      struct ompi_status_public_t *status);
 typedef int (*mca_io_base_module_file_read_at_all_fn_t)
     (struct ompi_file_t *fh, MPI_Offset offset, void *buf,
-     int count, struct ompi_datatype_t *datatype,
+     size_t count, struct ompi_datatype_t *datatype,
      struct ompi_status_public_t *status);
 typedef int (*mca_io_base_module_file_write_at_fn_t)
     (struct ompi_file_t *fh, MPI_Offset offset, const void *buf,
-     int count, struct ompi_datatype_t *datatype,
+     size_t count, struct ompi_datatype_t *datatype,
      struct ompi_status_public_t *status);
 typedef int (*mca_io_base_module_file_write_at_all_fn_t)
     (struct ompi_file_t *fh, MPI_Offset offset, const void *buf,
-     int count, struct ompi_datatype_t *datatype,
+     size_t count, struct ompi_datatype_t *datatype,
      struct ompi_status_public_t *status);
 
 typedef int (*mca_io_base_module_file_iread_at_fn_t)
     (struct ompi_file_t *fh, MPI_Offset offset, void *buf,
-     int count, struct ompi_datatype_t *datatype,
+     size_t count, struct ompi_datatype_t *datatype,
      struct ompi_request_t **request);
 typedef int (*mca_io_base_module_file_iwrite_at_fn_t)
     (struct ompi_file_t *fh, MPI_Offset offset, const void *buf,
-     int count, struct ompi_datatype_t *datatype,
+     size_t count, struct ompi_datatype_t *datatype,
      struct ompi_request_t **request);
 
 typedef int (*mca_io_base_module_file_iread_at_all_fn_t)
     (struct ompi_file_t *fh, MPI_Offset offset, void *buf,
-     int count, struct ompi_datatype_t *datatype,
+     size_t count, struct ompi_datatype_t *datatype,
      struct ompi_request_t **request);
 typedef int (*mca_io_base_module_file_iwrite_at_all_fn_t)
     (struct ompi_file_t *fh, MPI_Offset offset, const void *buf,
-     int count, struct ompi_datatype_t *datatype,
+     size_t count, struct ompi_datatype_t *datatype,
      struct ompi_request_t **request);
 
 typedef int (*mca_io_base_module_file_read_fn_t)
-    (struct ompi_file_t *fh, void *buf, int count, struct ompi_datatype_t *
+    (struct ompi_file_t *fh, void *buf, size_t count, struct ompi_datatype_t *
      datatype, struct ompi_status_public_t *status);
 typedef int (*mca_io_base_module_file_read_all_fn_t)
-    (struct ompi_file_t *fh, void *buf, int count, struct ompi_datatype_t *
+    (struct ompi_file_t *fh, void *buf, size_t count, struct ompi_datatype_t *
      datatype, struct ompi_status_public_t *status);
 typedef int (*mca_io_base_module_file_write_fn_t)
-    (struct ompi_file_t *fh, const void *buf, int count, struct ompi_datatype_t *
+    (struct ompi_file_t *fh, const void *buf, size_t count, struct ompi_datatype_t *
      datatype, struct ompi_status_public_t *status);
 typedef int (*mca_io_base_module_file_write_all_fn_t)
-    (struct ompi_file_t *fh, const void *buf, int count, struct ompi_datatype_t *
+    (struct ompi_file_t *fh, const void *buf, size_t count, struct ompi_datatype_t *
      datatype, struct ompi_status_public_t *status);
 
 typedef int (*mca_io_base_module_file_iread_fn_t)
-    (struct ompi_file_t *fh, void *buf, int count,
+    (struct ompi_file_t *fh, void *buf, size_t count,
      struct ompi_datatype_t *datatype, struct ompi_request_t **request);
 typedef int (*mca_io_base_module_file_iwrite_fn_t)
-    (struct ompi_file_t *fh, const void *buf, int count,
+    (struct ompi_file_t *fh, const void *buf, size_t count,
      struct ompi_datatype_t *datatype, struct ompi_request_t **request);
 
 typedef int (*mca_io_base_module_file_iread_all_fn_t)
-    (struct ompi_file_t *fh, void *buf, int count,
+    (struct ompi_file_t *fh, void *buf, size_t count,
      struct ompi_datatype_t *datatype, struct ompi_request_t **request);
 typedef int (*mca_io_base_module_file_iwrite_all_fn_t)
-    (struct ompi_file_t *fh, const void *buf, int count,
+    (struct ompi_file_t *fh, const void *buf, size_t count,
      struct ompi_datatype_t *datatype, struct ompi_request_t **request);
 
 typedef int (*mca_io_base_module_file_seek_fn_t)
@@ -237,22 +239,22 @@ typedef int (*mca_io_base_module_file_get_byte_offset_fn_t)
     (struct ompi_file_t *fh, MPI_Offset offset, MPI_Offset *disp);
 
 typedef int (*mca_io_base_module_file_read_shared_fn_t)
-    (struct ompi_file_t *fh, void *buf, int count,
+    (struct ompi_file_t *fh, void *buf, size_t count,
      struct ompi_datatype_t *datatype, struct ompi_status_public_t *status);
 typedef int (*mca_io_base_module_file_write_shared_fn_t)
-    (struct ompi_file_t *fh, const void *buf, int count,
+    (struct ompi_file_t *fh, const void *buf, size_t count,
      struct ompi_datatype_t *datatype, struct ompi_status_public_t *status);
 typedef int (*mca_io_base_module_file_iread_shared_fn_t)
-    (struct ompi_file_t *fh, void *buf, int count,
+    (struct ompi_file_t *fh, void *buf, size_t count,
      struct ompi_datatype_t *datatype, struct ompi_request_t **request);
 typedef int (*mca_io_base_module_file_iwrite_shared_fn_t)
-    (struct ompi_file_t *fh, const void *buf, int count,
+    (struct ompi_file_t *fh, const void *buf, size_t count,
      struct ompi_datatype_t *datatype, struct ompi_request_t **request);
 typedef int (*mca_io_base_module_file_read_ordered_fn_t)
-    (struct ompi_file_t *fh, void *buf, int count,
+    (struct ompi_file_t *fh, void *buf, size_t count,
      struct ompi_datatype_t *datatype, struct ompi_status_public_t *status);
 typedef int (*mca_io_base_module_file_write_ordered_fn_t)
-    (struct ompi_file_t *fh, const void *buf, int count,
+    (struct ompi_file_t *fh, const void *buf, size_t count,
      struct ompi_datatype_t *datatype, struct ompi_status_public_t *status);
 typedef int (*mca_io_base_module_file_seek_shared_fn_t)
     (struct ompi_file_t *fh, MPI_Offset offset, int whence);
@@ -261,31 +263,31 @@ typedef int (*mca_io_base_module_file_get_position_shared_fn_t)
 
 typedef int (*mca_io_base_module_file_read_at_all_begin_fn_t)
     (struct ompi_file_t *fh, MPI_Offset offset, void *buf,
-     int count, struct ompi_datatype_t *datatype);
+     size_t count, struct ompi_datatype_t *datatype);
 typedef int (*mca_io_base_module_file_read_at_all_end_fn_t)
     (struct ompi_file_t *fh, void *buf, struct ompi_status_public_t *status);
 typedef int (*mca_io_base_module_file_write_at_all_begin_fn_t)
     (struct ompi_file_t *fh, MPI_Offset offset, const void *buf,
-     int count, struct ompi_datatype_t *datatype);
+     size_t count, struct ompi_datatype_t *datatype);
 typedef int (*mca_io_base_module_file_write_at_all_end_fn_t)
     (struct ompi_file_t *fh, const void *buf, struct ompi_status_public_t *status);
 typedef int (*mca_io_base_module_file_read_all_begin_fn_t)
-    (struct ompi_file_t *fh, void *buf, int count,
+    (struct ompi_file_t *fh, void *buf, size_t count,
      struct ompi_datatype_t *datatype);
 typedef int (*mca_io_base_module_file_read_all_end_fn_t)
     (struct ompi_file_t *fh, void *buf, struct ompi_status_public_t *status);
 typedef int (*mca_io_base_module_file_write_all_begin_fn_t)
-    (struct ompi_file_t *fh, const void *buf, int count,
+    (struct ompi_file_t *fh, const void *buf, size_t count,
      struct ompi_datatype_t *datatype);
 typedef int (*mca_io_base_module_file_write_all_end_fn_t)
     (struct ompi_file_t *fh, const void *buf, struct ompi_status_public_t *status);
 typedef int (*mca_io_base_module_file_read_ordered_begin_fn_t)
-    (struct ompi_file_t *fh, void *buf, int count,
+    (struct ompi_file_t *fh, void *buf, size_t count,
      struct ompi_datatype_t *datatype);
 typedef int (*mca_io_base_module_file_read_ordered_end_fn_t)
     (struct ompi_file_t *fh, void *buf, struct ompi_status_public_t *status);
 typedef int (*mca_io_base_module_file_write_ordered_begin_fn_t)
-    (struct ompi_file_t *fh, const void *buf, int count,
+    (struct ompi_file_t *fh, const void *buf, size_t count,
      struct ompi_datatype_t *datatype);
 typedef int (*mca_io_base_module_file_write_ordered_end_fn_t)
     (struct ompi_file_t *fh, const void *buf, struct ompi_status_public_t *status);
@@ -300,7 +302,7 @@ typedef int (*mca_io_base_module_file_get_atomicity_fn_t)
     (struct ompi_file_t *fh, int *flag);
 typedef int (*mca_io_base_module_file_sync_fn_t)(struct ompi_file_t *fh);
 
-struct mca_io_base_module_2_0_0_t {
+struct mca_io_base_module_3_0_0_t {
 
     /* Back-ends to MPI API calls (pretty much a 1-to-1 mapping) */
 
@@ -369,14 +371,14 @@ struct mca_io_base_module_2_0_0_t {
     mca_io_base_module_file_get_atomicity_fn_t        io_module_file_get_atomicity;
     mca_io_base_module_file_sync_fn_t                 io_module_file_sync;
 };
-typedef struct mca_io_base_module_2_0_0_t mca_io_base_module_2_0_0_t;
+typedef struct mca_io_base_module_3_0_0_t mca_io_base_module_3_0_0_t;
 
 
 /*
  * All module versions
  */
 union mca_io_base_modules_t {
-    mca_io_base_module_2_0_0_t v2_0_0;
+    mca_io_base_module_3_0_0_t v3_0_0;
 };
 typedef union mca_io_base_modules_t mca_io_base_modules_t;
 
diff --git a/ompi/mca/io/ompio/io_ompio.c b/ompi/mca/io/ompio/io_ompio.c
index 8a806861db0..506b6897e46 100644
--- a/ompi/mca/io/ompio/io_ompio.c
+++ b/ompi/mca/io/ompio/io_ompio.c
@@ -15,6 +15,7 @@
  * Copyright (c) 2012-2013 Inria.  All rights reserved.
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -61,8 +62,8 @@ int ompi_io_ompio_generate_current_file_view (struct ompio_file_t *fh,
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
 
-    sum_previous_counts = fh->f_position_in_file_view;
-    j = fh->f_index_in_file_view;
+    sum_previous_counts = fh->f_fview.f_position_in_file_view;
+    j = fh->f_fview.f_index_in_file_view;
     bytes_to_write = max_data;
     k = 0;
 
@@ -79,46 +80,48 @@ int ompi_io_ompio_generate_current_file_view (struct ompio_file_t *fh,
             }
         }
 
-        if (fh->f_decoded_iov[j].iov_len -
-            (fh->f_total_bytes - sum_previous_counts) <= 0) {
-            sum_previous_counts += fh->f_decoded_iov[j].iov_len;
+        if (fh->f_fview.f_decoded_iov[j].iov_len -
+            (fh->f_fview.f_total_bytes - sum_previous_counts) <= 0) {
+            sum_previous_counts += fh->f_fview.f_decoded_iov[j].iov_len;
             j = j + 1;
-            if (j == (int)fh->f_iov_count) {
+            if (j == (int)fh->f_fview.f_iov_count) {
                 j = 0;
                 sum_previous_counts = 0;
-                fh->f_offset += fh->f_view_extent;
-                fh->f_position_in_file_view = sum_previous_counts;
-                fh->f_index_in_file_view = j;
-                fh->f_total_bytes = 0;
+                fh->f_fview.f_offset += fh->f_fview.f_view_extent;
+                fh->f_fview.f_position_in_file_view = sum_previous_counts;
+                fh->f_fview.f_index_in_file_view = j;
+                fh->f_fview.f_total_bytes = 0;
             }
         }
 
-        disp = (ptrdiff_t)(fh->f_decoded_iov[j].iov_base) +
-            (fh->f_total_bytes - sum_previous_counts);
-        iov[k].iov_base = (IOVBASE_TYPE *)(intptr_t)(disp + fh->f_offset);
+        disp = (ptrdiff_t)(fh->f_fview.f_decoded_iov[j].iov_base) +
+            (fh->f_fview.f_total_bytes - sum_previous_counts);
+        iov[k].iov_base = (IOVBASE_TYPE *)(intptr_t)(disp + fh->f_fview.f_offset);
 
-        if ((fh->f_decoded_iov[j].iov_len -
-             (fh->f_total_bytes - sum_previous_counts))
+        if ((fh->f_fview.f_decoded_iov[j].iov_len -
+             (fh->f_fview.f_total_bytes - sum_previous_counts))
             >= bytes_to_write) {
             iov[k].iov_len = bytes_to_write;
         }
         else {
-            iov[k].iov_len =  fh->f_decoded_iov[j].iov_len -
-                (fh->f_total_bytes - sum_previous_counts);
+            iov[k].iov_len =  fh->f_fview.f_decoded_iov[j].iov_len -
+                (fh->f_fview.f_total_bytes - sum_previous_counts);
         }
 
-        fh->f_total_bytes += iov[k].iov_len;
+        fh->f_fview.f_total_bytes += iov[k].iov_len;
         bytes_to_write -= iov[k].iov_len;
         k = k + 1;
     }
-    fh->f_position_in_file_view = sum_previous_counts;
-    fh->f_index_in_file_view = j;
+    fh->f_fview.f_position_in_file_view = sum_previous_counts;
+    fh->f_fview.f_index_in_file_view = j;
     *iov_count = k;
     *f_iov = iov;
 
     if (mca_io_ompio_record_offset_info){
 
 	int tot_entries=0, *recvcounts=NULL, *displs=NULL;
+	ompi_count_array_t recvcounts_desc;
+	ompi_disp_array_t displs_desc;
 	mca_io_ompio_offlen_array_t *per_process=NULL;
 	mca_io_ompio_offlen_array_t  *all_process=NULL;
 	int *sorted=NULL, *column_list=NULL, *values=NULL;
@@ -248,12 +251,14 @@ int ompi_io_ompio_generate_current_file_view (struct ompio_file_t *fh,
                 }
             }
 	}
+	OMPI_COUNT_ARRAY_INIT(&recvcounts_desc, recvcounts);
+	OMPI_DISP_ARRAY_INIT(&displs_desc, displs);
 	fh->f_comm->c_coll->coll_gatherv (per_process,
 					 k,
 					 io_array_type,
 					 all_process,
-					 recvcounts,
-					 displs,
+					 recvcounts_desc,
+					 displs_desc,
 					 io_array_type,
 					 OMPIO_ROOT,
 					 fh->f_comm,
@@ -536,6 +541,9 @@ int mca_io_ompio_get_mca_parameter_value ( char *mca_parameter_name, int name_le
     else if ( !strncmp ( mca_parameter_name, "cycle_buffer_size", name_length )) {
         return mca_io_ompio_cycle_buffer_size;
     }
+    else if ( !strncmp ( mca_parameter_name, "pipeline_buffer_size", name_length )) {
+        return mca_io_ompio_pipeline_buffer_size;
+    }
     else if ( !strncmp ( mca_parameter_name, "max_aggregators_ratio", name_length )) {
         return mca_io_ompio_max_aggregators_ratio;
     }
@@ -548,6 +556,9 @@ int mca_io_ompio_get_mca_parameter_value ( char *mca_parameter_name, int name_le
     else if ( !strncmp ( mca_parameter_name, "coll_timing_info", name_length )) {
         return mca_io_ompio_coll_timing_info;
     }
+    else if ( !strncmp (mca_parameter_name, "use_accelerator_buffers", name_length)) {
+	return mca_io_ompio_use_accelerator_buffers;
+    }
     else {
         opal_output (1, "Error in mca_io_ompio_get_mca_parameter_value: unknown parameter name");
     }
diff --git a/ompi/mca/io/ompio/io_ompio.h b/ompi/mca/io/ompio/io_ompio.h
index 4c019240892..d38bfb1acb3 100644
--- a/ompi/mca/io/ompio/io_ompio.h
+++ b/ompi/mca/io/ompio/io_ompio.h
@@ -14,6 +14,9 @@
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
+ * Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -45,6 +48,7 @@
 #include "ompi/mca/common/ompio/common_ompio.h"
 
 extern int mca_io_ompio_cycle_buffer_size;
+extern int mca_io_ompio_pipeline_buffer_size;
 extern int mca_io_ompio_bytes_per_agg;
 extern int mca_io_ompio_num_aggregators;
 extern int mca_io_ompio_record_offset_info;
@@ -53,7 +57,7 @@ extern int mca_io_ompio_max_aggregators_ratio;
 extern int mca_io_ompio_aggregators_cutoff_threshold;
 extern int mca_io_ompio_overwrite_amode;
 extern int mca_io_ompio_verbose_info_parsing;
-
+extern int mca_io_ompio_use_accelerator_buffers;
 OMPI_DECLSPEC extern int mca_io_ompio_coll_timing_info;
 
 #define QUEUESIZE 2048
@@ -63,6 +67,7 @@ OMPI_DECLSPEC extern int mca_io_ompio_coll_timing_info;
  */
 #define OMPIO_PREALLOC_MAX_BUF_SIZE   33554432
 #define OMPIO_DEFAULT_CYCLE_BUF_SIZE  536870912
+#define OMPIO_DEFAULT_PIPELINE_BUF_SIZE 1048576
 #define OMPIO_TAG_GATHER              -100
 #define OMPIO_TAG_GATHERV             -101
 #define OMPIO_TAG_BCAST               -102
@@ -83,13 +88,12 @@ OMPI_DECLSPEC extern int mca_io_ompio_coll_timing_info;
 
 BEGIN_C_DECLS
 
-OMPI_DECLSPEC extern mca_io_base_component_2_0_0_t mca_io_ompio_component;
+OMPI_DECLSPEC extern mca_io_base_component_3_0_0_t mca_io_ompio_component;
 /*
  * global variables, instantiated in module.c
  */
-extern opal_mutex_t mca_io_ompio_mutex;
-extern mca_io_base_module_2_0_0_t mca_io_ompio_module;
-OMPI_DECLSPEC extern mca_io_base_component_2_0_0_t mca_io_ompio_component;
+extern mca_io_base_module_3_0_0_t mca_io_ompio_module;
+OMPI_DECLSPEC extern mca_io_base_component_3_0_0_t mca_io_ompio_component;
 
 /*Used in extracting offset adj-matrix*/
 typedef struct mca_io_ompio_offlen_array_t{
@@ -179,92 +183,92 @@ int mca_io_ompio_file_get_view (struct ompi_file_t *fh,
 int mca_io_ompio_file_read_at (struct ompi_file_t *fh,
                                OMPI_MPI_OFFSET_TYPE offset,
                                void *buf,
-                               int count,
+                               size_t count,
                                struct ompi_datatype_t *datatype,
                                ompi_status_public_t *status);
 int mca_io_ompio_file_read_at_all (struct ompi_file_t *fh,
                                    OMPI_MPI_OFFSET_TYPE offset,
                                    void *buf,
-                                   int count,
+                                   size_t count,
                                    struct ompi_datatype_t *datatype,
                                    ompi_status_public_t *status);
 int mca_io_ompio_file_write_at (struct ompi_file_t *fh,
                                 OMPI_MPI_OFFSET_TYPE offset,
                                 const void *buf,
-                                int count,
+                                size_t count,
                                 struct ompi_datatype_t *datatype,
                                 ompi_status_public_t *status);
 int mca_io_ompio_file_write_at_all (struct ompi_file_t *fh,
                                     OMPI_MPI_OFFSET_TYPE offset,
                                     const void *buf,
-                                    int count,
+                                    size_t count,
                                     struct ompi_datatype_t *datatype,
                                     ompi_status_public_t *status);
 int mca_io_ompio_file_iread_at (struct ompi_file_t *fh,
                                 OMPI_MPI_OFFSET_TYPE offset,
                                 void *buf,
-                                int count,
+                                size_t count,
                                 struct ompi_datatype_t *datatype,
                                 ompi_request_t **request);
 int mca_io_ompio_file_iwrite_at (struct ompi_file_t *fh,
                                  OMPI_MPI_OFFSET_TYPE offset,
                                  const void *buf,
-                                 int count,
+                                 size_t count,
                                  struct ompi_datatype_t *datatype,
                                  ompi_request_t **request);
 
 /* Section 9.4.3 */
 int mca_io_ompio_file_read (struct ompi_file_t *fh,
                             void *buf,
-                            int count,
+                            size_t count,
                             struct ompi_datatype_t *datatype,
                             ompi_status_public_t *status);
 int mca_io_ompio_file_read_all (struct ompi_file_t *fh,
                                 void *buf,
-                                int count,
+                                size_t count,
                                 struct ompi_datatype_t *datatype,
                                 ompi_status_public_t *status);
 int mca_io_ompio_file_iread_all (ompi_file_t *fh,
 				void *buf,
-				int count,
+				size_t count,
 				struct ompi_datatype_t *datatype,
 				 ompi_request_t **request);
 int mca_io_ompio_file_iread_at_all (ompi_file_t *fh,
 				    OMPI_MPI_OFFSET_TYPE offset,
 				    void *buf,
-				    int count,
+				    size_t count,
 				    struct ompi_datatype_t *datatype,
 				    ompi_request_t **request);
 
 int mca_io_ompio_file_write (struct ompi_file_t *fh,
                              const void *buf,
-                             int count,
+                             size_t count,
                              struct ompi_datatype_t *datatype,
                              ompi_status_public_t *status);
 int mca_io_ompio_file_write_all (struct ompi_file_t *fh,
                                  const void *buf,
-                                 int count,
+                                 size_t count,
                                  struct ompi_datatype_t *datatype,
                                  ompi_status_public_t *status);
 int mca_io_ompio_file_iwrite_all (ompi_file_t *fh,
 				  const void *buf,
-				  int count,
+				  size_t count,
 				  struct ompi_datatype_t *datatype,
 				  ompi_request_t **request);
 int mca_io_ompio_file_iwrite_at_all (ompi_file_t *fh,
 				     OMPI_MPI_OFFSET_TYPE offset,
 				     const void *buf,
-				     int count,
+				     size_t count,
 				     struct ompi_datatype_t *datatype,
 				     ompi_request_t **request);
 int mca_io_ompio_file_iread (struct ompi_file_t *fh,
                              void *buf,
-                             int count,
+                             size_t count,
                              struct ompi_datatype_t *datatype,
                              ompi_request_t **request);
 int mca_io_ompio_file_iwrite (struct ompi_file_t *fh,
                               const void *buf,
-                              int count,
+                              size_t count,
                               struct ompi_datatype_t *datatype,
                               ompi_request_t **request);
 int mca_io_ompio_file_seek (struct ompi_file_t *fh,
@@ -279,32 +283,32 @@ int mca_io_ompio_file_get_byte_offset (struct ompi_file_t *fh,
 /* Section 9.4.4 */
 int mca_io_ompio_file_read_shared (struct ompi_file_t *fh,
                                    void *buf,
-                                   int count,
+                                   size_t count,
                                    struct ompi_datatype_t *datatype,
                                    ompi_status_public_t *status);
 int mca_io_ompio_file_write_shared (struct ompi_file_t *fh,
                                     const void *buf,
-                                    int count,
+                                    size_t count,
                                     struct ompi_datatype_t *datatype,
                                     ompi_status_public_t *status);
 int mca_io_ompio_file_iread_shared (struct ompi_file_t *fh,
                                     void *buf,
-                                    int count,
+                                    size_t count,
                                     struct ompi_datatype_t *datatype,
                                     ompi_request_t **request);
 int mca_io_ompio_file_iwrite_shared (struct ompi_file_t *fh,
                                      const void *buf,
-                                     int count,
+                                     size_t count,
                                      struct ompi_datatype_t *datatype,
                                      ompi_request_t **request);
 int mca_io_ompio_file_read_ordered (struct ompi_file_t *fh,
                                     void *buf,
-                                    int count,
+                                    size_t count,
                                     struct ompi_datatype_t *datatype,
                                     ompi_status_public_t *status);
 int mca_io_ompio_file_write_ordered (struct ompi_file_t *fh,
                                      const void *buf,
-                                     int count,
+                                     size_t count,
                                      struct ompi_datatype_t *datatype,
                                      ompi_status_public_t *status);
 int mca_io_ompio_file_seek_shared (struct ompi_file_t *fh,
@@ -317,7 +321,7 @@ int mca_io_ompio_file_get_position_shared (struct ompi_file_t *fh,
 int mca_io_ompio_file_read_at_all_begin (struct ompi_file_t *fh,
                                          OMPI_MPI_OFFSET_TYPE offset,
                                          void *buf,
-                                         int count,
+                                         size_t count,
                                          struct ompi_datatype_t *datatype);
 int mca_io_ompio_file_read_at_all_end (struct ompi_file_t *fh,
                                        void *buf,
@@ -325,35 +329,35 @@ int mca_io_ompio_file_read_at_all_end (struct ompi_file_t *fh,
 int mca_io_ompio_file_write_at_all_begin (struct ompi_file_t *fh,
                                           OMPI_MPI_OFFSET_TYPE offset,
                                           const void *buf,
-                                          int count,
+                                          size_t count,
                                           struct ompi_datatype_t *datatype);
 int mca_io_ompio_file_write_at_all_end (struct ompi_file_t *fh,
                                         const void *buf,
                                         ompi_status_public_t *status);
 int mca_io_ompio_file_read_all_begin (struct ompi_file_t *fh,
                                       void *buf,
-                                      int count,
+                                      size_t count,
                                       struct ompi_datatype_t *datatype);
 int mca_io_ompio_file_read_all_end (struct ompi_file_t *fh,
                                     void *buf,
                                     ompi_status_public_t *status);
 int mca_io_ompio_file_write_all_begin (struct ompi_file_t *fh,
                                        const void *buf,
-                                       int count,
+                                       size_t count,
                                        struct ompi_datatype_t *datatype);
 int mca_io_ompio_file_write_all_end (struct ompi_file_t *fh,
                                      const void *buf,
                                      ompi_status_public_t *status);
 int mca_io_ompio_file_read_ordered_begin (struct ompi_file_t *fh,
                                           void *buf,
-                                          int count,
+                                          size_t count,
                                           struct ompi_datatype_t *datatype);
 int mca_io_ompio_file_read_ordered_end (struct ompi_file_t *fh,
                                         void *buf,
                                         ompi_status_public_t *status);
 int mca_io_ompio_file_write_ordered_begin (struct ompi_file_t *fh,
                                            const void *buf,
-                                           int count,
+                                           size_t count,
                                            struct ompi_datatype_t *datatype);
 int mca_io_ompio_file_write_ordered_end (struct ompi_file_t *fh,
                                          const void *buf,
diff --git a/ompi/mca/io/ompio/io_ompio_component.c b/ompi/mca/io/ompio/io_ompio_component.c
index 09d18aad75c..0e6fcd01f6c 100644
--- a/ompi/mca/io/ompio/io_ompio_component.c
+++ b/ompi/mca/io/ompio/io_ompio_component.c
@@ -13,10 +13,13 @@
  * Copyright (c) 2008-2020 University of Houston. All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
  * Copyright (c) 2018      DataDirect Networks. All rights reserved.
+ * Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -36,12 +39,8 @@
 #include "ompi/mca/common/ompio/common_ompio_request.h"
 #include "ompi/mca/common/ompio/common_ompio_buffer.h"
 
-#ifdef HAVE_IME_NATIVE_H
-#include "ompi/mca/fs/ime/fs_ime.h"
-#endif
-
-
 int mca_io_ompio_cycle_buffer_size = OMPIO_DEFAULT_CYCLE_BUF_SIZE;
+int mca_io_ompio_pipeline_buffer_size = OMPIO_DEFAULT_PIPELINE_BUF_SIZE;
 int mca_io_ompio_bytes_per_agg = OMPIO_PREALLOC_MAX_BUF_SIZE;
 int mca_io_ompio_num_aggregators = -1;
 int mca_io_ompio_record_offset_info = 0;
@@ -50,7 +49,7 @@ int mca_io_ompio_max_aggregators_ratio=8;
 int mca_io_ompio_aggregators_cutoff_threshold=3;
 int mca_io_ompio_overwrite_amode = 1;
 int mca_io_ompio_verbose_info_parsing = 0;
-
+int mca_io_ompio_use_accelerator_buffers = 0;
 int mca_io_ompio_grouping_option=5;
 
 /*
@@ -61,7 +60,7 @@ static int open_component(void);
 static int close_component(void);
 static int init_query(bool enable_progress_threads,
                       bool enable_mpi_threads);
-static const struct mca_io_base_module_2_0_0_t *
+static const struct mca_io_base_module_3_0_0_t *
 file_query (struct ompi_file_t *file,
             struct mca_io_base_file_t **private_data,
             int *priority);
@@ -91,13 +90,7 @@ static int io_progress(void);
 static int priority_param = 30;
 static int delete_priority_param = 30;
 
-
-/*
- * Global, component-wide OMPIO mutex because OMPIO is not thread safe
- */
-opal_mutex_t mca_io_ompio_mutex = {{0}};
-
-
+static opal_mutex_t mca_io_ompio_mutex = OPAL_MUTEX_STATIC_INIT;
 
 /*
  * Public string showing this component's version number
@@ -106,12 +99,12 @@ const char *mca_io_ompio_component_version_string =
 "OMPI/MPI OMPIO io MCA component version " OMPI_VERSION;
 
 
-mca_io_base_component_2_0_0_t mca_io_ompio_component = {
+mca_io_base_component_3_0_0_t mca_io_ompio_component = {
     /* First, the mca_base_component_t struct containing meta information
        about the component itself */
 
     .io_version = {
-        MCA_IO_BASE_VERSION_2_0_0,
+        MCA_IO_BASE_VERSION_3_0_0,
         .mca_component_name = "ompio",
         MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION,
                               OMPI_RELEASE_VERSION),
@@ -182,6 +175,16 @@ static int register_component(void)
                                            MCA_BASE_VAR_SCOPE_READONLY,
                                            &mca_io_ompio_cycle_buffer_size);
 
+    mca_io_ompio_pipeline_buffer_size = OMPIO_DEFAULT_PIPELINE_BUF_SIZE;
+    (void) mca_base_component_var_register(&mca_io_ompio_component.io_version,
+                                           "pipeline_buffer_size",
+                                           "Size of temporary buffer used by individual reads/writes "
+                                           "in the pipeline protocol",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           &mca_io_ompio_pipeline_buffer_size);
+
     mca_io_ompio_bytes_per_agg = OMPIO_PREALLOC_MAX_BUF_SIZE;
     (void) mca_base_component_var_register(&mca_io_ompio_component.io_version,
                                            "bytes_per_agg",
@@ -229,7 +232,7 @@ static int register_component(void)
     mca_io_ompio_aggregators_cutoff_threshold=3;
     (void) mca_base_component_var_register(&mca_io_ompio_component.io_version,
                                            "aggregators_cutoff_threshold",
-                                           "Relativ cutoff threshold for incrementing the number of aggregators "
+                                           "Relative cutoff threshold for incrementing the number of aggregators "
                                            "in the simple aggregator selection algorithm (5). Lower value "
                                            "for this parameter will lead to higher no. of aggregators.",
                                            MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
@@ -260,14 +263,19 @@ static int register_component(void)
                                            MCA_BASE_VAR_SCOPE_READONLY,
                                            &mca_io_ompio_verbose_info_parsing);
 
+    mca_io_ompio_use_accelerator_buffers = 0;
+    (void) mca_base_component_var_register(&mca_io_ompio_component.io_version,
+                                           "use_accelerator_buffers", "Allow using accelerator buffers"
+                                           "for data aggregation in collective I/O if input buffer is device memory",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_READONLY, &mca_io_ompio_use_accelerator_buffers);
+
     return OMPI_SUCCESS;
 }
 
 static int open_component(void)
 {
-    /* Create the mutex */
-    OBJ_CONSTRUCT(&mca_io_ompio_mutex, opal_mutex_t);
-
     mca_common_ompio_request_init ();
 
     return mca_common_ompio_set_callbacks(ompi_io_ompio_generate_current_file_view,
@@ -279,11 +287,6 @@ static int close_component(void)
 {
     mca_common_ompio_request_fini ();
     mca_common_ompio_buffer_alloc_fini();
-    OBJ_DESTRUCT(&mca_io_ompio_mutex);
-
-#ifdef HAVE_IME_NATIVE_H
-    mca_fs_ime_native_fini();
-#endif
 
     return OMPI_SUCCESS;
 }
@@ -296,7 +299,7 @@ static int init_query(bool enable_progress_threads,
 }
 
 
-static const struct mca_io_base_module_2_0_0_t *
+static const struct mca_io_base_module_3_0_0_t *
 file_query(struct ompi_file_t *file,
            struct mca_io_base_file_t **private_data,
            int *priority)
diff --git a/ompi/mca/io/ompio/io_ompio_file_open.c b/ompi/mca/io/ompio/io_ompio_file_open.c
index e020f5f4c9a..76e4eb6c497 100644
--- a/ompi/mca/io/ompio/io_ompio_file_open.c
+++ b/ompi/mca/io/ompio/io_ompio_file_open.c
@@ -55,7 +55,7 @@ int mca_io_ompio_file_open (ompi_communicator_t *comm,
 
     /* No locking required for file_open according to my understanding
        There is virtually no way on how to reach this point from multiple
-       threads simultaniously 
+       threads simultaneously 
     */
     data = (mca_common_ompio_data_t *) fh->f_io_selected_data;
     if ( NULL == data ) {
@@ -411,25 +411,28 @@ static void mca_io_ompio_file_get_eof_offset (ompio_file_t *fh,
     size_t k=0, blocklen=0;
     size_t index_in_file_view=0;
 
-    in_offset -= fh->f_disp;
-    if ( fh->f_view_size  > 0 ) {
+    in_offset -= fh->f_fview.f_disp;
+    if ( fh->f_fview.f_view_size  > 0 ) {
         /* starting offset of the current copy of the filew view */
-        start_offset = in_offset / fh->f_view_extent;
+        start_offset = (in_offset / fh->f_fview.f_view_extent) * fh->f_fview.f_view_extent;
         
         index_in_file_view = 0;
         /* determine block id that the offset is located in and
            the starting offset of that block */
-        while ( offset <= in_offset && index_in_file_view < fh->f_iov_count) {
-            prev_offset = offset;
-            offset = start_offset + (OMPI_MPI_OFFSET_TYPE)(intptr_t) fh->f_decoded_iov[index_in_file_view++].iov_base;
+        while (offset <= in_offset && index_in_file_view < fh->f_fview.f_iov_count) {
+            offset = start_offset + (OMPI_MPI_OFFSET_TYPE)(intptr_t) fh->f_fview.f_decoded_iov[index_in_file_view++].iov_base;
+	    if (offset <= in_offset) {
+		prev_offset = offset;
+	    }
         }
         
         offset = prev_offset;
-        blocklen = fh->f_decoded_iov[index_in_file_view-1].iov_len;
-        while (  offset <= in_offset && k <= blocklen )  {
+        assert(index_in_file_view > 0);
+        blocklen = fh->f_fview.f_decoded_iov[index_in_file_view-1].iov_len;
+        while (offset <= in_offset && k <= blocklen)  {
             prev_offset = offset;
-            offset += fh->f_etype_size;
-            k += fh->f_etype_size;
+            offset += fh->f_fview.f_etype_size;
+            k += fh->f_fview.f_etype_size;
         }
 
         *out_offset = prev_offset;
@@ -443,12 +446,13 @@ int mca_io_ompio_file_seek (ompi_file_t *fh,
 {
     int ret = OMPI_SUCCESS;
     mca_common_ompio_data_t *data;
-    OMPI_MPI_OFFSET_TYPE offset, temp_offset, temp_offset2;
+    OMPI_MPI_OFFSET_TYPE offset, temp_offset2;
+    OMPI_MPI_OFFSET_TYPE temp_offset = 0;
 
     data = (mca_common_ompio_data_t *) fh->f_io_selected_data;
 
     OPAL_THREAD_LOCK(&fh->f_lock);
-    offset = off * data->ompio_fh.f_etype_size;
+    offset = off * data->ompio_fh.f_fview.f_etype_size;
 
     switch(whence) {
     case MPI_SEEK_SET:
@@ -460,7 +464,7 @@ int mca_io_ompio_file_seek (ompi_file_t *fh,
     case MPI_SEEK_CUR:
         ret = mca_common_ompio_file_get_position (&data->ompio_fh,
                                                   &temp_offset);
-        offset += temp_offset * data->ompio_fh.f_etype_size;
+        offset += temp_offset * data->ompio_fh.f_fview.f_etype_size;
         if (offset < 0) {
             OPAL_THREAD_UNLOCK(&fh->f_lock);
             return OMPI_ERROR;
@@ -483,7 +487,7 @@ int mca_io_ompio_file_seek (ompi_file_t *fh,
     }
 
     ret = mca_common_ompio_set_explicit_offset (&data->ompio_fh,
-                                             offset/data->ompio_fh.f_etype_size);
+                                             offset/data->ompio_fh.f_fview.f_etype_size);
     OPAL_THREAD_UNLOCK(&fh->f_lock);
 
     return ret;
@@ -519,19 +523,24 @@ int mca_io_ompio_file_get_byte_offset (ompi_file_t *fh,
     data = (mca_common_ompio_data_t *) fh->f_io_selected_data;
 
     OPAL_THREAD_LOCK(&fh->f_lock);
-    temp_offset = (long) data->ompio_fh.f_view_extent *
-        (offset*data->ompio_fh.f_etype_size / data->ompio_fh.f_view_size);
+    if ( data->ompio_fh.f_fview.f_view_size == 0 ) {
+        *disp = 0;
+        OPAL_THREAD_UNLOCK(&fh->f_lock);
+        return OMPI_SUCCESS;
+    }
+    temp_offset = (long) data->ompio_fh.f_fview.f_view_extent *
+        (offset*data->ompio_fh.f_fview.f_etype_size / data->ompio_fh.f_fview.f_view_size);
     if ( 0 > temp_offset ) {
         OPAL_THREAD_UNLOCK(&fh->f_lock);
         return MPI_ERR_ARG;
     }
     
-    i = (offset*data->ompio_fh.f_etype_size) % data->ompio_fh.f_view_size;
+    i = (offset*data->ompio_fh.f_fview.f_etype_size) % data->ompio_fh.f_fview.f_view_size;
     index = 0;
     k = 0;
 
     while (1) {
-        k = data->ompio_fh.f_decoded_iov[index].iov_len;
+        k = data->ompio_fh.f_fview.f_decoded_iov[index].iov_len;
         if (i >= k) {
             i -= k;
             index++;
@@ -546,8 +555,8 @@ int mca_io_ompio_file_get_byte_offset (ompi_file_t *fh,
         }
     }
 
-    *disp = data->ompio_fh.f_disp + temp_offset +
-        (OMPI_MPI_OFFSET_TYPE)(intptr_t)data->ompio_fh.f_decoded_iov[index].iov_base + k;
+    *disp = data->ompio_fh.f_fview.f_disp + temp_offset +
+        (OMPI_MPI_OFFSET_TYPE)(intptr_t)data->ompio_fh.f_fview.f_decoded_iov[index].iov_base + k;
     OPAL_THREAD_UNLOCK(&fh->f_lock);
 
     return OMPI_SUCCESS;
@@ -599,7 +608,7 @@ int mca_io_ompio_file_get_position_shared (ompi_file_t *fp,
     }
     OPAL_THREAD_LOCK(&fp->f_lock);
     ret = shared_fp_base_module->sharedfp_get_position(fh,offset);
-    *offset = *offset / fh->f_etype_size;
+    *offset = *offset / fh->f_fview.f_etype_size;
     OPAL_THREAD_UNLOCK(&fp->f_lock);
 
     return ret;
diff --git a/ompi/mca/io/ompio/io_ompio_file_read.c b/ompi/mca/io/ompio/io_ompio_file_read.c
index 8c3978c6878..39a11965711 100644
--- a/ompi/mca/io/ompio/io_ompio_file_read.c
+++ b/ompi/mca/io/ompio/io_ompio_file_read.c
@@ -12,6 +12,8 @@
  * Copyright (c) 2008-2019 University of Houston. All rights reserved.
  * Copyright (c) 2017-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -53,7 +55,7 @@
 
 int mca_io_ompio_file_read (ompi_file_t *fp,
 			    void *buf,
-			    int count,
+			    size_t count,
 			    struct ompi_datatype_t *datatype,
 			    ompi_status_public_t *status)
 {
@@ -71,7 +73,7 @@ int mca_io_ompio_file_read (ompi_file_t *fp,
 int mca_io_ompio_file_read_at (ompi_file_t *fh,
 			       OMPI_MPI_OFFSET_TYPE offset,
 			       void *buf,
-			       int count,
+			       size_t count,
 			       struct ompi_datatype_t *datatype,
 			       ompi_status_public_t * status)
 {
@@ -88,7 +90,7 @@ int mca_io_ompio_file_read_at (ompi_file_t *fh,
 
 int mca_io_ompio_file_iread (ompi_file_t *fh,
 			     void *buf,
-			     int count,
+			     size_t count,
 			     struct ompi_datatype_t *datatype,
 			     ompi_request_t **request)
 {
@@ -107,7 +109,7 @@ int mca_io_ompio_file_iread (ompi_file_t *fh,
 int mca_io_ompio_file_iread_at (ompi_file_t *fh,
 				OMPI_MPI_OFFSET_TYPE offset,
 				void *buf,
-				int count,
+				size_t count,
 				struct ompi_datatype_t *datatype,
 				ompi_request_t **request)
 {
@@ -127,7 +129,7 @@ int mca_io_ompio_file_iread_at (ompi_file_t *fh,
 /******************************************************/
 int mca_io_ompio_file_read_all (ompi_file_t *fh,
 				void *buf,
-				int count,
+				size_t count,
 				struct ompi_datatype_t *datatype,
 				ompi_status_public_t * status)
 {
@@ -155,7 +157,7 @@ int mca_io_ompio_file_read_all (ompi_file_t *fh,
 
 int mca_io_ompio_file_iread_all (ompi_file_t *fh,
 				void *buf,
-				int count,
+				size_t count,
 				struct ompi_datatype_t *datatype,
 				ompi_request_t **request)
 {
@@ -179,7 +181,7 @@ int mca_io_ompio_file_iread_all (ompi_file_t *fh,
 int mca_io_ompio_file_read_at_all (ompi_file_t *fh,
 				   OMPI_MPI_OFFSET_TYPE offset,
 				   void *buf,
-				   int count,
+				   size_t count,
 				   struct ompi_datatype_t *datatype,
 				   ompi_status_public_t * status)
 {
@@ -197,7 +199,7 @@ int mca_io_ompio_file_read_at_all (ompi_file_t *fh,
 int mca_io_ompio_file_iread_at_all (ompi_file_t *fh,
 				    OMPI_MPI_OFFSET_TYPE offset,
 				    void *buf,
-				    int count,
+				    size_t count,
 				    struct ompi_datatype_t *datatype,
 				    ompi_request_t **request)
 {
@@ -217,7 +219,7 @@ int mca_io_ompio_file_iread_at_all (ompi_file_t *fh,
 /******************************************************/
 int mca_io_ompio_file_read_shared (ompi_file_t *fp,
 				   void *buf,
-				   int count,
+				   size_t count,
 				   struct ompi_datatype_t *datatype,
 				   ompi_status_public_t * status)
 {
@@ -244,7 +246,7 @@ int mca_io_ompio_file_read_shared (ompi_file_t *fp,
 
 int mca_io_ompio_file_iread_shared (ompi_file_t *fh,
 				    void *buf,
-				    int count,
+				    size_t count,
 				    struct ompi_datatype_t *datatype,
 				    ompi_request_t **request)
 {
@@ -271,7 +273,7 @@ int mca_io_ompio_file_iread_shared (ompi_file_t *fh,
 
 int mca_io_ompio_file_read_ordered (ompi_file_t *fh,
 				    void *buf,
-				    int count,
+				    size_t count,
 				    struct ompi_datatype_t *datatype,
 				    ompi_status_public_t * status)
 {
@@ -297,7 +299,7 @@ int mca_io_ompio_file_read_ordered (ompi_file_t *fh,
 
 int mca_io_ompio_file_read_ordered_begin (ompi_file_t *fh,
 					  void *buf,
-					  int count,
+					  size_t count,
 					  struct ompi_datatype_t *datatype)
 {
     int ret = OMPI_SUCCESS;
@@ -351,7 +353,7 @@ int mca_io_ompio_file_read_ordered_end (ompi_file_t *fh,
 /**********************************************************************/
 int mca_io_ompio_file_read_all_begin (ompi_file_t *fh,
 				      void *buf,
-				      int count,
+				      size_t count,
 				      struct ompi_datatype_t *datatype)
 {
     int ret = OMPI_SUCCESS;
@@ -360,13 +362,18 @@ int mca_io_ompio_file_read_all_begin (ompi_file_t *fh,
 
     data = (mca_common_ompio_data_t *) fh->f_io_selected_data;
     fp = &data->ompio_fh;
-    if ( true == fp->f_split_coll_in_use ) {
-	printf("Only one split collective I/O operation allowed per file handle at any given point in time!\n");
-	return MPI_ERR_OTHER;
+
+    OPAL_THREAD_LOCK(&fh->f_lock);
+    if (true == fp->f_split_coll_in_use) {
+        printf("Only one split collective I/O operation allowed per file handle at any given point in time!\n");
+        OPAL_THREAD_UNLOCK(&fh->f_lock);
+        return MPI_ERR_OTHER;
     }
-    /* No need for locking fh->f_lock, that is done in file_iread_all */
-    ret = mca_io_ompio_file_iread_all ( fh, buf, count, datatype, &fp->f_split_coll_req );
     fp->f_split_coll_in_use = true;
+    OPAL_THREAD_UNLOCK(&fh->f_lock);
+
+    /* No need for locking fh->f_lock for the operation itself, that is done in io_ompio_file_iread_all */
+    ret = mca_io_ompio_file_iread_all ( fh, buf, count, datatype, &fp->f_split_coll_req );
 
     return ret;
 }
@@ -391,7 +398,7 @@ int mca_io_ompio_file_read_all_end (ompi_file_t *fh,
 int mca_io_ompio_file_read_at_all_begin (ompi_file_t *fh,
 					 OMPI_MPI_OFFSET_TYPE offset,
 					 void *buf,
-					 int count,
+					 size_t count,
 					 struct ompi_datatype_t *datatype)
 {
     int ret = OMPI_SUCCESS;
@@ -400,14 +407,17 @@ int mca_io_ompio_file_read_at_all_begin (ompi_file_t *fh,
     data = (mca_common_ompio_data_t *) fh->f_io_selected_data;
     fp = &data->ompio_fh;
 
-    if ( true == fp->f_split_coll_in_use ) {
-	printf("Only one split collective I/O operation allowed per file handle at any given point in time!\n");
-	return MPI_ERR_REQUEST;
-    }
     OPAL_THREAD_LOCK(&fh->f_lock);
+    if (true == fp->f_split_coll_in_use) {
+        printf("Only one split collective I/O operation allowed per file handle at any given point in time!\n");
+        OPAL_THREAD_UNLOCK(&fh->f_lock);
+        return MPI_ERR_OTHER;
+    }
+
+    fp->f_split_coll_in_use = true;
     ret = mca_common_ompio_file_iread_at_all ( fp, offset, buf, count, datatype, &fp->f_split_coll_req );
     OPAL_THREAD_UNLOCK(&fh->f_lock);
-    fp->f_split_coll_in_use = true;
+
     return ret;
 }
 
diff --git a/ompi/mca/io/ompio/io_ompio_file_set_view.c b/ompi/mca/io/ompio/io_ompio_file_set_view.c
index 5340370faa5..5a4f8136295 100644
--- a/ompi/mca/io/ompio/io_ompio_file_set_view.c
+++ b/ompi/mca/io/ompio/io_ompio_file_set_view.c
@@ -104,7 +104,7 @@ int mca_io_ompio_file_get_view (struct ompi_file_t *fp,
     fh = &data->ompio_fh;
 
     OPAL_THREAD_LOCK(&fp->f_lock);
-    *disp = fh->f_disp;
+    *disp = fh->f_fview.f_disp;
     datatype_duplicate (fh->f_etype, etype);
     datatype_duplicate (fh->f_orig_filetype, filetype);
     strcpy (datarep, fh->f_datarep);
diff --git a/ompi/mca/io/ompio/io_ompio_file_write.c b/ompi/mca/io/ompio/io_ompio_file_write.c
index 36d9f17d433..8efa2a5a3ec 100644
--- a/ompi/mca/io/ompio/io_ompio_file_write.c
+++ b/ompi/mca/io/ompio/io_ompio_file_write.c
@@ -12,6 +12,8 @@
  * Copyright (c) 2008-2019 University of Houston. All rights reserved.
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -56,7 +58,7 @@
 
 int mca_io_ompio_file_write (ompi_file_t *fp,
 			     const void *buf,
-			     int count,
+			     size_t count,
 			     struct ompi_datatype_t *datatype,
 			     ompi_status_public_t *status)
 {
@@ -76,7 +78,7 @@ int mca_io_ompio_file_write (ompi_file_t *fp,
 int mca_io_ompio_file_write_at (ompi_file_t *fh,
 				OMPI_MPI_OFFSET_TYPE offset,
 				const void *buf,
-				int count,
+				size_t count,
 				struct ompi_datatype_t *datatype,
 				ompi_status_public_t *status)
 {
@@ -93,7 +95,7 @@ int mca_io_ompio_file_write_at (ompi_file_t *fh,
 
 int mca_io_ompio_file_iwrite (ompi_file_t *fp,
 			      const void *buf,
-			      int count,
+			      size_t count,
 			      struct ompi_datatype_t *datatype,
 			      ompi_request_t **request)
 {
@@ -112,7 +114,7 @@ int mca_io_ompio_file_iwrite (ompi_file_t *fp,
 int mca_io_ompio_file_iwrite_at (ompi_file_t *fh,
 				 OMPI_MPI_OFFSET_TYPE offset,
 				 const void *buf,
-				 int count,
+				 size_t count,
 				 struct ompi_datatype_t *datatype,
 				 ompi_request_t **request)
 {
@@ -133,7 +135,7 @@ int mca_io_ompio_file_iwrite_at (ompi_file_t *fh,
 
 int mca_io_ompio_file_write_all (ompi_file_t *fh,
 				 const void *buf,
-				 int count,
+				 size_t count,
 				 struct ompi_datatype_t *datatype,
 				 ompi_status_public_t *status)
 {
@@ -162,7 +164,7 @@ int mca_io_ompio_file_write_all (ompi_file_t *fh,
 int mca_io_ompio_file_write_at_all (ompi_file_t *fh,
 				    OMPI_MPI_OFFSET_TYPE offset,
 				    const void *buf,
-				    int count,
+				    size_t count,
 				    struct ompi_datatype_t *datatype,
 				    ompi_status_public_t *status)
 {
@@ -179,7 +181,7 @@ int mca_io_ompio_file_write_at_all (ompi_file_t *fh,
 
 int mca_io_ompio_file_iwrite_all (ompi_file_t *fh,
 				  const void *buf,
-				  int count,
+				  size_t count,
 				  struct ompi_datatype_t *datatype,
 				  ompi_request_t **request)
 {
@@ -203,7 +205,7 @@ int mca_io_ompio_file_iwrite_all (ompi_file_t *fh,
 int mca_io_ompio_file_iwrite_at_all (ompi_file_t *fh,
 				     OMPI_MPI_OFFSET_TYPE offset,
 				     const void *buf,
-				     int count,
+				     size_t count,
 				     struct ompi_datatype_t *datatype,
 				     ompi_request_t **request)
 {
@@ -224,7 +226,7 @@ int mca_io_ompio_file_iwrite_at_all (ompi_file_t *fh,
 
 int mca_io_ompio_file_write_shared (ompi_file_t *fp,
 				    const void *buf,
-				    int count,
+				    size_t count,
 				    struct ompi_datatype_t *datatype,
 				    ompi_status_public_t * status)
 {
@@ -251,7 +253,7 @@ int mca_io_ompio_file_write_shared (ompi_file_t *fp,
 
 int mca_io_ompio_file_iwrite_shared (ompi_file_t *fp,
 				     const void *buf,
-				     int count,
+				     size_t count,
 				     struct ompi_datatype_t *datatype,
 				     ompi_request_t **request)
 {
@@ -278,7 +280,7 @@ int mca_io_ompio_file_iwrite_shared (ompi_file_t *fp,
 
 int mca_io_ompio_file_write_ordered (ompi_file_t *fp,
 				     const void *buf,
-				     int count,
+				     size_t count,
 				     struct ompi_datatype_t *datatype,
 				     ompi_status_public_t * status)
 {
@@ -305,7 +307,7 @@ int mca_io_ompio_file_write_ordered (ompi_file_t *fp,
 
 int mca_io_ompio_file_write_ordered_begin (ompi_file_t *fp,
 					   const void *buf,
-					   int count,
+					   size_t count,
 					   struct ompi_datatype_t *datatype)
 {
     int ret = OMPI_SUCCESS;
@@ -359,7 +361,7 @@ int mca_io_ompio_file_write_ordered_end (ompi_file_t *fp,
 /**********************************************************************/
 int mca_io_ompio_file_write_all_begin (ompi_file_t *fh,
 				       const void *buf,
-				       int count,
+				       size_t count,
 				       struct ompi_datatype_t *datatype)
 {
     int ret = OMPI_SUCCESS;
@@ -368,13 +370,18 @@ int mca_io_ompio_file_write_all_begin (ompi_file_t *fh,
 
     data = (mca_common_ompio_data_t *) fh->f_io_selected_data;
     fp = &data->ompio_fh;
-    if ( true == fp->f_split_coll_in_use ) {
-	printf("Only one split collective I/O operation allowed per file handle at any given point in time!\n");
-	return MPI_ERR_OTHER;
+
+    OPAL_THREAD_LOCK(&fh->f_lock);
+    if (true == fp->f_split_coll_in_use) {
+        printf("Only one split collective I/O operation allowed per file handle at any given point in time!\n");
+        OPAL_THREAD_UNLOCK(&fh->f_lock);
+        return MPI_ERR_OTHER;
     }
-    /* No need for locking fh->f_lock, that is done in file_iwrite_all */
-    ret = mca_io_ompio_file_iwrite_all ( fh, buf, count, datatype, &fp->f_split_coll_req );
     fp->f_split_coll_in_use = true;
+    OPAL_THREAD_UNLOCK(&fh->f_lock);
+
+    /* No need for locking fh->f_lock the operation itself, that is done in io_ompio_file_iwrite_all */
+    ret = mca_io_ompio_file_iwrite_all ( fh, buf, count, datatype, &fp->f_split_coll_req );
 
     return ret;
 }
@@ -400,7 +407,7 @@ int mca_io_ompio_file_write_all_end (ompi_file_t *fh,
 int mca_io_ompio_file_write_at_all_begin (ompi_file_t *fh,
 					  OMPI_MPI_OFFSET_TYPE offset,
 					  const void *buf,
-					  int count,
+					  size_t count,
 					  struct ompi_datatype_t *datatype)
 {
     int ret = OMPI_SUCCESS;
@@ -410,14 +417,15 @@ int mca_io_ompio_file_write_at_all_begin (ompi_file_t *fh,
     data = (mca_common_ompio_data_t *) fh->f_io_selected_data;
     fp = &data->ompio_fh;
 
-    if ( true == fp->f_split_coll_in_use ) {
-	printf("Only one split collective I/O operation allowed per file handle at any given point in time!\n");
-	return MPI_ERR_REQUEST;
-    }
     OPAL_THREAD_LOCK(&fh->f_lock);
+    if (true == fp->f_split_coll_in_use) {
+        printf("Only one split collective I/O operation allowed per file handle at any given point in time!\n");
+        OPAL_THREAD_UNLOCK(&fh->f_lock);
+        return MPI_ERR_OTHER;
+    }
+    fp->f_split_coll_in_use = true;
     ret = mca_common_ompio_file_iwrite_at_all ( fp, offset, buf, count, datatype, &fp->f_split_coll_req );
     OPAL_THREAD_UNLOCK(&fh->f_lock);
-    fp->f_split_coll_in_use = true;
 
     return ret;
 }
diff --git a/ompi/mca/io/ompio/io_ompio_module.c b/ompi/mca/io/ompio/io_ompio_module.c
index d80c93fb6ff..4b53e70d237 100644
--- a/ompi/mca/io/ompio/io_ompio_module.c
+++ b/ompi/mca/io/ompio/io_ompio_module.c
@@ -11,6 +11,8 @@
  *                         All rights reserved.
  * Copyright (c) 2008-2011 University of Houston. All rights reserved.
  * Copyright (c) 2016-2019 IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -27,7 +29,7 @@
 /*
  * The OMPIO module operations
  */
-mca_io_base_module_2_0_0_t mca_io_ompio_module = {
+mca_io_base_module_3_0_0_t mca_io_ompio_module = {
 
     mca_io_ompio_file_open,
     mca_io_ompio_file_close,
diff --git a/ompi/mca/io/romio321/.gitignore b/ompi/mca/io/romio321/.gitignore
deleted file mode 100644
index beaeb9ba92d..00000000000
--- a/ompi/mca/io/romio321/.gitignore
+++ /dev/null
@@ -1,16 +0,0 @@
-/Makefile
-/.deps
-/*.bb
-/*.bbg
-/*.gcda
-/*.gcno
-/.libs
-/.libstamp*
-/*.lo
-/.*-cache
-.state-cache
-romio/version.m4
-romio/confdb/._l*
-romio/confdb/config.rpath
-romio/adio/include/romioconf.h.in
-!romio/mpi2-other/**/Makefile.in
diff --git a/ompi/mca/io/romio321/Makefile.am b/ompi/mca/io/romio321/Makefile.am
deleted file mode 100644
index b6bdb972dd4..00000000000
--- a/ompi/mca/io/romio321/Makefile.am
+++ /dev/null
@@ -1,55 +0,0 @@
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2017      IBM Corporation.  All rights reserved.
-# Copyright (c) 2017      Research Organization for Information Science
-#                         and Technology (RIST). All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-sources =
-include src/Makefile.extra
-
-# Make the output library in this directory, and name it either
-# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
-# (for static builds).
-
-if MCA_BUILD_ompi_io_romio321_DSO
-lib =
-lib_sources =
-component = mca_io_romio321.la
-component_sources = $(sources)
-else
-lib = libmca_io_romio321.la
-lib_sources = $(sources)
-component =
-component_sources =
-endif
-
-libs = $(top_builddir)/3rd-party/romio321/libromio_dist.la
-
-mcacomponentdir = $(ompilibdir)
-mcacomponent_LTLIBRARIES = $(component)
-mca_io_romio321_la_SOURCES = $(component_sources)
-mca_io_romio321_la_LIBADD = $(libs)
-mca_io_romio321_la_DEPENDENCIES = $(libs)
-mca_io_romio321_la_LDFLAGS = -module -avoid-version
-
-noinst_LTLIBRARIES = $(lib)
-libmca_io_romio321_la_SOURCES = $(lib_sources)
-libmca_io_romio321_la_LIBADD = $(libs)
-libmca_io_romio321_la_DEPENDENCIES = $(libs)
-libmca_io_romio321_la_LDFLAGS = -module -avoid-version
diff --git a/ompi/mca/io/romio321/REFRESH_NOTES.txt b/ompi/mca/io/romio321/REFRESH_NOTES.txt
deleted file mode 100644
index e4dfa704068..00000000000
--- a/ompi/mca/io/romio321/REFRESH_NOTES.txt
+++ /dev/null
@@ -1,39 +0,0 @@
-1. prepare new romio directory
-cp -a <old romio> <new romio>
-# cp -a ompi/mca/io/romio ompi/mca/io/romio321
-rm -rf <new romio>/romio
-# rm -rf ompi/mca/io/romio321/romio
-git add <new romio>
-# git add ompi/mca/io/romio321
-# git commit
-
-2. import new romio
-cp -a <mpich dir>/src/mpi/romio <new romio>
-# cp -a /.../mpich-3.2b1/src/mpi/romio ompi/mca/io/romio321
-git add <new romio>
-# git add ompi/mca/io/romio321
-# git commit
-
-3. patch romio
-cd <new romio>
-# cd ompi/mca/io/romio321
-# git show 7a7968f02e2bad912426d1535119fb30d881b460 | patch -p5
-# cd ../../../..
-git add <new romio>
-# git add ompi/mca/io/romio321
-# git commit
-
-4. prepare ompi to use new romio
-vi <new romio>/Makefile.am <new romio>/configure.m4 <new romio>/src/Makefile.extra
-# vi ompi/mca/io/romio321/Makefile.am ompi/mca/io/romio321/configure.m4 ompi/mca/io/romio321/src/Makefile.extra
-git add <new romio>
-# git add ompi/mca/io/romio321
-# git commit
-
-
-5. remove old romio
-git rm -r <old romio>
-# git rm -r ompi/mca/io/romio
-# git commit
-rm -rf <old romio>
-# rm -rf ompi/mca/io/romio
diff --git a/ompi/mca/io/romio321/configure.m4 b/ompi/mca/io/romio321/configure.m4
deleted file mode 100644
index ae48dbef073..00000000000
--- a/ompi/mca/io/romio321/configure.m4
+++ /dev/null
@@ -1,113 +0,0 @@
-# -*- shell-script -*-
-#
-# Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2008-2015 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2015-2017 Research Organization for Information Science
-#                         and Technology (RIST). All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-AC_DEFUN([MCA_ompi_io_romio321_POST_CONFIG], [
-    AM_CONDITIONAL([MCA_io_romio321_SHOULD_BUILD], [test $1 -eq 1])
-])
-
-
-# MCA_io_romio321_CONFIG([action-if-found], [action-if-not-found])
-# -----------------------------------------------------------
-AC_DEFUN([MCA_ompi_io_romio321_CONFIG],[
-    AC_CONFIG_FILES([ompi/mca/io/romio321/Makefile])
-
-    OPAL_3RDPARTY_SUBDIRS="$OPAL_3RDPARTY_SUBDIRS romio321"
-    OPAL_3RDPARTY_DIST_SUBDIRS="$OPAL_3RDPARTY_DIST_SUBDIRS romio321"
-    
-    OPAL_VAR_SCOPE_PUSH([io_romio321_flags io_romio321_flags_define io_romio321_happy io_romio321_save_LIBS])
-    AC_ARG_ENABLE([io-romio],
-                  [AS_HELP_STRING([--disable-io-romio],
-                                  [Disable the ROMIO MPI-IO component])])
-    AC_ARG_WITH([io-romio-flags],
-                [AS_HELP_STRING([--with-io-romio-flags=FLAGS],
-                                [Pass FLAGS to the ROMIO distribution configuration script])])
-    AC_DEFINE_UNQUOTED([MCA_io_romio321_USER_CONFIGURE_FLAGS], ["$with_io_romio_flags"], [Set of user-defined configure flags given to ROMIOs configure script via --with-io-romio-flags])
-    AC_MSG_CHECKING([if want ROMIO component])
-    AS_IF([test "$enable_io_romio" = "no"],
-           [AC_MSG_RESULT([no])
-            $2],
-           [AC_MSG_RESULT([yes])
-            AC_MSG_CHECKING([if MPI profiling is enabled])
-            AS_IF([test "$enable_mpi_profile" = "no"],
-                  [AC_MSG_RESULT([no])
-                   AC_MSG_WARN([*** The ROMIO io component requires the MPI profiling layer])
-                   AS_IF([test "$enable_io_romio" = "yes"],
-                         [AC_MSG_ERROR([*** ROMIO requested but not available.  Aborting])])
-                   $2],
-                  [AC_MSG_RESULT([yes])
-
-                   AS_IF([test -n "$with_io_romio_flags" && test "$with_io_romio_flags" != "no"],
-                         [io_romio321_flags="$with_io_romio_flags $io_romio321_flags"],
-                         [io_romio321_flags=])
-                   # If ROMIO is going to end up in a DSO, all we need is
-                   # shared library-ized objects, as we're only building a
-                   # DSO (which is always shared).  Otherwise, build with
-                   # same flags as OMPI, as we might need any combination of
-                   # shared and static-ized objects...
-                   AS_IF([test "$compile_mode" = "dso"],
-                         [io_romio321_shared=enable
-                          io_romio321_static=disable],
-                         [AS_IF([test "$enable_shared" = "yes"],
-                                [io_romio321_shared=enable],
-                                [io_romio321_shared=disable])
-                          AS_IF([test "$enable_static" = "yes"],
-                                [io_romio321_static=enable],
-                                [io_romio321_static=disable])])
-                   AS_IF([test -n "$prefix" && test "$prefix" != "NONE"],
-                         [io_romio321_prefix_arg="--prefix=$prefix"],
-                         [io_romio321_prefix_arg=])
-
-                   AS_IF([test "$cross_compiling" = "yes"],
-                       [AS_IF([test ! -z $build], [io_romio321_flags="$io_romio321_flags --build=$build"])
-                        AS_IF([test ! -z $host], [io_romio321_flags="$io_romio321_flags --host=$host"])
-                        AS_IF([test ! -z $target], [io_romio321_flags="$io_romio321_flags --target=$target"])])
-                   AS_IF([test "$enable_grequest_extensions" = "yes"],
-                         [io_romio321_flags="$io_romio321_flags --enable-grequest-extensions"])
-                   io_romio321_flags_define="$io_romio321_flags FROM_OMPI=yes CC='$CC' CFLAGS='$CFLAGS -D__EXTENSIONS__' CPPFLAGS='$CPPFLAGS' FFLAGS='$FFLAGS' LDFLAGS='$LDFLAGS' --$io_romio321_shared-shared --$io_romio321_static-static $io_romio321_flags $io_romio321_prefix_arg --disable-aio --disable-weak-symbols --enable-strict --disable-f77 --disable-f90"
-                   AC_DEFINE_UNQUOTED([MCA_io_romio321_COMPLETE_CONFIGURE_FLAGS], ["$io_romio321_flags_define"], [Complete set of command line arguments given to ROMIOs configure script])
-
-                   io_romio321_flags="$io_romio321_flags FROM_OMPI=yes CC="'"'"$CC"'"'" CFLAGS="'"'"$CFLAGS -D__EXTENSIONS__"'"'" CPPFLAGS="'"'"$CPPFLAGS"'"'" FFLAGS="'"'"$FFLAGS"'"'" LDFLAGS="'"'"$LDFLAGS"'"'" --$io_romio321_shared-shared --$io_romio321_static-static $io_romio321_flags $io_romio321_prefix_arg --disable-aio --disable-weak-symbols --enable-strict --disable-f77 --disable-f90"
-
-                   opal_show_subtitle "Configuring ROMIO distribution"
-                   OPAL_CONFIG_SUBDIR([3rd-party/romio321],
-                                      [$io_romio321_flags],
-                                      [io_romio321_happy=1], [io_romio321_happy=0])
-
-                   AS_IF([test "$io_romio321_happy" = "1"],
-                         [ # grab the libraries list from ROMIO.  We don't
-                           # need this for building the component, as libtool
-                           # will figure that part out.  But we do need it for
-                           # the wrapper settings
-                          io_romio321_save_LIBS="$LIBS"
-                          LIBS=
-                          . 3rd-party/romio321/localdefs
-                          io_romio321_LIBS="$LIBS"
-                          LIBS="$io_romio321_save_LIBS"
-
-                          echo "ROMIO distribution configured successfully"
-                          $1],
-                         [AS_IF([test "$enable_io_romio" = "yes"],
-                                [AC_MSG_ERROR([ROMIO distribution did not configure successfully])],
-                                [AC_MSG_WARN([ROMIO distribution did not configure successfully])])
-                          $2])])])
-    OPAL_VAR_SCOPE_POP
-])
diff --git a/ompi/mca/io/romio321/ompi.patch b/ompi/mca/io/romio321/ompi.patch
deleted file mode 100644
index 9d6dd5ffc4c..00000000000
--- a/ompi/mca/io/romio321/ompi.patch
+++ /dev/null
@@ -1,717 +0,0 @@
-diff --git a/ompi/mca/io/romio321/romio/.config_params b/ompi/mca/io/romio321/romio/.config_params
-index 96f735f..fcc2f91 100644
---- a/ompi/mca/io/romio321/romio/.config_params
-+++ b/ompi/mca/io/romio321/romio/.config_params
-@@ -36,3 +36,4 @@ __sgi_mpi
- __hp_mpi
- __cray_mpi
- __lam_mpi
-+__open_mpi
-diff --git a/ompi/mca/io/romio321/romio/Makefile.am b/ompi/mca/io/romio321/romio/Makefile.am
-index b9d4e25..69d7014 100644
---- a/ompi/mca/io/romio321/romio/Makefile.am
-+++ b/ompi/mca/io/romio321/romio/Makefile.am
-@@ -1,9 +1,28 @@
- # -*- Mode: Makefile; -*-
-+# Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
-+#                         University Research and Technology
-+#                         Corporation.  All rights reserved.
-+# Copyright (c) 2004-2005 The University of Tennessee and The University
-+#                         of Tennessee Research Foundation.  All rights
-+#                         reserved.
-+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-+#                         University of Stuttgart.  All rights reserved.
-+# Copyright (c) 2004-2005 The Regents of the University of California.
-+#                         All rights reserved.
-+# Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
-+# $COPYRIGHT$
-+#
-+# Additional copyrights may follow
-+#
-+# $HEADER$
- #
- # (C) 2011 by Argonne National Laboratory.
- #     See COPYRIGHT in top-level directory.
- #
- 
-+# OMPI: include a top level makefile with some options
-+include $(top_srcdir)/Makefile.options
-+
- ## TODO: need to write an automakefile that handles two primary cases:
- ## 1) that ROMIO is being embedded within the MPI library, as in MPICH or Open
- ##    MPI
-@@ -17,7 +36,6 @@ ACLOCAL_AMFLAGS = -I confdb
- include_HEADERS = 
- nodist_include_HEADERS =
- noinst_HEADERS = 
--AM_CPPFLAGS =
- EXTRA_DIST =
- SUFFIXES = 
- doc1_src_txt =
-@@ -46,7 +64,9 @@ AM_CPPFLAGS += $(MPI_H_INCLUDE)
- # handle the "include" directory here
- AM_CPPFLAGS += -I$(top_builddir)/include -I$(top_srcdir)/include
- # nodist_ b/c these are created by config.status and should not be distributed
--nodist_include_HEADERS += include/mpio.h include/mpiof.h
-+# Open MPI: do not install mpio.h
-+noinst_HEADERS += include/mpio.h
-+noinst_HEADERS += include/io_romio_conv.h
- 
- # ------------------------------------------------------------------------
- 
-@@ -63,8 +83,8 @@ EXTRA_DIST += autogen.sh
- if BUILD_ROMIO_EMBEDDED
- # Build a libtool convenience library that the enclosing MPI implementation can
- # use by adding it to the right _LIBADD variable.
--noinst_LTLIBRARIES = libromio.la
--libromio_la_SOURCES = $(romio_mpi_sources) $(romio_other_sources) $(glue_sources)
-+noinst_LTLIBRARIES = libromio_dist.la
-+libromio_dist_la_SOURCES = $(romio_mpi_sources) $(romio_other_sources) $(glue_sources)
- 
- ## NOTE: ROMIO's old build system builds a bunch of _foo.o objects that contain
- ## PMPI_ implementations as well as calls to only other PMPI routines.  In
-diff --git a/ompi/mca/io/romio321/romio/adio/Makefile.mk b/ompi/mca/io/romio321/romio/adio/Makefile.mk
-index 505d518..ffc05cb 100644
---- a/ompi/mca/io/romio321/romio/adio/Makefile.mk
-+++ b/ompi/mca/io/romio321/romio/adio/Makefile.mk
-@@ -20,6 +20,7 @@ noinst_HEADERS +=                      \
-     adio/include/mpipr.h               \
-     adio/include/mpiu_greq.h           \
-     adio/include/nopackage.h           \
-+    adio/include/romioconf-undefs.h    \
-     adio/include/mpiu_external32.h     \
-     adio/include/hint_fns.h
- 
-diff --git a/ompi/mca/io/romio321/romio/adio/common/ad_end.c b/ompi/mca/io/romio321/romio/adio/common/ad_end.c
-index ea4dfeb..066c65c 100644
---- a/ompi/mca/io/romio321/romio/adio/common/ad_end.c
-+++ b/ompi/mca/io/romio321/romio/adio/common/ad_end.c
-@@ -16,7 +16,12 @@ void ADIO_End(int *error_code)
- 
-     /* if a default errhandler was set on MPI_FILE_NULL then we need to ensure
-      * that our reference to that errhandler is released */
-+/* Open MPI: The call to PMPI_File_set_errhandler has to be done in romio/src/io_romio_file_open.c
-+   in routine mca_io_romio_file_close()
-+*/
-+#if 0
-     PMPI_File_set_errhandler(MPI_FILE_NULL, MPI_ERRORS_RETURN);
-+#endif
- 
- /* delete the flattened datatype list */
-     curr = ADIOI_Flatlist;
-diff --git a/ompi/mca/io/romio321/romio/adio/common/ad_iread_coll.c b/ompi/mca/io/romio321/romio/adio/common/ad_iread_coll.c
-index 2ec23fb..8487c25 100644
---- a/ompi/mca/io/romio321/romio/adio/common/ad_iread_coll.c
-+++ b/ompi/mca/io/romio321/romio/adio/common/ad_iread_coll.c
-@@ -16,6 +16,8 @@
- #include "mpe.h"
- #endif
- 
-+#ifdef HAVE_MPI_GREQUEST_EXTENSIONS
-+
- /* ADIOI_GEN_IreadStridedColl */
- struct ADIOI_GEN_IreadStridedColl_vars {
-     /* requests */
-@@ -1340,3 +1342,4 @@ static int ADIOI_GEN_irc_wait_fn(int count, void **array_of_states,
-     return errcode;
- }
- 
-+#endif /* HAVE_MPI_GREQUEST_EXTENSIONS */
-diff --git a/ompi/mca/io/romio321/romio/adio/common/ad_iwrite_coll.c b/ompi/mca/io/romio321/romio/adio/common/ad_iwrite_coll.c
-index d275f78..9b27b42 100644
---- a/ompi/mca/io/romio321/romio/adio/common/ad_iwrite_coll.c
-+++ b/ompi/mca/io/romio321/romio/adio/common/ad_iwrite_coll.c
-@@ -13,6 +13,8 @@
- #include "mpe.h"
- #endif
- 
-+#ifdef HAVE_MPI_GREQUEST_EXTENSIONS
-+
- /* ADIOI_GEN_IwriteStridedColl */
- struct ADIOI_GEN_IwriteStridedColl_vars {
-     /* requests */
-@@ -1565,3 +1567,4 @@ static int ADIOI_GEN_iwc_wait_fn(int count, void **array_of_states,
-     return errcode;
- }
- 
-+#endif /* HAVE_MPI_GREQUEST_EXTENSIONS */
-diff --git a/ompi/mca/io/romio321/romio/adio/include/adioi.h b/ompi/mca/io/romio321/romio/adio/include/adioi.h
-index 0e91ead..13c0f25 100644
---- a/ompi/mca/io/romio321/romio/adio/include/adioi.h
-+++ b/ompi/mca/io/romio321/romio/adio/include/adioi.h
-@@ -440,18 +440,26 @@ void ADIOI_GEN_ReadStridedColl(ADIO_File fd, void *buf, int count,
-                        MPI_Datatype datatype, int file_ptr_type,
-                        ADIO_Offset offset, ADIO_Status *status, int
-                        *error_code);
-+#ifdef HAVE_MPI_GREQUEST_EXTENSIONS
- void ADIOI_GEN_IreadStridedColl(ADIO_File fd, void *buf, int count,
-                        MPI_Datatype datatype, int file_ptr_type,
-                        ADIO_Offset offset, MPI_Request *request,
-                        int *error_code);
-+#else
-+#define ADIOI_GEN_IreadStridedColl NULL
-+#endif
- void ADIOI_GEN_WriteStridedColl(ADIO_File fd, const void *buf, int count,
-                        MPI_Datatype datatype, int file_ptr_type,
-                        ADIO_Offset offset, ADIO_Status *status, int
-                        *error_code);
-+#ifdef HAVE_MPI_GREQUEST_EXTENSIONS
- void ADIOI_GEN_IwriteStridedColl(ADIO_File fd, const void *buf, int count,
-                        MPI_Datatype datatype, int file_ptr_type,
-                        ADIO_Offset offset, MPI_Request *request,
-                        int *error_code);
-+#else
-+#define ADIOI_GEN_IwriteStridedColl NULL
-+#endif
- void ADIOI_Calc_my_off_len(ADIO_File fd, int bufcount, MPI_Datatype
- 			    datatype, int file_ptr_type, ADIO_Offset 
- 			    offset, ADIO_Offset **offset_list_ptr, ADIO_Offset
-diff --git a/ompi/mca/io/romio321/romio/configure.ac b/ompi/mca/io/romio321/romio/configure.ac
-index 93f8b09..0b85d53 100644
---- a/ompi/mca/io/romio321/romio/configure.ac
-+++ b/ompi/mca/io/romio321/romio/configure.ac
-@@ -3,12 +3,21 @@
- # autoconf --localdir=../confdb configure.ac
- # (or wherever the confdb is)
- #
-+# irrelevant / unnecessary in an Open MPI environment, but are
-+# harmless and are left here solely for the sake of ease of future
-+# patching/importing.
- AC_PREREQ([2.63])
- 
--m4_include([version.m4])
--dnl 2nd arg is intentionally underquoted
-+# Open MPI: Modifications to this file were done on an "let's do the
-+# minimum possible" basis, not so that we can skip on the work or
-+# provide any less functionality, but more from a perspective that we
-+# want to be able to import new versions of ROMIO in as easy a fashion
-+# as possible.  Hence, there are some things in this file that are
-+# irrelevant / unnecessary in an Open MPI environment, but are
-+# harmless and are left here solely for the sake of ease of future
-+# patching/importing.
- AC_INIT([ROMIO],
--        MPICH_VERSION_m4,
-+        [Open MPI],
-         [discuss@mpich.org],
-         [romio],
-         [http://www.mpich.org/])
-@@ -22,7 +31,7 @@ dnl scripts.
- AC_CONFIG_AUX_DIR([confdb])
- AC_CONFIG_MACRO_DIR([confdb])
- 
--AM_INIT_AUTOMAKE([-Wall -Werror -Wno-portability-recursive foreign 1.12.3 silent-rules subdir-objects])
-+AM_INIT_AUTOMAKE([-Wall -Werror -Wno-portability-recursive foreign 1.12 silent-rules subdir-objects])
- AM_MAINTAINER_MODE([enable])
- 
- dnl must come before LT_INIT, which AC_REQUIREs AC_PROG_CC
-@@ -43,12 +52,15 @@ if test -n "$CONFIGURE_ARGS" ; then
- fi
- 
- AC_CONFIG_HEADER(adio/include/romioconf.h)
-+# Open MPI: modified AH_TOP
- AH_TOP([/*
-  *  (C) 2011 by Argonne National Laboratory.
-  *      See COPYRIGHT in top-level directory.
-  */
- #ifndef ROMIOCONF_H_INCLUDED
- #define ROMIOCONF_H_INCLUDED
-+
-+#include "romioconf-undefs.h"
- ])
- AH_BOTTOM([
- /* quash PACKAGE and PACKAGE_* vars, see MPICH top-level configure.ac for
-@@ -58,9 +70,37 @@ AH_BOTTOM([
- #endif /* !defined(ROMIOCONF_H_INCLUDED) */
- ])
- 
-+# Open MPI: this configure script doesn't seem to define these
-+# anywhere, so just do them manually here because "we know better"
-+# (i.e., Open MPI can be hard-wired to these values).
-+AC_DEFINE([HAVE_MPI_OFFSET], [1], [Will always be 1 - OMPI has MPI_OFFSET])
-+
-+# Open MPI: look for top Open MPI directory
-+AC_MSG_CHECKING([for Open MPI support files])
-+if test -f "$srcdir/../../../../../config/opal_mca.m4"; then
-+
-+    # This is needed for VPATH builds, so that it will -I the
-+    # appropriate include directory (don't know why automake
-+    # doesn't do this # automatically).
-+
-+    OMPI_TOP_SRCDIR='$(top_srcdir)/../../../../..'
-+    OMPI_TOP_BUILDDIR='$(top_builddir)/../../../../..'
-+    with_mpi="$OMPI_TOP_SRCDIR"
-+    AC_MSG_RESULT([in Open MPI source tree -- good])
-+    AC_SUBST(OMPI_TOP_SRCDIR)
-+    AC_SUBST(OMPI_TOP_BUILDDIR)
-+else
-+    AC_MSG_RESULT([not found])
-+    AC_MSG_WARN([*** Could not find Open MPI support files])
-+    AC_MSG_WARN([*** Can only build this version of ROMIO in an Open MPI source tree])
-+    AC_MSG_ERROR([*** Cannot continue])
-+fi
-+
- dnl
--NOF77=0
--NOF90=0
-+# Open MPI: disable the f77 and f90 tests, as we provide our own
-+# MPI interface and use only the C parts of ROMIO
-+NOF77=1
-+NOF90=1
- ARCH=""
- arch_IRIX=""
- MPI_IMPL=""
-@@ -95,6 +135,10 @@ AC_ARG_VAR([FROM_OMPI],[set to "yes" if building ROMIO inside of Open MPI])
- FROM_OMPI=${FROM_OMPI:-no}
- if test "$FROM_OMPI" = 1 ; then FROM_OMPI=yes ; fi
- 
-+AC_ARG_VAR([FROM_OMPI],[set to "yes" if building ROMIO inside of Open MPI])
-+FROM_OMPI=${FROM_OMPI:-no}
-+if test "$FROM_OMPI" = 1 ; then FROM_OMPI=yes ; fi
-+
- CFLAGS=${CFLAGS:-""}
- LL="lld"
- AR_LOCAL=""
-@@ -120,7 +164,8 @@ ROMIO_TFFLAGS=""
- NOPROFILE=0
- MPIRUN=""
- FORTRAN_TEST=""
--MAKE=${MAKE:-"make"}
-+# Open MPI: This (setting make) is a Bad Thing to do in Automake-based build systems
-+# MAKE=${MAKE:-"make"}
- # foll. needed for f77 test programs
- F77GETARG="call getarg(i,str)"
- F77IARGC="iargc()"
-@@ -133,6 +178,18 @@ MPI_OFFSET_KIND2="!"
- TEST_CC=""
- TEST_F77=""
- #
-+# Error handlers (not used with MPICH2, which provides its own routines)
-+MPIO_EXTRA_OBJECTS="get_errh.o set_errh.o"
-+MPIO_EXTRA_TMP_POBJECTS="get_errh.p set_errh.p"
-+MPIO_EXTRA_REAL_POBJECTS="_get_errh.o _set_errh.o"
-+#
-+# Completion routines for MPIO_Requests.  MPI Implementations with
-+# generalized requests do not need these
-+# ioreq_c2f and ioreq_f2c are not MPIO_Requests; rather, they
-+MPIO_REQOBJECTS="iotest.o iotestall.o iotestany.o iotestsome.o iowait.o iowaitall.o iowaitany.o iowaitsome.o ioreq_c2f.o ioreq_f2c.o"
-+MPIO_REQ_TMP_POBJECTS="iotest.p iowait.p iowaitall.p iowaitany.p iotestall.p iotestany.p iowaitsome.p iotestsome.p"
-+MPIO_REQ_REAL_POBJECTS="_iotest.o _iowait.o _iowaitall.o _iowaitany.o _iotestall.o _iotestany.o _iowaitsome.o _iotestsome.o"
-+#
- known_mpi_impls="mpich_mpi mpich_mpi sgi_mpi hp_mpi cray_mpi lam_mpi open_mpi_mpi"
- 
- dnl An m4 macro for use with m4_foreach_w and friends.  You should modify this
-@@ -169,7 +226,7 @@ AC_ARG_ENABLE(f77,
- AC_ARG_ENABLE(f90,
- [--enable-f90 - Turn on support for Fortran 90 (default)],,enable_f90=yes)
- AC_ARG_ENABLE(weak-symbols,
--[--enable-weak-symbols - Turn on support for weak symbols],,enable_weak_symbols=yes)
-+[--enable-weak-symbols - Turn on support for weak symbols],,enable_weak_symbols=no)
- AC_ARG_ENABLE(debug,
- [--enable-debug - Build a debugging version],,)
- AC_ARG_WITH(file-system,[
-@@ -192,9 +249,10 @@ if test "$enable_debug" = "yes" ; then
-     DEBUG=yes
- fi
- MPI=$with_mpi
--if test -n "$with_mpi"; then
--       CC=$MPI/bin/mpicc
--fi
-+# Open MPI: No!
-+#if test -n "$with_mpi"; then
-+#       CC=$MPI/bin/mpicc
-+#fi
- 
- # start with the set of file systems that the user asked for
- # FILE_SYSTEM=$with_file_system
-@@ -257,6 +315,7 @@ top_build_dir=`pwd`
- # used in romioinstall
- AC_SUBST(top_build_dir)
- 
-+# Open MPI: these shouldn't be needed with AM
- #
- # Create the "autoconf" style directory names...
- # Most of these are done for us; add the documentation directories
-@@ -378,8 +437,9 @@ if test "$CC" = "gcc" -a -z "$C_DEBUG_FLAG" ; then
- fi
- if test $DEBUG = "yes" ; then
-     CFLAGS="$CFLAGS $C_DEBUG_FLAG"
--else 
--    CFLAGS="$CFLAGS $C_OPT_FLAG"
-+# Open MPI: don't add optflags - they'll come from the top-level configure
-+#else
-+#    CFLAGS="$CFLAGS $C_OPT_FLAG"
- fi
- # ---------------------------------------------------------------------------
- # Here go the rest of the tests
-@@ -426,7 +486,8 @@ else
-     F77=":"
- fi
- #
--AC_C_INLINE
-+# Open MPI: We already do this test top-level
-+dnl AC_C_INLINE
- 
- AC_TYPE_SIZE_T
- AC_TYPE_SSIZE_T
-@@ -649,7 +710,9 @@ AM_CONDITIONAL([BUILD_MPIO_ERRHAN],[false])
- 
- # if we don't have weak symbol support, we must build a separate convenience
- # library in order to provide the "PMPI_" symbols
--AM_CONDITIONAL([BUILD_PROFILING_LIB],[test "x$HAVE_WEAK_SYMBOLS" = "x0"])
-+# Open MPI: Disable the profile library
-+#AM_CONDITIONAL([BUILD_PROFILING_LIB],[test "x$HAVE_WEAK_SYMBOLS" = "x0"])
-+AM_CONDITIONAL([BUILD_PROFILING_LIB],[false])
- 
- # weird: we have conflated "buid ROMIO's versions of the fortran bindings" and
- # "build ROMIO"s fortran I/O tests". Of course the common situaiton is that we
-@@ -1447,8 +1510,10 @@ if test $FROM_OMPI = yes ; then
-    HAVE_MPI_DARRAY_SUBARRAY="#define HAVE_MPI_DARRAY_SUBARRAY"
-    # Open MPI: see comments in mpi-io/mpioprof.h
-    AC_DEFINE(MPIO_BUILD_PROFILING, 1, [hack to make ROMIO build without profiling])
--   DEFINE_HAVE_MPI_GREQUEST="#define HAVE_MPI_GREQUEST"
-+   DEFINE_HAVE_MPI_GREQUEST="#define HAVE_MPI_GREQUEST 1"
-+   DEFINE_HAVE_MPI_GREQUEST_EXTENSIONS="#undef HAVE_MPI_GREQUEST_EXTENSIONS"
-    AC_DEFINE(HAVE_DECL_MPI_COMBINER_HINDEXED_BLOCK, 1, [Define if MPI library provides HINDEXED_BLOCK datatype])
-+   AC_DEFINE(HAVE_MPIIO_CONST, 1, Set if MPI-IO prototypes use const qualifier)
- elif test $FROM_LAM = yes ; then
-    # LAM does have the status set bytes functionality
-    AC_DEFINE(HAVE_STATUS_SET_BYTES,1,[Define if have MPIR_Status_set_bytes])
-@@ -1497,7 +1562,7 @@ elif test $FROM_MPICH = yes ; then
-    DEFINE_HAVE_MPI_GREQUEST_EXTENSIONS="#define HAVE_MPI_GREQUEST_EXTENSIONS 1"
-    AC_DEFINE(HAVE_MPIU_FUNCS,1,[Define if MPICH memory tracing macros defined])
-    AC_DEFINE(HAVE_MPIX_H, 1, [])
--   AC_DEFINE(HAVE_MPIIO_CONST, const, Set if MPI-IO prototypes use const qualifier)
-+   AC_DEFINE(HAVE_MPIIO_CONST, 1, Set if MPI-IO prototypes use const qualifier)
-    AC_DEFINE(HAVE_MPI_TYPE_SIZE_X, 1, [Define if MPI library provides MPI_TYPE_SIZE_X])
-    AC_DEFINE(HAVE_MPI_STATUS_SET_ELEMENTS_X, 1, [Define if MPI library provides MPI_STATUS_SET_ELEMENTS_X])
-    AC_DEFINE(HAVE_DECL_MPI_COMBINER_HINDEXED_BLOCK, 1, [Define if MPI library provides HINDEXED_BLOCK datatype])
-@@ -1620,9 +1685,22 @@ fi
- echo "setting SYSDEP_INC to $SYSDEP_INC"
- AC_SUBST(SYSDEP_INC)
- 
-+# Open MPI: use the exact same restrict test that we use in the
-+# upper-level Open MPI configure script so that we always get the same
-+# #define for "restrict" (there are a small number of files that will
-+# end up including both ROMIO's romioconf.h and opal_config.h, so we
-+# need to #defines to agree).
- AC_C_RESTRICT
- PAC_C_GNU_ATTRIBUTE
- 
-+# Open MPI: we need libtool
-+AM_PROG_LIBTOOL
-+
-+# Open MPI: setup the AM_CONDITIONALs to build the different adio devices
-+ m4_foreach([my_fs],
-+  [gpfs, gridftp, hfs, lustre, nfs, ntfs, panfs, pfs, pvfs, piofs, pvfs, pvfs2, sfs, testfs, ufs, xfs, zoidfs],
-+  [AM_CONDITIONAL(BUILD_[]AS_TR_CPP(my_fs), [test -n "$file_system_]my_fs["])])
-+
- # support gcov test coverage information
- PAC_ENABLE_COVERAGE
- 
-@@ -1634,23 +1712,29 @@ echo "setting CFLAGS to $CFLAGS"
- echo "setting USER_CFLAGS to $USER_CFLAGS"
- echo "setting USER_FFLAGS to $USER_FFLAGS"
- 
-+# Open MPI: Add on CFLAGS that we figured out up top.  They have
-+# makefile macros in them, so we couldn't substitute them until now.
-+CFLAGS="$CFLAGS $OMPI_CFLAGS "'-I$(top_builddir)/include'
-+#
-+# Open MPI - AM doesn't want the following:
-+#   VPATH, CC, CPPFLAGS, CFLAGS, AR, RANLIB, F77, MAKE
- AC_SUBST(ARCH)
- AC_SUBST(FILE_SYSTEM)
--AC_SUBST(CC)
--AC_SUBST(CPPFLAGS)
--AC_SUBST(CFLAGS)
-+#AC_SUBST(CC)
-+#AC_SUBST(CPPFLAGS)
-+#AC_SUBST(CFLAGS)
- AC_SUBST(USER_CFLAGS)
- AC_SUBST(USER_FFLAGS)
- AC_SUBST(MIPS)
- AC_SUBST(BITS)
--AC_SUBST(AR)
-+#AC_SUBST(AR)
- AC_SUBST(AR_FLAGS)
- AC_SUBST(MPI_INCLUDE_DIR)
- AC_SUBST(MPI_LIB)
--AC_SUBST(F77)
-+#AC_SUBST(F77)
- AC_SUBST(NOF77)
- AC_SUBST(NOPROFILE)
--AC_SUBST(MAKE)
-+#AC_SUBST(MAKE)
- AC_SUBST(arch_IRIX)
- AC_SUBST(ROMIO_HOME)
- AC_SUBST(LIBNAME)
-@@ -1671,6 +1755,7 @@ AC_SUBST(HAVE_MPI_DARRAY_SUBARRAY)
- AC_SUBST(BUILD_MPI_ARRAY)
- AC_SUBST(DEFINE_MPI_OFFSET)
- AC_SUBST(DEFINE_HAVE_MPI_GREQUEST)
-+AC_SUBST(DEFINE_HAVE_MPI_GREQUEST_EXTENSIONS)
- AC_SUBST(MPI_OFFSET_TYPE)
- AC_SUBST(MPI_FINFO1)
- AC_SUBST(MPI_FINFO2)
-@@ -1694,24 +1779,25 @@ AC_SUBST(ROMIO_TCPPFLAGS)
- AC_SUBST(ROMIO_TFFLAGS)
- AC_SUBST(MPIRUN)
- AC_SUBST(FORTRAN_TEST)
--dnl
--dnl Support shared libraries
--if test -z "$ENABLE_SHLIB" ; then
--   ENABLE_SHLIB=none
--fi
--AC_SUBST(ENABLE_SHLIB)
--AC_SUBST(CC_SHL)
--AC_SUBST(LIBTOOL)
-+#dnl
-+#dnl Support shared libraries
-+#if test -z "$ENABLE_SHLIB" ; then
-+#   ENABLE_SHLIB=none
-+#fi
-+#AC_SUBST(ENABLE_SHLIB)
-+#AC_SUBST(CC_SHL)
-+#AC_SUBST(LIBTOOL)
-+# Open MPI: This is no longer necessary with modern versions of autotools
- # Remove the .a from the library file name (so that we can use .so or
- # other appropriate suffix)
--SHLIBNAME=`echo $LIBNAME | sed 's/\.a$//'`
--AC_SUBST(SHLIBNAME)
--dnl
--if test ! -d adio ; then mkdir adio ; fi
--if test ! -d adio/include ; then mkdir adio/include ; fi
--if test ! -d mpi2-other ; then mkdir mpi2-other ; fi
--if test ! -d mpi-io ; then mkdir mpi-io ; fi
--if test ! -d mpi-io/glue ; then mkdir mpi-io/glue ; fi
-+#SHLIBNAME=`echo $LIBNAME | sed 's/\.a$//'`
-+#AC_SUBST(SHLIBNAME)
-+#dnl
-+#if test ! -d adio ; then mkdir adio ; fi
-+#if test ! -d adio/include ; then mkdir adio/include ; fi
-+#if test ! -d mpi2-other ; then mkdir mpi2-other ; fi
-+#if test ! -d mpi-io ; then mkdir mpi-io ; fi
-+#if test ! -d mpi-io/glue ; then mkdir mpi-io/glue ; fi
- 
- # Create makefiles for all of the adio devices.  Only the ones that 
- # are active will be called by the top level ROMIO make
-@@ -1719,8 +1805,6 @@ AC_OUTPUT_COMMANDS([chmod 755 util/romioinstall test/runtests])
- AC_CONFIG_FILES([
-     Makefile
-     localdefs
--    mpi2-other/info/Makefile
--    mpi2-other/array/Makefile
-     test/Makefile
-     test/misc.c
-     test/large_file.c
-@@ -1728,14 +1812,17 @@ AC_CONFIG_FILES([
-     test-internal/Makefile
-     util/romioinstall
-     include/mpio.h
--    include/mpiof.h
--    mpi2-other/info/fortran/Makefile
--    mpi2-other/array/fortran/Makefile
-     test/fmisc.f
-     test/fcoll_test.f
-     test/pfcoll_test.f
-     test/fperf.f
- ])
-+# Open MPI: intentionally skip the following:
-+# mpi2-other/info/Makefile
-+# mpi2-other/array/Makefile
-+# mpi2-other/info/fortran/Makefile
-+# mpi2-other/array/fortran/Makefile
-+# include/mpiof.h
- AC_OUTPUT
- 
- dnl PAC_SUBDIR_CACHE_CLEANUP
-diff --git a/ompi/mca/io/romio321/romio/doc/users-guide.tex b/ompi/mca/io/romio321/romio/doc/users-guide.tex
-index b33d483..3715431 100644
---- a/ompi/mca/io/romio321/romio/doc/users-guide.tex
-+++ b/ompi/mca/io/romio321/romio/doc/users-guide.tex
-@@ -807,13 +807,19 @@ to include the file {\tt mpio.h} for C or {\tt mpiof.h} for Fortran in
- your MPI-IO program.  
- 
- Note that on HP machines running HPUX and on NEC SX-4, you need to
--compile Fortran programs with {\tt mpifort}.
-+compile Fortran programs with {\tt mpifort}, because {\tt mpif77} does
-+not support 8-byte integers.
- 
- With MPICH, HP MPI, or NEC MPI, you can compile MPI-IO programs as \\
- \hspace*{.4in} {\tt mpicc foo.c}\\
- or \\
-+\hspace*{.4in} {\tt mpif77 foo.f }\\
-+or\\
- \hspace*{.4in} {\tt mpifort foo.f}\\
- 
-+As mentioned above, mpifort is preferred over mpif77 on HPUX and NEC
-+because the f77 compilers on those machines do not support 8-byte integers.
-+
- With SGI MPI, you can compile MPI-IO programs as \\
- \hspace*{.4in} {\tt cc foo.c -lmpi}\\
- or \\
-@@ -863,7 +869,8 @@ file systems because they don't support {\tt fcntl} file locks,
- and ROMIO uses that feature to implement shared file pointers.
- 
- \item On HP machines running HPUX and on NEC SX-4, you need to compile
--Fortran programs with {\tt mpifort}.
-+Fortran programs with {\tt mpifort} instead of {\tt mpif77}, because
-+the {\tt f77} compilers on these machines don't support 8-byte integers.
- 
- \item The file-open mode {\tt MPI\_MODE\_EXCL} does not work on Intel
- PFS file system, due to a bug in PFS.
-diff --git a/ompi/mca/io/romio321/romio/include/mpio.h.in b/ompi/mca/io/romio321/romio/include/mpio.h.in
-index 1f3f382..9ef709d 100644
---- a/ompi/mca/io/romio321/romio/include/mpio.h.in
-+++ b/ompi/mca/io/romio321/romio/include/mpio.h.in
-@@ -11,6 +11,16 @@
- #define MPIO_INCLUDE
- 
- #include "mpi.h"
-+/* Open MPI: We need to rename almost all of these functions, as well
-+   a the types to be names that conform to the prefix rule */
-+#include "io_romio_conv.h"
-+
-+#define MPIIMPL_HAVE_MPI_COMBINER_DARRAY 1
-+#define MPIIMPL_HAVE_MPI_TYPE_CREATE_DARRAY 1
-+#define MPIIMPL_HAVE_MPI_COMBINER_SUBARRAY 1
-+#define MPIIMPL_HAVE_MPI_TYPE_CREATE_DARRAY 1
-+#define MPIIMPL_HAVE_MPI_COMBINER_DUP 1
-+#define MPICH_ATTR_POINTER_WITH_TYPE_TAG(x,y)
- 
- #if defined(__cplusplus)
- extern "C" {
-@@ -38,6 +48,8 @@ typedef struct ADIOI_RequestD *MPIO_Request;
- #endif
- #define MPIO_REQUEST_DEFINED
- 
-+@DEFINE_HAVE_MPI_GREQUEST_EXTENSIONS@
-+
- #ifndef HAVE_MPI_OFFSET
- @DEFINE_MPI_OFFSET@
- /* If we needed to define MPI_Offset, then we also need to make
-@@ -279,9 +291,12 @@ int MPI_Type_create_darray(int size, int rank, int ndims, const int array_of_gsi
- #endif
- #endif
- /* above needed for some versions of mpi.h in MPICH!! */
-+/* Open MPI: I can't seem to make these #define properly.  Oh well --
-+   we don't need them anyway :-( */
-+#if 0
- MPI_File MPI_File_f2c(MPI_Fint file);
- MPI_Fint MPI_File_c2f(MPI_File file);
--
-+#endif
- 
- #ifndef HAVE_MPI_GREQUEST
- /* The following functions are required if generalized requests are not
-@@ -329,15 +344,21 @@ int MPI_Info_free(MPI_Info *info);
- #endif
- #endif
- /* above needed for some versions of mpi.h in MPICH!! */
-+/* Open MPI: we don't need these in ROMIO */
-+#if 0
- MPI_Fint MPI_Info_c2f(MPI_Info info);
- MPI_Info MPI_Info_f2c(MPI_Fint info);
- #endif
-+#endif
- 
- #endif   /* HAVE_PRAGMA_HP_SEC_DEF */
- 
- 
- /**************** BINDINGS FOR THE PROFILING INTERFACE ***************/
- 
-+/* Open MPI: We don't want any of the profiling layer */
-+#if 0
-+
- 
- /* Section 9.2 */
- int PMPI_File_open(MPI_Comm, const char *, int, MPI_Info, MPI_File *);
-@@ -528,6 +549,8 @@ MPI_Fint PMPI_Info_c2f(MPI_Info);
- MPI_Info PMPI_Info_f2c(MPI_Fint);
- #endif
- 
-+#endif /* Open MPI: We don't want any of the profiling layer */
-+
- #if defined(__cplusplus)
- }
- #endif
-diff --git a/ompi/mca/io/romio321/romio/mpi-io/Makefile.mk b/ompi/mca/io/romio321/romio/mpi-io/Makefile.mk
-index d4d5a29..264976e 100644
---- a/ompi/mca/io/romio321/romio/mpi-io/Makefile.mk
-+++ b/ompi/mca/io/romio321/romio/mpi-io/Makefile.mk
-@@ -14,8 +14,6 @@ noinst_HEADERS += mpi-io/mpioimpl.h mpi-io/mpioprof.h
- romio_mpi_sources +=          \
-     mpi-io/close.c            \
-     mpi-io/delete.c           \
--    mpi-io/file_c2f.c         \
--    mpi-io/file_f2c.c         \
-     mpi-io/fsync.c            \
-     mpi-io/get_amode.c        \
-     mpi-io/get_atom.c         \
-diff --git a/ompi/mca/io/romio321/romio/mpi-io/close.c b/ompi/mca/io/romio321/romio/mpi-io/close.c
-index 3f2b0b1..12d5c47 100644
---- a/ompi/mca/io/romio321/romio/mpi-io/close.c
-+++ b/ompi/mca/io/romio321/romio/mpi-io/close.c
-@@ -76,8 +76,13 @@ int MPI_File_close(MPI_File *fh)
-      * somehow inform the MPI library that we no longer hold a reference to any
-      * user defined error handler.  We do this by setting the errhandler at this
-      * point to MPI_ERRORS_RETURN. */
-+/* Open MPI: The call to PMPI_File_set_errhandler has to be done in romio/src/io_romio_file_open.c
-+   in routine mca_io_romio_file_close()
-+*/
-+#if 0
-     error_code = PMPI_File_set_errhandler(*fh, MPI_ERRORS_RETURN);
-     if (error_code != MPI_SUCCESS) goto fn_fail;
-+#endif
- 
-     ADIO_Close(adio_fh, &error_code);
-     MPIO_File_free(fh);
-diff --git a/ompi/mca/io/romio321/romio/mpi-io/glue/Makefile.mk b/ompi/mca/io/romio321/romio/mpi-io/glue/Makefile.mk
-index 05954a1..66f7f9e 100644
---- a/ompi/mca/io/romio321/romio/mpi-io/glue/Makefile.mk
-+++ b/ompi/mca/io/romio321/romio/mpi-io/glue/Makefile.mk
-@@ -7,6 +7,7 @@
- 
- include $(top_srcdir)/mpi-io/glue/default/Makefile.mk
- include $(top_srcdir)/mpi-io/glue/mpich/Makefile.mk
-+include $(top_srcdir)/mpi-io/glue/openmpi/Makefile.mk
- 
- if !BUILD_ROMIO_EMBEDDED
- romio_other_sources += \
-diff --git a/ompi/mca/io/romio321/romio/mpi-io/iread_all.c b/ompi/mca/io/romio321/romio/mpi-io/iread_all.c
-index 2615e56..00a0640 100644
---- a/ompi/mca/io/romio321/romio/mpi-io/iread_all.c
-+++ b/ompi/mca/io/romio321/romio/mpi-io/iread_all.c
-@@ -26,7 +26,7 @@ int MPI_File_iread_all(MPI_File fh, void *buf, int count, MPI_Datatype datatype,
- #include "mpioprof.h"
- #endif
- 
--#ifdef HAVE_MPI_GREQUEST
-+#if HAVE_MPI_GREQUEST
- #include "mpiu_greq.h"
- #endif
- 
-diff --git a/ompi/mca/io/romio321/romio/mpi-io/mpioprof.h b/ompi/mca/io/romio321/romio/mpi-io/mpioprof.h
-index f3ee3b4..c74413d 100644
---- a/ompi/mca/io/romio321/romio/mpi-io/mpioprof.h
-+++ b/ompi/mca/io/romio321/romio/mpi-io/mpioprof.h
-@@ -10,6 +10,15 @@
-    building the profiling interface
-  */
- 
-+/*
-+ * Open MPI: Unfortunately, ROMIO doesn't seem to be able to build
-+ * without a profiling interface, but we don't want a profiling
-+ * interface, since we are just using ROMIO "behind the scenes".
-+ * So enable all the profiling defines, only compile once, and don't
-+ * do all the name mangling.  The effect is about the same, but without
-+ * modification to all the files in the mpi-io directory.
-+ */
-+#if 0
- #ifdef MPIO_BUILD_PROFILING
- 
- #undef MPI_File_open
-@@ -212,3 +221,4 @@
- #define MPIX_Grequest_class_create PMPIX_Grequest_class_create
- 
- #endif
-+#endif
diff --git a/ompi/mca/io/romio321/owner.txt b/ompi/mca/io/romio321/owner.txt
deleted file mode 100644
index 61226d07ef4..00000000000
--- a/ompi/mca/io/romio321/owner.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-#
-# owner/status file
-# owner: institution that is responsible for this package
-# status: e.g. active, maintenance, unmaintained
-#
-owner: LANL/RIST
-status: active
diff --git a/ompi/mca/io/romio321/src/Makefile.extra b/ompi/mca/io/romio321/src/Makefile.extra
deleted file mode 100644
index dc684a8d6a9..00000000000
--- a/ompi/mca/io/romio321/src/Makefile.extra
+++ /dev/null
@@ -1,45 +0,0 @@
-# -*- makefile -*-
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2015      Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2017-2019 Research Organization for Information Science
-#                         and Technology (RIST).  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-AM_CPPFLAGS = \
-        -I$(OMPI_TOP_BUILDDIR)/include \
-        -I$(OMPI_TOP_BUILDDIR)/opal \
-        -I$(OMPI_TOP_BUILDDIR)/opal/include \
-        -I$(OMPI_TOP_SRCDIR)/opal \
-        -I$(OMPI_TOP_SRCDIR)/opal/include \
-        -I$(OMPI_TOP_BUILDDIR)/ompi \
-        -I$(OMPI_TOP_BUILDDIR)/ompi/include \
-        -I$(OMPI_TOP_SRCDIR)/ompi \
-        -I$(OMPI_TOP_SRCDIR)/ompi/include \
-        -I$(OMPI_TOP_SRCDIR)/ompi/mca/io/romio321 \
-        -I$(OMPI_TOP_SRCDIR)/3rd-party/romio321/include \
-        -I$(OMPI_TOP_SRCDIR)/3rd-party/romio321/adio/include \
-        -I$(top_builddir)
-
-sources += \
-	src/io_romio321.h \
-	src/io_romio321_component.c \
-	src/io_romio321_file_open.c \
-	src/io_romio321_file_read.c \
-	src/io_romio321_file_write.c \
-	src/io_romio321_module.c
-
diff --git a/ompi/mca/io/romio321/src/io_romio321.h b/ompi/mca/io/romio321/src/io_romio321.h
deleted file mode 100644
index 9fd44d353c5..00000000000
--- a/ompi/mca/io/romio321/src/io_romio321.h
+++ /dev/null
@@ -1,299 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015-2017 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2016-2019 IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef MCA_IO_ROMIO321_H
-#define MCA_IO_ROMIO321_H
-
-#include "ompi_config.h"
-#include "opal/mca/threads/mutex.h"
-#include "ompi/request/request.h"
-#include "ompi/file/file.h"
-#include "ompi/mca/io/io.h"
-#include "3rd-party/romio321/adio/include/romioconf.h"
-#include "3rd-party/romio321/include/mpio.h"
-
-
-BEGIN_C_DECLS
-
-OMPI_DECLSPEC extern mca_io_base_component_2_0_0_t mca_io_romio321_component;
-
-/*
- * global variables, instantiated in module.c
- */
-extern opal_mutex_t mca_io_romio321_mutex;
-extern mca_io_base_module_2_0_0_t mca_io_romio321_module;
-OMPI_DECLSPEC extern mca_io_base_component_2_0_0_t mca_io_romio321_component;
-
-/*
- * Private data for ROMIO modules
- */
-struct mca_io_romio321_data_t {
-    ROMIO_PREFIX (MPI_File) romio_fh;
-};
-typedef struct mca_io_romio321_data_t mca_io_romio321_data_t;
-
-
-/*
- * Module functions
- */
-
-
-/*
- *  mca->ROMIO module routines:
- *    ROMIO_PREFIX(file_XXX)
- *  ROMIO operations names:
- *    ROMIO_PREFIX(MPI_File_XXX)
- */
-/* Section 9.2 */
-int mca_io_romio321_file_open (struct ompi_communicator_t *comm,
-                            const char *filename,
-                            int amode,
-                            struct opal_info_t *info,
-                            ompi_file_t *fh);
-int mca_io_romio321_file_close (struct ompi_file_t *fh);
-int mca_io_romio321_file_delete (const char *filename,
-                              struct opal_info_t *info);
-int mca_io_romio321_file_set_size (struct ompi_file_t *fh,
-                                MPI_Offset size);
-int mca_io_romio321_file_preallocate (struct ompi_file_t *fh,
-                                   MPI_Offset size);
-int mca_io_romio321_file_get_size (struct ompi_file_t *fh,
-                                MPI_Offset * size);
-int mca_io_romio321_file_get_amode (struct ompi_file_t *fh,
-                                 int *amode);
-int mca_io_romio321_file_set_info (struct ompi_file_t *fh,
-                                struct ompi_info_t *info);
-int mca_io_romio321_file_get_info (struct ompi_file_t *fh,
-                                struct ompi_info_t ** info_used);
-
-/* Section 9.3 */
-int mca_io_romio321_file_set_view (struct ompi_file_t *fh,
-                                MPI_Offset disp,
-                                struct ompi_datatype_t *etype,
-                                struct ompi_datatype_t *filetype,
-                                const char *datarep,
-                                struct opal_info_t *info);
-int mca_io_romio321_file_get_view (struct ompi_file_t *fh,
-                                MPI_Offset * disp,
-                                struct ompi_datatype_t ** etype,
-                                struct ompi_datatype_t ** filetype,
-                                char *datarep);
-
-/* Section 9.4.2 */
-int mca_io_romio321_file_read_at (struct ompi_file_t *fh,
-                               MPI_Offset offset,
-                               void *buf,
-                               int count,
-                               struct ompi_datatype_t *datatype,
-                               ompi_status_public_t * status);
-int mca_io_romio321_file_read_at_all (struct ompi_file_t *fh,
-                                   MPI_Offset offset,
-                                   void *buf,
-                                   int count,
-                                   struct ompi_datatype_t *datatype,
-                                   ompi_status_public_t * status);
-int mca_io_romio321_file_write_at (struct ompi_file_t *fh,
-                                MPI_Offset offset,
-                                const void *buf,
-                                int count,
-                                struct ompi_datatype_t *datatype,
-                                ompi_status_public_t * status);
-int mca_io_romio321_file_write_at_all (struct ompi_file_t *fh,
-                                    MPI_Offset offset,
-                                    const void *buf,
-                                    int count,
-                                    struct ompi_datatype_t *datatype,
-                                    ompi_status_public_t * status);
-int mca_io_romio321_file_iread_at (struct ompi_file_t *fh,
-                                MPI_Offset offset,
-                                void *buf,
-                                int count,
-                                struct ompi_datatype_t *datatype,
-                                ompi_request_t **request);
-int mca_io_romio321_file_iread_at_all (struct ompi_file_t *fh,
-                                       MPI_Offset offset,
-                                       void *buf,
-                                       int count,
-                                       struct ompi_datatype_t *datatype,
-                                       ompi_request_t **request);
-int mca_io_romio321_file_iwrite_at (struct ompi_file_t *fh,
-                                 MPI_Offset offset,
-                                 const void *buf,
-                                 int count,
-                                 struct ompi_datatype_t *datatype,
-                                 ompi_request_t **request);
-int mca_io_romio321_file_iwrite_at_all (struct ompi_file_t *fh,
-                                        MPI_Offset offset,
-                                        const void *buf,
-                                        int count,
-                                        struct ompi_datatype_t *datatype,
-                                        ompi_request_t **request);
-
-/* Section 9.4.3 */
-int mca_io_romio321_file_read (struct ompi_file_t *fh,
-                            void *buf,
-                            int count,
-                            struct ompi_datatype_t *datatype,
-                            ompi_status_public_t * status);
-int mca_io_romio321_file_read_all (struct ompi_file_t *fh,
-                                void *buf,
-                                int count,
-                                struct ompi_datatype_t *datatype,
-                                ompi_status_public_t * status);
-int mca_io_romio321_file_write (struct ompi_file_t *fh,
-                             const void *buf,
-                             int count,
-                             struct ompi_datatype_t *datatype,
-                             ompi_status_public_t * status);
-int mca_io_romio321_file_write_all (struct ompi_file_t *fh,
-                                 const void *buf,
-                                 int count,
-                                 struct ompi_datatype_t *datatype,
-                                 ompi_status_public_t * status);
-int mca_io_romio321_file_iread (struct ompi_file_t *fh,
-                             void *buf,
-                             int count,
-                             struct ompi_datatype_t *datatype,
-                             ompi_request_t **request);
-int mca_io_romio321_file_iread_all (struct ompi_file_t *fh,
-                                    void *buf,
-                                    int count,
-                                    struct ompi_datatype_t *datatype,
-                                    ompi_request_t **request);
-int mca_io_romio321_file_iwrite (struct ompi_file_t *fh,
-                              const void *buf,
-                              int count,
-                              struct ompi_datatype_t *datatype,
-                              ompi_request_t **request);
-int mca_io_romio321_file_iwrite_all (struct ompi_file_t *fh,
-                                     const void *buf,
-                                     int count,
-                                     struct ompi_datatype_t *datatype,
-                                     ompi_request_t **request);
-int mca_io_romio321_file_seek (struct ompi_file_t *fh,
-                            MPI_Offset offset,
-                            int whence);
-int mca_io_romio321_file_get_position (struct ompi_file_t *fh,
-                                    MPI_Offset * offset);
-int mca_io_romio321_file_get_byte_offset (struct ompi_file_t *fh,
-                                       MPI_Offset offset,
-                                       MPI_Offset * disp);
-
-/* Section 9.4.4 */
-int mca_io_romio321_file_read_shared (struct ompi_file_t *fh,
-                                   void *buf,
-                                   int count,
-                                   struct ompi_datatype_t *datatype,
-                                   ompi_status_public_t * status);
-int mca_io_romio321_file_write_shared (struct ompi_file_t *fh,
-                                    const void *buf,
-                                    int count,
-                                    struct ompi_datatype_t *datatype,
-                                    ompi_status_public_t * status);
-int mca_io_romio321_file_iread_shared (struct ompi_file_t *fh,
-                                    void *buf,
-                                    int count,
-                                    struct ompi_datatype_t *datatype,
-                                    ompi_request_t **request);
-int mca_io_romio321_file_iwrite_shared (struct ompi_file_t *fh,
-                                     const void *buf,
-                                     int count,
-                                     struct ompi_datatype_t *datatype,
-                                     ompi_request_t **request);
-int mca_io_romio321_file_read_ordered (struct ompi_file_t *fh,
-                                    void *buf,
-                                    int count,
-                                    struct ompi_datatype_t *datatype,
-                                    ompi_status_public_t * status);
-int mca_io_romio321_file_write_ordered (struct ompi_file_t *fh,
-                                     const void *buf,
-                                     int count,
-                                     struct ompi_datatype_t *datatype,
-                                     ompi_status_public_t * status);
-int mca_io_romio321_file_seek_shared (struct ompi_file_t *fh,
-                                   MPI_Offset offset,
-                                   int whence);
-int mca_io_romio321_file_get_position_shared (struct ompi_file_t *fh,
-                                           MPI_Offset * offset);
-
-/* Section 9.4.5 */
-int mca_io_romio321_file_read_at_all_begin (struct ompi_file_t *fh,
-                                         MPI_Offset offset,
-                                         void *buf,
-                                         int count,
-                                         struct ompi_datatype_t *datatype);
-int mca_io_romio321_file_read_at_all_end (struct ompi_file_t *fh,
-                                       void *buf,
-                                       ompi_status_public_t * status);
-int mca_io_romio321_file_write_at_all_begin (struct ompi_file_t *fh,
-                                          MPI_Offset offset,
-                                          const void *buf,
-                                          int count,
-                                          struct ompi_datatype_t *datatype);
-int mca_io_romio321_file_write_at_all_end (struct ompi_file_t *fh,
-                                        const void *buf,
-                                        ompi_status_public_t * status);
-int mca_io_romio321_file_read_all_begin (struct ompi_file_t *fh,
-                                      void *buf,
-                                      int count,
-                                      struct ompi_datatype_t *datatype);
-int mca_io_romio321_file_read_all_end (struct ompi_file_t *fh,
-                                    void *buf,
-                                    ompi_status_public_t * status);
-int mca_io_romio321_file_write_all_begin (struct ompi_file_t *fh,
-                                       const void *buf,
-                                       int count,
-                                       struct ompi_datatype_t *datatype);
-int mca_io_romio321_file_write_all_end (struct ompi_file_t *fh,
-                                     const void *buf,
-                                     ompi_status_public_t * status);
-int mca_io_romio321_file_read_ordered_begin (struct ompi_file_t *fh,
-                                          void *buf,
-                                          int count,
-                                          struct ompi_datatype_t *datatype);
-int mca_io_romio321_file_read_ordered_end (struct ompi_file_t *fh,
-                                        void *buf,
-                                        ompi_status_public_t * status);
-int mca_io_romio321_file_write_ordered_begin (struct ompi_file_t *fh,
-                                           const void *buf,
-                                           int count,
-                                           struct ompi_datatype_t *datatype);
-int mca_io_romio321_file_write_ordered_end (struct ompi_file_t *fh,
-                                         const void *buf,
-                                         struct ompi_status_public_t * status);
-
-/* Section 9.5.1 */
-int mca_io_romio321_file_get_type_extent (struct ompi_file_t *fh,
-                                       struct ompi_datatype_t *datatype,
-                                       MPI_Aint * extent);
-
-/* Section 9.6.1 */
-int mca_io_romio321_file_set_atomicity (struct ompi_file_t *fh,
-                                     int flag);
-int mca_io_romio321_file_get_atomicity (struct ompi_file_t *fh,
-                                     int *flag);
-int mca_io_romio321_file_sync (struct ompi_file_t *fh);
-
-/* End Prototypes */
-
-END_C_DECLS
-#endif /* MCA_IO_ROMIO321_H */
diff --git a/ompi/mca/io/romio321/src/io_romio321_component.c b/ompi/mca/io/romio321/src/io_romio321_component.c
deleted file mode 100644
index 2f1b50a822f..00000000000
--- a/ompi/mca/io/romio321/src/io_romio321_component.c
+++ /dev/null
@@ -1,276 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008-2015 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015-2017 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "mpi.h"
-#include "opal/class/opal_list.h"
-#include "opal/mca/threads/mutex.h"
-#include "opal/mca/base/base.h"
-#include "ompi/mca/io/io.h"
-#include "io_romio321.h"
-
-#define ROMIO_VERSION_STRING "from MPICH v3.1.4"
-
-/*
- * Private functions
- */
-static int register_component(void);
-static int open_component(void);
-static int close_component(void);
-static int init_query(bool enable_progress_threads,
-                      bool enable_mpi_threads);
-static const struct mca_io_base_module_2_0_0_t *
-  file_query(struct ompi_file_t *file,
-             struct mca_io_base_file_t **private_data,
-             int *priority);
-static int file_unquery(struct ompi_file_t *file,
-                        struct mca_io_base_file_t *private_data);
-
-static int delete_query(const char *filename, struct opal_info_t *info,
-                        struct mca_io_base_delete_t **private_data,
-                        bool *usable, int *priorty);
-static int delete_select(const char *filename, struct opal_info_t *info,
-                         struct mca_io_base_delete_t *private_data);
-
-static int register_datarep(const char *,
-                            MPI_Datarep_conversion_function*,
-                            MPI_Datarep_conversion_function*,
-                            MPI_Datarep_extent_function*,
-                            void*);
-
-/*
- * Private variables
- */
-static int priority_param = 20;
-static int delete_priority_param = 20;
-
-
-/*
- * Global, component-wide ROMIO mutex because ROMIO is not thread safe
- */
-opal_mutex_t mca_io_romio321_mutex = {{0}};
-
-
-/*
- * Public string showing this component's version number
- */
-const char *mca_io_romio321_component_version_string =
-"OMPI/MPI ROMIO io MCA component version " OMPI_VERSION ", " ROMIO_VERSION_STRING;
-
-
-mca_io_base_component_2_0_0_t mca_io_romio321_component = {
-    /* First, the mca_base_component_t struct containing meta information
-       about the component itself */
-
-    .io_version = {
-        MCA_IO_BASE_VERSION_2_0_0,
-        .mca_component_name = "romio321",
-        MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION,
-                              OMPI_RELEASE_VERSION),
-        .mca_open_component = open_component,
-        .mca_close_component = close_component,
-        .mca_register_component_params = register_component,
-    },
-    .io_data = {
-        /* The component is checkpoint ready */
-        MCA_BASE_METADATA_PARAM_CHECKPOINT
-    },
-
-    /* Initial configuration / Open a new file */
-
-    .io_init_query = init_query,
-    .io_file_query = file_query,
-    .io_file_unquery = file_unquery,
-
-    /* Delete a file */
-
-    .io_delete_query = delete_query,
-    .io_delete_select = delete_select,
-
-    .io_register_datarep = register_datarep,
-};
-
-static char *ompi_io_romio321_version = ROMIO_VERSION_STRING;
-static char *ompi_io_romio321_user_configure_params = MCA_io_romio321_USER_CONFIGURE_FLAGS;
-static char *ompi_io_romio321_complete_configure_params = MCA_io_romio321_COMPLETE_CONFIGURE_FLAGS;
-
-static int register_component(void)
-{
-    /* Use a low priority, but allow other components to be lower */
-    priority_param = 10;
-    (void) mca_base_component_var_register(&mca_io_romio321_component.io_version,
-                                           "priority", "Priority of the io romio component",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY, &priority_param);
-    delete_priority_param = 10;
-    (void) mca_base_component_var_register(&mca_io_romio321_component.io_version,
-                                           "delete_priority", "Delete priority of the io romio component",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY, &delete_priority_param);
-    (void) mca_base_component_var_register(&mca_io_romio321_component.io_version,
-                                           "version", "Version of ROMIO", MCA_BASE_VAR_TYPE_STRING,
-                                           NULL, 0, MCA_BASE_VAR_FLAG_DEFAULT_ONLY,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY, &ompi_io_romio321_version);
-    (void) mca_base_component_var_register(&mca_io_romio321_component.io_version,
-                                           "user_configure_params",
-                                           "User-specified command line parameters passed to ROMIO's configure script",
-                                           MCA_BASE_VAR_TYPE_STRING, NULL, 0,
-                                           MCA_BASE_VAR_FLAG_DEFAULT_ONLY,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY, &ompi_io_romio321_user_configure_params);
-    (void) mca_base_component_var_register(&mca_io_romio321_component.io_version,
-                                           "complete_configure_params",
-                                           "Complete set of command line parameters passed to ROMIO's configure script",
-                                           MCA_BASE_VAR_TYPE_STRING, NULL, 0,
-                                           MCA_BASE_VAR_FLAG_DEFAULT_ONLY,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY, &ompi_io_romio321_complete_configure_params);
-
-    return OMPI_SUCCESS;
-}
-
-static int open_component(void)
-{
-    /* Create the mutex */
-    OBJ_CONSTRUCT(&mca_io_romio321_mutex, opal_mutex_t);
-
-    return OMPI_SUCCESS;
-}
-
-
-static int close_component(void)
-{
-    OBJ_DESTRUCT(&mca_io_romio321_mutex);
-
-    return OMPI_SUCCESS;
-}
-
-
-static int init_query(bool enable_progress_threads,
-                      bool enable_mpi_threads)
-{
-    /* Note that it's ok if mpi_enable_threads==true here because we
-       self-enforce only allowing one user thread into ROMIO at a time
-       -- this fact will be clearly documented for users (ROMIO itself
-       is not thread safe). */
-
-    return OMPI_SUCCESS;
-}
-
-
-static const struct mca_io_base_module_2_0_0_t *
-file_query(struct ompi_file_t *file,
-           struct mca_io_base_file_t **private_data,
-           int *priority)
-{
-    mca_io_romio321_data_t *data;
-
-    *priority = priority_param;
-
-    /* Allocate a space for this module to hang private data (e.g.,
-       the ROMIO file handle) */
-
-    data = malloc(sizeof(mca_io_romio321_data_t));
-    if (NULL == data) {
-        return NULL;
-    }
-    data->romio_fh = NULL;
-    *private_data = (struct mca_io_base_file_t*) data;
-
-    /* All done */
-
-    return &mca_io_romio321_module;
-}
-
-
-static int file_unquery(struct ompi_file_t *file,
-                        struct mca_io_base_file_t *private_data)
-{
-    /* Free the romio module-specific data that was allocated in
-       _file_query(), above */
-
-    if (NULL != private_data) {
-        free(private_data);
-    }
-
-    return OMPI_SUCCESS;
-}
-
-
-static int delete_query(const char *filename, struct opal_info_t *info,
-                        struct mca_io_base_delete_t **private_data,
-                        bool *usable, int *priority)
-{
-    *priority = delete_priority_param;
-    *usable = true;
-    *private_data = NULL;
-
-    return OMPI_SUCCESS;
-}
-
-
-static int delete_select(const char *filename, struct opal_info_t *info,
-                         struct mca_io_base_delete_t *private_data)
-{
-    int ret;
-
-// An opal_info_t isn't a full ompi_info_t. so if we're using an MPI call
-// below with an MPI_Info, we need to create an equivalent MPI_Info. This
-// isn't ideal but it only happens a few places.
-    ompi_info_t *ompi_info;
-    ompi_info = OBJ_NEW(ompi_info_t);
-    if (!ompi_info) { return(MPI_ERR_NO_MEM); }
-    opal_info_t *opal_info = &(ompi_info->super);
-    opal_info_dup (info, &opal_info);
-
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret = ROMIO_PREFIX(MPI_File_delete)(filename, ompi_info);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    ompi_info_free(&ompi_info);
-    return ret;
-}
-
-
-static int
-register_datarep(const char * datarep,
-                 MPI_Datarep_conversion_function* read_fn,
-                 MPI_Datarep_conversion_function* write_fn,
-                 MPI_Datarep_extent_function* extent_fn,
-                 void* state)
-{
-    int ret;
-
-    OPAL_THREAD_LOCK(&mca_io_romio321_mutex);
-    ret = ROMIO_PREFIX(MPI_Register_datarep(datarep, read_fn, write_fn,
-                                            extent_fn, state));
-    OPAL_THREAD_UNLOCK(&mca_io_romio321_mutex);
-
-    return ret;
-}
diff --git a/ompi/mca/io/romio321/src/io_romio321_file_open.c b/ompi/mca/io/romio321/src/io_romio321_file_open.c
deleted file mode 100644
index 5b641c3e4e5..00000000000
--- a/ompi/mca/io/romio321/src/io_romio321_file_open.c
+++ /dev/null
@@ -1,392 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015-2017 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2016-2019 IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/communicator/communicator.h"
-#include "ompi/info/info.h"
-#include "ompi/file/file.h"
-
-#include "io_romio321.h"
-
-
-int
-mca_io_romio321_file_open (ompi_communicator_t *comm,
-                        const char *filename,
-                        int amode,
-                        opal_info_t *info,
-                        ompi_file_t *fh)
-{
-    int ret;
-    mca_io_romio321_data_t *data;
-
-// An opal_info_t isn't a full ompi_info_t. so if we're using an MPI call
-// below with an MPI_Info, we need to create an equivalent MPI_Info. This
-// isn't ideal but it only happens a few places.
-    ompi_info_t *ompi_info;
-    ompi_info = OBJ_NEW(ompi_info_t);
-    if (!ompi_info) { return(MPI_ERR_NO_MEM); }
-    opal_info_t *opal_info = &(ompi_info->super);
-    opal_info_dup (info, &opal_info);
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-//    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret = ROMIO_PREFIX(MPI_File_open)(comm, filename, amode, ompi_info,
-                                      &data->romio_fh);
-//    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    ompi_info_free(&ompi_info);
-    return ret;
-}
-
-
-int
-mca_io_romio321_file_close (ompi_file_t *fh)
-{
-    int ret;
-    mca_io_romio321_data_t *data;
-    int finalized;
-
-    /* If we've already started MPI_Finalize by this point, then just
-       give up (because ROMIO's file close routine calls MPI_Barrier,
-       which we obviously can't do if we've started to MPI_Finalize).
-       The user didn't close the file, so they should expect
-       unexpected behavior. */
-    PMPI_Finalized(&finalized);
-    if (finalized) {
-        return OMPI_SUCCESS;
-    }
-
-    /* Because ROMIO expects the MPI library to provide error handler
-     * management routines but it doesn't ever participate in
-     * MPI_File_close, we have to somehow inform the MPI library that
-     * we no longer hold a reference to any user defined error
-     * handler.  We do this by setting the errhandler at this point to
-     * MPI_ERRORS_RETURN. */
-    if (fh->error_handler != &ompi_mpi_errors_return.eh) {
-        OBJ_RELEASE(fh->error_handler);
-        fh->error_handler = &ompi_mpi_errors_return.eh;
-        OBJ_RETAIN(fh->error_handler);
-    }
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret = ROMIO_PREFIX(MPI_File_close) (&data->romio_fh);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-
-int
-mca_io_romio321_file_set_size (ompi_file_t *fh,
-                            MPI_Offset size)
-{
-    int ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret = ROMIO_PREFIX(MPI_File_set_size) (data->romio_fh, size);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-int
-mca_io_romio321_file_preallocate (ompi_file_t *fh,
-                               MPI_Offset size)
-{
-    int ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret = ROMIO_PREFIX(MPI_File_preallocate) (data->romio_fh, size);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-
-int
-mca_io_romio321_file_get_size (ompi_file_t *fh,
-                            MPI_Offset * size)
-{
-    int ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret = ROMIO_PREFIX(MPI_File_get_size) (data->romio_fh, size);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-
-int
-mca_io_romio321_file_get_amode (ompi_file_t *fh,
-                             int *amode)
-{
-    int ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret = ROMIO_PREFIX(MPI_File_get_amode) (data->romio_fh, amode);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-
-int
-mca_io_romio321_file_set_info (ompi_file_t *fh,
-                            ompi_info_t *info)
-{
-    int ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret = ROMIO_PREFIX(MPI_File_set_info) (data->romio_fh, info);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-
-int
-mca_io_romio321_file_get_info (ompi_file_t *fh,
-                            ompi_info_t ** info_used)
-{
-    int ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret = ROMIO_PREFIX(MPI_File_get_info) (data->romio_fh, info_used);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-
-int
-mca_io_romio321_file_set_view (ompi_file_t *fh,
-                            MPI_Offset disp,
-                            struct ompi_datatype_t *etype,
-                            struct ompi_datatype_t *filetype,
-                            const char *datarep,
-                            opal_info_t *info)
-{
-    int ret;
-    mca_io_romio321_data_t *data;
-
-// An opal_info_t isn't a full ompi_info_t. so if we're using an MPI call
-// below with an MPI_Info, we need to create an equivalent MPI_Info. This
-// isn't ideal but it only happens a few places.
-    ompi_info_t *ompi_info;
-    ompi_info = OBJ_NEW(ompi_info_t);
-    if (!ompi_info) { return(MPI_ERR_NO_MEM); }
-    opal_info_t *opal_info = &(ompi_info->super);
-    opal_info_dup (info, &opal_info);
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret =
-        ROMIO_PREFIX(MPI_File_set_view) (data->romio_fh, disp, etype, filetype,
-                                        datarep, ompi_info);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    ompi_info_free(&ompi_info);
-    return ret;
-}
-
-
-int
-mca_io_romio321_file_get_view (ompi_file_t *fh,
-                            MPI_Offset * disp,
-                            struct ompi_datatype_t ** etype,
-                            struct ompi_datatype_t ** filetype,
-                            char *datarep)
-{
-    int ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret =
-        ROMIO_PREFIX(MPI_File_get_view) (data->romio_fh, disp, etype, filetype,
-                                        datarep);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-
-}
-
-
-int
-mca_io_romio321_file_get_type_extent (ompi_file_t *fh,
-                                   struct ompi_datatype_t *datatype,
-                                   MPI_Aint * extent)
-{
-    int ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret =
-        ROMIO_PREFIX(MPI_File_get_type_extent) (data->romio_fh, datatype, extent);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-
-int
-mca_io_romio321_file_set_atomicity (ompi_file_t *fh,
-                                 int flag)
-{
-    int ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret = ROMIO_PREFIX(MPI_File_set_atomicity) (data->romio_fh, flag);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-int
-mca_io_romio321_file_get_atomicity (ompi_file_t *fh,
-                                 int *flag)
-{
-    int ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret = ROMIO_PREFIX(MPI_File_get_atomicity) (data->romio_fh, flag);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-int
-mca_io_romio321_file_sync (ompi_file_t *fh)
-{
-    int ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret = ROMIO_PREFIX(MPI_File_sync) (data->romio_fh);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-
-int
-mca_io_romio321_file_seek_shared (ompi_file_t *fh,
-                               MPI_Offset offset,
-                               int whence)
-{
-    int ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret = ROMIO_PREFIX(MPI_File_seek_shared) (data->romio_fh, offset, whence);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-
-int
-mca_io_romio321_file_get_position_shared (ompi_file_t *fh,
-                                       MPI_Offset * offset)
-{
-    int ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret = ROMIO_PREFIX(MPI_File_get_position_shared) (data->romio_fh, offset);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-
-int
-mca_io_romio321_file_seek (ompi_file_t *fh,
-                        MPI_Offset offset,
-                        int whence)
-{
-    int ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret = ROMIO_PREFIX(MPI_File_seek) (data->romio_fh, offset, whence);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-
-int
-mca_io_romio321_file_get_position (ompi_file_t *fh,
-                                MPI_Offset * offset)
-{
-    int ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret = ROMIO_PREFIX(MPI_File_get_position) (data->romio_fh, offset);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-
-int
-mca_io_romio321_file_get_byte_offset (ompi_file_t *fh,
-                                   MPI_Offset offset,
-                                   MPI_Offset * disp)
-{
-    int ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret = ROMIO_PREFIX(MPI_File_get_byte_offset) (data->romio_fh, offset, disp);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
diff --git a/ompi/mca/io/romio321/src/io_romio321_file_read.c b/ompi/mca/io/romio321/src/io_romio321_file_read.c
deleted file mode 100644
index 8944d926cc9..00000000000
--- a/ompi/mca/io/romio321/src/io_romio321_file_read.c
+++ /dev/null
@@ -1,382 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2017      IBM Corporation.  All rights reserved.
- * Copyright (c) 2017      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include "mpi.h"
-#include "ompi/file/file.h"
-#include "io_romio321.h"
-
-
-int
-mca_io_romio321_file_read_at (ompi_file_t *fh,
-                           MPI_Offset offset,
-                           void *buf,
-                           int count,
-                           struct ompi_datatype_t *datatype,
-                           ompi_status_public_t * status)
-{
-    int         ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret =
-        ROMIO_PREFIX(MPI_File_read_at) (data->romio_fh, offset, buf, count,
-                                       datatype, status);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-
-int
-mca_io_romio321_file_read_at_all (ompi_file_t *fh,
-                               MPI_Offset offset,
-                               void *buf,
-                               int count,
-                               struct ompi_datatype_t *datatype,
-                               ompi_status_public_t * status)
-{
-    int         ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret =
-        ROMIO_PREFIX(MPI_File_read_at_all) (data->romio_fh, offset, buf, count,
-                                           datatype, status);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-
-int
-mca_io_romio321_file_iread_at (ompi_file_t *fh,
-                            MPI_Offset offset,
-                            void *buf,
-                            int count,
-                            struct ompi_datatype_t *datatype,
-                            ompi_request_t **request)
-{
-    int ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret =
-        ROMIO_PREFIX(MPI_File_iread_at) (data->romio_fh, offset, buf, count,
-                                         datatype, request);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-int
-mca_io_romio321_file_iread_at_all (ompi_file_t *fh,
-                                   MPI_Offset offset,
-                                   void *buf,
-                                   int count,
-                                   struct ompi_datatype_t *datatype,
-                                   ompi_request_t **request)
-{
-    int         ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    // ----------------------------------------------------
-    // NOTE: If you upgrade ROMIO, replace this with the actual ROMIO call.
-    // ----------------------------------------------------
-    // No support for non-blocking collective I/O operations.
-    // Fake it with individual non-blocking I/O operations.
-    // Similar to OMPIO
-    ret =
-        ROMIO_PREFIX(MPI_File_iread_at_all) (data->romio_fh, offset, buf, count,
-                                             datatype, request);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-
-int
-mca_io_romio321_file_read (ompi_file_t *fh,
-                        void *buf,
-                        int count,
-                        struct ompi_datatype_t *datatype,
-                        ompi_status_public_t * status)
-{
-    int         ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret =
-        ROMIO_PREFIX(MPI_File_read) (data->romio_fh, buf, count, datatype,
-                                    status);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-
-int
-mca_io_romio321_file_read_all (ompi_file_t *fh,
-                            void *buf,
-                            int count,
-                            struct ompi_datatype_t *datatype,
-                            ompi_status_public_t * status)
-{
-    int         ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret =
-        ROMIO_PREFIX(MPI_File_read_all) (data->romio_fh, buf, count, datatype,
-                                        status);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-
-int
-mca_io_romio321_file_iread (ompi_file_t *fh,
-                         void *buf,
-                         int count,
-                         struct ompi_datatype_t *datatype,
-                         ompi_request_t **request)
-{
-    int ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret =
-        ROMIO_PREFIX(MPI_File_iread) (data->romio_fh, buf, count, datatype,
-                                      request);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-int
-mca_io_romio321_file_iread_all (ompi_file_t *fh,
-                                void *buf,
-                                int count,
-                                struct ompi_datatype_t *datatype,
-                                ompi_request_t **request)
-{
-    int         ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    // ----------------------------------------------------
-    // NOTE: If you upgrade ROMIO, replace this with the actual ROMIO call.
-    // ----------------------------------------------------
-    // No support for non-blocking collective I/O operations.
-    // Fake it with individual non-blocking I/O operations.
-    // Similar to OMPIO
-    ret =
-        ROMIO_PREFIX(MPI_File_iread_all) (data->romio_fh, buf, count, datatype,
-                                          request);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-int
-mca_io_romio321_file_read_shared (ompi_file_t *fh,
-                               void *buf,
-                               int count,
-                               struct ompi_datatype_t *datatype,
-                               ompi_status_public_t * status)
-{
-    int         ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret =
-        ROMIO_PREFIX(MPI_File_read_shared) (data->romio_fh, buf, count,
-                                            datatype, status);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-
-int
-mca_io_romio321_file_iread_shared (ompi_file_t *fh,
-                                void *buf,
-                                int count,
-                                struct ompi_datatype_t *datatype,
-                                ompi_request_t **request)
-{
-    int ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret =
-        ROMIO_PREFIX(MPI_File_iread_shared) (data->romio_fh, buf, count,
-                                             datatype, request);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-
-int
-mca_io_romio321_file_read_ordered (ompi_file_t *fh,
-                                void *buf,
-                                int count,
-                                struct ompi_datatype_t *datatype,
-                                ompi_status_public_t * status)
-{
-    int         ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret =
-        ROMIO_PREFIX(MPI_File_read_ordered) (data->romio_fh, buf, count,
-                                             datatype, status);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-
-int
-mca_io_romio321_file_read_at_all_begin (ompi_file_t *fh,
-                                     MPI_Offset offset,
-                                     void *buf,
-                                     int count,
-                                     struct ompi_datatype_t *datatype)
-{
-    int         ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret =
-        ROMIO_PREFIX(MPI_File_read_at_all_begin) (data->romio_fh, offset, buf,
-                                                 count, datatype);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-
-int
-mca_io_romio321_file_read_at_all_end (ompi_file_t *fh,
-                                   void *buf,
-                                   ompi_status_public_t * status)
-{
-    int         ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret = ROMIO_PREFIX(MPI_File_read_at_all_end) (data->romio_fh, buf, status);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-
-int
-mca_io_romio321_file_read_all_begin (ompi_file_t *fh,
-                                  void *buf,
-                                  int count,
-                                  struct ompi_datatype_t *datatype)
-{
-    int         ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret =
-        ROMIO_PREFIX(MPI_File_read_all_begin) (data->romio_fh, buf, count,
-                                              datatype);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-
-int
-mca_io_romio321_file_read_all_end (ompi_file_t *fh,
-                                void *buf,
-                                ompi_status_public_t * status)
-{
-    int         ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret = ROMIO_PREFIX(MPI_File_read_all_end) (data->romio_fh, buf, status);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-
-int
-mca_io_romio321_file_read_ordered_begin (ompi_file_t *fh,
-                                      void *buf,
-                                      int count,
-                                      struct ompi_datatype_t *datatype)
-{
-    int         ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret =
-        ROMIO_PREFIX(MPI_File_read_ordered_begin) (data->romio_fh, buf, count,
-                                                  datatype);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-
-int
-mca_io_romio321_file_read_ordered_end (ompi_file_t *fh,
-                                    void *buf,
-                                    ompi_status_public_t * status)
-{
-    int         ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret = ROMIO_PREFIX(MPI_File_read_ordered_end) (data->romio_fh, buf,
-                                                   status);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
diff --git a/ompi/mca/io/romio321/src/io_romio321_file_write.c b/ompi/mca/io/romio321/src/io_romio321_file_write.c
deleted file mode 100644
index 2eb74d7f5a9..00000000000
--- a/ompi/mca/io/romio321/src/io_romio321_file_write.c
+++ /dev/null
@@ -1,375 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015-2017 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include "mpi.h"
-#include "ompi/file/file.h"
-#include "io_romio321.h"
-
-
-int
-mca_io_romio321_file_write_at (ompi_file_t *fh,
-                            MPI_Offset offset,
-                            const void *buf,
-                            int count,
-                            struct ompi_datatype_t *datatype,
-                            ompi_status_public_t * status)
-{
-    int         ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret =
-        ROMIO_PREFIX(MPI_File_write_at) (data->romio_fh, offset, buf, count,
-                                        datatype, status);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-
-
-int
-mca_io_romio321_file_write_at_all (ompi_file_t *fh,
-                                MPI_Offset offset,
-                                const void *buf,
-                                int count,
-                                struct ompi_datatype_t *datatype,
-                                ompi_status_public_t * status)
-{
-    int         ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret =
-        ROMIO_PREFIX(MPI_File_write_at_all) (data->romio_fh, offset, buf,
-                                             count, datatype, status);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-
-
-int
-mca_io_romio321_file_iwrite_at (ompi_file_t *fh,
-                             MPI_Offset offset,
-                             const void *buf,
-                             int count,
-                             struct ompi_datatype_t *datatype,
-                             ompi_request_t **request)
-{
-    int ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret =
-        ROMIO_PREFIX(MPI_File_iwrite_at) (data->romio_fh, offset, buf, count,
-                                          datatype, request);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-
-int
-mca_io_romio321_file_iwrite_at_all (ompi_file_t *fh,
-                                    MPI_Offset offset,
-                                    const void *buf,
-                                    int count,
-                                    struct ompi_datatype_t *datatype,
-                                    ompi_request_t **request)
-{
-    int         ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    // ----------------------------------------------------
-    // NOTE: If you upgrade ROMIO, replace this with the actual ROMIO call.
-    // ----------------------------------------------------
-    // No support for non-blocking collective I/O operations.
-    // Fake it with individual non-blocking I/O operations.
-    // Similar to OMPIO
-    ret =
-        ROMIO_PREFIX(MPI_File_iwrite_at_all) (data->romio_fh, offset, buf, count,
-                                             datatype, request);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-
-
-int
-mca_io_romio321_file_write (ompi_file_t *fh,
-                         const void *buf,
-                         int count,
-                         struct ompi_datatype_t *datatype,
-                         ompi_status_public_t * status)
-{
-    int         ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret =
-        ROMIO_PREFIX(MPI_File_write) (data->romio_fh, buf, count, datatype,
-                                     status);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-int
-mca_io_romio321_file_write_all (ompi_file_t *fh,
-                             const void *buf,
-                             int count,
-                             struct ompi_datatype_t *datatype,
-                             ompi_status_public_t * status)
-{
-    int         ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret =
-        ROMIO_PREFIX(MPI_File_write_all) (data->romio_fh, buf, count, datatype,
-                                         status);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-int
-mca_io_romio321_file_iwrite (ompi_file_t *fh,
-                          const void *buf,
-                          int count,
-                          struct ompi_datatype_t *datatype,
-                          ompi_request_t **request)
-{
-    int ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret =
-        ROMIO_PREFIX(MPI_File_iwrite) (data->romio_fh, buf, count, datatype,
-                                       request);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-int
-mca_io_romio321_file_iwrite_all (ompi_file_t *fh,
-                                const void *buf,
-                                 int count,
-                                 struct ompi_datatype_t *datatype,
-                                 ompi_request_t **request)
-{
-    int         ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    // ----------------------------------------------------
-    // NOTE: If you upgrade ROMIO, replace this with the actual ROMIO call.
-    // ----------------------------------------------------
-    // No support for non-blocking collective I/O operations.
-    // Fake it with individual non-blocking I/O operations.
-    // Similar to OMPIO
-    ret =
-        ROMIO_PREFIX(MPI_File_iwrite_all) (data->romio_fh, buf, count, datatype,
-                                          request);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-
-int
-mca_io_romio321_file_write_shared (ompi_file_t *fh,
-                                const void *buf,
-                                int count,
-                                struct ompi_datatype_t *datatype,
-                                ompi_status_public_t * status)
-{
-    int         ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret =
-        ROMIO_PREFIX(MPI_File_write_shared) (data->romio_fh, buf, count,
-                                             datatype, status);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-int
-mca_io_romio321_file_iwrite_shared (ompi_file_t *fh,
-                                 const void *buf,
-                                 int count,
-                                 struct ompi_datatype_t *datatype,
-                                 ompi_request_t **request)
-{
-    int ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret =
-        ROMIO_PREFIX(MPI_File_iwrite_shared) (data->romio_fh, buf, count,
-                                              datatype, request);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-int
-mca_io_romio321_file_write_ordered (ompi_file_t *fh,
-                                 const void *buf,
-                                 int count,
-                                 struct ompi_datatype_t *datatype,
-                                 ompi_status_public_t * status)
-{
-    int         ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret =
-        ROMIO_PREFIX(MPI_File_write_ordered) (data->romio_fh, buf, count,
-                                             datatype, status);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-int
-mca_io_romio321_file_write_at_all_begin (ompi_file_t *fh,
-                                      MPI_Offset offset,
-                                      const void *buf,
-                                      int count,
-                                      struct ompi_datatype_t *datatype)
-{
-    int         ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret = ROMIO_PREFIX(MPI_File_write_at_all_begin) (data->romio_fh, offset,
-                                                     buf, count, datatype);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-int
-mca_io_romio321_file_write_at_all_end (ompi_file_t *fh,
-                                    const void *buf,
-                                    ompi_status_public_t * status)
-{
-    int         ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret = ROMIO_PREFIX(MPI_File_write_at_all_end) (data->romio_fh, buf,
-                                                   status);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-int
-mca_io_romio321_file_write_all_begin (ompi_file_t *fh,
-                                   const void *buf,
-                                   int count,
-                                   struct ompi_datatype_t *datatype)
-{
-    int         ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret = ROMIO_PREFIX(MPI_File_write_all_begin) (data->romio_fh, buf, count,
-                                                 datatype);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-int
-mca_io_romio321_file_write_all_end (ompi_file_t *fh,
-                                 const void *buf,
-                                 ompi_status_public_t * status)
-{
-    int         ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret = ROMIO_PREFIX(MPI_File_write_all_end) (data->romio_fh, buf, status);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-int
-mca_io_romio321_file_write_ordered_begin (ompi_file_t *fh,
-                                       const void *buf,
-                                       int count,
-                                       struct ompi_datatype_t *datatype)
-{
-    int         ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret = ROMIO_PREFIX(MPI_File_write_ordered_begin) (data->romio_fh, buf,
-                                                     count, datatype);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
-
-int
-mca_io_romio321_file_write_ordered_end (ompi_file_t *fh,
-                                     const void *buf,
-                                     ompi_status_public_t * status)
-{
-    int         ret;
-    mca_io_romio321_data_t *data;
-
-    data = (mca_io_romio321_data_t *) fh->f_io_selected_data;
-    OPAL_THREAD_LOCK (&mca_io_romio321_mutex);
-    ret = ROMIO_PREFIX(MPI_File_write_ordered_end) (data->romio_fh, buf,
-                                                    status);
-    OPAL_THREAD_UNLOCK (&mca_io_romio321_mutex);
-
-    return ret;
-}
diff --git a/ompi/mca/io/romio321/src/io_romio321_module.c b/ompi/mca/io/romio321/src/io_romio321_module.c
deleted file mode 100644
index 24b17d8a111..00000000000
--- a/ompi/mca/io/romio321/src/io_romio321_module.c
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2008 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2017-2019 IBM Corporation. All rights reserved.
- * Copyright (c) 2017      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include "mpi.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/io/io.h"
-#include "io_romio321.h"
-
-
-/*
- * Global functions that do not need to be prototyped in a header
- * because ROMIO just expects these functions to exist.
- */
-int MPIR_Status_set_bytes(ompi_status_public_t *status,
-                          struct ompi_datatype_t *datatype, MPI_Count size);
-void ADIOI_Datatype_iscontig(MPI_Datatype datatype, int *flag);
-
-
-/*
- * The ROMIO module operations
- */
-mca_io_base_module_2_0_0_t mca_io_romio321_module = {
-    /* Back end to MPI API calls (pretty much a 1-to-1 mapping) */
-
-    mca_io_romio321_file_open,
-    mca_io_romio321_file_close,
-
-    mca_io_romio321_file_set_size,
-    mca_io_romio321_file_preallocate,
-    mca_io_romio321_file_get_size,
-    mca_io_romio321_file_get_amode,
-    mca_io_romio321_file_set_info,
-    mca_io_romio321_file_get_info,
-    mca_io_romio321_file_set_view,
-    mca_io_romio321_file_get_view,
-
-    /* Index IO operations */
-    mca_io_romio321_file_read_at,
-    mca_io_romio321_file_read_at_all,
-    mca_io_romio321_file_write_at,
-    mca_io_romio321_file_write_at_all,
-    mca_io_romio321_file_iread_at,
-    mca_io_romio321_file_iwrite_at,
-    mca_io_romio321_file_iread_at_all,
-    mca_io_romio321_file_iwrite_at_all,
-
-    /* non-indexed IO operations */
-    mca_io_romio321_file_read,
-    mca_io_romio321_file_read_all,
-    mca_io_romio321_file_write,
-    mca_io_romio321_file_write_all,
-    mca_io_romio321_file_iread,
-    mca_io_romio321_file_iwrite,
-    mca_io_romio321_file_iread_all,
-    mca_io_romio321_file_iwrite_all,
-
-    mca_io_romio321_file_seek,
-    mca_io_romio321_file_get_position,
-    mca_io_romio321_file_get_byte_offset,
-
-    mca_io_romio321_file_read_shared,
-    mca_io_romio321_file_write_shared,
-    mca_io_romio321_file_iread_shared,
-    mca_io_romio321_file_iwrite_shared,
-    mca_io_romio321_file_read_ordered,
-    mca_io_romio321_file_write_ordered,
-    mca_io_romio321_file_seek_shared,
-    mca_io_romio321_file_get_position_shared,
-
-    /* Split IO operations */
-    mca_io_romio321_file_read_at_all_begin,
-    mca_io_romio321_file_read_at_all_end,
-    mca_io_romio321_file_write_at_all_begin,
-    mca_io_romio321_file_write_at_all_end,
-    mca_io_romio321_file_read_all_begin,
-    mca_io_romio321_file_read_all_end,
-    mca_io_romio321_file_write_all_begin,
-    mca_io_romio321_file_write_all_end,
-    mca_io_romio321_file_read_ordered_begin,
-    mca_io_romio321_file_read_ordered_end,
-    mca_io_romio321_file_write_ordered_begin,
-    mca_io_romio321_file_write_ordered_end,
-
-    mca_io_romio321_file_get_type_extent,
-
-    /* Sync/atomic IO operations */
-    mca_io_romio321_file_set_atomicity,
-    mca_io_romio321_file_get_atomicity,
-    mca_io_romio321_file_sync
-};
-
-
-/*
- * This function is required by ROMIO to set information on an Open
- * MPI status.  Conveniently, it maps directly to
- * MPI_Status_set_elements (almost like they planned that... hmmm...).
- */
-int MPIR_Status_set_bytes(ompi_status_public_t *status,
-                          struct ompi_datatype_t *datatype, MPI_Count nbytes)
-{
-    /* Note that ROMIO is going to give a number of *bytes* here, but
-       MPI_STATUS_SET_ELEMENTS requires a number of *elements*.  So
-       rather than try to do a conversion up here, just set the number
-       of bytes with MPI_CHAR as the datatype.  If someone does a
-       GET_STATUS later, they have to supply their own datatype, and
-       we do the right calculations there.  This prevents roundoff
-       errors here, potentially "losing" bytes in the process. */
-
-    MPI_Status_set_elements_x(status, MPI_CHAR, nbytes);
-    return MPI_SUCCESS;
-}
-
-
-void ADIOI_Datatype_iscontig(MPI_Datatype datatype, int *flag)
-{
-    /*
-     * Open MPI contiguous check return true for datatype with
-     * gaps in the beginning and at the end. We have to provide
-     * a count of 2 in order to get these gaps taken into acount.
-     * In addition, if the data is contiguous but true_lb differes
-     * from zero, ROMIO will ignore the displacement. Thus, lie!
-     */
-    *flag = ompi_datatype_is_contiguous_memory_layout(datatype, 2);
-    if (*flag) {
-        MPI_Aint true_extent, true_lb;
-
-        ompi_datatype_get_true_extent(datatype, &true_lb, &true_extent);
-
-        if (true_lb > 0)
-            *flag = 0;
-    }
-}
diff --git a/ompi/mca/mca.h b/ompi/mca/mca.h
index f9e4ea341a4..b7c6df4c2a9 100644
--- a/ompi/mca/mca.h
+++ b/ompi/mca/mca.h
@@ -22,7 +22,7 @@
 /**
  * @file
  *
- * Top-level interface for \em all orte MCA components.
+ * Top-level interface for all MCA components.
  */
 
 #ifndef OMPI_MCA_H
diff --git a/ompi/mca/mtl/base/mtl_base_datatype.h b/ompi/mca/mtl/base/mtl_base_datatype.h
index 544ca32abc7..9ea03987c0d 100644
--- a/ompi/mca/mtl/base/mtl_base_datatype.h
+++ b/ompi/mca/mtl/base/mtl_base_datatype.h
@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2006 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -25,177 +26,87 @@
 #include "ompi/datatype/ompi_datatype.h"
 #include "opal/datatype/opal_convertor.h"
 #include "opal/datatype/opal_datatype_internal.h"
-#if OPAL_CUDA_SUPPORT
-#include "opal/mca/common/cuda/common_cuda.h"
-#include "opal/datatype/opal_convertor.h"
-#endif
+#include "opal/mca/accelerator/accelerator.h"
 
 #ifndef MTL_BASE_DATATYPE_H_INCLUDED
 #define MTL_BASE_DATATYPE_H_INCLUDED
 
-#if OPAL_CUDA_SUPPORT
-static int
-ompi_mtl_cuda_datatype_pack(struct opal_convertor_t *convertor,
-                            void **buffer,
-                            size_t *buffer_len,
-                            bool *freeAfter)
-{
-
-    struct iovec iov;
-    uint32_t iov_count = 1;
-    int is_cuda = convertor->flags & CONVERTOR_CUDA;
-
-#if !(OPAL_ENABLE_HETEROGENEOUS_SUPPORT)
-    if (convertor->pDesc &&
-	!(convertor->flags & CONVERTOR_COMPLETED) &&
-	opal_datatype_is_contiguous_memory_layout(convertor->pDesc,
-						  convertor->count)) {
-	    *freeAfter = false;
-	    *buffer = convertor->pBaseBuf;
-	    *buffer_len = convertor->local_size;
-	    return OPAL_SUCCESS;
-    }
-#endif
-
-    opal_convertor_get_packed_size(convertor, buffer_len);
-    *freeAfter  = false;
-    if( 0 == *buffer_len ) {
-        *buffer     = NULL;
-        return OMPI_SUCCESS;
-    }
-    iov.iov_len = *buffer_len;
-    iov.iov_base = NULL;
-    /* opal_convertor_need_buffers always returns true
-     * if CONVERTOR_CUDA is set, so unset temporarily
-     */
-    convertor->flags &= ~CONVERTOR_CUDA;
-
-    if (opal_convertor_need_buffers(convertor)) {
-        if (is_cuda) {
-            convertor->flags |= CONVERTOR_CUDA;
-        }
-        iov.iov_base = opal_cuda_malloc(*buffer_len, convertor);
-        if (NULL == iov.iov_base) return OMPI_ERR_OUT_OF_RESOURCE;
-        *freeAfter = true;
-    } else if (is_cuda) {
-            convertor->flags |= CONVERTOR_CUDA;
-    }
-
-    opal_convertor_pack( convertor, &iov, &iov_count, buffer_len );
-
-    *buffer = iov.iov_base;
-
-    return OMPI_SUCCESS;
-}
-#endif
-
 __opal_attribute_always_inline__ static inline int
 ompi_mtl_datatype_pack(struct opal_convertor_t *convertor,
                        void **buffer,
                        size_t *buffer_len,
-                       bool *freeAfter)
+                       bool *free_after)
 {
-#if OPAL_CUDA_SUPPORT
-    return ompi_mtl_cuda_datatype_pack(convertor,
-                                       buffer,
-                                       buffer_len,
-                                       freeAfter);
-#endif
     struct iovec iov;
     uint32_t iov_count = 1;
-
+    bool is_accelerator = opal_convertor_on_device(convertor);
 #if !(OPAL_ENABLE_HETEROGENEOUS_SUPPORT)
     if (convertor->pDesc &&
 	!(convertor->flags & CONVERTOR_COMPLETED) &&
 	opal_datatype_is_contiguous_memory_layout(convertor->pDesc,
-						  convertor->count)) {
-	    *freeAfter = false;
-	    *buffer = convertor->pBaseBuf;
+						  convertor->count) &&
+        !is_accelerator) {
+	    *free_after = false;
+	    *buffer = convertor->pBaseBuf + convertor->bConverted + convertor->pDesc->true_lb;
 	    *buffer_len = convertor->local_size;
 	    return OPAL_SUCCESS;
     }
 #endif
 
     opal_convertor_get_packed_size(convertor, buffer_len);
-    *freeAfter  = false;
+    *free_after = false;
     if( 0 == *buffer_len ) {
-        *buffer     = NULL;
+        *buffer = NULL;
         return OMPI_SUCCESS;
     }
     iov.iov_len = *buffer_len;
     iov.iov_base = NULL;
-    if (opal_convertor_need_buffers(convertor)) {
-        iov.iov_base = malloc(*buffer_len);
+    /* If we need buffers or we don't have accelerator support and it is a device buffer, we will need to copy */
+    if (opal_convertor_need_buffers(convertor) || (is_accelerator && false == ompi_mtl_base_selected_component->accelerator_support)) {
+        /* If this is a host buffer or an accelerator buffer and we don't have accelerator support we need to host malloc */
+        if (!is_accelerator || (is_accelerator && false == ompi_mtl_base_selected_component->accelerator_support)) {
+            iov.iov_base = malloc(*buffer_len);
+        /* The remaining case is if this is an accelerator buffer and we have accelerator support and we need buffers. We will need to do an accelerator malloc*/
+        } else {
+            opal_accelerator.mem_alloc(MCA_ACCELERATOR_NO_DEVICE_ID, &iov.iov_base, *buffer_len);
+        }
+
         if (NULL == iov.iov_base) return OMPI_ERR_OUT_OF_RESOURCE;
-        *freeAfter = true;
+        *free_after = true;
     }
 
-    opal_convertor_pack( convertor, &iov, &iov_count, buffer_len );
+    opal_convertor_pack(convertor, &iov, &iov_count, buffer_len);
 
     *buffer = iov.iov_base;
-
     return OMPI_SUCCESS;
 }
 
-#if OPAL_CUDA_SUPPORT
-static int
-ompi_mtl_cuda_datatype_recv_buf(struct opal_convertor_t *convertor,
-                                void ** buffer,
-                                size_t *buffer_len,
-                                bool *free_on_error)
-{
-    int is_cuda = convertor->flags & CONVERTOR_CUDA;
-    opal_convertor_get_packed_size(convertor, buffer_len);
-    *free_on_error = false;
-    if( 0 == *buffer_len ) {
-        *buffer = NULL;
-        *buffer_len = 0;
-        return OMPI_SUCCESS;
-    }
-    /* opal_convertor_need_buffers always returns true
-     * if CONVERTOR_CUDA is set, so unset temporarily
-     */
-    convertor->flags &= ~CONVERTOR_CUDA;
-    if (opal_convertor_need_buffers(convertor)) {
-        if (is_cuda) {
-            convertor->flags |= CONVERTOR_CUDA;
-        }
-        *buffer = opal_cuda_malloc(*buffer_len, convertor);
-        *free_on_error = true;
-    } else {
-        if (is_cuda) {
-            convertor->flags |= CONVERTOR_CUDA;
-        }
-        *buffer = convertor->pBaseBuf +
-            convertor->use_desc->desc[convertor->use_desc->used].end_loop.first_elem_disp;
-    }
-    return OMPI_SUCCESS;
-
-}
-#endif
-
 __opal_attribute_always_inline__ static inline int
 ompi_mtl_datatype_recv_buf(struct opal_convertor_t *convertor,
-                           void ** buffer,
+                           void **buffer,
                            size_t *buffer_len,
                            bool *free_on_error)
 {
-#if OPAL_CUDA_SUPPORT
-    return ompi_mtl_cuda_datatype_recv_buf(convertor,
-                                           buffer,
-                                           buffer_len,
-                                           free_on_error);
-#endif
-
     opal_convertor_get_packed_size(convertor, buffer_len);
     *free_on_error = false;
+    *buffer = NULL;
     if( 0 == *buffer_len ) {
-        *buffer = NULL;
         *buffer_len = 0;
         return OMPI_SUCCESS;
     }
-    if (opal_convertor_need_buffers(convertor)) {
-        *buffer = malloc(*buffer_len);
+    bool is_accelerator = opal_convertor_on_device(convertor);;
+
+    /* If we need buffers or we don't have accelerator support and it is a device buffer, we will need to copy */
+    if (opal_convertor_need_buffers(convertor) || (is_accelerator && false == ompi_mtl_base_selected_component->accelerator_support)) {
+        /* If this is a host buffer or an accelerator buffer and we don't have accelerator support we need to host malloc */
+        if (!is_accelerator || (is_accelerator && false == ompi_mtl_base_selected_component->accelerator_support)) {
+            *buffer = malloc(*buffer_len);
+        /* The remaining case is if this is an accelerator buffer and we have accelerator support and we need buffers. Wwe will need to do an accelerator malloc*/
+        } else {
+            opal_accelerator.mem_alloc(MCA_ACCELERATOR_NO_DEVICE_ID, buffer, *buffer_len);
+        }
+
+        if (NULL == *buffer) return OMPI_ERR_OUT_OF_RESOURCE;
         *free_on_error = true;
     } else {
         *buffer = convertor->pBaseBuf +
@@ -204,58 +115,28 @@ ompi_mtl_datatype_recv_buf(struct opal_convertor_t *convertor,
     return OMPI_SUCCESS;
 }
 
-#if OPAL_CUDA_SUPPORT
-static int
-ompi_mtl_cuda_datatype_unpack(struct opal_convertor_t *convertor,
-                              void *buffer,
-                              size_t buffer_len) {
-    struct iovec iov;
-    uint32_t iov_count = 1;
-    int is_cuda = convertor->flags & CONVERTOR_CUDA;
-    /* opal_convertor_need_buffers always returns true
-     * if CONVERTOR_CUDA is set, so unset temporarily
-     */
-     convertor->flags &= ~CONVERTOR_CUDA;
-
-    if (buffer_len > 0 && opal_convertor_need_buffers(convertor)) {
-        iov.iov_len = buffer_len;
-        iov.iov_base = buffer;
-
-        if (is_cuda) {
-            convertor->flags |= CONVERTOR_CUDA;
-        }
-        opal_convertor_unpack(convertor, &iov, &iov_count, &buffer_len );
-
-        opal_cuda_free(buffer, convertor);
-    } else if (is_cuda) {
-        convertor->flags |= CONVERTOR_CUDA;
-    }
-
-    return OMPI_SUCCESS;
-
-}
-#endif
-
 __opal_attribute_always_inline__ static inline int
 ompi_mtl_datatype_unpack(struct opal_convertor_t *convertor,
                          void *buffer,
                          size_t buffer_len)
 {
-#if OPAL_CUDA_SUPPORT
-    return ompi_mtl_cuda_datatype_unpack(convertor,
-                                         buffer,
-                                         buffer_len);
-#endif
     struct iovec iov;
     uint32_t iov_count = 1;
+    bool is_accelerator = opal_convertor_on_device(convertor);
 
-    if (buffer_len > 0 && opal_convertor_need_buffers(convertor)) {
+    /* If the buffer length is greater than 0 and we allocated buffers previously, we need to unpack them */
+    if (buffer_len > 0 && (opal_convertor_need_buffers(convertor) || (is_accelerator && false == ompi_mtl_base_selected_component->accelerator_support))) {
         iov.iov_len = buffer_len;
         iov.iov_base = buffer;
 
-        opal_convertor_unpack(convertor, &iov, &iov_count, &buffer_len );
-
-        free(buffer);
+        opal_convertor_unpack(convertor, &iov, &iov_count, &buffer_len);
+        /* If it's an accelerator buffer and we have accelerator support, we will need to free the accelerator buffer */
+        if (is_accelerator && true == ompi_mtl_base_selected_component->accelerator_support) {
+            opal_accelerator.mem_release(MCA_ACCELERATOR_NO_DEVICE_ID, buffer);
+        /* If it's not an accelerator buffer or we don't have accelerator support, we will need to free the host buffer */
+        } else {
+            free(buffer);
+        }
     }
 
     return OMPI_SUCCESS;
diff --git a/ompi/mca/mtl/base/mtl_base_frame.c b/ompi/mca/mtl/base/mtl_base_frame.c
index 757ad93ce7a..0d382d52b5a 100644
--- a/ompi/mca/mtl/base/mtl_base_frame.c
+++ b/ompi/mca/mtl/base/mtl_base_frame.c
@@ -76,7 +76,7 @@ ompi_mtl_base_select (bool enable_progress_threads,
                          best_component->mtl_version.mca_component_name );
 
     if (NULL == best_component->mtl_init(enable_progress_threads,
-                                          enable_mpi_threads)) {
+                                          enable_mpi_threads, &best_component->accelerator_support)) {
         opal_output_verbose( 10, ompi_mtl_base_framework.framework_output,
                              "select: init returned failure for component %s",
                              best_component->mtl_version.mca_component_name );
diff --git a/ompi/mca/mtl/mtl.h b/ompi/mca/mtl/mtl.h
index 24b2153064d..e88d7a9c9f2 100644
--- a/ompi/mca/mtl/mtl.h
+++ b/ompi/mca/mtl/mtl.h
@@ -62,9 +62,8 @@ typedef struct mca_mtl_request_t mca_mtl_request_t;
  * MTL module flags
  */
 #define MCA_MTL_BASE_FLAG_REQUIRE_WORLD 0x00000001
-#if OPAL_CUDA_SUPPORT
-#define MCA_MTL_BASE_FLAG_CUDA_INIT_DISABLE 0x00000002
-#endif
+#define MCA_MTL_BASE_FLAG_ACCELERATOR_INIT_DISABLE 0x00000002
+#define MCA_MTL_BASE_FLAG_SUPPORTS_EXT_CID 0x00000004
 
 /**
  * Initialization routine for MTL component
@@ -78,7 +77,7 @@ typedef struct mca_mtl_request_t mca_mtl_request_t;
  *
  * @param enable_progress_threads (IN) Progress threads have been
  *                  enabled by the user and the component must be
- *                  capable of making asycnhronous progress (either
+ *                  capable of making asynchronous progress (either
  *                  with its own thread, with the kernel, or with
  *                  the event library.
  * @param enable_mpi_threads (IN) MPI threads have been enabled by the
@@ -92,19 +91,23 @@ typedef struct mca_mtl_request_t mca_mtl_request_t;
  *                  support MPI_THREAD_MULTIPLE.  This variable only
  *                  needs to be set if enable_mpi_threads is true.
  *                  Otherwise, the return value will be ignored.
+ * @param accelerator_support (OUT) Component does / does not support
+ *                  direct transfers with an accelerator buffer.
  *
  * @retval NULL     component can not operate on the current machine
  * @retval non-NULL component interface function
  */
 typedef struct mca_mtl_base_module_t*
 (*mca_mtl_base_component_init_fn_t)(bool enable_progress_threads,
-                                    bool enable_mpi_threads);
+                                    bool enable_mpi_threads,
+                                    bool *accelerator_support);
 
 
 struct mca_mtl_base_component_2_0_0_t {
   mca_base_component_t mtl_version;
   mca_base_component_data_t mtl_data;
   mca_mtl_base_component_init_fn_t mtl_init;
+  bool accelerator_support;
 };
 typedef struct mca_mtl_base_component_2_0_0_t mca_mtl_base_component_2_0_0_t;
 typedef struct mca_mtl_base_component_2_0_0_t mca_mtl_base_component_t;
@@ -114,7 +117,7 @@ typedef struct mca_mtl_base_component_2_0_0_t mca_mtl_base_component_t;
  * MCA->MTL Clean up any resources held by MTL module
  *
  * Opposite of module_init.  Called when communication will no longer
- * be necessary.  ussually this is during MPI_FINALIZE, but it can be
+ * be necessary.  Usually this is during MPI_FINALIZE, but it can be
  * earlier if the component was not selected to run.  Assuming
  * module_init was called, finalize will always be called before the
  * component_close function is called.
@@ -204,7 +207,7 @@ typedef int (*mca_mtl_base_module_del_procs_fn_t)(
  * \note Open MPI is built around non-blocking operations.  This
  * function is provided for networks where progressing events outside
  * of point-to-point (for example, collectives, I/O, one-sided) can
- * occur without a progress function regularily being triggered.
+ * occur without a progress function regularly being triggered.
  *
  * \note While MPI does not allow users to specify negative tags, they
  * are used internally in Open MPI to provide a unique channel for
@@ -232,7 +235,7 @@ typedef int (*mca_mtl_base_module_send_fn_t)(
  * bytes requested in the module structure available for the MTL
  * directly after the ompi_request_t structure.  The PML will handle
  * proper destruction of the request once it can safely be destructed
- * (it has been completed and freeed by a call to REQUEST_FReE or
+ * (it has been completed and freed by a call to REQUEST_FReE or
  * TEST/WAIT).  The MTL should remove all resources associated with
  * the request when it is marked as completed.
  *
@@ -277,7 +280,7 @@ typedef int (*mca_mtl_base_module_isend_fn_t)(
  * MPI_Irecv must be implemented by this call.
  *
  * The PML will handle creation of the request, leaving the number of
- * bytes requested in teh module structure available for the MTL,
+ * bytes requested in the module structure available for the MTL,
  * directly after the ompi_request_t structure.  The PML will handle
  * proper destruction of the request once it can safely be destroyed
  * (it has been completed and free'ed by a call to REQUEST_FREE or
diff --git a/ompi/mca/mtl/ofi/Makefile.am b/ompi/mca/mtl/ofi/Makefile.am
index 832e2b8e6ec..5842abd3018 100644
--- a/ompi/mca/mtl/ofi/Makefile.am
+++ b/ompi/mca/mtl/ofi/Makefile.am
@@ -9,6 +9,8 @@
 #                         and Technology (RIST).  All rights reserved.
 # Copyright (c) 2020      Triad National Security, LLC. All rights
 #                         reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -17,14 +19,13 @@
 #
 
 EXTRA_DIST = post_configure.sh \
+    help-mtl-ofi.txt \
     $(generated_source_modules)
 
 MAINTAINERCLEANFILES = \
     $(generated_sources)
 
-AM_CPPFLAGS = $(opal_ofi_CPPFLAGS)
-
-dist_ompidata_DATA = help-mtl-ofi.txt
+AM_CPPFLAGS = $(mtl_ofi_CPPFLAGS)
 
 generated_source_modules = \
     mtl_ofi_send_opt.pm \
@@ -77,15 +78,15 @@ mcacomponentdir = $(ompilibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_mtl_ofi_la_SOURCES = $(mtl_ofi_sources)
 mca_mtl_ofi_la_LDFLAGS = \
-        $(opal_ofi_LDFLAGS) \
+        $(mtl_ofi_LDFLAGS) \
         -module -avoid-version
 mca_mtl_ofi_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
-        $(OPAL_TOP_BUILDDIR)/opal/mca/common/ofi/lib@OPAL_LIB_PREFIX@mca_common_ofi.la \
-        $(opal_ofi_LIBS)
+        $(OPAL_TOP_BUILDDIR)/opal/mca/common/ofi/lib@OPAL_LIB_NAME@mca_common_ofi.la \
+        $(mtl_ofi_LIBS)
 
 noinst_LTLIBRARIES = $(component_noinst)
 libmca_mtl_ofi_la_SOURCES = $(mtl_ofi_sources)
 libmca_mtl_ofi_la_LDFLAGS = \
-        $(opal_ofi_LDFLAGS) \
+        $(mtl_ofi_LDFLAGS) \
         -module -avoid-version
-libmca_mtl_ofi_la_LIBADD = $(opal_ofi_LIBS)
+libmca_mtl_ofi_la_LIBADD = $(mtl_ofi_LIBS)
diff --git a/ompi/mca/mtl/ofi/README.md b/ompi/mca/mtl/ofi/README.md
index 109ab23cd4a..0f9adc91c7f 100644
--- a/ompi/mca/mtl/ofi/README.md
+++ b/ompi/mca/mtl/ofi/README.md
@@ -93,7 +93,7 @@ instead of creating them all at once during init time and this
 approach also favours only creating as many contexts as needed.
 
 1. Multi-communicator model:
-   With this approach, the MPI application is requried to first duplicate
+   With this approach, the MPI application is required to first duplicate
    the communicators it wants to use with MPI operations (ideally creating
    as many communicators as the number of threads it wants to use to call
    into MPI). The duplicated communicators are then used by the
diff --git a/ompi/mca/mtl/ofi/configure.m4 b/ompi/mca/mtl/ofi/configure.m4
index 678247c5d35..c65d266adc3 100644
--- a/ompi/mca/mtl/ofi/configure.m4
+++ b/ompi/mca/mtl/ofi/configure.m4
@@ -1,10 +1,11 @@
-# -*- shell-script -*-
+# -*- autoconf -*-
 #
 # Copyright (c) 2013-2014 Intel, Inc. All rights reserved
 #
 # Copyright (c) 2014-2020 Cisco Systems, Inc.  All rights reserved
 # Copyright (c) 2017      Los Alamos National Security, LLC.  All rights
 #                         reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -23,29 +24,33 @@ AC_DEFUN([MCA_ompi_mtl_ofi_POST_CONFIG], [
 #                    [action-if-cant-compile])
 # ------------------------------------------------
 AC_DEFUN([MCA_ompi_mtl_ofi_CONFIG],[
-    AC_CONFIG_FILES([ompi/mca/mtl/ofi/Makefile])
+    OPAL_VAR_SCOPE_PUSH([mtl_ofi_happy])
 
-    # Check for OFI
-    OPAL_CHECK_OFI
+    AC_CONFIG_FILES([ompi/mca/mtl/ofi/Makefile])
 
-    # Check for CUDA
-    OPAL_CHECK_CUDA
+    OPAL_CHECK_OFI([mtl_ofi],
+                   [mtl_ofi_happy=1],
+                   [mtl_ofi_happy=0])
 
-    # Check for cuda support. If so, we require a minimum libfabric version
-    # of 1.9. FI_HMEM capabilities are only available starting from v1.9
-    opal_ofi_happy="yes"
-    AS_IF([test "$opal_check_cuda_happy" = "yes"],
+    dnl The OFI MTL requires at least OFI libfabric v1.9.
+    AS_IF([test ${mtl_ofi_happy} -eq 1],
           [OPAL_CHECK_OFI_VERSION_GE([1,9],
                                      [],
-                                     [opal_ofi_happy=no])])
+                                     [mtl_ofi_happy=0])])
 
-    # The OFI MTL requires at least OFI libfabric v1.5.
-    AS_IF([test "$opal_ofi_happy" = "yes"],
-          [OPAL_CHECK_OFI_VERSION_GE([1,5],
-                                     [],
-                                     [opal_ofi_happy=no])])
+    AS_IF([test ${mtl_ofi_happy} -eq 1],
+          [OPAL_MCA_CHECK_DEPENDENCY([opal], [btl], [ofi], [opal], [common], [ofi])])
 
-    AS_IF([test "$opal_ofi_happy" = "yes"],
+    AS_IF([test ${mtl_ofi_happy} -eq 1],
           [$1],
-          [$2])
+          [AS_IF([test -n "${with_ofi}" -a "${with_ofi}" != "no"],
+                 [AC_MSG_WARN([OFI libfabric support requested (via --with-ofi or --with-libfabric), but not found.])
+                  AC_MSG_ERROR([Cannot continue.])])
+           $2])
+
+    AC_SUBST([mtl_ofi_CPPFLAGS])
+    AC_SUBST([mtl_ofi_LDFLAGS])
+    AC_SUBST([mtl_ofi_LIBS])
+
+    OPAL_VAR_SCOPE_POP
 ])dnl
diff --git a/ompi/mca/mtl/ofi/generate-opt-funcs.pl b/ompi/mca/mtl/ofi/generate-opt-funcs.pl
old mode 100644
new mode 100755
diff --git a/ompi/mca/mtl/ofi/mtl_ofi.c b/ompi/mca/mtl/ofi/mtl_ofi.c
index 3987f275f29..35cb52443d9 100644
--- a/ompi/mca/mtl/ofi/mtl_ofi.c
+++ b/ompi/mca/mtl/ofi/mtl_ofi.c
@@ -1,5 +1,7 @@
 /*
  * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2021-2022 Triad National Security, LLC. All rights
+ *                         reserved.
  *
  * $COPYRIGHT$
  *
@@ -12,6 +14,8 @@
 
 OMPI_DECLSPEC extern mca_mtl_ofi_component_t mca_mtl_ofi_component;
 
+OBJ_CLASS_INSTANCE(mca_mtl_comm_t, opal_object_t, NULL, NULL);
+
 mca_mtl_ofi_module_t ompi_mtl_ofi = {
     {
         (int)((1ULL << MTL_OFI_CID_BIT_COUNT_1) - 1), /* max cid */
@@ -40,6 +44,178 @@ mca_mtl_ofi_module_t ompi_mtl_ofi = {
     NULL
 };
 
+
+static int ompi_mtl_ofi_init_contexts(struct mca_mtl_base_module_t *mtl,
+                                      struct ompi_communicator_t *comm,
+                                      mca_mtl_ofi_ep_type ep_type)
+{
+    int ret;
+    int ctxt_id = ompi_mtl_ofi.total_ctxts_used;
+    struct fi_cq_attr cq_attr = {0};
+    cq_attr.format = FI_CQ_FORMAT_TAGGED;
+    cq_attr.size = ompi_mtl_ofi.ofi_progress_event_count;
+
+    if (OFI_REGULAR_EP == ep_type) {
+        /*
+         * For regular endpoints, just create the Lock object and register
+         * progress function.
+         */
+        goto init_regular_ep;
+    }
+
+    /*
+     * We only create upto Max number of contexts asked for by the user.
+     * If user enables thread grouping feature and creates more number of
+     * communicators than available contexts, then we set the threshold
+     * context_id so that new communicators created beyond the threshold
+     * will be assigned to contexts in a round-robin fashion.
+     */
+    if (ompi_mtl_ofi.num_ofi_contexts <= ompi_mtl_ofi.total_ctxts_used) {
+        ompi_mtl_ofi.comm_to_context[comm->c_index] = comm->c_index %
+                                                          ompi_mtl_ofi.total_ctxts_used;
+        if (!ompi_mtl_ofi.threshold_comm_context_id) {
+            ompi_mtl_ofi.threshold_comm_context_id = comm->c_index;
+
+            opal_show_help("help-mtl-ofi.txt", "SEP thread grouping ctxt limit", true, ctxt_id,
+                           ompi_process_info.nodename, __FILE__, __LINE__);
+        }
+
+        return OMPI_SUCCESS;
+    }
+
+    /* Init context info for Scalable EPs */
+    ret = fi_tx_context(ompi_mtl_ofi.sep, ctxt_id, NULL, &ompi_mtl_ofi.ofi_ctxt[ctxt_id].tx_ep, NULL);
+    if (ret) {
+        MTL_OFI_LOG_FI_ERR(ret, "fi_tx_context failed");
+        goto init_error;
+    }
+
+    ret = fi_rx_context(ompi_mtl_ofi.sep, ctxt_id, NULL, &ompi_mtl_ofi.ofi_ctxt[ctxt_id].rx_ep, NULL);
+    if (ret) {
+        MTL_OFI_LOG_FI_ERR(ret, "fi_rx_context failed");
+        goto init_error;
+    }
+
+    ret = fi_cq_open(ompi_mtl_ofi.domain, &cq_attr, &ompi_mtl_ofi.ofi_ctxt[ctxt_id].cq, NULL);
+    if (ret) {
+        MTL_OFI_LOG_FI_ERR(ret, "fi_cq_open failed");
+        goto init_error;
+    }
+
+    /* Bind CQ to TX/RX context object */
+    ret = fi_ep_bind(ompi_mtl_ofi.ofi_ctxt[ctxt_id].tx_ep, (fid_t)ompi_mtl_ofi.ofi_ctxt[ctxt_id].cq,
+                     FI_TRANSMIT | FI_SELECTIVE_COMPLETION);
+    if (0 != ret) {
+        MTL_OFI_LOG_FI_ERR(ret, "fi_bind CQ-EP (FI_TRANSMIT) failed");
+        goto init_error;
+    }
+
+    ret = fi_ep_bind(ompi_mtl_ofi.ofi_ctxt[ctxt_id].rx_ep, (fid_t)ompi_mtl_ofi.ofi_ctxt[ctxt_id].cq,
+                     FI_RECV | FI_SELECTIVE_COMPLETION);
+    if (0 != ret) {
+        MTL_OFI_LOG_FI_ERR(ret, "fi_bind CQ-EP (FI_RECV) failed");
+        goto init_error;
+    }
+
+    /* Enable Endpoint for communication. This commits the bind operations */
+    ret = fi_enable(ompi_mtl_ofi.ofi_ctxt[ctxt_id].tx_ep);
+    if (0 != ret) {
+        MTL_OFI_LOG_FI_ERR(ret, "fi_enable (send context) failed");
+        goto init_error;
+    }
+
+    ret = fi_enable(ompi_mtl_ofi.ofi_ctxt[ctxt_id].rx_ep);
+    if (0 != ret) {
+        MTL_OFI_LOG_FI_ERR(ret, "fi_enable (recv context) failed");
+        goto init_error;
+    }
+
+init_regular_ep:
+    /* Initialize per-context lock */
+    OBJ_CONSTRUCT(&ompi_mtl_ofi.ofi_ctxt[ctxt_id].context_lock, opal_mutex_t);
+
+    if (!ompi_mtl_ofi.is_initialized) {
+        ret = opal_progress_register(ompi_mtl_ofi_progress_no_inline);
+        if (OMPI_SUCCESS != ret) {
+            opal_output_verbose(1, opal_common_ofi.output,
+                                "%s:%d: opal_progress_register failed: %d\n",
+                                __FILE__, __LINE__, ret);
+            goto init_error;
+        }
+    }
+
+    ompi_mtl_ofi.comm_to_context[comm->c_index] = ompi_mtl_ofi.total_ctxts_used;
+    ompi_mtl_ofi.total_ctxts_used++;
+
+    return OMPI_SUCCESS;
+
+init_error:
+    if (ompi_mtl_ofi.ofi_ctxt[ctxt_id].tx_ep) {
+        (void) fi_close((fid_t)ompi_mtl_ofi.ofi_ctxt[ctxt_id].tx_ep);
+    }
+
+    if (ompi_mtl_ofi.ofi_ctxt[ctxt_id].rx_ep) {
+        (void) fi_close((fid_t)ompi_mtl_ofi.ofi_ctxt[ctxt_id].rx_ep);
+    }
+
+    if (ompi_mtl_ofi.ofi_ctxt[ctxt_id].cq) {
+        (void) fi_close((fid_t)ompi_mtl_ofi.ofi_ctxt[ctxt_id].cq);
+    }
+
+    return ret;
+}
+
+static int ompi_mtl_ofi_finalize_contexts(struct mca_mtl_base_module_t *mtl,
+                                          struct ompi_communicator_t *comm,
+                                          mca_mtl_ofi_ep_type ep_type)
+{
+    int ret = OMPI_SUCCESS, ctxt_id = 0;
+
+    if (OFI_REGULAR_EP == ep_type) {
+        /* For regular EPs, simply destruct Lock object and exit */
+        goto finalize_regular_ep;
+    }
+
+    if (ompi_mtl_ofi.thread_grouping &&
+        ompi_mtl_ofi.threshold_comm_context_id &&
+        ((uint32_t) ompi_mtl_ofi.threshold_comm_context_id <= comm->c_index)) {
+        return OMPI_SUCCESS;
+    }
+
+    ctxt_id = ompi_mtl_ofi.thread_grouping ?
+           ompi_mtl_ofi.comm_to_context[comm->c_index] : 0;
+
+    /*
+     * For regular EPs, TX/RX contexts are aliased to SEP object which is
+     * closed in ompi_mtl_ofi_finalize(). So, skip handling those here.
+     */
+    if ((ret = fi_close((fid_t)ompi_mtl_ofi.ofi_ctxt[ctxt_id].tx_ep))) {
+        goto finalize_err;
+    }
+
+    if ((ret = fi_close((fid_t)ompi_mtl_ofi.ofi_ctxt[ctxt_id].rx_ep))) {
+        goto finalize_err;
+    }
+
+    if ((ret = fi_close((fid_t)ompi_mtl_ofi.ofi_ctxt[ctxt_id].cq))) {
+        goto finalize_err;
+    }
+
+finalize_regular_ep:
+    /* Destroy context lock */
+    OBJ_DESTRUCT(&ompi_mtl_ofi.ofi_ctxt[ctxt_id].context_lock);
+
+    return OMPI_SUCCESS;
+
+finalize_err:
+    opal_show_help("help-mtl-ofi.txt", "OFI call fail", true,
+                   "fi_close",
+                   ompi_process_info.nodename, __FILE__, __LINE__,
+                   fi_strerror(-ret), ret);
+
+    return OMPI_ERROR;
+}
+
 int
 ompi_mtl_ofi_add_procs(struct mca_mtl_base_module_t *mtl,
                        size_t nprocs,
@@ -48,16 +224,12 @@ ompi_mtl_ofi_add_procs(struct mca_mtl_base_module_t *mtl,
     int ret = OMPI_SUCCESS;
     size_t i;
     size_t size;
-    size_t namelen;
     int count = 0;
     char *ep_name = NULL;
-    char *ep_names = NULL;
     fi_addr_t *fi_addrs = NULL;
     mca_mtl_ofi_endpoint_t *endpoint = NULL;
     int num_peers_limit = (1 << ompi_mtl_ofi.num_bits_source_rank) - 1;
 
-    namelen = ompi_mtl_ofi.epnamelen;
-
     /* We cannot add more ranks than available tag bits */
     if ((false == ompi_mtl_ofi.fi_cq_data) &&
         OPAL_UNLIKELY(((int) (nprocs + ompi_mtl_ofi.num_peers) > num_peers_limit))) {
@@ -70,15 +242,6 @@ ompi_mtl_ofi_add_procs(struct mca_mtl_base_module_t *mtl,
         goto bail;
     }
 
-    /**
-     * Create array of EP names.
-     */
-    ep_names = malloc(nprocs * namelen);
-    if (NULL == ep_names) {
-        ret = OMPI_ERROR;
-        goto bail;
-    }
-
     /**
      * Create array of fi_addrs.
      */
@@ -88,10 +251,10 @@ ompi_mtl_ofi_add_procs(struct mca_mtl_base_module_t *mtl,
         goto bail;
     }
 
-    /**
-     * Retrieve the processes' EP names from modex.
-     */
     for (i = 0; i < nprocs; ++i) {
+        /**
+         * Retrieve the processes' EP name from modex.
+         */
         OFI_COMPAT_MODEX_RECV(ret,
                               &mca_mtl_ofi_component.super.mtl_version,
                               procs[i],
@@ -105,19 +268,18 @@ ompi_mtl_ofi_add_procs(struct mca_mtl_base_module_t *mtl,
             free(errhost);
             goto bail;
         }
-        memcpy(&ep_names[i*namelen], ep_name, namelen);
-    }
 
-    /**
-     * Map the EP names to fi_addrs.
-     */
-    count = fi_av_insert(ompi_mtl_ofi.av, ep_names, nprocs, fi_addrs, 0, NULL);
-    if ((count < 0) || (nprocs != (size_t)count)) {
-        opal_output_verbose(1, opal_common_ofi.output,
-                            "%s:%d: fi_av_insert failed: %d\n",
-                            __FILE__, __LINE__, count);
-        ret = OMPI_ERROR;
-        goto bail;
+        /**
+         * Map the EP name to fi_addr.
+         */
+        count = fi_av_insert(ompi_mtl_ofi.av, ep_name, 1, &fi_addrs[i], 0, NULL);
+        if ((count < 0) || (1 != (size_t)count)) {
+            opal_output_verbose(1, opal_common_ofi.output,
+                                "%s:%d: fi_av_insert failed for address %s: %d\n",
+                                __FILE__, __LINE__, ep_name, count);
+            ret = OMPI_ERROR;
+            goto bail;
+        }
     }
 
     /**
@@ -150,9 +312,6 @@ ompi_mtl_ofi_add_procs(struct mca_mtl_base_module_t *mtl,
     if (fi_addrs)
         free(fi_addrs);
 
-    if (ep_names)
-        free(ep_names);
-
     return ret;
 }
 
@@ -182,3 +341,92 @@ ompi_mtl_ofi_del_procs(struct mca_mtl_base_module_t *mtl,
 
     return OMPI_SUCCESS;
 }
+
+int ompi_mtl_ofi_add_comm(struct mca_mtl_base_module_t *mtl,
+                      struct ompi_communicator_t *comm)
+{
+    int ret = OMPI_SUCCESS;
+    uint32_t comm_size;
+    mca_mtl_comm_t* mtl_comm;
+
+    mca_mtl_ofi_ep_type ep_type = (0 == ompi_mtl_ofi.enable_sep) ?
+                                  OFI_REGULAR_EP : OFI_SCALABLE_EP;
+
+    if (!OMPI_COMM_IS_GLOBAL_INDEX(comm)) {
+        mtl_comm = OBJ_NEW(mca_mtl_comm_t);
+
+        if (OMPI_COMM_IS_INTER(comm)) {
+            comm_size = ompi_comm_remote_size(comm);
+        } else {
+            comm_size = ompi_comm_size(comm);
+        }
+        mtl_comm->c_index_vec = (c_index_vec_t *)malloc(sizeof(c_index_vec_t) * comm_size);
+        if (NULL == mtl_comm->c_index_vec) {
+            ret = OMPI_ERR_OUT_OF_RESOURCE;
+            OBJ_RELEASE(mtl_comm);
+            goto error;
+        } else {
+            for (uint32_t i=0; i < comm_size; i++) {
+                mtl_comm->c_index_vec[i].c_index_state = MCA_MTL_OFI_CID_NOT_EXCHANGED;
+            }
+        }
+        if (OMPI_COMM_IS_INTRA(comm)) {
+            mtl_comm->c_index_vec[comm->c_my_rank].c_index = comm->c_index;
+            mtl_comm->c_index_vec[comm->c_my_rank].c_index_state = MCA_MTL_OFI_CID_EXCHANGED;
+        }
+
+        comm->c_mtl_comm = mtl_comm;
+
+    } else  {
+
+        comm->c_mtl_comm = NULL;
+
+    }
+
+    /*
+     * If thread grouping enabled, add new OFI context for each communicator
+     * other than MPI_COMM_SELF.
+     */
+    if ((ompi_mtl_ofi.thread_grouping && (MPI_COMM_SELF != comm)) ||
+        /* If no thread grouping, add new OFI context only
+         * for MPI_COMM_WORLD.
+         */
+        (!ompi_mtl_ofi.thread_grouping && (!ompi_mtl_ofi.is_initialized))) {
+
+        ret = ompi_mtl_ofi_init_contexts(mtl, comm, ep_type);
+        ompi_mtl_ofi.is_initialized = true;
+
+        if (OMPI_SUCCESS != ret) {
+            goto error;
+        }
+    }
+
+error:
+    return ret;
+}
+
+int ompi_mtl_ofi_del_comm(struct mca_mtl_base_module_t *mtl,
+                          struct ompi_communicator_t *comm)
+{
+    int ret = OMPI_SUCCESS;
+    mca_mtl_ofi_ep_type ep_type = (0 == ompi_mtl_ofi.enable_sep) ?
+                                  OFI_REGULAR_EP : OFI_SCALABLE_EP;
+
+    if(NULL != comm->c_mtl_comm) {
+        free(comm->c_mtl_comm->c_index_vec);
+        OBJ_RELEASE(comm->c_mtl_comm);
+        comm->c_mtl_comm = NULL;
+    }
+
+    /*
+     * Clean up OFI contexts information.
+     */
+    if ((ompi_mtl_ofi.thread_grouping && (MPI_COMM_SELF != comm)) ||
+        (!ompi_mtl_ofi.thread_grouping && (MPI_COMM_WORLD == comm))) {
+
+        ret = ompi_mtl_ofi_finalize_contexts(mtl, comm, ep_type);
+    }
+
+    return ret;
+}
+
diff --git a/ompi/mca/mtl/ofi/mtl_ofi.h b/ompi/mca/mtl/ofi/mtl_ofi.h
index 6626c754a97..aae756b0518 100644
--- a/ompi/mca/mtl/ofi/mtl_ofi.h
+++ b/ompi/mca/mtl/ofi/mtl_ofi.h
@@ -2,11 +2,15 @@
  * Copyright (c) 2013-2018 Intel, Inc. All rights reserved
  * Copyright (c) 2017      Los Alamos National Security, LLC. All rights
  *                         reserved.
- * Copyright (c) 2019-2020 Triad National Security, LLC. All rights
+ * Copyright (c) 2019-2024 Triad National Security, LLC. All rights
  *                         reserved.
- * Copyright (c) 2018-2020 Amazon.com, Inc. or its affiliates. All rights
+ * Copyright (c) 2018-2023 Amazon.com, Inc. or its affiliates.  All Rights reserved.
  *                         reserved.
- *
+ * Copyright (c) 2021      Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2021      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -20,6 +24,7 @@
 #include "ompi/mca/mtl/mtl.h"
 #include "ompi/mca/mtl/base/base.h"
 #include "opal/datatype/opal_convertor.h"
+#include "opal/util/misc.h"
 #include "opal/util/show_help.h"
 #include "opal/util/printf.h"
 
@@ -40,6 +45,7 @@
 #include "ompi/mca/mtl/base/mtl_base_datatype.h"
 #include "ompi/message/message.h"
 #include "opal/mca/common/ofi/common_ofi.h"
+#include "opal/mca/accelerator/base/base.h"
 
 #include "mtl_ofi_opt.h"
 #include "mtl_ofi_types.h"
@@ -47,11 +53,6 @@
 #include "mtl_ofi_endpoint.h"
 #include "mtl_ofi_compat.h"
 
-#if OPAL_CUDA_SUPPORT
-#include "opal/mca/common/cuda/common_cuda.h"
-#include "opal/datatype/opal_convertor.h"
-#endif
-
 BEGIN_C_DECLS
 
 extern mca_mtl_ofi_module_t ompi_mtl_ofi;
@@ -65,19 +66,49 @@ extern int ompi_mtl_ofi_del_procs(struct mca_mtl_base_module_t *mtl,
                                   size_t nprocs,
                                   struct ompi_proc_t **procs);
 
+extern int ompi_mtl_ofi_add_comm(struct mca_mtl_base_module_t *mtl,
+                                 struct ompi_communicator_t *comm);
+extern int ompi_mtl_ofi_del_comm(struct mca_mtl_base_module_t *mtl,
+                                 struct ompi_communicator_t *comm);
+
 int ompi_mtl_ofi_progress_no_inline(void);
 
 #if OPAL_HAVE_THREAD_LOCAL
-extern opal_thread_local int per_thread_ctx;
-extern opal_thread_local struct fi_cq_tagged_entry wc[MTL_OFI_MAX_PROG_EVENT_COUNT];
+extern opal_thread_local int ompi_mtl_ofi_per_thread_ctx;
 #endif
 
+#define MCA_MTL_OFI_CID_NOT_EXCHANGED 2
+#define MCA_MTL_OFI_CID_EXCHANGING    1 
+#define MCA_MTL_OFI_CID_EXCHANGED     0
+
+typedef struct {
+    uint32_t c_index:30;
+    uint32_t c_index_state:2;
+} c_index_vec_t;
+
+typedef struct mca_mtl_comm_t {
+    opal_object_t super;
+    c_index_vec_t *c_index_vec;
+} mca_mtl_comm_t;
+
+OBJ_CLASS_DECLARATION(mca_mtl_comm_t);
+
+struct mca_mtl_ofi_cid_hdr_t {
+    ompi_comm_extended_cid_t hdr_cid;
+    int16_t                  hdr_src_c_index;
+    int32_t                  hdr_src;
+    bool                     need_response;
+    bool                     ofi_cq_data;
+};
+
+typedef struct mca_mtl_ofi_cid_hdr_t mca_mtl_ofi_cid_hdr_t;
+
 /* Set OFI context for operations which generate completion events */
 __opal_attribute_always_inline__ static inline void
 set_thread_context(int ctxt)
 {
 #if OPAL_HAVE_THREAD_LOCAL
-    per_thread_ctx = ctxt;
+    ompi_mtl_ofi_per_thread_ctx = ctxt;
     return;
 #endif
 }
@@ -87,7 +118,7 @@ __opal_attribute_always_inline__ static inline void
 get_thread_context(int *ctxt)
 {
 #if OPAL_HAVE_THREAD_LOCAL
-    *ctxt = per_thread_ctx;
+    *ctxt = ompi_mtl_ofi_per_thread_ctx;
 #endif
     return;
 }
@@ -101,41 +132,40 @@ opal_mutex_atomic_unlock(&ompi_mtl_ofi.ofi_ctxt[ctxt_id].context_lock)
 __opal_attribute_always_inline__ static inline int
 ompi_mtl_ofi_context_progress(int ctxt_id)
 {
-    int count = 0, i, events_read;
+    int count = 0, i, events_read, req_type = -1;
     ompi_mtl_ofi_request_t *ofi_req = NULL;
     struct fi_cq_err_entry error = { 0 };
     ssize_t ret;
-#if !OPAL_HAVE_THREAD_LOCAL
-    struct fi_cq_tagged_entry wc[MTL_OFI_MAX_PROG_EVENT_COUNT];
-#endif
+    struct fi_cq_tagged_entry ompi_mtl_ofi_wc[MTL_OFI_MAX_PROG_EVENT_COUNT];
 
     /**
      * Read the work completions from the CQ.
      * From the completion's op_context, we get the associated OFI request.
      * Call the request's callback.
      */
-    ret = fi_cq_read(ompi_mtl_ofi.ofi_ctxt[ctxt_id].cq, (void *)&wc,
+    ret = fi_cq_read(ompi_mtl_ofi.ofi_ctxt[ctxt_id].cq, (void *)&ompi_mtl_ofi_wc,
                      ompi_mtl_ofi.ofi_progress_event_count);
     if (ret > 0) {
         count+= ret;
         events_read = ret;
         for (i = 0; i < events_read; i++) {
-            if (NULL != wc[i].op_context) {
-                ofi_req = TO_OFI_REQ(wc[i].op_context);
+            if (NULL != ompi_mtl_ofi_wc[i].op_context) {
+                ofi_req = TO_OFI_REQ(ompi_mtl_ofi_wc[i].op_context);
                 assert(ofi_req);
-                ret = ofi_req->event_callback(&wc[i], ofi_req);
+                req_type = ofi_req->type;
+                ret = ofi_req->event_callback(&ompi_mtl_ofi_wc[i], ofi_req);
                 if (OMPI_SUCCESS != ret) {
-                    opal_output(0, "%s:%d: Error returned by request event callback: %zd.\n"
-                                   "*** The Open MPI OFI MTL is aborting the MPI job (via exit(3)).\n",
-                                   __FILE__, __LINE__, ret);
-                    fflush(stderr);
-                    exit(1);
+                    opal_output(0,
+                                "%s:%d: Error returned by request (type: %d) event callback: %zd.\n"
+                                "*** The Open MPI OFI MTL is aborting the MPI job (via exit(3)).\n",
+                                __FILE__, __LINE__, req_type, ret);
+                    goto bail;
                 }
             }
         }
     } else if (OPAL_UNLIKELY(ret == -FI_EAVAIL)) {
         /**
-         * An error occured and is being reported via the CQ.
+         * An error occurred and is being reported via the CQ.
          * Read the error and forward it to the upper layer.
          */
         ret = fi_cq_readerr(ompi_mtl_ofi.ofi_ctxt[ctxt_id].cq,
@@ -151,35 +181,46 @@ ompi_mtl_ofi_context_progress(int ctxt_id)
              * thread fetches the entry while others get -FI_EAGAIN
              * indicating an empty queue, which is not erroneous.
              */
-            if (ret == -FI_EAGAIN)
+            if (ret == -FI_EAGAIN) {
                 return count;
+            }
             opal_output(0, "%s:%d: Error returned from fi_cq_readerr: %s(%zd).\n"
                            "*** The Open MPI OFI MTL is aborting the MPI job (via exit(3)).\n",
                            __FILE__, __LINE__, fi_strerror(-ret), ret);
-            fflush(stderr);
-            exit(1);
+            goto bail;
+        }
+
+        if (!error.op_context) {
+            opal_output(0, "%s:%d: Error returned from fi_cq_readerr with null context. "
+                           "Completion flags: %016lx\n"
+                           "*** The Open MPI OFI MTL is aborting the MPI job (via exit(3)).\n",
+                           __FILE__, __LINE__, error.flags);
+            goto bail;
         }
 
-        assert(error.op_context);
         ofi_req = TO_OFI_REQ(error.op_context);
         assert(ofi_req);
+        req_type = ofi_req->type;
         ret = ofi_req->error_callback(&error, ofi_req);
         if (OMPI_SUCCESS != ret) {
-                opal_output(0, "%s:%d: Error returned by request error callback: %zd.\n"
-                               "*** The Open MPI OFI MTL is aborting the MPI job (via exit(3)).\n",
-                               __FILE__, __LINE__, ret);
-            fflush(stderr);
-            exit(1);
+            opal_output(0,
+                        "%s:%d: Error returned by request (type: %d) error callback: %zd.\n"
+                        "*** The Open MPI OFI MTL is aborting the MPI job (via exit(3)).\n",
+                        __FILE__, __LINE__, req_type, ret);
+            goto bail;
         }
     } else if (ret != -FI_EAGAIN && ret != -EINTR) {
         opal_output(0, "%s:%d: Error returned from fi_cq_read: %s(%zd).\n"
                        "*** The Open MPI OFI MTL is aborting the MPI job (via exit(3)).\n",
                        __FILE__, __LINE__, fi_strerror(-ret), ret);
-        fflush(stderr);
-        exit(1);
+        goto bail;
     }
 
     return count;
+
+bail:
+    fflush(stderr);
+    exit(1);
 }
 
 __opal_attribute_always_inline__ static inline int
@@ -226,25 +267,6 @@ ompi_mtl_ofi_progress(void)
     return count;
 }
 
-/**
- * When attempting to execute an OFI operation we need to handle
- * resource overrun cases. When a call to an OFI OP fails with -FI_EAGAIN
- * the OFI mtl will attempt to progress any pending Completion Queue
- * events that may prevent additional operations to be enqueued.
- * If the call to ofi progress is successful, then the function call
- * will be retried.
- */
-#define MTL_OFI_RETRY_UNTIL_DONE(FUNC, RETURN)         \
-    do {                                               \
-        do {                                           \
-            RETURN = FUNC;                             \
-            if (OPAL_LIKELY(0 == RETURN)) {break;}     \
-            if (OPAL_LIKELY(RETURN == -FI_EAGAIN)) {   \
-                ompi_mtl_ofi_progress();               \
-            }                                          \
-        } while (OPAL_LIKELY(-FI_EAGAIN == RETURN));   \
-    } while (0);
-
 #define MTL_OFI_LOG_FI_ERR(err, string)                                     \
     do {                                                                    \
         opal_output_verbose(1, opal_common_ofi.output,                      \
@@ -264,7 +286,7 @@ ompi_mtl_ofi_set_mr_null(ompi_mtl_ofi_request_t *ofi_req) {
 
 /**
  * Registers user buffer with Libfabric domain if
- * buffer is cuda and provider has fi_mr_hmem
+ * buffer is a device buffer and provider has fi_mr_hmem
  */
 static
 int ompi_mtl_ofi_register_buffer(struct opal_convertor_t *convertor,
@@ -275,8 +297,9 @@ int ompi_mtl_ofi_register_buffer(struct opal_convertor_t *convertor,
         return OMPI_SUCCESS;
     }
 
-#if OPAL_CUDA_SUPPORT
-    if (convertor->flags & CONVERTOR_CUDA) {
+#if OPAL_OFI_HAVE_FI_MR_IFACE
+
+    if ((convertor->flags & CONVERTOR_ACCELERATOR) && ompi_mtl_ofi.hmem_needs_reg) {
         /* Register buffer */
         int ret;
         struct fi_mr_attr attr = {0};
@@ -289,22 +312,39 @@ int ompi_mtl_ofi_register_buffer(struct opal_convertor_t *convertor,
         attr.access = FI_SEND | FI_RECV;
         attr.offset = 0;
         attr.context = NULL;
-
-        attr.iface = FI_HMEM_CUDA;
-        mca_common_cuda_get_device(&attr.device.cuda);
-
+        if (false == ompi_mtl_base_selected_component->accelerator_support) {
+            goto reg;
+        } else if (0 == strcmp(opal_accelerator_base_selected_component.base_version.mca_component_name, "cuda")) {
+            attr.iface = FI_HMEM_CUDA;
+            opal_accelerator.get_device(&attr.device.cuda);
+#if OPAL_OFI_HAVE_FI_HMEM_ROCR
+        } else if (0 == strcmp(opal_accelerator_base_selected_component.base_version.mca_component_name, "rocm")) {
+            attr.iface = FI_HMEM_ROCR;
+            opal_accelerator.get_device(&attr.device.cuda);
+#endif
+#if OPAL_OFI_HAVE_FI_HMEM_ZE
+        } else if (0 == strcmp(opal_accelerator_base_selected_component.base_version.mca_component_name, "ze")) {
+            attr.iface = FI_HMEM_ZE;
+            opal_accelerator.get_device(&attr.device.ze);
+#endif
+        } else {
+            return OPAL_ERROR;
+        }
+reg:
         ret = fi_mr_regattr(ompi_mtl_ofi.domain, &attr, 0, &ofi_req->mr);
 
         if (ret) {
             opal_show_help("help-mtl-ofi.txt", "Buffer Memory Registration Failed", true,
-                           "CUDA",
+                           opal_accelerator_base_selected_component.base_version.mca_component_name,
                            buffer, ofi_req->length,
                            fi_strerror(-ret), ret);
             ofi_req->mr = NULL;
             return OMPI_ERROR;
         }
     }
-#endif /* OPAL_CUDA_SUPPORT */
+
+#endif
+
     return OMPI_SUCCESS;
 }
 
@@ -335,11 +375,18 @@ int ompi_mtl_ofi_deregister_and_free_buffer(ompi_mtl_ofi_request_t *ofi_req) {
         return ret;
     }
     if (OPAL_UNLIKELY(NULL != ofi_req->buffer)) {
-#if OPAL_CUDA_SUPPORT
-        opal_cuda_free(ofi_req->buffer, ofi_req->convertor);
-#else
-        free(ofi_req->buffer);
-#endif
+        /* If no convertor has been provided, assume it's a host buffer */
+        if (NULL == ofi_req->convertor) {
+            free(ofi_req->buffer);
+        /* If the convertor is not an accelerator buffer, or we do not have
+           accelerator_support, it must be a host buffer. */
+        } else if (!(ofi_req->convertor->flags & CONVERTOR_ACCELERATOR) ||
+            false == ompi_mtl_base_selected_component->accelerator_support) {
+            free(ofi_req->buffer);
+        /* The buffer must be an accelerator buffer */
+        } else {
+            ret = opal_accelerator.mem_release(MCA_ACCELERATOR_NO_DEVICE_ID, ofi_req->buffer);
+        }
     }
     ofi_req->buffer = NULL;
     return ret;
@@ -373,6 +420,26 @@ ompi_mtl_ofi_send_callback(struct fi_cq_tagged_entry *wc,
     return OMPI_SUCCESS;
 }
 
+
+/*
+ * special send callback for excid send operation.
+ * Since the send excid operation cannot block
+ * waiting for completion of the send operation,
+ * we have to free the internal message buffer allocated
+ * as part of the excid operation here as well as the
+ * request itself.
+ */
+__opal_attribute_always_inline__ static inline int
+ompi_mtl_ofi_send_excid_callback(struct fi_cq_tagged_entry *wc,
+                           ompi_mtl_ofi_request_t *ofi_req)
+{
+    assert(ofi_req->completion_count > 0);
+    free(ofi_req->buffer);
+    ofi_req->completion_count--; /* no one's waiting on this */
+    free(ofi_req);
+    return OMPI_SUCCESS;
+}
+
 __opal_attribute_always_inline__ static inline int
 ompi_mtl_ofi_send_error_callback(struct fi_cq_err_entry *error,
                                  ompi_mtl_ofi_request_t *ofi_req)
@@ -446,6 +513,135 @@ ompi_mtl_ofi_map_comm_to_ctxt(uint32_t comm_id)
     return ompi_mtl_ofi.comm_to_context[comm_id];
 }
 
+__opal_attribute_always_inline__ static inline int
+ompi_mtl_ofi_post_recv_excid_buffer(bool blocking, struct ompi_communicator_t *comm, int src);
+
+__opal_attribute_always_inline__ static inline int
+ompi_mtl_ofi_send_excid(struct mca_mtl_base_module_t *mtl,
+                  struct ompi_communicator_t *comm,
+                  int dest,
+                  bool ofi_cq_data,
+                  bool is_send);
+
+__opal_attribute_always_inline__ static inline int
+ompi_mtl_ofi_recv_excid_error_callback(struct fi_cq_err_entry *error,
+                                 ompi_mtl_ofi_request_t *ofi_req)
+{
+    ompi_status_public_t *status;
+    assert(ofi_req->super.ompi_req);
+    status = &ofi_req->super.ompi_req->req_status;
+    status->MPI_TAG = MTL_OFI_GET_TAG(ofi_req->match_bits);
+    status->MPI_SOURCE = mtl_ofi_get_source((struct fi_cq_tagged_entry *) error);
+
+    switch (error->err) {
+        case FI_ETRUNC:
+            status->MPI_ERROR = MPI_ERR_TRUNCATE;
+            break;
+        case FI_ECANCELED:
+            status->_cancelled = true;
+            break;
+        default:
+            status->MPI_ERROR = MPI_ERR_INTERN;
+    }
+
+    ofi_req->super.completion_callback(&ofi_req->super);
+    return OMPI_SUCCESS;
+}
+
+__opal_attribute_always_inline__ static inline int
+ompi_mtl_ofi_post_recv_excid_buffer_callback(struct fi_cq_tagged_entry *wc,
+                                             ompi_mtl_ofi_request_t *ofi_req)
+{
+    ofi_req->completion_count--;
+    int ret;
+    mca_mtl_ofi_cid_hdr_t *buffer = (mca_mtl_ofi_cid_hdr_t *)ofi_req->buffer;
+    ompi_comm_extended_cid_t excid;
+    ompi_communicator_t *comm;
+    int src = buffer->hdr_src;
+    mca_mtl_comm_t *mtl_comm;
+
+    excid.cid_base = buffer->hdr_cid.cid_base;
+    excid.cid_sub.u64 = buffer->hdr_cid.cid_sub.u64;
+    for (int i = 0; i < 8; i++) {
+        excid.cid_sub.u8[i] = buffer->hdr_cid.cid_sub.u8[i];
+    }
+
+    comm = ompi_comm_lookup_cid(excid);
+    if (comm == NULL) {
+        comm = ompi_comm_lookup(buffer->hdr_src_c_index);
+    }
+
+    if (comm == NULL) {
+        return OMPI_SUCCESS;
+    }
+
+    mtl_comm = comm->c_mtl_comm;
+
+    if (mtl_comm->c_index_vec[src].c_index_state == MCA_MTL_OFI_CID_NOT_EXCHANGED
+        && buffer->need_response) {
+        mtl_comm->c_index_vec[src].c_index = buffer->hdr_src_c_index;
+        mtl_comm->c_index_vec[src].c_index_state = MCA_MTL_OFI_CID_EXCHANGED;
+        ret = ompi_mtl_ofi_send_excid(ofi_req->mtl, comm, src, buffer->ofi_cq_data, false);
+    } else {
+        mtl_comm->c_index_vec[src].c_index_state = MCA_MTL_OFI_CID_EXCHANGED;
+        mtl_comm->c_index_vec[src].c_index = buffer->hdr_src_c_index;
+    }
+
+    ret = ompi_mtl_ofi_post_recv_excid_buffer(false, comm, -1);
+    return ret;
+}
+
+__opal_attribute_always_inline__ static inline int
+ompi_mtl_ofi_post_recv_excid_buffer(bool blocking, struct ompi_communicator_t *comm, int src)
+{
+    int ctxt_id = 0;
+    ssize_t ret;
+    ompi_mtl_ofi_request_t *ofi_req = malloc(sizeof(ompi_mtl_ofi_request_t));
+    mca_mtl_ofi_cid_hdr_t *start = malloc(sizeof(mca_mtl_ofi_cid_hdr_t));
+    size_t length = sizeof(mca_mtl_ofi_cid_hdr_t);
+    mca_mtl_comm_t *mtl_comm;
+    
+    mtl_comm = comm->c_mtl_comm;
+
+    set_thread_context(ctxt_id);
+
+    ofi_req->type = OMPI_MTL_OFI_RECV;
+    ofi_req->event_callback = ompi_mtl_ofi_post_recv_excid_buffer_callback;
+    ofi_req->error_callback = ompi_mtl_ofi_recv_excid_error_callback;
+    ofi_req->buffer = start;
+    ofi_req->length = length;
+    ofi_req->convertor = NULL;
+    ofi_req->req_started = false;
+    ofi_req->status.MPI_ERROR = OMPI_SUCCESS;
+    ofi_req->remote_addr = 0UL;
+    ofi_req->match_bits = 0UL;
+    ofi_req->completion_count = 1;
+    ofi_req->comm = comm;
+
+    OFI_RETRY_UNTIL_DONE(fi_recv(ompi_mtl_ofi.ofi_ctxt[0].rx_ep,
+                                 start,
+                                 length,
+                                 NULL,
+                                 FI_ADDR_UNSPEC,
+                                 (void *)&ofi_req->ctx), ret);
+    if (OPAL_UNLIKELY(0 > ret)) {
+        if (NULL != ofi_req->buffer) {
+            free(ofi_req->buffer);
+        }
+        MTL_OFI_LOG_FI_ERR(ret, "fi_recv failed");
+        return ompi_mtl_ofi_get_error(ret);
+    }
+
+    if (blocking) {
+        assert(src != -1);
+        while (mtl_comm->c_index_vec[src].c_index_state > MCA_MTL_OFI_CID_EXCHANGED) {
+            ompi_mtl_ofi_progress();
+        }
+    }
+
+    return OMPI_SUCCESS;
+}
+
 __opal_attribute_always_inline__ static inline int
 ompi_mtl_ofi_ssend_recv(ompi_mtl_ofi_request_t *ack_req,
                   struct ompi_communicator_t *comm,
@@ -458,26 +654,31 @@ ompi_mtl_ofi_ssend_recv(ompi_mtl_ofi_request_t *ack_req,
     ssize_t ret = OMPI_SUCCESS;
     int ctxt_id = 0;
 
-    ctxt_id = ompi_mtl_ofi_map_comm_to_ctxt(comm->c_contextid);
+    if (ompi_mtl_ofi.total_ctxts_used > 0) {
+        ctxt_id = comm->c_contextid.cid_sub.u64 % ompi_mtl_ofi.total_ctxts_used;
+    } else {
+        ctxt_id = 0;
+    }
     set_thread_context(ctxt_id);
 
     ack_req = malloc(sizeof(ompi_mtl_ofi_request_t));
     assert(ack_req);
 
     ack_req->parent = ofi_req;
+    ack_req->type = OMPI_MTL_OFI_ACK;
     ack_req->event_callback = ompi_mtl_ofi_send_ack_callback;
     ack_req->error_callback = ompi_mtl_ofi_send_ack_error_callback;
 
     ofi_req->completion_count += 1;
 
-    MTL_OFI_RETRY_UNTIL_DONE(fi_trecv(ompi_mtl_ofi.ofi_ctxt[ctxt_id].rx_ep,
-                                      NULL,
-                                      0,
-                                      NULL,
-                                      *src_addr,
-                                      *match_bits | ompi_mtl_ofi.sync_send_ack,
-                                      0, /* Exact match, no ignore bits */
-                                      (void *) &ack_req->ctx), ret);
+    OFI_RETRY_UNTIL_DONE(fi_trecv(ompi_mtl_ofi.ofi_ctxt[ctxt_id].rx_ep,
+                                  NULL,
+                                  0,
+                                  NULL,
+                                  *src_addr,
+                                  *match_bits | ompi_mtl_ofi.sync_send_ack,
+                                  0, /* Exact match, no ignore bits */
+                                  (void *) &ack_req->ctx), ret);
     if (OPAL_UNLIKELY(0 > ret)) {
         opal_output_verbose(1, opal_common_ofi.output,
                             "%s:%d: fi_trecv failed: %s(%zd)",
@@ -491,18 +692,151 @@ ompi_mtl_ofi_ssend_recv(ompi_mtl_ofi_request_t *ack_req,
     return OMPI_SUCCESS;
 }
 
-__opal_attribute_always_inline__ static inline int
-ompi_mtl_ofi_send_generic(struct mca_mtl_base_module_t *mtl,
+/*
+ * this routine is invoked in the case of communicators which are not using a
+ * global cid, i.e. those created using MPI_Comm_create_from_group/
+ * MPI_Intercomm_create_from_groups in order to exchange the local cid used
+ * by the sender for this supplied communicator.  This function is only invoked
+ * for the first message sent to a given receiver.
+ */
+static int
+ompi_mtl_ofi_send_excid(struct mca_mtl_base_module_t *mtl,
                   struct ompi_communicator_t *comm,
                   int dest,
-                  int tag,
-                  struct opal_convertor_t *convertor,
-                  mca_pml_base_send_mode_t mode,
-                  bool ofi_cq_data)
+                  bool ofi_cq_data,
+                  bool is_send)
+{
+    ssize_t ret = OMPI_SUCCESS;
+    ompi_mtl_ofi_request_t *ofi_req = NULL;
+    int ctxt_id = 0;
+    mca_mtl_ofi_cid_hdr_t *start = NULL;
+    ompi_proc_t *ompi_proc = NULL;
+    mca_mtl_ofi_endpoint_t *endpoint = NULL;
+    fi_addr_t sep_peer_fiaddr = 0;
+    mca_mtl_comm_t *mtl_comm;
+    
+    ofi_req = (ompi_mtl_ofi_request_t *)malloc(sizeof(ompi_mtl_ofi_request_t));
+    if (NULL == ofi_req) {
+        ret =  OMPI_ERR_OUT_OF_RESOURCE;
+        goto fn_exit;
+    }
+
+    start = (mca_mtl_ofi_cid_hdr_t *)malloc(sizeof(mca_mtl_ofi_cid_hdr_t));
+    if (NULL == start) {
+        ret = OMPI_ERR_OUT_OF_RESOURCE;
+        goto fn_exit;
+    }
+
+    mtl_comm = comm->c_mtl_comm;
+
+    ctxt_id = 0;
+    set_thread_context(ctxt_id);
+
+    /**
+     * Create a send request, start it and wait until it completes.
+     */
+    ofi_req->type = OMPI_MTL_OFI_SEND;
+    ofi_req->event_callback = ompi_mtl_ofi_send_excid_callback;
+    ofi_req->error_callback = ompi_mtl_ofi_send_error_callback;
+    ofi_req->buffer = start;
+
+    ompi_proc = ompi_comm_peer_lookup(comm, dest);
+    endpoint = ompi_mtl_ofi_get_endpoint(mtl, ompi_proc);
+
+    /* For Scalable Endpoints, gather target receive context */
+    sep_peer_fiaddr = fi_rx_addr(endpoint->peer_fiaddr, ctxt_id, ompi_mtl_ofi.rx_ctx_bits);
+
+    start->hdr_cid = comm->c_contextid;
+    start->hdr_src = comm->c_my_rank;
+    start->hdr_src_c_index = comm->c_index;
+    start->ofi_cq_data = ofi_cq_data;
+    if (mtl_comm->c_index_vec[dest].c_index_state > MCA_MTL_OFI_CID_EXCHANGED) {
+        start->need_response = true;
+    } else {
+        start->need_response = false;
+    }
+    size_t length = sizeof(mca_mtl_ofi_cid_hdr_t);
+
+    ofi_req->length = length;
+    ofi_req->status.MPI_ERROR = OMPI_SUCCESS;
+    ofi_req->completion_count = 0;
+    if (OPAL_UNLIKELY(length > endpoint->mtl_ofi_module->max_msg_size)) {
+        opal_show_help("help-mtl-ofi.txt",
+            "message too big", false,
+            length, endpoint->mtl_ofi_module->max_msg_size);
+        ret = OMPI_ERROR;
+        goto fn_exit;
+    }
+
+    if (ompi_mtl_ofi.max_inject_size >= length) {
+        if (ofi_cq_data) {
+            OFI_RETRY_UNTIL_DONE(fi_injectdata(ompi_mtl_ofi.ofi_ctxt[0].tx_ep,
+                                               start,
+                                               length,
+                                               comm->c_my_rank,
+                                               sep_peer_fiaddr), ret);
+        } else {
+            OFI_RETRY_UNTIL_DONE(fi_inject(ompi_mtl_ofi.ofi_ctxt[0].tx_ep,
+                                           start,
+                                           length,
+                                           sep_peer_fiaddr), ret);
+        }
+        if (OPAL_UNLIKELY(0 > ret)) {
+            MTL_OFI_LOG_FI_ERR(ret,
+                               ofi_cq_data ? "fi_injectdata failed"
+                               : "fi_inject failed");
+
+        }
+    } else {
+        ofi_req->completion_count = 1;
+        if (ofi_cq_data) {
+            OFI_RETRY_UNTIL_DONE(fi_senddata(ompi_mtl_ofi.ofi_ctxt[0].tx_ep,
+                                             start,
+                                             length,
+                                             NULL,
+                                             comm->c_my_rank,
+                                             sep_peer_fiaddr,
+                                             (void *) &ofi_req->ctx), ret);
+        } else {
+            OFI_RETRY_UNTIL_DONE(fi_send(ompi_mtl_ofi.ofi_ctxt[0].tx_ep,
+                                         start,
+                                         length,
+                                         NULL,
+                                         sep_peer_fiaddr,
+                                         (void *) &ofi_req->ctx), ret);
+        }
+        if (OPAL_UNLIKELY(0 > ret)) {
+            MTL_OFI_LOG_FI_ERR(ret,
+                               ofi_cq_data ? "fi_tsenddata failed"
+                               : "fi_tsend failed");
+        }
+    }
+
+    ret = ompi_mtl_ofi_get_error(ret);
+    ofi_req->status.MPI_ERROR = ret;
+
+fn_exit:
+
+    if ((OMPI_SUCCESS != ret) || (ofi_req->completion_count == 0)) {
+        if (NULL != ofi_req) free(ofi_req);
+        if (NULL != start) free(start);
+    }
+
+    return ret;
+}
+
+__opal_attribute_always_inline__ static inline int
+ompi_mtl_ofi_send_generic(struct mca_mtl_base_module_t *mtl,
+                          struct ompi_communicator_t *comm,
+                          int dest,
+                          int tag,
+                          struct opal_convertor_t *convertor,
+                          mca_pml_base_send_mode_t mode,
+                          bool ofi_cq_data)
 {
     ssize_t ret = OMPI_SUCCESS;
     ompi_mtl_ofi_request_t ofi_req;
-    int ompi_ret, ctxt_id = 0;
+    int ompi_ret, ctxt_id = 0, c_index_for_tag;
     void *start;
     bool free_after;
     size_t length;
@@ -512,15 +846,44 @@ ompi_mtl_ofi_send_generic(struct mca_mtl_base_module_t *mtl,
     ompi_mtl_ofi_request_t *ack_req = NULL; /* For synchronous send */
     fi_addr_t src_addr = 0;
     fi_addr_t sep_peer_fiaddr = 0;
+    mca_mtl_comm_t *mtl_comm;
+
+    if (OPAL_LIKELY(OMPI_COMM_IS_GLOBAL_INDEX(comm))) {
+        c_index_for_tag = comm->c_index;
+    } else {
+        mtl_comm = comm->c_mtl_comm;
+        if (mtl_comm->c_index_vec[dest].c_index_state == MCA_MTL_OFI_CID_NOT_EXCHANGED) {
+            mtl_comm->c_index_vec[dest].c_index_state = MCA_MTL_OFI_CID_EXCHANGING;
+            ompi_ret = ompi_mtl_ofi_send_excid(mtl, comm, dest, ofi_cq_data, true);
+            if (OPAL_UNLIKELY(OMPI_SUCCESS != ompi_ret)) {
+                return ompi_ret;
+            }
+        }
+
+        if (mtl_comm->c_index_vec[dest].c_index_state > MCA_MTL_OFI_CID_EXCHANGED) {
+             while (mtl_comm->c_index_vec[dest].c_index_state > MCA_MTL_OFI_CID_EXCHANGED) {
+                ompi_ret = ompi_mtl_ofi_post_recv_excid_buffer(true, comm, dest);
+                if (OPAL_UNLIKELY(OMPI_SUCCESS != ompi_ret)) {
+                    return ompi_ret;
+                }
+            }
+        }
+        c_index_for_tag = mtl_comm->c_index_vec[dest].c_index;
+    }
 
     ompi_mtl_ofi_set_mr_null(&ofi_req);
+    if (ompi_mtl_ofi.total_ctxts_used > 0) {
+        ctxt_id = comm->c_contextid.cid_sub.u64 % ompi_mtl_ofi.total_ctxts_used;
+    } else {
+        ctxt_id = 0;
+    }
 
-    ctxt_id = ompi_mtl_ofi_map_comm_to_ctxt(comm->c_contextid);
     set_thread_context(ctxt_id);
 
     /**
      * Create a send request, start it and wait until it completes.
      */
+    ofi_req.type = OMPI_MTL_OFI_SEND;
     ofi_req.event_callback = ompi_mtl_ofi_send_callback;
     ofi_req.error_callback = ompi_mtl_ofi_send_error_callback;
 
@@ -539,6 +902,7 @@ ompi_mtl_ofi_send_generic(struct mca_mtl_base_module_t *mtl,
     ofi_req.length = length;
     ofi_req.status.MPI_ERROR = OMPI_SUCCESS;
     ofi_req.completion_count = 0;
+    ofi_req.convertor = convertor;
 
     if (OPAL_UNLIKELY(length > endpoint->mtl_ofi_module->max_msg_size)) {
         opal_show_help("help-mtl-ofi.txt",
@@ -548,10 +912,10 @@ ompi_mtl_ofi_send_generic(struct mca_mtl_base_module_t *mtl,
     }
 
     if (ofi_cq_data) {
-        match_bits = mtl_ofi_create_send_tag_CQD(comm->c_contextid, tag);
+        match_bits = mtl_ofi_create_send_tag_CQD(c_index_for_tag, tag);
         src_addr = sep_peer_fiaddr;
     } else {
-        match_bits = mtl_ofi_create_send_tag(comm->c_contextid,
+        match_bits = mtl_ofi_create_send_tag(c_index_for_tag,
                                              comm->c_my_rank, tag);
         /* src_addr is ignored when FI_DIRECTED_RECV is not supported */
     }
@@ -567,21 +931,17 @@ ompi_mtl_ofi_send_generic(struct mca_mtl_base_module_t *mtl,
     /** Inject does not currently support device memory
      *  https://github.com/ofiwg/libfabric/issues/5861
      */
-#if OPAL_CUDA_SUPPORT
-    if (!(convertor->flags & CONVERTOR_CUDA)
+    if (!(convertor->flags & CONVERTOR_ACCELERATOR)
         && (ompi_mtl_ofi.max_inject_size >= length)) {
-#else /* !(OPAL_CUDA_SUPPORT)*/
-    if (ompi_mtl_ofi.max_inject_size >= length) {
-#endif /* OPAL_CUDA_SUPPORT */
         if (ofi_cq_data) {
-            MTL_OFI_RETRY_UNTIL_DONE(fi_tinjectdata(ompi_mtl_ofi.ofi_ctxt[ctxt_id].tx_ep,
-                                            start,
-                                            length,
-                                            comm->c_my_rank,
-                                            sep_peer_fiaddr,
-                                            match_bits), ret);
+            OFI_RETRY_UNTIL_DONE(fi_tinjectdata(ompi_mtl_ofi.ofi_ctxt[ctxt_id].tx_ep,
+                                                start,
+                                                length,
+                                                comm->c_my_rank,
+                                                sep_peer_fiaddr,
+                                                match_bits), ret);
         } else {
-            MTL_OFI_RETRY_UNTIL_DONE(fi_tinject(ompi_mtl_ofi.ofi_ctxt[ctxt_id].tx_ep,
+            OFI_RETRY_UNTIL_DONE(fi_tinject(ompi_mtl_ofi.ofi_ctxt[ctxt_id].tx_ep,
                                             start,
                                             length,
                                             sep_peer_fiaddr,
@@ -606,16 +966,16 @@ ompi_mtl_ofi_send_generic(struct mca_mtl_base_module_t *mtl,
         }
         ofi_req.completion_count += 1;
         if (ofi_cq_data) {
-            MTL_OFI_RETRY_UNTIL_DONE(fi_tsenddata(ompi_mtl_ofi.ofi_ctxt[ctxt_id].tx_ep,
-                                          start,
-                                          length,
-                                          (NULL == ofi_req.mr) ? NULL : ofi_req.mr->mem_desc,
-                                          comm->c_my_rank,
-                                          sep_peer_fiaddr,
-                                          match_bits,
-                                          (void *) &ofi_req.ctx), ret);
+            OFI_RETRY_UNTIL_DONE(fi_tsenddata(ompi_mtl_ofi.ofi_ctxt[ctxt_id].tx_ep,
+                                              start,
+                                              length,
+                                              (NULL == ofi_req.mr) ? NULL : ofi_req.mr->mem_desc,
+                                              comm->c_my_rank,
+                                              sep_peer_fiaddr,
+                                              match_bits,
+                                              (void *) &ofi_req.ctx), ret);
         } else {
-            MTL_OFI_RETRY_UNTIL_DONE(fi_tsend(ompi_mtl_ofi.ofi_ctxt[ctxt_id].tx_ep,
+            OFI_RETRY_UNTIL_DONE(fi_tsend(ompi_mtl_ofi.ofi_ctxt[ctxt_id].tx_ep,
                                           start,
                                           length,
                                           (NULL == ofi_req.mr) ? NULL : ofi_req.mr->mem_desc,
@@ -646,6 +1006,83 @@ ompi_mtl_ofi_send_generic(struct mca_mtl_base_module_t *mtl,
     return ofi_req.status.MPI_ERROR;
 }
 
+/*
+ * This routine is invoked in the case where a Recv finds the
+ * MTL_OFI_IS_SYNC_SEND flag was set, indicating the sender issued an SSend and
+ * is blocking while it waits on an ACK message.
+ *
+ * Issue a fire-and-forget send back to the src with a matching tag so that
+ * the sender may continue progress.
+ * Requires ofi_req->remote_addr and ofi_req->comm to be set.
+ */
+static int
+ompi_mtl_ofi_gen_ssend_ack(struct fi_cq_tagged_entry *wc,
+                           ompi_mtl_ofi_request_t *ofi_req)
+{
+    /**
+     * If this recv is part of an MPI_Ssend operation, then we send an
+     * acknowledgment back to the sender.
+     * The ack message is sent without generating a completion event in
+     * the completion queue by not setting FI_COMPLETION in the flags to
+     * fi_tsendmsg(FI_SELECTIVE_COMPLETION).
+     * This is done since the 0 byte message requires no
+     * notification on the send side for a successful completion.
+     * If a failure occurs the provider will notify the error
+     * in the cq_readerr during OFI progress. Once the message has been
+     * successfully processed the request is marked as completed.
+     *
+     * Turns out that there is a bug in the argument checking
+     * in the CXI provider (at least the vendor 1.15.2.0 and earlier versions)
+     * fi_tsendmsg method.  So we have to feed a dummy iovec argument
+     * into fi_tsendmsg with a NULL buffer and zero iov_len, hence
+     * the d_iovect, etc.
+     */
+    int ctxt_id = 0;
+    ssize_t ret;
+    ompi_proc_t *ompi_proc = NULL;
+    mca_mtl_ofi_endpoint_t *endpoint = NULL;
+    int src = mtl_ofi_get_source(wc);
+    struct fi_msg_tagged tagged_msg;
+    struct iovec d_iovec = {.iov_base = NULL, .iov_len = 0};
+
+    if (ompi_mtl_ofi.total_ctxts_used > 0) {
+        ctxt_id = ofi_req->comm->c_contextid.cid_sub.u64 % ompi_mtl_ofi.total_ctxts_used;
+    } else {
+        ctxt_id = 0;
+    }
+
+    ret = MPI_SUCCESS;
+    
+    /**
+     * If the recv request was posted for any source,
+     * we need to extract the source's actual address.
+     */
+    ompi_proc = ompi_comm_peer_lookup(ofi_req->comm, src);
+    endpoint = ompi_mtl_ofi_get_endpoint(ofi_req->mtl, ompi_proc);
+    ofi_req->remote_addr = fi_rx_addr(endpoint->peer_fiaddr, ctxt_id, ompi_mtl_ofi.rx_ctx_bits);
+
+    tagged_msg.msg_iov = &d_iovec;
+    tagged_msg.desc = NULL;
+    tagged_msg.iov_count = 1;
+    tagged_msg.addr = ofi_req->remote_addr;
+    /**
+    * We must continue to use the user's original tag but remove the
+    * sync_send protocol tag bit and instead apply the sync_send_ack
+    * tag bit to complete the initiator's sync send receive.
+    */
+    tagged_msg.tag = (wc->tag | ompi_mtl_ofi.sync_send_ack) & ~ompi_mtl_ofi.sync_send;
+    tagged_msg.context = NULL;
+    tagged_msg.data = 0;
+
+    OFI_RETRY_UNTIL_DONE(fi_tsendmsg(ompi_mtl_ofi.ofi_ctxt[ctxt_id].tx_ep,
+                                     &tagged_msg, 0), ret);
+    if (OPAL_UNLIKELY(0 > ret)) {
+        MTL_OFI_LOG_FI_ERR(ret, "fi_tsendmsg failed during ompi_mtl_ofi_gen_ssend_ack");
+        ret = OMPI_ERROR;
+    }
+    return ret;
+}
+
 __opal_attribute_always_inline__ static inline int
 ompi_mtl_ofi_isend_generic(struct mca_mtl_base_module_t *mtl,
                    struct ompi_communicator_t *comm,
@@ -659,7 +1096,7 @@ ompi_mtl_ofi_isend_generic(struct mca_mtl_base_module_t *mtl,
 {
     ssize_t ret = OMPI_SUCCESS;
     ompi_mtl_ofi_request_t *ofi_req = (ompi_mtl_ofi_request_t *) mtl_request;
-    int ompi_ret, ctxt_id = 0;
+    int ompi_ret, ctxt_id = 0, c_index_for_tag;
     void *start;
     size_t length;
     bool free_after;
@@ -668,12 +1105,34 @@ ompi_mtl_ofi_isend_generic(struct mca_mtl_base_module_t *mtl,
     mca_mtl_ofi_endpoint_t *endpoint = NULL;
     ompi_mtl_ofi_request_t *ack_req = NULL; /* For synchronous send */
     fi_addr_t sep_peer_fiaddr = 0;
+    mca_mtl_comm_t *mtl_comm;
 
     ompi_mtl_ofi_set_mr_null(ofi_req);
 
-    ctxt_id = ompi_mtl_ofi_map_comm_to_ctxt(comm->c_contextid);
+    if (OMPI_COMM_IS_GLOBAL_INDEX(comm)) {
+        c_index_for_tag = comm->c_index;
+    } else {
+        mtl_comm = comm->c_mtl_comm;
+        if (mtl_comm->c_index_vec[dest].c_index_state == MCA_MTL_OFI_CID_NOT_EXCHANGED) {
+            mtl_comm->c_index_vec[dest].c_index_state = MCA_MTL_OFI_CID_EXCHANGING;
+            ompi_ret = ompi_mtl_ofi_send_excid(mtl, comm, dest, ofi_cq_data, true);
+        }
+        if (mtl_comm->c_index_vec[dest].c_index_state > MCA_MTL_OFI_CID_EXCHANGED) {
+            while (mtl_comm->c_index_vec[dest].c_index_state > MCA_MTL_OFI_CID_EXCHANGED) {
+                ompi_ret = ompi_mtl_ofi_post_recv_excid_buffer(true, comm, dest);
+            }
+        }
+        c_index_for_tag = mtl_comm->c_index_vec[dest].c_index;
+    }
+
+    if (ompi_mtl_ofi.total_ctxts_used > 0) {
+        ctxt_id = comm->c_contextid.cid_sub.u64 % ompi_mtl_ofi.total_ctxts_used;
+    } else {
+        ctxt_id = 0;
+    }
     set_thread_context(ctxt_id);
 
+    ofi_req->type = OMPI_MTL_OFI_SEND;
     ofi_req->event_callback = ompi_mtl_ofi_isend_callback;
     ofi_req->error_callback = ompi_mtl_ofi_send_error_callback;
 
@@ -690,6 +1149,7 @@ ompi_mtl_ofi_isend_generic(struct mca_mtl_base_module_t *mtl,
     ofi_req->length = length;
     ofi_req->status.MPI_ERROR = OMPI_SUCCESS;
     ofi_req->completion_count = 1;
+    ofi_req->convertor = convertor;
 
     if (OPAL_UNLIKELY(length > endpoint->mtl_ofi_module->max_msg_size)) {
         opal_show_help("help-mtl-ofi.txt",
@@ -699,9 +1159,9 @@ ompi_mtl_ofi_isend_generic(struct mca_mtl_base_module_t *mtl,
     }
 
     if (ofi_cq_data) {
-        match_bits = mtl_ofi_create_send_tag_CQD(comm->c_contextid, tag);
+        match_bits = mtl_ofi_create_send_tag_CQD(c_index_for_tag, tag);
     } else {
-        match_bits = mtl_ofi_create_send_tag(comm->c_contextid,
+        match_bits = mtl_ofi_create_send_tag(c_index_for_tag,
                           comm->c_my_rank, tag);
         /* src_addr is ignored when FI_DIRECTED_RECV  is not supported */
     }
@@ -719,17 +1179,57 @@ ompi_mtl_ofi_isend_generic(struct mca_mtl_base_module_t *mtl,
         return ompi_ret;
     }
 
+
+    /** Inject does not currently support device memory
+     *  https://github.com/ofiwg/libfabric/issues/5861
+     */
+    if (!(convertor->flags & CONVERTOR_ACCELERATOR)
+        && (ompi_mtl_ofi.max_inject_size >= length)) {
+        if (ofi_cq_data) {
+            ret = fi_tinjectdata(ompi_mtl_ofi.ofi_ctxt[ctxt_id].tx_ep,
+                    start,
+                    length,
+                    comm->c_my_rank,
+                    sep_peer_fiaddr,
+                    match_bits);
+        } else {
+            ret = fi_tinject(ompi_mtl_ofi.ofi_ctxt[ctxt_id].tx_ep,
+                    start,
+                    length,
+                    sep_peer_fiaddr,
+                    match_bits);
+        }
+
+        if(OPAL_LIKELY(ret == 0)) {
+            ofi_req->event_callback(NULL, ofi_req);
+            return ofi_req->status.MPI_ERROR;
+        } else if(ret != -FI_EAGAIN) {
+            MTL_OFI_LOG_FI_ERR(ret,
+                               ofi_cq_data ? "fi_tinjectdata failed"
+                               : "fi_tinject failed");
+            if (ack_req) {
+                fi_cancel((fid_t)ompi_mtl_ofi.ofi_ctxt[ctxt_id].tx_ep, &ack_req->ctx);
+                free(ack_req);
+            }
+            ofi_req->status.MPI_ERROR = ompi_mtl_ofi_get_error(ret);
+            ofi_req->event_callback(NULL, ofi_req);
+            return ofi_req->status.MPI_ERROR;
+        }
+        /* otherwise fall back to the standard fi_tsend path */
+    }
+
+
     if (ofi_cq_data) {
-        MTL_OFI_RETRY_UNTIL_DONE(fi_tsenddata(ompi_mtl_ofi.ofi_ctxt[ctxt_id].tx_ep,
-                                      start,
-                                      length,
-                                      (NULL == ofi_req->mr) ? NULL : ofi_req->mr->mem_desc,
-                                      comm->c_my_rank,
-                                      sep_peer_fiaddr,
-                                      match_bits,
-                                      (void *) &ofi_req->ctx), ret);
+        OFI_RETRY_UNTIL_DONE(fi_tsenddata(ompi_mtl_ofi.ofi_ctxt[ctxt_id].tx_ep,
+                                          start,
+                                          length,
+                                          (NULL == ofi_req->mr) ? NULL : ofi_req->mr->mem_desc,
+                                          comm->c_my_rank,
+                                          sep_peer_fiaddr,
+                                          match_bits,
+                                          (void *) &ofi_req->ctx), ret);
     } else {
-        MTL_OFI_RETRY_UNTIL_DONE(fi_tsend(ompi_mtl_ofi.ofi_ctxt[ctxt_id].tx_ep,
+        OFI_RETRY_UNTIL_DONE(fi_tsend(ompi_mtl_ofi.ofi_ctxt[ctxt_id].tx_ep,
                                       start,
                                       length,
                                       (NULL == ofi_req->mr) ? NULL : ofi_req->mr->mem_desc,
@@ -759,15 +1259,9 @@ __opal_attribute_always_inline__ static inline int
 ompi_mtl_ofi_recv_callback(struct fi_cq_tagged_entry *wc,
                            ompi_mtl_ofi_request_t *ofi_req)
 {
-    int ompi_ret, ctxt_id = 0;
-    ssize_t ret;
-    ompi_proc_t *ompi_proc = NULL;
-    mca_mtl_ofi_endpoint_t *endpoint = NULL;
+    int ompi_ret = OMPI_SUCCESS;
     int src = mtl_ofi_get_source(wc);
     ompi_status_public_t *status = NULL;
-    struct fi_msg_tagged tagged_msg;
-
-    ctxt_id = ompi_mtl_ofi_map_comm_to_ctxt(ofi_req->comm->c_contextid);
 
     assert(ofi_req->super.ompi_req);
     status = &ofi_req->super.ompi_req->req_status;
@@ -778,6 +1272,7 @@ ompi_mtl_ofi_recv_callback(struct fi_cq_tagged_entry *wc,
      */
     ofi_req->req_started = true;
 
+    status->MPI_ERROR = MPI_SUCCESS;
     status->MPI_SOURCE = src;
     status->MPI_TAG = MTL_OFI_GET_TAG(wc->tag);
     status->_ucount = wc->len;
@@ -813,57 +1308,26 @@ ompi_mtl_ofi_recv_callback(struct fi_cq_tagged_entry *wc,
     */
     assert(!MTL_OFI_IS_SYNC_SEND_ACK(wc->tag));
 
-    /**
-     * If this recv is part of an MPI_Ssend operation, then we send an
-     * acknowledgment back to the sender.
-     * The ack message is sent without generating a completion event in
-     * the completion queue by not setting FI_COMPLETION in the flags to
-     * fi_tsendmsg(FI_SELECTIVE_COMPLETION).
-     * This is done since the 0 byte message requires no
-     * notification on the send side for a successful completion.
-     * If a failure occurs the provider will notify the error
-     * in the cq_readerr during OFI progress. Once the message has been
-     * successfully processed the request is marked as completed.
-     */
     if (OPAL_UNLIKELY(MTL_OFI_IS_SYNC_SEND(wc->tag))) {
-        /**
-         * If the recv request was posted for any source,
-         * we need to extract the source's actual address.
-         */
-        if (ompi_mtl_ofi.any_addr == ofi_req->remote_addr) {
-            ompi_proc = ompi_comm_peer_lookup(ofi_req->comm, src);
-            endpoint = ompi_mtl_ofi_get_endpoint(ofi_req->mtl, ompi_proc);
-            ofi_req->remote_addr = fi_rx_addr(endpoint->peer_fiaddr, ctxt_id, ompi_mtl_ofi.rx_ctx_bits);
-        }
+        ompi_ret = ompi_mtl_ofi_gen_ssend_ack(wc, ofi_req);
 
-        tagged_msg.msg_iov = NULL;
-        tagged_msg.desc = NULL;
-        tagged_msg.iov_count = 0;
-        tagged_msg.addr = ofi_req->remote_addr;
-        /**
-        * We must continue to use the user's original tag but remove the
-        * sync_send protocol tag bit and instead apply the sync_send_ack
-        * tag bit to complete the initator's sync send receive.
-        */
-        tagged_msg.tag = (wc->tag | ompi_mtl_ofi.sync_send_ack) & ~ompi_mtl_ofi.sync_send;
-        tagged_msg.context = NULL;
-        tagged_msg.data = 0;
-
-        MTL_OFI_RETRY_UNTIL_DONE(fi_tsendmsg(ompi_mtl_ofi.ofi_ctxt[ctxt_id].tx_ep,
-                                 &tagged_msg, 0), ret);
-        if (OPAL_UNLIKELY(0 > ret)) {
-            MTL_OFI_LOG_FI_ERR(ret, "fi_tsendmsg failed");
-            status->MPI_ERROR = OMPI_ERROR;
+        if (OPAL_UNLIKELY(OMPI_SUCCESS != ompi_ret)) {
+            opal_output_verbose(1, opal_common_ofi.output,
+                                "%s:%d: ompi_mtl_ofi_gen_ssend_ack failed: %d",
+                                __FILE__, __LINE__, ompi_ret);
+            status->MPI_ERROR = ompi_ret;
         }
     }
 
+    ompi_ret = status->MPI_ERROR;
+
     ofi_req->super.completion_callback(&ofi_req->super);
 
-    return OMPI_SUCCESS;
+    return ompi_ret;
 }
 
 /**
- * Called when an error occured on a recv request.
+ * Called when an error occurred on a recv request.
  */
 __opal_attribute_always_inline__ static inline int
 ompi_mtl_ofi_recv_error_callback(struct fi_cq_err_entry *error,
@@ -909,10 +1373,29 @@ ompi_mtl_ofi_irecv_generic(struct mca_mtl_base_module_t *mtl,
     void *start;
     size_t length;
     bool free_after;
+    mca_mtl_comm_t *mtl_comm;
 
     ompi_mtl_ofi_set_mr_null(ofi_req);
 
-    ctxt_id = ompi_mtl_ofi_map_comm_to_ctxt(comm->c_contextid);
+    if (!OMPI_COMM_IS_GLOBAL_INDEX(comm)) {
+        mtl_comm = comm->c_mtl_comm;
+        if ((src == MPI_ANY_SOURCE || mtl_comm->c_index_vec[src].c_index_state > MCA_MTL_OFI_CID_EXCHANGED) &&
+             !ompi_mtl_ofi.has_posted_initial_buffer) {
+            ompi_mtl_ofi.has_posted_initial_buffer = true;
+            ompi_ret = ompi_mtl_ofi_post_recv_excid_buffer(false, comm, -1);
+        }
+        if (src >= 0 && mtl_comm->c_index_vec[src].c_index_state == MCA_MTL_OFI_CID_NOT_EXCHANGED) {
+            mtl_comm->c_index_vec[src].c_index_state = MCA_MTL_OFI_CID_EXCHANGING;
+            ompi_ret = ompi_mtl_ofi_send_excid(mtl, comm, src, ofi_cq_data, false);
+        }
+    }
+
+    if (ompi_mtl_ofi.total_ctxts_used > 0) {
+        ctxt_id = comm->c_contextid.cid_sub.u64 % ompi_mtl_ofi.total_ctxts_used;
+    } else {
+        ctxt_id = 0;
+    }
+
     set_thread_context(ctxt_id);
 
     if (ofi_cq_data) {
@@ -922,10 +1405,10 @@ ompi_mtl_ofi_irecv_generic(struct mca_mtl_base_module_t *mtl,
             remote_addr = fi_rx_addr(endpoint->peer_fiaddr, ctxt_id, ompi_mtl_ofi.rx_ctx_bits);
         }
 
-        mtl_ofi_create_recv_tag_CQD(&match_bits, &mask_bits, comm->c_contextid,
+        mtl_ofi_create_recv_tag_CQD(&match_bits, &mask_bits, comm->c_index,
                                     tag);
     } else {
-        mtl_ofi_create_recv_tag(&match_bits, &mask_bits, comm->c_contextid, src,
+        mtl_ofi_create_recv_tag(&match_bits, &mask_bits, comm->c_index, src,
                                 tag);
         /* src_addr is ignored when FI_DIRECTED_RECV is not used */
     }
@@ -955,7 +1438,7 @@ ompi_mtl_ofi_irecv_generic(struct mca_mtl_base_module_t *mtl,
         return ompi_ret;
     }
 
-    MTL_OFI_RETRY_UNTIL_DONE(fi_trecv(ompi_mtl_ofi.ofi_ctxt[ctxt_id].rx_ep,
+    OFI_RETRY_UNTIL_DONE(fi_trecv(ompi_mtl_ofi.ofi_ctxt[ctxt_id].rx_ep,
                                       start,
                                       length,
                                       (NULL == ofi_req->mr) ? NULL : ofi_req->mr->mem_desc,
@@ -980,6 +1463,7 @@ __opal_attribute_always_inline__ static inline int
 ompi_mtl_ofi_mrecv_callback(struct fi_cq_tagged_entry *wc,
                             ompi_mtl_ofi_request_t *ofi_req)
 {
+    int ompi_ret = OMPI_SUCCESS;
     struct mca_mtl_request_t *mrecv_req = ofi_req->mrecv_req;
     ompi_status_public_t *status = &mrecv_req->ompi_req->req_status;
     status->MPI_SOURCE = mtl_ofi_get_source(wc);
@@ -989,15 +1473,27 @@ ompi_mtl_ofi_mrecv_callback(struct fi_cq_tagged_entry *wc,
 
     ompi_mtl_ofi_deregister_and_free_buffer(ofi_req);
 
+    if (OPAL_UNLIKELY(MTL_OFI_IS_SYNC_SEND(wc->tag))) {
+        ompi_ret = ompi_mtl_ofi_gen_ssend_ack(wc, ofi_req);
+
+        if (OPAL_UNLIKELY(OMPI_SUCCESS != ompi_ret)) {
+            opal_output_verbose(1, opal_common_ofi.output,
+                                "%s:%d: ompi_mtl_ofi_gen_ssend_ack failed: %d",
+                                __FILE__, __LINE__, ompi_ret);
+            status->MPI_ERROR = ompi_ret;
+        }
+    }
+
+    ompi_ret = status->MPI_ERROR;
     free(ofi_req);
 
     mrecv_req->completion_callback(mrecv_req);
 
-    return OMPI_SUCCESS;
+    return ompi_ret;
 }
 
 /**
- * Called when an error occured on a mrecv request.
+ * Called when an error occurred on a mrecv request.
  */
 __opal_attribute_always_inline__ static inline int
 ompi_mtl_ofi_mrecv_error_callback(struct fi_cq_err_entry *error,
@@ -1048,7 +1544,12 @@ ompi_mtl_ofi_imrecv(struct mca_mtl_base_module_t *mtl,
 
     ompi_mtl_ofi_set_mr_null(ofi_req);
 
-    ctxt_id = ompi_mtl_ofi_map_comm_to_ctxt(comm->c_contextid);
+    if (ompi_mtl_ofi.total_ctxts_used > 0) {
+        ctxt_id = comm->c_contextid.cid_sub.u64 % ompi_mtl_ofi.total_ctxts_used;
+    } else {
+        ctxt_id = 0;
+    }
+
     set_thread_context(ctxt_id);
 
     ompi_ret = ompi_mtl_datatype_recv_buf(convertor,
@@ -1067,6 +1568,8 @@ ompi_mtl_ofi_imrecv(struct mca_mtl_base_module_t *mtl,
     ofi_req->convertor = convertor;
     ofi_req->status.MPI_ERROR = OMPI_SUCCESS;
     ofi_req->mrecv_req = mtl_request;
+    ofi_req->comm = comm;
+
 
     ompi_ret = ompi_mtl_ofi_register_buffer(convertor, ofi_req, start);
     if (OPAL_UNLIKELY(OMPI_SUCCESS != ompi_ret)) {
@@ -1087,7 +1590,7 @@ ompi_mtl_ofi_imrecv(struct mca_mtl_base_module_t *mtl,
     msg.context = (void *)&ofi_req->ctx;
     msg.data = 0;
 
-    MTL_OFI_RETRY_UNTIL_DONE(fi_trecvmsg(ompi_mtl_ofi.ofi_ctxt[ctxt_id].rx_ep, &msg, msgflags), ret);
+    OFI_RETRY_UNTIL_DONE(fi_trecvmsg(ompi_mtl_ofi.ofi_ctxt[ctxt_id].rx_ep, &msg, msgflags), ret);
     if (OPAL_UNLIKELY(0 > ret)) {
         ompi_mtl_ofi_deregister_and_free_buffer(ofi_req);
         MTL_OFI_LOG_FI_ERR(ret, "fi_trecvmsg failed");
@@ -1158,8 +1661,26 @@ ompi_mtl_ofi_iprobe_generic(struct mca_mtl_base_module_t *mtl,
     struct fi_msg_tagged msg;
     uint64_t msgflags = FI_PEEK | FI_COMPLETION;
     int ctxt_id = 0;
+    mca_mtl_comm_t *mtl_comm;
+
+    if (!OMPI_COMM_IS_GLOBAL_INDEX(comm)) {
+        mtl_comm = comm->c_mtl_comm;
+        if ((src == MPI_ANY_SOURCE || mtl_comm->c_index_vec[src].c_index_state > MCA_MTL_OFI_CID_EXCHANGED) &&
+                             !ompi_mtl_ofi.has_posted_initial_buffer) {
+            ompi_mtl_ofi.has_posted_initial_buffer = true;
+            ret = ompi_mtl_ofi_post_recv_excid_buffer(false, comm, -1);
+        }
+        if (src >= 0 && mtl_comm->c_index_vec[src].c_index_state == MCA_MTL_OFI_CID_NOT_EXCHANGED) {
+            mtl_comm->c_index_vec[src].c_index_state = MCA_MTL_OFI_CID_EXCHANGING;
+            ret = ompi_mtl_ofi_send_excid(mtl, comm, src, ofi_cq_data, false);
+        }
+    }
 
-    ctxt_id = ompi_mtl_ofi_map_comm_to_ctxt(comm->c_contextid);
+    if (ompi_mtl_ofi.total_ctxts_used > 0) {
+        ctxt_id = comm->c_contextid.cid_sub.u64 % ompi_mtl_ofi.total_ctxts_used;
+    } else {
+        ctxt_id = 0;
+    }
     set_thread_context(ctxt_id);
 
     if (ofi_cq_data) {
@@ -1170,11 +1691,11 @@ ompi_mtl_ofi_iprobe_generic(struct mca_mtl_base_module_t *mtl,
             remote_proc = fi_rx_addr(endpoint->peer_fiaddr, ctxt_id, ompi_mtl_ofi.rx_ctx_bits);
         }
 
-        mtl_ofi_create_recv_tag_CQD(&match_bits, &mask_bits, comm->c_contextid,
+        mtl_ofi_create_recv_tag_CQD(&match_bits, &mask_bits, comm->c_index,
                                     tag);
     }
     else {
-        mtl_ofi_create_recv_tag(&match_bits, &mask_bits, comm->c_contextid, src,
+        mtl_ofi_create_recv_tag(&match_bits, &mask_bits, comm->c_index, src,
                                 tag);
         /* src_addr is ignored when FI_DIRECTED_RECV is not used */
     }
@@ -1201,7 +1722,7 @@ ompi_mtl_ofi_iprobe_generic(struct mca_mtl_base_module_t *mtl,
     ofi_req.completion_count = 1;
     ofi_req.match_state = 0;
 
-    MTL_OFI_RETRY_UNTIL_DONE(fi_trecvmsg(ompi_mtl_ofi.ofi_ctxt[ctxt_id].rx_ep, &msg, msgflags), ret);
+    OFI_RETRY_UNTIL_DONE(fi_trecvmsg(ompi_mtl_ofi.ofi_ctxt[ctxt_id].rx_ep, &msg, msgflags), ret);
     if (OPAL_UNLIKELY(0 > ret)) {
         MTL_OFI_LOG_FI_ERR(ret, "fi_trecvmsg failed");
         return ompi_mtl_ofi_get_error(ret);
@@ -1214,7 +1735,7 @@ ompi_mtl_ofi_iprobe_generic(struct mca_mtl_base_module_t *mtl,
     *flag = ofi_req.match_state;
     if (1 == *flag) {
         if (MPI_STATUS_IGNORE != status) {
-            *status = ofi_req.status;
+            OMPI_COPY_STATUS(status, ofi_req.status, false);
         }
     }
 
@@ -1240,8 +1761,26 @@ ompi_mtl_ofi_improbe_generic(struct mca_mtl_base_module_t *mtl,
     struct fi_msg_tagged msg;
     uint64_t msgflags = FI_PEEK | FI_CLAIM | FI_COMPLETION;
     int ctxt_id = 0;
+    mca_mtl_comm_t *mtl_comm;
+
+    if (!OMPI_COMM_IS_GLOBAL_INDEX(comm)) {
+        mtl_comm = comm->c_mtl_comm;
+        if ((src == MPI_ANY_SOURCE || mtl_comm->c_index_vec[src].c_index_state > MCA_MTL_OFI_CID_EXCHANGED) 
+             && !ompi_mtl_ofi.has_posted_initial_buffer) {
+            ompi_mtl_ofi.has_posted_initial_buffer = true;
+            ret = ompi_mtl_ofi_post_recv_excid_buffer(false, comm, -1);
+        }
+        if (src >= 0 && mtl_comm->c_index_vec[src].c_index_state == MCA_MTL_OFI_CID_NOT_EXCHANGED) {
+            mtl_comm->c_index_vec[src].c_index_state = MCA_MTL_OFI_CID_EXCHANGING;
+            ret = ompi_mtl_ofi_send_excid(mtl, comm, src, ofi_cq_data, false);
+        }
+    }
 
-    ctxt_id = ompi_mtl_ofi_map_comm_to_ctxt(comm->c_contextid);
+    if (ompi_mtl_ofi.total_ctxts_used > 0) {
+        ctxt_id = comm->c_contextid.cid_sub.u64 % ompi_mtl_ofi.total_ctxts_used;
+    } else {
+        ctxt_id = 0;
+    }
     set_thread_context(ctxt_id);
 
     ofi_req = malloc(sizeof *ofi_req);
@@ -1260,12 +1799,12 @@ ompi_mtl_ofi_improbe_generic(struct mca_mtl_base_module_t *mtl,
             remote_proc = fi_rx_addr(endpoint->peer_fiaddr, ctxt_id, ompi_mtl_ofi.rx_ctx_bits);
         }
 
-        mtl_ofi_create_recv_tag_CQD(&match_bits, &mask_bits, comm->c_contextid,
+        mtl_ofi_create_recv_tag_CQD(&match_bits, &mask_bits, comm->c_index,
                                     tag);
     }
     else {
         /* src_addr is ignored when FI_DIRECTED_RECV is not used */
-        mtl_ofi_create_recv_tag(&match_bits, &mask_bits, comm->c_contextid, src,
+        mtl_ofi_create_recv_tag(&match_bits, &mask_bits, comm->c_index, src,
                                 tag);
     }
 
@@ -1292,7 +1831,7 @@ ompi_mtl_ofi_improbe_generic(struct mca_mtl_base_module_t *mtl,
     ofi_req->match_state = 0;
     ofi_req->mask_bits = mask_bits;
 
-    MTL_OFI_RETRY_UNTIL_DONE(fi_trecvmsg(ompi_mtl_ofi.ofi_ctxt[ctxt_id].rx_ep, &msg, msgflags), ret);
+    OFI_RETRY_UNTIL_DONE(fi_trecvmsg(ompi_mtl_ofi.ofi_ctxt[ctxt_id].rx_ep, &msg, msgflags), ret);
     if (OPAL_UNLIKELY(0 > ret)) {
         MTL_OFI_LOG_FI_ERR(ret, "fi_trecvmsg failed");
         free(ofi_req);
@@ -1306,7 +1845,7 @@ ompi_mtl_ofi_improbe_generic(struct mca_mtl_base_module_t *mtl,
     *matched = ofi_req->match_state;
     if (1 == *matched) {
         if (MPI_STATUS_IGNORE != status) {
-            *status = ofi_req->status;
+            OMPI_COPY_STATUS(status, ofi_req->status, false);
         }
 
         (*message) = ompi_message_alloc();
@@ -1335,9 +1874,8 @@ ompi_mtl_ofi_cancel(struct mca_mtl_base_module_t *mtl,
     int ret, ctxt_id = 0;
     ompi_mtl_ofi_request_t *ofi_req = (ompi_mtl_ofi_request_t*) mtl_request;
 
-    ctxt_id = ompi_mtl_ofi_map_comm_to_ctxt(ofi_req->comm->c_contextid);
-
     switch (ofi_req->type) {
+        case OMPI_MTL_OFI_PROBE:
         case OMPI_MTL_OFI_SEND:
             /**
              * Cannot cancel sends yet
@@ -1353,6 +1891,8 @@ ompi_mtl_ofi_cancel(struct mca_mtl_base_module_t *mtl,
             ompi_mtl_ofi_progress();
 
             if (!ofi_req->req_started) {
+                ctxt_id = ompi_mtl_ofi_map_comm_to_ctxt(ofi_req->comm->c_index);
+
                 ret = fi_cancel((fid_t)ompi_mtl_ofi.ofi_ctxt[ctxt_id].rx_ep,
                                &ofi_req->ctx);
                 if (0 == ret) {
@@ -1375,228 +1915,6 @@ ompi_mtl_ofi_cancel(struct mca_mtl_base_module_t *mtl,
     return OMPI_SUCCESS;
 }
 
-static int ompi_mtl_ofi_init_contexts(struct mca_mtl_base_module_t *mtl,
-                                      struct ompi_communicator_t *comm,
-                                      mca_mtl_ofi_ep_type ep_type)
-{
-    int ret;
-    int ctxt_id = ompi_mtl_ofi.total_ctxts_used;
-    struct fi_cq_attr cq_attr = {0};
-    cq_attr.format = FI_CQ_FORMAT_TAGGED;
-    cq_attr.size = ompi_mtl_ofi.ofi_progress_event_count;
-
-    if (OFI_REGULAR_EP == ep_type) {
-        /*
-         * For regular endpoints, just create the Lock object and register
-         * progress function.
-         */
-        goto init_regular_ep;
-    }
-
-    /*
-     * We only create upto Max number of contexts asked for by the user.
-     * If user enables thread grouping feature and creates more number of
-     * communicators than available contexts, then we set the threshold
-     * context_id so that new communicators created beyond the threshold
-     * will be assigned to contexts in a round-robin fashion.
-     */
-    if (ompi_mtl_ofi.num_ofi_contexts <= ompi_mtl_ofi.total_ctxts_used) {
-        ompi_mtl_ofi.comm_to_context[comm->c_contextid] = comm->c_contextid %
-                                                          ompi_mtl_ofi.total_ctxts_used;
-        if (!ompi_mtl_ofi.threshold_comm_context_id) {
-            ompi_mtl_ofi.threshold_comm_context_id = comm->c_contextid;
-
-            opal_show_help("help-mtl-ofi.txt", "SEP thread grouping ctxt limit", true, ctxt_id,
-                           ompi_process_info.nodename, __FILE__, __LINE__);
-        }
-
-        return OMPI_SUCCESS;
-    }
-
-    /* Init context info for Scalable EPs */
-    ret = fi_tx_context(ompi_mtl_ofi.sep, ctxt_id, NULL, &ompi_mtl_ofi.ofi_ctxt[ctxt_id].tx_ep, NULL);
-    if (ret) {
-        MTL_OFI_LOG_FI_ERR(ret, "fi_tx_context failed");
-        goto init_error;
-    }
-
-    ret = fi_rx_context(ompi_mtl_ofi.sep, ctxt_id, NULL, &ompi_mtl_ofi.ofi_ctxt[ctxt_id].rx_ep, NULL);
-    if (ret) {
-        MTL_OFI_LOG_FI_ERR(ret, "fi_rx_context failed");
-        goto init_error;
-    }
-
-    ret = fi_cq_open(ompi_mtl_ofi.domain, &cq_attr, &ompi_mtl_ofi.ofi_ctxt[ctxt_id].cq, NULL);
-    if (ret) {
-        MTL_OFI_LOG_FI_ERR(ret, "fi_cq_open failed");
-        goto init_error;
-    }
-
-    /* Bind CQ to TX/RX context object */
-    ret = fi_ep_bind(ompi_mtl_ofi.ofi_ctxt[ctxt_id].tx_ep, (fid_t)ompi_mtl_ofi.ofi_ctxt[ctxt_id].cq,
-                     FI_TRANSMIT | FI_SELECTIVE_COMPLETION);
-    if (0 != ret) {
-        MTL_OFI_LOG_FI_ERR(ret, "fi_bind CQ-EP (FI_TRANSMIT) failed");
-        goto init_error;
-    }
-
-    ret = fi_ep_bind(ompi_mtl_ofi.ofi_ctxt[ctxt_id].rx_ep, (fid_t)ompi_mtl_ofi.ofi_ctxt[ctxt_id].cq,
-                     FI_RECV | FI_SELECTIVE_COMPLETION);
-    if (0 != ret) {
-        MTL_OFI_LOG_FI_ERR(ret, "fi_bind CQ-EP (FI_RECV) failed");
-        goto init_error;
-    }
-
-    /* Enable Endpoint for communication. This commits the bind operations */
-    ret = fi_enable(ompi_mtl_ofi.ofi_ctxt[ctxt_id].tx_ep);
-    if (0 != ret) {
-        MTL_OFI_LOG_FI_ERR(ret, "fi_enable (send context) failed");
-        goto init_error;
-    }
-
-    ret = fi_enable(ompi_mtl_ofi.ofi_ctxt[ctxt_id].rx_ep);
-    if (0 != ret) {
-        MTL_OFI_LOG_FI_ERR(ret, "fi_enable (recv context) failed");
-        goto init_error;
-    }
-
-init_regular_ep:
-    /* Initialize per-context lock */
-    OBJ_CONSTRUCT(&ompi_mtl_ofi.ofi_ctxt[ctxt_id].context_lock, opal_mutex_t);
-
-    if (MPI_COMM_WORLD == comm) {
-        ret = opal_progress_register(ompi_mtl_ofi_progress_no_inline);
-        if (OMPI_SUCCESS != ret) {
-            opal_output_verbose(1, opal_common_ofi.output,
-                                "%s:%d: opal_progress_register failed: %d\n",
-                                __FILE__, __LINE__, ret);
-            goto init_error;
-        }
-    }
-
-    ompi_mtl_ofi.comm_to_context[comm->c_contextid] = ompi_mtl_ofi.total_ctxts_used;
-    ompi_mtl_ofi.total_ctxts_used++;
-
-    return OMPI_SUCCESS;
-
-init_error:
-    if (ompi_mtl_ofi.ofi_ctxt[ctxt_id].tx_ep) {
-        (void) fi_close((fid_t)ompi_mtl_ofi.ofi_ctxt[ctxt_id].tx_ep);
-    }
-
-    if (ompi_mtl_ofi.ofi_ctxt[ctxt_id].rx_ep) {
-        (void) fi_close((fid_t)ompi_mtl_ofi.ofi_ctxt[ctxt_id].rx_ep);
-    }
-
-    if (ompi_mtl_ofi.ofi_ctxt[ctxt_id].cq) {
-        (void) fi_close((fid_t)ompi_mtl_ofi.ofi_ctxt[ctxt_id].cq);
-    }
-
-    return ret;
-}
-
-static int ompi_mtl_ofi_finalize_contexts(struct mca_mtl_base_module_t *mtl,
-                                          struct ompi_communicator_t *comm,
-                                          mca_mtl_ofi_ep_type ep_type)
-{
-    int ret = OMPI_SUCCESS, ctxt_id = 0;
-
-    if (OFI_REGULAR_EP == ep_type) {
-        /* For regular EPs, simply destruct Lock object and exit */
-        goto finalize_regular_ep;
-    }
-
-    if (ompi_mtl_ofi.thread_grouping &&
-        ompi_mtl_ofi.threshold_comm_context_id &&
-        ((uint32_t) ompi_mtl_ofi.threshold_comm_context_id <= comm->c_contextid)) {
-        return OMPI_SUCCESS;
-    }
-
-    ctxt_id = ompi_mtl_ofi.thread_grouping ?
-           ompi_mtl_ofi.comm_to_context[comm->c_contextid] : 0;
-
-    /*
-     * For regular EPs, TX/RX contexts are aliased to SEP object which is
-     * closed in ompi_mtl_ofi_finalize(). So, skip handling those here.
-     */
-    if ((ret = fi_close((fid_t)ompi_mtl_ofi.ofi_ctxt[ctxt_id].tx_ep))) {
-        goto finalize_err;
-    }
-
-    if ((ret = fi_close((fid_t)ompi_mtl_ofi.ofi_ctxt[ctxt_id].rx_ep))) {
-        goto finalize_err;
-    }
-
-    if ((ret = fi_close((fid_t)ompi_mtl_ofi.ofi_ctxt[ctxt_id].cq))) {
-        goto finalize_err;
-    }
-
-finalize_regular_ep:
-    /* Destroy context lock */
-    OBJ_DESTRUCT(&ompi_mtl_ofi.ofi_ctxt[ctxt_id].context_lock);
-
-    return OMPI_SUCCESS;
-
-finalize_err:
-    opal_show_help("help-mtl-ofi.txt", "OFI call fail", true,
-                   "fi_close",
-                   ompi_process_info.nodename, __FILE__, __LINE__,
-                   fi_strerror(-ret), ret);
-
-    return OMPI_ERROR;
-}
-
-__opal_attribute_always_inline__ static inline int
-ompi_mtl_ofi_add_comm(struct mca_mtl_base_module_t *mtl,
-                      struct ompi_communicator_t *comm)
-{
-    int ret;
-    mca_mtl_ofi_ep_type ep_type = (0 == ompi_mtl_ofi.enable_sep) ?
-                                  OFI_REGULAR_EP : OFI_SCALABLE_EP;
-
-    /*
-     * If thread grouping enabled, add new OFI context for each communicator
-     * other than MPI_COMM_SELF.
-     */
-    if ((ompi_mtl_ofi.thread_grouping && (MPI_COMM_SELF != comm)) ||
-        /* If no thread grouping, add new OFI context only
-         * for MPI_COMM_WORLD.
-         */
-        (!ompi_mtl_ofi.thread_grouping && (MPI_COMM_WORLD == comm))) {
-
-        ret = ompi_mtl_ofi_init_contexts(mtl, comm, ep_type);
-
-        if (OMPI_SUCCESS != ret) {
-            goto error;
-        }
-    }
-
-    return OMPI_SUCCESS;
-
-error:
-    return OMPI_ERROR;
-}
-
-__opal_attribute_always_inline__ static inline int
-ompi_mtl_ofi_del_comm(struct mca_mtl_base_module_t *mtl,
-                      struct ompi_communicator_t *comm)
-{
-    int ret = OMPI_SUCCESS;
-    mca_mtl_ofi_ep_type ep_type = (0 == ompi_mtl_ofi.enable_sep) ?
-                                  OFI_REGULAR_EP : OFI_SCALABLE_EP;
-
-    /*
-     * Clean up OFI contexts information.
-     */
-    if ((ompi_mtl_ofi.thread_grouping && (MPI_COMM_SELF != comm)) ||
-        (!ompi_mtl_ofi.thread_grouping && (MPI_COMM_WORLD == comm))) {
-
-        ret = ompi_mtl_ofi_finalize_contexts(mtl, comm, ep_type);
-    }
-
-    return ret;
-}
-
 #ifdef MCA_ompi_mtl_DIRECT_CALL
 
 __opal_attribute_always_inline__ static inline int
diff --git a/ompi/mca/mtl/ofi/mtl_ofi_component.c b/ompi/mca/mtl/ofi/mtl_ofi_component.c
index a57dd51f571..27e52107638 100644
--- a/ompi/mca/mtl/ofi/mtl_ofi_component.c
+++ b/ompi/mca/mtl/ofi/mtl_ofi_component.c
@@ -2,11 +2,11 @@
 /*
  * Copyright (c) 2013-2018 Intel, Inc. All rights reserved
  *
- * Copyright (c) 2014-2017 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2014-2021 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2015-2016 Los Alamos National Security, LLC.  All rights
  *                         reserved.
- * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
- * Copyright (c) 2020      Triad National Security, LLC. All rights
+ * Copyright (c) 2018-2022 Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2020-2023 Triad National Security, LLC. All rights
  *                         reserved.
  * $COPYRIGHT$
  *
@@ -20,9 +20,6 @@
 #include "opal/util/argv.h"
 #include "opal/util/printf.h"
 #include "opal/mca/common/ofi/common_ofi.h"
-#if OPAL_CUDA_SUPPORT
-#include "opal/mca/common/cuda/common_cuda.h"
-#endif /* OPAL_CUDA_SUPPORT */
 
 static int ompi_mtl_ofi_component_open(void);
 static int ompi_mtl_ofi_component_query(mca_base_module_t **module, int *priority);
@@ -31,7 +28,8 @@ static int ompi_mtl_ofi_component_register(void);
 
 static mca_mtl_base_module_t*
 ompi_mtl_ofi_component_init(bool enable_progress_threads,
-                            bool enable_mpi_threads);
+                            bool enable_mpi_threads,
+                            bool *accelerator_support);
 
 static int param_priority;
 static int control_progress;
@@ -40,8 +38,7 @@ static int av_type;
 static int ofi_tag_mode;
 
 #if OPAL_HAVE_THREAD_LOCAL
-    opal_thread_local int per_thread_ctx;
-    opal_thread_local struct fi_cq_tagged_entry wc[MTL_OFI_MAX_PROG_EVENT_COUNT];
+    opal_thread_local int ompi_mtl_ofi_per_thread_ctx;
 #endif
 
 /*
@@ -185,7 +182,7 @@ ompi_mtl_ofi_component_register(void)
     control_progress = MTL_OFI_PROG_UNSPEC;
     mca_base_component_var_register (&mca_mtl_ofi_component.super.mtl_version,
                                      "control_progress",
-                                     "Specify control progress model (default: unspecificed, use provider's default). Set to auto or manual for auto or manual progress respectively.",
+                                     "Specify control progress model (default: unspecified, use provider's default). Set to auto or manual for auto or manual progress respectively.",
                                      MCA_BASE_VAR_TYPE_INT, new_enum, 0, 0,
                                      OPAL_INFO_LVL_3,
                                      MCA_BASE_VAR_SCOPE_READONLY,
@@ -254,9 +251,16 @@ ompi_mtl_ofi_component_register(void)
                                     MCA_BASE_VAR_SCOPE_READONLY,
                                     &ompi_mtl_ofi.num_ofi_contexts);
 
-    opal_common_ofi_register_mca_variables(&mca_mtl_ofi_component.super.mtl_version);
+    ompi_mtl_ofi.disable_hmem = false;
+    mca_base_component_var_register(&mca_mtl_ofi_component.super.mtl_version,
+                                    "disable_hmem",
+                                    "Disable HMEM usage",
+                                    MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
+                                    OPAL_INFO_LVL_3,
+                                    MCA_BASE_VAR_SCOPE_READONLY,
+                                    &ompi_mtl_ofi.disable_hmem);
 
-    return OMPI_SUCCESS;
+    return opal_common_ofi_mca_register(&mca_mtl_ofi_component.super.mtl_version);
 }
 
 
@@ -285,8 +289,7 @@ ompi_mtl_ofi_component_open(void)
             "provider_exclude")) {
         return OMPI_ERR_NOT_AVAILABLE;
     }
-
-    return OMPI_SUCCESS;
+    return opal_common_ofi_open();
 }
 
 static int
@@ -300,11 +303,7 @@ ompi_mtl_ofi_component_query(mca_base_module_t **module, int *priority)
 static int
 ompi_mtl_ofi_component_close(void)
 {
-#if OPAL_CUDA_SUPPORT
-    mca_common_cuda_fini();
-#endif
-    opal_common_ofi_mca_deregister();
-    return OMPI_SUCCESS;
+    return opal_common_ofi_close();
 }
 
 int
@@ -340,7 +339,7 @@ select_ofi_provider(struct fi_info *providers,
     }
 
     opal_output_verbose(1, opal_common_ofi.output,
-                        "%s:%d: mtl:ofi:prov: %s\n",
+                        "%s:%d: mtl:ofi:provider: %s\n",
                         __FILE__, __LINE__,
                         (prov ? prov->fabric_attr->prov_name : "none"));
 
@@ -356,16 +355,16 @@ select_ofi_provider(struct fi_info *providers,
       * however some may have multiple info objects with different
       * attributes for the same NIC. The initial provider attributes
       * are used to ensure that all NICs we return provide the same
-      * capabilities as the inital one.
+      * capabilities as the initial one.
       *
       * We use package rank to select between NICs of equal distance
       * if we cannot calculate a package_rank, we fall back to using the
       * process id.
       */
     if (NULL != prov) {
-        prov = opal_mca_common_ofi_select_provider(prov, &ompi_process_info);
-        opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
-                            "%s:%d: mtl:ofi:provider: %s\n",
+        prov = opal_common_ofi_select_provider(prov, &ompi_process_info);
+        opal_output_verbose(1, opal_common_ofi.output,
+                            "%s:%d: mtl:ofi:provider:domain: %s\n",
                             __FILE__, __LINE__,
                             (prov ? prov->domain_attr->name : "none"));
     }
@@ -374,8 +373,8 @@ select_ofi_provider(struct fi_info *providers,
 }
 
 static void
-ompi_mtl_ofi_define_tag_mode(int ofi_tag_mode, int *bits_for_cid) {
-    switch (ofi_tag_mode) {
+ompi_mtl_ofi_define_tag_mode(int ofi_tag_mode_arg, int *bits_for_cid) {
+    switch (ofi_tag_mode_arg) {
         case MTL_OFI_TAG_1:
             *bits_for_cid = (int) MTL_OFI_CID_BIT_COUNT_1;
             ompi_mtl_ofi.base.mtl_max_tag = (int)((1ULL << (MTL_OFI_TAG_BIT_COUNT_1 - 1)) - 1);
@@ -479,7 +478,7 @@ static int ompi_mtl_ofi_init_sep(struct fi_info *prov, int universe_size)
 
     /*
      * If SEP supported and Thread Grouping feature enabled, use
-     * num_ofi_contexts + 2. Extra 2 items is to accomodate Open MPI contextid
+     * num_ofi_contexts + 2. Extra 2 items is to accommodate Open MPI contextid
      * numbering- COMM_WORLD is 0, COMM_SELF is 1. Other user created
      * Comm contextid values are assigned sequentially starting with 3.
      */
@@ -566,9 +565,10 @@ static int ompi_mtl_ofi_init_regular_ep(struct fi_info * prov, int universe_size
 
 static mca_mtl_base_module_t*
 ompi_mtl_ofi_component_init(bool enable_progress_threads,
-                            bool enable_mpi_threads)
+                            bool enable_mpi_threads,
+                            bool *accelerator_support)
 {
-    int ret, fi_version;
+    int ret, fi_primary_version, fi_alternate_version;
     int num_local_ranks, sep_support_in_provider, max_ofi_ctxts;
     int ofi_tag_leading_zeros, ofi_tag_bits_for_cid;
     char **include_list = NULL;
@@ -576,14 +576,11 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
     struct fi_info *hints, *hints_dup = NULL;
     struct fi_info *providers = NULL;
     struct fi_info *prov = NULL;
-    struct fi_info *prov_cq_data = NULL;
-    char ep_name[FI_NAME_MAX] = {0};
-    size_t namelen;
+    void *ep_name = NULL;
+    size_t namelen = 0;
     int universe_size;
     char *univ_size_str;
 
-    opal_common_ofi_mca_register();
-
     opal_output_verbose(1, opal_common_ofi.output,
                         "%s:%d: mtl:ofi:provider_include = \"%s\"\n",
                         __FILE__, __LINE__, *opal_common_ofi.prov_include);
@@ -603,8 +600,19 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
      * that checking -- e.g., the shared memory provider supports
      * intranode communication (FI_LOCAL_COMM), but not internode
      * (FI_REMOTE_COMM), which is insufficient for MTL selection.
+     *
+     * Note: API version 1.9 is the first version that supports FI_HMEM
+     *
+     * Note: API version 1.18 is the first version that clearly define
+     * provider's behavior in making CUDA API calls that all provider
+     * by default is permitted to make CUDA calls if application uses >= 1.18 API.
+     *
+     * If application is using < 1.18 API, some provider will not claim support
+     * of FI_HMEM (even if they are capable of) because it does not know
+     * whether application permits it to make CUDA calls.
      */
-    fi_version = FI_VERSION(1, 5);
+    fi_primary_version = FI_VERSION(1, 18);
+    fi_alternate_version = FI_VERSION(1, 9);
 
     /**
      * Hints to filter providers
@@ -624,22 +632,21 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
         goto error;
     }
 
-#if OPAL_CUDA_SUPPORT
-    /** If Open MPI is built with CUDA, request device transfer
-     *  capabilities */
-    hints->caps |= FI_HMEM;
-    hints->domain_attr->mr_mode |= FI_MR_HMEM;
-    /**
-     * Note: API version 1.9 is the first version that supports FI_HMEM
-     */
-    fi_version = FI_VERSION(1, 9);
-#endif /* OPAL_CUDA_SUPPORT */
+    /* Request device transfer capabilities */
+#if defined(FI_HMEM)
+    if (false == ompi_mtl_ofi.disable_hmem) {
+        hints->caps |= FI_HMEM;
+        hints->domain_attr->mr_mode |= FI_MR_HMEM | FI_MR_ALLOCATED;
+    }
+#endif
+
+no_hmem:
 
     /* Make sure to get a RDM provider that can do the tagged matching
        interface and local communication and remote communication. */
-    hints->mode               = FI_CONTEXT;
+    hints->mode               = FI_CONTEXT | FI_CONTEXT2;
     hints->ep_attr->type      = FI_EP_RDM;
-    hints->caps               |= FI_TAGGED | FI_LOCAL_COMM | FI_REMOTE_COMM | FI_DIRECTED_RECV;
+    hints->caps               |= FI_MSG | FI_TAGGED | FI_LOCAL_COMM | FI_REMOTE_COMM | FI_DIRECTED_RECV;
     hints->tx_attr->msg_order = FI_ORDER_SAS;
     hints->rx_attr->msg_order = FI_ORDER_SAS;
     hints->rx_attr->op_flags  = FI_COMPLETION;
@@ -706,7 +713,11 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
         hints_dup->caps &= ~(FI_LOCAL_COMM | FI_REMOTE_COMM);
         hints_dup->fabric_attr->prov_name = strdup("efa");
 
-        ret = fi_getinfo(fi_version, NULL, NULL, 0ULL, hints_dup, &providers);
+        ret = fi_getinfo(fi_primary_version, NULL, NULL, 0ULL, hints_dup, &providers);
+        if (FI_ENOSYS == -ret) {
+            /* libfabric is not new enough, fallback to use older version of API */
+           ret = fi_getinfo(fi_alternate_version, NULL, NULL, 0ULL, hints_dup, &providers);
+	}
 
         opal_output_verbose(1, opal_common_ofi.output,
                             "%s:%d: EFA specific fi_getinfo(): %s\n",
@@ -738,19 +749,38 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
      * remote node or service.  this does not necessarily allocate resources.
      * Pass NULL for name/service because we want a list of providers supported.
      */
-    ret = fi_getinfo(fi_version,    /* OFI version requested                    */
+    ret = fi_getinfo(fi_primary_version,    /* OFI version requested            */
                      NULL,          /* Optional name or fabric to resolve       */
                      NULL,          /* Optional service name or port to request */
                      0ULL,          /* Optional flag                            */
                      hints,         /* In: Hints to filter providers            */
                      &providers);   /* Out: List of matching providers          */
+    if (FI_ENOSYS == -ret) {
+        ret = fi_getinfo(fi_alternate_version, NULL, NULL, 0ULL, hints, &providers);
+    }
 
     opal_output_verbose(1, opal_common_ofi.output,
                         "%s:%d: fi_getinfo(): %s\n",
                         __FILE__, __LINE__, fi_strerror(-ret));
 
-    if (FI_ENODATA == -ret) {
-        // It is not an error if no information is returned.
+    if ((FI_ENODATA == -ret)
+        || (0 == ret && include_list
+            && 0 == opal_common_ofi_count_providers_in_list(providers, include_list))
+        || (0 == ret && !include_list && exclude_list
+            && opal_common_ofi_providers_subset_of_list(providers, exclude_list))) {
+#if defined(FI_HMEM)
+        /* Attempt selecting a provider without FI_HMEM hints */
+        if (hints->caps & FI_HMEM) {
+            hints->caps &= ~FI_HMEM;
+            hints->domain_attr->mr_mode &= ~FI_MR_HMEM;
+            if (providers) {
+                (void) fi_freeinfo(providers);
+                providers = NULL;
+            }
+            goto no_hmem;
+        }
+#endif
+        /* It is not an error if no information is returned. */
         goto error;
     } else if (0 != ret) {
         opal_show_help("help-mtl-ofi.txt", "OFI call fail", true,
@@ -777,14 +807,41 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
     opal_argv_free(exclude_list);
     exclude_list = NULL;
 
-#if OPAL_CUDA_SUPPORT
-    if (!(prov->caps & FI_HMEM)) {
-        opal_output_verbose(1, opal_common_ofi.output,
-                            "%s:%d: Libfabric provider does not support CUDA buffers\n",
-                            __FILE__, __LINE__);
-        goto error;
+    *accelerator_support = false;
+#if defined(FI_HMEM)
+    if (!(prov->caps & FI_HMEM) || (true == ompi_mtl_ofi.disable_hmem)) {
+        if (!(prov->caps & FI_HMEM) && (false == ompi_mtl_ofi.disable_hmem)) {
+            opal_output_verbose(50, opal_common_ofi.output,
+                                "%s:%d: Libfabric provider does not support device buffers. Continuing with device to host copies.\n",
+                               __FILE__, __LINE__);
+        }
+        if (true == ompi_mtl_ofi.disable_hmem) {
+            opal_output_verbose(50, opal_common_ofi.output,
+                                "%s:%d: Support for device buffers disabled by MCA parameter. Continuing with device to host copies.\n",
+                               __FILE__, __LINE__);
+        }
+    } else {
+        *accelerator_support = true;
+        ompi_mtl_ofi.hmem_needs_reg = true;
+        /*
+         * Workaround for the fact that the CXI provider actually doesn't need for accelerator memory to be registered
+         * for local buffers, but if one does do so using fi_mr_regattr, one actually needs to manage the
+         * requested_key field in the fi_mr_attr attr argument, and the OFI MTL doesn't track which requested_keys
+         * have already been registered. So just set a flag to disable local registration.  Note the OFI BTL doesn't
+         * have a problem here since it uses fi_mr_regattr only within the context of an rcache, and manages the
+         * requested_key field in this way.
+         */
+         if ((NULL != strstr(prov->fabric_attr->prov_name, "cxi")) ||
+             (NULL != strstr(prov->fabric_attr->prov_name, "CXI")) ) {
+             ompi_mtl_ofi.hmem_needs_reg = false;
+         }
+
     }
-#endif /* OPAL_CUDA_SUPPORT */
+#else
+    opal_output_verbose(50, opal_common_ofi.output,
+                        "%s:%d: Libfabric provider does not support device buffers. Continuing with device to host copies.\n",
+                        __FILE__, __LINE__);
+#endif
 
     /**
      * Select the format of the OFI tag
@@ -893,6 +950,20 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
         }
     }
 
+    /* this must be called during single threaded part of the code and
+     * before Libfabric configures its memory monitors.  Easiest to do
+     * that before domain open.  Silently ignore not-supported errors,
+     * as they are not critical to program correctness, but only
+     * indicate that LIbfabric will have to pick a different, possibly
+     * less optimal, monitor. */
+    ret = opal_common_ofi_export_memory_monitor();
+    if (0 != ret && -FI_ENOSYS != ret) {
+        opal_output_verbose(1, opal_common_ofi.output,
+                            "Failed to inject Libfabric memory monitor: %s",
+                             fi_strerror(-ret));
+    }
+
+
     /**
      * Open fabric
      * The getinfo struct returns a fabric attribute struct that can be used to
@@ -928,7 +999,7 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
      */
     ret = fi_domain(ompi_mtl_ofi.fabric,  /* In:  Fabric object                 */
                     prov,                 /* In:  Provider                      */
-                    &ompi_mtl_ofi.domain, /* Out: Domain oject                  */
+                    &ompi_mtl_ofi.domain, /* Out: Domain object                 */
                     NULL);                /* Optional context for domain events */
     if (0 != ret) {
         opal_show_help("help-mtl-ofi.txt", "OFI call fail", true,
@@ -1031,15 +1102,11 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
     fi_freeinfo(providers);
     providers = NULL;
 
-    /**
-     * Get our address and publish it with modex.
-     */
-    namelen = sizeof(ep_name);
-    ret = fi_getname((fid_t)ompi_mtl_ofi.sep,
-                     &ep_name[0],
-                     &namelen);
-    if (ret) {
-        MTL_OFI_LOG_FI_ERR(ret, "fi_getname failed");
+    ret = opal_common_ofi_fi_getname((fid_t)ompi_mtl_ofi.sep,
+                                     &ep_name,
+                                     &namelen);
+    if (OMPI_SUCCESS != ret) {
+        MTL_OFI_LOG_FI_ERR(ret, "opal_common_ofi_fi_getname failed");
         goto error;
     }
 
@@ -1055,15 +1122,16 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
     }
 
     ompi_mtl_ofi.epnamelen = namelen;
+    free(ep_name);
 
     /**
      * Set the ANY_SRC address.
      */
     ompi_mtl_ofi.any_addr = FI_ADDR_UNSPEC;
-
-#if OPAL_CUDA_SUPPORT
-    mca_common_cuda_stage_one_init();
-#endif
+    ompi_mtl_ofi.is_initialized = false;
+    ompi_mtl_ofi.has_posted_initial_buffer = false;
+    
+    ompi_mtl_ofi.base.mtl_flags |= MCA_MTL_BASE_FLAG_SUPPORTS_EXT_CID;
 
     return &ompi_mtl_ofi.base;
 
@@ -1077,9 +1145,6 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
     if (providers) {
         (void) fi_freeinfo(providers);
     }
-    if (prov_cq_data) {
-        (void) fi_freeinfo(prov_cq_data);
-    }
     if (hints) {
         (void) fi_freeinfo(hints);
     }
@@ -1110,6 +1175,9 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
     if (ompi_mtl_ofi.ofi_ctxt) {
         free(ompi_mtl_ofi.ofi_ctxt);
     }
+    if (ep_name) {
+        free(ep_name);
+    }
 
     return NULL;
 }
diff --git a/ompi/mca/mtl/ofi/mtl_ofi_request.h b/ompi/mca/mtl/ofi/mtl_ofi_request.h
index d1c05e9680e..cb746f341db 100644
--- a/ompi/mca/mtl/ofi/mtl_ofi_request.h
+++ b/ompi/mca/mtl/ofi/mtl_ofi_request.h
@@ -34,7 +34,7 @@ struct ompi_mtl_ofi_request_t {
     ompi_mtl_ofi_request_type_t type;
 
     /** OFI context */
-    struct fi_context ctx;
+    struct fi_context2 ctx;
 
     /** Completion count used by blocking and/or synchronous operations */
     volatile int completion_count;
@@ -89,7 +89,7 @@ struct ompi_mtl_ofi_request_t {
     struct mca_mtl_request_t *mrecv_req;
 
     /** Stores reference to memory region from registration */
-    /*  Set to NULL if memory not registered or if non CUDA buffer */
+    /*  Set to NULL if memory not registered or if non accelerator buffer */
     struct fid_mr *mr;
 };
 typedef struct ompi_mtl_ofi_request_t ompi_mtl_ofi_request_t;
diff --git a/ompi/mca/mtl/ofi/mtl_ofi_types.h b/ompi/mca/mtl/ofi/mtl_ofi_types.h
index a2c2f3d4308..a925f0ec28e 100644
--- a/ompi/mca/mtl/ofi/mtl_ofi_types.h
+++ b/ompi/mca/mtl/ofi/mtl_ofi_types.h
@@ -2,6 +2,8 @@
  * Copyright (c) 2013-2018 Intel, Inc. All rights reserved
  *
  * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2022-2023 Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -57,6 +59,7 @@ typedef struct mca_mtl_ofi_module_t {
     int enable_sep;                     /* MCA to enable/disable SEP feature */
     int thread_grouping;                /* MCA for thread grouping feature */
     int num_ofi_contexts;               /* MCA for number of contexts to use */
+    bool disable_hmem;                   /* MCA to enable/disable request for FI_HMEM support from provider */
 
     /** Endpoint name length */
     size_t epnamelen;
@@ -95,6 +98,10 @@ typedef struct mca_mtl_ofi_module_t {
     /** Optimized function Symbol Tables **/
     struct ompi_mtl_ofi_symtable sym_table;
 
+    bool is_initialized;
+    bool has_posted_initial_buffer;
+    bool hmem_needs_reg;
+
 } mca_mtl_ofi_module_t;
 
 extern mca_mtl_ofi_module_t ompi_mtl_ofi;
diff --git a/ompi/mca/mtl/portals4/configure.m4 b/ompi/mca/mtl/portals4/configure.m4
index ef4c5aa5184..b26b87c8837 100644
--- a/ompi/mca/mtl/portals4/configure.m4
+++ b/ompi/mca/mtl/portals4/configure.m4
@@ -40,7 +40,7 @@ AC_DEFUN([MCA_ompi_mtl_portals4_CONFIG],[
           [$1],
           [$2])
 
-    # need to propogate CPPFLAGS to all of OMPI
+    # need to propagate CPPFLAGS to all of OMPI
     AS_IF([test "$DIRECT_mtl" = "portals4"],
           [mtl_portals4_WRAPPER_EXTRA_CPPFLAGS="$mtl_portals4_CPPFLAGS"
            CPPFLAGS="$CPPFLAGS $mtl_portals4_CPPFLAGS"])
diff --git a/ompi/mca/mtl/portals4/mtl_portals4.h b/ompi/mca/mtl/portals4/mtl_portals4.h
index 70db436b917..2729a383543 100644
--- a/ompi/mca/mtl/portals4/mtl_portals4.h
+++ b/ompi/mca/mtl/portals4/mtl_portals4.h
@@ -20,19 +20,21 @@
 #ifndef MTL_PORTALS_H_HAS_BEEN_INCLUDED
 #define MTL_PORTALS_H_HAS_BEEN_INCLUDED
 
-#include <portals4.h>
+#include "ompi_config.h"
 
-#include "opal/include/opal_config.h"
 #include "opal/class/opal_free_list.h"
 #include "opal/class/opal_list.h"
 #include "opal/datatype/opal_convertor.h"
-#include "ompi/proc/proc.h"
-#include "ompi/mca/mtl/mtl.h"
-#include "ompi/mca/mtl/base/base.h"
 
 #include "ompi/communicator/communicator.h"
+#include "ompi/mca/mtl/base/base.h"
+#include "ompi/mca/mtl/mtl.h"
+#include "ompi/proc/proc.h"
 
 #include "mtl_portals4_flowctl.h"
+#include "mtl_portals4_request.h"
+
+#include <portals4.h>
 
 BEGIN_C_DECLS
 
@@ -44,7 +46,7 @@ struct mca_mtl_portals4_module_t {
     /* add_procs() can get called multiple times.  this prevents multiple calls to portals4_init_interface(). */
     int32_t need_init;
 
-    /* Use the logical to physical table to accelerate portals4 adressing: 1 (true) : 0 (false) */
+    /* Use the logical to physical table to accelerate portals4 addressing: 1 (true) : 0 (false) */
     int32_t use_logical;
 
     /* Process_id */
diff --git a/ompi/mca/mtl/portals4/mtl_portals4_cancel.c b/ompi/mca/mtl/portals4/mtl_portals4_cancel.c
index 1e2e5a43ddf..8994676046b 100644
--- a/ompi/mca/mtl/portals4/mtl_portals4_cancel.c
+++ b/ompi/mca/mtl/portals4/mtl_portals4_cancel.c
@@ -45,7 +45,7 @@ ompi_mtl_portals4_cancel(struct mca_mtl_base_module_t* mtl,
                receive completion event... */
             ompi_mtl_portals4_progress();
 
-            if (PTL_INVALID_HANDLE != recvreq->me_h) {
+            if (!PtlHandleIsEqual(recvreq->me_h, PTL_INVALID_HANDLE)) {
                 ret = PtlMEUnlink(recvreq->me_h);
                 if (PTL_OK == ret) {
                     recvreq->super.super.ompi_req->req_status._cancelled = true;
diff --git a/ompi/mca/mtl/portals4/mtl_portals4_component.c b/ompi/mca/mtl/portals4/mtl_portals4_component.c
index 132358b5638..e9610940e44 100644
--- a/ompi/mca/mtl/portals4/mtl_portals4_component.c
+++ b/ompi/mca/mtl/portals4/mtl_portals4_component.c
@@ -43,9 +43,10 @@ static int ompi_mtl_portals4_component_close(void);
 static int ompi_mtl_portals4_component_query(mca_base_module_t **module, int *priority);
 static mca_mtl_base_module_t*
 ompi_mtl_portals4_component_init(bool enable_progress_threads,
-                                 bool enable_mpi_threads);
+                                 bool enable_mpi_threads,
+                                 bool *accelerator_support);
 
-OMPI_MODULE_DECLSPEC extern mca_mtl_base_component_2_0_0_t mca_mtl_portals4_component;
+OMPI_DECLSPEC extern mca_mtl_base_component_2_0_0_t mca_mtl_portals4_component;
 
 mca_mtl_base_component_2_0_0_t mca_mtl_portals4_component = {
 
@@ -90,7 +91,7 @@ ompi_mtl_portals4_component_register(void)
     ompi_mtl_portals4.use_logical = 0;
     (void) mca_base_component_var_register(&mca_mtl_portals4_component.mtl_version,
                                            "use_logical",
-                                           "Use the logical to physical table to accelerate portals4 adressing: 1 (true) : 0 (false)",
+                                           "Use the logical to physical table to accelerate portals4 addressing: 1 (true) : 0 (false)",
                                            MCA_BASE_VAR_TYPE_INT,
                                            NULL,
                                            0,
@@ -229,36 +230,6 @@ ompi_mtl_portals4_component_open(void)
         sizeof(ompi_mtl_portals4_request_t) -
         sizeof(struct mca_mtl_request_t);
 
-    opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
-                        "Flow control: "
-#if OMPI_MTL_PORTALS4_FLOW_CONTROL
-                        "yes"
-#else
-                        "no"
-#endif
-                        );
-    opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
-                        "Max message size: %lu", (unsigned long)
-                        ompi_mtl_portals4.max_msg_size_mtl);
-    opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
-                        "Short limit: %d", (int)
-                        ompi_mtl_portals4.short_limit);
-    opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
-                        "Eager limit: %d", (int)
-                        ompi_mtl_portals4.eager_limit);
-    opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
-                        "Short receive blocks: %d",
-                        ompi_mtl_portals4.recv_short_num);
-    opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
-                        "Send queue size: %d", ompi_mtl_portals4.send_queue_size);
-    opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
-                        "Recv queue size: %d", ompi_mtl_portals4.recv_queue_size);
-    opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
-                        "Long protocol: %s",
-                        (ompi_mtl_portals4.protocol == eager) ? "Eager" :
-                        (ompi_mtl_portals4.protocol == rndv) ? "Rendezvous" :
-                         "Other");
-
     OBJ_CONSTRUCT(&ompi_mtl_portals4.fl_message, opal_free_list_t);
     opal_free_list_init(&ompi_mtl_portals4.fl_message,
                         sizeof(ompi_mtl_portals4_message_t) +
@@ -291,6 +262,31 @@ ompi_mtl_portals4_component_open(void)
     ompi_mtl_portals4.use_flowctl=0;
 #endif
 
+    opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
+                        "Flow control: %s", 
+                        ompi_mtl_portals4.use_flowctl ? "yes" : "no");
+    opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
+                        "Max message size: %lu", (unsigned long)
+                        ompi_mtl_portals4.max_msg_size_mtl);
+    opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
+                        "Short limit: %d", (int)
+                        ompi_mtl_portals4.short_limit);
+    opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
+                        "Eager limit: %d", (int)
+                        ompi_mtl_portals4.eager_limit);
+    opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
+                        "Short receive blocks: %d",
+                        ompi_mtl_portals4.recv_short_num);
+    opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
+                        "Send queue size: %d", ompi_mtl_portals4.send_queue_size);
+    opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
+                        "Recv queue size: %d", ompi_mtl_portals4.recv_queue_size);
+    opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
+                        "Long protocol: %s",
+                        (ompi_mtl_portals4.protocol == eager) ? "Eager" :
+                        (ompi_mtl_portals4.protocol == rndv) ? "Rendezvous" :
+                         "Other");
+
     return OMPI_SUCCESS;
 }
 
@@ -327,7 +323,8 @@ ompi_mtl_portals4_component_close(void)
 
 static mca_mtl_base_module_t*
 ompi_mtl_portals4_component_init(bool enable_progress_threads,
-                                 bool enable_mpi_threads)
+                                 bool enable_mpi_threads,
+                                 bool *accelerator_support)
 {
     int ret;
     ptl_process_t id;
@@ -429,10 +426,8 @@ ompi_mtl_portals4_component_init(bool enable_progress_threads,
 
     ompi_mtl_portals4.base.mtl_max_tag = MTL_PORTALS4_MAX_TAG;
 
-    /* Disable opal from checking if buffer being sent is cuda */
-#if OPAL_CUDA_SUPPORT
-    ompi_mtl_portals4.base.mtl_flags |= MCA_MTL_BASE_FLAG_CUDA_INIT_DISABLE;
-#endif /* OPAL_CUDA_SUPPORT */
+    /* Disable opal from checking if buffer being sent is device */
+    ompi_mtl_portals4.base.mtl_flags |= MCA_MTL_BASE_FLAG_ACCELERATOR_INIT_DISABLE;
 
     return &ompi_mtl_portals4.base;
 
diff --git a/ompi/mca/mtl/portals4/mtl_portals4_endpoint.h b/ompi/mca/mtl/portals4/mtl_portals4_endpoint.h
index 5a352da7633..975c2e520db 100644
--- a/ompi/mca/mtl/portals4/mtl_portals4_endpoint.h
+++ b/ompi/mca/mtl/portals4/mtl_portals4_endpoint.h
@@ -20,8 +20,11 @@
 #ifndef OMPI_MTL_PORTALS_ENDPOINT_H
 #define OMPI_MTL_PORTALS_ENDPOINT_H
 
+#include "ompi_config.h"
+
 #include "ompi/mca/pml/pml.h"
-#include "ompi/mca/mtl/portals4/mtl_portals4.h"
+
+#include "mtl_portals4.h"
 
 struct mca_mtl_base_endpoint_t {
     ptl_process_t ptl_proc;
diff --git a/ompi/mca/mtl/portals4/mtl_portals4_flowctl.h b/ompi/mca/mtl/portals4/mtl_portals4_flowctl.h
index ee1fbd9a760..bdf0eb6c174 100644
--- a/ompi/mca/mtl/portals4/mtl_portals4_flowctl.h
+++ b/ompi/mca/mtl/portals4/mtl_portals4_flowctl.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012      Sandia National Laboratories.  All rights reserved.
+ * Copyright (c) 2012-2022 Sandia National Laboratories.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -10,10 +10,15 @@
 #ifndef MTL_PORTALS_FLOWCTL_H
 #define MTL_PORTALS_FLOWCTL_H
 
+#include "ompi_config.h"
+
 #include "opal/class/opal_free_list.h"
 
+#include "mtl_portals4.h"
 #include "mtl_portals4_request.h"
 
+#include <portals4.h>
+
 struct mca_mtl_base_endpoint_t;
 struct ompi_mtl_portals4_isend_request_t;
 
@@ -22,7 +27,7 @@ struct ompi_mtl_portals4_pending_request_t {
     mca_pml_base_send_mode_t mode;
     void *start;
     size_t length;
-    int contextid;
+    uint32_t contextid;
     int tag;
     int my_rank;
     int fc_notified;
@@ -34,7 +39,7 @@ OBJ_CLASS_DECLARATION(ompi_mtl_portals4_pending_request_t);
 
 
 struct ompi_mtl_portals4_flowctl_t {
-    int32_t flowctl_active;
+    opal_atomic_int32_t flowctl_active;
 
     opal_atomic_int32_t send_slots;
     int32_t max_send_slots;
diff --git a/ompi/mca/mtl/portals4/mtl_portals4_message.h b/ompi/mca/mtl/portals4/mtl_portals4_message.h
index db16755c2ef..ff73bbb72d0 100644
--- a/ompi/mca/mtl/portals4/mtl_portals4_message.h
+++ b/ompi/mca/mtl/portals4/mtl_portals4_message.h
@@ -13,6 +13,10 @@
 #ifndef MTL_PORTALS4_MESSAGE_H
 #define MTL_PORTALS4_MESSAGE_H
 
+#include "ompi_config.h"
+
+#include "mtl_portals4.h"
+
 struct ompi_mtl_portals4_message_t {
     opal_free_list_item_t super;
     ptl_event_t ev;
diff --git a/ompi/mca/mtl/portals4/mtl_portals4_probe.c b/ompi/mca/mtl/portals4/mtl_portals4_probe.c
index 5f2a991cfe7..127d2c86b6c 100644
--- a/ompi/mca/mtl/portals4/mtl_portals4_probe.c
+++ b/ompi/mca/mtl/portals4/mtl_portals4_probe.c
@@ -2,14 +2,14 @@
  * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2010 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2010-2012 Sandia National Laboratories.  All rights reserved.
+ * Copyright (c) 2010-2022 Sandia National Laboratories.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -82,7 +82,8 @@ ompi_mtl_portals4_iprobe(struct mca_mtl_base_module_t* mtl,
         remote_proc = *((ptl_process_t*) ompi_mtl_portals4_get_endpoint (mtl, ompi_proc));
     }
 
-    MTL_PORTALS4_SET_RECV_BITS(match_bits, ignore_bits, comm->c_contextid,
+    MTL_PORTALS4_SET_RECV_BITS(match_bits, ignore_bits,
+                               ompi_comm_get_local_cid(comm),
                                src, tag);
 
     me.start = NULL;
@@ -120,7 +121,9 @@ ompi_mtl_portals4_iprobe(struct mca_mtl_base_module_t* mtl,
 
     *flag = request.found_match;
     if (1 == *flag) {
-        *status = request.status;
+        if (MPI_STATUS_IGNORE != status) {
+            OMPI_COPY_STATUS(status, request.status, false);
+        }
     }
 
     return OMPI_SUCCESS;
@@ -144,7 +147,7 @@ ompi_mtl_portals4_improbe(struct mca_mtl_base_module_t *mtl,
 
     opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                         "%s:%d: improbe %d %d %d",
-                        __FILE__, __LINE__, comm->c_contextid, src, tag);
+                        __FILE__, __LINE__, ompi_comm_get_local_cid(comm), src, tag);
 
     if  (MPI_ANY_SOURCE == src) {
         if (ompi_mtl_portals4.use_logical) {
@@ -160,7 +163,8 @@ ompi_mtl_portals4_improbe(struct mca_mtl_base_module_t *mtl,
         remote_proc = *((ptl_process_t*) ompi_mtl_portals4_get_endpoint (mtl, ompi_proc));
     }
 
-    MTL_PORTALS4_SET_RECV_BITS(match_bits, ignore_bits, comm->c_contextid,
+    MTL_PORTALS4_SET_RECV_BITS(match_bits, ignore_bits,
+                               ompi_comm_get_local_cid(comm),
                                src, tag);
 
     me.start = NULL;
@@ -198,7 +202,9 @@ ompi_mtl_portals4_improbe(struct mca_mtl_base_module_t *mtl,
 
     *matched = request.found_match;
     if (1 == *matched) {
-        *status = request.status;
+        if (MPI_STATUS_IGNORE != status) {
+            OMPI_COPY_STATUS(status, request.status, false);
+        }
 
         (*message) = ompi_message_alloc();
         if (NULL == (*message)) {
diff --git a/ompi/mca/mtl/portals4/mtl_portals4_recv.c b/ompi/mca/mtl/portals4/mtl_portals4_recv.c
index f2737428e26..21c1de5bb3b 100644
--- a/ompi/mca/mtl/portals4/mtl_portals4_recv.c
+++ b/ompi/mca/mtl/portals4/mtl_portals4_recv.c
@@ -9,7 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2010-2012 Sandia National Laboratories.  All rights reserved.
+ * Copyright (c) 2010-2022 Sandia National Laboratories.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -49,7 +49,7 @@ read_msg(void *start, ptl_size_t length, ptl_process_t target,
          ompi_mtl_portals4_recv_request_t *request)
 {
     int ret, i;
-    ptl_size_t rest = length, asked = 0;
+    ptl_size_t rest = length;
     int32_t frag_count;
 
 #if OMPI_MTL_PORTALS4_FLOW_CONTROL
@@ -102,7 +102,6 @@ read_msg(void *start, ptl_size_t length, ptl_process_t target,
             return OMPI_ERR_OUT_OF_RESOURCE;
         }
         rest -= frag->frag_length;
-        asked += frag->frag_length;
     }
 
     return OMPI_SUCCESS;
@@ -457,7 +456,8 @@ ompi_mtl_portals4_irecv(struct mca_mtl_base_module_t* mtl,
         remote_proc = *((ptl_process_t*) ompi_mtl_portals4_get_endpoint (mtl, ompi_proc));
     }
 
-    MTL_PORTALS4_SET_RECV_BITS(match_bits, ignore_bits, comm->c_contextid,
+    MTL_PORTALS4_SET_RECV_BITS(match_bits, ignore_bits,
+                               ompi_comm_get_local_cid(comm),
                                src, tag);
 
     ret = ompi_mtl_datatype_recv_buf(convertor, &start, &length, &free_after);
@@ -468,7 +468,7 @@ ompi_mtl_portals4_irecv(struct mca_mtl_base_module_t* mtl,
     ptl_request->super.type = portals4_req_recv;
     ptl_request->super.event_callback = ompi_mtl_portals4_recv_progress;
 #if OPAL_ENABLE_DEBUG
-    ptl_request->opcount = OPAL_THREAD_ADD_FETCH64((int64_t*) &ompi_mtl_portals4.recv_opcount, 1);
+    ptl_request->opcount = OPAL_THREAD_ADD_FETCH64((opal_atomic_int64_t*) &ompi_mtl_portals4.recv_opcount, 1);
     ptl_request->hdr_data = 0;
 #endif
     ptl_request->buffer_ptr = (free_after) ? start : NULL;
@@ -549,7 +549,7 @@ ompi_mtl_portals4_imrecv(struct mca_mtl_base_module_t* mtl,
     }
 
 #if OPAL_ENABLE_DEBUG
-    ptl_request->opcount = OPAL_THREAD_ADD_FETCH64((int64_t*) &ompi_mtl_portals4.recv_opcount, 1);
+    ptl_request->opcount = OPAL_THREAD_ADD_FETCH64((opal_atomic_int64_t*) &ompi_mtl_portals4.recv_opcount, 1);
     ptl_request->hdr_data = 0;
 #endif
     ptl_request->super.type = portals4_req_recv;
diff --git a/ompi/mca/mtl/portals4/mtl_portals4_recv_short.c b/ompi/mca/mtl/portals4/mtl_portals4_recv_short.c
index 23cd049022b..d712338b498 100644
--- a/ompi/mca/mtl/portals4/mtl_portals4_recv_short.c
+++ b/ompi/mca/mtl/portals4/mtl_portals4_recv_short.c
@@ -162,7 +162,7 @@ ompi_mtl_portals4_recv_short_block_alloc(bool release_on_free)
 static int
 ompi_mtl_portals4_recv_short_block_free(ompi_mtl_portals4_recv_short_block_t *block)
 {
-    if (PTL_INVALID_HANDLE != block->me_h) {
+    if (!PtlHandleIsEqual(block->me_h, PTL_INVALID_HANDLE)) {
         PtlMEUnlink(block->me_h);
         block->me_h = PTL_INVALID_HANDLE;
     }
diff --git a/ompi/mca/mtl/portals4/mtl_portals4_recv_short.h b/ompi/mca/mtl/portals4/mtl_portals4_recv_short.h
index 5aab083bedb..bcc9f9c4743 100644
--- a/ompi/mca/mtl/portals4/mtl_portals4_recv_short.h
+++ b/ompi/mca/mtl/portals4/mtl_portals4_recv_short.h
@@ -20,7 +20,9 @@
 #ifndef OMPI_MTL_PORTALS_RECV_SHORT_H
 #define OMPI_MTL_PORTALS_RECV_SHORT_H
 
-#include "mtl_portals4_request.h"
+#include "ompi_config.h"
+
+#include "mtl_portals4.h"
 
 struct ompi_mtl_portals4_recv_short_block_t {
     opal_list_item_t base;
diff --git a/ompi/mca/mtl/portals4/mtl_portals4_request.h b/ompi/mca/mtl/portals4/mtl_portals4_request.h
index ede07b4e500..5644bd9192a 100644
--- a/ompi/mca/mtl/portals4/mtl_portals4_request.h
+++ b/ompi/mca/mtl/portals4/mtl_portals4_request.h
@@ -20,10 +20,17 @@
 #ifndef OMPI_MTL_PORTALS_REQUEST_H
 #define OMPI_MTL_PORTALS_REQUEST_H
 
+#include "ompi_config.h"
+
 #include "opal/datatype/opal_convertor.h"
-#include "ompi/mca/mtl/mtl.h"
 #include "opal/mca/timer/base/base.h"
 
+#include "ompi/mca/mtl/mtl.h"
+
+#include "mtl_portals4.h"
+
+#include <portals4.h>
+
 struct ompi_mtl_portals4_message_t;
 struct ompi_mtl_portals4_pending_request_t;
 
@@ -55,7 +62,7 @@ struct ompi_mtl_portals4_isend_request_t {
 #endif
     ptl_size_t length;
     opal_atomic_int32_t pending_get;
-    opal_atomic_uint32_t event_count;
+    opal_atomic_int32_t event_count;
 };
 typedef struct ompi_mtl_portals4_isend_request_t ompi_mtl_portals4_isend_request_t;
 
diff --git a/ompi/mca/mtl/portals4/mtl_portals4_send.c b/ompi/mca/mtl/portals4/mtl_portals4_send.c
index 27291eed559..69a741da127 100644
--- a/ompi/mca/mtl/portals4/mtl_portals4_send.c
+++ b/ompi/mca/mtl/portals4/mtl_portals4_send.c
@@ -10,7 +10,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2010      Sandia National Laboratories.  All rights reserved.
+ * Copyright (c) 2010-2022 Sandia National Laboratories.  All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * $COPYRIGHT$
@@ -40,7 +40,7 @@ ompi_mtl_portals4_callback(ptl_event_t *ev,
                            ompi_mtl_portals4_base_request_t* ptl_base_request,
                            bool *complete)
 {
-    int retval = OMPI_SUCCESS, ret, val, add = 1;
+    int retval = OMPI_SUCCESS, ret = 0, val = 0, add = 1;
     ompi_mtl_portals4_isend_request_t* ptl_request =
         (ompi_mtl_portals4_isend_request_t*) ptl_base_request;
 
@@ -114,7 +114,7 @@ ompi_mtl_portals4_callback(ptl_event_t *ev,
                          ptl_request->opcount, ev->type));
 
     /* First put achieved successfully (In the Priority List), so it may be necessary to decrement the number of pending get
-     * If the protocol is eager, just decrement pending_get 
+     * If the protocol is eager, just decrement pending_get
      * Else (the protocol is rndv), decrement pending_get only if length % max_msg_size <= eager_limit
      * (This is the case where the eager part allows to save one get)
      */
@@ -146,7 +146,7 @@ ompi_mtl_portals4_callback(ptl_event_t *ev,
         (!PtlHandleIsEqual(ptl_request->me_h, PTL_INVALID_HANDLE))) {
         /* long expected messages with the eager protocol
            (and also with the rndv protocol if the length
-           is less or egal to eager_limit) won't see a
+           is less or equal to eager_limit) won't see a
            get event to complete the message.  Give them an extra
            count to cause the message to complete with just the SEND
            and ACK events and remove the ME. (we wait for the counter
@@ -161,7 +161,7 @@ ompi_mtl_portals4_callback(ptl_event_t *ev,
         ptl_request->me_h = PTL_INVALID_HANDLE;
         add++;
     }
-    val = OPAL_THREAD_ADD_FETCH32((int32_t*)&ptl_request->event_count, add);
+    val = OPAL_THREAD_ADD_FETCH32(&ptl_request->event_count, add);
     assert(val <= 3);
 
     if (val == 3) {
@@ -229,7 +229,7 @@ ompi_mtl_portals4_isend_callback(ptl_event_t *ev,
 
 static inline int
 ompi_mtl_portals4_short_isend(mca_pml_base_send_mode_t mode,
-                              void *start, int length, int contextid, int tag,
+                              void *start, int length, uint32_t contextid, int tag,
                               int localrank,
                               ptl_process_t ptl_proc,
                               ompi_mtl_portals4_isend_request_t *ptl_request)
@@ -239,7 +239,9 @@ ompi_mtl_portals4_short_isend(mca_pml_base_send_mode_t mode,
     ptl_me_t me;
     ptl_hdr_data_t hdr_data;
 
-    MTL_PORTALS4_SET_SEND_BITS(match_bits, contextid, localrank, tag,
+    MTL_PORTALS4_SET_SEND_BITS(match_bits,
+                               contextid,
+                               localrank, tag,
                                MTL_PORTALS4_SHORT_MSG);
 
     MTL_PORTALS4_SET_HDR_DATA(hdr_data, ptl_request->opcount, length,
@@ -316,7 +318,7 @@ ompi_mtl_portals4_short_isend(mca_pml_base_send_mode_t mode,
 }
 
 static inline int
-ompi_mtl_portals4_long_isend(void *start, size_t length, int contextid, int tag,
+ompi_mtl_portals4_long_isend(void *start, size_t length, uint32_t contextid, int tag,
                              int localrank,
                              ptl_process_t ptl_proc,
                              ompi_mtl_portals4_isend_request_t *ptl_request)
@@ -327,7 +329,9 @@ ompi_mtl_portals4_long_isend(void *start, size_t length, int contextid, int tag,
     ptl_hdr_data_t hdr_data;
     ptl_size_t put_length;
 
-    MTL_PORTALS4_SET_SEND_BITS(match_bits, contextid, localrank, tag,
+    MTL_PORTALS4_SET_SEND_BITS(match_bits,
+                               contextid,
+                               localrank, tag,
                                MTL_PORTALS4_LONG_MSG);
 
     MTL_PORTALS4_SET_HDR_DATA(hdr_data, ptl_request->opcount, length, 0);
@@ -381,7 +385,7 @@ ompi_mtl_portals4_long_isend(void *start, size_t length, int contextid, int tag,
     }
 
     /* We have to wait for some GET events.
-       If the first put falls in overflow list, the number of GET event is egal to:
+       If the first put falls in overflow list, the number of GET event is equal to:
            (length - 1) / ompi_mtl_portals4.max_msg_size_mtl + 1
        else we will re-calculate this number when we received the first ACK event (with remote overflow list)
      */
@@ -414,7 +418,7 @@ ompi_mtl_portals4_long_isend(void *start, size_t length, int contextid, int tag,
 
 #if OMPI_MTL_PORTALS4_FLOW_CONTROL
 void
-ompi_mtl_portals4_pending_list_progress()
+ompi_mtl_portals4_pending_list_progress(void)
 {
     int ret, val;
     opal_list_item_t *item;
@@ -492,7 +496,7 @@ ompi_mtl_portals4_send_start(struct mca_mtl_base_module_t* mtl,
     ret = ompi_mtl_datatype_pack(convertor, &start, &length, &free_after);
     if (OMPI_SUCCESS != ret) return ret;
 
-    ptl_request->opcount = OPAL_THREAD_ADD_FETCH64((int64_t*)&ompi_mtl_portals4.opcount, 1);
+    ptl_request->opcount = OPAL_THREAD_ADD_FETCH64((opal_atomic_int64_t*)&ompi_mtl_portals4.opcount, 1);
     ptl_request->buffer_ptr = (free_after) ? start : NULL;
     ptl_request->length = length;
     ptl_request->event_count = 0;
@@ -513,7 +517,7 @@ ompi_mtl_portals4_send_start(struct mca_mtl_base_module_t* mtl,
     pending->mode = mode;
     pending->start = start;
     pending->length = length;
-    pending->contextid = comm->c_contextid;
+    pending->contextid = ompi_comm_get_local_cid(comm);
     pending->tag = tag;
     pending->my_rank = comm->c_my_rank;
     pending->fc_notified = 0;
@@ -546,7 +550,7 @@ ompi_mtl_portals4_send_start(struct mca_mtl_base_module_t* mtl,
         ret = ompi_mtl_portals4_short_isend(mode,
                                             start,
                                             length,
-                                            comm->c_contextid,
+                                            ompi_comm_get_local_cid(comm),
                                             tag,
                                             comm->c_my_rank,
                                             ptl_proc,
@@ -554,7 +558,7 @@ ompi_mtl_portals4_send_start(struct mca_mtl_base_module_t* mtl,
     } else {
         ret = ompi_mtl_portals4_long_isend(start,
                                            length,
-                                           comm->c_contextid,
+                                           ompi_comm_get_local_cid(comm),
                                            tag,
                                            comm->c_my_rank,
                                            ptl_proc,
diff --git a/ompi/mca/mtl/psm2/Makefile.am b/ompi/mca/mtl/psm2/Makefile.am
index 741c65e4638..8b889bf726a 100644
--- a/ompi/mca/mtl/psm2/Makefile.am
+++ b/ompi/mca/mtl/psm2/Makefile.am
@@ -13,8 +13,9 @@
 # Copyright (c) 2015      Intel, Inc. All rights reserved
 # Copyright (c) 2017      Los Alamos National Security, LLC.
 #                         All rights reserved.
-#   
+#
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -22,11 +23,12 @@
 # $HEADER$
 #
 
-EXTRA_DIST = post_configure.sh
+EXTRA_DIST = \
+        post_configure.sh \
+        help-mtl-psm2.txt
 
 AM_CPPFLAGS = $(mtl_psm2_CPPFLAGS)
 
-dist_ompidata_DATA = help-mtl-psm2.txt
 
 mtl_psm2_sources = \
 	       mtl_psm2.c \
diff --git a/ompi/mca/mtl/psm2/configure.m4 b/ompi/mca/mtl/psm2/configure.m4
index c72c5fd03f6..598874321ca 100644
--- a/ompi/mca/mtl/psm2/configure.m4
+++ b/ompi/mca/mtl/psm2/configure.m4
@@ -13,6 +13,7 @@
 # Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2013      Sandia National Laboratories.  All rights reserved.
 # Copyright (c) 2014      Intel Corporation. All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -42,7 +43,6 @@ AC_DEFUN([MCA_ompi_mtl_psm2_CONFIG],[
           [$2])
 
     # substitute in the things needed to build psm2
-    AC_SUBST([mtl_psm2_CFLAGS])
     AC_SUBST([mtl_psm2_CPPFLAGS])
     AC_SUBST([mtl_psm2_LDFLAGS])
     AC_SUBST([mtl_psm2_LIBS])
diff --git a/ompi/mca/mtl/psm2/help-mtl-psm2.txt b/ompi/mca/mtl/psm2/help-mtl-psm2.txt
index c1878641af2..b8b8b7e43b5 100644
--- a/ompi/mca/mtl/psm2/help-mtl-psm2.txt
+++ b/ompi/mca/mtl/psm2/help-mtl-psm2.txt
@@ -2,6 +2,7 @@
 #
 # Copyright (C) 2009. QLogic Corporation.  All rights reserved.
 # Copyright (c) 2013-2017 Intel, Inc. All rights reserved.
+# Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -13,11 +14,6 @@ Initialization of PSM2 library failed.
 
   Error: %s
 #
-[debug level]
-Unable to set PSM2 debug level.
-
-  Error: %s
-#
 [unable to open endpoint]
 PSM2 was unable to open an endpoint. Please make sure that the network link is
 active on the node and the hardware is functioning.
@@ -40,9 +36,6 @@ Unable to post application receive buffer (psm2_mq_irecv or psm2_mq_imrecv).
   Buffer:     %p
   Length:     %llu
 #
-[path query mechanism unknown]
-Unknown path record query mechanism %s. Supported mechanisms are %s.
-#
 [message too big]
 Message size %llu bigger than supported by PSM2 API. Max = %llu
 #
diff --git a/ompi/mca/mtl/psm2/mtl_psm2.c b/ompi/mca/mtl/psm2/mtl_psm2.c
index fab0034a2d5..e6f222321fe 100644
--- a/ompi/mca/mtl/psm2/mtl_psm2.c
+++ b/ompi/mca/mtl/psm2/mtl_psm2.c
@@ -31,6 +31,7 @@
 #include "ompi/mca/mtl/mtl.h"
 #include "ompi/mca/mtl/base/mtl_base_datatype.h"
 #include "opal/util/show_help.h"
+#include "opal/util/minmax.h"
 #include "ompi/proc/proc.h"
 
 #include "mtl_psm2.h"
@@ -191,9 +192,7 @@ int ompi_mtl_psm2_module_init(int local_rank, int num_local_procs) {
     /* register the psm2 progress function */
     opal_progress_register(ompi_mtl_psm2_progress);
 
-#if OPAL_CUDA_SUPPORT
-    ompi_mtl_psm2.super.mtl_flags |= MCA_MTL_BASE_FLAG_CUDA_INIT_DISABLE;
-#endif
+    ompi_mtl_psm2.super.mtl_flags |= MCA_MTL_BASE_FLAG_ACCELERATOR_INIT_DISABLE;
 
     return OMPI_SUCCESS;
 }
diff --git a/ompi/mca/mtl/psm2/mtl_psm2_component.c b/ompi/mca/mtl/psm2/mtl_psm2_component.c
index f4d73fbc032..e6192e05169 100644
--- a/ompi/mca/mtl/psm2/mtl_psm2_component.c
+++ b/ompi/mca/mtl/psm2/mtl_psm2_component.c
@@ -17,6 +17,8 @@
  * Copyright (c) 2018      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2018-2020 Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2021      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -57,8 +59,9 @@ static int ompi_mtl_psm2_component_close(void);
 static int ompi_mtl_psm2_component_query(mca_base_module_t **module, int *priority);
 static int ompi_mtl_psm2_component_register(void);
 
-static mca_mtl_base_module_t* ompi_mtl_psm2_component_init( bool enable_progress_threads,
-                                                          bool enable_mpi_threads );
+static mca_mtl_base_module_t* ompi_mtl_psm2_component_init(bool enable_progress_threads,
+                                                           bool enable_mpi_threads,
+                                                           bool *accelerator_support);
 
 mca_mtl_psm2_component_t mca_mtl_psm2_component = {
 
@@ -153,8 +156,8 @@ static void ompi_mtl_psm2_set_shadow_env (struct ompi_mtl_psm2_shadow_variable *
         break;
     case MCA_BASE_VAR_TYPE_UNSIGNED_LONG:
         if (0 == strcmp (variable->env_name, "PSM2_TRACEMASK")) {
-            /* PSM2 documentation shows the tracemask as a hexidecimal number. to be consitent
-             * use hexidecimal here. */
+            /* PSM2 documentation shows the tracemask as a hexadecimal number. to be consistent
+             * use hexadecimal here. */
             ret = opal_asprintf (&env_value, "%s=0x%lx", variable->env_name, storage->ulval);
         } else {
             ret = opal_asprintf (&env_value, "%s=%lu", variable->env_name, storage->ulval);
@@ -324,16 +327,21 @@ static int
 ompi_mtl_psm2_component_query(mca_base_module_t **module, int *priority)
 {
 
+#if HAVE_PSM2_LIB_REFCOUNT_CAP
     /*
      * Mixing the PSM2 MTL with the OFI BTL (using PSM2) 
      * can cause an issue when they both call psm2_finalize
      * in older versions of libpsm2.
+     * 
+     * An installer may know what they are doing and disabled
+     * checking psm2 version, hence making this code conditional.
      */
     if (!psm2_get_capability_mask(PSM2_LIB_REFCOUNT_CAP)) {
         opal_output_verbose(2, ompi_mtl_base_framework.framework_output, 
             "This version of the PSM2 MTL needs version 11.2.173 or later of the libpsm2 library for correct operation.\n");
         return OMPI_ERR_FATAL;
     }   
+#endif
 
     /*
      * if we get here it means that PSM2 is available so give high priority
@@ -369,7 +377,8 @@ get_local_rank(int *out_rank)
 
 static mca_mtl_base_module_t *
 ompi_mtl_psm2_component_init(bool enable_progress_threads,
-                            bool enable_mpi_threads)
+                             bool enable_mpi_threads,
+                             bool *accelerator_support)
 {
     psm2_error_t	err;
     int	verno_major = PSM2_VERNO_MAJOR;
diff --git a/ompi/mca/mtl/psm2/mtl_psm2_probe.c b/ompi/mca/mtl/psm2/mtl_psm2_probe.c
index b81317507be..c5b7f7ab93a 100644
--- a/ompi/mca/mtl/psm2/mtl_psm2_probe.c
+++ b/ompi/mca/mtl/psm2/mtl_psm2_probe.c
@@ -39,7 +39,7 @@ int ompi_mtl_psm2_iprobe(struct mca_mtl_base_module_t* mtl,
     psm2_mq_status2_t mqstat;
     psm2_error_t err;
 
-    PSM2_MAKE_TAGSEL(src, tag, comm->c_contextid, mqtag, tagsel);
+    PSM2_MAKE_TAGSEL(src, tag, comm->c_index, mqtag, tagsel);
 
     err = psm2_mq_iprobe2(ompi_mtl_psm2.mq,
             PSM2_MQ_ANY_ADDR, &mqtag, &tagsel, &mqstat);
@@ -88,7 +88,7 @@ ompi_mtl_psm2_improbe(struct mca_mtl_base_module_t *mtl,
     psm2_mq_req_t mqreq;
     psm2_error_t err;
 
-    PSM2_MAKE_TAGSEL(src, tag, comm->c_contextid, mqtag, tagsel);
+    PSM2_MAKE_TAGSEL(src, tag, comm->c_index, mqtag, tagsel);
 
     err = psm2_mq_improbe2(ompi_mtl_psm2.mq,
             PSM2_MQ_ANY_ADDR, &mqtag, &tagsel, &mqreq, &mqstat);
diff --git a/ompi/mca/mtl/psm2/mtl_psm2_recv.c b/ompi/mca/mtl/psm2/mtl_psm2_recv.c
index ff5c54067ce..83fdfcfec81 100644
--- a/ompi/mca/mtl/psm2/mtl_psm2_recv.c
+++ b/ompi/mca/mtl/psm2/mtl_psm2_recv.c
@@ -63,7 +63,7 @@ ompi_mtl_psm2_irecv(struct mca_mtl_base_module_t* mtl,
     mtl_psm2_request->convertor = convertor;
     mtl_psm2_request->type = OMPI_mtl_psm2_IRECV;
 
-    PSM2_MAKE_TAGSEL(src, tag, comm->c_contextid, mqtag, tagsel);
+    PSM2_MAKE_TAGSEL(src, tag, comm->c_index, mqtag, tagsel);
 
     err = psm2_mq_irecv2(ompi_mtl_psm2.mq,
 		       PSM2_MQ_ANY_ADDR,
diff --git a/ompi/mca/mtl/psm2/mtl_psm2_send.c b/ompi/mca/mtl/psm2/mtl_psm2_send.c
index 6acb30cf6d2..59742ace546 100644
--- a/ompi/mca/mtl/psm2/mtl_psm2_send.c
+++ b/ompi/mca/mtl/psm2/mtl_psm2_send.c
@@ -48,7 +48,7 @@ ompi_mtl_psm2_send(struct mca_mtl_base_module_t* mtl,
 
     assert(mtl == &ompi_mtl_psm2.super);
 
-    PSM2_MAKE_MQTAG(comm->c_contextid, comm->c_my_rank, tag, mqtag);
+    PSM2_MAKE_MQTAG(comm->c_index, comm->c_my_rank, tag, mqtag);
 
     ret = ompi_mtl_datatype_pack(convertor,
                                  &mtl_psm2_request.buf,
@@ -106,7 +106,7 @@ ompi_mtl_psm2_isend(struct mca_mtl_base_module_t* mtl,
 
     assert(mtl == &ompi_mtl_psm2.super);
 
-    PSM2_MAKE_MQTAG(comm->c_contextid, comm->c_my_rank, tag, mqtag);
+    PSM2_MAKE_MQTAG(comm->c_index, comm->c_my_rank, tag, mqtag);
 
 
     ret = ompi_mtl_datatype_pack(convertor,
diff --git a/ompi/mca/op/aarch64/Makefile.am b/ompi/mca/op/aarch64/Makefile.am
new file mode 100644
index 00000000000..4100ecebe16
--- /dev/null
+++ b/ompi/mca/op/aarch64/Makefile.am
@@ -0,0 +1,82 @@
+#
+# Copyright (c) 2019      The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# This is an aarch64 op component.  This Makefile.am is a typical
+# aarch64 of how to integrate into Open MPI's Automake-based build
+# system.
+#
+# See https://github.com/open-mpi/ompi/wiki/devel-CreateComponent
+# for more details on how to make Open MPI components.
+
+# First, list all .h and .c sources.  It is necessary to list all .h
+# files so that they will be picked up in the distribution tarball.
+
+sources = \
+    op_aarch64.h \
+    op_aarch64_component.c
+
+# Open MPI components can be compiled two ways:
+#
+# 1. As a standalone dynamic shared object (DSO), sometimes called a
+# dynamically loadable library (DLL).
+#
+# 2. As a static library that is slurped up into the upper-level
+# libmpi library (regardless of whether libmpi is a static or dynamic
+# library).  This is called a "Libtool convenience library".
+#
+# The component needs to create an output library in this top-level
+# component directory, and named either mca_<type>_<name>.la (for DSO
+# builds) or libmca_<type>_<name>.la (for static builds).  The OMPI
+# build system will have set the
+# MCA_BUILD_ompi_<framework>_<component>_DSO AM_CONDITIONAL to indicate
+# which way this component should be built.
+specialized_op_libs =
+if MCA_BUILD_ompi_op_has_neon_support
+specialized_op_libs += liblocal_ops_neon.la
+liblocal_ops_neon_la_SOURCES = op_aarch64_functions.c
+liblocal_ops_neon_la_CPPFLAGS = -DGENERATE_NEON_CODE
+endif
+if MCA_BUILD_ompi_op_has_sve_support
+specialized_op_libs += liblocal_ops_sve.la
+liblocal_ops_sve_la_SOURCES = op_aarch64_functions.c
+liblocal_ops_sve_la_CPPFLAGS = -DGENERATE_SVE_CODE
+endif
+
+component_noinst  = $(specialized_op_libs)
+if MCA_BUILD_ompi_op_aarch64_DSO
+component_install = mca_op_aarch64.la
+else
+component_install =
+component_noinst  += libmca_op_aarch64.la
+endif
+
+# Specific information for DSO builds.
+#
+# The DSO should install itself in $(ompilibdir) (by default,
+# $prefix/lib/openmpi).
+
+mcacomponentdir = $(ompilibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_op_aarch64_la_SOURCES = $(sources)
+mca_op_aarch64_la_LIBADD = $(specialized_op_libs)
+mca_op_aarch64_la_LDFLAGS = -module -avoid-version $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la
+
+
+# Specific information for static builds.
+#
+# Note that we *must* "noinst"; the upper-layer Makefile.am's will
+# slurp in the resulting .la library into libmpi.
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_op_aarch64_la_SOURCES = $(sources)
+libmca_op_aarch64_la_LIBADD = $(specialized_op_libs)
+libmca_op_aarch64_la_LDFLAGS = -module -avoid-version
diff --git a/ompi/mca/op/aarch64/configure.m4 b/ompi/mca/op/aarch64/configure.m4
new file mode 100644
index 00000000000..08d62b2e2f8
--- /dev/null
+++ b/ompi/mca/op/aarch64/configure.m4
@@ -0,0 +1,119 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2019-2020 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
+#
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# MCA_ompi_op_arm_CONFIG([action-if-can-compile],
+#		         [action-if-cant-compile])
+# ------------------------------------------------
+AC_DEFUN([MCA_ompi_op_aarch64_CONFIG],[
+    AC_CONFIG_FILES([ompi/mca/op/aarch64/Makefile])
+    case "${host}" in
+        aarch64*|arm64*)
+            op_aarch64_check="yes";;
+        *)
+            op_aarch64_check="no";;
+    esac
+    AS_IF([test "$op_aarch64_check" = "yes"],
+          [AC_LANG_PUSH([C])
+
+           #
+           # Check for NEON support
+           #
+           AC_CACHE_CHECK([for NEON support], op_cv_neon_support, 
+                [
+                  AC_LINK_IFELSE(
+                      [AC_LANG_PROGRAM([[
+#if defined(__aarch64__) && defined(__ARM_NEON)
+#include <arm_neon.h>
+#else
+#error "No support for __aarch64__"
+#endif
+                                       ]],
+                                       [[
+#if defined(__aarch64__) && defined(__ARM_NEON)
+    int32x4_t vA;
+    vA = vmovq_n_s32(0)
+#endif
+                                       ]])],
+                      [op_cv_neon_support=yes],
+                      [op_cv_neon_support=no])])
+
+           #
+           # Check for NEON FP support
+           #
+           AC_CACHE_CHECK([for NEON FP support], op_cv_neon_fp_support,
+                [AS_IF([test "$op_cv_neon_support" = "yes"],
+                        [
+                          AC_LINK_IFELSE(
+                              [AC_LANG_PROGRAM([[
+#if defined(__aarch64__) && defined(__ARM_NEON) && (defined(__ARM_NEON_FP) || defined(__ARM_FP))
+#include <arm_neon.h>
+#else
+#error "No support for __aarch64__ or NEON FP"
+#endif
+                                             ]],
+                                             [[
+#if defined(__aarch64__) && defined(__ARM_NEON) && (defined(__ARM_NEON_FP) || defined(__ARM_FP))
+    float32x4_t vA;
+    vA = vmovq_n_f32(0)
+#endif
+                                             ]])],
+                            [op_cv_neon_fp_support=yes],
+                            [op_cv_neon_fp_support=no])])])
+
+           #
+           # Check for SVE support
+           #
+           AC_CACHE_CHECK([for SVE support], op_cv_sve_support,
+                 [AS_IF([test "$op_cv_neon_support" = "yes"],
+                        [
+                          AC_LINK_IFELSE(
+                              [AC_LANG_PROGRAM([[
+#if defined(__aarch64__) && defined(__ARM_FEATURE_SVE)
+#include <arm_sve.h>
+#else
+#error "No support for __aarch64__ or SVE"
+#endif
+                                             ]],
+                                             [[
+#if defined(__aarch64__) && defined(_ARM_FEATURE_SVE)
+    svfloat32_t vA;
+    vA = svdup_n_f32(0)
+#endif
+                                             ]])],
+                      [op_cv_sve_support=yes],
+                      [op_cv_sve_support=no])])])
+          ])
+
+    AM_CONDITIONAL([MCA_BUILD_ompi_op_has_neon_support],
+                   [test "$op_cv_neon_support" = "yes"])
+    AM_CONDITIONAL([MCA_BUILD_ompi_op_has_neon_fp_support],
+                   [test "$op_cv_neon_fp_support" = "yes"])
+    AM_CONDITIONAL([MCA_BUILD_ompi_op_has_sve_support],
+                   [test "$op_cv_sve_support" = "yes"])
+    AC_SUBST(MCA_BUILD_ompi_op_has_neon_support)
+    AC_SUBST(MCA_BUILD_ompi_op_has_neon_fp_support)
+    AC_SUBST(MCA_BUILD_ompi_op_has_sve_support)
+
+    AS_IF([test "$op_cv_neon_support" = "yes"],
+          [AC_DEFINE([OMPI_MCA_OP_HAVE_NEON], [1],[NEON supported in the current build])])
+    AS_IF([test "$op_cv_neon_fp_support" = "yes"],
+          [AC_DEFINE([OMPI_MCA_OP_HAVE_NEON_FP], [1],[NEON FP supported in the current build])])
+    AS_IF([test "$op_cv_sve_support" = "yes"],
+          [AC_DEFINE([OMPI_MCA_OP_HAVE_SVE], [1],[SVE supported in the current build])])
+
+    # If we have at least support for Neon
+    AS_IF([test "$op_cv_neon_support" = "yes"],
+          [$1],
+          [$2])
+])dnl
diff --git a/ompi/mca/op/aarch64/op_aarch64.h b/ompi/mca/op/aarch64/op_aarch64.h
new file mode 100644
index 00000000000..22be89614c6
--- /dev/null
+++ b/ompi/mca/op/aarch64/op_aarch64.h
@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2019      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2019      Arm Ltd.  All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef MCA_OP_AARCH64_EXPORT_H
+#define MCA_OP_AARCH64_EXPORT_H
+
+#include "ompi_config.h"
+
+#include "ompi/mca/mca.h"
+#include "opal/class/opal_object.h"
+
+#include "ompi/mca/op/op.h"
+
+BEGIN_C_DECLS
+
+/**
+ * Derive a struct from the base op component struct, allowing us to
+ * cache some component-specific information on our well-known
+ * component struct.
+ */
+typedef struct {
+    /** The base op component struct */
+    ompi_op_base_component_1_0_0_t super;
+
+    /* What follows is aarch64-component-specific cached information.  We
+       tend to use this scheme (caching information on the aarch64
+       component itself) instead of lots of individual global
+       variables for the component.  */
+
+    /** A simple boolean indicating that the hardware is available. */
+    uint32_t hardware_available;
+
+    /** A simple boolean indicating whether double precision is
+        supported. */
+    bool double_supported;
+} ompi_op_aarch64_component_t;
+
+/**
+ * Globally exported variable.  Note that it is a *aarch64* component
+ * (defined above), which has the ompi_op_base_component_t as its
+ * first member.  Hence, the MCA/op framework will find the data that
+ * it expects in the first memory locations, but then the component
+ * itself can cache additional information after that that can be used
+ * by both the component and modules.
+ */
+OMPI_DECLSPEC extern ompi_op_aarch64_component_t
+    mca_op_aarch64_component;
+
+END_C_DECLS
+
+#endif /* MCA_OP_AARCH64_EXPORT_H */
diff --git a/ompi/mca/op/aarch64/op_aarch64_component.c b/ompi/mca/op/aarch64/op_aarch64_component.c
new file mode 100644
index 00000000000..92bf4fa3656
--- /dev/null
+++ b/ompi/mca/op/aarch64/op_aarch64_component.c
@@ -0,0 +1,246 @@
+/*
+ * Copyright (c) 2019      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2019      ARM Ltd.  All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2024      Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/** @file
+ *
+ * This is the "AARCH64" component source code.
+ *
+ */
+
+#include "ompi_config.h"
+
+#include "opal/util/printf.h"
+
+#include "ompi/constants.h"
+#include "ompi/mca/op/aarch64/op_aarch64.h"
+#include "ompi/mca/op/base/base.h"
+#include "ompi/mca/op/op.h"
+#include "ompi/op/op.h"
+
+static int mca_op_aarch64_component_open(void);
+static int mca_op_aarch64_component_close(void);
+static int mca_op_aarch64_component_init_query(bool enable_progress_threads,
+                                     bool enable_mpi_thread_multiple);
+static struct ompi_op_base_module_1_0_0_t *
+    mca_op_aarch64_component_op_query(struct ompi_op_t *op, int *priority);
+static int mca_op_aarch64_component_register(void);
+
+ompi_op_aarch64_component_t mca_op_aarch64_component = {
+    /* First, the mca_base_component_t struct containing meta
+       information about the component itself */
+    {
+        .opc_version = {
+            OMPI_OP_BASE_VERSION_1_0_0,
+
+            .mca_component_name = "aarch64",
+            MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION,
+                                  OMPI_RELEASE_VERSION),
+            .mca_open_component = mca_op_aarch64_component_open,
+            .mca_close_component = mca_op_aarch64_component_close,
+            .mca_register_component_params = mca_op_aarch64_component_register,
+        },
+        .opc_data = {
+            /* The component is checkpoint ready */
+            MCA_BASE_METADATA_PARAM_CHECKPOINT
+        },
+
+        .opc_init_query = mca_op_aarch64_component_init_query,
+        .opc_op_query = mca_op_aarch64_component_op_query,
+    },
+};
+
+/*
+ * Component open
+ */
+static int mca_op_aarch64_component_open(void)
+{
+
+    /* A first level check to see if NEON or SVE ISA is even available in this
+       process.  E.g., you may want to do a first-order check to see
+       if hardware is available.  If so, return OMPI_SUCCESS.  If not,
+       return anything other than OMPI_SUCCESS and the component will
+       silently be ignored.
+
+       Note that if this function returns non-OMPI_SUCCESS, then this
+       component won't even be shown in ompi_info output (which is
+       probably not what you want).
+    */
+
+    return OMPI_SUCCESS;
+}
+
+/*
+ * Component close
+ */
+static int mca_op_aarch64_component_close(void)
+{
+
+    /* If the aarch64 was opened successfully, close it (i.e., release any
+       resources that may have been allocated on this component).
+       Note that _component_close() will always be called at the end
+       of the process, so it may have been after any/all of the other
+       component functions have been invoked (and possibly even after
+       modules have been created and/or destroyed). */
+
+    return OMPI_SUCCESS;
+}
+
+/*
+ * Register MCA params.
+ */
+static int mca_op_aarch64_component_register(void)
+{
+
+    mca_op_aarch64_component.hardware_available = 1;  /* Check for Neon */
+#if defined(OMPI_MCA_OP_HAVE_SVE)
+    uint64_t id_aa64pfr0_el1 = (1UL << 32);
+    __asm__("mrs %0, ID_AA64PFR0_EL1" : "=r"(id_aa64pfr0_el1) : :);
+    /* Check for SVE support */
+    mca_op_aarch64_component.hardware_available |= ((id_aa64pfr0_el1 & (1UL << 32)) ? 2 : 0);
+#endif  /* defined(OMPI_MCA_OP_HAVE_SVE) */
+    (void) mca_base_component_var_register(&mca_op_aarch64_component.super.opc_version,
+                                           "hardware_available",
+                                           "Whether the Neon (1) or SVE (2) hardware is available",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           &mca_op_aarch64_component.hardware_available);
+    uint64_t id_aa64zfr0_el1 = 0;
+#if defined(OMPI_MCA_OP_HAVE_SVE)
+    __asm__("mrs %0, ID_AA64ZFR0_EL1" : "=r"(id_aa64zfr0_el1) : :);
+#endif  /* defined(OMPI_MCA_OP_HAVE_SVE) */
+    mca_op_aarch64_component.double_supported = id_aa64zfr0_el1 & (1UL << 56);
+    /* Bit 1: mandatory SVE2 instructions */
+    /* Bit 2: mandatory SVE2.1 instructions */
+    (void) mca_base_component_var_register(&mca_op_aarch64_component.super.opc_version,
+                                           "double_supported",
+                                           "Whether the double precision data types are supported or not",
+                                           MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
+                                           OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           &mca_op_aarch64_component.double_supported);
+
+    return OMPI_SUCCESS;
+}
+
+/*
+ * Query whether this component wants to be used in this process.
+ */
+static int mca_op_aarch64_component_init_query(bool enable_progress_threads,
+                                               bool enable_mpi_thread_multiple)
+{
+    if (mca_op_aarch64_component.hardware_available) {
+        return OMPI_SUCCESS;
+    }
+    return OMPI_ERR_NOT_SUPPORTED;
+}
+
+#if defined(OMPI_MCA_OP_HAVE_NEON)
+extern ompi_op_base_handler_fn_t
+ompi_op_aarch64_functions_neon[OMPI_OP_BASE_FORTRAN_OP_MAX][OMPI_OP_BASE_TYPE_MAX];
+extern ompi_op_base_3buff_handler_fn_t
+ompi_op_aarch64_3buff_functions_neon[OMPI_OP_BASE_FORTRAN_OP_MAX][OMPI_OP_BASE_TYPE_MAX];
+#endif  /* defined(OMPI_MCA_OP_HAVE_NEON) */
+#if defined(OMPI_MCA_OP_HAVE_SVE)
+extern ompi_op_base_handler_fn_t
+ompi_op_aarch64_functions_sve[OMPI_OP_BASE_FORTRAN_OP_MAX][OMPI_OP_BASE_TYPE_MAX];
+extern ompi_op_base_3buff_handler_fn_t
+ompi_op_aarch64_3buff_functions_sve[OMPI_OP_BASE_FORTRAN_OP_MAX][OMPI_OP_BASE_TYPE_MAX];
+#endif  /* defined(OMPI_MCA_OP_HAVE_SVE) */
+
+/*
+ * Query whether this component can be used for a specific op
+ */
+static struct ompi_op_base_module_1_0_0_t *
+    mca_op_aarch64_component_op_query(struct ompi_op_t *op, int *priority)
+{
+    /* Sanity check -- although the framework should never invoke the
+       _component_op_query() on non-intrinsic MPI_Op's, we'll put a
+       check here just to be sure. */
+    if (0 == (OMPI_OP_FLAGS_INTRINSIC & op->o_flags)) {
+        return NULL;
+    }
+    ompi_op_base_module_t *module = OBJ_NEW(ompi_op_base_module_t);
+    if (NULL == module) {
+        return NULL;
+    }
+
+    switch (op->o_f_to_c_index) {
+    case OMPI_OP_BASE_FORTRAN_MAX:
+    case OMPI_OP_BASE_FORTRAN_MIN:
+    case OMPI_OP_BASE_FORTRAN_SUM:
+    case OMPI_OP_BASE_FORTRAN_PROD:
+    case OMPI_OP_BASE_FORTRAN_BOR:
+    case OMPI_OP_BASE_FORTRAN_BAND:
+    case OMPI_OP_BASE_FORTRAN_BXOR:
+        for (int i = 0; i < OMPI_OP_BASE_TYPE_MAX; ++i) {
+            module->opm_fns[i] = NULL;
+            module->opm_3buff_fns[i] = NULL;
+#if defined(OMPI_MCA_OP_HAVE_SVE)
+            if( mca_op_aarch64_component.hardware_available & 2 ) {
+                module->opm_fns[i] = ompi_op_aarch64_functions_sve[op->o_f_to_c_index][i];
+                module->opm_3buff_fns[i] = ompi_op_aarch64_3buff_functions_sve[op->o_f_to_c_index][i];
+            }
+#endif  /* defined(OMPI_MCA_OP_HAVE_SVE) */
+#if defined(OMPI_MCA_OP_HAVE_NEON)
+            if( mca_op_aarch64_component.hardware_available & 1 ) {
+                if( NULL == module->opm_fns[i] ) {
+                    module->opm_fns[i] = ompi_op_aarch64_functions_neon[op->o_f_to_c_index][i];
+                }
+                if( NULL == module->opm_3buff_fns[i] ) {
+                    module->opm_3buff_fns[i] = ompi_op_aarch64_3buff_functions_neon[op->o_f_to_c_index][i];
+                }
+            }
+#endif  /* defined(OMPI_MCA_OP_HAVE_NEON) */
+        }
+        break;
+    case OMPI_OP_BASE_FORTRAN_LAND:
+        OBJ_RELEASE(module);
+        module = NULL;
+        break;
+    case OMPI_OP_BASE_FORTRAN_LOR:
+        OBJ_RELEASE(module);
+        module = NULL;
+        break;
+    case OMPI_OP_BASE_FORTRAN_LXOR:
+        OBJ_RELEASE(module);
+        module = NULL;
+        break;
+    case OMPI_OP_BASE_FORTRAN_MAXLOC:
+        OBJ_RELEASE(module);
+        module = NULL;
+        break;
+    case OMPI_OP_BASE_FORTRAN_MINLOC:
+        OBJ_RELEASE(module);
+        module= NULL;
+        break;
+    default:
+        OBJ_RELEASE(module);
+        module= NULL;
+    }
+    /* If we got a module from above, we'll return it.  Otherwise,
+       we'll return NULL, indicating that this component does not want
+       to be considered for selection for this MPI_Op.  Note that the
+       functions each returned a *aarch64* component pointer
+       (vs. a *base* component pointer -- where an *aarch64* component
+       is a base component plus some other module-specific cached
+       information), so we have to cast it to the right pointer type
+       before returning. */
+    if (NULL != module) {
+        *priority = 50;
+    }
+    return (ompi_op_base_module_1_0_0_t *) module;
+}
diff --git a/ompi/mca/op/aarch64/op_aarch64_functions.c b/ompi/mca/op/aarch64/op_aarch64_functions.c
new file mode 100644
index 00000000000..5515ddb9567
--- /dev/null
+++ b/ompi/mca/op/aarch64/op_aarch64_functions.c
@@ -0,0 +1,622 @@
+/*
+ * Copyright (c) 2019      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2019      Arm Ltd.  All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2024      Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#include "opal/util/output.h"
+
+#include "ompi/op/op.h"
+#include "ompi/mca/op/op.h"
+#include "ompi/mca/op/base/base.h"
+#include "ompi/mca/op/aarch64/op_aarch64.h"
+
+#if defined(GENERATE_SVE_CODE)
+#    include <arm_sve.h>
+#define OMPI_OP_TYPE_PREPEND sv
+#define OMPI_OP_OP_PREPEND sv
+#define APPEND   _sve
+#elif defined(GENERATE_NEON_CODE)
+#    include <arm_neon.h>
+#define OMPI_OP_TYPE_PREPEND
+#define OMPI_OP_OP_PREPEND v
+#define APPEND   _neon
+#else
+#error we should not reach this
+#endif /* OMPI_MCA_OP_HAVE_SVE */
+
+/*
+ * Concatenate preprocessor tokens A and B without expanding macro definitions
+ * (however, if invoked from a macro, macro arguments are expanded).
+ */
+#define OP_CONCAT_NX(A, B) A ## B
+
+/*
+ * Concatenate preprocessor tokens A and B after macro-expanding them.
+ */
+#define OP_CONCAT(A, B) OP_CONCAT_NX(A, B)
+
+#if defined(GENERATE_SVE_CODE)
+#    define svcnt(X)           \
+        _Generic((X),          \
+            int8_t: svcntb,    \
+            uint8_t: svcntb,   \
+            int16_t: svcnth,   \
+            uint16_t: svcnth,  \
+            int32_t: svcntw,   \
+            uint32_t: svcntw,  \
+            int64_t: svcntd,   \
+            uint64_t: svcntd,  \
+            float32_t: svcntw, \
+            float64_t: svcntd)()
+#else
+#define DUMP2(out, in1, in2) \
+        case 2: (out)[1] = current_func((in1)[1], (in2)[1]); \
+        case 1: (out)[0] = current_func((in1)[0], (in2)[0]);
+#define DUMP4(out, in1, in2) \
+        case 4: (out)[3] = current_func((in1)[3], (in2)[3]); \
+        case 3: (out)[2] = current_func((in1)[2], (in2)[2]); \
+        DUMP2(out, in1, in2)
+#define DUMP8(out, in1, in2) \
+        case 8: (out)[7] = current_func((in1)[7], (in2)[7]); \
+        case 7: (out)[6] = current_func((in1)[6], (in2)[6]); \
+        case 6: (out)[5] = current_func((in1)[5], (in2)[5]); \
+        case 5: (out)[4] = current_func((in1)[4], (in2)[4]); \
+        DUMP4(out, in1, in2)
+#define DUMP16(out, in1, in2) \
+        case 16: (out)[15] = current_func((in1)[15], (in2)[15]); \
+        case 15: (out)[14] = current_func((in1)[14], (in2)[14]); \
+        case 14: (out)[13] = current_func((in1)[13], (in2)[13]); \
+        case 13: (out)[12] = current_func((in1)[12], (in2)[12]); \
+        case 12: (out)[11] = current_func((in1)[11], (in2)[11]); \
+        case 11: (out)[10] = current_func((in1)[10], (in2)[10]); \
+        case 10: (out)[ 9] = current_func((in1)[ 9], (in2)[ 9]); \
+        case  9: (out)[ 8] = current_func((in1)[ 8], (in2)[ 8]); \
+        DUMP8(out, in1, in2)
+
+#    define neon_loop(how_much, out, in1, in2)   \
+_Generic((*(out)), \
+         int8_t:    __extension__({ switch ((how_much)) { DUMP16(out, in1, in2) }}), \
+         uint8_t:   __extension__({ switch ((how_much)) { DUMP16(out, in1, in2) }}), \
+         int16_t:   __extension__({ switch ((how_much)) { DUMP8(out, in1, in2) }}), \
+         uint16_t:  __extension__({ switch ((how_much)) { DUMP8(out, in1, in2) }}), \
+         int32_t:   __extension__({ switch ((how_much)) { DUMP4(out, in1, in2) }}), \
+         uint32_t:  __extension__({ switch ((how_much)) { DUMP4(out, in1, in2) }}), \
+         int64_t:   __extension__({ switch ((how_much)) { DUMP2(out, in1, in2) }}), \
+         uint64_t:  __extension__({ switch ((how_much)) { DUMP2(out, in1, in2) }}), \
+         float32_t: __extension__({ switch ((how_much)) { DUMP4(out, in1, in2) }}), \
+         float64_t: __extension__({ switch ((how_much)) { DUMP2(out, in1, in2) }}))
+#endif /* defined(GENERATE_SVE_CODE) */
+
+/*
+ * Since all the functions in this file are essentially identical, we
+ * use a macro to substitute in names and types.  The core operation
+ * in all functions that use this macro is the same.
+ *
+ * This macro is for (out op in).
+ *
+ */
+#if defined(GENERATE_NEON_CODE)
+#define OP_AARCH64_FUNC(name, type_name, type_size, type_cnt, type, op)                       \
+    static void OP_CONCAT(ompi_op_aarch64_2buff_##name##_##type##type_size##_t,               \
+                          APPEND)(const void *_in, void *_out, int *count,                    \
+                                  struct ompi_datatype_t **dtype,                             \
+                                  struct ompi_op_base_module_1_0_0_t *module)                 \
+    {                                                                                         \
+        int left_over = *count;                                                               \
+        type##type_size##_t *in = (type##type_size##_t *) _in,                                \
+                            *out = (type##type_size##_t *) _out;                              \
+        OP_CONCAT(OMPI_OP_TYPE_PREPEND, type##type_size##x##type_cnt##_t) vsrc, vdst;         \
+        for (; left_over >= type_cnt; left_over -= type_cnt) {                                \
+            vsrc = vld1q##_##type_name##type_size(in);                                        \
+            vdst = vld1q##_##type_name##type_size(out);                                       \
+            in += type_cnt;                                                                   \
+            vdst = OP_CONCAT(OMPI_OP_OP_PREPEND, op##q##_##type_name##type_size)(vdst, vsrc); \
+            vst1q##_##type_name##type_size(out, vdst);                                        \
+            out += type_cnt;                                                                  \
+        }                                                                                     \
+                                                                                              \
+        if(left_over > 0) {                                                                   \
+            neon_loop(left_over, out, out, in);                                               \
+        }                                                                                     \
+    }
+#elif defined(GENERATE_SVE_CODE)
+#define OP_AARCH64_FUNC(name, type_name, type_size, type_cnt, type, op)           \
+    static void OP_CONCAT(ompi_op_aarch64_2buff_##name##_##type##type_size##_t, APPEND) \
+                            (const void *_in, void *_out, int *count,             \
+                             struct ompi_datatype_t **dtype,                      \
+                             struct ompi_op_base_module_1_0_0_t *module)          \
+    {                                                                             \
+        const int types_per_step = svcnt(*((type##type_size##_t *) _in));         \
+        const int cnt = *count;                                                   \
+        type##type_size##_t *in = (type##type_size##_t *) _in,                    \
+                            *out = (type##type_size##_t *) _out;                  \
+        OP_CONCAT(OMPI_OP_TYPE_PREPEND, type##type_size##_t) vsrc, vdst;          \
+        for (int idx=0; idx < cnt; idx += types_per_step) {                    \
+            svbool_t pred = svwhilelt_b##type_size(idx, cnt);                     \
+            vsrc = svld1(pred, &in[idx]);                                         \
+            vdst = svld1(pred, &out[idx]);                                        \
+            vdst = OP_CONCAT(OMPI_OP_OP_PREPEND, op##_x)(pred, vdst, vsrc);       \
+            OP_CONCAT(OMPI_OP_OP_PREPEND, st1)(pred, &out[idx], vdst);            \
+        }                                                                         \
+    }
+#endif
+
+/*************************************************************************
+ * Max
+ *************************************************************************/
+#undef current_func
+#define current_func(a, b) ((a) > (b) ? (a) : (b))
+    OP_AARCH64_FUNC(max, s,  8, 16,   int, max)
+    OP_AARCH64_FUNC(max, u,  8, 16,  uint, max)
+    OP_AARCH64_FUNC(max, s, 16,  8,   int, max)
+    OP_AARCH64_FUNC(max, u, 16,  8,  uint, max)
+    OP_AARCH64_FUNC(max, s, 32,  4,   int, max)
+    OP_AARCH64_FUNC(max, u, 32,  4,  uint, max)
+#if defined(GENERATE_SVE_CODE)
+    OP_AARCH64_FUNC(max, s, 64,  2,   int, max)
+    OP_AARCH64_FUNC(max, u, 64,  2,  uint, max)
+#endif  /* defined(GENERATE_SVE_CODE) */
+
+    OP_AARCH64_FUNC(max, f, 32,  4, float, max)
+    OP_AARCH64_FUNC(max, f, 64,  2, float, max)
+
+/*************************************************************************
+ * Min
+ *************************************************************************/
+#undef current_func
+#define current_func(a, b) ((a) < (b) ? (a) : (b))
+    OP_AARCH64_FUNC(min, s,  8, 16,   int, min)
+    OP_AARCH64_FUNC(min, u,  8, 16,  uint, min)
+    OP_AARCH64_FUNC(min, s, 16,  8,   int, min)
+    OP_AARCH64_FUNC(min, u, 16,  8,  uint, min)
+    OP_AARCH64_FUNC(min, s, 32,  4,   int, min)
+    OP_AARCH64_FUNC(min, u, 32,  4,  uint, min)
+#if defined(GENERATE_SVE_CODE)
+    OP_AARCH64_FUNC(min, s, 64,  2,   int, min)
+    OP_AARCH64_FUNC(min, u, 64,  2,  uint, min)
+#endif  /* defined(GENERATE_SVE_CODE) */
+
+    OP_AARCH64_FUNC(min, f, 32,  4, float, min)
+    OP_AARCH64_FUNC(min, f, 64,  2, float, min)
+    /*************************************************************************
+     * Sum
+     ************************************************************************/
+#undef current_func
+#define current_func(a, b) ((a) + (b))
+    OP_AARCH64_FUNC(sum, s,  8, 16,   int, add)
+    OP_AARCH64_FUNC(sum, u,  8, 16,  uint, add)
+    OP_AARCH64_FUNC(sum, s, 16,  8,   int, add)
+    OP_AARCH64_FUNC(sum, u, 16,  8,  uint, add)
+    OP_AARCH64_FUNC(sum, s, 32,  4,   int, add)
+    OP_AARCH64_FUNC(sum, u, 32,  4,  uint, add)
+    OP_AARCH64_FUNC(sum, s, 64,  2,   int, add)
+    OP_AARCH64_FUNC(sum, u, 64,  2,  uint, add)
+
+    OP_AARCH64_FUNC(sum, f, 32,  4, float, add)
+    OP_AARCH64_FUNC(sum, f, 64,  2, float, add)
+
+/*************************************************************************
+ * Product
+ *************************************************************************/
+#undef current_func
+#define current_func(a, b) ((a) * (b))
+    OP_AARCH64_FUNC(prod, s,  8, 16,   int, mul)
+    OP_AARCH64_FUNC(prod, u,  8, 16,  uint, mul)
+    OP_AARCH64_FUNC(prod, s, 16,  8,   int, mul)
+    OP_AARCH64_FUNC(prod, u, 16,  8,  uint, mul)
+    OP_AARCH64_FUNC(prod, s, 32,  4,   int, mul)
+    OP_AARCH64_FUNC(prod, u, 32,  4,  uint, mul)
+#if defined(GENERATE_SVE_CODE)
+    OP_AARCH64_FUNC(prod, s, 64,  2,   int, mul)
+    OP_AARCH64_FUNC(prod, u, 64,  2,  uint, mul)
+#endif  /* defined(GENERATE_SVE_CODE) */
+
+    OP_AARCH64_FUNC(prod, f, 32,  4, float, mul)
+    OP_AARCH64_FUNC(prod, f, 64,  2, float, mul)
+
+/*************************************************************************
+ * Bitwise AND
+ *************************************************************************/
+#undef current_func
+#define current_func(a, b) ((a) & (b))
+    OP_AARCH64_FUNC(band, s,  8, 16,  int, and)
+    OP_AARCH64_FUNC(band, u,  8, 16, uint, and)
+    OP_AARCH64_FUNC(band, s, 16,  8,  int, and)
+    OP_AARCH64_FUNC(band, u, 16,  8, uint, and)
+    OP_AARCH64_FUNC(band, s, 32,  4,  int, and)
+    OP_AARCH64_FUNC(band, u, 32,  4, uint, and)
+    OP_AARCH64_FUNC(band, s, 64,  2,  int, and)
+    OP_AARCH64_FUNC(band, u, 64,  2, uint, and)
+
+ /*************************************************************************
+ * Bitwise OR
+ *************************************************************************/
+#undef current_func
+#define current_func(a, b) ((a) | (b))
+    OP_AARCH64_FUNC(bor, s,  8, 16,  int, orr)
+    OP_AARCH64_FUNC(bor, u,  8, 16, uint, orr)
+    OP_AARCH64_FUNC(bor, s, 16,  8,  int, orr)
+    OP_AARCH64_FUNC(bor, u, 16,  8, uint, orr)
+    OP_AARCH64_FUNC(bor, s, 32,  4,  int, orr)
+    OP_AARCH64_FUNC(bor, u, 32,  4, uint, orr)
+    OP_AARCH64_FUNC(bor, s, 64,  2,  int, orr)
+    OP_AARCH64_FUNC(bor, u, 64,  2, uint, orr)
+
+/*************************************************************************
+ * Bitwise XOR
+ *************************************************************************/
+#undef current_func
+#define current_func(a, b) ((a) ^ (b))
+    OP_AARCH64_FUNC(bxor, s,  8, 16,  int, eor)
+    OP_AARCH64_FUNC(bxor, u,  8, 16, uint, eor)
+    OP_AARCH64_FUNC(bxor, s, 16,  8,  int, eor)
+    OP_AARCH64_FUNC(bxor, u, 16,  8, uint, eor)
+    OP_AARCH64_FUNC(bxor, s, 32,  4,  int, eor)
+    OP_AARCH64_FUNC(bxor, u, 32,  4, uint, eor)
+    OP_AARCH64_FUNC(bxor, s, 64,  2,  int, eor)
+    OP_AARCH64_FUNC(bxor, u, 64,  2, uint, eor)
+
+    /*
+     *  This is a three buffer (2 input and 1 output) version of the reduction
+     *  routines, needed for some optimizations.
+     */
+#if defined(GENERATE_NEON_CODE)
+#define OP_AARCH64_FUNC_3BUFF(name, type_name, type_size, type_cnt, type, op)                 \
+static void OP_CONCAT(ompi_op_aarch64_3buff_##name##_##type##type_size##_t, APPEND)       \
+                             (const void *_in1, const void *_in2, void *_out, int *count, \
+                              struct ompi_datatype_t **dtype,                             \
+                              struct ompi_op_base_module_1_0_0_t *module)                 \
+{                                                                                         \
+    int left_over = *count;                                                               \
+    type##type_size##_t *in1 = (type##type_size##_t *) _in1,                              \
+                        *in2 = (type##type_size##_t *) _in2,                              \
+                        *out = (type##type_size##_t *) _out;                              \
+    OP_CONCAT(OMPI_OP_TYPE_PREPEND, type##type_size##x##type_cnt##_t) vsrc, vdst;         \
+    for (; left_over >= type_cnt; left_over -= type_cnt) {                                \
+        vsrc = vld1q##_##type_name##type_size(in1);                                       \
+        vdst = vld1q##_##type_name##type_size(in2);                                       \
+        in1 += type_cnt;                                                                  \
+        in2 += type_cnt;                                                                  \
+        vdst = OP_CONCAT(OMPI_OP_OP_PREPEND, op##q##_##type_name##type_size)(vdst, vsrc); \
+        vst1q##_##type_name##type_size(out, vdst);                                        \
+        out += type_cnt;                                                                  \
+    }                                                                                     \
+    if (left_over > 0) {                                                                  \
+        neon_loop(left_over, out, in1, in2);                                              \
+    }                                                                                     \
+}
+#elif defined(GENERATE_SVE_CODE)
+#define OP_AARCH64_FUNC_3BUFF(name, type_name, type_size, type_cnt, type, op)             \
+static void OP_CONCAT(ompi_op_aarch64_3buff_##name##_##type##type_size##_t, APPEND)       \
+                             (const void *_in1, const void *_in2, void *_out, int *count, \
+                              struct ompi_datatype_t **dtype,                             \
+                              struct ompi_op_base_module_1_0_0_t *module)                 \
+{                                                                                         \
+    const int types_per_step = svcnt(*((type##type_size##_t *) _in1));                    \
+    type##type_size##_t *in1 = (type##type_size##_t *) _in1,                              \
+                        *in2 = (type##type_size##_t *) _in2,                              \
+                        *out = (type##type_size##_t *) _out;                              \
+    const int cnt = *count;                                                               \
+    OP_CONCAT(OMPI_OP_TYPE_PREPEND, type##type_size##_t) vsrc, vdst;                      \
+    for (int idx=0; idx < cnt; idx += types_per_step) {                                   \
+        svbool_t pred = svwhilelt_b##type_size(idx, cnt);                                 \
+        vsrc = svld1(pred, &in1[idx]);                                                    \
+        vdst = svld1(pred, &in2[idx]);                                                    \
+        vdst = OP_CONCAT(OMPI_OP_OP_PREPEND, op##_x)(pred, vdst, vsrc);                   \
+        OP_CONCAT(OMPI_OP_OP_PREPEND, st1)(pred, &out[idx], vdst);                        \
+    }                                                                                     \
+}
+#endif  /* defined(GENERATE_SVE_CODE) */
+
+/*************************************************************************
+ * Max
+ *************************************************************************/
+#undef current_func
+#define current_func(a, b) ((a) > (b) ? (a) : (b))
+    OP_AARCH64_FUNC_3BUFF(max, s,  8, 16,   int, max)
+    OP_AARCH64_FUNC_3BUFF(max, u,  8, 16,  uint, max)
+    OP_AARCH64_FUNC_3BUFF(max, s, 16,  8,   int, max)
+    OP_AARCH64_FUNC_3BUFF(max, u, 16,  8,  uint, max)
+    OP_AARCH64_FUNC_3BUFF(max, s, 32,  4,   int, max)
+    OP_AARCH64_FUNC_3BUFF(max, u, 32,  4,  uint, max)
+#if defined(GENERATE_SVE_CODE)
+    OP_AARCH64_FUNC_3BUFF(max, s, 64,  2,   int, max)
+    OP_AARCH64_FUNC_3BUFF(max, u, 64,  2,  uint, max)
+#endif  /* defined(GENERATE_SVE_CODE) */
+
+    OP_AARCH64_FUNC_3BUFF(max, f, 32,  4, float, max)
+    OP_AARCH64_FUNC_3BUFF(max, f, 64,  2, float, max)
+
+    /*************************************************************************
+     * Min
+     *************************************************************************/
+#undef current_func
+#define current_func(a, b) ((a) < (b) ? (a) : (b))
+    OP_AARCH64_FUNC_3BUFF(min, s,  8, 16,   int, min)
+    OP_AARCH64_FUNC_3BUFF(min, u,  8, 16,  uint, min)
+    OP_AARCH64_FUNC_3BUFF(min, s, 16,  8,   int, min)
+    OP_AARCH64_FUNC_3BUFF(min, u, 16,  8,  uint, min)
+    OP_AARCH64_FUNC_3BUFF(min, s, 32,  4,   int, min)
+    OP_AARCH64_FUNC_3BUFF(min, u, 32,  4,  uint, min)
+#if defined(GENERATE_SVE_CODE)
+    OP_AARCH64_FUNC_3BUFF(min, s, 64,  2,   int, min)
+    OP_AARCH64_FUNC_3BUFF(min, u, 64,  2,  uint, min)
+#endif  /* defined(GENERATE_SVE_CODE) */
+
+    OP_AARCH64_FUNC_3BUFF(min, f, 32,  4, float, min)
+    OP_AARCH64_FUNC_3BUFF(min, f, 64,  2, float, min)
+
+ /*************************************************************************
+ * Sum
+ ************************************************************************/
+#undef current_func
+#define current_func(a, b) ((a) + (b))
+    OP_AARCH64_FUNC_3BUFF(sum, s,  8, 16,  int, add)
+    OP_AARCH64_FUNC_3BUFF(sum, u,  8, 16, uint, add)
+    OP_AARCH64_FUNC_3BUFF(sum, s, 16,  8,  int, add)
+    OP_AARCH64_FUNC_3BUFF(sum, u, 16,  8, uint, add)
+    OP_AARCH64_FUNC_3BUFF(sum, s, 32,  4,  int, add)
+    OP_AARCH64_FUNC_3BUFF(sum, u, 32,  4, uint, add)
+    OP_AARCH64_FUNC_3BUFF(sum, s, 64,  2,  int, add)
+    OP_AARCH64_FUNC_3BUFF(sum, u, 64,  2, uint, add)
+
+    OP_AARCH64_FUNC_3BUFF(sum, f, 32,  4, float, add)
+    OP_AARCH64_FUNC_3BUFF(sum, f, 64,  2, float, add)
+
+/*************************************************************************
+ * Product
+ *************************************************************************/
+#undef current_func
+#define current_func(a, b) ((a) * (b))
+    OP_AARCH64_FUNC_3BUFF(prod, s,  8, 16,   int, mul)
+    OP_AARCH64_FUNC_3BUFF(prod, u,  8, 16,  uint, mul)
+    OP_AARCH64_FUNC_3BUFF(prod, s, 16,  8,   int, mul)
+    OP_AARCH64_FUNC_3BUFF(prod, u, 16,  8,  uint, mul)
+    OP_AARCH64_FUNC_3BUFF(prod, s, 32,  4,   int, mul)
+    OP_AARCH64_FUNC_3BUFF(prod, u, 32,  4,  uint, mul)
+#if defined(GENERATE_SVE_CODE)
+    OP_AARCH64_FUNC_3BUFF(prod, s, 64,  2,   int, mul)
+    OP_AARCH64_FUNC_3BUFF(prod, u, 64,  2,  uint, mul)
+#endif  /* defined(GENERATE_SVE_CODE) */
+
+    OP_AARCH64_FUNC_3BUFF(prod, f, 32,  4, float, mul)
+    OP_AARCH64_FUNC_3BUFF(prod, f, 64,  2, float, mul)
+
+/*************************************************************************
+ * Bitwise AND
+ *************************************************************************/
+#undef current_func
+#define current_func(a, b) ((a) & (b))
+    OP_AARCH64_FUNC_3BUFF(band, s,  8, 16,  int, and)
+    OP_AARCH64_FUNC_3BUFF(band, u,  8, 16, uint, and)
+    OP_AARCH64_FUNC_3BUFF(band, s, 16,  8,  int, and)
+    OP_AARCH64_FUNC_3BUFF(band, u, 16,  8, uint, and)
+    OP_AARCH64_FUNC_3BUFF(band, s, 32,  4,  int, and)
+    OP_AARCH64_FUNC_3BUFF(band, u, 32,  4, uint, and)
+    OP_AARCH64_FUNC_3BUFF(band, s, 64,  2,  int, and)
+    OP_AARCH64_FUNC_3BUFF(band, u, 64,  2, uint, and)
+
+ /*************************************************************************
+ * Bitwise OR
+ *************************************************************************/
+#undef current_func
+#define current_func(a, b) ((a) | (b))
+    OP_AARCH64_FUNC_3BUFF(bor, s,  8, 16,  int, orr)
+    OP_AARCH64_FUNC_3BUFF(bor, u,  8, 16, uint, orr)
+    OP_AARCH64_FUNC_3BUFF(bor, s, 16,  8,  int, orr)
+    OP_AARCH64_FUNC_3BUFF(bor, u, 16,  8, uint, orr)
+    OP_AARCH64_FUNC_3BUFF(bor, s, 32,  4,  int, orr)
+    OP_AARCH64_FUNC_3BUFF(bor, u, 32,  4, uint, orr)
+    OP_AARCH64_FUNC_3BUFF(bor, s, 64,  2,  int, orr)
+    OP_AARCH64_FUNC_3BUFF(bor, u, 64,  2, uint, orr)
+
+/*************************************************************************
+ * Bitwise XOR
+ *************************************************************************/
+#undef current_func
+#define current_func(a, b) ((a) ^ (b))
+    OP_AARCH64_FUNC_3BUFF(bxor, s,  8, 16,  int, eor)
+    OP_AARCH64_FUNC_3BUFF(bxor, u,  8, 16, uint, eor)
+    OP_AARCH64_FUNC_3BUFF(bxor, s, 16,  8,  int, eor)
+    OP_AARCH64_FUNC_3BUFF(bxor, u, 16,  8, uint, eor)
+    OP_AARCH64_FUNC_3BUFF(bxor, s, 32,  4,  int, eor)
+    OP_AARCH64_FUNC_3BUFF(bxor, u, 32,  4, uint, eor)
+    OP_AARCH64_FUNC_3BUFF(bxor, s, 64,  2,  int, eor)
+    OP_AARCH64_FUNC_3BUFF(bxor, u, 64,  2, uint, eor)
+
+    /** C integer ***********************************************************/
+#define C_INTEGER_BASE(name, ftype)                                         \
+    [OMPI_OP_BASE_TYPE_INT8_T]   = OP_CONCAT(ompi_op_aarch64_##ftype##_##name##_int8_t, APPEND), \
+    [OMPI_OP_BASE_TYPE_UINT8_T]  = OP_CONCAT(ompi_op_aarch64_##ftype##_##name##_uint8_t, APPEND), \
+    [OMPI_OP_BASE_TYPE_INT16_T]  = OP_CONCAT(ompi_op_aarch64_##ftype##_##name##_int16_t, APPEND), \
+    [OMPI_OP_BASE_TYPE_UINT16_T] = OP_CONCAT(ompi_op_aarch64_##ftype##_##name##_uint16_t, APPEND), \
+    [OMPI_OP_BASE_TYPE_INT32_T]  = OP_CONCAT(ompi_op_aarch64_##ftype##_##name##_int32_t, APPEND), \
+    [OMPI_OP_BASE_TYPE_UINT32_T] = OP_CONCAT(ompi_op_aarch64_##ftype##_##name##_uint32_t, APPEND)
+#define C_INTEGER_EX(name, ftype)                                         \
+    [OMPI_OP_BASE_TYPE_INT64_T]  = OP_CONCAT(ompi_op_aarch64_##ftype##_##name##_int64_t, APPEND), \
+    [OMPI_OP_BASE_TYPE_UINT64_T] = OP_CONCAT(ompi_op_aarch64_##ftype##_##name##_uint64_t, APPEND)
+
+    /** Floating point, including all the Fortran reals *********************/
+#define FLOAT(name, ftype)  OP_CONCAT(ompi_op_aarch64_##ftype##_##name##_float32_t, APPEND)
+#define DOUBLE(name, ftype) OP_CONCAT(ompi_op_aarch64_##ftype##_##name##_float64_t, APPEND)
+
+#define FLOATING_POINT(name, ftype)                                    \
+    [OMPI_OP_BASE_TYPE_FLOAT] = FLOAT(name, ftype),                    \
+    [OMPI_OP_BASE_TYPE_DOUBLE] = DOUBLE(name, ftype)
+
+/*
+ * MPI_OP_NULL
+ * All types
+ */
+#define FLAGS_NO_FLOAT \
+        (OMPI_OP_FLAGS_INTRINSIC | OMPI_OP_FLAGS_ASSOC | OMPI_OP_FLAGS_COMMUTE)
+#define FLAGS \
+        (OMPI_OP_FLAGS_INTRINSIC | OMPI_OP_FLAGS_ASSOC | \
+         OMPI_OP_FLAGS_FLOAT_ASSOC | OMPI_OP_FLAGS_COMMUTE)
+
+    ompi_op_base_handler_fn_t OP_CONCAT(ompi_op_aarch64_functions, APPEND) [OMPI_OP_BASE_FORTRAN_OP_MAX][OMPI_OP_BASE_TYPE_MAX] =
+{
+    /* Corresponds to MPI_OP_NULL */
+    [OMPI_OP_BASE_FORTRAN_NULL] = {
+        /* Leaving this empty puts in NULL for all entries */
+        NULL,
+    },
+    /* Corresponds to MPI_MAX */
+    [OMPI_OP_BASE_FORTRAN_MAX] = {
+        C_INTEGER_BASE(max, 2buff),
+#if defined(GENERATE_SVE_CODE)
+        C_INTEGER_EX(max, 2buff),
+#endif /* defined(GENERATE_SVE_CODE) */
+        FLOATING_POINT(max, 2buff),
+    },
+    /* Corresponds to MPI_MIN */
+    [OMPI_OP_BASE_FORTRAN_MIN] = {
+        C_INTEGER_BASE(min, 2buff),
+#if defined(GENERATE_SVE_CODE)
+        C_INTEGER_EX(min, 2buff),
+#endif /* defined(GENERATE_SVE_CODE) */
+        FLOATING_POINT(min, 2buff),
+    },
+    /* Corresponds to MPI_SUM */
+    [OMPI_OP_BASE_FORTRAN_SUM] = {
+        C_INTEGER_BASE(sum, 2buff),
+        C_INTEGER_EX(sum, 2buff),
+        FLOATING_POINT(sum, 2buff),
+    },
+    /* Corresponds to MPI_PROD */
+    [OMPI_OP_BASE_FORTRAN_PROD] = {
+        C_INTEGER_BASE(prod, 2buff),
+#if defined(GENERATE_SVE_CODE)
+        C_INTEGER_EX(prod, 2buff),
+#endif /* defined(GENERATE_SVE_CODE) */
+        FLOATING_POINT(prod, 2buff),
+    },
+    /* Corresponds to MPI_LAND */
+    [OMPI_OP_BASE_FORTRAN_LAND] = {
+        NULL,
+    },
+    /* Corresponds to MPI_BAND */
+    [OMPI_OP_BASE_FORTRAN_BAND] = {
+        C_INTEGER_BASE(band, 2buff),
+        C_INTEGER_EX(band, 2buff),
+    },
+    /* Corresponds to MPI_LOR */
+    [OMPI_OP_BASE_FORTRAN_LOR] = {
+        NULL,
+    },
+    /* Corresponds to MPI_BOR */
+    [OMPI_OP_BASE_FORTRAN_BOR] = {
+        C_INTEGER_BASE(bor, 2buff),
+        C_INTEGER_EX(bor, 2buff),
+    },
+    /* Corresponds to MPI_LXOR */
+    [OMPI_OP_BASE_FORTRAN_LXOR] = {
+        NULL,
+    },
+    /* Corresponds to MPI_BXOR */
+    [OMPI_OP_BASE_FORTRAN_BXOR] = {
+        C_INTEGER_BASE(bxor, 2buff),
+        C_INTEGER_EX(bxor, 2buff),
+    },
+    /* Corresponds to MPI_REPLACE */
+    [OMPI_OP_BASE_FORTRAN_REPLACE] = {
+        /* (MPI_ACCUMULATE is handled differently than the other
+           reductions, so just zero out its function
+           implementations here to ensure that users don't invoke
+           MPI_REPLACE with any reduction operations other than
+           ACCUMULATE) */
+        NULL,
+    },
+
+};
+
+    ompi_op_base_3buff_handler_fn_t OP_CONCAT(ompi_op_aarch64_3buff_functions, APPEND)[OMPI_OP_BASE_FORTRAN_OP_MAX][OMPI_OP_BASE_TYPE_MAX] =
+{
+    /* Corresponds to MPI_OP_NULL */
+    [OMPI_OP_BASE_FORTRAN_NULL] = {
+        /* Leaving this empty puts in NULL for all entries */
+        NULL,
+    },
+    /* Corresponds to MPI_MAX */
+    [OMPI_OP_BASE_FORTRAN_MAX] = {
+        C_INTEGER_BASE(max, 3buff),
+#if defined(GENERATE_SVE_CODE)
+        C_INTEGER_EX(max, 3buff),
+#endif  /* defined(GENERATE_SVE_CODE) */
+        FLOATING_POINT(max, 3buff),
+    },
+    /* Corresponds to MPI_MIN */
+    [OMPI_OP_BASE_FORTRAN_MIN] = {
+        C_INTEGER_BASE(min, 3buff),
+#if defined(GENERATE_SVE_CODE)
+        C_INTEGER_EX(min, 3buff),
+#endif  /* defined(GENERATE_SVE_CODE) */
+        FLOATING_POINT(min, 3buff),
+    },
+    /* Corresponds to MPI_SUM */
+    [OMPI_OP_BASE_FORTRAN_SUM] = {
+        C_INTEGER_BASE(sum, 3buff),
+        C_INTEGER_EX(sum, 3buff),
+        FLOATING_POINT(sum, 3buff),
+    },
+    /* Corresponds to MPI_PROD */
+    [OMPI_OP_BASE_FORTRAN_PROD] = {
+        C_INTEGER_BASE(prod, 3buff),
+#if defined(GENERATE_SVE_CODE)
+        C_INTEGER_EX(prod, 3buff),
+#endif /* defined(GENERATE_SVE_CODE) */
+        FLOATING_POINT(prod, 3buff),
+    },
+    /* Corresponds to MPI_LAND */
+    [OMPI_OP_BASE_FORTRAN_LAND] ={
+        NULL,
+    },
+    /* Corresponds to MPI_BAND */
+    [OMPI_OP_BASE_FORTRAN_BAND] = {
+        C_INTEGER_BASE(band, 3buff),
+        C_INTEGER_EX(band, 3buff),
+    },
+    /* Corresponds to MPI_LOR */
+    [OMPI_OP_BASE_FORTRAN_LOR] = {
+        NULL,
+    },
+    /* Corresponds to MPI_BOR */
+    [OMPI_OP_BASE_FORTRAN_BOR] = {
+        C_INTEGER_BASE(bor, 3buff),
+        C_INTEGER_EX(bor, 3buff),
+    },
+    /* Corresponds to MPI_LXOR */
+    [OMPI_OP_BASE_FORTRAN_LXOR] = {
+        NULL,
+    },
+    /* Corresponds to MPI_BXOR */
+    [OMPI_OP_BASE_FORTRAN_BXOR] = {
+        C_INTEGER_BASE(bxor, 3buff),
+        C_INTEGER_EX(bxor, 3buff),
+    },
+    /* Corresponds to MPI_REPLACE */
+    [OMPI_OP_BASE_FORTRAN_REPLACE] = {
+        /* MPI_ACCUMULATE is handled differently than the other
+           reductions, so just zero out its function
+           implementations here to ensure that users don't invoke
+           MPI_REPLACE with any reduction operations other than
+           ACCUMULATE */
+        NULL,
+    },
+};
diff --git a/ompi/mca/op/avx/configure.m4 b/ompi/mca/op/avx/configure.m4
index 72490f5cc7d..29945e52a43 100644
--- a/ompi/mca/op/avx/configure.m4
+++ b/ompi/mca/op/avx/configure.m4
@@ -38,7 +38,13 @@ AC_DEFUN([MCA_ompi_op_avx_CONFIG],[
 
     OPAL_VAR_SCOPE_PUSH([op_avx_cflags_save])
 
-    AS_IF([test "$opal_cv_asm_arch" = "X86_64"],
+    case "${host}" in
+        x86_64-*x32|i?86-*|x86_64*|amd64*)
+            check_avx="yes";;
+        *)
+            check_avx="no";;
+    esac
+    AS_IF([test "$check_avx" = "yes"],
           [AC_LANG_PUSH([C])
 
            #
@@ -51,7 +57,7 @@ AC_DEFUN([MCA_ompi_op_avx_CONFIG],[
                       [AC_LANG_PROGRAM([[#include <immintrin.h>]],
                                        [[
 #if defined(__ICC) && !defined(__AVX512F__)
-#error "icc needs the -m flags to provide the AVX* detection macros
+#error "icc needs the -m flags to provide the AVX* detection macros"
 #endif
     __m512 vA, vB;
     _mm512_add_ps(vA, vB)
@@ -68,7 +74,7 @@ AC_DEFUN([MCA_ompi_op_avx_CONFIG],[
                              [AC_LANG_PROGRAM([[#include <immintrin.h>]],
                                               [[
 #if defined(__ICC) && !defined(__AVX512F__)
-#error "icc needs the -m flags to provide the AVX* detection macros
+#error "icc needs the -m flags to provide the AVX* detection macros"
 #endif
     __m512 vA, vB;
     _mm512_add_ps(vA, vB)
@@ -91,7 +97,7 @@ AC_DEFUN([MCA_ompi_op_avx_CONFIG],[
                              [AC_LANG_PROGRAM([[#include <immintrin.h>]],
                                       [[
 #if defined(__ICC) && !defined(__AVX512F__)
-#error "icc needs the -m flags to provide the AVX* detection macros
+#error "icc needs the -m flags to provide the AVX* detection macros"
 #endif
     int A[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
     __m512i vA = _mm512_loadu_si512((__m512i*)&(A[1]))
@@ -113,7 +119,7 @@ AC_DEFUN([MCA_ompi_op_avx_CONFIG],[
                              [AC_LANG_PROGRAM([[#include <immintrin.h>]],
                                       [[
 #if defined(__ICC) && !defined(__AVX512F__)
-#error "icc needs the -m flags to provide the AVX* detection macros
+#error "icc needs the -m flags to provide the AVX* detection macros"
 #endif
     __m512i vA, vB;
     _mm512_mullo_epi64(vA, vB)
@@ -123,6 +129,28 @@ AC_DEFUN([MCA_ompi_op_avx_CONFIG],[
                               MCA_BUILD_OP_AVX512_FLAGS=""
                               AC_MSG_RESULT([no])])
                          CFLAGS="$op_avx_cflags_save"
+                        ])
+                  #
+                  # Check for combination of AVX512F + AVX512VL
+                  #
+                  AS_IF([test $op_avx512_support -eq 1],
+                        [AC_MSG_CHECKING([if _mm_max_epi64 generates code that can be compiled])
+                         op_avx_cflags_save="$CFLAGS"
+                         CFLAGS="$CFLAGS_WITHOUT_OPTFLAGS -O0 $MCA_BUILD_OP_AVX512_FLAGS"
+                         AC_LINK_IFELSE(
+                             [AC_LANG_PROGRAM([[#include <immintrin.h>]],
+                                      [[
+#if !defined(__AVX512F__) || !defined(__AVX512VL__) || !defined(__AVX512BW__)
+#error "icc needs the -m flags to provide the AVX* detection macros"
+#endif
+    __m128i vA, vB;
+    _mm_max_epi64(vA, vB)
+                                      ]])],
+                             [AC_MSG_RESULT([yes])],
+                             [op_avx512_support=0
+                              MCA_BUILD_OP_AVX512_FLAGS=""
+                              AC_MSG_RESULT([no])])
+                         CFLAGS="$op_avx_cflags_save"
                         ])])
            #
            # Check support for AVX2
@@ -134,7 +162,7 @@ AC_DEFUN([MCA_ompi_op_avx_CONFIG],[
                       [AC_LANG_PROGRAM([[#include <immintrin.h>]],
                               [[
 #if defined(__ICC) && !defined(__AVX2__)
-#error "icc needs the -m flags to provide the AVX* detection macros
+#error "icc needs the -m flags to provide the AVX* detection macros"
 #endif
     __m256i vA, vB, vC;
     vC = _mm256_and_si256(vA, vB)
@@ -150,7 +178,7 @@ AC_DEFUN([MCA_ompi_op_avx_CONFIG],[
                            [AC_LANG_PROGRAM([[#include <immintrin.h>]],
                                    [[
 #if defined(__ICC) && !defined(__AVX2__)
-#error "icc needs the -m flags to provide the AVX* detection macros
+#error "icc needs the -m flags to provide the AVX* detection macros"
 #endif
     __m256i vA, vB, vC;
     vC = _mm256_and_si256(vA, vB)
@@ -173,7 +201,7 @@ AC_DEFUN([MCA_ompi_op_avx_CONFIG],[
                              [AC_LANG_PROGRAM([[#include <immintrin.h>]],
                                       [[
 #if defined(__ICC) && !defined(__AVX2__)
-#error "icc needs the -m flags to provide the AVX* detection macros
+#error "icc needs the -m flags to provide the AVX* detection macros"
 #endif
     int A[8] = {0, 1, 2, 3, 4, 5, 6, 7};
     __m256i vA = _mm256_loadu_si256((__m256i*)&A)
@@ -199,7 +227,7 @@ AC_DEFUN([MCA_ompi_op_avx_CONFIG],[
                       [AC_LANG_PROGRAM([[#include <immintrin.h>]],
                               [[
 #if defined(__ICC) && !defined(__AVX__)
-#error "icc needs the -m flags to provide the AVX* detection macros
+#error "icc needs the -m flags to provide the AVX* detection macros"
 #endif
     __m256 vA, vB, vC;
     vC = _mm256_add_ps(vA, vB)
@@ -217,7 +245,7 @@ AC_DEFUN([MCA_ompi_op_avx_CONFIG],[
                       [AC_LANG_PROGRAM([[#include <immintrin.h>]],
                               [[
 #if defined(__ICC) && !defined(__SSE4_1__)
-#error "icc needs the -m flags to provide the AVX* detection macros
+#error "icc needs the -m flags to provide the AVX* detection macros"
 #endif
     __m128i vA, vB;
     (void)_mm_max_epi8(vA, vB)
@@ -236,7 +264,7 @@ AC_DEFUN([MCA_ompi_op_avx_CONFIG],[
                       [AC_LANG_PROGRAM([[#include <immintrin.h>]],
                               [[
 #if defined(__ICC) && !defined(__SSE3__)
-#error "icc needs the -m flags to provide the AVX* detection macros
+#error "icc needs the -m flags to provide the AVX* detection macros"
 #endif
     int A[4] = {0, 1, 2, 3};
     __m128i vA = _mm_lddqu_si128((__m128i*)&A)
@@ -256,7 +284,7 @@ AC_DEFUN([MCA_ompi_op_avx_CONFIG],[
                                    [[
     __m256 vA, vB, vC;
 #if defined(__ICC) && !defined(__AVX__)
-#error "icc needs the -m flags to provide the AVX* detection macros
+#error "icc needs the -m flags to provide the AVX* detection macros"
 #endif
     vC = _mm256_add_ps(vA, vB)
                             ]])],
@@ -273,7 +301,7 @@ AC_DEFUN([MCA_ompi_op_avx_CONFIG],[
                              [AC_LANG_PROGRAM([[#include <immintrin.h>]],
                                      [[
 #if defined(__ICC) && !defined(__SSE4_1__)
-#error "icc needs the -m flags to provide the AVX* detection macros
+#error "icc needs the -m flags to provide the AVX* detection macros"
 #endif
     __m128i vA, vB;
     (void)_mm_max_epi8(vA, vB)
@@ -287,7 +315,7 @@ AC_DEFUN([MCA_ompi_op_avx_CONFIG],[
                              [AC_LANG_PROGRAM([[#include <immintrin.h>]],
                                  [[
 #if defined(__ICC) && !defined(__SSE3__)
-#error "icc needs the -m flags to provide the AVX* detection macros
+#error "icc needs the -m flags to provide the AVX* detection macros"
 #endif
     int A[4] = {0, 1, 2, 3};
     __m128i vA = _mm_lddqu_si128((__m128i*)&A)
@@ -315,15 +343,15 @@ AC_DEFUN([MCA_ompi_op_avx_CONFIG],[
                        [$op_sse3_support],
                        [SSE3 supported in the current build])
     AM_CONDITIONAL([MCA_BUILD_ompi_op_has_avx512_support],
-                   [test "$op_avx512_support" == "1"])
+                   [test "$op_avx512_support" = "1"])
     AM_CONDITIONAL([MCA_BUILD_ompi_op_has_avx2_support],
-                   [test "$op_avx2_support" == "1"])
+                   [test "$op_avx2_support" = "1"])
     AM_CONDITIONAL([MCA_BUILD_ompi_op_has_avx_support],
-                   [test "$op_avx_support" == "1"])
+                   [test "$op_avx_support" = "1"])
     AM_CONDITIONAL([MCA_BUILD_ompi_op_has_sse41_support],
-                   [test "$op_sse41_support" == "1"])
+                   [test "$op_sse41_support" = "1"])
     AM_CONDITIONAL([MCA_BUILD_ompi_op_has_sse3_support],
-                   [test "$op_sse3_support" == "1"])
+                   [test "$op_sse3_support" = "1"])
     AC_SUBST(MCA_BUILD_OP_AVX512_FLAGS)
     AC_SUBST(MCA_BUILD_OP_AVX2_FLAGS)
     AC_SUBST(MCA_BUILD_OP_AVX_FLAGS)
diff --git a/ompi/mca/op/avx/op_avx_component.c b/ompi/mca/op/avx/op_avx_component.c
index f116e3b8192..a2f01a373e2 100644
--- a/ompi/mca/op/avx/op_avx_component.c
+++ b/ompi/mca/op/avx/op_avx_component.c
@@ -21,7 +21,6 @@
 #include "ompi_config.h"
 
 #include "opal/util/printf.h"
-#include "ompi/include/mpi_portable_platform.h"
 
 #include "ompi/constants.h"
 #include "ompi/op/op.h"
diff --git a/ompi/mca/op/avx/op_avx_functions.c b/ompi/mca/op/avx/op_avx_functions.c
index ef3f0932906..d97d2d96338 100644
--- a/ompi/mca/op/avx/op_avx_functions.c
+++ b/ompi/mca/op/avx/op_avx_functions.c
@@ -32,16 +32,18 @@
  * to a lesser support (AVX512 -> AVX2, AVX2 -> AVX, AVX -> error out).
  */
 #if defined(GENERATE_AVX512_CODE)
+#  define PREPEND _avx512
 #  if defined(__AVX512BW__) && defined(__AVX512F__) && defined(__AVX512VL__)
-#    define PREPEND _avx512
+/* all good */
 #  else
 #    undef GENERATE_AVX512_CODE
 #  endif  /* defined(__AVX512BW__) && defined(__AVX512F__) && defined(__AVX512VL__) */
 #endif  /* defined(GENERATE_AVX512_CODE) */
 
 #if !defined(PREPEND) && defined(GENERATE_AVX2_CODE)
+#  define PREPEND _avx2
 #  if defined(__AVX2__)
-#    define PREPEND _avx2
+/* all good */
 #  else
 #    undef GENERATE_AVX2_CODE
 #  endif  /* defined(__AVX2__) */
@@ -302,7 +304,7 @@ static void OP_CONCAT(ompi_op_avx_2buff_##name##_##type,PREPEND)(const void *_in
 #endif  /* defined(OMPI_MCA_OP_HAVE_AVX512) && (1 == OMPI_MCA_OP_HAVE_AVX512) */
 /**
  * There is no support for 16 to 8 conversion without AVX512BW and AVX512VL, so
- * there is no AVX-only optimized function posible for OP_AVX_AVX2_MUL.
+ * there is no AVX-only optimized function possible for OP_AVX_AVX2_MUL.
  */
 
 /* special case for int8 mul */
@@ -879,7 +881,7 @@ static void OP_CONCAT(ompi_op_avx_3buff_##name##_##type,PREPEND)(const void * re
 #endif  /* defined(OMPI_MCA_OP_HAVE_AVX512) && (1 == OMPI_MCA_OP_HAVE_AVX512) */
 /**
  * There is no support for 16 to 8 conversion without AVX512BW and AVX512VL, so
- * there is no AVX-only optimized function posible for OP_AVX_AVX2_MUL.
+ * there is no AVX-only optimized function possible for OP_AVX_AVX2_MUL.
  */
 
 /* special case for int8 mul */
diff --git a/ompi/mca/op/base/op_base_functions.c b/ompi/mca/op/base/op_base_functions.c
index cf7512a9198..411059e9ccc 100644
--- a/ompi/mca/op/base/op_base_functions.c
+++ b/ompi/mca/op/base/op_base_functions.c
@@ -45,7 +45,7 @@
       int i;                                                             \
       type *a = (type *) in;                                             \
       type *b = (type *) out;                                            \
-      for (i = 0; i < *count; ++i) {                                     \
+      for (i = *count; i > 0; i--) {                                     \
           *(b++) op *(a++);                                              \
       }                                                                  \
   }
@@ -65,7 +65,7 @@
       int i;                                                             \
       type *a = (type *) in;                                             \
       type *b = (type *) out;                                            \
-      for (i = 0; i < *count; ++i) {                                     \
+      for (i = *count; i > 0; i--) {                                     \
           *(b) = current_func(*(b), *(a));                               \
           ++b;                                                           \
           ++a;                                                           \
@@ -93,7 +93,7 @@
         int i;                                                          \
         ompi_op_predefined_##type_name##_t *a = (ompi_op_predefined_##type_name##_t*) in; \
         ompi_op_predefined_##type_name##_t *b = (ompi_op_predefined_##type_name##_t*) out; \
-        for (i = 0; i < *count; ++i, ++a, ++b) {                        \
+        for (i = *count; i > 0; i--, ++a, ++b) {                        \
             if (a->v op b->v) {                                         \
                 b->v = a->v;                                            \
                 b->k = a->k;                                            \
@@ -117,7 +117,7 @@
       int i;                                                             \
       type (*a)[2] = (type (*)[2]) in;                                   \
       type (*b)[2] = (type (*)[2]) out;                                  \
-      for (i = 0; i < *count; ++i, ++a, ++b) {                           \
+      for (i = *count; i > 0; i--, ++a, ++b) {                           \
           (*b)[0] += (*a)[0];                                            \
           (*b)[1] += (*a)[1];                                            \
       }                                                                  \
@@ -138,7 +138,7 @@
       type (*a)[2] = (type (*)[2]) in;                                   \
       type (*b)[2] = (type (*)[2]) out;                                  \
       type c[2];                                                         \
-      for (i = 0; i < *count; ++i, ++a, ++b) {                           \
+      for (i = *count; i > 0; i--, ++a, ++b) {                           \
           c[0] = (*a)[0] * (*b)[0] - (*a)[1] * (*b)[1];                  \
           c[1] = (*a)[0] * (*b)[1] + (*a)[1] * (*b)[0];                  \
           (*b)[0] = c[0];                                                \
@@ -161,6 +161,9 @@ FUNC_FUNC(max,  int32_t,  int32_t)
 FUNC_FUNC(max, uint32_t, uint32_t)
 FUNC_FUNC(max,  int64_t,  int64_t)
 FUNC_FUNC(max, uint64_t, uint64_t)
+FUNC_FUNC(max,  long,  long)
+FUNC_FUNC(max,  unsigned_long, unsigned long)
+
 /* Fortran integer */
 #if OMPI_HAVE_FORTRAN_INTEGER
 FUNC_FUNC(max, fortran_integer, ompi_fortran_integer_t)
@@ -224,6 +227,9 @@ FUNC_FUNC(min,  int32_t,  int32_t)
 FUNC_FUNC(min, uint32_t, uint32_t)
 FUNC_FUNC(min,  int64_t,  int64_t)
 FUNC_FUNC(min, uint64_t, uint64_t)
+FUNC_FUNC(min,  long,  long)
+FUNC_FUNC(min,  unsigned_long, unsigned long)
+
 /* Fortran integer */
 #if OMPI_HAVE_FORTRAN_INTEGER
 FUNC_FUNC(min, fortran_integer, ompi_fortran_integer_t)
@@ -284,6 +290,9 @@ OP_FUNC(sum,  int32_t,  int32_t, +=)
 OP_FUNC(sum, uint32_t, uint32_t, +=)
 OP_FUNC(sum,  int64_t,  int64_t, +=)
 OP_FUNC(sum, uint64_t, uint64_t, +=)
+OP_FUNC(sum,  long,  long, +=)
+OP_FUNC(sum,  unsigned_long, unsigned long, +=)
+
 /* Fortran integer */
 #if OMPI_HAVE_FORTRAN_INTEGER
 OP_FUNC(sum, fortran_integer, ompi_fortran_integer_t, +=)
@@ -353,6 +362,9 @@ OP_FUNC(prod,  int32_t,  int32_t, *=)
 OP_FUNC(prod, uint32_t, uint32_t, *=)
 OP_FUNC(prod,  int64_t,  int64_t, *=)
 OP_FUNC(prod, uint64_t, uint64_t, *=)
+OP_FUNC(prod,  long,  long, *=)
+OP_FUNC(prod,  unsigned_long, unsigned long, *=)
+
 /* Fortran integer */
 #if OMPI_HAVE_FORTRAN_INTEGER
 OP_FUNC(prod, fortran_integer, ompi_fortran_integer_t, *=)
@@ -424,6 +436,9 @@ FUNC_FUNC(land,  int32_t,  int32_t)
 FUNC_FUNC(land, uint32_t, uint32_t)
 FUNC_FUNC(land,  int64_t,  int64_t)
 FUNC_FUNC(land, uint64_t, uint64_t)
+FUNC_FUNC(land,  long,  long)
+FUNC_FUNC(land,  unsigned_long, unsigned long)
+
 /* Logical */
 #if OMPI_HAVE_FORTRAN_LOGICAL
 FUNC_FUNC(land, fortran_logical, ompi_fortran_logical_t)
@@ -446,6 +461,9 @@ FUNC_FUNC(lor,  int32_t,  int32_t)
 FUNC_FUNC(lor, uint32_t, uint32_t)
 FUNC_FUNC(lor,  int64_t,  int64_t)
 FUNC_FUNC(lor, uint64_t, uint64_t)
+FUNC_FUNC(lor,  long,  long)
+FUNC_FUNC(lor,  unsigned_long, unsigned long)
+
 /* Logical */
 #if OMPI_HAVE_FORTRAN_LOGICAL
 FUNC_FUNC(lor, fortran_logical, ompi_fortran_logical_t)
@@ -468,6 +486,10 @@ FUNC_FUNC(lxor,  int32_t,  int32_t)
 FUNC_FUNC(lxor, uint32_t, uint32_t)
 FUNC_FUNC(lxor,  int64_t,  int64_t)
 FUNC_FUNC(lxor, uint64_t, uint64_t)
+FUNC_FUNC(lxor,  long,  long)
+FUNC_FUNC(lxor,  unsigned_long, unsigned long)
+
+
 /* Logical */
 #if OMPI_HAVE_FORTRAN_LOGICAL
 FUNC_FUNC(lxor, fortran_logical, ompi_fortran_logical_t)
@@ -490,6 +512,9 @@ FUNC_FUNC(band,  int32_t,  int32_t)
 FUNC_FUNC(band, uint32_t, uint32_t)
 FUNC_FUNC(band,  int64_t,  int64_t)
 FUNC_FUNC(band, uint64_t, uint64_t)
+FUNC_FUNC(band,  long,  long)
+FUNC_FUNC(band,  unsigned_long, unsigned long)
+
 /* Fortran integer */
 #if OMPI_HAVE_FORTRAN_INTEGER
 FUNC_FUNC(band, fortran_integer, ompi_fortran_integer_t)
@@ -527,6 +552,9 @@ FUNC_FUNC(bor,  int32_t,  int32_t)
 FUNC_FUNC(bor, uint32_t, uint32_t)
 FUNC_FUNC(bor,  int64_t,  int64_t)
 FUNC_FUNC(bor, uint64_t, uint64_t)
+FUNC_FUNC(bor,  long,  long)
+FUNC_FUNC(bor,  unsigned_long, unsigned long)
+
 /* Fortran integer */
 #if OMPI_HAVE_FORTRAN_INTEGER
 FUNC_FUNC(bor, fortran_integer, ompi_fortran_integer_t)
@@ -564,6 +592,9 @@ FUNC_FUNC(bxor,  int32_t,  int32_t)
 FUNC_FUNC(bxor, uint32_t, uint32_t)
 FUNC_FUNC(bxor,  int64_t,  int64_t)
 FUNC_FUNC(bxor, uint64_t, uint64_t)
+FUNC_FUNC(bxor,  long,  long)
+FUNC_FUNC(bxor,  unsigned_long, unsigned long)
+
 /* Fortran integer */
 #if OMPI_HAVE_FORTRAN_INTEGER
 FUNC_FUNC(bxor, fortran_integer, ompi_fortran_integer_t)
@@ -605,6 +636,7 @@ LOC_STRUCT(long_int, long, int)
 LOC_STRUCT(2int, int, int)
 LOC_STRUCT(short_int, short, int)
 LOC_STRUCT(long_double_int, long double, int)
+LOC_STRUCT(unsigned_long, unsigned long, int)
 
 /*************************************************************************
  * Max location
@@ -661,7 +693,7 @@ LOC_FUNC(minloc, long_double_int, <)
         type *a1 = (type *) in1;                                        \
         type *a2 = (type *) in2;                                        \
         type *b = (type *) out;                                         \
-        for (i = 0; i < *count; ++i) {                                  \
+        for (i = *count; i > 0; i--) {                                  \
             *(b++) =  *(a1++) op *(a2++);                               \
         }                                                               \
     }
@@ -683,7 +715,7 @@ LOC_FUNC(minloc, long_double_int, <)
         type *a1 = (type *) in1;                                        \
         type *a2 = (type *) in2;                                        \
         type *b = (type *) out;                                         \
-        for (i = 0; i < *count; ++i) {                                  \
+        for (i = *count; i > 0; i--) {                                  \
             *(b) = current_func(*(a1), *(a2));                          \
             ++b;                                                        \
             ++a1;                                                       \
@@ -716,7 +748,7 @@ LOC_FUNC(minloc, long_double_int, <)
       ompi_op_predefined_##type_name##_t *a1 = (ompi_op_predefined_##type_name##_t*) in1; \
       ompi_op_predefined_##type_name##_t *a2 = (ompi_op_predefined_##type_name##_t*) in2; \
       ompi_op_predefined_##type_name##_t *b = (ompi_op_predefined_##type_name##_t*) out; \
-      for (i = 0; i < *count; ++i, ++a1, ++a2, ++b ) {                  \
+      for (i = *count; i > 0; i--, ++a1, ++a2, ++b ) {                  \
           if (a1->v op a2->v) {                                         \
               b->v = a1->v;                                             \
               b->k = a1->k;                                             \
@@ -746,7 +778,7 @@ LOC_FUNC(minloc, long_double_int, <)
       type (*a1)[2] = (type (*)[2]) in1;                                 \
       type (*a2)[2] = (type (*)[2]) in2;                                 \
       type (*b)[2] = (type (*)[2]) out;                                  \
-      for (i = 0; i < *count; ++i, ++a1, ++a2, ++b) {                    \
+      for (i = *count; i > 0; i--, ++a1, ++a2, ++b) {                    \
           (*b)[0] = (*a1)[0] + (*a2)[0];                                 \
           (*b)[1] = (*a1)[1] + (*a2)[1];                                 \
       }                                                                  \
@@ -768,7 +800,7 @@ LOC_FUNC(minloc, long_double_int, <)
       type (*a1)[2] = (type (*)[2]) in1;                                 \
       type (*a2)[2] = (type (*)[2]) in2;                                 \
       type (*b)[2] = (type (*)[2]) out;                                  \
-      for (i = 0; i < *count; ++i, ++a1, ++a2, ++b) {                    \
+      for (i = *count; i > 0; i--, ++a1, ++a2, ++b) {                    \
           (*b)[0] = (*a1)[0] * (*a2)[0] - (*a1)[1] * (*a2)[1];           \
           (*b)[1] = (*a1)[0] * (*a2)[1] + (*a1)[1] * (*a2)[0];           \
       }                                                                  \
@@ -789,6 +821,9 @@ FUNC_FUNC_3BUF(max,  int32_t,  int32_t)
 FUNC_FUNC_3BUF(max, uint32_t, uint32_t)
 FUNC_FUNC_3BUF(max,  int64_t,  int64_t)
 FUNC_FUNC_3BUF(max, uint64_t, uint64_t)
+FUNC_FUNC_3BUF(max,  long,  long)
+FUNC_FUNC_3BUF(max,  unsigned_long, unsigned long)
+
 /* Fortran integer */
 #if OMPI_HAVE_FORTRAN_INTEGER
 FUNC_FUNC_3BUF(max, fortran_integer, ompi_fortran_integer_t)
@@ -852,6 +887,9 @@ FUNC_FUNC_3BUF(min,  int32_t,  int32_t)
 FUNC_FUNC_3BUF(min, uint32_t, uint32_t)
 FUNC_FUNC_3BUF(min,  int64_t,  int64_t)
 FUNC_FUNC_3BUF(min, uint64_t, uint64_t)
+FUNC_FUNC_3BUF(min,  long,  long)
+FUNC_FUNC_3BUF(min,  unsigned_long, unsigned long)
+
 /* Fortran integer */
 #if OMPI_HAVE_FORTRAN_INTEGER
 FUNC_FUNC_3BUF(min, fortran_integer, ompi_fortran_integer_t)
@@ -912,6 +950,9 @@ OP_FUNC_3BUF(sum,  int32_t,  int32_t, +)
 OP_FUNC_3BUF(sum, uint32_t, uint32_t, +)
 OP_FUNC_3BUF(sum,  int64_t,  int64_t, +)
 OP_FUNC_3BUF(sum, uint64_t, uint64_t, +)
+OP_FUNC_3BUF(sum,  long,  long, +)
+OP_FUNC_3BUF(sum,  unsigned_long, unsigned long, +)
+
 /* Fortran integer */
 #if OMPI_HAVE_FORTRAN_INTEGER
 OP_FUNC_3BUF(sum, fortran_integer, ompi_fortran_integer_t, +)
@@ -981,6 +1022,9 @@ OP_FUNC_3BUF(prod,  int32_t,  int32_t, *)
 OP_FUNC_3BUF(prod, uint32_t, uint32_t, *)
 OP_FUNC_3BUF(prod,  int64_t,  int64_t, *)
 OP_FUNC_3BUF(prod, uint64_t, uint64_t, *)
+OP_FUNC_3BUF(prod,  long,  long, *)
+OP_FUNC_3BUF(prod,  unsigned_long, unsigned long, *)
+
 /* Fortran integer */
 #if OMPI_HAVE_FORTRAN_INTEGER
 OP_FUNC_3BUF(prod, fortran_integer, ompi_fortran_integer_t, *)
@@ -1052,6 +1096,9 @@ FUNC_FUNC_3BUF(land,  int32_t,  int32_t)
 FUNC_FUNC_3BUF(land, uint32_t, uint32_t)
 FUNC_FUNC_3BUF(land,  int64_t,  int64_t)
 FUNC_FUNC_3BUF(land, uint64_t, uint64_t)
+FUNC_FUNC_3BUF(land,  long,  long)
+FUNC_FUNC_3BUF(land,  unsigned_long, unsigned long)
+
 /* Logical */
 #if OMPI_HAVE_FORTRAN_LOGICAL
 FUNC_FUNC_3BUF(land, fortran_logical, ompi_fortran_logical_t)
@@ -1074,6 +1121,9 @@ FUNC_FUNC_3BUF(lor,  int32_t,  int32_t)
 FUNC_FUNC_3BUF(lor, uint32_t, uint32_t)
 FUNC_FUNC_3BUF(lor,  int64_t,  int64_t)
 FUNC_FUNC_3BUF(lor, uint64_t, uint64_t)
+FUNC_FUNC_3BUF(lor,  long,  long)
+FUNC_FUNC_3BUF(lor,  unsigned_long, unsigned long)
+
 /* Logical */
 #if OMPI_HAVE_FORTRAN_LOGICAL
 FUNC_FUNC_3BUF(lor, fortran_logical, ompi_fortran_logical_t)
@@ -1096,6 +1146,9 @@ FUNC_FUNC_3BUF(lxor,  int32_t,  int32_t)
 FUNC_FUNC_3BUF(lxor, uint32_t, uint32_t)
 FUNC_FUNC_3BUF(lxor,  int64_t,  int64_t)
 FUNC_FUNC_3BUF(lxor, uint64_t, uint64_t)
+FUNC_FUNC_3BUF(lxor,  long,  long)
+FUNC_FUNC_3BUF(lxor,  unsigned_long, unsigned long)
+
 /* Logical */
 #if OMPI_HAVE_FORTRAN_LOGICAL
 FUNC_FUNC_3BUF(lxor, fortran_logical, ompi_fortran_logical_t)
@@ -1118,6 +1171,9 @@ FUNC_FUNC_3BUF(band,  int32_t,  int32_t)
 FUNC_FUNC_3BUF(band, uint32_t, uint32_t)
 FUNC_FUNC_3BUF(band,  int64_t,  int64_t)
 FUNC_FUNC_3BUF(band, uint64_t, uint64_t)
+FUNC_FUNC_3BUF(band,  long,  long)
+FUNC_FUNC_3BUF(band,  unsigned_long, unsigned long)
+
 /* Fortran integer */
 #if OMPI_HAVE_FORTRAN_INTEGER
 FUNC_FUNC_3BUF(band, fortran_integer, ompi_fortran_integer_t)
@@ -1155,6 +1211,9 @@ FUNC_FUNC_3BUF(bor,  int32_t,  int32_t)
 FUNC_FUNC_3BUF(bor, uint32_t, uint32_t)
 FUNC_FUNC_3BUF(bor,  int64_t,  int64_t)
 FUNC_FUNC_3BUF(bor, uint64_t, uint64_t)
+FUNC_FUNC_3BUF(bor,  long,  long)
+FUNC_FUNC_3BUF(bor,  unsigned_long, unsigned long)
+
 /* Fortran integer */
 #if OMPI_HAVE_FORTRAN_INTEGER
 FUNC_FUNC_3BUF(bor, fortran_integer, ompi_fortran_integer_t)
@@ -1192,6 +1251,9 @@ FUNC_FUNC_3BUF(bxor,  int32_t,  int32_t)
 FUNC_FUNC_3BUF(bxor, uint32_t, uint32_t)
 FUNC_FUNC_3BUF(bxor,  int64_t,  int64_t)
 FUNC_FUNC_3BUF(bxor, uint64_t, uint64_t)
+FUNC_FUNC_3BUF(bxor,  long,  long)
+FUNC_FUNC_3BUF(bxor,  unsigned_long, unsigned long)
+
 /* Fortran integer */
 #if OMPI_HAVE_FORTRAN_INTEGER
 FUNC_FUNC_3BUF(bxor, fortran_integer, ompi_fortran_integer_t)
@@ -1281,11 +1343,11 @@ LOC_FUNC_3BUF(minloc, long_double_int, <)
  *
  * **NOTE** These #define's used to be strictly ordered but the use of
  * designated initializers removed this restrictions. When adding new
- * operators ALWAYS use a designated initalizer!
+ * operators ALWAYS use a designated initializer!
  */
 
 /** C integer ***********************************************************/
-#define C_INTEGER(name, ftype)                                              \
+#define C_INTEGER(name, ftype)                                             \
   [OMPI_OP_BASE_TYPE_INT8_T] = ompi_op_base_##ftype##_##name##_int8_t,     \
   [OMPI_OP_BASE_TYPE_UINT8_T] = ompi_op_base_##ftype##_##name##_uint8_t,   \
   [OMPI_OP_BASE_TYPE_INT16_T] = ompi_op_base_##ftype##_##name##_int16_t,   \
@@ -1293,6 +1355,8 @@ LOC_FUNC_3BUF(minloc, long_double_int, <)
   [OMPI_OP_BASE_TYPE_INT32_T] = ompi_op_base_##ftype##_##name##_int32_t,   \
   [OMPI_OP_BASE_TYPE_UINT32_T] = ompi_op_base_##ftype##_##name##_uint32_t, \
   [OMPI_OP_BASE_TYPE_INT64_T] = ompi_op_base_##ftype##_##name##_int64_t,   \
+  [OMPI_OP_BASE_TYPE_LONG] = ompi_op_base_##ftype##_##name##_long,   \
+  [OMPI_OP_BASE_TYPE_UNSIGNED_LONG] = ompi_op_base_##ftype##_##name##_unsigned_long,   \
   [OMPI_OP_BASE_TYPE_UINT64_T] = ompi_op_base_##ftype##_##name##_uint64_t
 
 /** All the Fortran integers ********************************************/
@@ -1560,7 +1624,7 @@ ompi_op_base_handler_fn_t ompi_op_base_functions[OMPI_OP_BASE_FORTRAN_OP_MAX][OM
         [OMPI_OP_BASE_FORTRAN_REPLACE] = {
             /* (MPI_ACCUMULATE is handled differently than the other
                reductions, so just zero out its function
-               impementations here to ensure that users don't invoke
+               implementations here to ensure that users don't invoke
                MPI_REPLACE with any reduction operations other than
                ACCUMULATE) */
             NULL,
@@ -1647,7 +1711,7 @@ ompi_op_base_3buff_handler_fn_t ompi_op_base_3buff_functions[OMPI_OP_BASE_FORTRA
         [OMPI_OP_BASE_FORTRAN_REPLACE] = {
             /* MPI_ACCUMULATE is handled differently than the other
                reductions, so just zero out its function
-               impementations here to ensure that users don't invoke
+               implementations here to ensure that users don't invoke
                MPI_REPLACE with any reduction operations other than
                ACCUMULATE */
             NULL,
diff --git a/ompi/mca/op/base/op_base_op_select.c b/ompi/mca/op/base/op_base_op_select.c
index 837bc51580c..c032172bf19 100644
--- a/ompi/mca/op/base/op_base_op_select.c
+++ b/ompi/mca/op/base/op_base_op_select.c
@@ -14,7 +14,7 @@
  *                         rights reserved.
  * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
  * Copyright (c) 2008-2015 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2020      Research Organization for Information Science
+ * Copyright (c) 2020-2024 Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
  * $COPYRIGHT$
  *
@@ -163,7 +163,7 @@ int ompi_op_base_op_select(ompi_op_t *op)
 
             /* 3-buffer variants */
             if (NULL != avail->ao_module->opm_3buff_fns[i]) {
-                OBJ_RELEASE(op->o_func.intrinsic.modules[i]);
+                OBJ_RELEASE(op->o_3buff_intrinsic.modules[i]);
                 op->o_3buff_intrinsic.fns[i] =
                     avail->ao_module->opm_3buff_fns[i];
                 op->o_3buff_intrinsic.modules[i] = avail->ao_module;
@@ -192,7 +192,7 @@ int ompi_op_base_op_select(ompi_op_t *op)
             /* Oops -- we found a mismatch.  This shouldn't happen; so
                go release everything and return an error (yes, re-use
                the "i" index because we're going to return without
-               completing the outter loop). */
+               completing the outer loop). */
             for (i = 0; i < OMPI_OP_BASE_TYPE_MAX; ++i) {
                 OBJ_RELEASE(op->o_func.intrinsic.modules[i]);
                 op->o_func.intrinsic.modules[i] = NULL;
diff --git a/ompi/mca/op/example/README.md b/ompi/mca/op/example/README.md
index 15ffc68cb11..7999cff389b 100644
--- a/ompi/mca/op/example/README.md
+++ b/ompi/mca/op/example/README.md
@@ -10,9 +10,9 @@ programming examples, there are many different ways to program the
 same end effect.  Feel free to customize / simplify / strip out what
 you don't need from this example.
 
-This example component supports a ficticious set of hardware that
+This example component supports a fictitious set of hardware that
 provides acceleation for the `MPI_MAX` and `MPI_BXOR` `MPI_Ops`.  The
-ficticious hardware has multiple versions, too: some versions only
+fictitious hardware has multiple versions, too: some versions only
 support single precision floating point types for `MAX` and single
 precision integer types for `BXOR`, whereas later versions support
 both single and double precision floating point types for `MAX` and
diff --git a/ompi/mca/op/example/configure.m4 b/ompi/mca/op/example/configure.m4
index a15970ccdc2..9290b54ae8a 100644
--- a/ompi/mca/op/example/configure.m4
+++ b/ompi/mca/op/example/configure.m4
@@ -27,8 +27,8 @@
 
 # Do *NOT* invoke AC_MSG_ERROR, or any other macro that will abort
 # configure, except upon catastrophic error.  For example, it *is* a
-# catastropic error if the user specifically requested your component
-# but it cannot be built.  If it *not* a catastropic error if your
+# catastrophic error if the user specifically requested your component
+# but it cannot be built.  If it *not* a catastrophic error if your
 # component cannot be built (but was not specifically requested).
 
 # See https://github.com/open-mpi/ompi/wiki/devel-CreateComponent
diff --git a/ompi/mca/op/op.h b/ompi/mca/op/op.h
index f81127a74f4..34d26376ab9 100644
--- a/ompi/mca/op/op.h
+++ b/ompi/mca/op/op.h
@@ -191,6 +191,11 @@ enum {
     /** 2 location C: long double int */
     OMPI_OP_BASE_TYPE_LONG_DOUBLE_INT,
 
+    /** long */
+    OMPI_OP_BASE_TYPE_LONG,
+    /** unsigned long */
+    OMPI_OP_BASE_TYPE_UNSIGNED_LONG,
+
     /** 2 location C: wchar_t */
     OMPI_OP_BASE_TYPE_WCHAR,
 
@@ -280,7 +285,7 @@ typedef ompi_op_base_3buff_handler_fn_1_0_0_t ompi_op_base_3buff_handler_fn_t;
  * MPI_INIT.
  *
  * @note The component framework is not lazily opened, so attempts
- * should be made to minimze the amount of memory allocated during
+ * should be made to minimize the amount of memory allocated during
  * this function.
  *
  * @param[in] enable_progress_threads True if the component needs to
@@ -336,14 +341,14 @@ typedef struct ompi_op_base_component_1_0_0_t {
 
     /** Component initialization function */
     ompi_op_base_component_init_query_fn_t opc_init_query;
-    /** Query whether component is useable for given op */
+    /** Query whether component is usable for given op */
     ompi_op_base_component_op_query_1_0_0_fn_t opc_op_query;
 } ompi_op_base_component_1_0_0_t;
 
 
-/** Per guidence in mca.h, use the unversioned struct name if you just
+/** Per guidance in mca.h, use the unversioned struct name if you just
     want to always keep up with the most recent version of the
-    interace. */
+    interface. */
 typedef struct ompi_op_base_component_1_0_0_t ompi_op_base_component_t;
 
 /**
diff --git a/ompi/mca/osc/base/base.h b/ompi/mca/osc/base/base.h
index 1445510ee65..8b9f7b525b2 100644
--- a/ompi/mca/osc/base/base.h
+++ b/ompi/mca/osc/base/base.h
@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2005 The Trustees of Indiana University.
  *                         All rights reserved.
@@ -7,7 +8,10 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2016-2021 IBM Corporation. All rights reserved.
  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
+ * Copyright (c) 2018      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -37,19 +41,30 @@ BEGIN_C_DECLS
 int ompi_osc_base_find_available(bool enable_progress_threads,
                                  bool enable_mpi_threads);
 
+void ompi_osc_base_set_memory_alignment(struct opal_info_t *info,
+                                        size_t *memory_alignment);
+
 int ompi_osc_base_select(ompi_win_t *win,
                          void **base,
                          size_t size,
-                         int disp_unit,
+                         ptrdiff_t disp_unit,
                          ompi_communicator_t *comm,
-                         opal_info_t *info,
                          int flavor,
                          int *model);
 
-int ompi_osc_base_finalize(void);
-
 OMPI_DECLSPEC extern mca_base_framework_t ompi_osc_base_framework;
 
+
+/* Helper to check whether osc can support atomic operation on the size the operands
+ * Currently used with rdma and ucx.
+ */
+static inline __opal_attribute_always_inline__ bool ompi_osc_base_is_atomic_size_supported(uint64_t remote_addr,
+                                                                                           size_t size)
+{
+    return ((sizeof(uint32_t) == size && !(remote_addr & 0x3)) ||
+            (sizeof(uint64_t) == size && !(remote_addr & 0x7)));
+}
+
 END_C_DECLS
 
 #endif
diff --git a/ompi/mca/osc/base/osc_base_frame.c b/ompi/mca/osc/base/osc_base_frame.c
index 24a6a9dc126..8b9d56414fb 100644
--- a/ompi/mca/osc/base/osc_base_frame.c
+++ b/ompi/mca/osc/base/osc_base_frame.c
@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2005 The Trustees of Indiana University.
  *                         All rights reserved.
@@ -9,6 +10,8 @@
  *                         All rights reserved.
  * Copyright (c) 2014      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -22,7 +25,7 @@
 #include "ompi/mca/mca.h"
 #include "opal/util/output.h"
 #include "opal/mca/base/base.h"
-
+#include "opal/include/opal/align.h"
 
 #include "ompi/mca/osc/osc.h"
 #include "ompi/mca/osc/base/base.h"
@@ -36,6 +39,39 @@
 
 #include "ompi/mca/osc/base/static-components.h"
 
+void
+ompi_osc_base_set_memory_alignment(struct opal_info_t *info,
+                                   size_t *memory_alignment)
+{
+    int flag;
+    opal_cstring_t *align_info_str;
+
+    opal_info_get(info, "mpi_minimum_memory_alignment", &align_info_str, &flag);
+    if (flag) {
+        long long tmp_align = atoll(align_info_str->string);
+        OBJ_RELEASE(align_info_str);
+        if ((long long) OPAL_ALIGN_MIN < tmp_align) {
+            *memory_alignment = tmp_align;
+        }
+    }
+}
+
+static int ompi_osc_base_finalize(void)
+{
+    opal_list_item_t* item;
+
+    /* Finalize all available modules */
+    while (NULL !=
+           (item = opal_list_remove_first(&ompi_osc_base_framework.framework_components))) {
+        ompi_osc_base_component_t *component = (ompi_osc_base_component_t*)
+            ((mca_base_component_list_item_t*) item)->cli_component;
+        component->osc_finalize();
+        OBJ_RELEASE(item);
+    }
+    return OMPI_SUCCESS;
+}
+
+
 int
 ompi_osc_base_find_available(bool enable_progress_threads,
                             bool enable_mpi_threads)
@@ -56,22 +92,9 @@ ompi_osc_base_find_available(bool enable_progress_threads,
             OBJ_RELEASE(cli);
         }
     }
-    return OMPI_SUCCESS;
-}
 
-int
-ompi_osc_base_finalize(void)
-{
-    opal_list_item_t* item;
+    ompi_mpi_instance_append_finalize (ompi_osc_base_finalize);
 
-    /* Finalize all available modules */
-    while (NULL !=
-           (item = opal_list_remove_first(&ompi_osc_base_framework.framework_components))) {
-        ompi_osc_base_component_t *component = (ompi_osc_base_component_t*)
-            ((mca_base_component_list_item_t*) item)->cli_component;
-        component->osc_finalize();
-        OBJ_RELEASE(item);
-    }
     return OMPI_SUCCESS;
 }
 
diff --git a/ompi/mca/osc/base/osc_base_init.c b/ompi/mca/osc/base/osc_base_init.c
index 02229c51060..5dea1fae8fa 100644
--- a/ompi/mca/osc/base/osc_base_init.c
+++ b/ompi/mca/osc/base/osc_base_init.c
@@ -34,9 +34,8 @@ int
 ompi_osc_base_select(ompi_win_t *win,
                      void **base,
                      size_t size,
-                     int disp_unit,
+                     ptrdiff_t disp_unit,
                      ompi_communicator_t *comm,
-                     opal_info_t *info,
                      int flavor,
                      int *model)
 {
@@ -55,12 +54,9 @@ ompi_osc_base_select(ompi_win_t *win,
         ompi_osc_base_component_t *component = (ompi_osc_base_component_t*)
             ((mca_base_component_list_item_t*) item)->cli_component;
 
-        priority = component->osc_query(win, base, size, disp_unit, comm, info, flavor);
+        priority = component->osc_query(win, base, size, disp_unit, comm,
+                                        win->super.s_info, flavor);
         if (priority < 0) {
-            if (MPI_WIN_FLAVOR_SHARED == flavor && OMPI_ERR_RMA_SHARED == priority) {
-                /* NTH: quick fix to return OMPI_ERR_RMA_SHARED */
-                return OMPI_ERR_RMA_SHARED;
-            }
             continue;
         }
 
@@ -86,5 +82,6 @@ ompi_osc_base_select(ompi_win_t *win,
                          "select: component %s selected",
                          best_component->osc_version.mca_component_name );
 
-    return best_component->osc_select(win, base, size, disp_unit, comm, info, flavor, model);
+    return best_component->osc_select(win, base, size, disp_unit, comm,
+                                      win->super.s_info, flavor, model);
 }
diff --git a/ompi/mca/osc/base/osc_base_obj_convert.c b/ompi/mca/osc/base/osc_base_obj_convert.c
index 298b8c056c8..fee9c8120db 100644
--- a/ompi/mca/osc/base/osc_base_obj_convert.c
+++ b/ompi/mca/osc/base/osc_base_obj_convert.c
@@ -213,7 +213,7 @@ int ompi_osc_base_sndrcv_op (const void *origin, int32_t origin_count,
 
         done = opal_convertor_raw (&origin_convertor, origin_iovec, &origin_iov_count, &origin_size);
 
-        /* loop on the target segments until we have exhaused the decoded source data */
+        /* loop on the target segments until we have exhausted the decoded source data */
         while (origin_iov_index != origin_iov_count) {
             if (target_iov_index == target_iov_count) {
                 /* decode segments of the target buffer */
diff --git a/ompi/mca/osc/base/osc_base_obj_convert.h b/ompi/mca/osc/base/osc_base_obj_convert.h
index 86b2bc63452..e64986e71cf 100644
--- a/ompi/mca/osc/base/osc_base_obj_convert.h
+++ b/ompi/mca/osc/base/osc_base_obj_convert.h
@@ -35,7 +35,7 @@ BEGIN_C_DECLS
 /**
  * Create datatype based on packed payload
  *
- * Create a useable MPI datatype based on it's packed description.
+ * Create a usable MPI datatype based on its packed description.
  * The datatype is owned by the calling process and must be
  * OBJ_RELEASEd when no longer in use.
  *
@@ -44,7 +44,7 @@ BEGIN_C_DECLS
  *                    pointer to the payload will be moved past the
  *                    datatype information upon successful return.
  *
- * @retval NULL       A failure occrred
+ * @retval NULL       A failure occurred
  * @retval non-NULL   A fully operational datatype
  */
 static inline
@@ -61,14 +61,14 @@ ompi_osc_base_datatype_create(ompi_proc_t *remote_proc,  void **payload)
 /**
  * Create operation based on Fortran Index
  *
- * Create a useable MPI operation based on it's Fortran index, which is
+ * Create a usable MPI operation based on its Fortran index, which is
  * globally the same for predefined operations.  The op handle is
  * owned by the calling process and must be OBJ_RELEASEd when no
  * longer in use.
  *
- * @param op_id       The fortran index for the operaton
+ * @param op_id       The fortran index for the operation
  *
- * @retval NULL       A failure occrred
+ * @retval NULL       A failure occurred
  * @retval non-NULL   An op handle
  */
 static inline
@@ -102,7 +102,7 @@ OMPI_DECLSPEC int ompi_osc_base_get_primitive_type_info(ompi_datatype_t *datatyp
 
 
 /**
- * Apply the operation specified from inbuf to outbut
+ * Apply the operation specified from inbuf to outbuf
  *
  * Apply the specified reduction operation from inbuf to outbuf.
  * inbuf must contain count instances of datatype, in the local
diff --git a/ompi/mca/osc/monitoring/configure.m4 b/ompi/mca/osc/monitoring/configure.m4
index f3cd355b8d7..8ef5dc01e27 100644
--- a/ompi/mca/osc/monitoring/configure.m4
+++ b/ompi/mca/osc/monitoring/configure.m4
@@ -91,10 +91,7 @@ EOF
 AC_DEFUN(
     [MCA_ompi_osc_monitoring_CONFIG],
     [AC_CONFIG_FILES([ompi/mca/osc/monitoring/Makefile])
-
-     AS_IF([test "$MCA_BUILD_ompi_common_monitoring_DSO_TRUE" = ''],
-           [$1],
-           [$2])
+    [$1]
 
      MCA_OMPI_OSC_MONITORING_GENERATE_TEMPLATES(
          [ompi/mca/osc/monitoring/osc_monitoring_template_gen.h],
diff --git a/ompi/mca/osc/monitoring/osc_monitoring_accumulate.h b/ompi/mca/osc/monitoring/osc_monitoring_accumulate.h
index 5dcf6550760..ae678fbeaed 100644
--- a/ompi/mca/osc/monitoring/osc_monitoring_accumulate.h
+++ b/ompi/mca/osc/monitoring/osc_monitoring_accumulate.h
@@ -29,7 +29,7 @@
         int world_rank;                                                 \
         /**                                                             \
          * If this fails the destination is not part of my MPI_COM_WORLD \
-         * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank \
+         * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank \
          */                                                             \
         if(OPAL_SUCCESS == mca_common_monitoring_get_world_rank(target_rank, win->w_group, &world_rank)) { \
             size_t type_size;                                           \
@@ -42,21 +42,21 @@
     }                                                                   \
                                                                         \
     static int ompi_osc_monitoring_## template ##_get_accumulate (const void *origin_addr, \
-                                                                  int origin_count, \
+                                                                  size_t origin_count, \
                                                                   ompi_datatype_t*origin_datatype, \
                                                                   void *result_addr, \
-                                                                  int result_count, \
+                                                                  size_t result_count, \
                                                                   ompi_datatype_t*result_datatype, \
                                                                   int target_rank, \
                                                                   MPI_Aint target_disp, \
-                                                                  int target_count, \
+                                                                  size_t target_count, \
                                                                   ompi_datatype_t*target_datatype, \
                                                                   ompi_op_t *op, ompi_win_t*win) \
     {                                                                   \
         int world_rank;                                                 \
         /**                                                             \
          * If this fails the destination is not part of my MPI_COM_WORLD \
-         * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank \
+         * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank \
          */                                                             \
         if(OPAL_SUCCESS == mca_common_monitoring_get_world_rank(target_rank, win->w_group, &world_rank)) { \
             size_t type_size, data_size;                                \
@@ -72,14 +72,14 @@
     }                                                                   \
                                                                         \
     static int ompi_osc_monitoring_## template ##_rget_accumulate (const void *origin_addr, \
-                                                                   int origin_count, \
+                                                                   size_t origin_count, \
                                                                    ompi_datatype_t *origin_datatype, \
                                                                    void *result_addr, \
-                                                                   int result_count, \
+                                                                   size_t result_count, \
                                                                    ompi_datatype_t *result_datatype, \
                                                                    int target_rank, \
                                                                    MPI_Aint target_disp, \
-                                                                   int target_count, \
+                                                                   size_t target_count, \
                                                                    ompi_datatype_t*target_datatype, \
                                                                    ompi_op_t *op, \
                                                                    ompi_win_t *win, \
@@ -88,7 +88,7 @@
         int world_rank;                                                 \
         /**                                                             \
          * If this fails the destination is not part of my MPI_COM_WORLD \
-         * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank \
+         * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank \
          */                                                             \
         if(OPAL_SUCCESS == mca_common_monitoring_get_world_rank(target_rank, win->w_group, &world_rank)) { \
             size_t type_size, data_size;                                \
@@ -104,11 +104,11 @@
     }                                                                   \
                                                                         \
     static int ompi_osc_monitoring_## template ##_raccumulate (const void *origin_addr, \
-                                                               int origin_count, \
+                                                               size_t origin_count, \
                                                                ompi_datatype_t *origin_datatype, \
                                                                int target_rank, \
                                                                ptrdiff_t target_disp, \
-                                                               int target_count, \
+                                                               size_t target_count, \
                                                                ompi_datatype_t *target_datatype, \
                                                                ompi_op_t *op, ompi_win_t *win, \
                                                                ompi_request_t **request) \
@@ -116,7 +116,7 @@
         int world_rank;                                                 \
         /**                                                             \
          * If this fails the destination is not part of my MPI_COM_WORLD \
-         * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank \
+         * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank \
          */                                                             \
         if(OPAL_SUCCESS == mca_common_monitoring_get_world_rank(target_rank, win->w_group, &world_rank)) { \
             size_t type_size, data_size;                                \
@@ -129,18 +129,18 @@
     }                                                                   \
                                                                         \
     static int ompi_osc_monitoring_## template ##_accumulate (const void *origin_addr, \
-                                                              int origin_count, \
+                                                              size_t origin_count, \
                                                               ompi_datatype_t *origin_datatype, \
                                                               int target_rank, \
                                                               ptrdiff_t target_disp, \
-                                                              int target_count, \
+                                                              size_t target_count, \
                                                               ompi_datatype_t *target_datatype, \
                                                               ompi_op_t *op, ompi_win_t *win) \
     {                                                                   \
         int world_rank;                                                 \
         /**                                                             \
          * If this fails the destination is not part of my MPI_COM_WORLD \
-         * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank \
+         * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank \
          */                                                             \
         if(OPAL_SUCCESS == mca_common_monitoring_get_world_rank(target_rank, win->w_group, &world_rank)) { \
             size_t type_size, data_size;                                \
@@ -162,7 +162,7 @@
         int world_rank;                                                 \
         /**                                                             \
          * If this fails the destination is not part of my MPI_COM_WORLD \
-         * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank \
+         * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank \
          */                                                             \
         if(OPAL_SUCCESS == mca_common_monitoring_get_world_rank(target_rank, win->w_group, &world_rank)) { \
             size_t type_size;                                           \
diff --git a/ompi/mca/osc/monitoring/osc_monitoring_active_target.h b/ompi/mca/osc/monitoring/osc_monitoring_active_target.h
index 645fe3fccd6..53d46d707ac 100644
--- a/ompi/mca/osc/monitoring/osc_monitoring_active_target.h
+++ b/ompi/mca/osc/monitoring/osc_monitoring_active_target.h
@@ -17,14 +17,14 @@
 
 #define OSC_MONITORING_GENERATE_TEMPLATE_ACTIVE_TARGET(template)        \
                                                                         \
-    static int ompi_osc_monitoring_## template ##_post (ompi_group_t *group, int assert, ompi_win_t *win) \
+    static int ompi_osc_monitoring_## template ##_post (ompi_group_t *group, int mpi_assert, ompi_win_t *win) \
     {                                                                   \
-        return OMPI_OSC_MONITORING_MODULE_VARIABLE(template).osc_post(group, assert, win); \
+        return OMPI_OSC_MONITORING_MODULE_VARIABLE(template).osc_post(group, mpi_assert, win); \
     }                                                                   \
                                                                         \
-    static int ompi_osc_monitoring_## template ##_start (ompi_group_t *group, int assert, ompi_win_t *win) \
+    static int ompi_osc_monitoring_## template ##_start (ompi_group_t *group, int mpi_assert, ompi_win_t *win) \
     {                                                                   \
-        return OMPI_OSC_MONITORING_MODULE_VARIABLE(template).osc_start(group, assert, win); \
+        return OMPI_OSC_MONITORING_MODULE_VARIABLE(template).osc_start(group, mpi_assert, win); \
     }                                                                   \
                                                                         \
     static int ompi_osc_monitoring_## template ##_complete (ompi_win_t *win) \
@@ -42,9 +42,9 @@
         return OMPI_OSC_MONITORING_MODULE_VARIABLE(template).osc_test(win, flag); \
     }                                                                   \
                                                                         \
-    static int ompi_osc_monitoring_## template ##_fence (int assert, ompi_win_t *win) \
+    static int ompi_osc_monitoring_## template ##_fence (int mpi_assert, ompi_win_t *win) \
     {                                                                   \
-        return OMPI_OSC_MONITORING_MODULE_VARIABLE(template).osc_fence(assert, win); \
+        return OMPI_OSC_MONITORING_MODULE_VARIABLE(template).osc_fence(mpi_assert, win); \
     }
 
 #endif /* MCA_OSC_MONITORING_ACTIVE_TARGET_H */
diff --git a/ompi/mca/osc/monitoring/osc_monitoring_comm.h b/ompi/mca/osc/monitoring/osc_monitoring_comm.h
index c0ebb5f9383..99c10b5b2f3 100644
--- a/ompi/mca/osc/monitoring/osc_monitoring_comm.h
+++ b/ompi/mca/osc/monitoring/osc_monitoring_comm.h
@@ -19,18 +19,18 @@
 #define OSC_MONITORING_GENERATE_TEMPLATE_COMM(template)                 \
                                                                         \
     static int ompi_osc_monitoring_## template ##_put (const void *origin_addr, \
-                                                       int origin_count, \
+                                                       size_t origin_count, \
                                                        ompi_datatype_t *origin_datatype, \
                                                        int target_rank, \
                                                        ptrdiff_t target_disp, \
-                                                       int target_count, \
+                                                       size_t target_count, \
                                                        ompi_datatype_t *target_datatype, \
                                                        ompi_win_t *win) \
     {                                                                   \
         int world_rank;                                                 \
         /**                                                             \
          * If this fails the destination is not part of my MPI_COM_WORLD \
-         * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank \
+         * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank \
          */                                                             \
         if(OPAL_SUCCESS == mca_common_monitoring_get_world_rank(target_rank, win->w_group, &world_rank)) { \
             size_t type_size, data_size;                                \
@@ -43,11 +43,11 @@
     }                                                                   \
                                                                         \
     static int ompi_osc_monitoring_## template ##_rput (const void *origin_addr, \
-                                                        int origin_count, \
+                                                        size_t origin_count, \
                                                         ompi_datatype_t *origin_datatype, \
                                                         int target_rank, \
                                                         ptrdiff_t target_disp, \
-                                                        int target_count, \
+                                                        size_t target_count, \
                                                         ompi_datatype_t *target_datatype, \
                                                         ompi_win_t *win, \
                                                         ompi_request_t **request) \
@@ -55,7 +55,7 @@
         int world_rank;                                                 \
         /**                                                             \
          * If this fails the destination is not part of my MPI_COM_WORLD \
-         * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank \
+         * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank \
          */                                                             \
         if(OPAL_SUCCESS == mca_common_monitoring_get_world_rank(target_rank, win->w_group, &world_rank)) { \
             size_t type_size, data_size;                                \
@@ -67,18 +67,18 @@
         return OMPI_OSC_MONITORING_MODULE_VARIABLE(template).osc_rput(origin_addr, origin_count, origin_datatype, target_rank, target_disp, target_count, target_datatype, win, request); \
     }                                                                   \
                                                                         \
-    static int ompi_osc_monitoring_## template ##_get (void *origin_addr, int origin_count, \
+    static int ompi_osc_monitoring_## template ##_get (void *origin_addr, size_t origin_count, \
                                                        ompi_datatype_t *origin_datatype, \
                                                        int source_rank, \
                                                        ptrdiff_t source_disp, \
-                                                       int source_count, \
+                                                       size_t source_count, \
                                                        ompi_datatype_t *source_datatype, \
                                                        ompi_win_t *win) \
     {                                                                   \
         int world_rank;                                                 \
         /**                                                             \
          * If this fails the destination is not part of my MPI_COM_WORLD \
-         * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank \
+         * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank \
          */                                                             \
         if(OPAL_SUCCESS == mca_common_monitoring_get_world_rank(source_rank, win->w_group, &world_rank)) { \
             size_t type_size, data_size;                                \
@@ -91,11 +91,11 @@
         return OMPI_OSC_MONITORING_MODULE_VARIABLE(template).osc_get(origin_addr, origin_count, origin_datatype, source_rank, source_disp, source_count, source_datatype, win); \
     }                                                                   \
                                                                         \
-    static int ompi_osc_monitoring_## template ##_rget (void *origin_addr, int origin_count, \
+    static int ompi_osc_monitoring_## template ##_rget (void *origin_addr, size_t origin_count, \
                                                         ompi_datatype_t *origin_datatype, \
                                                         int source_rank, \
                                                         ptrdiff_t source_disp, \
-                                                        int source_count, \
+                                                        size_t source_count, \
                                                         ompi_datatype_t *source_datatype, \
                                                         ompi_win_t *win, \
                                                         ompi_request_t **request) \
@@ -103,7 +103,7 @@
         int world_rank;                                                 \
         /**                                                             \
          * If this fails the destination is not part of my MPI_COM_WORLD \
-         * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank \
+         * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank \
          */                                                             \
         if(OPAL_SUCCESS == mca_common_monitoring_get_world_rank(source_rank, win->w_group, &world_rank)) { \
             size_t type_size, data_size;                                \
diff --git a/ompi/mca/osc/monitoring/osc_monitoring_component.c b/ompi/mca/osc/monitoring/osc_monitoring_component.c
index f91cd8f1160..dcb8a06492d 100644
--- a/ompi/mca/osc/monitoring/osc_monitoring_component.c
+++ b/ompi/mca/osc/monitoring/osc_monitoring_component.c
@@ -47,7 +47,7 @@ static int mca_osc_monitoring_component_register(void)
     return OMPI_SUCCESS;
 }
 
-static int mca_osc_monitoring_component_query(struct ompi_win_t *win, void **base, size_t size, int disp_unit,
+static int mca_osc_monitoring_component_query(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t disp_unit,
                                               struct ompi_communicator_t *comm, struct opal_info_t *info,
                                               int flavor)
 {
@@ -69,7 +69,7 @@ ompi_mca_osc_monitoring_set_template(ompi_osc_base_component_t *best_component,
     return OMPI_ERR_NOT_SUPPORTED;
 }
 
-static int mca_osc_monitoring_component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit,
+static int mca_osc_monitoring_component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t disp_unit,
                                                struct ompi_communicator_t *comm, struct opal_info_t *info,
                                                int flavor, int *model)
 {
@@ -90,10 +90,6 @@ static int mca_osc_monitoring_component_select(struct ompi_win_t *win, void **ba
         
         priority = component->osc_query(win, base, size, disp_unit, comm, info, flavor);
         if (priority < 0) {
-            if (MPI_WIN_FLAVOR_SHARED == flavor && OMPI_ERR_RMA_SHARED == priority) {
-                /* NTH: quick fix to return OMPI_ERR_RMA_SHARED */
-                return OMPI_ERR_RMA_SHARED;
-            }
             continue;
         }
 
@@ -123,7 +119,7 @@ ompi_osc_monitoring_component_t mca_osc_monitoring_component = {
         /* First, the mca_base_component_t struct containing meta
            information about the component itself */
         .osc_version = {
-            OMPI_OSC_BASE_VERSION_3_0_0,
+            OMPI_OSC_BASE_VERSION_4_0_0,
 
             .mca_component_name = "monitoring", /* MCA component name */
             MCA_MONITORING_MAKE_VERSION,
diff --git a/ompi/mca/osc/monitoring/osc_monitoring_passive_target.h b/ompi/mca/osc/monitoring/osc_monitoring_passive_target.h
index 9bf9f01a2f3..4ada1683f69 100644
--- a/ompi/mca/osc/monitoring/osc_monitoring_passive_target.h
+++ b/ompi/mca/osc/monitoring/osc_monitoring_passive_target.h
@@ -41,9 +41,9 @@
         return OMPI_OSC_MONITORING_MODULE_VARIABLE(template).osc_flush_local_all(win); \
     }                                                                   \
                                                                         \
-    static int ompi_osc_monitoring_## template ##_lock (int lock_type, int target, int assert, ompi_win_t *win) \
+    static int ompi_osc_monitoring_## template ##_lock (int lock_type, int target, int mpi_assert, ompi_win_t *win) \
     {                                                                   \
-        return OMPI_OSC_MONITORING_MODULE_VARIABLE(template).osc_lock(lock_type, target, assert, win); \
+        return OMPI_OSC_MONITORING_MODULE_VARIABLE(template).osc_lock(lock_type, target, mpi_assert, win); \
     }                                                                   \
                                                                         \
     static int ompi_osc_monitoring_## template ##_unlock (int target, ompi_win_t *win) \
@@ -51,9 +51,9 @@
         return OMPI_OSC_MONITORING_MODULE_VARIABLE(template).osc_unlock(target, win); \
     }                                                                   \
                                                                         \
-    static int ompi_osc_monitoring_## template ##_lock_all (int assert, struct ompi_win_t *win) \
+    static int ompi_osc_monitoring_## template ##_lock_all (int mpi_assert, struct ompi_win_t *win) \
     {                                                                   \
-        return OMPI_OSC_MONITORING_MODULE_VARIABLE(template).osc_lock_all(assert, win); \
+        return OMPI_OSC_MONITORING_MODULE_VARIABLE(template).osc_lock_all(mpi_assert, win); \
     }                                                                   \
                                                                         \
     static int ompi_osc_monitoring_## template ##_unlock_all (struct ompi_win_t *win) \
diff --git a/ompi/mca/osc/osc.h b/ompi/mca/osc/osc.h
index a7892dfc7b0..82c1879ce93 100644
--- a/ompi/mca/osc/osc.h
+++ b/ompi/mca/osc/osc.h
@@ -115,7 +115,7 @@ typedef int (*ompi_osc_base_component_finalize_fn_t)(void);
 typedef int (*ompi_osc_base_component_query_fn_t)(struct ompi_win_t *win,
                                                   void **base,
                                                   size_t size,
-                                                  int disp_unit,
+                                                  ptrdiff_t disp_unit,
                                                   struct ompi_communicator_t *comm,
                                                   struct opal_info_t *info,
                                                   int flavor);
@@ -147,7 +147,7 @@ typedef int (*ompi_osc_base_component_query_fn_t)(struct ompi_win_t *win,
 typedef int (*ompi_osc_base_component_select_fn_t)(struct ompi_win_t *win,
                                                    void **base,
                                                    size_t size,
-                                                   int disp_unit,
+                                                   ptrdiff_t disp_unit,
                                                    struct ompi_communicator_t *comm,
                                                    struct opal_info_t *info,
                                                    int flavor,
@@ -160,28 +160,28 @@ typedef int (*ompi_osc_base_component_select_fn_t)(struct ompi_win_t *win,
  * this structure, called mca_osc_[component_name]_component, must
  * exist in any OSC component.
  */
-struct ompi_osc_base_component_2_0_0_t {
+struct ompi_osc_base_component_3_0_0_t {
     /** Base component description */
     mca_base_component_t osc_version;
     /** Base component data block */
     mca_base_component_data_t osc_data;
     /** Component initialization function */
     ompi_osc_base_component_init_fn_t osc_init;
-    /** Query whether component is useable for give comm/info */
+    /** Query whether component is usable for given comm/info */
     ompi_osc_base_component_query_fn_t osc_query;
     /** Create module for the given window */
     ompi_osc_base_component_select_fn_t osc_select;
     /* Finalize the component infrastructure */
     ompi_osc_base_component_finalize_fn_t osc_finalize;
 };
-typedef struct ompi_osc_base_component_2_0_0_t ompi_osc_base_component_2_0_0_t;
-typedef ompi_osc_base_component_2_0_0_t ompi_osc_base_component_t;
+typedef struct ompi_osc_base_component_3_0_0_t ompi_osc_base_component_3_0_0_t;
+typedef ompi_osc_base_component_3_0_0_t ompi_osc_base_component_t;
 
 
 /* ******************************************************************** */
 
 typedef int (*ompi_osc_base_module_win_shared_query_fn_t)(struct ompi_win_t *win, int rank,
-                                                          size_t *size, int *disp_unit, void *baseptr);
+                                                          size_t *size, ptrdiff_t *disp_unit, void *baseptr);
 
 typedef int (*ompi_osc_base_module_win_attach_fn_t)(struct ompi_win_t *win, void *base, size_t size);
 typedef int (*ompi_osc_base_module_win_detach_fn_t)(struct ompi_win_t *win, const void *base);
@@ -205,31 +205,31 @@ typedef int (*ompi_osc_base_module_free_fn_t)(struct ompi_win_t *win);
 
 
 typedef int (*ompi_osc_base_module_put_fn_t)(const void *origin_addr,
-                                            int origin_count,
+                                            size_t origin_count,
                                             struct ompi_datatype_t *origin_dt,
                                             int target,
                                             ptrdiff_t target_disp,
-                                            int target_count,
+                                            size_t target_count,
                                             struct ompi_datatype_t *target_dt,
                                             struct ompi_win_t *win);
 
 
 typedef int (*ompi_osc_base_module_get_fn_t)(void *origin_addr,
-                                            int origin_count,
+                                            size_t origin_count,
                                             struct ompi_datatype_t *origin_dt,
                                             int target,
                                             ptrdiff_t target_disp,
-                                            int target_count,
+                                            size_t target_count,
                                             struct ompi_datatype_t *target_dt,
                                             struct ompi_win_t *win);
 
 
 typedef int (*ompi_osc_base_module_accumulate_fn_t)(const void *origin_addr,
-                                                   int origin_count,
+                                                   size_t origin_count,
                                                    struct ompi_datatype_t *origin_dt,
                                                    int target,
                                                    ptrdiff_t target_disp,
-                                                   int target_count,
+                                                   size_t target_count,
                                                    struct ompi_datatype_t *target_dt,
                                                    struct ompi_op_t *op,
                                                    struct ompi_win_t *win);
@@ -251,69 +251,69 @@ typedef int (*ompi_osc_base_module_fetch_and_op_fn_t)(const void *origin_addr,
                                                       struct ompi_win_t *win);
 
 typedef int (*ompi_osc_base_module_get_accumulate_fn_t)(const void *origin_addr,
-                                                        int origin_count,
+                                                        size_t origin_count,
                                                         struct ompi_datatype_t *origin_datatype,
                                                         void *result_addr,
-                                                        int result_count,
+                                                        size_t result_count,
                                                         struct ompi_datatype_t *result_datatype,
                                                         int target_rank,
                                                         ptrdiff_t target_disp,
-                                                        int target_count,
+                                                        size_t target_count,
                                                         struct ompi_datatype_t *target_datatype,
                                                         struct ompi_op_t *op,
                                                         struct ompi_win_t *win);
 
 typedef int (*ompi_osc_base_module_rput_fn_t)(const void *origin_addr,
-                                              int origin_count,
+                                              size_t origin_count,
                                               struct ompi_datatype_t *origin_dt,
                                               int target,
                                               ptrdiff_t target_disp,
-                                              int target_count,
+                                              size_t target_count,
                                               struct ompi_datatype_t *target_dt,
                                               struct ompi_win_t *win,
                                               struct ompi_request_t **request);
 
 typedef int (*ompi_osc_base_module_rget_fn_t)(void *origin_addr,
-                                              int origin_count,
+                                              size_t origin_count,
                                               struct ompi_datatype_t *origin_dt,
                                               int target,
                                               ptrdiff_t target_disp,
-                                              int target_count,
+                                              size_t target_count,
                                               struct ompi_datatype_t *target_dt,
                                               struct ompi_win_t *win,
                                               struct ompi_request_t **request);
 
 
 typedef int (*ompi_osc_base_module_raccumulate_fn_t)(const void *origin_addr,
-                                                     int origin_count,
+                                                     size_t origin_count,
                                                      struct ompi_datatype_t *origin_dt,
                                                      int target,
                                                      ptrdiff_t target_disp,
-                                                     int target_count,
+                                                     size_t target_count,
                                                      struct ompi_datatype_t *target_dt,
                                                      struct ompi_op_t *op,
                                                      struct ompi_win_t *win,
                                                      struct ompi_request_t **request);
 
 typedef int (*ompi_osc_base_module_rget_accumulate_fn_t)(const void *origin_addr,
-                                                         int origin_count,
+                                                         size_t origin_count,
                                                          struct ompi_datatype_t *origin_datatype,
                                                          void *result_addr,
-                                                         int result_count,
+                                                         size_t result_count,
                                                          struct ompi_datatype_t *result_datatype,
                                                          int target_rank,
                                                          ptrdiff_t target_disp,
-                                                         int target_count,
+                                                         size_t target_count,
                                                          struct ompi_datatype_t *target_datatype,
                                                          struct ompi_op_t *op,
                                                          struct ompi_win_t *win,
                                                          struct ompi_request_t **request);
 
-typedef int (*ompi_osc_base_module_fence_fn_t)(int assert, struct ompi_win_t *win);
+typedef int (*ompi_osc_base_module_fence_fn_t)(int mpi_assert, struct ompi_win_t *win);
 
 
 typedef int (*ompi_osc_base_module_start_fn_t)(struct ompi_group_t *group,
-                                              int assert,
+                                              int mpi_assert,
                                               struct ompi_win_t *win);
 
 
@@ -321,7 +321,7 @@ typedef int (*ompi_osc_base_module_complete_fn_t)(struct ompi_win_t *win);
 
 
 typedef int (*ompi_osc_base_module_post_fn_t)(struct ompi_group_t *group,
-                                             int assert,
+                                             int mpi_assert,
                                              struct ompi_win_t *win);
 
 
@@ -334,13 +334,13 @@ typedef int (*ompi_osc_base_module_test_fn_t)(struct ompi_win_t *win,
 
 typedef int (*ompi_osc_base_module_lock_fn_t)(int lock_type,
                                              int target,
-                                             int assert,
+                                             int mpi_assert,
                                              struct ompi_win_t *win);
 
 typedef int (*ompi_osc_base_module_unlock_fn_t)(int target,
                                              struct ompi_win_t *win);
 
-typedef int (*ompi_osc_base_module_lock_all_fn_t)(int assert,
+typedef int (*ompi_osc_base_module_lock_all_fn_t)(int mpi_assert,
                                                   struct ompi_win_t *win);
 
 typedef int (*ompi_osc_base_module_unlock_all_fn_t)(struct ompi_win_t *win);
@@ -367,7 +367,7 @@ typedef int (*ompi_osc_base_module_flush_local_all_fn_t)(struct ompi_win_t *win)
  * free to create a structure that inherits this one for use as the
  * module structure.
  */
-struct ompi_osc_base_module_3_0_0_t {
+struct ompi_osc_base_module_4_0_0_t {
     ompi_osc_base_module_win_shared_query_fn_t osc_win_shared_query;
 
     ompi_osc_base_module_win_attach_fn_t osc_win_attach;
@@ -405,16 +405,16 @@ struct ompi_osc_base_module_3_0_0_t {
     ompi_osc_base_module_flush_local_fn_t osc_flush_local;
     ompi_osc_base_module_flush_local_all_fn_t osc_flush_local_all;
 };
-typedef struct ompi_osc_base_module_3_0_0_t ompi_osc_base_module_3_0_0_t;
-typedef ompi_osc_base_module_3_0_0_t ompi_osc_base_module_t;
+typedef struct ompi_osc_base_module_4_0_0_t ompi_osc_base_module_4_0_0_t;
+typedef ompi_osc_base_module_4_0_0_t ompi_osc_base_module_t;
 
 
 /* ******************************************************************** */
 
 
 /** Macro for use in components that are of type osc */
-#define OMPI_OSC_BASE_VERSION_3_0_0 \
-    OMPI_MCA_BASE_VERSION_2_1_0("osc", 3, 0, 0)
+#define OMPI_OSC_BASE_VERSION_4_0_0 \
+    OMPI_MCA_BASE_VERSION_2_1_0("osc", 4, 0, 0)
 
 
 /* ******************************************************************** */
diff --git a/ompi/mca/osc/portals4/configure.m4 b/ompi/mca/osc/portals4/configure.m4
index 1b33b6607d7..726004d4a61 100644
--- a/ompi/mca/osc/portals4/configure.m4
+++ b/ompi/mca/osc/portals4/configure.m4
@@ -31,7 +31,7 @@ AC_DEFUN([MCA_ompi_osc_portals4_CONFIG],[
            $1],
           [$2])
 
-    # need to propogate CPPFLAGS to all of OMPI
+    # need to propagate CPPFLAGS to all of OMPI
     AS_IF([test "$DIRECT_osc" = "portals4"],
           [CPPFLAGS="$CPPFLAGS $osc_portals4_CPPFLAGS"])
 
diff --git a/ompi/mca/osc/portals4/osc_portals4.h b/ompi/mca/osc/portals4/osc_portals4.h
index 6b58554489f..8497f7e1245 100644
--- a/ompi/mca/osc/portals4/osc_portals4.h
+++ b/ompi/mca/osc/portals4/osc_portals4.h
@@ -16,12 +16,15 @@
 #ifndef OSC_PORTALS4_PORTALS4_H
 #define OSC_PORTALS4_PORTALS4_H
 
-#include <portals4.h>
-#include "ompi/group/group.h"
+#include "ompi_config.h"
+
 #include "ompi/communicator/communicator.h"
+#include "ompi/group/group.h"
 
 #include "ompi/mca/mtl/portals4/mtl_portals4.h"
 
+#include <portals4.h>
+
 #define REQ_OSC_TABLE_ID     4
 
 #define OSC_PORTALS4_IOVEC_MAX 64
@@ -85,8 +88,8 @@ struct ompi_osc_portals4_module_t {
     ompi_osc_base_module_t super;
     void *free_after; /* if non-null, this pointer should be free()ed when window destroyed */
     struct ompi_communicator_t *comm; /* communicator which backs this window (unique to this window) */
-    int disp_unit; /* if -1, have to look at disp_units */
-    int *disp_units; /* array (possibly NULL!) of displacement units, per peer */
+    ptrdiff_t disp_unit; /* if -1, have to look at disp_units */
+    ptrdiff_t *disp_units; /* array (possibly NULL!) of displacement units, per peer */
     ptl_handle_ni_t ni_h; /* network interface used by this window */
     ptl_pt_index_t pt_idx; /* portal table index used by this window (this will be same across window) */
     ptl_handle_ct_t ct_h; /* Counting event handle used for completion in this window */
@@ -141,29 +144,29 @@ int ompi_osc_portals4_detach(struct ompi_win_t *win, const void *base);
 int ompi_osc_portals4_free(struct ompi_win_t *win);
 
 int ompi_osc_portals4_put(const void *origin_addr,
-                          int origin_count,
+                          size_t origin_count,
                           struct ompi_datatype_t *origin_dt,
                           int target,
                           ptrdiff_t target_disp,
-                          int target_count,
+                          size_t target_count,
                           struct ompi_datatype_t *target_dt,
                           struct ompi_win_t *win);
 
 int ompi_osc_portals4_get(void *origin_addr,
-                          int origin_count,
+                          size_t origin_count,
                           struct ompi_datatype_t *origin_dt,
                           int target,
                           ptrdiff_t target_disp,
-                          int target_count,
+                          size_t target_count,
                           struct ompi_datatype_t *target_dt,
                           struct ompi_win_t *win);
 
 int ompi_osc_portals4_accumulate(const void *origin_addr,
-                                 int origin_count,
+                                 size_t origin_count,
                                  struct ompi_datatype_t *origin_dt,
                                  int target,
                                  ptrdiff_t target_disp,
-                                 int target_count,
+                                 size_t target_count,
                                  struct ompi_datatype_t *target_dt,
                                  struct ompi_op_t *op,
                                  struct ompi_win_t *win);
@@ -185,73 +188,73 @@ int ompi_osc_portals4_fetch_and_op(const void *origin_addr,
                                    struct ompi_win_t *win);
 
 int ompi_osc_portals4_get_accumulate(const void *origin_addr,
-                                     int origin_count,
+                                     size_t origin_count,
                                      struct ompi_datatype_t *origin_datatype,
                                      void *result_addr,
-                                     int result_count,
+                                     size_t result_count,
                                      struct ompi_datatype_t *result_datatype,
                                      int target_rank,
                                      ptrdiff_t target_disp,
-                                     int target_count,
+                                     size_t target_count,
                                      struct ompi_datatype_t *target_datatype,
                                      struct ompi_op_t *op,
                                      struct ompi_win_t *win);
 
 int ompi_osc_portals4_rput(const void *origin_addr,
-                           int origin_count,
+                           size_t origin_count,
                            struct ompi_datatype_t *origin_dt,
                            int target,
                            ptrdiff_t target_disp,
-                           int target_count,
+                           size_t target_count,
                            struct ompi_datatype_t *target_dt,
                            struct ompi_win_t *win,
                            struct ompi_request_t **request);
 
 int ompi_osc_portals4_rget(void *origin_addr,
-                           int origin_count,
+                           size_t origin_count,
                            struct ompi_datatype_t *origin_dt,
                            int target,
                            ptrdiff_t target_disp,
-                           int target_count,
+                           size_t target_count,
                            struct ompi_datatype_t *target_dt,
                            struct ompi_win_t *win,
                            struct ompi_request_t **request);
 
 int ompi_osc_portals4_raccumulate(const void *origin_addr,
-                                  int origin_count,
+                                  size_t origin_count,
                                   struct ompi_datatype_t *origin_dt,
                                   int target,
                                   ptrdiff_t target_disp,
-                                  int target_count,
+                                  size_t target_count,
                                   struct ompi_datatype_t *target_dt,
                                   struct ompi_op_t *op,
                                   struct ompi_win_t *win,
                                   struct ompi_request_t **request);
 
 int ompi_osc_portals4_rget_accumulate(const void *origin_addr,
-                                      int origin_count,
+                                      size_t origin_count,
                                       struct ompi_datatype_t *origin_datatype,
                                       void *result_addr,
-                                      int result_count,
+                                      size_t result_count,
                                       struct ompi_datatype_t *result_datatype,
                                       int target_rank,
                                       ptrdiff_t target_disp,
-                                      int target_count,
+                                      size_t target_count,
                                       struct ompi_datatype_t *target_datatype,
                                       struct ompi_op_t *op,
                                       struct ompi_win_t *win,
                                       struct ompi_request_t **request);
 
-int ompi_osc_portals4_fence(int assert, struct ompi_win_t *win);
+int ompi_osc_portals4_fence(int mpi_assert, struct ompi_win_t *win);
 
 int ompi_osc_portals4_start(struct ompi_group_t *group,
-                            int assert,
+                            int mpi_assert,
                             struct ompi_win_t *win);
 
 int ompi_osc_portals4_complete(struct ompi_win_t *win);
 
 int ompi_osc_portals4_post(struct ompi_group_t *group,
-                           int assert,
+                           int mpi_assert,
                            struct ompi_win_t *win);
 
 int ompi_osc_portals4_wait(struct ompi_win_t *win);
@@ -261,14 +264,14 @@ int ompi_osc_portals4_test(struct ompi_win_t *win,
 
 int ompi_osc_portals4_lock(int lock_type,
                            int target,
-                           int assert,
+                           int mpi_assert,
                            struct ompi_win_t *win);
 
 int ompi_osc_portals4_unlock(int target,
                              struct ompi_win_t *win);
 
 
-int ompi_osc_portals4_lock_all(int assert,
+int ompi_osc_portals4_lock_all(int mpi_assert,
                                struct ompi_win_t *win);
 
 int ompi_osc_portals4_unlock_all(struct ompi_win_t *win);
diff --git a/ompi/mca/osc/portals4/osc_portals4_active_target.c b/ompi/mca/osc/portals4/osc_portals4_active_target.c
index 23a763efe8e..5e7f6fa4897 100644
--- a/ompi/mca/osc/portals4/osc_portals4_active_target.c
+++ b/ompi/mca/osc/portals4/osc_portals4_active_target.c
@@ -12,15 +12,15 @@
 
 #include "ompi_config.h"
 
-#include "ompi/mca/osc/osc.h"
 #include "ompi/mca/osc/base/base.h"
 #include "ompi/mca/osc/base/osc_base_obj_convert.h"
+#include "ompi/mca/osc/osc.h"
 
 #include "osc_portals4.h"
 
 
 int
-ompi_osc_portals4_fence(int assert, struct ompi_win_t *win)
+ompi_osc_portals4_fence(int mpi_assert, struct ompi_win_t *win)
 {
     ompi_osc_portals4_module_t *module =
         (ompi_osc_portals4_module_t*) win->w_osc_module;
@@ -42,7 +42,7 @@ ompi_osc_portals4_fence(int assert, struct ompi_win_t *win)
 
 int
 ompi_osc_portals4_start(struct ompi_group_t *group,
-                        int assert,
+                        int mpi_assert,
                         struct ompi_win_t *win)
 {
     ompi_osc_portals4_module_t *module =
@@ -53,7 +53,7 @@ ompi_osc_portals4_start(struct ompi_group_t *group,
         return OMPI_ERR_RMA_SYNC;
     }
 
-    if (0 == (assert & MPI_MODE_NOCHECK)) {
+    if (0 == (mpi_assert & MPI_MODE_NOCHECK)) {
         int size;
 
         OBJ_RETAIN(group);
@@ -115,14 +115,14 @@ ompi_osc_portals4_complete(struct ompi_win_t *win)
 
 int
 ompi_osc_portals4_post(struct ompi_group_t *group,
-                       int assert,
+                       int mpi_assert,
                        struct ompi_win_t *win)
 {
     ompi_osc_portals4_module_t *module =
         (ompi_osc_portals4_module_t*) win->w_osc_module;
     int ret, i, size;
 
-    if (0 == (assert & MPI_MODE_NOCHECK)) {
+    if (0 == (mpi_assert & MPI_MODE_NOCHECK)) {
         OBJ_RETAIN(group);
         module->post_group = group;
 
diff --git a/ompi/mca/osc/portals4/osc_portals4_comm.c b/ompi/mca/osc/portals4/osc_portals4_comm.c
index 6d7c0001044..c12796b3d1a 100644
--- a/ompi/mca/osc/portals4/osc_portals4_comm.c
+++ b/ompi/mca/osc/portals4/osc_portals4_comm.c
@@ -14,9 +14,9 @@
 
 #include "ompi_config.h"
 
-#include "ompi/mca/osc/osc.h"
 #include "ompi/mca/osc/base/base.h"
 #include "ompi/mca/osc/base/osc_base_obj_convert.h"
+#include "ompi/mca/osc/osc.h"
 
 #include "osc_portals4.h"
 #include "osc_portals4_request.h"
@@ -175,7 +175,7 @@ number_of_fragments(ptl_size_t length, ptl_size_t maxlength)
 {
     ptl_size_t nb_frag = length == 0 ? 1 : (length - 1) / maxlength + 1;
     OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
-                         "%s,%d : %ld fragment(s)", __FUNCTION__, __LINE__, nb_frag));
+                         "%s,%d : %ld fragment(s)", __FILE__, __LINE__, nb_frag));
     return nb_frag;
 }
 
@@ -217,7 +217,7 @@ segmentedPut(opal_atomic_int64_t *opcount,
             opal_atomic_add_fetch_64(opcount, -1);
             opal_output_verbose(1, ompi_osc_base_framework.framework_output,
                                  "%s:%d PtlPut failed with return value %d",
-                                 __FUNCTION__, __LINE__, ret);
+                                 __FILE__, __LINE__, ret);
             return ret;
         }
         put_length -= frag_length;
@@ -261,7 +261,7 @@ segmentedGet(opal_atomic_int64_t *opcount,
             opal_atomic_add_fetch_64(opcount, -1);
             opal_output_verbose(1, ompi_osc_base_framework.framework_output,
                                  "%s:%d PtlGet failed with return value %d",
-                                 __FUNCTION__, __LINE__, ret);
+                                 __FILE__, __LINE__, ret);
             return ret;
         }
         get_length -= frag_length;
@@ -310,7 +310,7 @@ segmentedAtomic(opal_atomic_int64_t *opcount,
             opal_atomic_add_fetch_64(opcount, -1);
             opal_output_verbose(1, ompi_osc_base_framework.framework_output,
                                  "%s:%d PtlAtomic failed with return value %d",
-                                 __FUNCTION__, __LINE__, ret);
+                                 __FILE__, __LINE__, ret);
             return ret;
         }
         length -= frag_length;
@@ -362,7 +362,7 @@ segmentedFetchAtomic(opal_atomic_int64_t *opcount,
             opal_atomic_add_fetch_64(opcount, -1);
             opal_output_verbose(1, ompi_osc_base_framework.framework_output,
                                  "%s:%d PtlFetchAtomic failed with return value %d",
-                                 __FUNCTION__, __LINE__, ret);
+                                 __FILE__, __LINE__, ret);
             return ret;
         }
         length -= frag_length;
@@ -414,7 +414,7 @@ segmentedSwap(opal_atomic_int64_t *opcount,
             opal_atomic_add_fetch_64(opcount, -1);
             opal_output_verbose(1, ompi_osc_base_framework.framework_output,
                                  "%s:%d PtlSwap failed with return value %d",
-                                 __FUNCTION__, __LINE__, ret);
+                                 __FILE__, __LINE__, ret);
             return ret;
         }
         length -= frag_length;
@@ -425,7 +425,7 @@ segmentedSwap(opal_atomic_int64_t *opcount,
 
 static int
 create_iov_list(const void       *address,
-                int               count,
+                size_t           count,
                 ompi_datatype_t  *datatype,
                 ptl_iovec_t     **ptl_iovec,
                 ptl_size_t       *ptl_iovec_count)
@@ -481,10 +481,10 @@ create_iov_list(const void       *address,
 static int
 get_to_iovec(ompi_osc_portals4_module_t *module,
              const void       *origin_address,
-             int               origin_count,
+             size_t            origin_count,
              ompi_datatype_t  *origin_datatype,
              ptl_process_t     peer,
-             int               target_count,
+             size_t            target_count,
              ompi_datatype_t  *target_datatype,
              size_t            offset,
              ptl_pt_index_t    pt_index,
@@ -496,7 +496,7 @@ get_to_iovec(ompi_osc_portals4_module_t *module,
     ptrdiff_t length, origin_lb, target_lb, extent;
     ptl_md_t md;
 
-    if (module->origin_iovec_md_h != PTL_INVALID_HANDLE) {
+    if (!PtlHandleIsEqual(module->origin_iovec_md_h,PTL_INVALID_HANDLE)) {
         PtlMDRelease(module->origin_iovec_md_h);
         free(module->origin_iovec_list);
         module->origin_iovec_md_h = PTL_INVALID_HANDLE;
@@ -542,8 +542,8 @@ get_to_iovec(ompi_osc_portals4_module_t *module,
     opal_atomic_add_fetch_64(&module->opcount, 1);
 
     OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
-                 "%s,%d Get(origin_count=%d, origin_lb=%lu, target_count=%d, target_lb=%lu, size=%lu, length=%lu, offset=%lu, op_count=%ld)",
-                 __FUNCTION__, __LINE__, origin_count, origin_lb, target_count, target_lb, size, length, offset, module->opcount));
+                 "%s,%d Get(origin_count=%d, origin_lb=%lu, target_count=%zu, target_lb=%lu, size=%zu, length=%lu, offset=%lu, op_count=%ld)",
+                 __FILE__, __LINE__, origin_count, origin_lb, target_count, target_lb, size, length, offset, module->opcount));
     ret = PtlGet(module->origin_iovec_md_h,
                  (ptl_size_t) origin_lb,
                  length,
@@ -555,7 +555,7 @@ get_to_iovec(ompi_osc_portals4_module_t *module,
     if (PTL_OK != ret) {
         OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
                      "%s,%d PtlGet() failed: ret = %d",
-                     __FUNCTION__, __LINE__, ret));
+                     __FILE__, __LINE__, ret));
         opal_atomic_add_fetch_64(&module->opcount, -1);
         return ret;
     }
@@ -568,10 +568,10 @@ get_to_iovec(ompi_osc_portals4_module_t *module,
 static int
 atomic_get_to_iovec(ompi_osc_portals4_module_t *module,
                     const void       *origin_address,
-                    int               origin_count,
+                    size_t            origin_count,
                     ompi_datatype_t  *origin_datatype,
                     ptl_process_t     peer,
-                    int               target_count,
+                    size_t            target_count,
                     ompi_datatype_t  *target_datatype,
                     size_t            offset,
                     ptl_pt_index_t    pt_index,
@@ -583,7 +583,7 @@ atomic_get_to_iovec(ompi_osc_portals4_module_t *module,
     ptrdiff_t length, origin_lb, target_lb, extent;
     ptl_md_t md;
 
-    if (module->origin_iovec_md_h != PTL_INVALID_HANDLE) {
+    if (!PtlHandleIsEqual(module->origin_iovec_md_h,PTL_INVALID_HANDLE)) {
         PtlMDRelease(module->origin_iovec_md_h);
         free(module->origin_iovec_list);
         module->origin_iovec_md_h = PTL_INVALID_HANDLE;
@@ -627,8 +627,8 @@ atomic_get_to_iovec(ompi_osc_portals4_module_t *module,
     }
 
     OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
-                 "%s,%d Get(origin_count=%d, origin_lb=%lu, target_count=%d, target_lb=%lu, size=%lu, length=%lu, offset=%lu, op_count=%ld)",
-                 __FUNCTION__, __LINE__, origin_count, origin_lb, target_count, target_lb, size, length, offset, module->opcount));
+                 "%s,%d Get(origin_count=%zu, origin_lb=%lu, target_count=%zu, target_lb=%lu, size=%zu, length=%lu, offset=%lu, op_count=%ld)",
+                 __FILE__, __LINE__, origin_count, origin_lb, target_count, target_lb, size, length, offset, module->opcount));
     ret = segmentedGet(&module->opcount,
                        module->origin_iovec_md_h,
                        (ptl_size_t) origin_lb,
@@ -650,10 +650,10 @@ atomic_get_to_iovec(ompi_osc_portals4_module_t *module,
 static int
 put_from_iovec(ompi_osc_portals4_module_t *module,
                const void       *origin_address,
-               int               origin_count,
+               size_t            origin_count,
                ompi_datatype_t  *origin_datatype,
                ptl_process_t     peer,
-               int               target_count,
+               size_t            target_count,
                ompi_datatype_t  *target_datatype,
                size_t            offset,
                ptl_pt_index_t    pt_index,
@@ -665,7 +665,7 @@ put_from_iovec(ompi_osc_portals4_module_t *module,
     ptrdiff_t length, origin_lb, target_lb, extent;
     ptl_md_t md;
 
-    if (module->origin_iovec_md_h != PTL_INVALID_HANDLE) {
+    if (!PtlHandleIsEqual(module->origin_iovec_md_h,PTL_INVALID_HANDLE)) {
         PtlMDRelease(module->origin_iovec_md_h);
         free(module->origin_iovec_list);
         module->origin_iovec_md_h = PTL_INVALID_HANDLE;
@@ -711,8 +711,8 @@ put_from_iovec(ompi_osc_portals4_module_t *module,
     opal_atomic_add_fetch_64(&module->opcount, 1);
 
     OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
-                 "%s,%d Put(origin_count=%d, origin_lb=%lu, target_count=%d, target_lb=%lu, size=%lu, length=%lu, offset=%lu, op_count=%ld)",
-                 __FUNCTION__, __LINE__, origin_count, origin_lb, target_count, target_lb, size, length, offset, module->opcount));
+                 "%s,%d Put(origin_count=%zu, origin_lb=%lu, target_count=%zu, target_lb=%lu, size=%zu, length=%lu, offset=%lu, op_count=%ld)",
+                 __FILE__, __LINE__, origin_count, origin_lb, target_count, target_lb, size, length, offset, module->opcount));
     ret = PtlPut(module->origin_iovec_md_h,
                  (ptl_size_t) origin_lb,
                  length,
@@ -726,7 +726,7 @@ put_from_iovec(ompi_osc_portals4_module_t *module,
     if (PTL_OK != ret) {
         OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
                      "%s,%d PtlPut() failed: ret = %d",
-                     __FUNCTION__, __LINE__, ret));
+                     __FILE__, __LINE__, ret));
         opal_atomic_add_fetch_64(&module->opcount, -1);
         return ret;
     }
@@ -739,10 +739,10 @@ put_from_iovec(ompi_osc_portals4_module_t *module,
 static int
 atomic_put_from_iovec(ompi_osc_portals4_module_t *module,
                       const void       *origin_address,
-                      int               origin_count,
+                      size_t            origin_count,
                       ompi_datatype_t  *origin_datatype,
                       ptl_process_t     peer,
-                      int               target_count,
+                      size_t            target_count,
                       ompi_datatype_t  *target_datatype,
                       size_t            offset,
                       ptl_pt_index_t    pt_index,
@@ -754,7 +754,7 @@ atomic_put_from_iovec(ompi_osc_portals4_module_t *module,
     ptrdiff_t length, origin_lb, target_lb, extent;
     ptl_md_t md;
 
-    if (module->origin_iovec_md_h != PTL_INVALID_HANDLE) {
+    if (!PtlHandleIsEqual(module->origin_iovec_md_h,PTL_INVALID_HANDLE)) {
         PtlMDRelease(module->origin_iovec_md_h);
         free(module->origin_iovec_list);
         module->origin_iovec_md_h = PTL_INVALID_HANDLE;
@@ -798,8 +798,8 @@ atomic_put_from_iovec(ompi_osc_portals4_module_t *module,
     }
 
     OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
-                 "%s,%d Put(origin_count=%d, origin_lb=%lu, target_count=%d, target_lb=%lu, length=%lu, op_count=%ld)",
-                 __FUNCTION__, __LINE__, origin_count, origin_lb, target_count, target_lb, length, module->opcount));
+                 "%s,%d Put(origin_count=%zu, origin_lb=%lu, target_count=%zu, target_lb=%lu, length=%lu, op_count=%ld)",
+                 __FILE__, __LINE__, origin_count, origin_lb, target_count, target_lb, length, module->opcount));
     ret = segmentedPut(&module->opcount,
                        module->origin_iovec_md_h,
                        (ptl_size_t) origin_lb,
@@ -823,10 +823,10 @@ atomic_put_from_iovec(ompi_osc_portals4_module_t *module,
 static int
 atomic_from_iovec(ompi_osc_portals4_module_t *module,
                   const void       *origin_address,
-                  int               origin_count,
+                  size_t            origin_count,
                   ompi_datatype_t  *origin_datatype,
                   ptl_process_t     peer,
-                  int               target_count,
+                  size_t            target_count,
                   ompi_datatype_t  *target_datatype,
                   size_t            offset,
                   ptl_pt_index_t    pt_index,
@@ -841,7 +841,7 @@ atomic_from_iovec(ompi_osc_portals4_module_t *module,
     ptl_op_t ptl_op;
     ptl_datatype_t ptl_dt;
 
-    if (module->origin_iovec_md_h != PTL_INVALID_HANDLE) {
+    if (!PtlHandleIsEqual(module->origin_iovec_md_h,PTL_INVALID_HANDLE)) {
         PtlMDRelease(module->origin_iovec_md_h);
         free(module->origin_iovec_list);
         module->origin_iovec_md_h = PTL_INVALID_HANDLE;
@@ -920,13 +920,13 @@ atomic_from_iovec(ompi_osc_portals4_module_t *module,
 static int
 swap_to_iovec(ompi_osc_portals4_module_t *module,
               const void       *result_address,
-              int               result_count,
+              size_t            result_count,
               ompi_datatype_t  *result_datatype,
               const void       *origin_address,
-              int               origin_count,
+              size_t            origin_count,
               ompi_datatype_t  *origin_datatype,
               ptl_process_t     peer,
-              int               target_count,
+              size_t            target_count,
               ompi_datatype_t  *target_datatype,
               size_t            offset,
               ptl_pt_index_t    pt_index,
@@ -940,7 +940,7 @@ swap_to_iovec(ompi_osc_portals4_module_t *module,
     ptl_md_t md;
     ptl_datatype_t ptl_dt;
 
-    if (module->result_iovec_md_h != PTL_INVALID_HANDLE) {
+    if (!PtlHandleIsEqual(module->result_iovec_md_h,PTL_INVALID_HANDLE)) {
         PtlMDRelease(module->result_iovec_md_h);
         free(module->result_iovec_list);
         module->result_iovec_md_h = PTL_INVALID_HANDLE;
@@ -971,7 +971,7 @@ swap_to_iovec(ompi_osc_portals4_module_t *module,
         return ret;
     }
 
-    if (module->origin_iovec_md_h != PTL_INVALID_HANDLE) {
+    if (!PtlHandleIsEqual(module->origin_iovec_md_h,PTL_INVALID_HANDLE)) {
         PtlMDRelease(module->origin_iovec_md_h);
         free(module->origin_iovec_list);
         module->origin_iovec_md_h = PTL_INVALID_HANDLE;
@@ -1044,13 +1044,13 @@ swap_to_iovec(ompi_osc_portals4_module_t *module,
 static int
 fetch_atomic_to_iovec(ompi_osc_portals4_module_t *module,
                       const void       *result_address,
-                      int               result_count,
+                      size_t            result_count,
                       ompi_datatype_t  *result_datatype,
                       const void       *origin_address,
-                      int               origin_count,
+                      size_t            origin_count,
                       ompi_datatype_t  *origin_datatype,
                       ptl_process_t     peer,
-                      int               target_count,
+                      size_t            target_count,
                       ompi_datatype_t  *target_datatype,
                       size_t            offset,
                       ptl_pt_index_t    pt_index,
@@ -1066,7 +1066,7 @@ fetch_atomic_to_iovec(ompi_osc_portals4_module_t *module,
     ptl_op_t ptl_op;
     ptl_datatype_t ptl_dt;
 
-    if (module->result_iovec_md_h != PTL_INVALID_HANDLE) {
+    if (!PtlHandleIsEqual(module->result_iovec_md_h,PTL_INVALID_HANDLE)) {
         PtlMDRelease(module->result_iovec_md_h);
         free(module->result_iovec_list);
         module->result_iovec_md_h = PTL_INVALID_HANDLE;
@@ -1097,7 +1097,7 @@ fetch_atomic_to_iovec(ompi_osc_portals4_module_t *module,
         return ret;
     }
 
-    if (module->origin_iovec_md_h != PTL_INVALID_HANDLE) {
+    if (!PtlHandleIsEqual(module->origin_iovec_md_h,PTL_INVALID_HANDLE)) {
         PtlMDRelease(module->origin_iovec_md_h);
         free(module->origin_iovec_list);
         module->origin_iovec_md_h = PTL_INVALID_HANDLE;
@@ -1182,10 +1182,10 @@ static int
 put_to_noncontig(opal_atomic_int64_t *opcount,
                  ptl_handle_md_t   md_h,
                  const void       *origin_address,
-                 int               origin_count,
+                 size_t            origin_count,
                  ompi_datatype_t  *origin_datatype,
                  ptl_process_t     peer,
-                 int               target_count,
+                 size_t            target_count,
                  ompi_datatype_t  *target_datatype,
                  size_t            offset,
                  ptl_pt_index_t    pt_index,
@@ -1229,7 +1229,7 @@ put_to_noncontig(opal_atomic_int64_t *opcount,
         /* opal_convertor_raw returns done when it has reached the end of the data */
         done = opal_convertor_raw (&target_convertor, target_iovec, &target_iov_count, &target_size);
 
-        /* loop on the target segments until we have exhaused the decoded source data */
+        /* loop on the target segments until we have exhausted the decoded source data */
         while (target_iov_index != target_iov_count) {
             if (origin_iov_index == origin_iov_count) {
                 /* decode segments of the target buffer */
@@ -1291,10 +1291,10 @@ static int
 atomic_put_to_noncontig(ompi_osc_portals4_module_t *module,
                         ptl_handle_md_t   md_h,
                         const void       *origin_address,
-                        int               origin_count,
+                        size_t            origin_count,
                         ompi_datatype_t  *origin_datatype,
                         ptl_process_t     peer,
-                        int               target_count,
+                        size_t            target_count,
                         ompi_datatype_t  *target_datatype,
                         size_t            offset,
                         ptl_pt_index_t    pt_index,
@@ -1338,7 +1338,7 @@ atomic_put_to_noncontig(ompi_osc_portals4_module_t *module,
         /* opal_convertor_raw returns done when it has reached the end of the data */
         done = opal_convertor_raw (&target_convertor, target_iovec, &target_iov_count, &target_size);
 
-        /* loop on the target segments until we have exhaused the decoded source data */
+        /* loop on the target segments until we have exhausted the decoded source data */
         while (target_iov_index != target_iov_count) {
             if (origin_iov_index == origin_iov_count) {
                 /* decode segments of the target buffer */
@@ -1394,10 +1394,10 @@ static int
 atomic_to_noncontig(ompi_osc_portals4_module_t *module,
                     ptl_handle_md_t   md_h,
                     const void       *origin_address,
-                    int               origin_count,
+                    size_t            origin_count,
                     ompi_datatype_t  *origin_datatype,
                     ptl_process_t     peer,
-                    int               target_count,
+                    size_t            target_count,
                     ompi_datatype_t  *target_datatype,
                     size_t            offset,
                     ptl_pt_index_t    pt_index,
@@ -1456,7 +1456,7 @@ atomic_to_noncontig(ompi_osc_portals4_module_t *module,
         /* opal_convertor_raw returns done when it has reached the end of the data */
         done = opal_convertor_raw (&target_convertor, target_iovec, &target_iov_count, &target_size);
 
-        /* loop on the target segments until we have exhaused the decoded source data */
+        /* loop on the target segments until we have exhausted the decoded source data */
         while (target_iov_index != target_iov_count) {
             if (origin_iov_index == origin_iov_count) {
                 /* decode segments of the target buffer */
@@ -1479,7 +1479,7 @@ atomic_to_noncontig(ompi_osc_portals4_module_t *module,
                              (unsigned long) target_iovec[target_iov_index].iov_len));
 
             OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
-                         "%s,%d Atomic", __FUNCTION__, __LINE__));
+                         "%s,%d Atomic", __FILE__, __LINE__));
             ret = PtlAtomic(md_h,
                             (ptl_size_t)origin_iovec[origin_iov_index].iov_base,
                             atomic_len,
@@ -1516,10 +1516,10 @@ static int
 get_from_noncontig(opal_atomic_int64_t *opcount,
                    ptl_handle_md_t   md_h,
                    const void       *origin_address,
-                   int               origin_count,
+                   size_t            origin_count,
                    ompi_datatype_t  *origin_datatype,
                    ptl_process_t     peer,
-                   int               target_count,
+                   size_t            target_count,
                    ompi_datatype_t  *target_datatype,
                    size_t            offset,
                    ptl_pt_index_t    pt_index,
@@ -1563,7 +1563,7 @@ get_from_noncontig(opal_atomic_int64_t *opcount,
         /* opal_convertor_raw returns done when it has reached the end of the data */
         done = opal_convertor_raw (&target_convertor, target_iovec, &target_iov_count, &target_size);
 
-        /* loop on the target segments until we have exhaused the decoded source data */
+        /* loop on the target segments until we have exhausted the decoded source data */
         while (target_iov_index != target_iov_count) {
             if (origin_iov_index == origin_iov_count) {
                 /* decode segments of the target buffer */
@@ -1617,10 +1617,10 @@ static int
 atomic_get_from_noncontig(ompi_osc_portals4_module_t *module,
                           ptl_handle_md_t   md_h,
                           const void       *origin_address,
-                          int               origin_count,
+                          size_t            origin_count,
                           ompi_datatype_t  *origin_datatype,
                           ptl_process_t     peer,
-                          int               target_count,
+                          size_t            target_count,
                           ompi_datatype_t  *target_datatype,
                           size_t            offset,
                           ptl_pt_index_t    pt_index,
@@ -1664,7 +1664,7 @@ atomic_get_from_noncontig(ompi_osc_portals4_module_t *module,
         /* opal_convertor_raw returns done when it has reached the end of the data */
         done = opal_convertor_raw (&target_convertor, target_iovec, &target_iov_count, &target_size);
 
-        /* loop on the target segments until we have exhaused the decoded source data */
+        /* loop on the target segments until we have exhausted the decoded source data */
         while (target_iov_index != target_iov_count) {
             if (origin_iov_index == origin_iov_count) {
                 /* decode segments of the target buffer */
@@ -1718,14 +1718,14 @@ static int
 swap_from_noncontig(ompi_osc_portals4_module_t *module,
                     ptl_handle_md_t   result_md_h,
                     const void       *result_address,
-                    int               result_count,
+                    size_t            result_count,
                     ompi_datatype_t  *result_datatype,
                     ptl_handle_md_t   origin_md_h,
                     const void       *origin_address,
-                    int               origin_count,
+                    size_t            origin_count,
                     ompi_datatype_t  *origin_datatype,
                     ptl_process_t     peer,
-                    int               target_count,
+                    size_t            target_count,
                     ompi_datatype_t  *target_datatype,
                     size_t            offset,
                     ptl_pt_index_t    pt_index,
@@ -1787,7 +1787,7 @@ swap_from_noncontig(ompi_osc_portals4_module_t *module,
         /* opal_convertor_raw returns done when it has reached the end of the data */
         done = opal_convertor_raw (&target_convertor, target_iovec, &target_iov_count, &target_size);
 
-        /* loop on the target segments until we have exhaused the decoded source data */
+        /* loop on the target segments until we have exhausted the decoded source data */
         while (target_iov_index != target_iov_count) {
             if (result_iov_index == result_iov_count) {
                 /* decode segments of the target buffer */
@@ -1835,7 +1835,7 @@ swap_from_noncontig(ompi_osc_portals4_module_t *module,
             if (PTL_OK != ret) {
                 opal_output_verbose(1, ompi_osc_base_framework.framework_output,
                                      "%s:%d PtlSwap failed with return value %d",
-                                     __FUNCTION__, __LINE__, ret);
+                                     __FILE__, __LINE__, ret);
                 opal_atomic_add_fetch_64(&module->opcount, -1);
                 return ret;
             }
@@ -1862,14 +1862,14 @@ static int
 fetch_atomic_from_noncontig(ompi_osc_portals4_module_t *module,
                             ptl_handle_md_t   result_md_h,
                             const void       *result_address,
-                            int               result_count,
+                            size_t            result_count,
                             ompi_datatype_t  *result_datatype,
                             ptl_handle_md_t   origin_md_h,
                             const void       *origin_address,
-                            int               origin_count,
+                            size_t            origin_count,
                             ompi_datatype_t  *origin_datatype,
                             ptl_process_t     peer,
-                            int               target_count,
+                            size_t            target_count,
                             ompi_datatype_t  *target_datatype,
                             size_t            offset,
                             ptl_pt_index_t    pt_index,
@@ -1939,7 +1939,7 @@ fetch_atomic_from_noncontig(ompi_osc_portals4_module_t *module,
         /* opal_convertor_raw returns done when it has reached the end of the data */
         done = opal_convertor_raw (&target_convertor, target_iovec, &target_iov_count, &target_size);
 
-        /* loop on the target segments until we have exhaused the decoded source data */
+        /* loop on the target segments until we have exhausted the decoded source data */
         while (target_iov_index != target_iov_count) {
             if (result_iov_index == result_iov_count) {
                 /* decode segments of the target buffer */
@@ -1986,7 +1986,7 @@ fetch_atomic_from_noncontig(ompi_osc_portals4_module_t *module,
             if (PTL_OK != ret) {
                 opal_output_verbose(1, ompi_osc_base_framework.framework_output,
                                      "%s:%d PtlFetchAtomic failed with return value %d",
-                                     __FUNCTION__, __LINE__, ret);
+                                     __FILE__, __LINE__, ret);
                 opal_atomic_add_fetch_64(&module->opcount, -1);
                 return ret;
             }
@@ -2010,11 +2010,11 @@ fetch_atomic_from_noncontig(ompi_osc_portals4_module_t *module,
 
 int
 ompi_osc_portals4_rput(const void *origin_addr,
-                       int origin_count,
+                       size_t origin_count,
                        struct ompi_datatype_t *origin_dt,
                        int target,
                        ptrdiff_t target_disp,
-                       int target_count,
+                       size_t target_count,
                        struct ompi_datatype_t *target_dt,
                        struct ompi_win_t *win,
                        struct ompi_request_t **ompi_req)
@@ -2028,7 +2028,7 @@ ompi_osc_portals4_rput(const void *origin_addr,
     ptrdiff_t length, origin_lb, target_lb, extent;
 
     OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
-                         "rput: 0x%lx, %d, %s, %d, %lu, %d, %s, 0x%lx",
+                         "rput: 0x%lx, %zu, %s, %d, %lu, %zu, %s, 0x%lx",
                          (unsigned long) origin_addr, origin_count,
                          origin_dt->name, target, (unsigned long) target_disp,
                          target_count, target_dt->name,
@@ -2057,7 +2057,7 @@ ompi_osc_portals4_rput(const void *origin_addr,
             OMPI_OSC_PORTALS4_REQUEST_RETURN(request);
             OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
                          "%s,%d put_to_noncontig() failed: ret = %d",
-                         __FUNCTION__, __LINE__, ret));
+                         __FILE__, __LINE__, ret));
             return ret;
         }
     } else if (!ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count)) {
@@ -2076,7 +2076,7 @@ ompi_osc_portals4_rput(const void *origin_addr,
             OMPI_OSC_PORTALS4_REQUEST_RETURN(request);
             OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
                          "%s,%d put_from_iovec() failed: ret = %d",
-                         __FUNCTION__, __LINE__, ret));
+                         __FILE__, __LINE__, ret));
             return ret;
         }
     } else {
@@ -2096,8 +2096,8 @@ ompi_osc_portals4_rput(const void *origin_addr,
         request->ops_expected += number_of_fragments(length, mca_osc_portals4_component.ptl_max_msg_size);
 
         OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
-                     "%s,%d RPut(origin_count=%d, origin_lb=%lu, target_count=%d, target_lb=%lu, length=%lu, op_count=%ld)",
-                     __FUNCTION__, __LINE__, origin_count, origin_lb, target_count, target_lb, length, module->opcount));
+                     "%s,%d RPut(origin_count=%zu, origin_lb=%lu, target_count=%zu, target_lb=%lu, length=%lu, op_count=%ld)",
+                     __FILE__, __LINE__, origin_count, origin_lb, target_count, target_lb, length, module->opcount));
         ret = segmentedPut(&module->opcount,
                            module->req_md_h,
                            (ptl_size_t) origin_addr + origin_lb,
@@ -2122,11 +2122,11 @@ ompi_osc_portals4_rput(const void *origin_addr,
 
 int
 ompi_osc_portals4_rget(void *origin_addr,
-                       int origin_count,
+                       size_t origin_count,
                        struct ompi_datatype_t *origin_dt,
                        int target,
                        ptrdiff_t target_disp,
-                       int target_count,
+                       size_t target_count,
                        struct ompi_datatype_t *target_dt,
                        struct ompi_win_t *win,
                        struct ompi_request_t **ompi_req)
@@ -2140,7 +2140,7 @@ ompi_osc_portals4_rget(void *origin_addr,
     ptrdiff_t length, origin_lb, target_lb, extent;
 
     OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
-                         "rget: 0x%lx, %d, %s, %d, %lu, %d, %s, 0x%lx",
+                         "rget: 0x%lx, %zu, %s, %d, %lu, %zu, %s, 0x%lx",
                          (unsigned long) origin_addr, origin_count,
                          origin_dt->name, target, (unsigned long) target_disp,
                          target_count, target_dt->name,
@@ -2168,7 +2168,7 @@ ompi_osc_portals4_rget(void *origin_addr,
         if (PTL_OK != ret) {
             OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
                          "%s,%d get_from_noncontig() failed: ret = %d",
-                         __FUNCTION__, __LINE__, ret));
+                         __FILE__, __LINE__, ret));
             return ret;
         }
     } else if (!ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count)) {
@@ -2186,7 +2186,7 @@ ompi_osc_portals4_rget(void *origin_addr,
         if (PTL_OK != ret) {
             OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
                          "%s,%d get_to_iovec() failed: ret = %d",
-                         __FUNCTION__, __LINE__, ret));
+                         __FILE__, __LINE__, ret));
             return ret;
         }
     } else {
@@ -2204,7 +2204,7 @@ ompi_osc_portals4_rget(void *origin_addr,
         request->ops_expected += number_of_fragments(length, mca_osc_portals4_component.ptl_max_msg_size);
 
         OPAL_OUTPUT_VERBOSE((90,ompi_osc_base_framework.framework_output,
-                              "%s,%d RGet", __FUNCTION__, __LINE__));
+                              "%s,%d RGet", __FILE__, __LINE__));
         ret = segmentedGet(&module->opcount,
                            module->req_md_h,
                            (ptl_size_t) origin_addr + origin_lb,
@@ -2227,11 +2227,11 @@ ompi_osc_portals4_rget(void *origin_addr,
 
 int
 ompi_osc_portals4_raccumulate(const void *origin_addr,
-                              int origin_count,
+                              size_t origin_count,
                               struct ompi_datatype_t *origin_dt,
                               int target,
                               ptrdiff_t target_disp,
-                              int target_count,
+                              size_t target_count,
                               struct ompi_datatype_t *target_dt,
                               struct ompi_op_t *op,
                               struct ompi_win_t *win,
@@ -2245,10 +2245,11 @@ ompi_osc_portals4_raccumulate(const void *origin_addr,
     size_t offset, size;
     ptl_op_t ptl_op;
     ptl_datatype_t ptl_dt;
-    ptrdiff_t sent, length, origin_lb, target_lb, extent;
+    ptl_size_t sent, length;
+    ptrdiff_t origin_lb, target_lb, extent;
 
     OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
-                         "raccumulate: 0x%lx, %d, %s, %d, %lu, %d, %s, %s 0x%lx",
+                         "raccumulate: 0x%lx, %zu, %s, %d, %lu, %zu, %s, %s 0x%lx",
                          (unsigned long) origin_addr, origin_count,
                          origin_dt->name, target, (unsigned long) target_disp,
                          target_count, target_dt->name,
@@ -2279,7 +2280,7 @@ ompi_osc_portals4_raccumulate(const void *origin_addr,
                 OMPI_OSC_PORTALS4_REQUEST_RETURN(request);
                 OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
                     "%s,%d atomic_put_to_noncontig() failed: ret = %d",
-                    __FUNCTION__, __LINE__, ret));
+                    __FILE__, __LINE__, ret));
                 return ret;
             }
         } else {
@@ -2300,7 +2301,7 @@ ompi_osc_portals4_raccumulate(const void *origin_addr,
                 OMPI_OSC_PORTALS4_REQUEST_RETURN(request);
                 OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
                     "%s,%d atomic_to_noncontig() failed: ret = %d",
-                    __FUNCTION__, __LINE__, ret));
+                    __FILE__, __LINE__, ret));
                 return ret;
             }
         }
@@ -2321,7 +2322,7 @@ ompi_osc_portals4_raccumulate(const void *origin_addr,
                 OMPI_OSC_PORTALS4_REQUEST_RETURN(request);
                 OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
                     "%s,%d atomic_put_from_iovec() failed: ret = %d",
-                    __FUNCTION__, __LINE__, ret));
+                    __FILE__, __LINE__, ret));
                 return ret;
             }
         } else {
@@ -2341,7 +2342,7 @@ ompi_osc_portals4_raccumulate(const void *origin_addr,
                 OMPI_OSC_PORTALS4_REQUEST_RETURN(request);
                 OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
                     "%s,%d atomic_from_iovec() failed: ret = %d",
-                    __FUNCTION__, __LINE__, ret));
+                    __FILE__, __LINE__, ret));
                 return ret;
             }
         }
@@ -2368,7 +2369,7 @@ ompi_osc_portals4_raccumulate(const void *origin_addr,
 
         if (MPI_REPLACE == op) {
             OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
-                                 "%s,%d Put", __FUNCTION__, __LINE__));
+                                 "%s,%d Put", __FILE__, __LINE__));
             ret = segmentedPut(&module->opcount,
                                module->req_md_h,
                                md_offset + origin_lb,
@@ -2406,7 +2407,7 @@ ompi_osc_portals4_raccumulate(const void *origin_addr,
                 (void)opal_atomic_add_fetch_64(&module->opcount, 1);
 
                 OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
-                             "%s,%d Atomic", __FUNCTION__, __LINE__));
+                             "%s,%d Atomic", __FILE__, __LINE__));
                 ret = PtlAtomic(module->req_md_h,
                                 md_offset + sent + origin_lb,
                                 msg_length,
@@ -2435,14 +2436,14 @@ ompi_osc_portals4_raccumulate(const void *origin_addr,
 
 int
 ompi_osc_portals4_rget_accumulate(const void *origin_addr,
-                                  int origin_count,
+                                  size_t origin_count,
                                   struct ompi_datatype_t *origin_dt,
                                   void *result_addr,
-                                  int result_count,
+                                  size_t result_count,
                                   struct ompi_datatype_t *result_dt,
                                   int target,
                                   ptrdiff_t target_disp,
-                                  int target_count,
+                                  size_t target_count,
                                   struct ompi_datatype_t *target_dt,
                                   struct ompi_op_t *op,
                                   struct ompi_win_t *win,
@@ -2459,7 +2460,7 @@ ompi_osc_portals4_rget_accumulate(const void *origin_addr,
     ptrdiff_t length, origin_lb, target_lb, result_lb, extent;
 
     OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
-                         "rget_accumulate: 0x%lx, %d, %s, 0x%lx, %d, %s, %d, %lu, %d, %s, %s, 0x%lx",
+                         "rget_accumulate: 0x%lx, %zu, %s, 0x%lx, %zu, %s, %d, %lu, %zu, %s, %s, 0x%lx",
                          (unsigned long) origin_addr, origin_count,
                          origin_dt->name, (unsigned long) result_addr,
                          result_count, result_dt->name,
@@ -2497,7 +2498,7 @@ ompi_osc_portals4_rget_accumulate(const void *origin_addr,
             if (PTL_OK != ret) {
                 OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
                              "%s,%d swap_from_noncontig() failed: ret = %d",
-                             __FUNCTION__, __LINE__, ret));
+                             __FILE__, __LINE__, ret));
                 OMPI_OSC_PORTALS4_REQUEST_RETURN(request);
                 return ret;
             }
@@ -2519,7 +2520,7 @@ ompi_osc_portals4_rget_accumulate(const void *origin_addr,
             if (PTL_OK != ret) {
                 OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
                              "%s,%d atomic_get_from_noncontig() failed: ret = %d",
-                             __FUNCTION__, __LINE__, ret));
+                             __FILE__, __LINE__, ret));
                 OMPI_OSC_PORTALS4_REQUEST_RETURN(request);
                 return ret;
             }
@@ -2546,7 +2547,7 @@ ompi_osc_portals4_rget_accumulate(const void *origin_addr,
             if (PTL_OK != ret) {
                 OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
                     "%s,%d fetch_atomic_from_noncontig() failed: ret = %d",
-                    __FUNCTION__, __LINE__, ret));
+                    __FILE__, __LINE__, ret));
                 OMPI_OSC_PORTALS4_REQUEST_RETURN(request);
                 return ret;
             }
@@ -2573,7 +2574,7 @@ ompi_osc_portals4_rget_accumulate(const void *origin_addr,
             if (PTL_OK != ret) {
                 OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
                              "%s,%d swap_to_iovec() failed: ret = %d",
-                             __FUNCTION__, __LINE__, ret));
+                             __FILE__, __LINE__, ret));
                 OMPI_OSC_PORTALS4_REQUEST_RETURN(request);
                 return ret;
             }
@@ -2594,7 +2595,7 @@ ompi_osc_portals4_rget_accumulate(const void *origin_addr,
             if (PTL_OK != ret) {
                 OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
                              "%s,%d atomic_get_to_iovec() failed: ret = %d",
-                             __FUNCTION__, __LINE__, ret));
+                             __FILE__, __LINE__, ret));
                 OMPI_OSC_PORTALS4_REQUEST_RETURN(request);
                 return ret;
             }
@@ -2619,7 +2620,7 @@ ompi_osc_portals4_rget_accumulate(const void *origin_addr,
             if (PTL_OK != ret) {
                 OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
                              "%s,%d fetch_atomic_to_iovec() failed: ret = %d",
-                             __FUNCTION__, __LINE__, ret));
+                             __FILE__, __LINE__, ret));
                 OMPI_OSC_PORTALS4_REQUEST_RETURN(request);
                 return ret;
             }
@@ -2701,7 +2702,7 @@ ompi_osc_portals4_rget_accumulate(const void *origin_addr,
             request->ops_expected += number_of_fragments(length, module->fetch_atomic_max);
 
             OPAL_OUTPUT_VERBOSE((90,ompi_osc_base_framework.framework_output,
-                                  "%s,%d MPI_Get_accumulate", __FUNCTION__, __LINE__));
+                                  "%s,%d MPI_Get_accumulate", __FILE__, __LINE__));
             ret = segmentedGet(&module->opcount,
                                module->req_md_h,
                                (ptl_size_t) md_offset + result_lb,
@@ -2787,11 +2788,11 @@ ompi_osc_portals4_rget_accumulate(const void *origin_addr,
 
 int
 ompi_osc_portals4_put(const void *origin_addr,
-                      int origin_count,
+                      size_t origin_count,
                       struct ompi_datatype_t *origin_dt,
                       int target,
                       ptrdiff_t target_disp,
-                      int target_count,
+                      size_t target_count,
                       struct ompi_datatype_t *target_dt,
                       struct ompi_win_t *win)
 {
@@ -2803,7 +2804,7 @@ ompi_osc_portals4_put(const void *origin_addr,
     ptrdiff_t length, origin_lb, target_lb, extent;
 
     OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
-                         "put: 0x%lx, %d, %s, %d, %lu, %d, %s, 0x%lx",
+                         "put: 0x%lx, %zu, %s, %d, %lu, %zu, %s, 0x%lx",
                          (unsigned long) origin_addr, origin_count,
                          origin_dt->name, target, (unsigned long) target_disp,
                          target_count, target_dt->name,
@@ -2827,7 +2828,7 @@ ompi_osc_portals4_put(const void *origin_addr,
         if (PTL_OK != ret) {
             OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
                          "%s,%d put_to_noncontig() failed: ret = %d",
-                         __FUNCTION__, __LINE__, ret));
+                         __FILE__, __LINE__, ret));
             return ret;
         }
     } else if (!ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count)) {
@@ -2845,7 +2846,7 @@ ompi_osc_portals4_put(const void *origin_addr,
         if (PTL_OK != ret) {
             OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
                          "%s,%d put_from_iovec() failed: ret = %d",
-                         __FUNCTION__, __LINE__, ret));
+                         __FILE__, __LINE__, ret));
             return ret;
         }
     } else {
@@ -2861,8 +2862,8 @@ ompi_osc_portals4_put(const void *origin_addr,
         length = size * origin_count;
 
         OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
-                     "%s,%d Put(origin_count=%d, origin_lb=%lu, target_count=%d, target_lb=%lu, length=%lu, op_count=%ld)",
-                     __FUNCTION__, __LINE__, origin_count, origin_lb, target_count, target_lb, length, module->opcount));
+                     "%s,%d Put(origin_count=%zu, origin_lb=%lu, target_count=%zu, target_lb=%lu, length=%lu, op_count=%ld)",
+                     __FILE__, __LINE__, origin_count, origin_lb, target_count, target_lb, length, module->opcount));
         ret = segmentedPut(&module->opcount,
                            module->md_h,
                            (ptl_size_t) origin_addr + origin_lb,
@@ -2886,11 +2887,11 @@ ompi_osc_portals4_put(const void *origin_addr,
 
 int
 ompi_osc_portals4_get(void *origin_addr,
-                      int origin_count,
+                      size_t origin_count,
                       struct ompi_datatype_t *origin_dt,
                       int target,
                       ptrdiff_t target_disp,
-                      int target_count,
+                      size_t target_count,
                       struct ompi_datatype_t *target_dt,
                       struct ompi_win_t *win)
 {
@@ -2902,7 +2903,7 @@ ompi_osc_portals4_get(void *origin_addr,
     ptrdiff_t length, origin_lb, target_lb, extent;
 
     OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
-                         "get: 0x%lx, %d, %s, %d, %lu, %d, %s, 0x%lx",
+                         "get: 0x%lx, %zu, %s, %d, %lu, %zu, %s, 0x%lx",
                          (unsigned long) origin_addr, origin_count,
                          origin_dt->name, target, (unsigned long) target_disp,
                          target_count, target_dt->name,
@@ -2926,7 +2927,7 @@ ompi_osc_portals4_get(void *origin_addr,
         if (PTL_OK != ret) {
             OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
                          "%s,%d get_from_noncontig() failed: ret = %d",
-                         __FUNCTION__, __LINE__, ret));
+                         __FILE__, __LINE__, ret));
             return ret;
         }
     } else if (!ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count)) {
@@ -2944,7 +2945,7 @@ ompi_osc_portals4_get(void *origin_addr,
         if (PTL_OK != ret) {
             OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
                          "%s,%d get_to_iovec() failed: ret = %d",
-                         __FUNCTION__, __LINE__, ret));
+                         __FILE__, __LINE__, ret));
             return ret;
         }
     } else {
@@ -2960,7 +2961,7 @@ ompi_osc_portals4_get(void *origin_addr,
         length = size * origin_count;
 
         OPAL_OUTPUT_VERBOSE((90,ompi_osc_base_framework.framework_output,
-                              "%s,%d Get", __FUNCTION__, __LINE__));
+                              "%s,%d Get", __FILE__, __LINE__));
         ret = segmentedGet(&module->opcount,
                            module->md_h,
                            (ptl_size_t) origin_addr + origin_lb,
@@ -2982,11 +2983,11 @@ ompi_osc_portals4_get(void *origin_addr,
 
 int
 ompi_osc_portals4_accumulate(const void *origin_addr,
-                             int origin_count,
+                             size_t origin_count,
                              struct ompi_datatype_t *origin_dt,
                              int target,
                              ptrdiff_t target_disp,
-                             int target_count,
+                             size_t target_count,
                              struct ompi_datatype_t *target_dt,
                              struct ompi_op_t *op,
                              struct ompi_win_t *win)
@@ -2998,10 +2999,11 @@ ompi_osc_portals4_accumulate(const void *origin_addr,
     size_t offset, size;
     ptl_op_t ptl_op;
     ptl_datatype_t ptl_dt;
-    ptrdiff_t sent, length, origin_lb, target_lb, extent;
+    ptl_size_t sent, length;
+    ptrdiff_t origin_lb, target_lb, extent;
 
     OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
-                         "accumulate: 0x%lx, %d, %s, %d, %lu, %d, %s, %s, 0x%lx",
+                         "accumulate: 0x%lx, %zu, %s, %d, %lu, %zu, %s, %s, 0x%lx",
                          (unsigned long) origin_addr, origin_count,
                          origin_dt->name, target, (unsigned long) target_disp,
                          target_count, target_dt->name,
@@ -3027,7 +3029,7 @@ ompi_osc_portals4_accumulate(const void *origin_addr,
             if (PTL_OK != ret) {
                 OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
                     "%s,%d atomic_put_to_noncontig() failed: ret = %d",
-                    __FUNCTION__, __LINE__, ret));
+                    __FILE__, __LINE__, ret));
                 return ret;
             }
         } else {
@@ -3047,7 +3049,7 @@ ompi_osc_portals4_accumulate(const void *origin_addr,
             if (PTL_OK != ret) {
                 OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
                     "%s,%d atomic_to_noncontig() failed: ret = %d",
-                    __FUNCTION__, __LINE__, ret));
+                    __FILE__, __LINE__, ret));
                 return ret;
             }
         }
@@ -3067,7 +3069,7 @@ ompi_osc_portals4_accumulate(const void *origin_addr,
             if (PTL_OK != ret) {
                 OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
                     "%s,%d atomic_put_from_iovec() failed: ret = %d",
-                    __FUNCTION__, __LINE__, ret));
+                    __FILE__, __LINE__, ret));
                 return ret;
             }
         } else {
@@ -3086,7 +3088,7 @@ ompi_osc_portals4_accumulate(const void *origin_addr,
             if (PTL_OK != ret) {
                 OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
                     "%s,%d atomic_from_iovec() failed: ret = %d",
-                    __FUNCTION__, __LINE__, ret));
+                    __FILE__, __LINE__, ret));
                 return ret;
             }
         }
@@ -3109,7 +3111,7 @@ ompi_osc_portals4_accumulate(const void *origin_addr,
 
         if (MPI_REPLACE == op) {
             OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
-                                 "%s,%d Put", __FUNCTION__, __LINE__));
+                                 "%s,%d Put", __FILE__, __LINE__));
             ret = segmentedPut(&module->opcount,
                                module->md_h,
                                md_offset + origin_lb,
@@ -3144,7 +3146,7 @@ ompi_osc_portals4_accumulate(const void *origin_addr,
                 (void)opal_atomic_add_fetch_64(&module->opcount, 1);
 
                 OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
-                             "%s,%d Atomic", __FUNCTION__, __LINE__));
+                             "%s,%d Atomic", __FILE__, __LINE__));
                 ret = PtlAtomic(module->md_h,
                                 md_offset + sent + origin_lb,
                                 msg_length,
@@ -3172,14 +3174,14 @@ ompi_osc_portals4_accumulate(const void *origin_addr,
 
 int
 ompi_osc_portals4_get_accumulate(const void *origin_addr,
-                                 int origin_count,
+                                 size_t origin_count,
                                  struct ompi_datatype_t *origin_dt,
                                  void *result_addr,
-                                 int result_count,
+                                 size_t result_count,
                                  struct ompi_datatype_t *result_dt,
                                  int target,
                                  ptrdiff_t target_disp,
-                                 int target_count,
+                                 size_t target_count,
                                  struct ompi_datatype_t *target_dt,
                                  struct ompi_op_t *op,
                                  struct ompi_win_t *win)
@@ -3194,7 +3196,7 @@ ompi_osc_portals4_get_accumulate(const void *origin_addr,
     ptrdiff_t length, origin_lb, target_lb, result_lb, extent;
 
     OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
-                         "get_accumulate: 0x%lx, %d, %s, 0x%lx, %d, %s, %d, %lu, %d, %s, %s, 0x%lx",
+                         "get_accumulate: 0x%lx, %zu, %s, 0x%lx, %zu, %s, %d, %lu, %zu, %s, %s, 0x%lx",
                          (unsigned long) origin_addr, origin_count,
                          origin_dt->name, (unsigned long) result_addr,
                          result_count, result_dt->name,
@@ -3228,7 +3230,7 @@ ompi_osc_portals4_get_accumulate(const void *origin_addr,
             if (PTL_OK != ret) {
                 OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
                              "%s,%d swap_from_noncontig() failed: ret = %d",
-                             __FUNCTION__, __LINE__, ret));
+                             __FILE__, __LINE__, ret));
                 return ret;
             }
         } else if (MPI_NO_OP == op) {
@@ -3249,7 +3251,7 @@ ompi_osc_portals4_get_accumulate(const void *origin_addr,
             if (PTL_OK != ret) {
                 OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
                              "%s,%d atomic_get_from_noncontig() failed: ret = %d",
-                             __FUNCTION__, __LINE__, ret));
+                             __FILE__, __LINE__, ret));
                 return ret;
             }
         } else {
@@ -3275,7 +3277,7 @@ ompi_osc_portals4_get_accumulate(const void *origin_addr,
             if (PTL_OK != ret) {
                 OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
                     "%s,%d fetch_atomic_from_noncontig() failed: ret = %d",
-                    __FUNCTION__, __LINE__, ret));
+                    __FILE__, __LINE__, ret));
                 return ret;
             }
         }
@@ -3301,7 +3303,7 @@ ompi_osc_portals4_get_accumulate(const void *origin_addr,
             if (PTL_OK != ret) {
                 OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
                              "%s,%d swap_to_iovec() failed: ret = %d",
-                             __FUNCTION__, __LINE__, ret));
+                             __FILE__, __LINE__, ret));
                 return ret;
             }
         } else if (MPI_NO_OP == op) {
@@ -3321,7 +3323,7 @@ ompi_osc_portals4_get_accumulate(const void *origin_addr,
             if (PTL_OK != ret) {
                 OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
                              "%s,%d atomic_get_to_iovec() failed: ret = %d",
-                             __FUNCTION__, __LINE__, ret));
+                             __FILE__, __LINE__, ret));
                 return ret;
             }
         } else {
@@ -3345,7 +3347,7 @@ ompi_osc_portals4_get_accumulate(const void *origin_addr,
             if (PTL_OK != ret) {
                 OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
                              "%s,%d fetch_atomic_to_iovec() failed: ret = %d",
-                             __FUNCTION__, __LINE__, ret));
+                             __FILE__, __LINE__, ret));
                 return ret;
             }
         }
@@ -3415,7 +3417,7 @@ ompi_osc_portals4_get_accumulate(const void *origin_addr,
             md_offset = (ptl_size_t) result_addr;
 
             OPAL_OUTPUT_VERBOSE((90,ompi_osc_base_framework.framework_output,
-                                  "%s,%d MPI_Get_accumulate", __FUNCTION__, __LINE__));
+                                  "%s,%d MPI_Get_accumulate", __FILE__, __LINE__));
             ret = segmentedGet(&module->opcount,
                                module->md_h,
                                (ptl_size_t) md_offset + result_lb,
@@ -3536,7 +3538,7 @@ ompi_osc_portals4_compare_and_swap(const void *origin_addr,
     (void)opal_atomic_add_fetch_64(&module->opcount, 1);
 
     OPAL_OUTPUT_VERBOSE((90,ompi_osc_base_framework.framework_output,
-                         "%s,%d Swap", __FUNCTION__, __LINE__));
+                         "%s,%d Swap", __FILE__, __LINE__));
     ret = PtlSwap(module->md_h,
                   result_md_offset,
                   module->md_h,
@@ -3607,7 +3609,7 @@ ompi_osc_portals4_fetch_and_op(const void *origin_addr,
 
         (void)opal_atomic_add_fetch_64(&module->opcount, 1);
         OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
-                             "%s,%d Swap", __FUNCTION__, __LINE__));
+                             "%s,%d Swap", __FILE__, __LINE__));
         ret = PtlSwap(module->md_h,
                       result_md_offset,
                       module->md_h,
@@ -3629,7 +3631,7 @@ ompi_osc_portals4_fetch_and_op(const void *origin_addr,
 
         (void)opal_atomic_add_fetch_64(&module->opcount, 1);
         OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
-                             "%s,%d Get", __FUNCTION__, __LINE__));
+                             "%s,%d Get", __FILE__, __LINE__));
         ret = PtlGet(module->md_h,
                      md_offset,
                      length,
@@ -3653,7 +3655,7 @@ ompi_osc_portals4_fetch_and_op(const void *origin_addr,
         origin_md_offset = (ptl_size_t) origin_addr;
 
         OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output,
-                             "%s,%d FetchAtomic", __FUNCTION__, __LINE__));
+                             "%s,%d FetchAtomic", __FILE__, __LINE__));
         ret = PtlFetchAtomic(module->md_h,
                              result_md_offset,
                              module->md_h,
diff --git a/ompi/mca/osc/portals4/osc_portals4_component.c b/ompi/mca/osc/portals4/osc_portals4_component.c
index f9148312522..3e2b99ad039 100644
--- a/ompi/mca/osc/portals4/osc_portals4_component.c
+++ b/ompi/mca/osc/portals4/osc_portals4_component.c
@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2011-2017 Sandia National Laboratories.  All rights reserved.
+ * Copyright (c) 2011-2022 Sandia National Laboratories.  All rights reserved.
  * Copyright (c) 2015-2018 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2015-2017 Research Organization for Information Science
@@ -9,7 +9,9 @@
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
- * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2018-2022 Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2020      High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -20,10 +22,13 @@
 #include "ompi_config.h"
 
 #include "opal/util/printf.h"
+#include "opal/include/opal/align.h"
+#include "opal/mca/mpool/base/base.h"
+#include "opal/opal_portable_platform.h"
 
-#include "ompi/mca/osc/osc.h"
 #include "ompi/mca/osc/base/base.h"
 #include "ompi/mca/osc/base/osc_base_obj_convert.h"
+#include "ompi/mca/osc/osc.h"
 #include "ompi/request/request.h"
 
 #include "osc_portals4.h"
@@ -33,10 +38,10 @@ static int component_open(void);
 static int component_register(void);
 static int component_init(bool enable_progress_threads, bool enable_mpi_threads);
 static int component_finalize(void);
-static int component_query(struct ompi_win_t *win, void **base, size_t size, int disp_unit,
+static int component_query(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t disp_unit,
                            struct ompi_communicator_t *comm, struct opal_info_t *info,
                            int flavor);
-static int component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit,
+static int component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t disp_unit,
                             struct ompi_communicator_t *comm, struct opal_info_t *info,
                             int flavor, int *model);
 
@@ -44,7 +49,7 @@ static int component_select(struct ompi_win_t *win, void **base, size_t size, in
 ompi_osc_portals4_component_t mca_osc_portals4_component = {
     { /* ompi_osc_base_component_t */
         .osc_version = {
-            OMPI_OSC_BASE_VERSION_3_0_0,
+            OMPI_OSC_BASE_VERSION_4_0_0,
             .mca_component_name = "portals4",
             MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION,
                                   OMPI_RELEASE_VERSION),
@@ -109,36 +114,19 @@ ompi_osc_portals4_module_t ompi_osc_portals4_module_template = {
 static bool
 check_config_value_bool(char *key, opal_info_t *info)
 {
-    char *value_string;
-    int value_len, ret, flag, param;
-    const bool *flag_value;
-    bool result;
-
-    ret = opal_info_get_valuelen(info, key, &value_len, &flag);
-    if (OMPI_SUCCESS != ret) goto info_not_found;
-    if (flag == 0) goto info_not_found;
-    value_len++;
-
-    value_string = (char*)malloc(sizeof(char) * value_len + 1); /* Should malloc 1 char for NUL-termination */
-    if (NULL == value_string) goto info_not_found;
+    int ret, flag, param;
+    bool result = false;
+    const bool *flag_value = &result;
 
-    ret = opal_info_get(info, key, value_len, value_string, &flag);
-    if (OMPI_SUCCESS != ret) {
-        free(value_string);
-        goto info_not_found;
+    ret = opal_info_get_bool(info, key, &result, &flag);
+    if (OMPI_SUCCESS == ret && flag) {
+        return result;
     }
-    assert(flag != 0);
-    ret = opal_info_value_to_bool(value_string, &result);
-    free(value_string);
-    if (OMPI_SUCCESS != ret) goto info_not_found;
-    return result;
 
- info_not_found:
     param = mca_base_var_find("ompi", "osc", "portals4", key);
-    if (0 > param) return false;
-
-    ret = mca_base_var_get_value(param, &flag_value, NULL, NULL);
-    if (OMPI_SUCCESS != ret) return false;
+    if (0 <= param) {
+        (void) mca_base_var_get_value(param, &flag_value, NULL, NULL);
+    }
 
     return flag_value[0];
 }
@@ -147,37 +135,31 @@ check_config_value_bool(char *key, opal_info_t *info)
 static bool
 check_config_value_equal(char *key, opal_info_t *info, char *value)
 {
-    char *value_string;
-    int value_len, ret, flag, param;
-    const bool *flag_value;
+    int ret, flag, param;
+    const char *mca_value;
     bool result = false;
+    opal_cstring_t *value_string;
 
-    ret = opal_info_get_valuelen(info, key, &value_len, &flag);
-    if (OMPI_SUCCESS != ret) goto info_not_found;
-    if (flag == 0) goto info_not_found;
-    value_len++;
-
-    value_string = (char*)malloc(sizeof(char) * value_len + 1); /* Should malloc 1 char for NUL-termination */
-    if (NULL == value_string) goto info_not_found;
-
-    ret = opal_info_get(info, key, value_len, value_string, &flag);
-    if (OMPI_SUCCESS != ret) {
-        free(value_string);
+    ret = opal_info_get(info, key, &value_string, &flag);
+    if (OMPI_SUCCESS != ret || !flag) {
         goto info_not_found;
     }
-    assert(flag != 0);
-    if (0 == strcmp(value_string, value)) result = true;
-    free(value_string);
+    if (0 == strcmp(value_string->string, value)) {
+        result = true;
+    }
+    OBJ_RELEASE(value_string);
     return result;
 
  info_not_found:
     param = mca_base_var_find("ompi", "osc", "portals4", key);
     if (0 > param) return false;
 
-    ret = mca_base_var_get_value(param, &flag_value, NULL, NULL);
+    ret = mca_base_var_get_value(param, &mca_value, NULL, NULL);
     if (OMPI_SUCCESS != ret) return false;
 
-    if (0 == strcmp(value_string, value)) result = true;
+    if (0 == strcmp(mca_value, value)) {
+        result = true;
+    }
 
     return result;
 }
@@ -228,7 +210,7 @@ progress_callback(void)
             }
 
             req = (ompi_osc_portals4_request_t*) ev.user_ptr;
-            opal_atomic_add_fetch_size_t(&req->super.req_status._ucount, ev.mlength);
+            req->super.req_status._ucount = opal_atomic_add_fetch_32(&req->bytes_committed, ev.mlength);
             ops = opal_atomic_add_fetch_32(&req->ops_committed, 1);
             if (ops == req->ops_expected) {
                 ompi_request_complete(&req->super, true);
@@ -380,7 +362,7 @@ component_finalize(void)
 
 
 static int
-component_query(struct ompi_win_t *win, void **base, size_t size, int disp_unit,
+component_query(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t disp_unit,
                 struct ompi_communicator_t *comm, struct opal_info_t *info,
                 int flavor)
 {
@@ -401,19 +383,33 @@ component_query(struct ompi_win_t *win, void **base, size_t size, int disp_unit,
 
 
 static int
-component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit,
+component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t disp_unit,
                  struct ompi_communicator_t *comm, struct opal_info_t *info,
                  int flavor, int *model)
 {
     ompi_osc_portals4_module_t *module = NULL;
     int ret = OMPI_ERROR;
-    int tmp;
+    int tmp, flag;
     ptl_md_t md;
     ptl_me_t me;
     char *name;
+    size_t memory_alignment = OPAL_ALIGN_MIN;
 
     if (MPI_WIN_FLAVOR_SHARED == flavor) return OMPI_ERR_NOT_SUPPORTED;
 
+    if (NULL != info) {
+        opal_cstring_t *align_info_str;
+        opal_info_get(info, "mpi_minimum_memory_alignment",
+                      &align_info_str, &flag);
+        if (flag) {
+            size_t tmp_align = atoll(align_info_str->string);
+            OBJ_RELEASE(align_info_str);
+            if (OPAL_ALIGN_MIN < tmp_align) {
+                memory_alignment = tmp_align;
+            }
+        }
+    }
+
     /* create module structure */
     module = (ompi_osc_portals4_module_t*)
         calloc(1, sizeof(ompi_osc_portals4_module_t));
@@ -425,8 +421,10 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit
 
     /* fill in our part */
     if (MPI_WIN_FLAVOR_ALLOCATE == flavor) {
-        module->free_after = *base = malloc(size);
+        *base = mca_mpool_base_default_module->mpool_alloc(mca_mpool_base_default_module, size,
+                                                           memory_alignment, 0);
         if (NULL == *base) goto error;
+        module->free_after = *base;
     } else {
         module->free_after = NULL;
     }
@@ -436,9 +434,9 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit
 
     opal_output_verbose(1, ompi_osc_base_framework.framework_output,
                         "portals4 component creating window with id %d",
-                        ompi_comm_get_cid(module->comm));
+                        ompi_comm_get_local_cid(module->comm));
 
-    opal_asprintf(&name, "portals4 window %d", ompi_comm_get_cid(module->comm));
+    opal_asprintf(&name, "portals4 window %d", ompi_comm_get_local_cid(module->comm));
     ompi_win_set_name(win, name);
     free(name);
 
@@ -463,9 +461,9 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit
         module->disp_units = NULL;
     } else {
         module->disp_unit = -1;
-        module->disp_units = malloc(sizeof(int) * ompi_comm_size(module->comm));
-        ret = module->comm->c_coll->coll_allgather(&disp_unit, 1, MPI_INT,
-                                                  module->disp_units, 1, MPI_INT,
+        module->disp_units = malloc(sizeof(ptrdiff_t) * ompi_comm_size(module->comm));
+        ret = module->comm->c_coll->coll_allgather(&disp_unit, sizeof(ptrdiff_t), MPI_BYTE,
+                                                  module->disp_units, sizeof(ptrdiff_t), MPI_BYTE,
                                                   module->comm,
                                                   module->comm->c_coll->coll_allgather_module);
         if (OMPI_SUCCESS != ret) goto error;
@@ -525,7 +523,7 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit
     me.options = PTL_ME_OP_PUT | PTL_ME_OP_GET | PTL_ME_NO_TRUNCATE | PTL_ME_EVENT_SUCCESS_DISABLE;
     me.match_id.phys.nid = PTL_NID_ANY;
     me.match_id.phys.pid = PTL_PID_ANY;
-    me.match_bits = module->comm->c_contextid;
+    me.match_bits = ompi_comm_get_local_cid(module->comm);
     me.ignore_bits = 0;
 
     ret = PtlMEAppend(module->ni_h,
@@ -548,7 +546,7 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit
     me.options = PTL_ME_OP_PUT | PTL_ME_OP_GET | PTL_ME_NO_TRUNCATE | PTL_ME_EVENT_SUCCESS_DISABLE;
     me.match_id.phys.nid = PTL_NID_ANY;
     me.match_id.phys.pid = PTL_PID_ANY;
-    me.match_bits = module->comm->c_contextid | OSC_PORTALS4_MB_CONTROL;
+    me.match_bits = ompi_comm_get_local_cid(module->comm) | OSC_PORTALS4_MB_CONTROL;
     me.ignore_bits = 0;
 
     ret = PtlMEAppend(module->ni_h,
@@ -565,7 +563,7 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit
     }
 
     module->opcount = 0;
-    module->match_bits = module->comm->c_contextid;
+    module->match_bits = ompi_comm_get_local_cid(module->comm);
     module->atomic_max = (check_config_value_equal("accumulate_ordering", info, "none")) ?
         mca_osc_portals4_component.matching_atomic_max :
         MIN(mca_osc_portals4_component.matching_atomic_max,
@@ -592,7 +590,7 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit
 
     module->passive_target_access_epoch = false;
 
-#if OPAL_ASSEMBLY_ARCH == OPAL_X86_64 || OPAL_ASSEMBLY_ARCH == OPAL_IA32
+#if defined(PLATFORM_ARCH_X86) || defined(PLATFORM_ARCH_X86_64)
     *model = MPI_WIN_UNIFIED;
 #else
     *model = MPI_WIN_SEPARATE;
@@ -617,10 +615,10 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit
 
  error:
     /* BWB: FIX ME: This is all wrong... */
-    if (0 != module->ct_h) PtlCTFree(module->ct_h);
-    if (0 != module->data_me_h) PtlMEUnlink(module->data_me_h);
-    if (0 != module->req_md_h) PtlMDRelease(module->req_md_h);
-    if (0 != module->md_h) PtlMDRelease(module->md_h);
+    if (!PtlHandleIsEqual(module->ct_h, PTL_INVALID_HANDLE)) PtlCTFree(module->ct_h);
+    if (!PtlHandleIsEqual(module->data_me_h, PTL_INVALID_HANDLE)) PtlMEUnlink(module->data_me_h);
+    if (!PtlHandleIsEqual(module->req_md_h, PTL_INVALID_HANDLE)) PtlMDRelease(module->req_md_h);
+    if (!PtlHandleIsEqual(module->md_h, PTL_INVALID_HANDLE)) PtlMDRelease(module->md_h);
     if (NULL != module->comm) ompi_comm_free(&module->comm);
     if (NULL != module) free(module);
 
@@ -657,11 +655,11 @@ ompi_osc_portals4_free(struct ompi_win_t *win)
     PtlMEUnlink(module->control_me_h);
     PtlMEUnlink(module->data_me_h);
     PtlMDRelease(module->md_h);
-    if (module->origin_iovec_md_h != PTL_INVALID_HANDLE) {
+    if (!PtlHandleIsEqual(module->origin_iovec_md_h,PTL_INVALID_HANDLE)) {
         PtlMDRelease(module->origin_iovec_md_h);
         free(module->origin_iovec_list);
     }
-    if (module->result_iovec_md_h != PTL_INVALID_HANDLE) {
+    if (!PtlHandleIsEqual(module->result_iovec_md_h,PTL_INVALID_HANDLE)) {
         PtlMDRelease(module->result_iovec_md_h);
         free(module->result_iovec_list);
     }
@@ -669,7 +667,8 @@ ompi_osc_portals4_free(struct ompi_win_t *win)
     PtlCTFree(module->ct_h);
     if (NULL != module->disp_units) free(module->disp_units);
     ompi_comm_free(&module->comm);
-    if (NULL != module->free_after) free(module->free_after);
+    mca_mpool_base_default_module->mpool_free(mca_mpool_base_default_module,
+                                              module->free_after);
 
     if (!opal_list_is_empty(&module->outstanding_locks)) {
         ret = OMPI_ERR_RMA_SYNC;
diff --git a/ompi/mca/osc/portals4/osc_portals4_passive_target.c b/ompi/mca/osc/portals4/osc_portals4_passive_target.c
index b9baeea6f1c..1fdcfe7bb84 100644
--- a/ompi/mca/osc/portals4/osc_portals4_passive_target.c
+++ b/ompi/mca/osc/portals4/osc_portals4_passive_target.c
@@ -12,9 +12,9 @@
 
 #include "ompi_config.h"
 
-#include "ompi/mca/osc/osc.h"
 #include "ompi/mca/osc/base/base.h"
 #include "ompi/mca/osc/base/osc_base_obj_convert.h"
+#include "ompi/mca/osc/osc.h"
 
 #include "osc_portals4.h"
 
@@ -198,7 +198,7 @@ end_shared(ompi_osc_portals4_module_t *module,
 int
 ompi_osc_portals4_lock(int lock_type,
                        int target,
-                       int assert,
+                       int mpi_assert,
                        struct ompi_win_t *win)
 {
     ompi_osc_portals4_module_t *module =
@@ -211,7 +211,7 @@ ompi_osc_portals4_lock(int lock_type,
     lock = OBJ_NEW(ompi_osc_portals4_outstanding_lock_t);
     lock->target = target;
 
-    if (0 == (assert & MPI_MODE_NOCHECK)) {
+    if (0 == (mpi_assert & MPI_MODE_NOCHECK)) {
         if (MPI_LOCK_EXCLUSIVE == lock_type) {
             lock->lock_type = lock_exclusive;
             ret = start_exclusive(module, target);
@@ -276,7 +276,7 @@ ompi_osc_portals4_unlock(int target,
 
 
 int
-ompi_osc_portals4_lock_all(int assert,
+ompi_osc_portals4_lock_all(int mpi_assert,
                            struct ompi_win_t *win)
 {
     ompi_osc_portals4_module_t *module =
@@ -289,7 +289,7 @@ ompi_osc_portals4_lock_all(int assert,
     lock = OBJ_NEW(ompi_osc_portals4_outstanding_lock_t);
     lock->target = -1;
 
-    if (0 == (assert & MPI_MODE_NOCHECK)) {
+    if (0 == (mpi_assert & MPI_MODE_NOCHECK)) {
         int i, comm_size;
 
         lock->lock_type = lock_shared;
diff --git a/ompi/mca/osc/portals4/osc_portals4_request.c b/ompi/mca/osc/portals4/osc_portals4_request.c
index 19461ec0580..170a37e4819 100644
--- a/ompi/mca/osc/portals4/osc_portals4_request.c
+++ b/ompi/mca/osc/portals4/osc_portals4_request.c
@@ -9,10 +9,10 @@
 
 #include "ompi_config.h"
 
-#include "ompi/request/request.h"
-#include "ompi/mca/osc/osc.h"
 #include "ompi/mca/osc/base/base.h"
 #include "ompi/mca/osc/base/osc_base_obj_convert.h"
+#include "ompi/mca/osc/osc.h"
+#include "ompi/request/request.h"
 
 #include "osc_portals4.h"
 #include "osc_portals4_request.h"
@@ -29,7 +29,7 @@ request_free(struct ompi_request_t **ompi_req)
     ompi_osc_portals4_request_t *request =
         (ompi_osc_portals4_request_t*) *ompi_req;
 
-    if (true != request->super.req_complete) {
+    if (!REQUEST_COMPLETE(&request->super)) {
         return MPI_ERR_REQUEST;
     }
 
diff --git a/ompi/mca/osc/portals4/osc_portals4_request.h b/ompi/mca/osc/portals4/osc_portals4_request.h
index 42e38566eeb..0cade501217 100644
--- a/ompi/mca/osc/portals4/osc_portals4_request.h
+++ b/ompi/mca/osc/portals4/osc_portals4_request.h
@@ -13,12 +13,15 @@
 #ifndef OSC_PORTALS4_REQUEST_H
 #define OSC_PORTALS4_REQUEST_H
 
+#include "ompi_config.h"
+
 #include "ompi/request/request.h"
 
 struct ompi_osc_portals4_request_t {
     ompi_request_t super;
     int32_t ops_expected;
     opal_atomic_int32_t ops_committed;
+    opal_atomic_int32_t bytes_committed;
 };
 typedef struct ompi_osc_portals4_request_t ompi_osc_portals4_request_t;
 
@@ -31,11 +34,11 @@ OBJ_CLASS_DECLARATION(ompi_osc_portals4_request_t);
         req = (ompi_osc_portals4_request_t*) item;                      \
         OMPI_REQUEST_INIT(&req->super, false);                          \
         req->super.req_mpi_object.win = win;                            \
-        req->super.req_complete = false;                                \
         req->super.req_state = OMPI_REQUEST_ACTIVE;                     \
         req->super.req_status.MPI_ERROR = MPI_SUCCESS;                  \
         req->ops_expected = 0;                                          \
         req->ops_committed = 0;                                         \
+        req->bytes_committed = 0;                                       \
     } while (0)
 
 #define OMPI_OSC_PORTALS4_REQUEST_RETURN(req)                           \
diff --git a/ompi/mca/osc/rdma/Makefile.am b/ompi/mca/osc/rdma/Makefile.am
index e52d0087743..4757ce6aa93 100644
--- a/ompi/mca/osc/rdma/Makefile.am
+++ b/ompi/mca/osc/rdma/Makefile.am
@@ -11,6 +11,8 @@
 # Copyright (c) 2014-2015 Los Alamos National Security, LLC. All rights
 #                         reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+#                         All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -21,6 +23,8 @@
 rdma_sources = \
 	osc_rdma.h \
 	osc_rdma_module.c \
+	osc_rdma_btl_comm.h \
+	osc_rdma_btl_comm.c \
 	osc_rdma_comm.h \
 	osc_rdma_comm.c \
 	osc_rdma_accumulate.c \
diff --git a/ompi/mca/osc/rdma/osc_rdma.h b/ompi/mca/osc/rdma/osc_rdma.h
index e1f265dfd03..b18d2d7c4d0 100644
--- a/ompi/mca/osc/rdma/osc_rdma.h
+++ b/ompi/mca/osc/rdma/osc_rdma.h
@@ -4,7 +4,7 @@
  *                         All rights reserved.
  * Copyright (c) 2004-2006 The Trustees of the University of Tennessee.
  *                         All rights reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ * Copyright (c) 2004-2020 High Performance Computing Center Stuttgart,
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
@@ -16,6 +16,9 @@
  * Copyright (c) 2019      Triad National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2020-2021 Google, LLC. All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -42,6 +45,8 @@
 #include "ompi/mca/osc/osc.h"
 #include "ompi/mca/osc/base/base.h"
 #include "opal/mca/btl/btl.h"
+#include "opal/mca/btl/base/base.h"
+#include "opal/mca/btl/base/btl_base_am_rdma.h"
 #include "ompi/memchecker.h"
 #include "ompi/op/op.h"
 #include "opal/align.h"
@@ -55,8 +60,6 @@
 
 #define RANK_ARRAY_COUNT(module) ((ompi_comm_size ((module)->comm) + (module)->node_count - 1) / (module)->node_count)
 
-#define MCA_OSC_RDMA_MAX_USED_BTLS 2
-
 enum {
     OMPI_OSC_RDMA_LOCKING_TWO_LEVEL,
     OMPI_OSC_RDMA_LOCKING_ON_DEMAND,
@@ -108,14 +111,14 @@ struct ompi_osc_rdma_component_t {
     /** Priority of the osc/rdma component */
     unsigned int priority;
 
-    /** Priority of the osc/rdma component when using non-RDMA BTLs */
-    unsigned int alternate_priority;
-
     /** directory where to place backing files */
     char *backing_directory;
 
     /** maximum count for network AMO usage */
     unsigned long network_amo_max_count;
+
+    /** memory alignment to be used for new windows */
+    size_t memory_alignment;
 };
 typedef struct ompi_osc_rdma_component_t ompi_osc_rdma_component_t;
 
@@ -148,9 +151,6 @@ struct ompi_osc_rdma_module_t {
     /** value of same_size info key for this window */
     bool same_size;
 
-    /** CPU atomics can be used */
-    bool use_cpu_atomics;
-
     /** passive-target synchronization will not be used in this window */
     bool no_locks;
 
@@ -168,7 +168,7 @@ struct ompi_osc_rdma_module_t {
     size_t size;
 
     /** Local displacement unit. */
-    int disp_unit;
+    ptrdiff_t disp_unit;
 
     /** maximum count for network AMO usage */
     unsigned long network_amo_max_count;
@@ -221,6 +221,9 @@ struct ompi_osc_rdma_module_t {
     /** offset in the shared memory segment where the state array starts */
     size_t state_offset;
 
+    /** memory alignment to be used for new windows */
+    size_t memory_alignment;
+
     /* ********************* sync data ************************ */
 
     /** global sync object (PSCW, fence, lock all) */
@@ -255,17 +258,38 @@ struct ompi_osc_rdma_module_t {
     /** lock for peer hash table/array */
     opal_mutex_t peer_lock;
 
+    /* ******************* communication *********************** */
+
+    /* we currently support two modes of operation, a single
+     * accelerated btl (which can use memory registration and can use
+     * btl_flush() and one or more alternate btls, which cannot use
+     * flush() or rely on memory registration.  Since it is an
+     * either/or situation, we use a union to simplify the code.
+     */
+    bool use_accelerated_btl;
+
+    union {
+        struct {
+            mca_btl_base_module_t *accelerated_btl;
+        };
+        struct {
+            mca_btl_base_am_rdma_module_t **alternate_am_rdmas;
+            uint8_t alternate_btl_count;
+        };
+    };
+
+    /** Does the selected BTL require memory registration?  This field
+        will be false when alternate BTLs are used, and the value
+        when an accelerated BTL is used depends on the registration
+        requirements of the underlying BTL. */
+    bool use_memory_registration;
 
-    /** BTL(s) in use. Currently this is only used to support RDMA emulation over
-     * non-RDMA BTLs. The typical usage is btl/sm + btl/tcp. In the future this
-     * could be used to support multiple RDMA-capable BTLs but the memory registration
-     * paths will need to be updated to pack/unpack multiple registration handles. */
-    struct mca_btl_base_module_t *selected_btls[MCA_OSC_RDMA_MAX_USED_BTLS];
-    uint8_t btls_in_use;
+    size_t put_alignment;
+    size_t get_alignment;
+    size_t put_limit;
+    size_t get_limit;
 
-    /** Only true if one BTL is in use. Memory registration is only supported when
-     * using a single BTL. */
-    bool use_memory_registration;
+    uint32_t atomic_flags;
 
     /** registered fragment used for locally buffered RDMA transfers */
     struct ompi_osc_rdma_frag_t *rdma_frag;
@@ -295,7 +319,7 @@ struct ompi_osc_rdma_module_t {
     opal_atomic_int32_t pending_ops;
 };
 typedef struct ompi_osc_rdma_module_t ompi_osc_rdma_module_t;
-OMPI_MODULE_DECLSPEC extern ompi_osc_rdma_component_t mca_osc_rdma_component;
+OMPI_DECLSPEC extern ompi_osc_rdma_component_t mca_osc_rdma_component;
 
 #define GET_MODULE(win) ((ompi_osc_rdma_module_t*) win->w_osc_module)
 
@@ -377,10 +401,11 @@ static inline int _ompi_osc_rdma_register (ompi_osc_rdma_module_t *module, struc
                                            size_t size, uint32_t flags, mca_btl_base_registration_handle_t **handle, int line, const char *file)
 {
     if (module->use_memory_registration) {
-        OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_INFO, "registering segment with btl. range: %p - %p (%lu bytes)",
+        assert(module->use_accelerated_btl);
+        OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_INFO, "registering segment with btl. range: %p - %p (%zu bytes)",
                          ptr, (void*)((char *) ptr + size), size);
 
-        *handle = module->selected_btls[0]->btl_register_mem (module->selected_btls[0], endpoint, ptr, size, flags);
+        *handle = module->accelerated_btl->btl_register_mem(module->accelerated_btl, endpoint, ptr, size, flags);
         if (OPAL_UNLIKELY(NULL == *handle)) {
             OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_DEBUG, "failed to register pointer with selected BTL. base: %p, "
                              "size: %lu. file: %s, line: %d", ptr, (unsigned long) size, file, line);
@@ -398,7 +423,9 @@ static inline int _ompi_osc_rdma_register (ompi_osc_rdma_module_t *module, struc
 static inline void _ompi_osc_rdma_deregister (ompi_osc_rdma_module_t *module, mca_btl_base_registration_handle_t *handle, int line, const char *file)
 {
     if (handle) {
-        module->selected_btls[0]->btl_deregister_mem (module->selected_btls[0], handle);
+        assert(module->use_memory_registration);
+        assert(module->use_accelerated_btl);
+        module->accelerated_btl->btl_deregister_mem(module->accelerated_btl, handle);
     }
 }
 
@@ -530,10 +557,11 @@ static inline ompi_osc_rdma_sync_t *ompi_osc_rdma_module_sync_lookup (ompi_osc_r
 static bool ompi_osc_rdma_use_btl_flush (ompi_osc_rdma_module_t *module)
 {
 #if defined(BTL_VERSION) && (BTL_VERSION >= 310)
-    return !!(module->selected_btls[0]->btl_flush);
-#else
-    return false;
+    if (module->use_accelerated_btl) {
+        return (NULL != module->accelerated_btl->btl_flush);
+    }
 #endif
+    return false;
 }
 
 /**
@@ -595,13 +623,13 @@ static inline void ompi_osc_rdma_sync_rdma_complete (ompi_osc_rdma_sync_t *sync)
         opal_progress ();
     }  while (ompi_osc_rdma_sync_get_count (sync));
 #else
-    mca_btl_base_module_t *btl_module = sync->module->selected_btls[0];
-
     do {
         if (!ompi_osc_rdma_use_btl_flush (sync->module)) {
             opal_progress ();
         } else {
-            btl_module->btl_flush (btl_module, NULL);
+            assert(sync->module->use_accelerated_btl);
+            mca_btl_base_module_t *btl_module = sync->module->accelerated_btl;
+            btl_module->btl_flush(btl_module, NULL);
         }
     }  while (ompi_osc_rdma_sync_get_count (sync) || (sync->module->rdma_frag && (sync->module->rdma_frag->pending > 1)));
 #endif
@@ -631,7 +659,137 @@ static inline bool ompi_osc_rdma_oor (int rc)
 
 __opal_attribute_always_inline__
 static inline mca_btl_base_module_t *ompi_osc_rdma_selected_btl (ompi_osc_rdma_module_t *module, uint8_t btl_index) {
-    return module->selected_btls[btl_index];
+    if (module->use_accelerated_btl) {
+        assert(0 == btl_index);
+        return module->accelerated_btl;
+    } else {
+        assert(btl_index < module->alternate_btl_count);
+        return module->alternate_am_rdmas[btl_index]->btl;
+    }
 }
 
+
+static inline mca_btl_base_am_rdma_module_t *ompi_osc_rdma_selected_am_rdma(ompi_osc_rdma_module_t *module, uint8_t btl_index) {
+    assert(!module->use_accelerated_btl);
+    assert(btl_index < module->alternate_btl_count);
+    return module->alternate_am_rdmas[btl_index];
+}
+
+/**
+ * @brief Copies from src to dest buffers that may or may not be accelerator
+ *        buffers
+ *
+ * @param[in]  dest           destination buffer
+ * @param[in]  src            source buffer
+ * @param[in]  size           length to copy
+ *
+ * @returns OMPI_SUCCESS or error
+ */
+static inline int osc_rdma_accelerator_mem_copy(void *dest, const void *src, size_t size)
+{
+    int res, dev_id;
+    uint64_t flags;
+    int dest_is_accel = opal_accelerator.check_addr(dest, &dev_id, &flags);
+    int src_is_accel = opal_accelerator.check_addr(src, &dev_id, &flags);
+
+    if (0 == dest_is_accel && 0 == src_is_accel) {
+        memcpy(dest, src, size);
+        return OMPI_SUCCESS;
+    } else if (0 > dest_is_accel) {
+        return dest_is_accel;
+    } else if (0 > src_is_accel) {
+        return src_is_accel;
+    }
+
+    res = opal_accelerator.mem_copy(MCA_ACCELERATOR_NO_DEVICE_ID, MCA_ACCELERATOR_NO_DEVICE_ID,
+                                  dest, src, size, MCA_ACCELERATOR_TRANSFER_UNSPEC);
+    if (OPAL_SUCCESS != res) {
+        opal_output(0, "Error in accelerator memcpy");
+    }
+    return res;
+}
+
+/**
+ * @brief Moves memory from src to dest buffers that may or may not be accelerator
+ *        buffers
+ *
+ * @param[in]  dest           destination buffer
+ * @param[in]  src            source buffer
+ * @param[in]  size           length to copy
+ *
+ * @returns OMPI_SUCCESS or error;
+ */
+static inline int osc_rdma_accelerator_mem_move(void *dest, const void *src, size_t size)
+{
+    int res, dev_id;
+    uint64_t flags;
+    int dest_is_accel = opal_accelerator.check_addr(dest, &dev_id, &flags);
+    int src_is_accel = opal_accelerator.check_addr(src, &dev_id, &flags);
+
+    if (0 == dest_is_accel && 0 == src_is_accel) {
+        memmove(dest, src, size);
+        return OMPI_SUCCESS;
+    } else if (0 > dest_is_accel) {
+        return dest_is_accel;
+    } else if (0 > src_is_accel) {
+        return src_is_accel;
+    }
+
+    res = opal_accelerator.mem_move(MCA_ACCELERATOR_NO_DEVICE_ID, MCA_ACCELERATOR_NO_DEVICE_ID,
+                                  dest, src, size, MCA_ACCELERATOR_TRANSFER_UNSPEC);
+    if (OPAL_SUCCESS != res) {
+        opal_output(0, "Error in accelerator memmove");
+    }
+    return res;
+}
+
+/**
+ * @brief Identifies whether the provided buffer is an accelerator buffer.
+ *
+ * @param[in]  buf            buffer
+ *
+ * @retval <0                 An error has occurred.
+ * @retval 0                  The buffer does not belong to a managed buffer
+ *                            in device memory.
+ * @retval >0                 The buffer belongs to a managed buffer in
+ *                            device memory.
+ */
+static inline int osc_rdma_is_accel(const void *buf)
+{
+    int dev_id;
+    uint64_t flags;
+    return opal_accelerator.check_addr(buf, &dev_id, &flags);
+}
+
+/**
+ * @brief Checks whether any btl's are capable of accelerator support.
+ *
+ * @param[in] btls            The modules to check for btl accelerator support
+ *
+ * @returns true              If any btl module has accelerator support
+ * @returns false             Otherwise
+ */
+static inline bool osc_rdma_btl_accel_support(opal_list_t *btls)
+{
+    mca_btl_base_selected_module_t* selected_btl;
+    /* verify if we have any btls available.  Since we do not verify
+     * connectivity across all btls in the alternate case, this is as
+     * good a test as we are going to have for success. */
+    if (opal_list_is_empty(btls)) {
+        return false;
+    }
+
+    /* If we have any btls, we check again if any btl supports
+     * MCA_BTL_FLAGS_ACCELERATOR_RDMA */
+    OPAL_LIST_FOREACH(selected_btl, btls, mca_btl_base_selected_module_t) {
+        opal_output_verbose(MCA_BASE_VERBOSE_INFO, ompi_osc_base_framework.framework_output,
+                    "osc_rdma_component_query: check ACCELERATOR_RDMA flag: %s",
+                    selected_btl->btl_component->btl_version.mca_component_name);
+        mca_btl_base_module_t *btl = selected_btl->btl_module;
+        if (btl->btl_flags & MCA_BTL_FLAGS_ACCELERATOR_RDMA) {
+            return true;
+        }
+    }
+    return false;
+}
 #endif /* OMPI_OSC_RDMA_H */
diff --git a/ompi/mca/osc/rdma/osc_rdma_accumulate.c b/ompi/mca/osc/rdma/osc_rdma_accumulate.c
index d02c0ee5377..06315df89d3 100644
--- a/ompi/mca/osc/rdma/osc_rdma_accumulate.c
+++ b/ompi/mca/osc/rdma/osc_rdma_accumulate.c
@@ -5,9 +5,14 @@
  * Copyright (c) 2016-2019 Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2016-2018 Intel, Inc. All rights reserved.
- * Copyright (c) 2019      Triad National Security, LLC. All rights
+ * Copyright (c) 2019-2022 Triad National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2019-2021 Google, LLC. All rights reserved.
+ * Copyright (c) 2021      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -15,10 +20,16 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "osc_rdma_accumulate.h"
 #include "osc_rdma_request.h"
 #include "osc_rdma_comm.h"
+#include "osc_rdma_lock.h"
+#include "osc_rdma_btl_comm.h"
 
+#include "opal/util/minmax.h"
+#include "ompi/mca/osc/base/base.h"
 #include "ompi/mca/osc/base/osc_base_obj_convert.h"
 
 static inline void ompi_osc_rdma_peer_accumulate_cleanup (ompi_osc_rdma_module_t *module, ompi_osc_rdma_peer_t *peer, bool lock_acquired)
@@ -53,14 +64,15 @@ struct ompi_osc_rdma_event_t {
 
 typedef struct ompi_osc_rdma_event_t ompi_osc_rdma_event_t;
 
-static int ompi_osc_rdma_gacc_local (const void *source_buffer, int source_count, ompi_datatype_t *source_datatype,
-                                     void *result_buffer, int result_count, ompi_datatype_t *result_datatype,
+static int ompi_osc_rdma_gacc_local (const void *source_buffer, size_t source_count, ompi_datatype_t *source_datatype,
+                                     void *result_buffer, size_t result_count, ompi_datatype_t *result_datatype,
                                      ompi_osc_rdma_peer_t *peer, uint64_t target_address,
-                                     mca_btl_base_registration_handle_t *target_handle, int target_count,
+                                     mca_btl_base_registration_handle_t *target_handle, size_t target_count,
                                      ompi_datatype_t *target_datatype, ompi_op_t *op, ompi_osc_rdma_module_t *module,
                                      ompi_osc_rdma_request_t *request, bool lock_acquired)
 {
     int ret = OMPI_SUCCESS;
+    void *tmp_source;
 
     do {
         OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "performing accumulate with local region(s)");
@@ -78,8 +90,21 @@ static int ompi_osc_rdma_gacc_local (const void *source_buffer, int source_count
 
         if (&ompi_mpi_op_no_op.op != op) {
             if (&ompi_mpi_op_replace.op != op) {
-                ret = ompi_osc_base_sndrcv_op (source_buffer, source_count, source_datatype, (void *) (intptr_t) target_address,
-                                               target_count, target_datatype, op);
+                /* sendrcv op does not work with accel buffers for non managed memory, do a copy to host buffers first */
+                ret = osc_rdma_is_accel(source_buffer);
+                if (0 < ret) {
+                    tmp_source = malloc(source_count * source_datatype->super.size);
+                    ret = opal_accelerator.mem_copy(MCA_ACCELERATOR_NO_DEVICE_ID, MCA_ACCELERATOR_NO_DEVICE_ID,
+                                                tmp_source, source_buffer, source_count * source_datatype->super.size, MCA_ACCELERATOR_TRANSFER_DTOH);
+                    ret = ompi_osc_base_sndrcv_op (tmp_source, source_count, source_datatype, (void *) (intptr_t) target_address,
+                                                   target_count, target_datatype, op);
+                    free(tmp_source);
+                } else if (0 == ret) {
+                    ret = ompi_osc_base_sndrcv_op (source_buffer, source_count, source_datatype, (void *) (intptr_t) target_address,
+                                                   target_count, target_datatype, op);
+                } else {
+                    return ret;
+                }
             } else {
                 ret = ompi_datatype_sndrcv (source_buffer, source_count, source_datatype, (void *) (intptr_t) target_address,
                                             target_count, target_datatype);
@@ -107,17 +132,74 @@ static inline int ompi_osc_rdma_cas_local (const void *source_addr, const void *
                                            uint64_t target_address, mca_btl_base_registration_handle_t *target_handle,
                                            ompi_osc_rdma_module_t *module, bool lock_acquired)
 {
+    int ret, result_is_accel, target_is_accel, compare_is_accel;
+    const void *compare_copy;
+    void *compare_to_free = NULL;
+    void *result_copy = NULL;
+    bool compare_copied = false;
+    bool result_copied = false;
+
     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "performing compare-and-swap with local regions");
+    result_is_accel = osc_rdma_is_accel(result_addr);
+    target_is_accel = osc_rdma_is_accel((void *)target_address);
+    compare_is_accel = osc_rdma_is_accel(compare_addr);
+    if (0 < result_is_accel || 0 < target_is_accel) {
+        ret = opal_accelerator.mem_copy(MCA_ACCELERATOR_NO_DEVICE_ID, MCA_ACCELERATOR_NO_DEVICE_ID,
+                                        result_addr, (void *) (uintptr_t) target_address, datatype->super.size, MCA_ACCELERATOR_TRANSFER_UNSPEC);
+
+    } else if (0 == result_is_accel && 0 == target_is_accel) {
+        memcpy(result_addr, (void *) (uintptr_t) target_address, datatype->super.size);
+    } else if (0 > result_is_accel) {
+        return result_is_accel;
+    } else if (0 > target_is_accel) {
+        return target_is_accel;
+    }
 
-    memcpy (result_addr, (void *) (uintptr_t) target_address, datatype->super.size);
+    /* Check if either addr is accelerator. Will have to perform some copies if so */
+    if (result_is_accel) {
+        result_copy = malloc(datatype->super.size);
+        ret = opal_accelerator.mem_copy(MCA_ACCELERATOR_NO_DEVICE_ID, MCA_ACCELERATOR_NO_DEVICE_ID,
+                                        result_copy, result_addr, datatype->super.size, MCA_ACCELERATOR_TRANSFER_DTOH);
+        result_copied = true;
+        if (OPAL_SUCCESS != ret) {
+            goto out;
+	}
+    } else {
+        result_copy = result_addr;
+    }
+
+    if (compare_is_accel) {
+        compare_copy = compare_to_free = malloc(datatype->super.size);
+        ret = opal_accelerator.mem_copy(MCA_ACCELERATOR_NO_DEVICE_ID, MCA_ACCELERATOR_NO_DEVICE_ID,
+                                        compare_to_free, compare_addr, datatype->super.size, MCA_ACCELERATOR_TRANSFER_DTOH);
+        compare_copied = true;
+        if (OPAL_SUCCESS != ret) {
+            goto out;
+        }
+    } else {
+        compare_copy = compare_addr;
+    }
 
-    if (0 == memcmp (compare_addr, result_addr, datatype->super.size)) {
-        memcpy ((void *) (uintptr_t) target_address, source_addr, datatype->super.size);
+    if (0 == memcmp (compare_copy, result_copy, datatype->super.size)) {
+        ret = osc_rdma_accelerator_mem_copy((void *) (uintptr_t) target_address, source_addr, datatype->super.size);
+        if (ret) {
+            goto out;
+        }
+    } else {
+        ret = OMPI_SUCCESS;
     }
 
     ompi_osc_rdma_peer_accumulate_cleanup (module, peer, lock_acquired);
 
-    return OMPI_SUCCESS;
+out:
+    if (compare_copied) {
+        free(compare_to_free);
+    }
+    if (result_copied) {
+        free(result_copy);
+    }
+
+    return ret;
 }
 
 static int ompi_osc_rdma_op_mapping[OMPI_OP_NUM_OF_TYPES + 1] = {
@@ -133,33 +215,47 @@ static int ompi_osc_rdma_op_mapping[OMPI_OP_NUM_OF_TYPES + 1] = {
     [OMPI_OP_REPLACE] = MCA_BTL_ATOMIC_SWAP,
 };
 
+/* set the appropriate flags for this atomic */
+static inline int ompi_osc_rdma_set_btl_flags(ompi_osc_rdma_module_t *module, ompi_datatype_t *dt, ptrdiff_t extent) {
+
+    int flags = 0;
+
+    if(4 == extent) {
+        flags = MCA_BTL_ATOMIC_FLAG_32BIT;
+    }
+
+    if (OMPI_DATATYPE_FLAG_DATA_FLOAT & dt->super.flags) {
+        flags |= MCA_BTL_ATOMIC_FLAG_FLOAT;
+    }
+
+    return flags;
+}
+
 static int ompi_osc_rdma_fetch_and_op_atomic (ompi_osc_rdma_sync_t *sync, const void *origin_addr, void *result_addr, ompi_datatype_t *dt,
                                               ptrdiff_t extent, ompi_osc_rdma_peer_t *peer, uint64_t target_address,
                                               mca_btl_base_registration_handle_t *target_handle, ompi_op_t *op, ompi_osc_rdma_request_t *req)
 {
     ompi_osc_rdma_module_t *module = sync->module;
-    mca_btl_base_module_t *selected_btl = ompi_osc_rdma_selected_btl (module, peer->data_btl_index);
-    int32_t atomic_flags = selected_btl->btl_atomic_flags;
-    int btl_op, flags;
+    int btl_op, flags, ret;
     int64_t origin;
 
-    if ((8 != extent && !((MCA_BTL_ATOMIC_SUPPORTS_32BIT & atomic_flags) && 4 == extent)) ||
-        (!(OMPI_DATATYPE_FLAG_DATA_INT & dt->super.flags) && !(MCA_BTL_ATOMIC_SUPPORTS_FLOAT & atomic_flags)) ||
+    if ((8 != extent && !((MCA_BTL_ATOMIC_SUPPORTS_32BIT & module->atomic_flags) && 4 == extent)) ||
+        (!(OMPI_DATATYPE_FLAG_DATA_INT & dt->super.flags) && !(MCA_BTL_ATOMIC_SUPPORTS_FLOAT & module->atomic_flags)) ||
+        ((OMPI_DATATYPE_FLAG_DATA_COMPLEX & dt->super.flags) == OMPI_DATATYPE_FLAG_DATA_COMPLEX)  ||
         !ompi_op_is_intrinsic (op) || (0 == ompi_osc_rdma_op_mapping[op->op_type])) {
         return OMPI_ERR_NOT_SUPPORTED;
     }
 
     btl_op = ompi_osc_rdma_op_mapping[op->op_type];
 
-    flags = (4 == extent) ? MCA_BTL_ATOMIC_FLAG_32BIT : 0;
-    if (OMPI_DATATYPE_FLAG_DATA_FLOAT & dt->super.flags) {
-        flags |= MCA_BTL_ATOMIC_FLAG_FLOAT;
-    }
+    flags = ompi_osc_rdma_set_btl_flags(module, dt, extent);
 
     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "initiating fetch-and-op using %d-bit btl atomics. origin: 0x%" PRIx64,
                      (4 == extent) ? 32 : 64, *((int64_t *) origin_addr));
-
-    origin = (8 == extent) ? ((int64_t *) origin_addr)[0] : ((int32_t *) origin_addr)[0];
+    ret = osc_rdma_accelerator_mem_copy(&origin, origin_addr, extent);
+    if (ret) {
+        return ret;
+    }
 
     return ompi_osc_rdma_btl_fop (module, peer->data_btl_index, peer->data_endpoint, target_address, target_handle, btl_op, origin, flags,
                                  result_addr, true, NULL, NULL, NULL);
@@ -172,6 +268,7 @@ static int ompi_osc_rdma_fetch_and_op_cas (ompi_osc_rdma_sync_t *sync, const voi
     ompi_osc_rdma_module_t *module = sync->module;
     uint64_t address, offset, new_value, old_value;
     int ret;
+    void *tmp_origin;
 
     if (extent > 8) {
         return OMPI_ERR_NOT_SUPPORTED;
@@ -193,9 +290,23 @@ static int ompi_osc_rdma_fetch_and_op_cas (ompi_osc_rdma_sync_t *sync, const voi
         new_value = old_value;
 
         if (&ompi_mpi_op_replace.op == op) {
-            memcpy ((void *)((intptr_t) &new_value + offset), (void *)((intptr_t) origin_addr + dt->super.true_lb), extent);
-        } else if (&ompi_mpi_op_no_op.op != op) {
-            ompi_op_reduce (op, (void *) ((intptr_t) origin_addr + dt->super.true_lb), (void*)((intptr_t) &new_value + offset), 1, dt);
+            ret = osc_rdma_accelerator_mem_copy((void *)((ptrdiff_t) &new_value + offset), (void *)((ptrdiff_t) origin_addr + dt->super.true_lb), extent);
+            if (ret) {
+                return ret;
+            }
+	} else if (&ompi_mpi_op_no_op.op != op) {
+            ret = osc_rdma_is_accel(((const char*) origin_addr) + dt->super.true_lb);
+            if (0 < ret) {
+                tmp_origin = malloc(dt->super.size);
+                ret = opal_accelerator.mem_copy(MCA_ACCELERATOR_NO_DEVICE_ID, MCA_ACCELERATOR_NO_DEVICE_ID,
+                                                tmp_origin, ((const char*) origin_addr) + dt->super.true_lb, dt->super.size, MCA_ACCELERATOR_TRANSFER_DTOH);
+                ompi_op_reduce (op, (void *) tmp_origin, (void*)((ptrdiff_t) &new_value + offset), 1, dt);
+                free(tmp_origin);
+            } else if (0 == ret) {
+                ompi_op_reduce (op, (void *) ((ptrdiff_t) origin_addr + dt->super.true_lb), (void*)((ptrdiff_t) &new_value + offset), 1, dt);
+            } else {
+                return ret;
+            }
         }
 
         ret = ompi_osc_rdma_btl_cswap (module, peer->data_btl_index, peer->data_endpoint, address, target_handle,
@@ -208,7 +319,7 @@ static int ompi_osc_rdma_fetch_and_op_cas (ompi_osc_rdma_sync_t *sync, const voi
     } while (1);
 
     if (result_addr) {
-        memcpy (result_addr, (void *)((intptr_t) &new_value + offset), extent);
+        ret = osc_rdma_accelerator_mem_copy(result_addr, (void *)((intptr_t) &new_value + offset), extent);
     }
 
     return ret;
@@ -219,30 +330,22 @@ static int ompi_osc_rdma_acc_single_atomic (ompi_osc_rdma_sync_t *sync, const vo
                                             ompi_op_t *op, ompi_osc_rdma_request_t *req)
 {
     ompi_osc_rdma_module_t *module = sync->module;
-    mca_btl_base_module_t *selected_btl = ompi_osc_rdma_selected_btl (module, peer->data_btl_index);
-    int32_t atomic_flags = selected_btl->btl_atomic_flags;
-    int btl_op, flags;
+    int btl_op, flags, ret;
     int64_t origin;
 
-    if (!(selected_btl->btl_flags & MCA_BTL_FLAGS_ATOMIC_OPS)) {
-        /* btl put atomics not supported or disabled. fall back on fetch-and-op */
-        return ompi_osc_rdma_fetch_and_op_atomic (sync, origin_addr, NULL, dt, extent, peer, target_address, target_handle,
-                                                  op, req);
-    }
-
-    if ((8 != extent && !((MCA_BTL_ATOMIC_SUPPORTS_32BIT & atomic_flags) && 4 == extent)) ||
-        (!(OMPI_DATATYPE_FLAG_DATA_INT & dt->super.flags) && !(MCA_BTL_ATOMIC_SUPPORTS_FLOAT & atomic_flags)) ||
+    if ((8 != extent && !((MCA_BTL_ATOMIC_SUPPORTS_32BIT & module->atomic_flags) && 4 == extent)) ||
+        (!(OMPI_DATATYPE_FLAG_DATA_INT & dt->super.flags) && !(MCA_BTL_ATOMIC_SUPPORTS_FLOAT & module->atomic_flags)) ||
+        ((OMPI_DATATYPE_FLAG_DATA_COMPLEX & dt->super.flags) == OMPI_DATATYPE_FLAG_DATA_COMPLEX)  ||
         !ompi_op_is_intrinsic (op) || (0 == ompi_osc_rdma_op_mapping[op->op_type])) {
         return OMPI_ERR_NOT_SUPPORTED;
     }
 
-    origin = (8 == extent) ? ((uint64_t *) origin_addr)[0] : ((uint32_t *) origin_addr)[0];
-
-    /* set the appropriate flags for this atomic */
-    flags = (4 == extent) ? MCA_BTL_ATOMIC_FLAG_32BIT : 0;
-    if (OMPI_DATATYPE_FLAG_DATA_FLOAT & dt->super.flags) {
-        flags |= MCA_BTL_ATOMIC_FLAG_FLOAT;
+    ret = osc_rdma_accelerator_mem_copy(&origin, origin_addr, extent);
+    if (ret) {
+        return ret;
     }
+    /* set the appropriate flags for this atomic */
+    flags = ompi_osc_rdma_set_btl_flags(module, dt, extent);
 
     btl_op = ompi_osc_rdma_op_mapping[op->op_type];
 
@@ -255,26 +358,27 @@ static int ompi_osc_rdma_acc_single_atomic (ompi_osc_rdma_sync_t *sync, const vo
 }
 
 static inline int ompi_osc_rdma_gacc_amo (ompi_osc_rdma_module_t *module, ompi_osc_rdma_sync_t *sync, const void *source, void *result,
-                                          int result_count, ompi_datatype_t *result_datatype, opal_convertor_t *result_convertor,
+                                          size_t result_count, ompi_datatype_t *result_datatype, opal_convertor_t *result_convertor,
                                           ompi_osc_rdma_peer_t *peer, uint64_t target_address,
-                                          mca_btl_base_registration_handle_t *target_handle, int count,
+                                          mca_btl_base_registration_handle_t *target_handle, size_t count,
                                           ompi_datatype_t *datatype, ompi_op_t *op, ompi_osc_rdma_request_t *request)
 {
     const bool use_amo = module->acc_use_amo;
     const size_t dt_size = datatype->super.size;
+    void *result_start = result;
     void *to_free = NULL;
     int ret;
 
-    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "using network atomics for accumulate operation with count %d", count);
+    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "using network atomics for accumulate operation with count %zu", count);
 
     if (NULL == result) {
-        to_free = result = malloc (request->len);
+        to_free = result_start = result = malloc (request->len);
         if (OPAL_UNLIKELY(NULL == result)) {
             return OMPI_ERR_OUT_OF_RESOURCE;
        }
     }
 
-    for (int i = 0 ; i < count ; ) {
+    for (size_t i = 0 ; i < count ; ) {
         if (use_amo) {
             if (NULL == result) {
                 ret = ompi_osc_rdma_acc_single_atomic (sync, source, datatype, dt_size, peer, target_address, target_handle, op, request);
@@ -297,6 +401,7 @@ static inline int ompi_osc_rdma_gacc_amo (ompi_osc_rdma_module_t *module, ompi_o
             target_address += dt_size;
             ++i;
         } else if (OPAL_UNLIKELY(OMPI_ERR_NOT_SUPPORTED == ret)) {
+            free(to_free);
             return OMPI_ERR_NOT_SUPPORTED;
         }
     }
@@ -304,7 +409,8 @@ static inline int ompi_osc_rdma_gacc_amo (ompi_osc_rdma_module_t *module, ompi_o
     if (NULL != result_convertor) {
         /* result buffer is not necessarily contiguous. use the opal datatype engine to
          * copy the data over in this case */
-        struct iovec iov = {.iov_base = result, .iov_len = request->len};
+        size_t len = count * dt_size;
+        struct iovec iov = {.iov_base = result_start, .iov_len = len};
         uint32_t iov_count = 1;
         size_t size = request->len;
 
@@ -320,19 +426,21 @@ static inline int ompi_osc_rdma_gacc_amo (ompi_osc_rdma_module_t *module, ompi_o
     return OMPI_SUCCESS;
 }
 
-static inline int ompi_osc_rdma_gacc_contig (ompi_osc_rdma_sync_t *sync, const void *source, int source_count,
-                                             ompi_datatype_t *source_datatype, void *result, int result_count,
+static inline int ompi_osc_rdma_gacc_contig (ompi_osc_rdma_sync_t *sync, const void *source, size_t source_count,
+                                             ompi_datatype_t *source_datatype, void *result, size_t result_count,
                                              ompi_datatype_t *result_datatype, opal_convertor_t *result_convertor,
                                              ompi_osc_rdma_peer_t *peer, uint64_t target_address,
-                                             mca_btl_base_registration_handle_t *target_handle, int target_count,
+                                             mca_btl_base_registration_handle_t *target_handle, size_t target_count,
                                              ompi_datatype_t *target_datatype, ompi_op_t *op, ompi_osc_rdma_request_t *request)
 {
     ompi_osc_rdma_module_t *module = sync->module;
-    unsigned long len = target_count * target_datatype->super.size;
+    size_t target_dtype_size = target_datatype->super.size;
+    unsigned long len = target_count * target_dtype_size;
     char *ptr = NULL;
     int ret;
+    void *tmp_source;
 
-    request->len = target_datatype->super.size * module->network_amo_max_count;
+    request->len = target_dtype_size * module->network_amo_max_count;
 
     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "initiating accumulate on contiguous region of %lu bytes to remote address %" PRIx64
                      ", sync %p", len, target_address, (void *) sync);
@@ -340,7 +448,8 @@ static inline int ompi_osc_rdma_gacc_contig (ompi_osc_rdma_sync_t *sync, const v
     /* if the datatype is small enough (and the count is 1) then try to directly use the hardware to execute
      * the atomic operation. this should be safe in all cases as either 1) the user has assured us they will
      * never use atomics with count > 1, 2) we have the accumulate lock, or 3) we have an exclusive lock */
-    if ((target_datatype->super.size <= 8) && (((unsigned long) target_count) <= module->network_amo_max_count)) {
+    if ((target_dtype_size <= 8) && (((unsigned long) target_count) <= module->network_amo_max_count) &&
+         ompi_osc_base_is_atomic_size_supported(target_address, target_dtype_size)) {
         ret = ompi_osc_rdma_gacc_amo (module, sync, source, result, result_count, result_datatype, result_convertor,
                                       peer, target_address, target_handle, target_count, target_datatype, op, request);
         if (OPAL_LIKELY(OMPI_SUCCESS == ret)) {
@@ -349,7 +458,7 @@ static inline int ompi_osc_rdma_gacc_contig (ompi_osc_rdma_sync_t *sync, const v
         OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "network atomics not available. falling back to get-op-put implementation...");
     }
 
-    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "using get-op-put to execute accumulate with count %d", target_count);
+    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "using get-op-put to execute accumulate with count %zu", target_count);
 
     if (&ompi_mpi_op_replace.op != op || OMPI_OSC_RDMA_TYPE_GET_ACC == request->type) {
         ptr = malloc (len);
@@ -387,8 +496,21 @@ static inline int ompi_osc_rdma_gacc_contig (ompi_osc_rdma_sync_t *sync, const v
         }
 
         if (&ompi_mpi_op_no_op.op != op) {
-            /* NTH: need to cast away const for the source buffer. the buffer will not be modified by this call */
-            ompi_op_reduce (op, (void *) source, ptr, source_count, source_datatype);
+            /* Cannot call ompi_op_reduce on a device buffer for non managed
+             * memory. Copy into temporary buffer first */
+            ret = osc_rdma_is_accel(source);
+            if (0 < ret) {
+                tmp_source = malloc(len);
+                ret = opal_accelerator.mem_copy(MCA_ACCELERATOR_NO_DEVICE_ID, MCA_ACCELERATOR_NO_DEVICE_ID,
+                                                tmp_source, source, len, MCA_ACCELERATOR_TRANSFER_DTOH);
+                ompi_op_reduce (op, (void *) tmp_source, ptr, source_count, source_datatype);
+                free(tmp_source);
+            } else if (0 == ret){
+                /* NTH: need to cast away const for the source buffer. the buffer will not be modified by this call */
+                ompi_op_reduce (op, (void *) source, ptr, source_count, source_datatype);
+            } else {
+                return ret;
+            }
 
             return ompi_osc_rdma_put_contig (sync, peer, target_address, target_handle, ptr, len, request);
         }
@@ -407,10 +529,10 @@ static void ompi_osc_rdma_gacc_master_cleanup (ompi_osc_rdma_request_t *request)
     ompi_osc_rdma_peer_accumulate_cleanup (request->module, request->peer, !ompi_osc_rdma_peer_is_exclusive (request->peer));
 }
 
-static inline int ompi_osc_rdma_gacc_master (ompi_osc_rdma_sync_t *sync, const void *source_addr, int source_count,
-                                             ompi_datatype_t *source_datatype, void *result_addr, int result_count,
+static inline int ompi_osc_rdma_gacc_master (ompi_osc_rdma_sync_t *sync, const void *source_addr, size_t source_count,
+                                             ompi_datatype_t *source_datatype, void *result_addr, size_t result_count,
                                              ompi_datatype_t *result_datatype, ompi_osc_rdma_peer_t *peer, uint64_t target_address,
-                                             mca_btl_base_registration_handle_t *target_handle, int target_count,
+                                             mca_btl_base_registration_handle_t *target_handle, size_t target_count,
                                              ompi_datatype_t *target_datatype, ompi_op_t *op, ompi_osc_rdma_request_t *request)
 {
     ompi_osc_rdma_module_t *module = sync->module;
@@ -424,7 +546,6 @@ static inline int ompi_osc_rdma_gacc_master (ompi_osc_rdma_sync_t *sync, const v
     /* needed for opal_convertor_raw but not used */
     size_t source_size, target_size;
     ompi_osc_rdma_request_t *subreq;
-    size_t result_position;
     ptrdiff_t lb, extent;
     int ret, acc_len;
     bool done;
@@ -547,7 +668,6 @@ static inline int ompi_osc_rdma_gacc_master (ompi_osc_rdma_sync_t *sync, const v
     target_iov_index = 0;
     target_iov_count = 0;
     source_iov_index = 0;
-    result_position = 0;
     subreq = NULL;
 
     do {
@@ -556,7 +676,7 @@ static inline int ompi_osc_rdma_gacc_master (ompi_osc_rdma_sync_t *sync, const v
         target_iov_index = 0;
         done = opal_convertor_raw (&target_convertor, target_iovec, &target_iov_count, &target_size);
 
-        /* loop on the source segments (if any) until we have exhaused the decoded target data */
+        /* loop on the source segments (if any) until we have exhausted the decoded target data */
         while (target_iov_index != target_iov_count) {
             if (source_iov_count == source_iov_index) {
                 /* decode segments of the source data */
@@ -570,9 +690,9 @@ static inline int ompi_osc_rdma_gacc_master (ompi_osc_rdma_sync_t *sync, const v
 
             /* determine how much to put in this operation */
             if (source_count) {
-                acc_len = min(min(target_iovec[target_iov_index].iov_len, source_iovec[source_iov_index].iov_len), acc_limit);
+                acc_len = opal_min(opal_min(target_iovec[target_iov_index].iov_len, source_iovec[source_iov_index].iov_len), acc_limit);
             } else {
-                acc_len = min(target_iovec[target_iov_index].iov_len, acc_limit);
+                acc_len = opal_min(target_iovec[target_iov_index].iov_len, acc_limit);
             }
 
             if (0 != acc_len) {
@@ -610,8 +730,6 @@ static inline int ompi_osc_rdma_gacc_master (ompi_osc_rdma_sync_t *sync, const v
             target_iovec[target_iov_index].iov_base = (void *)((intptr_t) target_iovec[target_iov_index].iov_base + acc_len);
             target_iov_index += (0 == target_iovec[target_iov_index].iov_len);
 
-            result_position += acc_len;
-
             if (source_datatype) {
                 source_iov_index += (0 == source_iovec[source_iov_index].iov_len);
                 source_iovec[source_iov_index].iov_len -= acc_len;
@@ -647,19 +765,24 @@ static inline int ompi_osc_rdma_cas_atomic (ompi_osc_rdma_sync_t *sync, const vo
                                             bool lock_acquired)
 {
     ompi_osc_rdma_module_t *module = sync->module;
-    mca_btl_base_module_t *btl = ompi_osc_rdma_selected_btl (module, peer->data_btl_index);
-    int32_t atomic_flags = btl->btl_atomic_flags;
     const size_t size = datatype->super.size;
     int64_t compare, source;
     int flags, ret;
 
-    if (8 != size && !(4 == size && (MCA_BTL_ATOMIC_SUPPORTS_32BIT & atomic_flags))) {
+    if (8 != size && !(4 == size && (MCA_BTL_ATOMIC_SUPPORTS_32BIT & module->atomic_flags))) {
         return OMPI_ERR_NOT_SUPPORTED;
     }
 
-    compare = (8 == size) ? ((int64_t *) compare_addr)[0] : ((int32_t *) compare_addr)[0];
-    source = (8 == size) ? ((int64_t *) source_addr)[0] : ((int32_t *) source_addr)[0];
-    flags = (4 == size) ? MCA_BTL_ATOMIC_FLAG_32BIT : 0;
+    ret = osc_rdma_accelerator_mem_copy(&compare, compare_addr, size);
+    if (ret) {
+        return ret;
+    }
+    ret = osc_rdma_accelerator_mem_copy(&source, source_addr, size);
+    if (ret) {
+        return ret;
+    }
+
+    flags = ompi_osc_rdma_set_btl_flags(module, datatype, size);
 
     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "initiating compare-and-swap using %d-bit btl atomics. compare: 0x%"
                      PRIx64 ", origin: 0x%" PRIx64, (int) size * 8, *((int64_t *) compare_addr), *((int64_t *) source_addr));
@@ -700,11 +823,17 @@ static inline int cas_rdma (ompi_osc_rdma_sync_t *sync, const void *source_addr,
                             mca_btl_base_registration_handle_t *target_handle, bool lock_acquired)
 {
     ompi_osc_rdma_module_t *module = sync->module;
-    mca_btl_base_module_t *btl = ompi_osc_rdma_selected_btl (module, peer->data_btl_index);
     unsigned long len = datatype->super.size;
     mca_btl_base_registration_handle_t *local_handle = NULL;
     ompi_osc_rdma_frag_t *frag = NULL;
     volatile bool complete = false;
+    void *result_copy;
+    const void *compare_copy;
+    void *compare_to_free = NULL;
+    bool result_copied = false;
+    bool compare_copied = false;
+    int mem_compare;
+
     /* drop the const. this code will not attempt to change the value */
     char *ptr = (char *) source_addr;
     int ret;
@@ -719,32 +848,72 @@ static inline int cas_rdma (ompi_osc_rdma_sync_t *sync, const void *source_addr,
         return ret;
     }
 
-    if (0 != memcmp (result_addr, compare_addr, len)) {
+    /* Check if either addr is accelerator. Will have to perform some copies if so */
+    ret = osc_rdma_is_accel(result_addr);
+    if (0 < ret) {
+        result_copy = malloc(len);
+        ret = opal_accelerator.mem_copy(MCA_ACCELERATOR_NO_DEVICE_ID, MCA_ACCELERATOR_NO_DEVICE_ID,
+                                        result_copy, result_addr, len, MCA_ACCELERATOR_TRANSFER_DTOH);
+        result_copied = true;
+    } else if (0 == ret) {
+        result_copy = result_addr;
+    } else {
+        return ret;
+    }
+    ret = osc_rdma_is_accel(compare_addr);
+    if (0 < ret) {
+        compare_copy = compare_to_free = malloc(len);
+        ret = opal_accelerator.mem_copy(MCA_ACCELERATOR_NO_DEVICE_ID, MCA_ACCELERATOR_NO_DEVICE_ID,
+                                        compare_to_free, compare_addr, len, MCA_ACCELERATOR_TRANSFER_DTOH);
+        compare_copied = true;
+    } else if (0 == ret) {
+        compare_copy = compare_addr;
+    } else {
+        if (result_copied) {
+            free(result_copy);
+        }
+        return ret;
+    }
+
+    mem_compare = memcmp(result_copy, compare_copy, len);
+    if (compare_copied) {
+        free(compare_to_free);
+    }
+    if (result_copied) {
+        free(result_copy);
+    }
+    if (0 != mem_compare) {
         /* value does not match compare value, nothing more to do*/
         ompi_osc_rdma_peer_accumulate_cleanup (module, peer, lock_acquired);
         return OMPI_SUCCESS;
     }
 
-    if (btl->btl_register_mem && len > btl->btl_put_local_registration_threshold) {
-        do {
-            ret = ompi_osc_rdma_frag_alloc (module, len, &frag, &ptr);
-            if (OPAL_UNLIKELY(OMPI_SUCCESS == ret)) {
-                break;
-            }
-
-            ompi_osc_rdma_progress (module);
-        } while (1);
+    if (module->use_memory_registration) {
+        mca_btl_base_module_t *btl = ompi_osc_rdma_selected_btl (module, peer->data_btl_index);
+        if (len > btl->btl_put_local_registration_threshold) {
+            do {
+                ret = ompi_osc_rdma_frag_alloc(module, len, &frag, &ptr);
+                if (OPAL_UNLIKELY(OMPI_SUCCESS == ret)) {
+                    break;
+                }
 
-        memcpy (ptr, source_addr, len);
-        local_handle = frag->handle;
+                ompi_osc_rdma_progress (module);
+            } while (1);
+            ret = osc_rdma_accelerator_mem_copy(ptr, source_addr, len);
+            if (ret) {
+                return ret;
+            }
+            local_handle = frag->handle;
+        }
     }
 
     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "RDMA compare-and-swap initiating blocking btl put...");
 
     do {
-        ret = btl->btl_put (btl, peer->data_endpoint, ptr, target_address,
-                            local_handle, target_handle, len, 0, MCA_BTL_NO_ORDER,
-                            ompi_osc_rdma_cas_put_complete, (void *) &complete, NULL);
+        ret = ompi_osc_rdma_btl_put(module, peer->data_btl_index, peer->data_endpoint,
+                                    ptr, target_address, local_handle, target_handle,
+                                    len, 0, MCA_BTL_NO_ORDER,
+                                    ompi_osc_rdma_cas_put_complete, (void *) &complete, NULL);
         if (OPAL_SUCCESS == ret || (OPAL_ERR_OUT_OF_RESOURCE != ret && OPAL_ERR_TEMP_OUT_OF_RESOURCE != ret)) {
             break;
         }
@@ -830,10 +999,12 @@ int ompi_osc_rdma_compare_and_swap (const void *origin_addr, const void *compare
          * user has indicated that they will only use the same op (or same op and no op) for
          * operations on overlapping memory ranges. that indicates it is safe to go ahead and
          * use network atomic operations. */
-        ret = ompi_osc_rdma_cas_atomic (sync, origin_addr, compare_addr, result_addr, dt,
-                                        peer, target_address, target_handle, lock_acquired);
-        if (OMPI_SUCCESS == ret) {
-            return OMPI_SUCCESS;
+        if(ompi_osc_base_is_atomic_size_supported(target_address, dt->super.size)) {
+            ret = ompi_osc_rdma_cas_atomic (sync, origin_addr, compare_addr, result_addr, dt,
+                                            peer, target_address, target_handle, lock_acquired);
+            if (OMPI_SUCCESS == ret) {
+                return OMPI_SUCCESS;
+            }
         }
     }
 
@@ -863,10 +1034,10 @@ int ompi_osc_rdma_compare_and_swap (const void *origin_addr, const void *compare
 
 
 static inline
-int ompi_osc_rdma_rget_accumulate_internal (ompi_win_t *win, const void *origin_addr, int origin_count,
-                                            ompi_datatype_t *origin_datatype, void *result_addr, int result_count,
+int ompi_osc_rdma_rget_accumulate_internal (ompi_win_t *win, const void *origin_addr, size_t origin_count,
+                                            ompi_datatype_t *origin_datatype, void *result_addr, size_t result_count,
                                             ompi_datatype_t *result_datatype,  int target_rank, MPI_Aint target_disp,
-                                            int target_count, ompi_datatype_t *target_datatype, ompi_op_t *op,
+                                            size_t target_count, ompi_datatype_t *target_datatype, ompi_op_t *op,
                                             ompi_request_t **request_out)
 {
     ompi_osc_rdma_module_t *module = GET_MODULE(win);
@@ -948,12 +1119,12 @@ int ompi_osc_rdma_rget_accumulate_internal (ompi_win_t *win, const void *origin_
     return ret;
 }
 
-int ompi_osc_rdma_get_accumulate (const void *origin_addr, int origin_count, ompi_datatype_t *origin_datatype,
-                                  void *result_addr, int result_count, ompi_datatype_t *result_datatype,
-                                  int target_rank, MPI_Aint target_disp, int target_count, ompi_datatype_t *target_datatype,
+int ompi_osc_rdma_get_accumulate (const void *origin_addr, size_t origin_count, ompi_datatype_t *origin_datatype,
+                                  void *result_addr, size_t result_count, ompi_datatype_t *result_datatype,
+                                  int target_rank, MPI_Aint target_disp, size_t target_count, ompi_datatype_t *target_datatype,
                                   ompi_op_t *op, ompi_win_t *win)
 {
-    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "get_acc: 0x%lx, %d, %s, 0x%lx, %d, %s, %d, 0x%lx, %d, %s, %s, %s",
+    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "get_acc: 0x%lx, %zu, %s, 0x%lx, %zu, %s, %d, 0x%lx, %zu, %s, %s, %s",
                      (unsigned long) origin_addr, origin_count, origin_datatype->name,
                      (unsigned long) result_addr, result_count, result_datatype->name, target_rank,
                      (unsigned long) target_disp, target_count, target_datatype->name, op->o_name,
@@ -966,12 +1137,12 @@ int ompi_osc_rdma_get_accumulate (const void *origin_addr, int origin_count, omp
 }
 
 
-int ompi_osc_rdma_rget_accumulate (const void *origin_addr, int origin_count, ompi_datatype_t *origin_datatype,
-                                   void *result_addr, int result_count, ompi_datatype_t *result_datatype,
-                                   int target_rank, MPI_Aint target_disp, int target_count, ompi_datatype_t *target_datatype,
+int ompi_osc_rdma_rget_accumulate (const void *origin_addr, size_t origin_count, ompi_datatype_t *origin_datatype,
+                                   void *result_addr, size_t result_count, ompi_datatype_t *result_datatype,
+                                   int target_rank, MPI_Aint target_disp, size_t target_count, ompi_datatype_t *target_datatype,
                                    ompi_op_t *op, ompi_win_t *win, ompi_request_t **request)
 {
-    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "rget_acc: 0x%lx, %d, %s, 0x%lx, %d, %s, %d, 0x%lx, %d, %s, %s, %s",
+    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "rget_acc: 0x%lx, %zu, %s, 0x%lx, %zu, %s, %d, 0x%lx, %zu, %s, %s, %s",
                      (unsigned long) origin_addr, origin_count, origin_datatype->name,
                      (unsigned long) result_addr, result_count, result_datatype->name, target_rank,
                      (unsigned long) target_disp, target_count, target_datatype->name, op->o_name,
@@ -982,11 +1153,11 @@ int ompi_osc_rdma_rget_accumulate (const void *origin_addr, int origin_count, om
                                                    target_count, target_datatype, op, request);
 }
 
-int ompi_osc_rdma_raccumulate (const void *origin_addr, int origin_count, ompi_datatype_t *origin_datatype, int target_rank,
-                               ptrdiff_t target_disp, int target_count, ompi_datatype_t *target_datatype, ompi_op_t *op,
+int ompi_osc_rdma_raccumulate (const void *origin_addr, size_t origin_count, ompi_datatype_t *origin_datatype, int target_rank,
+                               ptrdiff_t target_disp, size_t target_count, ompi_datatype_t *target_datatype, ompi_op_t *op,
                                ompi_win_t *win, ompi_request_t **request)
 {
-    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "racc: 0x%lx, %d, %s, %d, 0x%lx, %d, %s, %s, %s",
+    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "racc: 0x%lx, %zu, %s, %d, 0x%lx, %zu, %s, %s, %s",
                      (unsigned long) origin_addr, origin_count, origin_datatype->name, target_rank,
                      (unsigned long) target_disp, target_count, target_datatype->name, op->o_name, win->w_name);
 
@@ -995,11 +1166,11 @@ int ompi_osc_rdma_raccumulate (const void *origin_addr, int origin_count, ompi_d
                                                    op, request);
 }
 
-int ompi_osc_rdma_accumulate (const void *origin_addr, int origin_count, ompi_datatype_t *origin_datatype, int target_rank,
-                              ptrdiff_t target_disp, int target_count, ompi_datatype_t *target_datatype, ompi_op_t *op,
+int ompi_osc_rdma_accumulate (const void *origin_addr, size_t origin_count, ompi_datatype_t *origin_datatype, int target_rank,
+                              ptrdiff_t target_disp, size_t target_count, ompi_datatype_t *target_datatype, ompi_op_t *op,
                               ompi_win_t *win)
 {
-    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "acc: 0x%lx, %d, %s, %d, 0x%lx, %d, %s, %s, %s",
+    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "acc: 0x%lx, %zu, %s, %d, 0x%lx, %zu, %s, %s, %s",
                      (unsigned long) origin_addr, origin_count, origin_datatype->name, target_rank,
                      (unsigned long) target_disp, target_count, target_datatype->name, op->o_name, win->w_name);
 
diff --git a/ompi/mca/osc/rdma/osc_rdma_accumulate.h b/ompi/mca/osc/rdma/osc_rdma_accumulate.h
index 74f41abf6ef..8060f1e5590 100644
--- a/ompi/mca/osc/rdma/osc_rdma_accumulate.h
+++ b/ompi/mca/osc/rdma/osc_rdma_accumulate.h
@@ -11,7 +11,7 @@
  * $HEADER$
  */
 
-#if !defined(OSC_RDMA_ACCUMULATE_H)
+#ifndef OSC_RDMA_ACCUMULATE_H
 #define OSC_RDMA_ACCUMULATE_H
 
 #include "osc_rdma.h"
@@ -20,25 +20,25 @@ int ompi_osc_rdma_compare_and_swap (const void *origin_addr, const void *compare
                                     ompi_datatype_t *dt, int target_rank, ptrdiff_t target_disp,
                                     ompi_win_t *win);
 
-int ompi_osc_rdma_accumulate (const void *origin_addr, int origin_count, ompi_datatype_t *origin_datatype, int target_rank,
-                              ptrdiff_t target_disp, int target_count, ompi_datatype_t *target_datatype, ompi_op_t *op,
+int ompi_osc_rdma_accumulate (const void *origin_addr, size_t origin_count, ompi_datatype_t *origin_datatype, int target_rank,
+                              ptrdiff_t target_disp, size_t target_count, ompi_datatype_t *target_datatype, ompi_op_t *op,
                               ompi_win_t *win);
 
 int ompi_osc_rdma_fetch_and_op (const void *origin_addr, void *result_addr, ompi_datatype_t *dt, int target_rank,
                                 ptrdiff_t target_disp, ompi_op_t *op, ompi_win_t *win);
 
-int ompi_osc_rdma_get_accumulate (const void *origin_addr, int origin_count, ompi_datatype_t *origin_datatype,
-                                  void *result_addr, int result_count, ompi_datatype_t *result_datatype,
-                                  int target_rank, MPI_Aint target_disp, int target_count, ompi_datatype_t *target_datatype,
+int ompi_osc_rdma_get_accumulate (const void *origin_addr, size_t origin_count, ompi_datatype_t *origin_datatype,
+                                  void *result_addr, size_t result_count, ompi_datatype_t *result_datatype,
+                                  int target_rank, MPI_Aint target_disp, size_t target_count, ompi_datatype_t *target_datatype,
                                   ompi_op_t *op, ompi_win_t *win);
 
-int ompi_osc_rdma_raccumulate (const void *origin_addr, int origin_count, ompi_datatype_t *origin_datatype, int target_rank,
-                               ptrdiff_t target_disp, int target_count, ompi_datatype_t *target_datatype, ompi_op_t *op,
+int ompi_osc_rdma_raccumulate (const void *origin_addr, size_t origin_count, ompi_datatype_t *origin_datatype, int target_rank,
+                               ptrdiff_t target_disp, size_t target_count, ompi_datatype_t *target_datatype, ompi_op_t *op,
                                ompi_win_t *win, ompi_request_t **request);
 
-int ompi_osc_rdma_rget_accumulate (const void *origin_addr, int origin_count, ompi_datatype_t *origin_datatype,
-                                   void *result_addr, int result_count, ompi_datatype_t *result_datatype,
-                                   int target_rank, MPI_Aint target_disp, int target_count, ompi_datatype_t *target_datatype,
+int ompi_osc_rdma_rget_accumulate (const void *origin_addr, size_t origin_count, ompi_datatype_t *origin_datatype,
+                                   void *result_addr, size_t result_count, ompi_datatype_t *result_datatype,
+                                   int target_rank, MPI_Aint target_disp, size_t target_count, ompi_datatype_t *target_datatype,
                                    ompi_op_t *op, ompi_win_t *win, ompi_request_t **request);
 
 
diff --git a/ompi/mca/osc/rdma/osc_rdma_active_target.c b/ompi/mca/osc/rdma/osc_rdma_active_target.c
index f6cf6bb2820..9858151fa67 100644
--- a/ompi/mca/osc/rdma/osc_rdma_active_target.c
+++ b/ompi/mca/osc/rdma/osc_rdma_active_target.c
@@ -17,6 +17,7 @@
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2017-2018 Intel, Inc. All rights reserved.
+ * Copyright (c) 2021      Google, LLC. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -76,33 +77,6 @@ OBJ_CLASS_INSTANCE(ompi_osc_rdma_pending_op_t, opal_list_item_t,
                    ompi_osc_rdma_pending_op_construct,
                    ompi_osc_rdma_pending_op_destruct);
 
-/**
- * Dummy completion function for atomic operations
- */
-void ompi_osc_rdma_atomic_complete (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
-                                    void *local_address, mca_btl_base_registration_handle_t *local_handle,
-                                    void *context, void *data, int status)
-{
-    ompi_osc_rdma_pending_op_t *pending_op = (ompi_osc_rdma_pending_op_t *) context;
-
-    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_INFO, "pending atomic %p complete with status %d", (void*)pending_op, status);
-
-    if (pending_op->op_result) {
-        memmove (pending_op->op_result, pending_op->op_buffer, pending_op->op_size);
-    }
-
-    if (NULL != pending_op->cbfunc) {
-        pending_op->cbfunc (pending_op->cbdata, pending_op->cbcontext, status);
-    }
-
-    if (NULL != pending_op->op_frag) {
-        ompi_osc_rdma_frag_complete (pending_op->op_frag);
-        pending_op->op_frag = NULL;
-    }
-
-    pending_op->op_complete = true;
-    OBJ_RELEASE(pending_op);
-}
 
 /**
  * compare_ranks:
@@ -297,14 +271,14 @@ static int ompi_osc_rdma_post_peer (ompi_osc_rdma_module_t *module, ompi_osc_rdm
     return OMPI_SUCCESS;
 }
 
-int ompi_osc_rdma_post_atomic (ompi_group_t *group, int assert, ompi_win_t *win)
+int ompi_osc_rdma_post_atomic (ompi_group_t *group, int mpi_assert, ompi_win_t *win)
 {
     ompi_osc_rdma_module_t *module = GET_MODULE(win);
     ompi_osc_rdma_peer_t **peers;
     ompi_osc_rdma_state_t *state = module->state;
     int ret = OMPI_SUCCESS;
 
-    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "post: %p, %d, %s", (void*) group, assert, win->w_name);
+    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "post: %p, %d, %s", (void*) group, mpi_assert, win->w_name);
 
     /* check if we are already in a post epoch */
     if (module->pw_group) {
@@ -329,7 +303,7 @@ int ompi_osc_rdma_post_atomic (ompi_group_t *group, int assert, ompi_win_t *win)
     state->num_complete_msgs = 0;
     OPAL_THREAD_UNLOCK(&module->lock);
 
-    if ((assert & MPI_MODE_NOCHECK) || 0 == ompi_group_size (group)) {
+    if ((mpi_assert & MPI_MODE_NOCHECK) || 0 == ompi_group_size (group)) {
         return OMPI_SUCCESS;
     }
 
@@ -356,7 +330,7 @@ int ompi_osc_rdma_post_atomic (ompi_group_t *group, int assert, ompi_win_t *win)
     return ret;
 }
 
-int ompi_osc_rdma_start_atomic (ompi_group_t *group, int assert, ompi_win_t *win)
+int ompi_osc_rdma_start_atomic (ompi_group_t *group, int mpi_assert, ompi_win_t *win)
 {
     ompi_osc_rdma_module_t *module = GET_MODULE(win);
     ompi_osc_rdma_pending_post_t *pending_post, *next;
@@ -364,7 +338,7 @@ int ompi_osc_rdma_start_atomic (ompi_group_t *group, int assert, ompi_win_t *win
     ompi_osc_rdma_sync_t *sync = &module->all_sync;
     int group_size = ompi_group_size (group);
 
-    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "start: %p, %d, %s", (void*) group, assert,
+    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "start: %p, %d, %s", (void*) group, mpi_assert,
                      win->w_name);
 
     OPAL_THREAD_LOCK(&module->lock);
@@ -379,21 +353,22 @@ int ompi_osc_rdma_start_atomic (ompi_group_t *group, int assert, ompi_win_t *win
     sync->num_peers = ompi_group_size (group);
     sync->sync.pscw.group = group;
 
-    /* haven't processed any post messaes yet */
+    /* haven't processed any post messages yet */
     state->num_post_msgs = 0;
 
     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "start group size %d", sync->num_peers);
 
+    sync->type = OMPI_OSC_RDMA_SYNC_TYPE_PSCW;
+
     if (0 == ompi_group_size (group)) {
         /* nothing more to do. this is an empty start epoch */
+        sync->peer_list.peers = NULL;
         OPAL_THREAD_UNLOCK(&module->lock);
         return OMPI_SUCCESS;
     }
 
     opal_atomic_wmb ();
 
-    sync->type = OMPI_OSC_RDMA_SYNC_TYPE_PSCW;
-
     /* prevent us from entering a passive-target, fence, or another pscw access epoch until
      * the matching complete is called */
     sync->epoch_active = true;
@@ -408,7 +383,7 @@ int ompi_osc_rdma_start_atomic (ompi_group_t *group, int assert, ompi_win_t *win
     /* save the group */
     OBJ_RETAIN(group);
 
-    if (!(assert & MPI_MODE_NOCHECK)) {
+    if (!(mpi_assert & MPI_MODE_NOCHECK)) {
         /* look through list of pending posts */
         OPAL_LIST_FOREACH_SAFE(pending_post, next, &module->pending_posts, ompi_osc_rdma_pending_post_t) {
             for (int i = 0 ; i < group_size ; ++i) {
@@ -459,23 +434,25 @@ int ompi_osc_rdma_complete_atomic (ompi_win_t *win)
         return OMPI_ERR_RMA_SYNC;
     }
 
+    if (0 == sync->num_peers) {
+        OPAL_THREAD_UNLOCK(&module->lock);
+        return OMPI_SUCCESS;
+    }
+
     /* phase 1 cleanup sync object */
     group = sync->sync.pscw.group;
     group_size = sync->num_peers;
     sync->type = OMPI_OSC_RDMA_SYNC_TYPE_NONE;
     sync->epoch_active = false;
 
-    /* phase 2 cleanup group */
-    OBJ_RELEASE(group);
-
     peers = sync->peer_list.peers;
     if (NULL == peers) {
-        /* empty peer list */
         OPAL_THREAD_UNLOCK(&(module->lock));
-        OBJ_RELEASE(group);
         return OMPI_SUCCESS;
     }
 
+    OBJ_RELEASE(group);
+
     sync->peer_list.peers = NULL;
 
     OPAL_THREAD_UNLOCK(&(module->lock));
@@ -586,12 +563,12 @@ int ompi_osc_rdma_test_atomic (ompi_win_t *win, int *flag)
     return OMPI_SUCCESS;
 }
 
-int ompi_osc_rdma_fence_atomic (int assert, ompi_win_t *win)
+int ompi_osc_rdma_fence_atomic (int mpi_assert, ompi_win_t *win)
 {
     ompi_osc_rdma_module_t *module = GET_MODULE(win);
     int ret = OMPI_SUCCESS;
 
-    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "fence: %d, %s", assert, win->w_name);
+    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "fence: %d, %s", mpi_assert, win->w_name);
 
     /* can't enter an active target epoch while a lock is active */
     if (ompi_osc_rdma_in_passive_epoch (module) || module->pw_group) {
@@ -600,11 +577,11 @@ int ompi_osc_rdma_fence_atomic (int assert, ompi_win_t *win)
     }
 
     /* NTH: locking here isn't really needed per-se but it may make user synchronization errors more
-     * predicable. if the user is using RMA correctly then there will be no contention on this lock. */
+     * predictable. if the user is using RMA correctly then there will be no contention on this lock. */
     OPAL_THREAD_LOCK(&module->lock);
 
     /* active sends are now active (we will close the epoch if NOSUCCEED is specified) */
-    if (0 == (assert & MPI_MODE_NOSUCCEED)) {
+    if (0 == (mpi_assert & MPI_MODE_NOSUCCEED)) {
         module->all_sync.type = OMPI_OSC_RDMA_SYNC_TYPE_FENCE;
         module->all_sync.num_peers = ompi_comm_size (module->comm);
         /* NTH: should add a fast access array for peers here later. for now just use the
@@ -625,7 +602,7 @@ int ompi_osc_rdma_fence_atomic (int assert, ompi_win_t *win)
     /* ensure all writes to my memory are complete (both local stores, and RMA operations) */
     ret = module->comm->c_coll->coll_barrier(module->comm, module->comm->c_coll->coll_barrier_module);
 
-    if (assert & MPI_MODE_NOSUCCEED) {
+    if (mpi_assert & MPI_MODE_NOSUCCEED) {
         /* as specified in MPI-3 p 438 3-5 the fence can end an epoch. it isn't explicitly
          * stated that MPI_MODE_NOSUCCEED ends the epoch but it is a safe assumption. */
         module->all_sync.type = OMPI_OSC_RDMA_SYNC_TYPE_NONE;
diff --git a/ompi/mca/osc/rdma/osc_rdma_active_target.h b/ompi/mca/osc/rdma/osc_rdma_active_target.h
index 605d006c029..9da1945d36d 100644
--- a/ompi/mca/osc/rdma/osc_rdma_active_target.h
+++ b/ompi/mca/osc/rdma/osc_rdma_active_target.h
@@ -12,6 +12,7 @@
  *                         reserved.
  * Copyright (c) 2010      IBM Corporation.  All rights reserved.
  * Copyright (c) 2012-2013 Sandia National Laboratories.  All rights reserved.
+ * Copyright (c) 2021      Google, LLC. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -19,21 +20,21 @@
  * $HEADER$
  */
 
-#if !defined(OSC_RDMA_ACTIVE_TARGET_H)
+#ifndef OSC_RDMA_ACTIVE_TARGET_H
 #define OSC_RDMA_ACTIVE_TARGET_H
 
 #include "osc_rdma.h"
 #include "osc_rdma_sync.h"
 #include "osc_rdma_lock.h"
 
-int ompi_osc_rdma_fence_atomic (int assert, struct ompi_win_t *win);
+int ompi_osc_rdma_fence_atomic (int mpi_assert, struct ompi_win_t *win);
 
 int ompi_osc_rdma_start_atomic (struct ompi_group_t *group,
-                                int assert, struct ompi_win_t *win);
+                                int mpi_assert, struct ompi_win_t *win);
 int ompi_osc_rdma_complete_atomic (struct ompi_win_t *win);
 
 int ompi_osc_rdma_post_atomic (struct ompi_group_t *group,
-                               int assert, struct ompi_win_t *win);
+                               int mpi_assert, struct ompi_win_t *win);
 
 int ompi_osc_rdma_wait_atomic (struct ompi_win_t *win);
 
diff --git a/ompi/mca/osc/rdma/osc_rdma_btl_comm.c b/ompi/mca/osc/rdma/osc_rdma_btl_comm.c
new file mode 100644
index 00000000000..f02fc21d0fd
--- /dev/null
+++ b/ompi/mca/osc/rdma/osc_rdma_btl_comm.c
@@ -0,0 +1,61 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007-2018 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2010      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2012-2013 Sandia National Laboratories.  All rights reserved.
+ * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2017      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2017-2018 Intel, Inc. All rights reserved.
+ * Copyright (c) 2021      Google, LLC. All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "osc_rdma.h"
+#include "osc_rdma_frag.h"
+#include "osc_rdma_btl_comm.h"
+
+#include "opal/mca/btl/base/base.h"
+
+void ompi_osc_rdma_atomic_complete (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+                                    void *local_address, mca_btl_base_registration_handle_t *local_handle,
+                                    void *context, void *data, int status)
+{
+    ompi_osc_rdma_pending_op_t *pending_op = (ompi_osc_rdma_pending_op_t *) context;
+
+    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_INFO, "pending atomic %p complete with status %d", (void*)pending_op, status);
+
+    if (pending_op->op_result) {
+        osc_rdma_accelerator_mem_move (pending_op->op_result, pending_op->op_buffer, pending_op->op_size);
+    }
+
+    if (NULL != pending_op->cbfunc) {
+        pending_op->cbfunc (pending_op->cbdata, pending_op->cbcontext, status);
+    }
+
+    if (NULL != pending_op->op_frag) {
+        ompi_osc_rdma_frag_complete (pending_op->op_frag);
+        pending_op->op_frag = NULL;
+    }
+
+    pending_op->op_complete = true;
+    OBJ_RELEASE(pending_op);
+}
diff --git a/ompi/mca/osc/rdma/osc_rdma_btl_comm.h b/ompi/mca/osc/rdma/osc_rdma_btl_comm.h
new file mode 100644
index 00000000000..718036d0a76
--- /dev/null
+++ b/ompi/mca/osc/rdma/osc_rdma_btl_comm.h
@@ -0,0 +1,338 @@
+/*
+ * Copyright (c) 2014-2018 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2019      Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2021      Google, LLC. All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef OSC_RDMA_BTL_COMM_H
+#define OSC_RDMA_BTL_COMM_H
+
+#include "osc_rdma_frag.h"
+
+#include "opal/mca/btl/btl.h"
+
+
+void ompi_osc_rdma_atomic_complete(mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+                                    void *local_address, mca_btl_base_registration_handle_t *local_handle,
+                                    void *context, void *data, int status);
+
+
+static inline int
+ompi_osc_rdma_btl_put(ompi_osc_rdma_module_t *module, uint8_t btl_index,
+		      struct mca_btl_base_endpoint_t *endpoint,
+		      void *local_address, uint64_t remote_address,
+		      struct mca_btl_base_registration_handle_t *local_handle,
+		      struct mca_btl_base_registration_handle_t *remote_handle,
+		      size_t size, int flags, int order,
+		      mca_btl_base_rdma_completion_fn_t cbfunc,
+		      void *cbcontext, void *cbdata)
+{
+    if (module->use_accelerated_btl) {
+        mca_btl_base_module_t *btl = ompi_osc_rdma_selected_btl(module, btl_index);
+        return btl->btl_put(btl, endpoint, local_address, remote_address,
+                            local_handle, remote_handle, size, flags, order,
+                            cbfunc, cbcontext, cbdata);
+    } else {
+        mca_btl_base_am_rdma_module_t *am_rdma = ompi_osc_rdma_selected_am_rdma(module, btl_index);
+        return am_rdma->am_btl_put(am_rdma, endpoint, local_address, remote_address,
+                                   local_handle, remote_handle, size, flags, order,
+                                   cbfunc, cbcontext, cbdata);
+    }
+}
+
+
+static inline int
+ompi_osc_rdma_btl_get(ompi_osc_rdma_module_t *module, uint8_t btl_index,
+		      struct mca_btl_base_endpoint_t *endpoint,
+                      void *local_address, uint64_t remote_address,
+		      struct mca_btl_base_registration_handle_t *local_handle,
+		      struct mca_btl_base_registration_handle_t *remote_handle,
+		      size_t size, int flags, int order,
+		      mca_btl_base_rdma_completion_fn_t cbfunc,
+		      void *cbcontext, void *cbdata)
+{
+
+    if (module->use_accelerated_btl) {
+        mca_btl_base_module_t *btl = ompi_osc_rdma_selected_btl(module, btl_index);
+        return btl->btl_get(btl, endpoint, local_address, remote_address,
+                            local_handle, remote_handle, size, flags, order,
+                            cbfunc, cbcontext, cbdata);
+    } else {
+        mca_btl_base_am_rdma_module_t *am_rdma = ompi_osc_rdma_selected_am_rdma(module, btl_index);
+        return am_rdma->am_btl_get(am_rdma, endpoint, local_address, remote_address,
+                                   local_handle, remote_handle, size, flags, order,
+                                   cbfunc, cbcontext, cbdata);
+    }
+}
+
+
+static inline int
+ompi_osc_rdma_btl_atomic_op(ompi_osc_rdma_module_t *module, uint8_t btl_index,
+                            struct mca_btl_base_endpoint_t *endpoint,
+                            uint64_t remote_address, struct mca_btl_base_registration_handle_t *remote_handle,
+                            mca_btl_base_atomic_op_t op, uint64_t operand, int flags, int order,
+                            mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
+{
+    mca_btl_base_module_t *btl = ompi_osc_rdma_selected_btl(module, btl_index);
+
+    /* the AM BTL interface does not currently support op calls */
+    assert(module->use_accelerated_btl);
+
+    return btl->btl_atomic_op(btl, endpoint, remote_address, remote_handle,
+                              op, operand, flags, order,
+                              cbfunc, cbcontext, cbdata);
+}
+
+
+static inline int
+ompi_osc_rdma_btl_atomic_fop(ompi_osc_rdma_module_t *module, uint8_t btl_index,
+                             struct mca_btl_base_endpoint_t *endpoint,
+                             void *local_address, uint64_t remote_address,
+                             struct mca_btl_base_registration_handle_t *local_handle,
+                             struct mca_btl_base_registration_handle_t *remote_handle,
+                             mca_btl_base_atomic_op_t op, uint64_t operand, int flags, int order,
+                             mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
+
+{
+    if (module->use_accelerated_btl) {
+        mca_btl_base_module_t *btl = ompi_osc_rdma_selected_btl(module, btl_index);
+        return btl->btl_atomic_fop(btl, endpoint, local_address, remote_address,
+                                   local_handle, remote_handle,
+                                   op, operand, flags, order,
+                                   cbfunc, cbcontext, cbdata);
+    } else {
+        mca_btl_base_am_rdma_module_t *am_rdma = ompi_osc_rdma_selected_am_rdma(module, btl_index);
+        return am_rdma->am_btl_atomic_fop(am_rdma, endpoint, local_address, remote_address,
+                                          local_handle, remote_handle,
+                                          op, operand, flags, order,
+                                          cbfunc, cbcontext, cbdata);
+    }
+}
+
+
+static inline int
+ompi_osc_rdma_btl_atomic_cswap(ompi_osc_rdma_module_t *module, uint8_t btl_index,
+                               struct mca_btl_base_endpoint_t *endpoint,
+                               void *local_address, uint64_t remote_address,
+                               struct mca_btl_base_registration_handle_t *local_handle,
+                               struct mca_btl_base_registration_handle_t *remote_handle,
+                               uint64_t compare, uint64_t value, int flags, int order,
+                               mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
+{
+    if (module->use_accelerated_btl) {
+        mca_btl_base_module_t *btl = ompi_osc_rdma_selected_btl(module, btl_index);
+        return btl->btl_atomic_cswap(btl, endpoint, local_address, remote_address,
+                                     local_handle, remote_handle,
+                                     compare, value, flags, order,
+                                     cbfunc, cbcontext, cbdata);
+    } else {
+        mca_btl_base_am_rdma_module_t *am_rdma = ompi_osc_rdma_selected_am_rdma(module, btl_index);
+        return am_rdma->am_btl_atomic_cswap(am_rdma, endpoint, local_address, remote_address,
+                                            local_handle, remote_handle,
+                                            compare, value, flags, order,
+                                            cbfunc, cbcontext, cbdata);
+    }
+}
+
+
+static inline int
+ompi_osc_rdma_btl_fop(ompi_osc_rdma_module_t *module, uint8_t btl_index,
+		      struct mca_btl_base_endpoint_t *endpoint, uint64_t address,
+		      mca_btl_base_registration_handle_t *address_handle, int op,
+		      int64_t operand, int flags, int64_t *result, const bool wait_for_completion,
+		      ompi_osc_rdma_pending_op_cb_fn_t cbfunc, void *cbdata, void *cbcontext)
+{
+    ompi_osc_rdma_pending_op_t *pending_op;
+    mca_btl_base_module_t *selected_btl = ompi_osc_rdma_selected_btl (module, btl_index);
+    int ret = OPAL_ERROR;
+
+    pending_op = OBJ_NEW(ompi_osc_rdma_pending_op_t);
+    assert (NULL != pending_op);
+
+    if (!wait_for_completion) {
+        /* NTH: need to keep track of pending ops to avoid a potential teardown problem */
+        pending_op->module = module;
+        (void) opal_atomic_fetch_add_32 (&module->pending_ops, 1);
+    }
+
+    pending_op->op_result = (void *) result;
+    pending_op->op_size = (MCA_BTL_ATOMIC_FLAG_32BIT & flags) ? 4 : 8;
+    OBJ_RETAIN(pending_op);
+    if (cbfunc) {
+        pending_op->cbfunc = cbfunc;
+        pending_op->cbdata = cbdata;
+        pending_op->cbcontext = cbcontext;
+    }
+
+    /* spin until the btl has accepted the operation */
+    do {
+        if (NULL == pending_op->op_frag) {
+            ret = ompi_osc_rdma_frag_alloc (module, 8, &pending_op->op_frag, (char **) &pending_op->op_buffer);
+        }
+
+        if (NULL != pending_op->op_frag) {
+            ret = ompi_osc_rdma_btl_atomic_fop(module, btl_index, endpoint, pending_op->op_buffer,
+                                               (intptr_t) address, pending_op->op_frag->handle, address_handle,
+                                               op, operand, flags, MCA_BTL_NO_ORDER, ompi_osc_rdma_atomic_complete,
+                                               (void *) pending_op, NULL);
+        }
+
+        if (OPAL_LIKELY(!ompi_osc_rdma_oor(ret))) {
+            break;
+        }
+        ompi_osc_rdma_progress (module);
+    } while (1);
+
+    if (OPAL_SUCCESS != ret) {
+        if (OPAL_LIKELY(1 == ret)) {
+            *result = ((int64_t *) pending_op->op_buffer)[0];
+            ret = OMPI_SUCCESS;
+            ompi_osc_rdma_atomic_complete (selected_btl, endpoint, pending_op->op_buffer,
+                                           pending_op->op_frag->handle, (void *) pending_op, NULL, OPAL_SUCCESS);
+        } else {
+            /* need to release here because ompi_osc_rdma_atomic_complete was not called */
+            OBJ_RELEASE(pending_op);
+        }
+    } else if (wait_for_completion) {
+        while (!pending_op->op_complete) {
+            ompi_osc_rdma_progress (module);
+        }
+    }
+
+    OBJ_RELEASE(pending_op);
+
+    return ret;
+}
+
+
+static inline int
+ompi_osc_rdma_btl_op(ompi_osc_rdma_module_t *module, uint8_t btl_index,
+		     struct mca_btl_base_endpoint_t *endpoint, uint64_t address,
+		     mca_btl_base_registration_handle_t *address_handle,
+		     int op, int64_t operand, int flags, const bool wait_for_completion,
+		     ompi_osc_rdma_pending_op_cb_fn_t cbfunc, void *cbdata, void *cbcontext)
+{
+    ompi_osc_rdma_pending_op_t *pending_op;
+    mca_btl_base_module_t *selected_btl = ompi_osc_rdma_selected_btl (module, btl_index);
+    int ret;
+
+    /* if using the AM RDMA interface with alternate BTLs or if the
+       accelerated BTL does not support atomic ops, emulate the atomic
+       op over a fetch and atomic op */
+    if (!module->use_accelerated_btl || !(selected_btl->btl_flags & MCA_BTL_FLAGS_ATOMIC_OPS)) {
+        return ompi_osc_rdma_btl_fop (module, btl_index, endpoint, address, address_handle, op, operand, flags,
+                                      NULL, wait_for_completion, cbfunc, cbdata, cbcontext);
+    }
+
+    pending_op = OBJ_NEW(ompi_osc_rdma_pending_op_t);
+    assert (NULL != pending_op);
+    OBJ_RETAIN(pending_op);
+    if (cbfunc) {
+        pending_op->cbfunc = cbfunc;
+        pending_op->cbdata = cbdata;
+        pending_op->cbcontext = cbcontext;
+    }
+
+    if (!wait_for_completion) {
+        /* NTH: need to keep track of pending ops to avoid a potential teardown problem */
+        pending_op->module = module;
+        (void) opal_atomic_fetch_add_32 (&module->pending_ops, 1);
+    }
+
+    /* spin until the btl has accepted the operation */
+    do {
+        ret = ompi_osc_rdma_btl_atomic_op(module, btl_index, endpoint, (intptr_t) address, address_handle,
+                                          op, operand, flags, MCA_BTL_NO_ORDER, ompi_osc_rdma_atomic_complete,
+                                          (void *) pending_op, NULL);
+
+        if (OPAL_LIKELY(!ompi_osc_rdma_oor(ret))) {
+            break;
+        }
+        ompi_osc_rdma_progress (module);
+    } while (1);
+
+    if (OPAL_SUCCESS != ret) {
+        /* need to release here because ompi_osc_rdma_atomic_complete was not called */
+        OBJ_RELEASE(pending_op);
+        if (OPAL_LIKELY(1 == ret)) {
+            if (cbfunc) {
+                cbfunc (cbdata, cbcontext, OMPI_SUCCESS);
+            }
+            ret = OMPI_SUCCESS;
+        }
+    } else if (wait_for_completion) {
+        while (!pending_op->op_complete) {
+            ompi_osc_rdma_progress (module);
+        }
+    }
+
+    OBJ_RELEASE(pending_op);
+
+    return ret;
+}
+
+
+static inline int
+ompi_osc_rdma_btl_cswap(ompi_osc_rdma_module_t *module, uint8_t btl_index,
+			struct mca_btl_base_endpoint_t *endpoint, uint64_t address,
+			mca_btl_base_registration_handle_t *address_handle,
+			int64_t compare, int64_t value, int flags, int64_t *result)
+{
+    ompi_osc_rdma_pending_op_t *pending_op;
+    int ret;
+
+    pending_op = OBJ_NEW(ompi_osc_rdma_pending_op_t);
+    assert (NULL != pending_op);
+
+    OBJ_RETAIN(pending_op);
+
+    pending_op->op_result = (void *) result;
+    pending_op->op_size = (MCA_BTL_ATOMIC_FLAG_32BIT & flags) ? 4 : 8;
+
+    /* spin until the btl has accepted the operation */
+    do {
+        if (NULL == pending_op->op_frag) {
+            ret = ompi_osc_rdma_frag_alloc (module, 8, &pending_op->op_frag, (char **) &pending_op->op_buffer);
+        }
+        if (NULL != pending_op->op_frag) {
+            ret = ompi_osc_rdma_btl_atomic_cswap(module, btl_index, endpoint, pending_op->op_buffer,
+                                                  address, pending_op->op_frag->handle, address_handle, compare,
+                                                  value, flags, 0, ompi_osc_rdma_atomic_complete, (void *) pending_op,
+                                                  NULL);
+        }
+
+        if (OPAL_LIKELY(!ompi_osc_rdma_oor(ret))) {
+            break;
+        }
+        ompi_osc_rdma_progress (module);
+    } while (1);
+
+    if (OPAL_SUCCESS != ret) {
+        if (OPAL_LIKELY(1 == ret)) {
+            *result = ((int64_t *) pending_op->op_buffer)[0];
+            ret = OMPI_SUCCESS;
+        }
+
+        /* need to release here because ompi_osc_rdma_atomic_complete was not called */
+        OBJ_RELEASE(pending_op);
+    } else {
+        while (!pending_op->op_complete) {
+            ompi_osc_rdma_progress (module);
+        }
+    }
+
+    OBJ_RELEASE(pending_op);
+
+    return ret;
+}
+
+#endif /* OSC_RDMA_BTL_COMM_H */
diff --git a/ompi/mca/osc/rdma/osc_rdma_comm.c b/ompi/mca/osc/rdma/osc_rdma_comm.c
index 449bbea0641..9b519e36eaf 100644
--- a/ompi/mca/osc/rdma/osc_rdma_comm.c
+++ b/ompi/mca/osc/rdma/osc_rdma_comm.c
@@ -6,6 +6,8 @@
  * Copyright (c) 2017      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -13,13 +15,19 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "osc_rdma_comm.h"
+#include "osc_rdma_frag.h"
 #include "osc_rdma_sync.h"
 #include "osc_rdma_request.h"
 #include "osc_rdma_dynamic.h"
+#include "osc_rdma_btl_comm.h"
 
-#include "ompi/mca/osc/base/osc_base_obj_convert.h"
 #include "opal/align.h"
+#include "opal/util/minmax.h"
+#include "ompi/mca/osc/base/osc_base_obj_convert.h"
+#include "opal/mca/accelerator/accelerator.h"
 
 /* helper functions */
 static inline void ompi_osc_rdma_cleanup_rdma (ompi_osc_rdma_sync_t *sync, bool dec_always, ompi_osc_rdma_frag_t *frag,
@@ -58,8 +66,7 @@ int ompi_osc_get_data_blocking (ompi_osc_rdma_module_t *module, uint8_t btl_inde
                                 struct mca_btl_base_endpoint_t *endpoint, uint64_t source_address,
                                 mca_btl_base_registration_handle_t *source_handle, void *data, size_t len)
 {
-    mca_btl_base_module_t *btl = ompi_osc_rdma_selected_btl (module, btl_index);
-    const size_t btl_alignment_mask = ALIGNMENT_MASK(btl->btl_get_alignment);
+    const size_t btl_alignment_mask = ALIGNMENT_MASK(module->get_alignment);
     mca_btl_base_registration_handle_t *local_handle = NULL;
     ompi_osc_rdma_frag_t *frag = NULL;
     volatile bool read_complete = false;
@@ -75,30 +82,33 @@ int ompi_osc_get_data_blocking (ompi_osc_rdma_module_t *module, uint8_t btl_inde
                      "), len: %lu (aligned: %lu)", (void *) endpoint, source_address, aligned_addr, (unsigned long) len,
                      (unsigned long) aligned_len);
 
-    if (btl->btl_register_mem && len >= btl->btl_get_local_registration_threshold) {
-        do {
-            ret = ompi_osc_rdma_frag_alloc (module, aligned_len, &frag, &ptr);
-            if (OPAL_UNLIKELY(OMPI_ERR_OUT_OF_RESOURCE == ret)) {
-                ompi_osc_rdma_progress (module);
+    if (module->use_memory_registration) {
+        mca_btl_base_module_t *btl = ompi_osc_rdma_selected_btl(module, btl_index);
+        if (len >= btl->btl_get_local_registration_threshold) {
+            do {
+                ret = ompi_osc_rdma_frag_alloc(module, aligned_len, &frag, &ptr);
+                if (OPAL_UNLIKELY(OMPI_ERR_OUT_OF_RESOURCE == ret)) {
+                    ompi_osc_rdma_progress(module);
+                }
+            } while (OMPI_ERR_OUT_OF_RESOURCE == ret);
+
+            if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
+                OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_ERROR, "error allocating temporary buffer");
+                return ret;
             }
-        } while (OMPI_ERR_OUT_OF_RESOURCE == ret);
 
-        if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
-            OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_ERROR, "error allocating temporary buffer");
-            return ret;
+            local_handle = frag->handle;
+            OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "allocated temporary buffer %p in fragment %p", (void*)ptr,
+                             (void *) frag);
         }
-
-        local_handle = frag->handle;
-        OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "allocated temporary buffer %p in fragment %p", (void*)ptr,
-                         (void *) frag);
     }
 
-    assert (!(source_address & ALIGNMENT_MASK(btl->btl_get_alignment)));
+    assert (!(source_address & ALIGNMENT_MASK(module->get_alignment)));
 
     do {
-        ret = btl->btl_get (btl, endpoint, ptr, aligned_addr,
-                            local_handle, source_handle, aligned_len, 0, MCA_BTL_NO_ORDER,
-                            ompi_osc_get_data_complete, (void *) &read_complete, NULL);
+        ret = ompi_osc_rdma_btl_get(module, btl_index, endpoint, ptr, aligned_addr,
+                                    local_handle, source_handle, aligned_len, 0, MCA_BTL_NO_ORDER,
+                                    ompi_osc_get_data_complete, (void *) &read_complete, NULL);
         if (!ompi_osc_rdma_oor (ret)) {
             break;
         }
@@ -126,8 +136,10 @@ int ompi_osc_get_data_blocking (ompi_osc_rdma_module_t *module, uint8_t btl_inde
     opal_memchecker_base_mem_defined (ptr, len);
 
     if (frag) {
-        memcpy (data, ptr + offset, len);
-
+        ret = osc_rdma_accelerator_mem_copy(data, ptr + offset, len);
+        if (ret) {
+            return ret;
+        }
         /* done with the fragment */
         ompi_osc_rdma_frag_complete (frag);
     }
@@ -176,9 +188,9 @@ typedef int (*ompi_osc_rdma_fn_t) (ompi_osc_rdma_sync_t *sync, ompi_osc_rdma_pee
  * This function does the work of breaking a non-contiguous rdma transfer into contiguous components. It will
  * continue to submit rdma transfers until the entire region is transferred or a fatal error occurs.
  */
-static int ompi_osc_rdma_master_noncontig (ompi_osc_rdma_sync_t *sync, void *local_address, int local_count, ompi_datatype_t *local_datatype,
+static int ompi_osc_rdma_master_noncontig (ompi_osc_rdma_sync_t *sync, void *local_address, size_t local_count, ompi_datatype_t *local_datatype,
                                            ompi_osc_rdma_peer_t *peer, uint64_t remote_address,
-                                           mca_btl_base_registration_handle_t *remote_handle, int remote_count,
+                                           mca_btl_base_registration_handle_t *remote_handle, size_t remote_count,
                                            ompi_datatype_t *remote_datatype, ompi_osc_rdma_request_t *request, const size_t max_rdma_len,
                                            const ompi_osc_rdma_fn_t rdma_fn, const bool alloc_reqs)
 {
@@ -229,7 +241,7 @@ static int ompi_osc_rdma_master_noncontig (ompi_osc_rdma_sync_t *sync, void *loc
         /* opal_convertor_raw returns true when it has reached the end of the data */
         done = opal_convertor_raw (&remote_convertor, remote_iovec, &remote_iov_count, &remote_size);
 
-        /* loop on the target segments until we have exhaused the decoded source data */
+        /* loop on the target segments until we have exhausted the decoded source data */
         while (remote_iov_index != remote_iov_count) {
             if (local_iov_index == local_iov_count) {
                 /* decode segments of the target buffer */
@@ -242,7 +254,7 @@ static int ompi_osc_rdma_master_noncontig (ompi_osc_rdma_sync_t *sync, void *loc
             assert (0 != local_iov_count);
 
             /* determine how much to transfer in this operation */
-            rdma_len = min(min(local_iovec[local_iov_index].iov_len, remote_iovec[remote_iov_index].iov_len), max_rdma_len);
+            rdma_len = opal_min(opal_min(local_iovec[local_iov_index].iov_len, remote_iovec[remote_iov_index].iov_len), max_rdma_len);
 
             /* execute the get */
             if (!subreq && alloc_reqs) {
@@ -311,10 +323,10 @@ static int ompi_osc_rdma_master_noncontig (ompi_osc_rdma_sync_t *sync, void *loc
     return OMPI_SUCCESS;
 }
 
-static inline int ompi_osc_rdma_master (ompi_osc_rdma_sync_t *sync, void *local_address, int local_count,
+static inline int ompi_osc_rdma_master (ompi_osc_rdma_sync_t *sync, void *local_address, size_t local_count,
                                         ompi_datatype_t *local_datatype, ompi_osc_rdma_peer_t *peer,
                                         uint64_t remote_address, mca_btl_base_registration_handle_t *remote_handle,
-                                        int remote_count, ompi_datatype_t *remote_datatype,
+                                        size_t remote_count, ompi_datatype_t *remote_datatype,
                                         ompi_osc_rdma_request_t *request, const size_t max_rdma_len,
                                         const ompi_osc_rdma_fn_t rdma_fn, const bool alloc_reqs)
 {
@@ -361,8 +373,8 @@ static inline int ompi_osc_rdma_master (ompi_osc_rdma_sync_t *sync, void *local_
                                            max_rdma_len, rdma_fn, alloc_reqs);
 }
 
-static int ompi_osc_rdma_copy_local (const void *source, int source_count, ompi_datatype_t *source_datatype,
-                                     void *target, int target_count, ompi_datatype_t *target_datatype,
+static int ompi_osc_rdma_copy_local (const void *source, size_t source_count, ompi_datatype_t *source_datatype,
+                                     void *target, size_t target_count, ompi_datatype_t *target_datatype,
                                      ompi_osc_rdma_request_t *request)
 {
     int ret;
@@ -389,7 +401,7 @@ static void ompi_osc_rdma_put_complete (struct mca_btl_base_module_t *btl, struc
 
     /* the lowest bit is used as a flag indicating this put operation has a request */
     if ((intptr_t) context & 0x1) {
-        ompi_osc_rdma_request_t *request = request = (ompi_osc_rdma_request_t *) ((intptr_t) context & ~1);
+        ompi_osc_rdma_request_t *request = (ompi_osc_rdma_request_t *) ((intptr_t) context & ~1);
         sync = request->sync;
 
         if (0 == OPAL_THREAD_ADD_FETCH32 (&request->outstanding_requests, -1)) {
@@ -420,7 +432,7 @@ static void ompi_osc_rdma_put_complete_flush (struct mca_btl_base_module_t *btl,
 
     /* the lowest bit is used as a flag indicating this put operation has a request */
     if ((intptr_t) context & 0x1) {
-        ompi_osc_rdma_request_t *request = request = (ompi_osc_rdma_request_t *) ((intptr_t) context & ~1);
+        ompi_osc_rdma_request_t *request = (ompi_osc_rdma_request_t *) ((intptr_t) context & ~1);
         module = request->module;
 
         if (0 == OPAL_THREAD_ADD_FETCH32 (&request->outstanding_requests, -1)) {
@@ -444,7 +456,6 @@ static int ompi_osc_rdma_put_real (ompi_osc_rdma_sync_t *sync, ompi_osc_rdma_pee
                                    mca_btl_base_registration_handle_t *local_handle, size_t size,
                                    mca_btl_base_rdma_completion_fn_t cb, void *context, void *cbdata) {
     ompi_osc_rdma_module_t *module = sync->module;
-    mca_btl_base_module_t *btl = ompi_osc_rdma_selected_btl (module, peer->data_btl_index);
     int ret;
 
     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "initiating btl put of %lu bytes to remote address %" PRIx64 ", sync "
@@ -454,9 +465,9 @@ static int ompi_osc_rdma_put_real (ompi_osc_rdma_sync_t *sync, ompi_osc_rdma_pee
     ompi_osc_rdma_sync_rdma_inc (sync);
 
     do {
-        ret = btl->btl_put (btl, peer->data_endpoint, ptr, target_address,
-                            local_handle, target_handle, size, 0, MCA_BTL_NO_ORDER,
-                            cb, context, cbdata);
+        ret = ompi_osc_rdma_btl_put(module, peer->data_btl_index, peer->data_endpoint,
+                                    ptr, target_address, local_handle, target_handle,
+                                    size, 0, MCA_BTL_NO_ORDER, cb, context, cbdata);
         if (OPAL_UNLIKELY(OMPI_SUCCESS == ret)) {
             return OMPI_SUCCESS;
         }
@@ -481,7 +492,6 @@ int ompi_osc_rdma_put_contig (ompi_osc_rdma_sync_t *sync, ompi_osc_rdma_peer_t *
                               ompi_osc_rdma_request_t *request)
 {
     ompi_osc_rdma_module_t *module = sync->module;
-    mca_btl_base_module_t *btl = ompi_osc_rdma_selected_btl (module, peer->data_btl_index);
     mca_btl_base_registration_handle_t *local_handle = NULL;
     mca_btl_base_rdma_completion_fn_t cbfunc = NULL;
     ompi_osc_rdma_frag_t *frag = NULL;
@@ -489,16 +499,22 @@ int ompi_osc_rdma_put_contig (ompi_osc_rdma_sync_t *sync, ompi_osc_rdma_peer_t *
     void *cbcontext;
     int ret;
 
-    if (btl->btl_register_mem && size > btl->btl_put_local_registration_threshold) {
-        ret = ompi_osc_rdma_frag_alloc (module, size, &frag, &ptr);
-        if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
-            ret = ompi_osc_rdma_register (module, peer->data_endpoint, source_buffer, size, 0, &local_handle);
+    if (module->use_memory_registration) {
+        mca_btl_base_module_t *btl = ompi_osc_rdma_selected_btl(module, peer->data_btl_index);
+        if (size > btl->btl_put_local_registration_threshold) {
+            ret = ompi_osc_rdma_frag_alloc(module, size, &frag, &ptr);
             if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
-                return ret;
+                ret = ompi_osc_rdma_register(module, peer->data_endpoint, source_buffer, size, 0, &local_handle);
+                if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
+                    return ret;
+                }
+            } else {
+                ret = osc_rdma_accelerator_mem_copy(ptr, source_buffer, size);
+                if (ret) {
+                    return ret;
+                }
+                local_handle = frag->handle;
             }
-        } else {
-            memcpy (ptr, source_buffer, size);
-            local_handle = frag->handle;
         }
     }
 
@@ -506,7 +522,7 @@ int ompi_osc_rdma_put_contig (ompi_osc_rdma_sync_t *sync, ompi_osc_rdma_peer_t *
         /* NTH: when using the btl_flush function there is no guarantee that the callback will happen
          * before the flush is complete. because of this there is a chance that the sync object will be
          * released before there is a callback. to handle this case we call different callback that doesn't
-         * use the sync object. its possible the btl sematics will change in the future and the callback
+         * use the sync object. its possible the btl semantics will change in the future and the callback
          * will happen *before* flush is considered complete. if that is the case this workaround can be
          * removed */
         cbcontext = (void *) module;
@@ -544,6 +560,7 @@ static void ompi_osc_rdma_get_complete (struct mca_btl_base_module_t *btl, struc
     ompi_osc_rdma_frag_t *frag = (ompi_osc_rdma_frag_t *) data;
     ompi_osc_rdma_sync_t *sync = request->sync;
     void *origin_addr = request->origin_addr;
+    int ret;
 
     OSC_RDMA_VERBOSE(status ? MCA_BASE_VERBOSE_ERROR : MCA_BASE_VERBOSE_TRACE, "btl get complete on sync %p. local "
                      "address %p. origin %p. opal status %d", (void *) sync, local_address, origin_addr, status);
@@ -552,7 +569,11 @@ static void ompi_osc_rdma_get_complete (struct mca_btl_base_module_t *btl, struc
 
     if (request->buffer || frag) {
         if (OPAL_LIKELY(OMPI_SUCCESS == status)) {
-            memcpy (origin_addr, (void *) source, request->len);
+            ret = osc_rdma_accelerator_mem_copy(origin_addr, (void *) source, request->len);
+            if (ret) {
+                /* Can't bubble up this failure, abort */
+                abort();
+            }
         }
     }
 
@@ -600,8 +621,7 @@ static int ompi_osc_rdma_get_contig (ompi_osc_rdma_sync_t *sync, ompi_osc_rdma_p
                                      ompi_osc_rdma_request_t *request)
 {
     ompi_osc_rdma_module_t *module = sync->module;
-    mca_btl_base_module_t *btl = ompi_osc_rdma_selected_btl (module, peer->data_btl_index);
-    const size_t btl_alignment_mask = ALIGNMENT_MASK(btl->btl_get_alignment);
+    const size_t btl_alignment_mask = ALIGNMENT_MASK(module->get_alignment);
     mca_btl_base_registration_handle_t *local_handle = NULL;
     ompi_osc_rdma_frag_t *frag = NULL;
     osc_rdma_size_t aligned_len;
@@ -617,70 +637,73 @@ static int ompi_osc_rdma_get_contig (ompi_osc_rdma_sync_t *sync, ompi_osc_rdma_p
     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "initiating get of %lu bytes from remote ptr %" PRIx64 " to local ptr %p",
                      size, source_address, target_buffer);
 
-    if ((btl->btl_register_mem && size > btl->btl_get_local_registration_threshold) ||
-        (((uint64_t) target_buffer | size | source_address) & btl_alignment_mask)) {
-
-        ret = ompi_osc_rdma_frag_alloc (module, aligned_len, &frag, &ptr);
-        if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
-            if (OMPI_ERR_VALUE_OUT_OF_BOUNDS == ret) {
-                /* region is too large for a buffered read */
-                size_t subsize;
+    if (module->use_memory_registration) {
+        mca_btl_base_module_t *btl = ompi_osc_rdma_selected_btl(module, peer->data_btl_index);
+        if (size > btl->btl_get_local_registration_threshold ||
+            (((uint64_t) target_buffer | size | source_address) & btl_alignment_mask)) {
 
-                if ((source_address & btl_alignment_mask) && (source_address & btl_alignment_mask) == ((intptr_t) target_buffer & btl_alignment_mask)) {
-                    /* remote region has the same alignment but the base is not aligned. perform a small
-                     * buffered get of the beginning of the remote region */
-                    aligned_source_base = OPAL_ALIGN(source_address, btl->btl_get_alignment, osc_rdma_base_t);
-                    subsize = (size_t) (aligned_source_base - source_address);
-
-                    ret = ompi_osc_rdma_get_partial (sync, peer, source_address, source_handle, target_buffer, subsize, request);
-                    if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
-                        return ret;
+            ret = ompi_osc_rdma_frag_alloc(module, aligned_len, &frag, &ptr);
+            if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
+                if (OMPI_ERR_VALUE_OUT_OF_BOUNDS == ret) {
+                    /* region is too large for a buffered read */
+                    size_t subsize;
+
+                    if ((source_address & btl_alignment_mask) && (source_address & btl_alignment_mask) == ((intptr_t) target_buffer & btl_alignment_mask)) {
+                        /* remote region has the same alignment but the base is not aligned. perform a small
+                         * buffered get of the beginning of the remote region */
+                        aligned_source_base = OPAL_ALIGN(source_address, btl->btl_get_alignment, osc_rdma_base_t);
+                        subsize = (size_t) (aligned_source_base - source_address);
+
+                        ret = ompi_osc_rdma_get_partial(sync, peer, source_address, source_handle, target_buffer, subsize, request);
+                        if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
+                            return ret;
+                        }
+
+                        source_address += subsize;
+                        target_buffer = (void *) ((intptr_t) target_buffer + subsize);
+                        size -= subsize;
+
+                        aligned_len = aligned_source_bound - aligned_source_base;
                     }
 
-                    source_address += subsize;
-                    target_buffer = (void *) ((intptr_t) target_buffer + subsize);
-                    size -= subsize;
+                    if (!(((uint64_t) target_buffer | source_address) & btl_alignment_mask) &&
+                        (size & btl_alignment_mask)) {
+                        /* remote region bases are aligned but the bounds are not. perform a
+                         * small buffered get of the end of the remote region */
+                        aligned_len = size & ~btl_alignment_mask;
+                        subsize = size - aligned_len;
+                        size = aligned_len;
+                        ret = ompi_osc_rdma_get_partial(sync, peer, source_address + aligned_len, source_handle,
+                                                        (void *) ((intptr_t) target_buffer + aligned_len), subsize, request);
+                        if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
+                            return ret;
+                        }
+                    }
+                    /* (remaining) user request is now correctly aligned */
+                }
 
-                    aligned_len = aligned_source_bound - aligned_source_base;
+                if ((((uint64_t) target_buffer | size | source_address) & btl_alignment_mask)) {
+                    /* local and remote alignments differ */
+                    request->buffer = ptr = malloc(aligned_len);
+                } else {
+                    ptr = target_buffer;
                 }
 
-                if (!(((uint64_t) target_buffer | source_address) & btl_alignment_mask) &&
-                    (size & btl_alignment_mask)) {
-                    /* remote region bases are aligned but the bounds are not. perform a
-                     * small buffered get of the end of the remote region */
-                    aligned_len = size & ~btl_alignment_mask;
-                    subsize = size - aligned_len;
-                    size = aligned_len;
-                    ret = ompi_osc_rdma_get_partial (sync, peer, source_address + aligned_len, source_handle,
-                                                     (void *) ((intptr_t) target_buffer + aligned_len), subsize, request);
-                    if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
-                        return ret;
-                    }
+                if (NULL != ptr) {
+                    (void)ompi_osc_rdma_register(module, peer->data_endpoint, ptr, aligned_len, MCA_BTL_REG_FLAG_LOCAL_WRITE,
+                                                 &local_handle);
                 }
-                /* (remaining) user request is now correctly aligned */
-            }
 
-            if ((((uint64_t) target_buffer | size | source_address) & btl_alignment_mask)) {
-                /* local and remote alignments differ */
-                request->buffer = ptr = malloc (aligned_len);
+                if (OPAL_UNLIKELY(NULL == local_handle)) {
+                    free(request->buffer);
+                    request->buffer = NULL;
+                    return ret;
+                }
             } else {
-                ptr = target_buffer;
-            }
-
-            if (NULL != ptr) {
-                (void) ompi_osc_rdma_register (module, peer->data_endpoint, ptr, aligned_len, MCA_BTL_REG_FLAG_LOCAL_WRITE,
-                                               &local_handle);
+                OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "using internal buffer %p in fragment %p for get of size %lu bytes, source address 0x%lx",
+                                 (void*)ptr, (void *) frag, (unsigned long) aligned_len, (unsigned long) aligned_source_base);
+                local_handle = frag->handle;
             }
-
-            if (OPAL_UNLIKELY(NULL == local_handle)) {
-                free (request->buffer);
-                request->buffer = NULL;
-                return ret;
-            }
-        } else {
-            OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "using internal buffer %p in fragment %p for get of size %lu bytes, source address 0x%lx",
-                             (void*)ptr, (void *) frag, (unsigned long) aligned_len, (unsigned long) aligned_source_base);
-            local_handle = frag->handle;
         }
     }
 
@@ -703,10 +726,10 @@ static int ompi_osc_rdma_get_contig (ompi_osc_rdma_sync_t *sync, ompi_osc_rdma_p
     }
 
     do {
-        ret = btl->btl_get (btl, peer->data_endpoint, ptr,
-                            aligned_source_base, local_handle, source_handle,
-                            aligned_len, 0, MCA_BTL_NO_ORDER, ompi_osc_rdma_get_complete,
-                            request, frag);
+        ret = ompi_osc_rdma_btl_get(module, peer->data_btl_index, peer->data_endpoint,
+                                    ptr, aligned_source_base, local_handle, source_handle,
+                                    aligned_len, 0, MCA_BTL_NO_ORDER,
+                                    ompi_osc_rdma_get_complete, request, frag);
         if (OPAL_LIKELY(OMPI_SUCCESS == ret)) {
             return OMPI_SUCCESS;
         }
@@ -736,7 +759,6 @@ static inline int ompi_osc_rdma_put_w_req (ompi_osc_rdma_sync_t *sync, const voi
                                            ompi_datatype_t *target_datatype, ompi_osc_rdma_request_t *request)
 {
     ompi_osc_rdma_module_t *module = sync->module;
-    mca_btl_base_module_t *btl = ompi_osc_rdma_selected_btl (module, peer->data_btl_index);
     mca_btl_base_registration_handle_t *target_handle;
     uint64_t target_address;
     int ret;
@@ -771,15 +793,14 @@ static inline int ompi_osc_rdma_put_w_req (ompi_osc_rdma_sync_t *sync, const voi
 
     return ompi_osc_rdma_master (sync, (void *) origin_addr, origin_count, origin_datatype, peer,
                                  target_address, target_handle, target_count, target_datatype, request,
-                                 btl->btl_put_limit, ompi_osc_rdma_put_contig, false);
+                                 module->put_limit, ompi_osc_rdma_put_contig, false);
 }
 
-static inline int ompi_osc_rdma_get_w_req (ompi_osc_rdma_sync_t *sync, void *origin_addr, int origin_count, ompi_datatype_t *origin_datatype,
-                                           ompi_osc_rdma_peer_t *peer, ptrdiff_t source_disp, int source_count,
+static inline int ompi_osc_rdma_get_w_req (ompi_osc_rdma_sync_t *sync, void *origin_addr, size_t origin_count, ompi_datatype_t *origin_datatype,
+                                           ompi_osc_rdma_peer_t *peer, ptrdiff_t source_disp, size_t source_count,
                                            ompi_datatype_t *source_datatype, ompi_osc_rdma_request_t *request)
 {
     ompi_osc_rdma_module_t *module = sync->module;
-    mca_btl_base_module_t *btl = ompi_osc_rdma_selected_btl (module, peer->data_btl_index);
     mca_btl_base_registration_handle_t *source_handle;
     uint64_t source_address;
     ptrdiff_t source_span, source_lb;
@@ -812,17 +833,17 @@ static inline int ompi_osc_rdma_get_w_req (ompi_osc_rdma_sync_t *sync, void *ori
 
     return ompi_osc_rdma_master (sync, origin_addr, origin_count, origin_datatype, peer, source_address,
                                  source_handle, source_count, source_datatype, request,
-                                 btl->btl_get_limit, ompi_osc_rdma_get_contig, true);
+                                 module->get_limit, ompi_osc_rdma_get_contig, true);
 }
-int ompi_osc_rdma_put (const void *origin_addr, int origin_count, ompi_datatype_t *origin_datatype,
-                       int target_rank, ptrdiff_t target_disp, int target_count,
+int ompi_osc_rdma_put (const void *origin_addr, size_t origin_count, ompi_datatype_t *origin_datatype,
+                       int target_rank, ptrdiff_t target_disp, size_t target_count,
                        ompi_datatype_t *target_datatype, ompi_win_t *win)
 {
     ompi_osc_rdma_module_t *module = GET_MODULE(win);
     ompi_osc_rdma_peer_t *peer;
     ompi_osc_rdma_sync_t *sync;
 
-    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "put: 0x%lx, %d, %s, %d, %d, %d, %s, %s", (unsigned long) origin_addr,
+    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "put: 0x%lx, %zu, %s, %d, %d, %zu, %s, %s", (unsigned long) origin_addr,
                      origin_count, origin_datatype->name, target_rank, (int) target_disp, target_count,
                      target_datatype->name, win->w_name);
 
@@ -835,8 +856,8 @@ int ompi_osc_rdma_put (const void *origin_addr, int origin_count, ompi_datatype_
                                     target_count, target_datatype, NULL);
 }
 
-int ompi_osc_rdma_rput (const void *origin_addr, int origin_count, ompi_datatype_t *origin_datatype,
-                        int target_rank, ptrdiff_t target_disp, int target_count,
+int ompi_osc_rdma_rput (const void *origin_addr, size_t origin_count, ompi_datatype_t *origin_datatype,
+                        int target_rank, ptrdiff_t target_disp, size_t target_count,
                         ompi_datatype_t *target_datatype, ompi_win_t *win,
                         ompi_request_t **request)
 {
@@ -846,7 +867,7 @@ int ompi_osc_rdma_rput (const void *origin_addr, int origin_count, ompi_datatype
     ompi_osc_rdma_sync_t *sync;
     int ret;
 
-    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "rput: 0x%lx, %d, %s, %d, %d, %d, %s, %s", (unsigned long) origin_addr, origin_count,
+    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "rput: 0x%lx, %zu, %s, %d, %d, %zu, %s, %s", (unsigned long) origin_addr, origin_count,
                      origin_datatype->name, target_rank, (int) target_disp, target_count, target_datatype->name, win->w_name);
 
     sync = ompi_osc_rdma_module_sync_lookup (module, target_rank, &peer);
@@ -870,15 +891,15 @@ int ompi_osc_rdma_rput (const void *origin_addr, int origin_count, ompi_datatype
     return OMPI_SUCCESS;
 }
 
-int ompi_osc_rdma_get (void *origin_addr, int origin_count, ompi_datatype_t *origin_datatype,
-                       int source_rank, ptrdiff_t source_disp, int source_count,
+int ompi_osc_rdma_get (void *origin_addr, size_t origin_count, ompi_datatype_t *origin_datatype,
+                       int source_rank, ptrdiff_t source_disp, size_t source_count,
                        ompi_datatype_t *source_datatype, ompi_win_t *win)
 {
     ompi_osc_rdma_module_t *module = GET_MODULE(win);
     ompi_osc_rdma_peer_t *peer;
     ompi_osc_rdma_sync_t *sync;
 
-    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "get: 0x%lx, %d, %s, %d, %d, %d, %s, %s", (unsigned long) origin_addr,
+    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "get: 0x%lx, %zu, %s, %d, %d, %zu, %s, %s", (unsigned long) origin_addr,
                      origin_count, origin_datatype->name, source_rank, (int) source_disp, source_count,
                      source_datatype->name, win->w_name);
 
@@ -891,8 +912,8 @@ int ompi_osc_rdma_get (void *origin_addr, int origin_count, ompi_datatype_t *ori
                                     source_disp, source_count, source_datatype, NULL);
 }
 
-int ompi_osc_rdma_rget (void *origin_addr, int origin_count, ompi_datatype_t *origin_datatype,
-                        int source_rank, ptrdiff_t source_disp, int source_count,
+int ompi_osc_rdma_rget (void *origin_addr, size_t origin_count, ompi_datatype_t *origin_datatype,
+                        int source_rank, ptrdiff_t source_disp, size_t source_count,
                         ompi_datatype_t *source_datatype, ompi_win_t *win,
                         ompi_request_t **request)
 {
@@ -902,7 +923,7 @@ int ompi_osc_rdma_rget (void *origin_addr, int origin_count, ompi_datatype_t *or
     ompi_osc_rdma_sync_t *sync;
     int ret;
 
-    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "rget: 0x%lx, %d, %s, %d, %d, %d, %s, %s", (unsigned long) origin_addr,
+    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "rget: 0x%lx, %zu, %s, %d, %d, %zu, %s, %s", (unsigned long) origin_addr,
                      origin_count, origin_datatype->name, source_rank, (int) source_disp, source_count,
                      source_datatype->name, win->w_name);
 
diff --git a/ompi/mca/osc/rdma/osc_rdma_comm.h b/ompi/mca/osc/rdma/osc_rdma_comm.h
index d399bf244b4..153633846a0 100644
--- a/ompi/mca/osc/rdma/osc_rdma_comm.h
+++ b/ompi/mca/osc/rdma/osc_rdma_comm.h
@@ -4,6 +4,8 @@
  *                         reserved.
  * Copyright (c) 2017      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -11,17 +13,15 @@
  * $HEADER$
  */
 
-#if !defined(OMPI_OSC_RDMA_COMM_H)
+#ifndef OMPI_OSC_RDMA_COMM_H
 #define OMPI_OSC_RDMA_COMM_H
 
 #include "osc_rdma_dynamic.h"
 #include "osc_rdma_request.h"
 #include "osc_rdma_sync.h"
-#include "osc_rdma_lock.h"
 
 #define OMPI_OSC_RDMA_DECODE_MAX 64
 
-#define min(a,b) ((a) < (b) ? (a) : (b))
 #define ALIGNMENT_MASK(x) ((x) ? (x) - 1 : 0)
 
 /**
@@ -59,7 +59,7 @@ static inline int osc_rdma_get_remote_segment (ompi_osc_rdma_module_t *module, o
         *remote_handle = (mca_btl_base_registration_handle_t *) region->btl_handle_data;
     } else {
         ompi_osc_rdma_peer_extended_t *ex_peer = (ompi_osc_rdma_peer_extended_t *) peer;
-        int disp_unit = (module->same_disp_unit) ? module->disp_unit : ex_peer->disp_unit;
+        ptrdiff_t disp_unit = (module->same_disp_unit) ? module->disp_unit : ex_peer->disp_unit;
         size_t size = (module->same_size) ? module->size : (size_t) ex_peer->size;
 
         *remote_address = ex_peer->super.base + disp_unit * target_disp;
@@ -81,21 +81,21 @@ static inline int osc_rdma_get_remote_segment (ompi_osc_rdma_module_t *module, o
 
 /* prototypes for implementations of MPI RMA window functions. these will be called from the
  * mpi interface (ompi/mpi/c) */
-int ompi_osc_rdma_put (const void *origin_addr, int origin_count, ompi_datatype_t *origin_dt,
-                       int target, ptrdiff_t target_disp, int target_count,
+int ompi_osc_rdma_put (const void *origin_addr, size_t origin_count, ompi_datatype_t *origin_dt,
+                       int target, ptrdiff_t target_disp, size_t target_count,
                        ompi_datatype_t *target_dt, ompi_win_t *win);
 
-int ompi_osc_rdma_get (void *origin_addr, int origin_count, ompi_datatype_t *origin_dt,
-                       int target, ptrdiff_t target_disp, int target_count,
+int ompi_osc_rdma_get (void *origin_addr, size_t origin_count, ompi_datatype_t *origin_dt,
+                       int target, ptrdiff_t target_disp, size_t target_count,
                        ompi_datatype_t *target_dt, ompi_win_t *win);
 
-int ompi_osc_rdma_rput (const void *origin_addr, int origin_count, ompi_datatype_t *origin_dt,
-                        int target, ptrdiff_t target_disp, int target_count,
+int ompi_osc_rdma_rput (const void *origin_addr, size_t origin_count, ompi_datatype_t *origin_dt,
+                        int target, ptrdiff_t target_disp, size_t target_count,
                         ompi_datatype_t *target_dt, ompi_win_t *win,
                         ompi_request_t **request);
 
-int ompi_osc_rdma_rget (void *origin_addr, int origin_count, ompi_datatype_t *origin_dt,
-                        int target, ptrdiff_t target_disp, int target_count,
+int ompi_osc_rdma_rget (void *origin_addr, size_t origin_count, ompi_datatype_t *origin_dt,
+                        int target, ptrdiff_t target_disp, size_t target_count,
                         ompi_datatype_t *target_dt, ompi_win_t *win,
                         ompi_request_t **request);
 
diff --git a/ompi/mca/osc/rdma/osc_rdma_component.c b/ompi/mca/osc/rdma/osc_rdma_component.c
index dd0668aba8e..a7a005a8afe 100644
--- a/ompi/mca/osc/rdma/osc_rdma_component.c
+++ b/ompi/mca/osc/rdma/osc_rdma_component.c
@@ -5,7 +5,7 @@
  * Copyright (c) 2004-2017 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ * Copyright (c) 2004-2020 High Performance Computing Center Stuttgart,
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
@@ -18,7 +18,8 @@
  * Copyright (c) 2015-2017 Intel, Inc. All rights reserved.
  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
  * Copyright (c) 2018      Cisco Systems, Inc.  All rights reserved
- * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2018-2022 Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * Copyright (c) 2019      Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2020-2021 Google, LLC. All rights reserved.
@@ -34,6 +35,7 @@
 #include "ompi_config.h"
 
 #include <string.h>
+#include <stdlib.h>
 
 #include "osc_rdma.h"
 #include "osc_rdma_frag.h"
@@ -48,12 +50,11 @@
 #include "opal/util/arch.h"
 #include "opal/util/argv.h"
 #include "opal/util/printf.h"
-#include "opal/align.h"
 #include "opal/util/sys_limits.h"
-#if OPAL_CUDA_SUPPORT
-#include "opal/mca/common/cuda/common_cuda.h"
-#endif /* OPAL_CUDA_SUPPORT */
+#include "opal/util/minmax.h"
+#include "opal/mca/accelerator/accelerator.h"
 #include "opal/util/info_subscriber.h"
+#include "opal/mca/mpool/base/base.h"
 
 #include "ompi/info/info.h"
 #include "ompi/communicator/communicator.h"
@@ -71,21 +72,18 @@
 static int ompi_osc_rdma_component_register (void);
 static int ompi_osc_rdma_component_init (bool enable_progress_threads, bool enable_mpi_threads);
 static int ompi_osc_rdma_component_finalize (void);
-static int ompi_osc_rdma_component_query (struct ompi_win_t *win, void **base, size_t size, int disp_unit,
+static int ompi_osc_rdma_component_query (struct ompi_win_t *win, void **base, size_t size, ptrdiff_t disp_unit,
                                           struct ompi_communicator_t *comm, struct opal_info_t *info,
                                           int flavor);
-static int ompi_osc_rdma_component_select (struct ompi_win_t *win, void **base, size_t size, int disp_unit,
+static int ompi_osc_rdma_component_select (struct ompi_win_t *win, void **base, size_t size, ptrdiff_t disp_unit,
                                            struct ompi_communicator_t *comm, struct opal_info_t *info,
                                            int flavor, int *model);
-static int ompi_osc_rdma_query_btls (ompi_communicator_t *comm, ompi_osc_rdma_module_t *module);
+static int ompi_osc_rdma_query_accelerated_btls (ompi_communicator_t *comm, ompi_osc_rdma_module_t *module);
 static int ompi_osc_rdma_query_alternate_btls (ompi_communicator_t *comm, ompi_osc_rdma_module_t *module);
-static int ompi_osc_rdma_query_mtls (void);
 
-static char* ompi_osc_rdma_set_no_lock_info(opal_infosubscriber_t *obj, char *key, char *value);
+static const char* ompi_osc_rdma_set_no_lock_info(opal_infosubscriber_t *obj, const char *key, const char *value);
 
-static char *ompi_osc_rdma_btl_names;
-static char *ompi_osc_rdma_mtl_names;
-static char *ompi_osc_rdma_btl_alternate_names;
+static char *ompi_osc_rdma_full_connectivity_btls;
 
 static const mca_base_var_enum_value_t ompi_osc_rdma_locking_modes[] = {
     {.value = OMPI_OSC_RDMA_LOCKING_TWO_LEVEL, .string = "two_level"},
@@ -96,7 +94,7 @@ static const mca_base_var_enum_value_t ompi_osc_rdma_locking_modes[] = {
 ompi_osc_rdma_component_t mca_osc_rdma_component = {
     .super = {
         .osc_version = {
-            OMPI_OSC_BASE_VERSION_3_0_0,
+            OMPI_OSC_BASE_VERSION_4_0_0,
             .mca_component_name = "rdma",
             MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION,
                                   OMPI_RELEASE_VERSION),
@@ -209,7 +207,7 @@ static int ompi_osc_rdma_component_register (void)
     mca_osc_rdma_component.acc_use_amo = true;
     opal_asprintf(&description_str, "Enable the use of network atomic memory operations when using single "
              "intrinsic optimizations. If not set network compare-and-swap will be "
-             "used instread (default: %s)", mca_osc_rdma_component.acc_use_amo ? "true" : "false");
+             "used instead (default: %s)", mca_osc_rdma_component.acc_use_amo ? "true" : "false");
     (void) mca_base_component_var_register(&mca_osc_rdma_component.super.osc_version, "acc_use_amo", description_str,
                                            MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_GROUP,
                                            &mca_osc_rdma_component.acc_use_amo);
@@ -231,7 +229,7 @@ static int ompi_osc_rdma_component_register (void)
                                            MCA_BASE_VAR_SCOPE_GROUP, &mca_osc_rdma_component.max_attach);
     free(description_str);
 
-    mca_osc_rdma_component.priority = 101;
+    mca_osc_rdma_component.priority = 20;
     opal_asprintf(&description_str, "Priority of the osc/rdma component (default: %d)",
              mca_osc_rdma_component.priority);
     (void) mca_base_component_var_register (&mca_osc_rdma_component.super.osc_version, "priority", description_str,
@@ -239,14 +237,6 @@ static int ompi_osc_rdma_component_register (void)
                                             MCA_BASE_VAR_SCOPE_GROUP, &mca_osc_rdma_component.priority);
     free(description_str);
 
-    mca_osc_rdma_component.alternate_priority = 37;
-    opal_asprintf(&description_str, "Priority of the osc/rdma component when using non-RDMA btls (default: %d)",
-             mca_osc_rdma_component.alternate_priority);
-    (void) mca_base_component_var_register (&mca_osc_rdma_component.super.osc_version, "alternate_priority", description_str,
-                                            MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, 0, OPAL_INFO_LVL_3,
-                                            MCA_BASE_VAR_SCOPE_GROUP, &mca_osc_rdma_component.alternate_priority);
-    free(description_str);
-
     (void) mca_base_var_enum_create ("osc_rdma_locking_mode", ompi_osc_rdma_locking_modes, &new_enum);
 
     mca_osc_rdma_component.locking_mode = OMPI_OSC_RDMA_LOCKING_TWO_LEVEL;
@@ -256,30 +246,14 @@ static int ompi_osc_rdma_component_register (void)
                                             MCA_BASE_VAR_SCOPE_GROUP, &mca_osc_rdma_component.locking_mode);
     OBJ_RELEASE(new_enum);
 
-    ompi_osc_rdma_btl_names = "ugni,uct";
+    ompi_osc_rdma_full_connectivity_btls = "uct,ofi";
     opal_asprintf(&description_str, "Comma-delimited list of BTL component names to allow without verifying "
              "connectivity. Do not add a BTL to to this list unless it can reach all "
              "processes in any communicator used with an MPI window (default: %s)",
-             ompi_osc_rdma_btl_names);
+             ompi_osc_rdma_full_connectivity_btls);
     (void) mca_base_component_var_register (&mca_osc_rdma_component.super.osc_version, "btls", description_str,
                                             MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_3,
-                                            MCA_BASE_VAR_SCOPE_GROUP, &ompi_osc_rdma_btl_names);
-    free(description_str);
-
-    ompi_osc_rdma_btl_alternate_names = "sm,tcp";
-    opal_asprintf(&description_str, "Comma-delimited list of alternate BTL component names to allow without verifying "
-                  "connectivity (default: %s)", ompi_osc_rdma_btl_alternate_names);
-    (void) mca_base_component_var_register (&mca_osc_rdma_component.super.osc_version, "alternate_btls", description_str,
-                                            MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_3,
-                                            MCA_BASE_VAR_SCOPE_GROUP, &ompi_osc_rdma_btl_alternate_names);
-    free(description_str);
-
-    ompi_osc_rdma_mtl_names = "psm2";
-    opal_asprintf(&description_str, "Comma-delimited list of MTL component names to lower the priority of rdma "
-             "osc component (default: %s)", ompi_osc_rdma_mtl_names);
-    (void) mca_base_component_var_register (&mca_osc_rdma_component.super.osc_version, "mtls", description_str,
-                                            MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_3,
-                                            MCA_BASE_VAR_SCOPE_GROUP, &ompi_osc_rdma_mtl_names);
+                                            MCA_BASE_VAR_SCOPE_GROUP, &ompi_osc_rdma_full_connectivity_btls);
     free(description_str);
 
     if (0 == access ("/dev/shm", W_OK)) {
@@ -305,6 +279,16 @@ static int ompi_osc_rdma_component_register (void)
                                             MCA_BASE_VAR_TYPE_UNSIGNED_LONG, NULL, 0, 0, OPAL_INFO_LVL_3,
                                             MCA_BASE_VAR_SCOPE_LOCAL, &mca_osc_rdma_component.network_amo_max_count);
 
+    mca_osc_rdma_component.memory_alignment = opal_getpagesize();
+    opal_asprintf(&description_str, "The minimum memory alignment used to allocate local window memory (default: %zu). "
+                  "This is a best effort approach. Alignments larger than the page size may not be supported.",
+                  mca_osc_rdma_component.memory_alignment);
+    (void) mca_base_component_var_register (&mca_osc_rdma_component.super.osc_version, "minimum_memory_alignment",
+                                            description_str,
+                                            MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_3,
+                                            MCA_BASE_VAR_SCOPE_READONLY, &mca_osc_rdma_component.memory_alignment);
+    free(description_str);
+
     /* register performance variables */
 
     (void) mca_base_component_pvar_register (&mca_osc_rdma_component.super.osc_version, "put_retry_count",
@@ -384,7 +368,7 @@ int ompi_osc_rdma_component_finalize (void)
 }
 
 
-static int ompi_osc_rdma_component_query (struct ompi_win_t *win, void **base, size_t size, int disp_unit,
+static int ompi_osc_rdma_component_query (struct ompi_win_t *win, void **base, size_t size, ptrdiff_t disp_unit,
                                           struct ompi_communicator_t *comm, struct opal_info_t *info,
                                           int flavor)
 {
@@ -393,25 +377,21 @@ static int ompi_osc_rdma_component_query (struct ompi_win_t *win, void **base, s
         return -1;
     }
 
-#if OPAL_CUDA_SUPPORT
-    /* GPU buffers are not supported by the rdma component */
     if (MPI_WIN_FLAVOR_CREATE == flavor) {
-        if (opal_cuda_check_bufs(*base, NULL)) {
-            return -1;
+        uint64_t flags;
+        int dev_id;
+        if (opal_accelerator.check_addr(*base, &dev_id, &flags)) {
+            if (!osc_rdma_btl_accel_support(&mca_btl_base_modules_initialized)) {
+                return -1;
+            }
         }
     }
-#endif /* OPAL_CUDA_SUPPORT */
 
-    if (OMPI_SUCCESS == ompi_osc_rdma_query_mtls ()) {
-        return 5;
-    }
-
-    if (OMPI_SUCCESS == ompi_osc_rdma_query_btls (comm, NULL)) {
-        return mca_osc_rdma_component.priority;
-    }
-
-    if (OMPI_SUCCESS == ompi_osc_rdma_query_alternate_btls (comm, NULL)) {
-        return mca_osc_rdma_component.alternate_priority;
+    /* verify if we have any btls available.  Since we do not verify
+     * connectivity across all btls in the alternate case, this is as
+     * good a test as we are going to have for success. */
+    if (opal_list_is_empty(&mca_btl_base_modules_initialized)) {
+        return -1;
     }
 
     return mca_osc_rdma_component.priority;
@@ -430,6 +410,7 @@ static int ompi_osc_rdma_initialize_region (ompi_osc_rdma_module_t *module, void
     region->len = size;
 
     if (module->use_memory_registration && size) {
+        assert(module->use_accelerated_btl);
         if (MPI_WIN_FLAVOR_ALLOCATE != module->flavor || NULL == module->state_handle) {
             ret = ompi_osc_rdma_register (module, MCA_BTL_ENDPOINT_ANY, *base, size, MCA_BTL_REG_FLAG_ACCESS_ANY,
                                           &module->base_handle);
@@ -437,22 +418,24 @@ static int ompi_osc_rdma_initialize_region (ompi_osc_rdma_module_t *module, void
                 return OMPI_ERR_OUT_OF_RESOURCE;
             }
 
-            memcpy (region->btl_handle_data, module->base_handle, module->selected_btls[0]->btl_registration_handle_size);
+            memcpy (region->btl_handle_data, module->base_handle, module->accelerated_btl->btl_registration_handle_size);
         } else {
-            memcpy (region->btl_handle_data, module->state_handle, module->selected_btls[0]->btl_registration_handle_size);
+            memcpy (region->btl_handle_data, module->state_handle, module->accelerated_btl->btl_registration_handle_size);
         }
     }
 
     return OMPI_SUCCESS;
 }
 
-static int allocate_state_single (ompi_osc_rdma_module_t *module, void **base, size_t size)
+static int allocate_state_single (ompi_osc_rdma_module_t *module, void **base, size_t size, bool use_cpu_atomics)
 {
-    size_t total_size, local_rank_array_size, leader_peer_data_size;
+    size_t total_size, local_rank_array_size, leader_peer_data_size, base_data_size;
     ompi_osc_rdma_peer_t *my_peer;
     int ret, my_rank;
+    size_t memory_alignment = module->memory_alignment;
 
-    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "allocating private internal state");
+    opal_output_verbose(MCA_BASE_VERBOSE_TRACE, ompi_osc_base_framework.framework_output,
+                        "allocating private internal state");
 
     my_rank = ompi_comm_rank (module->comm);
 
@@ -463,32 +446,35 @@ static int allocate_state_single (ompi_osc_rdma_module_t *module, void **base, s
      * registration handles needed to access this data. */
     total_size = local_rank_array_size + module->region_size +
         module->state_size + leader_peer_data_size;
-    total_size += OPAL_ALIGN_PAD_AMOUNT(total_size, OPAL_ALIGN_MIN);
+    base_data_size = total_size;
 
     if (MPI_WIN_FLAVOR_ALLOCATE == module->flavor) {
-        total_size += size;
+        base_data_size += OPAL_ALIGN_PAD_AMOUNT(base_data_size, memory_alignment);
+        total_size = base_data_size + size;
     }
 
     /* the local data is ordered as follows: rank array (leader, offset mapping), state, leader peer data, and base
      * (if using MPI_Win_allocate). In this case the leader peer data array does not need to be stored in the same
      * segment but placing it there simplifies the peer data fetch and cleanup code. */
 
-    module->rank_array = calloc (total_size, 1);
+    module->rank_array = mca_mpool_base_default_module->mpool_alloc(mca_mpool_base_default_module, total_size,
+                                                                    memory_alignment, 0);
     if (OPAL_UNLIKELY(NULL == module->rank_array)) {
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
+    memset(module->rank_array, 0, total_size);
 
-// Note, the extra module->region_size space added after local_rank_array_size
-// is unused but is there to match what happens in allocte_state_shared()
-// This allows module->state_offset to be uniform across the ranks which
-// is part of how they pull peer info from each other.
+    /* Note, the extra module->region_size space added after local_rank_array_size
+     * is unused but is there to match what happens in allocte_state_shared()
+     * This allows module->state_offset to be uniform across the ranks which
+     * is part of how they pull peer info from each other. */
     module->state_offset = local_rank_array_size + module->region_size;
 
     module->state = (ompi_osc_rdma_state_t *) ((intptr_t) module->rank_array + module->state_offset);
     module->node_comm_info = (unsigned char *) ((intptr_t) module->state + module->state_size);
 
     if (MPI_WIN_FLAVOR_ALLOCATE == module->flavor) {
-        *base = (void *) ((intptr_t) module->node_comm_info + leader_peer_data_size);
+        *base = (void *) ((intptr_t) module->rank_array + base_data_size);
     }
 
     /* just go ahead and register the whole segment */
@@ -521,7 +507,7 @@ static int allocate_state_single (ompi_osc_rdma_module_t *module, void **base, s
     my_peer->flags |= OMPI_OSC_RDMA_PEER_LOCAL_BASE;
     my_peer->state = (uint64_t) (uintptr_t) module->state;
 
-    if (module->use_cpu_atomics) {
+    if (use_cpu_atomics) {
         /* all peers are local or it is safe to mix cpu and nic atomics */
         my_peer->flags |= OMPI_OSC_RDMA_PEER_LOCAL_STATE;
     } else {
@@ -540,7 +526,7 @@ static int allocate_state_single (ompi_osc_rdma_module_t *module, void **base, s
             ex_peer->size = size;
         }
 
-        if (!module->use_cpu_atomics) {
+        if (!use_cpu_atomics) {
             if (MPI_WIN_FLAVOR_ALLOCATE == module->flavor) {
                 /* base is local and cpu atomics are available */
                 ex_peer->super.base_handle = module->state_handle;
@@ -583,7 +569,8 @@ static int allocate_state_shared (ompi_osc_rdma_module_t *module, void **base, s
     ompi_osc_rdma_region_t *state_region;
     struct _local_data *temp;
     char *data_file;
-    int page_size = opal_getpagesize();
+    size_t memory_alignment = module->memory_alignment;
+    bool use_cpu_atomics;
 
     shared_comm = module->shared_comm;
 
@@ -592,24 +579,30 @@ static int allocate_state_shared (ompi_osc_rdma_module_t *module, void **base, s
 
     /* CPU atomics can be used if every process is on the same node or the NIC allows mixing CPU and NIC atomics */
     module->single_node     = local_size == global_size;
-    module->use_cpu_atomics = module->single_node;
 
-    if (!module->single_node) {
-        for (int i = 0 ; i < module->btls_in_use ; ++i) {
-            module->use_cpu_atomics = module->use_cpu_atomics && !!(module->selected_btls[i]->btl_flags & MCA_BTL_ATOMIC_SUPPORTS_GLOB);
-        }
+    if (module->single_node) {
+        use_cpu_atomics = true;
+    } else if (module->use_accelerated_btl) {
+        use_cpu_atomics = !!(module->accelerated_btl->btl_flags & MCA_BTL_ATOMIC_SUPPORTS_GLOB);
+    } else {
+        /* using the shared state optimization that is enabled by
+         * being able to use cpu atomics was never enabled for
+         * alternate btls, due to a previous bug in the enablement
+         * logic when alternate btls were first supported.  It is
+         * likely that this optimization could work with sufficient
+         * testing, but for now, always disable to not introduce new
+         * correctness risks.
+         */
+        use_cpu_atomics = false;
     }
 
     if (1 == local_size) {
         /* no point using a shared segment if there are no other processes on this node */
-        return allocate_state_single (module, base, size);
+        return allocate_state_single (module, base, size, use_cpu_atomics);
     }
 
-    if (local_size == global_size) {
-      module->use_memory_registration = false;
-    }
-
-    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "allocating shared internal state");
+    opal_output_verbose(MCA_BASE_VERBOSE_TRACE, ompi_osc_base_framework.framework_output,
+                        "allocating shared internal state");
 
     local_rank_array_size = sizeof (ompi_osc_rdma_rank_data_t) * RANK_ARRAY_COUNT (module);
     leader_peer_data_size = module->region_size * module->node_count;
@@ -620,8 +613,8 @@ static int allocate_state_shared (ompi_osc_rdma_module_t *module, void **base, s
 
     /* ensure proper alignment */
     if (MPI_WIN_FLAVOR_ALLOCATE == module->flavor) {
-        data_base += OPAL_ALIGN_PAD_AMOUNT(data_base, page_size);
-        size += OPAL_ALIGN_PAD_AMOUNT(size, page_size);
+        data_base += OPAL_ALIGN_PAD_AMOUNT(data_base, memory_alignment);
+        size += OPAL_ALIGN_PAD_AMOUNT(size, memory_alignment);
     }
 
     do {
@@ -649,23 +642,25 @@ static int allocate_state_shared (ompi_osc_rdma_module_t *module, void **base, s
                     my_base_offset = total_size;
                 }
                 total_size += temp[i].size;
+                total_size += OPAL_ALIGN_PAD_AMOUNT(total_size, memory_alignment);
             }
         }
 
         if (0 == local_rank) {
             /* allocate the shared memory segment */
-            ret = opal_asprintf (&data_file, "%s" OPAL_PATH_SEP "osc_rdma.%s.%x.%d",
+            ret = opal_asprintf (&data_file, "%s" OPAL_PATH_SEP "osc_rdma.%s.%x.%s.%d",
                             mca_osc_rdma_component.backing_directory, ompi_process_info.nodename,
-                            OMPI_PROC_MY_NAME->jobid, ompi_comm_get_cid(module->comm));
+                            OMPI_PROC_MY_NAME->jobid, ompi_comm_print_cid(module->comm), getpid());
             if (0 > ret) {
                 ret = OMPI_ERR_OUT_OF_RESOURCE;
             } else {
-              /* allocate enough space for the state + data for all local ranks */
-              ret = opal_shmem_segment_create (&module->seg_ds, data_file, total_size);
-              free (data_file);
-              if (OPAL_SUCCESS != ret) {
-                  OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_ERROR, "failed to create shared memory segment");
-              }
+                /* allocate enough space for the state + data for all local ranks */
+                ret = opal_shmem_segment_create (&module->seg_ds, data_file, total_size);
+                free (data_file);
+                if (OPAL_SUCCESS != ret) {
+                    opal_output_verbose(MCA_BASE_VERBOSE_ERROR, ompi_osc_base_framework.framework_output,
+                                        "failed to create shared memory segment");
+                }
             }
         }
 
@@ -682,7 +677,8 @@ static int allocate_state_shared (ompi_osc_rdma_module_t *module, void **base, s
 
         module->segment_base = opal_shmem_segment_attach (&module->seg_ds);
         if (NULL == module->segment_base) {
-            OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_ERROR, "failed to attach to the shared memory segment");
+            opal_output_verbose(MCA_BASE_VERBOSE_ERROR, ompi_osc_base_framework.framework_output,
+                                "failed to attach to the shared memory segment");
             ret = OPAL_ERROR;
         }
 
@@ -692,6 +688,7 @@ static int allocate_state_shared (ompi_osc_rdma_module_t *module, void **base, s
         }
 
         if (size && MPI_WIN_FLAVOR_ALLOCATE == module->flavor) {
+            size_t page_size = opal_getpagesize();
             char *baseptr = (char *)((intptr_t) module->segment_base + my_base_offset);
             *base = (void *)baseptr;
             // touch each page to force allocation on local NUMA node
@@ -717,14 +714,17 @@ static int allocate_state_shared (ompi_osc_rdma_module_t *module, void **base, s
         if (0 == local_rank) {
             /* unlink the shared memory backing file */
             opal_shmem_unlink (&module->seg_ds);
-            /* just go ahead and register the whole segment */
-            ret = ompi_osc_rdma_register (module, MCA_BTL_ENDPOINT_ANY, module->segment_base, total_size,
-                                          MCA_BTL_REG_FLAG_ACCESS_ANY, &module->state_handle);
-            if (OPAL_LIKELY(OMPI_SUCCESS == ret)) {
-                state_region->base = (intptr_t) module->segment_base;
-                if (module->state_handle) {
-                    memcpy (state_region->btl_handle_data, module->state_handle,
-                            module->selected_btls[0]->btl_registration_handle_size);
+
+            state_region->base = (intptr_t) module->segment_base;
+            if (module->use_accelerated_btl) {
+                /* just go ahead and register the whole segment */
+                ret = ompi_osc_rdma_register(module, MCA_BTL_ENDPOINT_ANY, module->segment_base, total_size,
+                                             MCA_BTL_REG_FLAG_ACCESS_ANY, &module->state_handle);
+                if (OPAL_LIKELY(OMPI_SUCCESS == ret)) {
+                    if (module->state_handle) {
+                        memcpy(state_region->btl_handle_data, module->state_handle,
+                               module->accelerated_btl->btl_registration_handle_size);
+                    }
                 }
             }
         }
@@ -744,8 +744,9 @@ static int allocate_state_shared (ompi_osc_rdma_module_t *module, void **base, s
             region->base = state_region->base + my_base_offset;
             region->len = size;
             if (module->use_memory_registration) {
-                memcpy (region->btl_handle_data, state_region->btl_handle_data,
-                        module->selected_btls[0]->btl_registration_handle_size);
+                assert(module->use_accelerated_btl);
+                memcpy(region->btl_handle_data, state_region->btl_handle_data,
+                       module->accelerated_btl->btl_registration_handle_size);
             }
         }
 
@@ -777,7 +778,7 @@ static int allocate_state_shared (ompi_osc_rdma_module_t *module, void **base, s
             ex_peer = (ompi_osc_rdma_peer_extended_t *) peer;
 
             /* set up peer state */
-            if (module->use_cpu_atomics) {
+            if (use_cpu_atomics) {
                 /* all peers are local or it is safe to mix cpu and nic atomics */
                 peer->flags |= OMPI_OSC_RDMA_PEER_LOCAL_STATE;
                 peer->state = (osc_rdma_counter_t) peer_state;
@@ -788,18 +789,21 @@ static int allocate_state_shared (ompi_osc_rdma_module_t *module, void **base, s
                     peer->state_handle = (mca_btl_base_registration_handle_t *) state_region->btl_handle_data;
                 }
                 peer->state = (osc_rdma_counter_t) ((uintptr_t) state_region->base + state_base + module->state_size * i);
-                if (i > 0) {
+                if (i==0) {
+                    peer->state_endpoint = peer->data_endpoint;
+                    peer->state_btl_index = peer->data_btl_index;
+                } else {
                     peer->state_endpoint = local_leader->state_endpoint;
                     peer->state_btl_index = local_leader->state_btl_index;
                 }
             }
 
             if (my_rank == peer_rank) {
-	        module->my_peer = peer;
+                module->my_peer = peer;
             }
 
             if (MPI_WIN_FLAVOR_DYNAMIC != module->flavor && MPI_WIN_FLAVOR_CREATE != module->flavor &&
-                !module->use_cpu_atomics && temp[i].size && i > 0) {
+                !use_cpu_atomics && temp[i].size && i > 0) {
                 /* use the local leader's endpoint */
                 peer->data_endpoint = local_leader->data_endpoint;
                 peer->data_btl_index = local_leader->data_btl_index;
@@ -808,7 +812,7 @@ static int allocate_state_shared (ompi_osc_rdma_module_t *module, void **base, s
             ompi_osc_module_add_peer (module, peer);
 
             if (MPI_WIN_FLAVOR_DYNAMIC == module->flavor) {
-                if (module->use_cpu_atomics && peer_rank == my_rank) {
+                if (use_cpu_atomics && peer_rank == my_rank) {
                     peer->flags |= OMPI_OSC_RDMA_PEER_LOCAL_BASE;
                 }
                 /* nothing more to do */
@@ -824,7 +828,7 @@ static int allocate_state_shared (ompi_osc_rdma_module_t *module, void **base, s
                 ex_peer->size = temp[i].size;
             }
 
-            if (module->use_cpu_atomics && (MPI_WIN_FLAVOR_ALLOCATE == module->flavor || peer_rank == my_rank)) {
+            if (use_cpu_atomics && (MPI_WIN_FLAVOR_ALLOCATE == module->flavor || peer_rank == my_rank)) {
                 /* base is local and cpu atomics are available */
                 if (MPI_WIN_FLAVOR_ALLOCATE == module->flavor) {
                     ex_peer->super.base = (uintptr_t) module->segment_base + offset;
@@ -849,23 +853,6 @@ static int allocate_state_shared (ompi_osc_rdma_module_t *module, void **base, s
     return ret;
 }
 
-static int ompi_osc_rdma_query_mtls (void)
-{
-    char **mtls_to_use;
-
-    mtls_to_use = opal_argv_split (ompi_osc_rdma_mtl_names, ',');
-    if (mtls_to_use && ompi_mtl_base_selected_component) {
-	for (int i = 0 ; mtls_to_use[i] ; ++i) {
-	    if (0 == strcmp (mtls_to_use[i], ompi_mtl_base_selected_component->mtl_version.mca_component_name)) {
-                opal_argv_free(mtls_to_use);
-		return OMPI_SUCCESS;
-	    }
-	}
-    }
-    opal_argv_free(mtls_to_use);
-    return -1;
-}
-
 /**
  * @brief ensure that all local procs are added to the bml
  *
@@ -894,93 +881,184 @@ static void ompi_osc_rdma_ensure_local_add_procs (void)
     free(procs);
 }
 
+
+/*
+ * qsort() sorting function for ompi_osc_rdma_query_alternate_btls(),
+ * using latency as the sorting metric.
+ */
+static int btl_latency_sort_fn(const void *a, const void *b)
+{
+    const mca_btl_base_am_rdma_module_t * const *am_rdma_a_p = a;
+    const mca_btl_base_am_rdma_module_t * const *am_rdma_b_p = b;
+    const mca_btl_base_am_rdma_module_t *am_rdma_a = *am_rdma_a_p;
+    const mca_btl_base_am_rdma_module_t *am_rdma_b = *am_rdma_b_p;
+
+    if (am_rdma_a->btl->btl_latency < am_rdma_b->btl->btl_latency) {
+        return -1;
+    } else if (am_rdma_a->btl->btl_latency == am_rdma_b->btl->btl_latency) {
+        return 0;
+    } else {
+        return 1;
+    }
+}
+
+
 /**
  * @brief query for alternate BTLs
  *
  * @in  comm   Communicator to query
- * @out module OSC module to store BTLs/count to (optional)
- * @out 
+ * @inout module OSC module to store BTLs/count to (optional)
  *
  * @return OMPI_SUCCESS if BTLs can be found
  * @return OMPI_ERR_UNREACH if no BTLs can be found that match
  *
- * In this case an "alternate" BTL is a BTL that does not provide true RDMA but
- * can use active messages using the BTL base AM RDMA/atomics. Since more than
- * one BTL may be needed for this support the OSC component will disable the
- * use of registration-based RDMA (these BTLs will not be used) and will use
- * any remaining BTL. By default the BTLs used will be tcp and sm but any single
- * (or pair) of BTLs may be used.
+ * We directly use the active message rdma wrappers for alternate
+ * BTLs, in all cases.  This greatly simplifies the alternate BTL
+ * implementation, at the expense of some performance.  With the
+ * AM wrappers, we can always enforce remote completion and the lack
+ * of memory registration, at some performance cost.  But we can use
+ * as many BTLs as we like.  The module's btl list is sorted by
+ * latency, so that ompi_osc_rdma_peer_btl_endpoint() picks the lowest
+ * available latency btl to communicate with the peer.  Unlike the OB1
+ * PML, we only use one BTL per peer.
+ *
+ * Like the OB1 PML, there is no verification that there is at least
+ * one BTL that can communicate with every other peer in the window.
  */
 static int ompi_osc_rdma_query_alternate_btls (ompi_communicator_t *comm, ompi_osc_rdma_module_t *module)
 {
+    size_t btl_count;
+    size_t index = 0;
     mca_btl_base_selected_module_t *item;
-    char **btls_to_use = opal_argv_split (ompi_osc_rdma_btl_alternate_names, ',');
-    int btls_found = 0;
-    void *tmp;
-
-    btls_to_use = opal_argv_split (ompi_osc_rdma_btl_alternate_names, ',');
-    if (NULL == btls_to_use) {
-        OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_INFO, "no alternate BTLs requested: %s", ompi_osc_rdma_btl_alternate_names);
-        return OMPI_ERR_UNREACH;
-    }
-
-    if (module) {
-        module->btls_in_use = 0;
-    }
-
-    /* rdma and atomics are only supported with BTLs at the moment */
-    for (int i = 0 ; btls_to_use[i] ; ++i) {
-        OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_INFO, "checking for btl %s", btls_to_use[i]);
-        OPAL_LIST_FOREACH(item, &mca_btl_base_modules_initialized, mca_btl_base_selected_module_t) {
-            if (NULL != item->btl_module->btl_register_mem) { 
-                OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_INFO, "skipping RDMA btl when searching for alternate BTL");
-                continue;
-            }
+    int ret;
 
-            if (0 != strcmp (btls_to_use[i], item->btl_module->btl_component->btl_version.mca_component_name)) {
-                OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_INFO, "skipping btl %s",
-                                 item->btl_module->btl_component->btl_version.mca_component_name);
-                continue;
-            }
+    assert(NULL != module);
+
+    module->put_alignment = 1;
+    module->get_alignment = 1;
+    module->put_limit = SIZE_MAX;
+    module->get_limit = SIZE_MAX;
+
+    btl_count = opal_list_get_size(&mca_btl_base_modules_initialized);
+    if (btl_count > UINT8_MAX) {
+        return OMPI_ERROR;
+    }
+
+    module->alternate_btl_count = btl_count;
+    module->alternate_am_rdmas = malloc(sizeof(struct mca_btl_base_am_rdma_module_t *) * module->alternate_btl_count);
+    if (NULL == module->alternate_am_rdmas) {
+        return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
+    }
+
+    /* add all alternate btls to the selected_btls list, not worrying
+       about ordering yet.  We have to add all btls unless we want to
+       iterate over all endpoints to build the minimum set of btls
+       needed to communicate with all peers.  An MCA parameter just
+       for osc rdma also wouldn't work, as the BML can decide not to
+       add an endpoint for a btl given the priority of another btl.
+       For example, it is not uncommon that the only endpoint created
+       to a peer on the same host is the sm btl's endpoint.  If we
+       had an osc rdma specific parameter list, and the user
+       specified a combination not including sm, that would result in
+       an eventual failure, as no btl would be found to talk to ranks
+       on the same host.*/
+    OPAL_LIST_FOREACH(item, &mca_btl_base_modules_initialized, mca_btl_base_selected_module_t) {
+        opal_output_verbose(MCA_BASE_VERBOSE_INFO, ompi_osc_base_framework.framework_output,
+                            "found alternate btl %s",
+                            item->btl_module->btl_component->btl_version.mca_component_name);
+
+        ret = opal_btl_base_am_rdma_create(item->btl_module,
+                                           MCA_BTL_FLAGS_RDMA_REMOTE_COMPLETION,
+                                           true /* no_memory_registration */,
+                                           &(module->alternate_am_rdmas[index]));
+        if (OMPI_SUCCESS != ret) {
+            return ret;
+        }
 
-            OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_INFO, "found alternate btl %s", btls_to_use[i]);
+        module->put_alignment = opal_max(module->put_alignment,
+                                         module->alternate_am_rdmas[index]->am_btl_put_alignment);
+        module->get_alignment = opal_max(module->get_alignment,
+                                         module->alternate_am_rdmas[index]->am_btl_get_alignment);
+        module->put_limit = opal_min(module->put_limit,
+                                     module->alternate_am_rdmas[index]->am_btl_put_limit);
+        module->get_limit = opal_min(module->get_limit,
+                                     module->alternate_am_rdmas[index]->am_btl_get_limit);
+
+        index++;
+    }
+    assert(index == module->alternate_btl_count);
+
+    /* sort based on latency, lowest first */
+    qsort(module->alternate_am_rdmas, module->alternate_btl_count,
+          sizeof(module->alternate_am_rdmas[0]), btl_latency_sort_fn);
+
+    module->use_memory_registration = false;
+    module->atomic_flags = MCA_BTL_ATOMIC_SUPPORTS_ADD |
+            MCA_BTL_ATOMIC_SUPPORTS_AND |
+            MCA_BTL_ATOMIC_SUPPORTS_OR |
+            MCA_BTL_ATOMIC_SUPPORTS_XOR |
+            MCA_BTL_ATOMIC_SUPPORTS_SWAP |
+            MCA_BTL_ATOMIC_SUPPORTS_MIN |
+            MCA_BTL_ATOMIC_SUPPORTS_MAX |
+            MCA_BTL_ATOMIC_SUPPORTS_32BIT |
+            MCA_BTL_ATOMIC_SUPPORTS_CSWAP |
+            MCA_BTL_ATOMIC_SUPPORTS_GLOB;
 
-            ++btls_found;
-            if (module) {
-                mca_btl_base_am_rdma_init(item->btl_module);
-                module->selected_btls[module->btls_in_use++] = item->btl_module;
-            }
-            
-        }
-    }
+    return OMPI_SUCCESS;
+}
 
-    opal_argv_free (btls_to_use);
 
-    return btls_found > 0 ? OMPI_SUCCESS : OMPI_ERR_UNREACH;
+/* Check for BTL requirements:
+ *  1) RDMA (put/get) and ATOMIC operations.  We only require cswap
+ *     and fetch and add and will emulate other opterations with those
+ *     two as necessary.
+ *  2) Remote Completion
+ */
+static bool ompi_osc_rdma_check_accelerated_btl(struct mca_btl_base_module_t *btl)
+{
+    return ((btl->btl_flags & MCA_BTL_FLAGS_RDMA) &&
+            (btl->btl_flags & MCA_BTL_FLAGS_ATOMIC_FOPS) &&
+            (btl->btl_flags & MCA_BTL_FLAGS_RDMA_REMOTE_COMPLETION) &&
+            (btl->btl_atomic_flags & MCA_BTL_ATOMIC_SUPPORTS_ADD));
 }
 
-static int ompi_osc_rdma_query_btls (ompi_communicator_t *comm, ompi_osc_rdma_module_t *module)
+/*
+ * Attempt to find a BTL that can be used for native RDMA
+ *
+ * Attempt to find an "accelerated" BTL that can be used directly, as
+ * opposed to emulated rdma semantics with the alternate BTLs.  To be
+ * an accelerated BTL, four conditions must be true:
+ *
+ *  1) The BTL must be able to communicate with all peers in the
+ *     Window
+ *  2) The BTL must provide remote completion
+ *  3) The BTL must be able to register the entire target window
+ *  4) The BTL must natively support put/get/atomic operations
+ *
+ * Testing (1) is expensive, so as an optimization, the
+ * ompi_osc_rdma_full_connectivity_btls list contains the list of BTL
+ * components we know can achieve (1) in almost all usage scenarios.
+ */
+static int ompi_osc_rdma_query_accelerated_btls (ompi_communicator_t *comm, ompi_osc_rdma_module_t *module)
 {
-    struct mca_btl_base_module_t **possible_btls = NULL;
     int comm_size = ompi_comm_size (comm);
-    int comm_rank = ompi_comm_rank (comm);
-    int rc = OMPI_SUCCESS, max_btls = 0;
-    unsigned int selected_latency = INT_MAX;
-    struct mca_btl_base_module_t *selected_btl = NULL;
-    mca_btl_base_selected_module_t *item;
-    int *btl_counts = NULL;
+    struct mca_btl_base_module_t *selected_btl;
+    mca_bml_base_endpoint_t *base_endpoint;
     char **btls_to_use;
-    void *tmp;
 
-    btls_to_use = opal_argv_split (ompi_osc_rdma_btl_names, ',');
+    assert(NULL != module);
 
-    if (module) {
-        module->selected_btls[0] = NULL;
-        module->btls_in_use = 0;
-        module->use_memory_registration = false;
-    }
+    module->use_accelerated_btl = false;
+    module->use_memory_registration = false;
 
+    /* Check for BTLs in the list of BTLs we know can reach all peers
+       in general usage. */
+    btls_to_use = opal_argv_split (ompi_osc_rdma_full_connectivity_btls, ',');
     if (btls_to_use) {
+        mca_btl_base_selected_module_t *item;
+
+        selected_btl = NULL;
+
         /* rdma and atomics are only supported with BTLs at the moment */
         OPAL_LIST_FOREACH(item, &mca_btl_base_modules_initialized, mca_btl_base_selected_module_t) {
             for (int i = 0 ; btls_to_use[i] ; ++i) {
@@ -988,9 +1066,8 @@ static int ompi_osc_rdma_query_btls (ompi_communicator_t *comm, ompi_osc_rdma_mo
                     continue;
                 }
 
-                if ((item->btl_module->btl_flags & (MCA_BTL_FLAGS_RDMA)) == MCA_BTL_FLAGS_RDMA &&
-                    (item->btl_module->btl_flags & (MCA_BTL_FLAGS_ATOMIC_FOPS | MCA_BTL_FLAGS_ATOMIC_OPS))) {
-                    if (!selected_btl || item->btl_module->btl_latency < selected_btl->btl_latency) {
+                if (ompi_osc_rdma_check_accelerated_btl(item->btl_module)) {
+                    if (NULL == selected_btl || item->btl_module->btl_latency < selected_btl->btl_latency) {
                         selected_btl = item->btl_module;
                     }
                 }
@@ -998,134 +1075,105 @@ static int ompi_osc_rdma_query_btls (ompi_communicator_t *comm, ompi_osc_rdma_mo
         }
 
         opal_argv_free (btls_to_use);
-    }
 
-    if (NULL != selected_btl) {
-        if (module) {
-            module->selected_btls[0] = selected_btl;
-            module->btls_in_use = 1;
-            module->use_memory_registration = selected_btl->btl_register_mem != NULL;
+        if (NULL != selected_btl) {
+            goto btl_selection_complete;
         }
-
-        OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_INFO, "selected btl: %s",
-                         selected_btl->btl_component->btl_version.mca_component_name);
-        return OMPI_SUCCESS;
     }
 
     /* if osc/rdma gets selected we need to ensure that all local procs have been added */
     ompi_osc_rdma_ensure_local_add_procs ();
-    
-    for (int rank = 0 ; rank < comm_size ; ++rank) {
-        ompi_proc_t *proc = ompi_comm_peer_lookup (comm, rank);
-        mca_bml_base_endpoint_t *endpoint;
-        int num_btls, prev_max;
-        bool found_btl = false;
-
-        endpoint = mca_bml_base_get_endpoint (proc);
-        if (NULL == endpoint) {
-            /* can't continue if some peer is unreachable */
-            rc = OMPI_ERR_UNREACH;
-            break;
-        }
-
-        num_btls = mca_bml_base_btl_array_get_size (&endpoint->btl_rdma);
-        if (0 == num_btls) {
-            rc = OMPI_ERR_NOT_AVAILABLE;
-            /* at least one rank doesn't have an RDMA capable btl */
-            break;
-        }
-
-        prev_max = max_btls;
-
-        max_btls = (max_btls > num_btls) ? max_btls : num_btls;
 
-        tmp = realloc (possible_btls, sizeof (void *) * max_btls);
-        if (NULL == tmp) {
-            rc = OMPI_ERR_OUT_OF_RESOURCE;
-            break;
-        }
-        possible_btls = tmp;
+    /*
+     * A BTL in the list of known can reach all peers that met our
+     * other requirements was not found.  Look for BTLs that may be
+     * able to talk to all peers.  This is obviously more expensive
+     * than the check above.
+     *
+     * This algorithm skips a potential use case: it requires
+     * reachability to self, which is not strictly needed if BTL and
+     * CPU atomics are atomic with each other.  However, the set of
+     * BTLs which can not send to self, which have RDMA semantics, an
+     * which have the required atomicity is currently the null set and
+     * almost certain to remain the null set, so we keep it simple.
+     *
+     * We only want BTLs that can reach all peers, so use rank 0's BTL
+     * list as the list of all available BTLs.  Any BTL that cannot
+     * be used to communicate with rank 0 necessarily is not in the
+     * list of all available BTLs for this algorithm.
+     */
+    base_endpoint = mca_bml_base_get_endpoint(ompi_comm_peer_lookup(comm, 0));
+    if (NULL == base_endpoint) {
+        return OMPI_ERR_UNREACH;
+    }
 
-        for (int j = prev_max ; j < max_btls ; ++j) {
-            possible_btls[j] = NULL;
-        }
+    selected_btl = NULL;
+    for (size_t i_btl = 0 ;
+         i_btl < mca_bml_base_btl_array_get_size(&base_endpoint->btl_rdma);
+         ++i_btl) {
+        bool have_connectivity = true;
+        struct mca_bml_base_btl_t *examine_bml_btl;
+        struct mca_btl_base_module_t *examine_btl;
 
-        tmp = realloc (btl_counts, sizeof (int) * max_btls);
-        if (NULL == tmp) {
-            rc = OMPI_ERR_OUT_OF_RESOURCE;
-            break;
-        }
-        btl_counts = tmp;
-
-        for (int i_btl = 0 ; i_btl < num_btls ; ++i_btl) {
-            /* for this implementation we need only compare-and-swap and fetch-and-add */
-            if ((endpoint->btl_rdma.bml_btls[i_btl].btl->btl_flags & (MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_ATOMIC_FOPS)) ==
-                (MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_ATOMIC_FOPS) && (endpoint->btl_rdma.bml_btls[i_btl].btl->btl_atomic_flags &
-                                                                     MCA_BTL_ATOMIC_SUPPORTS_ADD)) {
-                for (int j = 0 ; j < max_btls ; ++j) {
-                    if (endpoint->btl_rdma.bml_btls[i_btl].btl == possible_btls[j]) {
-                        ++btl_counts[j];
-                        found_btl = true;
-                        break;
-                    } else if (NULL == possible_btls[j]) {
-                        possible_btls[j] = endpoint->btl_rdma.bml_btls[i_btl].btl;
-                        btl_counts[j] = 1;
-                        found_btl = true;
-                        break;
-                    }
-                }
-            }
+        examine_bml_btl = mca_bml_base_btl_array_get_index(&base_endpoint->btl_rdma, i_btl);
+        if (NULL == examine_bml_btl) {
+            return OMPI_ERR_NOT_FOUND;
         }
+        examine_btl = examine_bml_btl->btl;
 
-        /* any non-local rank must have a usable btl */
-        if (!found_btl && comm_rank != rank) {
-            /* no btl = no rdma/atomics */
-            rc = OMPI_ERR_UNREACH;
-            break;
+        /* skip any BTL which doesn't meet our requirements */
+        if (!ompi_osc_rdma_check_accelerated_btl(examine_btl)) {
+            continue;
         }
-    }
-
-    if (OMPI_SUCCESS != rc) {
-        free (possible_btls);
-        free (btl_counts);
-        return rc;
-    }
 
-    for (int i = 0 ; i < max_btls ; ++i) {
-        int btl_count = btl_counts[i];
+        /* check connectivity across all ranks */
+        for (int rank = 0 ; rank < comm_size ; ++rank) {
+            ompi_proc_t *proc = ompi_comm_peer_lookup(comm, rank);
+            mca_bml_base_endpoint_t *endpoint;
 
-        if (NULL == possible_btls[i]) {
-            break;
-        }
+            endpoint = mca_bml_base_get_endpoint(proc);
+            if (NULL == endpoint) {
+                have_connectivity = false;
+                break;
+            }
 
-        if (possible_btls[i]->btl_atomic_flags & MCA_BTL_ATOMIC_SUPPORTS_GLOB) {
-            /* do not need to use the btl for self communication */
-            btl_count++;
+            if (NULL == mca_bml_base_btl_array_find(&endpoint->btl_rdma,
+                                                    examine_btl)) {
+                have_connectivity = false;
+                break;
+            }
         }
 
-        if (btl_count >= comm_size && possible_btls[i]->btl_latency < selected_latency) {
-            selected_btl = possible_btls[i];
-            selected_latency = possible_btls[i]->btl_latency;
+        /* if we have connectivity, displace currently selected btl if
+         * this one has lower latency; we prioritize latency over all
+         * other parameters
+         */
+        if (have_connectivity) {
+            if (NULL == selected_btl || examine_btl->btl_latency < selected_btl->btl_latency) {
+                selected_btl = examine_btl;
+            }
         }
     }
 
-    free (possible_btls);
-    free (btl_counts);
-
     if (NULL == selected_btl) {
-        OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_INFO, "no suitable btls found");
-        /* no btl = no rdma/atomics */
+        opal_output_verbose(MCA_BASE_VERBOSE_INFO, ompi_osc_base_framework.framework_output,
+                            "accelerated_query: no suitable btls found");
         return OMPI_ERR_NOT_AVAILABLE;
     }
 
-    if (module) {
-        module->selected_btls[0] = selected_btl;
-        module->btls_in_use = 1;
-        module->use_memory_registration = selected_btl->btl_register_mem != NULL;
-    }
+btl_selection_complete:
+    module->use_accelerated_btl = true;
+    module->accelerated_btl = selected_btl;
+    module->use_memory_registration = (selected_btl->btl_register_mem != NULL);
+    module->put_alignment = selected_btl->btl_put_alignment;
+    module->get_alignment = selected_btl->btl_get_alignment;
+    module->put_limit = selected_btl->btl_put_limit;
+    module->get_limit = selected_btl->btl_get_limit;
+    module->atomic_flags = selected_btl->btl_atomic_flags;
 
-    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_INFO, "selected btl: %s",
-                     selected_btl->btl_component->btl_version.mca_component_name);
+    opal_output_verbose(MCA_BASE_VERBOSE_INFO, ompi_osc_base_framework.framework_output,
+                        "accelerated_query: selected btl: %s",
+                        selected_btl->btl_component->btl_version.mca_component_name);
 
     return OMPI_SUCCESS;
 }
@@ -1165,7 +1213,8 @@ static int ompi_osc_rdma_share_data (ompi_osc_rdma_module_t *module)
             my_data->len = (osc_rdma_size_t) my_rank;
 
             if (module->use_memory_registration && module->state_handle) {
-                memcpy (my_data->btl_handle_data, module->state_handle, module->selected_btls[0]->btl_registration_handle_size);
+                assert(module->use_accelerated_btl);
+                memcpy (my_data->btl_handle_data, module->state_handle, module->accelerated_btl->btl_registration_handle_size);
             }
 
             /* gather state data at each node leader */
@@ -1174,7 +1223,8 @@ static int ompi_osc_rdma_share_data (ompi_osc_rdma_module_t *module)
                                                                     module->region_size, MPI_BYTE, module->local_leaders,
                                                                     module->local_leaders->c_coll->coll_allgather_module);
                 if (OMPI_SUCCESS != ret) {
-                    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_ERROR, "leader allgather failed with ompi error code %d", ret);
+                    opal_output_verbose(MCA_BASE_VERBOSE_ERROR, ompi_osc_base_framework.framework_output,
+                                        "leader allgather failed with ompi error code %d", ret);
                     break;
                 }
             }
@@ -1217,7 +1267,8 @@ static int ompi_osc_rdma_create_groups (ompi_osc_rdma_module_t *module)
     /* create a shared communicator to handle communication about the local segment */
     ret = ompi_comm_split_type (module->comm, MPI_COMM_TYPE_SHARED, 0, NULL, &module->shared_comm);
     if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
-        OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_ERROR, "failed to create a shared memory communicator. error code %d", ret);
+        opal_output_verbose(MCA_BASE_VERBOSE_ERROR, ompi_osc_base_framework.framework_output,
+                            "failed to create a shared memory communicator. error code %d", ret);
         return ret;
     }
 
@@ -1228,7 +1279,8 @@ static int ompi_osc_rdma_create_groups (ompi_osc_rdma_module_t *module)
     ret = ompi_comm_split (module->comm, (0 == local_rank) ? 0 : MPI_UNDEFINED, comm_rank, &module->local_leaders,
                            false);
     if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
-        OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_ERROR, "failed to create local leaders communicator. error code %d", ret);
+        opal_output_verbose(MCA_BASE_VERBOSE_ERROR, ompi_osc_base_framework.framework_output,
+                            "failed to create local leaders communicator. error code %d", ret);
         return ret;
     }
 
@@ -1241,7 +1293,8 @@ static int ompi_osc_rdma_create_groups (ompi_osc_rdma_module_t *module)
         ret = module->shared_comm->c_coll->coll_bcast (values, 2, MPI_INT, 0, module->shared_comm,
                                                       module->shared_comm->c_coll->coll_bcast_module);
         if (OMPI_SUCCESS != ret) {
-            OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_ERROR, "failed to broadcast local data. error code %d", ret);
+            opal_output_verbose(MCA_BASE_VERBOSE_ERROR, ompi_osc_base_framework.framework_output,
+                                "failed to broadcast local data. error code %d", ret);
             return ret;
         }
     }
@@ -1262,7 +1315,7 @@ static int ompi_osc_rdma_create_groups (ompi_osc_rdma_module_t *module)
  * This function checks if all ranks have the same displacement unit or size and sets the appropriate
  * flags on the module.
  */
-static int ompi_osc_rdma_check_parameters (ompi_osc_rdma_module_t *module, int disp_unit, size_t size)
+static int ompi_osc_rdma_check_parameters (ompi_osc_rdma_module_t *module, ptrdiff_t disp_unit, size_t size)
 {
     long values[4];
     int ret;
@@ -1298,7 +1351,7 @@ static int ompi_osc_rdma_check_parameters (ompi_osc_rdma_module_t *module, int d
 }
 
 
-static int ompi_osc_rdma_component_select (struct ompi_win_t *win, void **base, size_t size, int disp_unit,
+static int ompi_osc_rdma_component_select (struct ompi_win_t *win, void **base, size_t size, ptrdiff_t disp_unit,
                                            struct ompi_communicator_t *comm, struct opal_info_t *info,
                                            int flavor, int *model)
 {
@@ -1341,6 +1394,10 @@ static int ompi_osc_rdma_component_select (struct ompi_win_t *win, void **base,
     module->win = win;
     module->disp_unit = disp_unit;
     module->size = size;
+    module->memory_alignment = mca_osc_rdma_component.memory_alignment;
+    if (NULL != info) {
+        ompi_osc_base_set_memory_alignment(info, &module->memory_alignment);
+    }
 
     /* set the module so we properly cleanup */
     win->w_osc_module = (ompi_osc_base_module_t*) module;
@@ -1367,8 +1424,9 @@ static int ompi_osc_rdma_component_select (struct ompi_win_t *win, void **base,
         return ret;
     }
 
-    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_INFO, "creating osc/rdma window of flavor %d with id %d",
-                     flavor, ompi_comm_get_cid(module->comm));
+    opal_output_verbose(MCA_BASE_VERBOSE_INFO, ompi_osc_base_framework.framework_output,
+                        "creating osc/rdma window of flavor %d with id %s",
+                        flavor, ompi_comm_print_cid (module->comm));
 
     /* peer data */
     if (world_size > init_limit) {
@@ -1387,13 +1445,16 @@ static int ompi_osc_rdma_component_select (struct ompi_win_t *win, void **base,
     }
 
     /* find rdma capable endpoints */
-    ret = ompi_osc_rdma_query_btls (module->comm, module);
+    module->use_accelerated_btl = false;
+    ret = ompi_osc_rdma_query_accelerated_btls (module->comm, module);
     if (OMPI_SUCCESS != ret) {
-        OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_WARN, "could not find a suitable btl. falling back on "
-                         "active-message BTLs");
+        opal_output_verbose(MCA_BASE_VERBOSE_WARN, ompi_osc_base_framework.framework_output,
+                            "could not find an accelerated btl. falling back on "
+                            "active-message BTLs");
         ret = ompi_osc_rdma_query_alternate_btls (module->comm, module);
         if (OMPI_SUCCESS != ret) {
-            OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_WARN, "no BTL available for RMA window");
+            opal_output_verbose(MCA_BASE_VERBOSE_WARN, ompi_osc_base_framework.framework_output,
+                                "no BTL available for RMA window");
             ompi_osc_rdma_free (win);
             return ret;
         }
@@ -1403,7 +1464,8 @@ static int ompi_osc_rdma_component_select (struct ompi_win_t *win, void **base,
 
     module->region_size = sizeof (ompi_osc_rdma_region_t);
     if (module->use_memory_registration) {
-        module->region_size += module->selected_btls[0]->btl_registration_handle_size;
+        assert(module->use_accelerated_btl);
+        module->region_size += module->accelerated_btl->btl_registration_handle_size;
     }
 
     module->state_size = sizeof (ompi_osc_rdma_state_t);
@@ -1413,15 +1475,16 @@ static int ompi_osc_rdma_component_select (struct ompi_win_t *win, void **base,
     } else {
         module->state_size += mca_osc_rdma_component.max_attach * module->region_size;
     }
-/*
- * These are the info's that this module is interested in
- */
+
+    /*
+     * These are the info's that this module is interested in
+     */
     opal_infosubscribe_subscribe(&win->super, "no_locks", "false", ompi_osc_rdma_set_no_lock_info);
 
-/*
- * TODO: same_size, same_disp_unit have w_flag entries, but do not appear
- * to be used anywhere.  If that changes, they should be subscribed
- */
+    /*
+     * TODO: same_size, same_disp_unit have w_flag entries, but do not appear
+     * to be used anywhere.  If that changes, they should be subscribed
+     */
 
     /* fill in the function pointer part */
     memcpy(&module->super, &ompi_osc_rdma_module_rdma_template, sizeof(module->super));
@@ -1444,7 +1507,8 @@ static int ompi_osc_rdma_component_select (struct ompi_win_t *win, void **base,
     /* notify all others if something went wrong */
     ret = synchronize_errorcode(ret, module->comm);
     if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
-        OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_ERROR, "failed to allocate internal state");
+        opal_output_verbose(MCA_BASE_VERBOSE_ERROR, ompi_osc_base_framework.framework_output,
+                            "failed to allocate internal state");
         ompi_osc_rdma_free (win);
         return ret;
     }
@@ -1475,7 +1539,7 @@ static int ompi_osc_rdma_component_select (struct ompi_win_t *win, void **base,
     /* update component data */
     OPAL_THREAD_LOCK(&mca_osc_rdma_component.lock);
     ret = opal_hash_table_set_value_uint32(&mca_osc_rdma_component.modules,
-                                           ompi_comm_get_cid(module->comm),
+                                           ompi_comm_get_local_cid(module->comm),
                                            module);
     OPAL_THREAD_UNLOCK(&mca_osc_rdma_component.lock);
     if (OMPI_SUCCESS != ret) {
@@ -1486,7 +1550,7 @@ static int ompi_osc_rdma_component_select (struct ompi_win_t *win, void **base,
     /* fill in window information */
     *model = MPI_WIN_UNIFIED;
     win->w_osc_module = (ompi_osc_base_module_t*) module;
-    opal_asprintf(&name, "rdma window %d", ompi_comm_get_cid(module->comm));
+    opal_asprintf(&name, "rdma window %s", ompi_comm_print_cid(module->comm));
     ompi_win_set_name(win, name);
     free(name);
 
@@ -1495,21 +1559,24 @@ static int ompi_osc_rdma_component_select (struct ompi_win_t *win, void **base,
 
     ret = ompi_osc_rdma_share_data (module);
     if (OMPI_SUCCESS != ret) {
-        OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_ERROR, "failed to share window data with peers");
+        opal_output_verbose(MCA_BASE_VERBOSE_ERROR, ompi_osc_base_framework.framework_output,
+                            "failed to share window data with peers");
         ompi_osc_rdma_free (win);
     } else {
         /* for now the leader is always rank 0 in the communicator */
         module->leader = ompi_osc_rdma_module_peer (module, 0);
 
-        OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_INFO, "finished creating osc/rdma window with id %d",
-                         ompi_comm_get_cid(module->comm));
+        opal_output_verbose(MCA_BASE_VERBOSE_INFO, ompi_osc_base_framework.framework_output,
+                            "finished creating osc/rdma window with id %s",
+                            ompi_comm_print_cid(module->comm));
     }
 
     return ret;
 }
 
 
-static char* ompi_osc_rdma_set_no_lock_info(opal_infosubscriber_t *obj, char *key, char *value)
+static const char*
+ompi_osc_rdma_set_no_lock_info(opal_infosubscriber_t *obj, const char *key, const char *value)
 {
 
     struct ompi_win_t *win = (struct ompi_win_t*) obj;
@@ -1538,8 +1605,8 @@ static char* ompi_osc_rdma_set_no_lock_info(opal_infosubscriber_t *obj, char *ke
     }
     /* enforce collectiveness... */
     module->comm->c_coll->coll_barrier(module->comm, module->comm->c_coll->coll_barrier_module);
-/*
- * Accept any value
- */
+    /*
+     * Accept any value
+     */
     return module->no_locks ? "true" : "false";
 }
diff --git a/ompi/mca/osc/rdma/osc_rdma_dynamic.c b/ompi/mca/osc/rdma/osc_rdma_dynamic.c
index 8adfa7f8159..358ff064238 100644
--- a/ompi/mca/osc/rdma/osc_rdma_dynamic.c
+++ b/ompi/mca/osc/rdma/osc_rdma_dynamic.c
@@ -172,6 +172,8 @@ int ompi_osc_rdma_attach (struct ompi_win_t *win, void *base, size_t len)
     intptr_t page_size = opal_getpagesize ();
     int region_index, ret;
     size_t aligned_len;
+    int dev_id;
+    uint64_t flags;
 
     if (module->flavor != MPI_WIN_FLAVOR_DYNAMIC) {
         return OMPI_ERR_RMA_FLAVOR;
@@ -182,6 +184,16 @@ int ompi_osc_rdma_attach (struct ompi_win_t *win, void *base, size_t len)
         return OMPI_SUCCESS;
     }
 
+    /* Component query check doesn't apply for dynamic window types.
+     * We have to check whether a GPU window is supported here. */
+    if (opal_accelerator.check_addr(base, &dev_id, &flags)) {
+        if (!osc_rdma_btl_accel_support(&mca_btl_base_modules_initialized)) {
+            OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "Failed to attach %p. No components capable of attaching accelerator buffers.",
+                             base);
+            return OMPI_ERR_NOT_SUPPORTED;
+        }
+    }
+
     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "attach: %s, %p, %lu", win->w_name, base, (unsigned long) len);
 
     OPAL_THREAD_LOCK(&module->lock);
@@ -214,7 +226,7 @@ int ompi_osc_rdma_attach (struct ompi_win_t *win, void *base, size_t len)
         return ret;
     }
 
-    /* do a binary seach for where the region should be inserted */
+    /* do a binary search for where the region should be inserted */
     if (region_count) {
         region = find_insertion_point ((ompi_osc_rdma_region_t *) module->state->regions, 0, region_count - 1,
                                        (intptr_t) base, module->region_size, &region_index);
@@ -252,7 +264,8 @@ int ompi_osc_rdma_attach (struct ompi_win_t *win, void *base, size_t len)
             return OMPI_ERR_RMA_ATTACH;
         }
 
-        memcpy (region->btl_handle_data, handle, module->selected_btls[0]->btl_registration_handle_size);
+        assert(module->use_accelerated_btl);
+        memcpy(region->btl_handle_data, handle, module->accelerated_btl->btl_registration_handle_size);
         rdma_region_handle->btl_handle = handle;
     } else {
         rdma_region_handle->btl_handle = NULL;
@@ -467,12 +480,9 @@ int ompi_osc_rdma_find_dynamic_region (ompi_osc_rdma_module_t *module, ompi_osc_
                      " (len %lu)", base, base + len, (unsigned long) len);
 
     OPAL_THREAD_LOCK(&module->lock);
-    // Make sure region isn't being touched.
-    ompi_osc_rdma_lock_acquire_exclusive (module, peer, offsetof (ompi_osc_rdma_state_t, regions_lock));
     if (!ompi_osc_rdma_peer_local_state (peer)) {
         ret = ompi_osc_rdma_refresh_dynamic_region (module, dy_peer);
         if (OMPI_SUCCESS != ret) {
-            ompi_osc_rdma_lock_release_exclusive (module, peer, offsetof (ompi_osc_rdma_state_t, regions_lock));
             return ret;
         }
 
@@ -489,7 +499,6 @@ int ompi_osc_rdma_find_dynamic_region (ompi_osc_rdma_module_t *module, ompi_osc_
         ret = OMPI_ERR_RMA_RANGE;
     }
     OPAL_THREAD_UNLOCK(&module->lock);
-    ompi_osc_rdma_lock_release_exclusive (module, peer, offsetof (ompi_osc_rdma_state_t, regions_lock));
 
     /* round a matching region */
     return ret;
diff --git a/ompi/mca/osc/rdma/osc_rdma_frag.h b/ompi/mca/osc/rdma/osc_rdma_frag.h
index 50cccdf1e3b..ead1a8adc43 100644
--- a/ompi/mca/osc/rdma/osc_rdma_frag.h
+++ b/ompi/mca/osc/rdma/osc_rdma_frag.h
@@ -25,11 +25,7 @@ static inline void ompi_osc_rdma_frag_complete (ompi_osc_rdma_frag_t *frag)
         opal_atomic_rmb ();
 
         (void) opal_atomic_swap_32 (&frag->pending, 1);
-#if OPAL_HAVE_ATOMIC_MATH_64
         (void) opal_atomic_swap_64 (&frag->curr_index, 0);
-#else
-        (void) opal_atomic_swap_32 (&frag->curr_index, 0);
-#endif
     }
 }
 
@@ -88,11 +84,7 @@ static inline int ompi_osc_rdma_frag_alloc (ompi_osc_rdma_module_t *module, size
     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_INFO, "allocating frag. pending = %d", curr->pending);
     OPAL_THREAD_ADD_FETCH32(&curr->pending, 1);
 
-#if OPAL_HAVE_ATOMIC_MATH_64
     my_index = opal_atomic_fetch_add_64 (&curr->curr_index, request_len);
-#else
-    my_index = opal_atomic_fetch_add_32 (&curr->curr_index, request_len);
-#endif
     if (my_index + request_len > mca_osc_rdma_component.buffer_size) {
         if (my_index <= mca_osc_rdma_component.buffer_size) {
             /* this thread caused the buffer to spill over */
diff --git a/ompi/mca/osc/rdma/osc_rdma_lock.h b/ompi/mca/osc/rdma/osc_rdma_lock.h
index 2664c876588..19a3249bdbe 100644
--- a/ompi/mca/osc/rdma/osc_rdma_lock.h
+++ b/ompi/mca/osc/rdma/osc_rdma_lock.h
@@ -5,6 +5,8 @@
  * Copyright (c) 2019      Triad National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2021      Google, LLC. All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -12,11 +14,12 @@
  * $HEADER$
  */
 
-#if !defined(OMPI_OSC_RDMA_LOCK_H)
+#ifndef OMPI_OSC_RDMA_LOCK_H
 #define OMPI_OSC_RDMA_LOCK_H
 
 #include "osc_rdma_types.h"
 #include "osc_rdma_frag.h"
+#include "osc_rdma_btl_comm.h"
 
 static inline int ompi_osc_rdma_trylock_local (ompi_osc_rdma_atomic_lock_t *lock)
 {
@@ -29,82 +32,6 @@ static inline void ompi_osc_rdma_unlock_local (ompi_osc_rdma_atomic_lock_t *lock
     (void) ompi_osc_rdma_lock_add (lock, -OMPI_OSC_RDMA_LOCK_EXCLUSIVE);
 }
 
-/**
- * Dummy completion function for atomic operations
- */
-void ompi_osc_rdma_atomic_complete (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
-                                    void *local_address, mca_btl_base_registration_handle_t *local_handle,
-                                    void *context, void *data, int status);
-
-__opal_attribute_always_inline__
-static inline int ompi_osc_rdma_btl_fop (ompi_osc_rdma_module_t *module, uint8_t btl_index,
-                                         struct mca_btl_base_endpoint_t *endpoint, uint64_t address,
-                                         mca_btl_base_registration_handle_t *address_handle, int op,
-                                         int64_t operand, int flags, int64_t *result, const bool wait_for_completion,
-                                         ompi_osc_rdma_pending_op_cb_fn_t cbfunc, void *cbdata, void *cbcontext)
-{
-    ompi_osc_rdma_pending_op_t *pending_op;
-    mca_btl_base_module_t *selected_btl = ompi_osc_rdma_selected_btl (module, btl_index);
-    int ret = OPAL_ERROR;
-
-    pending_op = OBJ_NEW(ompi_osc_rdma_pending_op_t);
-    assert (NULL != pending_op);
-
-    if (!wait_for_completion) {
-        /* NTH: need to keep track of pending ops to avoid a potential teardown problem */
-        pending_op->module = module;
-        (void) opal_atomic_fetch_add_32 (&module->pending_ops, 1);
-    }
-
-    pending_op->op_result = (void *) result;
-    pending_op->op_size = (MCA_BTL_ATOMIC_FLAG_32BIT & flags) ? 4 : 8;
-    OBJ_RETAIN(pending_op);
-    if (cbfunc) {
-        pending_op->cbfunc = cbfunc;
-        pending_op->cbdata = cbdata;
-        pending_op->cbcontext = cbcontext;
-    }
-
-    /* spin until the btl has accepted the operation */
-    do {
-        if (NULL == pending_op->op_frag) {
-            ret = ompi_osc_rdma_frag_alloc (module, 8, &pending_op->op_frag, (char **) &pending_op->op_buffer);
-        }
-
-        if (NULL != pending_op->op_frag) {
-            ret = selected_btl->btl_atomic_fop (selected_btl, endpoint, pending_op->op_buffer,
-                                                (intptr_t) address, pending_op->op_frag->handle, address_handle,
-                                                op, operand, flags, MCA_BTL_NO_ORDER, ompi_osc_rdma_atomic_complete,
-                                                (void *) pending_op, NULL);
-        }
-
-        if (OPAL_LIKELY(!ompi_osc_rdma_oor(ret))) {
-            break;
-        }
-        ompi_osc_rdma_progress (module);
-    } while (1);
-
-    if (OPAL_SUCCESS != ret) {
-        if (OPAL_LIKELY(1 == ret)) {
-            *result = ((int64_t *) pending_op->op_buffer)[0];
-            ret = OMPI_SUCCESS;
-            ompi_osc_rdma_atomic_complete (selected_btl, endpoint, pending_op->op_buffer,
-                                           pending_op->op_frag->handle, (void *) pending_op, NULL, OPAL_SUCCESS);
-        } else {
-            /* need to release here because ompi_osc_rdma_atomic_complete was not called */
-            OBJ_RELEASE(pending_op);
-        }
-    } else if (wait_for_completion) {
-        while (!pending_op->op_complete) {
-            ompi_osc_rdma_progress (module);
-        }
-    }
-
-    OBJ_RELEASE(pending_op);
-
-    return ret;
-}
-
 __opal_attribute_always_inline__
 static inline int ompi_osc_rdma_lock_btl_fop (ompi_osc_rdma_module_t *module, ompi_osc_rdma_peer_t *peer, uint64_t address,
                                               int op, ompi_osc_rdma_lock_t operand, ompi_osc_rdma_lock_t *result,
@@ -114,69 +41,6 @@ static inline int ompi_osc_rdma_lock_btl_fop (ompi_osc_rdma_module_t *module, om
                                   operand, 0, result, wait_for_completion, NULL, NULL, NULL);
 }
 
-__opal_attribute_always_inline__
-static inline int ompi_osc_rdma_btl_op (ompi_osc_rdma_module_t *module, uint8_t btl_index,
-                                        struct mca_btl_base_endpoint_t *endpoint, uint64_t address,
-                                        mca_btl_base_registration_handle_t *address_handle,
-                                        int op, int64_t operand, int flags, const bool wait_for_completion,
-                                        ompi_osc_rdma_pending_op_cb_fn_t cbfunc, void *cbdata, void *cbcontext)
-{
-    ompi_osc_rdma_pending_op_t *pending_op;
-    mca_btl_base_module_t *selected_btl = ompi_osc_rdma_selected_btl (module, btl_index);
-    int ret;
-
-    if (!(selected_btl->btl_flags & MCA_BTL_FLAGS_ATOMIC_OPS)) {
-        return ompi_osc_rdma_btl_fop (module, btl_index, endpoint, address, address_handle, op, operand, flags,
-                                      NULL, wait_for_completion, cbfunc, cbdata, cbcontext);
-    }
-
-    pending_op = OBJ_NEW(ompi_osc_rdma_pending_op_t);
-    assert (NULL != pending_op);
-    OBJ_RETAIN(pending_op);
-    if (cbfunc) {
-        pending_op->cbfunc = cbfunc;
-        pending_op->cbdata = cbdata;
-        pending_op->cbcontext = cbcontext;
-    }
-
-    if (!wait_for_completion) {
-        /* NTH: need to keep track of pending ops to avoid a potential teardown problem */
-        pending_op->module = module;
-        (void) opal_atomic_fetch_add_32 (&module->pending_ops, 1);
-    }
-
-    /* spin until the btl has accepted the operation */
-    do {
-        ret = selected_btl->btl_atomic_op (selected_btl, endpoint, (intptr_t) address, address_handle,
-                                           op, operand, flags, MCA_BTL_NO_ORDER, ompi_osc_rdma_atomic_complete,
-                                           (void *) pending_op, NULL);
-
-        if (OPAL_LIKELY(!ompi_osc_rdma_oor(ret))) {
-            break;
-        }
-        ompi_osc_rdma_progress (module);
-    } while (1);
-
-    if (OPAL_SUCCESS != ret) {
-        /* need to release here because ompi_osc_rdma_atomic_complete was not called */
-        OBJ_RELEASE(pending_op);
-        if (OPAL_LIKELY(1 == ret)) {
-            if (cbfunc) {
-                cbfunc (cbdata, cbcontext, OMPI_SUCCESS);
-            }
-            ret = OMPI_SUCCESS;
-        }
-    } else if (wait_for_completion) {
-        while (!pending_op->op_complete) {
-            ompi_osc_rdma_progress (module);
-        }
-    }
-
-    OBJ_RELEASE(pending_op);
-
-    return ret;
-}
-
 __opal_attribute_always_inline__
 static inline int ompi_osc_rdma_lock_btl_op (ompi_osc_rdma_module_t *module, ompi_osc_rdma_peer_t *peer, uint64_t address,
                                              int op, ompi_osc_rdma_lock_t operand, const bool wait_for_completion)
@@ -185,61 +49,6 @@ static inline int ompi_osc_rdma_lock_btl_op (ompi_osc_rdma_module_t *module, omp
                                  operand, 0, wait_for_completion, NULL, NULL, NULL);
 }
 
-__opal_attribute_always_inline__
-static inline int ompi_osc_rdma_btl_cswap (ompi_osc_rdma_module_t *module, uint8_t btl_index,
-                                           struct mca_btl_base_endpoint_t *endpoint, uint64_t address,
-                                           mca_btl_base_registration_handle_t *address_handle,
-                                           int64_t compare, int64_t value, int flags, int64_t *result)
-{
-    ompi_osc_rdma_pending_op_t *pending_op;
-    mca_btl_base_module_t *selected_btl = ompi_osc_rdma_selected_btl (module, btl_index);
-    int ret;
-
-    pending_op = OBJ_NEW(ompi_osc_rdma_pending_op_t);
-    assert (NULL != pending_op);
-
-    OBJ_RETAIN(pending_op);
-
-    pending_op->op_result = (void *) result;
-    pending_op->op_size = (MCA_BTL_ATOMIC_FLAG_32BIT & flags) ? 4 : 8;
-
-    /* spin until the btl has accepted the operation */
-    do {
-        if (NULL == pending_op->op_frag) {
-            ret = ompi_osc_rdma_frag_alloc (module, 8, &pending_op->op_frag, (char **) &pending_op->op_buffer);
-        }
-        if (NULL != pending_op->op_frag) {
-            ret = selected_btl->btl_atomic_cswap (selected_btl, endpoint, pending_op->op_buffer,
-                                                  address, pending_op->op_frag->handle, address_handle, compare,
-                                                  value, flags, 0, ompi_osc_rdma_atomic_complete, (void *) pending_op,
-                                                  NULL);
-        }
-
-        if (OPAL_LIKELY(!ompi_osc_rdma_oor(ret))) {
-            break;
-        }
-        ompi_osc_rdma_progress (module);
-    } while (1);
-
-    if (OPAL_SUCCESS != ret) {
-        if (OPAL_LIKELY(1 == ret)) {
-            *result = ((int64_t *) pending_op->op_buffer)[0];
-            ret = OMPI_SUCCESS;
-        }
-
-        /* need to release here because ompi_osc_rdma_atomic_complete was not called */
-        OBJ_RELEASE(pending_op);
-    } else {
-        while (!pending_op->op_complete) {
-            ompi_osc_rdma_progress (module);
-        }
-    }
-
-    OBJ_RELEASE(pending_op);
-
-    return ret;
-}
-
 __opal_attribute_always_inline__
 static inline int ompi_osc_rdma_lock_btl_cswap (ompi_osc_rdma_module_t *module, ompi_osc_rdma_peer_t *peer, uint64_t address,
                                                 ompi_osc_rdma_lock_t compare, ompi_osc_rdma_lock_t value, ompi_osc_rdma_lock_t *result)
diff --git a/ompi/mca/osc/rdma/osc_rdma_module.c b/ompi/mca/osc/rdma/osc_rdma_module.c
index 90e8b439c78..8a080e1e4eb 100644
--- a/ompi/mca/osc/rdma/osc_rdma_module.c
+++ b/ompi/mca/osc/rdma/osc_rdma_module.c
@@ -4,7 +4,7 @@
  *                         All rights reserved.
  * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
  *                         All rights reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ * Copyright (c) 2004-2020 High Performance Computing Center Stuttgart,
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
@@ -22,6 +22,8 @@
  * $HEADER$
  */
 
+#include "opal/mca/mpool/base/base.h"
+
 #include "osc_rdma.h"
 #include "osc_rdma_lock.h"
 
@@ -58,8 +60,8 @@ int ompi_osc_rdma_free(ompi_win_t *win)
 
     if (NULL != module->comm) {
         opal_output_verbose(1, ompi_osc_base_framework.framework_output,
-                            "rdma component destroying window with id %d",
-                            ompi_comm_get_cid(module->comm));
+                            "rdma component destroying window with id %s",
+                            ompi_comm_print_cid(module->comm));
 
         /* finish with a barrier */
         if (ompi_group_size(win->w_group) > 1) {
@@ -70,7 +72,7 @@ int ompi_osc_rdma_free(ompi_win_t *win)
         /* remove from component information */
         OPAL_THREAD_LOCK(&mca_osc_rdma_component.lock);
         opal_hash_table_remove_value_uint32(&mca_osc_rdma_component.modules,
-                                            ompi_comm_get_cid(module->comm));
+                                            ompi_comm_get_local_cid(module->comm));
         OPAL_THREAD_UNLOCK(&mca_osc_rdma_component.lock);
     }
 
@@ -140,7 +142,14 @@ int ompi_osc_rdma_free(ompi_win_t *win)
 
     free (module->peer_array);
     free (module->outstanding_lock_array);
-    free (module->free_after);
+    mca_mpool_base_default_module->mpool_free(mca_mpool_base_default_module,
+                                              module->free_after);
+    if (!module->use_accelerated_btl) {
+        for (int i = 0 ; i < module->alternate_btl_count ; ++i) {
+            OBJ_RELEASE(module->alternate_am_rdmas[i]);
+        }
+        free(module->alternate_am_rdmas);
+    }
     free (module);
 
     return OMPI_SUCCESS;
diff --git a/ompi/mca/osc/rdma/osc_rdma_passive_target.c b/ompi/mca/osc/rdma/osc_rdma_passive_target.c
index dc11c5e31df..8154c8da176 100644
--- a/ompi/mca/osc/rdma/osc_rdma_passive_target.c
+++ b/ompi/mca/osc/rdma/osc_rdma_passive_target.c
@@ -13,6 +13,7 @@
  * Copyright (c) 2010      IBM Corporation.  All rights reserved.
  * Copyright (c) 2012-2013 Sandia National Laboratories.  All rights reserved.
  * Copyright (c) 2018      Intel, Inc. All rights reserved.
+ * Copyright (c) 2021      Google, LLC. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -206,14 +207,14 @@ int ompi_osc_rdma_demand_lock_peer (ompi_osc_rdma_module_t *module, ompi_osc_rdm
     return ret;
 }
 
-int ompi_osc_rdma_lock_atomic (int lock_type, int target, int assert, ompi_win_t *win)
+int ompi_osc_rdma_lock_atomic (int lock_type, int target, int mpi_assert, ompi_win_t *win)
 {
     ompi_osc_rdma_module_t *module = GET_MODULE(win);
     ompi_osc_rdma_peer_t *peer = ompi_osc_rdma_module_peer (module, target);
     ompi_osc_rdma_sync_t *lock;
     int ret = OMPI_SUCCESS;
 
-    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "lock: %d, %d, %d, %s", lock_type, target, assert, win->w_name);
+    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "lock: %d, %d, %d, %s", lock_type, target, mpi_assert, win->w_name);
 
     if (module->no_locks) {
         OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_INFO, "attempted to lock with no_locks set");
@@ -238,13 +239,13 @@ int ompi_osc_rdma_lock_atomic (int lock_type, int target, int assert, ompi_win_t
     lock->type = OMPI_OSC_RDMA_SYNC_TYPE_LOCK;
     lock->sync.lock.target = target;
     lock->sync.lock.type = lock_type;
-    lock->sync.lock.assert = assert;
+    lock->sync.lock.mpi_assert = mpi_assert;
 
     lock->peer_list.peer = peer;
     lock->num_peers = 1;
     OBJ_RETAIN(peer);
 
-    if (0 == (assert & MPI_MODE_NOCHECK)) {
+    if (0 == (mpi_assert & MPI_MODE_NOCHECK)) {
         ret = ompi_osc_rdma_lock_atomic_internal (module, peer, lock);
     }
 
@@ -288,7 +289,7 @@ int ompi_osc_rdma_unlock_atomic (int target, ompi_win_t *win)
     /* finish all outstanding fragments */
     ompi_osc_rdma_sync_rdma_complete (lock);
 
-    if (!(lock->sync.lock.assert & MPI_MODE_NOCHECK)) {
+    if (!(lock->sync.lock.mpi_assert & MPI_MODE_NOCHECK)) {
         ret = ompi_osc_rdma_unlock_atomic_internal (module, peer, lock);
     }
 
@@ -309,13 +310,13 @@ int ompi_osc_rdma_unlock_atomic (int target, ompi_win_t *win)
     return ret;
 }
 
-int ompi_osc_rdma_lock_all_atomic (int assert, struct ompi_win_t *win)
+int ompi_osc_rdma_lock_all_atomic (int mpi_assert, struct ompi_win_t *win)
 {
     ompi_osc_rdma_module_t *module = GET_MODULE(win);
     ompi_osc_rdma_sync_t *lock;
     int ret = OMPI_SUCCESS;
 
-    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "lock_all: %d, %s", assert, win->w_name);
+    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "lock_all: %d, %s", mpi_assert, win->w_name);
 
     if (module->no_locks) {
         OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_INFO, "attempted to lock with no_locks set");
@@ -339,7 +340,7 @@ int ompi_osc_rdma_lock_all_atomic (int assert, struct ompi_win_t *win)
     lock->type = OMPI_OSC_RDMA_SYNC_TYPE_LOCK;
     lock->sync.lock.target = -1;
     lock->sync.lock.type   = MPI_LOCK_SHARED;
-    lock->sync.lock.assert = assert;
+    lock->sync.lock.mpi_assert = mpi_assert;
     lock->num_peers = ompi_comm_size (module->comm);
 
     lock->epoch_active = true;
@@ -348,7 +349,7 @@ int ompi_osc_rdma_lock_all_atomic (int assert, struct ompi_win_t *win)
      * at the expense of memory usage. Ex. if a window has 1M peers then 8MB per process would
      * be needed for this array. */
 
-    if (0 == (assert & MPI_MODE_NOCHECK)) {
+    if (0 == (mpi_assert & MPI_MODE_NOCHECK)) {
         /* increment the global shared lock */
         if (OMPI_OSC_RDMA_LOCKING_TWO_LEVEL == module->locking_mode) {
             ret = ompi_osc_rdma_lock_acquire_shared (module, module->leader, 0x0000000100000000UL,
@@ -396,7 +397,7 @@ int ompi_osc_rdma_unlock_all_atomic (struct ompi_win_t *win)
     /* finish all outstanding fragments */
     ompi_osc_rdma_sync_rdma_complete (lock);
 
-    if (0 == (lock->sync.lock.assert & MPI_MODE_NOCHECK)) {
+    if (0 == (lock->sync.lock.mpi_assert & MPI_MODE_NOCHECK)) {
         if (OMPI_OSC_RDMA_LOCKING_ON_DEMAND == module->locking_mode) {
             ompi_osc_rdma_peer_t *peer, *next;
 
diff --git a/ompi/mca/osc/rdma/osc_rdma_passive_target.h b/ompi/mca/osc/rdma/osc_rdma_passive_target.h
index ee4a1f91281..5f29be9ca88 100644
--- a/ompi/mca/osc/rdma/osc_rdma_passive_target.h
+++ b/ompi/mca/osc/rdma/osc_rdma_passive_target.h
@@ -2,6 +2,7 @@
 /*
  * Copyright (c) 2014-2015 Los Alamos National Security, LLC.  All rights
  *                         reserved.
+ * Copyright (c) 2021      Google, LLC. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -9,7 +10,7 @@
  * $HEADER$
  */
 
-#if !defined(OSC_RDMA_PASSIVE_TARGET_H)
+#ifndef OSC_RDMA_PASSIVE_TARGET_H
 #define OSC_RDMA_PASSIVE_TARGET_H
 
 #include "osc_rdma.h"
@@ -21,13 +22,13 @@
  *
  * @param[in] lock_type        mpi lock type (MPI_LOCK_SHARED, MPI_LOCK_EXCLUSIVE)
  * @param[in] target           target process
- * @param[in] assert           asserts
+ * @param[in] mpi_assert       asserts
  * @param[in] win              mpi window
  *
  * @returns OMPI_SUCCESS on success
  * @returns OMPI_ERR_RMA_SYNC if there is a conflicting RMA epoch
  */
-int ompi_osc_rdma_lock_atomic (int lock_type, int target, int assert, ompi_win_t *win);
+int ompi_osc_rdma_lock_atomic (int lock_type, int target, int mpi_assert, ompi_win_t *win);
 
 /**
  * @brief unlock the target in the window using network/cpu atomics
@@ -43,18 +44,18 @@ int ompi_osc_rdma_unlock_atomic (int target, ompi_win_t *win);
 /**
  * @brief lock all targets in window using network/cpu atomics
  *
- * @param[in] assert           asserts
+ * @param[in] mpi_assert       asserts
  * @param[in] win              mpi window
  *
  * @returns OMPI_SUCCESS on success
  * @returns OMPI_ERR_RMA_SYNC if there is a conflicting RMA epoch
  */
-int ompi_osc_rdma_lock_all_atomic (int assert, struct ompi_win_t *win);
+int ompi_osc_rdma_lock_all_atomic (int mpi_assert, struct ompi_win_t *win);
 
 /**
  * @brief unlock all targets in window using network/cpu atomics
  *
- * @param[in] assert           asserts
+ * @param[in] mpi_assert       asserts
  * @param[in] win              mpi window
  *
  * @returns OMPI_SUCCESS on success
@@ -128,4 +129,4 @@ int ompi_osc_rdma_flush_local (int target, struct ompi_win_t *win);
  */
 int ompi_osc_rdma_flush_local_all (struct ompi_win_t *win);
 
-#endif
+#endif /* OSC_RDMA_PASSIVE_TARGET_H */
diff --git a/ompi/mca/osc/rdma/osc_rdma_peer.c b/ompi/mca/osc/rdma/osc_rdma_peer.c
index ebfe1e8aadc..e4fee6bf9de 100644
--- a/ompi/mca/osc/rdma/osc_rdma_peer.c
+++ b/ompi/mca/osc/rdma/osc_rdma_peer.c
@@ -40,60 +40,85 @@ static int ompi_osc_rdma_peer_btl_endpoint (struct ompi_osc_rdma_module_t *modul
                                             struct mca_btl_base_endpoint_t **endpoint)
 {
     ompi_proc_t *proc = ompi_comm_peer_lookup (module->comm, peer_id);
-    mca_bml_base_endpoint_t *bml_endpoint;
-    int num_btls;
-
-    /* for now just use the bml to get the btl endpoint */
-    bml_endpoint = mca_bml_base_get_endpoint (proc);
-
-    num_btls = mca_bml_base_btl_array_get_size (&bml_endpoint->btl_rdma);
-
-    for (int module_btl_index = 0 ; module_btl_index < module->btls_in_use ; ++module_btl_index) {
-        for (int btl_index = 0 ; btl_index < num_btls ; ++btl_index) {
-            if (bml_endpoint->btl_rdma.bml_btls[btl_index].btl == module->selected_btls[module_btl_index]) {
-                *btl_index_out = module_btl_index;
-                *endpoint = bml_endpoint->btl_rdma.bml_btls[btl_index].btl_endpoint;
-                return OMPI_SUCCESS;
-            }
+    mca_bml_base_endpoint_t *bml_endpoint = mca_bml_base_get_endpoint(proc);
+
+    if (module->use_accelerated_btl) {
+        opal_output_verbose(MCA_BASE_VERBOSE_TRACE, ompi_osc_base_framework.framework_output,
+                            "rank %d: accelerated btl search for peer %d",
+                            ompi_comm_rank(module->comm), peer_id);
+        mca_bml_base_btl_t *bml_btl = mca_bml_base_btl_array_find(&bml_endpoint->btl_rdma,
+                                                                  module->accelerated_btl);
+        if (NULL != bml_btl) {
+            *btl_index_out = 0;
+            *endpoint = bml_btl->btl_endpoint;
+
+            return OMPI_SUCCESS;
         }
-    }
+    } else {
+        mca_bml_base_btl_t *bml_btl;
+        opal_output_verbose(MCA_BASE_VERBOSE_TRACE, ompi_osc_base_framework.framework_output,
+                            "rank %d: alternate btl search for peer %d",
+                            ompi_comm_rank(module->comm), peer_id);
+
+        /* the non accelerated case is a bit difficult compared to the
+         * accelerated case.  The right BTL could be in either the
+         * rdma or eager endpoint list, because we're using the am
+         * rdma interface to provide RDMA semantics.  The important
+         * part is that we search the alternate_btls list in order,
+         * since it is sorted by latency.
+         */
+        for (int osc_btl_idx = 0 ; osc_btl_idx < module->alternate_btl_count ; ++osc_btl_idx) {
+            mca_btl_base_module_t *search_btl = ompi_osc_rdma_selected_btl(module, osc_btl_idx);
+            const char *source = NULL;
+
+            opal_output_verbose(MCA_BASE_VERBOSE_TRACE, ompi_osc_base_framework.framework_output,
+                                "rank %d comparing with btl %s, %d",
+                                ompi_comm_rank(module->comm),
+                                search_btl->btl_component->btl_version.mca_component_name,
+                                osc_btl_idx);
+
+            source = "rdma";
+            bml_btl = mca_bml_base_btl_array_find(&bml_endpoint->btl_rdma, search_btl);
+            if (NULL == bml_btl) {
+                source = "eager";
+                bml_btl = mca_bml_base_btl_array_find(&bml_endpoint->btl_eager, search_btl);
+            }
+            if (NULL != bml_btl) {
+                *btl_index_out = osc_btl_idx;
+                *endpoint = bml_btl->btl_endpoint;
 
-    /* if this is a non-RDMA btl then the endpoint may be listed under eager */
-    num_btls = mca_bml_base_btl_array_get_size (&bml_endpoint->btl_eager);
+                opal_output_verbose(MCA_BASE_VERBOSE_TRACE, ompi_osc_base_framework.framework_output,
+                                    "rank %d found btl for peer %d (%s, %d, %s)",
+                                    ompi_comm_rank(module->comm), peer_id,
+                                    bml_btl->btl->btl_component->btl_version.mca_component_name,
+                                    osc_btl_idx, source);
 
-    for (int module_btl_index = 0 ; module_btl_index < module->btls_in_use ; ++module_btl_index) {
-        for (int btl_index = 0 ; btl_index < num_btls ; ++btl_index) {
-            if (bml_endpoint->btl_eager.bml_btls[btl_index].btl == module->selected_btls[module_btl_index]) {
-                *btl_index_out = module_btl_index;
-                *endpoint = bml_endpoint->btl_eager.bml_btls[btl_index].btl_endpoint;
                 return OMPI_SUCCESS;
             }
         }
     }
 
+    opal_output_verbose(MCA_BASE_VERBOSE_ERROR, ompi_osc_base_framework.framework_output,
+                        "rank %d: failed peer search for peer %d",
+                        ompi_comm_rank(module->comm), peer_id);
+
     /* unlikely but can happen when creating a peer for self */
     return OMPI_ERR_UNREACH;
 }
 
 int ompi_osc_rdma_new_peer (struct ompi_osc_rdma_module_t *module, int peer_id, ompi_osc_rdma_peer_t **peer_out) {
-    struct mca_btl_base_endpoint_t *endpoint;
+    struct mca_btl_base_endpoint_t *endpoint = NULL;
     ompi_osc_rdma_peer_t *peer;
     uint8_t module_btl_index = UINT8_MAX;
-    mca_btl_base_module_t *btl = NULL;
 
     *peer_out = NULL;
 
     /* find a btl/endpoint to use for this peer */
     int ret = ompi_osc_rdma_peer_btl_endpoint (module, peer_id, &module_btl_index, &endpoint);
-    if (OPAL_UNLIKELY(OMPI_SUCCESS != ret && !((module->selected_btls[0]->btl_atomic_flags & MCA_BTL_ATOMIC_SUPPORTS_GLOB) &&
-                                               peer_id == ompi_comm_rank (module->comm)))) {
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
         return ret;
     }
 
-    if (endpoint) {
-        btl = ompi_osc_rdma_selected_btl (module, module_btl_index);
-    }
-
     if (MPI_WIN_FLAVOR_DYNAMIC == module->flavor) {
         peer = (ompi_osc_rdma_peer_t *) OBJ_NEW(ompi_osc_rdma_peer_dynamic_t);
     } else if (module->same_size && module->same_disp_unit) {
@@ -120,14 +145,13 @@ int ompi_osc_rdma_new_peer (struct ompi_osc_rdma_module_t *module, int peer_id,
  *
  * This function reads the registration handle and state pointer from the peer that holds that data. If necessary
  * it will then ready information about the peer from its state data structure. This information includes the
- * displacement unit, base pointer, window size, and registation handle (if applicable).
+ * displacement unit, base pointer, window size, and registration handle (if applicable).
  */
 static int ompi_osc_rdma_peer_setup (ompi_osc_rdma_module_t *module, ompi_osc_rdma_peer_t *peer)
 {
     ompi_osc_rdma_peer_extended_t *ex_peer = (ompi_osc_rdma_peer_extended_t *) peer;
     uint64_t peer_data_size;
     uint64_t peer_data_offset, array_pointer;
-    mca_btl_base_module_t *array_btl;
     struct mca_btl_base_endpoint_t *array_endpoint;
     ompi_osc_rdma_region_t *array_peer_data, *node_peer_data;
     ompi_osc_rdma_rank_data_t rank_data;
@@ -140,7 +164,8 @@ static int ompi_osc_rdma_peer_setup (ompi_osc_rdma_module_t *module, ompi_osc_rd
     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_DEBUG, "configuring peer for rank %d", peer->rank);
 
     if (module->use_memory_registration) {
-        registration_handle_size = module->selected_btls[0]->btl_registration_handle_size;
+        assert(module->use_accelerated_btl);
+        registration_handle_size = module->accelerated_btl->btl_registration_handle_size;
     }
 
     /* each node is responsible for holding a part of the rank -> node/local rank mapping array. this code
diff --git a/ompi/mca/osc/rdma/osc_rdma_peer.h b/ompi/mca/osc/rdma/osc_rdma_peer.h
index ef8f8e0605c..cea680a44d3 100644
--- a/ompi/mca/osc/rdma/osc_rdma_peer.h
+++ b/ompi/mca/osc/rdma/osc_rdma_peer.h
@@ -101,7 +101,7 @@ struct ompi_osc_rdma_peer_extended_t {
     osc_rdma_size_t size;
 
     /** displacement unit */
-    int disp_unit;
+    ptrdiff_t disp_unit;
 };
 
 typedef struct ompi_osc_rdma_peer_extended_t ompi_osc_rdma_peer_extended_t;
diff --git a/ompi/mca/osc/rdma/osc_rdma_sync.h b/ompi/mca/osc/rdma/osc_rdma_sync.h
index 31bf5bc3295..c494b0b8c74 100644
--- a/ompi/mca/osc/rdma/osc_rdma_sync.h
+++ b/ompi/mca/osc/rdma/osc_rdma_sync.h
@@ -2,6 +2,7 @@
 /*
  * Copyright (c) 2015-2018 Los Alamos National Security, LLC.  All rights
  *                         reserved.
+ * Copyright (c) 2021      Google, LLC. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -9,9 +10,11 @@
  * $HEADER$
  */
 
-#if !defined(OSC_RDMA_SYNC_H)
+#ifndef OSC_RDMA_SYNC_H
 #define OSC_RDMA_SYNC_H
 
+#include "ompi/group/group.h"
+#include "opal/mca/btl/btl.h"
 #include "osc_rdma_types.h"
 #include "opal/class/opal_object.h"
 #include "opal/mca/threads/threads.h"
@@ -66,8 +69,8 @@ struct ompi_osc_rdma_sync_t {
 
             /** assert specified at lock acquire time. at this time Open MPI
              * only uses 5-bits for asserts. if this number goes over 16 this
-             * will need to be changed to accomodate. */
-            int16_t assert;
+             * will need to be changed to accommodate. */
+            int16_t mpi_assert;
         } lock;
 
         /** post/start/complete/wait specific synchronization data */
diff --git a/ompi/mca/osc/rdma/osc_rdma_types.h b/ompi/mca/osc/rdma/osc_rdma_types.h
index 50f411fd0d7..69157f6bef0 100644
--- a/ompi/mca/osc/rdma/osc_rdma_types.h
+++ b/ompi/mca/osc/rdma/osc_rdma_types.h
@@ -21,8 +21,6 @@ struct ompi_osc_rdma_frag_t;
 struct ompi_osc_rdma_sync_t;
 struct ompi_osc_rdma_peer_t;
 
-#if OPAL_HAVE_ATOMIC_MATH_64
-
 typedef int64_t osc_rdma_base_t;
 typedef uint64_t osc_rdma_size_t;
 typedef int64_t osc_rdma_counter_t;
@@ -30,19 +28,6 @@ typedef opal_atomic_int64_t osc_rdma_atomic_counter_t;
 
 #define ompi_osc_rdma_counter_add opal_atomic_add_fetch_64
 
-#else
-
-typedef int32_t osc_rdma_base_t;
-typedef uint32_t osc_rdma_size_t;
-typedef int32_t osc_rdma_counter_t;
-typedef opal_atomic_int32_t osc_rdma_atomic_counter_t;
-
-#define ompi_osc_rdma_counter_add opal_atomic_add_fetch_32
-
-#endif
-
-#if OPAL_HAVE_ATOMIC_MATH_64
-
 #define OMPI_OSC_RDMA_LOCK_EXCLUSIVE   0x8000000000000000l
 
 typedef int64_t  ompi_osc_rdma_lock_t;
@@ -70,38 +55,6 @@ static inline int ompi_osc_rdma_lock_compare_exchange (opal_atomic_int64_t *p, i
     return ret;
 }
 
-#else
-
-#define OMPI_OSC_RDMA_LOCK_EXCLUSIVE 0x80000000l
-
-typedef int32_t  ompi_osc_rdma_lock_t;
-typedef opal_atomic_int32_t  ompi_osc_rdma_atomic_lock_t;
-
-static inline int32_t ompi_osc_rdma_lock_add (opal_atomic_int32_t *p, int32_t value)
-{
-    int32_t new;
-
-    opal_atomic_mb ();
-    /* opal_atomic_add_fetch_32 differs from normal atomics in that is returns the new value */
-    new = opal_atomic_add_fetch_32 (p, value) - value;
-    opal_atomic_mb ();
-
-    return new;
-}
-
-static inline int ompi_osc_rdma_lock_compare_exchange (opal_atomic_int32_t *p, int32_t *comp, int32_t value)
-{
-    int ret;
-
-    opal_atomic_mb ();
-    ret = opal_atomic_compare_exchange_strong_32 (p, comp, value);
-    opal_atomic_mb ();
-
-    return ret;
-}
-
-#endif /* OPAL_HAVE_ATOMIC_MATH_64 */
-
 /**
  * @brief structure describing a window memory region
  */
@@ -219,11 +172,7 @@ struct ompi_osc_rdma_frag_t {
 
     /* Number of operations which have started writing into the frag, but not yet completed doing so */
     opal_atomic_int32_t pending;
-#if OPAL_HAVE_ATOMIC_MATH_64
     opal_atomic_int64_t curr_index;
-#else
-    opal_atomic_int32_t curr_index;
-#endif
 
     struct ompi_osc_rdma_module_t *module;
     mca_btl_base_registration_handle_t *handle;
diff --git a/ompi/mca/osc/sm/osc_sm.h b/ompi/mca/osc/sm/osc_sm.h
index 01e96982050..363d3429a63 100644
--- a/ompi/mca/osc/sm/osc_sm.h
+++ b/ompi/mca/osc/sm/osc_sm.h
@@ -18,22 +18,11 @@
 
 #include "opal/mca/shmem/base/base.h"
 
-#if OPAL_HAVE_ATOMIC_MATH_64
-
 typedef uint64_t osc_sm_post_type_t;
 typedef opal_atomic_uint64_t osc_sm_post_atomic_type_t;
 #define OSC_SM_POST_BITS 6
 #define OSC_SM_POST_MASK 0x3f
 
-#else
-
-typedef uint32_t osc_sm_post_type_t;
-typedef opal_atomic_uint32_t osc_sm_post_atomic_type_t;
-#define OSC_SM_POST_BITS 5
-#define OSC_SM_POST_MASK 0x1f
-
-#endif
-
 /* data shared across all peers */
 struct ompi_osc_sm_global_state_t {
     int use_barrier_for_fence;
@@ -64,6 +53,9 @@ typedef struct ompi_osc_sm_node_state_t ompi_osc_sm_node_state_t;
 struct ompi_osc_sm_component_t {
     ompi_osc_base_component_t super;
 
+    /** Priority of the osc/sm component */
+    unsigned int priority;
+
     char *backing_directory;
 };
 typedef struct ompi_osc_sm_component_t ompi_osc_sm_component_t;
@@ -86,7 +78,7 @@ struct ompi_osc_sm_module_t {
 
     size_t *sizes;
     void **bases;
-    int *disp_units;
+    ptrdiff_t *disp_units;
 
     ompi_group_t *start_group;
     ompi_group_t *post_group;
@@ -106,7 +98,7 @@ struct ompi_osc_sm_module_t {
 };
 typedef struct ompi_osc_sm_module_t ompi_osc_sm_module_t;
 
-int ompi_osc_sm_shared_query(struct ompi_win_t *win, int rank, size_t *size, int *disp_unit, void *baseptr);
+int ompi_osc_sm_shared_query(struct ompi_win_t *win, int rank, size_t *size, ptrdiff_t *disp_unit, void *baseptr);
 
 int ompi_osc_sm_attach(struct ompi_win_t *win, void *base, size_t len);
 int ompi_osc_sm_detach(struct ompi_win_t *win, const void *base);
@@ -114,29 +106,29 @@ int ompi_osc_sm_detach(struct ompi_win_t *win, const void *base);
 int ompi_osc_sm_free(struct ompi_win_t *win);
 
 int ompi_osc_sm_put(const void *origin_addr,
-                          int origin_count,
+                          size_t origin_count,
                           struct ompi_datatype_t *origin_dt,
                           int target,
                           ptrdiff_t target_disp,
-                          int target_count,
+                          size_t target_count,
                           struct ompi_datatype_t *target_dt,
                           struct ompi_win_t *win);
 
 int ompi_osc_sm_get(void *origin_addr,
-                          int origin_count,
+                          size_t origin_count,
                           struct ompi_datatype_t *origin_dt,
                           int target,
                           ptrdiff_t target_disp,
-                          int target_count,
+                          size_t target_count,
                           struct ompi_datatype_t *target_dt,
                           struct ompi_win_t *win);
 
 int ompi_osc_sm_accumulate(const void *origin_addr,
-                                 int origin_count,
+                                 size_t origin_count,
                                  struct ompi_datatype_t *origin_dt,
                                  int target,
                                  ptrdiff_t target_disp,
-                                 int target_count,
+                                 size_t target_count,
                                  struct ompi_datatype_t *target_dt,
                                  struct ompi_op_t *op,
                                  struct ompi_win_t *win);
@@ -158,73 +150,73 @@ int ompi_osc_sm_fetch_and_op(const void *origin_addr,
                                    struct ompi_win_t *win);
 
 int ompi_osc_sm_get_accumulate(const void *origin_addr,
-                                     int origin_count,
+                                     size_t origin_count,
                                      struct ompi_datatype_t *origin_datatype,
                                      void *result_addr,
-                                     int result_count,
+                                     size_t result_count,
                                      struct ompi_datatype_t *result_datatype,
                                      int target_rank,
                                      MPI_Aint target_disp,
-                                     int target_count,
+                                     size_t target_count,
                                      struct ompi_datatype_t *target_datatype,
                                      struct ompi_op_t *op,
                                      struct ompi_win_t *win);
 
 int ompi_osc_sm_rput(const void *origin_addr,
-                           int origin_count,
+                           size_t origin_count,
                            struct ompi_datatype_t *origin_dt,
                            int target,
                            ptrdiff_t target_disp,
-                           int target_count,
+                           size_t target_count,
                            struct ompi_datatype_t *target_dt,
                            struct ompi_win_t *win,
                            struct ompi_request_t **request);
 
 int ompi_osc_sm_rget(void *origin_addr,
-                           int origin_count,
+                           size_t origin_count,
                            struct ompi_datatype_t *origin_dt,
                            int target,
                            ptrdiff_t target_disp,
-                           int target_count,
+                           size_t target_count,
                            struct ompi_datatype_t *target_dt,
                            struct ompi_win_t *win,
                            struct ompi_request_t **request);
 
 int ompi_osc_sm_raccumulate(const void *origin_addr,
-                                  int origin_count,
+                                  size_t origin_count,
                                   struct ompi_datatype_t *origin_dt,
                                   int target,
                                   ptrdiff_t target_disp,
-                                  int target_count,
+                                  size_t target_count,
                                   struct ompi_datatype_t *target_dt,
                                   struct ompi_op_t *op,
                                   struct ompi_win_t *win,
                                   struct ompi_request_t **request);
 
 int ompi_osc_sm_rget_accumulate(const void *origin_addr,
-                                      int origin_count,
+                                      size_t origin_count,
                                       struct ompi_datatype_t *origin_datatype,
                                       void *result_addr,
-                                      int result_count,
+                                      size_t result_count,
                                       struct ompi_datatype_t *result_datatype,
                                       int target_rank,
                                       MPI_Aint target_disp,
-                                      int target_count,
+                                      size_t target_count,
                                       struct ompi_datatype_t *target_datatype,
                                       struct ompi_op_t *op,
                                       struct ompi_win_t *win,
                                       struct ompi_request_t **request);
 
-int ompi_osc_sm_fence(int assert, struct ompi_win_t *win);
+int ompi_osc_sm_fence(int mpi_assert, struct ompi_win_t *win);
 
 int ompi_osc_sm_start(struct ompi_group_t *group,
-                            int assert,
+                            int mpi_assert,
                             struct ompi_win_t *win);
 
 int ompi_osc_sm_complete(struct ompi_win_t *win);
 
 int ompi_osc_sm_post(struct ompi_group_t *group,
-                           int assert,
+                           int mpi_assert,
                            struct ompi_win_t *win);
 
 int ompi_osc_sm_wait(struct ompi_win_t *win);
@@ -234,14 +226,14 @@ int ompi_osc_sm_test(struct ompi_win_t *win,
 
 int ompi_osc_sm_lock(int lock_type,
                            int target,
-                           int assert,
+                           int mpi_assert,
                            struct ompi_win_t *win);
 
 int ompi_osc_sm_unlock(int target,
                              struct ompi_win_t *win);
 
 
-int ompi_osc_sm_lock_all(int assert,
+int ompi_osc_sm_lock_all(int mpi_assert,
                                struct ompi_win_t *win);
 
 int ompi_osc_sm_unlock_all(struct ompi_win_t *win);
diff --git a/ompi/mca/osc/sm/osc_sm_active_target.c b/ompi/mca/osc/sm/osc_sm_active_target.c
index aafc6fdd82d..64e37e354cb 100644
--- a/ompi/mca/osc/sm/osc_sm_active_target.c
+++ b/ompi/mca/osc/sm/osc_sm_active_target.c
@@ -92,7 +92,7 @@ static int *ompi_osc_sm_group_ranks (ompi_group_t *group, ompi_group_t *sub_grou
 
 
 int
-ompi_osc_sm_fence(int assert, struct ompi_win_t *win)
+ompi_osc_sm_fence(int mpi_assert, struct ompi_win_t *win)
 {
     ompi_osc_sm_module_t *module =
         (ompi_osc_sm_module_t*) win->w_osc_module;
@@ -124,7 +124,7 @@ ompi_osc_sm_fence(int assert, struct ompi_win_t *win)
 
 int
 ompi_osc_sm_start(struct ompi_group_t *group,
-                  int assert,
+                  int mpi_assert,
                   struct ompi_win_t *win)
 {
     ompi_osc_sm_module_t *module =
@@ -139,7 +139,7 @@ ompi_osc_sm_start(struct ompi_group_t *group,
         return OMPI_ERR_RMA_SYNC;
     }
 
-    if (0 == (assert & MPI_MODE_NOCHECK)) {
+    if (0 == (mpi_assert & MPI_MODE_NOCHECK)) {
         int size;
 
         int *ranks = ompi_osc_sm_group_ranks (module->comm->c_local_group, group);
@@ -161,11 +161,7 @@ ompi_osc_sm_start(struct ompi_group_t *group,
 
             opal_atomic_rmb ();
 
-#if OPAL_HAVE_ATOMIC_MATH_64
             (void) opal_atomic_fetch_xor_64 ((opal_atomic_int64_t *) module->posts[my_rank] + rank_byte, rank_bit);
-#else
-            (void) opal_atomic_fetch_xor_32 ((opal_atomic_int32_t *) module->posts[my_rank] + rank_byte, rank_bit);
-#endif
        }
 
         free (ranks);
@@ -215,7 +211,7 @@ ompi_osc_sm_complete(struct ompi_win_t *win)
 
 int
 ompi_osc_sm_post(struct ompi_group_t *group,
-                       int assert,
+                       int mpi_assert,
                        struct ompi_win_t *win)
 {
     ompi_osc_sm_module_t *module =
@@ -236,7 +232,7 @@ ompi_osc_sm_post(struct ompi_group_t *group,
 
     OBJ_RETAIN(group);
 
-    if (0 == (assert & MPI_MODE_NOCHECK)) {
+    if (0 == (mpi_assert & MPI_MODE_NOCHECK)) {
         int *ranks = ompi_osc_sm_group_ranks (module->comm->c_local_group, group);
         if (NULL == ranks) {
             return OMPI_ERR_OUT_OF_RESOURCE;
@@ -247,11 +243,7 @@ ompi_osc_sm_post(struct ompi_group_t *group,
 
         gsize = ompi_group_size(module->post_group);
         for (int i = 0 ; i < gsize ; ++i) {
-#if OPAL_HAVE_ATOMIC_MATH_64
             (void) opal_atomic_fetch_add_64 ((opal_atomic_int64_t *) module->posts[ranks[i]] + my_byte, my_bit);
-#else
-            (void) opal_atomic_fetch_add_32 ((opal_atomic_int32_t *) module->posts[ranks[i]] + my_byte, my_bit);
-#endif
         }
 
         opal_atomic_wmb ();
diff --git a/ompi/mca/osc/sm/osc_sm_comm.c b/ompi/mca/osc/sm/osc_sm_comm.c
index b6094dd16eb..bbd5873bf96 100644
--- a/ompi/mca/osc/sm/osc_sm_comm.c
+++ b/ompi/mca/osc/sm/osc_sm_comm.c
@@ -22,11 +22,11 @@
 
 int
 ompi_osc_sm_rput(const void *origin_addr,
-                 int origin_count,
+                 size_t origin_count,
                  struct ompi_datatype_t *origin_dt,
                  int target,
                  ptrdiff_t target_disp,
-                 int target_count,
+                 size_t target_count,
                  struct ompi_datatype_t *target_dt,
                  struct ompi_win_t *win,
                  struct ompi_request_t **ompi_req)
@@ -37,7 +37,7 @@ ompi_osc_sm_rput(const void *origin_addr,
     void *remote_address;
 
     OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
-                         "rput: 0x%lx, %d, %s, %d, %d, %d, %s, 0x%lx",
+                         "rput: 0x%lx, %zu, %s, %d, %d, %zu, %s, 0x%lx",
                          (unsigned long) origin_addr, origin_count,
                          origin_dt->name, target, (int) target_disp,
                          target_count, target_dt->name,
@@ -62,11 +62,11 @@ ompi_osc_sm_rput(const void *origin_addr,
 
 int
 ompi_osc_sm_rget(void *origin_addr,
-                 int origin_count,
+                 size_t origin_count,
                  struct ompi_datatype_t *origin_dt,
                  int target,
                  ptrdiff_t target_disp,
-                 int target_count,
+                 size_t target_count,
                  struct ompi_datatype_t *target_dt,
                  struct ompi_win_t *win,
                  struct ompi_request_t **ompi_req)
@@ -77,7 +77,7 @@ ompi_osc_sm_rget(void *origin_addr,
     void *remote_address;
 
     OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
-                         "rget: 0x%lx, %d, %s, %d, %d, %d, %s, 0x%lx",
+                         "rget: 0x%lx, %zu, %s, %d, %d, %zu, %s, 0x%lx",
                          (unsigned long) origin_addr, origin_count,
                          origin_dt->name, target, (int) target_disp,
                          target_count, target_dt->name,
@@ -102,11 +102,11 @@ ompi_osc_sm_rget(void *origin_addr,
 
 int
 ompi_osc_sm_raccumulate(const void *origin_addr,
-                        int origin_count,
+                        size_t origin_count,
                         struct ompi_datatype_t *origin_dt,
                         int target,
                         ptrdiff_t target_disp,
-                        int target_count,
+                        size_t target_count,
                         struct ompi_datatype_t *target_dt,
                         struct ompi_op_t *op,
                         struct ompi_win_t *win,
@@ -118,7 +118,7 @@ ompi_osc_sm_raccumulate(const void *origin_addr,
     void *remote_address;
 
     OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
-                         "raccumulate: 0x%lx, %d, %s, %d, %d, %d, %s, %s, 0x%lx",
+                         "raccumulate: 0x%lx, %zu, %s, %d, %d, %zu, %s, %s, 0x%lx",
                          (unsigned long) origin_addr, origin_count,
                          origin_dt->name, target, (int) target_disp,
                          target_count, target_dt->name,
@@ -150,14 +150,14 @@ ompi_osc_sm_raccumulate(const void *origin_addr,
 
 int
 ompi_osc_sm_rget_accumulate(const void *origin_addr,
-                                  int origin_count,
+                                  size_t origin_count,
                                   struct ompi_datatype_t *origin_dt,
                                   void *result_addr,
-                                  int result_count,
+                                  size_t result_count,
                                   struct ompi_datatype_t *result_dt,
                                   int target,
                                   MPI_Aint target_disp,
-                                  int target_count,
+                                  size_t target_count,
                                   struct ompi_datatype_t *target_dt,
                                   struct ompi_op_t *op,
                                   struct ompi_win_t *win,
@@ -169,7 +169,7 @@ ompi_osc_sm_rget_accumulate(const void *origin_addr,
     void *remote_address;
 
     OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
-                         "rget_accumulate: 0x%lx, %d, %s, %d, %d, %d, %s, %s, 0x%lx",
+                         "rget_accumulate: 0x%lx, %zu, %s, %d, %d, %zu, %s, %s, 0x%lx",
                          (unsigned long) origin_addr, origin_count,
                          origin_dt->name, target, (int) target_disp,
                          target_count, target_dt->name,
@@ -207,11 +207,11 @@ ompi_osc_sm_rget_accumulate(const void *origin_addr,
 
 int
 ompi_osc_sm_put(const void *origin_addr,
-                      int origin_count,
+                      size_t origin_count,
                       struct ompi_datatype_t *origin_dt,
                       int target,
                       ptrdiff_t target_disp,
-                      int target_count,
+                      size_t target_count,
                       struct ompi_datatype_t *target_dt,
                       struct ompi_win_t *win)
 {
@@ -221,7 +221,7 @@ ompi_osc_sm_put(const void *origin_addr,
     void *remote_address;
 
     OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
-                         "put: 0x%lx, %d, %s, %d, %d, %d, %s, 0x%lx",
+                         "put: 0x%lx, %zu, %s, %d, %d, %zu, %s, 0x%lx",
                          (unsigned long) origin_addr, origin_count,
                          origin_dt->name, target, (int) target_disp,
                          target_count, target_dt->name,
@@ -238,11 +238,11 @@ ompi_osc_sm_put(const void *origin_addr,
 
 int
 ompi_osc_sm_get(void *origin_addr,
-                      int origin_count,
+                      size_t origin_count,
                       struct ompi_datatype_t *origin_dt,
                       int target,
                       ptrdiff_t target_disp,
-                      int target_count,
+                      size_t target_count,
                       struct ompi_datatype_t *target_dt,
                       struct ompi_win_t *win)
 {
@@ -252,7 +252,7 @@ ompi_osc_sm_get(void *origin_addr,
     void *remote_address;
 
     OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
-                         "get: 0x%lx, %d, %s, %d, %d, %d, %s, 0x%lx",
+                         "get: 0x%lx, %zu, %s, %d, %d, %zu, %s, 0x%lx",
                          (unsigned long) origin_addr, origin_count,
                          origin_dt->name, target, (int) target_disp,
                          target_count, target_dt->name,
@@ -269,11 +269,11 @@ ompi_osc_sm_get(void *origin_addr,
 
 int
 ompi_osc_sm_accumulate(const void *origin_addr,
-                       int origin_count,
+                       size_t origin_count,
                        struct ompi_datatype_t *origin_dt,
                        int target,
                        ptrdiff_t target_disp,
-                       int target_count,
+                       size_t target_count,
                        struct ompi_datatype_t *target_dt,
                        struct ompi_op_t *op,
                        struct ompi_win_t *win)
@@ -284,7 +284,7 @@ ompi_osc_sm_accumulate(const void *origin_addr,
     void *remote_address;
 
     OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
-                         "accumulate: 0x%lx, %d, %s, %d, %d, %d, %s, %s, 0x%lx",
+                         "accumulate: 0x%lx, %zu, %s, %d, %d, %zu, %s, %s, 0x%lx",
                          (unsigned long) origin_addr, origin_count,
                          origin_dt->name, target, (int) target_disp,
                          target_count, target_dt->name,
@@ -310,14 +310,14 @@ ompi_osc_sm_accumulate(const void *origin_addr,
 
 int
 ompi_osc_sm_get_accumulate(const void *origin_addr,
-                           int origin_count,
+                           size_t origin_count,
                            struct ompi_datatype_t *origin_dt,
                            void *result_addr,
-                           int result_count,
+                           size_t result_count,
                            struct ompi_datatype_t *result_dt,
                            int target,
                            MPI_Aint target_disp,
-                           int target_count,
+                           size_t target_count,
                            struct ompi_datatype_t *target_dt,
                            struct ompi_op_t *op,
                            struct ompi_win_t *win)
@@ -328,7 +328,7 @@ ompi_osc_sm_get_accumulate(const void *origin_addr,
     void *remote_address;
 
     OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
-                         "get_accumulate: 0x%lx, %d, %s, %d, %d, %d, %s, %s, 0x%lx",
+                         "get_accumulate: 0x%lx, %zu, %s, %d, %d, %zu, %s, %s, 0x%lx",
                          (unsigned long) origin_addr, origin_count,
                          origin_dt->name, target, (int) target_disp,
                          target_count, target_dt->name,
diff --git a/ompi/mca/osc/sm/osc_sm_component.c b/ompi/mca/osc/sm/osc_sm_component.c
index d404128c98d..ae87b960a9d 100644
--- a/ompi/mca/osc/sm/osc_sm_component.c
+++ b/ompi/mca/osc/sm/osc_sm_component.c
@@ -11,8 +11,10 @@
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
- * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2018-2022 Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * Copyright (c) 2020      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2020      High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -27,30 +29,27 @@
 #include "ompi/mca/osc/base/osc_base_obj_convert.h"
 #include "ompi/request/request.h"
 #include "opal/util/sys_limits.h"
-#include "opal/include/opal/align.h"
-#include "opal/util/info_subscriber.h"
+#include "opal/align.h"
 #include "opal/util/printf.h"
+#include "opal/mca/mpool/base/base.h"
 
 #include "osc_sm.h"
 
 static int component_open(void);
 static int component_init(bool enable_progress_threads, bool enable_mpi_threads);
 static int component_finalize(void);
-static int component_query(struct ompi_win_t *win, void **base, size_t size, int disp_unit,
+static int component_query(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t disp_unit,
                            struct ompi_communicator_t *comm, struct opal_info_t *info,
                            int flavor);
 static int component_register (void);
-static int component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit,
+static int component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t disp_unit,
                             struct ompi_communicator_t *comm, struct opal_info_t *info,
                             int flavor, int *model);
-static char* component_set_blocking_fence_info(opal_infosubscriber_t *obj, char *key, char *val);
-static char* component_set_alloc_shared_noncontig_info(opal_infosubscriber_t *obj, char *key, char *val);
-
 
 ompi_osc_sm_component_t mca_osc_sm_component = {
     { /* ompi_osc_base_component_t */
         .osc_version = {
-            OMPI_OSC_BASE_VERSION_3_0_0,
+            OMPI_OSC_BASE_VERSION_4_0_0,
             .mca_component_name = "sm",
             MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION,
                                   OMPI_RELEASE_VERSION),
@@ -112,6 +111,8 @@ ompi_osc_sm_module_t ompi_osc_sm_module_template = {
 
 static int component_register (void)
 {
+    char *description_str;
+
     if (0 == access ("/dev/shm", W_OK)) {
         mca_osc_sm_component.backing_directory = "/dev/shm";
     } else {
@@ -125,6 +126,16 @@ static int component_register (void)
                                             MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_3,
                                             MCA_BASE_VAR_SCOPE_READONLY, &mca_osc_sm_component.backing_directory);
 
+    mca_osc_sm_component.priority = 100;
+    opal_asprintf(&description_str, "Priority of the osc/sm component (default: %d)",
+                  mca_osc_sm_component.priority);
+    (void)mca_base_component_var_register(&mca_osc_sm_component.super.osc_version,
+                                          "priority", description_str,
+                                          MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, 0,
+                                          OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_GROUP,
+                                          &mca_osc_sm_component.priority);
+    free(description_str);
+
     return OPAL_SUCCESS;
 }
 
@@ -152,40 +163,34 @@ component_finalize(void)
 
 
 static int
-check_win_ok(ompi_communicator_t *comm, int flavor)
-{
-    if (! (MPI_WIN_FLAVOR_SHARED == flavor
-           || MPI_WIN_FLAVOR_ALLOCATE == flavor) ) {
-        return OMPI_ERR_NOT_SUPPORTED;
-    }
-
-    if (ompi_group_have_remote_peers (comm->c_local_group)) {
-        return OMPI_ERR_RMA_SHARED;
-    }
-
-    return OMPI_SUCCESS;
-}
-
-
-static int
-component_query(struct ompi_win_t *win, void **base, size_t size, int disp_unit,
+component_query(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t disp_unit,
                 struct ompi_communicator_t *comm, struct opal_info_t *info,
                 int flavor)
 {
-    int ret;
-    if (OMPI_SUCCESS != (ret = check_win_ok(comm, flavor))) {
-        if (OMPI_ERR_NOT_SUPPORTED == ret) {
+    /* component only supports shared or allocate flavors */
+    if (! (MPI_WIN_FLAVOR_SHARED == flavor ||
+           MPI_WIN_FLAVOR_ALLOCATE == flavor)) {
+        return -1;
+    }
+
+    /* If flavor is win_allocate, we can't run if there are remote
+     * peers in the group.  The same check for flavor_shared happens
+     * in select(), so that we can return an error to the user (since
+     * we should be able to run for all flavor_shared use cases.
+     * There's no way to return an error from component_query to the
+     * user, hence the delayed check.  */
+    if (MPI_WIN_FLAVOR_ALLOCATE == flavor) {
+        if (ompi_group_have_remote_peers(comm->c_local_group)) {
             return -1;
         }
-        return ret;
     }
 
-    return 100;
+    return mca_osc_sm_component.priority;
 }
 
 
 static int
-component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit,
+component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t disp_unit,
                  struct ompi_communicator_t *comm, struct opal_info_t *info,
                  int flavor, int *model)
 {
@@ -193,9 +198,12 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit
     int comm_size = ompi_comm_size (comm);
     bool unlink_needed = false;
     int ret = OMPI_ERROR;
+    size_t memory_alignment = OPAL_ALIGN_MIN;
 
-    if (OMPI_SUCCESS != (ret = check_win_ok(comm, flavor))) {
-        return ret;
+    assert(MPI_WIN_FLAVOR_SHARED == flavor || MPI_WIN_FLAVOR_ALLOCATE == flavor);
+
+    if (ompi_group_have_remote_peers(comm->c_local_group)) {
+        return OMPI_ERR_RMA_SHARED;
     }
 
     /* create module structure */
@@ -207,9 +215,9 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit
 
     OBJ_CONSTRUCT(&module->lock, opal_mutex_t);
 
-    ret = opal_infosubscribe_subscribe(&(win->super), "alloc_shared_noncontig", "false", component_set_alloc_shared_noncontig_info);
-
-    if (OPAL_SUCCESS != ret) goto error;
+    if (NULL != info) {
+        ompi_osc_base_set_memory_alignment(info, &memory_alignment);
+    }
 
     /* fill in the function pointer part */
     memcpy(module, &ompi_osc_sm_module_template,
@@ -230,7 +238,8 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit
         if (NULL == module->bases) return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
 
         module->sizes[0] = size;
-        module->bases[0] = malloc(size);
+        module->bases[0] = mca_mpool_base_default_module->mpool_alloc(mca_mpool_base_default_module, size,
+                                                                      memory_alignment, 0);
         if (NULL == module->bases[0]) return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
 
         module->global_state = malloc(sizeof(ompi_osc_sm_global_state_t));
@@ -246,9 +255,10 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit
         size_t pagesize;
         size_t state_size;
         size_t posts_size, post_size = (comm_size + OSC_SM_POST_MASK) / (OSC_SM_POST_MASK + 1);
+        size_t data_base_size;
 
-        OPAL_OUTPUT_VERBOSE((1, ompi_osc_base_framework.framework_output,
-                             "allocating shared memory region of size %ld\n", (long) size));
+        opal_output_verbose(MCA_BASE_VERBOSE_DEBUG, ompi_osc_base_framework.framework_output,
+                            "allocating shared memory region of size %ld\n", (long) size);
 
         /* get the pagesize */
         pagesize = opal_getpagesize();
@@ -256,22 +266,36 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit
         rbuf = malloc(sizeof(unsigned long) * comm_size);
         if (NULL == rbuf) return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
 
+        /* Note that the alloc_shared_noncontig info key only has
+         * meaning during window creation.  Once the window is
+         * created, we can't move memory around without making
+         * everything miserable.  So we intentionally do not subscribe
+         * to updates on the info key, because there's no useful
+         * update to occur. */
         module->noncontig = false;
         if (OMPI_SUCCESS != opal_info_get_bool(info, "alloc_shared_noncontig",
                                                &module->noncontig, &flag)) {
+            free(rbuf);
             goto error;
         }
 
         if (module->noncontig) {
+            opal_output_verbose(MCA_BASE_VERBOSE_DEBUG, ompi_osc_base_framework.framework_output,
+                                "allocating window using non-contiguous strategy");
             total = ((size - 1) / pagesize + 1) * pagesize;
         } else {
+            opal_output_verbose(MCA_BASE_VERBOSE_DEBUG, ompi_osc_base_framework.framework_output,
+                                "allocating window using contiguous strategy");
             total = size;
         }
         ret = module->comm->c_coll->coll_allgather(&total, 1, MPI_UNSIGNED_LONG,
                                                   rbuf, 1, MPI_UNSIGNED_LONG,
                                                   module->comm,
                                                   module->comm->c_coll->coll_allgather_module);
-        if (OMPI_SUCCESS != ret) return ret;
+        if (OMPI_SUCCESS != ret) {
+            free(rbuf);
+            goto error;
+        }
 
         total = 0;
         for (i = 0 ; i < comm_size ; ++i) {
@@ -283,17 +307,21 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit
         state_size += OPAL_ALIGN_PAD_AMOUNT(state_size, 64);
         posts_size = comm_size * post_size * sizeof (module->posts[0][0]);
         posts_size += OPAL_ALIGN_PAD_AMOUNT(posts_size, 64);
+        data_base_size = state_size + posts_size;
+        data_base_size += OPAL_ALIGN_PAD_AMOUNT(data_base_size, pagesize);
         if (0 == ompi_comm_rank (module->comm)) {
             char *data_file;
-            ret = opal_asprintf (&data_file, "%s" OPAL_PATH_SEP "osc_sm.%s.%x.%d.%d",
-                            mca_osc_sm_component.backing_directory, ompi_process_info.nodename,
-                            OMPI_PROC_MY_NAME->jobid, (int) OMPI_PROC_MY_NAME->vpid, ompi_comm_get_cid(module->comm));
+            ret = opal_asprintf (&data_file, "%s" OPAL_PATH_SEP "osc_sm.%s.%x.%d.%s",
+                                 mca_osc_sm_component.backing_directory, ompi_process_info.nodename,
+                                 OMPI_PROC_MY_NAME->jobid, (int) OMPI_PROC_MY_NAME->vpid,
+                                 ompi_comm_print_cid(module->comm));
             if (ret < 0) {
                 free(rbuf);
-                return OMPI_ERR_OUT_OF_RESOURCE;
+                ret = OMPI_ERR_OUT_OF_RESOURCE;
+                goto error;
             }
 
-            ret = opal_shmem_segment_create (&module->seg_ds, data_file, total + pagesize + state_size + posts_size);
+            ret = opal_shmem_segment_create (&module->seg_ds, data_file, total + data_base_size);
             free(data_file);
             if (OPAL_SUCCESS != ret) {
                 free(rbuf);
@@ -340,7 +368,7 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit
         module->global_state = (ompi_osc_sm_global_state_t *) (module->posts[0] + comm_size * post_size);
         module->node_states = (ompi_osc_sm_node_state_t *) (module->global_state + 1);
 
-        for (i = 0, total = state_size + posts_size ; i < comm_size ; ++i) {
+        for (i = 0, total = data_base_size ; i < comm_size ; ++i) {
             if (i > 0) {
                 module->posts[i] = module->posts[i - 1] + post_size;
             }
@@ -366,9 +394,9 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit
     opal_atomic_lock_init(&module->my_node_state->accumulate_lock, OPAL_ATOMIC_LOCK_UNLOCKED);
 
     /* share everyone's displacement units. */
-    module->disp_units = malloc(sizeof(int) * comm_size);
-    ret = module->comm->c_coll->coll_allgather(&disp_unit, 1, MPI_INT,
-                                              module->disp_units, 1, MPI_INT,
+    module->disp_units = malloc(sizeof(ptrdiff_t) * comm_size);
+    ret = module->comm->c_coll->coll_allgather(&disp_unit, sizeof(ptrdiff_t), MPI_BYTE,
+                                              module->disp_units, sizeof(ptrdiff_t), MPI_BYTE,
                                               module->comm,
                                               module->comm->c_coll->coll_allgather_module);
     if (OMPI_SUCCESS != ret) goto error;
@@ -426,11 +454,6 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit
 #endif
     }
 
-    ret = opal_infosubscribe_subscribe(&(win->super), "blocking_fence", module->global_state->use_barrier_for_fence ? "true" : "false",
-        component_set_blocking_fence_info);
-
-    if (OPAL_SUCCESS != ret) goto error;
-
     ret = module->comm->c_coll->coll_barrier(module->comm,
                                             module->comm->c_coll->coll_barrier_module);
     if (OMPI_SUCCESS != ret) goto error;
@@ -452,7 +475,7 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit
 
 
 int
-ompi_osc_sm_shared_query(struct ompi_win_t *win, int rank, size_t *size, int *disp_unit, void *baseptr)
+ompi_osc_sm_shared_query(struct ompi_win_t *win, int rank, size_t *size, ptrdiff_t *disp_unit, void *baseptr)
 {
     ompi_osc_sm_module_t *module =
         (ompi_osc_sm_module_t*) win->w_osc_module;
@@ -523,12 +546,13 @@ ompi_osc_sm_free(struct ompi_win_t *win)
         module->comm->c_coll->coll_barrier(module->comm,
                                           module->comm->c_coll->coll_barrier_module);
 
-	opal_shmem_segment_detach (&module->seg_ds);
+        opal_shmem_segment_detach (&module->seg_ds);
     } else {
         free(module->node_states);
         free(module->global_state);
         if (NULL != module->bases) {
-            free(module->bases[0]);
+            mca_mpool_base_default_module->mpool_free(mca_mpool_base_default_module,
+                                                      module->bases[0]);
         }
     }
     free(module->disp_units);
@@ -561,29 +585,6 @@ ompi_osc_sm_set_info(struct ompi_win_t *win, struct opal_info_t *info)
 }
 
 
-static char*
-component_set_blocking_fence_info(opal_infosubscriber_t *obj, char *key, char *val)
-{
-    ompi_osc_sm_module_t *module = (ompi_osc_sm_module_t*) ((struct ompi_win_t*) obj)->w_osc_module;
-/*
- * Assuming that you can't change the default.  
- */
-    return module->global_state->use_barrier_for_fence ? "true" : "false";
-}
-
-
-static char*
-component_set_alloc_shared_noncontig_info(opal_infosubscriber_t *obj, char *key, char *val)
-{
-
-    ompi_osc_sm_module_t *module = (ompi_osc_sm_module_t*) ((struct ompi_win_t*) obj)->w_osc_module;
-/*
- * Assuming that you can't change the default.  
- */
-    return module->noncontig ? "true" : "false";
-}
-
-
 int
 ompi_osc_sm_get_info(struct ompi_win_t *win, struct opal_info_t **info_used)
 {
diff --git a/ompi/mca/osc/sm/osc_sm_passive_target.c b/ompi/mca/osc/sm/osc_sm_passive_target.c
index 8c27561e543..27f47d51375 100644
--- a/ompi/mca/osc/sm/osc_sm_passive_target.c
+++ b/ompi/mca/osc/sm/osc_sm_passive_target.c
@@ -112,7 +112,7 @@ end_shared(ompi_osc_sm_module_t *module,
 int
 ompi_osc_sm_lock(int lock_type,
                  int target,
-                 int assert,
+                 int mpi_assert,
                  struct ompi_win_t *win)
 {
     ompi_osc_sm_module_t *module =
@@ -123,7 +123,7 @@ ompi_osc_sm_lock(int lock_type,
         return OMPI_ERR_RMA_SYNC;
     }
 
-    if (0 == (assert & MPI_MODE_NOCHECK)) {
+    if (0 == (mpi_assert & MPI_MODE_NOCHECK)) {
         if (MPI_LOCK_EXCLUSIVE == lock_type) {
             module->outstanding_locks[target] = lock_exclusive;
             ret = start_exclusive(module, target);
@@ -188,7 +188,7 @@ ompi_osc_sm_unlock(int target,
 
 
 int
-ompi_osc_sm_lock_all(int assert,
+ompi_osc_sm_lock_all(int mpi_assert,
                            struct ompi_win_t *win)
 {
     ompi_osc_sm_module_t *module =
@@ -197,7 +197,7 @@ ompi_osc_sm_lock_all(int assert,
 
     comm_size = ompi_comm_size(module->comm);
     for (i = 0 ; i < comm_size ; ++i) {
-        ret = ompi_osc_sm_lock(MPI_LOCK_SHARED, i, assert, win);
+        ret = ompi_osc_sm_lock(MPI_LOCK_SHARED, i, mpi_assert, win);
         if (OMPI_SUCCESS != ret) return ret;
     }
 
diff --git a/ompi/mca/osc/ucx/Makefile.am b/ompi/mca/osc/ucx/Makefile.am
index a7dae148e80..77184d83584 100644
--- a/ompi/mca/osc/ucx/Makefile.am
+++ b/ompi/mca/osc/ucx/Makefile.am
@@ -35,7 +35,7 @@ mcacomponentdir = $(pkglibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_osc_ucx_la_SOURCES = $(ucx_sources)
 mca_osc_ucx_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la $(osc_ucx_LIBS) \
-                        $(OPAL_TOP_BUILDDIR)/opal/mca/common/ucx/lib@OPAL_LIB_PREFIX@mca_common_ucx.la
+                        $(OPAL_TOP_BUILDDIR)/opal/mca/common/ucx/lib@OPAL_LIB_NAME@mca_common_ucx.la
 mca_osc_ucx_la_LDFLAGS = -module -avoid-version $(osc_ucx_LDFLAGS)
 
 noinst_LTLIBRARIES = $(component_noinst)
diff --git a/ompi/mca/osc/ucx/configure.m4 b/ompi/mca/osc/ucx/configure.m4
index 72f5527d97b..9dcc93adde2 100644
--- a/ompi/mca/osc/ucx/configure.m4
+++ b/ompi/mca/osc/ucx/configure.m4
@@ -1,6 +1,7 @@
 # -*- shell-script -*-
 #
 # Copyright (C) Mellanox Technologies Ltd. 2001-2017.  ALL RIGHTS RESERVED.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -25,6 +26,9 @@ AC_DEFUN([MCA_ompi_osc_ucx_CONFIG],[
                    [osc_ucx_happy="yes"],
                    [osc_ucx_happy="no"])
 
+    AS_IF([test "${osc_ucx_happy}" = "yes"],
+          [OPAL_MCA_CHECK_DEPENDENCY([ompi], [osc], [ucx], [opal], [common], [ucx])])
+
     AS_IF([test "$osc_ucx_happy" = "yes"],
           [$1],
           [$2])
diff --git a/ompi/mca/osc/ucx/osc_ucx.h b/ompi/mca/osc/ucx/osc_ucx.h
index 3265dc2fbc2..1c349f30592 100644
--- a/ompi/mca/osc/ucx/osc_ucx.h
+++ b/ompi/mca/osc/ucx/osc_ucx.h
@@ -16,30 +16,48 @@
 #include "ompi/communicator/communicator.h"
 #include "opal/mca/common/ucx/common_ucx.h"
 #include "opal/mca/common/ucx/common_ucx_wpool.h"
+#include "opal/mca/shmem/shmem.h"
+#include "opal/mca/shmem/base/base.h"
 
 #define OSC_UCX_ASSERT  MCA_COMMON_UCX_ASSERT
 #define OSC_UCX_ERROR   MCA_COMMON_UCX_ERROR
 #define OSC_UCX_VERBOSE MCA_COMMON_UCX_VERBOSE
 
 #define OMPI_OSC_UCX_POST_PEER_MAX 32
-#define OMPI_OSC_UCX_ATTACH_MAX    32
+#define OMPI_OSC_UCX_ATTACH_MAX    48
 #define OMPI_OSC_UCX_MEM_ADDR_MAX_LEN  1024
 
+
 typedef struct ompi_osc_ucx_component {
     ompi_osc_base_component_t super;
     opal_common_ucx_wpool_t *wpool;
     bool enable_mpi_threads;
     opal_free_list_t requests; /* request free list for the r* communication variants */
+    opal_free_list_t accumulate_requests; /* request free list for the r* communication variants */
     bool env_initialized; /* UCX environment is initialized or not */
-    int num_incomplete_req_ops;
+    bool priority_is_set; /* Is ucp_ctx created and component priority has been set */
+    int comm_world_size;
+    ucp_ep_h *endpoints;
     int num_modules;
     bool no_locks; /* Default value of the no_locks info key for new windows */
     bool acc_single_intrinsic;
     unsigned int priority;
+    /* directory where to place backing files */
+    char *backing_directory;
 } ompi_osc_ucx_component_t;
 
 OMPI_DECLSPEC extern ompi_osc_ucx_component_t mca_osc_ucx_component;
 
+#define OSC_UCX_INCREMENT_OUTSTANDING_NB_OPS(_module)                               \
+    do {                                                                            \
+        opal_atomic_add_fetch_64(&_module->ctx->num_incomplete_req_ops, 1);         \
+    } while(0);
+
+#define OSC_UCX_DECREMENT_OUTSTANDING_NB_OPS(_module)                               \
+    do {                                                                            \
+        opal_atomic_add_fetch_64(&_module->ctx->num_incomplete_req_ops, -1);        \
+    } while(0);
+
 typedef enum ompi_osc_ucx_epoch {
     NONE_EPOCH,
     FENCE_EPOCH,
@@ -65,7 +83,8 @@ typedef struct ompi_osc_ucx_epoch_type {
 #define OSC_UCX_STATE_COMPLETE_COUNT_OFFSET (sizeof(uint64_t) * 3)
 #define OSC_UCX_STATE_POST_INDEX_OFFSET (sizeof(uint64_t) * 4)
 #define OSC_UCX_STATE_POST_STATE_OFFSET (sizeof(uint64_t) * 5)
-#define OSC_UCX_STATE_DYNAMIC_WIN_CNT_OFFSET (sizeof(uint64_t) * (5 + OMPI_OSC_UCX_POST_PEER_MAX))
+#define OSC_UCX_STATE_DYNAMIC_LOCK_OFFSET (sizeof(uint64_t) * (5 + OMPI_OSC_UCX_POST_PEER_MAX))
+#define OSC_UCX_STATE_DYNAMIC_WIN_CNT_OFFSET (sizeof(uint64_t) * (6 + OMPI_OSC_UCX_POST_PEER_MAX))
 
 typedef struct ompi_osc_dynamic_win_info {
     uint64_t base;
@@ -87,10 +106,16 @@ typedef struct ompi_osc_ucx_state {
     volatile uint64_t complete_count; /* # msgs received from complete processes */
     volatile uint64_t post_index;
     volatile uint64_t post_state[OMPI_OSC_UCX_POST_PEER_MAX];
+    volatile uint64_t dynamic_lock;
     volatile uint64_t dynamic_win_count;
     volatile ompi_osc_dynamic_win_info_t dynamic_wins[OMPI_OSC_UCX_ATTACH_MAX];
 } ompi_osc_ucx_state_t;
 
+typedef struct ompi_osc_ucx_mem_ranges {
+    uint64_t base;
+    uint64_t tail;
+} ompi_osc_ucx_mem_ranges_t;
+
 typedef struct ompi_osc_ucx_module {
     ompi_osc_base_module_t super;
     struct ompi_communicator_t *comm;
@@ -98,10 +123,11 @@ typedef struct ompi_osc_ucx_module {
     size_t size;
     uint64_t *addrs;
     uint64_t *state_addrs;
-    int disp_unit; /* if disp_unit >= 0, then everyone has the same
-                    * disp unit size; if disp_unit == -1, then we
-                    * need to look at disp_units */
-    int *disp_units;
+    uint64_t *comm_world_ranks;
+    ptrdiff_t disp_unit; /* if disp_unit >= 0, then everyone has the same
+                          * disp unit size; if disp_unit == -1, then we
+                          * need to look at disp_units */
+    ptrdiff_t *disp_units;
 
     ompi_osc_ucx_state_t state; /* remote accessible flags */
     ompi_osc_local_dynamic_win_info_t local_dynamic_win_info[OMPI_OSC_UCX_ATTACH_MAX];
@@ -120,6 +146,16 @@ typedef struct ompi_osc_ucx_module {
     opal_common_ucx_ctx_t *ctx;
     opal_common_ucx_wpmem_t *mem;
     opal_common_ucx_wpmem_t *state_mem;
+    ompi_osc_ucx_mem_ranges_t *epoc_outstanding_ops_mems;
+    bool skip_sync_check;
+    bool noncontig_shared_win;
+    size_t *sizes;
+    /* in shared windows, shmem_addrs can be used for direct load store to
+     * remote windows */
+    uint64_t *shmem_addrs;
+    void *segment_base;
+    /** opal shared memory structure for the shared memory segment */
+    opal_shmem_ds_t seg_ds;
 } ompi_osc_ucx_module_t;
 
 typedef enum locktype {
@@ -134,24 +170,40 @@ typedef struct ompi_osc_ucx_lock {
     bool is_nocheck;
 } ompi_osc_ucx_lock_t;
 
-#define OSC_UCX_GET_EP(comm_, rank_) (ompi_comm_peer_lookup(comm_, rank_)->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_UCX])
+#define OSC_UCX_GET_EP(_module, rank_) (mca_osc_ucx_component.endpoints[_module->comm_world_ranks[rank_]])
 #define OSC_UCX_GET_DISP(module_, rank_) ((module_->disp_unit < 0) ? module_->disp_units[rank_] : module_->disp_unit)
 
+#define OSC_UCX_GET_DEFAULT_EP(_ep_ptr, _module, _target)                   \
+    if (opal_common_ucx_thread_enabled) {                  \
+        _ep_ptr = NULL;                                                     \
+    } else {                                                                \
+        _ep_ptr = (ucp_ep_h *)&(OSC_UCX_GET_EP(_module, _target));          \
+    }
+
+extern size_t ompi_osc_ucx_outstanding_ops_flush_threshold;
+
+int ompi_osc_ucx_shared_query(struct ompi_win_t *win, int rank, size_t *size,
+        ptrdiff_t *disp_unit, void * baseptr);
 int ompi_osc_ucx_win_attach(struct ompi_win_t *win, void *base, size_t len);
 int ompi_osc_ucx_win_detach(struct ompi_win_t *win, const void *base);
 int ompi_osc_ucx_free(struct ompi_win_t *win);
 
-int ompi_osc_ucx_put(const void *origin_addr, int origin_count,
+int ompi_osc_ucx_put(const void *origin_addr, size_t origin_count,
                      struct ompi_datatype_t *origin_dt,
-                     int target, ptrdiff_t target_disp, int target_count,
+                     int target, ptrdiff_t target_disp, size_t target_count,
                      struct ompi_datatype_t *target_dt, struct ompi_win_t *win);
-int ompi_osc_ucx_get(void *origin_addr, int origin_count,
+int ompi_osc_ucx_get(void *origin_addr, size_t origin_count,
                      struct ompi_datatype_t *origin_dt,
-                     int target, ptrdiff_t target_disp, int target_count,
+                     int target, ptrdiff_t target_disp, size_t target_count,
                      struct ompi_datatype_t *target_dt, struct ompi_win_t *win);
-int ompi_osc_ucx_accumulate(const void *origin_addr, int origin_count,
+int ompi_osc_ucx_accumulate(const void *origin_addr, size_t origin_count,
                             struct ompi_datatype_t *origin_dt,
-                            int target, ptrdiff_t target_disp, int target_count,
+                            int target, ptrdiff_t target_disp, size_t target_count,
+                            struct ompi_datatype_t *target_dt,
+                            struct ompi_op_t *op, struct ompi_win_t *win);
+int ompi_osc_ucx_accumulate_nb(const void *origin_addr, size_t origin_count,
+                            struct ompi_datatype_t *origin_dt,
+                            int target, ptrdiff_t target_disp, size_t target_count,
                             struct ompi_datatype_t *target_dt,
                             struct ompi_op_t *op, struct ompi_win_t *win);
 int ompi_osc_ucx_compare_and_swap(const void *origin_addr, const void *compare_addr,
@@ -162,47 +214,54 @@ int ompi_osc_ucx_fetch_and_op(const void *origin_addr, void *result_addr,
                               struct ompi_datatype_t *dt, int target,
                               ptrdiff_t target_disp, struct ompi_op_t *op,
                               struct ompi_win_t *win);
-int ompi_osc_ucx_get_accumulate(const void *origin_addr, int origin_count,
+int ompi_osc_ucx_get_accumulate(const void *origin_addr, size_t origin_count,
+                                struct ompi_datatype_t *origin_datatype,
+                                void *result_addr, size_t result_count,
+                                struct ompi_datatype_t *result_datatype,
+                                int target_rank, ptrdiff_t target_disp,
+                                size_t target_count, struct ompi_datatype_t *target_datatype,
+                                struct ompi_op_t *op, struct ompi_win_t *win);
+int ompi_osc_ucx_get_accumulate_nb(const void *origin_addr, size_t origin_count,
                                 struct ompi_datatype_t *origin_datatype,
-                                void *result_addr, int result_count,
+                                void *result_addr, size_t result_count,
                                 struct ompi_datatype_t *result_datatype,
                                 int target_rank, ptrdiff_t target_disp,
-                                int target_count, struct ompi_datatype_t *target_datatype,
+                                size_t target_count, struct ompi_datatype_t *target_datatype,
                                 struct ompi_op_t *op, struct ompi_win_t *win);
-int ompi_osc_ucx_rput(const void *origin_addr, int origin_count,
+int ompi_osc_ucx_rput(const void *origin_addr, size_t origin_count,
                       struct ompi_datatype_t *origin_dt,
-                      int target, ptrdiff_t target_disp, int target_count,
+                      int target, ptrdiff_t target_disp, size_t target_count,
                       struct ompi_datatype_t *target_dt,
                       struct ompi_win_t *win, struct ompi_request_t **request);
-int ompi_osc_ucx_rget(void *origin_addr, int origin_count,
+int ompi_osc_ucx_rget(void *origin_addr, size_t origin_count,
                       struct ompi_datatype_t *origin_dt,
-                      int target, ptrdiff_t target_disp, int target_count,
+                      int target, ptrdiff_t target_disp, size_t target_count,
                       struct ompi_datatype_t *target_dt, struct ompi_win_t *win,
                       struct ompi_request_t **request);
-int ompi_osc_ucx_raccumulate(const void *origin_addr, int origin_count,
+int ompi_osc_ucx_raccumulate(const void *origin_addr, size_t origin_count,
                              struct ompi_datatype_t *origin_dt,
-                             int target, ptrdiff_t target_disp, int target_count,
+                             int target, ptrdiff_t target_disp, size_t target_count,
                              struct ompi_datatype_t *target_dt, struct ompi_op_t *op,
                              struct ompi_win_t *win, struct ompi_request_t **request);
-int ompi_osc_ucx_rget_accumulate(const void *origin_addr, int origin_count,
+int ompi_osc_ucx_rget_accumulate(const void *origin_addr, size_t origin_count,
                                  struct ompi_datatype_t *origin_datatype,
-                                 void *result_addr, int result_count,
+                                 void *result_addr, size_t result_count,
                                  struct ompi_datatype_t *result_datatype,
-                                 int target_rank, ptrdiff_t target_disp, int target_count,
+                                 int target_rank, ptrdiff_t target_disp, size_t target_count,
                                  struct ompi_datatype_t *target_datatype,
                                  struct ompi_op_t *op, struct ompi_win_t *win,
                                  struct ompi_request_t **request);
 
-int ompi_osc_ucx_fence(int assert, struct ompi_win_t *win);
-int ompi_osc_ucx_start(struct ompi_group_t *group, int assert, struct ompi_win_t *win);
+int ompi_osc_ucx_fence(int mpi_assert, struct ompi_win_t *win);
+int ompi_osc_ucx_start(struct ompi_group_t *group, int mpi_assert, struct ompi_win_t *win);
 int ompi_osc_ucx_complete(struct ompi_win_t *win);
-int ompi_osc_ucx_post(struct ompi_group_t *group, int assert, struct ompi_win_t *win);
+int ompi_osc_ucx_post(struct ompi_group_t *group, int mpi_assert, struct ompi_win_t *win);
 int ompi_osc_ucx_wait(struct ompi_win_t *win);
 int ompi_osc_ucx_test(struct ompi_win_t *win, int *flag);
 
-int ompi_osc_ucx_lock(int lock_type, int target, int assert, struct ompi_win_t *win);
+int ompi_osc_ucx_lock(int lock_type, int target, int mpi_assert, struct ompi_win_t *win);
 int ompi_osc_ucx_unlock(int target, struct ompi_win_t *win);
-int ompi_osc_ucx_lock_all(int assert, struct ompi_win_t *win);
+int ompi_osc_ucx_lock_all(int mpi_assert, struct ompi_win_t *win);
 int ompi_osc_ucx_unlock_all(struct ompi_win_t *win);
 int ompi_osc_ucx_sync(struct ompi_win_t *win);
 int ompi_osc_ucx_flush(int target, struct ompi_win_t *win);
@@ -213,5 +272,7 @@ int ompi_osc_ucx_flush_local_all(struct ompi_win_t *win);
 int ompi_osc_find_attached_region_position(ompi_osc_dynamic_win_info_t *dynamic_wins,
                                            int min_index, int max_index,
                                            uint64_t base, size_t len, int *insert);
+int ompi_osc_ucx_dynamic_lock(ompi_osc_ucx_module_t *module, int target);
+int ompi_osc_ucx_dynamic_unlock(ompi_osc_ucx_module_t *module, int target);
 
 #endif /* OMPI_OSC_UCX_H */
diff --git a/ompi/mca/osc/ucx/osc_ucx_active_target.c b/ompi/mca/osc/ucx/osc_ucx_active_target.c
index 1c1aff65c16..57084e6ee7c 100644
--- a/ompi/mca/osc/ucx/osc_ucx_active_target.c
+++ b/ompi/mca/osc/ucx/osc_ucx_active_target.c
@@ -58,7 +58,7 @@ static inline void ompi_osc_ucx_handle_incoming_post(ompi_osc_ucx_module_t *modu
     opal_list_append(&module->pending_posts, &pending_post->super);
 }
 
-int ompi_osc_ucx_fence(int assert, struct ompi_win_t *win) {
+int ompi_osc_ucx_fence(int mpi_assert, struct ompi_win_t *win) {
     ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module;
     int ret = OMPI_SUCCESS;
 
@@ -67,14 +67,14 @@ int ompi_osc_ucx_fence(int assert, struct ompi_win_t *win) {
         return OMPI_ERR_RMA_SYNC;
     }
 
-    if (assert & MPI_MODE_NOSUCCEED) {
+    if (mpi_assert & MPI_MODE_NOSUCCEED) {
         module->epoch_type.access = NONE_EPOCH;
     } else {
         module->epoch_type.access = FENCE_EPOCH;
     }
 
-    if (!(assert & MPI_MODE_NOPRECEDE)) {
-        ret = opal_common_ucx_wpmem_flush(module->mem, OPAL_COMMON_UCX_SCOPE_WORKER, 0/*ignore*/);
+    if (!(mpi_assert & MPI_MODE_NOPRECEDE)) {
+        ret = opal_common_ucx_ctx_flush(module->ctx, OPAL_COMMON_UCX_SCOPE_WORKER, 0/*ignore*/);
         if (ret != OMPI_SUCCESS) {
             return ret;
         }
@@ -84,7 +84,7 @@ int ompi_osc_ucx_fence(int assert, struct ompi_win_t *win) {
                                               module->comm->c_coll->coll_barrier_module);
 }
 
-int ompi_osc_ucx_start(struct ompi_group_t *group, int assert, struct ompi_win_t *win) {
+int ompi_osc_ucx_start(struct ompi_group_t *group, int mpi_assert, struct ompi_win_t *win) {
     ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module;
     int i, size, *ranks_in_grp = NULL, *ranks_in_win_grp = NULL;
     ompi_group_t *win_group = NULL;
@@ -123,7 +123,7 @@ int ompi_osc_ucx_start(struct ompi_group_t *group, int assert, struct ompi_win_t
         return OMPI_ERROR;
     }
 
-    if ((assert & MPI_MODE_NOCHECK) == 0) {
+    if ((mpi_assert & MPI_MODE_NOCHECK) == 0) {
         ompi_osc_ucx_pending_post_t *pending_post, *next;
 
         /* first look through the pending list */
@@ -165,31 +165,33 @@ int ompi_osc_ucx_complete(struct ompi_win_t *win) {
     ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module;
     int i, size;
     int ret = OMPI_SUCCESS;
+    ucp_ep_h *ep;
 
     if (module->epoch_type.access != START_COMPLETE_EPOCH) {
         return OMPI_ERR_RMA_SYNC;
     }
 
-    module->epoch_type.access = NONE_EPOCH;
-
-    ret = opal_common_ucx_wpmem_flush(module->mem, OPAL_COMMON_UCX_SCOPE_WORKER, 0/*ignore*/);
+    ret = opal_common_ucx_ctx_flush(module->ctx, OPAL_COMMON_UCX_SCOPE_WORKER, 0/*ignore*/);
     if (ret != OMPI_SUCCESS) {
         return ret;
     }
 
+    module->epoch_type.access = NONE_EPOCH;
+
     size = ompi_group_size(module->start_group);
     for (i = 0; i < size; i++) {
         uint64_t remote_addr = module->state_addrs[module->start_grp_ranks[i]] + OSC_UCX_STATE_COMPLETE_COUNT_OFFSET; // write to state.complete_count on remote side
 
-        ret = opal_common_ucx_wpmem_post(module->mem, UCP_ATOMIC_POST_OP_ADD,
+        OSC_UCX_GET_DEFAULT_EP(ep, module, module->start_grp_ranks[i]);
+
+        ret = opal_common_ucx_wpmem_post(module->state_mem, UCP_ATOMIC_POST_OP_ADD,
                                        1, module->start_grp_ranks[i], sizeof(uint64_t),
-                                       remote_addr);
+                                       remote_addr, ep);
         if (ret != OMPI_SUCCESS) {
             OSC_UCX_VERBOSE(1, "opal_common_ucx_mem_post failed: %d", ret);
         }
 
-        ret = opal_common_ucx_wpmem_flush(module->mem, OPAL_COMMON_UCX_SCOPE_EP,
-                                        module->start_grp_ranks[i]);
+        ret = opal_common_ucx_ctx_flush(module->ctx, OPAL_COMMON_UCX_SCOPE_EP, module->start_grp_ranks[i]);
         if (ret != OMPI_SUCCESS) {
             return ret;
         }
@@ -202,8 +204,9 @@ int ompi_osc_ucx_complete(struct ompi_win_t *win) {
     return ret;
 }
 
-int ompi_osc_ucx_post(struct ompi_group_t *group, int assert, struct ompi_win_t *win) {
+int ompi_osc_ucx_post(struct ompi_group_t *group, int mpi_assert, struct ompi_win_t *win) {
     ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module;
+    ucp_ep_h *ep;
     int ret = OMPI_SUCCESS;
 
     if (module->epoch_type.exposure != NONE_EPOCH) {
@@ -213,7 +216,7 @@ int ompi_osc_ucx_post(struct ompi_group_t *group, int assert, struct ompi_win_t
     OBJ_RETAIN(group);
     module->post_group = group;
 
-    if ((assert & MPI_MODE_NOCHECK) == 0) {
+    if ((mpi_assert & MPI_MODE_NOCHECK) == 0) {
         int i, j, size;
         ompi_group_t *win_group = NULL;
         int *ranks_in_grp = NULL, *ranks_in_win_grp = NULL;
@@ -243,10 +246,13 @@ int ompi_osc_ucx_post(struct ompi_group_t *group, int assert, struct ompi_win_t
             uint64_t remote_addr = module->state_addrs[ranks_in_win_grp[i]] + OSC_UCX_STATE_POST_INDEX_OFFSET; // write to state.post_index on remote side
             uint64_t curr_idx = 0, result = 0;
 
+            OSC_UCX_GET_DEFAULT_EP(ep, module, ranks_in_win_grp[i]);
+
             /* do fop first to get an post index */
-            ret = opal_common_ucx_wpmem_fetch(module->mem, UCP_ATOMIC_FETCH_OP_FADD,
+            ret = opal_common_ucx_wpmem_fetch(module->state_mem, UCP_ATOMIC_FETCH_OP_FADD,
                                             1, ranks_in_win_grp[i], &result,
-                                            sizeof(result), remote_addr);
+                                            sizeof(result), remote_addr, ep);
+
             if (ret != OMPI_SUCCESS) {
                 ret = OMPI_ERROR;
                 goto cleanup;
@@ -258,9 +264,12 @@ int ompi_osc_ucx_post(struct ompi_group_t *group, int assert, struct ompi_win_t
 
             /* do cas to send post message */
             do {
-                ret = opal_common_ucx_wpmem_cmpswp(module->mem, 0, result,
-                                                 myrank + 1, &result, sizeof(result),
-                                                 remote_addr);
+
+                result =  myrank + 1;
+                ret = opal_common_ucx_wpmem_cmpswp(module->state_mem, 0, result,
+                                                 ranks_in_win_grp[i], &result, sizeof(result),
+                                                 remote_addr, ep);
+
                 if (ret != OMPI_SUCCESS) {
                     ret = OMPI_ERROR;
                     goto cleanup;
diff --git a/ompi/mca/osc/ucx/osc_ucx_comm.c b/ompi/mca/osc/ucx/osc_ucx_comm.c
index 72c1fdf92fd..ab122e67263 100644
--- a/ompi/mca/osc/ucx/osc_ucx_comm.c
+++ b/ompi/mca/osc/ucx/osc_ucx_comm.c
@@ -1,7 +1,8 @@
 /*
  * Copyright (C) 2001-2017 Mellanox Technologies Ltd. ALL RIGHTS RESERVED.
- * Copyright (c) 2019-2020 High Performance Computing Center Stuttgart,
+ * Copyright (c) 2019-2022 High Performance Computing Center Stuttgart,
  *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2021      IBM Corporation. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -26,18 +27,26 @@
         return OMPI_ERROR;                                                       \
     }
 
-/* macro to check whether UCX supports atomic operation on the size the operands */
-#define ATOMIC_SIZE_SUPPORTED(_remote_addr, _size) \
-    ((sizeof(uint32_t) == (_size) && !((_remote_addr) & 0x3)) || \
-     (sizeof(uint64_t) == (_size) && !((_remote_addr) & 0x7)))
+#define CHECK_DYNAMIC_WIN(_remote_addr, _module, _target, _ret)                         \
+    if (_module->flavor == MPI_WIN_FLAVOR_DYNAMIC) {                                    \
+        _ret = get_dynamic_win_info(_remote_addr, _module, _target);                    \
+        if (_ret != OMPI_SUCCESS) {                                                     \
+            return _ret;                                                                \
+        }                                                                               \
+    }
 
 typedef struct ucx_iovec {
     void *addr;
     size_t len;
 } ucx_iovec_t;
 
+size_t ompi_osc_ucx_outstanding_ops_flush_threshold = 64;
+
 static inline int check_sync_state(ompi_osc_ucx_module_t *module, int target,
                                    bool is_req_ops) {
+
+    if (module->skip_sync_check) return OMPI_SUCCESS;
+
     if (is_req_ops == false) {
         if (module->epoch_type.access == NONE_EPOCH) {
             return OMPI_ERR_RMA_SYNC;
@@ -124,12 +133,14 @@ static inline int create_iov_list(const void *addr, int count, ompi_datatype_t *
 }
 
 static inline int ddt_put_get(ompi_osc_ucx_module_t *module,
-                              const void *origin_addr, int origin_count,
+                              const void *origin_addr, size_t origin_count,
                               struct ompi_datatype_t *origin_dt,
                               bool is_origin_contig, ptrdiff_t origin_lb,
                               int target, uint64_t remote_addr,
-                              int target_count, struct ompi_datatype_t *target_dt,
+                              size_t target_count, struct ompi_datatype_t *target_dt,
                               bool is_target_contig, ptrdiff_t target_lb, bool is_get) {
+    ucp_ep_h *ep;
+    OSC_UCX_GET_DEFAULT_EP(ep, module, target);
     ucx_iovec_t *origin_ucx_iov = NULL, *target_ucx_iov = NULL;
     uint32_t origin_ucx_iov_count = 0, target_ucx_iov_count = 0;
     uint32_t origin_ucx_iov_idx = 0, target_ucx_iov_idx = 0;
@@ -165,7 +176,7 @@ static inline int ddt_put_get(ompi_osc_ucx_module_t *module,
             }
             status = opal_common_ucx_wpmem_putget(module->mem, op, target,
                                                 origin_ucx_iov[origin_ucx_iov_idx].addr, curr_len,
-                                                remote_addr + (uint64_t)(target_ucx_iov[target_ucx_iov_idx].addr));
+                                                remote_addr + (uint64_t)(target_ucx_iov[target_ucx_iov_idx].addr), ep);
             if (OPAL_SUCCESS != status) {
                 OSC_UCX_VERBOSE(1, "opal_common_ucx_mem_putget failed: %d", status);
                 ret = OMPI_ERROR;
@@ -200,7 +211,7 @@ static inline int ddt_put_get(ompi_osc_ucx_module_t *module,
             status = opal_common_ucx_wpmem_putget(module->mem, op, target,
                                                 origin_ucx_iov[origin_ucx_iov_idx].addr,
                                                 origin_ucx_iov[origin_ucx_iov_idx].len,
-                                                remote_addr + target_lb + prev_len);
+                                                remote_addr + target_lb + prev_len, ep);
             if (OPAL_SUCCESS != status) {
                 OSC_UCX_VERBOSE(1, "opal_common_ucx_mem_putget failed: %d", status);
                 ret = OMPI_ERROR;
@@ -223,7 +234,7 @@ static inline int ddt_put_get(ompi_osc_ucx_module_t *module,
             status = opal_common_ucx_wpmem_putget(module->mem, op, target,
                                                 (void *)((intptr_t)origin_addr + origin_lb + prev_len),
                                                 target_ucx_iov[target_ucx_iov_idx].len,
-                                                remote_addr + (uint64_t)(target_ucx_iov[target_ucx_iov_idx].addr));
+                                                remote_addr + (uint64_t)(target_ucx_iov[target_ucx_iov_idx].addr), ep);
             if (OPAL_SUCCESS != status) {
                 OSC_UCX_VERBOSE(1, "opal_common_ucx_mem_putget failed: %d", status);
                 ret = OMPI_ERROR;
@@ -236,7 +247,6 @@ static inline int ddt_put_get(ompi_osc_ucx_module_t *module,
     }
 
 cleanup:
-
     if (origin_ucx_iov != NULL) {
         free(origin_ucx_iov);
     }
@@ -247,135 +257,150 @@ static inline int ddt_put_get(ompi_osc_ucx_module_t *module,
     return ret;
 }
 
-static inline bool need_acc_lock(ompi_osc_ucx_module_t *module, int target)
-{
-    ompi_osc_ucx_lock_t *lock = NULL;
-    opal_hash_table_get_value_uint32(&module->outstanding_locks,
-                                     (uint32_t) target, (void **) &lock);
-
-    /* if there is an exclusive lock there is no need to acqurie the accumulate lock */
-    return !(NULL != lock && LOCK_EXCLUSIVE == lock->type);
-}
-
-static inline int start_atomicity(
-    ompi_osc_ucx_module_t *module,
-    int                    target,
-    bool                  *lock_acquired) {
-    uint64_t result_value = -1;
-    uint64_t remote_addr = (module->state_addrs)[target] + OSC_UCX_STATE_ACC_LOCK_OFFSET;
-    int ret = OMPI_SUCCESS;
-
-    if (need_acc_lock(module, target)) {
-        for (;;) {
-            ret = opal_common_ucx_wpmem_cmpswp(module->state_mem,
-                                            TARGET_LOCK_UNLOCKED, TARGET_LOCK_EXCLUSIVE,
-                                            target, &result_value, sizeof(result_value),
-                                            remote_addr);
-            if (ret != OMPI_SUCCESS) {
-                OSC_UCX_VERBOSE(1, "opal_common_ucx_mem_cmpswp failed: %d", ret);
-                return OMPI_ERROR;
-            }
-            if (result_value == TARGET_LOCK_UNLOCKED) {
-                break;
-            }
-
-            opal_common_ucx_wpool_progress(mca_osc_ucx_component.wpool);
-        }
-
-        *lock_acquired = true;
-    } else {
-        *lock_acquired = false;
-    }
-
-    return OMPI_SUCCESS;
-}
-
-static inline int end_atomicity(
-    ompi_osc_ucx_module_t *module,
-    int                    target,
-    bool                   lock_acquired,
-    void                  *free_ptr) {
-    uint64_t remote_addr = (module->state_addrs)[target] + OSC_UCX_STATE_ACC_LOCK_OFFSET;
-    int ret = OMPI_SUCCESS;
-
-    if (lock_acquired) {
-        uint64_t result_value = 0;
-        /* fence any still active operations */
-        ret = opal_common_ucx_wpmem_fence(module->mem);
-        if (ret != OMPI_SUCCESS) {
-            OSC_UCX_VERBOSE(1, "opal_common_ucx_mem_fence failed: %d", ret);
-            return OMPI_ERROR;
-        }
-
-        ret = opal_common_ucx_wpmem_fetch(module->state_mem,
-                                        UCP_ATOMIC_FETCH_OP_SWAP, TARGET_LOCK_UNLOCKED,
-                                        target, &result_value, sizeof(result_value),
-                                        remote_addr);
-        assert(result_value == TARGET_LOCK_EXCLUSIVE);
-    } else if (NULL != free_ptr){
-        /* flush before freeing the buffer */
-        ret = opal_common_ucx_wpmem_flush(module->state_mem, OPAL_COMMON_UCX_SCOPE_EP, target);
-    }
-    /* TODO: encapsulate in a request and make the release non-blocking */
-    if (NULL != free_ptr) {
-        free(free_ptr);
-    }
-    if (ret != OMPI_SUCCESS) {
-        OSC_UCX_VERBOSE(1, "opal_common_ucx_mem_fetch failed: %d", ret);
-        return OMPI_ERROR;
-    }
-
-    return ret;
-}
-
-static inline int get_dynamic_win_info(uint64_t remote_addr, ompi_osc_ucx_module_t *module,
-                                       int target) {
+static inline int get_dynamic_win_info(uint64_t remote_addr,
+                                        ompi_osc_ucx_module_t *module, int target) {
+    ucp_ep_h *ep;
+    OSC_UCX_GET_DEFAULT_EP(ep, module, target);
     uint64_t remote_state_addr = (module->state_addrs)[target] + OSC_UCX_STATE_DYNAMIC_WIN_CNT_OFFSET;
-    size_t len = sizeof(uint64_t) + sizeof(ompi_osc_dynamic_win_info_t) * OMPI_OSC_UCX_ATTACH_MAX;
-    char *temp_buf = malloc(len);
+    size_t remote_state_len = sizeof(uint64_t) + sizeof(ompi_osc_dynamic_win_info_t) * OMPI_OSC_UCX_ATTACH_MAX;
+    char *temp_buf = NULL;
     ompi_osc_dynamic_win_info_t *temp_dynamic_wins;
+    int contain = 0;
     uint64_t win_count;
-    int contain, insert = -1;
+    int insert = -1;
     int ret;
+    int ret_unlock;
+
+    /* We need to lock dyn-lock even if the process has an exclusive lock.
+     * Remote process protects its window attach/detach operations with a
+     * dynamic lock */
+    ret = ompi_osc_ucx_dynamic_lock(module, target);
+    if (ret != OPAL_SUCCESS) {
+        return OMPI_ERROR;
+    }
+
+    temp_buf = calloc(remote_state_len, 1);
+    if (NULL == temp_buf) {
+        ret = OMPI_ERR_OUT_OF_RESOURCE;
+        goto cleanup;
+    }
 
     ret = opal_common_ucx_wpmem_putget(module->state_mem, OPAL_COMMON_UCX_GET, target,
                                        (void *)((intptr_t)temp_buf),
-                                       len, remote_state_addr);
+                                       remote_state_len, remote_state_addr, ep);
     if (OPAL_SUCCESS != ret) {
         OSC_UCX_VERBOSE(1, "opal_common_ucx_mem_putget failed: %d", ret);
         ret = OMPI_ERROR;
         goto cleanup;
     }
 
-    ret = opal_common_ucx_wpmem_flush(module->state_mem, OPAL_COMMON_UCX_SCOPE_EP, target);
+    ret = opal_common_ucx_ctx_flush(module->ctx, OPAL_COMMON_UCX_SCOPE_EP, target);
     if (ret != OPAL_SUCCESS) {
         ret = OMPI_ERROR;
         goto cleanup;
     }
 
     memcpy(&win_count, temp_buf, sizeof(uint64_t));
-    assert(win_count > 0 && win_count <= OMPI_OSC_UCX_ATTACH_MAX);
+    if (win_count > OMPI_OSC_UCX_ATTACH_MAX) {
+        ret = MPI_ERR_RMA_RANGE;
+        goto cleanup;
+    }
 
     temp_dynamic_wins = (ompi_osc_dynamic_win_info_t *)(temp_buf + sizeof(uint64_t));
-    contain = ompi_osc_find_attached_region_position(temp_dynamic_wins, 0, win_count,
+    contain = ompi_osc_find_attached_region_position(temp_dynamic_wins, 0, win_count - 1,
                                                      remote_addr, 1, &insert);
-    assert(contain >= 0 && (uint64_t)contain < win_count);
+    if (contain < 0 || contain >= (int)win_count) {
+        OSC_UCX_ERROR("Dynamic window index not found contain: %d win_count: %" PRIu64 "\n",
+                contain, win_count);
+        ret = MPI_ERR_RMA_RANGE;
+        goto cleanup;
+    }
+    
+    assert(module->mem != NULL);
 
-    if (module->local_dynamic_win_info[contain].mem->mem_addrs == NULL) {
-        module->local_dynamic_win_info[contain].mem->mem_addrs = calloc(ompi_comm_size(module->comm),
-                                                                        OMPI_OSC_UCX_MEM_ADDR_MAX_LEN);
-        module->local_dynamic_win_info[contain].mem->mem_displs =calloc(ompi_comm_size(module->comm),
-                                                                        sizeof(int));
+    _mem_record_t *mem_rec = NULL;
+    ret = opal_tsd_tracked_key_get(&module->mem->tls_key, (void **) &mem_rec);
+    if (OPAL_SUCCESS != ret) {
+        ret = OMPI_ERROR;
+        goto cleanup;
     }
+    
+     if (mem_rec == NULL) {
+        OSC_UCX_GET_DEFAULT_EP(ep, module, target);
+        ret = opal_common_ucx_tlocal_fetch_spath(module->mem, target, ep);
+        if (OPAL_SUCCESS != ret) {
+            ret = OMPI_ERROR;
+            goto cleanup;
+        }
+        ret = opal_tsd_tracked_key_get(&module->mem->tls_key, (void **) &mem_rec);
+        if (OPAL_SUCCESS != ret) {
+            ret = OMPI_ERROR;
+            goto cleanup;
+        }
 
-    memcpy(module->local_dynamic_win_info[contain].mem->mem_addrs + target * OMPI_OSC_UCX_MEM_ADDR_MAX_LEN,
-           temp_dynamic_wins[contain].mem_addr, OMPI_OSC_UCX_MEM_ADDR_MAX_LEN);
-    module->local_dynamic_win_info[contain].mem->mem_displs[target] = target * OMPI_OSC_UCX_MEM_ADDR_MAX_LEN;
+     }
+
+    assert(NULL != mem_rec && NULL != mem_rec->rkeys);
+
+    // destroy the previous rkey[target] and create a new one
+    opal_mutex_lock(&mem_rec->winfo->mutex);
+
+    if (mem_rec->rkeys[target] != NULL) {
+        ucp_rkey_destroy(mem_rec->rkeys[target]);
+        OPAL_COMMON_UCX_DEBUG_ATOMIC_ADD(opal_common_ucx_unpacked_rkey_counts, -1);
+    }
+
+    void *rkey_buffer = &temp_dynamic_wins[contain].mem_addr;
+
+    ret = ucp_ep_rkey_unpack(mem_rec->winfo->endpoints[target], rkey_buffer,
+            &mem_rec->rkeys[target]);
+
+    OPAL_COMMON_UCX_DEBUG_ATOMIC_ADD(opal_common_ucx_unpacked_rkey_counts, 1);
+
+    opal_mutex_unlock(&mem_rec->winfo->mutex);
+
+    if (ret != UCS_OK) {
+        MCA_COMMON_UCX_VERBOSE(1, "ucp_ep_rkey_unpack failed: %d", ret);
+        ret = OPAL_ERROR;
+        goto cleanup;
+    }
 
 cleanup:
     free(temp_buf);
 
-    return ret;
+    /* unlock the dynamic lock */
+    ret_unlock = ompi_osc_ucx_dynamic_unlock(module, target);
+    /* ignore unlock result in case of error */
+    return (OMPI_SUCCESS != ret) ? ret : ret_unlock;
+}
+
+static inline
+bool osc_is_atomic_dt_op_supported(
+        struct ompi_datatype_t  *dt,
+        struct ompi_op_t        *op,
+        size_t                  dt_bytes,
+        uint64_t                remote_addr)
+{
+    /* UCX atomics are only supported on 32 and 64 bit values */
+    if (!ompi_datatype_is_predefined(dt) ||
+            !ompi_osc_base_is_atomic_size_supported(remote_addr, dt_bytes)) {
+        return false;
+    }
+    /* Hardware-based atomic add for floating point is not supported */
+    else if ((
+                op == &ompi_mpi_op_no_op.op
+             || op == &ompi_mpi_op_replace.op
+             || op == &ompi_mpi_op_sum.op
+             )
+         && !(
+                op == &ompi_mpi_op_sum.op
+                && (dt == MPI_FLOAT || dt == MPI_DOUBLE
+                   || dt == MPI_LONG_DOUBLE || dt == MPI_FLOAT_INT)
+            )) {
+        return true;
+    }
+
+    return false;
 }
 
 static inline
@@ -385,29 +410,19 @@ bool use_atomic_op(
     uint64_t                remote_addr,
     struct ompi_datatype_t *origin_dt,
     struct ompi_datatype_t *target_dt,
-    int                     origin_count,
-    int                     target_count)
+    size_t                  origin_count,
+    size_t                  target_count)
 {
+    size_t origin_dt_bytes;
 
-    if (module->acc_single_intrinsic &&
-        ompi_datatype_is_predefined(origin_dt) &&
-        origin_count == 1 &&
-        (op == &ompi_mpi_op_replace.op ||
-         op == &ompi_mpi_op_sum.op     ||
-         op == &ompi_mpi_op_no_op.op)) {
-        size_t origin_dt_bytes;
-        size_t target_dt_bytes;
+    if (!module->acc_single_intrinsic || origin_count != 1 || target_count != 1
+            || origin_dt != target_dt) {
+        return false;
+    } else {
         ompi_datatype_type_size(origin_dt, &origin_dt_bytes);
-        ompi_datatype_type_size(target_dt, &target_dt_bytes);
-        /* UCX only supports 32 and 64-bit operands atm */
-        if (ATOMIC_SIZE_SUPPORTED(remote_addr, origin_dt_bytes) &&
-            origin_dt_bytes  == target_dt_bytes    &&
-            origin_count     == target_count) {
-            return true;
-        }
+        return osc_is_atomic_dt_op_supported(origin_dt, op, origin_dt_bytes,
+                remote_addr);
     }
-
-    return false;
 }
 
 static int do_atomic_op_intrinsic(
@@ -419,20 +434,18 @@ static int do_atomic_op_intrinsic(
     struct ompi_datatype_t *dt,
     ptrdiff_t               target_disp,
     void                   *result_addr,
-    ompi_osc_ucx_request_t *ucx_req)
+    ompi_osc_ucx_accumulate_request_t *ucx_req)
 {
     int ret = OMPI_SUCCESS;
     size_t origin_dt_bytes;
+    opal_common_ucx_wpmem_t *mem = module->mem;
     ompi_datatype_type_size(dt, &origin_dt_bytes);
+    ucp_ep_h *ep;
+    OSC_UCX_GET_DEFAULT_EP(ep, module, target);
 
     uint64_t remote_addr = (module->addrs[target]) + target_disp * OSC_UCX_GET_DISP(module, target);
 
-    if (module->flavor == MPI_WIN_FLAVOR_DYNAMIC) {
-        ret = get_dynamic_win_info(remote_addr, module, target);
-        if (ret != OMPI_SUCCESS) {
-            return ret;
-        }
-    }
+    CHECK_DYNAMIC_WIN(remote_addr, module, target, ret);
 
     ucp_atomic_fetch_op_t opcode;
     bool is_no_op = false;
@@ -455,11 +468,11 @@ static int do_atomic_op_intrinsic(
         uint64_t value = 0;
         if ((count - 1) == i && NULL != ucx_req) {
             // the last item is used to feed the request, if needed
-            user_req_cb = &req_completion;
+            user_req_cb = &ompi_osc_ucx_req_completion;
             user_req_ptr = ucx_req;
             // issue a fence if this is the last but not the only element
             if (0 < i) {
-                ret = opal_common_ucx_wpmem_fence(module->mem);
+                ret = opal_common_ucx_wpmem_fence(mem);
                 if (ret != OMPI_SUCCESS) {
                     OSC_UCX_VERBOSE(1, "opal_common_ucx_mem_fence failed: %d", ret);
                     return OMPI_ERROR;
@@ -471,9 +484,9 @@ static int do_atomic_op_intrinsic(
         } else {
             value = opal_common_ucx_load_uint64(origin_addr, origin_dt_bytes);
         }
-        ret = opal_common_ucx_wpmem_fetch_nb(module->mem, opcode, value, target,
+        ret = opal_common_ucx_wpmem_fetch_nb(mem, opcode, value, target,
                                              output_addr, origin_dt_bytes, remote_addr,
-                                             user_req_cb, user_req_ptr);
+                                             user_req_cb, user_req_ptr, ep);
 
         // advance origin and remote address
         origin_addr  = (void*)((intptr_t)origin_addr + origin_dt_bytes);
@@ -486,10 +499,13 @@ static int do_atomic_op_intrinsic(
     return ret;
 }
 
-int ompi_osc_ucx_put(const void *origin_addr, int origin_count, struct ompi_datatype_t *origin_dt,
-                     int target, ptrdiff_t target_disp, int target_count,
+int ompi_osc_ucx_put(const void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt,
+                     int target, ptrdiff_t target_disp, size_t target_count,
                      struct ompi_datatype_t *target_dt, struct ompi_win_t *win) {
     ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module;
+    opal_common_ucx_wpmem_t *mem = module->mem;
+    ucp_ep_h *ep;
+    OSC_UCX_GET_DEFAULT_EP(ep, module, target);
     uint64_t remote_addr = (module->addrs[target]) + target_disp * OSC_UCX_GET_DISP(module, target);
     bool is_origin_contig = false, is_target_contig = false;
     ptrdiff_t origin_lb, origin_extent, target_lb, target_extent;
@@ -500,12 +516,7 @@ int ompi_osc_ucx_put(const void *origin_addr, int origin_count, struct ompi_data
         return ret;
     }
 
-    if (module->flavor == MPI_WIN_FLAVOR_DYNAMIC) {
-        ret = get_dynamic_win_info(remote_addr, module, target);
-        if (ret != OMPI_SUCCESS) {
-            return ret;
-        }
-    }
+    CHECK_DYNAMIC_WIN(remote_addr, module, target, ret);
 
     if (!target_count) {
         return OMPI_SUCCESS;
@@ -524,9 +535,9 @@ int ompi_osc_ucx_put(const void *origin_addr, int origin_count, struct ompi_data
         ompi_datatype_type_size(origin_dt, &origin_len);
         origin_len *= origin_count;
 
-        ret = opal_common_ucx_wpmem_putget(module->mem, OPAL_COMMON_UCX_PUT, target,
+        ret = opal_common_ucx_wpmem_putget(mem, OPAL_COMMON_UCX_PUT, target,
                                          (void *)((intptr_t)origin_addr + origin_lb),
-                                         origin_len, remote_addr + target_lb);
+                                         origin_len, remote_addr + target_lb, ep);
         if (OPAL_SUCCESS != ret) {
             OSC_UCX_VERBOSE(1, "opal_common_ucx_mem_putget failed: %d", ret);
             return OMPI_ERROR;
@@ -539,11 +550,14 @@ int ompi_osc_ucx_put(const void *origin_addr, int origin_count, struct ompi_data
     }
 }
 
-int ompi_osc_ucx_get(void *origin_addr, int origin_count,
+int ompi_osc_ucx_get(void *origin_addr, size_t origin_count,
                      struct ompi_datatype_t *origin_dt,
-                     int target, ptrdiff_t target_disp, int target_count,
+                     int target, ptrdiff_t target_disp, size_t target_count,
                      struct ompi_datatype_t *target_dt, struct ompi_win_t *win) {
     ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module;
+    opal_common_ucx_wpmem_t *mem = module->mem;
+    ucp_ep_h *ep;
+    OSC_UCX_GET_DEFAULT_EP(ep, module, target);
     uint64_t remote_addr = (module->addrs[target]) + target_disp * OSC_UCX_GET_DISP(module, target);
     ptrdiff_t origin_lb, origin_extent, target_lb, target_extent;
     bool is_origin_contig = false, is_target_contig = false;
@@ -554,18 +568,12 @@ int ompi_osc_ucx_get(void *origin_addr, int origin_count,
         return ret;
     }
 
-    if (module->flavor == MPI_WIN_FLAVOR_DYNAMIC) {
-        ret = get_dynamic_win_info(remote_addr, module, target);
-        if (ret != OMPI_SUCCESS) {
-            return ret;
-        }
-    }
+    CHECK_DYNAMIC_WIN(remote_addr, module, target, ret);
 
     if (!target_count) {
         return OMPI_SUCCESS;
     }
 
-
     ompi_datatype_get_true_extent(origin_dt, &origin_lb, &origin_extent);
     ompi_datatype_get_true_extent(target_dt, &target_lb, &target_extent);
 
@@ -579,9 +587,9 @@ int ompi_osc_ucx_get(void *origin_addr, int origin_count,
         ompi_datatype_type_size(origin_dt, &origin_len);
         origin_len *= origin_count;
 
-        ret = opal_common_ucx_wpmem_putget(module->mem, OPAL_COMMON_UCX_GET, target,
+        ret = opal_common_ucx_wpmem_putget(mem, OPAL_COMMON_UCX_GET, target,
                                          (void *)((intptr_t)origin_addr + origin_lb),
-                                         origin_len, remote_addr + target_lb);
+                                         origin_len, remote_addr + target_lb, ep);
         if (OPAL_SUCCESS != ret) {
             OSC_UCX_VERBOSE(1, "opal_common_ucx_mem_putget failed: %d", ret);
             return OMPI_ERROR;
@@ -595,16 +603,100 @@ int ompi_osc_ucx_get(void *origin_addr, int origin_count,
     }
 }
 
+static inline bool ompi_osc_need_acc_lock(ompi_osc_ucx_module_t *module, int target)
+{
+    ompi_osc_ucx_lock_t *lock = NULL;
+    if (module->no_locks) {
+        return true; // if we have no locks we always need to acquire the accumulate lock
+    }
+    opal_hash_table_get_value_uint32(&module->outstanding_locks,
+                                     (uint32_t) target, (void **) &lock);
+
+    /* if there is an exclusive lock there is no need to acqurie the accumulate lock */
+    return !(NULL != lock && LOCK_EXCLUSIVE == lock->type);
+}
+
+static inline int ompi_osc_ucx_acc_lock(ompi_osc_ucx_module_t *module, int target, bool *lock_acquired) {
+    uint64_t result_value = -1;
+    uint64_t remote_addr = (module->state_addrs)[target] + OSC_UCX_STATE_ACC_LOCK_OFFSET;
+    ucp_ep_h *ep;
+    OSC_UCX_GET_DEFAULT_EP(ep, module, target);
+    int ret = OMPI_SUCCESS;
+
+    if (ompi_osc_need_acc_lock(module, target)) {
+        for (;;) {
+            ret = opal_common_ucx_wpmem_cmpswp(module->state_mem,
+                                            TARGET_LOCK_UNLOCKED, TARGET_LOCK_EXCLUSIVE,
+                                            target, &result_value, sizeof(result_value),
+                                            remote_addr, ep);
+            if (ret != OMPI_SUCCESS) {
+                OSC_UCX_VERBOSE(1, "opal_common_ucx_mem_cmpswp failed: %d", ret);
+                return OMPI_ERROR;
+            }
+            if (result_value == TARGET_LOCK_UNLOCKED) {
+                break;
+            }
+
+            opal_common_ucx_wpool_progress(mca_osc_ucx_component.wpool);
+        }
+
+        *lock_acquired = true;
+    } else {
+        *lock_acquired = false;
+    }
+
+    return OMPI_SUCCESS;
+}
+
+static inline int ompi_osc_ucx_acc_unlock(ompi_osc_ucx_module_t *module, int target,
+                                    bool lock_acquired, void *free_ptr) {
+    uint64_t remote_addr = (module->state_addrs)[target] + OSC_UCX_STATE_ACC_LOCK_OFFSET;
+    ucp_ep_h *ep;
+    OSC_UCX_GET_DEFAULT_EP(ep, module, target);
+    int ret = OMPI_SUCCESS;
+
+    if (lock_acquired) {
+        uint64_t result_value = 0;
+        /* fence any still active operations */
+        ret = opal_common_ucx_wpmem_fence(module->mem);
+        if (ret != OMPI_SUCCESS) {
+            OSC_UCX_VERBOSE(1, "opal_common_ucx_mem_fence failed: %d", ret);
+            return OMPI_ERROR;
+        }
+
+        ret = opal_common_ucx_wpmem_fetch(module->state_mem,
+                                        UCP_ATOMIC_FETCH_OP_SWAP, TARGET_LOCK_UNLOCKED,
+                                        target, &result_value, sizeof(result_value),
+                                        remote_addr, ep);
+        assert(result_value == TARGET_LOCK_EXCLUSIVE);
+    } else if (NULL != free_ptr){
+        /* flush before freeing the buffer */
+        ret = opal_common_ucx_ctx_flush(module->ctx, OPAL_COMMON_UCX_SCOPE_EP, target);
+    }
+    /* TODO: encapsulate in a request and make the release non-blocking */
+    if (NULL != free_ptr) {
+        free(free_ptr);
+    }
+    if (ret != OMPI_SUCCESS) {
+        OSC_UCX_VERBOSE(1, "opal_common_ucx_mem_fetch failed: %d", ret);
+        return OMPI_ERROR;
+    }
+
+    return ret;
+}
+
 static
-int accumulate_req(const void *origin_addr, int origin_count,
+int accumulate_req(const void *origin_addr, size_t origin_count,
                    struct ompi_datatype_t *origin_dt,
-                   int target, ptrdiff_t target_disp, int target_count,
+                   int target, ptrdiff_t target_disp, size_t target_count,
                    struct ompi_datatype_t *target_dt,
                    struct ompi_op_t *op, struct ompi_win_t *win,
-                   ompi_osc_ucx_request_t *ucx_req) {
+                   ompi_osc_ucx_accumulate_request_t *ucx_req) {
 
     ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module;
     int ret = OMPI_SUCCESS;
+    uint64_t remote_addr = (module->addrs[target]) + target_disp *
+        OSC_UCX_GET_DISP(module, target);
     void *free_ptr = NULL;
     bool lock_acquired = false;
 
@@ -624,11 +716,14 @@ int accumulate_req(const void *origin_addr, int origin_count,
                                       target_disp, NULL, ucx_req);
     }
 
-    ret = start_atomicity(module, target, &lock_acquired);
+    /* Start atomicity by acquiring acc lock  */
+    ret = ompi_osc_ucx_acc_lock(module, target, &lock_acquired);
     if (ret != OMPI_SUCCESS) {
         return ret;
     }
 
+    CHECK_DYNAMIC_WIN(remote_addr, module, target, ret);
+
     if (op == &ompi_mpi_op_replace.op) {
         ret = ompi_osc_ucx_put(origin_addr, origin_count, origin_dt, target,
                                target_disp, target_count, target_dt, win);
@@ -650,6 +745,7 @@ int accumulate_req(const void *origin_addr, int origin_count,
             if (ret != OMPI_SUCCESS) {
                 return ret;
             }
+            temp_count *= target_count;
         }
         ompi_datatype_get_true_extent(temp_dt, &temp_lb, &temp_extent);
         temp_addr = free_ptr = malloc(temp_extent * temp_count);
@@ -664,7 +760,7 @@ int accumulate_req(const void *origin_addr, int origin_count,
             return ret;
         }
 
-        ret = opal_common_ucx_wpmem_flush(module->mem, OPAL_COMMON_UCX_SCOPE_EP, target);
+        ret = opal_common_ucx_ctx_flush(module->ctx, OPAL_COMMON_UCX_SCOPE_EP, target);
         if (ret != OMPI_SUCCESS) {
             free(temp_addr);
             return ret;
@@ -724,21 +820,337 @@ int accumulate_req(const void *origin_addr, int origin_count,
 
     }
 
+    ret = opal_common_ucx_ctx_flush(module->ctx, OPAL_COMMON_UCX_SCOPE_EP, target);
+    if (ret != OPAL_SUCCESS) {
+        return ret;
+    }
+
     if (NULL != ucx_req) {
         // nothing to wait for, mark request as completed
-        ompi_request_complete(&ucx_req->super, true);
+        ompi_request_complete(&ucx_req->super.super, true);
     }
 
-    return end_atomicity(module, target, lock_acquired, free_ptr);
+    return ompi_osc_ucx_acc_unlock(module, target, lock_acquired, free_ptr);
 }
 
-int ompi_osc_ucx_accumulate(const void *origin_addr, int origin_count,
+int ompi_osc_ucx_accumulate(const void *origin_addr, size_t origin_count,
                             struct ompi_datatype_t *origin_dt,
-                            int target, ptrdiff_t target_disp, int target_count,
+                            int target, ptrdiff_t target_disp, size_t target_count,
                             struct ompi_datatype_t *target_dt,
                             struct ompi_op_t *op, struct ompi_win_t *win) {
+
     return accumulate_req(origin_addr, origin_count, origin_dt, target,
-                          target_disp, target_count, target_dt, op, win, NULL);
+                    target_disp, target_count, target_dt, op, win, NULL);
+}
+
+static inline int ompi_osc_ucx_acc_rputget(void *stage_addr, size_t stage_count,
+                    struct ompi_datatype_t *stage_dt, int target, ptrdiff_t target_disp,
+                    size_t target_count, struct ompi_datatype_t *target_dt, struct ompi_op_t
+                    *op, struct ompi_win_t *win, bool lock_acquired, const void
+                    *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, bool is_put,
+                    int phase, int acc_type) {
+    ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module;
+    ucp_ep_h *ep;
+    OSC_UCX_GET_DEFAULT_EP(ep, module, target);
+    opal_common_ucx_wpmem_t *mem = module->mem;
+    uint64_t remote_addr = (module->state_addrs[target]) + OSC_UCX_STATE_REQ_FLAG_OFFSET;
+    ompi_osc_ucx_accumulate_request_t *ucx_req = NULL;
+    bool sync_check;
+    int ret = OMPI_SUCCESS;
+
+    if (acc_type != NONE) {
+        OMPI_OSC_UCX_ACCUMULATE_REQUEST_ALLOC(win, ucx_req);
+        assert(NULL != ucx_req);
+        ucx_req->op = op;
+        ucx_req->acc_type = acc_type;
+        ucx_req->phase = phase;
+        ucx_req->super.module = module;
+        ucx_req->target = target;
+        ucx_req->lock_acquired = lock_acquired;
+        ucx_req->win = win;
+        ucx_req->origin_addr = origin_addr;
+        ucx_req->origin_count = origin_count;
+        if (origin_dt != NULL) {
+            ucx_req->origin_dt = origin_dt;
+            if (!ompi_datatype_is_predefined(origin_dt)) {
+                OBJ_RETAIN(ucx_req->origin_dt);
+            }
+        }
+        ucx_req->stage_addr = stage_addr;
+        ucx_req->stage_count = stage_count;
+        if (stage_dt != NULL) {
+            ucx_req->stage_dt = stage_dt;
+            if (!ompi_datatype_is_predefined(stage_dt)) {
+                OBJ_RETAIN(ucx_req->stage_dt);
+            }
+        }
+        ucx_req->target = target;
+        assert(target_dt != NULL);
+        ucx_req->target_dt = target_dt;
+        if (!ompi_datatype_is_predefined(target_dt)) {
+            OBJ_RETAIN(ucx_req->target_dt);
+        }
+        ucx_req->target_disp = target_disp;
+        ucx_req->target_count = target_count;
+        ucx_req->free_ptr = NULL;
+    }
+    sync_check = module->skip_sync_check;
+    module->skip_sync_check = true; /* we already hold the acc lock, so no need for sync check*/
+
+    if (is_put) {
+        assert(origin_dt != NULL);
+        ret = ompi_osc_ucx_put(origin_addr, origin_count, origin_dt, target, target_disp,
+                               target_count, target_dt, win);
+    } else {
+        assert(stage_dt != NULL);
+        ret = ompi_osc_ucx_get(stage_addr, stage_count, stage_dt, target, target_disp,
+                target_count, target_dt, win);
+    }
+    if (ret != OMPI_SUCCESS) {
+        return ret;
+    }
+
+    module->skip_sync_check = sync_check;
+    if (acc_type != NONE) {
+        OSC_UCX_INCREMENT_OUTSTANDING_NB_OPS(module);
+        ret = opal_common_ucx_wpmem_flush_ep_nb(mem, target, ompi_osc_ucx_req_completion, ucx_req, ep);
+
+        if (ret != OMPI_SUCCESS) {
+            /* fallback to using an atomic op to acquire a request handle */
+            ret = opal_common_ucx_wpmem_fence(mem);
+            if (ret != OMPI_SUCCESS) {
+                OSC_UCX_VERBOSE(1, "opal_common_ucx_mem_fence failed: %d", ret);
+                return OMPI_ERROR;
+            }
+
+            ret = opal_common_ucx_wpmem_fetch_nb(mem, UCP_ATOMIC_FETCH_OP_FADD,
+                                                0, target, &(module->req_result),
+                                                sizeof(uint64_t), remote_addr & (~0x7),
+                                                ompi_osc_ucx_req_completion, ucx_req, ep);
+            if (ret != OMPI_SUCCESS) {
+                OMPI_OSC_UCX_REQUEST_RETURN(ucx_req);
+                return ret;
+            }
+        }
+    }
+
+    return ret;
+}
+
+static inline int ompi_osc_ucx_check_ops_and_flush (ompi_osc_ucx_module_t *module,
+        int target, ptrdiff_t target_disp, size_t target_count, struct
+        ompi_datatype_t *target_dt, bool lock_acquired) {
+    ptrdiff_t target_lb, target_extent;
+    uint64_t base_tmp, tail_tmp;
+    int ret = OMPI_SUCCESS;
+
+    if (module->ctx->num_incomplete_req_ops > ompi_osc_ucx_outstanding_ops_flush_threshold) {
+        ret = opal_common_ucx_ctx_flush(module->ctx, OPAL_COMMON_UCX_SCOPE_WORKER, 0);
+        if (ret != OPAL_SUCCESS) {
+            ret = OMPI_ERROR;
+            return ret;
+        }
+        opal_mutex_lock(&module->ctx->mutex);
+        /* Check if we need to clear the list */
+        if (ompi_osc_ucx_outstanding_ops_flush_threshold != 0
+                && module->ctx->num_incomplete_req_ops == 0) {
+            memset(module->epoc_outstanding_ops_mems, 0,
+                    sizeof(ompi_osc_ucx_mem_ranges_t) *
+                    ompi_osc_ucx_outstanding_ops_flush_threshold);
+        }
+        opal_mutex_unlock(&module->ctx->mutex);
+    }
+
+    if (ompi_osc_ucx_outstanding_ops_flush_threshold == 0) {
+        return ret;
+    }
+
+    if (!lock_acquired) {
+        /* We are not acquiring the acc lock as we already may have an exclusive lock
+         * to the target window. However, in the nb acc operation, we must
+         * preserve the atomicy of back to back calls to same target buffer
+         * even when acc lock is not available */
+
+        /* Calculate the base and range of the target buffer for this call */
+        ompi_datatype_get_true_extent(target_dt, &target_lb, &target_extent);
+        uint64_t base = (module->addrs[target]) + target_disp * OSC_UCX_GET_DISP(module, target);
+        uint64_t tail = base + target_extent * target_count;
+
+        assert((void *)base != NULL);
+
+        opal_mutex_lock(&module->ctx->mutex);
+
+        bool overlap = false;
+        /* Find overlapping outstanidng acc ops to same target */
+        size_t i;
+        for (i = 0; i < ompi_osc_ucx_outstanding_ops_flush_threshold; i++) {
+           base_tmp = module->epoc_outstanding_ops_mems[i].base;
+           tail_tmp = module->epoc_outstanding_ops_mems[i].tail;
+           if (base_tmp == tail_tmp) {
+               continue;
+           }
+           if (!(tail_tmp < base || tail < base_tmp)) {
+               overlap = true;
+               break;
+           }
+        }
+
+        /* If there are overlaps, then flush */
+        if (overlap) {
+            ret = opal_common_ucx_ctx_flush(module->ctx, OPAL_COMMON_UCX_SCOPE_WORKER, 0);
+            if (ret != OPAL_SUCCESS) {
+                ret = OMPI_ERROR;
+                return ret;
+            }
+        }
+
+        /* Add the new base and tail to the list of outstanding
+         * ops of this epoc */
+        bool op_added = false;
+        while (!op_added) {
+            /* Check if we need to clear the list */
+            if (module->ctx->num_incomplete_req_ops == 0) {
+                memset(module->epoc_outstanding_ops_mems, 0,
+                        sizeof(ompi_osc_ucx_mem_ranges_t) *
+                        ompi_osc_ucx_outstanding_ops_flush_threshold);
+            }
+
+            for (i = 0; i < ompi_osc_ucx_outstanding_ops_flush_threshold; i++) {
+               base_tmp = module->epoc_outstanding_ops_mems[i].base;
+               tail_tmp = module->epoc_outstanding_ops_mems[i].tail;
+               if (base_tmp == tail_tmp) {
+                    module->epoc_outstanding_ops_mems[i].base = base;
+                    module->epoc_outstanding_ops_mems[i].tail = tail;
+                    op_added = true;
+                    break;
+               };
+            }
+
+            if (!op_added) {
+                /* no more space so flush */
+                ret = opal_common_ucx_ctx_flush(module->ctx, OPAL_COMMON_UCX_SCOPE_WORKER, 0);
+                if (ret != OPAL_SUCCESS) {
+                    ret = OMPI_ERROR;
+                    return ret;
+                }
+            }
+        }
+        opal_mutex_unlock(&module->ctx->mutex);
+    }
+
+    return ret;
+}
+
+/* Nonblocking variant of accumulate. reduce+put happens inside completion call back
+ * of rget */
+static int ompi_osc_ucx_get_accumulate_nonblocking(const void *origin_addr, size_t origin_count,
+                    struct ompi_datatype_t *origin_dt, void *result_addr, size_t result_count,
+                    struct ompi_datatype_t *result_dt, int target, ptrdiff_t target_disp,
+                    size_t target_count, struct ompi_datatype_t *target_dt, struct ompi_op_t
+                    *op, struct ompi_win_t *win, int acc_type) {
+    ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module;
+    int ret = OMPI_SUCCESS;
+    uint64_t remote_addr = (module->addrs[target]) + target_disp *
+        OSC_UCX_GET_DISP(module, target);
+    void *free_ptr = NULL;
+    bool lock_acquired = false;
+
+    ret = check_sync_state(module, target, false);
+    if (ret != OMPI_SUCCESS) {
+        return ret;
+    }
+
+    if (ACCUMULATE == acc_type && op == &ompi_mpi_op_no_op.op) {
+        /* This is an accumulate (not get-accumulate) operation, so return */
+        return ret;
+    }
+
+    /* rely on UCX network atomics if the user told us that it safe */
+    if (use_atomic_op(module, op, target_disp, origin_dt, target_dt, origin_count, target_count)) {
+        return do_atomic_op_intrinsic(module, op, target,
+                                      origin_addr, origin_count, origin_dt,
+                                      target_disp, result_addr, NULL);
+    }
+
+    /* Start atomicity by acquiring acc lock  */
+    ret = ompi_osc_ucx_acc_lock(module, target, &lock_acquired);
+    if (ret != OMPI_SUCCESS) {
+        return ret;
+    }
+
+    ret = ompi_osc_ucx_check_ops_and_flush(module, target, target_disp, target_count,
+            target_dt, lock_acquired);
+    if (ret != OMPI_SUCCESS) {
+        return ret;
+    }
+
+    CHECK_DYNAMIC_WIN(remote_addr, module, target, ret);
+
+    if (GET_ACCUMULATE == acc_type) {
+        /* This is a get-accumulate operation, so read the target data into result addr */
+        ret = ompi_osc_ucx_acc_rputget(result_addr, (int)result_count, result_dt, target,
+                target_disp, target_count, target_dt, op,  win, lock_acquired,
+                origin_addr, origin_count, origin_dt, false, ACC_GET_RESULTS_DATA, GET_ACCUMULATE);
+        if (ret != OMPI_SUCCESS) {
+            return ret;
+        } else if (op == &ompi_mpi_op_no_op.op || op == &ompi_mpi_op_replace.op) {
+            /* Nothing else to do, so return */
+            return MPI_SUCCESS;
+        }
+    }
+
+    if (op == &ompi_mpi_op_replace.op) {
+        assert(ACCUMULATE == acc_type);
+        /* No need for get, just use put and realize when to release the lock */
+        ret = ompi_osc_ucx_acc_rputget(NULL, 0, NULL, target, target_disp,
+                target_count, target_dt, op,  win, lock_acquired, origin_addr,
+                origin_count, origin_dt, true, ACC_PUT_TARGET_DATA, ACCUMULATE);
+        if (ret != OMPI_SUCCESS) {
+            return ret;
+        }
+    } else {
+        void *temp_addr = NULL;
+        uint32_t temp_count;
+        ompi_datatype_t *temp_dt;
+        ptrdiff_t temp_lb, temp_extent;
+
+        if (ompi_datatype_is_predefined(target_dt)) {
+            temp_dt = target_dt;
+            temp_count = target_count;
+        } else {
+            ret = ompi_osc_base_get_primitive_type_info(target_dt, &temp_dt, &temp_count);
+            if (ret != OMPI_SUCCESS) {
+                return ret;
+            }
+            temp_count *= target_count;
+        }
+        ompi_datatype_get_true_extent(temp_dt, &temp_lb, &temp_extent);
+        temp_addr = free_ptr = malloc(temp_extent * temp_count);
+        if (temp_addr == NULL) {
+            return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
+        }
+
+        ret = ompi_osc_ucx_acc_rputget(temp_addr, (int)temp_count, temp_dt, target,
+                target_disp, target_count, target_dt, op,  win, lock_acquired,
+                origin_addr, origin_count, origin_dt, false, ACC_GET_STAGE_DATA,
+                acc_type);
+        if (ret != OMPI_SUCCESS) {
+            return ret;
+        }
+    }
+
+    return ret;
+}
+
+int ompi_osc_ucx_accumulate_nb(const void *origin_addr, size_t origin_count,
+                            struct ompi_datatype_t *origin_dt,
+                            int target, ptrdiff_t target_disp, size_t target_count,
+                            struct ompi_datatype_t *target_dt,
+                            struct ompi_op_t *op, struct ompi_win_t *win) {
+
+    return ompi_osc_ucx_get_accumulate_nonblocking(origin_addr, origin_count,
+                            origin_dt, (void *)NULL, 0, NULL, target, target_disp,
+                            target_count, target_dt, op, win, ACCUMULATE);
 }
 
 static int
@@ -750,25 +1162,31 @@ do_atomic_compare_and_swap(const void *origin_addr, const void *compare_addr,
     int ret;
     bool lock_acquired = false;
     size_t dt_bytes;
+    opal_common_ucx_wpmem_t *mem = module->mem;
+    ucp_ep_h *ep;
+    OSC_UCX_GET_DEFAULT_EP(ep, module, target);
     if (!module->acc_single_intrinsic) {
-        ret = start_atomicity(module, target, &lock_acquired);
+        /* Start atomicity by acquiring acc lock  */
+        ret = ompi_osc_ucx_acc_lock(module, target, &lock_acquired);
         if (ret != OMPI_SUCCESS) {
             return ret;
         }
     }
 
+    CHECK_DYNAMIC_WIN(remote_addr, module, target, ret);
+
     ompi_datatype_type_size(dt, &dt_bytes);
     uint64_t compare_val = opal_common_ucx_load_uint64(compare_addr, dt_bytes);
     uint64_t value       = opal_common_ucx_load_uint64(origin_addr,  dt_bytes);
-    ret = opal_common_ucx_wpmem_cmpswp_nb(module->mem, compare_val, value, target,
+    ret = opal_common_ucx_wpmem_cmpswp_nb(mem, compare_val, value, target,
                                           result_addr, dt_bytes, remote_addr,
-                                          NULL, NULL);
+                                          NULL, NULL, ep);
 
     if (module->acc_single_intrinsic) {
         return ret;
     }
 
-    return end_atomicity(module, target, lock_acquired, NULL);
+    return ompi_osc_ucx_acc_unlock(module, target, lock_acquired, NULL);
 }
 
 int ompi_osc_ucx_compare_and_swap(const void *origin_addr, const void *compare_addr,
@@ -776,6 +1194,9 @@ int ompi_osc_ucx_compare_and_swap(const void *origin_addr, const void *compare_a
                                   int target, ptrdiff_t target_disp,
                                   struct ompi_win_t *win) {
     ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t *)win->w_osc_module;
+    opal_common_ucx_wpmem_t *mem = module->mem;
+    ucp_ep_h *ep;
+    OSC_UCX_GET_DEFAULT_EP(ep, module, target);
     uint64_t remote_addr = (module->addrs[target]) + target_disp * OSC_UCX_GET_DISP(module, target);
     size_t dt_bytes;
     int ret = OMPI_SUCCESS;
@@ -786,15 +1207,8 @@ int ompi_osc_ucx_compare_and_swap(const void *origin_addr, const void *compare_a
         return ret;
     }
 
-    if (module->flavor == MPI_WIN_FLAVOR_DYNAMIC) {
-        ret = get_dynamic_win_info(remote_addr, module, target);
-        if (ret != OMPI_SUCCESS) {
-            return ret;
-        }
-    }
-
     ompi_datatype_type_size(dt, &dt_bytes);
-    if (ATOMIC_SIZE_SUPPORTED(remote_addr, dt_bytes)) {
+    if (ompi_osc_base_is_atomic_size_supported(remote_addr, dt_bytes)) {
         // fast path using UCX atomic operations
         return do_atomic_compare_and_swap(origin_addr, compare_addr,
                                           result_addr, dt, target,
@@ -803,34 +1217,37 @@ int ompi_osc_ucx_compare_and_swap(const void *origin_addr, const void *compare_a
 
     /* fall back to get-compare-put */
 
-    ret = start_atomicity(module, target, &lock_acquired);
+    /* Start atomicity by acquiring acc lock  */
+    ret = ompi_osc_ucx_acc_lock(module, target, &lock_acquired);
     if (ret != OMPI_SUCCESS) {
         return ret;
     }
 
-    ret = opal_common_ucx_wpmem_putget(module->mem, OPAL_COMMON_UCX_GET, target,
-                                       &result_addr, dt_bytes, remote_addr);
+    CHECK_DYNAMIC_WIN(remote_addr, module, target, ret);
+
+    ret = opal_common_ucx_wpmem_putget(mem, OPAL_COMMON_UCX_GET, target,
+                                       result_addr, dt_bytes, remote_addr, ep);
     if (OPAL_SUCCESS != ret) {
         OSC_UCX_VERBOSE(1, "opal_common_ucx_mem_putget failed: %d", ret);
         return OMPI_ERROR;
     }
 
-    ret = opal_common_ucx_wpmem_flush(module->mem, OPAL_COMMON_UCX_SCOPE_EP, target);
+    ret = opal_common_ucx_ctx_flush(module->ctx, OPAL_COMMON_UCX_SCOPE_EP, target);
     if (ret != OPAL_SUCCESS) {
         return ret;
     }
 
     if (0 == memcmp(result_addr, compare_addr, dt_bytes)) {
         // write the new value
-        ret = opal_common_ucx_wpmem_putget(module->mem, OPAL_COMMON_UCX_PUT, target,
-                                           (void*)origin_addr, dt_bytes, remote_addr);
+        ret = opal_common_ucx_wpmem_putget(mem, OPAL_COMMON_UCX_PUT, target,
+                                           (void*)origin_addr, dt_bytes, remote_addr, ep);
         if (OPAL_SUCCESS != ret) {
             OSC_UCX_VERBOSE(1, "opal_common_ucx_mem_putget failed: %d", ret);
             return OMPI_ERROR;
         }
     }
 
-    return end_atomicity(module, target, lock_acquired, NULL);
+    return ompi_osc_ucx_acc_unlock(module, target, lock_acquired, NULL);
 }
 
 int ompi_osc_ucx_fetch_and_op(const void *origin_addr, void *result_addr,
@@ -839,6 +1256,9 @@ int ompi_osc_ucx_fetch_and_op(const void *origin_addr, void *result_addr,
                               struct ompi_win_t *win) {
     size_t dt_bytes;
     ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module;
+    opal_common_ucx_wpmem_t *mem = module->mem;
+    ucp_ep_h *ep;
+    OSC_UCX_GET_DEFAULT_EP(ep, module, target);
     int ret = OMPI_SUCCESS;
 
     ret = check_sync_state(module, target, false);
@@ -849,27 +1269,20 @@ int ompi_osc_ucx_fetch_and_op(const void *origin_addr, void *result_addr,
     uint64_t remote_addr = (module->addrs[target]) + target_disp * OSC_UCX_GET_DISP(module, target);
     ompi_datatype_type_size(dt, &dt_bytes);
 
-    /* UCX atomics are only supported on 32 and 64 bit values */
-    if (ATOMIC_SIZE_SUPPORTED(remote_addr, dt_bytes) &&
-        (op == &ompi_mpi_op_no_op.op || op == &ompi_mpi_op_replace.op ||
-         op == &ompi_mpi_op_sum.op)) {
+    if (osc_is_atomic_dt_op_supported(dt, op, dt_bytes, remote_addr)) {
         uint64_t value;
         ucp_atomic_fetch_op_t opcode;
         bool lock_acquired = false;
 
         if (!module->acc_single_intrinsic) {
-            ret = start_atomicity(module, target, &lock_acquired);
+            /* Start atomicity by acquiring acc lock  */
+            ret = ompi_osc_ucx_acc_lock(module, target, &lock_acquired);
             if (ret != OMPI_SUCCESS) {
                 return ret;
             }
         }
 
-        if (module->flavor == MPI_WIN_FLAVOR_DYNAMIC) {
-            ret = get_dynamic_win_info(remote_addr, module, target);
-            if (ret != OMPI_SUCCESS) {
-                return ret;
-            }
-        }
+        CHECK_DYNAMIC_WIN(remote_addr, module, target, ret);
 
         value = origin_addr ? opal_common_ucx_load_uint64(origin_addr, dt_bytes) : 0;
 
@@ -882,15 +1295,15 @@ int ompi_osc_ucx_fetch_and_op(const void *origin_addr, void *result_addr,
             }
         }
 
-        ret = opal_common_ucx_wpmem_fetch_nb(module->mem, opcode, value, target,
+        ret = opal_common_ucx_wpmem_fetch_nb(mem, opcode, value, target,
                                              (void *)result_addr, dt_bytes,
-                                             remote_addr, NULL, NULL);
+                                             remote_addr, NULL, NULL, ep);
 
         if (module->acc_single_intrinsic) {
             return ret;
         }
 
-        return end_atomicity(module, target, lock_acquired, NULL);
+        return ompi_osc_ucx_acc_unlock(module, target, lock_acquired, NULL);
     } else {
         return ompi_osc_ucx_get_accumulate(origin_addr, 1, dt, result_addr, 1, dt,
                                            target, target_disp, 1, dt, op, win);
@@ -898,16 +1311,18 @@ int ompi_osc_ucx_fetch_and_op(const void *origin_addr, void *result_addr,
 }
 
 static
-int get_accumulate_req(const void *origin_addr, int origin_count,
+int get_accumulate_req(const void *origin_addr, size_t origin_count,
                        struct ompi_datatype_t *origin_dt,
-                       void *result_addr, int result_count,
+                       void *result_addr, size_t result_count,
                        struct ompi_datatype_t *result_dt,
                        int target, ptrdiff_t target_disp,
-                       int target_count, struct ompi_datatype_t *target_dt,
+                       size_t target_count, struct ompi_datatype_t *target_dt,
                        struct ompi_op_t *op, struct ompi_win_t *win,
-                       ompi_osc_ucx_request_t *ucx_req) {
+                       ompi_osc_ucx_accumulate_request_t *ucx_req) {
     ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module;
     int ret = OMPI_SUCCESS;
+    uint64_t remote_addr = (module->addrs[target]) + target_disp *
+        OSC_UCX_GET_DISP(module, target);
     void *free_addr = NULL;
     bool lock_acquired = false;
 
@@ -923,11 +1338,14 @@ int get_accumulate_req(const void *origin_addr, int origin_count,
                                       target_disp, result_addr, ucx_req);
     }
 
-    ret = start_atomicity(module, target, &lock_acquired);
+    /* Start atomicity by acquiring acc lock  */
+    ret = ompi_osc_ucx_acc_lock(module, target, &lock_acquired);
     if (ret != OMPI_SUCCESS) {
         return ret;
     }
 
+    CHECK_DYNAMIC_WIN(remote_addr, module, target, ret);
+
     ret = ompi_osc_ucx_get(result_addr, result_count, result_dt, target,
                            target_disp, target_count, target_dt, win);
     if (ret != OMPI_SUCCESS) {
@@ -936,6 +1354,11 @@ int get_accumulate_req(const void *origin_addr, int origin_count,
 
     if (op != &ompi_mpi_op_no_op.op) {
         if (op == &ompi_mpi_op_replace.op) {
+            ret = opal_common_ucx_ctx_flush(module->ctx, OPAL_COMMON_UCX_SCOPE_EP, target);
+            if (ret != OMPI_SUCCESS) {
+                return ret;
+            }
+
             ret = ompi_osc_ucx_put(origin_addr, origin_count, origin_dt,
                                    target, target_disp, target_count,
                                    target_dt, win);
@@ -957,6 +1380,7 @@ int get_accumulate_req(const void *origin_addr, int origin_count,
                 if (ret != OMPI_SUCCESS) {
                     return ret;
                 }
+                temp_count *= target_count;
             }
             ompi_datatype_get_true_extent(temp_dt, &temp_lb, &temp_extent);
             temp_addr = free_addr = malloc(temp_extent * temp_count);
@@ -970,7 +1394,7 @@ int get_accumulate_req(const void *origin_addr, int origin_count,
                 return ret;
             }
 
-            ret = opal_common_ucx_wpmem_flush(module->mem, OPAL_COMMON_UCX_SCOPE_EP, target);
+            ret = opal_common_ucx_ctx_flush(module->ctx, OPAL_COMMON_UCX_SCOPE_EP, target);
             if (ret != OMPI_SUCCESS) {
                 return ret;
             }
@@ -1026,21 +1450,26 @@ int get_accumulate_req(const void *origin_addr, int origin_count,
         }
     }
 
+    ret = opal_common_ucx_ctx_flush(module->ctx, OPAL_COMMON_UCX_SCOPE_EP, target);
+    if (ret != OPAL_SUCCESS) {
+        return ret;
+    }
+
     if (NULL != ucx_req) {
         // nothing to wait for, mark request as completed
-        ompi_request_complete(&ucx_req->super, true);
+        ompi_request_complete(&ucx_req->super.super, true);
     }
 
 
-    return end_atomicity(module, target, lock_acquired, free_addr);
+    return ompi_osc_ucx_acc_unlock(module, target, lock_acquired, free_addr);
 }
 
-int ompi_osc_ucx_get_accumulate(const void *origin_addr, int origin_count,
+int ompi_osc_ucx_get_accumulate(const void *origin_addr, size_t origin_count,
                                 struct ompi_datatype_t *origin_dt,
-                                void *result_addr, int result_count,
+                                void *result_addr, size_t result_count,
                                 struct ompi_datatype_t *result_dt,
                                 int target, ptrdiff_t target_disp,
-                                int target_count, struct ompi_datatype_t *target_dt,
+                                size_t target_count, struct ompi_datatype_t *target_dt,
                                 struct ompi_op_t *op, struct ompi_win_t *win) {
 
     return get_accumulate_req(origin_addr, origin_count, origin_dt, result_addr,
@@ -1048,14 +1477,30 @@ int ompi_osc_ucx_get_accumulate(const void *origin_addr, int origin_count,
                               target_count, target_dt, op, win, NULL);
 }
 
-int ompi_osc_ucx_rput(const void *origin_addr, int origin_count,
+int ompi_osc_ucx_get_accumulate_nb(const void *origin_addr, size_t origin_count,
+                                struct ompi_datatype_t *origin_dt,
+                                void *result_addr, size_t result_count,
+                                struct ompi_datatype_t *result_dt,
+                                int target, ptrdiff_t target_disp,
+                                size_t target_count, struct ompi_datatype_t *target_dt,
+                                struct ompi_op_t *op, struct ompi_win_t *win) {
+
+    return ompi_osc_ucx_get_accumulate_nonblocking(origin_addr, origin_count, origin_dt,
+                            result_addr, result_count, result_dt, target, target_disp,
+                            target_count, target_dt, op, win, GET_ACCUMULATE);
+}
+
+int ompi_osc_ucx_rput(const void *origin_addr, size_t origin_count,
                       struct ompi_datatype_t *origin_dt,
-                      int target, ptrdiff_t target_disp, int target_count,
+                      int target, ptrdiff_t target_disp, size_t target_count,
                       struct ompi_datatype_t *target_dt,
                       struct ompi_win_t *win, struct ompi_request_t **request) {
     ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module;
-    uint64_t remote_addr = (module->addrs[target]) + target_disp * OSC_UCX_GET_DISP(module, target);
-    ompi_osc_ucx_request_t *ucx_req = NULL;
+    ucp_ep_h *ep;
+    OSC_UCX_GET_DEFAULT_EP(ep, module, target);
+    opal_common_ucx_wpmem_t *mem = module->mem;
+    uint64_t remote_addr = (module->state_addrs[target]) + OSC_UCX_STATE_REQ_FLAG_OFFSET;
+    ompi_osc_ucx_generic_request_t *ucx_req = NULL;
     int ret = OMPI_SUCCESS;
 
     ret = check_sync_state(module, target, true);
@@ -1063,52 +1508,53 @@ int ompi_osc_ucx_rput(const void *origin_addr, int origin_count,
         return ret;
     }
 
-    if (module->flavor == MPI_WIN_FLAVOR_DYNAMIC) {
-        ret = get_dynamic_win_info(remote_addr, module, target);
-        if (ret != OMPI_SUCCESS) {
-            return ret;
-        }
-    }
-
-    OMPI_OSC_UCX_REQUEST_ALLOC(win, ucx_req);
-    assert(NULL != ucx_req);
-
     ret = ompi_osc_ucx_put(origin_addr, origin_count, origin_dt, target, target_disp,
                            target_count, target_dt, win);
     if (ret != OMPI_SUCCESS) {
         return ret;
     }
 
-    ret = opal_common_ucx_wpmem_fence(module->mem);
-    if (ret != OMPI_SUCCESS) {
-        OSC_UCX_VERBOSE(1, "opal_common_ucx_mem_fence failed: %d", ret);
-        return OMPI_ERROR;
-    }
+    OMPI_OSC_UCX_GENERIC_REQUEST_ALLOC(win, ucx_req, RPUT_REQ);
+    ucx_req->super.module = module;
+
+    OSC_UCX_INCREMENT_OUTSTANDING_NB_OPS(module);
+    ret = opal_common_ucx_wpmem_flush_ep_nb(mem, target, ompi_osc_ucx_req_completion, ucx_req, ep);
 
-    mca_osc_ucx_component.num_incomplete_req_ops++;
-    /* TODO: investigate whether ucp_worker_flush_nb is a better choice here */
-    ret = opal_common_ucx_wpmem_fetch_nb(module->mem, UCP_ATOMIC_FETCH_OP_FADD,
-                                         0, target, &(module->req_result),
-                                         sizeof(uint64_t), remote_addr & (~0x7),
-                                         req_completion, ucx_req);
     if (ret != OMPI_SUCCESS) {
-        OMPI_OSC_UCX_REQUEST_RETURN(ucx_req);
-        return ret;
+        /* fallback to using an atomic op to acquire a request handle */
+        ret = opal_common_ucx_wpmem_fence(mem);
+        if (ret != OMPI_SUCCESS) {
+            OSC_UCX_VERBOSE(1, "opal_common_ucx_mem_fence failed: %d", ret);
+            OMPI_OSC_UCX_REQUEST_RETURN(ucx_req);
+            return OMPI_ERROR;
+        }
+
+        ret = opal_common_ucx_wpmem_fetch_nb(mem, UCP_ATOMIC_FETCH_OP_FADD,
+                                            0, target, &(module->req_result),
+                                            sizeof(uint64_t), remote_addr & (~0x7),
+                                            ompi_osc_ucx_req_completion, ucx_req, ep);
+        if (ret != OMPI_SUCCESS) {
+            OMPI_OSC_UCX_REQUEST_RETURN(ucx_req);
+            return ret;
+        }
     }
 
-    *request = &ucx_req->super;
+    *request = &ucx_req->super.super;
 
     return ret;
 }
 
-int ompi_osc_ucx_rget(void *origin_addr, int origin_count,
+int ompi_osc_ucx_rget(void *origin_addr, size_t origin_count,
                       struct ompi_datatype_t *origin_dt,
-                      int target, ptrdiff_t target_disp, int target_count,
+                      int target, ptrdiff_t target_disp, size_t target_count,
                       struct ompi_datatype_t *target_dt, struct ompi_win_t *win,
                       struct ompi_request_t **request) {
     ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module;
-    uint64_t remote_addr = (module->addrs[target]) + target_disp * OSC_UCX_GET_DISP(module, target);
-    ompi_osc_ucx_request_t *ucx_req = NULL;
+    ucp_ep_h *ep;
+    OSC_UCX_GET_DEFAULT_EP(ep, module, target);
+    opal_common_ucx_wpmem_t *mem = module->mem;
+    uint64_t remote_addr = (module->state_addrs[target]) + OSC_UCX_STATE_REQ_FLAG_OFFSET;
+    ompi_osc_ucx_generic_request_t *ucx_req = NULL;
     int ret = OMPI_SUCCESS;
 
     ret = check_sync_state(module, target, true);
@@ -1116,51 +1562,49 @@ int ompi_osc_ucx_rget(void *origin_addr, int origin_count,
         return ret;
     }
 
-    if (module->flavor == MPI_WIN_FLAVOR_DYNAMIC) {
-        ret = get_dynamic_win_info(remote_addr, module, target);
-        if (ret != OMPI_SUCCESS) {
-            return ret;
-        }
-    }
-
-    OMPI_OSC_UCX_REQUEST_ALLOC(win, ucx_req);
-    assert(NULL != ucx_req);
-
     ret = ompi_osc_ucx_get(origin_addr, origin_count, origin_dt, target, target_disp,
                            target_count, target_dt, win);
     if (ret != OMPI_SUCCESS) {
         return ret;
     }
 
-    ret = opal_common_ucx_wpmem_fence(module->mem);
-    if (ret != OMPI_SUCCESS) {
-        OSC_UCX_VERBOSE(1, "opal_common_ucx_mem_fence failed: %d", ret);
-        return OMPI_ERROR;
-    }
+    OMPI_OSC_UCX_GENERIC_REQUEST_ALLOC(win, ucx_req, RGET_REQ);
+    ucx_req->super.module = module;
+
+    OSC_UCX_INCREMENT_OUTSTANDING_NB_OPS(module);
+    ret = opal_common_ucx_wpmem_flush_ep_nb(mem, target, ompi_osc_ucx_req_completion, ucx_req, ep);
 
-    mca_osc_ucx_component.num_incomplete_req_ops++;
-    /* TODO: investigate whether ucp_worker_flush_nb is a better choice here */
-    ret = opal_common_ucx_wpmem_fetch_nb(module->mem, UCP_ATOMIC_FETCH_OP_FADD,
-                                         0, target, &(module->req_result),
-                                         sizeof(uint64_t), remote_addr & (~0x7),
-                                         req_completion, ucx_req);
     if (ret != OMPI_SUCCESS) {
-        OMPI_OSC_UCX_REQUEST_RETURN(ucx_req);
-        return ret;
+        /* fallback to using an atomic op to acquire a request handle */
+        ret = opal_common_ucx_wpmem_fence(mem);
+        if (ret != OMPI_SUCCESS) {
+            OSC_UCX_VERBOSE(1, "opal_common_ucx_mem_fence failed: %d", ret);
+            OMPI_OSC_UCX_REQUEST_RETURN(ucx_req);
+            return OMPI_ERROR;
+        }
+
+        ret = opal_common_ucx_wpmem_fetch_nb(mem, UCP_ATOMIC_FETCH_OP_FADD,
+                                            0, target, &(module->req_result),
+                                            sizeof(uint64_t), remote_addr & (~0x7),
+                                            ompi_osc_ucx_req_completion, ucx_req, ep);
+        if (ret != OMPI_SUCCESS) {
+            OMPI_OSC_UCX_REQUEST_RETURN(ucx_req);
+            return ret;
+        }
     }
 
-    *request = &ucx_req->super;
+    *request = &ucx_req->super.super;
 
     return ret;
 }
 
-int ompi_osc_ucx_raccumulate(const void *origin_addr, int origin_count,
+int ompi_osc_ucx_raccumulate(const void *origin_addr, size_t origin_count,
                              struct ompi_datatype_t *origin_dt,
-                             int target, ptrdiff_t target_disp, int target_count,
+                             int target, ptrdiff_t target_disp, size_t target_count,
                              struct ompi_datatype_t *target_dt, struct ompi_op_t *op,
                              struct ompi_win_t *win, struct ompi_request_t **request) {
     ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module;
-    ompi_osc_ucx_request_t *ucx_req = NULL;
+    ompi_osc_ucx_accumulate_request_t *ucx_req = NULL;
     int ret = OMPI_SUCCESS;
 
     ret = check_sync_state(module, target, true);
@@ -1168,7 +1612,8 @@ int ompi_osc_ucx_raccumulate(const void *origin_addr, int origin_count,
         return ret;
     }
 
-    OMPI_OSC_UCX_REQUEST_ALLOC(win, ucx_req);
+    OMPI_OSC_UCX_ACCUMULATE_REQUEST_ALLOC(win, ucx_req);
+    ucx_req->super.module = module;
     assert(NULL != ucx_req);
 
     ret = accumulate_req(origin_addr, origin_count, origin_dt, target, target_disp,
@@ -1178,21 +1623,21 @@ int ompi_osc_ucx_raccumulate(const void *origin_addr, int origin_count,
         return ret;
     }
 
-    *request = &ucx_req->super;
+    *request = &ucx_req->super.super;
 
     return ret;
 }
 
-int ompi_osc_ucx_rget_accumulate(const void *origin_addr, int origin_count,
+int ompi_osc_ucx_rget_accumulate(const void *origin_addr, size_t origin_count,
                                  struct ompi_datatype_t *origin_datatype,
-                                 void *result_addr, int result_count,
+                                 void *result_addr, size_t result_count,
                                  struct ompi_datatype_t *result_datatype,
-                                 int target, ptrdiff_t target_disp, int target_count,
+                                 int target, ptrdiff_t target_disp, size_t target_count,
                                  struct ompi_datatype_t *target_datatype,
                                  struct ompi_op_t *op, struct ompi_win_t *win,
                                  struct ompi_request_t **request) {
     ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module;
-    ompi_osc_ucx_request_t *ucx_req = NULL;
+    ompi_osc_ucx_accumulate_request_t *ucx_req = NULL;
     int ret = OMPI_SUCCESS;
 
     ret = check_sync_state(module, target, true);
@@ -1200,7 +1645,8 @@ int ompi_osc_ucx_rget_accumulate(const void *origin_addr, int origin_count,
         return ret;
     }
 
-    OMPI_OSC_UCX_REQUEST_ALLOC(win, ucx_req);
+    OMPI_OSC_UCX_ACCUMULATE_REQUEST_ALLOC(win, ucx_req);
+    ucx_req->super.module = module;
     assert(NULL != ucx_req);
 
     ret = get_accumulate_req(origin_addr, origin_count, origin_datatype,
@@ -1212,7 +1658,248 @@ int ompi_osc_ucx_rget_accumulate(const void *origin_addr, int origin_count,
         return ret;
     }
 
-    *request = &ucx_req->super;
+    *request = &ucx_req->super.super;
+
+    return ret;
+}
+
+static inline int ompi_osc_ucx_nonblocking_ops_finalize(ompi_osc_ucx_module_t *module, int
+                target, bool lock_acquired, struct ompi_win_t *win, void *free_ptr) {
+    uint64_t remote_addr = (module->state_addrs)[target] + OSC_UCX_STATE_ACC_LOCK_OFFSET;
+    ucp_ep_h *ep;
+    OSC_UCX_GET_DEFAULT_EP(ep, module, target);
+    int ret = OMPI_SUCCESS;
+    ompi_osc_ucx_accumulate_request_t *ucx_req = NULL;
+
+    OMPI_OSC_UCX_ACCUMULATE_REQUEST_ALLOC(win, ucx_req);
+    assert(NULL != ucx_req);
+    ucx_req->free_ptr = free_ptr;
+    ucx_req->phase = ACC_FINALIZE;
+    ucx_req->acc_type = ANY;
+    ucx_req->super.module = module;
+
+    /* Fence any still active operations */
+    ret = opal_common_ucx_wpmem_fence(module->mem);
+    if (ret != OMPI_SUCCESS) {
+        OSC_UCX_VERBOSE(1, "opal_common_ucx_mem_fence failed: %d", ret);
+        return OMPI_ERROR;
+    }
+
+    if (lock_acquired) {
+        OSC_UCX_INCREMENT_OUTSTANDING_NB_OPS(module);
+        ret = opal_common_ucx_wpmem_fetch_nb(module->state_mem,
+                                        UCP_ATOMIC_FETCH_OP_SWAP, TARGET_LOCK_UNLOCKED,
+                                        target, &(module->req_result), sizeof(module->req_result),
+                                        remote_addr, ompi_osc_ucx_req_completion, ucx_req, ep);
+        if (ret != OMPI_SUCCESS) {
+            OSC_UCX_VERBOSE(1, "opal_common_ucx_wpmem_fetch_nb failed: %d", ret);
+            OMPI_OSC_UCX_REQUEST_RETURN(ucx_req);
+            return ret;
+        }
+    } else {
+        /* Lock is not acquired, but still, we need to know when the 
+         * acc is finalized so that we can free the temp buffers */
+        OSC_UCX_INCREMENT_OUTSTANDING_NB_OPS(module);
+        ret = opal_common_ucx_wpmem_flush_ep_nb(module->mem, target, ompi_osc_ucx_req_completion, ucx_req, ep);
+
+        if (ret != OMPI_SUCCESS) {
+            /* fallback to using an atomic op to acquire a request handle */
+            ret = opal_common_ucx_wpmem_fetch_nb(module->mem, UCP_ATOMIC_FETCH_OP_FADD,
+                                                0, target, &(module->req_result),
+                                                sizeof(uint64_t), remote_addr & (~0x7),
+                                                ompi_osc_ucx_req_completion, ucx_req, ep);
+            if (ret != OMPI_SUCCESS) {
+                OMPI_OSC_UCX_REQUEST_RETURN(ucx_req);
+                return ret;
+            }
+        }
+    }
 
     return ret;
 }
+
+void ompi_osc_ucx_req_completion(void *request) {
+    ompi_osc_ucx_generic_request_t *ucx_req = (ompi_osc_ucx_generic_request_t *)request;
+    int ret = OMPI_SUCCESS;
+    ompi_osc_ucx_module_t *module = ucx_req->super.module;
+    if (ucx_req->super.request_type == ACCUMULATE_REQ) {
+        /* This is an accumulate request */
+        ompi_osc_ucx_accumulate_request_t *req = (ompi_osc_ucx_accumulate_request_t *)request;
+        assert(req->phase != ACC_INIT);
+        void *free_addr = NULL;
+        bool release_lock = false;
+        ptrdiff_t temp_extent, temp_lb;
+        const void *origin_addr = req->origin_addr;
+        int origin_count = req->origin_count;
+        struct ompi_datatype_t *origin_dt = req->origin_dt;
+        void *temp_addr = req->stage_addr;
+        int temp_count = req->stage_count;
+        struct ompi_datatype_t *temp_dt = req->stage_dt;
+        int target = req->target;
+        int target_count = req->target_count;
+        int target_disp = req->target_disp;
+        struct ompi_datatype_t *target_dt = req->target_dt;
+        struct ompi_win_t *win = req->win;
+        struct ompi_op_t *op = req->op;
+
+        if (req->phase != ACC_FINALIZE) {
+            /* Avoid calling flush while we are already in progress */
+            module->mem->skip_periodic_flush = true;
+            module->state_mem->skip_periodic_flush = true;
+        }
+
+        switch (req->phase) {
+            case ACC_FINALIZE:
+            {
+                if (req->free_ptr != NULL) {
+                    free(req->free_ptr);
+                    req->free_ptr = NULL;
+                }
+                if (origin_dt != NULL && !ompi_datatype_is_predefined(origin_dt)) {
+                    OBJ_RELEASE(origin_dt);
+                }
+                if (target_dt != NULL && !ompi_datatype_is_predefined(target_dt)) {
+                    OBJ_RELEASE(target_dt);
+                }
+                if (temp_dt != NULL && !ompi_datatype_is_predefined(temp_dt)) {
+                    OBJ_RELEASE(temp_dt);
+                }
+                break;
+            }
+            case ACC_GET_RESULTS_DATA:
+            {
+                /* This is a get-accumulate operation */
+                if (op == &ompi_mpi_op_no_op.op) {
+                    /* Done with reading the target data, so release the
+                     * acc lock and return */
+                    release_lock = true;
+                } else if (op == &ompi_mpi_op_replace.op) {
+                    assert(target_dt != NULL && origin_dt != NULL);
+                    /* Now that we have the results data, replace the target
+                     * buffer with origin buffer and then release the lock  */
+                    ret = ompi_osc_ucx_acc_rputget(NULL, 0, NULL, target, target_disp,
+                            target_count, target_dt, op, win, 0, origin_addr, origin_count,
+                            origin_dt, true, -1, NONE);
+                    if (ret != OMPI_SUCCESS) {
+                        OSC_UCX_ERROR("ompi_osc_ucx_acc_rputget failed ret= %d\n", ret);
+                        free(temp_addr);
+                        abort();
+                    }
+                    release_lock = true;
+                }
+                break;
+            }
+
+            case ACC_PUT_TARGET_DATA:
+            {
+                /* This is an accumulate (not get-accumulate) operation */
+                assert(op == &ompi_mpi_op_replace.op);
+                release_lock = true;
+                break;
+            }
+
+            case ACC_GET_STAGE_DATA:
+            {
+                assert(op != &ompi_mpi_op_replace.op && op != &ompi_mpi_op_no_op.op);
+                assert(origin_dt != NULL && temp_dt != NULL);
+
+                bool is_origin_contig =
+                    ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count);
+            
+                if (ompi_datatype_is_predefined(origin_dt) || is_origin_contig) {
+                    ompi_op_reduce(op, (void *)origin_addr, temp_addr, (int)temp_count, temp_dt);
+                } else {
+                    ucx_iovec_t *origin_ucx_iov = NULL;
+                    uint32_t origin_ucx_iov_count = 0;
+                    uint32_t origin_ucx_iov_idx = 0;
+
+                    ret = create_iov_list(origin_addr, origin_count, origin_dt,
+                                          &origin_ucx_iov, &origin_ucx_iov_count);
+                    if (ret != OMPI_SUCCESS) {
+                        OSC_UCX_ERROR("create_iov_list failed ret= %d\n", ret);
+                        free(temp_addr);
+                        abort();
+                    }
+
+                    if ((op != &ompi_mpi_op_maxloc.op && op != &ompi_mpi_op_minloc.op) ||
+                        ompi_datatype_is_contiguous_memory_layout(temp_dt, temp_count)) {
+                        size_t temp_size;
+                        char *curr_temp_addr = (char *)temp_addr;
+                        ompi_datatype_type_size(temp_dt, &temp_size);
+                        while (origin_ucx_iov_idx < origin_ucx_iov_count) {
+                            int curr_count = origin_ucx_iov[origin_ucx_iov_idx].len / temp_size;
+                            ompi_op_reduce(op, origin_ucx_iov[origin_ucx_iov_idx].addr,
+                                           curr_temp_addr, curr_count, temp_dt);
+                            curr_temp_addr += curr_count * temp_size;
+                            origin_ucx_iov_idx++;
+                        }
+                    } else {
+                        int i;
+                        void *curr_origin_addr = origin_ucx_iov[origin_ucx_iov_idx].addr;
+                        ompi_datatype_get_true_extent(temp_dt, &temp_lb, &temp_extent);
+                        for (i = 0; i < (int)temp_count; i++) {
+                            ompi_op_reduce(op, curr_origin_addr,
+                                           (void *)((char *)temp_addr + i * temp_extent),
+                                           1, temp_dt);
+                            curr_origin_addr = (void *)((char *)curr_origin_addr + temp_extent);
+                            origin_ucx_iov_idx++;
+                            if (curr_origin_addr >= (void *)((char
+                                            *)origin_ucx_iov[origin_ucx_iov_idx].addr
+                                        +
+                                        origin_ucx_iov[origin_ucx_iov_idx].len))
+                            {
+                                origin_ucx_iov_idx++;
+                                curr_origin_addr = origin_ucx_iov[origin_ucx_iov_idx].addr;
+                            }
+                        }
+                    }
+
+                    free(origin_ucx_iov);
+                }
+            
+                if (req->acc_type == GET_ACCUMULATE) {
+                    /* Do fence to make sure target results are received before
+                     * writing into target */
+                    ret = opal_common_ucx_wpmem_fence(module->mem);
+                    if (ret != OMPI_SUCCESS) {
+                        OSC_UCX_ERROR("opal_common_ucx_mem_fence failed: %d", ret);
+                        abort();
+                    }
+                }
+
+                ret = ompi_osc_ucx_acc_rputget(NULL, 0, NULL, target, target_disp,
+                        target_count, target_dt, op, win, 0, temp_addr, temp_count,
+                        temp_dt, true, -1, NONE);
+                if (ret != OMPI_SUCCESS) {
+                    OSC_UCX_ERROR("ompi_osc_ucx_acc_rputget failed ret= %d\n", ret);
+                    free(temp_addr);
+                    abort();
+                }
+                release_lock = true;
+                free_addr = temp_addr;
+                break;
+            }
+
+            default:
+            {
+                OSC_UCX_ERROR("accumulate progress failed\n");
+                abort();
+            }
+        }
+
+        if (release_lock) {
+            /* Ordering between previous put/get operations and unlock will be realized
+             * through the ucp fence inside the finalize function */
+            ompi_osc_ucx_nonblocking_ops_finalize(module, target,
+                    req->lock_acquired, win, free_addr);
+        }
+
+        if (req->phase != ACC_FINALIZE) {
+            module->mem->skip_periodic_flush = false;
+            module->state_mem->skip_periodic_flush = false;
+        }
+    }
+    assert(module->ctx->num_incomplete_req_ops > 0);
+    OSC_UCX_DECREMENT_OUTSTANDING_NB_OPS(module);
+    ompi_request_complete(&(ucx_req->super.super), true);
+}
diff --git a/ompi/mca/osc/ucx/osc_ucx_component.c b/ompi/mca/osc/ucx/osc_ucx_component.c
index 5136b1d49d5..91886233fd6 100644
--- a/ompi/mca/osc/ucx/osc_ucx_component.c
+++ b/ompi/mca/osc/ucx/osc_ucx_component.c
@@ -1,6 +1,10 @@
 /*
  * Copyright (C) Mellanox Technologies Ltd. 2001-2017. ALL RIGHTS RESERVED.
  * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2021-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ *
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -19,6 +23,7 @@
 
 #include "osc_ucx.h"
 #include "osc_ucx_request.h"
+#include "opal/util/sys_limits.h"
 
 #define memcpy_off(_dst, _src, _len, _off)        \
     memcpy(((char*)(_dst)) + (_off), _src, _len); \
@@ -38,14 +43,16 @@ static void _osc_ucx_init_unlock(void)
     }
 }
 
+static bool enable_nonblocking_accumulate = false;
 
 static int component_open(void);
+static int component_close(void);
 static int component_register(void);
 static int component_init(bool enable_progress_threads, bool enable_mpi_threads);
 static int component_finalize(void);
-static int component_query(struct ompi_win_t *win, void **base, size_t size, int disp_unit,
+static int component_query(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t disp_unit,
                            struct ompi_communicator_t *comm, struct opal_info_t *info, int flavor);
-static int component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit,
+static int component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t disp_unit,
                             struct ompi_communicator_t *comm, struct opal_info_t *info,
                             int flavor, int *model);
 static void ompi_osc_ucx_unregister_progress(void);
@@ -53,11 +60,12 @@ static void ompi_osc_ucx_unregister_progress(void);
 ompi_osc_ucx_component_t mca_osc_ucx_component = {
     { /* ompi_osc_base_component_t */
         .osc_version = {
-            OMPI_OSC_BASE_VERSION_3_0_0,
+            OMPI_OSC_BASE_VERSION_4_0_0,
             .mca_component_name = "ucx",
             MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION,
                                   OMPI_RELEASE_VERSION),
             .mca_open_component = component_open,
+            .mca_close_component = component_close,
             .mca_register_component_params = component_register,
         },
         .osc_data = {
@@ -71,13 +79,16 @@ ompi_osc_ucx_component_t mca_osc_ucx_component = {
     },
     .wpool                  = NULL,
     .env_initialized        = false,
-    .num_incomplete_req_ops = 0,
+    .priority_is_set        = false,
     .num_modules            = 0,
-    .acc_single_intrinsic   = false
+    .acc_single_intrinsic   = false,
+    .comm_world_size        = 0,
+    .endpoints              = NULL
 };
 
 ompi_osc_ucx_module_t ompi_osc_ucx_module_template = {
     {
+        .osc_win_shared_query = ompi_osc_ucx_shared_query,
         .osc_win_attach = ompi_osc_ucx_win_attach,
         .osc_win_detach = ompi_osc_ucx_win_detach,
         .osc_free = ompi_osc_ucx_free,
@@ -138,6 +149,14 @@ static bool check_config_value_bool (char *key, opal_info_t *info)
 }
 
 static int component_open(void) {
+    opal_common_ucx_mca_register();
+
+    return OMPI_SUCCESS;
+}
+
+static int component_close(void) {
+    opal_common_ucx_mca_deregister();
+
     return OMPI_SUCCESS;
 }
 
@@ -168,17 +187,51 @@ static int component_register(void) {
                                            MCA_BASE_VAR_SCOPE_GROUP, &mca_osc_ucx_component.no_locks);
     free(description_str);
 
+    opal_common_ucx_thread_enabled = opal_using_threads();
     mca_osc_ucx_component.acc_single_intrinsic = false;
+
     opal_asprintf(&description_str, "Enable optimizations for MPI_Fetch_and_op, MPI_Accumulate, etc for codes "
              "that will not use anything more than a single predefined datatype (default: %s)",
              mca_osc_ucx_component.acc_single_intrinsic  ? "true" : "false");
     (void) mca_base_component_var_register(&mca_osc_ucx_component.super.osc_version, "acc_single_intrinsic",
                                            description_str, MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_5,
                                            MCA_BASE_VAR_SCOPE_GROUP, &mca_osc_ucx_component.acc_single_intrinsic);
+
+    opal_asprintf(&description_str, "Enable nonblocking MPI_Accumulate and MPI_Get_accumulate  (default: %s)",
+                                           enable_nonblocking_accumulate  ? "true" : "false");
+    (void) mca_base_component_var_register(&mca_osc_ucx_component.super.osc_version, "enable_nonblocking_accumulate",
+                                           description_str, MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                           MCA_BASE_VAR_SCOPE_GROUP, &enable_nonblocking_accumulate);
+
+    opal_asprintf(&description_str, "Enable optimizations for multi-threaded applications by allocating a separate worker "
+                                    "for each thread and a separate endpoint for each window  (default: %s)",
+                                         opal_common_ucx_thread_enabled  ? "true" : "false");
+    (void) mca_base_component_var_register(&mca_osc_ucx_component.super.osc_version, "enable_wpool_thread_multiple",
+                                           description_str, MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                           MCA_BASE_VAR_SCOPE_GROUP, &opal_common_ucx_thread_enabled);
+
+    opal_asprintf(&description_str, "Threshold on number of nonblocking accumulate calls on which there is a  periodical "
+                                        "flush (default: %ld)", ompi_osc_ucx_outstanding_ops_flush_threshold);
+    (void) mca_base_component_var_register(&mca_osc_ucx_component.super.osc_version, "outstanding_ops_flush_threshold",
+                                           description_str, MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                           MCA_BASE_VAR_SCOPE_GROUP, &ompi_osc_ucx_outstanding_ops_flush_threshold);
     free(description_str);
 
     opal_common_ucx_mca_var_register(&mca_osc_ucx_component.super.osc_version);
 
+    if (0 == access ("/dev/shm", W_OK)) {
+        mca_osc_ucx_component.backing_directory = "/dev/shm";
+    } else {
+        mca_osc_ucx_component.backing_directory = ompi_process_info.proc_session_dir;
+    }
+
+    (void) mca_base_component_var_register (&mca_osc_ucx_component.super.osc_version, "backing_directory",
+                                            "Directory to place backing files for memory windows. "
+                                            "This directory should be on a local filesystem such as /tmp or "
+                                            "/dev/shm (default: (linux) /dev/shm, (others) session directory)",
+                                            MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_3,
+                                            MCA_BASE_VAR_SCOPE_READONLY, &mca_osc_ucx_component.backing_directory);
+
     return OMPI_SUCCESS;
 }
 
@@ -189,25 +242,136 @@ static int progress_callback(void) {
     return 0;
 }
 
+static int ucp_context_init(bool enable_mt, int proc_world_size) {
+    int ret = OMPI_SUCCESS;
+    ucs_status_t status;
+    ucp_config_t *config = NULL;
+    ucp_params_t context_params;
+
+    status = ucp_config_read("MPI", NULL, &config);
+    if (UCS_OK != status) {
+        OSC_UCX_VERBOSE(1, "ucp_config_read failed: %d", status);
+        return OMPI_ERROR;
+    }
+
+    /* initialize UCP context */
+    memset(&context_params, 0, sizeof(context_params));
+    context_params.field_mask = UCP_PARAM_FIELD_FEATURES | UCP_PARAM_FIELD_MT_WORKERS_SHARED
+                                | UCP_PARAM_FIELD_ESTIMATED_NUM_EPS | UCP_PARAM_FIELD_REQUEST_INIT
+                                | UCP_PARAM_FIELD_REQUEST_SIZE;
+    context_params.features = UCP_FEATURE_RMA | UCP_FEATURE_AMO32 | UCP_FEATURE_AMO64;
+    context_params.mt_workers_shared = (enable_mt ? 1 : 0);
+    context_params.estimated_num_eps = proc_world_size;
+    context_params.request_init = opal_common_ucx_req_init;
+    context_params.request_size = sizeof(opal_common_ucx_request_t);
+
+#if HAVE_DECL_UCP_PARAM_FIELD_ESTIMATED_NUM_PPN
+    context_params.estimated_num_ppn = opal_process_info.num_local_peers + 1;
+    context_params.field_mask |= UCP_PARAM_FIELD_ESTIMATED_NUM_PPN;
+#endif
+
+    status = ucp_init(&context_params, config, &mca_osc_ucx_component.wpool->ucp_ctx);
+    if (UCS_OK != status) {
+        OSC_UCX_VERBOSE(1, "ucp_init failed: %d", status);
+        ret = OMPI_ERROR;
+    }
+    ucp_config_release(config);
+
+    return ret;
+}
+
 static int component_init(bool enable_progress_threads, bool enable_mpi_threads) {
+
     mca_osc_ucx_component.enable_mpi_threads = enable_mpi_threads;
     mca_osc_ucx_component.wpool = opal_common_ucx_wpool_allocate();
-    opal_common_ucx_mca_register();
+    mca_osc_ucx_component.priority_is_set = false;
+
+    return OMPI_SUCCESS;
+}
+
+static int component_set_priority() {
+    int param, ret;
+    opal_common_ucx_support_level_t support_level = OPAL_COMMON_UCX_SUPPORT_NONE;
+    mca_base_var_source_t param_source = MCA_BASE_VAR_SOURCE_DEFAULT;
+
+    if (mca_osc_ucx_component.priority_is_set == true) {
+        return OMPI_SUCCESS;
+    }
+
+    if (mca_osc_ucx_component.wpool == NULL) {
+        mca_osc_ucx_component.wpool = opal_common_ucx_wpool_allocate();
+    }
+
+    if (mca_osc_ucx_component.wpool->ucp_ctx == NULL) {
+        ret = ucp_context_init(mca_osc_ucx_component.enable_mpi_threads,  ompi_proc_world_size());
+        if (OMPI_ERROR == ret) {
+            return OMPI_ERR_NOT_AVAILABLE;
+        }
+    }
+
+    support_level = opal_common_ucx_support_level(mca_osc_ucx_component.wpool->ucp_ctx);
+    if (OPAL_COMMON_UCX_SUPPORT_NONE == support_level) {
+        ucp_cleanup(mca_osc_ucx_component.wpool->ucp_ctx);
+        mca_osc_ucx_component.wpool->ucp_ctx = NULL;
+        return OMPI_ERR_NOT_AVAILABLE;
+    }
+
+    param = mca_base_var_find("ompi","osc","ucx","priority");
+    if (0 <= param) {
+        (void) mca_base_var_get_value(param, NULL, &param_source, NULL);
+    }
+
+    /*
+     * Retain priority if we have supported devices and transports.
+     * Lower priority if we have supported transports, but not supported devices.
+     */
+    if(MCA_BASE_VAR_SOURCE_DEFAULT == param_source) {
+        mca_osc_ucx_component.priority = (support_level == OPAL_COMMON_UCX_SUPPORT_DEVICE) ?
+                    mca_osc_ucx_component.priority : 9;
+    }
+    OSC_UCX_VERBOSE(2, "returning priority %d", mca_osc_ucx_component.priority);
+
+    mca_osc_ucx_component.priority_is_set = true;
+
     return OMPI_SUCCESS;
 }
 
 static int component_finalize(void) {
+
+    if (!opal_common_ucx_thread_enabled) {
+        int i;
+        for (i = 0; i < mca_osc_ucx_component.comm_world_size; i++) {
+            ucp_ep_h ep = mca_osc_ucx_component.endpoints[i];
+            if (ep != NULL) {
+                ucp_ep_destroy(ep);
+                OPAL_COMMON_UCX_DEBUG_ATOMIC_ADD(opal_common_ucx_ep_counts, -1);
+            }
+        }
+        free(mca_osc_ucx_component.endpoints);
+    }
     opal_common_ucx_mca_deregister();
     if (mca_osc_ucx_component.env_initialized) {
         opal_common_ucx_wpool_finalize(mca_osc_ucx_component.wpool);
+        OBJ_DESTRUCT(&mca_osc_ucx_component.accumulate_requests);
+        OBJ_DESTRUCT(&mca_osc_ucx_component.requests);
     }
     opal_common_ucx_wpool_free(mca_osc_ucx_component.wpool);
+
+    assert(opal_common_ucx_ep_counts == 0);
+    assert(opal_common_ucx_unpacked_rkey_counts == 0);
     return OMPI_SUCCESS;
 }
 
-static int component_query(struct ompi_win_t *win, void **base, size_t size, int disp_unit,
+static int component_query(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t disp_unit,
                            struct ompi_communicator_t *comm, struct opal_info_t *info, int flavor) {
-    if (MPI_WIN_FLAVOR_SHARED == flavor) return -1;
+    int ret;
+    if (mca_osc_ucx_component.priority_is_set == false) {
+        ret = component_set_priority();
+        if (OMPI_SUCCESS != ret) {
+            OSC_UCX_ERROR("OSC UCX component priority set inside component query failed \n ");
+            return ret;
+        }
+    }
     return mca_osc_ucx_component.priority;
 }
 
@@ -217,13 +381,16 @@ static int exchange_len_info(void *my_info, size_t my_info_len, char **recv_info
     int ret = OMPI_SUCCESS;
     struct ompi_communicator_t *comm = (struct ompi_communicator_t *)metadata;
     int comm_size = ompi_comm_size(comm);
-    int lens[comm_size];
+    int *lens = calloc(comm_size, sizeof(int));
     int total_len, i;
+    ompi_count_array_t lens_desc;
+    ompi_disp_array_t disps_desc;
 
     ret = comm->c_coll->coll_allgather(&my_info_len, 1, MPI_INT,
                                        lens, 1, MPI_INT, comm,
                                        comm->c_coll->coll_allgather_module);
     if (OMPI_SUCCESS != ret) {
+        free(lens);
         return ret;
     }
 
@@ -235,13 +402,17 @@ static int exchange_len_info(void *my_info, size_t my_info_len, char **recv_info
     }
 
     (*recv_info_ptr) = (char *)calloc(total_len, sizeof(char));
+    OMPI_COUNT_ARRAY_INIT(&lens_desc, lens);
+    OMPI_DISP_ARRAY_INIT(&disps_desc, *disps_ptr);
     ret = comm->c_coll->coll_allgatherv(my_info, my_info_len, MPI_BYTE,
-                                        (void *)(*recv_info_ptr), lens, (*disps_ptr), MPI_BYTE,
+                                        (void *)(*recv_info_ptr), lens_desc, disps_desc, MPI_BYTE,
                                         comm, comm->c_coll->coll_allgatherv_module);
     if (OMPI_SUCCESS != ret) {
+        free(lens);
         return ret;
     }
 
+    free(lens);
     return ret;
 }
 
@@ -264,7 +435,7 @@ static void ompi_osc_ucx_unregister_progress()
     _osc_ucx_init_unlock();
 }
 
-static char* ompi_osc_ucx_set_no_lock_info(opal_infosubscriber_t *obj, char *key, char *value)
+static const char* ompi_osc_ucx_set_no_lock_info(opal_infosubscriber_t *obj, const char *key, const char *value)
 {
 
     struct ompi_win_t *win = (struct ompi_win_t*) obj;
@@ -296,58 +467,117 @@ static char* ompi_osc_ucx_set_no_lock_info(opal_infosubscriber_t *obj, char *key
     return module->no_locks ? "true" : "false";
 }
 
-static int component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit,
+int ompi_osc_ucx_shared_query(struct ompi_win_t *win, int rank, size_t *size,
+        ptrdiff_t *disp_unit, void *baseptr)
+{
+    ompi_osc_ucx_module_t *module =
+        (ompi_osc_ucx_module_t*) win->w_osc_module;
+
+    if (module->flavor != MPI_WIN_FLAVOR_SHARED) {
+        return MPI_ERR_WIN;
+    }
+
+    if (MPI_PROC_NULL != rank) {
+        *size = module->sizes[rank];
+        *((void**) baseptr) = (void *)module->shmem_addrs[rank];
+        if (module->disp_unit == -1) {
+            *disp_unit = module->disp_units[rank];
+        } else {
+            *disp_unit = module->disp_unit;
+        }
+    } else {
+        int i = 0;
+
+        *size = 0;
+        *((void**) baseptr) = NULL;
+        *disp_unit = 0;
+        for (i = 0 ; i < ompi_comm_size(module->comm) ; ++i) {
+            if (0 != module->sizes[i]) {
+                *size = module->sizes[i];
+                *((void**) baseptr) = (void *)module->shmem_addrs[i];
+                if (module->disp_unit == -1) {
+                    *disp_unit = module->disp_units[rank];
+                } else {
+                    *disp_unit = module->disp_unit;
+                }
+                break;
+            }
+        }
+    }
+
+    return OMPI_SUCCESS;
+}
+
+static int component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t disp_unit,
                             struct ompi_communicator_t *comm, struct opal_info_t *info,
                             int flavor, int *model) {
     ompi_osc_ucx_module_t *module = NULL;
     char *name = NULL;
     long values[2];
     int ret = OMPI_SUCCESS;
-    //ucs_status_t status;
     int i, comm_size = ompi_comm_size(comm);
     bool env_initialized = false;
     void *state_base = NULL;
     opal_common_ucx_mem_type_t mem_type;
-    uint64_t zero = 0;
     char *my_mem_addr;
     int my_mem_addr_size;
-    void * my_info = NULL;
+    uint64_t my_info[3] = {0};
     char *recv_buf = NULL;
-
-    /* the osc/sm component is the exclusive provider for support for
-     * shared memory windows */
-    if (flavor == MPI_WIN_FLAVOR_SHARED) {
-        return OMPI_ERR_NOT_SUPPORTED;
-    }
+    void *dynamic_base = NULL;
+    unsigned long total, *rbuf;
+    int flag;
+    size_t pagesize;
+    bool unlink_needed = false;
 
     /* May be called concurrently - protect */
     _osc_ucx_init_lock();
 
+
     if (mca_osc_ucx_component.env_initialized == false) {
         /* Lazy initialization of the global state.
          * As not all of the MPI applications are using One-Sided functionality
          * we don't want to initialize in the component_init()
          */
 
+        if (mca_osc_ucx_component.priority_is_set == false) {
+            ret = component_set_priority();
+            if (OMPI_SUCCESS != ret) {
+                OSC_UCX_ERROR("OSC UCX component priority set inside component select failed \n ");
+                return ret;
+            }
+        }
+
         OBJ_CONSTRUCT(&mca_osc_ucx_component.requests, opal_free_list_t);
         ret = opal_free_list_init (&mca_osc_ucx_component.requests,
-                                   sizeof(ompi_osc_ucx_request_t),
+                                   sizeof(ompi_osc_ucx_generic_request_t),
                                    opal_cache_line_size,
-                                   OBJ_CLASS(ompi_osc_ucx_request_t),
+                                   OBJ_CLASS(ompi_osc_ucx_generic_request_t),
                                    0, 0, 8, 0, 8, NULL, 0, NULL, NULL, NULL);
         if (OMPI_SUCCESS != ret) {
             OSC_UCX_VERBOSE(1, "opal_free_list_init failed: %d", ret);
             goto select_unlock;
         }
 
-        ret = opal_common_ucx_wpool_init(mca_osc_ucx_component.wpool,
-                                         ompi_proc_world_size(),
-                                         mca_osc_ucx_component.enable_mpi_threads);
+        OBJ_CONSTRUCT(&mca_osc_ucx_component.accumulate_requests, opal_free_list_t);
+        ret = opal_free_list_init (&mca_osc_ucx_component.accumulate_requests,
+                                   sizeof(ompi_osc_ucx_accumulate_request_t),
+                                   opal_cache_line_size,
+                                   OBJ_CLASS(ompi_osc_ucx_accumulate_request_t),
+                                   0, 0, 8, 0, 8, NULL, 0, NULL, NULL, NULL);
         if (OMPI_SUCCESS != ret) {
-            OSC_UCX_VERBOSE(1, "opal_common_ucx_wpool_init failed: %d", ret);
+            OSC_UCX_VERBOSE(1, "opal_free_list_init failed: %d", ret);
             goto select_unlock;
         }
 
+        ret = opal_common_ucx_wpool_init(mca_osc_ucx_component.wpool);
+        if (OMPI_SUCCESS != ret) {
+            OSC_UCX_VERBOSE(1, "opal_common_ucx_wpool_init failed: %d", ret);
+            goto select_unlock;
+        }
+        if (!opal_common_ucx_thread_enabled) {
+            mca_osc_ucx_component.comm_world_size = ompi_proc_world_size();
+            mca_osc_ucx_component.endpoints = calloc(mca_osc_ucx_component.comm_world_size, sizeof(ucp_ep_h));
+        }
         /* Make sure that all memory updates performed above are globally
          * observable before (mca_osc_ucx_component.env_initialized = true)
          */
@@ -386,13 +616,19 @@ static int component_select(struct ompi_win_t *win, void **base, size_t size, in
     /* fill in the function pointer part */
     memcpy(module, &ompi_osc_ucx_module_template, sizeof(ompi_osc_base_module_t));
 
+    /* TODO Provide support for nonblocking operations with dynamic windows */
+    if (enable_nonblocking_accumulate && flavor != MPI_WIN_FLAVOR_DYNAMIC) {
+        module->super.osc_accumulate = ompi_osc_ucx_accumulate_nb;
+        module->super.osc_get_accumulate = ompi_osc_ucx_get_accumulate_nb;
+    }
+
     ret = ompi_comm_dup(comm, &module->comm);
     if (ret != OMPI_SUCCESS) {
         goto error;
     }
 
     *model = MPI_WIN_UNIFIED;
-    opal_asprintf(&name, "ucx window %d", ompi_comm_get_cid(module->comm));
+    opal_asprintf(&name, "ucx window %s", ompi_comm_print_cid(module->comm));
     ompi_win_set_name(win, name);
     free(name);
 
@@ -400,6 +636,7 @@ static int component_select(struct ompi_win_t *win, void **base, size_t size, in
     module->size = size;
     module->no_locks = check_config_value_bool ("no_locks", info);
     module->acc_single_intrinsic = check_config_value_bool ("acc_single_intrinsic", info);
+    module->skip_sync_check = false;
 
     /* share everyone's displacement units. Only do an allgather if
        strictly necessary, since it requires O(p) state. */
@@ -417,14 +654,14 @@ static int component_select(struct ompi_win_t *win, void **base, size_t size, in
         module->disp_unit = disp_unit;
     } else { /* different disp_unit sizes, allocate O(p) space to store them */
         module->disp_unit = -1;
-        module->disp_units = calloc(comm_size, sizeof(int));
+        module->disp_units = calloc(comm_size, sizeof(ptrdiff_t));
         if (module->disp_units == NULL) {
             ret = OMPI_ERR_TEMP_OUT_OF_RESOURCE;
             goto error;
         }
 
-        ret = module->comm->c_coll->coll_allgather(&disp_unit, 1, MPI_INT,
-                                                   module->disp_units, 1, MPI_INT,
+        ret = module->comm->c_coll->coll_allgather(&disp_unit, sizeof(ptrdiff_t), MPI_BYTE,
+                                                   module->disp_units, sizeof(ptrdiff_t), MPI_BYTE,
                                                    module->comm,
                                                    module->comm->c_coll->coll_allgather_module);
         if (OMPI_SUCCESS != ret) {
@@ -439,65 +676,219 @@ static int component_select(struct ompi_win_t *win, void **base, size_t size, in
         goto error;
     }
 
-    if (flavor == MPI_WIN_FLAVOR_ALLOCATE || flavor == MPI_WIN_FLAVOR_CREATE) {
-        switch (flavor) {
-        case MPI_WIN_FLAVOR_ALLOCATE:
-            mem_type = OPAL_COMMON_UCX_MEM_ALLOCATE_MAP;
-            break;
-        case MPI_WIN_FLAVOR_CREATE:
-            mem_type = OPAL_COMMON_UCX_MEM_MAP;
-            break;
+    if (flavor == MPI_WIN_FLAVOR_SHARED) {
+        /* create the segment */
+        opal_output_verbose(MCA_BASE_VERBOSE_DEBUG, ompi_osc_base_framework.framework_output,
+                            "allocating shared memory region of size %ld\n", (long) size);
+        /* get the pagesize */
+        pagesize = opal_getpagesize();
+
+        rbuf = malloc(sizeof(unsigned long) * comm_size);
+        if (NULL == rbuf) return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
+
+        /* Note that the alloc_shared_noncontig info key only has
+         * meaning during window creation.  Once the window is
+         * created, we can't move memory around without making
+         * everything miserable.  So we intentionally do not subscribe
+         * to updates on the info key, because there's no useful
+         * update to occur. */
+        module->noncontig_shared_win = false;
+        if (OMPI_SUCCESS != opal_info_get_bool(info, "alloc_shared_noncontig",
+                                               &module->noncontig_shared_win, &flag)) {
+            free(rbuf);
+            goto error;
+        }
+
+        if (module->noncontig_shared_win) {
+            opal_output_verbose(MCA_BASE_VERBOSE_DEBUG, ompi_osc_base_framework.framework_output,
+                                "allocating window using non-contiguous strategy");
+            total = ((size - 1) / pagesize + 1) * pagesize;
+        } else {
+            opal_output_verbose(MCA_BASE_VERBOSE_DEBUG, ompi_osc_base_framework.framework_output,
+                                "allocating window using contiguous strategy");
+            total = size;
+        }
+        ret = module->comm->c_coll->coll_allgather(&total, 1, MPI_UNSIGNED_LONG,
+                                                  rbuf, 1, MPI_UNSIGNED_LONG,
+                                                  module->comm,
+                                                  module->comm->c_coll->coll_allgather_module);
+        if (OMPI_SUCCESS != ret) return ret;
+
+        total = 0;
+        for (i = 0 ; i < comm_size ; ++i) {
+            total += rbuf[i];
+        }
+
+        module->segment_base = NULL;
+        module->shmem_addrs = NULL;
+        module->sizes = NULL;
+
+        if (total != 0) {
+            /* user opal/shmem directly to create a shared memory segment */
+            if (0 == ompi_comm_rank (module->comm)) {
+                char *data_file;
+                ret = opal_asprintf (&data_file, "%s" OPAL_PATH_SEP "osc_ucx.%s.%x.%d.%s",
+                                     mca_osc_ucx_component.backing_directory, ompi_process_info.nodename,
+                                     OMPI_PROC_MY_NAME->jobid, (int) OMPI_PROC_MY_NAME->vpid,
+                                     ompi_comm_print_cid(module->comm));
+                if (ret < 0) {
+                    free(rbuf);
+                    return OMPI_ERR_OUT_OF_RESOURCE;
+                }
+
+                ret = opal_shmem_segment_create (&module->seg_ds, data_file, total);
+                free(data_file);
+                if (OPAL_SUCCESS != ret) {
+                    free(rbuf);
+                    goto error;
+                }
+
+                unlink_needed = true;
+            }
+
+            ret = module->comm->c_coll->coll_bcast (&module->seg_ds, sizeof (module->seg_ds), MPI_BYTE, 0,
+                                                    module->comm, module->comm->c_coll->coll_bcast_module);
+            if (OMPI_SUCCESS != ret) {
+                free(rbuf);
+                goto error;
+            }
+
+            module->segment_base = opal_shmem_segment_attach (&module->seg_ds);
+            if (NULL == module->segment_base) {
+                free(rbuf);
+                goto error;
+            }
+
+            /* wait for all processes to attach */
+            ret = module->comm->c_coll->coll_barrier (module->comm, module->comm->c_coll->coll_barrier_module);
+            if (OMPI_SUCCESS != ret) {
+                free(rbuf);
+                goto error;
+            }
+
+            if (0 == ompi_comm_rank (module->comm)) {
+                opal_shmem_unlink (&module->seg_ds);
+                unlink_needed = false;
+            }
         }
 
-        ret = opal_common_ucx_wpmem_create(module->ctx, base, size,
-                                         mem_type, &exchange_len_info,
-                                         (void *)module->comm,
-                                           &my_mem_addr, &my_mem_addr_size,
-                                           &module->mem);
-        if (ret != OMPI_SUCCESS) {
+        /* Although module->segment_base is pointing to a same physical address
+         * for all the processes, its value which is a virtual address can be
+         * different between different processes. To use direct load/store,
+         * shmem_addrs can be used, however, for RDMA, virtual address of
+         * remote process that will be stored in module->addrs should be used */
+        module->sizes = malloc(sizeof(size_t) * comm_size);
+        if (NULL == module->sizes) {
+            free(rbuf);
+            ret = OMPI_ERR_TEMP_OUT_OF_RESOURCE;
+            goto error;
+        }
+        module->shmem_addrs = malloc(sizeof(uint64_t) * comm_size);
+        if (NULL == module->shmem_addrs) {
+            free(module->sizes);
+            free(rbuf);
+            ret =  OMPI_ERR_TEMP_OUT_OF_RESOURCE;
             goto error;
         }
 
+
+        for (i = 0, total = 0; i < comm_size ; ++i) {
+            module->sizes[i] = rbuf[i];
+            if (module->sizes[i] || !module->noncontig_shared_win) {
+                module->shmem_addrs[i] = ((uint64_t) module->segment_base) + total;
+                total += rbuf[i];
+            } else {
+                module->shmem_addrs[i] = (uint64_t)NULL;
+            }
+        }
+
+        free(rbuf);
+
+        module->size = module->sizes[ompi_comm_rank(module->comm)];
+        *base = (void *)module->shmem_addrs[ompi_comm_rank(module->comm)];
+    }
+
+    void **mem_base = base;
+    switch (flavor) {
+    case MPI_WIN_FLAVOR_DYNAMIC:
+        mem_type = OPAL_COMMON_UCX_MEM_ALLOCATE_MAP;
+        module->size = 0;
+        mem_base = &dynamic_base;
+        break;
+    case MPI_WIN_FLAVOR_ALLOCATE:
+        mem_type = OPAL_COMMON_UCX_MEM_ALLOCATE_MAP;
+        break;
+    case MPI_WIN_FLAVOR_CREATE:
+        mem_type = OPAL_COMMON_UCX_MEM_MAP;
+        break;
+    case MPI_WIN_FLAVOR_SHARED:
+        mem_type = OPAL_COMMON_UCX_MEM_MAP;
+        break;
+    default:
+        ret = OMPI_ERR_BAD_PARAM;
+        goto error;
+    }
+    ret = opal_common_ucx_wpmem_create(module->ctx, mem_base, module->size,
+                                     mem_type, &exchange_len_info,
+                                     OPAL_COMMON_UCX_WPMEM_ADDR_EXCHANGE_FULL,
+                                     (void *)module->comm,
+                                       &my_mem_addr, &my_mem_addr_size,
+                                       &module->mem);
+    if (ret != OMPI_SUCCESS) {
+        goto error;
+    }
+
+    if (my_mem_addr_size != 0) {
+        /* rkey object is already distributed among comm processes */
+        ucp_rkey_buffer_release(my_mem_addr);
     }
 
     state_base = (void *)&(module->state);
     ret = opal_common_ucx_wpmem_create(module->ctx, &state_base,
                                      sizeof(ompi_osc_ucx_state_t),
-                                     OPAL_COMMON_UCX_MEM_MAP, &exchange_len_info,
+                                     OPAL_COMMON_UCX_MEM_MAP,
+                                     &exchange_len_info,
+                                     OPAL_COMMON_UCX_WPMEM_ADDR_EXCHANGE_FULL,
                                      (void *)module->comm,
-                                       &my_mem_addr, &my_mem_addr_size,
-                                       &module->state_mem);
+                                     &my_mem_addr, &my_mem_addr_size,
+                                     &module->state_mem);
     if (ret != OMPI_SUCCESS) {
         goto error;
     }
 
-    /* exchange window addrs */
-    my_info = malloc(2 * sizeof(uint64_t));
-    if (my_info == NULL) {
-        ret = OMPI_ERR_TEMP_OUT_OF_RESOURCE;
-        goto error;
+    if (my_mem_addr_size != 0) {
+        /* rkey object is already distributed among comm processes */
+        ucp_rkey_buffer_release(my_mem_addr);
     }
 
-    if (flavor == MPI_WIN_FLAVOR_ALLOCATE || flavor == MPI_WIN_FLAVOR_CREATE) {
-        memcpy(my_info, base, sizeof(uint64_t));
-    } else {
-        memcpy(my_info, &zero, sizeof(uint64_t));
+    /* exchange window addrs */
+    if (flavor == MPI_WIN_FLAVOR_ALLOCATE || flavor == MPI_WIN_FLAVOR_CREATE ||
+            flavor == MPI_WIN_FLAVOR_SHARED) {
+        my_info[0] = (uint64_t)*base;
+    } else if (flavor == MPI_WIN_FLAVOR_DYNAMIC) {
+        my_info[0] = (uint64_t)dynamic_base;
     }
-    memcpy((char*)my_info + sizeof(uint64_t), &state_base, sizeof(uint64_t));
+    my_info[1] = (uint64_t)state_base;
+    my_info[2] = ompi_comm_rank(&ompi_mpi_comm_world.comm);
 
-    recv_buf = (char *)calloc(comm_size, 2 * sizeof(uint64_t));
-    ret = comm->c_coll->coll_allgather((void *)my_info, 2 * sizeof(uint64_t),
-                                       MPI_BYTE, recv_buf, 2 * sizeof(uint64_t),
+    recv_buf = (char *)calloc(comm_size, sizeof(my_info));
+    ret = comm->c_coll->coll_allgather((void *)my_info, sizeof(my_info),
+                                       MPI_BYTE, recv_buf, sizeof(my_info),
                                        MPI_BYTE, comm, comm->c_coll->coll_allgather_module);
     if (ret != OMPI_SUCCESS) {
         goto error;
     }
 
+    module->epoc_outstanding_ops_mems =
+        calloc(ompi_osc_ucx_outstanding_ops_flush_threshold,
+                sizeof(ompi_osc_ucx_mem_ranges_t));
     module->addrs = calloc(comm_size, sizeof(uint64_t));
     module->state_addrs = calloc(comm_size, sizeof(uint64_t));
+    module->comm_world_ranks = calloc(comm_size, sizeof(uint64_t));
     for (i = 0; i < comm_size; i++) {
-        memcpy(&(module->addrs[i]), recv_buf + i * 2 * sizeof(uint64_t), sizeof(uint64_t));
-        memcpy(&(module->state_addrs[i]), recv_buf + i * 2 * sizeof(uint64_t) + sizeof(uint64_t), sizeof(uint64_t));
+        memcpy(&(module->addrs[i]), recv_buf + i * 3 * sizeof(uint64_t), sizeof(uint64_t));
+        memcpy(&(module->state_addrs[i]), recv_buf + i * 3 * sizeof(uint64_t) + sizeof(uint64_t), sizeof(uint64_t));
+        memcpy(&(module->comm_world_ranks[i]), recv_buf + i * 3 * sizeof(uint64_t) + 2 * sizeof(uint64_t), sizeof(uint64_t));
     }
     free(recv_buf);
 
@@ -508,6 +899,7 @@ static int component_select(struct ompi_win_t *win, void **base, size_t size, in
     module->state.complete_count = 0;
     module->state.req_flag = 0;
     module->state.acc_lock = TARGET_LOCK_UNLOCKED;
+    module->state.dynamic_lock = TARGET_LOCK_UNLOCKED;
     module->state.dynamic_win_count = 0;
     for (i = 0; i < OMPI_OSC_UCX_ATTACH_MAX; i++) {
         module->local_dynamic_win_info[i].refcnt = 0;
@@ -550,6 +942,7 @@ static int component_select(struct ompi_win_t *win, void **base, size_t size, in
     if (module->disp_units) free(module->disp_units);
     if (module->comm) ompi_comm_free(&module->comm);
     free(module);
+    module = NULL;
 
 error_nomem:
     if (env_initialized == true) {
@@ -558,6 +951,9 @@ static int component_select(struct ompi_win_t *win, void **base, size_t size, in
         mca_osc_ucx_component.env_initialized = false;
     }
 
+    if ((NULL != module) && (0 == ompi_comm_rank (module->comm)) && unlink_needed) {
+        opal_shmem_unlink (&module->seg_ds);
+    }
     ompi_osc_ucx_unregister_progress();
     return ret;
 }
@@ -567,6 +963,10 @@ int ompi_osc_find_attached_region_position(ompi_osc_dynamic_win_info_t *dynamic_
                                            uint64_t base, size_t len, int *insert) {
     int mid_index = (max_index + min_index) >> 1;
 
+    if (dynamic_wins[mid_index].size == 1) {
+        len = 0;
+    }
+
     if (min_index > max_index) {
         (*insert) = min_index;
         return -1;
@@ -575,7 +975,7 @@ int ompi_osc_find_attached_region_position(ompi_osc_dynamic_win_info_t *dynamic_
     if (dynamic_wins[mid_index].base > base) {
         return ompi_osc_find_attached_region_position(dynamic_wins, min_index, mid_index-1,
                                                       base, len, insert);
-    } else if (base + len < dynamic_wins[mid_index].base + dynamic_wins[mid_index].size) {
+    } else if (base + len <= dynamic_wins[mid_index].base + dynamic_wins[mid_index].size) {
         return mid_index;
     } else {
         return ompi_osc_find_attached_region_position(dynamic_wins, mid_index+1, max_index,
@@ -583,25 +983,86 @@ int ompi_osc_find_attached_region_position(ompi_osc_dynamic_win_info_t *dynamic_
     }
 }
 
+int ompi_osc_ucx_dynamic_lock(ompi_osc_ucx_module_t *module, int target) {
+    uint64_t result_value = -1;
+    uint64_t remote_addr = (module->state_addrs)[target] + OSC_UCX_STATE_DYNAMIC_LOCK_OFFSET;
+    ucp_ep_h *ep;
+    OSC_UCX_GET_DEFAULT_EP(ep, module, target);
+    int ret;
+
+    for (;;) {
+        ret = opal_common_ucx_wpmem_cmpswp(module->state_mem,
+                                        TARGET_LOCK_UNLOCKED, TARGET_LOCK_EXCLUSIVE,
+                                        target, &result_value, sizeof(result_value),
+                                        remote_addr, ep);
+        if (OPAL_SUCCESS != ret) {
+            OSC_UCX_VERBOSE(1, "opal_common_ucx_mem_cmpswp failed: %d", ret);
+            return OMPI_ERROR;
+        }
+        if (result_value == TARGET_LOCK_UNLOCKED) {
+            break;
+        }
+
+        opal_common_ucx_wpool_progress(mca_osc_ucx_component.wpool);
+    }
+
+    return OMPI_SUCCESS;
+}
+
+int ompi_osc_ucx_dynamic_unlock(ompi_osc_ucx_module_t *module, int target) {
+    uint64_t result_value = -1;
+    uint64_t remote_addr = (module->state_addrs)[target] + OSC_UCX_STATE_DYNAMIC_LOCK_OFFSET;
+    ucp_ep_h *ep;
+    OSC_UCX_GET_DEFAULT_EP(ep, module, target);
+    int ret;
+
+    ret = opal_common_ucx_wpmem_fence(module->mem);
+    if (OPAL_SUCCESS != ret) {
+        OSC_UCX_VERBOSE(1, "opal_common_ucx_mem_fence failed: %d", ret);
+        return OMPI_ERROR;
+    }
+
+    ret = opal_common_ucx_wpmem_fetch(module->state_mem,
+                                    UCP_ATOMIC_FETCH_OP_SWAP, TARGET_LOCK_UNLOCKED,
+                                    target, &result_value, sizeof(result_value),
+                                    remote_addr, ep);
+    if (OPAL_SUCCESS != ret) {
+        return OMPI_ERROR;
+    }
+
+    assert(result_value == TARGET_LOCK_EXCLUSIVE);
+    return OMPI_SUCCESS;
+}
+
 int ompi_osc_ucx_win_attach(struct ompi_win_t *win, void *base, size_t len) {
     ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module;
     int insert_index = -1, contain_index;
     int ret = OMPI_SUCCESS;
 
     if (module->state.dynamic_win_count >= OMPI_OSC_UCX_ATTACH_MAX) {
+        OSC_UCX_ERROR("Dynamic window attach failed: Cannot satisfy %" PRIu64 "attached windows. "
+                "Max attached windows is %d \n",
+                module->state.dynamic_win_count+1,
+                OMPI_OSC_UCX_ATTACH_MAX);
         return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
     }
 
+    ret = ompi_osc_ucx_dynamic_lock(module, ompi_comm_rank(module->comm));
+    if (ret != OMPI_SUCCESS) {
+        return ret;
+    }
+
     if (module->state.dynamic_win_count > 0) {
         contain_index = ompi_osc_find_attached_region_position((ompi_osc_dynamic_win_info_t *)module->state.dynamic_wins,
-                                                               0, (int)module->state.dynamic_win_count,
+                                                               0, (int)module->state.dynamic_win_count - 1,
                                                                (uint64_t)base, len, &insert_index);
         if (contain_index >= 0) {
             module->local_dynamic_win_info[contain_index].refcnt++;
+            ret = ompi_osc_ucx_dynamic_unlock(module, ompi_comm_rank(module->comm));
             return ret;
         }
 
-        assert(insert_index >= 0 && (uint64_t)insert_index < module->state.dynamic_win_count);
+        assert(insert_index >= 0 && (uint64_t)insert_index <= module->state.dynamic_win_count);
 
         memmove((void *)&module->local_dynamic_win_info[insert_index+1],
                 (void *)&module->local_dynamic_win_info[insert_index],
@@ -609,17 +1070,20 @@ int ompi_osc_ucx_win_attach(struct ompi_win_t *win, void *base, size_t len) {
         memmove((void *)&module->state.dynamic_wins[insert_index+1],
                 (void *)&module->state.dynamic_wins[insert_index],
                 (OMPI_OSC_UCX_ATTACH_MAX - (insert_index + 1)) * sizeof(ompi_osc_dynamic_win_info_t));
+        module->local_dynamic_win_info[insert_index].refcnt = 0;
     } else {
         insert_index = 0;
     }
 
     ret = opal_common_ucx_wpmem_create(module->ctx, &base, len,
                                        OPAL_COMMON_UCX_MEM_MAP, &exchange_len_info,
+                                       OPAL_COMMON_UCX_WPMEM_ADDR_EXCHANGE_DIRECT,
                                        (void *)module->comm,
                                        &(module->local_dynamic_win_info[insert_index].my_mem_addr),
                                        &(module->local_dynamic_win_info[insert_index].my_mem_addr_size),
                                        &(module->local_dynamic_win_info[insert_index].mem));
     if (ret != OMPI_SUCCESS) {
+        ompi_osc_ucx_dynamic_unlock(module, ompi_comm_rank(module->comm));
         return ret;
     }
 
@@ -633,12 +1097,18 @@ int ompi_osc_ucx_win_attach(struct ompi_win_t *win, void *base, size_t len) {
     module->local_dynamic_win_info[insert_index].refcnt++;
     module->state.dynamic_win_count++;
 
-    return ret;
+    return ompi_osc_ucx_dynamic_unlock(module, ompi_comm_rank(module->comm));
 }
 
 int ompi_osc_ucx_win_detach(struct ompi_win_t *win, const void *base) {
     ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module;
     int insert, contain;
+    int ret = OMPI_SUCCESS;
+
+    ret = ompi_osc_ucx_dynamic_lock(module, ompi_comm_rank(module->comm));
+    if (ret != OMPI_SUCCESS) {
+        return ret;
+    }
 
     assert(module->state.dynamic_win_count > 0);
 
@@ -649,7 +1119,7 @@ int ompi_osc_ucx_win_detach(struct ompi_win_t *win, const void *base) {
 
     /* if we can't find region - just exit */
     if (contain < 0) {
-        return OMPI_SUCCESS;
+        return ompi_osc_ucx_dynamic_unlock(module, ompi_comm_rank(module->comm));
     }
 
     module->local_dynamic_win_info[contain].refcnt--;
@@ -665,12 +1135,14 @@ int ompi_osc_ucx_win_detach(struct ompi_win_t *win, const void *base) {
         module->state.dynamic_win_count--;
     }
 
-    return OMPI_SUCCESS;
+    return ompi_osc_ucx_dynamic_unlock(module, ompi_comm_rank(module->comm));
+
 }
 
 int ompi_osc_ucx_free(struct ompi_win_t *win) {
     ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module;
     int ret;
+    uint64_t i;
 
     assert(module->lock_count == 0);
     assert(opal_list_is_empty(&module->pending_posts) == true);
@@ -679,7 +1151,10 @@ int ompi_osc_ucx_free(struct ompi_win_t *win) {
     }
     OBJ_DESTRUCT(&module->pending_posts);
 
-    opal_common_ucx_wpmem_flush(module->mem, OPAL_COMMON_UCX_SCOPE_WORKER, 0);
+    ret = opal_common_ucx_ctx_flush(module->ctx, OPAL_COMMON_UCX_SCOPE_WORKER, 0);
+    if (OPAL_SUCCESS != ret) {
+        return ret;
+    }
 
     ret = module->comm->c_coll->coll_barrier(module->comm,
                                              module->comm->c_coll->coll_barrier_module);
@@ -687,11 +1162,39 @@ int ompi_osc_ucx_free(struct ompi_win_t *win) {
         return ret;
     }
 
+    if (module->flavor == MPI_WIN_FLAVOR_SHARED) {
+        if (module->segment_base != NULL)
+            opal_shmem_segment_detach(&module->seg_ds);
+        if (module->shmem_addrs != NULL)
+            free(module->shmem_addrs);
+        if (module->sizes != NULL)
+            free(module->sizes);
+    }
+
+    if (module->flavor == MPI_WIN_FLAVOR_DYNAMIC) {
+       /* MPI_Win_free should detach any memory attached to dynamic windows */
+        for (i = 0; i < module->state.dynamic_win_count; i++) {
+            assert(module->local_dynamic_win_info[i].refcnt >= 1);
+            opal_common_ucx_wpmem_free(module->local_dynamic_win_info[i].mem);
+        }
+        module->state.dynamic_win_count = 0;
+
+        if (module->addrs[ompi_comm_rank(module->comm)] != 0) {
+            free((void *)module->addrs[ompi_comm_rank(module->comm)]);
+        }
+    }
+
+    if (NULL != module->epoc_outstanding_ops_mems) {
+        free(module->epoc_outstanding_ops_mems);
+    }
     free(module->addrs);
     free(module->state_addrs);
+    free(module->comm_world_ranks);
 
     opal_common_ucx_wpmem_free(module->state_mem);
-    opal_common_ucx_wpmem_free(module->mem);
+    if (NULL != module->mem) {
+        opal_common_ucx_wpmem_free(module->mem);
+    }
 
     opal_common_ucx_wpctx_release(module->ctx);
 
diff --git a/ompi/mca/osc/ucx/osc_ucx_passive_target.c b/ompi/mca/osc/ucx/osc_ucx_passive_target.c
index 92fd15d187f..455f7ae3302 100644
--- a/ompi/mca/osc/ucx/osc_ucx_passive_target.c
+++ b/ompi/mca/osc/ucx/osc_ucx_passive_target.c
@@ -21,12 +21,14 @@ OBJ_CLASS_INSTANCE(ompi_osc_ucx_lock_t, opal_object_t, NULL, NULL);
 static inline int start_shared(ompi_osc_ucx_module_t *module, int target) {
     uint64_t result_value = -1;
     uint64_t remote_addr = (module->state_addrs)[target] + OSC_UCX_STATE_LOCK_OFFSET;
+    ucp_ep_h *ep;
+    OSC_UCX_GET_DEFAULT_EP(ep, module, target);
     int ret = OMPI_SUCCESS;
 
     while (true) {
         ret = opal_common_ucx_wpmem_fetch(module->state_mem, UCP_ATOMIC_FETCH_OP_FADD, 1,
                                         target, &result_value, sizeof(result_value),
-                                        remote_addr);
+                                        remote_addr, ep);
         if (OMPI_SUCCESS != ret) {
             return ret;
         }
@@ -35,7 +37,7 @@ static inline int start_shared(ompi_osc_ucx_module_t *module, int target) {
         if (result_value >= TARGET_LOCK_EXCLUSIVE) {
             ret = opal_common_ucx_wpmem_post(module->state_mem,
                                            UCP_ATOMIC_POST_OP_ADD, (-1), target,
-                                           sizeof(uint64_t), remote_addr);
+                                           sizeof(uint64_t), remote_addr, ep);
             if (OMPI_SUCCESS != ret) {
                 return ret;
             }
@@ -50,20 +52,24 @@ static inline int start_shared(ompi_osc_ucx_module_t *module, int target) {
 
 static inline int end_shared(ompi_osc_ucx_module_t *module, int target) {
     uint64_t remote_addr = (module->state_addrs)[target] + OSC_UCX_STATE_LOCK_OFFSET;
+    ucp_ep_h *ep;
+    OSC_UCX_GET_DEFAULT_EP(ep, module, target);
     return opal_common_ucx_wpmem_post(module->state_mem, UCP_ATOMIC_POST_OP_ADD,
-                                    (-1), target, sizeof(uint64_t), remote_addr);
+                                    (-1), target, sizeof(uint64_t), remote_addr, ep);
 }
 
 static inline int start_exclusive(ompi_osc_ucx_module_t *module, int target) {
     uint64_t result_value = -1;
     uint64_t remote_addr = (module->state_addrs)[target] + OSC_UCX_STATE_LOCK_OFFSET;
+    ucp_ep_h *ep;
+    OSC_UCX_GET_DEFAULT_EP(ep, module, target);
     int ret = OMPI_SUCCESS;
 
     for (;;) {
         ret = opal_common_ucx_wpmem_cmpswp(module->state_mem,
                                          TARGET_LOCK_UNLOCKED, TARGET_LOCK_EXCLUSIVE,
                                          target, &result_value, sizeof(result_value),
-                                         remote_addr);
+                                         remote_addr, ep);
         if (OMPI_SUCCESS != ret) {
             return ret;
         }
@@ -76,12 +82,14 @@ static inline int start_exclusive(ompi_osc_ucx_module_t *module, int target) {
 
 static inline int end_exclusive(ompi_osc_ucx_module_t *module, int target) {
     uint64_t remote_addr = (module->state_addrs)[target] + OSC_UCX_STATE_LOCK_OFFSET;
+    ucp_ep_h *ep;
+    OSC_UCX_GET_DEFAULT_EP(ep, module, target);
     return opal_common_ucx_wpmem_post(module->state_mem, UCP_ATOMIC_POST_OP_ADD,
                                       -((int64_t)TARGET_LOCK_EXCLUSIVE), target,
-                                      sizeof(uint64_t), remote_addr);
+                                      sizeof(uint64_t), remote_addr, ep);
 }
 
-int ompi_osc_ucx_lock(int lock_type, int target, int assert, struct ompi_win_t *win) {
+int ompi_osc_ucx_lock(int lock_type, int target, int mpi_assert, struct ompi_win_t *win) {
     ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t *)win->w_osc_module;
     ompi_osc_ucx_lock_t *lock = NULL;
     ompi_osc_ucx_epoch_t original_epoch = module->epoch_type.access;
@@ -113,7 +121,7 @@ int ompi_osc_ucx_lock(int lock_type, int target, int assert, struct ompi_win_t *
     lock = OBJ_NEW(ompi_osc_ucx_lock_t);
     lock->target_rank = target;
 
-    if ((assert & MPI_MODE_NOCHECK) == 0) {
+    if ((mpi_assert & MPI_MODE_NOCHECK) == 0) {
         lock->is_nocheck = false;
         if (lock_type == MPI_LOCK_EXCLUSIVE) {
             ret = start_exclusive(module, target);
@@ -124,6 +132,11 @@ int ompi_osc_ucx_lock(int lock_type, int target, int assert, struct ompi_win_t *
         }
     } else {
         lock->is_nocheck = true;
+        if (lock_type == MPI_LOCK_EXCLUSIVE) {
+            lock->type = LOCK_EXCLUSIVE;
+        } else {
+            lock->type = LOCK_SHARED;
+        }
     }
 
     if (ret == OMPI_SUCCESS) {
@@ -150,13 +163,12 @@ int ompi_osc_ucx_unlock(int target, struct ompi_win_t *win) {
         return OMPI_ERR_RMA_SYNC;
     }
 
-    opal_hash_table_remove_value_uint32(&module->outstanding_locks,
-                                        (uint32_t)target);
-
-    ret = opal_common_ucx_wpmem_flush(module->mem, OPAL_COMMON_UCX_SCOPE_EP, target);
+    ret = opal_common_ucx_ctx_flush(module->ctx, OPAL_COMMON_UCX_SCOPE_WORKER, 0);
     if (ret != OMPI_SUCCESS) {
         return ret;
     }
+    opal_hash_table_remove_value_uint32(&module->outstanding_locks,
+                                        (uint32_t)target);
 
     if (lock->is_nocheck == false) {
         if (lock->type == LOCK_EXCLUSIVE) {
@@ -177,7 +189,7 @@ int ompi_osc_ucx_unlock(int target, struct ompi_win_t *win) {
     return ret;
 }
 
-int ompi_osc_ucx_lock_all(int assert, struct ompi_win_t *win) {
+int ompi_osc_ucx_lock_all(int mpi_assert, struct ompi_win_t *win) {
     ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module;
     int ret = OMPI_SUCCESS;
 
@@ -193,7 +205,7 @@ int ompi_osc_ucx_lock_all(int assert, struct ompi_win_t *win) {
 
     module->epoch_type.access = PASSIVE_ALL_EPOCH;
 
-    if (0 == (assert & MPI_MODE_NOCHECK)) {
+    if (0 == (mpi_assert & MPI_MODE_NOCHECK)) {
         int i, comm_size;
         module->lock_all_is_nocheck = false;
         comm_size = ompi_comm_size(module->comm);
@@ -216,16 +228,15 @@ int ompi_osc_ucx_lock_all(int assert, struct ompi_win_t *win) {
 
 int ompi_osc_ucx_unlock_all(struct ompi_win_t *win) {
     ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*)win->w_osc_module;
-    int comm_size = ompi_comm_size(module->comm);
-    int ret = OMPI_SUCCESS;
+    int comm_size = ompi_comm_size(module->comm), ret = OMPI_SUCCESS;
 
     if (module->epoch_type.access != PASSIVE_ALL_EPOCH) {
         return OMPI_ERR_RMA_SYNC;
     }
 
     assert(module->lock_count == 0);
-
-    ret = opal_common_ucx_wpmem_flush(module->mem, OPAL_COMMON_UCX_SCOPE_WORKER, 0);
+ 
+    ret = opal_common_ucx_ctx_flush(module->ctx, OPAL_COMMON_UCX_SCOPE_WORKER, 0);
     if (ret != OMPI_SUCCESS) {
         return ret;
     }
@@ -270,7 +281,7 @@ int ompi_osc_ucx_flush(int target, struct ompi_win_t *win) {
         return OMPI_ERR_RMA_SYNC;
     }
 
-    ret = opal_common_ucx_wpmem_flush(module->mem, OPAL_COMMON_UCX_SCOPE_EP, target);
+    ret = opal_common_ucx_ctx_flush(module->ctx, OPAL_COMMON_UCX_SCOPE_EP, target);
     if (ret != OMPI_SUCCESS) {
         return ret;
     }
@@ -287,7 +298,7 @@ int ompi_osc_ucx_flush_all(struct ompi_win_t *win) {
         return OMPI_ERR_RMA_SYNC;
     }
 
-    ret = opal_common_ucx_wpmem_flush(module->mem, OPAL_COMMON_UCX_SCOPE_WORKER, 0);
+    ret = opal_common_ucx_ctx_flush(module->ctx, OPAL_COMMON_UCX_SCOPE_WORKER, 0);
     if (ret != OMPI_SUCCESS) {
         return ret;
     }
diff --git a/ompi/mca/osc/ucx/osc_ucx_request.c b/ompi/mca/osc/ucx/osc_ucx_request.c
index d2a3fd494fa..73b5efc733b 100644
--- a/ompi/mca/osc/ucx/osc_ucx_request.c
+++ b/ompi/mca/osc/ucx/osc_ucx_request.c
@@ -24,9 +24,9 @@ static int request_cancel(struct ompi_request_t *request, int complete)
 
 static int request_free(struct ompi_request_t **ompi_req)
 {
-    ompi_osc_ucx_request_t *request = (ompi_osc_ucx_request_t*) *ompi_req;
+    ompi_osc_ucx_generic_request_t *request = (ompi_osc_ucx_generic_request_t*) *ompi_req;
 
-    if (true != (bool)(request->super.req_complete)) {
+    if (true != (bool)(request->super.super.req_complete)) {
         return MPI_ERR_REQUEST;
     }
 
@@ -37,20 +37,15 @@ static int request_free(struct ompi_request_t **ompi_req)
     return OMPI_SUCCESS;
 }
 
-static void request_construct(ompi_osc_ucx_request_t *request)
+static void request_construct(ompi_osc_ucx_generic_request_t *request)
 {
-    request->super.req_type = OMPI_REQUEST_WIN;
-    request->super.req_status._cancelled = 0;
-    request->super.req_free = request_free;
-    request->super.req_cancel = request_cancel;
+    request->super.super.req_type = OMPI_REQUEST_WIN;
+    request->super.super.req_status._cancelled = 0;
+    request->super.super.req_free = request_free;
+    request->super.super.req_cancel = request_cancel;
 }
 
-void req_completion(void *request) {
-    ompi_osc_ucx_request_t *req = (ompi_osc_ucx_request_t *)request;
-    ompi_request_complete(&(req->super), true);
-    mca_osc_ucx_component.num_incomplete_req_ops--;
-    assert(mca_osc_ucx_component.num_incomplete_req_ops >= 0);
-}
-
-OBJ_CLASS_INSTANCE(ompi_osc_ucx_request_t, ompi_request_t,
+OBJ_CLASS_INSTANCE(ompi_osc_ucx_generic_request_t, ompi_request_t,
+                   request_construct, NULL);
+OBJ_CLASS_INSTANCE(ompi_osc_ucx_accumulate_request_t, ompi_request_t,
                    request_construct, NULL);
diff --git a/ompi/mca/osc/ucx/osc_ucx_request.h b/ompi/mca/osc/ucx/osc_ucx_request.h
index b471e671bad..dfa718bc699 100644
--- a/ompi/mca/osc/ucx/osc_ucx_request.h
+++ b/ompi/mca/osc/ucx/osc_ucx_request.h
@@ -16,38 +16,132 @@
 
 #include "ompi/request/request.h"
 
+
+enum req_type {
+    ACCUMULATE_REQ,
+    RPUT_REQ,
+    RGET_REQ
+};
+
+enum acc_rma_type {
+    NONE,
+    ACCUMULATE,
+    GET_ACCUMULATE,
+    ANY
+};
+
+enum acc_phases {
+    ACC_INIT,
+    ACC_GET_RESULTS_DATA,
+    ACC_GET_STAGE_DATA,
+    ACC_PUT_TARGET_DATA,
+    ACC_FINALIZE
+};
+
 typedef struct ompi_osc_ucx_request {
     ompi_request_t super;
+    int request_type;
+    ompi_osc_ucx_module_t *module;
 } ompi_osc_ucx_request_t;
 
+typedef struct ompi_osc_ucx_generic_request {
+    ompi_osc_ucx_request_t super;
+} ompi_osc_ucx_generic_request_t;
+
+typedef struct ompi_osc_ucx_accumulate_request {
+    ompi_osc_ucx_request_t super;
+    struct ompi_op_t *op;
+    int phase;
+    int acc_type;
+    bool lock_acquired;
+    int target;
+    struct ompi_win_t *win;
+    const void *origin_addr;
+    int origin_count;
+    struct ompi_datatype_t *origin_dt;
+    void *stage_addr;
+    int stage_count;
+    struct ompi_datatype_t *stage_dt;
+    struct ompi_datatype_t *target_dt;
+    int target_disp;
+    int target_count;
+    void *free_ptr;
+} ompi_osc_ucx_accumulate_request_t;
+
 OBJ_CLASS_DECLARATION(ompi_osc_ucx_request_t);
+OBJ_CLASS_DECLARATION(ompi_osc_ucx_generic_request_t);
+OBJ_CLASS_DECLARATION(ompi_osc_ucx_accumulate_request_t);
+
+#define OMPI_OSC_UCX_GENERIC_REQUEST_ALLOC(win, req, _req_type)                         \
+    do {                                                                                \
+        opal_free_list_item_t *item;                                                    \
+        do {                                                                            \
+            item = opal_free_list_get(&mca_osc_ucx_component.requests);                 \
+            if (item == NULL) {                                                         \
+                if (module->ctx->num_incomplete_req_ops > 0) {                          \
+                    opal_common_ucx_wpool_progress(mca_osc_ucx_component.wpool);        \
+                }                                                                       \
+            }                                                                           \
+        } while (item == NULL);                                                         \
+        req = (ompi_osc_ucx_generic_request_t*) item;                                   \
+        OMPI_REQUEST_INIT(&req->super.super, false);                                          \
+        req->super.super.req_mpi_object.win = win;                                            \
+        req->super.super.req_complete = false;                                                \
+        req->super.super.req_state = OMPI_REQUEST_ACTIVE;                                     \
+        req->super.super.req_status.MPI_ERROR = MPI_SUCCESS;                                  \
+        req->super.module = NULL;                                                             \
+        req->super.request_type = _req_type;                                                  \
+    } while (0)
 
-#define OMPI_OSC_UCX_REQUEST_ALLOC(win, req)                            \
-    do {                                                                \
-        opal_free_list_item_t *item;                                    \
-        do {                                                            \
-            item = opal_free_list_get(&mca_osc_ucx_component.requests); \
-            if (item == NULL) {                                         \
-                if (mca_osc_ucx_component.num_incomplete_req_ops > 0) { \
-                    opal_common_ucx_wpool_progress(mca_osc_ucx_component.wpool); \
-                }                                                       \
-            }                                                           \
-        } while (item == NULL);                                         \
-        req = (ompi_osc_ucx_request_t*) item;                           \
-        OMPI_REQUEST_INIT(&req->super, false);                          \
-        req->super.req_mpi_object.win = win;                            \
-        req->super.req_complete = false;                                \
-        req->super.req_state = OMPI_REQUEST_ACTIVE;                     \
-        req->super.req_status.MPI_ERROR = MPI_SUCCESS;                  \
+#define OMPI_OSC_UCX_ACCUMULATE_REQUEST_ALLOC(win, req)                                 \
+    do {                                                                                \
+        opal_free_list_item_t *item;                                                    \
+        do {                                                                            \
+            item = opal_free_list_get(&mca_osc_ucx_component.accumulate_requests);      \
+            if (item == NULL) {                                                         \
+                if (module->ctx->num_incomplete_req_ops > 0) {                          \
+                    opal_common_ucx_wpool_progress(mca_osc_ucx_component.wpool);        \
+                }                                                                       \
+            }                                                                           \
+        } while (item == NULL);                                                         \
+        req = (ompi_osc_ucx_accumulate_request_t*) item;                                \
+        OMPI_REQUEST_INIT(&req->super.super, false);                                          \
+        req->super.super.req_mpi_object.win = win;                                            \
+        req->super.super.req_complete = false;                                                \
+        req->super.super.req_state = OMPI_REQUEST_ACTIVE;                                     \
+        req->super.super.req_status.MPI_ERROR = MPI_SUCCESS;                                  \
+        req->super.module = NULL;                                                             \
+        req->super.request_type = ACCUMULATE_REQ;                                             \
+        req->acc_type = NONE;                                                   \
+        req->op = MPI_NO_OP;                                                    \
+        req->phase = ACC_INIT;                                                  \
+        req->target = -1;                                                       \
+        req->lock_acquired = false;                                             \
+        req->win = NULL;                                                        \
+        req->origin_addr = NULL;                                                \
+        req->origin_count = 0;                                                  \
+        req->origin_dt = NULL;                                                  \
+        req->stage_addr = NULL;                                                 \
+        req->stage_count = 0;                                                   \
+        req->stage_dt = NULL;                                                   \
+        req->target_dt = NULL;                                                  \
+        req->target_count = 0;                                                  \
+        req->target_disp = 0;                                                   \
+        req->free_ptr = NULL;                                                   \
     } while (0)
 
-#define OMPI_OSC_UCX_REQUEST_RETURN(req)                                \
-    do {                                                                \
-        OMPI_REQUEST_FINI(&req->super);                                 \
-        opal_free_list_return (&mca_osc_ucx_component.requests,         \
-                               (opal_free_list_item_t*) req);           \
+#define OMPI_OSC_UCX_REQUEST_RETURN(req)                                                \
+    do {                                                                                \
+        OMPI_REQUEST_FINI(&req->super.super);                                           \
+        if (req->super.request_type == ACCUMULATE_REQ) {                                \
+            opal_free_list_return (&mca_osc_ucx_component.accumulate_requests,          \
+                                   (opal_free_list_item_t*) req);                       \
+        } else {                                                                        \
+            opal_free_list_return (&mca_osc_ucx_component.requests,                     \
+                                   (opal_free_list_item_t*) req);                       \
+        }                                                                               \
     } while (0)
 
-void req_completion(void *request);
+void ompi_osc_ucx_req_completion(void *request);
 
 #endif /* OMPI_OSC_UCX_REQUEST_H */
diff --git a/ompi/mca/part/Makefile.am b/ompi/mca/part/Makefile.am
new file mode 100644
index 00000000000..3eab15c827d
--- /dev/null
+++ b/ompi/mca/part/Makefile.am
@@ -0,0 +1,39 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# main library setup
+noinst_LTLIBRARIES = libmca_part.la
+libmca_part_la_SOURCES =
+
+# local files
+headers = part.h 
+libmca_part_la_SOURCES += $(headers) $(nodist_headers)
+
+# Conditionally install the header files
+if WANT_INSTALL_HEADERS
+ompidir = $(ompiincludedir)/$(subdir)
+nobase_ompi_HEADERS = $(headers)
+nobase_nodist_ompi_HEADERS = $(nodist_headers)
+endif
+
+include base/Makefile.am
+
+distclean-local:
+	rm -f base/static-components.h
diff --git a/ompi/mca/part/base/Makefile.am b/ompi/mca/part/base/Makefile.am
new file mode 100644
index 00000000000..05a7aba9783
--- /dev/null
+++ b/ompi/mca/part/base/Makefile.am
@@ -0,0 +1,21 @@
+#
+# Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+headers += \
+        base/base.h \
+        base/part_base_prequest.h \
+        base/part_base_precvreq.h \
+        base/part_base_psendreq.h 
+
+libmca_part_la_SOURCES += \
+        base/part_base_frame.c \
+        base/part_base_precvreq.c \
+        base/part_base_prequest.c \
+        base/part_base_select.c \
+        base/part_base_psendreq.c 
diff --git a/ompi/mca/part/base/base.h b/ompi/mca/part/base/base.h
new file mode 100644
index 00000000000..a65b2aba51c
--- /dev/null
+++ b/ompi/mca/part/base/base.h
@@ -0,0 +1,71 @@
+/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights reserved.
+ * Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef MCA_PART_BASE_H
+#define MCA_PART_BASE_H
+
+#include "ompi_config.h"
+
+#include "ompi/mca/mca.h"
+#include "opal/mca/base/mca_base_framework.h"
+#include "opal/class/opal_list.h"
+#include "opal/class/opal_pointer_array.h"
+
+#include "ompi/mca/part/part.h"
+
+/*
+ * Global functions for PART
+ */
+
+BEGIN_C_DECLS
+
+/*
+ * This is the base priority for a PART wrapper component
+ * If there exists more than one then it is undefined
+ * which one is picked.
+ */
+#define PART_SELECT_WRAPPER_PRIORITY -128
+
+/*
+ * MCA framework
+ */
+OMPI_DECLSPEC extern mca_base_framework_t ompi_part_base_framework;
+
+/*
+ * Select an available component.
+ */
+OMPI_DECLSPEC  int mca_part_base_select(bool enable_progress_threads,
+                                        bool enable_mpi_threads);
+
+OMPI_DECLSPEC  int mca_part_base_progress(void);
+
+OMPI_DECLSPEC int mca_part_base_finalize(void);
+
+/*
+ * Globals
+ */
+OMPI_DECLSPEC extern mca_part_base_component_t mca_part_base_selected_component;
+OMPI_DECLSPEC extern mca_part_base_module_t mca_part;
+OMPI_DECLSPEC extern opal_pointer_array_t mca_part_base_part;
+
+END_C_DECLS
+
+#endif /* MCA_PART_BASE_H */
diff --git a/ompi/mca/part/base/part_base_frame.c b/ompi/mca/part/base/part_base_frame.c
new file mode 100644
index 00000000000..f9da3548456
--- /dev/null
+++ b/ompi/mca/part/base/part_base_frame.c
@@ -0,0 +1,148 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2007 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018      IBM Corporation. All rights reserved.
+ * Copyright (c) 2020      Intel, Inc.  All rights reserved.
+ * Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include <string.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif  /* HAVE_UNIST_H */
+#include "ompi/mca/mca.h"
+#include "opal/util/output.h"
+#include "opal/mca/base/base.h"
+
+
+#include "ompi/constants.h"
+#include "ompi/mca/part/part.h"
+#include "ompi/mca/part/base/base.h"
+#include "ompi/mca/part/base/part_base_prequest.h"
+
+/*
+ * The following file was created by configure.  It contains extern
+ * statements and the definition of an array of pointers to each
+ * component's public mca_base_component_t struct.
+ */
+
+#include "ompi/mca/part/base/static-components.h"
+
+/*
+ * This function allows modules to not declare a progress function,
+ * by defaulting to doing nothing.
+ */
+int mca_part_base_progress(void)
+{
+    return OMPI_SUCCESS;
+}
+
+#define xstringify(part) #part
+#define stringify(part) xstringify(part)
+
+/*
+ * Global variables
+ */
+mca_part_base_module_t mca_part = {
+    .part_progress = mca_part_base_progress   /* part_progress */
+};
+
+mca_part_base_component_t mca_part_base_selected_component = {{0}};
+opal_pointer_array_t mca_part_base_part = {{0}};
+
+static int mca_part_base_register(mca_base_register_flag_t flags)
+{
+    return OMPI_SUCCESS;
+}
+
+int mca_part_base_finalize(void) {
+  if (NULL != mca_part_base_selected_component.partm_finalize) {
+    return mca_part_base_selected_component.partm_finalize();
+  }
+  return OMPI_SUCCESS;
+}
+
+
+static int mca_part_base_close(void)
+{
+    int i, j;
+
+    /* unregister the progress function */
+    if( NULL != mca_part.part_progress ) {
+        opal_progress_unregister(mca_part.part_progress);
+    }
+
+    /* reset the progress function to do nothing */
+    mca_part.part_progress = mca_part_base_progress;
+
+    /* Free all the strings in the array of components */
+    j = opal_pointer_array_get_size(&mca_part_base_part);
+    for (i = 0; i < j; ++i) {
+        char *str;
+        str = (char*) opal_pointer_array_get_item(&mca_part_base_part, i);
+        free(str);
+    }
+    OBJ_DESTRUCT(&mca_part_base_part);
+
+    OBJ_DESTRUCT(&mca_part_base_psend_requests);
+    OBJ_DESTRUCT(&mca_part_base_precv_requests);
+
+    /* Close all remaining available components */
+    return mca_base_framework_components_close(&ompi_part_base_framework, NULL);
+}
+
+/**
+ * Function for finding and opening either all MCA components, or the one
+ * that was specifically requested via a MCA parameter.
+ */
+static int mca_part_base_open(mca_base_open_flag_t flags)
+{
+    OBJ_CONSTRUCT(&mca_part_base_part, opal_pointer_array_t);
+
+    
+    OBJ_CONSTRUCT(&mca_part_base_psend_requests, opal_free_list_t);
+    OBJ_CONSTRUCT(&mca_part_base_precv_requests, opal_free_list_t);
+    /* Open up all available components */
+
+    if (OPAL_SUCCESS !=
+        mca_base_framework_components_open(&ompi_part_base_framework, flags)) {
+        return OMPI_ERROR;
+    }
+
+    /* Set a sentinel in case we don't select any components (e.g.,
+       ompi_info) */
+
+    mca_part_base_selected_component.partm_finalize = NULL;
+
+    /* Currently this uses a default with no selection criteria as there is only 1 module. */
+    opal_pointer_array_add(&mca_part_base_part, strdup("persist"));
+
+    return OMPI_SUCCESS;
+}
+
+MCA_BASE_FRAMEWORK_DECLARE(ompi, part, "OMPI PART", mca_part_base_register,
+                           mca_part_base_open, mca_part_base_close,
+                           mca_part_base_static_components, 0);
diff --git a/ompi/mca/part/base/part_base_precvreq.c b/ompi/mca/part/base/part_base_precvreq.c
new file mode 100644
index 00000000000..ae50422bbc8
--- /dev/null
+++ b/ompi/mca/part/base/part_base_precvreq.c
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2007 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+/*%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%*/
+
+#include "ompi_config.h"
+#include "ompi/types.h"
+#include "ompi/mca/part/part.h"
+#include "ompi/mca/part/base/part_base_precvreq.h"
+
+
+static void mca_part_base_precv_request_construct(mca_part_base_precv_request_t* request)
+{
+    OBJ_CONSTRUCT(&request->req_base.req_convertor, opal_convertor_t);
+}
+
+
+/* For each request the convertor get cleaned after each message
+ * (in the base _FINI macro). Therefore, as the convertor is a static object
+ * we don't have to call OBJ_DESTRUCT here.
+ */
+
+OBJ_CLASS_INSTANCE(
+    mca_part_base_precv_request_t,
+    mca_part_base_prequest_t,
+    mca_part_base_precv_request_construct,
+    0
+);
+
+
diff --git a/ompi/mca/part/base/part_base_precvreq.h b/ompi/mca/part/base/part_base_precvreq.h
new file mode 100644
index 00000000000..64836757e0d
--- /dev/null
+++ b/ompi/mca/part/base/part_base_precvreq.h
@@ -0,0 +1,50 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2016 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+/**
+ * @file
+ */
+#ifndef MCA_PART_BASE_PRECV_REQUEST_H
+#define MCA_PART_BASE_PRECV_REQUEST_H
+
+#include "ompi_config.h"
+#include "ompi/mca/part/base/part_base_prequest.h"
+#include "opal/datatype/opal_convertor.h"
+#include "ompi/peruse/peruse-internal.h"
+
+BEGIN_C_DECLS
+
+/**
+ * Base type for receive requests.
+ */
+struct mca_part_base_precv_request_t {
+   mca_part_base_prequest_t req_base;  /**< base request */
+   size_t parts;                       /**< number of partitions */
+};
+typedef struct mca_part_base_precv_request_t mca_part_base_precv_request_t;
+
+OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_part_base_precv_request_t);
+
+END_C_DECLS
+
+#endif
+
diff --git a/ompi/mca/part/base/part_base_prequest.c b/ompi/mca/part/base/part_base_prequest.c
new file mode 100644
index 00000000000..e96fe059b0d
--- /dev/null
+++ b/ompi/mca/part/base/part_base_prequest.c
@@ -0,0 +1,45 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2015 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2007 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "ompi/mca/part/part.h"
+#include "ompi/mca/part/base/part_base_prequest.h"
+
+/**
+ * If you wonder why these 2 freelists are declared here read the comment
+ * in the part_base_request.h file.
+ */
+opal_free_list_t mca_part_base_psend_requests = {{{0}}};
+opal_free_list_t mca_part_base_precv_requests = {{{0}}};
+
+static void mca_part_base_prequest_construct(mca_part_base_prequest_t* req)
+{
+    req->req_ompi.req_type = OMPI_REQUEST_PART;
+}
+
+OBJ_CLASS_INSTANCE(
+    mca_part_base_prequest_t,
+    ompi_request_t,
+    mca_part_base_prequest_construct,
+    NULL
+);
+
diff --git a/ompi/mca/part/base/part_base_prequest.h b/ompi/mca/part/base/part_base_prequest.h
new file mode 100644
index 00000000000..676a5596b4f
--- /dev/null
+++ b/ompi/mca/part/base/part_base_prequest.h
@@ -0,0 +1,78 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2016 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2009      Sun Microsystems, Inc. All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+/**
+ * @file
+ */
+#ifndef MCA_PART_BASE_REQUEST_H
+#define MCA_PART_BASE_REQUEST_H
+
+#include "ompi_config.h"
+#include "opal/class/opal_free_list.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/request/request.h"
+#include "opal/datatype/opal_convertor.h"
+BEGIN_C_DECLS
+
+/**
+ * External list for the partitioned requests. 
+ */
+OMPI_DECLSPEC extern opal_free_list_t mca_part_base_psend_requests;
+OMPI_DECLSPEC extern opal_free_list_t mca_part_base_precv_requests;
+
+typedef enum {
+    MCA_PART_REQUEST_NULL,
+    MCA_PART_REQUEST_PSEND,
+    MCA_PART_REQUEST_PRECV
+} mca_part_base_request_type_t;
+
+
+/**
+ *  Base type for Partitioned P2P requests
+ */
+struct mca_part_base_prequest_t {
+
+    ompi_request_t req_ompi;              /**< base request */
+    volatile int32_t req_part_complete;   /**< flag indicating if the pt-2-pt layer is done with this request */
+    volatile int32_t req_free_called;     /**< flag indicating if the user has freed this request */
+    mca_part_base_request_type_t req_type; /**< MPI request type - used for test */
+    struct ompi_communicator_t *req_comm; /**< communicator pointer */
+    struct ompi_datatype_t *req_datatype; /**< pointer to data type */
+    opal_convertor_t req_convertor;       /**< always need the convertor */
+
+    void *req_addr;                       /**< pointer to application buffer */
+    size_t req_parts;                     /**< number of partitions */
+    size_t req_count;                     /**< count of user datatype elements */
+    int32_t req_peer;                     /**< peer process - rank w/in this communicator */
+    int32_t req_tag;                      /**< user defined tag */
+    struct ompi_proc_t* req_proc;         /**< peer process */
+};
+typedef struct mca_part_base_prequest_t mca_part_base_prequest_t;
+
+OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_part_base_prequest_t);
+
+END_C_DECLS
+
+#endif
+
diff --git a/ompi/mca/part/base/part_base_psendreq.c b/ompi/mca/part/base/part_base_psendreq.c
new file mode 100644
index 00000000000..768388c179e
--- /dev/null
+++ b/ompi/mca/part/base/part_base_psendreq.c
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2007 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <string.h>
+#include "ompi/mca/part/part.h"
+#include "ompi/mca/part/base/part_base_psendreq.h"
+
+static void mca_part_base_psend_request_construct(mca_part_base_psend_request_t* request)
+{
+    /* no need to reinit for every send -- never changes */
+    OBJ_CONSTRUCT(&request->req_base.req_convertor, opal_convertor_t);
+}
+
+OBJ_CLASS_INSTANCE(
+    mca_part_base_psend_request_t,
+    mca_part_base_prequest_t,
+    mca_part_base_psend_request_construct,
+    0
+);
+
diff --git a/ompi/mca/part/base/part_base_psendreq.h b/ompi/mca/part/base/part_base_psendreq.h
new file mode 100644
index 00000000000..ee7121d2554
--- /dev/null
+++ b/ompi/mca/part/base/part_base_psendreq.h
@@ -0,0 +1,54 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2016 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2017      Intel, Inc. All rights reserved.
+ * Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+/**
+ * @file
+ */
+#ifndef MCA_PART_BASE_PSEND_REQUEST_H
+#define MCA_PART_BASE_PSEND_REQUEST_H
+
+#include "ompi_config.h"
+#include "ompi/mca/part/part.h"
+#include "ompi/mca/part/base/part_base_prequest.h"
+#include "opal/datatype/opal_convertor.h"
+#include "ompi/peruse/peruse-internal.h"
+
+BEGIN_C_DECLS
+
+/**
+ * Base type for send requests
+ */
+struct mca_part_base_psend_request_t {
+    mca_part_base_prequest_t req_base;       /**< base request type - common data structure for use by wait/test */
+    const void *req_addr;                    /**< pointer to send buffer - may not be application buffer */
+    size_t req_bytes_packed;                 /**< packed size of a message given the datatype and count */
+};
+typedef struct mca_part_base_psend_request_t mca_part_base_psend_request_t;
+
+OMPI_DECLSPEC OBJ_CLASS_DECLARATION( mca_part_base_psend_request_t );
+
+END_C_DECLS
+
+#endif
diff --git a/ompi/mca/part/base/part_base_select.c b/ompi/mca/part/base/part_base_select.c
new file mode 100644
index 00000000000..71525aaec8a
--- /dev/null
+++ b/ompi/mca/part/base/part_base_select.c
@@ -0,0 +1,233 @@
+/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2012      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2013-2014 Intel, Inc. All rights reserved
+ * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include <string.h>
+
+#include "opal/class/opal_list.h"
+#include "opal/util/output.h"
+#include "opal/util/show_help.h"
+#include "opal/runtime/opal_progress.h"
+#include "ompi/mca/mca.h"
+#include "opal/mca/base/base.h"
+#include "opal/runtime/opal.h"
+#include "opal/mca/pmix/pmix-internal.h"
+
+#include "ompi/constants.h"
+#include "ompi/mca/part/part.h"
+#include "ompi/mca/part/base/base.h"
+#include "ompi/proc/proc.h"
+
+typedef struct opened_component_t {
+  opal_list_item_t super;
+  mca_part_base_component_t *om_component;
+} opened_component_t;
+
+static bool modex_reqd=false;
+
+/**
+ * Function for selecting one component from all those that are
+ * available.
+ *
+ * Call the init function on all available components and get their
+ * priorities.  Select the component with the highest priority.  All
+ * other components will be closed and unloaded.  The selected component
+ * will have all of its function pointers saved and returned to the
+ * caller.
+ */
+int mca_part_base_select(bool enable_progress_threads,
+                        bool enable_mpi_threads)
+{
+    int i, priority = 0, best_priority = 0, num_part = 0;
+    opal_list_item_t *item = NULL;
+    mca_base_component_list_item_t *cli = NULL;
+    mca_part_base_component_t *component = NULL, *best_component = NULL;
+    mca_part_base_module_t *module = NULL, *best_module = NULL;
+    opal_list_t opened;
+    opened_component_t *om = NULL;
+    bool found_part;
+
+    /* Traverse the list of available components; call their init
+       functions. */
+
+    best_priority = -1;
+    best_component = NULL;
+    module = NULL;
+    OBJ_CONSTRUCT(&opened, opal_list_t);
+    OPAL_LIST_FOREACH(cli, &ompi_part_base_framework.framework_components, mca_base_component_list_item_t) {
+        component = (mca_part_base_component_t *) cli->cli_component;
+
+        /* if there is an include list - item must be in the list to be included */
+        found_part = false;
+        for( i = 0; i < opal_pointer_array_get_size(&mca_part_base_part); i++) {
+            char * tmp_val = NULL;
+            tmp_val = (char *) opal_pointer_array_get_item(&mca_part_base_part, i);
+            if( NULL == tmp_val) {
+                continue;
+            }
+
+            if(0 == strncmp(component->partm_version.mca_component_name,
+                            tmp_val, strlen(component->partm_version.mca_component_name)) ) {
+                found_part = true;
+                break;
+            }
+        }
+
+        if(!found_part && opal_pointer_array_get_size(&mca_part_base_part)) {
+            opal_output_verbose( 10, ompi_part_base_framework.framework_output,
+                                     "select: component %s not in the include list",
+                                     component->partm_version.mca_component_name );
+
+            continue;
+        }
+
+        /* if there is no init function - ignore it */
+        if (NULL == component->partm_init) {
+            opal_output_verbose( 10, ompi_part_base_framework.framework_output,
+                                 "select: no init function; ignoring component %s",
+                                 component->partm_version.mca_component_name );
+            continue;
+        }
+
+        /* this is a part that could be considered */
+        num_part++;
+
+        /* Init component to get its priority */
+        opal_output_verbose( 10, ompi_part_base_framework.framework_output,
+                             "select: initializing %s component %s",
+                             component->partm_version.mca_type_name,
+                             component->partm_version.mca_component_name );
+        priority = best_priority;
+        module = component->partm_init(&priority, enable_progress_threads,
+                                      enable_mpi_threads);
+        if (NULL == module) {
+            opal_output_verbose( 10, ompi_part_base_framework.framework_output,
+                                 "select: init returned failure for component %s",
+                                 component->partm_version.mca_component_name );
+            continue;
+        }
+
+        opal_output_verbose( 10, ompi_part_base_framework.framework_output,
+                             "select: init returned priority %d", priority );
+        if (priority > best_priority) {
+            best_priority = priority;
+            best_component = component;
+            best_module = module;
+        }
+
+        om = (opened_component_t*)malloc(sizeof(opened_component_t));
+        if (NULL == om) {
+            return OMPI_ERR_OUT_OF_RESOURCE;
+        }
+        OBJ_CONSTRUCT(om, opal_list_item_t);
+        om->om_component = component;
+        opal_list_append(&opened, (opal_list_item_t*) om);
+    }
+
+    /* Finished querying all components.  Check for the bozo case. */
+
+    if( NULL == best_component ) {
+        opal_show_help("help-mca-base.txt", "find-available:none found",
+                       true, "part",
+                       opal_process_info.nodename,
+                       "part");
+        for( i = 0; i < opal_pointer_array_get_size(&mca_part_base_part); i++) {
+            char * tmp_val = NULL;
+            tmp_val = (char *) opal_pointer_array_get_item(&mca_part_base_part, i);
+            if( NULL == tmp_val) {
+                continue;
+            }
+            ompi_rte_abort(1, "PART %s cannot be selected", tmp_val);
+        }
+        if(0 == i) {
+            ompi_rte_abort(2, "No part component available.  This shouldn't happen.");
+        }
+    }
+
+    opal_output_verbose( 10, ompi_part_base_framework.framework_output,
+                         "selected %s best priority %d\n",
+                         best_component->partm_version.mca_component_name, best_priority);
+
+    /* if more than one PART could be considered, then we still need the
+     * modex since we cannot know which one will be selected on all procs
+     */
+    if (1 < num_part) {
+        modex_reqd = true;
+    }
+
+    /* Finalize all non-selected components */
+
+    for (item = opal_list_remove_first(&opened);
+         NULL != item;
+         item = opal_list_remove_first(&opened)) {
+        om = (opened_component_t *) item;
+
+        if (om->om_component != best_component
+            ) {
+            /* Finalize */
+
+            if (NULL != om->om_component->partm_finalize) {
+
+                /* Blatantly ignore the return code (what would we do to
+                   recover, anyway?  This component is going away, so errors
+                   don't matter anymore) */
+
+                om->om_component->partm_finalize();
+                opal_output_verbose(10, ompi_part_base_framework.framework_output,
+                                    "select: component %s not selected / finalized",
+                                    om->om_component->partm_version.mca_component_name);
+            }
+        }
+        OBJ_DESTRUCT( om );
+        free(om);
+    }
+    OBJ_DESTRUCT( &opened );
+
+    /* Save the winner */
+
+    mca_part_base_selected_component = *best_component;
+    mca_part = *best_module;
+    opal_output_verbose( 10, ompi_part_base_framework.framework_output,
+                         "select: component %s selected",
+                         mca_part_base_selected_component.partm_version.mca_component_name );
+
+    /* This base function closes, unloads, and removes from the
+       available list all unselected components.  The available list will
+       contain only the selected component. */
+
+    mca_base_components_close(ompi_part_base_framework.framework_output,
+                              &ompi_part_base_framework.framework_components,
+                              (mca_base_component_t *) best_component);
+
+    /* register the winner's callback */
+    if( NULL != mca_part.part_progress ) {
+        opal_progress_register(mca_part.part_progress);
+    }
+
+    /* All done */
+
+    return OMPI_SUCCESS;
+}
+
diff --git a/ompi/mca/part/configure.m4 b/ompi/mca/part/configure.m4
new file mode 100644
index 00000000000..88475c5e445
--- /dev/null
+++ b/ompi/mca/part/configure.m4
@@ -0,0 +1,18 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2013-2020 Sandia National Laboratories.  All rights reserved.
+#
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+AC_DEFUN([MCA_ompi_part_CONFIG],[
+    # configure all the components
+    MCA_CONFIGURE_FRAMEWORK($1, $2, 1)
+
+    # this is a direct callable component, so set that up.
+    MCA_SETUP_DIRECT_CALL($1, $2)
+])
diff --git a/ompi/mca/part/part.h b/ompi/mca/part/part.h
new file mode 100644
index 00000000000..aa3977bc84c
--- /dev/null
+++ b/ompi/mca/part/part.h
@@ -0,0 +1,249 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2006 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2015 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2011-2020 Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2021      Bull S.A.S. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+/**
+ * @file
+ *
+ * Partitioned Communication (PART)
+ *
+ * An MCA component type that provides the partitioned interface functionality
+ * required by the MPI-4 specification. Part is designed to act as intermediary 
+ * between the MPI layer and another transfer layer. This differs from other 
+ * components, such as PML, by allowing the component to leverage the underlying 
+ * transfer mechanism to be another MPI layer, such as the osc component/the 
+ * RMA interface.
+ *
+ *   ------------------------------------
+ *   |                MPI               |
+ *   ------------------------------------
+ *   |               PART               |
+ *   ------------------------------------
+ *   |             OSC (RDMA)           |
+ *   ------------------------------------
+ * 
+ * The initial implementation is currently leveraging the RMA interface, 
+ * with the intent to remove the MPI layer and directly call the osc component.
+ * Other transport mechanisms could be used in future implementation (such as 
+ * the MTL and BTL components). 
+ *
+ * This component and it's initial module are under development and have 
+ * extra restrictions on use than described in the MPI-4 specification. 
+ * Currently, MPI_Psend_init and MPI_Precv_init are both blocking in the RMA 
+ * component which requires careful use to avoid deadlocks. This will 
+ * be addressed in future updates. 
+ */
+
+#ifndef MCA_PART_H
+#define MCA_PART_H
+
+#include "ompi_config.h"
+#include "ompi/mca/mca.h"
+#include "mpi.h" /* needed for MPI_ANY_TAG */
+#include "ompi/request/request.h"
+
+BEGIN_C_DECLS
+
+struct ompi_proc_t;
+
+/**
+ * MCA->PART Called by MCA framework to initialize the component.
+ *
+ * @param priority (OUT) Relative priority or ranking used by MCA to
+ * selected a component.
+ *
+ * @param enable_progress_threads (IN) Whether this component is
+ * allowed to run a hidden/progress thread or not.
+ *
+ * @param enable_mpi_threads (IN) Whether support for multiple MPI
+ * threads is enabled or not (i.e., MPI_THREAD_MULTIPLE), which
+ * indicates whether multiple threads may invoke this component
+ * simultaneously or not.
+ */
+typedef struct mca_part_base_module_1_0_1_t * (*mca_part_base_component_init_fn_t)(
+    int *priority,
+    bool enable_progress_threads,
+    bool enable_mpi_threads);
+
+typedef int (*mca_part_base_component_finalize_fn_t)(void);
+
+/**
+ * PART component version and interface functions.
+ */
+
+struct mca_part_base_component_4_0_0_t {
+   mca_base_component_t partm_version;
+   mca_base_component_data_t partm_data;
+   mca_part_base_component_init_fn_t partm_init;
+   mca_part_base_component_finalize_fn_t partm_finalize;
+};
+typedef struct mca_part_base_component_4_0_0_t mca_part_base_component_4_0_0_t;
+typedef mca_part_base_component_4_0_0_t mca_part_base_component_t;
+
+
+/**
+ * MCA management functions.
+ */
+
+
+/**
+ * For non-threaded case, provides MCA the opportunity to
+ * progress outstanding requests on all btls.
+ *
+ *  * @return        Count of "completions", a metric of
+ *                   how many items where completed in the call
+ *                   to progress.
+*/
+typedef int (*mca_part_base_module_progress_fn_t)(void);
+
+/**
+ * MPI Interface Functions
+ */
+
+/**
+ *  Initialize a partitioned receive request.
+ *
+ *  @param buf (IN)         User buffer.
+ *  @param parts (IN)       Number of partitions.
+ *  @param count (IN)       Number of elements of the specified datatype.
+ *  @param datatype (IN)    User defined datatype.
+ *  @param src (IN)         Source rank w/in communicator.
+ *  @param tag (IN)         User defined tag.
+ *  @param comm (IN)        Communicator.
+ *  @param request (OUT)    Request handle.
+ *  @return                 OMPI_SUCCESS or failure status.
+ */
+
+typedef int (*mca_part_base_module_precv_init_fn_t)(
+    void *buf,
+    size_t parts,
+    size_t count,
+    struct ompi_datatype_t *datatype,
+    int src,
+    int tag,
+    struct ompi_communicator_t* comm,
+    struct ompi_info_t * info,
+    struct ompi_request_t **request
+);
+
+/**
+ *  Initialize a partitioned send request.
+ *
+ *  @param buf (IN)         User buffer.
+ *  @param parts (IN)       Number of partitions.
+ *  @param count (IN)       Number of elements of the specified datatype.
+ *  @param datatype (IN)    User defined datatype.
+ *  @param dst (IN)         Peer rank w/in communicator.
+ *  @param tag (IN)         User defined tag.
+ *  @param mode (IN)        Send mode (STANDARD,BUFFERED,SYNCHRONOUS,READY)
+ *  @param comm (IN)        Communicator.
+ *  @param request (OUT)    Request handle.
+ *  @return                 OMPI_SUCCESS or failure status.
+ */
+typedef int (*mca_part_base_module_psend_init_fn_t)(
+    const void *buf,
+    size_t parts, 
+    size_t count,
+    struct ompi_datatype_t *datatype,
+    int dst,
+    int tag,
+    struct ompi_communicator_t* comm,
+    struct ompi_info_t * info,
+    struct ompi_request_t **request
+);
+
+/**
+ * Initiate one or more partitioned requests.
+ *
+ * @param count (IN)        Number of requests
+ * @param requests (IN/OUT) Array of persistent requests
+ * @return                  OMPI_SUCCESS or failure status.
+ */
+typedef ompi_request_start_fn_t mca_part_base_module_start_fn_t;
+
+/**
+ * Mark a range of partitions ready in a partitioned send request.
+ *
+ * @param min_part         Minimum partition to mark ready for transfer. 
+ * @param max_part         Maximum partition to mark ready for transfer.
+ * @param request (IN/OUT) Request
+ * @return                 OMPI_SUCCESS or failure status.
+ *
+ */
+typedef int (*mca_part_base_module_pready_fn_t)(
+    size_t min_part,
+    size_t max_part,
+    struct ompi_request_t* request
+);
+
+/**
+ * Check a range of partitions in a partitioned receive request.
+ *
+ * @param min_part         Minimum partition to check. 
+ * @param max_part         Maximum partition to check.
+ * @param flag             Flag for completion of entire range. 
+ * @param request (IN/OUT) Request
+ * @return                 OMPI_SUCCESS or failure status.
+ *
+ */
+typedef int (*mca_part_base_module_parrived_fn_t)(
+    size_t min_part,
+    size_t max_part,
+    int* flag,
+    struct ompi_request_t* request
+);
+
+/**
+ *  PART instance.
+ */
+
+struct mca_part_base_module_1_0_1_t {
+
+    /* downcalls from MCA to PART */
+    mca_part_base_module_progress_fn_t    part_progress;
+
+    /* downcalls from MPI to PART */
+    mca_part_base_module_precv_init_fn_t  part_precv_init;
+    mca_part_base_module_psend_init_fn_t  part_psend_init;
+    mca_part_base_module_start_fn_t       part_start; 
+    mca_part_base_module_pready_fn_t      part_pready;
+    mca_part_base_module_parrived_fn_t    part_parrived;
+    /* diagnostics */
+
+    /* FT Event */
+
+    /* maximum constant sizes */
+};
+typedef struct mca_part_base_module_1_0_1_t mca_part_base_module_1_0_1_t;
+typedef mca_part_base_module_1_0_1_t mca_part_base_module_t;
+
+/*
+ * Macro for use in components that are of type part
+ */
+#define MCA_PART_BASE_VERSION_2_0_0 \
+    OMPI_MCA_BASE_VERSION_2_1_0("part", 4, 0, 0)
+
+OMPI_DECLSPEC extern mca_part_base_module_t mca_part;
+
+END_C_DECLS
+#endif /* MCA_PART_H */
diff --git a/ompi/mca/part/persist/Makefile.am b/ompi/mca/part/persist/Makefile.am
new file mode 100644
index 00000000000..fab2975e92b
--- /dev/null
+++ b/ompi/mca/part/persist/Makefile.am
@@ -0,0 +1,53 @@
+#
+# Copyright (c) 2004-2006 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2009      High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2020-2021 Sandia National Laboratories. All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+EXTRA_DIST = post_configure.sh
+
+if MCA_BUILD_ompi_part_persist_DSO
+component_noinst =
+component_install = mca_part_persist.la
+else
+component_noinst = libmca_part_persist.la
+component_install =
+endif
+
+local_sources = \
+	part_persist.c \
+	part_persist.h \
+	part_persist_component.c \
+	part_persist_component.h \
+	part_persist_recvreq.h \
+	part_persist_recvreq.c \
+	part_persist_request.h \
+	part_persist_request.c \
+	part_persist_sendreq.h \
+	part_persist_sendreq.c 
+
+mcacomponentdir = $(ompilibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_part_persist_la_SOURCES = $(local_sources)
+mca_part_persist_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
+	$(part_persist_LIBS)
+mca_part_persist_la_LDFLAGS = -module -avoid-version $(part_persist_LDFLAGS)
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_part_persist_la_SOURCES = $(local_sources)
+libmca_part_persist_la_LIBADD = $(part_persist_LIBS)
+libmca_part_persist_la_LDFLAGS = -module -avoid-version $(part_persist_LDFLAGS)
+
diff --git a/ompi/mca/part/persist/part_persist.c b/ompi/mca/part/persist/part_persist.c
new file mode 100644
index 00000000000..b90a180a601
--- /dev/null
+++ b/ompi/mca/part/persist/part_persist.c
@@ -0,0 +1,47 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2006-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2006 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2011-2021 Sandia National Laboratories. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/communicator/communicator.h"
+#include "ompi/mca/part/base/part_base_prequest.h"
+#include "ompi/mca/part/base/base.h"
+
+#include "ompi/mca/part/persist/part_persist.h"
+#include "ompi/mca/part/persist/part_persist_sendreq.h"
+#include "ompi/mca/part/persist/part_persist_recvreq.h"
+
+ompi_part_persist_t ompi_part_persist = {
+    .super = {
+        .part_progress = mca_part_persist_progress,
+        .part_precv_init = mca_part_persist_precv_init,
+        .part_psend_init = mca_part_persist_psend_init,
+        .part_start = mca_part_persist_start,
+        .part_pready = mca_part_persist_pready,
+        .part_parrived = mca_part_persist_parrived, 
+    }
+};
+
+
+OBJ_CLASS_INSTANCE(mca_part_persist_list_t,
+                   opal_list_item_t,
+                   NULL,
+                   NULL);
+
diff --git a/ompi/mca/part/persist/part_persist.h b/ompi/mca/part/persist/part_persist.h
new file mode 100644
index 00000000000..ccc8f8f1971
--- /dev/null
+++ b/ompi/mca/part/persist/part_persist.h
@@ -0,0 +1,613 @@
+/*
+ * Copyright (c) 2004-2006 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      Intel, Inc. All rights reserved
+ * Copyright (c) 2019-2021 The University of Tennessee at Chattanooga and The University
+ *                         of Tennessee Research Foundation. All rights reserved.
+ * Copyright (c) 2019-2021 Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2021      University of Alabama at Birmingham. All rights reserved.
+ * Copyright (c) 2021      Tennessee Technological University. All rights reserved.
+ * Copyright (c) 2021      Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2021      Bull S.A.S. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef PART_PERSIST_H
+#define PART_PERSIST_H
+
+#ifdef HAVE_ALLOCA_H
+#include <alloca.h>
+#endif
+
+#include<math.h>
+
+#include "ompi_config.h"
+#include "ompi/request/request.h"
+#include "ompi/mca/part/part.h"
+#include "ompi/mca/part/base/base.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/request/request.h"
+#include "opal/sys/atomic.h"
+
+#include "ompi/mca/part/persist/part_persist_request.h"
+#include "ompi/mca/part/base/part_base_precvreq.h"
+#include "ompi/mca/part/persist/part_persist_recvreq.h"
+#include "ompi/mca/part/persist/part_persist_sendreq.h"
+#include "ompi/message/message.h"
+#include "ompi/mca/pml/pml.h"
+BEGIN_C_DECLS
+
+typedef struct mca_part_persist_list_t {
+    opal_list_item_t        super;
+    mca_part_persist_request_t *item;
+} mca_part_persist_list_t;
+
+OPAL_DECLSPEC OBJ_CLASS_DECLARATION(mca_part_persist_list_t);
+
+
+struct ompi_part_persist_t {
+    mca_part_base_module_t super;
+    int                    free_list_num;
+    int                    free_list_max;
+    int                    free_list_inc;
+    opal_list_t           *progress_list;
+
+    int32_t next_send_tag;                /**< This is a counter for send tags for the actual data transfer. */
+    int32_t next_recv_tag; 
+    ompi_communicator_t   *part_comm; /* This approach requires a separate tag space, so we need a dedicated communicator. */
+    ompi_request_t        *part_comm_req;
+    int32_t                part_comm_ready;
+    ompi_communicator_t   *part_comm_setup; /* We create a second communicator to send set-up messages (rational: these 
+                                               messages go in the opposite direction of normal messages, need to use MPI_ANY_SOURCE 
+                                               to support different communicators, and thus need to have a unique tag. Because tags 
+                                               are controlled by the sender in this model, we cannot assume that the tag will be 
+                                               unused in part_comm. */
+    ompi_request_t        *part_comm_sreq;
+    int32_t                part_comm_sready;
+    int32_t                init_comms;    
+    int32_t                init_world;
+    int32_t                my_world_rank; /* Because the back end communicators use a world rank, we need to communicate ours 
+                                             to set up the requests. */
+    opal_atomic_int32_t    block_entry;
+    opal_mutex_t lock; 
+};
+typedef struct ompi_part_persist_t ompi_part_persist_t;
+extern ompi_part_persist_t ompi_part_persist;
+
+
+/**
+ * This is a helper function that frees a request. This requires ompi_part_persist.lock be held before calling.
+ */
+__opal_attribute_always_inline__ static inline int
+mca_part_persist_free_req(struct mca_part_persist_request_t* req)
+{
+    int err = OMPI_SUCCESS;
+    size_t i;
+    opal_list_remove_item(ompi_part_persist.progress_list, (opal_list_item_t*)req->progress_elem);
+    OBJ_RELEASE(req->progress_elem);
+
+    for(i = 0; i < req->real_parts; i++) {
+        ompi_request_free(&(req->persist_reqs[i]));
+    }
+    free(req->persist_reqs);
+    free(req->flags);
+
+    if( MCA_PART_PERSIST_REQUEST_PRECV == req->req_type ) {
+        MCA_PART_PERSIST_PRECV_REQUEST_RETURN(req);
+    } else {
+        MCA_PART_PERSIST_PSEND_REQUEST_RETURN(req);
+    }
+    return err;
+}
+
+
+__opal_attribute_always_inline__ static inline void mca_part_persist_init_lists(void)
+{
+    opal_free_list_init (&mca_part_base_precv_requests,
+                         sizeof(mca_part_persist_precv_request_t),
+                         opal_cache_line_size,
+                         OBJ_CLASS(mca_part_persist_precv_request_t),
+                         0,opal_cache_line_size,
+                         ompi_part_persist.free_list_num,
+                         ompi_part_persist.free_list_max,
+                         ompi_part_persist.free_list_inc,
+                         NULL, 0, NULL, NULL, NULL);
+    opal_free_list_init (&mca_part_base_psend_requests,
+                         sizeof(mca_part_persist_psend_request_t),
+                         opal_cache_line_size,
+                         OBJ_CLASS(mca_part_persist_psend_request_t),
+                         0,opal_cache_line_size,
+                         ompi_part_persist.free_list_num,
+                         ompi_part_persist.free_list_max,
+                         ompi_part_persist.free_list_inc,
+                         NULL, 0, NULL, NULL, NULL);
+     ompi_part_persist.progress_list = OBJ_NEW(opal_list_t);
+}
+
+__opal_attribute_always_inline__ static inline void
+mca_part_persist_complete(struct mca_part_persist_request_t* request)
+{
+    if(MCA_PART_PERSIST_REQUEST_PRECV == request->req_type) {
+        request->req_ompi.req_status.MPI_SOURCE = request->req_peer; 
+    } else {
+        request->req_ompi.req_status.MPI_SOURCE = request->req_comm->c_my_rank;
+    }
+    request->req_ompi.req_complete_cb = NULL;
+    request->req_ompi.req_status.MPI_TAG = request->req_tag;  
+    request->req_ompi.req_status._ucount = request->req_bytes;
+    request->req_ompi.req_status.MPI_ERROR = OMPI_SUCCESS;
+    request->req_part_complete = true;
+    ompi_request_complete(&(request->req_ompi), true );
+}
+
+/**
+ * mca_part_persist_progress is the progress function that will be registered. It handles 
+ * both send and recv request testing and completion. It also handles freeing requests,
+ * after MPI_Free is called and the requests have become inactive.
+ */
+__opal_attribute_always_inline__ static inline int
+mca_part_persist_progress(void)
+{
+    mca_part_persist_list_t *current;
+    int err;
+    size_t i;
+
+    /* prevent re-entry, */
+    int block_entry = opal_atomic_add_fetch_32(&(ompi_part_persist.block_entry), 1);
+    if(1 < block_entry)
+    {
+        block_entry = opal_atomic_add_fetch_32(&(ompi_part_persist.block_entry), -1);
+        return OMPI_SUCCESS;
+    }
+
+    OPAL_THREAD_LOCK(&ompi_part_persist.lock);
+ 
+    mca_part_persist_request_t* to_delete = NULL;
+
+    /* Don't do anything till a function in the module is called. */
+    if(-1 == ompi_part_persist.init_world)
+    {
+        OPAL_THREAD_UNLOCK(&ompi_part_persist.lock);
+        block_entry = opal_atomic_add_fetch_32(&(ompi_part_persist.block_entry), -1);
+        return OMPI_SUCCESS;
+    }
+
+    /* Can't do anything if we don't have world */
+    if(0 == ompi_part_persist.init_world) {
+        ompi_part_persist.my_world_rank = ompi_comm_rank(&ompi_mpi_comm_world.comm);
+        err = ompi_comm_idup(&ompi_mpi_comm_world.comm, &ompi_part_persist.part_comm, &ompi_part_persist.part_comm_req);
+        if(err != OMPI_SUCCESS) {
+             exit(-1);
+        }
+        ompi_part_persist.part_comm_ready = 0;
+        err = ompi_comm_idup(&ompi_mpi_comm_world.comm, &ompi_part_persist.part_comm_setup, &ompi_part_persist.part_comm_sreq);
+        if(err != OMPI_SUCCESS) {
+            exit(-1);
+        }
+        ompi_part_persist.part_comm_sready = 0;
+        ompi_part_persist.init_world = 1;
+
+        OPAL_THREAD_UNLOCK(&ompi_part_persist.lock);
+        block_entry = opal_atomic_add_fetch_32(&(ompi_part_persist.block_entry), -1);
+        return OMPI_SUCCESS;
+    }
+
+    /* Check to see if Comms are setup */
+    if(0 == ompi_part_persist.init_comms) {
+        if(0 == ompi_part_persist.part_comm_ready) {
+            ompi_request_test(&ompi_part_persist.part_comm_req, &ompi_part_persist.part_comm_ready, MPI_STATUS_IGNORE);
+        }
+        if(0 == ompi_part_persist.part_comm_sready) {
+            ompi_request_test(&ompi_part_persist.part_comm_sreq, &ompi_part_persist.part_comm_sready, MPI_STATUS_IGNORE);
+        }
+        if(0 != ompi_part_persist.part_comm_ready && 0 != ompi_part_persist.part_comm_sready) {
+            ompi_part_persist.init_comms = 1;
+        }
+        OPAL_THREAD_UNLOCK(&ompi_part_persist.lock);
+        block_entry = opal_atomic_add_fetch_32(&(ompi_part_persist.block_entry), -1);
+        return OMPI_SUCCESS;
+    }
+
+    OPAL_LIST_FOREACH(current, ompi_part_persist.progress_list, mca_part_persist_list_t) {
+        mca_part_persist_request_t *req = (mca_part_persist_request_t *) current->item;
+
+        /* Check to see if request is initilaized */
+        if(false == req->initialized) {
+            int done = 0; 
+            
+            if(true == req->flag_post_setup_recv) {
+                err = MCA_PML_CALL(irecv(&(req->setup_info[1]), sizeof(struct ompi_mca_persist_setup_t), MPI_BYTE, OMPI_ANY_SOURCE, req->my_recv_tag, ompi_part_persist.part_comm_setup, &req->setup_req[1]));
+                req->flag_post_setup_recv = false;
+            } 
+ 
+            ompi_request_test(&(req->setup_req[1]), &done, MPI_STATUS_IGNORE);
+
+            if(done) {
+                size_t dt_size_;
+                uint32_t dt_size;
+
+                if(MCA_PART_PERSIST_REQUEST_PSEND == req->req_type) {
+                    /* parse message */
+                    req->world_peer  = req->setup_info[1].world_rank; 
+
+                    err = opal_datatype_type_size(&(req->req_datatype->super), &dt_size_);
+                    if(OMPI_SUCCESS != err) return OMPI_ERROR;
+                    dt_size = (dt_size_ > (size_t) UINT_MAX) ? MPI_UNDEFINED : (uint32_t) dt_size_;
+                    uint32_t bytes = req->real_count * dt_size;
+
+                    /* Set up persistent sends */
+                    req->persist_reqs = (ompi_request_t**) malloc(sizeof(ompi_request_t*)*(req->real_parts));
+                    for(i = 0; i < req->real_parts; i++) {
+                         void *buf = ((void*) (((char*)req->req_addr) + (bytes * i)));
+                         err = MCA_PML_CALL(isend_init(buf, req->real_count, req->req_datatype, req->world_peer, req->my_send_tag+i, MCA_PML_BASE_SEND_STANDARD, ompi_part_persist.part_comm, &(req->persist_reqs[i])));
+                    }    
+                } else {
+                    /* parse message */
+                    req->world_peer   = req->setup_info[1].world_rank; 
+                    req->my_send_tag  = req->setup_info[1].start_tag;
+                    req->my_recv_tag  = req->setup_info[1].setup_tag;
+                    req->real_parts   = req->setup_info[1].num_parts;
+                    req->real_count   = req->setup_info[1].count;
+                    req->real_dt_size = req->setup_info[1].dt_size;
+
+
+                    err = opal_datatype_type_size(&(req->req_datatype->super), &dt_size_);
+                    if(OMPI_SUCCESS != err) return OMPI_ERROR;
+                    dt_size = (dt_size_ > (size_t) UINT_MAX) ? MPI_UNDEFINED : (uint32_t) dt_size_;
+                    uint32_t bytes = req->real_count * dt_size;
+
+
+
+		    /* Set up persistent sends */
+                    req->persist_reqs = (ompi_request_t**) malloc(sizeof(ompi_request_t*)*(req->real_parts));
+                    req->flags = (int*) calloc(req->real_parts,sizeof(int));
+
+                    if(req->real_dt_size == dt_size) {
+
+     	                for(i = 0; i < req->real_parts; i++) {
+                            void *buf = ((void*) (((char*)req->req_addr) + (bytes * i)));
+                            err = MCA_PML_CALL(irecv_init(buf, req->real_count, req->req_datatype, req->world_peer, req->my_send_tag+i, ompi_part_persist.part_comm, &(req->persist_reqs[i])));
+                        }
+                    } else {
+                        for(i = 0; i < req->real_parts; i++) {
+                            void *buf = ((void*) (((char*)req->req_addr) + (req->real_count * req->real_dt_size * i)));
+                            err = MCA_PML_CALL(irecv_init(buf, req->real_count * req->real_dt_size, MPI_BYTE, req->world_peer, req->my_send_tag+i, ompi_part_persist.part_comm, &(req->persist_reqs[i])));
+                        }
+		    }
+                    err = req->persist_reqs[0]->req_start(req->real_parts, (&(req->persist_reqs[0])));                     
+
+                    /* Send back a message */
+                    req->setup_info[0].world_rank = ompi_part_persist.my_world_rank;
+                    err = MCA_PML_CALL(isend(&(req->setup_info[0]), sizeof(struct ompi_mca_persist_setup_t), MPI_BYTE, req->world_peer, req->my_recv_tag, MCA_PML_BASE_SEND_STANDARD, ompi_part_persist.part_comm_setup, &req->setup_req[0]));
+                    if(OMPI_SUCCESS != err) return OMPI_ERROR;
+                }
+
+                req->initialized = true; 
+            }
+        } else {
+            if(false == req->req_part_complete && REQUEST_COMPLETED != req->req_ompi.req_complete && OMPI_REQUEST_ACTIVE == req->req_ompi.req_state) {
+               for(i = 0; i < req->real_parts; i++) {
+
+                    /* Check to see if partition is queued for being started. Only applicable to sends. */ 
+                    if(-2 ==  req->flags[i]) {
+                        err = req->persist_reqs[i]->req_start(1, (&(req->persist_reqs[i])));
+                        req->flags[i] = 0;
+                    }
+
+                    if(0 == req->flags[i])
+                    {
+                        ompi_request_test(&(req->persist_reqs[i]), &(req->flags[i]), MPI_STATUS_IGNORE);
+                        if(0 != req->flags[i]) req->done_count++;
+                    }
+                }
+
+                /* Check for completion and complete the requests */
+                if(req->done_count == req->real_parts)
+                {
+                    req->first_send = false;
+                    mca_part_persist_complete(req);
+                } 
+            }
+
+            if(true == req->req_free_called && true == req->req_part_complete && REQUEST_COMPLETED == req->req_ompi.req_complete &&  OMPI_REQUEST_INACTIVE == req->req_ompi.req_state) {
+                to_delete = req;
+            }
+        }
+
+    }
+    OPAL_THREAD_UNLOCK(&ompi_part_persist.lock);
+    block_entry = opal_atomic_add_fetch_32(&(ompi_part_persist.block_entry), -1);
+    if(to_delete) {
+        err =  mca_part_persist_free_req(to_delete);
+        if (OMPI_SUCCESS != err) {
+            return OMPI_ERROR;
+        }
+    }
+
+    return OMPI_SUCCESS;
+}
+
+__opal_attribute_always_inline__ static inline int
+mca_part_persist_precv_init(void *buf,
+                        size_t parts, 
+                        size_t count,
+                        ompi_datatype_t * datatype,
+                        int src,
+                        int tag,
+                        struct ompi_communicator_t *comm,
+                        struct ompi_info_t * info,
+                        struct ompi_request_t **request)
+{
+    int err = OMPI_SUCCESS;
+    size_t dt_size_;
+    uint32_t dt_size;
+    mca_part_persist_list_t* new_progress_elem = NULL;
+
+    mca_part_persist_precv_request_t *recvreq;
+
+    /* if module hasn't been called before, flag module to init. */
+    if(-1 == ompi_part_persist.init_world)
+    {
+        ompi_part_persist.init_world = 0;
+    }
+
+    /* Allocate a new request */
+    MCA_PART_PERSIST_PRECV_REQUEST_ALLOC(recvreq);
+    if (OPAL_UNLIKELY(NULL == recvreq)) return OMPI_ERR_OUT_OF_RESOURCE;
+
+    MCA_PART_PERSIST_PRECV_REQUEST_INIT(recvreq, ompi_proc, comm, tag, src,
+                                     datatype, buf, parts, count, flags);
+
+    mca_part_persist_request_t *req = (mca_part_persist_request_t *) recvreq;
+
+    /* Set lazy initializion flags */
+    req->initialized = false;
+    req->first_send  = true; 
+    req->flag_post_setup_recv = false;
+    req->flags = NULL;
+    /* Non-blocking receive on setup info */
+    err	= MCA_PML_CALL(irecv(&req->setup_info[1], sizeof(struct ompi_mca_persist_setup_t), MPI_BYTE, src, tag, comm, &req->setup_req[1])); 
+    if(OMPI_SUCCESS != err) return OMPI_ERROR;
+
+    /* Compute total number of bytes */
+    err = opal_datatype_type_size(&(req->req_datatype->super), &dt_size_);
+    if(OMPI_SUCCESS != err) return OMPI_ERROR;
+    dt_size = (dt_size_ > (size_t) UINT_MAX) ? MPI_UNDEFINED : (uint32_t) dt_size_;
+    req->req_bytes = parts * count * dt_size;
+
+    /* Set ompi request initial values */
+    req->req_ompi.req_persistent = true;
+    req->req_part_complete = true;
+    req->req_ompi.req_complete = REQUEST_COMPLETED;
+    req->req_ompi.req_state = OMPI_REQUEST_INACTIVE;
+
+    /* Add element to progress engine */
+    new_progress_elem = OBJ_NEW(mca_part_persist_list_t);
+    new_progress_elem->item = req;
+    req->progress_elem = new_progress_elem; 
+    OPAL_THREAD_LOCK(&ompi_part_persist.lock);
+    opal_list_append(ompi_part_persist.progress_list, (opal_list_item_t*)new_progress_elem);
+    OPAL_THREAD_UNLOCK(&ompi_part_persist.lock);
+
+    /* set return values */
+    *request = (ompi_request_t*) recvreq;
+    return err;
+}
+
+__opal_attribute_always_inline__ static inline int
+mca_part_persist_psend_init(const void* buf,
+                        size_t parts,
+                        size_t count,
+                        ompi_datatype_t* datatype,
+                        int dst,
+                        int tag,
+                        ompi_communicator_t* comm,
+                        struct ompi_info_t * info,
+                        ompi_request_t** request)
+{
+    int err = OMPI_SUCCESS;
+    size_t dt_size_;
+    uint32_t dt_size;
+    mca_part_persist_list_t* new_progress_elem = NULL;
+    mca_part_persist_psend_request_t *sendreq;
+
+    /* if module hasn't been called before, flag module to init. */
+    if(-1 == ompi_part_persist.init_world)
+    {
+        ompi_part_persist.init_world = 0;
+    }
+
+    /* Create new request object */
+    MCA_PART_PERSIST_PSEND_REQUEST_ALLOC(sendreq, comm, dst, ompi_proc);
+    if (OPAL_UNLIKELY(NULL == sendreq)) return OMPI_ERR_OUT_OF_RESOURCE;
+    MCA_PART_PERSIST_PSEND_REQUEST_INIT(sendreq, ompi_proc, comm, tag, dst,
+                                    datatype, buf, parts, count, flags);
+    mca_part_persist_request_t *req = (mca_part_persist_request_t *) sendreq;
+
+    /* Set lazy initialization variables */
+    req->initialized = false;
+    req->first_send  = true; 
+    
+
+    /* Determine total bytes to send. */
+    err = opal_datatype_type_size(&(req->req_datatype->super), &dt_size_);
+    if(OMPI_SUCCESS != err) return OMPI_ERROR;
+    dt_size = (dt_size_ > (size_t) UINT_MAX) ? MPI_UNDEFINED : (uint32_t) dt_size_;
+    req->req_bytes = parts * count * dt_size;
+
+    /* non-blocking send set-up data */
+    req->setup_info[0].world_rank = ompi_comm_rank(&ompi_mpi_comm_world.comm);
+    req->setup_info[0].start_tag = ompi_part_persist.next_send_tag; ompi_part_persist.next_send_tag += parts; 
+    req->my_send_tag = req->setup_info[0].start_tag;
+    req->setup_info[0].setup_tag = ompi_part_persist.next_recv_tag; ompi_part_persist.next_recv_tag++;
+    req->my_recv_tag = req->setup_info[0].setup_tag;
+    req->setup_info[0].num_parts = parts;
+    req->real_parts = parts;
+    req->setup_info[0].count = count;
+    req->real_count = count;
+    req->setup_info[0].dt_size = dt_size;
+
+    req->flags = (int*) calloc(req->real_parts, sizeof(int));
+
+    err = MCA_PML_CALL(isend(&(req->setup_info[0]), sizeof(struct ompi_mca_persist_setup_t), MPI_BYTE, dst, tag, MCA_PML_BASE_SEND_STANDARD, comm, &req->setup_req[0]));
+    if(OMPI_SUCCESS != err) return OMPI_ERROR;
+
+    /* Non-blocking receive on setup info */
+    if(1 == ompi_part_persist.init_comms) {
+        err = MCA_PML_CALL(irecv(&(req->setup_info[1]), sizeof(struct ompi_mca_persist_setup_t), MPI_BYTE, MPI_ANY_SOURCE, req->my_recv_tag, ompi_part_persist.part_comm_setup, &req->setup_req[1]));
+        if(OMPI_SUCCESS != err) return OMPI_ERROR;
+        req->flag_post_setup_recv = false;
+    } else {
+        req->flag_post_setup_recv = true;
+    }
+
+    /* Initilaize completion variables */
+    sendreq->req_base.req_ompi.req_persistent = true;
+    req->req_part_complete = true;
+    req->req_ompi.req_complete = REQUEST_COMPLETED;
+    req->req_ompi.req_state = OMPI_REQUEST_INACTIVE;
+ 
+    /* add element to progress queue */
+    new_progress_elem = OBJ_NEW(mca_part_persist_list_t);
+    new_progress_elem->item = req;
+    req->progress_elem = new_progress_elem;
+    OPAL_THREAD_LOCK(&ompi_part_persist.lock);
+    opal_list_append(ompi_part_persist.progress_list, (opal_list_item_t*)new_progress_elem);
+    OPAL_THREAD_UNLOCK(&ompi_part_persist.lock);
+
+    /* Set return values */
+    *request = (ompi_request_t*) sendreq;
+    return err;
+}
+
+__opal_attribute_always_inline__ static inline int
+mca_part_persist_start(size_t count, ompi_request_t** requests)
+{
+    int err = OMPI_SUCCESS;
+    size_t _count = count;
+    size_t i;
+
+    for(i = 0; i < _count && OMPI_SUCCESS == err; i++) {
+        mca_part_persist_request_t *req = (mca_part_persist_request_t *)(requests[i]);
+        /* First use is a special case, to support lazy initialization */
+        if(false == req->first_send)
+        {
+            if(MCA_PART_PERSIST_REQUEST_PSEND == req->req_type) {
+                req->done_count = 0;
+                memset((void*)req->flags,0,sizeof(uint32_t)*req->real_parts);
+            } else {
+                req->done_count = 0;
+                err = req->persist_reqs[0]->req_start(req->real_parts, req->persist_reqs);
+                memset((void*)req->flags,0,sizeof(uint32_t)*req->real_parts);
+            } 
+        } else {
+            if(MCA_PART_PERSIST_REQUEST_PSEND == req->req_type) {
+                req->done_count = 0;
+                for(i = 0; i < req->real_parts && OMPI_SUCCESS == err; i++) {
+                    req->flags[i] = -1;
+                }
+            } else {
+                req->done_count = 0;
+            } 
+        } 
+        req->req_ompi.req_state = OMPI_REQUEST_ACTIVE;    
+        req->req_ompi.req_status.MPI_TAG = MPI_ANY_TAG;
+        req->req_ompi.req_status.MPI_ERROR = OMPI_SUCCESS;
+        req->req_ompi.req_status._cancelled = 0;
+        req->req_part_complete = false;
+        req->req_ompi.req_complete = false;
+        OPAL_ATOMIC_SWAP_PTR(&req->req_ompi.req_complete, REQUEST_PENDING);   
+    }
+
+    return err;
+}
+
+__opal_attribute_always_inline__ static inline int
+mca_part_persist_pready(size_t min_part,
+                    size_t max_part,
+                    ompi_request_t* request)
+{
+    int err = OMPI_SUCCESS;
+    size_t i;
+
+    mca_part_persist_request_t *req = (mca_part_persist_request_t *)(request);
+    if(true == req->initialized)
+    {
+        err = req->persist_reqs[min_part]->req_start(max_part-min_part+1, (&(req->persist_reqs[min_part])));
+        for(i = min_part; i <= max_part && OMPI_SUCCESS == err; i++) {
+            req->flags[i] = 0; /* Mark partition as ready for testing */
+        }
+    }
+    else
+    {
+        for(i = min_part; i <= max_part && OMPI_SUCCESS == err; i++) {
+            req->flags[i] = -2; /* Mark partition as queued */
+        }
+    }
+    return err;
+}
+
+__opal_attribute_always_inline__ static inline int
+mca_part_persist_parrived(size_t min_part,
+                      size_t max_part,
+                      int* flag, 
+                      ompi_request_t* request)
+{
+    int err = OMPI_SUCCESS;
+    size_t i;
+    int _flag = false;
+    mca_part_persist_request_t *req = (mca_part_persist_request_t *)request;
+
+    if(0 != req->flags) {
+        _flag = 1;
+        if(req->req_parts == req->real_parts) {
+            for(i = min_part; i <= max_part; i++) {
+                _flag = _flag && req->flags[i];            
+            }
+        } else {
+            float convert = ((float)req->real_parts) / ((float)req->req_parts);
+            size_t _min = floor(convert * min_part);
+            size_t _max = ceil(convert * max_part);
+            for(i = _min; i <= _max; i++) {
+                _flag = _flag && req->flags[i];
+            }
+        }
+    }
+
+    if(!_flag) {
+        opal_progress();
+    } 
+    *flag = _flag;
+    return err;
+}
+
+
+/**
+ * mca_part_persist_free marks an entry as free called and sets the request to 
+ * MPI_REQUEST_NULL. Note: requests get freed in the progress engine. 
+ */
+__opal_attribute_always_inline__ static inline int
+mca_part_persist_free(ompi_request_t** request)
+{
+    mca_part_persist_request_t* req = *(mca_part_persist_request_t**)request;
+
+    if(true == req->req_free_called) return OMPI_ERROR;
+    req->req_free_called = true;
+
+    *request = MPI_REQUEST_NULL;
+    return OMPI_SUCCESS;
+}
+
+END_C_DECLS
+
+#endif  /* PART_PERSIST_H_HAS_BEEN_INCLUDED */
diff --git a/ompi/mca/part/persist/part_persist_component.c b/ompi/mca/part/persist/part_persist_component.c
new file mode 100644
index 00000000000..919284476b9
--- /dev/null
+++ b/ompi/mca/part/persist/part_persist_component.c
@@ -0,0 +1,139 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2006-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2006 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013-2021 Sandia National Laboratories.  All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mca/part/persist/part_persist.h"
+
+#include "ompi/mca/part/persist/part_persist_sendreq.h"
+#include "ompi/mca/part/persist/part_persist_recvreq.h"
+#include "ompi/mca/part/persist/part_persist_component.h"
+
+static int mca_part_persist_component_register(void);
+static int mca_part_persist_component_open(void);
+static int mca_part_persist_component_close(void);
+static mca_part_base_module_t* mca_part_persist_component_init( int* priority,
+                            bool enable_progress_threads, bool enable_mpi_threads);
+static int mca_part_persist_component_fini(void);
+
+mca_part_base_component_4_0_0_t mca_part_persist_component = {
+    /* First, the mca_base_component_t struct containing meta
+     * information about the component itself */
+
+    .partm_version = {
+        MCA_PART_BASE_VERSION_2_0_0,
+
+        .mca_component_name = "persist",
+        MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION,
+                              OMPI_RELEASE_VERSION),
+        .mca_open_component = mca_part_persist_component_open,
+        .mca_close_component = mca_part_persist_component_close,
+        .mca_register_component_params = mca_part_persist_component_register,
+    },
+    .partm_data = {
+        /* This component is not checkpoint ready */
+        MCA_BASE_METADATA_PARAM_NONE
+    },
+
+    .partm_init = mca_part_persist_component_init,
+    .partm_finalize = mca_part_persist_component_fini,
+};
+
+static int
+mca_part_persist_component_register(void)
+{
+    ompi_part_persist.free_list_num = 4;
+    (void) mca_base_component_var_register(&mca_part_persist_component.partm_version, "free_list_num",
+                                           "Initial size of request free lists",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           &ompi_part_persist.free_list_num);
+
+    ompi_part_persist.free_list_max = -1;
+    (void) mca_base_component_var_register(&mca_part_persist_component.partm_version, "free_list_max",
+                                           "Maximum size of request free lists",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           &ompi_part_persist.free_list_max);
+
+    ompi_part_persist.free_list_inc = 64;
+    (void) mca_base_component_var_register(&mca_part_persist_component.partm_version, "free_list_inc",
+                                           "Number of elements to add when growing request free lists",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           &ompi_part_persist.free_list_inc);
+
+
+    return OPAL_SUCCESS;
+}
+
+static int
+mca_part_persist_component_open(void)
+{
+    OBJ_CONSTRUCT(&ompi_part_persist.lock, opal_mutex_t);
+
+    ompi_part_persist.next_send_tag = 0;                /**< This is a counter for send tags for the actual data transfer. */
+    ompi_part_persist.next_recv_tag = 0; 
+
+    mca_part_persist_init_lists(); 
+
+    ompi_part_persist.init_comms = 0;
+    ompi_part_persist.init_world = -1;
+
+    ompi_part_persist.part_comm_ready = 0;
+    ompi_part_persist.part_comm_ready = 0;
+
+    ompi_part_persist.block_entry = 0;
+    return OMPI_SUCCESS;
+}
+
+
+static int
+mca_part_persist_component_close(void)
+{
+    OBJ_DESTRUCT(&ompi_part_persist.lock);
+    return OMPI_SUCCESS; 
+}
+
+
+static mca_part_base_module_t*
+mca_part_persist_component_init(int* priority,
+                            bool enable_progress_threads,
+                            bool enable_mpi_threads)
+{
+    *priority = 1;
+
+    opal_output_verbose( 10, 0,
+                         "in persist part priority is %d\n", *priority);
+
+    return &ompi_part_persist.super;
+}
+
+
+static int
+mca_part_persist_component_fini(void)
+{
+    return OMPI_SUCCESS;
+}
+
diff --git a/ompi/mca/part/persist/part_persist_component.h b/ompi/mca/part/persist/part_persist_component.h
new file mode 100644
index 00000000000..6644e2a3332
--- /dev/null
+++ b/ompi/mca/part/persist/part_persist_component.h
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2025 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2020-2021 Sandia National Laboratories. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+/**
+ *  @file
+ */
+
+#ifndef MCA_PART_RMA_COMPONENT_H
+#define MCA_PART_RMA_COMPONENT_H
+
+BEGIN_C_DECLS
+
+/*
+ * PART module functions.
+ */
+OMPI_DECLSPEC extern mca_part_base_component_4_0_0_t mca_part_persist_component;
+
+END_C_DECLS
+
+#endif
diff --git a/ompi/mca/part/persist/part_persist_recvreq.c b/ompi/mca/part/persist/part_persist_recvreq.c
new file mode 100644
index 00000000000..f66bf96a78f
--- /dev/null
+++ b/ompi/mca/part/persist/part_persist_recvreq.c
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2006 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2020-2021 Sandia National Laboratories. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mca/part/persist/part_persist.h"
+#include "ompi/mca/part/persist/part_persist_recvreq.h"
+
+
+static void
+mca_part_persist_precv_request_construct(mca_part_persist_precv_request_t* recvreq)
+{
+    recvreq->req_base.req_ompi.req_start = mca_part_persist_start;
+    recvreq->req_base.req_ompi.req_free = mca_part_persist_free;
+    recvreq->req_base.req_ompi.req_cancel = NULL; 
+    recvreq->req_base.req_ompi.req_persistent = true; 
+    OBJ_CONSTRUCT( &(recvreq->req_base.req_convertor), opal_convertor_t );
+}
+
+
+OBJ_CLASS_INSTANCE(mca_part_persist_precv_request_t,
+                   mca_part_persist_request_t,
+                   mca_part_persist_precv_request_construct,
+                   NULL);
+
diff --git a/ompi/mca/part/persist/part_persist_recvreq.h b/ompi/mca/part/persist/part_persist_recvreq.h
new file mode 100644
index 00000000000..f1d70e1bc8b
--- /dev/null
+++ b/ompi/mca/part/persist/part_persist_recvreq.h
@@ -0,0 +1,103 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2013 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2006 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2012-2021 Sandia National Laboratories.  All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2017      Intel, Inc. All rights reserved
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef PART_PERSIST_RECVREQ_H
+#define PART_PERSIST_RECVREQ_H
+
+#include "ompi/mca/part/persist/part_persist_request.h"
+#include "ompi/mca/part/base/part_base_precvreq.h"
+
+struct mca_part_persist_precv_request_t {
+    mca_part_persist_request_t req_base;
+};
+typedef struct mca_part_persist_precv_request_t mca_part_persist_precv_request_t;
+OBJ_CLASS_DECLARATION(mca_part_persist_precv_request_t);
+
+/**
+ *  Allocate a recv request from the modules free list.
+ *
+ *  @param rc (OUT)  OMPI_SUCCESS or error status on failure.
+ *  @return          Receive request.
+ */
+#define MCA_PART_PERSIST_PRECV_REQUEST_ALLOC(precvreq)                           \
+do {                                                                         \
+    precvreq = (mca_part_persist_precv_request_t*)                               \
+      opal_free_list_get (&mca_part_base_precv_requests);                    \
+    precvreq->req_base.req_type = MCA_PART_PERSIST_REQUEST_PRECV;            \
+ } while (0)
+
+/**
+ * Initialize a receive request with call parameters.
+ *
+ * @param request (IN)       Receive request.
+ * @param addr (IN)          User buffer.
+ * @param count (IN)         Number of elements of indicated datatype.
+ * @param datatype (IN)      User defined datatype.
+ * @param src (IN)           Source rank w/in the communicator.
+ * @param comm (IN)          Communicator.
+ * @param persistent (IN)    Is this a ersistent request.
+ */
+#define MCA_PART_PERSIST_PRECV_REQUEST_INIT( request,                     \
+                                         ompi_proc,                   \
+                                         comm,                        \
+                                         tag,                         \
+                                         src,                         \
+                                         datatype,                    \
+                                         addr,                        \
+                                         parts,                       \
+                                         count,                       \
+                                         flags )                      \
+do {                                                                  \
+        OBJ_RETAIN(comm);                                             \
+        OMPI_DATATYPE_RETAIN(datatype);                               \
+        (request)->req_base.req_comm = comm;                         \
+        (request)->req_base.req_datatype = datatype;                 \
+        (request)->req_base.req_ompi.req_mpi_object.comm = comm;     \
+        (request)->req_base.req_ompi.req_status.MPI_SOURCE = src;    \
+        (request)->req_base.req_ompi.req_status.MPI_TAG = tag;       \
+        (request)->req_base.req_part_complete = true;                \
+        (request)->req_base.req_ompi.req_status._ucount = count;     \
+        (request)->req_base.req_free_called = false;                 \
+        (request)->req_base.req_addr = addr;                       /**< pointer to application buffer */\
+        (request)->req_base.req_parts = parts;                /**< number of partitions */\
+        (request)->req_base.req_count = count;                     /**< count of user datatype elements */\
+        (request)->req_base.req_peer = src;                     /**< peer process - rank w/in this communicator */\
+        (request)->req_base.req_tag = tag;                           \
+} while(0)
+
+/**
+ *  Free the PART receive request
+ */
+#define MCA_PART_PERSIST_PRECV_REQUEST_RETURN(recvreq)                      \
+{                                                                       \
+    OBJ_RELEASE((recvreq)->req_comm);                          \
+    OMPI_DATATYPE_RELEASE((recvreq)->req_datatype);            \
+    OMPI_REQUEST_FINI(&(recvreq)->req_ompi);                   \
+    opal_convertor_cleanup( &((recvreq)->req_convertor) );     \
+    opal_free_list_return ( &mca_part_base_precv_requests,               \
+                           (opal_free_list_item_t*)(recvreq));          \
+}
+
+#endif
+
+
diff --git a/ompi/mca/part/persist/part_persist_request.c b/ompi/mca/part/persist/part_persist_request.c
new file mode 100644
index 00000000000..56362f35656
--- /dev/null
+++ b/ompi/mca/part/persist/part_persist_request.c
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2006 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2020-2021 Sandia National Laboratories. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mca/part/persist/part_persist.h"
+#include "ompi/mca/part/persist/part_persist_request.h"
+
+
+static void mca_part_persist_request_construct( mca_part_persist_request_t* req) {
+    OBJ_CONSTRUCT(&req->req_convertor, opal_convertor_t);
+    req->req_ompi.req_type = OMPI_REQUEST_PART;
+}
+
+static void mca_part_persist_request_destruct( mca_part_persist_request_t* req) {
+    OBJ_DESTRUCT(&req->req_convertor);
+}
+
+OBJ_CLASS_INSTANCE(mca_part_persist_request_t,
+                   ompi_request_t,
+                   mca_part_persist_request_construct,
+                   mca_part_persist_request_destruct);
diff --git a/ompi/mca/part/persist/part_persist_request.h b/ompi/mca/part/persist/part_persist_request.h
new file mode 100644
index 00000000000..ba55c6bd920
--- /dev/null
+++ b/ompi/mca/part/persist/part_persist_request.h
@@ -0,0 +1,105 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2016 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2006 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2017      Intel, Inc. All rights reserved
+ * Copyright (c) 2020-2021 Sandia National Laboratories. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef PART_PERSIST_REQUEST_H
+#define PART_PERSIST_REQUEST_H
+
+#include "ompi/mca/part/base/part_base_psendreq.h"
+#include "ompi/mca/part/part.h"
+#include "opal/sys/atomic.h"
+/**
+ * Type of request.
+ */
+typedef enum {
+    MCA_PART_PERSIST_REQUEST_PSEND,
+    MCA_PART_PERSIST_REQUEST_PRECV,
+    MCA_PART_PERSIST_REQUEST_NULL
+} mca_part_persist_request_type_t;
+
+struct mca_part_persist_list_t;
+
+struct ompi_mca_persist_setup_t {
+   int world_rank;
+   int start_tag;
+   int setup_tag;
+   size_t num_parts;
+   size_t dt_size;
+   size_t count;
+};
+
+
+/**
+ *  Base type for PART PERSIST requests
+ */
+struct mca_part_persist_request_t {
+
+/* START: These fields have to match the definition of the mca_part_persist_request_t */
+    ompi_request_t req_ompi;              /**< base request */
+    volatile int32_t req_part_complete;   /**< flag indicating if the pt-2-pt layer is done with this request */
+    volatile int32_t req_free_called;     /**< flag indicating if the user has freed this request */
+    mca_part_persist_request_type_t req_type; /**< MPI request type - used for test */
+    struct ompi_communicator_t *req_comm; /**< communicator pointer */
+    struct ompi_datatype_t *req_datatype; /**< pointer to data type */
+    opal_convertor_t req_convertor;       /**< always need the convertor */
+
+    const void *req_addr;                       /**< pointer to application buffer */
+    size_t req_parts;                     /**< number of partitions */
+    size_t req_count;                     /**< count of user datatype elements */
+    int32_t req_peer;                     /**< peer process - rank w/in this communicator */
+    int32_t req_tag;                      /**< user defined tag */
+    struct ompi_proc_t* req_proc;         /**< peer process */
+
+/* END: These fields have to match the definition of the mca_part_persist_request_t */
+
+    size_t  req_bytes;                    /**< bytes for completion status */
+
+    size_t real_parts;                   /**< internal number of partitions */
+    size_t real_count;
+    size_t real_dt_size;                 /**< receiver needs to know how large the sender's datatype is. */
+    size_t part_size; 
+
+    ompi_request_t** persist_reqs;            /**< requests for persistent sends/recvs */
+    ompi_request_t* setup_req [2];                /**< Request structure for setup messages */
+
+
+    int32_t req_partitions_send;          /**< Send side number of partitions */
+    int32_t req_partitions_recv;          /**< Recv side number of partitions */
+
+    int32_t my_send_tag;                  /**< This is a counter for send tags for the actual data transfer. */
+    int32_t my_recv_tag;                  /**< This is a counter for receive tags, for incoming setup messages. */ 
+
+    int32_t world_peer;                   /**< peer's rank in MPI_COMM_WORLD */
+
+    int32_t initialized;                  /**< flag for initialized state */
+    int32_t first_send;                   /**< flag for whether the first send has happened */
+    int32_t flag_post_setup_recv;  
+    size_t done_count;             /**< counter for the number of partitions marked ready */
+
+    int32_t *flags;               /**< array of flags to determine whether a partition has arrived */
+
+    struct ompi_mca_persist_setup_t setup_info[2]; /**< Setup info to send during initialization. */
+  
+    struct mca_part_persist_list_t* progress_elem; /**< pointer to progress list element for removal during free. */ 
+
+};
+typedef struct mca_part_persist_request_t mca_part_persist_request_t;
+OBJ_CLASS_DECLARATION(mca_part_persist_request_t);
+
+#endif
diff --git a/ompi/mca/part/persist/part_persist_sendreq.c b/ompi/mca/part/persist/part_persist_sendreq.c
new file mode 100644
index 00000000000..b4526bd16e3
--- /dev/null
+++ b/ompi/mca/part/persist/part_persist_sendreq.c
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2006 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2020-2021 Sandia National Laboratories. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mca/part/persist/part_persist.h"
+#include "ompi/mca/part/persist/part_persist_sendreq.h"
+
+
+static void mca_part_persist_psend_request_construct(mca_part_persist_psend_request_t* sendreq)
+{
+    /* no need to reinit for every send -- never changes */
+    sendreq->req_base.req_ompi.req_start = mca_part_persist_start;
+    sendreq->req_base.req_ompi.req_free = mca_part_persist_free;
+    sendreq->req_base.req_ompi.req_persistent = true;
+    sendreq->req_base.req_ompi.req_cancel = NULL;
+}
+
+OBJ_CLASS_INSTANCE(mca_part_persist_psend_request_t,
+                   mca_part_persist_request_t,
+                   mca_part_persist_psend_request_construct,
+                   NULL);
+
diff --git a/ompi/mca/part/persist/part_persist_sendreq.h b/ompi/mca/part/persist/part_persist_sendreq.h
new file mode 100644
index 00000000000..b3bf08386ce
--- /dev/null
+++ b/ompi/mca/part/persist/part_persist_sendreq.h
@@ -0,0 +1,94 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2013 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2006 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      Intel, Inc. All rights reserved
+ * Copyright (c) 2020-2021 Sandia National Laboratories. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef PART_PERSIST_PSENDREQ_H
+#define PART_PERSIST_PSENDREQ_H
+
+#include "ompi/mca/part/persist/part_persist_request.h"
+#include "ompi/mca/part/base/part_base_psendreq.h"
+#include "ompi/mca/part/part.h"
+#include "opal/prefetch.h"
+
+struct mca_part_persist_psend_request_t {
+    mca_part_persist_request_t req_base;
+};
+typedef struct mca_part_persist_psend_request_t mca_part_persist_psend_request_t;
+OBJ_CLASS_DECLARATION(mca_part_persist_psend_request_t);
+
+
+#define MCA_PART_PERSIST_PSEND_REQUEST_ALLOC(sendreq, comm, dst,          \
+                                           ompi_proc)                 \
+do {                                                                  \
+    sendreq = (mca_part_persist_psend_request_t*)                         \
+        opal_free_list_wait (&mca_part_base_psend_requests);          \
+    sendreq->req_base.req_type = MCA_PART_PERSIST_REQUEST_PSEND;          \
+} while(0)
+
+#define MCA_PART_PERSIST_PSEND_REQUEST_INIT( req_send,                    \
+                                         ompi_proc,                   \
+                                         comm,                        \
+                                         tag,                         \
+                                         dst,                         \
+                                         datatype,                    \
+                                         buf,                         \
+                                         parts,                       \
+                                         count,                       \
+                                         flags )                      \
+    do {                                                              \
+        OMPI_REQUEST_INIT(&(sendreq->req_base.req_ompi),     \
+                          false);                                     \
+        OBJ_RETAIN(comm);                                             \
+        OMPI_DATATYPE_RETAIN(datatype);                               \
+        (req_send)->req_base.req_comm = comm;                         \
+        (req_send)->req_base.req_datatype = datatype;                 \
+        (req_send)->req_base.req_ompi.req_mpi_object.comm = comm;     \
+        (req_send)->req_base.req_ompi.req_status.MPI_SOURCE =         \
+        comm->c_my_rank;                                              \
+        (req_send)->req_base.req_ompi.req_status.MPI_TAG = tag;       \
+        (req_send)->req_base.req_part_complete = true;                  \
+        (req_send)->req_base.req_ompi.req_status._ucount = count;     \
+        (req_send)->req_base.req_free_called = false;                 \
+        (req_send)->req_base.req_addr = buf;                       /**< pointer to application buffer */\
+        (req_send)->req_base.req_parts = parts;                /**< number of partitions */\
+        (req_send)->req_base.req_count = count;                     /**< count of user datatype elements */\
+        (req_send)->req_base.req_peer = dst;                     /**< peer process - rank w/in this communicator */\
+        (req_send)->req_base.req_tag = tag;                      /**< user defined tag */\
+    } while(0)
+
+/*
+ * Release resources associated with a request
+ */
+#define MCA_PART_PERSIST_PSEND_REQUEST_RETURN(sendreq)                      \
+    {                                                                   \
+        /*  Let the base handle the reference counts */                 \
+        OMPI_DATATYPE_RELEASE(sendreq->req_datatype);  \
+        OBJ_RELEASE(sendreq->req_comm);               \
+        OMPI_REQUEST_FINI(&sendreq->req_ompi);        \
+        opal_convertor_cleanup( &(sendreq->req_convertor) ); \
+        opal_free_list_return ( &mca_part_base_psend_requests,            \
+                               (opal_free_list_item_t*)sendreq);        \
+    }
+
+#endif
diff --git a/ompi/mca/part/persist/post_configure.sh b/ompi/mca/part/persist/post_configure.sh
new file mode 100644
index 00000000000..d06ec572027
--- /dev/null
+++ b/ompi/mca/part/persist/post_configure.sh
@@ -0,0 +1 @@
+DIRECT_CALL_HEADER="ompi/mca/part/rma/part_rma.h"
diff --git a/ompi/mca/pml/base/base.h b/ompi/mca/pml/base/base.h
index 8eb37e48448..60a3755eb7e 100644
--- a/ompi/mca/pml/base/base.h
+++ b/ompi/mca/pml/base/base.h
@@ -1,4 +1,4 @@
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -11,6 +11,9 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights reserved.
+ * Copyright (c) 2018      Triad National Security, LLC. All rights
+ *                         reserved.
+
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -60,8 +63,6 @@ OMPI_DECLSPEC int mca_pml_base_pml_check_selected(const char *my_pml,
                                                   struct ompi_proc_t **procs,
                                                   size_t nprocs);
 
-OMPI_DECLSPEC int mca_pml_base_finalize(void);
-
 /* not #if conditional on OPAL_ENABLE_FT_MPI for ABI */
 OMPI_DECLSPEC int mca_pml_base_revoke_comm(struct ompi_communicator_t *comm, bool coll_only);
 
@@ -71,6 +72,7 @@ OMPI_DECLSPEC int mca_pml_base_revoke_comm(struct ompi_communicator_t *comm, boo
 OMPI_DECLSPEC extern mca_pml_base_component_t mca_pml_base_selected_component;
 OMPI_DECLSPEC extern mca_pml_base_module_t mca_pml;
 OMPI_DECLSPEC extern opal_pointer_array_t mca_pml_base_pml;
+OMPI_DECLSPEC extern bool ompi_pml_base_check_pml;
 
 END_C_DECLS
 
diff --git a/ompi/mca/pml/base/pml_base_bsend.c b/ompi/mca/pml/base/pml_base_bsend.c
index 3826253e2ae..d1c38e90209 100644
--- a/ompi/mca/pml/base/pml_base_bsend.c
+++ b/ompi/mca/pml/base/pml_base_bsend.c
@@ -16,6 +16,8 @@
  * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2018      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -34,6 +36,7 @@
 #include "ompi/mca/pml/base/pml_base_sendreq.h"
 #include "ompi/mca/pml/base/pml_base_bsend.h"
 #include "opal/mca/mpool/mpool.h"
+#include "ompi/runtime/mpiruntime.h"
 
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
@@ -56,6 +59,8 @@ static opal_atomic_int32_t          mca_pml_bsend_init = 0;
 /* defined in pml_base_open.c */
 extern char *ompi_pml_base_bsend_allocator_name;
 
+static int mca_pml_base_bsend_fini (void);
+
 /*
  * Routine to return pages to sub-allocator as needed
  */
@@ -77,7 +82,7 @@ static void* mca_pml_bsend_alloc_segment(void *ctx, size_t *size_inout)
 /*
  * One time initialization at startup
  */
-int mca_pml_base_bsend_init(bool thread_safe)
+int mca_pml_base_bsend_init (void)
 {
     size_t tmp;
 
@@ -100,6 +105,9 @@ int mca_pml_base_bsend_init(bool thread_safe)
         tmp >>= 1;
         mca_pml_bsend_pagebits++;
     }
+
+    ompi_mpi_instance_append_finalize (mca_pml_base_bsend_fini);
+
     return OMPI_SUCCESS;
 }
 
@@ -107,7 +115,7 @@ int mca_pml_base_bsend_init(bool thread_safe)
 /*
  * One-time cleanup at shutdown - release any resources.
  */
-int mca_pml_base_bsend_fini(void)
+static int mca_pml_base_bsend_fini (void)
 {
     if(OPAL_THREAD_ADD_FETCH32(&mca_pml_bsend_init,-1) > 0)
         return OMPI_SUCCESS;
@@ -125,11 +133,10 @@ int mca_pml_base_bsend_fini(void)
 /*
  * User-level call to attach buffer.
  */
-int mca_pml_base_bsend_attach(void* addr, int size)
+int mca_pml_base_bsend_attach(void* addr, size_t size)
 {
     int align;
 
-    bool thread_safe = ompi_mpi_thread_multiple;
     if(NULL == addr || size <= 0) {
         return OMPI_ERR_BUFFER;
     }
@@ -142,7 +149,7 @@ int mca_pml_base_bsend_attach(void* addr, int size)
     }
 
     /* try to create an instance of the allocator - to determine thread safety level */
-    mca_pml_bsend_allocator = mca_pml_bsend_allocator_component->allocator_init(thread_safe, mca_pml_bsend_alloc_segment, NULL, NULL);
+    mca_pml_bsend_allocator = mca_pml_bsend_allocator_component->allocator_init(ompi_mpi_thread_multiple, mca_pml_bsend_alloc_segment, NULL, NULL);
     if(NULL == mca_pml_bsend_allocator) {
         OPAL_THREAD_UNLOCK(&mca_pml_bsend_mutex);
         return OMPI_ERR_BUFFER;
@@ -172,7 +179,7 @@ int mca_pml_base_bsend_attach(void* addr, int size)
 /*
  * User-level call to detach buffer
  */
-int mca_pml_base_bsend_detach(void* addr, int* size)
+int mca_pml_base_bsend_detach(void* addr, size_t* size)
 {
     OPAL_THREAD_LOCK(&mca_pml_bsend_mutex);
 
@@ -194,7 +201,7 @@ int mca_pml_base_bsend_detach(void* addr, int* size)
     if(NULL != addr)
         *((void**)addr) = mca_pml_bsend_userbase;
     if(NULL != size)
-        *size = (int)mca_pml_bsend_usersize;
+        *size = mca_pml_bsend_usersize;
 
     /* reset local variables */
     mca_pml_bsend_userbase = NULL;
@@ -242,7 +249,7 @@ int mca_pml_base_bsend_request_start(ompi_request_t* request)
 
         OPAL_THREAD_UNLOCK(&mca_pml_bsend_mutex);
 
-        /* The convertor is already initialized in the begining so we just have to
+        /* The convertor is already initialized in the beginning so we just have to
          * pack the data in the newly allocated buffer.
          */
         iov.iov_base = (IOVBASE_TYPE*)sendreq->req_addr;
diff --git a/ompi/mca/pml/base/pml_base_bsend.h b/ompi/mca/pml/base/pml_base_bsend.h
index e50bdc7b5e8..0d3c47a7ca5 100644
--- a/ompi/mca/pml/base/pml_base_bsend.h
+++ b/ompi/mca/pml/base/pml_base_bsend.h
@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -10,6 +11,8 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2018      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -25,11 +28,10 @@
 
 BEGIN_C_DECLS
 
-OMPI_DECLSPEC int mca_pml_base_bsend_init(bool enable_mpi_threads);
-OMPI_DECLSPEC int mca_pml_base_bsend_fini(void);
+OMPI_DECLSPEC int mca_pml_base_bsend_init (void);
 
-int mca_pml_base_bsend_attach(void* addr, int size);
-int mca_pml_base_bsend_detach(void* addr, int* size);
+int mca_pml_base_bsend_attach(void* addr, size_t size);
+int mca_pml_base_bsend_detach(void* addr, size_t* size);
 
 OMPI_DECLSPEC int mca_pml_base_bsend_request_alloc(ompi_request_t*);
 OMPI_DECLSPEC int mca_pml_base_bsend_request_start(ompi_request_t*);
diff --git a/ompi/mca/pml/base/pml_base_frame.c b/ompi/mca/pml/base/pml_base_frame.c
index 5481095d486..8468c5c4f17 100644
--- a/ompi/mca/pml/base/pml_base_frame.c
+++ b/ompi/mca/pml/base/pml_base_frame.c
@@ -15,7 +15,7 @@
  *                         reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2018 IBM Corporation. All rights reserved.
+ * Copyright (c) 2018-2022 IBM Corporation.  All rights reserved.
  * Copyright (c) 2020      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
@@ -42,6 +42,9 @@
 #include "ompi/mca/pml/pml.h"
 #include "ompi/mca/pml/base/base.h"
 #include "ompi/mca/pml/base/pml_base_request.h"
+#if MPI_VERSION >= 4
+#include "ompi/mca/pml/base/pml_base_sendreq.h"
+#endif
 
 /*
  * The following file was created by configure.  It contains extern
@@ -88,23 +91,41 @@ mca_pml_base_module_t mca_pml = {
     NULL,                    /* pml_dump */
     0,                       /* pml_max_contextid */
     0,                       /* pml_max_tag */
-    0                        /* pml_flags */
+    0,                       /* pml_flags */
+    NULL                     /* pml_get_transports */
 };
 
 mca_pml_base_component_t mca_pml_base_selected_component = {{0}};
 opal_pointer_array_t mca_pml_base_pml = {{0}};
 char *ompi_pml_base_bsend_allocator_name = NULL;
+bool ompi_pml_base_check_pml = true;
 
 #if !MCA_ompi_pml_DIRECT_CALL
 static char *ompi_pml_base_wrapper = NULL;
 #endif
 
+#if MPI_VERSION >= 4
+#define OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_DEFAULT OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_ONCE
+int ompi_pml_base_warn_dep_cancel_send_level = OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_DEFAULT;
+mca_base_var_enum_value_t ompi_pml_base_warn_dep_cancel_send_values[] = {
+    {.value = OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_ALWAYS, .string = "always"},
+    {.value = OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_ONCE,   .string = "once"},
+    {.value = OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_NEVER,  .string = "never"},
+    {0, NULL}
+};
+#endif
+
 static int mca_pml_base_register(mca_base_register_flag_t flags)
 {
 #if !MCA_ompi_pml_DIRECT_CALL
     int var_id;
 #endif
 
+#if MPI_VERSION >= 4
+    mca_base_var_enum_t *ompi_pml_base_warn_dep_cancel_send_enum = NULL;
+    int rc;
+#endif
+
     ompi_pml_base_bsend_allocator_name = "basic";
     (void) mca_base_var_register("ompi", "pml", "base", "bsend_allocator", NULL,
                                  MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
@@ -123,16 +144,32 @@ static int mca_pml_base_register(mca_base_register_flag_t flags)
     (void) mca_base_var_register_synonym(var_id, "ompi", "pml", NULL, "wrapper", 0);
 #endif
 
-    return OMPI_SUCCESS;
-}
+#if MPI_VERSION >= 4
+    mca_base_var_enum_create("pml_base_deprecate_warnings", ompi_pml_base_warn_dep_cancel_send_values,
+                             &ompi_pml_base_warn_dep_cancel_send_enum);
+    rc = mca_base_var_register("ompi", "pml", "base", "warn_dep_cancel_send",
+                               "How often to issue warnings for deprecated cancellation of send requests",
+                               MCA_BASE_VAR_TYPE_INT, ompi_pml_base_warn_dep_cancel_send_enum, 0, 0,
+                               OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+                               &ompi_pml_base_warn_dep_cancel_send_level);
+    OBJ_RELEASE(ompi_pml_base_warn_dep_cancel_send_enum);
+    if (OPAL_ERR_VALUE_OUT_OF_BOUNDS == rc) {
+        ompi_pml_base_warn_dep_cancel_send_level = OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_DEFAULT;
+        opal_output(0, "pml:base:register: Warning invalid deprecation warning value specified. Using default: %d",
+                    ompi_pml_base_warn_dep_cancel_send_level);
+    }
+#endif
 
-int mca_pml_base_finalize(void) {
-  if (NULL != mca_pml_base_selected_component.pmlm_finalize) {
-    return mca_pml_base_selected_component.pmlm_finalize();
-  }
-  return OMPI_SUCCESS;
-}
+    ompi_pml_base_check_pml = true;
+    (void) mca_base_var_register("ompi", "pml", "base", "check_pml",
+                                 "Whether to check the pml selections to ensure they all match",
+                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
+                                 OPAL_INFO_LVL_9,
+                                 MCA_BASE_VAR_SCOPE_READONLY,
+                                 &ompi_pml_base_check_pml);
 
+    return OMPI_SUCCESS;
+}
 
 static int mca_pml_base_close(void)
 {
@@ -143,7 +180,7 @@ static int mca_pml_base_close(void)
         opal_progress_unregister(mca_pml.pml_progress);
     }
 
-    /* Blatently ignore the return code (what would we do to recover,
+    /* Blatantly ignore the return code (what would we do to recover,
        anyway?  This module is going away, so errors don't matter
        anymore) */
 
diff --git a/ompi/mca/pml/base/pml_base_select.c b/ompi/mca/pml/base/pml_base_select.c
index 1b9c1de13d4..e66ec02b5e4 100644
--- a/ompi/mca/pml/base/pml_base_select.c
+++ b/ompi/mca/pml/base/pml_base_select.c
@@ -1,4 +1,4 @@
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -14,7 +14,8 @@
  *                         reserved.
  * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015-2020 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2020      Amazon.com, Inc. or its affiliates.  All Rights
+ * Copyright (c) 2020-2022 Amazon.com, Inc. or its affiliates.  All Rights
+ * Copyright (c) 2018-2020 Triad National Security, LLC. All rights
  *                         reserved.
  * $COPYRIGHT$
  *
@@ -37,6 +38,7 @@
 #include "opal/mca/pmix/pmix-internal.h"
 
 #include "ompi/constants.h"
+#include "ompi/instance/instance.h"
 #include "ompi/mca/pml/pml.h"
 #include "ompi/mca/pml/base/base.h"
 #include "ompi/proc/proc.h"
@@ -46,6 +48,15 @@ typedef struct opened_component_t {
   mca_pml_base_component_t *om_component;
 } opened_component_t;
 
+
+static int mca_pml_base_finalize (void) {
+  if (NULL != mca_pml_base_selected_component.pmlm_finalize) {
+      return mca_pml_base_selected_component.pmlm_finalize();
+  }
+
+  return OMPI_SUCCESS;
+}
+
 /**
  * Function for selecting one component from all those that are
  * available.
@@ -59,7 +70,7 @@ typedef struct opened_component_t {
 int mca_pml_base_select(bool enable_progress_threads,
                         bool enable_mpi_threads)
 {
-    int i, priority = 0, best_priority = 0, num_pml = 0, ret = 0;
+    int i, priority = 0, best_priority = -1, ret = 0;
     opal_list_item_t *item = NULL;
     mca_base_component_list_item_t *cli = NULL;
     mca_pml_base_component_t *component = NULL, *best_component = NULL;
@@ -71,9 +82,6 @@ int mca_pml_base_select(bool enable_progress_threads,
     /* Traverse the list of available components; call their init
        functions. */
 
-    best_priority = -1;
-    best_component = NULL;
-    module = NULL;
     OBJ_CONSTRUCT(&opened, opal_list_t);
     OPAL_LIST_FOREACH(cli, &ompi_pml_base_framework.framework_components, mca_base_component_list_item_t) {
         component = (mca_pml_base_component_t *) cli->cli_component;
@@ -110,9 +118,6 @@ int mca_pml_base_select(bool enable_progress_threads,
             continue;
         }
 
-        /* this is a pml that could be considered */
-        num_pml++;
-
         /* Init component to get its priority */
         opal_output_verbose( 10, ompi_pml_base_framework.framework_output,
                              "select: initializing %s component %s",
@@ -189,7 +194,7 @@ int mca_pml_base_select(bool enable_progress_threads,
 
             if (NULL != om->om_component->pmlm_finalize) {
 
-                /* Blatently ignore the return code (what would we do to
+                /* Blatantly ignore the return code (what would we do to
                    recover, anyway?  This component is going away, so errors
                    don't matter anymore) */
 
@@ -229,6 +234,7 @@ int mca_pml_base_select(bool enable_progress_threads,
     ret = mca_pml_base_pml_selected(best_component->pmlm_version.mca_component_name);
 
     /* All done */
+    ompi_mpi_instance_append_finalize (mca_pml_base_finalize);
 
     return ret;
 }
@@ -340,6 +346,10 @@ mca_pml_base_pml_check_selected(const char *my_pml,
     int ret = 0;
     size_t i;
 
+    if (!ompi_pml_base_check_pml) {
+        return OMPI_SUCCESS;
+    }
+
     if (!opal_pmix_collect_all_data) {
         /*
          * If direct modex, then compare our PML with the peer's PML
diff --git a/ompi/mca/pml/base/pml_base_sendreq.c b/ompi/mca/pml/base/pml_base_sendreq.c
index fabd711129b..fd25b2f4898 100644
--- a/ompi/mca/pml/base/pml_base_sendreq.c
+++ b/ompi/mca/pml/base/pml_base_sendreq.c
@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -19,6 +20,9 @@
 #include <string.h>
 #include "ompi/mca/pml/pml.h"
 #include "ompi/mca/pml/base/pml_base_sendreq.h"
+#if MPI_VERSION >= 4
+#include "opal/sys/atomic.h"
+#endif
 
 static void mca_pml_base_send_request_construct(mca_pml_base_send_request_t* req);
 static void mca_pml_base_send_request_destruct(mca_pml_base_send_request_t* req);
@@ -47,3 +51,17 @@ static void mca_pml_base_send_request_destruct(mca_pml_base_send_request_t* req)
      */
 }
 
+#if MPI_VERSION >= 4
+int mca_pml_cancel_send_callback(struct ompi_request_t *request, int flag)
+{
+    static opal_atomic_int32_t send_deprecate_count = 0;
+    int32_t val;
+
+    val = opal_atomic_add_fetch_32(&send_deprecate_count, 1);
+    if ( (OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_ONCE   == ompi_pml_base_warn_dep_cancel_send_level && (1 == val)) ||
+         (OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_ALWAYS == ompi_pml_base_warn_dep_cancel_send_level) ) {
+        opal_output(0, "WARNING: Calling MPI_Cancel for a request created by a non-blocking send is deprecated.");
+    }
+    return OMPI_SUCCESS;
+}
+#endif
diff --git a/ompi/mca/pml/base/pml_base_sendreq.h b/ompi/mca/pml/base/pml_base_sendreq.h
index 9ecec0a6d87..9c0987ce0c6 100644
--- a/ompi/mca/pml/base/pml_base_sendreq.h
+++ b/ompi/mca/pml/base/pml_base_sendreq.h
@@ -16,6 +16,7 @@
  * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2017      Intel, Inc. All rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -67,6 +68,19 @@ OMPI_DECLSPEC OBJ_CLASS_DECLARATION( mca_pml_base_send_request_t );
  * is done in the send request start routine.
  */
 
+#if MPI_VERSION >= 4
+/*
+ * Deprecation of MPI_Cancel for non blocking sends in MPI 4.0 and later
+ * is handled by adding a callback for the cancel request.
+ */
+#define MCA_PML_BASE_SEND_CANCEL_CALLBACK(req)                              \
+    if (NULL == (req)->req_base.req_ompi.req_cancel) {                      \
+        (req)->req_base.req_ompi.req_cancel = mca_pml_cancel_send_callback; \
+    }
+#else
+#define MCA_PML_BASE_SEND_CANCEL_CALLBACK(req)
+#endif
+
 #define MCA_PML_BASE_SEND_REQUEST_INIT( request,                          \
                                         addr,                             \
                                         count,                            \
@@ -95,6 +109,7 @@ OMPI_DECLSPEC OBJ_CLASS_DECLARATION( mca_pml_base_send_request_t );
       (request)->req_base.req_pml_complete = false;                       \
       (request)->req_base.req_free_called = false;                        \
       (request)->req_base.req_ompi.req_status._cancelled = 0;             \
+      MCA_PML_BASE_SEND_CANCEL_CALLBACK(request)                          \
       (request)->req_bytes_packed = 0;                                    \
                                                                           \
       /* initialize datatype convertor for this request */                \
@@ -153,6 +168,21 @@ OMPI_DECLSPEC OBJ_CLASS_DECLARATION( mca_pml_base_send_request_t );
     } while (0)
 
 
+#if MPI_VERSION >= 4
+enum {
+      OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_ALWAYS = 1,
+      OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_ONCE   = 2,
+      OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_NEVER  = 3,
+};
+OMPI_DECLSPEC extern int ompi_pml_base_warn_dep_cancel_send_level;
+
+/**
+ * Cancellation callback used to issue a deprecation warning on
+ * nonblocking sends (per MPI 4.0).
+ */
+OMPI_DECLSPEC extern int mca_pml_cancel_send_callback(struct ompi_request_t *request, int flag);
+#endif
+
 END_C_DECLS
 
 #endif
diff --git a/ompi/mca/pml/cm/pml_cm.c b/ompi/mca/pml/cm/pml_cm.c
index 567b00bc331..51a63fdba4c 100644
--- a/ompi/mca/pml/cm/pml_cm.c
+++ b/ompi/mca/pml/cm/pml_cm.c
@@ -93,7 +93,7 @@ int
 mca_pml_cm_add_comm(ompi_communicator_t* comm)
 {
     /* should never happen, but it was, so check */
-    if (comm->c_contextid > ompi_pml_cm.super.pml_max_contextid) {
+    if (comm->c_index > ompi_pml_cm.super.pml_max_contextid) {
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
 
diff --git a/ompi/mca/pml/cm/pml_cm.h b/ompi/mca/pml/cm/pml_cm.h
index fa563e0b313..4abf40c5f94 100644
--- a/ompi/mca/pml/cm/pml_cm.h
+++ b/ompi/mca/pml/cm/pml_cm.h
@@ -1,12 +1,13 @@
 /*
  * Copyright (c) 2004-2006 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2004-2007 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2017      Intel, Inc. All rights reserved
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -17,10 +18,6 @@
 #ifndef PML_CM_H
 #define PML_CM_H
 
-#ifdef HAVE_ALLOCA_H
-#include <alloca.h>
-#endif
-
 #include "ompi_config.h"
 #include "ompi/request/request.h"
 #include "ompi/mca/pml/pml.h"
@@ -43,13 +40,6 @@ BEGIN_C_DECLS
 
 struct mca_mtl_request_t;
 
-/* Array of send completion callback - one per send type
- * These are called internally by the library when the send
- * is completed from its perspective.
- */
-extern void (*send_completion_callbacks[])
-    (struct mca_mtl_request_t *mtl_request);
-
 struct ompi_pml_cm_t {
     mca_pml_base_module_t super;
     int                   free_list_num;
@@ -131,14 +121,6 @@ mca_pml_cm_irecv(void *addr,
     return ret;
 }
 
-__opal_attribute_always_inline__ static inline void
-mca_pml_cm_recv_fast_completion(struct mca_mtl_request_t *mtl_request)
-{
-    // Do nothing!
-    ompi_request_complete(mtl_request->ompi_req, true);
-    return;
-}
-
 __opal_attribute_always_inline__ static inline int
 mca_pml_cm_recv(void *addr,
                 size_t count,
@@ -150,73 +132,41 @@ mca_pml_cm_recv(void *addr,
 {
     int ret;
     uint32_t flags = 0;
+    mca_pml_cm_thin_recv_request_t *recvreq;
+
+    MCA_PML_CM_THIN_RECV_REQUEST_ALLOC(recvreq);
+    if (OPAL_UNLIKELY(NULL == recvreq))
+        return OMPI_ERR_OUT_OF_RESOURCE;
+
 #if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
-    ompi_proc_t *ompi_proc;
+    ompi_proc_t *ompi_proc = NULL;
 #endif
-    opal_convertor_t convertor;
-    mca_pml_cm_request_t req;
-    mca_mtl_request_t *req_mtl =
-            alloca(sizeof(mca_mtl_request_t) + ompi_mtl->mtl_request_size);
-
-    OBJ_CONSTRUCT(&convertor, opal_convertor_t);
-    req_mtl->ompi_req = &req.req_ompi;
-    req_mtl->completion_callback = mca_pml_cm_recv_fast_completion;
-
-    req.req_pml_type = MCA_PML_CM_REQUEST_RECV_THIN;
-    req.req_free_called = false;
-    req.req_ompi.req_complete = false;
-    req.req_ompi.req_complete_cb = NULL;
-    req.req_ompi.req_state = OMPI_REQUEST_ACTIVE;
-    req.req_ompi.req_status.MPI_TAG = OMPI_ANY_TAG;
-    req.req_ompi.req_status.MPI_ERROR = OMPI_SUCCESS;
-    req.req_ompi.req_status._cancelled = 0;
 
-#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
-    if( MPI_ANY_SOURCE == src ) {
-        ompi_proc = ompi_proc_local_proc;
-    } else {
-        ompi_proc = ompi_comm_peer_lookup( comm, src );
-    }
+    MCA_PML_CM_THIN_RECV_REQUEST_INIT(recvreq,
+                                      ompi_proc,
+                                      comm,
+                                      src,
+                                      datatype,
+                                      addr,
+                                      count,
+                                      flags);
 
-    MCA_PML_CM_SWITCH_CUDA_CONVERTOR_OFF(flags, datatype, count);
-
-    opal_convertor_copy_and_prepare_for_recv(
-	ompi_proc->super.proc_convertor,
-		&(datatype->super),
-		count,
-		addr,
-                flags,
-		&convertor );
-#else
-    MCA_PML_CM_SWITCH_CUDA_CONVERTOR_OFF(flags, datatype, count);
-
-    opal_convertor_copy_and_prepare_for_recv(
-	ompi_mpi_local_convertor,
-		&(datatype->super),
-		count,
-		addr,
-                flags,
-		&convertor );
-#endif
+    assert(NULL == recvreq->req_base.req_ompi.req_complete_cb);
 
-    ret = OMPI_MTL_CALL(irecv(ompi_mtl,
-                              comm,
-                              src,
-                              tag,
-                              &convertor,
-                              req_mtl));
-    if( OPAL_UNLIKELY(OMPI_SUCCESS != ret) ) {
-	OBJ_DESTRUCT(&convertor);
+    MCA_PML_CM_THIN_RECV_REQUEST_START(recvreq, comm, tag, src, ret);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
+        MCA_PML_CM_THIN_RECV_REQUEST_RETURN(recvreq);
         return ret;
     }
 
-    ompi_request_wait_completion(&req.req_ompi);
+    ompi_request_wait_completion(&recvreq->req_base.req_ompi);
 
-    if (NULL != status) {  /* return status */
-        *status = req.req_ompi.req_status;
+    if (MPI_STATUS_IGNORE != status) {
+        OMPI_COPY_STATUS(status, recvreq->req_base.req_ompi.req_status, false);
     }
-    ret = req.req_ompi.req_status.MPI_ERROR;
-    OBJ_DESTRUCT(&convertor);
+
+    ret = recvreq->req_base.req_ompi.req_status.MPI_ERROR;
+    ompi_request_free((ompi_request_t **) &recvreq);
     return ret;
 }
 
@@ -243,8 +193,8 @@ mca_pml_cm_isend_init(const void* buf,
                                      datatype, sendmode, true, false, buf, count, flags);
 
     /* Work around a leak in start by marking this request as complete. The
-     * problem occured because we do not have a way to differentiate an
-     * inital request and an incomplete pml request in start. This line
+     * problem occurred because we do not have a way to differentiate an
+     * initial request and an incomplete pml request in start. This line
      * allows us to detect this state. */
     sendreq->req_send.req_base.req_pml_complete = true;
 
@@ -375,25 +325,18 @@ mca_pml_cm_send(const void *buf,
 		convertor.flags      = ompi_mpi_local_convertor->flags;
 		convertor.master     = ompi_mpi_local_convertor->master;
 
-		convertor.local_size = count * datatype->super.size;
-		convertor.pBaseBuf   = (unsigned char*)buf + datatype->super.true_lb;
-		convertor.count      = count;
-		convertor.pDesc      = &datatype->super;
-
-#if OPAL_CUDA_SUPPORT
-        /* Switches off CUDA detection if
-           MTL set MCA_MTL_BASE_FLAG_CUDA_INIT_DISABLE during init */
-        MCA_PML_CM_SWITCH_CUDA_CONVERTOR_OFF(flags, datatype, count);
-        convertor.flags      |= flags;
-        /* Sets CONVERTOR_CUDA flag if CUDA buffer */
-        opal_convertor_prepare_for_send( &convertor, &datatype->super, count, buf );
-#endif
+                /* Switches off device detection if
+                   MTL set MCA_MTL_BASE_FLAG_ACCELERATOR_INIT_DISABLE during init */
+                MCA_PML_CM_SWITCH_ACCELERATOR_CONVERTOR_OFF(flags, datatype, count);
+                convertor.flags      |= flags;
+                /* Sets CONVERTOR_ACCELERATOR flag if device buffer */
+                opal_convertor_prepare_for_send(&convertor, &datatype->super, count, (unsigned char *)buf);
     } else
 #endif
 	{
 		ompi_proc = ompi_comm_peer_lookup(comm, dst);
 
-                MCA_PML_CM_SWITCH_CUDA_CONVERTOR_OFF(flags, datatype, count);
+                MCA_PML_CM_SWITCH_ACCELERATOR_CONVERTOR_OFF(flags, datatype, count);
 
 		opal_convertor_copy_and_prepare_for_send(
 		ompi_proc->super.proc_convertor,
@@ -548,8 +491,8 @@ mca_pml_cm_mrecv(void *buf,
 
     ompi_request_wait_completion(&recvreq->req_base.req_ompi);
 
-    if (NULL != status) {  /* return status */
-        *status = recvreq->req_base.req_ompi.req_status;
+    if (MPI_STATUS_IGNORE != status) {
+        OMPI_COPY_STATUS(status, recvreq->req_base.req_ompi.req_status, false);
     }
     ret = recvreq->req_base.req_ompi.req_status.MPI_ERROR;
     ompi_request_free( (ompi_request_t**)&recvreq );
diff --git a/ompi/mca/pml/cm/pml_cm_component.c b/ompi/mca/pml/cm/pml_cm_component.c
index 40def94feb2..7ea3678cb6e 100644
--- a/ompi/mca/pml/cm/pml_cm_component.c
+++ b/ompi/mca/pml/cm/pml_cm_component.c
@@ -14,6 +14,8 @@
  *                         reserved.
  * Copyright (c) 2020      Amazon.com, Inc. or its affiliates.
  *                         All Rights reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -63,18 +65,6 @@ mca_pml_base_component_2_1_0_t mca_pml_cm_component = {
     .pmlm_finalize = mca_pml_cm_component_fini,
 };
 
-/* Array of send completion callback - one per send type
- * These are called internally by the library when the send
- * is completed from its perspective.
- */
-void (*send_completion_callbacks[MCA_PML_BASE_SEND_SIZE])
-    (struct mca_mtl_request_t *mtl_request) =
-  { mca_pml_cm_send_request_completion,
-    mca_pml_cm_send_request_completion,
-    mca_pml_cm_send_request_completion,
-    mca_pml_cm_send_request_completion,
-    mca_pml_cm_send_request_completion } ;
-
 static int
 mca_pml_cm_component_register(void)
 {
@@ -141,7 +131,7 @@ mca_pml_cm_component_init(int* priority,
 
     opal_output_verbose( 10, 0,
                          "in cm pml priority is %d\n", *priority);
-    /* find a useable MTL */
+    /* find a usable MTL */
     ret = ompi_mtl_base_select(enable_progress_threads, enable_mpi_threads, priority);
     if (OMPI_SUCCESS != ret) {
         return NULL;
@@ -151,8 +141,10 @@ mca_pml_cm_component_init(int* priority,
         ompi_pml_cm.super.pml_flags |= MCA_PML_BASE_FLAG_REQUIRE_WORLD;
     }
 
-    /* update our tag / context id max values based on MTL
-       information */
+    if (ompi_mtl->mtl_flags & MCA_MTL_BASE_FLAG_SUPPORTS_EXT_CID) {
+        ompi_pml_cm.super.pml_flags |= MCA_PML_BASE_FLAG_SUPPORTS_EXT_CID;
+    }
+
     ompi_pml_cm.super.pml_max_contextid = ompi_mtl->mtl_max_contextid;
     ompi_pml_cm.super.pml_max_tag = ompi_mtl->mtl_max_tag;
 
diff --git a/ompi/mca/pml/cm/pml_cm_component.h b/ompi/mca/pml/cm/pml_cm_component.h
index bd19eaeeca8..94dc91dd732 100644
--- a/ompi/mca/pml/cm/pml_cm_component.h
+++ b/ompi/mca/pml/cm/pml_cm_component.h
@@ -25,7 +25,7 @@ BEGIN_C_DECLS
 /*
  * PML module functions.
  */
-OMPI_MODULE_DECLSPEC extern mca_pml_base_component_2_1_0_t mca_pml_cm_component;
+OMPI_DECLSPEC extern mca_pml_base_component_2_1_0_t mca_pml_cm_component;
 
 END_C_DECLS
 
diff --git a/ompi/mca/pml/cm/pml_cm_recvreq.h b/ompi/mca/pml/cm/pml_cm_recvreq.h
index 6729cac886e..1c1cca4616d 100644
--- a/ompi/mca/pml/cm/pml_cm_recvreq.h
+++ b/ompi/mca/pml/cm/pml_cm_recvreq.h
@@ -110,7 +110,7 @@ do {                                                                    \
     } else {                                                            \
         ompi_proc = ompi_comm_peer_lookup( comm, src );                 \
     }                                                                   \
-    MCA_PML_CM_SWITCH_CUDA_CONVERTOR_OFF(flags, datatype, count);       \
+    MCA_PML_CM_SWITCH_ACCELERATOR_CONVERTOR_OFF(flags, datatype, count);       \
     opal_convertor_copy_and_prepare_for_recv(                           \
                                   ompi_proc->super.proc_convertor,      \
                                   &(datatype->super),                   \
@@ -138,7 +138,7 @@ do {                                                                    \
     OBJ_RETAIN(comm);                                                   \
     OMPI_DATATYPE_RETAIN(datatype);                                     \
                                                                         \
-    MCA_PML_CM_SWITCH_CUDA_CONVERTOR_OFF(flags, datatype, count);       \
+    MCA_PML_CM_SWITCH_ACCELERATOR_CONVERTOR_OFF(flags, datatype, count);       \
     opal_convertor_copy_and_prepare_for_recv(                           \
         ompi_mpi_local_convertor,                                       \
         &(datatype->super),                                             \
@@ -179,7 +179,7 @@ do {                                                                    \
     } else {                                                            \
         ompi_proc = ompi_comm_peer_lookup( comm, src );                 \
     }                                                                   \
-    MCA_PML_CM_SWITCH_CUDA_CONVERTOR_OFF(flags, datatype, count);       \
+    MCA_PML_CM_SWITCH_ACCELERATOR_CONVERTOR_OFF(flags, datatype, count);       \
     opal_convertor_copy_and_prepare_for_recv(                           \
                                   ompi_proc->super.proc_convertor,      \
                                   &(datatype->super),                   \
@@ -213,7 +213,7 @@ do {                                                                    \
     OBJ_RETAIN(comm);                                                   \
     OMPI_DATATYPE_RETAIN(datatype);                                     \
                                                                         \
-    MCA_PML_CM_SWITCH_CUDA_CONVERTOR_OFF(flags, datatype, count);       \
+    MCA_PML_CM_SWITCH_ACCELERATOR_CONVERTOR_OFF(flags, datatype, count);       \
     opal_convertor_copy_and_prepare_for_recv(                           \
         ompi_mpi_local_convertor,                                       \
         &(datatype->super),                                             \
diff --git a/ompi/mca/pml/cm/pml_cm_request.h b/ompi/mca/pml/cm/pml_cm_request.h
index 8f2f9b6547a..07eaee06be2 100644
--- a/ompi/mca/pml/cm/pml_cm_request.h
+++ b/ompi/mca/pml/cm/pml_cm_request.h
@@ -55,19 +55,15 @@ typedef struct mca_pml_cm_request_t mca_pml_cm_request_t;
 OBJ_CLASS_DECLARATION(mca_pml_cm_request_t);
 
 /*
- * Avoid CUDA convertor inits only for contiguous memory and if indicated by
- * the MTL. For non-contiguous memory, do not skip CUDA convertor init phases.
+ * Avoid accelerator convertor inits only for contiguous memory and if indicated by
+ * the MTL. For non-contiguous memory, do not skip accelerator convertor init phases.
  */
-#if OPAL_CUDA_SUPPORT
-#define MCA_PML_CM_SWITCH_CUDA_CONVERTOR_OFF(flags, datatype, count)            \
+#define MCA_PML_CM_SWITCH_ACCELERATOR_CONVERTOR_OFF(flags, datatype, count)            \
     {                                                                           \
         if (opal_datatype_is_contiguous_memory_layout(&datatype->super, count)  \
-            && (ompi_mtl->mtl_flags & MCA_MTL_BASE_FLAG_CUDA_INIT_DISABLE)) {   \
-            flags |= CONVERTOR_SKIP_CUDA_INIT;                                  \
+            && (ompi_mtl->mtl_flags & MCA_MTL_BASE_FLAG_ACCELERATOR_INIT_DISABLE)) {   \
+            flags |= CONVERTOR_SKIP_ACCELERATOR_INIT;                                  \
         }                                                                       \
     }
-#else
-#define MCA_PML_CM_SWITCH_CUDA_CONVERTOR_OFF(flags, datatype, count)
-#endif
 
 #endif
diff --git a/ompi/mca/pml/cm/pml_cm_sendreq.h b/ompi/mca/pml/cm/pml_cm_sendreq.h
index c229f3e6899..4872943c05a 100644
--- a/ompi/mca/pml/cm/pml_cm_sendreq.h
+++ b/ompi/mca/pml/cm/pml_cm_sendreq.h
@@ -15,6 +15,8 @@
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2017      Intel, Inc. All rights reserved
+ * Copyright (c) 2021      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -31,6 +33,7 @@
 #include "ompi/mca/pml/pml.h"
 #include "ompi/mca/mtl/mtl.h"
 #include "opal/prefetch.h"
+#include "opal/runtime/opal_params.h"
 
 struct mca_pml_cm_send_request_t {
     mca_pml_cm_request_t req_base;
@@ -133,7 +136,7 @@ do {                                                                    \
     OMPI_DATATYPE_RETAIN(datatype);                                     \
     (req_send)->req_base.req_comm = comm;                               \
     (req_send)->req_base.req_datatype = datatype;                       \
-    MCA_PML_CM_SWITCH_CUDA_CONVERTOR_OFF(flags, datatype, count);       \
+    MCA_PML_CM_SWITCH_ACCELERATOR_CONVERTOR_OFF(flags, datatype, count);       \
     opal_convertor_copy_and_prepare_for_send(                           \
                                              ompi_proc->super.proc_convertor, \
                                              &(datatype->super),        \
@@ -164,7 +167,7 @@ do {                                                                    \
     OMPI_DATATYPE_RETAIN(datatype);                                     \
     (req_send)->req_base.req_comm = comm;                               \
     (req_send)->req_base.req_datatype = datatype;                       \
-    MCA_PML_CM_SWITCH_CUDA_CONVERTOR_OFF(flags, datatype, count);       \
+    MCA_PML_CM_SWITCH_ACCELERATOR_CONVERTOR_OFF(flags, datatype, count);       \
     opal_convertor_copy_and_prepare_for_send(                           \
         ompi_mpi_local_convertor,                                       \
         &(datatype->super),                                             \
@@ -197,7 +200,7 @@ do {                                                                    \
     OMPI_DATATYPE_RETAIN(datatype);                                     \
     (req_send)->req_base.req_comm = comm;                               \
     (req_send)->req_base.req_datatype = datatype;                       \
-    MCA_PML_CM_SWITCH_CUDA_CONVERTOR_OFF(flags, datatype, count);       \
+    MCA_PML_CM_SWITCH_ACCELERATOR_CONVERTOR_OFF(flags, datatype, count);       \
     opal_convertor_copy_and_prepare_for_send(                           \
                                              ompi_proc->super.proc_convertor, \
                                              &(datatype->super),        \
@@ -236,22 +239,16 @@ do {                                                                    \
             ompi_mpi_local_convertor->flags;                            \
         (req_send)->req_base.req_convertor.master     =                 \
             ompi_mpi_local_convertor->master;                           \
-        (req_send)->req_base.req_convertor.local_size =                 \
-            count * datatype->super.size;                               \
-        (req_send)->req_base.req_convertor.pBaseBuf   =                 \
-            (unsigned char*)buf + datatype->super.true_lb;              \
-        (req_send)->req_base.req_convertor.count      = count;          \
-        (req_send)->req_base.req_convertor.pDesc      = &datatype->super; \
-        /* Switches off CUDA detection if                               \
-         MTL set MCA_MTL_BASE_FLAG_CUDA_INIT_DISABLE during init */     \
-        MCA_PML_CM_SWITCH_CUDA_CONVERTOR_OFF(flags, datatype, count);   \
+        /* Switches off device buffer  detection if                 \
+         MTL set MCA_MTL_BASE_FLAG_ACCELERATOR_INIT_DISABLE during init */     \
+        MCA_PML_CM_SWITCH_ACCELERATOR_CONVERTOR_OFF(flags, datatype, count);   \
         (req_send)->req_base.req_convertor.flags |= flags;              \
-        /* Sets CONVERTOR_CUDA flag if CUDA buffer */                   \
-        opal_convertor_prepare_for_send(                                \
-            &req_send->req_base.req_convertor,                          \
-            &datatype->super, count, buf );                             \
+        /* Sets CONVERTOR_ACCELERATOR flag if device buffer */                   \
+        opal_convertor_prepare_for_send(                            \
+            &req_send->req_base.req_convertor,                      \
+            &datatype->super, count, (unsigned char*)buf); \
     } else {                                                            \
-        MCA_PML_CM_SWITCH_CUDA_CONVERTOR_OFF(flags, datatype, count);   \
+        MCA_PML_CM_SWITCH_ACCELERATOR_CONVERTOR_OFF(flags, datatype, count);   \
         opal_convertor_copy_and_prepare_for_send(                       \
             ompi_mpi_local_convertor,                                   \
             &(datatype->super),                                         \
@@ -378,6 +375,10 @@ do {                                                                    \
             iov.iov_base = (IOVBASE_TYPE*)sendreq->req_buff;            \
             max_data = iov.iov_len = sendreq->req_count;                \
             iov_count = 1;                                              \
+            opal_convertor_prepare_for_send( &sendreq->req_send.req_base.req_convertor, \
+                                             &sendreq->req_send.req_base.req_datatype->super,   \
+                                             max_data,                  \
+                                             sendreq->req_addr);        \
             opal_convertor_pack( &sendreq->req_send.req_base.req_convertor, \
                                  &iov,                                  \
                                  &iov_count,                            \
@@ -458,7 +459,7 @@ do {
 #define MCA_PML_CM_HVY_SEND_REQUEST_RETURN(sendreq)                     \
     {                                                                   \
         /*  Let the base handle the reference counts */                 \
-        OMPI_DATATYPE_RETAIN(sendreq->req_send.req_base.req_datatype);  \
+        OMPI_DATATYPE_RELEASE(sendreq->req_send.req_base.req_datatype);  \
         OBJ_RELEASE(sendreq->req_send.req_base.req_comm);               \
         OMPI_REQUEST_FINI(&sendreq->req_send.req_base.req_ompi);        \
         opal_convertor_cleanup( &(sendreq->req_send.req_base.req_convertor) ); \
@@ -495,7 +496,7 @@ do {                                                                         \
 #define MCA_PML_CM_THIN_SEND_REQUEST_RETURN(sendreq)                    \
     {                                                                   \
         /*  Let the base handle the reference counts */                 \
-        OMPI_DATATYPE_RETAIN(sendreq->req_send.req_base.req_datatype);  \
+        OMPI_DATATYPE_RELEASE(sendreq->req_send.req_base.req_datatype);  \
         OBJ_RELEASE(sendreq->req_send.req_base.req_comm);               \
         OMPI_REQUEST_FINI(&sendreq->req_send.req_base.req_ompi);        \
         opal_convertor_cleanup( &(sendreq->req_send.req_base.req_convertor) ); \
diff --git a/ompi/mca/pml/monitoring/configure.m4 b/ompi/mca/pml/monitoring/configure.m4
index 27815f22957..ffc2bcf5fc4 100644
--- a/ompi/mca/pml/monitoring/configure.m4
+++ b/ompi/mca/pml/monitoring/configure.m4
@@ -15,10 +15,7 @@
 # ------------------------------------------------
 AC_DEFUN([MCA_ompi_pml_monitoring_CONFIG],[
     AC_CONFIG_FILES([ompi/mca/pml/monitoring/Makefile])
-
-    AS_IF([test "$MCA_BUILD_ompi_common_monitoring_DSO_TRUE" = ''],
-          [$1],
-          [$2])
+    [$1]
 ])dnl
 
 
diff --git a/ompi/mca/pml/monitoring/pml_monitoring_irecv.c b/ompi/mca/pml/monitoring/pml_monitoring_irecv.c
index 3bc07f7e91a..eaf8e07356b 100644
--- a/ompi/mca/pml/monitoring/pml_monitoring_irecv.c
+++ b/ompi/mca/pml/monitoring/pml_monitoring_irecv.c
@@ -16,7 +16,7 @@
 #include "pml_monitoring.h"
 
 
-/* EJ: loging is done on the sender. Nothing to do here */
+/* EJ: logging is done on the sender. Nothing to do here */
 
 int mca_pml_monitoring_irecv_init(void *buf,
                                   size_t count,
diff --git a/ompi/mca/pml/monitoring/pml_monitoring_isend.c b/ompi/mca/pml/monitoring/pml_monitoring_isend.c
index 9e64b1e83f7..c1ab7908441 100644
--- a/ompi/mca/pml/monitoring/pml_monitoring_isend.c
+++ b/ompi/mca/pml/monitoring/pml_monitoring_isend.c
@@ -40,7 +40,7 @@ int mca_pml_monitoring_isend(const void *buf,
     int world_rank;
     /**
      * If this fails the destination is not part of my MPI_COM_WORLD
-     * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank
+     * Lookup its name in the rank hashtable to get its MPI_COMM_WORLD rank
      */
     if(OPAL_SUCCESS == mca_common_monitoring_get_world_rank(dst, comm->c_remote_group, &world_rank)) {
         size_t type_size, data_size;
diff --git a/ompi/mca/pml/monitoring/pml_monitoring_start.c b/ompi/mca/pml/monitoring/pml_monitoring_start.c
index c275d298155..f10ebbc7d48 100644
--- a/ompi/mca/pml/monitoring/pml_monitoring_start.c
+++ b/ompi/mca/pml/monitoring/pml_monitoring_start.c
@@ -16,7 +16,7 @@
 #include "pml_monitoring.h"
 #include "ompi/mca/pml/base/pml_base_request.h"
 
-/* manage persistant requests*/
+/* manage persistent requests*/
 int mca_pml_monitoring_start(size_t count,
                              ompi_request_t** requests)
 {
diff --git a/ompi/mca/pml/ob1/Makefile.am b/ompi/mca/pml/ob1/Makefile.am
index 3364d001b1e..38487f5a2e6 100644
--- a/ompi/mca/pml/ob1/Makefile.am
+++ b/ompi/mca/pml/ob1/Makefile.am
@@ -13,6 +13,8 @@
 # Copyright (c) 2009-2014 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2012      NVIDIA Corporation.  All rights reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -20,11 +22,10 @@
 # $HEADER$
 #
 
-dist_ompidata_DATA = \
+EXTRA_DIST = \
+        post_configure.sh \
 	help-mpi-pml-ob1.txt
 
-EXTRA_DIST = post_configure.sh
-
 ob1_sources  = \
 	pml_ob1.c \
 	pml_ob1.h \
@@ -48,6 +49,8 @@ ob1_sources  = \
 	pml_ob1_sendreq.c \
 	pml_ob1_sendreq.h \
 	pml_ob1_start.c \
+	pml_ob1_accelerator.h \
+	pml_ob1_accelerator.c \
 	custommatch/pml_ob1_custom_match.h \
 	custommatch/pml_ob1_custom_match_arrays.h \
 	custommatch/pml_ob1_custom_match_vectors.h \
@@ -56,12 +59,6 @@ ob1_sources  = \
 	custommatch/pml_ob1_custom_match_fuzzy512-short.h \
 	custommatch/pml_ob1_custom_match_fuzzy512-word.h
 
-# If we have CUDA support requested, build the CUDA file also
-if OPAL_cuda_support
-ob1_sources += \
-    pml_ob1_cuda.c
-endif
-
 if MCA_BUILD_ompi_pml_ob1_DSO
 component_noinst =
 component_install = mca_pml_ob1.la
@@ -75,10 +72,7 @@ mcacomponent_LTLIBRARIES = $(component_install)
 mca_pml_ob1_la_SOURCES = $(ob1_sources)
 mca_pml_ob1_la_LDFLAGS = -module -avoid-version
 
-if OPAL_cuda_support
-mca_pml_ob1_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
-    $(OMPI_TOP_BUILDDIR)/opal/mca/common/cuda/lib@OPAL_LIB_PREFIX@mca_common_cuda.la
-endif
+mca_pml_ob1_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la
 
 noinst_LTLIBRARIES = $(component_noinst)
 libmca_pml_ob1_la_SOURCES = $(ob1_sources)
diff --git a/ompi/mca/pml/ob1/custommatch/pml_ob1_custom_match_arrays.h b/ompi/mca/pml/ob1/custommatch/pml_ob1_custom_match_arrays.h
index 3e59a5a35e5..482b7129ab0 100644
--- a/ompi/mca/pml/ob1/custommatch/pml_ob1_custom_match_arrays.h
+++ b/ompi/mca/pml/ob1/custommatch/pml_ob1_custom_match_arrays.h
@@ -297,7 +297,7 @@ static inline void custom_match_print(custom_match_prq* list)
     custom_match_prq_node* elem;
     int i = 0;
     int j = 0;
-    printf("Elements in the list (this is currenly only partialy implemented):\n");
+    printf("Elements in the list (this is currently only partially implemented):\n");
     for(elem = list->head; elem; elem = elem->next)
     {
         printf("This is the %d linked list element\n", ++i);
diff --git a/ompi/mca/pml/ob1/custommatch/pml_ob1_custom_match_fuzzy512-byte.h b/ompi/mca/pml/ob1/custommatch/pml_ob1_custom_match_fuzzy512-byte.h
index fca65806f3f..4b1c133fafb 100644
--- a/ompi/mca/pml/ob1/custommatch/pml_ob1_custom_match_fuzzy512-byte.h
+++ b/ompi/mca/pml/ob1/custommatch/pml_ob1_custom_match_fuzzy512-byte.h
@@ -464,7 +464,7 @@ static inline void custom_match_umq_append(custom_match_umq* list, int tag, int
 {
     int8_t key = source ^ tag;
 #if CUSTOM_MATCH_DEBUG
-    ((int8_t*)&key)[3] = (int8_t) tag; // MGFD TODO verify this set higer order bits...
+    ((int8_t*)&key)[3] = (int8_t) tag; // MGFD TODO verify this set higher order bits...
     mca_pml_ob1_recv_frag_t *req = (mca_pml_ob1_recv_frag_t *)payload;
     printf("custom_match_umq_append list: %x key: %x payload: %x tag: %d src: %d\n", list, key, payload, req->hdr.hdr_match.hdr_tag, req->hdr.hdr_match.hdr_src);
 #endif
diff --git a/ompi/mca/pml/ob1/custommatch/pml_ob1_custom_match_fuzzy512-short.h b/ompi/mca/pml/ob1/custommatch/pml_ob1_custom_match_fuzzy512-short.h
index 73d47debf1b..af22c9daef4 100644
--- a/ompi/mca/pml/ob1/custommatch/pml_ob1_custom_match_fuzzy512-short.h
+++ b/ompi/mca/pml/ob1/custommatch/pml_ob1_custom_match_fuzzy512-short.h
@@ -455,7 +455,7 @@ static inline void custom_match_umq_append(custom_match_umq* list, int tag, int
 {
     int16_t key = source ^ tag;
 #if CUSTOM_MATCH_DEBUG_VERBOSE
-    ((int8_t*)&key)[3] = (int8_t) tag; // MGFD TODO verify this set higer order bits...
+    ((int8_t*)&key)[3] = (int8_t) tag; // MGFD TODO verify this set higher order bits...
     mca_pml_ob1_recv_frag_t *req = (mca_pml_ob1_recv_frag_t *)payload;
     printf("custom_match_umq_append list: %x key: %x payload: %x tag: %d src: %d\n", list, key, payload, req->hdr.hdr_match.hdr_tag, req->hdr.hdr_match.hdr_src);
 #endif
diff --git a/ompi/mca/pml/ob1/custommatch/pml_ob1_custom_match_fuzzy512-word.h b/ompi/mca/pml/ob1/custommatch/pml_ob1_custom_match_fuzzy512-word.h
index 35ff477f2a5..6fc45344aa4 100644
--- a/ompi/mca/pml/ob1/custommatch/pml_ob1_custom_match_fuzzy512-word.h
+++ b/ompi/mca/pml/ob1/custommatch/pml_ob1_custom_match_fuzzy512-word.h
@@ -97,7 +97,7 @@ static inline void* custom_match_prq_find_verify(custom_match_prq* list, int tag
     custom_match_prq_node* elem = list->head;
     int i;
     int32_t key = peer;
-    ((int8_t*)&key)[3] = (int8_t) tag; // MGFD TODO verify this set higer order bits...
+    ((int8_t*)&key)[3] = (int8_t) tag; // MGFD TODO verify this set higher order bits...
     __m512i search = _mm512_set1_epi32(key);
     while(elem)
     {
@@ -134,7 +134,7 @@ static inline void* custom_match_prq_find_dequeue_verify(custom_match_prq* list,
     custom_match_prq_node* elem = list->head;
     int i;
     int32_t key = peer;
-    ((int8_t*)&key)[3] = (int8_t) tag; // MGFD TODO verify this set higer order bits...
+    ((int8_t*)&key)[3] = (int8_t) tag; // MGFD TODO verify this set higher order bits...
     __m512i search = _mm512_set1_epi32(key);
     while(elem)
     {
@@ -384,7 +384,7 @@ static inline void* custom_match_umq_find_verify_hold(custom_match_umq* list, in
     custom_match_umq_node* elem = list->head;
     int i;
     int32_t key = peer;
-    ((int8_t*)&key)[3] = (int8_t) tag; // MGFD TODO verify this set higer order bits...
+    ((int8_t*)&key)[3] = (int8_t) tag; // MGFD TODO verify this set higher order bits...
     __m512i search = _mm512_set1_epi32(key);
 
     int32_t mask = ~0;
@@ -472,7 +472,7 @@ static inline void custom_match_umq_remove_hold(custom_match_umq* list, custom_m
 static inline void custom_match_umq_append(custom_match_umq* list, int tag, int source, void* payload)
 {
     int32_t key = source;
-    ((int8_t*)&key)[3] = (int8_t) tag; // MGFD TODO verify this set higer order bits...
+    ((int8_t*)&key)[3] = (int8_t) tag; // MGFD TODO verify this set higher order bits...
 #if CUSTOM_MATCH_DEBUG_VERBOSE
     mca_pml_ob1_recv_frag_t *req = (mca_pml_ob1_recv_frag_t *)payload;
     printf("custom_match_umq_append list: %x key: %x payload: %x tag: %d src: %d\n", list, key, payload, req->hdr.hdr_match.hdr_tag, req->hdr.hdr_match.hdr_src);
diff --git a/ompi/mca/pml/ob1/custommatch/pml_ob1_custom_match_linkedlist.h b/ompi/mca/pml/ob1/custommatch/pml_ob1_custom_match_linkedlist.h
index 042959c9c37..e95ff618f6b 100644
--- a/ompi/mca/pml/ob1/custommatch/pml_ob1_custom_match_linkedlist.h
+++ b/ompi/mca/pml/ob1/custommatch/pml_ob1_custom_match_linkedlist.h
@@ -170,7 +170,7 @@ static inline void custom_match_prq_append(custom_match_prq* list, void* payload
     }
     mca_pml_base_request_t *req = (mca_pml_base_request_t *)payload;
 #if CUSTOM_MATCH_DEBUG_VERBOSE
-    printf("custom_match_prq_append list: %x tag: %x soruce: %x tag: %x peer: %x\n", list, tag, source, req->req_tag, req->req_peer);
+    printf("custom_match_prq_append list: %x tag: %x source: %x tag: %x peer: %x\n", list, tag, source, req->req_tag, req->req_peer);
 #endif
     int i;
     custom_match_prq_node* elem;
@@ -261,7 +261,7 @@ static inline void custom_match_print(custom_match_prq* list)
     custom_match_prq_node* elem;
     int i = 0;
     int j = 0;
-    printf("Elements in the list (this is currenly only partialy implemented):\n");
+    printf("Elements in the list (this is currently only partially implemented):\n");
     for(elem = list->head; elem; elem = elem->next)
     {
         printf("This is the %d linked list element\n", ++i);
diff --git a/ompi/mca/pml/ob1/custommatch/pml_ob1_custom_match_vectors.h b/ompi/mca/pml/ob1/custommatch/pml_ob1_custom_match_vectors.h
index bde8e881c32..6b4e94ca210 100644
--- a/ompi/mca/pml/ob1/custommatch/pml_ob1_custom_match_vectors.h
+++ b/ompi/mca/pml/ob1/custommatch/pml_ob1_custom_match_vectors.h
@@ -315,7 +315,7 @@ static inline void custom_match_print(custom_match_prq* list)
     custom_match_prq_node* elem;
     int i = 0;
     int j = 0;
-    printf("Elements in the list (this is currenly only partialy implemented):\n");
+    printf("Elements in the list (this is currently only partially implemented):\n");
     for(elem = list->head; elem; elem = elem->next)
     {
         printf("This is the %d linked list element\n", ++i);
@@ -476,7 +476,7 @@ static inline void custom_match_umq_append(custom_match_umq* list, int tag, int
 {
 #if CUSTOM_MATCH_DEBUG_VERBOSE
     int32_t key = source;
-    ((int8_t*)&key)[3] = (int8_t) tag; // MGFD TODO verify this set higer order bits...
+    ((int8_t*)&key)[3] = (int8_t) tag; // MGFD TODO verify this set higher order bits...
 #endif
 #if CUSTOM_MATCH_DEBUG_VERBOSE
     mca_pml_ob1_recv_frag_t *req = (mca_pml_ob1_recv_frag_t *)payload;
diff --git a/ompi/mca/pml/ob1/pml_ob1.c b/ompi/mca/pml/ob1/pml_ob1.c
index a25caf46d35..e0516d16fe0 100644
--- a/ompi/mca/pml/ob1/pml_ob1.c
+++ b/ompi/mca/pml/ob1/pml_ob1.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
+ * Copyright (c) 2004-2022 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -22,6 +22,11 @@
  *                         All rights reserved.
  * Copyright (c) 2018 IBM Corporation. All rights reserved.
  * Copyright (c) 2019-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2018-2021 Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -57,43 +62,44 @@
 #include "pml_ob1_sendreq.h"
 #include "pml_ob1_recvreq.h"
 #include "pml_ob1_rdmafrag.h"
+#include "pml_ob1_accelerator.h"
 
 mca_pml_ob1_t mca_pml_ob1 = {
     {
-        mca_pml_ob1_add_procs,
-        mca_pml_ob1_del_procs,
-        mca_pml_ob1_enable,
-        NULL,  /* mca_pml_ob1_progress, */
-        mca_pml_ob1_add_comm,
-        mca_pml_ob1_del_comm,
+        .pml_add_procs      = mca_pml_ob1_add_procs,
+        .pml_del_procs      = mca_pml_ob1_del_procs,
+        .pml_enable         = mca_pml_ob1_enable,
+        .pml_progress       = NULL,  /* mca_pml_ob1_progress, */
+        .pml_add_comm       = mca_pml_ob1_add_comm,
+        .pml_del_comm       = mca_pml_ob1_del_comm,
 #if OPAL_ENABLE_FT_MPI
-        mca_pml_ob1_revoke_comm,
+        .pml_revoke_comm    = mca_pml_ob1_revoke_comm,
+#else
+        .pml_revoke_comm    = NULL,
 #endif
-        mca_pml_ob1_irecv_init,
-        mca_pml_ob1_irecv,
-        mca_pml_ob1_recv,
-        mca_pml_ob1_isend_init,
-        mca_pml_ob1_isend,
-        mca_pml_ob1_send,
-        mca_pml_ob1_iprobe,
-        mca_pml_ob1_probe,
-        mca_pml_ob1_start,
-        mca_pml_ob1_improbe,
-        mca_pml_ob1_mprobe,
-        mca_pml_ob1_imrecv,
-        mca_pml_ob1_mrecv,
-        mca_pml_ob1_dump,
-        65535,
-        INT_MAX,
-        0 /* flags */
+        .pml_irecv_init     = mca_pml_ob1_irecv_init,
+        .pml_irecv          = mca_pml_ob1_irecv,
+        .pml_recv           = mca_pml_ob1_recv,
+        .pml_isend_init     = mca_pml_ob1_isend_init,
+        .pml_isend          = mca_pml_ob1_isend,
+        .pml_send           = mca_pml_ob1_send,
+        .pml_iprobe         = mca_pml_ob1_iprobe,
+        .pml_probe          = mca_pml_ob1_probe,
+        .pml_start          = mca_pml_ob1_start,
+        .pml_improbe        = mca_pml_ob1_improbe,
+        .pml_mprobe         = mca_pml_ob1_mprobe,
+        .pml_imrecv         = mca_pml_ob1_imrecv,
+        .pml_mrecv          = mca_pml_ob1_mrecv,
+        .pml_dump           = mca_pml_ob1_dump,
+        .pml_max_contextid  = 65535,
+        .pml_max_tag        = INT_MAX,
+        .pml_flags          = 0 /* flags */
     }
 };
 
-#if OPAL_CUDA_SUPPORT
-extern void mca_pml_ob1_cuda_add_ipc_support(struct mca_btl_base_module_t* btl,
+extern void mca_pml_ob1_accelerator_add_ipc_support(struct mca_btl_base_module_t* btl,
                                              int32_t flags, ompi_proc_t* errproc,
                                              char* btlinfo);
-#endif /* OPAL_CUDA_SUPPORT */
 
 void mca_pml_ob1_error_handler( struct mca_btl_base_module_t* btl,
                                 int32_t flags, opal_proc_t* errproc,
@@ -192,10 +198,50 @@ int mca_pml_ob1_enable(bool enable)
                           mca_pml_ob1.free_list_inc,
                           NULL, 0, NULL, NULL, NULL);
 
+    mca_pml_ob1.accelerator_enabled = (0 == mca_pml_ob1_accelerator_init()) ? true : false;
+
     mca_pml_ob1.enabled = true;
+
     return OMPI_SUCCESS;
 }
 
+static const char*
+mca_pml_ob1_set_allow_overtake(opal_infosubscriber_t* obj,
+                               const char* key,
+                               const char* value)
+{
+    ompi_communicator_t *ompi_comm = (ompi_communicator_t *) obj;
+    bool allow_overtake_was_set = OMPI_COMM_CHECK_ASSERT_ALLOW_OVERTAKE(ompi_comm);
+
+    /* As we keep the out-of-sequence messages ordered by their sequence, as a receiver we
+     * can just move the previously considered out-of-order messages into the unexpected queue,
+     * and we maintain some form of logical consistency with the message order.
+     */
+    if (opal_str_to_bool(value)) {
+        if (!allow_overtake_was_set) {
+            ompi_comm->c_flags |= OMPI_COMM_ASSERT_ALLOW_OVERTAKE;
+            mca_pml_ob1_merge_cant_match(ompi_comm);
+        }
+        return "true";
+    }
+    if (allow_overtake_was_set) {
+        /* However, in the case we are trying to turn off allow_overtake, it is not clear what
+         * should be done with the previous messages that are pending on our peers, nor with
+         * the messages currently in the network. Similarly, if one process turns off allow
+         * overtake, before any potential sender start sending valid sequence numbers there
+         * is no way to order the messages in a sensible order.
+         * The possible solution is cumbersome, it would force a network quiescence followed by
+         * a synchronization of all processes in the communicator, and then all peers will
+         * start sending messages starting with sequence number 0.
+         * A lot of code for minimal benefit, especially taking in account that the MPI standard
+         * does not define this. Instead, refuse to disable allow overtake, and at least the
+         * user has the opportunity to check if we accepted to change it.
+         */
+        return "true";
+    }
+    return "false";
+}
+
 int mca_pml_ob1_add_comm(ompi_communicator_t* comm)
 {
     /* allocate pml specific comm data */
@@ -209,24 +255,43 @@ int mca_pml_ob1_add_comm(ompi_communicator_t* comm)
     }
 
     /* should never happen, but it was, so check */
-    if (comm->c_contextid > mca_pml_ob1.super.pml_max_contextid) {
+    if (comm->c_index > mca_pml_ob1.super.pml_max_contextid) {
         OBJ_RELEASE(pml_comm);
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
 
     ompi_comm_assert_subscribe (comm, OMPI_COMM_ASSERT_NO_ANY_SOURCE);
-    ompi_comm_assert_subscribe (comm, OMPI_COMM_ASSERT_ALLOW_OVERTAKE);
 
     mca_pml_ob1_comm_init_size(pml_comm, comm->c_remote_group->grp_proc_count);
     comm->c_pml_comm = pml_comm;
 
+    /* Register the subscriber alert for the mpi_assert_allow_overtaking info. */
+    opal_infosubscribe_subscribe (&comm->super, "mpi_assert_allow_overtaking",
+                                  "false", mca_pml_ob1_set_allow_overtake);
+
     /* Grab all related messages from the non_existing_communicator pending queue */
     OPAL_LIST_FOREACH_SAFE(frag, next_frag, &mca_pml_ob1.non_existing_communicator_pending, mca_pml_ob1_recv_frag_t) {
         hdr = &frag->hdr.hdr_match;
 
+        if (MCA_PML_OB1_HDR_TYPE_CID == frag->hdr.hdr_common.hdr_type) {
+            if (!ompi_comm_cid_compare (comm, frag->hdr.hdr_cid.hdr_cid)) {
+                continue;
+            }
+
+            /* handle this CID*/
+            mca_pml_ob1_handle_cid (comm, frag->hdr.hdr_ext_match.hdr_match.hdr_src, &frag->hdr.hdr_cid);
+
+            hdr = &frag->hdr.hdr_ext_match.hdr_match;
+            hdr->hdr_ctx = comm->c_index;
+
+            /* NTH: this is ok because the pointer that will be freed is stored in frag->addr[] */
+            frag->segments[0].seg_addr.pval = (void *)((uintptr_t) frag->segments[0].seg_addr.pval + sizeof (frag->hdr.hdr_cid));
+        }
+
         /* Is this fragment for the current communicator ? */
-        if( frag->hdr.hdr_match.hdr_ctx != comm->c_contextid )
+        if (hdr->hdr_ctx != comm->c_index) {
             continue;
+        }
 
         /* As we now know we work on a fragment for this communicator
          * we should remove it from the
@@ -275,20 +340,20 @@ int mca_pml_ob1_add_comm(ompi_communicator_t* comm)
             PERUSE_TRACE_MSG_EVENT(PERUSE_COMM_MSG_INSERT_IN_UNEX_Q, comm,
                                    hdr->hdr_src, hdr->hdr_tag, PERUSE_RECV);
             /* And now the ugly part. As some fragments can be inserted in the cant_match list,
-             * every time we succesfully add a fragment in the unexpected list we have to make
+             * every time we successfully add a fragment in the unexpected list we have to make
              * sure the next one is not in the cant_match. Otherwise, we will endup in a deadlock
              * situation as the cant_match is only checked when a new fragment is received from
              * the network.
              */
             if( NULL != pml_proc->frags_cant_match ) {
-                frag = check_cantmatch_for_match(pml_proc);
+                frag = ompi_pml_ob1_check_cantmatch_for_match(pml_proc);
                 if( NULL != frag ) {
                     hdr = &frag->hdr.hdr_match;
                     goto add_fragment_to_unexpected;
                 }
             }
         } else {
-            append_frag_to_ordered_list(&pml_proc->frags_cant_match, frag,
+            ompi_pml_ob1_append_frag_to_ordered_list(&pml_proc->frags_cant_match, frag,
                                         pml_proc->expected_sequence);
         }
     }
@@ -318,11 +383,6 @@ int mca_pml_ob1_add_procs(ompi_proc_t** procs, size_t nprocs)
     if(nprocs == 0)
         return OMPI_SUCCESS;
 
-    OBJ_CONSTRUCT(&reachable, opal_bitmap_t);
-    rc = opal_bitmap_init(&reachable, (int)nprocs);
-    if(OMPI_SUCCESS != rc)
-        return rc;
-
     /* make sure remote procs are using the same PML as us */
     if (OMPI_SUCCESS != (rc = mca_pml_base_pml_check_selected("ob1",
                                                               procs,
@@ -330,11 +390,21 @@ int mca_pml_ob1_add_procs(ompi_proc_t** procs, size_t nprocs)
         return rc;
     }
 
-    rc = mca_bml.bml_add_procs( nprocs,
-                                procs,
-                                &reachable );
-    if(OMPI_SUCCESS != rc)
-        goto cleanup_and_return;
+    if (NULL == mca_bml.bml_add_procs) {
+        return OMPI_ERR_UNREACH;
+    }
+
+    OBJ_CONSTRUCT(&reachable, opal_bitmap_t);
+    rc = opal_bitmap_init(&reachable, (int)nprocs);
+    if (OMPI_SUCCESS != rc) {
+        return rc;
+    }
+
+    rc = mca_bml.bml_add_procs (nprocs, procs, &reachable);
+    OBJ_DESTRUCT(&reachable);
+    if (OMPI_SUCCESS != rc) {
+        return rc;
+    }
 
     /* Check that values supplied by all initialized btls will work
        for us.  Note that this is the list of all initialized BTLs,
@@ -358,48 +428,45 @@ int mca_pml_ob1_add_procs(ompi_proc_t** procs, size_t nprocs)
                            sm->btl_component->btl_version.mca_component_name,
                            sizeof(mca_pml_ob1_hdr_t),
                            sm->btl_component->btl_version.mca_component_name);
-            rc = OMPI_ERR_BAD_PARAM;
-            goto cleanup_and_return;
+            return OMPI_ERR_BAD_PARAM;
         }
 #if OPAL_CUDA_GDR_SUPPORT
         /* If size is SIZE_MAX, then we know we want to set this to the minimum possible
          * value which is the size of the PML header. */
-        if (SIZE_MAX == sm->btl_module->btl_cuda_eager_limit) {
-            sm->btl_module->btl_cuda_eager_limit = sizeof(mca_pml_ob1_hdr_t);
+        if (SIZE_MAX == sm->btl_module->btl_accelerator_eager_limit) {
+            sm->btl_module->btl_accelerator_eager_limit = sizeof(mca_pml_ob1_hdr_t);
         }
         /* If size is 0, then this value is unused.  If it is non-zero then do some
          * extra checking of it. */
-        if (0 != sm->btl_module->btl_cuda_eager_limit) {
-            if (sm->btl_module->btl_cuda_eager_limit < sizeof(mca_pml_ob1_hdr_t)) {
+        if (0 != sm->btl_module->btl_accelerator_eager_limit) {
+            if (sm->btl_module->btl_accelerator_eager_limit < sizeof(mca_pml_ob1_hdr_t)) {
                 opal_show_help("help-mpi-pml-ob1.txt", "cuda_eager_limit_too_small",
                                true,
                                sm->btl_component->btl_version.mca_component_name,
                                ompi_process_info.nodename,
                                sm->btl_component->btl_version.mca_component_name,
-                               sm->btl_module->btl_cuda_eager_limit,
+                               sm->btl_module->btl_accelerator_eager_limit,
                                sm->btl_component->btl_version.mca_component_name,
                                sizeof(mca_pml_ob1_hdr_t),
                                sm->btl_component->btl_version.mca_component_name);
-                rc = OMPI_ERR_BAD_PARAM;
-                goto cleanup_and_return;
+                return OMPI_ERR_BAD_PARAM;
             }
         }
-        if (0 == sm->btl_module->btl_cuda_rdma_limit) {
+        if (0 == sm->btl_module->btl_accelerator_rdma_limit) {
             /* All is fine.  0 means to ignore value so set to SIZE_MAX */
-            sm->btl_module->btl_cuda_rdma_limit = SIZE_MAX;
+            sm->btl_module->btl_accelerator_rdma_limit = SIZE_MAX;
         } else {
-            if (sm->btl_module->btl_cuda_rdma_limit < sm->btl_module->btl_cuda_eager_limit) {
+            if (sm->btl_module->btl_accelerator_rdma_limit < sm->btl_module->btl_accelerator_eager_limit) {
                 opal_show_help("help-mpi-pml-ob1.txt", "cuda_rdma_limit_too_small",
                                true,
                                sm->btl_component->btl_version.mca_component_name,
                                ompi_process_info.nodename,
                                sm->btl_component->btl_version.mca_component_name,
-                               sm->btl_module->btl_cuda_rdma_limit,
+                               sm->btl_module->btl_accelerator_rdma_limit,
                                sm->btl_component->btl_version.mca_component_name,
-                               sm->btl_module->btl_cuda_eager_limit,
+                               sm->btl_module->btl_accelerator_eager_limit,
                                sm->btl_component->btl_version.mca_component_name);
-                rc = OMPI_ERR_BAD_PARAM;
-                goto cleanup_and_return;
+                return OMPI_ERR_BAD_PARAM;
             }
         }
 #endif /* OPAL_CUDA_GDR_SUPPORT */
@@ -410,54 +477,61 @@ int mca_pml_ob1_add_procs(ompi_proc_t** procs, size_t nprocs)
     rc = mca_bml.bml_register( MCA_PML_OB1_HDR_TYPE_MATCH,
                                mca_pml_ob1_recv_frag_callback_match,
                                NULL );
-    if(OMPI_SUCCESS != rc)
-        goto cleanup_and_return;
+    if (OMPI_SUCCESS != rc) {
+        return rc;
+    }
 
     rc = mca_bml.bml_register( MCA_PML_OB1_HDR_TYPE_RNDV,
                                mca_pml_ob1_recv_frag_callback_rndv,
                                NULL );
-    if(OMPI_SUCCESS != rc)
-        goto cleanup_and_return;
+    if (OMPI_SUCCESS != rc) {
+        return rc;
+    }
 
     rc = mca_bml.bml_register( MCA_PML_OB1_HDR_TYPE_RGET,
                                mca_pml_ob1_recv_frag_callback_rget,
                                NULL );
-    if(OMPI_SUCCESS != rc)
-        goto cleanup_and_return;
+    if (OMPI_SUCCESS != rc) {
+        return rc;
+    }
 
     rc = mca_bml.bml_register( MCA_PML_OB1_HDR_TYPE_ACK,
                                mca_pml_ob1_recv_frag_callback_ack,
                                NULL );
-    if(OMPI_SUCCESS != rc)
-        goto cleanup_and_return;
+    if (OMPI_SUCCESS != rc) {
+        return rc;
+    }
 
     rc = mca_bml.bml_register( MCA_PML_OB1_HDR_TYPE_FRAG,
                                mca_pml_ob1_recv_frag_callback_frag,
                                NULL );
-    if(OMPI_SUCCESS != rc)
-        goto cleanup_and_return;
+    if (OMPI_SUCCESS != rc) {
+        return rc;
+    }
 
     rc = mca_bml.bml_register( MCA_PML_OB1_HDR_TYPE_PUT,
                                mca_pml_ob1_recv_frag_callback_put,
                                NULL );
-    if(OMPI_SUCCESS != rc)
-        goto cleanup_and_return;
+    if (OMPI_SUCCESS != rc) {
+        return rc;
+    }
 
     rc = mca_bml.bml_register( MCA_PML_OB1_HDR_TYPE_FIN,
                                mca_pml_ob1_recv_frag_callback_fin,
                                NULL );
-    if(OMPI_SUCCESS != rc)
-        goto cleanup_and_return;
-
-    /* register error handlers */
-    rc = mca_bml.bml_register_error(mca_pml_ob1_error_handler);
-    if(OMPI_SUCCESS != rc)
-        goto cleanup_and_return;
+    if (OMPI_SUCCESS != rc) {
+        return rc;
+    }
 
-  cleanup_and_return:
-    OBJ_DESTRUCT(&reachable);
+    rc = mca_bml.bml_register (MCA_PML_OB1_HDR_TYPE_CID,
+                               mca_pml_ob1_recv_frag_callback_cid,
+                               NULL);
+    if (OMPI_SUCCESS != rc) {
+        return rc;
+    }
 
-    return rc;
+    /* register error handlers */
+    return  mca_bml.bml_register_error(mca_pml_ob1_error_handler);
 }
 
 /*
@@ -594,8 +668,8 @@ int mca_pml_ob1_dump(struct ompi_communicator_t* comm, int verbose)
 
     /* TODO: don't forget to dump mca_pml_ob1.non_existing_communicator_pending */
 
-    opal_output(0, "Communicator %s [%p](%d) rank %d recv_seq %d num_procs %lu last_probed %lu\n",
-                comm->c_name, (void*) comm, comm->c_contextid, comm->c_my_rank,
+    opal_output(0, "Communicator %s [%p](%s) rank %d recv_seq %d num_procs %lu last_probed %lu\n",
+                comm->c_name, (void*) comm, ompi_comm_print_cid (comm), comm->c_my_rank,
                 pml_comm->recv_sequence, pml_comm->num_procs, pml_comm->last_probed);
 
 #if !MCA_PML_OB1_CUSTOM_MATCH
@@ -653,10 +727,8 @@ int mca_pml_ob1_dump(struct ompi_communicator_t* comm, int verbose)
     return OMPI_SUCCESS;
 }
 
-static void mca_pml_ob1_fin_completion( mca_btl_base_module_t* btl,
-                                        struct mca_btl_base_endpoint_t* ep,
-                                        struct mca_btl_base_descriptor_t* des,
-                                        int status )
+static void mca_pml_ob1_control_completion (mca_btl_base_module_t* btl, struct mca_btl_base_endpoint_t *endpoint,
+                                            mca_btl_base_descriptor_t *des, int status)
 {
 
     mca_bml_base_btl_t* bml_btl = (mca_bml_base_btl_t*) des->des_context;
@@ -665,40 +737,36 @@ static void mca_pml_ob1_fin_completion( mca_btl_base_module_t* btl,
     MCA_PML_OB1_PROGRESS_PENDING(bml_btl);
 }
 
-/**
- * Send an FIN to the peer. If we fail to send this ack (no more available
- * fragments or the send failed) this function automatically add the FIN
- * to the list of pending FIN, Which guarantee that the FIN will be sent
- * later.
- */
-int mca_pml_ob1_send_fin( ompi_proc_t* proc,
-                          mca_bml_base_btl_t* bml_btl,
-                          opal_ptr_t hdr_frag,
-                          uint64_t rdma_size,
-                          uint8_t order,
-                          int status )
+
+int mca_pml_ob1_send_control_btl (mca_bml_base_btl_t *bml_btl, int order, mca_pml_ob1_hdr_t *hdr, size_t hdr_size,
+                                  bool add_to_pending)
 {
-    mca_btl_base_descriptor_t* fin;
+    int des_flags = MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP | MCA_BTL_DES_FLAGS_SIGNAL;
+    mca_btl_base_descriptor_t *des;
     int rc;
 
-    mca_bml_base_alloc(bml_btl, &fin, order, sizeof(mca_pml_ob1_fin_hdr_t),
-                       MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP | MCA_BTL_DES_FLAGS_SIGNAL);
+    if (NULL != bml_btl->btl->btl_sendi) {
+        rc = mca_bml_base_sendi (bml_btl, NULL, hdr, hdr_size, 0, order, des_flags, hdr->hdr_common.hdr_type, &des);
+        if (OPAL_LIKELY(OPAL_SUCCESS == rc)) {
+            return rc;
+        }
+    } else {
+        (void) mca_bml_base_alloc (bml_btl, &des, order, hdr_size, des_flags);
+    }
 
-    if(NULL == fin) {
-        MCA_PML_OB1_ADD_FIN_TO_PENDING(proc, hdr_frag, rdma_size, bml_btl, order, status);
+    if (OPAL_UNLIKELY(NULL == des)) {
+        if (add_to_pending) {
+            mca_pml_ob1_add_to_pending (NULL, bml_btl, order, hdr, hdr_size);
+        }
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
-    fin->des_cbfunc = mca_pml_ob1_fin_completion;
-    fin->des_cbdata = NULL;
 
-    /* fill in header */
-    mca_pml_ob1_fin_hdr_prepare ((mca_pml_ob1_fin_hdr_t *) fin->des_segments->seg_addr.pval,
-                                 0, hdr_frag.lval, status ? status : (int64_t) rdma_size);
+    des->des_cbfunc = mca_pml_ob1_control_completion;
 
-    ob1_hdr_hton((mca_pml_ob1_hdr_t *) fin->des_segments->seg_addr.pval, MCA_PML_OB1_HDR_TYPE_FIN, proc);
+    memcpy (des->des_segments->seg_addr.pval, hdr, hdr_size);
 
     /* queue request */
-    rc = mca_bml_base_send( bml_btl, fin, MCA_PML_OB1_HDR_TYPE_FIN );
+    rc = mca_bml_base_send (bml_btl, des, hdr->hdr_common.hdr_type);
     if( OPAL_LIKELY( rc >= 0 ) ) {
         if( OPAL_LIKELY( 1 == rc ) ) {
             MCA_PML_OB1_PROGRESS_PENDING(bml_btl);
@@ -706,76 +774,98 @@ int mca_pml_ob1_send_fin( ompi_proc_t* proc,
         SPC_RECORD(OMPI_SPC_BYTES_SENT_MPI, (ompi_spc_value_t)sizeof(mca_pml_ob1_fin_hdr_t));
         return OMPI_SUCCESS;
     }
-    mca_bml_base_free(bml_btl, fin);
-    MCA_PML_OB1_ADD_FIN_TO_PENDING(proc, hdr_frag, rdma_size, bml_btl, order, status);
+
+    mca_bml_base_free(bml_btl, des);
+    if (add_to_pending) {
+        mca_pml_ob1_add_to_pending (NULL, bml_btl, order, hdr, hdr_size);
+    }
+
+    return OMPI_ERR_OUT_OF_RESOURCE;
+}
+
+int mca_pml_ob1_send_control_any (ompi_proc_t *proc, int order, mca_pml_ob1_hdr_t *hdr, size_t hdr_size,
+                                  bool add_to_pending)
+{
+    mca_bml_base_endpoint_t* endpoint = mca_bml_base_get_endpoint (proc);
+    int rc;
+
+    assert (NULL != endpoint);
+
+    for (size_t i = 0 ; i < mca_bml_base_btl_array_get_size(&endpoint->btl_eager) ; ++i) {
+        mca_bml_base_btl_t *bml_btl = mca_bml_base_btl_array_get_next (&endpoint->btl_eager);
+
+        rc = mca_pml_ob1_send_control_btl (bml_btl, order, hdr, hdr_size, false);
+        if (OMPI_SUCCESS == rc) {
+            return OMPI_SUCCESS;
+        }
+    }
+
+    if (add_to_pending) {
+        mca_pml_ob1_add_to_pending (proc, NULL, order, hdr, hdr_size);
+    }
+
     return OMPI_ERR_OUT_OF_RESOURCE;
 }
 
+/**
+ * Send an FIN to the peer. If we fail to send this ack (no more available
+ * fragments or the send failed) this function automatically add the FIN
+ * to the list of pending FIN, Which guarantee that the FIN will be sent
+ * later.
+ */
+int mca_pml_ob1_send_fin (ompi_proc_t* proc, mca_bml_base_btl_t* bml_btl, opal_ptr_t hdr_frag, uint64_t rdma_size,
+                          uint8_t order, int status)
+{
+    mca_pml_ob1_fin_hdr_t fin;
+
+    /* fill in header */
+    mca_pml_ob1_fin_hdr_prepare (&fin, 0, hdr_frag.lval, status ? status : (int64_t) rdma_size);
+
+    ob1_hdr_hton((mca_pml_ob1_hdr_t *) &fin, MCA_PML_OB1_HDR_TYPE_FIN, proc);
+
+    return mca_pml_ob1_send_control_btl (bml_btl, order, (mca_pml_ob1_hdr_t *) &fin, sizeof (fin), true);
+}
+
+int mca_pml_ob1_send_cid (ompi_proc_t *proc, ompi_communicator_t *comm)
+{
+    mca_pml_ob1_cid_hdr_t cid;
+
+    mca_pml_ob1_cid_hdr_prepare (&cid, comm);
+    ob1_hdr_hton ((mca_pml_ob1_hdr_t *) &cid, cid.hdr_common.hdr_type, proc);
+
+    return mca_pml_ob1_send_control_any (proc, MCA_BTL_NO_ORDER, (mca_pml_ob1_hdr_t *) &cid, sizeof (cid), true);
+}
+
 void mca_pml_ob1_process_pending_packets(mca_bml_base_btl_t* bml_btl)
 {
     mca_pml_ob1_pckt_pending_t *pckt;
-    int32_t i, rc, s = (int32_t)opal_list_get_size(&mca_pml_ob1.pckt_pending);
-
-    for(i = 0; i < s; i++) {
-        mca_bml_base_btl_t *send_dst = NULL;
-        OPAL_THREAD_LOCK(&mca_pml_ob1.lock);
-        pckt = (mca_pml_ob1_pckt_pending_t*)
-            opal_list_remove_first(&mca_pml_ob1.pckt_pending);
-        OPAL_THREAD_UNLOCK(&mca_pml_ob1.lock);
-        if(NULL == pckt)
+    int32_t rc, max = (int32_t) opal_list_get_size (&mca_pml_ob1.pckt_pending);
+
+    for (int32_t i = 0; i < max ; ++i) {
+        OPAL_THREAD_SCOPED_LOCK(&mca_pml_ob1.lock, {
+                pckt = (mca_pml_ob1_pckt_pending_t*)
+                    opal_list_remove_first(&mca_pml_ob1.pckt_pending);
+            });
+        if (NULL == pckt) {
             break;
-        if(pckt->bml_btl != NULL &&
-                pckt->bml_btl->btl == bml_btl->btl) {
-            send_dst = pckt->bml_btl;
-        } else {
-            mca_bml_base_endpoint_t* endpoint =
-                (mca_bml_base_endpoint_t*) pckt->proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
-            send_dst = mca_bml_base_btl_array_find(
-                    &endpoint->btl_eager, bml_btl->btl);
         }
-        if(NULL == send_dst) {
-            OPAL_THREAD_LOCK(&mca_pml_ob1.lock);
-            opal_list_append(&mca_pml_ob1.pckt_pending,
-                             (opal_list_item_t*)pckt);
-            OPAL_THREAD_UNLOCK(&mca_pml_ob1.lock);
-            continue;
+
+        if (pckt->bml_btl) {
+            rc = mca_pml_ob1_send_control_btl (pckt->bml_btl, pckt->order, &pckt->hdr, pckt->hdr_size, false);
+        } else {
+            rc = mca_pml_ob1_send_control_any (pckt->proc, pckt->order, &pckt->hdr, pckt->hdr_size, false);
         }
 
-        switch(pckt->hdr.hdr_common.hdr_type) {
-            case MCA_PML_OB1_HDR_TYPE_ACK:
-                rc = mca_pml_ob1_recv_request_ack_send_btl(pckt->proc,
-                        send_dst,
-                        pckt->hdr.hdr_ack.hdr_src_req.lval,
-                        pckt->hdr.hdr_ack.hdr_dst_req.pval,
-                        pckt->hdr.hdr_ack.hdr_send_offset,
-                        pckt->hdr.hdr_ack.hdr_send_size,
-                        pckt->hdr.hdr_common.hdr_flags & MCA_PML_OB1_HDR_FLAGS_NORDMA);
-                if( OPAL_UNLIKELY(OMPI_ERR_OUT_OF_RESOURCE == rc) ) {
-                    OPAL_THREAD_LOCK(&mca_pml_ob1.lock);
+        if (OPAL_SUCCESS != rc) {
+            /* could not send the packet. readd it to the pending list */
+            OPAL_THREAD_SCOPED_LOCK(&mca_pml_ob1.lock, {
                     opal_list_append(&mca_pml_ob1.pckt_pending,
                                      (opal_list_item_t*)pckt);
-                    OPAL_THREAD_UNLOCK(&mca_pml_ob1.lock);
-                    return;
-                }
-                break;
-            case MCA_PML_OB1_HDR_TYPE_FIN:
-                rc = mca_pml_ob1_send_fin(pckt->proc, send_dst,
-                                          pckt->hdr.hdr_fin.hdr_frag,
-                                          pckt->hdr.hdr_fin.hdr_size,
-                                          pckt->order,
-                                          pckt->status);
-                if( OPAL_UNLIKELY(OMPI_ERR_OUT_OF_RESOURCE == rc) ) {
-                    MCA_PML_OB1_PCKT_PENDING_RETURN(pckt);
-                    return;
-                }
-                break;
-            default:
-                opal_output(0, "[%s:%d] wrong header type\n",
-                            __FILE__, __LINE__);
-                break;
+                });
+        } else {
+            /* We're done with this packet, return it back to the free list */
+            MCA_PML_OB1_PCKT_PENDING_RETURN(pckt);
         }
-        /* We're done with this packet, return it back to the free list */
-        MCA_PML_OB1_PCKT_PENDING_RETURN(pckt);
     }
 }
 
@@ -808,12 +898,11 @@ void mca_pml_ob1_process_pending_rdma(void)
 void mca_pml_ob1_error_handler(
         struct mca_btl_base_module_t* btl, int32_t flags,
         opal_proc_t* errproc, char* btlinfo ) {
-#if OPAL_CUDA_SUPPORT
-    if (flags & MCA_BTL_ERROR_FLAGS_ADD_CUDA_IPC) {
-        mca_pml_ob1_cuda_add_ipc_support(btl, flags, (struct ompi_proc_t*)errproc, btlinfo);
+
+    if (flags & MCA_BTL_ERROR_FLAGS_ADD_ACCELERATOR_IPC) {
+        mca_pml_ob1_accelerator_add_ipc_support(btl, flags, (struct ompi_proc_t*)errproc, btlinfo);
         return;
     }
-#endif /* OPAL_CUDA_SUPPORT */
 
     /* Some BTL report unreachable errors during normal MPI_Finalize
      * termination. Lets simply ignore such errors after MPI is not supposed to
diff --git a/ompi/mca/pml/ob1/pml_ob1.h b/ompi/mca/pml/ob1/pml_ob1.h
index f425e98d6ff..a0091793ab4 100644
--- a/ompi/mca/pml/ob1/pml_ob1.h
+++ b/ompi/mca/pml/ob1/pml_ob1.h
@@ -12,10 +12,12 @@
  *                         All rights reserved.
  * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved
  * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
- * Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018-2019 Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -41,6 +43,7 @@
 #include "ompi/mca/bml/base/base.h"
 #include "ompi/proc/proc.h"
 #include "opal/mca/allocator/base/base.h"
+#include "ompi/runtime/mpiruntime.h"
 
 BEGIN_C_DECLS
 
@@ -83,12 +86,16 @@ struct mca_pml_ob1_t {
     char* allocator_name;
     mca_allocator_base_module_t* allocator;
     unsigned int unexpected_limit;
+    /* Accelerator support initialized */
+    bool accelerator_enabled;
 };
 typedef struct mca_pml_ob1_t mca_pml_ob1_t;
 
 extern mca_pml_ob1_t mca_pml_ob1;
 extern int mca_pml_ob1_output;
 extern bool mca_pml_ob1_matching_protection;
+extern int mca_pml_ob1_accelerator_events_max;
+
 /*
  * PML interface functions.
  */
@@ -226,11 +233,11 @@ END_C_DECLS
 
 struct mca_pml_ob1_pckt_pending_t {
     opal_free_list_item_t super;
-    ompi_proc_t* proc;
+    ompi_proc_t *proc;
     mca_pml_ob1_hdr_t hdr;
+    size_t hdr_size;
     struct mca_bml_base_btl_t *bml_btl;
     uint8_t order;
-    int status;
 };
 typedef struct mca_pml_ob1_pckt_pending_t mca_pml_ob1_pckt_pending_t;
 OBJ_CLASS_DECLARATION(mca_pml_ob1_pckt_pending_t);
@@ -248,22 +255,22 @@ do {                                                            \
         (opal_free_list_item_t*)pckt);                          \
 } while(0)
 
-#define MCA_PML_OB1_ADD_FIN_TO_PENDING(P, D, Sz, B, O, S)           \
-    do {                                                            \
-        mca_pml_ob1_pckt_pending_t *_pckt;                          \
-                                                                    \
-        MCA_PML_OB1_PCKT_PENDING_ALLOC(_pckt);                      \
-        mca_pml_ob1_fin_hdr_prepare (&_pckt->hdr.hdr_fin, 0,        \
-                                     (D).lval, (Sz));               \
-        _pckt->proc = (P);                                          \
-        _pckt->bml_btl = (B);                                       \
-        _pckt->order = (O);                                         \
-        _pckt->status = (S);                                        \
-        OPAL_THREAD_LOCK(&mca_pml_ob1.lock);                        \
-        opal_list_append(&mca_pml_ob1.pckt_pending,                 \
-                (opal_list_item_t*)_pckt);                          \
-        OPAL_THREAD_UNLOCK(&mca_pml_ob1.lock);                      \
-    } while(0)
+static inline void mca_pml_ob1_add_to_pending (ompi_proc_t *proc, mca_bml_base_btl_t *bml_btl,
+                                               int order, mca_pml_ob1_hdr_t *hdr, size_t hdr_size)
+{
+    mca_pml_ob1_pckt_pending_t *pckt;
+
+    MCA_PML_OB1_PCKT_PENDING_ALLOC(pckt);
+    assert (sizeof (pckt->hdr) >= hdr_size);
+    pckt->proc = proc;
+    pckt->order = order;
+    pckt->hdr_size = hdr_size;
+    pckt->bml_btl = bml_btl;
+    memcpy (&pckt->hdr, hdr, hdr_size);
+    OPAL_THREAD_SCOPED_LOCK(&mca_pml_ob1.lock, {
+            opal_list_append(&mca_pml_ob1.pckt_pending, &pckt->super.super);
+        });
+}
 
 #define OB1_MATCHING_LOCK(lock)                                  \
     do {                                                         \
@@ -287,6 +294,8 @@ do {                                                            \
 int mca_pml_ob1_send_fin(ompi_proc_t* proc, mca_bml_base_btl_t* bml_btl,
         opal_ptr_t hdr_frag, uint64_t size, uint8_t order, int status);
 
+int mca_pml_ob1_send_cid (ompi_proc_t *proc, ompi_communicator_t *comm);
+
 /* This function tries to resend FIN/ACK packets from pckt_pending queue.
  * Packets are added to the queue when sending of FIN or ACK is failed due to
  * resource unavailability. bml_btl passed to the function doesn't represents
@@ -408,4 +417,9 @@ mca_pml_ob1_calc_weighted_length( mca_pml_ob1_com_btl_t *btls, int num_btls, siz
  */
 int mca_pml_ob1_enable_progress(int32_t count);
 
+int mca_pml_ob1_send_control_any (ompi_proc_t *proc, int order, mca_pml_ob1_hdr_t *hdr, size_t hdr_size,
+                                  bool add_to_pending);
+int mca_pml_ob1_send_control_btl (mca_bml_base_btl_t *bml_btl, int order, mca_pml_ob1_hdr_t *hdr, size_t hdr_size,
+                                  bool add_to_pending);
+
 #endif
diff --git a/ompi/mca/pml/ob1/pml_ob1_accelerator.c b/ompi/mca/pml/ob1/pml_ob1_accelerator.c
new file mode 100644
index 00000000000..6d63dd59114
--- /dev/null
+++ b/ompi/mca/pml/ob1/pml_ob1_accelerator.c
@@ -0,0 +1,506 @@
+/*
+ * Copyright (c) 2008      UT-Battelle, LLC. All rights reserved.
+ * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2020      Intel, Inc.  All rights reserved.
+ * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2022 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2006 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2015 NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2012-2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018-2022 Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "opal/prefetch.h"
+#include "opal/runtime/opal_params.h"
+#include "opal/mca/btl/btl.h"
+#include "opal/mca/mpool/mpool.h"
+#include "ompi/constants.h"
+#include "ompi/mca/pml/pml.h"
+#include "pml_ob1.h"
+#include "pml_ob1_hdr.h"
+#include "pml_ob1_rdmafrag.h"
+#include "pml_ob1_recvreq.h"
+#include "pml_ob1_sendreq.h"
+#include "ompi/mca/bml/base/base.h"
+
+#include "opal/types.h"
+#include "pml_ob1_accelerator.h"
+#include "pml_ob1.h"
+#include "opal/mca/accelerator/base/base.h"
+
+static opal_accelerator_stream_t *dtoh_stream = NULL;
+static opal_accelerator_stream_t *htod_stream = NULL;
+
+static opal_mutex_t pml_ob1_accelerator_htod_lock;
+static opal_mutex_t pml_ob1_accelerator_dtoh_lock;
+
+/* Array of accelerator events to be queried for sending side and
+ * receiving side. */
+static opal_accelerator_event_t **accelerator_event_dtoh_array = NULL;
+static opal_accelerator_event_t **accelerator_event_htod_array = NULL;
+
+/* Array of fragments currently being moved by accelerator async non-blocking
+ * operations */
+static struct mca_btl_base_descriptor_t **accelerator_event_dtoh_frag_array = NULL;
+static struct mca_btl_base_descriptor_t **accelerator_event_htod_frag_array = NULL;
+
+/* First free/available location in accelerator_event_status_array */
+static int accelerator_event_dtoh_first_avail, accelerator_event_htod_first_avail;
+
+/* First currently-being used location in the accelerator_event_status_array */
+static int accelerator_event_dtoh_first_used, accelerator_event_htod_first_used;
+
+/* Number of status items currently in use */
+static volatile int accelerator_event_dtoh_num_used, accelerator_event_htod_num_used;
+
+/* Size of array holding events */
+static int accelerator_event_htod_most = 0;
+
+int mca_pml_ob1_record_htod_event(char *msg, struct mca_btl_base_descriptor_t *frag)
+{
+    int result;
+
+    if (0 == strcmp(opal_accelerator_base_selected_component.base_version.mca_component_name, "null")) {
+        return 0;
+    }
+
+    /* First make sure there is room to store the event.  If not, then
+     * return an error.  The error message will tell the user to try and
+     * run again, but with a larger array for storing events. */
+    OPAL_THREAD_LOCK(&pml_ob1_accelerator_htod_lock);
+    if (accelerator_event_htod_num_used == mca_pml_ob1_accelerator_events_max) {
+        opal_output_verbose(1, mca_pml_ob1_output, "Out of event handles. Max: %d. Suggested to rerun with new max with --mca pml_ob1_accelerator_events_max %d.",
+                            mca_pml_ob1_accelerator_events_max, mca_pml_ob1_accelerator_events_max + 100);
+        OPAL_THREAD_UNLOCK(&pml_ob1_accelerator_htod_lock);
+        return OPAL_ERR_OUT_OF_RESOURCE;
+    }
+
+    if (accelerator_event_htod_num_used > accelerator_event_htod_most) {
+        accelerator_event_htod_most = accelerator_event_htod_num_used;
+        /* Just print multiples of 10 */
+        if (0 == (accelerator_event_htod_most % 10)) {
+            opal_output_verbose(20, mca_pml_ob1_output, "Maximum HtoD events used is now %d",
+                                accelerator_event_htod_most);
+        }
+    }
+
+    result = opal_accelerator.record_event(MCA_ACCELERATOR_NO_DEVICE_ID, accelerator_event_htod_array[accelerator_event_htod_first_avail], htod_stream);
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != result)) {
+        opal_output_verbose(1, mca_pml_ob1_output, "Event Record failed.");
+        OPAL_THREAD_UNLOCK(&pml_ob1_accelerator_htod_lock);
+        return OPAL_ERROR;
+    }
+    accelerator_event_htod_frag_array[accelerator_event_htod_first_avail] = frag;
+
+    /* Bump up the first available slot and number used by 1 */
+    accelerator_event_htod_first_avail++;
+    if (accelerator_event_htod_first_avail >= mca_pml_ob1_accelerator_events_max) {
+        accelerator_event_htod_first_avail = 0;
+    }
+    accelerator_event_htod_num_used++;
+
+    OPAL_THREAD_UNLOCK(&pml_ob1_accelerator_htod_lock);
+    return OPAL_SUCCESS;
+}
+
+opal_accelerator_stream_t *mca_pml_ob1_get_dtoh_stream(void)
+{
+    return dtoh_stream;
+}
+opal_accelerator_stream_t *mca_pml_ob1_get_htod_stream(void)
+{
+    return htod_stream;
+}
+
+/**
+ * Progress any htod event completions.
+ */
+int mca_pml_ob1_progress_one_htod_event(struct mca_btl_base_descriptor_t **frag)
+{
+    int result;
+
+    if (0 == strcmp(opal_accelerator_base_selected_component.base_version.mca_component_name, "null")) {
+        return 0;
+    }
+
+    OPAL_THREAD_LOCK(&pml_ob1_accelerator_htod_lock);
+    if (accelerator_event_htod_num_used > 0) {
+        opal_output_verbose(30, mca_pml_ob1_output,
+                            "mca_pml_ob1_progress_one_htod_event, outstanding_events=%d",
+                            accelerator_event_htod_num_used);
+
+        result = opal_accelerator.query_event(MCA_ACCELERATOR_NO_DEVICE_ID, accelerator_event_htod_array[accelerator_event_htod_first_used]);
+
+        /* We found an event that is not ready, so return. */
+        if (OPAL_ERR_RESOURCE_BUSY == result) {
+            opal_output_verbose(30, mca_pml_ob1_output,
+                                "Accelerator event query returned OPAL_ERR_RESOURCE_BUSY");
+            *frag = NULL;
+            OPAL_THREAD_UNLOCK(&pml_ob1_accelerator_htod_lock);
+            return 0;
+        } else if (OPAL_SUCCESS != result) {
+            opal_output_verbose(1, mca_pml_ob1_output, "Accelerator event query failed: %d,", result);
+            *frag = NULL;
+            OPAL_THREAD_UNLOCK(&pml_ob1_accelerator_htod_lock);
+            return OPAL_ERROR;
+        }
+
+        *frag = accelerator_event_htod_frag_array[accelerator_event_htod_first_used];
+
+        /* Bump counters, loop around the circular buffer if necessary */
+        --accelerator_event_htod_num_used;
+        ++accelerator_event_htod_first_used;
+        if (accelerator_event_htod_first_used >= mca_pml_ob1_accelerator_events_max) {
+            accelerator_event_htod_first_used = 0;
+        }
+        /* A return value of 1 indicates an event completed and a frag was returned */
+        OPAL_THREAD_UNLOCK(&pml_ob1_accelerator_htod_lock);
+        return 1;
+    }
+    OPAL_THREAD_UNLOCK(&pml_ob1_accelerator_htod_lock);
+    return 0;
+}
+
+int mca_pml_ob1_accelerator_init(void)
+{
+    int rc = OPAL_SUCCESS;
+    int result = OPAL_SUCCESS;
+    int i;
+
+    if (0 == strcmp(opal_accelerator_base_selected_component.base_version.mca_component_name, "null")) {
+        return 0;
+    }
+
+    OBJ_CONSTRUCT(&pml_ob1_accelerator_htod_lock, opal_mutex_t);
+    OBJ_CONSTRUCT(&pml_ob1_accelerator_dtoh_lock, opal_mutex_t);
+
+    /* Create Streams */
+    result = opal_accelerator.create_stream(MCA_ACCELERATOR_NO_DEVICE_ID, &dtoh_stream);
+    if (OPAL_SUCCESS != result) {
+        opal_output_verbose(1, mca_pml_ob1_output, "Failed to create accelerator dtoh_stream stream.");
+        rc = result;
+        goto cleanup_and_error;
+    }
+
+    result = opal_accelerator.create_stream(MCA_ACCELERATOR_NO_DEVICE_ID, &htod_stream);
+    if (OPAL_SUCCESS != result) {
+        opal_output_verbose(1, mca_pml_ob1_output, "Failed to create accelerator htod_stream stream.");
+        rc = result;
+        goto cleanup_and_error;
+    }
+
+    /* Set up an array of pointers to store outstanding async dtoh events.
+     * Used on the sending side for asynchronous copies. */
+    accelerator_event_dtoh_num_used = 0;
+    accelerator_event_dtoh_first_avail = 0;
+    accelerator_event_dtoh_first_used = 0;
+
+    accelerator_event_dtoh_array = calloc(mca_pml_ob1_accelerator_events_max, sizeof(opal_accelerator_event_t *));
+    if (NULL == accelerator_event_dtoh_array) {
+        opal_output_verbose(1, mca_pml_ob1_output, "No memory.");
+        rc = OPAL_ERROR;
+        goto cleanup_and_error;
+    }
+
+    /* Create the events since they can be reused. */
+    for (i = 0; i < mca_pml_ob1_accelerator_events_max; i++) {
+        result = opal_accelerator.create_event(MCA_ACCELERATOR_NO_DEVICE_ID, &accelerator_event_dtoh_array[i], false);
+        if (OPAL_SUCCESS != result) {
+            opal_output_verbose(1, mca_pml_ob1_output, "Accelerator create event failed.");
+            rc = OPAL_ERROR;
+            goto cleanup_and_error;
+        }
+    }
+
+    /* The first available status index is 0.  Make an empty frag
+       array. */
+    accelerator_event_dtoh_frag_array = (struct mca_btl_base_descriptor_t **) malloc(
+        sizeof(struct mca_btl_base_descriptor_t *) * mca_pml_ob1_accelerator_events_max);
+    if (NULL == accelerator_event_dtoh_frag_array) {
+        opal_output_verbose(1, mca_pml_ob1_output, "No memory.");
+        rc = OPAL_ERROR;
+        goto cleanup_and_error;
+    }
+
+    /* Set up an array of pointers to store outstanding async htod events.
+     * Used on the receiving side for asynchronous copies. */
+    accelerator_event_htod_num_used = 0;
+    accelerator_event_htod_first_avail = 0;
+    accelerator_event_htod_first_used = 0;
+
+    accelerator_event_htod_array = calloc(mca_pml_ob1_accelerator_events_max, sizeof(opal_accelerator_event_t *));
+    if (NULL == accelerator_event_htod_array) {
+        opal_output_verbose(1, mca_pml_ob1_output, "No memory.");
+        rc = OPAL_ERROR;
+        goto cleanup_and_error;
+    }
+
+    /* Create the events since they can be reused. */
+    for (i = 0; i < mca_pml_ob1_accelerator_events_max; i++) {
+        result = opal_accelerator.create_event(MCA_ACCELERATOR_NO_DEVICE_ID, &accelerator_event_htod_array[i], false);
+        if (OPAL_SUCCESS != result) {
+            opal_output_verbose(1, mca_pml_ob1_output, "Accelerator create event failed.");
+            rc = OPAL_ERROR;
+            goto cleanup_and_error;
+        }
+    }
+
+    /* The first available status index is 0.  Make an empty frag
+       array. */
+    accelerator_event_htod_frag_array = (struct mca_btl_base_descriptor_t **) malloc(
+        sizeof(struct mca_btl_base_descriptor_t *) * mca_pml_ob1_accelerator_events_max);
+    if (NULL == accelerator_event_htod_frag_array) {
+        opal_output_verbose(1, mca_pml_ob1_output, "No memory.");
+        rc = OPAL_ERROR;
+        goto cleanup_and_error;
+    }
+
+cleanup_and_error:
+    if (OPAL_SUCCESS != rc) {
+        if (NULL != accelerator_event_dtoh_array) {
+            free(accelerator_event_dtoh_array);
+        }
+        if (NULL != accelerator_event_dtoh_frag_array) {
+            free(accelerator_event_dtoh_frag_array);
+        }
+        if (NULL != accelerator_event_htod_array) {
+            free(accelerator_event_htod_array);
+        }
+        if (NULL != accelerator_event_htod_frag_array) {
+            free(accelerator_event_htod_frag_array);
+        }
+        OBJ_DESTRUCT(&pml_ob1_accelerator_htod_lock);
+        OBJ_DESTRUCT(&pml_ob1_accelerator_dtoh_lock);
+    }
+
+    return rc;
+}
+
+void mca_pml_ob1_accelerator_fini(void)
+{
+    int i;
+
+    if (0 == strcmp(opal_accelerator_base_selected_component.base_version.mca_component_name, "null")) {
+        return;
+    }
+
+    if (NULL != accelerator_event_htod_array) {
+        for (i = 0; i < mca_pml_ob1_accelerator_events_max; i++) {
+            if (NULL != accelerator_event_htod_array[i]) {
+                OBJ_RELEASE(accelerator_event_htod_array[i]);
+            }
+        }
+        free(accelerator_event_htod_array);
+    }
+
+    if (NULL != accelerator_event_dtoh_array) {
+        for (i = 0; i < mca_pml_ob1_accelerator_events_max; i++) {
+            if (NULL != accelerator_event_dtoh_array[i]) {
+                OBJ_RELEASE(accelerator_event_dtoh_array[i]);
+            }
+        }
+        free(accelerator_event_dtoh_array);
+    }
+    if (NULL != accelerator_event_dtoh_frag_array) {
+        free(accelerator_event_dtoh_frag_array);
+    }
+    if (NULL != accelerator_event_htod_frag_array) {
+        free(accelerator_event_htod_frag_array);
+    }
+
+    OBJ_RELEASE(htod_stream);
+    OBJ_RELEASE(dtoh_stream);
+
+    OBJ_DESTRUCT(&pml_ob1_accelerator_htod_lock);
+    OBJ_DESTRUCT(&pml_ob1_accelerator_dtoh_lock);
+}
+
+size_t mca_pml_ob1_rdma_cuda_btls(
+    mca_bml_base_endpoint_t* bml_endpoint,
+    unsigned char* base,
+    size_t size,
+    mca_pml_ob1_com_btl_t* rdma_btls);
+
+int mca_pml_ob1_accelerator_need_buffers(void * rreq,
+                                         mca_btl_base_module_t* btl);
+
+void mca_pml_ob1_accelerator_add_ipc_support(struct mca_btl_base_module_t* btl, int32_t flags,
+                                             ompi_proc_t* errproc, char* btlinfo);
+
+/**
+ * Handle the accelerator buffer.
+ */
+int mca_pml_ob1_send_request_start_accelerator(mca_pml_ob1_send_request_t* sendreq,
+                                               mca_bml_base_btl_t* bml_btl,
+                                               size_t size) {
+    int rc;
+#if OPAL_CUDA_GDR_SUPPORT
+    /* With some BTLs, switch to RNDV from RGET at large messages */
+    if ((sendreq->req_send.req_base.req_convertor.flags & CONVERTOR_ACCELERATOR) &&
+        (sendreq->req_send.req_bytes_packed > (bml_btl->btl->btl_accelerator_rdma_limit - sizeof(mca_pml_ob1_hdr_t)))) {
+        return mca_pml_ob1_send_request_start_rndv(sendreq, bml_btl, 0, 0);
+    }
+#endif /* OPAL_CUDA_GDR_SUPPORT */
+
+    if (opal_convertor_need_buffers(&sendreq->req_send.req_base.req_convertor) == false) {
+        unsigned char *base;
+        opal_convertor_get_current_pointer( &sendreq->req_send.req_base.req_convertor, (void**)&base );
+        if( 0 != (sendreq->req_rdma_cnt = (uint32_t)mca_pml_ob1_rdma_cuda_btls(
+                                                                           sendreq->req_endpoint,
+                                                                           base,
+                                                                           sendreq->req_send.req_bytes_packed,
+                                                                           sendreq->req_rdma))) {
+            rc = mca_pml_ob1_send_request_start_rdma(sendreq, bml_btl,
+                                                     sendreq->req_send.req_bytes_packed);
+            if( OPAL_UNLIKELY(OMPI_SUCCESS != rc) ) {
+                mca_pml_ob1_free_rdma_resources(sendreq);
+            }
+        } else {
+            if (bml_btl->btl_flags & MCA_BTL_FLAGS_ACCELERATOR_PUT) {
+                rc = mca_pml_ob1_send_request_start_rndv(sendreq, bml_btl, size,
+                                                         MCA_PML_OB1_HDR_FLAGS_CONTIG);
+            } else {
+                rc = mca_pml_ob1_send_request_start_rndv(sendreq, bml_btl, 0, 0);
+            }
+        }
+    } else {
+        /* Do not send anything with first rendezvous message as copying GPU
+         * memory into RNDV message is expensive. */
+        rc = mca_pml_ob1_send_request_start_rndv(sendreq, bml_btl, 0, 0);
+    }
+    return rc;
+}
+
+
+
+size_t mca_pml_ob1_rdma_cuda_btls(
+    mca_bml_base_endpoint_t* bml_endpoint,
+    unsigned char* base,
+    size_t size,
+    mca_pml_ob1_com_btl_t* rdma_btls)
+{
+    int num_btls = mca_bml_base_btl_array_get_size(&bml_endpoint->btl_send);
+    double weight_total = 0;
+    int num_btls_used = 0, n;
+
+    /* shortcut when there are no rdma capable btls */
+    if(num_btls == 0) {
+        return 0;
+    }
+
+    /* check to see if memory is registered */
+    for(n = 0; n < num_btls && num_btls_used < mca_pml_ob1.max_rdma_per_request;
+            n++) {
+        mca_bml_base_btl_t* bml_btl =
+            mca_bml_base_btl_array_get_index(&bml_endpoint->btl_send, n);
+
+        if (bml_btl->btl_flags & MCA_BTL_FLAGS_ACCELERATOR_GET) {
+            mca_btl_base_registration_handle_t *handle = NULL;
+
+            if( NULL != bml_btl->btl->btl_register_mem ) {
+                /* register the memory */
+                handle = bml_btl->btl->btl_register_mem (bml_btl->btl, bml_btl->btl_endpoint,
+                                                         base, size,
+#if OPAL_CUDA_GDR_SUPPORT
+                                                         MCA_BTL_REG_FLAG_CUDA_GPU_MEM |
+#endif
+                                                         MCA_BTL_REG_FLAG_REMOTE_READ);
+            }
+
+            if(NULL == handle)
+                continue;
+
+            rdma_btls[num_btls_used].bml_btl = bml_btl;
+            rdma_btls[num_btls_used].btl_reg = handle;
+            weight_total += bml_btl->btl_weight;
+            num_btls_used++;
+        }
+    }
+
+    /* if we don't use leave_pinned and all BTLs that already have this memory
+     * registered amount to less then half of available bandwidth - fall back to
+     * pipeline protocol */
+    if(0 == num_btls_used || (!opal_leave_pinned && weight_total < 0.5))
+        return 0;
+
+    mca_pml_ob1_calc_weighted_length(rdma_btls, num_btls_used, size,
+                                     weight_total);
+
+    return num_btls_used;
+}
+
+int mca_pml_ob1_accelerator_need_buffers(void * rreq,
+                                         mca_btl_base_module_t* btl)
+{
+    mca_pml_ob1_recv_request_t* recvreq = (mca_pml_ob1_recv_request_t*)rreq;
+    mca_bml_base_endpoint_t* bml_endpoint = mca_bml_base_get_endpoint (recvreq->req_recv.req_base.req_proc);
+    mca_bml_base_btl_t *bml_btl = mca_bml_base_btl_array_find(&bml_endpoint->btl_send, btl);
+
+    /* A btl could be in the rdma list but not in the send list so check there also */
+    if (NULL == bml_btl) {
+        bml_btl = mca_bml_base_btl_array_find(&bml_endpoint->btl_rdma, btl);
+    }
+    /* We should always be able to find back the bml_btl based on the btl */
+    assert(NULL != bml_btl);
+
+    if ((recvreq->req_recv.req_base.req_convertor.flags & CONVERTOR_ACCELERATOR) &&
+        (bml_btl->btl_flags & MCA_BTL_FLAGS_ACCELERATOR_GET)) {
+        return opal_convertor_need_buffers(&recvreq->req_recv.req_base.req_convertor);
+    }
+    return true;
+}
+
+/*
+ * This function enables us to start using RDMA get protocol with GPU buffers.
+ * We do this by adjusting the flags in the BML structure.  This is not the
+ * best thing, but this may go away if CUDA IPC is supported everywhere in the
+ * future. */
+void mca_pml_ob1_accelerator_add_ipc_support(struct mca_btl_base_module_t* btl, int32_t flags,
+                                      ompi_proc_t* errproc, char* btlinfo)
+{
+    mca_bml_base_endpoint_t* ep;
+    int btl_verbose_stream = 0;
+    int i;
+
+    assert(NULL != errproc);
+    assert(NULL != errproc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]);
+    if (NULL != btlinfo) {
+        btl_verbose_stream = *(int *)btlinfo;
+    }
+    ep = (mca_bml_base_endpoint_t*)errproc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
+
+    /* Find the corresponding bml and adjust the flag to support CUDA get */
+    for( i = 0; i < (int)ep->btl_send.arr_size; i++ ) {
+        if( ep->btl_send.bml_btls[i].btl == btl ) {
+            if (4 < opal_output_get_verbosity(btl_verbose_stream)) {
+                char *errhost = opal_get_proc_hostname(&errproc->super);
+                opal_output(0, "BTL %s: rank=%d enabling accelerator IPC "
+                            "to rank=%d on node=%s \n",
+                            btl->btl_component->btl_version.mca_component_name,
+                            OMPI_PROC_MY_NAME->vpid,
+                            ((ompi_process_name_t*)&errproc->super.proc_name)->vpid,
+                            errhost);
+                free(errhost);
+            }
+            ep->btl_send.bml_btls[i].btl_flags |= MCA_BTL_FLAGS_ACCELERATOR_GET;
+        }
+    }
+}
diff --git a/ompi/mca/pml/ob1/pml_ob1_accelerator.h b/ompi/mca/pml/ob1/pml_ob1_accelerator.h
new file mode 100644
index 00000000000..993790aaaa3
--- /dev/null
+++ b/ompi/mca/pml/ob1/pml_ob1_accelerator.h
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2013 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2006 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2015 NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/* Implements a progress engine based accelerator asynchronous copy implementation */
+
+#ifndef OMPI_PML_OB1_ACCELERATOR_H
+#define OMPI_PML_OB1_ACCELERATOR_H
+
+#include "opal/mca/accelerator/accelerator.h"
+#include "opal/mca/btl/btl.h"
+
+OPAL_DECLSPEC int mca_pml_ob1_record_htod_event(char *msg, struct mca_btl_base_descriptor_t *frag);
+OPAL_DECLSPEC opal_accelerator_stream_t *mca_pml_ob1_get_dtoh_stream(void);
+OPAL_DECLSPEC opal_accelerator_stream_t *mca_pml_ob1_get_htod_stream(void);
+OPAL_DECLSPEC int mca_pml_ob1_progress_one_htod_event(struct mca_btl_base_descriptor_t **);
+OPAL_DECLSPEC int mca_pml_ob1_accelerator_init(void);
+OPAL_DECLSPEC void mca_pml_ob1_accelerator_fini(void);
+
+#endif /* OMPI_PML_OB1_ACCELERATOR_H */
diff --git a/ompi/mca/pml/ob1/pml_ob1_comm.c b/ompi/mca/pml/ob1/pml_ob1_comm.c
index 9eeedd6b05d..aa0f2046638 100644
--- a/ompi/mca/pml/ob1/pml_ob1_comm.c
+++ b/ompi/mca/pml/ob1/pml_ob1_comm.c
@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -12,6 +13,9 @@
  *
  * Copyright (c) 2018      Sandia National Laboratories
  * 			   All rights reserved.
+ * Copyright (c) 2019      Triad National Security, LLC. All rights
+ *                         reserved.
+ *
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -33,6 +37,8 @@ static void mca_pml_ob1_comm_proc_construct(mca_pml_ob1_comm_proc_t* proc)
     proc->expected_sequence = 1;
     proc->send_sequence = 0;
     proc->frags_cant_match = NULL;
+    /* don't know the index of this communicator yet */
+    proc->comm_index = -1;
 #if !MCA_PML_OB1_CUSTOM_MATCH
     OBJ_CONSTRUCT(&proc->specific_receives, opal_list_t);
     OBJ_CONSTRUCT(&proc->unexpected_frags, opal_list_t);
@@ -84,7 +90,7 @@ static void mca_pml_ob1_comm_destruct(mca_pml_ob1_comm_t* comm)
             }
         }
 
-        free(comm->procs);
+        free ((void *) comm->procs);
     }
 
 #if !MCA_PML_OB1_CUSTOM_MATCH
@@ -116,4 +122,26 @@ int mca_pml_ob1_comm_init_size (mca_pml_ob1_comm_t* comm, size_t size)
     return OMPI_SUCCESS;
 }
 
+mca_pml_ob1_comm_proc_t *mca_pml_ob1_peer_create (ompi_communicator_t *comm, mca_pml_ob1_comm_t *pml_comm, int rank)
+{
+    mca_pml_ob1_comm_proc_t *proc = OBJ_NEW(mca_pml_ob1_comm_proc_t);
+    uintptr_t old_proc = 0;
 
+    proc->ompi_proc = ompi_comm_peer_lookup (comm, rank);
+    if (OMPI_COMM_IS_GLOBAL_INDEX (comm)) {
+	/* the index is global so we can save it on the proc now */
+	proc->comm_index = comm->c_index;
+    }
+    OBJ_RETAIN(proc->ompi_proc);
+    /* make sure proc structure is filled in before adding it to the array */
+    opal_atomic_wmb ();
+
+    if (!OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR((opal_atomic_intptr_t *) pml_comm->procs + rank, &old_proc,
+						(uintptr_t) proc)) {
+	/* proc was created by a competing thread. go ahead and throw this one away. */
+	OBJ_RELEASE(proc);
+	return (mca_pml_ob1_comm_proc_t *) old_proc;
+    }
+
+    return proc;
+}
diff --git a/ompi/mca/pml/ob1/pml_ob1_comm.h b/ompi/mca/pml/ob1/pml_ob1_comm.h
index 25313b4d204..7b7e978ec96 100644
--- a/ompi/mca/pml/ob1/pml_ob1_comm.h
+++ b/ompi/mca/pml/ob1/pml_ob1_comm.h
@@ -44,6 +44,7 @@ struct mca_pml_ob1_comm_proc_t {
     opal_object_t super;
     struct ompi_proc_t* ompi_proc;
     uint16_t expected_sequence;    /**< send message sequence number - receiver side */
+    int16_t comm_index;           /**< index of this communicator on the receiver size (-1 - not set) */
     opal_atomic_int32_t send_sequence; /**< send side sequence number */
     struct mca_pml_ob1_recv_frag_t* frags_cant_match;  /**< out-of-order fragment queues */
 #if !MCA_PML_OB1_CUSTOM_MATCH
@@ -54,6 +55,8 @@ struct mca_pml_ob1_comm_proc_t {
 
 OBJ_CLASS_DECLARATION(mca_pml_ob1_comm_proc_t);
 
+#define MCA_PML_OB1_PROC_REQUIRES_EXT_MATCH(proc) (-1 == (proc)->comm_index)
+
 /**
  *  Cached on ompi_communicator_t to hold queues/state
  *  used by the PML<->PTL interface for matching logic.
@@ -66,7 +69,7 @@ struct mca_pml_comm_t {
     opal_list_t wild_receives;    /**< queue of unmatched wild (source process not specified) receives */
 #endif
     opal_mutex_t proc_lock;
-    mca_pml_ob1_comm_proc_t **procs;
+    mca_pml_ob1_comm_proc_t * volatile * procs;
     size_t num_procs;
     size_t last_probed;
 #if MCA_PML_OB1_CUSTOM_MATCH
@@ -78,6 +81,11 @@ typedef struct mca_pml_comm_t mca_pml_ob1_comm_t;
 
 OBJ_CLASS_DECLARATION(mca_pml_ob1_comm_t);
 
+/**
+ * @brief Helper function to allocate/fill in ob1 proc for a comm/rank
+ */
+mca_pml_ob1_comm_proc_t *mca_pml_ob1_peer_create (ompi_communicator_t *comm, mca_pml_ob1_comm_t *pml_comm, int rank);
+
 static inline mca_pml_ob1_comm_proc_t *mca_pml_ob1_peer_lookup (struct ompi_communicator_t *comm, int rank)
 {
     mca_pml_ob1_comm_t *pml_comm = (mca_pml_ob1_comm_t *)comm->c_pml_comm;
@@ -93,15 +101,7 @@ static inline mca_pml_ob1_comm_proc_t *mca_pml_ob1_peer_lookup (struct ompi_comm
                        " valid range of the communicator. Please submit a bug request!");
     }
     if (OPAL_UNLIKELY(NULL == pml_comm->procs[rank])) {
-        OPAL_THREAD_LOCK(&pml_comm->proc_lock);
-        if (NULL == pml_comm->procs[rank]) {
-            mca_pml_ob1_comm_proc_t* proc = OBJ_NEW(mca_pml_ob1_comm_proc_t);
-            proc->ompi_proc = ompi_comm_peer_lookup (comm, rank);
-            OBJ_RETAIN(proc->ompi_proc);
-            opal_atomic_wmb ();
-            pml_comm->procs[rank] = proc;
-        }
-        OPAL_THREAD_UNLOCK(&pml_comm->proc_lock);
+        mca_pml_ob1_peer_create (comm, pml_comm, rank);
     }
 
     return pml_comm->procs[rank];
diff --git a/ompi/mca/pml/ob1/pml_ob1_component.c b/ompi/mca/pml/ob1/pml_ob1_component.c
index 0feb982ae60..74786121e4f 100644
--- a/ompi/mca/pml/ob1/pml_ob1_component.c
+++ b/ompi/mca/pml/ob1/pml_ob1_component.c
@@ -39,6 +39,7 @@
 #include "pml_ob1_recvreq.h"
 #include "pml_ob1_rdmafrag.h"
 #include "pml_ob1_recvfrag.h"
+#include "pml_ob1_accelerator.h"
 #include "ompi/mca/bml/base/base.h"
 #include "pml_ob1_component.h"
 #include "opal/mca/allocator/base/base.h"
@@ -61,6 +62,7 @@ static int mca_pml_ob1_component_fini(void);
 int mca_pml_ob1_output = 0;
 static int mca_pml_ob1_verbose = 0;
 bool mca_pml_ob1_matching_protection = false;
+int mca_pml_ob1_accelerator_events_max = 400;
 
 mca_pml_base_component_2_1_0_t mca_pml_ob1_component = {
     /* First, the mca_base_component_t struct containing meta
@@ -241,6 +243,12 @@ static int mca_pml_ob1_component_register(void)
                                            MCA_BASE_PVAR_FLAG_READONLY | MCA_BASE_PVAR_FLAG_CONTINUOUS,
                                            mca_pml_ob1_get_posted_recvq_size, NULL, mca_pml_ob1_comm_size_notify, NULL);
 
+    mca_pml_ob1_accelerator_events_max = 400;
+    (void) mca_base_component_var_register(&mca_pml_ob1_component.pmlm_version, "accelerator_events_max",
+                                           "Number of events created by the ob1 component internally",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                           MCA_BASE_VAR_SCOPE_READONLY, &mca_pml_ob1_accelerator_events_max);
+
     return OMPI_SUCCESS;
 }
 
@@ -314,6 +322,9 @@ mca_pml_ob1_component_init( int* priority,
 
     }
 
+    /** this pml supports the extended CID space */
+    mca_pml_ob1.super.pml_flags |= MCA_PML_BASE_FLAG_SUPPORTS_EXT_CID;
+
     return &mca_pml_ob1.super;
 }
 
@@ -322,8 +333,10 @@ int mca_pml_ob1_component_fini(void)
     int rc;
 
     /* Shutdown BML */
-    if(OMPI_SUCCESS != (rc = mca_bml.bml_finalize()))
-        return rc;
+    if (NULL != mca_bml.bml_finalize) {
+        if(OMPI_SUCCESS != (rc = mca_bml.bml_finalize()))
+            return rc;
+    }
 
     if(!mca_pml_ob1.enabled) {
         if( NULL != mca_pml_ob1.allocator ) {
@@ -359,6 +372,10 @@ int mca_pml_ob1_component_fini(void)
     OBJ_DESTRUCT(&mca_pml_ob1.lock);
     OBJ_DESTRUCT(&mca_pml_ob1.send_ranges);
 
+    if (mca_pml_ob1.accelerator_enabled) {
+        mca_pml_ob1_accelerator_fini();
+    }
+
     if( NULL != mca_pml_ob1.allocator ) {
         (void)mca_pml_ob1.allocator->alc_finalize(mca_pml_ob1.allocator);
         mca_pml_ob1.allocator = NULL;
diff --git a/ompi/mca/pml/ob1/pml_ob1_component.h b/ompi/mca/pml/ob1/pml_ob1_component.h
index 0e73d2f6806..4e36d85e91d 100644
--- a/ompi/mca/pml/ob1/pml_ob1_component.h
+++ b/ompi/mca/pml/ob1/pml_ob1_component.h
@@ -20,12 +20,14 @@
 #ifndef MCA_PML_OB1_COMPONENT_H
 #define MCA_PML_OB1_COMPONENT_H
 
+#include "ompi/mca/pml/pml.h"
+
 BEGIN_C_DECLS
 
 /*
  * PML module functions.
  */
-OMPI_MODULE_DECLSPEC extern mca_pml_base_component_2_1_0_t mca_pml_ob1_component;
+OMPI_DECLSPEC extern mca_pml_base_component_2_1_0_t mca_pml_ob1_component;
 
 END_C_DECLS
 
diff --git a/ompi/mca/pml/ob1/pml_ob1_cuda.c b/ompi/mca/pml/ob1/pml_ob1_cuda.c
deleted file mode 100644
index a71776e03e0..00000000000
--- a/ompi/mca/pml/ob1/pml_ob1_cuda.c
+++ /dev/null
@@ -1,219 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2008 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      UT-Battelle, LLC. All rights reserved.
- * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved.
- * Copyright (c) 2012-2015 NVIDIA Corporation.  All rights reserved.
- * Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2020      Intel, Inc.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-
-#include "ompi_config.h"
-#include "opal/prefetch.h"
-#include "opal/runtime/opal_params.h"
-#include "opal/mca/btl/btl.h"
-#include "opal/mca/mpool/mpool.h"
-#include "ompi/constants.h"
-#include "ompi/mca/pml/pml.h"
-#include "pml_ob1.h"
-#include "pml_ob1_hdr.h"
-#include "pml_ob1_rdmafrag.h"
-#include "pml_ob1_recvreq.h"
-#include "pml_ob1_sendreq.h"
-#include "ompi/mca/bml/base/base.h"
-
-size_t mca_pml_ob1_rdma_cuda_btls(
-    mca_bml_base_endpoint_t* bml_endpoint,
-    unsigned char* base,
-    size_t size,
-    mca_pml_ob1_com_btl_t* rdma_btls);
-
-int mca_pml_ob1_cuda_need_buffers(void * rreq,
-                                  mca_btl_base_module_t* btl);
-
-void mca_pml_ob1_cuda_add_ipc_support(struct mca_btl_base_module_t* btl, int32_t flags,
-                                      ompi_proc_t* errproc, char* btlinfo);
-
-/**
- * Handle the CUDA buffer.
- */
-int mca_pml_ob1_send_request_start_cuda(mca_pml_ob1_send_request_t* sendreq,
-                                        mca_bml_base_btl_t* bml_btl,
-                                        size_t size) {
-    int rc;
-#if OPAL_CUDA_GDR_SUPPORT
-    /* With some BTLs, switch to RNDV from RGET at large messages */
-    if ((sendreq->req_send.req_base.req_convertor.flags & CONVERTOR_CUDA) &&
-        (sendreq->req_send.req_bytes_packed > (bml_btl->btl->btl_cuda_rdma_limit - sizeof(mca_pml_ob1_hdr_t)))) {
-        return mca_pml_ob1_send_request_start_rndv(sendreq, bml_btl, 0, 0);
-    }
-#endif /* OPAL_CUDA_GDR_SUPPORT */
-
-    sendreq->req_send.req_base.req_convertor.flags &= ~CONVERTOR_CUDA;
-    if (opal_convertor_need_buffers(&sendreq->req_send.req_base.req_convertor) == false) {
-        unsigned char *base;
-        opal_convertor_get_current_pointer( &sendreq->req_send.req_base.req_convertor, (void**)&base );
-        /* Set flag back */
-        sendreq->req_send.req_base.req_convertor.flags |= CONVERTOR_CUDA;
-        if( 0 != (sendreq->req_rdma_cnt = (uint32_t)mca_pml_ob1_rdma_cuda_btls(
-                                                                           sendreq->req_endpoint,
-                                                                           base,
-                                                                           sendreq->req_send.req_bytes_packed,
-                                                                           sendreq->req_rdma))) {
-            rc = mca_pml_ob1_send_request_start_rdma(sendreq, bml_btl,
-                                                     sendreq->req_send.req_bytes_packed);
-            if( OPAL_UNLIKELY(OMPI_SUCCESS != rc) ) {
-                mca_pml_ob1_free_rdma_resources(sendreq);
-            }
-        } else {
-            if (bml_btl->btl_flags & MCA_BTL_FLAGS_CUDA_PUT) {
-                rc = mca_pml_ob1_send_request_start_rndv(sendreq, bml_btl, size,
-                                                         MCA_PML_OB1_HDR_FLAGS_CONTIG);
-            } else {
-                rc = mca_pml_ob1_send_request_start_rndv(sendreq, bml_btl, 0, 0);
-            }
-        }
-    } else {
-        /* Do not send anything with first rendezvous message as copying GPU
-         * memory into RNDV message is expensive. */
-        sendreq->req_send.req_base.req_convertor.flags |= CONVERTOR_CUDA;
-        rc = mca_pml_ob1_send_request_start_rndv(sendreq, bml_btl, 0, 0);
-    }
-    return rc;
-}
-
-
-
-size_t mca_pml_ob1_rdma_cuda_btls(
-    mca_bml_base_endpoint_t* bml_endpoint,
-    unsigned char* base,
-    size_t size,
-    mca_pml_ob1_com_btl_t* rdma_btls)
-{
-    int num_btls = mca_bml_base_btl_array_get_size(&bml_endpoint->btl_send);
-    double weight_total = 0;
-    int num_btls_used = 0, n;
-
-    /* shortcut when there are no rdma capable btls */
-    if(num_btls == 0) {
-        return 0;
-    }
-
-    /* check to see if memory is registered */
-    for(n = 0; n < num_btls && num_btls_used < mca_pml_ob1.max_rdma_per_request;
-            n++) {
-        mca_bml_base_btl_t* bml_btl =
-            mca_bml_base_btl_array_get_index(&bml_endpoint->btl_send, n);
-
-        if (bml_btl->btl_flags & MCA_BTL_FLAGS_CUDA_GET) {
-            mca_btl_base_registration_handle_t *handle = NULL;
-
-            if( NULL != bml_btl->btl->btl_register_mem ) {
-                /* register the memory */
-                handle = bml_btl->btl->btl_register_mem (bml_btl->btl, bml_btl->btl_endpoint,
-                                                         base, size, MCA_BTL_REG_FLAG_CUDA_GPU_MEM |
-                                                         MCA_BTL_REG_FLAG_REMOTE_READ);
-            }
-
-            if(NULL == handle)
-                continue;
-
-            rdma_btls[num_btls_used].bml_btl = bml_btl;
-            rdma_btls[num_btls_used].btl_reg = handle;
-            weight_total += bml_btl->btl_weight;
-            num_btls_used++;
-        }
-    }
-
-    /* if we don't use leave_pinned and all BTLs that already have this memory
-     * registered amount to less then half of available bandwidth - fall back to
-     * pipeline protocol */
-    if(0 == num_btls_used || (!opal_leave_pinned && weight_total < 0.5))
-        return 0;
-
-    mca_pml_ob1_calc_weighted_length(rdma_btls, num_btls_used, size,
-                                     weight_total);
-
-    return num_btls_used;
-}
-
-int mca_pml_ob1_cuda_need_buffers(void * rreq,
-                                  mca_btl_base_module_t* btl)
-{
-    mca_pml_ob1_recv_request_t* recvreq = (mca_pml_ob1_recv_request_t*)rreq;
-    mca_bml_base_endpoint_t* bml_endpoint = mca_bml_base_get_endpoint (recvreq->req_recv.req_base.req_proc);
-    mca_bml_base_btl_t *bml_btl = mca_bml_base_btl_array_find(&bml_endpoint->btl_send, btl);
-
-    /* A btl could be in the rdma list but not in the send list so check there also */
-    if (NULL == bml_btl) {
-        bml_btl = mca_bml_base_btl_array_find(&bml_endpoint->btl_rdma, btl);
-    }
-    /* We should always be able to find back the bml_btl based on the btl */
-    assert(NULL != bml_btl);
-
-    if ((recvreq->req_recv.req_base.req_convertor.flags & CONVERTOR_CUDA) &&
-        (bml_btl->btl_flags & MCA_BTL_FLAGS_CUDA_GET)) {
-        recvreq->req_recv.req_base.req_convertor.flags &= ~CONVERTOR_CUDA;
-        if(opal_convertor_need_buffers(&recvreq->req_recv.req_base.req_convertor) == true) {
-            recvreq->req_recv.req_base.req_convertor.flags |= CONVERTOR_CUDA;
-            return true;
-        } else {
-            recvreq->req_recv.req_base.req_convertor.flags |= CONVERTOR_CUDA;
-            return false;
-        }
-    }
-    return true;
-}
-
-/*
- * This function enables us to start using RDMA get protocol with GPU buffers.
- * We do this by adjusting the flags in the BML structure.  This is not the
- * best thing, but this may go away if CUDA IPC is supported everywhere in the
- * future. */
-void mca_pml_ob1_cuda_add_ipc_support(struct mca_btl_base_module_t* btl, int32_t flags,
-                                      ompi_proc_t* errproc, char* btlinfo)
-{
-    mca_bml_base_endpoint_t* ep;
-    int btl_verbose_stream = 0;
-    int i;
-
-    assert(NULL != errproc);
-    assert(NULL != errproc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]);
-    if (NULL != btlinfo) {
-        btl_verbose_stream = *(int *)btlinfo;
-    }
-    ep = (mca_bml_base_endpoint_t*)errproc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML];
-
-    /* Find the corresponding bml and adjust the flag to support CUDA get */
-    for( i = 0; i < (int)ep->btl_send.arr_size; i++ ) {
-        if( ep->btl_send.bml_btls[i].btl == btl ) {
-            if (4 < opal_output_get_verbosity(btl_verbose_stream)) {
-                char *errhost = opal_get_proc_hostname(&errproc->super);
-                opal_output(0, "BTL %s: rank=%d enabling CUDA IPC "
-                            "to rank=%d on node=%s \n",
-                            btl->btl_component->btl_version.mca_component_name,
-                            OMPI_PROC_MY_NAME->vpid,
-                            ((ompi_process_name_t*)&errproc->super.proc_name)->vpid,
-                            errhost);
-                free(errhost);
-            }
-            ep->btl_send.bml_btls[i].btl_flags |= MCA_BTL_FLAGS_CUDA_GET;
-        }
-    }
-}
diff --git a/ompi/mca/pml/ob1/pml_ob1_hdr.h b/ompi/mca/pml/ob1/pml_ob1_hdr.h
index 716dd841511..62173915932 100644
--- a/ompi/mca/pml/ob1/pml_ob1_hdr.h
+++ b/ompi/mca/pml/ob1/pml_ob1_hdr.h
@@ -49,13 +49,14 @@
 #define MCA_PML_OB1_HDR_TYPE_GET       (MCA_BTL_TAG_PML + 7)
 #define MCA_PML_OB1_HDR_TYPE_PUT       (MCA_BTL_TAG_PML + 8)
 #define MCA_PML_OB1_HDR_TYPE_FIN       (MCA_BTL_TAG_PML + 9)
+#define MCA_PML_OB1_HDR_TYPE_CID       (MCA_BTL_TAG_PML + 10)
 
-#define MCA_PML_OB1_HDR_FLAGS_ACK     1  /* is an ack required */
-#define MCA_PML_OB1_HDR_FLAGS_NBO     2  /* is the hdr in network byte order */
-#define MCA_PML_OB1_HDR_FLAGS_PIN     4  /* is user buffer pinned */
-#define MCA_PML_OB1_HDR_FLAGS_CONTIG  8  /* is user buffer contiguous */
-#define MCA_PML_OB1_HDR_FLAGS_NORDMA  16 /* rest will be send by copy-in-out */
-#define MCA_PML_OB1_HDR_FLAGS_SIGNAL  32 /* message can be optionally signalling */
+#define MCA_PML_OB1_HDR_FLAGS_ACK     0x01  /* is an ack required */
+#define MCA_PML_OB1_HDR_FLAGS_NBO     0x02  /* is the hdr in network byte order */
+#define MCA_PML_OB1_HDR_FLAGS_PIN     0x04  /* is user buffer pinned */
+#define MCA_PML_OB1_HDR_FLAGS_CONTIG  0x08  /* is user buffer contiguous */
+#define MCA_PML_OB1_HDR_FLAGS_NORDMA  0x10  /* rest will be send by copy-in-out */
+#define MCA_PML_OB1_HDR_FLAGS_SIGNAL  0x20  /* message can be optionally signalling */
 
 /**
  * Common hdr attributes - must be first element in each hdr type
@@ -76,6 +77,26 @@ static inline void mca_pml_ob1_common_hdr_prepare (mca_pml_ob1_common_hdr_t *hdr
 #define MCA_PML_OB1_COMMON_HDR_NTOH(h)
 #define MCA_PML_OB1_COMMON_HDR_HTON(h)
 
+/**
+ * Header definition for sending a CID/local comm index combo
+ */
+struct mca_pml_ob1_cid_hdr_t {
+    mca_pml_ob1_common_hdr_t hdr_common;
+    ompi_comm_extended_cid_t hdr_cid;
+    int16_t                  hdr_src_comm_index;
+    int32_t                  hdr_src;
+};
+
+typedef struct mca_pml_ob1_cid_hdr_t mca_pml_ob1_cid_hdr_t;
+
+static inline void mca_pml_ob1_cid_hdr_prepare (mca_pml_ob1_cid_hdr_t *hdr, ompi_communicator_t *comm)
+{
+    mca_pml_ob1_common_hdr_prepare (&hdr->hdr_common, MCA_PML_OB1_HDR_TYPE_CID, 0);
+    hdr->hdr_cid = ompi_comm_get_extended_cid (comm);
+    hdr->hdr_src_comm_index = comm->c_index;
+    hdr->hdr_src = ompi_comm_rank (comm);
+}
+
 /**
  *  Header definition for the first fragment, contains the
  *  attributes required to match the corresponding posted receive.
@@ -130,7 +151,35 @@ do { \
     (h).hdr_seq = htons((h).hdr_seq); \
 } while (0)
 
-/**
+#define MCA_PML_OB1_EXT_MATCH_HDR_NTOH(h) \
+do {\
+    MCA_PML_OB1_COMMON_HDR_NTOH((h).hdr_common); \
+    (h).hdr_cid.cid_base = ntoh64((h).hdr_cid.cid_base); \
+    (h).hdr_cid.cid_sub.u64 = ntoh64((h).hdr_cid.cid_sub.u64); \
+    (h).hdr_src_comm_index = ntohs((h).hdr_src_comm_index); \
+    (h).hdr_src = ntohl((h).hdr_src); \
+} while (0)
+
+#define MCA_PML_OB1_EXT_MATCH_HDR_HTON(h) \
+do {\
+    MCA_PML_OB1_COMMON_HDR_HTON((h).hdr_common); \
+    (h).hdr_cid.cid_base = hton64((h).hdr_cid.cid_base); \
+    (h).hdr_cid.cid_sub.u64 = hton64((h).hdr_cid.cid_sub.u64); \
+    (h).hdr_src_comm_index = htons((h).hdr_src_comm_index); \
+    (h).hdr_src = htonl((h).hdr_src); \
+} while (0)
+
+struct mca_pml_ob1_ext_match_hdr_t {
+    mca_pml_ob1_cid_hdr_t hdr_ext_cid;
+
+    /* actual match */
+    mca_pml_ob1_match_hdr_t hdr_match;
+};
+
+typedef struct mca_pml_ob1_ext_match_hdr_t mca_pml_ob1_ext_match_hdr_t;
+
+/*
+*
  * Header definition for the first fragment when an acknowledgment
  * is required. This could be the first fragment of a large message
  * or a short message that requires an ack (synchronous).
@@ -142,6 +191,14 @@ struct mca_pml_ob1_rendezvous_hdr_t {
 };
 typedef struct mca_pml_ob1_rendezvous_hdr_t mca_pml_ob1_rendezvous_hdr_t;
 
+struct mca_pml_ob1_ext_rendezvous_hdr_t {
+    mca_pml_ob1_cid_hdr_t hdr_ext_cid;
+
+    /* actual match */
+    mca_pml_ob1_rendezvous_hdr_t hdr_rndv;
+};
+typedef struct mca_pml_ob1_ext_rendezvous_hdr_t mca_pml_ob1_ext_rendezvous_hdr_t;
+
 static inline void mca_pml_ob1_rendezvous_hdr_prepare (mca_pml_ob1_rendezvous_hdr_t *hdr, uint8_t hdr_type, uint8_t hdr_flags,
                                                        uint16_t hdr_ctx, int32_t hdr_src, int32_t hdr_tag, uint16_t hdr_seq,
                                                        uint64_t hdr_msg_length, void *hdr_src_req)
@@ -180,6 +237,15 @@ struct mca_pml_ob1_rget_hdr_t {
 };
 typedef struct mca_pml_ob1_rget_hdr_t mca_pml_ob1_rget_hdr_t;
 
+struct mca_pml_ob1_ext_rget_hdr_t {
+    mca_pml_ob1_cid_hdr_t hdr_ext_cid;
+
+    /* actual match */
+    mca_pml_ob1_rget_hdr_t hdr_rget;
+};
+
+typedef struct mca_pml_ob1_ext_rget_hdr_t mca_pml_ob1_ext_rget_hdr_t;
+
 static inline void mca_pml_ob1_rget_hdr_prepare (mca_pml_ob1_rget_hdr_t *hdr, uint8_t hdr_flags,
                                                  uint16_t hdr_ctx, int32_t hdr_src, int32_t hdr_tag, uint16_t hdr_seq,
                                                  uint64_t hdr_msg_length, void *hdr_src_req, void *hdr_frag,
@@ -425,11 +491,16 @@ union mca_pml_ob1_hdr_t {
     mca_pml_ob1_ack_hdr_t hdr_ack;
     mca_pml_ob1_rdma_hdr_t hdr_rdma;
     mca_pml_ob1_fin_hdr_t hdr_fin;
+    /* extended CID support */
+    mca_pml_ob1_cid_hdr_t hdr_cid;
+    mca_pml_ob1_ext_match_hdr_t hdr_ext_match;
+    mca_pml_ob1_ext_rendezvous_hdr_t hdr_ext_rndv;
+    mca_pml_ob1_ext_rget_hdr_t hdr_ext_rget;
 };
 typedef union mca_pml_ob1_hdr_t mca_pml_ob1_hdr_t;
 
 #if !defined(WORDS_BIGENDIAN) && OPAL_ENABLE_HETEROGENEOUS_SUPPORT
-static inline __opal_attribute_always_inline__ void
+static inline void
 ob1_hdr_ntoh(mca_pml_ob1_hdr_t *hdr, const uint8_t hdr_type)
 {
     if(!(hdr->hdr_common.hdr_flags & MCA_PML_OB1_HDR_FLAGS_NBO))
@@ -457,6 +528,15 @@ ob1_hdr_ntoh(mca_pml_ob1_hdr_t *hdr, const uint8_t hdr_type)
         case MCA_PML_OB1_HDR_TYPE_FIN:
             MCA_PML_OB1_FIN_HDR_NTOH(hdr->hdr_fin);
             break;
+        case MCA_PML_OB1_HDR_TYPE_CID:
+	{
+	    mca_pml_ob1_hdr_t *next_hdr = (mca_pml_ob1_hdr_t *) ((uintptr_t) hdr + sizeof (hdr->hdr_cid));
+
+	    MCA_PML_OB1_EXT_MATCH_HDR_NTOH(hdr->hdr_cid);
+	    /* now swap the real header */
+	    ob1_hdr_ntoh (next_hdr, next_hdr->hdr_common.hdr_type);
+	    break;
+	}
         default:
             assert(0);
             break;
@@ -469,7 +549,7 @@ ob1_hdr_ntoh(mca_pml_ob1_hdr_t *hdr, const uint8_t hdr_type)
 #if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
 #define ob1_hdr_hton(h, t, p) \
     ob1_hdr_hton_intr((mca_pml_ob1_hdr_t*)h, t, p)
-static inline __opal_attribute_always_inline__ void
+static inline  void
 ob1_hdr_hton_intr(mca_pml_ob1_hdr_t *hdr, const uint8_t hdr_type,
         const ompi_proc_t *proc)
 {
@@ -503,6 +583,15 @@ ob1_hdr_hton_intr(mca_pml_ob1_hdr_t *hdr, const uint8_t hdr_type,
         case MCA_PML_OB1_HDR_TYPE_FIN:
             MCA_PML_OB1_FIN_HDR_HTON(hdr->hdr_fin);
             break;
+        case MCA_PML_OB1_HDR_TYPE_CID:
+	{
+	    mca_pml_ob1_hdr_t *next_hdr = (mca_pml_ob1_hdr_t *) ((uintptr_t) hdr + sizeof (hdr->hdr_cid));
+
+	    MCA_PML_OB1_EXT_MATCH_HDR_HTON(hdr->hdr_cid);
+	    /* now swap the real header */
+	    ob1_hdr_hton (next_hdr, next_hdr->hdr_common.hdr_type, proc);
+	    break;
+	}
         default:
             assert(0);
             break;
@@ -516,7 +605,8 @@ ob1_hdr_hton_intr(mca_pml_ob1_hdr_t *hdr, const uint8_t hdr_type,
 static inline __opal_attribute_always_inline__ void
 ob1_hdr_copy(mca_pml_ob1_hdr_t *src, mca_pml_ob1_hdr_t *dst)
 {
-    switch(src->hdr_common.hdr_type) {
+    do {
+        switch(src->hdr_common.hdr_type) {
         case MCA_PML_OB1_HDR_TYPE_MATCH:
             memcpy( &(dst->hdr_match), &(src->hdr_match), sizeof(mca_pml_ob1_match_hdr_t) );
             break;
@@ -538,10 +628,24 @@ ob1_hdr_copy(mca_pml_ob1_hdr_t *src, mca_pml_ob1_hdr_t *dst)
         case MCA_PML_OB1_HDR_TYPE_FIN:
             memcpy( &(dst->hdr_fin), &(src->hdr_fin), sizeof(mca_pml_ob1_fin_hdr_t) );
             break;
+        case MCA_PML_OB1_HDR_TYPE_CID:
+	{
+	    mca_pml_ob1_hdr_t *next_src = (mca_pml_ob1_hdr_t *) ((uintptr_t) src + sizeof (src->hdr_cid));
+	    mca_pml_ob1_hdr_t *next_dst = (mca_pml_ob1_hdr_t *) ((uintptr_t) dst + sizeof (dst->hdr_cid));
+
+	    memcpy (&dst->hdr_cid, &src->hdr_cid, sizeof (src->hdr_cid));
+            /* can't call recursively and expect inlining */
+            src = next_src;
+            dst = next_dst;
+            continue;
+	}
         default:
             memcpy( &(dst->hdr_common), &(src->hdr_common), sizeof(mca_pml_ob1_common_hdr_t) );
             break;
-    }
+        }
+
+        break;
+    } while (1);
 }
 
 #endif  /* MCA_PML_OB1_HEADER_H */
diff --git a/ompi/mca/pml/ob1/pml_ob1_iprobe.c b/ompi/mca/pml/ob1/pml_ob1_iprobe.c
index 9175a6cc4d5..4d6a0eb8dfd 100644
--- a/ompi/mca/pml/ob1/pml_ob1_iprobe.c
+++ b/ompi/mca/pml/ob1/pml_ob1_iprobe.c
@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2016 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -40,10 +40,10 @@ int mca_pml_ob1_iprobe(int src,
     MCA_PML_OB1_RECV_REQUEST_START(&recvreq);
 
     if( REQUEST_COMPLETE( &(recvreq.req_recv.req_base.req_ompi)) ) {
-        if( NULL != status ) {
-            *status = recvreq.req_recv.req_base.req_ompi.req_status;
-        }
         rc = recvreq.req_recv.req_base.req_ompi.req_status.MPI_ERROR;
+        if( MPI_STATUS_IGNORE != status ) {
+            OMPI_COPY_STATUS(status, recvreq.req_recv.req_base.req_ompi.req_status, false);
+        }
         *matched = 1;
     } else {
         *matched = 0;
@@ -71,8 +71,8 @@ int mca_pml_ob1_probe(int src,
 
     ompi_request_wait_completion(&recvreq.req_recv.req_base.req_ompi);
     rc = recvreq.req_recv.req_base.req_ompi.req_status.MPI_ERROR;
-    if (NULL != status) {
-        *status = recvreq.req_recv.req_base.req_ompi.req_status;
+    if( MPI_STATUS_IGNORE != status ) {
+        OMPI_COPY_STATUS(status, recvreq.req_recv.req_base.req_ompi.req_status, false);
     }
 
     MCA_PML_BASE_RECV_REQUEST_FINI( &recvreq.req_recv );
@@ -107,8 +107,9 @@ mca_pml_ob1_improbe(int src,
     MCA_PML_OB1_RECV_REQUEST_START(recvreq);
 
     if( REQUEST_COMPLETE( &(recvreq->req_recv.req_base.req_ompi)) ) {
-        if( NULL != status ) {
-            *status = recvreq->req_recv.req_base.req_ompi.req_status;
+        rc = recvreq->req_recv.req_base.req_ompi.req_status.MPI_ERROR;
+        if( MPI_STATUS_IGNORE != status ) {
+            OMPI_COPY_STATUS(status, recvreq->req_recv.req_base.req_ompi.req_status, false);
         }
         *matched = 1;
 
@@ -116,8 +117,6 @@ mca_pml_ob1_improbe(int src,
         (*message)->req_ptr = recvreq;
         (*message)->peer = recvreq->req_recv.req_base.req_ompi.req_status.MPI_SOURCE;
         (*message)->count = recvreq->req_recv.req_base.req_ompi.req_status._ucount;
-
-        rc = recvreq->req_recv.req_base.req_ompi.req_status.MPI_ERROR;
     } else {
         *matched = 0;
 
@@ -162,9 +161,8 @@ mca_pml_ob1_mprobe(int src,
 
     ompi_request_wait_completion(&recvreq->req_recv.req_base.req_ompi);
     rc = recvreq->req_recv.req_base.req_ompi.req_status.MPI_ERROR;
-
-    if( NULL != status ) {
-        *status = recvreq->req_recv.req_base.req_ompi.req_status;
+    if( MPI_STATUS_IGNORE != status ) {
+       OMPI_COPY_STATUS(status, recvreq->req_recv.req_base.req_ompi.req_status, false);
     }
 
     if( OMPI_SUCCESS == rc ) {
diff --git a/ompi/mca/pml/ob1/pml_ob1_irecv.c b/ompi/mca/pml/ob1/pml_ob1_irecv.c
index 3497074743d..4ccb8ea00f2 100644
--- a/ompi/mca/pml/ob1/pml_ob1_irecv.c
+++ b/ompi/mca/pml/ob1/pml_ob1_irecv.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2016 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -65,8 +65,8 @@ int mca_pml_ob1_irecv_init(void *addr,
                              PERUSE_RECV);
 
     /* Work around a leak in start by marking this request as complete. The
-     * problem occured because we do not have a way to differentiate an
-     * inital request and an incomplete pml request in start. This line
+     * problem occurred because we do not have a way to differentiate an
+     * initial request and an incomplete pml request in start. This line
      * allows us to detect this state. */
     recvreq->req_recv.req_base.req_pml_complete = true;
 
@@ -145,8 +145,8 @@ int mca_pml_ob1_recv(void *addr,
         );
     }
 
-    if (NULL != status) {  /* return status */
-        *status = recvreq->req_recv.req_base.req_ompi.req_status;
+    if (MPI_STATUS_IGNORE != status) {
+        OMPI_COPY_STATUS(status, recvreq->req_recv.req_base.req_ompi.req_status, false);
     }
 
     rc = recvreq->req_recv.req_base.req_ompi.req_status.MPI_ERROR;
@@ -362,8 +362,8 @@ mca_pml_ob1_mrecv( void *buf,
 
     MCA_PML_OB1_RECV_FRAG_RETURN(frag);
 
-    if (NULL != status) {  /* return status */
-        *status = recvreq->req_recv.req_base.req_ompi.req_status;
+    if (MPI_STATUS_IGNORE != status) {
+        OMPI_COPY_STATUS(status, recvreq->req_recv.req_base.req_ompi.req_status, false);
     }
     rc = recvreq->req_recv.req_base.req_ompi.req_status.MPI_ERROR;
 #if OPAL_ENABLE_FT_MPI
diff --git a/ompi/mca/pml/ob1/pml_ob1_isend.c b/ompi/mca/pml/ob1/pml_ob1_isend.c
index 2b7e7047708..b3e1741a239 100644
--- a/ompi/mca/pml/ob1/pml_ob1_isend.c
+++ b/ompi/mca/pml/ob1/pml_ob1_isend.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2018 The University of Tennessee and The University
+ * Copyright (c) 2004-2022 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -12,9 +12,10 @@
  *                         All rights reserved.
  * Copyright (c) 2007-2016 Los Alamos National Security, LLC.  All rights
  *                         reserved.
- * Copyright (c) 2014      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2014-2021 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -29,6 +30,9 @@
 #include "pml_ob1_recvreq.h"
 #include "ompi/peruse/peruse-internal.h"
 #include "ompi/runtime/ompi_spc.h"
+#if MPI_VERSION >= 4
+#include "ompi/mca/pml/base/pml_base_sendreq.h"
+#endif
 
 /**
  * Single usage request. As we allow recursive calls (as an
@@ -48,25 +52,22 @@ int mca_pml_ob1_isend_init(const void *buf,
                            ompi_communicator_t * comm,
                            ompi_request_t ** request)
 {
+    mca_pml_ob1_comm_proc_t *ob1_proc = mca_pml_ob1_peer_lookup (comm, dst);
     mca_pml_ob1_send_request_t *sendreq = NULL;
     MCA_PML_OB1_SEND_REQUEST_ALLOC(comm, dst, sendreq);
     if (NULL == sendreq)
         return OMPI_ERR_OUT_OF_RESOURCE;
 
-    MCA_PML_OB1_SEND_REQUEST_INIT(sendreq,
-                                  buf,
-                                  count,
-                                  datatype,
-                                  dst, tag,
-                                  comm, sendmode, true);
+    MCA_PML_OB1_SEND_REQUEST_INIT(sendreq, buf, count, datatype, dst, tag,
+                                  comm, sendmode, true, ob1_proc);
 
     PERUSE_TRACE_COMM_EVENT (PERUSE_COMM_REQ_ACTIVATE,
                              &(sendreq)->req_send.req_base,
                              PERUSE_SEND);
 
     /* Work around a leak in start by marking this request as complete. The
-     * problem occured because we do not have a way to differentiate an
-     * inital request and an incomplete pml request in start. This line
+     * problem occurred because we do not have a way to differentiate an
+     * initial request and an incomplete pml request in start. This line
      * allows us to detect this state. */
     sendreq->req_send.req_base.req_pml_complete = true;
 
@@ -78,7 +79,8 @@ int mca_pml_ob1_isend_init(const void *buf,
 static inline int mca_pml_ob1_send_inline (const void *buf, size_t count,
                                            ompi_datatype_t * datatype,
                                            int dst, int tag, int16_t seqn,
-                                           ompi_proc_t *dst_proc, mca_bml_base_endpoint_t* endpoint,
+                                           ompi_proc_t *dst_proc, mca_pml_ob1_comm_proc_t *ob1_proc,
+                                           mca_bml_base_endpoint_t* endpoint,
                                            ompi_communicator_t * comm)
 {
     mca_pml_ob1_match_hdr_t match;
@@ -88,11 +90,14 @@ static inline int mca_pml_ob1_send_inline (const void *buf, size_t count,
     int rc;
 
     bml_btl = mca_bml_base_btl_array_get_next(&endpoint->btl_eager);
-    if( NULL == bml_btl->btl->btl_sendi)
+    if( NULL == bml_btl || NULL == bml_btl->btl->btl_sendi)
         return OMPI_ERR_NOT_AVAILABLE;
 
     ompi_datatype_type_size (datatype, &size);
-    if ((size * count) > 256) {  /* some random number */
+
+    /* the size used here was picked based on performance on a Cray XE-6. it should probably
+     * be provided by the btl module */
+    if ((size * count) > 256 || -1 == ob1_proc->comm_index) {
         return OMPI_ERR_NOT_AVAILABLE;
     }
 
@@ -111,7 +116,7 @@ static inline int mca_pml_ob1_send_inline (const void *buf, size_t count,
     }
 
     mca_pml_ob1_match_hdr_prepare (&match, MCA_PML_OB1_HDR_TYPE_MATCH, 0,
-                                   comm->c_contextid, comm->c_my_rank,
+                                   ob1_proc->comm_index, comm->c_my_rank,
                                    tag, seqn);
 
     ob1_hdr_hton(&match, MCA_PML_OB1_HDR_TYPE_MATCH, dst_proc);
@@ -138,7 +143,7 @@ static inline int mca_pml_ob1_send_inline (const void *buf, size_t count,
     }
 
     if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
-	return rc;
+        return rc;
     }
 
     return (int) size;
@@ -169,17 +174,23 @@ int mca_pml_ob1_isend(const void *buf,
         return OMPI_ERR_UNREACH;
     }
 
-    if (!OMPI_COMM_CHECK_ASSERT_ALLOW_OVERTAKE(comm)) {
+    if (!OMPI_COMM_CHECK_ASSERT_ALLOW_OVERTAKE(comm) || 0 > tag) {
         seqn = (uint16_t) OPAL_THREAD_ADD_FETCH32(&ob1_proc->send_sequence, 1);
     }
 
     if (MCA_PML_BASE_SEND_SYNCHRONOUS != sendmode) {
-        rc = mca_pml_ob1_send_inline (buf, count, datatype, dst, tag, seqn, dst_proc,
+        rc = mca_pml_ob1_send_inline (buf, count, datatype, dst, tag, seqn, dst_proc, ob1_proc,
                                       endpoint, comm);
         if (OPAL_LIKELY(0 <= rc)) {
             /* NTH: it is legal to return ompi_request_empty since the only valid
              * field in a send completion status is whether or not the send was
              * cancelled (which it can't be at this point anyway). */
+#if MPI_VERSION >= 4
+            if (OPAL_UNLIKELY(OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_NEVER != ompi_pml_base_warn_dep_cancel_send_level)) {
+                *request = &ompi_request_empty_send;
+                return OMPI_SUCCESS;
+            }
+#endif
             *request = &ompi_request_empty;
             return OMPI_SUCCESS;
         }
@@ -194,7 +205,7 @@ int mca_pml_ob1_isend(const void *buf,
                                   count,
                                   datatype,
                                   dst, tag,
-                                  comm, sendmode, false);
+                                  comm, sendmode, false, ob1_proc);
 
     PERUSE_TRACE_COMM_EVENT (PERUSE_COMM_REQ_ACTIVATE,
                              &(sendreq)->req_send.req_base,
@@ -206,7 +217,6 @@ int mca_pml_ob1_isend(const void *buf,
 
 #if OPAL_ENABLE_FT_MPI
 alloc_ft_req:
-#endif /* OPAL_ENABLE_FT_MPI */
     MCA_PML_OB1_SEND_REQUEST_ALLOC(comm, dst, sendreq);
     if (NULL == sendreq)
         return OMPI_ERR_OUT_OF_RESOURCE;
@@ -216,7 +226,7 @@ int mca_pml_ob1_isend(const void *buf,
                                   count,
                                   datatype,
                                   dst, tag,
-                                  comm, sendmode, false);
+                                  comm, sendmode, false, ob1_proc);
 
     PERUSE_TRACE_COMM_EVENT (PERUSE_COMM_REQ_ACTIVATE,
                              &(sendreq)->req_send.req_base,
@@ -224,10 +234,16 @@ int mca_pml_ob1_isend(const void *buf,
 
     /* No point in starting the request, it won't go through, mark completed
      * in error for collection in future wait */
-    sendreq->req_send.req_base.req_ompi.req_status.MPI_ERROR = MPI_ERR_PROC_FAILED;
+    sendreq->req_send.req_base.req_ompi.req_status.MPI_ERROR = ompi_comm_is_revoked(comm)? MPI_ERR_REVOKED: MPI_ERR_PROC_FAILED;
     MCA_PML_OB1_SEND_REQUEST_MPI_COMPLETE(sendreq, false);
+    OPAL_OUTPUT_VERBOSE((2, ompi_ftmpi_output_handle, "Allocating request in error %p (peer %d, seq %" PRIu64 ") with error code %d",
+                         (void*) sendreq,
+                         dst,
+                         sendreq->req_send.req_base.req_sequence,
+                         sendreq->req_send.req_base.req_ompi.req_status.MPI_ERROR));
     *request = (ompi_request_t *) sendreq;
     return OMPI_SUCCESS;
+#endif /* OPAL_ENABLE_FT_MPI */
 }
 
 int mca_pml_ob1_send(const void *buf,
@@ -268,7 +284,7 @@ int mca_pml_ob1_send(const void *buf,
         return OMPI_SUCCESS;
     }
 
-    if (!OMPI_COMM_CHECK_ASSERT_ALLOW_OVERTAKE(comm)) {
+    if (!OMPI_COMM_CHECK_ASSERT_ALLOW_OVERTAKE(comm) || 0 > tag) {
         seqn = (uint16_t) OPAL_THREAD_ADD_FETCH32(&ob1_proc->send_sequence, 1);
     }
 
@@ -279,7 +295,7 @@ int mca_pml_ob1_send(const void *buf,
      */
     if (MCA_PML_BASE_SEND_SYNCHRONOUS != sendmode) {
         rc = mca_pml_ob1_send_inline (buf, count, datatype, dst, tag, seqn, dst_proc,
-                                      endpoint, comm);
+                                      ob1_proc, endpoint, comm);
         if (OPAL_LIKELY(0 <= rc)) {
             return OMPI_SUCCESS;
         }
@@ -299,12 +315,8 @@ int mca_pml_ob1_send(const void *buf,
     sendreq->req_send.req_base.req_proc = dst_proc;
     sendreq->rdma_frag = NULL;
 
-    MCA_PML_OB1_SEND_REQUEST_INIT(sendreq,
-                                  buf,
-                                  count,
-                                  datatype,
-                                  dst, tag,
-                                  comm, sendmode, false);
+    MCA_PML_OB1_SEND_REQUEST_INIT(sendreq, buf, count, datatype, dst, tag,
+                                  comm, sendmode, false, ob1_proc);
 
     PERUSE_TRACE_COMM_EVENT (PERUSE_COMM_REQ_ACTIVATE,
                              &sendreq->req_send.req_base,
diff --git a/ompi/mca/pml/ob1/pml_ob1_progress.c b/ompi/mca/pml/ob1/pml_ob1_progress.c
index b17f2ee0a69..930d5b7311e 100644
--- a/ompi/mca/pml/ob1/pml_ob1_progress.c
+++ b/ompi/mca/pml/ob1/pml_ob1_progress.c
@@ -12,6 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2017      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -23,9 +24,8 @@
 
 #include "pml_ob1.h"
 #include "pml_ob1_sendreq.h"
+#include "pml_ob1_accelerator.h"
 #include "ompi/mca/bml/base/base.h"
-#if OPAL_CUDA_SUPPORT
-#include "opal/mca/common/cuda/common_cuda.h"
 #include "pml_ob1_recvreq.h"
 #include "opal/runtime/opal_params.h"
 
@@ -34,13 +34,13 @@
  * to know when no pending events are expected so that it can
  * unregister the progress function.
  */
-static inline int mca_pml_ob1_process_pending_cuda_async_copies(void)
+static inline int mca_pml_ob1_process_pending_accelerator_async_copies(void)
 {
     mca_btl_base_descriptor_t *frag;
     int progress, count = 0;
 
     do {
-        progress = progress_one_cuda_htod_event(&frag);
+        progress = mca_pml_ob1_progress_one_htod_event(&frag);
         if (1 == progress) {
             /* Call the finish function to make progress. */
             mca_pml_ob1_recv_request_frag_copy_finished(NULL, NULL, frag, 0);
@@ -51,7 +51,6 @@ static inline int mca_pml_ob1_process_pending_cuda_async_copies(void)
 
     return count;
 }
-#endif /* OPAL_CUDA_SUPPORT */
 
 static opal_atomic_int32_t mca_pml_ob1_progress_needed = 0;
 int mca_pml_ob1_enable_progress(int32_t count)
@@ -68,12 +67,9 @@ int mca_pml_ob1_progress(void)
 {
     int i, queue_length = opal_list_get_size(&mca_pml_ob1.send_pending);
     int j, completed_requests = 0;
-    bool send_succedded;
+    bool send_succeeded;
 
-#if OPAL_CUDA_SUPPORT
-    if (opal_cuda_support)
-        completed_requests += mca_pml_ob1_process_pending_cuda_async_copies();
-#endif /* OPAL_CUDA_SUPPORT */
+    completed_requests += mca_pml_ob1_process_pending_accelerator_async_copies();
 
     for( i = 0; i < queue_length; i++ ) {
         mca_pml_ob1_send_pending_t pending_type = MCA_PML_OB1_SEND_PENDING_NONE;
@@ -98,7 +94,7 @@ int mca_pml_ob1_progress(void)
         case MCA_PML_OB1_SEND_PENDING_START:
             MCA_PML_OB1_SEND_REQUEST_RESET(sendreq);
             endpoint = sendreq->req_endpoint;
-            send_succedded = false;
+            send_succeeded = false;
             for(j = 0; j < (int)mca_bml_base_btl_array_get_size(&endpoint->btl_eager); j++) {
                 mca_bml_base_btl_t* bml_btl;
                 int rc;
@@ -107,12 +103,12 @@ int mca_pml_ob1_progress(void)
                 bml_btl = mca_bml_base_btl_array_get_next(&endpoint->btl_eager);
                 rc = mca_pml_ob1_send_request_start_btl(sendreq, bml_btl);
                 if( OPAL_LIKELY(OMPI_SUCCESS == rc) ) {
-                    send_succedded = true;
+                    send_succeeded = true;
                     completed_requests++;
                     break;
                 }
             }
-            if( false == send_succedded ) {
+            if( false == send_succeeded ) {
                 add_request_to_send_pending(sendreq, MCA_PML_OB1_SEND_PENDING_START, true);
             }
         }
diff --git a/ompi/mca/pml/ob1/pml_ob1_rdma.c b/ompi/mca/pml/ob1/pml_ob1_rdma.c
index 8cae10b0322..f27b281959a 100644
--- a/ompi/mca/pml/ob1/pml_ob1_rdma.c
+++ b/ompi/mca/pml/ob1/pml_ob1_rdma.c
@@ -124,8 +124,8 @@ size_t mca_pml_ob1_rdma_pipeline_btls_count (mca_bml_base_endpoint_t* bml_endpoi
         /* NTH: go ahead and use an rdma btl if is the only one */
         bool ignore = !mca_pml_ob1.use_all_rdma;
 
-        for (int i = 0 ; i < num_eager_btls && ignore ; ++i) {
-            mca_bml_base_btl_t *eager_btl = mca_bml_base_btl_array_get_index (&bml_endpoint->btl_eager, i);
+        for (int j = 0 ; j < num_eager_btls && ignore ; ++j) {
+            mca_bml_base_btl_t *eager_btl = mca_bml_base_btl_array_get_index (&bml_endpoint->btl_eager, j);
             if (eager_btl->btl_endpoint == bml_btl->btl_endpoint) {
                 ignore = false;
                 break;
@@ -154,8 +154,8 @@ size_t mca_pml_ob1_rdma_pipeline_btls( mca_bml_base_endpoint_t* bml_endpoint,
         /* NTH: go ahead and use an rdma btl if is the only one */
         bool ignore = !mca_pml_ob1.use_all_rdma;
 
-        for (int i = 0 ; i < num_eager_btls && ignore ; ++i) {
-            mca_bml_base_btl_t *eager_btl = mca_bml_base_btl_array_get_index (&bml_endpoint->btl_eager, i);
+        for (int j = 0 ; j < num_eager_btls && ignore ; ++j) {
+            mca_bml_base_btl_t *eager_btl = mca_bml_base_btl_array_get_index (&bml_endpoint->btl_eager, j);
             if (eager_btl->btl_endpoint == bml_btl->btl_endpoint) {
                 ignore = false;
                 break;
diff --git a/ompi/mca/pml/ob1/pml_ob1_rdma.h b/ompi/mca/pml/ob1/pml_ob1_rdma.h
index 7729043abe6..2cb295dbe03 100644
--- a/ompi/mca/pml/ob1/pml_ob1_rdma.h
+++ b/ompi/mca/pml/ob1/pml_ob1_rdma.h
@@ -25,6 +25,9 @@
 #ifndef MCA_PML_OB1_RDMA_H
 #define MCA_PML_OB1_RDMA_H
 
+#include "ompi/mca/bml/bml.h"
+#include "ompi/mca/pml/ob1/pml_ob1_comm.h"
+
 struct mca_bml_base_endpoint_t;
 
 /*
diff --git a/ompi/mca/pml/ob1/pml_ob1_recvfrag.c b/ompi/mca/pml/ob1/pml_ob1_recvfrag.c
index 3c1e2762e85..a3db1458938 100644
--- a/ompi/mca/pml/ob1/pml_ob1_recvfrag.c
+++ b/ompi/mca/pml/ob1/pml_ob1_recvfrag.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2019 The University of Tennessee and The University
+ * Copyright (c) 2004-2022 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2007 High Performance Computing Center Stuttgart,
@@ -20,6 +20,11 @@
  * Copyright (c) 2018      Sandia National Laboratories
  *                         All rights reserved.
  * Copyright (c) 2020      Google, LLC. All rights reserved.
+ * Copyright (c) 2020-2021 Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2021      Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -49,9 +54,7 @@
 #include "pml_ob1_recvreq.h"
 #include "pml_ob1_sendreq.h"
 #include "pml_ob1_hdr.h"
-#if OPAL_CUDA_SUPPORT
-#include "opal/mca/common/cuda/common_cuda.h"
-#endif /* OPAL_CUDA_SUPPORT */
+#include "pml_ob1_accelerator.h"
 
 OBJ_CLASS_INSTANCE( mca_pml_ob1_buffer_t,
                     opal_free_list_item_t,
@@ -113,7 +116,7 @@ append_frag_to_umq(custom_match_umq *queue, mca_btl_base_module_t *btl,
  * messages. On the vertical layer, messages with contiguous sequence
  * number organize themselves in a way to minimize the search space.
  */
-void append_frag_to_ordered_list (mca_pml_ob1_recv_frag_t **queue,
+void ompi_pml_ob1_append_frag_to_ordered_list (mca_pml_ob1_recv_frag_t **queue,
                                   mca_pml_ob1_recv_frag_t *frag,
                                   uint16_t seq)
 {
@@ -286,7 +289,7 @@ remove_head_from_ordered_list(mca_pml_ob1_recv_frag_t** queue)
  *
  * @param hdr (IN)                  Header of received recv_frag.
  * @param segments (IN)             Received recv_frag descriptor.
- * @param num_segments (IN)         Flag indicating wether a match was made.
+ * @param num_segments (IN)         Flag indicating whether a match was made.
  * @param type (IN)                 Type of the message header.
  * @return                          OMPI_SUCCESS or error status on failure.
  */
@@ -305,7 +308,7 @@ static int mca_pml_ob1_recv_frag_match (mca_btl_base_module_t *btl,
  * @param proc (IN)                 Proc for which we have received the message.
  * @param hdr (IN)                  Header of received recv_frag.
  * @param segments (IN)             Received recv_frag descriptor.
- * @param num_segments (IN)         Flag indicating wether a match was made.
+ * @param num_segments (IN)         Flag indicating whether a match was made.
  * @param type (IN)                 Type of the message header.
  * @return                          OMPI_SUCCESS or error status on failure.
  */
@@ -349,7 +352,7 @@ int mca_pml_ob1_revoke_comm( struct ompi_communicator_t* ompi_comm, bool coll_on
     OBJ_CONSTRUCT(&nack_list, opal_list_t);
 
     OPAL_THREAD_LOCK(&comm->matching_lock);
-    /* these assignement need to be here because we need the matching_lock */
+    /* these assignments need to be here because we need the matching_lock */
     ompi_comm->coll_revoked = true;
     if( !coll_only ) ompi_comm->comm_revoked = true;
 
@@ -366,7 +369,7 @@ int mca_pml_ob1_revoke_comm( struct ompi_communicator_t* ompi_comm, bool coll_on
         /* note this is not an ompi_proc, but a ob1_comm_proc, thus we don't
          * use ompi_proc_is_sentinel to verify if initialized. */
         if( NULL == proc ) continue;
-        /* remove the frag from the unexpected list, add to the nack list 
+        /* remove the frag from the unexpected list, add to the nack list
          * so that we can send the nack as needed to remote cancel the send
          * from outside the match lock.
          */
@@ -381,7 +384,7 @@ int mca_pml_ob1_revoke_comm( struct ompi_communicator_t* ompi_comm, bool coll_on
             }
         }
         /* same for the cantmatch queue/heap; this list is more complicated
-         * Keep it simple: we pop all of the complex list, put the bad items 
+         * Keep it simple: we pop all of the complex list, put the bad items
          * in the nack_list, and keep the good items in the keep_list;
          * then we reinsert the good items in the cantmatch heaplist */
         mca_pml_ob1_recv_frag_t* frag;
@@ -396,7 +399,7 @@ int mca_pml_ob1_revoke_comm( struct ompi_communicator_t* ompi_comm, bool coll_on
             }
         }
         while( NULL != (it = opal_list_remove_first(&keep_list)) ) {
-            append_frag_to_ordered_list(&proc->frags_cant_match, (mca_pml_ob1_recv_frag_t*)it, proc->expected_sequence);
+            ompi_pml_ob1_append_frag_to_ordered_list(&proc->frags_cant_match, (mca_pml_ob1_recv_frag_t*)it, proc->expected_sequence);
         }
         OBJ_DESTRUCT(&keep_list);
     }
@@ -404,8 +407,8 @@ int mca_pml_ob1_revoke_comm( struct ompi_communicator_t* ompi_comm, bool coll_on
 #if OPAL_ENABLE_DEBUG
     if( opal_list_get_size(&nack_list) ) {
         OPAL_OUTPUT_VERBOSE((15, ompi_ftmpi_output_handle,
-                             "ob1_revoke_comm: purging unexpected and cantmatch frags for in comm %d (%s): nacking %zu frags",
-                             ompi_comm->c_contextid, coll_only ? "collective frags only" : "all revoked",
+                             "ob1_revoke_comm: purging unexpected and cantmatch frags for in comm %s (%s): nacking %zu frags",
+                             ompi_comm_print_cid(ompi_comm), coll_only ? "collective frags only" : "all revoked",
                              opal_list_get_size(&nack_list)));
         if( verbose > 15) mca_pml_ob1_dump(ompi_comm, verbose);
     }
@@ -419,7 +422,7 @@ int mca_pml_ob1_revoke_comm( struct ompi_communicator_t* ompi_comm, bool coll_on
             assert( MCA_PML_OB1_HDR_TYPE_RGET == hdr->hdr_common.hdr_type ||
                     MCA_PML_OB1_HDR_TYPE_RNDV == hdr->hdr_common.hdr_type );
             OPAL_OUTPUT_VERBOSE((2, ompi_ftmpi_output_handle,
-                                 "ob1_revoke_comm: sending NACK to %d", hdr->hdr_rndv.hdr_match.hdr_src));
+                                 "ob1_revoke_comm: sending NACK to %d for seq %d", hdr->hdr_rndv.hdr_match.hdr_src, hdr->hdr_rndv.hdr_match.hdr_seq));
             /* Send a ACK with a NULL request to signify revocation */
             proc = mca_pml_ob1_peer_lookup(ompi_comm, hdr->hdr_rndv.hdr_match.hdr_src);
             mca_pml_ob1_recv_request_ack_send(NULL, proc->ompi_proc, hdr->hdr_rndv.hdr_src_req.lval, NULL, 0, 0, false);
@@ -428,7 +431,7 @@ int mca_pml_ob1_revoke_comm( struct ompi_communicator_t* ompi_comm, bool coll_on
             /* if it's a TYPE_MATCH, the sender is not expecting anything
              * from us. So we are done. */
             OPAL_OUTPUT_VERBOSE((15, ompi_ftmpi_output_handle,
-                                 "ob1_revoke_comm: dropping silently frag from %d", hdr->hdr_rndv.hdr_match.hdr_src));
+                                 "ob1_revoke_comm: dropping silently frag from %d for seq %d", hdr->hdr_rndv.hdr_match.hdr_src, hdr->hdr_rndv.hdr_match.hdr_seq));
         }
         MCA_PML_OB1_RECV_FRAG_RETURN(frag);
     }
@@ -437,7 +440,7 @@ int mca_pml_ob1_revoke_comm( struct ompi_communicator_t* ompi_comm, bool coll_on
 }
 #endif /*OPAL_ENABLE_FT_MPI*/
 
-mca_pml_ob1_recv_frag_t *check_cantmatch_for_match (mca_pml_ob1_comm_proc_t *proc)
+mca_pml_ob1_recv_frag_t *ompi_pml_ob1_check_cantmatch_for_match (mca_pml_ob1_comm_proc_t *proc)
 {
     mca_pml_ob1_recv_frag_t *frag = proc->frags_cant_match;
 
@@ -476,8 +479,8 @@ void mca_pml_ob1_recv_frag_callback_match (mca_btl_base_module_t *btl,
          * this pending queue will be searched and all matching fragments
          * moved to the right communicator.
          */
-        append_frag_to_list( &mca_pml_ob1.non_existing_communicator_pending,
-                             btl, hdr, segments, num_segments, NULL );
+        append_frag_to_list( &mca_pml_ob1.non_existing_communicator_pending, btl,
+                             hdr, segments, num_segments, NULL );
         return;
     }
     comm = (mca_pml_ob1_comm_t *)comm_ptr->c_pml_comm;
@@ -498,7 +501,7 @@ void mca_pml_ob1_recv_frag_callback_match (mca_btl_base_module_t *btl,
      * run, lock to make sure that if another thread is processing
      * a frag from the same message a match is made only once.
      * Also, this prevents other posted receives (for a pair of
-     * end points) from being processed, and potentially "loosing"
+     * end points) from being processed, and potentially "losing"
      * the fragment.
      */
     OB1_MATCHING_LOCK(&comm->matching_lock);
@@ -515,7 +518,7 @@ void mca_pml_ob1_recv_frag_callback_match (mca_btl_base_module_t *btl,
     }
 #endif
 
-    if (!OMPI_COMM_CHECK_ASSERT_ALLOW_OVERTAKE(comm_ptr)) {
+    if (!OMPI_COMM_CHECK_ASSERT_ALLOW_OVERTAKE(comm_ptr) || 0 > hdr->hdr_tag) {
         /* get sequence number of next message that can be processed.
          * If this frag is out of sequence, queue it up in the list
          * now as we still have the lock.
@@ -524,7 +527,7 @@ void mca_pml_ob1_recv_frag_callback_match (mca_btl_base_module_t *btl,
             mca_pml_ob1_recv_frag_t* frag;
             MCA_PML_OB1_RECV_FRAG_ALLOC(frag);
             MCA_PML_OB1_RECV_FRAG_INIT(frag, hdr, segments, num_segments, btl);
-            append_frag_to_ordered_list(&proc->frags_cant_match, frag, proc->expected_sequence);
+            ompi_pml_ob1_append_frag_to_ordered_list(&proc->frags_cant_match, frag, proc->expected_sequence);
             SPC_RECORD(OMPI_SPC_OUT_OF_SEQUENCE, 1);
             OB1_MATCHING_UNLOCK(&comm->matching_lock);
             return;
@@ -568,7 +571,7 @@ void mca_pml_ob1_recv_frag_callback_match (mca_btl_base_module_t *btl,
             uint32_t iov_count = 1;
 
             /*
-             *  Make user buffer accessable(defined) before unpacking.
+             *  Make user buffer accessible(defined) before unpacking.
              */
             MEMCHECKER(
                        memchecker_call(&opal_memchecker_base_mem_defined,
@@ -594,7 +597,7 @@ void mca_pml_ob1_recv_frag_callback_match (mca_btl_base_module_t *btl,
             SPC_USER_OR_MPI(match->req_recv.req_base.req_ompi.req_status.MPI_TAG, (ompi_spc_value_t)bytes_received,
                             OMPI_SPC_BYTES_RECEIVED_USER, OMPI_SPC_BYTES_RECEIVED_MPI);
             /*
-             *  Unpacking finished, make the user buffer unaccessable again.
+             *  Unpacking finished, make the user buffer unaccessible again.
              */
             MEMCHECKER(
                        memchecker_call(&opal_memchecker_base_mem_noaccess,
@@ -621,7 +624,7 @@ void mca_pml_ob1_recv_frag_callback_match (mca_btl_base_module_t *btl,
         mca_pml_ob1_recv_frag_t* frag;
 
         OB1_MATCHING_LOCK(&comm->matching_lock);
-        if((frag = check_cantmatch_for_match(proc))) {
+        if((frag = ompi_pml_ob1_check_cantmatch_for_match(proc))) {
             /* mca_pml_ob1_recv_frag_match_proc() will release the lock. */
             mca_pml_ob1_recv_frag_match_proc(frag->btl, comm_ptr, proc,
                                              &frag->hdr.hdr_match,
@@ -633,6 +636,38 @@ void mca_pml_ob1_recv_frag_callback_match (mca_btl_base_module_t *btl,
     }
 }
 
+/**
+ * Merge all out of sequence fragments into the matching queue, as if they were received now.
+ */
+int mca_pml_ob1_merge_cant_match( ompi_communicator_t * ompi_comm )
+{
+    mca_pml_ob1_comm_t * pml_comm = (mca_pml_ob1_comm_t *)ompi_comm->c_pml_comm;
+    mca_pml_ob1_recv_frag_t *frag, *frags_cant_match;
+    mca_pml_ob1_comm_proc_t* proc;
+    int cnt = 0;
+
+    OB1_MATCHING_LOCK(&pml_comm->matching_lock);
+    for (uint32_t i = 0; i < pml_comm->num_procs; i++) {
+        if ((NULL == (proc = pml_comm->procs[i])) || (NULL != proc->frags_cant_match)) {
+            continue;
+        }
+
+        /* Acquire all cant_match frags from the peer */
+        frags_cant_match = proc->frags_cant_match;
+        proc->frags_cant_match = NULL;
+        while(NULL != (frag = remove_head_from_ordered_list(&frags_cant_match))) {
+            /* mca_pml_ob1_recv_frag_match_proc() will release the lock. */
+            mca_pml_ob1_recv_frag_match_proc(frag->btl, ompi_comm, proc,
+                                             &frag->hdr.hdr_match,
+                                             frag->segments, frag->num_segments,
+                                             frag->hdr.hdr_match.hdr_common.hdr_type, frag);
+            OB1_MATCHING_LOCK(&pml_comm->matching_lock);
+            cnt++;
+        }
+    }
+    OB1_MATCHING_UNLOCK(&pml_comm->matching_lock);
+    return cnt;
+}
 
 void mca_pml_ob1_recv_frag_callback_rndv (mca_btl_base_module_t *btl,
                                           const mca_btl_base_receive_descriptor_t *descriptor)
@@ -681,7 +716,7 @@ void mca_pml_ob1_recv_frag_callback_ack (mca_btl_base_module_t *btl,
 #if OPAL_ENABLE_FT_MPI
     /* if the req_recv is NULL, the comm has been revoked at the receiver */
     if( OPAL_UNLIKELY(NULL == sendreq->req_recv.pval) ) {
-        OPAL_OUTPUT_VERBOSE((2, ompi_ftmpi_output_handle, "Recvfrag: Received a NACK to the RDV/RGET match to %d on comm %d\n", sendreq->req_send.req_base.req_peer, sendreq->req_send.req_base.req_comm->c_contextid));
+        OPAL_OUTPUT_VERBOSE((2, ompi_ftmpi_output_handle, "Recvfrag: Received a NACK to the RDV/RGET match to %d for seq %" PRIu64 " on comm %s\n", sendreq->req_send.req_base.req_peer, sendreq->req_send.req_base.req_sequence, ompi_comm_print_cid(sendreq->req_send.req_base.req_comm)));
         if (NULL != sendreq->rdma_frag) {
             MCA_PML_OB1_RDMA_FRAG_RETURN(sendreq->rdma_frag);
             sendreq->rdma_frag = NULL;
@@ -723,16 +758,15 @@ void mca_pml_ob1_recv_frag_callback_ack (mca_btl_base_module_t *btl,
         OPAL_THREAD_ADD_FETCH32(&sendreq->req_state, -1);
     }
 
-#if OPAL_CUDA_SUPPORT /* CUDA_ASYNC_SEND */
-    if ((sendreq->req_send.req_base.req_convertor.flags & CONVERTOR_CUDA) &&
-        (btl->btl_flags & MCA_BTL_FLAGS_CUDA_COPY_ASYNC_SEND)) {
+    if ((sendreq->req_send.req_base.req_convertor.flags & CONVERTOR_ACCELERATOR) &&
+        (btl->btl_flags & MCA_BTL_FLAGS_ACCELERATOR_COPY_ASYNC_SEND)) {
         /* The user's buffer is GPU and this BTL can support asynchronous copies,
          * so adjust the convertor accordingly.  All the subsequent fragments will
          * use the asynchronous copy. */
-        void *strm = mca_common_cuda_get_dtoh_stream();
-        opal_cuda_set_copy_function_async(&sendreq->req_send.req_base.req_convertor, strm);
+        opal_accelerator_stream_t *stream = mca_pml_ob1_get_dtoh_stream();
+        sendreq->req_send.req_base.req_convertor.flags |= CONVERTOR_ACCELERATOR_ASYNC;
+        sendreq->req_send.req_base.req_convertor.stream = stream;
     }
-#endif /* OPAL_CUDA_SUPPORT */
 
     if (send_request_pml_complete_check(sendreq) == false)
         mca_pml_ob1_send_request_schedule(sendreq);
@@ -751,12 +785,11 @@ void mca_pml_ob1_recv_frag_callback_frag (mca_btl_base_module_t *btl,
 
     ob1_hdr_ntoh((mca_pml_ob1_hdr_t*)hdr, MCA_PML_OB1_HDR_TYPE_FRAG);
     recvreq = (mca_pml_ob1_recv_request_t*)hdr->hdr_frag.hdr_dst_req.pval;
-#if OPAL_CUDA_SUPPORT /* CUDA_ASYNC_RECV */
     /* If data is destined for GPU buffer and convertor was set up for asynchronous
      * copies, then start the copy and return.  The copy completion will trigger
      * the next phase. */
-    if (recvreq->req_recv.req_base.req_convertor.flags & CONVERTOR_CUDA_ASYNC) {
-        assert(btl->btl_flags & MCA_BTL_FLAGS_CUDA_COPY_ASYNC_RECV);
+    if (recvreq->req_recv.req_base.req_convertor.flags & CONVERTOR_ACCELERATOR_ASYNC) {
+        assert(btl->btl_flags & MCA_BTL_FLAGS_ACCELERATOR_COPY_ASYNC_RECV);
 
         /* This will trigger the opal_convertor_pack to start asynchronous copy. */
         mca_pml_ob1_recv_request_frag_copy_start(recvreq, btl, segments, descriptor->des_segment_count, NULL);
@@ -766,7 +799,6 @@ void mca_pml_ob1_recv_frag_callback_frag (mca_btl_base_module_t *btl,
 
         return;
     }
-#endif /* OPAL_CUDA_SUPPORT */
 
     mca_pml_ob1_recv_request_progress_frag(recvreq,btl,segments,descriptor->des_segment_count);
 }
@@ -988,7 +1020,7 @@ static mca_pml_ob1_recv_request_t *match_one (mca_btl_base_module_t *btl,
  * RCS/CTS receive side matching
  *
  * @param hdr list of parameters needed for matching
- *                    This list is also embeded in frag,
+ *                    This list is also embedded in frag,
  *                    but this allows to save a memory copy when
  *                    a match is made in this routine. (IN)
  * @param frag   pointer to receive fragment which we want
@@ -1037,8 +1069,8 @@ static int mca_pml_ob1_recv_frag_match (mca_btl_base_module_t *btl,
          * this pending queue will be searched and all matching fragments
          * moved to the right communicator.
          */
-        append_frag_to_list( &mca_pml_ob1.non_existing_communicator_pending,
-                             btl, hdr, segments, num_segments, NULL );
+        append_frag_to_list( &mca_pml_ob1.non_existing_communicator_pending, btl,
+                             hdr, segments, num_segments, NULL );
         return OMPI_SUCCESS;
     }
     comm = (mca_pml_ob1_comm_t *)comm_ptr->c_pml_comm;
@@ -1059,7 +1091,7 @@ static int mca_pml_ob1_recv_frag_match (mca_btl_base_module_t *btl,
      * run, lock to make sure that if another thread is processing
      * a frag from the same message a match is made only once.
      * Also, this prevents other posted receives (for a pair of
-     * end points) from being processed, and potentially "loosing"
+     * end points) from being processed, and potentially "losing"
      * the fragment.
      */
     OB1_MATCHING_LOCK(&comm->matching_lock);
@@ -1074,7 +1106,7 @@ static int mca_pml_ob1_recv_frag_match (mca_btl_base_module_t *btl,
             /* Send a ACK with a NULL request to signify revocation */
             mca_pml_ob1_rendezvous_hdr_t* hdr_rndv = (mca_pml_ob1_rendezvous_hdr_t*) hdr;
             mca_pml_ob1_recv_request_ack_send(NULL, proc->ompi_proc, hdr_rndv->hdr_src_req.lval, NULL, 0, 0, false);
-            OPAL_OUTPUT_VERBOSE((2, ompi_ftmpi_output_handle, "Recvfrag: comm %d is revoked or collectives force errors, sending a NACK to the RDV/RGET match from %d\n", hdr->hdr_ctx, hdr->hdr_src));
+            OPAL_OUTPUT_VERBOSE((2, ompi_ftmpi_output_handle, "Recvfrag: comm %d is revoked or collectives force errors, sending a NACK to the RDV/RGET match from %d for seq %d\n", hdr->hdr_ctx, hdr->hdr_src, hdr->hdr_seq));
         }
         else {
             OPAL_OUTPUT_VERBOSE((15, ompi_ftmpi_output_handle,
@@ -1088,13 +1120,13 @@ static int mca_pml_ob1_recv_frag_match (mca_btl_base_module_t *btl,
     frag_msg_seq = hdr->hdr_seq;
     next_msg_seq_expected = (uint16_t)proc->expected_sequence;
 
-    if (!OMPI_COMM_CHECK_ASSERT_ALLOW_OVERTAKE(comm_ptr)) {
+    if (!OMPI_COMM_CHECK_ASSERT_ALLOW_OVERTAKE(comm_ptr) || 0 > hdr->hdr_tag) {
         /* If the sequence number is wrong, queue it up for later. */
         if(OPAL_UNLIKELY(frag_msg_seq != next_msg_seq_expected)) {
             mca_pml_ob1_recv_frag_t* frag;
             MCA_PML_OB1_RECV_FRAG_ALLOC(frag);
             MCA_PML_OB1_RECV_FRAG_INIT(frag, hdr, segments, num_segments, btl);
-            append_frag_to_ordered_list(&proc->frags_cant_match, frag, next_msg_seq_expected);
+            ompi_pml_ob1_append_frag_to_ordered_list(&proc->frags_cant_match, frag, next_msg_seq_expected);
 
             SPC_RECORD(OMPI_SPC_OUT_OF_SEQUENCE, 1);
             SPC_RECORD(OMPI_SPC_OOS_IN_QUEUE, 1);
@@ -1183,11 +1215,11 @@ mca_pml_ob1_recv_frag_match_proc (mca_btl_base_module_t *btl,
     /*
      * Now that new message has arrived, check to see if
      * any fragments on the frags_cant_match list
-     * may now be used to form new matchs
+     * may now be used to form new matches
      */
     if(OPAL_UNLIKELY(NULL != proc->frags_cant_match)) {
         OB1_MATCHING_LOCK(&comm->matching_lock);
-        if((frag = check_cantmatch_for_match(proc))) {
+        if((frag = ompi_pml_ob1_check_cantmatch_for_match(proc))) {
             hdr = &frag->hdr.hdr_match;
             segments = frag->segments;
             num_segments = frag->num_segments;
@@ -1201,3 +1233,71 @@ mca_pml_ob1_recv_frag_match_proc (mca_btl_base_module_t *btl,
     return OMPI_SUCCESS;
 }
 
+void mca_pml_ob1_handle_cid (ompi_communicator_t *comm, int src, mca_pml_ob1_cid_hdr_t *hdr_cid)
+{
+    mca_pml_ob1_comm_proc_t *ob1_proc = mca_pml_ob1_peer_lookup (comm, src);
+    bool had_comm_index = (-1 != ob1_proc->comm_index);
+
+    if (!had_comm_index) {
+        /* avoid sending too many extra packets. if this doesn't work well then a flag can be added to
+         * the proc to indicate that this packet has been sent */
+        ob1_proc->comm_index = hdr_cid->hdr_src_comm_index;
+
+        /*
+         * if the proc to send to is myself,  no need to do the send
+         */
+        if(ob1_proc->ompi_proc != ompi_proc_local()) {
+            (void) mca_pml_ob1_send_cid (ob1_proc->ompi_proc, comm);
+        }
+    }
+}
+
+void mca_pml_ob1_recv_frag_callback_cid (mca_btl_base_module_t* btl,
+                                         const mca_btl_base_receive_descriptor_t* des)
+{
+    mca_btl_base_segment_t segments[MCA_BTL_DES_MAX_SEGMENTS];
+    mca_pml_ob1_hdr_t *hdr = (mca_pml_ob1_hdr_t *) des->des_segments[0].seg_addr.pval;
+    mca_pml_ob1_match_hdr_t *hdr_match = &hdr->hdr_ext_match.hdr_match;
+    size_t num_segments = des->des_segment_count;
+    ompi_communicator_t *comm;
+
+    memcpy (segments, des->des_segments, num_segments * sizeof (segments[0]));
+    assert (segments->seg_len >= sizeof (hdr->hdr_cid));
+
+    ob1_hdr_ntoh (hdr, hdr->hdr_common.hdr_type);
+
+    /* NTH: this should be ok as as all BTLs create a dummy segment */
+    segments->seg_len -= offsetof (mca_pml_ob1_ext_match_hdr_t, hdr_match);
+    segments->seg_addr.pval = (void *) hdr_match;
+
+    /* find the communicator with this extended CID */
+    comm = ompi_comm_lookup_cid (hdr->hdr_cid.hdr_cid);
+    if (OPAL_UNLIKELY(NULL == comm)) {
+        if (segments->seg_len > 0) {
+            /* This is a special case. A message for a not yet existing
+             * communicator can happens. Instead of doing a matching we
+             * will temporarily add it the a pending queue in the PML.
+             * Later on, when the communicator is completely instantiated,
+             * this pending queue will be searched and all matching fragments
+             * moved to the right communicator.
+             */
+            append_frag_to_list (&mca_pml_ob1.non_existing_communicator_pending,
+                                 btl, (const mca_pml_ob1_match_hdr_t *)hdr, des->des_segments,
+                                 num_segments, NULL);
+        }
+
+        /* nothing more to do */
+        return;
+    }
+
+    mca_pml_ob1_handle_cid (comm, hdr->hdr_cid.hdr_src, &hdr->hdr_cid);
+    hdr_match->hdr_ctx = comm->c_index;
+
+    if (segments->seg_len == 0) {
+        /* just a response */
+        return;
+    }
+
+    mca_pml_ob1_recv_frag_match (btl, hdr_match, segments, des->des_segment_count,
+                                 hdr_match->hdr_common.hdr_type);
+}
diff --git a/ompi/mca/pml/ob1/pml_ob1_recvfrag.h b/ompi/mca/pml/ob1/pml_ob1_recvfrag.h
index f803046fb16..00d869d4d42 100644
--- a/ompi/mca/pml/ob1/pml_ob1_recvfrag.h
+++ b/ompi/mca/pml/ob1/pml_ob1_recvfrag.h
@@ -15,6 +15,7 @@
  * Copyright (c) 2012-2015 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2020      Google, LLC. All rights reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -28,7 +29,8 @@
 #ifndef MCA_PML_OB1_RECVFRAG_H
 #define MCA_PML_OB1_RECVFRAG_H
 
-#include "pml_ob1_hdr.h"
+#include "ompi/mca/pml/ob1/pml_ob1_comm.h"
+#include "ompi/mca/pml/ob1/pml_ob1_hdr.h"
 
 BEGIN_C_DECLS
 
@@ -158,16 +160,30 @@ extern void mca_pml_ob1_recv_frag_callback_put (mca_btl_base_module_t *btl,
 extern void mca_pml_ob1_recv_frag_callback_fin (mca_btl_base_module_t *btl,
                                                 const mca_btl_base_receive_descriptor_t *descriptor);
 
+/**
+ * Callback from BTL on receipt of an extended CID header
+ */
+extern void mca_pml_ob1_recv_frag_callback_cid( mca_btl_base_module_t *btl,
+                                                const mca_btl_base_receive_descriptor_t* descriptor);
+
 /**
  * Extract the next fragment from the cant_match ordered list. This fragment
  * will be the next in sequence.
  */
 extern mca_pml_ob1_recv_frag_t*
-check_cantmatch_for_match(mca_pml_ob1_comm_proc_t *proc);
+ompi_pml_ob1_check_cantmatch_for_match(mca_pml_ob1_comm_proc_t *proc);
+
+/**
+ * Move for all peers all pending cant_match fragments into the matching queues. This
+ * function is necessary when allow_overtake info key is transition to set.
+ */
+int mca_pml_ob1_merge_cant_match( ompi_communicator_t * ompi_comm );
+
+void ompi_pml_ob1_append_frag_to_ordered_list(mca_pml_ob1_recv_frag_t** queue,
+                                                  mca_pml_ob1_recv_frag_t* frag,
+                                                  uint16_t seq);
 
-void append_frag_to_ordered_list(mca_pml_ob1_recv_frag_t** queue,
-                                 mca_pml_ob1_recv_frag_t* frag,
-                                 uint16_t seq);
+void mca_pml_ob1_handle_cid (ompi_communicator_t *comm, int src, mca_pml_ob1_cid_hdr_t *hdr_cid);
 
 extern void mca_pml_ob1_dump_cant_match(mca_pml_ob1_recv_frag_t* queue);
 END_C_DECLS
diff --git a/ompi/mca/pml/ob1/pml_ob1_recvreq.c b/ompi/mca/pml/ob1/pml_ob1_recvreq.c
index cd089c01db3..57aba677a8a 100644
--- a/ompi/mca/pml/ob1/pml_ob1_recvreq.c
+++ b/ompi/mca/pml/ob1/pml_ob1_recvreq.c
@@ -21,6 +21,9 @@
  * Copyright (c) 2018      Sandia National Laboratories
  *                         All rights reserved.
  * Copyright (c) 2020      Google, LLC. All rights reserved.
+ * Copyright (c) 2021      Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -40,16 +43,11 @@
 #include "pml_ob1_recvfrag.h"
 #include "pml_ob1_sendreq.h"
 #include "pml_ob1_rdmafrag.h"
+#include "pml_ob1_accelerator.h"
 #include "ompi/mca/bml/base/base.h"
 
-#if OPAL_CUDA_SUPPORT
-#include "opal/mca/common/cuda/common_cuda.h"
-#endif /* OPAL_CUDA_SUPPORT */
-
-#if OPAL_CUDA_SUPPORT
-int mca_pml_ob1_cuda_need_buffers(mca_pml_ob1_recv_request_t* recvreq,
-                                  mca_btl_base_module_t* btl);
-#endif /* OPAL_CUDA_SUPPORT */
+int mca_pml_ob1_accelerator_need_buffers(mca_pml_ob1_recv_request_t* recvreq,
+                                         mca_btl_base_module_t* btl);
 
 void mca_pml_ob1_recv_request_process_pending(void)
 {
@@ -282,6 +280,10 @@ int mca_pml_ob1_recv_request_ack_send_btl(
     return OMPI_ERR_OUT_OF_RESOURCE;
 }
 
+/*
+ *
+ */
+
 static int mca_pml_ob1_recv_request_ack(
     mca_pml_ob1_recv_request_t* recvreq,
     mca_btl_base_module_t* btl,
@@ -300,11 +302,15 @@ static int mca_pml_ob1_recv_request_ack(
         /*
          * lookup request buffer to determine if memory is already
          * registered.
+         *
+         * We only want this code to be hit if these buffers don't need to be packed
+         * and they're not device buffers.
          */
-
-        if(opal_convertor_need_buffers(&recvreq->req_recv.req_base.req_convertor) == 0 &&
-           hdr->hdr_match.hdr_common.hdr_flags & MCA_PML_OB1_HDR_FLAGS_CONTIG &&
-           rdma_num != 0) {
+        if (opal_convertor_need_buffers(&recvreq->req_recv.req_base.req_convertor) == 0 &&
+            !(recvreq->req_recv.req_base.req_convertor.flags & CONVERTOR_ACCELERATOR) &&
+            !(recvreq->req_recv.req_base.req_convertor.flags & CONVERTOR_ACCELERATOR_UNIFIED) &&
+            hdr->hdr_match.hdr_common.hdr_flags & MCA_PML_OB1_HDR_FLAGS_CONTIG &&
+            rdma_num != 0) {
             unsigned char *base;
             opal_convertor_get_current_pointer( &recvreq->req_recv.req_base.req_convertor, (void**)&(base) );
 
@@ -343,7 +349,7 @@ static int mca_pml_ob1_recv_request_ack(
             return OMPI_SUCCESS;
     }
 
-    /* let know to shedule function there is no need to put ACK flag. If not all message went over
+    /* let know to schedule function there is no need to put ACK flag. If not all message went over
      * RDMA then we cancel the GET protocol in order to switch back to send/recv. In this case send
      * back the remote send request, the peer kept a pointer to the frag locally. In the future we
      * might want to cancel the fragment itself, in which case we will have to send back the remote
@@ -376,7 +382,7 @@ static int mca_pml_ob1_recv_request_get_frag_failed (mca_pml_ob1_rdma_frag_t *fr
         }
     }
 
-    if (++frag->retries < mca_pml_ob1.rdma_retries_limit &&
+    if (frag->retries < mca_pml_ob1.rdma_retries_limit &&
         OMPI_ERR_OUT_OF_RESOURCE == rc) {
         OPAL_THREAD_LOCK(&mca_pml_ob1.lock);
         opal_list_append(&mca_pml_ob1.rdma_pending, (opal_list_item_t*)frag);
@@ -407,6 +413,7 @@ static void mca_pml_ob1_rget_completion (mca_btl_base_module_t* btl, struct mca_
     /* check completion status */
     if (OPAL_UNLIKELY(OMPI_SUCCESS != status)) {
         status = mca_pml_ob1_recv_request_get_frag_failed (frag, status);
+        /* fragment was returned or queue by the above call */
         if (OPAL_UNLIKELY(OMPI_SUCCESS != status)) {
             size_t skipped_bytes = recvreq->req_send_offset - recvreq->req_rdma_offset;
             opal_output_verbose(mca_pml_ob1_output, 1, "pml:ob1: %s: operation failed with code %d", __func__, status);
@@ -429,12 +436,12 @@ static void mca_pml_ob1_rget_completion (mca_btl_base_module_t* btl, struct mca_
         mca_pml_ob1_send_fin (recvreq->req_recv.req_base.req_proc,
                               bml_btl, frag->rdma_hdr.hdr_rget.hdr_frag,
                               frag->rdma_length, 0, 0);
+
+        MCA_PML_OB1_RDMA_FRAG_RETURN(frag);
     }
 
     recv_request_pml_complete_check(recvreq);
 
-    MCA_PML_OB1_RDMA_FRAG_RETURN(frag);
-
     MCA_PML_OB1_PROGRESS_PENDING(bml_btl);
 }
 
@@ -577,7 +584,7 @@ void mca_pml_ob1_recv_request_progress_frag( mca_pml_ob1_recv_request_t* recvreq
                                      bytes_received,
                                      bytes_delivered );
     /*
-     *  Unpacking finished, make the user buffer unaccessable again.
+     *  Unpacking finished, make the user buffer unaccessible again.
      */
     MEMCHECKER(
                memchecker_call(&opal_memchecker_base_mem_noaccess,
@@ -597,7 +604,6 @@ void mca_pml_ob1_recv_request_progress_frag( mca_pml_ob1_recv_request_t* recvreq
     }
 }
 
-#if OPAL_CUDA_SUPPORT /* CUDA_ASYNC_RECV */
 /**
  * This function is basically the first half of the code in the
  * mca_pml_ob1_recv_request_progress_frag function.  This fires off
@@ -637,7 +643,7 @@ void mca_pml_ob1_recv_request_frag_copy_start( mca_pml_ob1_recv_request_t* recvr
     /* Then record an event that will get triggered by a PML progress call which
      * checks the stream events.  If we get an error, abort.  Should get message
      * from CUDA code about what went wrong. */
-    result = mca_common_cuda_record_htod_event("pml", des);
+    result = mca_pml_ob1_record_htod_event("pml", des);
     if (OMPI_SUCCESS != result) {
         opal_output(0, "%s:%d FATAL", __FILE__, __LINE__);
         ompi_rte_abort(-1, NULL);
@@ -675,7 +681,6 @@ void mca_pml_ob1_recv_request_frag_copy_finished( mca_btl_base_module_t* btl,
         mca_pml_ob1_recv_request_schedule(recvreq, NULL);
     }
 }
-#endif /* OPAL_CUDA_SUPPORT */
 
 /*
  * Update the recv request status to reflect the number of bytes
@@ -706,9 +711,7 @@ void mca_pml_ob1_recv_request_progress_rget( mca_pml_ob1_recv_request_t* recvreq
      * sender side is already registered. We need to be smarter here, perhaps
      * do couple of RDMA reads */
     if (opal_convertor_need_buffers(&recvreq->req_recv.req_base.req_convertor) == true) {
-#if OPAL_CUDA_SUPPORT
-        if (mca_pml_ob1_cuda_need_buffers(recvreq, btl))
-#endif /* OPAL_CUDA_SUPPORT */
+        if (mca_pml_ob1_accelerator_need_buffers(recvreq, btl))
         {
             mca_pml_ob1_recv_request_ack(recvreq, btl, &hdr->hdr_rndv, 0);
             return;
@@ -719,13 +722,12 @@ void mca_pml_ob1_recv_request_progress_rget( mca_pml_ob1_recv_request_t* recvreq
     bml_endpoint = mca_bml_base_get_endpoint (recvreq->req_recv.req_base.req_proc);
     rdma_bml = mca_bml_base_btl_array_find(&bml_endpoint->btl_rdma, btl);
 
-#if OPAL_CUDA_SUPPORT
     if (OPAL_UNLIKELY(NULL == rdma_bml)) {
-        if (recvreq->req_recv.req_base.req_convertor.flags & CONVERTOR_CUDA) {
+        if (recvreq->req_recv.req_base.req_convertor.flags & CONVERTOR_ACCELERATOR) {
             mca_bml_base_btl_t *bml_btl;
             bml_btl = mca_bml_base_btl_array_find(&bml_endpoint->btl_send, btl);
-            /* Check to see if this is a CUDA get */
-            if (bml_btl->btl_flags & MCA_BTL_FLAGS_CUDA_GET) {
+            /* Check to see if this is an accelerator get */
+            if (bml_btl->btl_flags & MCA_BTL_FLAGS_ACCELERATOR_GET) {
                 rdma_bml = bml_btl;
             }
         } else {
@@ -734,7 +736,6 @@ void mca_pml_ob1_recv_request_progress_rget( mca_pml_ob1_recv_request_t* recvreq
             return;
         }
     }
-#endif /* OPAL_CUDA_SUPPORT */
 
     if (OPAL_UNLIKELY(NULL == rdma_bml)) {
         opal_output(0, "[%s:%d] invalid bml for rdma get", __FILE__, __LINE__);
@@ -752,7 +753,7 @@ void mca_pml_ob1_recv_request_progress_rget( mca_pml_ob1_recv_request_t* recvreq
         void *data_ptr;
         uint32_t flags = MCA_BTL_REG_FLAG_LOCAL_WRITE | MCA_BTL_REG_FLAG_REMOTE_WRITE;
 #if OPAL_CUDA_GDR_SUPPORT
-        if (recvreq->req_recv.req_base.req_convertor.flags & CONVERTOR_CUDA) {
+        if (recvreq->req_recv.req_base.req_convertor.flags & CONVERTOR_ACCELERATOR) {
             flags |= MCA_BTL_REG_FLAG_CUDA_GPU_MEM;
         }
 #endif /* OPAL_CUDA_GDR_SUPPORT */
@@ -890,16 +891,14 @@ void mca_pml_ob1_recv_request_progress_rndv( mca_pml_ob1_recv_request_t* recvreq
         mca_pml_ob1_recv_request_schedule(recvreq, NULL);
     }
 
-#if OPAL_CUDA_SUPPORT /* CUDA_ASYNC_RECV */
-    /* If BTL supports it and this is a CUDA buffer being received into,
+    /* If BTL supports it and this is an ACCELERATOR buffer being received into,
      * have all subsequent FRAGS copied in asynchronously. */
-    if ((recvreq->req_recv.req_base.req_convertor.flags & CONVERTOR_CUDA) &&
-        (btl->btl_flags & MCA_BTL_FLAGS_CUDA_COPY_ASYNC_RECV)) {
-        void *strm = mca_common_cuda_get_htod_stream();
-        opal_cuda_set_copy_function_async(&recvreq->req_recv.req_base.req_convertor, strm);
+    if ((recvreq->req_recv.req_base.req_convertor.flags & CONVERTOR_ACCELERATOR) &&
+        (btl->btl_flags & MCA_BTL_FLAGS_ACCELERATOR_COPY_ASYNC_RECV)) {
+        opal_accelerator_stream_t *stream = mca_pml_ob1_get_htod_stream();
+        recvreq->req_recv.req_base.req_convertor.flags |= CONVERTOR_ACCELERATOR_ASYNC;
+        recvreq->req_recv.req_base.req_convertor.stream = stream;
     }
-#endif
-
 }
 
 /*
@@ -922,7 +921,7 @@ void mca_pml_ob1_recv_request_progress_match( mca_pml_ob1_recv_request_t* recvre
 
     MCA_PML_OB1_RECV_REQUEST_MATCHED(recvreq, &hdr->hdr_match);
     /*
-     *  Make user buffer accessable(defined) before unpacking.
+     *  Make user buffer accessible(defined) before unpacking.
      */
     MEMCHECKER(
                memchecker_call(&opal_memchecker_base_mem_defined,
@@ -938,7 +937,7 @@ void mca_pml_ob1_recv_request_progress_match( mca_pml_ob1_recv_request_t* recvre
                                      bytes_received,
                                      bytes_delivered);
     /*
-     *  Unpacking finished, make the user buffer unaccessable again.
+     *  Unpacking finished, make the user buffer unaccessible again.
      */
     MEMCHECKER(
                memchecker_call(&opal_memchecker_base_mem_noaccess,
@@ -1198,8 +1197,8 @@ recv_req_match_wild( mca_pml_ob1_recv_request_t* req,
                      mca_pml_ob1_comm_proc_t **p)
 #endif
 {
-    mca_pml_ob1_comm_t* comm = req->req_recv.req_base.req_comm->c_pml_comm;
-    mca_pml_ob1_comm_proc_t **procp = comm->procs;
+    mca_pml_ob1_comm_t *comm = (mca_pml_ob1_comm_t *) req->req_recv.req_base.req_comm->c_pml_comm;
+    mca_pml_ob1_comm_proc_t **procp = (mca_pml_ob1_comm_proc_t **) comm->procs;
 
 #if MCA_PML_OB1_CUSTOM_MATCH
     mca_pml_ob1_recv_frag_t* frag;
@@ -1303,8 +1302,8 @@ void mca_pml_ob1_recv_req_start(mca_pml_ob1_recv_request_t *req)
         ompi_communicator_t* comm_ptr = req->req_recv.req_base.req_comm;
         if( ((ompi_comm_is_revoked(comm_ptr) && !ompi_request_tag_is_ft(req->req_recv.req_base.req_tag) )
          || (ompi_comm_coll_revoked(comm_ptr) && ompi_request_tag_is_collective(req->req_recv.req_base.req_tag)))) {
-            OPAL_OUTPUT_VERBOSE((2, ompi_ftmpi_output_handle, "Recvreq: Posting a new recv req peer %d, tag %d on a revoked/coll_revoked communicator %d, discarding it.\n",
-                req->req_recv.req_base.req_peer, req->req_recv.req_base.req_tag, comm_ptr->c_contextid));
+            OPAL_OUTPUT_VERBOSE((2, ompi_ftmpi_output_handle, "Recvreq: Posting a new recv req peer %d, tag %d on a revoked/coll_revoked communicator %s, discarding it.\n",
+                req->req_recv.req_base.req_peer, req->req_recv.req_base.req_tag, ompi_comm_print_cid(comm_ptr)));
             req->req_recv.req_base.req_ompi.req_status.MPI_ERROR = ompi_comm_is_revoked(comm_ptr)? MPI_ERR_REVOKED: MPI_ERR_PROC_FAILED;
             recv_request_pml_complete( req );
             PERUSE_TRACE_COMM_EVENT(PERUSE_COMM_SEARCH_UNEX_Q_END,
diff --git a/ompi/mca/pml/ob1/pml_ob1_recvreq.h b/ompi/mca/pml/ob1/pml_ob1_recvreq.h
index b0b89388dea..402d3f4dcec 100644
--- a/ompi/mca/pml/ob1/pml_ob1_recvreq.h
+++ b/ompi/mca/pml/ob1/pml_ob1_recvreq.h
@@ -234,8 +234,8 @@ static inline void prepare_recv_req_converter(mca_pml_ob1_recv_request_t *req)
                 req->req_recv.req_base.req_addr,
                 0,
                 &req->req_recv.req_base.req_convertor);
-        opal_convertor_get_unpacked_size(&req->req_recv.req_base.req_convertor,
-                                         &req->req_bytes_expected);
+        opal_convertor_get_packed_size(&req->req_recv.req_base.req_convertor,
+                                       &req->req_bytes_expected);
     }
 }
 
@@ -331,7 +331,6 @@ void mca_pml_ob1_recv_request_progress_frag(
     const mca_btl_base_segment_t* segments,
     size_t num_segments);
 
-#if OPAL_CUDA_SUPPORT
 void mca_pml_ob1_recv_request_frag_copy_start(
     mca_pml_ob1_recv_request_t* req,
     struct mca_btl_base_module_t* btl,
@@ -343,7 +342,6 @@ void mca_pml_ob1_recv_request_frag_copy_finished(struct mca_btl_base_module_t* b
     struct mca_btl_base_endpoint_t* ep,
     struct mca_btl_base_descriptor_t* des,
     int status );
-#endif /* OPAL_CUDA_SUPPORT */
 /**
  *
  */
diff --git a/ompi/mca/pml/ob1/pml_ob1_sendreq.c b/ompi/mca/pml/ob1/pml_ob1_sendreq.c
index bae8fc10bc9..0dd246917c0 100644
--- a/ompi/mca/pml/ob1/pml_ob1_sendreq.c
+++ b/ompi/mca/pml/ob1/pml_ob1_sendreq.c
@@ -19,6 +19,10 @@
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2018-2019 Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      Google, LLC. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -125,6 +129,10 @@ static int mca_pml_ob1_send_request_free(struct ompi_request_t** request)
 
 static int mca_pml_ob1_send_request_cancel(struct ompi_request_t* request, int complete)
 {
+#if MPI_VERSION >= 4
+    mca_pml_cancel_send_callback(request, complete);
+#endif
+
 #if OPAL_ENABLE_FT_MPI
     ompi_communicator_t* comm = request->req_mpi_object.comm;
     mca_pml_ob1_send_request_t* pml_req = (mca_pml_ob1_send_request_t*)request;
@@ -445,7 +453,6 @@ mca_pml_ob1_frag_completion( mca_btl_base_module_t* btl,
     MCA_PML_OB1_PROGRESS_PENDING(bml_btl);
 }
 
-#if OPAL_CUDA_SUPPORT /* CUDA_ASYNC_SEND */
 /**
  * This function is called when the copy of the frag from the GPU buffer
  * to the internal buffer is complete.  Used to support asynchronous
@@ -461,7 +468,7 @@ mca_pml_ob1_copy_frag_completion( mca_btl_base_module_t* btl,
     mca_bml_base_btl_t* bml_btl = (mca_bml_base_btl_t*) des->des_context;
 
     des->des_cbfunc = mca_pml_ob1_frag_completion;
-    /* Reset the BTL onwership flag as the BTL can free it after completion. */
+    /* Reset the BTL ownership flag as the BTL can free it after completion. */
     des->des_flags |= MCA_BTL_DES_FLAGS_BTL_OWNERSHIP;
     OPAL_OUTPUT((-1, "copy_frag_completion FRAG frag=%p", (void *)des));
     /* Currently, we cannot support a failure in the send.  In the blocking
@@ -476,7 +483,6 @@ mca_pml_ob1_copy_frag_completion( mca_btl_base_module_t* btl,
         ompi_rte_abort(-1, NULL);
     }
 }
-#endif /* OPAL_CUDA_SUPPORT */
 
 /**
  *  Buffer the entire message and mark as complete.
@@ -487,18 +493,23 @@ int mca_pml_ob1_send_request_start_buffered(
     mca_bml_base_btl_t* bml_btl,
     size_t size)
 {
+    const bool need_ext_match = MCA_PML_OB1_SEND_REQUEST_REQUIRES_EXT_MATCH(sendreq);
+    size_t hdr_size = sizeof (mca_pml_ob1_rendezvous_hdr_t);
     mca_btl_base_descriptor_t* des;
     mca_btl_base_segment_t* segment;
     mca_pml_ob1_hdr_t* hdr;
+    mca_pml_ob1_rendezvous_hdr_t *hdr_rndv;
     struct iovec iov;
     unsigned int iov_count;
     size_t max_data, req_bytes_delivered;
     int rc;
 
+    if (OPAL_UNLIKELY(need_ext_match)) {
+        hdr_size = sizeof (hdr->hdr_ext_rndv);
+    }
+
     /* allocate descriptor */
-    mca_bml_base_alloc(bml_btl, &des,
-                       MCA_BTL_NO_ORDER,
-                       sizeof(mca_pml_ob1_rendezvous_hdr_t) + size,
+    mca_bml_base_alloc(bml_btl, &des, MCA_BTL_NO_ORDER, hdr_size + size,
                        MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP |
                        MCA_BTL_DES_FLAGS_SIGNAL);
     if( OPAL_UNLIKELY(NULL == des) ) {
@@ -507,8 +518,7 @@ int mca_pml_ob1_send_request_start_buffered(
     segment = des->des_segments;
 
     /* pack the data into the BTL supplied buffer */
-    iov.iov_base = (IOVBASE_TYPE*)((unsigned char*)segment->seg_addr.pval +
-                                    sizeof(mca_pml_ob1_rendezvous_hdr_t));
+    iov.iov_base = (IOVBASE_TYPE*)((unsigned char*)segment->seg_addr.pval + hdr_size);
     iov.iov_len = size;
     iov_count = 1;
     max_data = size;
@@ -523,17 +533,24 @@ int mca_pml_ob1_send_request_start_buffered(
 
     /* build rendezvous header */
     hdr = (mca_pml_ob1_hdr_t*)segment->seg_addr.pval;
-    mca_pml_ob1_rendezvous_hdr_prepare (&hdr->hdr_rndv, MCA_PML_OB1_HDR_TYPE_RNDV, 0,
-                                        sendreq->req_send.req_base.req_comm->c_contextid,
+    if (OPAL_UNLIKELY(need_ext_match)) {
+        hdr_rndv = &hdr->hdr_ext_rndv.hdr_rndv;
+        mca_pml_ob1_cid_hdr_prepare (&hdr->hdr_cid, sendreq->req_send.req_base.req_comm);
+    } else {
+        hdr_rndv = &hdr->hdr_rndv;
+    }
+
+    mca_pml_ob1_rendezvous_hdr_prepare (hdr_rndv, MCA_PML_OB1_HDR_TYPE_RNDV, 0,
+                                        sendreq->ob1_proc->comm_index,
                                         sendreq->req_send.req_base.req_comm->c_my_rank,
                                         sendreq->req_send.req_base.req_tag,
                                         (uint16_t)sendreq->req_send.req_base.req_sequence,
                                         sendreq->req_send.req_bytes_packed, sendreq);
 
-    ob1_hdr_hton(hdr, MCA_PML_OB1_HDR_TYPE_RNDV, sendreq->req_send.req_base.req_proc);
+    ob1_hdr_hton(hdr, hdr->hdr_common.hdr_type, sendreq->req_send.req_base.req_proc);
 
     /* update lengths */
-    segment->seg_len = sizeof(mca_pml_ob1_rendezvous_hdr_t) + max_data;
+    segment->seg_len = hdr_size + max_data;
 
     des->des_cbfunc = mca_pml_ob1_rndv_completion;
     des->des_cbdata = sendreq;
@@ -571,7 +588,7 @@ int mca_pml_ob1_send_request_start_buffered(
     MCA_PML_OB1_SEND_REQUEST_MPI_COMPLETE(sendreq, true);
 
     /* send */
-    rc = mca_bml_base_send(bml_btl, des, MCA_PML_OB1_HDR_TYPE_RNDV);
+    rc = mca_bml_base_send (bml_btl, des, hdr->hdr_common.hdr_type);
     if( OPAL_LIKELY( rc >= 0 ) ) {
         if( OPAL_LIKELY( 1 == rc ) ) {
             mca_pml_ob1_rndv_completion_request( bml_btl, sendreq, req_bytes_delivered);
@@ -593,18 +610,22 @@ int mca_pml_ob1_send_request_start_copy( mca_pml_ob1_send_request_t* sendreq,
                                          mca_bml_base_btl_t* bml_btl,
                                          size_t size )
 {
+    const bool need_ext_match = MCA_PML_OB1_SEND_REQUEST_REQUIRES_EXT_MATCH(sendreq);
+    size_t hdr_size = OMPI_PML_OB1_MATCH_HDR_LEN;
     mca_btl_base_descriptor_t* des = NULL;
     mca_btl_base_segment_t* segment;
     mca_pml_ob1_hdr_t* hdr;
+    mca_pml_ob1_match_hdr_t *hdr_match;
     struct iovec iov;
     unsigned int iov_count;
     size_t max_data = size;
     int rc;
 
-    if(NULL != bml_btl->btl->btl_sendi) {
+    if(NULL != bml_btl->btl->btl_sendi && !need_ext_match) {
         mca_pml_ob1_match_hdr_t match;
+
         mca_pml_ob1_match_hdr_prepare (&match, MCA_PML_OB1_HDR_TYPE_MATCH, 0,
-                                       sendreq->req_send.req_base.req_comm->c_contextid,
+                                       sendreq->ob1_proc->comm_index,
                                        sendreq->req_send.req_base.req_comm->c_my_rank,
                                        sendreq->req_send.req_base.req_tag,
                                        (uint16_t)sendreq->req_send.req_base.req_sequence);
@@ -632,9 +653,11 @@ int mca_pml_ob1_send_request_start_copy( mca_pml_ob1_send_request_t* sendreq,
         }
     } else {
         /* allocate descriptor */
-        mca_bml_base_alloc( bml_btl, &des,
-                            MCA_BTL_NO_ORDER,
-                            OMPI_PML_OB1_MATCH_HDR_LEN + size,
+        if (OPAL_UNLIKELY(need_ext_match)) {
+            hdr_size += sizeof (hdr->hdr_cid);
+        }
+
+        mca_bml_base_alloc (bml_btl, &des, MCA_BTL_NO_ORDER, hdr_size + size,
                             MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
     }
     if( OPAL_UNLIKELY(NULL == des) ) {
@@ -645,8 +668,7 @@ int mca_pml_ob1_send_request_start_copy( mca_pml_ob1_send_request_t* sendreq,
 
     if(size > 0) {
         /* pack the data into the supplied buffer */
-        iov.iov_base = (IOVBASE_TYPE*)((unsigned char*)segment->seg_addr.pval +
-                                       OMPI_PML_OB1_MATCH_HDR_LEN);
+        iov.iov_base = (IOVBASE_TYPE*)((unsigned char*)segment->seg_addr.pval + hdr_size);
         iov.iov_len  = size;
         iov_count    = 1;
         /*
@@ -662,7 +684,7 @@ int mca_pml_ob1_send_request_start_copy( mca_pml_ob1_send_request_t* sendreq,
         (void)opal_convertor_pack( &sendreq->req_send.req_base.req_convertor,
                                    &iov, &iov_count, &max_data );
          /*
-          *  Packing finished, make the user buffer unaccessable.
+          *  Packing finished, make the user buffer unaccessible.
           */
         MEMCHECKER(
             memchecker_call(&opal_memchecker_base_mem_noaccess,
@@ -672,26 +694,32 @@ int mca_pml_ob1_send_request_start_copy( mca_pml_ob1_send_request_t* sendreq,
         );
     }
 
-
     /* build match header */
     hdr = (mca_pml_ob1_hdr_t*)segment->seg_addr.pval;
-    mca_pml_ob1_match_hdr_prepare (&hdr->hdr_match, MCA_PML_OB1_HDR_TYPE_MATCH, 0,
-                                   sendreq->req_send.req_base.req_comm->c_contextid,
+    if (OPAL_UNLIKELY(need_ext_match)) {
+        hdr_match = &hdr->hdr_ext_match.hdr_match;
+        mca_pml_ob1_cid_hdr_prepare (&hdr->hdr_cid, sendreq->req_send.req_base.req_comm);
+    } else {
+        hdr_match = &hdr->hdr_match;
+    }
+
+    mca_pml_ob1_match_hdr_prepare (hdr_match, MCA_PML_OB1_HDR_TYPE_MATCH, 0,
+                                   sendreq->ob1_proc->comm_index,
                                    sendreq->req_send.req_base.req_comm->c_my_rank,
                                    sendreq->req_send.req_base.req_tag,
                                    (uint16_t)sendreq->req_send.req_base.req_sequence);
 
-    ob1_hdr_hton(hdr, MCA_PML_OB1_HDR_TYPE_MATCH, sendreq->req_send.req_base.req_proc);
+    ob1_hdr_hton(hdr, hdr->hdr_common.hdr_type, sendreq->req_send.req_base.req_proc);
 
     /* update lengths */
-    segment->seg_len = OMPI_PML_OB1_MATCH_HDR_LEN + max_data;
+    segment->seg_len = hdr_size + max_data;
 
     /* short message */
     des->des_cbdata = sendreq;
     des->des_cbfunc = mca_pml_ob1_match_completion_free;
 
     /* send */
-    rc = mca_bml_base_send_status(bml_btl, des, MCA_PML_OB1_HDR_TYPE_MATCH);
+    rc = mca_bml_base_send_status(bml_btl, des, hdr->hdr_common.hdr_type);
     SPC_USER_OR_MPI(sendreq->req_send.req_base.req_ompi.req_status.MPI_TAG, (ompi_spc_value_t)size,
                     OMPI_SPC_BYTES_SENT_USER, OMPI_SPC_BYTES_SENT_MPI);
     if( OPAL_LIKELY( rc >= OPAL_SUCCESS ) ) {
@@ -720,19 +748,23 @@ int mca_pml_ob1_send_request_start_prepare( mca_pml_ob1_send_request_t* sendreq,
                                             mca_bml_base_btl_t* bml_btl,
                                             size_t size )
 {
+    const bool need_ext_match = MCA_PML_OB1_SEND_REQUEST_REQUIRES_EXT_MATCH(sendreq);
+    size_t hdr_size = OMPI_PML_OB1_MATCH_HDR_LEN;
     mca_btl_base_descriptor_t* des;
     mca_btl_base_segment_t* segment;
     mca_pml_ob1_hdr_t* hdr;
+    mca_pml_ob1_match_hdr_t *hdr_match;
     int rc;
 
+    if (OPAL_UNLIKELY(need_ext_match)) {
+        hdr_size += sizeof (hdr->hdr_cid);
+    }
+
     /* prepare descriptor */
-    mca_bml_base_prepare_src( bml_btl,
-                              &sendreq->req_send.req_base.req_convertor,
-                              MCA_BTL_NO_ORDER,
-                              OMPI_PML_OB1_MATCH_HDR_LEN,
-                              &size,
+    mca_bml_base_prepare_src (bml_btl, &sendreq->req_send.req_base.req_convertor,
+                              MCA_BTL_NO_ORDER, hdr_size, &size,
                               MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP,
-                              &des );
+                              &des);
     if( OPAL_UNLIKELY(NULL == des) ) {
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
@@ -740,20 +772,27 @@ int mca_pml_ob1_send_request_start_prepare( mca_pml_ob1_send_request_t* sendreq,
 
     /* build match header */
     hdr = (mca_pml_ob1_hdr_t*)segment->seg_addr.pval;
-    mca_pml_ob1_match_hdr_prepare (&hdr->hdr_match, MCA_PML_OB1_HDR_TYPE_MATCH, 0,
-                                   sendreq->req_send.req_base.req_comm->c_contextid,
+    if (OPAL_UNLIKELY(need_ext_match)) {
+        hdr_match = &hdr->hdr_ext_match.hdr_match;
+        mca_pml_ob1_cid_hdr_prepare (&hdr->hdr_cid, sendreq->req_send.req_base.req_comm);
+    } else {
+        hdr_match = &hdr->hdr_match;
+    }
+
+    mca_pml_ob1_match_hdr_prepare (hdr_match, MCA_PML_OB1_HDR_TYPE_MATCH, 0,
+                                   sendreq->ob1_proc->comm_index,
                                    sendreq->req_send.req_base.req_comm->c_my_rank,
                                    sendreq->req_send.req_base.req_tag,
                                    (uint16_t)sendreq->req_send.req_base.req_sequence);
 
-    ob1_hdr_hton(hdr, MCA_PML_OB1_HDR_TYPE_MATCH, sendreq->req_send.req_base.req_proc);
+    ob1_hdr_hton(hdr, hdr->hdr_common.hdr_type, sendreq->req_send.req_base.req_proc);
 
     /* short message */
     des->des_cbfunc = mca_pml_ob1_match_completion_free;
     des->des_cbdata = sendreq;
 
     /* send */
-    rc = mca_bml_base_send(bml_btl, des, MCA_PML_OB1_HDR_TYPE_MATCH);
+    rc = mca_bml_base_send(bml_btl, des, hdr->hdr_common.hdr_type);
     SPC_USER_OR_MPI(sendreq->req_send.req_base.req_ompi.req_status.MPI_TAG, (ompi_spc_value_t)size,
                     OMPI_SPC_BYTES_SENT_USER, OMPI_SPC_BYTES_SENT_MPI);
     if( OPAL_LIKELY( rc >= OPAL_SUCCESS ) ) {
@@ -768,7 +807,7 @@ int mca_pml_ob1_send_request_start_prepare( mca_pml_ob1_send_request_t* sendreq,
 
 
 /**
- *  We have contigous data that is registered - schedule across
+ *  We have contiguous data that is registered - schedule across
  *  available nics.
  */
 
@@ -782,16 +821,18 @@ int mca_pml_ob1_send_request_start_rdma( mca_pml_ob1_send_request_t* sendreq,
      * one RDMA capable BTLs). This way round robin distribution of RDMA
      * operation is achieved.
      */
+    const bool need_ext_match = MCA_PML_OB1_SEND_REQUEST_REQUIRES_EXT_MATCH(sendreq);
+    size_t reg_size, hdr_size = sizeof (mca_pml_ob1_rget_hdr_t);
     mca_btl_base_registration_handle_t *local_handle;
     mca_btl_base_descriptor_t *des;
     mca_pml_ob1_rdma_frag_t *frag;
-    mca_pml_ob1_rget_hdr_t *hdr;
-    size_t reg_size;
+    mca_pml_ob1_hdr_t *hdr;
+    mca_pml_ob1_rget_hdr_t *hdr_rget;
     void *data_ptr;
     int rc;
 
     bml_btl = sendreq->req_rdma[0].bml_btl;
-    if (!(bml_btl->btl_flags & (MCA_BTL_FLAGS_GET | MCA_BTL_FLAGS_CUDA_GET))) {
+    if (!(bml_btl->btl_flags & (MCA_BTL_FLAGS_GET | MCA_BTL_FLAGS_ACCELERATOR_GET))) {
         sendreq->rdma_frag = NULL;
         /* This BTL does not support get. Use rendezvous to start the RDMA operation using put instead. */
         return mca_pml_ob1_send_request_start_rndv (sendreq, bml_btl, 0, MCA_PML_OB1_HDR_FLAGS_CONTIG |
@@ -818,10 +859,15 @@ int mca_pml_ob1_send_request_start_rdma( mca_pml_ob1_send_request_t* sendreq,
     frag->cbfunc = mca_pml_ob1_rget_completion;
     /* do not store the local handle in the fragment. it will be released by mca_pml_ob1_free_rdma_resources */
 
+    if (OPAL_UNLIKELY(need_ext_match)) {
+        hdr_size = sizeof (hdr->hdr_ext_rget);
+    }
+
     reg_size = bml_btl->btl->btl_registration_handle_size;
+    hdr_size += reg_size;
 
     /* allocate space for get hdr + segment list */
-    mca_bml_base_alloc(bml_btl, &des, MCA_BTL_NO_ORDER, sizeof (*hdr) + reg_size,
+    mca_bml_base_alloc(bml_btl, &des, MCA_BTL_NO_ORDER, hdr_size,
                        MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP |
                        MCA_BTL_DES_FLAGS_SIGNAL);
     if( OPAL_UNLIKELY(NULL == des) ) {
@@ -834,17 +880,24 @@ int mca_pml_ob1_send_request_start_rdma( mca_pml_ob1_send_request_t* sendreq,
     sendreq->rdma_frag = frag;
 
     /* build match header */
-    hdr = (mca_pml_ob1_rget_hdr_t *) des->des_segments->seg_addr.pval;
+    hdr = (mca_pml_ob1_hdr_t *) des->des_segments->seg_addr.pval;
+    if (need_ext_match) {
+        hdr_rget = &hdr->hdr_ext_rget.hdr_rget;
+        mca_pml_ob1_cid_hdr_prepare (&hdr->hdr_cid, sendreq->req_send.req_base.req_comm);
+    } else {
+        hdr_rget = &hdr->hdr_rget;
+    }
+
     /* TODO -- Add support for multiple segments for get */
-    mca_pml_ob1_rget_hdr_prepare (hdr, MCA_PML_OB1_HDR_FLAGS_CONTIG | MCA_PML_OB1_HDR_FLAGS_PIN,
-                                  sendreq->req_send.req_base.req_comm->c_contextid,
+    mca_pml_ob1_rget_hdr_prepare (hdr_rget, MCA_PML_OB1_HDR_FLAGS_CONTIG | MCA_PML_OB1_HDR_FLAGS_PIN,
+                                  sendreq->ob1_proc->comm_index,
                                   sendreq->req_send.req_base.req_comm->c_my_rank,
                                   sendreq->req_send.req_base.req_tag,
                                   (uint16_t)sendreq->req_send.req_base.req_sequence,
                                   sendreq->req_send.req_bytes_packed, sendreq,
                                   frag, data_ptr, local_handle, reg_size);
 
-    ob1_hdr_hton(hdr, MCA_PML_OB1_HDR_TYPE_RGET, sendreq->req_send.req_base.req_proc);
+    ob1_hdr_hton(hdr, hdr->hdr_common.hdr_type, sendreq->req_send.req_base.req_proc);
 
     des->des_cbfunc = mca_pml_ob1_send_ctl_completion;
     des->des_cbdata = sendreq;
@@ -860,7 +913,7 @@ int mca_pml_ob1_send_request_start_rdma( mca_pml_ob1_send_request_t* sendreq,
     }
 
     /* send */
-    rc = mca_bml_base_send(bml_btl, des, MCA_PML_OB1_HDR_TYPE_RGET);
+    rc = mca_bml_base_send(bml_btl, des, hdr->hdr_common.hdr_type);
     if (OPAL_UNLIKELY(rc < 0)) {
         MCA_PML_OB1_RDMA_FRAG_RETURN(frag);
         sendreq->rdma_frag = NULL;
@@ -882,18 +935,22 @@ int mca_pml_ob1_send_request_start_rndv( mca_pml_ob1_send_request_t* sendreq,
                                          size_t size,
                                          int flags )
 {
+    const bool need_ext_match = MCA_PML_OB1_SEND_REQUEST_REQUIRES_EXT_MATCH(sendreq);
+    size_t hdr_size = sizeof (mca_pml_ob1_rendezvous_hdr_t);
     mca_btl_base_descriptor_t* des;
     mca_btl_base_segment_t* segment;
     mca_pml_ob1_hdr_t* hdr;
+    mca_pml_ob1_rendezvous_hdr_t *hdr_rndv;
     int rc;
 
+    if (OPAL_UNLIKELY(need_ext_match)) {
+        hdr_size = sizeof (hdr->hdr_ext_rndv);
+    }
+
     /* prepare descriptor */
     if(size == 0) {
-        mca_bml_base_alloc( bml_btl,
-                            &des,
-                            MCA_BTL_NO_ORDER,
-                            sizeof(mca_pml_ob1_rendezvous_hdr_t),
-                            MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP );
+        mca_bml_base_alloc (bml_btl, &des, MCA_BTL_NO_ORDER, hdr_size, MCA_BTL_DES_FLAGS_PRIORITY |
+                            MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
     } else {
         MEMCHECKER(
             memchecker_call(&opal_memchecker_base_mem_defined,
@@ -901,14 +958,10 @@ int mca_pml_ob1_send_request_start_rndv( mca_pml_ob1_send_request_t* sendreq,
                             sendreq->req_send.req_base.req_count,
                             sendreq->req_send.req_base.req_datatype);
         );
-        mca_bml_base_prepare_src( bml_btl,
-                                  &sendreq->req_send.req_base.req_convertor,
-                                  MCA_BTL_NO_ORDER,
-                                  sizeof(mca_pml_ob1_rendezvous_hdr_t),
-                                  &size,
+        mca_bml_base_prepare_src (bml_btl, &sendreq->req_send.req_base.req_convertor,
+                                  MCA_BTL_NO_ORDER, hdr_size, &size,
                                   MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP |
-                                  MCA_BTL_DES_FLAGS_SIGNAL,
-                                  &des );
+                                  MCA_BTL_DES_FLAGS_SIGNAL, &des);
         MEMCHECKER(
             memchecker_call(&opal_memchecker_base_mem_noaccess,
                             sendreq->req_send.req_base.req_addr,
@@ -924,15 +977,23 @@ int mca_pml_ob1_send_request_start_rndv( mca_pml_ob1_send_request_t* sendreq,
 
     /* build hdr */
     hdr = (mca_pml_ob1_hdr_t*)segment->seg_addr.pval;
-    mca_pml_ob1_rendezvous_hdr_prepare (&hdr->hdr_rndv, MCA_PML_OB1_HDR_TYPE_RNDV, flags |
+
+    if (OPAL_UNLIKELY(need_ext_match)) {
+        hdr_rndv = &hdr->hdr_ext_rndv.hdr_rndv;
+        mca_pml_ob1_cid_hdr_prepare (&hdr->hdr_cid, sendreq->req_send.req_base.req_comm);
+    } else {
+        hdr_rndv = &hdr->hdr_rndv;
+    }
+
+    mca_pml_ob1_rendezvous_hdr_prepare (hdr_rndv, MCA_PML_OB1_HDR_TYPE_RNDV, flags |
                                         MCA_PML_OB1_HDR_FLAGS_SIGNAL,
-                                        sendreq->req_send.req_base.req_comm->c_contextid,
+                                        sendreq->ob1_proc->comm_index,
                                         sendreq->req_send.req_base.req_comm->c_my_rank,
                                         sendreq->req_send.req_base.req_tag,
                                         (uint16_t)sendreq->req_send.req_base.req_sequence,
                                         sendreq->req_send.req_bytes_packed, sendreq);
 
-    ob1_hdr_hton(hdr, MCA_PML_OB1_HDR_TYPE_RNDV, sendreq->req_send.req_base.req_proc);
+    ob1_hdr_hton(hdr, hdr->hdr_common.hdr_type, sendreq->req_send.req_base.req_proc);
 
     /* first fragment of a long message */
     des->des_cbdata = sendreq;
@@ -942,7 +1003,7 @@ int mca_pml_ob1_send_request_start_rndv( mca_pml_ob1_send_request_t* sendreq,
     sendreq->req_state = 2;
 
     /* send */
-    rc = mca_bml_base_send(bml_btl, des, MCA_PML_OB1_HDR_TYPE_RNDV);
+    rc = mca_bml_base_send(bml_btl, des, hdr->hdr_common.hdr_type);
     if( OPAL_LIKELY( rc >= 0 ) ) {
         if( OPAL_LIKELY( 1 == rc ) ) {
             mca_pml_ob1_rndv_completion_request( bml_btl, sendreq, size );
@@ -1050,6 +1111,12 @@ mca_pml_ob1_send_request_schedule_once(mca_pml_ob1_send_request_t* sendreq)
 
     range = get_send_range(sendreq);
 
+    if (NULL != sendreq->rdma_frag) {
+        /* this request was first attempted with RDMA but is now using send/recv */
+        MCA_PML_OB1_RDMA_FRAG_RETURN(sendreq->rdma_frag);
+        sendreq->rdma_frag = NULL;
+    }
+
     while(range && (false == sendreq->req_throttle_sends ||
           sendreq->req_pipeline_depth < mca_pml_ob1.send_pipeline_depth)) {
         mca_pml_ob1_frag_hdr_t* hdr;
@@ -1072,7 +1139,7 @@ mca_pml_ob1_send_request_schedule_once(mca_pml_ob1_send_request_t* sendreq)
             add_request_to_send_pending(sendreq,
                     MCA_PML_OB1_SEND_PENDING_SCHEDULE, true);
             /* Note that request remains locked. send_request_process_pending()
-             * function will call shedule_exclusive() directly without taking
+             * function will call schedule_exclusive() directly without taking
              * the lock */
             return OMPI_ERR_OUT_OF_RESOURCE;
         }
@@ -1092,17 +1159,12 @@ mca_pml_ob1_send_request_schedule_once(mca_pml_ob1_send_request_t* sendreq)
 
         /* makes sure that we don't exceed BTL max send size */
         if(bml_btl->btl->btl_max_send_size != 0) {
-#if OPAL_CUDA_SUPPORT
             size_t max_send_size;
-            if ((sendreq->req_send.req_base.req_convertor.flags & CONVERTOR_CUDA) && (bml_btl->btl->btl_cuda_max_send_size != 0)) {
-                max_send_size = bml_btl->btl->btl_cuda_max_send_size - sizeof(mca_pml_ob1_frag_hdr_t);
+            if ((sendreq->req_send.req_base.req_convertor.flags & CONVERTOR_ACCELERATOR) && (bml_btl->btl->btl_accelerator_max_send_size != 0)) {
+                max_send_size = bml_btl->btl->btl_accelerator_max_send_size - sizeof(mca_pml_ob1_frag_hdr_t);
             } else {
                 max_send_size = bml_btl->btl->btl_max_send_size - sizeof(mca_pml_ob1_frag_hdr_t);
             }
-#else /* OPAL_CUDA_SUPPORT */
-            size_t max_send_size = bml_btl->btl->btl_max_send_size -
-                sizeof(mca_pml_ob1_frag_hdr_t);
-#endif /* OPAL_CUDA_SUPPORT */
             if (size > max_send_size) {
                 size = max_send_size;
             }
@@ -1159,13 +1221,12 @@ mca_pml_ob1_send_request_schedule_once(mca_pml_ob1_send_request_t* sendreq)
                  &(sendreq->req_send.req_base), size, PERUSE_SEND);
 #endif  /* OMPI_WANT_PERUSE */
 
-#if OPAL_CUDA_SUPPORT /* CUDA_ASYNC_SEND */
          /* At this point, check to see if the BTL is doing an asynchronous
           * copy.  This would have been initiated in the mca_bml_base_prepare_src
           * called above.  The flag is checked here as we let the hdr be
           * set up prior to checking.
           */
-        if (des->des_flags & MCA_BTL_DES_FLAGS_CUDA_COPY_ASYNC) {
+        if (des->des_flags & MCA_BTL_DES_FLAGS_ACCELERATOR_COPY_ASYNC) {
             OPAL_OUTPUT((-1, "Initiating async copy on FRAG frag=%p", (void *)des));
             /* Need to make sure BTL does not free frag after completion
              * of asynchronous copy as we still need to send the fragment. */
@@ -1183,7 +1244,6 @@ mca_pml_ob1_send_request_schedule_once(mca_pml_ob1_send_request_t* sendreq)
             }
             continue;
         }
-#endif /* OPAL_CUDA_SUPPORT */
 
         /* initiate send - note that this may complete before the call returns */
         rc = mca_bml_base_send(bml_btl, des, MCA_PML_OB1_HDR_TYPE_FRAG);
@@ -1215,25 +1275,31 @@ static void mca_pml_ob1_send_request_put_frag_failed (mca_pml_ob1_rdma_frag_t *f
     mca_pml_ob1_send_request_t* sendreq = (mca_pml_ob1_send_request_t *) frag->rdma_req;
     mca_bml_base_btl_t *bml_btl = frag->rdma_bml;
 
-    if (++frag->retries < mca_pml_ob1.rdma_retries_limit && OMPI_ERR_OUT_OF_RESOURCE == rc) {
+    if (frag->retries < mca_pml_ob1.rdma_retries_limit && OMPI_ERR_OUT_OF_RESOURCE == rc) {
         /* queue the frag for later if there was a resource error */
         OPAL_THREAD_LOCK(&mca_pml_ob1.lock);
         opal_list_append(&mca_pml_ob1.rdma_pending, (opal_list_item_t*)frag);
         OPAL_THREAD_UNLOCK(&mca_pml_ob1.lock);
-    } else {
-        /* tell receiver to deregister memory */
-        mca_pml_ob1_send_fin (sendreq->req_send.req_base.req_proc, bml_btl,
-                              frag->rdma_hdr.hdr_rdma.hdr_frag, 0, MCA_BTL_NO_ORDER,
-                              OPAL_ERR_TEMP_OUT_OF_RESOURCE);
-
-        /* send fragment by copy in/out */
-        mca_pml_ob1_send_request_copy_in_out(sendreq, frag->rdma_hdr.hdr_rdma.hdr_rdma_offset,
-                                             frag->rdma_length);
-        /* if a pointer to a receive request is not set it means that
-         * ACK was not yet received. Don't schedule sends before ACK */
-        if (NULL != sendreq->req_recv.pval)
-            mca_pml_ob1_send_request_schedule (sendreq);
+        return;
+    }
+
+#if OPAL_ENABLE_FT
+    if(!ompi_proc_is_active(sendreq->req_send.req_base.req_proc)) {
+        return;
     }
+#endif /* OPAL_ENABLE_FT */
+    /* tell receiver to deregister memory */
+    mca_pml_ob1_send_fin (sendreq->req_send.req_base.req_proc, bml_btl,
+                          frag->rdma_hdr.hdr_rdma.hdr_frag, 0, MCA_BTL_NO_ORDER,
+                          OPAL_ERR_TEMP_OUT_OF_RESOURCE);
+
+    /* send fragment by copy in/out */
+    mca_pml_ob1_send_request_copy_in_out(sendreq, frag->rdma_hdr.hdr_rdma.hdr_rdma_offset,
+                                         frag->rdma_length);
+    /* if a pointer to a receive request is not set it means that
+     * ACK was not yet received. Don't schedule sends before ACK */
+    if (NULL != sendreq->req_recv.pval)
+        mca_pml_ob1_send_request_schedule (sendreq);
 }
 
 /**
diff --git a/ompi/mca/pml/ob1/pml_ob1_sendreq.h b/ompi/mca/pml/ob1/pml_ob1_sendreq.h
index 80a4ae2f6b4..1d50ecc033d 100644
--- a/ompi/mca/pml/ob1/pml_ob1_sendreq.h
+++ b/ompi/mca/pml/ob1/pml_ob1_sendreq.h
@@ -46,6 +46,7 @@ typedef enum {
 struct mca_pml_ob1_send_request_t {
     mca_pml_base_send_request_t req_send;
     mca_bml_base_endpoint_t* req_endpoint;
+    mca_pml_ob1_comm_proc_t *ob1_proc;
     opal_ptr_t req_recv;
     opal_atomic_int32_t  req_state;
     opal_atomic_int32_t  req_lock;
@@ -143,7 +144,8 @@ get_request_from_send_pending(mca_pml_ob1_send_pending_t *type)
                                        tag,                             \
                                        comm,                            \
                                        sendmode,                        \
-                                       persistent)                      \
+                                       persistent,                      \
+                                       ob1_proc)                        \
     {                                                                   \
         MCA_PML_BASE_SEND_REQUEST_INIT(&(sendreq)->req_send,            \
                                        buf,                             \
@@ -156,11 +158,14 @@ get_request_from_send_pending(mca_pml_ob1_send_pending_t *type)
                                        persistent,                      \
                                        0); /* convertor_flags */        \
         (sendreq)->req_recv.pval = NULL;                                \
+        (sendreq)->ob1_proc = ob1_proc;                                 \
     }
 
 #define MCA_PML_OB1_SEND_REQUEST_RESET(sendreq)                         \
     MCA_PML_BASE_SEND_REQUEST_RESET(&(sendreq)->req_send)
 
+#define MCA_PML_OB1_SEND_REQUEST_REQUIRES_EXT_MATCH(sendreq) (-1 == sendreq->ob1_proc->comm_index)
+
 static inline void mca_pml_ob1_free_rdma_resources (mca_pml_ob1_send_request_t* sendreq)
 {
     size_t r;
@@ -346,12 +351,10 @@ mca_pml_ob1_send_request_schedule(mca_pml_ob1_send_request_t* sendreq)
     mca_pml_ob1_send_request_schedule_exclusive(sendreq);
 }
 
-#if OPAL_CUDA_SUPPORT
-int mca_pml_ob1_send_request_start_cuda(
+int mca_pml_ob1_send_request_start_accelerator(
     mca_pml_ob1_send_request_t* sendreq,
     mca_bml_base_btl_t* bml_btl,
     size_t size);
-#endif /* OPAL_CUDA_SUPPORT */
 
 /**
  *  Start the specified request
@@ -393,8 +396,8 @@ mca_pml_ob1_send_request_start_btl( mca_pml_ob1_send_request_t* sendreq,
     int rc;
 
 #if OPAL_CUDA_GDR_SUPPORT
-    if (btl->btl_cuda_eager_limit && (sendreq->req_send.req_base.req_convertor.flags & CONVERTOR_CUDA)) {
-        eager_limit = btl->btl_cuda_eager_limit - sizeof(mca_pml_ob1_hdr_t);
+    if (btl->btl_accelerator_eager_limit && (sendreq->req_send.req_base.req_convertor.flags & CONVERTOR_ACCELERATOR)) {
+        eager_limit = btl->btl_accelerator_eager_limit - sizeof(mca_pml_ob1_hdr_t);
     }
 #endif /* OPAL_CUDA_GDR_SUPPORT */
 
@@ -424,7 +427,9 @@ mca_pml_ob1_send_request_start_btl( mca_pml_ob1_send_request_t* sendreq,
         if(sendreq->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED) {
             rc = mca_pml_ob1_send_request_start_buffered(sendreq, bml_btl, size);
         } else if
-                (opal_convertor_need_buffers(&sendreq->req_send.req_base.req_convertor) == false) {
+                (opal_convertor_need_buffers(&sendreq->req_send.req_base.req_convertor) == false &&
+                !(sendreq->req_send.req_base.req_convertor.flags & CONVERTOR_ACCELERATOR) &&
+                !(sendreq->req_send.req_base.req_convertor.flags & CONVERTOR_ACCELERATOR_UNIFIED)) {
             unsigned char *base;
             opal_convertor_get_current_pointer( &sendreq->req_send.req_base.req_convertor, (void**)&base );
 
@@ -443,11 +448,9 @@ mca_pml_ob1_send_request_start_btl( mca_pml_ob1_send_request_t* sendreq,
                                                          MCA_PML_OB1_HDR_FLAGS_CONTIG);
             }
         } else {
-#if OPAL_CUDA_SUPPORT
-            if (sendreq->req_send.req_base.req_convertor.flags & CONVERTOR_CUDA) {
-                return mca_pml_ob1_send_request_start_cuda(sendreq, bml_btl, size);
+            if (sendreq->req_send.req_base.req_convertor.flags & CONVERTOR_ACCELERATOR) {
+                return mca_pml_ob1_send_request_start_accelerator(sendreq, bml_btl, size);
             }
-#endif /* OPAL_CUDA_SUPPORT */
             rc = mca_pml_ob1_send_request_start_rndv(sendreq, bml_btl, size, 0);
         }
     }
diff --git a/ompi/mca/pml/ob1/pml_ob1_start.c b/ompi/mca/pml/ob1/pml_ob1_start.c
index 41795e007e5..311866e4014 100644
--- a/ompi/mca/pml/ob1/pml_ob1_start.c
+++ b/ompi/mca/pml/ob1/pml_ob1_start.c
@@ -84,13 +84,16 @@ int mca_pml_ob1_start(size_t count, ompi_request_t** requests)
                     sendreq = (mca_pml_ob1_send_request_t *) request;
                     requests[i] = request;
                 } else if (sendreq->req_send.req_bytes_packed != 0) {
-                    size_t offset = 0;
                     /**
                      * Reset the convertor in case we're dealing with the original
-                     * request, which when completed do not reset the convertor.
+                     * request, which when completed do not reset the convertor but
+                     * leaves it pointing to the buffered send location, with the
+                     * packed datatype and count.
                      */
-                    opal_convertor_set_position (&sendreq->req_send.req_base.req_convertor,
-                                                 &offset);
+                    opal_convertor_prepare_for_send(&sendreq->req_send.req_base.req_convertor,
+                                                    &sendreq->req_send.req_base.req_datatype->super,
+                                                    sendreq->req_send.req_base.req_count,
+                                                    sendreq->req_send.req_base.req_addr);
                 }
 
                 /* reset the completion flag */
diff --git a/ompi/mca/pml/pml.h b/ompi/mca/pml/pml.h
index b356d224a22..9446e45887a 100644
--- a/ompi/mca/pml/pml.h
+++ b/ompi/mca/pml/pml.h
@@ -99,6 +99,29 @@ typedef struct mca_pml_base_module_2_1_0_t * (*mca_pml_base_component_init_fn_t)
     bool enable_progress_threads,
     bool enable_mpi_threads);
 
+/**
+ * @brief A transport name and device name pair used by an endpoint.
+ * @ref mca_pml.pml_get_transports
+ */
+typedef struct mca_pml_transport_entry_t {
+    // The name of a transport layer used by this endpoint.
+    const char *transport_name;
+    // The name of the device used with this transport by this endpoint.
+    const char *device_name;
+} mca_pml_transport_entry_t;
+
+/**
+ * @brief  Structure containing an array of transport layers and device names
+ * used by a PML.
+ * @ref mca_pml.pml_get_transports
+ */
+typedef struct mca_pml_transports_t {
+    // Number of transport/device name pairs.
+    unsigned int count;
+    // Pointer to array of transport/device name pairs used by this endpoint.
+    mca_pml_transport_entry_t *entries;
+} mca_pml_transports_t;
+
 typedef int (*mca_pml_base_component_finalize_fn_t)(void);
 
 /**
@@ -168,6 +191,16 @@ typedef int (*mca_pml_base_module_enable_fn_t)(
 */
 typedef int (*mca_pml_base_module_progress_fn_t)(void);
 
+/**
+ * Get the set of transports and devices used to communicate with specified rank
+ *
+ * @param comm Communicator
+ * @param rank Task rank of other task
+ * @return Pointer to structure containing set of transports and device names or NULL
+ */
+typedef mca_pml_transports_t *(*mca_pml_base_module_get_transports_t)(struct ompi_communicator_t *comm,
+                                                                      int rank);
+
 /**
  * MPI Interface Functions
  */
@@ -489,7 +522,12 @@ typedef int (*mca_pml_base_module_dump_fn_t)(
  */
 /** PML requires requires all procs in the job on the first call to
  * add_procs */
-#define MCA_PML_BASE_FLAG_REQUIRE_WORLD 0x00000001
+#define MCA_PML_BASE_FLAG_REQUIRE_WORLD    0x00000001
+
+/**
+ * PML supports the extended CID space (doesn't need a global communicator index)
+ */
+#define MCA_PML_BASE_FLAG_SUPPORTS_EXT_CID 0x00000002
 
 /**
  *  PML instance.
@@ -528,6 +566,8 @@ struct mca_pml_base_module_2_1_0_t {
     uint32_t                              pml_max_contextid;
     int                                   pml_max_tag;
     int                                   pml_flags;
+
+    mca_pml_base_module_get_transports_t pml_get_transports;
 };
 typedef struct mca_pml_base_module_2_1_0_t mca_pml_base_module_2_1_0_t;
 typedef mca_pml_base_module_2_1_0_t mca_pml_base_module_t;
@@ -560,5 +600,10 @@ static inline bool mca_pml_base_requires_world (void)
     return !!(mca_pml.pml_flags & MCA_PML_BASE_FLAG_REQUIRE_WORLD);
 }
 
+static inline bool mca_pml_base_supports_extended_cid (void)
+{
+    return !!(mca_pml.pml_flags & MCA_PML_BASE_FLAG_SUPPORTS_EXT_CID);
+}
+
 END_C_DECLS
 #endif /* MCA_PML_H */
diff --git a/ompi/mca/pml/ucx/Makefile.am b/ompi/mca/pml/ucx/Makefile.am
index 6b8355031f2..fe6c8e47c35 100644
--- a/ompi/mca/pml/ucx/Makefile.am
+++ b/ompi/mca/pml/ucx/Makefile.am
@@ -38,7 +38,7 @@ mcacomponentdir = $(ompilibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_pml_ucx_la_SOURCES = $(local_sources)
 mca_pml_ucx_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la $(pml_ucx_LIBS) \
-                        $(OPAL_TOP_BUILDDIR)/opal/mca/common/ucx/lib@OPAL_LIB_PREFIX@mca_common_ucx.la
+                        $(OPAL_TOP_BUILDDIR)/opal/mca/common/ucx/lib@OPAL_LIB_NAME@mca_common_ucx.la
 mca_pml_ucx_la_LDFLAGS = -module -avoid-version $(pml_ucx_LDFLAGS)
 
 noinst_LTLIBRARIES = $(component_noinst)
diff --git a/ompi/mca/pml/ucx/configure.m4 b/ompi/mca/pml/ucx/configure.m4
index 9ee0273b398..0683262ae8c 100644
--- a/ompi/mca/pml/ucx/configure.m4
+++ b/ompi/mca/pml/ucx/configure.m4
@@ -1,5 +1,6 @@
 #
 # Copyright (C) Mellanox Technologies Ltd. 2001-2015.  ALL RIGHTS RESERVED.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -19,6 +20,9 @@ AC_DEFUN([MCA_ompi_pml_ucx_CONFIG], [
                    [pml_ucx_happy="yes"],
                    [pml_ucx_happy="no"])
 
+    AS_IF([test "${pml_ucx_happy}" = "yes"],
+          [OPAL_MCA_CHECK_DEPENDENCY([ompi], [pml], [ucx], [opal], [common], [ucx])])
+
     AS_IF([test "$pml_ucx_happy" = "yes"],
           [$1],
           [$2])
diff --git a/ompi/mca/pml/ucx/pml_ucx.c b/ompi/mca/pml/ucx/pml_ucx.c
index 488f02b2573..c748b02e12f 100644
--- a/ompi/mca/pml/ucx/pml_ucx.c
+++ b/ompi/mca/pml/ucx/pml_ucx.c
@@ -1,12 +1,14 @@
 /*
  * Copyright (C) 2001-2011 Mellanox Technologies Ltd. 2001-2011.  ALL RIGHTS RESERVED.
- * Copyright (c) 2016-2020 The University of Tennessee and The University
+ * Copyright (c) 2016-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2018-2019 Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2018 IBM Corporation. All rights reserved.
+ * Copyright (c) 2018-2022 IBM Corporation.  All rights reserved.
  * Copyright (c) 2019      Intel, Inc.  All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -23,63 +25,68 @@
 #include "ompi/runtime/ompi_spc.h"
 #include "ompi/mca/pml/base/pml_base_bsend.h"
 #include "opal/mca/common/ucx/common_ucx.h"
-#if OPAL_CUDA_SUPPORT
-#include "opal/mca/common/cuda/common_cuda.h"
-#endif /* OPAL_CUDA_SUPPORT */
 #include "pml_ucx_request.h"
 
 #include <inttypes.h>
 
+/*
+ * Maximum number of transport entries that we can expect from UCX
+ */
+#define PML_UCX_MAX_TRANSPORT_ENTRIES 100
 
 #define PML_UCX_TRACE_SEND(_msg, _buf, _count, _datatype, _dst, _tag, _mode, _comm, ...) \
     PML_UCX_VERBOSE(8, _msg " buf %p count %zu type '%s' dst %d tag %d mode %s comm %d '%s'", \
                     __VA_ARGS__, \
                     (_buf), (_count), (_datatype)->name, (_dst), (_tag), \
-                    mca_pml_ucx_send_mode_name(_mode), (_comm)->c_contextid, \
+                    mca_pml_ucx_send_mode_name(_mode), (_comm)->c_index, \
                     (_comm)->c_name);
 
 #define PML_UCX_TRACE_RECV(_msg, _buf, _count, _datatype, _src, _tag, _comm, ...) \
     PML_UCX_VERBOSE(8, _msg " buf %p count %zu type '%s' src %d tag %d comm %d '%s'", \
                     __VA_ARGS__, \
                     (_buf), (_count), (_datatype)->name, (_src), (_tag), \
-                    (_comm)->c_contextid, (_comm)->c_name);
+                    (_comm)->c_index, (_comm)->c_name);
 
 #define PML_UCX_TRACE_PROBE(_msg, _src, _tag, _comm) \
     PML_UCX_VERBOSE(8, _msg " src %d tag %d comm %d '%s'", \
-                    _src, (_tag), (_comm)->c_contextid, (_comm)->c_name);
+                    _src, (_tag), (_comm)->c_index, (_comm)->c_name);
 
 #define PML_UCX_TRACE_MRECV(_msg, _buf, _count, _datatype, _message) \
     PML_UCX_VERBOSE(8, _msg " buf %p count %zu type '%s' msg *%p=%p (%p)", \
                     (_buf), (_count), (_datatype)->name, (void*)(_message), \
                     (void*)*(_message), (*(_message))->req_ptr);
 
+mca_pml_transports_t *mca_pml_ucx_get_transports(ompi_communicator_t *comm,
+                                                 int rank);
+
 #define MODEX_KEY "pml-ucx"
 
 mca_pml_ucx_module_t ompi_pml_ucx = {
     .super = {
-        .pml_add_procs     = mca_pml_ucx_add_procs,
-        .pml_del_procs     = mca_pml_ucx_del_procs,
-        .pml_enable        = mca_pml_ucx_enable,
-        .pml_progress      = NULL,
-        .pml_add_comm      = mca_pml_ucx_add_comm,
-        .pml_del_comm      = mca_pml_ucx_del_comm,
-        .pml_irecv_init    = mca_pml_ucx_irecv_init,
-        .pml_irecv         = mca_pml_ucx_irecv,
-        .pml_recv          = mca_pml_ucx_recv,
-        .pml_isend_init    = mca_pml_ucx_isend_init,
-        .pml_isend         = mca_pml_ucx_isend,
-        .pml_send          = mca_pml_ucx_send,
-        .pml_iprobe        = mca_pml_ucx_iprobe,
-        .pml_probe         = mca_pml_ucx_probe,
-        .pml_start         = mca_pml_ucx_start,
-        .pml_improbe       = mca_pml_ucx_improbe,
-        .pml_mprobe        = mca_pml_ucx_mprobe,
-        .pml_imrecv        = mca_pml_ucx_imrecv,
-        .pml_mrecv         = mca_pml_ucx_mrecv,
-        .pml_dump          = mca_pml_ucx_dump,
-        .pml_max_contextid = (1ul << (PML_UCX_CONTEXT_BITS)) - 1,
-        .pml_max_tag       = (1ul << (PML_UCX_TAG_BITS - 1)) - 1,
-        .pml_flags         = 0 /* flags */
+        .pml_add_procs      = mca_pml_ucx_add_procs,
+        .pml_del_procs      = mca_pml_ucx_del_procs,
+        .pml_enable         = mca_pml_ucx_enable,
+        .pml_progress       = NULL,
+        .pml_add_comm       = mca_pml_ucx_add_comm,
+        .pml_del_comm       = mca_pml_ucx_del_comm,
+        .pml_irecv_init     = mca_pml_ucx_irecv_init,
+        .pml_irecv          = mca_pml_ucx_irecv,
+        .pml_recv           = mca_pml_ucx_recv,
+        .pml_isend_init     = mca_pml_ucx_isend_init,
+        .pml_isend          = mca_pml_ucx_isend,
+        .pml_send           = mca_pml_ucx_send,
+        .pml_iprobe         = mca_pml_ucx_iprobe,
+        .pml_probe          = mca_pml_ucx_probe,
+        .pml_start          = mca_pml_ucx_start,
+        .pml_improbe        = mca_pml_ucx_improbe,
+        .pml_mprobe         = mca_pml_ucx_mprobe,
+        .pml_imrecv         = mca_pml_ucx_imrecv,
+        .pml_mrecv          = mca_pml_ucx_mrecv,
+        .pml_dump           = mca_pml_ucx_dump,
+        .pml_max_contextid  = (1ul << (PML_UCX_CONTEXT_BITS)) - 1,
+        .pml_max_tag        = (1ul << (PML_UCX_TAG_BITS - 1)) - 1,
+        .pml_flags          = 0, /* flags */
+        .pml_get_transports = mca_pml_ucx_get_transports
     },
     .ucp_context           = NULL,
     .ucp_worker            = NULL
@@ -207,6 +214,12 @@ int mca_pml_ucx_open(void)
         return OMPI_ERROR;
     }
 
+    if ((major_version == 1) && (minor_version < 9)) {
+        /* show warning due to issue #8549 */
+        PML_UCX_WARN("UCX version %u.%u.%u is too old, please install 1.9.x or "
+                     "newer", major_version, minor_version, release_number);
+    }
+
     /* Read options */
     status = ucp_config_read("MPI", NULL, &config);
     if (UCS_OK != status) {
@@ -255,14 +268,7 @@ int mca_pml_ucx_open(void)
     }
 
     ompi_pml_ucx.request_size     = attr.request_size;
-    ompi_pml_ucx.cuda_initialized = false;
 
-#if HAVE_UCP_ATTR_MEMORY_TYPES && OPAL_CUDA_SUPPORT
-    if (attr.memory_types & UCS_BIT(UCS_MEMORY_TYPE_CUDA)) {
-        mca_common_cuda_stage_one_init();
-        ompi_pml_ucx.cuda_initialized = true;
-    }
-#endif
     return OMPI_SUCCESS;
 }
 
@@ -270,11 +276,6 @@ int mca_pml_ucx_close(void)
 {
     PML_UCX_VERBOSE(1, "mca_pml_ucx_close");
 
-#if OPAL_CUDA_SUPPORT
-    if (ompi_pml_ucx.cuda_initialized) {
-        mca_common_cuda_fini();
-    }
-#endif
     if (ompi_pml_ucx.ucp_context != NULL) {
         ucp_cleanup(ompi_pml_ucx.ucp_context);
         ompi_pml_ucx.ucp_context = NULL;
@@ -291,14 +292,21 @@ int mca_pml_ucx_init(int enable_mpi_threads)
 
     PML_UCX_VERBOSE(1, "mca_pml_ucx_init");
 
-    /* TODO check MPI thread mode */
     params.field_mask  = UCP_WORKER_PARAM_FIELD_THREAD_MODE;
     if (enable_mpi_threads) {
         params.thread_mode = UCS_THREAD_MODE_MULTI;
     } else {
-        params.thread_mode = UCS_THREAD_MODE_SINGLE;
+        params.thread_mode =
+            opal_common_ucx_thread_mode(ompi_mpi_thread_provided);
     }
 
+#if HAVE_DECL_UCP_WORKER_FLAG_IGNORE_REQUEST_LEAK
+    if (!ompi_pml_ucx.request_leak_check) {
+        params.field_mask |= UCP_WORKER_PARAM_FIELD_FLAGS;
+        params.flags      |= UCP_WORKER_FLAG_IGNORE_REQUEST_LEAK;
+    }
+#endif
+
     status = ucp_worker_create(ompi_pml_ucx.ucp_context, &params,
                                &ompi_pml_ucx.ucp_worker);
     if (UCS_OK != status) {
@@ -341,6 +349,9 @@ int mca_pml_ucx_init(int enable_mpi_threads)
     /* Create a completed request to be returned from isend */
     OBJ_CONSTRUCT(&ompi_pml_ucx.completed_send_req, ompi_request_t);
     mca_pml_ucx_completed_request_init(&ompi_pml_ucx.completed_send_req);
+#if MPI_VERSION >= 4
+    ompi_pml_ucx.completed_send_req.req_cancel = mca_pml_cancel_send_callback;
+#endif
 
     opal_progress_register(mca_pml_ucx_progress);
 
@@ -399,6 +410,11 @@ static ucp_ep_h mca_pml_ucx_add_proc_common(ompi_proc_t *proc)
     ucp_ep_h ep;
     int ret;
 
+    /* Do not add a new endpoint if we already created one */
+    if (NULL != proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PML]) {
+        return proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PML];
+    }
+
     ret = mca_pml_ucx_recv_worker_address(proc, &address, &addrlen);
     if (ret < 0) {
         return NULL;
@@ -440,6 +456,65 @@ int mca_pml_ucx_add_procs(struct ompi_proc_t **procs, size_t nprocs)
     return OMPI_SUCCESS;
 }
 
+mca_pml_transports_t *mca_pml_ucx_get_transports(struct ompi_communicator_t *comm,
+                                                 int rank)
+{
+#if HAVE_DECL_UCP_EP_ATTR_FIELD_TRANSPORTS
+    mca_pml_transports_t *transports;
+    unsigned i;
+    ucs_status_t status;
+    ucp_ep_attr_t ep_attrs;
+    ompi_proc_t *proc = ompi_comm_peer_lookup(comm, rank);
+
+    if (NULL == proc) {
+        return NULL;
+    }
+    if (NULL == proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PML]) {
+        return NULL;
+    }
+
+    ep_attrs.field_mask = UCP_EP_ATTR_FIELD_TRANSPORTS;
+    ep_attrs.transports.num_entries = PML_UCX_MAX_TRANSPORT_ENTRIES;
+    ep_attrs.transports.entry_size = sizeof(ucp_transport_entry_t);
+    ep_attrs.transports.entries = (ucp_transport_entry_t *) malloc(
+             sizeof(ucp_transport_entry_t) * ep_attrs.transports.num_entries);
+    if (NULL == ep_attrs.transports.entries) {
+        return NULL;
+    }
+
+    status = ucp_ep_query(proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PML],
+                          &ep_attrs);
+    if (UCS_OK != status) {
+        return NULL;
+    }
+
+    transports = (mca_pml_transports_t *) malloc(sizeof(mca_pml_transports_t));
+    if (NULL == transports) {
+        return NULL;
+    }
+
+    transports->entries = (mca_pml_transport_entry_t *) malloc(
+        ep_attrs.transports.num_entries * sizeof(mca_pml_transport_entry_t));
+    if (NULL == transports->entries) {
+        free(transports);
+        return NULL;
+    }
+
+    for (i = 0; i < ep_attrs.transports.num_entries; i++) {
+        transports->entries[i].transport_name =
+                ep_attrs.transports.entries[i].transport_name;
+        transports->entries[i].device_name = 
+                ep_attrs.transports.entries[i].device_name;
+    }
+    transports->count = ep_attrs.transports.num_entries;
+
+    free(ep_attrs.transports.entries);
+    return transports;
+#else
+    return NULL;
+#endif
+}
+
 __opal_attribute_always_inline__
 static inline ucp_ep_h mca_pml_ucx_get_ep(ompi_communicator_t *comm, int rank)
 {
@@ -489,8 +564,7 @@ int mca_pml_ucx_enable(bool enable)
     int ret;
 
     /* Create a key for adding custom attributes to datatypes */
-    copy_fn.attr_datatype_copy_fn  =
-                    (MPI_Type_internal_copy_attr_function*)MPI_TYPE_NULL_COPY_FN;
+    copy_fn.attr_datatype_copy_fn  = MPI_TYPE_NULL_COPY_FN;
     del_fn.attr_datatype_delete_fn = mca_pml_ucx_datatype_attr_del_fn;
     ret = ompi_attr_create_keyval(TYPE_ATTR, copy_fn, del_fn,
                                   &ompi_pml_ucx.datatype_attr_keyval, NULL, 0,
@@ -543,7 +617,9 @@ int mca_pml_ucx_irecv_init(void *buf, size_t count, ompi_datatype_t *datatype,
     req->flags                    = 0;
     req->buffer                   = buf;
     req->count                    = count;
-    req->datatype.datatype        = mca_pml_ucx_get_datatype(datatype);
+    req->ompi_datatype            = datatype;
+    req->datatype                 = mca_pml_ucx_get_datatype(datatype);
+    OMPI_DATATYPE_RETAIN(datatype);
 
     PML_UCX_MAKE_RECV_TAG(req->tag, req->recv.tag_mask, tag, src, comm);
 
@@ -557,7 +633,7 @@ int mca_pml_ucx_irecv(void *buf, size_t count, ompi_datatype_t *datatype,
 {
 #if HAVE_DECL_UCP_TAG_RECV_NBX
     pml_ucx_datatype_t *op_data = mca_pml_ucx_get_op_data(datatype);
-    ucp_request_param_t *param  = &op_data->op_param.recv;
+    ucp_request_param_t *param  = &op_data->op_param.irecv;
 #endif
 
     ucp_tag_t ucp_tag, ucp_tag_mask;
@@ -624,7 +700,7 @@ int mca_pml_ucx_recv(void *buf, size_t count, ompi_datatype_t *datatype, int src
     MCA_COMMON_UCX_PROGRESS_LOOP(ompi_pml_ucx.ucp_worker) {
         status = ucp_request_test(req, &info);
         if (status != UCS_INPROGRESS) {
-            result = mca_pml_ucx_set_recv_status_safe(mpi_status, status, &info);
+            result = mca_pml_ucx_set_recv_status_public(mpi_status, status, &info);
 
 #if SPC_ENABLE == 1
             size_t dt_size;
@@ -663,6 +739,8 @@ int mca_pml_ucx_isend_init(const void *buf, size_t count, ompi_datatype_t *datat
                            struct ompi_communicator_t* comm,
                            struct ompi_request_t **request)
 {
+    int rc;
+    uint32_t cid;
     mca_pml_ucx_persistent_request_t *req;
     ucp_ep_h ep;
 
@@ -679,20 +757,26 @@ int mca_pml_ucx_isend_init(const void *buf, size_t count, ompi_datatype_t *datat
         return OMPI_ERROR;
     }
 
+    rc = ompi_comm_get_remote_cid(comm, dst, &cid);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+         return rc;
+    }
+
     req->ompi.req_state           = OMPI_REQUEST_INACTIVE;
     req->ompi.req_mpi_object.comm = comm;
     req->flags                    = MCA_PML_UCX_REQUEST_FLAG_SEND;
     req->buffer                   = (void *)buf;
     req->count                    = count;
-    req->tag                      = PML_UCX_MAKE_SEND_TAG(tag, comm);
+    req->tag                      = PML_UCX_MAKE_SEND_TAG(tag, comm, cid);
     req->send.mode                = mode;
     req->send.ep                  = ep;
+    req->ompi_datatype            = datatype;
+    OMPI_DATATYPE_RETAIN(datatype);
 
     if (MCA_PML_BASE_SEND_BUFFERED == mode) {
-        req->datatype.ompi_datatype = datatype;
-        OBJ_RETAIN(datatype);
+        req->datatype = (ucp_datatype_t)NULL;
     } else {
-        req->datatype.datatype = mca_pml_ucx_get_datatype(datatype);
+        req->datatype = mca_pml_ucx_get_datatype(datatype);
     }
 
     *request = &req->ompi;
@@ -808,7 +892,9 @@ int mca_pml_ucx_isend(const void *buf, size_t count, ompi_datatype_t *datatype,
                       struct ompi_communicator_t* comm,
                       struct ompi_request_t **request)
 {
+    int rc;
     ompi_request_t *req;
+    uint32_t cid;
     ucp_ep_h ep;
 
     PML_UCX_TRACE_SEND("i%ssend request *%p",
@@ -820,15 +906,18 @@ int mca_pml_ucx_isend(const void *buf, size_t count, ompi_datatype_t *datatype,
     if (OPAL_UNLIKELY(NULL == ep)) {
         return OMPI_ERROR;
     }
-
+    rc = ompi_comm_get_remote_cid(comm, dst, &cid);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+        return rc;
+    }
 #if HAVE_DECL_UCP_TAG_SEND_NBX
     req = (ompi_request_t*)mca_pml_ucx_common_send_nbx(ep, buf, count, datatype,
-                                                       PML_UCX_MAKE_SEND_TAG(tag, comm), mode,
-                                                       &mca_pml_ucx_get_op_data(datatype)->op_param.send);
+                                                       PML_UCX_MAKE_SEND_TAG(tag, comm, cid), mode,
+                                                       &mca_pml_ucx_get_op_data(datatype)->op_param.isend);
 #else
     req = (ompi_request_t*)mca_pml_ucx_common_send(ep, buf, count, datatype,
                                                    mca_pml_ucx_get_datatype(datatype),
-                                                   PML_UCX_MAKE_SEND_TAG(tag, comm), mode,
+                                                   PML_UCX_MAKE_SEND_TAG(tag, comm, cid), mode,
                                                    mca_pml_ucx_send_completion);
 #endif
 
@@ -846,6 +935,11 @@ int mca_pml_ucx_isend(const void *buf, size_t count, ompi_datatype_t *datatype,
     } else if (!UCS_PTR_IS_ERR(req)) {
         PML_UCX_VERBOSE(8, "got request %p", (void*)req);
         req->req_mpi_object.comm = comm;
+#if MPI_VERSION >= 4
+        if (OPAL_LIKELY(mca_pml_ucx_request_cancel == req->req_cancel)) {
+            req->req_cancel      = mca_pml_ucx_request_cancel_send;
+        }
+#endif
         *request                 = req;
         return OMPI_SUCCESS;
     } else {
@@ -857,19 +951,19 @@ int mca_pml_ucx_isend(const void *buf, size_t count, ompi_datatype_t *datatype,
 static inline __opal_attribute_always_inline__ int
 mca_pml_ucx_send_nb(ucp_ep_h ep, const void *buf, size_t count,
                     ompi_datatype_t *datatype, ucp_datatype_t ucx_datatype,
-                    ucp_tag_t tag, mca_pml_base_send_mode_t mode,
-                    ucp_send_callback_t cb)
+                    ucp_tag_t tag, mca_pml_base_send_mode_t mode)
 {
     ompi_request_t *req;
 
     req = (ompi_request_t*)mca_pml_ucx_common_send(ep, buf, count, datatype,
                                                    mca_pml_ucx_get_datatype(datatype),
-                                                   tag, mode, cb);
+                                                   tag, mode,
+                                                   mca_pml_ucx_send_completion_empty);
     if (OPAL_LIKELY(req == NULL)) {
         return OMPI_SUCCESS;
     } else if (!UCS_PTR_IS_ERR(req)) {
         PML_UCX_VERBOSE(8, "got request %p", (void*)req);
-        MCA_COMMON_UCX_WAIT_LOOP(req, ompi_pml_ucx.ucp_worker, "ucx send", ompi_request_free(&req));
+        MCA_COMMON_UCX_WAIT_LOOP(req, ompi_pml_ucx.ucp_worker, "ucx send", ucp_request_free(req));
     } else {
         PML_UCX_ERROR("ucx send failed: %s", ucs_status_string(UCS_PTR_STATUS(req)));
         return OMPI_ERROR;
@@ -920,7 +1014,9 @@ int mca_pml_ucx_send(const void *buf, size_t count, ompi_datatype_t *datatype, i
                      int tag, mca_pml_base_send_mode_t mode,
                      struct ompi_communicator_t* comm)
 {
+    int rc;
     ucp_ep_h ep;
+    uint32_t cid;
 
     PML_UCX_TRACE_SEND("%s", buf, count, datatype, dst, tag, mode, comm,
                        mode == MCA_PML_BASE_SEND_BUFFERED ? "bsend" : "send");
@@ -937,18 +1033,22 @@ int mca_pml_ucx_send(const void *buf, size_t count, ompi_datatype_t *datatype, i
                     OMPI_SPC_BYTES_SENT_USER, OMPI_SPC_BYTES_SENT_MPI);
 #endif
 
+    rc = ompi_comm_get_remote_cid(comm, dst, &cid);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+        return rc;
+    }
+
 #if HAVE_DECL_UCP_TAG_SEND_NBR
     if (OPAL_LIKELY((MCA_PML_BASE_SEND_BUFFERED != mode) &&
                     (MCA_PML_BASE_SEND_SYNCHRONOUS != mode))) {
         return mca_pml_ucx_send_nbr(ep, buf, count, datatype,
-                                    PML_UCX_MAKE_SEND_TAG(tag, comm));
+                                    PML_UCX_MAKE_SEND_TAG(tag, comm, cid));
     }
 #endif
 
     return mca_pml_ucx_send_nb(ep, buf, count, datatype,
                                mca_pml_ucx_get_datatype(datatype),
-                               PML_UCX_MAKE_SEND_TAG(tag, comm), mode,
-                               mca_pml_ucx_send_completion);
+                               PML_UCX_MAKE_SEND_TAG(tag, comm, cid), mode);
 }
 
 int mca_pml_ucx_iprobe(int src, int tag, struct ompi_communicator_t* comm,
@@ -967,10 +1067,10 @@ int mca_pml_ucx_iprobe(int src, int tag, struct ompi_communicator_t* comm,
                                0, &info);
     if (ucp_msg != NULL) {
         *matched = 1;
-        mca_pml_ucx_set_recv_status_safe(mpi_status, UCS_OK, &info);
+        mca_pml_ucx_set_recv_status_public(mpi_status, UCS_OK, &info);
     } else  {
         (++progress_count % opal_common_ucx.progress_iterations) ?
-            (void)ucp_worker_progress(ompi_pml_ucx.ucp_worker) : opal_progress();
+            (int)ucp_worker_progress(ompi_pml_ucx.ucp_worker) : opal_progress();
         *matched = 0;
     }
     return OMPI_SUCCESS;
@@ -991,7 +1091,7 @@ int mca_pml_ucx_probe(int src, int tag, struct ompi_communicator_t* comm,
         ucp_msg = ucp_tag_probe_nb(ompi_pml_ucx.ucp_worker, ucp_tag,
                                    ucp_tag_mask, 0, &info);
         if (ucp_msg != NULL) {
-            mca_pml_ucx_set_recv_status_safe(mpi_status, UCS_OK, &info);
+            mca_pml_ucx_set_recv_status_public(mpi_status, UCS_OK, &info);
             return OMPI_SUCCESS;
         }
     }
@@ -1016,10 +1116,10 @@ int mca_pml_ucx_improbe(int src, int tag, struct ompi_communicator_t* comm,
         PML_UCX_MESSAGE_NEW(comm, ucp_msg, &info, message);
         PML_UCX_VERBOSE(8, "got message %p (%p)", (void*)*message, (void*)ucp_msg);
         *matched         = 1;
-        mca_pml_ucx_set_recv_status_safe(mpi_status, UCS_OK, &info);
+        mca_pml_ucx_set_recv_status_public(mpi_status, UCS_OK, &info);
     } else  {
         (++progress_count % opal_common_ucx.progress_iterations) ?
-            (void)ucp_worker_progress(ompi_pml_ucx.ucp_worker) : opal_progress();
+            (int)ucp_worker_progress(ompi_pml_ucx.ucp_worker) : opal_progress();
         *matched = 0;
     }
     return OMPI_SUCCESS;
@@ -1042,7 +1142,7 @@ int mca_pml_ucx_mprobe(int src, int tag, struct ompi_communicator_t* comm,
         if (ucp_msg != NULL) {
             PML_UCX_MESSAGE_NEW(comm, ucp_msg, &info, message);
             PML_UCX_VERBOSE(8, "got message %p (%p)", (void*)*message, (void*)ucp_msg);
-            mca_pml_ucx_set_recv_status_safe(mpi_status, UCS_OK, &info);
+            mca_pml_ucx_set_recv_status_public(mpi_status, UCS_OK, &info);
             return OMPI_SUCCESS;
         }
     }
@@ -1115,8 +1215,8 @@ int mca_pml_ucx_start(size_t count, ompi_request_t** requests)
             tmp_req = (ompi_request_t*)mca_pml_ucx_common_send(preq->send.ep,
                                                                preq->buffer,
                                                                preq->count,
-                                                               preq->datatype.ompi_datatype,
-                                                               preq->datatype.datatype,
+                                                               preq->ompi_datatype,
+                                                               preq->datatype,
                                                                preq->tag,
                                                                preq->send.mode,
                                                                mca_pml_ucx_psend_completion);
@@ -1124,7 +1224,7 @@ int mca_pml_ucx_start(size_t count, ompi_request_t** requests)
             PML_UCX_VERBOSE(8, "start recv request %p", (void*)preq);
             tmp_req = (ompi_request_t*)ucp_tag_recv_nb(ompi_pml_ucx.ucp_worker,
                                                        preq->buffer, preq->count,
-                                                       preq->datatype.datatype,
+                                                       preq->datatype,
                                                        preq->tag,
                                                        preq->recv.tag_mask,
                                                        mca_pml_ucx_precv_completion);
diff --git a/ompi/mca/pml/ucx/pml_ucx.h b/ompi/mca/pml/ucx/pml_ucx.h
index 5fd98f4ee9b..88837f997bc 100644
--- a/ompi/mca/pml/ucx/pml_ucx.h
+++ b/ompi/mca/pml/ucx/pml_ucx.h
@@ -57,7 +57,8 @@ struct mca_pml_ucx_module {
     mca_pml_ucx_freelist_t    convs;
 
     int                       priority;
-    bool                      cuda_initialized;
+    bool                      request_leak_check;
+    uint32_t                  op_attr_nonblocking;
 };
 
 extern mca_pml_base_component_2_1_0_t mca_pml_ucx_component;
diff --git a/ompi/mca/pml/ucx/pml_ucx_component.c b/ompi/mca/pml/ucx/pml_ucx_component.c
index 4d583110095..ec095e19fef 100644
--- a/ompi/mca/pml/ucx/pml_ucx_component.c
+++ b/ompi/mca/pml/ucx/pml_ucx_component.c
@@ -49,6 +49,10 @@ mca_pml_base_component_2_1_0_t mca_pml_ucx_component = {
 
 static int mca_pml_ucx_component_register(void)
 {
+#if HAVE_DECL_UCP_OP_ATTR_FLAG_MULTI_SEND
+    int multi_send_op_attr_enable;
+#endif
+
     ompi_pml_ucx.priority = 51;
     (void) mca_base_component_var_register(&mca_pml_ucx_component.pmlm_version, "priority",
                                            "Priority of the UCX component",
@@ -64,6 +68,35 @@ static int mca_pml_ucx_component_register(void)
                                            OPAL_INFO_LVL_3,
                                            MCA_BASE_VAR_SCOPE_LOCAL,
                                            &ompi_pml_ucx.num_disconnect);
+
+#if HAVE_DECL_UCP_WORKER_FLAG_IGNORE_REQUEST_LEAK
+    ompi_pml_ucx.request_leak_check = false;
+    (void) mca_base_component_var_register(&mca_pml_ucx_component.pmlm_version, "request_leak_check",
+                                           "Enable showing a warning during MPI_Finalize if some "
+                                           "non-blocking MPI requests have not been released",
+                                           MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
+                                           OPAL_INFO_LVL_3,
+                                           MCA_BASE_VAR_SCOPE_LOCAL,
+                                           &ompi_pml_ucx.request_leak_check);
+#else
+    /* If UCX does not support ignoring leak check, then it's always enabled */
+    ompi_pml_ucx.request_leak_check = true;
+#endif
+
+    ompi_pml_ucx.op_attr_nonblocking = 0;
+#if HAVE_DECL_UCP_OP_ATTR_FLAG_MULTI_SEND
+    multi_send_op_attr_enable        = 1;
+    (void) mca_base_component_var_register(&mca_pml_ucx_component.pmlm_version, "multi_send_nb",
+                                           "Enable passing multi-send optimization flag for nonblocking operations",
+                                           MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
+                                           OPAL_INFO_LVL_3,
+                                           MCA_BASE_VAR_SCOPE_LOCAL,
+                                           &multi_send_op_attr_enable);
+    if (multi_send_op_attr_enable) {
+        ompi_pml_ucx.op_attr_nonblocking = UCP_OP_ATTR_FLAG_MULTI_SEND;
+    }
+#endif
+
     opal_common_ucx_mca_var_register(&mca_pml_ucx_component.pmlm_version);
     return 0;
 }
@@ -112,6 +145,10 @@ mca_pml_ucx_component_init(int* priority, bool enable_progress_threads,
     *priority = (support_level == OPAL_COMMON_UCX_SUPPORT_DEVICE) ?
                 ompi_pml_ucx.priority : 19;
     PML_UCX_VERBOSE(2, "returning priority %d", *priority);
+
+    /** this pml supports the extended CID space */
+    ompi_pml_ucx.super.pml_flags |= MCA_PML_BASE_FLAG_SUPPORTS_EXT_CID;
+
     return &ompi_pml_ucx.super;
 }
 
diff --git a/ompi/mca/pml/ucx/pml_ucx_datatype.c b/ompi/mca/pml/ucx/pml_ucx_datatype.c
index 212cb1de702..13cc49e04ff 100644
--- a/ompi/mca/pml/ucx/pml_ucx_datatype.c
+++ b/ompi/mca/pml/ucx/pml_ucx_datatype.c
@@ -24,7 +24,6 @@
 #ifdef HAVE_UCP_REQUEST_PARAM_T
 #define PML_UCX_DATATYPE_SET_VALUE(_datatype, _val) \
     (_datatype)->op_param.send._val; \
-    (_datatype)->op_param.bsend._val; \
     (_datatype)->op_param.recv._val;
 #endif
 
@@ -168,6 +167,19 @@ static inline int mca_pml_ucx_datatype_is_contig(ompi_datatype_t *datatype)
            (lb == 0);
 }
 
+static unsigned mca_pml_ucx_ilog2_u64(uint64_t n)
+{
+#if OPAL_C_HAVE_BUILTIN_CLZ
+    return (sizeof(n) * 8) - 1 - __builtin_clzll(n);
+#else
+    unsigned i;
+    for (i = 0; n > 1; ++i) {
+        n >>= 1;
+    }
+    return i;
+#endif
+}
+
 #ifdef HAVE_UCP_REQUEST_PARAM_T
 __opal_attribute_always_inline__ static inline
 pml_ucx_datatype_t *mca_pml_ucx_init_nbx_datatype(ompi_datatype_t *datatype,
@@ -190,8 +202,6 @@ pml_ucx_datatype_t *mca_pml_ucx_init_nbx_datatype(ompi_datatype_t *datatype,
     pml_datatype->datatype                    = ucp_datatype;
     pml_datatype->op_param.send.op_attr_mask  = UCP_OP_ATTR_FIELD_CALLBACK;
     pml_datatype->op_param.send.cb.send       = mca_pml_ucx_send_nbx_completion;
-    pml_datatype->op_param.bsend.op_attr_mask = UCP_OP_ATTR_FIELD_CALLBACK;
-    pml_datatype->op_param.bsend.cb.send      = mca_pml_ucx_bsend_nbx_completion;
     pml_datatype->op_param.recv.op_attr_mask  = UCP_OP_ATTR_FIELD_CALLBACK |
                                                 UCP_OP_ATTR_FLAG_NO_IMM_CMPL;
     pml_datatype->op_param.recv.cb.recv       = mca_pml_ucx_recv_nbx_completion;
@@ -199,39 +209,57 @@ pml_ucx_datatype_t *mca_pml_ucx_init_nbx_datatype(ompi_datatype_t *datatype,
     is_contig_pow2 = mca_pml_ucx_datatype_is_contig(datatype) &&
                      (size && !(size & (size - 1))); /* is_pow2(size) */
     if (is_contig_pow2) {
-        pml_datatype->size_shift = (int)(log(size) / log(2.0)); /* log2(size) */
+        pml_datatype->size_shift = mca_pml_ucx_ilog2_u64(size);
     } else {
         pml_datatype->size_shift = 0;
         PML_UCX_DATATYPE_SET_VALUE(pml_datatype, op_attr_mask |= UCP_OP_ATTR_FIELD_DATATYPE);
         PML_UCX_DATATYPE_SET_VALUE(pml_datatype, datatype = ucp_datatype);
     }
 
+    pml_datatype->op_param.isend = pml_datatype->op_param.send;
+    pml_datatype->op_param.irecv = pml_datatype->op_param.recv;
+    pml_datatype->op_param.isend.op_attr_mask |= ompi_pml_ucx.op_attr_nonblocking;
+    pml_datatype->op_param.irecv.op_attr_mask |= ompi_pml_ucx.op_attr_nonblocking;
+
     return pml_datatype;
 }
 #endif
 
 ucp_datatype_t mca_pml_ucx_init_datatype(ompi_datatype_t *datatype)
 {
+    static opal_thread_internal_mutex_t lock = OPAL_THREAD_INTERNAL_MUTEX_INITIALIZER;
     size_t size = 0; /* init to suppress compiler warning */
     ucp_datatype_t ucp_datatype;
     ucs_status_t status;
     int ret;
 
-    if (mca_pml_ucx_datatype_is_contig(datatype)) {
-        ompi_datatype_type_size(datatype, &size);
-        ucp_datatype = ucp_dt_make_contig(size);
+    opal_thread_internal_mutex_lock(&lock);
+
+    if (datatype->pml_data != PML_UCX_DATATYPE_INVALID) {
+        /* datatype is already initialized in concurrent thread */
         goto out;
     }
 
-    status = ucp_dt_create_generic(&pml_ucx_generic_datatype_ops,
-                                   datatype, &ucp_datatype);
-    if (status != UCS_OK) {
-        int err = MPI_ERR_INTERN;
-        PML_UCX_ERROR("Failed to create UCX datatype for %s", datatype->name);
-        /* TODO: this error should return to the caller and invoke an error
-         * handler from the MPI API call.
-         * For now, it is fatal. */
-        ompi_mpi_errors_are_fatal_comm_handler(NULL, &err, "Failed to allocate datatype structure");
+    if (mca_pml_ucx_datatype_is_contig(datatype)) {
+        ompi_datatype_type_size(datatype, &size);
+        ucp_datatype = ucp_dt_make_contig(size);
+        PML_UCX_VERBOSE(7, "created contig UCX datatype 0x%"PRIx64,
+                        ucp_datatype)
+    } else {
+        status = ucp_dt_create_generic(&pml_ucx_generic_datatype_ops,
+                                       datatype, &ucp_datatype);
+        if (status != UCS_OK) {
+            int err = MPI_ERR_INTERN;
+            PML_UCX_ERROR("Failed to create UCX datatype for %s",
+                          datatype->name);
+            /* TODO: this error should return to the caller and invoke an error
+             * handler from the MPI API call.
+             * For now, it is fatal. */
+            ompi_mpi_errors_are_fatal_comm_handler(NULL, &err,
+                                                   "Failed to allocate "
+                                                   "datatype structure");
+        }
+        PML_UCX_VERBOSE(7, "created generic UCX datatype 0x%"PRIx64, ucp_datatype)
     }
 
     /* Add custom attribute, to clean up UCX resources when OMPI datatype is
@@ -254,8 +282,6 @@ ucp_datatype_t mca_pml_ucx_init_datatype(ompi_datatype_t *datatype)
             ompi_mpi_errors_are_fatal_comm_handler(NULL, &err, "Failed to allocate datatype structure");
         }
     }
-out:
-    PML_UCX_VERBOSE(7, "created generic UCX datatype 0x%"PRIx64, ucp_datatype)
 
 #ifdef HAVE_UCP_REQUEST_PARAM_T
     UCS_STATIC_ASSERT(sizeof(datatype->pml_data) >= sizeof(pml_ucx_datatype_t*));
@@ -266,7 +292,10 @@ ucp_datatype_t mca_pml_ucx_init_datatype(ompi_datatype_t *datatype)
     datatype->pml_data = ucp_datatype;
 #endif
 
-    return ucp_datatype;
+out:
+    opal_thread_internal_mutex_unlock(&lock);
+
+    return mca_pml_ucx_from_ompi_datatype(datatype);
 }
 
 static void mca_pml_ucx_convertor_construct(mca_pml_ucx_convertor_t *convertor)
diff --git a/ompi/mca/pml/ucx/pml_ucx_datatype.h b/ompi/mca/pml/ucx/pml_ucx_datatype.h
index 921aab0d399..c2d7707f1f9 100644
--- a/ompi/mca/pml/ucx/pml_ucx_datatype.h
+++ b/ompi/mca/pml/ucx/pml_ucx_datatype.h
@@ -21,8 +21,9 @@ typedef struct {
     int                     size_shift;
     struct {
         ucp_request_param_t send;
-        ucp_request_param_t bsend;
+        ucp_request_param_t isend;
         ucp_request_param_t recv;
+        ucp_request_param_t irecv;
     } op_param;
 } pml_ucx_datatype_t;
 #endif
@@ -43,23 +44,25 @@ OBJ_CLASS_DECLARATION(mca_pml_ucx_convertor_t);
 
 
 __opal_attribute_always_inline__
-static inline ucp_datatype_t mca_pml_ucx_get_datatype(ompi_datatype_t *datatype)
+static inline ucp_datatype_t
+mca_pml_ucx_from_ompi_datatype(ompi_datatype_t *datatype)
 {
 #ifdef HAVE_UCP_REQUEST_PARAM_T
-    pml_ucx_datatype_t *ucp_type = (pml_ucx_datatype_t*)datatype->pml_data;
-
-    if (OPAL_LIKELY(ucp_type != PML_UCX_DATATYPE_INVALID)) {
-        return ucp_type->datatype;
-    }
+    return ((pml_ucx_datatype_t*)datatype->pml_data)->datatype;
 #else
-    ucp_datatype_t ucp_type = datatype->pml_data;
+    return (ucp_datatype_t)datatype->pml_data;
+#endif
+}
 
-    if (OPAL_LIKELY(ucp_type != PML_UCX_DATATYPE_INVALID)) {
-        return ucp_type;
+
+__opal_attribute_always_inline__
+static inline ucp_datatype_t mca_pml_ucx_get_datatype(ompi_datatype_t *datatype)
+{
+    if (OPAL_UNLIKELY(datatype->pml_data == PML_UCX_DATATYPE_INVALID)) {
+        return mca_pml_ucx_init_datatype(datatype);
     }
-#endif
 
-    return mca_pml_ucx_init_datatype(datatype);
+    return mca_pml_ucx_from_ompi_datatype(datatype);
 }
 
 #ifdef HAVE_UCP_REQUEST_PARAM_T
diff --git a/ompi/mca/pml/ucx/pml_ucx_request.c b/ompi/mca/pml/ucx/pml_ucx_request.c
index 744d5803587..1a8d0dbc043 100644
--- a/ompi/mca/pml/ucx/pml_ucx_request.c
+++ b/ompi/mca/pml/ucx/pml_ucx_request.c
@@ -3,6 +3,7 @@
  * Copyright (c) 2016      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -14,6 +15,7 @@
 #include "ompi/mca/pml/base/pml_base_bsend.h"
 #include "ompi/message/message.h"
 #include "ompi/runtime/ompi_spc.h"
+#include "ompi/request/request.h"
 #include <inttypes.h>
 
 
@@ -29,12 +31,20 @@ static int mca_pml_ucx_request_free(ompi_request_t **rptr)
     return OMPI_SUCCESS;
 }
 
-static int mca_pml_ucx_request_cancel(ompi_request_t *req, int flag)
+int mca_pml_ucx_request_cancel(ompi_request_t *req, int flag)
 {
     ucp_request_cancel(ompi_pml_ucx.ucp_worker, req);
     return OMPI_SUCCESS;
 }
 
+#if MPI_VERSION >= 4
+int mca_pml_ucx_request_cancel_send(ompi_request_t *req, int flag)
+{
+    mca_pml_cancel_send_callback(req, flag);
+    return mca_pml_ucx_request_cancel(req, flag);
+}
+#endif
+
 __opal_attribute_always_inline__ static inline void
 mca_pml_ucx_send_completion_internal(void *request, ucs_status_t status)
 {
@@ -85,6 +95,11 @@ void mca_pml_ucx_send_completion(void *request, ucs_status_t status)
     mca_pml_ucx_send_completion_internal(request, status);
 }
 
+void mca_pml_ucx_send_completion_empty(void *request, ucs_status_t status)
+{
+    /* empty */
+}
+
 void mca_pml_ucx_bsend_completion(void *request, ucs_status_t status)
 {
     mca_pml_ucx_bsend_completion_internal(request, status);
@@ -181,10 +196,10 @@ static void mca_pml_ucx_request_init_common(ompi_request_t* ompi_req,
     ompi_req->req_start            = mca_pml_ucx_start;
     ompi_req->req_free             = req_free;
     ompi_req->req_cancel           = req_cancel;
-    /* This field is used to attach persistant request to a temporary req.
+    /* This field is used to attach persistent request to a temporary req.
      * Receive (ucp_tag_recv_nb) may call completion callback
      * before the field is set. If the field is not NULL then mca_pml_ucx_preq_completion() 
-     * will try to complete bogus persistant request.
+     * will try to complete bogus persistent request.
      */ 
     ompi_req->req_complete_cb_data = NULL;
 }
@@ -216,10 +231,7 @@ static int mca_pml_ucx_persistent_request_free(ompi_request_t **rptr)
         mca_pml_ucx_persistent_request_detach(preq, tmp_req);
         ucp_request_free(tmp_req);
     }
-    if ((preq->flags & MCA_PML_UCX_REQUEST_FLAG_SEND) &&
-         (MCA_PML_BASE_SEND_BUFFERED == preq->send.mode)) {
-        OBJ_RELEASE(preq->datatype.ompi_datatype);
-    }
+    OMPI_DATATYPE_RELEASE(preq->ompi_datatype);
     PML_UCX_FREELIST_RETURN(&ompi_pml_ucx.persistent_reqs, &preq->ompi.super);
     *rptr = MPI_REQUEST_NULL;
     return OMPI_SUCCESS;
@@ -270,7 +282,7 @@ void mca_pml_ucx_completed_request_init(ompi_request_t *ompi_req)
     mca_pml_ucx_request_init_common(ompi_req, false, OMPI_REQUEST_ACTIVE,
                                     mca_pml_completed_request_free,
                                     mca_pml_completed_request_cancel);
-    ompi_req->req_mpi_object.comm = &ompi_mpi_comm_world.comm;
+    ompi_req->req_mpi_object.comm = &ompi_mpi_comm_null.comm;
     ompi_request_complete(ompi_req, false);
 }
 
diff --git a/ompi/mca/pml/ucx/pml_ucx_request.h b/ompi/mca/pml/ucx/pml_ucx_request.h
index bfa3d6c858c..9e901794f8d 100644
--- a/ompi/mca/pml/ucx/pml_ucx_request.h
+++ b/ompi/mca/pml/ucx/pml_ucx_request.h
@@ -1,8 +1,9 @@
 /*
  * Copyright (C) Mellanox Technologies Ltd. 2001-2015.  ALL RIGHTS RESERVED.
- * Copyright (c) 2016      The University of Tennessee and The University
+ * Copyright (c) 2016-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -15,6 +16,9 @@
 
 #include "pml_ucx.h"
 #include "pml_ucx_datatype.h"
+#if MPI_VERSION >= 4
+#include "ompi/mca/pml/base/pml_base_sendreq.h"
+#endif
 
 
 enum {
@@ -39,10 +43,10 @@ enum {
 #define PML_UCX_TAG_MASK                       0x7fffff0000000000ul
 
 
-#define PML_UCX_MAKE_SEND_TAG(_tag, _comm) \
+#define PML_UCX_MAKE_SEND_TAG(_tag, _comm, _c_index) \
     ((((uint64_t) (_tag)            ) << (PML_UCX_RANK_BITS + PML_UCX_CONTEXT_BITS)) | \
      (((uint64_t)(_comm)->c_my_rank ) << PML_UCX_CONTEXT_BITS) | \
-     ((uint64_t)(_comm)->c_contextid))
+     ((uint64_t)(_c_index)))
 
 
 #define PML_UCX_MAKE_RECV_TAG(_ucp_tag, _ucp_tag_mask, _tag, _src, _comm) \
@@ -54,7 +58,7 @@ enum {
         } \
         \
         _ucp_tag = (((uint64_t)(_src) & UCS_MASK(PML_UCX_RANK_BITS)) << PML_UCX_CONTEXT_BITS) | \
-                   (_comm)->c_contextid; \
+                   (_comm)->c_index; \
         \
         if ((_tag) != MPI_ANY_TAG) { \
             _ucp_tag_mask |= PML_UCX_TAG_MASK; \
@@ -99,10 +103,8 @@ struct pml_ucx_persistent_request {
     unsigned                          flags;
     void                              *buffer;
     size_t                            count;
-    union {
-        ucp_datatype_t                datatype;
-        ompi_datatype_t              *ompi_datatype;
-    } datatype;
+    ucp_datatype_t                    datatype;
+    ompi_datatype_t                   *ompi_datatype;
     ucp_tag_t                         tag;
     struct {
         mca_pml_base_send_mode_t      mode;
@@ -119,6 +121,8 @@ void mca_pml_ucx_send_completion(void *request, ucs_status_t status);
 void mca_pml_ucx_recv_completion(void *request, ucs_status_t status,
                                  ucp_tag_recv_info_t *info);
 
+void mca_pml_ucx_send_completion_empty(void *request, ucs_status_t status);
+
 void mca_pml_ucx_psend_completion(void *request, ucs_status_t status);
 
 void mca_pml_ucx_bsend_completion(void *request, ucs_status_t status);
@@ -145,12 +149,23 @@ void mca_pml_ucx_request_init(void *request);
 
 void mca_pml_ucx_request_cleanup(void *request);
 
+int mca_pml_ucx_request_cancel(ompi_request_t *req, int flag);
+#if MPI_VERSION >= 4
+int mca_pml_ucx_request_cancel_send(ompi_request_t *req, int flag);
+#endif
+
 
 static inline void mca_pml_ucx_request_reset(ompi_request_t *req)
 {
     req->req_complete          = REQUEST_PENDING;
 }
 
+/* Use when setting a request's status field.
+ * Note that a new function 'mca_mpl_ucx_set_send_status_public' shall
+ * be created and used instead if updating a publicly visible status becomes
+ * necessary (i.e., the status argument in an user-visible procedure), see the
+ * recv_status case below for rationale.
+ */
 __opal_attribute_always_inline__
 static inline void mca_pml_ucx_set_send_status(ompi_status_public_t* mpi_status,
                                                ucs_status_t status)
@@ -165,14 +180,18 @@ static inline void mca_pml_ucx_set_send_status(ompi_status_public_t* mpi_status,
     }
 }
 
+/* Use when setting a request's status field.
+ * Note that the next function 'mca_mpl_ucx_set_recv_status_public' shall
+ * be used instead when updating a  publicly visible status (i.e., the 
+ * status argument in an user-visible procedure).
+ */
 static inline int mca_pml_ucx_set_recv_status(ompi_status_public_t* mpi_status,
                                                ucs_status_t ucp_status,
                                                const ucp_tag_recv_info_t *info)
 {
-    int64_t tag;
+    int64_t tag = info->sender_tag;
 
     if (OPAL_LIKELY(ucp_status == UCS_OK)) {
-        tag = info->sender_tag;
         mpi_status->MPI_ERROR  = MPI_SUCCESS;
         mpi_status->MPI_SOURCE = PML_UCX_TAG_GET_SOURCE(tag);
         mpi_status->MPI_TAG    = PML_UCX_TAG_GET_MPI_TAG(tag);
@@ -180,6 +199,10 @@ static inline int mca_pml_ucx_set_recv_status(ompi_status_public_t* mpi_status,
         mpi_status->_ucount    = info->length;
     } else if (ucp_status == UCS_ERR_MESSAGE_TRUNCATED) {
         mpi_status->MPI_ERROR = MPI_ERR_TRUNCATE;
+        mpi_status->MPI_SOURCE = PML_UCX_TAG_GET_SOURCE(tag);
+        mpi_status->MPI_TAG    = PML_UCX_TAG_GET_MPI_TAG(tag);
+        mpi_status->_cancelled = false;
+        mpi_status->_ucount    = info->length;
     } else if (ucp_status == UCS_ERR_CANCELED) {
         mpi_status->MPI_ERROR  = MPI_SUCCESS;
         mpi_status->_cancelled = true;
@@ -190,16 +213,41 @@ static inline int mca_pml_ucx_set_recv_status(ompi_status_public_t* mpi_status,
     return mpi_status->MPI_ERROR;
 }
 
-static inline int mca_pml_ucx_set_recv_status_safe(ompi_status_public_t* mpi_status,
+/* Use when setting a publicly visible status (i.e., the status argument in an
+ * user-visible procedure).
+ * Except in procedures that return MPI_ERR_IN_STATUS, the MPI_ERROR
+ * field of a status object shall never be modified
+ * See MPI-1.1 doc, sec 3.2.5, p.22
+ */
+static inline int mca_pml_ucx_set_recv_status_public(ompi_status_public_t* mpi_status,
                                                    ucs_status_t ucp_status,
                                                    const ucp_tag_recv_info_t *info)
 {
     if (mpi_status != MPI_STATUS_IGNORE) {
-        return mca_pml_ucx_set_recv_status(mpi_status, ucp_status, info);
-    } else if (OPAL_LIKELY(ucp_status == UCS_OK) || (ucp_status == UCS_ERR_CANCELED)) {
-        return UCS_OK;
+        if (OPAL_LIKELY(ucp_status == UCS_OK)) {
+            uint64_t tag = info->sender_tag;
+            mpi_status->MPI_SOURCE = PML_UCX_TAG_GET_SOURCE(tag);
+            mpi_status->MPI_TAG = PML_UCX_TAG_GET_MPI_TAG(tag);
+            mpi_status->_cancelled = false;
+            mpi_status->_ucount = info->length;
+            return MPI_SUCCESS;
+        } else if (ucp_status == UCS_ERR_MESSAGE_TRUNCATED) {
+            uint64_t tag = info->sender_tag;
+            mpi_status->MPI_SOURCE = PML_UCX_TAG_GET_SOURCE(tag);
+            mpi_status->MPI_TAG = PML_UCX_TAG_GET_MPI_TAG(tag);
+            mpi_status->_cancelled = false;
+            mpi_status->_ucount = info->length;
+            return MPI_ERR_TRUNCATE;
+        } else if (ucp_status == UCS_ERR_CANCELED) {
+            mpi_status->_cancelled = true;
+            return MPI_SUCCESS;
+        } else {
+            return MPI_ERR_INTERN;
+        }
     } else if (ucp_status == UCS_ERR_MESSAGE_TRUNCATED) {
         return MPI_ERR_TRUNCATE;
+    } else if (OPAL_LIKELY(ucp_status == UCS_OK) || (ucp_status == UCS_ERR_CANCELED)) {
+        return MPI_SUCCESS;
     }
 
     return MPI_ERR_INTERN;
diff --git a/ompi/mca/pml/v/pml_v.h b/ompi/mca/pml/v/pml_v.h
index 1c0c609972d..cc718471c12 100644
--- a/ompi/mca/pml/v/pml_v.h
+++ b/ompi/mca/pml/v/pml_v.h
@@ -19,7 +19,7 @@
 
 BEGIN_C_DECLS
 
-OMPI_MODULE_DECLSPEC extern mca_pml_base_component_2_1_0_t mca_pml_v_component;
+OMPI_DECLSPEC extern mca_pml_base_component_2_1_0_t mca_pml_v_component;
 
 END_C_DECLS
 
diff --git a/ompi/mca/sharedfp/base/Makefile.am b/ompi/mca/sharedfp/base/Makefile.am
index 419d0de87e1..017e4efd332 100644
--- a/ompi/mca/sharedfp/base/Makefile.am
+++ b/ompi/mca/sharedfp/base/Makefile.am
@@ -21,8 +21,12 @@
 headers += \
         base/base.h
 
+libmca_sharedfp_la_SOURCES += \
+        base/sharedfp_base_frame.c \
+        base/sharedfp_base_find_available.c
+
+if OMPI_OMPIO_SUPPORT
 libmca_sharedfp_la_SOURCES += \
         base/sharedfp_base_file_select.c \
-        base/sharedfp_base_file_unselect.c \
-        base/sharedfp_base_find_available.c \
-        base/sharedfp_base_frame.c
+        base/sharedfp_base_file_unselect.c
+endif
diff --git a/ompi/mca/sharedfp/base/sharedfp_base_file_select.c b/ompi/mca/sharedfp/base/sharedfp_base_file_select.c
index a6fe773df42..adaeea2b487 100644
--- a/ompi/mca/sharedfp/base/sharedfp_base_file_select.c
+++ b/ompi/mca/sharedfp/base/sharedfp_base_file_select.c
@@ -50,7 +50,7 @@ static OBJ_CLASS_INSTANCE(queried_module_t, opal_list_item_t, NULL, NULL);
 /*
  * Only one sharedfp module can be attached to each file.
  *
- * This module calls the query funtion on all the components that were
+ * This module calls the query function on all the components that were
  * detected by sharedfp_base_open. This function is called on a
  * per-file basis. This function has the following function.
  *
@@ -220,14 +220,14 @@ int mca_sharedfp_base_file_select (struct ompio_file_t *file,
             * module of this component.
             *
             * ANJU: a component might not have all the functions
-            * defined.  Whereever a function pointer is null in the
+            * defined.  Wherever a function pointer is null in the
             * module structure we need to fill it in with the base
             * structure function pointers. This is yet to be done
             */
 
             /*
-             * We don return here coz we still need to go through and
-             * elease the other objects
+             * We don't return here coz we still need to go through and
+             * release the other objects
              */
 
             /*fill_null_pointers (om->om_module);*/
@@ -237,7 +237,7 @@ int mca_sharedfp_base_file_select (struct ompio_file_t *file,
 
          } else {
             /*
-             * this is not the "choosen one", finalize
+             * this is not the "chosen one", finalize
              */
              if (NULL != om->om_component->sharedfpm_file_unquery) {
                 /* unquery the component only if they have some clean
diff --git a/ompi/mca/sharedfp/base/sharedfp_base_find_available.c b/ompi/mca/sharedfp/base/sharedfp_base_find_available.c
index 54d5f2fe2da..085d8fb0251 100644
--- a/ompi/mca/sharedfp/base/sharedfp_base_find_available.c
+++ b/ompi/mca/sharedfp/base/sharedfp_base_find_available.c
@@ -10,6 +10,8 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2008-2011 University of Houston. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -36,7 +38,7 @@ static int init_query(const mca_base_component_t *m,
                       mca_base_component_list_item_t *entry,
                       bool enable_progress_threads,
                       bool enable_mpi_threads);
-static int init_query_2_0_0(const mca_base_component_t *component,
+static int init_query_3_0_0(const mca_base_component_t *component,
                             mca_base_component_list_item_t *entry,
                             bool enable_progress_threads,
                             bool enable_mpi_threads);
@@ -95,10 +97,10 @@ static int init_query(const mca_base_component_t *m,
                         m->mca_component_name);
 
     /* This component has been successfully opened, now try to query it */
-    if (2 == m->mca_type_major_version &&
+    if (3 == m->mca_type_major_version &&
         0 == m->mca_type_minor_version &&
         0 == m->mca_type_release_version) {
-        ret = init_query_2_0_0(m, entry, enable_progress_threads,
+        ret = init_query_3_0_0(m, entry, enable_progress_threads,
                                enable_mpi_threads);
     } else {
         /* unrecognised API version */
@@ -129,13 +131,13 @@ static int init_query(const mca_base_component_t *m,
 }
 
 
-static int init_query_2_0_0(const mca_base_component_t *component,
+static int init_query_3_0_0(const mca_base_component_t *component,
                             mca_base_component_list_item_t *entry,
                             bool enable_progress_threads,
                             bool enable_mpi_threads)
 {
-    mca_sharedfp_base_component_2_0_0_t *sharedfp =
-        (mca_sharedfp_base_component_2_0_0_t *) component;
+    mca_sharedfp_base_component_3_0_0_t *sharedfp =
+        (mca_sharedfp_base_component_3_0_0_t *) component;
 
     return sharedfp->sharedfpm_init_query(enable_progress_threads,
                               enable_mpi_threads);
diff --git a/ompi/mca/sharedfp/individual/sharedfp_individual.c b/ompi/mca/sharedfp/individual/sharedfp_individual.c
index 2d66e201978..d6e06dd450e 100644
--- a/ompi/mca/sharedfp/individual/sharedfp_individual.c
+++ b/ompi/mca/sharedfp/individual/sharedfp_individual.c
@@ -13,6 +13,8 @@
  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
  * Copyright (c) 2018      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -38,8 +40,8 @@
  * *******************************************************************
  */
  /* IMPORTANT: Update here when adding sharedfp component interface functions*/
-static mca_sharedfp_base_module_1_0_0_t individual =  {
-    mca_sharedfp_individual_module_init, /* initalise after being selected */
+static mca_sharedfp_base_module_2_0_0_t individual =  {
+    mca_sharedfp_individual_module_init, /* initialise after being selected */
     mca_sharedfp_individual_module_finalize, /* close a module on a communicator */
     mca_sharedfp_individual_seek,
     mca_sharedfp_individual_get_position,
@@ -70,15 +72,14 @@ int mca_sharedfp_individual_component_init_query(bool enable_progress_threads,
    return OMPI_SUCCESS;
 }
 
-struct mca_sharedfp_base_module_1_0_0_t * mca_sharedfp_individual_component_file_query (ompio_file_t *fh, int *priority) {
+struct mca_sharedfp_base_module_2_0_0_t * mca_sharedfp_individual_component_file_query (ompio_file_t *fh, int *priority) {
 
     int amode;
     bool wronly_flag=false;
     bool relaxed_order_flag=false;
     opal_info_t *info;
     int flag;
-    int valuelen;
-    char value[MPI_MAX_INFO_VAL+1];
+    opal_cstring_t *info_str;
     *priority = 0;
 
     /*test, and update priority*/
@@ -105,16 +106,16 @@ struct mca_sharedfp_base_module_1_0_0_t * mca_sharedfp_individual_component_file
     /* 2. Did the user specify MPI_INFO relaxed ordering flag? */
     info = fh->f_info;
     if ( info != &(MPI_INFO_NULL->super) ){
-        valuelen = MPI_MAX_INFO_VAL;
-        opal_info_get ( info,"OMPIO_SHAREDFP_RELAXED_ORDERING", valuelen, value, &flag);
+        opal_info_get ( info,"OMPIO_SHAREDFP_RELAXED_ORDERING", &info_str, &flag);
         if ( flag ) {
            if ( mca_sharedfp_individual_verbose ) {
                 opal_output(ompi_sharedfp_base_framework.framework_output,
                         "mca_sharedfp_individual_component_file_query: "
-                        "OMPIO_SHAREDFP_RELAXED_ORDERING=%s\n",value);
+                        "OMPIO_SHAREDFP_RELAXED_ORDERING=%s\n", info_str->string);
 	    }
             /* flag - Returns true if key defined, false if not (boolean). */
             relaxed_order_flag=true;
+            OBJ_RELEASE(info_str);
         }
         else {
             if ( mca_sharedfp_individual_verbose ) {
diff --git a/ompi/mca/sharedfp/individual/sharedfp_individual.h b/ompi/mca/sharedfp/individual/sharedfp_individual.h
index bad60ce0e6a..d31ef0b64aa 100644
--- a/ompi/mca/sharedfp/individual/sharedfp_individual.h
+++ b/ompi/mca/sharedfp/individual/sharedfp_individual.h
@@ -13,6 +13,8 @@
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -33,7 +35,7 @@ BEGIN_C_DECLS
 
 int mca_sharedfp_individual_component_init_query(bool enable_progress_threads,
                                                  bool enable_mpi_threads);
-struct mca_sharedfp_base_module_1_0_0_t *
+struct mca_sharedfp_base_module_2_0_0_t *
         mca_sharedfp_individual_component_file_query (ompio_file_t *file, int *priority);
 int mca_sharedfp_individual_component_file_unquery (ompio_file_t *file);
 
@@ -44,7 +46,7 @@ extern int mca_sharedfp_individual_priority;
 extern int mca_sharedfp_individual_verbose;
 extern int mca_sharedfp_individual_usage_counter;
 
-OMPI_MODULE_DECLSPEC extern mca_sharedfp_base_component_2_0_0_t mca_sharedfp_individual_component;
+OMPI_DECLSPEC extern mca_sharedfp_base_component_3_0_0_t mca_sharedfp_individual_component;
 /*
  * ******************************************************************
  * ********* functions which are implemented in this module *********
@@ -63,42 +65,42 @@ int mca_sharedfp_individual_file_open (struct ompi_communicator_t *comm,
                                        ompio_file_t *fh);
 int mca_sharedfp_individual_file_close (ompio_file_t *fh);
 int mca_sharedfp_individual_read (ompio_file_t *fh,
-                                  void *buf, int count, MPI_Datatype datatype, MPI_Status *status);
+                                  void *buf, size_t count, MPI_Datatype datatype, MPI_Status *status);
 int mca_sharedfp_individual_read_ordered (ompio_file_t *fh,
-                                          void *buf, int count, struct ompi_datatype_t *datatype,
+                                          void *buf, size_t count, struct ompi_datatype_t *datatype,
                                           ompi_status_public_t *status);
 int mca_sharedfp_individual_read_ordered_begin (ompio_file_t *fh,
                                                  void *buf,
-                                                 int count,
+                                                 size_t count,
                                                  struct ompi_datatype_t *datatype);
 int mca_sharedfp_individual_read_ordered_end (ompio_file_t *fh,
                                                void *buf,
                                                ompi_status_public_t *status);
 int mca_sharedfp_individual_iread (ompio_file_t *fh,
                                     void *buf,
-                                    int count,
+                                    size_t count,
                                     struct ompi_datatype_t *datatype,
                                     ompi_request_t **request);
 int mca_sharedfp_individual_write (ompio_file_t *fh,
                                    const void *buf,
-                                   int count,
+                                   size_t count,
                                    struct ompi_datatype_t *datatype,
                                    ompi_status_public_t *status);
 int mca_sharedfp_individual_write_ordered (ompio_file_t *fh,
                                            const void *buf,
-                                           int count,
+                                           size_t count,
                                            struct ompi_datatype_t *datatype,
                                            ompi_status_public_t *status);
 int mca_sharedfp_individual_write_ordered_begin (ompio_file_t *fh,
                                                  const void *buf,
-                                                 int count,
+                                                 size_t count,
                                                  struct ompi_datatype_t *datatype);
 int mca_sharedfp_individual_write_ordered_end (ompio_file_t *fh,
                                                const void *buf,
                                                ompi_status_public_t *status);
 int mca_sharedfp_individual_iwrite (ompio_file_t *fh,
                                     const void *buf,
-                                    int count,
+                                    size_t count,
                                     struct ompi_datatype_t *datatype,
                                     ompi_request_t **request);
 
diff --git a/ompi/mca/sharedfp/individual/sharedfp_individual_collaborate_data.c b/ompi/mca/sharedfp/individual/sharedfp_individual_collaborate_data.c
index 52d8ad4bad3..d3edc3ec5be 100644
--- a/ompi/mca/sharedfp/individual/sharedfp_individual_collaborate_data.c
+++ b/ompi/mca/sharedfp/individual/sharedfp_individual_collaborate_data.c
@@ -44,6 +44,8 @@ int mca_sharedfp_individual_collaborate_data(struct mca_sharedfp_base_data_t *sh
     OMPI_MPI_OFFSET_TYPE *offsetbuff = NULL;
     int *countbuff = NULL;
     int *displ = NULL;
+    ompi_count_array_t countbuff_desc;
+    ompi_disp_array_t displ_desc;
     double *ind_ts = NULL;
     long *ind_recordlength = NULL;
     OMPI_MPI_OFFSET_TYPE *local_off = NULL;
@@ -142,12 +144,15 @@ int mca_sharedfp_individual_collaborate_data(struct mca_sharedfp_base_data_t *sh
 	goto exit;
     }
 
+    OMPI_COUNT_ARRAY_INIT(&countbuff_desc, countbuff);
+    OMPI_DISP_ARRAY_INIT(&displ_desc, displ);
+
     ret = ompio_fh->f_comm->c_coll->coll_allgatherv ( ind_ts, 
                                                       countbuff[ompio_fh->f_rank], 
                                                       MPI_DOUBLE,
                                                       timestampbuff, 
-                                                      countbuff, 
-                                                      displ, 
+                                                      countbuff_desc,
+                                                      displ_desc,
                                                       MPI_DOUBLE,
                                                       ompio_fh->f_comm, 
                                                       ompio_fh->f_comm->c_coll->coll_allgatherv_module );
@@ -159,8 +164,8 @@ int mca_sharedfp_individual_collaborate_data(struct mca_sharedfp_base_data_t *sh
                                                       countbuff[ompio_fh->f_rank], 
                                                       OMPI_OFFSET_DATATYPE,
                                                       offsetbuff, 
-                                                      countbuff, 
-                                                      displ, 
+                                                      countbuff_desc,
+                                                      displ_desc,
                                                       OMPI_OFFSET_DATATYPE,
                                                       ompio_fh->f_comm, 
                                                       ompio_fh->f_comm->c_coll->coll_allgatherv_module );
diff --git a/ompi/mca/sharedfp/individual/sharedfp_individual_component.c b/ompi/mca/sharedfp/individual/sharedfp_individual_component.c
index 93fda2ff485..fff66f8bbf8 100644
--- a/ompi/mca/sharedfp/individual/sharedfp_individual_component.c
+++ b/ompi/mca/sharedfp/individual/sharedfp_individual_component.c
@@ -13,6 +13,8 @@
  * Copyright (c) 2013      University of Houston. All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -48,13 +50,13 @@ static int individual_register(void);
  * Instantiate the public struct with all of our public information
  * and pointers to our public functions in it
  */
-mca_sharedfp_base_component_2_0_0_t mca_sharedfp_individual_component = {
+mca_sharedfp_base_component_3_0_0_t mca_sharedfp_individual_component = {
 
     /* First, the mca_component_t struct containing meta information
        about the component itself */
 
     .sharedfpm_version = {
-        MCA_SHAREDFP_BASE_VERSION_2_0_0,
+        MCA_SHAREDFP_BASE_VERSION_3_0_0,
 
         /* Component name and version */
         .mca_component_name = "individual",
diff --git a/ompi/mca/sharedfp/individual/sharedfp_individual_file_open.c b/ompi/mca/sharedfp/individual/sharedfp_individual_file_open.c
index 5bff7247362..fbe0907167a 100644
--- a/ompi/mca/sharedfp/individual/sharedfp_individual_file_open.c
+++ b/ompi/mca/sharedfp/individual/sharedfp_individual_file_open.c
@@ -192,7 +192,7 @@ int mca_sharedfp_individual_file_close (ompio_file_t *fh)
     }
     sh = fh->f_sharedfp_data;
 
-    /* Merge data from individal files to final output file */
+    /* Merge data from individual files to final output file */
     err = mca_sharedfp_individual_collaborate_data (sh, fh);
 
     headnode = (mca_sharedfp_individual_header_record*)(sh->selected_module_data);
@@ -201,7 +201,7 @@ int mca_sharedfp_individual_file_close (ompio_file_t *fh)
         if (headnode->datafilehandle)  {
             /*TODO: properly deal with returned error code*/
             err = mca_common_ompio_file_close(headnode->datafilehandle);
-            /* NOTE: No neeed to manually delete the file,
+            /* NOTE: No need to manually delete the file,
 	    ** the amode should have been set to delete on close
 	    */
         }
@@ -213,7 +213,7 @@ int mca_sharedfp_individual_file_close (ompio_file_t *fh)
         if (headnode->metadatafilehandle)  {
             /*TODO: properly deal with returned error code*/
             err = mca_common_ompio_file_close(headnode->metadatafilehandle);
-            /* NOTE: No neeed to manually delete the file,
+            /* NOTE: No need to manually delete the file,
 	    ** the amode should have been set to delete on close
 	    */
         }
diff --git a/ompi/mca/sharedfp/individual/sharedfp_individual_iwrite.c b/ompi/mca/sharedfp/individual/sharedfp_individual_iwrite.c
index 82d5d3bb63a..caccbea9f97 100644
--- a/ompi/mca/sharedfp/individual/sharedfp_individual_iwrite.c
+++ b/ompi/mca/sharedfp/individual/sharedfp_individual_iwrite.c
@@ -12,6 +12,8 @@
  * Copyright (c) 2013-2018 University of Houston. All rights reserved.
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -30,7 +32,7 @@
 
 int mca_sharedfp_individual_iwrite(ompio_file_t *fh,
                                    const void *buf,
-                                   int count,
+                                   size_t count,
                                    ompi_datatype_t *datatype,
                                    MPI_Request * request)
 {
@@ -79,7 +81,7 @@ int mca_sharedfp_individual_iwrite(ompio_file_t *fh,
 
 int mca_sharedfp_individual_write_ordered_begin(ompio_file_t *fh,
                                                 const void *buf,
-                                                int count,
+                                                size_t count,
                                                 struct ompi_datatype_t *datatype)
 {
     int ret = OMPI_SUCCESS;
diff --git a/ompi/mca/sharedfp/individual/sharedfp_individual_read.c b/ompi/mca/sharedfp/individual/sharedfp_individual_read.c
index fa6953e3854..bb7aa164c2c 100644
--- a/ompi/mca/sharedfp/individual/sharedfp_individual_read.c
+++ b/ompi/mca/sharedfp/individual/sharedfp_individual_read.c
@@ -12,6 +12,8 @@
  * Copyright (c) 2008      University of Houston. All rights reserved.
  * Copyright (c) 2018      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -29,14 +31,14 @@
 #include "ompi/mca/sharedfp/base/base.h"
 
 int mca_sharedfp_individual_read ( ompio_file_t *fh,
-				   void *buf, int count, MPI_Datatype datatype, MPI_Status *status)
+				   void *buf, size_t count, MPI_Datatype datatype, MPI_Status *status)
 {
     opal_output(0,"mca_sharedfp_individual_read: NOT SUPPORTED by this component\n");
     return OMPI_ERROR;
 }
 
 int mca_sharedfp_individual_read_ordered ( ompio_file_t *fh,
-                                   void *buf, int count, MPI_Datatype datatype, MPI_Status *status)
+                                   void *buf, size_t count, MPI_Datatype datatype, MPI_Status *status)
 {
     opal_output(0,"mca_sharedfp_individual_read_ordered: NOT SUPPORTED by this component\n");
     return OMPI_ERROR;
@@ -44,7 +46,7 @@ int mca_sharedfp_individual_read_ordered ( ompio_file_t *fh,
 
 int mca_sharedfp_individual_iread(ompio_file_t *fh,
                                    void *buf,
-                                   int count,
+                                   size_t count,
                                    ompi_datatype_t *datatype,
                                    MPI_Request * request)
 {
@@ -54,7 +56,7 @@ int mca_sharedfp_individual_iread(ompio_file_t *fh,
 
 int mca_sharedfp_individual_read_ordered_begin(ompio_file_t *fh,
                                                 void *buf,
-                                                int count,
+                                                size_t count,
                                                 struct ompi_datatype_t *datatype)
 {
     opal_output(0,"mca_sharedfp_individual_read_ordered_begin: NOT SUPPORTED by this component\n");
diff --git a/ompi/mca/sharedfp/individual/sharedfp_individual_write.c b/ompi/mca/sharedfp/individual/sharedfp_individual_write.c
index a1d21fb96c9..8928dbe9b05 100644
--- a/ompi/mca/sharedfp/individual/sharedfp_individual_write.c
+++ b/ompi/mca/sharedfp/individual/sharedfp_individual_write.c
@@ -12,6 +12,8 @@
  * Copyright (c) 2013-2018 University of Houston. All rights reserved.
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -30,7 +32,7 @@
 
 int mca_sharedfp_individual_write (ompio_file_t *fh,
                                        const void *buf,
-                                       int count,
+                                       size_t count,
                                        struct ompi_datatype_t *datatype,
                                        ompi_status_public_t *status)
 {
@@ -79,7 +81,7 @@ int mca_sharedfp_individual_write (ompio_file_t *fh,
 
 int mca_sharedfp_individual_write_ordered (ompio_file_t *fh,
                                            const void *buf,
-                                           int count,
+                                           size_t count,
                                            struct ompi_datatype_t *datatype,
                                            ompi_status_public_t *status)
 {
diff --git a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile.c b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile.c
index c49f21cc27f..cbeedd12c34 100644
--- a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile.c
+++ b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile.c
@@ -12,6 +12,8 @@
  * Copyright (c) 2013      University of Houston. All rights reserved.
  * Copyright (c) 2018      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -42,8 +44,8 @@
  * *******************************************************************
  */
  /* IMPORTANT: Update here when adding sharedfp component interface functions*/
-static mca_sharedfp_base_module_1_0_0_t lockedfile =  {
-    mca_sharedfp_lockedfile_module_init, /* initalise after being selected */
+static mca_sharedfp_base_module_2_0_0_t lockedfile =  {
+    mca_sharedfp_lockedfile_module_init, /* initialise after being selected */
     mca_sharedfp_lockedfile_module_finalize, /* close a module on a communicator */
     mca_sharedfp_lockedfile_seek,
     mca_sharedfp_lockedfile_get_position,
@@ -74,7 +76,7 @@ int mca_sharedfp_lockedfile_component_init_query(bool enable_progress_threads,
     return OMPI_SUCCESS;
 }
 
-struct mca_sharedfp_base_module_1_0_0_t * mca_sharedfp_lockedfile_component_file_query(ompio_file_t *fh, int *priority) {
+struct mca_sharedfp_base_module_2_0_0_t * mca_sharedfp_lockedfile_component_file_query(ompio_file_t *fh, int *priority) {
     struct flock lock;
     int fd, err;
     /*char *filename;*/
@@ -126,30 +128,30 @@ struct mca_sharedfp_base_module_1_0_0_t * mca_sharedfp_lockedfile_component_file
     fd = open(filename, O_RDWR | O_CREAT, 0644);
 
     if ( -1 == fd ){
-        opal_output(ompi_sharedfp_base_framework.framework_output,
+        opal_output_verbose(10, ompi_sharedfp_base_framework.framework_output,
 		    "mca_sharedfp_lockedfile_component_file_query: error opening file %s %s", filename, strerror(errno));
         has_file_lock_support=false;
     }
     else{
         err = fcntl(fd, F_SETLKW, &lock);
-	opal_output(ompi_sharedfp_base_framework.framework_output,
+        opal_output_verbose(10, ompi_sharedfp_base_framework.framework_output,
 		    "mca_sharedfp_lockedfile_component_file_query: returned err=%d, for fd=%d\n",err,fd);
 
         if (err) {
-            opal_output(ompi_sharedfp_base_framework.framework_output,
+            opal_output_verbose(10, ompi_sharedfp_base_framework.framework_output,
 			"mca_sharedfp_lockedfile_component_file_query: Failed to set a file lock on %s %s\n", filename, strerror(errno) );
-            opal_output(ompi_sharedfp_base_framework.framework_output,
+            opal_output_verbose(10, ompi_sharedfp_base_framework.framework_output,
 			"err=%d, errno=%d, EOPNOTSUPP=%d, EINVAL=%d, ENOSYS=%d, EACCES=%d, EAGAIN=%d, EBADF=%d\n",
                         err, errno, EOPNOTSUPP, EINVAL, ENOSYS, EACCES, EAGAIN, EBADF);
 
             if (errno == EACCES || errno == EAGAIN) {
-                opal_output(ompi_sharedfp_base_framework.framework_output,
+                opal_output_verbose(10, ompi_sharedfp_base_framework.framework_output,
 			    "errno=EACCES || EAGAIN, Already locked by another process\n");
             }
 
         }
         else {
-	    opal_output(ompi_sharedfp_base_framework.framework_output,
+            opal_output_verbose(10, ompi_sharedfp_base_framework.framework_output,
 			"mca_sharedfp_lockedfile_component_file_query: fcntl claims success in setting a file lock on %s\n", filename );
 
             has_file_lock_support=true;
@@ -166,7 +168,7 @@ struct mca_sharedfp_base_module_1_0_0_t * mca_sharedfp_lockedfile_component_file
     *priority = 0;
     /*module can not run!, return NULL to indicate that we are unable to run*/
 
-    opal_output(ompi_sharedfp_base_framework.framework_output,
+    opal_output_verbose(10, ompi_sharedfp_base_framework.framework_output,
 		"mca_sharedfp_lockedfile_component_file_query: Can not run!, file locking not supported\n");
     return NULL;
 }
diff --git a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile.h b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile.h
index ba1d68fcf6c..486ab5e18e5 100644
--- a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile.h
+++ b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile.h
@@ -13,6 +13,8 @@
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -33,7 +35,7 @@ BEGIN_C_DECLS
 
 int mca_sharedfp_lockedfile_component_init_query(bool enable_progress_threads,
                                                  bool enable_mpi_threads);
-struct mca_sharedfp_base_module_1_0_0_t *
+struct mca_sharedfp_base_module_2_0_0_t *
         mca_sharedfp_lockedfile_component_file_query (ompio_file_t *file, int *priority);
 int mca_sharedfp_lockedfile_component_file_unquery (ompio_file_t *file);
 
@@ -43,7 +45,7 @@ int mca_sharedfp_lockedfile_module_finalize (ompio_file_t *file);
 extern int mca_sharedfp_lockedfile_priority;
 extern int mca_sharedfp_lockedfile_verbose;
 
-OMPI_MODULE_DECLSPEC extern mca_sharedfp_base_component_2_0_0_t mca_sharedfp_lockedfile_component;
+OMPI_DECLSPEC extern mca_sharedfp_base_component_3_0_0_t mca_sharedfp_lockedfile_component;
 /*
  * ******************************************************************
  * ********* functions which are implemented in this module *********
@@ -62,43 +64,43 @@ int mca_sharedfp_lockedfile_file_open (struct ompi_communicator_t *comm,
                                        ompio_file_t *fh);
 int mca_sharedfp_lockedfile_file_close (ompio_file_t *fh);
 int mca_sharedfp_lockedfile_read (ompio_file_t *fh,
-                                  void *buf, int count, MPI_Datatype datatype, MPI_Status *status);
+                                  void *buf, size_t count, MPI_Datatype datatype, MPI_Status *status);
 int mca_sharedfp_lockedfile_read_ordered (ompio_file_t *fh,
-                                          void *buf, int count, struct ompi_datatype_t *datatype,
+                                          void *buf, size_t count, struct ompi_datatype_t *datatype,
                                           ompi_status_public_t *status
                                           );
 int mca_sharedfp_lockedfile_read_ordered_begin (ompio_file_t *fh,
                                                  void *buf,
-                                                 int count,
+                                                 size_t count,
                                                  struct ompi_datatype_t *datatype);
 int mca_sharedfp_lockedfile_read_ordered_end (ompio_file_t *fh,
                                                void *buf,
                                                ompi_status_public_t *status);
 int mca_sharedfp_lockedfile_iread (ompio_file_t *fh,
                                     void *buf,
-                                    int count,
+                                    size_t count,
                                     struct ompi_datatype_t *datatype,
                                     ompi_request_t **request);
 int mca_sharedfp_lockedfile_write (ompio_file_t *fh,
                                    const void *buf,
-                                   int count,
+                                   size_t count,
                                    struct ompi_datatype_t *datatype,
                                    ompi_status_public_t *status);
 int mca_sharedfp_lockedfile_write_ordered (ompio_file_t *fh,
                                            const void *buf,
-                                           int count,
+                                           size_t count,
                                            struct ompi_datatype_t *datatype,
                                            ompi_status_public_t *status);
 int mca_sharedfp_lockedfile_write_ordered_begin (ompio_file_t *fh,
                                                  const void *buf,
-                                                 int count,
+                                                 size_t count,
                                                  struct ompi_datatype_t *datatype);
 int mca_sharedfp_lockedfile_write_ordered_end (ompio_file_t *fh,
                                                const void *buf,
                                                ompi_status_public_t *status);
 int mca_sharedfp_lockedfile_iwrite (ompio_file_t *fh,
                                     const void *buf,
-                                    int count,
+                                    size_t count,
                                     struct ompi_datatype_t *datatype,
                                     ompi_request_t **request);
 
@@ -115,7 +117,7 @@ struct mca_sharedfp_lockedfile_data
     char* filename;
 };
 
-typedef struct mca_sharedfp_lockedfile_data lockedfile_data;
+typedef struct mca_sharedfp_lockedfile_data lockedfile_data_global;
 
 
 int mca_sharedfp_lockedfile_request_position (struct mca_sharedfp_base_data_t * sh,
diff --git a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_component.c b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_component.c
index 3d142c4e25f..044fcb8fb97 100644
--- a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_component.c
+++ b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_component.c
@@ -13,6 +13,8 @@
  * Copyright (c) 2013      University of Houston. All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -47,13 +49,13 @@ static int lockedfile_register(void);
  * Instantiate the public struct with all of our public information
  * and pointers to our public functions in it
  */
-mca_sharedfp_base_component_2_0_0_t mca_sharedfp_lockedfile_component = {
+mca_sharedfp_base_component_3_0_0_t mca_sharedfp_lockedfile_component = {
 
     /* First, the mca_component_t struct containing meta information
        about the component itself */
 
     .sharedfpm_version = {
-        MCA_SHAREDFP_BASE_VERSION_2_0_0,
+        MCA_SHAREDFP_BASE_VERSION_3_0_0,
 
         /* Component name and version */
         .mca_component_name = "lockedfile",
diff --git a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_file_open.c b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_file_open.c
index b82be1f8932..48aae544a41 100644
--- a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_file_open.c
+++ b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_file_open.c
@@ -37,6 +37,11 @@
 #include <fcntl.h>
 #include <unistd.h>
 
+#include "opal/util/output.h"
+#include "opal/util/opal_getcwd.h"
+#include "opal/util/path.h"
+#include "opal/util/os_path.h"
+
 int mca_sharedfp_lockedfile_file_open (struct ompi_communicator_t *comm,
 				       const char* filename,
 				       int amode,
@@ -110,8 +115,26 @@ int mca_sharedfp_lockedfile_file_open (struct ompi_communicator_t *comm,
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
     snprintf(lockedfilename, filenamelen, "%s-%u-%d%s",filename,masterjobid,int_pid,".lock");
-    module_data->filename = lockedfilename;
-    
+    if (opal_path_is_absolute(lockedfilename) ) {
+        module_data->filename = lockedfilename;
+    } else {
+        char path[OPAL_PATH_MAX];
+        err = opal_getcwd(path, OPAL_PATH_MAX);
+        if (OPAL_SUCCESS != err) {
+            free (sh);
+            free (module_data);
+            free (lockedfilename);
+            return err;
+        }
+        module_data->filename = opal_os_path(0, path, lockedfilename, NULL);
+        if (NULL == module_data->filename){
+            free (sh);
+            free (module_data);
+            free (lockedfilename);
+            return OMPI_ERROR;
+        }
+    }
+
     /*-------------------------------------------------*/
     /*Open the lockedfile without shared file pointer  */
     /*-------------------------------------------------*/
@@ -125,12 +148,19 @@ int mca_sharedfp_lockedfile_file_open (struct ompi_communicator_t *comm,
             opal_output(0, "[%d]mca_sharedfp_lockedfile_file_open: Error during file open\n", 
                         fh->f_rank);
             free (sh);
-            free(module_data);
+            free (module_data);
             free (lockedfilename);
             return OMPI_ERROR;
         }
-	write ( handle, &position, sizeof(OMPI_MPI_OFFSET_TYPE) );
-	close ( handle );
+	err = opal_best_effort_write (handle, &position, sizeof(OMPI_MPI_OFFSET_TYPE));
+        if (OPAL_SUCCESS != err)  {
+            free (sh);
+            free (module_data);
+            free (lockedfilename);
+            close (handle);
+            return err;
+        }
+	close (handle);
     }
     err = comm->c_coll->coll_barrier ( comm, comm->c_coll->coll_barrier_module );
     if ( OMPI_SUCCESS != err ) {
@@ -172,7 +202,7 @@ int mca_sharedfp_lockedfile_file_close (ompio_file_t *fh)
     }
     sh = fh->f_sharedfp_data;
 
-    module_data = (lockedfile_data*)(sh->selected_module_data);
+    module_data = (lockedfile_data_global*)(sh->selected_module_data);
     if ( module_data)   {
         /*Close lockedfile handle*/
         if ( module_data->handle)  {
diff --git a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_iread.c b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_iread.c
index 646bed50664..856f2f054b0 100644
--- a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_iread.c
+++ b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_iread.c
@@ -12,6 +12,8 @@
  * Copyright (c) 2013-2018 University of Houston. All rights reserved.
  * Copyright (c) 2018      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -31,7 +33,7 @@
 
 int mca_sharedfp_lockedfile_iread(ompio_file_t *fh,
                                   void *buf,
-                                  int count,
+                                  size_t count,
                                   ompi_datatype_t *datatype,
                                   MPI_Request * request)
 {
@@ -62,7 +64,7 @@ int mca_sharedfp_lockedfile_iread(ompio_file_t *fh,
 
     /*Request the offset to write bytesRequested bytes*/
     ret = mca_sharedfp_lockedfile_request_position(sh,bytesRequested,&offset);
-    offset /= fh->f_etype_size;
+    offset /= fh->f_fview.f_etype_size;
 
     if ( -1 != ret )  {
 	if ( mca_sharedfp_lockedfile_verbose ) {
@@ -79,7 +81,7 @@ int mca_sharedfp_lockedfile_iread(ompio_file_t *fh,
 
 int mca_sharedfp_lockedfile_read_ordered_begin(ompio_file_t *fh,
                                        void *buf,
-                                       int count,
+                                       size_t count,
                                        struct ompi_datatype_t *datatype)
 {
     int ret = OMPI_SUCCESS;
@@ -175,7 +177,7 @@ int mca_sharedfp_lockedfile_read_ordered_begin(ompio_file_t *fh,
 
     /*Each process now has its own individual offset*/
     offset = offsetBuff - sendBuff;
-    offset /= fh->f_etype_size;
+    offset /= fh->f_fview.f_etype_size;
 
     if ( mca_sharedfp_lockedfile_verbose ) {
 	opal_output(ompi_sharedfp_base_framework.framework_output,
diff --git a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_iwrite.c b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_iwrite.c
index 653e32fd579..c4d51d4abeb 100644
--- a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_iwrite.c
+++ b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_iwrite.c
@@ -12,6 +12,8 @@
  * Copyright (c) 2013-2018 University of Houston. All rights reserved.
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -31,7 +33,7 @@
 
 int mca_sharedfp_lockedfile_iwrite(ompio_file_t *fh,
                                    const void *buf,
-                                   int count,
+                                   size_t count,
                                    ompi_datatype_t *datatype,
                                    MPI_Request * request)
 {
@@ -60,7 +62,7 @@ int mca_sharedfp_lockedfile_iwrite(ompio_file_t *fh,
 
     /*Request the offset to write bytesRequested bytes*/
     ret = mca_sharedfp_lockedfile_request_position(sh,bytesRequested,&offset);
-    offset /= fh->f_etype_size;
+    offset /= fh->f_fview.f_etype_size;
 
     if ( -1 != ret) {
 	if ( mca_sharedfp_lockedfile_verbose ) {
@@ -77,7 +79,7 @@ int mca_sharedfp_lockedfile_iwrite(ompio_file_t *fh,
 
 int mca_sharedfp_lockedfile_write_ordered_begin(ompio_file_t *fh,
                                                 const void *buf,
-                                                int count,
+                                                size_t count,
                                                 struct ompi_datatype_t *datatype)
 {
     int ret = OMPI_SUCCESS;
@@ -184,7 +186,7 @@ int mca_sharedfp_lockedfile_write_ordered_begin(ompio_file_t *fh,
 
     /*Each process now has its own individual offset*/
     offset = offsetBuff - sendBuff;
-    offset /= fh->f_etype_size;
+    offset /= fh->f_fview.f_etype_size;
 
     if ( mca_sharedfp_lockedfile_verbose ) {
 	opal_output(ompi_sharedfp_base_framework.framework_output,
diff --git a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_read.c b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_read.c
index 53e62fdeac9..94ca3cb254a 100644
--- a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_read.c
+++ b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_read.c
@@ -12,6 +12,8 @@
  * Copyright (c) 2013-2018 University of Houston. All rights reserved.
  * Copyright (c) 2018      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -29,7 +31,7 @@
 #include "ompi/mca/sharedfp/base/base.h"
 
 int mca_sharedfp_lockedfile_read ( ompio_file_t *fh,
-                                   void *buf, int count, MPI_Datatype datatype, MPI_Status *status)
+                                   void *buf, size_t count, MPI_Datatype datatype, MPI_Status *status)
 {
     int ret = OMPI_SUCCESS;
     OMPI_MPI_OFFSET_TYPE offset = 0;
@@ -59,7 +61,7 @@ int mca_sharedfp_lockedfile_read ( ompio_file_t *fh,
 
     /*Request the offset to write bytesRequested bytes*/
     ret = mca_sharedfp_lockedfile_request_position(sh,bytesRequested,&offset);
-    offset /= fh->f_etype_size;
+    offset /= fh->f_fview.f_etype_size;
 
     if (-1 != ret )  {
 	if ( mca_sharedfp_lockedfile_verbose ) {
@@ -76,7 +78,7 @@ int mca_sharedfp_lockedfile_read ( ompio_file_t *fh,
 
 int mca_sharedfp_lockedfile_read_ordered (ompio_file_t *fh,
                                            void *buf,
-                                           int count,
+                                           size_t count,
                                            struct ompi_datatype_t *datatype,
                                            ompi_status_public_t *status)
 {
@@ -162,7 +164,7 @@ int mca_sharedfp_lockedfile_read_ordered (ompio_file_t *fh,
 
     /*Each process now has its own individual offset in recvBUFF*/
     offset = offsetBuff - sendBuff;
-    offset /= fh->f_etype_size;
+    offset /= fh->f_fview.f_etype_size;
 
     if ( mca_sharedfp_lockedfile_verbose ) {
         opal_output(ompi_sharedfp_base_framework.framework_output,
diff --git a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_request_position.c b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_request_position.c
index 8edfa2bf8c3..88e79c4663a 100644
--- a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_request_position.c
+++ b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_request_position.c
@@ -25,6 +25,9 @@
 #include "ompi/constants.h"
 #include "ompi/mca/sharedfp/sharedfp.h"
 #include "ompi/mca/sharedfp/base/base.h"
+#include "opal/util/output.h"
+#include "opal/util/fd.h"
+
 
 /*Use fcntl to lock the hidden file which stores the current position*/
 #include <fcntl.h>
@@ -61,9 +64,9 @@ int mca_sharedfp_lockedfile_request_position(struct mca_sharedfp_base_data_t * s
     fl.l_pid    = getpid();
 
 
-    /* Aquire an exclusive lock */
+    /* Acquire an exclusive lock */
     if (fcntl(fd, F_SETLKW, &fl) == -1) {
-        opal_output(0,"sharedfp_lockedfile_request_position: errorr acquiring lock: fcntl(%d,F_SETLKW,&fl)\n",fd);
+        opal_output(0,"sharedfp_lockedfile_request_position: error acquiring lock: fcntl(%d,F_SETLKW,&fl)\n",fd);
         opal_output(0,"sharedfp_lockedfile_request_position: error(%i): %s", errno, strerror(errno));
         return OMPI_ERROR;
     }
@@ -76,7 +79,10 @@ int mca_sharedfp_lockedfile_request_position(struct mca_sharedfp_base_data_t * s
 
     /* read from the file */
     lseek ( fd, 0, SEEK_SET );
-    read ( fd, &buf, sizeof(OMPI_MPI_OFFSET_TYPE));
+    ret = opal_fd_read ( fd,  sizeof(OMPI_MPI_OFFSET_TYPE), &buf);
+    if (OPAL_SUCCESS != ret ) {
+        goto exit;
+    }
     if ( mca_sharedfp_lockedfile_verbose ) {
         opal_output(ompi_sharedfp_base_framework.framework_output,
                     "sharedfp_lockedfile_request_position: Read last_offset=%lld! ret=%d\n",buf, ret);
@@ -92,8 +98,11 @@ int mca_sharedfp_lockedfile_request_position(struct mca_sharedfp_base_data_t * s
 
     /* write to the file  */
     lseek ( fd, 0, SEEK_SET );
-    write ( fd, &position, sizeof(OMPI_MPI_OFFSET_TYPE));
-
+    ret = opal_best_effort_write ( fd, &position, sizeof(OMPI_MPI_OFFSET_TYPE));
+    /* No need to handle error case here, the subsequent steps are identical
+       in case of ret != OPAL_SUCCESS, namely release lock and return ret */
+    
+exit:
     /* unlock the file */
     if ( mca_sharedfp_lockedfile_verbose ) {
         opal_output(ompi_sharedfp_base_framework.framework_output,
@@ -115,7 +124,10 @@ int mca_sharedfp_lockedfile_request_position(struct mca_sharedfp_base_data_t * s
     if (fcntl(fd, F_SETLK, &fl) == -1) {
         opal_output(0,"sharedfp_lockedfile_request_position:failed to release lock for fd: %d\n",fd);
         opal_output(0,"error(%i): %s", errno, strerror(errno));
-        return OMPI_ERROR;
+        /* Only overwrite error code if it was OPAL_SUCCESS previously */
+        if (OPAL_SUCCESS == ret ) {
+            ret = OMPI_ERROR;
+        }
     }
     else {
 	if ( mca_sharedfp_lockedfile_verbose ) {
diff --git a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_seek.c b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_seek.c
index 8ff623c764e..e03c19aeea8 100644
--- a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_seek.c
+++ b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_seek.c
@@ -28,6 +28,8 @@
 #include "ompi/mca/sharedfp/sharedfp.h"
 #include "ompi/mca/sharedfp/base/base.h"
 
+#include "opal/util/fd.h"
+
 /*Use fcntl to lock the file which stores the current position*/
 #include <fcntl.h>
 
@@ -50,7 +52,7 @@ mca_sharedfp_lockedfile_seek (ompio_file_t *fh,
     }
 
     sh = fh->f_sharedfp_data;
-    offset = off * fh->f_etype_size;
+    offset = off * fh->f_fview.f_etype_size;
 
     if( 0 == fh->f_rank ){
         if ( MPI_SEEK_SET == whence ){
@@ -94,7 +96,7 @@ mca_sharedfp_lockedfile_seek (ompio_file_t *fh,
         fd_lockedfilehandle = lockedfile_data->handle;
 
 	opal_output(ompi_sharedfp_base_framework.framework_output,
-		    "sharedfp_lockedfile_seek: Aquiring lock...");
+		    "sharedfp_lockedfile_seek: Acquiring lock...");
 
         /* set up the flock structure */
         fl.l_type   = F_WRLCK;
@@ -103,9 +105,9 @@ mca_sharedfp_lockedfile_seek (ompio_file_t *fh,
         fl.l_len    = 0;
         fl.l_pid    = getpid();
 
-        /* Aquire an exclusive lock */
+        /* Acquire an exclusive lock */
         if ( fcntl(fd_lockedfilehandle, F_SETLKW, &fl) == -1) {
-            opal_output(0, "Erorr acquiring lock: fcntl(%d,F_SETLKW,&fl)\n",fd_lockedfilehandle);
+            opal_output(0, "Error acquiring lock: fcntl(%d,F_SETLKW,&fl)\n",fd_lockedfilehandle);
             opal_output(0,"error(%i): %s", errno, strerror(errno));
             return OMPI_ERROR;
         }
@@ -119,7 +121,7 @@ mca_sharedfp_lockedfile_seek (ompio_file_t *fh,
 	 *--------------------
 	 */
 	lseek ( fd_lockedfilehandle, 0, SEEK_SET);
-        write ( fd_lockedfilehandle, &offset, sizeof(OMPI_MPI_OFFSET_TYPE));
+        opal_fd_write ( fd_lockedfilehandle, sizeof(OMPI_MPI_OFFSET_TYPE), &offset);
 
         /*-------------------
 	 * unlock the file
diff --git a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_write.c b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_write.c
index 2f5877137d5..2240d2b4cdd 100644
--- a/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_write.c
+++ b/ompi/mca/sharedfp/lockedfile/sharedfp_lockedfile_write.c
@@ -12,6 +12,8 @@
  * Copyright (c) 2013-2018 University of Houston. All rights reserved.
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -30,7 +32,7 @@
 
 int mca_sharedfp_lockedfile_write (ompio_file_t *fh,
                                    const void *buf,
-                                   int count,
+                                   size_t count,
                                    struct ompi_datatype_t *datatype,
                                    ompi_status_public_t *status)
 {
@@ -59,7 +61,7 @@ int mca_sharedfp_lockedfile_write (ompio_file_t *fh,
 
     /* Request the offset to write bytesRequested bytes */
     ret = mca_sharedfp_lockedfile_request_position ( sh, bytesRequested, &offset);
-    offset /= fh->f_etype_size;
+    offset /= fh->f_fview.f_etype_size;
 
     if (-1 != ret )  {
 	if ( mca_sharedfp_lockedfile_verbose ) {
@@ -75,7 +77,7 @@ int mca_sharedfp_lockedfile_write (ompio_file_t *fh,
 
 int mca_sharedfp_lockedfile_write_ordered (ompio_file_t *fh,
                                            const void *buf,
-                                           int count,
+                                           size_t count,
                                            struct ompi_datatype_t *datatype,
                                            ompi_status_public_t *status)
 {
@@ -177,7 +179,7 @@ int mca_sharedfp_lockedfile_write_ordered (ompio_file_t *fh,
 
     /*Each process now has its own individual offset*/
     offset = offsetBuff - sendBuff;
-    offset /= fh->f_etype_size;
+    offset /= fh->f_fview.f_etype_size;
 
     if ( mca_sharedfp_lockedfile_verbose ) {
         opal_output(ompi_sharedfp_base_framework.framework_output,
diff --git a/ompi/mca/sharedfp/sharedfp.h b/ompi/mca/sharedfp/sharedfp.h
index 8f37d2d53ab..ed0e3a2dc32 100644
--- a/ompi/mca/sharedfp/sharedfp.h
+++ b/ompi/mca/sharedfp/sharedfp.h
@@ -16,6 +16,8 @@
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -41,8 +43,8 @@ struct ompi_file_t;
 /*
  * Macro for use in components that are of type coll
  */
-#define MCA_SHAREDFP_BASE_VERSION_2_0_0 \
-    OMPI_MCA_BASE_VERSION_2_1_0("sharedfp", 2, 0, 0)
+#define MCA_SHAREDFP_BASE_VERSION_3_0_0 \
+    OMPI_MCA_BASE_VERSION_2_1_0("sharedfp", 3, 0, 0)
 
 /*
  * This framework abstracts out operations of the shared filepointer
@@ -78,7 +80,7 @@ typedef int (*mca_sharedfp_base_component_init_query_1_0_0_fn_t)
 (bool enable_progress_threads,
  bool enable_mpi_threads);
 
-typedef struct mca_sharedfp_base_module_1_0_0_t *
+typedef struct mca_sharedfp_base_module_2_0_0_t *
         (*mca_sharedfp_base_component_file_query_1_0_0_fn_t)
         (struct ompio_file_t *file, int *priority);
 
@@ -87,11 +89,11 @@ typedef int (*mca_sharedfp_base_component_file_unquery_1_0_0_fn_t)
 
 /*
  * ****************** component struct ******************************
- * Structure for sharedfp v2.0.0 components.This is chained to MCA v2.0.0
+ * Structure for sharedfp v3.0.0 components.This is chained to MCA v2.0.0
  * ****************** component struct ******************************
  */
 
-struct mca_sharedfp_base_component_2_0_0_t {
+struct mca_sharedfp_base_component_3_0_0_t {
     mca_base_component_t sharedfpm_version;
     mca_base_component_data_t sharedfpm_data;
 
@@ -99,8 +101,8 @@ struct mca_sharedfp_base_component_2_0_0_t {
     mca_sharedfp_base_component_file_query_1_0_0_fn_t sharedfpm_file_query;
     mca_sharedfp_base_component_file_unquery_1_0_0_fn_t sharedfpm_file_unquery;
 };
-typedef struct mca_sharedfp_base_component_2_0_0_t mca_sharedfp_base_component_2_0_0_t;
-typedef struct mca_sharedfp_base_component_2_0_0_t mca_sharedfp_base_component_t;
+typedef struct mca_sharedfp_base_component_3_0_0_t mca_sharedfp_base_component_3_0_0_t;
+typedef struct mca_sharedfp_base_component_3_0_0_t mca_sharedfp_base_component_t;
 
 /*
  * ***********************************************************************
@@ -125,19 +127,19 @@ typedef int (*mca_sharedfp_base_module_get_position_fn_t)(
 typedef int (*mca_sharedfp_base_module_write_fn_t)(
         struct ompio_file_t *fh,
         const void *buf,
-        int count,
+        size_t count,
         struct ompi_datatype_t *datatype,
         ompi_status_public_t *status);
 typedef int (*mca_sharedfp_base_module_write_ordered_fn_t)(
         struct ompio_file_t *fh,
         const void *buf,
-        int count,
+        size_t count,
         struct ompi_datatype_t *datatype,
         ompi_status_public_t *status);
 typedef int (*mca_sharedfp_base_module_write_ordered_begin_fn_t)(
         struct ompio_file_t *fh,
         const void *buf,
-        int count,
+        size_t count,
         struct ompi_datatype_t *datatype);
 typedef int (*mca_sharedfp_base_module_write_ordered_end_fn_t)(
         struct ompio_file_t *fh,
@@ -146,31 +148,31 @@ typedef int (*mca_sharedfp_base_module_write_ordered_end_fn_t)(
 typedef int (*mca_sharedfp_base_module_iwrite_fn_t)(
         struct ompio_file_t *fh,
         const void *buf,
-        int count,
+        size_t count,
         struct ompi_datatype_t *datatype,
         ompi_request_t ** request);
 typedef int (*mca_sharedfp_base_module_read_fn_t)(
         struct ompio_file_t *fh,
         void *buf,
-        int count,
+        size_t count,
         struct ompi_datatype_t *datatype,
         ompi_status_public_t *status);
 typedef int (*mca_sharedfp_base_module_read_ordered_fn_t)(
         struct ompio_file_t *fh,
         void *buf,
-        int count,
+        size_t count,
         struct ompi_datatype_t *datatype,
         ompi_status_public_t *status);
 typedef int (*mca_sharedfp_base_module_iread_fn_t)(
         struct ompio_file_t *fh,
         void *buf,
-        int count,
+        size_t count,
         struct ompi_datatype_t *datatype,
         ompi_request_t ** request);
 typedef int (*mca_sharedfp_base_module_read_ordered_begin_fn_t)(
         struct ompio_file_t *fh,
         void *buf,
-        int count,
+        size_t count,
         struct ompi_datatype_t *datatype);
 typedef int (*mca_sharedfp_base_module_read_ordered_end_fn_t)(
         struct ompio_file_t *fh,
@@ -187,7 +189,7 @@ typedef int (*mca_sharedfp_base_module_file_close_fn_t)(struct ompio_file_t *fh)
  * ***************************  module structure *************************
  * ***********************************************************************
  */
-struct mca_sharedfp_base_module_1_0_0_t {
+struct mca_sharedfp_base_module_2_0_0_t {
     /*
      * Per-file initialization function. This is called only
      * on the module which is selected. The finalize corresponding to
@@ -213,8 +215,8 @@ struct mca_sharedfp_base_module_1_0_0_t {
     mca_sharedfp_base_module_file_open_fn_t   sharedfp_file_open;
     mca_sharedfp_base_module_file_close_fn_t  sharedfp_file_close;
 };
-typedef struct mca_sharedfp_base_module_1_0_0_t mca_sharedfp_base_module_1_0_0_t;
-typedef mca_sharedfp_base_module_1_0_0_t mca_sharedfp_base_module_t;
+typedef struct mca_sharedfp_base_module_2_0_0_t mca_sharedfp_base_module_2_0_0_t;
+typedef mca_sharedfp_base_module_2_0_0_t mca_sharedfp_base_module_t;
 
 
 /* This structure keeps all of the data needed by a sharedfp module.
diff --git a/ompi/mca/sharedfp/sm/sharedfp_sm.c b/ompi/mca/sharedfp/sm/sharedfp_sm.c
index 498c02d716a..5882ec8e30c 100644
--- a/ompi/mca/sharedfp/sm/sharedfp_sm.c
+++ b/ompi/mca/sharedfp/sm/sharedfp_sm.c
@@ -9,9 +9,12 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2008-2013  University of Houston. All rights reserved.
+ * Copyright (c) 2008-2021 University of Houston. All rights reserved.
  * Copyright (c) 2018      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2021      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -26,19 +29,27 @@
  */
 
 #include "ompi_config.h"
+
+#if HAVE_SYS_STAT_H
+#    include <sys/stat.h>
+#endif /* HAVE_SYS_STAT_H */
+
 #include "mpi.h"
+#include "opal/util/printf.h"
 #include "ompi/mca/sharedfp/sharedfp.h"
 #include "ompi/mca/sharedfp/base/base.h"
 #include "ompi/mca/sharedfp/sm/sharedfp_sm.h"
 
+#include "opal/util/basename.h"
+
 /*
  * *******************************************************************
  * ************************ actions structure ************************
  * *******************************************************************
  */
  /* IMPORTANT: Update here when adding sharedfp component interface functions*/
-static mca_sharedfp_base_module_1_0_0_t sm =  {
-    mca_sharedfp_sm_module_init, /* initalise after being selected */
+static mca_sharedfp_base_module_2_0_0_t sm =  {
+    mca_sharedfp_sm_module_init, /* initialise after being selected */
     mca_sharedfp_sm_module_finalize, /* close a module on a communicator */
     mca_sharedfp_sm_seek,
     mca_sharedfp_sm_get_position,
@@ -69,7 +80,7 @@ int mca_sharedfp_sm_component_init_query(bool enable_progress_threads,
    return OMPI_SUCCESS;
 }
 
-struct mca_sharedfp_base_module_1_0_0_t * mca_sharedfp_sm_component_file_query(ompio_file_t *fh, int *priority)
+struct mca_sharedfp_base_module_2_0_0_t * mca_sharedfp_sm_component_file_query(ompio_file_t *fh, int *priority)
 {
     int i;
     ompi_proc_t *proc;
@@ -87,13 +98,39 @@ struct mca_sharedfp_base_module_1_0_0_t * mca_sharedfp_sm_component_file_query(o
     for (i = 0; i < size; ++i) {
         proc = ompi_group_peer_lookup(group,i);
         if (!OPAL_PROC_ON_LOCAL_NODE(proc->super.proc_flags)){
-            opal_output(ompi_sharedfp_base_framework.framework_output,
-                        "mca_sharedfp_sm_component_file_query: Disqualifying myself: (%d/%s) "
+            opal_output_verbose(10, ompi_sharedfp_base_framework.framework_output,
+                        "mca_sharedfp_sm_component_file_query: Disqualifying myself: (%s/%s) "
                         "not all processes are on the same node.",
-                        comm->c_contextid, comm->c_name);
+                        ompi_comm_print_cid (comm), comm->c_name);
             return NULL;
         }
     }
+
+
+    /* Check that we can actually open the required file */
+    char *filename_basename = opal_basename((char*)fh->f_filename);
+    char *sm_filename;
+    int comm_cid = -1;
+    int pid = ompi_comm_rank (comm);
+    
+    opal_asprintf(&sm_filename, "%s/%s_cid-%d-%d.sm", ompi_process_info.job_session_dir,
+             filename_basename, comm_cid, pid);
+    free(filename_basename);
+
+    int sm_fd = open(sm_filename, O_RDWR | O_CREAT,
+                     S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+    if ( sm_fd == -1){
+        /*error opening file*/
+        opal_output_verbose(10, ompi_sharedfp_base_framework.framework_output,
+		    "mca_sharedfp_sm_component_file_query: Error, unable to open file "
+		    "for mmap: %s\n",sm_filename);
+        free(sm_filename);
+        return NULL;
+    }
+    close (sm_fd);
+    unlink(sm_filename);
+    free (sm_filename);
+    
     /* This module can run */
     *priority = mca_sharedfp_sm_priority;
     return &sm;
diff --git a/ompi/mca/sharedfp/sm/sharedfp_sm.h b/ompi/mca/sharedfp/sm/sharedfp_sm.h
index 69b463803b1..f6411232858 100644
--- a/ompi/mca/sharedfp/sm/sharedfp_sm.h
+++ b/ompi/mca/sharedfp/sm/sharedfp_sm.h
@@ -14,6 +14,8 @@
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -34,7 +36,7 @@ BEGIN_C_DECLS
 
 int mca_sharedfp_sm_component_init_query(bool enable_progress_threads,
                                                  bool enable_mpi_threads);
-struct mca_sharedfp_base_module_1_0_0_t *
+struct mca_sharedfp_base_module_2_0_0_t *
         mca_sharedfp_sm_component_file_query (ompio_file_t *file, int *priority);
 int mca_sharedfp_sm_component_file_unquery (ompio_file_t *file);
 
@@ -44,7 +46,7 @@ int mca_sharedfp_sm_module_finalize (ompio_file_t *file);
 extern int mca_sharedfp_sm_priority;
 extern int mca_sharedfp_sm_verbose;
 
-OMPI_MODULE_DECLSPEC extern mca_sharedfp_base_component_2_0_0_t mca_sharedfp_sm_component;
+OMPI_DECLSPEC extern mca_sharedfp_base_component_3_0_0_t mca_sharedfp_sm_component;
 /*
  * ******************************************************************
  * ********* functions which are implemented in this module *********
@@ -62,43 +64,43 @@ int mca_sharedfp_sm_file_open (struct ompi_communicator_t *comm,
                                        ompio_file_t *fh);
 int mca_sharedfp_sm_file_close (ompio_file_t *fh);
 int mca_sharedfp_sm_read (ompio_file_t *fh,
-                                  void *buf, int count, MPI_Datatype datatype, MPI_Status *status);
+                                  void *buf, size_t count, MPI_Datatype datatype, MPI_Status *status);
 int mca_sharedfp_sm_read_ordered (ompio_file_t *fh,
-                                          void *buf, int count, struct ompi_datatype_t *datatype,
+                                          void *buf, size_t count, struct ompi_datatype_t *datatype,
                                           ompi_status_public_t *status
                                           );
 int mca_sharedfp_sm_read_ordered_begin (ompio_file_t *fh,
                                                  void *buf,
-                                                 int count,
+                                                 size_t count,
                                                  struct ompi_datatype_t *datatype);
 int mca_sharedfp_sm_read_ordered_end (ompio_file_t *fh,
                                                void *buf,
                                                ompi_status_public_t *status);
 int mca_sharedfp_sm_iread (ompio_file_t *fh,
                                     void *buf,
-                                    int count,
+                                    size_t count,
                                     struct ompi_datatype_t *datatype,
                                     ompi_request_t **request);
 int mca_sharedfp_sm_write (ompio_file_t *fh,
                                    const void *buf,
-                                   int count,
+                                   size_t count,
                                    struct ompi_datatype_t *datatype,
                                    ompi_status_public_t *status);
 int mca_sharedfp_sm_write_ordered (ompio_file_t *fh,
                                            const void *buf,
-                                           int count,
+                                           size_t count,
                                            struct ompi_datatype_t *datatype,
                                            ompi_status_public_t *status);
 int mca_sharedfp_sm_write_ordered_begin (ompio_file_t *fh,
                                                  const void *buf,
-                                                 int count,
+                                                 size_t count,
                                                  struct ompi_datatype_t *datatype);
 int mca_sharedfp_sm_write_ordered_end (ompio_file_t *fh,
                                                const void *buf,
                                                ompi_status_public_t *status);
 int mca_sharedfp_sm_iwrite (ompio_file_t *fh,
                                     const void *buf,
-                                    int count,
+                                    size_t count,
                                     struct ompi_datatype_t *datatype,
                                     ompi_request_t **request);
 /*--------------------------------------------------------------*
@@ -124,11 +126,11 @@ struct mca_sharedfp_sm_data
     char *sem_name;    /* Name of the semaphore */
 };
 
-typedef struct mca_sharedfp_sm_data sm_data;
+typedef struct mca_sharedfp_sm_data sm_data_global;
 
 
 int mca_sharedfp_sm_request_position (ompio_file_t *fh,
-                                      int bytes_requested,
+                                      long long bytes_requested,
                                       OMPI_MPI_OFFSET_TYPE * offset);
 /*
  * ******************************************************************
diff --git a/ompi/mca/sharedfp/sm/sharedfp_sm_component.c b/ompi/mca/sharedfp/sm/sharedfp_sm_component.c
index 9abb8e387f2..8198c99c042 100644
--- a/ompi/mca/sharedfp/sm/sharedfp_sm_component.c
+++ b/ompi/mca/sharedfp/sm/sharedfp_sm_component.c
@@ -13,6 +13,8 @@
  * Copyright (c) 2013-2015 University of Houston. All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -47,13 +49,13 @@ static int sm_register(void);
  * Instantiate the public struct with all of our public information
  * and pointers to our public functions in it
  */
-mca_sharedfp_base_component_2_0_0_t mca_sharedfp_sm_component = {
+mca_sharedfp_base_component_3_0_0_t mca_sharedfp_sm_component = {
 
     /* First, the mca_component_t struct containing meta information
        about the component itself */
 
     .sharedfpm_version = {
-        MCA_SHAREDFP_BASE_VERSION_2_0_0,
+        MCA_SHAREDFP_BASE_VERSION_3_0_0,
 
         /* Component name and version */
         .mca_component_name = "sm",
diff --git a/ompi/mca/sharedfp/sm/sharedfp_sm_file_open.c b/ompi/mca/sharedfp/sm/sharedfp_sm_file_open.c
index 6526ee52480..edc453a7add 100644
--- a/ompi/mca/sharedfp/sm/sharedfp_sm_file_open.c
+++ b/ompi/mca/sharedfp/sm/sharedfp_sm_file_open.c
@@ -9,11 +9,11 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2013-2018 University of Houston. All rights reserved.
+ * Copyright (c) 2013-2021 University of Houston. All rights reserved.
  * Copyright (c) 2013      Intel, Inc. All rights reserved.
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2015      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2021 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
  * $COPYRIGHT$
  *
@@ -35,12 +35,16 @@
 #include "sharedfp_sm.h"
 
 #include "mpi.h"
+#include "opal/util/printf.h"
+#include "opal/util/output.h"
 #include "ompi/constants.h"
 #include "ompi/group/group.h"
 #include "ompi/proc/proc.h"
 #include "ompi/mca/sharedfp/sharedfp.h"
 #include "ompi/mca/sharedfp/base/base.h"
 
+#include "opal/util/basename.h"
+
 #include <semaphore.h>
 #include <sys/mman.h>
 #include <libgen.h>
@@ -57,11 +61,9 @@ int mca_sharedfp_sm_file_open (struct ompi_communicator_t *comm,
     struct mca_sharedfp_sm_data * sm_data = NULL;
     char * filename_basename;
     char * sm_filename;
-    int sm_filename_length;
     struct mca_sharedfp_sm_offset * sm_offset_ptr;
     struct mca_sharedfp_sm_offset sm_offset;
     int sm_fd;
-    uint32_t comm_cid;
     int int_pid;
     pid_t my_pid;
 
@@ -101,18 +103,9 @@ int mca_sharedfp_sm_file_open (struct ompi_communicator_t *comm,
     ** and then mapping it to memory
     ** For sharedfp we also want to put the file backed shared memory into the tmp directory
     */
-    filename_basename = basename((char*)filename);
-    /* format is "%s/%s_cid-%d-%d.sm", see below */
-    sm_filename_length = strlen(ompi_process_info.job_session_dir) + 1 + strlen(filename_basename) + 5 + (3*sizeof(uint32_t)+1) + 4;
-    sm_filename = (char*) malloc( sizeof(char) * sm_filename_length);
-    if (NULL == sm_filename) {
-        opal_output(0, "mca_sharedfp_sm_file_open: Error, unable to malloc sm_filename\n");
-        free(sm_data);
-        free(sh);
-        return OMPI_ERR_OUT_OF_RESOURCE;
-    }
+    filename_basename = opal_basename((char*)filename);
+    /* format is "%s/%s_cid-%s-%d.sm", see below */
 
-    comm_cid = ompi_comm_get_cid(comm);
     if ( 0 == fh->f_rank ) {
         my_pid = getpid();
         int_pid = (int) my_pid;
@@ -120,20 +113,21 @@ int mca_sharedfp_sm_file_open (struct ompi_communicator_t *comm,
     err = comm->c_coll->coll_bcast (&int_pid, 1, MPI_INT, 0, comm, comm->c_coll->coll_bcast_module );
     if ( OMPI_SUCCESS != err ) {
         opal_output(0,"mca_sharedfp_sm_file_open: Error in bcast operation \n");
-        free(sm_filename);
+        free(filename_basename);
         free(sm_data);
         free(sh);
         return err;
     }
 
-    snprintf(sm_filename, sm_filename_length, "%s/%s_cid-%d-%d.sm", ompi_process_info.job_session_dir,
-             filename_basename, comm_cid, int_pid);
+    opal_asprintf(&sm_filename, "%s/%s_cid-%s-%d.sm", ompi_process_info.job_session_dir,
+                  filename_basename, ompi_comm_print_cid(comm), int_pid);
     /* open shared memory file, initialize to 0, map into memory */
     sm_fd = open(sm_filename, O_RDWR | O_CREAT,
                  S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
     if ( sm_fd == -1){
         /*error opening file*/
         opal_output(0,"mca_sharedfp_sm_file_open: Error, unable to open file for mmap: %s\n",sm_filename);
+        free(filename_basename);
         free(sm_filename);
         free(sm_data);
         free(sh);
@@ -145,11 +139,20 @@ int mca_sharedfp_sm_file_open (struct ompi_communicator_t *comm,
     /* TODO: is it necessary to write to the file first? */
     if( 0 == fh->f_rank ){
         memset ( &sm_offset, 0, sizeof (struct mca_sharedfp_sm_offset ));
-        write ( sm_fd, &sm_offset, sizeof(struct mca_sharedfp_sm_offset));
+        err = opal_best_effort_write ( sm_fd, &sm_offset, sizeof(struct mca_sharedfp_sm_offset));
+        if (OPAL_SUCCESS != err) {
+            free(filename_basename);
+            free(sm_filename);
+            free(sm_data);
+            free(sh);
+            close (sm_fd);
+            return err;
+        }
     }
     err = comm->c_coll->coll_barrier (comm, comm->c_coll->coll_barrier_module );
     if ( OMPI_SUCCESS != err ) {
         opal_output(0,"mca_sharedfp_sm_file_open: Error in barrier operation \n");
+        free(filename_basename);
         free(sm_filename);
         free(sm_data);
         free(sh);
@@ -167,6 +170,7 @@ int mca_sharedfp_sm_file_open (struct ompi_communicator_t *comm,
         err = OMPI_ERROR;
         opal_output(0, "mca_sharedfp_sm_file_open: Error, unable to mmap file: %s\n",sm_filename);
         opal_output(0, "%s\n", strerror(errno));
+        free(filename_basename);
         free(sm_filename);
         free(sm_data);
         free(sh);
@@ -185,6 +189,10 @@ int mca_sharedfp_sm_file_open (struct ompi_communicator_t *comm,
     sm_data->sem_name = (char*) malloc( sizeof(char) * 253);
     snprintf(sm_data->sem_name,252,"OMPIO_%s",filename_basename);
 #endif
+    // We're now done with filename_basename.  Free it here so that we
+    // don't have to keep freeing it in the error/return cases.
+    free(filename_basename);
+    filename_basename = NULL;
 
     if( (sm_data->mutex = sem_open(sm_data->sem_name, O_CREAT, 0644, 1)) != SEM_FAILED ) {
 #elif defined(HAVE_SEM_INIT)
@@ -253,7 +261,7 @@ int mca_sharedfp_sm_file_close (ompio_file_t *fh)
      */
     fh->f_comm->c_coll->coll_barrier (fh->f_comm, fh->f_comm->c_coll->coll_barrier_module );
 
-    file_data = (sm_data*)(sh->selected_module_data);
+    file_data = (sm_data_global*)(sh->selected_module_data);
     if (file_data)  {
         /*Close sm handle*/
         if (file_data->sm_offset_ptr) {
diff --git a/ompi/mca/sharedfp/sm/sharedfp_sm_iread.c b/ompi/mca/sharedfp/sm/sharedfp_sm_iread.c
index 4f97f4f868b..5c699c34028 100644
--- a/ompi/mca/sharedfp/sm/sharedfp_sm_iread.c
+++ b/ompi/mca/sharedfp/sm/sharedfp_sm_iread.c
@@ -12,6 +12,8 @@
  * Copyright (c) 2013-2018 University of Houston. All rights reserved.
  * Copyright (c) 2018      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -30,13 +32,13 @@
 
 int mca_sharedfp_sm_iread(ompio_file_t *fh,
                           void *buf,
-                          int count,
+                          size_t count,
                           ompi_datatype_t *datatype,
                           MPI_Request * request)
 {
     int ret = OMPI_SUCCESS;
     OMPI_MPI_OFFSET_TYPE offset = 0;
-    long bytesRequested = 0;
+    long long bytesRequested = 0;
     size_t numofBytes;
 
     if( NULL == fh->f_sharedfp_data){
@@ -51,11 +53,11 @@ int mca_sharedfp_sm_iread(ompio_file_t *fh,
 
     if ( mca_sharedfp_sm_verbose ) {
         opal_output(ompi_sharedfp_base_framework.framework_output,
-                    "sharedfp_sm_iread: Bytes Requested is %ld\n",bytesRequested);
+                    "sharedfp_sm_iread: Bytes Requested is %lld\n",bytesRequested);
     }
     /*Request the offset to write bytesRequested bytes*/
     ret = mca_sharedfp_sm_request_position(fh,bytesRequested,&offset);
-    offset /= fh->f_etype_size;
+    offset /= fh->f_fview.f_etype_size;
 
     if (  -1 != ret ) {
         if ( mca_sharedfp_sm_verbose ) {
@@ -71,7 +73,7 @@ int mca_sharedfp_sm_iread(ompio_file_t *fh,
 
 int mca_sharedfp_sm_read_ordered_begin(ompio_file_t *fh,
                                        void *buf,
-                                       int count,
+                                       size_t count,
                                        struct ompi_datatype_t *datatype)
 {
     int ret = OMPI_SUCCESS;
@@ -80,7 +82,7 @@ int mca_sharedfp_sm_read_ordered_begin(ompio_file_t *fh,
     long *buff=NULL;
     long offsetBuff;
     OMPI_MPI_OFFSET_TYPE offsetReceived = 0;
-    long bytesRequested = 0;
+    long long bytesRequested = 0;
     int recvcnt = 1, sendcnt = 1;
     size_t numofBytes;
     int i;
@@ -129,7 +131,7 @@ int mca_sharedfp_sm_read_ordered_begin(ompio_file_t *fh,
 	    bytesRequested += buff[i];
 	    if ( mca_sharedfp_sm_verbose ) {
 		opal_output(ompi_sharedfp_base_framework.framework_output,
-			    "mca_sharedfp_sm_read_ordered_begin: Bytes requested are %ld\n",
+			    "mca_sharedfp_sm_read_ordered_begin: Bytes requested are %lld\n",
 			    bytesRequested);
 	    }
         }
@@ -171,7 +173,7 @@ int mca_sharedfp_sm_read_ordered_begin(ompio_file_t *fh,
 
     /*Each process now has its own individual offset in recvBUFF*/
     offset = offsetBuff - sendBuff;
-    offset /= fh->f_etype_size;
+    offset /= fh->f_fview.f_etype_size;
 
     if ( mca_sharedfp_sm_verbose ) {
 	opal_output(ompi_sharedfp_base_framework.framework_output,
diff --git a/ompi/mca/sharedfp/sm/sharedfp_sm_iwrite.c b/ompi/mca/sharedfp/sm/sharedfp_sm_iwrite.c
index 4ce43089110..84e8a8ee5b5 100644
--- a/ompi/mca/sharedfp/sm/sharedfp_sm_iwrite.c
+++ b/ompi/mca/sharedfp/sm/sharedfp_sm_iwrite.c
@@ -12,6 +12,8 @@
  * Copyright (c) 2013-2018 University of Houston. All rights reserved.
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -30,13 +32,13 @@
 
 int mca_sharedfp_sm_iwrite(ompio_file_t *fh,
                            const void *buf,
-                           int count,
+                           size_t count,
                            ompi_datatype_t *datatype,
                            MPI_Request * request)
 {
      int ret = OMPI_SUCCESS;
      OMPI_MPI_OFFSET_TYPE offset = 0;
-     long bytesRequested = 0;
+     long long bytesRequested = 0;
      size_t numofBytes;
 
      if( NULL == fh->f_sharedfp_data){
@@ -51,11 +53,11 @@ int mca_sharedfp_sm_iwrite(ompio_file_t *fh,
 
      if ( mca_sharedfp_sm_verbose ) {
          opal_output(ompi_sharedfp_base_framework.framework_output,
-		     "sharedfp_sm_iwrite: Bytes Requested is %ld\n",bytesRequested);
+		     "sharedfp_sm_iwrite: Bytes Requested is %lld\n",bytesRequested);
      }
     /* Request the offset to write bytesRequested bytes */
      ret = mca_sharedfp_sm_request_position(fh,bytesRequested,&offset);
-     offset /= fh->f_etype_size;
+     offset /= fh->f_fview.f_etype_size;
 
      if ( -1 != ret ) {
         if ( mca_sharedfp_sm_verbose ) {
@@ -72,7 +74,7 @@ int mca_sharedfp_sm_iwrite(ompio_file_t *fh,
 
 int mca_sharedfp_sm_write_ordered_begin(ompio_file_t *fh,
                                         const void *buf,
-                                        int count,
+                                        size_t count,
                                         struct ompi_datatype_t *datatype)
 {
     int ret = OMPI_SUCCESS;
@@ -81,7 +83,7 @@ int mca_sharedfp_sm_write_ordered_begin(ompio_file_t *fh,
     long *buff=NULL;
     long offsetBuff;
     OMPI_MPI_OFFSET_TYPE offsetReceived = 0;
-    long bytesRequested = 0;
+    long long bytesRequested = 0;
     int recvcnt = 1, sendcnt = 1;
     size_t numofBytes;
     int i;
@@ -123,7 +125,7 @@ int mca_sharedfp_sm_write_ordered_begin(ompio_file_t *fh,
 	    bytesRequested += buff[i];
 	    if ( mca_sharedfp_sm_verbose ) {
 		opal_output(ompi_sharedfp_base_framework.framework_output,
-			    "mca_sharedfp_sm_write_ordered_begin: Bytes requested are %ld\n",
+			    "mca_sharedfp_sm_write_ordered_begin: Bytes requested are %lld\n",
 			    bytesRequested);
 	    }
         }
@@ -159,7 +161,7 @@ int mca_sharedfp_sm_write_ordered_begin(ompio_file_t *fh,
 
     /*Each process now has its own individual offset in recvBUFF*/
     offset = offsetBuff - sendBuff;
-    offset /= fh->f_etype_size;
+    offset /= fh->f_fview.f_etype_size;
 
     if ( mca_sharedfp_sm_verbose ) {
 	opal_output(ompi_sharedfp_base_framework.framework_output,
diff --git a/ompi/mca/sharedfp/sm/sharedfp_sm_read.c b/ompi/mca/sharedfp/sm/sharedfp_sm_read.c
index 846926b69f2..77157a5509f 100644
--- a/ompi/mca/sharedfp/sm/sharedfp_sm_read.c
+++ b/ompi/mca/sharedfp/sm/sharedfp_sm_read.c
@@ -12,6 +12,8 @@
  * Copyright (c) 2013-2018 University of Houston. All rights reserved.
  * Copyright (c) 2018      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -29,11 +31,11 @@
 #include "ompi/mca/sharedfp/base/base.h"
 
 int mca_sharedfp_sm_read ( ompio_file_t *fh,
-                           void *buf, int count, MPI_Datatype datatype, MPI_Status *status)
+                           void *buf, size_t count, MPI_Datatype datatype, MPI_Status *status)
 {
     int ret = OMPI_SUCCESS;
     OMPI_MPI_OFFSET_TYPE offset = 0;
-    long bytesRequested = 0;
+    long long bytesRequested = 0;
     size_t numofBytes;
 
     if( NULL == fh->f_sharedfp_data){
@@ -48,12 +50,12 @@ int mca_sharedfp_sm_read ( ompio_file_t *fh,
 
     if ( mca_sharedfp_sm_verbose ) {
         opal_output(ompi_sharedfp_base_framework.framework_output,
-                    "sharedfp_sm_read: Bytes Requested is %ld\n",bytesRequested);
+                    "sharedfp_sm_read: Bytes Requested is %lld\n",bytesRequested);
     }
 
     /*Request the offset to write bytesRequested bytes*/
     ret = mca_sharedfp_sm_request_position(fh,bytesRequested,&offset);
-    offset /= fh->f_etype_size;
+    offset /= fh->f_fview.f_etype_size;
 
     if (  -1 != ret ) {
         if ( mca_sharedfp_sm_verbose ) {
@@ -70,7 +72,7 @@ int mca_sharedfp_sm_read ( ompio_file_t *fh,
 
 int mca_sharedfp_sm_read_ordered (ompio_file_t *fh,
                                   void *buf,
-                                  int count,
+                                  size_t count,
                                   struct ompi_datatype_t *datatype,
                                   ompi_status_public_t *status)
 {
@@ -80,7 +82,7 @@ int mca_sharedfp_sm_read_ordered (ompio_file_t *fh,
     long *buff=NULL;
     long offsetBuff;
     OMPI_MPI_OFFSET_TYPE offsetReceived = 0;
-    long bytesRequested = 0;
+    long long bytesRequested = 0;
     int recvcnt = 1, sendcnt = 1;
     size_t numofBytes;
     int i;
@@ -122,7 +124,7 @@ int mca_sharedfp_sm_read_ordered (ompio_file_t *fh,
             bytesRequested += buff[i];
             if ( mca_sharedfp_sm_verbose ) {
                 opal_output(ompi_sharedfp_base_framework.framework_output,
-                            "mca_sharedfp_sm_read_ordered: Bytes requested are %ld\n",bytesRequested);
+                            "mca_sharedfp_sm_read_ordered: Bytes requested are %lld\n",bytesRequested);
             }
         }
 
@@ -163,7 +165,7 @@ int mca_sharedfp_sm_read_ordered (ompio_file_t *fh,
 
     /*Each process now has its own individual offset in recvBUFF*/
     offset = offsetBuff - sendBuff;
-    offset /= fh->f_etype_size;
+    offset /= fh->f_fview.f_etype_size;
 
     if ( mca_sharedfp_sm_verbose ) {
         opal_output(ompi_sharedfp_base_framework.framework_output,
diff --git a/ompi/mca/sharedfp/sm/sharedfp_sm_request_position.c b/ompi/mca/sharedfp/sm/sharedfp_sm_request_position.c
index a4e9edade55..53e20a7678e 100644
--- a/ompi/mca/sharedfp/sm/sharedfp_sm_request_position.c
+++ b/ompi/mca/sharedfp/sm/sharedfp_sm_request_position.c
@@ -11,6 +11,8 @@
  *                         All rights reserved.
  * Copyright (c) 2013-2015 University of Houston. All rights reserved.
  * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -31,7 +33,7 @@
 #include <semaphore.h>
 
 int mca_sharedfp_sm_request_position(ompio_file_t *fh, 
-                                     int bytes_requested,
+                                     long long bytes_requested,
                                      OMPI_MPI_OFFSET_TYPE *offset)
 {
     int ret = OMPI_SUCCESS;
@@ -47,12 +49,12 @@ int mca_sharedfp_sm_request_position(ompio_file_t *fh,
     *offset = 0;
     if ( mca_sharedfp_sm_verbose ) {
         opal_output(ompi_sharedfp_base_framework.framework_output,
-                    "Aquiring lock, rank=%d...",fh->f_rank);
+                    "Acquiring lock, rank=%d...",fh->f_rank);
     }
 
     sm_offset_ptr = sm_data->sm_offset_ptr;
 
-    /* Aquire an exclusive lock */
+    /* Acquire an exclusive lock */
 
     sem_wait(sm_data->mutex);
 
@@ -70,7 +72,7 @@ int mca_sharedfp_sm_request_position(ompio_file_t *fh,
     position = old_offset + bytes_requested;
     if ( mca_sharedfp_sm_verbose ) {
         opal_output(ompi_sharedfp_base_framework.framework_output,
-                    "old_offset=%lld, bytes_requested=%d, new offset=%lld!\n",old_offset,bytes_requested,position);
+                    "old_offset=%lld, bytes_requested=%lld, new offset=%lld!\n",old_offset,bytes_requested,position);
     }
     sm_offset_ptr->offset=position;
 
diff --git a/ompi/mca/sharedfp/sm/sharedfp_sm_seek.c b/ompi/mca/sharedfp/sm/sharedfp_sm_seek.c
index 17aa72c326b..511b8eda0d0 100644
--- a/ompi/mca/sharedfp/sm/sharedfp_sm_seek.c
+++ b/ompi/mca/sharedfp/sm/sharedfp_sm_seek.c
@@ -50,7 +50,7 @@ mca_sharedfp_sm_seek (ompio_file_t *fh,
     }
 
     sh = fh->f_sharedfp_data;
-    offset = off * fh->f_etype_size;
+    offset = off * fh->f_fview.f_etype_size;
 
     if( 0 == fh->f_rank ){
         if ( MPI_SEEK_SET == whence){
@@ -112,10 +112,10 @@ mca_sharedfp_sm_seek (ompio_file_t *fh,
         /*--------------------*/
         if ( mca_sharedfp_sm_verbose ) {
             opal_output(ompi_sharedfp_base_framework.framework_output,
-                        "sharedfp_sm_seek: Aquiring lock, rank=%d...",fh->f_rank); fflush(stdout);
+                        "sharedfp_sm_seek: Acquiring lock, rank=%d...",fh->f_rank); fflush(stdout);
         }
 
-        /* Aquire an exclusive lock */
+        /* Acquire an exclusive lock */
         sm_offset_ptr = sm_data->sm_offset_ptr;
 
         sem_wait(sm_data->mutex);
diff --git a/ompi/mca/sharedfp/sm/sharedfp_sm_write.c b/ompi/mca/sharedfp/sm/sharedfp_sm_write.c
index 86d4c7c6d59..0a916901ce2 100644
--- a/ompi/mca/sharedfp/sm/sharedfp_sm_write.c
+++ b/ompi/mca/sharedfp/sm/sharedfp_sm_write.c
@@ -12,6 +12,8 @@
  * Copyright (c) 2013-2018 University of Houston. All rights reserved.
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -30,13 +32,13 @@
 
 int mca_sharedfp_sm_write (ompio_file_t *fh,
                            const void *buf,
-                           int count,
+                           size_t count,
                            struct ompi_datatype_t *datatype,
                            ompi_status_public_t *status)
 {
     int ret = OMPI_SUCCESS;
     OMPI_MPI_OFFSET_TYPE offset = 0;
-    long bytesRequested = 0;
+    long long bytesRequested = 0;
     size_t numofBytes;
 
     if( NULL == fh->f_sharedfp_data ){
@@ -53,12 +55,12 @@ int mca_sharedfp_sm_write (ompio_file_t *fh,
 
     if ( mca_sharedfp_sm_verbose ) {
         opal_output(ompi_sharedfp_base_framework.framework_output,
-                    "sharedfp_sm_write: Requested is %ld\n",bytesRequested);
+                    "sharedfp_sm_write: Requested is %lld\n",bytesRequested);
     }
 
     /*Request the offset to write bytesRequested bytes*/
     ret = mca_sharedfp_sm_request_position(fh, bytesRequested,&offset);
-    offset /= fh->f_etype_size;
+    offset /= fh->f_fview.f_etype_size;
     if ( -1 != ret ) {
         if ( mca_sharedfp_sm_verbose ) {
             opal_output(ompi_sharedfp_base_framework.framework_output,
@@ -74,7 +76,7 @@ int mca_sharedfp_sm_write (ompio_file_t *fh,
 
 int mca_sharedfp_sm_write_ordered (ompio_file_t *fh,
                                    const void *buf,
-                                   int count,
+                                   size_t count,
                                    struct ompi_datatype_t *datatype,
                                    ompi_status_public_t *status)
 {
@@ -84,14 +86,14 @@ int mca_sharedfp_sm_write_ordered (ompio_file_t *fh,
     long *buff=NULL;
     long offsetBuff;
     OMPI_MPI_OFFSET_TYPE offsetReceived = 0;
-    long bytesRequested = 0;
+    long long bytesRequested = 0;
     int recvcnt = 1, sendcnt = 1;
     size_t numofBytes;
     int i;
 
     if( NULL == fh->f_sharedfp_data){
         opal_output(ompi_sharedfp_base_framework.framework_output,
-                    "sharedfp_sm_write_ordered: module not initialzed \n");
+                    "sharedfp_sm_write_ordered: module not initialized \n");
         return OMPI_ERROR;
     }
 
@@ -120,7 +122,7 @@ int mca_sharedfp_sm_write_ordered (ompio_file_t *fh,
             bytesRequested += buff[i];
             if ( mca_sharedfp_sm_verbose ) {
                 opal_output(ompi_sharedfp_base_framework.framework_output,
-                            "sharedfp_sm_write_ordered: Bytes requested are %ld\n",bytesRequested);
+                            "sharedfp_sm_write_ordered: Bytes requested are %lld\n",bytesRequested);
             }
         }
 
@@ -156,7 +158,7 @@ int mca_sharedfp_sm_write_ordered (ompio_file_t *fh,
 
     /* Each process now has its own individual offset */
     offset = offsetBuff - sendBuff;
-    offset /= fh->f_etype_size;
+    offset /= fh->f_fview.f_etype_size;
 
     if ( mca_sharedfp_sm_verbose ) {
         opal_output(ompi_sharedfp_base_framework.framework_output,
diff --git a/ompi/mca/topo/base/base.h b/ompi/mca/topo/base/base.h
index 9ab1a4b927a..f04a1cbff9f 100644
--- a/ompi/mca/topo/base/base.h
+++ b/ompi/mca/topo/base/base.h
@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -16,6 +17,8 @@
  * Copyright (c) 2014-2015 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
+ * Copyright (c) 2018      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -56,6 +59,13 @@ mca_topo_base_comm_select(const ompi_communicator_t*  comm,
                           mca_topo_base_module_t**    selected_module,
                           uint32_t                    type);
 
+/* Select a topo module for a particular type of topology */
+OMPI_DECLSPEC int
+mca_topo_base_group_select (const ompi_group_t *group,
+                            mca_topo_base_module_t *preferred_module,
+                            mca_topo_base_module_t **selected_module,
+                            uint32_t type);
+
 /* Find all components that want to be considered in this job */
 OMPI_DECLSPEC int
 mca_topo_base_find_available(bool enable_progress_threads,
diff --git a/ompi/mca/topo/base/topo_base_cart_create.c b/ompi/mca/topo/base/topo_base_cart_create.c
index 2e7257d5443..a1d562c813a 100644
--- a/ompi/mca/topo/base/topo_base_cart_create.c
+++ b/ompi/mca/topo/base/topo_base_cart_create.c
@@ -16,6 +16,8 @@
  *                         reserved.
  * Copyright (c) 2014-2015 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -29,6 +31,81 @@
 #include "ompi/mca/topo/base/base.h"
 #include "ompi/mca/topo/topo.h"
 
+static int mca_topo_base_cart_allocate (ompi_group_t *group, int ndims, const int *dims, const int *periods,
+                                        int *my_rank, int *num_procs, mca_topo_base_comm_cart_2_2_0_t **cart_out)
+{
+    mca_topo_base_comm_cart_2_2_0_t *cart = OBJ_NEW(mca_topo_base_comm_cart_2_2_0_t);
+    if (OPAL_UNLIKELY(NULL == cart)) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+    int nprocs = 1;
+
+    *num_procs = group->grp_proc_count;
+    *my_rank = group->grp_my_rank;
+
+    /* Calculate the number of processes in this grid */
+    for (int i = 0 ; i < ndims ; ++i) {
+        if (dims[i] <= 0) {
+            OBJ_RELEASE(cart);
+            return OMPI_ERROR;
+        }
+        nprocs *= dims[i];
+    }
+
+    /* check for the error condition */
+    if (OPAL_UNLIKELY(*num_procs < nprocs)) {
+        OBJ_RELEASE(cart);
+        return MPI_ERR_DIMS;
+    }
+
+    /* check if we have to trim the list of processes */
+    if (nprocs < *num_procs) {
+        *num_procs = nprocs;
+    }
+
+    if (*my_rank > (nprocs - 1)) {
+        *my_rank = MPI_UNDEFINED;
+    }
+
+    if (MPI_UNDEFINED == *my_rank) {
+        /* nothing more to do */
+        OBJ_RELEASE(cart);
+        *cart_out = NULL;
+        return OMPI_SUCCESS;
+    }
+
+    cart->ndims = ndims;
+
+    /* MPI-2.1 allows 0-dimension cartesian communicators, so prevent
+       a 0-byte malloc -- leave dims as NULL */
+    if (0 == ndims) {
+        *cart_out = cart;
+        return OMPI_SUCCESS;
+    }
+
+    cart->dims = (int *) malloc (sizeof (int) * ndims);
+    cart->periods = (int *) malloc (sizeof (int) * ndims);
+    cart->coords = (int *) malloc (sizeof (int) * ndims);
+    if (OPAL_UNLIKELY(NULL == cart->dims || NULL == cart->periods || NULL == cart->coords)) {
+        OBJ_RELEASE(cart);
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    /* Cartesian communicator; copy the right data to the common information */
+    memcpy(cart->dims, dims, ndims * sizeof(int));
+    memcpy(cart->periods, periods, ndims * sizeof(int));
+
+    nprocs = *num_procs;
+    for (int i = 0, rank = *my_rank ; i < ndims ; ++i) {
+        nprocs /= cart->dims[i];
+        cart->coords[i] = rank / nprocs;
+        rank %= nprocs;
+    }
+
+    *cart_out = cart;
+    return OMPI_SUCCESS;
+}
+
 /*
  * function - makes a new communicator to which topology information
  *            has been attached
@@ -55,135 +132,50 @@ int mca_topo_base_cart_create(mca_topo_base_module_t *topo,
                               bool reorder,
                               ompi_communicator_t** comm_topo)
 {
-    int nprocs = 1, i, new_rank, num_procs, ret;
+    int new_rank, num_procs, ret;
     ompi_communicator_t *new_comm;
-    ompi_proc_t **topo_procs = NULL;
     mca_topo_base_comm_cart_2_2_0_t* cart;
+    ompi_group_t *c_local_group;
 
-    num_procs = old_comm->c_local_group->grp_proc_count;
-    new_rank = old_comm->c_local_group->grp_my_rank;
     assert(topo->type == OMPI_COMM_CART);
 
-    /* Calculate the number of processes in this grid */
-    for (i = 0; i < ndims; ++i) {
-        if(dims[i] <= 0) {
-            return OMPI_ERROR;
-        }
-        nprocs *= dims[i];
-    }
-
-    /* check for the error condition */
-    if (num_procs < nprocs) {
-        return MPI_ERR_DIMS;
-    }
-
-    /* check if we have to trim the list of processes */
-    if (nprocs < num_procs) {
-        num_procs = nprocs;
-    }
-
-    if (new_rank > (nprocs-1)) {
-        ndims = 0;
-        new_rank = MPI_UNDEFINED;
-        num_procs = 0;
+    ret = mca_topo_base_cart_allocate (old_comm->c_local_group, ndims, dims, periods,
+                                       &new_rank, &num_procs, &cart);
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != ret)) {
+        return ret;
     }
 
-    cart = OBJ_NEW(mca_topo_base_comm_cart_2_2_0_t);
-    if( NULL == cart ) {
+    /* Copy the proc structure from the previous communicator over to
+       the new one.  The topology module is then able to work on this
+       copy and rearrange it as it deems fit. NTH: seems odd that this
+       function has always clipped the group size here. It might be
+       worthwhile to clip the group in the module (if reordering) */
+    c_local_group = ompi_group_flatten (old_comm->c_local_group, num_procs);
+    if (OPAL_UNLIKELY(NULL == c_local_group)) {
+        OBJ_RELEASE(cart);
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
-    cart->ndims = ndims;
 
-    /* MPI-2.1 allows 0-dimension cartesian communicators, so prevent
-       a 0-byte malloc -- leave dims as NULL */
-    if( ndims > 0 ) {
-        cart->dims = (int*)malloc(sizeof(int) * ndims);
-        if (NULL == cart->dims) {
-            OBJ_RELEASE(cart);
-            return OMPI_ERROR;
-        }
-        memcpy(cart->dims, dims, ndims * sizeof(int));
+    ret = ompi_comm_create (old_comm, c_local_group, &new_comm);
 
-        /* Cartesian communicator; copy the right data to the common information */
-        cart->periods = (int*)malloc(sizeof(int) * ndims);
-        if (NULL == cart->periods) {
-            OBJ_RELEASE(cart);
-            return OMPI_ERR_OUT_OF_RESOURCE;
-        }
-        memcpy(cart->periods, periods, ndims * sizeof(int));
+    ompi_group_free (&c_local_group);
 
-        cart->coords = (int*)malloc(sizeof(int) * ndims);
-        if (NULL == cart->coords) {
-            OBJ_RELEASE(cart);
-            return OMPI_ERR_OUT_OF_RESOURCE;
-        }
-        {  /* setup the cartesian topology */
-            int nprocs = num_procs, rank = new_rank;
-
-            for (i = 0; i < ndims; ++i) {
-                nprocs /= cart->dims[i];
-                cart->coords[i] = rank / nprocs;
-                rank %= nprocs;
-            }
-        }
-    }
-
-    /* JMS: This should really be refactored to use
-       comm_create_group(), because ompi_comm_allocate() still
-       complains about 0-byte mallocs in debug builds for 0-member
-       groups. */
-    if (num_procs > 0) {
-        /* Copy the proc structure from the previous communicator over to
-           the new one.  The topology module is then able to work on this
-           copy and rearrange it as it deems fit. */
-        topo_procs = (ompi_proc_t**)malloc(num_procs * sizeof(ompi_proc_t *));
-        if (NULL == topo_procs) {
-            OBJ_RELEASE(cart);
-            return OMPI_ERR_OUT_OF_RESOURCE;
-        }
-        if(OMPI_GROUP_IS_DENSE(old_comm->c_local_group)) {
-            memcpy(topo_procs,
-                   old_comm->c_local_group->grp_proc_pointers,
-                   num_procs * sizeof(ompi_proc_t *));
-        } else {
-            for(i = 0 ; i < num_procs; i++) {
-                topo_procs[i] = ompi_group_peer_lookup(old_comm->c_local_group,i);
-            }
-        }
-    }
-
-    /* allocate a new communicator */
-    new_comm = ompi_comm_allocate(num_procs, 0);
-    if (NULL == new_comm) {
-        free(topo_procs);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
         OBJ_RELEASE(cart);
-        return MPI_ERR_INTERN;
+        return ret;
     }
 
-    ret = ompi_comm_enable(old_comm, new_comm,
-                           new_rank, num_procs, topo_procs);
-    if (OMPI_SUCCESS != ret) {
-        /* something wrong happened during setting the communicator */
-        free(topo_procs);
-        OBJ_RELEASE(cart);
-        if (MPI_COMM_NULL != new_comm) {
-            new_comm->c_topo = NULL;
-            new_comm->c_flags &= ~OMPI_COMM_CART;
-            ompi_comm_free (&new_comm);
-        }
-        return ret;
+    *comm_topo = new_comm;
+
+    if (MPI_COMM_NULL == new_comm) {
+        /* not part of this new communicator */
+        return OMPI_SUCCESS;
     }
 
     new_comm->c_topo           = topo;
     new_comm->c_topo->mtc.cart = cart;
     new_comm->c_topo->reorder  = reorder;
     new_comm->c_flags         |= OMPI_COMM_CART;
-    *comm_topo = new_comm;
-
-    if( MPI_UNDEFINED == new_rank ) {
-        ompi_comm_free(&new_comm);
-        *comm_topo = MPI_COMM_NULL;
-    }
 
     /* end here */
     return OMPI_SUCCESS;
@@ -197,15 +189,9 @@ static void mca_topo_base_comm_cart_2_2_0_construct(mca_topo_base_comm_cart_2_2_
 }
 
 static void mca_topo_base_comm_cart_2_2_0_destruct(mca_topo_base_comm_cart_2_2_0_t * cart) {
-    if (NULL != cart->dims) {
-        free(cart->dims);
-    }
-    if (NULL != cart->periods) {
-        free(cart->periods);
-    }
-    if (NULL != cart->coords) {
-        free(cart->coords);
-    }
+    free(cart->dims);
+    free(cart->periods);
+    free(cart->coords);
 }
 
 OBJ_CLASS_INSTANCE(mca_topo_base_comm_cart_2_2_0_t, opal_object_t,
diff --git a/ompi/mca/topo/base/topo_base_comm_select.c b/ompi/mca/topo/base/topo_base_comm_select.c
index 165727fd393..b6d1847392d 100644
--- a/ompi/mca/topo/base/topo_base_comm_select.c
+++ b/ompi/mca/topo/base/topo_base_comm_select.c
@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -11,6 +12,8 @@
  *                         All rights reserved.
  * Copyright (c) 2008-2013 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2012-2013 Inria.  All rights reserved.
+ * Copyright (c) 2018      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -58,7 +61,7 @@ static OBJ_CLASS_INSTANCE(queried_module_t, opal_list_item_t, NULL, NULL);
  * communicator provided as argument is the old communicator, and the
  * newly selected topology is __not__ supposed to be attached to it.
  *
- * This module calls the query funtion on all the components that were
+ * This module calls the query function on all the components that were
  * detected by topo_base_open. This function is called on a
  * per-communicator basis. This function has the following function.
  *
@@ -68,10 +71,10 @@ static OBJ_CLASS_INSTANCE(queried_module_t, opal_list_item_t, NULL, NULL);
  * 4. Select the module with the highest priority.
  * 5. OBJ_RELEASE all the "losing" modules.
  */
-int mca_topo_base_comm_select(const ompi_communicator_t*  comm,
-                              mca_topo_base_module_t*     preferred_module,
-                              mca_topo_base_module_t**    selected_module,
-                              uint32_t                    type)
+static int _mca_topo_base_select (const ompi_communicator_t *comm, const ompi_group_t *group,
+                                  mca_topo_base_module_t *preferred_module,
+                                  mca_topo_base_module_t **selected_module,
+                                  uint32_t type)
 {
     int priority;
     int best_priority;
@@ -88,9 +91,15 @@ int mca_topo_base_comm_select(const ompi_communicator_t*  comm,
     if (OMPI_SUCCESS != (err = mca_topo_base_lazy_init())) {
         return err;
     }
-    opal_output_verbose(10, ompi_topo_base_framework.framework_output,
-                        "topo:base:comm_select: new communicator: %s (cid %d)",
-                        comm->c_name, comm->c_contextid);
+
+    if (comm) {
+        opal_output_verbose(10, ompi_topo_base_framework.framework_output,
+                            "topo:base:comm_select: new communicator: %s (cid %s)",
+                            comm->c_name, ompi_comm_print_cid (comm));
+    } else {
+        opal_output_verbose(10, ompi_topo_base_framework.framework_output,
+                            "topo:base:group_select: new communicator");
+    }
 
     /* Check and see if a preferred component was provided. If it was
        provided then it should be used (if possible) */
@@ -106,7 +115,7 @@ int mca_topo_base_comm_select(const ompi_communicator_t*  comm,
          /* query the component for its priority and get its module
             structure. This is necessary to proceed */
          component = (mca_topo_base_component_t *)preferred_module->topo_component;
-         module = component->topoc_comm_query(comm, &priority, type);
+         module = component->topoc_query(comm, group, &priority, type);
          if (NULL != module) {
 
              /* this query seems to have returned something legitimate
@@ -149,14 +158,14 @@ int mca_topo_base_comm_select(const ompi_communicator_t*  comm,
        /*
         * we can call the query function only if there is a function :-)
         */
-       if (NULL == component->topoc_comm_query) {
+       if (NULL == component->topoc_query) {
           opal_output_verbose(10, ompi_topo_base_framework.framework_output,
                              "select: no query, ignoring the component");
        } else {
            /*
             * call the query function and see what it returns
             */
-           module = component->topoc_comm_query(comm, &priority, type);
+           module = component->topoc_query(comm, group, &priority, type);
 
            if (NULL == module) {
                /*
@@ -218,7 +227,7 @@ int mca_topo_base_comm_select(const ompi_communicator_t*  comm,
             * module of this component.
             *
             * ANJU: a component might not have all the functions
-            * defined.  Whereever a function pointer is null in the
+            * defined.  Wherever a function pointer is null in the
             * module structure we need to fill it in with the base
             * structure function pointers. This is yet to be done
             */
@@ -226,7 +235,7 @@ int mca_topo_base_comm_select(const ompi_communicator_t*  comm,
             om->om_module->topo_component = best_component;
             *selected_module = om->om_module;
          } else {
-             /* this is not the "choosen one", finalize */
+             /* this is not the "chosen one", finalize */
               opal_output_verbose(10, ompi_topo_base_framework.framework_output,
                                   "select: component %s is not selected",
                                   om->om_component->topoc_version.mca_component_name);
@@ -242,7 +251,7 @@ int mca_topo_base_comm_select(const ompi_communicator_t*  comm,
     if(ompi_ftmpi_enabled) {
         /* check if module is tested for FT, warn if not. */
         const char* ft_whitelist="";
-        opal_show_help("help-ft-mpi.txt", "module:untested:failundef", true,
+        opal_show_help("help-mpi-ft.txt", "module:untested:failundef", true,
             best_component->topoc_version.mca_type_name,
             best_component->topoc_version.mca_component_name,
             ft_whitelist);
@@ -251,12 +260,23 @@ int mca_topo_base_comm_select(const ompi_communicator_t*  comm,
     return OMPI_SUCCESS;
 }
 
+int mca_topo_base_comm_select (const ompi_communicator_t *comm, mca_topo_base_module_t *preferred_module,
+                               mca_topo_base_module_t **selected_module, uint32_t type)
+{
+    return _mca_topo_base_select (comm, NULL, preferred_module, selected_module, type);
+}
+
+int mca_topo_base_group_select(const ompi_group_t *group, mca_topo_base_module_t *preferred_module,
+                               mca_topo_base_module_t **selected_module, uint32_t type)
+{
+    return _mca_topo_base_select (NULL, group, preferred_module, selected_module, type);
+}
 
 /*
  * This function fills in the null function pointers, in other words,
  * those functions which are not implemented by the module with the
  * pointers from the base function. Somewhere, I need to incoroporate
- * a check for the common minimum funtions being implemented by the
+ * a check for the common minimum functions being implemented by the
  * module.
  */
 static void fill_null_pointers(int type, mca_topo_base_module_t *module)
diff --git a/ompi/mca/topo/base/topo_base_dist_graph_create.c b/ompi/mca/topo/base/topo_base_dist_graph_create.c
index fdc202f879a..66e2976deb5 100644
--- a/ompi/mca/topo/base/topo_base_dist_graph_create.c
+++ b/ompi/mca/topo/base/topo_base_dist_graph_create.c
@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2008      The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -11,6 +12,8 @@
  * Copyright (c) 2014-2015 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016-2017 IBM Corporation.  All rights reserved.
+ * Copyright (c) 2018      Triad National Security, LLC. All rights
+ *                         reserved.
  */
 
 #include "ompi_config.h"
@@ -289,20 +292,10 @@ int mca_topo_base_dist_graph_create(mca_topo_base_module_t* module,
 {
     int err;
 
-    if( OMPI_SUCCESS != (err = ompi_comm_create(comm_old,
-                                                comm_old->c_local_group,
-                                                newcomm)) ) {
+    if (OMPI_SUCCESS != (err = ompi_comm_dup_with_info (comm_old, info, newcomm))) {
         OBJ_RELEASE(module);
         return err;
     }
-    // But if there is an info object, the above call didn't make use
-    // of it, so we'll do a dup-with-info to get the final comm and
-    // free the above intermediate newcomm:
-    if (info && info != &(MPI_INFO_NULL->super)) {
-        ompi_communicator_t *intermediate_comm = *newcomm;
-        ompi_comm_dup_with_info (intermediate_comm, info, newcomm);
-        ompi_comm_free(&intermediate_comm);
-    }
 
     assert(NULL == (*newcomm)->c_topo);
     (*newcomm)->c_topo             = module;
@@ -332,18 +325,10 @@ static void mca_topo_base_comm_dist_graph_2_2_0_construct(mca_topo_base_comm_dis
 }
 
 static void mca_topo_base_comm_dist_graph_2_2_0_destruct(mca_topo_base_comm_dist_graph_2_2_0_t * dist_graph) {
-    if (NULL != dist_graph->in) {
-        free(dist_graph->in);
-    }
-    if (NULL != dist_graph->inw) {
-        free(dist_graph->inw);
-    }
-    if (NULL != dist_graph->out) {
-        free(dist_graph->out);
-    }
-    if (NULL != dist_graph->outw) {
-        free(dist_graph->outw);
-    }
+    free(dist_graph->in);
+    free(dist_graph->inw);
+    free(dist_graph->out);
+    free(dist_graph->outw);
 }
 
 OBJ_CLASS_INSTANCE(mca_topo_base_comm_dist_graph_2_2_0_t, opal_object_t,
diff --git a/ompi/mca/topo/base/topo_base_dist_graph_create_adjacent.c b/ompi/mca/topo/base/topo_base_dist_graph_create_adjacent.c
index 5b12042708b..336aa05c733 100644
--- a/ompi/mca/topo/base/topo_base_dist_graph_create_adjacent.c
+++ b/ompi/mca/topo/base/topo_base_dist_graph_create_adjacent.c
@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2008      The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -11,6 +12,8 @@
  * Copyright (c) 2014-2015 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2017      IBM Corp.  All rights reserved.
+ * Copyright (c) 2018      Triad National Security, LLC. All rights
+ *                         reserved.
  */
 
 #include "ompi_config.h"
@@ -20,40 +23,23 @@
 #include "ompi/mca/topo/base/base.h"
 
 
-int mca_topo_base_dist_graph_create_adjacent(mca_topo_base_module_t* module,
-                                             ompi_communicator_t *comm_old,
-                                             int indegree, const int sources[],
-                                             const int sourceweights[],
-                                             int outdegree,
-                                             const int destinations[],
-                                             const int destweights[],
-                                             opal_info_t *info, int reorder,
-                                             ompi_communicator_t **newcomm)
+static int _mca_topo_base_dist_graph_create_adjacent (mca_topo_base_module_t* module, int indegree,
+                                                      const int sources[], const int sourceweights[],
+                                                      int outdegree, const int destinations[],
+                                                      const int destweights[], int reorder,
+                                                      ompi_communicator_t **newcomm)
 {
     mca_topo_base_comm_dist_graph_2_2_0_t *topo = NULL;
     int err;
 
-    if( OMPI_SUCCESS != (err = ompi_comm_create(comm_old,
-                                                comm_old->c_local_group,
-                                                newcomm)) ) {
-        return err;
-    }
-    // But if there is an info object, the above call didn't make use
-    // of it, so we'll do a dup-with-info to get the final comm and
-    // free the above intermediate newcomm:
-    if (info && info != &(MPI_INFO_NULL->super)) {
-        ompi_communicator_t *intermediate_comm = *newcomm;
-        ompi_comm_dup_with_info (intermediate_comm, info, newcomm);
-        ompi_comm_free(&intermediate_comm);
-    }
-
     err = OMPI_ERR_OUT_OF_RESOURCE;  /* suppose by default something bad will happens */
 
     assert( NULL == (*newcomm)->c_topo );
 
     topo = OBJ_NEW(mca_topo_base_comm_dist_graph_2_2_0_t);
-    if( NULL == topo ) {
-        goto bail_out;
+    if (NULL == topo) {
+        ompi_comm_free (newcomm);
+        return OMPI_ERR_OUT_OF_RESOURCE;
     }
     topo->in = topo->inw = NULL;
     topo->out = topo->outw = NULL;
@@ -103,16 +89,29 @@ int mca_topo_base_dist_graph_create_adjacent(mca_topo_base_module_t* module,
 
  bail_out:
     if (NULL != topo) {
-        if( NULL != topo->in ) free(topo->in);
-        if( MPI_UNWEIGHTED != sourceweights ) {
-            if( NULL != topo->inw ) free(topo->inw);
-        }
-        if( NULL != topo->out ) free(topo->out);
-        if( MPI_UNWEIGHTED != destweights ) {
-            if( NULL != topo->outw ) free(topo->outw);
-        }
         OBJ_RELEASE(topo);
     }
+
     ompi_comm_free(newcomm);
     return err;
 }
+
+int mca_topo_base_dist_graph_create_adjacent(mca_topo_base_module_t* module,
+                                             ompi_communicator_t *comm_old,
+                                             int indegree, const int sources[],
+                                             const int sourceweights[],
+                                             int outdegree,
+                                             const int destinations[],
+                                             const int destweights[],
+                                             opal_info_t *info, int reorder,
+                                             ompi_communicator_t **newcomm)
+{
+    int err;
+
+    if (OMPI_SUCCESS != (err = ompi_comm_dup_with_info (comm_old, info, newcomm))) {
+        return err;
+    }
+
+    return _mca_topo_base_dist_graph_create_adjacent (module, indegree, sources, sourceweights, outdegree,
+                                                      destinations, destweights, reorder, newcomm);
+}
diff --git a/ompi/mca/topo/base/topo_base_find_available.c b/ompi/mca/topo/base/topo_base_find_available.c
index 64a831c4cef..579f82e09fa 100644
--- a/ompi/mca/topo/base/topo_base_find_available.c
+++ b/ompi/mca/topo/base/topo_base_find_available.c
@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
diff --git a/ompi/mca/topo/base/topo_base_graph_create.c b/ompi/mca/topo/base/topo_base_graph_create.c
index f41cd033d9d..dfd2708bd53 100644
--- a/ompi/mca/topo/base/topo_base_graph_create.c
+++ b/ompi/mca/topo/base/topo_base_graph_create.c
@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -24,6 +25,46 @@
 #include "ompi/mca/topo/base/base.h"
 #include "ompi/mca/topo/topo.h"
 
+static int mca_topo_base_graph_allocate (ompi_group_t *group, int nnodes, const int *index, const int *edges,
+                                         int *num_procs, mca_topo_base_comm_graph_2_2_0_t **graph_out)
+{
+    mca_topo_base_comm_graph_2_2_0_t *graph;
+
+    *num_procs = group->grp_proc_count;
+
+    if (*num_procs < nnodes) {
+        return MPI_ERR_DIMS;
+    }
+
+    if (*num_procs > nnodes) {
+        *num_procs = nnodes;
+    }
+
+    if (group->grp_my_rank > (nnodes - 1) || MPI_UNDEFINED == group->grp_my_rank) {
+        *graph_out = NULL;
+        return OMPI_SUCCESS;
+    }
+
+    graph = OBJ_NEW(mca_topo_base_comm_graph_2_2_0_t);
+    if( NULL == graph ) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+    graph->nnodes = nnodes;
+    graph->index = (int *) malloc (sizeof (int) * nnodes);
+    graph->edges = (int *) malloc (sizeof (int) * index[nnodes-1]);
+    if (OPAL_UNLIKELY(NULL == graph->index || NULL == graph->edges)) {
+        OBJ_RELEASE(graph);
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    memcpy(graph->index, index, nnodes * sizeof(int));
+    memcpy(graph->edges, edges, index[nnodes-1] * sizeof(int));
+
+    *graph_out = graph;
+
+    return OMPI_SUCCESS;
+}
+
 /*
  *
  * function - makes a new communicator to which topology information
@@ -40,111 +81,41 @@
  * @retval MPI_ERR_OUT_OF_RESOURCE
  */
 
-int mca_topo_base_graph_create(mca_topo_base_module_t *topo,
-                               ompi_communicator_t* old_comm,
-                               int nnodes,
-                               const int *index,
-                               const int *edges,
-                               bool reorder,
-                               ompi_communicator_t** comm_topo)
+int mca_topo_base_graph_create (mca_topo_base_module_t *topo, ompi_communicator_t *old_comm,
+                                int nnodes, const int *index, const int *edges, bool reorder,
+                                ompi_communicator_t **comm_topo)
 {
-    ompi_communicator_t *new_comm;
-    int new_rank, num_procs, ret, i;
-    ompi_proc_t **topo_procs = NULL;
-    mca_topo_base_comm_graph_2_2_0_t* graph;
+    mca_topo_base_comm_graph_2_2_0_t *graph;
+    ompi_group_t *c_local_group;
+    int num_procs, ret;
 
-    num_procs = old_comm->c_local_group->grp_proc_count;
-    new_rank = old_comm->c_local_group->grp_my_rank;
     assert(topo->type == OMPI_COMM_GRAPH);
 
-    if( num_procs < nnodes ) {
-        return MPI_ERR_DIMS;
-    }
-    if( num_procs > nnodes ) {
-        num_procs = nnodes;
-    }
-    if( new_rank > (nnodes - 1) ) {
-        new_rank = MPI_UNDEFINED;
-        num_procs = 0;
-        nnodes = 0;
-    }
-
-    graph = OBJ_NEW(mca_topo_base_comm_graph_2_2_0_t);
-    if( NULL == graph ) {
-        return OMPI_ERR_OUT_OF_RESOURCE;
-    }
-    graph->nnodes = nnodes;
+    *comm_topo = MPI_COMM_NULL;
 
-    /* Don't do any of the other initialization if we're not supposed
-       to be part of the new communicator (because nnodes has been
-       reset to 0, making things like index[nnodes-1] be junk).
-
-       JMS: This should really be refactored to use
-       comm_create_group(), because ompi_comm_allocate() still
-       complains about 0-byte mallocs in debug builds for 0-member
-       groups. */
-    if (MPI_UNDEFINED != new_rank) {
-        graph->index = (int*)malloc(sizeof(int) * nnodes);
-        if (NULL == graph->index) {
-            OBJ_RELEASE(graph);
-            return OMPI_ERR_OUT_OF_RESOURCE;
-        }
-        memcpy(graph->index, index, nnodes * sizeof(int));
-
-        /* Graph communicator; copy the right data to the common information */
-        graph->edges = (int*)malloc(sizeof(int) * index[nnodes-1]);
-        if (NULL == graph->edges) {
-            OBJ_RELEASE(graph);
-            return OMPI_ERR_OUT_OF_RESOURCE;
-        }
-        memcpy(graph->edges, edges, index[nnodes-1] * sizeof(int));
-
-        topo_procs = (ompi_proc_t**)malloc(num_procs * sizeof(ompi_proc_t *));
-        if (NULL == topo_procs) {
-           OBJ_RELEASE(graph);
-           return OMPI_ERR_OUT_OF_RESOURCE;
-        }
-        if(OMPI_GROUP_IS_DENSE(old_comm->c_local_group)) {
-            memcpy(topo_procs,
-                   old_comm->c_local_group->grp_proc_pointers,
-                   num_procs * sizeof(ompi_proc_t *));
-        } else {
-            for(i = 0 ; i < num_procs; i++) {
-                topo_procs[i] = ompi_group_peer_lookup(old_comm->c_local_group,i);
-            }
-        }
+    ret = mca_topo_base_graph_allocate (old_comm->c_local_group, nnodes, index, edges, &num_procs,
+                                        &graph);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
+        return ret;
     }
 
-    /* allocate a new communicator */
-    new_comm = ompi_comm_allocate(nnodes, 0);
-    if (NULL == new_comm) {
-        free(topo_procs);
+    c_local_group = ompi_group_flatten (old_comm->c_local_group, nnodes);
+    if (OPAL_UNLIKELY(NULL == c_local_group)) {
         OBJ_RELEASE(graph);
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
 
-    ret = ompi_comm_enable(old_comm, new_comm,
-                           new_rank, num_procs, topo_procs);
-    if (OMPI_SUCCESS != ret) {
-        free(topo_procs);
+    ret = ompi_comm_create (old_comm, c_local_group, comm_topo);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
         OBJ_RELEASE(graph);
-        if (MPI_COMM_NULL != new_comm) {
-            new_comm->c_topo            = NULL;
-            new_comm->c_flags          &= ~OMPI_COMM_GRAPH;
-            ompi_comm_free (&new_comm);
-        }
         return ret;
     }
-    
-    new_comm->c_topo            = topo;
-    new_comm->c_topo->mtc.graph = graph;
-    new_comm->c_flags          |= OMPI_COMM_GRAPH;
-    new_comm->c_topo->reorder   = reorder;
-    *comm_topo = new_comm;
-
-    if( MPI_UNDEFINED == new_rank ) {
-        ompi_comm_free(&new_comm);
-        *comm_topo = MPI_COMM_NULL;
+
+    if (MPI_COMM_NULL != *comm_topo) {
+        (*comm_topo)->c_topo            = topo;
+        (*comm_topo)->c_topo->mtc.graph = graph;
+        (*comm_topo)->c_flags          |= OMPI_COMM_GRAPH;
+        (*comm_topo)->c_topo->reorder   = reorder;
     }
 
     return OMPI_SUCCESS;
diff --git a/ompi/mca/topo/basic/topo_basic.h b/ompi/mca/topo/basic/topo_basic.h
index 006005bffcd..32540335121 100644
--- a/ompi/mca/topo/basic/topo_basic.h
+++ b/ompi/mca/topo/basic/topo_basic.h
@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2011-2013 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
@@ -23,7 +24,7 @@ BEGIN_C_DECLS
 
 typedef mca_topo_base_component_2_2_0_t mca_topo_basic_component_t;
 /* Public component instance */
-OMPI_MODULE_DECLSPEC extern mca_topo_basic_component_t
+OMPI_DECLSPEC extern mca_topo_basic_component_t
     mca_topo_basic_component;
 
 END_C_DECLS
diff --git a/ompi/mca/topo/basic/topo_basic_component.c b/ompi/mca/topo/basic/topo_basic_component.c
index 83f26519273..f8d2fd494c8 100644
--- a/ompi/mca/topo/basic/topo_basic_component.c
+++ b/ompi/mca/topo/basic/topo_basic_component.c
@@ -26,11 +26,11 @@ const char *mca_topo_basic_component_version_string =
     "Open MPI basic topology MCA component version" OMPI_VERSION;
 
 /*
- * Local funtions
+ * Local functions
  */
 static int init_query(bool enable_progress_threads, bool enable_mpi_threads);
 static struct mca_topo_base_module_t *
-comm_query(const ompi_communicator_t *comm, int *priority, uint32_t type);
+mca_topo_basic_query(const ompi_communicator_t *comm, const ompi_group_t *group, int *priority, uint32_t type);
 
 /*
  * Public component structure
@@ -40,9 +40,8 @@ mca_topo_basic_component_t mca_topo_basic_component =
     .topoc_version = {
         MCA_TOPO_BASE_VERSION_2_2_0,
         .mca_component_name = "basic",
-        .mca_component_major_version = OMPI_MAJOR_VERSION,
-        .mca_component_minor_version = OMPI_MINOR_VERSION,
-        .mca_component_release_version = OMPI_RELEASE_VERSION,
+        MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION,
+                              OMPI_RELEASE_VERSION),
         /* NULLs for the rest of the function pointers */
     },
 
@@ -52,7 +51,7 @@ mca_topo_basic_component_t mca_topo_basic_component =
     },
 
     .topoc_init_query = init_query,
-    .topoc_comm_query = comm_query,
+    .topoc_query = mca_topo_basic_query,
 };
 
 
@@ -64,7 +63,7 @@ static int init_query(bool enable_progress_threads, bool enable_mpi_threads)
 
 
 static struct mca_topo_base_module_t *
-comm_query(const ompi_communicator_t *comm, int *priority, uint32_t type)
+mca_topo_basic_query (const ompi_communicator_t *comm, const ompi_group_t *group, int *priority, uint32_t type)
 {
     /* Don't use OBJ_NEW, we need to zero the memory or the functions pointers
      * will not be correctly copied over from the base.
@@ -81,5 +80,3 @@ comm_query(const ompi_communicator_t *comm, int *priority, uint32_t type)
     basic->type = type;
     return basic;
 }
-
-
diff --git a/ompi/mca/topo/example/topo_example.h b/ompi/mca/topo/example/topo_example.h
index 9d14e89d1d9..8e2b2903d7d 100644
--- a/ompi/mca/topo/example/topo_example.h
+++ b/ompi/mca/topo/example/topo_example.h
@@ -26,7 +26,7 @@
 
 /*
  * ******************************************************************
- * ******** functions which provide MCA interface comppliance *******
+ * ******** functions which provide MCA interface compliance ********
  * ******************************************************************
  * These functions are:
  *       - mca_topo_example_module_open
@@ -42,7 +42,7 @@ BEGIN_C_DECLS
 /*
  * Public component instance
  */
-OMPI_MODULE_DECLSPEC extern mca_topo_base_component_2_2_0_t
+OMPI_DECLSPEC extern mca_topo_base_component_2_2_0_t
     mca_topo_example_component;
 
 /*
diff --git a/ompi/mca/topo/example/topo_example_component.c b/ompi/mca/topo/example/topo_example_component.c
index 3874a89b7bc..799471a8a37 100644
--- a/ompi/mca/topo/example/topo_example_component.c
+++ b/ompi/mca/topo/example/topo_example_component.c
@@ -31,7 +31,7 @@ const char *mca_topo_example_component_version_string =
     "Open MPI example topology MCA component version" OMPI_VERSION;
 
 /*
- * Local funtions
+ * Local functions
  */
 static int init_query(bool enable_progress_threads, bool enable_mpi_threads);
 static struct mca_topo_base_module_t *
diff --git a/ompi/mca/topo/topo.h b/ompi/mca/topo/topo.h
index 7735250f290..5ff6186183f 100644
--- a/ompi/mca/topo/topo.h
+++ b/ompi/mca/topo/topo.h
@@ -45,12 +45,11 @@ typedef int (*mca_topo_base_component_init_query_2_2_0_fn_t)
      bool enable_mpi_threads);
 
 /*
- * Communicator query, called during cart and graph communicator
- * creation.
+ * Communicator/group query, called during cart and graph communicator creation.
  */
 typedef struct mca_topo_base_module_t*
-(*mca_topo_base_component_comm_query_2_2_0_fn_t)
-    (const ompi_communicator_t *comm, int *priority, uint32_t type);
+(*mca_topo_base_component_query_2_2_0_fn_t)
+    (const ompi_communicator_t *comm, const ompi_group_t *group, int *priority, uint32_t type);
 
 /*
  * Structure for topo v2.1.0 components.This is chained to MCA v2.0.0
@@ -60,7 +59,7 @@ typedef struct mca_topo_base_component_2_2_0_t {
     mca_base_component_data_t topoc_data;
 
     mca_topo_base_component_init_query_2_2_0_fn_t topoc_init_query;
-    mca_topo_base_component_comm_query_2_2_0_fn_t topoc_comm_query;
+    mca_topo_base_component_query_2_2_0_fn_t topoc_query;
 } mca_topo_base_component_2_2_0_t;
 typedef mca_topo_base_component_2_2_0_t mca_topo_base_component_t;
 
@@ -344,7 +343,7 @@ OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_topo_base_module_t);
 
 /*
  * ******************************************************************
- * ********** Use in components that are of type topo v2.2.0 ********
+ * ********** Use in components that are of type topo v2.3.0 ********
  * ******************************************************************
  */
 #define MCA_TOPO_BASE_VERSION_2_2_0 \
diff --git a/ompi/mca/topo/treematch/configure.m4 b/ompi/mca/topo/treematch/configure.m4
index 18774d7b93d..8301d173f5e 100644
--- a/ompi/mca/topo/treematch/configure.m4
+++ b/ompi/mca/topo/treematch/configure.m4
@@ -9,7 +9,7 @@
 # Copyright (c) 2015      Intel, Inc. All rights reserved.
 # Copyright (c) 2019      Research Organization for Information Science
 #                         and Technology (RIST).  All rights reserved.
-# Copyright (c) 2020      Amazon.com, Inc. or its affiliates.
+# Copyright (c) 2020-2022 Amazon.com, Inc. or its affiliates.  All Rights reserved.
 #                         All Rights reserved.
 # $COPYRIGHT$
 #
@@ -25,7 +25,6 @@ AC_DEFUN([MCA_ompi_topo_treematch_CONFIG], [
     OPAL_CONFIG_TREEMATCH([topo_treematch], [$1], [$2])
     AC_CONFIG_FILES([ompi/mca/topo/treematch/Makefile])
 
-    AC_SUBST([topo_treematch_CFLAGS])
     AC_SUBST([topo_treematch_CPPFLAGS])
     AC_SUBST([topo_treematch_LDFLAGS])
     AC_SUBST([topo_treematch_LIBS])
diff --git a/ompi/mca/topo/treematch/topo_treematch.h b/ompi/mca/topo/treematch/topo_treematch.h
index bcc4d748bfd..5a885dfd77a 100644
--- a/ompi/mca/topo/treematch/topo_treematch.h
+++ b/ompi/mca/topo/treematch/topo_treematch.h
@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2011-2015 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
@@ -22,7 +23,7 @@
 
 /*
  * ******************************************************************
- * ******** functions which provide MCA interface comppliance *******
+ * ******** functions which provide MCA interface compliance ********
  * ******************************************************************
  * These functions are:
  *       - mca_topo_treematch_module_open
@@ -44,7 +45,7 @@ typedef struct mca_topo_treematch_component_2_2_0_t {
     int reorder_mode;
 } mca_topo_treematch_component_2_2_0_t;
 
-OMPI_MODULE_DECLSPEC extern mca_topo_treematch_component_2_2_0_t
+OMPI_DECLSPEC extern mca_topo_treematch_component_2_2_0_t
     mca_topo_treematch_component;
 
 /*
diff --git a/ompi/mca/topo/treematch/topo_treematch_component.c b/ompi/mca/topo/treematch/topo_treematch_component.c
index fca7e5b71b0..ba1d870e5e3 100644
--- a/ompi/mca/topo/treematch/topo_treematch_component.c
+++ b/ompi/mca/topo/treematch/topo_treematch_component.c
@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2011-2015 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
@@ -22,11 +23,11 @@ const char *mca_topo_treematch_component_version_string =
     "Open MPI treematch topology MCA component version" OMPI_VERSION;
 
 /*
- * Local funtions
+ * Local functions
  */
 static int init_query(bool enable_progress_threads, bool enable_mpi_threads);
 static struct mca_topo_base_module_t *
-comm_query(const ompi_communicator_t *comm, int *priority, uint32_t type);
+mca_topo_treematch_query(const ompi_communicator_t *comm, const ompi_group_t *group, int *priority, uint32_t type);
 static int mca_topo_treematch_component_register(void);
 
 /*
@@ -34,29 +35,24 @@ static int mca_topo_treematch_component_register(void);
  */
 mca_topo_treematch_component_2_2_0_t mca_topo_treematch_component =
     {
-        {
-            {
+        .super = {
+            .topoc_version = {
                 MCA_TOPO_BASE_VERSION_2_2_0,
-
-                "treematch",
-                OMPI_MAJOR_VERSION,
-                OMPI_MINOR_VERSION,
-                OMPI_RELEASE_VERSION,
-                NULL,  /* component open */
-                NULL,  /* component close */
-                NULL, /* component query */
-                mca_topo_treematch_component_register, /* component register */
+                .mca_component_name = "treematch",
+                MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION,
+                                      OMPI_RELEASE_VERSION),
+                .mca_register_component_params = mca_topo_treematch_component_register,
             },
 
-            {
+            .topoc_data = {
                 /* The component is checkpoint ready */
                 MCA_BASE_METADATA_PARAM_CHECKPOINT
             },
 
-            init_query,
-            comm_query
+            .topoc_init_query = init_query,
+            .topoc_query = mca_topo_treematch_query,
         },
-        0  /* reorder: by default centralized */
+        .reorder_mode = 0  /* reorder: by default centralized */
     };
 
 
@@ -72,7 +68,7 @@ static int init_query(bool enable_progress_threads, bool enable_mpi_threads)
 
 
 static struct mca_topo_base_module_t *
-comm_query(const ompi_communicator_t *comm, int *priority, uint32_t type)
+mca_topo_treematch_query(const ompi_communicator_t *comm, const ompi_group_t *group, int *priority, uint32_t type)
 {
     mca_topo_treematch_module_t *treematch;
 
diff --git a/ompi/mca/topo/treematch/topo_treematch_dist_graph_create.c b/ompi/mca/topo/treematch/topo_treematch_dist_graph_create.c
index e4887eea213..785d8977142 100644
--- a/ompi/mca/topo/treematch/topo_treematch_dist_graph_create.c
+++ b/ompi/mca/topo/treematch/topo_treematch_dist_graph_create.c
@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2011-2017 The University of Tennessee and The University
+ * Copyright (c) 2011-2023 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2011-2018 Inria.  All rights reserved.
@@ -382,7 +382,6 @@ int mca_topo_treematch_dist_graph_create(mca_topo_base_module_t* topo_module,
 
     /* Centralized Reordering */
     if (0 == mca_topo_treematch_component.reorder_mode) {
-        int *k = NULL;
         int *obj_mapping = NULL;
         int num_objs_total = 0;
 
@@ -693,18 +692,19 @@ int mca_topo_treematch_dist_graph_create(mca_topo_base_module_t* topo_module,
                                                                   &newrank, 1, MPI_INT,
                                                                   0, comm_old,
                                                                   comm_old->c_coll->coll_scatter_module))) {
-            if (NULL != k) free(k);
+            if (NULL != k) { free(k); k = NULL; }
             goto release_and_return;
         }
 
-        if ( 0 == rank )
+        if ( 0 == rank ) {
             free(k);
+            k = NULL;
+        }
 
         /* this needs to be optimized but will do for now */
         if (OMPI_SUCCESS != (err = ompi_comm_split(comm_old, 0, newrank, newcomm, false))) {
             goto release_and_return;
         }
-        /* end of TODO */
 
         /* Attach the dist_graph to the newly created communicator */
         (*newcomm)->c_flags        |= OMPI_COMM_DIST_GRAPH;
@@ -903,7 +903,7 @@ int mca_topo_treematch_dist_graph_create(mca_topo_base_module_t* topo_module,
                                                                    &newrank, 1, MPI_INT,
                                                                    0, localcomm,
                                                                    localcomm->c_coll->coll_scatter_module))) {
-            if (NULL != k) free(k);
+            if (NULL != k) { free(k); k = NULL; };
             ompi_comm_free(&localcomm);
             free(lrank_to_grank);
             free(grank_to_lrank);
@@ -932,8 +932,10 @@ int mca_topo_treematch_dist_graph_create(mca_topo_base_module_t* topo_module,
         newrank += offset;
         free(marked);
 
-        if (rank == lindex_to_grank[0])
+        if (rank == lindex_to_grank[0]) {
             free(k);
+            k = NULL;
+        }
 
         /* this needs to be optimized but will do for now */
         if (OMPI_SUCCESS != (err = ompi_comm_split(comm_old, 0, newrank, newcomm, false))) {
@@ -942,7 +944,6 @@ int mca_topo_treematch_dist_graph_create(mca_topo_base_module_t* topo_module,
             free(grank_to_lrank);
             goto release_and_return;
         }
-        /* end of TODO */
 
         /* Attach the dist_graph to the newly created communicator */
         (*newcomm)->c_flags        |= OMPI_COMM_DIST_GRAPH;
@@ -953,6 +954,17 @@ int mca_topo_treematch_dist_graph_create(mca_topo_base_module_t* topo_module,
         free(lrank_to_grank);
     } /* distributed reordering end */
 
+    /* Translate the ranks provided by the user to account for the reordered communicator.
+     * Note that this operation is safe to be done in place, directly into the in/out arrays.
+     */
+    ompi_group_translate_ranks(comm_old->c_remote_group, topo->indegree,
+                               topo->in,
+                               (*newcomm)->c_remote_group,
+                               topo->in);
+    ompi_group_translate_ranks(comm_old->c_remote_group, topo->outdegree,
+                               topo->out,
+                               (*newcomm)->c_remote_group,
+                               topo->out);
   release_and_return:
     if (NULL != reqs ) free(reqs);
     if (NULL != tracker) free(tracker);
diff --git a/ompi/mca/vprotocol/base/vprotocol_base_select.c b/ompi/mca/vprotocol/base/vprotocol_base_select.c
index 47e8ea6190d..f8d8f833e03 100644
--- a/ompi/mca/vprotocol/base/vprotocol_base_select.c
+++ b/ompi/mca/vprotocol/base/vprotocol_base_select.c
@@ -114,7 +114,7 @@ int mca_vprotocol_base_select(bool enable_progress_threads,
             /* Finalize */
             V_OUTPUT_VERBOSE(500, "vprotocol select: component %s not selected / finalized", om->om_component->pmlm_version.mca_component_name);
             if (NULL != om->om_component->pmlm_finalize) {
-                /* Blatently ignore the return code (what would we do to
+                /* Blatantly ignore the return code (what would we do to
                 recover, anyway?  This component is going away, so errors
                 don't matter anymore) */
                 om->om_component->pmlm_finalize();
diff --git a/ompi/mca/vprotocol/example/vprotocol_example_recv.c b/ompi/mca/vprotocol/example/vprotocol_example_recv.c
index 349e340270a..dbaaa9c88cf 100644
--- a/ompi/mca/vprotocol/example/vprotocol_example_recv.c
+++ b/ompi/mca/vprotocol/example/vprotocol_example_recv.c
@@ -56,7 +56,7 @@ int mca_vprotocol_example_recv(void *addr,
   int ret;
   V_OUTPUT_VERBOSE(50, "posted\trecv \tcomm %d\tfrom %d\ttag %d\tsize %ld", comm->c_contextid, src, tag, (long) count);
   ret = mca_pml_v.host_pml.pml_recv(addr, count, datatype, src, tag, comm, status);
-# ifdef OPAL_ENABLE_DEBUG
+# if OPAL_ENABLE_DEBUG
     if(status)
       V_OUTPUT_VERBOSE(75, "deliver\trecv \tcomm %d\tfrom %d(%d)\ttag %d(%d)\tsize %ld(%ld)\tstatus %d", comm->c_contextid, src, status->MPI_SOURCE, tag, status->MPI_TAG, (long) count, (long) status->_ucount, status->MPI_ERROR);
     else
diff --git a/ompi/mca/vprotocol/pessimist/vprotocol_pessimist.c b/ompi/mca/vprotocol/pessimist/vprotocol_pessimist.c
index b3ba707c746..4ca64d7c830 100644
--- a/ompi/mca/vprotocol/pessimist/vprotocol_pessimist.c
+++ b/ompi/mca/vprotocol/pessimist/vprotocol_pessimist.c
@@ -50,6 +50,6 @@ mca_vprotocol_pessimist_module_t mca_vprotocol_pessimist =
 
 int mca_vprotocol_pessimist_dump(struct ompi_communicator_t* comm, int verbose)
 {
-  V_OUTPUT_VERBOSE(verbose, "vprotocol_pessimist: dump for comm %d", comm->c_contextid);
+  V_OUTPUT_VERBOSE(verbose, "vprotocol_pessimist: dump for comm %s", ompi_comm_print_cid (comm));
   return mca_pml_v.host_pml.pml_dump(comm, verbose);
 }
diff --git a/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_component.c b/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_component.c
index 61d968ec332..005f55bc33f 100644
--- a/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_component.c
+++ b/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_component.c
@@ -4,6 +4,7 @@
  *                         All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -183,13 +184,13 @@ static int mca_vprotocol_pessimist_component_finalize(void)
 int mca_vprotocol_pessimist_enable(bool enable) {
     if(enable) {
         int ret;
-        if((ret = vprotocol_pessimist_sender_based_init(_mmap_file_name,
+        if((ret = ompi_vprotocol_pessimist_sender_based_init(_mmap_file_name,
                                                  _sender_based_size)) != OMPI_SUCCESS)
             return ret;
     }
     else {
-        vprotocol_pessimist_sender_based_finalize();
-        vprotocol_pessimist_event_logger_disconnect(mca_vprotocol_pessimist.el_comm);
+        ompi_vprotocol_pessimist_sender_based_finalize();
+        ompi_vprotocol_pessimist_event_logger_disconnect(mca_vprotocol_pessimist.el_comm);
     }
     return OMPI_SUCCESS;
 }
diff --git a/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_eventlog.c b/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_eventlog.c
index 8625e5ca6a7..29e82e7a51c 100644
--- a/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_eventlog.c
+++ b/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_eventlog.c
@@ -6,6 +6,7 @@
  * Copyright (c) 2015-2019 Intel, Inc.  All rights reserved.
  * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -19,7 +20,7 @@
 #include "opal/util/printf.h"
 #include "ompi/dpm/dpm.h"
 
-int vprotocol_pessimist_event_logger_connect(int el_rank, ompi_communicator_t **el_comm)
+int ompi_vprotocol_pessimist_event_logger_connect(int el_rank, ompi_communicator_t **el_comm)
 {
     int rc;
     pmix_status_t prc;
@@ -68,13 +69,13 @@ int vprotocol_pessimist_event_logger_connect(int el_rank, ompi_communicator_t **
     return rc;
 }
 
-int vprotocol_pessimist_event_logger_disconnect(ompi_communicator_t *el_comm)
+int ompi_vprotocol_pessimist_event_logger_disconnect(ompi_communicator_t *el_comm)
 {
     ompi_dpm_disconnect(el_comm);
     return OMPI_SUCCESS;
 }
 
-void vprotocol_pessimist_matching_replay(int *src) {
+void ompi_vprotocol_pessimist_matching_replay(int *src) {
 #if OPAL_ENABLE_DEBUG
     vprotocol_pessimist_clock_t max = 0;
 #endif
@@ -102,7 +103,7 @@ void vprotocol_pessimist_matching_replay(int *src) {
         else if(mevent->reqid > max)
             max = mevent->reqid;
     }
-    /* not forcing a ANY SOURCE event whose recieve clock is lower than max
+    /* not forcing a ANY SOURCE event whose receive clock is lower than max
      * is a bug indicating we have missed an event during logging ! */
     assert(((*src) != MPI_ANY_SOURCE) || (mca_vprotocol_pessimist.clock > max));
 #else
@@ -110,7 +111,7 @@ void vprotocol_pessimist_matching_replay(int *src) {
 #endif
 }
 
-void vprotocol_pessimist_delivery_replay(size_t n, ompi_request_t **reqs,
+void ompi_vprotocol_pessimist_delivery_replay(size_t n, ompi_request_t **reqs,
                                          int *outcount, int *index,
                                          ompi_status_public_t *status) {
     mca_vprotocol_pessimist_event_t *event;
diff --git a/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_eventlog.h b/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_eventlog.h
index af0631812b3..1ebd9db8b7a 100644
--- a/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_eventlog.h
+++ b/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_eventlog.h
@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2004-2007 The Trustees of the University of Tennessee.
  *                         All rights reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -11,6 +12,7 @@
 #ifndef __VPROTOCOL_PESSIMIST_EVENTLOG_H__
 #define __VPROTOCOL_PESSIMIST_EVENTLOG_H__
 
+#include "ompi/mca/pml/v/pml_v_output.h"
 #include "vprotocol_pessimist.h"
 #include "vprotocol_pessimist_request.h"
 #include "vprotocol_pessimist_eventlog_protocol.h"
@@ -20,12 +22,12 @@ BEGIN_C_DECLS
 /** Initialize the MPI connexion with the event logger
  * @return OMPI_SUCCESS or error code
  */
-int vprotocol_pessimist_event_logger_connect(int el_rank, ompi_communicator_t **el_comm);
+int ompi_vprotocol_pessimist_event_logger_connect(int el_rank, ompi_communicator_t **el_comm);
 
 /** Finalize the MPI connexion with the event logger
  * @return OMPI_SUCCESS or error code
  */
-int vprotocol_pessimist_event_logger_disconnect(ompi_communicator_t *el_comm);
+int ompi_vprotocol_pessimist_event_logger_disconnect(ompi_communicator_t *el_comm);
 
 /*******************************************************************************
   * ANY_SOURCE MATCHING
@@ -84,7 +86,7 @@ static inline void vprotocol_pessimist_matching_log_finish(ompi_request_t *req)
         vprotocol_pessimist_clock_t max_clock;                                \
         if(OPAL_UNLIKELY(ompi_comm_invalid(mca_vprotocol_pessimist.el_comm))) \
         {                                                                     \
-            rc = vprotocol_pessimist_event_logger_connect(0,                  \
+            rc = ompi_vprotocol_pessimist_event_logger_connect(0,             \
                                         &mca_vprotocol_pessimist.el_comm);    \
             if(OMPI_SUCCESS != rc)                                            \
                 OMPI_ERRHANDLER_INVOKE(mca_vprotocol_pessimist.el_comm, rc,   \
@@ -170,9 +172,9 @@ static inline void vprotocol_pessimist_event_flush(void)
  */
 #define VPROTOCOL_PESSIMIST_MATCHING_REPLAY(src) do {                         \
   if(mca_vprotocol_pessimist.replay && ((src) == MPI_ANY_SOURCE))             \
-    vprotocol_pessimist_matching_replay(&(src));                              \
+    ompi_vprotocol_pessimist_matching_replay(&(src));                         \
 } while(0)
-void vprotocol_pessimist_matching_replay(int *src);
+void ompi_vprotocol_pessimist_matching_replay(int *src);
 
 /*******************************************************************************
   * WAIT/TEST-SOME/ANY & PROBES
@@ -231,11 +233,11 @@ static inline void vprotocol_pessimist_delivery_log(ompi_request_t *req)
   * i (OUT): index(es) of the delivered request
   * status (OUT): status of the delivered request
   */
-#define VPROTOCOL_PESSIMIST_DELIVERY_REPLAY(n, reqs, outcount, i, status) do {\
-  if(mca_vprotocol_pessimist.replay)                                          \
-    vprotocol_pessimist_delivery_replay(n, reqs, outcount, i, status);        \
+#define VPROTOCOL_PESSIMIST_DELIVERY_REPLAY(n, reqs, outcount, i, status) do {   \
+  if(mca_vprotocol_pessimist.replay)                                             \
+    ompi_vprotocol_pessimist_delivery_replay(n, reqs, outcount, i, status);      \
 } while(0)
-void vprotocol_pessimist_delivery_replay(size_t, ompi_request_t **,
+void ompi_vprotocol_pessimist_delivery_replay(size_t, ompi_request_t **,
                                          int *, int *, ompi_status_public_t *);
 
 END_C_DECLS
diff --git a/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_proc.c b/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_proc.c
index 1b7b9d90499..4f633edc732 100644
--- a/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_proc.c
+++ b/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_proc.c
@@ -13,7 +13,7 @@
 
 int mca_vprotocol_pessimist_add_procs(struct ompi_proc_t **procs, size_t nprocs)
 {
-  /* TODO: for each proc, retrieve post send of sender based request, post recieve of list
+  /* TODO: for each proc, retrieve post send of sender based request, post receive of list
       block any other communications until we are up. To be determined how i manage to send (or resend) data to failed nodes
   */
   return mca_pml_v.host_pml.pml_add_procs(procs, nprocs);
diff --git a/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_progress.c b/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_progress.c
index 3b993ce61dc..be6323cf797 100644
--- a/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_progress.c
+++ b/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_progress.c
@@ -21,7 +21,7 @@ int mca_vprotocol_pessimist_progress(void)
     printf("PROGRESS\n");
     /* First let the real progress take place */
     ret = mca_pml_v.host_pml.pml_progress();
-    /* Then progess the sender_based copies */
+    /* Then progress the sender_based copies */
     vprotocol_pessimist_sb_progress_all_reqs();
     return ret;
 }
diff --git a/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_request.c b/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_request.c
index 7408157e63d..ceb59c7c671 100644
--- a/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_request.c
+++ b/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_request.c
@@ -31,7 +31,7 @@ static void vprotocol_pessimist_request_construct(mca_pml_base_request_t *req)
     ftreq->pml_req_free = req->req_ompi.req_free;
     ftreq->event = NULL;
     ftreq->sb.bytes_progressed = 0;
-    assert(ftreq->pml_req_free == req->req_ompi.req_free); /* detection of aligment issues on different arch */
+    assert(ftreq->pml_req_free == req->req_ompi.req_free); /* detection of alignment issues on different arch */
     req->req_ompi.req_free = mca_vprotocol_pessimist_request_free;
     OBJ_CONSTRUCT(& ftreq->list_item, opal_list_item_t);
 }
diff --git a/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_sender_based.c b/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_sender_based.c
index 50ed0a16c85..6db34752f32 100644
--- a/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_sender_based.c
+++ b/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_sender_based.c
@@ -3,6 +3,7 @@
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -80,7 +81,7 @@ static void sb_mmap_free(void)
                      (void *) sb.sb_addr, strerror(errno));
 }
 
-int vprotocol_pessimist_sender_based_init(const char *mmapfile, size_t size)
+int ompi_vprotocol_pessimist_sender_based_init(const char *mmapfile, size_t size)
 {
     char *path;
 #ifdef SB_USE_CONVERTOR_METHOD
@@ -107,7 +108,7 @@ int vprotocol_pessimist_sender_based_init(const char *mmapfile, size_t size)
     return OMPI_SUCCESS;
 }
 
-void vprotocol_pessimist_sender_based_finalize(void)
+void ompi_vprotocol_pessimist_sender_based_finalize(void)
 {
     if(((uintptr_t) NULL) != sb.sb_addr)
         sb_mmap_free();
@@ -118,7 +119,7 @@ void vprotocol_pessimist_sender_based_finalize(void)
 /** Manage mmap floating window, allocating enough memory for the message to be
   * asynchronously copied to disk.
   */
-void vprotocol_pessimist_sender_based_alloc(size_t len)
+void ompi_vprotocol_pessimist_sender_based_alloc(size_t len)
 {
     if(((uintptr_t) NULL) != sb.sb_addr)
         sb_mmap_free();
@@ -127,7 +128,7 @@ void vprotocol_pessimist_sender_based_alloc(size_t len)
         ompi_comm_dup(MPI_COMM_SELF, &sb.sb_comm, 1);
 #endif
 
-    /* Take care of alignement of sb_offset                             */
+    /* Take care of alignment of sb_offset                             */
     sb.sb_offset += sb.sb_cursor - sb.sb_addr;
     sb.sb_cursor = sb.sb_offset % sb.sb_pagesize;
     sb.sb_offset -= sb.sb_cursor;
diff --git a/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_sender_based.h b/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_sender_based.h
index b639b47b394..c0ee12926fe 100644
--- a/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_sender_based.h
+++ b/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_sender_based.h
@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2004-2007 The Trustees of the University of Tennessee.
  *                         All rights reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -22,16 +23,16 @@ BEGIN_C_DECLS
 
 /** Prepare for using the sender based storage
   */
-int vprotocol_pessimist_sender_based_init(const char *mmapfile, size_t size);
+int ompi_vprotocol_pessimist_sender_based_init(const char *mmapfile, size_t size);
 
 /** Cleanup mmap etc
   */
-void vprotocol_pessimist_sender_based_finalize(void);
+void ompi_vprotocol_pessimist_sender_based_finalize(void);
 
 /** Manage mmap floating window, allocating enough memory for the message to be
   * asynchronously copied to disk.
   */
-void vprotocol_pessimist_sender_based_alloc(size_t len);
+void ompi_vprotocol_pessimist_sender_based_alloc(size_t len);
 
 
 /*******************************************************************************
@@ -177,7 +178,7 @@ static inline void vprotocol_pessimist_sender_based_copy_start(ompi_request_t *r
             pmlreq->req_bytes_packed +
             sizeof(vprotocol_pessimist_sender_based_header_t))
     {
-        vprotocol_pessimist_sender_based_alloc(pmlreq->req_bytes_packed);
+        ompi_vprotocol_pessimist_sender_based_alloc(pmlreq->req_bytes_packed);
     }
 
     /* Copy message header to the sender-based space */
@@ -195,7 +196,7 @@ static inline void vprotocol_pessimist_sender_based_copy_start(ompi_request_t *r
     sbhdr->size = pmlreq->req_bytes_packed;
     sbhdr->dst = pmlreq->req_base.req_peer;
     sbhdr->tag = pmlreq->req_base.req_tag;
-    sbhdr->contextid = pmlreq->req_base.req_comm->c_contextid;
+    sbhdr->contextid = ompi_comm_get_extended_cid (pmlreq->req_base.req_comm);
     sbhdr->sequence = pmlreq->req_base.req_sequence;
     ftreq->sb.cursor += sizeof(vprotocol_pessimist_sender_based_header_t);
     V_OUTPUT_VERBOSE(70, "pessimist:\tsb\tsend\t%"PRIpclock"\tsize %lu (+%lu header)", VPESSIMIST_FTREQ(req)->reqid, (long unsigned)pmlreq->req_bytes_packed, (long unsigned)sizeof(vprotocol_pessimist_sender_based_header_t));
diff --git a/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_sender_based_types.h b/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_sender_based_types.h
index c00dfff70e3..c19ade18e76 100644
--- a/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_sender_based_types.h
+++ b/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_sender_based_types.h
@@ -46,7 +46,7 @@ typedef struct vprotocol_pessimist_sender_based_header_t
     size_t size;
     int dst;
     int tag;
-    uint32_t contextid;
+    ompi_comm_extended_cid_t contextid;
     vprotocol_pessimist_clock_t sequence;
 } vprotocol_pessimist_sender_based_header_t;
 
diff --git a/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_wait.c b/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_wait.c
index 7dce3f92375..4d8481d883d 100644
--- a/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_wait.c
+++ b/ompi/mca/vprotocol/pessimist/vprotocol_pessimist_wait.c
@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2004-2007 The Trustees of the University of Tennessee.
  *                         All rights reserved.
+ * Copyright (c) 2021      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -19,8 +20,7 @@ static int vprotocol_pessimist_request_no_free(ompi_request_t **req) {
 }
 
 #define PREPARE_REQUESTS_WITH_NO_FREE(count, requests) do { \
-    size_t i; \
-    for(i = 0; i < count; i++) \
+    for (size_t i = 0; i < count; i++)     \
     { \
         if(requests[i] == MPI_REQUEST_NULL) continue; \
         requests[i]->req_free = vprotocol_pessimist_request_no_free; \
@@ -74,7 +74,6 @@ int mca_vprotocol_pessimist_test_any(size_t count, ompi_request_t ** requests,
                                      ompi_status_public_t * status)
 {
     int ret;
-    size_t i;
 
     VPROTOCOL_PESSIMIST_DELIVERY_REPLAY(count, requests, completed, index, status);
 
@@ -86,7 +85,7 @@ int mca_vprotocol_pessimist_test_any(size_t count, ompi_request_t ** requests,
 
     if(completed)
     {   /* Parse the result */
-        for(i = 0; i < count; i++)
+        for(size_t i = 0; i < count; i++)
         {
             ompi_request_t *req = requests[i];
             if(req == MPI_REQUEST_NULL) continue;
@@ -116,7 +115,6 @@ int mca_vprotocol_pessimist_wait_any(size_t count, ompi_request_t ** requests,
                                      int *index, ompi_status_public_t * status)
 {
     int ret;
-    size_t i;
     int dummy;
 
     VPROTOCOL_PESSIMIST_DELIVERY_REPLAY(count, requests, &dummy, index, status);
@@ -127,7 +125,7 @@ int mca_vprotocol_pessimist_wait_any(size_t count, ompi_request_t ** requests,
     ret = mca_pml_v.host_request_fns.req_wait_any(count, requests, index, status);
 
     /* Parse the result */
-    for(i = 0; i < count; i++)
+    for(size_t i = 0; i < count; i++)
     {
         ompi_request_t *req = requests[i];
         if(req == MPI_REQUEST_NULL) continue;
diff --git a/ompi/message/message.c b/ompi/message/message.c
index deb0a4697f9..9bbe5f3d1f5 100644
--- a/ompi/message/message.c
+++ b/ompi/message/message.c
@@ -6,6 +6,8 @@
  *                         reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -20,6 +22,7 @@
 #include "opal/class/opal_object.h"
 #include "ompi/message/message.h"
 #include "ompi/constants.h"
+#include "ompi/instance/instance.h"
 
 static void ompi_message_constructor(ompi_message_t *msg);
 
@@ -27,6 +30,8 @@ OBJ_CLASS_INSTANCE(ompi_message_t,
                    opal_free_list_item_t,
                    ompi_message_constructor, NULL);
 
+static int ompi_message_finalize (void);
+
 opal_free_list_t ompi_message_free_list = {{{0}}};
 opal_pointer_array_t  ompi_message_f_to_c_table = {{0}};
 
@@ -67,11 +72,12 @@ ompi_message_init(void)
         return OMPI_ERR_NOT_FOUND;
     }
 
+    ompi_mpi_instance_append_finalize (ompi_message_finalize);
+
     return rc;
 }
 
-int
-ompi_message_finalize(void)
+static int ompi_message_finalize (void)
 {
     OBJ_DESTRUCT(&ompi_message_no_proc);
     OBJ_DESTRUCT(&ompi_message_free_list);
diff --git a/ompi/message/message.h b/ompi/message/message.h
index 0f0f1eacfac..a0819ac5d7c 100644
--- a/ompi/message/message.h
+++ b/ompi/message/message.h
@@ -4,6 +4,8 @@
  * Copyright (c) 2012-2017 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
  *                         reserved.
+ * Copyright (c) 2018      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -34,7 +36,7 @@ typedef struct ompi_message_t ompi_message_t;
 OMPI_DECLSPEC OBJ_CLASS_DECLARATION(ompi_message_t);
 
 /**
- * Padded struct to maintain back compatibiltiy.
+ * Padded struct to maintain back compatibility.
  * See ompi/communicator/communicator.h comments with struct ompi_communicator_t
  * for full explanation why we chose the following padding construct for predefines.
  */
@@ -49,8 +51,6 @@ typedef struct ompi_predefined_message_t ompi_predefined_message_t;
 
 int ompi_message_init(void);
 
-int ompi_message_finalize(void);
-
 OMPI_DECLSPEC extern opal_free_list_t ompi_message_free_list;
 OMPI_DECLSPEC extern opal_pointer_array_t  ompi_message_f_to_c_table;
 OMPI_DECLSPEC extern ompi_predefined_message_t  ompi_message_no_proc;
diff --git a/ompi/mpi/Makefile.am b/ompi/mpi/Makefile.am
index 388076307c9..80ae278fde2 100644
--- a/ompi/mpi/Makefile.am
+++ b/ompi/mpi/Makefile.am
@@ -12,6 +12,7 @@
 # Copyright (c) 2006-2018 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2015      Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -19,7 +20,13 @@
 # $HEADER$
 #
 
-EXTRA_DIST = \
-        mpi/fortran/configure-fortran-output-bottom.h
-
-dist_ompidata_DATA += mpi/help-mpi-api.txt
+EXTRA_DIST += \
+        mpi/fortran/configure-fortran-output-bottom.h \
+        mpi/help-mpi-api.txt \
+        mpi/bindings/bindings.py \
+        mpi/bindings/ompi_bindings/consts.py \
+        mpi/bindings/ompi_bindings/c.py \
+        mpi/bindings/ompi_bindings/c_type.py \
+        mpi/bindings/ompi_bindings/fortran.py \
+        mpi/bindings/ompi_bindings/fortran_type.py \
+        mpi/bindings/ompi_bindings/util.py
diff --git a/ompi/mpi/bindings/bindings.py b/ompi/mpi/bindings/bindings.py
new file mode 100644
index 00000000000..951d651a74d
--- /dev/null
+++ b/ompi/mpi/bindings/bindings.py
@@ -0,0 +1,77 @@
+# Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+#                         reserved.
+#
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+"""Main binding generation script.
+
+This script is used for generating the bindings for both C and Fortran. See the
+scripts in 'ompi_bindings/' for the bulk of the code.
+"""
+import argparse
+import os
+import sys
+
+
+def handle_missing_command(args, out):
+    print('missing subcommand (one of {fortran,c} required)', file=sys.stderr)
+    sys.exit(1)
+
+
+def main():
+    parser = argparse.ArgumentParser(description='generate fortran binding files')
+    parser.add_argument('--builddir', required=True, help='absolute path to automake builddir (abs_top_builddir)')
+    parser.add_argument('--output', required=True, help='output file to use')
+    parser.add_argument('--srcdir', required=True, help='absolute path to automake srcdir (abs_top_srcdir)')
+    parser.set_defaults(handler=handle_missing_command)
+    subparsers = parser.add_subparsers()
+
+    # Fortran set up code
+    parser_fortran = subparsers.add_parser('fortran', help='subcommand for generating Fortran code')
+    parser_fortran.add_argument('--generate-ts-suffix', action="store_true",
+                                help='generate ts suffixes for appropriate routines')
+    # Handler for generating actual code
+    subparsers_fortran = parser_fortran.add_subparsers()
+    parser_code = subparsers_fortran.add_parser('code', help='generate binding code')
+    parser_code.set_defaults(handler=lambda args, out: fortran.generate_code(args, out))
+    parser_code.add_argument('--lang', choices=('fortran', 'c'),
+                             help='language to generate (only for code subparser)')
+    # Handler for generating the Fortran interface files
+    parser_interface = subparsers_fortran.add_parser('interface',
+                                                     help='generate Fortran interface specifications')
+    parser_interface.set_defaults(handler=lambda args, out: fortran.generate_interface(args, out))
+    # The prototype files argument must come last and be specified for both subparsers
+    for f_subparser in [parser_code, parser_interface]:
+        f_subparser.add_argument('--prototype-files', nargs='+', help='prototype files to generate code for')
+
+    # C set up code
+    parser_c = subparsers.add_parser('c', help='subcommand for generating C code')
+    subparsers_c = parser_c.add_subparsers()
+    parser_header = subparsers_c.add_parser('header', help='generate header file from template files')
+    parser_header.add_argument('file', nargs='+', help='list of template source files')
+    parser_header.add_argument('--external', action='store_true', help='generate external mpi.h header file')
+    parser_header.add_argument('--srcdir', help='source directory')
+    parser_header.set_defaults(handler=lambda args, out: c.generate_header(args, out))
+    parser_gen = subparsers_c.add_parser('source', help='generate source file from template file')
+    # parser = argparse.ArgumentParser(description='C ABI binding generation code')
+    parser_gen.add_argument('type', choices=('ompi', 'standard'),
+                            help='generate the OMPI ABI functions or the standard ABI functions')
+    parser_gen.add_argument('source_file', help='template file to use for C code generation')
+    parser_gen.set_defaults(handler=lambda args, out: c.generate_source(args, out))
+    args = parser.parse_args()
+
+    # Pull in both generated python files and src files on import
+    sys.path.insert(0, os.path.join(args.builddir, 'ompi/mpi/bindings'))
+    sys.path.insert(0, os.path.join(args.srcdir, 'ompi/mpi/bindings'))
+    from ompi_bindings import c, fortran
+    from ompi_bindings.util import OutputFile
+
+    with open(args.output, 'w') as f:
+        args.handler(args, OutputFile(f))
+
+
+if __name__ == '__main__':
+    main()
diff --git a/ompi/mpi/bindings/ompi_bindings/c.py b/ompi/mpi/bindings/ompi_bindings/c.py
new file mode 100644
index 00000000000..362f481d11b
--- /dev/null
+++ b/ompi/mpi/bindings/ompi_bindings/c.py
@@ -0,0 +1,386 @@
+# Copyright (c) 2024      Triad National Security, LLC. All rights reserved.
+# Copyright (c) 2023      Research Organization for Information Science
+#                         and Technology (RIST).  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADERS$
+#
+#
+"""MPI C Binding Code.
+
+This file is used for generating C bindings, as well as bigcount interfaces,
+from individual *.c.in template files. This also currently includes unused ABI
+code, in preparation for the standard ABI.
+
+TEMPLATE SOURCE FILE ASSUMPTIONS:
+* Only one function per file
+* Nothing (other than blank lines) after closing '}'
+* Function prototype is preceded by PROTOTYPE
+* All types in the function prototype are converted to one-word capital types
+  as defined here (to be later converted to ompi or standard ABI types)
+* Functions requiring a bigcount implementation should have type COUNT in
+  place of MPI_Count or int for each count parameter. Bigcount functions will
+  be generated automatically for any function that includes a COUNT type.
+"""
+from abc import ABC, abstractmethod
+import argparse
+import re
+import sys
+import os
+from ompi_bindings import consts, util
+from ompi_bindings.consts import ConvertFuncs, ConvertOMPIToStandard
+from ompi_bindings.c_type import Type
+from ompi_bindings.parser import SourceTemplate
+
+
+class ABIHeaderBuilder:
+    """ABI header builder code."""
+
+    def __init__(self, prototypes, out, external=False):
+        self.out = out
+        self.external = external
+
+        if external:
+            mangle_name = lambda name: name
+        else:
+            mangle_name = util.abi_internal_name
+
+        # Build up the list of standard ABI signatures
+        signatures = []
+        for prototype in prototypes:
+            base_name = util.mpi_fn_name_from_base_fn_name(prototype.name)
+            signatures.append(prototype.signature(base_name, abi_type='standard',
+                                                  mangle_name=mangle_name))
+            # Profiling prototype
+            signatures.append(prototype.signature(f'P{base_name}', abi_type='standard',
+                                                  mangle_name=mangle_name))
+            if util.prototype_has_bigcount(prototype):
+                signatures.append(prototype.signature(f'{base_name}_c', abi_type='standard',
+                                                      enable_count=True,
+                                                      mangle_name=mangle_name))
+                # Profiling prototype
+                signatures.append(prototype.signature(f'P{base_name}_c', abi_type='standard',
+                                                      enable_count=True,
+                                                      mangle_name=mangle_name))
+        self.signatures = signatures
+
+    def mangle_name(self, extname):
+        """Mangle names, depending on whether building external or internal header."""
+        if self.external:
+            return extname
+        return util.abi_internal_name(extname)
+
+    def dump(self, *pargs, **kwargs):
+        self.out.dump(*pargs, **kwargs)
+
+    def dump_lines(self, lines):
+        lines = util.indent_lines(lines, 4 * ' ', start=1)
+        for line in lines:
+            self.dump(line)
+
+    def generate_error_convert_fn(self):
+        self.dump(f'{consts.INLINE_ATTRS} int {ConvertFuncs.ERROR_CLASS}(int error_class)')
+        self.dump('{')
+        lines = []
+        lines.append('switch (error_class) {')
+        for error in consts.ERROR_CLASSES:
+            lines.append(f'case {self.mangle_name(error)}:')
+            lines.append(f'return {error};')
+        lines.append('default:')
+        lines.append('return error_class;')
+        lines.append('}')
+        self.dump_lines(lines)
+        self.dump('}')
+
+    def generic_convert(self, fn_name, param_name, type_, value_names):
+        intern_type = self.mangle_name(type_)
+        self.dump(f'{consts.INLINE_ATTRS} {type_} {fn_name}({intern_type} {param_name})')
+        self.dump('{')
+        lines = []
+        for i, value_name in enumerate(value_names):
+            intern_name = self.mangle_name(value_name)
+            if i == 0:
+                lines.append('if (%s == %s) {' % (intern_name, param_name))
+            else:
+                lines.append('} else if (%s == %s) {' % (intern_name, param_name))
+            lines.append(f'return {value_name};')
+        lines.append('}')
+        lines.append(f'return ({type_}) {param_name};')
+        self.dump_lines(lines)
+        self.dump('}')
+
+    def generic_convert_reverse(self, fn_name, param_name, type_, value_names):
+        intern_type = self.mangle_name(type_)
+        self.dump(f'{consts.INLINE_ATTRS} {intern_type} {fn_name}({type_} {param_name})')
+        self.dump('{')
+        lines = []
+        for i, value_name in enumerate(value_names):
+            intern_name = self.mangle_name(value_name)
+            if i == 0:
+                lines.append('if (%s == %s) {' % (value_name, param_name))
+            else:
+                lines.append('} else if (%s == %s) {' % (value_name, param_name))
+            lines.append(f'return {intern_name};')
+        lines.append('}')
+        lines.append(f'return ({intern_type}) {param_name};')
+        self.dump_lines(lines)
+        self.dump('}')
+
+    def generate_comm_convert_fn(self):
+        self.generic_convert(ConvertFuncs.COMM, 'comm', 'MPI_Comm', consts.RESERVED_COMMUNICATORS)
+
+    def generate_comm_convert_fn_intern_to_abi(self):
+        self.generic_convert_reverse(ConvertOMPIToStandard.COMM, 'comm', 'MPI_Comm', consts.RESERVED_COMMUNICATORS)
+
+    def generate_info_convert_fn(self):
+        self.generic_convert(ConvertFuncs.INFO, 'info', 'MPI_Info', consts.RESERVED_INFOS)
+
+    def generate_file_convert_fn_intern_to_abi(self):
+        self.generic_convert_reverse(ConvertFuncs.FILE, 'file', 'MPI_File', consts.RESERVED_FILES)
+
+    def generate_datatype_convert_fn(self):
+        self.generic_convert(ConvertFuncs.DATATYPE, 'datatype', 'MPI_Datatype', consts.PREDEFINED_DATATYPES)
+
+    def generate_op_convert_fn(self):
+        self.generic_convert(ConvertFuncs.OP, 'op', 'MPI_Op', consts.COLLECTIVE_OPERATIONS)
+
+    def generate_win_convert_fn(self):
+        self.generic_convert(ConvertFuncs.WIN, 'win', 'MPI_Win', consts.RESERVED_WINDOWS)
+
+    def generate_pointer_convert_fn(self, type_, fn_name, constants):
+        abi_type = self.mangle_name(type_)
+        self.dump(f'{consts.INLINE_ATTRS} void {fn_name}({abi_type} *ptr)')
+        self.dump('{')
+        lines = []
+        for i, ompi_name in enumerate(constants):
+            abi_name = self.mangle_name(ompi_name)
+            if i == 0:
+                lines.append('if (%s == (%s) *ptr) {' % (ompi_name, type_))
+            else:
+                lines.append('} else if (%s == (%s) *ptr) {' % (ompi_name, type_))
+            lines.append(f'*ptr = {abi_name};')
+        lines.append('}')
+        self.dump_lines(lines)
+        self.dump('}')
+
+    def generate_request_convert_fn(self):
+        self.generate_pointer_convert_fn('MPI_Request', ConvertFuncs.REQUEST, consts.RESERVED_REQUESTS)
+
+    def generate_file_convert_fn(self):
+        self.generate_pointer_convert_fn('MPI_File', ConvertFuncs.FILE, consts.RESERVED_FILES)
+
+    def generate_status_convert_fn(self):
+        type_ = 'MPI_Status'
+        abi_type = self.mangle_name(type_)
+        self.dump(f'{consts.INLINE_ATTRS} void {ConvertFuncs.STATUS}({abi_type} *out, {type_} *inp)')
+        self.dump('{')
+        self.dump('    out->MPI_SOURCE = inp->MPI_SOURCE;')
+        self.dump('    out->MPI_TAG = inp->MPI_TAG;')
+        self.dump(f'    out->MPI_ERROR = {ConvertFuncs.ERROR_CLASS}(inp->MPI_ERROR);')
+        # Ignoring the private fields for now
+        self.dump('}')
+
+    def define(self, type_, name, value):
+        self.dump(f'#define {name} OMPI_CAST_CONSTANT({type_}, {value})')
+
+    def define_all(self, type_, constants):
+        for i, const in enumerate(constants):
+            self.define(self.mangle_name(type_), self.mangle_name(const), i + 1)
+        self.dump()
+
+    def dump_header(self):
+        header_guard = '_ABI_INTERNAL_'
+        self.dump(f'#ifndef {header_guard}')
+        self.dump(f'#define {header_guard}')
+
+        self.dump('#include "stddef.h"')
+        self.dump('#include "stdint.h"')
+
+        self.dump("""
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+""")
+
+        self.dump("""
+#if defined(c_plusplus) || defined(__cplusplus)
+#define OMPI_CAST_CONSTANT(type, value) (static_cast<type> (static_cast<void *> (value)))
+#else
+#define OMPI_CAST_CONSTANT(type, value) ((type) ((void *) value))
+#endif
+""")
+
+        for i, err in enumerate(consts.ERROR_CLASSES):
+            self.dump(f'#define {self.mangle_name(err)} {i + 1}')
+        self.dump()
+
+        self.define_all('MPI_Datatype', consts.PREDEFINED_DATATYPES)
+        self.define_all('MPI_Op', COLLECTIVE_OPERATIONS)
+        self.define_all('MPI_Comm', consts.RESERVED_COMMUNICATORS)
+        self.define_all('MPI_Request', consts.RESERVED_REQUESTS)
+        self.define_all('MPI_Win', consts.RESERVED_WINDOWS)
+        self.define_all('MPI_Info', consts.RESERVED_INFOS)
+        self.define_all('MPI_File', consts.RESERVED_FILES)
+
+        for name, value in consts.VARIOUS_CONSTANTS.items():
+            self.dump(f'#define {self.mangle_name(name)} {value}')
+        self.dump()
+
+        status_type = self.mangle_name('MPI_Status')
+        for i, name in enumerate(consts.IGNORED_STATUS_HANDLES):
+            self.define(f'{status_type} *', self.mangle_name(name), i + 1)
+        self.dump()
+
+        for i, name in enumerate(consts.COMMUNICATOR_SPLIT_TYPES):
+            self.dump(f'#define {self.mangle_name(name)} {i}')
+        self.dump()
+
+        for mpi_type, c_type in consts.C_OPAQUE_TYPES.items():
+            self.dump(f'typedef {c_type} {self.mangle_name(mpi_type)};')
+        self.dump()
+
+        for handle in consts.C_HANDLES:
+            prefix, suffix = handle.split('_')
+            name = f'{prefix}_ABI_{suffix}'
+            self.dump(f'typedef struct {self.mangle_name(name)} *{self.mangle_name(handle)};')
+        self.dump()
+        self.dump("""
+struct MPI_Status_ABI {
+    int MPI_SOURCE;
+    int MPI_TAG;
+    int MPI_ERROR;
+    int mpi_abi_private[5];
+};""")
+        self.dump(f'typedef struct MPI_Status_ABI {self.mangle_name("MPI_Status")};')
+        self.dump()
+        # Function signatures
+        for sig in self.signatures:
+            self.dump(f'{sig};')
+        self.dump('int MPI_Abi_details(int *buflen, char *details, MPI_Info *info);')
+        self.dump('int MPI_Abi_supported(int *flag);')
+        self.dump('int MPI_Abi_version(int *abi_major, int *abi_minor);')
+        if not self.external:
+            # Now generate the conversion code
+            self.generate_error_convert_fn()
+            self.generate_comm_convert_fn()
+            self.generate_comm_convert_fn_intern_to_abi()
+            self.generate_info_convert_fn()
+            self.generate_file_convert_fn()
+            self.generate_datatype_convert_fn()
+            self.generate_op_convert_fn()
+            self.generate_win_convert_fn()
+            self.generate_request_convert_fn()
+            self.generate_status_convert_fn()
+
+        self.dump("""
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+""")
+        self.dump(f'#endif /* {header_guard} */')
+
+
+def print_profiling_header(fn_name, out):
+    """Print the profiling header code."""
+    out.dump('#if OMPI_BUILD_MPI_PROFILING')
+    out.dump('#if OPAL_HAVE_WEAK_SYMBOLS')
+    out.dump(f'#pragma weak {fn_name} = P{fn_name}')
+    out.dump('#endif')
+    out.dump(f'#define {fn_name} P{fn_name}')
+    out.dump('#endif')
+
+
+def print_cdefs_for_bigcount(fn_name, out, enable_count=False):
+    if enable_count:
+        out.dump('#undef OMPI_BIGCOUNT_SRC')
+        out.dump('#define OMPI_BIGCOUNT_SRC 1')
+    else:
+        out.dump('#undef OMPI_BIGCOUNT_SRC')
+        out.dump('#define OMPI_BIGCOUNT_SRC 0')
+
+def ompi_abi(base_name, template, out):
+    """Generate the OMPI ABI functions."""
+    template.print_header(out)
+    print_profiling_header(base_name, out)
+    print_cdefs_for_bigcount(base_name, out)
+    out.dump(template.prototype.signature(base_name, abi_type='ompi'))
+    template.print_body(func_name=base_name, out=out)
+    # Check if we need to generate the bigcount interface
+    if util.prototype_has_bigcount(template.prototype):
+        base_name_c = f'{base_name}_c'
+        print_profiling_header(base_name_c, out)
+        print_cdefs_for_bigcount(base_name_c, out, enable_count=True)
+        out.dump(template.prototype.signature(base_name_c, abi_type='ompi', enable_count=True))
+        template.print_body(func_name=base_name_c, out=out)
+
+
+ABI_INTERNAL_HEADER = 'ompi/mpi/c/abi.h'
+
+
+def standard_abi(base_name, template, out):
+    """Generate the standard ABI functions."""
+    template.print_header(out)
+    out.dump(f'#include "{ABI_INTERNAL_HEADER}"')
+
+    # Static internal function (add a random component to avoid conflicts)
+    internal_name = f'ompi_abi_{template.prototype.name}'
+    internal_sig = template.prototype.signature(internal_name, abi_type='ompi',
+                                                enable_count=True)
+    out.dump(consts.INLINE_ATTRS, internal_sig)
+    template.print_body(func_name=base_name, out=out)
+
+    def generate_function(prototype, fn_name, internal_fn, enable_count=False):
+        """Generate a function for the standard ABI."""
+        print_profiling_header(fn_name)
+        print_cdefs_for_bigcount(fn_name,enable_count)
+
+        # Handle type conversions and arguments
+        params = [param.construct(abi_type='standard') for param in prototype.params]
+        out.dump(prototype.signature(fn_name, abi_type='standard', enable_count=enable_count))
+        out.dump('{')
+        lines = []
+        return_type = prototype.return_type.construct(abi_type='standard')
+        lines.append(f'{return_type.tmp_type_text()} ret_value;')
+        for param in params:
+            if param.init_code:
+                lines.extend(param.init_code)
+        pass_args = ', '.join(param.argument for param in params)
+        lines.append(f'ret_value = {internal_fn}({pass_args});')
+        for param in params:
+            if param.final_code:
+                lines.extend(param.final_code)
+        lines.extend(return_type.return_code('ret_value'))
+
+        # Indent the lines
+        lines = util.indent_lines(lines, 4 * ' ', start=1)
+        for line in lines:
+            out.dump(line)
+        out.dump('}')
+
+    generate_function(template.prototype, base_name, internal_name)
+    if util.prototype_has_bigcount(template.prototype):
+        base_name_c = f'{base_name}_c'
+        generate_function(template.prototype, base_name_c, internal_name,
+                          enable_count=True)
+
+
+def generate_header(args, out):
+    """Generate an ABI header and conversion code."""
+    out.dump(f'/* {consts.GENERATED_MESSAGE} */')
+    prototypes = [SourceTemplate.load(file_, args.srcdir, type_constructor=Type.construct).prototype
+                  for file_ in args.file]
+    builder = ABIHeaderBuilder(prototypes, out, external=args.external)
+    builder.dump_header()
+
+
+def generate_source(args, out):
+    """Generate source file."""
+    out.dump(f'/* {consts.GENERATED_MESSAGE} */')
+    template = SourceTemplate.load(args.source_file, type_constructor=Type.construct)
+    base_name = util.mpi_fn_name_from_base_fn_name(template.prototype.name)
+    if args.type == 'ompi':
+        ompi_abi(base_name, template, out)
+    else:
+        standard_abi(base_name, template, out)
diff --git a/ompi/mpi/bindings/ompi_bindings/c_type.py b/ompi/mpi/bindings/ompi_bindings/c_type.py
new file mode 100644
index 00000000000..532dfb88e37
--- /dev/null
+++ b/ompi/mpi/bindings/ompi_bindings/c_type.py
@@ -0,0 +1,1237 @@
+# Copyright (c) 2024      Triad National Security, LLC. All rights
+#                         reserved.
+#
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+"""C type definitions."""
+from abc import ABC, abstractmethod
+from ompi_bindings.consts import ConvertFuncs, ConvertOMPIToStandard
+
+
+class Type(ABC):
+    """Type representation."""
+
+    PARAMS_OMPI_ABI = {}
+
+    PARAMS_STANDARD_ABI = {}
+
+    def __init__(self, type_name, name=None,
+                 mangle_name=lambda name: abi_internal_name(name),
+                 count_param=None, **kwargs):
+        self.type = type_name
+        self.name = name
+        self.count_param = count_param
+        self.mangle_name = mangle_name
+
+    @staticmethod
+    def construct(abi_type, type_name, **kwargs):
+        """Construct the parameter for the given ABI and type."""
+        if abi_type == 'ompi':
+            return Type.PARAMS_OMPI_ABI[type_name](type_name, **kwargs)
+        elif abi_type == 'standard':
+            return Type.PARAMS_STANDARD_ABI[type_name](type_name, **kwargs)
+        else:
+            raise RuntimeError(f'invalid ABI type {abi_type}')
+
+    @staticmethod
+    def add_type(type_name, abi_type=('ompi', 'standard')):
+        """Add a new class corresponding to a type."""
+        def wrapper(class_):
+            if 'ompi' in abi_type:
+                Type.PARAMS_OMPI_ABI[type_name] = class_
+            if 'standard' in abi_type:
+                Type.PARAMS_STANDARD_ABI[type_name] = class_
+            return class_
+        return wrapper
+
+    @property
+    def is_count(self):
+        """Return True if this parameter is a count (requiring bigcount API)."""
+        return False
+
+    @property
+    def init_code(self):
+        """Return the initialization code needed for an ABI wrapper."""
+        return []
+
+    @property
+    def final_code(self):
+        """Return the finalization code needed for an ABI wrapper."""
+        return []
+
+    def return_code(self, name):
+        """Process a value and then build up a return statement."""
+        return [f'return {name};']
+
+    @property
+    def argument(self):
+        """Return the argument text required for passing an argument to a function."""
+        return self.name
+
+    @abstractmethod
+    def type_text(self, enable_count=False):
+        """Return the source text corresponding to a type definition."""
+
+    def tmp_type_text(self, enable_count=False):
+        """Return source text corresponding to a temporary type definition before conversion."""
+        return self.type_text(enable_count=enable_count)
+
+    def parameter(self, enable_count=False, **kwargs):
+        return f'{self.type_text(enable_count=enable_count)} {self.name}'
+
+
+@Type.add_type('ERROR_CLASS')
+class TypeErrorClass(Type):
+
+    def type_text(self, enable_count=False):
+        return 'int'
+
+    def return_code(self, name):
+        return [f'return {ConvertFuncs.ERROR_CLASS}({name});']
+
+
+@Type.add_type('BUFFER')
+class TypeBuffer(Type):
+
+    def type_text(self, enable_count=False):
+        return 'const void *'
+
+
+@Type.add_type('BUFFER_OUT')
+class TypeBufferOut(Type):
+
+    def type_text(self, enable_count=False):
+        return f'void *'
+
+
+@Type.add_type('COUNT')
+class TypeCount(Type):
+
+    @property
+    def is_count(self):
+        return True
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Count' if enable_count else 'int'
+
+
+@Type.add_type('COUNT_ARRAY')
+class TypeCountArray(Type):
+    """Array of counts (either int or MPI_Count)."""
+
+    @property
+    def is_count(self):
+        return True
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Count *' if enable_count else 'int *'
+
+    def parameter(self, enable_count=False, **kwargs):
+        count_type = 'MPI_Count' if enable_count else 'int'
+        return f'const {count_type} {self.name}[]'
+
+@Type.add_type('AINT_COUNT_ARRAY')
+class TypeAintCountArray(Type):
+    """Array of counts (either MPI_Aint or MPI_Count)."""
+
+    @property
+    def is_count(self):
+        return True
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Count *' if enable_count else 'MPI_Aint *'
+
+    def parameter(self, enable_count=False, **kwargs):
+        count_type = 'MPI_Count' if enable_count else 'MPI_Aint'
+        return f'const {count_type} {self.name}[]'
+
+@Type.add_type('ELEMENT_COUNT')
+class ElementCountType(Type):
+    """Special count type for MPI_Get_element_x"""
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Count *'
+
+
+@Type.add_type('PARTITIONED_COUNT')
+class TypePartitionedCount(Type):
+    """Count type for partitioned communication functions."""
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Count'
+
+
+@Type.add_type('DISP')
+class TypeDisp(Type):
+
+    @property
+    def is_count(self):
+        return True
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Aint' if enable_count else 'int'
+
+
+@Type.add_type('DISP_ARRAY')
+class TypeDispArray(Type):
+
+    @property
+    def is_count(self):
+        return True
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Aint *' if enable_count else 'int *'
+
+    def parameter(self, enable_count=False, **kwargs):
+        count_type = 'MPI_Aint' if enable_count else 'int'
+        return f'const {count_type} {self.name}[]'
+
+
+@Type.add_type('INT')
+class TypeInt(Type):
+
+    def type_text(self, enable_count=False):
+        return 'int'
+
+
+@Type.add_type('AINT')
+class TypeAint(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Aint'
+
+
+@Type.add_type('AINT_OUT')
+class TypeAintOut(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Aint *'
+
+
+@Type.add_type('AINT_ARRAY')
+class TypeAintArray(Type):
+
+    def type_text(self, enable_count=False):
+        return 'const MPI_Aint *'
+
+    def parameter(self, enable_count=False, **kwargs):
+        return f'const MPI_Aint {self.name}[]'
+
+
+@Type.add_type('INT_OUT')
+class TypeIntOut(Type):
+
+    def type_text(self, enable_count=False):
+        return 'int *'
+
+    def parameter(self, enable_count=False, **kwargs):
+        if self.count_param is None:
+            return f'int *{self.name}'
+        else:
+            return f'int {self.name}[]'
+
+
+@Type.add_type('COUNT_OUT')
+class TypeCountOut(Type):
+
+    @property
+    def is_count(self):
+        return True
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Count *' if enable_count else 'int *'
+
+
+@Type.add_type('AINT_COUNT')
+class TypeAintCountOut(Type):
+
+    @property
+    def is_count(self):
+        return True
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Count' if enable_count else 'MPI_Aint'
+
+
+@Type.add_type('AINT_COUNT_OUT')
+class TypeAintCountOut(Type):
+
+    @property
+    def is_count(self):
+        return True
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Count *' if enable_count else 'MPI_Aint *'
+
+
+@Type.add_type('INT_ARRAY')
+class TypeIntArray(Type):
+
+    def type_text(self, enable_count=False):
+        return 'const int *'
+
+    def parameter(self, enable_count=False, **kwargs):
+        return f'const int {self.name}[]'
+
+@Type.add_type('INT_AINT_OUT')
+class TypeIntAintOut(Type):
+
+    @property
+    def is_count(self):
+        return True
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Aint *' if enable_count else 'int *'
+
+@Type.add_type('RANGE_ARRAY')
+class TypeRangeArray(Type):
+
+    def type_text(self, enable_count=False):
+        return 'int *'
+
+    def parameter(self, enable_count=False, **kwargs):
+        return f'int {self.name}[][3]'
+
+
+@Type.add_type('OFFSET')
+class TypeOffset(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Offset'
+
+
+@Type.add_type('OFFSET_OUT')
+class TypeOffsetOut(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Offset *'
+
+
+@Type.add_type('DOUBLE')
+class TypeDouble(Type):
+
+    def type_text(self, enable_count=False):
+        return 'double'
+
+
+@Type.add_type('ARGV')
+class TypeArgv(Type):
+
+    def type_text(self, enable_count=False):
+        return 'char ***'
+
+
+@Type.add_type('STRING_ARRAY')
+class TypeStringArray(Type):
+
+    def type_text(self, enable_count=False):
+        return 'char **'
+
+    def parameter(self, enable_count=False, **kwargs):
+        return f'char *{self.name}[]'
+
+
+@Type.add_type('DATATYPE', abi_type=['ompi'])
+class TypeDatatype(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Datatype'
+
+
+@Type.add_type('DATATYPE_ARRAY', abi_type=['ompi'])
+class TypeDatatypeArray(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Datatype'
+
+    def parameter(self, enable_count=False, **kwargs):
+        return f'const {self.type_text(enable_count=enable_count)} {self.name}[]'
+
+
+class StandardABIType(Type):
+
+    @property
+    def tmpname(self):
+        return f'{self.name}_tmp'
+
+    @property
+    def argument(self):
+        return self.tmpname
+
+
+@Type.add_type('DATATYPE', abi_type=['standard'])
+class TypeDatatypeStandard(StandardABIType):
+
+    @property
+    def init_code(self):
+        return [f'MPI_Datatype {self.tmpname} = {ConvertFuncs.DATATYPE}({self.name});']
+
+    def type_text(self, enable_count=False):
+        return self.mangle_name('MPI_Datatype')
+
+
+@Type.add_type('DATATYPE_OUT', abi_type=['ompi'])
+class TypeDatatypeOut(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Datatype *'
+
+
+@Type.add_type('DATATYPE_OUT', abi_type=['standard'])
+class TypeDatatypeStandard(Type):
+
+    @property
+    def final_code(self):
+        return [f'*{self.name} = {ConvertOMPIToStandard.DATATYPE}((MPI_Datatype) *{self.name});']
+
+    def type_text(self, enable_count=False):
+        type_name = self.mangle_name('MPI_Datatype')
+        return f'{type_name} *'
+
+    @property
+    def argument(self):
+        return f'(MPI_Datatype *) {self.name}'
+
+
+@Type.add_type('OP', abi_type=['ompi'])
+class TypeDatatype(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Op'
+
+
+@Type.add_type('OP', abi_type=['standard'])
+class TypeDatatype(StandardABIType):
+
+    @property
+    def init_code(self):
+        return [f'MPI_Op {self.tmpname} = {ConvertFuncs.OP}({self.name});']
+
+    def type_text(self, enable_count=False):
+        return self.mangle_name('MPI_Op')
+
+
+@Type.add_type('OP_OUT', abi_type=['ompi'])
+class TypeOpOut(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Op *'
+
+
+@Type.add_type('OP_OUT', abi_type=['standard'])
+class TypeOpOutStandard(Type):
+
+    @property
+    def final_code(self):
+        return [f'*{self.name} = {ConvertOMPIToStandard.OP}((MPI_Op) *{self.name});']
+
+    def type_text(self, enable_count=False):
+        type_name = self.mangle_name('MPI_Op')
+        return f'{type_name} *'
+
+    @property
+    def argument(self):
+        return f'(MPI_Op *) {self.name}'
+
+
+@Type.add_type('RANK')
+class TypeRank(Type):
+
+    def type_text(self, enable_count=False):
+        return 'int'
+
+
+@Type.add_type('TAG')
+class TypeRank(Type):
+
+    def type_text(self, enable_count=False):
+        return 'int'
+
+
+@Type.add_type('COMM', abi_type=['ompi'])
+class TypeCommunicator(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Comm'
+
+
+@Type.add_type('COMM', abi_type=['standard'])
+class TypeCommunicatorStandard(StandardABIType):
+
+    @property
+    def init_code(self):
+        return [f'MPI_Comm {self.tmpname} = {ConvertFuncs.COMM}({self.name});']
+
+    def tmp_type_text(self, enable_count=False):
+        return 'MPI_Comm'
+
+    def return_code(self, name):
+        return [f'return {ConvertOMPIToStandard.COMM}({name});']
+
+    def type_text(self, enable_count=False):
+        return self.mangle_name('MPI_Comm')
+
+
+@Type.add_type('COMM_OUT', abi_type=['ompi'])
+class TypeCommunicator(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Comm *'
+
+
+@Type.add_type('COMM_OUT', abi_type=['standard'])
+class TypeCommunicator(Type):
+
+    @property
+    def final_code(self):
+        return [f'*{self.name} = {ConvertOMPIToStandard.COMM}((MPI_Comm) *{self.name});']
+
+    def type_text(self, enable_count=False):
+        type_name = self.mangle_name('MPI_Comm')
+        return f'{type_name} *'
+
+    @property
+    def argument(self):
+        return f'(MPI_Comm *) {self.name}'
+
+
+@Type.add_type('WIN', abi_type=['ompi'])
+class TypeWindow(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Win'
+
+
+@Type.add_type('WIN', abi_type=['standard'])
+class TypeWindowStandard(StandardABIType):
+
+    @property
+    def init_code(self):
+        return [f'MPI_Win {self.tmpname} = {ConvertFuncs.WIN}({self.name});']
+
+    def type_text(self, enable_count=False):
+        return self.mangle_name('MPI_Win')
+
+
+@Type.add_type('WIN_OUT', abi_type=['ompi'])
+class TypeWindowOut(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Win *'
+
+
+@Type.add_type('WIN_OUT', abi_type=['standard'])
+class TypeWindowOutStandard(StandardABIType):
+
+    @property
+    def final_code(self):
+        return [f'*{self.name} = {ConvertOMPIToStandard.WIN}((MPI_Win) *{self.name});']
+
+    def type_text(self, enable_count=False):
+        type_name = self.mangle_name('MPI_Win')
+        return f'{type_name} *'
+
+    @property
+    def argument(self):
+        return f'(MPI_Win *) {self.name}'
+
+
+@Type.add_type('REQUEST', abi_type=['ompi'])
+class TypeRequest(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Request'
+
+
+@Type.add_type('REQUEST', abi_type=['standard'])
+class TypeRequestStandard(Type):
+
+    def type_text(self, enable_count=False):
+        return self.mangle_name('MPI_Request')
+
+    @property
+    def argument(self):
+        return f'(MPI_Request) {self.name}'
+
+
+@Type.add_type('REQUEST_INOUT', abi_type=['ompi'])
+class TypeRequestInOut(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Request *'
+
+
+@Type.add_type('REQUEST_INOUT', abi_type=['standard'])
+class TypeRequestInOutStandard(Type):
+
+    @property
+    def final_code(self):
+        if self.count_param is None:
+            return [f'{ConvertFuncs.REQUEST}({self.name});']
+        else:
+            return [
+                'for (int i = 0; i < %s; ++i) {' % (self.count_param,),
+                f'{ConvertFuncs.REQUEST}(&{self.name}[i]);',
+                '}',
+            ]
+
+    @property
+    def argument(self):
+        return f'(MPI_Request *) {self.name}'
+
+    def type_text(self, enable_count=False):
+        type_name = self.mangle_name('MPI_Request')
+        return f'{type_name} *'
+
+    def parameter(self, enable_count=False, **kwargs):
+        type_name = self.mangle_name('MPI_Request')
+        if self.count_param is None:
+            return f'{type_name} *{self.name}'
+        else:
+            return f'{type_name} {self.name}[]'
+
+
+@Type.add_type('STATUS', abi_type=['ompi'])
+class TypeStatus(Type):
+
+    def type_text(self, enable_count=False):
+        return 'const MPI_Status *'
+
+
+@Type.add_type('STATUS', abi_type=['standard'])
+class TypeStatusStandard(StandardABIType):
+
+    @property
+    def init_code(self):
+        # TODO: Need to ensure this is the correct conversion function for MPI_Status
+        return [f'{ConvertFuncs.STATUS}({self.name}, &{self.tmpname});']
+
+    def typ_text(self, enable_count=False):
+        type_name = self.mangle_name('MPI_Status')
+        return f'const {type_name} *'
+
+
+@Type.add_type('STATUS_OUT', abi_type=['ompi'])
+class TypeStatusOut(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Status *'
+
+    def parameter(self, enable_count=False, **kwargs):
+        if self.count_param is None:
+            return f'MPI_Status *{self.name}'
+        else:
+            return f'MPI_Status {self.name}[]'
+
+
+@Type.add_type('STATUS_OUT', abi_type=['standard'])
+class TypeStausOutStandard(StandardABIType):
+
+    def if_should_set_status(self):
+        """Generate the condition to check if the status(es) should be set."""
+        condition = ' && '.join(f'{self.mangle_name(const)} != {self.name}'
+                                for const in IGNORED_STATUS_HANDLES)
+        return 'if (%s) {' % (condition,)
+
+    @property
+    def status_argument(self):
+        return f'{self.name}_arg'
+
+    @property
+    def init_code(self):
+        code = [f'MPI_Status *{self.status_argument} = NULL;']
+        if self.count_param is None:
+            code.append(f'MPI_Status {self.tmpname};')
+        else:
+            code.append(f'MPI_Status *{self.tmpname} = NULL;')
+        code.append(self.if_should_set_status())
+        if self.count_param is not None:
+            code.append(f'{self.tmpname} = malloc({self.count_param} * sizeof(MPI_Status));')
+            code.append(f'{self.status_argument} = {self.tmpname};')
+        else:
+            code.append(f'{self.status_argument} = &{self.tmpname};')
+        code.append('} else {')
+        if self.count_param is not None:
+            code.append(f'{self.status_argument} = MPI_STATUSES_IGNORE;')
+        else:
+            code.append(f'{self.status_argument} = MPI_STATUS_IGNORE;')
+        code.append('}')
+        return code
+
+    @property
+    def final_code(self):
+        code = [self.if_should_set_status()]
+        if self.count_param is None:
+            code.append(f'{ConvertFuncs.STATUS}({self.name}, &{self.tmpname});')
+        else:
+            code.extend([
+                'for (int i = 0; i < %s; ++i) {' % (self.count_param,),
+                f'{ConvertFuncs.STATUS}(&{self.name}[i], &{self.tmpname}[i]);',
+                '}',
+                f'free({self.tmpname});',
+            ])
+        code.append('}')
+        return code
+
+    @property
+    def argument(self):
+        return self.status_argument
+
+    def type_text(self, enable_count=False):
+        type_name = self.mangle_name('MPI_Status')
+        return f'{type_name} *'
+
+    def parameter(self, enable_count=False, **kwargs):
+        type_name = self.mangle_name('MPI_Status')
+        if self.count_param is None:
+            return f'{type_name} *{self.name}'
+        else:
+            return f'{type_name} {self.name}[]'
+
+
+@Type.add_type('F08_STATUS')
+class TypeF08Status(Type):
+
+    def type_text(self, enable_count=False):
+        return 'const MPI_F08_status *'
+
+
+@Type.add_type('F08_STATUS_OUT')
+class TypeF08StatusOut(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_F08_status *'
+
+
+# TODO: For now this just assumes that MPI_Fint doesn't need any conversions
+@Type.add_type('FINT')
+class TypeFint(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Fint'
+
+
+@Type.add_type('FINT_CONST')
+class TypeFintRef(Type):
+
+    def type_text(self, enable_count=False):
+        return 'const MPI_Fint *'
+
+
+@Type.add_type('FINT_OUT')
+class TypeFintOut(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Fint *'
+
+
+@Type.add_type('STRING')
+class TypeString(Type):
+
+    def type_text(self, enable_count=False):
+        return 'const char *'
+
+
+@Type.add_type('STRING_OUT')
+class TypeStringOut(Type):
+
+    def type_text(self, enable_count=False):
+        return 'char *'
+
+
+@Type.add_type('INFO', abi_type=['ompi'])
+class TypeInfo(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Info'
+
+
+@Type.add_type('INFO', abi_type=['standard'])
+class TypeInfoStandard(StandardABIType):
+
+    @property
+    def init_code(self):
+        return [f'MPI_Info {self.tmpname} = {ConvertFuncs.INFO}({self.name});']
+
+    def type_text(self, enable_count=False):
+        return self.mangle_name('MPI_Info')
+
+
+@Type.add_type('INFO_OUT', abi_type=['ompi'])
+class TypeInfoOut(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Info *'
+
+
+@Type.add_type('INFO_OUT', abi_type=['standard'])
+class TypeInfoOutStandard(Type):
+
+    @property
+    def argument(self):
+        return f'(MPI_Info *) {self.name}'
+
+    def type_text(self, enable_count=False):
+        type_name = self.mangle_name('MPI_Info')
+        return f'{type_name} *'
+
+
+@Type.add_type('INFO_ARRAY', abi_type=['ompi'])
+class TypeInfoArray(Type):
+
+    def type_text(self, enable_count=False):
+        return 'const MPI_Info *'
+
+    def parameter(self, enable_count=False, **kwargs):
+        return f'const MPI_Info {self.name}[]'
+
+
+@Type.add_type('INFO_ARRAY', abi_type=['standard'])
+class TypeInfoArray(Type):
+
+    def type_text(self, enable_count=False):
+        type_name = self.mangle_name('MPI_Info')
+        return f'const {type_name} *'
+
+    def parameter(self, enable_count=False, **kwargs):
+        type_name = self.mangle_name('MPI_Info')
+        return f'const {type_name} {self.name}[]'
+
+
+@Type.add_type('FILE', abi_type=['ompi'])
+class TypeFile(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_File'
+
+
+@Type.add_type('FILE', abi_type=['standard'])
+class TypeFileStandard(Type):
+
+    @property
+    def argument(self):
+        return f'(MPI_File) {self.name}'
+
+    def type_text(self, enable_count=False):
+        return self.mangle_name('MPI_File')
+
+
+@Type.add_type('FILE_OUT', abi_type=['ompi'])
+class TypeFileOut(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_File *'
+
+
+@Type.add_type('FILE_OUT', abi_type=['standard'])
+class TypeFileOutStandard(Type):
+
+    @property
+    def argument(self):
+        return f'(MPI_File *) {self.name}'
+
+    @property
+    def final_code(self):
+        return [f'{ConvertFuncs.FILE}({self.name});']
+
+    def type_text(self, enable_count=False):
+        type_name = self.mangle_name('MPI_File')
+        return f'{type_name} *'
+
+
+@Type.add_type('MESSAGE', abi_type=['ompi'])
+class TypeMessage(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Message'
+
+
+@Type.add_type('MESSAGE', abi_type=['standard'])
+class TypeMessageStandard(Type):
+
+    @property
+    def argument(self):
+        return f'(MPI_File) {self.name}'
+
+    def type_text(self, enable_count=False):
+        return self.mangle_name('MPI_Message')
+
+
+@Type.add_type('MESSAGE_OUT', abi_type=['ompi'])
+class TypeMessageOut(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Message *'
+
+
+@Type.add_type('MESSAGE_OUT', abi_type=['standard'])
+class TypeMessageOutStandard(Type):
+
+    @property
+    def argument(self):
+        return f'(MPI_Message *) {self.name}'
+
+    def type_text(self, enable_count=False):
+        type_name = self.mangle_name('MPI_Message')
+        return f'{type_name} *'
+
+
+@Type.add_type('COMM_ERRHANDLER_FUNCTION', abi_type=['ompi'])
+class TypeCommErrhandlerFunction(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Comm_errhandler_function *'
+
+
+@Type.add_type('COMM_ERRHANDLER_FUNCTION', abi_type=['standard'])
+class TypeCommErrhandlerFunctionStandard(Type):
+    # TODO: This may require a special function to wrap the calllback
+    pass
+
+
+@Type.add_type('FILE_ERRHANDLER_FUNCTION', abi_type=['ompi'])
+class TypeFileErrhandlerFunction(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_File_errhandler_function *'
+
+
+@Type.add_type('FILE_ERRHANDLER_FUNCTION', abi_type=['standard'])
+class TypeFileErrhandlerFunction(Type):
+    # TODO: This may require a special function to wrap the callback
+    pass
+
+
+@Type.add_type('COPY_FUNCTION', abi_type=['ompi'])
+class TypeCopyFunction(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Copy_function *'
+
+
+@Type.add_type('COPY_FUNCTION', abi_type=['standard'])
+class TypeCopyFunctionStandard(Type):
+    # TODO: This may require a special function to wrap the callback
+    pass
+
+
+@Type.add_type('DELETE_FUNCTION', abi_type=['ompi'])
+class TypeDeleteFunction(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Delete_function *'
+
+
+@Type.add_type('DELETE_FUNCTION', abi_type=['standard'])
+class TypeDeleteFunctionStandard(Type):
+    # TODO: This may require a special function to wrap the callback
+    pass
+
+
+@Type.add_type('USER_FUNCTION', abi_type=['ompi'])
+class TypeUserFunction(Type):
+
+    @property
+    def is_count(self):
+        return True
+
+    def type_text(self, enable_count=False):
+        return 'MPI_User_function_c *' if enable_count else 'MPI_User_function *'
+
+
+@Type.add_type('USER_FUNCTION', abi_type=['standard'])
+class TypeUserFunctionStandard(Type):
+    # TODO: This may require a special function to wrap the callback
+    pass
+
+
+@Type.add_type('COMM_COPY_ATTR_FUNCTION', abi_type=['ompi'])
+class TypeCommCopyAttrFunction(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Comm_copy_attr_function *'
+
+
+@Type.add_type('COMM_COPY_ATTR_FUNCTION', abi_type=['standard'])
+class TypeCommCopyAttrFunctionStandard(Type):
+    # TODO: This may require a special function to wrap the callback
+    pass
+
+
+@Type.add_type('COMM_DELETE_ATTR_FUNCTION', abi_type=['ompi'])
+class TypeCommDeleteAttrFunction(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Comm_delete_attr_function *'
+
+
+@Type.add_type('COMM_DELETE_ATTR_FUNCTION', abi_type=['standard'])
+class TypeCommDeleteAttrFunctionStandard(Type):
+    # TODO: This may require a special function to wrap the callback
+    pass
+
+
+@Type.add_type('GREQUEST_QUERY_FUNCTION', abi_type=['ompi'])
+class TypeGrequestQueryFunction(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Grequest_query_function *'
+
+
+@Type.add_type('GREQUEST_QUERY_FUNCTION', abi_type=['standard'])
+class TypeGrequestQueryFunctionStandard(Type):
+    # TODO: This may require a special function to wrap the callback
+    pass
+
+
+@Type.add_type('GREQUEST_FREE_FUNCTION', abi_type=['ompi'])
+class TypeGrequestFreeFunction(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Grequest_free_function *'
+
+
+@Type.add_type('GREQUEST_FREE_FUNCTION', abi_type=['standard'])
+class TypeGrequestFreeFunctionStandard(Type):
+    # TODO: This may require a special function to wrap the callback
+    pass
+
+
+@Type.add_type('GREQUEST_CANCEL_FUNCTION', abi_type=['ompi'])
+class TypeGrequestCancelFunction(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Grequest_cancel_function *'
+
+
+@Type.add_type('GREQUEST_CANCEL_FUNCTION', abi_type=['standard'])
+class TypeGrequestCancelFunctionStandard(Type):
+    # TODO: This may require a special function to wrap the callback
+    pass
+
+
+@Type.add_type('DATAREP_CONVERSION_FUNCTION', abi_type=['ompi'])
+class TypeDatarepConversionFunction(Type):
+
+    @property
+    def is_count(self):
+        return True
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Datarep_conversion_function_c *' if enable_count else 'MPI_Datarep_conversion_function *'
+
+@Type.add_type('DATAREP_CONVERSION_FUNCTION', abi_type=['standard'])
+class TypeDatarepConversionFunctionStandard(Type):
+    # TODO: This may require a special function to wrap the callback
+    pass
+
+
+@Type.add_type('DATAREP_EXTENT_FUNCTION', abi_type=['ompi'])
+class TypeDatarepExtentFunction(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Datarep_extent_function *'
+
+
+@Type.add_type('DATAREP_EXTENT_FUNCTION', abi_type=['standard'])
+class TypeDatarepExtentFunctionStandard(Type):
+    # TODO: This may require a special function to wrap the callback
+    pass
+
+
+@Type.add_type('SESSION_ERRHANDLER_FUNCTION', abi_type=['ompi'])
+class TypeSessionErrhandlerFunction(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Session_errhandler_function *'
+
+
+@Type.add_type('SESSION_ERRHANDLER_FUNCTION', abi_type=['standard'])
+class TypeSessionErrhandlerFunctionStandard(Type):
+    # TODO: This may require a special function to wrap the callback
+    pass
+
+
+@Type.add_type('TYPE_COPY_ATTR_FUNCTION', abi_type=['ompi'])
+class TypeTypeCopyAttrFunction(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Type_copy_attr_function *'
+
+
+@Type.add_type('TYPE_COPY_ATTR_FUNCTION', abi_type=['standard'])
+class TypeTypeCopyAttrFunctionStandard(Type):
+    # TODO: This may require a special function to wrap the callback
+    pass
+
+
+@Type.add_type('TYPE_DELETE_ATTR_FUNCTION', abi_type=['ompi'])
+class TypeTypeDeleteAttrFunction(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Type_delete_attr_function *'
+
+
+@Type.add_type('TYPE_DELETE_ATTR_FUNCTION', abi_type=['standard'])
+class TypeTypeDeleteAttrFunctionStandard(Type):
+    # TODO: This may require a special function to wrap the callback
+    pass
+
+
+@Type.add_type('WIN_ERRHANLDER_FUNCTION', abi_type=['ompi'])
+class TypeWinErrhandlerFunction(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Win_errhandler_function *'
+
+
+@Type.add_type('WIN_ERRHANDLER_FUNCTION', abi_type=['standard'])
+class TypeWinErrhandlerFunctionStandard(Type):
+    # TODO: This may require a special function to wrap the callback
+    pass
+
+
+@Type.add_type('WIN_COPY_ATTR_FUNCTION', abi_type=['ompi'])
+class TypeWinCopyAttrFunction(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Win_copy_attr_function *'
+
+
+@Type.add_type('WIN_COPY_ATTR_FUNCTION', abi_type=['standard'])
+class TypeWinCopyAttrFunctionStandard(Type):
+    # TODO: This may require a special function to wrap the callback
+    pass
+
+
+@Type.add_type('WIN_DELETE_ATTR_FUNCTION', abi_type=['ompi'])
+class TypeWinDeleteAttrFunction(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Win_delete_attr_function *'
+
+
+@Type.add_type('WIN_DELETE_ATTR_FUNCTION', abi_type=['standard'])
+class TypeWinDeleteAttrFunctionStandard(Type):
+    # TODO: This may require a special function to wrap the callback
+    pass
+
+
+@Type.add_type('ERRHANDLER', abi_type=['ompi'])
+class TypeErrhandler(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Errhandler'
+
+
+@Type.add_type('ERRHANDLER', abi_type=['standard'])
+class TypeErrhandlerStandard(Type):
+
+    @property
+    def argument(self):
+        return f'(MPI_Errhandler) {self.name}'
+
+    def type_text(self, enable_count=False):
+        return self.mangle_name('MPI_Errhandler')
+
+
+@Type.add_type('ERRHANDLER_OUT', abi_type=['ompi'])
+class TypeErrhandlerOut(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Errhandler *'
+
+
+@Type.add_type('ERRHANDLER_OUT', abi_type=['standard'])
+class TypeErrhandlerOutStandard(Type):
+
+    @property
+    def argument(self):
+        return f'(MPI_Errhandler *) {self.name}'
+
+    def type_text(self, enable_count=False):
+        type_name = self.mangle_name('MPI_Errhandler')
+        return f'{MPI_Errhandler} *'
+
+
+@Type.add_type('GROUP', abi_type=['ompi'])
+class TypeGroup(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Group'
+
+
+@Type.add_type('GROUP', abi_type=['standard'])
+class TypeGroupStandard(Type):
+
+    @property
+    def argument(self):
+        return f'(MPI_Group) {self.name}'
+
+    def type_text(self, enable_count=False):
+        return self.mangle_name('MPI_Group')
+
+
+@Type.add_type('GROUP_OUT', abi_type=['ompi'])
+class TypeGroupOut(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Group *'
+
+
+@Type.add_type('GROUP_OUT', abi_type=['standard'])
+class TypeGroupOutStandard(Type):
+
+    @property
+    def final_code(self):
+        return [f'*{self.name} = {ConvertOMPIToStandard.GROUP}((MPI_Group) *{self.name});']
+
+    def type_text(self, enable_count=False):
+        type_name = self.mangle_name('MPI_Group')
+        return f'{type_name} *'
+
+    @property
+    def argument(self):
+        return f'(MPI_Group *) {self.name}'
+
+
+@Type.add_type('SESSION_OUT', abi_type=['ompi'])
+class TypeSessionOut(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Session *'
+
+
+@Type.add_type('SESSION_OUT', abi_type=['standard'])
+class TypeSessionOutStandard(Type):
+
+    # TODO: This will require some conversion code for the ABI
+    @property
+    def argument(self):
+        return f'(MPI_Session *) {self.name}'
+
+    def type_text(self):
+        type_name = self.mangle_name('MPI_Session')
+        return f'{type_name} *'
+
+
+@Type.add_type('SESSION', abi_type=['ompi'])
+class TypeSession(Type):
+
+    def type_text(self, enable_count=False):
+        return 'MPI_Session'
+
+
+@Type.add_type('SESSION', abi_type=['standard'])
+class TypeSessionStandard(Type):
+
+    # TODO: This will require some conversion code for the ABI
+    @property
+    def argument(self):
+        return f'(MPI_Session) {self.name}'
+
+    def type_text(self, enable_count=False):
+        return self.mangle_name('MPI_Session')
diff --git a/ompi/mpi/bindings/ompi_bindings/consts.py b/ompi/mpi/bindings/ompi_bindings/consts.py
new file mode 100644
index 00000000000..48ef249223f
--- /dev/null
+++ b/ompi/mpi/bindings/ompi_bindings/consts.py
@@ -0,0 +1,253 @@
+# Copyright (c) 2024      Triad National Security, LLC. All rights
+#                         reserved.
+#
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+"""Constants used for generating bindings."""
+import re
+
+FORTRAN_ERROR_NAME = 'ierror'
+C_ERROR_NAME = 'ierr'
+C_ERROR_TMP_NAME = 'c_ierr'
+GENERATED_MESSAGE = 'THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT EDIT BY HAND.'
+
+#
+# C and ABI constants
+#
+# C type: const int
+ERROR_CLASSES = [
+    'MPI_SUCCESS',
+    'MPI_ERR_BUFFER',
+    'MPI_ERR_COUNT',
+    'MPI_ERR_TYPE',
+    'MPI_ERR_TAG',
+    'MPI_ERR_COMM',
+    'MPI_ERR_RANK',
+    'MPI_ERR_REQUEST',
+    'MPI_ERR_ROOT',
+    'MPI_ERR_GROUP',
+    'MPI_ERR_OP',
+    'MPI_ERR_TOPOLOGY',
+    'MPI_ERR_DIMS',
+    'MPI_ERR_ARG',
+    'MPI_ERR_UNKNOWN',
+    'MPI_ERR_TRUNCATE',
+    'MPI_ERR_OTHER',
+    'MPI_ERR_INTERN',
+    'MPI_ERR_PENDING',
+    'MPI_ERR_IN_STATUS',
+    'MPI_ERR_ACCESS',
+    'MPI_ERR_AMODE',
+    'MPI_ERR_ASSERT',
+    'MPI_ERR_BAD_FILE',
+    'MPI_ERR_BASE',
+    'MPI_ERR_CONVERSION',
+    'MPI_ERR_DISP',
+    'MPI_ERR_DUP_DATAREP',
+    'MPI_ERR_FILE_EXISTS',
+    'MPI_ERR_FILE_IN_USE',
+    'MPI_ERR_FILE',
+    'MPI_ERR_INFO_KEY',
+    'MPI_ERR_INFO_NOKEY',
+    'MPI_ERR_INFO_VALUE',
+    'MPI_ERR_INFO',
+    'MPI_ERR_IO',
+    'MPI_ERR_KEYVAL',
+    'MPI_ERR_LOCKTYPE',
+    'MPI_ERR_NAME',
+    'MPI_ERR_NO_MEM',
+    'MPI_ERR_NOT_SAME',
+    'MPI_ERR_NO_SPACE',
+    'MPI_ERR_NO_SUCH_FILE',
+    'MPI_ERR_PORT',
+    'MPI_ERR_PROC_ABORTED',
+    'MPI_ERR_QUOTA',
+    'MPI_ERR_READ_ONLY',
+    'MPI_ERR_RMA_ATTACH',
+    'MPI_ERR_RMA_CONFLICT',
+    'MPI_ERR_RMA_RANGE',
+    'MPI_ERR_RMA_SHARED',
+    'MPI_ERR_RMA_SYNC',
+    'MPI_ERR_RMA_FLAVOR',
+    'MPI_ERR_SERVICE',
+    'MPI_ERR_SESSION',
+    'MPI_ERR_SIZE',
+    'MPI_ERR_SPAWN',
+    'MPI_ERR_UNSUPPORTED_DATAREP',
+    'MPI_ERR_UNSUPPORTED_OPERATION',
+    'MPI_ERR_WIN',
+    'MPI_T_ERR_CANNOT_INIT',
+    'MPI_T_ERR_NOT_INITIALIZED',
+    'MPI_T_ERR_MEMORY',
+    'MPI_T_ERR_INVALID',
+    'MPI_T_ERR_INVALID_INDEX',
+    'MPI_T_ERR_INVALID_ITEM',
+    'MPI_T_ERR_INVALID_SESSION',
+    'MPI_T_ERR_INVALID_HANDLE',
+    'MPI_T_ERR_INVALID_NAME',
+    'MPI_T_ERR_OUT_OF_HANDLES',
+    'MPI_T_ERR_OUT_OF_SESSIONS',
+    'MPI_T_ERR_CVAR_SET_NOT_NOW',
+    'MPI_T_ERR_CVAR_SET_NEVER',
+    'MPI_T_ERR_PVAR_NO_WRITE',
+    'MPI_T_ERR_PVAR_NO_STARTSTOP',
+    'MPI_T_ERR_PVAR_NO_ATOMIC',
+    'MPI_ERR_LASTCODE',
+]
+
+PREDEFINED_DATATYPES = [
+    'MPI_CHAR',
+    'MPI_SHORT',
+    'MPI_INT',
+    'MPI_LONG',
+    'MPI_LONG_LONG_INT',
+    'MPI_LONG_LONG',
+    'MPI_SIGNED_CHAR',
+    'MPI_UNSIGNED_CHAR',
+    'MPI_UNSIGNED_SHORT',
+    'MPI_UNSIGNED',
+    'MPI_UNSIGNED_LONG',
+    'MPI_UNSIGNED_LONG_LONG',
+    'MPI_FLOAT',
+    'MPI_DOUBLE',
+    'MPI_LONG_DOUBLE',
+    'MPI_WCHAR',
+    'MPI_C_BOOL',
+    'MPI_INT8_T',
+    'MPI_INT16_T',
+    'MPI_INT32_T',
+    'MPI_INT64_T',
+    'MPI_UINT8_T',
+    'MPI_UINT16_T',
+    'MPI_UINT32_T',
+    'MPI_UINT64_T',
+    'MPI_AINT',
+    'MPI_COUNT',
+    'MPI_OFFSET',
+    'MPI_C_COMPLEX',
+    'MPI_C_FLOAT_COMPLEX',
+    'MPI_C_DOUBLE_COMPLEX',
+    'MPI_C_LONG_DOUBLE_COMPLEX',
+    'MPI_BYTE',
+    'MPI_PACKED',
+    'MPI_CXX_BOOL',
+    'MPI_CXX_FLOAT_COMPLEX',
+    'MPI_CXX_DOUBLE_COMPLEX',
+    'MPI_CXX_LONG_DOUBLE_COMPLEX',
+    'MPI_FLOAT_INT',
+    'MPI_DOUBLE_INT',
+    'MPI_LONG_INT',
+    'MPI_2INT',
+    'MPI_SHORT_INT',
+    'MPI_LONG_DOUBLE_INT',
+]
+
+# C type: MPI_Comm
+RESERVED_COMMUNICATORS = [
+    'MPI_COMM_NULL',
+    'MPI_COMM_WORLD',
+    'MPI_COMM_SELF',
+]
+
+COMMUNICATOR_SPLIT_TYPES = [
+    'MPI_COMM_TYPE_SHARED',
+    'MPI_COMM_TYPE_HW_UNGUIDED',
+    'MPI_COMM_TYPE_HW_GUIDED',
+]
+
+RESERVED_WINDOWS = [
+    'MPI_WIN_NULL',
+]
+
+RESERVED_REQUESTS = [
+    'MPI_REQUEST_NULL',
+]
+
+RESERVED_INFOS = [
+    'MPI_INFO_ENV',
+    'MPI_INFO_NULL',
+]
+
+RESERVED_FILES = [
+    'MPI_FILE_NULL',
+]
+
+IGNORED_STATUS_HANDLES = [
+    'MPI_STATUSES_IGNORE',
+    'MPI_STATUS_IGNORE',
+]
+
+COLLECTIVE_OPERATIONS = [
+    'MPI_MAX',
+    'MPI_MIN',
+    'MPI_SUM',
+    'MPI_PROD',
+    'MPI_MAXLOC',
+    'MPI_MINLOC',
+    'MPI_BAND',
+    'MPI_BOR',
+    'MPI_BXOR',
+    'MPI_LAND',
+    'MPI_LOR',
+    'MPI_LXOR',
+    'MPI_REPLACE',
+    'MPI_NO_OP',
+]
+
+VARIOUS_CONSTANTS = {
+    # Just setting this to the same as ompi ABI for right now, but will need to
+    # match the standard ABI value when defined
+    'MPI_MAX_LIBRARY_VERSION_STRING': 256,
+    'MPI_MAX_PROCESSOR_NAME': 256,
+}
+
+# Types
+
+C_OPAQUE_TYPES = {
+    'MPI_Aint': 'intptr_t',
+    'MPI_Offset': 'int64_t',
+    'MPI_Count': 'size_t',
+    # The below type needs to be set externally depending on Fortran compiler
+    'MPI_Fint': 'int64_t',
+}
+
+C_HANDLES = [
+    'MPI_Comm',
+    'MPI_Datatype',
+    'MPI_Errhandler',
+    'MPI_File',
+    'MPI_Group',
+    'MPI_Info',
+    'MPI_Message',
+    'MPI_Op',
+    'MPI_Request',
+    'MPI_Session',
+    'MPI_Win',
+]
+
+class ConvertFuncs:
+    """Names of conversion functions (between standard ABI and OMPI ABI)."""
+
+    ERROR_CLASS = 'ompi_convert_intern_error_abi_error'
+    COMM = 'ompi_convert_abi_comm_intern_comm'
+    DATATYPE = 'ompi_convert_abi_datatype_intern_datatype'
+    REQUEST = 'ompi_convert_abi_request_intern_request'
+    STATUS = 'ompi_convert_intern_status_abi_status'
+    OP = 'ompi_convert_abi_op_intern_op'
+    WIN = 'ompi_convert_abi_win_intern_win'
+    INFO = 'ompi_convert_abi_info_intern_info'
+    FILE = 'ompi_convert_abi_file_intern_file'
+
+
+class ConvertOMPIToStandard:
+    """Generated function for converting from OMPI to standard ABI."""
+
+    COMM = 'ompi_convert_comm_ompi_to_standard'
+
+
+# Inline function attributes
+INLINE_ATTRS = '__opal_attribute_always_inline__ static inline'
diff --git a/ompi/mpi/bindings/ompi_bindings/fortran.py b/ompi/mpi/bindings/ompi_bindings/fortran.py
new file mode 100644
index 00000000000..42d185d4fba
--- /dev/null
+++ b/ompi/mpi/bindings/ompi_bindings/fortran.py
@@ -0,0 +1,300 @@
+# Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+#                         reserved.
+#
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+"""Fortran binding generation code.
+
+This takes as input a *.in file containing a list of prototypes for Fortran
+subroutines with generic types. Using this file, it can generate the Fortran
+subroutines in one file and the C wraping code in another for all prototypes
+listed.
+"""
+from collections import namedtuple
+import json
+import re
+from ompi_bindings import consts, util
+from ompi_bindings.fortran_type import FortranType
+from ompi_bindings.parser import SourceTemplate
+
+
+class FortranBinding:
+    """Class for generating the binding for a single function."""
+
+    def __init__(self, prototype, out, template=None, bigcount=False, needs_ts=False):
+        # Generate bigcount interface version
+        self.bigcount = bigcount
+        self.fn_name = template.prototype.name
+        self.out = out
+        self.template = template
+        self.needs_ts = needs_ts
+        self.parameters = []
+        for param in self.template.prototype.params:
+            self.parameters.append(param.construct(fn_name=self.fn_name,
+                                                   bigcount=bigcount))
+
+    def dump(self, *pargs, **kwargs):
+        """Write to the output file."""
+        self.out.dump(*pargs, **kwargs)
+
+    def _fn_name_suffix(self):
+        """Return a suffix for function names."""
+        return '_c' if self.bigcount else ''
+
+    @property
+    def c_func_name(self):
+        """Produce the final C func name from base_name."""
+        return f'ompi_{self.fn_name}_wrapper_f08{self._fn_name_suffix()}'
+
+    @property
+    def inner_call(self):
+        """Produce the name of the function to call in the body of the C code."""
+        return f'PMPI_{self.fn_name.capitalize()}{self._fn_name_suffix()}'
+
+    def _use(self):
+        """Determine the Fortran use-statements needed."""
+        use = {}
+        for param in self.parameters:
+            for mod, name in param.use():
+                if mod not in use:
+                    use[mod] = set()
+                use[mod].add(name)
+        return use
+
+    def _use_stmts(self):
+        """Return a list of required use statments."""
+        use = self._use()
+        stmts = []
+        for mod, names in use.items():
+            names = ', '.join(names)
+            stmts.append(f'use :: {mod}, only: {names}')
+        return stmts
+
+    def _print_fortran_interface(self):
+        """Output the C subroutine binding for the Fortran code."""
+        name = self.c_func_name
+        self.dump('    interface')
+
+        # Print the subroutine and parameter list, breaking parameters across lines
+        subroutine_start = f'        subroutine {name}('
+        params = [param.name for param in self.parameters]
+        params.append(consts.FORTRAN_ERROR_NAME)
+        lines = util.break_param_lines_fortran(start=subroutine_start, params=params, end=') &')
+        for line in lines:
+            self.dump(line)
+        self.dump(f'            BIND(C, name="{name}")')
+
+        use_stmts = self._use_stmts()
+        for stmt in use_stmts:
+            self.dump(f'            {stmt}')
+        self.dump('            implicit none')
+        for param in self.parameters:
+            self.dump(f'            {param.declare_cbinding_fortran()}')
+        self.dump(f'            INTEGER, INTENT(OUT) :: {consts.FORTRAN_ERROR_NAME}')
+        self.dump(f'        end subroutine {name}')
+        self.dump('    end interface')
+
+    def _print_fortran_header(self, is_interface=False):
+        """Print the header, including use stmts, dummy variable decls, etc..
+
+        This does not include the subroutine line.
+        """
+        # Use statements
+        use_stmts = self._use_stmts()
+        for stmt in use_stmts:
+            self.dump(f'    {stmt}')
+        self.dump('    implicit none')
+        # Parameters/dummy variable declarations
+        for param in self.parameters:
+            if is_interface:
+                self.dump_lines(param.interface_predeclare())
+            self.dump_lines(param.declare())
+        # Add the integer error manually
+        self.dump(f'    INTEGER, OPTIONAL, INTENT(OUT) :: {consts.FORTRAN_ERROR_NAME}')
+
+    def _print_fortran_subroutine(self):
+        """Output the Fortran subroutine line."""
+        sub_name = util.fortran_f08_name(self.fn_name, bigcount=self.bigcount, needs_ts=self.needs_ts)
+        params = [param.name for param in self.parameters]
+        params.append(consts.FORTRAN_ERROR_NAME)
+        lines = util.break_param_lines_fortran(f'subroutine {sub_name}(', params, ')')
+        for line in lines:
+            self.dump(line)
+
+    def _print_fortran_subroutine_end(self):
+        """Output the Fortran end subroutine line."""
+        sub_name = util.fortran_f08_name(self.fn_name, bigcount=self.bigcount, needs_ts=self.needs_ts)
+        self.dump(f'end subroutine {sub_name}')
+
+    def dump_lines(self, line_text):
+        for line in line_text.split('\n'):
+            line = line.rstrip()
+            if line:
+                self.dump(f'    {line}')
+
+    def print_f_source(self):
+        """Output the main MPI Fortran subroutine."""
+        self._print_fortran_subroutine()
+        self._print_fortran_header()
+
+        # Temporaries
+        self.dump(f'    INTEGER :: {consts.C_ERROR_TMP_NAME}')
+        for param in self.parameters:
+            self.dump_lines(param.declare_tmp())
+
+        # Interface for call to C function
+        self.dump()
+        self._print_fortran_interface()
+        self.dump()
+
+        # Call into the C function
+        call_start = f'    call {self.c_func_name}('
+        params = [param.argument() for param in self.parameters]
+        params.append(consts.C_ERROR_TMP_NAME)
+        lines = util.break_param_lines_fortran(start=call_start, params=params, end=')')
+        for line in lines:
+            self.dump(line)
+
+        # Convert error type
+        self.dump(f'    if (present({consts.FORTRAN_ERROR_NAME})) {consts.FORTRAN_ERROR_NAME} = {consts.C_ERROR_TMP_NAME}')
+
+        for param in self.parameters:
+            self.dump_lines(param.post())
+
+        self._print_fortran_subroutine_end()
+
+    def print_c_source(self):
+        """Output the C source and function that the Fortran calls into."""
+        if self.template is None:
+            return
+        parameters = [param.c_parameter() for param in self.parameters]
+        # Always append the integer error
+        parameters.append(f'MPI_Fint *{consts.C_ERROR_NAME}')
+        parameters = ', '.join(parameters)
+        # Just put the signature here to silence `-Wmissing-prototypes`
+        c_func = self.c_func_name
+        self.dump(f'void {c_func}({parameters});')
+        self.dump(f'void {c_func}({parameters})')
+        count_type, disp_type, count_fint_type = ('MPI_Count', 'MPI_Aint', 'MPI_Count') if self.bigcount else ('int', 'int', 'MPI_Fint')
+        self.template.print_body(c_func, out=self.out,
+                                 replacements={'INNER_CALL': self.inner_call,
+                                               'COUNT_TYPE': count_type,
+                                               'COUNT_FINT_TYPE': count_fint_type,
+                                               'DISP_TYPE': disp_type})
+
+    def print_interface(self):
+        """Output just the Fortran interface for this binding."""
+        self._print_fortran_subroutine()
+        self._print_fortran_header(is_interface=True)
+        self._print_fortran_subroutine_end()
+
+
+def print_f_source_header(out):
+    """Print the fortran f08 file header."""
+    out.dump(f'! {consts.GENERATED_MESSAGE}')
+    out.dump('#include "ompi/mpi/fortran/configure-fortran-output.h"')
+
+
+def print_profiling_rename_macros(templates, out, args):
+    """Print macros for renaming functions for the profiling interface.
+
+    Previously hardcoded in mpi-f08-rename.h.
+    """
+    out.dump('#if OMPI_BUILD_MPI_PROFILING')
+    for template in templates:
+        has_buffers = util.prototype_has_buffers(template.prototype)
+        needs_ts = has_buffers and args.generate_ts_suffix
+        name = util.fortran_f08_name(template.prototype.name, needs_ts=needs_ts)
+        out.dump(f'#define {name} P{name}')
+        # Check for bigcount version
+        if util.prototype_has_bigcount(template.prototype):
+            bigcount_name = util.fortran_f08_name(template.prototype.name, bigcount=True, needs_ts=needs_ts)
+            out.dump(f'#define {bigcount_name} P{bigcount_name}')
+        name = util.fortran_f08_generic_interface_name(template.prototype.name)
+        out.dump(f'#define {name} P{name}')
+    out.dump('#endif /* OMPI_BUILD_MPI_PROFILING */')
+
+
+def print_c_source_header(out):
+    """Print the header of the C source file."""
+    out.dump(f'/* {consts.GENERATED_MESSAGE} */')
+    out.dump('#include "ompi_config.h"')
+    out.dump('#include "mpi.h"')
+    out.dump('#include "ompi/errhandler/errhandler.h"')
+    out.dump('#include "ompi/mpi/fortran/mpif-h/status-conversion.h"')
+    out.dump('#include "ompi/mpi/fortran/base/constants.h"')
+    out.dump('#include "ompi/mpi/fortran/base/fint_2_int.h"')
+    out.dump('#include "ompi/request/request.h"')
+    out.dump('#include "ompi/communicator/communicator.h"')
+    out.dump('#include "ompi/win/win.h"')
+    out.dump('#include "ompi/file/file.h"')
+    out.dump('#include "ompi/errhandler/errhandler.h"')
+    out.dump('#include "ompi/datatype/ompi_datatype.h"')
+    out.dump('#include "ompi/mca/coll/base/coll_base_util.h"')
+    out.dump('#include "ts.h"')
+    out.dump('#include "bigcount.h"')
+
+
+def print_binding(prototype, lang, out, bigcount=False, template=None, needs_ts=False):
+    """Print the binding with or without bigcount."""
+    binding = FortranBinding(prototype, out=out, bigcount=bigcount, template=template, needs_ts=needs_ts)
+    if lang == 'fortran':
+        binding.print_f_source()
+    else:
+        binding.print_c_source()
+
+
+def load_function_templates(prototype_files):
+    """Load the templates from a file list."""
+    return [
+        SourceTemplate.load(fname, type_constructor=FortranType.construct)
+        for fname in prototype_files
+    ]
+
+
+def generate_code(args, out):
+    """Generate binding code based on arguments."""
+    templates = load_function_templates(args.prototype_files)
+
+    if args.lang == 'fortran':
+        print_f_source_header(out)
+        out.dump()
+        print_profiling_rename_macros(templates, out, args)
+        out.dump()
+    else:
+        print_c_source_header(out)
+
+    for template in templates:
+        out.dump()
+        has_buffers = util.prototype_has_buffers(template.prototype)
+        needs_ts = has_buffers and args.generate_ts_suffix
+        print_binding(template.prototype, args.lang, out, template=template, needs_ts=needs_ts)
+        if util.prototype_has_bigcount(template.prototype):
+            out.dump()
+            print_binding(template.prototype, args.lang, bigcount=True, out=out, template=template, needs_ts=needs_ts)
+
+
+def generate_interface(args, out):
+    """Generate the Fortran interface files."""
+    out.dump(f'! {consts.GENERATED_MESSAGE}')
+
+    templates = load_function_templates(args.prototype_files)
+    print_profiling_rename_macros(templates, out, args)
+
+    for template in templates:
+        ext_name = util.ext_api_func_name(template.prototype.name)
+        out.dump(f'interface {ext_name}')
+        has_buffers = util.prototype_has_buffers(template.prototype)
+        needs_ts = has_buffers and args.generate_ts_suffix
+        binding = FortranBinding(template.prototype, template=template, needs_ts=needs_ts, out=out)
+        binding.print_interface()
+        if util.prototype_has_bigcount(template.prototype):
+            out.dump()
+            binding_c = FortranBinding(template.prototype, out=out, template=template,
+                                       needs_ts=needs_ts, bigcount=True)
+            binding_c.print_interface()
+        out.dump(f'end interface {ext_name}')
diff --git a/ompi/mpi/bindings/ompi_bindings/fortran_type.py b/ompi/mpi/bindings/ompi_bindings/fortran_type.py
new file mode 100644
index 00000000000..38627123536
--- /dev/null
+++ b/ompi/mpi/bindings/ompi_bindings/fortran_type.py
@@ -0,0 +1,761 @@
+# Copyright (c) 2024      Triad National Security, LLC. All rights
+#                         reserved.
+#
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+"""Fortran types and corresponding template code.
+
+All types used in the Fortran bindings are defined here as classes that derive
+from the FortranType base class. These are used for generating both Fortran and
+supporting C code for the mpi_f08 bindings.
+"""
+from abc import ABC, abstractmethod
+from ompi_bindings import consts, util
+
+
+class FortranType(ABC):
+
+    def __init__(self, name, fn_name, bigcount=False, count_param=None, **kwargs):
+        self.name = name
+        self.fn_name = fn_name
+        # Generate the bigcount interface version?
+        self.bigcount = bigcount
+        self.count_param = count_param
+        self.used_counters = 0
+
+    TYPES = {}
+
+    @classmethod
+    def add(cls, type_name):
+        """Decorator for adding types."""
+        def wrapper(class_):
+            cls.TYPES[type_name] = class_
+            return class_
+        return wrapper
+
+    @classmethod
+    def get(cls, type_name):
+        return cls.TYPES[type_name]
+
+    @classmethod
+    def construct(cls, type_name, **kwargs):
+        type_ = cls.TYPES[type_name]
+        return type_(**kwargs)
+
+    @property
+    def fn_api_name(self):
+        """Return the MPI API name to be used in error messages, etc.."""
+        return util.ext_api_func_name(self.fn_name, bigcount=self.bigcount).upper()
+
+    @property
+    def tmp_name(self):
+        """Return a temporary name for use in C."""
+        return f'c_{self.name}'
+
+    @property
+    def tmp_name2(self):
+        """Return a secondary temporary name for use in C."""
+        return f'c_{self.name}2'
+
+    def tmp_counter(self):
+        """Get a temporary counter variable to be used in a loop."""
+        name = f'{self.name}_i_{self.used_counters}'
+        self.used_counters += 1
+        return name
+
+    def interface_predeclare(self):
+        """Return predeclaration code, if required for the interface."""
+        return ''
+
+    @abstractmethod
+    def declare(self):
+        """Return a declaration for the type."""
+
+    def declare_tmp(self):
+        """Declare temporaries on in the subroutine."""
+        return ''
+
+    def declare_cbinding_fortran(self):
+        """Return the C binding declaration as seen from Fortran."""
+        return self.declare()
+
+    def argument(self):
+        """Return the value to pass as an argument."""
+        return self.name
+
+    def use(self):
+        """Return list of (module, name) for a Fortran use-statement."""
+        return []
+
+    def post(self):
+        """Return post-processing code to be run after the call."""
+        return ''
+
+    @abstractmethod
+    def c_parameter(self):
+        """Return the parameter expression to be used in the C function."""
+
+#
+# Definitions of generic types in Fortran and how these can be converted
+# to and from C.
+#
+
+@FortranType.add('BUFFER')
+class BufferType(FortranType):
+    def interface_predeclare(self):
+        return f'!OMPI_F08_IGNORE_TKR_PREDECL {self.name}'
+
+    def declare(self):
+        return f'OMPI_F08_IGNORE_TKR_TYPE, INTENT(IN) :: {self.name}'
+
+    def c_parameter(self):
+        # See fortran/use-mpi-f08/base/ts.h; OMPI_CFI_BUFFER is expanded based
+        # on whether or not the compiler supports TS 29113.
+        return f'OMPI_CFI_BUFFER *{self.name}'
+
+
+@FortranType.add('BUFFER_ASYNC')
+class BufferAsyncType(BufferType):
+    def declare(self):
+        return f'OMPI_F08_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: {self.name}'
+
+
+@FortranType.add('BUFFER_OUT')
+class BufferOutType(BufferType):
+    def declare(self):
+        return f'OMPI_F08_IGNORE_TKR_TYPE :: {self.name}'
+
+
+@FortranType.add('BUFFER_ASYNC_OUT')
+class BufferAsyncOutType(BufferType):
+    def declare(self):
+        return f'OMPI_F08_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: {self.name}'
+
+
+@FortranType.add('VBUFFER')
+class VBufferType(FortranType):
+    """Variable buffer type, as used by MPI_*v() functions."""
+    def declare(self):
+        return f'OMPI_F08_IGNORE_TKR_TYPE, INTENT(IN) :: {self.name}'
+
+    def c_parameter(self):
+        return f'OMPI_CFI_BUFFER *{self.name}'
+
+
+@FortranType.add('VBUFFER_OUT')
+class VBufferType(FortranType):
+    """Variable buffer receive type, as used by MPI_*v() functions."""
+    def declare(self):
+        return f'OMPI_F08_IGNORE_TKR_TYPE :: {self.name}'
+
+    def c_parameter(self):
+        return f'OMPI_CFI_BUFFER *{self.name}'
+
+
+@FortranType.add('WBUFFER')
+class WBufferType(FortranType):
+    """Variable buffer send type, used with MPI_*w() functions."""
+    def declare(self):
+        return f'OMPI_F08_IGNORE_TKR_TYPE, INTENT(IN) :: {self.name}'
+
+    def c_parameter(self):
+        return f'OMPI_CFI_BUFFER *{self.name}'
+
+
+@FortranType.add('WBUFFER_OUT')
+class WBufferType(FortranType):
+    """Variable buffer receive type, used with MPI_*w() functions."""
+    def declare(self):
+        return f'OMPI_F08_IGNORE_TKR_TYPE :: {self.name}'
+
+    def c_parameter(self):
+        return f'OMPI_CFI_BUFFER *{self.name}'
+
+
+@FortranType.add('C_PTR_OUT')
+class CptrType(FortranType):
+    def declare(self):
+        return f'TYPE(C_PTR), INTENT(OUT) :: {self.name}'
+
+    def use(self):
+        return [('ISO_C_BINDING', 'C_PTR')]
+
+    def c_parameter(self):
+        return f'char *{self.name}'
+
+@FortranType.add('COUNT')
+class CountType(FortranType):
+    def declare(self):
+        if self.bigcount:
+            return f'INTEGER(KIND=MPI_COUNT_KIND), INTENT(IN) :: {self.name}'
+        else:
+            return f'INTEGER, INTENT(IN) :: {self.name}'
+
+    def use(self):
+        return [('mpi_f08_types', 'MPI_COUNT_KIND')]
+
+    def c_parameter(self):
+        type_ = 'MPI_Count' if self.bigcount else 'MPI_Fint'
+        return f'{type_} *{self.name}'
+
+@FortranType.add('COUNT_INOUT')
+class CountTypeInOut(FortranType):
+    """COUNT type with INOUT INTENT"""
+    def declare(self):
+        if self.bigcount:
+            return f'INTEGER(KIND=MPI_COUNT_KIND), INTENT(INOUT) :: {self.name}'
+        else:
+            return f'INTEGER, INTENT(INOUT) :: {self.name}'
+
+    def use(self):
+        return [('mpi_f08_types', 'MPI_COUNT_KIND')]
+
+    def c_parameter(self):
+        type_ = 'MPI_Count' if self.bigcount else 'MPI_Fint'
+        return f'{type_} *{self.name}'
+
+@FortranType.add('COUNT_OUT')
+class CountTypeInOut(FortranType):
+    """COUNT type with OUT INTENT"""
+    def declare(self):
+        if self.bigcount:
+            return f'INTEGER(KIND=MPI_COUNT_KIND), INTENT(OUT) :: {self.name}'
+        else:
+            return f'INTEGER, INTENT(IN) :: {self.name}'
+
+    def use(self):
+        return [('mpi_f08_types', 'MPI_COUNT_KIND')]
+
+    def c_parameter(self):
+        type_ = 'MPI_Count' if self.bigcount else 'MPI_Fint'
+        return f'{type_} *{self.name}'
+
+
+@FortranType.add('PARTITIONED_COUNT')
+class PartitionedCountType(FortranType):
+    def declare(self):
+            return f'INTEGER(KIND=MPI_COUNT_KIND), INTENT(IN) :: {self.name}'
+
+    def use(self):
+        return [('mpi_f08_types', 'MPI_COUNT_KIND')]
+
+    def c_parameter(self):
+        return f'MPI_Count *{self.name}'
+
+
+@FortranType.add('DATATYPE')
+class DatatypeType(FortranType):
+    def declare(self):
+        return f'TYPE(MPI_Datatype), INTENT(IN) :: {self.name}'
+
+    def declare_cbinding_fortran(self):
+        return f'INTEGER, INTENT(IN) :: {self.name}'
+
+    def argument(self):
+        return f'{self.name}%MPI_VAL'
+
+    def use(self):
+        return [('mpi_f08_types', 'MPI_Datatype')]
+
+    def c_parameter(self):
+        return f'MPI_Fint *{self.name}'
+
+@FortranType.add('DATATYPE_OUT')
+class DatatypeTypeOut(FortranType):
+    def declare(self):
+        return f'TYPE(MPI_Datatype), INTENT(OUT) :: {self.name}'
+
+    def declare_cbinding_fortran(self):
+        return f'INTEGER, INTENT(OUT) :: {self.name}'
+
+    def argument(self):
+        return f'{self.name}%MPI_VAL'
+
+    def use(self):
+        return [('mpi_f08_types', 'MPI_Datatype')]
+
+    def c_parameter(self):
+        return f'MPI_Fint *{self.name}'
+
+
+
+@FortranType.add('DATATYPE_ARRAY')
+class DatatypeArrayType(FortranType):
+    def declare(self):
+        return f'TYPE(MPI_Datatype), INTENT(IN) :: {self.name}(*)'
+
+    def use(self):
+        return [('mpi_f08_types', 'MPI_Datatype')]
+
+    def c_parameter(self):
+        return f'MPI_Fint *{self.name}'
+
+
+@FortranType.add('INT')
+class IntType(FortranType):
+    def declare(self):
+        return f'INTEGER, INTENT(IN) :: {self.name}'
+
+    def c_parameter(self):
+        return f'MPI_Fint *{self.name}'
+
+
+@FortranType.add('RANK')
+class RankType(IntType):
+    pass
+
+
+@FortranType.add('TAG')
+class TagType(IntType):
+    pass
+
+
+@FortranType.add('INDEX_OUT')
+class IndexOutType(IntType):
+    def declare(self):
+        return f'INTEGER, INTENT(OUT) :: {self.name}'
+
+
+@FortranType.add('LOGICAL_OUT')
+class LogicalOutType(IntType):
+    """Logical type.
+
+    NOTE: Since the logical type causes difficulties when passed to C code,
+    this code uses a temporary integer in Fortran to pass to the C code. On
+    completion the logical type is set based on C's true/false rules.
+    """
+
+    def declare(self):
+        return f'LOGICAL, INTENT(OUT) :: {self.name}'
+
+    def declare_tmp(self):
+        return f'INTEGER :: {self.tmp_name} = 0'
+
+    def declare_cbinding_fortran(self):
+        return f'INTEGER, INTENT(OUT) :: {self.name}'
+
+    def argument(self):
+        return self.tmp_name
+
+    def post(self):
+        return f'{self.name} = {self.tmp_name} /= 0'
+
+    def c_parameter(self):
+        return f'MPI_Fint *{self.name}'
+
+
+@FortranType.add('COMM')
+class CommType(FortranType):
+    def declare(self):
+        return f'TYPE(MPI_Comm), INTENT(IN) :: {self.name}'
+
+    def declare_cbinding_fortran(self):
+        return f'INTEGER, INTENT(IN) :: {self.name}'
+
+    def argument(self):
+        return f'{self.name}%MPI_VAL'
+
+    def use(self):
+        return [('mpi_f08_types', 'MPI_Comm')]
+
+    def c_parameter(self):
+        return f'MPI_Fint *{self.name}'
+
+
+@FortranType.add('STATUS')
+class StatusType(FortranType):
+    def declare(self):
+        return f'TYPE(MPI_Status) :: {self.name}'
+
+    def use(self):
+        return [('mpi_f08_types', 'MPI_Status')]
+
+    def c_parameter(self):
+        return f'MPI_Fint *{self.name}'
+
+
+@FortranType.add('STATUS_OUT')
+class StatusOutType(FortranType):
+    def declare(self):
+        return f'TYPE(MPI_Status), INTENT(OUT) :: {self.name}'
+
+    def use(self):
+        return [('mpi_f08_types', 'MPI_Status')]
+
+    def c_parameter(self):
+        # TODO: Is this correct? (I've listed it as TYPE(MPI_Status) in the binding)
+        return f'MPI_Fint *{self.name}'
+
+
+@FortranType.add('REQUEST_OUT')
+class RequestType(FortranType):
+    def declare(self):
+        return f'TYPE(MPI_Request), INTENT(OUT) :: {self.name}'
+
+    def declare_cbinding_fortran(self):
+        return f'INTEGER, INTENT(OUT) :: {self.name}'
+
+    def argument(self):
+        return f'{self.name}%MPI_VAL'
+
+    def use(self):
+        return [('mpi_f08_types', 'MPI_Request')]
+
+    def c_parameter(self):
+        return f'MPI_Fint *{self.name}'
+
+
+@FortranType.add('REQUEST_ARRAY')
+class RequestArrayType(FortranType):
+    def declare(self):
+        return f'TYPE(MPI_Request), INTENT(INOUT) :: {self.name}({self.count_param})'
+
+    def declare_cbinding_fortran(self):
+        return f'INTEGER, INTENT(INOUT) :: {self.name}({self.count_param})'
+
+    def argument(self):
+        return f'{self.name}(:)%MPI_VAL'
+
+    def use(self):
+        return [('mpi_f08_types', 'MPI_Request')]
+
+    def c_parameter(self):
+        return f'MPI_Fint *{self.name}'
+
+
+@FortranType.add('STATUS_ARRAY')
+class StatusArrayType(FortranType):
+    def declare(self):
+        return f'TYPE(MPI_Status), INTENT(OUT) :: {self.name}(*)'
+
+    def use(self):
+        return [('mpi_f08_types', 'MPI_Status')]
+
+    def c_parameter(self):
+        return f'MPI_Fint *{self.name}'
+
+
+@FortranType.add('INT_ARRAY')
+class IntArray(FortranType):
+    """Integer array as used for MPI_*v() variable length functions."""
+
+    def declare(self):
+        return f'INTEGER, INTENT(IN) :: {self.name}(*)'
+
+    def c_parameter(self):
+        return f'MPI_Fint *{self.name}'
+
+
+@FortranType.add('COUNT_ARRAY')
+class CountArray(IntArray):
+    """Array of MPI_Count or int."""
+
+    def declare(self):
+        kind = '(KIND=MPI_COUNT_KIND)' if self.bigcount else ''
+        return f'INTEGER{kind}, INTENT(IN) :: {self.name}(*)'
+
+    def use(self):
+        if self.bigcount:
+            return [('mpi_f08_types', 'MPI_COUNT_KIND')]
+        return []
+
+    def c_parameter(self):
+        count_type = 'MPI_Count' if self.bigcount else 'MPI_Fint'
+        return f'{count_type} *{self.name}'
+
+@FortranType.add('AINT_COUNT_ARRAY')
+class CountArray(IntArray):
+    """Array of MPI_Count or int."""
+
+    def declare(self):
+        kind = '(KIND=MPI_COUNT_KIND)' if self.bigcount else '(KIND=MPI_ADDRESS_KIND)'
+        return f'INTEGER{kind}, INTENT(IN) :: {self.name}(*)'
+
+    def use(self):
+        if self.bigcount:
+            return [('mpi_f08_types', 'MPI_COUNT_KIND')]
+        else:
+            return [('mpi_f08_types', 'MPI_ADDRESS_KIND')]
+
+    def c_parameter(self):
+        count_type = 'MPI_Count' if self.bigcount else 'MPI_Aint'
+        return f'{count_type} *{self.name}'
+
+
+
+@FortranType.add('AINT')
+class Aint(FortranType):
+    """MPI_Aint type."""
+
+    def declare(self):
+        return f'INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: {self.name}'
+
+    def use(self):
+        return [('mpi_f08_types', 'MPI_ADDRESS_KIND')]
+
+    def c_parameter(self):
+        return f'MPI_Aint *{self.name}'
+
+
+@FortranType.add('AINT_OUT')
+class AintOut(FortranType):
+    """MPI_Aint out type."""
+
+    def declare(self):
+        return f'INTEGER(MPI_ADDRESS_KIND), INTENT(OUT) :: {self.name}'
+
+    def use(self):
+        return [('mpi_f08_types', 'MPI_ADDRESS_KIND')]
+
+    def c_parameter(self):
+        return f'MPI_Aint *{self.name}'
+
+
+@FortranType.add('AINT_COUNT')
+class AintCountTypeIn(FortranType):
+    """AINT/COUNT type with ININTENT"""
+    def declare(self):
+        if self.bigcount:
+            return f'INTEGER(KIND=MPI_COUNT_KIND), INTENT(IN) :: {self.name}'
+        else:
+            return f'INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) :: {self.name}'
+
+    def use(self):
+        if self.bigcount:
+            return [('mpi_f08_types', 'MPI_COUNT_KIND')]
+        else:
+            return [('mpi_f08_types', 'MPI_ADDRESS_KIND')]
+
+    def c_parameter(self):
+        type_ = 'MPI_Count' if self.bigcount else 'MPI_Aint'
+        return f'{type_} *{self.name}'
+
+
+@FortranType.add('AINT_COUNT_INOUT')
+class AintCountTypeInOut(FortranType):
+    """AINT/COUNT type with INOUT INTENT"""
+    def declare(self):
+        if self.bigcount:
+            return f'INTEGER(KIND=MPI_COUNT_KIND), INTENT(INOUT) :: {self.name}'
+        else:
+            return f'INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(INOUT) :: {self.name}'
+
+    def use(self):
+        if self.bigcount:
+            return [('mpi_f08_types', 'MPI_COUNT_KIND')]
+        else:
+            return [('mpi_f08_types', 'MPI_ADDRESS_KIND')]
+
+    def c_parameter(self):
+        type_ = 'MPI_Count' if self.bigcount else 'MPI_Aint'
+        return f'{type_} *{self.name}'
+
+
+@FortranType.add('AINT_COUNT_OUT')
+class AintCountTypeOut(FortranType):
+    """AINT/COUNT type with OUT INTENT"""
+    def declare(self):
+        if self.bigcount:
+            return f'INTEGER(KIND=MPI_COUNT_KIND), INTENT(OUT) :: {self.name}'
+        else:
+            return f'INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(OUT) :: {self.name}'
+
+    def use(self):
+        if self.bigcount:
+            return [('mpi_f08_types', 'MPI_COUNT_KIND')]
+        else:
+            return [('mpi_f08_types', 'MPI_ADDRESS_KIND')]
+
+    def c_parameter(self):
+        type_ = 'MPI_Count' if self.bigcount else 'MPI_Aint'
+        return f'{type_} *{self.name}'
+
+
+@FortranType.add('AINT_ARRAY')
+class AintArrayType(FortranType):
+    """Array of MPI_Aint."""
+
+    def declare(self):
+        # TODO: Should there be a separate ASYNC version here, when the OMPI_ASYNCHRONOUS attr is required?
+        return f'INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) OMPI_ASYNCHRONOUS :: {self.name}(*)'
+
+    def use(self):
+        return [('mpi_f08_types', 'MPI_ADDRESS_KIND')]
+
+    def c_parameter(self):
+        return f'MPI_Aint *{self.name}'
+
+
+@FortranType.add('DISP')
+class Disp(FortranType):
+    """Displacecment type."""
+
+    def declare(self):
+        kind = '(KIND=MPI_ADDRESS_KIND)' if self.bigcount else ''
+        return f'INTEGER{kind}, INTENT(IN) :: {self.name}'
+
+    def use(self):
+        if self.bigcount:
+            return [('mpi_f08_types', 'MPI_ADDRESS_KIND')]
+        return []
+
+    def c_parameter(self):
+        count_type = 'MPI_Aint' if self.bigcount else 'MPI_Fint'
+        return f'{count_type} *{self.name}'
+
+@FortranType.add('DISP_OUT')
+class DispOut(FortranType):
+    """Displacecment out type."""
+
+    def declare(self):
+        kind = '(KIND=MPI_ADDRESS_KIND)' if self.bigcount else ''
+        return f'INTEGER{kind}, INTENT(OUT) :: {self.name}'
+
+    def use(self):
+        if self.bigcount:
+            return [('mpi_f08_types', 'MPI_ADDRESS_KIND')]
+        return []
+
+    def c_parameter(self):
+        count_type = 'MPI_Aint' if self.bigcount else 'MPI_Fint'
+        return f'{count_type} *{self.name}'
+
+
+@FortranType.add('DISP_ARRAY')
+class DispArray(IntArray):
+    """Array of MPI_Aint or int."""
+
+    def declare(self):
+        kind = '(KIND=MPI_ADDRESS_KIND)' if self.bigcount else ''
+        return f'INTEGER{kind}, INTENT(IN) :: {self.name}(*)'
+
+    def use(self):
+        if self.bigcount:
+            return [('mpi_f08_types', 'MPI_ADDRESS_KIND')]
+        return []
+
+    def c_parameter(self):
+        count_type = 'MPI_Aint' if self.bigcount else 'MPI_Fint'
+        return f'{count_type} *{self.name}'
+
+
+@FortranType.add('OP')
+class Op(FortranType):
+    """MPI_Op type."""
+
+    def declare(self):
+        return f'TYPE(MPI_Op), INTENT(IN) :: {self.name}'
+
+    def use(self):
+        return [('mpi_f08_types', 'MPI_Op')]
+
+    def c_parameter(self):
+        return f'MPI_Fint *{self.name}'
+
+
+@FortranType.add('WIN')
+class Win(FortranType):
+    """MPI_Win type."""
+
+    def declare(self):
+        return f'TYPE(MPI_Win), INTENT(IN) :: {self.name}'
+
+    def use(self):
+        return [('mpi_f08_types', 'MPI_Win')]
+
+    def c_parameter(self):
+        return f'MPI_Fint *{self.name}'
+
+@FortranType.add('WIN_OUT')
+class WinOut(FortranType):
+    """MPI_Win out type."""
+
+    def declare(self):
+        return f'TYPE(MPI_Win), INTENT(OUT) :: {self.name}'
+
+    def declare_cbinding_fortran(self):
+        return f'INTEGER, INTENT(OUT) :: {self.name}'
+
+    def argument(self):
+        return f'{self.name}%MPI_VAL'
+
+    def use(self):
+        return [('mpi_f08_types', 'MPI_Win')]
+
+    def c_parameter(self):
+        return f'MPI_Fint *{self.name}'
+
+
+@FortranType.add('FILE')
+class File(FortranType):
+    """MPI_File type."""
+
+    def declare(self):
+        return f'TYPE(MPI_File), INTENT(IN) :: {self.name}'
+
+    def use(self):
+        return [('mpi_f08_types', 'MPI_File')]
+
+    def c_parameter(self):
+        return f'MPI_Fint *{self.name}'
+
+@FortranType.add('INFO')
+class Info(FortranType):
+    """MPI_Info type."""
+
+    def declare(self):
+        return f'TYPE(MPI_Info), INTENT(IN) :: {self.name}'
+
+    def use(self):
+        return [('mpi_f08_types', 'MPI_Info')]
+
+    def c_parameter(self):
+        return f'MPI_Fint *{self.name}'
+
+@FortranType.add('OFFSET')
+class Offset(FortranType):
+    """MPI_Offset type."""
+
+    def declare(self):
+        return f'INTEGER(MPI_OFFSET_KIND), INTENT(IN) :: {self.name}'
+
+    def use(self):
+        return [('mpi_f08_types', 'MPI_OFFSET_KIND')]
+
+    def c_parameter(self):
+        return f'MPI_Offset *{self.name}'
+
+
+@FortranType.add('CHAR_ARRAY')
+class CharArray(FortranType):
+    """Fortran CHAR type."""
+
+    def declare(self):
+        return f'CHARACTER(LEN=*), INTENT(IN) :: {self.name}'
+
+    def use(self):
+        return [('iso_c_binding', 'c_char')]
+
+    def declare_cbinding_fortran(self):
+        return f'CHARACTER(KIND=C_CHAR), INTENT(IN) :: {self.name}(*)'
+
+    def c_parameter(self):
+        return f'char *{self.name}'
+
+
+@FortranType.add('MESSAGE_INOUT')
+class MessageInOut(FortranType):
+    """MPI_Message INOUT type."""
+
+    def declare(self):
+        return f'TYPE(MPI_Message), INTENT(INOUT) :: {self.name}'
+
+    def use(self):
+        return [('mpi_f08_types', 'MPI_Message')]
+
+    def c_parameter(self):
+        return f'MPI_Fint *{self.name}'
diff --git a/ompi/mpi/bindings/ompi_bindings/parser.py b/ompi/mpi/bindings/ompi_bindings/parser.py
new file mode 100644
index 00000000000..b7c6b36eba4
--- /dev/null
+++ b/ompi/mpi/bindings/ompi_bindings/parser.py
@@ -0,0 +1,145 @@
+# Copyright (c) 2024      Triad National Security, LLC. All rights
+#                         reserved.
+#
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+"""Source parsing code."""
+
+class Parameter:
+
+    def __init__(self, text, type_constructor):
+        """Parse a parameter."""
+        # parameter in the form "TYPE NAME" or "TYPE NAME:COUNT_VAR"
+        type_name, namecount = text.split()
+        if ':' in namecount:
+            name, count_param = namecount.split(':')
+        else:
+            name, count_param = namecount, None
+        self.type_name = type_name
+        self.name = name
+        self.count_param = count_param
+        self.type_constructor = type_constructor
+
+    def construct(self, **kwargs):
+        """Construct the type parameter for the given ABI."""
+        return self.type_constructor(type_name=self.type_name, name=self.name,
+                                     count_param=self.count_param, **kwargs)
+
+
+class ReturnType:
+    """Return type wrapper."""
+
+    def __init__(self, type_name, type_constructor):
+        self.type_name = type_name
+        self.type_constructor = type_constructor
+
+    def construct(self, **kwargs):
+        """Construct the return type for the given ABI."""
+        return self.type_constructor(type_name=self.type_name, **kwargs)
+
+
+class Prototype:
+    """MPI function prototype."""
+
+    def __init__(self, name, return_type, params):
+        self.name = name
+        self.return_type = return_type
+        self.params = params
+
+    def signature(self, fn_name, enable_count=False, **kwargs):
+        """Build a signature with the given name and if count is enabled."""
+        params = ', '.join(param.construct(**kwargs).parameter(enable_count=enable_count, **kwargs)
+                           for param in self.params)
+        if not params:
+            params = 'void'
+        return_type_text = self.return_type.construct(**kwargs).type_text(enable_count=enable_count)
+        return f'{return_type_text} {fn_name}({params})'
+
+
+def validate_body(body):
+    """Validate the body of a template."""
+    # Just do a simple bracket balance test to determine the bounds of the
+    # function body. All lines after the function body should be blank. There
+    # are cases where this will break, such as if someone puts code all on one
+    # line.
+    bracket_balance = 0
+    line_count = 0
+    for line in body:
+        line = line.strip()
+        if bracket_balance == 0 and line_count > 0 and line:
+            raise util.BindingError('Extra code found in template; only one function body is allowed')
+
+        update = line.count('{') - line.count('}')
+        bracket_balance += update
+        if bracket_balance != 0:
+            line_count += 1
+
+    if bracket_balance != 0:
+        raise util.BindingError('Mismatched brackets found in template')
+
+
+class SourceTemplate:
+    """Source template for a single API function."""
+
+    def __init__(self, prototype, header, body):
+        self.prototype = prototype
+        self.header = header
+        self.body = body
+
+    @staticmethod
+    def load(fname, prefix=None, type_constructor=None):
+        """Load a template file and return the SourceTemplate."""
+        if prefix is not None:
+            fname = os.path.join(prefix, fname)
+        with open(fname) as fp:
+            header = []
+            prototype = []
+            body = []
+
+            for line in fp:
+                line = line.rstrip()
+                if prototype and line.startswith('PROTOTYPE'):
+                    raise util.BindingError('more than one prototype found in template file')
+                elif ((prototype and not any(')' in s for s in prototype))
+                      or line.startswith('PROTOTYPE')):
+                    prototype.append(line)
+                elif prototype:
+                    # Validate bracket balance
+                    body.append(line)
+                else:
+                    header.append(line)
+
+            if not prototype:
+                raise RuntimeError(f'missing prototype for {fname}')
+            # Parse the prototype
+            prototype = ''.join(prototype)
+            prototype = prototype[len('PROTOTYPE'):]
+            i = prototype.index('(')
+            j = prototype.index(')')
+            return_type, name = prototype[:i].split()
+            return_type = ReturnType(return_type, type_constructor=type_constructor)
+            params = [param.strip() for param in prototype[i + 1:j].split(',') if param.strip()]
+            params = [Parameter(param, type_constructor=type_constructor) for param in params]
+            prototype = Prototype(name, return_type, params)
+            # Ensure the body contains only one function
+            validate_body(body)
+            return SourceTemplate(prototype, header, body)
+
+    def print_header(self, out):
+        """Print the source header."""
+        for line in self.header:
+            out.dump(line)
+
+    def print_body(self, func_name, out, replacements=None):
+        """Print the body."""
+        replacements = {} if replacements is None else replacements
+        for line in self.body:
+            # FUNC_NAME is used for error messages
+            line = line.replace('FUNC_NAME', f'"{func_name}"')
+            # Replace other parts in the body of the form '@KEY_NAME@'
+            for key, value in replacements.items():
+                line = line.replace(f'@{key}@', value)
+            out.dump(line)
diff --git a/ompi/mpi/bindings/ompi_bindings/util.py b/ompi/mpi/bindings/ompi_bindings/util.py
new file mode 100644
index 00000000000..68d03eaa563
--- /dev/null
+++ b/ompi/mpi/bindings/ompi_bindings/util.py
@@ -0,0 +1,167 @@
+# Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+#                         reserved.
+#
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+"""Utility code for OMPI binding generation."""
+import textwrap
+
+
+class OutputFile:
+    """Output file of script."""
+
+    def __init__(self, fp):
+        self.fp = fp
+
+    def dump(self, *pargs, **kwargs):
+        print(*pargs, **kwargs, file=self.fp)
+
+
+def prepare_text(text):
+    """Prepare text to be output, removing extra lines and whitespace."""
+    text = textwrap.dedent(text)
+    lines = text.split('\n')
+    out_lines = []
+    new_lines = 0
+    for line in lines:
+        line = line.rstrip()
+        # Only allow one blank line
+        if not line:
+            new_lines += 1
+        else:
+            new_lines = 0
+        if new_lines > 1:
+            continue
+        out_lines.append(line)
+    return '\n'.join(line for line in lines if line.strip())
+
+
+class BindingError(Exception):
+    """Thrown when a binding error is encountered."""
+
+
+def validate_allowed_keys(keys, req_keys, type_name, param_name):
+    """Validate allowed keys for a type, raising an error on failure."""
+    missing_keys = [key for key in req_keys if key not in keys]
+    invalid_keys = [key for key in keys if key not in req_keys]
+    init_message = f'Param {param_name} with type {type_name}'
+    if missing_keys and invalid_keys:
+        raise BindingError(f'{init_message} has missing keys ({missing_keys}) and invalid keys ({invalid_keys})')
+    elif missing_keys:
+        raise BindingError(f'{init_message} has missing keys: {missing_keys}')
+    elif invalid_keys:
+        raise BindingError(f'{init_message} has invalid keys: {invalid_keys}')
+
+
+def ext_api_func_name(fn_name, bigcount=False):
+    """Produce the external MPI API function name."""
+    suffix = '_c' if bigcount else ''
+    return f'MPI_{fn_name.capitalize()}{suffix}'
+
+
+def ext_api_func_name_profile(fn_name, bigcount=False):
+    """Produce the external PMPI API function name."""
+    return f'P{ext_api_func_name(fn_name, bigcount)}'
+
+
+def fortran_f08_name(fn_name, bigcount=False, needs_ts=False):
+    """Produce the final f08 name from base_name. See section 19.2 of MPI 4.1 standard."""
+    suffix = '_c' if bigcount else ''
+    ts = 'ts' if needs_ts else ''
+    return f'MPI_{fn_name.capitalize()}{suffix}_f08{ts}'
+
+def fortran_f08_generic_interface_name(fn_name):
+    """Produce the generic interface name from the base_name."""
+    return f'MPI_{fn_name.capitalize()}'
+
+def break_param_lines_fortran(start, params, end):
+    """Break paramters for a fortran call onto multiple lines.
+
+    This is often necessary to avoid going over the max line length of 132
+    characters.
+    """
+    assert len(params) > 1, 'expected more than one parameter'
+    indent = len(start) * ' '
+    lines = [f'{start}{params[0]},']
+    for param in params[1:-1]:
+        lines.append(f'{indent}{param},')
+    last_line = f'{indent}{params[-1]}{end}'
+    max_len = max(len(line) for line in lines)
+    max_len = max(max_len, len(last_line))
+    result_lines = []
+    for line in lines:
+        spaces = (max_len - len(line) + 1) * ' '
+        result_lines.append(f'{line}{spaces}&')
+    result_lines.append(last_line)
+    return result_lines
+
+
+def indent_lines(lines, tab, start=0):
+    """Crude pretty-printing function."""
+    new_lines = []
+    indent_count = start
+    for line in lines:
+        # Closing bracket
+        if '}' in line:
+            indent_count -= 1
+
+        prefix = indent_count * tab
+        new_lines.append(f'{prefix}{line}')
+
+        # Opening bracket
+        if '{' in line:
+            indent_count += 1
+    return new_lines
+
+
+def mpi_fn_name_from_base_fn_name(name):
+    """Convert from a base name to the standard 'MPI_*' name."""
+    return f'MPI_{name.capitalize()}'
+
+
+def abi_internal_name(extname):
+    """Convert from the ABI external name to an internal name.
+
+    Used to avoid conflicts with existing MPI names.
+    """
+    return f'{extname}_ABI_INTERNAL'
+
+
+BIGCOUNT_TYPE_NAMES = [
+    'COUNT',
+    'COUNT_ARRAY',
+    'DISP',
+    'DISP_ARRAY',
+    'DISP_OUT',
+    'COUNT_INOUT',
+    'COUNT_OUT',
+    'AINT_COUNT',
+    'AINT_COUNT_ARRAY',
+    'AINT_COUNT_OUT',
+    'AINT_COUNT_INOUT',
+    'INT_AINT_OUT',
+    'USER_FUNCTION',
+    'DATAREP_CONVERSION_FUNCTION',
+]
+
+
+def prototype_has_bigcount(prototype):
+    """Should this prototype have a bigcount version?"""
+    return any(param.type_name in BIGCOUNT_TYPE_NAMES for param in prototype.params)
+
+BUFFER_TYPE_NAMES = [
+    'BUFFER',
+    'BUFFER_ASYNC',
+    'BUFFER_OUT',
+    'BUFFER_ASYNC_OUT',
+]
+
+def prototype_has_buffers(prototype):
+    """Does the prototype have buffer arguments?"""
+    if any(param.type_name in BUFFER_TYPE_NAMES for param in prototype.params):
+        return True
+    else:
+        return False
diff --git a/ompi/mpi/c/Makefile.am b/ompi/mpi/c/Makefile.am
index d58b564f225..67eed568e4f 100644
--- a/ompi/mpi/c/Makefile.am
+++ b/ompi/mpi/c/Makefile.am
@@ -9,7 +9,7 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2009-2014 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2009-2021 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
 # Copyright (c) 2012      Oak Rigde National Laboratory. All rights reserved.
 # Copyright (c) 2012-2013 Inria.  All rights reserved.
@@ -17,6 +17,9 @@
 #                         reserved.
 # Copyright (c) 2015-2020 Research Organization for Information Science
 #                         and Technology (RIST).  All rights reserved.
+# Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+#                         reserved.
+# Copyright (c) 2025      Advanced Micro Devices, Inc. All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -24,425 +27,500 @@
 # $HEADER$
 #
 
-SUBDIRS = profile
-
-# OMPI_BUILD_MPI_PROFILING is enabled when we want our generated MPI_* symbols
-# to be replaced by PMPI_*.
-# In this directory, we need it to be 0
+# The purpose of the profiling layer is to allow intercept libraries
+# which override the MPI_ namespace symbols.  We potentially compile
+# every MPI function twice.  We always build the profiling layer,
+# because the symbols that are always implemented as functions are the
+# PMPI_ namespace symbols.  We sometimes also build the non-profiling
+# layer, if weak symbols can't be used to alias the MPI_ namespace
+# into the PMPI_ namespace.
 
-AM_CPPFLAGS     = -DOMPI_BUILD_MPI_PROFILING=0
+include $(top_srcdir)/Makefile.ompi-rules
 
-#
-# The top directory always builds MPI_* bindings. The bottom directory
-# always builds PMPI_* bindings. The cases where the top directory
-# needs to be built are:
-#
-# 1. When profiling is disabled.
-# 2. When profiling is enabled but weak symbol support is absent.
-#
-
-noinst_LTLIBRARIES = libmpi_c.la
+noinst_LTLIBRARIES = libmpi_c.la libmpi_c_profile.la
 if BUILD_MPI_BINDINGS_LAYER
-noinst_LTLIBRARIES += libmpi_c_mpi.la
+noinst_LTLIBRARIES += libmpi_c_noprofile.la
 endif
 
 headers = bindings.h
 
 #
-# libmpi_c.la is always build because it contains some non-profilied
-# functions.
+# Template/prototype source files used for generating MPI functions
 #
 
+prototype_sources = \
+        abort.c.in \
+        accumulate.c.in \
+        add_error_class.c.in \
+        add_error_code.c.in \
+        add_error_string.c.in \
+        allgather.c.in \
+        allgather_init.c.in \
+        allgatherv.c.in \
+        allgatherv_init.c.in \
+        alloc_mem.c.in \
+        allreduce.c.in \
+        allreduce_init.c.in \
+        alltoall.c.in \
+        alltoall_init.c.in \
+        alltoallv.c.in \
+        alltoallv_init.c.in \
+        alltoallw.c.in \
+        alltoallw_init.c.in \
+        attr_delete.c.in \
+        attr_get.c.in \
+        attr_put.c.in \
+        barrier.c.in \
+        barrier_init.c.in \
+        bcast.c.in \
+        bcast_init.c.in \
+        bsend.c.in \
+        bsend_init.c.in \
+        buffer_attach.c.in \
+        buffer_detach.c.in \
+        cancel.c.in \
+        cart_coords.c.in \
+        cart_create.c.in \
+        cartdim_get.c.in \
+        cart_get.c.in \
+        cart_map.c.in \
+        cart_rank.c.in \
+        cart_shift.c.in \
+        cart_sub.c.in \
+        close_port.c.in \
+        comm_accept.c.in \
+        comm_c2f.c.in \
+        comm_call_errhandler.c.in \
+	comm_compare.c.in \
+        comm_connect.c.in \
+	comm_create.c.in \
+	comm_create_errhandler.c.in \
+	comm_create_group.c.in \
+	comm_create_from_group.c.in \
+	comm_create_keyval.c.in \
+        comm_delete_attr.c.in \
+        comm_disconnect.c.in \
+        comm_dup.c.in \
+        comm_dup_with_info.c.in \
+        comm_f2c.c.in \
+        comm_free.c.in \
+        comm_free_keyval.c.in \
+        comm_get_attr.c.in \
+	comm_get_errhandler.c.in \
+	comm_get_info.c.in \
+	comm_get_name.c.in \
+	comm_get_parent.c.in \
+	comm_group.c.in \
+        comm_idup.c.in \
+        comm_idup_with_info.c.in \
+	comm_join.c.in \
+        comm_rank.c.in \
+	comm_remote_group.c.in \
+	comm_remote_size.c.in \
+	comm_set_attr.c.in \
+	comm_set_errhandler.c.in \
+	comm_set_info.c.in \
+	comm_set_name.c.in \
+        comm_size.c.in \
+	comm_spawn.c.in \
+	comm_spawn_multiple.c.in \
+        comm_split.c.in \
+        comm_split_type.c.in \
+	comm_test_inter.c.in \
+	compare_and_swap.c.in \
+	dims_create.c.in \
+	dist_graph_create_adjacent.c.in \
+	dist_graph_create.c.in \
+	dist_graph_neighbors.c.in \
+	dist_graph_neighbors_count.c.in \
+        errhandler_c2f.c.in \
+        errhandler_f2c.c.in \
+	errhandler_free.c.in \
+	error_class.c.in \
+	error_string.c.in \
+	exscan.c.in \
+	exscan_init.c.in \
+	fetch_and_op.c.in \
+	file_c2f.c.in \
+	file_call_errhandler.c.in \
+	file_close.c.in \
+	file_create_errhandler.c.in \
+	file_delete.c.in \
+	file_f2c.c.in \
+	file_get_amode.c.in \
+	file_get_atomicity.c.in \
+	file_get_byte_offset.c.in \
+	file_get_errhandler.c.in \
+	file_get_group.c.in \
+	file_get_info.c.in \
+	file_get_position.c.in \
+	file_get_position_shared.c.in \
+	file_get_size.c.in \
+	file_get_type_extent.c.in \
+	file_get_view.c.in \
+	file_iread_all.c.in \
+	file_iread_at_all.c.in \
+	file_iread_at.c.in \
+	file_iread.c.in \
+	file_iread_shared.c.in \
+	file_iwrite_all.c.in \
+	file_iwrite_at_all.c.in \
+	file_iwrite_at.c.in \
+	file_iwrite.c.in \
+	file_iwrite_shared.c.in \
+        file_open.c.in \
+	file_preallocate.c.in \
+	file_read_all_begin.c.in \
+	file_read_all.c.in \
+	file_read_all_end.c.in \
+	file_read_at_all_begin.c.in \
+	file_read_at_all.c.in \
+	file_read_at_all_end.c.in \
+	file_read_at.c.in \
+	file_read.c.in \
+	file_read_ordered_begin.c.in \
+	file_read_ordered.c.in \
+	file_read_ordered_end.c.in \
+	file_read_shared.c.in \
+	file_seek.c.in \
+	file_seek_shared.c.in \
+	file_set_atomicity.c.in \
+	file_set_errhandler.c.in \
+	file_set_info.c.in \
+	file_set_size.c.in \
+	file_set_view.c.in \
+	file_sync.c.in \
+	file_write_all_begin.c.in \
+	file_write_all.c.in \
+	file_write_all_end.c.in \
+	file_write_at_all_begin.c.in \
+	file_write_at_all.c.in \
+	file_write_at_all_end.c.in \
+	file_write_at.c.in \
+	file_write.c.in \
+	file_write_ordered_begin.c.in \
+	file_write_ordered.c.in \
+	file_write_ordered_end.c.in \
+	file_write_shared.c.in \
+        finalize.c.in \
+        finalized.c.in \
+	free_mem.c.in \
+	gather.c.in \
+	gather_init.c.in \
+	gatherv.c.in \
+	gatherv_init.c.in \
+	get_accumulate.c.in \
+	get_address.c.in \
+	get.c.in \
+	get_count.c.in \
+	get_elements.c.in \
+	get_elements_x.c.in \
+        get_library_version.c.in \
+        get_processor_name.c.in \
+	get_version.c.in \
+	graph_create.c.in \
+	graphdims_get.c.in \
+	graph_get.c.in \
+	graph_map.c.in \
+	graph_neighbors.c.in \
+	graph_neighbors_count.c.in \
+	grequest_complete.c.in \
+	grequest_start.c.in \
+	group_c2f.c.in \
+	group_compare.c.in \
+	group_difference.c.in \
+	group_excl.c.in \
+	group_f2c.c.in \
+	group_free.c.in \
+	group_from_session_pset.c.in \
+	group_incl.c.in \
+	group_intersection.c.in \
+	group_range_excl.c.in \
+	group_range_incl.c.in \
+	group_rank.c.in \
+	group_size.c.in \
+	group_translate_ranks.c.in \
+	group_union.c.in \
+        iallgather.c.in \
+        iallgatherv.c.in \
+        ialltoall.c.in \
+        ialltoallv.c.in \
+        ialltoallw.c.in \
+        ibarrier.c.in \
+        ibcast.c.in \
+	ibsend.c.in \
+	iexscan.c.in \
+	igather.c.in \
+	igatherv.c.in \
+	improbe.c.in \
+	imrecv.c.in \
+        ineighbor_allgather.c.in \
+        ineighbor_allgatherv.c.in \
+        ineighbor_alltoall.c.in \
+	ineighbor_alltoallv.c.in \
+	ineighbor_alltoallw.c.in \
+	info_c2f.c.in \
+	info_create.c.in \
+	info_create_env.c.in \
+	info_delete.c.in \
+	info_dup.c.in \
+	info_f2c.c.in \
+	info_free.c.in \
+	info_get.c.in \
+	info_get_nkeys.c.in \
+	info_get_nthkey.c.in \
+	info_get_string.c.in \
+	info_get_valuelen.c.in \
+	info_set.c.in \
+        init.c.in \
+        initialized.c.in \
+	init_thread.c.in \
+	intercomm_create.c.in \
+	intercomm_create_from_groups.c.in \
+	intercomm_merge.c.in \
+	iprobe.c.in \
+        iallreduce.c.in \
+        irecv.c.in \
+	ireduce.c.in \
+	ireduce_scatter_block.c.in \
+	ireduce_scatter.c.in \
+	irsend.c.in \
+	iscan.c.in \
+	iscatter.c.in \
+	iscatterv.c.in \
+        isend.c.in \
+	isendrecv.c.in \
+	isendrecv_replace.c.in \
+	issend.c.in \
+	is_thread_main.c.in \
+	keyval_create.c.in \
+	keyval_free.c.in \
+	lookup_name.c.in \
+        message_c2f.c.in \
+        message_f2c.c.in \
+	mprobe.c.in \
+	mrecv.c.in \
+	neighbor_allgather.c.in \
+        neighbor_allgather_init.c.in \
+        neighbor_allgatherv.c.in \
+        neighbor_allgatherv_init.c.in \
+        neighbor_alltoall.c.in \
+        neighbor_alltoall_init.c.in \
+        neighbor_alltoallv.c.in \
+        neighbor_alltoallv_init.c.in\
+	neighbor_alltoallw.c.in \
+	neighbor_alltoallw_init.c.in \
+        open_port.c.in \
+	op_c2f.c.in \
+	op_commutative.c.in \
+	op_create.c.in \
+	op_free.c.in \
+	op_f2c.c.in \
+	pack.c.in \
+	pack_external.c.in \
+	pack_external_size.c.in \
+	pack_size.c.in \
+	parrived.c.in \
+	pready.c.in \
+	pready_list.c.in \
+	pready_range.c.in \
+	precv_init.c.in \
+	probe.c.in \
+	psend_init.c.in \
+	publish_name.c.in \
+	put.c.in \
+	query_thread.c.in \
+	raccumulate.c.in \
+        recv.c.in \
+	recv_init.c.in \
+	reduce.c.in \
+	reduce_init.c.in \
+	reduce_local.c.in \
+	reduce_scatter_block.c.in \
+	reduce_scatter_block_init.c.in \
+	reduce_scatter.c.in \
+	reduce_scatter_init.c.in \
+	register_datarep.c.in \
+        request_c2f.c.in \
+        request_f2c.c.in \
+	request_free.c.in \
+	request_get_status.c.in \
+	rget_accumulate.c.in \
+	rget.c.in \
+	rput.c.in \
+	rsend.c.in \
+	rsend_init.c.in \
+        scan.c.in \
+	scan_init.c.in \
+	scatter.c.in \
+	scatter_init.c.in \
+	scatterv.c.in \
+	scatterv_init.c.in \
+        send.c.in \
+	send_init.c.in \
+	sendrecv.c.in \
+	sendrecv_replace.c.in \
+	session_c2f.c.in \
+	session_call_errhandler.c.in \
+	session_create_errhandler.c.in \
+	session_f2c.c.in \
+	session_finalize.c.in \
+	session_get_errhandler.c.in \
+	session_get_info.c.in \
+	session_get_nth_pset.c.in \
+	session_get_num_psets.c.in \
+	session_get_pset_info.c.in \
+	session_init.c.in \
+	session_set_errhandler.c.in \
+	session_set_info.c.in \
+	ssend.c.in \
+	ssend_init.c.in \
+	startall.c.in \
+	start.c.in \
+	status_c2f08.c.in \
+	status_c2f.c.in \
+	status_f082c.c.in \
+	status_f082f.c.in \
+	status_f2c.c.in \
+	status_f2f08.c.in \
+        status_get_error.c.in \
+        status_get_source.c.in \
+        status_get_tag.c.in \
+	status_set_cancelled.c.in \
+	status_set_elements.c.in \
+	status_set_elements_x.c.in \
+        status_set_error.c.in \
+        status_set_source.c.in \
+        status_set_tag.c.in \
+	testall.c.in \
+	testany.c.in \
+	test.c.in \
+	test_cancelled.c.in \
+	testsome.c.in \
+	topo_test.c.in \
+	type_c2f.c.in \
+	type_commit.c.in \
+	type_contiguous.c.in \
+	type_create_darray.c.in \
+	type_create_f90_complex.c.in \
+	type_create_f90_integer.c.in \
+	type_create_f90_real.c.in \
+        type_create_hindexed.c.in \
+	type_create_hindexed_block.c.in \
+	type_create_hvector.c.in \
+	type_create_indexed_block.c.in \
+	type_create_keyval.c.in \
+	type_create_resized.c.in \
+        type_create_struct.c.in \
+	type_create_subarray.c.in \
+	type_delete_attr.c.in \
+	type_dup.c.in \
+	type_f2c.c.in \
+	type_free.c.in \
+	type_free_keyval.c.in \
+	type_get_attr.c.in \
+	type_get_extent.c.in \
+	type_get_extent_x.c.in \
+	type_get_name.c.in \
+	type_get_true_extent.c.in \
+	type_get_true_extent_x.c.in \
+	type_indexed.c.in \
+	type_match_size.c.in \
+	type_set_attr.c.in \
+	type_set_name.c.in \
+	type_size.c.in \
+	type_size_x.c.in \
+	type_vector.c.in \
+	unpack.c.in \
+	unpack_external.c.in \
+	unpublish_name.c.in \
+        waitall.c.in \
+	waitany.c.in \
+	wait.c.in \
+        waitsome.c.in \
+	win_allocate.c.in \
+	win_allocate_shared.c.in \
+        win_attach.c.in \
+        win_c2f.c.in \
+        win_call_errhandler.c.in \
+        win_complete.c.in  \
+	win_create.c.in \
+	win_create_dynamic.c.in \
+	win_create_errhandler.c.in \
+	win_create_keyval.c.in \
+        win_delete_attr.c.in \
+        win_detach.c.in \
+        win_fence.c.in \
+        win_flush_all.c.in \
+        win_flush.c.in \
+        win_flush_local_all.c.in \
+        win_flush_local.c.in \
+	win_free.c.in \
+        win_free_keyval.c.in \
+	win_get_attr.c.in \
+	win_get_errhandler.c.in \
+	win_get_group.c.in \
+	win_get_info.c.in \
+	win_get_name.c.in \
+	win_lock_all.c.in \
+	win_lock.c.in \
+	win_post.c.in \
+	win_set_attr.c.in \
+	win_set_errhandler.c.in \
+	win_set_info.c.in \
+	win_set_name.c.in \
+	win_shared_query.c.in \
+	win_start.c.in \
+	win_sync.c.in \
+	win_test.c.in \
+	win_unlock_all.c.in \
+	win_unlock.c.in \
+	win_wait.c.in \
+        wtime.c.in
+
+# Include template files in case someone wants to update them
+EXTRA_DIST = $(prototype_sources)
+
+# attr_fn.c contains attribute manipulation functions which do not
+# profiling implications, and so are always built.
 libmpi_c_la_SOURCES = \
         attr_fn.c
+libmpi_c_la_LIBADD = libmpi_c_profile.la
+if BUILD_MPI_BINDINGS_LAYER
+libmpi_c_la_LIBADD += libmpi_c_noprofile.la
+endif
+
+# Conditionally install the header files
+if WANT_INSTALL_HEADERS
+ompidir = $(ompiincludedir)/$(subdir)
+ompi_HEADERS = $(headers)
+endif
 
 #
-# libmpi_c_mpi.la is only built in some cases (see above)
+# List of all C files that have profile versions (generated_*.c files were
+# generated from prototype_sources above).
 #
+#
+interface_profile_sources = $(prototype_sources:.c.in=_generated.c)
+
 
-libmpi_c_mpi_la_SOURCES = \
-        abort.c \
-        add_error_class.c \
-        add_error_code.c \
-        add_error_string.c \
-        allgather.c \
-        iallgather.c \
-        allgatherv.c \
-        iallgatherv.c \
-        alloc_mem.c \
-        allreduce.c \
-        iallreduce.c \
-        alltoall.c \
-        ialltoall.c \
-        alltoallv.c \
-        ialltoallv.c \
-        alltoallw.c \
-        ialltoallw.c \
-        attr_delete.c \
-        attr_get.c \
-        attr_put.c \
-        barrier.c \
-        ibarrier.c \
-        bcast.c \
-        ibcast.c \
-        bsend.c \
-        bsend_init.c \
-        buffer_attach.c \
-        buffer_detach.c \
-        cancel.c \
-        cart_coords.c \
-        cart_create.c \
-        cartdim_get.c \
-        cart_get.c \
-        cart_map.c \
-        cart_rank.c \
-        cart_shift.c \
-        cart_sub.c \
-        close_port.c \
-        comm_accept.c \
-        comm_c2f.c \
-        comm_call_errhandler.c \
-        comm_compare.c \
-        comm_connect.c \
-        comm_create.c \
-        comm_create_errhandler.c \
-        comm_create_group.c \
-        comm_create_keyval.c \
-        comm_delete_attr.c \
-        comm_disconnect.c \
-        comm_dup.c \
-        comm_dup_with_info.c \
-        comm_idup.c \
-        comm_f2c.c \
-        comm_free.c \
-        comm_free_keyval.c \
-        comm_get_attr.c \
-        comm_get_errhandler.c \
-        comm_get_info.c \
-        comm_get_name.c \
-        comm_get_parent.c \
-        comm_group.c \
-        comm_join.c \
-        comm_rank.c \
-        comm_remote_group.c \
-        comm_remote_size.c \
-        comm_set_attr.c \
-        comm_set_info.c \
-        dist_graph_create.c \
-        dist_graph_create_adjacent.c \
-        dist_graph_neighbors.c \
-        dist_graph_neighbors_count.c \
-        comm_set_errhandler.c \
-        comm_set_name.c \
-        comm_size.c \
-        comm_spawn.c \
-        comm_spawn_multiple.c \
-        comm_split.c \
-        comm_split_type.c \
-        comm_test_inter.c \
-	compare_and_swap.c \
-        dims_create.c \
-        errhandler_c2f.c \
-        errhandler_f2c.c \
-        errhandler_free.c \
-        error_class.c \
-        error_string.c \
-        exscan.c \
-	fetch_and_op.c \
-        iexscan.c \
-        file_c2f.c \
-        file_call_errhandler.c \
-        file_close.c \
-        file_create_errhandler.c \
-        file_delete.c \
-        file_f2c.c \
-        file_get_amode.c \
-        file_get_atomicity.c \
-        file_get_byte_offset.c \
-        file_get_errhandler.c \
-        file_get_group.c \
-        file_get_info.c \
-        file_get_position.c \
-        file_get_position_shared.c \
-        file_get_size.c \
-        file_get_type_extent.c \
-        file_get_view.c \
-        file_iread_at.c \
-        file_iread_at_all.c \
-        file_iread.c \
-        file_iread_all.c \
-        file_iread_shared.c \
-        file_iwrite_at.c \
-        file_iwrite_at_all.c \
-        file_iwrite.c \
-        file_iwrite_all.c \
-        file_iwrite_shared.c \
-        file_open.c \
-        file_preallocate.c \
-        file_read_all_begin.c \
-        file_read_all.c \
-        file_read_all_end.c \
-        file_read_at_all_begin.c \
-        file_read_at_all.c \
-        file_read_at_all_end.c \
-        file_read_at.c \
-        file_read.c \
-        file_read_ordered_begin.c \
-        file_read_ordered.c \
-        file_read_ordered_end.c \
-        file_read_shared.c \
-        file_seek.c \
-        file_seek_shared.c \
-        file_set_atomicity.c \
-        file_set_errhandler.c \
-        file_set_info.c \
-        file_set_size.c \
-        file_set_view.c \
-        file_sync.c \
-        file_write_all_begin.c \
-        file_write_all.c \
-        file_write_all_end.c \
-        file_write_at_all_begin.c \
-        file_write_at_all.c \
-        file_write_at_all_end.c \
-        file_write_at.c \
-        file_write.c \
-        file_write_ordered_begin.c \
-        file_write_ordered.c \
-        file_write_ordered_end.c \
-        file_write_shared.c \
-        finalize.c \
-        finalized.c \
-        free_mem.c \
-        gather.c \
-        igather.c \
-        gatherv.c \
-        igatherv.c \
-        get_address.c \
-        get_count.c \
-        get_elements.c \
-        get_elements_x.c \
-	get_accumulate.c \
-        get_library_version.c \
-        get_processor_name.c \
-        get_version.c \
-        graph_create.c \
-        graph_get.c \
-        graph_map.c \
-        graph_neighbors_count.c \
-        graph_neighbors.c \
-        graphdims_get.c \
-        grequest_complete.c \
-        grequest_start.c \
-        group_c2f.c \
-        group_compare.c \
-        group_difference.c \
-        group_excl.c \
-        group_f2c.c \
-        group_free.c \
-        group_incl.c \
-        group_intersection.c \
-        group_range_excl.c \
-        group_range_incl.c \
-        group_rank.c \
-        group_size.c \
-        group_translate_ranks.c \
-        group_union.c \
-        ibsend.c \
-        improbe.c \
-        imrecv.c \
-        info_c2f.c \
-        info_create.c \
-        info_delete.c \
-        info_dup.c \
-        info_f2c.c \
-        info_free.c \
-        info_get.c \
-        info_get_nkeys.c \
-        info_get_nthkey.c \
-        info_get_valuelen.c \
-        info_set.c \
-        init.c \
-        init_thread.c \
-        initialized.c \
-        intercomm_create.c \
-        intercomm_merge.c \
-        iprobe.c \
-        irecv.c \
-        irsend.c \
-        is_thread_main.c \
-        isend.c \
-        issend.c \
-        lookup_name.c \
-        message_f2c.c \
-        message_c2f.c \
-        mprobe.c \
-        mrecv.c \
-        neighbor_allgather.c \
-        ineighbor_allgather.c \
-        neighbor_allgatherv.c \
-        ineighbor_allgatherv.c \
-        neighbor_alltoall.c \
-        ineighbor_alltoall.c \
-        neighbor_alltoallv.c \
-        ineighbor_alltoallv.c \
-        neighbor_alltoallw.c \
-        ineighbor_alltoallw.c \
-        keyval_create.c \
-        keyval_free.c \
-        op_c2f.c \
-        op_commutative.c \
-        op_create.c \
-        op_f2c.c \
-        op_free.c \
-        open_port.c \
-        pack_external.c \
-        pack_external_size.c \
-        pack.c \
-        pack_size.c \
+# The following are special case functions where we
+# have to deal manually
+#
+interface_profile_sources += \
         pcontrol.c \
-        probe.c \
-        publish_name.c \
-        query_thread.c \
-	raccumulate.c \
-        recv_init.c \
-        recv.c \
-        reduce.c \
-        register_datarep.c \
-        ireduce.c \
-        reduce_local.c \
-        reduce_scatter.c \
-        ireduce_scatter.c \
-        reduce_scatter_block.c \
-        ireduce_scatter_block.c \
-        request_c2f.c \
-        request_f2c.c \
-        request_free.c \
-        request_get_status.c \
-	rget.c \
-	rget_accumulate.c \
-	rput.c \
-        rsend_init.c \
-        rsend.c \
-        scan.c \
-        iscan.c \
-        scatter.c \
-        iscatter.c \
-        scatterv.c \
-        iscatterv.c \
-        send.c \
-        send_init.c \
-        sendrecv.c \
-        sendrecv_replace.c \
-        ssend_init.c \
-        ssend.c \
-        start.c \
-        startall.c \
-        status_c2f.c \
-        status_c2f08.c \
-        status_f082c.c \
-        status_f082f.c \
-        status_f2c.c \
-        status_f2f08.c \
-        status_set_cancelled.c \
-        status_set_elements.c \
-        status_set_elements_x.c \
-        testall.c \
-        testany.c \
-        test.c \
-        test_cancelled.c \
-        testsome.c \
-        topo_test.c \
-        type_c2f.c \
-        type_commit.c \
-        type_contiguous.c \
-        type_create_darray.c \
-        type_create_f90_complex.c \
-        type_create_f90_integer.c \
-        type_create_f90_real.c \
-        type_create_hindexed.c \
-        type_create_hvector.c \
-        type_create_indexed_block.c \
-        type_create_hindexed_block.c \
-        type_create_keyval.c \
-        type_create_resized.c \
-        type_create_struct.c \
-        type_create_subarray.c \
-        type_delete_attr.c \
-        type_dup.c \
-        type_f2c.c \
-        type_free.c \
-        type_free_keyval.c \
-        type_get_attr.c \
-        type_get_contents.c \
+	type_get_contents.c \
+	type_get_contents_c.c \
         type_get_envelope.c \
-        type_get_extent.c \
-        type_get_extent_x.c \
-        type_get_name.c \
-        type_get_true_extent.c \
-        type_get_true_extent_x.c \
-        type_indexed.c \
-        type_match_size.c \
-        type_set_attr.c \
-        type_set_name.c \
-        type_size.c \
-        type_size_x.c \
-        type_vector.c \
-        unpack_external.c \
-        unpack.c \
-        unpublish_name.c \
-        wait.c \
-        waitall.c \
-        waitany.c \
-        waitsome.c \
-        wtime.c \
-        wtick.c \
-        accumulate.c \
-        get.c \
-        put.c \
-	win_allocate.c \
-	win_allocate_shared.c \
-	win_attach.c \
-        win_c2f.c \
-        win_call_errhandler.c \
-        win_complete.c  \
-        win_create_errhandler.c \
-        win_create_keyval.c \
-        win_create.c \
-	win_create_dynamic.c \
-        win_delete_attr.c \
-	win_detach.c \
+        type_get_envelope_c.c \
         win_f2c.c \
-        win_fence.c \
-	win_flush.c \
-	win_flush_all.c \
-	win_flush_local.c \
-	win_flush_local_all.c \
-        win_free_keyval.c \
-        win_free.c \
-        win_get_attr.c \
-        win_get_errhandler.c \
-        win_get_group.c \
-        win_get_info.c \
-        win_get_name.c  \
-        win_lock.c \
-	win_lock_all.c \
-        win_post.c \
-        win_set_attr.c \
-        win_set_errhandler.c \
-        win_set_info.c \
-        win_set_name.c \
-	win_shared_query.c \
-	win_sync.c \
-        win_start.c \
-        win_test.c \
-        win_unlock.c \
-	win_unlock_all.c \
-        win_wait.c
-
+        wtick.c
 
-if OMPI_ENABLE_MPI1_COMPAT
-libmpi_c_mpi_la_SOURCES += \
+# The following functions were removed from the MPI standard, but are
+# retained for ABI compliance reasons.  They are listed independently
+# of the other MPI functions in case we one day change behavior around
+# ABI compliance.
+interface_profile_sources += \
         address.c \
         errhandler_create.c \
         errhandler_get.c \
@@ -453,11 +531,25 @@ libmpi_c_mpi_la_SOURCES += \
         type_lb.c \
         type_struct.c \
         type_ub.c
-endif
 
-# Conditionally install the header files
+libmpi_c_profile_la_SOURCES = $(interface_profile_sources)
+libmpi_c_profile_la_CPPFLAGS = -DOMPI_BUILD_MPI_PROFILING=1
+
+libmpi_c_noprofile_la_SOURCES = $(interface_profile_sources)
+libmpi_c_noprofile_la_CPPFLAGS = -DOMPI_BUILD_MPI_PROFILING=0
+
+# ABI generation rules
+if OMPI_GENERATE_BINDINGS
+%_generated.c: %.c.in
+	$(OMPI_V_GEN) $(PYTHON) $(top_srcdir)/ompi/mpi/bindings/bindings.py \
+	    --builddir $(abs_top_builddir) \
+	    --srcdir $(abs_top_srcdir) \
+	    --output $@ \
+	    c \
+	    source \
+	    ompi \
+	    $<
 
-if WANT_INSTALL_HEADERS
-ompidir = $(ompiincludedir)/$(subdir)
-ompi_HEADERS = $(headers)
 endif
+# Delete generated files on maintainer-clean
+MAINTAINERCLEANFILES = *_generated.c
diff --git a/ompi/mpi/c/abort.c b/ompi/mpi/c/abort.c
deleted file mode 100644
index 889fe1738b3..00000000000
--- a/ompi/mpi/c/abort.c
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007-2008 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include "opal/util/show_help.h"
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/runtime/mpiruntime.h"
-#include "ompi/memchecker.h"
-#include "ompi/communicator/communicator.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Abort = PMPI_Abort
-#endif
-#define MPI_Abort PMPI_Abort
-#endif
-
-static const char FUNC_NAME[] = "MPI_Abort";
-
-
-int MPI_Abort(MPI_Comm comm, int errorcode)
-{
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    /* Don't even bother checking comm and errorcode values for
-       errors */
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-    }
-
-    opal_show_help("help-mpi-api.txt", "mpi-abort", true,
-                   ompi_comm_rank(comm),
-                   ('\0' != comm->c_name[0]) ? comm->c_name : "<Unknown>",
-                   OMPI_NAME_PRINT(OMPI_PROC_MY_NAME),
-                   errorcode);
-    return ompi_mpi_abort(comm, errorcode);
-}
diff --git a/ompi/mpi/c/abort.c.in b/ompi/mpi/c/abort.c.in
new file mode 100644
index 00000000000..f8fe6381ec3
--- /dev/null
+++ b/ompi/mpi/c/abort.c.in
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007-2008 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "opal/util/show_help.h"
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/runtime/mpiruntime.h"
+#include "ompi/memchecker.h"
+#include "ompi/communicator/communicator.h"
+
+PROTOTYPE ERROR_CLASS abort(COMM comm, INT errorcode)
+{
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    /* Don't even bother checking comm and errorcode values for
+       errors */
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+    }
+
+    opal_show_help("help-mpi-api.txt", "mpi-abort", true,
+                   ompi_comm_rank(comm),
+                   ('\0' != comm->c_name[0]) ? comm->c_name : "<Unknown>",
+                   OMPI_NAME_PRINT(OMPI_PROC_MY_NAME),
+                   errorcode);
+    return ompi_mpi_abort(comm, errorcode);
+}
diff --git a/ompi/mpi/c/accumulate.c b/ompi/mpi/c/accumulate.c
deleted file mode 100644
index ae9343dce3b..00000000000
--- a/ompi/mpi/c/accumulate.c
+++ /dev/null
@@ -1,137 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2009      Sun Microsystmes, Inc.  All rights reserved.
- * Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-#include "ompi/mca/osc/osc.h"
-#include "ompi/op/op.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/datatype/ompi_datatype_internal.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Accumulate = PMPI_Accumulate
-#endif
-#define MPI_Accumulate PMPI_Accumulate
-#endif
-
-static const char FUNC_NAME[] = "MPI_Accumulate";
-
-int MPI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype,
-                   int target_rank, MPI_Aint target_disp, int target_count,
-                   MPI_Datatype target_datatype, MPI_Op op, MPI_Win win)
-{
-    int rc;
-    ompi_win_t *ompi_win = (ompi_win_t*) win;
-
-    MEMCHECKER(
-        memchecker_datatype(origin_datatype);
-        memchecker_datatype(target_datatype);
-        memchecker_call(&opal_memchecker_base_isdefined, (void *) origin_addr, origin_count, origin_datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = OMPI_SUCCESS;
-
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        } else if (origin_count < 0 || target_count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else if (ompi_win_peer_invalid(win, target_rank) &&
-                   (MPI_PROC_NULL != target_rank)) {
-            rc = MPI_ERR_RANK;
-        } else if (MPI_OP_NULL == op || MPI_NO_OP == op) {
-            rc = MPI_ERR_OP;
-        } else if (!ompi_op_is_intrinsic(op)) {
-            rc = MPI_ERR_OP;
-        } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) {
-            rc = MPI_ERR_DISP;
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, origin_datatype, origin_count);
-            if (OMPI_SUCCESS == rc) {
-                OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, target_datatype, target_count);
-            }
-            if (OMPI_SUCCESS == rc) {
-                /* While technically the standard probably requires that the
-                   datatypes used with MPI_REPLACE conform to all the rules
-                   for other reduction operators, we don't require such
-                   behavior, as checking for it is expensive here and we don't
-                   care in implementation.. */
-                if (op != &ompi_mpi_op_replace.op && op != &ompi_mpi_op_no_op.op) {
-                    ompi_datatype_t *op_check_dt, *origin_check_dt;
-                    char *msg;
-
-                    /* ACCUMULATE, unlike REDUCE, can use with derived
-                       datatypes with predefinied operations, with some
-                       restrictions outlined in MPI-3:11.3.4.  The derived
-                       datatype must be composed entierly from one predefined
-                       datatype (so you can do all the construction you want,
-                       but at the bottom, you can only use one datatype, say,
-                       MPI_INT).  If the datatype at the target isn't
-                       predefined, then make sure it's composed of only one
-                       datatype, and check that datatype against
-                       ompi_op_is_valid(). */
-                    origin_check_dt = ompi_datatype_get_single_predefined_type_from_args(origin_datatype);
-                    op_check_dt = ompi_datatype_get_single_predefined_type_from_args(target_datatype);
-
-                    if( !((origin_check_dt == op_check_dt) & (NULL != op_check_dt)) ) {
-                        OMPI_ERRHANDLER_RETURN(MPI_ERR_ARG, win, MPI_ERR_ARG, FUNC_NAME);
-                    }
-
-                    /* check to make sure primitive type is valid for
-                       reduction.  Should do this on the target, but
-                       then can't get the errcode back for this
-                       call */
-                    if (!ompi_op_is_valid(op, op_check_dt, &msg, FUNC_NAME)) {
-                        int ret = OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_OP, msg);
-                        free(msg);
-                        return ret;
-                    }
-                }
-            }
-        }
-        OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME);
-    }
-
-    if (MPI_PROC_NULL == target_rank) {
-        return MPI_SUCCESS;
-    }
-
-    rc = ompi_win->w_osc_module->osc_accumulate(origin_addr,
-                                                origin_count,
-                                                origin_datatype,
-                                                target_rank,
-                                                target_disp,
-                                                target_count,
-                                                target_datatype,
-                                                op, win);
-    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/accumulate.c.in b/ompi/mpi/c/accumulate.c.in
new file mode 100644
index 00000000000..d778f5dc822
--- /dev/null
+++ b/ompi/mpi/c/accumulate.c.in
@@ -0,0 +1,131 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2009      Sun Microsystmes, Inc.  All rights reserved.
+ * Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+#include "ompi/mca/osc/osc.h"
+#include "ompi/op/op.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/datatype/ompi_datatype_internal.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS accumulate(BUFFER origin_addr, COUNT origin_count, DATATYPE origin_datatype,
+                                 INT target_rank, AINT target_disp, COUNT target_count,
+                                 DATATYPE target_datatype, OP op, WIN win)
+{
+    int rc;
+    ompi_win_t *ompi_win = (ompi_win_t*) win;
+
+    MEMCHECKER(
+        memchecker_datatype(origin_datatype);
+        memchecker_datatype(target_datatype);
+        memchecker_call(&opal_memchecker_base_isdefined, (void *) origin_addr, origin_count, origin_datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = OMPI_SUCCESS;
+
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        } else if (origin_count < 0 || target_count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else if (ompi_win_peer_invalid(win, target_rank) &&
+                   (MPI_PROC_NULL != target_rank)) {
+            rc = MPI_ERR_RANK;
+        } else if (MPI_OP_NULL == op || MPI_NO_OP == op) {
+            rc = MPI_ERR_OP;
+        } else if (!ompi_op_is_intrinsic(op)) {
+            rc = MPI_ERR_OP;
+        } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) {
+            rc = MPI_ERR_DISP;
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, origin_datatype, origin_count);
+            if (OMPI_SUCCESS == rc) {
+                OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, target_datatype, target_count);
+            }
+            if (OMPI_SUCCESS == rc) {
+                /* While technically the standard probably requires that the
+                   datatypes used with MPI_REPLACE conform to all the rules
+                   for other reduction operators, we don't require such
+                   behavior, as checking for it is expensive here and we don't
+                   care in implementation.. */
+                if (op != &ompi_mpi_op_replace.op && op != &ompi_mpi_op_no_op.op) {
+                    ompi_datatype_t *op_check_dt, *origin_check_dt;
+                    char *msg;
+
+                    /* ACCUMULATE, unlike REDUCE, can use with derived
+                       datatypes with predefinied operations, with some
+                       restrictions outlined in MPI-3:11.3.4.  The derived
+                       datatype must be composed entirely from one predefined
+                       datatype (so you can do all the construction you want,
+                       but at the bottom, you can only use one datatype, say,
+                       MPI_INT).  If the datatype at the target isn't
+                       predefined, then make sure it's composed of only one
+                       datatype, and check that datatype against
+                       ompi_op_is_valid(). */
+                    origin_check_dt = ompi_datatype_get_single_predefined_type_from_args(origin_datatype);
+                    op_check_dt = ompi_datatype_get_single_predefined_type_from_args(target_datatype);
+
+                    if( !((origin_check_dt == op_check_dt) & (NULL != op_check_dt)) ) {
+                        OMPI_ERRHANDLER_RETURN(MPI_ERR_ARG, win, MPI_ERR_ARG, FUNC_NAME);
+                    }
+
+                    /* check to make sure primitive type is valid for
+                       reduction.  Should do this on the target, but
+                       then can't get the errcode back for this
+                       call */
+                    if (!ompi_op_is_valid(op, op_check_dt, &msg, FUNC_NAME)) {
+                        int ret = OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_OP, msg);
+                        free(msg);
+                        return ret;
+                    }
+                }
+            }
+        }
+        OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME);
+    }
+
+    if (MPI_PROC_NULL == target_rank) {
+        return MPI_SUCCESS;
+    }
+
+    rc = ompi_win->w_osc_module->osc_accumulate(origin_addr,
+                                                origin_count,
+                                                origin_datatype,
+                                                target_rank,
+                                                target_disp,
+                                                target_count,
+                                                target_datatype,
+                                                op, win);
+    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/add_error_class.c b/ompi/mpi/c/add_error_class.c
deleted file mode 100644
index f0756e0ad5f..00000000000
--- a/ompi/mpi/c/add_error_class.c
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      University of Houston. All rights reserved.
- * Copyright (c) 2015-2017 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/errhandler/errcode.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/attribute/attribute.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Add_error_class = PMPI_Add_error_class
-#endif
-#define MPI_Add_error_class PMPI_Add_error_class
-#endif
-
-static const char FUNC_NAME[] = "MPI_Add_error_class";
-
-
-int MPI_Add_error_class(int *errorclass)
-{
-    int err_class;
-    int rc;
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (NULL == errorclass) {
-            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD,
-                                           MPI_ERR_ARG, FUNC_NAME);
-        }
-    }
-
-    err_class = ompi_mpi_errclass_add();
-    if ( 0 > err_class ) {
-        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INTERN,
-                                      FUNC_NAME);
-    }
-
-
-    /*
-    ** Update the attribute value. See the comments
-    ** in attribute/attribute.c and attribute/attribute_predefined.c
-    ** why we have to call the fortran attr_set function
-    */
-    rc  = ompi_attr_set_fint (COMM_ATTR,
-                              MPI_COMM_WORLD,
-                              &MPI_COMM_WORLD->c_keyhash,
-                              MPI_LASTUSEDCODE,
-                              ompi_mpi_errcode_lastused,
-                              true);
-    if ( MPI_SUCCESS != rc ) {
-	return OMPI_ERRHANDLER_NOHANDLE_INVOKE(rc, FUNC_NAME);
-    }
-
-    *errorclass = err_class;
-    return MPI_SUCCESS;
-}
-
diff --git a/ompi/mpi/c/add_error_class.c.in b/ompi/mpi/c/add_error_class.c.in
new file mode 100644
index 00000000000..624be311675
--- /dev/null
+++ b/ompi/mpi/c/add_error_class.c.in
@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      University of Houston. All rights reserved.
+ * Copyright (c) 2015-2017 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/errhandler/errcode.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/attribute/attribute.h"
+
+PROTOTYPE ERROR_CLASS add_error_class(INT_OUT errorclass)
+{
+    int err_class;
+    int rc;
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (NULL == errorclass) {
+            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD,
+                                           MPI_ERR_ARG, FUNC_NAME);
+        }
+    }
+
+    err_class = ompi_mpi_errclass_add();
+    if ( 0 > err_class ) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INTERN,
+                                      FUNC_NAME);
+    }
+
+
+    /*
+    ** Update the attribute value. See the comments
+    ** in attribute/attribute.c and attribute/attribute_predefined.c
+    ** why we have to call the fortran attr_set function
+    */
+    rc  = ompi_attr_set_fint (COMM_ATTR,
+                              MPI_COMM_WORLD,
+                              &MPI_COMM_WORLD->c_keyhash,
+                              MPI_LASTUSEDCODE,
+                              ompi_mpi_errcode_lastused,
+                              true);
+    if ( MPI_SUCCESS != rc ) {
+	return OMPI_ERRHANDLER_NOHANDLE_INVOKE(rc, FUNC_NAME);
+    }
+
+    *errorclass = err_class;
+    return MPI_SUCCESS;
+}
+
diff --git a/ompi/mpi/c/add_error_code.c b/ompi/mpi/c/add_error_code.c
deleted file mode 100644
index 288926e7f38..00000000000
--- a/ompi/mpi/c/add_error_code.c
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      University of Houston. All rights reserved.
- * Copyright (c) 2015-2017 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/errhandler/errcode.h"
-#include "ompi/attribute/attribute.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Add_error_code = PMPI_Add_error_code
-#endif
-#define MPI_Add_error_code PMPI_Add_error_code
-#endif
-
-static const char FUNC_NAME[] = "MPI_Add_error_code";
-
-
-int MPI_Add_error_code(int errorclass, int *errorcode)
-{
-    int code;
-    int rc;
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if ( ompi_mpi_errcode_is_invalid(errorclass) )
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-
-	if ( !ompi_mpi_errnum_is_class ( errorclass) )
-	    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-					  FUNC_NAME);
-
-        if (NULL == errorcode) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(
-                                          MPI_ERR_ARG, FUNC_NAME);
-        }
-    }
-
-    code = ompi_mpi_errcode_add ( errorclass);
-    if ( 0 > code ) {
-        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INTERN,
-                                      FUNC_NAME);
-    }
-
-    /*
-    ** Update the attribute value. See the comments
-    ** in attribute/attribute.c and attribute/attribute_predefined.c
-    ** why we have to call the fortran attr_set function
-    */
-    rc  = ompi_attr_set_fint (COMM_ATTR,
-                              MPI_COMM_WORLD,
-                              &MPI_COMM_WORLD->c_keyhash,
-                              MPI_LASTUSEDCODE,
-                              ompi_mpi_errcode_lastused,
-                              true);
-    if ( MPI_SUCCESS != rc ) {
-	return OMPI_ERRHANDLER_NOHANDLE_INVOKE(rc, FUNC_NAME);
-    }
-
-    *errorcode = code;
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/add_error_code.c.in b/ompi/mpi/c/add_error_code.c.in
new file mode 100644
index 00000000000..a4755ff275e
--- /dev/null
+++ b/ompi/mpi/c/add_error_code.c.in
@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      University of Houston. All rights reserved.
+ * Copyright (c) 2015-2017 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/errhandler/errcode.h"
+#include "ompi/attribute/attribute.h"
+
+PROTOTYPE ERROR_CLASS add_error_code(INT errorclass, INT_OUT errorcode)
+{
+    int code;
+    int rc;
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if ( ompi_mpi_errcode_is_invalid(errorclass) )
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+
+	if ( !ompi_mpi_errnum_is_class ( errorclass) )
+	    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+					  FUNC_NAME);
+
+        if (NULL == errorcode) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(
+                                          MPI_ERR_ARG, FUNC_NAME);
+        }
+    }
+
+    code = ompi_mpi_errcode_add ( errorclass);
+    if ( 0 > code ) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INTERN,
+                                      FUNC_NAME);
+    }
+
+    /*
+    ** Update the attribute value. See the comments
+    ** in attribute/attribute.c and attribute/attribute_predefined.c
+    ** why we have to call the fortran attr_set function
+    */
+    rc  = ompi_attr_set_fint (COMM_ATTR,
+                              MPI_COMM_WORLD,
+                              &MPI_COMM_WORLD->c_keyhash,
+                              MPI_LASTUSEDCODE,
+                              ompi_mpi_errcode_lastused,
+                              true);
+    if ( MPI_SUCCESS != rc ) {
+	return OMPI_ERRHANDLER_NOHANDLE_INVOKE(rc, FUNC_NAME);
+    }
+
+    *errorcode = code;
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/add_error_string.c b/ompi/mpi/c/add_error_string.c
deleted file mode 100644
index e367745c308..00000000000
--- a/ompi/mpi/c/add_error_string.c
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      University of Houston. All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-#include <string.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/errhandler/errcode.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Add_error_string = PMPI_Add_error_string
-#endif
-#define MPI_Add_error_string PMPI_Add_error_string
-#endif
-
-static const char FUNC_NAME[] = "MPI_Add_error_string";
-
-
-int MPI_Add_error_string(int errorcode, const char *string)
-{
-    int rc;
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if ( ompi_mpi_errcode_is_invalid(errorcode) )
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-
-	if ( ompi_mpi_errcode_is_predefined(errorcode) )
-	    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-					  FUNC_NAME);
-
-	if ( MPI_MAX_ERROR_STRING < (strlen(string)+1) )
-	    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-					  FUNC_NAME);
-    }
-
-    rc = ompi_mpi_errnum_add_string (errorcode, string, (int)(strlen(string)+1));
-    if ( OMPI_SUCCESS != rc ) {
-        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INTERN,
-                                      FUNC_NAME);
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/add_error_string.c.in b/ompi/mpi/c/add_error_string.c.in
new file mode 100644
index 00000000000..d5e1e0e5d74
--- /dev/null
+++ b/ompi/mpi/c/add_error_string.c.in
@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      University of Houston. All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+#include <string.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/errhandler/errcode.h"
+
+PROTOTYPE ERROR_CLASS add_error_string(INT errorcode, STRING string)
+{
+    int rc;
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if ( ompi_mpi_errcode_is_invalid(errorcode) )
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+
+	if ( ompi_mpi_errcode_is_predefined(errorcode) )
+	    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+					  FUNC_NAME);
+
+	if ( MPI_MAX_ERROR_STRING < (strlen(string)+1) )
+	    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+					  FUNC_NAME);
+    }
+
+    rc = ompi_mpi_errnum_add_string (errorcode, string, (int)(strlen(string)+1));
+    if ( OMPI_SUCCESS != rc ) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INTERN,
+                                      FUNC_NAME);
+    }
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/allgather.c b/ompi/mpi/c/allgather.c
deleted file mode 100644
index 5cd9858d8e6..00000000000
--- a/ompi/mpi/c/allgather.c
+++ /dev/null
@@ -1,140 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010      University of Houston.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Allgather = PMPI_Allgather
-#endif
-#define MPI_Allgather PMPI_Allgather
-#endif
-
-static const char FUNC_NAME[] = "MPI_Allgather";
-
-
-int MPI_Allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                  void *recvbuf, int recvcount, MPI_Datatype recvtype,
-                  MPI_Comm comm)
-{
-    int err;
-
-    SPC_RECORD(OMPI_SPC_ALLGATHER, 1);
-
-    MEMCHECKER(
-        int rank;
-        ptrdiff_t ext;
-
-        rank = ompi_comm_rank(comm);
-        ompi_datatype_type_extent(recvtype, &ext);
-
-        memchecker_datatype(recvtype);
-        memchecker_comm(comm);
-        /* check whether the actual send buffer is defined. */
-        if (MPI_IN_PLACE == sendbuf) {
-            memchecker_call(&opal_memchecker_base_isdefined,
-                            (char *)(recvbuf)+rank*recvcount*ext,
-                            recvcount, recvtype);
-        } else {
-            memchecker_datatype(sendtype);
-            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-        }
-        /* check whether the receive buffer is addressable. */
-        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-
-        /* Unrooted operation -- same checks for all ranks on both
-           intracommunicators and intercommunicators */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-          OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
-        } else if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
-          err = MPI_ERR_TYPE;
-        } else if (recvcount < 0) {
-          err = MPI_ERR_COUNT;
-        } else if ((MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
-                   MPI_IN_PLACE == recvbuf) {
-          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        } else if (MPI_IN_PLACE != sendbuf) {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-        }
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
-        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-    }
-#endif
-
-    /* Do we need to do anything?  Everyone had to give the same send
-       signature, which means that everyone must have given a
-       sendcount > 0 if there's anything to send for the intra-communicator
-       case.  If we're doing IN_PLACE, however, check recvcount,
-       not sendcount. */
-    if ( OMPI_COMM_IS_INTRA(comm) ) {
-       if ((MPI_IN_PLACE != sendbuf && 0 == sendcount) ||
-            (0 == recvcount)) {
-            return MPI_SUCCESS;
-	   }
-    }
-    else if ( OMPI_COMM_IS_INTER(comm) ){
-        /* for inter comunicators, the communication pattern
-           need not be symmetric. Specifically, one group is
-	   allows to have sendcount=0, while the other has
-           a valid sendcount. Thus, the only way not to do
-           anything is if both sendcount and recvcount are zero. */
-	    if ( 0 == sendcount && 0 == recvcount ) {
-	        return MPI_SUCCESS;
-	    }
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-
-    err = comm->c_coll->coll_allgather(sendbuf, sendcount, sendtype,
-                                      recvbuf, recvcount, recvtype, comm,
-                                      comm->c_coll->coll_allgather_module);
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/allgather.c.in b/ompi/mpi/c/allgather.c.in
new file mode 100644
index 00000000000..ec9e57c1079
--- /dev/null
+++ b/ompi/mpi/c/allgather.c.in
@@ -0,0 +1,132 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010      University of Houston.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2018 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS allgather(BUFFER sendbuf, COUNT sendcount, DATATYPE sendtype,
+                                BUFFER_OUT recvbuf, COUNT recvcount, DATATYPE recvtype,
+                                COMM comm)
+{
+    int err;
+
+    SPC_RECORD(OMPI_SPC_ALLGATHER, 1);
+
+    MEMCHECKER(
+        int rank;
+        ptrdiff_t ext;
+
+        rank = ompi_comm_rank(comm);
+        ompi_datatype_type_extent(recvtype, &ext);
+
+        memchecker_datatype(recvtype);
+        memchecker_comm(comm);
+        /* check whether the actual send buffer is defined. */
+        if (MPI_IN_PLACE == sendbuf) {
+            memchecker_call(&opal_memchecker_base_isdefined,
+                            (char *)(recvbuf)+rank*recvcount*ext,
+                            recvcount, recvtype);
+        } else {
+            memchecker_datatype(sendtype);
+            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+        }
+        /* check whether the receive buffer is addressable. */
+        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+
+        /* Unrooted operation -- same checks for all ranks on both
+           intracommunicators and intercommunicators */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+          OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+        } else if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
+          err = MPI_ERR_TYPE;
+        } else if (recvcount < 0) {
+          err = MPI_ERR_COUNT;
+        } else if ((MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
+                   MPI_IN_PLACE == recvbuf) {
+          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        } else if (MPI_IN_PLACE != sendbuf) {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+        }
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
+        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+    }
+#endif
+
+    /* Do we need to do anything?  Everyone had to give the same send
+       signature, which means that everyone must have given a
+       sendcount > 0 if there's anything to send for the intra-communicator
+       case.  If we're doing IN_PLACE, however, check recvcount,
+       not sendcount. */
+    if ( OMPI_COMM_IS_INTRA(comm) ) {
+       if ((MPI_IN_PLACE != sendbuf && 0 == sendcount) ||
+            (0 == recvcount)) {
+            return MPI_SUCCESS;
+	   }
+    }
+    else if ( OMPI_COMM_IS_INTER(comm) ){
+        /* for inter comunicators, the communication pattern
+           need not be symmetric. Specifically, one group is
+	   allows to have sendcount=0, while the other has
+           a valid sendcount. Thus, the only way not to do
+           anything is if both sendcount and recvcount are zero. */
+	    if ( 0 == sendcount && 0 == recvcount ) {
+	        return MPI_SUCCESS;
+	    }
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+
+    err = comm->c_coll->coll_allgather(sendbuf, sendcount, sendtype,
+                                      recvbuf, recvcount, recvtype, comm,
+                                      comm->c_coll->coll_allgather_module);
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/allgather_init.c.in b/ompi/mpi/c/allgather_init.c.in
new file mode 100644
index 00000000000..54a14756231
--- /dev/null
+++ b/ompi/mpi/c/allgather_init.c.in
@@ -0,0 +1,101 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oak Ridge National Laboratory. All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS allgather_init(BUFFER sendbuf, COUNT sendcount, DATATYPE sendtype,
+                       BUFFER_OUT recvbuf, COUNT recvcount, DATATYPE recvtype,
+                       COMM comm, INFO info, REQUEST_INOUT request)
+{
+    int err;
+
+    SPC_RECORD(OMPI_SPC_ALLGATHER_INIT, 1);
+
+    MEMCHECKER(
+        int rank;
+        ptrdiff_t ext;
+
+        rank = ompi_comm_rank(comm);
+        ompi_datatype_type_extent(recvtype, &ext);
+
+        memchecker_datatype(recvtype);
+        memchecker_comm(comm);
+        /* check whether the actual send buffer is defined. */
+        if (MPI_IN_PLACE == sendbuf) {
+            memchecker_call(&opal_memchecker_base_isdefined,
+                            (char *)(recvbuf)+rank*ext,
+                            recvcount, recvtype);
+        } else {
+            memchecker_datatype(sendtype);
+            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+        }
+        /* check whether the receive buffer is addressable. */
+        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+
+        /* Unrooted operation -- same checks for all ranks on both
+           intracommunicators and intercommunicators */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+          OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+        } else if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
+          err = MPI_ERR_TYPE;
+        } else if (recvcount < 0) {
+          err = MPI_ERR_COUNT;
+        } else if ((MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
+                   MPI_IN_PLACE == recvbuf) {
+          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        } else if (MPI_IN_PLACE != sendbuf) {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+        }
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    err = comm->c_coll->coll_allgather_init(sendbuf, sendcount, sendtype,
+                                            recvbuf, recvcount, recvtype, comm,
+                                            info, request, comm->c_coll->coll_allgather_init_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_datatypes(*request, (MPI_IN_PLACE==sendbuf)?NULL:sendtype, recvtype);
+    }
+
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/allgatherv.c b/ompi/mpi/c/allgatherv.c
deleted file mode 100644
index 8d3ab33113c..00000000000
--- a/ompi/mpi/c/allgatherv.c
+++ /dev/null
@@ -1,161 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2010      University of Houston.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Allgatherv = PMPI_Allgatherv
-#endif
-#define MPI_Allgatherv PMPI_Allgatherv
-#endif
-
-static const char FUNC_NAME[] = "MPI_Allgatherv";
-
-
-int MPI_Allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                   void *recvbuf, const int recvcounts[],
-                   const int displs[], MPI_Datatype recvtype, MPI_Comm comm)
-{
-    int i, size, err;
-
-    SPC_RECORD(OMPI_SPC_ALLGATHERV, 1);
-
-    MEMCHECKER(
-        int rank;
-        ptrdiff_t ext;
-
-        rank = ompi_comm_rank(comm);
-        size = ompi_comm_size(comm);
-        ompi_datatype_type_extent(recvtype, &ext);
-
-        memchecker_datatype(recvtype);
-        memchecker_comm (comm);
-        /* check whether the receive buffer is addressable. */
-        for (i = 0; i < size; i++) {
-            memchecker_call(&opal_memchecker_base_isaddressable,
-                            (char *)(recvbuf)+displs[i]*ext,
-                            recvcounts[i], recvtype);
-        }
-
-        /* check whether the actual send buffer is defined. */
-        if (MPI_IN_PLACE == sendbuf) {
-            memchecker_call(&opal_memchecker_base_isdefined,
-                            (char *)(recvbuf)+displs[rank]*ext,
-                            recvcounts[rank], recvtype);
-        } else {
-            memchecker_datatype(sendtype);
-            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-
-        /* Unrooted operation -- same checks for all ranks on both
-           intracommunicators and intercommunicators */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if ((MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
-                   MPI_IN_PLACE == recvbuf) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        } else if (MPI_DATATYPE_NULL == recvtype) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
-        }
-
-        if (MPI_IN_PLACE != sendbuf) {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-        }
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-
-      /* We always define the remote group to be the same as the local
-         group in the case of an intracommunicator, so it's safe to
-         get the size of the remote group here for both intra- and
-         intercommunicators */
-
-        size = ompi_comm_remote_size(comm);
-        for (i = 0; i < size; ++i) {
-          if (recvcounts[i] < 0) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-          }
-        }
-
-        if (NULL == displs) {
-          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_BUFFER, FUNC_NAME);
-        }
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
-        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-    }
-#endif
-
-    /* Do we need to do anything?  Everyone had to give the same
-       signature, which means that everyone must have given a
-       sum(recvounts) > 0 if there's anything to do. */
-
-    if ( OMPI_COMM_IS_INTRA( comm) ) {
-	for (i = 0; i < ompi_comm_size(comm); ++i) {
-	    if (0 != recvcounts[i]) {
-		break;
-	    }
-	}
-	if (i >= ompi_comm_size(comm)) {
-	    return MPI_SUCCESS;
-	}
-    }
-    /* There is no rule that can be applied for inter-communicators, since
-       recvcount(s)=0 only indicates that the processes in the other group
-       do not send anything, sendcount=0 only indicates that I do not send
-       anything. However, other processes in my group might very well send
-       something */
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_allgatherv(sendbuf, sendcount, sendtype,
-                                       recvbuf, (int *) recvcounts,
-                                       (int *) displs, recvtype, comm,
-                                       comm->c_coll->coll_allgatherv_module);
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/allgatherv.c.in b/ompi/mpi/c/allgatherv.c.in
new file mode 100644
index 00000000000..e0501ce3887
--- /dev/null
+++ b/ompi/mpi/c/allgatherv.c.in
@@ -0,0 +1,158 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010      University of Houston.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+
+PROTOTYPE ERROR_CLASS allgatherv(BUFFER sendbuf, COUNT sendcount, DATATYPE sendtype,
+                   BUFFER_OUT recvbuf, COUNT_ARRAY recvcounts,
+                   DISP_ARRAY displs, DATATYPE recvtype, COMM comm)
+{
+    int i, size, err;
+    ompi_count_array_t recvcounts_desc;
+    ompi_disp_array_t displs_desc;
+
+    SPC_RECORD(OMPI_SPC_ALLGATHERV, 1);
+
+    MEMCHECKER(
+        int rank;
+        ptrdiff_t ext;
+
+        rank = ompi_comm_rank(comm);
+        size = ompi_comm_size(comm);
+        ompi_datatype_type_extent(recvtype, &ext);
+
+        memchecker_datatype(recvtype);
+        memchecker_comm (comm);
+        /* check whether the receive buffer is addressable. */
+        for (i = 0; i < size; i++) {
+            memchecker_call(&opal_memchecker_base_isaddressable,
+                            (char *)(recvbuf)+displs[i]*ext,
+                            recvcounts[i], recvtype);
+        }
+
+        /* check whether the actual send buffer is defined. */
+        if (MPI_IN_PLACE == sendbuf) {
+            memchecker_call(&opal_memchecker_base_isdefined,
+                            (char *)(recvbuf)+displs[rank]*ext,
+                            recvcounts[rank], recvtype);
+        } else {
+            memchecker_datatype(sendtype);
+            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+
+        /* Unrooted operation -- same checks for all ranks on both
+           intracommunicators and intercommunicators */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if ((MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
+                   MPI_IN_PLACE == recvbuf) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        } else if (MPI_DATATYPE_NULL == recvtype) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
+        }
+
+        if (MPI_IN_PLACE != sendbuf) {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+        }
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+
+      /* We always define the remote group to be the same as the local
+         group in the case of an intracommunicator, so it's safe to
+         get the size of the remote group here for both intra- and
+         intercommunicators */
+
+        size = ompi_comm_remote_size(comm);
+        for (i = 0; i < size; ++i) {
+          if (recvcounts[i] < 0) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+          }
+        }
+
+        if (NULL == displs) {
+          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_BUFFER, FUNC_NAME);
+        }
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
+        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+    }
+#endif
+
+    /* Do we need to do anything?  Everyone had to give the same
+       signature, which means that everyone must have given a
+       sum(recvounts) > 0 if there's anything to do. */
+
+    if ( OMPI_COMM_IS_INTRA( comm) ) {
+	for (i = 0; i < ompi_comm_size(comm); ++i) {
+	    if (0 != recvcounts[i]) {
+		break;
+	    }
+	}
+	if (i >= ompi_comm_size(comm)) {
+	    return MPI_SUCCESS;
+	}
+    }
+    /* There is no rule that can be applied for inter-communicators, since
+       recvcount(s)=0 only indicates that the processes in the other group
+       do not send anything, sendcount=0 only indicates that I do not send
+       anything. However, other processes in my group might very well send
+       something */
+
+    /* Invoke the coll component to perform the back-end operation */
+    OMPI_COUNT_ARRAY_INIT(&recvcounts_desc, recvcounts);
+    OMPI_DISP_ARRAY_INIT(&displs_desc, displs);
+    err = comm->c_coll->coll_allgatherv(sendbuf, sendcount, sendtype,
+                                       recvbuf, recvcounts_desc,
+                                       displs_desc, recvtype, comm,
+                                       comm->c_coll->coll_allgatherv_module);
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/allgatherv_init.c.in b/ompi/mpi/c/allgatherv_init.c.in
new file mode 100644
index 00000000000..090b1b9befb
--- /dev/null
+++ b/ompi/mpi/c/allgatherv_init.c.in
@@ -0,0 +1,130 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010      University of Houston.  All rights reserved.
+ * Copyright (c) 2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS allgatherv_init(BUFFER sendbuf, COUNT sendcount, DATATYPE sendtype,
+                                      BUFFER_OUT recvbuf, COUNT_ARRAY recvcounts, DISP_ARRAY displs,
+                                      DATATYPE recvtype, COMM comm,
+                                      INFO info, REQUEST_INOUT request)
+{
+    int i, size, err;
+    ompi_count_array_t recvcounts_desc;
+    ompi_disp_array_t displs_desc;
+
+    SPC_RECORD(OMPI_SPC_ALLGATHERV_INIT, 1);
+
+    MEMCHECKER(
+        int rank;
+        ptrdiff_t ext;
+
+        rank = ompi_comm_rank(comm);
+        size = ompi_comm_size(comm);
+        ompi_datatype_type_extent(recvtype, &ext);
+
+        memchecker_datatype(recvtype);
+        memchecker_comm (comm);
+        /* check whether the receive buffer is addressable. */
+        for (i = 0; i < size; i++) {
+            memchecker_call(&opal_memchecker_base_isaddressable,
+                            (char *)(recvbuf)+displs[i]*ext,
+                            recvcounts[i], recvtype);
+        }
+
+        /* check whether the actual send buffer is defined. */
+        if (MPI_IN_PLACE == sendbuf) {
+            memchecker_call(&opal_memchecker_base_isdefined,
+                            (char *)(recvbuf)+displs[rank]*ext,
+                            recvcounts[rank], recvtype);
+        } else {
+            memchecker_datatype(sendtype);
+            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+
+        /* Unrooted operation -- same checks for all ranks on both
+           intracommunicators and intercommunicators */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if ((MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
+                   MPI_IN_PLACE == recvbuf) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        } else if (MPI_DATATYPE_NULL == recvtype) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
+        }
+
+        if (MPI_IN_PLACE != sendbuf) {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+        }
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+
+      /* We always define the remote group to be the same as the local
+         group in the case of an intracommunicator, so it's safe to
+         get the size of the remote group here for both intra- and
+         intercommunicators */
+
+        size = ompi_comm_remote_size(comm);
+        for (i = 0; i < size; ++i) {
+          if (recvcounts[i] < 0) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+          }
+        }
+
+        if (NULL == displs) {
+          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_BUFFER, FUNC_NAME);
+        }
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    OMPI_COUNT_ARRAY_INIT(&recvcounts_desc, recvcounts);
+    OMPI_DISP_ARRAY_INIT(&displs_desc, displs);
+    err = comm->c_coll->coll_allgatherv_init(sendbuf, sendcount, sendtype,
+                                             recvbuf, recvcounts_desc, displs_desc,
+                                             recvtype, comm, info, request,
+                                             comm->c_coll->coll_allgatherv_init_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_datatypes(*request, (MPI_IN_PLACE==sendbuf)?NULL:sendtype, recvtype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/alloc_mem.c b/ompi/mpi/c/alloc_mem.c
deleted file mode 100644
index 44f7f5e39d7..00000000000
--- a/ompi/mpi/c/alloc_mem.c
+++ /dev/null
@@ -1,93 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007-2020 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-#include "opal/mca/mpool/mpool.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Alloc_mem = PMPI_Alloc_mem
-#endif
-#define MPI_Alloc_mem PMPI_Alloc_mem
-#endif
-
-static const char FUNC_NAME[] = "MPI_Alloc_mem";
-
-
-int MPI_Alloc_mem(MPI_Aint size, MPI_Info info, void *baseptr)
-{
-    char info_value[MPI_MAX_INFO_VAL + 1];
-    char *mpool_hints = NULL;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (size < 0 || NULL == baseptr) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        } else if (NULL == info || ompi_info_is_freed(info)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
-                                          FUNC_NAME);
-        }
-    }
-
-    /* Per these threads:
-
-         https://www.open-mpi.org/community/lists/devel/2007/07/1977.php
-         https://www.open-mpi.org/community/lists/devel/2007/07/1979.php
-
-       If you call MPI_ALLOC_MEM with a size of 0, you get NULL
-       back .*/
-    if (0 == size) {
-        *((void **) baseptr) = NULL;
-        return MPI_SUCCESS;
-    }
-
-    if (MPI_INFO_NULL != info) {
-        int flag;
-        (void) ompi_info_get (info, "mpool_hints", MPI_MAX_INFO_VAL, info_value, &flag);
-        if (flag) {
-            mpool_hints = info_value;
-        }
-    }
-
-    *((void **) baseptr) = mca_mpool_base_alloc ((size_t) size, (struct opal_info_t*)info,
-                                                 mpool_hints);
-    if (NULL == *((void **) baseptr)) {
-        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_NO_MEM,
-                                      FUNC_NAME);
-    }
-
-    /* All done */
-    return MPI_SUCCESS;
-}
-
diff --git a/ompi/mpi/c/alloc_mem.c.in b/ompi/mpi/c/alloc_mem.c.in
new file mode 100644
index 00000000000..4dfd3d5bbfe
--- /dev/null
+++ b/ompi/mpi/c/alloc_mem.c.in
@@ -0,0 +1,90 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007-2020 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include "opal/mca/mpool/mpool.h"
+
+PROTOTYPE ERROR_CLASS alloc_mem(AINT size, INFO info, BUFFER_OUT baseptr)
+{
+    opal_cstring_t *info_str = NULL;
+    const char *mpool_hints = NULL;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (size < 0 || NULL == baseptr) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        } else if (NULL == info || ompi_info_is_freed(info)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
+                                          FUNC_NAME);
+        }
+    }
+
+    /* Per these threads:
+
+         https://www.open-mpi.org/community/lists/devel/2007/07/1977.php
+         https://www.open-mpi.org/community/lists/devel/2007/07/1979.php
+
+       If you call MPI_ALLOC_MEM with a size of 0, you get NULL
+       back .*/
+    if (0 == size) {
+        *((void **) baseptr) = NULL;
+        return MPI_SUCCESS;
+    }
+
+    if (MPI_INFO_NULL != info) {
+        int flag;
+        (void) ompi_info_get (info, "mpool_hints", &info_str, &flag);
+        if (flag) {
+            mpool_hints = info_str->string;
+        }
+    }
+
+    *((void **) baseptr) = mca_mpool_base_alloc ((size_t) size, (struct opal_info_t*)info,
+                                                 mpool_hints);
+
+    if (NULL != info_str) {
+        OBJ_RELEASE(info_str);
+    }
+
+    if (NULL == *((void **) baseptr)) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_NO_MEM,
+                                      FUNC_NAME);
+    }
+
+    /* All done */
+    return MPI_SUCCESS;
+}
+
diff --git a/ompi/mpi/c/allreduce.c b/ompi/mpi/c/allreduce.c
deleted file mode 100644
index 9f2c5023cac..00000000000
--- a/ompi/mpi/c/allreduce.c
+++ /dev/null
@@ -1,129 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/op/op.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Allreduce = PMPI_Allreduce
-#endif
-#define MPI_Allreduce PMPI_Allreduce
-#endif
-
-static const char FUNC_NAME[] = "MPI_Allreduce";
-
-
-int MPI_Allreduce(const void *sendbuf, void *recvbuf, int count,
-                  MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
-{
-    int err;
-
-    SPC_RECORD(OMPI_SPC_ALLREDUCE, 1);
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_comm(comm);
-
-        /* check whether receive buffer is defined. */
-        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, count, datatype);
-
-        /* check whether the actual send buffer is defined. */
-        if (MPI_IN_PLACE == sendbuf) {
-            memchecker_call(&opal_memchecker_base_isdefined, recvbuf, count, datatype);
-        } else {
-            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        char *msg;
-
-        /* Unrooted operation -- same checks for all ranks on both
-           intracommunicators and intercommunicators */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-	if (ompi_comm_invalid(comm)) {
-	    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if (MPI_OP_NULL == op) {
-            err = MPI_ERR_OP;
-        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
-            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
-            free(msg);
-            return ret;
-        } else if ((MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
-                   MPI_IN_PLACE == recvbuf ) {
-	    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_BUFFER,
-                                          FUNC_NAME);
-        } else if( (sendbuf == recvbuf) &&
-                   (MPI_BOTTOM != sendbuf) &&
-                   (count > 1) ) {
-	    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_BUFFER,
-                                          FUNC_NAME);
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
-        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-    }
-#endif
-
-    /* MPI-1, p114, says that each process must supply at least
-       one element.  But at least the Pallas benchmarks call
-       MPI_REDUCE with a count of 0.  So be sure to handle it. */
-
-    if (0 == count) {
-        return MPI_SUCCESS;
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-
-    OBJ_RETAIN(op);
-    err = comm->c_coll->coll_allreduce(sendbuf, recvbuf, count,
-                                      datatype, op, comm,
-                                      comm->c_coll->coll_allreduce_module);
-    OBJ_RELEASE(op);
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/allreduce.c.in b/ompi/mpi/c/allreduce.c.in
new file mode 100644
index 00000000000..182fa1fd02d
--- /dev/null
+++ b/ompi/mpi/c/allreduce.c.in
@@ -0,0 +1,121 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/op/op.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS allreduce(BUFFER sendbuf, BUFFER_OUT recvbuf, COUNT count,
+                                    DATATYPE datatype, OP op, COMM comm)
+{
+    int err;
+
+    SPC_RECORD(OMPI_SPC_ALLREDUCE, 1);
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_comm(comm);
+
+        /* check whether receive buffer is defined. */
+        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, count, datatype);
+
+        /* check whether the actual send buffer is defined. */
+        if (MPI_IN_PLACE == sendbuf) {
+            memchecker_call(&opal_memchecker_base_isdefined, recvbuf, count, datatype);
+        } else {
+            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        char *msg;
+
+        /* Unrooted operation -- same checks for all ranks on both
+           intracommunicators and intercommunicators */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+	if (ompi_comm_invalid(comm)) {
+	    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if (MPI_OP_NULL == op) {
+            err = MPI_ERR_OP;
+        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
+            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
+            free(msg);
+            return ret;
+        } else if ((MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
+                   MPI_IN_PLACE == recvbuf ) {
+	    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_BUFFER,
+                                          FUNC_NAME);
+        } else if( (sendbuf == recvbuf) &&
+                   (MPI_BOTTOM != sendbuf) &&
+                   (count > 1) ) {
+	    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_BUFFER,
+                                          FUNC_NAME);
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
+        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+    }
+#endif
+
+    /* MPI-1, p114, says that each process must supply at least
+       one element.  But at least the Pallas benchmarks call
+       MPI_REDUCE with a count of 0.  So be sure to handle it. */
+
+    if (0 == count) {
+        return MPI_SUCCESS;
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+
+    OBJ_RETAIN(op);
+    err = comm->c_coll->coll_allreduce(sendbuf, recvbuf, count,
+                                      datatype, op, comm,
+                                      comm->c_coll->coll_allreduce_module);
+    OBJ_RELEASE(op);
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/allreduce_init.c.in b/ompi/mpi/c/allreduce_init.c.in
new file mode 100644
index 00000000000..3f613c2d587
--- /dev/null
+++ b/ompi/mpi/c/allreduce_init.c.in
@@ -0,0 +1,114 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2016      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/op/op.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS allreduce_init(BUFFER sendbuf, BUFFER_OUT recvbuf, COUNT count,
+                       DATATYPE datatype, OP op, COMM comm,
+                       INFO info, REQUEST_INOUT request)
+{
+    int err;
+
+    SPC_RECORD(OMPI_SPC_ALLREDUCE_INIT, 1);
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_comm(comm);
+
+        /* check whether receive buffer is defined. */
+        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, count, datatype);
+
+        /* check whether the actual send buffer is defined. */
+        if (MPI_IN_PLACE == sendbuf) {
+            memchecker_call(&opal_memchecker_base_isdefined, recvbuf, count, datatype);
+        } else {
+            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        char *msg;
+
+        /* Unrooted operation -- same checks for all ranks on both
+           intracommunicators and intercommunicators */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if (MPI_OP_NULL == op) {
+            err = MPI_ERR_OP;
+        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
+            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
+            free(msg);
+            return ret;
+        } else if ((MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
+                   MPI_IN_PLACE == recvbuf ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_BUFFER,
+                                          FUNC_NAME);
+        } else if( (sendbuf == recvbuf) &&
+                   (MPI_BOTTOM != sendbuf) &&
+                   (count > 1) ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_BUFFER,
+                                          FUNC_NAME);
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+    }
+
+
+    /* MPI standard says that reductions have to have a count of at least 1,
+     * but some benchmarks (e.g., IMB) calls this function with a count of 0.
+     * So handle that case.
+     */
+    if (0 == count) {
+        err = ompi_request_persistent_noop_create(request);
+        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+
+    err = comm->c_coll->coll_allreduce_init(sendbuf, recvbuf, count, datatype,
+                                            op, comm, info, request, comm->c_coll->coll_allreduce_init_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_op(*request, op, datatype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/alltoall.c b/ompi/mpi/c/alltoall.c
deleted file mode 100644
index 069538a53c5..00000000000
--- a/ompi/mpi/c/alltoall.c
+++ /dev/null
@@ -1,125 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2014-2015 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2015      Mellanox Technologies. All rights reserved.
- *
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Alltoall = PMPI_Alltoall
-#endif
-#define MPI_Alltoall PMPI_Alltoall
-#endif
-
-static const char FUNC_NAME[] = "MPI_Alltoall";
-
-
-int MPI_Alltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                 void *recvbuf, int recvcount, MPI_Datatype recvtype,
-                 MPI_Comm comm)
-{
-    int err;
-    size_t recvtype_size;
-
-    SPC_RECORD(OMPI_SPC_ALLTOALL, 1);
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-        if (MPI_IN_PLACE != sendbuf) {
-            memchecker_datatype(sendtype);
-            memchecker_call(&opal_memchecker_base_isdefined, (void *)sendbuf, sendcount, sendtype);
-        }
-        memchecker_datatype(recvtype);
-        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-
-        /* Unrooted operation -- same checks for all ranks on both
-           intracommunicators and intercommunicators */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if ((MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
-                   MPI_IN_PLACE == recvbuf) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        } else {
-            if(MPI_IN_PLACE != sendbuf) {
-                OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-                OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-            }
-            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtype, recvcount);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-
-        if (MPI_IN_PLACE != sendbuf && !OMPI_COMM_IS_INTER(comm)) {
-            size_t sendtype_size, recvtype_size;
-            ompi_datatype_type_size(sendtype, &sendtype_size);
-            ompi_datatype_type_size(recvtype, &recvtype_size);
-            if ((sendtype_size*sendcount) != (recvtype_size*recvcount)) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TRUNCATE, FUNC_NAME);
-            }
-        }
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
-        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-    }
-#endif
-
-    if (! OMPI_COMM_IS_INTER(comm)) {
-        ompi_datatype_type_size(recvtype, &recvtype_size);
-        if( (0 == recvcount) || (0 == recvtype_size) ) {
-            return MPI_SUCCESS;
-        }
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_alltoall(sendbuf, sendcount, sendtype,
-                                     recvbuf, recvcount, recvtype,
-                                     comm, comm->c_coll->coll_alltoall_module);
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/alltoall.c.in b/ompi/mpi/c/alltoall.c.in
new file mode 100644
index 00000000000..63124937903
--- /dev/null
+++ b/ompi/mpi/c/alltoall.c.in
@@ -0,0 +1,116 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2014-2015 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015      Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS alltoall(BUFFER sendbuf, COUNT sendcount, DATATYPE sendtype,
+                               BUFFER_OUT recvbuf, COUNT recvcount, DATATYPE recvtype,
+                               COMM comm)
+{
+    int err;
+    size_t recvtype_size;
+
+    SPC_RECORD(OMPI_SPC_ALLTOALL, 1);
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+        if (MPI_IN_PLACE != sendbuf) {
+            memchecker_datatype(sendtype);
+            memchecker_call(&opal_memchecker_base_isdefined, (void *)sendbuf, sendcount, sendtype);
+        }
+        memchecker_datatype(recvtype);
+        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+
+        /* Unrooted operation -- same checks for all ranks on both
+           intracommunicators and intercommunicators */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if ((MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
+                   MPI_IN_PLACE == recvbuf) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        } else {
+            if(MPI_IN_PLACE != sendbuf) {
+                OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+                OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+            }
+            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtype, recvcount);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+
+        if (MPI_IN_PLACE != sendbuf && !OMPI_COMM_IS_INTER(comm)) {
+            size_t sendtype_size, recvtype_size_tmp;
+            ompi_datatype_type_size(sendtype, &sendtype_size);
+            ompi_datatype_type_size(recvtype, &recvtype_size_tmp);
+            if ((sendtype_size*sendcount) != (recvtype_size_tmp*recvcount)) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TRUNCATE, FUNC_NAME);
+            }
+        }
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
+        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+    }
+#endif
+
+    if (! OMPI_COMM_IS_INTER(comm)) {
+        ompi_datatype_type_size(recvtype, &recvtype_size);
+        if( (0 == recvcount) || (0 == recvtype_size) ) {
+            return MPI_SUCCESS;
+        }
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    err = comm->c_coll->coll_alltoall(sendbuf, sendcount, sendtype,
+                                     recvbuf, recvcount, recvtype,
+                                     comm, comm->c_coll->coll_alltoall_module);
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/alltoall_init.c.in b/ompi/mpi/c/alltoall_init.c.in
new file mode 100644
index 00000000000..becb9f77d54
--- /dev/null
+++ b/ompi/mpi/c/alltoall_init.c.in
@@ -0,0 +1,99 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oak Ridge National Laboratory. All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS alltoall_init(BUFFER sendbuf, COUNT sendcount, DATATYPE sendtype,
+                      BUFFER_OUT recvbuf, COUNT recvcount, DATATYPE recvtype,
+                      COMM comm, INFO info, REQUEST_INOUT request)
+{
+    size_t sendtype_size, recvtype_size;
+    int err;
+
+    SPC_RECORD(OMPI_SPC_ALLTOALL_INIT, 1);
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+        if (MPI_IN_PLACE != sendbuf) {
+            memchecker_datatype(sendtype);
+            memchecker_call(&opal_memchecker_base_isdefined, (void *)sendbuf, sendcount, sendtype);
+        }
+        memchecker_datatype(recvtype);
+        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+
+        /* Unrooted operation -- same checks for all ranks on both
+           intracommunicators and intercommunicators */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if ((MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
+                   MPI_IN_PLACE == recvbuf) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        } else {
+            if (MPI_IN_PLACE != sendbuf) {
+                OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+                OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+            }
+            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtype, recvcount);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+
+        if (MPI_IN_PLACE != sendbuf && !OMPI_COMM_IS_INTER(comm)) {
+            ompi_datatype_type_size(sendtype, &sendtype_size);
+            ompi_datatype_type_size(recvtype, &recvtype_size);
+            if ((sendtype_size*sendcount) != (recvtype_size*recvcount)) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TRUNCATE, FUNC_NAME);
+            }
+        }
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    err = comm->c_coll->coll_alltoall_init(sendbuf, sendcount, sendtype,
+                                           recvbuf, recvcount, recvtype, comm, info,
+                                           request, comm->c_coll->coll_alltoall_init_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_datatypes(*request, (MPI_IN_PLACE==sendbuf)?NULL:sendtype, recvtype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/alltoallv.c b/ompi/mpi/c/alltoallv.c
deleted file mode 100644
index 4106d96d249..00000000000
--- a/ompi/mpi/c/alltoallv.c
+++ /dev/null
@@ -1,144 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2014-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Alltoallv = PMPI_Alltoallv
-#endif
-#define MPI_Alltoallv PMPI_Alltoallv
-#endif
-
-static const char FUNC_NAME[] = "MPI_Alltoallv";
-
-
-int MPI_Alltoallv(const void *sendbuf, const int sendcounts[],
-                  const int sdispls[], MPI_Datatype sendtype,
-                  void *recvbuf, const int recvcounts[], const int rdispls[],
-                  MPI_Datatype recvtype, MPI_Comm comm)
-{
-    int i, size, err;
-
-    SPC_RECORD(OMPI_SPC_ALLTOALLV, 1);
-
-    MEMCHECKER(
-        ptrdiff_t recv_ext;
-        ptrdiff_t send_ext;
-
-        if (MPI_IN_PLACE != sendbuf) {
-            memchecker_datatype(sendtype);
-            ompi_datatype_type_extent(sendtype, &send_ext);
-        }
-        memchecker_datatype(recvtype);
-        ompi_datatype_type_extent(recvtype, &recv_ext);
-
-        memchecker_comm(comm);
-
-        size = OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm);
-        for ( i = 0; i < size; i++ ) {
-            if (MPI_IN_PLACE != sendbuf) {
-                /* check if send chunks are defined. */
-                memchecker_call(&opal_memchecker_base_isdefined,
-                                (char *)(sendbuf)+sdispls[i]*send_ext,
-                                sendcounts[i], sendtype);
-            }
-            /* check if receive chunks are addressable. */
-            memchecker_call(&opal_memchecker_base_isaddressable,
-                            (char *)(recvbuf)+rdispls[i]*recv_ext,
-                            recvcounts[i], recvtype);
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-
-        /* Unrooted operation -- same checks for all ranks */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-
-        if (MPI_IN_PLACE == sendbuf) {
-            sendcounts = recvcounts;
-            sdispls = rdispls;
-            sendtype = recvtype;
-        }
-
-        if ((NULL == sendcounts) || (NULL == sdispls) ||
-            (NULL == recvcounts) || (NULL == rdispls) ||
-            (MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
-            MPI_IN_PLACE == recvbuf) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-
-        size = OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm);
-        for (i = 0; i < size; ++i) {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcounts[i]);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtype, recvcounts[i]);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-
-        if (MPI_IN_PLACE != sendbuf && !OMPI_COMM_IS_INTER(comm)) {
-            size_t sendtype_size, recvtype_size;
-            int me = ompi_comm_rank(comm);
-            ompi_datatype_type_size(sendtype, &sendtype_size);
-            ompi_datatype_type_size(recvtype, &recvtype_size);
-            if ((sendtype_size*sendcounts[me]) != (recvtype_size*recvcounts[me])) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TRUNCATE, FUNC_NAME);
-            }
-        }
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
-        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-    }
-#endif
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_alltoallv(sendbuf, sendcounts, sdispls, sendtype,
-                                      recvbuf, recvcounts, rdispls, recvtype,
-                                      comm, comm->c_coll->coll_alltoallv_module);
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/alltoallv.c.in b/ompi/mpi/c/alltoallv.c.in
new file mode 100644
index 00000000000..dd9fabc5db1
--- /dev/null
+++ b/ompi/mpi/c/alltoallv.c.in
@@ -0,0 +1,142 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2018 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS alltoallv(BUFFER sendbuf, COUNT_ARRAY sendcounts,
+                                DISP_ARRAY sdispls, DATATYPE sendtype,
+                                BUFFER_OUT recvbuf, COUNT_ARRAY recvcounts, DISP_ARRAY rdispls,
+                                DATATYPE recvtype, COMM comm)
+{
+    int i, size, err;
+    ompi_count_array_t sendcounts_desc, recvcounts_desc;
+    ompi_disp_array_t sdispls_desc, rdispls_desc;
+
+    size = OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm);
+    SPC_RECORD(OMPI_SPC_ALLTOALLV, 1);
+
+    MEMCHECKER(
+        ptrdiff_t recv_ext;
+        ptrdiff_t send_ext;
+
+        if (MPI_IN_PLACE != sendbuf) {
+            memchecker_datatype(sendtype);
+            ompi_datatype_type_extent(sendtype, &send_ext);
+        }
+        memchecker_datatype(recvtype);
+        ompi_datatype_type_extent(recvtype, &recv_ext);
+
+        memchecker_comm(comm);
+
+        for ( i = 0; i < size; i++ ) {
+            if (MPI_IN_PLACE != sendbuf) {
+                /* check if send chunks are defined. */
+                memchecker_call(&opal_memchecker_base_isdefined,
+                                (char *)(sendbuf)+sdispls[i]*send_ext,
+                                sendcounts[i], sendtype);
+            }
+            /* check if receive chunks are addressable. */
+            memchecker_call(&opal_memchecker_base_isaddressable,
+                            (char *)(recvbuf)+rdispls[i]*recv_ext,
+                            recvcounts[i], recvtype);
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+
+        /* Unrooted operation -- same checks for all ranks */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+
+        if (MPI_IN_PLACE == sendbuf) {
+            sendcounts = recvcounts;
+            sdispls = rdispls;
+            sendtype = recvtype;
+        }
+
+        if ((NULL == sendcounts) || (NULL == sdispls) ||
+            (NULL == recvcounts) || (NULL == rdispls) ||
+            (MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
+            MPI_IN_PLACE == recvbuf) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+
+        size = OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm);
+        for (i = 0; i < size; ++i) {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcounts[i]);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtype, recvcounts[i]);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+
+        if (MPI_IN_PLACE != sendbuf && !OMPI_COMM_IS_INTER(comm)) {
+            size_t sendtype_size, recvtype_size;
+            int me = ompi_comm_rank(comm);
+            ompi_datatype_type_size(sendtype, &sendtype_size);
+            ompi_datatype_type_size(recvtype, &recvtype_size);
+            if ((sendtype_size*sendcounts[me]) != (recvtype_size*recvcounts[me])) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TRUNCATE, FUNC_NAME);
+            }
+        }
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
+        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+    }
+#endif
+
+    /* Invoke the coll component to perform the back-end operation */
+    OMPI_COUNT_ARRAY_INIT(&sendcounts_desc, sendcounts);
+    OMPI_DISP_ARRAY_INIT(&sdispls_desc, sdispls);
+    OMPI_COUNT_ARRAY_INIT(&recvcounts_desc, recvcounts);
+    OMPI_DISP_ARRAY_INIT(&rdispls_desc, rdispls);
+    err = comm->c_coll->coll_alltoallv(sendbuf, sendcounts_desc, sdispls_desc, sendtype,
+                                      recvbuf, recvcounts_desc, rdispls_desc, recvtype,
+                                      comm, comm->c_coll->coll_alltoallv_module);
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/alltoallv_init.c.in b/ompi/mpi/c/alltoallv_init.c.in
new file mode 100644
index 00000000000..da7a9358426
--- /dev/null
+++ b/ompi/mpi/c/alltoallv_init.c.in
@@ -0,0 +1,135 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS alltoallv_init(BUFFER sendbuf, COUNT_ARRAY sendcounts, DISP_ARRAY sdispls,
+                                     DATATYPE sendtype, BUFFER_OUT recvbuf, COUNT_ARRAY recvcounts,
+                                     DISP_ARRAY rdispls, DATATYPE recvtype, COMM comm,
+                                     INFO info, REQUEST_INOUT request)
+{
+    int i, size, err;
+    ompi_count_array_t sendcounts_desc, recvcounts_desc;
+    ompi_disp_array_t sdispls_desc, rdispls_desc;
+
+    SPC_RECORD(OMPI_SPC_ALLTOALLV_INIT, 1);
+
+    MEMCHECKER(
+        ptrdiff_t recv_ext;
+        ptrdiff_t send_ext;
+
+        memchecker_comm(comm);
+
+        if (MPI_IN_PLACE != sendbuf) {
+            memchecker_datatype(sendtype);
+            ompi_datatype_type_extent(sendtype, &send_ext);
+        }
+
+        memchecker_datatype(recvtype);
+        ompi_datatype_type_extent(recvtype, &recv_ext);
+
+        size = OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm);
+        for ( i = 0; i < size; i++ ) {
+            if (MPI_IN_PLACE != sendbuf) {
+                /* check if send chunks are defined. */
+                memchecker_call(&opal_memchecker_base_isdefined,
+                                (char *)(sendbuf)+sdispls[i]*send_ext,
+                                sendcounts[i], sendtype);
+            }
+            /* check if receive chunks are addressable. */
+            memchecker_call(&opal_memchecker_base_isaddressable,
+                            (char *)(recvbuf)+rdispls[i]*recv_ext,
+                            recvcounts[i], recvtype);
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+
+        /* Unrooted operation -- same checks for all ranks */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+
+        if (MPI_IN_PLACE == sendbuf) {
+            sendcounts = recvcounts;
+            sdispls = rdispls;
+            sendtype = recvtype;
+        }
+
+        if ((NULL == sendcounts) || (NULL == sdispls) ||
+            (NULL == recvcounts) || (NULL == rdispls) ||
+            (MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
+            MPI_IN_PLACE == recvbuf) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+
+        size = OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm);
+        for (i = 0; i < size; ++i) {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcounts[i]);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtype, recvcounts[i]);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+
+        if (MPI_IN_PLACE != sendbuf && !OMPI_COMM_IS_INTER(comm)) {
+            int me = ompi_comm_rank(comm);
+            size_t sendtype_size, recvtype_size;
+            ompi_datatype_type_size(sendtype, &sendtype_size);
+            ompi_datatype_type_size(recvtype, &recvtype_size);
+            if ((sendtype_size*sendcounts[me]) != (recvtype_size*recvcounts[me])) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TRUNCATE, FUNC_NAME);
+            }
+        }
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    OMPI_COUNT_ARRAY_INIT(&sendcounts_desc, sendcounts);
+    OMPI_COUNT_ARRAY_INIT(&recvcounts_desc, recvcounts);
+    OMPI_DISP_ARRAY_INIT(&sdispls_desc, sdispls);
+    OMPI_DISP_ARRAY_INIT(&rdispls_desc, rdispls);
+    err = comm->c_coll->coll_alltoallv_init(sendbuf, sendcounts_desc, sdispls_desc,
+                                            sendtype, recvbuf, recvcounts_desc, rdispls_desc,
+                                            recvtype, comm, info, request, comm->c_coll->coll_alltoallv_init_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_datatypes(*request, (MPI_IN_PLACE==sendbuf)?NULL:sendtype, recvtype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/alltoallw.c b/ompi/mpi/c/alltoallw.c
deleted file mode 100644
index e8850326df4..00000000000
--- a/ompi/mpi/c/alltoallw.c
+++ /dev/null
@@ -1,134 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2014-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Alltoallw = PMPI_Alltoallw
-#endif
-#define MPI_Alltoallw PMPI_Alltoallw
-#endif
-
-static const char FUNC_NAME[] = "MPI_Alltoallw";
-
-
-int MPI_Alltoallw(const void *sendbuf, const int sendcounts[],
-                  const int sdispls[], const MPI_Datatype sendtypes[],
-                  void *recvbuf, const int recvcounts[], const int rdispls[],
-                  const MPI_Datatype recvtypes[], MPI_Comm comm)
-{
-    int i, size, err;
-
-    SPC_RECORD(OMPI_SPC_ALLTOALLW, 1);
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-
-        size = OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm);
-        for ( i = 0; i < size; i++ ) {
-            if (MPI_IN_PLACE != sendbuf) {
-                memchecker_datatype(sendtypes[i]);
-                memchecker_call(&opal_memchecker_base_isdefined,
-                                (char *)(sendbuf)+sdispls[i],
-                                sendcounts[i], sendtypes[i]);
-            }
-            memchecker_datatype(recvtypes[i]);
-            memchecker_call(&opal_memchecker_base_isaddressable,
-                            (char *)(recvbuf)+rdispls[i],
-                            recvcounts[i], recvtypes[i]);
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-
-        /* Unrooted operation -- same checks for all ranks */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-
-        if (MPI_IN_PLACE == sendbuf) {
-            sendcounts = recvcounts;
-            sdispls    = rdispls;
-            sendtypes  = recvtypes;
-        }
-
-        if ((NULL == sendcounts) || (NULL == sdispls) || (NULL == sendtypes) ||
-            (NULL == recvcounts) || (NULL == rdispls) || (NULL == recvtypes) ||
-            (MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
-            MPI_IN_PLACE == recvbuf) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-
-        size = OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm);
-        for (i = 0; i < size; ++i) {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtypes[i], sendcounts[i]);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtypes[i], recvcounts[i]);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-
-        if (MPI_IN_PLACE != sendbuf && !OMPI_COMM_IS_INTER(comm)) {
-            size_t sendtype_size, recvtype_size;
-            int me = ompi_comm_rank(comm);
-            ompi_datatype_type_size(sendtypes[me], &sendtype_size);
-            ompi_datatype_type_size(recvtypes[me], &recvtype_size);
-            if ((sendtype_size*sendcounts[me]) != (recvtype_size*recvcounts[me])) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TRUNCATE, FUNC_NAME);
-            }
-        }
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
-        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-    }
-#endif
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_alltoallw(sendbuf, sendcounts, sdispls, (ompi_datatype_t **) sendtypes,
-                                      recvbuf, recvcounts, rdispls, (ompi_datatype_t **) recvtypes,
-                                      comm, comm->c_coll->coll_alltoallw_module);
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/alltoallw.c.in b/ompi/mpi/c/alltoallw.c.in
new file mode 100644
index 00000000000..75f0a131ec2
--- /dev/null
+++ b/ompi/mpi/c/alltoallw.c.in
@@ -0,0 +1,132 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2018 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS alltoallw(BUFFER sendbuf, COUNT_ARRAY sendcounts,
+                                DISP_ARRAY sdispls, DATATYPE_ARRAY sendtypes,
+                                BUFFER_OUT recvbuf, COUNT_ARRAY recvcounts, DISP_ARRAY rdispls,
+                                DATATYPE_ARRAY recvtypes, COMM comm)
+{
+    int i, size, err;
+    ompi_count_array_t sendcounts_desc, recvcounts_desc;
+    ompi_disp_array_t sdispls_desc, rdispls_desc;
+
+    SPC_RECORD(OMPI_SPC_ALLTOALLW, 1);
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+
+        size = OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm);
+        for ( i = 0; i < size; i++ ) {
+            if (MPI_IN_PLACE != sendbuf) {
+                memchecker_datatype(sendtypes[i]);
+                memchecker_call(&opal_memchecker_base_isdefined,
+                                (char *)(sendbuf)+sdispls[i],
+                                sendcounts[i], sendtypes[i]);
+            }
+            memchecker_datatype(recvtypes[i]);
+            memchecker_call(&opal_memchecker_base_isaddressable,
+                            (char *)(recvbuf)+rdispls[i],
+                            recvcounts[i], recvtypes[i]);
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+
+        /* Unrooted operation -- same checks for all ranks */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+
+        if (MPI_IN_PLACE == sendbuf) {
+            sendcounts = recvcounts;
+            sdispls    = rdispls;
+            sendtypes  = recvtypes;
+        }
+
+        if ((NULL == sendcounts) || (NULL == sdispls) || (NULL == sendtypes) ||
+            (NULL == recvcounts) || (NULL == rdispls) || (NULL == recvtypes) ||
+            (MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
+            MPI_IN_PLACE == recvbuf) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+
+        size = OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm);
+        for (i = 0; i < size; ++i) {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtypes[i], sendcounts[i]);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtypes[i], recvcounts[i]);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+
+        if (MPI_IN_PLACE != sendbuf && !OMPI_COMM_IS_INTER(comm)) {
+            size_t sendtype_size, recvtype_size;
+            int me = ompi_comm_rank(comm);
+            ompi_datatype_type_size(sendtypes[me], &sendtype_size);
+            ompi_datatype_type_size(recvtypes[me], &recvtype_size);
+            if ((sendtype_size*sendcounts[me]) != (recvtype_size*recvcounts[me])) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TRUNCATE, FUNC_NAME);
+            }
+        }
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
+        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+    }
+#endif
+
+    /* Invoke the coll component to perform the back-end operation */
+    OMPI_COUNT_ARRAY_INIT(&sendcounts_desc, sendcounts);
+    OMPI_COUNT_ARRAY_INIT(&recvcounts_desc, recvcounts);
+    OMPI_DISP_ARRAY_INIT(&sdispls_desc, sdispls);
+    OMPI_DISP_ARRAY_INIT(&rdispls_desc, rdispls);
+    err = comm->c_coll->coll_alltoallw(sendbuf, sendcounts_desc, sdispls_desc, (ompi_datatype_t **) sendtypes,
+                                      recvbuf, recvcounts_desc, rdispls_desc, (ompi_datatype_t **) recvtypes,
+                                      comm, comm->c_coll->coll_alltoallw_module);
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/alltoallw_init.c.in b/ompi/mpi/c/alltoallw_init.c.in
new file mode 100644
index 00000000000..bb37f852993
--- /dev/null
+++ b/ompi/mpi/c/alltoallw_init.c.in
@@ -0,0 +1,132 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2022 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS alltoallw_init(BUFFER sendbuf, COUNT_ARRAY sendcounts, DISP_ARRAY sdispls,
+                                     DATATYPE_ARRAY sendtypes, BUFFER_OUT recvbuf, COUNT_ARRAY recvcounts,
+                                     DISP_ARRAY rdispls, DATATYPE_ARRAY recvtypes, COMM comm,
+                                     INFO info, REQUEST_INOUT request)
+{
+    int i, size, err;
+    ompi_count_array_t sendcounts_desc, recvcounts_desc;
+    ompi_disp_array_t sdispls_desc, rdispls_desc;
+
+    SPC_RECORD(OMPI_SPC_ALLTOALLW_INIT, 1);
+
+    MEMCHECKER(
+        ptrdiff_t recv_ext;
+        ptrdiff_t send_ext;
+
+
+        memchecker_comm(comm);
+
+        size = OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm);
+        for ( i = 0; i < size; i++ ) {
+            if (MPI_IN_PLACE != sendbuf) {
+                memchecker_datatype(sendtypes[i]);
+                ompi_datatype_type_extent(sendtypes[i], &send_ext);
+                memchecker_call(&opal_memchecker_base_isdefined,
+                                (char *)(sendbuf)+sdispls[i]*send_ext,
+                                sendcounts[i], sendtypes[i]);
+            }
+
+            memchecker_datatype(recvtypes[i]);
+            ompi_datatype_type_extent(recvtypes[i], &recv_ext);
+            memchecker_call(&opal_memchecker_base_isaddressable,
+                            (char *)(recvbuf)+rdispls[i]*recv_ext,
+                            recvcounts[i], recvtypes[i]);
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+
+        /* Unrooted operation -- same checks for all ranks */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+
+        if (MPI_IN_PLACE == sendbuf) {
+            sendcounts = recvcounts;
+            sdispls    = rdispls;
+            sendtypes  = recvtypes;
+        }
+
+        if ((NULL == sendcounts) || (NULL == sdispls) || (NULL == sendtypes) ||
+            (NULL == recvcounts) || (NULL == rdispls) || (NULL == recvtypes) ||
+            (MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
+            MPI_IN_PLACE == recvbuf) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+
+        size = OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm);
+        for (i = 0; i < size; ++i) {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtypes[i], sendcounts[i]);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtypes[i], recvcounts[i]);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+
+        if (MPI_IN_PLACE != sendbuf && !OMPI_COMM_IS_INTER(comm)) {
+            int me = ompi_comm_rank(comm);
+            size_t sendtype_size, recvtype_size;
+            ompi_datatype_type_size(sendtypes[me], &sendtype_size);
+            ompi_datatype_type_size(recvtypes[me], &recvtype_size);
+            if ((sendtype_size*sendcounts[me]) != (recvtype_size*recvcounts[me])) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TRUNCATE, FUNC_NAME);
+            }
+        }
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    OMPI_COUNT_ARRAY_INIT(&sendcounts_desc, sendcounts);
+    OMPI_COUNT_ARRAY_INIT(&recvcounts_desc, recvcounts);
+    OMPI_DISP_ARRAY_INIT(&sdispls_desc, sdispls);
+    OMPI_DISP_ARRAY_INIT(&rdispls_desc, rdispls);
+    err = comm->c_coll->coll_alltoallw_init(sendbuf, sendcounts_desc, sdispls_desc,
+                                            sendtypes, recvbuf, recvcounts_desc,
+                                            rdispls_desc, recvtypes, comm, info, request,
+                                            comm->c_coll->coll_alltoallw_init_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_datatypes_w(*request, (MPI_IN_PLACE==sendbuf)?NULL:sendtypes, recvtypes, false);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/attr_delete.c b/ompi/mpi/c/attr_delete.c
deleted file mode 100644
index e412b06ea47..00000000000
--- a/ompi/mpi/c/attr_delete.c
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/attribute/attribute.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Attr_delete = PMPI_Attr_delete
-#endif
-#define MPI_Attr_delete PMPI_Attr_delete
-#endif
-
-static const char FUNC_NAME[] = "MPI_Attr_delete";
-
-
-int MPI_Attr_delete(MPI_Comm comm, int keyval)
-{
-    int ret;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-    }
-
-    ret = ompi_attr_delete(COMM_ATTR, comm, comm->c_keyhash, keyval,
-                           false);
-
-    OMPI_ERRHANDLER_RETURN(ret, comm, MPI_ERR_OTHER, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/attr_delete.c.in b/ompi/mpi/c/attr_delete.c.in
new file mode 100644
index 00000000000..8bc920ad4f5
--- /dev/null
+++ b/ompi/mpi/c/attr_delete.c.in
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/attribute/attribute.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS attr_delete(COMM comm, INT keyval)
+{
+    int ret;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+    }
+
+    ret = ompi_attr_delete(COMM_ATTR, comm, comm->c_keyhash, keyval,
+                           false);
+
+    OMPI_ERRHANDLER_RETURN(ret, comm, MPI_ERR_OTHER, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/attr_get.c b/ompi/mpi/c/attr_get.c
deleted file mode 100644
index 3adf3f207a5..00000000000
--- a/ompi/mpi/c/attr_get.c
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/attribute/attribute.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Attr_get = PMPI_Attr_get
-#endif
-#define MPI_Attr_get PMPI_Attr_get
-#endif
-
-static const char FUNC_NAME[] = "MPI_Attr_get";
-
-int MPI_Attr_get(MPI_Comm comm, int keyval, void *attribute_val, int *flag)
-{
-    int ret;
-
-    MEMCHECKER(
-         memchecker_comm(comm);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if ((NULL == attribute_val) || (NULL == flag)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-    }
-
-    /* This stuff is very confusing.  Be sure to see
-       src/attribute/attribute.c for a lengthy comment explaining Open
-       MPI attribute behavior. */
-
-    ret = ompi_attr_get_c(comm->c_keyhash, keyval, (void**)attribute_val, flag);
-    OMPI_ERRHANDLER_RETURN(ret, comm, ret, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/attr_get.c.in b/ompi/mpi/c/attr_get.c.in
new file mode 100644
index 00000000000..36b1e1f7a9b
--- /dev/null
+++ b/ompi/mpi/c/attr_get.c.in
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/attribute/attribute.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS attr_get(COMM comm, INT keyval, BUFFER_OUT attribute_val, INT_OUT flag)
+{
+    int ret;
+
+    MEMCHECKER(
+         memchecker_comm(comm);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if ((NULL == attribute_val) || (NULL == flag)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+    /* This stuff is very confusing.  Be sure to see
+       src/attribute/attribute.c for a lengthy comment explaining Open
+       MPI attribute behavior. */
+
+    ret = ompi_attr_get_c(comm->c_keyhash, keyval, (void**)attribute_val, flag);
+    OMPI_ERRHANDLER_RETURN(ret, comm, ret, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/attr_put.c b/ompi/mpi/c/attr_put.c
deleted file mode 100644
index 653b35d309b..00000000000
--- a/ompi/mpi/c/attr_put.c
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/attribute/attribute.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Attr_put = PMPI_Attr_put
-#endif
-#define MPI_Attr_put PMPI_Attr_put
-#endif
-
-static const char FUNC_NAME[] = "MPI_Attr_put";
-
-int MPI_Attr_put(MPI_Comm comm, int keyval, void *attribute_val)
-{
-    int ret;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-    }
-
-    ret = ompi_attr_set_c(COMM_ATTR, comm, &comm->c_keyhash,
-                          keyval, attribute_val, false);
-
-    OMPI_ERRHANDLER_RETURN(ret, comm, MPI_ERR_OTHER, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/attr_put.c.in b/ompi/mpi/c/attr_put.c.in
new file mode 100644
index 00000000000..4b6e5cd8b5c
--- /dev/null
+++ b/ompi/mpi/c/attr_put.c.in
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/attribute/attribute.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS attr_put(COMM comm, INT keyval, BUFFER_OUT attribute_val)
+{
+    int ret;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+    }
+
+    ret = ompi_attr_set_c(COMM_ATTR, comm, &comm->c_keyhash,
+                          keyval, attribute_val, false);
+
+    OMPI_ERRHANDLER_RETURN(ret, comm, MPI_ERR_OTHER, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/barrier.c b/ompi/mpi/c/barrier.c
deleted file mode 100644
index 35e629b0524..00000000000
--- a/ompi/mpi/c/barrier.c
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Barrier = PMPI_Barrier
-#endif
-#define MPI_Barrier PMPI_Barrier
-#endif
-
-static const char FUNC_NAME[] = "MPI_Barrier";
-
-
-int MPI_Barrier(MPI_Comm comm)
-{
-  int err = MPI_SUCCESS;
-
-  SPC_RECORD(OMPI_SPC_BARRIER, 1);
-
-  MEMCHECKER(
-    memchecker_comm(comm);
-  );
-
-  /* Error checking */
-
-  if (MPI_PARAM_CHECK) {
-    OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-    if (ompi_comm_invalid(comm)) {
-      return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
-    }
-  }
-
-#if OPAL_ENABLE_FT_MPI
-  /*
-   * An early check, so as to return early if we are using a broken
-   * communicator. This is not absolutely necessary since we will
-   * check for this, and other, error conditions during the operation.
-   */
-  if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
-      OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-  }
-#endif
-
-  /* Intracommunicators: Only invoke the back-end coll module barrier
-     function if there's more than one process in the communicator */
-
-  if (OMPI_COMM_IS_INTRA(comm)) {
-    if (ompi_comm_size(comm) > 1) {
-      err = comm->c_coll->coll_barrier(comm, comm->c_coll->coll_barrier_module);
-    }
-  }
-
-  /* Intercommunicators -- always invoke, because, by definition,
-     there's always at least 2 processes in an intercommunicator. */
-
-  else {
-      err = comm->c_coll->coll_barrier(comm, comm->c_coll->coll_barrier_module);
-  }
-
-  /* All done */
-
-  OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/barrier.c.in b/ompi/mpi/c/barrier.c.in
new file mode 100644
index 00000000000..1ba1c42d9e1
--- /dev/null
+++ b/ompi/mpi/c/barrier.c.in
@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS barrier(COMM comm)
+{
+  int err = MPI_SUCCESS;
+
+  SPC_RECORD(OMPI_SPC_BARRIER, 1);
+
+  MEMCHECKER(
+    memchecker_comm(comm);
+  );
+
+  /* Error checking */
+
+  if (MPI_PARAM_CHECK) {
+    OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+    if (ompi_comm_invalid(comm)) {
+      return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+    }
+  }
+
+#if OPAL_ENABLE_FT_MPI
+  /*
+   * An early check, so as to return early if we are using a broken
+   * communicator. This is not absolutely necessary since we will
+   * check for this, and other, error conditions during the operation.
+   */
+  if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
+      OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+  }
+#endif
+
+  /* Intracommunicators: Only invoke the back-end coll module barrier
+     function if there's more than one process in the communicator */
+
+  if (OMPI_COMM_IS_INTRA(comm)) {
+    if (ompi_comm_size(comm) > 1) {
+      err = comm->c_coll->coll_barrier(comm, comm->c_coll->coll_barrier_module);
+    }
+  }
+
+  /* Intercommunicators -- always invoke, because, by definition,
+     there's always at least 2 processes in an intercommunicator. */
+
+  else {
+      err = comm->c_coll->coll_barrier(comm, comm->c_coll->coll_barrier_module);
+  }
+
+  /* All done */
+
+  OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/barrier_init.c.in b/ompi/mpi/c/barrier_init.c.in
new file mode 100644
index 00000000000..dde5d3f669e
--- /dev/null
+++ b/ompi/mpi/c/barrier_init.c.in
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2012      Oak Rigde National Laboratory. All rights reserved.
+ * Copyright (c) 2015-2018 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS barrier_init(COMM comm, INFO info, REQUEST_INOUT request)
+{
+    int err = MPI_SUCCESS;
+
+    SPC_RECORD(OMPI_SPC_BARRIER_INIT, 1);
+
+    MEMCHECKER(
+            memchecker_comm(comm);
+            );
+
+    /* Error checking */
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+        }
+    }
+
+    err = comm->c_coll->coll_barrier_init(comm, info, request, comm->c_coll->coll_barrier_init_module);
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/bcast.c b/ompi/mpi/c/bcast.c
deleted file mode 100644
index b3c128215a8..00000000000
--- a/ompi/mpi/c/bcast.c
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Bcast = PMPI_Bcast
-#endif
-#define MPI_Bcast PMPI_Bcast
-#endif
-
-static const char FUNC_NAME[] = "MPI_Bcast";
-
-
-int MPI_Bcast(void *buffer, int count, MPI_Datatype datatype,
-              int root, MPI_Comm comm)
-{
-    int err;
-
-    SPC_RECORD(OMPI_SPC_BCAST, 1);
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_comm(comm);
-        if (OMPI_COMM_IS_INTRA(comm)) {
-            if (ompi_comm_rank(comm) == root) {
-                /* check whether root's send buffer is defined. */
-                memchecker_call(&opal_memchecker_base_isdefined, buffer, count, datatype);
-            } else {
-                /* check whether receive buffer is addressable. */
-                memchecker_call(&opal_memchecker_base_isaddressable, buffer, count, datatype);
-            }
-        } else {
-            if (MPI_ROOT == root) {
-                /* check whether root's send buffer is defined. */
-                memchecker_call(&opal_memchecker_base_isdefined, buffer, count, datatype);
-            } else if (MPI_PROC_NULL != root) {
-                /* check whether receive buffer is addressable. */
-                memchecker_call(&opal_memchecker_base_isaddressable, buffer, count, datatype);
-            }
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-      err = MPI_SUCCESS;
-      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-      if (ompi_comm_invalid(comm)) {
-          return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                     FUNC_NAME);
-      }
-
-      /* Errors for all ranks */
-
-      OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
-      OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-      if (MPI_IN_PLACE == buffer) {
-          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-      }
-
-      /* Errors for intracommunicators */
-
-      if (OMPI_COMM_IS_INTRA(comm)) {
-        if ((root >= ompi_comm_size(comm)) || (root < 0)) {
-          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
-        }
-      }
-
-      /* Errors for intercommunicators */
-
-      else {
-        if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
-               MPI_ROOT == root || MPI_PROC_NULL == root)) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
-        }
-      }
-    } 
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
-        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-    }
-#endif
-
-    /* If there's only one node, or if the count is 0, we're done */
-
-    if ((OMPI_COMM_IS_INTRA(comm) && ompi_comm_size(comm) <= 1) ||
-        0 == count) {
-        return MPI_SUCCESS;
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-
-    err = comm->c_coll->coll_bcast(buffer, count, datatype, root, comm,
-                                  comm->c_coll->coll_bcast_module);
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/bcast.c.in b/ompi/mpi/c/bcast.c.in
new file mode 100644
index 00000000000..bbb54d24b04
--- /dev/null
+++ b/ompi/mpi/c/bcast.c.in
@@ -0,0 +1,120 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2015-2018 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS bcast(BUFFER_OUT buffer, COUNT count, DATATYPE datatype,
+                            INT root, COMM comm)
+{
+    int err;
+
+    SPC_RECORD(OMPI_SPC_BCAST, 1);
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_comm(comm);
+        if (OMPI_COMM_IS_INTRA(comm)) {
+            if (ompi_comm_rank(comm) == root) {
+                /* check whether root's send buffer is defined. */
+                memchecker_call(&opal_memchecker_base_isdefined, buffer, count, datatype);
+            } else {
+                /* check whether receive buffer is addressable. */
+                memchecker_call(&opal_memchecker_base_isaddressable, buffer, count, datatype);
+            }
+        } else {
+            if (MPI_ROOT == root) {
+                /* check whether root's send buffer is defined. */
+                memchecker_call(&opal_memchecker_base_isdefined, buffer, count, datatype);
+            } else if (MPI_PROC_NULL != root) {
+                /* check whether receive buffer is addressable. */
+                memchecker_call(&opal_memchecker_base_isaddressable, buffer, count, datatype);
+            }
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+      err = MPI_SUCCESS;
+      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+      if (ompi_comm_invalid(comm)) {
+          return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                     FUNC_NAME);
+      }
+
+      /* Errors for all ranks */
+
+      OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
+      OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+      if (MPI_IN_PLACE == buffer) {
+          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+      }
+
+      /* Errors for intracommunicators */
+
+      if (OMPI_COMM_IS_INTRA(comm)) {
+        if ((root >= ompi_comm_size(comm)) || (root < 0)) {
+          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
+        }
+      }
+
+      /* Errors for intercommunicators */
+
+      else {
+        if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
+               MPI_ROOT == root || MPI_PROC_NULL == root)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
+        }
+      }
+    } 
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
+        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+    }
+#endif
+
+    /* If there's only one node, or if the count is 0, we're done */
+
+    if ((OMPI_COMM_IS_INTRA(comm) && ompi_comm_size(comm) <= 1) ||
+        0 == count) {
+        return MPI_SUCCESS;
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+
+    err = comm->c_coll->coll_bcast(buffer, count, datatype, root, comm,
+                                  comm->c_coll->coll_bcast_module);
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/bcast_init.c.in b/ompi/mpi/c/bcast_init.c.in
new file mode 100644
index 00000000000..08213a90a0a
--- /dev/null
+++ b/ompi/mpi/c/bcast_init.c.in
@@ -0,0 +1,89 @@
+/*
+ * Copyright (c) 2012      Oak Rigde National Laboratory. All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2017-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS bcast_init(BUFFER_OUT buffer, COUNT count, DATATYPE datatype,
+                                 INT root, COMM comm, INFO info, REQUEST_INOUT request)
+{
+    int err;
+
+    SPC_RECORD(OMPI_SPC_BCAST_INIT, 1);
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_call(&opal_memchecker_base_isdefined, buffer, count, datatype);
+        memchecker_comm(comm);
+    );
+
+    if (MPI_PARAM_CHECK) {
+      err = MPI_SUCCESS;
+      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+      if (ompi_comm_invalid(comm)) {
+          return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                     FUNC_NAME);
+      }
+
+      /* Errors for all ranks */
+
+      OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
+      OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+      if (MPI_IN_PLACE == buffer) {
+          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+      }
+
+      /* Errors for intracommunicators */
+
+      if (OMPI_COMM_IS_INTRA(comm)) {
+        if ((root >= ompi_comm_size(comm)) || (root < 0)) {
+          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
+        }
+      }
+
+      /* Errors for intercommunicators */
+
+      else {
+        if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
+               MPI_ROOT == root || MPI_PROC_NULL == root)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
+        }
+      }
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+
+    err = comm->c_coll->coll_bcast_init(buffer, count, datatype, root, comm,
+                                        info, request,
+                                        comm->c_coll->coll_bcast_init_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        if (!OMPI_COMM_IS_INTRA(comm)) {
+            if (MPI_PROC_NULL == root) {
+                datatype = NULL;
+            }
+        }
+        ompi_coll_base_retain_datatypes(*request, datatype, NULL);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/bindings.h b/ompi/mpi/c/bindings.h
index c00e6b0331c..2a849feea8d 100644
--- a/ompi/mpi/c/bindings.h
+++ b/ompi/mpi/c/bindings.h
@@ -34,10 +34,10 @@ BEGIN_C_DECLS
    doesn't work to simply list all of the pragmas in a top-level
    header file. */
 
-/* These macros have to be used to check the corectness of the datatype depending on the
+/* These macros have to be used to check the correctness of the datatype depending on the
  * operations that we have to do with them. They can be used on all functions, not only
  * on the top level MPI functions, as they does not trigger the error handler. Is the user
- * responsability to do it.
+ * responsibility to do it.
  */
 #define OMPI_CHECK_DATATYPE_FOR_SEND( RC, DDT, COUNT )                  \
     do {                                                                \
@@ -89,18 +89,33 @@ BEGIN_C_DECLS
             if ( (DDT)->super.flags & OPAL_DATATYPE_FLAG_PREDEFINED ) { \
                 (RC) = MPI_ERR_BUFFER;                                  \
             } else {                                                    \
-                size_t size = 0;                                        \
-                ptrdiff_t true_lb       = 0;                            \
-                ptrdiff_t true_extended = 0;                            \
-                ompi_datatype_type_size((DDT), &size);                       \
-                ompi_datatype_get_true_extent((DDT), &true_lb, &true_extended); \
-                if ( 0 < size && 0 == true_lb ) {                       \
+                size_t ompi_chk_usr_buf_size = 0;                       \
+                ptrdiff_t ompi_chk_usr_buf_true_lb = 0;                 \
+                ptrdiff_t ompi_chk_usr_buf_true_extended = 0;           \
+                ompi_datatype_type_size((DDT), &ompi_chk_usr_buf_size); \
+                ompi_datatype_get_true_extent((DDT),                    \
+                                             &ompi_chk_usr_buf_true_lb, \
+                                       &ompi_chk_usr_buf_true_extended);\
+                if ( 0 < ompi_chk_usr_buf_size &&                       \
+                    0 == ompi_chk_usr_buf_true_extended ) {             \
                     (RC) = MPI_ERR_BUFFER;                              \
                 }                                                       \
             }                                                           \
         }                                                               \
     } while (0)
 
+
+/* check for integer overflow - needed while parts of Open MPI have not been embiggened */
+#define OMPI_CHECK_MPI_COUNT_INT_CONVERSION_OVERFLOW(RC, x) \
+    do { \
+        if ((x) > INT_MAX) { \
+            (RC) = MPI_ERR_VALUE_TOO_LARGE; \
+        } else { \
+            (RC) = MPI_SUCCESS; \
+       }  \
+    } while (0)
+
+
 END_C_DECLS
 
 #endif /* OMPI_C_BINDINGS_H */
diff --git a/ompi/mpi/c/bsend.c b/ompi/mpi/c/bsend.c
deleted file mode 100644
index 881a16ce9bf..00000000000
--- a/ompi/mpi/c/bsend.c
+++ /dev/null
@@ -1,98 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/mca/pml/base/pml_base_bsend.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Bsend = PMPI_Bsend
-#endif
-#define MPI_Bsend PMPI_Bsend
-#endif
-
-static const char FUNC_NAME[] = "MPI_Bsend";
-
-
-int MPI_Bsend(const void *buf, int count, MPI_Datatype type, int dest, int tag, MPI_Comm comm)
-{
-    int rc = MPI_SUCCESS;
-
-    SPC_RECORD(OMPI_SPC_BSEND, 1);
-
-    MEMCHECKER(
-        memchecker_datatype(type);
-        memchecker_call(&opal_memchecker_base_isdefined, buf, count, type);
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else if (MPI_DATATYPE_NULL == type || NULL == type) {
-            rc = MPI_ERR_TYPE;
-        } else if (tag < 0 || tag > mca_pml.pml_max_tag) {
-            rc = MPI_ERR_TAG;
-        } else if (ompi_comm_peer_invalid(comm, dest) &&
-                   (MPI_PROC_NULL != dest)) {
-            rc = MPI_ERR_RANK;
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_SEND(rc, type, count);
-            OMPI_CHECK_USER_BUFFER(rc, buf, type, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-    }
-
-#if OPAL_ENABLE_FT_MPI
-        /*
-         * An early check, so as to return early if we are communicating with
-         * a failed process. This is not absolutely necessary since we will
-         * check for this, and other, error conditions during the completion
-         * call in the PML.
-         */
-        if( OPAL_UNLIKELY(!ompi_comm_iface_p2p_check_proc(comm, dest, &rc)) ) {
-            OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-        }
-#endif
-
-    if (MPI_PROC_NULL == dest) {
-        return MPI_SUCCESS;
-    }
-
-    rc = MCA_PML_CALL(send(buf, count, type, dest, tag, MCA_PML_BASE_SEND_BUFFERED, comm));
-    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/bsend.c.in b/ompi/mpi/c/bsend.c.in
new file mode 100644
index 00000000000..833f096d15f
--- /dev/null
+++ b/ompi/mpi/c/bsend.c.in
@@ -0,0 +1,91 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/mca/pml/base/pml_base_bsend.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS bsend(BUFFER buf, COUNT count, DATATYPE type, INT dest,
+                            INT tag, COMM comm)
+{
+    int rc = MPI_SUCCESS;
+
+    SPC_RECORD(OMPI_SPC_BSEND, 1);
+
+    MEMCHECKER(
+        memchecker_datatype(type);
+        memchecker_call(&opal_memchecker_base_isdefined, buf, count, type);
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else if (MPI_DATATYPE_NULL == type || NULL == type) {
+            rc = MPI_ERR_TYPE;
+        } else if (tag < 0 || tag > mca_pml.pml_max_tag) {
+            rc = MPI_ERR_TAG;
+        } else if (ompi_comm_peer_invalid(comm, dest) &&
+                   (MPI_PROC_NULL != dest)) {
+            rc = MPI_ERR_RANK;
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_SEND(rc, type, count);
+            OMPI_CHECK_USER_BUFFER(rc, buf, type, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+#if OPAL_ENABLE_FT_MPI
+        /*
+         * An early check, so as to return early if we are communicating with
+         * a failed process. This is not absolutely necessary since we will
+         * check for this, and other, error conditions during the completion
+         * call in the PML.
+         */
+        if( OPAL_UNLIKELY(!ompi_comm_iface_p2p_check_proc(comm, dest, &rc)) ) {
+            OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+        }
+#endif
+
+    if (MPI_PROC_NULL == dest) {
+        return MPI_SUCCESS;
+    }
+
+    rc = MCA_PML_CALL(send(buf, count, type, dest, tag, MCA_PML_BASE_SEND_BUFFERED, comm));
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/bsend_init.c b/ompi/mpi/c/bsend_init.c
deleted file mode 100644
index 48982dc7357..00000000000
--- a/ompi/mpi/c/bsend_init.c
+++ /dev/null
@@ -1,97 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/mca/pml/base/pml_base_bsend.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Bsend_init = PMPI_Bsend_init
-#endif
-#define MPI_Bsend_init PMPI_Bsend_init
-#endif
-
-static const char FUNC_NAME[] = "MPI_Bsend_init";
-
-
-int MPI_Bsend_init(const void *buf, int count, MPI_Datatype type,
-                   int dest, int tag, MPI_Comm comm, MPI_Request *request)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(type);
-        memchecker_call(&opal_memchecker_base_isaddressable, buf, count, type);
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else if (type == MPI_DATATYPE_NULL) {
-            rc = MPI_ERR_TYPE;
-        } else if (tag < 0 || tag > mca_pml.pml_max_tag) {
-            rc = MPI_ERR_TAG;
-        } else if (ompi_comm_peer_invalid(comm, dest) &&
-                   (MPI_PROC_NULL != dest)) {
-            rc = MPI_ERR_RANK;
-        } else if (request == NULL) {
-            rc = MPI_ERR_REQUEST;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-    }
-
-    if (MPI_PROC_NULL == dest) {
-        rc = ompi_request_persistent_noop_create(request);
-        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * The request will be checked for process failure errors during the
-     * completion calls. So no need to check here.
-     */
-#endif
-
-    /*
-     * Here, we just initialize the request -- memchecker should set the buffer in MPI_Start.
-     */
-    rc = MCA_PML_CALL(isend_init(buf, count, type, dest, tag,
-				 MCA_PML_BASE_SEND_BUFFERED, comm, request));
-    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/bsend_init.c.in b/ompi/mpi/c/bsend_init.c.in
new file mode 100644
index 00000000000..7a7bb5b0224
--- /dev/null
+++ b/ompi/mpi/c/bsend_init.c.in
@@ -0,0 +1,89 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/mca/pml/base/pml_base_bsend.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS bsend_init(BUFFER buf, COUNT count, DATATYPE type,
+                                 INT dest, INT tag, COMM comm, REQUEST_INOUT request)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(type);
+        memchecker_call(&opal_memchecker_base_isaddressable, buf, count, type);
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else if (type == MPI_DATATYPE_NULL) {
+            rc = MPI_ERR_TYPE;
+        } else if (tag < 0 || tag > mca_pml.pml_max_tag) {
+            rc = MPI_ERR_TAG;
+        } else if (ompi_comm_peer_invalid(comm, dest) &&
+                   (MPI_PROC_NULL != dest)) {
+            rc = MPI_ERR_RANK;
+        } else if (request == NULL) {
+            rc = MPI_ERR_REQUEST;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+    if (MPI_PROC_NULL == dest) {
+        rc = ompi_request_persistent_noop_create(request);
+        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * The request will be checked for process failure errors during the
+     * completion calls. So no need to check here.
+     */
+#endif
+
+    /*
+     * Here, we just initialize the request -- memchecker should set the buffer in MPI_Start.
+     */
+    rc = MCA_PML_CALL(isend_init(buf, count, type, dest, tag,
+				 MCA_PML_BASE_SEND_BUFFERED, comm, request));
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/buffer_attach.c b/ompi/mpi/c/buffer_attach.c
deleted file mode 100644
index be9a43adad6..00000000000
--- a/ompi/mpi/c/buffer_attach.c
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/mca/pml/base/pml_base_bsend.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Buffer_attach = PMPI_Buffer_attach
-#endif
-#define MPI_Buffer_attach PMPI_Buffer_attach
-#endif
-
-static const char FUNC_NAME[] = "MPI_Buffer_attach";
-
-
-int MPI_Buffer_attach(void *buffer, int size)
-{
-    int ret = OMPI_SUCCESS;
-
-  if (MPI_PARAM_CHECK) {
-    OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-    if (NULL == buffer || size < 0) {
-      return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-    }
-  }
-
-  ret = mca_pml_base_bsend_attach(buffer, size);
-
-  return ret;
-}
-
diff --git a/ompi/mpi/c/buffer_attach.c.in b/ompi/mpi/c/buffer_attach.c.in
new file mode 100644
index 00000000000..53d5421db8e
--- /dev/null
+++ b/ompi/mpi/c/buffer_attach.c.in
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/mca/pml/base/pml_base_bsend.h"
+
+PROTOTYPE ERROR_CLASS buffer_attach(BUFFER_OUT buffer, COUNT size)
+{
+    int ret = OMPI_SUCCESS;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == buffer || size < 0) {
+           return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+        }
+    }
+
+    ret = mca_pml_base_bsend_attach(buffer, size);
+
+    return ret;
+}
+
diff --git a/ompi/mpi/c/buffer_detach.c b/ompi/mpi/c/buffer_detach.c
deleted file mode 100644
index 9c5cd9e60a2..00000000000
--- a/ompi/mpi/c/buffer_detach.c
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/mca/pml/base/pml_base_bsend.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Buffer_detach = PMPI_Buffer_detach
-#endif
-#define MPI_Buffer_detach PMPI_Buffer_detach
-#endif
-
-static const char FUNC_NAME[] = "MPI_Buffer_detach";
-
-
-int MPI_Buffer_detach(void *buffer, int *size)
-{
-    int ret = OMPI_SUCCESS;
-
-  if (MPI_PARAM_CHECK) {
-    OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-    if (NULL == buffer || NULL == size) {
-      return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-    }
-  }
-
-  ret = mca_pml_base_bsend_detach(buffer, size);
-
-  return ret;
-}
diff --git a/ompi/mpi/c/buffer_detach.c.in b/ompi/mpi/c/buffer_detach.c.in
new file mode 100644
index 00000000000..6e15b64e3ef
--- /dev/null
+++ b/ompi/mpi/c/buffer_detach.c.in
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/mca/pml/base/pml_base_bsend.h"
+
+PROTOTYPE ERROR_CLASS buffer_detach(BUFFER_OUT buffer, COUNT_OUT size)
+{
+    size_t size_arg;
+    int ret = OMPI_SUCCESS;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == buffer || NULL == size) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+        }
+    }
+
+    ret = mca_pml_base_bsend_detach(buffer, &size_arg);
+    if (MPI_SUCCESS == ret) {
+        *size = (int)size_arg;
+    }
+
+    return ret;
+}
diff --git a/ompi/mpi/c/cancel.c b/ompi/mpi/c/cancel.c
deleted file mode 100644
index c0e356d542c..00000000000
--- a/ompi/mpi/c/cancel.c
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/request/request.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Cancel = PMPI_Cancel
-#endif
-#define MPI_Cancel PMPI_Cancel
-#endif
-
-static const char FUNC_NAME[] = "MPI_Cancel";
-
-
-int MPI_Cancel(MPI_Request *request)
-{
-    int rc;
-
-    SPC_RECORD(OMPI_SPC_CANCEL, 1);
-
-    MEMCHECKER(
-        memchecker_request(request);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == request || NULL == *request ||
-            MPI_REQUEST_NULL == *request) {
-            OMPI_ERRHANDLER_NOHANDLE_RETURN(MPI_ERR_REQUEST, 
-                                   MPI_ERR_REQUEST, FUNC_NAME);
-        }
-    }
-
-    if (MPI_REQUEST_NULL == *request) {
-        return MPI_SUCCESS;
-    }
-
-    rc = ompi_request_cancel(*request);
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/cancel.c.in b/ompi/mpi/c/cancel.c.in
new file mode 100644
index 00000000000..c721098b6e8
--- /dev/null
+++ b/ompi/mpi/c/cancel.c.in
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/request/request.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS cancel(REQUEST_INOUT request)
+{
+    int rc;
+
+    SPC_RECORD(OMPI_SPC_CANCEL, 1);
+
+    MEMCHECKER(
+        memchecker_request(request);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == request || NULL == *request ||
+            MPI_REQUEST_NULL == *request) {
+            OMPI_ERRHANDLER_NOHANDLE_RETURN(MPI_ERR_REQUEST, 
+                                   MPI_ERR_REQUEST, FUNC_NAME);
+        }
+    }
+
+    if (MPI_REQUEST_NULL == *request) {
+        return MPI_SUCCESS;
+    }
+
+    rc = ompi_request_cancel(*request);
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/cart_coords.c b/ompi/mpi/c/cart_coords.c
deleted file mode 100644
index fcc6ca105fb..00000000000
--- a/ompi/mpi/c/cart_coords.c
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2013 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012      Los Alamos Nat Security, LLC. All rights reserved.
- * Copyright (c) 2012-2013 Inria.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/topo/topo.h"
-#include "ompi/group/group.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Cart_coords = PMPI_Cart_coords
-#endif
-#define MPI_Cart_coords PMPI_Cart_coords
-#endif
-
-static const char FUNC_NAME[] = "MPI_Cart_coords";
-
-int MPI_Cart_coords(MPI_Comm comm, int rank, int maxdims, int coords[])
-{
-    int err;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    /* check the arguments */
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-        if (OMPI_COMM_IS_INTER(comm)) {
-            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-        if ( (0 > maxdims) || ((0 < maxdims) && (NULL == coords))) {
-            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-        if ((0 > rank) || (rank > ompi_group_size(comm->c_local_group))) {
-            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_RANK,
-                                          FUNC_NAME);
-        }
-    }
-
-    if (!OMPI_COMM_IS_CART(comm)) {
-        return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_TOPOLOGY,
-                                      FUNC_NAME);
-    }
-
-    err = comm->c_topo->topo.cart.cart_coords(comm, rank, maxdims, coords);
-
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/cart_coords.c.in b/ompi/mpi/c/cart_coords.c.in
new file mode 100644
index 00000000000..edd02f68a91
--- /dev/null
+++ b/ompi/mpi/c/cart_coords.c.in
@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2013 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Los Alamos Nat Security, LLC. All rights reserved.
+ * Copyright (c) 2012-2013 Inria.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/group/group.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS cart_coords(COMM comm, INT rank, INT maxdims, INT_OUT coords)
+{
+    int err;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    /* check the arguments */
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+        if (OMPI_COMM_IS_INTER(comm)) {
+            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+        if ( (0 > maxdims) || ((0 < maxdims) && (NULL == coords))) {
+            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+        if ((0 > rank) || (rank > ompi_group_size(comm->c_local_group))) {
+            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_RANK,
+                                          FUNC_NAME);
+        }
+    }
+
+    if (!OMPI_COMM_IS_CART(comm)) {
+        return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_TOPOLOGY,
+                                      FUNC_NAME);
+    }
+
+    err = comm->c_topo->topo.cart.cart_coords(comm, rank, maxdims, coords);
+
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/cart_create.c b/ompi/mpi/c/cart_create.c
deleted file mode 100644
index 64f18827959..00000000000
--- a/ompi/mpi/c/cart_create.c
+++ /dev/null
@@ -1,126 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007-2008 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2012-2013 Inria.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/topo/base/base.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Cart_create = PMPI_Cart_create
-#endif
-#define MPI_Cart_create PMPI_Cart_create
-#endif
-
-static const char FUNC_NAME[] = "MPI_Cart_create";
-
-
-int MPI_Cart_create(MPI_Comm old_comm, int ndims, const int dims[],
-                    const int periods[], int reorder, MPI_Comm *comm_cart)
-{
-    mca_topo_base_module_t* topo;
-    int err;
-
-    MEMCHECKER(
-        memchecker_comm(old_comm);
-    );
-
-    /* check the arguments */
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(old_comm)) {
-            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if (OMPI_COMM_IS_INTER(old_comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-        if (ndims < 0) {
-            return OMPI_ERRHANDLER_INVOKE (old_comm, MPI_ERR_ARG,
-                                          FUNC_NAME);
-        } else if (ndims >= 1 &&
-                   (NULL == dims || NULL == periods || NULL == comm_cart)) {
-            return OMPI_ERRHANDLER_INVOKE (old_comm, MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-
-        /* check if the number of processes on the grid are correct */
-        {
-           int i, count_nodes = 1;
-           const int *p = dims;
-           int parent_procs = ompi_comm_size(old_comm);
-
-           for (i=0; i < ndims; i++, p++) {
-               count_nodes *= *p;
-           }
-
-           if (parent_procs < count_nodes) {
-               return OMPI_ERRHANDLER_INVOKE (old_comm, MPI_ERR_ARG,
-                                              FUNC_NAME);
-           }
-        }
-    }
-
-    /*
-     * everything seems to be alright with the communicator, we can go
-     * ahead and select a topology module for this purpose and create
-     * the new graph communicator
-     */
-    if (OMPI_SUCCESS != (err = mca_topo_base_comm_select(old_comm,
-                                                         NULL,
-                                                         &topo,
-                                                         OMPI_COMM_CART))) {
-        return err;
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_create_check(old_comm, &err)) ) {
-       OMPI_ERRHANDLER_RETURN(err, old_comm, err, FUNC_NAME);
-    }
-#endif
-
-    /* Now let that topology module rearrange procs/ranks if it wants to */
-    err = topo->topo.cart.cart_create(topo, old_comm,
-                                      ndims, dims, periods,
-                                      (0 == reorder) ? false : true, comm_cart);
-
-    if (MPI_SUCCESS != err) {
-        OBJ_RELEASE(topo);
-        return OMPI_ERRHANDLER_INVOKE(old_comm, err, FUNC_NAME);
-    }
-
-    /* All done */
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/cart_create.c.in b/ompi/mpi/c/cart_create.c.in
new file mode 100644
index 00000000000..54c84f5d746
--- /dev/null
+++ b/ompi/mpi/c/cart_create.c.in
@@ -0,0 +1,118 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007-2008 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2012-2013 Inria.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/topo/base/base.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS cart_create(COMM old_comm, INT  ndims, INT_ARRAY dims,
+                                  INT_ARRAY periods, INT reorder, COMM_OUT comm_cart)
+{
+    mca_topo_base_module_t* topo;
+    int err;
+
+    MEMCHECKER(
+        memchecker_comm(old_comm);
+    );
+
+    /* check the arguments */
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(old_comm)) {
+            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if (OMPI_COMM_IS_INTER(old_comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+        if (ndims < 0) {
+            return OMPI_ERRHANDLER_INVOKE (old_comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+        } else if (ndims >= 1 &&
+                   (NULL == dims || NULL == periods || NULL == comm_cart)) {
+            return OMPI_ERRHANDLER_INVOKE (old_comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+
+        /* check if the number of processes on the grid are correct */
+        {
+           int i, count_nodes = 1;
+           const int *p = dims;
+           int parent_procs = ompi_comm_size(old_comm);
+
+           for (i=0; i < ndims; i++, p++) {
+               count_nodes *= *p;
+           }
+
+           if (parent_procs < count_nodes) {
+               return OMPI_ERRHANDLER_INVOKE (old_comm, MPI_ERR_ARG,
+                                              FUNC_NAME);
+           }
+        }
+    }
+
+    /*
+     * everything seems to be alright with the communicator, we can go
+     * ahead and select a topology module for this purpose and create
+     * the new graph communicator
+     */
+    if (OMPI_SUCCESS != (err = mca_topo_base_comm_select(old_comm,
+                                                         NULL,
+                                                         &topo,
+                                                         OMPI_COMM_CART))) {
+        return err;
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_create_check(old_comm, &err)) ) {
+       OMPI_ERRHANDLER_RETURN(err, old_comm, err, FUNC_NAME);
+    }
+#endif
+
+    /* Now let that topology module rearrange procs/ranks if it wants to */
+    err = topo->topo.cart.cart_create(topo, old_comm,
+                                      ndims, dims, periods,
+                                      (0 == reorder) ? false : true, comm_cart);
+
+    if (MPI_SUCCESS != err) {
+        OBJ_RELEASE(topo);
+        return OMPI_ERRHANDLER_INVOKE(old_comm, err, FUNC_NAME);
+    }
+
+    /* All done */
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/cart_get.c b/ompi/mpi/c/cart_get.c
deleted file mode 100644
index 93242234d9a..00000000000
--- a/ompi/mpi/c/cart_get.c
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2013 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012      Los Alamos Nat Security, LLC. All rights reserved.
- * Copyright (c) 2012-2013 Inria.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/topo/topo.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Cart_get = PMPI_Cart_get
-#endif
-#define MPI_Cart_get PMPI_Cart_get
-#endif
-
-static const char FUNC_NAME[] = "MPI_Cart_get";
-
-int MPI_Cart_get(MPI_Comm comm, int maxdims, int dims[],
-                 int periods[], int coords[])
-{
-    int err;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    /* check the arguments */
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
-            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-        if ((0 > maxdims) || (0 < maxdims &&
-                              ((NULL == dims) || (NULL == periods) ||
-                               (NULL == coords)))) {
-            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-    }
-
-    if (!OMPI_COMM_IS_CART(comm)) {
-        return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_TOPOLOGY,
-                                      FUNC_NAME);
-    }
-
-    err = comm->c_topo->topo.cart.cart_get(comm, maxdims, dims, periods, coords);
-
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/cart_get.c.in b/ompi/mpi/c/cart_get.c.in
new file mode 100644
index 00000000000..d7d222def11
--- /dev/null
+++ b/ompi/mpi/c/cart_get.c.in
@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2013 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Los Alamos Nat Security, LLC. All rights reserved.
+ * Copyright (c) 2012-2013 Inria.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS cart_get(COMM comm, INT maxdims, INT_OUT dims,
+                               INT_OUT periods, INT_OUT coords)
+{
+    int err;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    /* check the arguments */
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
+            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+        if ((0 > maxdims) || (0 < maxdims &&
+                              ((NULL == dims) || (NULL == periods) ||
+                               (NULL == coords)))) {
+            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+    if (!OMPI_COMM_IS_CART(comm)) {
+        return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_TOPOLOGY,
+                                      FUNC_NAME);
+    }
+
+    err = comm->c_topo->topo.cart.cart_get(comm, maxdims, dims, periods, coords);
+
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/cart_map.c b/ompi/mpi/c/cart_map.c
deleted file mode 100644
index 62bb04909d2..00000000000
--- a/ompi/mpi/c/cart_map.c
+++ /dev/null
@@ -1,82 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2013 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2012-2013 Inria.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/topo/topo.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Cart_map = PMPI_Cart_map
-#endif
-#define MPI_Cart_map PMPI_Cart_map
-#endif
-
-static const char FUNC_NAME[] = "MPI_Cart_map";
-
-
-int MPI_Cart_map(MPI_Comm comm, int ndims, const int dims[],
-                 const int periods[], int *newrank)
-{
-    int err = MPI_SUCCESS;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    /* check the arguments */
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-        if (OMPI_COMM_IS_INTER(comm)) {
-            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-        if ((NULL == dims) || (NULL == periods) || (NULL == newrank)) {
-            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-    }
-
-    if(!OMPI_COMM_IS_CART(comm)) {
-        /* In case the communicator has no topo-module attached to
-           it, we just return the "default" value suggested by MPI:
-           newrank = rank */
-        *newrank = ompi_comm_rank(comm);
-    } else {
-        err = comm->c_topo->topo.cart.cart_map(comm, ndims, dims,
-                                               periods, newrank);
-    }
-
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/cart_map.c.in b/ompi/mpi/c/cart_map.c.in
new file mode 100644
index 00000000000..c5d6c12649b
--- /dev/null
+++ b/ompi/mpi/c/cart_map.c.in
@@ -0,0 +1,74 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2013 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2012-2013 Inria.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS cart_map(COMM comm, INT ndims, INT_ARRAY dims,
+                               INT_ARRAY periods, INT_OUT newrank)
+{
+    int err = MPI_SUCCESS;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    /* check the arguments */
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+        if (OMPI_COMM_IS_INTER(comm)) {
+            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+        if ((NULL == dims) || (NULL == periods) || (NULL == newrank)) {
+            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+    if(!OMPI_COMM_IS_CART(comm)) {
+        /* In case the communicator has no topo-module attached to
+           it, we just return the "default" value suggested by MPI:
+           newrank = rank */
+        *newrank = ompi_comm_rank(comm);
+    } else {
+        err = comm->c_topo->topo.cart.cart_map(comm, ndims, dims,
+                                               periods, newrank);
+    }
+
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/cart_rank.c b/ompi/mpi/c/cart_rank.c
deleted file mode 100644
index 979dbd22434..00000000000
--- a/ompi/mpi/c/cart_rank.c
+++ /dev/null
@@ -1,104 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2013 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007-2015 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2012-2013 Inria.  All rights reserved.
- * Copyright (c) 2014-2015 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/topo/topo.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Cart_rank = PMPI_Cart_rank
-#endif
-#define MPI_Cart_rank PMPI_Cart_rank
-#endif
-
-static const char FUNC_NAME[] = "MPI_Cart_rank";
-
-int MPI_Cart_rank(MPI_Comm comm, const int coords[], int *rank)
-{
-    int i, err;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    /* check the arguments */
-    if (MPI_PARAM_CHECK) {
-        mca_topo_base_comm_cart_2_2_0_t* cart;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-        if (OMPI_COMM_IS_INTER(comm)) {
-            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-
-        /* Need this check here to protect the access to "cart",
-           below.  I.e., if OMPI_COMM_IS_CART is true, then cart is
-           guaranteed to be != NULL. */
-        if (!OMPI_COMM_IS_CART(comm)) {
-            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_TOPOLOGY,
-                                           FUNC_NAME);
-        }
-
-        cart = comm->c_topo->mtc.cart;
-        /* Per MPI-2.1, coords is only relevant if the dimension of
-           the cartesian comm is >0 */
-        if (((NULL == coords) &&
-             (cart->ndims >= 1)) ||
-            (NULL == rank)){
-            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-
-        /* Check if coords[i] is within the acceptable range if
-           dimension i is not periodic */
-        for (i = 0; i < cart->ndims; ++i) {
-            if (!cart->periods[i] &&
-                (coords[i] < 0 ||
-                 coords[i] >= cart->dims[i])) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-    } else {
-        /* Need to always test for cartesian communicators, even in
-           the !MPI_PARAM_CHECK case. */
-        if (!OMPI_COMM_IS_CART(comm)) {
-            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_TOPOLOGY,
-                                           FUNC_NAME);
-        }
-    }
-
-    err = comm->c_topo->topo.cart.cart_rank(comm, coords, rank);
-
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/cart_rank.c.in b/ompi/mpi/c/cart_rank.c.in
new file mode 100644
index 00000000000..64f4fe72e76
--- /dev/null
+++ b/ompi/mpi/c/cart_rank.c.in
@@ -0,0 +1,97 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2013 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007-2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2012-2013 Inria.  All rights reserved.
+ * Copyright (c) 2014-2015 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS cart_rank(COMM comm, INT_ARRAY coords, INT_OUT rank)
+{
+    int i, err;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    /* check the arguments */
+    if (MPI_PARAM_CHECK) {
+        mca_topo_base_comm_cart_2_2_0_t* cart;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+        if (OMPI_COMM_IS_INTER(comm)) {
+            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+
+        /* Need this check here to protect the access to "cart",
+           below.  I.e., if OMPI_COMM_IS_CART is true, then cart is
+           guaranteed to be != NULL. */
+        if (!OMPI_COMM_IS_CART(comm)) {
+            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_TOPOLOGY,
+                                           FUNC_NAME);
+        }
+
+        cart = comm->c_topo->mtc.cart;
+        /* Per MPI-2.1, coords is only relevant if the dimension of
+           the cartesian comm is >0 */
+        if (((NULL == coords) &&
+             (cart->ndims >= 1)) ||
+            (NULL == rank)){
+            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+
+        /* Check if coords[i] is within the acceptable range if
+           dimension i is not periodic */
+        for (i = 0; i < cart->ndims; ++i) {
+            if (!cart->periods[i] &&
+                (coords[i] < 0 ||
+                 coords[i] >= cart->dims[i])) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+    } else {
+        /* Need to always test for cartesian communicators, even in
+           the !MPI_PARAM_CHECK case. */
+        if (!OMPI_COMM_IS_CART(comm)) {
+            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_TOPOLOGY,
+                                           FUNC_NAME);
+        }
+    }
+
+    err = comm->c_topo->topo.cart.cart_rank(comm, coords, rank);
+
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/cart_shift.c b/ompi/mpi/c/cart_shift.c
deleted file mode 100644
index 99d07693114..00000000000
--- a/ompi/mpi/c/cart_shift.c
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2013 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Inria.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/topo/topo.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Cart_shift = PMPI_Cart_shift
-#endif
-#define MPI_Cart_shift PMPI_Cart_shift
-#endif
-
-static const char FUNC_NAME[] = "MPI_Cart_shift";
-
-
-int MPI_Cart_shift(MPI_Comm comm, int direction, int disp,
-                   int *rank_source, int *rank_dest)
-{
-    int err;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    /* check the arguments */
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-        if (OMPI_COMM_IS_INTER(comm)) {
-            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-        if (0 > direction) { /* yet to detect direction >= comm->c_topo_ndims */
-            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_DIMS,
-                                          FUNC_NAME);
-        }
-        if (NULL == rank_source || NULL == rank_dest) {
-            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-    }
-
-    if (!OMPI_COMM_IS_CART(comm)) {
-        return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_TOPOLOGY,
-                                      FUNC_NAME);
-    }
-
-    /* call the function */
-    err = comm->c_topo->topo.cart.cart_shift(comm, direction, disp, rank_source, rank_dest);
-
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/cart_shift.c.in b/ompi/mpi/c/cart_shift.c.in
new file mode 100644
index 00000000000..3b25ead1fee
--- /dev/null
+++ b/ompi/mpi/c/cart_shift.c.in
@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2013 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Inria.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS cart_shift(COMM comm, INT direction, INT disp,
+                                 INT_OUT rank_source, INT_OUT rank_dest)
+{
+    int err;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    /* check the arguments */
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+        if (OMPI_COMM_IS_INTER(comm)) {
+            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+        if (0 > direction) { /* yet to detect direction >= comm->c_topo_ndims */
+            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_DIMS,
+                                          FUNC_NAME);
+        }
+        if (NULL == rank_source || NULL == rank_dest) {
+            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+    if (!OMPI_COMM_IS_CART(comm)) {
+        return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_TOPOLOGY,
+                                      FUNC_NAME);
+    }
+
+    /* call the function */
+    err = comm->c_topo->topo.cart.cart_shift(comm, direction, disp, rank_source, rank_dest);
+
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/cart_sub.c b/ompi/mpi/c/cart_sub.c
deleted file mode 100644
index 9cf0ea9717c..00000000000
--- a/ompi/mpi/c/cart_sub.c
+++ /dev/null
@@ -1,90 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007-2009 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2012-2013 Inria.  All rights reserved.
- * Copyright (c) 2014-2015 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/topo/topo.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Cart_sub = PMPI_Cart_sub
-#endif
-#define MPI_Cart_sub PMPI_Cart_sub
-#endif
-
-static const char FUNC_NAME[] = "MPI_Cart_sub";
-
-
-int MPI_Cart_sub(MPI_Comm comm, const int remain_dims[], MPI_Comm *new_comm)
-{
-    int err;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    /* check the arguments */
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-        if (OMPI_COMM_IS_INTER(comm)) {
-            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-        if (((NULL == remain_dims) && (0 != comm->c_topo->mtc.cart->ndims))
-            && (NULL == new_comm)) {
-            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-    }
-
-    if (!OMPI_COMM_IS_CART(comm)) {
-        return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_TOPOLOGY,
-                                      FUNC_NAME);
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_create_check(comm, &err)) ) {
-        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-    }
-#endif
-
-    err = comm->c_topo->topo.cart.cart_sub(comm, remain_dims, new_comm);
-
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/cart_sub.c.in b/ompi/mpi/c/cart_sub.c.in
new file mode 100644
index 00000000000..939cace0e91
--- /dev/null
+++ b/ompi/mpi/c/cart_sub.c.in
@@ -0,0 +1,82 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2017 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007-2009 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2012-2013 Inria.  All rights reserved.
+ * Copyright (c) 2014-2015 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS cart_sub(COMM comm, INT_ARRAY remain_dims, COMM_OUT new_comm)
+{
+    int err;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    /* check the arguments */
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+        if (OMPI_COMM_IS_INTER(comm)) {
+            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+        if (((NULL == remain_dims) && (0 != comm->c_topo->mtc.cart->ndims))
+            && (NULL == new_comm)) {
+            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+    if (!OMPI_COMM_IS_CART(comm)) {
+        return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_TOPOLOGY,
+                                      FUNC_NAME);
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_create_check(comm, &err)) ) {
+        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+    }
+#endif
+
+    err = comm->c_topo->topo.cart.cart_sub(comm, remain_dims, new_comm);
+
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/cartdim_get.c b/ompi/mpi/c/cartdim_get.c
deleted file mode 100644
index 17e1fbaead4..00000000000
--- a/ompi/mpi/c/cartdim_get.c
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2013 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Inria.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/topo/topo.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Cartdim_get = PMPI_Cartdim_get
-#endif
-#define MPI_Cartdim_get PMPI_Cartdim_get
-#endif
-
-
-static const char FUNC_NAME[] = "MPI_Cartdim_get";
-
-
-int MPI_Cartdim_get(MPI_Comm comm, int *ndims)
-{
-    int err;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    if (MPI_PARAM_CHECK) {
-       OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-       if (ompi_comm_invalid(comm)) {
-           return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_COMM,
-                                        FUNC_NAME);
-       }
-       if (OMPI_COMM_IS_INTER(comm)) {
-           return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_COMM,
-                                         FUNC_NAME);
-       }
-       if (NULL == ndims) {
-           return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_ARG,
-                                          FUNC_NAME);
-       }
-    }
-
-    if (!OMPI_COMM_IS_CART(comm)) {
-        return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_TOPOLOGY,
-                                       FUNC_NAME);
-    }
-
-    err = comm->c_topo->topo.cart.cartdim_get(comm, ndims);
-
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/cartdim_get.c.in b/ompi/mpi/c/cartdim_get.c.in
new file mode 100644
index 00000000000..cad3d1a9af0
--- /dev/null
+++ b/ompi/mpi/c/cartdim_get.c.in
@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2013 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Inria.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS cartdim_get(COMM comm, INT_OUT ndims)
+{
+    int err;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if (MPI_PARAM_CHECK) {
+       OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+       if (ompi_comm_invalid(comm)) {
+           return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_COMM,
+                                        FUNC_NAME);
+       }
+       if (OMPI_COMM_IS_INTER(comm)) {
+           return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_COMM,
+                                         FUNC_NAME);
+       }
+       if (NULL == ndims) {
+           return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+       }
+    }
+
+    if (!OMPI_COMM_IS_CART(comm)) {
+        return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_TOPOLOGY,
+                                       FUNC_NAME);
+    }
+
+    err = comm->c_topo->topo.cart.cartdim_get(comm, ndims);
+
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/close_port.c b/ompi/mpi/c/close_port.c
deleted file mode 100644
index 062d2f0ae51..00000000000
--- a/ompi/mpi/c/close_port.c
+++ /dev/null
@@ -1,59 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Intel, Inc. All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/dpm/dpm.h"
-
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Close_port = PMPI_Close_port
-#endif
-#define MPI_Close_port PMPI_Close_port
-#endif
-
-static const char FUNC_NAME[] = "MPI_Close_port";
-
-
-int MPI_Close_port(const char *port_name)
-{
-    int ret;
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if ( NULL == port_name )
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-    }
-
-    ret = ompi_dpm_close_port(port_name);
-
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(ret, ret, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/close_port.c.in b/ompi/mpi/c/close_port.c.in
new file mode 100644
index 00000000000..a351858a48e
--- /dev/null
+++ b/ompi/mpi/c/close_port.c.in
@@ -0,0 +1,50 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Intel, Inc. All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/dpm/dpm.h"
+
+PROTOTYPE ERROR_CLASS close_port(STRING port_name)
+{
+    int ret;
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if ( NULL == port_name )
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+    }
+
+    ret = ompi_dpm_close_port(port_name);
+
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(ret, ret, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/comm_accept.c b/ompi/mpi/c/comm_accept.c
deleted file mode 100644
index 52ceaf3ea36..00000000000
--- a/ompi/mpi/c/comm_accept.c
+++ /dev/null
@@ -1,134 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2015 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2008      University of Houston, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Intel, Inc. All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "opal/util/show_help.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/runtime/mpiruntime.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-#include "ompi/dpm/dpm.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_accept = PMPI_Comm_accept
-#endif
-#define MPI_Comm_accept PMPI_Comm_accept
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_accept";
-
-
-int MPI_Comm_accept(const char *port_name, MPI_Info info, int root,
-                    MPI_Comm comm, MPI_Comm *newcomm)
-{
-    int rank, rc;
-    bool send_first=false; /* we receive first */
-    ompi_communicator_t *newcomp=MPI_COMM_NULL;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_comm_invalid (comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-        if ( OMPI_COMM_IS_INTER(comm)) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-        if ( (0 > root) || (ompi_comm_size(comm) <= root) ) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-        if ( NULL == newcomm ) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-        if (NULL == info || ompi_info_is_freed(info)) {
-          return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
-                                        FUNC_NAME);
-        }
-    }
-
-    rank = ompi_comm_rank ( comm );
-    if ( MPI_PARAM_CHECK ) {
-        if ( rank == root ) {
-            if ( NULL == port_name )
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
-                                              FUNC_NAME);
-        }
-    }
-
-    if (!ompi_mpi_dynamics_is_enabled(FUNC_NAME)) {
-        return OMPI_ERRHANDLER_INVOKE(comm, OMPI_ERR_NOT_SUPPORTED, FUNC_NAME);
-    }
-
-    /* parse info object. no prefedined values for this function in MPI-2
-     * so lets ignore it for the moment.
-     * if ( rank == root && MPI_INFO_NULL != info ) {
-     * }
-     */
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * We must not call ompi_comm_iface_create_check() here, because that
-     * risks leaving the connect side dangling on an unmatched operation.
-     * We will let the connect_accept logic proceed and discover the
-     * issue internally so that all sides get informed.
-     */
-#endif
-
-    if ( rank == root ) {
-	rc = ompi_dpm_connect_accept (comm, root, port_name, send_first,
-				      &newcomp);
-    }
-    else {
-	rc = ompi_dpm_connect_accept (comm, root, NULL, send_first,
-				      &newcomp);
-    }
-
-
-    if (OPAL_ERR_NOT_SUPPORTED == rc) {
-        opal_show_help("help-mpi-api.txt",
-                       "MPI function not supported",
-                       true,
-                       FUNC_NAME,
-                       "Underlying runtime environment does not support accept/connect functionality");
-    }
-
-    *newcomm = newcomp;
-    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME );
-}
diff --git a/ompi/mpi/c/comm_accept.c.in b/ompi/mpi/c/comm_accept.c.in
new file mode 100644
index 00000000000..5e04389c633
--- /dev/null
+++ b/ompi/mpi/c/comm_accept.c.in
@@ -0,0 +1,126 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2008      University of Houston, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Intel, Inc. All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "opal/util/show_help.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/runtime/mpiruntime.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include "ompi/dpm/dpm.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS comm_accept(STRING port_name, INFO info, INT root,
+                                  COMM comm, COMM_OUT newcomm)
+{
+    int rank, rc;
+    bool send_first=false; /* we receive first */
+    ompi_communicator_t *newcomp=MPI_COMM_NULL;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_comm_invalid (comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+        if ( OMPI_COMM_IS_INTER(comm)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+        if ( (0 > root) || (ompi_comm_size(comm) <= root) ) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+        if ( NULL == newcomm ) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+        if (NULL == info || ompi_info_is_freed(info)) {
+          return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
+                                        FUNC_NAME);
+        }
+    }
+
+    rank = ompi_comm_rank ( comm );
+    if ( MPI_PARAM_CHECK ) {
+        if ( rank == root ) {
+            if ( NULL == port_name )
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
+                                              FUNC_NAME);
+        }
+    }
+
+    if (!ompi_mpi_dynamics_is_enabled(FUNC_NAME)) {
+        return OMPI_ERRHANDLER_INVOKE(comm, OMPI_ERR_NOT_SUPPORTED, FUNC_NAME);
+    }
+
+    /* parse info object. no prefedined values for this function in MPI-2
+     * so lets ignore it for the moment.
+     * if ( rank == root && MPI_INFO_NULL != info ) {
+     * }
+     */
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * We must not call ompi_comm_iface_create_check() here, because that
+     * risks leaving the connect side dangling on an unmatched operation.
+     * We will let the connect_accept logic proceed and discover the
+     * issue internally so that all sides get informed.
+     */
+#endif
+
+    if ( rank == root ) {
+	rc = ompi_dpm_connect_accept (comm, root, port_name, send_first,
+				      &newcomp);
+    }
+    else {
+	rc = ompi_dpm_connect_accept (comm, root, NULL, send_first,
+				      &newcomp);
+    }
+
+
+    if (OPAL_ERR_NOT_SUPPORTED == rc) {
+        opal_show_help("help-mpi-api.txt",
+                       "MPI function not supported",
+                       true,
+                       FUNC_NAME,
+                       "Underlying runtime environment does not support accept/connect functionality");
+    }
+
+    *newcomm = newcomp;
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME );
+}
diff --git a/ompi/mpi/c/comm_c2f.c b/ompi/mpi/c/comm_c2f.c
deleted file mode 100644
index 794cc3c90f1..00000000000
--- a/ompi/mpi/c/comm_c2f.c
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mpi/fortran/base/fint_2_int.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_c2f = PMPI_Comm_c2f
-#endif
-#define MPI_Comm_c2f PMPI_Comm_c2f
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_c2f";
-
-
-MPI_Fint MPI_Comm_c2f(MPI_Comm comm)
-{
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        /* Note that ompi_comm_invalid() explicitly checks for
-           MPI_COMM_NULL, but MPI_COMM_C2F is supposed to treat
-           MPI_COMM_NULL as a valid communicator (and therefore return
-           a valid Fortran handle for it).  Hence, this function
-           should not return an error if MPI_COMM_NULL is passed in.
-
-           See a big comment in ompi/communicator/communicator.h about
-           this. */
-        if (ompi_comm_invalid (comm) && MPI_COMM_NULL != comm) {
-            return OMPI_INT_2_FINT(-1);
-        }
-    }
-
-    return OMPI_INT_2_FINT(comm->c_f_to_c_index);
-}
diff --git a/ompi/mpi/c/comm_c2f.c.in b/ompi/mpi/c/comm_c2f.c.in
new file mode 100644
index 00000000000..2a70b75995f
--- /dev/null
+++ b/ompi/mpi/c/comm_c2f.c.in
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE FINT comm_c2f(COMM comm)
+{
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        /* Note that ompi_comm_invalid() explicitly checks for
+           MPI_COMM_NULL, but MPI_COMM_C2F is supposed to treat
+           MPI_COMM_NULL as a valid communicator (and therefore return
+           a valid Fortran handle for it).  Hence, this function
+           should not return an error if MPI_COMM_NULL is passed in.
+
+           See a big comment in ompi/communicator/communicator.h about
+           this. */
+        if (ompi_comm_invalid (comm) && MPI_COMM_NULL != comm) {
+            return OMPI_INT_2_FINT(-1);
+        }
+    }
+
+    return OMPI_INT_2_FINT(comm->c_f_to_c_index);
+}
diff --git a/ompi/mpi/c/comm_call_errhandler.c b/ompi/mpi/c/comm_call_errhandler.c
deleted file mode 100644
index b9db6285c16..00000000000
--- a/ompi/mpi/c/comm_call_errhandler.c
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_call_errhandler = PMPI_Comm_call_errhandler
-#endif
-#define MPI_Comm_call_errhandler PMPI_Comm_call_errhandler
-#endif
-
-
-static const char FUNC_NAME[] = "MPI_Comm_call_errhandler";
-
-
-int MPI_Comm_call_errhandler(MPI_Comm comm, int errorcode)
-{
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-  /* Error checking */
-
-  if (MPI_PARAM_CHECK) {
-    OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-    if (ompi_comm_invalid(comm)) {
-      return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                    FUNC_NAME);
-    }
-  }
-
-  /* Invoke the errhandler */
-
-  OMPI_ERRHANDLER_INVOKE(comm, errorcode, FUNC_NAME);
-
-  /* See MPI-2 8.5 why this function has to return MPI_SUCCESS */
-  return MPI_SUCCESS;
-}
-
diff --git a/ompi/mpi/c/comm_call_errhandler.c.in b/ompi/mpi/c/comm_call_errhandler.c.in
new file mode 100644
index 00000000000..ffe26023940
--- /dev/null
+++ b/ompi/mpi/c/comm_call_errhandler.c.in
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS comm_call_errhandler(COMM comm, INT errorcode)
+{
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+  /* Error checking */
+
+  if (MPI_PARAM_CHECK) {
+    OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+    if (ompi_comm_invalid(comm)) {
+      return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                    FUNC_NAME);
+    }
+  }
+
+  /* Invoke the errhandler */
+
+  OMPI_ERRHANDLER_INVOKE(comm, errorcode, FUNC_NAME);
+
+  /* See MPI-2 8.5 why this function has to return MPI_SUCCESS */
+  return MPI_SUCCESS;
+}
+
diff --git a/ompi/mpi/c/comm_compare.c b/ompi/mpi/c/comm_compare.c
deleted file mode 100644
index 55d4b0d38d1..00000000000
--- a/ompi/mpi/c/comm_compare.c
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_compare = PMPI_Comm_compare
-#endif
-#define MPI_Comm_compare PMPI_Comm_compare
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_compare";
-
-
-int MPI_Comm_compare(MPI_Comm comm1, MPI_Comm comm2, int *result) {
-
-    int rc;
-
-    MEMCHECKER(
-        memchecker_comm(comm1);
-        memchecker_comm(comm2);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_comm_invalid(comm1) || ompi_comm_invalid(comm2)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-
-        if ( NULL == result ) {
-            return OMPI_ERRHANDLER_INVOKE(comm1, MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-    }
-
-    rc = ompi_comm_compare ( (ompi_communicator_t*)comm1,
-                             (ompi_communicator_t*)comm2,
-                             result);
-    OMPI_ERRHANDLER_RETURN ( rc, comm1, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/comm_compare.c.in b/ompi/mpi/c/comm_compare.c.in
new file mode 100644
index 00000000000..7cb247d6baf
--- /dev/null
+++ b/ompi/mpi/c/comm_compare.c.in
@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS comm_compare(COMM comm1, COMM comm2, INT_OUT result)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_comm(comm1);
+        memchecker_comm(comm2);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_comm_invalid(comm1) || ompi_comm_invalid(comm2))
+	{
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+
+        if ( NULL == result ) {
+            return OMPI_ERRHANDLER_INVOKE(comm1, MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+    rc = ompi_comm_compare ( (ompi_communicator_t*)comm1,
+                             (ompi_communicator_t*)comm2,
+                             result);
+    OMPI_ERRHANDLER_RETURN ( rc, comm1, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/comm_connect.c b/ompi/mpi/c/comm_connect.c
deleted file mode 100644
index dfd4eacc544..00000000000
--- a/ompi/mpi/c/comm_connect.c
+++ /dev/null
@@ -1,134 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2015 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2008      University of Houston.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015      Intel, Inc. All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "opal/util/show_help.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/runtime/mpiruntime.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-#include "ompi/dpm/dpm.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_connect = PMPI_Comm_connect
-#endif
-#define MPI_Comm_connect PMPI_Comm_connect
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_connect";
-
-
-int MPI_Comm_connect(const char *port_name, MPI_Info info, int root,
-                     MPI_Comm comm, MPI_Comm *newcomm)
-{
-    int rank, rc;
-    bool send_first=true;   /* yes, we are the active part in this game */
-    ompi_communicator_t *newcomp=MPI_COMM_NULL;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_comm_invalid (comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-        if ( OMPI_COMM_IS_INTER(comm)) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-        if ( (0 > root) || (ompi_comm_size(comm) <= root) ) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-        if ( NULL == newcomm ) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-        if (NULL == info || ompi_info_is_freed(info)) {
-          return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
-                                        FUNC_NAME);
-        }
-    }
-
-    rank = ompi_comm_rank ( comm );
-    if ( MPI_PARAM_CHECK ) {
-        if ( rank == root ) {
-            if ( NULL == port_name )
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
-                                              FUNC_NAME);
-        }
-    }
-
-    if (!ompi_mpi_dynamics_is_enabled(FUNC_NAME)) {
-        return OMPI_ERRHANDLER_INVOKE(comm, OMPI_ERR_NOT_SUPPORTED, FUNC_NAME);
-    }
-
-    /* parse info object. No prefedined values for this function in MPI-2,
-     * so lets ignore it for the moment.
-     *
-     * if ( rank == root && MPI_INFO_NULL != info ) {
-     * }
-     */
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * We must not call ompi_comm_iface_create_check() here, because that
-     * risks leaving the accept side dangling on an unmatched operation.
-     * We will let the connect_accept logic proceed and discover the
-     * issue internally so that all sides get informed.
-     */
-#endif
-
-    if ( rank == root ) {
-        rc = ompi_dpm_connect_accept (comm, root, port_name, send_first,
-				      &newcomp);
-    }
-    else {
-        rc = ompi_dpm_connect_accept (comm, root, NULL, send_first,
-				      &newcomp);
-    }
-
-    if (OPAL_ERR_NOT_SUPPORTED == rc) {
-        opal_show_help("help-mpi-api.txt",
-                       "MPI function not supported",
-                       true,
-                       FUNC_NAME,
-                       "Underlying runtime environment does not support accept/connect functionality");
-    }
-
-    *newcomm = newcomp;
-    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/comm_connect.c.in b/ompi/mpi/c/comm_connect.c.in
new file mode 100644
index 00000000000..006a7a8e042
--- /dev/null
+++ b/ompi/mpi/c/comm_connect.c.in
@@ -0,0 +1,126 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2008      University of Houston.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Intel, Inc. All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "opal/util/show_help.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/runtime/mpiruntime.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include "ompi/dpm/dpm.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS comm_connect(STRING port_name, INFO info, INT root,
+                                   COMM comm, COMM_OUT newcomm)
+{
+    int rank, rc;
+    bool send_first=true;   /* yes, we are the active part in this game */
+    ompi_communicator_t *newcomp=MPI_COMM_NULL;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_comm_invalid (comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+        if ( OMPI_COMM_IS_INTER(comm)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+        if ( (0 > root) || (ompi_comm_size(comm) <= root) ) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+        if ( NULL == newcomm ) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+        if (NULL == info || ompi_info_is_freed(info)) {
+          return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
+                                        FUNC_NAME);
+        }
+    }
+
+    rank = ompi_comm_rank ( comm );
+    if ( MPI_PARAM_CHECK ) {
+        if ( rank == root ) {
+            if ( NULL == port_name )
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
+                                              FUNC_NAME);
+        }
+    }
+
+    if (!ompi_mpi_dynamics_is_enabled(FUNC_NAME)) {
+        return OMPI_ERRHANDLER_INVOKE(comm, OMPI_ERR_NOT_SUPPORTED, FUNC_NAME);
+    }
+
+    /* parse info object. No prefedined values for this function in MPI-2,
+     * so lets ignore it for the moment.
+     *
+     * if ( rank == root && MPI_INFO_NULL != info ) {
+     * }
+     */
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * We must not call ompi_comm_iface_create_check() here, because that
+     * risks leaving the accept side dangling on an unmatched operation.
+     * We will let the connect_accept logic proceed and discover the
+     * issue internally so that all sides get informed.
+     */
+#endif
+
+    if ( rank == root ) {
+        rc = ompi_dpm_connect_accept (comm, root, port_name, send_first,
+				      &newcomp);
+    }
+    else {
+        rc = ompi_dpm_connect_accept (comm, root, NULL, send_first,
+				      &newcomp);
+    }
+
+    if (OPAL_ERR_NOT_SUPPORTED == rc) {
+        opal_show_help("help-mpi-api.txt",
+                       "MPI function not supported",
+                       true,
+                       FUNC_NAME,
+                       "Underlying runtime environment does not support accept/connect functionality");
+    }
+
+    *newcomm = newcomp;
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/comm_create.c b/ompi/mpi/c/comm_create.c
deleted file mode 100644
index fc29be5b6ee..00000000000
--- a/ompi/mpi/c/comm_create.c
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2008 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_create = PMPI_Comm_create
-#endif
-#define MPI_Comm_create PMPI_Comm_create
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_create";
-
-
-int MPI_Comm_create(MPI_Comm comm, MPI_Group group, MPI_Comm *newcomm) {
-
-    int rc;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_comm_invalid (comm))
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-
-        if ( MPI_GROUP_NULL == group || NULL == group )
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_GROUP,
-                                          FUNC_NAME);
-
-        if ( NULL == newcomm )
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
-                                          FUNC_NAME);
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_create_check(comm, &rc)) ) {
-        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-    }
-#endif
-
-    rc = ompi_comm_create ( (ompi_communicator_t*)comm, (ompi_group_t*)group,
-                           (ompi_communicator_t**)newcomm );
-    OMPI_ERRHANDLER_RETURN ( rc, comm, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/comm_create.c.in b/ompi/mpi/c/comm_create.c.in
new file mode 100644
index 00000000000..3b315ac1002
--- /dev/null
+++ b/ompi/mpi/c/comm_create.c.in
@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2008 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS comm_create(COMM comm, GROUP group, COMM_OUT newcomm)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_comm_invalid (comm))
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+
+        if ( MPI_GROUP_NULL == group || NULL == group )
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_GROUP,
+                                          FUNC_NAME);
+
+        if ( NULL == newcomm )
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_create_check(comm, &rc)) ) {
+        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+    }
+#endif
+
+    rc = ompi_comm_create ( (ompi_communicator_t*)comm, (ompi_group_t*)group,
+                           (ompi_communicator_t**)newcomm );
+    OMPI_ERRHANDLER_RETURN ( rc, comm, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/comm_create_errhandler.c b/ompi/mpi/c/comm_create_errhandler.c
deleted file mode 100644
index e342037cc92..00000000000
--- a/ompi/mpi/c/comm_create_errhandler.c
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008-2009 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_create_errhandler = PMPI_Comm_create_errhandler
-#endif
-#define MPI_Comm_create_errhandler PMPI_Comm_create_errhandler
-#endif
-
-
-static const char FUNC_NAME[] = "MPI_Comm_create_errhandler";
-
-
-int MPI_Comm_create_errhandler(MPI_Comm_errhandler_function *function,
-                               MPI_Errhandler *errhandler)
-{
-  int err = MPI_SUCCESS;
-
-  /* Error checking */
-
-  if (MPI_PARAM_CHECK) {
-    OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-    if (NULL == function ||
-        NULL == errhandler) {
-      return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                    FUNC_NAME);
-    }
-  }
-
-  /* Create and cache the errhandler.  Sets a refcount of 1. */
-
-  *errhandler =
-    ompi_errhandler_create(OMPI_ERRHANDLER_TYPE_COMM,
-                           (ompi_errhandler_generic_handler_fn_t*) function,
-                           OMPI_ERRHANDLER_LANG_C);
-  if (NULL == *errhandler) {
-    err = MPI_ERR_INTERN;
-  }
-
-  OMPI_ERRHANDLER_NOHANDLE_RETURN(err, MPI_ERR_INTERN, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/comm_create_errhandler.c.in b/ompi/mpi/c/comm_create_errhandler.c.in
new file mode 100644
index 00000000000..2d1ba927233
--- /dev/null
+++ b/ompi/mpi/c/comm_create_errhandler.c.in
@@ -0,0 +1,62 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008-2009 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018-2021 Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+
+PROTOTYPE ERROR_CLASS comm_create_errhandler(COMM_ERRHANDLER_FUNCTION function,
+                                             ERRHANDLER_OUT errhandler)
+{
+    int err = MPI_SUCCESS;
+
+    /* Error checking */
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (NULL == function ||
+            NULL == errhandler) {
+          return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                                 FUNC_NAME);
+        }
+    }
+
+    /* Create and cache the errhandler.  Sets a refcount of 1. */
+
+    *errhandler =
+        ompi_errhandler_create(OMPI_ERRHANDLER_TYPE_COMM,
+                               (ompi_errhandler_generic_handler_fn_t*) function,
+                               OMPI_ERRHANDLER_LANG_C);
+    if (NULL == *errhandler) {
+        err = MPI_ERR_INTERN;
+    }
+
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, MPI_ERR_INTERN, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/comm_create_from_group.c.in b/ompi/mpi/c/comm_create_from_group.c.in
new file mode 100644
index 00000000000..26e4049fac5
--- /dev/null
+++ b/ompi/mpi/c/comm_create_from_group.c.in
@@ -0,0 +1,106 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2008 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2021-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS comm_create_from_group (GROUP group, STRING tag, INFO info, ERRHANDLER errhandler,
+                                              COMM_OUT newcomm)
+{
+    int rc;
+    char *pmix_group_tag = NULL;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (NULL == errhandler ||
+               MPI_ERRHANDLER_NULL == errhandler ||
+                    ( OMPI_ERRHANDLER_TYPE_COMM != errhandler->eh_mpi_object_type &&
+                     OMPI_ERRHANDLER_TYPE_PREDEFINED != errhandler->eh_mpi_object_type) ) {
+               return ompi_errhandler_invoke (NULL, MPI_COMM_NULL, OMPI_ERRHANDLER_TYPE_COMM,
+                                              MPI_ERR_ARG,FUNC_NAME);
+        }
+
+        if (NULL == tag) {
+            return ompi_errhandler_invoke (errhandler, MPI_COMM_NULL, OMPI_ERRHANDLER_TYPE_COMM,
+                                           MPI_ERR_TAG, FUNC_NAME);
+        }
+
+        if (NULL == group) {
+            return ompi_errhandler_invoke (errhandler, MPI_COMM_NULL, OMPI_ERRHANDLER_TYPE_COMM,
+                                           MPI_ERR_GROUP, FUNC_NAME);
+        }
+
+        if (NULL == info || ompi_info_is_freed(info)) {
+            return ompi_errhandler_invoke (errhandler, MPI_COMM_NULL, OMPI_ERRHANDLER_TYPE_COMM,
+                                           MPI_ERR_INFO, FUNC_NAME);
+        }
+
+        if (NULL == newcomm) {
+            return ompi_errhandler_invoke (errhandler, MPI_COMM_NULL, OMPI_ERRHANDLER_TYPE_COMM,
+                                           MPI_ERR_ARG, FUNC_NAME);
+        }
+    }
+
+    if (MPI_GROUP_NULL == group || MPI_UNDEFINED == ompi_group_rank (group)) {
+        *newcomm = MPI_COMM_NULL;
+        return MPI_SUCCESS;
+    }
+
+
+    /*
+     * we use PMIx group operations to implement comm/intercomm create from group/groups.
+     * PMIx group constructors require a unique tag across the processes using the same
+     * PMIx server.  This is not equivalent to the uniqueness requirements of the tag argument
+     * to MPI_Comm_create_from_group and MPI_Intercomm_create_from_groups, hence an
+     * additional discriminator needs to be added to the user supplied tag argument.
+     */
+    opal_asprintf (&pmix_group_tag, "%s-%s.%d", tag, OPAL_NAME_PRINT(ompi_group_get_proc_name (group, 0)),
+                   ompi_group_size(group));
+    if (OPAL_UNLIKELY(NULL == pmix_group_tag)) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    rc = ompi_comm_create_from_group ((ompi_group_t *) group, pmix_group_tag, &info->super, errhandler,
+                                      (ompi_communicator_t **) newcomm);
+    free(pmix_group_tag);
+    if (MPI_SUCCESS != rc) {
+        return ompi_errhandler_invoke (errhandler, MPI_COMM_NULL, errhandler->eh_mpi_object_type,
+                                       rc, FUNC_NAME);
+    }
+
+    return rc;
+}
diff --git a/ompi/mpi/c/comm_create_group.c b/ompi/mpi/c/comm_create_group.c
deleted file mode 100644
index 507812df3d8..00000000000
--- a/ompi/mpi/c/comm_create_group.c
+++ /dev/null
@@ -1,90 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2008 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_create_group = PMPI_Comm_create_group
-#endif
-#define MPI_Comm_create_group PMPI_Comm_create_group
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_create_group";
-
-
-int MPI_Comm_create_group (MPI_Comm comm, MPI_Group group, int tag, MPI_Comm *newcomm) {
-    int rc;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_comm_invalid (comm))
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-
-        if (tag < 0 || tag > mca_pml.pml_max_tag)
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TAG,
-                                          FUNC_NAME);
-
-        if ( NULL == group )
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_GROUP,
-                                          FUNC_NAME);
-
-        if ( NULL == newcomm )
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
-                                          FUNC_NAME);
-    }
-
-    if (MPI_GROUP_NULL == group || MPI_UNDEFINED == ompi_group_rank (group)) {
-        *newcomm = MPI_COMM_NULL;
-        return MPI_SUCCESS;
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_create_check(comm, &rc)) ) {
-        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-    }
-#endif
-
-    rc = ompi_comm_create_group ((ompi_communicator_t *) comm, (ompi_group_t *) group,
-                                 tag, (ompi_communicator_t **) newcomm);
-    OMPI_ERRHANDLER_RETURN (rc, comm, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/comm_create_group.c.in b/ompi/mpi/c/comm_create_group.c.in
new file mode 100644
index 00000000000..44852ef4eba
--- /dev/null
+++ b/ompi/mpi/c/comm_create_group.c.in
@@ -0,0 +1,83 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2008 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS comm_create_group (COMM comm, GROUP group, INT tag, COMM_OUT newcomm)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_comm_invalid (comm))
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+
+        if (tag < 0 || tag > mca_pml.pml_max_tag)
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TAG,
+                                          FUNC_NAME);
+
+        if ( NULL == group )
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_GROUP,
+                                          FUNC_NAME);
+
+        if ( NULL == newcomm )
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+    }
+
+    if (MPI_GROUP_NULL == group || MPI_UNDEFINED == ompi_group_rank (group)) {
+        *newcomm = MPI_COMM_NULL;
+        return MPI_SUCCESS;
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_create_check(comm, &rc)) ) {
+        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+    }
+#endif
+
+    rc = ompi_comm_create_group ((ompi_communicator_t *) comm, (ompi_group_t *) group,
+                                 tag, (ompi_communicator_t **) newcomm);
+    OMPI_ERRHANDLER_RETURN (rc, comm, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/comm_create_keyval.c b/ompi/mpi/c/comm_create_keyval.c
deleted file mode 100644
index 5feb61a3256..00000000000
--- a/ompi/mpi/c/comm_create_keyval.c
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/attribute/attribute.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_create_keyval = PMPI_Comm_create_keyval
-#endif
-#define MPI_Comm_create_keyval PMPI_Comm_create_keyval
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_create_keyval";
-
-
-int MPI_Comm_create_keyval(MPI_Comm_copy_attr_function *comm_copy_attr_fn,
-                           MPI_Comm_delete_attr_function *comm_delete_attr_fn,
-                           int *comm_keyval, void *extra_state)
-{
-    int ret;
-    ompi_attribute_fn_ptr_union_t copy_fn;
-    ompi_attribute_fn_ptr_union_t del_fn;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if ((NULL == comm_copy_attr_fn) || (NULL == comm_delete_attr_fn) ||
-            (NULL == comm_keyval)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-    }
-
-    copy_fn.attr_communicator_copy_fn = (MPI_Comm_internal_copy_attr_function*)comm_copy_attr_fn;
-    del_fn.attr_communicator_delete_fn = comm_delete_attr_fn;
-
-    ret = ompi_attr_create_keyval(COMM_ATTR, copy_fn,
-                                  del_fn, comm_keyval, extra_state, 0, NULL);
-
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(ret, MPI_ERR_OTHER, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/comm_create_keyval.c.in b/ompi/mpi/c/comm_create_keyval.c.in
new file mode 100644
index 00000000000..d69c245fc49
--- /dev/null
+++ b/ompi/mpi/c/comm_create_keyval.c.in
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/attribute/attribute.h"
+
+PROTOTYPE ERROR_CLASS comm_create_keyval(COMM_COPY_ATTR_FUNCTION comm_copy_attr_fn,
+                                         COMM_DELETE_ATTR_FUNCTION comm_delete_attr_fn,
+                                         INT_OUT comm_keyval, BUFFER_OUT extra_state)
+{
+    int ret;
+    ompi_attribute_fn_ptr_union_t copy_fn;
+    ompi_attribute_fn_ptr_union_t del_fn;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if ((NULL == comm_copy_attr_fn) || (NULL == comm_delete_attr_fn) ||
+            (NULL == comm_keyval)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+    copy_fn.attr_communicator_copy_fn = comm_copy_attr_fn;
+    del_fn.attr_communicator_delete_fn = comm_delete_attr_fn;
+
+    ret = ompi_attr_create_keyval(COMM_ATTR, copy_fn,
+                                  del_fn, comm_keyval, extra_state, 0, NULL);
+
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(ret, MPI_ERR_OTHER, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/comm_delete_attr.c b/ompi/mpi/c/comm_delete_attr.c
deleted file mode 100644
index 19a714def7a..00000000000
--- a/ompi/mpi/c/comm_delete_attr.c
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/attribute/attribute.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_delete_attr = PMPI_Comm_delete_attr
-#endif
-#define MPI_Comm_delete_attr PMPI_Comm_delete_attr
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_delete_attr";
-
-
-int MPI_Comm_delete_attr(MPI_Comm comm, int comm_keyval)
-{
-    int ret;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-    }
-
-    ret = ompi_attr_delete(COMM_ATTR, comm, comm->c_keyhash, comm_keyval,
-                           false);
-
-    OMPI_ERRHANDLER_RETURN(ret, comm, MPI_ERR_OTHER, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/comm_delete_attr.c.in b/ompi/mpi/c/comm_delete_attr.c.in
new file mode 100644
index 00000000000..7d792cd993c
--- /dev/null
+++ b/ompi/mpi/c/comm_delete_attr.c.in
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/attribute/attribute.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS comm_delete_attr(COMM comm, INT comm_keyval)
+{
+    int ret;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+    }
+
+    ret = ompi_attr_delete(COMM_ATTR, comm, comm->c_keyhash, comm_keyval,
+                           false);
+
+    OMPI_ERRHANDLER_RETURN(ret, comm, MPI_ERR_OTHER, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/comm_disconnect.c b/ompi/mpi/c/comm_disconnect.c
deleted file mode 100644
index 7d9b1519ba4..00000000000
--- a/ompi/mpi/c/comm_disconnect.c
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Intel, Inc. All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_disconnect = PMPI_Comm_disconnect
-#endif
-#define MPI_Comm_disconnect PMPI_Comm_disconnect
-#endif
-
-#include "ompi/dpm/dpm.h"
-
-
-static const char FUNC_NAME[] = "MPI_Comm_disconnect";
-
-
-int MPI_Comm_disconnect(MPI_Comm *comm)
-{
-    int ret = MPI_SUCCESS;
-
-    MEMCHECKER(
-        memchecker_comm(*comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if ( ompi_comm_invalid (*comm))
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-    }
-
-    if (MPI_COMM_WORLD == *comm || MPI_COMM_SELF == *comm ) {
-        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
-    }
-
-    if ( OMPI_COMM_IS_DYNAMIC(*comm)) {
-        if (OMPI_SUCCESS != ompi_dpm_disconnect (*comm)) {
-            ret = OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
-        }
-    }
-    else {
-        (*comm)->c_coll->coll_barrier(*comm, (*comm)->c_coll->coll_barrier_module);
-    }
-
-    ompi_comm_free(comm);
-
-    return ret;
-}
diff --git a/ompi/mpi/c/comm_disconnect.c.in b/ompi/mpi/c/comm_disconnect.c.in
new file mode 100644
index 00000000000..17e2ef4e43e
--- /dev/null
+++ b/ompi/mpi/c/comm_disconnect.c.in
@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Intel, Inc. All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+#include "ompi/dpm/dpm.h"
+
+PROTOTYPE ERROR_CLASS comm_disconnect(COMM_OUT comm)
+{
+    int ret = MPI_SUCCESS;
+
+    MEMCHECKER(
+        memchecker_comm(*comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if ( ompi_comm_invalid (*comm))
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+    }
+
+    if (MPI_COMM_WORLD == *comm || MPI_COMM_SELF == *comm ) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+    }
+
+    if ( OMPI_COMM_IS_DYNAMIC(*comm)) {
+        if (OMPI_SUCCESS != ompi_dpm_disconnect (*comm)) {
+            ret = OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+        }
+    }
+    else {
+        (*comm)->c_coll->coll_barrier(*comm, (*comm)->c_coll->coll_barrier_module);
+    }
+
+    ompi_comm_free(comm);
+
+    return ret;
+}
diff --git a/ompi/mpi/c/comm_dup.c b/ompi/mpi/c/comm_dup.c
deleted file mode 100644
index 5f747e7c7a0..00000000000
--- a/ompi/mpi/c/comm_dup.c
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2006-2008 University of Houston.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_dup = PMPI_Comm_dup
-#endif
-#define MPI_Comm_dup PMPI_Comm_dup
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_dup";
-
-int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm)
-{
-    int rc=MPI_SUCCESS;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    /* argument checking */
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_comm_invalid (comm))
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-
-        if ( NULL == newcomm )
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
-                                          FUNC_NAME);
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_create_check(comm, &rc)) ) {
-        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-    }
-#endif
-
-    rc = ompi_comm_dup ( comm, newcomm );
-    OMPI_ERRHANDLER_RETURN ( rc, comm, rc, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/comm_dup.c.in b/ompi/mpi/c/comm_dup.c.in
new file mode 100644
index 00000000000..c595673298b
--- /dev/null
+++ b/ompi/mpi/c/comm_dup.c.in
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2006-2008 University of Houston.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS comm_dup(COMM comm, COMM_OUT newcomm)
+{
+    int rc=MPI_SUCCESS;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    /* argument checking */
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_comm_invalid (comm))
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+
+        if ( NULL == newcomm )
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_create_check(comm, &rc)) ) {
+        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+    }
+#endif
+
+    rc = ompi_comm_dup ( comm, newcomm );
+    OMPI_ERRHANDLER_RETURN ( rc, comm, rc, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/comm_dup_with_info.c b/ompi/mpi/c/comm_dup_with_info.c
deleted file mode 100644
index 9b2df72134a..00000000000
--- a/ompi/mpi/c/comm_dup_with_info.c
+++ /dev/null
@@ -1,83 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2006-2008 University of Houston.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_dup_with_info = PMPI_Comm_dup_with_info
-#endif
-#define MPI_Comm_dup_with_info PMPI_Comm_dup_with_info
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_dup_with_info";
-
-int MPI_Comm_dup_with_info(MPI_Comm comm, MPI_Info info, MPI_Comm *newcomm)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    /* argument checking */
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_comm_invalid (comm))
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        if (NULL == info || ompi_info_is_freed(info)) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_INFO,
-                                          FUNC_NAME);
-        }
-
-        if ( NULL == newcomm )
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
-                                          FUNC_NAME);
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_create_check(comm, &rc)) ) {
-        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-    }
-#endif
-
-    rc = ompi_comm_dup_with_info (comm, &info->super, newcomm);
-    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/comm_dup_with_info.c.in b/ompi/mpi/c/comm_dup_with_info.c.in
new file mode 100644
index 00000000000..4e6e2ad7b94
--- /dev/null
+++ b/ompi/mpi/c/comm_dup_with_info.c.in
@@ -0,0 +1,76 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2006-2008 University of Houston.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS comm_dup_with_info(COMM comm, INFO info, COMM_OUT newcomm)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    /* argument checking */
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_comm_invalid (comm))
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        if (NULL == info || ompi_info_is_freed(info)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_INFO,
+                                          FUNC_NAME);
+        }
+
+        if ( NULL == newcomm )
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_create_check(comm, &rc)) ) {
+        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+    }
+#endif
+
+    rc = ompi_comm_dup_with_info (comm, &info->super, newcomm);
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/comm_f2c.c b/ompi/mpi/c/comm_f2c.c
deleted file mode 100644
index a0bd6be2acb..00000000000
--- a/ompi/mpi/c/comm_f2c.c
+++ /dev/null
@@ -1,59 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2007 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mpi/fortran/base/fint_2_int.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_f2c = PMPI_Comm_f2c
-#endif
-#define MPI_Comm_f2c PMPI_Comm_f2c
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_f2c";
-
-
-MPI_Comm MPI_Comm_f2c(MPI_Fint comm)
-{
-    int o_index= OMPI_FINT_2_INT(comm);
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-    }
-
-    /* Per MPI-2:4.12.4, do not invoke an error handler if we get an
-       invalid fortran handle.  If we get an invalid fortran handle,
-       return an invalid C handle. */
-
-    if ( 0 > o_index ||
-         o_index >= opal_pointer_array_get_size(&ompi_comm_f_to_c_table)) {
-        return NULL;
-    }
-
-    return (MPI_Comm)opal_pointer_array_get_item(&ompi_comm_f_to_c_table, o_index);
-}
diff --git a/ompi/mpi/c/comm_f2c.c.in b/ompi/mpi/c/comm_f2c.c.in
new file mode 100644
index 00000000000..769b7906a52
--- /dev/null
+++ b/ompi/mpi/c/comm_f2c.c.in
@@ -0,0 +1,51 @@
+/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+
+PROTOTYPE COMM comm_f2c(FINT comm)
+{
+    int o_index= OMPI_FINT_2_INT(comm);
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+    }
+
+    /* Per MPI-2:4.12.4, do not invoke an error handler if we get an
+       invalid fortran handle.  If we get an invalid fortran handle,
+       return an invalid C handle. */
+
+    if ( 0 > o_index ||
+         o_index >= opal_pointer_array_get_size(&ompi_comm_f_to_c_table)) {
+        return NULL;
+    }
+
+    return (MPI_Comm)opal_pointer_array_get_item(&ompi_comm_f_to_c_table, o_index);
+}
diff --git a/ompi/mpi/c/comm_free.c b/ompi/mpi/c/comm_free.c
deleted file mode 100644
index f190011aabd..00000000000
--- a/ompi/mpi/c/comm_free.c
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_free = PMPI_Comm_free
-#endif
-#define MPI_Comm_free PMPI_Comm_free
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_free";
-
-
-int MPI_Comm_free(MPI_Comm *comm)
-{
-    int ret;
-
-    MEMCHECKER(
-        memchecker_comm(*comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if ( NULL == *comm ||
-             ompi_comm_invalid (*comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if (MPI_COMM_WORLD == *comm ||
-                   MPI_COMM_SELF == *comm) {
-            return OMPI_ERRHANDLER_INVOKE(*comm, MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-    }
-
-    ret = ompi_comm_free ( comm );
-    OMPI_ERRHANDLER_CHECK(ret, *comm, ret, FUNC_NAME);
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/comm_free.c.in b/ompi/mpi/c/comm_free.c.in
new file mode 100644
index 00000000000..9540ab668fd
--- /dev/null
+++ b/ompi/mpi/c/comm_free.c.in
@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS comm_free(COMM_OUT comm)
+{
+    int ret;
+
+    MEMCHECKER(
+        memchecker_comm(*comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if ( NULL == *comm ||
+             ompi_comm_invalid (*comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if (MPI_COMM_WORLD == *comm ||
+                   MPI_COMM_SELF == *comm) {
+            return OMPI_ERRHANDLER_INVOKE(*comm, MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+    }
+
+    ret = ompi_comm_free ( comm );
+    OMPI_ERRHANDLER_CHECK(ret, *comm, ret, FUNC_NAME);
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/comm_free_keyval.c b/ompi/mpi/c/comm_free_keyval.c
deleted file mode 100644
index 7db43297090..00000000000
--- a/ompi/mpi/c/comm_free_keyval.c
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/attribute/attribute.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_free_keyval = PMPI_Comm_free_keyval
-#endif
-#define MPI_Comm_free_keyval PMPI_Comm_free_keyval
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_free_keyval";
-
-
-int MPI_Comm_free_keyval(int *comm_keyval)
-{
-    int ret;
-
-    /* Check for valid key pointer */
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == comm_keyval) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-    }
-
-    ret = ompi_attr_free_keyval(COMM_ATTR, comm_keyval, 0);
-
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(ret, MPI_ERR_OTHER, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/comm_free_keyval.c.in b/ompi/mpi/c/comm_free_keyval.c.in
new file mode 100644
index 00000000000..85ddf34f124
--- /dev/null
+++ b/ompi/mpi/c/comm_free_keyval.c.in
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/attribute/attribute.h"
+
+PROTOTYPE ERROR_CLASS comm_free_keyval(INT_OUT comm_keyval)
+{
+    int ret;
+
+    /* Check for valid key pointer */
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == comm_keyval) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+    ret = ompi_attr_free_keyval(COMM_ATTR, comm_keyval, 0);
+
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(ret, MPI_ERR_OTHER, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/comm_get_attr.c b/ompi/mpi/c/comm_get_attr.c
deleted file mode 100644
index 93091f5511e..00000000000
--- a/ompi/mpi/c/comm_get_attr.c
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/attribute/attribute.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_get_attr = PMPI_Comm_get_attr
-#endif
-#define MPI_Comm_get_attr PMPI_Comm_get_attr
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_get_attr";
-
-
-int MPI_Comm_get_attr(MPI_Comm comm, int comm_keyval,
-                      void *attribute_val, int *flag)
-{
-    int ret;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if ((NULL == attribute_val) || (NULL == flag)) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        } else if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if (MPI_KEYVAL_INVALID == comm_keyval) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_KEYVAL, FUNC_NAME);
-        }
-    }
-
-    /* This stuff is very confusing.  Be sure to see
-       src/attribute/attribute.c for a lengthy comment explaining Open
-       MPI attribute behavior. */
-
-    ret = ompi_attr_get_c(comm->c_keyhash, comm_keyval,
-                          (void**)attribute_val, flag);
-    OMPI_ERRHANDLER_RETURN(ret, comm, MPI_ERR_OTHER, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/comm_get_attr.c.in b/ompi/mpi/c/comm_get_attr.c.in
new file mode 100644
index 00000000000..1accf0c2c5e
--- /dev/null
+++ b/ompi/mpi/c/comm_get_attr.c.in
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/attribute/attribute.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS comm_get_attr(COMM comm, INT comm_keyval,
+                                    BUFFER_OUT attribute_val, INT_OUT flag)
+{
+    int ret;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if ((NULL == attribute_val) || (NULL == flag)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        } else if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if (MPI_KEYVAL_INVALID == comm_keyval) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_KEYVAL, FUNC_NAME);
+        }
+    }
+
+    /* This stuff is very confusing.  Be sure to see
+       src/attribute/attribute.c for a lengthy comment explaining Open
+       MPI attribute behavior. */
+
+    ret = ompi_attr_get_c(comm->c_keyhash, comm_keyval,
+                          (void**)attribute_val, flag);
+    OMPI_ERRHANDLER_RETURN(ret, comm, MPI_ERR_OTHER, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/comm_get_errhandler.c b/ompi/mpi/c/comm_get_errhandler.c
deleted file mode 100644
index 301cdd91862..00000000000
--- a/ompi/mpi/c/comm_get_errhandler.c
+++ /dev/null
@@ -1,74 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007-2009 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015-2017 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_get_errhandler = PMPI_Comm_get_errhandler
-#endif
-#define MPI_Comm_get_errhandler PMPI_Comm_get_errhandler
-#endif
-
-
-static const char FUNC_NAME[] = "MPI_Comm_get_errhandler";
-
-
-int MPI_Comm_get_errhandler(MPI_Comm comm, MPI_Errhandler *errhandler)
-{
-    /* Error checking */
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    /* Error checking */
-
-    if (MPI_PARAM_CHECK) {
-      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-      if (ompi_comm_invalid(comm)) {
-        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                      FUNC_NAME);
-      } else if (NULL == errhandler) {
-        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                      FUNC_NAME);
-      }
-    }
-
-    OPAL_THREAD_LOCK(&(comm->c_lock));
-    /* Retain the errhandler, corresponding to object refcount decrease
-       in errhandler_free.c. */
-    OBJ_RETAIN(comm->error_handler);
-    *errhandler = comm->error_handler;
-    OPAL_THREAD_UNLOCK(&(comm->c_lock));
-
-    /* All done */
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/comm_get_errhandler.c.in b/ompi/mpi/c/comm_get_errhandler.c.in
new file mode 100644
index 00000000000..170a36fa841
--- /dev/null
+++ b/ompi/mpi/c/comm_get_errhandler.c.in
@@ -0,0 +1,71 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007-2009 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2017 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2020-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+#include "ompi/instance/instance.h"
+
+PROTOTYPE ERROR_CLASS comm_get_errhandler(COMM comm, ERRHANDLER_OUT errhandler)
+{
+    int ret = MPI_SUCCESS;
+
+    /* Error checking */
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    /* Error checking */
+
+    if (MPI_PARAM_CHECK) {
+      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+      if (ompi_comm_invalid(comm)) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                      FUNC_NAME);
+      } else if (NULL == errhandler) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                      FUNC_NAME);
+      }
+    }
+
+    OPAL_THREAD_LOCK(&(comm->c_lock));
+    /* Retain the errhandler, corresponding to object refcount decrease
+       in errhandler_free.c. */
+    OBJ_RETAIN(comm->error_handler);
+    *errhandler = comm->error_handler;
+    OPAL_THREAD_UNLOCK(&(comm->c_lock));
+
+   /* make sure the infrastructure is initialized */
+    ret = ompi_mpi_instance_retain ();
+
+    /* All done */
+
+    return ret;
+}
diff --git a/ompi/mpi/c/comm_get_info.c b/ompi/mpi/c/comm_get_info.c
deleted file mode 100644
index cb9ff2146cf..00000000000
--- a/ompi/mpi/c/comm_get_info.c
+++ /dev/null
@@ -1,66 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-#include <stdlib.h>
-#include <string.h>
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_get_info = PMPI_Comm_get_info
-#endif
-#define MPI_Comm_get_info PMPI_Comm_get_info
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_get_info";
-
-
-int MPI_Comm_get_info(MPI_Comm comm, MPI_Info *info_used)
-{
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == info_used) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
-                                          FUNC_NAME);
-        }
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-    }
-
-    if (NULL == comm->super.s_info) {
-/*
- * Setup any defaults if MPI_Win_set_info was never called
- */
-        opal_infosubscribe_change_info(&comm->super, &MPI_INFO_NULL->super);
-    }
-
-
-    (*info_used) = OBJ_NEW(ompi_info_t);
-    if (NULL == (*info_used)) {
-       return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_NO_MEM,
-                                      FUNC_NAME);
-    }
-    opal_info_t *opal_info_used = &(*info_used)->super;
-
-    opal_info_dup_mpistandard(comm->super.s_info, &opal_info_used);
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/comm_get_info.c.in b/ompi/mpi/c/comm_get_info.c.in
new file mode 100644
index 00000000000..132a27fb413
--- /dev/null
+++ b/ompi/mpi/c/comm_get_info.c.in
@@ -0,0 +1,59 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include <stdlib.h>
+#include <string.h>
+
+PROTOTYPE ERROR_CLASS comm_get_info(COMM comm, INFO_OUT info_used)
+{
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == info_used) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
+                                          FUNC_NAME);
+        }
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+    }
+
+    if (NULL == comm->super.s_info) {
+        /*
+        * Setup any defaults if MPI_Win_set_info was never called
+        */
+        opal_infosubscribe_change_info(&comm->super, &MPI_INFO_NULL->super);
+    }
+
+
+    *info_used = ompi_info_allocate ();
+    if (NULL == (*info_used)) {
+       return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_NO_MEM,
+                                      FUNC_NAME);
+    }
+
+    opal_info_t *opal_info_used = &(*info_used)->super;
+
+    opal_info_dup_public(comm->super.s_info, &opal_info_used);
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/comm_get_name.c b/ompi/mpi/c/comm_get_name.c
deleted file mode 100644
index e635b768505..00000000000
--- a/ompi/mpi/c/comm_get_name.c
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2018 Cisco Systems, Inc.  All rights reserved
- * Copyright (c) 2015-2017 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <string.h>
-
-#include "opal/mca/threads/mutex.h"
-#include "opal/util/string_copy.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/totalview.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_get_name = PMPI_Comm_get_name
-#endif
-#define MPI_Comm_get_name PMPI_Comm_get_name
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_get_name";
-
-
-int MPI_Comm_get_name(MPI_Comm comm, char *name, int *length)
-{
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if ( ompi_comm_invalid ( comm ) )
-            return OMPI_ERRHANDLER_INVOKE ( MPI_COMM_WORLD, MPI_ERR_COMM,
-                                            FUNC_NAME);
-
-        if ( NULL == name || NULL == length )
-            return OMPI_ERRHANDLER_INVOKE ( comm, MPI_ERR_ARG,
-                                            FUNC_NAME);
-    }
-    OPAL_THREAD_LOCK(&(comm->c_lock));
-    /* Note that MPI-2.1 requires:
-       - terminating the string with a \0
-       - name[*resultlen] == '\0'
-       - and therefore (*resultlen) cannot be > (MPI_MAX_OBJECT_NAME-1)
-
-       The Fortran API version will pad to the right if necessary.
-
-       Note that comm->c_name is guaranteed to be \0-terminated and
-       able to completely fit into MPI_MAX_OBJECT_NAME bytes (i.e.,
-       name+\0). */
-    if ( comm->c_flags & OMPI_COMM_NAMEISSET ) {
-        opal_string_copy(name, comm->c_name, MPI_MAX_OBJECT_NAME);
-        *length = (int) strlen(comm->c_name);
-    } else {
-        name[0] = '\0';
-        *length = 0;
-    }
-    OPAL_THREAD_UNLOCK(&(comm->c_lock));
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/comm_get_name.c.in b/ompi/mpi/c/comm_get_name.c.in
new file mode 100644
index 00000000000..f0714830f27
--- /dev/null
+++ b/ompi/mpi/c/comm_get_name.c.in
@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2018 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2015-2017 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include <string.h>
+
+#include "opal/mca/threads/mutex.h"
+#include "opal/util/string_copy.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/totalview.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS comm_get_name(COMM comm, STRING_OUT name, INT_OUT length)
+{
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if ( ompi_comm_invalid ( comm ) )
+            return OMPI_ERRHANDLER_INVOKE ( MPI_COMM_WORLD, MPI_ERR_COMM,
+                                            FUNC_NAME);
+
+        if ( NULL == name || NULL == length )
+            return OMPI_ERRHANDLER_INVOKE ( comm, MPI_ERR_ARG,
+                                            FUNC_NAME);
+    }
+    OPAL_THREAD_LOCK(&(comm->c_lock));
+    /* Note that MPI-2.1 requires:
+       - terminating the string with a \0
+       - name[*resultlen] == '\0'
+       - and therefore (*resultlen) cannot be > (MPI_MAX_OBJECT_NAME-1)
+
+       The Fortran API version will pad to the right if necessary.
+
+       Note that comm->c_name is guaranteed to be \0-terminated and
+       able to completely fit into MPI_MAX_OBJECT_NAME bytes (i.e.,
+       name+\0). */
+    if ( comm->c_flags & OMPI_COMM_NAMEISSET ) {
+        opal_string_copy(name, comm->c_name, MPI_MAX_OBJECT_NAME);
+        *length = (int) strlen(comm->c_name);
+    } else {
+        name[0] = '\0';
+        *length = 0;
+    }
+    OPAL_THREAD_UNLOCK(&(comm->c_lock));
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/comm_get_parent.c b/ompi/mpi/c/comm_get_parent.c
deleted file mode 100644
index a6909ef2459..00000000000
--- a/ompi/mpi/c/comm_get_parent.c
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_get_parent = PMPI_Comm_get_parent
-#endif
-#define MPI_Comm_get_parent PMPI_Comm_get_parent
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_get_parent";
-
-
-int MPI_Comm_get_parent(MPI_Comm *parent)
-{
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if ( NULL == parent ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-    }
-
-    /*
-     * ompi_mpi_comm_parent is MPI_COMM_NULL, in case this
-     * world has not been spawned by another MPI job.
-     * This is also the return value required by MPI-2.
-     */
-
-     *parent = ompi_mpi_comm_parent;
-     return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/comm_get_parent.c.in b/ompi/mpi/c/comm_get_parent.c.in
new file mode 100644
index 00000000000..49e4f38177b
--- /dev/null
+++ b/ompi/mpi/c/comm_get_parent.c.in
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+
+PROTOTYPE ERROR_CLASS comm_get_parent(COMM_OUT parent)
+{
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if ( NULL == parent ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+    /*
+     * ompi_mpi_comm_parent is MPI_COMM_NULL, in case this
+     * world has not been spawned by another MPI job.
+     * This is also the return value required by MPI-2.
+     */
+
+     *parent = ompi_mpi_comm_parent;
+     return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/comm_group.c b/ompi/mpi/c/comm_group.c
deleted file mode 100644
index 7a563578dcc..00000000000
--- a/ompi/mpi/c/comm_group.c
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_group = PMPI_Comm_group
-#endif
-#define MPI_Comm_group PMPI_Comm_group
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_group";
-
-
-int MPI_Comm_group(MPI_Comm comm, MPI_Group *group) {
-
-    int rc;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    /* argument checking */
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if ( ompi_comm_invalid (comm) )
-           return  OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-
-        if ( NULL == group )
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
-                                          FUNC_NAME);
-    } /* end if ( MPI_PARAM_CHECK) */
-
-   rc = ompi_comm_group ( (ompi_communicator_t*)comm, (ompi_group_t**)group );
-   OMPI_ERRHANDLER_RETURN ( rc, comm, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/comm_group.c.in b/ompi/mpi/c/comm_group.c.in
new file mode 100644
index 00000000000..f354e3e759d
--- /dev/null
+++ b/ompi/mpi/c/comm_group.c.in
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS comm_group(COMM comm, GROUP_OUT group) 
+{
+
+    int rc;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    /* argument checking */
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if ( ompi_comm_invalid (comm) )
+           return  OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+
+        if ( NULL == group )
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+    } /* end if ( MPI_PARAM_CHECK) */
+
+   rc = ompi_comm_group ( (ompi_communicator_t*)comm, (ompi_group_t**)group );
+   OMPI_ERRHANDLER_RETURN ( rc, comm, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/comm_idup.c b/ompi/mpi/c/comm_idup.c
deleted file mode 100644
index 278b9b1706f..00000000000
--- a/ompi/mpi/c/comm_idup.c
+++ /dev/null
@@ -1,67 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2006-2008 University of Houston.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_idup = PMPI_Comm_idup
-#endif
-#define MPI_Comm_idup PMPI_Comm_idup
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_idup";
-
-int MPI_Comm_idup(MPI_Comm comm, MPI_Comm *newcomm, MPI_Request *request)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    /* argument checking */
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_comm_invalid (comm))
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-
-        if ( NULL == newcomm )
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
-                                          FUNC_NAME);
-    }
-
-    rc = ompi_comm_idup (comm, newcomm, request);
-    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/comm_idup.c.in b/ompi/mpi/c/comm_idup.c.in
new file mode 100644
index 00000000000..e51fd1b5493
--- /dev/null
+++ b/ompi/mpi/c/comm_idup.c.in
@@ -0,0 +1,60 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2006-2008 University of Houston.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS comm_idup(COMM comm, COMM_OUT newcomm, REQUEST_INOUT request)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    /* argument checking */
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_comm_invalid (comm))
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+
+        if ( NULL == newcomm )
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+    }
+
+    rc = ompi_comm_idup (comm, newcomm, request);
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/comm_idup_with_info.c.in b/ompi/mpi/c/comm_idup_with_info.c.in
new file mode 100644
index 00000000000..983a2c35d0b
--- /dev/null
+++ b/ompi/mpi/c/comm_idup_with_info.c.in
@@ -0,0 +1,78 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2006-2008 University of Houston.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
+ * Copyright (c) 2021-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS comm_idup_with_info(COMM comm, INFO info,
+                                          COMM_OUT newcomm, REQUEST_INOUT request)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    /* argument checking */
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_comm_invalid (comm))
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        if (NULL == info || ompi_info_is_freed(info)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_INFO,
+                                          FUNC_NAME);
+        }
+
+        if ( NULL == newcomm )
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_create_check(comm, &rc)) ) {
+        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+    }
+#endif
+
+    rc = ompi_comm_idup_with_info (comm, &info->super, newcomm, request);
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/comm_join.c b/ompi/mpi/c/comm_join.c
deleted file mode 100644
index 6e8b728674a..00000000000
--- a/ompi/mpi/c/comm_join.c
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2012      Los Alamos National Security, LLC.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2015-2018 Cisco Systems, Inc.  All rights reserved
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-#include <string.h>
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-#ifdef HAVE_SYS_SOCKET_H
-#include <sys/socket.h>
-#endif
-#include <errno.h>
-#ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
-#endif
-
-#include "opal/util/show_help.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/runtime/mpiruntime.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/dpm/dpm.h"
-
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_join = PMPI_Comm_join
-#endif
-#define MPI_Comm_join PMPI_Comm_join
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_join";
-
-static int ompi_socket_send (int fd, char *buf, int len );
-static int ompi_socket_recv (int fd, char *buf, int len );
-
-int MPI_Comm_join(int fd, MPI_Comm *intercomm)
-{
-    int rc;
-    uint32_t len, rlen, llen, lrlen;
-    int send_first=0;
-    ompi_process_name_t rname, tmp_name;
-
-    ompi_communicator_t *newcomp;
-    char port_name[MPI_MAX_PORT_NAME];
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if ( NULL == intercomm ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-    }
-
-    if (!ompi_mpi_dynamics_is_enabled(FUNC_NAME)) {
-        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(OMPI_ERR_NOT_SUPPORTED,
-                                      FUNC_NAME);
-    }
-
-    /* send my process name */
-    tmp_name = *OMPI_PROC_MY_NAME;
-    OMPI_PROCESS_NAME_HTON(tmp_name);
-    ompi_socket_send(fd, (char*) &tmp_name, sizeof(tmp_name));
-
-    /* recv the remote name */
-    ompi_socket_recv(fd, (char*) &rname, sizeof(rname));
-    OMPI_PROCESS_NAME_NTOH(rname);
-
-    /* compare the two to get send_first */
-    if (OMPI_PROC_MY_NAME->jobid == rname.jobid) {
-        if (OMPI_PROC_MY_NAME->vpid < rname.vpid) {
-            send_first = true;
-        } else if (OMPI_PROC_MY_NAME->vpid == rname.vpid) {
-            /* joining to myself is not allowed */
-            *intercomm = MPI_COMM_NULL;
-            return MPI_ERR_INTERN;
-        } else {
-            send_first = false;
-        }
-    } else if (OMPI_PROC_MY_NAME->jobid < rname.jobid) {
-        send_first = true;
-    }
-
-    /* Assumption: socket_send should not block, even if the socket
-       is not configured to be non-blocking, because the message length are
-       so short. */
-
-    /* we will only use the send_first proc's port name,
-     * so pass it to the recv_first participant */
-    if (send_first) {
-        // The port_name that we get back will be \0-terminated.  The
-        // strlen+\0 will be <= MPI_MAX_PORT_NAME characters.
-        if (OMPI_SUCCESS != (rc = ompi_dpm_open_port(port_name))) {
-            goto error;
-        }
-        // Send the strlen+1 so that we both send the \0 and the
-        // receiver receives the \0.
-        llen   = (uint32_t)(strlen(port_name)+1);
-        len    = htonl(llen);
-        ompi_socket_send( fd, (char *) &len, sizeof(uint32_t));
-        ompi_socket_send (fd, port_name, llen);
-    } else {
-        ompi_socket_recv (fd, (char *) &rlen, sizeof(uint32_t));
-        // The lrlen that we receive will be the strlen+1 (to account
-        // for \0), and will be <= MPI_MAX_PORT_NAME.
-        lrlen  = ntohl(rlen);
-        ompi_socket_recv (fd, port_name, lrlen);
-    }
-
-    /* use the port to connect/accept */
-    rc = ompi_dpm_connect_accept (MPI_COMM_SELF, 0, port_name, send_first, &newcomp);
-
-    *intercomm = newcomp;
-
- error:
-    if (OPAL_ERR_NOT_SUPPORTED == rc) {
-        opal_show_help("help-mpi-api.txt",
-                       "MPI function not supported",
-                       true,
-                       FUNC_NAME,
-                       "Underlying runtime environment does not support join functionality");
-    }
-
-    OMPI_ERRHANDLER_RETURN (rc, MPI_COMM_SELF, rc, FUNC_NAME);
-}
-
-
-static int ompi_socket_send (int fd, char *buf, int len )
-{
-    int num;
-    size_t s_num;
-    ssize_t a;
-    char *c_ptr;
-    int ret = OMPI_SUCCESS;
-
-    num   = len;
-    c_ptr = buf;
-
-    do {
-        s_num = (size_t) num;
-        a = write ( fd, c_ptr, s_num );
-        if ( a == -1 ) {
-            if ( errno == EINTR ) {
-                /* Catch EINTR on, mainly on IBM RS6000 */
-                continue;
-            }
-#ifdef __SR8000
-            else if ( errno == EWOULDBLOCK ) {
-                /*Catch EWOULDBLOCK on Hitachi SR8000 */
-                continue;
-            }
-            else if ( errno == EAGAIN ) {
-                /* Catch EAGAIN on Hitachi SR8000 */
-                continue;
-            }
-#endif
-            else {
-                /* Another error occured */
-                fprintf (stderr,"ompi_socket_send: error while writing to socket"
-                         " error:%s", strerror (errno) );
-                return MPI_ERR_OTHER;
-            }
-        }
-        num      -= a;
-        c_ptr    += a;
-    }   while ( num > 0 );
-
-
-    if ( num < 0 )  {
-        fprintf (stderr, "ompi_socket_send: more data written then available");
-        ret = MPI_ERR_INTERN;
-    }
-
-    return ret;
-}
-
-static int ompi_socket_recv (int fd, char *buf, int len )
-{
-    int num;
-    size_t s_num;
-    ssize_t a;
-    char *c_ptr;
-    int ret = MPI_SUCCESS;
-
-    num      = len;
-    c_ptr      = buf;
-
-    do {
-        s_num = (size_t ) num;
-        a = read ( fd, c_ptr, s_num );
-        if ( a == -1 ) {
-            if ( errno == EINTR ) {
-                /* Catch EINTR on, mainly on IBM RS6000 */
-                continue;
-            }
-#ifdef __SR8000
-            else if ( errno == EWOULDBLOCK ) {
-                /*Catch EWOULDBLOCK on Hitachi SR8000 */
-                continue;
-            }
-            else if ( errno == EAGAIN ) {
-                /* Catch EAGAIN on Hitachi SR8000 */
-                continue;
-            }
-#endif
-            else {
-                /* Another error occured */
-                fprintf (stderr,"ompi_socket_recv: error while reading from socket"
-                         " error:%s", strerror (errno) );
-                return MPI_ERR_OTHER;
-            }
-        }
-        num    -= a;
-        c_ptr  += a;
-    }   while ( num > 0 );
-
-    if ( num < 0 )  {
-        fprintf (stderr, "ompi_socket_recv: more data read then available");
-        ret = MPI_ERR_INTERN;
-    }
-
-    return ret;
-}
diff --git a/ompi/mpi/c/comm_join.c.in b/ompi/mpi/c/comm_join.c.in
new file mode 100644
index 00000000000..28d6c545c60
--- /dev/null
+++ b/ompi/mpi/c/comm_join.c.in
@@ -0,0 +1,243 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2012      Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015-2018 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+#include <string.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+#include <errno.h>
+#ifdef HAVE_NETINET_IN_H
+#include <netinet/in.h>
+#endif
+
+#include "opal/util/show_help.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/runtime/mpiruntime.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/dpm/dpm.h"
+
+
+
+static int ompi_socket_send (int fd, char *buf, int len );
+static int ompi_socket_recv (int fd, char *buf, int len );
+
+static int ompi_socket_send (int fd, char *buf, int len )
+{
+    int num;
+    size_t s_num;
+    ssize_t a;
+    char *c_ptr;
+    int ret = OMPI_SUCCESS;
+
+    num   = len;
+    c_ptr = buf;
+
+    do {
+        s_num = (size_t) num;
+        a = write ( fd, c_ptr, s_num );
+        if ( a == -1 ) {
+            if ( errno == EINTR ) {
+                /* Catch EINTR on, mainly on IBM RS6000 */
+                continue;
+            }
+#ifdef __SR8000
+            else if ( errno == EWOULDBLOCK ) {
+                /*Catch EWOULDBLOCK on Hitachi SR8000 */
+                continue;
+            }
+            else if ( errno == EAGAIN ) {
+                /* Catch EAGAIN on Hitachi SR8000 */
+                continue;
+            }
+#endif
+            else {
+                /* Another error occurred */
+                fprintf (stderr,"ompi_socket_send: error while writing to socket"
+                         " error:%s", strerror (errno) );
+                return MPI_ERR_OTHER;
+            }
+        }
+        num      -= a;
+        c_ptr    += a;
+    }   while ( num > 0 );
+
+
+    if ( num < 0 )  {
+        fprintf (stderr, "ompi_socket_send: more data written then available");
+        ret = MPI_ERR_INTERN;
+    }
+
+    return ret;
+}
+
+static int ompi_socket_recv (int fd, char *buf, int len )
+{
+    int num;
+    size_t s_num;
+    ssize_t a;
+    char *c_ptr;
+    int ret = MPI_SUCCESS;
+
+    num      = len;
+    c_ptr      = buf;
+
+    do {
+        s_num = (size_t ) num;
+        a = read ( fd, c_ptr, s_num );
+        if ( a == -1 ) {
+            if ( errno == EINTR ) {
+                /* Catch EINTR on, mainly on IBM RS6000 */
+                continue;
+            }
+#ifdef __SR8000
+            else if ( errno == EWOULDBLOCK ) {
+                /*Catch EWOULDBLOCK on Hitachi SR8000 */
+                continue;
+            }
+            else if ( errno == EAGAIN ) {
+                /* Catch EAGAIN on Hitachi SR8000 */
+                continue;
+            }
+#endif
+            else {
+                /* Another error occurred */
+                fprintf (stderr,"ompi_socket_recv: error while reading from socket"
+                         " error:%s", strerror (errno) );
+                return MPI_ERR_OTHER;
+            }
+        }
+        num    -= a;
+        c_ptr  += a;
+    }   while ( num > 0 );
+
+    if ( num < 0 )  {
+        fprintf (stderr, "ompi_socket_recv: more data read then available");
+        ret = MPI_ERR_INTERN;
+    }
+
+    return ret;
+}
+
+PROTOTYPE ERROR_CLASS comm_join(INT fd, COMM_OUT intercomm)
+{
+
+    int rc;
+    uint32_t len, rlen, llen, lrlen;
+    int send_first=0;
+    ompi_process_name_t rname, tmp_name;
+
+    ompi_communicator_t *newcomp;
+    char port_name[MPI_MAX_PORT_NAME];
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if ( NULL == intercomm ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+    if (!ompi_mpi_dynamics_is_enabled(FUNC_NAME)) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(OMPI_ERR_NOT_SUPPORTED,
+                                      FUNC_NAME);
+    }
+
+    /* send my process name */
+    tmp_name = *OMPI_PROC_MY_NAME;
+    OMPI_PROCESS_NAME_HTON(tmp_name);
+    ompi_socket_send(fd, (char*) &tmp_name, sizeof(tmp_name));
+
+    /* recv the remote name */
+    ompi_socket_recv(fd, (char*) &rname, sizeof(rname));
+    OMPI_PROCESS_NAME_NTOH(rname);
+
+    /* compare the two to get send_first */
+    if (OMPI_PROC_MY_NAME->jobid == rname.jobid) {
+        if (OMPI_PROC_MY_NAME->vpid < rname.vpid) {
+            send_first = true;
+        } else if (OMPI_PROC_MY_NAME->vpid == rname.vpid) {
+            /* joining to myself is not allowed */
+            *intercomm = MPI_COMM_NULL;
+            return MPI_ERR_INTERN;
+        } else {
+            send_first = false;
+        }
+    } else if (OMPI_PROC_MY_NAME->jobid < rname.jobid) {
+        send_first = true;
+    }
+
+    /* Assumption: socket_send should not block, even if the socket
+       is not configured to be non-blocking, because the message length are
+       so short. */
+
+    /* we will only use the send_first proc's port name,
+     * so pass it to the recv_first participant */
+    if (send_first) {
+        // The port_name that we get back will be \0-terminated.  The
+        // strlen+\0 will be <= MPI_MAX_PORT_NAME characters.
+        if (OMPI_SUCCESS != (rc = ompi_dpm_open_port(port_name))) {
+            goto error;
+        }
+        // Send the strlen+1 so that we both send the \0 and the
+        // receiver receives the \0.
+        llen   = (uint32_t)(strlen(port_name)+1);
+        len    = htonl(llen);
+        ompi_socket_send( fd, (char *) &len, sizeof(uint32_t));
+        ompi_socket_send (fd, port_name, llen);
+    } else {
+        ompi_socket_recv (fd, (char *) &rlen, sizeof(uint32_t));
+        // The lrlen that we receive will be the strlen+1 (to account
+        // for \0), and will be <= MPI_MAX_PORT_NAME.
+        lrlen  = ntohl(rlen);
+        ompi_socket_recv (fd, port_name, lrlen);
+    }
+
+    /* use the port to connect/accept */
+    rc = ompi_dpm_connect_accept (MPI_COMM_SELF, 0, port_name, send_first, &newcomp);
+
+    *intercomm = newcomp;
+
+ error:
+    if (OPAL_ERR_NOT_SUPPORTED == rc) {
+        opal_show_help("help-mpi-api.txt",
+                       "MPI function not supported",
+                       true,
+                       FUNC_NAME,
+                       "Underlying runtime environment does not support join functionality");
+    }
+
+    OMPI_ERRHANDLER_RETURN (rc, MPI_COMM_SELF, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/comm_rank.c b/ompi/mpi/c/comm_rank.c
deleted file mode 100644
index 8709d71f34d..00000000000
--- a/ompi/mpi/c/comm_rank.c
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_rank = PMPI_Comm_rank
-#endif
-#define MPI_Comm_rank PMPI_Comm_rank
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_rank";
-
-
-int MPI_Comm_rank(MPI_Comm comm, int *rank)
-{
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_comm_invalid (comm))
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-
-        if ( NULL == rank )
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
-                                          FUNC_NAME);
-    }
-
-    *rank = ompi_comm_rank((ompi_communicator_t*)comm);
-    return MPI_SUCCESS;
-}
-
diff --git a/ompi/mpi/c/comm_rank.c.in b/ompi/mpi/c/comm_rank.c.in
new file mode 100644
index 00000000000..8580bc3555d
--- /dev/null
+++ b/ompi/mpi/c/comm_rank.c.in
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS comm_rank(COMM comm, INT_OUT rank)
+{
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_comm_invalid (comm))
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+
+        if ( NULL == rank )
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+    }
+
+    *rank = ompi_comm_rank((ompi_communicator_t*)comm);
+    return MPI_SUCCESS;
+}
+
diff --git a/ompi/mpi/c/comm_remote_group.c b/ompi/mpi/c/comm_remote_group.c
deleted file mode 100644
index 5412d422d0d..00000000000
--- a/ompi/mpi/c/comm_remote_group.c
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-#include <string.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_remote_group = PMPI_Comm_remote_group
-#endif
-#define MPI_Comm_remote_group PMPI_Comm_remote_group
-#endif
-
-
-static const char FUNC_NAME[] = "MPI_Comm_remote_group";
-
-
-int MPI_Comm_remote_group(MPI_Comm comm, MPI_Group *group)
-{
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid (comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-
-        if ( NULL == group ) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-    }
-
-    if ( OMPI_COMM_IS_INTER(comm) ) {
-        OBJ_RETAIN(comm->c_remote_group);
-    }
-    else {
-        return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_COMM,
-                                       FUNC_NAME);
-    }
-
-    *group = (MPI_Group) comm->c_remote_group;
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/comm_remote_group.c.in b/ompi/mpi/c/comm_remote_group.c.in
new file mode 100644
index 00000000000..32cf053e040
--- /dev/null
+++ b/ompi/mpi/c/comm_remote_group.c.in
@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+#include <string.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS comm_remote_group(COMM comm, GROUP_OUT group)
+{
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid (comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+
+        if ( NULL == group ) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+    if ( OMPI_COMM_IS_INTER(comm) ) {
+        OBJ_RETAIN(comm->c_remote_group);
+    }
+    else {
+        return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_COMM,
+                                       FUNC_NAME);
+    }
+
+    *group = (MPI_Group) comm->c_remote_group;
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/comm_remote_size.c b/ompi/mpi/c/comm_remote_size.c
deleted file mode 100644
index e02cb6d4790..00000000000
--- a/ompi/mpi/c/comm_remote_size.c
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_remote_size = PMPI_Comm_remote_size
-#endif
-#define MPI_Comm_remote_size PMPI_Comm_remote_size
-#endif
-
-
-static const char FUNC_NAME[] = "MPI_Comm_remote_size";
-
-
-int MPI_Comm_remote_size(MPI_Comm comm, int *size) {
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_comm_invalid (comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-
-        if ( NULL == size ) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-    }
-
-    *size = ompi_comm_remote_size ((ompi_communicator_t*)comm);
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/comm_remote_size.c.in b/ompi/mpi/c/comm_remote_size.c.in
new file mode 100644
index 00000000000..dcdc03bd924
--- /dev/null
+++ b/ompi/mpi/c/comm_remote_size.c.in
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS comm_remote_size(COMM comm, INT_OUT size)
+{
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_comm_invalid (comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+
+        if ( NULL == size ) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+    }
+
+    *size = ompi_comm_remote_size ((ompi_communicator_t*)comm);
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/comm_set_attr.c b/ompi/mpi/c/comm_set_attr.c
deleted file mode 100644
index 0de8da979b2..00000000000
--- a/ompi/mpi/c/comm_set_attr.c
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/attribute/attribute.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_set_attr = PMPI_Comm_set_attr
-#endif
-#define MPI_Comm_set_attr PMPI_Comm_set_attr
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_set_attr";
-
-
-int MPI_Comm_set_attr(MPI_Comm comm, int comm_keyval, void *attribute_val)
-{
-    int ret;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-    }
-
-    ret = ompi_attr_set_c(COMM_ATTR, comm, &comm->c_keyhash,
-                          comm_keyval, attribute_val, false);
-    OMPI_ERRHANDLER_RETURN(ret, comm, MPI_ERR_OTHER, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/comm_set_attr.c.in b/ompi/mpi/c/comm_set_attr.c.in
new file mode 100644
index 00000000000..9baa10aeac0
--- /dev/null
+++ b/ompi/mpi/c/comm_set_attr.c.in
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/attribute/attribute.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS comm_set_attr(COMM comm, INT comm_keyval,
+                                    BUFFER_OUT attribute_val)
+{
+    int ret;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+    }
+
+    ret = ompi_attr_set_c(COMM_ATTR, comm, &comm->c_keyhash,
+                          comm_keyval, attribute_val, false);
+    OMPI_ERRHANDLER_RETURN(ret, comm, MPI_ERR_OTHER, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/comm_set_errhandler.c b/ompi/mpi/c/comm_set_errhandler.c
deleted file mode 100644
index 0be31b76406..00000000000
--- a/ompi/mpi/c/comm_set_errhandler.c
+++ /dev/null
@@ -1,79 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015-2017 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_set_errhandler = PMPI_Comm_set_errhandler
-#endif
-#define MPI_Comm_set_errhandler PMPI_Comm_set_errhandler
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_set_errhandler";
-
-
-int MPI_Comm_set_errhandler(MPI_Comm comm, MPI_Errhandler errhandler)
-{
-    MPI_Errhandler tmp;
-
-    /* Error checking */
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    /* Error checking */
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if (NULL == errhandler ||
-                   MPI_ERRHANDLER_NULL == errhandler ||
-                   ( OMPI_ERRHANDLER_TYPE_COMM != errhandler->eh_mpi_object_type &&
-                     OMPI_ERRHANDLER_TYPE_PREDEFINED != errhandler->eh_mpi_object_type) ) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-    }
-
-    /* Prepare the new error handler */
-    OBJ_RETAIN(errhandler);
-
-    OPAL_THREAD_LOCK(&(comm->c_lock));
-    /* Ditch the old errhandler, and decrement its refcount. */
-    tmp = comm->error_handler;
-    comm->error_handler = errhandler;
-    OBJ_RELEASE(tmp);
-    OPAL_THREAD_UNLOCK(&(comm->c_lock));
-
-    /* All done */
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/comm_set_errhandler.c.in b/ompi/mpi/c/comm_set_errhandler.c.in
new file mode 100644
index 00000000000..9cfabe8e856
--- /dev/null
+++ b/ompi/mpi/c/comm_set_errhandler.c.in
@@ -0,0 +1,71 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015-2017 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS comm_set_errhandler(COMM comm, ERRHANDLER errhandler)
+{
+    MPI_Errhandler tmp;
+
+    /* Error checking */
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    /* Error checking */
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if (NULL == errhandler ||
+                   MPI_ERRHANDLER_NULL == errhandler ||
+                   ( OMPI_ERRHANDLER_TYPE_COMM != errhandler->eh_mpi_object_type &&
+                     OMPI_ERRHANDLER_TYPE_PREDEFINED != errhandler->eh_mpi_object_type) ) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+    /* Prepare the new error handler */
+    OBJ_RETAIN(errhandler);
+
+    OPAL_THREAD_LOCK(&(comm->c_lock));
+    /* Ditch the old errhandler, and decrement its refcount. */
+    tmp = comm->error_handler;
+    comm->error_handler = errhandler;
+    OBJ_RELEASE(tmp);
+    OPAL_THREAD_UNLOCK(&(comm->c_lock));
+
+    /* All done */
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/comm_set_info.c b/ompi/mpi/c/comm_set_info.c
deleted file mode 100644
index 9eca61263be..00000000000
--- a/ompi/mpi/c/comm_set_info.c
+++ /dev/null
@@ -1,52 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "opal/util/info_subscriber.h"
-#include <stdlib.h>
-#include <string.h>
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_set_info = PMPI_Comm_set_info
-#endif
-#define MPI_Comm_set_info PMPI_Comm_set_info
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_set_info";
-
-
-int MPI_Comm_set_info(MPI_Comm comm, MPI_Info info)
-{
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == info || MPI_INFO_NULL == info ||
-            ompi_info_is_freed(info)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
-                                          FUNC_NAME);
-        }
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-    }
-
-    opal_infosubscribe_change_info(&(comm->super), &(info->super));
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/comm_set_info.c.in b/ompi/mpi/c/comm_set_info.c.in
new file mode 100644
index 00000000000..30cd717eaf9
--- /dev/null
+++ b/ompi/mpi/c/comm_set_info.c.in
@@ -0,0 +1,44 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "opal/util/info_subscriber.h"
+#include <stdlib.h>
+#include <string.h>
+
+PROTOTYPE ERROR_CLASS comm_set_info(COMM comm, INFO info)
+{
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == info || MPI_INFO_NULL == info ||
+            ompi_info_is_freed(info)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
+                                          FUNC_NAME);
+        }
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+    }
+
+    opal_infosubscribe_change_info(&(comm->super), &(info->super));
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/comm_set_name.c b/ompi/mpi/c/comm_set_name.c
deleted file mode 100644
index 538d3b81646..00000000000
--- a/ompi/mpi/c/comm_set_name.c
+++ /dev/null
@@ -1,76 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <string.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/totalview.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_set_name = PMPI_Comm_set_name
-#endif
-#define MPI_Comm_set_name PMPI_Comm_set_name
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_set_name";
-
-
-int MPI_Comm_set_name(MPI_Comm comm, const char *name)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if ( ompi_comm_invalid ( comm ) ) {
-            return OMPI_ERRHANDLER_INVOKE ( MPI_COMM_WORLD, MPI_ERR_COMM,
-                                            FUNC_NAME);
-        }
-
-        if ( NULL == name ) {
-            return OMPI_ERRHANDLER_INVOKE ( comm, MPI_ERR_ARG,
-                                            FUNC_NAME);
-        }
-    }
-
-    rc = ompi_comm_set_name (comm, name );
-    /* -- Tracing information for new communicator name -- */
-#if 0
-  /* Force TotalView DLL to take note of this name setting */
-
-  ++ompi_tv_comm_sequence_number;
-#endif
-   OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/comm_set_name.c.in b/ompi/mpi/c/comm_set_name.c.in
new file mode 100644
index 00000000000..46dbd725bad
--- /dev/null
+++ b/ompi/mpi/c/comm_set_name.c.in
@@ -0,0 +1,68 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include <string.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/totalview.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS comm_set_name(COMM comm, STRING name)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if ( ompi_comm_invalid ( comm ) ) {
+            return OMPI_ERRHANDLER_INVOKE ( MPI_COMM_WORLD, MPI_ERR_COMM,
+                                            FUNC_NAME);
+        }
+
+        if ( NULL == name ) {
+            return OMPI_ERRHANDLER_INVOKE ( comm, MPI_ERR_ARG,
+                                            FUNC_NAME);
+        }
+    }
+
+    rc = ompi_comm_set_name (comm, name );
+    /* -- Tracing information for new communicator name -- */
+#if 0
+  /* Force TotalView DLL to take note of this name setting */
+
+  ++ompi_tv_comm_sequence_number;
+#endif
+   OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/comm_size.c b/ompi/mpi/c/comm_size.c
deleted file mode 100644
index 2c9f0232f48..00000000000
--- a/ompi/mpi/c/comm_size.c
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_size = PMPI_Comm_size
-#endif
-#define MPI_Comm_size PMPI_Comm_size
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_size";
-
-
-int MPI_Comm_size(MPI_Comm comm, int *size)
-{
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if ( ompi_comm_invalid (comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(
-                                          MPI_ERR_COMM, FUNC_NAME);
-        }
-
-        if ( NULL == size ) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-    }
-
-    *size = ompi_comm_size((ompi_communicator_t*)comm);
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/comm_size.c.in b/ompi/mpi/c/comm_size.c.in
new file mode 100644
index 00000000000..544a2f56256
--- /dev/null
+++ b/ompi/mpi/c/comm_size.c.in
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS comm_size(COMM comm, INT_OUT size)
+{
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if ( ompi_comm_invalid (comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(
+                                          MPI_ERR_COMM, FUNC_NAME);
+        }
+
+        if ( NULL == size ) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+    }
+
+    *size = ompi_comm_size((ompi_communicator_t*)comm);
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/comm_spawn.c b/ompi/mpi/c/comm_spawn.c
deleted file mode 100644
index dfeb9bffe07..00000000000
--- a/ompi/mpi/c/comm_spawn.c
+++ /dev/null
@@ -1,186 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2015 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2009      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Intel, Inc. All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "opal/util/show_help.h"
-#include "opal/util/printf.h"
-
-#include "ompi/info/info.h"
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/runtime/mpiruntime.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/dpm/dpm.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_spawn = PMPI_Comm_spawn
-#endif
-#define MPI_Comm_spawn PMPI_Comm_spawn
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_spawn";
-
-
-int MPI_Comm_spawn(const char *command, char *argv[], int maxprocs, MPI_Info info,
-		   int root, MPI_Comm comm, MPI_Comm *intercomm,
-		   int array_of_errcodes[])
-{
-    int rank, rc=OMPI_SUCCESS, i, flag;
-    bool send_first = false; /* we wait to be contacted */
-    ompi_communicator_t *newcomp=MPI_COMM_NULL;
-    char port_name[MPI_MAX_PORT_NAME]; char *port_string = NULL;
-    bool non_mpi = false;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if ( ompi_comm_invalid (comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-        if ( OMPI_COMM_IS_INTER(comm)) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-        if ( (0 > root) || (ompi_comm_size(comm) <= root) ) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-        if ( NULL == intercomm ) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &rc)) ) {
-        return OMPI_ERRHANDLER_INVOKE(comm, rc, FUNC_NAME);
-    }
-#endif
-
-    rank = ompi_comm_rank ( comm );
-    if ( MPI_PARAM_CHECK ) {
-        if ( rank == root ) {
-            if ( NULL == command ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
-                                              FUNC_NAME);
-            }
-            if ( 0 > maxprocs ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
-                                              FUNC_NAME);
-            }
-            if (NULL == info || ompi_info_is_freed(info)) {
-                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
-                                              FUNC_NAME);
-            }
-        }
-    }
-
-    if (!ompi_mpi_dynamics_is_enabled(FUNC_NAME)) {
-        return OMPI_ERRHANDLER_INVOKE(comm, OMPI_ERR_NOT_SUPPORTED, FUNC_NAME);
-    }
-
-    /* initialize the port name to avoid problems */
-    memset(port_name, 0, MPI_MAX_PORT_NAME);
-
-    /* See if the info key "ompi_non_mpi" was set to true */
-    if (rank == root) {
-        ompi_info_get_bool(info, "ompi_non_mpi", &non_mpi, &flag);
-    }
-
-    if ( rank == root ) {
-        if (!non_mpi) {
-            /* Open a port. The port_name is passed as an environment
-               variable to the children. */
-            if (OMPI_SUCCESS != (rc = ompi_dpm_open_port (port_name))) {
-                goto error;
-            }
-        } else if (1 < ompi_comm_size(comm)) {
-            /* we do not support non_mpi spawns on comms this size */
-            rc = OMPI_ERR_NOT_SUPPORTED;
-            goto error;
-        }
-        if (OMPI_SUCCESS != (rc = ompi_dpm_spawn (1, &command, &argv, &maxprocs,
-                                                  &info, port_name))) {
-            goto error;
-        }
-    }
-
-error:
-    if (OMPI_SUCCESS != rc) {
-        /* There was an error in one of the above stages,
-         * we still need to do the connect_accept stage so that
-         * non-root ranks do not deadlock.
-         * Add the error code to the port string for connect_accept
-         * to propagate the error code. */
-        (void)opal_asprintf(&port_string, "%s:error=%d", port_name, rc);
-    }
-    else {
-        port_string = port_name;
-    }
-
-    if (non_mpi) {
-        newcomp = MPI_COMM_NULL;
-    } else {
-        rc = ompi_dpm_connect_accept (comm, root, port_string, send_first, &newcomp);
-    }
-
-    if (OPAL_ERR_NOT_SUPPORTED == rc) {
-        opal_show_help("help-mpi-api.txt",
-                       "MPI function not supported",
-                       true,
-                       FUNC_NAME,
-                       "Underlying runtime environment does not support spawn functionality");
-    }
-
-    if(port_string != port_name) {
-        free(port_string);
-    }
-
-    /* close the port */
-    if (rank == root && !non_mpi) {
-        ompi_dpm_close_port(port_name);
-    }
-
-    /* set error codes */
-    if (MPI_ERRCODES_IGNORE != array_of_errcodes) {
-        for ( i=0; i < maxprocs; i++ ) {
-            array_of_errcodes[i]=rc;
-        }
-    }
-
-    *intercomm = newcomp;
-    OMPI_ERRHANDLER_RETURN (rc, comm, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/comm_spawn.c.in b/ompi/mpi/c/comm_spawn.c.in
new file mode 100644
index 00000000000..548fb113d02
--- /dev/null
+++ b/ompi/mpi/c/comm_spawn.c.in
@@ -0,0 +1,178 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2009      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Intel, Inc. All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "opal/util/show_help.h"
+#include "opal/util/printf.h"
+
+#include "ompi/info/info.h"
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/runtime/mpiruntime.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/dpm/dpm.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS comm_spawn(STRING command, STRING_ARRAY argv, INT maxprocs, INFO info,
+                                 INT root, COMM comm, COMM_OUT intercomm,
+                                 INT_OUT array_of_errcodes)
+{
+    int rank, rc=OMPI_SUCCESS, i, flag;
+    bool send_first = false; /* we wait to be contacted */
+    ompi_communicator_t *newcomp=MPI_COMM_NULL;
+    char port_name[MPI_MAX_PORT_NAME]; char *port_string = NULL;
+    bool non_mpi = false;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if ( ompi_comm_invalid (comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+        if ( OMPI_COMM_IS_INTER(comm)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+        if ( (0 > root) || (ompi_comm_size(comm) <= root) ) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+        if ( NULL == intercomm ) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &rc)) ) {
+        return OMPI_ERRHANDLER_INVOKE(comm, rc, FUNC_NAME);
+    }
+#endif
+
+    rank = ompi_comm_rank ( comm );
+    if ( MPI_PARAM_CHECK ) {
+        if ( rank == root ) {
+            if ( NULL == command ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
+                                              FUNC_NAME);
+            }
+            if ( 0 > maxprocs ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
+                                              FUNC_NAME);
+            }
+            if (NULL == info || ompi_info_is_freed(info)) {
+                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
+                                              FUNC_NAME);
+            }
+        }
+    }
+
+    if (!ompi_mpi_dynamics_is_enabled(FUNC_NAME)) {
+        return OMPI_ERRHANDLER_INVOKE(comm, OMPI_ERR_NOT_SUPPORTED, FUNC_NAME);
+    }
+
+    /* initialize the port name to avoid problems */
+    memset(port_name, 0, MPI_MAX_PORT_NAME);
+
+    /* See if the info key "ompi_non_mpi" was set to true */
+    if (rank == root) {
+        ompi_info_get_bool(info, "ompi_non_mpi", &non_mpi, &flag);
+    }
+
+    if ( rank == root ) {
+        if (!non_mpi) {
+            /* Open a port. The port_name is passed as an environment
+               variable to the children. */
+            if (OMPI_SUCCESS != (rc = ompi_dpm_open_port (port_name))) {
+                goto error;
+            }
+        } else if (1 < ompi_comm_size(comm)) {
+            /* we do not support non_mpi spawns on comms this size */
+            rc = OMPI_ERR_NOT_SUPPORTED;
+            goto error;
+        }
+        if (OMPI_SUCCESS != (rc = ompi_dpm_spawn (1, &command, &argv, &maxprocs,
+                                                  &info, port_name))) {
+            goto error;
+        }
+    }
+
+error:
+    if (OMPI_SUCCESS != rc) {
+        /* There was an error in one of the above stages,
+         * we still need to do the connect_accept stage so that
+         * non-root ranks do not deadlock.
+         * Add the error code to the port string for connect_accept
+         * to propagate the error code. */
+        (void)opal_asprintf(&port_string, "%s:error=%d", port_name, rc);
+    }
+    else {
+        port_string = port_name;
+    }
+
+    if (non_mpi) {
+        newcomp = MPI_COMM_NULL;
+    } else {
+        rc = ompi_dpm_connect_accept (comm, root, port_string, send_first, &newcomp);
+    }
+
+    if (OPAL_ERR_NOT_SUPPORTED == rc) {
+        opal_show_help("help-mpi-api.txt",
+                       "MPI function not supported",
+                       true,
+                       FUNC_NAME,
+                       "Underlying runtime environment does not support spawn functionality");
+    }
+
+    if(port_string != port_name) {
+        free(port_string);
+    }
+
+    /* close the port */
+    if (rank == root && !non_mpi) {
+        ompi_dpm_close_port(port_name);
+    }
+
+    /* set error codes */
+    if (MPI_ERRCODES_IGNORE != array_of_errcodes) {
+        for ( i=0; i < maxprocs; i++ ) {
+            array_of_errcodes[i]=rc;
+        }
+    }
+
+    *intercomm = newcomp;
+    OMPI_ERRHANDLER_RETURN (rc, comm, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/comm_spawn_multiple.c b/ompi/mpi/c/comm_spawn_multiple.c
deleted file mode 100644
index 58b36c855c6..00000000000
--- a/ompi/mpi/c/comm_spawn_multiple.c
+++ /dev/null
@@ -1,236 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2015 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2009      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Intel, Inc. All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "opal/util/show_help.h"
-#include "opal/util/printf.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/runtime/mpiruntime.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-#include "ompi/dpm/dpm.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_spawn_multiple = PMPI_Comm_spawn_multiple
-#endif
-#define MPI_Comm_spawn_multiple PMPI_Comm_spawn_multiple
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_spawn_multiple";
-
-
-int MPI_Comm_spawn_multiple(int count, char *array_of_commands[], char **array_of_argv[],
-                            const int array_of_maxprocs[], const MPI_Info array_of_info[],
-                            int root, MPI_Comm comm, MPI_Comm *intercomm,
-                            int array_of_errcodes[])
-{
-    int i=0, rc=0, rank=0, size=0, flag;
-    ompi_communicator_t *newcomp=MPI_COMM_NULL;
-    bool send_first=false; /* they are contacting us first */
-    char port_name[MPI_MAX_PORT_NAME]; char *port_string = NULL;
-    bool non_mpi = false, cumulative = false;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if ( ompi_comm_invalid (comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-        if ( OMPI_COMM_IS_INTER(comm)) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COMM, FUNC_NAME);
-        }
-        if ( (0 > root) || (ompi_comm_size(comm) <= root) ) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-        if ( NULL == intercomm ) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-    }
-
-    rank = ompi_comm_rank ( comm );
-    if ( MPI_PARAM_CHECK ) {
-        if ( rank == root ) {
-            if ( 0 > count ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-            if ( NULL == array_of_commands ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-            if ( NULL ==  array_of_maxprocs ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-            if ( NULL == array_of_info ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_INFO, FUNC_NAME);
-            }
-            for (i = 0; i < count; ++i) {
-                if (NULL == array_of_info[i] ||
-                    ompi_info_is_freed(array_of_info[i])) {
-                    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
-                                                  FUNC_NAME);
-                }
-                /* If ompi_non_mpi is set to true on any info, it must
-                   be set to true on all of them.  Note that not
-                   setting ompi_non_mpi is the same as setting it to
-                   false. */
-                ompi_info_get_bool(array_of_info[i], "ompi_non_mpi", &non_mpi,
-                                   &flag);
-                if (flag && 0 == i) {
-                    /* If this is the first info, save its
-                       ompi_non_mpi value */
-                    cumulative = non_mpi;
-                } else if (!flag) {
-                    non_mpi = false;
-                }
-                /* If this info's effective value doesn't agree with
-                   the rest of them, error */
-                if (cumulative != non_mpi) {
-                    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(
-                                                  MPI_ERR_INFO,
-                                                  FUNC_NAME);
-                }
-            }
-            for ( i=0; i<count; i++ ) {
-                if ( NULL == array_of_commands[i] ) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-                }
-                if ( 0 > array_of_maxprocs[i] ) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-                }
-            }
-        }
-    }
-
-    if (!ompi_mpi_dynamics_is_enabled(FUNC_NAME)) {
-        return OMPI_ERRHANDLER_INVOKE(comm, OMPI_ERR_NOT_SUPPORTED, FUNC_NAME);
-    }
-
-    if (rank == root) {
-        if (MPI_INFO_NULL == array_of_info[0]) {
-            non_mpi = false;
-        } else {
-            ompi_info_get_bool(array_of_info[0], "ompi_non_mpi", &non_mpi,
-                               &flag);
-            if (!flag) {
-                non_mpi = false;
-            }
-        }
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &rc)) ) {
-        return OMPI_ERRHANDLER_INVOKE(comm, rc, FUNC_NAME);
-    }
-#endif
-
-    /* initialize the port name to avoid problems */
-    memset(port_name, 0, MPI_MAX_PORT_NAME);
-
-
-    if ( rank == root ) {
-        if (!non_mpi) {
-            /* Open a port. The port_name is passed as an environment
-               variable to the children. */
-            if (OMPI_SUCCESS != (rc = ompi_dpm_open_port (port_name))) {
-                goto error;
-            }
-        } else if (1 < ompi_comm_size(comm)) {
-            /* we do not support non_mpi spawns on comms this size */
-            rc = OMPI_ERR_NOT_SUPPORTED;
-            goto error;
-        }
-        if (OMPI_SUCCESS != (rc = ompi_dpm_spawn(count, (const char **) array_of_commands,
-                                                 array_of_argv, array_of_maxprocs,
-                                                 array_of_info, port_name))) {
-            goto error;
-        }
-    }
-
-error:
-    if (OMPI_SUCCESS != rc) {
-        /* There was an error in one of the above stages,
-         * we still need to do the connect_accept stage so that
-         * non-root ranks do not deadlock.
-         * Add the error code to the port string for connect_accept
-         * to propagate the error code. */
-        (void)opal_asprintf(&port_string, "%s:error=%d", port_name, rc);
-    }
-    else {
-        port_string = port_name;
-    }
-
-    if (non_mpi) {
-        newcomp = MPI_COMM_NULL;
-    } else {
-        rc = ompi_dpm_connect_accept (comm, root, port_string, send_first, &newcomp);
-    }
-
-    if (OPAL_ERR_NOT_SUPPORTED == rc) {
-        opal_show_help("help-mpi-api.txt",
-                       "MPI function not supported",
-                       true,
-                       FUNC_NAME,
-                       "Underlying runtime environment does not support spawn functionality");
-    }
-
-    if(port_string != port_name) {
-        free(port_string);
-    }
-
-    /* close the port */
-    if (rank == root && !non_mpi) {
-        ompi_dpm_close_port(port_name);
-    }
-
-    /* set array of errorcodes */
-    if (MPI_ERRCODES_IGNORE != array_of_errcodes) {
-        if (MPI_COMM_NULL != newcomp) {
-            size = newcomp->c_remote_group->grp_proc_count;
-        } else {
-            for ( i=0; i < count; i++) {
-                size = size + array_of_maxprocs[i];
-            }
-        }
-        for ( i=0; i < size; i++ ) {
-            array_of_errcodes[i]=rc;
-        }
-    }
-
-    *intercomm = newcomp;
-    OMPI_ERRHANDLER_RETURN (rc, comm, rc, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/comm_spawn_multiple.c.in b/ompi/mpi/c/comm_spawn_multiple.c.in
new file mode 100644
index 00000000000..a729df6dfd0
--- /dev/null
+++ b/ompi/mpi/c/comm_spawn_multiple.c.in
@@ -0,0 +1,228 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2009      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Intel, Inc. All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "opal/util/show_help.h"
+#include "opal/util/printf.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/runtime/mpiruntime.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include "ompi/dpm/dpm.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS comm_spawn_multiple(INT count, STRING_ARRAY array_of_commands, ARGV array_of_argv,
+                                          INT_ARRAY array_of_maxprocs, INFO_ARRAY array_of_info,
+                                          INT root, COMM comm, COMM_OUT intercomm,
+                                          INT_OUT array_of_errcodes)
+{
+    int i=0, rc=0, rank=0, size=0, flag;
+    ompi_communicator_t *newcomp=MPI_COMM_NULL;
+    bool send_first=false; /* they are contacting us first */
+    char port_name[MPI_MAX_PORT_NAME]; char *port_string = NULL;
+    bool non_mpi = false, cumulative = false;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if ( ompi_comm_invalid (comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+        if ( OMPI_COMM_IS_INTER(comm)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COMM, FUNC_NAME);
+        }
+        if ( (0 > root) || (ompi_comm_size(comm) <= root) ) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+        if ( NULL == intercomm ) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+    }
+
+    rank = ompi_comm_rank ( comm );
+    if ( MPI_PARAM_CHECK ) {
+        if ( rank == root ) {
+            if ( 0 > count ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+            if ( NULL == array_of_commands ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+            if ( NULL ==  array_of_maxprocs ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+            if ( NULL == array_of_info ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_INFO, FUNC_NAME);
+            }
+            for (i = 0; i < count; ++i) {
+                if (NULL == array_of_info[i] ||
+                    ompi_info_is_freed(array_of_info[i])) {
+                    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
+                                                  FUNC_NAME);
+                }
+                /* If ompi_non_mpi is set to true on any info, it must
+                   be set to true on all of them.  Note that not
+                   setting ompi_non_mpi is the same as setting it to
+                   false. */
+                ompi_info_get_bool(array_of_info[i], "ompi_non_mpi", &non_mpi,
+                                   &flag);
+                if (flag && 0 == i) {
+                    /* If this is the first info, save its
+                       ompi_non_mpi value */
+                    cumulative = non_mpi;
+                } else if (!flag) {
+                    non_mpi = false;
+                }
+                /* If this info's effective value doesn't agree with
+                   the rest of them, error */
+                if (cumulative != non_mpi) {
+                    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(
+                                                  MPI_ERR_INFO,
+                                                  FUNC_NAME);
+                }
+            }
+            for ( i=0; i<count; i++ ) {
+                if ( NULL == array_of_commands[i] ) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+                }
+                if ( 0 > array_of_maxprocs[i] ) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+                }
+            }
+        }
+    }
+
+    if (!ompi_mpi_dynamics_is_enabled(FUNC_NAME)) {
+        return OMPI_ERRHANDLER_INVOKE(comm, OMPI_ERR_NOT_SUPPORTED, FUNC_NAME);
+    }
+
+    if (rank == root) {
+        if (MPI_INFO_NULL == array_of_info[0]) {
+            non_mpi = false;
+        } else {
+            ompi_info_get_bool(array_of_info[0], "ompi_non_mpi", &non_mpi,
+                               &flag);
+            if (!flag) {
+                non_mpi = false;
+            }
+        }
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &rc)) ) {
+        return OMPI_ERRHANDLER_INVOKE(comm, rc, FUNC_NAME);
+    }
+#endif
+
+    /* initialize the port name to avoid problems */
+    memset(port_name, 0, MPI_MAX_PORT_NAME);
+
+
+    if ( rank == root ) {
+        if (!non_mpi) {
+            /* Open a port. The port_name is passed as an environment
+               variable to the children. */
+            if (OMPI_SUCCESS != (rc = ompi_dpm_open_port (port_name))) {
+                goto error;
+            }
+        } else if (1 < ompi_comm_size(comm)) {
+            /* we do not support non_mpi spawns on comms this size */
+            rc = OMPI_ERR_NOT_SUPPORTED;
+            goto error;
+        }
+        if (OMPI_SUCCESS != (rc = ompi_dpm_spawn(count, (const char **) array_of_commands,
+                                                 array_of_argv, array_of_maxprocs,
+                                                 array_of_info, port_name))) {
+            goto error;
+        }
+    }
+
+error:
+    if (OMPI_SUCCESS != rc) {
+        /* There was an error in one of the above stages,
+         * we still need to do the connect_accept stage so that
+         * non-root ranks do not deadlock.
+         * Add the error code to the port string for connect_accept
+         * to propagate the error code. */
+        (void)opal_asprintf(&port_string, "%s:error=%d", port_name, rc);
+    }
+    else {
+        port_string = port_name;
+    }
+
+    if (non_mpi) {
+        newcomp = MPI_COMM_NULL;
+    } else {
+        rc = ompi_dpm_connect_accept (comm, root, port_string, send_first, &newcomp);
+    }
+
+    if (OPAL_ERR_NOT_SUPPORTED == rc) {
+        opal_show_help("help-mpi-api.txt",
+                       "MPI function not supported",
+                       true,
+                       FUNC_NAME,
+                       "Underlying runtime environment does not support spawn functionality");
+    }
+
+    if(port_string != port_name) {
+        free(port_string);
+    }
+
+    /* close the port */
+    if (rank == root && !non_mpi) {
+        ompi_dpm_close_port(port_name);
+    }
+
+    /* set array of errorcodes */
+    if (MPI_ERRCODES_IGNORE != array_of_errcodes) {
+        if (MPI_COMM_NULL != newcomp) {
+            size = newcomp->c_remote_group->grp_proc_count;
+        } else {
+            for ( i=0; i < count; i++) {
+                size = size + array_of_maxprocs[i];
+            }
+        }
+        for ( i=0; i < size; i++ ) {
+            array_of_errcodes[i]=rc;
+        }
+    }
+
+    *intercomm = newcomp;
+    OMPI_ERRHANDLER_RETURN (rc, comm, rc, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/comm_split.c b/ompi/mpi/c/comm_split.c
deleted file mode 100644
index a55f0fa7204..00000000000
--- a/ompi/mpi/c/comm_split.c
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_split = PMPI_Comm_split
-#endif
-#define MPI_Comm_split PMPI_Comm_split
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_split";
-
-
-int MPI_Comm_split(MPI_Comm comm, int color, int key, MPI_Comm *newcomm) {
-
-    int rc;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if ( ompi_comm_invalid ( comm )) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-
-        if ( color < 0 &&  MPI_UNDEFINED != color ) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-
-        if ( NULL == newcomm ) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_create_check(comm, &rc)) ) {
-        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-    }
-#endif
-
-    rc = ompi_comm_split ( (ompi_communicator_t*)comm, color, key,
-                          (ompi_communicator_t**)newcomm, false);
-    OMPI_ERRHANDLER_RETURN ( rc, comm, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/comm_split.c.in b/ompi/mpi/c/comm_split.c.in
new file mode 100644
index 00000000000..6d361ce539f
--- /dev/null
+++ b/ompi/mpi/c/comm_split.c.in
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS comm_split(COMM comm, INT color, INT key, COMM_OUT newcomm)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if ( ompi_comm_invalid ( comm )) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+
+        if ( color < 0 &&  MPI_UNDEFINED != color ) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+
+        if ( NULL == newcomm ) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_create_check(comm, &rc)) ) {
+        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+    }
+#endif
+
+    rc = ompi_comm_split ( (ompi_communicator_t*)comm, color, key,
+                          (ompi_communicator_t**)newcomm, false);
+    OMPI_ERRHANDLER_RETURN ( rc, comm, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/comm_split_type.c b/ompi/mpi/c/comm_split_type.c
deleted file mode 100644
index ba24cae5c12..00000000000
--- a/ompi/mpi/c/comm_split_type.c
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012      Sandia National Laboratories. All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_split_type = PMPI_Comm_split_type
-#endif
-#define MPI_Comm_split_type PMPI_Comm_split_type
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_split_type";
-
-
-int MPI_Comm_split_type(MPI_Comm comm, int split_type, int key,
-                        MPI_Info info, MPI_Comm *newcomm) {
-
-    int rc;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if ( ompi_comm_invalid ( comm )) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-
-        if (NULL == info || ompi_info_is_freed(info)) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_INFO,
-                                          FUNC_NAME);
-        }
-
-        if ( MPI_COMM_TYPE_SHARED != split_type && // Same as OMPI_COMM_TYPE_NODE
-	     OMPI_COMM_TYPE_CLUSTER != split_type &&
-	     OMPI_COMM_TYPE_CU != split_type &&
-	     OMPI_COMM_TYPE_HOST != split_type &&
-	     OMPI_COMM_TYPE_BOARD != split_type &&
-	     OMPI_COMM_TYPE_NODE != split_type && // Same as MPI_COMM_TYPE_SHARED
-	     OMPI_COMM_TYPE_NUMA != split_type &&
-	     OMPI_COMM_TYPE_SOCKET != split_type &&
-	     OMPI_COMM_TYPE_L3CACHE != split_type &&
-	     OMPI_COMM_TYPE_L2CACHE != split_type &&
-	     OMPI_COMM_TYPE_L1CACHE != split_type &&
-	     OMPI_COMM_TYPE_CORE != split_type &&
-	     OMPI_COMM_TYPE_HWTHREAD != split_type &&
-             MPI_UNDEFINED != split_type ) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-
-        if ( NULL == newcomm ) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_create_check(comm, &rc)) ) {
-        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-    }
-#endif
-
-    if( (MPI_COMM_SELF == comm) && (MPI_UNDEFINED == split_type) ) {
-        *newcomm = MPI_COMM_NULL;
-        rc = MPI_SUCCESS;
-    } else {
-        rc = ompi_comm_split_type( (ompi_communicator_t*)comm, split_type, key, &(info->super),
-                                   (ompi_communicator_t**)newcomm);
-    }
-    OMPI_ERRHANDLER_RETURN ( rc, comm, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/comm_split_type.c.in b/ompi/mpi/c/comm_split_type.c.in
new file mode 100644
index 00000000000..26f2b39cea9
--- /dev/null
+++ b/ompi/mpi/c/comm_split_type.c.in
@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017-2022 IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS comm_split_type(COMM comm, INT split_type, INT key,
+                                      INFO info, COMM_OUT newcomm)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if ( ompi_comm_invalid ( comm )) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+
+        if (NULL == info || ompi_info_is_freed(info)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_INFO,
+                                          FUNC_NAME);
+        }
+
+        if ( MPI_COMM_TYPE_SHARED != split_type && // Same as OMPI_COMM_TYPE_NODE
+             MPI_COMM_TYPE_HW_UNGUIDED != split_type &&
+             MPI_COMM_TYPE_HW_GUIDED != split_type &&
+	     OMPI_COMM_TYPE_CLUSTER != split_type &&
+	     OMPI_COMM_TYPE_CU != split_type &&
+	     OMPI_COMM_TYPE_HOST != split_type &&
+	     OMPI_COMM_TYPE_BOARD != split_type &&
+	     OMPI_COMM_TYPE_NODE != split_type && // Same as MPI_COMM_TYPE_SHARED
+	     OMPI_COMM_TYPE_NUMA != split_type &&
+	     OMPI_COMM_TYPE_SOCKET != split_type &&
+	     OMPI_COMM_TYPE_L3CACHE != split_type &&
+	     OMPI_COMM_TYPE_L2CACHE != split_type &&
+	     OMPI_COMM_TYPE_L1CACHE != split_type &&
+	     OMPI_COMM_TYPE_CORE != split_type &&
+	     OMPI_COMM_TYPE_HWTHREAD != split_type &&
+             MPI_UNDEFINED != split_type ) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+
+        if ( NULL == newcomm ) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_create_check(comm, &rc)) ) {
+        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+    }
+#endif
+
+    if ( MPI_COMM_TYPE_HW_GUIDED == split_type ) {
+        int flag;
+        opal_cstring_t *value = NULL;
+
+        /* MPI_Info is required for this split_type.
+         * Not an error condition, per MPI 4.0.
+         */
+        if ( MPI_INFO_NULL == info ) {
+            *newcomm = MPI_COMM_NULL;
+            rc = MPI_SUCCESS;
+            OMPI_ERRHANDLER_RETURN ( rc, comm, rc, FUNC_NAME);
+        }
+
+        /* MPI_Info with key "mpi_hw_resource_type" is required for this split_type.
+         * Not an error condition, per MPI 4.0.
+         */
+        ompi_info_get(info, "mpi_hw_resource_type", &value, &flag);
+        if ( !flag ) {
+            *newcomm = MPI_COMM_NULL;
+            rc = MPI_SUCCESS;
+            OMPI_ERRHANDLER_RETURN ( rc, comm, rc, FUNC_NAME);
+        }
+    }
+
+    if( (MPI_COMM_SELF == comm) && (MPI_UNDEFINED == split_type) ) {
+        *newcomm = MPI_COMM_NULL;
+        rc = MPI_SUCCESS;
+    } else {
+        rc = ompi_comm_split_type( (ompi_communicator_t*)comm, split_type, key, &(info->super),
+                                   (ompi_communicator_t**)newcomm);
+    }
+    OMPI_ERRHANDLER_RETURN ( rc, comm, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/comm_test_inter.c b/ompi/mpi/c/comm_test_inter.c
deleted file mode 100644
index 141a90d3478..00000000000
--- a/ompi/mpi/c/comm_test_inter.c
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Comm_test_inter = PMPI_Comm_test_inter
-#endif
-#define MPI_Comm_test_inter PMPI_Comm_test_inter
-#endif
-
-static const char FUNC_NAME[] = "MPI_Comm_test_inter";
-
-
-int MPI_Comm_test_inter(MPI_Comm comm, int *flag) {
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if ( ompi_comm_invalid ( comm ) ) {
-             return OMPI_ERRHANDLER_INVOKE ( MPI_COMM_WORLD, MPI_ERR_COMM,
-                                             FUNC_NAME);
-        }
-
-        if ( NULL == flag ) {
-             return OMPI_ERRHANDLER_INVOKE ( comm, MPI_ERR_ARG,
-                                             FUNC_NAME);
-        }
-    }
-
-    *flag = (comm->c_flags & OMPI_COMM_INTER);
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/comm_test_inter.c.in b/ompi/mpi/c/comm_test_inter.c.in
new file mode 100644
index 00000000000..51aa348b0f5
--- /dev/null
+++ b/ompi/mpi/c/comm_test_inter.c.in
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS comm_test_inter(COMM comm, INT_OUT flag)
+{
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if ( ompi_comm_invalid ( comm )) {
+             return OMPI_ERRHANDLER_INVOKE ( MPI_COMM_WORLD, MPI_ERR_COMM,
+                                             FUNC_NAME);
+        }
+
+        if ( NULL == flag ){
+             return OMPI_ERRHANDLER_INVOKE ( comm, MPI_ERR_ARG,
+                                             FUNC_NAME);
+        }
+    }
+
+    *flag = ( comm->c_flags & OMPI_COMM_INTER );
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/compare_and_swap.c b/ompi/mpi/c/compare_and_swap.c
deleted file mode 100644
index 014c31d5c27..00000000000
--- a/ompi/mpi/c/compare_and_swap.c
+++ /dev/null
@@ -1,74 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-#include "ompi/mca/osc/osc.h"
-#include "ompi/datatype/ompi_datatype.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Compare_and_swap = PMPI_Compare_and_swap
-#endif
-#define MPI_Compare_and_swap PMPI_Compare_and_swap
-#endif
-
-static const char FUNC_NAME[] = "MPI_Compare_and_swap";
-
-
-int MPI_Compare_and_swap(const void *origin_addr, const void *compare_addr, void *result_addr,
-                         MPI_Datatype datatype, int target_rank, MPI_Aint target_disp, MPI_Win win)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        rc = OMPI_SUCCESS;
-
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        } else if (ompi_win_peer_invalid(win, target_rank) &&
-                   (MPI_PROC_NULL != target_rank)) {
-            rc = MPI_ERR_RANK;
-        } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) {
-            rc = MPI_ERR_DISP;
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, datatype, 1);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME);
-    }
-
-    if (MPI_PROC_NULL == target_rank) return MPI_SUCCESS;
-
-    rc = win->w_osc_module->osc_compare_and_swap(origin_addr, compare_addr, result_addr,
-                                                 datatype, target_rank, target_disp, win);
-    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/compare_and_swap.c.in b/ompi/mpi/c/compare_and_swap.c.in
new file mode 100644
index 00000000000..5dc5e94e524
--- /dev/null
+++ b/ompi/mpi/c/compare_and_swap.c.in
@@ -0,0 +1,67 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+#include "ompi/mca/osc/osc.h"
+#include "ompi/datatype/ompi_datatype.h"
+
+PROTOTYPE ERROR_CLASS compare_and_swap(BUFFER origin_addr, BUFFER compare_addr, BUFFER_OUT result_addr,
+                                       DATATYPE datatype, INT target_rank, AINT target_disp, WIN win)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        rc = OMPI_SUCCESS;
+
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        } else if (ompi_win_peer_invalid(win, target_rank) &&
+                   (MPI_PROC_NULL != target_rank)) {
+            rc = MPI_ERR_RANK;
+        } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) {
+            rc = MPI_ERR_DISP;
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, datatype, 1);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME);
+    }
+
+    if (MPI_PROC_NULL == target_rank) return MPI_SUCCESS;
+
+    rc = win->w_osc_module->osc_compare_and_swap(origin_addr, compare_addr, result_addr,
+                                                 datatype, target_rank, target_disp, win);
+    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/dims_create.c b/ompi/mpi/c/dims_create.c
deleted file mode 100644
index f38a76c767e..00000000000
--- a/ompi/mpi/c/dims_create.c
+++ /dev/null
@@ -1,253 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2014 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2012      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2014      Intel, Inc. All rights reserved
- * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015-2016 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <math.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Dims_create = PMPI_Dims_create
-#endif
-#define MPI_Dims_create PMPI_Dims_create
-#endif
-
-static const char FUNC_NAME[] = "MPI_Dims_create";
-
-/* static functions */
-static int assignnodes(int ndim, int nfactor, int *pfacts,int **pdims);
-static int getfactors(int num, int *nfators, int **factors);
-
-
-/*
- * This is a utility function, no need to have anything in the lower
- * layer for this at all
- */
-int MPI_Dims_create(int nnodes, int ndims, int dims[])
-{
-    int i;
-    int freeprocs;
-    int freedims;
-    int nfactors;
-    int *factors;
-    int *procs;
-    int *p;
-    int err;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (0 > ndims) {
-            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD,
-                                           MPI_ERR_DIMS, FUNC_NAME);
-        }
-
-        if ((0 != ndims) && (NULL == dims)) {
-            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD,
-                                           MPI_ERR_ARG, FUNC_NAME);
-        }
-
-        if (1 > nnodes) {
-            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD,
-                                           MPI_ERR_DIMS, FUNC_NAME);
-        }
-    }
-
-    /* Get # of free-to-be-assigned processes and # of free dimensions */
-    freeprocs = nnodes;
-    freedims = 0;
-    for (i = 0, p = dims; i < ndims; ++i,++p) {
-        if (*p == 0) {
-            ++freedims;
-        } else if ((*p < 0) || ((nnodes % *p) != 0)) {
-            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_DIMS,
-                                           FUNC_NAME);
-        } else {
-            freeprocs /= *p;
-        }
-    }
-
-    if (freedims == 0) {
-       if (freeprocs == 1) {
-          return MPI_SUCCESS;
-       }
-       return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_DIMS,
-                                     FUNC_NAME);
-    }
-
-    if (freeprocs == 1) {
-        for (i = 0; i < ndims; ++i, ++dims) {
-            if (*dims == 0) {
-               *dims = 1;
-            }
-        }
-        return MPI_SUCCESS;
-    }
-
-    /* Factor the number of free processes */
-    if (MPI_SUCCESS != (err = getfactors(freeprocs, &nfactors, &factors))) {
-       return OMPI_ERRHANDLER_NOHANDLE_INVOKE(err,
-                                     FUNC_NAME);
-    }
-
-    /* Assign free processes to free dimensions */
-    if (MPI_SUCCESS != (err = assignnodes(freedims, nfactors, factors, &procs))) {
-       free(factors);
-       return OMPI_ERRHANDLER_NOHANDLE_INVOKE(err,
-                                     FUNC_NAME);
-    }
-
-    /* Return assignment results */
-    p = procs;
-    for (i = 0; i < ndims; ++i, ++dims) {
-        if (*dims == 0) {
-           *dims = *p++;
-        }
-    }
-
-    free((char *) factors);
-    free((char *) procs);
-
-    /* all done */
-    return MPI_SUCCESS;
-}
-
-/*
- *  assignnodes
- *
- *  Function:   - assign processes to dimensions
- *          - get "best-balanced" grid
- *          - greedy bin-packing algorithm used
- *          - sort dimensions in decreasing order
- *          - dimensions array dynamically allocated
- *  Accepts:    - # of dimensions
- *          - # of prime factors
- *          - array of prime factors
- *          - ptr to array of dimensions (returned value)
- *  Returns:    - 0 or ERROR
- */
-static int
-assignnodes(int ndim, int nfactor, int *pfacts, int **pdims)
-{
-    int *bins;
-    int i, j;
-    int n;
-    int f;
-    int *p;
-    int *pmin;
-
-    if (0 >= ndim) {
-       return MPI_ERR_DIMS;
-    }
-
-    /* Allocate and initialize the bins */
-    bins = (int *) malloc((unsigned) ndim * sizeof(int));
-    if (NULL == bins) {
-       return MPI_ERR_NO_MEM;
-    }
-    *pdims = bins;
-
-    for (i = 0, p = bins; i < ndim; ++i, ++p) {
-        *p = 1;
-     }
-
-    /* Loop assigning factors from the highest to the lowest */
-    for (j = nfactor - 1; j >= 0; --j) {
-        f = pfacts[j];
-        /* Assign a factor to the smallest bin */
-        pmin = bins;
-        for (i = 1, p = pmin + 1; i < ndim; ++i, ++p) {
-            if (*p < *pmin) {
-                pmin = p;
-            }
-        }
-        *pmin *= f;
-     }
-
-     /* Sort dimensions in decreasing order (O(n^2) for now) */
-     for (i = 0, pmin = bins; i < ndim - 1; ++i, ++pmin) {
-         for (j = i + 1, p = pmin + 1; j < ndim; ++j, ++p) {
-             if (*p > *pmin) {
-                n = *p;
-                *p = *pmin;
-                *pmin = n;
-             }
-         }
-     }
-
-     return MPI_SUCCESS;
-}
-
-/*
- *  getfactors
- *
- *  Function:   - factorize a number
- *  Accepts:    - number
- *          - # prime factors
- *          - array of prime factors
- *  Returns:    - MPI_SUCCESS or ERROR
- */
-static int
-getfactors(int num, int *nfactors, int **factors) {
-    int size;
-    int d;
-    int i;
-    int sqrtnum;
-
-    if(num  < 2) {
-        (*nfactors) = 0;
-        (*factors) = NULL;
-        return MPI_SUCCESS;
-    }
-    /* Allocate the array of prime factors which cannot exceed log_2(num) entries */
-    sqrtnum = ceil(sqrt(num));
-    size = ceil(log(num) / log(2));
-    *factors = (int *) malloc((unsigned) size * sizeof(int));
-
-    i = 0;
-    /* determine all occurences of factor 2 */
-    while((num % 2) == 0) {
-        num /= 2;
-        (*factors)[i++] = 2;
-    }
-    /* determine all occurences of uneven prime numbers up to sqrt(num) */
-    d = 3;
-    for(d = 3; (num > 1) && (d <= sqrtnum); d += 2) {
-        while((num % d) == 0) {
-            num /= d;
-            (*factors)[i++] = d;
-        }
-    }
-    /* as we looped only up to sqrt(num) one factor > sqrt(num) may be left over */
-    if(num != 1) {
-        (*factors)[i++] = num;
-    }
-    (*nfactors) = i;
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/dims_create.c.in b/ompi/mpi/c/dims_create.c.in
new file mode 100644
index 00000000000..a669173e085
--- /dev/null
+++ b/ompi/mpi/c/dims_create.c.in
@@ -0,0 +1,242 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2014 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2012      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2014      Intel, Inc. All rights reserved
+ * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2016 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include <math.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+
+/* static functions */
+static int assignnodes(int ndim, int nfactor, int *pfacts,int **pdims);
+static int getfactors(int num, int *nfators, int **factors);
+
+/*
+ *  assignnodes
+ *
+ *  Function:   - assign processes to dimensions
+ *          - get "best-balanced" grid
+ *          - greedy bin-packing algorithm used
+ *          - sort dimensions in decreasing order
+ *          - dimensions array dynamically allocated
+ *  Accepts:    - # of dimensions
+ *          - # of prime factors
+ *          - array of prime factors
+ *          - ptr to array of dimensions (returned value)
+ *  Returns:    - 0 or ERROR
+ */
+static int assignnodes(int ndim, int nfactor, int *pfacts, int **pdims)
+{
+    int *bins;
+    int i, j;
+    int n;
+    int f;
+    int *p;
+    int *pmin;
+
+    if (0 >= ndim) {
+       return MPI_ERR_DIMS;
+    }
+
+    /* Allocate and initialize the bins */
+    bins = (int *) malloc((unsigned) ndim * sizeof(int));
+    if (NULL == bins) {
+       return MPI_ERR_NO_MEM;
+    }
+    *pdims = bins;
+
+    for (i = 0, p = bins; i < ndim; ++i, ++p) {
+        *p = 1;
+     }
+
+    /* Loop assigning factors from the highest to the lowest */
+    for (j = nfactor - 1; j >= 0; --j) {
+        f = pfacts[j];
+        /* Assign a factor to the smallest bin */
+        pmin = bins;
+        for (i = 1, p = pmin + 1; i < ndim; ++i, ++p) {
+            if (*p < *pmin) {
+                pmin = p;
+            }
+        }
+        *pmin *= f;
+     }
+
+     /* Sort dimensions in decreasing order (O(n^2) for now) */
+     for (i = 0, pmin = bins; i < ndim - 1; ++i, ++pmin) {
+         for (j = i + 1, p = pmin + 1; j < ndim; ++j, ++p) {
+             if (*p > *pmin) {
+                n = *p;
+                *p = *pmin;
+                *pmin = n;
+             }
+         }
+     }
+
+     return MPI_SUCCESS;
+}
+
+/*
+ *  getfactors
+ *
+ *  Function:   - factorize a number
+ *  Accepts:    - number
+ *          - # prime factors
+ *          - array of prime factors
+ *  Returns:    - MPI_SUCCESS or ERROR
+ */
+static int getfactors(int num, int *nfactors, int **factors) {
+    int size;
+    int d;
+    int i;
+    int sqrtnum;
+
+    if(num  < 2) {
+        (*nfactors) = 0;
+        (*factors) = NULL;
+        return MPI_SUCCESS;
+    }
+    /* Allocate the array of prime factors which cannot exceed log_2(num) entries */
+    sqrtnum = ceil(sqrt(num));
+    size = ceil(log(num) / log(2));
+    *factors = (int *) malloc((unsigned) size * sizeof(int));
+
+    i = 0;
+    /* determine all occurrences of factor 2 */
+    while((num % 2) == 0) {
+        num /= 2;
+        (*factors)[i++] = 2;
+    }
+    /* determine all occurrences of uneven prime numbers up to sqrt(num) */
+    d = 3;
+    for(d = 3; (num > 1) && (d <= sqrtnum); d += 2) {
+        while((num % d) == 0) {
+            num /= d;
+            (*factors)[i++] = d;
+        }
+    }
+    /* as we looped only up to sqrt(num) one factor > sqrt(num) may be left over */
+    if(num != 1) {
+        (*factors)[i++] = num;
+    }
+    (*nfactors) = i;
+    return MPI_SUCCESS;
+}
+
+/*
+ * This is a utility function, no need to have anything in the lower
+ * layer for this at all
+ */
+PROTOTYPE ERROR_CLASS dims_create(INT nnodes, INT ndims, INT_OUT dims)
+{
+    int i;
+    int freeprocs;
+    int freedims;
+    int nfactors;
+    int *factors;
+    int *procs;
+    int *p;
+    int err;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (0 > ndims) {
+            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD,
+                                           MPI_ERR_DIMS, FUNC_NAME);
+        }
+
+        if ((0 != ndims) && (NULL == dims)) {
+            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD,
+                                           MPI_ERR_ARG, FUNC_NAME);
+        }
+
+        if (1 > nnodes) {
+            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD,
+                                           MPI_ERR_DIMS, FUNC_NAME);
+        }
+    }
+
+    /* Get # of free-to-be-assigned processes and # of free dimensions */
+    freeprocs = nnodes;
+    freedims = 0;
+    for (i = 0, p = dims; i < ndims; ++i,++p) {
+        if (*p == 0) {
+            ++freedims;
+        } else if ((*p < 0) || ((nnodes % *p) != 0)) {
+            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_DIMS,
+                                           FUNC_NAME);
+        } else {
+            freeprocs /= *p;
+        }
+    }
+
+    if (freedims == 0) {
+       if (freeprocs == 1) {
+          return MPI_SUCCESS;
+       }
+       return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_DIMS,
+                                     FUNC_NAME);
+    }
+
+    if (freeprocs == 1) {
+        for (i = 0; i < ndims; ++i, ++dims) {
+            if (*dims == 0) {
+               *dims = 1;
+            }
+        }
+        return MPI_SUCCESS;
+    }
+    /* Factor the number of free processes */
+    if (MPI_SUCCESS != (err = getfactors(freeprocs, &nfactors, &factors))) {
+       return OMPI_ERRHANDLER_NOHANDLE_INVOKE(err,
+                                     FUNC_NAME);
+    }
+
+    /* Assign free processes to free dimensions */
+    if (MPI_SUCCESS != (err = assignnodes(freedims, nfactors, factors, &procs))) {
+       free(factors);
+       return OMPI_ERRHANDLER_NOHANDLE_INVOKE(err,
+                                     FUNC_NAME);
+    }
+
+    /* Return assignment results */
+    p = procs;
+    for (i = 0; i < ndims; ++i, ++dims) {
+        if (*dims == 0) {
+           *dims = *p++;
+        }
+    }
+
+    free((char *) factors);
+    free((char *) procs);
+
+    /* all done */
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/dist_graph_create.c b/ompi/mpi/c/dist_graph_create.c
deleted file mode 100644
index 22234916a1d..00000000000
--- a/ompi/mpi/c/dist_graph_create.c
+++ /dev/null
@@ -1,107 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2012-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2012-2013 Inria.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/memchecker.h"
-#include "ompi/mca/topo/topo.h"
-#include "ompi/mca/topo/base/base.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Dist_graph_create = PMPI_Dist_graph_create
-#endif
-#define MPI_Dist_graph_create PMPI_Dist_graph_create
-#endif
-
-static const char FUNC_NAME[] = "MPI_Dist_graph_create";
-
-int MPI_Dist_graph_create(MPI_Comm comm_old, int n, const int sources[],
-                          const int degrees[], const int destinations[], const int weights[],
-                          MPI_Info info, int reorder, MPI_Comm * newcomm)
-{
-    mca_topo_base_module_t* topo;
-    int i, j, index, err, comm_size;
-
-    MEMCHECKER(
-         memchecker_comm(comm_old);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm_old)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if (OMPI_COMM_IS_INTER(comm_old)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if (n < 0 || NULL == newcomm) {
-            return OMPI_ERRHANDLER_INVOKE(comm_old, MPI_ERR_ARG, FUNC_NAME);
-        } else if (n > 0 && (NULL == sources || NULL == degrees ||
-                              NULL == destinations || NULL == weights)) {
-            return OMPI_ERRHANDLER_INVOKE(comm_old, MPI_ERR_ARG, FUNC_NAME);
-        }
-        /* Ensure the arrays are full of valid-valued integers */
-        comm_size = ompi_comm_size(comm_old);
-        for( i = index = 0; i < n; ++i ) {
-            if (((sources[i] < 0) && (sources[i] != MPI_PROC_NULL)) || sources[i] >= comm_size) {
-                return OMPI_ERRHANDLER_INVOKE(comm_old, MPI_ERR_ARG,
-                                              FUNC_NAME);
-            } else if (degrees[i] < 0) {
-                return OMPI_ERRHANDLER_INVOKE(comm_old, MPI_ERR_ARG,
-                                              FUNC_NAME);
-            }
-            for( j = 0; j < degrees[i]; ++j ) {
-                if (((destinations[index] < 0) && (destinations[index] != MPI_PROC_NULL)) || destinations[index] >= comm_size) {
-                    return OMPI_ERRHANDLER_INVOKE(comm_old, MPI_ERR_ARG,
-                                                  FUNC_NAME);
-                } else if (MPI_UNWEIGHTED != weights && weights[index] < 0) {
-                    return OMPI_ERRHANDLER_INVOKE(comm_old, MPI_ERR_ARG,
-                                                  FUNC_NAME);
-                }
-                index++;
-            }
-        }
-    }
-
-    /* Ensure there is a topo attached to this communicator */
-    if(OMPI_SUCCESS != (err = mca_topo_base_comm_select(comm_old, NULL,
-                                                        &topo, OMPI_COMM_DIST_GRAPH))) {
-        return OMPI_ERRHANDLER_INVOKE(comm_old, err, FUNC_NAME);
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_create_check(comm_old, &err)) ) {
-        OMPI_ERRHANDLER_RETURN(err, comm_old, err, FUNC_NAME);
-    }
-#endif
-
-    err = topo->topo.dist_graph.dist_graph_create(topo, comm_old, n, sources, degrees,
-                                                  destinations, weights, &(info->super),
-                                                  reorder, newcomm);
-    OMPI_ERRHANDLER_RETURN(err, comm_old, err, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/dist_graph_create.c.in b/ompi/mpi/c/dist_graph_create.c.in
new file mode 100644
index 00000000000..693ffb8ac8d
--- /dev/null
+++ b/ompi/mpi/c/dist_graph_create.c.in
@@ -0,0 +1,100 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2012-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2012-2013 Inria.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/mca/topo/base/base.h"
+
+PROTOTYPE ERROR_CLASS dist_graph_create(COMM comm_old, INT n, INT_ARRAY sources,
+                                        INT_ARRAY degrees, INT_ARRAY destinations, INT_ARRAY weights,
+                                        INFO info, INT reorder, COMM_OUT newcomm)
+{
+    mca_topo_base_module_t* topo;
+    int i, j, index, err, comm_size;
+
+    MEMCHECKER(
+         memchecker_comm(comm_old);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm_old)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if (OMPI_COMM_IS_INTER(comm_old)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if (n < 0 || NULL == newcomm) {
+            return OMPI_ERRHANDLER_INVOKE(comm_old, MPI_ERR_ARG, FUNC_NAME);
+        } else if (n > 0 && (NULL == sources || NULL == degrees ||
+                              NULL == destinations || NULL == weights)) {
+            return OMPI_ERRHANDLER_INVOKE(comm_old, MPI_ERR_ARG, FUNC_NAME);
+        }
+        /* Ensure the arrays are full of valid-valued integers */
+        comm_size = ompi_comm_size(comm_old);
+        for( i = index = 0; i < n; ++i ) {
+            if (((sources[i] < 0) && (sources[i] != MPI_PROC_NULL)) || sources[i] >= comm_size) {
+                return OMPI_ERRHANDLER_INVOKE(comm_old, MPI_ERR_ARG,
+                                              FUNC_NAME);
+            } else if (degrees[i] < 0) {
+                return OMPI_ERRHANDLER_INVOKE(comm_old, MPI_ERR_ARG,
+                                              FUNC_NAME);
+            }
+            for( j = 0; j < degrees[i]; ++j ) {
+                if (((destinations[index] < 0) && (destinations[index] != MPI_PROC_NULL)) || destinations[index] >= comm_size) {
+                    return OMPI_ERRHANDLER_INVOKE(comm_old, MPI_ERR_ARG,
+                                                  FUNC_NAME);
+                } else if (MPI_UNWEIGHTED != weights && weights[index] < 0) {
+                    return OMPI_ERRHANDLER_INVOKE(comm_old, MPI_ERR_ARG,
+                                                  FUNC_NAME);
+                }
+                index++;
+            }
+        }
+    }
+
+    /* Ensure there is a topo attached to this communicator */
+    if(OMPI_SUCCESS != (err = mca_topo_base_comm_select(comm_old, NULL,
+                                                        &topo, OMPI_COMM_DIST_GRAPH))) {
+        return OMPI_ERRHANDLER_INVOKE(comm_old, err, FUNC_NAME);
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_create_check(comm_old, &err)) ) {
+        OMPI_ERRHANDLER_RETURN(err, comm_old, err, FUNC_NAME);
+    }
+#endif
+
+    err = topo->topo.dist_graph.dist_graph_create(topo, comm_old, n, sources, degrees,
+                                                  destinations, weights, &(info->super),
+                                                  reorder, newcomm);
+    OMPI_ERRHANDLER_RETURN(err, comm_old, err, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/dist_graph_create_adjacent.c b/ompi/mpi/c/dist_graph_create_adjacent.c
deleted file mode 100644
index a21070b214d..00000000000
--- a/ompi/mpi/c/dist_graph_create_adjacent.c
+++ /dev/null
@@ -1,119 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2008      The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2011-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013-2014 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2012-2013 Inria.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * Author(s): Torsten Hoefler
- *
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/memchecker.h"
-#include "ompi/mca/topo/topo.h"
-#include "ompi/mca/topo/base/base.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Dist_graph_create_adjacent = PMPI_Dist_graph_create_adjacent
-#endif
-#define MPI_Dist_graph_create_adjacent PMPI_Dist_graph_create_adjacent
-#endif
-
-static const char FUNC_NAME[] = "MPI_Dist_graph_create_adjacent";
-
-
-int MPI_Dist_graph_create_adjacent(MPI_Comm comm_old,
-                                   int indegree, const int sources[],
-                                   const int sourceweights[], int outdegree,
-                                   const int destinations[], const int destweights[],
-                                   MPI_Info info, int reorder,
-                                   MPI_Comm *comm_dist_graph)
-{
-    mca_topo_base_module_t* topo;
-    int i, comm_size, err;
-
-    MEMCHECKER(
-        memchecker_comm(comm_old);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm_old)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if (OMPI_COMM_IS_INTER(comm_old)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if (indegree < 0 || outdegree < 0 || NULL == comm_dist_graph) {
-            return OMPI_ERRHANDLER_INVOKE(comm_old, MPI_ERR_ARG,
-                                          "MPI_Dist_graph_create_adjacent negative degree");
-        } else if ((indegree > 0 &&
-                    (NULL == sources || NULL == sourceweights)) ||
-                   (outdegree > 0 &&
-                    (NULL == destinations || NULL == destweights))) {
-            return OMPI_ERRHANDLER_INVOKE(comm_old, MPI_ERR_ARG, "MPI_Dist_graph_create_adjacent mismatched sources or destinations");
-        }
-        comm_size = ompi_comm_size(comm_old);
-        for (i = 0; i < indegree; ++i) {
-            if (((sources[i] < 0) && (sources[i] != MPI_PROC_NULL)) || sources[i] >= comm_size) {
-                return OMPI_ERRHANDLER_INVOKE(comm_old, MPI_ERR_ARG,
-                                              "MPI_Dist_graph_create_adjacent invalid sources");
-            } else if (MPI_UNWEIGHTED != sourceweights && sourceweights[i] < 0) {
-                return OMPI_ERRHANDLER_INVOKE(comm_old, MPI_ERR_ARG,
-                                              "MPI_Dist_graph_create_adjacent invalid sourceweights");
-            }
-        }
-        for (i = 0; i < outdegree; ++i) {
-            if (((destinations[i] < 0) && (destinations[i] != MPI_PROC_NULL)) || destinations[i] >= comm_size) {
-                return OMPI_ERRHANDLER_INVOKE(comm_old, MPI_ERR_ARG,
-                                              "MPI_Dist_graph_create_adjacent invalid destinations");
-            } else if (MPI_UNWEIGHTED != destweights && destweights[i] < 0) {
-                return OMPI_ERRHANDLER_INVOKE(comm_old, MPI_ERR_ARG,
-                                              "MPI_Dist_graph_create_adjacent invalid destweights");
-            }
-        }
-    }
-
-    /* Ensure there is a topo attached to this communicator */
-    if(OMPI_SUCCESS != (err = mca_topo_base_comm_select(comm_old, NULL,
-                                                        &topo, OMPI_COMM_DIST_GRAPH))) {
-        return OMPI_ERRHANDLER_INVOKE(comm_old, err, FUNC_NAME);
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_create_check(comm_old, &err)) ) {
-        OMPI_ERRHANDLER_RETURN(err, comm_old, err, FUNC_NAME);
-    }
-#endif
-
-    err = topo->topo.dist_graph.dist_graph_create_adjacent(topo, comm_old, indegree,
-                                                           sources, sourceweights, outdegree,
-                                                           destinations, destweights, &(info->super),
-                                                           reorder, comm_dist_graph);
-    OMPI_ERRHANDLER_RETURN(err, comm_old, err, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/dist_graph_create_adjacent.c.in b/ompi/mpi/c/dist_graph_create_adjacent.c.in
new file mode 100644
index 00000000000..3423f7dd607
--- /dev/null
+++ b/ompi/mpi/c/dist_graph_create_adjacent.c.in
@@ -0,0 +1,111 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2008      The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2011-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013-2014 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2012-2013 Inria.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * Author(s): Torsten Hoefler
+ *
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/mca/topo/base/base.h"
+
+PROTOTYPE ERROR_CLASS dist_graph_create_adjacent(COMM comm_old,
+                                                 INT indegree, INT_ARRAY sources,
+                                                 INT_ARRAY sourceweights, INT outdegree,
+                                                 INT_ARRAY destinations, INT_ARRAY destweights,
+                                                 INFO info, INT reorder,
+                                                 COMM_OUT comm_dist_graph)
+{
+    mca_topo_base_module_t* topo;
+    int i, comm_size, err;
+
+    MEMCHECKER(
+        memchecker_comm(comm_old);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm_old)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if (OMPI_COMM_IS_INTER(comm_old)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if (indegree < 0 || outdegree < 0 || NULL == comm_dist_graph) {
+            return OMPI_ERRHANDLER_INVOKE(comm_old, MPI_ERR_ARG,
+                                          "MPI_Dist_graph_create_adjacent negative degree");
+        } else if ((indegree > 0 &&
+                    (NULL == sources || NULL == sourceweights)) ||
+                   (outdegree > 0 &&
+                    (NULL == destinations || NULL == destweights))) {
+            return OMPI_ERRHANDLER_INVOKE(comm_old, MPI_ERR_ARG, "MPI_Dist_graph_create_adjacent mismatched sources or destinations");
+        }
+        comm_size = ompi_comm_size(comm_old);
+        for (i = 0; i < indegree; ++i) {
+            if (((sources[i] < 0) && (sources[i] != MPI_PROC_NULL)) || sources[i] >= comm_size) {
+                return OMPI_ERRHANDLER_INVOKE(comm_old, MPI_ERR_ARG,
+                                              "MPI_Dist_graph_create_adjacent invalid sources");
+            } else if (MPI_UNWEIGHTED != sourceweights && sourceweights[i] < 0) {
+                return OMPI_ERRHANDLER_INVOKE(comm_old, MPI_ERR_ARG,
+                                              "MPI_Dist_graph_create_adjacent invalid sourceweights");
+            }
+        }
+        for (i = 0; i < outdegree; ++i) {
+            if (((destinations[i] < 0) && (destinations[i] != MPI_PROC_NULL)) || destinations[i] >= comm_size) {
+                return OMPI_ERRHANDLER_INVOKE(comm_old, MPI_ERR_ARG,
+                                              "MPI_Dist_graph_create_adjacent invalid destinations");
+            } else if (MPI_UNWEIGHTED != destweights && destweights[i] < 0) {
+                return OMPI_ERRHANDLER_INVOKE(comm_old, MPI_ERR_ARG,
+                                              "MPI_Dist_graph_create_adjacent invalid destweights");
+            }
+        }
+    }
+
+    /* Ensure there is a topo attached to this communicator */
+    if(OMPI_SUCCESS != (err = mca_topo_base_comm_select(comm_old, NULL,
+                                                        &topo, OMPI_COMM_DIST_GRAPH))) {
+        return OMPI_ERRHANDLER_INVOKE(comm_old, err, FUNC_NAME);
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_create_check(comm_old, &err)) ) {
+        OMPI_ERRHANDLER_RETURN(err, comm_old, err, FUNC_NAME);
+    }
+#endif
+
+    err = topo->topo.dist_graph.dist_graph_create_adjacent(topo, comm_old, indegree,
+                                                           sources, sourceweights, outdegree,
+                                                           destinations, destweights, &(info->super),
+                                                           reorder, comm_dist_graph);
+    OMPI_ERRHANDLER_RETURN(err, comm_old, err, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/dist_graph_neighbors.c b/ompi/mpi/c/dist_graph_neighbors.c
deleted file mode 100644
index bbe86ce189b..00000000000
--- a/ompi/mpi/c/dist_graph_neighbors.c
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Copyright (c) 2008      The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2012-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Inria.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/memchecker.h"
-#include "ompi/mca/topo/topo.h"
-#include "ompi/mca/topo/base/base.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Dist_graph_neighbors = PMPI_Dist_graph_neighbors
-#endif
-#define MPI_Dist_graph_neighbors PMPI_Dist_graph_neighbors
-#endif
-
-static const char FUNC_NAME[] = "MPI_Dist_graph_neighbors";
-
-
-int MPI_Dist_graph_neighbors(MPI_Comm comm, int maxindegree,
-                             int sources[], int sourceweights[],
-                             int maxoutdegree, int destinations[],
-                             int destweights[])
-{
-    int err;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if (maxindegree < 0 || maxoutdegree < 0 ||
-                   (maxindegree > 0 &&
-                    (NULL == sources || NULL == sourceweights)) ||
-                   (maxoutdegree > 0 &&
-                    (NULL == destinations || NULL == destweights))) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-    }
-
-    if (!OMPI_COMM_IS_DIST_GRAPH(comm)) {
-        return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_TOPOLOGY,
-                                       FUNC_NAME);
-    }
-
-    err = comm->c_topo->topo.dist_graph.dist_graph_neighbors(comm, maxindegree,
-                                                             sources, sourceweights, maxoutdegree,
-                                                             destinations, destweights);
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/dist_graph_neighbors.c.in b/ompi/mpi/c/dist_graph_neighbors.c.in
new file mode 100644
index 00000000000..29c9427e5b1
--- /dev/null
+++ b/ompi/mpi/c/dist_graph_neighbors.c.in
@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2008      The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2012-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Inria.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/mca/topo/base/base.h"
+
+PROTOTYPE ERROR_CLASS dist_graph_neighbors(COMM comm, INT maxindegree,
+                                           INT_OUT sources, INT_OUT sourceweights,
+                                           INT maxoutdegree, INT_OUT destinations,
+                                           INT_OUT destweights)
+{
+    int err;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if (maxindegree < 0 || maxoutdegree < 0 ||
+                   (maxindegree > 0 &&
+                    (NULL == sources || NULL == sourceweights)) ||
+                   (maxoutdegree > 0 &&
+                    (NULL == destinations || NULL == destweights))) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+    }
+
+    if (!OMPI_COMM_IS_DIST_GRAPH(comm)) {
+        return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_TOPOLOGY,
+                                       FUNC_NAME);
+    }
+
+    err = comm->c_topo->topo.dist_graph.dist_graph_neighbors(comm, maxindegree,
+                                                             sources, sourceweights, maxoutdegree,
+                                                             destinations, destweights);
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/dist_graph_neighbors_count.c b/ompi/mpi/c/dist_graph_neighbors_count.c
deleted file mode 100644
index 3a9f8c75d57..00000000000
--- a/ompi/mpi/c/dist_graph_neighbors_count.c
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Copyright (c) 2008      The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2011-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Inria.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- */
-#include <assert.h>
-#include <stdlib.h>
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/memchecker.h"
-#include "ompi/mca/topo/topo.h"
-#include "ompi/mca/topo/base/base.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Dist_graph_neighbors_count = PMPI_Dist_graph_neighbors_count
-#endif
-#define MPI_Dist_graph_neighbors_count PMPI_Dist_graph_neighbors_count
-#endif
-
-static const char FUNC_NAME[] = "MPI_Dist_graph_neighbors_count";
-
-
-int MPI_Dist_graph_neighbors_count(MPI_Comm comm, int *inneighbors,
-                                   int *outneighbors, int *weighted)
-{
-    int err;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if (NULL == inneighbors || NULL == outneighbors ||
-                   NULL == weighted) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-    }
-
-    if (!OMPI_COMM_IS_DIST_GRAPH(comm)) {
-        return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_TOPOLOGY,
-                                       FUNC_NAME);
-    }
-    err = comm->c_topo->topo.dist_graph.dist_graph_neighbors_count(comm, inneighbors,
-                                                                   outneighbors, weighted);
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/dist_graph_neighbors_count.c.in b/ompi/mpi/c/dist_graph_neighbors_count.c.in
new file mode 100644
index 00000000000..51e3843a337
--- /dev/null
+++ b/ompi/mpi/c/dist_graph_neighbors_count.c.in
@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2008      The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2011-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Inria.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ */
+#include <assert.h>
+#include <stdlib.h>
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/mca/topo/base/base.h"
+
+PROTOTYPE ERROR_CLASS dist_graph_neighbors_count(COMM comm, INT_OUT inneighbors,
+                                                 INT_OUT outneighbors, INT_OUT weighted)
+{
+    int err;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if (NULL == inneighbors || NULL == outneighbors ||
+                   NULL == weighted) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+    }
+
+    if (!OMPI_COMM_IS_DIST_GRAPH(comm)) {
+        return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_TOPOLOGY,
+                                       FUNC_NAME);
+    }
+    err = comm->c_topo->topo.dist_graph.dist_graph_neighbors_count(comm, inneighbors,
+                                                                   outneighbors, weighted);
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/errhandler_c2f.c b/ompi/mpi/c/errhandler_c2f.c
deleted file mode 100644
index f798fb7b561..00000000000
--- a/ompi/mpi/c/errhandler_c2f.c
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mpi/fortran/base/fint_2_int.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Errhandler_c2f = PMPI_Errhandler_c2f
-#endif
-#define MPI_Errhandler_c2f PMPI_Errhandler_c2f
-#endif
-
-static const char FUNC_NAME[] = "MPI_Errhandler_c2f";
-
-
-MPI_Fint MPI_Errhandler_c2f(MPI_Errhandler errhandler)
-{
-  /* Error checking */
-
-  if (MPI_PARAM_CHECK) {
-    OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-	/* mapping an invalid handle to a null handle */
-    if (NULL == errhandler) {
-        return OMPI_INT_2_FINT(-1);
-    }
-  }
-
-
-  return OMPI_INT_2_FINT(errhandler->eh_f_to_c_index);
-}
diff --git a/ompi/mpi/c/errhandler_c2f.c.in b/ompi/mpi/c/errhandler_c2f.c.in
new file mode 100644
index 00000000000..b563a49c333
--- /dev/null
+++ b/ompi/mpi/c/errhandler_c2f.c.in
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+
+PROTOTYPE FINT errhandler_c2f(ERRHANDLER errhandler)
+{
+  /* Error checking */
+
+  if (MPI_PARAM_CHECK) {
+	/* mapping an invalid handle to a null handle */
+    if (NULL == errhandler) {
+        return OMPI_INT_2_FINT(-1);
+    }
+  }
+
+
+  return OMPI_INT_2_FINT(errhandler->eh_f_to_c_index);
+}
diff --git a/ompi/mpi/c/errhandler_f2c.c b/ompi/mpi/c/errhandler_f2c.c
deleted file mode 100644
index bf4dce1994f..00000000000
--- a/ompi/mpi/c/errhandler_f2c.c
+++ /dev/null
@@ -1,62 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2007 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mpi/fortran/base/fint_2_int.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Errhandler_f2c = PMPI_Errhandler_f2c
-#endif
-#define MPI_Errhandler_f2c PMPI_Errhandler_f2c
-#endif
-
-static const char FUNC_NAME[] = "MPI_Errhandler_f2c";
-
-
-MPI_Errhandler MPI_Errhandler_f2c(MPI_Fint errhandler_f)
-{
-    int eh_index = OMPI_FINT_2_INT(errhandler_f);
-
-    /* Error checking */
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-    }
-
-    /* Per MPI-2:4.12.4, do not invoke an error handler if we get an
-       invalid fortran handle.  If we get an invalid fortran handle,
-       return an invalid C handle. */
-
-    if (eh_index < 0 ||
-        eh_index >=
-        opal_pointer_array_get_size(&ompi_errhandler_f_to_c_table)) {
-        return NULL;
-    }
-
-    return (MPI_Errhandler)opal_pointer_array_get_item(&ompi_errhandler_f_to_c_table,
-                                                       eh_index);
-}
diff --git a/ompi/mpi/c/errhandler_f2c.c.in b/ompi/mpi/c/errhandler_f2c.c.in
new file mode 100644
index 00000000000..464ad3cf337
--- /dev/null
+++ b/ompi/mpi/c/errhandler_f2c.c.in
@@ -0,0 +1,72 @@
+/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2020-2024 Triad National Security, LLC.  
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+
+PROTOTYPE ERRHANDLER errhandler_f2c(FINT errhandler_f)
+{
+    int eh_index = OMPI_FINT_2_INT(errhandler_f);
+    MPI_Errhandler c_err_handler;
+
+    /* Per MPI-2:4.12.4, do not invoke an error handler if we get an
+       invalid fortran handle.  If we get an invalid fortran handle,
+       return an invalid C handle. */
+
+    /*
+     * special cases for MPI_ERRORS_ARE_FATAL and MPI_ERRORS_RETURN -
+     * needed for MPI 4.0
+     */
+
+    switch(eh_index) {
+    case  OMPI_ERRHANDLER_NULL_FORTRAN:
+        c_err_handler = MPI_ERRHANDLER_NULL;
+        break;
+    case  OMPI_ERRORS_ARE_FATAL_FORTRAN:
+        c_err_handler = MPI_ERRORS_ARE_FATAL;
+        break;
+    case  OMPI_ERRORS_RETURN_FORTRAN:
+        c_err_handler = MPI_ERRORS_RETURN;
+        break;
+    default:
+        if (MPI_PARAM_CHECK) {
+            OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        }
+        if (eh_index < 0 ||
+            eh_index >=
+                 opal_pointer_array_get_size(&ompi_errhandler_f_to_c_table)) {
+                c_err_handler = NULL;
+        } else {
+            c_err_handler = (MPI_Errhandler)opal_pointer_array_get_item(&ompi_errhandler_f_to_c_table,
+                                                                           eh_index);
+        }
+        break;
+    }
+
+    return c_err_handler;
+}
diff --git a/ompi/mpi/c/errhandler_free.c b/ompi/mpi/c/errhandler_free.c
deleted file mode 100644
index a87038f4707..00000000000
--- a/ompi/mpi/c/errhandler_free.c
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Errhandler_free = PMPI_Errhandler_free
-#endif
-#define MPI_Errhandler_free PMPI_Errhandler_free
-#endif
-
-static const char FUNC_NAME[] = "MPI_Errhandler_free";
-
-
-int MPI_Errhandler_free(MPI_Errhandler *errhandler)
-{
-  /* Error checking */
-
-  if (MPI_PARAM_CHECK) {
-    OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-    /* Raise an MPI error if we got NULL or if we got an intrinsic
-       *and* the reference count is 1 (meaning that this FREE would
-       actually free the underlying intrinsic object).  This is ugly
-       but necessary -- see below. */
-    if (NULL == errhandler ||
-        (ompi_errhandler_is_intrinsic(*errhandler) &&
-         1 == (*errhandler)->super.obj_reference_count)) {
-      return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                   "MPI_Errhandler_free");
-    }
-  }
-
-  /* Return the errhandler.  According to MPI-2 errata, any errhandler
-     obtained by MPI_*_GET_ERRHANDLER or MPI_ERRHANDLER_GET must also
-     be freed by MPI_ERRHANDLER_FREE (including intrinsic error
-     handlers).  For example, this is valid:
-
-     int main() {
-         MPI_Errhandler errhdl;
-         MPI_Init(NULL, NULL);
-         MPI_Comm_get_errhandler(MPI_COMM_WORLD, &errhdl);
-         MPI_Errhandler_free(&errhdl);
-         MPI_Finalize();
-         return 0;
-     }
-
-     So decrease the refcount here. */
-
-  OBJ_RELEASE(*errhandler);
-  *errhandler = MPI_ERRHANDLER_NULL;
-
-  /* All done */
-
-  return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/errhandler_free.c.in b/ompi/mpi/c/errhandler_free.c.in
new file mode 100644
index 00000000000..8f888f1965e
--- /dev/null
+++ b/ompi/mpi/c/errhandler_free.c.in
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+
+PROTOTYPE ERROR_CLASS errhandler_free(ERRHANDLER_OUT errhandler)
+{
+  /* Error checking */
+
+  if (MPI_PARAM_CHECK) {
+    /* Raise an MPI exception if we got NULL or if we got an intrinsic
+       *and* the reference count is 1 (meaning that this FREE would
+       actually free the underlying intrinsic object).  This is ugly
+       but necessary -- see below. */
+    if (NULL == errhandler ||
+        (ompi_errhandler_is_intrinsic(*errhandler) &&
+         1 == (*errhandler)->super.obj_reference_count)) {
+      return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                   "MPI_Errhandler_free");
+    }
+  }
+
+  /* Return the errhandler.  According to MPI-2 errata, any errhandler
+     obtained by MPI_*_GET_ERRHANDLER or MPI_ERRHANDLER_GET must also
+     be freed by MPI_ERRHANDLER_FREE (including intrinsic error
+     handlers).  For example, this is valid:
+
+     int main() {
+         MPI_Errhandler errhdl;
+         MPI_Init(NULL, NULL);
+         MPI_Comm_get_errhandler(MPI_COMM_WORLD, &errhdl);
+         MPI_Errhandler_free(&errhdl);
+         MPI_Finalize();
+         return 0;
+     }
+
+     So decrease the refcount here. */
+
+  ompi_errhandler_free (*errhandler);
+  *errhandler = MPI_ERRHANDLER_NULL;
+
+  /* All done */
+
+  return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/error_class.c b/ompi/mpi/c/error_class.c
deleted file mode 100644
index b8d4ac7db0e..00000000000
--- a/ompi/mpi/c/error_class.c
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/errhandler/errcode.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Error_class = PMPI_Error_class
-#endif
-#define MPI_Error_class PMPI_Error_class
-#endif
-
-static const char FUNC_NAME[] = "MPI_Error_class";
-
-
-int MPI_Error_class(int errorcode, int *errorclass)
-{
-    if ( MPI_PARAM_CHECK ) {
-        if ( ompi_mpi_errcode_is_invalid(errorcode)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                                   FUNC_NAME);
-        }
-    }
-
-
-    *errorclass = ompi_mpi_errcode_get_class(errorcode);
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/error_class.c.in b/ompi/mpi/c/error_class.c.in
new file mode 100644
index 00000000000..28a3116126b
--- /dev/null
+++ b/ompi/mpi/c/error_class.c.in
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/errhandler/errcode.h"
+
+PROTOTYPE ERROR_CLASS error_class(INT errorcode, INT_OUT errorclass)
+{
+    int ret;
+
+    /* make sure the infrastructure is initialized */
+    ret = ompi_mpi_instance_retain ();
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(ret, FUNC_NAME);
+    }
+
+    if ( MPI_PARAM_CHECK ) {
+        if ( ompi_mpi_errcode_is_invalid(errorcode)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                                   FUNC_NAME);
+        }
+    }
+
+
+    *errorclass = ompi_mpi_errcode_get_class(errorcode);
+    ompi_mpi_instance_release ();
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/error_string.c b/ompi/mpi/c/error_string.c
deleted file mode 100644
index 2493f5be637..00000000000
--- a/ompi/mpi/c/error_string.c
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      University of Houston. All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2018      Cisco Systems, Inc.  All rights reserved
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <string.h>
-
-#include "opal/util/string_copy.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/errhandler/errcode.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Error_string = PMPI_Error_string
-#endif
-#define MPI_Error_string PMPI_Error_string
-#endif
-
-static const char FUNC_NAME[] = "MPI_Error_string";
-
-
-int MPI_Error_string(int errorcode, char *string, int *resultlen)
-{
-    char *tmpstring;
-
-    if ( MPI_PARAM_CHECK ) {
-        if ( ompi_mpi_errcode_is_invalid(errorcode)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                                   FUNC_NAME);
-        }
-    }
-
-    tmpstring = ompi_mpi_errnum_get_string (errorcode);
-    opal_string_copy(string, tmpstring, MPI_MAX_ERROR_STRING);
-    *resultlen = (int)strlen(string);
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/error_string.c.in b/ompi/mpi/c/error_string.c.in
new file mode 100644
index 00000000000..0b3bf36edbe
--- /dev/null
+++ b/ompi/mpi/c/error_string.c.in
@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      University of Houston. All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018      Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2022-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <string.h>
+
+#include "opal/util/string_copy.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/errhandler/errcode.h"
+
+PROTOTYPE ERROR_CLASS error_string(INT errorcode, STRING_OUT string,
+                                   INT_OUT resultlen)
+{
+    int ret;
+    char *tmpstring;
+
+    /* make sure the infrastructure is initialized */
+    ret = ompi_mpi_instance_retain ();
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(ret,
+                                               FUNC_NAME);
+    }
+
+    if ( MPI_PARAM_CHECK ) {
+        if ( ompi_mpi_errcode_is_invalid(errorcode)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                                   FUNC_NAME);
+        }
+    }
+
+    tmpstring = ompi_mpi_errnum_get_string (errorcode);
+    opal_string_copy(string, tmpstring, MPI_MAX_ERROR_STRING);
+    *resultlen = (int)strlen(string);
+
+    ompi_mpi_instance_release();
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/exscan.c b/ompi/mpi/c/exscan.c
deleted file mode 100644
index 812812a97b6..00000000000
--- a/ompi/mpi/c/exscan.c
+++ /dev/null
@@ -1,114 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/op/op.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Exscan = PMPI_Exscan
-#endif
-#define MPI_Exscan PMPI_Exscan
-#endif
-
-static const char FUNC_NAME[] = "MPI_Exscan";
-
-
-int MPI_Exscan(const void *sendbuf, void *recvbuf, int count,
-               MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
-{
-    int err;
-
-    SPC_RECORD(OMPI_SPC_EXSCAN, 1);
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        if (MPI_IN_PLACE != sendbuf) {
-            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
-        } else {
-            memchecker_call(&opal_memchecker_base_isdefined, recvbuf, count, datatype);
-        }
-        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, count, datatype);
-        memchecker_comm(comm);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        char *msg;
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-
-        /* Unrooted operation -- same checks for intracommunicators
-           and intercommunicators */
-        else if (MPI_OP_NULL == op) {
-            err = MPI_ERR_OP;
-        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
-            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
-            free(msg);
-            return ret;
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
-        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-    }
-#endif
-
-    /* Do we need to do anything? (MPI says that reductions have to
-       have a count of at least 1, but at least IMB calls reduce with
-       a count of 0 -- blah!) */
-
-    if (0 == count) {
-        return MPI_SUCCESS;
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-
-    OBJ_RETAIN(op);
-    err = comm->c_coll->coll_exscan(sendbuf, recvbuf, count,
-                                   datatype, op, comm,
-                                   comm->c_coll->coll_exscan_module);
-    OBJ_RELEASE(op);
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/exscan.c.in b/ompi/mpi/c/exscan.c.in
new file mode 100644
index 00000000000..0268810658c
--- /dev/null
+++ b/ompi/mpi/c/exscan.c.in
@@ -0,0 +1,106 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2018 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/op/op.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS exscan(BUFFER sendbuf, BUFFER_OUT recvbuf, COUNT count,
+                             DATATYPE datatype, OP op, COMM comm)
+{
+    int err;
+
+    SPC_RECORD(OMPI_SPC_EXSCAN, 1);
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        if (MPI_IN_PLACE != sendbuf) {
+            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
+        } else {
+            memchecker_call(&opal_memchecker_base_isdefined, recvbuf, count, datatype);
+        }
+        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, count, datatype);
+        memchecker_comm(comm);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        char *msg;
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+
+        /* Unrooted operation -- same checks for intracommunicators
+           and intercommunicators */
+        else if (MPI_OP_NULL == op) {
+            err = MPI_ERR_OP;
+        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
+            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
+            free(msg);
+            return ret;
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
+        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+    }
+#endif
+
+    /* Do we need to do anything? (MPI says that reductions have to
+       have a count of at least 1, but at least IMB calls reduce with
+       a count of 0 -- blah!) */
+
+    if (0 == count) {
+        return MPI_SUCCESS;
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+
+    OBJ_RETAIN(op);
+    err = comm->c_coll->coll_exscan(sendbuf, recvbuf, count,
+                                   datatype, op, comm,
+                                   comm->c_coll->coll_exscan_module);
+    OBJ_RELEASE(op);
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/exscan_init.c.in b/ompi/mpi/c/exscan_init.c.in
new file mode 100644
index 00000000000..7f155d78fe0
--- /dev/null
+++ b/ompi/mpi/c/exscan_init.c.in
@@ -0,0 +1,84 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/op/op.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS exscan_init(BUFFER sendbuf, BUFFER_OUT recvbuf, COUNT count,
+                                  DATATYPE datatype, OP op, COMM comm,
+                                  INFO info, REQUEST_INOUT request)
+{
+    int err;
+
+    SPC_RECORD(OMPI_SPC_EXSCAN_INIT, 1);
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
+        memchecker_comm(comm);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        char *msg;
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+
+        /* Unrooted operation -- same checks for intracommunicators
+           and intercommunicators */
+        else if (MPI_OP_NULL == op) {
+            err = MPI_ERR_OP;
+        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
+            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
+            free(msg);
+            return ret;
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+
+    err = comm->c_coll->coll_exscan_init(sendbuf, recvbuf, count,
+                                         datatype, op, comm, info, request,
+                                         comm->c_coll->coll_exscan_init_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_op(*request, op, datatype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/fetch_and_op.c b/ompi/mpi/c/fetch_and_op.c
deleted file mode 100644
index 47d61e403d2..00000000000
--- a/ompi/mpi/c/fetch_and_op.c
+++ /dev/null
@@ -1,74 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
- * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-#include "ompi/mca/osc/osc.h"
-#include "ompi/datatype/ompi_datatype.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Fetch_and_op = PMPI_Fetch_and_op
-#endif
-#define MPI_Fetch_and_op PMPI_Fetch_and_op
-#endif
-
-static const char FUNC_NAME[] = "MPI_Fetch_and_op";
-
-
-int MPI_Fetch_and_op(const void *origin_addr, void *result_addr, MPI_Datatype datatype,
-                     int target_rank, MPI_Aint target_disp, MPI_Op op, MPI_Win win)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        rc = OMPI_SUCCESS;
-
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        } else if (ompi_win_peer_invalid(win, target_rank) &&
-                   (MPI_PROC_NULL != target_rank)) {
-            rc = MPI_ERR_RANK;
-        } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) {
-            rc = MPI_ERR_DISP;
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, datatype, 1);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME);
-    }
-
-    if (MPI_PROC_NULL == target_rank) return MPI_SUCCESS;
-
-    rc = win->w_osc_module->osc_fetch_and_op(origin_addr, result_addr, datatype,
-                                             target_rank, target_disp, op, win);
-    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/fetch_and_op.c.in b/ompi/mpi/c/fetch_and_op.c.in
new file mode 100644
index 00000000000..5d26f2ca187
--- /dev/null
+++ b/ompi/mpi/c/fetch_and_op.c.in
@@ -0,0 +1,66 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+#include "ompi/mca/osc/osc.h"
+#include "ompi/datatype/ompi_datatype.h"
+
+PROTOTYPE ERROR_CLASS fetch_and_op(BUFFER origin_addr, BUFFER_OUT result_addr, DATATYPE datatype,
+                                   INT target_rank, AINT target_disp, OP op, WIN win)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        rc = OMPI_SUCCESS;
+
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        } else if (ompi_win_peer_invalid(win, target_rank) &&
+                   (MPI_PROC_NULL != target_rank)) {
+            rc = MPI_ERR_RANK;
+        } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) {
+            rc = MPI_ERR_DISP;
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, datatype, 1);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME);
+    }
+
+    if (MPI_PROC_NULL == target_rank) return MPI_SUCCESS;
+
+    rc = win->w_osc_module->osc_fetch_and_op(origin_addr, result_addr, datatype,
+                                             target_rank, target_disp, op, win);
+    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_c2f.c b/ompi/mpi/c/file_c2f.c
deleted file mode 100644
index 3ba4d40c5c9..00000000000
--- a/ompi/mpi/c/file_c2f.c
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mpi/fortran/base/fint_2_int.h"
-#include "ompi/file/file.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_c2f = PMPI_File_c2f
-#endif
-#define MPI_File_c2f PMPI_File_c2f
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_c2f";
-
-
-MPI_Fint MPI_File_c2f(MPI_File file)
-{
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        /* Note that ompi_file_invalid() explicitly checks for
-           MPI_FILE_NULL, but MPI_FILE_C2F is supposed to treat
-           MPI_FILE_NULL as a valid file (and therefore return a valid
-           Fortran handle for it).  Hence, this function should not
-           return an error if MPI_FILE_NULL is passed in.
-
-           See a big comment in ompi/communicator/communicator.h about
-           this. */
-        if (ompi_file_invalid(file) && MPI_FILE_NULL != file) {
-            return OMPI_INT_2_FINT(-1);
-        }
-    }
-
-    return OMPI_INT_2_FINT(file->f_f_to_c_index);
-}
diff --git a/ompi/mpi/c/file_c2f.c.in b/ompi/mpi/c/file_c2f.c.in
new file mode 100644
index 00000000000..509c0216aac
--- /dev/null
+++ b/ompi/mpi/c/file_c2f.c.in
@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+#include "ompi/file/file.h"
+
+PROTOTYPE FINT file_c2f(FILE file)
+{
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        /* Note that ompi_file_invalid() explicitly checks for
+           MPI_FILE_NULL, but MPI_FILE_C2F is supposed to treat
+           MPI_FILE_NULL as a valid file (and therefore return a valid
+           Fortran handle for it).  Hence, this function should not
+           return an error if MPI_FILE_NULL is passed in.
+
+           See a big comment in ompi/communicator/communicator.h about
+           this. */
+        if (ompi_file_invalid(file) && MPI_FILE_NULL != file) {
+            return OMPI_INT_2_FINT(-1);
+        }
+    }
+
+    return OMPI_INT_2_FINT(file->f_f_to_c_index);
+}
diff --git a/ompi/mpi/c/file_call_errhandler.c b/ompi/mpi/c/file_call_errhandler.c
deleted file mode 100644
index 64f2336f7de..00000000000
--- a/ompi/mpi/c/file_call_errhandler.c
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_call_errhandler = PMPI_File_call_errhandler
-#endif
-#define MPI_File_call_errhandler PMPI_File_call_errhandler
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_call_errhandler";
-
-
-int MPI_File_call_errhandler(MPI_File fh, int errorcode)
-{
-  /* Error checking */
-
-  if (MPI_PARAM_CHECK) {
-    OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-    if (NULL == fh ||
-        MPI_FILE_NULL == fh) {
-      return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-    }
-  }
-
-  /* Invoke the errhandler */
-  OMPI_ERRHANDLER_INVOKE(fh, errorcode, FUNC_NAME);
-
-  /* See MPI-2 8.5 why this function has to return MPI_SUCCESS */
-  return MPI_SUCCESS;
-
-}
diff --git a/ompi/mpi/c/file_call_errhandler.c.in b/ompi/mpi/c/file_call_errhandler.c.in
new file mode 100644
index 00000000000..830e3eefc6d
--- /dev/null
+++ b/ompi/mpi/c/file_call_errhandler.c.in
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+
+PROTOTYPE ERROR_CLASS file_call_errhandler(FILE fh, INT errorcode)
+{
+  /* Error checking */
+
+  if (MPI_PARAM_CHECK) {
+    OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+    if (NULL == fh ||
+        MPI_FILE_NULL == fh) {
+      return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+    }
+  }
+
+  /* Invoke the errhandler */
+  OMPI_ERRHANDLER_INVOKE(fh, errorcode, FUNC_NAME);
+
+  /* See MPI-2 8.5 why this function has to return MPI_SUCCESS */
+  return MPI_SUCCESS;
+
+}
diff --git a/ompi/mpi/c/file_close.c b/ompi/mpi/c/file_close.c
deleted file mode 100644
index ce11ba11bd2..00000000000
--- a/ompi/mpi/c/file_close.c
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_close = PMPI_File_close
-#endif
-#define MPI_File_close PMPI_File_close
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_close";
-
-
-int MPI_File_close(MPI_File *fh)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        /* Note that MPI-2:9.7 (p265) says that errors in
-           MPI_FILE_CLOSE should invoke the default error handler on
-           MPI_FILE_NULL */
-
-        if (NULL == fh || ompi_file_invalid(*fh)) {
-            return OMPI_ERRHANDLER_INVOKE(MPI_FILE_NULL, MPI_ERR_FILE,
-                                          FUNC_NAME);
-        }
-    }
-
-    /* Release the MPI_File; the destructor releases the component,
-       zeroes out fiels, etc. */
-
-    rc = ompi_file_close(fh);
-    OMPI_ERRHANDLER_RETURN(rc, *fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_close.c.in b/ompi/mpi/c/file_close.c.in
new file mode 100644
index 00000000000..4111c6a3026
--- /dev/null
+++ b/ompi/mpi/c/file_close.c.in
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+
+PROTOTYPE ERROR_CLASS file_close(FILE_OUT fh)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        /* Note that MPI-2:9.7 (p265) says that errors in
+           MPI_FILE_CLOSE should invoke the default error handler on
+           MPI_FILE_NULL */
+
+        if (NULL == fh || ompi_file_invalid(*fh)) {
+            return OMPI_ERRHANDLER_INVOKE(MPI_FILE_NULL, MPI_ERR_FILE,
+                                          FUNC_NAME);
+        }
+    }
+
+    /* Release the MPI_File; the destructor releases the component,
+       zeroes out fields, etc. */
+
+    rc = ompi_file_close(fh);
+    OMPI_ERRHANDLER_RETURN(rc, *fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_create_errhandler.c b/ompi/mpi/c/file_create_errhandler.c
deleted file mode 100644
index a839ec3a9fa..00000000000
--- a/ompi/mpi/c/file_create_errhandler.c
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008-2009 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_create_errhandler = PMPI_File_create_errhandler
-#endif
-#define MPI_File_create_errhandler PMPI_File_create_errhandler
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_create_errhandler";
-
-
-int MPI_File_create_errhandler(MPI_File_errhandler_function *function,
-		                        MPI_Errhandler *errhandler) {
-  int err = MPI_SUCCESS;
-
-  /* Error checking */
-
-  if (MPI_PARAM_CHECK) {
-    OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-    if (NULL == function ||
-        NULL == errhandler) {
-      return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                   "MPI_File_create_errhandler");
-    }
-  }
-
-  /* Create and cache the errhandler.  Sets a refcount of 1. */
-
-  *errhandler =
-    ompi_errhandler_create(OMPI_ERRHANDLER_TYPE_FILE,
-                           (ompi_errhandler_generic_handler_fn_t*) function,
-                           OMPI_ERRHANDLER_LANG_C);
-  if (NULL == *errhandler) {
-    err = MPI_ERR_INTERN;
-  }
-
-  OMPI_ERRHANDLER_NOHANDLE_RETURN(err, MPI_ERR_INTERN,
-                        "MPI_File_create_errhandler");
-}
diff --git a/ompi/mpi/c/file_create_errhandler.c.in b/ompi/mpi/c/file_create_errhandler.c.in
new file mode 100644
index 00000000000..f5040e98a12
--- /dev/null
+++ b/ompi/mpi/c/file_create_errhandler.c.in
@@ -0,0 +1,61 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008-2009 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+
+PROTOTYPE ERROR_CLASS file_create_errhandler (FILE_ERRHANDLER_FUNCTION function,
+                                              ERRHANDLER_OUT errhandler)
+{
+    int err = MPI_SUCCESS;
+
+    /* Error checking */
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == function ||
+            NULL == errhandler) {
+          return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                   "MPI_File_create_errhandler");
+        }
+    }
+
+    /* Create and cache the errhandler.  Sets a refcount of 1. */
+
+    *errhandler =
+        ompi_errhandler_create(OMPI_ERRHANDLER_TYPE_FILE,
+                               (ompi_errhandler_generic_handler_fn_t*) function,
+                               OMPI_ERRHANDLER_LANG_C);
+    if (NULL == *errhandler) {
+        err = MPI_ERR_INTERN;
+    }
+
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, MPI_ERR_INTERN,
+                          "MPI_File_create_errhandler");
+}
diff --git a/ompi/mpi/c/file_delete.c b/ompi/mpi/c/file_delete.c
deleted file mode 100644
index d666544b880..00000000000
--- a/ompi/mpi/c/file_delete.c
+++ /dev/null
@@ -1,82 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-#include "ompi/file/file.h"
-#include "ompi/mca/io/io.h"
-#include "ompi/mca/io/base/base.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_delete = PMPI_File_delete
-#endif
-#define MPI_File_delete PMPI_File_delete
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_delete";
-
-
-int MPI_File_delete(const char *filename, MPI_Info info)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == info || ompi_info_is_freed(info)) {
-            rc = MPI_ERR_INFO;
-        } else if (NULL == filename) {
-            rc = MPI_ERR_ARG;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, MPI_FILE_NULL, rc, FUNC_NAME);
-    }
-
-    /* Note that MPI-2:9.7 (p265 in the ps; 261 in the pdf) says that
-       errors in MPI_FILE_OPEN (before the file handle is created)
-       should invoke the default error handler on MPI_FILE_NULL.
-       Hence, if we get a file handle out of ompi_file_open(), invoke
-       the error handler on that.  If not, invoke the error handler on
-       MPI_FILE_NULL. */
-
-    /* The io framework is only initialized lazily.  If it hasn't
-       already been initialized, do so now (note that MPI_FILE_OPEN
-       and MPI_FILE_DELETE are the only two places that it will be
-       initialized). We might want to add a check to see if the
-       framework is open instead of just incrementing the open count. */
-
-    if (OMPI_SUCCESS != (rc = mca_base_framework_open(&ompi_io_base_framework, 0))) {
-        return OMPI_ERRHANDLER_INVOKE(MPI_FILE_NULL, rc, FUNC_NAME);
-    }
-
-    /* Since there is no MPI_File handle associated with this
-       function, the MCA has to do a selection and perform the
-       action */
-    rc = mca_io_base_delete(filename, &(info->super));
-    OMPI_ERRHANDLER_RETURN(rc, MPI_FILE_NULL, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_delete.c.in b/ompi/mpi/c/file_delete.c.in
new file mode 100644
index 00000000000..f85442b50c4
--- /dev/null
+++ b/ompi/mpi/c/file_delete.c.in
@@ -0,0 +1,74 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include "ompi/file/file.h"
+#include "ompi/mca/io/io.h"
+#include "ompi/mca/io/base/base.h"
+
+PROTOTYPE ERROR_CLASS file_delete(STRING filename, INFO info)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == info || ompi_info_is_freed(info)) {
+            rc = MPI_ERR_INFO;
+        } else if (NULL == filename) {
+            rc = MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, MPI_FILE_NULL, rc, FUNC_NAME);
+    }
+
+    /* Note that MPI-2:9.7 (p265 in the ps; 261 in the pdf) says that
+       errors in MPI_FILE_OPEN (before the file handle is created)
+       should invoke the default error handler on MPI_FILE_NULL.
+       Hence, if we get a file handle out of ompi_file_open(), invoke
+       the error handler on that.  If not, invoke the error handler on
+       MPI_FILE_NULL. */
+
+    /* The io framework is only initialized lazily.  If it hasn't
+       already been initialized, do so now (note that MPI_FILE_OPEN
+       and MPI_FILE_DELETE are the only two places that it will be
+       initialized). We might want to add a check to see if the
+       framework is open instead of just incrementing the open count. */
+
+    if (OMPI_SUCCESS != (rc = mca_base_framework_open(&ompi_io_base_framework, 0))) {
+        return OMPI_ERRHANDLER_INVOKE(MPI_FILE_NULL, rc, FUNC_NAME);
+    }
+
+    /* Since there is no MPI_File handle associated with this
+       function, the MCA has to do a selection and perform the
+       action */
+    rc = mca_io_base_delete(filename, &(info->super));
+    OMPI_ERRHANDLER_RETURN(rc, MPI_FILE_NULL, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_f2c.c b/ompi/mpi/c/file_f2c.c
deleted file mode 100644
index 3640a2a9ae5..00000000000
--- a/ompi/mpi/c/file_f2c.c
+++ /dev/null
@@ -1,60 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2007 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mpi/fortran/base/fint_2_int.h"
-#include "ompi/file/file.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_f2c = PMPI_File_f2c
-#endif
-#define MPI_File_f2c PMPI_File_f2c
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_f2c";
-
-
-MPI_File MPI_File_f2c(MPI_Fint file_f)
-{
-    int file_index = OMPI_FINT_2_INT(file_f);
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-    }
-
-    /* Per MPI-2:4.12.4, do not invoke an error handler if we get an
-       invalid fortran handle.  If we get an invalid fortran handle,
-       return an invalid C handle. */
-
-    if (file_index < 0 ||
-        file_index >=
-        opal_pointer_array_get_size(&ompi_file_f_to_c_table)) {
-        return NULL;
-    }
-
-    return (MPI_File)opal_pointer_array_get_item(&ompi_file_f_to_c_table, file_index);
-}
diff --git a/ompi/mpi/c/file_f2c.c.in b/ompi/mpi/c/file_f2c.c.in
new file mode 100644
index 00000000000..283f1ff497d
--- /dev/null
+++ b/ompi/mpi/c/file_f2c.c.in
@@ -0,0 +1,52 @@
+/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+#include "ompi/file/file.h"
+
+PROTOTYPE FILE file_f2c(FINT file_f)
+{
+    int file_index = OMPI_FINT_2_INT(file_f);
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+    }
+
+    /* Per MPI-2:4.12.4, do not invoke an error handler if we get an
+       invalid fortran handle.  If we get an invalid fortran handle,
+       return an invalid C handle. */
+
+    if (file_index < 0 ||
+        file_index >=
+        opal_pointer_array_get_size(&ompi_file_f_to_c_table)) {
+        return NULL;
+    }
+
+    return (MPI_File)opal_pointer_array_get_item(&ompi_file_f_to_c_table, file_index);
+}
diff --git a/ompi/mpi/c/file_get_amode.c b/ompi/mpi/c/file_get_amode.c
deleted file mode 100644
index f000948862a..00000000000
--- a/ompi/mpi/c/file_get_amode.c
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_get_amode = PMPI_File_get_amode
-#endif
-#define MPI_File_get_amode PMPI_File_get_amode
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_get_amode";
-
-
-int MPI_File_get_amode(MPI_File fh, int *amode)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            rc = MPI_ERR_FILE;
-            fh = MPI_FILE_NULL;
-        } else if (NULL == amode) {
-            rc = MPI_ERR_ARG;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-            io_module_file_get_amode(fh, amode);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_get_amode.c.in b/ompi/mpi/c/file_get_amode.c.in
new file mode 100644
index 00000000000..8fac4fbb98d
--- /dev/null
+++ b/ompi/mpi/c/file_get_amode.c.in
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+
+PROTOTYPE ERROR_CLASS file_get_amode(FILE fh, INT_OUT amode)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            rc = MPI_ERR_FILE;
+            fh = MPI_FILE_NULL;
+        } else if (NULL == amode) {
+            rc = MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+            io_module_file_get_amode(fh, amode);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_get_atomicity.c b/ompi/mpi/c/file_get_atomicity.c
deleted file mode 100644
index e1424643c03..00000000000
--- a/ompi/mpi/c/file_get_atomicity.c
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_get_atomicity = PMPI_File_get_atomicity
-#endif
-#define MPI_File_get_atomicity PMPI_File_get_atomicity
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_get_atomicity";
-
-
-int MPI_File_get_atomicity(MPI_File fh, int *flag)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            rc = MPI_ERR_FILE;
-            fh = MPI_FILE_NULL;
-        } else if (NULL == flag) {
-            rc = MPI_ERR_ARG;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-            io_module_file_get_atomicity(fh, flag);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_get_atomicity.c.in b/ompi/mpi/c/file_get_atomicity.c.in
new file mode 100644
index 00000000000..904e7875b18
--- /dev/null
+++ b/ompi/mpi/c/file_get_atomicity.c.in
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+
+PROTOTYPE ERROR_CLASS file_get_atomicity(FILE fh, INT_OUT flag)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            rc = MPI_ERR_FILE;
+            fh = MPI_FILE_NULL;
+        } else if (NULL == flag) {
+            rc = MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+            io_module_file_get_atomicity(fh, flag);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_get_byte_offset.c b/ompi/mpi/c/file_get_byte_offset.c
deleted file mode 100644
index 92ab09d4523..00000000000
--- a/ompi/mpi/c/file_get_byte_offset.c
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_get_byte_offset = PMPI_File_get_byte_offset
-#endif
-#define MPI_File_get_byte_offset PMPI_File_get_byte_offset
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_get_byte_offset";
-
-
-int MPI_File_get_byte_offset(MPI_File fh, MPI_Offset offset,
-                             MPI_Offset *disp)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            rc = MPI_ERR_FILE;
-            fh = MPI_FILE_NULL;
-        } else if (NULL == disp) {
-            rc = MPI_ERR_ARG;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-            io_module_file_get_byte_offset(fh, offset, disp);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_get_byte_offset.c.in b/ompi/mpi/c/file_get_byte_offset.c.in
new file mode 100644
index 00000000000..c86572ee5de
--- /dev/null
+++ b/ompi/mpi/c/file_get_byte_offset.c.in
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+
+PROTOTYPE ERROR_CLASS file_get_byte_offset(FILE fh, OFFSET offset,
+                                           OFFSET_OUT disp)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            rc = MPI_ERR_FILE;
+            fh = MPI_FILE_NULL;
+        } else if (NULL == disp) {
+            rc = MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+            io_module_file_get_byte_offset(fh, offset, disp);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_get_errhandler.c b/ompi/mpi/c/file_get_errhandler.c
deleted file mode 100644
index b78de9ce292..00000000000
--- a/ompi/mpi/c/file_get_errhandler.c
+++ /dev/null
@@ -1,73 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015-2017 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_get_errhandler = PMPI_File_get_errhandler
-#endif
-#define MPI_File_get_errhandler PMPI_File_get_errhandler
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_get_errhandler";
-
-
-int MPI_File_get_errhandler( MPI_File file, MPI_Errhandler *errhandler)
-{
-  /* Error checking */
-
-  if (MPI_PARAM_CHECK) {
-    OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-    /* Note that MPI-2:9.7 (p265 in the ps; 261 in the pdf) explicitly
-       says that you are allowed to set the error handler on
-       MPI_FILE_NULL */
-
-    if (NULL == file) {
-      return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_FILE,
-                                   "MPI_File_get_errhandler");
-    } else if (NULL == errhandler) {
-      return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                   "MPI_File_get_errhandler");
-    }
-  }
-
-  OPAL_THREAD_LOCK(&file->f_lock);
-  /* Retain the errhandler, corresponding to object refcount
-     decrease in errhandler_free.c. */
-  *errhandler = file->error_handler;
-  OBJ_RETAIN(file->error_handler);
-  OPAL_THREAD_UNLOCK(&file->f_lock);
-
-  /* All done */
-
-  return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/file_get_errhandler.c.in b/ompi/mpi/c/file_get_errhandler.c.in
new file mode 100644
index 00000000000..8940b77bb3b
--- /dev/null
+++ b/ompi/mpi/c/file_get_errhandler.c.in
@@ -0,0 +1,70 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2017 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2020-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+
+PROTOTYPE ERROR_CLASS file_get_errhandler(FILE file, ERRHANDLER_OUT errhandler)
+{
+  int ret = MPI_SUCCESS;
+
+  /* Error checking */
+
+  if (MPI_PARAM_CHECK) {
+    OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+    /* Note that MPI-2:9.7 (p265 in the ps; 261 in the pdf) explicitly
+       says that you are allowed to set the error handler on
+       MPI_FILE_NULL */
+
+    if (NULL == file) {
+      return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_FILE,
+                                   "MPI_File_get_errhandler");
+    } else if (NULL == errhandler) {
+      return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                   "MPI_File_get_errhandler");
+    }
+  }
+
+  OPAL_THREAD_LOCK(&file->f_lock);
+  /* Retain the errhandler, corresponding to object refcount
+     decrease in errhandler_free.c. */
+  *errhandler = file->error_handler;
+  OBJ_RETAIN(file->error_handler);
+  OPAL_THREAD_UNLOCK(&file->f_lock);
+
+   /* make sure the infrastructure is initialized */
+  ret = ompi_mpi_instance_retain ();
+
+  /* All done */
+
+  return ret;
+}
diff --git a/ompi/mpi/c/file_get_group.c b/ompi/mpi/c/file_get_group.c
deleted file mode 100644
index 3e09186377b..00000000000
--- a/ompi/mpi/c/file_get_group.c
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_get_group = PMPI_File_get_group
-#endif
-#define MPI_File_get_group PMPI_File_get_group
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_get_group";
-
-
-int MPI_File_get_group(MPI_File fh, MPI_Group *group)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            rc = MPI_ERR_FILE;
-            fh = MPI_FILE_NULL;
-        } else if (NULL == group) {
-            rc = MPI_ERR_GROUP;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Does not need to invoke a back-end io function */
-
-    rc = ompi_comm_group (fh->f_comm, group);
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_get_group.c.in b/ompi/mpi/c/file_get_group.c.in
new file mode 100644
index 00000000000..cb2ef7ebdf5
--- /dev/null
+++ b/ompi/mpi/c/file_get_group.c.in
@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+
+PROTOTYPE ERROR_CLASS file_get_group(FILE fh, GROUP_OUT group)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            rc = MPI_ERR_FILE;
+            fh = MPI_FILE_NULL;
+        } else if (NULL == group) {
+            rc = MPI_ERR_GROUP;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Does not need to invoke a back-end io function */
+
+    rc = ompi_comm_group (fh->f_comm, group);
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_get_info.c b/ompi/mpi/c/file_get_info.c
deleted file mode 100644
index 8d1411dedc8..00000000000
--- a/ompi/mpi/c/file_get_info.c
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2016-2019 IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/file/file.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_get_info = PMPI_File_get_info
-#endif
-#define MPI_File_get_info PMPI_File_get_info
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_get_info";
-
-
-int MPI_File_get_info(MPI_File fh, MPI_Info *info_used)
-{
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == info_used) {
-            return OMPI_ERRHANDLER_INVOKE(fh, MPI_ERR_INFO, FUNC_NAME);
-        }
-        if (ompi_file_invalid(fh)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-    }
-
-// Some components we're still letting handle info internally, eg romio321.
-// Components that want to handle it themselves will fill in the get/set
-// info function pointers, components that don't will use NULL.
-    if (fh->f_io_selected_module.v2_0_0.io_module_file_get_info != NULL) {
-        int rc;
-        switch (fh->f_io_version) {
-        case MCA_IO_BASE_V_2_0_0:
-            rc = fh->f_io_selected_module.v2_0_0.
-              io_module_file_get_info(fh, info_used);
-            break;
-
-        default:
-            rc = MPI_ERR_INTERN;
-            break;
-        }
-        OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-    }
-
-    if (NULL == fh->super.s_info) {
-/*
- * Setup any defaults if MPI_Win_set_info was never called
- */
-        opal_infosubscribe_change_info(&fh->super, &MPI_INFO_NULL->super);
-    }
-
-
-    (*info_used) = OBJ_NEW(ompi_info_t);
-    if (NULL == (*info_used)) {
-       return OMPI_ERRHANDLER_INVOKE(fh, MPI_ERR_NO_MEM, FUNC_NAME);
-    }
-    opal_info_t *opal_info_used = &(*info_used)->super;
-
-    opal_info_dup_mpistandard(fh->super.s_info, &opal_info_used);
-
-    return OMPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/file_get_info.c.in b/ompi/mpi/c/file_get_info.c.in
new file mode 100644
index 00000000000..9d158e11bc3
--- /dev/null
+++ b/ompi/mpi/c/file_get_info.c.in
@@ -0,0 +1,82 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016-2019 IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/file/file.h"
+
+PROTOTYPE ERROR_CLASS file_get_info(FILE fh, INFO_OUT info_used)
+{
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == info_used) {
+            return OMPI_ERRHANDLER_INVOKE(fh, MPI_ERR_INFO, FUNC_NAME);
+        }
+        if (ompi_file_invalid(fh)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+    }
+
+// Some components we're still letting handle info internally, eg romio321.
+// Components that want to handle it themselves will fill in the get/set
+// info function pointers, components that don't will use NULL.
+    if (fh->f_io_selected_module.v3_0_0.io_module_file_get_info != NULL) {
+        int rc;
+        switch (fh->f_io_version) {
+        case MCA_IO_BASE_V_3_0_0:
+            rc = fh->f_io_selected_module.v3_0_0.
+              io_module_file_get_info(fh, info_used);
+            break;
+
+        default:
+            rc = MPI_ERR_INTERN;
+            break;
+        }
+        OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+    }
+
+    if (NULL == fh->super.s_info) {
+        /*
+         * Setup any defaults if MPI_Win_set_info was never called
+         */
+        opal_infosubscribe_change_info(&fh->super, &MPI_INFO_NULL->super);
+    }
+
+
+    *info_used = ompi_info_allocate ();
+    if (NULL == *info_used) {
+       return OMPI_ERRHANDLER_INVOKE(fh, MPI_ERR_NO_MEM, FUNC_NAME);
+    }
+    opal_info_t *opal_info_used = &(*info_used)->super;
+
+    opal_info_dup_public(fh->super.s_info, &opal_info_used);
+
+    return OMPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/file_get_position.c b/ompi/mpi/c/file_get_position.c
deleted file mode 100644
index 1f0d44c9c7e..00000000000
--- a/ompi/mpi/c/file_get_position.c
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_get_position = PMPI_File_get_position
-#endif
-#define MPI_File_get_position PMPI_File_get_position
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_get_position";
-
-
-int MPI_File_get_position(MPI_File fh, MPI_Offset *offset)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            rc = MPI_ERR_FILE;
-            fh = MPI_FILE_NULL;
-        } else if (NULL == offset) {
-            rc = MPI_ERR_ARG;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-            io_module_file_get_position(fh, offset);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_get_position.c.in b/ompi/mpi/c/file_get_position.c.in
new file mode 100644
index 00000000000..ff36d078fab
--- /dev/null
+++ b/ompi/mpi/c/file_get_position.c.in
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+
+PROTOTYPE ERROR_CLASS file_get_position(FILE fh, OFFSET_OUT offset)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            rc = MPI_ERR_FILE;
+            fh = MPI_FILE_NULL;
+        } else if (NULL == offset) {
+            rc = MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+            io_module_file_get_position(fh, offset);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_get_position_shared.c b/ompi/mpi/c/file_get_position_shared.c
deleted file mode 100644
index 944d3dd1008..00000000000
--- a/ompi/mpi/c/file_get_position_shared.c
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_get_position_shared = PMPI_File_get_position_shared
-#endif
-#define MPI_File_get_position_shared PMPI_File_get_position_shared
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_get_position_shared";
-
-
-int MPI_File_get_position_shared(MPI_File fh, MPI_Offset *offset)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            rc = MPI_ERR_FILE;
-            fh = MPI_FILE_NULL;
-        } else if (NULL == offset) {
-            rc = MPI_ERR_ARG;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-            io_module_file_get_position_shared(fh, offset);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_get_position_shared.c.in b/ompi/mpi/c/file_get_position_shared.c.in
new file mode 100644
index 00000000000..588d2d0bbcc
--- /dev/null
+++ b/ompi/mpi/c/file_get_position_shared.c.in
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+
+PROTOTYPE ERROR_CLASS file_get_position_shared(FILE fh, OFFSET_OUT offset)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            rc = MPI_ERR_FILE;
+            fh = MPI_FILE_NULL;
+        } else if (NULL == offset) {
+            rc = MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+            io_module_file_get_position_shared(fh, offset);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_get_size.c b/ompi/mpi/c/file_get_size.c
deleted file mode 100644
index e9d94efed19..00000000000
--- a/ompi/mpi/c/file_get_size.c
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_get_size = PMPI_File_get_size
-#endif
-#define MPI_File_get_size PMPI_File_get_size
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_get_size";
-
-
-int MPI_File_get_size(MPI_File fh, MPI_Offset *size)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            rc = MPI_ERR_FILE;
-            fh = MPI_FILE_NULL;
-        } else if (NULL == size) {
-            rc = MPI_ERR_ARG;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-            io_module_file_get_size(fh, size);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_get_size.c.in b/ompi/mpi/c/file_get_size.c.in
new file mode 100644
index 00000000000..5fee0729b53
--- /dev/null
+++ b/ompi/mpi/c/file_get_size.c.in
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+
+PROTOTYPE ERROR_CLASS file_get_size(FILE fh, OFFSET_OUT size)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            rc = MPI_ERR_FILE;
+            fh = MPI_FILE_NULL;
+        } else if (NULL == size) {
+            rc = MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+            io_module_file_get_size(fh, size);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_get_type_extent.c b/ompi/mpi/c/file_get_type_extent.c
deleted file mode 100644
index fb664adba17..00000000000
--- a/ompi/mpi/c/file_get_type_extent.c
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_get_type_extent = PMPI_File_get_type_extent
-#endif
-#define MPI_File_get_type_extent PMPI_File_get_type_extent
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_get_type_extent";
-
-
-int MPI_File_get_type_extent(MPI_File fh, MPI_Datatype datatype,
-                             MPI_Aint *extent)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            rc = MPI_ERR_FILE;
-            fh = MPI_FILE_NULL;
-        } else if (NULL == extent) {
-            rc = MPI_ERR_ARG;
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, 1);
-        }
-
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-            io_module_file_get_type_extent(fh, datatype, extent);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_get_type_extent.c.in b/ompi/mpi/c/file_get_type_extent.c.in
new file mode 100644
index 00000000000..09748110e11
--- /dev/null
+++ b/ompi/mpi/c/file_get_type_extent.c.in
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS file_get_type_extent(FILE fh, DATATYPE datatype,
+                                           AINT_COUNT_OUT extent)
+{
+    int rc;
+    MPI_Aint tmp_extent = 0;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            rc = MPI_ERR_FILE;
+            fh = MPI_FILE_NULL;
+        } else if (NULL == extent) {
+            rc = MPI_ERR_ARG;
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, 1);
+        }
+
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+            io_module_file_get_type_extent(fh, datatype, &tmp_extent);
+        *extent = tmp_extent;
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_get_view.c b/ompi/mpi/c/file_get_view.c
deleted file mode 100644
index ac828b3e125..00000000000
--- a/ompi/mpi/c/file_get_view.c
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_get_view = PMPI_File_get_view
-#endif
-#define MPI_File_get_view PMPI_File_get_view
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_get_view";
-
-
-int MPI_File_get_view(MPI_File fh, MPI_Offset *disp,
-                      MPI_Datatype *etype,
-                      MPI_Datatype *filetype, char *datarep)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            rc = MPI_ERR_FILE;
-            fh = MPI_FILE_NULL;
-        } else if (NULL == etype || NULL == filetype) {
-            rc = MPI_ERR_TYPE;
-        } else if (NULL == disp || NULL == datarep) {
-            rc = MPI_ERR_ARG;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-            io_module_file_get_view(fh, disp, etype, filetype, datarep);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_get_view.c.in b/ompi/mpi/c/file_get_view.c.in
new file mode 100644
index 00000000000..52f8c2e2cd6
--- /dev/null
+++ b/ompi/mpi/c/file_get_view.c.in
@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+
+PROTOTYPE ERROR_CLASS file_get_view(FILE fh, OFFSET_OUT disp, DATATYPE_OUT etype,
+                                    DATATYPE_OUT filetype, STRING_OUT datarep)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            rc = MPI_ERR_FILE;
+            fh = MPI_FILE_NULL;
+        } else if (NULL == etype || NULL == filetype) {
+            rc = MPI_ERR_TYPE;
+        } else if (NULL == disp || NULL == datarep) {
+            rc = MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+            io_module_file_get_view(fh, disp, etype, filetype, datarep);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_iread.c b/ompi/mpi/c/file_iread.c
deleted file mode 100644
index 029d81151c5..00000000000
--- a/ompi/mpi/c/file_iread.c
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/file/file.h"
-#include "ompi/mca/io/io.h"
-#include "ompi/mca/io/base/io_base_request.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_iread = PMPI_File_iread
-#endif
-#define MPI_File_iread PMPI_File_iread
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_iread";
-
-
-int MPI_File_iread(MPI_File fh, void *buf, int count,
-                   MPI_Datatype datatype, MPI_Request *request)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else if (NULL == request) {
-            rc = MPI_ERR_REQUEST;
-        } else {
-           OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-            io_module_file_iread(fh, buf, count, datatype, request);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_iread.c.in b/ompi/mpi/c/file_iread.c.in
new file mode 100644
index 00000000000..792d9204fd2
--- /dev/null
+++ b/ompi/mpi/c/file_iread.c.in
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/file/file.h"
+#include "ompi/mca/io/io.h"
+#include "ompi/mca/io/base/io_base_request.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS file_iread(FILE fh, BUFFER_OUT buf, COUNT count,
+                                 DATATYPE datatype, REQUEST_INOUT request)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else if (NULL == request) {
+            rc = MPI_ERR_REQUEST;
+        } else {
+           OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+            io_module_file_iread(fh, buf, count, datatype, request);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_iread_all.c b/ompi/mpi/c/file_iread_all.c
deleted file mode 100644
index 50c9732a4e1..00000000000
--- a/ompi/mpi/c/file_iread_all.c
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      University of Houston.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/file/file.h"
-#include "ompi/mca/io/io.h"
-#include "ompi/mca/io/base/io_base_request.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_iread_all = PMPI_File_iread_all
-#endif
-#define MPI_File_iread_all PMPI_File_iread_all
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_iread_all";
-
-
-int MPI_File_iread_all(MPI_File fh, void *buf, int count,
-                       MPI_Datatype datatype, MPI_Request *request)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else if (NULL == request) {
-            rc = MPI_ERR_REQUEST;
-        } else {
-           OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        if( OPAL_UNLIKELY(NULL == fh->f_io_selected_module.v2_0_0.io_module_file_iread_all) ) {
-            rc = MPI_ERR_UNSUPPORTED_OPERATION;
-        }
-        else {
-            rc = fh->f_io_selected_module.v2_0_0.
-                io_module_file_iread_all(fh, buf, count, datatype, request);
-        }
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_iread_all.c.in b/ompi/mpi/c/file_iread_all.c.in
new file mode 100644
index 00000000000..2e65290dd6d
--- /dev/null
+++ b/ompi/mpi/c/file_iread_all.c.in
@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      University of Houston.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/file/file.h"
+#include "ompi/mca/io/io.h"
+#include "ompi/mca/io/base/io_base_request.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS file_iread_all(FILE fh, BUFFER_OUT buf, COUNT count,
+                                     DATATYPE datatype, REQUEST_INOUT request)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else if (NULL == request) {
+            rc = MPI_ERR_REQUEST;
+        } else {
+           OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        if( OPAL_UNLIKELY(NULL == fh->f_io_selected_module.v3_0_0.io_module_file_iread_all) ) {
+            rc = MPI_ERR_UNSUPPORTED_OPERATION;
+        }
+        else {
+            rc = fh->f_io_selected_module.v3_0_0.
+                io_module_file_iread_all(fh, buf, count, datatype, request);
+        }
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_iread_at.c b/ompi/mpi/c/file_iread_at.c
deleted file mode 100644
index b10df7a7451..00000000000
--- a/ompi/mpi/c/file_iread_at.c
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/file/file.h"
-#include "ompi/mca/io/io.h"
-#include "ompi/mca/io/base/io_base_request.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_iread_at = PMPI_File_iread_at
-#endif
-#define MPI_File_iread_at PMPI_File_iread_at
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_iread_at";
-
-
-int MPI_File_iread_at(MPI_File fh, MPI_Offset offset, void *buf,
-                      int count, MPI_Datatype datatype, MPI_Request *request)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else if (NULL == request) {
-            rc = MPI_ERR_REQUEST;
-        } else {
-           OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-            io_module_file_iread_at(fh, offset, buf, count, datatype,
-                                    request);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_iread_at.c.in b/ompi/mpi/c/file_iread_at.c.in
new file mode 100644
index 00000000000..f62d07eeab5
--- /dev/null
+++ b/ompi/mpi/c/file_iread_at.c.in
@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/file/file.h"
+#include "ompi/mca/io/io.h"
+#include "ompi/mca/io/base/io_base_request.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS file_iread_at(FILE fh, OFFSET offset, BUFFER_OUT buf,
+                                    COUNT count, DATATYPE datatype, REQUEST_INOUT request)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else if (NULL == request) {
+            rc = MPI_ERR_REQUEST;
+        } else {
+           OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+            io_module_file_iread_at(fh, offset, buf, count, datatype,
+                                    request);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_iread_at_all.c b/ompi/mpi/c/file_iread_at_all.c
deleted file mode 100644
index ec790aed050..00000000000
--- a/ompi/mpi/c/file_iread_at_all.c
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      University of Houston.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/file/file.h"
-#include "ompi/mca/io/io.h"
-#include "ompi/mca/io/base/io_base_request.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_iread_at_all = PMPI_File_iread_at_all
-#endif
-#define MPI_File_iread_at_all PMPI_File_iread_at_all
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_iread_at_all";
-
-
-int MPI_File_iread_at_all(MPI_File fh, MPI_Offset offset, void *buf,
-                          int count, MPI_Datatype datatype, MPI_Request *request)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else if (NULL == request) {
-            rc = MPI_ERR_REQUEST;
-        } else {
-           OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        if( OPAL_UNLIKELY(NULL == fh->f_io_selected_module.v2_0_0.io_module_file_iread_at_all) ) {
-            rc = MPI_ERR_UNSUPPORTED_OPERATION;
-        }
-        else {
-            rc = fh->f_io_selected_module.v2_0_0.
-                io_module_file_iread_at_all(fh, offset, buf, count, datatype,
-                                            request);
-        }
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_iread_at_all.c.in b/ompi/mpi/c/file_iread_at_all.c.in
new file mode 100644
index 00000000000..6613a85e1ff
--- /dev/null
+++ b/ompi/mpi/c/file_iread_at_all.c.in
@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      University of Houston.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/file/file.h"
+#include "ompi/mca/io/io.h"
+#include "ompi/mca/io/base/io_base_request.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS file_iread_at_all(FILE fh, OFFSET offset, BUFFER_OUT buf,
+                                        COUNT count, DATATYPE datatype, REQUEST_INOUT request)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else if (NULL == request) {
+            rc = MPI_ERR_REQUEST;
+        } else {
+           OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        if( OPAL_UNLIKELY(NULL == fh->f_io_selected_module.v3_0_0.io_module_file_iread_at_all) ) {
+            rc = MPI_ERR_UNSUPPORTED_OPERATION;
+        }
+        else {
+            rc = fh->f_io_selected_module.v3_0_0.
+                io_module_file_iread_at_all(fh, offset, buf, count, datatype,
+                                            request);
+        }
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_iread_shared.c b/ompi/mpi/c/file_iread_shared.c
deleted file mode 100644
index 6c49f751e43..00000000000
--- a/ompi/mpi/c/file_iread_shared.c
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/file/file.h"
-#include "ompi/mca/io/io.h"
-#include "ompi/mca/io/base/io_base_request.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_iread_shared = PMPI_File_iread_shared
-#endif
-#define MPI_File_iread_shared PMPI_File_iread_shared
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_iread_shared";
-
-
-int MPI_File_iread_shared(MPI_File fh, void *buf, int count,
-                          MPI_Datatype datatype, MPI_Request *request)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else if (NULL == request) {
-            rc = MPI_ERR_REQUEST;
-        } else {
-           OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-            io_module_file_iread_shared(fh, buf, count, datatype, request);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_iread_shared.c.in b/ompi/mpi/c/file_iread_shared.c.in
new file mode 100644
index 00000000000..9c17a98c25a
--- /dev/null
+++ b/ompi/mpi/c/file_iread_shared.c.in
@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/file/file.h"
+#include "ompi/mca/io/io.h"
+#include "ompi/mca/io/base/io_base_request.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS file_iread_shared(FILE fh, BUFFER_OUT buf, COUNT count,
+                                        DATATYPE datatype, REQUEST_INOUT request)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else if (NULL == request) {
+            rc = MPI_ERR_REQUEST;
+        } else {
+           OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+            io_module_file_iread_shared(fh, buf, count, datatype, request);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_iwrite.c b/ompi/mpi/c/file_iwrite.c
deleted file mode 100644
index e2f04db2f83..00000000000
--- a/ompi/mpi/c/file_iwrite.c
+++ /dev/null
@@ -1,86 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/file/file.h"
-#include "ompi/mca/io/io.h"
-#include "ompi/mca/io/base/io_base_request.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_iwrite = PMPI_File_iwrite
-#endif
-#define MPI_File_iwrite PMPI_File_iwrite
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_iwrite";
-
-
-int MPI_File_iwrite(MPI_File fh, const void *buf, int count, MPI_Datatype
-                    datatype, MPI_Request *request)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else if (NULL == request) {
-            rc = MPI_ERR_REQUEST;
-        } else {
-           OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-          io_module_file_iwrite(fh, buf, count, datatype, request);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_iwrite.c.in b/ompi/mpi/c/file_iwrite.c.in
new file mode 100644
index 00000000000..dd6c3819b86
--- /dev/null
+++ b/ompi/mpi/c/file_iwrite.c.in
@@ -0,0 +1,78 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/file/file.h"
+#include "ompi/mca/io/io.h"
+#include "ompi/mca/io/base/io_base_request.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS file_iwrite(FILE fh, BUFFER buf, COUNT count,
+                                  DATATYPE datatype, REQUEST_INOUT request)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else if (NULL == request) {
+            rc = MPI_ERR_REQUEST;
+        } else {
+           OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+          io_module_file_iwrite(fh, buf, count, datatype, request);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_iwrite_all.c b/ompi/mpi/c/file_iwrite_all.c
deleted file mode 100644
index 77349d0e046..00000000000
--- a/ompi/mpi/c/file_iwrite_all.c
+++ /dev/null
@@ -1,93 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      University of Houston.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/file/file.h"
-#include "ompi/mca/io/io.h"
-#include "ompi/mca/io/base/io_base_request.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_iwrite_all = PMPI_File_iwrite_all
-#endif
-#define MPI_File_iwrite_all PMPI_File_iwrite_all
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_iwrite_all";
-
-
-int MPI_File_iwrite_all(MPI_File fh, const void *buf, int count, MPI_Datatype
-                        datatype, MPI_Request *request)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else if (NULL == request) {
-            rc = MPI_ERR_REQUEST;
-        } else {
-           OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        if( OPAL_UNLIKELY(NULL == fh->f_io_selected_module.v2_0_0.io_module_file_iwrite_all) ) {
-            rc = MPI_ERR_UNSUPPORTED_OPERATION;
-        }
-        else {
-            rc = fh->f_io_selected_module.v2_0_0.
-                io_module_file_iwrite_all(fh, buf, count, datatype, request);
-        }
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_iwrite_all.c.in b/ompi/mpi/c/file_iwrite_all.c.in
new file mode 100644
index 00000000000..d543bf76855
--- /dev/null
+++ b/ompi/mpi/c/file_iwrite_all.c.in
@@ -0,0 +1,85 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      University of Houston.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/file/file.h"
+#include "ompi/mca/io/io.h"
+#include "ompi/mca/io/base/io_base_request.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS file_iwrite_all(FILE fh, BUFFER buf, COUNT count,
+                                      DATATYPE datatype, REQUEST_INOUT request)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else if (NULL == request) {
+            rc = MPI_ERR_REQUEST;
+        } else {
+           OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        if( OPAL_UNLIKELY(NULL == fh->f_io_selected_module.v3_0_0.io_module_file_iwrite_all) ) {
+            rc = MPI_ERR_UNSUPPORTED_OPERATION;
+        }
+        else {
+            rc = fh->f_io_selected_module.v3_0_0.
+                io_module_file_iwrite_all(fh, buf, count, datatype, request);
+        }
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_iwrite_at.c b/ompi/mpi/c/file_iwrite_at.c
deleted file mode 100644
index 7dca089212d..00000000000
--- a/ompi/mpi/c/file_iwrite_at.c
+++ /dev/null
@@ -1,89 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/file/file.h"
-#include "ompi/mca/io/io.h"
-#include "ompi/mca/io/base/io_base_request.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_iwrite_at = PMPI_File_iwrite_at
-#endif
-#define MPI_File_iwrite_at PMPI_File_iwrite_at
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_iwrite_at";
-
-
-int MPI_File_iwrite_at(MPI_File fh, MPI_Offset offset, const void *buf,
-                       int count, MPI_Datatype datatype,
-                       MPI_Request *request)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else if (NULL == request) {
-            rc = MPI_ERR_REQUEST;
-        } else {
-           OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-            io_module_file_iwrite_at(fh, offset, buf, count, datatype,
-                                     request);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_iwrite_at.c.in b/ompi/mpi/c/file_iwrite_at.c.in
new file mode 100644
index 00000000000..2b0d05b851c
--- /dev/null
+++ b/ompi/mpi/c/file_iwrite_at.c.in
@@ -0,0 +1,81 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/file/file.h"
+#include "ompi/mca/io/io.h"
+#include "ompi/mca/io/base/io_base_request.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS file_iwrite_at(FILE fh, OFFSET offset, BUFFER buf,
+                                     COUNT count, DATATYPE datatype,
+                                     REQUEST_INOUT request)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else if (NULL == request) {
+            rc = MPI_ERR_REQUEST;
+        } else {
+           OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+            io_module_file_iwrite_at(fh, offset, buf, count, datatype,
+                                     request);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_iwrite_at_all.c b/ompi/mpi/c/file_iwrite_at_all.c
deleted file mode 100644
index d0f6d1e783f..00000000000
--- a/ompi/mpi/c/file_iwrite_at_all.c
+++ /dev/null
@@ -1,96 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      University of Houston.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/file/file.h"
-#include "ompi/mca/io/io.h"
-#include "ompi/mca/io/base/io_base_request.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_iwrite_at_all = PMPI_File_iwrite_at_all
-#endif
-#define MPI_File_iwrite_at_all PMPI_File_iwrite_at_all
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_iwrite_at_all";
-
-
-int MPI_File_iwrite_at_all(MPI_File fh, MPI_Offset offset, const void *buf,
-                           int count, MPI_Datatype datatype,
-                           MPI_Request *request)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else if (NULL == request) {
-            rc = MPI_ERR_REQUEST;
-        } else {
-           OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        if( OPAL_UNLIKELY(NULL == fh->f_io_selected_module.v2_0_0.io_module_file_iwrite_at_all) ) {
-            rc = MPI_ERR_UNSUPPORTED_OPERATION;
-        }
-        else {
-            rc = fh->f_io_selected_module.v2_0_0.
-                io_module_file_iwrite_at_all(fh, offset, buf, count, datatype,
-                                             request);
-        }
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_iwrite_at_all.c.in b/ompi/mpi/c/file_iwrite_at_all.c.in
new file mode 100644
index 00000000000..d11c109f4d4
--- /dev/null
+++ b/ompi/mpi/c/file_iwrite_at_all.c.in
@@ -0,0 +1,88 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      University of Houston.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/file/file.h"
+#include "ompi/mca/io/io.h"
+#include "ompi/mca/io/base/io_base_request.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS file_iwrite_at_all(FILE fh, OFFSET offset, BUFFER buf,
+                                         COUNT count, DATATYPE datatype,
+                                         REQUEST_INOUT request)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else if (NULL == request) {
+            rc = MPI_ERR_REQUEST;
+        } else {
+           OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        if( OPAL_UNLIKELY(NULL == fh->f_io_selected_module.v3_0_0.io_module_file_iwrite_at_all) ) {
+            rc = MPI_ERR_UNSUPPORTED_OPERATION;
+        }
+        else {
+            rc = fh->f_io_selected_module.v3_0_0.
+                io_module_file_iwrite_at_all(fh, offset, buf, count, datatype,
+                                             request);
+        }
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_iwrite_shared.c b/ompi/mpi/c/file_iwrite_shared.c
deleted file mode 100644
index 90af5affa15..00000000000
--- a/ompi/mpi/c/file_iwrite_shared.c
+++ /dev/null
@@ -1,86 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/file/file.h"
-#include "ompi/mca/io/io.h"
-#include "ompi/mca/io/base/io_base_request.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_iwrite_shared = PMPI_File_iwrite_shared
-#endif
-#define MPI_File_iwrite_shared PMPI_File_iwrite_shared
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_iwrite_shared";
-
-
-int MPI_File_iwrite_shared(MPI_File fh, const void *buf, int count,
-                           MPI_Datatype datatype, MPI_Request *request)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else if (NULL == request) {
-            rc = MPI_ERR_REQUEST;
-        } else {
-           OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-          io_module_file_iwrite_shared(fh, buf, count, datatype, request);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_iwrite_shared.c.in b/ompi/mpi/c/file_iwrite_shared.c.in
new file mode 100644
index 00000000000..4c0bb039391
--- /dev/null
+++ b/ompi/mpi/c/file_iwrite_shared.c.in
@@ -0,0 +1,78 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/file/file.h"
+#include "ompi/mca/io/io.h"
+#include "ompi/mca/io/base/io_base_request.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS file_iwrite_shared(FILE fh, BUFFER buf, COUNT count,
+                                         DATATYPE datatype, REQUEST_INOUT request)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else if (NULL == request) {
+            rc = MPI_ERR_REQUEST;
+        } else {
+           OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+          io_module_file_iwrite_shared(fh, buf, count, datatype, request);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_open.c b/ompi/mpi/c/file_open.c
deleted file mode 100644
index 204e92e9e44..00000000000
--- a/ompi/mpi/c/file_open.c
+++ /dev/null
@@ -1,113 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2013      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2016      University of Houston. All rights reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-#include "ompi/file/file.h"
-#include "ompi/mca/io/io.h"
-#include "ompi/mca/io/base/base.h"
-#include "ompi/memchecker.h"
-
-
-extern opal_mutex_t ompi_mpi_file_bootstrap_mutex;
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_open = PMPI_File_open
-#endif
-#define MPI_File_open PMPI_File_open
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_open";
-
-
-int MPI_File_open(MPI_Comm comm, const char *filename, int amode,
-                  MPI_Info info, MPI_File *fh)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == info || ompi_info_is_freed(info)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
-                                          FUNC_NAME);
-        } else if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-        if (OMPI_COMM_IS_INTER(comm)) {
-            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-
-    }
-
-    /* Note that MPI-2:9.7 (p265 in the ps; p261 in the pdf) says that
-       errors in MPI_FILE_OPEN (before the file handle is created)
-       should invoke the default error handler on MPI_FILE_NULL.
-       Hence, if we get a file handle out of ompi_file_open(), invoke
-       the error handler on that.  If not, invoke the error handler on
-       MPI_FILE_NULL. */
-
-    /* The io framework is only initialized lazily.  If it hasn't
-       already been initialized, do so now (note that MPI_FILE_OPEN
-       and MPI_FILE_DELETE are the only two places that it will be
-       initialized). */
-
-    /* For multi-threaded scenarios, initializing the file i/o
-       framework and mca infrastructure needs to be protected
-       by a mutex, similarly to the other frameworks in
-       ompi/runtime/ompi_mpi_init.c
-    */
-
-    opal_mutex_lock(&ompi_mpi_file_bootstrap_mutex);
-
-    rc = mca_base_framework_open(&ompi_io_base_framework, 0);
-    if (OMPI_SUCCESS != rc) {
-        opal_mutex_unlock(&ompi_mpi_file_bootstrap_mutex);
-        return OMPI_ERRHANDLER_INVOKE(MPI_FILE_NULL, rc, FUNC_NAME);
-    }
-    opal_mutex_unlock(&ompi_mpi_file_bootstrap_mutex);
-
-    /* Create an empty MPI_File handle */
-
-    *fh = MPI_FILE_NULL;
-    rc = ompi_file_open(comm, filename, amode, &(info->super), fh);
-
-    /* Creating the file handle also selects a component to use,
-       creates a module, and calls file_open() on the module.  So
-       we're good to go. */
-    OMPI_ERRHANDLER_RETURN(rc, *fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_open.c.in b/ompi/mpi/c/file_open.c.in
new file mode 100644
index 00000000000..a6278068471
--- /dev/null
+++ b/ompi/mpi/c/file_open.c.in
@@ -0,0 +1,104 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2013      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016      University of Houston. All rights reserved.
+ * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include "ompi/file/file.h"
+#include "ompi/mca/io/io.h"
+#include "ompi/mca/io/base/base.h"
+#include "ompi/memchecker.h"
+
+extern opal_mutex_t ompi_mpi_file_bootstrap_mutex;
+
+PROTOTYPE ERROR_CLASS file_open(COMM comm, STRING filename, INT amode,
+                                INFO info, FILE_OUT fh)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == info || ompi_info_is_freed(info)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
+                                          FUNC_NAME);
+        } else if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+        if (OMPI_COMM_IS_INTER(comm)) {
+            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+
+    }
+
+    /* Note that MPI-2:9.7 (p265 in the ps; p261 in the pdf) says that
+       errors in MPI_FILE_OPEN (before the file handle is created)
+       should invoke the default error handler on MPI_FILE_NULL.
+       Hence, if we get a file handle out of ompi_file_open(), invoke
+       the error handler on that.  If not, invoke the error handler on
+       MPI_FILE_NULL. */
+
+    /* The io framework is only initialized lazily.  If it hasn't
+       already been initialized, do so now (note that MPI_FILE_OPEN
+       and MPI_FILE_DELETE are the only two places that it will be
+       initialized). */
+
+    /* For multi-threaded scenarios, initializing the file i/o
+       framework and mca infrastructure needs to be protected
+       by a mutex, similarly to the other frameworks in
+       ompi/runtime/ompi_mpi_init.c
+    */
+
+    opal_mutex_lock(&ompi_mpi_file_bootstrap_mutex);
+
+    rc = mca_base_framework_open(&ompi_io_base_framework, 0);
+    if (OMPI_SUCCESS != rc) {
+        opal_mutex_unlock(&ompi_mpi_file_bootstrap_mutex);
+        return OMPI_ERRHANDLER_INVOKE(MPI_FILE_NULL, rc, FUNC_NAME);
+    }
+    opal_mutex_unlock(&ompi_mpi_file_bootstrap_mutex);
+
+    /* Create an empty MPI_File handle */
+
+    *fh = MPI_FILE_NULL;
+    rc = ompi_file_open(comm, filename, amode, &(info->super), fh);
+
+    /* Creating the file handle also selects a component to use,
+       creates a module, and calls file_open() on the module.  So
+       we're good to go. */
+    OMPI_ERRHANDLER_RETURN(rc, *fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_preallocate.c b/ompi/mpi/c/file_preallocate.c
deleted file mode 100644
index 0f13c4d91ed..00000000000
--- a/ompi/mpi/c/file_preallocate.c
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_preallocate = PMPI_File_preallocate
-#endif
-#define MPI_File_preallocate PMPI_File_preallocate
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_preallocate";
-
-
-int MPI_File_preallocate(MPI_File fh, MPI_Offset size)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            rc = MPI_ERR_FILE;
-            fh = MPI_FILE_NULL;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-            io_module_file_preallocate(fh, size);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_preallocate.c.in b/ompi/mpi/c/file_preallocate.c.in
new file mode 100644
index 00000000000..f188738045f
--- /dev/null
+++ b/ompi/mpi/c/file_preallocate.c.in
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+
+PROTOTYPE ERROR_CLASS file_preallocate(FILE fh, OFFSET size)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            rc = MPI_ERR_FILE;
+            fh = MPI_FILE_NULL;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+            io_module_file_preallocate(fh, size);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_read.c b/ompi/mpi/c/file_read.c
deleted file mode 100644
index 4df5785a4e4..00000000000
--- a/ompi/mpi/c/file_read.c
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_read = PMPI_File_read
-#endif
-#define MPI_File_read PMPI_File_read
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_read";
-
-
-int MPI_File_read(MPI_File fh, void *buf, int count,
-                  MPI_Datatype datatype, MPI_Status *status)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else {
-           OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-            io_module_file_read(fh, buf, count, datatype, status);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_read.c.in b/ompi/mpi/c/file_read.c.in
new file mode 100644
index 00000000000..5346df22d4d
--- /dev/null
+++ b/ompi/mpi/c/file_read.c.in
@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS File_read(FILE fh, BUFFER_OUT buf, COUNT count,
+                                DATATYPE datatype, STATUS_OUT status)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else {
+           OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+            io_module_file_read(fh, buf, count, datatype, status);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_read_all.c b/ompi/mpi/c/file_read_all.c
deleted file mode 100644
index 9d8d7cdc170..00000000000
--- a/ompi/mpi/c/file_read_all.c
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_read_all = PMPI_File_read_all
-#endif
-#define MPI_File_read_all PMPI_File_read_all
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_read_all";
-
-
-int MPI_File_read_all(MPI_File fh, void *buf, int count, MPI_Datatype
-                      datatype, MPI_Status *status)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else {
-           OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-            io_module_file_read_all(fh, buf, count, datatype, status);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_read_all.c.in b/ompi/mpi/c/file_read_all.c.in
new file mode 100644
index 00000000000..3407258cadd
--- /dev/null
+++ b/ompi/mpi/c/file_read_all.c.in
@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS file_read_all(FILE fh, BUFFER_OUT buf, COUNT count,
+                                    DATATYPE datatype, STATUS_OUT status)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else {
+           OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+            io_module_file_read_all(fh, buf, count, datatype, status);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_read_all_begin.c b/ompi/mpi/c/file_read_all_begin.c
deleted file mode 100644
index c818623f877..00000000000
--- a/ompi/mpi/c/file_read_all_begin.c
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_read_all_begin = PMPI_File_read_all_begin
-#endif
-#define MPI_File_read_all_begin PMPI_File_read_all_begin
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_read_all_begin";
-
-
-int MPI_File_read_all_begin(MPI_File fh, void *buf, int count,
-                            MPI_Datatype datatype)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else {
-           OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-            io_module_file_read_all_begin(fh, buf, count, datatype);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_read_all_begin.c.in b/ompi/mpi/c/file_read_all_begin.c.in
new file mode 100644
index 00000000000..58552037653
--- /dev/null
+++ b/ompi/mpi/c/file_read_all_begin.c.in
@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS file_read_all_begin(FILE fh, BUFFER_OUT buf, COUNT count,
+                                          DATATYPE datatype)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else {
+           OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+            io_module_file_read_all_begin(fh, buf, count, datatype);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_read_all_end.c b/ompi/mpi/c/file_read_all_end.c
deleted file mode 100644
index 1a4b0451f25..00000000000
--- a/ompi/mpi/c/file_read_all_end.c
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_read_all_end = PMPI_File_read_all_end
-#endif
-#define MPI_File_read_all_end PMPI_File_read_all_end
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_read_all_end";
-
-
-int MPI_File_read_all_end(MPI_File fh, void *buf, MPI_Status *status)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-            io_module_file_read_all_end(fh, buf, status);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_read_all_end.c.in b/ompi/mpi/c/file_read_all_end.c.in
new file mode 100644
index 00000000000..c038aa710fc
--- /dev/null
+++ b/ompi/mpi/c/file_read_all_end.c.in
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+
+PROTOTYPE ERROR_CLASS file_read_all_end(FILE fh, BUFFER_OUT buf, STATUS_OUT status)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+            io_module_file_read_all_end(fh, buf, status);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_read_at.c b/ompi/mpi/c/file_read_at.c
deleted file mode 100644
index 87eb9d3ec2d..00000000000
--- a/ompi/mpi/c/file_read_at.c
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_read_at = PMPI_File_read_at
-#endif
-#define MPI_File_read_at PMPI_File_read_at
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_read_at";
-
-
-int MPI_File_read_at(MPI_File fh, MPI_Offset offset, void *buf,
-                     int count, MPI_Datatype datatype, MPI_Status *status)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else {
-           OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-            io_module_file_read_at(fh, offset, buf, count, datatype, status);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_read_at.c.in b/ompi/mpi/c/file_read_at.c.in
new file mode 100644
index 00000000000..37a2e1b86e8
--- /dev/null
+++ b/ompi/mpi/c/file_read_at.c.in
@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS file_read_at(FILE fh, OFFSET offset, BUFFER_OUT buf,
+                                   COUNT count, DATATYPE datatype, STATUS_OUT status)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else {
+           OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+            io_module_file_read_at(fh, offset, buf, count, datatype, status);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_read_at_all.c b/ompi/mpi/c/file_read_at_all.c
deleted file mode 100644
index 9f988da1db6..00000000000
--- a/ompi/mpi/c/file_read_at_all.c
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_read_at_all = PMPI_File_read_at_all
-#endif
-#define MPI_File_read_at_all PMPI_File_read_at_all
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_read_at_all";
-
-
-int MPI_File_read_at_all(MPI_File fh, MPI_Offset offset, void *buf,
-                         int count, MPI_Datatype datatype,
-                         MPI_Status *status)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else {
-           OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-            io_module_file_read_at_all(fh, offset, buf, count, datatype,
-                                       status);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_read_at_all.c.in b/ompi/mpi/c/file_read_at_all.c.in
new file mode 100644
index 00000000000..504fdd490b9
--- /dev/null
+++ b/ompi/mpi/c/file_read_at_all.c.in
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS file_read_at_all(FILE fh, OFFSET offset, BUFFER_OUT buf,
+                                       COUNT count, DATATYPE datatype,
+                                       STATUS_OUT status)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else {
+           OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+            io_module_file_read_at_all(fh, offset, buf, count, datatype,
+                                       status);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_read_at_all_begin.c b/ompi/mpi/c/file_read_at_all_begin.c
deleted file mode 100644
index 99b4a512311..00000000000
--- a/ompi/mpi/c/file_read_at_all_begin.c
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_read_at_all_begin = PMPI_File_read_at_all_begin
-#endif
-#define MPI_File_read_at_all_begin PMPI_File_read_at_all_begin
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_read_at_all_begin";
-
-
-int MPI_File_read_at_all_begin(MPI_File fh, MPI_Offset offset, void *buf,
-                               int count, MPI_Datatype datatype)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else {
-           OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-            io_module_file_read_at_all_begin(fh, offset, buf, count, datatype);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_read_at_all_begin.c.in b/ompi/mpi/c/file_read_at_all_begin.c.in
new file mode 100644
index 00000000000..8ea532e3473
--- /dev/null
+++ b/ompi/mpi/c/file_read_at_all_begin.c.in
@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS file_read_at_all_begin(FILE fh, OFFSET offset, BUFFER_OUT buf,
+                                             COUNT count, DATATYPE datatype)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else {
+           OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+            io_module_file_read_at_all_begin(fh, offset, buf, count, datatype);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_read_at_all_end.c b/ompi/mpi/c/file_read_at_all_end.c
deleted file mode 100644
index b83ebdc265d..00000000000
--- a/ompi/mpi/c/file_read_at_all_end.c
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_read_at_all_end = PMPI_File_read_at_all_end
-#endif
-#define MPI_File_read_at_all_end PMPI_File_read_at_all_end
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_read_at_all_end";
-
-
-int MPI_File_read_at_all_end(MPI_File fh, void *buf, MPI_Status *status)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-            io_module_file_read_at_all_end(fh, buf, status);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_read_at_all_end.c.in b/ompi/mpi/c/file_read_at_all_end.c.in
new file mode 100644
index 00000000000..54584aaa78c
--- /dev/null
+++ b/ompi/mpi/c/file_read_at_all_end.c.in
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+
+PROTOTYPE ERROR_CLASS file_read_at_all_end(FILE fh, BUFFER_OUT buf, STATUS_OUT status)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+            io_module_file_read_at_all_end(fh, buf, status);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_read_ordered.c b/ompi/mpi/c/file_read_ordered.c
deleted file mode 100644
index b784e411ed6..00000000000
--- a/ompi/mpi/c/file_read_ordered.c
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-#include "ompi/datatype/ompi_datatype.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_read_ordered = PMPI_File_read_ordered
-#endif
-#define MPI_File_read_ordered PMPI_File_read_ordered
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_read_ordered";
-
-
-int MPI_File_read_ordered(MPI_File fh, void *buf, int count,
-                          MPI_Datatype datatype, MPI_Status *status)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else {
-           OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-            io_module_file_read_ordered(fh, buf, count, datatype, status);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_read_ordered.c.in b/ompi/mpi/c/file_read_ordered.c.in
new file mode 100644
index 00000000000..4a38fa16749
--- /dev/null
+++ b/ompi/mpi/c/file_read_ordered.c.in
@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+#include "ompi/datatype/ompi_datatype.h"
+
+PROTOTYPE ERROR_CLASS file_read_ordered(FILE fh, BUFFER_OUT buf, COUNT count,
+                                        DATATYPE datatype, STATUS_OUT status)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else {
+           OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+            io_module_file_read_ordered(fh, buf, count, datatype, status);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_read_ordered_begin.c b/ompi/mpi/c/file_read_ordered_begin.c
deleted file mode 100644
index d9a339f3fac..00000000000
--- a/ompi/mpi/c/file_read_ordered_begin.c
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_read_ordered_begin = PMPI_File_read_ordered_begin
-#endif
-#define MPI_File_read_ordered_begin PMPI_File_read_ordered_begin
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_read_ordered_begin";
-
-
-int MPI_File_read_ordered_begin(MPI_File fh, void *buf, int count,
-                                MPI_Datatype datatype)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else {
-           OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-            io_module_file_read_ordered_begin(fh, buf, count, datatype);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_read_ordered_begin.c.in b/ompi/mpi/c/file_read_ordered_begin.c.in
new file mode 100644
index 00000000000..c1f0e6dfba8
--- /dev/null
+++ b/ompi/mpi/c/file_read_ordered_begin.c.in
@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS file_read_ordered_begin(FILE fh, BUFFER_OUT buf, COUNT count,
+                                              DATATYPE datatype)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else {
+           OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+            io_module_file_read_ordered_begin(fh, buf, count, datatype);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_read_ordered_end.c b/ompi/mpi/c/file_read_ordered_end.c
deleted file mode 100644
index 179932cc07d..00000000000
--- a/ompi/mpi/c/file_read_ordered_end.c
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_read_ordered_end = PMPI_File_read_ordered_end
-#endif
-#define MPI_File_read_ordered_end PMPI_File_read_ordered_end
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_read_ordered_end";
-
-
-int MPI_File_read_ordered_end(MPI_File fh, void *buf, MPI_Status *status)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-            io_module_file_read_ordered_end(fh, buf, status);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_read_ordered_end.c.in b/ompi/mpi/c/file_read_ordered_end.c.in
new file mode 100644
index 00000000000..4ef9e623ba7
--- /dev/null
+++ b/ompi/mpi/c/file_read_ordered_end.c.in
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+
+PROTOTYPE ERROR_CLASS file_read_ordered_end(FILE fh, BUFFER_OUT buf, STATUS_OUT status)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+            io_module_file_read_ordered_end(fh, buf, status);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_read_shared.c b/ompi/mpi/c/file_read_shared.c
deleted file mode 100644
index e2eda745c6e..00000000000
--- a/ompi/mpi/c/file_read_shared.c
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_read_shared = PMPI_File_read_shared
-#endif
-#define MPI_File_read_shared PMPI_File_read_shared
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_read_shared";
-
-
-int MPI_File_read_shared(MPI_File fh, void *buf, int count,
-                         MPI_Datatype datatype, MPI_Status *status)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else {
-           OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-            io_module_file_read_shared(fh, buf, count, datatype, status);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_read_shared.c.in b/ompi/mpi/c/file_read_shared.c.in
new file mode 100644
index 00000000000..b473851aff0
--- /dev/null
+++ b/ompi/mpi/c/file_read_shared.c.in
@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS file_read_shared(FILE fh, BUFFER_OUT buf, COUNT count,
+                                       DATATYPE datatype, STATUS_OUT status)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else {
+           OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+            io_module_file_read_shared(fh, buf, count, datatype, status);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_seek.c b/ompi/mpi/c/file_seek.c
deleted file mode 100644
index 0055b530d2b..00000000000
--- a/ompi/mpi/c/file_seek.c
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_seek = PMPI_File_seek
-#endif
-#define MPI_File_seek PMPI_File_seek
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_seek";
-
-
-int MPI_File_seek(MPI_File fh, MPI_Offset offset, int whence)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            rc = MPI_ERR_FILE;
-            fh = MPI_FILE_NULL;
-        } else if (MPI_SEEK_SET != whence && MPI_SEEK_CUR != whence &&
-                   MPI_SEEK_END != whence) {
-            rc = MPI_ERR_ARG;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-            io_module_file_seek(fh, offset, whence);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_seek.c.in b/ompi/mpi/c/file_seek.c.in
new file mode 100644
index 00000000000..697940771b1
--- /dev/null
+++ b/ompi/mpi/c/file_seek.c.in
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+
+PROTOTYPE ERROR_CLASS file_seek(FILE fh, OFFSET offset, INT whence)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            rc = MPI_ERR_FILE;
+            fh = MPI_FILE_NULL;
+        } else if (MPI_SEEK_SET != whence && MPI_SEEK_CUR != whence &&
+                   MPI_SEEK_END != whence) {
+            rc = MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+            io_module_file_seek(fh, offset, whence);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_seek_shared.c b/ompi/mpi/c/file_seek_shared.c
deleted file mode 100644
index b96e5503de0..00000000000
--- a/ompi/mpi/c/file_seek_shared.c
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_seek_shared = PMPI_File_seek_shared
-#endif
-#define MPI_File_seek_shared PMPI_File_seek_shared
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_seek_shared";
-
-
-int MPI_File_seek_shared(MPI_File fh, MPI_Offset offset, int whence)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            rc = MPI_ERR_FILE;
-            fh = MPI_FILE_NULL;
-        } else if (MPI_SEEK_SET != whence && MPI_SEEK_CUR != whence &&
-                   MPI_SEEK_END != whence) {
-            rc = MPI_ERR_ARG;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-            io_module_file_seek_shared(fh, offset, whence);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_seek_shared.c.in b/ompi/mpi/c/file_seek_shared.c.in
new file mode 100644
index 00000000000..e2caa35bb0f
--- /dev/null
+++ b/ompi/mpi/c/file_seek_shared.c.in
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+
+PROTOTYPE ERROR_CLASS file_seek_shared(FILE fh, OFFSET offset, INT whence)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            rc = MPI_ERR_FILE;
+            fh = MPI_FILE_NULL;
+        } else if (MPI_SEEK_SET != whence && MPI_SEEK_CUR != whence &&
+                   MPI_SEEK_END != whence) {
+            rc = MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+            io_module_file_seek_shared(fh, offset, whence);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_set_atomicity.c b/ompi/mpi/c/file_set_atomicity.c
deleted file mode 100644
index 670021adc53..00000000000
--- a/ompi/mpi/c/file_set_atomicity.c
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_set_atomicity = PMPI_File_set_atomicity
-#endif
-#define MPI_File_set_atomicity PMPI_File_set_atomicity
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_set_atomicity";
-
-
-int MPI_File_set_atomicity(MPI_File fh, int flag)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            rc = MPI_ERR_FILE;
-            fh = MPI_FILE_NULL;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-            io_module_file_set_atomicity(fh, flag);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_set_atomicity.c.in b/ompi/mpi/c/file_set_atomicity.c.in
new file mode 100644
index 00000000000..bb80f922813
--- /dev/null
+++ b/ompi/mpi/c/file_set_atomicity.c.in
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+
+PROTOTYPE ERROR_CLASS file_set_atomicity(FILE fh, INT flag)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            rc = MPI_ERR_FILE;
+            fh = MPI_FILE_NULL;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+            io_module_file_set_atomicity(fh, flag);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_set_errhandler.c b/ompi/mpi/c/file_set_errhandler.c
deleted file mode 100644
index 7590ab6c853..00000000000
--- a/ompi/mpi/c/file_set_errhandler.c
+++ /dev/null
@@ -1,78 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015-2017 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_set_errhandler = PMPI_File_set_errhandler
-#endif
-#define MPI_File_set_errhandler PMPI_File_set_errhandler
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_set_errhandler";
-
-
-int MPI_File_set_errhandler( MPI_File file, MPI_Errhandler errhandler)
-{
-    MPI_Errhandler tmp;
-
-    /* Error checking */
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        /* Note that MPI-2:9.7 (p265 in the ps; p261 in the pdf)
-           explicitly says that you are allowed to set the error
-           handler on MPI_FILE_NULL */
-
-        if (NULL == file) {
-            return OMPI_ERRHANDLER_INVOKE(MPI_FILE_NULL, MPI_ERR_FILE,
-                                          FUNC_NAME);
-        } else if (NULL == errhandler ||
-                   MPI_ERRHANDLER_NULL == errhandler ||
-                   (OMPI_ERRHANDLER_TYPE_FILE != errhandler->eh_mpi_object_type &&
-		    OMPI_ERRHANDLER_TYPE_PREDEFINED != errhandler->eh_mpi_object_type) ) {
-            return OMPI_ERRHANDLER_INVOKE(file, MPI_ERR_ARG, FUNC_NAME);
-        }
-    }
-
-    /* Prepare the new error handler */
-    OBJ_RETAIN(errhandler);
-
-    OPAL_THREAD_LOCK(&file->f_lock);
-    /* Ditch the old errhandler, and decrement its refcount. */
-    tmp = file->error_handler;
-    file->error_handler = errhandler;
-    OBJ_RELEASE(tmp);
-    OPAL_THREAD_UNLOCK(&file->f_lock);
-
-    /* All done */
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/file_set_errhandler.c.in b/ompi/mpi/c/file_set_errhandler.c.in
new file mode 100644
index 00000000000..12edd9c65b2
--- /dev/null
+++ b/ompi/mpi/c/file_set_errhandler.c.in
@@ -0,0 +1,70 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2017 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+
+PROTOTYPE ERROR_CLASS file_set_errhandler(FILE file, ERRHANDLER errhandler)
+{
+    MPI_Errhandler tmp;
+
+    /* Error checking */
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        /* Note that MPI-2:9.7 (p265 in the ps; p261 in the pdf)
+           explicitly says that you are allowed to set the error
+           handler on MPI_FILE_NULL */
+
+        if (NULL == file) {
+            return OMPI_ERRHANDLER_INVOKE(MPI_FILE_NULL, MPI_ERR_FILE,
+                                          FUNC_NAME);
+        } else if (NULL == errhandler ||
+                   MPI_ERRHANDLER_NULL == errhandler ||
+                   (OMPI_ERRHANDLER_TYPE_FILE != errhandler->eh_mpi_object_type &&
+		    OMPI_ERRHANDLER_TYPE_PREDEFINED != errhandler->eh_mpi_object_type) ) {
+            return OMPI_ERRHANDLER_INVOKE(file, MPI_ERR_ARG, FUNC_NAME);
+        }
+    }
+
+    /* Prepare the new error handler */
+    OBJ_RETAIN(errhandler);
+
+    OPAL_THREAD_LOCK(&file->f_lock);
+    /* Ditch the old errhandler, and decrement its refcount. */
+    tmp = file->error_handler;
+    file->error_handler = errhandler;
+    OBJ_RELEASE(tmp);
+    OPAL_THREAD_UNLOCK(&file->f_lock);
+
+    /* All done */
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/file_set_info.c b/ompi/mpi/c/file_set_info.c
deleted file mode 100644
index c6d4c49ecdf..00000000000
--- a/ompi/mpi/c/file_set_info.c
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2016-2019 IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-#include "ompi/communicator/communicator.h"
-#include "opal/util/info_subscriber.h"
-#include "ompi/file/file.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_set_info = PMPI_File_set_info
-#endif
-#define MPI_File_set_info PMPI_File_set_info
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_set_info";
-
-
-int MPI_File_set_info(MPI_File fh, MPI_Info info)
-{
-    int ret; 
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_file_invalid(fh)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_FILE, FUNC_NAME);
-        }
-
-	if (NULL == info || MPI_INFO_NULL == info ||
-            ompi_info_is_freed(info)) {
-            return OMPI_ERRHANDLER_INVOKE(fh, MPI_ERR_INFO,
-                                          FUNC_NAME);
-        }
-    }
-
-// Some components we're still letting handle info internally, eg romio321.
-// Components that want to handle it themselves will fill in the get/set
-// info function pointers, components that don't will use NULL.
-    if (fh->f_io_selected_module.v2_0_0.io_module_file_set_info != NULL) {
-        int rc;
-        switch (fh->f_io_version) {
-        case MCA_IO_BASE_V_2_0_0:
-            rc = fh->f_io_selected_module.v2_0_0.
-              io_module_file_set_info(fh, info);
-            break;
-
-        default:
-            rc = MPI_ERR_INTERN;
-            break;
-        }
-        OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-    }
-
-    ret = opal_infosubscribe_change_info(&fh->super, &info->super);
-
-    OMPI_ERRHANDLER_RETURN(ret, fh, ret, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_set_info.c.in b/ompi/mpi/c/file_set_info.c.in
new file mode 100644
index 00000000000..6d0d9eb5225
--- /dev/null
+++ b/ompi/mpi/c/file_set_info.c.in
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016-2019 IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include "ompi/communicator/communicator.h"
+#include "opal/util/info_subscriber.h"
+#include "ompi/file/file.h"
+
+PROTOTYPE ERROR_CLASS file_set_info(FILE fh, INFO info)
+{
+    int ret; 
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_file_invalid(fh)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_FILE, FUNC_NAME);
+        }
+
+	if (NULL == info || MPI_INFO_NULL == info ||
+            ompi_info_is_freed(info)) {
+            return OMPI_ERRHANDLER_INVOKE(fh, MPI_ERR_INFO,
+                                          FUNC_NAME);
+        }
+    }
+
+// Some components we're still letting handle info internally, eg romio321.
+// Components that want to handle it themselves will fill in the get/set
+// info function pointers, components that don't will use NULL.
+    if (fh->f_io_selected_module.v3_0_0.io_module_file_set_info != NULL) {
+        int rc;
+        switch (fh->f_io_version) {
+        case MCA_IO_BASE_V_3_0_0:
+            rc = fh->f_io_selected_module.v3_0_0.
+              io_module_file_set_info(fh, info);
+            break;
+
+        default:
+            rc = MPI_ERR_INTERN;
+            break;
+        }
+        OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+    }
+
+    ret = opal_infosubscribe_change_info(&fh->super, &info->super);
+
+    OMPI_ERRHANDLER_RETURN(ret, fh, ret, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_set_size.c b/ompi/mpi/c/file_set_size.c
deleted file mode 100644
index a966915e97c..00000000000
--- a/ompi/mpi/c/file_set_size.c
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_set_size = PMPI_File_set_size
-#endif
-#define MPI_File_set_size PMPI_File_set_size
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_set_size";
-
-
-int MPI_File_set_size(MPI_File fh, MPI_Offset size)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            rc = MPI_ERR_FILE;
-            fh = MPI_FILE_NULL;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-            io_module_file_set_size(fh, size);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_set_size.c.in b/ompi/mpi/c/file_set_size.c.in
new file mode 100644
index 00000000000..ebd426bcf94
--- /dev/null
+++ b/ompi/mpi/c/file_set_size.c.in
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+
+PROTOTYPE ERROR_CLASS file_set_size(FILE fh, OFFSET size)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            rc = MPI_ERR_FILE;
+            fh = MPI_FILE_NULL;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+            io_module_file_set_size(fh, size);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_set_view.c b/ompi/mpi/c/file_set_view.c
deleted file mode 100644
index 19143546b5b..00000000000
--- a/ompi/mpi/c/file_set_view.c
+++ /dev/null
@@ -1,90 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/info/info.h"
-#include "ompi/file/file.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_set_view = PMPI_File_set_view
-#endif
-#define MPI_File_set_view PMPI_File_set_view
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_set_view";
-
-
-int MPI_File_set_view(MPI_File fh, MPI_Offset disp, MPI_Datatype etype,
-                      MPI_Datatype filetype, const char *datarep, MPI_Info info)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(etype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            rc = MPI_ERR_FILE;
-            fh = MPI_FILE_NULL;
-        } else {
-           OMPI_CHECK_DATATYPE_FOR_VIEW(rc, etype, 0);
-           if (MPI_SUCCESS == rc) {
-               OMPI_CHECK_DATATYPE_FOR_VIEW(rc, filetype, 0);
-           }
-        }
-        if ( NULL == datarep) {
-            rc = MPI_ERR_UNSUPPORTED_DATAREP;
-            fh = MPI_FILE_NULL;            
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-          io_module_file_set_view(fh, disp, etype, filetype, datarep, &(info->super));
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_set_view.c.in b/ompi/mpi/c/file_set_view.c.in
new file mode 100644
index 00000000000..79ac5c04353
--- /dev/null
+++ b/ompi/mpi/c/file_set_view.c.in
@@ -0,0 +1,82 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2018 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/info/info.h"
+#include "ompi/file/file.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS file_set_view(FILE fh, OFFSET disp, DATATYPE etype,
+                                    DATATYPE filetype, STRING datarep, INFO info)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(etype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            rc = MPI_ERR_FILE;
+            fh = MPI_FILE_NULL;
+        } else {
+           OMPI_CHECK_DATATYPE_FOR_VIEW(rc, etype, 0);
+           if (MPI_SUCCESS == rc) {
+               OMPI_CHECK_DATATYPE_FOR_VIEW(rc, filetype, 0);
+           }
+        }
+        if ( NULL == datarep) {
+            rc = MPI_ERR_UNSUPPORTED_DATAREP;
+            fh = MPI_FILE_NULL;            
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+          io_module_file_set_view(fh, disp, etype, filetype, datarep, &(info->super));
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_sync.c b/ompi/mpi/c/file_sync.c
deleted file mode 100644
index 6369f59ff87..00000000000
--- a/ompi/mpi/c/file_sync.c
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_sync = PMPI_File_sync
-#endif
-#define MPI_File_sync PMPI_File_sync
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_sync";
-
-
-int MPI_File_sync(MPI_File fh)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            rc = MPI_ERR_FILE;
-            fh = MPI_FILE_NULL;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-            io_module_file_sync(fh);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_sync.c.in b/ompi/mpi/c/file_sync.c.in
new file mode 100644
index 00000000000..e8c37ca17cd
--- /dev/null
+++ b/ompi/mpi/c/file_sync.c.in
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+
+PROTOTYPE ERROR_CLASS file_sync(FILE fh)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            rc = MPI_ERR_FILE;
+            fh = MPI_FILE_NULL;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+            io_module_file_sync(fh);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_write.c b/ompi/mpi/c/file_write.c
deleted file mode 100644
index 8bf967d1964..00000000000
--- a/ompi/mpi/c/file_write.c
+++ /dev/null
@@ -1,84 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_write = PMPI_File_write
-#endif
-#define MPI_File_write PMPI_File_write
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_write";
-
-
-int MPI_File_write(MPI_File fh, const void *buf, int count,
-                   MPI_Datatype datatype, MPI_Status *status)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else {
-           OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-          io_module_file_write(fh, buf, count, datatype, status);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_write.c.in b/ompi/mpi/c/file_write.c.in
new file mode 100644
index 00000000000..0f737e7f667
--- /dev/null
+++ b/ompi/mpi/c/file_write.c.in
@@ -0,0 +1,76 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS file_write(FILE fh, BUFFER buf, COUNT count,
+                                 DATATYPE datatype, STATUS_OUT status)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else {
+           OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+          io_module_file_write(fh, buf, count, datatype, status);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_write_all.c b/ompi/mpi/c/file_write_all.c
deleted file mode 100644
index 5feb488e254..00000000000
--- a/ompi/mpi/c/file_write_all.c
+++ /dev/null
@@ -1,84 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_write_all = PMPI_File_write_all
-#endif
-#define MPI_File_write_all PMPI_File_write_all
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_write_all";
-
-
-int MPI_File_write_all(MPI_File fh, const void *buf, int count, MPI_Datatype
-                       datatype, MPI_Status *status)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else {
-           OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-          io_module_file_write_all(fh, buf, count, datatype, status);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_write_all.c.in b/ompi/mpi/c/file_write_all.c.in
new file mode 100644
index 00000000000..ace251e326e
--- /dev/null
+++ b/ompi/mpi/c/file_write_all.c.in
@@ -0,0 +1,76 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS file_write_all(FILE fh, BUFFER buf, COUNT count,
+                                     DATATYPE datatype, STATUS_OUT status)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else {
+           OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+          io_module_file_write_all(fh, buf, count, datatype, status);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_write_all_begin.c b/ompi/mpi/c/file_write_all_begin.c
deleted file mode 100644
index 0dd82cfdc2b..00000000000
--- a/ompi/mpi/c/file_write_all_begin.c
+++ /dev/null
@@ -1,84 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_write_all_begin = PMPI_File_write_all_begin
-#endif
-#define MPI_File_write_all_begin PMPI_File_write_all_begin
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_write_all_begin";
-
-
-int MPI_File_write_all_begin(MPI_File fh, const void *buf, int count,
-                             MPI_Datatype datatype)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else {
-           OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-          io_module_file_write_all_begin(fh, buf, count, datatype);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_write_all_begin.c.in b/ompi/mpi/c/file_write_all_begin.c.in
new file mode 100644
index 00000000000..2ee5280b860
--- /dev/null
+++ b/ompi/mpi/c/file_write_all_begin.c.in
@@ -0,0 +1,76 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS file_write_all_begin(FILE fh, BUFFER buf, COUNT count,
+                                           DATATYPE datatype)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else {
+           OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+          io_module_file_write_all_begin(fh, buf, count, datatype);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_write_all_end.c b/ompi/mpi/c/file_write_all_end.c
deleted file mode 100644
index 01368c70b8a..00000000000
--- a/ompi/mpi/c/file_write_all_end.c
+++ /dev/null
@@ -1,72 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_write_all_end = PMPI_File_write_all_end
-#endif
-#define MPI_File_write_all_end PMPI_File_write_all_end
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_write_all_end";
-
-
-int MPI_File_write_all_end(MPI_File fh, const void *buf, MPI_Status *status)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-          io_module_file_write_all_end(fh, buf, status);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_write_all_end.c.in b/ompi/mpi/c/file_write_all_end.c.in
new file mode 100644
index 00000000000..31f3fa70750
--- /dev/null
+++ b/ompi/mpi/c/file_write_all_end.c.in
@@ -0,0 +1,64 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+
+PROTOTYPE ERROR_CLASS file_write_all_end(FILE fh, BUFFER buf, STATUS_OUT status)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+          io_module_file_write_all_end(fh, buf, status);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_write_at.c b/ompi/mpi/c/file_write_at.c
deleted file mode 100644
index 0c7596c93b6..00000000000
--- a/ompi/mpi/c/file_write_at.c
+++ /dev/null
@@ -1,85 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_write_at = PMPI_File_write_at
-#endif
-#define MPI_File_write_at PMPI_File_write_at
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_write_at";
-
-
-int MPI_File_write_at(MPI_File fh, MPI_Offset offset, const void *buf,
-                      int count, MPI_Datatype datatype,
-                      MPI_Status *status)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else {
-           OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-          io_module_file_write_at(fh, offset, buf, count, datatype, status);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_write_at.c.in b/ompi/mpi/c/file_write_at.c.in
new file mode 100644
index 00000000000..517046900f2
--- /dev/null
+++ b/ompi/mpi/c/file_write_at.c.in
@@ -0,0 +1,77 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS file_write_at(FILE fh, OFFSET offset, BUFFER buf,
+                                    COUNT count, DATATYPE datatype,
+                                    STATUS_OUT status)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else {
+           OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+          io_module_file_write_at(fh, offset, buf, count, datatype, status);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_write_at_all.c b/ompi/mpi/c/file_write_at_all.c
deleted file mode 100644
index 1974cbd8dbc..00000000000
--- a/ompi/mpi/c/file_write_at_all.c
+++ /dev/null
@@ -1,86 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_write_at_all = PMPI_File_write_at_all
-#endif
-#define MPI_File_write_at_all PMPI_File_write_at_all
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_write_at_all";
-
-
-int MPI_File_write_at_all(MPI_File fh, MPI_Offset offset, const void *buf,
-                          int count, MPI_Datatype datatype,
-                          MPI_Status *status)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else {
-           OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-          io_module_file_write_at_all(fh, offset, buf, count, datatype,
-                                        status);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_write_at_all.c.in b/ompi/mpi/c/file_write_at_all.c.in
new file mode 100644
index 00000000000..8bfe767c6fe
--- /dev/null
+++ b/ompi/mpi/c/file_write_at_all.c.in
@@ -0,0 +1,78 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS file_write_at_all(FILE fh, OFFSET offset, BUFFER buf,
+                                        COUNT count, DATATYPE datatype,
+                                        STATUS_OUT status)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else {
+           OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+          io_module_file_write_at_all(fh, offset, buf, count, datatype,
+                                        status);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_write_at_all_begin.c b/ompi/mpi/c/file_write_at_all_begin.c
deleted file mode 100644
index 086bdab8a2c..00000000000
--- a/ompi/mpi/c/file_write_at_all_begin.c
+++ /dev/null
@@ -1,85 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_write_at_all_begin = PMPI_File_write_at_all_begin
-#endif
-#define MPI_File_write_at_all_begin PMPI_File_write_at_all_begin
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_write_at_all_begin";
-
-
-int MPI_File_write_at_all_begin(MPI_File fh, MPI_Offset offset, const void *buf,
-                                int count, MPI_Datatype datatype)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else {
-           OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-          io_module_file_write_at_all_begin(fh, offset, buf, count,
-                                              datatype);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_write_at_all_begin.c.in b/ompi/mpi/c/file_write_at_all_begin.c.in
new file mode 100644
index 00000000000..90e487d816d
--- /dev/null
+++ b/ompi/mpi/c/file_write_at_all_begin.c.in
@@ -0,0 +1,77 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS file_write_at_all_begin(FILE fh, OFFSET offset, BUFFER buf,
+                                              COUNT count, DATATYPE datatype)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else {
+           OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+          io_module_file_write_at_all_begin(fh, offset, buf, count,
+                                              datatype);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_write_at_all_end.c b/ompi/mpi/c/file_write_at_all_end.c
deleted file mode 100644
index 22fa297a841..00000000000
--- a/ompi/mpi/c/file_write_at_all_end.c
+++ /dev/null
@@ -1,72 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_write_at_all_end = PMPI_File_write_at_all_end
-#endif
-#define MPI_File_write_at_all_end PMPI_File_write_at_all_end
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_write_at_all_end";
-
-
-int MPI_File_write_at_all_end(MPI_File fh, const void *buf, MPI_Status *status)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-          io_module_file_write_at_all_end(fh, buf, status);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_write_at_all_end.c.in b/ompi/mpi/c/file_write_at_all_end.c.in
new file mode 100644
index 00000000000..e4c372974f3
--- /dev/null
+++ b/ompi/mpi/c/file_write_at_all_end.c.in
@@ -0,0 +1,64 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+
+PROTOTYPE ERROR_CLASS file_write_at_all_end(FILE fh, BUFFER buf, STATUS_OUT status)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+          io_module_file_write_at_all_end(fh, buf, status);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_write_ordered.c b/ompi/mpi/c/file_write_ordered.c
deleted file mode 100644
index 39dd105c367..00000000000
--- a/ompi/mpi/c/file_write_ordered.c
+++ /dev/null
@@ -1,84 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_write_ordered = PMPI_File_write_ordered
-#endif
-#define MPI_File_write_ordered PMPI_File_write_ordered
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_write_ordered";
-
-
-int MPI_File_write_ordered(MPI_File fh, const void *buf, int count,
-                           MPI_Datatype datatype, MPI_Status *status)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else {
-           OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-          io_module_file_write_ordered(fh, buf, count, datatype, status);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_write_ordered.c.in b/ompi/mpi/c/file_write_ordered.c.in
new file mode 100644
index 00000000000..6b9b1a42e35
--- /dev/null
+++ b/ompi/mpi/c/file_write_ordered.c.in
@@ -0,0 +1,76 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS file_write_ordered(FILE fh, BUFFER buf, COUNT count,
+                                         DATATYPE datatype, STATUS_OUT status)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else {
+           OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+          io_module_file_write_ordered(fh, buf, count, datatype, status);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_write_ordered_begin.c b/ompi/mpi/c/file_write_ordered_begin.c
deleted file mode 100644
index 32b76d78016..00000000000
--- a/ompi/mpi/c/file_write_ordered_begin.c
+++ /dev/null
@@ -1,84 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_write_ordered_begin = PMPI_File_write_ordered_begin
-#endif
-#define MPI_File_write_ordered_begin PMPI_File_write_ordered_begin
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_write_ordered_begin";
-
-
-int MPI_File_write_ordered_begin(MPI_File fh, const void *buf, int count,
-                                 MPI_Datatype datatype)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else {
-           OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-          io_module_file_write_ordered_begin(fh, buf, count, datatype);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_write_ordered_begin.c.in b/ompi/mpi/c/file_write_ordered_begin.c.in
new file mode 100644
index 00000000000..68ce2c2fe62
--- /dev/null
+++ b/ompi/mpi/c/file_write_ordered_begin.c.in
@@ -0,0 +1,76 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS file_write_ordered_begin(FILE fh, BUFFER buf, COUNT count,
+                                               DATATYPE datatype)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else {
+           OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+          io_module_file_write_ordered_begin(fh, buf, count, datatype);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_write_ordered_end.c b/ompi/mpi/c/file_write_ordered_end.c
deleted file mode 100644
index 9369788e48b..00000000000
--- a/ompi/mpi/c/file_write_ordered_end.c
+++ /dev/null
@@ -1,72 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_write_ordered_end = PMPI_File_write_ordered_end
-#endif
-#define MPI_File_write_ordered_end PMPI_File_write_ordered_end
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_write_ordered_end";
-
-
-int MPI_File_write_ordered_end(MPI_File fh, const void *buf, MPI_Status *status)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-          io_module_file_write_ordered_end(fh, buf, status);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_write_ordered_end.c.in b/ompi/mpi/c/file_write_ordered_end.c.in
new file mode 100644
index 00000000000..d2c95c5bcf5
--- /dev/null
+++ b/ompi/mpi/c/file_write_ordered_end.c.in
@@ -0,0 +1,64 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+
+PROTOTYPE ERROR_CLASS file_write_ordered_end(FILE fh, BUFFER buf, STATUS_OUT status)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+          io_module_file_write_ordered_end(fh, buf, status);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/file_write_shared.c b/ompi/mpi/c/file_write_shared.c
deleted file mode 100644
index 239be859170..00000000000
--- a/ompi/mpi/c/file_write_shared.c
+++ /dev/null
@@ -1,84 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/file/file.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_File_write_shared = PMPI_File_write_shared
-#endif
-#define MPI_File_write_shared PMPI_File_write_shared
-#endif
-
-static const char FUNC_NAME[] = "MPI_File_write_shared";
-
-
-int MPI_File_write_shared(MPI_File fh, const void *buf, int count,
-                          MPI_Datatype datatype, MPI_Status *status)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_file_invalid(fh)) {
-            fh = MPI_FILE_NULL;
-            rc = MPI_ERR_FILE;
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else {
-           OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-
-    switch (fh->f_io_version) {
-    case MCA_IO_BASE_V_2_0_0:
-        rc = fh->f_io_selected_module.v2_0_0.
-          io_module_file_write_shared(fh, buf, count, datatype, status);
-        break;
-
-    default:
-        rc = MPI_ERR_INTERN;
-        break;
-    }
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/file_write_shared.c.in b/ompi/mpi/c/file_write_shared.c.in
new file mode 100644
index 00000000000..be4183cfe70
--- /dev/null
+++ b/ompi/mpi/c/file_write_shared.c.in
@@ -0,0 +1,76 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS file_write_shared(FILE fh, BUFFER buf, COUNT count,
+                                        DATATYPE datatype, STATUS_OUT status)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_file_invalid(fh)) {
+            fh = MPI_FILE_NULL;
+            rc = MPI_ERR_FILE;
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else {
+           OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, fh, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+
+    switch (fh->f_io_version) {
+    case MCA_IO_BASE_V_3_0_0:
+        rc = fh->f_io_selected_module.v3_0_0.
+          io_module_file_write_shared(fh, buf, count, datatype, status);
+        break;
+
+    default:
+        rc = MPI_ERR_INTERN;
+        break;
+    }
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, fh, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/finalize.c b/ompi/mpi/c/finalize.c
deleted file mode 100644
index be7989261ba..00000000000
--- a/ompi/mpi/c/finalize.c
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2018 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Finalize = PMPI_Finalize
-#endif
-#define MPI_Finalize PMPI_Finalize
-#endif
-
-static const char FUNC_NAME[] = "MPI_Finalize";
-
-
-int MPI_Finalize(void)
-{
-    /* If --with-spc and ompi_mpi_spc_dump_enabled were specified, print
-     * all of the final SPC values aggregated across the whole MPI run.
-     * Also, free all SPC memory.
-     */
-    SPC_FINI();
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-    }
-
-    /* Pretty simple */
-
-    return ompi_mpi_finalize();
-}
diff --git a/ompi/mpi/c/finalize.c.in b/ompi/mpi/c/finalize.c.in
new file mode 100644
index 00000000000..d260e699510
--- /dev/null
+++ b/ompi/mpi/c/finalize.c.in
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS finalize()
+{
+    /* If --with-spc and ompi_mpi_spc_dump_enabled were specified, print
+     * all of the final SPC values aggregated across the whole MPI run.
+     * Also, free all SPC memory.
+     */
+    SPC_FINI();
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+    }
+
+    /* Pretty simple */
+
+    return ompi_mpi_finalize();
+}
diff --git a/ompi/mpi/c/finalized.c b/ompi/mpi/c/finalized.c
deleted file mode 100644
index 514e91b3b25..00000000000
--- a/ompi/mpi/c/finalized.c
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2015-2018 Cisco Systems, Inc.  All rights reserved
- * Copyright (c) 2015      Intel, Inc. All rights reserved
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/hook/base/base.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Finalized = PMPI_Finalized
-#endif
-#define MPI_Finalized PMPI_Finalized
-#endif
-
-static const char FUNC_NAME[] = "MPI_Finalized";
-
-
-int MPI_Finalized(int *flag)
-{
-    ompi_hook_base_mpi_finalized_top(flag);
-
-    int32_t state = ompi_mpi_state;
-
-    if (MPI_PARAM_CHECK) {
-        if (NULL == flag) {
-
-            /* If we have an error, the action that we take depends on
-               whether we're currently (after MPI_Init and before
-               MPI_Finalize) or not */
-
-            if (state >= OMPI_MPI_STATE_INIT_COMPLETED &&
-                state < OMPI_MPI_STATE_FINALIZE_PAST_COMM_SELF_DESTRUCT) {
-                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                              FUNC_NAME);
-            } else {
-                /* We have no MPI object here so call ompi_errhandle_invoke
-                 * directly */
-                return ompi_errhandler_invoke(NULL, NULL, -1,
-                                              ompi_errcode_get_mpi_code(MPI_ERR_ARG),
-                                              FUNC_NAME);
-            }
-        }
-    }
-
-    *flag = (state >= OMPI_MPI_STATE_FINALIZE_PAST_COMM_SELF_DESTRUCT);
-
-    ompi_hook_base_mpi_finalized_bottom(flag);
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/finalized.c.in b/ompi/mpi/c/finalized.c.in
new file mode 100644
index 00000000000..76abbbf4eb4
--- /dev/null
+++ b/ompi/mpi/c/finalized.c.in
@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015-2018 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2015      Intel, Inc. All rights reserved
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/hook/base/base.h"
+
+PROTOTYPE ERROR_CLASS finalized(INT_OUT flag)
+{
+    ompi_hook_base_mpi_finalized_top(flag);
+
+    int32_t state = ompi_mpi_state;
+
+    if (MPI_PARAM_CHECK) {
+        if (NULL == flag) {
+
+            /* If we have an error, the action that we take depends on
+               whether we're currently (after MPI_Init and before
+               MPI_Finalize) or not */
+
+            if (state >= OMPI_MPI_STATE_INIT_COMPLETED &&
+                state < OMPI_MPI_STATE_FINALIZE_PAST_COMM_SELF_DESTRUCT) {
+                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                              FUNC_NAME);
+            } else {
+                /* We have no MPI object here so call ompi_errhandle_invoke
+                 * directly */
+                return ompi_errhandler_invoke(NULL, NULL, -1,
+                                              ompi_errcode_get_mpi_code(MPI_ERR_ARG),
+                                              FUNC_NAME);
+            }
+        }
+    }
+
+    *flag = (state >= OMPI_MPI_STATE_FINALIZE_PAST_COMM_SELF_DESTRUCT);
+
+    ompi_hook_base_mpi_finalized_bottom(flag);
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/free_mem.c b/ompi/mpi/c/free_mem.c
deleted file mode 100644
index 5c0bbc4eb6a..00000000000
--- a/ompi/mpi/c/free_mem.c
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007-2020 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "opal/mca/mpool/mpool.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Free_mem = PMPI_Free_mem
-#endif
-#define MPI_Free_mem PMPI_Free_mem
-#endif
-
-static const char FUNC_NAME[] = "MPI_Free_mem";
-
-
-int MPI_Free_mem(void *baseptr)
-{
-    /* Per these threads:
-
-         https://www.open-mpi.org/community/lists/devel/2007/07/1977.php
-         https://www.open-mpi.org/community/lists/devel/2007/07/1979.php
-
-       If you call MPI_ALLOC_MEM with a size of 0, you get NULL
-       back.  So don't consider a NULL==baseptr an error. */
-    if (NULL != baseptr && OMPI_SUCCESS != mca_mpool_base_free(baseptr)) {
-        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_NO_MEM, FUNC_NAME);
-    }
-
-    return MPI_SUCCESS;
-}
-
diff --git a/ompi/mpi/c/free_mem.c.in b/ompi/mpi/c/free_mem.c.in
new file mode 100644
index 00000000000..fa68d9fb67f
--- /dev/null
+++ b/ompi/mpi/c/free_mem.c.in
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007-2020 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "opal/mca/mpool/mpool.h"
+
+PROTOTYPE ERROR_CLASS free_mem(BUFFER_OUT baseptr)
+{
+    /* Per these threads:
+
+         https://www.open-mpi.org/community/lists/devel/2007/07/1977.php
+         https://www.open-mpi.org/community/lists/devel/2007/07/1979.php
+
+       If you call MPI_ALLOC_MEM with a size of 0, you get NULL
+       back.  So don't consider a NULL==baseptr an error. */
+    if (NULL != baseptr && OMPI_SUCCESS != mca_mpool_base_free(baseptr)) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_NO_MEM, FUNC_NAME);
+    }
+
+    return MPI_SUCCESS;
+}
+
diff --git a/ompi/mpi/c/gather.c b/ompi/mpi/c/gather.c
deleted file mode 100644
index b946023caff..00000000000
--- a/ompi/mpi/c/gather.c
+++ /dev/null
@@ -1,198 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2008      University of Houston.  All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Gather = PMPI_Gather
-#endif
-#define MPI_Gather PMPI_Gather
-#endif
-
-static const char FUNC_NAME[] = "MPI_Gather";
-
-
-int MPI_Gather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-               void *recvbuf, int recvcount, MPI_Datatype recvtype,
-               int root, MPI_Comm comm)
-{
-    int err;
-
-    SPC_RECORD(OMPI_SPC_GATHER, 1);
-
-    MEMCHECKER(
-        int rank;
-        ptrdiff_t ext;
-
-        rank = ompi_comm_rank(comm);
-        ompi_datatype_type_extent(recvtype, &ext);
-
-        memchecker_comm(comm);
-        if(OMPI_COMM_IS_INTRA(comm)) {
-            if(ompi_comm_rank(comm) == root) {
-                /* check whether root's send buffer is defined. */
-                if (MPI_IN_PLACE == sendbuf) {
-                  memchecker_call(&opal_memchecker_base_isdefined,
-                                  (char *)(recvbuf)+rank*ext,
-                                  recvcount, recvtype);
-                } else {
-                    memchecker_datatype(sendtype);
-                    memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-                }
-
-                memchecker_datatype(recvtype);
-                /* check whether root's receive buffer is addressable. */
-                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-            } else {
-                memchecker_datatype(sendtype);
-                /* check whether send buffer is defined on other processes. */
-                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-            }
-        } else {
-            if (MPI_ROOT == root) {
-                memchecker_datatype(recvtype);
-                /* check whether root's receive buffer is addressable. */
-                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-            } else if (MPI_PROC_NULL != root) {
-                memchecker_datatype(sendtype);
-                /* check whether send buffer is defined. */
-                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-            }
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if ((ompi_comm_rank(comm) != root && MPI_IN_PLACE == sendbuf) ||
-                   (ompi_comm_rank(comm) == root && MPI_IN_PLACE == recvbuf)) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-
-        /* Errors for intracommunicators */
-
-        if (OMPI_COMM_IS_INTRA(comm)) {
-
-            /* Errors for all ranks */
-
-            if ((root >= ompi_comm_size(comm)) || (root < 0)) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
-            }
-
-            if (MPI_IN_PLACE != sendbuf) {
-                OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-            }
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-
-            /* Errors for the root.  Some of these could have been
-               combined into compound if statements above, but since
-               this whole section can be compiled out (or turned off at
-               run time) for efficiency, it's more clear to separate
-               them out into individual tests. */
-
-            if (ompi_comm_rank(comm) == root) {
-                if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
-                }
-
-                if (recvcount < 0) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-                }
-            }
-        }
-
-        /* Errors for intercommunicators */
-
-        else {
-            if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
-                   MPI_ROOT == root || MPI_PROC_NULL == root)) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
-            }
-
-            /* Errors for the senders */
-
-            if (MPI_ROOT != root && MPI_PROC_NULL != root) {
-                OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-                OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-            }
-
-            /* Errors for the root.  Ditto on the comment above -- these
-               error checks could have been combined above, but let's
-               make the code easier to read. */
-
-            else if (MPI_ROOT == root) {
-                if (recvcount < 0) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-                }
-
-                if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
-                }
-            }
-        }
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
-        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-    }
-#endif
-
-    /* Do we need to do anything? */
-
-    if ((0 == sendcount && MPI_ROOT != root &&
-         (ompi_comm_rank(comm) != root ||
-          (ompi_comm_rank(comm) == root && MPI_IN_PLACE != sendbuf))) ||
-        (ompi_comm_rank(comm) == root && MPI_IN_PLACE == sendbuf &&
-         0 == recvcount) ||
-        (0 == recvcount && (MPI_ROOT == root || MPI_PROC_NULL == root))) {
-        return MPI_SUCCESS;
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_gather(sendbuf, sendcount, sendtype, recvbuf,
-                                   recvcount, recvtype, root, comm,
-                                   comm->c_coll->coll_gather_module);
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/gather.c.in b/ompi/mpi/c/gather.c.in
new file mode 100644
index 00000000000..e071da8d748
--- /dev/null
+++ b/ompi/mpi/c/gather.c.in
@@ -0,0 +1,197 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2023 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2008      University of Houston.  All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS gather(BUFFER sendbuf, COUNT sendcount, DATATYPE sendtype,
+                             BUFFER_OUT recvbuf, COUNT recvcount, DATATYPE recvtype,
+                             INT root, COMM comm)
+{
+    int err;
+
+    SPC_RECORD(OMPI_SPC_GATHER, 1);
+
+    MEMCHECKER(
+        int rank;
+        ptrdiff_t ext;
+
+        rank = ompi_comm_rank(comm);
+
+        memchecker_comm(comm);
+        if(OMPI_COMM_IS_INTRA(comm)) {
+            if(ompi_comm_rank(comm) == root) {
+                /* check whether root's send buffer is defined. */
+                memchecker_datatype(recvtype);
+                ompi_datatype_type_extent(recvtype, &ext);
+                if (MPI_IN_PLACE == sendbuf) {
+                  memchecker_call(&opal_memchecker_base_isdefined,
+                                  (char *)(recvbuf)+rank*ext,
+                                  recvcount, recvtype);
+                } else {
+                    memchecker_datatype(sendtype);
+                    memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+                }
+
+                memchecker_datatype(recvtype);
+                /* check whether root's receive buffer is addressable. */
+                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+            } else {
+                memchecker_datatype(sendtype);
+                /* check whether send buffer is defined on other processes. */
+                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+            }
+        } else {
+            if (MPI_ROOT == root) {
+                memchecker_datatype(recvtype);
+                /* check whether root's receive buffer is addressable. */
+                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+            } else if (MPI_PROC_NULL != root) {
+                memchecker_datatype(sendtype);
+                /* check whether send buffer is defined. */
+                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+            }
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if ((ompi_comm_rank(comm) != root && MPI_IN_PLACE == sendbuf) ||
+                   (ompi_comm_rank(comm) == root && MPI_IN_PLACE == recvbuf)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+
+        /* Errors for intracommunicators */
+
+        if (OMPI_COMM_IS_INTRA(comm)) {
+
+            /* Errors for all ranks */
+
+            if ((root >= ompi_comm_size(comm)) || (root < 0)) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
+            }
+
+            if (MPI_IN_PLACE != sendbuf) {
+                OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+            }
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+
+            /* Errors for the root.  Some of these could have been
+               combined into compound if statements above, but since
+               this whole section can be compiled out (or turned off at
+               run time) for efficiency, it's more clear to separate
+               them out into individual tests. */
+
+            if (ompi_comm_rank(comm) == root) {
+                if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
+                }
+
+                if (recvcount < 0) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+                }
+            }
+        }
+
+        /* Errors for intercommunicators */
+
+        else {
+            if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
+                   MPI_ROOT == root || MPI_PROC_NULL == root)) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
+            }
+
+            /* Errors for the senders */
+
+            if (MPI_ROOT != root && MPI_PROC_NULL != root) {
+                OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+                OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+            }
+
+            /* Errors for the root.  Ditto on the comment above -- these
+               error checks could have been combined above, but let's
+               make the code easier to read. */
+
+            else if (MPI_ROOT == root) {
+                if (recvcount < 0) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+                }
+
+                if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
+                }
+            }
+        }
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
+        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+    }
+#endif
+
+    /* Do we need to do anything? */
+
+    if ((0 == sendcount && MPI_ROOT != root &&
+         (ompi_comm_rank(comm) != root ||
+          (ompi_comm_rank(comm) == root && MPI_IN_PLACE != sendbuf))) ||
+        (ompi_comm_rank(comm) == root && MPI_IN_PLACE == sendbuf &&
+         0 == recvcount) ||
+        (0 == recvcount && (MPI_ROOT == root || MPI_PROC_NULL == root))) {
+        return MPI_SUCCESS;
+    }
+    void* updated_recvbuf;
+    if (OMPI_COMM_IS_INTRA(comm)) {
+        updated_recvbuf = (ompi_comm_rank(comm) == root) ? recvbuf : NULL;
+    } else {
+        updated_recvbuf = (root == MPI_ROOT) ? recvbuf : NULL;
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    err = comm->c_coll->coll_gather(sendbuf, sendcount, sendtype, updated_recvbuf,
+                                    recvcount, recvtype, root, comm,
+                                    comm->c_coll->coll_gather_module);
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/gather_init.c.in b/ompi/mpi/c/gather_init.c.in
new file mode 100644
index 00000000000..6022a42bbf9
--- /dev/null
+++ b/ompi/mpi/c/gather_init.c.in
@@ -0,0 +1,195 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2023 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2008      University of Houston.  All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS gather_init(BUFFER sendbuf, COUNT sendcount, DATATYPE sendtype,
+                                  BUFFER_OUT recvbuf, COUNT recvcount, DATATYPE recvtype,
+                                  INT root, COMM comm, INFO info, REQUEST_INOUT request)
+{
+    int err;
+
+    SPC_RECORD(OMPI_SPC_GATHER_INIT, 1);
+
+    MEMCHECKER(
+        int rank;
+        ptrdiff_t ext;
+
+        rank = ompi_comm_rank(comm);
+
+        memchecker_comm(comm);
+        if(OMPI_COMM_IS_INTRA(comm)) {
+            if(ompi_comm_rank(comm) == root) {
+                /* check whether root's send buffer is defined. */
+                if (MPI_IN_PLACE == sendbuf) {
+                  memchecker_datatype(recvtype);
+                  ompi_datatype_type_extent(recvtype, &ext);
+                  memchecker_call(&opal_memchecker_base_isdefined,
+                                  (char *)(recvbuf)+rank*ext,
+                                  recvcount, recvtype);
+                } else {
+                    memchecker_datatype(sendtype);
+                    memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+                }
+
+                memchecker_datatype(recvtype);
+                /* check whether root's receive buffer is addressable. */
+                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+            } else {
+                memchecker_datatype(sendtype);
+                /* check whether send buffer is defined on other processes. */
+                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+            }
+        } else {
+            if (MPI_ROOT == root) {
+                memchecker_datatype(recvtype);
+                /* check whether root's receive buffer is addressable. */
+                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+            } else if (MPI_PROC_NULL != root) {
+                memchecker_datatype(sendtype);
+                /* check whether send buffer is defined. */
+                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+            }
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if ((ompi_comm_rank(comm) != root && MPI_IN_PLACE == sendbuf) ||
+                   (ompi_comm_rank(comm) == root && MPI_IN_PLACE == recvbuf)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+
+        /* Errors for intracommunicators */
+
+        if (OMPI_COMM_IS_INTRA(comm)) {
+
+            /* Errors for all ranks */
+
+            if ((root >= ompi_comm_size(comm)) || (root < 0)) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
+            }
+
+            if (MPI_IN_PLACE != sendbuf) {
+                OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+            }
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+
+            /* Errors for the root.  Some of these could have been
+               combined into compound if statements above, but since
+               this whole section can be compiled out (or turned off at
+               run time) for efficiency, it's more clear to separate
+               them out into individual tests. */
+
+            if (ompi_comm_rank(comm) == root) {
+                if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
+                }
+
+                if (recvcount < 0) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+                }
+            }
+        }
+
+        /* Errors for intercommunicators */
+
+        else {
+            if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
+                   MPI_ROOT == root || MPI_PROC_NULL == root)) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
+            }
+
+            /* Errors for the senders */
+
+            if (MPI_ROOT != root && MPI_PROC_NULL != root) {
+                OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+                OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+            }
+
+            /* Errors for the root.  Ditto on the comment above -- these
+               error checks could have been combined above, but let's
+               make the code easier to read. */
+
+            else if (MPI_ROOT == root) {
+                if (recvcount < 0) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+                }
+
+                if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
+                }
+            }
+        }
+    }
+
+    void* updated_recvbuf;
+    if (OMPI_COMM_IS_INTRA(comm)) {
+        updated_recvbuf = (ompi_comm_rank(comm) == root) ? recvbuf : NULL;
+    } else {
+        updated_recvbuf = (root == MPI_ROOT) ? recvbuf : NULL;
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    err = comm->c_coll->coll_gather_init(sendbuf, sendcount, sendtype, updated_recvbuf,
+                                         recvcount, recvtype, root, comm, info, request,
+                                         comm->c_coll->coll_gather_init_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        if (OMPI_COMM_IS_INTRA(comm)) {
+            if (MPI_IN_PLACE == sendbuf) {
+                sendtype = NULL;
+            } else if (ompi_comm_rank(comm) != root) {
+                recvtype = NULL;
+            }
+        } else {
+            if (MPI_ROOT == root) {
+                sendtype = NULL;
+            } else if (MPI_PROC_NULL == root) {
+                sendtype = NULL;
+                recvtype = NULL;
+            } else {
+                recvtype = NULL;
+            }
+        }
+        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/gatherv.c b/ompi/mpi/c/gatherv.c
deleted file mode 100644
index 3aefc403ac0..00000000000
--- a/ompi/mpi/c/gatherv.c
+++ /dev/null
@@ -1,211 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2016 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Gatherv = PMPI_Gatherv
-#endif
-#define MPI_Gatherv PMPI_Gatherv
-#endif
-
-static const char FUNC_NAME[] = "MPI_Gatherv";
-
-
-int MPI_Gatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                void *recvbuf, const int recvcounts[], const int displs[],
-                MPI_Datatype recvtype, int root, MPI_Comm comm)
-{
-    int i, size, err;
-
-    SPC_RECORD(OMPI_SPC_GATHERV, 1);
-
-    MEMCHECKER(
-        ptrdiff_t ext;
-
-        size = ompi_comm_remote_size(comm);
-        ompi_datatype_type_extent(recvtype, &ext);
-
-        memchecker_comm(comm);
-        if(OMPI_COMM_IS_INTRA(comm)) {
-            if(ompi_comm_rank(comm) == root) {
-                /* check whether root's send buffer is defined. */
-                if (MPI_IN_PLACE == sendbuf) {
-                    for (i = 0; i < size; i++) {
-                        memchecker_call(&opal_memchecker_base_isdefined,
-                                        (char *)(recvbuf)+displs[i]*ext,
-                                        recvcounts[i], recvtype);
-                    }
-                } else {
-                    memchecker_datatype(sendtype);
-                    memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-                }
-
-                memchecker_datatype(recvtype);
-                /* check whether root's receive buffer is addressable. */
-                for (i = 0; i < size; i++) {
-                    memchecker_call(&opal_memchecker_base_isaddressable,
-                                    (char *)(recvbuf)+displs[i]*ext,
-                                    recvcounts[i], recvtype);
-                }
-            } else {
-                memchecker_datatype(sendtype);
-                /* check whether send buffer is defined on other processes. */
-                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-            }
-        } else {
-            if (MPI_ROOT == root) {
-                memchecker_datatype(recvtype);
-                /* check whether root's receive buffer is addressable. */
-                for (i = 0; i < size; i++) {
-                    memchecker_call(&opal_memchecker_base_isaddressable,
-                                    (char *)(recvbuf)+displs[i]*ext,
-                                    recvcounts[i], recvtype);
-                }
-            } else if (MPI_PROC_NULL != root) {
-                memchecker_datatype(sendtype);
-                /* check whether send buffer is defined. */
-                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-            }
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if ((ompi_comm_rank(comm) != root && MPI_IN_PLACE == sendbuf) ||
-                   (ompi_comm_rank(comm) == root && MPI_IN_PLACE == recvbuf)) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-
-        /* Errors for intracommunicators */
-
-        if (OMPI_COMM_IS_INTRA(comm)) {
-
-            /* Errors for all ranks */
-
-            if ((root >= ompi_comm_size(comm)) || (root < 0)) {
-                err = MPI_ERR_ROOT;
-            } else if (MPI_IN_PLACE != sendbuf) {
-                OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-            }
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-
-            /* Errors for the root.  Some of these could have been
-               combined into compound if statements above, but since
-               this whole section can be compiled out (or turned off at
-               run time) for efficiency, it's more clear to separate
-               them out into individual tests. */
-
-            if (ompi_comm_rank(comm) == root) {
-                if (NULL == displs) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-                }
-
-                if (NULL == recvcounts) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-                }
-
-                size = ompi_comm_size(comm);
-                for (i = 0; i < size; ++i) {
-                    if (recvcounts[i] < 0) {
-                        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-                    } else if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
-                        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
-                    }
-                }
-            }
-        }
-
-        /* Errors for intercommunicators */
-
-        else {
-            if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
-                   MPI_ROOT == root || MPI_PROC_NULL == root)) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
-            }
-
-            /* Errors for the senders */
-
-            if (MPI_ROOT != root && MPI_PROC_NULL != root) {
-                OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-                OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-            }
-
-            /* Errors for the root.  Ditto on the comment above -- these
-               error checks could have been combined above, but let's
-               make the code easier to read. */
-
-            else if (MPI_ROOT == root) {
-                if (NULL == displs) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-                }
-
-                if (NULL == recvcounts) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-                }
-
-                size = ompi_comm_remote_size(comm);
-                for (i = 0; i < size; ++i) {
-                    if (recvcounts[i] < 0) {
-                        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-                    } else if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
-                        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
-                    }
-                }
-            }
-        }
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
-        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-    }
-#endif
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_gatherv(sendbuf, sendcount, sendtype, recvbuf,
-                                    recvcounts, displs,
-                                    recvtype, root, comm,
-                                    comm->c_coll->coll_gatherv_module);
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/gatherv.c.in b/ompi/mpi/c/gatherv.c.in
new file mode 100644
index 00000000000..1d5dbe469ab
--- /dev/null
+++ b/ompi/mpi/c/gatherv.c.in
@@ -0,0 +1,215 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2023 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2016 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS gatherv(BUFFER sendbuf, COUNT sendcount, DATATYPE sendtype,
+                              BUFFER_OUT recvbuf, COUNT_ARRAY recvcounts, DISP_ARRAY displs,
+                              DATATYPE recvtype, INT root, COMM comm)
+{
+    int i, size, err;
+    ompi_count_array_t recvcounts_desc;
+    ompi_disp_array_t displs_desc;
+
+    SPC_RECORD(OMPI_SPC_GATHERV, 1);
+
+    MEMCHECKER(
+        ptrdiff_t ext;
+
+        size = ompi_comm_remote_size(comm);
+
+        memchecker_comm(comm);
+        if(OMPI_COMM_IS_INTRA(comm)) {
+            if(ompi_comm_rank(comm) == root) {
+                memchecker_datatype(recvtype);
+                ompi_datatype_type_extent(recvtype, &ext);
+                /* check whether root's send buffer is defined. */
+                if (MPI_IN_PLACE == sendbuf) {
+                    for (i = 0; i < size; i++) {
+                        memchecker_call(&opal_memchecker_base_isdefined,
+                                        (char *)(recvbuf)+displs[i]*ext,
+                                        recvcounts[i], recvtype);
+                    }
+                } else {
+                    memchecker_datatype(sendtype);
+                    memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+                }
+
+                /* check whether root's receive buffer is addressable. */
+                for (i = 0; i < size; i++) {
+                    memchecker_call(&opal_memchecker_base_isaddressable,
+                                    (char *)(recvbuf)+displs[i]*ext,
+                                    recvcounts[i], recvtype);
+                }
+            } else {
+                memchecker_datatype(sendtype);
+                /* check whether send buffer is defined on other processes. */
+                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+            }
+        } else {
+            if (MPI_ROOT == root) {
+                memchecker_datatype(recvtype);
+                /* check whether root's receive buffer is addressable. */
+                ompi_datatype_type_extent(recvtype, &ext);
+                for (i = 0; i < size; i++) {
+                    memchecker_call(&opal_memchecker_base_isaddressable,
+                                    (char *)(recvbuf)+displs[i]*ext,
+                                    recvcounts[i], recvtype);
+                }
+            } else if (MPI_PROC_NULL != root) {
+                memchecker_datatype(sendtype);
+                /* check whether send buffer is defined. */
+                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+            }
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if ((ompi_comm_rank(comm) != root && MPI_IN_PLACE == sendbuf) ||
+                   (ompi_comm_rank(comm) == root && MPI_IN_PLACE == recvbuf)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+
+        /* Errors for intracommunicators */
+
+        if (OMPI_COMM_IS_INTRA(comm)) {
+
+            /* Errors for all ranks */
+
+            if ((root >= ompi_comm_size(comm)) || (root < 0)) {
+                err = MPI_ERR_ROOT;
+            } else if (MPI_IN_PLACE != sendbuf) {
+                OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+            }
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+
+            /* Errors for the root.  Some of these could have been
+               combined into compound if statements above, but since
+               this whole section can be compiled out (or turned off at
+               run time) for efficiency, it's more clear to separate
+               them out into individual tests. */
+
+            if (ompi_comm_rank(comm) == root) {
+                if (NULL == displs) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+                }
+
+                if (NULL == recvcounts) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+                }
+
+                size = ompi_comm_size(comm);
+                for (i = 0; i < size; ++i) {
+                    if (recvcounts[i] < 0) {
+                        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+                    } else if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
+                        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
+                    }
+                }
+            }
+        }
+
+        /* Errors for intercommunicators */
+
+        else {
+            if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
+                   MPI_ROOT == root || MPI_PROC_NULL == root)) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
+            }
+
+            /* Errors for the senders */
+
+            if (MPI_ROOT != root && MPI_PROC_NULL != root) {
+                OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+                OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+            }
+
+            /* Errors for the root.  Ditto on the comment above -- these
+               error checks could have been combined above, but let's
+               make the code easier to read. */
+
+            else if (MPI_ROOT == root) {
+                if (NULL == displs) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+                }
+
+                if (NULL == recvcounts) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+                }
+
+                size = ompi_comm_remote_size(comm);
+                for (i = 0; i < size; ++i) {
+                    if (recvcounts[i] < 0) {
+                        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+                    } else if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
+                        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
+                    }
+                }
+            }
+        }
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
+        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+    }
+#endif
+
+    void* updated_recvbuf;
+    if (OMPI_COMM_IS_INTRA(comm)) {
+        updated_recvbuf = (ompi_comm_rank(comm) == root) ? recvbuf : NULL;
+    } else {
+        updated_recvbuf = (root == MPI_ROOT) ? recvbuf : NULL;
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    OMPI_COUNT_ARRAY_INIT(&recvcounts_desc, recvcounts);
+    OMPI_DISP_ARRAY_INIT(&displs_desc, displs);
+    err = comm->c_coll->coll_gatherv(sendbuf, sendcount, sendtype, updated_recvbuf,
+                                    recvcounts_desc, displs_desc,
+                                    recvtype, root, comm,
+                                    comm->c_coll->coll_gatherv_module);
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/gatherv_init.c.in b/ompi/mpi/c/gatherv_init.c.in
new file mode 100644
index 00000000000..afb89860f24
--- /dev/null
+++ b/ompi/mpi/c/gatherv_init.c.in
@@ -0,0 +1,223 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2023 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS gatherv_init(BUFFER sendbuf, COUNT sendcount, DATATYPE sendtype,
+                                   BUFFER_OUT recvbuf, COUNT_ARRAY recvcounts, DISP_ARRAY displs,
+                                   DATATYPE recvtype, INT root, COMM comm,
+                                   INFO info, REQUEST_INOUT request)
+{
+    int i, size, err;
+    ompi_count_array_t recvcounts_desc;
+    ompi_disp_array_t displs_desc;
+
+    SPC_RECORD(OMPI_SPC_GATHERV_INIT, 1);
+
+    MEMCHECKER(
+        ptrdiff_t ext;
+
+        size = ompi_comm_remote_size(comm);
+        ompi_datatype_type_extent(recvtype, &ext);
+
+        memchecker_comm(comm);
+        if(OMPI_COMM_IS_INTRA(comm)) {
+            if(ompi_comm_rank(comm) == root) {
+                /* check whether root's send buffer is defined. */
+                if (MPI_IN_PLACE == sendbuf) {
+                    for (i = 0; i < size; i++) {
+                        memchecker_call(&opal_memchecker_base_isdefined,
+                                        (char *)(recvbuf)+displs[i]*ext,
+                                        recvcounts[i], recvtype);
+                    }
+                } else {
+                    memchecker_datatype(sendtype);
+                    memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+                }
+
+                memchecker_datatype(recvtype);
+                /* check whether root's receive buffer is addressable. */
+                for (i = 0; i < size; i++) {
+                    memchecker_call(&opal_memchecker_base_isaddressable,
+                                    (char *)(recvbuf)+displs[i]*ext,
+                                    recvcounts[i], recvtype);
+                }
+            } else {
+                memchecker_datatype(sendtype);
+                /* check whether send buffer is defined on other processes. */
+                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+            }
+        } else {
+            if (MPI_ROOT == root) {
+                memchecker_datatype(recvtype);
+                /* check whether root's receive buffer is addressable. */
+                for (i = 0; i < size; i++) {
+                    memchecker_call(&opal_memchecker_base_isaddressable,
+                                    (char *)(recvbuf)+displs[i]*ext,
+                                    recvcounts[i], recvtype);
+                }
+            } else if (MPI_PROC_NULL != root) {
+                memchecker_datatype(sendtype);
+                /* check whether send buffer is defined. */
+                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+            }
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if ((ompi_comm_rank(comm) != root && MPI_IN_PLACE == sendbuf) ||
+                   (ompi_comm_rank(comm) == root && MPI_IN_PLACE == recvbuf)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+
+        /* Errors for intracommunicators */
+
+        if (OMPI_COMM_IS_INTRA(comm)) {
+
+            /* Errors for all ranks */
+
+            if ((root >= ompi_comm_size(comm)) || (root < 0)) {
+                err = MPI_ERR_ROOT;
+            } else if (MPI_IN_PLACE != sendbuf) {
+                OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+            }
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+
+            /* Errors for the root.  Some of these could have been
+               combined into compound if statements above, but since
+               this whole section can be compiled out (or turned off at
+               run time) for efficiency, it's more clear to separate
+               them out into individual tests. */
+
+            if (ompi_comm_rank(comm) == root) {
+                if (NULL == displs) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+                }
+
+                if (NULL == recvcounts) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+                }
+
+                size = ompi_comm_size(comm);
+                for (i = 0; i < size; ++i) {
+                    if (recvcounts[i] < 0) {
+                        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+                    } else if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
+                        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
+                    }
+                }
+            }
+        }
+
+        /* Errors for intercommunicators */
+
+        else {
+            if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
+                   MPI_ROOT == root || MPI_PROC_NULL == root)) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
+            }
+
+            /* Errors for the senders */
+
+            if (MPI_ROOT != root && MPI_PROC_NULL != root) {
+                OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+                OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+            }
+
+            /* Errors for the root.  Ditto on the comment above -- these
+               error checks could have been combined above, but let's
+               make the code easier to read. */
+
+            else if (MPI_ROOT == root) {
+                if (NULL == displs) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+                }
+
+                if (NULL == recvcounts) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+                }
+
+                size = ompi_comm_remote_size(comm);
+                for (i = 0; i < size; ++i) {
+                    if (recvcounts[i] < 0) {
+                        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+                    } else if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
+                        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
+                    }
+                }
+            }
+        }
+    }
+
+    void* updated_recvbuf;
+    if (OMPI_COMM_IS_INTRA(comm)) {
+        updated_recvbuf = (ompi_comm_rank(comm) == root) ? recvbuf : NULL;
+    } else {
+        updated_recvbuf = (root == MPI_ROOT) ? recvbuf : NULL;
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    OMPI_COUNT_ARRAY_INIT(&recvcounts_desc, recvcounts);
+    OMPI_DISP_ARRAY_INIT(&displs_desc, displs);
+    err = comm->c_coll->coll_gatherv_init(sendbuf, sendcount, sendtype, updated_recvbuf,
+                                          recvcounts_desc, displs_desc, recvtype,
+                                          root, comm, info, request,
+                                          comm->c_coll->coll_gatherv_init_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        if (OMPI_COMM_IS_INTRA(comm)) {
+            if (MPI_IN_PLACE == sendbuf) {
+                sendtype = NULL;
+            } else if (ompi_comm_rank(comm) != root) {
+                recvtype = NULL;
+            }
+        } else {
+            if (MPI_ROOT == root) {
+                sendtype = NULL;
+            } else if (MPI_PROC_NULL == root) {
+                sendtype = NULL;
+                recvtype = NULL;
+            } else {
+                recvtype = NULL;
+            }
+        }
+        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/get.c b/ompi/mpi/c/get.c
deleted file mode 100644
index d378012cfc4..00000000000
--- a/ompi/mpi/c/get.c
+++ /dev/null
@@ -1,83 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-#include "ompi/mca/osc/osc.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Get = PMPI_Get
-#endif
-#define MPI_Get PMPI_Get
-#endif
-
-static const char FUNC_NAME[] = "MPI_Get";
-
-
-int MPI_Get(void *origin_addr, int origin_count,
-            MPI_Datatype origin_datatype, int target_rank,
-            MPI_Aint target_disp, int target_count,
-            MPI_Datatype target_datatype, MPI_Win win)
-{
-    int rc;
-
-    SPC_RECORD(OMPI_SPC_GET, 1);
-
-    if (MPI_PARAM_CHECK) {
-        rc = OMPI_SUCCESS;
-
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        } else if (origin_count < 0 || target_count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else if (ompi_win_peer_invalid(win, target_rank) &&
-                   (MPI_PROC_NULL != target_rank)) {
-            rc = MPI_ERR_RANK;
-        } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) {
-            rc = MPI_ERR_DISP;
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, origin_datatype, origin_count);
-            if (OMPI_SUCCESS == rc) {
-                OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, target_datatype, target_count);
-            }
-        }
-        OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME);
-    }
-
-    if (MPI_PROC_NULL == target_rank) return MPI_SUCCESS;
-
-    rc = win->w_osc_module->osc_get(origin_addr, origin_count, origin_datatype,
-                                    target_rank, target_disp, target_count,
-                                    target_datatype, win);
-    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/get.c.in b/ompi/mpi/c/get.c.in
new file mode 100644
index 00000000000..f8aa7674680
--- /dev/null
+++ b/ompi/mpi/c/get.c.in
@@ -0,0 +1,75 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+#include "ompi/mca/osc/osc.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS get(BUFFER_OUT origin_addr, COUNT origin_count,
+                          DATATYPE origin_datatype, INT target_rank,
+                          AINT target_disp, COUNT target_count,
+                          DATATYPE target_datatype, WIN win)
+{
+    int rc;
+
+    SPC_RECORD(OMPI_SPC_GET, 1);
+
+    if (MPI_PARAM_CHECK) {
+        rc = OMPI_SUCCESS;
+
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        } else if (origin_count < 0 || target_count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else if (ompi_win_peer_invalid(win, target_rank) &&
+                   (MPI_PROC_NULL != target_rank)) {
+            rc = MPI_ERR_RANK;
+        } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) {
+            rc = MPI_ERR_DISP;
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, origin_datatype, origin_count);
+            if (OMPI_SUCCESS == rc) {
+                OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, target_datatype, target_count);
+            }
+        }
+        OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME);
+    }
+
+    if (MPI_PROC_NULL == target_rank) return MPI_SUCCESS;
+
+    rc = win->w_osc_module->osc_get(origin_addr, origin_count, origin_datatype,
+                                    target_rank, target_disp, target_count,
+                                    target_datatype, win);
+    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/get_accumulate.c b/ompi/mpi/c/get_accumulate.c
deleted file mode 100644
index dde86eb385a..00000000000
--- a/ompi/mpi/c/get_accumulate.c
+++ /dev/null
@@ -1,147 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2009      Sun Microsystmes, Inc.  All rights reserved.
- * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
- * Copyright (c) 2014-2015 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-#include "ompi/mca/osc/osc.h"
-#include "ompi/op/op.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/datatype/ompi_datatype_internal.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Get_accumulate = PMPI_Get_accumulate
-#endif
-#define MPI_Get_accumulate PMPI_Get_accumulate
-#endif
-
-static const char FUNC_NAME[] = "MPI_Get_accumulate";
-
-int MPI_Get_accumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype,
-                       void *result_addr, int result_count, MPI_Datatype result_datatype,
-                       int target_rank, MPI_Aint target_disp, int target_count,
-                       MPI_Datatype target_datatype, MPI_Op op, MPI_Win win)
-{
-    int rc;
-    ompi_win_t *ompi_win = (ompi_win_t*) win;
-
-    MEMCHECKER(
-        memchecker_datatype(origin_datatype);
-        memchecker_datatype(target_datatype);
-        memchecker_call(&opal_memchecker_base_isdefined, (void *) origin_addr, origin_count, origin_datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = OMPI_SUCCESS;
-
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        } else if (origin_count < 0 || target_count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else if (ompi_win_peer_invalid(win, target_rank) &&
-                   (MPI_PROC_NULL != target_rank)) {
-            rc = MPI_ERR_RANK;
-        } else if (MPI_OP_NULL == op) {
-            rc = MPI_ERR_OP;
-        } else if (!ompi_op_is_intrinsic(op)) {
-            rc = MPI_ERR_OP;
-        } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) {
-            rc = MPI_ERR_DISP;
-        } else {
-            /* the origin datatype is meaningless when using MPI_OP_NO_OP */
-            if (&ompi_mpi_op_no_op.op != op) {
-                OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, origin_datatype, origin_count);
-            } else {
-                rc = OMPI_SUCCESS;
-            }
-            if (OMPI_SUCCESS == rc) {
-                OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, target_datatype, target_count);
-            }
-            if (OMPI_SUCCESS == rc) {
-                /* While technically the standard probably requires that the
-                   datatypes used with MPI_REPLACE conform to all the rules
-                   for other reduction operators, we don't require such
-                   behavior, as checking for it is expensive here and we don't
-                   care in implementation.. */
-                if (op != &ompi_mpi_op_replace.op && op != &ompi_mpi_op_no_op.op) {
-                    ompi_datatype_t *op_check_dt, *origin_check_dt;
-                    char *msg;
-
-                    /* GET_ACCUMULATE, unlike REDUCE, can use with derived
-                       datatypes with predefinied operations, with some
-                       restrictions outlined in MPI-3:11.3.4.  The derived
-                       datatype must be composed entierly from one predefined
-                       datatype (so you can do all the construction you want,
-                       but at the bottom, you can only use one datatype, say,
-                       MPI_INT).  If the datatype at the target isn't
-                       predefined, then make sure it's composed of only one
-                       datatype, and check that datatype against
-                       ompi_op_is_valid(). */
-                    origin_check_dt = ompi_datatype_get_single_predefined_type_from_args(origin_datatype);
-                    op_check_dt = ompi_datatype_get_single_predefined_type_from_args(target_datatype);
-
-                    if( !((origin_check_dt == op_check_dt) & (NULL != op_check_dt)) ) {
-                        OMPI_ERRHANDLER_RETURN(MPI_ERR_ARG, win, MPI_ERR_ARG, FUNC_NAME);
-                    }
-
-                    /* check to make sure primitive type is valid for
-                       reduction.  Should do this on the target, but
-                       then can't get the errcode back for this
-                       call */
-                    if (!ompi_op_is_valid(op, op_check_dt, &msg, FUNC_NAME)) {
-                        int ret = OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_OP, msg);
-                        free(msg);
-                        return ret;
-                    }
-                }
-            }
-        }
-        OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME);
-    }
-
-    if (MPI_PROC_NULL == target_rank) {
-        return MPI_SUCCESS;
-    }
-
-    rc = ompi_win->w_osc_module->osc_get_accumulate(origin_addr,
-                                                    origin_count,
-                                                    origin_datatype,
-                                                    result_addr,
-                                                    result_count,
-                                                    result_datatype,
-                                                    target_rank,
-                                                    target_disp,
-                                                    target_count,
-                                                    target_datatype,
-                                                    op, win);
-    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/get_accumulate.c.in b/ompi/mpi/c/get_accumulate.c.in
new file mode 100644
index 00000000000..9bcc26b1b3b
--- /dev/null
+++ b/ompi/mpi/c/get_accumulate.c.in
@@ -0,0 +1,140 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2009      Sun Microsystmes, Inc.  All rights reserved.
+ * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2014-2015 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+#include "ompi/mca/osc/osc.h"
+#include "ompi/op/op.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/datatype/ompi_datatype_internal.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS get_accumulate(BUFFER origin_addr, COUNT origin_count, DATATYPE origin_datatype,
+                                     BUFFER_OUT result_addr, COUNT result_count, DATATYPE result_datatype,
+                                     INT target_rank, AINT target_disp, COUNT target_count,
+                                     DATATYPE target_datatype, OP op, WIN win)
+{
+    int rc;
+    ompi_win_t *ompi_win = (ompi_win_t*) win;
+
+    MEMCHECKER(
+        memchecker_datatype(origin_datatype);
+        memchecker_datatype(target_datatype);
+        memchecker_call(&opal_memchecker_base_isdefined, (void *) origin_addr, origin_count, origin_datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = OMPI_SUCCESS;
+
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        } else if (origin_count < 0 || target_count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else if (ompi_win_peer_invalid(win, target_rank) &&
+                   (MPI_PROC_NULL != target_rank)) {
+            rc = MPI_ERR_RANK;
+        } else if (MPI_OP_NULL == op) {
+            rc = MPI_ERR_OP;
+        } else if (!ompi_op_is_intrinsic(op)) {
+            rc = MPI_ERR_OP;
+        } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) {
+            rc = MPI_ERR_DISP;
+        } else {
+            /* the origin datatype is meaningless when using MPI_OP_NO_OP */
+            if (&ompi_mpi_op_no_op.op != op) {
+                OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, origin_datatype, origin_count);
+            } else {
+                rc = OMPI_SUCCESS;
+            }
+            if (OMPI_SUCCESS == rc) {
+                OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, target_datatype, target_count);
+            }
+            if (OMPI_SUCCESS == rc) {
+                /* While technically the standard probably requires that the
+                   datatypes used with MPI_REPLACE conform to all the rules
+                   for other reduction operators, we don't require such
+                   behavior, as checking for it is expensive here and we don't
+                   care in implementation.. */
+                if (op != &ompi_mpi_op_replace.op && op != &ompi_mpi_op_no_op.op) {
+                    ompi_datatype_t *op_check_dt, *origin_check_dt;
+                    char *msg;
+
+                    /* GET_ACCUMULATE, unlike REDUCE, can use with derived
+                       datatypes with predefinied operations, with some
+                       restrictions outlined in MPI-3:11.3.4.  The derived
+                       datatype must be composed entirely from one predefined
+                       datatype (so you can do all the construction you want,
+                       but at the bottom, you can only use one datatype, say,
+                       MPI_INT).  If the datatype at the target isn't
+                       predefined, then make sure it's composed of only one
+                       datatype, and check that datatype against
+                       ompi_op_is_valid(). */
+                    origin_check_dt = ompi_datatype_get_single_predefined_type_from_args(origin_datatype);
+                    op_check_dt = ompi_datatype_get_single_predefined_type_from_args(target_datatype);
+
+                    if( !((origin_check_dt == op_check_dt) & (NULL != op_check_dt)) ) {
+                        OMPI_ERRHANDLER_RETURN(MPI_ERR_ARG, win, MPI_ERR_ARG, FUNC_NAME);
+                    }
+
+                    /* check to make sure primitive type is valid for
+                       reduction.  Should do this on the target, but
+                       then can't get the errcode back for this
+                       call */
+                    if (!ompi_op_is_valid(op, op_check_dt, &msg, FUNC_NAME)) {
+                        int ret = OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_OP, msg);
+                        free(msg);
+                        return ret;
+                    }
+                }
+            }
+        }
+        OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME);
+    }
+
+    if (MPI_PROC_NULL == target_rank) {
+        return MPI_SUCCESS;
+    }
+
+    rc = ompi_win->w_osc_module->osc_get_accumulate(origin_addr,
+                                                    origin_count,
+                                                    origin_datatype,
+                                                    result_addr,
+                                                    result_count,
+                                                    result_datatype,
+                                                    target_rank,
+                                                    target_disp,
+                                                    target_count,
+                                                    target_datatype,
+                                                    op, win);
+    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/get_address.c b/ompi/mpi/c/get_address.c
deleted file mode 100644
index 0b934b2b751..00000000000
--- a/ompi/mpi/c/get_address.c
+++ /dev/null
@@ -1,51 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2013-2016 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Get_address = PMPI_Get_address
-#endif
-#define MPI_Get_address PMPI_Get_address
-#endif
-
-static const char FUNC_NAME[] = "MPI_Get_address";
-
-
-int MPI_Get_address(const void *location, MPI_Aint *address)
-{
-    if( MPI_PARAM_CHECK ) {
-      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-      if (NULL == address) {
-        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-      }
-    }
-
-    *address = (MPI_Aint)location;
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/get_address.c.in b/ompi/mpi/c/get_address.c.in
new file mode 100644
index 00000000000..3aba3b67427
--- /dev/null
+++ b/ompi/mpi/c/get_address.c.in
@@ -0,0 +1,43 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2013-2016 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+
+PROTOTYPE ERROR_CLASS get_address(BUFFER location, AINT_OUT address)
+{
+    if( MPI_PARAM_CHECK ) {
+      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+      if (NULL == address) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+      }
+    }
+
+    *address = (MPI_Aint)location;
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/get_count.c b/ompi/mpi/c/get_count.c
deleted file mode 100644
index d2321a1dcc0..00000000000
--- a/ompi/mpi/c/get_count.c
+++ /dev/null
@@ -1,83 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-#include <limits.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Get_count = PMPI_Get_count
-#endif
-#define MPI_Get_count PMPI_Get_count
-#endif
-
-static const char FUNC_NAME[] = "MPI_Get_count";
-
-
-int MPI_Get_count(const MPI_Status *status, MPI_Datatype datatype, int *count)
-{
-    size_t size = 0, internal_count;
-    int rc      = MPI_SUCCESS;
-
-    MEMCHECKER(
-               if (status != MPI_STATUSES_IGNORE) {
-                   /*
-                    * Before checking the complete status, we need to reset the definedness
-                    * of the MPI_ERROR-field (single-completion calls wait/test).
-                    */
-                   opal_memchecker_base_mem_defined((void*)&status->MPI_ERROR, sizeof(int));
-                   memchecker_status(status);
-                   memchecker_datatype(datatype);
-               }
-               );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, 1);
-
-        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
-    }
-
-    if( ompi_datatype_type_size( datatype, &size ) == MPI_SUCCESS ) {
-        if( size == 0 ) {
-            *count = 0;
-        } else {
-            internal_count = status->_ucount / size; /* count the number of complete datatypes */
-            if( (internal_count * size) != status->_ucount ||
-                internal_count > ((size_t) INT_MAX) ) {
-                *count = MPI_UNDEFINED;
-            } else {
-                *count = (int)internal_count;
-            }
-        }
-    }
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/get_count.c.in b/ompi/mpi/c/get_count.c.in
new file mode 100644
index 00000000000..c2daea1543e
--- /dev/null
+++ b/ompi/mpi/c/get_count.c.in
@@ -0,0 +1,75 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+#include <limits.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS get_count(STATUS status, DATATYPE datatype, COUNT_OUT count)
+{
+    size_t size = 0, internal_count;
+    int rc      = MPI_SUCCESS;
+
+    MEMCHECKER(
+               if (status != MPI_STATUSES_IGNORE) {
+                   /*
+                    * Before checking the complete status, we need to reset the definedness
+                    * of the MPI_ERROR-field (single-completion calls wait/test).
+                    */
+                   opal_memchecker_base_mem_defined((void*)&status->MPI_ERROR, sizeof(int));
+                   memchecker_status(status);
+                   memchecker_datatype(datatype);
+               }
+               );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, 1);
+
+        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
+    }
+
+    if( ompi_datatype_type_size( datatype, &size ) == MPI_SUCCESS ) {
+        if( size == 0 ) {
+            *count = 0;
+        } else {
+            internal_count = status->_ucount / size; /* count the number of complete datatypes */
+            if( (internal_count * size) != status->_ucount ||
+                internal_count > ((size_t) INT_MAX) ) {
+                *count = MPI_UNDEFINED;
+            } else {
+                *count = (int)internal_count;
+            }
+        }
+    }
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/get_elements.c b/ompi/mpi/c/get_elements.c
deleted file mode 100644
index 3789c96f5ca..00000000000
--- a/ompi/mpi/c/get_elements.c
+++ /dev/null
@@ -1,90 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-#include <limits.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Get_elements = PMPI_Get_elements
-#endif
-#define MPI_Get_elements PMPI_Get_elements
-#endif
-
-static const char FUNC_NAME[] = "MPI_Get_elements";
-
-int MPI_Get_elements(const MPI_Status *status, MPI_Datatype datatype, int *count)
-{
-    size_t internal_count;
-    int ret;
-
-    MEMCHECKER(
-               if (status != MPI_STATUSES_IGNORE) {
-                   /*
-                    * Before checking the complete status, we need to reset the definedness
-                    * of the MPI_ERROR-field (single-completion calls wait/test).
-                    */
-                   opal_memchecker_base_mem_defined((void*)&status->MPI_ERROR, sizeof(int));
-                   memchecker_status(status);
-                   memchecker_datatype(datatype);
-               }
-               );
-
-    if (MPI_PARAM_CHECK) {
-        int err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == status || MPI_STATUSES_IGNORE == status ||
-            MPI_STATUS_IGNORE == status || NULL == count) {
-            err = MPI_ERR_ARG;
-        } else if (NULL == datatype || MPI_DATATYPE_NULL == datatype) {
-            err = MPI_ERR_TYPE;
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_RECV(err, datatype, 1);
-        }
-        OMPI_ERRHANDLER_NOHANDLE_CHECK(err, err, FUNC_NAME);
-    }
-
-    ret = ompi_datatype_get_elements (datatype, status->_ucount, &internal_count);
-    if (OMPI_SUCCESS == ret || OMPI_ERR_VALUE_OUT_OF_BOUNDS == ret) {
-        if (OMPI_SUCCESS == ret && internal_count <= INT_MAX) {
-            *count = internal_count;
-        } else {
-            /* If we have more elements that we can represent with a signed int then we must
-             * set count to MPI_UNDEFINED (MPI 3.0).
-             */
-            *count = MPI_UNDEFINED;
-        }
-
-        return MPI_SUCCESS;
-    }
-
-    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/get_elements.c.in b/ompi/mpi/c/get_elements.c.in
new file mode 100644
index 00000000000..49483f92f5d
--- /dev/null
+++ b/ompi/mpi/c/get_elements.c.in
@@ -0,0 +1,82 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+#include <limits.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS Get_elements(STATUS status, DATATYPE datatype, COUNT_OUT count)
+{
+    size_t internal_count;
+    int ret;
+
+    MEMCHECKER(
+               if (status != MPI_STATUSES_IGNORE) {
+                   /*
+                    * Before checking the complete status, we need to reset the definedness
+                    * of the MPI_ERROR-field (single-completion calls wait/test).
+                    */
+                   opal_memchecker_base_mem_defined((void*)&status->MPI_ERROR, sizeof(int));
+                   memchecker_status(status);
+                   memchecker_datatype(datatype);
+               }
+               );
+
+    if (MPI_PARAM_CHECK) {
+        int err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == status || MPI_STATUSES_IGNORE == status ||
+            MPI_STATUS_IGNORE == status || NULL == count) {
+            err = MPI_ERR_ARG;
+        } else if (NULL == datatype || MPI_DATATYPE_NULL == datatype) {
+            err = MPI_ERR_TYPE;
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_RECV(err, datatype, 1);
+        }
+        OMPI_ERRHANDLER_NOHANDLE_CHECK(err, err, FUNC_NAME);
+    }
+
+    ret = ompi_datatype_get_elements (datatype, status->_ucount, &internal_count);
+    if (OMPI_SUCCESS == ret || OMPI_ERR_VALUE_OUT_OF_BOUNDS == ret) {
+        if (OMPI_SUCCESS == ret && internal_count <= INT_MAX) {
+            *count = internal_count;
+        } else {
+            /* If we have more elements that we can represent with a signed int then we must
+             * set count to MPI_UNDEFINED (MPI 3.0).
+             */
+            *count = MPI_UNDEFINED;
+        }
+
+        return MPI_SUCCESS;
+    }
+
+    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/get_elements_x.c b/ompi/mpi/c/get_elements_x.c
deleted file mode 100644
index 3224d0f1cce..00000000000
--- a/ompi/mpi/c/get_elements_x.c
+++ /dev/null
@@ -1,90 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-#include <limits.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Get_elements_x = PMPI_Get_elements_x
-#endif
-#define MPI_Get_elements_x PMPI_Get_elements_x
-#endif
-
-static const char FUNC_NAME[] = "MPI_Get_elements_x";
-
-int MPI_Get_elements_x(const MPI_Status *status, MPI_Datatype datatype, MPI_Count *count)
-{
-    size_t internal_count;
-    int ret;
-
-    MEMCHECKER(
-               if (status != MPI_STATUSES_IGNORE) {
-                   /*
-                    * Before checking the complete status, we need to reset the definedness
-                    * of the MPI_ERROR-field (single-completion calls wait/test).
-                    */
-                   opal_memchecker_base_mem_defined((void*)&status->MPI_ERROR, sizeof(int));
-                   memchecker_status(status);
-                   memchecker_datatype(datatype);
-               }
-               );
-
-    if (MPI_PARAM_CHECK) {
-        int err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == status || MPI_STATUSES_IGNORE == status ||
-            MPI_STATUS_IGNORE == status || NULL == count) {
-            err = MPI_ERR_ARG;
-        } else if (NULL == datatype || MPI_DATATYPE_NULL == datatype) {
-            err = MPI_ERR_TYPE;
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_RECV(err, datatype, 1);
-        }
-        OMPI_ERRHANDLER_NOHANDLE_CHECK(err, err, FUNC_NAME);
-    }
-
-    ret = ompi_datatype_get_elements (datatype, status->_ucount, &internal_count);
-    if (OMPI_SUCCESS == ret || OMPI_ERR_VALUE_OUT_OF_BOUNDS == ret) {
-        if (OMPI_SUCCESS == ret && internal_count <= (size_t) SSIZE_MAX) {
-            *count = internal_count;
-        } else {
-            /* If we have more elements that we can represent with an MPI_Count then we must
-             * set count to MPI_UNDEFINED (MPI 3.0).
-             */
-            *count = MPI_UNDEFINED;
-        }
-
-        return MPI_SUCCESS;
-    }
-
-    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/get_elements_x.c.in b/ompi/mpi/c/get_elements_x.c.in
new file mode 100644
index 00000000000..d301f1336b2
--- /dev/null
+++ b/ompi/mpi/c/get_elements_x.c.in
@@ -0,0 +1,83 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+#include <limits.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS get_elements_x(STATUS status, DATATYPE datatype, ELEMENT_COUNT count)
+{
+    size_t internal_count;
+    int ret;
+
+    MEMCHECKER(
+               if (status != MPI_STATUSES_IGNORE) {
+                   /*
+                    * Before checking the complete status, we need to reset the definedness
+                    * of the MPI_ERROR-field (single-completion calls wait/test).
+                    */
+                   opal_memchecker_base_mem_defined((void*)&status->MPI_ERROR, sizeof(int));
+                   memchecker_status(status);
+                   memchecker_datatype(datatype);
+               }
+               );
+
+    if (MPI_PARAM_CHECK) {
+        int err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == status || MPI_STATUSES_IGNORE == status ||
+            MPI_STATUS_IGNORE == status || NULL == count) {
+            err = MPI_ERR_ARG;
+        } else if (NULL == datatype || MPI_DATATYPE_NULL == datatype) {
+            err = MPI_ERR_TYPE;
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_RECV(err, datatype, 1);
+        }
+        OMPI_ERRHANDLER_NOHANDLE_CHECK(err, err, FUNC_NAME);
+    }
+
+    ret = ompi_datatype_get_elements (datatype, status->_ucount, &internal_count);
+    if (OMPI_SUCCESS == ret || OMPI_ERR_VALUE_OUT_OF_BOUNDS == ret) {
+        if (OMPI_SUCCESS == ret && internal_count <= (size_t) SSIZE_MAX) {
+            *count = internal_count;
+        } else {
+            /* If we have more elements that we can represent with an MPI_Count then we must
+             * set count to MPI_UNDEFINED (MPI 3.0).
+             */
+            *count = MPI_UNDEFINED;
+        }
+
+        return MPI_SUCCESS;
+    }
+
+    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/get_library_version.c b/ompi/mpi/c/get_library_version.c
deleted file mode 100644
index cb82d91c108..00000000000
--- a/ompi/mpi/c/get_library_version.c
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2014-2018 Cisco Systems, Inc.  All rights reserved
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2015      Intel, Inc. All rights reserved
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Get_library_version = PMPI_Get_library_version
-#endif
-#define MPI_Get_library_version PMPI_Get_library_version
-#endif
-
-static const char FUNC_NAME[] = "MPI_Get_library_version";
-
-
-int MPI_Get_library_version(char *version, int *resultlen)
-{
-    int len_left;
-    char *ptr, tmp[MPI_MAX_LIBRARY_VERSION_STRING];
-
-    if (MPI_PARAM_CHECK) {
-        /* Per MPI-3, this function can be invoked before
-           MPI_INIT, so we don't invoke the normal
-           MPI_ERR_INIT_FINALIZE() macro here */
-
-        if (NULL == version || NULL == resultlen) {
-            /* Note that we have to check and see if we have
-               previously called MPI_INIT or not.  If so, use the
-               normal OMPI_ERRHANDLER_INVOKE, because the user may
-               have changed the default errhandler on MPI_COMM_WORLD.
-               If we have not invoked MPI_INIT, then just abort
-               (i.e., use a NULL communicator, which will end up at the
-               default errhandler, which is abort). */
-
-            int32_t state = ompi_mpi_state;
-            if (state >= OMPI_MPI_STATE_INIT_COMPLETED &&
-                state < OMPI_MPI_STATE_FINALIZE_PAST_COMM_SELF_DESTRUCT) {
-                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                              FUNC_NAME);
-            } else {
-                /* We have no MPI object here so call ompi_errhandle_invoke
-                 * directly */
-                return ompi_errhandler_invoke(NULL, NULL, -1,
-                                              ompi_errcode_get_mpi_code(MPI_ERR_ARG),
-                                              FUNC_NAME);
-            }
-        }
-    }
-
-    /* First write to a tmp variable so that we can write to *all* the
-       chars (MPI-3 says that we can only write resultlen chars to the
-       output string) */
-    ptr = tmp;
-    len_left = sizeof(tmp);
-    memset(tmp, 0, MPI_MAX_LIBRARY_VERSION_STRING);
-
-    snprintf(tmp, MPI_MAX_LIBRARY_VERSION_STRING, "Open MPI v%d.%d.%d",
-             OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION, OMPI_RELEASE_VERSION);
-    ptr += strlen(tmp);
-    len_left -= strlen(tmp);
-
-    if (strlen(OMPI_GREEK_VERSION) > 0) {
-        snprintf(ptr, len_left, "%s", OMPI_GREEK_VERSION);
-        ptr = tmp + strlen(tmp);
-        len_left = MPI_MAX_LIBRARY_VERSION_STRING - strlen(tmp);
-    }
-
-    /* Package name */
-    if (strlen(OPAL_PACKAGE_STRING) > 0) {
-        snprintf(ptr, len_left, ", package: %s", OPAL_PACKAGE_STRING);
-        ptr = tmp + strlen(tmp);
-        len_left = MPI_MAX_LIBRARY_VERSION_STRING - strlen(tmp);
-    }
-
-    /* Ident string */
-    if (strlen(OMPI_IDENT_STRING) > 0) {
-        snprintf(ptr, len_left, ", ident: %s", OMPI_IDENT_STRING);
-        ptr = tmp + strlen(tmp);
-        len_left = MPI_MAX_LIBRARY_VERSION_STRING - strlen(tmp);
-    }
-
-    /* Repository revision */
-    if (strlen(OMPI_REPO_REV) > 0) {
-        snprintf(ptr, len_left, ", repo rev: %s", OMPI_REPO_REV);
-        ptr = tmp + strlen(tmp);
-        len_left = MPI_MAX_LIBRARY_VERSION_STRING - strlen(tmp);
-    }
-
-    /* Release date */
-    if (strlen(OMPI_RELEASE_DATE) > 0) {
-        snprintf(ptr, len_left, ", %s", OMPI_RELEASE_DATE);
-        ptr = tmp + strlen(tmp);
-        len_left = MPI_MAX_LIBRARY_VERSION_STRING - strlen(tmp);
-    }
-
-    memcpy(version, tmp, strlen(tmp) + 1);
-    *resultlen = strlen(tmp) + 1;
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/get_library_version.c.in b/ompi/mpi/c/get_library_version.c.in
new file mode 100644
index 00000000000..b8e02e10a7d
--- /dev/null
+++ b/ompi/mpi/c/get_library_version.c.in
@@ -0,0 +1,116 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2014-2018 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015      Intel, Inc. All rights reserved
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+
+PROTOTYPE ERROR_CLASS get_library_version(STRING_OUT version, INT_OUT resultlen)
+{
+    int len_left;
+    char *ptr, tmp[MPI_MAX_LIBRARY_VERSION_STRING];
+
+    if (MPI_PARAM_CHECK) {
+        /* Per MPI-3, this function can be invoked before
+           MPI_INIT, so we don't invoke the normal
+           MPI_ERR_INIT_FINALIZE() macro here */
+
+        if (NULL == version || NULL == resultlen) {
+            /* Note that we have to check and see if we have
+               previously called MPI_INIT or not.  If so, use the
+               normal OMPI_ERRHANDLER_INVOKE, because the user may
+               have changed the default errhandler on MPI_COMM_WORLD.
+               If we have not invoked MPI_INIT, then just abort
+               (i.e., use a NULL communicator, which will end up at the
+               default errhandler, which is abort). */
+
+            int32_t state = ompi_mpi_state;
+            if (state >= OMPI_MPI_STATE_INIT_COMPLETED &&
+                state < OMPI_MPI_STATE_FINALIZE_PAST_COMM_SELF_DESTRUCT) {
+                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                              FUNC_NAME);
+            } else {
+                /* We have no MPI object here so call ompi_errhandle_invoke
+                 * directly */
+                return ompi_errhandler_invoke(NULL, NULL, -1,
+                                              ompi_errcode_get_mpi_code(MPI_ERR_ARG),
+                                              FUNC_NAME);
+            }
+        }
+    }
+
+    /* First write to a tmp variable so that we can write to *all* the
+       chars (MPI-3 says that we can only write resultlen chars to the
+       output string) */
+    ptr = tmp;
+    len_left = sizeof(tmp);
+    memset(tmp, 0, MPI_MAX_LIBRARY_VERSION_STRING);
+
+    snprintf(tmp, MPI_MAX_LIBRARY_VERSION_STRING, "Open MPI v%d.%d.%d",
+             OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION, OMPI_RELEASE_VERSION);
+    ptr += strlen(tmp);
+    len_left -= strlen(tmp);
+
+    if (strlen(OMPI_GREEK_VERSION) > 0) {
+        snprintf(ptr, len_left, "%s", OMPI_GREEK_VERSION);
+        ptr = tmp + strlen(tmp);
+        len_left = MPI_MAX_LIBRARY_VERSION_STRING - strlen(tmp);
+    }
+
+    /* Package name */
+    if (strlen(OPAL_PACKAGE_STRING) > 0) {
+        snprintf(ptr, len_left, ", package: %s", OPAL_PACKAGE_STRING);
+        ptr = tmp + strlen(tmp);
+        len_left = MPI_MAX_LIBRARY_VERSION_STRING - strlen(tmp);
+    }
+
+    /* Ident string */
+    if (strlen(OMPI_IDENT_STRING) > 0) {
+        snprintf(ptr, len_left, ", ident: %s", OMPI_IDENT_STRING);
+        ptr = tmp + strlen(tmp);
+        len_left = MPI_MAX_LIBRARY_VERSION_STRING - strlen(tmp);
+    }
+
+    /* Repository revision */
+    if (strlen(OMPI_REPO_REV) > 0) {
+        snprintf(ptr, len_left, ", repo rev: %s", OMPI_REPO_REV);
+        ptr = tmp + strlen(tmp);
+        len_left = MPI_MAX_LIBRARY_VERSION_STRING - strlen(tmp);
+    }
+
+    /* Release date */
+    if (strlen(OMPI_RELEASE_DATE) > 0) {
+        snprintf(ptr, len_left, ", %s", OMPI_RELEASE_DATE);
+        ptr = tmp + strlen(tmp);
+        len_left = MPI_MAX_LIBRARY_VERSION_STRING - strlen(tmp);
+    }
+
+    memcpy(version, tmp, strlen(tmp) + 1);
+    *resultlen = strlen(tmp) + 1;
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/get_processor_name.c b/ompi/mpi/c/get_processor_name.c
deleted file mode 100644
index c76fd6dcdd3..00000000000
--- a/ompi/mpi/c/get_processor_name.c
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2008 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#include <string.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Get_processor_name = PMPI_Get_processor_name
-#endif
-#define MPI_Get_processor_name PMPI_Get_processor_name
-#endif
-
-static const char FUNC_NAME[] = "MPI_Get_processor_name";
-
-
-int MPI_Get_processor_name(char *name, int *resultlen)
-{
-    if ( MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if ( NULL == name  ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-        if ( NULL == resultlen  ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-    }
-
-    /* A simple implementation of this function using gethostname.
-
-       Note that MPI-2.1 requires:
-       - terminating the string with a \0
-       - name[*resultlen] == '\0'
-       - and therefore (*resultlen) cannot be > (MPI_MAX_PROCESSOR_NAME-1)
-
-       Guard against gethostname() returning a *really long* hostname
-       and not null-terminating the string.  The Fortran API version
-       will pad to the right if necessary. */
-    gethostname(name, (MPI_MAX_PROCESSOR_NAME - 1));
-    name[MPI_MAX_PROCESSOR_NAME - 1] = '\0';
-    *resultlen = (int) strlen(name);
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/get_processor_name.c.in b/ompi/mpi/c/get_processor_name.c.in
new file mode 100644
index 00000000000..9902a653e8f
--- /dev/null
+++ b/ompi/mpi/c/get_processor_name.c.in
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2008 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <string.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+
+PROTOTYPE ERROR_CLASS get_processor_name(STRING_OUT name, INT_OUT resultlen)
+{
+    if ( MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if ( NULL == name  ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+        if ( NULL == resultlen  ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+    /* A simple implementation of this function using gethostname.
+
+       Note that MPI-2.1 requires:
+       - terminating the string with a \0
+       - name[*resultlen] == '\0'
+       - and therefore (*resultlen) cannot be > (MPI_MAX_PROCESSOR_NAME-1)
+
+       Guard against gethostname() returning a *really long* hostname
+       and not null-terminating the string.  The Fortran API version
+       will pad to the right if necessary. */
+    gethostname(name, (MPI_MAX_PROCESSOR_NAME - 1));
+    name[MPI_MAX_PROCESSOR_NAME - 1] = '\0';
+    *resultlen = (int) strlen(name);
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/get_version.c b/ompi/mpi/c/get_version.c
deleted file mode 100644
index 181e84987f0..00000000000
--- a/ompi/mpi/c/get_version.c
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2015      Intel, Inc. All rights reserved
- * Copyright (c) 2018      Cisco Systems, Inc.  All rights reserved
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Get_version = PMPI_Get_version
-#endif
-#define MPI_Get_version PMPI_Get_version
-#endif
-
-static const char FUNC_NAME[] = "MPI_Get_version";
-
-
-int MPI_Get_version(int *version, int *subversion)
-{
-    if (MPI_PARAM_CHECK) {
-        /* Per MPI-2:3.1, this function can be invoked before
-           MPI_INIT, so we don't invoke the normal
-           MPI_ERR_INIT_FINALIZE() macro here */
-
-        if (NULL == version || NULL == subversion) {
-            /* Note that we have to check and see if we have
-               previously called MPI_INIT or not.  If so, use the
-               normal OMPI_ERRHANDLER_INVOKE, because the user may
-               have changed the default errhandler on MPI_COMM_WORLD.
-               If we have not invoked MPI_INIT, then just abort
-               (i.e., use a NULL communicator, which will end up at the
-               default errhandler, which is abort). */
-
-            int32_t state = ompi_mpi_state;
-            if (state >= OMPI_MPI_STATE_INIT_COMPLETED &&
-                state < OMPI_MPI_STATE_FINALIZE_PAST_COMM_SELF_DESTRUCT) {
-                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                              FUNC_NAME);
-            } else {
-                /* We have no MPI object here so call ompi_errhandle_invoke
-                 * directly */
-                return ompi_errhandler_invoke(NULL, NULL, -1,
-                                              ompi_errcode_get_mpi_code(MPI_ERR_ARG),
-                                              FUNC_NAME);
-            }
-        }
-    }
-
-    /* According to the MPI-2 specification */
-
-    *version = MPI_VERSION;
-    *subversion = MPI_SUBVERSION;
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/get_version.c.in b/ompi/mpi/c/get_version.c.in
new file mode 100644
index 00000000000..88abe13653c
--- /dev/null
+++ b/ompi/mpi/c/get_version.c.in
@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015      Intel, Inc. All rights reserved
+ * Copyright (c) 2018      Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+
+PROTOTYPE ERROR_CLASS get_version(INT_OUT version, INT_OUT subversion)
+{
+    if (MPI_PARAM_CHECK) {
+        /* Per MPI-2:3.1, this function can be invoked before
+           MPI_INIT, so we don't invoke the normal
+           MPI_ERR_INIT_FINALIZE() macro here */
+
+        if (NULL == version || NULL == subversion) {
+            /* Note that we have to check and see if we have
+               previously called MPI_INIT or not.  If so, use the
+               normal OMPI_ERRHANDLER_INVOKE, because the user may
+               have changed the default errhandler on MPI_COMM_WORLD.
+               If we have not invoked MPI_INIT, then just abort
+               (i.e., use a NULL communicator, which will end up at the
+               default errhandler, which is abort). */
+
+            int32_t state = ompi_mpi_state;
+            if (state >= OMPI_MPI_STATE_INIT_COMPLETED &&
+                state < OMPI_MPI_STATE_FINALIZE_PAST_COMM_SELF_DESTRUCT) {
+                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                              FUNC_NAME);
+            } else {
+                /* We have no MPI object here so call ompi_errhandle_invoke
+                 * directly */
+                return ompi_errhandler_invoke(NULL, NULL, -1,
+                                              ompi_errcode_get_mpi_code(MPI_ERR_ARG),
+                                              FUNC_NAME);
+            }
+        }
+    }
+
+    /* According to the MPI-2 specification */
+
+    *version = MPI_VERSION;
+    *subversion = MPI_SUBVERSION;
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/graph_create.c b/ompi/mpi/c/graph_create.c
deleted file mode 100644
index c01aed97cc1..00000000000
--- a/ompi/mpi/c/graph_create.c
+++ /dev/null
@@ -1,125 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2012-2013 Inria.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/topo/base/base.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Graph_create = PMPI_Graph_create
-#endif
-#define MPI_Graph_create PMPI_Graph_create
-#endif
-
-static const char FUNC_NAME[] = "MPI_Graph_create";
-
-
-int MPI_Graph_create(MPI_Comm old_comm, int nnodes, const int indx[],
-                     const int edges[], int reorder, MPI_Comm *comm_graph)
-{
-    mca_topo_base_module_t* topo;
-    int err;
-
-    MEMCHECKER(
-        memchecker_comm(old_comm);
-    );
-
-    /* check the arguments */
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(old_comm)) {
-            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_COMM,
-                                           FUNC_NAME);
-        } else if (OMPI_COMM_IS_INTER(old_comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-        if (nnodes < 0) {
-            return OMPI_ERRHANDLER_INVOKE (old_comm, MPI_ERR_ARG,
-                                           FUNC_NAME);
-        } else if (nnodes >= 1 && ((NULL == indx) || (NULL == edges))) {
-            return OMPI_ERRHANDLER_INVOKE (old_comm, MPI_ERR_ARG,
-                                           FUNC_NAME);
-        }
-
-        if (nnodes > ompi_comm_size(old_comm)) {
-            return OMPI_ERRHANDLER_INVOKE (old_comm, MPI_ERR_ARG,
-                                           FUNC_NAME);
-        }
-    }
-
-    /* MPI-2.1 7.5.3 states that if nnodes == 0, all processes should
-       get MPI_COMM_NULL */
-    if (0 == nnodes) {
-        *comm_graph = MPI_COMM_NULL;
-        return MPI_SUCCESS;
-    }
-    if( nnodes > old_comm->c_local_group->grp_proc_count ) {
-        return OMPI_ERRHANDLER_INVOKE (old_comm, MPI_ERR_ARG,
-                                       FUNC_NAME);
-    }
-
-    /*
-     * everything seems to be alright with the communicator, we can go
-     * ahead and select a topology module for this purpose and create
-     * the new graph communicator
-     */
-    if (OMPI_SUCCESS != (err = mca_topo_base_comm_select(old_comm,
-                                                         NULL,
-                                                         &topo,
-                                                         OMPI_COMM_GRAPH))) {
-        return err;
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_create_check(old_comm, &err)) ) {
-        OMPI_ERRHANDLER_RETURN(err, old_comm, err, FUNC_NAME);
-    }
-#endif
-
-    /* Now let that topology module rearrange procs/ranks if it wants to */
-    err = topo->topo.graph.graph_create(topo, old_comm,
-                                        nnodes, indx, edges,
-                                        (0 == reorder) ? false : true, comm_graph);
-    if (MPI_SUCCESS != err) {
-        OBJ_RELEASE(topo);
-        return OMPI_ERRHANDLER_INVOKE(old_comm, err, FUNC_NAME);
-    }
-
-    /* All done */
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/graph_create.c.in b/ompi/mpi/c/graph_create.c.in
new file mode 100644
index 00000000000..e4b1fb4f04c
--- /dev/null
+++ b/ompi/mpi/c/graph_create.c.in
@@ -0,0 +1,117 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2012-2013 Inria.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/topo/base/base.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS graph_create(COMM old_comm, INT nnodes, INT_ARRAY indx,
+                                   INT_ARRAY edges, INT reorder, COMM_OUT comm_graph)
+{
+    mca_topo_base_module_t* topo;
+    int err;
+
+    MEMCHECKER(
+        memchecker_comm(old_comm);
+    );
+
+    /* check the arguments */
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(old_comm)) {
+            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_COMM,
+                                           FUNC_NAME);
+        } else if (OMPI_COMM_IS_INTER(old_comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+        if (nnodes < 0) {
+            return OMPI_ERRHANDLER_INVOKE (old_comm, MPI_ERR_ARG,
+                                           FUNC_NAME);
+        } else if (nnodes >= 1 && ((NULL == indx) || (NULL == edges))) {
+            return OMPI_ERRHANDLER_INVOKE (old_comm, MPI_ERR_ARG,
+                                           FUNC_NAME);
+        }
+
+        if (nnodes > ompi_comm_size(old_comm)) {
+            return OMPI_ERRHANDLER_INVOKE (old_comm, MPI_ERR_ARG,
+                                           FUNC_NAME);
+        }
+    }
+
+    /* MPI-2.1 7.5.3 states that if nnodes == 0, all processes should
+       get MPI_COMM_NULL */
+    if (0 == nnodes) {
+        *comm_graph = MPI_COMM_NULL;
+        return MPI_SUCCESS;
+    }
+    if( nnodes > old_comm->c_local_group->grp_proc_count ) {
+        return OMPI_ERRHANDLER_INVOKE (old_comm, MPI_ERR_ARG,
+                                       FUNC_NAME);
+    }
+
+    /*
+     * everything seems to be alright with the communicator, we can go
+     * ahead and select a topology module for this purpose and create
+     * the new graph communicator
+     */
+    if (OMPI_SUCCESS != (err = mca_topo_base_comm_select(old_comm,
+                                                         NULL,
+                                                         &topo,
+                                                         OMPI_COMM_GRAPH))) {
+        return err;
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_create_check(old_comm, &err)) ) {
+        OMPI_ERRHANDLER_RETURN(err, old_comm, err, FUNC_NAME);
+    }
+#endif
+
+    /* Now let that topology module rearrange procs/ranks if it wants to */
+    err = topo->topo.graph.graph_create(topo, old_comm,
+                                        nnodes, indx, edges,
+                                        (0 == reorder) ? false : true, comm_graph);
+    if (MPI_SUCCESS != err) {
+        OBJ_RELEASE(topo);
+        return OMPI_ERRHANDLER_INVOKE(old_comm, err, FUNC_NAME);
+    }
+
+    /* All done */
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/graph_get.c b/ompi/mpi/c/graph_get.c
deleted file mode 100644
index c96d5027d42..00000000000
--- a/ompi/mpi/c/graph_get.c
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2013 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012      Los Alamos Nat Security, LLC. All rights reserved.
- * Copyright (c) 2012-2013 Inria.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/topo/topo.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Graph_get = PMPI_Graph_get
-#endif
-#define MPI_Graph_get PMPI_Graph_get
-#endif
-
-static const char FUNC_NAME[] = "MPI_Graph_get";
-
-
-int MPI_Graph_get(MPI_Comm comm, int maxindx, int maxedges,
-                  int indx[], int edges[])
-{
-    int err;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    /* check the arguments */
-    if (MPI_PARAM_CHECK) {
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_COMM,
-                                           FUNC_NAME);
-        }
-        if (OMPI_COMM_IS_INTER(comm)) {
-            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_COMM,
-                                           FUNC_NAME);
-        }
-        if (0 > maxindx || 0 > maxedges || NULL == indx || NULL == edges) {
-            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_ARG,
-                                           FUNC_NAME);
-        }
-    }
-
-    if (!OMPI_COMM_IS_GRAPH(comm)) {
-        return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_TOPOLOGY,
-                                       FUNC_NAME);
-    }
-
-    /* call the function */
-    err = comm->c_topo->topo.graph.graph_get(comm, maxindx, maxedges, indx, edges);
-
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/graph_get.c.in b/ompi/mpi/c/graph_get.c.in
new file mode 100644
index 00000000000..ce61cc8a3c4
--- /dev/null
+++ b/ompi/mpi/c/graph_get.c.in
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2013 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Los Alamos Nat Security, LLC. All rights reserved.
+ * Copyright (c) 2012-2013 Inria.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS graph_get(COMM comm, INT maxindx, INT maxedges,
+                                INT_OUT indx, INT_OUT edges)
+{
+    int err;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    /* check the arguments */
+    if (MPI_PARAM_CHECK) {
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_COMM,
+                                           FUNC_NAME);
+        }
+        if (OMPI_COMM_IS_INTER(comm)) {
+            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_COMM,
+                                           FUNC_NAME);
+        }
+        if (0 > maxindx || 0 > maxedges || NULL == indx || NULL == edges) {
+            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_ARG,
+                                           FUNC_NAME);
+        }
+    }
+
+    if (!OMPI_COMM_IS_GRAPH(comm)) {
+        return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_TOPOLOGY,
+                                       FUNC_NAME);
+    }
+
+    /* call the function */
+    err = comm->c_topo->topo.graph.graph_get(comm, maxindx, maxedges, indx, edges);
+
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/graph_map.c b/ompi/mpi/c/graph_map.c
deleted file mode 100644
index 8088b2f3483..00000000000
--- a/ompi/mpi/c/graph_map.c
+++ /dev/null
@@ -1,81 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2013 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2012-2013 Inria.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/topo/topo.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Graph_map = PMPI_Graph_map
-#endif
-#define MPI_Graph_map PMPI_Graph_map
-#endif
-
-static const char FUNC_NAME[] = "MPI_Graph_map";
-
-
-int MPI_Graph_map(MPI_Comm comm, int nnodes, const int indx[], const int edges[],
-                  int *newrank)
-{
-    int err = MPI_SUCCESS;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    /* check the arguments */
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_COMM,
-                                           FUNC_NAME);
-        }
-        if (OMPI_COMM_IS_INTER(comm)) {
-            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_COMM,
-                                           FUNC_NAME);
-        }
-        if (1 > nnodes || NULL == indx || NULL == edges || NULL == newrank) {
-            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_ARG,
-                                           FUNC_NAME);
-        }
-    }
-
-    if(!OMPI_COMM_IS_GRAPH(comm)) {
-        /* In case the communicator has no topo-module attached to
-           it, we just return the "default" value suggested by MPI:
-           newrank = rank */
-        *newrank = ompi_comm_rank(comm);
-    } else {
-      err = comm->c_topo->topo.graph.graph_map(comm, nnodes, indx, edges, newrank);
-    }
-
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/graph_map.c.in b/ompi/mpi/c/graph_map.c.in
new file mode 100644
index 00000000000..d82a346f83b
--- /dev/null
+++ b/ompi/mpi/c/graph_map.c.in
@@ -0,0 +1,73 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2013 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2012-2013 Inria.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS graph_map(COMM comm, INT nnodes, INT_ARRAY indx, INT_ARRAY edges,
+                                INT_OUT newrank)
+{
+    int err = MPI_SUCCESS;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    /* check the arguments */
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_COMM,
+                                           FUNC_NAME);
+        }
+        if (OMPI_COMM_IS_INTER(comm)) {
+            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_COMM,
+                                           FUNC_NAME);
+        }
+        if (1 > nnodes || NULL == indx || NULL == edges || NULL == newrank) {
+            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_ARG,
+                                           FUNC_NAME);
+        }
+    }
+
+    if(!OMPI_COMM_IS_GRAPH(comm)) {
+        /* In case the communicator has no topo-module attached to
+           it, we just return the "default" value suggested by MPI:
+           newrank = rank */
+        *newrank = ompi_comm_rank(comm);
+    } else {
+      err = comm->c_topo->topo.graph.graph_map(comm, nnodes, indx, edges, newrank);
+    }
+
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/graph_neighbors.c b/ompi/mpi/c/graph_neighbors.c
deleted file mode 100644
index 867a841baa0..00000000000
--- a/ompi/mpi/c/graph_neighbors.c
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2013 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012      Los Alamos Nat Security, LLC. All rights reserved.
- * Copyright (c) 2012-2013 Inria.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/topo/topo.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Graph_neighbors = PMPI_Graph_neighbors
-#endif
-#define MPI_Graph_neighbors PMPI_Graph_neighbors
-#endif
-
-static const char FUNC_NAME[] = "MPI_Graph_neighbors";
-
-
-int MPI_Graph_neighbors(MPI_Comm comm, int rank, int maxneighbors,
-                        int neighbors[])
-{
-    int err;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    /* check the arguments */
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_COMM,
-                                           FUNC_NAME);
-        }
-        if (OMPI_COMM_IS_INTER(comm)) {
-            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_COMM,
-                                           FUNC_NAME);
-        }
-        if ((0 > maxneighbors) || ((0 < maxneighbors) && NULL == neighbors)) {
-            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_ARG,
-                                           FUNC_NAME);
-        }
-        if ((0 > rank) || (rank > ompi_group_size(comm->c_local_group))) {
-            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_RANK,
-                                           FUNC_NAME);
-        }
-    }
-
-    if (!OMPI_COMM_IS_GRAPH(comm)) {
-        return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_TOPOLOGY,
-                                       FUNC_NAME);
-    }
-
-    /* call the function */
-    err = comm->c_topo->topo.graph.graph_neighbors(comm, rank, maxneighbors, neighbors);
-
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/graph_neighbors.c.in b/ompi/mpi/c/graph_neighbors.c.in
new file mode 100644
index 00000000000..c5596ed24b4
--- /dev/null
+++ b/ompi/mpi/c/graph_neighbors.c.in
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2013 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Los Alamos Nat Security, LLC. All rights reserved.
+ * Copyright (c) 2012-2013 Inria.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS graph_neighbors(COMM comm, INT rank, INT maxneighbors,
+                                      INT_OUT neighbors)
+{
+    int err;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    /* check the arguments */
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_COMM,
+                                           FUNC_NAME);
+        }
+        if (OMPI_COMM_IS_INTER(comm)) {
+            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_COMM,
+                                           FUNC_NAME);
+        }
+        if ((0 > maxneighbors) || ((0 < maxneighbors) && NULL == neighbors)) {
+            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_ARG,
+                                           FUNC_NAME);
+        }
+        if ((0 > rank) || (rank > ompi_group_size(comm->c_local_group))) {
+            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_RANK,
+                                           FUNC_NAME);
+        }
+    }
+
+    if (!OMPI_COMM_IS_GRAPH(comm)) {
+        return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_TOPOLOGY,
+                                       FUNC_NAME);
+    }
+
+    /* call the function */
+    err = comm->c_topo->topo.graph.graph_neighbors(comm, rank, maxneighbors, neighbors);
+
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/graph_neighbors_count.c b/ompi/mpi/c/graph_neighbors_count.c
deleted file mode 100644
index db1fb11dc7c..00000000000
--- a/ompi/mpi/c/graph_neighbors_count.c
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2013 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Inria.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/topo/topo.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Graph_neighbors_count = PMPI_Graph_neighbors_count
-#endif
-#define MPI_Graph_neighbors_count PMPI_Graph_neighbors_count
-#endif
-
-static const char FUNC_NAME[] = "MPI_Graph_neighbors_count";
-
-
-int MPI_Graph_neighbors_count(MPI_Comm comm, int rank, int *nneighbors)
-{
-    int err;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    /* check the arguments */
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_COMM,
-                                           FUNC_NAME);
-        }
-        if (OMPI_COMM_IS_INTER(comm)) {
-            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_COMM,
-                                           FUNC_NAME);
-        }
-        if ((0 > rank) || (rank > ompi_group_size(comm->c_local_group))) {
-            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_RANK,
-                                           FUNC_NAME);
-        }
-        if (NULL == nneighbors) {
-            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_ARG,
-                                           FUNC_NAME);
-        }
-    }
-
-    if (!OMPI_COMM_IS_GRAPH(comm)) {
-        return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_TOPOLOGY,
-                                       FUNC_NAME);
-    }
-
-    err = comm->c_topo->topo.graph.graph_neighbors_count(comm, rank, nneighbors);
-
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/graph_neighbors_count.c.in b/ompi/mpi/c/graph_neighbors_count.c.in
new file mode 100644
index 00000000000..752c1deaee0
--- /dev/null
+++ b/ompi/mpi/c/graph_neighbors_count.c.in
@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2013 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Inria.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS graph_neighbors_count(COMM comm, INT rank, INT_OUT nneighbors)
+{
+    int err;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    /* check the arguments */
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_COMM,
+                                           FUNC_NAME);
+        }
+        if (OMPI_COMM_IS_INTER(comm)) {
+            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_COMM,
+                                           FUNC_NAME);
+        }
+        if ((0 > rank) || (rank > ompi_group_size(comm->c_local_group))) {
+            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_RANK,
+                                           FUNC_NAME);
+        }
+        if (NULL == nneighbors) {
+            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_ARG,
+                                           FUNC_NAME);
+        }
+    }
+
+    if (!OMPI_COMM_IS_GRAPH(comm)) {
+        return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_TOPOLOGY,
+                                       FUNC_NAME);
+    }
+
+    err = comm->c_topo->topo.graph.graph_neighbors_count(comm, rank, nneighbors);
+
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/graphdims_get.c b/ompi/mpi/c/graphdims_get.c
deleted file mode 100644
index 8af76da95ee..00000000000
--- a/ompi/mpi/c/graphdims_get.c
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2013 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Inria.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/topo/topo.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Graphdims_get = PMPI_Graphdims_get
-#endif
-#define MPI_Graphdims_get PMPI_Graphdims_get
-#endif
-
-static const char FUNC_NAME[] = "MPI_Graphdims_get";
-
-
-int MPI_Graphdims_get(MPI_Comm comm, int *nnodes, int *nedges)
-{
-    int err;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    /* check the arguments */
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_COMM,
-                                           FUNC_NAME);
-        }
-        if (OMPI_COMM_IS_INTER(comm)) {
-            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_COMM,
-                                           FUNC_NAME);
-        }
-        if (NULL == nnodes || NULL == nedges) {
-            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_ARG,
-                                           FUNC_NAME);
-        }
-    }
-
-    if (!OMPI_COMM_IS_GRAPH(comm)) {
-        return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_TOPOLOGY,
-                                       FUNC_NAME);
-    }
-
-    err = comm->c_topo->topo.graph.graphdims_get(comm, nnodes, nedges);
-
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/graphdims_get.c.in b/ompi/mpi/c/graphdims_get.c.in
new file mode 100644
index 00000000000..f505040b5f9
--- /dev/null
+++ b/ompi/mpi/c/graphdims_get.c.in
@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2013 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Inria.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS graphdims_get(COMM comm, INT_OUT nnodes, INT_OUT nedges)
+{
+    int err;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    /* check the arguments */
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_COMM,
+                                           FUNC_NAME);
+        }
+        if (OMPI_COMM_IS_INTER(comm)) {
+            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_COMM,
+                                           FUNC_NAME);
+        }
+        if (NULL == nnodes || NULL == nedges) {
+            return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_ARG,
+                                           FUNC_NAME);
+        }
+    }
+
+    if (!OMPI_COMM_IS_GRAPH(comm)) {
+        return OMPI_ERRHANDLER_INVOKE (comm, MPI_ERR_TOPOLOGY,
+                                       FUNC_NAME);
+    }
+
+    err = comm->c_topo->topo.graph.graphdims_get(comm, nnodes, nedges);
+
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/grequest_complete.c b/ompi/mpi/c/grequest_complete.c
deleted file mode 100644
index 477441af161..00000000000
--- a/ompi/mpi/c/grequest_complete.c
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/request/grequest.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Grequest_complete = PMPI_Grequest_complete
-#endif
-#define MPI_Grequest_complete PMPI_Grequest_complete
-#endif
-
-static const char FUNC_NAME[] = "MPI_Grequest_complete";
-
-
-int MPI_Grequest_complete(MPI_Request request)
-{
-    int rc = MPI_SUCCESS;
-
-    MEMCHECKER(
-        memchecker_request(&request);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (MPI_REQUEST_NULL == request || NULL == request) {
-           rc = MPI_ERR_REQUEST;
-        } else if (OMPI_REQUEST_GEN != request->req_type) {
-            rc = MPI_ERR_REQUEST;
-        }
-        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
-    }
-
-    rc = ompi_grequest_complete(request);
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, MPI_ERR_INTERN, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/grequest_complete.c.in b/ompi/mpi/c/grequest_complete.c.in
new file mode 100644
index 00000000000..8fcf8cf1c1b
--- /dev/null
+++ b/ompi/mpi/c/grequest_complete.c.in
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/request/grequest.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS grequest_complete(REQUEST request)
+{
+    int rc = MPI_SUCCESS;
+
+    MEMCHECKER(
+        memchecker_request(&request);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (MPI_REQUEST_NULL == request || NULL == request) {
+           rc = MPI_ERR_REQUEST;
+        } else if (OMPI_REQUEST_GEN != request->req_type) {
+            rc = MPI_ERR_REQUEST;
+        }
+        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
+    }
+
+    rc = ompi_grequest_complete(request);
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, MPI_ERR_INTERN, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/grequest_start.c b/ompi/mpi/c/grequest_start.c
deleted file mode 100644
index 20077d5a027..00000000000
--- a/ompi/mpi/c/grequest_start.c
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/request/grequest.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Grequest_start = PMPI_Grequest_start
-#endif
-#define MPI_Grequest_start PMPI_Grequest_start
-#endif
-
-static const char FUNC_NAME[] = "MPI_Grequest_start";
-
-
-int MPI_Grequest_start(MPI_Grequest_query_function *query_fn,
-                       MPI_Grequest_free_function *free_fn,
-                       MPI_Grequest_cancel_function *cancel_fn,
-                       void *extra_state, MPI_Request *request)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == request) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_REQUEST,
-                                          FUNC_NAME);
-        }
-    }
-
-    rc = ompi_grequest_start(query_fn,free_fn,cancel_fn,extra_state,request);
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/grequest_start.c.in b/ompi/mpi/c/grequest_start.c.in
new file mode 100644
index 00000000000..75acd2eb372
--- /dev/null
+++ b/ompi/mpi/c/grequest_start.c.in
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/request/grequest.h"
+
+PROTOTYPE ERROR_CLASS grequest_start(GREQUEST_QUERY_FUNCTION query_fn,
+                                     GREQUEST_FREE_FUNCTION free_fn,
+                                     GREQUEST_CANCEL_FUNCTION cancel_fn,
+                                     BUFFER_OUT extra_state, REQUEST_INOUT request)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == request) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_REQUEST,
+                                          FUNC_NAME);
+        }
+    }
+
+    rc = ompi_grequest_start(query_fn,free_fn,cancel_fn,extra_state,request);
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/group_c2f.c b/ompi/mpi/c/group_c2f.c
deleted file mode 100644
index 0c4022548ff..00000000000
--- a/ompi/mpi/c/group_c2f.c
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mpi/fortran/base/fint_2_int.h"
-#include "ompi/group/group.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Group_c2f = PMPI_Group_c2f
-#endif
-#define MPI_Group_c2f PMPI_Group_c2f
-#endif
-
-static const char FUNC_NAME[] = "MPI_Group_c2f";
-
-
-MPI_Fint MPI_Group_c2f(MPI_Group group)
-{
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if( (NULL == group) ) {
-            return OMPI_INT_2_FINT(-1);
-        }
-    }
-
-    return OMPI_INT_2_FINT(group->grp_f_to_c_index) ;
-}
diff --git a/ompi/mpi/c/group_c2f.c.in b/ompi/mpi/c/group_c2f.c.in
new file mode 100644
index 00000000000..c2251619a90
--- /dev/null
+++ b/ompi/mpi/c/group_c2f.c.in
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+#include "ompi/group/group.h"
+
+PROTOTYPE FINT group_c2f(GROUP group)
+{
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if( (NULL == group) ) {
+            return OMPI_INT_2_FINT(-1);
+        }
+    }
+
+    return OMPI_INT_2_FINT(group->grp_f_to_c_index) ;
+}
diff --git a/ompi/mpi/c/group_compare.c b/ompi/mpi/c/group_compare.c
deleted file mode 100644
index f51230713bb..00000000000
--- a/ompi/mpi/c/group_compare.c
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2009      University of Houston. All rights reserved.
- * Copyright (c) 2012      Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/group/group.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/proc/proc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Group_compare = PMPI_Group_compare
-#endif
-#define MPI_Group_compare PMPI_Group_compare
-#endif
-
-static const char FUNC_NAME[] = "MPI_Group_compare";
-
-
-int MPI_Group_compare(MPI_Group group1, MPI_Group group2, int *result) {
-    int return_value = MPI_SUCCESS;
-
-    /* check for errors */
-    if( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if( ( MPI_GROUP_NULL == group1 ) || ( MPI_GROUP_NULL == group2 ) ||
-                (NULL == group1) || (NULL==group2) ){
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_GROUP,
-                                          FUNC_NAME);
-        } else if (NULL == result) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-    }
-
-    return_value = ompi_group_compare((ompi_group_t *)group1, (ompi_group_t *)group2, result);
-
-    return return_value;
-}
diff --git a/ompi/mpi/c/group_compare.c.in b/ompi/mpi/c/group_compare.c.in
new file mode 100644
index 00000000000..728a091396f
--- /dev/null
+++ b/ompi/mpi/c/group_compare.c.in
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2009      University of Houston. All rights reserved.
+ * Copyright (c) 2012      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/group/group.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/proc/proc.h"
+
+PROTOTYPE ERROR_CLASS group_compare(GROUP group1, GROUP group2, INT_OUT result)
+{
+    int return_value = MPI_SUCCESS;
+
+    /* check for errors */
+    if( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if( ( MPI_GROUP_NULL == group1 ) || ( MPI_GROUP_NULL == group2 ) ||
+                (NULL == group1) || (NULL==group2) ){
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_GROUP,
+                                          FUNC_NAME);
+        } else if (NULL == result) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+    return_value = ompi_group_compare((ompi_group_t *)group1, (ompi_group_t *)group2, result);
+
+    return return_value;
+}
diff --git a/ompi/mpi/c/group_difference.c b/ompi/mpi/c/group_difference.c
deleted file mode 100644
index 4036b7f61d4..00000000000
--- a/ompi/mpi/c/group_difference.c
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      University of Houston. All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/group/group.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Group_difference = PMPI_Group_difference
-#endif
-#define MPI_Group_difference PMPI_Group_difference
-#endif
-
-static const char FUNC_NAME[] = "MPI_Group_difference";
-
-
-int MPI_Group_difference(MPI_Group group1, MPI_Group group2,
-                         MPI_Group *new_group) {
-  int err;
-
-  /* error checking */
-  if( MPI_PARAM_CHECK ) {
-      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-      if( (MPI_GROUP_NULL == group1) || (MPI_GROUP_NULL == group2) ||
-          (NULL == group1) || (NULL == group2) ||
-          (NULL == new_group) ) {
-          return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_GROUP,
-                                        FUNC_NAME);
-      }
-  }
-
-  err = ompi_group_difference ( group1, group2,  new_group );
-  OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME );
-}
diff --git a/ompi/mpi/c/group_difference.c.in b/ompi/mpi/c/group_difference.c.in
new file mode 100644
index 00000000000..c3a1cca2fe0
--- /dev/null
+++ b/ompi/mpi/c/group_difference.c.in
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      University of Houston. All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/group/group.h"
+
+PROTOTYPE ERROR_CLASS group_difference(GROUP group1, GROUP group2,
+                         GROUP_OUT new_group)
+{
+  int err;
+
+  /* error checking */
+  if( MPI_PARAM_CHECK ) {
+      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+      if( (MPI_GROUP_NULL == group1) || (MPI_GROUP_NULL == group2) ||
+          (NULL == group1) || (NULL == group2) ||
+          (NULL == new_group) ) {
+          return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_GROUP,
+                                        FUNC_NAME);
+      }
+  }
+
+  err = ompi_group_difference ( group1, group2,  new_group );
+  OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME );
+}
diff --git a/ompi/mpi/c/group_excl.c b/ompi/mpi/c/group_excl.c
deleted file mode 100644
index 7280f592ae0..00000000000
--- a/ompi/mpi/c/group_excl.c
+++ /dev/null
@@ -1,87 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      University of Houston. All rights reserved.
- * Copyright (c) 2006-2009 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos Nat Security, LLC. All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/group/group.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/communicator/communicator.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Group_excl = PMPI_Group_excl
-#endif
-#define MPI_Group_excl PMPI_Group_excl
-#endif
-
-static const char FUNC_NAME[] = "MPI_Group_excl";
-
-
-int MPI_Group_excl(MPI_Group group, int n, const int ranks[],
-                   MPI_Group *new_group)
-{
-    ompi_group_t *group_pointer = (ompi_group_t *)group;
-    int i, err, group_size;
-
-    group_size = ompi_group_size ( group_pointer);
-    if( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        /* verify that group is valid group */
-        if ( (MPI_GROUP_NULL == group)  || (NULL == group) ||
-             (NULL == new_group) ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_GROUP,
-                                          FUNC_NAME);
-        } else if (NULL == ranks && n > 0) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-
-        /* check that new group is no larger than old group */
-        if ( n > group_size) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_GROUP,
-                                          FUNC_NAME);
-        }
-
-        /* check to see if procs are within range */
-        for( i=0 ; i  < n ; i++ ) {
-            if( ( 0 > ranks[i] ) || (ranks[i] >= group_size)){
-                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_RANK,
-                                              FUNC_NAME );
-            }
-        }
-
-    }  /* end if( MPI_PARAM_CHECK ) */
-
-    if ( n == group_size ) {
-        *new_group = MPI_GROUP_EMPTY;
-        OBJ_RETAIN(MPI_GROUP_EMPTY);
-        return MPI_SUCCESS;
-    }
-
-    err = ompi_group_excl ( group, n, ranks, new_group );
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME );
-}
diff --git a/ompi/mpi/c/group_excl.c.in b/ompi/mpi/c/group_excl.c.in
new file mode 100644
index 00000000000..2165594fad1
--- /dev/null
+++ b/ompi/mpi/c/group_excl.c.in
@@ -0,0 +1,79 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      University of Houston. All rights reserved.
+ * Copyright (c) 2006-2009 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos Nat Security, LLC. All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/group/group.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/communicator/communicator.h"
+
+PROTOTYPE ERROR_CLASS group_excl(GROUP group, INT n, INT_ARRAY ranks,
+                                 GROUP_OUT new_group)
+{
+    ompi_group_t *group_pointer = (ompi_group_t *)group;
+    int i, err, group_size;
+
+    group_size = ompi_group_size ( group_pointer);
+    if( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        /* verify that group is valid group */
+        if ( (MPI_GROUP_NULL == group)  || (NULL == group) ||
+             (NULL == new_group) ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_GROUP,
+                                          FUNC_NAME);
+        } else if (NULL == ranks && n > 0) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+
+        /* check that new group is no larger than old group */
+        if ( n > group_size) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_GROUP,
+                                          FUNC_NAME);
+        }
+
+        /* check to see if procs are within range */
+        for( i=0 ; i  < n ; i++ ) {
+            if( ( 0 > ranks[i] ) || (ranks[i] >= group_size)){
+                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_RANK,
+                                              FUNC_NAME );
+            }
+        }
+
+    }  /* end if( MPI_PARAM_CHECK ) */
+
+    if ( n == group_size ) {
+        *new_group = MPI_GROUP_EMPTY;
+        OBJ_RETAIN(MPI_GROUP_EMPTY);
+        return MPI_SUCCESS;
+    }
+
+    err = ompi_group_excl ( group, n, ranks, new_group );
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME );
+}
diff --git a/ompi/mpi/c/group_f2c.c b/ompi/mpi/c/group_f2c.c
deleted file mode 100644
index d6ee5c68f50..00000000000
--- a/ompi/mpi/c/group_f2c.c
+++ /dev/null
@@ -1,60 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2007 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mpi/fortran/base/fint_2_int.h"
-#include "ompi/group/group.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Group_f2c = PMPI_Group_f2c
-#endif
-#define MPI_Group_f2c PMPI_Group_f2c
-#endif
-
-static const char FUNC_NAME[] = "MPI_Group_f2c";
-
-
-MPI_Group MPI_Group_f2c(MPI_Fint group_f)
-{
-    int group_index = OMPI_FINT_2_INT(group_f);
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-    }
-
-    /* Per MPI-2:4.12.4, do not invoke an error handler if we get an
-       invalid fortran handle.  If we get an invalid fortran handle,
-       return an invalid C handle. */
-
-    if (group_index < 0 ||
-        group_index >=
-        opal_pointer_array_get_size(&ompi_group_f_to_c_table)) {
-        return NULL;
-    }
-
-    return (MPI_Group)opal_pointer_array_get_item(&ompi_group_f_to_c_table, group_index);
-}
diff --git a/ompi/mpi/c/group_f2c.c.in b/ompi/mpi/c/group_f2c.c.in
new file mode 100644
index 00000000000..cd328493e95
--- /dev/null
+++ b/ompi/mpi/c/group_f2c.c.in
@@ -0,0 +1,51 @@
+/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+#include "ompi/group/group.h"
+
+PROTOTYPE GROUP group_f2c(FINT group_f)
+{
+    int group_index = OMPI_FINT_2_INT(group_f);
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+    }
+
+    /* Per MPI-2:4.12.4, do not invoke an error handler if we get an
+       invalid fortran handle.  If we get an invalid fortran handle,
+       return an invalid C handle. */
+
+    if (group_index < 0 ||
+        group_index >=
+        opal_pointer_array_get_size(&ompi_group_f_to_c_table)) {
+        return NULL;
+    }
+
+    return (MPI_Group)opal_pointer_array_get_item(&ompi_group_f_to_c_table, group_index);
+}
diff --git a/ompi/mpi/c/group_free.c b/ompi/mpi/c/group_free.c
deleted file mode 100644
index f0b5788046c..00000000000
--- a/ompi/mpi/c/group_free.c
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2020 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/group/group.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Group_free = PMPI_Group_free
-#endif
-#define MPI_Group_free PMPI_Group_free
-#endif
-
-static const char FUNC_NAME[] = "MPI_Group_free";
-
-
-int MPI_Group_free(MPI_Group *group)
-{
-    int ret;
-
-    /* check to make sure we don't free GROUP_NULL.  Note that we *do*
-       allow freeing GROUP_EMPTY after much debate in the OMPI core
-       group.  The final thread about this, and the decision to
-       support freeing GROUP_EMPTY can be found here:
-
-       https://www.open-mpi.org/community/lists/devel/2007/12/2750.php
-
-       The short version: other MPI's allow it (LAM/MPI, CT6, MPICH2)
-       probably mainly because the Intel MPI test suite expects it to
-       happen and there's now several years worth of expected behavior
-       to allow this behavior.  Rather than have to explain every time
-       why OMPI is the only one who completely adheres to the standard
-       / fails the intel tests, it seemed easier to just let this one
-       slide.  It's not really that important, after all! */
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if ((NULL == group) ||
-            (MPI_GROUP_NULL == *group) || (NULL == *group) ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_GROUP,
-                                          FUNC_NAME);
-        }
-
-    }
-
-    ret = ompi_group_free ( group);
-    OMPI_ERRHANDLER_NOHANDLE_CHECK(ret, ret, FUNC_NAME);
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/group_free.c.in b/ompi/mpi/c/group_free.c.in
new file mode 100644
index 00000000000..6fbc3ee6a36
--- /dev/null
+++ b/ompi/mpi/c/group_free.c.in
@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2020 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/group/group.h"
+
+PROTOTYPE ERROR_CLASS group_free(GROUP_OUT group)
+{
+    int ret;
+
+    /* check to make sure we don't free GROUP_NULL.  Note that we *do*
+       allow freeing GROUP_EMPTY after much debate in the OMPI core
+       group.  The final thread about this, and the decision to
+       support freeing GROUP_EMPTY can be found here:
+
+       https://www.open-mpi.org/community/lists/devel/2007/12/2750.php
+
+       The short version: other MPI's allow it (LAM/MPI, CT6, MPICH2)
+       probably mainly because the Intel MPI test suite expects it to
+       happen and there's now several years worth of expected behavior
+       to allow this behavior.  Rather than have to explain every time
+       why OMPI is the only one who completely adheres to the standard
+       / fails the intel tests, it seemed easier to just let this one
+       slide.  It's not really that important, after all! */
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if ((NULL == group) ||
+            (MPI_GROUP_NULL == *group) || (NULL == *group) ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_GROUP,
+                                          FUNC_NAME);
+        }
+
+    }
+
+    ret = ompi_group_free ( group);
+    OMPI_ERRHANDLER_NOHANDLE_CHECK(ret, ret, FUNC_NAME);
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/group_from_session_pset.c.in b/ompi/mpi/c/group_from_session_pset.c.in
new file mode 100644
index 00000000000..ec7bacc9415
--- /dev/null
+++ b/ompi/mpi/c/group_from_session_pset.c.in
@@ -0,0 +1,48 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/instance/instance.h"
+
+PROTOTYPE ERROR_CLASS group_from_session_pset (SESSION session, STRING pset_name, GROUP_OUT newgroup)
+{
+    int rc;
+
+    if ( MPI_PARAM_CHECK ) {
+        if (ompi_instance_invalid(session)) {
+            if (NULL != session) {
+                return OMPI_ERRHANDLER_INVOKE(session, MPI_ERR_SESSION, FUNC_NAME);
+            } else {
+                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_SESSION, FUNC_NAME);
+            }
+	} else if (NULL == pset_name || NULL == newgroup) {
+            return OMPI_ERRHANDLER_INVOKE(session, MPI_ERR_ARG, FUNC_NAME);
+        }
+    }
+
+    rc = ompi_group_from_pset (session, pset_name, newgroup);
+    /*
+     * if process set was not found, OMPI_ERR_NOT_FOUND is the return value.
+     * we want to map this to MPI_ERR_ARG but we have to do it manually here
+     * since the OMPI error to MPI error code code maps this to MPI_ERR_INTERN
+     */
+    if (OMPI_ERR_NOT_FOUND == rc) {
+        rc = MPI_ERR_ARG;
+    }
+
+
+    OMPI_ERRHANDLER_RETURN (rc, (NULL == session) ? MPI_SESSION_NULL : session,
+                            rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/group_incl.c b/ompi/mpi/c/group_incl.c
deleted file mode 100644
index 7c1fb84531f..00000000000
--- a/ompi/mpi/c/group_incl.c
+++ /dev/null
@@ -1,87 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      University of Houston. All rights reserved.
- * Copyright (c) 2006-2009 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/group/group.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/communicator/communicator.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Group_incl = PMPI_Group_incl
-#endif
-#define MPI_Group_incl PMPI_Group_incl
-#endif
-
-static const char FUNC_NAME[] = "MPI_Group_incl";
-
-
-int MPI_Group_incl(MPI_Group group, int n, const int ranks[], MPI_Group *new_group)
-{
-  int i, group_size, err;
-  ompi_group_t *group_pointer;
-
-  group_pointer = (ompi_group_t *)group;
-  group_size = ompi_group_size ( group_pointer );
-
-  if( MPI_PARAM_CHECK ) {
-      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-      /* verify that group is valid group */
-      if ( (MPI_GROUP_NULL == group) || ( NULL == group) ||
-           (NULL == new_group) ) {
-          return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_GROUP,
-                                        FUNC_NAME);
-      } else if (NULL == ranks && n > 0) {
-          return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                        FUNC_NAME);
-      }
-
-      /* check that new group is no larger than old group */
-      if ( n > group_size ) {
-          return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_RANK,
-                                        FUNC_NAME);
-      }
-
-      for (i = 0; i < n; i++) {
-          if ((ranks[i] < 0) || (ranks[i] >= group_size)){
-              return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_RANK,
-                                             FUNC_NAME);
-          }
-      }
-  }  /* end if( MPI_PARAM_CHECK ) */
-
-  if ( 0 == n ) {
-      *new_group = MPI_GROUP_EMPTY;
-      OBJ_RETAIN(MPI_GROUP_EMPTY);
-      return MPI_SUCCESS;
-  }
-
-  err = ompi_group_incl(group,n,ranks,new_group);
-  OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err,FUNC_NAME);
-}
diff --git a/ompi/mpi/c/group_incl.c.in b/ompi/mpi/c/group_incl.c.in
new file mode 100644
index 00000000000..2380238c1d7
--- /dev/null
+++ b/ompi/mpi/c/group_incl.c.in
@@ -0,0 +1,79 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      University of Houston. All rights reserved.
+ * Copyright (c) 2006-2009 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/group/group.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/communicator/communicator.h"
+
+PROTOTYPE ERROR_CLASS group_incl(GROUP group, INT n, INT_ARRAY ranks, GROUP_OUT new_group)
+{
+  int i, group_size, err;
+  ompi_group_t *group_pointer;
+
+  group_pointer = (ompi_group_t *)group;
+  group_size = ompi_group_size ( group_pointer );
+
+  if( MPI_PARAM_CHECK ) {
+      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+      /* verify that group is valid group */
+      if ( (MPI_GROUP_NULL == group) || ( NULL == group) ||
+           (NULL == new_group) ) {
+          return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_GROUP,
+                                        FUNC_NAME);
+      } else if (NULL == ranks && n > 0) {
+          return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                        FUNC_NAME);
+      }
+
+      /* check that new group is no larger than old group */
+      if ( n > group_size ) {
+          return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_RANK,
+                                        FUNC_NAME);
+      }
+
+      for (i = 0; i < n; i++) {
+          if ((ranks[i] < 0) || (ranks[i] >= group_size)){
+              return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_RANK,
+                                             FUNC_NAME);
+          }
+      }
+  }  /* end if( MPI_PARAM_CHECK ) */
+
+  if ( 0 == n ) {
+      *new_group = MPI_GROUP_EMPTY;
+      OBJ_RETAIN(MPI_GROUP_EMPTY);
+      return MPI_SUCCESS;
+  }
+
+  err = ompi_group_incl(group,n,ranks,new_group);
+  OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err,FUNC_NAME);
+}
diff --git a/ompi/mpi/c/group_intersection.c b/ompi/mpi/c/group_intersection.c
deleted file mode 100644
index 93e56d54b79..00000000000
--- a/ompi/mpi/c/group_intersection.c
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      University of Houston. All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/group/group.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/communicator/communicator.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Group_intersection = PMPI_Group_intersection
-#endif
-#define MPI_Group_intersection PMPI_Group_intersection
-#endif
-
-static const char FUNC_NAME[] = "MPI_Group_intersection";
-
-
-int MPI_Group_intersection(MPI_Group group1, MPI_Group group2,
-        MPI_Group *new_group)
-{
-  int err;
-
-  if ( MPI_PARAM_CHECK ) {
-      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-      /* verify that groups are valid */
-      if ( (MPI_GROUP_NULL == group1) || (MPI_GROUP_NULL == group2) ||
-           ( NULL == group1) || (NULL == group2) ||
-           (NULL == new_group) ) {
-          return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_GROUP,
-                                        FUNC_NAME);
-      }
-  }
-
-  err = ompi_group_intersection ( group1, group2,  new_group );
-  OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME );
-}
diff --git a/ompi/mpi/c/group_intersection.c.in b/ompi/mpi/c/group_intersection.c.in
new file mode 100644
index 00000000000..0464ad5b570
--- /dev/null
+++ b/ompi/mpi/c/group_intersection.c.in
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      University of Houston. All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/group/group.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/communicator/communicator.h"
+
+PROTOTYPE ERROR_CLASS group_intersection(GROUP group1, GROUP group2,
+        GROUP_OUT new_group)
+{
+  int err;
+
+  if ( MPI_PARAM_CHECK ) {
+      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+      /* verify that groups are valid */
+      if ( (MPI_GROUP_NULL == group1) || (MPI_GROUP_NULL == group2) ||
+           ( NULL == group1) || (NULL == group2) ||
+           (NULL == new_group) ) {
+          return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_GROUP,
+                                        FUNC_NAME);
+      }
+  }
+
+  err = ompi_group_intersection ( group1, group2,  new_group );
+  OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME );
+}
diff --git a/ompi/mpi/c/group_range_excl.c b/ompi/mpi/c/group_range_excl.c
deleted file mode 100644
index 48bf5fb56cb..00000000000
--- a/ompi/mpi/c/group_range_excl.c
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      University of Houston. All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/group/group.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/communicator/communicator.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Group_range_excl = PMPI_Group_range_excl
-#endif
-#define MPI_Group_range_excl PMPI_Group_range_excl
-#endif
-
-static const char FUNC_NAME[] = "MPI_Group_range_excl";
-
-
-int MPI_Group_range_excl(MPI_Group group, int n_triplets, int ranges[][3],
-                         MPI_Group *new_group)
-{
-    int err, i, group_size, indx;
-    int * elements_int_list;
-
-    /* can't act on NULL group */
-    if( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if ( (MPI_GROUP_NULL == group) || (NULL == group) ||
-             (NULL == new_group) ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_GROUP,
-                                          FUNC_NAME);
-        }
-
-        group_size = ompi_group_size ( group );
-        elements_int_list = (int *) malloc(sizeof(int) * (group_size+1));
-        if (NULL == elements_int_list) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_OTHER,
-                                          FUNC_NAME);
-        }
-        for (i = 0; i < group_size; i++) {
-            elements_int_list[i] = -1;
-        }
-
-        for (i = 0; i < n_triplets; i++) {
-            if ((0 > ranges[i][0]) || (ranges[i][0] > group_size)) {
-                goto error_rank;
-            }
-            if ((0 > ranges[i][1]) || (ranges[i][1] > group_size)) {
-                goto error_rank;
-            }
-            if (ranges[i][2] == 0) {
-                goto error_rank;
-            }
-
-            if (ranges[i][0] < ranges[i][1]) {
-                if (ranges[i][2] < 0) {
-                    goto error_rank;
-                }
-                /* positive stride */
-                for (indx = ranges[i][0]; indx <= ranges[i][1]; indx += ranges[i][2]) {
-                    /* make sure rank has not already been selected */
-                    if (elements_int_list[indx] != -1) {
-                        goto error_rank;
-                    }
-                    elements_int_list[indx] = i;
-                }
-            } else if (ranges[i][0] > ranges[i][1]) {
-                if (ranges[i][2] > 0) {
-                    goto error_rank;
-                }
-                /* negative stride */
-                for (indx = ranges[i][0]; indx >= ranges[i][1]; indx += ranges[i][2]) {
-                    /* make sure rank has not already been selected */
-                    if (elements_int_list[indx] != -1) {
-                        goto error_rank;
-                    }
-                    elements_int_list[indx] = i;
-                }
-            } else {
-                /* first_rank == last_rank */
-                indx = ranges[i][0];
-                if (elements_int_list[indx] != -1) {
-                    goto error_rank;
-                }
-                elements_int_list[indx] = i;
-            }
-        }
-
-        free (elements_int_list);
-    }
-
-    err = ompi_group_range_excl(group,n_triplets,ranges,new_group);
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err,FUNC_NAME);
-
-error_rank:
-    free(elements_int_list);
-    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_RANK, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/group_range_excl.c.in b/ompi/mpi/c/group_range_excl.c.in
new file mode 100644
index 00000000000..483d3a75685
--- /dev/null
+++ b/ompi/mpi/c/group_range_excl.c.in
@@ -0,0 +1,112 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      University of Houston. All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/group/group.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/communicator/communicator.h"
+
+PROTOTYPE ERROR_CLASS group_range_excl(GROUP group, INT n_triplets, RANGE_ARRAY ranges,
+                                       GROUP_OUT new_group)
+{
+    int err, i, group_size, indx;
+    int * elements_int_list;
+
+    /* can't act on NULL group */
+    if( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if ( (MPI_GROUP_NULL == group) || (NULL == group) ||
+             (NULL == new_group) ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_GROUP,
+                                          FUNC_NAME);
+        }
+
+        group_size = ompi_group_size ( group );
+        elements_int_list = (int *) malloc(sizeof(int) * (group_size+1));
+        if (NULL == elements_int_list) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_OTHER,
+                                          FUNC_NAME);
+        }
+        for (i = 0; i < group_size; i++) {
+            elements_int_list[i] = -1;
+        }
+
+        for (i = 0; i < n_triplets; i++) {
+            if ((0 > ranges[i][0]) || (ranges[i][0] > group_size)) {
+                goto error_rank;
+            }
+            if ((0 > ranges[i][1]) || (ranges[i][1] > group_size)) {
+                goto error_rank;
+            }
+            if (ranges[i][2] == 0) {
+                goto error_rank;
+            }
+
+            if (ranges[i][0] < ranges[i][1]) {
+                if (ranges[i][2] < 0) {
+                    goto error_rank;
+                }
+                /* positive stride */
+                for (indx = ranges[i][0]; indx <= ranges[i][1]; indx += ranges[i][2]) {
+                    /* make sure rank has not already been selected */
+                    if (elements_int_list[indx] != -1) {
+                        goto error_rank;
+                    }
+                    elements_int_list[indx] = i;
+                }
+            } else if (ranges[i][0] > ranges[i][1]) {
+                if (ranges[i][2] > 0) {
+                    goto error_rank;
+                }
+                /* negative stride */
+                for (indx = ranges[i][0]; indx >= ranges[i][1]; indx += ranges[i][2]) {
+                    /* make sure rank has not already been selected */
+                    if (elements_int_list[indx] != -1) {
+                        goto error_rank;
+                    }
+                    elements_int_list[indx] = i;
+                }
+            } else {
+                /* first_rank == last_rank */
+                indx = ranges[i][0];
+                if (elements_int_list[indx] != -1) {
+                    goto error_rank;
+                }
+                elements_int_list[indx] = i;
+            }
+        }
+
+        free (elements_int_list);
+    }
+
+    err = ompi_group_range_excl(group,n_triplets,ranges,new_group);
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err,FUNC_NAME);
+
+error_rank:
+    free(elements_int_list);
+    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_RANK, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/group_range_incl.c b/ompi/mpi/c/group_range_incl.c
deleted file mode 100644
index 3559b88f872..00000000000
--- a/ompi/mpi/c/group_range_incl.c
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      University of Houston. All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/group/group.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/communicator/communicator.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Group_range_incl = PMPI_Group_range_incl
-#endif
-#define MPI_Group_range_incl PMPI_Group_range_incl
-#endif
-
-static const char FUNC_NAME[] = "MPI_Group_range_incl";
-
-
-int MPI_Group_range_incl(MPI_Group group, int n_triplets, int ranges[][3],
-                         MPI_Group *new_group)
-{
-    int err, i,indx;
-    int group_size;
-    int * elements_int_list;
-
-    /* can't act on NULL group */
-    if( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if ( (MPI_GROUP_NULL == group) || (NULL == group) ||
-             (NULL == new_group) ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_GROUP,
-                                          FUNC_NAME);
-        }
-
-        group_size = ompi_group_size ( group);
-        elements_int_list = (int *) malloc(sizeof(int) * (group_size+1));
-        if (NULL == elements_int_list) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_OTHER, FUNC_NAME);
-        }
-        for (i = 0; i < group_size; i++) {
-            elements_int_list[i] = -1;
-        }
-
-        for ( i=0; i < n_triplets; i++) {
-            if ((0 > ranges[i][0]) || (ranges[i][0] > group_size)) {
-                goto error_rank;
-            }
-            if ((0 > ranges[i][1]) || (ranges[i][1] > group_size)) {
-                goto error_rank;
-            }
-            if (ranges[i][2] == 0) {
-                goto error_rank;
-            }
-
-            if ((ranges[i][0] < ranges[i][1])) {
-                if (ranges[i][2] < 0) {
-                    goto error_rank;
-                }
-                /* positive stride */
-                for (indx = ranges[i][0]; indx <= ranges[i][1]; indx += ranges[i][2]) {
-                    /* make sure rank has not already been selected */
-                    if (elements_int_list[indx] != -1) {
-                        goto error_rank;
-                    }
-                    elements_int_list[indx] = i;
-                }
-            } else if (ranges[i][0] > ranges[i][1]) {
-                if (ranges[i][2] > 0) {
-                    goto error_rank;
-                }
-                /* negative stride */
-                for (indx = ranges[i][0]; indx >= ranges[i][1]; indx += ranges[i][2]) {
-                    /* make sure rank has not already been selected */
-                    if (elements_int_list[indx] != -1) {
-                        goto error_rank;
-                    }
-                    elements_int_list[indx] = i;
-                }
-            } else {
-                /* first_rank == last_rank */
-                indx = ranges[i][0];
-                if (elements_int_list[indx] != -1) {
-                    goto error_rank;
-                }
-                elements_int_list[indx] = i;
-            }
-        }
-
-        free ( elements_int_list);
-    }
-
-    err = ompi_group_range_incl ( group, n_triplets, ranges, new_group );
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME );
-
-error_rank:
-    free(elements_int_list);
-    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_RANK, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/group_range_incl.c.in b/ompi/mpi/c/group_range_incl.c.in
new file mode 100644
index 00000000000..deee2458a53
--- /dev/null
+++ b/ompi/mpi/c/group_range_incl.c.in
@@ -0,0 +1,113 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      University of Houston. All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/group/group.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/communicator/communicator.h"
+
+PROTOTYPE ERROR_CLASS group_range_incl(GROUP group, INT n_triplets, RANGE_ARRAY ranges,
+                                       GROUP_OUT new_group)
+{
+    int err, i,indx;
+    int group_size;
+    int * elements_int_list;
+
+    /* can't act on NULL group */
+    if( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if ( (MPI_GROUP_NULL == group) || (NULL == group) ||
+             (NULL == new_group) ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_GROUP,
+                                          FUNC_NAME);
+        }
+
+        group_size = ompi_group_size ( group);
+        elements_int_list = (int *) malloc(sizeof(int) * (group_size+1));
+        if (NULL == elements_int_list) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_OTHER, FUNC_NAME);
+        }
+        for (i = 0; i < group_size; i++) {
+            elements_int_list[i] = -1;
+        }
+
+        for ( i=0; i < n_triplets; i++) {
+            if ((0 > ranges[i][0]) || (ranges[i][0] > group_size)) {
+                goto error_rank;
+            }
+            if ((0 > ranges[i][1]) || (ranges[i][1] > group_size)) {
+                goto error_rank;
+            }
+            if (ranges[i][2] == 0) {
+                goto error_rank;
+            }
+
+            if ((ranges[i][0] < ranges[i][1])) {
+                if (ranges[i][2] < 0) {
+                    goto error_rank;
+                }
+                /* positive stride */
+                for (indx = ranges[i][0]; indx <= ranges[i][1]; indx += ranges[i][2]) {
+                    /* make sure rank has not already been selected */
+                    if (elements_int_list[indx] != -1) {
+                        goto error_rank;
+                    }
+                    elements_int_list[indx] = i;
+                }
+            } else if (ranges[i][0] > ranges[i][1]) {
+                if (ranges[i][2] > 0) {
+                    goto error_rank;
+                }
+                /* negative stride */
+                for (indx = ranges[i][0]; indx >= ranges[i][1]; indx += ranges[i][2]) {
+                    /* make sure rank has not already been selected */
+                    if (elements_int_list[indx] != -1) {
+                        goto error_rank;
+                    }
+                    elements_int_list[indx] = i;
+                }
+            } else {
+                /* first_rank == last_rank */
+                indx = ranges[i][0];
+                if (elements_int_list[indx] != -1) {
+                    goto error_rank;
+                }
+                elements_int_list[indx] = i;
+            }
+        }
+
+        free ( elements_int_list);
+    }
+
+    err = ompi_group_range_incl ( group, n_triplets, ranges, new_group );
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME );
+
+error_rank:
+    free(elements_int_list);
+    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_RANK, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/group_rank.c b/ompi/mpi/c/group_rank.c
deleted file mode 100644
index ea42ce8b11c..00000000000
--- a/ompi/mpi/c/group_rank.c
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/group/group.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Group_rank = PMPI_Group_rank
-#endif
-#define MPI_Group_rank PMPI_Group_rank
-#endif
-
-static const char FUNC_NAME[] = "MPI_Group_rank";
-
-
-int MPI_Group_rank(MPI_Group group, int *rank)
-{
-    /* error checking */
-    if( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if( (MPI_GROUP_NULL == group) || ( NULL == group) ){
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_GROUP,
-                                          FUNC_NAME);
-        } else if (NULL == rank) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-    }
-
-    *rank=ompi_group_rank((ompi_group_t *)group);
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/group_rank.c.in b/ompi/mpi/c/group_rank.c.in
new file mode 100644
index 00000000000..550c18b347b
--- /dev/null
+++ b/ompi/mpi/c/group_rank.c.in
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/group/group.h"
+
+PROTOTYPE ERROR_CLASS group_rank(GROUP group, INT_OUT rank)
+{
+    /* error checking */
+    if( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if( (MPI_GROUP_NULL == group) || ( NULL == group) ){
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_GROUP,
+                                          FUNC_NAME);
+        } else if (NULL == rank) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+    *rank=ompi_group_rank((ompi_group_t *)group);
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/group_size.c b/ompi/mpi/c/group_size.c
deleted file mode 100644
index 51e4bbda3de..00000000000
--- a/ompi/mpi/c/group_size.c
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/group/group.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Group_size = PMPI_Group_size
-#endif
-#define MPI_Group_size PMPI_Group_size
-#endif
-
-static const char FUNC_NAME[] = "MPI_Group_size";
-
-
-int MPI_Group_size(MPI_Group group, int *size)
-{
-    /* error checking */
-    if( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if( (MPI_GROUP_NULL == group) || (NULL == group) ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_GROUP,
-                                          FUNC_NAME);
-        } else if (NULL == size) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-    }
-
-    *size=ompi_group_size((ompi_group_t *)group);
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/group_size.c.in b/ompi/mpi/c/group_size.c.in
new file mode 100644
index 00000000000..2179e5b9f63
--- /dev/null
+++ b/ompi/mpi/c/group_size.c.in
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/group/group.h"
+
+PROTOTYPE ERROR_CLASS group_size(GROUP group, INT_OUT size)
+{
+    /* error checking */
+    if( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if( (MPI_GROUP_NULL == group) || (NULL == group) ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_GROUP,
+                                          FUNC_NAME);
+        } else if (NULL == size) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+    *size=ompi_group_size((ompi_group_t *)group);
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/group_translate_ranks.c b/ompi/mpi/c/group_translate_ranks.c
deleted file mode 100644
index c1949fe1a94..00000000000
--- a/ompi/mpi/c/group_translate_ranks.c
+++ /dev/null
@@ -1,74 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/group/group.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Group_translate_ranks = PMPI_Group_translate_ranks
-#endif
-#define MPI_Group_translate_ranks PMPI_Group_translate_ranks
-#endif
-
-static const char FUNC_NAME[] = "MPI_Group_translate_ranks";
-
-
-int MPI_Group_translate_ranks(MPI_Group group1, int n_ranks, const int ranks1[],
-                              MPI_Group group2, int ranks2[])
-{
-    int err;
-
-    /* check for errors */
-    if( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if ((MPI_GROUP_NULL == group1) || (MPI_GROUP_NULL == group2) ||
-            (NULL == group1) || (NULL == group2)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_GROUP,
-                                          FUNC_NAME);
-        }
-        if (n_ranks < 0) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_GROUP,
-                                          FUNC_NAME);
-        }
-        if (n_ranks > 0 && ((NULL == ranks1) || (NULL == ranks2 ))) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_GROUP,
-                                          FUNC_NAME);
-        }
-    }
-
-    if (0 == n_ranks) {
-        return MPI_SUCCESS;
-    }
-
-    err = ompi_group_translate_ranks ( group1, n_ranks, ranks1,
-                                       group2, ranks2 );
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME );
-}
diff --git a/ompi/mpi/c/group_translate_ranks.c.in b/ompi/mpi/c/group_translate_ranks.c.in
new file mode 100644
index 00000000000..b84f7212f56
--- /dev/null
+++ b/ompi/mpi/c/group_translate_ranks.c.in
@@ -0,0 +1,66 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/group/group.h"
+
+PROTOTYPE ERROR_CLASS group_translate_ranks(GROUP group1, INT n_ranks, INT_ARRAY ranks1,
+                                            GROUP group2, INT_OUT ranks2)
+{
+    int err;
+
+    /* check for errors */
+    if( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if ((MPI_GROUP_NULL == group1) || (MPI_GROUP_NULL == group2) ||
+            (NULL == group1) || (NULL == group2)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_GROUP,
+                                          FUNC_NAME);
+        }
+        if (n_ranks < 0) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_GROUP,
+                                          FUNC_NAME);
+        }
+        if (n_ranks > 0 && ((NULL == ranks1) || (NULL == ranks2 ))) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_GROUP,
+                                          FUNC_NAME);
+        }
+    }
+
+    if (0 == n_ranks) {
+        return MPI_SUCCESS;
+    }
+
+    err = ompi_group_translate_ranks ( group1, n_ranks, ranks1,
+                                       group2, ranks2 );
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME );
+}
diff --git a/ompi/mpi/c/group_union.c b/ompi/mpi/c/group_union.c
deleted file mode 100644
index cdfc3efb3c9..00000000000
--- a/ompi/mpi/c/group_union.c
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      University of Houston. All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/group/group.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/communicator/communicator.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Group_union = PMPI_Group_union
-#endif
-#define MPI_Group_union PMPI_Group_union
-#endif
-
-static const char FUNC_NAME[] = "MPI_Group_union";
-
-
-int MPI_Group_union(MPI_Group group1, MPI_Group group2, MPI_Group *new_group)
-{
-    int err;
-
-    /* check for errors */
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if ((MPI_GROUP_NULL == group1) || (MPI_GROUP_NULL == group2) ||
-            (NULL == group1) || (NULL == group2) ||
-            (NULL == new_group)) {
-            return
-                OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_GROUP,
-                                       FUNC_NAME);
-        }
-    }
-
-    err = ompi_group_union ( group1, group2,  new_group );
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME );
-}
diff --git a/ompi/mpi/c/group_union.c.in b/ompi/mpi/c/group_union.c.in
new file mode 100644
index 00000000000..af291660747
--- /dev/null
+++ b/ompi/mpi/c/group_union.c.in
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      University of Houston. All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/group/group.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/communicator/communicator.h"
+
+PROTOTYPE ERROR_CLASS group_union(GROUP group1, GROUP group2, GROUP_OUT new_group)
+{
+    int err;
+
+    /* check for errors */
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if ((MPI_GROUP_NULL == group1) || (MPI_GROUP_NULL == group2) ||
+            (NULL == group1) || (NULL == group2) ||
+            (NULL == new_group)) {
+            return
+                OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_GROUP,
+                                       FUNC_NAME);
+        }
+    }
+
+    err = ompi_group_union ( group1, group2,  new_group );
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME );
+}
diff --git a/ompi/mpi/c/iallgather.c b/ompi/mpi/c/iallgather.c
deleted file mode 100644
index 75cfb5d3ceb..00000000000
--- a/ompi/mpi/c/iallgather.c
+++ /dev/null
@@ -1,109 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012      Oak Ridge National Laboratory. All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2020 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Iallgather = PMPI_Iallgather
-#endif
-#define MPI_Iallgather PMPI_Iallgather
-#endif
-
-static const char FUNC_NAME[] = "MPI_Iallgather";
-
-
-int MPI_Iallgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                   void *recvbuf, int recvcount, MPI_Datatype recvtype,
-                   MPI_Comm comm,  MPI_Request *request)
-{
-    int err;
-
-    SPC_RECORD(OMPI_SPC_IALLGATHER, 1);
-
-    MEMCHECKER(
-        int rank;
-        ptrdiff_t ext;
-
-        rank = ompi_comm_rank(comm);
-        ompi_datatype_type_extent(recvtype, &ext);
-
-        memchecker_datatype(recvtype);
-        memchecker_comm(comm);
-        /* check whether the actual send buffer is defined. */
-        if (MPI_IN_PLACE == sendbuf) {
-            memchecker_call(&opal_memchecker_base_isdefined,
-                            (char *)(recvbuf)+rank*recvcount*ext,
-                            recvcount, recvtype);
-        } else {
-            memchecker_datatype(sendtype);
-            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-        }
-        /* check whether the receive buffer is addressable. */
-        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-
-        /* Unrooted operation -- same checks for all ranks on both
-           intracommunicators and intercommunicators */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-          OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
-        } else if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
-          err = MPI_ERR_TYPE;
-        } else if (recvcount < 0) {
-          err = MPI_ERR_COUNT;
-        } else if ((MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
-                   MPI_IN_PLACE == recvbuf) {
-          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        } else if (MPI_IN_PLACE != sendbuf) {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-        }
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_iallgather(sendbuf, sendcount, sendtype,
-                                       recvbuf, recvcount, recvtype, comm,
-                                       request, comm->c_coll->coll_iallgather_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_datatypes(*request, (MPI_IN_PLACE==sendbuf)?NULL:sendtype, recvtype);
-    }
-
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/iallgather.c.in b/ompi/mpi/c/iallgather.c.in
new file mode 100644
index 00000000000..6bc3d09adee
--- /dev/null
+++ b/ompi/mpi/c/iallgather.c.in
@@ -0,0 +1,100 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oak Ridge National Laboratory. All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2020 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS iallgather(BUFFER sendbuf, COUNT sendcount, DATATYPE sendtype,
+                                 BUFFER_OUT recvbuf, COUNT recvcount, DATATYPE recvtype,
+                                 COMM comm, REQUEST_INOUT request)
+{
+    int err;
+
+    SPC_RECORD(OMPI_SPC_IALLGATHER, 1);
+
+    MEMCHECKER(
+        int rank;
+        ptrdiff_t ext;
+
+        rank = ompi_comm_rank(comm);
+        ompi_datatype_type_extent(recvtype, &ext);
+
+        memchecker_datatype(recvtype);
+        memchecker_comm(comm);
+        /* check whether the actual send buffer is defined. */
+        if (MPI_IN_PLACE == sendbuf) {
+            memchecker_call(&opal_memchecker_base_isdefined,
+                            (char *)(recvbuf)+rank*recvcount*ext,
+                            recvcount, recvtype);
+        } else {
+            memchecker_datatype(sendtype);
+            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+        }
+        /* check whether the receive buffer is addressable. */
+        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+
+        /* Unrooted operation -- same checks for all ranks on both
+           intracommunicators and intercommunicators */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+          OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+        } else if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
+          err = MPI_ERR_TYPE;
+        } else if (recvcount < 0) {
+          err = MPI_ERR_COUNT;
+        } else if ((MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
+                   MPI_IN_PLACE == recvbuf) {
+          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        } else if (MPI_IN_PLACE != sendbuf) {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+        }
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    err = comm->c_coll->coll_iallgather(sendbuf, sendcount, sendtype,
+                                       recvbuf, recvcount, recvtype, comm,
+                                       request, comm->c_coll->coll_iallgather_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_datatypes(*request, (MPI_IN_PLACE==sendbuf)?NULL:sendtype, recvtype);
+    }
+
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/iallgatherv.c b/ompi/mpi/c/iallgatherv.c
deleted file mode 100644
index 67427b916ac..00000000000
--- a/ompi/mpi/c/iallgatherv.c
+++ /dev/null
@@ -1,133 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2010      University of Houston.  All rights reserved.
- * Copyright (c) 2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Iallgatherv = PMPI_Iallgatherv
-#endif
-#define MPI_Iallgatherv PMPI_Iallgatherv
-#endif
-
-static const char FUNC_NAME[] = "MPI_Iallgatherv";
-
-
-int MPI_Iallgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                    void *recvbuf, const int recvcounts[], const int displs[],
-                    MPI_Datatype recvtype, MPI_Comm comm, MPI_Request *request)
-{
-    int i, size, err;
-
-    SPC_RECORD(OMPI_SPC_IALLGATHERV, 1);
-
-    MEMCHECKER(
-        int rank;
-        ptrdiff_t ext;
-
-        rank = ompi_comm_rank(comm);
-        size = ompi_comm_size(comm);
-        ompi_datatype_type_extent(recvtype, &ext);
-
-        memchecker_datatype(recvtype);
-        memchecker_comm (comm);
-        /* check whether the receive buffer is addressable. */
-        for (i = 0; i < size; i++) {
-            memchecker_call(&opal_memchecker_base_isaddressable,
-                            (char *)(recvbuf)+displs[i]*ext,
-                            recvcounts[i], recvtype);
-        }
-
-        /* check whether the actual send buffer is defined. */
-        if (MPI_IN_PLACE == sendbuf) {
-            memchecker_call(&opal_memchecker_base_isdefined,
-                            (char *)(recvbuf)+displs[rank]*ext,
-                            recvcounts[rank], recvtype);
-        } else {
-            memchecker_datatype(sendtype);
-            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-
-        /* Unrooted operation -- same checks for all ranks on both
-           intracommunicators and intercommunicators */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if ((MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
-                   MPI_IN_PLACE == recvbuf) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        } else if (MPI_DATATYPE_NULL == recvtype) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
-        }
-
-        if (MPI_IN_PLACE != sendbuf) {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-        }
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-
-      /* We always define the remote group to be the same as the local
-         group in the case of an intracommunicator, so it's safe to
-         get the size of the remote group here for both intra- and
-         intercommunicators */
-
-        size = ompi_comm_remote_size(comm);
-        for (i = 0; i < size; ++i) {
-          if (recvcounts[i] < 0) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-          }
-        }
-
-        if (NULL == displs) {
-          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_BUFFER, FUNC_NAME);
-        }
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_iallgatherv(sendbuf, sendcount, sendtype,
-                                        recvbuf, recvcounts, displs,
-                                        recvtype, comm, request,
-                                        comm->c_coll->coll_iallgatherv_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_datatypes(*request, (MPI_IN_PLACE==sendbuf)?NULL:sendtype, recvtype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/iallgatherv.c.in b/ompi/mpi/c/iallgatherv.c.in
new file mode 100644
index 00000000000..aa2b543a423
--- /dev/null
+++ b/ompi/mpi/c/iallgatherv.c.in
@@ -0,0 +1,129 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010      University of Houston.  All rights reserved.
+ * Copyright (c) 2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS iallgatherv(BUFFER sendbuf, COUNT sendcount, DATATYPE sendtype,
+                                  BUFFER_OUT recvbuf, COUNT_ARRAY recvcounts, DISP_ARRAY displs,
+                                  DATATYPE recvtype, COMM comm, REQUEST_INOUT request)
+{
+    int i, size, err;
+    ompi_count_array_t recvcounts_desc;
+    ompi_disp_array_t displs_desc;
+
+    SPC_RECORD(OMPI_SPC_IALLGATHERV, 1);
+
+    MEMCHECKER(
+        int rank;
+        ptrdiff_t ext;
+
+        rank = ompi_comm_rank(comm);
+        size = ompi_comm_size(comm);
+        ompi_datatype_type_extent(recvtype, &ext);
+
+        memchecker_datatype(recvtype);
+        memchecker_comm (comm);
+        /* check whether the receive buffer is addressable. */
+        for (i = 0; i < size; i++) {
+            memchecker_call(&opal_memchecker_base_isaddressable,
+                            (char *)(recvbuf)+displs[i]*ext,
+                            recvcounts[i], recvtype);
+        }
+
+        /* check whether the actual send buffer is defined. */
+        if (MPI_IN_PLACE == sendbuf) {
+            memchecker_call(&opal_memchecker_base_isdefined,
+                            (char *)(recvbuf)+displs[rank]*ext,
+                            recvcounts[rank], recvtype);
+        } else {
+            memchecker_datatype(sendtype);
+            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+
+        /* Unrooted operation -- same checks for all ranks on both
+           intracommunicators and intercommunicators */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if ((MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
+                   MPI_IN_PLACE == recvbuf) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        } else if (MPI_DATATYPE_NULL == recvtype) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
+        }
+
+        if (MPI_IN_PLACE != sendbuf) {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+        }
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+
+      /* We always define the remote group to be the same as the local
+         group in the case of an intracommunicator, so it's safe to
+         get the size of the remote group here for both intra- and
+         intercommunicators */
+
+        size = ompi_comm_remote_size(comm);
+        for (i = 0; i < size; ++i) {
+          if (recvcounts[i] < 0) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+          }
+        }
+
+        if (NULL == displs) {
+          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_BUFFER, FUNC_NAME);
+        }
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    OMPI_COUNT_ARRAY_INIT(&recvcounts_desc, recvcounts);
+    OMPI_DISP_ARRAY_INIT(&displs_desc, displs);
+    err = comm->c_coll->coll_iallgatherv(sendbuf, sendcount, sendtype,
+                                        recvbuf, recvcounts_desc, displs_desc,
+                                        recvtype, comm, request,
+                                        comm->c_coll->coll_iallgatherv_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_datatypes(*request, (MPI_IN_PLACE==sendbuf)?NULL:sendtype, recvtype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/iallreduce.c b/ompi/mpi/c/iallreduce.c
deleted file mode 100644
index ee9cf01e7a3..00000000000
--- a/ompi/mpi/c/iallreduce.c
+++ /dev/null
@@ -1,121 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2016      IBM Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/op/op.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Iallreduce = PMPI_Iallreduce
-#endif
-#define MPI_Iallreduce PMPI_Iallreduce
-#endif
-
-static const char FUNC_NAME[] = "MPI_Iallreduce";
-
-
-int MPI_Iallreduce(const void *sendbuf, void *recvbuf, int count,
-                   MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, MPI_Request *request)
-{
-    int err;
-
-    SPC_RECORD(OMPI_SPC_IALLREDUCE, 1);
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_comm(comm);
-
-        /* check whether receive buffer is defined. */
-        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, count, datatype);
-
-        /* check whether the actual send buffer is defined. */
-        if (MPI_IN_PLACE == sendbuf) {
-            memchecker_call(&opal_memchecker_base_isdefined, recvbuf, count, datatype);
-        } else {
-            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        char *msg;
-
-        /* Unrooted operation -- same checks for all ranks on both
-           intracommunicators and intercommunicators */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if (MPI_OP_NULL == op) {
-            err = MPI_ERR_OP;
-        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
-            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
-            free(msg);
-            return ret;
-        } else if ((MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
-                   MPI_IN_PLACE == recvbuf ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_BUFFER,
-                                          FUNC_NAME);
-        } else if( (sendbuf == recvbuf) &&
-                   (MPI_BOTTOM != sendbuf) &&
-                   (count > 1) ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_BUFFER,
-                                          FUNC_NAME);
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-    }
-
-
-    /* MPI standard says that reductions have to have a count of at least 1,
-     * but some benchmarks (e.g., IMB) calls this function with a count of 0.
-     * So handle that case.
-     */
-    if (0 == count) {
-        *request = &ompi_request_empty;
-        return MPI_SUCCESS;
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-
-    err = comm->c_coll->coll_iallreduce(sendbuf, recvbuf, count, datatype,
-                                       op, comm, request, comm->c_coll->coll_iallreduce_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_op(*request, op, datatype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/iallreduce.c.in b/ompi/mpi/c/iallreduce.c.in
new file mode 100644
index 00000000000..60be428fa2e
--- /dev/null
+++ b/ompi/mpi/c/iallreduce.c.in
@@ -0,0 +1,113 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2016      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/op/op.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS iallreduce(BUFFER sendbuf, BUFFER_OUT recvbuf, COUNT count,
+                                 DATATYPE datatype, OP op, COMM comm, REQUEST_INOUT request)
+{
+    int err;
+
+    SPC_RECORD(OMPI_SPC_IALLREDUCE, 1);
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_comm(comm);
+
+        /* check whether receive buffer is defined. */
+        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, count, datatype);
+
+        /* check whether the actual send buffer is defined. */
+        if (MPI_IN_PLACE == sendbuf) {
+            memchecker_call(&opal_memchecker_base_isdefined, recvbuf, count, datatype);
+        } else {
+            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        char *msg;
+
+        /* Unrooted operation -- same checks for all ranks on both
+           intracommunicators and intercommunicators */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if (MPI_OP_NULL == op) {
+            err = MPI_ERR_OP;
+        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
+            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
+            free(msg);
+            return ret;
+        } else if ((MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
+                   MPI_IN_PLACE == recvbuf ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_BUFFER,
+                                          FUNC_NAME);
+        } else if( (sendbuf == recvbuf) &&
+                   (MPI_BOTTOM != sendbuf) &&
+                   (count > 1) ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_BUFFER,
+                                          FUNC_NAME);
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+    }
+
+
+    /* MPI standard says that reductions have to have a count of at least 1,
+     * but some benchmarks (e.g., IMB) calls this function with a count of 0.
+     * So handle that case.
+     */
+    if (0 == count) {
+        *request = &ompi_request_empty;
+        return MPI_SUCCESS;
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+
+    err = comm->c_coll->coll_iallreduce(sendbuf, recvbuf, count, datatype,
+                                       op, comm, request, comm->c_coll->coll_iallreduce_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_op(*request, op, datatype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/ialltoall.c b/ompi/mpi/c/ialltoall.c
deleted file mode 100644
index 15451ba1e63..00000000000
--- a/ompi/mpi/c/ialltoall.c
+++ /dev/null
@@ -1,107 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012      Oak Ridge National Laboratory. All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2014-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Ialltoall = PMPI_Ialltoall
-#endif
-#define MPI_Ialltoall PMPI_Ialltoall
-#endif
-
-static const char FUNC_NAME[] = "MPI_Ialltoall";
-
-
-int MPI_Ialltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                  void *recvbuf, int recvcount, MPI_Datatype recvtype,
-                  MPI_Comm comm,  MPI_Request *request)
-{
-    size_t sendtype_size, recvtype_size;
-    int err;
-
-    SPC_RECORD(OMPI_SPC_IALLTOALL, 1);
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-        if (MPI_IN_PLACE != sendbuf) {
-            memchecker_datatype(sendtype);
-            memchecker_call(&opal_memchecker_base_isdefined, (void *)sendbuf, sendcount, sendtype);
-        }
-        memchecker_datatype(recvtype);
-        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-
-        /* Unrooted operation -- same checks for all ranks on both
-           intracommunicators and intercommunicators */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if ((MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
-                   MPI_IN_PLACE == recvbuf) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        } else {
-            if (MPI_IN_PLACE != sendbuf) {
-                OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-                OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-            }
-            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtype, recvcount);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-
-        if (MPI_IN_PLACE != sendbuf && !OMPI_COMM_IS_INTER(comm)) {
-            ompi_datatype_type_size(sendtype, &sendtype_size);
-            ompi_datatype_type_size(recvtype, &recvtype_size);
-            if ((sendtype_size*sendcount) != (recvtype_size*recvcount)) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TRUNCATE, FUNC_NAME);
-            }
-        }
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_ialltoall(sendbuf, sendcount, sendtype,
-                                      recvbuf, recvcount, recvtype, comm,
-                                      request, comm->c_coll->coll_ialltoall_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_datatypes(*request, (MPI_IN_PLACE==sendbuf)?NULL:sendtype, recvtype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/ialltoall.c.in b/ompi/mpi/c/ialltoall.c.in
new file mode 100644
index 00000000000..f6ccdb72815
--- /dev/null
+++ b/ompi/mpi/c/ialltoall.c.in
@@ -0,0 +1,99 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oak Ridge National Laboratory. All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS ialltoall(BUFFER sendbuf, COUNT sendcount, DATATYPE sendtype,
+                                BUFFER_OUT recvbuf, COUNT recvcount, DATATYPE recvtype,
+                                COMM comm, REQUEST_INOUT request)
+{
+    size_t sendtype_size, recvtype_size;
+    int err;
+
+    SPC_RECORD(OMPI_SPC_IALLTOALL, 1);
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+        if (MPI_IN_PLACE != sendbuf) {
+            memchecker_datatype(sendtype);
+            memchecker_call(&opal_memchecker_base_isdefined, (void *)sendbuf, sendcount, sendtype);
+        }
+        memchecker_datatype(recvtype);
+        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+
+        /* Unrooted operation -- same checks for all ranks on both
+           intracommunicators and intercommunicators */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if ((MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
+                   MPI_IN_PLACE == recvbuf) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        } else {
+            if (MPI_IN_PLACE != sendbuf) {
+                OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+                OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+            }
+            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtype, recvcount);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+
+        if (MPI_IN_PLACE != sendbuf && !OMPI_COMM_IS_INTER(comm)) {
+            ompi_datatype_type_size(sendtype, &sendtype_size);
+            ompi_datatype_type_size(recvtype, &recvtype_size);
+            if ((sendtype_size*sendcount) != (recvtype_size*recvcount)) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TRUNCATE, FUNC_NAME);
+            }
+        }
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    err = comm->c_coll->coll_ialltoall(sendbuf, sendcount, sendtype,
+                                      recvbuf, recvcount, recvtype, comm,
+                                      request, comm->c_coll->coll_ialltoall_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_datatypes(*request, (MPI_IN_PLACE==sendbuf)?NULL:sendtype, recvtype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/ialltoallv.c b/ompi/mpi/c/ialltoallv.c
deleted file mode 100644
index 0bf6a51d394..00000000000
--- a/ompi/mpi/c/ialltoallv.c
+++ /dev/null
@@ -1,137 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2014-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Ialltoallv = PMPI_Ialltoallv
-#endif
-#define MPI_Ialltoallv PMPI_Ialltoallv
-#endif
-
-static const char FUNC_NAME[] = "MPI_Ialltoallv";
-
-
-int MPI_Ialltoallv(const void *sendbuf, const int sendcounts[], const int sdispls[],
-                   MPI_Datatype sendtype, void *recvbuf, const int recvcounts[],
-                   const int rdispls[], MPI_Datatype recvtype, MPI_Comm comm,
-                   MPI_Request *request)
-{
-    int i, size, err;
-
-    SPC_RECORD(OMPI_SPC_IALLTOALLV, 1);
-
-    MEMCHECKER(
-        ptrdiff_t recv_ext;
-        ptrdiff_t send_ext;
-
-        memchecker_comm(comm);
-
-        if (MPI_IN_PLACE != sendbuf) {
-            memchecker_datatype(sendtype);
-            ompi_datatype_type_extent(sendtype, &send_ext);
-        }
-
-        memchecker_datatype(recvtype);
-        ompi_datatype_type_extent(recvtype, &recv_ext);
-
-        size = OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm);
-        for ( i = 0; i < size; i++ ) {
-            if (MPI_IN_PLACE != sendbuf) {
-                /* check if send chunks are defined. */
-                memchecker_call(&opal_memchecker_base_isdefined,
-                                (char *)(sendbuf)+sdispls[i]*send_ext,
-                                sendcounts[i], sendtype);
-            }
-            /* check if receive chunks are addressable. */
-            memchecker_call(&opal_memchecker_base_isaddressable,
-                            (char *)(recvbuf)+rdispls[i]*recv_ext,
-                            recvcounts[i], recvtype);
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-
-        /* Unrooted operation -- same checks for all ranks */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-
-        if (MPI_IN_PLACE == sendbuf) {
-            sendcounts = recvcounts;
-            sdispls = rdispls;
-            sendtype = recvtype;
-        }
-
-        if ((NULL == sendcounts) || (NULL == sdispls) ||
-            (NULL == recvcounts) || (NULL == rdispls) ||
-            (MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
-            MPI_IN_PLACE == recvbuf) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-
-        size = OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm);
-        for (i = 0; i < size; ++i) {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcounts[i]);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtype, recvcounts[i]);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-
-        if (MPI_IN_PLACE != sendbuf && !OMPI_COMM_IS_INTER(comm)) {
-            int me = ompi_comm_rank(comm);
-            size_t sendtype_size, recvtype_size;
-            ompi_datatype_type_size(sendtype, &sendtype_size);
-            ompi_datatype_type_size(recvtype, &recvtype_size);
-            if ((sendtype_size*sendcounts[me]) != (recvtype_size*recvcounts[me])) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TRUNCATE, FUNC_NAME);
-            }
-        }
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_ialltoallv(sendbuf, sendcounts, sdispls,
-                                       sendtype, recvbuf, recvcounts, rdispls,
-                                       recvtype, comm, request, comm->c_coll->coll_ialltoallv_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_datatypes(*request, (MPI_IN_PLACE==sendbuf)?NULL:sendtype, recvtype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/ialltoallv.c.in b/ompi/mpi/c/ialltoallv.c.in
new file mode 100644
index 00000000000..838286975ee
--- /dev/null
+++ b/ompi/mpi/c/ialltoallv.c.in
@@ -0,0 +1,135 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS ialltoallv(BUFFER sendbuf, COUNT_ARRAY sendcounts, DISP_ARRAY sdispls,
+                                 DATATYPE sendtype, BUFFER_OUT recvbuf, COUNT_ARRAY recvcounts,
+                                 DISP_ARRAY rdispls, DATATYPE recvtype, COMM comm,
+                                 REQUEST_INOUT request)
+{
+    int i, size, err;
+    ompi_count_array_t sendcounts_desc, recvcounts_desc;
+    ompi_disp_array_t sdispls_desc, rdispls_desc;
+
+    SPC_RECORD(OMPI_SPC_IALLTOALLV, 1);
+
+    MEMCHECKER(
+        ptrdiff_t recv_ext;
+        ptrdiff_t send_ext;
+
+        memchecker_comm(comm);
+
+        if (MPI_IN_PLACE != sendbuf) {
+            memchecker_datatype(sendtype);
+            ompi_datatype_type_extent(sendtype, &send_ext);
+        }
+
+        memchecker_datatype(recvtype);
+        ompi_datatype_type_extent(recvtype, &recv_ext);
+
+        size = OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm);
+        for ( i = 0; i < size; i++ ) {
+            if (MPI_IN_PLACE != sendbuf) {
+                /* check if send chunks are defined. */
+                memchecker_call(&opal_memchecker_base_isdefined,
+                                (char *)(sendbuf)+sdispls[i]*send_ext,
+                                sendcounts[i], sendtype);
+            }
+            /* check if receive chunks are addressable. */
+            memchecker_call(&opal_memchecker_base_isaddressable,
+                            (char *)(recvbuf)+rdispls[i]*recv_ext,
+                            recvcounts[i], recvtype);
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+
+        /* Unrooted operation -- same checks for all ranks */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+
+        if (MPI_IN_PLACE == sendbuf) {
+            sendcounts = recvcounts;
+            sdispls = rdispls;
+            sendtype = recvtype;
+        }
+
+        if ((NULL == sendcounts) || (NULL == sdispls) ||
+            (NULL == recvcounts) || (NULL == rdispls) ||
+            (MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
+            MPI_IN_PLACE == recvbuf) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+
+        size = OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm);
+        for (i = 0; i < size; ++i) {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcounts[i]);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtype, recvcounts[i]);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+
+        if (MPI_IN_PLACE != sendbuf && !OMPI_COMM_IS_INTER(comm)) {
+            int me = ompi_comm_rank(comm);
+            size_t sendtype_size, recvtype_size;
+            ompi_datatype_type_size(sendtype, &sendtype_size);
+            ompi_datatype_type_size(recvtype, &recvtype_size);
+            if ((sendtype_size*sendcounts[me]) != (recvtype_size*recvcounts[me])) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TRUNCATE, FUNC_NAME);
+            }
+        }
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    OMPI_COUNT_ARRAY_INIT(&sendcounts_desc, sendcounts);
+    OMPI_COUNT_ARRAY_INIT(&recvcounts_desc, recvcounts);
+    OMPI_DISP_ARRAY_INIT(&sdispls_desc, sdispls);
+    OMPI_DISP_ARRAY_INIT(&rdispls_desc, rdispls);
+    err = comm->c_coll->coll_ialltoallv(sendbuf, sendcounts_desc, sdispls_desc,
+                                       sendtype, recvbuf, recvcounts_desc, rdispls_desc,
+                                       recvtype, comm, request, comm->c_coll->coll_ialltoallv_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_datatypes(*request, (MPI_IN_PLACE==sendbuf)?NULL:sendtype, recvtype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/ialltoallw.c b/ompi/mpi/c/ialltoallw.c
deleted file mode 100644
index 7c1150d6106..00000000000
--- a/ompi/mpi/c/ialltoallw.c
+++ /dev/null
@@ -1,128 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2014-2020 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Ialltoallw = PMPI_Ialltoallw
-#endif
-#define MPI_Ialltoallw PMPI_Ialltoallw
-#endif
-
-static const char FUNC_NAME[] = "MPI_Ialltoallw";
-
-
-int MPI_Ialltoallw(const void *sendbuf, const int sendcounts[], const int sdispls[],
-                   const MPI_Datatype sendtypes[], void *recvbuf, const int recvcounts[],
-                   const int rdispls[], const MPI_Datatype recvtypes[], MPI_Comm comm,
-                   MPI_Request *request)
-{
-    int i, size, err;
-
-    SPC_RECORD(OMPI_SPC_IALLTOALLW, 1);
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-
-        size = OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm);
-        for ( i = 0; i < size; i++ ) {
-            if (MPI_IN_PLACE != sendbuf) {
-                memchecker_datatype(sendtypes[i]);
-                memchecker_call(&opal_memchecker_base_isdefined,
-                                (char *)(sendbuf)+sdispls[i],
-                                sendcounts[i], sendtypes[i]);
-            }
-
-            memchecker_datatype(recvtypes[i]);
-            memchecker_call(&opal_memchecker_base_isaddressable,
-                            (char *)(recvbuf)+rdispls[i],
-                            recvcounts[i], recvtypes[i]);
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-
-        /* Unrooted operation -- same checks for all ranks */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-
-        if (MPI_IN_PLACE == sendbuf) {
-            sendcounts = recvcounts;
-            sdispls    = rdispls;
-            sendtypes  = recvtypes;
-        }
-
-        if ((NULL == sendcounts) || (NULL == sdispls) || (NULL == sendtypes) ||
-            (NULL == recvcounts) || (NULL == rdispls) || (NULL == recvtypes) ||
-            (MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
-            MPI_IN_PLACE == recvbuf) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-
-        size = OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm);
-        for (i = 0; i < size; ++i) {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtypes[i], sendcounts[i]);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtypes[i], recvcounts[i]);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-
-        if (MPI_IN_PLACE != sendbuf && !OMPI_COMM_IS_INTER(comm)) {
-            int me = ompi_comm_rank(comm);
-            size_t sendtype_size, recvtype_size;
-            ompi_datatype_type_size(sendtypes[me], &sendtype_size);
-            ompi_datatype_type_size(recvtypes[me], &recvtype_size);
-            if ((sendtype_size*sendcounts[me]) != (recvtype_size*recvcounts[me])) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TRUNCATE, FUNC_NAME);
-            }
-        }
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_ialltoallw(sendbuf, sendcounts, sdispls,
-                                       sendtypes, recvbuf, recvcounts,
-                                       rdispls, recvtypes, comm, request,
-                                       comm->c_coll->coll_ialltoallw_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_datatypes_w(*request, (MPI_IN_PLACE==sendbuf)?NULL:sendtypes, recvtypes);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/ialltoallw.c.in b/ompi/mpi/c/ialltoallw.c.in
new file mode 100644
index 00000000000..44e5f4a9e7a
--- /dev/null
+++ b/ompi/mpi/c/ialltoallw.c.in
@@ -0,0 +1,126 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2022 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2020 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS ialltoallw(BUFFER sendbuf, COUNT_ARRAY sendcounts, DISP_ARRAY sdispls,
+                                 DATATYPE_ARRAY sendtypes, BUFFER_OUT recvbuf, COUNT_ARRAY recvcounts,
+                                 DISP_ARRAY rdispls, DATATYPE_ARRAY recvtypes, COMM comm,
+                                 REQUEST_INOUT request)
+{
+    int i, size, err;
+    ompi_count_array_t sendcounts_desc, recvcounts_desc;
+    ompi_disp_array_t sdispls_desc, rdispls_desc;
+
+    SPC_RECORD(OMPI_SPC_IALLTOALLW, 1);
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+
+        size = OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm);
+        for ( i = 0; i < size; i++ ) {
+            if (MPI_IN_PLACE != sendbuf) {
+                memchecker_datatype(sendtypes[i]);
+                memchecker_call(&opal_memchecker_base_isdefined,
+                                (char *)(sendbuf)+sdispls[i],
+                                sendcounts[i], sendtypes[i]);
+            }
+
+            memchecker_datatype(recvtypes[i]);
+            memchecker_call(&opal_memchecker_base_isaddressable,
+                            (char *)(recvbuf)+rdispls[i],
+                            recvcounts[i], recvtypes[i]);
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+
+        /* Unrooted operation -- same checks for all ranks */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+
+        if (MPI_IN_PLACE == sendbuf) {
+            sendcounts = recvcounts;
+            sdispls    = rdispls;
+            sendtypes  = recvtypes;
+        }
+
+        if ((NULL == sendcounts) || (NULL == sdispls) || (NULL == sendtypes) ||
+            (NULL == recvcounts) || (NULL == rdispls) || (NULL == recvtypes) ||
+            (MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
+            MPI_IN_PLACE == recvbuf) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+
+        size = OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm);
+        for (i = 0; i < size; ++i) {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtypes[i], sendcounts[i]);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtypes[i], recvcounts[i]);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+
+        if (MPI_IN_PLACE != sendbuf && !OMPI_COMM_IS_INTER(comm)) {
+            int me = ompi_comm_rank(comm);
+            size_t sendtype_size, recvtype_size;
+            ompi_datatype_type_size(sendtypes[me], &sendtype_size);
+            ompi_datatype_type_size(recvtypes[me], &recvtype_size);
+            if ((sendtype_size*sendcounts[me]) != (recvtype_size*recvcounts[me])) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TRUNCATE, FUNC_NAME);
+            }
+        }
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    OMPI_COUNT_ARRAY_INIT(&sendcounts_desc, sendcounts);
+    OMPI_COUNT_ARRAY_INIT(&recvcounts_desc, recvcounts);
+    OMPI_DISP_ARRAY_INIT(&sdispls_desc, sdispls);
+    OMPI_DISP_ARRAY_INIT(&rdispls_desc, rdispls);
+    err = comm->c_coll->coll_ialltoallw(sendbuf, sendcounts_desc, sdispls_desc,
+                                       sendtypes, recvbuf, recvcounts_desc,
+                                       rdispls_desc, recvtypes, comm, request,
+                                       comm->c_coll->coll_ialltoallw_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_datatypes_w(*request, (MPI_IN_PLACE==sendbuf)?NULL:sendtypes, recvtypes, false);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/ibarrier.c b/ompi/mpi/c/ibarrier.c
deleted file mode 100644
index e5672a14580..00000000000
--- a/ompi/mpi/c/ibarrier.c
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c)      2012 Oak Rigde National Laboratory. All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Ibarrier = PMPI_Ibarrier
-#endif
-#define MPI_Ibarrier PMPI_Ibarrier
-#endif
-
-static const char FUNC_NAME[] = "MPI_Ibarrier";
-
-
-int MPI_Ibarrier(MPI_Comm comm, MPI_Request *request)
-{
-    int err = MPI_SUCCESS;
-
-    SPC_RECORD(OMPI_SPC_IBARRIER, 1);
-
-    MEMCHECKER(
-            memchecker_comm(comm);
-            );
-
-    /* Error checking */
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
-        }
-    }
-
-    err = comm->c_coll->coll_ibarrier(comm, request, comm->c_coll->coll_ibarrier_module);
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/ibarrier.c.in b/ompi/mpi/c/ibarrier.c.in
new file mode 100644
index 00000000000..a0c23d31dec
--- /dev/null
+++ b/ompi/mpi/c/ibarrier.c.in
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c)      2012 Oak Rigde National Laboratory. All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS ibarrier(COMM comm, REQUEST_INOUT request)
+{
+    int err = MPI_SUCCESS;
+
+    SPC_RECORD(OMPI_SPC_IBARRIER, 1);
+
+    MEMCHECKER(
+            memchecker_comm(comm);
+            );
+
+    /* Error checking */
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+        }
+    }
+
+    err = comm->c_coll->coll_ibarrier(comm, request, comm->c_coll->coll_ibarrier_module);
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/ibcast.c b/ompi/mpi/c/ibcast.c
deleted file mode 100644
index acb55f93b9b..00000000000
--- a/ompi/mpi/c/ibcast.c
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Copyright (c) 2012      Oak Rigde National Laboratory. All rights reserved.
- * Copyright (c) 2015-2020 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2017-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Ibcast = PMPI_Ibcast
-#endif
-#define MPI_Ibcast PMPI_Ibcast
-#endif
-
-static const char FUNC_NAME[] = "MPI_Ibcast";
-
-
-int MPI_Ibcast(void *buffer, int count, MPI_Datatype datatype,
-              int root, MPI_Comm comm,  MPI_Request *request)
-{
-    int err;
-
-    SPC_RECORD(OMPI_SPC_IBCAST, 1);
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_comm(comm);
-        if (OMPI_COMM_IS_INTRA(comm)) {
-            if (ompi_comm_rank(comm) == root) {
-                /* check whether root's send buffer is defined. */
-                memchecker_call(&opal_memchecker_base_isdefined, buffer, count, datatype);
-            } else {
-                /* check whether receive buffer is addressable. */
-                memchecker_call(&opal_memchecker_base_isaddressable, buffer, count, datatype);
-            }
-        } else {
-            if (MPI_ROOT == root) {
-                /* check whether root's send buffer is defined. */
-                memchecker_call(&opal_memchecker_base_isdefined, buffer, count, datatype);
-            } else if (MPI_PROC_NULL != root) {
-                /* check whether receive buffer is addressable. */
-                memchecker_call(&opal_memchecker_base_isaddressable, buffer, count, datatype);
-            }
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-      err = MPI_SUCCESS;
-      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-      if (ompi_comm_invalid(comm)) {
-          return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                     FUNC_NAME);
-      }
-
-      /* Errors for all ranks */
-
-      OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
-      OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-      if (MPI_IN_PLACE == buffer) {
-          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-      }
-
-      /* Errors for intracommunicators */
-
-      if (OMPI_COMM_IS_INTRA(comm)) {
-        if ((root >= ompi_comm_size(comm)) || (root < 0)) {
-          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
-        }
-      }
-
-      /* Errors for intercommunicators */
-
-      else {
-        if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
-               MPI_ROOT == root || MPI_PROC_NULL == root)) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
-        }
-      }
-    }
-
-    /* If there's only one node, or if the count is 0, we're done */
-
-    if ((OMPI_COMM_IS_INTRA(comm) && ompi_comm_size(comm) <= 1) ||
-        0 == count) {
-        *request = &ompi_request_empty;
-        return MPI_SUCCESS;
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-
-    err = comm->c_coll->coll_ibcast(buffer, count, datatype, root, comm,
-                                  request,
-                                  comm->c_coll->coll_ibcast_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        if (!OMPI_COMM_IS_INTRA(comm)) {
-            if (MPI_PROC_NULL == root) {
-                datatype = NULL;
-            }
-        }
-        ompi_coll_base_retain_datatypes(*request, datatype, NULL);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/ibcast.c.in b/ompi/mpi/c/ibcast.c.in
new file mode 100644
index 00000000000..70eaea4a18f
--- /dev/null
+++ b/ompi/mpi/c/ibcast.c.in
@@ -0,0 +1,113 @@
+/*
+ * Copyright (c) 2012      Oak Rigde National Laboratory. All rights reserved.
+ * Copyright (c) 2015-2020 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2017-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS ibcast(BUFFER_OUT buffer, COUNT count, DATATYPE datatype,
+                             INT root, COMM comm,  REQUEST_INOUT request)
+{
+    int err;
+
+    SPC_RECORD(OMPI_SPC_IBCAST, 1);
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_comm(comm);
+        if (OMPI_COMM_IS_INTRA(comm)) {
+            if (ompi_comm_rank(comm) == root) {
+                /* check whether root's send buffer is defined. */
+                memchecker_call(&opal_memchecker_base_isdefined, buffer, count, datatype);
+            } else {
+                /* check whether receive buffer is addressable. */
+                memchecker_call(&opal_memchecker_base_isaddressable, buffer, count, datatype);
+            }
+        } else {
+            if (MPI_ROOT == root) {
+                /* check whether root's send buffer is defined. */
+                memchecker_call(&opal_memchecker_base_isdefined, buffer, count, datatype);
+            } else if (MPI_PROC_NULL != root) {
+                /* check whether receive buffer is addressable. */
+                memchecker_call(&opal_memchecker_base_isaddressable, buffer, count, datatype);
+            }
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+      err = MPI_SUCCESS;
+      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+      if (ompi_comm_invalid(comm)) {
+          return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                     FUNC_NAME);
+      }
+
+      /* Errors for all ranks */
+
+      OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
+      OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+      if (MPI_IN_PLACE == buffer) {
+          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+      }
+
+      /* Errors for intracommunicators */
+
+      if (OMPI_COMM_IS_INTRA(comm)) {
+        if ((root >= ompi_comm_size(comm)) || (root < 0)) {
+          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
+        }
+      }
+
+      /* Errors for intercommunicators */
+
+      else {
+        if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
+               MPI_ROOT == root || MPI_PROC_NULL == root)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
+        }
+      }
+    }
+
+    /* If there's only one node, or if the count is 0, we're done */
+
+    if ((OMPI_COMM_IS_INTRA(comm) && ompi_comm_size(comm) <= 1) ||
+        0 == count) {
+        *request = &ompi_request_empty;
+        return MPI_SUCCESS;
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+
+    err = comm->c_coll->coll_ibcast(buffer, count, datatype, root, comm,
+                                  request,
+                                  comm->c_coll->coll_ibcast_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        if (!OMPI_COMM_IS_INTRA(comm)) {
+            if (MPI_PROC_NULL == root) {
+                datatype = NULL;
+            }
+        }
+        ompi_coll_base_retain_datatypes(*request, datatype, NULL);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/ibsend.c b/ompi/mpi/c/ibsend.c
deleted file mode 100644
index fa22b745213..00000000000
--- a/ompi/mpi/c/ibsend.c
+++ /dev/null
@@ -1,98 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/mca/pml/base/pml_base_bsend.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Ibsend = PMPI_Ibsend
-#endif
-#define MPI_Ibsend PMPI_Ibsend
-#endif
-
-static const char FUNC_NAME[] = "MPI_Ibsend";
-
-
-int MPI_Ibsend(const void *buf, int count, MPI_Datatype type, int dest,
-               int tag, MPI_Comm comm, MPI_Request *request)
-{
-    int rc = MPI_SUCCESS;
-
-    SPC_RECORD(OMPI_SPC_IBSEND, 1);
-
-    MEMCHECKER(
-        memchecker_datatype(type);
-        memchecker_call(&opal_memchecker_base_isdefined, buf, count, type);
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else if (tag < 0 || tag > mca_pml.pml_max_tag) {
-            rc = MPI_ERR_TAG;
-        } else if (ompi_comm_peer_invalid(comm, dest) &&
-                   (MPI_PROC_NULL != dest)) {
-            rc = MPI_ERR_RANK;
-        } else if (request == NULL) {
-            rc = MPI_ERR_REQUEST;
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_SEND(rc, type, count);
-            OMPI_CHECK_USER_BUFFER(rc, buf, type, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-    }
-
-    if (MPI_PROC_NULL == dest) {
-        *request = &ompi_request_empty;
-        return MPI_SUCCESS;
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * The request will be checked for process failure errors during the
-     * completion calls. So no need to check here.
-     */
-#endif
-
-    MEMCHECKER (
-        memchecker_call(&opal_memchecker_base_mem_noaccess, buf, count, type);
-    );
-    rc = MCA_PML_CALL(isend(buf, count, type, dest, tag, MCA_PML_BASE_SEND_BUFFERED, comm, request));
-    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/ibsend.c.in b/ompi/mpi/c/ibsend.c.in
new file mode 100644
index 00000000000..dbe9b9e0c29
--- /dev/null
+++ b/ompi/mpi/c/ibsend.c.in
@@ -0,0 +1,90 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/mca/pml/base/pml_base_bsend.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS ibsend(BUFFER buf, COUNT count, DATATYPE type, INT dest,
+                             INT tag, COMM comm, REQUEST_INOUT request)
+{
+    int rc = MPI_SUCCESS;
+
+    SPC_RECORD(OMPI_SPC_IBSEND, 1);
+
+    MEMCHECKER(
+        memchecker_datatype(type);
+        memchecker_call(&opal_memchecker_base_isdefined, buf, count, type);
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else if (tag < 0 || tag > mca_pml.pml_max_tag) {
+            rc = MPI_ERR_TAG;
+        } else if (ompi_comm_peer_invalid(comm, dest) &&
+                   (MPI_PROC_NULL != dest)) {
+            rc = MPI_ERR_RANK;
+        } else if (request == NULL) {
+            rc = MPI_ERR_REQUEST;
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_SEND(rc, type, count);
+            OMPI_CHECK_USER_BUFFER(rc, buf, type, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+    if (MPI_PROC_NULL == dest) {
+        *request = &ompi_request_empty;
+        return MPI_SUCCESS;
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * The request will be checked for process failure errors during the
+     * completion calls. So no need to check here.
+     */
+#endif
+
+    MEMCHECKER (
+        memchecker_call(&opal_memchecker_base_mem_noaccess, buf, count, type);
+    );
+    rc = MCA_PML_CALL(isend(buf, count, type, dest, tag, MCA_PML_BASE_SEND_BUFFERED, comm, request));
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/iexscan.c b/ompi/mpi/c/iexscan.c
deleted file mode 100644
index 0c90e47af12..00000000000
--- a/ompi/mpi/c/iexscan.c
+++ /dev/null
@@ -1,96 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/op/op.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Iexscan = PMPI_Iexscan
-#endif
-#define MPI_Iexscan PMPI_Iexscan
-#endif
-
-static const char FUNC_NAME[] = "MPI_Iexscan";
-
-
-int MPI_Iexscan(const void *sendbuf, void *recvbuf, int count,
-                MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, MPI_Request *request)
-{
-    int err;
-
-    SPC_RECORD(OMPI_SPC_IEXSCAN, 1);
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        if (MPI_IN_PLACE != sendbuf) {
-            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
-        } else {
-            memchecker_call(&opal_memchecker_base_isdefined, recvbuf, count, datatype);
-        }
-        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, count, datatype);
-        memchecker_comm(comm);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        char *msg;
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-
-        /* Unrooted operation -- same checks for intracommunicators
-           and intercommunicators */
-        else if (MPI_OP_NULL == op) {
-            err = MPI_ERR_OP;
-        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
-            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
-            free(msg);
-            return ret;
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-
-    err = comm->c_coll->coll_iexscan(sendbuf, recvbuf, count,
-                                    datatype, op, comm, request,
-                                    comm->c_coll->coll_iexscan_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_op(*request, op, datatype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/iexscan.c.in b/ompi/mpi/c/iexscan.c.in
new file mode 100644
index 00000000000..a945d98ae53
--- /dev/null
+++ b/ompi/mpi/c/iexscan.c.in
@@ -0,0 +1,88 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/op/op.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS iexscan(BUFFER sendbuf, BUFFER_OUT recvbuf, COUNT count,
+                              DATATYPE datatype, OP op, COMM comm, REQUEST_INOUT request)
+{
+    int err;
+
+    SPC_RECORD(OMPI_SPC_IEXSCAN, 1);
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        if (MPI_IN_PLACE != sendbuf) {
+            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
+        } else {
+            memchecker_call(&opal_memchecker_base_isdefined, recvbuf, count, datatype);
+        }
+        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, count, datatype);
+        memchecker_comm(comm);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        char *msg;
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+
+        /* Unrooted operation -- same checks for intracommunicators
+           and intercommunicators */
+        else if (MPI_OP_NULL == op) {
+            err = MPI_ERR_OP;
+        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
+            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
+            free(msg);
+            return ret;
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+
+    err = comm->c_coll->coll_iexscan(sendbuf, recvbuf, count,
+                                    datatype, op, comm, request,
+                                    comm->c_coll->coll_iexscan_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_op(*request, op, datatype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/igather.c b/ompi/mpi/c/igather.c
deleted file mode 100644
index b501a757d78..00000000000
--- a/ompi/mpi/c/igather.c
+++ /dev/null
@@ -1,195 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2008      University of Houston.  All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Igather = PMPI_Igather
-#endif
-#define MPI_Igather PMPI_Igather
-#endif
-
-static const char FUNC_NAME[] = "MPI_Igather";
-
-
-int MPI_Igather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                void *recvbuf, int recvcount, MPI_Datatype recvtype,
-                int root, MPI_Comm comm, MPI_Request *request)
-{
-    int err;
-
-    SPC_RECORD(OMPI_SPC_IGATHER, 1);
-
-    MEMCHECKER(
-        int rank;
-        ptrdiff_t ext;
-
-        rank = ompi_comm_rank(comm);
-        ompi_datatype_type_extent(recvtype, &ext);
-
-        memchecker_comm(comm);
-        if(OMPI_COMM_IS_INTRA(comm)) {
-            if(ompi_comm_rank(comm) == root) {
-                /* check whether root's send buffer is defined. */
-                if (MPI_IN_PLACE == sendbuf) {
-                  memchecker_call(&opal_memchecker_base_isdefined,
-                                  (char *)(recvbuf)+rank*ext,
-                                  recvcount, recvtype);
-                } else {
-                    memchecker_datatype(sendtype);
-                    memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-                }
-
-                memchecker_datatype(recvtype);
-                /* check whether root's receive buffer is addressable. */
-                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-            } else {
-                memchecker_datatype(sendtype);
-                /* check whether send buffer is defined on other processes. */
-                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-            }
-        } else {
-            if (MPI_ROOT == root) {
-                memchecker_datatype(recvtype);
-                /* check whether root's receive buffer is addressable. */
-                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-            } else if (MPI_PROC_NULL != root) {
-                memchecker_datatype(sendtype);
-                /* check whether send buffer is defined. */
-                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-            }
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if ((ompi_comm_rank(comm) != root && MPI_IN_PLACE == sendbuf) ||
-                   (ompi_comm_rank(comm) == root && MPI_IN_PLACE == recvbuf)) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-
-        /* Errors for intracommunicators */
-
-        if (OMPI_COMM_IS_INTRA(comm)) {
-
-            /* Errors for all ranks */
-
-            if ((root >= ompi_comm_size(comm)) || (root < 0)) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
-            }
-
-            if (MPI_IN_PLACE != sendbuf) {
-                OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-            }
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-
-            /* Errors for the root.  Some of these could have been
-               combined into compound if statements above, but since
-               this whole section can be compiled out (or turned off at
-               run time) for efficiency, it's more clear to separate
-               them out into individual tests. */
-
-            if (ompi_comm_rank(comm) == root) {
-                if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
-                }
-
-                if (recvcount < 0) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-                }
-            }
-        }
-
-        /* Errors for intercommunicators */
-
-        else {
-            if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
-                   MPI_ROOT == root || MPI_PROC_NULL == root)) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
-            }
-
-            /* Errors for the senders */
-
-            if (MPI_ROOT != root && MPI_PROC_NULL != root) {
-                OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-                OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-            }
-
-            /* Errors for the root.  Ditto on the comment above -- these
-               error checks could have been combined above, but let's
-               make the code easier to read. */
-
-            else if (MPI_ROOT == root) {
-                if (recvcount < 0) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-                }
-
-                if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
-                }
-            }
-        }
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_igather(sendbuf, sendcount, sendtype, recvbuf,
-                                    recvcount, recvtype, root, comm, request,
-                                    comm->c_coll->coll_igather_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        if (OMPI_COMM_IS_INTRA(comm)) {
-            if (MPI_IN_PLACE == sendbuf) {
-                sendtype = NULL;
-            } else if (ompi_comm_rank(comm) != root) {
-                recvtype = NULL;
-            }
-        } else {
-            if (MPI_ROOT == root) {
-                sendtype = NULL;
-            } else if (MPI_PROC_NULL == root) {
-                sendtype = NULL;
-                recvtype = NULL;
-            } else {
-                recvtype = NULL;
-            }
-        }
-        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/igather.c.in b/ompi/mpi/c/igather.c.in
new file mode 100644
index 00000000000..1786e7056eb
--- /dev/null
+++ b/ompi/mpi/c/igather.c.in
@@ -0,0 +1,196 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2023 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2008      University of Houston.  All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS igather(BUFFER sendbuf, COUNT sendcount, DATATYPE sendtype,
+                              BUFFER_OUT recvbuf, COUNT recvcount, DATATYPE recvtype,
+                              INT root, COMM comm, REQUEST_INOUT request)
+{
+    int err;
+
+    SPC_RECORD(OMPI_SPC_IGATHER, 1);
+
+    MEMCHECKER(
+        int rank;
+        ptrdiff_t ext;
+
+        rank = ompi_comm_rank(comm);
+
+        memchecker_comm(comm);
+        if(OMPI_COMM_IS_INTRA(comm)) {
+            if(ompi_comm_rank(comm) == root) {
+                /* check whether root's send buffer is defined. */
+                if (MPI_IN_PLACE == sendbuf) {
+                  memchecker_datatype(recvtype);
+                  ompi_datatype_type_extent(recvtype, &ext);
+                  memchecker_call(&opal_memchecker_base_isdefined,
+                                  (char *)(recvbuf)+rank*ext,
+                                  recvcount, recvtype);
+                } else {
+                    memchecker_datatype(sendtype);
+                    memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+                }
+
+                memchecker_datatype(recvtype);
+                /* check whether root's receive buffer is addressable. */
+                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+            } else {
+                memchecker_datatype(sendtype);
+                /* check whether send buffer is defined on other processes. */
+                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+            }
+        } else {
+            if (MPI_ROOT == root) {
+                memchecker_datatype(recvtype);
+                /* check whether root's receive buffer is addressable. */
+                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+            } else if (MPI_PROC_NULL != root) {
+                memchecker_datatype(sendtype);
+                /* check whether send buffer is defined. */
+                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+            }
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if ((ompi_comm_rank(comm) != root && MPI_IN_PLACE == sendbuf) ||
+                   (ompi_comm_rank(comm) == root && MPI_IN_PLACE == recvbuf)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+
+        /* Errors for intracommunicators */
+
+        if (OMPI_COMM_IS_INTRA(comm)) {
+
+            /* Errors for all ranks */
+
+            if ((root >= ompi_comm_size(comm)) || (root < 0)) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
+            }
+
+            if (MPI_IN_PLACE != sendbuf) {
+                OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+            }
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+
+            /* Errors for the root.  Some of these could have been
+               combined into compound if statements above, but since
+               this whole section can be compiled out (or turned off at
+               run time) for efficiency, it's more clear to separate
+               them out into individual tests. */
+
+            if (ompi_comm_rank(comm) == root) {
+                if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
+                }
+
+                if (recvcount < 0) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+                }
+            }
+        }
+
+        /* Errors for intercommunicators */
+
+        else {
+            if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
+                   MPI_ROOT == root || MPI_PROC_NULL == root)) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
+            }
+
+            /* Errors for the senders */
+
+            if (MPI_ROOT != root && MPI_PROC_NULL != root) {
+                OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+                OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+            }
+
+            /* Errors for the root.  Ditto on the comment above -- these
+               error checks could have been combined above, but let's
+               make the code easier to read. */
+
+            else if (MPI_ROOT == root) {
+                if (recvcount < 0) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+                }
+
+                if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
+                }
+            }
+        }
+    }
+
+    void* updated_recvbuf;
+    if (OMPI_COMM_IS_INTRA(comm)) {
+        updated_recvbuf = (ompi_comm_rank(comm) == root) ? recvbuf : NULL;
+    } else {
+        updated_recvbuf = (root == MPI_ROOT) ? recvbuf : NULL;
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    err = comm->c_coll->coll_igather(sendbuf, sendcount, sendtype, updated_recvbuf,
+                                    recvcount, recvtype, root, comm, request,
+                                    comm->c_coll->coll_igather_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        if (OMPI_COMM_IS_INTRA(comm)) {
+            if (MPI_IN_PLACE == sendbuf) {
+                sendtype = NULL;
+            } else if (ompi_comm_rank(comm) != root) {
+                recvtype = NULL;
+            }
+        } else {
+            if (MPI_ROOT == root) {
+                sendtype = NULL;
+            } else if (MPI_PROC_NULL == root) {
+                sendtype = NULL;
+                recvtype = NULL;
+            } else {
+                recvtype = NULL;
+            }
+        }
+        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/igatherv.c b/ompi/mpi/c/igatherv.c
deleted file mode 100644
index e1b5c585cc7..00000000000
--- a/ompi/mpi/c/igatherv.c
+++ /dev/null
@@ -1,218 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Igatherv = PMPI_Igatherv
-#endif
-#define MPI_Igatherv PMPI_Igatherv
-#endif
-
-static const char FUNC_NAME[] = "MPI_Igatherv";
-
-
-int MPI_Igatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                 void *recvbuf, const int recvcounts[], const int displs[],
-                 MPI_Datatype recvtype, int root, MPI_Comm comm, MPI_Request *request)
-{
-    int i, size, err;
-
-    SPC_RECORD(OMPI_SPC_IGATHERV, 1);
-
-    MEMCHECKER(
-        ptrdiff_t ext;
-
-        size = ompi_comm_remote_size(comm);
-        ompi_datatype_type_extent(recvtype, &ext);
-
-        memchecker_comm(comm);
-        if(OMPI_COMM_IS_INTRA(comm)) {
-            if(ompi_comm_rank(comm) == root) {
-                /* check whether root's send buffer is defined. */
-                if (MPI_IN_PLACE == sendbuf) {
-                    for (i = 0; i < size; i++) {
-                        memchecker_call(&opal_memchecker_base_isdefined,
-                                        (char *)(recvbuf)+displs[i]*ext,
-                                        recvcounts[i], recvtype);
-                    }
-                } else {
-                    memchecker_datatype(sendtype);
-                    memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-                }
-
-                memchecker_datatype(recvtype);
-                /* check whether root's receive buffer is addressable. */
-                for (i = 0; i < size; i++) {
-                    memchecker_call(&opal_memchecker_base_isaddressable,
-                                    (char *)(recvbuf)+displs[i]*ext,
-                                    recvcounts[i], recvtype);
-                }
-            } else {
-                memchecker_datatype(sendtype);
-                /* check whether send buffer is defined on other processes. */
-                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-            }
-        } else {
-            if (MPI_ROOT == root) {
-                memchecker_datatype(recvtype);
-                /* check whether root's receive buffer is addressable. */
-                for (i = 0; i < size; i++) {
-                    memchecker_call(&opal_memchecker_base_isaddressable,
-                                    (char *)(recvbuf)+displs[i]*ext,
-                                    recvcounts[i], recvtype);
-                }
-            } else if (MPI_PROC_NULL != root) {
-                memchecker_datatype(sendtype);
-                /* check whether send buffer is defined. */
-                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-            }
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if ((ompi_comm_rank(comm) != root && MPI_IN_PLACE == sendbuf) ||
-                   (ompi_comm_rank(comm) == root && MPI_IN_PLACE == recvbuf)) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-
-        /* Errors for intracommunicators */
-
-        if (OMPI_COMM_IS_INTRA(comm)) {
-
-            /* Errors for all ranks */
-
-            if ((root >= ompi_comm_size(comm)) || (root < 0)) {
-                err = MPI_ERR_ROOT;
-            } else if (MPI_IN_PLACE != sendbuf) {
-                OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-            }
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-
-            /* Errors for the root.  Some of these could have been
-               combined into compound if statements above, but since
-               this whole section can be compiled out (or turned off at
-               run time) for efficiency, it's more clear to separate
-               them out into individual tests. */
-
-            if (ompi_comm_rank(comm) == root) {
-                if (NULL == displs) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-                }
-
-                if (NULL == recvcounts) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-                }
-
-                size = ompi_comm_size(comm);
-                for (i = 0; i < size; ++i) {
-                    if (recvcounts[i] < 0) {
-                        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-                    } else if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
-                        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
-                    }
-                }
-            }
-        }
-
-        /* Errors for intercommunicators */
-
-        else {
-            if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
-                   MPI_ROOT == root || MPI_PROC_NULL == root)) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
-            }
-
-            /* Errors for the senders */
-
-            if (MPI_ROOT != root && MPI_PROC_NULL != root) {
-                OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-                OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-            }
-
-            /* Errors for the root.  Ditto on the comment above -- these
-               error checks could have been combined above, but let's
-               make the code easier to read. */
-
-            else if (MPI_ROOT == root) {
-                if (NULL == displs) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-                }
-
-                if (NULL == recvcounts) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-                }
-
-                size = ompi_comm_remote_size(comm);
-                for (i = 0; i < size; ++i) {
-                    if (recvcounts[i] < 0) {
-                        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-                    } else if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
-                        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
-                    }
-                }
-            }
-        }
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_igatherv(sendbuf, sendcount, sendtype, recvbuf,
-                                     recvcounts, displs, recvtype,
-                                     root, comm, request, comm->c_coll->coll_igatherv_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        if (OMPI_COMM_IS_INTRA(comm)) {
-            if (MPI_IN_PLACE == sendbuf) {
-                sendtype = NULL;
-            } else if (ompi_comm_rank(comm) != root) {
-                recvtype = NULL;
-            }
-        } else {
-            if (MPI_ROOT == root) {
-                sendtype = NULL;
-            } else if (MPI_PROC_NULL == root) {
-                sendtype = NULL;
-                recvtype = NULL;
-            } else {
-                recvtype = NULL;
-            }
-        }
-        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/igatherv.c.in b/ompi/mpi/c/igatherv.c.in
new file mode 100644
index 00000000000..346db74a4c9
--- /dev/null
+++ b/ompi/mpi/c/igatherv.c.in
@@ -0,0 +1,221 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2023 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS igatherv(BUFFER sendbuf, COUNT sendcount, DATATYPE sendtype,
+                               BUFFER_OUT recvbuf, COUNT_ARRAY recvcounts, DISP_ARRAY displs,
+                               DATATYPE recvtype, INT root, COMM comm, REQUEST_INOUT request)
+{
+    int i, size, err;
+    ompi_count_array_t recvcounts_desc;
+    ompi_disp_array_t displs_desc;
+
+    SPC_RECORD(OMPI_SPC_IGATHERV, 1);
+
+    MEMCHECKER(
+        ptrdiff_t ext;
+
+        size = ompi_comm_remote_size(comm);
+        ompi_datatype_type_extent(recvtype, &ext);
+
+        memchecker_comm(comm);
+        if(OMPI_COMM_IS_INTRA(comm)) {
+            if(ompi_comm_rank(comm) == root) {
+                /* check whether root's send buffer is defined. */
+                if (MPI_IN_PLACE == sendbuf) {
+                    for (i = 0; i < size; i++) {
+                        memchecker_call(&opal_memchecker_base_isdefined,
+                                        (char *)(recvbuf)+displs[i]*ext,
+                                        recvcounts[i], recvtype);
+                    }
+                } else {
+                    memchecker_datatype(sendtype);
+                    memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+                }
+
+                memchecker_datatype(recvtype);
+                /* check whether root's receive buffer is addressable. */
+                for (i = 0; i < size; i++) {
+                    memchecker_call(&opal_memchecker_base_isaddressable,
+                                    (char *)(recvbuf)+displs[i]*ext,
+                                    recvcounts[i], recvtype);
+                }
+            } else {
+                memchecker_datatype(sendtype);
+                /* check whether send buffer is defined on other processes. */
+                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+            }
+        } else {
+            if (MPI_ROOT == root) {
+                memchecker_datatype(recvtype);
+                /* check whether root's receive buffer is addressable. */
+                for (i = 0; i < size; i++) {
+                    memchecker_call(&opal_memchecker_base_isaddressable,
+                                    (char *)(recvbuf)+displs[i]*ext,
+                                    recvcounts[i], recvtype);
+                }
+            } else if (MPI_PROC_NULL != root) {
+                memchecker_datatype(sendtype);
+                /* check whether send buffer is defined. */
+                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+            }
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if ((ompi_comm_rank(comm) != root && MPI_IN_PLACE == sendbuf) ||
+                   (ompi_comm_rank(comm) == root && MPI_IN_PLACE == recvbuf)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+
+        /* Errors for intracommunicators */
+
+        if (OMPI_COMM_IS_INTRA(comm)) {
+
+            /* Errors for all ranks */
+
+            if ((root >= ompi_comm_size(comm)) || (root < 0)) {
+                err = MPI_ERR_ROOT;
+            } else if (MPI_IN_PLACE != sendbuf) {
+                OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+            }
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+
+            /* Errors for the root.  Some of these could have been
+               combined into compound if statements above, but since
+               this whole section can be compiled out (or turned off at
+               run time) for efficiency, it's more clear to separate
+               them out into individual tests. */
+
+            if (ompi_comm_rank(comm) == root) {
+                if (NULL == displs) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+                }
+
+                if (NULL == recvcounts) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+                }
+
+                size = ompi_comm_size(comm);
+                for (i = 0; i < size; ++i) {
+                    if (recvcounts[i] < 0) {
+                        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+                    } else if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
+                        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
+                    }
+                }
+            }
+        }
+
+        /* Errors for intercommunicators */
+
+        else {
+            if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
+                   MPI_ROOT == root || MPI_PROC_NULL == root)) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
+            }
+
+            /* Errors for the senders */
+
+            if (MPI_ROOT != root && MPI_PROC_NULL != root) {
+                OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+                OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+            }
+
+            /* Errors for the root.  Ditto on the comment above -- these
+               error checks could have been combined above, but let's
+               make the code easier to read. */
+
+            else if (MPI_ROOT == root) {
+                if (NULL == displs) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+                }
+
+                if (NULL == recvcounts) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+                }
+
+                size = ompi_comm_remote_size(comm);
+                for (i = 0; i < size; ++i) {
+                    if (recvcounts[i] < 0) {
+                        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+                    } else if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
+                        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
+                    }
+                }
+            }
+        }
+    }
+
+    void* updated_recvbuf;
+    if (OMPI_COMM_IS_INTRA(comm)) {
+        updated_recvbuf = (ompi_comm_rank(comm) == root) ? recvbuf : NULL;
+    } else {
+        updated_recvbuf = (root == MPI_ROOT) ? recvbuf : NULL;
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    OMPI_COUNT_ARRAY_INIT(&recvcounts_desc, recvcounts);
+    OMPI_DISP_ARRAY_INIT(&displs_desc, displs);
+    err = comm->c_coll->coll_igatherv(sendbuf, sendcount, sendtype, updated_recvbuf,
+                                     recvcounts_desc, displs_desc, recvtype,
+                                     root, comm, request, comm->c_coll->coll_igatherv_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        if (OMPI_COMM_IS_INTRA(comm)) {
+            if (MPI_IN_PLACE == sendbuf) {
+                sendtype = NULL;
+            } else if (ompi_comm_rank(comm) != root) {
+                recvtype = NULL;
+            }
+        } else {
+            if (MPI_ROOT == root) {
+                sendtype = NULL;
+            } else if (MPI_PROC_NULL == root) {
+                sendtype = NULL;
+                recvtype = NULL;
+            } else {
+                recvtype = NULL;
+            }
+        }
+        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/improbe.c b/ompi/mpi/c/improbe.c
deleted file mode 100644
index 22831c4bad3..00000000000
--- a/ompi/mpi/c/improbe.c
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
- * Copyright (c) 2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2020      The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/memchecker.h"
-#include "ompi/request/request.h"
-#include "ompi/message/message.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Improbe = PMPI_Improbe
-#endif
-#define MPI_Improbe PMPI_Improbe
-#endif
-
-static const char FUNC_NAME[] = "MPI_Improbe";
-
-
-int MPI_Improbe(int source, int tag, MPI_Comm comm, int *flag,
-                MPI_Message *message, MPI_Status *status)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (((tag < 0) && (tag != MPI_ANY_TAG)) || (tag > mca_pml.pml_max_tag)) {
-            rc = MPI_ERR_TAG;
-        } else if (ompi_comm_invalid(comm)) {
-            rc = MPI_ERR_COMM;
-        } else if ((source != MPI_ANY_SOURCE) &&
-                   (MPI_PROC_NULL != source) &&
-                   ompi_comm_peer_invalid(comm, source)) {
-            rc = MPI_ERR_RANK;
-        } else if (NULL == message) {
-            rc = MPI_ERR_REQUEST;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-    }
-
-    if (MPI_PROC_NULL == source) {
-        if (MPI_STATUS_IGNORE != status) {
-            *status = ompi_request_empty.req_status;
-            /* Per MPI-1, the MPI_ERROR field is not defined for
-               single-completion calls */
-            MEMCHECKER(
-                opal_memchecker_base_mem_undefined(&status->MPI_ERROR, sizeof(int));
-            );
-        }
-        *message = &ompi_message_no_proc.message;
-        *flag = 1;
-        return MPI_SUCCESS;
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * The request will be checked for process failure errors during the
-     * completion calls. So no need to check here.
-     */
-#endif
-
-    rc = MCA_PML_CALL(improbe(source, tag, comm, flag, message, status));
-    /* Per MPI-1, the MPI_ERROR field is not defined for
-       single-completion calls */
-    MEMCHECKER(
-        opal_memchecker_base_mem_undefined(&status->MPI_ERROR, sizeof(int));
-    );
-
-    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/improbe.c.in b/ompi/mpi/c/improbe.c.in
new file mode 100644
index 00000000000..a0f1c7c221f
--- /dev/null
+++ b/ompi/mpi/c/improbe.c.in
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2012      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2020-2021 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/memchecker.h"
+#include "ompi/request/request.h"
+#include "ompi/message/message.h"
+
+PROTOTYPE ERROR_CLASS improbe(INT source, INT tag, COMM comm, INT_OUT flag,
+                              MESSAGE_OUT message, STATUS_OUT status)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (((tag < 0) && (tag != MPI_ANY_TAG)) || (tag > mca_pml.pml_max_tag)) {
+            rc = MPI_ERR_TAG;
+        } else if (ompi_comm_invalid(comm)) {
+            rc = MPI_ERR_COMM;
+        } else if ((source != MPI_ANY_SOURCE) &&
+                   (MPI_PROC_NULL != source) &&
+                   ompi_comm_peer_invalid(comm, source)) {
+            rc = MPI_ERR_RANK;
+        } else if (NULL == message) {
+            rc = MPI_ERR_REQUEST;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+    if (MPI_PROC_NULL == source) {
+        if (MPI_STATUS_IGNORE != status) {
+            OMPI_COPY_STATUS(status, ompi_request_empty.req_status, false);
+            /* Per MPI-1, the MPI_ERROR field is not defined for
+               single-completion calls */
+            MEMCHECKER(
+                opal_memchecker_base_mem_undefined(&status->MPI_ERROR, sizeof(int));
+            );
+        }
+        *message = &ompi_message_no_proc.message;
+        *flag = 1;
+        return MPI_SUCCESS;
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * The request will be checked for process failure errors during the
+     * completion calls. So no need to check here.
+     */
+#endif
+
+    rc = MCA_PML_CALL(improbe(source, tag, comm, flag, message, status));
+    /* Per MPI-1, the MPI_ERROR field is not defined for
+       single-completion calls */
+    MEMCHECKER(
+        opal_memchecker_base_mem_undefined(&status->MPI_ERROR, sizeof(int));
+    );
+
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/imrecv.c b/ompi/mpi/c/imrecv.c
deleted file mode 100644
index 6e7295adcb2..00000000000
--- a/ompi/mpi/c/imrecv.c
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
- * Copyright (c) 2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012      Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/memchecker.h"
-#include "ompi/request/request.h"
-#include "ompi/message/message.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Imrecv = PMPI_Imrecv
-#endif
-#define MPI_Imrecv PMPI_Imrecv
-#endif
-
-static const char FUNC_NAME[] = "MPI_Imrecv";
-
-
-int MPI_Imrecv(void *buf, int count, MPI_Datatype type,
-               MPI_Message *message, MPI_Request *request)
-{
-    int rc = MPI_SUCCESS;
-    ompi_communicator_t *comm;
-
-    MEMCHECKER(
-        memchecker_datatype(type);
-        memchecker_message(message);
-        memchecker_call(&opal_memchecker_base_isaddressable, buf, count, type);
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        OMPI_CHECK_DATATYPE_FOR_RECV(rc, type, count);
-        OMPI_CHECK_USER_BUFFER(rc, buf, type, count);
-
-        if (NULL == message || MPI_MESSAGE_NULL == *message) {
-            rc = MPI_ERR_REQUEST;
-            comm = MPI_COMM_NULL;
-        } else {
-            comm = (*message)->comm;
-        }
-
-        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-    } else {
-        comm = (*message)->comm;
-    }
-
-    if (&ompi_message_no_proc.message == *message) {
-        *request = &ompi_request_empty;
-        *message = MPI_MESSAGE_NULL;
-        return MPI_SUCCESS;
-     }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * The message and associated request will be checked by the PML, and
-     * handled approprately. So no need to check here.
-     */
-#endif
-
-    rc = MCA_PML_CALL(imrecv(buf, count, type, message, request));
-    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/imrecv.c.in b/ompi/mpi/c/imrecv.c.in
new file mode 100644
index 00000000000..ea8823790c0
--- /dev/null
+++ b/ompi/mpi/c/imrecv.c.in
@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/memchecker.h"
+#include "ompi/request/request.h"
+#include "ompi/message/message.h"
+
+PROTOTYPE ERROR_CLASS imrecv(BUFFER_OUT buf, COUNT count, DATATYPE type,
+                             MESSAGE_OUT message, REQUEST_INOUT request)
+{
+    int rc = MPI_SUCCESS;
+    ompi_communicator_t *comm;
+
+    MEMCHECKER(
+        memchecker_datatype(type);
+        memchecker_message(message);
+        memchecker_call(&opal_memchecker_base_isaddressable, buf, count, type);
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        OMPI_CHECK_DATATYPE_FOR_RECV(rc, type, count);
+        OMPI_CHECK_USER_BUFFER(rc, buf, type, count);
+
+        if (NULL == message || MPI_MESSAGE_NULL == *message) {
+            rc = MPI_ERR_REQUEST;
+            comm = MPI_COMM_NULL;
+        } else {
+            comm = (*message)->comm;
+        }
+
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    } else {
+        comm = (*message)->comm;
+    }
+
+    if (&ompi_message_no_proc.message == *message) {
+        *request = &ompi_request_empty;
+        *message = MPI_MESSAGE_NULL;
+        return MPI_SUCCESS;
+     }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * The message and associated request will be checked by the PML, and
+     * handled appropriately. So no need to check here.
+     */
+#endif
+
+    rc = MCA_PML_CALL(imrecv(buf, count, type, message, request));
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/ineighbor_allgather.c b/ompi/mpi/c/ineighbor_allgather.c
deleted file mode 100644
index a6a5603ecd4..00000000000
--- a/ompi/mpi/c/ineighbor_allgather.c
+++ /dev/null
@@ -1,129 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012      Oak Rigde National Laboratory. All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2017      IBM Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/mca/topo/topo.h"
-#include "ompi/mca/topo/base/base.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Ineighbor_allgather = PMPI_Ineighbor_allgather
-#endif
-#define MPI_Ineighbor_allgather PMPI_Ineighbor_allgather
-#endif
-
-static const char FUNC_NAME[] = "MPI_Ineighbor_allgather";
-
-
-int MPI_Ineighbor_allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                            void *recvbuf, int recvcount, MPI_Datatype recvtype,
-                            MPI_Comm comm,  MPI_Request *request)
-{
-    int err;
-
-    SPC_RECORD(OMPI_SPC_INEIGHBOR_ALLGATHER, 1);
-
-    MEMCHECKER(
-        ptrdiff_t ext;
-
-        ompi_datatype_type_extent(recvtype, &ext);
-
-        memchecker_datatype(recvtype);
-        memchecker_comm(comm);
-        /* check whether the actual send buffer is defined. */
-        if (MPI_IN_PLACE != sendbuf) {
-            memchecker_datatype(sendtype);
-            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-        }
-        /* check whether the receive buffer is addressable. */
-        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-
-        /* Unrooted operation -- same checks for all ranks on both
-           intracommunicators and intercommunicators */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
-          OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
-        } else if (! OMPI_COMM_IS_TOPO(comm)) {
-          OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TOPOLOGY, FUNC_NAME);
-        } else if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
-          err = MPI_ERR_TYPE;
-        } else if (recvcount < 0) {
-          err = MPI_ERR_COUNT;
-        } else if (MPI_IN_PLACE == sendbuf || MPI_IN_PLACE == recvbuf) {
-          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-        }
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-
-        if( OMPI_COMM_IS_CART(comm) ) {
-            const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
-            if( 0 > cart->ndims ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-        else if( OMPI_COMM_IS_GRAPH(comm) ) {
-            int degree;
-            mca_topo_base_graph_neighbors_count(comm, ompi_comm_rank(comm), &degree);
-            if( 0 > degree ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-        else if( OMPI_COMM_IS_DIST_GRAPH(comm) ) {
-            const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
-            int indegree  = dist_graph->indegree;
-            int outdegree = dist_graph->outdegree;
-            if( indegree <  0 || outdegree <  0 ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_ineighbor_allgather(sendbuf, sendcount, sendtype, recvbuf,
-                                                recvcount, recvtype, comm, request,
-                                                comm->c_coll->coll_ineighbor_allgather_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
-    }
-
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/ineighbor_allgather.c.in b/ompi/mpi/c/ineighbor_allgather.c.in
new file mode 100644
index 00000000000..b434176f31e
--- /dev/null
+++ b/ompi/mpi/c/ineighbor_allgather.c.in
@@ -0,0 +1,121 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oak Rigde National Laboratory. All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/mca/topo/base/base.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS ineighbor_allgather(BUFFER sendbuf, COUNT sendcount, DATATYPE sendtype,
+                                          BUFFER_OUT recvbuf, COUNT recvcount, DATATYPE recvtype,
+                                          COMM comm, REQUEST_INOUT request)
+{
+    int err;
+
+    SPC_RECORD(OMPI_SPC_INEIGHBOR_ALLGATHER, 1);
+
+    MEMCHECKER(
+        ptrdiff_t ext;
+
+        ompi_datatype_type_extent(recvtype, &ext);
+
+        memchecker_datatype(recvtype);
+        memchecker_comm(comm);
+        /* check whether the actual send buffer is defined. */
+        if (MPI_IN_PLACE != sendbuf) {
+            memchecker_datatype(sendtype);
+            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+        }
+        /* check whether the receive buffer is addressable. */
+        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+
+        /* Unrooted operation -- same checks for all ranks on both
+           intracommunicators and intercommunicators */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
+          OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+        } else if (! OMPI_COMM_IS_TOPO(comm)) {
+          OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TOPOLOGY, FUNC_NAME);
+        } else if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
+          err = MPI_ERR_TYPE;
+        } else if (recvcount < 0) {
+          err = MPI_ERR_COUNT;
+        } else if (MPI_IN_PLACE == sendbuf || MPI_IN_PLACE == recvbuf) {
+          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+        }
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+
+        if( OMPI_COMM_IS_CART(comm) ) {
+            const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
+            if( 0 > cart->ndims ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+        else if( OMPI_COMM_IS_GRAPH(comm) ) {
+            int degree;
+            mca_topo_base_graph_neighbors_count(comm, ompi_comm_rank(comm), &degree);
+            if( 0 > degree ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+        else if( OMPI_COMM_IS_DIST_GRAPH(comm) ) {
+            const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
+            int indegree  = dist_graph->indegree;
+            int outdegree = dist_graph->outdegree;
+            if( indegree <  0 || outdegree <  0 ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    err = comm->c_coll->coll_ineighbor_allgather(sendbuf, sendcount, sendtype, recvbuf,
+                                                recvcount, recvtype, comm, request,
+                                                comm->c_coll->coll_ineighbor_allgather_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
+    }
+
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/ineighbor_allgatherv.c b/ompi/mpi/c/ineighbor_allgatherv.c
deleted file mode 100644
index 33f9ff74a56..00000000000
--- a/ompi/mpi/c/ineighbor_allgatherv.c
+++ /dev/null
@@ -1,152 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2010      University of Houston.  All rights reserved.
- * Copyright (c) 2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2017      IBM Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/mca/topo/topo.h"
-#include "ompi/mca/topo/base/base.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Ineighbor_allgatherv = PMPI_Ineighbor_allgatherv
-#endif
-#define MPI_Ineighbor_allgatherv PMPI_Ineighbor_allgatherv
-#endif
-
-static const char FUNC_NAME[] = "MPI_Ineighbor_allgatherv";
-
-
-int MPI_Ineighbor_allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                             void *recvbuf, const int recvcounts[], const int displs[],
-                             MPI_Datatype recvtype, MPI_Comm comm, MPI_Request *request)
-{
-    int i, size, err;
-
-    SPC_RECORD(OMPI_SPC_INEIGHBOR_ALLGATHERV, 1);
-
-    MEMCHECKER(
-        ptrdiff_t ext;
-
-        size = ompi_comm_size(comm);
-        ompi_datatype_type_extent(recvtype, &ext);
-
-        memchecker_datatype(recvtype);
-        memchecker_comm (comm);
-        /* check whether the receive buffer is addressable. */
-        for (i = 0; i < size; i++) {
-            memchecker_call(&opal_memchecker_base_isaddressable,
-                            (char *)(recvbuf)+displs[i]*ext,
-                            recvcounts[i], recvtype);
-        }
-
-        /* check whether the actual send buffer is defined. */
-        if (MPI_IN_PLACE != sendbuf) {
-            memchecker_datatype(sendtype);
-            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-
-        /* Unrooted operation -- same checks for all ranks on both
-           intracommunicators and intercommunicators */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if (! OMPI_COMM_IS_TOPO(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TOPOLOGY,
-                                          FUNC_NAME);
-        } else if (MPI_IN_PLACE == sendbuf || MPI_IN_PLACE == recvbuf) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        } else if (MPI_DATATYPE_NULL == recvtype) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
-        }
-
-        OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-
-      /* We always define the remote group to be the same as the local
-         group in the case of an intracommunicator, so it's safe to
-         get the size of the remote group here for both intra- and
-         intercommunicators */
-
-        size = ompi_comm_remote_size(comm);
-        for (i = 0; i < size; ++i) {
-          if (recvcounts[i] < 0) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-          }
-        }
-
-        if (NULL == displs) {
-          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_BUFFER, FUNC_NAME);
-        }
-
-        if( OMPI_COMM_IS_CART(comm) ) {
-            const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
-            if( 0 > cart->ndims ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-        else if( OMPI_COMM_IS_GRAPH(comm) ) {
-            int degree;
-            mca_topo_base_graph_neighbors_count(comm, ompi_comm_rank(comm), &degree);
-            if( 0 > degree ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-        else if( OMPI_COMM_IS_DIST_GRAPH(comm) ) {
-            const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
-            int indegree  = dist_graph->indegree;
-            int outdegree = dist_graph->outdegree;
-            if( indegree <  0 || outdegree <  0 ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_ineighbor_allgatherv(sendbuf, sendcount, sendtype,
-                                                 recvbuf, (int *) recvcounts, (int *) displs,
-                                                 recvtype, comm, request,
-                                                 comm->c_coll->coll_ineighbor_allgatherv_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/ineighbor_allgatherv.c.in b/ompi/mpi/c/ineighbor_allgatherv.c.in
new file mode 100644
index 00000000000..5f1eb6086b9
--- /dev/null
+++ b/ompi/mpi/c/ineighbor_allgatherv.c.in
@@ -0,0 +1,148 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010      University of Houston.  All rights reserved.
+ * Copyright (c) 2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/mca/topo/base/base.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS ineighbor_allgatherv(BUFFER sendbuf, COUNT sendcount, DATATYPE sendtype,
+                                           BUFFER_OUT recvbuf, COUNT_ARRAY recvcounts, DISP_ARRAY displs,
+                                           DATATYPE recvtype, COMM comm, REQUEST_INOUT request)
+{
+    int i, size, err;
+    ompi_count_array_t recvcounts_desc;
+    ompi_disp_array_t displs_desc;
+
+    SPC_RECORD(OMPI_SPC_INEIGHBOR_ALLGATHERV, 1);
+
+    MEMCHECKER(
+        ptrdiff_t ext;
+
+        size = ompi_comm_size(comm);
+        ompi_datatype_type_extent(recvtype, &ext);
+
+        memchecker_datatype(recvtype);
+        memchecker_comm (comm);
+        /* check whether the receive buffer is addressable. */
+        for (i = 0; i < size; i++) {
+            memchecker_call(&opal_memchecker_base_isaddressable,
+                            (char *)(recvbuf)+displs[i]*ext,
+                            recvcounts[i], recvtype);
+        }
+
+        /* check whether the actual send buffer is defined. */
+        if (MPI_IN_PLACE != sendbuf) {
+            memchecker_datatype(sendtype);
+            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+
+        /* Unrooted operation -- same checks for all ranks on both
+           intracommunicators and intercommunicators */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if (! OMPI_COMM_IS_TOPO(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TOPOLOGY,
+                                          FUNC_NAME);
+        } else if (MPI_IN_PLACE == sendbuf || MPI_IN_PLACE == recvbuf) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        } else if (MPI_DATATYPE_NULL == recvtype) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
+        }
+
+        OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+
+      /* We always define the remote group to be the same as the local
+         group in the case of an intracommunicator, so it's safe to
+         get the size of the remote group here for both intra- and
+         intercommunicators */
+
+        size = ompi_comm_remote_size(comm);
+        for (i = 0; i < size; ++i) {
+          if (recvcounts[i] < 0) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+          }
+        }
+
+        if (NULL == displs) {
+          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_BUFFER, FUNC_NAME);
+        }
+
+        if( OMPI_COMM_IS_CART(comm) ) {
+            const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
+            if( 0 > cart->ndims ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+        else if( OMPI_COMM_IS_GRAPH(comm) ) {
+            int degree;
+            mca_topo_base_graph_neighbors_count(comm, ompi_comm_rank(comm), &degree);
+            if( 0 > degree ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+        else if( OMPI_COMM_IS_DIST_GRAPH(comm) ) {
+            const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
+            int indegree  = dist_graph->indegree;
+            int outdegree = dist_graph->outdegree;
+            if( indegree <  0 || outdegree <  0 ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    OMPI_COUNT_ARRAY_INIT(&recvcounts_desc, recvcounts);
+    OMPI_DISP_ARRAY_INIT(&displs_desc, displs);
+    err = comm->c_coll->coll_ineighbor_allgatherv(sendbuf, sendcount, sendtype,
+                                                 recvbuf, recvcounts_desc, displs_desc,
+                                                 recvtype, comm, request,
+                                                 comm->c_coll->coll_ineighbor_allgatherv_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/ineighbor_alltoall.c b/ompi/mpi/c/ineighbor_alltoall.c
deleted file mode 100644
index 18ef991ecc5..00000000000
--- a/ompi/mpi/c/ineighbor_alltoall.c
+++ /dev/null
@@ -1,130 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012      Oak Ridge National Laboratory. All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2014-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2017      IBM Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/mca/topo/topo.h"
-#include "ompi/mca/topo/base/base.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Ineighbor_alltoall = PMPI_Ineighbor_alltoall
-#endif
-#define MPI_Ineighbor_alltoall PMPI_Ineighbor_alltoall
-#endif
-
-static const char FUNC_NAME[] = "MPI_Ineighbor_alltoall";
-
-
-int MPI_Ineighbor_alltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                           void *recvbuf, int recvcount, MPI_Datatype recvtype,
-                           MPI_Comm comm, MPI_Request *request)
-{
-    size_t sendtype_size, recvtype_size;
-    int err;
-
-    SPC_RECORD(OMPI_SPC_INEIGHBOR_ALLTOALL, 1);
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-        if (MPI_IN_PLACE != sendbuf) {
-            memchecker_datatype(sendtype);
-            memchecker_call(&opal_memchecker_base_isdefined, (void *)sendbuf, sendcount, sendtype);
-        }
-        memchecker_datatype(recvtype);
-        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-
-        /* Unrooted operation -- same checks for all ranks on both
-           intracommunicators and intercommunicators */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if (! OMPI_COMM_IS_TOPO(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TOPOLOGY,
-                                          FUNC_NAME);
-        } else if (MPI_IN_PLACE == sendbuf || MPI_IN_PLACE == recvbuf) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtype, recvcount);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-
-        ompi_datatype_type_size(sendtype, &sendtype_size);
-        ompi_datatype_type_size(recvtype, &recvtype_size);
-        if ((sendtype_size*sendcount) != (recvtype_size*recvcount)) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TRUNCATE, FUNC_NAME);
-        }
-
-        if( OMPI_COMM_IS_CART(comm) ) {
-            const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
-            if( 0 > cart->ndims ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-        else if( OMPI_COMM_IS_GRAPH(comm) ) {
-            int degree;
-            mca_topo_base_graph_neighbors_count(comm, ompi_comm_rank(comm), &degree);
-            if( 0 > degree ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-        else if( OMPI_COMM_IS_DIST_GRAPH(comm) ) {
-            const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
-            int indegree  = dist_graph->indegree;
-            int outdegree = dist_graph->outdegree;
-            if( indegree <  0 || outdegree <  0 ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_ineighbor_alltoall(sendbuf, sendcount, sendtype,
-                                               recvbuf, recvcount, recvtype, comm,
-                                               request, comm->c_coll->coll_ineighbor_alltoall_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/ineighbor_alltoall.c.in b/ompi/mpi/c/ineighbor_alltoall.c.in
new file mode 100644
index 00000000000..c93bb5a248e
--- /dev/null
+++ b/ompi/mpi/c/ineighbor_alltoall.c.in
@@ -0,0 +1,122 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oak Ridge National Laboratory. All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2014-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/mca/topo/base/base.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS ineighbor_alltoall(BUFFER sendbuf, COUNT sendcount, DATATYPE sendtype,
+                                         BUFFER_OUT recvbuf, COUNT recvcount, DATATYPE recvtype,
+                                         COMM comm, REQUEST_INOUT request)
+{
+    size_t sendtype_size, recvtype_size;
+    int err;
+
+    SPC_RECORD(OMPI_SPC_INEIGHBOR_ALLTOALL, 1);
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+        if (MPI_IN_PLACE != sendbuf) {
+            memchecker_datatype(sendtype);
+            memchecker_call(&opal_memchecker_base_isdefined, (void *)sendbuf, sendcount, sendtype);
+        }
+        memchecker_datatype(recvtype);
+        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+
+        /* Unrooted operation -- same checks for all ranks on both
+           intracommunicators and intercommunicators */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if (! OMPI_COMM_IS_TOPO(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TOPOLOGY,
+                                          FUNC_NAME);
+        } else if (MPI_IN_PLACE == sendbuf || MPI_IN_PLACE == recvbuf) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtype, recvcount);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+
+        ompi_datatype_type_size(sendtype, &sendtype_size);
+        ompi_datatype_type_size(recvtype, &recvtype_size);
+        if ((sendtype_size*sendcount) != (recvtype_size*recvcount)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TRUNCATE, FUNC_NAME);
+        }
+
+        if( OMPI_COMM_IS_CART(comm) ) {
+            const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
+            if( 0 > cart->ndims ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+        else if( OMPI_COMM_IS_GRAPH(comm) ) {
+            int degree;
+            mca_topo_base_graph_neighbors_count(comm, ompi_comm_rank(comm), &degree);
+            if( 0 > degree ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+        else if( OMPI_COMM_IS_DIST_GRAPH(comm) ) {
+            const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
+            int indegree  = dist_graph->indegree;
+            int outdegree = dist_graph->outdegree;
+            if( indegree <  0 || outdegree <  0 ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    err = comm->c_coll->coll_ineighbor_alltoall(sendbuf, sendcount, sendtype,
+                                               recvbuf, recvcount, recvtype, comm,
+                                               request, comm->c_coll->coll_ineighbor_alltoall_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/ineighbor_alltoallv.c b/ompi/mpi/c/ineighbor_alltoallv.c
deleted file mode 100644
index 91cecde7d2c..00000000000
--- a/ompi/mpi/c/ineighbor_alltoallv.c
+++ /dev/null
@@ -1,155 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2017 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2014-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2017      IBM Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/mca/topo/topo.h"
-#include "ompi/mca/topo/base/base.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Ineighbor_alltoallv = PMPI_Ineighbor_alltoallv
-#endif
-#define MPI_Ineighbor_alltoallv PMPI_Ineighbor_alltoallv
-#endif
-
-static const char FUNC_NAME[] = "MPI_Ineighbor_alltoallv";
-
-
-int MPI_Ineighbor_alltoallv(const void *sendbuf, const int sendcounts[], const int sdispls[],
-                            MPI_Datatype sendtype, void *recvbuf, const int recvcounts[],
-                            const int rdispls[], MPI_Datatype recvtype, MPI_Comm comm,
-                            MPI_Request *request)
-{
-    int i, err;
-    int indegree, outdegree;
-
-    SPC_RECORD(OMPI_SPC_INEIGHBOR_ALLTOALLV, 1);
-
-    MEMCHECKER(
-        ptrdiff_t recv_ext;
-        ptrdiff_t send_ext;
-
-        memchecker_comm(comm);
-
-        if (MPI_IN_PLACE != sendbuf) {
-            memchecker_datatype(sendtype);
-            ompi_datatype_type_extent(recvtype, &recv_ext);
-        }
-
-        memchecker_datatype(recvtype);
-        ompi_datatype_type_extent(sendtype, &send_ext);
-
-        err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree);
-        if (MPI_SUCCESS == err) {
-            if (MPI_IN_PLACE != sendbuf) {
-                for ( i = 0; i < outdegree; i++ ) {
-                    /* check if send chunks are defined. */
-                    memchecker_call(&opal_memchecker_base_isdefined,
-                                    (char *)(sendbuf)+sdispls[i]*send_ext,
-                                    sendcounts[i], sendtype);
-                }
-            }
-            for ( i = 0; i < indegree; i++ ) {
-                /* check if receive chunks are addressable. */
-                memchecker_call(&opal_memchecker_base_isaddressable,
-                                (char *)(recvbuf)+rdispls[i]*recv_ext,
-                                recvcounts[i], recvtype);
-            }
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-
-        /* Unrooted operation -- same checks for all ranks */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if (! OMPI_COMM_IS_TOPO(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TOPOLOGY,
-                                          FUNC_NAME);
-        }
-
-        err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree);
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        if (((0 < outdegree) && ((NULL == sendcounts) || (NULL == sdispls))) ||
-            ((0 < indegree) && ((NULL == recvcounts) || (NULL == rdispls))) ||
-            MPI_IN_PLACE == sendbuf || MPI_IN_PLACE == recvbuf) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-        for (i = 0; i < outdegree; ++i) {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcounts[i]);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-        for (i = 0; i < indegree; ++i) {
-            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtype, recvcounts[i]);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-
-        if( OMPI_COMM_IS_CART(comm) ) {
-            const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
-            if( 0 > cart->ndims ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-        else if( OMPI_COMM_IS_GRAPH(comm) ) {
-            int degree;
-            mca_topo_base_graph_neighbors_count(comm, ompi_comm_rank(comm), &degree);
-            if( 0 > degree ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-        else if( OMPI_COMM_IS_DIST_GRAPH(comm) ) {
-            const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
-            indegree  = dist_graph->indegree;
-            outdegree = dist_graph->outdegree;
-            if( indegree <  0 || outdegree <  0 ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_ineighbor_alltoallv(sendbuf, sendcounts, sdispls,
-                                                sendtype, recvbuf, recvcounts, rdispls,
-                                                recvtype, comm, request, comm->c_coll->coll_ineighbor_alltoallv_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/ineighbor_alltoallv.c.in b/ompi/mpi/c/ineighbor_alltoallv.c.in
new file mode 100644
index 00000000000..38e7bf82abd
--- /dev/null
+++ b/ompi/mpi/c/ineighbor_alltoallv.c.in
@@ -0,0 +1,153 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2017 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/mca/topo/base/base.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS ineighbor_alltoallv(BUFFER sendbuf, COUNT_ARRAY sendcounts, DISP_ARRAY sdispls,
+                                          DATATYPE sendtype, BUFFER_OUT recvbuf, COUNT_ARRAY recvcounts,
+                                          DISP_ARRAY rdispls, DATATYPE recvtype, COMM comm,
+                                          REQUEST_INOUT request)
+{
+    int i, err;
+    int indegree, outdegree;
+    ompi_count_array_t sendcounts_desc, recvcounts_desc;
+    ompi_disp_array_t sdispls_desc, rdispls_desc;
+
+    SPC_RECORD(OMPI_SPC_INEIGHBOR_ALLTOALLV, 1);
+
+    MEMCHECKER(
+        ptrdiff_t recv_ext;
+        ptrdiff_t send_ext;
+
+        memchecker_comm(comm);
+
+        if (MPI_IN_PLACE != sendbuf) {
+            memchecker_datatype(sendtype);
+            ompi_datatype_type_extent(recvtype, &recv_ext);
+        }
+
+        memchecker_datatype(recvtype);
+        ompi_datatype_type_extent(sendtype, &send_ext);
+
+        err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree);
+        if (MPI_SUCCESS == err) {
+            if (MPI_IN_PLACE != sendbuf) {
+                for ( i = 0; i < outdegree; i++ ) {
+                    /* check if send chunks are defined. */
+                    memchecker_call(&opal_memchecker_base_isdefined,
+                                    (char *)(sendbuf)+sdispls[i]*send_ext,
+                                    sendcounts[i], sendtype);
+                }
+            }
+            for ( i = 0; i < indegree; i++ ) {
+                /* check if receive chunks are addressable. */
+                memchecker_call(&opal_memchecker_base_isaddressable,
+                                (char *)(recvbuf)+rdispls[i]*recv_ext,
+                                recvcounts[i], recvtype);
+            }
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+
+        /* Unrooted operation -- same checks for all ranks */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if (! OMPI_COMM_IS_TOPO(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TOPOLOGY,
+                                          FUNC_NAME);
+        }
+
+        err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree);
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        if (((0 < outdegree) && ((NULL == sendcounts) || (NULL == sdispls))) ||
+            ((0 < indegree) && ((NULL == recvcounts) || (NULL == rdispls))) ||
+            MPI_IN_PLACE == sendbuf || MPI_IN_PLACE == recvbuf) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+        for (i = 0; i < outdegree; ++i) {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcounts[i]);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+        for (i = 0; i < indegree; ++i) {
+            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtype, recvcounts[i]);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+
+        if( OMPI_COMM_IS_CART(comm) ) {
+            const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
+            if( 0 > cart->ndims ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+        else if( OMPI_COMM_IS_GRAPH(comm) ) {
+            int degree;
+            mca_topo_base_graph_neighbors_count(comm, ompi_comm_rank(comm), &degree);
+            if( 0 > degree ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+        else if( OMPI_COMM_IS_DIST_GRAPH(comm) ) {
+            const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
+            indegree  = dist_graph->indegree;
+            outdegree = dist_graph->outdegree;
+            if( indegree <  0 || outdegree <  0 ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    OMPI_COUNT_ARRAY_INIT(&sendcounts_desc, sendcounts);
+    OMPI_COUNT_ARRAY_INIT(&recvcounts_desc, recvcounts);
+    OMPI_DISP_ARRAY_INIT(&sdispls_desc, sdispls);
+    OMPI_DISP_ARRAY_INIT(&rdispls_desc, rdispls);
+    err = comm->c_coll->coll_ineighbor_alltoallv(sendbuf, sendcounts_desc, sdispls_desc,
+                                                sendtype, recvbuf, recvcounts_desc, rdispls_desc,
+                                                recvtype, comm, request, comm->c_coll->coll_ineighbor_alltoallv_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/ineighbor_alltoallw.c b/ompi/mpi/c/ineighbor_alltoallw.c
deleted file mode 100644
index 79c0785d2b4..00000000000
--- a/ompi/mpi/c/ineighbor_alltoallw.c
+++ /dev/null
@@ -1,153 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2017 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2014-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2017      IBM Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/mca/topo/topo.h"
-#include "ompi/mca/topo/base/base.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Ineighbor_alltoallw = PMPI_Ineighbor_alltoallw
-#endif
-#define MPI_Ineighbor_alltoallw PMPI_Ineighbor_alltoallw
-#endif
-
-static const char FUNC_NAME[] = "MPI_Ineighbor_alltoallw";
-
-
-int MPI_Ineighbor_alltoallw(const void *sendbuf, const int sendcounts[], const MPI_Aint sdispls[],
-                            const MPI_Datatype sendtypes[], void *recvbuf, const int recvcounts[],
-                            const MPI_Aint rdispls[], const MPI_Datatype recvtypes[], MPI_Comm comm,
-                            MPI_Request *request)
-{
-    int i, err;
-    int indegree, outdegree;
-
-    SPC_RECORD(OMPI_SPC_INEIGHBOR_ALLTOALLW, 1);
-
-    MEMCHECKER(
-        ptrdiff_t recv_ext;
-        ptrdiff_t send_ext;
-
-        memchecker_comm(comm);
-
-        err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree);
-        if (MPI_SUCCESS == err) {
-            if (MPI_IN_PLACE != sendbuf) {
-                for ( i = 0; i < outdegree; i++ ) {
-                    memchecker_datatype(sendtypes[i]);
-
-                    ompi_datatype_type_extent(sendtypes[i], &send_ext);
-
-                    memchecker_call(&opal_memchecker_base_isdefined,
-                                    (char *)(sendbuf)+sdispls[i]*send_ext,
-                                    sendcounts[i], sendtypes[i]);
-                }
-            }
-            for ( i = 0; i < indegree; i++ ) {
-                memchecker_datatype(recvtypes[i]);
-
-                ompi_datatype_type_extent(recvtypes[i], &recv_ext);
-
-                memchecker_call(&opal_memchecker_base_isaddressable,
-                                (char *)(recvbuf)+sdispls[i]*recv_ext,
-                                recvcounts[i], recvtypes[i]);
-            }
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-
-        /* Unrooted operation -- same checks for all ranks */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if (! OMPI_COMM_IS_TOPO(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TOPOLOGY,
-                                          FUNC_NAME);
-        }
-
-        err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree);
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        if (((0 < outdegree) && ((NULL == sendcounts) || (NULL == sdispls) || (NULL == sendtypes))) ||
-            ((0 < indegree) && ((NULL == recvcounts) || (NULL == rdispls) || (NULL == recvtypes))) ||
-            MPI_IN_PLACE == sendbuf || MPI_IN_PLACE == recvbuf) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-        for (i = 0; i < outdegree; ++i) {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtypes[i], sendcounts[i]);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-        for (i = 0; i < indegree; ++i) {
-            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtypes[i], recvcounts[i]);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-
-        if( OMPI_COMM_IS_CART(comm) ) {
-            const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
-            if( 0 > cart->ndims ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-        else if( OMPI_COMM_IS_GRAPH(comm) ) {
-            int degree;
-            mca_topo_base_graph_neighbors_count(comm, ompi_comm_rank(comm), &degree);
-            if( 0 > degree ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-        else if( OMPI_COMM_IS_DIST_GRAPH(comm) ) {
-            const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
-            indegree  = dist_graph->indegree;
-            outdegree = dist_graph->outdegree;
-            if( indegree <  0 || outdegree <  0 ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_ineighbor_alltoallw(sendbuf, sendcounts, sdispls, sendtypes,
-                                                recvbuf, recvcounts, rdispls, recvtypes, comm, request,
-                                                comm->c_coll->coll_ineighbor_alltoallw_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_datatypes_w(*request, sendtypes, recvtypes);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/ineighbor_alltoallw.c.in b/ompi/mpi/c/ineighbor_alltoallw.c.in
new file mode 100644
index 00000000000..8f9d29591de
--- /dev/null
+++ b/ompi/mpi/c/ineighbor_alltoallw.c.in
@@ -0,0 +1,151 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2022 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2017 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/mca/topo/base/base.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS ineighbor_alltoallw(BUFFER sendbuf, COUNT_ARRAY sendcounts, AINT_ARRAY sdispls,
+                                          DATATYPE_ARRAY sendtypes, BUFFER_OUT recvbuf, COUNT_ARRAY recvcounts,
+                                          AINT_ARRAY rdispls, DATATYPE_ARRAY recvtypes, COMM comm,
+                                          REQUEST_INOUT request)
+{
+    int i, err;
+    int indegree, outdegree;
+    ompi_count_array_t sendcounts_desc, recvcounts_desc;
+    ompi_disp_array_t sdispls_desc, rdispls_desc;
+
+    SPC_RECORD(OMPI_SPC_INEIGHBOR_ALLTOALLW, 1);
+
+    MEMCHECKER(
+        ptrdiff_t recv_ext;
+        ptrdiff_t send_ext;
+
+        memchecker_comm(comm);
+
+        err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree);
+        if (MPI_SUCCESS == err) {
+            if (MPI_IN_PLACE != sendbuf) {
+                for ( i = 0; i < outdegree; i++ ) {
+                    memchecker_datatype(sendtypes[i]);
+
+                    ompi_datatype_type_extent(sendtypes[i], &send_ext);
+
+                    memchecker_call(&opal_memchecker_base_isdefined,
+                                    (char *)(sendbuf)+sdispls[i]*send_ext,
+                                    sendcounts[i], sendtypes[i]);
+                }
+            }
+            for ( i = 0; i < indegree; i++ ) {
+                memchecker_datatype(recvtypes[i]);
+
+                ompi_datatype_type_extent(recvtypes[i], &recv_ext);
+
+                memchecker_call(&opal_memchecker_base_isaddressable,
+                                (char *)(recvbuf)+sdispls[i]*recv_ext,
+                                recvcounts[i], recvtypes[i]);
+            }
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+
+        /* Unrooted operation -- same checks for all ranks */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if (! OMPI_COMM_IS_TOPO(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TOPOLOGY,
+                                          FUNC_NAME);
+        }
+
+        err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree);
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        if (((0 < outdegree) && ((NULL == sendcounts) || (NULL == sdispls) || (NULL == sendtypes))) ||
+            ((0 < indegree) && ((NULL == recvcounts) || (NULL == rdispls) || (NULL == recvtypes))) ||
+            MPI_IN_PLACE == sendbuf || MPI_IN_PLACE == recvbuf) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+        for (i = 0; i < outdegree; ++i) {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtypes[i], sendcounts[i]);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+        for (i = 0; i < indegree; ++i) {
+            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtypes[i], recvcounts[i]);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+
+        if( OMPI_COMM_IS_CART(comm) ) {
+            const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
+            if( 0 > cart->ndims ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+        else if( OMPI_COMM_IS_GRAPH(comm) ) {
+            int degree;
+            mca_topo_base_graph_neighbors_count(comm, ompi_comm_rank(comm), &degree);
+            if( 0 > degree ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+        else if( OMPI_COMM_IS_DIST_GRAPH(comm) ) {
+            const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
+            indegree  = dist_graph->indegree;
+            outdegree = dist_graph->outdegree;
+            if( indegree <  0 || outdegree <  0 ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    OMPI_COUNT_ARRAY_INIT(&sendcounts_desc, sendcounts);
+    OMPI_DISP_ARRAY_INIT(&sdispls_desc, sdispls);
+    OMPI_COUNT_ARRAY_INIT(&recvcounts_desc, recvcounts);
+    OMPI_DISP_ARRAY_INIT(&rdispls_desc, rdispls);
+    err = comm->c_coll->coll_ineighbor_alltoallw(sendbuf, sendcounts_desc, sdispls_desc, sendtypes,
+                                                recvbuf, recvcounts_desc, rdispls_desc, recvtypes, comm, request,
+                                                comm->c_coll->coll_ineighbor_alltoallw_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_datatypes_w(*request, sendtypes, recvtypes, true);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/info_c2f.c b/ompi/mpi/c/info_c2f.c
deleted file mode 100644
index 55b795016b8..00000000000
--- a/ompi/mpi/c/info_c2f.c
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mpi/fortran/base/fint_2_int.h"
-#include "ompi/info/info.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Info_c2f = PMPI_Info_c2f
-#endif
-#define MPI_Info_c2f PMPI_Info_c2f
-#endif
-
-static const char FUNC_NAME[] = "MPI_Info_c2f";
-
-
-MPI_Fint MPI_Info_c2f(MPI_Info info)
-{
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (NULL == info || ompi_info_is_freed(info)) {
-            return OMPI_INT_2_FINT(-1);
-        }
-    }
-
-    return OMPI_INT_2_FINT(info->i_f_to_c_index);
-}
diff --git a/ompi/mpi/c/info_c2f.c.in b/ompi/mpi/c/info_c2f.c.in
new file mode 100644
index 00000000000..827dde289ae
--- /dev/null
+++ b/ompi/mpi/c/info_c2f.c.in
@@ -0,0 +1,42 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+#include "ompi/info/info.h"
+
+PROTOTYPE FINT info_c2f(INFO info)
+{
+    if (MPI_PARAM_CHECK) {
+        if (NULL == info || ompi_info_is_freed(info)) {
+            return OMPI_INT_2_FINT(-1);
+        }
+    }
+
+    return OMPI_INT_2_FINT(info->i_f_to_c_index);
+}
diff --git a/ompi/mpi/c/info_create.c b/ompi/mpi/c/info_create.c
deleted file mode 100644
index a41433e116d..00000000000
--- a/ompi/mpi/c/info_create.c
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Info_create = PMPI_Info_create
-#endif
-#define MPI_Info_create PMPI_Info_create
-#endif
-
-static const char FUNC_NAME[] = "MPI_Info_create";
-
-/**
- * Create a new info object
- *
- * @param info Pointer to the MPI_Info handle
- *
- * @retval MPI_SUCCESS
- * @retval MPI_ERR_INFO
- * @retval MPI_ERR_NO_MEM
- *
- * When an MPI_Info object is not being used, it should be freed using
- * MPI_Info_free
- */
-int MPI_Info_create(MPI_Info *info)
-{
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == info) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
-                                          FUNC_NAME);
-        }
-    }
-
-    /*
-     * Call the object create function. This function not only
-     * allocates the space for MPI_Info, but also calls all the
-     * relevant init functions. Should I check if the fortran
-     * handle is valid
-     */
-    (*info) = OBJ_NEW(ompi_info_t);
-    if (NULL == (*info)) {
-        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_NO_MEM,
-                                      FUNC_NAME);
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/info_create.c.in b/ompi/mpi/c/info_create.c.in
new file mode 100644
index 00000000000..ad60e3f306d
--- /dev/null
+++ b/ompi/mpi/c/info_create.c.in
@@ -0,0 +1,60 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+
+/**
+ * Create a new info object
+ *
+ * @param info Pointer to the MPI_Info handle
+ *
+ * @retval MPI_SUCCESS
+ * @retval MPI_ERR_INFO
+ * @retval MPI_ERR_NO_MEM
+ *
+ * When an MPI_Info object is not being used, it should be freed using
+ * MPI_Info_free
+ */
+PROTOTYPE ERROR_CLASS info_create(INFO_OUT info)
+{
+    if (MPI_PARAM_CHECK) {
+        if (NULL == info) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
+                                          FUNC_NAME);
+        }
+    }
+
+    *info = ompi_info_allocate ();
+    if (NULL == (*info)) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_NO_MEM,
+                                      FUNC_NAME);
+    }
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/info_create_env.c.in b/ompi/mpi/c/info_create_env.c.in
new file mode 100644
index 00000000000..9cdd7fc0120
--- /dev/null
+++ b/ompi/mpi/c/info_create_env.c.in
@@ -0,0 +1,70 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+
+/**
+ *  Returns an info object with the same construction as MPI_INFO_ENV as created 
+ *  during MPI_INIT or MPI_INIT_THREAD when the same arguments are used.
+ *
+ * @param argc number or arguments (Integer)
+ * @param argv Pointer to array of arguments
+ * @param info Pointer to the MPI_Info handle
+ *
+ * @retval MPI_SUCCESS
+ * @retval MPI_ERR_INFO
+ * @retval MPI_ERR_NO_MEM
+ *
+ * When an MPI_Info object is not being used, it should be freed using
+ * MPI_Info_free
+ */
+PROTOTYPE ERROR_CLASS info_create_env(INT argc, STRING_ARRAY argv, INFO_OUT info)
+{
+    int rc;
+    ompi_info_t *the_info;
+
+    if (MPI_PARAM_CHECK) {
+        if (NULL == info) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
+                                          FUNC_NAME);
+        }
+    }
+
+    the_info = ompi_info_allocate ();
+    if (NULL == the_info) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_NO_MEM,
+                                      FUNC_NAME);
+    }
+
+    *info = the_info;
+
+    rc = ompi_mpiinfo_init_env(argc, argv, the_info);
+    OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/info_delete.c b/ompi/mpi/c/info_delete.c
deleted file mode 100644
index e1f53539e0f..00000000000
--- a/ompi/mpi/c/info_delete.c
+++ /dev/null
@@ -1,90 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      Cisco Systems, Inc.  All rights reserved
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-#include <stdlib.h>
-#include <string.h>
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Info_delete = PMPI_Info_delete
-#endif
-#define MPI_Info_delete PMPI_Info_delete
-#endif
-
-static const char FUNC_NAME[] = "MPI_Info_delete";
-
-
-/**
- * Delete a (key,value) pair from "info"
- *
- * @param info MPI_Info handle on which we need to operate
- * @param key The key portion of the (key,value) pair that
- *            needs to be deleted
- *
- * @retval MPI_SUCCESS If the (key,val) pair was deleted
- * @retval MPI_ERR_INFO
- * @retval MPI_ERR_INFO_KEYY
- */
-int MPI_Info_delete(MPI_Info info, const char *key) {
-    int key_length;
-    int err;
-
-    /**
-     * This function merely deletes the (key,val) pair in info
-     */
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == info || MPI_INFO_NULL == info ||
-            ompi_info_is_freed(info)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
-                                          FUNC_NAME);
-        }
-
-        key_length = (key) ? (int)strlen (key) : 0;
-        if ((NULL == key) || (0 == key_length) ||
-            (MPI_MAX_INFO_KEY <= key_length)) {
-          return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO_KEY,
-                                        FUNC_NAME);
-        }
-    }
-
-    err = ompi_info_delete (info, key);
-
-    // Note that ompi_info_delete() (i.e., opal_info_delete()) will
-    // return OPAL_ERR_NOT_FOUND if there was no corresponding key to
-    // delete.  Per MPI-3.1, we need to convert that to
-    // MPI_ERR_INFO_NOKEY.
-    if (OPAL_ERR_NOT_FOUND == err) {
-        err = MPI_ERR_INFO_NOKEY;
-    }
-
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/info_delete.c.in b/ompi/mpi/c/info_delete.c.in
new file mode 100644
index 00000000000..1c0126803b6
--- /dev/null
+++ b/ompi/mpi/c/info_delete.c.in
@@ -0,0 +1,82 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include <stdlib.h>
+#include <string.h>
+
+/**
+ * Delete a (key,value) pair from "info"
+ *
+ * @param info MPI_Info handle on which we need to operate
+ * @param key The key portion of the (key,value) pair that
+ *            needs to be deleted
+ *
+ * @retval MPI_SUCCESS If the (key,val) pair was deleted
+ * @retval MPI_ERR_INFO
+ * @retval MPI_ERR_INFO_KEYY
+ */
+PROTOTYPE ERROR_CLASS info_delete(INFO info, STRING key)
+{
+    int key_length;
+    int err;
+
+    /**
+     * This function merely deletes the (key,val) pair in info
+     */
+    if (MPI_PARAM_CHECK) {
+        if (NULL == info || MPI_INFO_NULL == info ||
+            ompi_info_is_freed(info)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
+                                          FUNC_NAME);
+        }
+
+        key_length = (key) ? (int)strlen (key) : 0;
+        if ((NULL == key) || (0 == key_length) ||
+            (MPI_MAX_INFO_KEY <= key_length)) {
+          return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO_KEY,
+                                        FUNC_NAME);
+        }
+    }
+
+    err = ompi_info_delete (info, key);
+
+    // Note that ompi_info_delete() (i.e., opal_info_delete()) will
+    // return OPAL_ERR_NOT_FOUND if there was no corresponding key to
+    // delete.  Per MPI-3.1, we need to convert that to
+    // MPI_ERR_INFO_NOKEY.
+    if (OPAL_ERR_NOT_FOUND == err) {
+        err = MPI_ERR_INFO_NOKEY;
+    }
+
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/info_dup.c b/ompi/mpi/c/info_dup.c
deleted file mode 100644
index 7c738d7b01c..00000000000
--- a/ompi/mpi/c/info_dup.c
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Info_dup = PMPI_Info_dup
-#endif
-#define MPI_Info_dup PMPI_Info_dup
-#endif
-
-static const char FUNC_NAME[] = "MPI_Info_dup";
-
-
-/**
- *   MPI_Info_dup - Duplicate an 'MPI_Info' object
- *
- *   @param info source info object (handle)
- *   @param newinfo pointer to the new info object (handle)
- *
- *   @retval MPI_SUCCESS
- *   @retval MPI_ERR_INFO
- *   @retval MPI_ERR_NO_MEM
- *
- *   Not only will the (key, value) pairs be duplicated, the order of keys
- *   will be the same in 'newinfo' as it is in 'info'.
- *   When an info object is no longer being used, it should be freed with
- *   'MPI_Info_free'.
- */
-int MPI_Info_dup(MPI_Info info, MPI_Info *newinfo) {
-    int err;
-
-    /**
-     * Here we need to do 2 things
-     * 1. Create a newinfo object using MPI_Info_create
-     * 2. Fetch all the values from info and copy them to
-     *    newinfo using MPI_Info_set
-     * The new implementation facilitates traversal in many ways.
-     * I have chosen to get the number of elements on the list
-     * and copy them to newinfo one by one
-     */
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == info || MPI_INFO_NULL == info || NULL == newinfo ||
-            ompi_info_is_freed(info)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
-                                          FUNC_NAME);
-        }
-    }
-
-    *newinfo = OBJ_NEW(ompi_info_t);
-    if (NULL == *newinfo) {
-        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_NO_MEM,
-                                      FUNC_NAME);
-    }
-
-    /*
-     * Now to actually duplicate all the values
-     */
-    err = ompi_info_dup (info, newinfo);
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/info_dup.c.in b/ompi/mpi/c/info_dup.c.in
new file mode 100644
index 00000000000..b18c11a36ec
--- /dev/null
+++ b/ompi/mpi/c/info_dup.c.in
@@ -0,0 +1,80 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+
+/**
+ *   MPI_Info_dup - Duplicate an 'MPI_Info' object
+ *
+ *   @param info source info object (handle)
+ *   @param newinfo pointer to the new info object (handle)
+ *
+ *   @retval MPI_SUCCESS
+ *   @retval MPI_ERR_INFO
+ *   @retval MPI_ERR_NO_MEM
+ *
+ *   Not only will the (key, value) pairs be duplicated, the order of keys
+ *   will be the same in 'newinfo' as it is in 'info'.
+ *   When an info object is no longer being used, it should be freed with
+ *   'MPI_Info_free'.
+ */
+PROTOTYPE ERROR_CLASS info_dup(INFO info, INFO_OUT newinfo)
+{
+    int err;
+
+    /**
+     * Here we need to do 2 things
+     * 1. Create a newinfo object using MPI_Info_create
+     * 2. Fetch all the values from info and copy them to
+     *    newinfo using MPI_Info_set
+     * The new implementation facilitates traversal in many ways.
+     * I have chosen to get the number of elements on the list
+     * and copy them to newinfo one by one
+     */
+
+    if (MPI_PARAM_CHECK) {
+        if (NULL == info || MPI_INFO_NULL == info || NULL == newinfo ||
+            ompi_info_is_freed(info)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
+                                          FUNC_NAME);
+        }
+    }
+
+    *newinfo = ompi_info_allocate();
+    if (NULL == *newinfo) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_NO_MEM,
+                                      FUNC_NAME);
+    }
+
+    /*
+     * Now to actually duplicate all the values
+     */
+    err = ompi_info_dup (info, newinfo);
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/info_f2c.c b/ompi/mpi/c/info_f2c.c
deleted file mode 100644
index 8c6383a20e6..00000000000
--- a/ompi/mpi/c/info_f2c.c
+++ /dev/null
@@ -1,68 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2007 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mpi/fortran/base/fint_2_int.h"
-#include "ompi/info/info.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Info_f2c = PMPI_Info_f2c
-#endif
-#define MPI_Info_f2c PMPI_Info_f2c
-#endif
-
-static const char FUNC_NAME[] = "MPI_Info_f2c";
-
-
-/**
- * Converts the MPI_Fint info into a valid C MPI_Info handle
- *
- * @param info Integer handle to an MPI_INFO object
- * @retval C handle corresponding to MPI_INFO object
- */
-MPI_Info MPI_Info_f2c(MPI_Fint info)
-{
-    int info_index = OMPI_FINT_2_INT(info);
-
-    /* check the arguments */
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-    }
-
-    /* Per MPI-2:4.12.4, do not invoke an error handler if we get an
-       invalid fortran handle.  If we get an invalid fortran handle,
-       return an invalid C handle. */
-
-    if (info_index < 0 ||
-        info_index >=
-        opal_pointer_array_get_size(&ompi_info_f_to_c_table)) {
-        return NULL;
-    }
-
-    return (MPI_Info)opal_pointer_array_get_item(&ompi_info_f_to_c_table, info_index);
-}
diff --git a/ompi/mpi/c/info_f2c.c.in b/ompi/mpi/c/info_f2c.c.in
new file mode 100644
index 00000000000..44f1c9e9c0d
--- /dev/null
+++ b/ompi/mpi/c/info_f2c.c.in
@@ -0,0 +1,78 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+#include "ompi/info/info.h"
+
+/**
+ * Converts the MPI_Fint info into a valid C MPI_Info handle
+ *
+ * @param info Integer handle to an MPI_INFO object
+ * @retval C handle corresponding to MPI_INFO object
+ */
+PROTOTYPE INFO info_f2c(FINT info)
+{
+    int info_index = OMPI_FINT_2_INT(info);
+
+    /* Per MPI-2:4.12.4, do not invoke an error handler if we get an
+       invalid fortran handle.  If we get an invalid fortran handle,
+       return an invalid C handle. */
+    /*
+     * Deal with special pre-defined cases for MPI 4.0
+     */
+
+    if (info_index == 0) {
+        return MPI_INFO_NULL;
+    }
+
+    if (info_index == 1) {
+        return MPI_INFO_ENV;
+    }
+
+    /* 
+     * if the application has not created an info object yet
+     * then the size of the ompi_info_f_to_c_table is zero
+     * so this check can be done even if an info object has not
+     * previously been created.
+     */
+
+    if (info_index < 0 ||
+        info_index >=
+        opal_pointer_array_get_size(&ompi_info_f_to_c_table)) {
+        return NULL;
+    }
+
+    /*
+     * if we get here, then the info support infrastructure has been initialized
+     * either via a prior call to MPI_Info_create or one of the MPI initialization
+     * methods.
+     */
+    return (MPI_Info)opal_pointer_array_get_item(&ompi_info_f_to_c_table, info_index);
+}
diff --git a/ompi/mpi/c/info_free.c b/ompi/mpi/c/info_free.c
deleted file mode 100644
index 32220c84c04..00000000000
--- a/ompi/mpi/c/info_free.c
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Info_free = PMPI_Info_free
-#endif
-#define MPI_Info_free PMPI_Info_free
-#endif
-
-static const char FUNC_NAME[] = "MPI_Info_free";
-
-
-/**
- *   MPI_Info_free - Free an 'MPI_Info' object.
- *
- *   @param info pointer to info object to be freed (handle)
- *
- *   @retval MPI_SUCCESS
- *   @retval MPI_ERR_INFO
- *
- *   Upon successful completion, 'info' will be set to 'MPI_INFO_NULL'.
- */
-int MPI_Info_free(MPI_Info *info)
-{
-    int err;
-
-    /*
-     * Free all the alloced items from MPI_Info info.
-     * Make sure the items are freed in an orderly
-     * fashion so that there are no dangling pointers.
-     */
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == info || MPI_INFO_NULL == *info ||
-            ompi_info_is_freed(*info)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
-                                          FUNC_NAME);
-        }
-    }
-
-    err = ompi_info_free(info);
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/info_free.c.in b/ompi/mpi/c/info_free.c.in
new file mode 100644
index 00000000000..96e45184187
--- /dev/null
+++ b/ompi/mpi/c/info_free.c.in
@@ -0,0 +1,62 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+
+/**
+ *   MPI_Info_free - Free an 'MPI_Info' object.
+ *
+ *   @param info pointer to info object to be freed (handle)
+ *
+ *   @retval MPI_SUCCESS
+ *   @retval MPI_ERR_INFO
+ *
+ *   Upon successful completion, 'info' will be set to 'MPI_INFO_NULL'.
+ */
+PROTOTYPE ERROR_CLASS info_free(INFO_OUT info)
+{
+    int err;
+
+    /*
+     * Free all the alloced items from MPI_Info info.
+     * Make sure the items are freed in an orderly
+     * fashion so that there are no dangling pointers.
+     */
+    if (MPI_PARAM_CHECK) {
+        if (NULL == info || MPI_INFO_NULL == *info ||
+            ompi_info_is_freed(*info)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
+                                          FUNC_NAME);
+        }
+    }
+
+    err = ompi_info_free(info);
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/info_get.c b/ompi/mpi/c/info_get.c
deleted file mode 100644
index b8e8566c767..00000000000
--- a/ompi/mpi/c/info_get.c
+++ /dev/null
@@ -1,104 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-#include <stdlib.h>
-#include <string.h>
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Info_get = PMPI_Info_get
-#endif
-#define MPI_Info_get PMPI_Info_get
-#endif
-
-static const char FUNC_NAME[] = "MPI_Info_get";
-
-/**
- *   MPI_Info_get - Get a (key, value) pair from an 'MPI_Info' object
- *
- *   @param info info object (handle)
- *   @param key null-terminated character string of the index key
- *   @param valuelen maximum length of 'value' (integer)
- *   @param value null-terminated character string of the value
- *   @param flag true (1) if 'key' defined on 'info', false (0) if not
- *               (logical)
- *
- *   @retval MPI_SUCCESS
- *   @retval MPI_ERR_ARG
- *   @retval MPI_ERR_INFO
- *   @retval MPI_ERR_INFO_KEY
- *   @retval MPI_ERR_INFO_VALUE
- *
- *   In C and C++, 'valuelen' should be one less than the allocated space
- *   to allow for for the null terminator.
- */
-int MPI_Info_get(MPI_Info info, const char *key, int valuelen,
-                 char *value, int *flag)
-{
-    int err;
-    int key_length;
-
-    /*
-     * Simple function. All we need to do is search for the value
-     * having the "key" associated with it and then populate the
-     * necessary structures.
-     */
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == info || MPI_INFO_NULL == info ||
-            ompi_info_is_freed(info)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
-                                          FUNC_NAME);
-        }
-        if (0 > valuelen){
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-
-        key_length = (key) ? (int)strlen (key) : 0;
-        if ((NULL == key) || (0 == key_length) ||
-            (MPI_MAX_INFO_KEY <= key_length)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO_KEY,
-                                          FUNC_NAME);
-        }
-        if (NULL == value) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO_VALUE,
-                                          FUNC_NAME);
-        }
-        if (NULL == flag) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-    }
-
-    err = ompi_info_get(info, key, valuelen, value, flag);
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/info_get.c.in b/ompi/mpi/c/info_get.c.in
new file mode 100644
index 00000000000..4784eafcf26
--- /dev/null
+++ b/ompi/mpi/c/info_get.c.in
@@ -0,0 +1,103 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include "opal/util/string_copy.h"
+#include <stdlib.h>
+#include <string.h>
+
+/**
+ *   MPI_Info_get - Get a (key, value) pair from an 'MPI_Info' object
+ *
+ *   @param info info object (handle)
+ *   @param key null-terminated character string of the index key
+ *   @param valuelen maximum length of 'value' (integer)
+ *   @param value null-terminated character string of the value
+ *   @param flag true (1) if 'key' defined on 'info', false (0) if not
+ *               (logical)
+ *
+ *   @retval MPI_SUCCESS
+ *   @retval MPI_ERR_ARG
+ *   @retval MPI_ERR_INFO
+ *   @retval MPI_ERR_INFO_KEY
+ *   @retval MPI_ERR_INFO_VALUE
+ *
+ *   In C and C++, 'valuelen' should be one less than the allocated space
+ *   to allow for for the null terminator.
+ */
+PROTOTYPE ERROR_CLASS info_get(INFO info, STRING key, INT valuelen,
+                               STRING_OUT value, INT_OUT flag)
+{
+    int err;
+    int key_length;
+    opal_cstring_t *info_str;
+
+    /*
+     * Simple function. All we need to do is search for the value
+     * having the "key" associated with it and then populate the
+     * necessary structures.
+     */
+    if (MPI_PARAM_CHECK) {
+        if (NULL == info || MPI_INFO_NULL == info ||
+            ompi_info_is_freed(info)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
+                                          FUNC_NAME);
+        }
+        if (0 > valuelen){
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+
+        key_length = (key) ? (int)strlen (key) : 0;
+        if ((NULL == key) || (0 == key_length) ||
+            (MPI_MAX_INFO_KEY <= key_length)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO_KEY,
+                                          FUNC_NAME);
+        }
+        if (NULL == value) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO_VALUE,
+                                          FUNC_NAME);
+        }
+        if (NULL == flag) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+    err = ompi_info_get(info, key, &info_str, flag);
+    if (*flag) {
+        opal_string_copy(value, info_str->string, valuelen+1);
+        OBJ_RELEASE(info_str);
+    }
+
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/info_get_nkeys.c b/ompi/mpi/c/info_get_nkeys.c
deleted file mode 100644
index fe79178af95..00000000000
--- a/ompi/mpi/c/info_get_nkeys.c
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Info_get_nkeys = PMPI_Info_get_nkeys
-#endif
-#define MPI_Info_get_nkeys PMPI_Info_get_nkeys
-#endif
-
-static const char FUNC_NAME[] = "MPI_Info_get_nkeys";
-
-
-/**
- * MPI_Info_get_nkeys - Returns the number of keys defined on an
- * 'MPI_Info' object
- *
- * @param info info object (handle)
- * @param nkeys number of keys defined on 'info' (integer)
- *
- * @retval MPI_SUCCESS
- * @retval MPI_ERR_ARG
- * @retval MPI_ERR_INFO
- *
- * This function returns the number of elements in the list
- * containing the key-value pairs
- */
-int MPI_Info_get_nkeys(MPI_Info info, int *nkeys)
-{
-    int err;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == info || MPI_INFO_NULL == info ||
-            ompi_info_is_freed(info)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
-                                          FUNC_NAME);
-        }
-        if (NULL == nkeys) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-    }
-
-    err = ompi_info_get_nkeys(info, nkeys);
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/info_get_nkeys.c.in b/ompi/mpi/c/info_get_nkeys.c.in
new file mode 100644
index 00000000000..95e2bd9a540
--- /dev/null
+++ b/ompi/mpi/c/info_get_nkeys.c.in
@@ -0,0 +1,65 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+
+
+/**
+ * MPI_Info_get_nkeys - Returns the number of keys defined on an
+ * 'MPI_Info' object
+ *
+ * @param info info object (handle)
+ * @param nkeys number of keys defined on 'info' (integer)
+ *
+ * @retval MPI_SUCCESS
+ * @retval MPI_ERR_ARG
+ * @retval MPI_ERR_INFO
+ *
+ * This function returns the number of elements in the list
+ * containing the key-value pairs
+ */
+PROTOTYPE ERROR_CLASS info_get_nkeys(INFO info, INT_OUT nkeys)
+{
+    int err;
+
+    if (MPI_PARAM_CHECK) {
+        if (NULL == info || MPI_INFO_NULL == info ||
+            ompi_info_is_freed(info)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
+                                          FUNC_NAME);
+        }
+        if (NULL == nkeys) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+    err = ompi_info_get_nkeys(info, nkeys);
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/info_get_nthkey.c b/ompi/mpi/c/info_get_nthkey.c
deleted file mode 100644
index 412664f0542..00000000000
--- a/ompi/mpi/c/info_get_nthkey.c
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-#include <string.h>
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Info_get_nthkey = PMPI_Info_get_nthkey
-#endif
-#define MPI_Info_get_nthkey PMPI_Info_get_nthkey
-#endif
-
-static const char FUNC_NAME[] = "MPI_Info_get_nthkey";
-
-
-/**
- *   MPI_Info_get_nthkey - Get a key indexed by integer from an 'MPI_Info' obje
- *
- *   @param info info object (handle)
- *   @param n index of key to retrieve (integer)
- *   @param key character string of at least 'MPI_MAX_INFO_KEY' characters
- *
- *   @retval MPI_SUCCESS
- *   @retval MPI_ERR_ARG
- *   @retval MPI_ERR_INFO
- *   @retval MPI_ERR_INFO_KEY
- */
-int MPI_Info_get_nthkey(MPI_Info info, int n, char *key)
-{
-    int nkeys;
-    int err;
-
-    /*
-     * 1. Check if info is a valid handle
-     * 2. Check if there are at least (n+1) elements
-     * 3. If so, give the nth defined key
-     */
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == info || MPI_INFO_NULL == info ||
-            ompi_info_is_freed(info)) {
-            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_INFO,
-                                           FUNC_NAME);
-        }
-        if (0 > n) {
-            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_ARG,
-                                           FUNC_NAME);
-        }
-        if (NULL == key) {
-            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_INFO_KEY,
-                                           FUNC_NAME);
-        }
-    }
-
-    /* Keys are indexed on 0, which makes the "n" parameter offset by
-       1 from the value returned by get_nkeys().  So be sure to
-       compare appropriately. */
-
-    err = ompi_info_get_nkeys(info, &nkeys);
-    OMPI_ERRHANDLER_NOHANDLE_CHECK(err, err, FUNC_NAME);
-    if (n > (nkeys - 1)) {
-        return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_INFO_KEY,
-                                       FUNC_NAME);
-    }
-
-    /* Everything seems alright. Call the back end key copy */
-
-    err = ompi_info_get_nthkey (info, n, key);
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/info_get_nthkey.c.in b/ompi/mpi/c/info_get_nthkey.c.in
new file mode 100644
index 00000000000..1795a2ac905
--- /dev/null
+++ b/ompi/mpi/c/info_get_nthkey.c.in
@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include "opal/util/string_copy.h"
+#include <string.h>
+
+/**
+ *   MPI_Info_get_nthkey - Get a key indexed by integer from an 'MPI_Info' obje
+ *
+ *   @param info info object (handle)
+ *   @param n index of key to retrieve (integer)
+ *   @param key character string of at least 'MPI_MAX_INFO_KEY' characters
+ *
+ *   @retval MPI_SUCCESS
+ *   @retval MPI_ERR_ARG
+ *   @retval MPI_ERR_INFO
+ *   @retval MPI_ERR_INFO_KEY
+ */
+PROTOTYPE ERROR_CLASS info_get_nthkey(INFO info, INT n, STRING_OUT key)
+{
+    int nkeys;
+    int err;
+
+    /*
+     * 1. Check if info is a valid handle 
+     * 2. Check if there are at least (n+1) elements
+     * 3. If so, give the nth defined key
+     */
+    if (NULL == info || MPI_INFO_NULL == info) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO, FUNC_NAME);
+    }
+
+    if (MPI_PARAM_CHECK) {
+        if (ompi_info_is_freed(info)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO, FUNC_NAME);
+        }
+        if (0 > n) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+        }
+        if (NULL == key) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO_KEY, FUNC_NAME);
+        }
+    }
+
+    /* Keys are indexed on 0, which makes the "n" parameter offset by
+       1 from the value returned by get_nkeys().  So be sure to
+       compare appropriately. */
+
+    err = ompi_info_get_nkeys(info, &nkeys);
+    OMPI_ERRHANDLER_NOHANDLE_CHECK(err, err, FUNC_NAME);
+    if (n > (nkeys - 1)) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE (MPI_ERR_INFO_KEY, FUNC_NAME);
+    }
+
+    /* Everything seems alright. Call the back end key copy */
+
+    opal_cstring_t *key_str = NULL;
+    err = ompi_info_get_nthkey (info, n, &key_str);
+    if (NULL != key_str) {
+        opal_string_copy(key, key_str->string, MPI_MAX_INFO_KEY);
+        OBJ_RELEASE(key_str);
+    }
+
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/info_get_string.c.in b/ompi/mpi/c/info_get_string.c.in
new file mode 100644
index 00000000000..a9b17c3f99c
--- /dev/null
+++ b/ompi/mpi/c/info_get_string.c.in
@@ -0,0 +1,110 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
+ * Copyright (c) 2021-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include "opal/util/string_copy.h"
+#include <stdlib.h>
+#include <string.h>
+
+
+/**
+ *   MPI_Info_get_string - Get a (key, value) pair from an 'MPI_Info' object
+ *
+ *   @param info info object (handle)
+ *   @param key null-terminated character string of the index key
+ *   @param buflen maximum length of 'value' (integer)
+ *   @param value null-terminated character string of the value
+ *   @param flag true (1) if 'key' defined on 'info', false (0) if not
+ *               (logical)
+ *
+ *   @retval MPI_SUCCESS
+ *   @retval MPI_ERR_ARG
+ *   @retval MPI_ERR_INFO
+ *   @retval MPI_ERR_INFO_KEY
+ *   @retval MPI_ERR_INFO_VALUE
+ *
+ */
+PROTOTYPE ERROR_CLASS Info_get_string(INFO info, STRING key, INT_OUT buflen,
+                        STRING_OUT value, INT_OUT flag)
+{
+    int err;
+    int key_length;
+    opal_cstring_t *info_str;
+
+    /*
+     * Simple function. All we need to do is search for the value
+     * having the "key" associated with it and then populate the
+     * necessary structures.
+     */
+    if (MPI_PARAM_CHECK) {
+        if (NULL == info || MPI_INFO_NULL == info ||
+            ompi_info_is_freed(info)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
+                                          FUNC_NAME);
+        }
+
+        key_length = (key) ? (int)strlen (key) : 0;
+        if ((NULL == key) || (0 == key_length) ||
+            (MPI_MAX_INFO_KEY <= key_length)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO_KEY,
+                                          FUNC_NAME);
+        }
+        if (NULL == buflen) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+        if ((NULL == value) && *buflen) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO_VALUE,
+                                          FUNC_NAME);
+        }
+        if (NULL == flag) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+    if (0 == *buflen) {
+        err = ompi_info_get_valuelen(info, key, buflen, flag);
+        if (1 == *flag) {
+            *buflen += 1; /* add on for the \0, see MPI 4.0 Standard */
+        }
+    } else {
+        err = ompi_info_get(info, key, &info_str, flag);
+        if (*flag) {
+            opal_string_copy(value, info_str->string, *buflen);
+            *buflen = info_str->length + 1; /* add on for the \0, see MPI 4.0 Standard */
+            OBJ_RELEASE(info_str);
+        }
+    }
+
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/info_get_valuelen.c b/ompi/mpi/c/info_get_valuelen.c
deleted file mode 100644
index 575de6d407d..00000000000
--- a/ompi/mpi/c/info_get_valuelen.c
+++ /dev/null
@@ -1,93 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-#include <stdlib.h>
-#include <string.h>
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Info_get_valuelen = PMPI_Info_get_valuelen
-#endif
-#define MPI_Info_get_valuelen PMPI_Info_get_valuelen
-#endif
-
-static const char FUNC_NAME[] = "MPI_Info_get_valuelen";
-
-
-/**
- *   MPI_Info_get_valuelen - Get the length of a value for a given key in an 'M
- *
- *   @param info - info object (handle)
- *   @param key - null-terminated character string of the index key
- *   @param valuelen - length of the value associated with 'key' (integer)
- *   @param flag - true (1) if 'key' defined on 'info', false (0) if not
- *   (logical)
- *
- *   @retval MPI_SUCCESS
- *   @retval MPI_ERR_ARG
- *   @retval MPI_ERR_INFO
- *   @retval MPI_ERR_INFO_KEY
- *
- *   The length returned in C and C++ does not include the end-of-string
- *   character.  If the 'key' is not found on 'info', 'valuelen' is left
- *   alone.
- */
-int MPI_Info_get_valuelen(MPI_Info info, const char *key, int *valuelen,
-                          int *flag)
-{
-    int key_length;
-    int err;
-
-    /*
-     * Simple function. All we need to do is search for the value
-     * having the "key" associated with it and return the length
-     */
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == info || MPI_INFO_NULL == info ||
-            ompi_info_is_freed(info)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
-                                          FUNC_NAME);
-        }
-        key_length = (key) ? (int)strlen (key) : 0;
-        if ((NULL == key) || (0 == key_length) ||
-            (MPI_MAX_INFO_KEY <= key_length)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO_KEY,
-                                          FUNC_NAME);
-        }
-        if (NULL == flag || NULL == valuelen) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-    }
-
-    err = ompi_info_get_valuelen (info, key, valuelen, flag);
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/info_get_valuelen.c.in b/ompi/mpi/c/info_get_valuelen.c.in
new file mode 100644
index 00000000000..0195d602e28
--- /dev/null
+++ b/ompi/mpi/c/info_get_valuelen.c.in
@@ -0,0 +1,84 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include <stdlib.h>
+#include <string.h>
+
+/**
+ *   MPI_Info_get_valuelen - Get the length of a value for a given key in an 'M
+ *
+ *   @param info - info object (handle)
+ *   @param key - null-terminated character string of the index key
+ *   @param valuelen - length of the value associated with 'key' (integer)
+ *   @param flag - true (1) if 'key' defined on 'info', false (0) if not
+ *   (logical)
+ *
+ *   @retval MPI_SUCCESS
+ *   @retval MPI_ERR_ARG
+ *   @retval MPI_ERR_INFO
+ *   @retval MPI_ERR_INFO_KEY
+ *
+ *   The length returned in C and C++ does not include the end-of-string
+ *   character.  If the 'key' is not found on 'info', 'valuelen' is left
+ *   alone.
+ */
+PROTOTYPE ERROR_CLASS info_get_valuelen(INFO info, STRING key, INT_OUT valuelen,
+                                        INT_OUT flag)
+{
+    int key_length;
+    int err;
+
+    /*
+     * Simple function. All we need to do is search for the value
+     * having the "key" associated with it and return the length
+     */
+    if (MPI_PARAM_CHECK) {
+        if (NULL == info || MPI_INFO_NULL == info ||
+            ompi_info_is_freed(info)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
+                                          FUNC_NAME);
+        }
+        key_length = (key) ? (int)strlen (key) : 0;
+        if ((NULL == key) || (0 == key_length) ||
+            (MPI_MAX_INFO_KEY <= key_length)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO_KEY,
+                                          FUNC_NAME);
+        }
+        if (NULL == flag || NULL == valuelen) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+    err = ompi_info_get_valuelen (info, key, valuelen, flag);
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/info_set.c b/ompi/mpi/c/info_set.c
deleted file mode 100644
index bb3c5c6198c..00000000000
--- a/ompi/mpi/c/info_set.c
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2012-2013 Inria.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2018 IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-#include "opal/util/show_help.h"
-#include <stdlib.h>
-#include <string.h>
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Info_set = PMPI_Info_set
-#endif
-#define MPI_Info_set PMPI_Info_set
-#endif
-
-static const char FUNC_NAME[] = "MPI_Info_set";
-
-
-/**
- *   MPI_Info_set - Set a (key, value) pair in an 'MPI_Info' object
- *
- *   @param key null-terminated character string of the index key
- *   @param value null-terminated character string of the value
- *   @param info info object (handle)
- *
- *   @retval MPI_SUCCESS
- *   @retval MPI_ERR_ARG
- *   @retval MPI_ERR_INFO_KEY
- *   @retval MPI_ERR_INFO_VAL
- *   @retval MPI_ERR_INFO_NOKEY
- *   @retval MPI_ERR_NO_MEM
- *
- *   MPI_Info_set adds the (key,value) pair to info, and overrides
- *   the value if for the same key a previsou value was set. key and
- *   value must be NULL terminated strings in C. In Fortan, leading
- *   and trailing spaces in key and value are stripped. If either
- *   key or value is greater than the allowed maxima, MPI_ERR_INFO_KEY
- *   and MPI_ERR_INFO_VALUE are raised
- */
-int MPI_Info_set(MPI_Info info, const char *key, const char *value)
-{
-    int err;
-    int key_length;
-    int value_length;
-
-    /*
-     * Error conditions are
-     *   - info is NULL
-     *   - No storage space available for the new value
-     *   - Key length exceeded MPI_MAX_KEY_VAL
-     *   - value length exceeded MPI_MAX_KEY_VAL
-     */
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == info || MPI_INFO_NULL == info ||
-            ompi_info_is_freed(info)) {
-            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_INFO,
-                                           FUNC_NAME);
-        }
-
-        key_length = (key) ? (int)strlen (key) : 0;
-        if ((NULL == key) || (0 == key_length) ||
-            (MPI_MAX_INFO_KEY <= key_length)) {
-            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_INFO_KEY,
-                                           FUNC_NAME);
-        }
-
-        value_length = (value) ? (int)strlen (value) : 0;
-        if ((NULL == value) || (0 == value_length) ||
-            (MPI_MAX_INFO_VAL <= value_length)) {
-            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_INFO_VALUE,
-                                           FUNC_NAME);
-        }
-    }
-
-// An extra warning condition is a key that uses our reserved prefix "__IN_".
-// That one is used internally to deal with the dynamic nature the key/val
-// pairs where we have callbacks that modify the val, and the MPI standard
-// wants the get_info call to give back the original setting rather than
-// the callback-modified setting. So if a user directly used a key __IN_foo
-// it would confuse our accounting slightly.
-    if (0 == strncmp(key, OPAL_INFO_SAVE_PREFIX, strlen(OPAL_INFO_SAVE_PREFIX))) {
-        opal_show_help("help-mpi-api.txt", "info-set-with-reserved-prefix", true,
-            key, OPAL_INFO_SAVE_PREFIX);
-    }
-
-    /*
-     * If all is right with the arguments, then call the back-end
-     * allocator.
-     */
-
-    err = ompi_info_set (info, key, value);
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/info_set.c.in b/ompi/mpi/c/info_set.c.in
new file mode 100644
index 00000000000..c58121ab9c9
--- /dev/null
+++ b/ompi/mpi/c/info_set.c.in
@@ -0,0 +1,101 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2012-2013 Inria.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018 IBM Corporation. All rights reserved.
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include "opal/util/show_help.h"
+#include <stdlib.h>
+#include <string.h>
+
+/**
+ *   MPI_Info_set - Set a (key, value) pair in an 'MPI_Info' object
+ *
+ *   @param key null-terminated character string of the index key
+ *   @param value null-terminated character string of the value
+ *   @param info info object (handle)
+ *
+ *   @retval MPI_SUCCESS
+ *   @retval MPI_ERR_ARG
+ *   @retval MPI_ERR_INFO_KEY
+ *   @retval MPI_ERR_INFO_VAL
+ *   @retval MPI_ERR_INFO_NOKEY
+ *   @retval MPI_ERR_NO_MEM
+ *
+ *   MPI_Info_set adds the (key,value) pair to info, and overrides
+ *   the value if for the same key a previous value was set. key and
+ *   value must be NULL terminated strings in C. In Fortran, leading
+ *   and trailing spaces in key and value are stripped. If either
+ *   key or value is greater than the allowed maxima, MPI_ERR_INFO_KEY
+ *   and MPI_ERR_INFO_VALUE are raised
+ */
+PROTOTYPE ERROR_CLASS info_set(INFO info, STRING key, STRING value)
+{
+    int err;
+    int key_length;
+    int value_length;
+
+    /*
+     * Error conditions are
+     *   - info is NULL
+     *   - No storage space available for the new value
+     *   - Key length exceeded MPI_MAX_KEY_VAL
+     *   - value length exceeded MPI_MAX_KEY_VAL
+     */
+
+    if (MPI_PARAM_CHECK) {
+        if (NULL == info || MPI_INFO_NULL == info ||
+            ompi_info_is_freed(info)) {
+            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_INFO,
+                                           FUNC_NAME);
+        }
+
+        key_length = (key) ? (int)strlen (key) : 0;
+        if ((NULL == key) || (0 == key_length) ||
+            (MPI_MAX_INFO_KEY <= key_length)) {
+            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_INFO_KEY,
+                                           FUNC_NAME);
+        }
+
+        value_length = (value) ? (int)strlen (value) : 0;
+        if ((NULL == value) || (0 == value_length) ||
+            (MPI_MAX_INFO_VAL <= value_length)) {
+            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_INFO_VALUE,
+                                           FUNC_NAME);
+        }
+    }
+
+    /*
+     * If all is right with the arguments, then call the back-end
+     * allocator.
+     */
+
+    err = ompi_info_set (info, key, value);
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/init.c b/ompi/mpi/c/init.c
deleted file mode 100644
index eb5a50a7643..00000000000
--- a/ompi/mpi/c/init.c
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2018 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2006 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007-2018 Cisco Systems, Inc.  All rights reserved
- * Copyright (c) 2007-2008 Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <stdlib.h>
-
-#include "opal/util/show_help.h"
-#include "ompi/runtime/ompi_spc.h"
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/constants.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Init = PMPI_Init
-#endif
-#define MPI_Init PMPI_Init
-#endif
-
-static const char FUNC_NAME[] = "MPI_Init";
-
-
-int MPI_Init(int *argc, char ***argv)
-{
-    int err;
-    int provided;
-    char *env;
-    int required = MPI_THREAD_SINGLE;
-
-    /* check for environment overrides for required thread level.  If
-       there is, check to see that it is a valid/supported thread level.
-       If not, default to MPI_THREAD_MULTIPLE. */
-
-    if (NULL != (env = getenv("OMPI_MPI_THREAD_LEVEL"))) {
-        required = atoi(env);
-        if (required < MPI_THREAD_SINGLE || required > MPI_THREAD_MULTIPLE) {
-            required = MPI_THREAD_MULTIPLE;
-        }
-    }
-
-    /* Call the back-end initialization function (we need to put as
-       little in this function as possible so that if it's profiled, we
-       don't lose anything) */
-
-    if (NULL != argc && NULL != argv) {
-        err = ompi_mpi_init(*argc, *argv, required, &provided, false);
-    } else {
-        err = ompi_mpi_init(0, NULL, required, &provided, false);
-    }
-
-    /* Since we don't have a communicator to invoke an errorhandler on
-       here, don't use the fancy-schmancy ERRHANDLER macros; they're
-       really designed for real communicator objects.  Just use the
-       back-end function directly. */
-
-    if (MPI_SUCCESS != err) {
-        return ompi_errhandler_invoke(NULL, NULL,
-                                      OMPI_ERRHANDLER_TYPE_COMM,
-                                      err <
-                                      0 ? ompi_errcode_get_mpi_code(err) :
-                                      err, FUNC_NAME);
-    }
-
-    SPC_INIT();
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/init.c.in b/ompi/mpi/c/init.c.in
new file mode 100644
index 00000000000..b9e5d513482
--- /dev/null
+++ b/ompi/mpi/c/init.c.in
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2006 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007-2018 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2007-2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include <stdlib.h>
+
+#include "opal/util/show_help.h"
+#include "ompi/runtime/ompi_spc.h"
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/constants.h"
+
+PROTOTYPE INT init(INT_OUT argc, ARGV argv)
+{
+    int err;
+    int provided;
+    char *env;
+    int required = MPI_THREAD_SINGLE;
+
+    /* check for environment overrides for required thread level.  If
+       there is, check to see that it is a valid/supported thread level.
+       If not, default to MPI_THREAD_MULTIPLE. */
+
+    if (NULL != (env = getenv("OMPI_MPI_THREAD_LEVEL"))) {
+        required = atoi(env);
+        if (required < MPI_THREAD_SINGLE || required > MPI_THREAD_MULTIPLE) {
+            required = MPI_THREAD_MULTIPLE;
+        }
+    }
+
+    /* Call the back-end initialization function (we need to put as
+       little in this function as possible so that if it's profiled, we
+       don't lose anything) */
+
+    if (NULL != argc && NULL != argv) {
+        err = ompi_mpi_init(*argc, *argv, required, &provided, false);
+    } else {
+        err = ompi_mpi_init(0, NULL, required, &provided, false);
+    }
+
+    /* Since we don't have a communicator to invoke an errorhandler on
+       here, don't use the fancy-schmancy ERRHANDLER macros; they're
+       really designed for real communicator objects.  Just use the
+       back-end function directly. */
+
+    if (MPI_SUCCESS != err) {
+        return ompi_errhandler_invoke(NULL, NULL,
+                                      OMPI_ERRHANDLER_TYPE_COMM,
+                                      err <
+                                      0 ? ompi_errcode_get_mpi_code(err) :
+                                      err, FUNC_NAME);
+    }
+
+    SPC_INIT();
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/init_thread.c b/ompi/mpi/c/init_thread.c
deleted file mode 100644
index 2cdad8ff3af..00000000000
--- a/ompi/mpi/c/init_thread.c
+++ /dev/null
@@ -1,98 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2018 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2006 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2010      Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2015-2018 Cisco Systems, Inc.  All rights reserved
- * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "opal/util/show_help.h"
-#include "ompi/runtime/ompi_spc.h"
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/constants.h"
-#include "ompi/mca/hook/base/base.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Init_thread = PMPI_Init_thread
-#endif
-#define MPI_Init_thread PMPI_Init_thread
-#endif
-
-static const char FUNC_NAME[] = "MPI_Init_thread";
-
-
-int MPI_Init_thread(int *argc, char ***argv, int required,
-                    int *provided)
-{
-    int err, safe_required = MPI_THREAD_SERIALIZED;
-
-    ompi_hook_base_mpi_init_thread_top(argc, argv, required, provided);
-
-    /* Detect an incorrect thread support level, but dont report until we have the minimum
-     * infrastructure setup.
-     */
-    if( (MPI_THREAD_SINGLE == required) || (MPI_THREAD_SERIALIZED == required) ||
-        (MPI_THREAD_FUNNELED == required) || (MPI_THREAD_MULTIPLE == required) ) {
-        safe_required = required;
-    }
-
-    *provided = safe_required;
-
-    /* Call the back-end initialization function (we need to put as
-       little in this function as possible so that if it's profiled, we
-       don't lose anything) */
-
-    if (NULL != argc && NULL != argv) {
-        err = ompi_mpi_init(*argc, *argv, safe_required, provided, false);
-    } else {
-        err = ompi_mpi_init(0, NULL, safe_required, provided, false);
-    }
-
-    if( safe_required != required ) {
-        /* Trigger the error handler for the incorrect argument. Keep it separate from the
-         * check on the ompi_mpi_init return and report a nice, meaningful error message to
-         * the user. */
-        return ompi_errhandler_invoke((ompi_errhandler_t*)&ompi_mpi_errors_are_fatal, NULL, OMPI_ERRHANDLER_TYPE_COMM,
-                                      MPI_ERR_ARG, FUNC_NAME);
-    }
-
-    /* Since we don't have a communicator to invoke an errorhandler on
-       here, don't use the fancy-schmancy ERRHANDLER macros; they're
-       really designed for real communicator objects.  Just use the
-       back-end function directly. */
-
-    if (MPI_SUCCESS != err) {
-        return ompi_errhandler_invoke(NULL, NULL, OMPI_ERRHANDLER_TYPE_COMM,
-                                      err < 0 ? ompi_errcode_get_mpi_code(err) :
-                                      err, FUNC_NAME);
-    }
-
-    SPC_INIT();
-
-    ompi_hook_base_mpi_init_thread_bottom(argc, argv, required, provided);
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/init_thread.c.in b/ompi/mpi/c/init_thread.c.in
new file mode 100644
index 00000000000..f56728ee262
--- /dev/null
+++ b/ompi/mpi/c/init_thread.c.in
@@ -0,0 +1,97 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2006 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015-2018 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "opal/util/show_help.h"
+#include "ompi/runtime/ompi_spc.h"
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/constants.h"
+#include "ompi/mca/hook/base/base.h"
+
+PROTOTYPE ERROR_CLASS init_thread(INT_OUT argc, ARGV argv, INT required,
+                                  INT_OUT provided)
+{
+    int err, safe_required = MPI_THREAD_SERIALIZED;
+    char *env;
+
+    ompi_hook_base_mpi_init_thread_top(argc, argv, required, provided);
+
+    /* Detect an incorrect thread support level, but dont report until we have the minimum
+     * infrastructure setup.
+     */
+    if( (MPI_THREAD_SINGLE == required) || (MPI_THREAD_SERIALIZED == required) ||
+        (MPI_THREAD_FUNNELED == required) || (MPI_THREAD_MULTIPLE == required) ) {
+
+        if (NULL != (env = getenv("OMPI_MPI_THREAD_LEVEL")))  {
+            safe_required = atoi(env);
+        }
+        else {
+            safe_required = required;
+        }
+    }
+
+    *provided = safe_required;
+
+    /* Call the back-end initialization function (we need to put as
+       little in this function as possible so that if it's profiled, we
+       don't lose anything) */
+
+    if (NULL != argc && NULL != argv) {
+        err = ompi_mpi_init(*argc, *argv, safe_required, provided, false);
+    } else {
+        err = ompi_mpi_init(0, NULL, safe_required, provided, false);
+    }
+
+    if( safe_required != required ) {
+        /* Trigger the error handler for the incorrect argument. Keep it separate from the
+         * check on the ompi_mpi_init return and report a nice, meaningful error message to
+         * the user. */
+        return ompi_errhandler_invoke((ompi_errhandler_t*)&ompi_mpi_errors_are_fatal, NULL, OMPI_ERRHANDLER_TYPE_COMM,
+                                      MPI_ERR_ARG, FUNC_NAME);
+    }
+
+    /* Since we don't have a communicator to invoke an errorhandler on
+       here, don't use the fancy-schmancy ERRHANDLER macros; they're
+       really designed for real communicator objects.  Just use the
+       back-end function directly. */
+
+    if (MPI_SUCCESS != err) {
+        return ompi_errhandler_invoke(NULL, NULL, OMPI_ERRHANDLER_TYPE_COMM,
+                                      err < 0 ? ompi_errcode_get_mpi_code(err) :
+                                      err, FUNC_NAME);
+    }
+
+    SPC_INIT();
+
+    ompi_hook_base_mpi_init_thread_bottom(argc, argv, required, provided);
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/initialized.c b/ompi/mpi/c/initialized.c
deleted file mode 100644
index 57f4e466243..00000000000
--- a/ompi/mpi/c/initialized.c
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2015-2018 Cisco Systems, Inc.  All rights reserved
- * Copyright (c) 2015      Intel, Inc. All rights reserved
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/hook/base/base.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Initialized = PMPI_Initialized
-#endif
-#define MPI_Initialized PMPI_Initialized
-#endif
-
-static const char FUNC_NAME[] = "MPI_Initialized";
-
-
-int MPI_Initialized(int *flag)
-{
-    ompi_hook_base_mpi_initialized_top(flag);
-
-    int32_t state = ompi_mpi_state;
-
-    if (MPI_PARAM_CHECK) {
-        if (NULL == flag) {
-
-            /* If we have an error, the action that we take depends on
-               whether we're currently (after MPI_Init and before
-               MPI_Finalize) or not */
-
-            if (state >= OMPI_MPI_STATE_INIT_COMPLETED &&
-                state < OMPI_MPI_STATE_FINALIZE_PAST_COMM_SELF_DESTRUCT) {
-                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                              FUNC_NAME);
-            } else {
-                /* We have no MPI object here so call ompi_errhandle_invoke
-                 * directly */
-                return ompi_errhandler_invoke(NULL, NULL, -1,
-                                              ompi_errcode_get_mpi_code(MPI_ERR_ARG),
-                                              FUNC_NAME);
-            }
-        }
-    }
-
-    *flag = (state >= OMPI_MPI_STATE_INIT_COMPLETED);
-
-    ompi_hook_base_mpi_initialized_bottom(flag);
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/initialized.c.in b/ompi/mpi/c/initialized.c.in
new file mode 100644
index 00000000000..db347371bd2
--- /dev/null
+++ b/ompi/mpi/c/initialized.c.in
@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015-2018 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2015      Intel, Inc. All rights reserved
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/hook/base/base.h"
+
+PROTOTYPE ERROR_CLASS initialized(INT_OUT flag)
+{
+    ompi_hook_base_mpi_initialized_top(flag);
+
+    int32_t state = ompi_mpi_state;
+
+    if (MPI_PARAM_CHECK) {
+        if (NULL == flag) {
+
+            /* If we have an error, the action that we take depends on
+               whether we're currently (after MPI_Init and before
+               MPI_Finalize) or not */
+
+            if (state >= OMPI_MPI_STATE_INIT_COMPLETED &&
+                state < OMPI_MPI_STATE_FINALIZE_PAST_COMM_SELF_DESTRUCT) {
+                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                              FUNC_NAME);
+            } else {
+                /* We have no MPI object here so call ompi_errhandle_invoke
+                 * directly */
+                return ompi_errhandler_invoke(NULL, NULL, -1,
+                                              ompi_errcode_get_mpi_code(MPI_ERR_ARG),
+                                              FUNC_NAME);
+            }
+        }
+    }
+
+    *flag = (state >= OMPI_MPI_STATE_INIT_COMPLETED);
+
+    ompi_hook_base_mpi_initialized_bottom(flag);
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/intercomm_create.c b/ompi/mpi/c/intercomm_create.c
deleted file mode 100644
index 5c582c7c0e5..00000000000
--- a/ompi/mpi/c/intercomm_create.c
+++ /dev/null
@@ -1,257 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2006-2009 University of Houston.  All rights reserved.
- * Copyright (c) 2012-2013 Inria.  All rights reserved.
- * Copyright (c) 2014-2015 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2016      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/request/request.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Intercomm_create = PMPI_Intercomm_create
-#endif
-#define MPI_Intercomm_create PMPI_Intercomm_create
-#endif
-
-static const char FUNC_NAME[] = "MPI_Intercomm_create";
-
-
-int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader,
-                         MPI_Comm bridge_comm, int remote_leader,
-                         int tag, MPI_Comm *newintercomm)
-{
-    int local_size=0, local_rank=0;
-    int lleader=0, rleader=0;
-    ompi_communicator_t *newcomp=NULL;
-    struct ompi_proc_t **rprocs=NULL;
-    int rc=0, rsize=0;
-    ompi_proc_t **proc_list=NULL;
-    int j;
-    ompi_group_t *new_group_pointer;
-
-    MEMCHECKER(
-        memchecker_comm(local_comm);
-        memchecker_comm(bridge_comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if ( ompi_comm_invalid ( local_comm ) ||
-             ( local_comm->c_flags & OMPI_COMM_INTER ) )
-            return OMPI_ERRHANDLER_INVOKE ( MPI_COMM_WORLD, MPI_ERR_COMM,
-                                            FUNC_NAME);
-
-        if ( NULL == newintercomm )
-            return OMPI_ERRHANDLER_INVOKE ( local_comm, MPI_ERR_ARG,
-                                            FUNC_NAME);
-
-        /* if ( tag < 0 || tag > MPI_TAG_UB )
-             return OMPI_ERRHANDLER_INVOKE ( local_comm, MPI_ERR_ARG,
-                                             FUNC_NAME);
-        */
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * We must not call ompi_comm_iface_create_check() here, because that
-     * risks leaving the remote group dangling on an unmatched operation.
-     * We will let the  logic proceed and discover the
-     * issue internally so that all sides get informed.
-     */
-#endif
-
-    local_size = ompi_comm_size ( local_comm );
-    local_rank = ompi_comm_rank ( local_comm );
-    lleader = local_leader;
-    rleader = remote_leader;
-
-    if ( MPI_PARAM_CHECK ) {
-        if ( (0 > local_leader) || (local_leader >= local_size) )
-            return OMPI_ERRHANDLER_INVOKE ( local_comm, MPI_ERR_ARG,
-                                            FUNC_NAME);
-
-        /* remember that the remote_leader and bridge_comm arguments
-           just have to be valid at the local_leader */
-        if ( local_rank == local_leader ) {
-            if ( ompi_comm_invalid ( bridge_comm ) ||
-                 (bridge_comm->c_flags & OMPI_COMM_INTER) ) {
-                return OMPI_ERRHANDLER_INVOKE ( local_comm, MPI_ERR_COMM,
-                                                FUNC_NAME);
-            }
-            if ( (remote_leader < 0) || (remote_leader >= ompi_comm_size(bridge_comm))) {
-                return OMPI_ERRHANDLER_INVOKE ( local_comm, MPI_ERR_ARG,
-                                                FUNC_NAME);
-            }
-        } /* if ( local_rank == local_leader ) */
-    }
-
-    if ( local_rank == local_leader ) {
-        MPI_Request req;
-
-        /* local leader exchange group sizes lists */
-        rc = MCA_PML_CALL(irecv(&rsize, 1, MPI_INT, rleader, tag, bridge_comm,
-                                &req));
-        if ( rc != MPI_SUCCESS ) {
-#if OPAL_ENABLE_FT_MPI
-            if( MPI_ERR_PROC_FAILED == rc ) {
-                rsize = 0;
-                goto skip_handshake;
-            }
-#endif  /* OPAL_ENABLE_FT_MPI */
-            goto err_exit;
-        }
-        rc = MCA_PML_CALL(send (&local_size, 1, MPI_INT, rleader, tag,
-                                MCA_PML_BASE_SEND_STANDARD, bridge_comm));
-        if ( rc != MPI_SUCCESS ) {
-#if OPAL_ENABLE_FT_MPI
-            if( MPI_ERR_PROC_FAILED == rc ) {
-                rsize = 0;
-                goto skip_handshake;
-            }
-#endif  /* OPAL_ENABLE_FT_MPI */
-            goto err_exit;
-        }
-#if OPAL_ENABLE_FT_MPI
-  skip_handshake:  /* nothing special */;
-#endif /* OPAL_ENABLE_FT_MPI */
-        rc = ompi_request_wait( &req, MPI_STATUS_IGNORE);
-        if ( rc != MPI_SUCCESS ) {
-            rsize = 0;  /* participate in the collective and then done */
-        }
-    }
-
-    /* bcast size and list of remote processes to all processes in local_comm */
-    rc = local_comm->c_coll->coll_bcast ( &rsize, 1, MPI_INT, lleader,
-                                         local_comm,
-                                         local_comm->c_coll->coll_bcast_module);
-    if ( rc != MPI_SUCCESS ) {
-#if OPAL_ENABLE_FT_MPI
-        if ( local_rank != local_leader ) {
-            goto err_exit;
-        }
-        /* the leaders must go in the ger_rprocs in order to avoid deadlocks */
-#else
-        goto err_exit;
-#endif  /* OPAL_ENABLE_FT_MPI */
-    }
-
-    rc = ompi_comm_get_rprocs( local_comm, bridge_comm, lleader,
-                               remote_leader, tag, rsize, &rprocs );
-    if ( OMPI_SUCCESS != rc ) {
-        goto err_exit;
-    }
-
-    if ( MPI_PARAM_CHECK ) {
-        if(OMPI_GROUP_IS_DENSE(local_comm->c_local_group)) {
-            rc = ompi_comm_overlapping_groups(local_comm->c_local_group->grp_proc_count,
-                                              local_comm->c_local_group->grp_proc_pointers,
-                                              rsize,
-                                              rprocs);
-        }
-        else {
-            proc_list = (ompi_proc_t **) calloc (local_comm->c_local_group->grp_proc_count,
-                                                 sizeof (ompi_proc_t *));
-            for(j=0 ; j<local_comm->c_local_group->grp_proc_count ; j++) {
-                proc_list[j] = ompi_group_peer_lookup(local_comm->c_local_group,j);
-            }
-            rc = ompi_comm_overlapping_groups(local_comm->c_local_group->grp_proc_count,
-                                              proc_list,
-                                              rsize,
-                                              rprocs);
-        }
-        if ( OMPI_SUCCESS != rc ) {
-            goto err_exit;
-        }
-    }
-    new_group_pointer = ompi_group_allocate(rsize);
-    if( NULL == new_group_pointer ) {
-        rc = MPI_ERR_GROUP;
-        goto err_exit;
-    }
-
-    /* put group elements in the list */
-    for (j = 0; j < rsize; j++) {
-        new_group_pointer->grp_proc_pointers[j] = rprocs[j];
-        OBJ_RETAIN(rprocs[j]);
-    }
-
-    rc = ompi_comm_set ( &newcomp,                                     /* new comm */
-                         local_comm,                                   /* old comm */
-                         local_comm->c_local_group->grp_proc_count,    /* local_size */
-                         NULL,                                         /* local_procs*/
-                         rsize,                                        /* remote_size */
-                         NULL,                                         /* remote_procs */
-                         NULL,                                         /* attrs */
-                         local_comm->error_handler,                    /* error handler*/
-                         false,                                        /* dont copy the topo */
-                         local_comm->c_local_group,                    /* local group */
-                         new_group_pointer                             /* remote group */
-                         );
-
-    if ( MPI_SUCCESS != rc ) {
-        goto err_exit;
-    }
-
-    OBJ_RELEASE(new_group_pointer);
-    new_group_pointer = MPI_GROUP_NULL;
-
-    /* Determine context id. It is identical to f_2_c_handle */
-    rc = ompi_comm_nextcid (newcomp, local_comm, bridge_comm, &lleader,
-                            &rleader, false, OMPI_COMM_CID_INTRA_BRIDGE);
-    if ( MPI_SUCCESS != rc ) {
-        goto err_exit;
-    }
-
-    /* activate comm and init coll-module */
-    rc = ompi_comm_activate (&newcomp, local_comm, bridge_comm, &lleader, &rleader,
-                             false, OMPI_COMM_CID_INTRA_BRIDGE);
-    if ( MPI_SUCCESS != rc ) {
-        goto err_exit;
-    }
-
- err_exit:
-    if ( NULL != rprocs ) {
-        free ( rprocs );
-    }
-    if ( NULL != proc_list ) {
-        free ( proc_list );
-    }
-    if ( OMPI_SUCCESS != rc ) {
-        *newintercomm = MPI_COMM_NULL;
-        return OMPI_ERRHANDLER_INVOKE(local_comm, rc,
-                                      FUNC_NAME);
-    }
-
-    *newintercomm = newcomp;
-    return MPI_SUCCESS;
-}
-
diff --git a/ompi/mpi/c/intercomm_create.c.in b/ompi/mpi/c/intercomm_create.c.in
new file mode 100644
index 00000000000..6eb055aae98
--- /dev/null
+++ b/ompi/mpi/c/intercomm_create.c.in
@@ -0,0 +1,82 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2006-2009 University of Houston.  All rights reserved.
+ * Copyright (c) 2012-2013 Inria.  All rights reserved.
+ * Copyright (c) 2014-2015 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/request/request.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS intercomm_create(COMM local_comm, INT local_leader,
+                                       COMM bridge_comm, INT remote_leader,
+                                       INT tag, COMM_OUT newintercomm)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_comm(local_comm);
+        memchecker_comm(bridge_comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if ( ompi_comm_invalid ( local_comm ) ||
+             ( local_comm->c_flags & OMPI_COMM_INTER ) )
+            return OMPI_ERRHANDLER_INVOKE ( MPI_COMM_WORLD, MPI_ERR_COMM,
+                                            FUNC_NAME);
+
+        if ( NULL == newintercomm )
+            return OMPI_ERRHANDLER_INVOKE ( local_comm, MPI_ERR_ARG,
+                                            FUNC_NAME);
+
+        /* if ( tag < 0 || tag > MPI_TAG_UB )
+             return OMPI_ERRHANDLER_INVOKE ( local_comm, MPI_ERR_ARG,
+                                             FUNC_NAME);
+        */
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * We must not call ompi_comm_iface_create_check() here, because that
+     * risks leaving the remote group dangling on an unmatched operation.
+     * We will let the  logic proceed and discover the
+     * issue internally so that all sides get informed.
+     */
+#endif
+
+    rc = ompi_intercomm_create (local_comm, local_leader, bridge_comm, remote_leader, tag,
+                                newintercomm);
+
+    OMPI_ERRHANDLER_RETURN (rc, local_comm, rc, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/intercomm_create_from_groups.c.in b/ompi/mpi/c/intercomm_create_from_groups.c.in
new file mode 100644
index 00000000000..ac1b88f257c
--- /dev/null
+++ b/ompi/mpi/c/intercomm_create_from_groups.c.in
@@ -0,0 +1,102 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2017 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2006-2009 University of Houston.  All rights reserved.
+ * Copyright (c) 2012-2013 Inria.  All rights reserved.
+ * Copyright (c) 2014-2015 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/request/request.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS intercomm_create_from_groups (GROUP local_group, INT local_leader, GROUP remote_group,
+                                                    INT remote_leader, STRING tag, INFO info, ERRHANDLER errhandler,
+                                                    COMM_OUT newintercomm)
+{
+    int rc, my_grp_rank, remote_grp_size;
+
+    MEMCHECKER(
+        memchecker_comm(local_comm);
+        memchecker_comm(bridge_comm);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (NULL == errhandler ||
+               MPI_ERRHANDLER_NULL == errhandler ||
+                    ( OMPI_ERRHANDLER_TYPE_COMM != errhandler->eh_mpi_object_type &&
+                     OMPI_ERRHANDLER_TYPE_PREDEFINED != errhandler->eh_mpi_object_type) ) {
+               return ompi_errhandler_invoke (NULL, MPI_COMM_NULL, OMPI_ERRHANDLER_TYPE_COMM,
+                                      MPI_ERR_ARG,FUNC_NAME);
+
+        }
+
+        if (NULL == info || ompi_info_is_freed(info)) {
+            return ompi_errhandler_invoke (errhandler, MPI_COMM_NULL, OMPI_ERRHANDLER_TYPE_COMM,
+                                           MPI_ERR_INFO, FUNC_NAME);
+        }
+        if (NULL == tag) {
+            return ompi_errhandler_invoke (errhandler, MPI_COMM_NULL, OMPI_ERRHANDLER_TYPE_COMM,
+                                           MPI_ERR_TAG, FUNC_NAME);
+        }
+        if (NULL == newintercomm) {
+            return ompi_errhandler_invoke (errhandler, MPI_COMM_NULL, OMPI_ERRHANDLER_TYPE_COMM,
+                                           MPI_ERR_ARG, FUNC_NAME);
+        }
+
+        my_grp_rank = ompi_group_rank((ompi_group_t *)local_group);
+        if (local_leader == my_grp_rank) {
+
+            if (NULL == local_group || NULL == remote_group) {
+                return ompi_errhandler_invoke (errhandler, MPI_COMM_NULL, OMPI_ERRHANDLER_TYPE_COMM,
+                                               MPI_ERR_GROUP, FUNC_NAME);
+            }
+
+            remote_grp_size = ompi_group_size((ompi_group_t *)remote_group);
+            if (remote_leader >= remote_grp_size) {
+                rc = ompi_errhandler_invoke (errhandler, MPI_COMM_NULL, OMPI_ERRHANDLER_TYPE_COMM,
+                                               MPI_ERR_ARG, FUNC_NAME);
+                return rc;
+            }
+        }
+    }
+
+    rc = ompi_intercomm_create_from_groups (local_group, local_leader, remote_group, remote_leader, tag,
+                                            &info->super, errhandler, newintercomm);
+
+    if (MPI_SUCCESS != rc) {
+        return ompi_errhandler_invoke (errhandler, MPI_COMM_NULL, errhandler->eh_mpi_object_type,
+                                       rc, FUNC_NAME);
+    }
+
+    return rc;
+}
+
diff --git a/ompi/mpi/c/intercomm_merge.c b/ompi/mpi/c/intercomm_merge.c
deleted file mode 100644
index 18c458e5ca9..00000000000
--- a/ompi/mpi/c/intercomm_merge.c
+++ /dev/null
@@ -1,158 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2006-2009 University of Houston.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2012-2013 Inria.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <string.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/proc/proc.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Intercomm_merge = PMPI_Intercomm_merge
-#endif
-#define MPI_Intercomm_merge PMPI_Intercomm_merge
-#endif
-
-static const char FUNC_NAME[] = "MPI_Intercomm_merge";
-
-
-int MPI_Intercomm_merge(MPI_Comm intercomm, int high,
-                        MPI_Comm *newcomm)
-{
-    ompi_communicator_t *newcomp=MPI_COMM_NULL;
-    ompi_proc_t **procs=NULL;
-    int local_size, remote_size;
-    int first;
-    int total_size;
-    int rc=MPI_SUCCESS;
-    int thigh = high;
-    ompi_group_t *new_group_pointer;
-
-    MEMCHECKER(
-        memchecker_comm(intercomm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_comm_invalid ( intercomm ) ||
-             !( intercomm->c_flags & OMPI_COMM_INTER ) )
-            return OMPI_ERRHANDLER_INVOKE ( MPI_COMM_WORLD, MPI_ERR_COMM,
-                                            FUNC_NAME);
-
-        if ( NULL == newcomm )
-            return OMPI_ERRHANDLER_INVOKE ( intercomm, MPI_ERR_ARG,
-                                            FUNC_NAME);
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_create_check(intercomm, &rc)) ) {
-        OMPI_ERRHANDLER_RETURN(rc, intercomm, rc, FUNC_NAME);
-    }
-#endif
-
-    local_size  = ompi_comm_size ( intercomm );
-    remote_size = ompi_comm_remote_size ( intercomm );
-    total_size  = local_size + remote_size;
-    procs = (ompi_proc_t **) malloc ( total_size * sizeof(ompi_proc_t *));
-    if ( NULL == procs ) {
-        rc = MPI_ERR_INTERN;
-        goto exit;
-    }
-
-    first = ompi_comm_determine_first ( intercomm, thigh );
-    if ( MPI_UNDEFINED == first ) {
-        rc = MPI_ERR_INTERN;
-        goto exit;
-    }
-
-    if ( first ) {
-        ompi_group_union ( intercomm->c_local_group, intercomm->c_remote_group, &new_group_pointer );
-    }
-    else {
-        ompi_group_union ( intercomm->c_remote_group, intercomm->c_local_group, &new_group_pointer );
-    }
-
-    rc = ompi_comm_set ( &newcomp,                 /* new comm */
-                         intercomm,                /* old comm */
-                         total_size,               /* local_size */
-                         NULL,                     /* local_procs*/
-                         0,                        /* remote_size */
-                         NULL,                     /* remote_procs */
-                         NULL,                     /* attrs */
-                         intercomm->error_handler, /* error handler*/
-                         false,                    /* don't copy the topo */
-                         new_group_pointer,        /* local group */
-                         NULL                      /* remote group */
-                         );
-    if ( MPI_SUCCESS != rc ) {
-        goto exit;
-    }
-
-    OBJ_RELEASE(new_group_pointer);
-    new_group_pointer = MPI_GROUP_NULL;
-
-    /* Determine context id */
-    rc = ompi_comm_nextcid (newcomp, intercomm, NULL, NULL, NULL, false,
-                            OMPI_COMM_CID_INTER);
-    if ( OMPI_SUCCESS != rc ) {
-        goto exit;
-    }
-
-    /* activate communicator and init coll-module */
-    rc = ompi_comm_activate (&newcomp, intercomm, NULL, NULL, NULL, false,
-                             OMPI_COMM_CID_INTER);
-    if ( OMPI_SUCCESS != rc ) {
-        goto exit;
-    }
-
- exit:
-    if ( NULL != procs ) {
-        free ( procs );
-    }
-    if ( MPI_SUCCESS != rc ) {
-        if ( MPI_COMM_NULL != newcomp && NULL != newcomp ) {
-            OBJ_RELEASE(newcomp);
-        }
-        *newcomm = MPI_COMM_NULL;
-        return OMPI_ERRHANDLER_INVOKE(intercomm, rc,  FUNC_NAME);
-    }
-
-    *newcomm = newcomp;
-    return MPI_SUCCESS;
-}
-
diff --git a/ompi/mpi/c/intercomm_merge.c.in b/ompi/mpi/c/intercomm_merge.c.in
new file mode 100644
index 00000000000..6e26f1e3fb6
--- /dev/null
+++ b/ompi/mpi/c/intercomm_merge.c.in
@@ -0,0 +1,157 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2017 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2006-2009 University of Houston.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2012-2013 Inria.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <string.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/proc/proc.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS intercomm_merge(COMM intercomm, INT high,
+                                      COMM_OUT newcomm)
+{
+    ompi_communicator_t *newcomp = MPI_COMM_NULL;
+    ompi_proc_t **procs=NULL;
+    int first, thigh = high;
+    int local_size, remote_size;
+    int total_size;
+    int rc=MPI_SUCCESS;
+    ompi_group_t *new_group_pointer;
+
+    MEMCHECKER(
+        memchecker_comm(intercomm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_comm_invalid ( intercomm ) ||
+             !( intercomm->c_flags & OMPI_COMM_INTER ) )
+            return OMPI_ERRHANDLER_INVOKE ( MPI_COMM_WORLD, MPI_ERR_COMM,
+                                            FUNC_NAME);
+
+        if ( NULL == newcomm )
+            return OMPI_ERRHANDLER_INVOKE ( intercomm, MPI_ERR_ARG,
+                                            FUNC_NAME);
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_create_check(intercomm, &rc)) ) {
+        OMPI_ERRHANDLER_RETURN(rc, intercomm, rc, FUNC_NAME);
+    }
+#endif
+
+    local_size  = ompi_comm_size ( intercomm );
+    remote_size = ompi_comm_remote_size ( intercomm );
+    total_size  = local_size + remote_size;
+    procs = (ompi_proc_t **) malloc ( total_size * sizeof(ompi_proc_t *));
+    if ( NULL == procs ) {
+        rc = MPI_ERR_INTERN;
+        goto exit;
+    }
+
+    first = ompi_comm_determine_first ( intercomm, thigh );
+    if ( MPI_UNDEFINED == first ) {
+        rc = MPI_ERR_INTERN;
+        goto exit;
+    }
+
+    if ( first ) {
+        ompi_group_union ( intercomm->c_local_group, intercomm->c_remote_group, &new_group_pointer );
+    }
+    else {
+        ompi_group_union ( intercomm->c_remote_group, intercomm->c_local_group, &new_group_pointer );
+    }
+
+    rc = ompi_comm_set ( &newcomp,                 /* new comm */
+                         intercomm,                /* old comm */
+                         total_size,               /* local_size */
+                         NULL,                     /* local_procs*/
+                         0,                        /* remote_size */
+                         NULL,                     /* remote_procs */
+                         NULL,                     /* attrs */
+                         intercomm->error_handler, /* error handler*/
+                         new_group_pointer,        /* local group */
+                         NULL,                     /* remote group */
+                         0);
+    if ( MPI_SUCCESS != rc ) {
+        goto exit;
+    }
+
+    OBJ_RELEASE(new_group_pointer);
+    new_group_pointer = MPI_GROUP_NULL;
+
+    /* Determine context id */
+    rc = ompi_comm_nextcid (newcomp, intercomm, NULL, NULL, NULL, false,
+                            OMPI_COMM_CID_INTER);
+    if ( OMPI_SUCCESS != rc ) {
+        goto exit;
+    }
+
+    /* activate communicator and init coll-module */
+    rc = ompi_comm_activate (&newcomp, intercomm, NULL, NULL, NULL, false,
+                             OMPI_COMM_CID_INTER);
+    if ( OMPI_SUCCESS != rc ) {
+        goto exit;
+    }
+
+    ompi_info_memkind_assert_type type;
+    newcomp->super.s_info = OBJ_NEW(opal_info_t);
+    ompi_info_memkind_copy_or_set (&intercomm->instance->super, &newcomp->super,
+                                   &ompi_mpi_info_null.info.super, &type);
+    if (OMPI_INFO_MEMKIND_ASSERT_NO_ACCEL == type) {
+        newcomp->c_assertions |= OMPI_COMM_ASSERT_NO_ACCEL_BUF;
+    }
+
+    
+ exit:
+
+    if ( NULL != procs ) {
+        free ( procs );
+    }
+    if ( MPI_SUCCESS != rc ) {
+        if ( MPI_COMM_NULL != newcomp && NULL != newcomp ) {
+            OBJ_RELEASE(newcomp);
+        }
+        *newcomm = MPI_COMM_NULL;
+        return OMPI_ERRHANDLER_INVOKE(intercomm, rc,  FUNC_NAME);
+    }
+
+    *newcomm = newcomp;
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/iprobe.c b/ompi/mpi/c/iprobe.c
deleted file mode 100644
index 7cea64d8bde..00000000000
--- a/ompi/mpi/c/iprobe.c
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/memchecker.h"
-#include "ompi/request/request.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Iprobe = PMPI_Iprobe
-#endif
-#define MPI_Iprobe PMPI_Iprobe
-#endif
-
-static const char FUNC_NAME[] = "MPI_Iprobe";
-
-
-int MPI_Iprobe(int source, int tag, MPI_Comm comm, int *flag, MPI_Status *status)
-{
-    int rc;
-
-    SPC_RECORD(OMPI_SPC_IPROBE, 1);
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (((tag < 0) && (tag != MPI_ANY_TAG)) || (tag > mca_pml.pml_max_tag)) {
-            rc = MPI_ERR_TAG;
-        } else if (ompi_comm_invalid(comm)) {
-            rc = MPI_ERR_COMM;
-        } else if ((source != MPI_ANY_SOURCE) &&
-                   (MPI_PROC_NULL != source) &&
-                   ompi_comm_peer_invalid(comm, source)) {
-            rc = MPI_ERR_RANK;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-    }
-
-    if (MPI_PROC_NULL == source) {
-        *flag = 1;
-        if (MPI_STATUS_IGNORE != status) {
-            *status = ompi_request_empty.req_status;
-            /*
-             * Per MPI-1, the MPI_ERROR field is not defined for single-completion calls
-             */
-            MEMCHECKER(
-                opal_memchecker_base_mem_undefined(&status->MPI_ERROR, sizeof(int));
-            );
-        }
-        return MPI_SUCCESS;
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * The call will check for process failure errors during the
-     * operation. So no need to check here.
-     */
-#endif
-
-    rc = MCA_PML_CALL(iprobe(source, tag, comm, flag, status));
-
-    /*
-     * Per MPI-1, the MPI_ERROR field is not defined for single-completion calls
-     */
-    MEMCHECKER(
-        opal_memchecker_base_mem_undefined(&status->MPI_ERROR, sizeof(int));
-    );
-
-    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/iprobe.c.in b/ompi/mpi/c/iprobe.c.in
new file mode 100644
index 00000000000..751f41ec8c3
--- /dev/null
+++ b/ompi/mpi/c/iprobe.c.in
@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/memchecker.h"
+#include "ompi/request/request.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS iprobe(INT source, INT tag, COMM comm, INT_OUT flag, STATUS_OUT status)
+{
+    int rc;
+
+    SPC_RECORD(OMPI_SPC_IPROBE, 1);
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (((tag < 0) && (tag != MPI_ANY_TAG)) || (tag > mca_pml.pml_max_tag)) {
+            rc = MPI_ERR_TAG;
+        } else if (ompi_comm_invalid(comm)) {
+            rc = MPI_ERR_COMM;
+        } else if ((source != MPI_ANY_SOURCE) &&
+                   (MPI_PROC_NULL != source) &&
+                   ompi_comm_peer_invalid(comm, source)) {
+            rc = MPI_ERR_RANK;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+    if (MPI_PROC_NULL == source) {
+        *flag = 1;
+        if (MPI_STATUS_IGNORE != status) {
+            OMPI_COPY_STATUS(status, ompi_request_empty.req_status, false);
+            /*
+             * Per MPI-1, the MPI_ERROR field is not defined for single-completion calls
+             */
+            MEMCHECKER(
+                opal_memchecker_base_mem_undefined(&status->MPI_ERROR, sizeof(int));
+            );
+        }
+        return MPI_SUCCESS;
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * The call will check for process failure errors during the
+     * operation. So no need to check here.
+     */
+#endif
+
+    rc = MCA_PML_CALL(iprobe(source, tag, comm, flag, status));
+
+    /*
+     * Per MPI-1, the MPI_ERROR field is not defined for single-completion calls
+     */
+    MEMCHECKER(
+        opal_memchecker_base_mem_undefined(&status->MPI_ERROR, sizeof(int));
+    );
+
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/irecv.c b/ompi/mpi/c/irecv.c
deleted file mode 100644
index 7173282de18..00000000000
--- a/ompi/mpi/c/irecv.c
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/request/request.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Irecv = PMPI_Irecv
-#endif
-#define MPI_Irecv PMPI_Irecv
-#endif
-
-static const char FUNC_NAME[] = "MPI_Irecv";
-
-
-int MPI_Irecv(void *buf, int count, MPI_Datatype type, int source,
-              int tag, MPI_Comm comm, MPI_Request *request)
-{
-    int rc = MPI_SUCCESS;
-
-    SPC_RECORD(OMPI_SPC_IRECV, 1);
-
-    MEMCHECKER(
-        memchecker_datatype(type);
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        OMPI_CHECK_DATATYPE_FOR_RECV(rc, type, count);
-        OMPI_CHECK_USER_BUFFER(rc, buf, type, count);
-
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
-        } else if (((tag < 0) && (tag != MPI_ANY_TAG)) || (tag > mca_pml.pml_max_tag)) {
-            rc = MPI_ERR_TAG;
-        } else if ((MPI_ANY_SOURCE != source) &&
-                   (MPI_PROC_NULL != source) &&
-                   ompi_comm_peer_invalid(comm, source)) {
-            rc = MPI_ERR_RANK;
-        } else if (NULL == request) {
-            rc = MPI_ERR_REQUEST;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-    }
-
-    if (source == MPI_PROC_NULL) {
-        *request = &ompi_request_empty;
-        return MPI_SUCCESS;
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * The request will be checked for process failure errors during the
-     * completion calls. So no need to check here.
-     */
-#endif
-
-    MEMCHECKER (
-        memchecker_call(&opal_memchecker_base_mem_noaccess, buf, count, type);
-    );
-    rc = MCA_PML_CALL(irecv(buf,count,type,source,tag,comm,request));
-    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/irecv.c.in b/ompi/mpi/c/irecv.c.in
new file mode 100644
index 00000000000..b2344cb5982
--- /dev/null
+++ b/ompi/mpi/c/irecv.c.in
@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/request/request.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS irecv(BUFFER_OUT buf, COUNT count, DATATYPE type,
+                            INT source, INT tag, COMM comm, REQUEST_INOUT request)
+{
+    int rc = MPI_SUCCESS;
+
+    SPC_RECORD(OMPI_SPC_IRECV, 1);
+
+    MEMCHECKER(
+        memchecker_datatype(type);
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        OMPI_CHECK_DATATYPE_FOR_RECV(rc, type, count);
+        OMPI_CHECK_USER_BUFFER(rc, buf, type, count);
+
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+        } else if (((tag < 0) && (tag != MPI_ANY_TAG)) || (tag > mca_pml.pml_max_tag)) {
+            rc = MPI_ERR_TAG;
+        } else if ((MPI_ANY_SOURCE != source) &&
+                   (MPI_PROC_NULL != source) &&
+                   ompi_comm_peer_invalid(comm, source)) {
+            rc = MPI_ERR_RANK;
+        } else if (NULL == request) {
+            rc = MPI_ERR_REQUEST;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+    if (source == MPI_PROC_NULL) {
+        *request = &ompi_request_empty;
+        return MPI_SUCCESS;
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * The request will be checked for process failure errors during the
+     * completion calls. So no need to check here.
+     */
+#endif
+
+    MEMCHECKER (
+        memchecker_call(&opal_memchecker_base_mem_noaccess, buf, count, type);
+    );
+    rc = MCA_PML_CALL(irecv(buf,count,type,source,tag,comm,request));
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/ireduce.c b/ompi/mpi/c/ireduce.c
deleted file mode 100644
index 34e773f43a9..00000000000
--- a/ompi/mpi/c/ireduce.c
+++ /dev/null
@@ -1,145 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2016      IBM Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/op/op.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Ireduce = PMPI_Ireduce
-#endif
-#define MPI_Ireduce PMPI_Ireduce
-#endif
-
-static const char FUNC_NAME[] = "MPI_Ireduce";
-
-
-int MPI_Ireduce(const void *sendbuf, void *recvbuf, int count,
-                MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm, MPI_Request *request)
-{
-    int err;
-
-    SPC_RECORD(OMPI_SPC_IREDUCE, 1);
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_comm(comm);
-
-        if(OMPI_COMM_IS_INTRA(comm)) {
-            if(ompi_comm_rank(comm) == root) {
-                /* check whether root's send buffer is defined. */
-                if (MPI_IN_PLACE == sendbuf) {
-                    memchecker_call(&opal_memchecker_base_isdefined, recvbuf, count, datatype);
-                } else {
-                    memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
-                }
-
-                /* check whether root's receive buffer is addressable. */
-                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, count, datatype);
-            } else {
-                /* check whether send buffer is defined on other processes. */
-                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
-            }
-        } else {
-            if (MPI_ROOT == root) {
-                /* check whether root's receive buffer is addressable. */
-                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, count, datatype);
-            } else if (MPI_PROC_NULL != root) {
-                /* check whether send buffer is defined. */
-                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
-            }
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        char *msg;
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-
-        /* Checks for all ranks */
-
-        else if (MPI_OP_NULL == op || NULL == op) {
-            err = MPI_ERR_OP;
-        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
-            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
-            free(msg);
-            return ret;
-        } else if ((ompi_comm_rank(comm) != root && MPI_IN_PLACE == sendbuf) ||
-                   (ompi_comm_rank(comm) == root && ((MPI_IN_PLACE == recvbuf) || (sendbuf == recvbuf)))) {
-            err = MPI_ERR_ARG;
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-
-        /* Intercommunicator errors */
-
-        if (!OMPI_COMM_IS_INTRA(comm)) {
-            if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
-                   MPI_ROOT == root || MPI_PROC_NULL == root)) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
-            }
-        }
-
-        /* Intracommunicator errors */
-
-        else {
-            if (root < 0 || root >= ompi_comm_size(comm)) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
-            }
-        }
-    }
-
-    /* MPI standard says that reductions have to have a count of at least 1,
-     * but some benchmarks (e.g., IMB) calls this function with a count of 0.
-     * So handle that case.
-     */
-    if (0 == count) {
-        *request = &ompi_request_empty;
-        return MPI_SUCCESS;
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_ireduce(sendbuf, recvbuf, count,
-                                    datatype, op, root, comm, request,
-                                    comm->c_coll->coll_ireduce_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_op(*request, op, datatype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/ireduce.c.in b/ompi/mpi/c/ireduce.c.in
new file mode 100644
index 00000000000..5dbdf1b102b
--- /dev/null
+++ b/ompi/mpi/c/ireduce.c.in
@@ -0,0 +1,148 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2023 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2016      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/op/op.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS ireduce(BUFFER sendbuf, BUFFER_OUT recvbuf, COUNT count,
+                              DATATYPE datatype, OP op, INT root, COMM comm, REQUEST_INOUT request)
+{
+    int err;
+
+    SPC_RECORD(OMPI_SPC_IREDUCE, 1);
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_comm(comm);
+
+        if(OMPI_COMM_IS_INTRA(comm)) {
+            if(ompi_comm_rank(comm) == root) {
+                /* check whether root's send buffer is defined. */
+                if (MPI_IN_PLACE == sendbuf) {
+                    memchecker_call(&opal_memchecker_base_isdefined, recvbuf, count, datatype);
+                } else {
+                    memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
+                }
+
+                /* check whether root's receive buffer is addressable. */
+                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, count, datatype);
+            } else {
+                /* check whether send buffer is defined on other processes. */
+                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
+            }
+        } else {
+            if (MPI_ROOT == root) {
+                /* check whether root's receive buffer is addressable. */
+                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, count, datatype);
+            } else if (MPI_PROC_NULL != root) {
+                /* check whether send buffer is defined. */
+                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
+            }
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        char *msg;
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+
+        /* Checks for all ranks */
+
+        else if (MPI_OP_NULL == op || NULL == op) {
+            err = MPI_ERR_OP;
+        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
+            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
+            free(msg);
+            return ret;
+        } else if ((ompi_comm_rank(comm) != root && MPI_IN_PLACE == sendbuf) ||
+                   (ompi_comm_rank(comm) == root && ((MPI_IN_PLACE == recvbuf) ||
+                   ((sendbuf == recvbuf) && (0 != count))))) {
+            err = MPI_ERR_ARG;
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+
+        /* Intercommunicator errors */
+
+        if (!OMPI_COMM_IS_INTRA(comm)) {
+            if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
+                   MPI_ROOT == root || MPI_PROC_NULL == root)) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
+            }
+        }
+
+        /* Intracommunicator errors */
+
+        else {
+            if (root < 0 || root >= ompi_comm_size(comm)) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
+            }
+        }
+    }
+
+    /* MPI standard says that reductions have to have a count of at least 1,
+     * but some benchmarks (e.g., IMB) calls this function with a count of 0.
+     * So handle that case.
+     */
+    if (0 == count) {
+        *request = &ompi_request_empty;
+        return MPI_SUCCESS;
+    }
+
+    void *updated_recvbuf;
+    const void *updated_sendbuf;
+    if(OMPI_COMM_IS_INTRA(comm)) {
+        updated_recvbuf = (ompi_comm_rank(comm) == root) ? recvbuf : NULL;
+        updated_sendbuf = sendbuf;
+    } else {
+        updated_sendbuf = ((MPI_ROOT == root) || (MPI_PROC_NULL == root)) ? NULL : sendbuf;
+        updated_recvbuf = (MPI_ROOT == root) ? recvbuf : NULL;
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    err = comm->c_coll->coll_ireduce(updated_sendbuf, updated_recvbuf, count,
+                                     datatype, op, root, comm, request,
+                                     comm->c_coll->coll_ireduce_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_op(*request, op, datatype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/ireduce_scatter.c b/ompi/mpi/c/ireduce_scatter.c
deleted file mode 100644
index 83df2a87301..00000000000
--- a/ompi/mpi/c/ireduce_scatter.c
+++ /dev/null
@@ -1,142 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2016      IBM Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/op/op.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Ireduce_scatter = PMPI_Ireduce_scatter
-#endif
-#define MPI_Ireduce_scatter PMPI_Ireduce_scatter
-#endif
-
-static const char FUNC_NAME[] = "MPI_Ireduce_scatter";
-
-
-int MPI_Ireduce_scatter(const void *sendbuf, void *recvbuf, const int recvcounts[],
-                        MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, MPI_Request *request)
-{
-    int i, err, size, count;
-
-    SPC_RECORD(OMPI_SPC_IREDUCE_SCATTER, 1);
-
-    MEMCHECKER(
-        int rank;
-        int count;
-
-        size = ompi_comm_size(comm);
-        rank = ompi_comm_rank(comm);
-        for (count = i = 0; i < size; ++i) {
-            if (0 == recvcounts[i]) {
-                count += recvcounts[i];
-            }
-        }
-
-        memchecker_comm(comm);
-        memchecker_datatype(datatype);
-
-        /* check receive buffer of current proccess, whether it's addressable. */
-        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf,
-                        recvcounts[rank], datatype);
-
-        /* check whether the actual send buffer is defined. */
-        if(MPI_IN_PLACE == sendbuf) {
-            memchecker_call(&opal_memchecker_base_isdefined, recvbuf, count, datatype);
-        } else {
-            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
-
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        char *msg;
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-
-        /* Unrooted operation; same checks for all ranks on both
-           intracommunicators and intercommunicators */
-
-        else if (MPI_OP_NULL == op || NULL == op) {
-          err = MPI_ERR_OP;
-        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
-            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
-            free(msg);
-            return ret;
-        } else if (NULL == recvcounts) {
-          err = MPI_ERR_COUNT;
-        } else if (MPI_IN_PLACE == recvbuf) {
-          err = MPI_ERR_ARG;
-        }
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-
-        /* Based on the standard each group has to provide the same total
-           number of elements, so the size of the recvcounts array depends
-           on the number of participants in the local group.  */
-        size = ompi_comm_size(comm);
-        for (i = 0; i < size; ++i) {
-          OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, recvcounts[i]);
-          OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-    }
-
-    /* MPI standard says that reductions have to have a count of at least 1,
-     * but some benchmarks (e.g., IMB) calls this function with a count of 0.
-     * So handle that case.
-     */
-    size = ompi_comm_size(comm);
-    for (count = i = 0; i < size; ++i) {
-        if (0 == recvcounts[i]) {
-            ++count;
-        }
-    }
-    if (size == count) {
-        *request = &ompi_request_empty;
-        return MPI_SUCCESS;
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-
-    err = comm->c_coll->coll_ireduce_scatter(sendbuf, recvbuf, recvcounts,
-                                            datatype, op, comm, request,
-                                            comm->c_coll->coll_ireduce_scatter_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_op(*request, op, datatype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/ireduce_scatter.c.in b/ompi/mpi/c/ireduce_scatter.c.in
new file mode 100644
index 00000000000..7b47a7b7888
--- /dev/null
+++ b/ompi/mpi/c/ireduce_scatter.c.in
@@ -0,0 +1,136 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2016      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/op/op.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS ireduce_scatter(BUFFER sendbuf, BUFFER_OUT recvbuf, COUNT_ARRAY recvcounts,
+                                      DATATYPE datatype, OP op, COMM comm, REQUEST_INOUT request)
+{
+    int i, err, size, count;
+    ompi_count_array_t recvcounts_desc;
+
+    SPC_RECORD(OMPI_SPC_IREDUCE_SCATTER, 1);
+
+    MEMCHECKER(
+        int rank;
+        int count;
+
+        size = ompi_comm_size(comm);
+        rank = ompi_comm_rank(comm);
+        for (count = i = 0; i < size; ++i) {
+            if (0 == recvcounts[i]) {
+                count += recvcounts[i];
+            }
+        }
+
+        memchecker_comm(comm);
+        memchecker_datatype(datatype);
+
+        /* check receive buffer of current process, whether it's addressable. */
+        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf,
+                        recvcounts[rank], datatype);
+
+        /* check whether the actual send buffer is defined. */
+        if(MPI_IN_PLACE == sendbuf) {
+            memchecker_call(&opal_memchecker_base_isdefined, recvbuf, count, datatype);
+        } else {
+            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
+
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        char *msg;
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+
+        /* Unrooted operation; same checks for all ranks on both
+           intracommunicators and intercommunicators */
+
+        else if (MPI_OP_NULL == op || NULL == op) {
+          err = MPI_ERR_OP;
+        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
+            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
+            free(msg);
+            return ret;
+        } else if (NULL == recvcounts) {
+          err = MPI_ERR_COUNT;
+        } else if (MPI_IN_PLACE == recvbuf) {
+          err = MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+
+        /* Based on the standard each group has to provide the same total
+           number of elements, so the size of the recvcounts array depends
+           on the number of participants in the local group.  */
+        size = ompi_comm_size(comm);
+        for (i = 0; i < size; ++i) {
+          OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, recvcounts[i]);
+          OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+    }
+
+    /* MPI standard says that reductions have to have a count of at least 1,
+     * but some benchmarks (e.g., IMB) calls this function with a count of 0.
+     * So handle that case.
+     */
+    size = ompi_comm_size(comm);
+    for (count = i = 0; i < size; ++i) {
+        if (0 == recvcounts[i]) {
+            ++count;
+        }
+    }
+    if (size == count) {
+        *request = &ompi_request_empty;
+        return MPI_SUCCESS;
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+
+    OMPI_COUNT_ARRAY_INIT(&recvcounts_desc, recvcounts);
+    err = comm->c_coll->coll_ireduce_scatter(sendbuf, recvbuf, recvcounts_desc,
+                                            datatype, op, comm, request,
+                                            comm->c_coll->coll_ireduce_scatter_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_op(*request, op, datatype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/ireduce_scatter_block.c b/ompi/mpi/c/ireduce_scatter_block.c
deleted file mode 100644
index 41fa369524b..00000000000
--- a/ompi/mpi/c/ireduce_scatter_block.c
+++ /dev/null
@@ -1,114 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012      Oak Ridge National Labs. All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/op/op.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Ireduce_scatter_block = PMPI_Ireduce_scatter_block
-#endif
-#define MPI_Ireduce_scatter_block PMPI_Ireduce_scatter_block
-#endif
-
-static const char FUNC_NAME[] = "MPI_Ireduce_scatter_block";
-
-
-int MPI_Ireduce_scatter_block(const void *sendbuf, void *recvbuf, int recvcount,
-                              MPI_Datatype datatype, MPI_Op op,
-                              MPI_Comm comm, MPI_Request *request)
-{
-    int err;
-
-    SPC_RECORD(OMPI_SPC_IREDUCE_SCATTER_BLOCK, 1);
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-        memchecker_datatype(datatype);
-
-        /* check receive buffer of current proccess, whether it's addressable. */
-        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf,
-                        recvcount, datatype);
-
-        /* check whether the actual send buffer is defined. */
-        if(MPI_IN_PLACE == sendbuf) {
-            memchecker_call(&opal_memchecker_base_isdefined, recvbuf, recvcount, datatype);
-        } else {
-            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, recvcount, datatype);
-
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        char *msg;
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-
-        /* Unrooted operation; same checks for all ranks on both
-           intracommunicators and intercommunicators */
-
-        else if (MPI_OP_NULL == op || NULL == op) {
-          err = MPI_ERR_OP;
-        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
-            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
-            free(msg);
-            return ret;
-        } else if (MPI_IN_PLACE == recvbuf) {
-          err = MPI_ERR_ARG;
-        }
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-
-        OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, recvcount);
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-    }
-
-    if (0 == recvcount) {
-        *request = &ompi_request_empty;
-        return MPI_SUCCESS;
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-
-    err = comm->c_coll->coll_ireduce_scatter_block(sendbuf, recvbuf, recvcount,
-                                                  datatype, op, comm, request,
-                                                  comm->c_coll->coll_ireduce_scatter_block_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_op(*request, op, datatype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/ireduce_scatter_block.c.in b/ompi/mpi/c/ireduce_scatter_block.c.in
new file mode 100644
index 00000000000..d4d7e3e7361
--- /dev/null
+++ b/ompi/mpi/c/ireduce_scatter_block.c.in
@@ -0,0 +1,107 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oak Ridge National Labs. All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/op/op.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS ireduce_scatter_block(BUFFER sendbuf, BUFFER_OUT recvbuf, COUNT recvcount,
+                                            DATATYPE datatype, OP op,
+                                            COMM comm, REQUEST_INOUT request)
+{
+    int err;
+
+    SPC_RECORD(OMPI_SPC_IREDUCE_SCATTER_BLOCK, 1);
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+        memchecker_datatype(datatype);
+
+        /* check receive buffer of current process, whether it's addressable. */
+        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf,
+                        recvcount, datatype);
+
+        /* check whether the actual send buffer is defined. */
+        if(MPI_IN_PLACE == sendbuf) {
+            memchecker_call(&opal_memchecker_base_isdefined, recvbuf, recvcount, datatype);
+        } else {
+            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, recvcount, datatype);
+
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        char *msg;
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+
+        /* Unrooted operation; same checks for all ranks on both
+           intracommunicators and intercommunicators */
+
+        else if (MPI_OP_NULL == op || NULL == op) {
+          err = MPI_ERR_OP;
+        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
+            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
+            free(msg);
+            return ret;
+        } else if (MPI_IN_PLACE == recvbuf) {
+          err = MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+
+        OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, recvcount);
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+    }
+
+    if (0 == recvcount) {
+        *request = &ompi_request_empty;
+        return MPI_SUCCESS;
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+
+    err = comm->c_coll->coll_ireduce_scatter_block(sendbuf, recvbuf, recvcount,
+                                                  datatype, op, comm, request,
+                                                  comm->c_coll->coll_ireduce_scatter_block_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_op(*request, op, datatype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/irsend.c b/ompi/mpi/c/irsend.c
deleted file mode 100644
index 4db6cf39e3f..00000000000
--- a/ompi/mpi/c/irsend.c
+++ /dev/null
@@ -1,101 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/request/request.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Irsend = PMPI_Irsend
-#endif
-#define MPI_Irsend PMPI_Irsend
-#endif
-
-static const char FUNC_NAME[] = "MPI_Irsend";
-
-
-int MPI_Irsend(const void *buf, int count, MPI_Datatype type, int dest,
-               int tag, MPI_Comm comm, MPI_Request *request)
-{
-    int rc;
-
-    SPC_RECORD(OMPI_SPC_IRSEND, 1);
-
-    MEMCHECKER(
-        memchecker_datatype(type);
-        memchecker_call(&opal_memchecker_base_isdefined, buf, count, type);
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else if (tag < 0 || tag > mca_pml.pml_max_tag) {
-            rc = MPI_ERR_TAG;
-        } else if (ompi_comm_peer_invalid(comm, dest) &&
-                   (MPI_PROC_NULL != dest)) {
-            rc = MPI_ERR_RANK;
-        } else if (request == NULL) {
-            rc = MPI_ERR_REQUEST;
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_SEND(rc, type, count);
-            OMPI_CHECK_USER_BUFFER(rc, buf, type, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-    }
-
-    if (MPI_PROC_NULL == dest) {
-        *request = &ompi_request_empty;
-        return MPI_SUCCESS;
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * The request will be checked for process failure errors during the
-     * completion calls. So no need to check here.
-     */
-#endif
-
-    MEMCHECKER (
-        memchecker_call(&opal_memchecker_base_mem_noaccess, buf, count, type);
-    );
-    rc = MCA_PML_CALL(isend(buf,count,type,dest,tag,
-                            MCA_PML_BASE_SEND_READY,comm,request));
-    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-}
-
-
diff --git a/ompi/mpi/c/irsend.c.in b/ompi/mpi/c/irsend.c.in
new file mode 100644
index 00000000000..777d8feec29
--- /dev/null
+++ b/ompi/mpi/c/irsend.c.in
@@ -0,0 +1,93 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/request/request.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS Irsend(BUFFER buf, COUNT count, DATATYPE type, INT dest,
+               INT tag, COMM comm, REQUEST_INOUT request)
+{
+    int rc;
+
+    SPC_RECORD(OMPI_SPC_IRSEND, 1);
+
+    MEMCHECKER(
+        memchecker_datatype(type);
+        memchecker_call(&opal_memchecker_base_isdefined, buf, count, type);
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else if (tag < 0 || tag > mca_pml.pml_max_tag) {
+            rc = MPI_ERR_TAG;
+        } else if (ompi_comm_peer_invalid(comm, dest) &&
+                   (MPI_PROC_NULL != dest)) {
+            rc = MPI_ERR_RANK;
+        } else if (request == NULL) {
+            rc = MPI_ERR_REQUEST;
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_SEND(rc, type, count);
+            OMPI_CHECK_USER_BUFFER(rc, buf, type, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+    if (MPI_PROC_NULL == dest) {
+        *request = &ompi_request_empty;
+        return MPI_SUCCESS;
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * The request will be checked for process failure errors during the
+     * completion calls. So no need to check here.
+     */
+#endif
+
+    MEMCHECKER (
+        memchecker_call(&opal_memchecker_base_mem_noaccess, buf, count, type);
+    );
+    rc = MCA_PML_CALL(isend(buf,count,type,dest,tag,
+                            MCA_PML_BASE_SEND_READY,comm,request));
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
+
+
diff --git a/ompi/mpi/c/is_thread_main.c b/ompi/mpi/c/is_thread_main.c
deleted file mode 100644
index b876dff4661..00000000000
--- a/ompi/mpi/c/is_thread_main.c
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/runtime/mpiruntime.h"
-#include "opal/mca/threads/threads.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Is_thread_main = PMPI_Is_thread_main
-#endif
-#define MPI_Is_thread_main PMPI_Is_thread_main
-#endif
-
-static const char FUNC_NAME[] = "MPI_Is_thread_main";
-
-
-int MPI_Is_thread_main(int *flag)
-{
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == flag) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
-        }
-    }
-
-    /* Compare this thread ID to the main thread ID */
-
-    *flag = (int) opal_thread_self_compare(ompi_mpi_main_thread);
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/is_thread_main.c.in b/ompi/mpi/c/is_thread_main.c.in
new file mode 100644
index 00000000000..5e80109f2b4
--- /dev/null
+++ b/ompi/mpi/c/is_thread_main.c.in
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/runtime/mpiruntime.h"
+#include "opal/mca/threads/threads.h"
+
+PROTOTYPE ERROR_CLASS is_thread_main(INT_OUT flag)
+{
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == flag) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+        }
+    }
+
+    /* Compare this thread ID to the main thread ID */
+
+    *flag = (int) opal_thread_self_compare(ompi_mpi_main_thread);
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/iscan.c b/ompi/mpi/c/iscan.c
deleted file mode 100644
index 165add74124..00000000000
--- a/ompi/mpi/c/iscan.c
+++ /dev/null
@@ -1,106 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/op/op.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Iscan = PMPI_Iscan
-#endif
-#define MPI_Iscan PMPI_Iscan
-#endif
-
-static const char FUNC_NAME[] = "MPI_Iscan";
-
-
-int MPI_Iscan(const void *sendbuf, void *recvbuf, int count,
-              MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, MPI_Request *request)
-{
-    int err;
-
-    SPC_RECORD(OMPI_SPC_ISCAN, 1);
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_comm(comm);
-        if (MPI_IN_PLACE != sendbuf) {
-            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
-        } else {
-            memchecker_call(&opal_memchecker_base_isdefined, recvbuf, count, datatype);
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        char *msg;
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-
-        /* No intercommunicators allowed! (MPI does not define
-           MPI_SCAN on intercommunicators) */
-
-        else if (OMPI_COMM_IS_INTER(comm)) {
-          err = MPI_ERR_COMM;
-        }
-
-        /* Unrooted operation; checks for all ranks */
-
-        else if (MPI_OP_NULL == op || NULL == op) {
-          err = MPI_ERR_OP;
-        } else if (MPI_IN_PLACE == recvbuf) {
-          err = MPI_ERR_ARG;
-        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
-            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
-            free(msg);
-            return ret;
-        } else {
-          OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-    }
-
-    /* Call the coll component to actually perform the allgather */
-
-    err = comm->c_coll->coll_iscan(sendbuf, recvbuf, count,
-                                  datatype, op, comm,
-                                  request,
-                                  comm->c_coll->coll_iscan_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_op(*request, op, datatype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/iscan.c.in b/ompi/mpi/c/iscan.c.in
new file mode 100644
index 00000000000..6d2c1d950ec
--- /dev/null
+++ b/ompi/mpi/c/iscan.c.in
@@ -0,0 +1,98 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/op/op.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS iscan(BUFFER sendbuf, BUFFER_OUT recvbuf, COUNT count,
+                            DATATYPE datatype, OP op, COMM comm, REQUEST_INOUT request)
+{
+    int err;
+
+    SPC_RECORD(OMPI_SPC_ISCAN, 1);
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_comm(comm);
+        if (MPI_IN_PLACE != sendbuf) {
+            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
+        } else {
+            memchecker_call(&opal_memchecker_base_isdefined, recvbuf, count, datatype);
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        char *msg;
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+
+        /* No intercommunicators allowed! (MPI does not define
+           MPI_SCAN on intercommunicators) */
+
+        else if (OMPI_COMM_IS_INTER(comm)) {
+          err = MPI_ERR_COMM;
+        }
+
+        /* Unrooted operation; checks for all ranks */
+
+        else if (MPI_OP_NULL == op || NULL == op) {
+          err = MPI_ERR_OP;
+        } else if (MPI_IN_PLACE == recvbuf) {
+          err = MPI_ERR_ARG;
+        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
+            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
+            free(msg);
+            return ret;
+        } else {
+          OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+    }
+
+    /* Call the coll component to actually perform the allgather */
+
+    err = comm->c_coll->coll_iscan(sendbuf, recvbuf, count,
+                                  datatype, op, comm,
+                                  request,
+                                  comm->c_coll->coll_iscan_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_op(*request, op, datatype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/iscatter.c b/ompi/mpi/c/iscatter.c
deleted file mode 100644
index e14fb03de48..00000000000
--- a/ompi/mpi/c/iscatter.c
+++ /dev/null
@@ -1,178 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2008      University of Houston.  All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Iscatter = PMPI_Iscatter
-#endif
-#define MPI_Iscatter PMPI_Iscatter
-#endif
-
-static const char FUNC_NAME[] = "MPI_Iscatter";
-
-
-int MPI_Iscatter(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                 void *recvbuf, int recvcount, MPI_Datatype recvtype,
-                 int root, MPI_Comm comm, MPI_Request *request)
-{
-    int err;
-
-    SPC_RECORD(OMPI_SPC_ISCATTER, 1);
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-        if(OMPI_COMM_IS_INTRA(comm)) {
-            if(ompi_comm_rank(comm) == root) {
-                memchecker_datatype(sendtype);
-                /* check whether root's send buffer is defined. */
-                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-                if(MPI_IN_PLACE != recvbuf) {
-                    memchecker_datatype(recvtype);
-                    /* check whether receive buffer is addressable. */
-                    memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-                }
-            } else {
-                memchecker_datatype(recvtype);
-                /* check whether receive buffer is addressable. */
-                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-            }
-        } else {
-            if(MPI_ROOT == root) {
-                memchecker_datatype(sendtype);
-                /* check whether root's send buffer is defined. */
-                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-            } else if (MPI_PROC_NULL != root) {
-                memchecker_datatype(recvtype);
-                /* check whether receive buffer is addressable. */
-                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-            }
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if ((ompi_comm_rank(comm) != root && MPI_IN_PLACE == recvbuf) ||
-                   (ompi_comm_rank(comm) == root && MPI_IN_PLACE == sendbuf)) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-
-        /* Errors for intracommunicators */
-
-        if (OMPI_COMM_IS_INTRA(comm)) {
-
-          /* Errors for all ranks */
-
-          if ((root >= ompi_comm_size(comm)) || (root < 0)) {
-              err = MPI_ERR_ROOT;
-          } else if (MPI_IN_PLACE != recvbuf) {
-              if (recvcount < 0) {
-                  err = MPI_ERR_COUNT;
-              } else if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
-                  err = MPI_ERR_TYPE;
-              }
-          }
-
-          /* Errors for the root.  Some of these could have been
-             combined into compound if statements above, but since
-             this whole section can be compiled out (or turned off at
-             run time) for efficiency, it's more clear to separate
-             them out into individual tests. */
-
-          else if (ompi_comm_rank(comm) == root) {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-          }
-          OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-
-        /* Errors for intercommunicators */
-
-        else {
-          if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
-                 MPI_ROOT == root || MPI_PROC_NULL == root)) {
-            err = MPI_ERR_ROOT;
-          }
-
-          /* Errors for the receivers */
-
-          else if (MPI_ROOT != root && MPI_PROC_NULL != root) {
-            if (recvcount < 0) {
-              err = MPI_ERR_COUNT;
-            } else if (MPI_DATATYPE_NULL == recvtype) {
-              err = MPI_ERR_TYPE;
-            }
-          }
-
-          /* Errors for the root.  Ditto on the comment above -- these
-             error checks could have been combined above, but let's
-             make the code easier to read. */
-
-          else if (MPI_ROOT == root) {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-          }
-          OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_iscatter(sendbuf, sendcount, sendtype, recvbuf,
-                                     recvcount, recvtype, root, comm, request,
-                                     comm->c_coll->coll_iscatter_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        if (OMPI_COMM_IS_INTRA(comm)) {
-            if (MPI_IN_PLACE == recvbuf) {
-                recvtype = NULL;
-            } else if (ompi_comm_rank(comm) != root) {
-                sendtype = NULL;
-            }
-        } else {
-            if (MPI_ROOT == root) {
-                recvtype = NULL;
-            } else if (MPI_PROC_NULL == root) {
-                sendtype = NULL;
-                recvtype = NULL;
-            } else {
-                sendtype = NULL;
-            }
-        }
-        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/iscatter.c.in b/ompi/mpi/c/iscatter.c.in
new file mode 100644
index 00000000000..b5450ba9f6f
--- /dev/null
+++ b/ompi/mpi/c/iscatter.c.in
@@ -0,0 +1,180 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2023 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2008      University of Houston.  All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS iscatter(BUFFER sendbuf, COUNT sendcount, DATATYPE sendtype,
+                               BUFFER_OUT recvbuf, COUNT recvcount, DATATYPE recvtype,
+                               INT root, COMM comm, REQUEST_INOUT request)
+{
+    int err;
+
+    SPC_RECORD(OMPI_SPC_ISCATTER, 1);
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+        if(OMPI_COMM_IS_INTRA(comm)) {
+            if(ompi_comm_rank(comm) == root) {
+                memchecker_datatype(sendtype);
+                /* check whether root's send buffer is defined. */
+                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+                if(MPI_IN_PLACE != recvbuf) {
+                    memchecker_datatype(recvtype);
+                    /* check whether receive buffer is addressable. */
+                    memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+                }
+            } else {
+                memchecker_datatype(recvtype);
+                /* check whether receive buffer is addressable. */
+                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+            }
+        } else {
+            if(MPI_ROOT == root) {
+                memchecker_datatype(sendtype);
+                /* check whether root's send buffer is defined. */
+                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+            } else if (MPI_PROC_NULL != root) {
+                memchecker_datatype(recvtype);
+                /* check whether receive buffer is addressable. */
+                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+            }
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if ((ompi_comm_rank(comm) != root && MPI_IN_PLACE == recvbuf) ||
+                   (ompi_comm_rank(comm) == root && MPI_IN_PLACE == sendbuf)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+
+        /* Errors for intracommunicators */
+
+        if (OMPI_COMM_IS_INTRA(comm)) {
+
+          /* Errors for all ranks */
+
+          if ((root >= ompi_comm_size(comm)) || (root < 0)) {
+              err = MPI_ERR_ROOT;
+          } else if (MPI_IN_PLACE != recvbuf) {
+              if (recvcount < 0) {
+                  err = MPI_ERR_COUNT;
+              } else if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
+                  err = MPI_ERR_TYPE;
+              }
+          }
+
+          /* Errors for the root.  Some of these could have been
+             combined into compound if statements above, but since
+             this whole section can be compiled out (or turned off at
+             run time) for efficiency, it's more clear to separate
+             them out into individual tests. */
+
+          else if (ompi_comm_rank(comm) == root) {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+          }
+          OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+
+        /* Errors for intercommunicators */
+
+        else {
+          if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
+                 MPI_ROOT == root || MPI_PROC_NULL == root)) {
+            err = MPI_ERR_ROOT;
+          }
+
+          /* Errors for the receivers */
+
+          else if (MPI_ROOT != root && MPI_PROC_NULL != root) {
+            if (recvcount < 0) {
+              err = MPI_ERR_COUNT;
+            } else if (MPI_DATATYPE_NULL == recvtype) {
+              err = MPI_ERR_TYPE;
+            }
+          }
+
+          /* Errors for the root.  Ditto on the comment above -- these
+             error checks could have been combined above, but let's
+             make the code easier to read. */
+
+          else if (MPI_ROOT == root) {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+          }
+          OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+    }
+
+    const void *updated_sendbuf;
+    void *updated_recvbuf;
+    if (OMPI_COMM_IS_INTRA(comm)) {
+        updated_sendbuf = (ompi_comm_rank(comm) != root) ? NULL : sendbuf;
+        updated_recvbuf = recvbuf;
+    } else {
+        updated_sendbuf = (MPI_ROOT != root ) ? NULL : sendbuf;
+        updated_recvbuf = ((MPI_ROOT == root) || (MPI_PROC_NULL == root)) ? NULL : recvbuf;
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    err = comm->c_coll->coll_iscatter(updated_sendbuf, sendcount, sendtype, updated_recvbuf,
+                                      recvcount, recvtype, root, comm, request,
+                                      comm->c_coll->coll_iscatter_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        if (OMPI_COMM_IS_INTRA(comm)) {
+            if (MPI_IN_PLACE == recvbuf) {
+                recvtype = NULL;
+            } else if (ompi_comm_rank(comm) != root) {
+                sendtype = NULL;
+            }
+        } else {
+            if (MPI_ROOT == root) {
+                recvtype = NULL;
+            } else if (MPI_PROC_NULL == root) {
+                sendtype = NULL;
+                recvtype = NULL;
+            } else {
+                sendtype = NULL;
+            }
+        }
+        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/iscatterv.c b/ompi/mpi/c/iscatterv.c
deleted file mode 100644
index 969fcd1fd55..00000000000
--- a/ompi/mpi/c/iscatterv.c
+++ /dev/null
@@ -1,218 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Iscatterv = PMPI_Iscatterv
-#endif
-#define MPI_Iscatterv PMPI_Iscatterv
-#endif
-
-static const char FUNC_NAME[] = "MPI_Iscatterv";
-
-
-int MPI_Iscatterv(const void *sendbuf, const int sendcounts[], const int displs[],
-                  MPI_Datatype sendtype, void *recvbuf, int recvcount,
-                  MPI_Datatype recvtype, int root, MPI_Comm comm, MPI_Request *request)
-{
-    int i, size, err;
-
-    SPC_RECORD(OMPI_SPC_ISCATTERV, 1);
-
-    MEMCHECKER(
-        ptrdiff_t ext;
-
-        size = ompi_comm_remote_size(comm);
-        ompi_datatype_type_extent(recvtype, &ext);
-
-        memchecker_comm(comm);
-        if(OMPI_COMM_IS_INTRA(comm)) {
-              if(ompi_comm_rank(comm) == root) {
-                memchecker_datatype(sendtype);
-                /* check whether root's send buffer is defined. */
-                for (i = 0; i < size; i++) {
-                    memchecker_call(&opal_memchecker_base_isdefined,
-                                    (char *)(sendbuf)+displs[i]*ext,
-                                    sendcounts[i], sendtype);
-                }
-                if(MPI_IN_PLACE != recvbuf) {
-                    memchecker_datatype(recvtype);
-                    /* check whether receive buffer is addressable. */
-                    memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-                }
-              } else {
-                  memchecker_datatype(recvtype);
-                  /* check whether receive buffer is addressable. */
-                  memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-              }
-        } else {
-            if(MPI_ROOT == root) {
-                  memchecker_datatype(sendtype);
-                  /* check whether root's send buffer is defined. */
-                  for (i = 0; i < size; i++) {
-                      memchecker_call(&opal_memchecker_base_isdefined,
-                                      (char *)(sendbuf)+displs[i]*ext,
-                                      sendcounts[i], sendtype);
-                  }
-            } else if (MPI_PROC_NULL != root) {
-                /* check whether receive buffer is addressable. */
-                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-            }
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if ((ompi_comm_rank(comm) != root && MPI_IN_PLACE == recvbuf) ||
-                   (ompi_comm_rank(comm) == root && MPI_IN_PLACE == sendbuf)) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-
-        /* Errors for intracommunicators */
-
-        if (OMPI_COMM_IS_INTRA(comm)) {
-
-            /* Errors for all ranks */
-
-            if ((root >= ompi_comm_size(comm)) || (root < 0)) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
-            }
-
-            if (MPI_IN_PLACE != recvbuf) {
-                if (recvcount < 0) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT,
-                                                  FUNC_NAME);
-                }
-
-                if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE,
-                                                  FUNC_NAME);
-                }
-            }
-
-            /* Errors for the root.  Some of these could have been
-               combined into compound if statements above, but since
-               this whole section can be compiled out (or turned off at
-               run time) for efficiency, it's more clear to separate
-               them out into individual tests. */
-
-            if (ompi_comm_rank(comm) == root) {
-                if (NULL == displs) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-                }
-
-                if (NULL == sendcounts) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-                }
-
-                size = ompi_comm_size(comm);
-                for (i = 0; i < size; ++i) {
-                    OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcounts[i]);
-                    OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-                }
-            }
-        }
-
-        /* Errors for intercommunicators */
-
-        else {
-            if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
-                   MPI_ROOT == root || MPI_PROC_NULL == root)) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
-            }
-
-          /* Errors for the receivers */
-
-            if (MPI_ROOT != root && MPI_PROC_NULL != root) {
-                if (recvcount < 0) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-                }
-
-                if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
-                }
-            }
-
-            /* Errors for the root.  Ditto on the comment above -- these
-               error checks could have been combined above, but let's
-               make the code easier to read. */
-
-            else if (MPI_ROOT == root) {
-                if (NULL == displs) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-                }
-
-                if (NULL == sendcounts) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-                }
-
-                size = ompi_comm_remote_size(comm);
-                for (i = 0; i < size; ++i) {
-                    OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcounts[i]);
-                    OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-                }
-            }
-        }
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_iscatterv(sendbuf, sendcounts, displs,
-                                      sendtype, recvbuf, recvcount, recvtype, root, comm,
-                                      request, comm->c_coll->coll_iscatterv_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        if (OMPI_COMM_IS_INTRA(comm)) {
-            if (MPI_IN_PLACE == recvbuf) {
-                recvtype = NULL;
-            } else if (ompi_comm_rank(comm) != root) {
-                sendtype = NULL;
-            }
-        } else {
-            if (MPI_ROOT == root) {
-                recvtype = NULL;
-            } else if (MPI_PROC_NULL == root) {
-                sendtype = NULL;
-                recvtype = NULL;
-            } else {
-                sendtype = NULL;
-            }
-        }
-        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/iscatterv.c.in b/ompi/mpi/c/iscatterv.c.in
new file mode 100644
index 00000000000..0bd4bb96065
--- /dev/null
+++ b/ompi/mpi/c/iscatterv.c.in
@@ -0,0 +1,225 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2023 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS iscatterv(BUFFER sendbuf, COUNT_ARRAY sendcounts, DISP_ARRAY displs,
+                                DATATYPE sendtype, BUFFER_OUT recvbuf, COUNT recvcount,
+                                DATATYPE recvtype, INT root, COMM comm, REQUEST_INOUT request)
+{
+    int i, size, err;
+    ompi_count_array_t sendcounts_desc;
+    ompi_disp_array_t displs_desc;
+
+    SPC_RECORD(OMPI_SPC_ISCATTERV, 1);
+
+    MEMCHECKER(
+        ptrdiff_t ext;
+
+        size = ompi_comm_remote_size(comm);
+
+        memchecker_comm(comm);
+        if(OMPI_COMM_IS_INTRA(comm)) {
+              if(ompi_comm_rank(comm) == root) {
+                memchecker_datatype(sendtype);
+                ompi_datatype_type_extent(sendtype, &ext);
+                /* check whether root's send buffer is defined. */
+                for (i = 0; i < size; i++) {
+                    memchecker_call(&opal_memchecker_base_isdefined,
+                                    (char *)(sendbuf)+displs[i]*ext,
+                                    sendcounts[i], sendtype);
+                }
+                if(MPI_IN_PLACE != recvbuf) {
+                    memchecker_datatype(recvtype);
+                    /* check whether receive buffer is addressable. */
+                    memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+                }
+              } else {
+                  memchecker_datatype(recvtype);
+                  /* check whether receive buffer is addressable. */
+                  memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+              }
+        } else {
+            if(MPI_ROOT == root) {
+                  memchecker_datatype(sendtype);
+                  ompi_datatype_type_extent(sendtype, &ext);
+                  /* check whether root's send buffer is defined. */
+                  for (i = 0; i < size; i++) {
+                      memchecker_call(&opal_memchecker_base_isdefined,
+                                      (char *)(sendbuf)+displs[i]*ext,
+                                      sendcounts[i], sendtype);
+                  }
+            } else if (MPI_PROC_NULL != root) {
+                /* check whether receive buffer is addressable. */
+                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+            }
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if ((ompi_comm_rank(comm) != root && MPI_IN_PLACE == recvbuf) ||
+                   (ompi_comm_rank(comm) == root && MPI_IN_PLACE == sendbuf)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+
+        /* Errors for intracommunicators */
+
+        if (OMPI_COMM_IS_INTRA(comm)) {
+
+            /* Errors for all ranks */
+
+            if ((root >= ompi_comm_size(comm)) || (root < 0)) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
+            }
+
+            if (MPI_IN_PLACE != recvbuf) {
+                if (recvcount < 0) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT,
+                                                  FUNC_NAME);
+                }
+
+                if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE,
+                                                  FUNC_NAME);
+                }
+            }
+
+            /* Errors for the root.  Some of these could have been
+               combined into compound if statements above, but since
+               this whole section can be compiled out (or turned off at
+               run time) for efficiency, it's more clear to separate
+               them out into individual tests. */
+
+            if (ompi_comm_rank(comm) == root) {
+                if (NULL == displs) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+                }
+
+                if (NULL == sendcounts) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+                }
+
+                size = ompi_comm_size(comm);
+                for (i = 0; i < size; ++i) {
+                    OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcounts[i]);
+                    OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+                }
+            }
+        }
+
+        /* Errors for intercommunicators */
+
+        else {
+            if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
+                   MPI_ROOT == root || MPI_PROC_NULL == root)) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
+            }
+
+          /* Errors for the receivers */
+
+            if (MPI_ROOT != root && MPI_PROC_NULL != root) {
+                if (recvcount < 0) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+                }
+
+                if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
+                }
+            }
+
+            /* Errors for the root.  Ditto on the comment above -- these
+               error checks could have been combined above, but let's
+               make the code easier to read. */
+
+            else if (MPI_ROOT == root) {
+                if (NULL == displs) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+                }
+
+                if (NULL == sendcounts) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+                }
+
+                size = ompi_comm_remote_size(comm);
+                for (i = 0; i < size; ++i) {
+                    OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcounts[i]);
+                    OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+                }
+            }
+        }
+    }
+
+    const void *updated_sendbuf;
+    void *updated_recvbuf;
+    if (OMPI_COMM_IS_INTRA(comm)) {
+        updated_sendbuf = (ompi_comm_rank(comm) != root) ? NULL : sendbuf;
+        updated_recvbuf = recvbuf;
+    } else {
+        updated_sendbuf = (MPI_ROOT != root ) ? NULL : sendbuf;
+        updated_recvbuf = ((MPI_ROOT == root) || (MPI_PROC_NULL == root)) ? NULL : recvbuf;
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    OMPI_COUNT_ARRAY_INIT(&sendcounts_desc, sendcounts);
+    OMPI_DISP_ARRAY_INIT(&displs_desc, displs);
+    err = comm->c_coll->coll_iscatterv(updated_sendbuf, sendcounts_desc, displs_desc,
+                                       sendtype, updated_recvbuf, recvcount, recvtype, root, comm,
+                                       request, comm->c_coll->coll_iscatterv_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        if (OMPI_COMM_IS_INTRA(comm)) {
+            if (MPI_IN_PLACE == recvbuf) {
+                recvtype = NULL;
+            } else if (ompi_comm_rank(comm) != root) {
+                sendtype = NULL;
+            }
+        } else {
+            if (MPI_ROOT == root) {
+                recvtype = NULL;
+            } else if (MPI_PROC_NULL == root) {
+                sendtype = NULL;
+                recvtype = NULL;
+            } else {
+                sendtype = NULL;
+            }
+        }
+        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/isend.c b/ompi/mpi/c/isend.c
deleted file mode 100644
index e7408e39ec1..00000000000
--- a/ompi/mpi/c/isend.c
+++ /dev/null
@@ -1,106 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2017 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/request/request.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Isend = PMPI_Isend
-#endif
-#define MPI_Isend PMPI_Isend
-#endif
-
-static const char FUNC_NAME[] = "MPI_Isend";
-
-
-int MPI_Isend(const void *buf, int count, MPI_Datatype type, int dest,
-              int tag, MPI_Comm comm, MPI_Request *request)
-{
-    int rc = MPI_SUCCESS;
-
-    SPC_RECORD(OMPI_SPC_ISEND, 1);
-
-    MEMCHECKER(
-        memchecker_datatype(type);
-        memchecker_call(&opal_memchecker_base_isdefined, buf, count, type);
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else if (MPI_DATATYPE_NULL == type || NULL == type) {
-            rc = MPI_ERR_TYPE;
-        } else if (tag < 0 || tag > mca_pml.pml_max_tag) {
-            rc = MPI_ERR_TAG;
-        } else if (ompi_comm_peer_invalid(comm, dest) &&
-                   (MPI_PROC_NULL != dest)) {
-            rc = MPI_ERR_RANK;
-        } else if (request == NULL) {
-            rc = MPI_ERR_REQUEST;
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_SEND(rc, type, count);
-            OMPI_CHECK_USER_BUFFER(rc, buf, type, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-    }
-
-    if (MPI_PROC_NULL == dest) {
-        *request = &ompi_request_empty;
-        return MPI_SUCCESS;
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * The request will be checked for process failure errors during the
-     * completion calls. So no need to check here.
-     */
-#endif
-
-    /*
-     * today's MPI standard mandates the send buffer remains accessible during the send operation
-     * hence memchecker cannot mark buf as non accessible, but it might mark buf as read-only in
-     * order to trap end user errors. Unfortunatly valgrind does not support marking buffers as read-only,
-     * so there is pretty much nothing we can do here.
-     */
-
-    rc = MCA_PML_CALL(isend(buf, count, type, dest, tag,
-                            MCA_PML_BASE_SEND_STANDARD, comm, request));
-    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-}
-
-
diff --git a/ompi/mpi/c/isend.c.in b/ompi/mpi/c/isend.c.in
new file mode 100644
index 00000000000..1be898b1b3d
--- /dev/null
+++ b/ompi/mpi/c/isend.c.in
@@ -0,0 +1,98 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2017 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/request/request.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS isend(BUFFER buf, COUNT count, DATATYPE type, INT dest,
+                            INT tag, COMM comm, REQUEST_INOUT request)
+{
+    int rc = MPI_SUCCESS;
+
+    SPC_RECORD(OMPI_SPC_ISEND, 1);
+
+    MEMCHECKER(
+        memchecker_datatype(type);
+        memchecker_call(&opal_memchecker_base_isdefined, buf, count, type);
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else if (MPI_DATATYPE_NULL == type || NULL == type) {
+            rc = MPI_ERR_TYPE;
+        } else if (tag < 0 || tag > mca_pml.pml_max_tag) {
+            rc = MPI_ERR_TAG;
+        } else if (ompi_comm_peer_invalid(comm, dest) &&
+                   (MPI_PROC_NULL != dest)) {
+            rc = MPI_ERR_RANK;
+        } else if (request == NULL) {
+            rc = MPI_ERR_REQUEST;
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_SEND(rc, type, count);
+            OMPI_CHECK_USER_BUFFER(rc, buf, type, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+    if (MPI_PROC_NULL == dest) {
+        *request = &ompi_request_empty;
+        return MPI_SUCCESS;
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * The request will be checked for process failure errors during the
+     * completion calls. So no need to check here.
+     */
+#endif
+
+    /*
+     * today's MPI standard mandates the send buffer remains accessible during the send operation
+     * hence memchecker cannot mark buf as non accessible, but it might mark buf as read-only in
+     * order to trap end user errors. Unfortunately valgrind does not support marking buffers as read-only,
+     * so there is pretty much nothing we can do here.
+     */
+
+    rc = MCA_PML_CALL(isend(buf, count, type, dest, tag,
+                            MCA_PML_BASE_SEND_STANDARD, comm, request));
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
+
+
diff --git a/ompi/mpi/c/isendrecv.c.in b/ompi/mpi/c/isendrecv.c.in
new file mode 100644
index 00000000000..39f410b6796
--- /dev/null
+++ b/ompi/mpi/c/isendrecv.c.in
@@ -0,0 +1,188 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2022 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2021-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/communicator/comm_request.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/request/request.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+
+struct ompi_isendrecv_context_t {
+    opal_object_t super;
+    int nreqs;
+    int source;
+    ompi_request_t *subreq[2];
+};
+
+typedef struct ompi_isendrecv_context_t ompi_isendrecv_context_t;
+#if OMPI_BUILD_MPI_PROFILING
+OBJ_CLASS_INSTANCE(ompi_isendrecv_context_t, opal_object_t, NULL, NULL);
+#else
+OBJ_CLASS_DECLARATION(ompi_isendrecv_context_t);
+#endif  /* OMPI_BUILD_MPI_PROFILING */
+
+static int ompi_isendrecv_complete_func (ompi_comm_request_t *request)
+{
+    ompi_isendrecv_context_t *context =
+        (ompi_isendrecv_context_t *) request->context;
+
+    /*
+     * Copy the status from the receive side of the sendrecv request?
+     * But what if the send failed?
+     *
+     * Probably need to bring up in the MPI forum.  
+     */
+
+    if (MPI_PROC_NULL != context->source) {
+        OMPI_COPY_STATUS(&request->super.req_status, 
+                         context->subreq[0]->req_status, false);
+    } else {
+        OMPI_COPY_STATUS(&request->super.req_status, 
+                         ompi_request_empty.req_status, false);
+    }
+
+    if(NULL != context->subreq[0]) {
+        ompi_request_free(&context->subreq[0]);
+    }
+    if(NULL != context->subreq[1]) {
+        ompi_request_free(&context->subreq[1]);
+    }
+
+    return OMPI_SUCCESS;
+}
+
+PROTOTYPE ERROR_CLASS isendrecv(BUFFER sendbuf, COUNT sendcount, DATATYPE sendtype,
+                                INT dest, INT sendtag, BUFFER_OUT recvbuf, COUNT recvcount,
+                                DATATYPE recvtype, INT source, INT recvtag,
+                                COMM comm,  REQUEST_INOUT request)
+{
+    ompi_isendrecv_context_t *context = NULL;
+    ompi_comm_request_t *crequest;
+    int rc = MPI_SUCCESS;
+    int nreqs = 0;
+    uint32_t flags;
+
+    SPC_RECORD(OMPI_SPC_ISENDRECV, 1);
+
+    MEMCHECKER(
+        memchecker_datatype(sendtype);
+        memchecker_datatype(recvtype);
+        memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        OMPI_CHECK_DATATYPE_FOR_SEND(rc, sendtype, sendcount);
+        OMPI_CHECK_DATATYPE_FOR_RECV(rc, recvtype, recvcount);
+        OMPI_CHECK_USER_BUFFER(rc, sendbuf, sendtype, sendcount);
+        OMPI_CHECK_USER_BUFFER(rc, recvbuf, recvtype, recvcount);
+
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+        } else if (dest != MPI_PROC_NULL && ompi_comm_peer_invalid(comm, dest)) {
+            rc = MPI_ERR_RANK;
+        } else if (sendtag < 0 || sendtag > mca_pml.pml_max_tag) {
+            rc = MPI_ERR_TAG;
+        } else if (source != MPI_PROC_NULL && source != MPI_ANY_SOURCE && ompi_comm_peer_invalid(comm, source)) {
+            rc = MPI_ERR_RANK;
+        } else if (((recvtag < 0) && (recvtag !=  MPI_ANY_TAG)) || (recvtag > mca_pml.pml_max_tag)) {
+            rc = MPI_ERR_TAG;
+        } else if (request == NULL) {
+            rc = MPI_ERR_REQUEST;
+        }
+
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+    crequest = ompi_comm_request_get ();
+    if (NULL == crequest) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    context = OBJ_NEW(ompi_isendrecv_context_t);
+    if (NULL == context) {
+        ompi_comm_request_return (crequest);
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    crequest->context = &context->super;
+    context->subreq[0] = MPI_REQUEST_NULL;
+    context->subreq[1] = MPI_REQUEST_NULL;
+    context->source = source;
+
+    if (source != MPI_PROC_NULL) { /* post recv */
+        rc = MCA_PML_CALL(irecv(recvbuf, recvcount, recvtype,
+                                source, recvtag, comm, &context->subreq[nreqs++]));
+        if (MPI_SUCCESS != rc) {
+            OBJ_RELEASE(context);
+            ompi_comm_request_return (crequest);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+    if (dest != MPI_PROC_NULL) { /* send */
+        rc = MCA_PML_CALL(isend(sendbuf, sendcount, sendtype, dest,
+                                sendtag, MCA_PML_BASE_SEND_STANDARD, comm, &context->subreq[nreqs++]));
+        if (MPI_SUCCESS != rc) {
+            OBJ_RELEASE(context);
+            ompi_comm_request_return (crequest);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+    /*
+     *  schedule the operation
+     */
+
+    context->nreqs = nreqs;
+    assert(nreqs <= 2);
+
+    flags = OMPI_COMM_REQ_FLAG_RETAIN_SUBREQ;
+
+    rc = ompi_comm_request_schedule_append_w_flags(crequest, ompi_isendrecv_complete_func, 
+                                                   context->subreq, nreqs, flags);
+    if (MPI_SUCCESS != rc) {
+        OBJ_RELEASE(context);
+        ompi_comm_request_return (crequest);
+    }
+
+    OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+
+    /* kick off the request */
+
+    ompi_comm_request_start (crequest);
+    *request = &crequest->super;
+
+    return rc;
+}
diff --git a/ompi/mpi/c/isendrecv_replace.c.in b/ompi/mpi/c/isendrecv_replace.c.in
new file mode 100644
index 00000000000..2dad8a3078e
--- /dev/null
+++ b/ompi/mpi/c/isendrecv_replace.c.in
@@ -0,0 +1,250 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2022 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2012 Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2021-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/communicator/comm_request.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "opal/datatype/opal_convertor.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/proc/proc.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+struct ompi_isendrecv_replace_context_t {
+    opal_object_t super;
+    opal_convertor_t convertor;
+    size_t packed_size;
+    unsigned char packed_data[2048];
+    struct iovec iov;
+    int nreqs;
+    int source;
+    ompi_request_t *subreq[2];
+};
+
+typedef struct ompi_isendrecv_replace_context_t ompi_isendrecv_replace_context_t;
+
+#if OMPI_BUILD_MPI_PROFILING
+static void ompi_isendrecv_context_constructor(ompi_isendrecv_replace_context_t *context)
+{
+    context->packed_size = 0;
+    OBJ_CONSTRUCT(&context->convertor, opal_convertor_t);
+}
+
+static void ompi_isendrecv_context_destructor(ompi_isendrecv_replace_context_t *context)
+{
+    if (context->packed_size > sizeof(context->packed_data)) {
+        PMPI_Free_mem(context->iov.iov_base);
+    }
+    OBJ_DESTRUCT(&context->convertor);
+}
+
+OBJ_CLASS_INSTANCE(ompi_isendrecv_replace_context_t, 
+                   opal_object_t, 
+                   ompi_isendrecv_context_constructor,
+                   ompi_isendrecv_context_destructor);
+#else
+OBJ_CLASS_DECLARATION(ompi_isendrecv_replace_context_t);
+#endif /* OMPI_BUILD_MPI_PROFILING */
+
+static int ompi_isendrecv_replace_complete_func (ompi_comm_request_t *request)
+{
+    ompi_isendrecv_replace_context_t *context =
+        (ompi_isendrecv_replace_context_t *) request->context;
+
+    /*
+     * Copy the status from the receive side of the sendrecv request?
+     * But what if the send failed?
+     *
+     * Probably need to bring up in the MPI forum.  
+     */
+
+    if (MPI_PROC_NULL != context->source) {
+        OMPI_COPY_STATUS(&request->super.req_status,
+                         context->subreq[0]->req_status, false);
+    } else {
+        OMPI_COPY_STATUS(&request->super.req_status, 
+                         ompi_request_empty.req_status, false);
+    }
+
+    if(NULL != context->subreq[0]) {
+        ompi_request_free(&context->subreq[0]);
+    }
+    if(NULL != context->subreq[1]) {
+        ompi_request_free(&context->subreq[1]);
+    }
+
+    return OMPI_SUCCESS;
+}
+
+PROTOTYPE ERROR_CLASS isendrecv_replace(BUFFER_OUT buf, COUNT count, DATATYPE datatype,
+                                        INT dest, INT sendtag, INT source, INT recvtag,
+                                        COMM comm, REQUEST_INOUT request)
+{
+    int rc = MPI_SUCCESS;
+    size_t max_data;
+    uint32_t iov_count;
+    ompi_comm_request_t *crequest = NULL;
+    ompi_isendrecv_replace_context_t *context = NULL;
+    int nreqs = 0;
+    uint32_t flags;
+
+    SPC_RECORD(OMPI_SPC_ISENDRECV_REPLACE, 1);
+
+    MEMCHECKER(
+               memchecker_datatype(datatype);
+               memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
+               memchecker_comm(comm);
+               );
+
+    if ( MPI_PARAM_CHECK ) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
+
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+        } else if (dest != MPI_PROC_NULL && ompi_comm_peer_invalid(comm, dest)) {
+            rc = MPI_ERR_RANK;
+        } else if (sendtag < 0 || sendtag > mca_pml.pml_max_tag) {
+            rc = MPI_ERR_TAG;
+        } else if (source != MPI_PROC_NULL && source != MPI_ANY_SOURCE && ompi_comm_peer_invalid(comm, source)) {
+            rc = MPI_ERR_RANK;
+        } else if (((recvtag < 0) && (recvtag !=  MPI_ANY_TAG)) || (recvtag > mca_pml.pml_max_tag)) {
+            rc = MPI_ERR_TAG;
+        } else if (request == NULL) {
+            rc = MPI_ERR_REQUEST;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+    /* simple case */
+    if ( source == MPI_PROC_NULL || dest == MPI_PROC_NULL || count == 0 ) {
+        rc = PMPI_Isendrecv(buf, count, datatype, dest, sendtag, buf, count, datatype, source, recvtag, comm, request);
+        return rc;
+    }
+
+    ompi_proc_t* proc = ompi_comm_peer_lookup(comm, dest);
+    if(proc == NULL) {
+        rc = MPI_ERR_RANK;
+        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+    }
+
+    crequest = ompi_comm_request_get ();
+    if (NULL == crequest) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    context = OBJ_NEW(ompi_isendrecv_replace_context_t);
+    if (NULL == context) {
+        ompi_comm_request_return (crequest);
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    context->iov.iov_base = context->packed_data;
+    context->iov.iov_len = sizeof(context->packed_data);
+
+    crequest->context = &context->super;
+    context->subreq[0] = NULL;
+    context->subreq[1] = NULL;
+    context->source = source;
+
+    /* initialize convertor to unpack recv buffer */
+    OBJ_CONSTRUCT(&context->convertor, opal_convertor_t);
+    opal_convertor_copy_and_prepare_for_send( proc->super.proc_convertor, &(datatype->super),
+                                              count, buf, 0, &context->convertor );
+
+    /* setup a buffer for recv */
+    opal_convertor_get_packed_size( &context->convertor, &context->packed_size );
+    if( context->packed_size > sizeof(context->packed_data) ) {
+        rc = PMPI_Alloc_mem(context->packed_size, MPI_INFO_NULL, &context->iov.iov_base);
+        if(OMPI_SUCCESS != rc) {
+            OBJ_RELEASE(context);
+            ompi_comm_request_return (crequest);
+            OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+        }
+        context->iov.iov_len = context->packed_size;
+    }
+    max_data = context->packed_size;
+    iov_count = 1;
+    rc = opal_convertor_pack(&context->convertor, &context->iov, &iov_count, &max_data);
+    if ( 0 > rc ) {
+        OBJ_RELEASE(context);
+        ompi_comm_request_return (crequest);
+        rc = MPI_ERR_UNKNOWN;
+        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+    }
+
+    if (source != MPI_PROC_NULL) { /* post recv */
+        rc = MCA_PML_CALL(irecv(buf, count, datatype,
+                                source, recvtag, comm, &context->subreq[nreqs++]));
+        if (MPI_SUCCESS != rc) {
+            OBJ_RELEASE(context);
+            ompi_comm_request_return (crequest);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+    if (dest != MPI_PROC_NULL) { /* send */
+        rc = MCA_PML_CALL(isend(context->iov.iov_base, context->packed_size, MPI_PACKED, dest,
+                               sendtag, MCA_PML_BASE_SEND_STANDARD, comm, 
+                                &context->subreq[nreqs++]));
+        if (MPI_SUCCESS != rc) {
+            OBJ_RELEASE(context);
+            ompi_comm_request_return (crequest);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+    /*
+     *  schedule the operation
+     */
+
+    context->nreqs = nreqs;
+    assert(nreqs <= 2);
+
+    flags = OMPI_COMM_REQ_FLAG_RETAIN_SUBREQ;
+
+    rc = ompi_comm_request_schedule_append_w_flags(crequest, 
+                                                   ompi_isendrecv_replace_complete_func, 
+                                                   context->subreq, 
+                                                   nreqs,
+                                                   flags);
+    if (MPI_SUCCESS != rc) {
+        OBJ_RELEASE(context);
+        ompi_comm_request_return (crequest);
+    }
+
+    OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+
+    /* kick off the request */
+
+    ompi_comm_request_start (crequest);
+    *request = &crequest->super;
+
+    return rc;
+}
diff --git a/ompi/mpi/c/issend.c b/ompi/mpi/c/issend.c
deleted file mode 100644
index e233b145da1..00000000000
--- a/ompi/mpi/c/issend.c
+++ /dev/null
@@ -1,99 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/request/request.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Issend = PMPI_Issend
-#endif
-#define MPI_Issend PMPI_Issend
-#endif
-
-static const char FUNC_NAME[] = "MPI_Issend";
-
-
-int MPI_Issend(const void *buf, int count, MPI_Datatype type, int dest,
-               int tag, MPI_Comm comm, MPI_Request *request)
-{
-    int rc = MPI_SUCCESS;
-
-    SPC_RECORD(OMPI_SPC_ISSEND, 1);
-
-    MEMCHECKER(
-        memchecker_datatype(type);
-        memchecker_call(&opal_memchecker_base_isdefined, buf, count, type);
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else if (tag < 0 || tag > mca_pml.pml_max_tag) {
-            rc = MPI_ERR_TAG;
-        } else if (ompi_comm_peer_invalid(comm, dest) &&
-                   (MPI_PROC_NULL != dest)) {
-            rc = MPI_ERR_RANK;
-        } else if (request == NULL) {
-            rc = MPI_ERR_REQUEST;
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_SEND(rc, type, count);
-            OMPI_CHECK_USER_BUFFER(rc, buf, type, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-    }
-
-    if (MPI_PROC_NULL == dest) {
-        *request = &ompi_request_empty;
-        return MPI_SUCCESS;
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * The request will be checked for process failure errors during the
-     * completion calls. So no need to check here.
-     */
-#endif
-
-    MEMCHECKER (
-        memchecker_call(&opal_memchecker_base_mem_noaccess, buf, count, type);
-    );
-    rc = MCA_PML_CALL(isend(buf, count, type, dest, tag,
-                            MCA_PML_BASE_SEND_SYNCHRONOUS, comm, request));
-    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-}
-
-
diff --git a/ompi/mpi/c/issend.c.in b/ompi/mpi/c/issend.c.in
new file mode 100644
index 00000000000..1a34c0a4c9c
--- /dev/null
+++ b/ompi/mpi/c/issend.c.in
@@ -0,0 +1,91 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/request/request.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS issend(BUFFER buf, COUNT count, DATATYPE type, INT dest,
+                             INT tag, COMM comm, REQUEST_INOUT request)
+{
+    int rc = MPI_SUCCESS;
+
+    SPC_RECORD(OMPI_SPC_ISSEND, 1);
+
+    MEMCHECKER(
+        memchecker_datatype(type);
+        memchecker_call(&opal_memchecker_base_isdefined, buf, count, type);
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else if (tag < 0 || tag > mca_pml.pml_max_tag) {
+            rc = MPI_ERR_TAG;
+        } else if (ompi_comm_peer_invalid(comm, dest) &&
+                   (MPI_PROC_NULL != dest)) {
+            rc = MPI_ERR_RANK;
+        } else if (request == NULL) {
+            rc = MPI_ERR_REQUEST;
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_SEND(rc, type, count);
+            OMPI_CHECK_USER_BUFFER(rc, buf, type, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+    if (MPI_PROC_NULL == dest) {
+        *request = &ompi_request_empty;
+        return MPI_SUCCESS;
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * The request will be checked for process failure errors during the
+     * completion calls. So no need to check here.
+     */
+#endif
+
+    MEMCHECKER (
+        memchecker_call(&opal_memchecker_base_mem_noaccess, buf, count, type);
+    );
+    rc = MCA_PML_CALL(isend(buf, count, type, dest, tag,
+                            MCA_PML_BASE_SEND_SYNCHRONOUS, comm, request));
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
+
+
diff --git a/ompi/mpi/c/keyval_create.c b/ompi/mpi/c/keyval_create.c
deleted file mode 100644
index 1a8a268fed8..00000000000
--- a/ompi/mpi/c/keyval_create.c
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/attribute/attribute.h"
-#include "ompi/communicator/communicator.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Keyval_create = PMPI_Keyval_create
-#endif
-#define MPI_Keyval_create PMPI_Keyval_create
-#endif
-
-static const char FUNC_NAME[] = "MPI_Keyval_create";
-
-
-int MPI_Keyval_create(MPI_Copy_function *copy_attr_fn,
-                      MPI_Delete_function *delete_attr_fn,
-                      int *keyval, void *extra_state)
-{
-    int ret;
-    ompi_attribute_fn_ptr_union_t copy_fn;
-    ompi_attribute_fn_ptr_union_t del_fn;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == keyval) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_KEYVAL,
-                                          FUNC_NAME);
-        } else if ((NULL == copy_attr_fn) || (NULL == delete_attr_fn)) {
-	    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-    }
-
-    copy_fn.attr_communicator_copy_fn = (MPI_Comm_internal_copy_attr_function*)copy_attr_fn;
-    del_fn.attr_communicator_delete_fn = delete_attr_fn;
-
-    ret = ompi_attr_create_keyval(COMM_ATTR, copy_fn,
-                                  del_fn, keyval, extra_state, 0, NULL);
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(ret, MPI_ERR_OTHER, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/keyval_create.c.in b/ompi/mpi/c/keyval_create.c.in
new file mode 100644
index 00000000000..8a318ad4da3
--- /dev/null
+++ b/ompi/mpi/c/keyval_create.c.in
@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/attribute/attribute.h"
+#include "ompi/communicator/communicator.h"
+
+PROTOTYPE ERROR_CLASS keyval_create(COPY_FUNCTION copy_attr_fn,
+                                    DELETE_FUNCTION delete_attr_fn,
+                                    INT_OUT keyval, BUFFER_OUT extra_state)
+{
+    int ret;
+    ompi_attribute_fn_ptr_union_t copy_fn;
+    ompi_attribute_fn_ptr_union_t del_fn;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == keyval) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_KEYVAL,
+                                          FUNC_NAME);
+        } else if ((NULL == copy_attr_fn) || (NULL == delete_attr_fn)) {
+	    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+    copy_fn.attr_communicator_copy_fn = copy_attr_fn;
+    del_fn.attr_communicator_delete_fn = delete_attr_fn;
+
+    ret = ompi_attr_create_keyval(COMM_ATTR, copy_fn,
+                                  del_fn, keyval, extra_state, 0, NULL);
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(ret, MPI_ERR_OTHER, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/keyval_free.c b/ompi/mpi/c/keyval_free.c
deleted file mode 100644
index 40183f72f14..00000000000
--- a/ompi/mpi/c/keyval_free.c
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/attribute/attribute.h"
-#include "ompi/communicator/communicator.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Keyval_free = PMPI_Keyval_free
-#endif
-#define MPI_Keyval_free PMPI_Keyval_free
-#endif
-
-static const char FUNC_NAME[] = "MPI_Keyval_free";
-
-
-int MPI_Keyval_free(int *keyval)
-{
-    int ret;
-
-    /* Check for valid key pointer */
-    if (MPI_PARAM_CHECK) {
-        if (NULL == keyval) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_KEYVAL,
-                                          FUNC_NAME);
-        }
-    }
-
-    ret = ompi_attr_free_keyval(COMM_ATTR, keyval, 0);
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(ret, MPI_ERR_OTHER, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/keyval_free.c.in b/ompi/mpi/c/keyval_free.c.in
new file mode 100644
index 00000000000..8705ada7a10
--- /dev/null
+++ b/ompi/mpi/c/keyval_free.c.in
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/attribute/attribute.h"
+#include "ompi/communicator/communicator.h"
+
+PROTOTYPE ERROR_CLASS keyval_free(INT_OUT keyval)
+{
+    int ret;
+
+    /* Check for valid key pointer */
+    if (MPI_PARAM_CHECK) {
+        if (NULL == keyval) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_KEYVAL,
+                                          FUNC_NAME);
+        }
+    }
+
+    ret = ompi_attr_free_keyval(COMM_ATTR, keyval, 0);
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(ret, MPI_ERR_OTHER, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/lookup_name.c b/ompi/mpi/c/lookup_name.c
deleted file mode 100644
index 7415ad88cf5..00000000000
--- a/ompi/mpi/c/lookup_name.c
+++ /dev/null
@@ -1,121 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2019 Intel, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2015-2018 Cisco Systems, Inc.  All rights reserved
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "opal/class/opal_list.h"
-#include "opal/mca/pmix/pmix-internal.h"
-#include "opal/util/show_help.h"
-#include "opal/util/string_copy.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-#include "ompi/communicator/communicator.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Lookup_name = PMPI_Lookup_name
-#endif
-#define MPI_Lookup_name PMPI_Lookup_name
-#endif
-
-static const char FUNC_NAME[] = "MPI_Lookup_name";
-
-
-int MPI_Lookup_name(const char *service_name, MPI_Info info, char *port_name)
-{
-    char range[OPAL_MAX_INFO_VAL];
-    int flag=0, ret;
-    pmix_status_t rc;
-    pmix_pdata_t pdat;
-    pmix_info_t pinfo;
-    pmix_data_range_t rng;
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if ( NULL == port_name ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-        if ( NULL == service_name ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-        if (NULL == info || ompi_info_is_freed(info)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
-                                          FUNC_NAME);
-        }
-    }
-
-    /* OMPI supports info keys to pass the range to
-     * be searched for the given key */
-    if (MPI_INFO_NULL != info) {
-        ompi_info_get (info, "range", sizeof(range) - 1, range, &flag);
-        if (flag) {
-            if (0 == strcmp(range, "nspace")) {
-                rng = PMIX_RANGE_NAMESPACE;  // share only with procs in same nspace
-            } else if (0 == strcmp(range, "session")) {
-                rng = PMIX_RANGE_SESSION; // share only with procs in same session
-            } else {
-                /* unrecognized scope */
-                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                            FUNC_NAME);
-            }
-        }
-    }
-    PMIX_INFO_LOAD(&pinfo, PMIX_RANGE, &rng, PMIX_DATA_RANGE);
-
-    /* collect the findings */
-    PMIX_PDATA_CONSTRUCT(&pdat);
-    PMIX_LOAD_KEY(pdat.key, service_name);
-
-    rc = PMIx_Lookup(&pdat, 1, &pinfo, 1);
-    PMIX_INFO_DESTRUCT(&pinfo);
-    if (PMIX_SUCCESS != rc ||
-        PMIX_STRING != pdat.value.type ||
-        NULL == pdat.value.data.string) {
-        if (PMIX_ERR_NOT_SUPPORTED == rc) {
-            ret = OMPI_ERR_NOT_SUPPORTED;
-            opal_show_help("help-mpi-api.txt",
-                           "MPI function not supported",
-                           true,
-                           FUNC_NAME,
-                           "Underlying runtime environment does not support name lookup functionality");
-        } else {
-            ret = MPI_ERR_NAME;
-        }
-
-        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(ret, FUNC_NAME);
-    }
-
-    opal_string_copy( port_name, pdat.value.data.string,
-                      MPI_MAX_PORT_NAME );
-    PMIX_PDATA_DESTRUCT(&pdat);
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/lookup_name.c.in b/ompi/mpi/c/lookup_name.c.in
new file mode 100644
index 00000000000..a49ee143e59
--- /dev/null
+++ b/ompi/mpi/c/lookup_name.c.in
@@ -0,0 +1,112 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015-2018 Cisco Systems, Inc.  All rights reserved
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "opal/class/opal_list.h"
+#include "opal/mca/pmix/pmix-internal.h"
+#include "opal/util/show_help.h"
+#include "opal/util/string_copy.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include "ompi/communicator/communicator.h"
+
+PROTOTYPE ERROR_CLASS lookup_name(STRING service_name, INFO info, STRING_OUT port_name)
+{
+    int flag=0, ret;
+    pmix_status_t rc;
+    pmix_pdata_t pdat;
+    pmix_info_t pinfo;
+    pmix_data_range_t rng;
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if ( NULL == port_name ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+        if ( NULL == service_name ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+        if (NULL == info || ompi_info_is_freed(info)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
+                                          FUNC_NAME);
+        }
+    }
+
+    /* OMPI supports info keys to pass the range to
+     * be searched for the given key */
+    if (MPI_INFO_NULL != info) {
+        opal_cstring_t *info_str;
+        ompi_info_get (info, "range", &info_str, &flag);
+        if (flag) {
+            if (0 == strcmp(info_str->string, "nspace")) {
+                rng = PMIX_RANGE_NAMESPACE;  // share only with procs in same nspace
+            } else if (0 == strcmp(info_str->string, "session")) {
+                rng = PMIX_RANGE_SESSION; // share only with procs in same session
+            } else {
+                /* unrecognized scope */
+                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                            FUNC_NAME);
+            }
+            OBJ_RELEASE(info_str);
+        }
+    }
+    PMIX_INFO_LOAD(&pinfo, PMIX_RANGE, &rng, PMIX_DATA_RANGE);
+
+    /* collect the findings */
+    PMIX_PDATA_CONSTRUCT(&pdat);
+    PMIX_LOAD_KEY(pdat.key, service_name);
+
+    rc = PMIx_Lookup(&pdat, 1, &pinfo, 1);
+    PMIX_INFO_DESTRUCT(&pinfo);
+    if (PMIX_SUCCESS != rc ||
+        PMIX_STRING != pdat.value.type ||
+        NULL == pdat.value.data.string) {
+        if (PMIX_ERR_NOT_SUPPORTED == rc) {
+            ret = OMPI_ERR_NOT_SUPPORTED;
+            opal_show_help("help-mpi-api.txt",
+                           "MPI function not supported",
+                           true,
+                           FUNC_NAME,
+                           "Underlying runtime environment does not support name lookup functionality");
+        } else {
+            ret = MPI_ERR_NAME;
+        }
+
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(ret, FUNC_NAME);
+    }
+
+    opal_string_copy( port_name, pdat.value.data.string,
+                      MPI_MAX_PORT_NAME );
+    PMIX_PDATA_DESTRUCT(&pdat);
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/message_c2f.c b/ompi/mpi/c/message_c2f.c
deleted file mode 100644
index c6406da434b..00000000000
--- a/ompi/mpi/c/message_c2f.c
+++ /dev/null
@@ -1,75 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2007 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mpi/fortran/base/fint_2_int.h"
-#include "ompi/message/message.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Message_c2f = PMPI_Message_c2f
-#endif
-#define MPI_Message_c2f PMPI_Message_c2f
-#endif
-
-static const char FUNC_NAME[] = "MPI_Message_c2f";
-
-
-MPI_Fint MPI_Message_c2f(MPI_Message message)
-{
-    MEMCHECKER(
-        memchecker_message(&message);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (NULL == message) {
-            return OMPI_INT_2_FINT(-1);
-        }
-    }
-
-    /* We only put messages in the f2c table when this function is
-       invoked.  This is because putting messages in the table
-       involves locking and unlocking the table, which would incur a
-       performance penalty (in the critical performance path) for C
-       applications.  In this way, at least only Fortran applications
-       are penalized.  :-\
-
-       Modifying this one function neatly fixes up all the Fortran
-       bindings because they all call MPI_Message_c2f in order to
-       transmorgify the C MPI_Message that they got back into a
-       fortran integer.
-    */
-
-    if (MPI_UNDEFINED == message->m_f_to_c_index) {
-        message->m_f_to_c_index =
-            opal_pointer_array_add(&ompi_message_f_to_c_table, message);
-    }
-
-    return OMPI_INT_2_FINT(message->m_f_to_c_index) ;
-}
diff --git a/ompi/mpi/c/message_c2f.c.in b/ompi/mpi/c/message_c2f.c.in
new file mode 100644
index 00000000000..407fc9f1ff3
--- /dev/null
+++ b/ompi/mpi/c/message_c2f.c.in
@@ -0,0 +1,67 @@
+/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+#include "ompi/message/message.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE FINT message_c2f(MESSAGE message)
+{
+    MEMCHECKER(
+        memchecker_message(&message);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (NULL == message) {
+            return OMPI_INT_2_FINT(-1);
+        }
+    }
+
+    /* We only put messages in the f2c table when this function is
+       invoked.  This is because putting messages in the table
+       involves locking and unlocking the table, which would incur a
+       performance penalty (in the critical performance path) for C
+       applications.  In this way, at least only Fortran applications
+       are penalized.  :-\
+
+       Modifying this one function neatly fixes up all the Fortran
+       bindings because they all call MPI_Message_c2f in order to
+       transmorgify the C MPI_Message that they got back into a
+       fortran integer.
+    */
+
+    if (MPI_UNDEFINED == message->m_f_to_c_index) {
+        message->m_f_to_c_index =
+            opal_pointer_array_add(&ompi_message_f_to_c_table, message);
+    }
+
+    return OMPI_INT_2_FINT(message->m_f_to_c_index) ;
+}
diff --git a/ompi/mpi/c/message_f2c.c b/ompi/mpi/c/message_f2c.c
deleted file mode 100644
index b667b573b19..00000000000
--- a/ompi/mpi/c/message_f2c.c
+++ /dev/null
@@ -1,60 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2007 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mpi/fortran/base/fint_2_int.h"
-#include "ompi/message/message.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Message_f2c = PMPI_Message_f2c
-#endif
-#define MPI_Message_f2c PMPI_Message_f2c
-#endif
-
-static const char FUNC_NAME[] = "MPI_Message_f2c";
-
-
-MPI_Message MPI_Message_f2c(MPI_Fint message)
-{
-    int message_index = OMPI_FINT_2_INT(message);
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-    }
-
-    /* Per MPI-2:4.12.4, do not invoke an error handler if we get an
-       invalid fortran handle.  If we get an invalid fortran handle,
-       return an invalid C handle. */
-
-    if (message_index < 0 ||
-        message_index >=
-        opal_pointer_array_get_size(&ompi_message_f_to_c_table)) {
-        return NULL;
-    }
-
-    return (MPI_Message)opal_pointer_array_get_item(&ompi_message_f_to_c_table,
-                                                    message_index);
-}
diff --git a/ompi/mpi/c/message_f2c.c.in b/ompi/mpi/c/message_f2c.c.in
new file mode 100644
index 00000000000..e9179e86ba0
--- /dev/null
+++ b/ompi/mpi/c/message_f2c.c.in
@@ -0,0 +1,52 @@
+/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+#include "ompi/message/message.h"
+
+PROTOTYPE MESSAGE message_f2c(FINT message)
+{
+    int message_index = OMPI_FINT_2_INT(message);
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+    }
+
+    /* Per MPI-2:4.12.4, do not invoke an error handler if we get an
+       invalid fortran handle.  If we get an invalid fortran handle,
+       return an invalid C handle. */
+
+    if (message_index < 0 ||
+        message_index >=
+        opal_pointer_array_get_size(&ompi_message_f_to_c_table)) {
+        return NULL;
+    }
+
+    return (MPI_Message)opal_pointer_array_get_item(&ompi_message_f_to_c_table,
+                                                    message_index);
+}
diff --git a/ompi/mpi/c/mprobe.c b/ompi/mpi/c/mprobe.c
deleted file mode 100644
index a608174ecca..00000000000
--- a/ompi/mpi/c/mprobe.c
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
- * Copyright (c) 2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012      Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2020      The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/memchecker.h"
-#include "ompi/request/request.h"
-#include "ompi/message/message.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Mprobe = PMPI_Mprobe
-#endif
-#define MPI_Mprobe PMPI_Mprobe
-#endif
-
-static const char FUNC_NAME[] = "MPI_Mprobe";
-
-int MPI_Mprobe(int source, int tag, MPI_Comm comm,
-               MPI_Message *message, MPI_Status *status)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (((tag < 0) && (tag != MPI_ANY_TAG)) || (tag > mca_pml.pml_max_tag)) {
-            rc = MPI_ERR_TAG;
-        } else if (ompi_comm_invalid(comm)) {
-            rc = MPI_ERR_COMM;
-        } else if ((source != MPI_ANY_SOURCE) &&
-                   (MPI_PROC_NULL != source) &&
-                   ompi_comm_peer_invalid(comm, source)) {
-            rc = MPI_ERR_RANK;
-        } else if (NULL == message) {
-            rc = MPI_ERR_REQUEST;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-    }
-
-    if (MPI_PROC_NULL == source) {
-        if (MPI_STATUS_IGNORE != status) {
-            *status = ompi_request_empty.req_status;
-            /* Per MPI-1, the MPI_ERROR field is not defined for
-               single-completion calls */
-            MEMCHECKER(
-                opal_memchecker_base_mem_undefined(&status->MPI_ERROR, sizeof(int));
-            );
-        }
-        *message = &ompi_message_no_proc.message;
-        return MPI_SUCCESS;
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * Check here for issues with the peer, so we do not have to duplicate the
-     * functionality in the PML.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_p2p_check_proc(comm, source, &rc)) ) {
-        if (MPI_STATUS_IGNORE != status) {
-            status->MPI_SOURCE = source;
-            status->MPI_TAG    = tag;
-        }
-        *message = &ompi_message_no_proc.message;
-        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-    }
-#endif
-
-    rc = MCA_PML_CALL(mprobe(source, tag, comm, message, status));
-    /* Per MPI-1, the MPI_ERROR field is not defined for
-       single-completion calls */
-    MEMCHECKER(
-        opal_memchecker_base_mem_undefined(&status->MPI_ERROR, sizeof(int));
-    );
-
-    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/mprobe.c.in b/ompi/mpi/c/mprobe.c.in
new file mode 100644
index 00000000000..2f060e52b49
--- /dev/null
+++ b/ompi/mpi/c/mprobe.c.in
@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2012      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2020-2021 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/memchecker.h"
+#include "ompi/request/request.h"
+#include "ompi/message/message.h"
+
+PROTOTYPE ERROR_CLASS mprobe(INT source, INT tag, COMM comm,
+                             MESSAGE_OUT message, STATUS_OUT status)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (((tag < 0) && (tag != MPI_ANY_TAG)) || (tag > mca_pml.pml_max_tag)) {
+            rc = MPI_ERR_TAG;
+        } else if (ompi_comm_invalid(comm)) {
+            rc = MPI_ERR_COMM;
+        } else if ((source != MPI_ANY_SOURCE) &&
+                   (MPI_PROC_NULL != source) &&
+                   ompi_comm_peer_invalid(comm, source)) {
+            rc = MPI_ERR_RANK;
+        } else if (NULL == message) {
+            rc = MPI_ERR_REQUEST;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+    if (MPI_PROC_NULL == source) {
+        if (MPI_STATUS_IGNORE != status) {
+            OMPI_COPY_STATUS(status, ompi_request_empty.req_status, false);
+            /* Per MPI-1, the MPI_ERROR field is not defined for
+               single-completion calls */
+            MEMCHECKER(
+                opal_memchecker_base_mem_undefined(&status->MPI_ERROR, sizeof(int));
+            );
+        }
+        *message = &ompi_message_no_proc.message;
+        return MPI_SUCCESS;
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * Check here for issues with the peer, so we do not have to duplicate the
+     * functionality in the PML.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_p2p_check_proc(comm, source, &rc)) ) {
+        if (MPI_STATUS_IGNORE != status) {
+            status->MPI_SOURCE = source;
+            status->MPI_TAG    = tag;
+        }
+        *message = &ompi_message_no_proc.message;
+        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+    }
+#endif
+
+    rc = MCA_PML_CALL(mprobe(source, tag, comm, message, status));
+    /* Per MPI-1, the MPI_ERROR field is not defined for
+       single-completion calls */
+    MEMCHECKER(
+        opal_memchecker_base_mem_undefined(&status->MPI_ERROR, sizeof(int));
+    );
+
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/mrecv.c b/ompi/mpi/c/mrecv.c
deleted file mode 100644
index 692bf87da8b..00000000000
--- a/ompi/mpi/c/mrecv.c
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
- * Copyright (c) 2012      Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2012-2013 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2018      The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/memchecker.h"
-#include "ompi/request/request.h"
-#include "ompi/message/message.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Mrecv = PMPI_Mrecv
-#endif
-#define MPI_Mrecv PMPI_Mrecv
-#endif
-
-static const char FUNC_NAME[] = "MPI_Mrecv";
-
-
-int MPI_Mrecv(void *buf, int count, MPI_Datatype type,
-              MPI_Message *message, MPI_Status *status)
-{
-    int rc = MPI_SUCCESS;
-    ompi_communicator_t *comm;
-
-    SPC_RECORD(OMPI_SPC_MRECV, 1);
-
-    MEMCHECKER(
-        memchecker_datatype(type);
-        memchecker_message(message);
-        memchecker_call(&opal_memchecker_base_isaddressable, buf, count, type);
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        OMPI_CHECK_DATATYPE_FOR_RECV(rc, type, count);
-        OMPI_CHECK_USER_BUFFER(rc, buf, type, count);
-
-        if (NULL == message || MPI_MESSAGE_NULL == *message) {
-            rc = MPI_ERR_REQUEST;
-            comm = MPI_COMM_NULL;
-        } else {
-            comm = (*message)->comm;
-        }
-
-        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-    } else {
-        comm = (*message)->comm;
-    }
-
-    if (&ompi_message_no_proc.message == *message) {
-        if (MPI_STATUS_IGNORE != status) {
-            *status = ompi_request_empty.req_status;
-        }
-        *message = MPI_MESSAGE_NULL;
-        return MPI_SUCCESS;
-     }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * The message and associated request will be checked by the PML, and
-     * handled approprately. SO no need to check here.
-     */
-#endif
-
-    rc = MCA_PML_CALL(mrecv(buf, count, type, message, status));
-    /* Per MPI-1, the MPI_ERROR field is not defined for
-       single-completion calls */
-    MEMCHECKER(
-        opal_memchecker_base_mem_undefined(&status->MPI_ERROR, sizeof(int));
-    );
-
-    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/mrecv.c.in b/ompi/mpi/c/mrecv.c.in
new file mode 100644
index 00000000000..a23368eaf51
--- /dev/null
+++ b/ompi/mpi/c/mrecv.c.in
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2012      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2012-2013 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018-2021 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/memchecker.h"
+#include "ompi/request/request.h"
+#include "ompi/message/message.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS mrecv(BUFFER_OUT buf, COUNT count, DATATYPE type,
+                            MESSAGE_OUT message, STATUS_OUT status)
+{
+    int rc = MPI_SUCCESS;
+    ompi_communicator_t *comm;
+
+    SPC_RECORD(OMPI_SPC_MRECV, 1);
+
+    MEMCHECKER(
+        memchecker_datatype(type);
+        memchecker_message(message);
+        memchecker_call(&opal_memchecker_base_isaddressable, buf, count, type);
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        OMPI_CHECK_DATATYPE_FOR_RECV(rc, type, count);
+        OMPI_CHECK_USER_BUFFER(rc, buf, type, count);
+
+        if (NULL == message || MPI_MESSAGE_NULL == *message) {
+            rc = MPI_ERR_REQUEST;
+            comm = MPI_COMM_NULL;
+        } else {
+            comm = (*message)->comm;
+        }
+
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    } else {
+        comm = (*message)->comm;
+    }
+
+    if (&ompi_message_no_proc.message == *message) {
+        if (MPI_STATUS_IGNORE != status) {
+            OMPI_COPY_STATUS(status, ompi_request_empty.req_status, false);
+        }
+        *message = MPI_MESSAGE_NULL;
+        return MPI_SUCCESS;
+     }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * The message and associated request will be checked by the PML, and
+     * handled appropriately. SO no need to check here.
+     */
+#endif
+
+    rc = MCA_PML_CALL(mrecv(buf, count, type, message, status));
+    /* Per MPI-1, the MPI_ERROR field is not defined for
+       single-completion calls */
+    MEMCHECKER(
+        opal_memchecker_base_mem_undefined(&status->MPI_ERROR, sizeof(int));
+    );
+
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/neighbor_allgather.c b/ompi/mpi/c/neighbor_allgather.c
deleted file mode 100644
index 872c2ffb292..00000000000
--- a/ompi/mpi/c/neighbor_allgather.c
+++ /dev/null
@@ -1,143 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010      University of Houston.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015-2017 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-#include "ompi/mca/topo/topo.h"
-#include "ompi/mca/topo/base/base.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Neighbor_allgather = PMPI_Neighbor_allgather
-#endif
-#define MPI_Neighbor_allgather PMPI_Neighbor_allgather
-#endif
-
-static const char FUNC_NAME[] = "MPI_Neighbor_allgather";
-
-
-int MPI_Neighbor_allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                           void *recvbuf, int recvcount, MPI_Datatype recvtype,
-                           MPI_Comm comm)
-{
-    int err;
-
-    SPC_RECORD(OMPI_SPC_NEIGHBOR_ALLGATHER, 1);
-
-    MEMCHECKER(
-        ptrdiff_t ext;
-
-        ompi_datatype_type_extent(recvtype, &ext);
-
-        memchecker_datatype(recvtype);
-        memchecker_comm(comm);
-        /* check whether the actual send buffer is defined. */
-        if (MPI_IN_PLACE != sendbuf) {
-            memchecker_datatype(sendtype);
-            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-        }
-        /* check whether the receive buffer is addressable. */
-        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-
-        /* Unrooted operation -- same checks for all ranks on both
-           intracommunicators and intercommunicators */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
-          OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
-        } else if (! OMPI_COMM_IS_TOPO(comm)) {
-          OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TOPOLOGY, FUNC_NAME);
-        } else if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
-          err = MPI_ERR_TYPE;
-        } else if (recvcount < 0) {
-          err = MPI_ERR_COUNT;
-        } else if (MPI_IN_PLACE == sendbuf || MPI_IN_PLACE == recvbuf) {
-          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-        }
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-
-        if( OMPI_COMM_IS_CART(comm) ) {
-            const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
-            if( 0 > cart->ndims ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-        else if( OMPI_COMM_IS_GRAPH(comm) ) {
-            int degree;
-            mca_topo_base_graph_neighbors_count(comm, ompi_comm_rank(comm), &degree);
-            if( 0 > degree ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-        else if( OMPI_COMM_IS_DIST_GRAPH(comm) ) {
-            const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
-            int indegree  = dist_graph->indegree;
-            int outdegree = dist_graph->outdegree;
-            if( indegree <  0 || outdegree <  0 ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-    }
-
-    /* Do we need to do anything?  Everyone had to give the same send
-       signature, which means that everyone must have given a
-       sendcount > 0. */
-    if ((0 == sendcount) || (0 == recvcount)) {
-        return MPI_SUCCESS;
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
-        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-    }
-#endif
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_neighbor_allgather(sendbuf, sendcount, sendtype,
-                                               recvbuf, recvcount, recvtype, comm,
-                                               comm->c_coll->coll_neighbor_allgather_module);
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/neighbor_allgather.c.in b/ompi/mpi/c/neighbor_allgather.c.in
new file mode 100644
index 00000000000..fbfbf8ef342
--- /dev/null
+++ b/ompi/mpi/c/neighbor_allgather.c.in
@@ -0,0 +1,135 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010      University of Houston.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2017 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/mca/topo/base/base.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS neighbor_allgather(BUFFER sendbuf, COUNT sendcount, DATATYPE sendtype,
+                                         BUFFER_OUT recvbuf, COUNT recvcount, DATATYPE recvtype,
+                                         COMM comm)
+{
+    int err;
+
+    SPC_RECORD(OMPI_SPC_NEIGHBOR_ALLGATHER, 1);
+
+    MEMCHECKER(
+        ptrdiff_t ext;
+
+        ompi_datatype_type_extent(recvtype, &ext);
+
+        memchecker_datatype(recvtype);
+        memchecker_comm(comm);
+        /* check whether the actual send buffer is defined. */
+        if (MPI_IN_PLACE != sendbuf) {
+            memchecker_datatype(sendtype);
+            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+        }
+        /* check whether the receive buffer is addressable. */
+        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+
+        /* Unrooted operation -- same checks for all ranks on both
+           intracommunicators and intercommunicators */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
+          OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+        } else if (! OMPI_COMM_IS_TOPO(comm)) {
+          OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TOPOLOGY, FUNC_NAME);
+        } else if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
+          err = MPI_ERR_TYPE;
+        } else if (recvcount < 0) {
+          err = MPI_ERR_COUNT;
+        } else if (MPI_IN_PLACE == sendbuf || MPI_IN_PLACE == recvbuf) {
+          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+        }
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+
+        if( OMPI_COMM_IS_CART(comm) ) {
+            const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
+            if( 0 > cart->ndims ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+        else if( OMPI_COMM_IS_GRAPH(comm) ) {
+            int degree;
+            mca_topo_base_graph_neighbors_count(comm, ompi_comm_rank(comm), &degree);
+            if( 0 > degree ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+        else if( OMPI_COMM_IS_DIST_GRAPH(comm) ) {
+            const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
+            int indegree  = dist_graph->indegree;
+            int outdegree = dist_graph->outdegree;
+            if( indegree <  0 || outdegree <  0 ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+    }
+
+    /* Do we need to do anything?  Everyone had to give the same send
+       signature, which means that everyone must have given a
+       sendcount > 0. */
+    if ((0 == sendcount) || (0 == recvcount)) {
+        return MPI_SUCCESS;
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
+        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+    }
+#endif
+
+    /* Invoke the coll component to perform the back-end operation */
+    err = comm->c_coll->coll_neighbor_allgather(sendbuf, sendcount, sendtype,
+                                               recvbuf, recvcount, recvtype, comm,
+                                               comm->c_coll->coll_neighbor_allgather_module);
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/neighbor_allgather_init.c.in b/ompi/mpi/c/neighbor_allgather_init.c.in
new file mode 100644
index 00000000000..ed93cf30c69
--- /dev/null
+++ b/ompi/mpi/c/neighbor_allgather_init.c.in
@@ -0,0 +1,121 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oak Rigde National Laboratory. All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/mca/topo/base/base.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS neighbor_allgather_init(BUFFER sendbuf, COUNT sendcount, DATATYPE sendtype,
+                                              BUFFER_OUT recvbuf, COUNT recvcount, DATATYPE recvtype,
+                                              COMM comm, INFO info, REQUEST_INOUT request)
+{
+    int err;
+
+    SPC_RECORD(OMPI_SPC_NEIGHBOR_ALLGATHER_INIT, 1);
+
+    MEMCHECKER(
+        ptrdiff_t ext;
+
+        ompi_datatype_type_extent(recvtype, &ext);
+
+        memchecker_datatype(recvtype);
+        memchecker_comm(comm);
+        /* check whether the actual send buffer is defined. */
+        if (MPI_IN_PLACE != sendbuf) {
+            memchecker_datatype(sendtype);
+            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+        }
+        /* check whether the receive buffer is addressable. */
+        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+
+        /* Unrooted operation -- same checks for all ranks on both
+           intracommunicators and intercommunicators */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
+          OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+        } else if (! OMPI_COMM_IS_TOPO(comm)) {
+          OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TOPOLOGY, FUNC_NAME);
+        } else if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
+          err = MPI_ERR_TYPE;
+        } else if (recvcount < 0) {
+          err = MPI_ERR_COUNT;
+        } else if (MPI_IN_PLACE == sendbuf || MPI_IN_PLACE == recvbuf) {
+          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+        }
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+
+        if( OMPI_COMM_IS_CART(comm) ) {
+            const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
+            if( 0 > cart->ndims ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+        else if( OMPI_COMM_IS_GRAPH(comm) ) {
+            int degree;
+            mca_topo_base_graph_neighbors_count(comm, ompi_comm_rank(comm), &degree);
+            if( 0 > degree ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+        else if( OMPI_COMM_IS_DIST_GRAPH(comm) ) {
+            const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
+            int indegree  = dist_graph->indegree;
+            int outdegree = dist_graph->outdegree;
+            if( indegree <  0 || outdegree <  0 ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    err = comm->c_coll->coll_neighbor_allgather_init(sendbuf, sendcount, sendtype, recvbuf,
+                                                     recvcount, recvtype, comm, info, request,
+                                                     comm->c_coll->coll_neighbor_allgather_init_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
+    }
+
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/neighbor_allgatherv.c b/ompi/mpi/c/neighbor_allgatherv.c
deleted file mode 100644
index 4e618fd7dec..00000000000
--- a/ompi/mpi/c/neighbor_allgatherv.c
+++ /dev/null
@@ -1,158 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2010      University of Houston.  All rights reserved.
- * Copyright (c) 2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2016 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2017 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/topo/base/base.h"
-#include "ompi/memchecker.h"
-#include "ompi/mca/topo/topo.h"
-#include "ompi/mca/topo/base/base.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Neighbor_allgatherv = PMPI_Neighbor_allgatherv
-#endif
-#define MPI_Neighbor_allgatherv PMPI_Neighbor_allgatherv
-#endif
-
-static const char FUNC_NAME[] = "MPI_Neighbor_allgatherv";
-
-
-int MPI_Neighbor_allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                            void *recvbuf, const int recvcounts[], const int displs[],
-                            MPI_Datatype recvtype, MPI_Comm comm)
-{
-    int in_size, out_size, err;
-
-    SPC_RECORD(OMPI_SPC_NEIGHBOR_ALLGATHERV, 1);
-
-    MEMCHECKER(
-        ptrdiff_t ext;
-
-        mca_topo_base_neighbor_count (comm, &in_size, &out_size);
-        ompi_datatype_type_extent(recvtype, &ext);
-
-        memchecker_datatype(recvtype);
-        memchecker_comm (comm);
-        /* check whether the receive buffer is addressable. */
-        for (int i = 0; i < in_size; ++i) {
-            memchecker_call(&opal_memchecker_base_isaddressable,
-                            (char *)(recvbuf)+displs[i]*ext,
-                            recvcounts[i], recvtype);
-        }
-
-        /* check whether the actual send buffer is defined. */
-        if (MPI_IN_PLACE != sendbuf) {
-            memchecker_datatype(sendtype);
-            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-
-        /* Unrooted operation -- same checks for all ranks on both
-           intracommunicators and intercommunicators */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if (! OMPI_COMM_IS_TOPO(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TOPOLOGY,
-                                          FUNC_NAME);
-        } else if (MPI_IN_PLACE == sendbuf || MPI_IN_PLACE == recvbuf) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        } else if (MPI_DATATYPE_NULL == recvtype) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
-        }
-
-        OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-
-      /* We always define the remote group to be the same as the local
-         group in the case of an intracommunicator, so it's safe to
-         get the size of the remote group here for both intra- and
-         intercommunicators */
-
-        mca_topo_base_neighbor_count (comm, &in_size, &out_size);
-        for (int i = 0; i < in_size; ++i) {
-          if (recvcounts[i] < 0) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-          }
-        }
-
-        if (NULL == displs) {
-          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_BUFFER, FUNC_NAME);
-        }
-
-        if( OMPI_COMM_IS_CART(comm) ) {
-            const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
-            if( 0 > cart->ndims ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-        else if( OMPI_COMM_IS_GRAPH(comm) ) {
-            int degree;
-            mca_topo_base_graph_neighbors_count(comm, ompi_comm_rank(comm), &degree);
-            if( 0 > degree ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-        else if( OMPI_COMM_IS_DIST_GRAPH(comm) ) {
-            const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
-            int indegree  = dist_graph->indegree;
-            int outdegree = dist_graph->outdegree;
-            if( indegree <  0 || outdegree <  0 ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
-        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-    }
-#endif
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_neighbor_allgatherv(sendbuf, sendcount, sendtype,
-                                                recvbuf, recvcounts, displs,
-                                                recvtype, comm, comm->c_coll->coll_neighbor_allgatherv_module);
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/neighbor_allgatherv.c.in b/ompi/mpi/c/neighbor_allgatherv.c.in
new file mode 100644
index 00000000000..508b8fdad7f
--- /dev/null
+++ b/ompi/mpi/c/neighbor_allgatherv.c.in
@@ -0,0 +1,163 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010      University of Houston.  All rights reserved.
+ * Copyright (c) 2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2016 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2017 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/topo/base/base.h"
+#include "ompi/memchecker.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/mca/topo/base/base.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS neighbor_allgatherv(BUFFER sendbuf, COUNT sendcount, DATATYPE sendtype,
+                                          BUFFER_OUT recvbuf, COUNT_ARRAY recvcounts, DISP_ARRAY displs,
+                                          DATATYPE recvtype, COMM comm)
+{
+    int in_size, out_size, err = MPI_SUCCESS;
+    ompi_count_array_t recvcounts_desc;
+    ompi_disp_array_t displs_desc;
+
+    SPC_RECORD(OMPI_SPC_NEIGHBOR_ALLGATHERV, 1);
+
+    MEMCHECKER(
+        ptrdiff_t ext;
+
+        err = mca_topo_base_neighbor_count (comm, &in_size, &out_size);
+        if (MPI_SUCCESS != err) {
+            return OMPI_ERRHANDLER_INVOKE(comm, err, FUNC_NAME);
+        }
+        ompi_datatype_type_extent(recvtype, &ext);
+
+        memchecker_datatype(recvtype);
+        memchecker_comm (comm);
+        /* check whether the receive buffer is addressable. */
+        for (int i = 0; i < in_size; ++i) {
+            memchecker_call(&opal_memchecker_base_isaddressable,
+                            (char *)(recvbuf)+displs[i]*ext,
+                            recvcounts[i], recvtype);
+        }
+
+        /* check whether the actual send buffer is defined. */
+        if (MPI_IN_PLACE != sendbuf) {
+            memchecker_datatype(sendtype);
+            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+
+        /* Unrooted operation -- same checks for all ranks on both
+           intracommunicators and intercommunicators */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if (! OMPI_COMM_IS_TOPO(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TOPOLOGY,
+                                          FUNC_NAME);
+        } else if (MPI_IN_PLACE == sendbuf || MPI_IN_PLACE == recvbuf) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        } else if (MPI_DATATYPE_NULL == recvtype) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
+        }
+
+        OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+
+      /* We always define the remote group to be the same as the local
+         group in the case of an intracommunicator, so it's safe to
+         get the size of the remote group here for both intra- and
+         intercommunicators */
+
+        err = mca_topo_base_neighbor_count (comm, &in_size, &out_size);
+        if (MPI_SUCCESS != err) {
+            return OMPI_ERRHANDLER_INVOKE(comm, err, FUNC_NAME);
+        }
+        for (int i = 0; i < in_size; ++i) {
+          if (recvcounts[i] < 0) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+          }
+        }
+
+        if (NULL == displs) {
+          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_BUFFER, FUNC_NAME);
+        }
+
+        if( OMPI_COMM_IS_CART(comm) ) {
+            const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
+            if( 0 > cart->ndims ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+        else if( OMPI_COMM_IS_GRAPH(comm) ) {
+            int degree;
+            err = mca_topo_base_graph_neighbors_count(comm, ompi_comm_rank(comm), &degree);
+            if (MPI_SUCCESS != err) {
+                return OMPI_ERRHANDLER_INVOKE(comm, err, FUNC_NAME);
+            }
+            if( 0 > degree ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+        else if( OMPI_COMM_IS_DIST_GRAPH(comm) ) {
+            const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
+            int indegree  = dist_graph->indegree;
+            int outdegree = dist_graph->outdegree;
+            if( indegree <  0 || outdegree <  0 ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
+        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+    }
+#endif
+
+    /* Invoke the coll component to perform the back-end operation */
+    OMPI_COUNT_ARRAY_INIT(&recvcounts_desc, recvcounts);
+    OMPI_DISP_ARRAY_INIT(&displs_desc, displs);
+    err = comm->c_coll->coll_neighbor_allgatherv(sendbuf, sendcount, sendtype,
+                                                recvbuf, recvcounts_desc, displs_desc,
+                                                recvtype, comm, comm->c_coll->coll_neighbor_allgatherv_module);
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/neighbor_allgatherv_init.c.in b/ompi/mpi/c/neighbor_allgatherv_init.c.in
new file mode 100644
index 00000000000..24dc3c9a503
--- /dev/null
+++ b/ompi/mpi/c/neighbor_allgatherv_init.c.in
@@ -0,0 +1,149 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010      University of Houston.  All rights reserved.
+ * Copyright (c) 2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/mca/topo/base/base.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS neighbor_allgatherv_init(BUFFER sendbuf, COUNT sendcount, DATATYPE sendtype,
+                                               BUFFER_OUT recvbuf, COUNT_ARRAY recvcounts, DISP_ARRAY displs,
+                                               DATATYPE recvtype, COMM comm,
+                                               INFO info, REQUEST_INOUT request)
+{
+    int i, size, err;
+    ompi_count_array_t recvcounts_desc;
+    ompi_disp_array_t displs_desc;
+
+    SPC_RECORD(OMPI_SPC_NEIGHBOR_ALLGATHERV_INIT, 1);
+
+    MEMCHECKER(
+        ptrdiff_t ext;
+
+        size = ompi_comm_size(comm);
+        ompi_datatype_type_extent(recvtype, &ext);
+
+        memchecker_datatype(recvtype);
+        memchecker_comm (comm);
+        /* check whether the receive buffer is addressable. */
+        for (i = 0; i < size; i++) {
+            memchecker_call(&opal_memchecker_base_isaddressable,
+                            (char *)(recvbuf)+displs[i]*ext,
+                            recvcounts[i], recvtype);
+        }
+
+        /* check whether the actual send buffer is defined. */
+        if (MPI_IN_PLACE != sendbuf) {
+            memchecker_datatype(sendtype);
+            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+
+        /* Unrooted operation -- same checks for all ranks on both
+           intracommunicators and intercommunicators */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if (! OMPI_COMM_IS_TOPO(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TOPOLOGY,
+                                          FUNC_NAME);
+        } else if (MPI_IN_PLACE == sendbuf || MPI_IN_PLACE == recvbuf) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        } else if (MPI_DATATYPE_NULL == recvtype) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
+        }
+
+        OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+
+      /* We always define the remote group to be the same as the local
+         group in the case of an intracommunicator, so it's safe to
+         get the size of the remote group here for both intra- and
+         intercommunicators */
+
+        size = ompi_comm_remote_size(comm);
+        for (i = 0; i < size; ++i) {
+          if (recvcounts[i] < 0) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+          }
+        }
+
+        if (NULL == displs) {
+          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_BUFFER, FUNC_NAME);
+        }
+
+        if( OMPI_COMM_IS_CART(comm) ) {
+            const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
+            if( 0 > cart->ndims ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+        else if( OMPI_COMM_IS_GRAPH(comm) ) {
+            int degree;
+            mca_topo_base_graph_neighbors_count(comm, ompi_comm_rank(comm), &degree);
+            if( 0 > degree ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+        else if( OMPI_COMM_IS_DIST_GRAPH(comm) ) {
+            const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
+            int indegree  = dist_graph->indegree;
+            int outdegree = dist_graph->outdegree;
+            if( indegree <  0 || outdegree <  0 ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    OMPI_COUNT_ARRAY_INIT(&recvcounts_desc, recvcounts);
+    OMPI_DISP_ARRAY_INIT(&displs_desc, displs);
+    err = comm->c_coll->coll_neighbor_allgatherv_init(sendbuf, sendcount, sendtype,
+                                                      recvbuf, recvcounts_desc, displs_desc,
+                                                      recvtype, comm, info, request,
+                                                      comm->c_coll->coll_neighbor_allgatherv_init_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/neighbor_alltoall.c b/ompi/mpi/c/neighbor_alltoall.c
deleted file mode 100644
index e50885fb1c3..00000000000
--- a/ompi/mpi/c/neighbor_alltoall.c
+++ /dev/null
@@ -1,146 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2014-2015 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-#include "ompi/mca/topo/topo.h"
-#include "ompi/mca/topo/base/base.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Neighbor_alltoall = PMPI_Neighbor_alltoall
-#endif
-#define MPI_Neighbor_alltoall PMPI_Neighbor_alltoall
-#endif
-
-static const char FUNC_NAME[] = "MPI_Neighbor_alltoall";
-
-
-int MPI_Neighbor_alltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                          void *recvbuf, int recvcount, MPI_Datatype recvtype,
-                          MPI_Comm comm)
-{
-    size_t sendtype_size, recvtype_size;
-    int err;
-
-    SPC_RECORD(OMPI_SPC_NEIGHBOR_ALLTOALL, 1);
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-        if (MPI_IN_PLACE != sendbuf) {
-            memchecker_datatype(sendtype);
-            memchecker_call(&opal_memchecker_base_isdefined, (void *)sendbuf, sendcount, sendtype);
-        }
-        memchecker_datatype(recvtype);
-        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-
-        /* Unrooted operation -- same checks for all ranks on both
-           intracommunicators and intercommunicators */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if (! OMPI_COMM_IS_TOPO(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TOPOLOGY,
-                                          FUNC_NAME);
-        } else if (MPI_IN_PLACE == sendbuf || MPI_IN_PLACE == recvbuf) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtype, recvcount);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-
-        ompi_datatype_type_size(sendtype, &sendtype_size);
-        ompi_datatype_type_size(recvtype, &recvtype_size);
-        if ((sendtype_size*sendcount) != (recvtype_size*recvcount)) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TRUNCATE, FUNC_NAME);
-        }
-
-        if( OMPI_COMM_IS_CART(comm) ) {
-            const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
-            if( 0 > cart->ndims ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-        else if( OMPI_COMM_IS_GRAPH(comm) ) {
-            int degree;
-            mca_topo_base_graph_neighbors_count(comm, ompi_comm_rank(comm), &degree);
-            if( 0 > degree ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-        else if( OMPI_COMM_IS_DIST_GRAPH(comm) ) {
-            const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
-            int indegree  = dist_graph->indegree;
-            int outdegree = dist_graph->outdegree;
-            if( indegree <  0 || outdegree <  0 ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-    }
-
-    /* Do we need to do anything? */
-
-    ompi_datatype_type_size(sendtype, &sendtype_size);
-    ompi_datatype_type_size(recvtype, &recvtype_size);
-    if (((0 == sendcount) || (0 == sendtype_size)) &&
-        ((0 == recvcount) || 0 == (recvtype_size))) {
-        return MPI_SUCCESS;
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
-        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-    }
-#endif
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_neighbor_alltoall(sendbuf, sendcount, sendtype, recvbuf,
-                                              recvcount, recvtype, comm,
-                                              comm->c_coll->coll_neighbor_alltoall_module);
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/neighbor_alltoall.c.in b/ompi/mpi/c/neighbor_alltoall.c.in
new file mode 100644
index 00000000000..d015c2742ba
--- /dev/null
+++ b/ompi/mpi/c/neighbor_alltoall.c.in
@@ -0,0 +1,138 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2014-2015 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/mca/topo/base/base.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS neighbor_alltoall(BUFFER sendbuf, COUNT sendcount, DATATYPE sendtype,
+                                        BUFFER_OUT recvbuf, COUNT recvcount, DATATYPE recvtype,
+                                        COMM comm)
+{
+    size_t sendtype_size, recvtype_size;
+    int err;
+
+    SPC_RECORD(OMPI_SPC_NEIGHBOR_ALLTOALL, 1);
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+        if (MPI_IN_PLACE != sendbuf) {
+            memchecker_datatype(sendtype);
+            memchecker_call(&opal_memchecker_base_isdefined, (void *)sendbuf, sendcount, sendtype);
+        }
+        memchecker_datatype(recvtype);
+        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+
+        /* Unrooted operation -- same checks for all ranks on both
+           intracommunicators and intercommunicators */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if (! OMPI_COMM_IS_TOPO(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TOPOLOGY,
+                                          FUNC_NAME);
+        } else if (MPI_IN_PLACE == sendbuf || MPI_IN_PLACE == recvbuf) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtype, recvcount);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+
+        ompi_datatype_type_size(sendtype, &sendtype_size);
+        ompi_datatype_type_size(recvtype, &recvtype_size);
+        if ((sendtype_size*sendcount) != (recvtype_size*recvcount)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TRUNCATE, FUNC_NAME);
+        }
+
+        if( OMPI_COMM_IS_CART(comm) ) {
+            const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
+            if( 0 > cart->ndims ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+        else if( OMPI_COMM_IS_GRAPH(comm) ) {
+            int degree;
+            mca_topo_base_graph_neighbors_count(comm, ompi_comm_rank(comm), &degree);
+            if( 0 > degree ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+        else if( OMPI_COMM_IS_DIST_GRAPH(comm) ) {
+            const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
+            int indegree  = dist_graph->indegree;
+            int outdegree = dist_graph->outdegree;
+            if( indegree <  0 || outdegree <  0 ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+    }
+
+    /* Do we need to do anything? */
+
+    ompi_datatype_type_size(sendtype, &sendtype_size);
+    ompi_datatype_type_size(recvtype, &recvtype_size);
+    if (((0 == sendcount) || (0 == sendtype_size)) &&
+        ((0 == recvcount) || 0 == (recvtype_size))) {
+        return MPI_SUCCESS;
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
+        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+    }
+#endif
+
+    /* Invoke the coll component to perform the back-end operation */
+    err = comm->c_coll->coll_neighbor_alltoall(sendbuf, sendcount, sendtype, recvbuf,
+                                              recvcount, recvtype, comm,
+                                              comm->c_coll->coll_neighbor_alltoall_module);
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/neighbor_alltoall_init.c.in b/ompi/mpi/c/neighbor_alltoall_init.c.in
new file mode 100644
index 00000000000..7541a5d7824
--- /dev/null
+++ b/ompi/mpi/c/neighbor_alltoall_init.c.in
@@ -0,0 +1,123 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oak Ridge National Laboratory. All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2014-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/mca/topo/base/base.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS neighbor_alltoall_init(BUFFER sendbuf, COUNT sendcount, DATATYPE sendtype,
+                                             BUFFER_OUT recvbuf, COUNT recvcount, DATATYPE recvtype,
+                                             COMM comm, INFO info, REQUEST_INOUT request)
+{
+    size_t sendtype_size, recvtype_size;
+    int err;
+
+    SPC_RECORD(OMPI_SPC_NEIGHBOR_ALLTOALL_INIT, 1);
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+        if (MPI_IN_PLACE != sendbuf) {
+            memchecker_datatype(sendtype);
+            memchecker_call(&opal_memchecker_base_isdefined, (void *)sendbuf, sendcount, sendtype);
+        }
+        memchecker_datatype(recvtype);
+        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+
+        /* Unrooted operation -- same checks for all ranks on both
+           intracommunicators and intercommunicators */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if (! OMPI_COMM_IS_TOPO(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TOPOLOGY,
+                                          FUNC_NAME);
+        } else if (MPI_IN_PLACE == sendbuf || MPI_IN_PLACE == recvbuf) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtype, recvcount);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+
+        ompi_datatype_type_size(sendtype, &sendtype_size);
+        ompi_datatype_type_size(recvtype, &recvtype_size);
+        if ((sendtype_size*sendcount) != (recvtype_size*recvcount)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TRUNCATE, FUNC_NAME);
+        }
+
+        if( OMPI_COMM_IS_CART(comm) ) {
+            const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
+            if( 0 > cart->ndims ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+        else if( OMPI_COMM_IS_GRAPH(comm) ) {
+            int degree;
+            mca_topo_base_graph_neighbors_count(comm, ompi_comm_rank(comm), &degree);
+            if( 0 > degree ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+        else if( OMPI_COMM_IS_DIST_GRAPH(comm) ) {
+            const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
+            int indegree  = dist_graph->indegree;
+            int outdegree = dist_graph->outdegree;
+            if( indegree <  0 || outdegree <  0 ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    err = comm->c_coll->coll_neighbor_alltoall_init(sendbuf, sendcount, sendtype,
+                                                    recvbuf, recvcount, recvtype, comm,
+                                                    info, request,
+                                                    comm->c_coll->coll_neighbor_alltoall_init_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/neighbor_alltoallv.c b/ompi/mpi/c/neighbor_alltoallv.c
deleted file mode 100644
index 8b17498cd05..00000000000
--- a/ompi/mpi/c/neighbor_alltoallv.c
+++ /dev/null
@@ -1,162 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2017 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2014-2019 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-#include "ompi/mca/topo/topo.h"
-#include "ompi/mca/topo/base/base.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Neighbor_alltoallv = PMPI_Neighbor_alltoallv
-#endif
-#define MPI_Neighbor_alltoallv PMPI_Neighbor_alltoallv
-#endif
-
-static const char FUNC_NAME[] = "MPI_Neighbor_alltoallv";
-
-
-int MPI_Neighbor_alltoallv(const void *sendbuf, const int sendcounts[], const int sdispls[],
-                           MPI_Datatype sendtype, void *recvbuf,
-                           const int recvcounts[], const int rdispls[],
-                           MPI_Datatype recvtype, MPI_Comm comm)
-{
-    int i, err;
-    int indegree, outdegree;
-
-    SPC_RECORD(OMPI_SPC_NEIGHBOR_ALLTOALLV, 1);
-
-    MEMCHECKER(
-        ptrdiff_t recv_ext;
-        ptrdiff_t send_ext;
-
-        memchecker_comm(comm);
-
-        if (MPI_IN_PLACE != sendbuf) {
-            memchecker_datatype(sendtype);
-            ompi_datatype_type_extent(recvtype, &recv_ext);
-        }
-
-        memchecker_datatype(recvtype);
-        ompi_datatype_type_extent(sendtype, &send_ext);
-
-        err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree);
-        if (MPI_SUCCESS == err) {
-            if (MPI_IN_PLACE != sendbuf) {
-                for ( i = 0; i < outdegree; i++ ) {
-                    /* check if send chunks are defined. */
-                    memchecker_call(&opal_memchecker_base_isdefined,
-                                    (char *)(sendbuf)+sdispls[i]*send_ext,
-                                    sendcounts[i], sendtype);
-                }
-            }
-            for ( i = 0; i < indegree; i++ ) {
-                /* check if receive chunks are addressable. */
-                memchecker_call(&opal_memchecker_base_isaddressable,
-                                (char *)(recvbuf)+rdispls[i]*recv_ext,
-                                recvcounts[i], recvtype);
-            }
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-
-        /* Unrooted operation -- same checks for all ranks */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if (! OMPI_COMM_IS_TOPO(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-
-        err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree);
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        if (((0 < outdegree) && ((NULL == sendcounts) || (NULL == sdispls))) ||
-            ((0 < indegree) && ((NULL == recvcounts) || (NULL == rdispls))) ||
-            MPI_IN_PLACE == sendbuf || MPI_IN_PLACE == recvbuf) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-        for (i = 0; i < outdegree; ++i) {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcounts[i]);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-        for (i = 0; i < indegree; ++i) {
-            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtype, recvcounts[i]);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-
-        if( OMPI_COMM_IS_CART(comm) ) {
-            const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
-            if( 0 > cart->ndims ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-        else if( OMPI_COMM_IS_GRAPH(comm) ) {
-            int degree;
-            mca_topo_base_graph_neighbors_count(comm, ompi_comm_rank(comm), &degree);
-            if( 0 > degree ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-        else if( OMPI_COMM_IS_DIST_GRAPH(comm) ) {
-            const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
-            indegree  = dist_graph->indegree;
-            outdegree = dist_graph->outdegree;
-            if( indegree <  0 || outdegree <  0 ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
-        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-    }
-#endif
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_neighbor_alltoallv(sendbuf, sendcounts, sdispls, sendtype,
-                                               recvbuf, recvcounts, rdispls, recvtype,
-                                               comm, comm->c_coll->coll_neighbor_alltoallv_module);
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/neighbor_alltoallv.c.in b/ompi/mpi/c/neighbor_alltoallv.c.in
new file mode 100644
index 00000000000..62a2051edbd
--- /dev/null
+++ b/ompi/mpi/c/neighbor_alltoallv.c.in
@@ -0,0 +1,160 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2017 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/mca/topo/base/base.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS neighbor_alltoallv(BUFFER sendbuf, COUNT_ARRAY sendcounts, DISP_ARRAY sdispls,
+                                         DATATYPE sendtype, BUFFER_OUT recvbuf,
+                                         COUNT_ARRAY recvcounts, DISP_ARRAY rdispls,
+                                         DATATYPE recvtype, COMM comm)
+{
+    int i, err;
+    int indegree, outdegree;
+    ompi_count_array_t sendcounts_desc, recvcounts_desc;
+    ompi_disp_array_t sdispls_desc, rdispls_desc;
+
+    SPC_RECORD(OMPI_SPC_NEIGHBOR_ALLTOALLV, 1);
+
+    MEMCHECKER(
+        ptrdiff_t recv_ext;
+        ptrdiff_t send_ext;
+
+        memchecker_comm(comm);
+
+        if (MPI_IN_PLACE != sendbuf) {
+            memchecker_datatype(sendtype);
+            ompi_datatype_type_extent(recvtype, &recv_ext);
+        }
+
+        memchecker_datatype(recvtype);
+        ompi_datatype_type_extent(sendtype, &send_ext);
+
+        err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree);
+        if (MPI_SUCCESS == err) {
+            if (MPI_IN_PLACE != sendbuf) {
+                for ( i = 0; i < outdegree; i++ ) {
+                    /* check if send chunks are defined. */
+                    memchecker_call(&opal_memchecker_base_isdefined,
+                                    (char *)(sendbuf)+sdispls[i]*send_ext,
+                                    sendcounts[i], sendtype);
+                }
+            }
+            for ( i = 0; i < indegree; i++ ) {
+                /* check if receive chunks are addressable. */
+                memchecker_call(&opal_memchecker_base_isaddressable,
+                                (char *)(recvbuf)+rdispls[i]*recv_ext,
+                                recvcounts[i], recvtype);
+            }
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+
+        /* Unrooted operation -- same checks for all ranks */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if (! OMPI_COMM_IS_TOPO(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+
+        err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree);
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        if (((0 < outdegree) && ((NULL == sendcounts) || (NULL == sdispls))) ||
+            ((0 < indegree) && ((NULL == recvcounts) || (NULL == rdispls))) ||
+            MPI_IN_PLACE == sendbuf || MPI_IN_PLACE == recvbuf) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+        for (i = 0; i < outdegree; ++i) {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcounts[i]);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+        for (i = 0; i < indegree; ++i) {
+            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtype, recvcounts[i]);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+
+        if( OMPI_COMM_IS_CART(comm) ) {
+            const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
+            if( 0 > cart->ndims ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+        else if( OMPI_COMM_IS_GRAPH(comm) ) {
+            int degree;
+            mca_topo_base_graph_neighbors_count(comm, ompi_comm_rank(comm), &degree);
+            if( 0 > degree ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+        else if( OMPI_COMM_IS_DIST_GRAPH(comm) ) {
+            const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
+            indegree  = dist_graph->indegree;
+            outdegree = dist_graph->outdegree;
+            if( indegree <  0 || outdegree <  0 ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
+        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+    }
+#endif
+
+    /* Invoke the coll component to perform the back-end operation */
+    OMPI_COUNT_ARRAY_INIT(&sendcounts_desc, sendcounts);
+    OMPI_COUNT_ARRAY_INIT(&recvcounts_desc, recvcounts);
+    OMPI_DISP_ARRAY_INIT(&sdispls_desc, sdispls);
+    OMPI_DISP_ARRAY_INIT(&rdispls_desc, rdispls);
+    err = comm->c_coll->coll_neighbor_alltoallv(sendbuf, sendcounts_desc, sdispls_desc, sendtype,
+                                               recvbuf, recvcounts_desc, rdispls_desc, recvtype,
+                                               comm, comm->c_coll->coll_neighbor_alltoallv_module);
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/neighbor_alltoallv_init.c.in b/ompi/mpi/c/neighbor_alltoallv_init.c.in
new file mode 100644
index 00000000000..e5a7fcfb541
--- /dev/null
+++ b/ompi/mpi/c/neighbor_alltoallv_init.c.in
@@ -0,0 +1,154 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2017 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2023 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/mca/topo/base/base.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS neighbor_alltoallv_init(BUFFER sendbuf, COUNT_ARRAY sendcounts, DISP_ARRAY sdispls,
+                                              DATATYPE sendtype, BUFFER_OUT recvbuf, COUNT_ARRAY recvcounts,
+                                              DISP_ARRAY rdispls, DATATYPE recvtype, COMM comm,
+                                              INFO info, REQUEST_INOUT request)
+{
+    int i, err;
+    int indegree, outdegree;
+    ompi_count_array_t sendcounts_desc, recvcounts_desc;
+    ompi_disp_array_t sdispls_desc, rdispls_desc;
+
+    SPC_RECORD(OMPI_SPC_NEIGHBOR_ALLTOALLV_INIT, 1);
+
+    MEMCHECKER(
+        ptrdiff_t recv_ext;
+        ptrdiff_t send_ext;
+
+        memchecker_comm(comm);
+
+        if (MPI_IN_PLACE != sendbuf) {
+            memchecker_datatype(sendtype);
+            ompi_datatype_type_extent(recvtype, &recv_ext);
+        }
+
+        memchecker_datatype(recvtype);
+        ompi_datatype_type_extent(sendtype, &send_ext);
+
+        err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree);
+        if (MPI_SUCCESS == err) {
+            if (MPI_IN_PLACE != sendbuf) {
+                for ( i = 0; i < outdegree; i++ ) {
+                    /* check if send chunks are defined. */
+                    memchecker_call(&opal_memchecker_base_isdefined,
+                                    (char *)(sendbuf)+sdispls[i]*send_ext,
+                                    sendcounts[i], sendtype);
+                }
+            }
+            for ( i = 0; i < indegree; i++ ) {
+                /* check if receive chunks are addressable. */
+                memchecker_call(&opal_memchecker_base_isaddressable,
+                                (char *)(recvbuf)+rdispls[i]*recv_ext,
+                                recvcounts[i], recvtype);
+            }
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+
+        /* Unrooted operation -- same checks for all ranks */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if (! OMPI_COMM_IS_TOPO(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TOPOLOGY,
+                                          FUNC_NAME);
+        }
+
+        err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree);
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        if (((0 < outdegree) && ((NULL == sendcounts) || (NULL == sdispls))) ||
+            ((0 < indegree) && ((NULL == recvcounts) || (NULL == rdispls))) ||
+            MPI_IN_PLACE == sendbuf || MPI_IN_PLACE == recvbuf) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+        for (i = 0; i < outdegree; ++i) {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcounts[i]);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+        for (i = 0; i < indegree; ++i) {
+            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtype, recvcounts[i]);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+
+        if( OMPI_COMM_IS_CART(comm) ) {
+            const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
+            if( 0 > cart->ndims ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+        else if( OMPI_COMM_IS_GRAPH(comm) ) {
+            int degree;
+            mca_topo_base_graph_neighbors_count(comm, ompi_comm_rank(comm), &degree);
+            if( 0 > degree ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+        else if( OMPI_COMM_IS_DIST_GRAPH(comm) ) {
+            const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
+            indegree  = dist_graph->indegree;
+            outdegree = dist_graph->outdegree;
+            if( indegree <  0 || outdegree <  0 ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    OMPI_COUNT_ARRAY_INIT(&sendcounts_desc, sendcounts);
+    OMPI_COUNT_ARRAY_INIT(&recvcounts_desc, recvcounts);
+    OMPI_DISP_ARRAY_INIT(&sdispls_desc, sdispls);
+    OMPI_DISP_ARRAY_INIT(&rdispls_desc, rdispls);
+    err = comm->c_coll->coll_neighbor_alltoallv_init(sendbuf, sendcounts_desc, sdispls_desc,
+                                                     sendtype, recvbuf, recvcounts_desc, rdispls_desc,
+                                                     recvtype, comm, info, request,
+                                                     comm->c_coll->coll_neighbor_alltoallv_init_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/neighbor_alltoallw.c b/ompi/mpi/c/neighbor_alltoallw.c
deleted file mode 100644
index df0a48becac..00000000000
--- a/ompi/mpi/c/neighbor_alltoallw.c
+++ /dev/null
@@ -1,158 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2017 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2014-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2017      IBM Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-#include "ompi/mca/topo/topo.h"
-#include "ompi/mca/topo/base/base.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Neighbor_alltoallw = PMPI_Neighbor_alltoallw
-#endif
-#define MPI_Neighbor_alltoallw PMPI_Neighbor_alltoallw
-#endif
-
-static const char FUNC_NAME[] = "MPI_Neighbor_alltoallw";
-
-
-int MPI_Neighbor_alltoallw(const void *sendbuf, const int sendcounts[], const MPI_Aint sdispls[],
-                           const MPI_Datatype sendtypes[], void *recvbuf,
-                           const int recvcounts[], const MPI_Aint rdispls[],
-                           const MPI_Datatype recvtypes[], MPI_Comm comm)
-{
-    int i, err;
-    int indegree, outdegree;
-
-    SPC_RECORD(OMPI_SPC_NEIGHBOR_ALLTOALLW, 1);
-
-    MEMCHECKER(
-        ptrdiff_t recv_ext;
-        ptrdiff_t send_ext;
-
-        memchecker_comm(comm);
-
-        err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree);
-        if (MPI_SUCCESS == err) {
-            if (MPI_IN_PLACE != sendbuf) {
-                for ( i = 0; i < outdegree; i++ ) {
-                    memchecker_datatype(sendtypes[i]);
-
-                    ompi_datatype_type_extent(sendtypes[i], &send_ext);
-
-                    memchecker_call(&opal_memchecker_base_isdefined,
-                                    (char *)(sendbuf)+sdispls[i]*send_ext,
-                                    sendcounts[i], sendtypes[i]);
-                }
-            }
-            for ( i = 0; i < indegree; i++ ) {
-                memchecker_datatype(recvtypes[i]);
-                ompi_datatype_type_extent(recvtypes[i], &recv_ext);
-                memchecker_call(&opal_memchecker_base_isaddressable,
-                                (char *)(recvbuf)+sdispls[i]*recv_ext,
-                                recvcounts[i], recvtypes[i]);
-            }
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-
-        /* Unrooted operation -- same checks for all ranks */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if (! OMPI_COMM_IS_TOPO(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TOPOLOGY,
-                                          FUNC_NAME);
-        }
-
-        err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree);
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        if (((0 < outdegree) && ((NULL == sendcounts) || (NULL == sdispls) || (NULL == sendtypes))) ||
-            ((0 < indegree) && ((NULL == recvcounts) || (NULL == rdispls) || (NULL == recvtypes))) ||
-            MPI_IN_PLACE == sendbuf || MPI_IN_PLACE == recvbuf) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-        for (i = 0; i < outdegree; ++i) {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtypes[i], sendcounts[i]);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-        for (i = 0; i < indegree; ++i) {
-            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtypes[i], recvcounts[i]);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-
-        if( OMPI_COMM_IS_CART(comm) ) {
-            const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
-            if( 0 > cart->ndims ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-        else if( OMPI_COMM_IS_GRAPH(comm) ) {
-            int degree;
-            mca_topo_base_graph_neighbors_count(comm, ompi_comm_rank(comm), &degree);
-            if( 0 > degree ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-        else if( OMPI_COMM_IS_DIST_GRAPH(comm) ) {
-            const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
-            indegree  = dist_graph->indegree;
-            outdegree = dist_graph->outdegree;
-            if( indegree <  0 || outdegree <  0 ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
-        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-    }
-#endif
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_neighbor_alltoallw(sendbuf, sendcounts, sdispls, sendtypes,
-                                               recvbuf, recvcounts, rdispls, recvtypes,
-                                               comm, comm->c_coll->coll_neighbor_alltoallw_module);
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/neighbor_alltoallw.c.in b/ompi/mpi/c/neighbor_alltoallw.c.in
new file mode 100644
index 00000000000..e44c1ed260a
--- /dev/null
+++ b/ompi/mpi/c/neighbor_alltoallw.c.in
@@ -0,0 +1,157 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2017 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/mca/topo/base/base.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS neighbor_alltoallw(BUFFER sendbuf, COUNT_ARRAY sendcounts, AINT_ARRAY sdispls,
+                                         DATATYPE_ARRAY sendtypes, BUFFER_OUT recvbuf,
+                                         COUNT_ARRAY recvcounts, AINT_ARRAY rdispls,
+                                         DATATYPE_ARRAY recvtypes, COMM comm)
+{
+    int i, err;
+    int indegree, outdegree;
+    ompi_count_array_t sendcounts_desc, recvcounts_desc;
+    ompi_disp_array_t sdispls_desc, rdispls_desc;
+
+    SPC_RECORD(OMPI_SPC_NEIGHBOR_ALLTOALLW, 1);
+
+    MEMCHECKER(
+        ptrdiff_t recv_ext;
+        ptrdiff_t send_ext;
+
+        memchecker_comm(comm);
+
+        err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree);
+        if (MPI_SUCCESS == err) {
+            if (MPI_IN_PLACE != sendbuf) {
+                for ( i = 0; i < outdegree; i++ ) {
+                    memchecker_datatype(sendtypes[i]);
+
+                    ompi_datatype_type_extent(sendtypes[i], &send_ext);
+
+                    memchecker_call(&opal_memchecker_base_isdefined,
+                                    (char *)(sendbuf)+sdispls[i]*send_ext,
+                                    sendcounts[i], sendtypes[i]);
+                }
+            }
+            for ( i = 0; i < indegree; i++ ) {
+                memchecker_datatype(recvtypes[i]);
+                ompi_datatype_type_extent(recvtypes[i], &recv_ext);
+                memchecker_call(&opal_memchecker_base_isaddressable,
+                                (char *)(recvbuf)+sdispls[i]*recv_ext,
+                                recvcounts[i], recvtypes[i]);
+            }
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+
+        /* Unrooted operation -- same checks for all ranks */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if (! OMPI_COMM_IS_TOPO(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TOPOLOGY,
+                                          FUNC_NAME);
+        }
+
+        err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree);
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        if (((0 < outdegree) && ((NULL == sendcounts) || (NULL == sdispls) || (NULL == sendtypes))) ||
+            ((0 < indegree) && ((NULL == recvcounts) || (NULL == rdispls) || (NULL == recvtypes))) ||
+            MPI_IN_PLACE == sendbuf || MPI_IN_PLACE == recvbuf) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+        for (i = 0; i < outdegree; ++i) {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtypes[i], sendcounts[i]);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+        for (i = 0; i < indegree; ++i) {
+            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtypes[i], recvcounts[i]);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+
+        if( OMPI_COMM_IS_CART(comm) ) {
+            const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
+            if( 0 > cart->ndims ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+        else if( OMPI_COMM_IS_GRAPH(comm) ) {
+            int degree;
+            mca_topo_base_graph_neighbors_count(comm, ompi_comm_rank(comm), &degree);
+            if( 0 > degree ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+        else if( OMPI_COMM_IS_DIST_GRAPH(comm) ) {
+            const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
+            indegree  = dist_graph->indegree;
+            outdegree = dist_graph->outdegree;
+            if( indegree <  0 || outdegree <  0 ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
+        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+    }
+#endif
+
+    /* Invoke the coll component to perform the back-end operation */
+    OMPI_COUNT_ARRAY_INIT(&sendcounts_desc, sendcounts);
+    OMPI_DISP_ARRAY_INIT(&sdispls_desc, sdispls);
+    OMPI_COUNT_ARRAY_INIT(&recvcounts_desc, recvcounts);
+    OMPI_DISP_ARRAY_INIT(&rdispls_desc, rdispls);
+    err = comm->c_coll->coll_neighbor_alltoallw(sendbuf, sendcounts_desc, sdispls_desc, sendtypes,
+                                               recvbuf, recvcounts_desc, rdispls_desc, recvtypes,
+                                               comm, comm->c_coll->coll_neighbor_alltoallw_module);
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
+
+
diff --git a/ompi/mpiext/pcollreq/c/neighbor_alltoallw_init.c b/ompi/mpi/c/neighbor_alltoallw_init.c
similarity index 80%
rename from ompi/mpiext/pcollreq/c/neighbor_alltoallw_init.c
rename to ompi/mpi/c/neighbor_alltoallw_init.c
index bece4e4a97b..808fc77a2f1 100644
--- a/ompi/mpiext/pcollreq/c/neighbor_alltoallw_init.c
+++ b/ompi/mpi/c/neighbor_alltoallw_init.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
+ * Copyright (c) 2004-2022 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -13,7 +13,7 @@
  * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2012-2017 Los Alamos National Security, LLC.  All rights
  *                         reserved.
- * Copyright (c) 2014-2019 Research Organization for Information Science
+ * Copyright (c) 2014-2023 Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2017      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
@@ -36,25 +36,26 @@
 #include "ompi/mca/topo/topo.h"
 #include "ompi/mca/topo/base/base.h"
 #include "ompi/runtime/ompi_spc.h"
-#include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_Neighbor_alltoallw_init = PMPIX_Neighbor_alltoallw_init
+#pragma weak MPI_Neighbor_alltoallw_init = PMPI_Neighbor_alltoallw_init
 #endif
-#define MPIX_Neighbor_alltoallw_init PMPIX_Neighbor_alltoallw_init
+#define MPI_Neighbor_alltoallw_init PMPI_Neighbor_alltoallw_init
 #endif
 
-static const char FUNC_NAME[] = "MPIX_Neighbor_alltoallw_init";
+static const char FUNC_NAME[] = "MPI_Neighbor_alltoallw_init";
 
 
-int MPIX_Neighbor_alltoallw_init(const void *sendbuf, const int sendcounts[], const MPI_Aint sdispls[],
-                                 const MPI_Datatype sendtypes[], void *recvbuf, const int recvcounts[],
-                                 const MPI_Aint rdispls[], const MPI_Datatype recvtypes[], MPI_Comm comm,
-                                 MPI_Info info, MPI_Request *request)
+int MPI_Neighbor_alltoallw_init(const void *sendbuf, const int sendcounts[], const MPI_Aint sdispls[],
+                                const MPI_Datatype sendtypes[], void *recvbuf, const int recvcounts[],
+                                const MPI_Aint rdispls[], const MPI_Datatype recvtypes[], MPI_Comm comm,
+                                MPI_Info info, MPI_Request *request)
 {
     int i, err;
     int indegree, outdegree;
+    ompi_count_array_t sendcounts_desc, recvcounts_desc;
+    ompi_disp_array_t sdispls_desc, rdispls_desc;
 
     SPC_RECORD(OMPI_SPC_NEIGHBOR_ALLTOALLW_INIT, 1);
 
@@ -103,14 +104,13 @@ int MPIX_Neighbor_alltoallw_init(const void *sendbuf, const int sendcounts[], co
                                           FUNC_NAME);
         }
 
-        if ((NULL == sendcounts) || (NULL == sdispls) || (NULL == sendtypes) ||
-            (NULL == recvcounts) || (NULL == rdispls) || (NULL == recvtypes) ||
+        err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree);
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        if (((0 < outdegree) && ((NULL == sendcounts) || (NULL == sdispls) || (NULL == sendtypes))) ||
+            ((0 < indegree) && ((NULL == recvcounts) || (NULL == rdispls) || (NULL == recvtypes))) ||
             MPI_IN_PLACE == sendbuf || MPI_IN_PLACE == recvbuf) {
             return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
         }
-
-        err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree);
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
         for (i = 0; i < outdegree; ++i) {
             OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtypes[i], sendcounts[i]);
             OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
@@ -144,12 +144,16 @@ int MPIX_Neighbor_alltoallw_init(const void *sendbuf, const int sendcounts[], co
     }
 
     /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_neighbor_alltoallw_init(sendbuf, sendcounts, sdispls, sendtypes,
-                                                     recvbuf, recvcounts, rdispls, recvtypes, comm,
+    OMPI_COUNT_ARRAY_INIT(&sendcounts_desc, sendcounts);
+    OMPI_COUNT_ARRAY_INIT(&recvcounts_desc, recvcounts);
+    OMPI_DISP_ARRAY_INIT(&sdispls_desc, sdispls);
+    OMPI_DISP_ARRAY_INIT(&rdispls_desc, rdispls);
+    err = comm->c_coll->coll_neighbor_alltoallw_init(sendbuf, sendcounts_desc, sdispls_desc, sendtypes,
+                                                     recvbuf, recvcounts_desc, rdispls_desc, recvtypes, comm,
                                                      info, request,
                                                      comm->c_coll->coll_neighbor_alltoallw_init_module);
     if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_datatypes_w(*request, sendtypes, recvtypes);
+        ompi_coll_base_retain_datatypes_w(*request, sendtypes, recvtypes, true);
     }
     OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
 }
diff --git a/ompi/mpi/c/neighbor_alltoallw_init.c.in b/ompi/mpi/c/neighbor_alltoallw_init.c.in
new file mode 100644
index 00000000000..a51fb2f7e1c
--- /dev/null
+++ b/ompi/mpi/c/neighbor_alltoallw_init.c.in
@@ -0,0 +1,152 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2022 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2017 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2023 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/mca/topo/base/base.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS neighbor_alltoallw_init(BUFFER sendbuf, COUNT_ARRAY sendcounts, AINT_ARRAY sdispls,
+                                              DATATYPE_ARRAY sendtypes, BUFFER_OUT recvbuf, COUNT_ARRAY recvcounts,
+                                              AINT_ARRAY rdispls, DATATYPE_ARRAY recvtypes, COMM comm,
+                                              INFO info, REQUEST_INOUT request)
+{
+    int i, err;
+    int indegree, outdegree;
+    ompi_count_array_t sendcounts_desc, recvcounts_desc;
+    ompi_disp_array_t sdispls_desc, rdispls_desc;
+
+    SPC_RECORD(OMPI_SPC_NEIGHBOR_ALLTOALLW_INIT, 1);
+
+    MEMCHECKER(
+        ptrdiff_t recv_ext;
+        ptrdiff_t send_ext;
+
+        memchecker_comm(comm);
+
+        err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree);
+        if (MPI_SUCCESS == err) {
+            if (MPI_IN_PLACE != sendbuf) {
+                for ( i = 0; i < outdegree; i++ ) {
+                    memchecker_datatype(sendtypes[i]);
+
+                    ompi_datatype_type_extent(sendtypes[i], &send_ext);
+
+                    memchecker_call(&opal_memchecker_base_isdefined,
+                                    (char *)(sendbuf)+sdispls[i]*send_ext,
+                                    sendcounts[i], sendtypes[i]);
+                }
+            }
+            for ( i = 0; i < indegree; i++ ) {
+                memchecker_datatype(recvtypes[i]);
+
+                ompi_datatype_type_extent(recvtypes[i], &recv_ext);
+
+                memchecker_call(&opal_memchecker_base_isaddressable,
+                                (char *)(recvbuf)+sdispls[i]*recv_ext,
+                                recvcounts[i], recvtypes[i]);
+            }
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+
+        /* Unrooted operation -- same checks for all ranks */
+
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if (! OMPI_COMM_IS_TOPO(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TOPOLOGY,
+                                          FUNC_NAME);
+        }
+
+        err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree);
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        if (((0 < outdegree) && ((NULL == sendcounts) || (NULL == sdispls) || (NULL == sendtypes))) ||
+            ((0 < indegree) && ((NULL == recvcounts) || (NULL == rdispls) || (NULL == recvtypes))) ||
+            MPI_IN_PLACE == sendbuf || MPI_IN_PLACE == recvbuf) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+        for (i = 0; i < outdegree; ++i) {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtypes[i], sendcounts[i]);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+        for (i = 0; i < indegree; ++i) {
+            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtypes[i], recvcounts[i]);
+            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+
+        if( OMPI_COMM_IS_CART(comm) ) {
+            const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
+            if( 0 > cart->ndims ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+        else if( OMPI_COMM_IS_GRAPH(comm) ) {
+            int degree;
+            mca_topo_base_graph_neighbors_count(comm, ompi_comm_rank(comm), &degree);
+            if( 0 > degree ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+        else if( OMPI_COMM_IS_DIST_GRAPH(comm) ) {
+            const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
+            indegree  = dist_graph->indegree;
+            outdegree = dist_graph->outdegree;
+            if( indegree <  0 || outdegree <  0 ) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    OMPI_COUNT_ARRAY_INIT(&sendcounts_desc, sendcounts);
+    OMPI_DISP_ARRAY_INIT(&sdispls_desc, sdispls);
+    OMPI_COUNT_ARRAY_INIT(&recvcounts_desc, recvcounts);
+    OMPI_DISP_ARRAY_INIT(&rdispls_desc, rdispls);
+    err = comm->c_coll->coll_neighbor_alltoallw_init(sendbuf, sendcounts_desc, sdispls_desc, sendtypes,
+                                                     recvbuf, recvcounts_desc, rdispls_desc, recvtypes, comm,
+                                                     info, request,
+                                                     comm->c_coll->coll_neighbor_alltoallw_init_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_datatypes_w(*request, sendtypes, recvtypes, true);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/op_c2f.c b/ompi/mpi/c/op_c2f.c
deleted file mode 100644
index bf09306c21e..00000000000
--- a/ompi/mpi/c/op_c2f.c
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mpi/fortran/base/fint_2_int.h"
-#include "ompi/op/op.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Op_c2f = PMPI_Op_c2f
-#endif
-#define MPI_Op_c2f PMPI_Op_c2f
-#endif
-
-static const char FUNC_NAME[] = "MPI_Op_c2f";
-
-
-MPI_Fint MPI_Op_c2f(MPI_Op op)
-{
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (NULL == op) {
-            return OMPI_INT_2_FINT(-1);
-        }
-    }
-
-    return OMPI_INT_2_FINT(op->o_f_to_c_index);
-}
diff --git a/ompi/mpi/c/op_c2f.c.in b/ompi/mpi/c/op_c2f.c.in
new file mode 100644
index 00000000000..be3a49fe04c
--- /dev/null
+++ b/ompi/mpi/c/op_c2f.c.in
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+#include "ompi/op/op.h"
+
+PROTOTYPE FINT op_c2f(OP op)
+{
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (NULL == op) {
+            return OMPI_INT_2_FINT(-1);
+        }
+    }
+
+    return OMPI_INT_2_FINT(op->o_f_to_c_index);
+}
diff --git a/ompi/mpi/c/op_commutative.c b/ompi/mpi/c/op_commutative.c
deleted file mode 100644
index 2bd6ea94cd7..00000000000
--- a/ompi/mpi/c/op_commutative.c
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2009-2014 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/op/op.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Op_commutative = PMPI_Op_commutative
-#endif
-#define MPI_Op_commutative PMPI_Op_commutative
-#endif
-
-static const char FUNC_NAME[] = "MPI_Op_commutative";
-
-
-int MPI_Op_commutative(MPI_Op op, int *commute)
-{
-    /* Error checking */
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == op || MPI_OP_NULL == op) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_OP,
-                                          FUNC_NAME);
-        }
-        if (NULL == commute) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-    }
-
-    /* We have a valid op, get the flag */
-
-    *commute = ompi_op_is_commute(op);
-
-    /* All done */
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/op_commutative.c.in b/ompi/mpi/c/op_commutative.c.in
new file mode 100644
index 00000000000..0346036efee
--- /dev/null
+++ b/ompi/mpi/c/op_commutative.c.in
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2009-2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/op/op.h"
+
+PROTOTYPE ERROR_CLASS op_commutative(OP op, INT_OUT commute)
+{
+    /* Error checking */
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == op || MPI_OP_NULL == op) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_OP,
+                                          FUNC_NAME);
+        }
+        if (NULL == commute) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+    /* We have a valid op, get the flag */
+
+    *commute = ompi_op_is_commute(op);
+
+    /* All done */
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/op_create.c b/ompi/mpi/c/op_create.c
deleted file mode 100644
index 28e00222ad9..00000000000
--- a/ompi/mpi/c/op_create.c
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008-2009 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/op/op.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Op_create = PMPI_Op_create
-#endif
-#define MPI_Op_create PMPI_Op_create
-#endif
-
-static const char FUNC_NAME[] = "MPI_Op_create";
-
-
-int MPI_Op_create(MPI_User_function * function, int commute, MPI_Op * op)
-{
-    int err = MPI_SUCCESS;
-
-    /* Error checking */
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == op) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_OP,
-                                          FUNC_NAME);
-        } else if (NULL == function) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-    }
-
-    /* Create and cache the op.  Sets a refcount of 1. */
-
-    *op = ompi_op_create_user(OPAL_INT_TO_BOOL(commute),
-                              (ompi_op_fortran_handler_fn_t *) function);
-    if (NULL == *op) {
-        err = MPI_ERR_INTERN;
-    }
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, MPI_ERR_INTERN, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/op_create.c.in b/ompi/mpi/c/op_create.c.in
new file mode 100644
index 00000000000..b3a39473ce1
--- /dev/null
+++ b/ompi/mpi/c/op_create.c.in
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008-2009 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2025      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/op/op.h"
+
+PROTOTYPE ERROR_CLASS op_create(USER_FUNCTION function, INT commute, OP_OUT op)
+{
+    int err = MPI_SUCCESS;
+#if OMPI_BIGCOUNT_SRC
+    bool bc_op = true;
+#else
+    bool bc_op = false;
+#endif
+
+    /* Error checking */
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == op) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_OP,
+                                          FUNC_NAME);
+        } else if (NULL == function) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+    /* Create and cache the op.  Sets a refcount of 1. */
+
+    *op = ompi_op_create_user(OPAL_INT_TO_BOOL(commute),
+                              bc_op,
+                              (ompi_op_fortran_handler_fn_t *) function);
+    if (NULL == *op) {
+        err = MPI_ERR_INTERN;
+    }
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, MPI_ERR_INTERN, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/op_f2c.c b/ompi/mpi/c/op_f2c.c
deleted file mode 100644
index d9be1f240b0..00000000000
--- a/ompi/mpi/c/op_f2c.c
+++ /dev/null
@@ -1,61 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2007 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mpi/fortran/base/fint_2_int.h"
-#include "ompi/op/op.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Op_f2c = PMPI_Op_f2c
-#endif
-#define MPI_Op_f2c PMPI_Op_f2c
-#endif
-
-static const char FUNC_NAME[] = "MPI_Op_f2c";
-
-
-MPI_Op MPI_Op_f2c(MPI_Fint op_f)
-{
-    int op_index = OMPI_FINT_2_INT(op_f);
-
-    /* Error checking */
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-    }
-
-    /* Per MPI-2:4.12.4, do not invoke an error handler if we get an
-       invalid fortran handle.  If we get an invalid fortran handle,
-       return an invalid C handle. */
-
-    if (op_index < 0 ||
-        op_index >=
-        opal_pointer_array_get_size(ompi_op_f_to_c_table)) {
-        return NULL;
-    }
-
-    return (MPI_Op)opal_pointer_array_get_item(ompi_op_f_to_c_table, op_index);
-}
diff --git a/ompi/mpi/c/op_f2c.c.in b/ompi/mpi/c/op_f2c.c.in
new file mode 100644
index 00000000000..59812d893d2
--- /dev/null
+++ b/ompi/mpi/c/op_f2c.c.in
@@ -0,0 +1,53 @@
+/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+#include "ompi/op/op.h"
+
+PROTOTYPE OP op_f2c(FINT op_f)
+{
+    int op_index = OMPI_FINT_2_INT(op_f);
+
+    /* Error checking */
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+    }
+
+    /* Per MPI-2:4.12.4, do not invoke an error handler if we get an
+       invalid fortran handle.  If we get an invalid fortran handle,
+       return an invalid C handle. */
+
+    if (op_index < 0 ||
+        op_index >=
+        opal_pointer_array_get_size(ompi_op_f_to_c_table)) {
+        return NULL;
+    }
+
+    return (MPI_Op)opal_pointer_array_get_item(ompi_op_f_to_c_table, op_index);
+}
diff --git a/ompi/mpi/c/op_free.c b/ompi/mpi/c/op_free.c
deleted file mode 100644
index 8920f271fd7..00000000000
--- a/ompi/mpi/c/op_free.c
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/op/op.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Op_free = PMPI_Op_free
-#endif
-#define MPI_Op_free PMPI_Op_free
-#endif
-
-static const char FUNC_NAME[] = "MPI_Op_free";
-
-
-int MPI_Op_free(MPI_Op *op)
-{
-  /* Error checking */
-
-  if (MPI_PARAM_CHECK) {
-    OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-    if (NULL == op ||
-        ompi_op_is_intrinsic(*op)) {
-      return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_OP,
-                                    FUNC_NAME);
-    }
-  }
-
-  /* We have a valid op, release it */
-
-  OBJ_RELEASE(*op);
-  *op = MPI_OP_NULL;
-
-  /* All done */
-
-  return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/op_free.c.in b/ompi/mpi/c/op_free.c.in
new file mode 100644
index 00000000000..93668a28abc
--- /dev/null
+++ b/ompi/mpi/c/op_free.c.in
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/op/op.h"
+
+PROTOTYPE ERROR_CLASS op_free(OP_OUT op)
+{
+  /* Error checking */
+
+  if (MPI_PARAM_CHECK) {
+    OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+    if (NULL == op ||
+        ompi_op_is_intrinsic(*op)) {
+      return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_OP,
+                                    FUNC_NAME);
+    }
+  }
+
+  /* We have a valid op, release it */
+
+  OBJ_RELEASE(*op);
+  *op = MPI_OP_NULL;
+
+  /* All done */
+
+  return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/open_port.c b/ompi/mpi/c/open_port.c
deleted file mode 100644
index f77ee6ae583..00000000000
--- a/ompi/mpi/c/open_port.c
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Intel, Inc. All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-#include "ompi/dpm/dpm.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Open_port = PMPI_Open_port
-#endif
-#define MPI_Open_port PMPI_Open_port
-#endif
-
-static const char FUNC_NAME[] = "MPI_Open_port";
-
-
-int MPI_Open_port(MPI_Info info, char *port_name)
-{
-    int rc;
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if ( NULL == port_name ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-        if (NULL == info || ompi_info_is_freed(info)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
-                                          FUNC_NAME);
-        }
-    }
-
-    if ( MPI_INFO_NULL != info ) {
-        /* in theory, they user might tell us here
-           how to establish the address. Since our communication
-           is relying on OOB, we probably won't use the info-object.
-
-           Potential values defined in MPI-2:
-           - "ip_port"    : value contains IP port number
-           - "ip_address" : value contains IP address
-        */
-    }
-
-    rc = ompi_dpm_open_port(port_name);
-
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/open_port.c.in b/ompi/mpi/c/open_port.c.in
new file mode 100644
index 00000000000..5020da632b2
--- /dev/null
+++ b/ompi/mpi/c/open_port.c.in
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Intel, Inc. All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include "ompi/dpm/dpm.h"
+
+PROTOTYPE ERROR_CLASS open_port(INFO info, STRING_OUT port_name)
+{
+    int rc;
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if ( NULL == port_name ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+        if (NULL == info || ompi_info_is_freed(info)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
+                                          FUNC_NAME);
+        }
+    }
+
+    if ( MPI_INFO_NULL != info ) {
+        /* in theory, they user might tell us here
+           how to establish the address. Since our communication
+           is relying on OOB, we probably won't use the info-object.
+
+           Potential values defined in MPI-2:
+           - "ip_port"    : value contains IP port number
+           - "ip_address" : value contains IP address
+        */
+    }
+
+    rc = ompi_dpm_open_port(port_name);
+
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/pack.c b/ompi/mpi/c/pack.c
deleted file mode 100644
index de70bd796bb..00000000000
--- a/ompi/mpi/c/pack.c
+++ /dev/null
@@ -1,125 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2021      IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "opal/datatype/opal_convertor.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Pack = PMPI_Pack
-#endif
-#define MPI_Pack PMPI_Pack
-#endif
-
-static const char FUNC_NAME[] = "MPI_Pack";
-
-
-int MPI_Pack(const void *inbuf, int incount, MPI_Datatype datatype,
-             void *outbuf, int outsize, int *position, MPI_Comm comm)
-{
-    int rc = MPI_SUCCESS, ret;
-    opal_convertor_t local_convertor;
-    struct iovec invec;
-    unsigned int iov_count;
-    size_t size;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_call(&opal_memchecker_base_isdefined, inbuf, incount, datatype);
-        memchecker_call(&opal_memchecker_base_isaddressable, (void *)((char *)outbuf + *position), outsize, MPI_PACKED);
-        memchecker_comm(comm);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
-        } else if ((NULL == outbuf) || (NULL == position)) {  /* inbuf can be MPI_BOTTOM */
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        } else if (incount < 0) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-        } else if (outsize < 0) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-        OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, incount);
-        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-        OMPI_CHECK_USER_BUFFER(rc, inbuf, datatype, incount);
-        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-    }
-
-    /*
-     * If a datatype's description contains a single element that describes
-     * a large vector that path is reasonably optimized in pack/unpack. On
-     * the other hand if the count and datatype combined describe the same
-     * vector, that gets processed one element at a time.
-     *
-     * So at the top level we morph the call if the count and datatype look
-     * like a good vector.
-     */
-    ompi_datatype_consolidate_t dtmod;
-    rc = ompi_datatype_consolidate_create(incount, datatype, &dtmod,
-        OMPI_DATATYPE_CONSOLIDATE_THRESHOLD);
-    OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-
-    OBJ_CONSTRUCT( &local_convertor, opal_convertor_t );
-    /* the resulting convertor will be set to the position ZERO */
-    opal_convertor_copy_and_prepare_for_send( ompi_mpi_local_convertor,
-                                              &(dtmod.dt->super), dtmod.count,
-                                              (void *) inbuf, 0, &local_convertor );
-
-    /* Check for truncation */
-    opal_convertor_get_packed_size( &local_convertor, &size );
-    if( (*position + size) > (unsigned int)outsize ) {  /* we can cast as we already checked for < 0 */
-        OBJ_DESTRUCT( &local_convertor );
-        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TRUNCATE, FUNC_NAME);
-    }
-
-    /* Prepare the iovec with all informations */
-    invec.iov_base = (char*) outbuf + (*position);
-    invec.iov_len = size;
-
-    /* Do the actual packing */
-    iov_count = 1;
-    ret = opal_convertor_pack( &local_convertor, &invec, &iov_count, &size );
-    *position += size;
-    OBJ_DESTRUCT( &local_convertor );
-
-    rc = ompi_datatype_consolidate_free(&dtmod);
-    OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-
-    /* All done.  Note that the convertor returns 1 upon success, not
-       OPAL_SUCCESS. */
-    if (1 != ret) {
-        rc = OMPI_ERROR;
-    }
-    OMPI_ERRHANDLER_RETURN(rc, comm, MPI_ERR_UNKNOWN, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/pack.c.in b/ompi/mpi/c/pack.c.in
new file mode 100644
index 00000000000..69a0bb358eb
--- /dev/null
+++ b/ompi/mpi/c/pack.c.in
@@ -0,0 +1,117 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2018 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2021      IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "opal/datatype/opal_convertor.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS pack(BUFFER inbuf, COUNT incount, DATATYPE datatype,
+                           BUFFER_OUT outbuf, COUNT outsize, COUNT_OUT position, COMM comm)
+{
+    int rc = MPI_SUCCESS, ret;
+    opal_convertor_t local_convertor;
+    struct iovec invec;
+    unsigned int iov_count;
+    size_t size;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_call(&opal_memchecker_base_isdefined, inbuf, incount, datatype);
+        memchecker_call(&opal_memchecker_base_isaddressable, (void *)((char *)outbuf + *position), outsize, MPI_PACKED);
+        memchecker_comm(comm);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+        } else if ((NULL == outbuf) || (NULL == position)) {  /* inbuf can be MPI_BOTTOM */
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        } else if (incount < 0) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+        } else if (outsize < 0) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+        OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, incount);
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+        OMPI_CHECK_USER_BUFFER(rc, inbuf, datatype, incount);
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+    /*
+     * If a datatype's description contains a single element that describes
+     * a large vector that path is reasonably optimized in pack/unpack. On
+     * the other hand if the count and datatype combined describe the same
+     * vector, that gets processed one element at a time.
+     *
+     * So at the top level we morph the call if the count and datatype look
+     * like a good vector.
+     */
+    ompi_datatype_consolidate_t dtmod;
+    rc = ompi_datatype_consolidate_create(incount, datatype, &dtmod,
+        OMPI_DATATYPE_CONSOLIDATE_THRESHOLD);
+    OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+
+    OBJ_CONSTRUCT( &local_convertor, opal_convertor_t );
+    /* the resulting convertor will be set to the position ZERO */
+    opal_convertor_copy_and_prepare_for_send( ompi_mpi_local_convertor,
+                                              &(dtmod.dt->super), dtmod.count,
+                                              (void *) inbuf, 0, &local_convertor );
+
+    /* Check for truncation */
+    opal_convertor_get_packed_size( &local_convertor, &size );
+    if( (*position + size) > (unsigned int)outsize ) {  /* we can cast as we already checked for < 0 */
+        OBJ_DESTRUCT( &local_convertor );
+        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TRUNCATE, FUNC_NAME);
+    }
+
+    /* Prepare the iovec with all information */
+    invec.iov_base = (char*) outbuf + (*position);
+    invec.iov_len = size;
+
+    /* Do the actual packing */
+    iov_count = 1;
+    ret = opal_convertor_pack( &local_convertor, &invec, &iov_count, &size );
+    *position += size;
+    OBJ_DESTRUCT( &local_convertor );
+
+    rc = ompi_datatype_consolidate_free(&dtmod);
+    OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+
+    /* All done.  Note that the convertor returns 1 upon success, not
+       OPAL_SUCCESS. */
+    if (1 != ret) {
+        rc = OMPI_ERROR;
+    }
+    OMPI_ERRHANDLER_RETURN(rc, comm, MPI_ERR_UNKNOWN, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/pack_external.c b/ompi/mpi/c/pack_external.c
deleted file mode 100644
index f8c59485134..00000000000
--- a/ompi/mpi/c/pack_external.c
+++ /dev/null
@@ -1,76 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2017 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "opal/datatype/opal_convertor.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Pack_external = PMPI_Pack_external
-#endif
-#define MPI_Pack_external PMPI_Pack_external
-#endif
-
-static const char FUNC_NAME[] = "MPI_Pack_external";
-
-
-int MPI_Pack_external(const char datarep[], const void *inbuf, int incount,
-                      MPI_Datatype datatype, void *outbuf,
-                      MPI_Aint outsize, MPI_Aint *position)
-{
-    int rc = MPI_SUCCESS;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_call(&opal_memchecker_base_isdefined, inbuf, incount, datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if ((NULL == outbuf) || (NULL == position)) {  /* inbuf can be MPI_BOTTOM */
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-        } else if (incount < 0) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COUNT, FUNC_NAME);
-        } else if (outsize < 0) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-        }
-        OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, incount);
-        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
-        OMPI_CHECK_USER_BUFFER(rc, inbuf, datatype, incount);
-        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
-    }
-
-    rc = ompi_datatype_pack_external(datarep, inbuf, incount,
-                                     datatype, outbuf, 
-                                     outsize, position);
-
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/pack_external.c.in b/ompi/mpi/c/pack_external.c.in
new file mode 100644
index 00000000000..cc880e91ee7
--- /dev/null
+++ b/ompi/mpi/c/pack_external.c.in
@@ -0,0 +1,68 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2017 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "opal/datatype/opal_convertor.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS pack_external(STRING datarep, BUFFER inbuf, COUNT incount,
+                                    DATATYPE datatype, BUFFER_OUT outbuf,
+                                    AINT_COUNT outsize, AINT_COUNT_OUT position)
+{
+    int rc = MPI_SUCCESS;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_call(&opal_memchecker_base_isdefined, inbuf, incount, datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if ((NULL == outbuf) || (NULL == position)) {  /* inbuf can be MPI_BOTTOM */
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+        } else if (incount < 0) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COUNT, FUNC_NAME);
+        } else if (outsize < 0) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+        }
+        OMPI_CHECK_DATATYPE_FOR_SEND(rc, datatype, incount);
+        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
+        OMPI_CHECK_USER_BUFFER(rc, inbuf, datatype, incount);
+        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
+    }
+
+    rc = ompi_datatype_pack_external(datarep, inbuf, incount,
+                                     datatype, outbuf, 
+                                     outsize, (MPI_Aint *)position);
+
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/pack_external_size.c b/ompi/mpi/c/pack_external_size.c
deleted file mode 100644
index 024aaf49de3..00000000000
--- a/ompi/mpi/c/pack_external_size.c
+++ /dev/null
@@ -1,67 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2017 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "opal/datatype/opal_convertor.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Pack_external_size = PMPI_Pack_external_size
-#endif
-#define MPI_Pack_external_size PMPI_Pack_external_size
-#endif
-
-static const char FUNC_NAME[] = "MPI_Pack_external_size";
-
-
-int MPI_Pack_external_size(const char datarep[], int incount,
-                           MPI_Datatype datatype, MPI_Aint *size)
-{
-    int rc = MPI_SUCCESS;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == size) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-        } else if (MPI_DATATYPE_NULL == datatype || NULL == datatype) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE, FUNC_NAME);
-        }
-    }
-
-    rc = ompi_datatype_pack_external_size(datarep, incount,
-                                          datatype, size);
-
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/pack_external_size.c.in b/ompi/mpi/c/pack_external_size.c.in
new file mode 100644
index 00000000000..8ff195bf34f
--- /dev/null
+++ b/ompi/mpi/c/pack_external_size.c.in
@@ -0,0 +1,61 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2017 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "opal/datatype/opal_convertor.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS pack_external_size(STRING datarep, COUNT incount,
+                                         DATATYPE datatype, AINT_COUNT_OUT size)
+{
+    int rc = MPI_SUCCESS;
+    MPI_Aint tmp_size = 0;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == size) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+        } else if (MPI_DATATYPE_NULL == datatype || NULL == datatype) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE, FUNC_NAME);
+        }
+    }
+
+    rc = ompi_datatype_pack_external_size(datarep, incount,
+                                          datatype, &tmp_size);
+    *size = tmp_size;
+
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/pack_size.c b/ompi/mpi/c/pack_size.c
deleted file mode 100644
index 306dbcf2c6a..00000000000
--- a/ompi/mpi/c/pack_size.c
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "opal/datatype/opal_convertor.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Pack_size = PMPI_Pack_size
-#endif
-#define MPI_Pack_size PMPI_Pack_size
-#endif
-
-static const char FUNC_NAME[] = "MPI_Pack_size";
-
-int MPI_Pack_size(int incount, MPI_Datatype datatype, MPI_Comm comm,
-                  int *size)
-{
-    opal_convertor_t local_convertor;
-    size_t length;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_comm(comm);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if (NULL == size) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        } else if (MPI_DATATYPE_NULL == datatype || NULL == datatype) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
-        }
-    }
-
-    OBJ_CONSTRUCT( &local_convertor, opal_convertor_t );
-    /* the resulting convertor will be set to the position ZERO */
-    opal_convertor_copy_and_prepare_for_send( ompi_mpi_local_convertor, &(datatype->super),
-                                              incount, NULL, 0, &local_convertor );
-
-    opal_convertor_get_packed_size( &local_convertor, &length );
-    *size = (int)length;
-    OBJ_DESTRUCT( &local_convertor );
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/pack_size.c.in b/ompi/mpi/c/pack_size.c.in
new file mode 100644
index 00000000000..f0931d00671
--- /dev/null
+++ b/ompi/mpi/c/pack_size.c.in
@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "opal/datatype/opal_convertor.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS pack_size(COUNT incount, DATATYPE datatype, COMM comm,
+                                COUNT_OUT size)
+{
+    opal_convertor_t local_convertor;
+    size_t length;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_comm(comm);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if (NULL == size) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        } else if (MPI_DATATYPE_NULL == datatype || NULL == datatype) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
+        }
+    }
+
+    OBJ_CONSTRUCT( &local_convertor, opal_convertor_t );
+    /* the resulting convertor will be set to the position ZERO */
+    opal_convertor_copy_and_prepare_for_send( ompi_mpi_local_convertor, &(datatype->super),
+                                              incount, NULL, 0, &local_convertor );
+
+    opal_convertor_get_packed_size( &local_convertor, &length );
+    *size = (int)length;
+    OBJ_DESTRUCT( &local_convertor );
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/parrived.c.in b/ompi/mpi/c/parrived.c.in
new file mode 100644
index 00000000000..562138ef1ca
--- /dev/null
+++ b/ompi/mpi/c/parrived.c.in
@@ -0,0 +1,56 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2022 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2013-2015 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/part/part.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS parrived(REQUEST request, INT partition, INT_OUT flag)
+{
+    int rc;
+
+    SPC_RECORD(OMPI_SPC_PARRIVED, 1);
+
+    if (MPI_PARAM_CHECK) {
+        rc = OMPI_SUCCESS;
+
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == request || OMPI_REQUEST_PART != request->req_type) {
+            rc = MPI_ERR_REQUEST;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, MPI_COMM_WORLD, rc, FUNC_NAME);
+    }
+
+    rc = mca_part.part_parrived(partition, partition, flag, request);
+    OMPI_ERRHANDLER_RETURN(rc, MPI_COMM_WORLD, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/pready.c.in b/ompi/mpi/c/pready.c.in
new file mode 100644
index 00000000000..8610688f420
--- /dev/null
+++ b/ompi/mpi/c/pready.c.in
@@ -0,0 +1,56 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013-2015 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/part/part.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS pready(INT partition, REQUEST request)
+{
+    int rc;
+
+    SPC_RECORD(OMPI_SPC_PREADY, 1);
+
+    if (MPI_PARAM_CHECK) {
+        rc = OMPI_SUCCESS;
+
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == request || OMPI_REQUEST_PART != request->req_type) {
+            rc = MPI_ERR_REQUEST;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, MPI_COMM_WORLD, rc, FUNC_NAME);
+    }
+
+    rc = mca_part.part_pready(partition, partition, request);
+    OMPI_ERRHANDLER_RETURN(rc, MPI_COMM_WORLD, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/pready_list.c.in b/ompi/mpi/c/pready_list.c.in
new file mode 100644
index 00000000000..086ff618954
--- /dev/null
+++ b/ompi/mpi/c/pready_list.c.in
@@ -0,0 +1,57 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013-2015 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/part/part.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS pready_list(INT length, INT_ARRAY partitions, REQUEST request)
+{
+    int rc = OMPI_SUCCESS;
+    SPC_RECORD(OMPI_SPC_PREADY, 1);
+
+    if (MPI_PARAM_CHECK) {
+        rc = OMPI_SUCCESS;
+
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == request || OMPI_REQUEST_PART != request->req_type) {
+            rc = MPI_ERR_REQUEST;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, MPI_COMM_WORLD, rc, FUNC_NAME);
+    }
+
+    for(int i = 0; i < length && OMPI_SUCCESS == rc; i++) {
+        rc = mca_part.part_pready(partitions[i], partitions[i], request);
+    }
+    OMPI_ERRHANDLER_RETURN(rc, MPI_COMM_WORLD, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/pready_range.c.in b/ompi/mpi/c/pready_range.c.in
new file mode 100644
index 00000000000..7dfc2df59ae
--- /dev/null
+++ b/ompi/mpi/c/pready_range.c.in
@@ -0,0 +1,56 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013-2015 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/part/part.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS pready_range(INT partition_low, INT partition_high, REQUEST request)
+{
+    int rc;
+
+    SPC_RECORD(OMPI_SPC_PREADY, 1);
+
+    if (MPI_PARAM_CHECK) {
+        rc = OMPI_SUCCESS;
+
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == request || OMPI_REQUEST_PART != request->req_type) {
+            rc = MPI_ERR_REQUEST;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, MPI_COMM_WORLD, rc, FUNC_NAME);
+    }
+
+    rc = mca_part.part_pready(partition_low, partition_high, request);
+    OMPI_ERRHANDLER_RETURN(rc, MPI_COMM_WORLD, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/precv_init.c.in b/ompi/mpi/c/precv_init.c.in
new file mode 100644
index 00000000000..31b1f3d97f8
--- /dev/null
+++ b/ompi/mpi/c/precv_init.c.in
@@ -0,0 +1,57 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2021 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013-2015 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2021      Bull S.A.S. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/part/part.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS precv_init(BUFFER_OUT buf, INT partitions, PARTITIONED_COUNT count,
+                                 DATATYPE datatype, INT source, INT tag,
+                                 COMM comm, INFO info, REQUEST_INOUT request)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        rc = OMPI_SUCCESS;
+
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == request) {
+            rc = MPI_ERR_REQUEST;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, MPI_COMM_WORLD, rc, FUNC_NAME);
+    }
+
+    rc = mca_part.part_precv_init(buf, partitions, count, datatype, source, tag, comm, info, request);
+    OMPI_ERRHANDLER_RETURN(rc, MPI_COMM_WORLD, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/probe.c b/ompi/mpi/c/probe.c
deleted file mode 100644
index 6c235315876..00000000000
--- a/ompi/mpi/c/probe.c
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/memchecker.h"
-#include "ompi/request/request.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Probe = PMPI_Probe
-#endif
-#define MPI_Probe PMPI_Probe
-#endif
-
-static const char FUNC_NAME[] = "MPI_Probe";
-
-
-int MPI_Probe(int source, int tag, MPI_Comm comm, MPI_Status *status)
-{
-    int rc;
-
-    SPC_RECORD(OMPI_SPC_PROBE, 1);
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (((tag < 0) && (tag != MPI_ANY_TAG)) || (tag > mca_pml.pml_max_tag)) {
-            rc = MPI_ERR_TAG;
-        } else if (ompi_comm_invalid(comm)) {
-            rc = MPI_ERR_COMM;
-        } else if ((source != MPI_ANY_SOURCE) &&
-                   (MPI_PROC_NULL != source) &&
-                   ompi_comm_peer_invalid(comm, source)) {
-            rc = MPI_ERR_RANK;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, comm, rc, "MPI_Probe");
-    }
-
-    if (MPI_PROC_NULL == source) {
-        if (MPI_STATUS_IGNORE != status) {
-            *status = ompi_request_empty.req_status;
-            /*
-             * Per MPI-1, the MPI_ERROR field is not defined for single-completion calls
-             */
-            MEMCHECKER(
-                opal_memchecker_base_mem_undefined(&status->MPI_ERROR, sizeof(int));
-            );
-        }
-        return MPI_SUCCESS;
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * Check here for issues with the peer, so we do not have to duplicate the
-     * functionality in the PML.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_p2p_check_proc(comm, source, &rc)) ) {
-        if (MPI_STATUS_IGNORE != status) {
-            status->MPI_SOURCE = source;
-            status->MPI_TAG    = tag;
-        }
-        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-    }
-#endif
-
-    rc = MCA_PML_CALL(probe(source, tag, comm, status));
-    /*
-     * Per MPI-1, the MPI_ERROR field is not defined for single-completion calls
-     */
-    MEMCHECKER(
-        opal_memchecker_base_mem_undefined(&status->MPI_ERROR, sizeof(int));
-    );
-
-    OMPI_ERRHANDLER_RETURN(rc, comm, rc, "MPI_Probe");
-}
diff --git a/ompi/mpi/c/probe.c.in b/ompi/mpi/c/probe.c.in
new file mode 100644
index 00000000000..906b164633e
--- /dev/null
+++ b/ompi/mpi/c/probe.c.in
@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/memchecker.h"
+#include "ompi/request/request.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS probe(INT source, INT tag, COMM comm, STATUS_OUT status)
+{
+    int rc;
+
+    SPC_RECORD(OMPI_SPC_PROBE, 1);
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (((tag < 0) && (tag != MPI_ANY_TAG)) || (tag > mca_pml.pml_max_tag)) {
+            rc = MPI_ERR_TAG;
+        } else if (ompi_comm_invalid(comm)) {
+            rc = MPI_ERR_COMM;
+        } else if ((source != MPI_ANY_SOURCE) &&
+                   (MPI_PROC_NULL != source) &&
+                   ompi_comm_peer_invalid(comm, source)) {
+            rc = MPI_ERR_RANK;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, "MPI_Probe");
+    }
+
+    if (MPI_PROC_NULL == source) {
+        if (MPI_STATUS_IGNORE != status) {
+            OMPI_COPY_STATUS(status, ompi_request_empty.req_status, false);
+            /*
+             * Per MPI-1, the MPI_ERROR field is not defined for single-completion calls
+             */
+            MEMCHECKER(
+                opal_memchecker_base_mem_undefined(&status->MPI_ERROR, sizeof(int));
+            );
+        }
+        return MPI_SUCCESS;
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * Check here for issues with the peer, so we do not have to duplicate the
+     * functionality in the PML.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_p2p_check_proc(comm, source, &rc)) ) {
+        if (MPI_STATUS_IGNORE != status) {
+            status->MPI_SOURCE = source;
+            status->MPI_TAG    = tag;
+        }
+        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+    }
+#endif
+
+    rc = MCA_PML_CALL(probe(source, tag, comm, status));
+    /*
+     * Per MPI-1, the MPI_ERROR field is not defined for single-completion calls
+     */
+    MEMCHECKER(
+        opal_memchecker_base_mem_undefined(&status->MPI_ERROR, sizeof(int));
+    );
+
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, "MPI_Probe");
+}
diff --git a/ompi/mpi/c/profile/Makefile.am b/ompi/mpi/c/profile/Makefile.am
deleted file mode 100644
index a15613a6dd8..00000000000
--- a/ompi/mpi/c/profile/Makefile.am
+++ /dev/null
@@ -1,462 +0,0 @@
-# -*- makefile.am	 -*-
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2013 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2009-2014 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
-# Copyright (c) 2012      Oak Ridge National Laboratory. All rights reserved.
-# Copyright (c) 2012-2013 Inria.  All rights reserved.
-# Copyright (c) 2013      Los Alamos National Security, LLC. All rights
-#                         reserved.
-# Copyright (c) 2015-2020 Research Organization for Information Science
-#                         and Technology (RIST).  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-include $(top_srcdir)/Makefile.ompi-rules
-
-# If OMPI_BUILD_MPI_PROFILING is enabled when we want our generated MPI_* symbols
-# to be replaced by PMPI_*.
-# In this directory, we definately need it to be 1.
-
-AM_CPPFLAGS = -DOMPI_BUILD_MPI_PROFILING=1
-
-#
-# This build needs to go through only if profiling is required.
-# Further, this build HAS to go through if profiling is required.
-#
-
-noinst_LTLIBRARIES = libmpi_c_pmpi.la
-
-nodist_libmpi_c_pmpi_la_SOURCES = \
-        pabort.c \
-        padd_error_class.c \
-        padd_error_code.c \
-        padd_error_string.c \
-        pallgather.c \
-        piallgather.c \
-        pallgatherv.c \
-        piallgatherv.c \
-        palloc_mem.c \
-        pallreduce.c \
-        piallreduce.c \
-        palltoall.c \
-        pialltoall.c \
-        palltoallv.c \
-        pialltoallv.c \
-        palltoallw.c \
-        pialltoallw.c \
-        pattr_delete.c \
-        pattr_get.c \
-        pattr_put.c \
-        pbarrier.c \
-        pibarrier.c \
-        pbcast.c \
-        pibcast.c \
-        pbsend.c \
-        pbsend_init.c \
-        pbuffer_attach.c \
-        pbuffer_detach.c \
-        pcancel.c \
-        pcart_coords.c \
-        pcart_create.c \
-        pcartdim_get.c \
-        pcart_get.c \
-        pcart_map.c \
-        pcart_rank.c \
-        pcart_shift.c \
-        pcart_sub.c \
-        pclose_port.c \
-        pcomm_accept.c \
-        pcomm_c2f.c \
-        pcomm_call_errhandler.c \
-        pcomm_compare.c \
-        pcomm_connect.c \
-        pcomm_create.c \
-        pcomm_create_errhandler.c \
-        pcomm_create_group.c \
-        pcomm_create_keyval.c \
-        pcomm_delete_attr.c \
-        pcomm_disconnect.c \
-        pcomm_dup.c \
-        pcomm_dup_with_info.c \
-        pcomm_idup.c \
-        pcomm_f2c.c \
-        pcomm_free.c \
-        pcomm_free_keyval.c \
-        pcomm_get_attr.c \
-        pcomm_get_errhandler.c \
-        pcomm_get_info.c \
-        pcomm_get_name.c \
-        pcomm_get_parent.c \
-        pcomm_group.c \
-        pcomm_join.c \
-        pcomm_rank.c \
-        pcomm_remote_group.c \
-        pcomm_remote_size.c \
-        pcomm_set_attr.c \
-        pcomm_set_info.c \
-        pdist_graph_create.c \
-        pdist_graph_create_adjacent.c \
-        pdist_graph_neighbors.c \
-        pdist_graph_neighbors_count.c \
-        pcomm_set_errhandler.c \
-        pcomm_set_name.c \
-        pcomm_size.c \
-        pcomm_spawn.c \
-        pcomm_spawn_multiple.c \
-        pcomm_split.c \
-        pcomm_split_type.c \
-        pcomm_test_inter.c \
-	pcompare_and_swap.c \
-        pdims_create.c \
-        perrhandler_c2f.c \
-        perrhandler_f2c.c \
-        perrhandler_free.c \
-        perror_class.c \
-        perror_string.c \
-        pexscan.c \
-	pfetch_and_op.c \
-        piexscan.c \
-        pfile_c2f.c \
-        pfile_call_errhandler.c \
-        pfile_close.c \
-        pfile_create_errhandler.c \
-        pfile_delete.c \
-        pfile_f2c.c \
-        pfile_get_amode.c \
-        pfile_get_atomicity.c \
-        pfile_get_byte_offset.c \
-        pfile_get_errhandler.c \
-        pfile_get_group.c \
-        pfile_get_info.c \
-        pfile_get_position.c \
-        pfile_get_position_shared.c \
-        pfile_get_size.c \
-        pfile_get_type_extent.c \
-        pfile_get_view.c \
-        pfile_iread_at.c \
-        pfile_iread.c \
-        pfile_iread_at_all.c \
-        pfile_iread_all.c \
-        pfile_iread_shared.c \
-        pfile_iwrite_at.c \
-        pfile_iwrite.c \
-        pfile_iwrite_at_all.c \
-        pfile_iwrite_all.c \
-        pfile_iwrite_shared.c \
-        pfile_open.c \
-        pfile_preallocate.c \
-        pfile_read_all_begin.c \
-        pfile_read_all.c \
-        pfile_read_all_end.c \
-        pfile_read_at_all_begin.c \
-        pfile_read_at_all.c \
-        pfile_read_at_all_end.c \
-        pfile_read_at.c \
-        pfile_read.c \
-        pfile_read_ordered_begin.c \
-        pfile_read_ordered.c \
-        pfile_read_ordered_end.c \
-        pfile_read_shared.c \
-        pfile_seek.c \
-        pfile_seek_shared.c \
-        pfile_set_atomicity.c \
-        pfile_set_errhandler.c \
-        pfile_set_info.c \
-        pfile_set_size.c \
-        pfile_set_view.c \
-        pfile_sync.c \
-        pfile_write_all_begin.c \
-        pfile_write_all.c \
-        pfile_write_all_end.c \
-        pfile_write_at_all_begin.c \
-        pfile_write_at_all.c \
-        pfile_write_at_all_end.c \
-        pfile_write_at.c \
-        pfile_write.c \
-        pfile_write_ordered_begin.c \
-        pfile_write_ordered.c \
-        pfile_write_ordered_end.c \
-        pfile_write_shared.c \
-        pfinalize.c \
-        pfinalized.c \
-        pfree_mem.c \
-        pgather.c \
-        pigather.c \
-        pgatherv.c \
-        pigatherv.c \
-        pget_address.c \
-        pget_count.c \
-        pget_elements.c \
-        pget_elements_x.c \
-	pget_accumulate.c \
-        pget_library_version.c \
-        pget_processor_name.c \
-        pget_version.c \
-        pgraph_create.c \
-        pgraph_get.c \
-        pgraph_map.c \
-        pgraph_neighbors_count.c \
-        pgraph_neighbors.c \
-        pgraphdims_get.c \
-        pgrequest_complete.c \
-        pgrequest_start.c \
-        pgroup_c2f.c \
-        pgroup_compare.c \
-        pgroup_difference.c \
-        pgroup_excl.c \
-        pgroup_f2c.c \
-        pgroup_free.c \
-        pgroup_incl.c \
-        pgroup_intersection.c \
-        pgroup_range_excl.c \
-        pgroup_range_incl.c \
-        pgroup_rank.c \
-        pgroup_size.c \
-        pgroup_translate_ranks.c \
-        pgroup_union.c \
-        pibsend.c \
-	pimprobe.c \
-	pimrecv.c \
-        pinfo_c2f.c \
-        pinfo_create.c \
-        pinfo_delete.c \
-        pinfo_dup.c \
-        pinfo_f2c.c \
-        pinfo_free.c \
-        pinfo_get.c \
-        pinfo_get_nkeys.c \
-        pinfo_get_nthkey.c \
-        pinfo_get_valuelen.c \
-        pinfo_set.c \
-        pinit.c \
-        pinit_thread.c \
-        pinitialized.c \
-        pintercomm_create.c \
-        pintercomm_merge.c \
-        piprobe.c \
-        pirecv.c \
-        pirsend.c \
-        pis_thread_main.c \
-        pisend.c \
-        pissend.c \
-        plookup_name.c \
-        pmessage_f2c.c \
-        pmessage_c2f.c \
-        pmprobe.c \
-        pmrecv.c \
-        pneighbor_allgather.c \
-        pineighbor_allgather.c \
-        pneighbor_allgatherv.c \
-        pineighbor_allgatherv.c \
-        pneighbor_alltoall.c \
-        pineighbor_alltoall.c \
-        pneighbor_alltoallv.c \
-        pineighbor_alltoallv.c \
-        pneighbor_alltoallw.c \
-        pineighbor_alltoallw.c \
-        pkeyval_create.c \
-        pkeyval_free.c \
-        pop_c2f.c \
-        pop_create.c \
-        pop_commutative.c \
-        pop_f2c.c \
-        pop_free.c \
-        popen_port.c \
-        ppack_external.c \
-        ppack_external_size.c \
-        ppack.c \
-        ppack_size.c \
-        ppcontrol.c \
-        pprobe.c \
-        ppublish_name.c \
-        pquery_thread.c \
-	praccumulate.c \
-        precv_init.c \
-        precv.c \
-        preduce.c \
-        pregister_datarep.c \
-        pireduce.c \
-        preduce_local.c \
-        preduce_scatter.c \
-        pireduce_scatter.c \
-        preduce_scatter_block.c \
-        pireduce_scatter_block.c \
-        prequest_c2f.c \
-        prequest_f2c.c \
-        prequest_free.c \
-        prequest_get_status.c \
-	prget.c \
-	prget_accumulate.c \
-	prput.c \
-        prsend_init.c \
-        prsend.c \
-        pscan.c \
-        piscan.c \
-        pscatter.c \
-        piscatter.c \
-        pscatterv.c \
-        piscatterv.c \
-        psend.c \
-        psend_init.c \
-        psendrecv.c \
-        psendrecv_replace.c \
-        pssend_init.c \
-        pssend.c \
-        pstart.c \
-        pstartall.c \
-        pstatus_c2f.c \
-        pstatus_c2f08.c \
-        pstatus_f082c.c \
-        pstatus_f082f.c \
-        pstatus_f2f08.c \
-        pstatus_f2c.c \
-        pstatus_set_cancelled.c \
-        pstatus_set_elements.c \
-        pstatus_set_elements_x.c \
-        ptestall.c \
-        ptestany.c \
-        ptest.c \
-        ptest_cancelled.c \
-        ptestsome.c \
-        ptopo_test.c \
-        ptype_c2f.c \
-        ptype_commit.c \
-        ptype_contiguous.c \
-        ptype_create_darray.c \
-        ptype_create_f90_complex.c \
-        ptype_create_f90_integer.c \
-        ptype_create_f90_real.c \
-        ptype_create_hindexed.c \
-        ptype_create_hvector.c \
-        ptype_create_indexed_block.c \
-        ptype_create_hindexed_block.c \
-        ptype_create_keyval.c \
-        ptype_create_resized.c \
-        ptype_create_struct.c \
-        ptype_create_subarray.c \
-        ptype_delete_attr.c \
-        ptype_dup.c \
-        ptype_f2c.c \
-        ptype_free.c \
-        ptype_free_keyval.c \
-        ptype_get_attr.c \
-        ptype_get_contents.c \
-        ptype_get_envelope.c \
-        ptype_get_extent.c \
-        ptype_get_extent_x.c \
-        ptype_get_name.c \
-        ptype_get_true_extent.c \
-        ptype_get_true_extent_x.c \
-        ptype_indexed.c \
-        ptype_match_size.c \
-        ptype_set_attr.c \
-        ptype_set_name.c \
-        ptype_size.c \
-        ptype_size_x.c \
-        ptype_vector.c  \
-        punpack_external.c \
-        punpack.c \
-        punpublish_name.c \
-        pwait.c \
-        pwaitall.c \
-        pwaitany.c \
-        pwaitsome.c \
-	pwtime.c \
-	pwtick.c \
-        paccumulate.c \
-        pget.c \
-        pput.c \
-	pwin_allocate.c \
-	pwin_allocate_shared.c \
-	pwin_attach.c \
-        pwin_c2f.c \
-        pwin_call_errhandler.c \
-        pwin_complete.c  \
-        pwin_create_errhandler.c \
-        pwin_create_keyval.c \
-        pwin_create.c \
-	pwin_create_dynamic.c \
-        pwin_delete_attr.c \
-	pwin_detach.c \
-        pwin_f2c.c \
-        pwin_fence.c \
-	pwin_flush.c \
-	pwin_flush_all.c \
-	pwin_flush_local.c \
-	pwin_flush_local_all.c \
-        pwin_free_keyval.c \
-        pwin_free.c \
-        pwin_get_attr.c \
-        pwin_get_errhandler.c \
-        pwin_get_group.c \
-        pwin_get_info.c  \
-        pwin_get_name.c  \
-        pwin_lock.c \
-	pwin_lock_all.c \
-        pwin_post.c \
-        pwin_set_attr.c \
-        pwin_set_errhandler.c \
-        pwin_set_info.c \
-        pwin_set_name.c \
-	pwin_shared_query.c \
-        pwin_start.c \
-	pwin_sync.c \
-        pwin_test.c \
-        pwin_unlock.c \
-	pwin_unlock_all.c \
-        pwin_wait.c
-
-if OMPI_ENABLE_MPI1_COMPAT
-nodist_libmpi_c_pmpi_la_SOURCES += \
-        paddress.c \
-        perrhandler_create.c \
-        perrhandler_get.c \
-        perrhandler_set.c \
-        ptype_extent.c \
-        ptype_hindexed.c \
-        ptype_hvector.c \
-        ptype_lb.c \
-        ptype_struct.c \
-        ptype_ub.c
-endif
-
-#
-# Sym link in the sources from the real MPI directory
-#
-$(nodist_libmpi_c_pmpi_la_SOURCES):
-	$(OMPI_V_LN_S) if test ! -r $@ ; then \
-		pname=`echo $@ | cut -b '2-'` ; \
-		$(LN_S) $(top_srcdir)/ompi/mpi/c/$$pname $@ ; \
-	fi
-
-# Conditionally install the header files
-
-if WANT_INSTALL_HEADERS
-ompidir = $(ompiincludedir)/$(subdir)
-endif
-
-# These files were created by targets above
-
-MAINTAINERCLEANFILES = $(nodist_libmpi_c_pmpi_la_SOURCES)
-
-# Don't want these targets in here
-
-tags-recursive:
-tags:
-TAGS:
-GTAGS:
-ID:
diff --git a/ompi/mpi/c/psend_init.c.in b/ompi/mpi/c/psend_init.c.in
new file mode 100644
index 00000000000..7ddf8335090
--- /dev/null
+++ b/ompi/mpi/c/psend_init.c.in
@@ -0,0 +1,57 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013-2015 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2021      Bull S.A.S. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/part/part.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS psend_init(BUFFER buf, INT partitions, PARTITIONED_COUNT count,
+                                 DATATYPE datatype, INT dest, INT tag, COMM comm,
+                                 INFO info, REQUEST_INOUT request)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        rc = OMPI_SUCCESS;
+
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == request) {
+            rc = MPI_ERR_REQUEST;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, MPI_COMM_WORLD, rc, FUNC_NAME);
+    }
+
+    rc = mca_part.part_psend_init(buf, partitions, count, datatype, dest, tag, comm, info, request);
+    OMPI_ERRHANDLER_RETURN(rc, MPI_COMM_WORLD, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/publish_name.c b/ompi/mpi/c/publish_name.c
deleted file mode 100644
index be930aa82d6..00000000000
--- a/ompi/mpi/c/publish_name.c
+++ /dev/null
@@ -1,139 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2019 Intel, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2015      Cisco Systems, Inc.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "opal/class/opal_list.h"
-#include "opal/mca/pmix/pmix-internal.h"
-#include "opal/util/show_help.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-#include "ompi/communicator/communicator.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Publish_name = PMPI_Publish_name
-#endif
-#define MPI_Publish_name PMPI_Publish_name
-#endif
-
-static const char FUNC_NAME[] = "MPI_Publish_name";
-
-
-int MPI_Publish_name(const char *service_name, MPI_Info info,
-                     const char *port_name)
-{
-    int ret;
-    char range[OPAL_MAX_INFO_VAL];
-    int flag=0;
-    pmix_status_t rc;
-    pmix_info_t pinfo[3];
-    pmix_data_range_t rng = PMIX_RANGE_SESSION;
-    pmix_persistence_t pers = PMIX_PERSIST_SESSION;
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if ( NULL == port_name || 0 == strlen(port_name) ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-        if ( NULL == service_name || 0 == strlen(service_name) ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-        if (NULL == info || ompi_info_is_freed(info)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
-                                          FUNC_NAME);
-        }
-    }
-
-    /* OMPI supports info keys to pass the range and persistence to
-     * be used for the given key */
-    if (MPI_INFO_NULL != info) {
-        ompi_info_get (info, "range", sizeof(range) - 1, range, &flag);
-        if (flag) {
-            if (0 == strcmp(range, "nspace")) {
-                rng = PMIX_RANGE_NAMESPACE;  // share only with procs in same nspace
-            } else if (0 == strcmp(range, "session")) {
-                rng = PMIX_RANGE_SESSION; // share only with procs in same session
-            } else {
-                /* unrecognized scope */
-                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                            FUNC_NAME);
-            }
-        }
-        ompi_info_get (info, "persistence", sizeof(range) - 1, range, &flag);
-        if (flag) {
-            if (0 == strcmp(range, "indef")) {
-                pers = PMIX_PERSIST_INDEF;   // retain until specifically deleted
-            } else if (0 == strcmp(range, "proc")) {
-                pers = PMIX_PERSIST_PROC;    // retain until publishing process terminates
-            } else if (0 == strcmp(range, "app")) {
-                pers = PMIX_PERSIST_APP;     // retain until application terminates
-            } else if (0 == strcmp(range, "session")) {
-                pers = PMIX_PERSIST_SESSION; // retain until session/allocation terminates
-            } else {
-                /* unrecognized persistence */
-                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                            FUNC_NAME);
-            }
-        }
-    }
-
-    /* publish the service name */
-    PMIX_INFO_LOAD(&pinfo[0], service_name, port_name, PMIX_STRING);
-    PMIX_INFO_LOAD(&pinfo[1], PMIX_RANGE, &rng, PMIX_DATA_RANGE);
-    PMIX_INFO_LOAD(&pinfo[2], PMIX_PERSISTENCE, &pers, PMIX_PERSIST);
-
-    rc = PMIx_Publish(pinfo, 3);
-    PMIX_INFO_DESTRUCT(&pinfo[0]);
-    PMIX_INFO_DESTRUCT(&pinfo[1]);
-    PMIX_INFO_DESTRUCT(&pinfo[2]);
-
-    if ( PMIX_SUCCESS != rc ) {
-        if (PMIX_EXISTS == rc) {
-            /* already exists - can't publish it */
-            ret = MPI_ERR_FILE_EXISTS;
-        } else if (PMIX_ERR_NOT_SUPPORTED == rc) {
-            /* this PMIX environment doesn't support publishing */
-            ret = OMPI_ERR_NOT_SUPPORTED;
-            opal_show_help("help-mpi-api.txt",
-                           "MPI function not supported",
-                           true,
-                           FUNC_NAME,
-                           "Underlying runtime environment does not support name publishing functionality");
-        } else {
-            ret = MPI_ERR_INTERN;
-        }
-
-        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(ret, FUNC_NAME);
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/publish_name.c.in b/ompi/mpi/c/publish_name.c.in
new file mode 100644
index 00000000000..967ee44033f
--- /dev/null
+++ b/ompi/mpi/c/publish_name.c.in
@@ -0,0 +1,133 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "opal/class/opal_list.h"
+#include "opal/mca/pmix/pmix-internal.h"
+#include "opal/util/show_help.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include "ompi/communicator/communicator.h"
+
+PROTOTYPE ERROR_CLASS publish_name(STRING service_name, INFO info,
+                                   STRING port_name)
+{
+    int ret;
+    opal_cstring_t *info_str;
+    int flag=0;
+    pmix_status_t rc;
+    pmix_info_t pinfo[3];
+    pmix_data_range_t rng = PMIX_RANGE_SESSION;
+    pmix_persistence_t pers = PMIX_PERSIST_SESSION;
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if ( NULL == port_name || 0 == strlen(port_name) ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+        if ( NULL == service_name || 0 == strlen(service_name) ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+        if (NULL == info || ompi_info_is_freed(info)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
+                                          FUNC_NAME);
+        }
+    }
+
+    /* OMPI supports info keys to pass the range and persistence to
+     * be used for the given key */
+    if (MPI_INFO_NULL != info) {
+        ompi_info_get (info, "range", &info_str, &flag);
+        if (flag) {
+            if (0 == strcmp(info_str->string, "nspace")) {
+                rng = PMIX_RANGE_NAMESPACE;  // share only with procs in same nspace
+            } else if (0 == strcmp(info_str->string, "session")) {
+                rng = PMIX_RANGE_SESSION; // share only with procs in same session
+            } else {
+                /* unrecognized scope */
+                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                            FUNC_NAME);
+            }
+            OBJ_RELEASE(info_str);
+        }
+        ompi_info_get (info, "persistence", &info_str, &flag);
+        if (flag) {
+            if (0 == strcmp(info_str->string, "indef")) {
+                pers = PMIX_PERSIST_INDEF;   // retain until specifically deleted
+            } else if (0 == strcmp(info_str->string, "proc")) {
+                pers = PMIX_PERSIST_PROC;    // retain until publishing process terminates
+            } else if (0 == strcmp(info_str->string, "app")) {
+                pers = PMIX_PERSIST_APP;     // retain until application terminates
+            } else if (0 == strcmp(info_str->string, "session")) {
+                pers = PMIX_PERSIST_SESSION; // retain until session/allocation terminates
+            } else {
+                /* unrecognized persistence */
+                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                            FUNC_NAME);
+            }
+            OBJ_RELEASE(info_str);
+        }
+    }
+
+    /* publish the service name */
+    PMIX_INFO_LOAD(&pinfo[0], service_name, port_name, PMIX_STRING);
+    PMIX_INFO_LOAD(&pinfo[1], PMIX_RANGE, &rng, PMIX_DATA_RANGE);
+    PMIX_INFO_LOAD(&pinfo[2], PMIX_PERSISTENCE, &pers, PMIX_PERSIST);
+
+    rc = PMIx_Publish(pinfo, 3);
+    PMIX_INFO_DESTRUCT(&pinfo[0]);
+    PMIX_INFO_DESTRUCT(&pinfo[1]);
+    PMIX_INFO_DESTRUCT(&pinfo[2]);
+
+    if ( PMIX_SUCCESS != rc ) {
+        if (PMIX_EXISTS == rc) {
+            /* already exists - can't publish it */
+            ret = MPI_ERR_FILE_EXISTS;
+        } else if (PMIX_ERR_NOT_SUPPORTED == rc) {
+            /* this PMIX environment doesn't support publishing */
+            ret = OMPI_ERR_NOT_SUPPORTED;
+            opal_show_help("help-mpi-api.txt",
+                           "MPI function not supported",
+                           true,
+                           FUNC_NAME,
+                           "Underlying runtime environment does not support name publishing functionality");
+        } else {
+            ret = MPI_ERR_INTERN;
+        }
+
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(ret, FUNC_NAME);
+    }
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/put.c b/ompi/mpi/c/put.c
deleted file mode 100644
index 097b41b5b9d..00000000000
--- a/ompi/mpi/c/put.c
+++ /dev/null
@@ -1,86 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013-2015 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-#include "ompi/mca/osc/osc.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Put = PMPI_Put
-#endif
-#define MPI_Put PMPI_Put
-#endif
-
-static const char FUNC_NAME[] = "MPI_Put";
-
-
-int MPI_Put(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype,
-            int target_rank, MPI_Aint target_disp, int target_count,
-            MPI_Datatype target_datatype, MPI_Win win)
-{
-    int rc;
-
-    SPC_RECORD(OMPI_SPC_PUT, 1);
-
-    if (MPI_PARAM_CHECK) {
-        rc = OMPI_SUCCESS;
-
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        } else if (origin_count < 0 || target_count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else if (ompi_win_peer_invalid(win, target_rank) &&
-                   (MPI_PROC_NULL != target_rank)) {
-            rc = MPI_ERR_RANK;
-        } else if (NULL == target_datatype ||
-                   MPI_DATATYPE_NULL == target_datatype) {
-            rc = MPI_ERR_TYPE;
-        } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) {
-            rc = MPI_ERR_DISP;
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, origin_datatype, origin_count);
-            if (OMPI_SUCCESS == rc) {
-                OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, target_datatype, target_count);
-            }
-        }
-        OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME);
-    }
-
-    if (MPI_PROC_NULL == target_rank) return MPI_SUCCESS;
-
-    rc = win->w_osc_module->osc_put(origin_addr, origin_count, origin_datatype,
-                                    target_rank, target_disp, target_count,
-                                    target_datatype, win);
-    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/put.c.in b/ompi/mpi/c/put.c.in
new file mode 100644
index 00000000000..c24cedd7e62
--- /dev/null
+++ b/ompi/mpi/c/put.c.in
@@ -0,0 +1,78 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013-2015 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+#include "ompi/mca/osc/osc.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS put(BUFFER origin_addr, COUNT origin_count, DATATYPE origin_datatype,
+                          INT target_rank, AINT target_disp, COUNT target_count,
+                          DATATYPE target_datatype, WIN win)
+{
+    int rc;
+
+    SPC_RECORD(OMPI_SPC_PUT, 1);
+
+    if (MPI_PARAM_CHECK) {
+        rc = OMPI_SUCCESS;
+
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        } else if (origin_count < 0 || target_count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else if (ompi_win_peer_invalid(win, target_rank) &&
+                   (MPI_PROC_NULL != target_rank)) {
+            rc = MPI_ERR_RANK;
+        } else if (NULL == target_datatype ||
+                   MPI_DATATYPE_NULL == target_datatype) {
+            rc = MPI_ERR_TYPE;
+        } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) {
+            rc = MPI_ERR_DISP;
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, origin_datatype, origin_count);
+            if (OMPI_SUCCESS == rc) {
+                OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, target_datatype, target_count);
+            }
+        }
+        OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME);
+    }
+
+    if (MPI_PROC_NULL == target_rank) return MPI_SUCCESS;
+
+    rc = win->w_osc_module->osc_put(origin_addr, origin_count, origin_datatype,
+                                    target_rank, target_disp, target_count,
+                                    target_datatype, win);
+    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/query_thread.c b/ompi/mpi/c/query_thread.c
deleted file mode 100644
index aaaef184d67..00000000000
--- a/ompi/mpi/c/query_thread.c
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Query_thread = PMPI_Query_thread
-#endif
-#define MPI_Query_thread PMPI_Query_thread
-#endif
-
-static const char FUNC_NAME[] = "MPI_Query_thread";
-
-
-int MPI_Query_thread(int *provided)
-{
-  if (MPI_PARAM_CHECK) {
-    OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-    if (NULL == provided) {
-      return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-    }
-  }
-
-  /* Simple */
-
-  *provided = ompi_mpi_thread_provided;
-  return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/query_thread.c.in b/ompi/mpi/c/query_thread.c.in
new file mode 100644
index 00000000000..493076c2ce9
--- /dev/null
+++ b/ompi/mpi/c/query_thread.c.in
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+
+PROTOTYPE ERROR_CLASS query_thread(INT_OUT provided)
+{
+  if (MPI_PARAM_CHECK) {
+    OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+    if (NULL == provided) {
+      return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+    }
+  }
+
+  /* Simple */
+
+  *provided = ompi_mpi_thread_provided;
+  return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/raccumulate.c b/ompi/mpi/c/raccumulate.c
deleted file mode 100644
index 1b9ad962598..00000000000
--- a/ompi/mpi/c/raccumulate.c
+++ /dev/null
@@ -1,139 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2009      Sun Microsystmes, Inc.  All rights reserved.
- * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
- * Copyright (c) 2014-2015 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-#include "ompi/mca/osc/osc.h"
-#include "ompi/op/op.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/datatype/ompi_datatype_internal.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Raccumulate = PMPI_Raccumulate
-#endif
-#define MPI_Raccumulate PMPI_Raccumulate
-#endif
-
-static const char FUNC_NAME[] = "MPI_Raccumulate";
-
-int MPI_Raccumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype,
-                   int target_rank, MPI_Aint target_disp, int target_count,
-                   MPI_Datatype target_datatype, MPI_Op op, MPI_Win win, MPI_Request *request)
-{
-    int rc;
-    ompi_win_t *ompi_win = (ompi_win_t*) win;
-
-    MEMCHECKER(
-        memchecker_datatype(origin_datatype);
-        memchecker_datatype(target_datatype);
-        memchecker_call(&opal_memchecker_base_isdefined, origin_addr, origin_count, origin_datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = OMPI_SUCCESS;
-
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        } else if (origin_count < 0 || target_count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else if (ompi_win_peer_invalid(win, target_rank) &&
-                   (MPI_PROC_NULL != target_rank)) {
-            rc = MPI_ERR_RANK;
-        } else if (MPI_OP_NULL == op || MPI_NO_OP == op) {
-            rc = MPI_ERR_OP;
-        } else if (!ompi_op_is_intrinsic(op)) {
-            rc = MPI_ERR_OP;
-        } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) {
-            rc = MPI_ERR_DISP;
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, origin_datatype, origin_count);
-            if (OMPI_SUCCESS == rc) {
-                OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, target_datatype, target_count);
-            }
-            if (OMPI_SUCCESS == rc) {
-                /* While technically the standard probably requires that the
-                   datatypes used with MPI_REPLACE conform to all the rules
-                   for other reduction operators, we don't require such
-                   behavior, as checking for it is expensive here and we don't
-                   care in implementation.. */
-                if (op != &ompi_mpi_op_replace.op && op != &ompi_mpi_op_no_op.op) {
-                    ompi_datatype_t *op_check_dt, *origin_check_dt;
-                    char *msg;
-
-                    /* RACCUMULATE, unlike REDUCE, can use with derived
-                       datatypes with predefinied operations, with some
-                       restrictions outlined in MPI-3:11.3.4.  The derived
-                       datatype must be composed entierly from one predefined
-                       datatype (so you can do all the construction you want,
-                       but at the bottom, you can only use one datatype, say,
-                       MPI_INT).  If the datatype at the target isn't
-                       predefined, then make sure it's composed of only one
-                       datatype, and check that datatype against
-                       ompi_op_is_valid(). */
-                    origin_check_dt = ompi_datatype_get_single_predefined_type_from_args(origin_datatype);
-                    op_check_dt = ompi_datatype_get_single_predefined_type_from_args(target_datatype);
-
-                    if( !((origin_check_dt == op_check_dt) & (NULL != op_check_dt)) ) {
-                        OMPI_ERRHANDLER_RETURN(MPI_ERR_ARG, win, MPI_ERR_ARG, FUNC_NAME);
-                    }
-
-                    /* check to make sure primitive type is valid for
-                       reduction.  Should do this on the target, but
-                       then can't get the errcode back for this
-                       call */
-                    if (!ompi_op_is_valid(op, op_check_dt, &msg, FUNC_NAME)) {
-                        int ret = OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_OP, msg);
-                        free(msg);
-                        return ret;
-                    }
-                }
-            }
-        }
-        OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME);
-    }
-
-    if (MPI_PROC_NULL == target_rank) {
-        *request = &ompi_request_empty;
-        return MPI_SUCCESS;
-    }
-
-    rc = ompi_win->w_osc_module->osc_raccumulate(origin_addr,
-                                                origin_count,
-                                                origin_datatype,
-                                                target_rank,
-                                                target_disp,
-                                                target_count,
-                                                target_datatype,
-                                                op, win, request);
-    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/raccumulate.c.in b/ompi/mpi/c/raccumulate.c.in
new file mode 100644
index 00000000000..8280c313295
--- /dev/null
+++ b/ompi/mpi/c/raccumulate.c.in
@@ -0,0 +1,132 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2009      Sun Microsystmes, Inc.  All rights reserved.
+ * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2014-2015 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+#include "ompi/mca/osc/osc.h"
+#include "ompi/op/op.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/datatype/ompi_datatype_internal.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS raccumulate(BUFFER origin_addr, COUNT origin_count, DATATYPE origin_datatype,
+                                  INT target_rank, AINT target_disp, COUNT target_count,
+                                  DATATYPE target_datatype, OP op, WIN win, REQUEST_INOUT request)
+{
+    int rc;
+    ompi_win_t *ompi_win = (ompi_win_t*) win;
+
+    MEMCHECKER(
+        memchecker_datatype(origin_datatype);
+        memchecker_datatype(target_datatype);
+        memchecker_call(&opal_memchecker_base_isdefined, origin_addr, origin_count, origin_datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = OMPI_SUCCESS;
+
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        } else if (origin_count < 0 || target_count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else if (ompi_win_peer_invalid(win, target_rank) &&
+                   (MPI_PROC_NULL != target_rank)) {
+            rc = MPI_ERR_RANK;
+        } else if (MPI_OP_NULL == op || MPI_NO_OP == op) {
+            rc = MPI_ERR_OP;
+        } else if (!ompi_op_is_intrinsic(op)) {
+            rc = MPI_ERR_OP;
+        } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) {
+            rc = MPI_ERR_DISP;
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, origin_datatype, origin_count);
+            if (OMPI_SUCCESS == rc) {
+                OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, target_datatype, target_count);
+            }
+            if (OMPI_SUCCESS == rc) {
+                /* While technically the standard probably requires that the
+                   datatypes used with MPI_REPLACE conform to all the rules
+                   for other reduction operators, we don't require such
+                   behavior, as checking for it is expensive here and we don't
+                   care in implementation.. */
+                if (op != &ompi_mpi_op_replace.op && op != &ompi_mpi_op_no_op.op) {
+                    ompi_datatype_t *op_check_dt, *origin_check_dt;
+                    char *msg;
+
+                    /* RACCUMULATE, unlike REDUCE, can use with derived
+                       datatypes with predefinied operations, with some
+                       restrictions outlined in MPI-3:11.3.4.  The derived
+                       datatype must be composed entirely from one predefined
+                       datatype (so you can do all the construction you want,
+                       but at the bottom, you can only use one datatype, say,
+                       MPI_INT).  If the datatype at the target isn't
+                       predefined, then make sure it's composed of only one
+                       datatype, and check that datatype against
+                       ompi_op_is_valid(). */
+                    origin_check_dt = ompi_datatype_get_single_predefined_type_from_args(origin_datatype);
+                    op_check_dt = ompi_datatype_get_single_predefined_type_from_args(target_datatype);
+
+                    if( !((origin_check_dt == op_check_dt) & (NULL != op_check_dt)) ) {
+                        OMPI_ERRHANDLER_RETURN(MPI_ERR_ARG, win, MPI_ERR_ARG, FUNC_NAME);
+                    }
+
+                    /* check to make sure primitive type is valid for
+                       reduction.  Should do this on the target, but
+                       then can't get the errcode back for this
+                       call */
+                    if (!ompi_op_is_valid(op, op_check_dt, &msg, FUNC_NAME)) {
+                        int ret = OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_OP, msg);
+                        free(msg);
+                        return ret;
+                    }
+                }
+            }
+        }
+        OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME);
+    }
+
+    if (MPI_PROC_NULL == target_rank) {
+        *request = &ompi_request_empty;
+        return MPI_SUCCESS;
+    }
+
+    rc = ompi_win->w_osc_module->osc_raccumulate(origin_addr,
+                                                origin_count,
+                                                origin_datatype,
+                                                target_rank,
+                                                target_disp,
+                                                target_count,
+                                                target_datatype,
+                                                op, win, request);
+    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/recv.c b/ompi/mpi/c/recv.c
deleted file mode 100644
index d51fb4d7acb..00000000000
--- a/ompi/mpi/c/recv.c
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/memchecker.h"
-#include "ompi/request/request.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Recv = PMPI_Recv
-#endif
-#define MPI_Recv PMPI_Recv
-#endif
-
-static const char FUNC_NAME[] = "MPI_Recv";
-
-
-int MPI_Recv(void *buf, int count, MPI_Datatype type, int source,
-             int tag, MPI_Comm comm, MPI_Status *status)
-{
-    int rc = MPI_SUCCESS;
-
-    SPC_RECORD(OMPI_SPC_RECV, 1);
-
-    MEMCHECKER(
-        memchecker_datatype(type);
-        memchecker_call(&opal_memchecker_base_isaddressable, buf, count, type);
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        OMPI_CHECK_DATATYPE_FOR_RECV(rc, type, count);
-        OMPI_CHECK_USER_BUFFER(rc, buf, type, count);
-
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
-        } else if (((tag < 0) && (tag != MPI_ANY_TAG)) || (tag > mca_pml.pml_max_tag)) {
-            rc = MPI_ERR_TAG;
-        } else if ((source != MPI_ANY_SOURCE) &&
-                   (MPI_PROC_NULL != source) &&
-                   ompi_comm_peer_invalid(comm, source)) {
-            rc = MPI_ERR_RANK;
-        }
-
-        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are communicating with
-     * a failed process. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the completion
-     * call in the PML.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_p2p_check_proc(comm, source, &rc)) ) {
-        if (MPI_STATUS_IGNORE != status) {
-            status->MPI_SOURCE = source;
-            status->MPI_TAG    = tag;
-        }
-        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-    }
-#endif
-
-    if (MPI_PROC_NULL == source) {
-        if (MPI_STATUS_IGNORE != status) {
-            *status = ompi_request_empty.req_status;
-        }
-        return MPI_SUCCESS;
-    }
-
-    rc = MCA_PML_CALL(recv(buf, count, type, source, tag, comm, status));
-    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/recv.c.in b/ompi/mpi/c/recv.c.in
new file mode 100644
index 00000000000..6a022fc14a8
--- /dev/null
+++ b/ompi/mpi/c/recv.c.in
@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/memchecker.h"
+#include "ompi/request/request.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS recv(BUFFER_OUT buf, COUNT count, DATATYPE type,
+                           INT source, INT tag, COMM comm, STATUS_OUT status)
+{
+    int rc = MPI_SUCCESS;
+
+    SPC_RECORD(OMPI_SPC_RECV, 1);
+
+    MEMCHECKER(
+        memchecker_datatype(type);
+        memchecker_call(&opal_memchecker_base_isaddressable, buf, count, type);
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        OMPI_CHECK_DATATYPE_FOR_RECV(rc, type, count);
+        OMPI_CHECK_USER_BUFFER(rc, buf, type, count);
+
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+        } else if (((tag < 0) && (tag != MPI_ANY_TAG)) || (tag > mca_pml.pml_max_tag)) {
+            rc = MPI_ERR_TAG;
+        } else if ((source != MPI_ANY_SOURCE) &&
+                   (MPI_PROC_NULL != source) &&
+                   ompi_comm_peer_invalid(comm, source)) {
+            rc = MPI_ERR_RANK;
+        }
+
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are communicating with
+     * a failed process. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the completion
+     * call in the PML.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_p2p_check_proc(comm, source, &rc)) ) {
+        if (MPI_STATUS_IGNORE != status) {
+            status->MPI_SOURCE = source;
+            status->MPI_TAG    = tag;
+        }
+        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+    }
+#endif
+
+    if (MPI_PROC_NULL == source) {
+        if (MPI_STATUS_IGNORE != status) {
+            OMPI_COPY_STATUS(status, ompi_request_empty.req_status, false);
+            /*
+             * Per MPI-1, the MPI_ERROR field is not defined for single-completion calls
+             */
+            MEMCHECKER(
+                opal_memchecker_base_mem_undefined(&status->MPI_ERROR, sizeof(int));
+            );
+        }
+        return MPI_SUCCESS;
+    }
+
+    rc = MCA_PML_CALL(recv(buf, count, type, source, tag, comm, status));
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/recv_init.c b/ompi/mpi/c/recv_init.c
deleted file mode 100644
index a6d34477967..00000000000
--- a/ompi/mpi/c/recv_init.c
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/request/request.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Recv_init = PMPI_Recv_init
-#endif
-#define MPI_Recv_init PMPI_Recv_init
-#endif
-
-static const char FUNC_NAME[] = "MPI_Recv_init";
-
-
-int MPI_Recv_init(void *buf, int count, MPI_Datatype type, int source,
-                  int tag, MPI_Comm comm, MPI_Request *request)
-{
-    int rc = MPI_SUCCESS;
-
-    MEMCHECKER(
-        memchecker_datatype(type);
-        memchecker_call(&opal_memchecker_base_isaddressable, buf, count, type);
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        OMPI_CHECK_DATATYPE_FOR_RECV(rc, type, count);
-        OMPI_CHECK_USER_BUFFER(rc, buf, type, count);
-
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
-        } else if (((tag < 0) && (tag != MPI_ANY_TAG)) || (tag > mca_pml.pml_max_tag)) {
-            rc = MPI_ERR_TAG;
-        } else if ((source != MPI_ANY_SOURCE) &&
-                   (MPI_PROC_NULL != source) &&
-                   ompi_comm_peer_invalid(comm, source)) {
-            rc = MPI_ERR_RANK;
-        } else if (NULL == request) {
-            rc = MPI_ERR_REQUEST;
-        }
-
-        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-    }
-
-    if (MPI_PROC_NULL == source) {
-        rc = ompi_request_persistent_noop_create(request);
-        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * The request will be checked for process failure errors during the
-     * completion calls. So no need to check here.
-     */
-#endif
-
-    /*
-     * Here, we just initialize the request -- memchecker should set the buffer in MPI_Start.
-     */
-    rc = MCA_PML_CALL(irecv_init(buf,count,type,source,tag,comm,request));
-    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/recv_init.c.in b/ompi/mpi/c/recv_init.c.in
new file mode 100644
index 00000000000..33386f94c25
--- /dev/null
+++ b/ompi/mpi/c/recv_init.c.in
@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/request/request.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS recv_init(BUFFER_OUT buf, COUNT count, DATATYPE type, INT source,
+                                INT tag, COMM comm, REQUEST_INOUT request)
+{
+    int rc = MPI_SUCCESS;
+
+    MEMCHECKER(
+        memchecker_datatype(type);
+        memchecker_call(&opal_memchecker_base_isaddressable, buf, count, type);
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        OMPI_CHECK_DATATYPE_FOR_RECV(rc, type, count);
+        OMPI_CHECK_USER_BUFFER(rc, buf, type, count);
+
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+        } else if (((tag < 0) && (tag != MPI_ANY_TAG)) || (tag > mca_pml.pml_max_tag)) {
+            rc = MPI_ERR_TAG;
+        } else if ((source != MPI_ANY_SOURCE) &&
+                   (MPI_PROC_NULL != source) &&
+                   ompi_comm_peer_invalid(comm, source)) {
+            rc = MPI_ERR_RANK;
+        } else if (NULL == request) {
+            rc = MPI_ERR_REQUEST;
+        }
+
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+    if (MPI_PROC_NULL == source) {
+        rc = ompi_request_persistent_noop_create(request);
+        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * The request will be checked for process failure errors during the
+     * completion calls. So no need to check here.
+     */
+#endif
+
+    /*
+     * Here, we just initialize the request -- memchecker should set the buffer in MPI_Start.
+     */
+    rc = MCA_PML_CALL(irecv_init(buf,count,type,source,tag,comm,request));
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/reduce.c b/ompi/mpi/c/reduce.c
deleted file mode 100644
index b1673576d5b..00000000000
--- a/ompi/mpi/c/reduce.c
+++ /dev/null
@@ -1,154 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/op/op.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Reduce = PMPI_Reduce
-#endif
-#define MPI_Reduce PMPI_Reduce
-#endif
-
-static const char FUNC_NAME[] = "MPI_Reduce";
-
-
-int MPI_Reduce(const void *sendbuf, void *recvbuf, int count,
-               MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm)
-{
-    int err;
-
-    SPC_RECORD(OMPI_SPC_REDUCE, 1);
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_comm(comm);
-
-        if(OMPI_COMM_IS_INTRA(comm)) {
-            if(ompi_comm_rank(comm) == root) {
-                /* check whether root's send buffer is defined. */
-                if (MPI_IN_PLACE == sendbuf) {
-                    memchecker_call(&opal_memchecker_base_isdefined, recvbuf, count, datatype);
-                } else {
-                    memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
-                }
-
-                /* check whether root's receive buffer is addressable. */
-                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, count, datatype);
-            } else {
-                /* check whether send buffer is defined on other processes. */
-                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
-            }
-        } else {
-            if (MPI_ROOT == root) {
-                /* check whether root's receive buffer is addressable. */
-                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, count, datatype);
-            } else if (MPI_PROC_NULL != root) {
-                /* check whether send buffer is defined. */
-                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
-            }
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        char *msg;
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-
-        /* Checks for all ranks */
-
-        else if (MPI_OP_NULL == op || NULL == op) {
-            err = MPI_ERR_OP;
-        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
-            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
-            free(msg);
-            return ret;
-        } else if ((ompi_comm_rank(comm) != root && MPI_IN_PLACE == sendbuf) ||
-                   (ompi_comm_rank(comm) == root && ((MPI_IN_PLACE == recvbuf) || (sendbuf == recvbuf)))) {
-            err = MPI_ERR_ARG;
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-
-        /* Intercommunicator errors */
-
-        if (!OMPI_COMM_IS_INTRA(comm)) {
-            if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
-                   MPI_ROOT == root || MPI_PROC_NULL == root)) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
-            }
-        }
-
-        /* Intracommunicator errors */
-
-        else {
-            if (root < 0 || root >= ompi_comm_size(comm)) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
-            }
-        }
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
-        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-    }
-#endif
-
-    /* Do we need to do anything? (MPI says that reductions have to
-       have a count of at least 1, but at least IMB calls reduce with
-       a count of 0 -- blah!) */
-
-    if (0 == count) {
-        return MPI_SUCCESS;
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-
-    OBJ_RETAIN(op);
-    err = comm->c_coll->coll_reduce(sendbuf, recvbuf, count,
-                                   datatype, op, root, comm,
-                                   comm->c_coll->coll_reduce_module);
-    OBJ_RELEASE(op);
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/reduce.c.in b/ompi/mpi/c/reduce.c.in
new file mode 100644
index 00000000000..04aecd91675
--- /dev/null
+++ b/ompi/mpi/c/reduce.c.in
@@ -0,0 +1,156 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2023 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/op/op.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS reduce(BUFFER sendbuf, BUFFER_OUT recvbuf, COUNT count,
+                             DATATYPE datatype, OP op, INT root, COMM comm)
+{
+    int err;
+
+    SPC_RECORD(OMPI_SPC_REDUCE, 1);
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_comm(comm);
+
+        if(OMPI_COMM_IS_INTRA(comm)) {
+            if(ompi_comm_rank(comm) == root) {
+                /* check whether root's send buffer is defined. */
+                if (MPI_IN_PLACE == sendbuf) {
+                    memchecker_call(&opal_memchecker_base_isdefined, recvbuf, count, datatype);
+                } else {
+                    memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
+                }
+
+                /* check whether root's receive buffer is addressable. */
+                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, count, datatype);
+            } else {
+                /* check whether send buffer is defined on other processes. */
+                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
+            }
+        } else {
+            if (MPI_ROOT == root) {
+                /* check whether root's receive buffer is addressable. */
+                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, count, datatype);
+            } else if (MPI_PROC_NULL != root) {
+                /* check whether send buffer is defined. */
+                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
+            }
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        char *msg;
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+
+        /* Checks for all ranks */
+
+        else if (MPI_OP_NULL == op || NULL == op) {
+            err = MPI_ERR_OP;
+        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
+            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
+            free(msg);
+            return ret;
+        } else if ((ompi_comm_rank(comm) != root && MPI_IN_PLACE == sendbuf) ||
+                   (ompi_comm_rank(comm) == root && ((MPI_IN_PLACE == recvbuf) ||
+                   ((sendbuf == recvbuf) && (0 != count))))) {
+            err = MPI_ERR_ARG;
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+
+        /* Intercommunicator errors */
+
+        if (!OMPI_COMM_IS_INTRA(comm)) {
+            if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
+                   MPI_ROOT == root || MPI_PROC_NULL == root)) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
+            }
+        }
+
+        /* Intracommunicator errors */
+
+        else {
+            if (root < 0 || root >= ompi_comm_size(comm)) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
+            }
+        }
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
+        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+    }
+#endif
+
+    /* Do we need to do anything? (MPI says that reductions have to
+       have a count of at least 1, but at least IMB calls reduce with
+       a count of 0 -- blah!) */
+
+    if (0 == count) {
+        return MPI_SUCCESS;
+    }
+
+    void *updated_recvbuf;
+    const void *updated_sendbuf;
+    if(OMPI_COMM_IS_INTRA(comm)) {
+        updated_recvbuf = (ompi_comm_rank(comm) == root) ? recvbuf : NULL;
+        updated_sendbuf = sendbuf;
+    } else {
+        updated_sendbuf = ((MPI_ROOT == root) || (MPI_PROC_NULL == root)) ? NULL : sendbuf;
+        updated_recvbuf = (MPI_ROOT == root) ? recvbuf : NULL;
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    OBJ_RETAIN(op);
+    err = comm->c_coll->coll_reduce(updated_sendbuf, updated_recvbuf, count,
+                                    datatype, op, root, comm,
+                                    comm->c_coll->coll_reduce_module);
+    OBJ_RELEASE(op);
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/reduce_init.c.in b/ompi/mpi/c/reduce_init.c.in
new file mode 100644
index 00000000000..1912420e798
--- /dev/null
+++ b/ompi/mpi/c/reduce_init.c.in
@@ -0,0 +1,149 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2023 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2016      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/op/op.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS reduce_init(BUFFER sendbuf, BUFFER_OUT recvbuf, COUNT count,
+                                  DATATYPE datatype, OP op, INT root, COMM comm,
+                                  INFO info, REQUEST_INOUT request)
+{
+    int err;
+
+    SPC_RECORD(OMPI_SPC_REDUCE_INIT, 1);
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_comm(comm);
+
+        if(OMPI_COMM_IS_INTRA(comm)) {
+            if(ompi_comm_rank(comm) == root) {
+                /* check whether root's send buffer is defined. */
+                if (MPI_IN_PLACE == sendbuf) {
+                    memchecker_call(&opal_memchecker_base_isdefined, recvbuf, count, datatype);
+                } else {
+                    memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
+                }
+
+                /* check whether root's receive buffer is addressable. */
+                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, count, datatype);
+            } else {
+                /* check whether send buffer is defined on other processes. */
+                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
+            }
+        } else {
+            if (MPI_ROOT == root) {
+                /* check whether root's receive buffer is addressable. */
+                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, count, datatype);
+            } else if (MPI_PROC_NULL != root) {
+                /* check whether send buffer is defined. */
+                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
+            }
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        char *msg;
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+
+        /* Checks for all ranks */
+
+        else if (MPI_OP_NULL == op || NULL == op) {
+            err = MPI_ERR_OP;
+        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
+            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
+            free(msg);
+            return ret;
+        } else if ((ompi_comm_rank(comm) != root && MPI_IN_PLACE == sendbuf) ||
+                   (ompi_comm_rank(comm) == root && ((MPI_IN_PLACE == recvbuf) || (sendbuf == recvbuf)))) {
+            err = MPI_ERR_ARG;
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+
+        /* Intercommunicator errors */
+
+        if (!OMPI_COMM_IS_INTRA(comm)) {
+            if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
+                   MPI_ROOT == root || MPI_PROC_NULL == root)) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
+            }
+        }
+
+        /* Intracommunicator errors */
+
+        else {
+            if (root < 0 || root >= ompi_comm_size(comm)) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
+            }
+        }
+    }
+
+    /* MPI standard says that reductions have to have a count of at least 1,
+     * but some benchmarks (e.g., IMB) calls this function with a count of 0.
+     * So handle that case.
+     */
+    if (0 == count) {
+        err = ompi_request_persistent_noop_create(request);
+        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+    }
+
+    void *updated_recvbuf;
+    const void *updated_sendbuf;
+    if(OMPI_COMM_IS_INTRA(comm)) {
+        updated_recvbuf = (ompi_comm_rank(comm) == root) ? recvbuf : NULL;
+        updated_sendbuf = sendbuf;
+    } else {
+        updated_sendbuf = ((MPI_ROOT == root) || (MPI_PROC_NULL == root)) ? NULL : sendbuf;
+        updated_recvbuf = (MPI_ROOT == root) ? recvbuf : NULL;
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    err = comm->c_coll->coll_reduce_init(updated_sendbuf, updated_recvbuf, count,
+                                         datatype, op, root, comm, info, request,
+                                         comm->c_coll->coll_reduce_init_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_op(*request, op, datatype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/reduce_local.c b/ompi/mpi/c/reduce_local.c
deleted file mode 100644
index 982edd7c6a1..00000000000
--- a/ompi/mpi/c/reduce_local.c
+++ /dev/null
@@ -1,89 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2009 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2016-2017 IBM Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/op/op.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Reduce_local = PMPI_Reduce_local
-#endif
-#define MPI_Reduce_local PMPI_Reduce_local
-#endif
-
-static const char FUNC_NAME[] = "MPI_Reduce_local";
-
-
-int MPI_Reduce_local(const void *inbuf, void *inoutbuf, int count,
-                     MPI_Datatype datatype, MPI_Op op)
-{
-    int err;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        char *msg;
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (MPI_OP_NULL == op || NULL == op) {
-            err = MPI_ERR_OP;
-        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
-            int ret = OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_OP, msg);
-            free(msg);
-            return ret;
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
-        }
-        OMPI_ERRHANDLER_NOHANDLE_CHECK(err, err, FUNC_NAME);
-    }
-
-    /* If the count is 0, just return */
-    if (0 == count) {
-        return MPI_SUCCESS;
-    }
-
-    /* Invoke the op component to perform the back-end operation */
-    OBJ_RETAIN(op);
-    OBJ_RETAIN(datatype);
-    // Since there is no 'comm' parameter to this interface use 'self' since
-    // this is a local operation to this process.
-    ompi_communicator_t *comm = &ompi_mpi_comm_self.comm;
-    err = comm->c_coll->coll_reduce_local(inbuf, inoutbuf, count, datatype, op,
-                                         comm->c_coll->coll_reduce_local_module);
-    OBJ_RELEASE(datatype);
-    OBJ_RELEASE(op);
-
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/reduce_local.c.in b/ompi/mpi/c/reduce_local.c.in
new file mode 100644
index 00000000000..29bce96f91c
--- /dev/null
+++ b/ompi/mpi/c/reduce_local.c.in
@@ -0,0 +1,81 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2009 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016-2017 IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/op/op.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS reduce_local(BUFFER inbuf, BUFFER_OUT inoutbuf, COUNT count,
+                                   DATATYPE datatype, OP op)
+{
+    int err;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        char *msg;
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (MPI_OP_NULL == op || NULL == op) {
+            err = MPI_ERR_OP;
+        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
+            int ret = OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_OP, msg);
+            free(msg);
+            return ret;
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
+        }
+        OMPI_ERRHANDLER_NOHANDLE_CHECK(err, err, FUNC_NAME);
+    }
+
+    /* If the count is 0, just return */
+    if (0 == count) {
+        return MPI_SUCCESS;
+    }
+
+    /* Invoke the op component to perform the back-end operation */
+    OBJ_RETAIN(op);
+    OBJ_RETAIN(datatype);
+    // Since there is no 'comm' parameter to this interface use 'self' since
+    // this is a local operation to this process.
+    ompi_communicator_t *comm = &ompi_mpi_comm_self.comm;
+    err = comm->c_coll->coll_reduce_local(inbuf, inoutbuf, count, datatype, op,
+                                         comm->c_coll->coll_reduce_local_module);
+    OBJ_RELEASE(datatype);
+    OBJ_RELEASE(op);
+
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/reduce_scatter.c b/ompi/mpi/c/reduce_scatter.c
deleted file mode 100644
index 19967582ae6..00000000000
--- a/ompi/mpi/c/reduce_scatter.c
+++ /dev/null
@@ -1,149 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/op/op.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Reduce_scatter = PMPI_Reduce_scatter
-#endif
-#define MPI_Reduce_scatter PMPI_Reduce_scatter
-#endif
-
-static const char FUNC_NAME[] = "MPI_Reduce_scatter";
-
-
-int MPI_Reduce_scatter(const void *sendbuf, void *recvbuf, const int recvcounts[],
-                       MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
-{
-    int i, err, size, count;
-
-    SPC_RECORD(OMPI_SPC_REDUCE_SCATTER, 1);
-
-    MEMCHECKER(
-        int rank;
-
-        size = ompi_comm_size(comm);
-        rank = ompi_comm_rank(comm);
-        for (count = i = 0; i < size; ++i) {
-            if (0 == recvcounts[i]) {
-                count += recvcounts[i];
-            }
-        }
-
-        memchecker_comm(comm);
-        memchecker_datatype(datatype);
-
-        /* check receive buffer of current proccess, whether it's addressable. */
-        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf,
-                        recvcounts[rank], datatype);
-
-        /* check whether the actual send buffer is defined. */
-        if(MPI_IN_PLACE == sendbuf) {
-            memchecker_call(&opal_memchecker_base_isdefined, recvbuf, count, datatype);
-        } else {
-            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
-
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        char *msg;
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-
-        /* Unrooted operation; same checks for all ranks on both
-           intracommunicators and intercommunicators */
-
-        else if (MPI_OP_NULL == op || NULL == op) {
-          err = MPI_ERR_OP;
-        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
-            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
-            free(msg);
-            return ret;
-        } else if (NULL == recvcounts) {
-          err = MPI_ERR_COUNT;
-        } else if (MPI_IN_PLACE == recvbuf) {
-          err = MPI_ERR_ARG;
-        }
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-
-        /* Based on the standard each group has to provide the same total
-           number of elements, so the size of the recvcounts array depends
-           on the number of participants in the local group.  */
-        size = ompi_comm_size(comm);
-        for (i = 0; i < size; ++i) {
-          OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, recvcounts[i]);
-          OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
-        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-    }
-#endif
-
-    /* MPI-1, p114, says that each process must supply at least one
-       element.  But at least the Pallas benchmarks call MPI_REDUCE
-       with a count of 0.  So be sure to handle it.  Grrr... */
-
-    size = ompi_comm_size(comm);
-    for (count = i = 0; i < size; ++i) {
-        if (0 == recvcounts[i]) {
-            ++count;
-        }
-    }
-    if (size == count) {
-        return MPI_SUCCESS;
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-
-    OBJ_RETAIN(op);
-    err = comm->c_coll->coll_reduce_scatter(sendbuf, recvbuf, recvcounts,
-                                           datatype, op, comm,
-                                           comm->c_coll->coll_reduce_scatter_module);
-    OBJ_RELEASE(op);
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/reduce_scatter.c.in b/ompi/mpi/c/reduce_scatter.c.in
new file mode 100644
index 00000000000..4a83f57ea26
--- /dev/null
+++ b/ompi/mpi/c/reduce_scatter.c.in
@@ -0,0 +1,143 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2018 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/op/op.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS reduce_scatter(BUFFER sendbuf, BUFFER_OUT recvbuf, COUNT_ARRAY recvcounts,
+                                     DATATYPE datatype, OP op, COMM comm)
+{
+    int i, err, size, count;
+    ompi_count_array_t recvcounts_desc;
+
+    SPC_RECORD(OMPI_SPC_REDUCE_SCATTER, 1);
+
+    MEMCHECKER(
+        int rank;
+
+        size = ompi_comm_size(comm);
+        rank = ompi_comm_rank(comm);
+        for (count = i = 0; i < size; ++i) {
+            if (0 == recvcounts[i]) {
+                count += recvcounts[i];
+            }
+        }
+
+        memchecker_comm(comm);
+        memchecker_datatype(datatype);
+
+        /* check receive buffer of current process, whether it's addressable. */
+        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf,
+                        recvcounts[rank], datatype);
+
+        /* check whether the actual send buffer is defined. */
+        if(MPI_IN_PLACE == sendbuf) {
+            memchecker_call(&opal_memchecker_base_isdefined, recvbuf, count, datatype);
+        } else {
+            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
+
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        char *msg;
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+
+        /* Unrooted operation; same checks for all ranks on both
+           intracommunicators and intercommunicators */
+
+        else if (MPI_OP_NULL == op || NULL == op) {
+          err = MPI_ERR_OP;
+        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
+            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
+            free(msg);
+            return ret;
+        } else if (NULL == recvcounts) {
+          err = MPI_ERR_COUNT;
+        } else if (MPI_IN_PLACE == recvbuf) {
+          err = MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+
+        /* Based on the standard each group has to provide the same total
+           number of elements, so the size of the recvcounts array depends
+           on the number of participants in the local group.  */
+        size = ompi_comm_size(comm);
+        for (i = 0; i < size; ++i) {
+          OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, recvcounts[i]);
+          OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
+        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+    }
+#endif
+
+    /* MPI-1, p114, says that each process must supply at least one
+       element.  But at least the Pallas benchmarks call MPI_REDUCE
+       with a count of 0.  So be sure to handle it.  Grrr... */
+
+    size = ompi_comm_size(comm);
+    for (count = i = 0; i < size; ++i) {
+        if (0 == recvcounts[i]) {
+            ++count;
+        }
+    }
+    if (size == count) {
+        return MPI_SUCCESS;
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+
+    OMPI_COUNT_ARRAY_INIT(&recvcounts_desc, recvcounts);
+    OBJ_RETAIN(op);
+    err = comm->c_coll->coll_reduce_scatter(sendbuf, recvbuf, recvcounts_desc,
+                                           datatype, op, comm,
+                                           comm->c_coll->coll_reduce_scatter_module);
+    OBJ_RELEASE(op);
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/reduce_scatter_block.c b/ompi/mpi/c/reduce_scatter_block.c
deleted file mode 100644
index 94386953178..00000000000
--- a/ompi/mpi/c/reduce_scatter_block.c
+++ /dev/null
@@ -1,120 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012      Oak Ridge National Labs. All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/op/op.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Reduce_scatter_block = PMPI_Reduce_scatter_block
-#endif
-#define MPI_Reduce_scatter_block PMPI_Reduce_scatter_block
-#endif
-
-static const char FUNC_NAME[] = "MPI_Reduce_scatter_block";
-
-
-int MPI_Reduce_scatter_block(const void *sendbuf, void *recvbuf, int recvcount,
-                             MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
-{
-    int err;
-
-    SPC_RECORD(OMPI_SPC_REDUCE_SCATTER_BLOCK, 1);
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-        memchecker_datatype(datatype);
-
-        /* check receive buffer of current proccess, whether it's addressable. */
-        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf,
-                        recvcount, datatype);
-
-        /* check whether the actual send buffer is defined. */
-        if(MPI_IN_PLACE == sendbuf) {
-            memchecker_call(&opal_memchecker_base_isdefined, recvbuf, recvcount, datatype);
-        } else {
-            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, recvcount, datatype);
-
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        char *msg;
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-
-        /* Unrooted operation; same checks for all ranks on both
-           intracommunicators and intercommunicators */
-
-        else if (MPI_OP_NULL == op || NULL == op) {
-          err = MPI_ERR_OP;
-        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
-            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
-            free(msg);
-            return ret;
-        } else if (MPI_IN_PLACE == recvbuf) {
-          err = MPI_ERR_ARG;
-        }
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-
-        OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, recvcount);
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-    }
-    if (0 == recvcount) {
-        return MPI_SUCCESS;
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
-        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-    }
-#endif
-
-    /* Invoke the coll component to perform the back-end operation */
-
-    OBJ_RETAIN(op);
-    err = comm->c_coll->coll_reduce_scatter_block(sendbuf, recvbuf, recvcount,
-                                                 datatype, op, comm,
-                                                 comm->c_coll->coll_reduce_scatter_block_module);
-    OBJ_RELEASE(op);
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/reduce_scatter_block.c.in b/ompi/mpi/c/reduce_scatter_block.c.in
new file mode 100644
index 00000000000..124a0e9ebf9
--- /dev/null
+++ b/ompi/mpi/c/reduce_scatter_block.c.in
@@ -0,0 +1,112 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oak Ridge National Labs. All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2018 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/op/op.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS reduce_scatter_block(BUFFER sendbuf, BUFFER_OUT recvbuf, COUNT recvcount,
+                                           DATATYPE datatype, OP op, COMM comm)
+{
+    int err;
+
+    SPC_RECORD(OMPI_SPC_REDUCE_SCATTER_BLOCK, 1);
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+        memchecker_datatype(datatype);
+
+        /* check receive buffer of current process, whether it's addressable. */
+        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf,
+                        recvcount, datatype);
+
+        /* check whether the actual send buffer is defined. */
+        if(MPI_IN_PLACE == sendbuf) {
+            memchecker_call(&opal_memchecker_base_isdefined, recvbuf, recvcount, datatype);
+        } else {
+            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, recvcount, datatype);
+
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        char *msg;
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+
+        /* Unrooted operation; same checks for all ranks on both
+           intracommunicators and intercommunicators */
+
+        else if (MPI_OP_NULL == op || NULL == op) {
+          err = MPI_ERR_OP;
+        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
+            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
+            free(msg);
+            return ret;
+        } else if (MPI_IN_PLACE == recvbuf) {
+          err = MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+
+        OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, recvcount);
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+    }
+    if (0 == recvcount) {
+        return MPI_SUCCESS;
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
+        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+    }
+#endif
+
+    /* Invoke the coll component to perform the back-end operation */
+
+    OBJ_RETAIN(op);
+    err = comm->c_coll->coll_reduce_scatter_block(sendbuf, recvbuf, recvcount,
+                                                 datatype, op, comm,
+                                                 comm->c_coll->coll_reduce_scatter_block_module);
+    OBJ_RELEASE(op);
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/reduce_scatter_block_init.c.in b/ompi/mpi/c/reduce_scatter_block_init.c.in
new file mode 100644
index 00000000000..46aec913ef1
--- /dev/null
+++ b/ompi/mpi/c/reduce_scatter_block_init.c.in
@@ -0,0 +1,101 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oak Ridge National Labs. All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/op/op.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS reduce_scatter_block_init(BUFFER sendbuf, BUFFER_OUT recvbuf, COUNT recvcount,
+                                                DATATYPE datatype, OP op,
+                                                COMM comm, INFO info, REQUEST_INOUT request)
+{
+    int err;
+
+    SPC_RECORD(OMPI_SPC_REDUCE_SCATTER_BLOCK_INIT, 1);
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+        memchecker_datatype(datatype);
+
+        /* check receive buffer of current process, whether it's addressable. */
+        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf,
+                        recvcount, datatype);
+
+        /* check whether the actual send buffer is defined. */
+        if(MPI_IN_PLACE == sendbuf) {
+            memchecker_call(&opal_memchecker_base_isdefined, recvbuf, recvcount, datatype);
+        } else {
+            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, recvcount, datatype);
+
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        char *msg;
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+
+        /* Unrooted operation; same checks for all ranks on both
+           intracommunicators and intercommunicators */
+
+        else if (MPI_OP_NULL == op || NULL == op) {
+          err = MPI_ERR_OP;
+        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
+            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
+            free(msg);
+            return ret;
+        } else if (MPI_IN_PLACE == recvbuf) {
+          err = MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+
+        OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, recvcount);
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+
+    err = comm->c_coll->coll_reduce_scatter_block_init(sendbuf, recvbuf, recvcount,
+                                                       datatype, op, comm, info, request,
+                                                       comm->c_coll->coll_reduce_scatter_block_init_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_op(*request, op, datatype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/reduce_scatter_init.c.in b/ompi/mpi/c/reduce_scatter_init.c.in
new file mode 100644
index 00000000000..717093ca410
--- /dev/null
+++ b/ompi/mpi/c/reduce_scatter_init.c.in
@@ -0,0 +1,137 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2016      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/op/op.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS reduce_scatter_init(BUFFER sendbuf, BUFFER_OUT recvbuf, COUNT_ARRAY recvcounts,
+                                          DATATYPE datatype, OP op, COMM comm, INFO info, REQUEST_INOUT request)
+{
+    int i, err, size, count;
+    ompi_count_array_t recvcounts_desc;
+
+    SPC_RECORD(OMPI_SPC_REDUCE_SCATTER_INIT, 1);
+
+    MEMCHECKER(
+        int rank;
+        int count;
+
+        size = ompi_comm_size(comm);
+        rank = ompi_comm_rank(comm);
+        for (count = i = 0; i < size; ++i) {
+            if (0 == recvcounts[i]) {
+                count += recvcounts[i];
+            }
+        }
+
+        memchecker_comm(comm);
+        memchecker_datatype(datatype);
+
+        /* check receive buffer of current process, whether it's addressable. */
+        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf,
+                        recvcounts[rank], datatype);
+
+        /* check whether the actual send buffer is defined. */
+        if(MPI_IN_PLACE == sendbuf) {
+            memchecker_call(&opal_memchecker_base_isdefined, recvbuf, count, datatype);
+        } else {
+            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
+
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        char *msg;
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+
+        /* Unrooted operation; same checks for all ranks on both
+           intracommunicators and intercommunicators */
+
+        else if (MPI_OP_NULL == op || NULL == op) {
+          err = MPI_ERR_OP;
+        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
+            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
+            free(msg);
+            return ret;
+        } else if (NULL == recvcounts) {
+          err = MPI_ERR_COUNT;
+        } else if (MPI_IN_PLACE == recvbuf) {
+          err = MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+
+        /* Based on the standard each group has to provide the same total
+           number of elements, so the size of the recvcounts array depends
+           on the number of participants in the local group.  */
+        size = ompi_comm_size(comm);
+        for (i = 0; i < size; ++i) {
+          OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, recvcounts[i]);
+          OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+    }
+
+    /* MPI standard says that reductions have to have a count of at least 1,
+     * but some benchmarks (e.g., IMB) calls this function with a count of 0.
+     * So handle that case.
+     */
+    size = ompi_comm_size(comm);
+    for (count = i = 0; i < size; ++i) {
+        if (0 == recvcounts[i]) {
+            ++count;
+        }
+    }
+    if (size == count) {
+        err = ompi_request_persistent_noop_create(request);
+        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+
+    OMPI_COUNT_ARRAY_INIT(&recvcounts_desc, recvcounts);
+    err = comm->c_coll->coll_reduce_scatter_init(sendbuf, recvbuf, recvcounts_desc,
+                                                 datatype, op, comm, info, request,
+                                                 comm->c_coll->coll_reduce_scatter_init_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_op(*request, op, datatype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/register_datarep.c b/ompi/mpi/c/register_datarep.c
deleted file mode 100644
index 2ce49dbe9f3..00000000000
--- a/ompi/mpi/c/register_datarep.c
+++ /dev/null
@@ -1,79 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/io/base/base.h"
-#include "ompi/file/file.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Register_datarep = PMPI_Register_datarep
-#endif
-#define MPI_Register_datarep PMPI_Register_datarep
-#endif
-
-static const char FUNC_NAME[] = "MPI_Register_datarep";
-
-
-int MPI_Register_datarep(const char *datarep,
-			 MPI_Datarep_conversion_function *read_conversion_fn,
-			 MPI_Datarep_conversion_function *write_conversion_fn,
-			 MPI_Datarep_extent_function *dtype_file_extent_fn,
-			 void *extra_state)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == datarep) {
-            rc = MPI_ERR_ARG;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, MPI_FILE_NULL, rc, FUNC_NAME);
-    }
-
-    /* The io framework is only initialized lazily.  If it hasn't
-       already been initialized, do so now (note that MPI_FILE_OPEN
-       and MPI_FILE_DELETE are the only two places that it will be
-       initialized). */
-
-    if (OMPI_SUCCESS != (rc = mca_base_framework_open(&ompi_io_base_framework, 0))) {
-        return OMPI_ERRHANDLER_INVOKE(MPI_FILE_NULL, rc, FUNC_NAME);
-    }
-
-    /* Call the back-end io component function */
-    rc = mca_io_base_register_datarep(datarep, read_conversion_fn,
-                                      write_conversion_fn,
-                                      dtype_file_extent_fn,
-                                      extra_state);
-
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(rc, MPI_FILE_NULL, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/register_datarep.c.in b/ompi/mpi/c/register_datarep.c.in
new file mode 100644
index 00000000000..4f49fc239fc
--- /dev/null
+++ b/ompi/mpi/c/register_datarep.c.in
@@ -0,0 +1,70 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/io/base/base.h"
+#include "ompi/file/file.h"
+
+PROTOTYPE ERROR_CLASS register_datarep(STRING datarep,
+                                       DATAREP_CONVERSION_FUNCTION read_conversion_fn,
+                                       DATAREP_CONVERSION_FUNCTION write_conversion_fn,
+                                       DATAREP_EXTENT_FUNCTION dtype_file_extent_fn,
+                                       BUFFER_OUT extra_state)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == datarep) {
+            rc = MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, MPI_FILE_NULL, rc, FUNC_NAME);
+    }
+
+    /* The io framework is only initialized lazily.  If it hasn't
+       already been initialized, do so now (note that MPI_FILE_OPEN
+       and MPI_FILE_DELETE are the only two places that it will be
+       initialized). */
+
+    if (OMPI_SUCCESS != (rc = mca_base_framework_open(&ompi_io_base_framework, 0))) {
+        return OMPI_ERRHANDLER_INVOKE(MPI_FILE_NULL, rc, FUNC_NAME);
+    }
+
+    /* Call the back-end io component function */
+    rc = mca_io_base_register_datarep(datarep, (MPI_Datarep_conversion_function *) read_conversion_fn,
+                                      (MPI_Datarep_conversion_function *) write_conversion_fn,
+                                      dtype_file_extent_fn,
+                                      extra_state);
+
+
+    /* All done */
+
+    OMPI_ERRHANDLER_RETURN(rc, MPI_FILE_NULL, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/request_c2f.c b/ompi/mpi/c/request_c2f.c
deleted file mode 100644
index 9d98b2928b5..00000000000
--- a/ompi/mpi/c/request_c2f.c
+++ /dev/null
@@ -1,75 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2007 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mpi/fortran/base/fint_2_int.h"
-#include "ompi/request/request.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Request_c2f = PMPI_Request_c2f
-#endif
-#define MPI_Request_c2f PMPI_Request_c2f
-#endif
-
-static const char FUNC_NAME[] = "MPI_Request_c2f";
-
-
-MPI_Fint MPI_Request_c2f(MPI_Request request)
-{
-    MEMCHECKER(
-        memchecker_request(&request);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (NULL == request) {
-            return OMPI_INT_2_FINT(-1);
-        }
-    }
-
-    /* We only put requests in the f2c table when this function is
-       invoked.  This is because putting requests in the table
-       involves locking and unlocking the table, which would incur a
-       performance penalty (in the critical performance path) for C
-       applications.  In this way, at least only Fortran applications
-       are penalized.  :-\
-
-       Modifying this one function neatly fixes up all the Fortran
-       bindings because they all call MPI_Request_c2f in order to
-       transmorgify the C MPI_Request that they got back into a
-       fortran integer.
-    */
-
-    if (MPI_UNDEFINED == request->req_f_to_c_index) {
-        request->req_f_to_c_index =
-            opal_pointer_array_add(&ompi_request_f_to_c_table, request);
-    }
-
-    return OMPI_INT_2_FINT(request->req_f_to_c_index) ;
-}
diff --git a/ompi/mpi/c/request_c2f.c.in b/ompi/mpi/c/request_c2f.c.in
new file mode 100644
index 00000000000..65a21fb692d
--- /dev/null
+++ b/ompi/mpi/c/request_c2f.c.in
@@ -0,0 +1,67 @@
+/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+#include "ompi/request/request.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE FINT request_c2f(REQUEST request)
+{
+    MEMCHECKER(
+        memchecker_request(&request);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (NULL == request) {
+            return OMPI_INT_2_FINT(-1);
+        }
+    }
+
+    /* We only put requests in the f2c table when this function is
+       invoked.  This is because putting requests in the table
+       involves locking and unlocking the table, which would incur a
+       performance penalty (in the critical performance path) for C
+       applications.  In this way, at least only Fortran applications
+       are penalized.  :-\
+
+       Modifying this one function neatly fixes up all the Fortran
+       bindings because they all call MPI_Request_c2f in order to
+       transmorgify the C MPI_Request that they got back into a
+       fortran integer.
+    */
+
+    if (MPI_UNDEFINED == request->req_f_to_c_index) {
+        request->req_f_to_c_index =
+            opal_pointer_array_add(&ompi_request_f_to_c_table, request);
+    }
+
+    return OMPI_INT_2_FINT(request->req_f_to_c_index) ;
+}
diff --git a/ompi/mpi/c/request_f2c.c b/ompi/mpi/c/request_f2c.c
deleted file mode 100644
index 993aa4383fe..00000000000
--- a/ompi/mpi/c/request_f2c.c
+++ /dev/null
@@ -1,60 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2007 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mpi/fortran/base/fint_2_int.h"
-#include "ompi/request/request.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Request_f2c = PMPI_Request_f2c
-#endif
-#define MPI_Request_f2c PMPI_Request_f2c
-#endif
-
-static const char FUNC_NAME[] = "MPI_Request_f2c";
-
-
-MPI_Request MPI_Request_f2c(MPI_Fint request)
-{
-    int request_index = OMPI_FINT_2_INT(request);
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-    }
-
-    /* Per MPI-2:4.12.4, do not invoke an error handler if we get an
-       invalid fortran handle.  If we get an invalid fortran handle,
-       return an invalid C handle. */
-
-    if (request_index < 0 ||
-        request_index >=
-        opal_pointer_array_get_size(&ompi_request_f_to_c_table)) {
-        return NULL;
-    }
-
-    return (MPI_Request)opal_pointer_array_get_item(&ompi_request_f_to_c_table,
-                                                    request_index);
-}
diff --git a/ompi/mpi/c/request_f2c.c.in b/ompi/mpi/c/request_f2c.c.in
new file mode 100644
index 00000000000..3f6b6345f59
--- /dev/null
+++ b/ompi/mpi/c/request_f2c.c.in
@@ -0,0 +1,52 @@
+/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+#include "ompi/request/request.h"
+
+PROTOTYPE REQUEST request_f2c(FINT request)
+{
+    int request_index = OMPI_FINT_2_INT(request);
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+    }
+
+    /* Per MPI-2:4.12.4, do not invoke an error handler if we get an
+       invalid fortran handle.  If we get an invalid fortran handle,
+       return an invalid C handle. */
+
+    if (request_index < 0 ||
+        request_index >=
+        opal_pointer_array_get_size(&ompi_request_f_to_c_table)) {
+        return NULL;
+    }
+
+    return (MPI_Request)opal_pointer_array_get_item(&ompi_request_f_to_c_table,
+                                                    request_index);
+}
diff --git a/ompi/mpi/c/request_free.c b/ompi/mpi/c/request_free.c
deleted file mode 100644
index 91f91786f13..00000000000
--- a/ompi/mpi/c/request_free.c
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2009 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/request/request.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Request_free = PMPI_Request_free
-#endif
-#define MPI_Request_free PMPI_Request_free
-#endif
-
-static const char FUNC_NAME[] = "MPI_Request_free";
-
-
-int MPI_Request_free(MPI_Request *request)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_request(request);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == request || NULL == *request ||
-            MPI_REQUEST_NULL == *request) {
-            rc = MPI_ERR_REQUEST;
-        }
-        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
-    }
-
-    rc = ompi_request_free(request);
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/request_free.c.in b/ompi/mpi/c/request_free.c.in
new file mode 100644
index 00000000000..28a18e95431
--- /dev/null
+++ b/ompi/mpi/c/request_free.c.in
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2009 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/request/request.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS request_free(REQUEST_INOUT request)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_request(request);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == request || NULL == *request ||
+            MPI_REQUEST_NULL == *request) {
+            rc = MPI_ERR_REQUEST;
+        }
+        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
+    }
+
+    rc = ompi_request_free(request);
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/request_get_status.c b/ompi/mpi/c/request_get_status.c
deleted file mode 100644
index 8c7a2bfaddd..00000000000
--- a/ompi/mpi/c/request_get_status.c
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2010 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/request/request.h"
-#include "ompi/request/grequest.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Request_get_status = PMPI_Request_get_status
-#endif
-#define MPI_Request_get_status PMPI_Request_get_status
-#endif
-
-static const char FUNC_NAME[] = "MPI_Request_get_status";
-
-/* Non blocking test for the request status. Upon completion, the request will
- * not be freed (unlike the test function). A subsequent call to test, wait
- * or free should be executed on the request.
- */
-int MPI_Request_get_status(MPI_Request request, int *flag,
-                           MPI_Status *status)
-{
-#if OPAL_ENABLE_PROGRESS_THREADS == 0
-    int do_it_once = 0;
-#endif
-
-    MEMCHECKER(
-        memchecker_request(&request);
-    );
-
-    if( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if( (NULL == flag) ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-        } else if (NULL == request) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_REQUEST,
-                                          FUNC_NAME);
-        }
-    }
-
-#if OPAL_ENABLE_PROGRESS_THREADS == 0
- recheck_request_status:
-#endif
-    opal_atomic_mb();
-    if( (request == MPI_REQUEST_NULL) || (request->req_state == OMPI_REQUEST_INACTIVE) ) {
-        *flag = true;
-        if( MPI_STATUS_IGNORE != status ) {
-            *status = ompi_status_empty;
-        }
-        return MPI_SUCCESS;
-    }
-    if( request->req_complete ) {
-        *flag = true;
-        /* If this is a generalized request, we *always* have to call
-           the query function to get the status (MPI-2:8.2), even if
-           the user passed STATUS_IGNORE. */
-        if (OMPI_REQUEST_GEN == request->req_type) {
-            ompi_grequest_invoke_query(request, &request->req_status);
-        }
-        if (MPI_STATUS_IGNORE != status) {
-            *status = request->req_status;
-        }
-        return MPI_SUCCESS;
-    }
-#if OPAL_ENABLE_PROGRESS_THREADS == 0
-    if( 0 == do_it_once ) {
-        /* If we run the opal_progress then check the status of the
-           request before leaving. We will call the opal_progress only
-           once per call. */
-        opal_progress();
-        do_it_once++;
-        goto recheck_request_status;
-    }
-#endif
-    *flag = false;
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/request_get_status.c.in b/ompi/mpi/c/request_get_status.c.in
new file mode 100644
index 00000000000..86ba237866b
--- /dev/null
+++ b/ompi/mpi/c/request_get_status.c.in
@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2010 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/request/request.h"
+#include "ompi/request/grequest.h"
+#include "ompi/memchecker.h"
+
+/* Non blocking test for the request status. Upon completion, the request will
+ * not be freed (unlike the test function). A subsequent call to test, wait
+ * or free should be executed on the request.
+ */
+PROTOTYPE ERROR_CLASS request_get_status(REQUEST request, INT_OUT flag,
+                                         STATUS_OUT status)
+{
+#if OPAL_ENABLE_PROGRESS_THREADS == 0
+    int do_it_once = 0;
+#endif
+
+    MEMCHECKER(
+        memchecker_request(&request);
+    );
+
+    if( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if( (NULL == flag) ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+        } else if (NULL == request) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_REQUEST,
+                                          FUNC_NAME);
+        }
+    }
+
+#if OPAL_ENABLE_PROGRESS_THREADS == 0
+ recheck_request_status:
+#endif
+    opal_atomic_mb();
+    if( (request == MPI_REQUEST_NULL) || (request->req_state == OMPI_REQUEST_INACTIVE) ) {
+        *flag = true;
+        if( MPI_STATUS_IGNORE != status ) {
+            OMPI_COPY_STATUS(status, ompi_status_empty, false);
+        }
+        return MPI_SUCCESS;
+    }
+    if( request->req_complete ) {
+        *flag = true;
+        /* If this is a generalized request, we *always* have to call
+           the query function to get the status (MPI-2:8.2), even if
+           the user passed STATUS_IGNORE. */
+        if (OMPI_REQUEST_GEN == request->req_type) {
+            ompi_grequest_invoke_query(request, &request->req_status);
+        }
+        if (MPI_STATUS_IGNORE != status) {
+            OMPI_COPY_STATUS(status, request->req_status, false);
+        }
+        return MPI_SUCCESS;
+    }
+#if OPAL_ENABLE_PROGRESS_THREADS == 0
+    if( 0 == do_it_once ) {
+        /* If we run the opal_progress then check the status of the
+           request before leaving. We will call the opal_progress only
+           once per call. */
+        opal_progress();
+        do_it_once++;
+        goto recheck_request_status;
+    }
+#endif
+    *flag = false;
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/rget.c b/ompi/mpi/c/rget.c
deleted file mode 100644
index a5e789b6de2..00000000000
--- a/ompi/mpi/c/rget.c
+++ /dev/null
@@ -1,86 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2014-2015 Los Alamos National Security, LLC. ALl rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-#include "ompi/mca/osc/osc.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Rget = PMPI_Rget
-#endif
-#define MPI_Rget PMPI_Rget
-#endif
-
-static const char FUNC_NAME[] = "MPI_Rget";
-
-
-int MPI_Rget(void *origin_addr, int origin_count,
-             MPI_Datatype origin_datatype, int target_rank,
-             MPI_Aint target_disp, int target_count,
-             MPI_Datatype target_datatype, MPI_Win win, MPI_Request *request)
-{
-    int rc;
-
-    SPC_RECORD(OMPI_SPC_RGET, 1);
-
-    if (MPI_PARAM_CHECK) {
-        rc = OMPI_SUCCESS;
-
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        } else if (origin_count < 0 || target_count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else if (ompi_win_peer_invalid(win, target_rank) &&
-                   (MPI_PROC_NULL != target_rank)) {
-            rc = MPI_ERR_RANK;
-        } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) {
-            rc = MPI_ERR_DISP;
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, origin_datatype, origin_count);
-            if (OMPI_SUCCESS == rc) {
-                OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, target_datatype, target_count);
-            }
-        }
-        OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME);
-    }
-
-    if (MPI_PROC_NULL == target_rank) {
-        *request = &ompi_request_empty;
-        return MPI_SUCCESS;
-    }
-
-    rc = win->w_osc_module->osc_rget(origin_addr, origin_count, origin_datatype,
-                                     target_rank, target_disp, target_count,
-                                     target_datatype, win, request);
-    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/rget.c.in b/ompi/mpi/c/rget.c.in
new file mode 100644
index 00000000000..a0c70e235e9
--- /dev/null
+++ b/ompi/mpi/c/rget.c.in
@@ -0,0 +1,78 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2014-2015 Los Alamos National Security, LLC. ALl rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+#include "ompi/mca/osc/osc.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS rget(BUFFER_OUT origin_addr, COUNT origin_count,
+                           DATATYPE origin_datatype, INT target_rank,
+                           AINT target_disp, COUNT target_count,
+                           DATATYPE target_datatype, WIN win, REQUEST_INOUT request)
+{
+    int rc;
+
+    SPC_RECORD(OMPI_SPC_RGET, 1);
+
+    if (MPI_PARAM_CHECK) {
+        rc = OMPI_SUCCESS;
+
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        } else if (origin_count < 0 || target_count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else if (ompi_win_peer_invalid(win, target_rank) &&
+                   (MPI_PROC_NULL != target_rank)) {
+            rc = MPI_ERR_RANK;
+        } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) {
+            rc = MPI_ERR_DISP;
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, origin_datatype, origin_count);
+            if (OMPI_SUCCESS == rc) {
+                OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, target_datatype, target_count);
+            }
+        }
+        OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME);
+    }
+
+    if (MPI_PROC_NULL == target_rank) {
+        *request = &ompi_request_empty;
+        return MPI_SUCCESS;
+    }
+
+    rc = win->w_osc_module->osc_rget(origin_addr, origin_count, origin_datatype,
+                                     target_rank, target_disp, target_count,
+                                     target_datatype, win, request);
+    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/rget_accumulate.c b/ompi/mpi/c/rget_accumulate.c
deleted file mode 100644
index 8efc0c413b1..00000000000
--- a/ompi/mpi/c/rget_accumulate.c
+++ /dev/null
@@ -1,149 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2009      Sun Microsystmes, Inc.  All rights reserved.
- * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
- * Copyright (c) 2014-2015 Los Alamos National Security, LLC. All right
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/request/request.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-#include "ompi/mca/osc/osc.h"
-#include "ompi/op/op.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/datatype/ompi_datatype_internal.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Rget_accumulate = PMPI_Rget_accumulate
-#endif
-#define MPI_Rget_accumulate PMPI_Rget_accumulate
-#endif
-
-static const char FUNC_NAME[] = "MPI_Rget_accumulate";
-
-int MPI_Rget_accumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype,
-                        void *result_addr, int result_count, MPI_Datatype result_datatype,
-                        int target_rank, MPI_Aint target_disp, int target_count,
-                        MPI_Datatype target_datatype, MPI_Op op, MPI_Win win, MPI_Request *request)
-{
-    int rc;
-    ompi_win_t *ompi_win = (ompi_win_t*) win;
-
-    MEMCHECKER(
-        memchecker_datatype(origin_datatype);
-        memchecker_datatype(target_datatype);
-        memchecker_call(&opal_memchecker_base_isdefined, (void *) origin_addr, origin_count, origin_datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        rc = OMPI_SUCCESS;
-
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        } else if (origin_count < 0 || target_count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else if (ompi_win_peer_invalid(win, target_rank) &&
-                   (MPI_PROC_NULL != target_rank)) {
-            rc = MPI_ERR_RANK;
-        } else if (MPI_OP_NULL == op) {
-            rc = MPI_ERR_OP;
-        } else if (!ompi_op_is_intrinsic(op)) {
-            rc = MPI_ERR_OP;
-        } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) {
-            rc = MPI_ERR_DISP;
-        } else {
-            /* the origin datatype is meaningless when using MPI_OP_NO_OP */
-            if (&ompi_mpi_op_no_op.op != op) {
-                OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, origin_datatype, origin_count);
-            } else {
-                rc = OMPI_SUCCESS;
-            }
-            if (OMPI_SUCCESS == rc) {
-                OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, target_datatype, target_count);
-            }
-            if (OMPI_SUCCESS == rc) {
-                /* While technically the standard probably requires that the
-                   datatypes used with MPI_REPLACE conform to all the rules
-                   for other reduction operators, we don't require such
-                   behavior, as checking for it is expensive here and we don't
-                   care in implementation.. */
-                if (op != &ompi_mpi_op_replace.op && op != &ompi_mpi_op_no_op.op) {
-                    ompi_datatype_t *op_check_dt, *origin_check_dt;
-                    char *msg;
-
-                    /* RGET_ACCUMULATE, unlike REDUCE, can use with derived
-                       datatypes with predefinied operations, with some
-                       restrictions outlined in MPI-3:11.3.4.  The derived
-                       datatype must be composed entierly from one predefined
-                       datatype (so you can do all the construction you want,
-                       but at the bottom, you can only use one datatype, say,
-                       MPI_INT).  If the datatype at the target isn't
-                       predefined, then make sure it's composed of only one
-                       datatype, and check that datatype against
-                       ompi_op_is_valid(). */
-                    origin_check_dt = ompi_datatype_get_single_predefined_type_from_args(origin_datatype);
-                    op_check_dt = ompi_datatype_get_single_predefined_type_from_args(target_datatype);
-
-                    if( !((origin_check_dt == op_check_dt) & (NULL != op_check_dt)) ) {
-                        OMPI_ERRHANDLER_RETURN(MPI_ERR_ARG, win, MPI_ERR_ARG, FUNC_NAME);
-                    }
-
-                    /* check to make sure primitive type is valid for
-                       reduction.  Should do this on the target, but
-                       then can't get the errcode back for this
-                       call */
-                    if (!ompi_op_is_valid(op, op_check_dt, &msg, FUNC_NAME)) {
-                        int ret = OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_OP, msg);
-                        free(msg);
-                        return ret;
-                    }
-                }
-            }
-        }
-        OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME);
-    }
-
-    if (MPI_PROC_NULL == target_rank) {
-        *request = &ompi_request_empty;
-        return MPI_SUCCESS;
-    }
-
-    rc = ompi_win->w_osc_module->osc_rget_accumulate(origin_addr,
-                                                    origin_count,
-                                                    origin_datatype,
-                                                    result_addr,
-                                                    result_count,
-                                                    result_datatype,
-                                                    target_rank,
-                                                    target_disp,
-                                                    target_count,
-                                                    target_datatype,
-                                                     op, win, request);
-    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/rget_accumulate.c.in b/ompi/mpi/c/rget_accumulate.c.in
new file mode 100644
index 00000000000..146a9416dbb
--- /dev/null
+++ b/ompi/mpi/c/rget_accumulate.c.in
@@ -0,0 +1,143 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2009      Sun Microsystmes, Inc.  All rights reserved.
+ * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2014-2015 Los Alamos National Security, LLC. All right
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/request/request.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+#include "ompi/mca/osc/osc.h"
+#include "ompi/op/op.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/datatype/ompi_datatype_internal.h"
+#include "ompi/memchecker.h"
+
+
+PROTOTYPE ERROR_CLASS Rget_accumulate(BUFFER origin_addr, COUNT origin_count, DATATYPE origin_datatype,
+                        BUFFER_OUT result_addr, COUNT result_count, DATATYPE result_datatype,
+                        INT target_rank, AINT target_disp, COUNT target_count,
+                        DATATYPE target_datatype, OP op, WIN win, REQUEST_INOUT request)
+{
+    int rc;
+    ompi_win_t *ompi_win = (ompi_win_t*) win;
+
+    MEMCHECKER(
+        memchecker_datatype(origin_datatype);
+        memchecker_datatype(target_datatype);
+        memchecker_call(&opal_memchecker_base_isdefined, (void *) origin_addr, origin_count, origin_datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        rc = OMPI_SUCCESS;
+
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        } else if (origin_count < 0 || target_count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else if (ompi_win_peer_invalid(win, target_rank) &&
+                   (MPI_PROC_NULL != target_rank)) {
+            rc = MPI_ERR_RANK;
+        } else if (MPI_OP_NULL == op) {
+            rc = MPI_ERR_OP;
+        } else if (!ompi_op_is_intrinsic(op)) {
+            rc = MPI_ERR_OP;
+        } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) {
+            rc = MPI_ERR_DISP;
+        } else {
+            /* the origin datatype is meaningless when using MPI_OP_NO_OP */
+            if (&ompi_mpi_op_no_op.op != op) {
+                OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, origin_datatype, origin_count);
+            } else {
+                rc = OMPI_SUCCESS;
+            }
+            if (OMPI_SUCCESS == rc) {
+                OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, target_datatype, target_count);
+            }
+            if (OMPI_SUCCESS == rc) {
+                /* While technically the standard probably requires that the
+                   datatypes used with MPI_REPLACE conform to all the rules
+                   for other reduction operators, we don't require such
+                   behavior, as checking for it is expensive here and we don't
+                   care in implementation.. */
+                if (op != &ompi_mpi_op_replace.op && op != &ompi_mpi_op_no_op.op) {
+                    ompi_datatype_t *op_check_dt, *origin_check_dt;
+                    char *msg;
+
+                    /* RGET_ACCUMULATE, unlike REDUCE, can use with derived
+                       datatypes with predefinied operations, with some
+                       restrictions outlined in MPI-3:11.3.4.  The derived
+                       datatype must be composed entirely from one predefined
+                       datatype (so you can do all the construction you want,
+                       but at the bottom, you can only use one datatype, say,
+                       MPI_INT).  If the datatype at the target isn't
+                       predefined, then make sure it's composed of only one
+                       datatype, and check that datatype against
+                       ompi_op_is_valid(). */
+                    origin_check_dt = ompi_datatype_get_single_predefined_type_from_args(origin_datatype);
+                    op_check_dt = ompi_datatype_get_single_predefined_type_from_args(target_datatype);
+
+                    if( !((origin_check_dt == op_check_dt) & (NULL != op_check_dt)) ) {
+                        OMPI_ERRHANDLER_RETURN(MPI_ERR_ARG, win, MPI_ERR_ARG, FUNC_NAME);
+                    }
+
+                    /* check to make sure primitive type is valid for
+                       reduction.  Should do this on the target, but
+                       then can't get the errcode back for this
+                       call */
+                    if (!ompi_op_is_valid(op, op_check_dt, &msg, FUNC_NAME)) {
+                        int ret = OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_OP, msg);
+                        free(msg);
+                        return ret;
+                    }
+                }
+            }
+        }
+        OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME);
+    }
+
+    if (MPI_PROC_NULL == target_rank) {
+        *request = &ompi_request_empty;
+        return MPI_SUCCESS;
+    }
+
+    rc = ompi_win->w_osc_module->osc_rget_accumulate(origin_addr,
+                                                    origin_count,
+                                                    origin_datatype,
+                                                    result_addr,
+                                                    result_count,
+                                                    result_datatype,
+                                                    target_rank,
+                                                    target_disp,
+                                                    target_count,
+                                                    target_datatype,
+                                                     op, win, request);
+    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/rput.c b/ompi/mpi/c/rput.c
deleted file mode 100644
index 8493ac14aba..00000000000
--- a/ompi/mpi/c/rput.c
+++ /dev/null
@@ -1,89 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2014-2015 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-#include "ompi/mca/osc/osc.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Rput = PMPI_Rput
-#endif
-#define MPI_Rput PMPI_Rput
-#endif
-
-static const char FUNC_NAME[] = "MPI_Rput";
-
-
-int MPI_Rput(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype,
-            int target_rank, MPI_Aint target_disp, int target_count,
-             MPI_Datatype target_datatype, MPI_Win win, MPI_Request *request)
-{
-    int rc;
-
-    SPC_RECORD(OMPI_SPC_RPUT, 1);
-
-    if (MPI_PARAM_CHECK) {
-        rc = OMPI_SUCCESS;
-
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        } else if (origin_count < 0 || target_count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else if (ompi_win_peer_invalid(win, target_rank) &&
-                   (MPI_PROC_NULL != target_rank)) {
-            rc = MPI_ERR_RANK;
-        } else if (NULL == target_datatype ||
-                   MPI_DATATYPE_NULL == target_datatype) {
-            rc = MPI_ERR_TYPE;
-        } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) {
-            rc = MPI_ERR_DISP;
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, origin_datatype, origin_count);
-            if (OMPI_SUCCESS == rc) {
-                OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, target_datatype, target_count);
-            }
-        }
-        OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME);
-    }
-
-    if (MPI_PROC_NULL == target_rank) {
-        *request = &ompi_request_empty;
-        return MPI_SUCCESS;
-    }
-
-    rc = win->w_osc_module->osc_rput(origin_addr, origin_count, origin_datatype,
-                                     target_rank, target_disp, target_count,
-                                     target_datatype, win, request);
-    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/rput.c.in b/ompi/mpi/c/rput.c.in
new file mode 100644
index 00000000000..8ee8b5adabb
--- /dev/null
+++ b/ompi/mpi/c/rput.c.in
@@ -0,0 +1,81 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2014-2015 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+#include "ompi/mca/osc/osc.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS rput(BUFFER origin_addr, COUNT origin_count, DATATYPE origin_datatype,
+                           INT target_rank, AINT target_disp, COUNT target_count,
+                           DATATYPE target_datatype, WIN win, REQUEST_INOUT request)
+{
+    int rc;
+
+    SPC_RECORD(OMPI_SPC_RPUT, 1);
+
+    if (MPI_PARAM_CHECK) {
+        rc = OMPI_SUCCESS;
+
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        } else if (origin_count < 0 || target_count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else if (ompi_win_peer_invalid(win, target_rank) &&
+                   (MPI_PROC_NULL != target_rank)) {
+            rc = MPI_ERR_RANK;
+        } else if (NULL == target_datatype ||
+                   MPI_DATATYPE_NULL == target_datatype) {
+            rc = MPI_ERR_TYPE;
+        } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) {
+            rc = MPI_ERR_DISP;
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, origin_datatype, origin_count);
+            if (OMPI_SUCCESS == rc) {
+                OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, target_datatype, target_count);
+            }
+        }
+        OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME);
+    }
+
+    if (MPI_PROC_NULL == target_rank) {
+        *request = &ompi_request_empty;
+        return MPI_SUCCESS;
+    }
+
+    rc = win->w_osc_module->osc_rput(origin_addr, origin_count, origin_datatype,
+                                     target_rank, target_disp, target_count,
+                                     target_datatype, win, request);
+    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/rsend.c b/ompi/mpi/c/rsend.c
deleted file mode 100644
index cce1334a04b..00000000000
--- a/ompi/mpi/c/rsend.c
+++ /dev/null
@@ -1,98 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Rsend = PMPI_Rsend
-#endif
-#define MPI_Rsend PMPI_Rsend
-#endif
-
-static const char FUNC_NAME[] = "MPI_Rsend";
-
-
-int MPI_Rsend(const void *buf, int count, MPI_Datatype type, int dest, int tag, MPI_Comm comm)
-{
-    int rc = MPI_SUCCESS;
-
-    SPC_RECORD(OMPI_SPC_RSEND, 1);
-
-    MEMCHECKER(
-        memchecker_datatype(type);
-        memchecker_call(&opal_memchecker_base_isdefined, buf, count, type);
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else if (MPI_DATATYPE_NULL == type || NULL == type) {
-            rc = MPI_ERR_TYPE;
-        } else if (tag < 0 || tag > mca_pml.pml_max_tag) {
-            rc = MPI_ERR_TAG;
-        } else if (ompi_comm_peer_invalid(comm, dest) &&
-                   (MPI_PROC_NULL != dest)) {
-            rc = MPI_ERR_RANK;
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_SEND(rc, type, count);
-            OMPI_CHECK_USER_BUFFER(rc, buf, type, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are communicating with
-     * a failed process. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the completion
-     * call in the PML.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_p2p_check_proc(comm, dest, &rc)) ) {
-        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-    }
-#endif
-
-    if (MPI_PROC_NULL == dest) {
-        return MPI_SUCCESS;
-    }
-
-    rc = MCA_PML_CALL(send(buf, count, type, dest, tag,
-                           MCA_PML_BASE_SEND_READY, comm));
-    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/rsend.c.in b/ompi/mpi/c/rsend.c.in
new file mode 100644
index 00000000000..2364e8178b5
--- /dev/null
+++ b/ompi/mpi/c/rsend.c.in
@@ -0,0 +1,90 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS rsend(BUFFER buf, COUNT count, DATATYPE type, INT dest, INT tag, COMM comm)
+{
+    int rc = MPI_SUCCESS;
+
+    SPC_RECORD(OMPI_SPC_RSEND, 1);
+
+    MEMCHECKER(
+        memchecker_datatype(type);
+        memchecker_call(&opal_memchecker_base_isdefined, buf, count, type);
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else if (MPI_DATATYPE_NULL == type || NULL == type) {
+            rc = MPI_ERR_TYPE;
+        } else if (tag < 0 || tag > mca_pml.pml_max_tag) {
+            rc = MPI_ERR_TAG;
+        } else if (ompi_comm_peer_invalid(comm, dest) &&
+                   (MPI_PROC_NULL != dest)) {
+            rc = MPI_ERR_RANK;
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_SEND(rc, type, count);
+            OMPI_CHECK_USER_BUFFER(rc, buf, type, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are communicating with
+     * a failed process. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the completion
+     * call in the PML.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_p2p_check_proc(comm, dest, &rc)) ) {
+        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+    }
+#endif
+
+    if (MPI_PROC_NULL == dest) {
+        return MPI_SUCCESS;
+    }
+
+    rc = MCA_PML_CALL(send(buf, count, type, dest, tag,
+                           MCA_PML_BASE_SEND_READY, comm));
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/rsend_init.c b/ompi/mpi/c/rsend_init.c
deleted file mode 100644
index d9fc9f4d1ee..00000000000
--- a/ompi/mpi/c/rsend_init.c
+++ /dev/null
@@ -1,98 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/request/request.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Rsend_init = PMPI_Rsend_init
-#endif
-#define MPI_Rsend_init PMPI_Rsend_init
-#endif
-
-static const char FUNC_NAME[] = "MPI_Rsend_init";
-
-
-int MPI_Rsend_init(const void *buf, int count, MPI_Datatype type,
-                   int dest, int tag, MPI_Comm comm,
-                   MPI_Request *request)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(type);
-        memchecker_call(&opal_memchecker_base_isaddressable, buf, count, type);
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else if (MPI_DATATYPE_NULL == type || NULL == type) {
-            rc = MPI_ERR_TYPE;
-        } else if (tag < 0 || tag > mca_pml.pml_max_tag) {
-            rc = MPI_ERR_TAG;
-        } else if (ompi_comm_peer_invalid(comm, dest) &&
-                   (MPI_PROC_NULL != dest)) {
-            rc = MPI_ERR_RANK;
-        } else if (request == NULL) {
-            rc = MPI_ERR_REQUEST;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-    }
-
-    if (MPI_PROC_NULL == dest) {
-        rc = ompi_request_persistent_noop_create(request);
-        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * The request will be checked for process failure errors during the
-     * completion calls. So no need to check here.
-     */
-#endif
-
-    /*
-     * Here, we just initialize the request -- memchecker should set the buffer in MPI_Start.
-     */
-    rc =  MCA_PML_CALL(isend_init(buf,count,type,dest,tag,
-                                  MCA_PML_BASE_SEND_READY,comm,request));
-    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/rsend_init.c.in b/ompi/mpi/c/rsend_init.c.in
new file mode 100644
index 00000000000..a6a90588c62
--- /dev/null
+++ b/ompi/mpi/c/rsend_init.c.in
@@ -0,0 +1,90 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/request/request.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS rsend_init(BUFFER buf, COUNT count, DATATYPE type,
+                                 INT dest, INT tag, COMM comm,
+                                 REQUEST_INOUT request)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(type);
+        memchecker_call(&opal_memchecker_base_isaddressable, buf, count, type);
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else if (MPI_DATATYPE_NULL == type || NULL == type) {
+            rc = MPI_ERR_TYPE;
+        } else if (tag < 0 || tag > mca_pml.pml_max_tag) {
+            rc = MPI_ERR_TAG;
+        } else if (ompi_comm_peer_invalid(comm, dest) &&
+                   (MPI_PROC_NULL != dest)) {
+            rc = MPI_ERR_RANK;
+        } else if (request == NULL) {
+            rc = MPI_ERR_REQUEST;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+    if (MPI_PROC_NULL == dest) {
+        rc = ompi_request_persistent_noop_create(request);
+        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * The request will be checked for process failure errors during the
+     * completion calls. So no need to check here.
+     */
+#endif
+
+    /*
+     * Here, we just initialize the request -- memchecker should set the buffer in MPI_Start.
+     */
+    rc =  MCA_PML_CALL(isend_init(buf,count,type,dest,tag,
+                                  MCA_PML_BASE_SEND_READY,comm,request));
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/scan.c b/ompi/mpi/c/scan.c
deleted file mode 100644
index 3d6aef421b8..00000000000
--- a/ompi/mpi/c/scan.c
+++ /dev/null
@@ -1,123 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/op/op.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Scan = PMPI_Scan
-#endif
-#define MPI_Scan PMPI_Scan
-#endif
-
-static const char FUNC_NAME[] = "MPI_Scan";
-
-
-int MPI_Scan(const void *sendbuf, void *recvbuf, int count,
-             MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
-{
-    int err;
-
-    SPC_RECORD(OMPI_SPC_SCAN, 1);
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_comm(comm);
-        if (MPI_IN_PLACE != sendbuf) {
-            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
-        } else {
-            memchecker_call(&opal_memchecker_base_isdefined, recvbuf, count, datatype);
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        char *msg;
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-
-        /* No intercommunicators allowed! (MPI does not define
-           MPI_SCAN on intercommunicators) */
-
-        else if (OMPI_COMM_IS_INTER(comm)) {
-          err = MPI_ERR_COMM;
-        }
-
-        /* Unrooted operation; checks for all ranks */
-
-        else if (MPI_OP_NULL == op || NULL == op) {
-          err = MPI_ERR_OP;
-        } else if (MPI_IN_PLACE == recvbuf) {
-          err = MPI_ERR_ARG;
-        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
-            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
-            free(msg);
-            return ret;
-        } else {
-          OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
-        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-    }
-#endif
-
-    /* Do we need to do anything? (MPI says that reductions have to
-       have a count of at least 1, but at least IMB calls reduce with
-       a count of 0 -- blah!) */
-
-    if (0 == count) {
-        return MPI_SUCCESS;
-    }
-
-    /* Call the coll component to actually perform the allgather */
-
-    OBJ_RETAIN(op);
-    err = comm->c_coll->coll_scan(sendbuf, recvbuf, count,
-                                 datatype, op, comm,
-                                 comm->c_coll->coll_scan_module);
-    OBJ_RELEASE(op);
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/scan.c.in b/ompi/mpi/c/scan.c.in
new file mode 100644
index 00000000000..4f028c7c709
--- /dev/null
+++ b/ompi/mpi/c/scan.c.in
@@ -0,0 +1,115 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/op/op.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS scan(BUFFER sendbuf, BUFFER_OUT recvbuf, COUNT count,
+                           DATATYPE datatype, OP op, COMM comm)
+{
+    int err;
+
+    SPC_RECORD(OMPI_SPC_SCAN, 1);
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_comm(comm);
+        if (MPI_IN_PLACE != sendbuf) {
+            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
+        } else {
+            memchecker_call(&opal_memchecker_base_isdefined, recvbuf, count, datatype);
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        char *msg;
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+
+        /* No intercommunicators allowed! (MPI does not define
+           MPI_SCAN on intercommunicators) */
+
+        else if (OMPI_COMM_IS_INTER(comm)) {
+          err = MPI_ERR_COMM;
+        }
+
+        /* Unrooted operation; checks for all ranks */
+
+        else if (MPI_OP_NULL == op || NULL == op) {
+          err = MPI_ERR_OP;
+        } else if (MPI_IN_PLACE == recvbuf) {
+          err = MPI_ERR_ARG;
+        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
+            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
+            free(msg);
+            return ret;
+        } else {
+          OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
+        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+    }
+#endif
+
+    /* Do we need to do anything? (MPI says that reductions have to
+       have a count of at least 1, but at least IMB calls reduce with
+       a count of 0 -- blah!) */
+
+    if (0 == count) {
+        return MPI_SUCCESS;
+    }
+
+    /* Call the coll component to actually perform the allgather */
+
+    OBJ_RETAIN(op);
+    err = comm->c_coll->coll_scan(sendbuf, recvbuf, count,
+                                 datatype, op, comm,
+                                 comm->c_coll->coll_scan_module);
+    OBJ_RELEASE(op);
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/scan_init.c.in b/ompi/mpi/c/scan_init.c.in
new file mode 100644
index 00000000000..b1625f138a6
--- /dev/null
+++ b/ompi/mpi/c/scan_init.c.in
@@ -0,0 +1,99 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/op/op.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS scan_init(BUFFER sendbuf, BUFFER_OUT recvbuf, COUNT count,
+                                DATATYPE datatype, OP op, COMM comm,
+                                INFO info, REQUEST_INOUT request)
+{
+    int err;
+
+    SPC_RECORD(OMPI_SPC_SCAN_INIT, 1);
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_comm(comm);
+        if (MPI_IN_PLACE != sendbuf) {
+            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
+        } else {
+            memchecker_call(&opal_memchecker_base_isdefined, recvbuf, count, datatype);
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        char *msg;
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+
+        /* No intercommunicators allowed! (MPI does not define
+           MPI_SCAN on intercommunicators) */
+
+        else if (OMPI_COMM_IS_INTER(comm)) {
+          err = MPI_ERR_COMM;
+        }
+
+        /* Unrooted operation; checks for all ranks */
+
+        else if (MPI_OP_NULL == op || NULL == op) {
+          err = MPI_ERR_OP;
+        } else if (MPI_IN_PLACE == recvbuf) {
+          err = MPI_ERR_ARG;
+        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
+            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
+            free(msg);
+            return ret;
+        } else {
+          OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
+        }
+        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+    }
+
+    /* Call the coll component to actually perform the allgather */
+
+    err = comm->c_coll->coll_scan_init(sendbuf, recvbuf, count,
+                                       datatype, op, comm,
+                                       info, request,
+                                       comm->c_coll->coll_scan_init_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        ompi_coll_base_retain_op(*request, op, datatype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/scatter.c b/ompi/mpi/c/scatter.c
deleted file mode 100644
index 63b6cf6abfb..00000000000
--- a/ompi/mpi/c/scatter.c
+++ /dev/null
@@ -1,181 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2008      University of Houston.  All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Scatter = PMPI_Scatter
-#endif
-#define MPI_Scatter PMPI_Scatter
-#endif
-
-static const char FUNC_NAME[] = "MPI_Scatter";
-
-
-int MPI_Scatter(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                void *recvbuf, int recvcount, MPI_Datatype recvtype,
-                int root, MPI_Comm comm)
-{
-    int err;
-
-    SPC_RECORD(OMPI_SPC_SCATTER, 1);
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-        if(OMPI_COMM_IS_INTRA(comm)) {
-            if(ompi_comm_rank(comm) == root) {
-                memchecker_datatype(sendtype);
-                /* check whether root's send buffer is defined. */
-                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-                if(MPI_IN_PLACE != recvbuf) {
-                    memchecker_datatype(recvtype);
-                    /* check whether receive buffer is addressable. */
-                    memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-                }
-            } else {
-                memchecker_datatype(recvtype);
-                /* check whether receive buffer is addressable. */
-                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-            }
-        } else {
-            if(MPI_ROOT == root) {
-                memchecker_datatype(sendtype);
-                /* check whether root's send buffer is defined. */
-                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-            } else if (MPI_PROC_NULL != root) {
-                memchecker_datatype(recvtype);
-                /* check whether receive buffer is addressable. */
-                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-            }
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if ((ompi_comm_rank(comm) != root && MPI_IN_PLACE == recvbuf) ||
-                   (ompi_comm_rank(comm) == root && MPI_IN_PLACE == sendbuf)) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-
-        /* Errors for intracommunicators */
-
-        if (OMPI_COMM_IS_INTRA(comm)) {
-
-          /* Errors for all ranks */
-
-          if ((root >= ompi_comm_size(comm)) || (root < 0)) {
-              err = MPI_ERR_ROOT;
-          } else if (MPI_IN_PLACE != recvbuf) {
-              if (recvcount < 0) {
-                  err = MPI_ERR_COUNT;
-              } else if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
-                  err = MPI_ERR_TYPE;
-              }
-          }
-
-          /* Errors for the root.  Some of these could have been
-             combined into compound if statements above, but since
-             this whole section can be compiled out (or turned off at
-             run time) for efficiency, it's more clear to separate
-             them out into individual tests. */
-
-          else if (ompi_comm_rank(comm) == root) {
-              OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-          }
-          OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-
-        /* Errors for intercommunicators */
-
-        else {
-          if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
-                 MPI_ROOT == root || MPI_PROC_NULL == root)) {
-              err = MPI_ERR_ROOT;
-          }
-
-          /* Errors for the receivers */
-
-          else if (MPI_ROOT != root && MPI_PROC_NULL != root) {
-            if (recvcount < 0) {
-                err = MPI_ERR_COUNT;
-            } else if (MPI_DATATYPE_NULL == recvtype) {
-                err = MPI_ERR_TYPE;
-            }
-          }
-
-          /* Errors for the root.  Ditto on the comment above -- these
-             error checks could have been combined above, but let's
-             make the code easier to read. */
-
-          else if (MPI_ROOT == root) {
-              OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-          }
-          OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
-        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-    }
-#endif
-
-    /* Do we need to do anything? */
-
-    if ((0 == recvcount && MPI_ROOT != root &&
-         (ompi_comm_rank(comm) != root ||
-          (ompi_comm_rank(comm) == root && MPI_IN_PLACE != recvbuf))) ||
-        (ompi_comm_rank(comm) == root && MPI_IN_PLACE == recvbuf &&
-         0 == sendcount) ||
-        (0 == sendcount && (MPI_ROOT == root || MPI_PROC_NULL == root))) {
-        return MPI_SUCCESS;
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_scatter(sendbuf, sendcount, sendtype, recvbuf,
-                                    recvcount, recvtype, root, comm,
-                                    comm->c_coll->coll_scatter_module);
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/scatter.c.in b/ompi/mpi/c/scatter.c.in
new file mode 100644
index 00000000000..e07ffde7d9f
--- /dev/null
+++ b/ompi/mpi/c/scatter.c.in
@@ -0,0 +1,183 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2023 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2008      University of Houston.  All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS scatter(BUFFER sendbuf, COUNT sendcount, DATATYPE sendtype,
+                              BUFFER_OUT recvbuf, COUNT recvcount, DATATYPE recvtype,
+                              INT root, COMM comm)
+{
+    int err;
+
+    SPC_RECORD(OMPI_SPC_SCATTER, 1);
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+        if(OMPI_COMM_IS_INTRA(comm)) {
+            if(ompi_comm_rank(comm) == root) {
+                memchecker_datatype(sendtype);
+                /* check whether root's send buffer is defined. */
+                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+                if(MPI_IN_PLACE != recvbuf) {
+                    memchecker_datatype(recvtype);
+                    /* check whether receive buffer is addressable. */
+                    memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+                }
+            } else {
+                memchecker_datatype(recvtype);
+                /* check whether receive buffer is addressable. */
+                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+            }
+        } else {
+            if(MPI_ROOT == root) {
+                memchecker_datatype(sendtype);
+                /* check whether root's send buffer is defined. */
+                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+            } else if (MPI_PROC_NULL != root) {
+                memchecker_datatype(recvtype);
+                /* check whether receive buffer is addressable. */
+                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+            }
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if ((ompi_comm_rank(comm) != root && MPI_IN_PLACE == recvbuf) ||
+                   (ompi_comm_rank(comm) == root && MPI_IN_PLACE == sendbuf)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+
+        /* Errors for intracommunicators */
+
+        if (OMPI_COMM_IS_INTRA(comm)) {
+
+          /* Errors for all ranks */
+
+          if ((root >= ompi_comm_size(comm)) || (root < 0)) {
+              err = MPI_ERR_ROOT;
+          } else if (MPI_IN_PLACE != recvbuf) {
+              if (recvcount < 0) {
+                  err = MPI_ERR_COUNT;
+              } else if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
+                  err = MPI_ERR_TYPE;
+              }
+          }
+
+          /* Errors for the root.  Some of these could have been
+             combined into compound if statements above, but since
+             this whole section can be compiled out (or turned off at
+             run time) for efficiency, it's more clear to separate
+             them out into individual tests. */
+
+          else if (ompi_comm_rank(comm) == root) {
+              OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+          }
+          OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+
+        /* Errors for intercommunicators */
+
+        else {
+          if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
+                 MPI_ROOT == root || MPI_PROC_NULL == root)) {
+              err = MPI_ERR_ROOT;
+          }
+
+          /* Errors for the receivers */
+
+          else if (MPI_ROOT != root && MPI_PROC_NULL != root) {
+            if (recvcount < 0) {
+                err = MPI_ERR_COUNT;
+            } else if (MPI_DATATYPE_NULL == recvtype) {
+                err = MPI_ERR_TYPE;
+            }
+          }
+
+          /* Errors for the root.  Ditto on the comment above -- these
+             error checks could have been combined above, but let's
+             make the code easier to read. */
+
+          else if (MPI_ROOT == root) {
+              OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+          }
+          OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
+        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+    }
+#endif
+
+    /* Do we need to do anything? */
+
+    if ((0 == recvcount && MPI_ROOT != root &&
+         (ompi_comm_rank(comm) != root ||
+          (ompi_comm_rank(comm) == root && MPI_IN_PLACE != recvbuf))) ||
+        (ompi_comm_rank(comm) == root && MPI_IN_PLACE == recvbuf &&
+         0 == sendcount) ||
+        (0 == sendcount && (MPI_ROOT == root || MPI_PROC_NULL == root))) {
+        return MPI_SUCCESS;
+    }
+
+    const void *updated_sendbuf;
+    void *updated_recvbuf;
+    if (OMPI_COMM_IS_INTRA(comm)) {
+        updated_sendbuf = (ompi_comm_rank(comm) != root) ? NULL : sendbuf;
+        updated_recvbuf = recvbuf;
+    } else {
+        updated_sendbuf = (MPI_ROOT != root ) ? NULL : sendbuf;
+        updated_recvbuf = ((MPI_ROOT == root) || (MPI_PROC_NULL == root)) ? NULL : recvbuf;
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    err = comm->c_coll->coll_scatter(updated_sendbuf, sendcount, sendtype, updated_recvbuf,
+                                     recvcount, recvtype, root, comm,
+                                     comm->c_coll->coll_scatter_module);
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/scatter_init.c.in b/ompi/mpi/c/scatter_init.c.in
new file mode 100644
index 00000000000..c17ef222d77
--- /dev/null
+++ b/ompi/mpi/c/scatter_init.c.in
@@ -0,0 +1,180 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2023 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2008      University of Houston.  All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS scatter_init(BUFFER sendbuf, COUNT sendcount, DATATYPE sendtype,
+                                   BUFFER_OUT recvbuf, COUNT recvcount, DATATYPE recvtype,
+                                   INT root, COMM comm, INFO info, REQUEST_INOUT request)
+{
+    int err;
+
+    SPC_RECORD(OMPI_SPC_SCATTER_INIT, 1);
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+        if(OMPI_COMM_IS_INTRA(comm)) {
+            if(ompi_comm_rank(comm) == root) {
+                memchecker_datatype(sendtype);
+                /* check whether root's send buffer is defined. */
+                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+                if(MPI_IN_PLACE != recvbuf) {
+                    memchecker_datatype(recvtype);
+                    /* check whether receive buffer is addressable. */
+                    memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+                }
+            } else {
+                memchecker_datatype(recvtype);
+                /* check whether receive buffer is addressable. */
+                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+            }
+        } else {
+            if(MPI_ROOT == root) {
+                memchecker_datatype(sendtype);
+                /* check whether root's send buffer is defined. */
+                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+            } else if (MPI_PROC_NULL != root) {
+                memchecker_datatype(recvtype);
+                /* check whether receive buffer is addressable. */
+                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+            }
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if ((ompi_comm_rank(comm) != root && MPI_IN_PLACE == recvbuf) ||
+                   (ompi_comm_rank(comm) == root && MPI_IN_PLACE == sendbuf)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+
+        /* Errors for intracommunicators */
+
+        if (OMPI_COMM_IS_INTRA(comm)) {
+
+          /* Errors for all ranks */
+
+          if ((root >= ompi_comm_size(comm)) || (root < 0)) {
+              err = MPI_ERR_ROOT;
+          } else if (MPI_IN_PLACE != recvbuf) {
+              if (recvcount < 0) {
+                  err = MPI_ERR_COUNT;
+              } else if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
+                  err = MPI_ERR_TYPE;
+              }
+          }
+
+          /* Errors for the root.  Some of these could have been
+             combined into compound if statements above, but since
+             this whole section can be compiled out (or turned off at
+             run time) for efficiency, it's more clear to separate
+             them out into individual tests. */
+
+          else if (ompi_comm_rank(comm) == root) {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+          }
+          OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+
+        /* Errors for intercommunicators */
+
+        else {
+          if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
+                 MPI_ROOT == root || MPI_PROC_NULL == root)) {
+            err = MPI_ERR_ROOT;
+          }
+
+          /* Errors for the receivers */
+
+          else if (MPI_ROOT != root && MPI_PROC_NULL != root) {
+            if (recvcount < 0) {
+              err = MPI_ERR_COUNT;
+            } else if (MPI_DATATYPE_NULL == recvtype) {
+              err = MPI_ERR_TYPE;
+            }
+          }
+
+          /* Errors for the root.  Ditto on the comment above -- these
+             error checks could have been combined above, but let's
+             make the code easier to read. */
+
+          else if (MPI_ROOT == root) {
+            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
+          }
+          OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+        }
+    }
+
+    const void *updated_sendbuf;
+    void *updated_recvbuf;
+    if (OMPI_COMM_IS_INTRA(comm)) {
+        updated_sendbuf = (ompi_comm_rank(comm) != root) ? NULL : sendbuf;
+        updated_recvbuf = recvbuf;
+    } else {
+        updated_sendbuf = (MPI_ROOT != root ) ? NULL : sendbuf;
+        updated_recvbuf = ((MPI_ROOT == root) || (MPI_PROC_NULL == root)) ? NULL : recvbuf;
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    err = comm->c_coll->coll_scatter_init(updated_sendbuf, sendcount, sendtype, updated_recvbuf,
+                                          recvcount, recvtype, root, comm, info, request,
+                                          comm->c_coll->coll_scatter_init_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        if (OMPI_COMM_IS_INTRA(comm)) {
+            if (MPI_IN_PLACE == recvbuf) {
+                recvtype = NULL;
+            } else if (ompi_comm_rank(comm) != root) {
+                sendtype = NULL;
+            }
+        } else {
+            if (MPI_ROOT == root) {
+                recvtype = NULL;
+            } else if (MPI_PROC_NULL == root) {
+                sendtype = NULL;
+                recvtype = NULL;
+            } else {
+                sendtype = NULL;
+            }
+        }
+        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/scatterv.c b/ompi/mpi/c/scatterv.c
deleted file mode 100644
index 6600f92b2c6..00000000000
--- a/ompi/mpi/c/scatterv.c
+++ /dev/null
@@ -1,210 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2012      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Scatterv = PMPI_Scatterv
-#endif
-#define MPI_Scatterv PMPI_Scatterv
-#endif
-
-static const char FUNC_NAME[] = "MPI_Scatterv";
-
-
-int MPI_Scatterv(const void *sendbuf, const int sendcounts[], const int displs[],
-                 MPI_Datatype sendtype, void *recvbuf, int recvcount,
-                 MPI_Datatype recvtype, int root, MPI_Comm comm)
-{
-    int i, size, err;
-
-    SPC_RECORD(OMPI_SPC_SCATTERV, 1);
-
-    MEMCHECKER(
-        ptrdiff_t ext;
-
-        size = ompi_comm_remote_size(comm);
-        ompi_datatype_type_extent(recvtype, &ext);
-
-        memchecker_comm(comm);
-        if(OMPI_COMM_IS_INTRA(comm)) {
-              if(ompi_comm_rank(comm) == root) {
-                memchecker_datatype(sendtype);
-                /* check whether root's send buffer is defined. */
-                for (i = 0; i < size; i++) {
-                    memchecker_call(&opal_memchecker_base_isdefined,
-                                    (char *)(sendbuf)+displs[i]*ext,
-                                    sendcounts[i], sendtype);
-                }
-                if(MPI_IN_PLACE != recvbuf) {
-                    memchecker_datatype(recvtype);
-                    /* check whether receive buffer is addressable. */
-                    memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-                }
-              } else {
-                  memchecker_datatype(recvtype);
-                  /* check whether receive buffer is addressable. */
-                  memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-              }
-        } else {
-            if(MPI_ROOT == root) {
-                  memchecker_datatype(sendtype);
-                  /* check whether root's send buffer is defined. */
-                  for (i = 0; i < size; i++) {
-                      memchecker_call(&opal_memchecker_base_isdefined,
-                                      (char *)(sendbuf)+displs[i]*ext,
-                                      sendcounts[i], sendtype);
-                  }
-            } else if (MPI_PROC_NULL != root) {
-                /* check whether receive buffer is addressable. */
-                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-            }
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if ((ompi_comm_rank(comm) != root && MPI_IN_PLACE == recvbuf) ||
-                   (ompi_comm_rank(comm) == root && MPI_IN_PLACE == sendbuf)) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-
-        /* Errors for intracommunicators */
-
-        if (OMPI_COMM_IS_INTRA(comm)) {
-
-            /* Errors for all ranks */
-
-            if ((root >= ompi_comm_size(comm)) || (root < 0)) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
-            }
-
-            if (MPI_IN_PLACE != recvbuf) {
-                if (recvcount < 0) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT,
-                                                  FUNC_NAME);
-                }
-
-                if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE,
-                                                  FUNC_NAME);
-                }
-            }
-
-            /* Errors for the root.  Some of these could have been
-               combined into compound if statements above, but since
-               this whole section can be compiled out (or turned off at
-               run time) for efficiency, it's more clear to separate
-               them out into individual tests. */
-
-            if (ompi_comm_rank(comm) == root) {
-                if (NULL == displs) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-                }
-
-                if (NULL == sendcounts) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-                }
-
-                size = ompi_comm_size(comm);
-                for (i = 0; i < size; ++i) {
-                    OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcounts[i]);
-                    OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-                }
-            }
-        }
-
-        /* Errors for intercommunicators */
-
-        else {
-            if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
-                   MPI_ROOT == root || MPI_PROC_NULL == root)) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
-            }
-
-          /* Errors for the receivers */
-
-            if (MPI_ROOT != root && MPI_PROC_NULL != root) {
-                if (recvcount < 0) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-                }
-
-                if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
-                }
-            }
-
-            /* Errors for the root.  Ditto on the comment above -- these
-               error checks could have been combined above, but let's
-               make the code easier to read. */
-
-            else if (MPI_ROOT == root) {
-                if (NULL == displs) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-                }
-
-                if (NULL == sendcounts) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-                }
-
-                size = ompi_comm_remote_size(comm);
-                for (i = 0; i < size; ++i) {
-                    OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcounts[i]);
-                    OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-                }
-            }
-        }
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are using a broken
-     * communicator. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the operation.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
-        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-    }
-#endif
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_scatterv(sendbuf, sendcounts, displs,
-                                     sendtype, recvbuf, recvcount, recvtype, root, comm,
-                                     comm->c_coll->coll_scatterv_module);
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/scatterv.c.in b/ompi/mpi/c/scatterv.c.in
new file mode 100644
index 00000000000..517f01bb503
--- /dev/null
+++ b/ompi/mpi/c/scatterv.c.in
@@ -0,0 +1,217 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2023 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2012      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS scatterv(BUFFER sendbuf, COUNT_ARRAY sendcounts, DISP_ARRAY displs,
+                               DATATYPE sendtype, BUFFER_OUT recvbuf, COUNT recvcount,
+                               DATATYPE recvtype, INT root, COMM comm)
+{
+    int i, size, err;
+    ompi_count_array_t sendcounts_desc;
+    ompi_disp_array_t displs_desc;
+
+    SPC_RECORD(OMPI_SPC_SCATTERV, 1);
+
+    MEMCHECKER(
+        ptrdiff_t ext;
+
+        size = ompi_comm_remote_size(comm);
+
+        memchecker_comm(comm);
+        if(OMPI_COMM_IS_INTRA(comm)) {
+              if(ompi_comm_rank(comm) == root) {
+                memchecker_datatype(sendtype);
+                ompi_datatype_type_extent(sendtype, &ext);
+                /* check whether root's send buffer is defined. */
+                for (i = 0; i < size; i++) {
+                    memchecker_call(&opal_memchecker_base_isdefined,
+                                    (char *)(sendbuf)+displs[i]*ext,
+                                    sendcounts[i], sendtype);
+                }
+                if(MPI_IN_PLACE != recvbuf) {
+                    memchecker_datatype(recvtype);
+                    /* check whether receive buffer is addressable. */
+                    memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+                }
+              } else {
+                  memchecker_datatype(recvtype);
+                  /* check whether receive buffer is addressable. */
+                  memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+              }
+        } else {
+            if(MPI_ROOT == root) {
+                  memchecker_datatype(sendtype);
+                  ompi_datatype_type_extent(sendtype, &ext);
+                  /* check whether root's send buffer is defined. */
+                  for (i = 0; i < size; i++) {
+                      memchecker_call(&opal_memchecker_base_isdefined,
+                                      (char *)(sendbuf)+displs[i]*ext,
+                                      sendcounts[i], sendtype);
+                  }
+            } else if (MPI_PROC_NULL != root) {
+                /* check whether receive buffer is addressable. */
+                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+            }
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if ((ompi_comm_rank(comm) != root && MPI_IN_PLACE == recvbuf) ||
+                   (ompi_comm_rank(comm) == root && MPI_IN_PLACE == sendbuf)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+
+        /* Errors for intracommunicators */
+
+        if (OMPI_COMM_IS_INTRA(comm)) {
+
+            /* Errors for all ranks */
+
+            if ((root >= ompi_comm_size(comm)) || (root < 0)) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
+            }
+
+            if (MPI_IN_PLACE != recvbuf) {
+                if (recvcount < 0) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT,
+                                                  FUNC_NAME);
+                }
+
+                if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE,
+                                                  FUNC_NAME);
+                }
+            }
+
+            /* Errors for the root.  Some of these could have been
+               combined into compound if statements above, but since
+               this whole section can be compiled out (or turned off at
+               run time) for efficiency, it's more clear to separate
+               them out into individual tests. */
+
+            if (ompi_comm_rank(comm) == root) {
+                if (NULL == displs) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+                }
+
+                if (NULL == sendcounts) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+                }
+
+                size = ompi_comm_size(comm);
+                for (i = 0; i < size; ++i) {
+                    OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcounts[i]);
+                    OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+                }
+            }
+        }
+
+        /* Errors for intercommunicators */
+
+        else {
+            if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
+                   MPI_ROOT == root || MPI_PROC_NULL == root)) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
+            }
+
+          /* Errors for the receivers */
+
+            if (MPI_ROOT != root && MPI_PROC_NULL != root) {
+                if (recvcount < 0) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+                }
+
+                if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
+                }
+            }
+
+            /* Errors for the root.  Ditto on the comment above -- these
+               error checks could have been combined above, but let's
+               make the code easier to read. */
+
+            else if (MPI_ROOT == root) {
+                if (NULL == displs) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+                }
+
+                if (NULL == sendcounts) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+                }
+
+                size = ompi_comm_remote_size(comm);
+                for (i = 0; i < size; ++i) {
+                    OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcounts[i]);
+                    OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+                }
+            }
+        }
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are using a broken
+     * communicator. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the operation.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_coll_check(comm, &err)) ) {
+        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+    }
+#endif
+
+    const void *updated_sendbuf;
+    void *updated_recvbuf;
+    if (OMPI_COMM_IS_INTRA(comm)) {
+        updated_sendbuf = (ompi_comm_rank(comm) != root) ? NULL : sendbuf;
+        updated_recvbuf = recvbuf;
+    } else {
+        updated_sendbuf = (MPI_ROOT != root ) ? NULL : sendbuf;
+        updated_recvbuf = ((MPI_ROOT == root) || (MPI_PROC_NULL == root)) ? NULL : recvbuf;
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    OMPI_COUNT_ARRAY_INIT(&sendcounts_desc, sendcounts);
+    OMPI_DISP_ARRAY_INIT(&displs_desc, displs);
+    err = comm->c_coll->coll_scatterv(updated_sendbuf, sendcounts_desc, displs_desc,
+                                      sendtype, updated_recvbuf, recvcount, recvtype, root, comm,
+                                      comm->c_coll->coll_scatterv_module);
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/scatterv_init.c.in b/ompi/mpi/c/scatterv_init.c.in
new file mode 100644
index 00000000000..18558e2b994
--- /dev/null
+++ b/ompi/mpi/c/scatterv_init.c.in
@@ -0,0 +1,225 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2023 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS scatterv_init(BUFFER sendbuf, COUNT_ARRAY sendcounts, DISP_ARRAY displs,
+                                    DATATYPE sendtype, BUFFER_OUT recvbuf, COUNT recvcount,
+                                    DATATYPE recvtype, INT root, COMM comm, INFO info, REQUEST_INOUT request)
+{
+    int i, size, err;
+    ompi_count_array_t sendcounts_desc;
+    ompi_disp_array_t displs_desc;
+
+    SPC_RECORD(OMPI_SPC_SCATTERV_INIT, 1);
+
+    MEMCHECKER(
+        ptrdiff_t ext;
+
+        size = ompi_comm_remote_size(comm);
+
+        memchecker_comm(comm);
+        if(OMPI_COMM_IS_INTRA(comm)) {
+              if(ompi_comm_rank(comm) == root) {
+                memchecker_datatype(sendtype);
+                ompi_datatype_type_extent(sendtype, &ext);
+                /* check whether root's send buffer is defined. */
+                for (i = 0; i < size; i++) {
+                    memchecker_call(&opal_memchecker_base_isdefined,
+                                    (char *)(sendbuf)+displs[i]*ext,
+                                    sendcounts[i], sendtype);
+                }
+                if(MPI_IN_PLACE != recvbuf) {
+                    memchecker_datatype(recvtype);
+                    /* check whether receive buffer is addressable. */
+                    memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+                }
+              } else {
+                  memchecker_datatype(recvtype);
+                  /* check whether receive buffer is addressable. */
+                  memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+              }
+        } else {
+            if(MPI_ROOT == root) {
+                  memchecker_datatype(sendtype);
+                  ompi_datatype_type_extent(sendtype, &ext);
+                  /* check whether root's send buffer is defined. */
+                  for (i = 0; i < size; i++) {
+                      memchecker_call(&opal_memchecker_base_isdefined,
+                                      (char *)(sendbuf)+displs[i]*ext,
+                                      sendcounts[i], sendtype);
+                  }
+            } else if (MPI_PROC_NULL != root) {
+                /* check whether receive buffer is addressable. */
+                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
+            }
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        err = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if ((ompi_comm_rank(comm) != root && MPI_IN_PLACE == recvbuf) ||
+                   (ompi_comm_rank(comm) == root && MPI_IN_PLACE == sendbuf)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+
+        /* Errors for intracommunicators */
+
+        if (OMPI_COMM_IS_INTRA(comm)) {
+
+            /* Errors for all ranks */
+
+            if ((root >= ompi_comm_size(comm)) || (root < 0)) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
+            }
+
+            if (MPI_IN_PLACE != recvbuf) {
+                if (recvcount < 0) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT,
+                                                  FUNC_NAME);
+                }
+
+                if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE,
+                                                  FUNC_NAME);
+                }
+            }
+
+            /* Errors for the root.  Some of these could have been
+               combined into compound if statements above, but since
+               this whole section can be compiled out (or turned off at
+               run time) for efficiency, it's more clear to separate
+               them out into individual tests. */
+
+            if (ompi_comm_rank(comm) == root) {
+                if (NULL == displs) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+                }
+
+                if (NULL == sendcounts) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+                }
+
+                size = ompi_comm_size(comm);
+                for (i = 0; i < size; ++i) {
+                    OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcounts[i]);
+                    OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+                }
+            }
+        }
+
+        /* Errors for intercommunicators */
+
+        else {
+            if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
+                   MPI_ROOT == root || MPI_PROC_NULL == root)) {
+                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
+            }
+
+          /* Errors for the receivers */
+
+            if (MPI_ROOT != root && MPI_PROC_NULL != root) {
+                if (recvcount < 0) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+                }
+
+                if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
+                }
+            }
+
+            /* Errors for the root.  Ditto on the comment above -- these
+               error checks could have been combined above, but let's
+               make the code easier to read. */
+
+            else if (MPI_ROOT == root) {
+                if (NULL == displs) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+                }
+
+                if (NULL == sendcounts) {
+                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+                }
+
+                size = ompi_comm_remote_size(comm);
+                for (i = 0; i < size; ++i) {
+                    OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcounts[i]);
+                    OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
+                }
+            }
+        }
+    }
+
+    const void *updated_sendbuf;
+    void *updated_recvbuf;
+    if (OMPI_COMM_IS_INTRA(comm)) {
+        updated_sendbuf = (ompi_comm_rank(comm) != root) ? NULL : sendbuf;
+        updated_recvbuf = recvbuf;
+    } else {
+        updated_sendbuf = (MPI_ROOT != root ) ? NULL : sendbuf;
+        updated_recvbuf = ((MPI_ROOT == root) || (MPI_PROC_NULL == root)) ? NULL : recvbuf;
+    }
+
+    /* Invoke the coll component to perform the back-end operation */
+    OMPI_COUNT_ARRAY_INIT(&sendcounts_desc, sendcounts);
+    OMPI_DISP_ARRAY_INIT(&displs_desc, displs);
+    err = comm->c_coll->coll_scatterv_init(updated_sendbuf, sendcounts_desc, displs_desc,
+                                           sendtype, updated_recvbuf, recvcount, recvtype, root, comm,
+                                           info, request, comm->c_coll->coll_scatterv_init_module);
+    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
+        if (OMPI_COMM_IS_INTRA(comm)) {
+            if (MPI_IN_PLACE == recvbuf) {
+                recvtype = NULL;
+            } else if (ompi_comm_rank(comm) != root) {
+                sendtype = NULL;
+            }
+        } else {
+            if (MPI_ROOT == root) {
+                recvtype = NULL;
+            } else if (MPI_PROC_NULL == root) {
+                sendtype = NULL;
+                recvtype = NULL;
+            } else {
+                sendtype = NULL;
+            }
+        }
+        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
+    }
+    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/send.c b/ompi/mpi/c/send.c
deleted file mode 100644
index b21bba2c7bc..00000000000
--- a/ompi/mpi/c/send.c
+++ /dev/null
@@ -1,95 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Send = PMPI_Send
-#endif
-#define MPI_Send PMPI_Send
-#endif
-
-static const char FUNC_NAME[] = "MPI_Send";
-
-
-int MPI_Send(const void *buf, int count, MPI_Datatype type, int dest,
-             int tag, MPI_Comm comm)
-{
-    int rc = MPI_SUCCESS;
-
-    SPC_RECORD(OMPI_SPC_SEND, 1);
-
-    MEMCHECKER(
-        memchecker_datatype(type);
-        memchecker_call(&opal_memchecker_base_isdefined, buf, count, type);
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else if (tag < 0 || tag > mca_pml.pml_max_tag) {
-            rc = MPI_ERR_TAG;
-        } else if (ompi_comm_peer_invalid(comm, dest) &&
-                   (MPI_PROC_NULL != dest)) {
-            rc = MPI_ERR_RANK;
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_SEND(rc, type, count);
-            OMPI_CHECK_USER_BUFFER(rc, buf, type, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are communicating with
-     * a failed process. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the completion
-     * call in the PML.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_p2p_check_proc(comm, dest, &rc)) ) {
-        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-    }
-#endif
-
-    if (MPI_PROC_NULL == dest) {
-        return MPI_SUCCESS;
-    }
-
-    rc = MCA_PML_CALL(send(buf, count, type, dest, tag, MCA_PML_BASE_SEND_STANDARD, comm));
-    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/send.c.in b/ompi/mpi/c/send.c.in
new file mode 100644
index 00000000000..797bc7630ad
--- /dev/null
+++ b/ompi/mpi/c/send.c.in
@@ -0,0 +1,87 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS send(BUFFER buf, COUNT count, DATATYPE type, RANK dest,
+                           TAG tag, COMM comm)
+{
+    int rc = MPI_SUCCESS;
+
+    SPC_RECORD(OMPI_SPC_SEND, 1);
+
+    MEMCHECKER(
+        memchecker_datatype(type);
+        memchecker_call(&opal_memchecker_base_isdefined, buf, count, type);
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else if (tag < 0 || tag > mca_pml.pml_max_tag) {
+            rc = MPI_ERR_TAG;
+        } else if (ompi_comm_peer_invalid(comm, dest) &&
+                   (MPI_PROC_NULL != dest)) {
+            rc = MPI_ERR_RANK;
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_SEND(rc, type, count);
+            OMPI_CHECK_USER_BUFFER(rc, buf, type, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are communicating with
+     * a failed process. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the completion
+     * call in the PML.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_p2p_check_proc(comm, dest, &rc)) ) {
+        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+    }
+#endif
+
+    if (MPI_PROC_NULL == dest) {
+        return MPI_SUCCESS;
+    }
+
+    rc = MCA_PML_CALL(send(buf, count, type, dest, tag, MCA_PML_BASE_SEND_STANDARD, comm));
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/send_init.c b/ompi/mpi/c/send_init.c
deleted file mode 100644
index 924b22948ff..00000000000
--- a/ompi/mpi/c/send_init.c
+++ /dev/null
@@ -1,97 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/request/request.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Send_init = PMPI_Send_init
-#endif
-#define MPI_Send_init PMPI_Send_init
-#endif
-
-static const char FUNC_NAME[] = "MPI_Send_init";
-
-
-int MPI_Send_init(const void *buf, int count, MPI_Datatype type,
-                  int dest, int tag, MPI_Comm comm,
-                  MPI_Request *request)
-{
-    int rc = MPI_SUCCESS;
-
-    MEMCHECKER(
-        memchecker_datatype(type);
-        memchecker_call(&opal_memchecker_base_isaddressable, buf, count, type);
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else if (tag < 0 || tag > mca_pml.pml_max_tag) {
-            rc = MPI_ERR_TAG;
-        } else if (ompi_comm_peer_invalid(comm, dest) &&
-                   (MPI_PROC_NULL != dest)) {
-            rc = MPI_ERR_RANK;
-        } else if (request == NULL) {
-            rc = MPI_ERR_REQUEST;
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_SEND(rc, type, count);
-            OMPI_CHECK_USER_BUFFER(rc, buf, type, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-    }
-
-    if (MPI_PROC_NULL == dest) {
-        rc = ompi_request_persistent_noop_create(request);
-        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * The request will be checked for process failure errors during the
-     * completion calls. So no need to check here.
-     */
-#endif
-
-    /*
-     * Here, we just initialize the request -- memchecker should set the buffer in MPI_Start.
-     */
-    rc = MCA_PML_CALL(isend_init(buf,count,type,dest,tag,MCA_PML_BASE_SEND_STANDARD,comm,request));
-    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/send_init.c.in b/ompi/mpi/c/send_init.c.in
new file mode 100644
index 00000000000..de0dd93f777
--- /dev/null
+++ b/ompi/mpi/c/send_init.c.in
@@ -0,0 +1,89 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/request/request.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS send_init(BUFFER buf, COUNT count, DATATYPE type,
+                                INT dest, INT tag, COMM comm,
+                                REQUEST_INOUT request)
+{
+    int rc = MPI_SUCCESS;
+
+    MEMCHECKER(
+        memchecker_datatype(type);
+        memchecker_call(&opal_memchecker_base_isaddressable, buf, count, type);
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else if (tag < 0 || tag > mca_pml.pml_max_tag) {
+            rc = MPI_ERR_TAG;
+        } else if (ompi_comm_peer_invalid(comm, dest) &&
+                   (MPI_PROC_NULL != dest)) {
+            rc = MPI_ERR_RANK;
+        } else if (request == NULL) {
+            rc = MPI_ERR_REQUEST;
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_SEND(rc, type, count);
+            OMPI_CHECK_USER_BUFFER(rc, buf, type, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+    if (MPI_PROC_NULL == dest) {
+        rc = ompi_request_persistent_noop_create(request);
+        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * The request will be checked for process failure errors during the
+     * completion calls. So no need to check here.
+     */
+#endif
+
+    /*
+     * Here, we just initialize the request -- memchecker should set the buffer in MPI_Start.
+     */
+    rc = MCA_PML_CALL(isend_init(buf,count,type,dest,tag,MCA_PML_BASE_SEND_STANDARD,comm,request));
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/sendrecv.c b/ompi/mpi/c/sendrecv.c
deleted file mode 100644
index bd9dee53c9f..00000000000
--- a/ompi/mpi/c/sendrecv.c
+++ /dev/null
@@ -1,129 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/request/request.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Sendrecv = PMPI_Sendrecv
-#endif
-#define MPI_Sendrecv PMPI_Sendrecv
-#endif
-
-static const char FUNC_NAME[] = "MPI_Sendrecv";
-
-
-int MPI_Sendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                 int dest, int sendtag, void *recvbuf, int recvcount,
-                 MPI_Datatype recvtype, int source, int recvtag,
-                 MPI_Comm comm,  MPI_Status *status)
-{
-    ompi_request_t* req;
-    int rc = MPI_SUCCESS;
-#if OPAL_ENABLE_FT_MPI
-    int rcs = MPI_SUCCESS;
-#endif
-
-    SPC_RECORD(OMPI_SPC_SENDRECV, 1);
-
-    MEMCHECKER(
-        memchecker_datatype(sendtype);
-        memchecker_datatype(recvtype);
-        memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        OMPI_CHECK_DATATYPE_FOR_SEND(rc, sendtype, sendcount);
-        OMPI_CHECK_DATATYPE_FOR_RECV(rc, recvtype, recvcount);
-        OMPI_CHECK_USER_BUFFER(rc, sendbuf, sendtype, sendcount);
-        OMPI_CHECK_USER_BUFFER(rc, recvbuf, recvtype, recvcount);
-
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
-        } else if (dest != MPI_PROC_NULL && ompi_comm_peer_invalid(comm, dest)) {
-            rc = MPI_ERR_RANK;
-        } else if (sendtag < 0 || sendtag > mca_pml.pml_max_tag) {
-            rc = MPI_ERR_TAG;
-        } else if (source != MPI_PROC_NULL && source != MPI_ANY_SOURCE && ompi_comm_peer_invalid(comm, source)) {
-            rc = MPI_ERR_RANK;
-        } else if (((recvtag < 0) && (recvtag !=  MPI_ANY_TAG)) || (recvtag > mca_pml.pml_max_tag)) {
-            rc = MPI_ERR_TAG;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-    }
-
-    if (source != MPI_PROC_NULL) { /* post recv */
-        rc = MCA_PML_CALL(irecv(recvbuf, recvcount, recvtype,
-                                source, recvtag, comm, &req));
-        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-    }
-
-    if (dest != MPI_PROC_NULL) { /* send */
-        rc = MCA_PML_CALL(send(sendbuf, sendcount, sendtype, dest,
-                               sendtag, MCA_PML_BASE_SEND_STANDARD, comm));
-#if OPAL_ENABLE_FT_MPI
-        /* If ULFM is enabled we need to wait for the posted receive to
-         * complete, hence we cannot return here */
-        rcs = rc;
-#else
-        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-#endif  /* OPAL_ENABLE_FT_MPI */
-    }
-
-    if (source != MPI_PROC_NULL) { /* wait for recv */
-        rc = ompi_request_wait(&req, status);
-#if OPAL_ENABLE_FT_MPI
-        /* Sendrecv never returns ERR_PROC_FAILED_PENDING because it is
-         * blocking. Lets complete now that irecv and promote the error
-         * to ERR_PROC_FAILED */
-        if( OPAL_UNLIKELY(MPI_ERR_PROC_FAILED_PENDING == rc) ) {
-            ompi_request_cancel(req);
-            ompi_request_wait(&req, MPI_STATUS_IGNORE);
-            rc = MPI_ERR_PROC_FAILED;
-        }
-#endif
-    } else {
-        if (MPI_STATUS_IGNORE != status) {
-            *status = ompi_request_empty.req_status;
-        }
-        rc = MPI_SUCCESS;
-    }
-#if OPAL_ENABLE_FT_MPI
-    if( OPAL_UNLIKELY(MPI_SUCCESS != rcs && MPI_SUCCESS == rc) ) {
-        rc = rcs;
-    }
-#endif
-
-    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/sendrecv.c.in b/ompi/mpi/c/sendrecv.c.in
new file mode 100644
index 00000000000..d6a30102a50
--- /dev/null
+++ b/ompi/mpi/c/sendrecv.c.in
@@ -0,0 +1,131 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/request/request.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS sendrecv(BUFFER sendbuf, COUNT sendcount, DATATYPE sendtype,
+                               INT dest, INT sendtag, BUFFER_OUT recvbuf, COUNT recvcount,
+                               DATATYPE recvtype, INT source, INT recvtag,
+                               COMM comm, STATUS_OUT status)
+{
+    ompi_request_t* req = MPI_REQUEST_NULL;
+    int rc = MPI_SUCCESS;
+    int rcs = MPI_SUCCESS;
+
+    SPC_RECORD(OMPI_SPC_SENDRECV, 1);
+
+    MEMCHECKER(
+        memchecker_datatype(sendtype);
+        memchecker_datatype(recvtype);
+        memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        OMPI_CHECK_DATATYPE_FOR_SEND(rc, sendtype, sendcount);
+        OMPI_CHECK_DATATYPE_FOR_RECV(rc, recvtype, recvcount);
+        OMPI_CHECK_USER_BUFFER(rc, sendbuf, sendtype, sendcount);
+        OMPI_CHECK_USER_BUFFER(rc, recvbuf, recvtype, recvcount);
+
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+        } else if (dest != MPI_PROC_NULL && ompi_comm_peer_invalid(comm, dest)) {
+            rc = MPI_ERR_RANK;
+        } else if (sendtag < 0 || sendtag > mca_pml.pml_max_tag) {
+            rc = MPI_ERR_TAG;
+        } else if (source != MPI_PROC_NULL && source != MPI_ANY_SOURCE && ompi_comm_peer_invalid(comm, source)) {
+            rc = MPI_ERR_RANK;
+        } else if (((recvtag < 0) && (recvtag !=  MPI_ANY_TAG)) || (recvtag > mca_pml.pml_max_tag)) {
+            rc = MPI_ERR_TAG;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+    if (source != MPI_PROC_NULL) { /* post recv */
+        rc = MCA_PML_CALL(irecv(recvbuf, recvcount, recvtype,
+                                source, recvtag, comm, &req));
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+    if (dest != MPI_PROC_NULL) { /* send */
+        rc = MCA_PML_CALL(send(sendbuf, sendcount, sendtype, dest,
+                               sendtag, MCA_PML_BASE_SEND_STANDARD, comm));
+        if (OPAL_UNLIKELY(MPI_SUCCESS != rc)) {
+            rcs = rc;
+#if OPAL_ENABLE_FT_MPI
+            /* If this is a PROC_FAILED error, we still need to proceed with
+             * the receive, so that we do not propagate errors to the sender in
+             * the case src != dst, and only dst is dead. In this case the 
+             * recv is guaranteed to complete (either in error if the source is 
+             * dead, or successfully if the source is live). */
+            if (OPAL_UNLIKELY(MPI_ERR_PROC_FAILED != rc))
+            /* if intentionally spills outside ifdef */
+#endif
+            ompi_request_cancel(req);
+        }
+    }
+
+    if (source != MPI_PROC_NULL) { /* wait for recv */
+        rc = ompi_request_wait(&req, status);
+#if OPAL_ENABLE_FT_MPI
+        /* Sendrecv never returns ERR_PROC_FAILED_PENDING because it is
+         * blocking. Lets cancel that irecv to complete it NOW and promote
+         * the error to ERR_PROC_FAILED */
+        if( OPAL_UNLIKELY(MPI_ERR_PROC_FAILED_PENDING == rc) ) {
+            ompi_request_cancel(req);
+            ompi_request_wait(&req, MPI_STATUS_IGNORE);
+            rc = MPI_ERR_PROC_FAILED;
+        }
+#endif
+    } else {
+        if (MPI_STATUS_IGNORE != status) {
+            OMPI_COPY_STATUS(status, ompi_request_empty.req_status, false);
+            /*
+             * Per MPI-1, the MPI_ERROR field is not defined for single-completion calls
+             */
+            MEMCHECKER(
+                opal_memchecker_base_mem_undefined(&status->MPI_ERROR, sizeof(int));
+            );
+        }
+        rc = MPI_SUCCESS;
+    }
+    if( OPAL_UNLIKELY(MPI_SUCCESS != rcs && MPI_SUCCESS == rc) ) {
+        rc = rcs;
+    }
+
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/sendrecv_replace.c b/ompi/mpi/c/sendrecv_replace.c
deleted file mode 100644
index 594a72831dc..00000000000
--- a/ompi/mpi/c/sendrecv_replace.c
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2010-2012 Oracle and/or its affiliates.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "opal/datatype/opal_convertor.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/proc/proc.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Sendrecv_replace = PMPI_Sendrecv_replace
-#endif
-#define MPI_Sendrecv_replace PMPI_Sendrecv_replace
-#endif
-
-static const char FUNC_NAME[] = "MPI_Sendrecv_replace";
-
-
-int MPI_Sendrecv_replace(void * buf, int count, MPI_Datatype datatype,
-                         int dest, int sendtag, int source, int recvtag,
-                         MPI_Comm comm, MPI_Status *status)
-
-{
-    int rc = MPI_SUCCESS;
-
-    SPC_RECORD(OMPI_SPC_SENDRECV_REPLACE, 1);
-
-    MEMCHECKER(
-               memchecker_datatype(datatype);
-               memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
-               memchecker_comm(comm);
-               );
-
-    if ( MPI_PARAM_CHECK ) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
-
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
-        } else if (dest != MPI_PROC_NULL && ompi_comm_peer_invalid(comm, dest)) {
-            rc = MPI_ERR_RANK;
-        } else if (sendtag < 0 || sendtag > mca_pml.pml_max_tag) {
-            rc = MPI_ERR_TAG;
-        } else if (source != MPI_PROC_NULL && source != MPI_ANY_SOURCE && ompi_comm_peer_invalid(comm, source)) {
-            rc = MPI_ERR_RANK;
-        } else if (((recvtag < 0) && (recvtag !=  MPI_ANY_TAG)) || (recvtag > mca_pml.pml_max_tag)) {
-            rc = MPI_ERR_TAG;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * The final call to Sendrecv will check for process failures inside
-     * So no need to check here.
-     */
-#endif /* OPAL_ENABLE_FT_MPI */
-
-    /* simple case */
-    if ( source == MPI_PROC_NULL || dest == MPI_PROC_NULL || count == 0 ) {
-        rc = PMPI_Sendrecv(buf, count, datatype, dest, sendtag, buf, count, datatype, source, recvtag, comm, status);
-
-        return rc;
-    }
-
-    /**
-     * If we look for an optimal solution, then we should receive the data into a temporary buffer
-     * and once the send completes we would unpack back into the original buffer. However, if the
-     * sender is unknown, this approach can only be implementing by receiving with the recv datatype
-     * (potentially non-contiguous) and thus the allocated memory will be larger than the size of the
-     * datatype. A simpler, but potentially less efficient approach is to work on the data we have
-     * control of, aka the sent data, and pack it into a contiguous buffer before posting the receive.
-     * Once the send completes, we free it.
-     */
-    opal_convertor_t convertor;
-    unsigned char packed_data[2048];
-    struct iovec iov = { .iov_base = packed_data, .iov_len = sizeof(packed_data) };
-    size_t packed_size, max_data;
-    uint32_t iov_count;
-    ompi_status_public_t recv_status;
-    ompi_proc_t* proc = ompi_comm_peer_lookup(comm, dest);
-    if(proc == NULL) {
-        rc = MPI_ERR_RANK;
-        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-    }
-
-    /* initialize convertor to unpack recv buffer */
-    OBJ_CONSTRUCT(&convertor, opal_convertor_t);
-    opal_convertor_copy_and_prepare_for_send( proc->super.proc_convertor, &(datatype->super),
-                                              count, buf, 0, &convertor );
-
-    /* setup a buffer for recv */
-    opal_convertor_get_packed_size( &convertor, &packed_size );
-    if( packed_size > sizeof(packed_data) ) {
-        rc = PMPI_Alloc_mem(packed_size, MPI_INFO_NULL, &iov.iov_base);
-        if(OMPI_SUCCESS != rc) {
-            rc = OMPI_ERR_OUT_OF_RESOURCE;
-            goto cleanup_and_return;
-        }
-        iov.iov_len = packed_size;
-    }
-    max_data = packed_size;
-    iov_count = 1;
-    rc = opal_convertor_pack(&convertor, &iov, &iov_count, &max_data);
-
-    /* recv into temporary buffer */
-    rc = PMPI_Sendrecv( iov.iov_base, packed_size, MPI_PACKED, dest, sendtag, buf, count,
-                        datatype, source, recvtag, comm, &recv_status );
-
- cleanup_and_return:
-    /* return status to user */
-    if(status != MPI_STATUS_IGNORE) {
-        *status = recv_status;
-    }
-
-    /* release resources */
-    if(packed_size > sizeof(packed_data)) {
-        PMPI_Free_mem(iov.iov_base);
-    }
-    OBJ_DESTRUCT(&convertor);
-
-    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/sendrecv_replace.c.in b/ompi/mpi/c/sendrecv_replace.c.in
new file mode 100644
index 00000000000..76aadb3ab30
--- /dev/null
+++ b/ompi/mpi/c/sendrecv_replace.c.in
@@ -0,0 +1,174 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2012 Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "opal/datatype/opal_convertor.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/proc/proc.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS sendrecv_replace(BUFFER_OUT buf, COUNT count, DATATYPE datatype,
+                                       INT dest, INT sendtag, INT source, INT recvtag,
+                                       COMM comm, STATUS_OUT status)
+{
+    ompi_request_t* req;
+    int rc = MPI_SUCCESS;
+#if OPAL_ENABLE_FT_MPI
+    int rcs = MPI_SUCCESS;
+#endif
+
+    SPC_RECORD(OMPI_SPC_SENDRECV_REPLACE, 1);
+
+    MEMCHECKER(
+               memchecker_datatype(datatype);
+               memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
+               memchecker_comm(comm);
+               );
+
+    if ( MPI_PARAM_CHECK ) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, count);
+
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+        } else if (dest != MPI_PROC_NULL && ompi_comm_peer_invalid(comm, dest)) {
+            rc = MPI_ERR_RANK;
+        } else if (sendtag < 0 || sendtag > mca_pml.pml_max_tag) {
+            rc = MPI_ERR_TAG;
+        } else if (source != MPI_PROC_NULL && source != MPI_ANY_SOURCE && ompi_comm_peer_invalid(comm, source)) {
+            rc = MPI_ERR_RANK;
+        } else if (((recvtag < 0) && (recvtag !=  MPI_ANY_TAG)) || (recvtag > mca_pml.pml_max_tag)) {
+            rc = MPI_ERR_TAG;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * The final call to Sendrecv will check for process failures inside
+     * So no need to check here.
+     */
+#endif /* OPAL_ENABLE_FT_MPI */
+
+    /* simple case */
+    if ( source == MPI_PROC_NULL || dest == MPI_PROC_NULL || count == 0 ) {
+        rc = PMPI_Sendrecv(buf, count, datatype, dest, sendtag, buf, count, datatype, source, recvtag, comm, status);
+
+        return rc;
+    }
+
+    /**
+     * If we look for an optimal solution, then we should receive the data into a temporary buffer
+     * and once the send completes we would unpack back into the original buffer. However, if the
+     * sender is unknown, this approach can only be implementing by receiving with the recv datatype
+     * (potentially non-contiguous) and thus the allocated memory will be larger than the size of the
+     * datatype. A simpler, but potentially less efficient approach is to work on the data we have
+     * control of, aka the sent data, and pack it into a contiguous buffer before posting the receive.
+     * Once the send completes, we free it.
+     */
+    opal_convertor_t convertor;
+    unsigned char packed_data[2048];
+    struct iovec iov = { .iov_base = packed_data, .iov_len = sizeof(packed_data) };
+    size_t packed_size, max_data;
+    uint32_t iov_count;
+    ompi_proc_t* proc = ompi_comm_peer_lookup(comm, dest);
+    if(proc == NULL) {
+        rc = MPI_ERR_RANK;
+        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+    }
+
+    /* initialize convertor to pack send buffer */
+    OBJ_CONSTRUCT(&convertor, opal_convertor_t);
+    opal_convertor_copy_and_prepare_for_send( proc->super.proc_convertor, &(datatype->super),
+                                              count, buf, 0, &convertor );
+
+    /* setup a temporary buffer to send */
+    opal_convertor_get_packed_size( &convertor, &packed_size );
+    if( packed_size > sizeof(packed_data) ) {
+        rc = PMPI_Alloc_mem(packed_size, MPI_INFO_NULL, &iov.iov_base);
+        if(OMPI_SUCCESS != rc) {
+            rc = OMPI_ERR_OUT_OF_RESOURCE;
+            goto cleanup_and_return;
+        }
+        iov.iov_len = packed_size;
+    }
+    max_data = packed_size;
+    iov_count = 1;
+    (void)opal_convertor_pack(&convertor, &iov, &iov_count, &max_data);
+
+    /* receive into the buffer */
+    rc = MCA_PML_CALL(irecv(buf, count, datatype,
+                            source, recvtag, comm, &req));
+    if(OMPI_SUCCESS != rc) {
+        goto cleanup_and_return;
+    }
+
+    /* send from the temporary buffer */
+    rc = MCA_PML_CALL(send(iov.iov_base, packed_size, MPI_PACKED, dest,
+                           sendtag, MCA_PML_BASE_SEND_STANDARD, comm));
+#if OPAL_ENABLE_FT_MPI
+    /* If ULFM is enabled we need to wait for the posted receive to
+     * complete, hence we cannot return here */
+    rcs = rc;
+#else
+    if(OMPI_SUCCESS != rc) {
+        goto cleanup_and_return;
+    }
+#endif  /* OPAL_ENABLE_FT_MPI */
+
+    rc = ompi_request_wait(&req, status);
+#if OPAL_ENABLE_FT_MPI
+    /* Sendrecv_replace never returns ERR_PROC_FAILED_PENDING because it is
+     * blocking. Lets complete now that irecv and promote the error
+     * to ERR_PROC_FAILED */
+    if( OPAL_UNLIKELY(MPI_ERR_PROC_FAILED_PENDING == rc) ) {
+        ompi_request_cancel(req);
+        ompi_request_wait(&req, MPI_STATUS_IGNORE);
+        rc = MPI_ERR_PROC_FAILED;
+    }
+#endif
+
+#if OPAL_ENABLE_FT_MPI
+    if( OPAL_UNLIKELY(MPI_SUCCESS != rcs && MPI_SUCCESS == rc) ) {
+        rc = rcs;
+    }
+#endif
+
+ cleanup_and_return:
+
+    /* release resources */
+    if(packed_size > sizeof(packed_data)) {
+        PMPI_Free_mem(iov.iov_base);
+    }
+    OBJ_DESTRUCT(&convertor);
+
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/session_c2f.c.in b/ompi/mpi/c/session_c2f.c.in
new file mode 100644
index 00000000000..0bb1fa2ef79
--- /dev/null
+++ b/ompi/mpi/c/session_c2f.c.in
@@ -0,0 +1,46 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include <stdio.h>
+
+#include "ompi/instance/instance.h"
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+
+PROTOTYPE FINT session_c2f(SESSION session)
+{
+
+    if ( MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (NULL == session) {
+            return OMPI_INT_2_FINT(-1);
+        }
+    }
+
+    return OMPI_INT_2_FINT(session->i_f_to_c_index);
+}
diff --git a/ompi/mpi/c/session_call_errhandler.c.in b/ompi/mpi/c/session_call_errhandler.c.in
new file mode 100644
index 00000000000..2966e85298b
--- /dev/null
+++ b/ompi/mpi/c/session_call_errhandler.c.in
@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/instance/instance.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS session_call_errhandler(SESSION session, INT errorcode)
+{
+  /* Error checking */
+
+  if (MPI_PARAM_CHECK) {
+    if (ompi_instance_invalid(session)) {
+        if (NULL != session) {
+            return OMPI_ERRHANDLER_INVOKE(session, MPI_ERR_SESSION, FUNC_NAME);
+        } else {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_SESSION, FUNC_NAME);
+        }
+    }
+  }
+
+  /* Invoke the errhandler */
+
+  OMPI_ERRHANDLER_INVOKE(session, errorcode, FUNC_NAME);
+
+  return MPI_SUCCESS;
+}
+
diff --git a/ompi/mpi/c/session_create_errhandler.c.in b/ompi/mpi/c/session_create_errhandler.c.in
new file mode 100644
index 00000000000..9a4a64435be
--- /dev/null
+++ b/ompi/mpi/c/session_create_errhandler.c.in
@@ -0,0 +1,42 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/instance/instance.h"
+
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS session_create_errhandler (SESSION_ERRHANDLER_FUNCTION session_errhandler_fn, ERRHANDLER_OUT errhandler)
+{
+    int err = MPI_SUCCESS;
+
+    if ( MPI_PARAM_CHECK ) {
+        if (NULL == errhandler || NULL == session_errhandler_fn) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+    /* Create and cache the errhandler.  Sets a refcount of 1. */
+    *errhandler =
+        ompi_errhandler_create(OMPI_ERRHANDLER_TYPE_INSTANCE,
+                               (ompi_errhandler_generic_handler_fn_t *) session_errhandler_fn,
+                               OMPI_ERRHANDLER_LANG_C);
+    if (NULL == *errhandler) {
+        err = MPI_ERR_INTERN;
+    }
+
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, MPI_ERR_INTERN, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/session_f2c.c.in b/ompi/mpi/c/session_f2c.c.in
new file mode 100644
index 00000000000..3f5f347a7f9
--- /dev/null
+++ b/ompi/mpi/c/session_f2c.c.in
@@ -0,0 +1,49 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include "ompi/instance/instance.h"
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+
+PROTOTYPE SESSION session_f2c(FINT session)
+{
+    int o_index= OMPI_FINT_2_INT(session);
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+    }
+
+    /* Per MPI-2:4.12.4, do not invoke an error handler if we get an
+       invalid fortran handle.  If we get an invalid fortran handle,
+       return an invalid C handle. */
+
+    if (0 > o_index || o_index >= opal_pointer_array_get_size(&ompi_instance_f_to_c_table)) {
+        return NULL;
+    }
+
+    return (MPI_Session) opal_pointer_array_get_item (&ompi_instance_f_to_c_table, o_index);
+}
diff --git a/ompi/mpi/c/session_finalize.c.in b/ompi/mpi/c/session_finalize.c.in
new file mode 100644
index 00000000000..44e67abeb8e
--- /dev/null
+++ b/ompi/mpi/c/session_finalize.c.in
@@ -0,0 +1,35 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS session_finalize (SESSION_OUT session)
+{
+    int rc;
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (NULL == session || NULL == *session || MPI_SESSION_NULL == *session) {
+            return MPI_ERR_ARG;
+        }
+    }
+
+    rc = ompi_mpi_instance_finalize (session);
+    /* if an error occurred raise it on the null session */
+    OMPI_ERRHANDLER_RETURN (rc, MPI_SESSION_NULL, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/session_get_errhandler.c.in b/ompi/mpi/c/session_get_errhandler.c.in
new file mode 100644
index 00000000000..70c9eb75ebb
--- /dev/null
+++ b/ompi/mpi/c/session_get_errhandler.c.in
@@ -0,0 +1,63 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007-2009 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2017 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2022-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/instance/instance.h"
+
+PROTOTYPE ERROR_CLASS session_get_errhandler(SESSION session, ERRHANDLER_OUT errhandler)
+{
+    int ret = MPI_SUCCESS;
+
+    /* Error checking */
+
+    if (MPI_PARAM_CHECK) {
+      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+      if (ompi_instance_invalid(session)) {
+        if (NULL != session) {
+            return OMPI_ERRHANDLER_INVOKE(session, MPI_ERR_SESSION, FUNC_NAME);
+        } else {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_SESSION, FUNC_NAME);
+        }
+      }
+    }
+
+    OPAL_THREAD_LOCK(&(session->s_lock));
+    /* Retain the errhandler, corresponding to object refcount decrease
+       in errhandler_free.c. */
+    OBJ_RETAIN(session->error_handler);
+    *errhandler = session->error_handler;
+    OPAL_THREAD_UNLOCK(&(session->s_lock));
+
+   /* make sure the infrastructure is initialized */
+    ret = ompi_mpi_instance_retain ();
+
+    /* All done */
+    OMPI_ERRHANDLER_RETURN (ret, session, ret, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/session_get_info.c.in b/ompi/mpi/c/session_get_info.c.in
new file mode 100644
index 00000000000..3c4d743b3d8
--- /dev/null
+++ b/ompi/mpi/c/session_get_info.c.in
@@ -0,0 +1,60 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/instance/instance.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include <stdlib.h>
+#include <string.h>
+
+PROTOTYPE ERROR_CLASS session_get_info (SESSION session, INFO_OUT info_used)
+{
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_instance_invalid(session)) {
+            if (NULL != session) {
+                return OMPI_ERRHANDLER_INVOKE(session, MPI_ERR_SESSION, FUNC_NAME);
+            } else {
+                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_SESSION, FUNC_NAME);
+            }
+        }
+        if (NULL == info_used) {
+            return OMPI_ERRHANDLER_INVOKE (session, MPI_ERR_INFO, FUNC_NAME);
+        }
+    }
+
+    if (NULL == session->super.s_info) {
+        /*
+         * Setup any defaults if MPI_Win_set_info was never called
+         */
+        opal_infosubscribe_change_info (&session->super, &MPI_INFO_NULL->super);
+    }
+
+
+    *info_used = ompi_info_allocate ();
+    if (OPAL_UNLIKELY(NULL == *info_used)) {
+        return OMPI_ERRHANDLER_INVOKE (session, MPI_ERR_NO_MEM, FUNC_NAME);
+    }
+
+    opal_info_t *opal_info_used = &(*info_used)->super;
+
+    opal_info_dup (session->super.s_info, &opal_info_used);
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/session_get_nth_pset.c.in b/ompi/mpi/c/session_get_nth_pset.c.in
new file mode 100644
index 00000000000..11d84569b15
--- /dev/null
+++ b/ompi/mpi/c/session_get_nth_pset.c.in
@@ -0,0 +1,39 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/instance/instance.h"
+
+PROTOTYPE ERROR_CLASS session_get_nth_pset (SESSION session, INFO info, INT n, INT_OUT len, STRING_OUT pset_name)
+{
+    int rc = MPI_SUCCESS;
+
+    if ( MPI_PARAM_CHECK ) {
+        if (ompi_instance_invalid(session)) {
+            if (NULL != session) {
+                return OMPI_ERRHANDLER_INVOKE(session, MPI_ERR_SESSION, FUNC_NAME);
+            } else {
+                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_SESSION, FUNC_NAME);
+            }
+        } else if ((NULL == pset_name && *len > 0) || n < 0) {
+            return OMPI_ERRHANDLER_INVOKE(session, MPI_ERR_ARG, FUNC_NAME);
+        }
+    }
+
+    rc = ompi_instance_get_nth_pset (session, n, len, pset_name);
+
+    OMPI_ERRHANDLER_RETURN (rc, (NULL == session) ? MPI_SESSION_NULL : session, 
+                            rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/session_get_num_psets.c.in b/ompi/mpi/c/session_get_num_psets.c.in
new file mode 100644
index 00000000000..93bb265424c
--- /dev/null
+++ b/ompi/mpi/c/session_get_num_psets.c.in
@@ -0,0 +1,39 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/instance/instance.h"
+
+PROTOTYPE ERROR_CLASS session_get_num_psets (SESSION session, INFO info, INT_OUT npset_names)
+{
+    int rc;
+
+    if ( MPI_PARAM_CHECK ) {
+        if (ompi_instance_invalid(session)) {
+            if (NULL != session) {
+                return OMPI_ERRHANDLER_INVOKE(session, MPI_ERR_SESSION, FUNC_NAME);
+            } else {
+                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_SESSION, FUNC_NAME);
+            }
+        } else if (NULL == npset_names) {
+            return OMPI_ERRHANDLER_INVOKE(session, MPI_ERR_ARG, FUNC_NAME);
+        }
+    }
+
+    rc = ompi_instance_get_num_psets (session, npset_names);
+
+    OMPI_ERRHANDLER_RETURN (rc, (NULL == session) ? MPI_SESSION_NULL : session,
+                            rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/session_get_pset_info.c.in b/ompi/mpi/c/session_get_pset_info.c.in
new file mode 100644
index 00000000000..660fe91ed6d
--- /dev/null
+++ b/ompi/mpi/c/session_get_pset_info.c.in
@@ -0,0 +1,58 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/instance/instance.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include <stdlib.h>
+#include <string.h>
+
+PROTOTYPE ERROR_CLASS session_get_pset_info (SESSION session, STRING pset_name, INFO_OUT info_used)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_instance_invalid(session)) {
+            if (NULL != session) {
+                return OMPI_ERRHANDLER_INVOKE(session, MPI_ERR_SESSION, FUNC_NAME);
+            } else {
+                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_SESSION, FUNC_NAME);
+            }
+        }
+        if (NULL == info_used) {
+            return OMPI_ERRHANDLER_INVOKE (session, MPI_ERR_INFO, FUNC_NAME);
+        }
+        if (NULL == pset_name) {
+            return OMPI_ERRHANDLER_INVOKE (session, MPI_ERR_ARG, FUNC_NAME);
+        }
+    }
+
+    rc = ompi_instance_get_pset_info (session, pset_name, (opal_info_t **) info_used);
+    /*
+     * if process set was not found, OMPI_ERR_NOT_FOUND is the return value.
+     * we want to map this to MPI_ERR_ARG but we have to do it manually here
+     * since the OMPI error to MPI error code code maps this to MPI_ERR_INTERN
+     */
+    if (OMPI_ERR_NOT_FOUND == rc) {
+        rc = MPI_ERR_ARG;
+    }
+
+    return OMPI_ERRHANDLER_INVOKE(session, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/session_init.c.in b/ompi/mpi/c/session_init.c.in
new file mode 100644
index 00000000000..bcc71c7eeb4
--- /dev/null
+++ b/ompi/mpi/c/session_init.c.in
@@ -0,0 +1,50 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/info/info.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/instance/instance.h"
+
+PROTOTYPE ERROR_CLASS session_init (INFO info, ERRHANDLER errhandler, SESSION_OUT session)
+{
+    int rc, flag;
+    int ts_level = MPI_THREAD_SINGLE;  /* for now we default to thread single for OMPI sessions */
+    opal_cstring_t *info_value;
+    const char ts_level_multi[] = "MPI_THREAD_MULTIPLE";
+
+    if ( MPI_PARAM_CHECK ) {
+        if (NULL == errhandler || NULL == session) {
+            return MPI_ERR_ARG;
+        }
+
+        if (NULL == info || ompi_info_is_freed (info)) {
+            return MPI_ERR_INFO;
+        }
+    }
+
+    if (MPI_INFO_NULL != info) {
+        (void) ompi_info_get (info, "thread_level", &info_value, &flag);
+        if (flag) {
+            if(strncmp(info_value->string, ts_level_multi, strlen(ts_level_multi)) == 0) {
+                ts_level = MPI_THREAD_MULTIPLE;
+            }
+            OBJ_RELEASE(info_value);
+        }
+    }
+
+    rc = ompi_mpi_instance_init (ts_level, &info->super, errhandler, session, 0, NULL);
+    /* if an error occurred raise it on the null session */
+    OMPI_ERRHANDLER_RETURN (rc, MPI_SESSION_NULL, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/session_set_errhandler.c.in b/ompi/mpi/c/session_set_errhandler.c.in
new file mode 100644
index 00000000000..a2ad8e36efe
--- /dev/null
+++ b/ompi/mpi/c/session_set_errhandler.c.in
@@ -0,0 +1,63 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015-2017 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2022-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/instance/instance.h"
+
+PROTOTYPE ERROR_CLASS session_set_errhandler(SESSION session, ERRHANDLER errhandler)
+{
+    MPI_Errhandler tmp;
+
+    /* Error checking */
+
+    if (MPI_PARAM_CHECK) {
+        if (ompi_instance_invalid(session)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_SESSION, FUNC_NAME);
+        } else if (NULL == errhandler ||
+                   MPI_ERRHANDLER_NULL == errhandler ||
+                   ( OMPI_ERRHANDLER_TYPE_INSTANCE != errhandler->eh_mpi_object_type &&
+                     OMPI_ERRHANDLER_TYPE_PREDEFINED != errhandler->eh_mpi_object_type) ) {
+            return OMPI_ERRHANDLER_INVOKE(session, MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+    /* Prepare the new error handler */
+    OBJ_RETAIN(errhandler);
+
+    OPAL_THREAD_LOCK(&(session->s_lock));
+    /* Ditch the old errhandler, and decrement its refcount. */
+    tmp = session->error_handler;
+    session->error_handler = errhandler;
+    OBJ_RELEASE(tmp);
+    OPAL_THREAD_UNLOCK(&(session->s_lock));
+
+    /* All done */
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/session_set_info.c.in b/ompi/mpi/c/session_set_info.c.in
new file mode 100644
index 00000000000..ba974b2f402
--- /dev/null
+++ b/ompi/mpi/c/session_set_info.c.in
@@ -0,0 +1,46 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/instance/instance.h"
+#include "ompi/errhandler/errhandler.h"
+#include "opal/util/info_subscriber.h"
+#include <stdlib.h>
+#include <string.h>
+
+PROTOTYPE ERROR_CLASS session_set_info (SESSION session, INFO info)
+{
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_instance_invalid(session)) {
+            if (NULL != session) {
+                return OMPI_ERRHANDLER_INVOKE(session, MPI_ERR_SESSION, FUNC_NAME);
+            } else {
+                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_SESSION, FUNC_NAME);
+            }
+
+        }
+        if (NULL == info || MPI_INFO_NULL == info || ompi_info_is_freed(info)) {
+            return OMPI_ERRHANDLER_INVOKE (session, MPI_ERR_INFO, FUNC_NAME);
+        }
+    }
+
+    opal_infosubscribe_change_info (&session->super, &info->super);
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/ssend.c b/ompi/mpi/c/ssend.c
deleted file mode 100644
index 55ec1c671ab..00000000000
--- a/ompi/mpi/c/ssend.c
+++ /dev/null
@@ -1,98 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Ssend = PMPI_Ssend
-#endif
-#define MPI_Ssend PMPI_Ssend
-#endif
-
-static const char FUNC_NAME[] = "MPI_Ssend";
-
-
-int MPI_Ssend(const void *buf, int count, MPI_Datatype type, int dest, int tag, MPI_Comm comm)
-{
-    int rc = MPI_SUCCESS;
-
-    SPC_RECORD(OMPI_SPC_SSEND, 1);
-
-    MEMCHECKER(
-        memchecker_datatype(type);
-        memchecker_call(&opal_memchecker_base_isdefined, buf, count, type);
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else if (MPI_DATATYPE_NULL == type || NULL == type) {
-            rc = MPI_ERR_TYPE;
-        } else if (tag < 0 || tag > mca_pml.pml_max_tag) {
-            rc = MPI_ERR_TAG;
-        } else if (ompi_comm_peer_invalid(comm, dest) &&
-                   (MPI_PROC_NULL != dest)) {
-            rc = MPI_ERR_RANK;
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_SEND(rc, type, count);
-            OMPI_CHECK_USER_BUFFER(rc, buf, type, count);
-        }
-        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * An early check, so as to return early if we are communicating with
-     * a failed process. This is not absolutely necessary since we will
-     * check for this, and other, error conditions during the completion
-     * call in the PML.
-     */
-    if( OPAL_UNLIKELY(!ompi_comm_iface_p2p_check_proc(comm, dest, &rc)) ) {
-        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-    }
-#endif
-
-    if (MPI_PROC_NULL == dest) {
-        return MPI_SUCCESS;
-    }
-
-    rc = MCA_PML_CALL(send(buf, count, type, dest, tag,
-                           MCA_PML_BASE_SEND_SYNCHRONOUS, comm));
-    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/ssend.c.in b/ompi/mpi/c/ssend.c.in
new file mode 100644
index 00000000000..a5bb55d0712
--- /dev/null
+++ b/ompi/mpi/c/ssend.c.in
@@ -0,0 +1,90 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS ssend(BUFFER buf, COUNT count, DATATYPE type, INT dest, INT tag, COMM comm)
+{
+    int rc = MPI_SUCCESS;
+
+    SPC_RECORD(OMPI_SPC_SSEND, 1);
+
+    MEMCHECKER(
+        memchecker_datatype(type);
+        memchecker_call(&opal_memchecker_base_isdefined, buf, count, type);
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else if (MPI_DATATYPE_NULL == type || NULL == type) {
+            rc = MPI_ERR_TYPE;
+        } else if (tag < 0 || tag > mca_pml.pml_max_tag) {
+            rc = MPI_ERR_TAG;
+        } else if (ompi_comm_peer_invalid(comm, dest) &&
+                   (MPI_PROC_NULL != dest)) {
+            rc = MPI_ERR_RANK;
+        } else {
+            OMPI_CHECK_DATATYPE_FOR_SEND(rc, type, count);
+            OMPI_CHECK_USER_BUFFER(rc, buf, type, count);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * An early check, so as to return early if we are communicating with
+     * a failed process. This is not absolutely necessary since we will
+     * check for this, and other, error conditions during the completion
+     * call in the PML.
+     */
+    if( OPAL_UNLIKELY(!ompi_comm_iface_p2p_check_proc(comm, dest, &rc)) ) {
+        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+    }
+#endif
+
+    if (MPI_PROC_NULL == dest) {
+        return MPI_SUCCESS;
+    }
+
+    rc = MCA_PML_CALL(send(buf, count, type, dest, tag,
+                           MCA_PML_BASE_SEND_SYNCHRONOUS, comm));
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/ssend_init.c b/ompi/mpi/c/ssend_init.c
deleted file mode 100644
index 78844583200..00000000000
--- a/ompi/mpi/c/ssend_init.c
+++ /dev/null
@@ -1,98 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/request/request.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Ssend_init = PMPI_Ssend_init
-#endif
-#define MPI_Ssend_init PMPI_Ssend_init
-#endif
-
-static const char FUNC_NAME[] = "MPI_Ssend_init";
-
-
-int MPI_Ssend_init(const void *buf, int count, MPI_Datatype type,
-                   int dest, int tag, MPI_Comm comm,
-                   MPI_Request *request)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(type);
-        memchecker_call(&opal_memchecker_base_isaddressable, buf, count, type);
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        } else if (MPI_DATATYPE_NULL == type || NULL == type) {
-            rc = MPI_ERR_TYPE;
-        } else if (tag < 0 || tag > mca_pml.pml_max_tag) {
-            rc = MPI_ERR_TAG;
-        } else if (ompi_comm_peer_invalid(comm, dest) &&
-                   (MPI_PROC_NULL != dest)) {
-            rc = MPI_ERR_RANK;
-        } else if (request == NULL) {
-            rc = MPI_ERR_REQUEST;
-        }
-        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-    }
-
-    if (MPI_PROC_NULL == dest) {
-        rc = ompi_request_persistent_noop_create(request);
-        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * The request will be checked for process failure errors during the
-     * completion calls. So no need to check here.
-     */
-#endif
-
-    /*
-     * Here, we just initialize the request -- memchecker should set the buffer in MPI_Start.
-     */
-    rc = MCA_PML_CALL(isend_init(buf, count, type, dest, tag,
-                                 MCA_PML_BASE_SEND_SYNCHRONOUS, comm, request));
-    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-}
-
diff --git a/ompi/mpi/c/ssend_init.c.in b/ompi/mpi/c/ssend_init.c.in
new file mode 100644
index 00000000000..33bc8616a16
--- /dev/null
+++ b/ompi/mpi/c/ssend_init.c.in
@@ -0,0 +1,90 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/request/request.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS ssend_init(BUFFER buf, COUNT count, DATATYPE type,
+                   INT dest, INT tag, COMM comm,
+                   REQUEST_INOUT request)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(type);
+        memchecker_call(&opal_memchecker_base_isaddressable, buf, count, type);
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        } else if (MPI_DATATYPE_NULL == type || NULL == type) {
+            rc = MPI_ERR_TYPE;
+        } else if (tag < 0 || tag > mca_pml.pml_max_tag) {
+            rc = MPI_ERR_TAG;
+        } else if (ompi_comm_peer_invalid(comm, dest) &&
+                   (MPI_PROC_NULL != dest)) {
+            rc = MPI_ERR_RANK;
+        } else if (request == NULL) {
+            rc = MPI_ERR_REQUEST;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+    if (MPI_PROC_NULL == dest) {
+        rc = ompi_request_persistent_noop_create(request);
+        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * The request will be checked for process failure errors during the
+     * completion calls. So no need to check here.
+     */
+#endif
+
+    /*
+     * Here, we just initialize the request -- memchecker should set the buffer in MPI_Start.
+     */
+    rc = MCA_PML_CALL(isend_init(buf, count, type, dest, tag,
+                                 MCA_PML_BASE_SEND_SYNCHRONOUS, comm, request));
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
+
diff --git a/ompi/mpi/c/start.c b/ompi/mpi/c/start.c
deleted file mode 100644
index c8862254cbe..00000000000
--- a/ompi/mpi/c/start.c
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2015-2017 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      FUJITSU LIMITED.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/request/request.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Start = PMPI_Start
-#endif
-#define MPI_Start PMPI_Start
-#endif
-
-static const char FUNC_NAME[] = "MPI_Start";
-
-
-int MPI_Start(MPI_Request *request)
-{
-    int ret = OMPI_SUCCESS;
-
-    MEMCHECKER(
-        memchecker_request(request);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        int rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (request == NULL) {
-            rc = MPI_ERR_REQUEST;
-        }
-        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
-    }
-    /**
-     * Per definition of the handling of persistent request in the
-     * MPI standard 3.1 page 78 line 19: we must have the following
-     * sequence CREATE (START COMPLETE)* FREE. The upper level is
-     * responsible for handling any concurency. The PML must handle
-     * this case, as it is the only one knowing if the request can
-     * be reused or not (it is PML completed or not?).
-     */
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * The request will be checked for process failure errors during the
-     * completion calls. So no need to check here.
-     */
-#endif
-
-    switch((*request)->req_type) {
-    case OMPI_REQUEST_PML:
-    case OMPI_REQUEST_COLL:
-        if ( MPI_PARAM_CHECK && !(*request)->req_persistent) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_REQUEST, FUNC_NAME);
-        }
-
-        ret = (*request)->req_start(1, request);
-
-        return ret;
-
-    case OMPI_REQUEST_NOOP:
-        /**
-         * We deal with a MPI_PROC_NULL request. If the request is
-         * already active, fall back to the error case in the default.
-         * Otherwise, mark it active so we can correctly handle it in
-         * the wait*.
-         */
-        if( OMPI_REQUEST_INACTIVE == (*request)->req_state ) {
-            (*request)->req_state = OMPI_REQUEST_ACTIVE;
-            return MPI_SUCCESS;
-        }
-
-    default:
-        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_REQUEST, FUNC_NAME);
-    }
-}
-
diff --git a/ompi/mpi/c/start.c.in b/ompi/mpi/c/start.c.in
new file mode 100644
index 00000000000..954724e5154
--- /dev/null
+++ b/ompi/mpi/c/start.c.in
@@ -0,0 +1,98 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2015-2017 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/request/request.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS start(REQUEST_INOUT request)
+{
+    int ret = OMPI_SUCCESS;
+
+    MEMCHECKER(
+        memchecker_request(request);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        int rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (request == NULL) {
+            rc = MPI_ERR_REQUEST;
+        }
+        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
+    }
+    /**
+     * Per definition of the handling of persistent request in the
+     * MPI standard 3.1 page 78 line 19: we must have the following
+     * sequence CREATE (START COMPLETE)* FREE. The upper level is
+     * responsible for handling any concurrency. The PML must handle
+     * this case, as it is the only one knowing if the request can
+     * be reused or not (it is PML completed or not?).
+     */
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * The request will be checked for process failure errors during the
+     * completion calls. So no need to check here.
+     */
+#endif
+
+    switch((*request)->req_type) {
+    case OMPI_REQUEST_PML:
+    case OMPI_REQUEST_COLL:
+    case OMPI_REQUEST_PART:
+        if ( MPI_PARAM_CHECK && !((*request)->req_persistent &&
+                                  OMPI_REQUEST_INACTIVE == (*request)->req_state)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_REQUEST, FUNC_NAME);
+        }
+
+        ret = (*request)->req_start(1, request);
+
+        OMPI_ERRHANDLER_NOHANDLE_RETURN(ret, ret, FUNC_NAME);
+
+    case OMPI_REQUEST_NOOP:
+        /**
+         * We deal with a MPI_PROC_NULL request. If the request is
+         * already active, fall back to the error case in the default.
+         * Otherwise, mark it active so we can correctly handle it in
+         * the wait*.
+         */
+        if( OMPI_REQUEST_INACTIVE == (*request)->req_state ) {
+            (*request)->req_state = OMPI_REQUEST_ACTIVE;
+            return MPI_SUCCESS;
+        }
+
+    default:
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_REQUEST, FUNC_NAME);
+    }
+}
+
diff --git a/ompi/mpi/c/startall.c b/ompi/mpi/c/startall.c
deleted file mode 100644
index 1b223bb88aa..00000000000
--- a/ompi/mpi/c/startall.c
+++ /dev/null
@@ -1,121 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2012-2016 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2014-2017 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017-2018 FUJITSU LIMITED.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/request/request.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Startall = PMPI_Startall
-#endif
-#define MPI_Startall PMPI_Startall
-#endif
-
-static const char FUNC_NAME[] = "MPI_Startall";
-
-
-int MPI_Startall(int count, MPI_Request requests[])
-{
-    int i, j;
-    int ret = OMPI_SUCCESS;
-    ompi_request_start_fn_t start_fn = NULL;
-
-    MEMCHECKER(
-        for (j = 0; j < count; j++){
-            memchecker_request(&requests[j]);
-        }
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        int rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == requests) {
-            rc = MPI_ERR_REQUEST;
-        } else if (count < 0) {
-            rc = MPI_ERR_ARG;
-        } else {
-            for (i = 0; i < count; ++i) {
-                if (NULL == requests[i] ||
-                    ! requests[i]->req_persistent ||
-                    (OMPI_REQUEST_PML  != requests[i]->req_type &&
-                     OMPI_REQUEST_COLL != requests[i]->req_type &&
-                     OMPI_REQUEST_NOOP != requests[i]->req_type)) {
-                    rc = MPI_ERR_REQUEST;
-                    break;
-                }
-            }
-        }
-        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
-    }
-
-#if OPAL_ENABLE_FT_MPI
-    /*
-     * The request will be checked for process failure errors during the
-     * completion calls. So no need to check here.
-     */
-#endif
-
-    for (i = 0, j = -1; i < count; ++i) {
-        /* Per MPI it is invalid to start an active request */
-        if (OMPI_REQUEST_INACTIVE != requests[i]->req_state) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_REQUEST, FUNC_NAME);
-        }
-
-        if (OMPI_REQUEST_NOOP == requests[i]->req_type) {
-            /**
-             * We deal with a MPI_PROC_NULL request. If the request is
-             * already active, fall back to the error case in the default.
-             * Otherwise, mark it active so we can correctly handle it in
-             * the wait*.
-             */
-            requests[i]->req_state = OMPI_REQUEST_ACTIVE;
-        }
-
-        /* Call a req_start callback function per requests which have the
-         * same req_start value. */
-        if (requests[i]->req_start != start_fn) {
-            if (NULL != start_fn && i != 0) {
-                start_fn(i - j, requests + j);
-            }
-            start_fn = requests[i]->req_start;
-            j = i;
-        }
-    }
-
-    if (NULL != start_fn) {
-        start_fn(i - j, requests + j);
-    }
-
-    return ret;
-}
-
diff --git a/ompi/mpi/c/startall.c.in b/ompi/mpi/c/startall.c.in
new file mode 100644
index 00000000000..d75c02debfe
--- /dev/null
+++ b/ompi/mpi/c/startall.c.in
@@ -0,0 +1,114 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2012-2016 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2017 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017-2018 FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/request/request.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS startall(INT count, REQUEST_INOUT requests)
+{
+    int i, j;
+    int ret = OMPI_SUCCESS;
+    ompi_request_start_fn_t start_fn = NULL;
+
+    MEMCHECKER(
+        for (j = 0; j < count; j++){
+            memchecker_request(&requests[j]);
+        }
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        int rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if ((NULL == requests) && (0 != count)) {
+            rc = MPI_ERR_REQUEST;
+        } else if (count < 0) {
+            rc = MPI_ERR_ARG;
+        } else {
+            for (i = 0; i < count; ++i) {
+                if (NULL == requests[i] ||
+                    ! requests[i]->req_persistent ||
+                    (OMPI_REQUEST_PML  != requests[i]->req_type &&
+                     OMPI_REQUEST_COLL != requests[i]->req_type &&
+                     OMPI_REQUEST_PART != requests[i]->req_type &&
+                     OMPI_REQUEST_NOOP != requests[i]->req_type)) {
+                    rc = MPI_ERR_REQUEST;
+                    break;
+                }
+            }
+        }
+        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
+    }
+
+#if OPAL_ENABLE_FT_MPI
+    /*
+     * The request will be checked for process failure errors during the
+     * completion calls. So no need to check here.
+     */
+#endif
+
+    for (i = 0, j = -1; i < count; ++i) {
+        /* Per MPI it is invalid to start an active request */
+        if (OMPI_REQUEST_INACTIVE != requests[i]->req_state) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_REQUEST, FUNC_NAME);
+        }
+
+        if (OMPI_REQUEST_NOOP == requests[i]->req_type) {
+            /**
+             * We deal with a MPI_PROC_NULL request. If the request is
+             * already active, fall back to the error case in the default.
+             * Otherwise, mark it active so we can correctly handle it in
+             * the wait*.
+             */
+            requests[i]->req_state = OMPI_REQUEST_ACTIVE;
+        }
+
+        /* Call a req_start callback function per requests which have the
+         * same req_start value. */
+        if (requests[i]->req_start != start_fn) {
+            if (NULL != start_fn && i != 0) {
+                start_fn(i - j, requests + j);
+            }
+            start_fn = requests[i]->req_start;
+            j = i;
+        }
+    }
+
+    if (NULL != start_fn) {
+        start_fn(i - j, requests + j);
+    }
+
+    return ret;
+}
+
diff --git a/ompi/mpi/c/status_c2f.c b/ompi/mpi/c/status_c2f.c
deleted file mode 100644
index dd0190cae66..00000000000
--- a/ompi/mpi/c/status_c2f.c
+++ /dev/null
@@ -1,101 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mpi/fortran/base/fint_2_int.h"
-#include "ompi/mpi/fortran/base/constants.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Status_c2f = PMPI_Status_c2f
-#endif
-#define MPI_Status_c2f PMPI_Status_c2f
-#endif
-
-static const char FUNC_NAME[] = "MPI_Status_c2f";
-
-
-int MPI_Status_c2f(const MPI_Status *c_status, MPI_Fint *f_status)
-{
-    const int *c_ints;
-    int i;
-    MEMCHECKER(
-        if(c_status != MPI_STATUSES_IGNORE) {
-            /*
-             * Before checking the complete status, we need to reset the definedness
-             * of the MPI_ERROR-field (single-completion calls wait/test).
-             */
-            opal_memchecker_base_mem_defined((void*)&c_status->MPI_ERROR, sizeof(int));
-            memchecker_status(c_status);
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        /* MPI-2:4.12.5 says that if you pass in
-           MPI_STATUS[ES]_IGNORE, it's erroneous */
-
-        if (NULL == c_status || MPI_STATUS_IGNORE == c_status ||
-            MPI_STATUSES_IGNORE == c_status || NULL == f_status) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(
-                                          MPI_ERR_IN_STATUS, FUNC_NAME);
-        }
-    }
-
-    /* Note that MPI-2.2 16.3.5 states that even the hidden data in a
-       status must be converted (!).  This is somewhat problematic
-       because the Fortran data is all INTEGERS while the C MPI_Status
-       contains a size_t.  That being said, note 2 things:
-
-       1. The _ucount and _canceled members are never accessed from
-          Fortran.
-       2. configure calculated a value of MPI_STATUS_SIZE to ensure
-          that the Fortran status is the Right size to hold the C
-          MPI_Status (including the size_t member).
-
-       So for the purposes of this function, just copy over all the
-       data as if they were int's.  This works because all OMPI
-       Fortran MPI API functions that take a status as an IN argument
-       first call MPI_Status_f2c on it before using it (in which case
-       we'll do the exact opposite copy, thereby rebuilding the size_t
-       value properly before it is accessed in C).
-
-       Note that if sizeof(int) > sizeof(INTEGER), we're potentially
-       hosed anyway (i.e., even the public values in the status could
-       get truncated).  But if sizeof(int) == sizeof(INTEGER) or
-       sizeof(int) < sizeof(INTEGER), everything should be kosher. */
-    c_ints = (const int*)c_status;
-    for( i = 0; i < (int)(sizeof(MPI_Status) / sizeof(int)); i++ ) {
-        f_status[i] = OMPI_INT_2_FINT(c_ints[i]);
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/status_c2f.c.in b/ompi/mpi/c/status_c2f.c.in
new file mode 100644
index 00000000000..d430bc7964f
--- /dev/null
+++ b/ompi/mpi/c/status_c2f.c.in
@@ -0,0 +1,93 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+#include "ompi/mpi/fortran/base/constants.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS status_c2f(STATUS c_status, FINT_OUT f_status)
+{
+    const int *c_ints;
+    int i;
+    MEMCHECKER(
+        if(c_status != MPI_STATUSES_IGNORE) {
+            /*
+             * Before checking the complete status, we need to reset the definedness
+             * of the MPI_ERROR-field (single-completion calls wait/test).
+             */
+            opal_memchecker_base_mem_defined((void*)&c_status->MPI_ERROR, sizeof(int));
+            memchecker_status(c_status);
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        /* MPI-2:4.12.5 says that if you pass in
+           MPI_STATUS[ES]_IGNORE, it's erroneous */
+
+        if (NULL == c_status || MPI_STATUS_IGNORE == c_status ||
+            MPI_STATUSES_IGNORE == c_status || NULL == f_status) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(
+                                          MPI_ERR_IN_STATUS, FUNC_NAME);
+        }
+    }
+
+    /* Note that MPI-2.2 16.3.5 states that even the hidden data in a
+       status must be converted (!).  This is somewhat problematic
+       because the Fortran data is all INTEGERS while the C MPI_Status
+       contains a size_t.  That being said, note 2 things:
+
+       1. The _ucount and _canceled members are never accessed from
+          Fortran.
+       2. configure calculated a value of MPI_STATUS_SIZE to ensure
+          that the Fortran status is the Right size to hold the C
+          MPI_Status (including the size_t member).
+
+       So for the purposes of this function, just copy over all the
+       data as if they were int's.  This works because all OMPI
+       Fortran MPI API functions that take a status as an IN argument
+       first call MPI_Status_f2c on it before using it (in which case
+       we'll do the exact opposite copy, thereby rebuilding the size_t
+       value properly before it is accessed in C).
+
+       Note that if sizeof(int) > sizeof(INTEGER), we're potentially
+       hosed anyway (i.e., even the public values in the status could
+       get truncated).  But if sizeof(int) == sizeof(INTEGER) or
+       sizeof(int) < sizeof(INTEGER), everything should be kosher. */
+    c_ints = (const int*)c_status;
+    for( i = 0; i < (int)(sizeof(MPI_Status) / sizeof(int)); i++ ) {
+        f_status[i] = OMPI_INT_2_FINT(c_ints[i]);
+    }
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/status_c2f08.c b/ompi/mpi/c/status_c2f08.c
deleted file mode 100644
index c5b5dd335b7..00000000000
--- a/ompi/mpi/c/status_c2f08.c
+++ /dev/null
@@ -1,85 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2020 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mpi/fortran/base/fint_2_int.h"
-#include "ompi/mpi/fortran/base/constants.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Status_c2f08 = PMPI_Status_c2f08
-#endif
-#define MPI_Status_c2f08 PMPI_Status_c2f08
-#endif
-
-static const char FUNC_NAME[] = "MPI_Status_c2f08";
-
-
-int MPI_Status_c2f08(const MPI_Status *c_status, MPI_F08_status *f08_status)
-{
-    const int *c_ints;
-    MEMCHECKER(
-        if(c_status != MPI_STATUSES_IGNORE) {
-            /*
-             * Before checking the complete status, we need to reset the definedness
-             * of the MPI_ERROR-field (single-completion calls wait/test).
-             */
-            opal_memchecker_base_mem_defined((void*)&c_status->MPI_ERROR, sizeof(int));
-            memchecker_status(c_status);
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        /* MPI-4:18.2.5 implies that if you pass in
-           MPI_STATUS[ES]_IGNORE, it's erroneous */
-
-        if (NULL == c_status || MPI_STATUS_IGNORE == c_status ||
-            MPI_STATUSES_IGNORE == c_status || NULL == f08_status) {
-            return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD,
-                                          MPI_ERR_IN_STATUS, FUNC_NAME);
-        }
-    }
-
-    /* ***NOTE*** See huge comment in status_c2f.c (yes, I know
-                  there's a size_t member in the C MPI_Status -- go
-                  read that comment for an explanation why copying
-                  everything as a bunch of int's is ok). */
-    f08_status->MPI_SOURCE = OMPI_INT_2_FINT(c_status->MPI_SOURCE);
-    f08_status->MPI_TAG = OMPI_INT_2_FINT(c_status->MPI_TAG);
-    f08_status->MPI_ERROR = OMPI_INT_2_FINT(c_status->MPI_ERROR);
-    c_ints = (const int *)c_status + 3;
-    for(int i = 0; i < (int)(sizeof(MPI_Status) / sizeof(int) - 3); i++ ) {
-        f08_status->internal[i] = OMPI_INT_2_FINT(c_ints[i]);
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/status_c2f08.c.in b/ompi/mpi/c/status_c2f08.c.in
new file mode 100644
index 00000000000..74919c4b1d7
--- /dev/null
+++ b/ompi/mpi/c/status_c2f08.c.in
@@ -0,0 +1,77 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2020 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+#include "ompi/mpi/fortran/base/constants.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS status_c2f08(STATUS c_status, F08_STATUS_OUT f08_status)
+{
+    const int *c_ints;
+    MEMCHECKER(
+        if(c_status != MPI_STATUSES_IGNORE) {
+            /*
+             * Before checking the complete status, we need to reset the definedness
+             * of the MPI_ERROR-field (single-completion calls wait/test).
+             */
+            opal_memchecker_base_mem_defined((void*)&c_status->MPI_ERROR, sizeof(int));
+            memchecker_status(c_status);
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        /* MPI-4:18.2.5 implies that if you pass in
+           MPI_STATUS[ES]_IGNORE, it's erroneous */
+
+        if (NULL == c_status || MPI_STATUS_IGNORE == c_status ||
+            MPI_STATUSES_IGNORE == c_status || NULL == f08_status) {
+            return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD,
+                                          MPI_ERR_IN_STATUS, FUNC_NAME);
+        }
+    }
+
+    /* ***NOTE*** See huge comment in status_c2f.c (yes, I know
+                  there's a size_t member in the C MPI_Status -- go
+                  read that comment for an explanation why copying
+                  everything as a bunch of int's is ok). */
+    f08_status->MPI_SOURCE = OMPI_INT_2_FINT(c_status->MPI_SOURCE);
+    f08_status->MPI_TAG = OMPI_INT_2_FINT(c_status->MPI_TAG);
+    f08_status->MPI_ERROR = OMPI_INT_2_FINT(c_status->MPI_ERROR);
+    c_ints = (const int *)c_status + 3;
+    for(int i = 0; i < (int)(sizeof(MPI_Status) / sizeof(int) - 3); i++ ) {
+        f08_status->internal[i] = OMPI_INT_2_FINT(c_ints[i]);
+    }
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/status_f082c.c b/ompi/mpi/c/status_f082c.c
deleted file mode 100644
index 0134b698f78..00000000000
--- a/ompi/mpi/c/status_f082c.c
+++ /dev/null
@@ -1,84 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2020 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mpi/fortran/base/fint_2_int.h"
-#include "ompi/mpi/fortran/base/constants.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Status_f082c = PMPI_Status_f082c
-#endif
-#define MPI_Status_f082c PMPI_Status_f082c
-#endif
-
-static const char FUNC_NAME[] = "MPI_Status_f082c";
-
-
-int MPI_Status_f082c(const MPI_F08_status  *f08_status, MPI_Status *c_status)
-{
-    int *c_ints;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        /* MPI-4:18.2.5 implies that if you pass in
-           MPI_STATUS[ES]_IGNORE, it's erroneous */
-
-        if (NULL == f08_status ||
-#if OMPI_BUILD_FORTRAN_BINDINGS
-            /* This section is #if'ed out if we are not building the
-               fortran bindings because these macros check values
-               against constants that only exist if the fortran
-               bindings exist. */
-            OMPI_IS_FORTRAN_STATUS_IGNORE(f08_status) ||
-            OMPI_IS_FORTRAN_STATUSES_IGNORE(f08_status) ||
-#endif
-            NULL == c_status) {
-            return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD,
-                                          MPI_ERR_IN_STATUS, FUNC_NAME);
-        }
-    }
-
-    /* ***NOTE*** See huge comment in status_c2f.c (yes, I know
-                  there's a size_t member in the C MPI_Status -- go
-                  read that comment for an explanation why copying
-                  everything as a bunch of int's is ok).
-
-       We can't use OMPI_FINT_2_INT here because of some complications
-       with include files.  :-( So just do the casting manually. */
-    c_status->MPI_SOURCE = (int)f08_status->MPI_SOURCE;
-    c_status->MPI_TAG = (int)f08_status->MPI_TAG;
-    c_status->MPI_ERROR = (int)f08_status->MPI_ERROR;
-    c_ints = (int *)c_status + 3;
-    for(int i=0; i < (int)(sizeof(MPI_Status) / sizeof(int) - 3); i++) {
-        c_ints[i] = (int)f08_status->internal[i];
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/status_f082c.c.in b/ompi/mpi/c/status_f082c.c.in
new file mode 100644
index 00000000000..dc7920a4e13
--- /dev/null
+++ b/ompi/mpi/c/status_f082c.c.in
@@ -0,0 +1,76 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2020 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+#include "ompi/mpi/fortran/base/constants.h"
+
+PROTOTYPE ERROR_CLASS status_f082c(F08_STATUS f08_status, STATUS_OUT c_status)
+{
+    int *c_ints;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        /* MPI-4:18.2.5 implies that if you pass in
+           MPI_STATUS[ES]_IGNORE, it's erroneous */
+
+        if (NULL == f08_status ||
+#if OMPI_BUILD_FORTRAN_BINDINGS
+            /* This section is #if'ed out if we are not building the
+               fortran bindings because these macros check values
+               against constants that only exist if the fortran
+               bindings exist. */
+            OMPI_IS_FORTRAN_STATUS_IGNORE(f08_status) ||
+            OMPI_IS_FORTRAN_STATUSES_IGNORE(f08_status) ||
+#endif
+            NULL == c_status) {
+            return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD,
+                                          MPI_ERR_IN_STATUS, FUNC_NAME);
+        }
+    }
+
+    /* ***NOTE*** See huge comment in status_c2f.c (yes, I know
+                  there's a size_t member in the C MPI_Status -- go
+                  read that comment for an explanation why copying
+                  everything as a bunch of int's is ok).
+
+       We can't use OMPI_FINT_2_INT here because of some complications
+       with include files.  :-( So just do the casting manually. */
+    c_status->MPI_SOURCE = (int)f08_status->MPI_SOURCE;
+    c_status->MPI_TAG = (int)f08_status->MPI_TAG;
+    c_status->MPI_ERROR = (int)f08_status->MPI_ERROR;
+    c_ints = (int *)c_status + 3;
+    for(int i=0; i < (int)(sizeof(MPI_Status) / sizeof(int) - 3); i++) {
+        c_ints[i] = (int)f08_status->internal[i];
+    }
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/status_f082f.c b/ompi/mpi/c/status_f082f.c
deleted file mode 100644
index 0abc1762b39..00000000000
--- a/ompi/mpi/c/status_f082f.c
+++ /dev/null
@@ -1,69 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2020 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mpi/fortran/base/fint_2_int.h"
-#include "ompi/mpi/fortran/base/constants.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Status_f082f = PMPI_Status_f082f
-#endif
-#define MPI_Status_f082f PMPI_Status_f082f
-#endif
-
-static const char FUNC_NAME[] = "MPI_Status_f082f";
-
-
-int MPI_Status_f082f(const MPI_F08_status  *f08_status, MPI_Fint *f_status)
-{
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        /* MPI-2:4.12.5 says that if you pass in
-           MPI_STATUS[ES]_IGNORE, it's erroneous */
-
-        if (NULL == f08_status ||
-#if OMPI_BUILD_FORTRAN_BINDINGS
-            /* This section is #if'ed out if we are not building the
-               fortran bindings because these macros check values
-               against constants that only exist if the fortran
-               bindings exist. */
-            OMPI_IS_FORTRAN_STATUS_IGNORE(f08_status) ||
-            OMPI_IS_FORTRAN_STATUSES_IGNORE(f08_status) ||
-#endif
-            NULL == f_status) {
-            return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD,
-                                          MPI_ERR_IN_STATUS, FUNC_NAME);
-        }
-    }
-
-    memcpy(f_status, f08_status, OMPI_FORTRAN_STATUS_SIZE * sizeof(MPI_Fint));
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/status_f082f.c.in b/ompi/mpi/c/status_f082f.c.in
new file mode 100644
index 00000000000..39ff75516c1
--- /dev/null
+++ b/ompi/mpi/c/status_f082f.c.in
@@ -0,0 +1,61 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2020 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+#include "ompi/mpi/fortran/base/constants.h"
+
+PROTOTYPE ERROR_CLASS status_f082f(F08_STATUS f08_status, FINT_OUT f_status)
+{
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        /* MPI-2:4.12.5 says that if you pass in
+           MPI_STATUS[ES]_IGNORE, it's erroneous */
+
+        if (NULL == f08_status ||
+#if OMPI_BUILD_FORTRAN_BINDINGS
+            /* This section is #if'ed out if we are not building the
+               fortran bindings because these macros check values
+               against constants that only exist if the fortran
+               bindings exist. */
+            OMPI_IS_FORTRAN_STATUS_IGNORE(f08_status) ||
+            OMPI_IS_FORTRAN_STATUSES_IGNORE(f08_status) ||
+#endif
+            NULL == f_status) {
+            return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD,
+                                          MPI_ERR_IN_STATUS, FUNC_NAME);
+        }
+    }
+
+    memcpy(f_status, f08_status, OMPI_FORTRAN_STATUS_SIZE * sizeof(MPI_Fint));
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/status_f2c.c b/ompi/mpi/c/status_f2c.c
deleted file mode 100644
index 32fa39b86e9..00000000000
--- a/ompi/mpi/c/status_f2c.c
+++ /dev/null
@@ -1,81 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mpi/fortran/base/fint_2_int.h"
-#include "ompi/mpi/fortran/base/constants.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Status_f2c = PMPI_Status_f2c
-#endif
-#define MPI_Status_f2c PMPI_Status_f2c
-#endif
-
-static const char FUNC_NAME[] = "MPI_Status_f2c";
-
-
-int MPI_Status_f2c(const MPI_Fint *f_status, MPI_Status *c_status)
-{
-    int i, *c_ints;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        /* MPI-2:4.12.5 says that if you pass in
-           MPI_STATUS[ES]_IGNORE, it's erroneous */
-
-        if (NULL == f_status ||
-#if OMPI_BUILD_FORTRAN_BINDINGS
-            /* This section is #if'ed out if we are not building the
-               fortran bindings because these macros check values
-               against constants that only exist if the fortran
-               bindings exist. */
-            OMPI_IS_FORTRAN_STATUS_IGNORE(f_status) ||
-            OMPI_IS_FORTRAN_STATUSES_IGNORE(f_status) ||
-#endif
-            NULL == c_status) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(
-                                          MPI_ERR_IN_STATUS, FUNC_NAME);
-        }
-    }
-
-    /* ***NOTE*** See huge comment in status_c2f.c (yes, I know
-                  there's a size_t member in the C MPI_Status -- go
-                  read that comment for an explanation why copying
-                  everything as a bunch of int's is ok).
-
-       We can't use OMPI_FINT_2_INT here because of some complications
-       with include files.  :-( So just do the casting manually. */
-    c_ints = (int*)c_status;
-    for( i = 0; i < (int)(sizeof(MPI_Status) / sizeof(int)); i++ ) {
-        c_ints[i] = (int)f_status[i];
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/status_f2c.c.in b/ompi/mpi/c/status_f2c.c.in
new file mode 100644
index 00000000000..3216193e3e9
--- /dev/null
+++ b/ompi/mpi/c/status_f2c.c.in
@@ -0,0 +1,73 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+#include "ompi/mpi/fortran/base/constants.h"
+
+PROTOTYPE ERROR_CLASS status_f2c(FINT_CONST f_status, STATUS_OUT c_status)
+{
+    int i, *c_ints;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        /* MPI-2:4.12.5 says that if you pass in
+           MPI_STATUS[ES]_IGNORE, it's erroneous */
+
+        if (NULL == f_status ||
+#if OMPI_BUILD_FORTRAN_BINDINGS
+            /* This section is #if'ed out if we are not building the
+               fortran bindings because these macros check values
+               against constants that only exist if the fortran
+               bindings exist. */
+            OMPI_IS_FORTRAN_STATUS_IGNORE(f_status) ||
+            OMPI_IS_FORTRAN_STATUSES_IGNORE(f_status) ||
+#endif
+            NULL == c_status) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(
+                                          MPI_ERR_IN_STATUS, FUNC_NAME);
+        }
+    }
+
+    /* ***NOTE*** See huge comment in status_c2f.c (yes, I know
+                  there's a size_t member in the C MPI_Status -- go
+                  read that comment for an explanation why copying
+                  everything as a bunch of int's is ok).
+
+       We can't use OMPI_FINT_2_INT here because of some complications
+       with include files.  :-( So just do the casting manually. */
+    c_ints = (int*)c_status;
+    for( i = 0; i < (int)(sizeof(MPI_Status) / sizeof(int)); i++ ) {
+        c_ints[i] = (int)f_status[i];
+    }
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/status_f2f08.c b/ompi/mpi/c/status_f2f08.c
deleted file mode 100644
index 3c7c31df312..00000000000
--- a/ompi/mpi/c/status_f2f08.c
+++ /dev/null
@@ -1,71 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2020 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mpi/fortran/base/fint_2_int.h"
-#include "ompi/mpi/fortran/base/constants.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Status_f2f08 = PMPI_Status_f2f08
-#endif
-#define MPI_Status_f2f08 PMPI_Status_f2f08
-#endif
-
-static const char FUNC_NAME[] = "MPI_Status_f2f08";
-
-
-int MPI_Status_f2f08(const MPI_Fint *f_status, MPI_F08_status *f08_status)
-{
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        /* MPI-2:4.12.5 says that if you pass in
-           MPI_STATUS[ES]_IGNORE, it's erroneous */
-
-        if (NULL == f_status ||
-#if OMPI_BUILD_FORTRAN_BINDINGS
-            /* This section is #if'ed out if we are not building the
-               fortran bindings because these macros check values
-               against constants that only exist if the fortran
-               bindings exist. */
-            OMPI_IS_FORTRAN_STATUS_IGNORE(f_status) ||
-            OMPI_IS_FORTRAN_STATUSES_IGNORE(f_status) ||
-#endif
-            NULL == f08_status) {
-            return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD,
-                                          MPI_ERR_IN_STATUS, FUNC_NAME);
-        }
-    }
-
-    memcpy(f08_status, f_status, OMPI_FORTRAN_STATUS_SIZE*sizeof(MPI_Fint));
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/status_f2f08.c.in b/ompi/mpi/c/status_f2f08.c.in
new file mode 100644
index 00000000000..5bc1972e958
--- /dev/null
+++ b/ompi/mpi/c/status_f2f08.c.in
@@ -0,0 +1,63 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2020 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+#include "ompi/mpi/fortran/base/constants.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS status_f2f08(FINT_CONST f_status, F08_STATUS_OUT f08_status)
+{
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        /* MPI-2:4.12.5 says that if you pass in
+           MPI_STATUS[ES]_IGNORE, it's erroneous */
+
+        if (NULL == f_status ||
+#if OMPI_BUILD_FORTRAN_BINDINGS
+            /* This section is #if'ed out if we are not building the
+               fortran bindings because these macros check values
+               against constants that only exist if the fortran
+               bindings exist. */
+            OMPI_IS_FORTRAN_STATUS_IGNORE(f_status) ||
+            OMPI_IS_FORTRAN_STATUSES_IGNORE(f_status) ||
+#endif
+            NULL == f08_status) {
+            return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD,
+                                          MPI_ERR_IN_STATUS, FUNC_NAME);
+        }
+    }
+
+    memcpy(f08_status, f_status, OMPI_FORTRAN_STATUS_SIZE*sizeof(MPI_Fint));
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/status_get_error.c.in b/ompi/mpi/c/status_get_error.c.in
new file mode 100644
index 00000000000..9f5e95e6c27
--- /dev/null
+++ b/ompi/mpi/c/status_get_error.c.in
@@ -0,0 +1,52 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2025      Advanced Micro Devices, Inc. All rights reserved.
+ * Copyright (c) 2025      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+#include <limits.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS status_get_error(STATUS status, INT_OUT error)
+{
+    int rc      = MPI_SUCCESS;
+
+    MEMCHECKER(
+               if (status != MPI_STATUSES_IGNORE) {
+                   /*
+                    * Before checking the complete status, we need to reset the definedness
+                    * of the MPI_ERROR-field (single-completion calls wait/test).
+                    */
+                   opal_memchecker_base_mem_defined((void*)&status->MPI_ERROR, sizeof(int));
+                   memchecker_status(status);
+               }
+               );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == status ||
+            MPI_STATUS_IGNORE == status ||
+            MPI_STATUSES_IGNORE == status) {
+            rc = MPI_ERR_ARG;
+        }
+        if (NULL == error) {
+            rc = MPI_ERR_ARG;
+        }
+
+        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
+    }
+
+    *error = status->MPI_ERROR;
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/status_get_source.c.in b/ompi/mpi/c/status_get_source.c.in
new file mode 100644
index 00000000000..a982b960cbc
--- /dev/null
+++ b/ompi/mpi/c/status_get_source.c.in
@@ -0,0 +1,52 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2025      Advanced Micro Devices, Inc. All rights reserved.
+ * Copyright (c) 2025      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+#include <limits.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS status_get_source(STATUS status, INT_OUT source)
+{
+    int rc      = MPI_SUCCESS;
+
+    MEMCHECKER(
+               if (status != MPI_STATUSES_IGNORE) {
+                   /*
+                    * Before checking the complete status, we need to reset the definedness
+                    * of the MPI_ERROR-field (single-completion calls wait/test).
+                    */
+                   opal_memchecker_base_mem_defined((void*)&status->MPI_ERROR, sizeof(int));
+                   memchecker_status(status);
+               }
+               );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == status ||
+            MPI_STATUS_IGNORE == status ||
+            MPI_STATUSES_IGNORE == status) {
+            rc = MPI_ERR_ARG;
+        }
+        if (NULL == source) {
+            rc = MPI_ERR_ARG;
+        }
+
+        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
+    }
+
+    *source = status->MPI_SOURCE;
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/status_get_tag.c.in b/ompi/mpi/c/status_get_tag.c.in
new file mode 100644
index 00000000000..b953cee1878
--- /dev/null
+++ b/ompi/mpi/c/status_get_tag.c.in
@@ -0,0 +1,54 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2025      Advanced Micro Devices, Inc. All rights reserved.
+ * Copyright (c) 2025      Triad National Security, LLC. All rights
+ *                         reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+#include <limits.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+
+
+PROTOTYPE ERROR_CLASS status_get_tag(STATUS status, INT_OUT tag)
+{
+    int rc      = MPI_SUCCESS;
+
+    MEMCHECKER(
+               if (status != MPI_STATUSES_IGNORE) {
+                   /*
+                    * Before checking the complete status, we need to reset the definedness
+                    * of the MPI_ERROR-field (single-completion calls wait/test).
+                    */
+                   opal_memchecker_base_mem_defined((void*)&status->MPI_ERROR, sizeof(int));
+                   memchecker_status(status);
+               }
+               );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == status ||
+            MPI_STATUS_IGNORE == status ||
+            MPI_STATUSES_IGNORE == status) {
+            rc = MPI_ERR_ARG;
+        }
+        if (NULL == tag) {
+            rc = MPI_ERR_ARG;
+        }
+
+        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
+    }
+
+    *tag = status->MPI_TAG;
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/status_set_cancelled.c b/ompi/mpi/c/status_set_cancelled.c
deleted file mode 100644
index 1c6cb835e9e..00000000000
--- a/ompi/mpi/c/status_set_cancelled.c
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Status_set_cancelled = PMPI_Status_set_cancelled
-#endif
-#define MPI_Status_set_cancelled PMPI_Status_set_cancelled
-#endif
-
-static const char FUNC_NAME[] = "MPI_Status_set_cancelled";
-
-
-int MPI_Status_set_cancelled(MPI_Status *status, int flag)
-{
-    MEMCHECKER(
-        if(status != MPI_STATUSES_IGNORE) {
-            /*
-             * Before checking the complete status, we need to reset the definedness
-             * of the MPI_ERROR-field (single-completion calls wait/test).
-             */
-            opal_memchecker_base_mem_defined(&status->MPI_ERROR, sizeof(int));
-            memchecker_status(status);
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        int rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == status ||
-            MPI_STATUS_IGNORE == status ||
-            MPI_STATUSES_IGNORE == status) {
-            rc = MPI_ERR_ARG;
-        }
-        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
-    }
-
-    status->_cancelled = flag;
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/status_set_cancelled.c.in b/ompi/mpi/c/status_set_cancelled.c.in
new file mode 100644
index 00000000000..3c47560527c
--- /dev/null
+++ b/ompi/mpi/c/status_set_cancelled.c.in
@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS status_set_cancelled(STATUS_OUT status, INT flag)
+{
+    MEMCHECKER(
+        if(status != MPI_STATUSES_IGNORE) {
+            /*
+             * Before checking the complete status, we need to reset the definedness
+             * of the MPI_ERROR-field (single-completion calls wait/test).
+             */
+            opal_memchecker_base_mem_defined(&status->MPI_ERROR, sizeof(int));
+            memchecker_status(status);
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        int rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == status ||
+            MPI_STATUS_IGNORE == status ||
+            MPI_STATUSES_IGNORE == status) {
+            rc = MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
+    }
+
+    status->_cancelled = flag;
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/status_set_elements.c b/ompi/mpi/c/status_set_elements.c
deleted file mode 100644
index 34c8888bc81..00000000000
--- a/ompi/mpi/c/status_set_elements.c
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Status_set_elements = PMPI_Status_set_elements
-#endif
-#define MPI_Status_set_elements PMPI_Status_set_elements
-#endif
-
-static const char FUNC_NAME[] = "MPI_Status_set_elements";
-
-int MPI_Status_set_elements(MPI_Status *status, MPI_Datatype datatype, int count)
-{
-    int rc = MPI_SUCCESS;
-    size_t size;
-
-    MEMCHECKER(
-        if(status != MPI_STATUSES_IGNORE) {
-            /*
-             * Before checking the complete status, we need to reset the definedness
-             * of the MPI_ERROR-field (single-completion calls wait/test).
-             */
-            opal_memchecker_base_mem_defined(&status->MPI_ERROR, sizeof(int));
-            memchecker_status (status);
-            memchecker_datatype(datatype);
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == datatype || MPI_DATATYPE_NULL == datatype) {
-            rc = MPI_ERR_TYPE;
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        }
-        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
-    }
-
-    /* ROMIO calls MPI_STATUS_SET_ELEMENTS with IGNORE values, so we
-       need to allow it.  Blah! */
-    if (MPI_STATUS_IGNORE == status || MPI_STATUSES_IGNORE == status) {
-        return MPI_SUCCESS;
-    }
-
-    if( ompi_datatype_is_predefined(datatype) ) {
-        ompi_datatype_type_size( datatype, &size );
-        status->_ucount = count * size;
-    } else {
-        ompi_datatype_set_element_count( datatype, count, &size );
-        status->_ucount = size;
-    }
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/status_set_elements.c.in b/ompi/mpi/c/status_set_elements.c.in
new file mode 100644
index 00000000000..953a7aef7d0
--- /dev/null
+++ b/ompi/mpi/c/status_set_elements.c.in
@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS status_set_elements(STATUS_OUT status, DATATYPE datatype, COUNT count)
+{
+    int rc = MPI_SUCCESS;
+    size_t size;
+
+    MEMCHECKER(
+        if(status != MPI_STATUSES_IGNORE) {
+            /*
+             * Before checking the complete status, we need to reset the definedness
+             * of the MPI_ERROR-field (single-completion calls wait/test).
+             */
+            opal_memchecker_base_mem_defined(&status->MPI_ERROR, sizeof(int));
+            memchecker_status (status);
+            memchecker_datatype(datatype);
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == datatype || MPI_DATATYPE_NULL == datatype) {
+            rc = MPI_ERR_TYPE;
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        }
+        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
+    }
+
+    /* ROMIO calls MPI_STATUS_SET_ELEMENTS with IGNORE values, so we
+       need to allow it.  Blah! */
+    if (MPI_STATUS_IGNORE == status || MPI_STATUSES_IGNORE == status) {
+        return MPI_SUCCESS;
+    }
+
+    if( ompi_datatype_is_predefined(datatype) ) {
+        ompi_datatype_type_size( datatype, &size );
+        status->_ucount = count * size;
+    } else {
+        ompi_datatype_set_element_count( datatype, count, &size );
+        status->_ucount = size;
+    }
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/status_set_elements_x.c b/ompi/mpi/c/status_set_elements_x.c
deleted file mode 100644
index 8d7d4656718..00000000000
--- a/ompi/mpi/c/status_set_elements_x.c
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Status_set_elements_x = PMPI_Status_set_elements_x
-#endif
-#define MPI_Status_set_elements_x PMPI_Status_set_elements_x
-#endif
-
-static const char FUNC_NAME[] = "MPI_Status_set_elements_x";
-
-int MPI_Status_set_elements_x(MPI_Status *status, MPI_Datatype datatype, MPI_Count count)
-{
-    int rc = MPI_SUCCESS;
-    size_t size;
-
-    MEMCHECKER(
-        if(status != MPI_STATUSES_IGNORE) {
-            /*
-             * Before checking the complete status, we need to reset the definedness
-             * of the MPI_ERROR-field (single-completion calls wait/test).
-             */
-            opal_memchecker_base_mem_defined(&status->MPI_ERROR, sizeof(int));
-            memchecker_status (status);
-            memchecker_datatype(datatype);
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == datatype || MPI_DATATYPE_NULL == datatype) {
-            rc = MPI_ERR_TYPE;
-        } else if (count < 0) {
-            rc = MPI_ERR_COUNT;
-        }
-        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
-    }
-
-    /* ROMIO calls MPI_STATUS_SET_ELEMENTS with IGNORE values, so we
-       need to allow it.  Blah! */
-    if (MPI_STATUS_IGNORE == status || MPI_STATUSES_IGNORE == status) {
-        return MPI_SUCCESS;
-    }
-
-    if( ompi_datatype_is_predefined(datatype) ) {
-        ompi_datatype_type_size( datatype, &size );
-        status->_ucount = count * size;
-    } else {
-        ompi_datatype_set_element_count( datatype, count, &size );
-        status->_ucount = size;
-    }
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/status_set_elements_x.c.in b/ompi/mpi/c/status_set_elements_x.c.in
new file mode 100644
index 00000000000..c24b24f9c55
--- /dev/null
+++ b/ompi/mpi/c/status_set_elements_x.c.in
@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS status_set_elements_x(STATUS_OUT status, DATATYPE datatype, PARTITIONED_COUNT count)
+{
+    int rc = MPI_SUCCESS;
+    size_t size;
+
+    MEMCHECKER(
+        if(status != MPI_STATUSES_IGNORE) {
+            /*
+             * Before checking the complete status, we need to reset the definedness
+             * of the MPI_ERROR-field (single-completion calls wait/test).
+             */
+            opal_memchecker_base_mem_defined(&status->MPI_ERROR, sizeof(int));
+            memchecker_status (status);
+            memchecker_datatype(datatype);
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == datatype || MPI_DATATYPE_NULL == datatype) {
+            rc = MPI_ERR_TYPE;
+        } else if (count < 0) {
+            rc = MPI_ERR_COUNT;
+        }
+        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
+    }
+
+    /* ROMIO calls MPI_STATUS_SET_ELEMENTS with IGNORE values, so we
+       need to allow it.  Blah! */
+    if (MPI_STATUS_IGNORE == status || MPI_STATUSES_IGNORE == status) {
+        return MPI_SUCCESS;
+    }
+
+    if( ompi_datatype_is_predefined(datatype) ) {
+        ompi_datatype_type_size( datatype, &size );
+        status->_ucount = count * size;
+    } else {
+        ompi_datatype_set_element_count( datatype, count, &size );
+        status->_ucount = size;
+    }
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/status_set_error.c.in b/ompi/mpi/c/status_set_error.c.in
new file mode 100644
index 00000000000..30a667cd5a5
--- /dev/null
+++ b/ompi/mpi/c/status_set_error.c.in
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2025      Advanced Micro Devices, Inc. All rights reserved.
+ * Copyright (c) 2025      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS status_set_error(STATUS_OUT status, INT error)
+{
+    MEMCHECKER(
+        if(status != MPI_STATUSES_IGNORE) {
+            /*
+             * Before checking the complete status, we need to reset the definedness
+             * of the MPI_ERROR-field (single-completion calls wait/test).
+             */
+            opal_memchecker_base_mem_defined(&status->MPI_ERROR, sizeof(int));
+            memchecker_status(status);
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        int rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == status ||
+            MPI_STATUS_IGNORE == status ||
+            MPI_STATUSES_IGNORE == status) {
+            rc = MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
+    }
+
+    status->MPI_ERROR = error;
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/status_set_source.c.in b/ompi/mpi/c/status_set_source.c.in
new file mode 100644
index 00000000000..46e1959bb85
--- /dev/null
+++ b/ompi/mpi/c/status_set_source.c.in
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2025      Advanced Micro Devices, Inc. All rights reserved.
+ * Copyright (c) 2025      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS  status_set_source(STATUS_OUT status, INT source)
+{
+    MEMCHECKER(
+        if(status != MPI_STATUSES_IGNORE) {
+            /*
+             * Before checking the complete status, we need to reset the definedness
+             * of the MPI_ERROR-field (single-completion calls wait/test).
+             */
+            opal_memchecker_base_mem_defined(&status->MPI_ERROR, sizeof(int));
+            memchecker_status(status);
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        int rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == status ||
+            MPI_STATUS_IGNORE == status ||
+            MPI_STATUSES_IGNORE == status) {
+            rc = MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
+    }
+
+    status->MPI_SOURCE = source;
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/status_set_tag.c.in b/ompi/mpi/c/status_set_tag.c.in
new file mode 100644
index 00000000000..2a85c3cb62f
--- /dev/null
+++ b/ompi/mpi/c/status_set_tag.c.in
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2025      Advanced Micro Devices, Inc. All rights reserved.
+ * Copyright (c) 2025      Triad National Security, LLC. All rights
+ *                         reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS status_set_tag(STATUS_OUT status, INT tag)
+{
+    MEMCHECKER(
+        if(status != MPI_STATUSES_IGNORE) {
+            /*
+             * Before checking the complete status, we need to reset the definedness
+             * of the MPI_ERROR-field (single-completion calls wait/test).
+             */
+            opal_memchecker_base_mem_defined(&status->MPI_ERROR, sizeof(int));
+            memchecker_status(status);
+        }
+    );
+
+    if (MPI_PARAM_CHECK) {
+        int rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == status ||
+            MPI_STATUS_IGNORE == status ||
+            MPI_STATUSES_IGNORE == status) {
+            rc = MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
+    }
+
+    status->MPI_TAG = tag;
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/test.c b/ompi/mpi/c/test.c
deleted file mode 100644
index 795843da21f..00000000000
--- a/ompi/mpi/c/test.c
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/request/request.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Test = PMPI_Test
-#endif
-#define MPI_Test PMPI_Test
-#endif
-
-static const char FUNC_NAME[] = "MPI_Test";
-
-
-int MPI_Test(MPI_Request *request, int *completed, MPI_Status *status)
-{
-    int rc;
-
-    SPC_RECORD(OMPI_SPC_TEST, 1);
-
-    MEMCHECKER(
-        memchecker_request (request);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (request == NULL) {
-            rc = MPI_ERR_REQUEST;
-        } else if (completed == NULL) {
-            rc = MPI_ERR_ARG;
-        }
-        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
-    }
-
-    rc = ompi_request_test(request, completed, status);
-    if (*completed < 0) {
-        *completed = 0;
-    }
-
-    MEMCHECKER(
-        opal_memchecker_base_mem_undefined(&status->MPI_ERROR, sizeof(int));
-    );
-
-    if (OMPI_SUCCESS == rc) {
-        return MPI_SUCCESS;
-    }
-    return ompi_errhandler_request_invoke(1, request, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/test.c.in b/ompi/mpi/c/test.c.in
new file mode 100644
index 00000000000..a44ac730458
--- /dev/null
+++ b/ompi/mpi/c/test.c.in
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/request/request.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS test(REQUEST_INOUT request, INT_OUT completed, STATUS_OUT status)
+{
+    int rc;
+
+    SPC_RECORD(OMPI_SPC_TEST, 1);
+
+    MEMCHECKER(
+        memchecker_request (request);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (request == NULL) {
+            rc = MPI_ERR_REQUEST;
+        } else if (completed == NULL) {
+            rc = MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
+    }
+
+    rc = ompi_request_test(request, completed, status);
+    if (*completed < 0) {
+        *completed = 0;
+    }
+
+    MEMCHECKER(
+        opal_memchecker_base_mem_undefined(&status->MPI_ERROR, sizeof(int));
+    );
+
+    if (OMPI_SUCCESS == rc) {
+        return MPI_SUCCESS;
+    }
+    return ompi_errhandler_request_invoke(1, request, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/test_cancelled.c b/ompi/mpi/c/test_cancelled.c
deleted file mode 100644
index bc13ade2b3b..00000000000
--- a/ompi/mpi/c/test_cancelled.c
+++ /dev/null
@@ -1,61 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Test_cancelled = PMPI_Test_cancelled
-#endif
-#define MPI_Test_cancelled PMPI_Test_cancelled
-#endif
-
-static const char FUNC_NAME[] = "MPI_Test_cancelled";
-
-
-int MPI_Test_cancelled(const MPI_Status *status, int *flag)
-{
-    int rc = MPI_SUCCESS;
-
-    MEMCHECKER (
-        if(status != MPI_STATUSES_IGNORE)
-            memchecker_status(status);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == flag || NULL == status) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-        }
-    }
-
-    *flag = status->_cancelled;
-    return rc;
-}
-
diff --git a/ompi/mpi/c/test_cancelled.c.in b/ompi/mpi/c/test_cancelled.c.in
new file mode 100644
index 00000000000..aa52f937d5f
--- /dev/null
+++ b/ompi/mpi/c/test_cancelled.c.in
@@ -0,0 +1,53 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS test_cancelled(STATUS status, INT_OUT flag)
+{
+    int rc = MPI_SUCCESS;
+
+    MEMCHECKER (
+        if(status != MPI_STATUSES_IGNORE)
+            memchecker_status(status);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == flag || NULL == status) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+        }
+    }
+
+    *flag = status->_cancelled;
+    return rc;
+}
+
diff --git a/ompi/mpi/c/testall.c b/ompi/mpi/c/testall.c
deleted file mode 100644
index 1f59fbe3377..00000000000
--- a/ompi/mpi/c/testall.c
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
- * Copyright (c) 2012      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/request/request.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Testall = PMPI_Testall
-#endif
-#define MPI_Testall PMPI_Testall
-#endif
-
-static const char FUNC_NAME[] = "MPI_Testall";
-
-
-int MPI_Testall(int count, MPI_Request requests[], int *flag,
-                MPI_Status statuses[])
-{
-    SPC_RECORD(OMPI_SPC_TESTALL, 1);
-
-    MEMCHECKER(
-        int j;
-        for (j = 0; j < count; j++){
-            memchecker_request(&requests[j]);
-        }
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        int i, rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if( (NULL == requests) && (0 != count) ) {
-            rc = MPI_ERR_REQUEST;
-        } else {
-            for (i = 0; i < count; i++) {
-                if (NULL == requests[i]) {
-                    rc = MPI_ERR_REQUEST;
-                    break;
-                }
-            }
-        }
-        if ((NULL == flag) || (count < 0)) {
-            rc = MPI_ERR_ARG;
-        }
-        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
-    }
-
-    if (OPAL_UNLIKELY(0 == count)) {
-        *flag = true;
-        return MPI_SUCCESS;
-    }
-
-    if (OMPI_SUCCESS == ompi_request_test_all(count, requests, flag,
-                                              statuses)) {
-        return MPI_SUCCESS;
-    }
-
-    if (MPI_SUCCESS !=
-        ompi_errhandler_request_invoke(count, requests, FUNC_NAME)) {
-        return MPI_ERR_IN_STATUS;
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/testall.c.in b/ompi/mpi/c/testall.c.in
new file mode 100644
index 00000000000..c1ba8e9e21e
--- /dev/null
+++ b/ompi/mpi/c/testall.c.in
@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2012      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2021-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/request/request.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS testall(INT count, REQUEST_INOUT requests, INT_OUT flag,
+                              STATUS_OUT statuses)
+{
+    SPC_RECORD(OMPI_SPC_TESTALL, 1);
+
+    MEMCHECKER(
+        int j;
+        for (j = 0; j < count; j++){
+            memchecker_request(&requests[j]);
+        }
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        int rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if( (NULL == requests) && (0 != count) ) {
+            rc = MPI_ERR_REQUEST;
+        } else {
+            if(!ompi_request_check_same_instance(requests, count) ) {
+                rc = MPI_ERR_REQUEST;
+            }
+        }
+        if ((NULL == flag) || (count < 0)) {
+            rc = MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
+    }
+
+    if (OPAL_UNLIKELY(0 == count)) {
+        *flag = true;
+        return MPI_SUCCESS;
+    }
+
+    if (OMPI_SUCCESS == ompi_request_test_all(count, requests, flag,
+                                              statuses)) {
+        return MPI_SUCCESS;
+    }
+
+    if (MPI_SUCCESS !=
+        ompi_errhandler_request_invoke(count, requests, FUNC_NAME)) {
+        return MPI_ERR_IN_STATUS;
+    }
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/testany.c b/ompi/mpi/c/testany.c
deleted file mode 100644
index 9740ce7b97e..00000000000
--- a/ompi/mpi/c/testany.c
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
- * Copyright (c) 2012      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2014-2015 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/request/request.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Testany = PMPI_Testany
-#endif
-#define MPI_Testany PMPI_Testany
-#endif
-
-static const char FUNC_NAME[] = "MPI_Testany";
-
-
-int MPI_Testany(int count, MPI_Request requests[], int *indx, int *completed, MPI_Status *status)
-{
-    SPC_RECORD(OMPI_SPC_TESTANY, 1);
-
-    MEMCHECKER(
-        int j;
-        for (j = 0; j < count; j++){
-            memchecker_request(&requests[j]);
-        }
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        int i, rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if ((NULL == requests) && (0 != count)) {
-            rc = MPI_ERR_REQUEST;
-        } else {
-            for (i = 0; i < count; i++) {
-                if (NULL == requests[i]) {
-                    rc = MPI_ERR_REQUEST;
-                    break;
-                }
-            }
-        }
-        if (((NULL == indx || NULL == completed) && count > 0) ||
-            count < 0) {
-            rc = MPI_ERR_ARG;
-        }
-        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
-    }
-
-    if (OPAL_UNLIKELY(0 == count)) {
-        *completed = true;
-        *indx = MPI_UNDEFINED;
-        if (MPI_STATUS_IGNORE != status) {
-            *status = ompi_status_empty;
-        }
-        return MPI_SUCCESS;
-    }
-
-    if (OMPI_SUCCESS == ompi_request_test_any(count, requests,
-                                              indx, completed, status)) {
-        return MPI_SUCCESS;
-    }
-
-    return ompi_errhandler_request_invoke(count, requests, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/testany.c.in b/ompi/mpi/c/testany.c.in
new file mode 100644
index 00000000000..50ed88b2fa5
--- /dev/null
+++ b/ompi/mpi/c/testany.c.in
@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2012      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2015 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2021-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/request/request.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS testany(INT count, REQUEST_INOUT requests, INT_OUT indx, INT_OUT completed, STATUS_OUT status)
+{
+    SPC_RECORD(OMPI_SPC_TESTANY, 1);
+
+    MEMCHECKER(
+        int j;
+        for (j = 0; j < count; j++){
+            memchecker_request(&requests[j]);
+        }
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        int rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if ((NULL == requests) && (0 != count)) {
+            rc = MPI_ERR_REQUEST;
+        } else {
+            if(!ompi_request_check_same_instance(requests, count) ) {
+                rc = MPI_ERR_REQUEST;
+            }
+        }
+        if (((NULL == indx || NULL == completed) && count > 0) ||
+            count < 0) {
+            rc = MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
+    }
+
+    if (OPAL_UNLIKELY(0 == count)) {
+        *completed = true;
+        *indx = MPI_UNDEFINED;
+        if (MPI_STATUS_IGNORE != status) {
+            OMPI_COPY_STATUS(status, ompi_status_empty, false);
+        }
+        return MPI_SUCCESS;
+    }
+
+    if (OMPI_SUCCESS == ompi_request_test_any(count, requests,
+                                              indx, completed, status)) {
+        return MPI_SUCCESS;
+    }
+
+    return ompi_errhandler_request_invoke(count, requests, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/testsome.c b/ompi/mpi/c/testsome.c
deleted file mode 100644
index 9f9782d505b..00000000000
--- a/ompi/mpi/c/testsome.c
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
- * Copyright (c) 2012      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/request/request.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Testsome = PMPI_Testsome
-#endif
-#define MPI_Testsome PMPI_Testsome
-#endif
-
-static const char FUNC_NAME[] = "MPI_Testsome";
-
-
-int MPI_Testsome(int incount, MPI_Request requests[],
-                 int *outcount, int indices[],
-                 MPI_Status statuses[])
-{
-    SPC_RECORD(OMPI_SPC_TESTSOME, 1);
-
-    MEMCHECKER(
-        int j;
-        for (j = 0; j < incount; j++){
-            memchecker_request(&requests[j]);
-        }
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        int indx, rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if ((NULL == requests) && (0 != incount)) {
-            rc = MPI_ERR_REQUEST;
-        } else {
-            for (indx = 0; indx < incount; ++indx) {
-                if (NULL == requests[indx]) {
-                    rc = MPI_ERR_REQUEST;
-                    break;
-                }
-            }
-        }
-        if (((NULL == outcount || NULL == indices) && incount > 0) ||
-            incount < 0) {
-            return MPI_ERR_ARG;
-        }
-        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
-    }
-
-    if (OPAL_UNLIKELY(0 == incount)) {
-        *outcount = MPI_UNDEFINED;
-        return OMPI_SUCCESS;
-    }
-
-    if (OMPI_SUCCESS == ompi_request_test_some(incount, requests, outcount,
-                                               indices, statuses)) {
-        return MPI_SUCCESS;
-    }
-
-    if (MPI_SUCCESS !=
-        ompi_errhandler_request_invoke(incount, requests, FUNC_NAME)) {
-        return MPI_ERR_IN_STATUS;
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/testsome.c.in b/ompi/mpi/c/testsome.c.in
new file mode 100644
index 00000000000..e5bccca0a28
--- /dev/null
+++ b/ompi/mpi/c/testsome.c.in
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2012      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2021-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/request/request.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS testsome(INT incount, REQUEST_INOUT requests,
+                               INT_OUT outcount, INT_OUT indices,
+                               STATUS_OUT statuses)
+{
+    SPC_RECORD(OMPI_SPC_TESTSOME, 1);
+
+    MEMCHECKER(
+        int j;
+        for (j = 0; j < incount; j++){
+            memchecker_request(&requests[j]);
+        }
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        int rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if ((NULL == requests) && (0 != incount)) {
+            rc = MPI_ERR_REQUEST;
+        } else {
+            if(!ompi_request_check_same_instance(requests, incount) ) {
+                rc = MPI_ERR_REQUEST;
+            }
+        }
+        if (((NULL == outcount || NULL == indices) && incount > 0) ||
+            incount < 0) {
+            return MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
+    }
+
+    if (OPAL_UNLIKELY(0 == incount)) {
+        *outcount = MPI_UNDEFINED;
+        return OMPI_SUCCESS;
+    }
+
+    if (OMPI_SUCCESS == ompi_request_test_some(incount, requests, outcount,
+                                               indices, statuses)) {
+        return MPI_SUCCESS;
+    }
+
+    if (MPI_SUCCESS !=
+        ompi_errhandler_request_invoke(incount, requests, FUNC_NAME)) {
+        return MPI_ERR_IN_STATUS;
+    }
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/topo_test.c b/ompi/mpi/c/topo_test.c
deleted file mode 100644
index 2fc9d82f25a..00000000000
--- a/ompi/mpi/c/topo_test.c
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2012-2013 Inria.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <stdio.h>
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/memchecker.h"
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Topo_test = PMPI_Topo_test
-#endif
-#define MPI_Topo_test PMPI_Topo_test
-#endif
-
-static const char FUNC_NAME[] = "MPI_Topo_test";
-
-
-int MPI_Topo_test(MPI_Comm comm, int *status)
-{
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid (comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if ( NULL == status ) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-    }
-
-    if (OMPI_COMM_IS_CART(comm)) {
-        *status = MPI_CART;
-    } else if (OMPI_COMM_IS_GRAPH(comm)) {
-        *status = MPI_GRAPH;
-    } else if (OMPI_COMM_IS_DIST_GRAPH(comm)) {
-        *status = MPI_DIST_GRAPH;
-    } else {
-        *status = MPI_UNDEFINED;
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/topo_test.c.in b/ompi/mpi/c/topo_test.c.in
new file mode 100644
index 00000000000..33c1d9e1fda
--- /dev/null
+++ b/ompi/mpi/c/topo_test.c.in
@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2012-2013 Inria.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include <stdio.h>
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS topo_test(COMM comm, INT_OUT status)
+{
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid (comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        } else if ( NULL == status ) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+    }
+
+    if (OMPI_COMM_IS_CART(comm)) {
+        *status = MPI_CART;
+    } else if (OMPI_COMM_IS_GRAPH(comm)) {
+        *status = MPI_GRAPH;
+    } else if (OMPI_COMM_IS_DIST_GRAPH(comm)) {
+        *status = MPI_DIST_GRAPH;
+    } else {
+        *status = MPI_UNDEFINED;
+    }
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/type_c2f.c b/ompi/mpi/c/type_c2f.c
deleted file mode 100644
index d660756f81b..00000000000
--- a/ompi/mpi/c/type_c2f.c
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mpi/fortran/base/fint_2_int.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_c2f = PMPI_Type_c2f
-#endif
-#define MPI_Type_c2f PMPI_Type_c2f
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_c2f";
-
-
-MPI_Fint MPI_Type_c2f(MPI_Datatype datatype)
-{
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (NULL == datatype) {
-            return OMPI_INT_2_FINT(-1);
-        }
-    }
-
-    /* If necessary add the datatype to the f2c translation table */
-    if( -1 == datatype->d_f_to_c_index ) {
-        datatype->d_f_to_c_index = opal_pointer_array_add(&ompi_datatype_f_to_c_table, datatype);
-        /* We don't check for error as returning a negative value is considered as an error */
-    }
-    return OMPI_INT_2_FINT(datatype->d_f_to_c_index);
-}
diff --git a/ompi/mpi/c/type_c2f.c.in b/ompi/mpi/c/type_c2f.c.in
new file mode 100644
index 00000000000..9ca8ec5faa7
--- /dev/null
+++ b/ompi/mpi/c/type_c2f.c.in
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2017 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE FINT type_c2f(DATATYPE datatype)
+{
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (NULL == datatype) {
+            return OMPI_INT_2_FINT(-1);
+        }
+    }
+
+    /* If necessary add the datatype to the f2c translation table */
+    if( -1 == datatype->d_f_to_c_index ) {
+        datatype->d_f_to_c_index = opal_pointer_array_add(&ompi_datatype_f_to_c_table, datatype);
+        /* We don't check for error as returning a negative value is considered as an error */
+    }
+    return OMPI_INT_2_FINT(datatype->d_f_to_c_index);
+}
diff --git a/ompi/mpi/c/type_commit.c b/ompi/mpi/c/type_commit.c
deleted file mode 100644
index d7ac77d87f6..00000000000
--- a/ompi/mpi/c/type_commit.c
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_commit = PMPI_Type_commit
-#endif
-#define MPI_Type_commit PMPI_Type_commit
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_commit";
-
-
-int MPI_Type_commit(MPI_Datatype *type)
-{
-    int rc;
-
-    MEMCHECKER(
-               memchecker_datatype(*type);
-               );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == type || NULL == *type || MPI_DATATYPE_NULL == *type) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE, FUNC_NAME);
-        }
-    }
-
-    rc = ompi_datatype_commit( type );
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME );
-}
diff --git a/ompi/mpi/c/type_commit.c.in b/ompi/mpi/c/type_commit.c.in
new file mode 100644
index 00000000000..cd67aa24501
--- /dev/null
+++ b/ompi/mpi/c/type_commit.c.in
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS type_commit(DATATYPE_OUT type)
+{
+    int rc;
+
+    MEMCHECKER(
+               memchecker_datatype(*type);
+               );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == type || NULL == *type || MPI_DATATYPE_NULL == *type) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE, FUNC_NAME);
+        }
+    }
+
+    rc = ompi_datatype_commit( type );
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME );
+}
diff --git a/ompi/mpi/c/type_contiguous.c b/ompi/mpi/c/type_contiguous.c
deleted file mode 100644
index 4bac82c79ee..00000000000
--- a/ompi/mpi/c/type_contiguous.c
+++ /dev/null
@@ -1,74 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_contiguous = PMPI_Type_contiguous
-#endif
-#define MPI_Type_contiguous PMPI_Type_contiguous
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_contiguous";
-
-
-int MPI_Type_contiguous(int count,
-                        MPI_Datatype oldtype,
-                        MPI_Datatype *newtype)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(oldtype);
-        );
-
-    if( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (MPI_DATATYPE_NULL == oldtype || NULL == oldtype ||
-            NULL == newtype) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE, FUNC_NAME);
-        } else if( count < 0 ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COUNT, FUNC_NAME);
-        }
-    }
-
-    rc = ompi_datatype_create_contiguous( count, oldtype, newtype );
-    OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME );
-
-    /* data description */
-    {
-        const int* a_i[1] = {&count};
-        ompi_datatype_set_args( *newtype, 1, a_i, 0, NULL, 1, &oldtype, MPI_COMBINER_CONTIGUOUS );
-    }
-
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME );
-}
diff --git a/ompi/mpi/c/type_contiguous.c.in b/ompi/mpi/c/type_contiguous.c.in
new file mode 100644
index 00000000000..cc88f3cab77
--- /dev/null
+++ b/ompi/mpi/c/type_contiguous.c.in
@@ -0,0 +1,82 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+/*
+ * TODO:BIGCOUNT this file will need to be updated once
+ * the datatype framework supports bigcount 
+ */
+
+
+PROTOTYPE ERROR_CLASS type_contiguous(COUNT count,
+                                      DATATYPE oldtype,
+                                      DATATYPE_OUT newtype)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(oldtype);
+        );
+
+    if( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (MPI_DATATYPE_NULL == oldtype || NULL == oldtype ||
+            NULL == newtype) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE, FUNC_NAME);
+        } else if( count < 0 ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COUNT, FUNC_NAME);
+        }
+#if OMPI_BIGCOUNT_SRC
+        OMPI_CHECK_MPI_COUNT_INT_CONVERSION_OVERFLOW(rc, count);
+        if (OMPI_SUCCESS != rc) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(rc, FUNC_NAME);
+        }
+#endif
+    }
+
+    rc = ompi_datatype_create_contiguous( count, oldtype, newtype );
+    OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME );
+
+    /* data description */
+    {
+#if OMPI_BIGCOUNT_SRC
+        int icount = (int)count;
+        const int* a_i[1] = {&icount};
+#else
+        const int* a_i[1] = {&count};
+#endif
+        ompi_datatype_set_args( *newtype, 1, a_i, 0, NULL, 1, &oldtype, MPI_COMBINER_CONTIGUOUS );
+    }
+
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME );
+}
diff --git a/ompi/mpi/c/type_create_darray.c b/ompi/mpi/c/type_create_darray.c
deleted file mode 100644
index 9dbe45d615a..00000000000
--- a/ompi/mpi/c/type_create_darray.c
+++ /dev/null
@@ -1,112 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2009      Sun Microsystems, Inc. All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_create_darray = PMPI_Type_create_darray
-#endif
-#define MPI_Type_create_darray PMPI_Type_create_darray
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_create_darray";
-
-int MPI_Type_create_darray(int size,
-                           int rank,
-                           int ndims,
-                           const int gsize_array[],
-                           const int distrib_array[],
-                           const int darg_array[],
-                           const int psize_array[],
-                           int order,
-                           MPI_Datatype oldtype,
-                           MPI_Datatype *newtype)
-
-{
-    int i, rc;
-
-    MEMCHECKER(
-        memchecker_datatype(oldtype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        int prod_psize = 1;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if( (rank < 0) || (size < 0) || (rank >= size) ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-        } else if( ndims < 0 ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COUNT, FUNC_NAME);
-        } else if( (ndims > 0) && ((NULL == gsize_array) || (NULL == distrib_array) ||
-                                   (NULL == darg_array) || (NULL == psize_array))) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-        } else if (NULL == newtype) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE, FUNC_NAME);
-        } else if( !(OPAL_DATATYPE_FLAG_DATA & oldtype->super.flags) ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE, FUNC_NAME);
-        } else if( (MPI_ORDER_C != order) && (MPI_ORDER_FORTRAN != order) ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-        }
-        if( ndims > 0 ) {
-            for( i = 0; i < ndims; i++ ) {
-                if( (MPI_DISTRIBUTE_BLOCK != distrib_array[i]) &&
-                    (MPI_DISTRIBUTE_CYCLIC != distrib_array[i]) &&
-                    (MPI_DISTRIBUTE_NONE != distrib_array[i]) ) {
-                    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-                } else if( (gsize_array[i] < 1) || (psize_array[i] < 0) ||
-                           ((darg_array[i] < 0) && (MPI_DISTRIBUTE_DFLT_DARG != darg_array[i]) ) ) {
-                    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-                } else if( (MPI_DISTRIBUTE_DFLT_DARG != darg_array[i]) &&
-                           (MPI_DISTRIBUTE_BLOCK == distrib_array[i]) &&
-                           ((darg_array[i] * psize_array[i]) < gsize_array[i]) ) {
-                    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-                } else if( 1 > psize_array[i] )
-                    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-                prod_psize *= psize_array[i];
-            }
-            if( prod_psize != size )
-                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-        }
-    }
-
-    rc = ompi_datatype_create_darray( size, rank, ndims,
-                                      gsize_array, distrib_array, darg_array, psize_array,
-                                      order, oldtype, newtype );
-    if( OMPI_SUCCESS == rc ) {
-        const int* a_i[8] = {&size, &rank, &ndims, gsize_array, distrib_array, darg_array,
-                             psize_array, &order};
-
-        ompi_datatype_set_args( *newtype, 4 * ndims + 4, a_i, 0, NULL, 1, &oldtype,
-                                MPI_COMBINER_DARRAY );
-    }
-
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/type_create_darray.c.in b/ompi/mpi/c/type_create_darray.c.in
new file mode 100644
index 00000000000..fcf00c8fe14
--- /dev/null
+++ b/ompi/mpi/c/type_create_darray.c.in
@@ -0,0 +1,133 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2009      Sun Microsystems, Inc. All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+/*
+ * TODO:BIGCOUNT this file will need to be updated once
+ * the datatype framework supports bigcount 
+ */
+
+
+PROTOTYPE ERROR_CLASS type_create_darray(INT size,
+                                         INT rank,
+                                         INT ndims,
+                                         COUNT_ARRAY gsize_array,
+                                         INT_ARRAY distrib_array,
+                                         INT_ARRAY darg_array,
+                                         INT_ARRAY psize_array,
+                                         INT order,
+                                         DATATYPE oldtype,
+                                         DATATYPE_OUT newtype)
+{
+    int i, rc;
+    int *igsize_array = NULL;
+
+    MEMCHECKER(
+        memchecker_datatype(oldtype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        int prod_psize = 1;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if( (rank < 0) || (size < 0) || (rank >= size) ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+        } else if( ndims < 0 ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COUNT, FUNC_NAME);
+        } else if( (ndims > 0) && ((NULL == gsize_array) || (NULL == distrib_array) ||
+                                   (NULL == darg_array) || (NULL == psize_array))) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+        } else if (NULL == newtype) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE, FUNC_NAME);
+        } else if( !(OPAL_DATATYPE_FLAG_DATA & oldtype->super.flags) ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE, FUNC_NAME);
+        } else if( (MPI_ORDER_C != order) && (MPI_ORDER_FORTRAN != order) ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+        }
+        if( ndims > 0 ) {
+#if OMPI_BIGCOUNT_SRC
+            for( i = 0; i < ndims; i++ ) {
+                OMPI_CHECK_MPI_COUNT_INT_CONVERSION_OVERFLOW(rc, gsize_array[i]);
+                if (OMPI_SUCCESS != rc) {
+                    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(rc, FUNC_NAME);
+                }
+            }
+#endif
+            for( i = 0; i < ndims; i++ ) {
+                if( (MPI_DISTRIBUTE_BLOCK != distrib_array[i]) &&
+                    (MPI_DISTRIBUTE_CYCLIC != distrib_array[i]) &&
+                    (MPI_DISTRIBUTE_NONE != distrib_array[i]) ) {
+                    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+                } else if( (gsize_array[i] < 1) || (psize_array[i] < 0) ||
+                           ((darg_array[i] < 0) && (MPI_DISTRIBUTE_DFLT_DARG != darg_array[i]) ) ) {
+                    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+                } else if( (MPI_DISTRIBUTE_DFLT_DARG != darg_array[i]) &&
+                           (MPI_DISTRIBUTE_BLOCK == distrib_array[i]) &&
+                           (((MPI_Count)darg_array[i] * (MPI_Count)psize_array[i]) < (MPI_Count)gsize_array[i]) ) {
+                    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+                } else if( 1 > psize_array[i] )
+                    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+                prod_psize *= psize_array[i];
+            }
+            if( prod_psize != size )
+                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+        }
+    }
+
+#if OMPI_BIGCOUNT_SRC
+    igsize_array = (int *)malloc(ndims * sizeof(int));
+    if (NULL == igsize_array) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_NO_MEM, FUNC_NAME);
+    }
+    for (int ii=0;ii<ndims;ii++) {
+        igsize_array[ii] = (int)gsize_array[ii];
+    }
+#else
+    igsize_array = (int *)gsize_array;
+#endif
+    rc = ompi_datatype_create_darray( size, rank, ndims,
+                                      igsize_array, distrib_array, darg_array, psize_array,
+                                      order, oldtype, newtype );
+    if( OMPI_SUCCESS == rc ) {
+        const int* a_i[8] = {&size, &rank, &ndims, igsize_array, distrib_array, darg_array,
+                             psize_array, &order};
+
+        ompi_datatype_set_args( *newtype, 4 * ndims + 4, a_i, 0, NULL, 1, &oldtype,
+                                MPI_COMBINER_DARRAY );
+    }
+#if OMPI_BIGCOUNT_SRC
+    free(igsize_array);
+#endif
+
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/type_create_f90_complex.c b/ompi/mpi/c/type_create_f90_complex.c
deleted file mode 100644
index 638afc2f262..00000000000
--- a/ompi/mpi/c/type_create_f90_complex.c
+++ /dev/null
@@ -1,135 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2009 Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2008-2018 Cisco Systems, Inc.  All rights reserved
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation.  All rights reserved.
- * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
- * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <float.h>
-
-#include "opal/util/printf.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_create_f90_complex = PMPI_Type_create_f90_complex
-#endif
-#define MPI_Type_create_f90_complex PMPI_Type_create_f90_complex
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_create_f90_complex";
-
-
-int MPI_Type_create_f90_complex(int p, int r, MPI_Datatype *newtype)
-{
-    uint64_t key;
-    int p_key, r_key;
-    int sflt_dig = 3, sflt_max_10_exp = +5, sflt_min_10_exp = -4;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        /* Note: These functions accept negative integers for the p and r
-         * arguments.  This is because for the SELECTED_COMPLEX_KIND,
-         * negative numbers are equivalent to zero values.  See section
-         * 13.14.95 of the Fortran 95 standard. */
-
-        if ((MPI_UNDEFINED == p && MPI_UNDEFINED == r)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-        }
-    }
-
-    /* if the user does not care about p or r set them to 0 so the
-     * test associate with them will always succeed.
-     */
-    p_key = p;
-    r_key = r;
-    if( MPI_UNDEFINED == p ) p_key = 0;
-    if( MPI_UNDEFINED == r ) r_key = 0;
-
-    /**
-     * With respect to the MPI standard, MPI-2.0 Sect. 10.2.5, MPI_TYPE_CREATE_F90_xxxx,
-     * page 295, line 47 we handle this nicely by caching the values in a hash table.
-     * However, as the value of might not always make sense, a little bit of optimization
-     * might be a good idea. Therefore, first we try to see if we can handle the value
-     * with some kind of default value, and if it's the case then we look into the
-     * cache.
-     */
-
-    if     ( (LDBL_DIG < p) || (LDBL_MAX_10_EXP < r) || (-LDBL_MIN_10_EXP < r) ) *newtype = &ompi_mpi_datatype_null.dt;
-    else if( (DBL_DIG  < p) || (DBL_MAX_10_EXP  < r) || (-DBL_MIN_10_EXP  < r) ) *newtype = &ompi_mpi_ldblcplex.dt;
-    else if( (FLT_DIG  < p) || (FLT_MAX_10_EXP  < r) || (-FLT_MIN_10_EXP  < r) ) *newtype = &ompi_mpi_dblcplex.dt;
-    else if( ! OMPI_HAVE_FORTRAN_COMPLEX4 ||
-             (sflt_dig < p) || (sflt_max_10_exp < r) || (-sflt_min_10_exp < r) ) *newtype = &ompi_mpi_cplex.dt;
-    else                                                                         *newtype = &ompi_mpi_complex4.dt;
-
-    if( *newtype != &ompi_mpi_datatype_null.dt ) {
-        ompi_datatype_t* datatype;
-        const int* a_i[2];
-        int rc;
-
-        key = (((uint64_t)p_key) << 32) | ((uint64_t)r_key);
-        if( OPAL_SUCCESS == opal_hash_table_get_value_uint64( &ompi_mpi_f90_complex_hashtable,
-                                                              key, (void**)newtype ) ) {
-            return MPI_SUCCESS;
-        }
-        /* Create the duplicate type corresponding to selected type, then
-         * set the argument to be a COMBINER with the correct value of r
-         * and add it to the hash table. */
-        if (OMPI_SUCCESS != ompi_datatype_duplicate( *newtype, &datatype)) {
-            OMPI_ERRHANDLER_RETURN (MPI_ERR_INTERN, MPI_COMM_WORLD,
-                                    MPI_ERR_INTERN, FUNC_NAME );
-        }
-        /* Make sure the user is not allowed to free this datatype as specified
-         * in the MPI standard.
-         */
-        datatype->super.flags |= OMPI_DATATYPE_FLAG_PREDEFINED;
-        /* Mark the datatype as a special F90 convenience type */
-        // Specifically using opal_snprintf() here (instead of
-        // snprintf()) so that over-eager compilers do not warn us
-        // that we may be truncating the output.  We *know* that the
-        // output may be truncated, and that's ok.
-        opal_snprintf(datatype->name, sizeof(datatype->name),
-                      "COMBINER %s", (*newtype)->name);
-
-        a_i[0] = &p;
-        a_i[1] = &r;
-        ompi_datatype_set_args( datatype, 2, a_i, 0, NULL, 0, NULL, MPI_COMBINER_F90_COMPLEX );
-
-        rc = opal_hash_table_set_value_uint64( &ompi_mpi_f90_complex_hashtable, key, datatype );
-        if (OMPI_SUCCESS != rc) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(rc, FUNC_NAME);
-        }
-        *newtype = datatype;
-        return MPI_SUCCESS;
-    }
-
-    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/type_create_f90_complex.c.in b/ompi/mpi/c/type_create_f90_complex.c.in
new file mode 100644
index 00000000000..b5b9713872b
--- /dev/null
+++ b/ompi/mpi/c/type_create_f90_complex.c.in
@@ -0,0 +1,123 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2009 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2008-2021 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include <float.h>
+
+#include "opal/util/printf.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+
+PROTOTYPE ERROR_CLASS Type_create_f90_complex(INT p, INT r, DATATYPE_OUT newtype)
+{
+    uint64_t key;
+    int p_key, r_key;
+    int sflt_dig = 3, sflt_max_10_exp = +5, sflt_min_10_exp = -4;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        /* Note: These functions accept negative integers for the p and r
+         * arguments.  This is because for the SELECTED_COMPLEX_KIND,
+         * negative numbers are equivalent to zero values.  See section
+         * 13.14.95 of the Fortran 95 standard. */
+
+        if ((MPI_UNDEFINED == p && MPI_UNDEFINED == r)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+        }
+    }
+
+    /* if the user does not care about p or r set them to 0 so the
+     * test associate with them will always succeed.
+     */
+    p_key = p;
+    r_key = r;
+    if( MPI_UNDEFINED == p ) p_key = 0;
+    if( MPI_UNDEFINED == r ) r_key = 0;
+
+    /**
+     * With respect to the MPI standard, MPI-2.0 Sect. 10.2.5, MPI_TYPE_CREATE_F90_xxxx,
+     * page 295, line 47 we handle this nicely by caching the values in a hash table.
+     * However, as the value of might not always make sense, a little bit of optimization
+     * might be a good idea. Therefore, first we try to see if we can handle the value
+     * with some kind of default value, and if it's the case then we look into the
+     * cache.
+     */
+
+    if     ( (LDBL_DIG < p) || (LDBL_MAX_10_EXP < r) || (-LDBL_MIN_10_EXP < r) ) *newtype = &ompi_mpi_datatype_null.dt;
+    else if( (DBL_DIG  < p) || (DBL_MAX_10_EXP  < r) || (-DBL_MIN_10_EXP  < r) ) *newtype = &ompi_mpi_ldblcplex.dt;
+    else if( (FLT_DIG  < p) || (FLT_MAX_10_EXP  < r) || (-FLT_MIN_10_EXP  < r) ) *newtype = &ompi_mpi_dblcplex.dt;
+    else if( ! OMPI_HAVE_FORTRAN_COMPLEX4 ||
+             (sflt_dig < p) || (sflt_max_10_exp < r) || (-sflt_min_10_exp < r) ) *newtype = &ompi_mpi_cplex.dt;
+    else                                                                         *newtype = &ompi_mpi_complex4.dt;
+
+    if( *newtype != &ompi_mpi_datatype_null.dt ) {
+        ompi_datatype_t* datatype;
+        const int* a_i[2];
+        int rc;
+
+        key = (((uint64_t)p_key) << 32) | ((uint64_t)r_key);
+        if( OPAL_SUCCESS == opal_hash_table_get_value_uint64( &ompi_mpi_f90_complex_hashtable,
+                                                              key, (void**)newtype ) ) {
+            return MPI_SUCCESS;
+        }
+        /* Create the duplicate type corresponding to selected type, then
+         * set the argument to be a COMBINER with the correct value of r
+         * and add it to the hash table. */
+        if (OMPI_SUCCESS != ompi_datatype_duplicate( *newtype, &datatype)) {
+            OMPI_ERRHANDLER_RETURN (MPI_ERR_INTERN, MPI_COMM_WORLD,
+                                    MPI_ERR_INTERN, FUNC_NAME );
+        }
+        /* Make sure the user is not allowed to free this datatype as specified
+         * in the MPI standard.
+         */
+        datatype->super.flags |= OMPI_DATATYPE_FLAG_PREDEFINED;
+        /* Mark the datatype as a special F90 convenience type */
+        snprintf(datatype->name, sizeof(datatype->name),
+                 "COMBINER %s", (*newtype)->name);
+
+        a_i[0] = &p;
+        a_i[1] = &r;
+        ompi_datatype_set_args( datatype, 2, a_i, 0, NULL, 0, NULL, MPI_COMBINER_F90_COMPLEX );
+
+        rc = opal_hash_table_set_value_uint64( &ompi_mpi_f90_complex_hashtable, key, datatype );
+        if (OMPI_SUCCESS != rc) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(rc, FUNC_NAME);
+        }
+        *newtype = datatype;
+        return MPI_SUCCESS;
+    }
+
+    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/type_create_f90_integer.c b/ompi/mpi/c/type_create_f90_integer.c
deleted file mode 100644
index cea7d11b394..00000000000
--- a/ompi/mpi/c/type_create_f90_integer.c
+++ /dev/null
@@ -1,126 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2009 Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2008-2018 Cisco Systems, Inc.  All rights reserved
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "opal/util/printf.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_create_f90_integer = PMPI_Type_create_f90_integer
-#endif
-#define MPI_Type_create_f90_integer PMPI_Type_create_f90_integer
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_create_f90_integer";
-
-
-int MPI_Type_create_f90_integer(int r, MPI_Datatype *newtype)
-
-{
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        /* Note: These functions accept negative integers for the p and r
-         * arguments.  This is because for the SELECTED_INTEGER_KIND,
-         * negative numbers are equivalent to zero values.  See section
-         * 13.14.95 of the Fortran 95 standard. */
-    }
-
-    /**
-     * With respect to the MPI standard, MPI-2.0 Sect. 10.2.5, MPI_TYPE_CREATE_F90_xxxx,
-     * page 295, line 47 we handle this nicely by caching the values in a hash table.
-     * However, as the value of might not always make sense, a little bit of optimization
-     * might be a good idea. Therefore, first we try to see if we can handle the value
-     * with some kind of default value, and if it's the case then we look into the
-     * cache.
-     */
-
-    if      (r > 38) *newtype = &ompi_mpi_datatype_null.dt;
-#if OMPI_HAVE_FORTRAN_INTEGER16
-    else if (r > 18) *newtype = &ompi_mpi_long_long_int.dt;
-#else
-    else if (r > 18) *newtype = &ompi_mpi_datatype_null.dt;
-#endif  /* OMPI_HAVE_F90_INTEGER16 */
-#if SIZEOF_LONG > SIZEOF_INT
-    else if (r >  9) *newtype = &ompi_mpi_long.dt;
-#else
-#if SIZEOF_LONG_LONG > SIZEOF_INT
-    else if (r >  9) *newtype = &ompi_mpi_long_long_int.dt;
-#else
-    else if (r >  9) *newtype = &ompi_mpi_datatype_null.dt;
-#endif  /* SIZEOF_LONG_LONG > SIZEOF_INT */
-#endif  /* SIZEOF_LONG > SIZEOF_INT */
-    else if (r >  4) *newtype = &ompi_mpi_int.dt;
-    else if (r >  2) *newtype = &ompi_mpi_short.dt;
-    else             *newtype = &ompi_mpi_byte.dt;
-
-    if( *newtype != &ompi_mpi_datatype_null.dt ) {
-        ompi_datatype_t* datatype;
-        const int* a_i[1];
-        int rc;
-
-        if( OPAL_SUCCESS == opal_hash_table_get_value_uint32( &ompi_mpi_f90_integer_hashtable,
-                                                              r, (void**)newtype ) ) {
-            return MPI_SUCCESS;
-        }
-        /* Create the duplicate type corresponding to selected type, then
-         * set the argument to be a COMBINER with the correct value of r
-         * and add it to the hash table. */
-        if (OMPI_SUCCESS != ompi_datatype_duplicate( *newtype, &datatype)) {
-            OMPI_ERRHANDLER_RETURN (MPI_ERR_INTERN, MPI_COMM_WORLD,
-                                    MPI_ERR_INTERN, FUNC_NAME );
-        }
-        /* Make sure the user is not allowed to free this datatype as specified
-         * in the MPI standard.
-         */
-        datatype->super.flags |= OMPI_DATATYPE_FLAG_PREDEFINED;
-        /* Mark the datatype as a special F90 convenience type */
-        // Specifically using opal_snprintf() here (instead of
-        // snprintf()) so that over-eager compilers do not warn us
-        // that we may be truncating the output.  We *know* that the
-        // output may be truncated, and that's ok.
-        opal_snprintf(datatype->name, sizeof(datatype->name),
-                      "COMBINER %s", (*newtype)->name);
-
-        a_i[0] = &r;
-        ompi_datatype_set_args( datatype, 1, a_i, 0, NULL, 0, NULL, MPI_COMBINER_F90_INTEGER );
-
-        rc = opal_hash_table_set_value_uint32( &ompi_mpi_f90_integer_hashtable, r, datatype );
-        if (OMPI_SUCCESS != rc) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(rc, FUNC_NAME);
-        }
-        *newtype = datatype;
-        return MPI_SUCCESS;
-    }
-
-    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/type_create_f90_integer.c.in b/ompi/mpi/c/type_create_f90_integer.c.in
new file mode 100644
index 00000000000..222794a9884
--- /dev/null
+++ b/ompi/mpi/c/type_create_f90_integer.c.in
@@ -0,0 +1,112 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2009 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2008-2021 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include "opal/util/printf.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+
+PROTOTYPE ERROR_CLASS type_create_f90_integer(INT r, DATATYPE_OUT newtype)
+{
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        /* Note: These functions accept negative integers for the p and r
+         * arguments.  This is because for the SELECTED_INTEGER_KIND,
+         * negative numbers are equivalent to zero values.  See section
+         * 13.14.95 of the Fortran 95 standard. */
+    }
+
+    /**
+     * With respect to the MPI standard, MPI-2.0 Sect. 10.2.5, MPI_TYPE_CREATE_F90_xxxx,
+     * page 295, line 47 we handle this nicely by caching the values in a hash table.
+     * However, as the value of might not always make sense, a little bit of optimization
+     * might be a good idea. Therefore, first we try to see if we can handle the value
+     * with some kind of default value, and if it's the case then we look into the
+     * cache.
+     */
+
+    if      (r > 38) *newtype = &ompi_mpi_datatype_null.dt;
+#if OMPI_HAVE_FORTRAN_INTEGER16
+    else if (r > 18) *newtype = &ompi_mpi_long_long_int.dt;
+#else
+    else if (r > 18) *newtype = &ompi_mpi_datatype_null.dt;
+#endif  /* OMPI_HAVE_F90_INTEGER16 */
+#if SIZEOF_LONG > SIZEOF_INT
+    else if (r >  9) *newtype = &ompi_mpi_long.dt;
+#else
+#if SIZEOF_LONG_LONG > SIZEOF_INT
+    else if (r >  9) *newtype = &ompi_mpi_long_long_int.dt;
+#else
+    else if (r >  9) *newtype = &ompi_mpi_datatype_null.dt;
+#endif  /* SIZEOF_LONG_LONG > SIZEOF_INT */
+#endif  /* SIZEOF_LONG > SIZEOF_INT */
+    else if (r >  4) *newtype = &ompi_mpi_int.dt;
+    else if (r >  2) *newtype = &ompi_mpi_short.dt;
+    else             *newtype = &ompi_mpi_byte.dt;
+
+    if( *newtype != &ompi_mpi_datatype_null.dt ) {
+        ompi_datatype_t* datatype;
+        const int* a_i[1];
+        int rc;
+
+        if( OPAL_SUCCESS == opal_hash_table_get_value_uint32( &ompi_mpi_f90_integer_hashtable,
+                                                              r, (void**)newtype ) ) {
+            return MPI_SUCCESS;
+        }
+        /* Create the duplicate type corresponding to selected type, then
+         * set the argument to be a COMBINER with the correct value of r
+         * and add it to the hash table. */
+        if (OMPI_SUCCESS != ompi_datatype_duplicate( *newtype, &datatype)) {
+            OMPI_ERRHANDLER_RETURN (MPI_ERR_INTERN, MPI_COMM_WORLD,
+                                    MPI_ERR_INTERN, FUNC_NAME );
+        }
+        /* Make sure the user is not allowed to free this datatype as specified
+         * in the MPI standard.
+         */
+        datatype->super.flags |= OMPI_DATATYPE_FLAG_PREDEFINED;
+        /* Mark the datatype as a special F90 convenience type */
+        snprintf(datatype->name, sizeof(datatype->name),
+                 "COMBINER %s", (*newtype)->name);
+
+        a_i[0] = &r;
+        ompi_datatype_set_args( datatype, 1, a_i, 0, NULL, 0, NULL, MPI_COMBINER_F90_INTEGER );
+
+        rc = opal_hash_table_set_value_uint32( &ompi_mpi_f90_integer_hashtable, r, datatype );
+        if (OMPI_SUCCESS != rc) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(rc, FUNC_NAME);
+        }
+        *newtype = datatype;
+        return MPI_SUCCESS;
+    }
+
+    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/type_create_f90_real.c b/ompi/mpi/c/type_create_f90_real.c
deleted file mode 100644
index 5518782cc22..00000000000
--- a/ompi/mpi/c/type_create_f90_real.c
+++ /dev/null
@@ -1,133 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2009 Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2008-2018 Cisco Systems, Inc.  All rights reserved
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation.  All rights reserved.
- * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
- * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <float.h>
-
-#include "opal/util/printf.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_create_f90_real = PMPI_Type_create_f90_real
-#endif
-#define MPI_Type_create_f90_real PMPI_Type_create_f90_real
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_create_f90_real";
-
-
-int MPI_Type_create_f90_real(int p, int r, MPI_Datatype *newtype)
-{
-    uint64_t key;
-    int p_key, r_key;
-    int sflt_dig = 3, sflt_max_10_exp = +5, sflt_min_10_exp = -4;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        /* Note: These functions accept negative integers for the p and r
-         * arguments.  This is because for the SELECTED_REAL_KIND,
-         * negative numbers are equivalent to zero values.  See section
-         * 13.14.95 of the Fortran 95 standard. */
-
-        if ((MPI_UNDEFINED == p && MPI_UNDEFINED == r)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-        }
-    }
-
-    /* if the user does not care about p or r set them to 0 so the
-     * test associate with them will always succeed.
-     */
-    p_key = p;
-    r_key = r;
-    if( MPI_UNDEFINED == p ) p_key = 0;
-    if( MPI_UNDEFINED == r ) r_key = 0;
-
-    /**
-     * With respect to the MPI standard, MPI-2.0 Sect. 10.2.5, MPI_TYPE_CREATE_F90_xxxx,
-     * page 295, line 47 we handle this nicely by caching the values in a hash table.
-     * However, as the value of might not always make sense, a little bit of optimization
-     * might be a good idea. Therefore, first we try to see if we can handle the value
-     * with some kind of default value, and if it's the case then we look into the
-     * cache.
-     */
-
-    if     ( (LDBL_DIG < p) || (LDBL_MAX_10_EXP < r) || (-LDBL_MIN_10_EXP < r) ) *newtype = &ompi_mpi_datatype_null.dt;
-    else if( (DBL_DIG  < p) || (DBL_MAX_10_EXP  < r) || (-DBL_MIN_10_EXP  < r) ) *newtype = &ompi_mpi_long_double.dt;
-    else if( (FLT_DIG  < p) || (FLT_MAX_10_EXP  < r) || (-FLT_MIN_10_EXP  < r) ) *newtype = &ompi_mpi_double.dt;
-    else if( ! OMPI_HAVE_FORTRAN_REAL2 ||
-             (sflt_dig < p) || (sflt_max_10_exp < r) || (-sflt_min_10_exp < r) ) *newtype = &ompi_mpi_float.dt;
-    else                                                                         *newtype = &ompi_mpi_real2.dt;
-
-    if( *newtype != &ompi_mpi_datatype_null.dt ) {
-        ompi_datatype_t* datatype;
-        const int* a_i[2] = {&p, &r};
-        int rc;
-
-        key = (((uint64_t)p_key) << 32) | ((uint64_t)r_key);
-        if( OPAL_SUCCESS == opal_hash_table_get_value_uint64( &ompi_mpi_f90_real_hashtable,
-                                                              key, (void**)newtype ) ) {
-            return MPI_SUCCESS;
-        }
-        /* Create the duplicate type corresponding to selected type, then
-         * set the argument to be a COMBINER with the correct value of r
-         * and add it to the hash table. */
-        if (OMPI_SUCCESS != ompi_datatype_duplicate( *newtype, &datatype)) {
-            OMPI_ERRHANDLER_RETURN (MPI_ERR_INTERN, MPI_COMM_WORLD,
-                                    MPI_ERR_INTERN, FUNC_NAME );
-        }
-        /* Make sure the user is not allowed to free this datatype as specified
-         * in the MPI standard.
-         */
-        datatype->super.flags |= OMPI_DATATYPE_FLAG_PREDEFINED;
-        /* Mark the datatype as a special F90 convenience type */
-        // Specifically using opal_snprintf() here (instead of
-        // snprintf()) so that over-eager compilers do not warn us
-        // that we may be truncating the output.  We *know* that the
-        // output may be truncated, and it's ok.
-        opal_snprintf(datatype->name, sizeof(datatype->name),
-                      "COMBINER %s", (*newtype)->name);
-
-        ompi_datatype_set_args( datatype, 2, a_i, 0, NULL, 0, NULL, MPI_COMBINER_F90_REAL );
-
-        rc = opal_hash_table_set_value_uint64( &ompi_mpi_f90_real_hashtable, key, datatype );
-        if (OMPI_SUCCESS != rc) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(rc, FUNC_NAME);
-        }
-        *newtype = datatype;
-        return MPI_SUCCESS;
-    }
-
-    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/type_create_f90_real.c.in b/ompi/mpi/c/type_create_f90_real.c.in
new file mode 100644
index 00000000000..e7d2e28bde9
--- /dev/null
+++ b/ompi/mpi/c/type_create_f90_real.c.in
@@ -0,0 +1,121 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2009 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2008-2021 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include <float.h>
+
+#include "opal/util/printf.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+
+PROTOTYPE ERROR_CLASS type_create_f90_real(INT p, INT r, DATATYPE_OUT newtype)
+{
+    uint64_t key;
+    int p_key, r_key;
+    int sflt_dig = 3, sflt_max_10_exp = +5, sflt_min_10_exp = -4;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        /* Note: These functions accept negative integers for the p and r
+         * arguments.  This is because for the SELECTED_REAL_KIND,
+         * negative numbers are equivalent to zero values.  See section
+         * 13.14.95 of the Fortran 95 standard. */
+
+        if ((MPI_UNDEFINED == p && MPI_UNDEFINED == r)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+        }
+    }
+
+    /* if the user does not care about p or r set them to 0 so the
+     * test associate with them will always succeed.
+     */
+    p_key = p;
+    r_key = r;
+    if( MPI_UNDEFINED == p ) p_key = 0;
+    if( MPI_UNDEFINED == r ) r_key = 0;
+
+    /**
+     * With respect to the MPI standard, MPI-2.0 Sect. 10.2.5, MPI_TYPE_CREATE_F90_xxxx,
+     * page 295, line 47 we handle this nicely by caching the values in a hash table.
+     * However, as the value of might not always make sense, a little bit of optimization
+     * might be a good idea. Therefore, first we try to see if we can handle the value
+     * with some kind of default value, and if it's the case then we look into the
+     * cache.
+     */
+
+    if     ( (LDBL_DIG < p) || (LDBL_MAX_10_EXP < r) || (-LDBL_MIN_10_EXP < r) ) *newtype = &ompi_mpi_datatype_null.dt;
+    else if( (DBL_DIG  < p) || (DBL_MAX_10_EXP  < r) || (-DBL_MIN_10_EXP  < r) ) *newtype = &ompi_mpi_long_double.dt;
+    else if( (FLT_DIG  < p) || (FLT_MAX_10_EXP  < r) || (-FLT_MIN_10_EXP  < r) ) *newtype = &ompi_mpi_double.dt;
+    else if( ! OMPI_HAVE_FORTRAN_REAL2 ||
+             (sflt_dig < p) || (sflt_max_10_exp < r) || (-sflt_min_10_exp < r) ) *newtype = &ompi_mpi_float.dt;
+    else                                                                         *newtype = &ompi_mpi_real2.dt;
+
+    if( *newtype != &ompi_mpi_datatype_null.dt ) {
+        ompi_datatype_t* datatype;
+        const int* a_i[2] = {&p, &r};
+        int rc;
+
+        key = (((uint64_t)p_key) << 32) | ((uint64_t)r_key);
+        if( OPAL_SUCCESS == opal_hash_table_get_value_uint64( &ompi_mpi_f90_real_hashtable,
+                                                              key, (void**)newtype ) ) {
+            return MPI_SUCCESS;
+        }
+        /* Create the duplicate type corresponding to selected type, then
+         * set the argument to be a COMBINER with the correct value of r
+         * and add it to the hash table. */
+        if (OMPI_SUCCESS != ompi_datatype_duplicate( *newtype, &datatype)) {
+            OMPI_ERRHANDLER_RETURN (MPI_ERR_INTERN, MPI_COMM_WORLD,
+                                    MPI_ERR_INTERN, FUNC_NAME );
+        }
+        /* Make sure the user is not allowed to free this datatype as specified
+         * in the MPI standard.
+         */
+        datatype->super.flags |= OMPI_DATATYPE_FLAG_PREDEFINED;
+        /* Mark the datatype as a special F90 convenience type */
+        snprintf(datatype->name, sizeof(datatype->name),
+                 "COMBINER %s", (*newtype)->name);
+
+        ompi_datatype_set_args( datatype, 2, a_i, 0, NULL, 0, NULL, MPI_COMBINER_F90_REAL );
+
+        rc = opal_hash_table_set_value_uint64( &ompi_mpi_f90_real_hashtable, key, datatype );
+        if (OMPI_SUCCESS != rc) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(rc, FUNC_NAME);
+        }
+        *newtype = datatype;
+        return MPI_SUCCESS;
+    }
+
+    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/type_create_hindexed.c b/ompi/mpi/c/type_create_hindexed.c
deleted file mode 100644
index 1de9d6d9cef..00000000000
--- a/ompi/mpi/c/type_create_hindexed.c
+++ /dev/null
@@ -1,92 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_create_hindexed = PMPI_Type_create_hindexed
-#endif
-#define MPI_Type_create_hindexed PMPI_Type_create_hindexed
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_create_hindexed";
-
-
-int MPI_Type_create_hindexed(int count,
-                             const int array_of_blocklengths[],
-                             const MPI_Aint array_of_displacements[],
-                             MPI_Datatype oldtype,
-                             MPI_Datatype *newtype)
-{
-    int rc, i;
-
-    MEMCHECKER(
-        memchecker_datatype(oldtype);
-        );
-
-    if( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if( count < 0 ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COUNT,
-                                          FUNC_NAME);
-        } else if ((count > 0) && (NULL == array_of_blocklengths ||
-                                   NULL == array_of_displacements)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        } else if (MPI_DATATYPE_NULL == oldtype || NULL == oldtype ||
-                   NULL == newtype) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
-                                          FUNC_NAME);
-        }
-        for ( i = 0; i < count; i++ ) {
-            if (array_of_blocklengths[i] < 0) {
-                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                              FUNC_NAME );
-            }
-        }
-    }
-
-    rc = ompi_datatype_create_hindexed( count, array_of_blocklengths, array_of_displacements,
-                                        oldtype, newtype );
-    if( rc != MPI_SUCCESS ) {
-        ompi_datatype_destroy( newtype );
-        OMPI_ERRHANDLER_NOHANDLE_RETURN( rc, rc, FUNC_NAME );
-    }
-    /* data description */
-    {
-        const int* a_i[2] = {&count, array_of_blocklengths};
-
-        ompi_datatype_set_args( *newtype, count + 1, a_i, count, array_of_displacements,
-                                1, &oldtype, MPI_COMBINER_HINDEXED );
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/type_create_hindexed.c.in b/ompi/mpi/c/type_create_hindexed.c.in
new file mode 100644
index 00000000000..79e03f91bb2
--- /dev/null
+++ b/ompi/mpi/c/type_create_hindexed.c.in
@@ -0,0 +1,122 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+
+/*
+ * TODO:BIGCOUNT this file will need to be updated once
+ * the datatype framework supports bigcount 
+ */
+
+
+PROTOTYPE ERROR_CLASS type_create_hindexed(COUNT count,
+                                           COUNT_ARRAY array_of_blocklengths,
+                                           AINT_COUNT_ARRAY array_of_displacements,
+                                           DATATYPE oldtype,
+                                           DATATYPE_OUT newtype)
+{
+    int rc, i;
+    int *iarray_of_blocklengths = NULL;
+    MPI_Aint *iarray_of_displacements = NULL;
+
+    MEMCHECKER(
+        memchecker_datatype(oldtype);
+        );
+
+    if( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if( count < 0 ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COUNT,
+                                          FUNC_NAME);
+        } else if ((count > 0) && (NULL == array_of_blocklengths ||
+                                   NULL == array_of_displacements)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        } else if (MPI_DATATYPE_NULL == oldtype || NULL == oldtype ||
+                   NULL == newtype) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
+                                          FUNC_NAME);
+        }
+        for ( i = 0; i < count; i++ ) {
+            if (array_of_blocklengths[i] < 0) {
+                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                              FUNC_NAME );
+            }
+        }
+#if OMPI_BIGCOUNT_SRC
+        OMPI_CHECK_MPI_COUNT_INT_CONVERSION_OVERFLOW(rc, count);
+        if (OMPI_SUCCESS != rc) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(rc, FUNC_NAME);
+        }
+#endif
+    }
+
+#if OMPI_BIGCOUNT_SRC
+    iarray_of_blocklengths = (int *)malloc(count * sizeof(int));
+    if (NULL == iarray_of_blocklengths) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_NO_MEM, FUNC_NAME);
+    }
+    iarray_of_displacements = (MPI_Aint *)malloc(count * sizeof(MPI_Aint));
+    if (NULL == iarray_of_displacements) {
+        free( iarray_of_blocklengths);
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_NO_MEM, FUNC_NAME);
+    }
+    for (i = 0; i < (int)count; i++) {
+        iarray_of_blocklengths[i] = (int)array_of_blocklengths[i];
+        iarray_of_displacements[i] = (MPI_Aint)array_of_displacements[i];
+    }
+#else
+    iarray_of_blocklengths = (int *)array_of_blocklengths;
+    iarray_of_displacements = (MPI_Aint *)array_of_displacements;
+#endif
+
+    rc = ompi_datatype_create_hindexed( count, iarray_of_blocklengths, iarray_of_displacements,
+                                        oldtype, newtype );
+    if( rc != MPI_SUCCESS ) {
+        ompi_datatype_destroy( newtype );
+        OMPI_ERRHANDLER_NOHANDLE_RETURN( rc, rc, FUNC_NAME );
+    }
+    /* data description */
+    {
+        const int* a_i[2] = {(int *)&count, iarray_of_blocklengths};
+
+        ompi_datatype_set_args( *newtype, count + 1, a_i, count, iarray_of_displacements,
+                                1, &oldtype, MPI_COMBINER_HINDEXED );
+    }
+
+#if OMPI_BIGCOUNT_SRC
+    free(iarray_of_blocklengths);
+    free(iarray_of_displacements);
+#endif
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/type_create_hindexed_block.c b/ompi/mpi/c/type_create_hindexed_block.c
deleted file mode 100644
index 43ecaeaef5c..00000000000
--- a/ompi/mpi/c/type_create_hindexed_block.c
+++ /dev/null
@@ -1,76 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2012-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_create_hindexed_block = PMPI_Type_create_hindexed_block
-#endif
-#define MPI_Type_create_hindexed_block PMPI_Type_create_hindexed_block
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_create_hindexed_block";
-
-
-int MPI_Type_create_hindexed_block(int count,
-                                   int blocklength,
-                                   const MPI_Aint array_of_displacements[],
-                                   MPI_Datatype oldtype,
-                                   MPI_Datatype *newtype)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(oldtype);
-        );
-
-    if( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if( count < 0 ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COUNT,
-                                          FUNC_NAME);
-        } else if( (count > 0) && (blocklength < 0 || NULL == array_of_displacements) ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME );
-        } else if (NULL == oldtype || MPI_DATATYPE_NULL == oldtype ||
-                   NULL == newtype) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
-                                          FUNC_NAME );
-        }
-    }
-
-    rc = ompi_datatype_create_hindexed_block( count, blocklength, array_of_displacements,
-                                              oldtype, newtype );
-    if( rc != MPI_SUCCESS ) {
-        ompi_datatype_destroy( newtype );
-        OMPI_ERRHANDLER_NOHANDLE_RETURN( rc, rc, FUNC_NAME );
-    }
-    {
-        const int* a_i[2] = {&count, &blocklength};
-        ompi_datatype_set_args( *newtype, 2, a_i, count, array_of_displacements, 1, &oldtype,
-                                MPI_COMBINER_HINDEXED_BLOCK );
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/type_create_hindexed_block.c.in b/ompi/mpi/c/type_create_hindexed_block.c.in
new file mode 100644
index 00000000000..f7bbadd3631
--- /dev/null
+++ b/ompi/mpi/c/type_create_hindexed_block.c.in
@@ -0,0 +1,95 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2012-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+/*
+ * TODO:BIGCOUNT this file will need to be updated once
+ * the datatype framework supports bigcount 
+ */
+
+PROTOTYPE ERROR_CLASS type_create_hindexed_block(COUNT count,
+                                                 COUNT blocklength,
+                                                 AINT_COUNT_ARRAY array_of_displacements,
+                                                 DATATYPE oldtype,
+                                                 DATATYPE_OUT newtype)
+{
+    int rc;
+    MPI_Aint *iarray_of_displacements = NULL;
+
+    MEMCHECKER(
+        memchecker_datatype(oldtype);
+        );
+
+    if( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if( count < 0 ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COUNT,
+                                          FUNC_NAME);
+        } else if( (count > 0) && (blocklength < 0 || NULL == array_of_displacements) ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME );
+        } else if (NULL == oldtype || MPI_DATATYPE_NULL == oldtype ||
+                   NULL == newtype) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
+                                          FUNC_NAME );
+        }
+#if OMPI_BIGCOUNT_SRC
+        OMPI_CHECK_MPI_COUNT_INT_CONVERSION_OVERFLOW(rc, count);
+        if (OMPI_SUCCESS != rc) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(rc, FUNC_NAME);
+        }
+#endif
+    }
+
+#if OMPI_BIGCOUNT_SRC
+    iarray_of_displacements = (MPI_Aint *)malloc(count * sizeof(MPI_Aint));
+    if (NULL == iarray_of_displacements) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_NO_MEM, FUNC_NAME);
+    }
+    for (int ii = 0; ii < (int)count; ii++) {
+        iarray_of_displacements[ii] = (MPI_Aint)array_of_displacements[ii];
+    }
+#else
+    iarray_of_displacements = (MPI_Aint *)array_of_displacements;
+#endif
+
+    rc = ompi_datatype_create_hindexed_block( count, blocklength, iarray_of_displacements,
+                                              oldtype, newtype );
+    if( rc != MPI_SUCCESS ) {
+        ompi_datatype_destroy( newtype );
+        OMPI_ERRHANDLER_NOHANDLE_RETURN( rc, rc, FUNC_NAME );
+    }
+    {
+        const int* a_i[2] = {(int *)&count, (int *)&blocklength};
+        ompi_datatype_set_args( *newtype, 2, a_i, count, iarray_of_displacements, 1, &oldtype,
+                                MPI_COMBINER_HINDEXED_BLOCK );
+    }
+#if OMPI_BIGCOUNT_SRC
+    free(iarray_of_displacements);
+#endif
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/type_create_hvector.c b/ompi/mpi/c/type_create_hvector.c
deleted file mode 100644
index 6a7b259842c..00000000000
--- a/ompi/mpi/c/type_create_hvector.c
+++ /dev/null
@@ -1,82 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_create_hvector = PMPI_Type_create_hvector
-#endif
-#define MPI_Type_create_hvector PMPI_Type_create_hvector
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_create_hvector";
-
-
-int MPI_Type_create_hvector(int count,
-                            int blocklength,
-                            MPI_Aint stride,
-                            MPI_Datatype oldtype,
-                            MPI_Datatype *newtype)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(oldtype);
-        );
-
-    if( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if( count < 0 ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COUNT,
-                                          FUNC_NAME );
-        } else if( blocklength < 0) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME );
-        } else if (NULL == oldtype || MPI_DATATYPE_NULL == oldtype ||
-                   NULL == newtype) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
-                                          FUNC_NAME );
-        }
-    }
-
-    rc = ompi_datatype_create_hvector ( count, blocklength, stride, oldtype,
-                                        newtype );
-    OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME );
-
-    {
-        const int* a_i[2] = {&count, &blocklength};
-        MPI_Aint a_a[1] = {stride};
-
-        ompi_datatype_set_args( *newtype, 2, a_i, 1, a_a, 1, &oldtype, MPI_COMBINER_HVECTOR );
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/type_create_hvector.c.in b/ompi/mpi/c/type_create_hvector.c.in
new file mode 100644
index 00000000000..cb93050d05c
--- /dev/null
+++ b/ompi/mpi/c/type_create_hvector.c.in
@@ -0,0 +1,85 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+/*
+ * TODO:BIGCOUNT this file will need to be updated once
+ * the datatype framework supports bigcount 
+ */
+
+PROTOTYPE ERROR_CLASS type_create_hvector(COUNT count,
+                                          COUNT blocklength,
+                                          AINT_COUNT stride,
+                                          DATATYPE oldtype,
+                                          DATATYPE_OUT newtype)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(oldtype);
+        );
+
+    if( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if( count < 0 ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COUNT,
+                                          FUNC_NAME );
+        } else if( blocklength < 0) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME );
+        } else if (NULL == oldtype || MPI_DATATYPE_NULL == oldtype ||
+                   NULL == newtype) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
+                                          FUNC_NAME );
+        }
+#if OMPI_BIGCOUNT_SRC
+        OMPI_CHECK_MPI_COUNT_INT_CONVERSION_OVERFLOW(rc, count);
+        if (OMPI_SUCCESS != rc) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(rc, FUNC_NAME);
+        }
+#endif
+    }
+
+    rc = ompi_datatype_create_hvector ( count, blocklength, stride, oldtype,
+                                        newtype );
+    OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME );
+
+    {
+        const int* a_i[2] = {(int *)&count, (int *)&blocklength};
+        MPI_Aint a_a[1] = {stride};
+
+        ompi_datatype_set_args( *newtype, 2, a_i, 1, a_a, 1, &oldtype, MPI_COMBINER_HVECTOR );
+    }
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/type_create_indexed_block.c b/ompi/mpi/c/type_create_indexed_block.c
deleted file mode 100644
index cec91fbfa5e..00000000000
--- a/ompi/mpi/c/type_create_indexed_block.c
+++ /dev/null
@@ -1,84 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_create_indexed_block = PMPI_Type_create_indexed_block
-#endif
-#define MPI_Type_create_indexed_block PMPI_Type_create_indexed_block
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_create_indexed_block";
-
-
-int MPI_Type_create_indexed_block(int count,
-                                  int blocklength,
-                                  const int array_of_displacements[],
-                                  MPI_Datatype oldtype,
-                                  MPI_Datatype *newtype)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(oldtype);
-        );
-
-    if( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if( count < 0 ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COUNT,
-                                          FUNC_NAME);
-        } else if( (count > 0) && (blocklength < 0 || NULL == array_of_displacements) ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME );
-        } else if (NULL == oldtype || MPI_DATATYPE_NULL == oldtype ||
-                   NULL == newtype) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
-                                          FUNC_NAME );
-        }
-    }
-
-    rc = ompi_datatype_create_indexed_block( count, blocklength, array_of_displacements,
-                                             oldtype, newtype );
-    if( rc != MPI_SUCCESS ) {
-        ompi_datatype_destroy( newtype );
-        OMPI_ERRHANDLER_NOHANDLE_RETURN( rc, rc, FUNC_NAME );
-    }
-    {
-        const int* a_i[3] = {&count, &blocklength, array_of_displacements};
-
-        ompi_datatype_set_args( *newtype, 2 + count, a_i, 0, NULL, 1, &oldtype,
-                                MPI_COMBINER_INDEXED_BLOCK );
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/type_create_indexed_block.c.in b/ompi/mpi/c/type_create_indexed_block.c.in
new file mode 100644
index 00000000000..24732bf5086
--- /dev/null
+++ b/ompi/mpi/c/type_create_indexed_block.c.in
@@ -0,0 +1,97 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+/*
+ * TODO:BIGCOUNT this file will need to be updated once
+ * the datatype framework supports bigcount 
+ */
+
+
+PROTOTYPE ERROR_CLASS type_create_indexed_block(COUNT count,
+                                                COUNT blocklength,
+                                                COUNT_ARRAY array_of_displacements,
+                                                DATATYPE oldtype,
+                                                DATATYPE_OUT newtype)
+{
+    int rc;
+    int *iarray_of_displacements = NULL;
+
+    MEMCHECKER(
+        memchecker_datatype(oldtype);
+        );
+
+    if( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if( count < 0 ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COUNT,
+                                          FUNC_NAME);
+        } else if( (count > 0) && (blocklength < 0 || NULL == array_of_displacements) ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME );
+        } else if (NULL == oldtype || MPI_DATATYPE_NULL == oldtype ||
+                   NULL == newtype) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
+                                          FUNC_NAME );
+        }
+    }
+
+#if OMPI_BIGCOUNT_SRC
+    iarray_of_displacements = (int *)malloc(count * sizeof(int));
+    if (NULL == iarray_of_displacements) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_NO_MEM, FUNC_NAME);
+    }
+    for (int ii = 0; ii < (int)count; ii++) {
+        iarray_of_displacements[ii] = (int)array_of_displacements[ii];
+    }
+#else
+    iarray_of_displacements = (int *)array_of_displacements;
+#endif
+    rc = ompi_datatype_create_indexed_block( count, blocklength, iarray_of_displacements,
+                                             oldtype, newtype );
+    if( rc != MPI_SUCCESS ) {
+        ompi_datatype_destroy( newtype );
+        OMPI_ERRHANDLER_NOHANDLE_RETURN( rc, rc, FUNC_NAME );
+    }
+    {
+        const int* a_i[3] = {(int *)&count, (int *)&blocklength, iarray_of_displacements};
+
+        ompi_datatype_set_args( *newtype, 2 + count, a_i, 0, NULL, 1, &oldtype,
+                                MPI_COMBINER_INDEXED_BLOCK );
+    }
+
+#if OMPI_BIGCOUNT_SRC
+    free(iarray_of_displacements);
+#endif
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/type_create_keyval.c b/ompi/mpi/c/type_create_keyval.c
deleted file mode 100644
index dc0e764b15d..00000000000
--- a/ompi/mpi/c/type_create_keyval.c
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/attribute/attribute.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_create_keyval = PMPI_Type_create_keyval
-#endif
-#define MPI_Type_create_keyval PMPI_Type_create_keyval
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_create_keyval";
-
-
-int MPI_Type_create_keyval(MPI_Type_copy_attr_function *type_copy_attr_fn,
-                           MPI_Type_delete_attr_function *type_delete_attr_fn,
-                           int *type_keyval,
-                           void *extra_state)
-{
-    int ret;
-    ompi_attribute_fn_ptr_union_t copy_fn;
-    ompi_attribute_fn_ptr_union_t del_fn;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if ((NULL == type_copy_attr_fn) || (NULL == type_delete_attr_fn) ||
-            (NULL == type_keyval)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(
-                                          MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-    }
-
-    copy_fn.attr_datatype_copy_fn = (MPI_Type_internal_copy_attr_function*)type_copy_attr_fn;
-    del_fn.attr_datatype_delete_fn = type_delete_attr_fn;
-
-    ret = ompi_attr_create_keyval(TYPE_ATTR, copy_fn, del_fn,
-                                  type_keyval, extra_state, 0, NULL);
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(ret, ret, FUNC_NAME);
-}
-
-
diff --git a/ompi/mpi/c/type_create_keyval.c.in b/ompi/mpi/c/type_create_keyval.c.in
new file mode 100644
index 00000000000..ea5cff2542d
--- /dev/null
+++ b/ompi/mpi/c/type_create_keyval.c.in
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/attribute/attribute.h"
+
+PROTOTYPE ERROR_CLASS type_create_keyval(TYPE_COPY_ATTR_FUNCTION type_copy_attr_fn,
+                                         TYPE_DELETE_ATTR_FUNCTION type_delete_attr_fn,
+                                         INT_OUT type_keyval,
+                                         BUFFER_OUT extra_state)
+{
+    int ret;
+    ompi_attribute_fn_ptr_union_t copy_fn;
+    ompi_attribute_fn_ptr_union_t del_fn;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if ((NULL == type_copy_attr_fn) || (NULL == type_delete_attr_fn) ||
+            (NULL == type_keyval)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(
+                                          MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+    copy_fn.attr_datatype_copy_fn = type_copy_attr_fn;
+    del_fn.attr_datatype_delete_fn = type_delete_attr_fn;
+
+    ret = ompi_attr_create_keyval(TYPE_ATTR, copy_fn, del_fn,
+                                  type_keyval, extra_state, 0, NULL);
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(ret, ret, FUNC_NAME);
+}
+
+
diff --git a/ompi/mpi/c/type_create_resized.c b/ompi/mpi/c/type_create_resized.c
deleted file mode 100644
index a88c5b9ad06..00000000000
--- a/ompi/mpi/c/type_create_resized.c
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_create_resized = PMPI_Type_create_resized
-#endif
-#define MPI_Type_create_resized PMPI_Type_create_resized
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_create_resized";
-
-
-int MPI_Type_create_resized(MPI_Datatype oldtype,
-                            MPI_Aint lb,
-                            MPI_Aint extent,
-                            MPI_Datatype *newtype)
-{
-   int rc;
-
-   MEMCHECKER(
-      memchecker_datatype(oldtype);
-   );
-
-   if( MPI_PARAM_CHECK ) {
-      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-      if (NULL == oldtype || MPI_DATATYPE_NULL == oldtype ||
-          NULL == newtype) {
-        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
-                                      FUNC_NAME );
-      }
-   }
-
-   rc = ompi_datatype_create_resized( oldtype, lb, extent, newtype );
-   if( rc != MPI_SUCCESS ) {
-      ompi_datatype_destroy( newtype );
-      OMPI_ERRHANDLER_NOHANDLE_RETURN( rc, rc, FUNC_NAME );
-   }
-
-   {
-      MPI_Aint a_a[2];
-      a_a[0] = lb;
-      a_a[1] = extent;
-      ompi_datatype_set_args( *newtype, 0, NULL, 2, a_a, 1, &oldtype, MPI_COMBINER_RESIZED );
-   }
-
-   return MPI_SUCCESS;
-}
-
-
diff --git a/ompi/mpi/c/type_create_resized.c.in b/ompi/mpi/c/type_create_resized.c.in
new file mode 100644
index 00000000000..9e07109ec7b
--- /dev/null
+++ b/ompi/mpi/c/type_create_resized.c.in
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS type_create_resized(DATATYPE oldtype,
+                                          AINT_COUNT lb,
+                                          AINT_COUNT extent,
+                                          DATATYPE_OUT newtype)
+{
+   int rc;
+
+   MEMCHECKER(
+      memchecker_datatype(oldtype);
+   );
+
+   if( MPI_PARAM_CHECK ) {
+      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+      if (NULL == oldtype || MPI_DATATYPE_NULL == oldtype ||
+          NULL == newtype) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
+                                      FUNC_NAME );
+      }
+   }
+
+   rc = ompi_datatype_create_resized( oldtype, lb, extent, newtype );
+   if( rc != MPI_SUCCESS ) {
+      ompi_datatype_destroy( newtype );
+      OMPI_ERRHANDLER_NOHANDLE_RETURN( rc, rc, FUNC_NAME );
+   }
+
+   {
+      MPI_Aint a_a[2];
+      a_a[0] = lb;
+      a_a[1] = extent;
+      ompi_datatype_set_args( *newtype, 0, NULL, 2, a_a, 1, &oldtype, MPI_COMBINER_RESIZED );
+   }
+
+   return MPI_SUCCESS;
+}
+
+
diff --git a/ompi/mpi/c/type_create_struct.c b/ompi/mpi/c/type_create_struct.c
deleted file mode 100644
index ec199c27915..00000000000
--- a/ompi/mpi/c/type_create_struct.c
+++ /dev/null
@@ -1,99 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_create_struct = PMPI_Type_create_struct
-#endif
-#define MPI_Type_create_struct PMPI_Type_create_struct
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_create_struct";
-
-
-int MPI_Type_create_struct(int count,
-                           const int array_of_blocklengths[],
-                           const MPI_Aint array_of_displacements[],
-                           const MPI_Datatype array_of_types[],
-                           MPI_Datatype *newtype)
-{
-    int i, rc;
-
-    if ( count > 0 ) {
-        for ( i = 0; i < count; i++ ) {
-            MEMCHECKER(
-                memchecker_datatype(array_of_types[i]);
-                );
-        }
-    }
-
-    if( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if( count < 0 ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COUNT,
-                                          FUNC_NAME);
-        } else if( (count > 0) && (NULL == array_of_blocklengths ||
-                                   NULL == array_of_displacements ||
-                                   NULL == array_of_types) ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-        } else if (NULL == newtype) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE, FUNC_NAME);
-        }
-        for ( i = 0; i < count; i++ ){
-            if (NULL == array_of_types[i] ||
-                MPI_DATATYPE_NULL == array_of_types[i]) {
-                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
-                                              FUNC_NAME);
-            } else if (array_of_blocklengths[i] < 0) {
-                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                              FUNC_NAME);
-            }
-        }
-    }
-
-    rc = ompi_datatype_create_struct( count, array_of_blocklengths, array_of_displacements,
-                                      array_of_types, newtype );
-    if( rc != MPI_SUCCESS ) {
-        ompi_datatype_destroy( newtype );
-        OMPI_ERRHANDLER_NOHANDLE_RETURN( rc, 
-                                rc, FUNC_NAME );
-    }
-
-    {
-        const int* a_i[2] = {&count, array_of_blocklengths};
-
-        ompi_datatype_set_args( *newtype, count + 1, a_i, count, array_of_displacements,
-                                count, array_of_types, MPI_COMBINER_STRUCT );
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/type_create_struct.c.in b/ompi/mpi/c/type_create_struct.c.in
new file mode 100644
index 00000000000..accea45f603
--- /dev/null
+++ b/ompi/mpi/c/type_create_struct.c.in
@@ -0,0 +1,130 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+/*
+ * TODO:BIGCOUNT this file will need to be updated once
+ * the datatype framework supports bigcount 
+ */
+
+PROTOTYPE ERROR_CLASS type_create_struct(COUNT count,
+                                         COUNT_ARRAY array_of_blocklengths,
+                                         AINT_COUNT_ARRAY array_of_displacements,
+                                         DATATYPE_ARRAY array_of_types,
+                                         DATATYPE_OUT newtype)
+{
+    int i, rc, icount = (int)count;
+    int *iarray_of_blocklengths = NULL;
+    MPI_Aint *iarray_of_displacements = NULL;
+
+    if ( count > 0 ) {
+        for ( i = 0; i < count; i++ ) {
+            MEMCHECKER(
+                memchecker_datatype(array_of_types[i]);
+                );
+        }
+    }
+
+    if( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if( count < 0 ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COUNT,
+                                          FUNC_NAME);
+        } else if( (count > 0) && (NULL == array_of_blocklengths ||
+                                   NULL == array_of_displacements ||
+                                   NULL == array_of_types) ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+        } else if (NULL == newtype) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE, FUNC_NAME);
+        }
+        for ( i = 0; i < count; i++ ){
+            if (NULL == array_of_types[i] ||
+                MPI_DATATYPE_NULL == array_of_types[i]) {
+                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
+                                              FUNC_NAME);
+            } else if (array_of_blocklengths[i] < 0) {
+                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                              FUNC_NAME);
+            }
+        }
+#if OMPI_BIGCOUNT_SRC
+        OMPI_CHECK_MPI_COUNT_INT_CONVERSION_OVERFLOW(rc, count);
+        if (OMPI_SUCCESS != rc) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(rc, FUNC_NAME);
+        }
+#endif
+    }
+
+#if OMPI_BIGCOUNT_SRC
+    iarray_of_blocklengths = (int *)malloc(count * sizeof(int));
+    if (NULL == iarray_of_blocklengths) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_NO_MEM, FUNC_NAME);
+    }
+    iarray_of_displacements = (MPI_Aint *)malloc(count * sizeof(MPI_Aint));
+    if (NULL == iarray_of_displacements) {
+        free(iarray_of_blocklengths);
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_NO_MEM, FUNC_NAME);
+    }
+    for (int ii = 0; ii < (int)count; ii++) {
+        iarray_of_blocklengths[ii] = (int)array_of_blocklengths[ii];
+        iarray_of_displacements[ii] = (MPI_Aint)array_of_displacements[ii];
+    }
+#else
+    iarray_of_blocklengths = (int *)array_of_blocklengths;
+    iarray_of_displacements = (MPI_Aint *)array_of_displacements;
+#endif
+    rc = ompi_datatype_create_struct( icount, iarray_of_blocklengths, iarray_of_displacements,
+                                      array_of_types, newtype );
+    if( rc != MPI_SUCCESS ) {
+        ompi_datatype_destroy( newtype );
+#if OMPI_BIGCOUNT_SRC
+        free(iarray_of_blocklengths);
+        free(iarray_of_displacements);
+#endif
+        OMPI_ERRHANDLER_NOHANDLE_RETURN( rc, 
+                                rc, FUNC_NAME );
+    }
+
+    {
+        const int* a_i[2] = {(int *)&icount, iarray_of_blocklengths};
+
+        ompi_datatype_set_args( *newtype, icount + 1, a_i, icount, iarray_of_displacements,
+                                icount, array_of_types, MPI_COMBINER_STRUCT );
+    }
+#if OMPI_BIGCOUNT_SRC
+    free(iarray_of_blocklengths);
+    free(iarray_of_displacements);
+#endif
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/type_create_subarray.c b/ompi/mpi/c/type_create_subarray.c
deleted file mode 100644
index daa68e634e6..00000000000
--- a/ompi/mpi/c/type_create_subarray.c
+++ /dev/null
@@ -1,88 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2009      Sun Microsystems, Inc. All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_create_subarray = PMPI_Type_create_subarray
-#endif
-#define MPI_Type_create_subarray PMPI_Type_create_subarray
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_create_subarray";
-
-
-int MPI_Type_create_subarray(int ndims,
-                             const int size_array[],
-                             const int subsize_array[],
-                             const int start_array[],
-                             int order,
-                             MPI_Datatype oldtype,
-                             MPI_Datatype *newtype)
-{
-    int32_t i, rc;
-
-    MEMCHECKER(
-        memchecker_datatype(oldtype);
-        );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if( ndims < 0 ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COUNT, FUNC_NAME);
-        } else if( (ndims > 0) && ((NULL == size_array) || (NULL == subsize_array) || (NULL == start_array)) ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-        } else if( (NULL == oldtype) || (MPI_DATATYPE_NULL == oldtype) || (NULL == newtype) ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE, FUNC_NAME);
-        } else if( (MPI_ORDER_C != order) && (MPI_ORDER_FORTRAN != order) ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-        }
-        for( i = 0; i < ndims; i++ ) {
-            if( (subsize_array[i] < 1) || (subsize_array[i] > size_array[i]) ) {
-                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-            } else if( (start_array[i] < 0) || (start_array[i] > (size_array[i] - subsize_array[i])) ) {
-                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-    }
-
-    rc = ompi_datatype_create_subarray( ndims, size_array, subsize_array, start_array,
-                                        order, oldtype, newtype);
-    if( OMPI_SUCCESS == rc ) {
-        const int* a_i[5] = {&ndims, size_array, subsize_array, start_array, &order};
-
-        ompi_datatype_set_args( *newtype, 3 * ndims + 2, a_i, 0, NULL, 1, &oldtype,
-                                MPI_COMBINER_SUBARRAY );
-    }
-
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/type_create_subarray.c.in b/ompi/mpi/c/type_create_subarray.c.in
new file mode 100644
index 00000000000..d25cde247c4
--- /dev/null
+++ b/ompi/mpi/c/type_create_subarray.c.in
@@ -0,0 +1,133 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2009      Sun Microsystems, Inc. All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+/*
+ * TODO:BIGCOUNT this file will need to be updated once
+ * the datatype framework supports bigcount 
+ */
+
+PROTOTYPE ERROR_CLASS type_create_subarray(INT ndims,
+                                           COUNT_ARRAY size_array,
+                                           COUNT_ARRAY subsize_array,
+                                           COUNT_ARRAY start_array,
+                                           INT order,
+                                           DATATYPE oldtype,
+                                           DATATYPE_OUT newtype)
+{
+    int32_t i, rc;
+    int *isize_array = NULL;
+    int *isubsize_array = NULL;
+    int *istart_array = NULL;
+
+    MEMCHECKER(
+        memchecker_datatype(oldtype);
+        );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if( ndims < 0 ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COUNT, FUNC_NAME);
+        } else if( (ndims > 0) && ((NULL == size_array) || (NULL == subsize_array) || (NULL == start_array)) ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+        } else if( (NULL == oldtype) || (MPI_DATATYPE_NULL == oldtype) || (NULL == newtype) ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE, FUNC_NAME);
+        } else if( (MPI_ORDER_C != order) && (MPI_ORDER_FORTRAN != order) ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+        }
+        for( i = 0; i < ndims; i++ ) {
+#if OMPI_BIGCOUNT_SRC
+            OMPI_CHECK_MPI_COUNT_INT_CONVERSION_OVERFLOW(rc, size_array[i]);
+            if (OMPI_SUCCESS != rc) {
+                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(rc, FUNC_NAME);
+            }
+            OMPI_CHECK_MPI_COUNT_INT_CONVERSION_OVERFLOW(rc, subsize_array[i]);
+            if (OMPI_SUCCESS != rc) {
+                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(rc, FUNC_NAME);
+            }
+            OMPI_CHECK_MPI_COUNT_INT_CONVERSION_OVERFLOW(rc, start_array[i]);
+            if (OMPI_SUCCESS != rc) {
+                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(rc, FUNC_NAME);
+            }
+#endif
+            if( (subsize_array[i] < 1) || (subsize_array[i] > size_array[i]) ) {
+                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+            } else if( (start_array[i] < 0) || (start_array[i] > (size_array[i] - subsize_array[i])) ) {
+                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+            }
+        }
+    }
+
+#if OMPI_BIGCOUNT_SRC
+    isize_array = (int *)malloc(ndims * sizeof(int));
+    if (NULL == isize_array) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_NO_MEM, FUNC_NAME);
+    }
+    isubsize_array = (int *)malloc(ndims * sizeof(int));
+    if (NULL == isubsize_array) {
+        free(isize_array);
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_NO_MEM, FUNC_NAME);
+    }
+    istart_array = (int *)malloc(ndims * sizeof(int));
+    if (NULL == istart_array) {
+        free(isize_array);
+        free(isubsize_array);
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_NO_MEM, FUNC_NAME);
+    }
+    for (int ii = 0; ii < ndims; ii++) {
+        isize_array[ii] = (int)size_array[ii];
+        isubsize_array[ii] = (int)subsize_array[ii];
+        istart_array[ii] = (int)start_array[ii];
+    }
+#else
+    isize_array = (int *)size_array;
+    isubsize_array = (int *)subsize_array;
+    istart_array = (int *)start_array;
+#endif
+    rc = ompi_datatype_create_subarray( ndims, isize_array, isubsize_array, istart_array,
+                                        order, oldtype, newtype);
+    if( OMPI_SUCCESS == rc ) {
+        const int* a_i[5] = {&ndims, isize_array, isubsize_array, istart_array, &order};
+
+        ompi_datatype_set_args( *newtype, 3 * ndims + 2, a_i, 0, NULL, 1, &oldtype,
+                                MPI_COMBINER_SUBARRAY );
+    }
+
+#if OMPI_BIGCOUNT_SRC
+    free(isize_array);
+    free(isubsize_array);
+    free(istart_array);
+#endif
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/type_delete_attr.c b/ompi/mpi/c/type_delete_attr.c
deleted file mode 100644
index 050df3586d0..00000000000
--- a/ompi/mpi/c/type_delete_attr.c
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/attribute/attribute.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_delete_attr = PMPI_Type_delete_attr
-#endif
-#define MPI_Type_delete_attr PMPI_Type_delete_attr
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_delete_attr";
-
-
-int MPI_Type_delete_attr (MPI_Datatype type, int type_keyval)
-{
-   int ret;
-
-   MEMCHECKER(
-      memchecker_datatype(type);
-   );
-
-   if (MPI_PARAM_CHECK) {
-      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-      if (NULL == type || MPI_DATATYPE_NULL == type) {
-         return OMPI_ERRHANDLER_NOHANDLE_INVOKE(
-                                       MPI_ERR_TYPE,
-                                       FUNC_NAME);
-      }
-   }
-
-   ret = ompi_attr_delete(TYPE_ATTR, type, type->d_keyhash, type_keyval,
-                          false);
-   OMPI_ERRHANDLER_NOHANDLE_RETURN(ret, 
-			  MPI_ERR_OTHER, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/type_delete_attr.c.in b/ompi/mpi/c/type_delete_attr.c.in
new file mode 100644
index 00000000000..68de235978c
--- /dev/null
+++ b/ompi/mpi/c/type_delete_attr.c.in
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/attribute/attribute.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS type_delete_attr(DATATYPE type, INT type_keyval)
+{
+   int ret;
+
+   MEMCHECKER(
+      memchecker_datatype(type);
+   );
+
+   if (MPI_PARAM_CHECK) {
+      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+      if (NULL == type || MPI_DATATYPE_NULL == type) {
+         return OMPI_ERRHANDLER_NOHANDLE_INVOKE(
+                                       MPI_ERR_TYPE,
+                                       FUNC_NAME);
+      }
+   }
+
+   ret = ompi_attr_delete(TYPE_ATTR, type, type->d_keyhash, type_keyval,
+                          false);
+   OMPI_ERRHANDLER_NOHANDLE_RETURN(ret, 
+			  MPI_ERR_OTHER, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/type_dup.c b/ompi/mpi/c/type_dup.c
deleted file mode 100644
index bd9e125e443..00000000000
--- a/ompi/mpi/c/type_dup.c
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/attribute/attribute.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_dup = PMPI_Type_dup
-#endif
-#define MPI_Type_dup PMPI_Type_dup
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_dup";
-
-
-int MPI_Type_dup (MPI_Datatype type,
-                  MPI_Datatype *newtype)
-{
-   MEMCHECKER(
-      memchecker_datatype(type);
-   );
-
-   if( MPI_PARAM_CHECK ) {
-      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-      if (NULL == type || MPI_DATATYPE_NULL == type ||
-          NULL == newtype) {
-        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
-                                      FUNC_NAME );
-      }
-   }
-
-   if (OMPI_SUCCESS != ompi_datatype_duplicate( type, newtype)) {
-       ompi_datatype_destroy( newtype );
-       OMPI_ERRHANDLER_RETURN (MPI_ERR_INTERN, MPI_COMM_WORLD,
-                               MPI_ERR_INTERN, FUNC_NAME );
-   }
-
-   ompi_datatype_set_args( *newtype, 0, NULL, 0, NULL, 1, &type, MPI_COMBINER_DUP );
-
-   /* Copy all the old attributes, if there were any.  This is done
-      here (vs. ompi_datatype_duplicate()) because MPI_TYPE_DUP is the
-      only MPI function that copies attributes.  All other MPI
-      functions that take an old type and generate a newtype do not
-      copy attributes.  Really. */
-   if (NULL != type->d_keyhash) {
-       ompi_attr_hash_init(&(*newtype)->d_keyhash);
-       if (OMPI_SUCCESS != ompi_attr_copy_all(TYPE_ATTR,
-                                              type, *newtype,
-                                              type->d_keyhash,
-                                              (*newtype)->d_keyhash)) {
-           ompi_datatype_destroy(newtype);
-           OMPI_ERRHANDLER_NOHANDLE_RETURN( MPI_ERR_INTERN, 
-                                   MPI_ERR_INTERN, FUNC_NAME );
-       }
-   }
-
-   return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/type_dup.c.in b/ompi/mpi/c/type_dup.c.in
new file mode 100644
index 00000000000..41abec68692
--- /dev/null
+++ b/ompi/mpi/c/type_dup.c.in
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/attribute/attribute.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS type_dup (DATATYPE type, DATATYPE_OUT newtype)
+{
+   int ret;
+
+   MEMCHECKER(
+      memchecker_datatype(type);
+   );
+
+   if( MPI_PARAM_CHECK ) {
+      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+      if (NULL == type || MPI_DATATYPE_NULL == type ||
+          NULL == newtype) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
+                                      FUNC_NAME );
+      }
+   }
+
+   if (OMPI_SUCCESS != (ret = ompi_datatype_duplicate( type, newtype))) {
+       ompi_datatype_destroy( newtype );
+       OMPI_ERRHANDLER_NOHANDLE_RETURN( ret, ret, FUNC_NAME );
+   }
+
+   ompi_datatype_set_args( *newtype, 0, NULL, 0, NULL, 1, &type, MPI_COMBINER_DUP );
+
+   /* Copy all the old attributes, if there were any.  This is done
+      here (vs. ompi_datatype_duplicate()) because MPI_TYPE_DUP is the
+      only MPI function that copies attributes.  All other MPI
+      functions that take an old type and generate a newtype do not
+      copy attributes.  Really. */
+   if (NULL != type->d_keyhash) {
+       ompi_attr_hash_init(&(*newtype)->d_keyhash);
+       if (OMPI_SUCCESS != (ret = ompi_attr_copy_all(TYPE_ATTR,
+                                                     type, *newtype,
+                                                     type->d_keyhash,
+                                                     (*newtype)->d_keyhash))) {
+           ompi_datatype_destroy(newtype);
+           OMPI_ERRHANDLER_NOHANDLE_RETURN( ret, ret, FUNC_NAME );
+       }
+   }
+
+   return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/type_f2c.c b/ompi/mpi/c/type_f2c.c
deleted file mode 100644
index 2afa2909ddd..00000000000
--- a/ompi/mpi/c/type_f2c.c
+++ /dev/null
@@ -1,65 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2007 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mpi/fortran/base/fint_2_int.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/datatype/ompi_datatype_internal.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_f2c = PMPI_Type_f2c
-#endif
-#define MPI_Type_f2c PMPI_Type_f2c
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_f2c";
-
-
-MPI_Datatype MPI_Type_f2c(MPI_Fint datatype)
-{
-    int datatype_index = OMPI_FINT_2_INT(datatype);
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-    }
-
-    /* Per MPI-2:4.12.4, do not invoke an error handler if we get an
-       invalid fortran handle.  If we get an invalid fortran handle,
-       return an invalid C handle. */
-    if (datatype_index < 0 ||
-        datatype_index >=
-        opal_pointer_array_get_size(&ompi_datatype_f_to_c_table)) {
-        return NULL;
-    }
-
-    return (MPI_Datatype)opal_pointer_array_get_item(&ompi_datatype_f_to_c_table, datatype_index);
-}
-
diff --git a/ompi/mpi/c/type_f2c.c.in b/ompi/mpi/c/type_f2c.c.in
new file mode 100644
index 00000000000..083e0dcae80
--- /dev/null
+++ b/ompi/mpi/c/type_f2c.c.in
@@ -0,0 +1,57 @@
+/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/datatype/ompi_datatype_internal.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE DATATYPE type_f2c(FINT datatype)
+{
+    int datatype_index = OMPI_FINT_2_INT(datatype);
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+    }
+
+    /* Per MPI-2:4.12.4, do not invoke an error handler if we get an
+       invalid fortran handle.  If we get an invalid fortran handle,
+       return an invalid C handle. */
+    if (datatype_index < 0 ||
+        datatype_index >=
+        opal_pointer_array_get_size(&ompi_datatype_f_to_c_table)) {
+        return NULL;
+    }
+
+    return (MPI_Datatype)opal_pointer_array_get_item(&ompi_datatype_f_to_c_table, datatype_index);
+}
+
diff --git a/ompi/mpi/c/type_free.c b/ompi/mpi/c/type_free.c
deleted file mode 100644
index c7b562d961b..00000000000
--- a/ompi/mpi/c/type_free.c
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_free = PMPI_Type_free
-#endif
-#define MPI_Type_free PMPI_Type_free
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_free";
-
-
-int MPI_Type_free(MPI_Datatype *type)
-{
-   int rc;
-
-   MEMCHECKER(
-      memchecker_datatype(*type);
-   );
-
-   if( MPI_PARAM_CHECK ) {
-      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-      if (NULL == type || NULL == *type || MPI_DATATYPE_NULL == *type ||
-          ompi_datatype_is_predefined(*type)) {
-        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
-                                      FUNC_NAME );
-      }
-   }
-
-   rc = ompi_datatype_destroy( type );
-   if( rc != MPI_SUCCESS ) {
-      OMPI_ERRHANDLER_NOHANDLE_RETURN( MPI_ERR_INTERN, 
-                             MPI_ERR_INTERN, FUNC_NAME );
-   }
-   *type = MPI_DATATYPE_NULL;
-
-   return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/type_free.c.in b/ompi/mpi/c/type_free.c.in
new file mode 100644
index 00000000000..fbdfb199434
--- /dev/null
+++ b/ompi/mpi/c/type_free.c.in
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS type_free(DATATYPE_OUT type)
+{
+   int rc;
+
+   MEMCHECKER(
+      memchecker_datatype(*type);
+   );
+
+   if( MPI_PARAM_CHECK ) {
+      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+      if (NULL == type || NULL == *type || MPI_DATATYPE_NULL == *type ||
+          ompi_datatype_is_predefined(*type)) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
+                                      FUNC_NAME );
+      }
+   }
+
+   rc = ompi_datatype_destroy( type );
+   if( rc != MPI_SUCCESS ) {
+      OMPI_ERRHANDLER_NOHANDLE_RETURN( MPI_ERR_INTERN, 
+                             MPI_ERR_INTERN, FUNC_NAME );
+   }
+   *type = MPI_DATATYPE_NULL;
+
+   return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/type_free_keyval.c b/ompi/mpi/c/type_free_keyval.c
deleted file mode 100644
index 3ea276a1cf2..00000000000
--- a/ompi/mpi/c/type_free_keyval.c
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/attribute/attribute.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_free_keyval = PMPI_Type_free_keyval
-#endif
-#define MPI_Type_free_keyval PMPI_Type_free_keyval
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_free_keyval";
-
-
-int MPI_Type_free_keyval(int *type_keyval)
-{
-    int ret;
-
-    /* Check for valid key pointer */
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == type_keyval) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(
-                                          MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-    }
-
-    ret = ompi_attr_free_keyval(TYPE_ATTR, type_keyval, 0);
-
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(ret, 
-			  MPI_ERR_OTHER, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/type_free_keyval.c.in b/ompi/mpi/c/type_free_keyval.c.in
new file mode 100644
index 00000000000..1c08edd3351
--- /dev/null
+++ b/ompi/mpi/c/type_free_keyval.c.in
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/attribute/attribute.h"
+
+PROTOTYPE ERROR_CLASS type_free_keyval(INT_OUT type_keyval)
+{
+    int ret;
+
+    /* Check for valid key pointer */
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == type_keyval) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(
+                                          MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+    ret = ompi_attr_free_keyval(TYPE_ATTR, type_keyval, 0);
+
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(ret, 
+			  MPI_ERR_OTHER, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/type_get_attr.c b/ompi/mpi/c/type_get_attr.c
deleted file mode 100644
index 1b42b5c8122..00000000000
--- a/ompi/mpi/c/type_get_attr.c
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/attribute/attribute.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_get_attr = PMPI_Type_get_attr
-#endif
-#define MPI_Type_get_attr PMPI_Type_get_attr
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_get_attr";
-
-
-int MPI_Type_get_attr (MPI_Datatype type,
-                       int type_keyval,
-                       void *attribute_val,
-                       int *flag)
-{
-    int ret;
-
-    MEMCHECKER(
-        memchecker_datatype(type);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == type || MPI_DATATYPE_NULL == type) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
-                                          FUNC_NAME );
-        } else if ((NULL == attribute_val) || (NULL == flag)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(
-                                          MPI_ERR_ARG,
-                                          FUNC_NAME);
-        } else if (MPI_KEYVAL_INVALID == type_keyval) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_KEYVAL,
-                                          FUNC_NAME);
-        }
-    }
-
-    /* This stuff is very confusing.  Be sure to see
-       src/attribute/attribute.c for a lengthy comment explaining Open
-       MPI attribute behavior. */
-
-    ret = ompi_attr_get_c(type->d_keyhash, type_keyval,
-                          (void**)attribute_val, flag);
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(ret, 
-                           MPI_ERR_OTHER, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/type_get_attr.c.in b/ompi/mpi/c/type_get_attr.c.in
new file mode 100644
index 00000000000..506adb9bf9d
--- /dev/null
+++ b/ompi/mpi/c/type_get_attr.c.in
@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/attribute/attribute.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS type_get_attr (DATATYPE type,
+                                     INT type_keyval,
+                                     BUFFER_OUT attribute_val,
+                                     INT_OUT flag)
+{
+    int ret;
+
+    MEMCHECKER(
+        memchecker_datatype(type);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == type || MPI_DATATYPE_NULL == type) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
+                                          FUNC_NAME );
+        } else if ((NULL == attribute_val) || (NULL == flag)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(
+                                          MPI_ERR_ARG,
+                                          FUNC_NAME);
+        } else if (MPI_KEYVAL_INVALID == type_keyval) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_KEYVAL,
+                                          FUNC_NAME);
+        }
+    }
+
+    /* This stuff is very confusing.  Be sure to see
+       src/attribute/attribute.c for a lengthy comment explaining Open
+       MPI attribute behavior. */
+
+    ret = ompi_attr_get_c(type->d_keyhash, type_keyval,
+                          (void**)attribute_val, flag);
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(ret, 
+                           MPI_ERR_OTHER, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/type_get_contents_c.c b/ompi/mpi/c/type_get_contents_c.c
new file mode 100644
index 00000000000..6be23ccc4c4
--- /dev/null
+++ b/ompi/mpi/c/type_get_contents_c.c
@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_Type_get_contents_c = PMPI_Type_get_contents_c
+#endif
+#define MPI_Type_get_contents_c PMPI_Type_get_contents_c
+#endif
+
+static const char FUNC_NAME[] = "MPI_Type_get_contents_c";
+
+
+int MPI_Type_get_contents_c(MPI_Datatype mtype,
+                            MPI_Count max_integers,
+                            MPI_Count max_addresses,
+                            MPI_Count max_large_counts,
+                            MPI_Count  max_datatypes,
+                            int array_of_integers[],
+                            MPI_Aint array_of_addresses[],
+                            MPI_Count  array_of_large_counts[],
+                            MPI_Datatype array_of_datatypes[])
+{
+    int rc, i;
+    MPI_Datatype newtype;
+
+    MEMCHECKER(
+        memchecker_datatype(mtype);
+    );
+
+    if( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == mtype || MPI_DATATYPE_NULL == mtype) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
+                                          FUNC_NAME );
+        } else if( ((NULL == array_of_integers) && (max_integers != 0)) ||
+                   ((NULL == array_of_addresses) && (max_addresses != 0)) ||
+                   ((NULL == array_of_datatypes) && (max_datatypes != 0)) ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME );
+        }
+    }
+
+/* TODO:BIGCOUNT: Need to embiggen ompi_datatype_get_args */
+    rc = ompi_datatype_get_args( mtype, 1, (int *)&max_integers, array_of_integers,
+                            (int *)&max_addresses, array_of_addresses,
+                            (int *)&max_datatypes, array_of_datatypes, NULL );
+    if( rc != MPI_SUCCESS ) {
+        OMPI_ERRHANDLER_NOHANDLE_RETURN( MPI_ERR_INTERN, 
+                                MPI_ERR_INTERN, FUNC_NAME );
+    }
+
+    for( i = 0; i < max_datatypes; i++ ) {
+        /* if we have a predefined datatype then we return directly a pointer to
+         * the datatype, otherwise we should create a copy and give back the copy.
+         */
+        if( !(ompi_datatype_is_predefined(array_of_datatypes[i])) ) {
+            if( (rc = ompi_datatype_duplicate( array_of_datatypes[i], &newtype )) != MPI_SUCCESS ) {
+                ompi_datatype_destroy( &newtype );
+                OMPI_ERRHANDLER_NOHANDLE_RETURN( MPI_ERR_INTERN, 
+                                        MPI_ERR_INTERN, FUNC_NAME );
+            }
+            ompi_datatype_copy_args( array_of_datatypes[i], newtype );
+            array_of_datatypes[i] = newtype;
+        }
+    }
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/type_get_envelope.c.in b/ompi/mpi/c/type_get_envelope.c.in
new file mode 100644
index 00000000000..e5395d796e4
--- /dev/null
+++ b/ompi/mpi/c/type_get_envelope.c.in
@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS Type_get_envelope(DATATYPE type,
+                          INT_OUT num_integers,
+                          INT_OUT num_addresses,
+                          INT_OUT num_datatypes,
+                          INT_OUT combiner)
+{
+   int rc;
+
+   MEMCHECKER(
+      memchecker_datatype(type);
+   );
+
+   if( MPI_PARAM_CHECK ) {
+      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+      if (NULL == type || MPI_DATATYPE_NULL == type) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
+                                      FUNC_NAME );
+      } else if (NULL == num_integers || NULL == num_addresses ||
+                 NULL == num_datatypes || NULL == combiner) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                      FUNC_NAME );
+      }
+   }
+
+   rc = ompi_datatype_get_args( type, 0, num_integers, NULL, num_addresses, NULL,
+                           num_datatypes, NULL, combiner );
+   OMPI_ERRHANDLER_NOHANDLE_RETURN( rc, rc, FUNC_NAME );
+}
diff --git a/ompi/mpi/c/type_get_envelope_c.c b/ompi/mpi/c/type_get_envelope_c.c
new file mode 100644
index 00000000000..24229e327cf
--- /dev/null
+++ b/ompi/mpi/c/type_get_envelope_c.c
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_Type_get_envelope_c = PMPI_Type_get_envelope_c
+#endif
+#define MPI_Type_get_envelope_c PMPI_Type_get_envelope_c
+#endif
+
+static const char FUNC_NAME[] = "MPI_Type_get_envelope_c";
+
+int MPI_Type_get_envelope_c(MPI_Datatype type,
+                            MPI_Count *num_integers,
+                            MPI_Count *num_addresses,
+                            MPI_Count *num_large_counts,
+                            MPI_Count *num_datatypes,
+                            int *combiner)
+{
+   int rc;
+
+   MEMCHECKER(
+      memchecker_datatype(type);
+   );
+
+   if( MPI_PARAM_CHECK ) {
+      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+      if (NULL == type || MPI_DATATYPE_NULL == type) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
+                                      FUNC_NAME );
+      } else if (NULL == num_integers || NULL == num_addresses ||
+                 NULL == num_datatypes || NULL == combiner) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                      FUNC_NAME );
+      }
+   }
+
+/* TODO:BIGCOUNT: Need to embiggen ompi_datatype_get_args */
+   rc = ompi_datatype_get_args( type, 0, (int *)num_integers, NULL, (int *)num_addresses, NULL,
+                           (int *)num_datatypes, NULL, combiner );
+   OMPI_ERRHANDLER_NOHANDLE_RETURN( rc, rc, FUNC_NAME );
+}
+
diff --git a/ompi/mpi/c/type_get_extent.c b/ompi/mpi/c/type_get_extent.c
deleted file mode 100644
index 0fddff2b0cb..00000000000
--- a/ompi/mpi/c/type_get_extent.c
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_get_extent = PMPI_Type_get_extent
-#endif
-#define MPI_Type_get_extent PMPI_Type_get_extent
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_get_extent";
-
-int MPI_Type_get_extent(MPI_Datatype type, MPI_Aint *lb, MPI_Aint *extent)
-{
-  int rc;
-
-  MEMCHECKER(
-    memchecker_datatype(type);
-  );
-
-  if (MPI_PARAM_CHECK) {
-    OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-    if (NULL == type || MPI_DATATYPE_NULL == type) {
-      return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE, FUNC_NAME);
-    } else if (NULL == lb || NULL == extent) {
-      return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-    }
-  }
-
-  rc = ompi_datatype_get_extent( type, lb, extent );
-  OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME );
-}
diff --git a/ompi/mpi/c/type_get_extent.c.in b/ompi/mpi/c/type_get_extent.c.in
new file mode 100644
index 00000000000..a2b044c9e33
--- /dev/null
+++ b/ompi/mpi/c/type_get_extent.c.in
@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS type_get_extent(DATATYPE type, AINT_COUNT_OUT lb, AINT_COUNT_OUT extent)
+{
+  int rc;
+  ptrdiff_t tmp_lb = 0;
+  ptrdiff_t tmp_extent = 0;
+
+  MEMCHECKER(
+    memchecker_datatype(type);
+  );
+
+  if (MPI_PARAM_CHECK) {
+    OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+    if (NULL == type || MPI_DATATYPE_NULL == type) {
+      return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE, FUNC_NAME);
+    } else if (NULL == lb || NULL == extent) {
+      return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+    }
+  }
+
+  rc = ompi_datatype_get_extent( type, &tmp_lb, &tmp_extent );
+  *lb = tmp_lb;
+  *extent = tmp_extent;
+  OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME );
+}
diff --git a/ompi/mpi/c/type_get_extent_x.c b/ompi/mpi/c/type_get_extent_x.c
deleted file mode 100644
index 5781a2f5468..00000000000
--- a/ompi/mpi/c/type_get_extent_x.c
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_get_extent_x = PMPI_Type_get_extent_x
-#endif
-#define MPI_Type_get_extent_x PMPI_Type_get_extent_x
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_get_extent_x";
-
-int MPI_Type_get_extent_x(MPI_Datatype type, MPI_Count *lb, MPI_Count *extent)
-{
-  MPI_Aint alb, aextent;
-  int rc;
-
-  MEMCHECKER(
-    memchecker_datatype(type);
-  );
-
-  if (MPI_PARAM_CHECK) {
-    OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-    if (NULL == type || MPI_DATATYPE_NULL == type) {
-      return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE, FUNC_NAME);
-    } else if (NULL == lb || NULL == extent) {
-      return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-    }
-  }
-
-  rc = ompi_datatype_get_extent( type, &alb, &aextent );
-  if (OMPI_SUCCESS == rc) {
-    *lb = (MPI_Count) alb;
-    *extent = (MPI_Count) aextent;
-  }
-
-  OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME );
-}
diff --git a/ompi/mpi/c/type_get_extent_x.c.in b/ompi/mpi/c/type_get_extent_x.c.in
new file mode 100644
index 00000000000..4ee710708b5
--- /dev/null
+++ b/ompi/mpi/c/type_get_extent_x.c.in
@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS type_get_extent_x(DATATYPE type, ELEMENT_COUNT lb, ELEMENT_COUNT extent)
+{
+  MPI_Aint alb, aextent;
+  int rc;
+
+  MEMCHECKER(
+    memchecker_datatype(type);
+  );
+
+  if (MPI_PARAM_CHECK) {
+    OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+    if (NULL == type || MPI_DATATYPE_NULL == type) {
+      return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE, FUNC_NAME);
+    } else if (NULL == lb || NULL == extent) {
+      return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+    }
+  }
+
+  rc = ompi_datatype_get_extent( type, &alb, &aextent );
+  if (OMPI_SUCCESS == rc) {
+    *lb = (MPI_Count) alb;
+    *extent = (MPI_Count) aextent;
+  }
+
+  OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME );
+}
diff --git a/ompi/mpi/c/type_get_name.c b/ompi/mpi/c/type_get_name.c
deleted file mode 100644
index b38697a1bc1..00000000000
--- a/ompi/mpi/c/type_get_name.c
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008-2018 Cisco Systems, Inc.  All rights reserved
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <string.h>
-
-#include "opal/util/string_copy.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_get_name = PMPI_Type_get_name
-#endif
-#define MPI_Type_get_name PMPI_Type_get_name
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_get_name";
-
-
-int MPI_Type_get_name(MPI_Datatype type, char *type_name, int *resultlen)
-{
-
-    MEMCHECKER(
-        memchecker_datatype(type);
-    );
-
-   if ( MPI_PARAM_CHECK ) {
-      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-      if (NULL == type || MPI_DATATYPE_NULL == type) {
-        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
-                                      FUNC_NAME );
-      } else if (NULL == type_name || NULL == resultlen) {
-        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                      FUNC_NAME );
-      }
-   }
-
-    /* Note that MPI-2.1 requires:
-       - terminating the string with a \0
-       - name[*resultlen] == '\0'
-       - and therefore (*resultlen) cannot be > (MPI_MAX_OBJECT_NAME-1)
-
-       The Fortran API version will pad to the right if necessary.
-
-       Note that type->name is guaranteed to be \0-terminated and
-       able to completely fit into MPI_MAX_OBJECT_NAME bytes (i.e.,
-       name+\0). */
-   *resultlen = (int)strlen(type->name);
-   opal_string_copy(type_name, type->name, MPI_MAX_OBJECT_NAME);
-   return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/type_get_name.c.in b/ompi/mpi/c/type_get_name.c.in
new file mode 100644
index 00000000000..68358762dce
--- /dev/null
+++ b/ompi/mpi/c/type_get_name.c.in
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008-2018 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include <string.h>
+
+#include "opal/util/string_copy.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS type_get_name(DATATYPE type, STRING_OUT type_name, INT_OUT resultlen)
+{
+
+    MEMCHECKER(
+        memchecker_datatype(type);
+    );
+
+   if ( MPI_PARAM_CHECK ) {
+      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+      if (NULL == type || MPI_DATATYPE_NULL == type) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
+                                      FUNC_NAME );
+      } else if (NULL == type_name || NULL == resultlen) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                      FUNC_NAME );
+      }
+   }
+
+    /* Note that MPI-2.1 requires:
+       - terminating the string with a \0
+       - name[*resultlen] == '\0'
+       - and therefore (*resultlen) cannot be > (MPI_MAX_OBJECT_NAME-1)
+
+       The Fortran API version will pad to the right if necessary.
+
+       Note that type->name is guaranteed to be \0-terminated and
+       able to completely fit into MPI_MAX_OBJECT_NAME bytes (i.e.,
+       name+\0). */
+   *resultlen = (int)strlen(type->name);
+   opal_string_copy(type_name, type->name, MPI_MAX_OBJECT_NAME);
+   return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/type_get_true_extent.c b/ompi/mpi/c/type_get_true_extent.c
deleted file mode 100644
index f2a6377593c..00000000000
--- a/ompi/mpi/c/type_get_true_extent.c
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_get_true_extent = PMPI_Type_get_true_extent
-#endif
-#define MPI_Type_get_true_extent PMPI_Type_get_true_extent
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_get_true_extent";
-
-int MPI_Type_get_true_extent(MPI_Datatype datatype,
-                             MPI_Aint *true_lb,
-                             MPI_Aint *true_extent)
-{
-   int rc;
-
-   MEMCHECKER(
-      memchecker_datatype(datatype);
-   );
-
-   if( MPI_PARAM_CHECK ) {
-      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-      if (NULL == datatype || MPI_DATATYPE_NULL == datatype) {
-        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
-                                      FUNC_NAME );
-      } else if (NULL == true_lb || NULL == true_extent) {
-        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                      FUNC_NAME );
-      }
-   }
-
-   rc = ompi_datatype_get_true_extent( datatype, true_lb, true_extent );
-   OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME );
-}
diff --git a/ompi/mpi/c/type_get_true_extent.c.in b/ompi/mpi/c/type_get_true_extent.c.in
new file mode 100644
index 00000000000..ab8b2e2cddf
--- /dev/null
+++ b/ompi/mpi/c/type_get_true_extent.c.in
@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS type_get_true_extent(DATATYPE datatype,
+                                           AINT_COUNT_OUT true_lb,
+                                           AINT_COUNT_OUT true_extent)
+{
+   int rc;
+
+   MEMCHECKER(
+      memchecker_datatype(datatype);
+   );
+
+   if( MPI_PARAM_CHECK ) {
+      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+      if (NULL == datatype || MPI_DATATYPE_NULL == datatype) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
+                                      FUNC_NAME );
+      } else if (NULL == true_lb || NULL == true_extent) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                      FUNC_NAME );
+      }
+   }
+
+   rc = ompi_datatype_get_true_extent( datatype, (MPI_Aint *)true_lb, (MPI_Aint *)true_extent );
+   OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME );
+}
diff --git a/ompi/mpi/c/type_get_true_extent_x.c b/ompi/mpi/c/type_get_true_extent_x.c
deleted file mode 100644
index bbbe7197992..00000000000
--- a/ompi/mpi/c/type_get_true_extent_x.c
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_get_true_extent_x = PMPI_Type_get_true_extent_x
-#endif
-#define MPI_Type_get_true_extent_x PMPI_Type_get_true_extent_x
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_get_true_extent_x";
-
-int MPI_Type_get_true_extent_x(MPI_Datatype datatype,
-                               MPI_Count *true_lb,
-                               MPI_Count *true_extent)
-{
-   MPI_Aint atrue_lb, atrue_extent;
-   int rc;
-
-   MEMCHECKER(
-      memchecker_datatype(datatype);
-   );
-
-   if( MPI_PARAM_CHECK ) {
-      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-      if (NULL == datatype || MPI_DATATYPE_NULL == datatype) {
-        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
-                                      FUNC_NAME );
-      } else if (NULL == true_lb || NULL == true_extent) {
-        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                      FUNC_NAME );
-      }
-   }
-
-   rc = ompi_datatype_get_true_extent( datatype, &atrue_lb, &atrue_extent );
-   if (OMPI_SUCCESS == rc) {
-      *true_lb = (MPI_Count) atrue_lb;
-      *true_extent = (MPI_Count) atrue_extent;
-   }
-
-   OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME );
-}
diff --git a/ompi/mpi/c/type_get_true_extent_x.c.in b/ompi/mpi/c/type_get_true_extent_x.c.in
new file mode 100644
index 00000000000..8999e924165
--- /dev/null
+++ b/ompi/mpi/c/type_get_true_extent_x.c.in
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS type_get_true_extent_x(DATATYPE datatype,
+                                             ELEMENT_COUNT true_lb,
+                                             ELEMENT_COUNT true_extent)
+{
+   MPI_Aint atrue_lb, atrue_extent;
+   int rc;
+
+   MEMCHECKER(
+      memchecker_datatype(datatype);
+   );
+
+   if( MPI_PARAM_CHECK ) {
+      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+      if (NULL == datatype || MPI_DATATYPE_NULL == datatype) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
+                                      FUNC_NAME );
+      } else if (NULL == true_lb || NULL == true_extent) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                      FUNC_NAME );
+      }
+   }
+
+   rc = ompi_datatype_get_true_extent( datatype, &atrue_lb, &atrue_extent );
+   if (OMPI_SUCCESS == rc) {
+      *true_lb = (MPI_Count) atrue_lb;
+      *true_extent = (MPI_Count) atrue_extent;
+   }
+
+   OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME );
+}
diff --git a/ompi/mpi/c/type_indexed.c b/ompi/mpi/c/type_indexed.c
deleted file mode 100644
index c4589d0583d..00000000000
--- a/ompi/mpi/c/type_indexed.c
+++ /dev/null
@@ -1,93 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_indexed = PMPI_Type_indexed
-#endif
-#define MPI_Type_indexed PMPI_Type_indexed
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_indexed";
-
-int MPI_Type_indexed(int count,
-                     const int array_of_blocklengths[],
-                     const int array_of_displacements[],
-                     MPI_Datatype oldtype,
-                     MPI_Datatype *newtype)
-{
-    int rc, i;
-
-    MEMCHECKER(
-        memchecker_datatype(oldtype);
-        );
-
-    if( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == oldtype || MPI_DATATYPE_NULL == oldtype ||
-            NULL == newtype) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
-                                          FUNC_NAME);
-        } else if( count < 0 ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COUNT,
-                                          FUNC_NAME);
-        } else if ((count > 0) && (NULL == array_of_blocklengths ||
-                                   NULL == array_of_displacements)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-        for( i = 0; i < count; i++ ) {
-            if( array_of_blocklengths[i] < 0 ) {
-                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                              FUNC_NAME);
-            }
-        }
-    }
-
-    rc = ompi_datatype_create_indexed ( count, array_of_blocklengths,
-                                        array_of_displacements,
-                                        oldtype, newtype );
-    if( rc != MPI_SUCCESS ) {
-        ompi_datatype_destroy( newtype );
-        OMPI_ERRHANDLER_NOHANDLE_RETURN( rc, 
-                                rc, FUNC_NAME );
-    }
-
-    {
-        const int* a_i[3] = {&count, array_of_blocklengths, array_of_displacements};
-
-        ompi_datatype_set_args( *newtype, 2 * count + 1, a_i, 0, NULL, 1, &oldtype,
-                                MPI_COMBINER_INDEXED );
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/type_indexed.c.in b/ompi/mpi/c/type_indexed.c.in
new file mode 100644
index 00000000000..c3ff80c1b1c
--- /dev/null
+++ b/ompi/mpi/c/type_indexed.c.in
@@ -0,0 +1,123 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+/*
+ * TODO:BIGCOUNT this file will need to be updated once
+ * the datatype framework supports bigcount 
+ */
+
+
+PROTOTYPE ERROR_CLASS type_indexed(COUNT count,
+                                   COUNT_ARRAY array_of_blocklengths,
+                                   COUNT_ARRAY array_of_displacements,
+                                   DATATYPE oldtype,
+                                   DATATYPE_OUT newtype)
+{
+    int rc, i;
+    int *iarray_of_blocklengths;
+    int *iarray_of_displacements;
+
+    MEMCHECKER(
+        memchecker_datatype(oldtype);
+        );
+
+    if( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == oldtype || MPI_DATATYPE_NULL == oldtype ||
+            NULL == newtype) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
+                                          FUNC_NAME);
+        } else if( count < 0 ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COUNT,
+                                          FUNC_NAME);
+        } else if ((count > 0) && (NULL == array_of_blocklengths ||
+                                   NULL == array_of_displacements)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+#if OMPI_BIGCOUNT_SRC
+        OMPI_CHECK_MPI_COUNT_INT_CONVERSION_OVERFLOW(rc, count);
+        if (OMPI_SUCCESS != rc) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(rc, FUNC_NAME);
+        }
+#endif
+        for( i = 0; i < count; i++ ) {
+            if( array_of_blocklengths[i] < 0 ) {
+                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                              FUNC_NAME);
+            }
+        }
+    }
+
+#if OMPI_BIGCOUNT_SRC
+    iarray_of_blocklengths = (int *)malloc(count * sizeof(int));
+    if (NULL == iarray_of_blocklengths) {
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_NO_MEM, FUNC_NAME);
+    }
+    iarray_of_displacements = (int *)malloc(count * sizeof(int));
+    if (NULL == iarray_of_displacements) {
+        free(iarray_of_blocklengths);
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_NO_MEM, FUNC_NAME);
+    }
+
+    for (int ii = 0; ii < (int)count; ii++) {
+        iarray_of_blocklengths[ii] = (int)array_of_blocklengths[ii];
+        iarray_of_displacements[ii] = (int)array_of_displacements[ii];
+    }
+#else
+    iarray_of_blocklengths = (int *)array_of_blocklengths;
+    iarray_of_displacements = (int *)array_of_displacements;
+#endif
+    rc = ompi_datatype_create_indexed ( count, iarray_of_blocklengths,
+                                        iarray_of_displacements,
+                                        oldtype, newtype );
+    if( rc != MPI_SUCCESS ) {
+        ompi_datatype_destroy( newtype );
+        OMPI_ERRHANDLER_NOHANDLE_RETURN( rc, 
+                                rc, FUNC_NAME );
+    }
+
+    {
+        const int* a_i[3] = {(int *)&count, iarray_of_blocklengths, iarray_of_displacements};
+
+        ompi_datatype_set_args( *newtype, 2 * count + 1, a_i, 0, NULL, 1, &oldtype,
+                                MPI_COMBINER_INDEXED );
+    }
+#if OMPI_BIGCOUNT_SRC
+    free(iarray_of_blocklengths);
+    free(iarray_of_displacements);
+#endif
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/type_match_size.c b/ompi/mpi/c/type_match_size.c
deleted file mode 100644
index d34f4825960..00000000000
--- a/ompi/mpi/c/type_match_size.c
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2009      Sun Microsystems, Inc. All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/datatype/ompi_datatype_internal.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_match_size = PMPI_Type_match_size
-#endif
-#define MPI_Type_match_size PMPI_Type_match_size
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_match_size";
-
-
-int MPI_Type_match_size(int typeclass, int size, MPI_Datatype *type)
-{
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-    }
-
-    switch( typeclass ) {
-    case MPI_TYPECLASS_REAL:
-        *type = (MPI_Datatype)ompi_datatype_match_size( size, OMPI_DATATYPE_FLAG_DATA_FLOAT, OMPI_DATATYPE_FLAG_DATA_FORTRAN );
-        break;
-    case MPI_TYPECLASS_INTEGER:
-        *type = (MPI_Datatype)ompi_datatype_match_size( size, OMPI_DATATYPE_FLAG_DATA_INT, OMPI_DATATYPE_FLAG_DATA_FORTRAN );
-        break;
-    case MPI_TYPECLASS_COMPLEX:
-        *type = (MPI_Datatype)ompi_datatype_match_size( size, OMPI_DATATYPE_FLAG_DATA_COMPLEX, OMPI_DATATYPE_FLAG_DATA_FORTRAN );
-        break;
-    default:
-        *type = &ompi_mpi_datatype_null.dt;
-    }
-
-    if( *type != &ompi_mpi_datatype_null.dt ) {
-        return MPI_SUCCESS;
-    }
-
-    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/type_match_size.c.in b/ompi/mpi/c/type_match_size.c.in
new file mode 100644
index 00000000000..cbb94a3305b
--- /dev/null
+++ b/ompi/mpi/c/type_match_size.c.in
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2009      Sun Microsystems, Inc. All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/datatype/ompi_datatype_internal.h"
+
+PROTOTYPE ERROR_CLASS type_match_size(INT typeclass, INT size, DATATYPE_OUT type)
+{
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+    }
+
+    switch( typeclass ) {
+    case MPI_TYPECLASS_REAL:
+        *type = (MPI_Datatype)ompi_datatype_match_size( size, OMPI_DATATYPE_FLAG_DATA_FLOAT, OMPI_DATATYPE_FLAG_DATA_FORTRAN );
+        break;
+    case MPI_TYPECLASS_INTEGER:
+        *type = (MPI_Datatype)ompi_datatype_match_size( size, OMPI_DATATYPE_FLAG_DATA_INT, OMPI_DATATYPE_FLAG_DATA_FORTRAN );
+        break;
+    case MPI_TYPECLASS_COMPLEX:
+        *type = (MPI_Datatype)ompi_datatype_match_size( size, OMPI_DATATYPE_FLAG_DATA_COMPLEX, OMPI_DATATYPE_FLAG_DATA_FORTRAN );
+        break;
+    default:
+        *type = &ompi_mpi_datatype_null.dt;
+    }
+
+    if( *type != &ompi_mpi_datatype_null.dt ) {
+        return MPI_SUCCESS;
+    }
+
+    return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/type_set_attr.c b/ompi/mpi/c/type_set_attr.c
deleted file mode 100644
index 6a4b2b3b27e..00000000000
--- a/ompi/mpi/c/type_set_attr.c
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/attribute/attribute.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_set_attr = PMPI_Type_set_attr
-#endif
-#define MPI_Type_set_attr PMPI_Type_set_attr
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_set_attr";
-
-
-int MPI_Type_set_attr (MPI_Datatype type,
-                       int type_keyval,
-                       void *attribute_val)
-{
-    int ret;
-
-    MEMCHECKER(
-        memchecker_datatype(type);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == type || MPI_DATATYPE_NULL == type) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE, FUNC_NAME);
-        }
-    }
-
-    ret = ompi_attr_set_c(TYPE_ATTR, type, &type->d_keyhash,
-                          type_keyval, attribute_val, false);
-
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(ret, 
-			  MPI_ERR_OTHER, FUNC_NAME);
-
-}
diff --git a/ompi/mpi/c/type_set_attr.c.in b/ompi/mpi/c/type_set_attr.c.in
new file mode 100644
index 00000000000..9c3c88ea20a
--- /dev/null
+++ b/ompi/mpi/c/type_set_attr.c.in
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/attribute/attribute.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS type_set_attr (DATATYPE type, INT type_keyval,
+                                     BUFFER_OUT attribute_val)
+{
+    int ret;
+
+    MEMCHECKER(
+        memchecker_datatype(type);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == type || MPI_DATATYPE_NULL == type) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE, FUNC_NAME);
+        }
+    }
+
+    ret = ompi_attr_set_c(TYPE_ATTR, type, &type->d_keyhash,
+                          type_keyval, attribute_val, false);
+
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(ret, 
+			  MPI_ERR_OTHER, FUNC_NAME);
+
+}
diff --git a/ompi/mpi/c/type_set_name.c b/ompi/mpi/c/type_set_name.c
deleted file mode 100644
index 86cb18d94a8..00000000000
--- a/ompi/mpi/c/type_set_name.c
+++ /dev/null
@@ -1,66 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2018      Cisco Systems, Inc.  All rights reserved
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <string.h>
-
-#include "opal/util/string_copy.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_set_name = PMPI_Type_set_name
-#endif
-#define MPI_Type_set_name PMPI_Type_set_name
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_set_name";
-
-
-int MPI_Type_set_name (MPI_Datatype type, const char *type_name)
-{
-    MEMCHECKER(
-        memchecker_datatype(type);
-        );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == type || MPI_DATATYPE_NULL == type) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE, FUNC_NAME);
-        } else if (NULL == type_name) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-        }
-    }
-
-    memset(type->name, 0, MPI_MAX_OBJECT_NAME);
-    opal_string_copy( type->name, type_name, MPI_MAX_OBJECT_NAME);
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/type_set_name.c.in b/ompi/mpi/c/type_set_name.c.in
new file mode 100644
index 00000000000..6414bb4bb46
--- /dev/null
+++ b/ompi/mpi/c/type_set_name.c.in
@@ -0,0 +1,58 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018      Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include <string.h>
+
+#include "opal/util/string_copy.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS type_set_name (DATATYPE type, STRING type_name)
+{
+    MEMCHECKER(
+        memchecker_datatype(type);
+        );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == type || MPI_DATATYPE_NULL == type) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE, FUNC_NAME);
+        } else if (NULL == type_name) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+        }
+    }
+
+    memset(type->name, 0, MPI_MAX_OBJECT_NAME);
+    opal_string_copy( type->name, type_name, MPI_MAX_OBJECT_NAME);
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/type_size.c b/ompi/mpi/c/type_size.c
deleted file mode 100644
index 9d709624f49..00000000000
--- a/ompi/mpi/c/type_size.c
+++ /dev/null
@@ -1,64 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2009      Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_size = PMPI_Type_size
-#endif
-#define MPI_Type_size PMPI_Type_size
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_size";
-
-int MPI_Type_size(MPI_Datatype type, int *size)
-{
-    size_t type_size;
-    MEMCHECKER(
-        memchecker_datatype(type);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == type || MPI_DATATYPE_NULL == type) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE, FUNC_NAME);
-        } else if (NULL == size) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-        }
-    }
-
-    opal_datatype_type_size ( &type->super, &type_size);
-
-    *size = (type_size > (size_t) INT_MAX) ? MPI_UNDEFINED : (int) type_size;
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/type_size.c.in b/ompi/mpi/c/type_size.c.in
new file mode 100644
index 00000000000..6e0897dd7aa
--- /dev/null
+++ b/ompi/mpi/c/type_size.c.in
@@ -0,0 +1,57 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2009      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS type_size(DATATYPE type, COUNT_OUT size)
+{
+    size_t type_size;
+    MEMCHECKER(
+        memchecker_datatype(type);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == type || MPI_DATATYPE_NULL == type) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE, FUNC_NAME);
+        } else if (NULL == size) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+        }
+    }
+
+    opal_datatype_type_size ( &type->super, &type_size);
+
+    *size = (type_size > (size_t) INT_MAX) ? MPI_UNDEFINED : (int) type_size;
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/type_size_x.c b/ompi/mpi/c/type_size_x.c
deleted file mode 100644
index 8a10be03606..00000000000
--- a/ompi/mpi/c/type_size_x.c
+++ /dev/null
@@ -1,64 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2009      Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_size_x = PMPI_Type_size_x
-#endif
-#define MPI_Type_size_x PMPI_Type_size_x
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_size_x";
-
-int MPI_Type_size_x(MPI_Datatype type, MPI_Count *size)
-{
-    size_t type_size;
-    MEMCHECKER(
-        memchecker_datatype(type);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == type || MPI_DATATYPE_NULL == type) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE, FUNC_NAME);
-        } else if (NULL == size) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-        }
-    }
-
-    opal_datatype_type_size ( &type->super, &type_size);
-
-    *size = (type_size > (size_t) MPI_COUNT_MAX) ? MPI_UNDEFINED : (MPI_Count) type_size;
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/type_size_x.c.in b/ompi/mpi/c/type_size_x.c.in
new file mode 100644
index 00000000000..ded1005acc6
--- /dev/null
+++ b/ompi/mpi/c/type_size_x.c.in
@@ -0,0 +1,57 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2009      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS type_size_x(DATATYPE type, ELEMENT_COUNT size)
+{
+    size_t type_size;
+    MEMCHECKER(
+        memchecker_datatype(type);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == type || MPI_DATATYPE_NULL == type) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE, FUNC_NAME);
+        } else if (NULL == size) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+        }
+    }
+
+    opal_datatype_type_size ( &type->super, &type_size);
+
+    *size = (type_size > (size_t) MPI_COUNT_MAX) ? MPI_UNDEFINED : (MPI_Count) type_size;
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/type_vector.c b/ompi/mpi/c/type_vector.c
deleted file mode 100644
index 12524f26271..00000000000
--- a/ompi/mpi/c/type_vector.c
+++ /dev/null
@@ -1,79 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Type_vector = PMPI_Type_vector
-#endif
-#define MPI_Type_vector PMPI_Type_vector
-#endif
-
-static const char FUNC_NAME[] = "MPI_Type_vector";
-
-int MPI_Type_vector(int count,
-                    int blocklength,
-                    int stride,
-                    MPI_Datatype oldtype,
-                    MPI_Datatype *newtype)
-{
-    int rc;
-
-    MEMCHECKER(
-        memchecker_datatype(oldtype);
-        );
-
-    if( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == oldtype || MPI_DATATYPE_NULL == oldtype ||
-            NULL == newtype) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
-                                          FUNC_NAME );
-        } else  if( count < 0 ) {
-            OMPI_ERRHANDLER_NOHANDLE_RETURN( MPI_ERR_COUNT, 
-                                    MPI_ERR_COUNT, FUNC_NAME );
-        } else if( blocklength < 0) {
-            OMPI_ERRHANDLER_NOHANDLE_RETURN( MPI_ERR_ARG, 
-                                    MPI_ERR_ARG, FUNC_NAME );
-        }
-    }
-
-    rc = ompi_datatype_create_vector ( count, blocklength, stride, oldtype, newtype );
-    OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME );
-
-    {
-        const int* a_i[3] = {&count, &blocklength, &stride};
-
-        ompi_datatype_set_args( *newtype, 3, a_i, 0, NULL, 1, &oldtype, MPI_COMBINER_VECTOR );
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/type_vector.c.in b/ompi/mpi/c/type_vector.c.in
new file mode 100644
index 00000000000..08bad80d75f
--- /dev/null
+++ b/ompi/mpi/c/type_vector.c.in
@@ -0,0 +1,83 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/memchecker.h"
+
+/*
+ * TODO:BIGCOUNT this file will need to be updated once
+ * the datatype framework supports bigcount 
+ */
+
+PROTOTYPE ERROR_CLASS type_vector(COUNT count,
+                                  COUNT blocklength,
+                                  COUNT stride,
+                                  DATATYPE oldtype,
+                                  DATATYPE_OUT newtype)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_datatype(oldtype);
+        );
+
+    if( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == oldtype || MPI_DATATYPE_NULL == oldtype ||
+            NULL == newtype) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TYPE,
+                                          FUNC_NAME );
+        } else  if( count < 0 ) {
+            OMPI_ERRHANDLER_NOHANDLE_RETURN( MPI_ERR_COUNT, 
+                                    MPI_ERR_COUNT, FUNC_NAME );
+        } else if( blocklength < 0) {
+            OMPI_ERRHANDLER_NOHANDLE_RETURN( MPI_ERR_ARG, 
+                                    MPI_ERR_ARG, FUNC_NAME );
+        }
+#if OMPI_BIGCOUNT_SRC
+        OMPI_CHECK_MPI_COUNT_INT_CONVERSION_OVERFLOW(rc, count);
+        if (OMPI_SUCCESS != rc) {
+            OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME);
+        }
+#endif
+    }
+
+    rc = ompi_datatype_create_vector ( count, blocklength, stride, oldtype, newtype );
+    OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME );
+
+    {
+        const int* a_i[3] = {(int *)&count, (int *)&blocklength, (int *)&stride};
+
+        ompi_datatype_set_args( *newtype, 3, a_i, 0, NULL, 1, &oldtype, MPI_COMBINER_VECTOR );
+    }
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/unpack.c b/ompi/mpi/c/unpack.c
deleted file mode 100644
index b9f5e7b8781..00000000000
--- a/ompi/mpi/c/unpack.c
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2013 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2021      IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "opal/datatype/opal_convertor.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Unpack = PMPI_Unpack
-#endif
-#define MPI_Unpack PMPI_Unpack
-#endif
-
-static const char FUNC_NAME[] = "MPI_Unpack";
-
-
-int MPI_Unpack(const void *inbuf, int insize, int *position,
-               void *outbuf, int outcount, MPI_Datatype datatype,
-               MPI_Comm comm)
-{
-    int rc = MPI_SUCCESS;
-    opal_convertor_t local_convertor;
-    struct iovec outvec;
-    unsigned int iov_count;
-    size_t size;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_call(&opal_memchecker_base_isdefined, inbuf, insize, MPI_PACKED);
-        memchecker_call(&opal_memchecker_base_isaddressable, outbuf, outcount, datatype);
-        memchecker_comm(comm);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-
-        if ((NULL == inbuf) || (NULL == position)) {  /* outbuf can be MPI_BOTTOM */
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-
-        if (outcount < 0) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-        }
-
-        OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, outcount);
-        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-        OMPI_CHECK_USER_BUFFER(rc, outbuf, datatype, outcount);
-        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-    }
-
-   /*
-    * If a datatype's description contains a single element that describes
-    * a large vector that path is reasonably optimized in pack/unpack. On
-    * the other hand if the count and datatype combined describe the same
-    * vector that is processed one element at a time.
-    *
-    * So at the top level we morph the call if the count and datatype look
-    * like a good vector.
-    */
-    ompi_datatype_consolidate_t dtmod;
-    rc = ompi_datatype_consolidate_create(outcount, datatype, &dtmod,
-        OMPI_DATATYPE_CONSOLIDATE_THRESHOLD);
-    OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
-
-    if( insize > 0 ) {
-        int ret;
-        OBJ_CONSTRUCT( &local_convertor, opal_convertor_t );
-        /* the resulting convertor will be set the the position ZERO */
-        opal_convertor_copy_and_prepare_for_recv( ompi_mpi_local_convertor,
-                                                  &(dtmod.dt->super), dtmod.count,
-                                                  outbuf, 0, &local_convertor );
-
-        /* Check for truncation */
-        opal_convertor_get_packed_size( &local_convertor, &size );
-        if( (*position + size) > (unsigned int)insize ) {
-            OBJ_DESTRUCT( &local_convertor );
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TRUNCATE, FUNC_NAME);
-        }
-
-        /* Prepare the iovec with all informations */
-        outvec.iov_base = (char*) inbuf + (*position);
-        outvec.iov_len = size;
-
-        /* Do the actual unpacking */
-        iov_count = 1;
-        ret = opal_convertor_unpack( &local_convertor, &outvec, &iov_count, &size );
-        *position += size;
-        OBJ_DESTRUCT( &local_convertor );
-        /* All done.  Note that the convertor returns 1 upon success, not
-           OPAL_SUCCESS. */
-        if (1 != ret) {
-            rc = OMPI_ERROR;
-        }
-    }
-
-    rc = ompi_datatype_consolidate_free(&dtmod);
-
-    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/unpack.c.in b/ompi/mpi/c/unpack.c.in
new file mode 100644
index 00000000000..94055ceef94
--- /dev/null
+++ b/ompi/mpi/c/unpack.c.in
@@ -0,0 +1,125 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2013 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2018 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2021      IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "opal/datatype/opal_convertor.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS unpack(BUFFER inbuf, 
+		             COUNT insize, 
+ 		             COUNT_OUT position,
+                             BUFFER_OUT outbuf, 
+			     COUNT outcount, DATATYPE datatype,
+                             COMM comm)
+{
+    int rc = MPI_SUCCESS, rc_keep = MPI_SUCCESS;
+    opal_convertor_t local_convertor;
+    struct iovec outvec;
+    unsigned int iov_count;
+    size_t size;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_call(&opal_memchecker_base_isdefined, inbuf, insize, MPI_PACKED);
+        memchecker_call(&opal_memchecker_base_isaddressable, outbuf, outcount, datatype);
+        memchecker_comm(comm);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+        }
+
+        if ((NULL == inbuf) || (NULL == position)) {  /* outbuf can be MPI_BOTTOM */
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
+        }
+
+        if (outcount < 0) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
+        }
+
+        OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, outcount);
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+        OMPI_CHECK_USER_BUFFER(rc, outbuf, datatype, outcount);
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+   /*
+    * If a datatype's description contains a single element that describes
+    * a large vector that path is reasonably optimized in pack/unpack. On
+    * the other hand if the count and datatype combined describe the same
+    * vector that is processed one element at a time.
+    *
+    * So at the top level we morph the call if the count and datatype look
+    * like a good vector.
+    */
+    ompi_datatype_consolidate_t dtmod;
+    rc = ompi_datatype_consolidate_create(outcount, datatype, &dtmod,
+        OMPI_DATATYPE_CONSOLIDATE_THRESHOLD);
+    OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+
+    if( insize > 0 ) {
+        int ret;
+        OBJ_CONSTRUCT( &local_convertor, opal_convertor_t );
+        /* the resulting convertor will be set the the position ZERO */
+        opal_convertor_copy_and_prepare_for_recv( ompi_mpi_local_convertor,
+                                                  &(dtmod.dt->super), dtmod.count,
+                                                  outbuf, 0, &local_convertor );
+
+        /* Check for truncation */
+        opal_convertor_get_packed_size( &local_convertor, &size );
+        if( (*position + size) > (unsigned int)insize ) {
+            OBJ_DESTRUCT( &local_convertor );
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TRUNCATE, FUNC_NAME);
+        }
+
+        /* Prepare the iovec with all information */
+        outvec.iov_base = (char*) inbuf + (*position);
+        outvec.iov_len = size;
+
+        /* Do the actual unpacking */
+        iov_count = 1;
+        ret = opal_convertor_unpack( &local_convertor, &outvec, &iov_count, &size );
+        *position += size;
+        OBJ_DESTRUCT( &local_convertor );
+        /* All done.  Note that the convertor returns 1 upon success, not
+           OPAL_SUCCESS. */
+        if (1 != ret) {
+            rc_keep = OMPI_ERROR;
+        }
+    }
+
+    rc = ompi_datatype_consolidate_free(&dtmod);
+    rc = (rc_keep != MPI_SUCCESS) ? rc_keep : rc;
+
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/unpack_external.c b/ompi/mpi/c/unpack_external.c
deleted file mode 100644
index 579f8b912f1..00000000000
--- a/ompi/mpi/c/unpack_external.c
+++ /dev/null
@@ -1,73 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2017 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "opal/datatype/opal_convertor.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Unpack_external = PMPI_Unpack_external
-#endif
-#define MPI_Unpack_external PMPI_Unpack_external
-#endif
-
-static const char FUNC_NAME[] = "MPI_Unpack_external";
-
-
-int MPI_Unpack_external (const char datarep[], const void *inbuf, MPI_Aint insize,
-                         MPI_Aint *position, void *outbuf, int outcount,
-                         MPI_Datatype datatype)
-{
-    int rc = MPI_SUCCESS;
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_call(&opal_memchecker_base_isdefined, outbuf, outcount, datatype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if ((NULL == inbuf) || (NULL == position)) {  /* outbuf can be MPI_BOTTOM */
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
-        } else if (outcount < 0) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COUNT, FUNC_NAME);
-        }
-        OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, outcount);
-        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
-        OMPI_CHECK_USER_BUFFER(rc, outbuf, datatype, outcount);
-        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
-    }
-
-    rc = ompi_datatype_unpack_external(datarep, inbuf, insize,
-                                       position, outbuf, outcount,
-                                       datatype);
-
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/unpack_external.c.in b/ompi/mpi/c/unpack_external.c.in
new file mode 100644
index 00000000000..645031cfd32
--- /dev/null
+++ b/ompi/mpi/c/unpack_external.c.in
@@ -0,0 +1,64 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2017 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "opal/datatype/opal_convertor.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS unpack_external (STRING datarep, BUFFER inbuf, AINT_COUNT insize,
+                                       AINT_COUNT_OUT position, BUFFER_OUT outbuf, COUNT outcount,
+                                       DATATYPE datatype)
+{
+    int rc = MPI_SUCCESS;
+
+    MEMCHECKER(
+        memchecker_datatype(datatype);
+        memchecker_call(&opal_memchecker_base_isdefined, outbuf, outcount, datatype);
+    );
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if ((NULL == inbuf) || (NULL == position)) {  /* outbuf can be MPI_BOTTOM */
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG, FUNC_NAME);
+        } else if (outcount < 0) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COUNT, FUNC_NAME);
+        }
+        OMPI_CHECK_DATATYPE_FOR_RECV(rc, datatype, outcount);
+        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
+        OMPI_CHECK_USER_BUFFER(rc, outbuf, datatype, outcount);
+        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
+    }
+
+    rc = ompi_datatype_unpack_external(datarep, inbuf, insize,
+                                       (MPI_Aint *)position, outbuf, outcount,
+                                       datatype);
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(rc, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/unpublish_name.c b/ompi/mpi/c/unpublish_name.c
deleted file mode 100644
index db03e70c39a..00000000000
--- a/ompi/mpi/c/unpublish_name.c
+++ /dev/null
@@ -1,125 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2019 Intel, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2015      Cisco Systems, Inc.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "opal/class/opal_list.h"
-#include "opal/mca/pmix/pmix-internal.h"
-#include "opal/util/argv.h"
-#include "opal/util/show_help.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Unpublish_name = PMPI_Unpublish_name
-#endif
-#define MPI_Unpublish_name PMPI_Unpublish_name
-#endif
-
-static const char FUNC_NAME[] = "MPI_Unpublish_name";
-
-
-int MPI_Unpublish_name(const char *service_name, MPI_Info info,
-                       const char *port_name)
-{
-    int ret;
-    char range[OPAL_MAX_INFO_VAL];
-    int flag=0;
-    pmix_status_t rc;
-    pmix_info_t pinfo;
-    pmix_data_range_t rng = PMIX_RANGE_SESSION;
-    char **keys = NULL;
-
-    if ( MPI_PARAM_CHECK ) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if ( NULL == port_name ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-        if ( NULL == service_name ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-        if (NULL == info || ompi_info_is_freed(info)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
-                                          FUNC_NAME);
-        }
-    }
-
-    /* OMPI supports info keys to pass the range to
-     * be searched for the given key */
-    if (MPI_INFO_NULL != info) {
-        ompi_info_get (info, "range", sizeof(range) - 1, range, &flag);
-        if (flag) {
-            if (0 == strcmp(range, "nspace")) {
-                rng = PMIX_RANGE_NAMESPACE;  // share only with procs in same nspace
-            } else if (0 == strcmp(range, "session")) {
-                rng = PMIX_RANGE_SESSION; // share only with procs in same session
-            } else {
-                /* unrecognized scope */
-                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                            FUNC_NAME);
-            }
-        }
-    }
-
-    /* unpublish the service_name */
-    opal_argv_append_nosize(&keys, service_name);
-    PMIX_INFO_LOAD(&pinfo, PMIX_RANGE, &rng, PMIX_DATA_RANGE);
-
-    rc = PMIx_Unpublish(keys, &pinfo, 1);
-    opal_argv_free(keys);
-    PMIX_INFO_DESTRUCT(&pinfo);
-
-    if ( PMIX_SUCCESS != rc ) {
-        if (PMIX_ERR_NOT_FOUND == rc) {
-            /* service couldn't be found */
-            ret = MPI_ERR_SERVICE;
-        } else if (PMIX_ERR_NO_PERMISSIONS == rc) {
-            /* this process didn't own the specified service */
-            ret = MPI_ERR_ACCESS;
-        } else if (PMIX_ERR_NOT_SUPPORTED == rc) {
-            /* this PMIX environment doesn't support publishing */
-            ret = OMPI_ERR_NOT_SUPPORTED;
-            opal_show_help("help-mpi-api.txt",
-                           "MPI function not supported",
-                           true,
-                           FUNC_NAME,
-                           "Underlying runtime environment does not support name publishing functionality");
-        } else {
-            ret = MPI_ERR_INTERN;
-        }
-
-        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(ret, FUNC_NAME);
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/unpublish_name.c.in b/ompi/mpi/c/unpublish_name.c.in
new file mode 100644
index 00000000000..ca4aa92f7a8
--- /dev/null
+++ b/ompi/mpi/c/unpublish_name.c.in
@@ -0,0 +1,118 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "opal/class/opal_list.h"
+#include "opal/mca/pmix/pmix-internal.h"
+#include "opal/util/argv.h"
+#include "opal/util/show_help.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+
+PROTOTYPE ERROR_CLASS unpublish_name(STRING service_name, INFO info,
+                                     STRING port_name)
+{
+    int ret;
+    opal_cstring_t *info_str;
+    int flag=0;
+    pmix_status_t rc;
+    pmix_info_t pinfo;
+    pmix_data_range_t rng = PMIX_RANGE_SESSION;
+    char **keys = NULL;
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if ( NULL == port_name ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+        if ( NULL == service_name ) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+        if (NULL == info || ompi_info_is_freed(info)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INFO,
+                                          FUNC_NAME);
+        }
+    }
+
+    /* OMPI supports info keys to pass the range to
+     * be searched for the given key */
+    if (MPI_INFO_NULL != info) {
+        ompi_info_get (info, "range", &info_str, &flag);
+        if (flag) {
+            if (0 == strcmp(info_str->string, "nspace")) {
+                rng = PMIX_RANGE_NAMESPACE;  // share only with procs in same nspace
+            } else if (0 == strcmp(info_str->string, "session")) {
+                rng = PMIX_RANGE_SESSION; // share only with procs in same session
+            } else {
+                /* unrecognized scope */
+                return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                            FUNC_NAME);
+            }
+            OBJ_RELEASE(info_str);
+        }
+    }
+
+    /* unpublish the service_name */
+    opal_argv_append_nosize(&keys, service_name);
+    PMIX_INFO_LOAD(&pinfo, PMIX_RANGE, &rng, PMIX_DATA_RANGE);
+
+    rc = PMIx_Unpublish(keys, &pinfo, 1);
+    opal_argv_free(keys);
+    PMIX_INFO_DESTRUCT(&pinfo);
+
+    if ( PMIX_SUCCESS != rc ) {
+        if (PMIX_ERR_NOT_FOUND == rc) {
+            /* service couldn't be found */
+            ret = MPI_ERR_SERVICE;
+        } else if (PMIX_ERR_NO_PERMISSIONS == rc) {
+            /* this process didn't own the specified service */
+            ret = MPI_ERR_ACCESS;
+        } else if (PMIX_ERR_NOT_SUPPORTED == rc) {
+            /* this PMIX environment doesn't support publishing */
+            ret = OMPI_ERR_NOT_SUPPORTED;
+            opal_show_help("help-mpi-api.txt",
+                           "MPI function not supported",
+                           true,
+                           FUNC_NAME,
+                           "Underlying runtime environment does not support name publishing functionality");
+        } else {
+            ret = MPI_ERR_INTERN;
+        }
+
+        return OMPI_ERRHANDLER_NOHANDLE_INVOKE(ret, FUNC_NAME);
+    }
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/wait.c b/ompi/mpi/c/wait.c
deleted file mode 100644
index aad340cc31a..00000000000
--- a/ompi/mpi/c/wait.c
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015-2017 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/request/request.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Wait = PMPI_Wait
-#endif
-#define MPI_Wait PMPI_Wait
-#endif
-
-static const char FUNC_NAME[] = "MPI_Wait";
-
-
-int MPI_Wait(MPI_Request *request, MPI_Status *status)
-{
-    SPC_RECORD(OMPI_SPC_WAIT, 1);
-
-    MEMCHECKER(
-        memchecker_request(request);
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        int rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (request == NULL) {
-            rc = MPI_ERR_REQUEST;
-        }
-        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
-    }
-
-    if (MPI_REQUEST_NULL == *request) {
-        if (MPI_STATUS_IGNORE != status) {
-            *status = ompi_status_empty;
-            /*
-             * Per MPI-1, the MPI_ERROR field is not defined for single-completion calls
-             */
-            MEMCHECKER(
-                opal_memchecker_base_mem_undefined(&status->MPI_ERROR, sizeof(int));
-            );
-        }
-        return MPI_SUCCESS;
-    }
-
-    if (OMPI_SUCCESS == ompi_request_wait(request, status)) {
-        /*
-         * Per MPI-1, the MPI_ERROR field is not defined for single-completion calls
-         */
-        MEMCHECKER(
-            if (MPI_STATUS_IGNORE != status) {
-                opal_memchecker_base_mem_undefined(&status->MPI_ERROR, sizeof(int));
-            }
-        );
-        return MPI_SUCCESS;
-    }
-
-    MEMCHECKER(
-        if (MPI_STATUS_IGNORE != status) {
-            opal_memchecker_base_mem_undefined(&status->MPI_ERROR, sizeof(int));
-        }
-    );
-    return ompi_errhandler_request_invoke(1, request, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/wait.c.in b/ompi/mpi/c/wait.c.in
new file mode 100644
index 00000000000..d2db5ab726f
--- /dev/null
+++ b/ompi/mpi/c/wait.c.in
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2017 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/request/request.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS wait(REQUEST_INOUT request, STATUS_OUT status)
+{
+    SPC_RECORD(OMPI_SPC_WAIT, 1);
+
+    MEMCHECKER(
+        memchecker_request(request);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        int rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (request == NULL) {
+            rc = MPI_ERR_REQUEST;
+        }
+        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
+    }
+
+    if (MPI_REQUEST_NULL == *request) {
+        if (MPI_STATUS_IGNORE != status) {
+            OMPI_COPY_STATUS(status, ompi_status_empty, false);
+            /*
+             * Per MPI-1, the MPI_ERROR field is not defined for single-completion calls
+             */
+            MEMCHECKER(
+                opal_memchecker_base_mem_undefined(&status->MPI_ERROR, sizeof(int));
+            );
+        }
+        return MPI_SUCCESS;
+    }
+
+    if (OMPI_SUCCESS == ompi_request_wait(request, status)) {
+        /*
+         * Per MPI-1, the MPI_ERROR field is not defined for single-completion calls
+         */
+        MEMCHECKER(
+            if (MPI_STATUS_IGNORE != status) {
+                opal_memchecker_base_mem_undefined(&status->MPI_ERROR, sizeof(int));
+            }
+        );
+        return MPI_SUCCESS;
+    }
+
+    MEMCHECKER(
+        if (MPI_STATUS_IGNORE != status) {
+            opal_memchecker_base_mem_undefined(&status->MPI_ERROR, sizeof(int));
+        }
+    );
+    return ompi_errhandler_request_invoke(1, request, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/waitall.c b/ompi/mpi/c/waitall.c
deleted file mode 100644
index 7b12e20cb89..00000000000
--- a/ompi/mpi/c/waitall.c
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/request/request.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Waitall = PMPI_Waitall
-#endif
-#define MPI_Waitall PMPI_Waitall
-#endif
-
-static const char FUNC_NAME[] = "MPI_Waitall";
-
-
-int MPI_Waitall(int count, MPI_Request requests[], MPI_Status statuses[])
-{
-    SPC_RECORD(OMPI_SPC_WAITALL, 1);
-
-    MEMCHECKER(
-        int j;
-        for (j = 0; j < count; j++){
-            memchecker_request(&requests[j]);
-        }
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        int i, rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if( (NULL == requests) && (0 != count) ) {
-            rc = MPI_ERR_REQUEST;
-        } else {
-            for (i = 0; i < count; i++) {
-                if (NULL == requests[i]) {
-                    rc = MPI_ERR_REQUEST;
-                    break;
-                }
-            }
-        }
-        if (count < 0) {
-            rc = MPI_ERR_ARG;
-        }
-        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
-    }
-
-    if (OPAL_UNLIKELY(0 == count)) {
-        return MPI_SUCCESS;
-    }
-
-    if (OMPI_SUCCESS == ompi_request_wait_all(count, requests, statuses)) {
-        return MPI_SUCCESS;
-    }
-
-    if (MPI_SUCCESS !=
-        ompi_errhandler_request_invoke(count, requests, FUNC_NAME)) {
-        return MPI_ERR_IN_STATUS;
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/waitall.c.in b/ompi/mpi/c/waitall.c.in
new file mode 100644
index 00000000000..b2f3c4fe7a1
--- /dev/null
+++ b/ompi/mpi/c/waitall.c.in
@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2021-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/request/request.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS waitall(INT count, REQUEST_INOUT requests:count,
+                              STATUS_OUT statuses:count)
+{
+    SPC_RECORD(OMPI_SPC_WAITALL, 1);
+
+    MEMCHECKER(
+        int j;
+        for (j = 0; j < count; j++){
+            memchecker_request(&requests[j]);
+        }
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        int rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if( (NULL == requests) && (0 != count) ) {
+            rc = MPI_ERR_REQUEST;
+        } else {
+            if(!ompi_request_check_same_instance(requests, count) ) {
+                rc = MPI_ERR_REQUEST;
+            }
+        }
+        if (count < 0) {
+            rc = MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
+    }
+
+    if (OPAL_UNLIKELY(0 == count)) {
+        return MPI_SUCCESS;
+    }
+
+    if (OMPI_SUCCESS == ompi_request_wait_all(count, requests, statuses)) {
+        return MPI_SUCCESS;
+    }
+
+    if (MPI_SUCCESS !=
+        ompi_errhandler_request_invoke(count, requests, FUNC_NAME)) {
+        return MPI_ERR_IN_STATUS;
+    }
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/waitany.c b/ompi/mpi/c/waitany.c
deleted file mode 100644
index 5878d7d5c35..00000000000
--- a/ompi/mpi/c/waitany.c
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
- * Copyright (c) 2012      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2014-2015 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/request/request.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Waitany = PMPI_Waitany
-#endif
-#define MPI_Waitany PMPI_Waitany
-#endif
-
-static const char FUNC_NAME[] = "MPI_Waitany";
-
-
-int MPI_Waitany(int count, MPI_Request requests[], int *indx, MPI_Status *status)
-{
-    SPC_RECORD(OMPI_SPC_WAITANY, 1);
-
-    MEMCHECKER(
-        int j;
-        for (j = 0; j < count; j++){
-            memchecker_request(&requests[j]);
-        }
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        int i, rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if ((NULL == requests) && (0 != count)) {
-            rc = MPI_ERR_REQUEST;
-        } else {
-            for (i = 0; i < count; i++) {
-                if (NULL == requests[i]) {
-                    rc = MPI_ERR_REQUEST;
-                    break;
-                }
-            }
-        }
-        if ((NULL == indx && count > 0) ||
-            count < 0) {
-            rc = MPI_ERR_ARG;
-        }
-        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
-    }
-
-    if (OPAL_UNLIKELY(0 == count)) {
-        *indx = MPI_UNDEFINED;
-        if (MPI_STATUS_IGNORE != status) {
-            *status = ompi_status_empty;
-        }
-        return MPI_SUCCESS;
-    }
-
-    if (OMPI_SUCCESS == ompi_request_wait_any(count, requests, indx, status)) {
-        return MPI_SUCCESS;
-    }
-
-    return ompi_errhandler_request_invoke(count, requests, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/waitany.c.in b/ompi/mpi/c/waitany.c.in
new file mode 100644
index 00000000000..5d2fb4424eb
--- /dev/null
+++ b/ompi/mpi/c/waitany.c.in
@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2012      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2015 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2021-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/request/request.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS waitany(INT count, REQUEST_INOUT requests, INT_OUT indx, STATUS_OUT status)
+{
+    SPC_RECORD(OMPI_SPC_WAITANY, 1);
+
+    MEMCHECKER(
+        int j;
+        for (j = 0; j < count; j++){
+            memchecker_request(&requests[j]);
+        }
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        int rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if ((NULL == requests) && (0 != count)) {
+            rc = MPI_ERR_REQUEST;
+        } else {
+            if(!ompi_request_check_same_instance(requests, count) ) {
+                rc = MPI_ERR_REQUEST;
+            }
+        }
+        if ((NULL == indx && count > 0) ||
+            count < 0) {
+            rc = MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
+    }
+
+    if (OPAL_UNLIKELY(0 == count)) {
+        *indx = MPI_UNDEFINED;
+        if (MPI_STATUS_IGNORE != status) {
+            OMPI_COPY_STATUS(status, ompi_status_empty, false);
+        }
+        return MPI_SUCCESS;
+    }
+
+    if (OMPI_SUCCESS == ompi_request_wait_any(count, requests, indx, status)) {
+        return MPI_SUCCESS;
+    }
+
+    return ompi_errhandler_request_invoke(count, requests, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/waitsome.c b/ompi/mpi/c/waitsome.c
deleted file mode 100644
index b6beb5da752..00000000000
--- a/ompi/mpi/c/waitsome.c
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
- * Copyright (c) 2012      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/request/request.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Waitsome = PMPI_Waitsome
-#endif
-#define MPI_Waitsome PMPI_Waitsome
-#endif
-
-static const char FUNC_NAME[] = "MPI_Waitsome";
-
-
-int MPI_Waitsome(int incount, MPI_Request requests[],
-                 int *outcount, int indices[],
-                 MPI_Status statuses[])
-{
-    SPC_RECORD(OMPI_SPC_WAITSOME, 1);
-
-    MEMCHECKER(
-        int j;
-        for (j = 0; j < incount; j++){
-            memchecker_request(&requests[j]);
-        }
-    );
-
-    if ( MPI_PARAM_CHECK ) {
-        int indx, rc = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if ((NULL == requests) && (0 != incount)) {
-            rc = MPI_ERR_REQUEST;
-        } else {
-            for (indx = 0; indx < incount; ++indx) {
-                if (NULL == requests[indx]) {
-                    rc = MPI_ERR_REQUEST;
-                    break;
-                }
-            }
-        }
-        if (((NULL == outcount || NULL == indices) && incount > 0) ||
-            incount < 0) {
-            rc = MPI_ERR_ARG;
-        }
-        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
-    }
-
-    if (OPAL_UNLIKELY(0 == incount)) {
-        *outcount = MPI_UNDEFINED;
-        return MPI_SUCCESS;
-    }
-
-    if (OMPI_SUCCESS == ompi_request_wait_some( incount, requests,
-                                                outcount, indices, statuses )) {
-        return MPI_SUCCESS;
-    }
-
-    if (MPI_SUCCESS !=
-        ompi_errhandler_request_invoke(incount, requests, FUNC_NAME)) {
-        return MPI_ERR_IN_STATUS;
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/waitsome.c.in b/ompi/mpi/c/waitsome.c.in
new file mode 100644
index 00000000000..ce25d1f47e8
--- /dev/null
+++ b/ompi/mpi/c/waitsome.c.in
@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2012      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2021-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/request/request.h"
+#include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
+
+PROTOTYPE ERROR_CLASS waitsome(INT incount, REQUEST_INOUT requests:incount,
+                               INT_OUT outcount, INT_OUT indices:incount,
+                               STATUS_OUT statuses:incount)
+{
+    SPC_RECORD(OMPI_SPC_WAITSOME, 1);
+
+    MEMCHECKER(
+        int j;
+        for (j = 0; j < incount; j++){
+            memchecker_request(&requests[j]);
+        }
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        int rc = MPI_SUCCESS;
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if ((NULL == requests) && (0 != incount)) {
+            rc = MPI_ERR_REQUEST;
+        } else {
+            if(!ompi_request_check_same_instance(requests, incount) ) {
+                rc = MPI_ERR_REQUEST;
+            }
+        }
+        if (((NULL == outcount || NULL == indices) && incount > 0) ||
+            incount < 0) {
+            rc = MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_NOHANDLE_CHECK(rc, rc, FUNC_NAME);
+    }
+
+    if (OPAL_UNLIKELY(0 == incount)) {
+        *outcount = MPI_UNDEFINED;
+        return MPI_SUCCESS;
+    }
+
+    if (OMPI_SUCCESS == ompi_request_wait_some( incount, requests,
+                                                outcount, indices, statuses )) {
+        return MPI_SUCCESS;
+    }
+
+    if (MPI_SUCCESS !=
+        ompi_errhandler_request_invoke(incount, requests, FUNC_NAME)) {
+        return MPI_ERR_IN_STATUS;
+    }
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/win_allocate.c b/ompi/mpi/c/win_allocate.c
deleted file mode 100644
index 6db2837a3a2..00000000000
--- a/ompi/mpi/c/win_allocate.c
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-#include "ompi/win/win.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_allocate = PMPI_Win_allocate
-#endif
-#define MPI_Win_allocate PMPI_Win_allocate
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_allocate";
-
-
-int MPI_Win_allocate(MPI_Aint size, int disp_unit, MPI_Info info,
-                     MPI_Comm comm, void *baseptr, MPI_Win *win)
-{
-    int ret = MPI_SUCCESS;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-    /* argument checking */
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_comm_invalid (comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-
-        } else if (NULL == info || ompi_info_is_freed(info)) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_INFO,
-                                          FUNC_NAME);
-
-        } else if (NULL == win) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_WIN, FUNC_NAME);
-        } else if ( size < 0 ) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_SIZE, FUNC_NAME);
-        } else if ( disp_unit <= 0 ) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_DISP, FUNC_NAME);
-        }
-    }
-
-    /* communicator must be an intracommunicator */
-    if (OMPI_COMM_IS_INTER(comm)) {
-        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COMM, FUNC_NAME);
-    }
-
-    /* create window and return */
-    ret = ompi_win_allocate((size_t)size, disp_unit, &(info->super),
-                            comm, baseptr, win);
-    if (OMPI_SUCCESS != ret) {
-        *win = MPI_WIN_NULL;
-        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_WIN, FUNC_NAME);
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/win_allocate.c.in b/ompi/mpi/c/win_allocate.c.in
new file mode 100644
index 00000000000..b1f40905b11
--- /dev/null
+++ b/ompi/mpi/c/win_allocate.c.in
@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include "ompi/win/win.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS win_allocate(AINT size, DISP disp_unit, INFO info,
+                                   COMM comm, BUFFER_OUT baseptr, WIN_OUT win)
+{
+    int ret = MPI_SUCCESS;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+    /* argument checking */
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_comm_invalid (comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+
+        } else if (NULL == info || ompi_info_is_freed(info)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_INFO,
+                                          FUNC_NAME);
+
+        } else if (NULL == win) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_WIN, FUNC_NAME);
+        } else if ( size < 0 ) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_SIZE, FUNC_NAME);
+        } else if ( disp_unit <= 0 ) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_DISP, FUNC_NAME);
+        }
+    }
+
+    /* communicator must be an intracommunicator */
+    if (OMPI_COMM_IS_INTER(comm)) {
+        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COMM, FUNC_NAME);
+    }
+
+/*
+ * TODO:BIGCOUNT remove this check once ompi-rma supports bigcount
+ */
+#if OMPI_BIGCOUNT_SRC
+    OMPI_CHECK_MPI_COUNT_INT_CONVERSION_OVERFLOW(ret, disp_unit);
+    if (OMPI_SUCCESS != ret) {
+        *win = MPI_WIN_NULL;
+        return OMPI_ERRHANDLER_INVOKE(comm, ret, FUNC_NAME);
+    }
+#endif
+
+    /* create window and return */
+/*
+ * TODO:BIGCOUNT remove (int) before disp_unit  once ompi-rma supports bigcount
+ */
+    ret = ompi_win_allocate((size_t)size, (int)disp_unit, &(info->super),
+                            comm, baseptr, win);
+    if (OMPI_SUCCESS != ret) {
+        *win = MPI_WIN_NULL;
+        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_WIN, FUNC_NAME);
+    }
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/win_allocate_shared.c b/ompi/mpi/c/win_allocate_shared.c
deleted file mode 100644
index aa6b72c2fcb..00000000000
--- a/ompi/mpi/c/win_allocate_shared.c
+++ /dev/null
@@ -1,88 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2014      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-#include "ompi/win/win.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_allocate_shared = PMPI_Win_allocate_shared
-#endif
-#define MPI_Win_allocate_shared PMPI_Win_allocate_shared
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_allocate_shared";
-
-
-int MPI_Win_allocate_shared(MPI_Aint size, int disp_unit, MPI_Info info,
-                            MPI_Comm comm, void *baseptr, MPI_Win *win)
-{
-    int ret = MPI_SUCCESS;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-    /* argument checking */
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_comm_invalid (comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-
-        } else if (NULL == info || ompi_info_is_freed(info)) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_INFO,
-                                          FUNC_NAME);
-
-        } else if (NULL == win) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_WIN, FUNC_NAME);
-        } else if ( size < 0 ) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_SIZE, FUNC_NAME);
-        }
-    }
-
-    /* communicator must be an intracommunicator */
-    if (OMPI_COMM_IS_INTER(comm)) {
-        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COMM, FUNC_NAME);
-    }
-
-    /* create window and return */
-    ret = ompi_win_allocate_shared((size_t)size, disp_unit, &(info->super),
-                                   comm, baseptr, win);
-    if (OMPI_SUCCESS != ret) {
-        *win = MPI_WIN_NULL;
-        OMPI_ERRHANDLER_RETURN (ret, comm, ret, FUNC_NAME);
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/win_allocate_shared.c.in b/ompi/mpi/c/win_allocate_shared.c.in
new file mode 100644
index 00000000000..bcec8b3a761
--- /dev/null
+++ b/ompi/mpi/c/win_allocate_shared.c.in
@@ -0,0 +1,93 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2014      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include "ompi/win/win.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS win_allocate_shared(AINT size, DISP disp_unit, INFO info,
+                                          COMM comm, BUFFER_OUT baseptr, WIN_OUT win)
+{
+    int ret = MPI_SUCCESS;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+    /* argument checking */
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_comm_invalid (comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+
+        } else if (NULL == info || ompi_info_is_freed(info)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_INFO,
+                                          FUNC_NAME);
+
+        } else if (NULL == win) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_WIN, FUNC_NAME);
+        } else if ( size < 0 ) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_SIZE, FUNC_NAME);
+        }
+/*
+ * TODO:BIGCOUNT remove this check once ompi-rma supports bigcount
+ */
+#if OMPI_BIGCOUNT_SRC
+        OMPI_CHECK_MPI_COUNT_INT_CONVERSION_OVERFLOW(ret, disp_unit);
+        if (OMPI_SUCCESS != ret) {
+            *win = MPI_WIN_NULL;
+            return OMPI_ERRHANDLER_INVOKE(comm, ret, FUNC_NAME);
+        }
+#endif
+    }
+
+    /* communicator must be an intracommunicator */
+    if (OMPI_COMM_IS_INTER(comm)) {
+        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COMM, FUNC_NAME);
+    }
+
+    /* create window and return */
+/*
+ * TODO:BIGCOUNT remove (int) from disp_unit once ompi-rma supports bigcount
+ */
+    ret = ompi_win_allocate_shared((size_t)size, (int)disp_unit, &(info->super),
+                                   comm, baseptr, win);
+    if (OMPI_SUCCESS != ret) {
+        *win = MPI_WIN_NULL;
+        OMPI_ERRHANDLER_RETURN (ret, comm, ret, FUNC_NAME);
+    }
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/win_attach.c b/ompi/mpi/c/win_attach.c
deleted file mode 100644
index 927a1b6278a..00000000000
--- a/ompi/mpi/c/win_attach.c
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-#include "ompi/win/win.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_attach = PMPI_Win_attach
-#endif
-#define MPI_Win_attach PMPI_Win_attach
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_attach";
-
-int MPI_Win_attach(MPI_Win win, void *base, MPI_Aint size)
-{
-    int ret = MPI_SUCCESS;
-
-    /* argument checking */
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        } else if (NULL == base) {
-            ret = MPI_ERR_ARG;
-        }
-        OMPI_ERRHANDLER_CHECK(ret, win, ret, FUNC_NAME);
-    }
-
-    /* create window and return */
-    ret = win->w_osc_module->osc_win_attach(win, base, size);
-    OMPI_ERRHANDLER_RETURN(ret, win, ret, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/win_attach.c.in b/ompi/mpi/c/win_attach.c.in
new file mode 100644
index 00000000000..18529108967
--- /dev/null
+++ b/ompi/mpi/c/win_attach.c.in
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include "ompi/win/win.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS win_attach(WIN win, BUFFER_OUT base, AINT size)
+{
+    int ret = MPI_SUCCESS;
+
+    /* argument checking */
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        } else if (NULL == base) {
+            ret = MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_CHECK(ret, win, ret, FUNC_NAME);
+    }
+
+    /* create window and return */
+    ret = win->w_osc_module->osc_win_attach(win, base, size);
+    OMPI_ERRHANDLER_RETURN(ret, win, ret, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/win_c2f.c b/ompi/mpi/c/win_c2f.c
deleted file mode 100644
index 67a5f47b3be..00000000000
--- a/ompi/mpi/c/win_c2f.c
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "ompi/win/win.h"
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/mpi/fortran/base/fint_2_int.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_c2f = PMPI_Win_c2f
-#endif
-#define MPI_Win_c2f PMPI_Win_c2f
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_c2f";
-
-
-MPI_Fint MPI_Win_c2f(MPI_Win win)
-{
-    if ( MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        /* Note that ompi_win_invalid() explicitly checks for
-           MPI_WIN_NULL, but MPI_WIN_C2F is supposed to treat
-           MPI_WIN_NULL as a valid window (and therefore return
-           a valid Fortran handle for it).  Hence, this function
-           should not return an error if MPI_WIN_NULL is passed in.
-
-           See a big comment in ompi/communicator/communicator.h about
-           this (I know that's not win.h, but the issues are related,
-           and that's where the explanation is). */
-        if (ompi_win_invalid(win) && MPI_WIN_NULL != win) {
-            return OMPI_INT_2_FINT(-1);
-        }
-    }
-
-    return OMPI_INT_2_FINT(win->w_f_to_c_index);
-}
diff --git a/ompi/mpi/c/win_c2f.c.in b/ompi/mpi/c/win_c2f.c.in
new file mode 100644
index 00000000000..44e0f138907
--- /dev/null
+++ b/ompi/mpi/c/win_c2f.c.in
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include <stdio.h>
+
+#include "ompi/win/win.h"
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+
+PROTOTYPE FINT win_c2f(WIN win)
+{
+    if ( MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        /* Note that ompi_win_invalid() explicitly checks for
+           MPI_WIN_NULL, but MPI_WIN_C2F is supposed to treat
+           MPI_WIN_NULL as a valid window (and therefore return
+           a valid Fortran handle for it).  Hence, this function
+           should not return an error if MPI_WIN_NULL is passed in.
+
+           See a big comment in ompi/communicator/communicator.h about
+           this (I know that's not win.h, but the issues are related,
+           and that's where the explanation is). */
+        if (ompi_win_invalid(win) && MPI_WIN_NULL != win) {
+            return OMPI_INT_2_FINT(-1);
+        }
+    }
+
+    return OMPI_INT_2_FINT(win->w_f_to_c_index);
+}
diff --git a/ompi/mpi/c/win_call_errhandler.c b/ompi/mpi/c/win_call_errhandler.c
deleted file mode 100644
index 7584653c501..00000000000
--- a/ompi/mpi/c/win_call_errhandler.c
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_call_errhandler = PMPI_Win_call_errhandler
-#endif
-#define MPI_Win_call_errhandler PMPI_Win_call_errhandler
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_call_errhandler";
-
-
-int MPI_Win_call_errhandler(MPI_Win win, int errorcode)
-{
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN,
-                                          FUNC_NAME);
-        }
-    }
-
-  /* Invoke the errhandler */
-  OMPI_ERRHANDLER_INVOKE(win, errorcode, FUNC_NAME);
-
-  /* See MPI-2 8.5 why this function has to return MPI_SUCCESS */
-  return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/win_call_errhandler.c.in b/ompi/mpi/c/win_call_errhandler.c.in
new file mode 100644
index 00000000000..644b0e5e93d
--- /dev/null
+++ b/ompi/mpi/c/win_call_errhandler.c.in
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+
+PROTOTYPE ERROR_CLASS win_call_errhandler(WIN win, INT errorcode)
+{
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN,
+                                          FUNC_NAME);
+        }
+    }
+
+  /* Invoke the errhandler */
+  OMPI_ERRHANDLER_INVOKE(win, errorcode, FUNC_NAME);
+
+  /* See MPI-2 8.5 why this function has to return MPI_SUCCESS */
+  return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/win_complete.c b/ompi/mpi/c/win_complete.c
deleted file mode 100644
index 98cc831af7c..00000000000
--- a/ompi/mpi/c/win_complete.c
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-#include "ompi/mca/osc/osc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_complete = PMPI_Win_complete
-#endif
-#define MPI_Win_complete PMPI_Win_complete
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_complete";
-
-
-int MPI_Win_complete(MPI_Win win)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        }
-    }
-
-    rc = win->w_osc_module->osc_complete(win);
-    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/win_complete.c.in b/ompi/mpi/c/win_complete.c.in
new file mode 100644
index 00000000000..a9fd3d2b2a9
--- /dev/null
+++ b/ompi/mpi/c/win_complete.c.in
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+#include "ompi/mca/osc/osc.h"
+
+PROTOTYPE ERROR_CLASS win_complete(WIN win)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        }
+    }
+
+    rc = win->w_osc_module->osc_complete(win);
+    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/win_create.c b/ompi/mpi/c/win_create.c
deleted file mode 100644
index 419ff3ddf9f..00000000000
--- a/ompi/mpi/c/win_create.c
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-#include "ompi/win/win.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_create = PMPI_Win_create
-#endif
-#define MPI_Win_create PMPI_Win_create
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_create";
-
-
-int MPI_Win_create(void *base, MPI_Aint size, int disp_unit,
-                   MPI_Info info, MPI_Comm comm, MPI_Win *win)
-{
-    int ret = MPI_SUCCESS;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-    /* argument checking */
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_comm_invalid (comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-
-        } else if (NULL == info || ompi_info_is_freed(info)) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_INFO,
-                                          FUNC_NAME);
-
-        } else if (NULL == win) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_WIN, FUNC_NAME);
-        } else if ( size < 0 ) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_SIZE, FUNC_NAME);
-        } else if ( disp_unit <= 0 ) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_DISP, FUNC_NAME);
-        }
-    }
-
-    /* communicator must be an intracommunicator */
-    if (OMPI_COMM_IS_INTER(comm)) {
-        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COMM, FUNC_NAME);
-    }
-
-    /* create window and return */
-    ret = ompi_win_create(base, (size_t)size, disp_unit, comm,
-                          &(info->super), win);
-    if (OMPI_SUCCESS != ret) {
-        *win = MPI_WIN_NULL;
-        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_WIN, FUNC_NAME);
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/win_create.c.in b/ompi/mpi/c/win_create.c.in
new file mode 100644
index 00000000000..c737d22370f
--- /dev/null
+++ b/ompi/mpi/c/win_create.c.in
@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include "ompi/win/win.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS win_create(BUFFER_OUT base, AINT size, DISP disp_unit,
+                                 INFO info, COMM comm, WIN_OUT win)
+{
+    int ret = MPI_SUCCESS;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+    /* argument checking */
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_comm_invalid (comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+
+        } else if (NULL == info || ompi_info_is_freed(info)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_INFO,
+                                          FUNC_NAME);
+
+        } else if (NULL == win) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_WIN, FUNC_NAME);
+        } else if ( size < 0 ) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_SIZE, FUNC_NAME);
+        } else if ( disp_unit <= 0 ) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_DISP, FUNC_NAME);
+        }
+/*
+ * TODO:BIGCOUNT remove this check once ompi-rma supports bigcount
+ */
+#if OMPI_BIGCOUNT_SRC
+        OMPI_CHECK_MPI_COUNT_INT_CONVERSION_OVERFLOW(ret, disp_unit);
+        if (OMPI_SUCCESS != ret) {
+            *win = MPI_WIN_NULL;
+            return OMPI_ERRHANDLER_INVOKE(comm, ret, FUNC_NAME);
+        }
+#endif
+    }
+
+    /* communicator must be an intracommunicator */
+    if (OMPI_COMM_IS_INTER(comm)) {
+        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COMM, FUNC_NAME);
+    }
+
+/*
+ * TODO:BIGCOUNT remove (int) before disp_unit once ompi-rma supports bigcount
+ */
+    /* create window and return */
+    ret = ompi_win_create(base, (size_t)size, (int)disp_unit, comm,
+                          &(info->super), win);
+    if (OMPI_SUCCESS != ret) {
+        *win = MPI_WIN_NULL;
+        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_WIN, FUNC_NAME);
+    }
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/win_create_dynamic.c b/ompi/mpi/c/win_create_dynamic.c
deleted file mode 100644
index a8b218e2b90..00000000000
--- a/ompi/mpi/c/win_create_dynamic.c
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-#include "ompi/win/win.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_create_dynamic = PMPI_Win_create_dynamic
-#endif
-#define MPI_Win_create_dynamic PMPI_Win_create_dynamic
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_create_dynamic";
-
-
-int MPI_Win_create_dynamic(MPI_Info info, MPI_Comm comm, MPI_Win *win)
-{
-    int ret = MPI_SUCCESS;
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-    );
-    /* argument checking */
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_comm_invalid (comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-
-        } else if (NULL == info || ompi_info_is_freed(info)) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_INFO,
-                                          FUNC_NAME);
-
-        } else if (NULL == win) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_WIN, FUNC_NAME);
-        }
-    }
-
-    /* communicator must be an intracommunicator */
-    if (OMPI_COMM_IS_INTER(comm)) {
-        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COMM, FUNC_NAME);
-    }
-
-    /* create_dynamic window and return */
-    ret = ompi_win_create_dynamic(&(info->super), comm, win);
-    if (OMPI_SUCCESS != ret) {
-        *win = MPI_WIN_NULL;
-        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_WIN, FUNC_NAME);
-    }
-
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/win_create_dynamic.c.in b/ompi/mpi/c/win_create_dynamic.c.in
new file mode 100644
index 00000000000..f50a2d697ff
--- /dev/null
+++ b/ompi/mpi/c/win_create_dynamic.c.in
@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include "ompi/win/win.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS win_create_dynamic(INFO info, COMM comm, WIN_OUT win)
+{
+    int ret = MPI_SUCCESS;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+    /* argument checking */
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_comm_invalid (comm)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
+                                          FUNC_NAME);
+
+        } else if (NULL == info || ompi_info_is_freed(info)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_INFO,
+                                          FUNC_NAME);
+
+        } else if (NULL == win) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_WIN, FUNC_NAME);
+        }
+    }
+
+    /* communicator must be an intracommunicator */
+    if (OMPI_COMM_IS_INTER(comm)) {
+        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COMM, FUNC_NAME);
+    }
+
+    /* create_dynamic window and return */
+    ret = ompi_win_create_dynamic(&(info->super), comm, win);
+    if (OMPI_SUCCESS != ret) {
+        *win = MPI_WIN_NULL;
+        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_WIN, FUNC_NAME);
+    }
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/win_create_errhandler.c b/ompi/mpi/c/win_create_errhandler.c
deleted file mode 100644
index c18a736857d..00000000000
--- a/ompi/mpi/c/win_create_errhandler.c
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008-2009 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_create_errhandler = PMPI_Win_create_errhandler
-#endif
-#define MPI_Win_create_errhandler PMPI_Win_create_errhandler
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_create_errhandler";
-
-
-int MPI_Win_create_errhandler(MPI_Win_errhandler_function *function,
-                              MPI_Errhandler *errhandler)
-{
-    int err = MPI_SUCCESS;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (NULL == function ||
-            NULL == errhandler) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-    }
-
-    /* Create and cache the errhandler.  Sets a refcount of 1. */
-    *errhandler =
-        ompi_errhandler_create(OMPI_ERRHANDLER_TYPE_WIN,
-                               (ompi_errhandler_generic_handler_fn_t*) function,
-                               OMPI_ERRHANDLER_LANG_C);
-    if (NULL == *errhandler) {
-        err = MPI_ERR_INTERN;
-    }
-
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, MPI_ERR_INTERN, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/win_create_errhandler.c.in b/ompi/mpi/c/win_create_errhandler.c.in
new file mode 100644
index 00000000000..27a48e33aa4
--- /dev/null
+++ b/ompi/mpi/c/win_create_errhandler.c.in
@@ -0,0 +1,57 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008-2009 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+
+PROTOTYPE ERROR_CLASS win_create_errhandler(WIN_ERRHANLDER_FUNCTION function,
+                                            ERRHANDLER_OUT errhandler)
+{
+    int err = MPI_SUCCESS;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (NULL == function ||
+            NULL == errhandler) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+    /* Create and cache the errhandler.  Sets a refcount of 1. */
+    *errhandler =
+        ompi_errhandler_create(OMPI_ERRHANDLER_TYPE_WIN,
+                               (ompi_errhandler_generic_handler_fn_t*) function,
+                               OMPI_ERRHANDLER_LANG_C);
+    if (NULL == *errhandler) {
+        err = MPI_ERR_INTERN;
+    }
+
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(err, MPI_ERR_INTERN, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/win_create_keyval.c b/ompi/mpi/c/win_create_keyval.c
deleted file mode 100644
index 68c45b776ad..00000000000
--- a/ompi/mpi/c/win_create_keyval.c
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/attribute/attribute.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_create_keyval = PMPI_Win_create_keyval
-#endif
-#define MPI_Win_create_keyval PMPI_Win_create_keyval
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_create_keyval";
-
-
-int MPI_Win_create_keyval(MPI_Win_copy_attr_function *win_copy_attr_fn,
-                          MPI_Win_delete_attr_function *win_delete_attr_fn,
-                          int *win_keyval, void *extra_state)
-{
-    int ret;
-    ompi_attribute_fn_ptr_union_t copy_fn;
-    ompi_attribute_fn_ptr_union_t del_fn;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if ((NULL == win_copy_attr_fn) || (NULL == win_delete_attr_fn) ||
-            (NULL == win_keyval)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-    }
-
-    copy_fn.attr_win_copy_fn = (MPI_Win_internal_copy_attr_function*)win_copy_attr_fn;
-    del_fn.attr_win_delete_fn = win_delete_attr_fn;
-
-    ret = ompi_attr_create_keyval(WIN_ATTR, copy_fn, del_fn,
-                                  win_keyval, extra_state, 0, NULL);
-    OMPI_ERRHANDLER_NOHANDLE_RETURN(ret, MPI_ERR_OTHER, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/win_create_keyval.c.in b/ompi/mpi/c/win_create_keyval.c.in
new file mode 100644
index 00000000000..f58401685bf
--- /dev/null
+++ b/ompi/mpi/c/win_create_keyval.c.in
@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/attribute/attribute.h"
+
+PROTOTYPE ERROR_CLASS win_create_keyval(WIN_COPY_ATTR_FUNCTION win_copy_attr_fn,
+                                        WIN_DELETE_ATTR_FUNCTION win_delete_attr_fn,
+                                        INT_OUT win_keyval, BUFFER_OUT extra_state)
+{
+    int ret;
+    ompi_attribute_fn_ptr_union_t copy_fn;
+    ompi_attribute_fn_ptr_union_t del_fn;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if ((NULL == win_copy_attr_fn) || (NULL == win_delete_attr_fn) ||
+            (NULL == win_keyval)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+    copy_fn.attr_win_copy_fn = win_copy_attr_fn;
+    del_fn.attr_win_delete_fn = win_delete_attr_fn;
+
+    ret = ompi_attr_create_keyval(WIN_ATTR, copy_fn, del_fn,
+                                  win_keyval, extra_state, 0, NULL);
+    OMPI_ERRHANDLER_NOHANDLE_RETURN(ret, MPI_ERR_OTHER, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/win_delete_attr.c b/ompi/mpi/c/win_delete_attr.c
deleted file mode 100644
index e6bfbd3f9a2..00000000000
--- a/ompi/mpi/c/win_delete_attr.c
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/attribute/attribute.h"
-#include "ompi/win/win.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_delete_attr = PMPI_Win_delete_attr
-#endif
-#define MPI_Win_delete_attr PMPI_Win_delete_attr
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_delete_attr";
-
-
-int MPI_Win_delete_attr(MPI_Win win, int win_keyval)
-{
-    int ret;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        }
-    }
-
-    ret = ompi_attr_delete(WIN_ATTR, win, win->w_keyhash, win_keyval,
-                           false);
-    OMPI_ERRHANDLER_RETURN(ret, win, MPI_ERR_OTHER, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/win_delete_attr.c.in b/ompi/mpi/c/win_delete_attr.c.in
new file mode 100644
index 00000000000..0baa3cc01c3
--- /dev/null
+++ b/ompi/mpi/c/win_delete_attr.c.in
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/attribute/attribute.h"
+#include "ompi/win/win.h"
+
+PROTOTYPE ERROR_CLASS win_delete_attr(WIN win, INT win_keyval)
+{
+    int ret;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        }
+    }
+
+    ret = ompi_attr_delete(WIN_ATTR, win, win->w_keyhash, win_keyval,
+                           false);
+    OMPI_ERRHANDLER_RETURN(ret, win, MPI_ERR_OTHER, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/win_detach.c b/ompi/mpi/c/win_detach.c
deleted file mode 100644
index 00dd0eec52a..00000000000
--- a/ompi/mpi/c/win_detach.c
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-#include "ompi/win/win.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_detach = PMPI_Win_detach
-#endif
-#define MPI_Win_detach PMPI_Win_detach
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_detach";
-
-int MPI_Win_detach(MPI_Win win, const void *base)
-{
-    int ret = MPI_SUCCESS;
-
-    /* argument checking */
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        } else if (NULL == base) {
-            ret = MPI_ERR_ARG;
-        }
-        OMPI_ERRHANDLER_CHECK(ret, win, ret, FUNC_NAME);
-    }
-
-    /* create window and return */
-    ret = win->w_osc_module->osc_win_detach(win, base);
-    OMPI_ERRHANDLER_RETURN(ret, win, ret, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/win_detach.c.in b/ompi/mpi/c/win_detach.c.in
new file mode 100644
index 00000000000..e6b6836adec
--- /dev/null
+++ b/ompi/mpi/c/win_detach.c.in
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include "ompi/win/win.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS win_detach(WIN win, BUFFER base)
+{
+    int ret = MPI_SUCCESS;
+
+    /* argument checking */
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        } else if (NULL == base) {
+            ret = MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_CHECK(ret, win, ret, FUNC_NAME);
+    }
+
+    /* create window and return */
+    ret = win->w_osc_module->osc_win_detach(win, base);
+    OMPI_ERRHANDLER_RETURN(ret, win, ret, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/win_f2c.c b/ompi/mpi/c/win_f2c.c
index bc8bdb7dcab..82bf2b9cc71 100644
--- a/ompi/mpi/c/win_f2c.c
+++ b/ompi/mpi/c/win_f2c.c
@@ -36,7 +36,6 @@
 
 static const char FUNC_NAME[] = "MPI_Win_f2c";
 
-
 MPI_Win MPI_Win_f2c(MPI_Fint win)
 {
     int o_index= OMPI_FINT_2_INT(win);
diff --git a/ompi/mpi/c/win_fence.c b/ompi/mpi/c/win_fence.c
deleted file mode 100644
index 3fd394f133c..00000000000
--- a/ompi/mpi/c/win_fence.c
+++ /dev/null
@@ -1,60 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2014      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-#include "ompi/mca/osc/osc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_fence = PMPI_Win_fence
-#endif
-#define MPI_Win_fence PMPI_Win_fence
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_fence";
-
-
-int MPI_Win_fence(int assert, MPI_Win win)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        } else if (0 != (assert & ~(MPI_MODE_NOSTORE | MPI_MODE_NOPUT |
-                                    MPI_MODE_NOPRECEDE | MPI_MODE_NOSUCCEED))) {
-            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ASSERT, FUNC_NAME);
-        }
-    }
-
-    rc = win->w_osc_module->osc_fence(assert, win);
-    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/win_fence.c.in b/ompi/mpi/c/win_fence.c.in
new file mode 100644
index 00000000000..e71ad6831b6
--- /dev/null
+++ b/ompi/mpi/c/win_fence.c.in
@@ -0,0 +1,52 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2014      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+#include "ompi/mca/osc/osc.h"
+
+PROTOTYPE ERROR_CLASS win_fence(INT mpi_assert, WIN win)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        } else if (0 != (mpi_assert & ~(MPI_MODE_NOSTORE | MPI_MODE_NOPUT |
+                                    MPI_MODE_NOPRECEDE | MPI_MODE_NOSUCCEED))) {
+            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ASSERT, FUNC_NAME);
+        }
+    }
+
+    rc = win->w_osc_module->osc_fence(mpi_assert, win);
+    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/win_flush.c b/ompi/mpi/c/win_flush.c
deleted file mode 100644
index 99030dffc06..00000000000
--- a/ompi/mpi/c/win_flush.c
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-#include "ompi/win/win.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_flush = PMPI_Win_flush
-#endif
-#define MPI_Win_flush PMPI_Win_flush
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_flush";
-
-int MPI_Win_flush(int rank, MPI_Win win)
-{
-    int ret = MPI_SUCCESS;
-
-    /* argument checking */
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        }
-        OMPI_ERRHANDLER_CHECK(ret, win, ret, FUNC_NAME);
-    }
-
-    /* create window and return */
-    ret = win->w_osc_module->osc_flush(rank, win);
-    OMPI_ERRHANDLER_RETURN(ret, win, ret, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/win_flush.c.in b/ompi/mpi/c/win_flush.c.in
new file mode 100644
index 00000000000..01a78419822
--- /dev/null
+++ b/ompi/mpi/c/win_flush.c.in
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include "ompi/win/win.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS win_flush(INT rank, WIN win)
+{
+    int ret = MPI_SUCCESS;
+
+    /* argument checking */
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        }
+        OMPI_ERRHANDLER_CHECK(ret, win, ret, FUNC_NAME);
+    }
+
+    /* create window and return */
+    ret = win->w_osc_module->osc_flush(rank, win);
+    OMPI_ERRHANDLER_RETURN(ret, win, ret, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/win_flush_all.c b/ompi/mpi/c/win_flush_all.c
deleted file mode 100644
index d21f86fdf92..00000000000
--- a/ompi/mpi/c/win_flush_all.c
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-#include "ompi/win/win.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_flush_all = PMPI_Win_flush_all
-#endif
-#define MPI_Win_flush_all PMPI_Win_flush_all
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_flush_all";
-
-int MPI_Win_flush_all(MPI_Win win)
-{
-    int ret = MPI_SUCCESS;
-
-    /* argument checking */
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        }
-        OMPI_ERRHANDLER_CHECK(ret, win, ret, FUNC_NAME);
-    }
-
-    /* create window and return */
-    ret = win->w_osc_module->osc_flush_all(win);
-    OMPI_ERRHANDLER_RETURN(ret, win, ret, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/win_flush_all.c.in b/ompi/mpi/c/win_flush_all.c.in
new file mode 100644
index 00000000000..559d99926f6
--- /dev/null
+++ b/ompi/mpi/c/win_flush_all.c.in
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include "ompi/win/win.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS win_flush_all(WIN win)
+{
+    int ret = MPI_SUCCESS;
+
+    /* argument checking */
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        }
+        OMPI_ERRHANDLER_CHECK(ret, win, ret, FUNC_NAME);
+    }
+
+    /* create window and return */
+    ret = win->w_osc_module->osc_flush_all(win);
+    OMPI_ERRHANDLER_RETURN(ret, win, ret, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/win_flush_local.c b/ompi/mpi/c/win_flush_local.c
deleted file mode 100644
index 07fa1d689dd..00000000000
--- a/ompi/mpi/c/win_flush_local.c
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-#include "ompi/win/win.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_flush_local = PMPI_Win_flush_local
-#endif
-#define MPI_Win_flush_local PMPI_Win_flush_local
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_flush_local";
-
-int MPI_Win_flush_local(int rank, MPI_Win win)
-{
-    int ret = MPI_SUCCESS;
-
-    /* argument checking */
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        }
-        OMPI_ERRHANDLER_CHECK(ret, win, ret, FUNC_NAME);
-    }
-
-    /* create window and return */
-    ret = win->w_osc_module->osc_flush_local(rank, win);
-    OMPI_ERRHANDLER_RETURN(ret, win, ret, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/win_flush_local.c.in b/ompi/mpi/c/win_flush_local.c.in
new file mode 100644
index 00000000000..a61afa2542e
--- /dev/null
+++ b/ompi/mpi/c/win_flush_local.c.in
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include "ompi/win/win.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS win_flush_local(INT rank, WIN win)
+{
+    int ret = MPI_SUCCESS;
+
+    /* argument checking */
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        }
+        OMPI_ERRHANDLER_CHECK(ret, win, ret, FUNC_NAME);
+    }
+
+    /* create window and return */
+    ret = win->w_osc_module->osc_flush_local(rank, win);
+    OMPI_ERRHANDLER_RETURN(ret, win, ret, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/win_flush_local_all.c b/ompi/mpi/c/win_flush_local_all.c
deleted file mode 100644
index 4498a98ca83..00000000000
--- a/ompi/mpi/c/win_flush_local_all.c
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-#include "ompi/win/win.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_flush_local_all = PMPI_Win_flush_local_all
-#endif
-#define MPI_Win_flush_local_all PMPI_Win_flush_local_all
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_flush_local_all";
-
-int MPI_Win_flush_local_all(MPI_Win win)
-{
-    int ret = MPI_SUCCESS;
-
-    /* argument checking */
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        }
-        OMPI_ERRHANDLER_CHECK(ret, win, ret, FUNC_NAME);
-    }
-
-    /* create window and return */
-    ret = win->w_osc_module->osc_flush_local_all(win);
-    OMPI_ERRHANDLER_RETURN(ret, win, ret, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/win_flush_local_all.c.in b/ompi/mpi/c/win_flush_local_all.c.in
new file mode 100644
index 00000000000..5d43b5b4bcf
--- /dev/null
+++ b/ompi/mpi/c/win_flush_local_all.c.in
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include "ompi/win/win.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS win_flush_local_all(WIN win)
+{
+    int ret = MPI_SUCCESS;
+
+    /* argument checking */
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        }
+        OMPI_ERRHANDLER_CHECK(ret, win, ret, FUNC_NAME);
+    }
+
+    /* create window and return */
+    ret = win->w_osc_module->osc_flush_local_all(win);
+    OMPI_ERRHANDLER_RETURN(ret, win, ret, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/win_free.c b/ompi/mpi/c/win_free.c
deleted file mode 100644
index 809b5dbd702..00000000000
--- a/ompi/mpi/c/win_free.c
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_free = PMPI_Win_free
-#endif
-#define MPI_Win_free PMPI_Win_free
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_free";
-
-
-int MPI_Win_free(MPI_Win *win)
-{
-    int ret;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(*win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        }
-    }
-
-    ret = ompi_win_free(*win);
-    if (OMPI_SUCCESS == ret) {
-        *win = MPI_WIN_NULL;
-    }
-
-    OMPI_ERRHANDLER_RETURN(ret, *win, ret, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/win_free.c.in b/ompi/mpi/c/win_free.c.in
new file mode 100644
index 00000000000..fb8604f6688
--- /dev/null
+++ b/ompi/mpi/c/win_free.c.in
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+
+PROTOTYPE ERROR_CLASS win_free(WIN_OUT win)
+{
+    int ret;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(*win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        }
+    }
+
+    ret = ompi_win_free(*win);
+    if (OMPI_SUCCESS == ret) {
+        *win = MPI_WIN_NULL;
+    }
+
+    OMPI_ERRHANDLER_RETURN(ret, *win, ret, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/win_free_keyval.c b/ompi/mpi/c/win_free_keyval.c
deleted file mode 100644
index 1ef42ec7e38..00000000000
--- a/ompi/mpi/c/win_free_keyval.c
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/attribute/attribute.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_free_keyval = PMPI_Win_free_keyval
-#endif
-#define MPI_Win_free_keyval PMPI_Win_free_keyval
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_free_keyval";
-
-
-int MPI_Win_free_keyval(int *win_keyval)
-{
-   int ret;
-
-   /* Check for valid key pointer */
-
-   if (MPI_PARAM_CHECK) {
-      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-      if (NULL == win_keyval) {
-         return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                       FUNC_NAME);
-      }
-   }
-
-   ret = ompi_attr_free_keyval(WIN_ATTR, win_keyval, 0);
-   OMPI_ERRHANDLER_NOHANDLE_RETURN(ret, MPI_ERR_OTHER, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/win_free_keyval.c.in b/ompi/mpi/c/win_free_keyval.c.in
new file mode 100644
index 00000000000..8b7b5448cac
--- /dev/null
+++ b/ompi/mpi/c/win_free_keyval.c.in
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/attribute/attribute.h"
+
+PROTOTYPE ERROR_CLASS win_free_keyval(INT_OUT win_keyval)
+{
+   int ret;
+
+   /* Check for valid key pointer */
+
+   if (MPI_PARAM_CHECK) {
+      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+      if (NULL == win_keyval) {
+         return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
+                                       FUNC_NAME);
+      }
+   }
+
+   ret = ompi_attr_free_keyval(WIN_ATTR, win_keyval, 0);
+   OMPI_ERRHANDLER_NOHANDLE_RETURN(ret, MPI_ERR_OTHER, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/win_get_attr.c b/ompi/mpi/c/win_get_attr.c
deleted file mode 100644
index cf89c3fa1bd..00000000000
--- a/ompi/mpi/c/win_get_attr.c
+++ /dev/null
@@ -1,68 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/attribute/attribute.h"
-#include "ompi/win/win.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_get_attr = PMPI_Win_get_attr
-#endif
-#define MPI_Win_get_attr PMPI_Win_get_attr
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_get_attr";
-
-
-int MPI_Win_get_attr(MPI_Win win, int win_keyval,
-                     void *attribute_val, int *flag)
-{
-    int ret;
-
-    if (MPI_PARAM_CHECK) {
-       OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-       if (ompi_win_invalid(win)) {
-           return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-       } else if ((NULL == attribute_val) || (NULL == flag)) {
-           return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ARG, FUNC_NAME);
-        } else if (MPI_KEYVAL_INVALID == win_keyval) {
-            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_KEYVAL, FUNC_NAME);
-       }
-    }
-
-    /* This stuff is very confusing.  Be sure to see
-       src/attribute/attribute.c for a lengthy comment explaining Open
-       MPI attribute behavior. */
-
-    ret = ompi_attr_get_c(win->w_keyhash, win_keyval,
-                          (void**)attribute_val, flag);
-
-    OMPI_ERRHANDLER_RETURN(ret, win, MPI_ERR_OTHER, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/win_get_attr.c.in b/ompi/mpi/c/win_get_attr.c.in
new file mode 100644
index 00000000000..bcca259625c
--- /dev/null
+++ b/ompi/mpi/c/win_get_attr.c.in
@@ -0,0 +1,60 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/attribute/attribute.h"
+#include "ompi/win/win.h"
+
+PROTOTYPE ERROR_CLASS win_get_attr(WIN win, INT win_keyval,
+                                   BUFFER_OUT attribute_val, INT_OUT flag)
+{
+    int ret;
+
+    if (MPI_PARAM_CHECK) {
+       OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+       if (ompi_win_invalid(win)) {
+           return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+       } else if ((NULL == attribute_val) || (NULL == flag)) {
+           return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ARG, FUNC_NAME);
+        } else if (MPI_KEYVAL_INVALID == win_keyval) {
+            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_KEYVAL, FUNC_NAME);
+       }
+    }
+
+    /* This stuff is very confusing.  Be sure to see
+       src/attribute/attribute.c for a lengthy comment explaining Open
+       MPI attribute behavior. */
+
+    ret = ompi_attr_get_c(win->w_keyhash, win_keyval,
+                          (void**)attribute_val, flag);
+
+    OMPI_ERRHANDLER_RETURN(ret, win, MPI_ERR_OTHER, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/win_get_errhandler.c b/ompi/mpi/c/win_get_errhandler.c
deleted file mode 100644
index 9196e607b83..00000000000
--- a/ompi/mpi/c/win_get_errhandler.c
+++ /dev/null
@@ -1,65 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008-2009 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015-2017 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_get_errhandler = PMPI_Win_get_errhandler
-#endif
-#define MPI_Win_get_errhandler PMPI_Win_get_errhandler
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_get_errhandler";
-
-
-int MPI_Win_get_errhandler(MPI_Win win, MPI_Errhandler *errhandler)
-{
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN,
-                                          FUNC_NAME);
-        } else if (NULL == errhandler) {
-            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ARG,
-                                          FUNC_NAME);
-        }
-    }
-
-    OPAL_THREAD_LOCK(&win->w_lock);
-    /* Retain the errhandler, corresponding to object refcount
-       decrease in errhandler_free.c. */
-    OBJ_RETAIN(win->error_handler);
-    *errhandler = win->error_handler;
-    OPAL_THREAD_UNLOCK(&win->w_lock);
-
-    /* All done */
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/win_get_errhandler.c.in b/ompi/mpi/c/win_get_errhandler.c.in
new file mode 100644
index 00000000000..8a027da4fd1
--- /dev/null
+++ b/ompi/mpi/c/win_get_errhandler.c.in
@@ -0,0 +1,64 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008-2009 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2017 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2020-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+#include "ompi/instance/instance.h"
+
+PROTOTYPE ERROR_CLASS win_get_errhandler(WIN win, ERRHANDLER_OUT errhandler)
+{
+    int ret = MPI_SUCCESS;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN,
+                                          FUNC_NAME);
+        } else if (NULL == errhandler) {
+            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ARG,
+                                          FUNC_NAME);
+        }
+    }
+
+    OPAL_THREAD_LOCK(&win->w_lock);
+    /* Retain the errhandler, corresponding to object refcount
+       decrease in errhandler_free.c. */
+    OBJ_RETAIN(win->error_handler);
+    *errhandler = win->error_handler;
+    OPAL_THREAD_UNLOCK(&win->w_lock);
+
+   /* make sure the infrastructure is initialized */
+    ret = ompi_mpi_instance_retain ();
+
+
+    /* All done */
+    return ret;
+}
diff --git a/ompi/mpi/c/win_get_group.c b/ompi/mpi/c/win_get_group.c
deleted file mode 100644
index d4b895ece57..00000000000
--- a/ompi/mpi/c/win_get_group.c
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_get_group = PMPI_Win_get_group
-#endif
-#define MPI_Win_get_group PMPI_Win_get_group
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_get_group";
-
-
-int MPI_Win_get_group(MPI_Win win, MPI_Group *group)
-{
-    int ret;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        } else if (NULL == group) {
-            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ARG, FUNC_NAME);
-        }
-    }
-
-    ret = ompi_win_group(win, (ompi_group_t**) group);
-    OMPI_ERRHANDLER_RETURN(ret, win, ret, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/win_get_group.c.in b/ompi/mpi/c/win_get_group.c.in
new file mode 100644
index 00000000000..5330d2dbbec
--- /dev/null
+++ b/ompi/mpi/c/win_get_group.c.in
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+
+PROTOTYPE ERROR_CLASS win_get_group(WIN win, GROUP_OUT group)
+{
+    int ret;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        } else if (NULL == group) {
+            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ARG, FUNC_NAME);
+        }
+    }
+
+    ret = ompi_win_group(win, (ompi_group_t**) group);
+    OMPI_ERRHANDLER_RETURN(ret, win, ret, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/win_get_info.c b/ompi/mpi/c/win_get_info.c
deleted file mode 100644
index 83e4fbaf0ee..00000000000
--- a/ompi/mpi/c/win_get_info.c
+++ /dev/null
@@ -1,66 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2013      Sandia National Laboratories.  All rights reserved.
- * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-#include "opal/util/info.h"
-#include "opal/util/info_subscriber.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_get_info = PMPI_Win_get_info
-#endif
-#define MPI_Win_get_info PMPI_Win_get_info
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_get_info";
-
-int MPI_Win_get_info(MPI_Win win, MPI_Info *info_used)
-{
-    int ret;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        }
-
-        if (NULL == info_used) {
-            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ARG, FUNC_NAME);
-        }
-    }
-
-    if (NULL == win->super.s_info) {
-/*
- * Setup any defaults if MPI_Win_set_info was never called
- */
-	opal_infosubscribe_change_info(&win->super, &MPI_INFO_NULL->super); 	
-    }
-
-    (*info_used) = OBJ_NEW(ompi_info_t);
-    if (NULL == (*info_used)) {
-       return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_NO_MEM, FUNC_NAME);
-    }
-    opal_info_t *opal_info_used = &(*info_used)->super;
-
-    ret = opal_info_dup_mpistandard(win->super.s_info, &opal_info_used);
-
-    OMPI_ERRHANDLER_RETURN(ret, win, ret, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/win_get_info.c.in b/ompi/mpi/c/win_get_info.c.in
new file mode 100644
index 00000000000..abd1622756f
--- /dev/null
+++ b/ompi/mpi/c/win_get_info.c.in
@@ -0,0 +1,59 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2013      Sandia National Laboratories.  All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+#include "opal/util/info.h"
+#include "opal/util/info_subscriber.h"
+
+PROTOTYPE ERROR_CLASS win_get_info(WIN win, INFO_OUT info_used)
+{
+    int ret;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        }
+
+        if (NULL == info_used) {
+            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ARG, FUNC_NAME);
+        }
+    }
+
+    if (NULL == win->super.s_info) {
+        /*
+         * Setup any defaults if MPI_Win_set_info was never called
+         */
+        opal_infosubscribe_change_info(&win->super, &MPI_INFO_NULL->super);
+    }
+
+    *info_used = ompi_info_allocate ();
+    if (NULL == (*info_used)) {
+       return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_NO_MEM, FUNC_NAME);
+    }
+    opal_info_t *opal_info_used = &(*info_used)->super;
+
+    ret = opal_info_dup_public(win->super.s_info, &opal_info_used);
+
+    OMPI_ERRHANDLER_RETURN(ret, win, ret, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/win_get_name.c b/ompi/mpi/c/win_get_name.c
deleted file mode 100644
index 6d745c05a5f..00000000000
--- a/ompi/mpi/c/win_get_name.c
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_get_name = PMPI_Win_get_name
-#endif
-#define MPI_Win_get_name PMPI_Win_get_name
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_get_name";
-
-
-int MPI_Win_get_name(MPI_Win win, char *win_name, int *resultlen)
-{
-    int ret;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        } else if (NULL == win_name || NULL == resultlen) {
-            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ARG, FUNC_NAME);
-        }
-    }
-
-    /* Note that MPI-2.1 requires:
-       - terminating the string with a \0
-       - name[*resultlen] == '\0'
-       - and therefore (*resultlen) cannot be > (MPI_MAX_OBJECT_NAME-1)
-
-       The Fortran API version will pad to the right if necessary.
-
-       Note that win->name is guaranteed to be \0-terminated and
-       able to completely fit into MPI_MAX_OBJECT_NAME bytes (i.e.,
-       name+\0).  ompi_win_get_name() does the Right things. */
-    ret = ompi_win_get_name(win, win_name, resultlen);
-    OMPI_ERRHANDLER_RETURN(ret, win, ret, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/win_get_name.c.in b/ompi/mpi/c/win_get_name.c.in
new file mode 100644
index 00000000000..49ab8b8fd75
--- /dev/null
+++ b/ompi/mpi/c/win_get_name.c.in
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+
+PROTOTYPE ERROR_CLASS win_get_name(WIN win, STRING_OUT win_name, INT_OUT resultlen)
+{
+    int ret;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        } else if (NULL == win_name || NULL == resultlen) {
+            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ARG, FUNC_NAME);
+        }
+    }
+
+    /* Note that MPI-2.1 requires:
+       - terminating the string with a \0
+       - name[*resultlen] == '\0'
+       - and therefore (*resultlen) cannot be > (MPI_MAX_OBJECT_NAME-1)
+
+       The Fortran API version will pad to the right if necessary.
+
+       Note that win->name is guaranteed to be \0-terminated and
+       able to completely fit into MPI_MAX_OBJECT_NAME bytes (i.e.,
+       name+\0).  ompi_win_get_name() does the Right things. */
+    ret = ompi_win_get_name(win, win_name, resultlen);
+    OMPI_ERRHANDLER_RETURN(ret, win, ret, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/win_lock.c b/ompi/mpi/c/win_lock.c
deleted file mode 100644
index 7b7ac98b2ef..00000000000
--- a/ompi/mpi/c/win_lock.c
+++ /dev/null
@@ -1,66 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2014      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015-2017 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-#include "ompi/mca/osc/osc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_lock = PMPI_Win_lock
-#endif
-#define MPI_Win_lock PMPI_Win_lock
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_lock";
-
-
-int MPI_Win_lock(int lock_type, int rank, int assert, MPI_Win win)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        } else if (lock_type != MPI_LOCK_EXCLUSIVE &&
-                   lock_type != MPI_LOCK_SHARED) {
-            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_LOCKTYPE, FUNC_NAME);
-        } else if (ompi_win_peer_invalid(win, rank)) {
-            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_RANK, FUNC_NAME);
-        } else if (0 != (assert & ~(MPI_MODE_NOCHECK))) {
-            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ASSERT, FUNC_NAME);
-        } else if (! ompi_win_allow_locks(win)) {
-            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_RMA_SYNC, FUNC_NAME);
-        }
-    }
-
-    rc = win->w_osc_module->osc_lock(lock_type, rank, assert, win);
-    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/win_lock.c.in b/ompi/mpi/c/win_lock.c.in
new file mode 100644
index 00000000000..34aa454026b
--- /dev/null
+++ b/ompi/mpi/c/win_lock.c.in
@@ -0,0 +1,58 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2014      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2017 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+#include "ompi/mca/osc/osc.h"
+
+PROTOTYPE ERROR_CLASS win_lock(INT lock_type, INT rank, INT mpi_assert, WIN win)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        } else if (lock_type != MPI_LOCK_EXCLUSIVE &&
+                   lock_type != MPI_LOCK_SHARED) {
+            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_LOCKTYPE, FUNC_NAME);
+        } else if (ompi_win_peer_invalid(win, rank)) {
+            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_RANK, FUNC_NAME);
+        } else if (0 != (mpi_assert & ~(MPI_MODE_NOCHECK))) {
+            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ASSERT, FUNC_NAME);
+        } else if (! ompi_win_allow_locks(win)) {
+            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_RMA_SYNC, FUNC_NAME);
+        }
+    }
+
+    rc = win->w_osc_module->osc_lock(lock_type, rank, mpi_assert, win);
+    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/win_lock_all.c b/ompi/mpi/c/win_lock_all.c
deleted file mode 100644
index 91608ea793e..00000000000
--- a/ompi/mpi/c/win_lock_all.c
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-#include "ompi/mca/osc/osc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_lock_all = PMPI_Win_lock_all
-#endif
-#define MPI_Win_lock_all PMPI_Win_lock_all
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_lock_all";
-
-
-int MPI_Win_lock_all(int assert, MPI_Win win)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        } else if (0 != (assert & ~(MPI_MODE_NOCHECK))) {
-            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ASSERT, FUNC_NAME);
-        } else if (! ompi_win_allow_locks(win)) {
-            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_RMA_SYNC, FUNC_NAME);
-        }
-    }
-
-    rc = win->w_osc_module->osc_lock_all(assert, win);
-    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/win_lock_all.c.in b/ompi/mpi/c/win_lock_all.c.in
new file mode 100644
index 00000000000..3dde356ec4d
--- /dev/null
+++ b/ompi/mpi/c/win_lock_all.c.in
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+#include "ompi/mca/osc/osc.h"
+
+PROTOTYPE ERROR_CLASS win_lock_all(INT mpi_assert, WIN win)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        } else if (0 != (mpi_assert & ~(MPI_MODE_NOCHECK))) {
+            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ASSERT, FUNC_NAME);
+        } else if (! ompi_win_allow_locks(win)) {
+            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_RMA_SYNC, FUNC_NAME);
+        }
+    }
+
+    rc = win->w_osc_module->osc_lock_all(mpi_assert, win);
+    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/win_post.c b/ompi/mpi/c/win_post.c
deleted file mode 100644
index 6cd4914ca8d..00000000000
--- a/ompi/mpi/c/win_post.c
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-#include "ompi/mca/osc/osc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_post = PMPI_Win_post
-#endif
-#define MPI_Win_post PMPI_Win_post
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_post";
-
-
-int MPI_Win_post(MPI_Group group, int assert, MPI_Win win)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        } else if (0 != (assert & ~(MPI_MODE_NOCHECK | MPI_MODE_NOSTORE |
-                                    MPI_MODE_NOPUT))) {
-            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ASSERT, FUNC_NAME);
-        }
-    }
-
-    rc = win->w_osc_module->osc_post(group, assert, win);
-    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/win_post.c.in b/ompi/mpi/c/win_post.c.in
new file mode 100644
index 00000000000..be0587aacd5
--- /dev/null
+++ b/ompi/mpi/c/win_post.c.in
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+#include "ompi/mca/osc/osc.h"
+
+PROTOTYPE ERROR_CLASS win_post(GROUP group, INT mpi_assert, WIN win)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        } else if (0 != (mpi_assert & ~(MPI_MODE_NOCHECK | MPI_MODE_NOSTORE |
+                                    MPI_MODE_NOPUT))) {
+            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ASSERT, FUNC_NAME);
+        }
+    }
+
+    rc = win->w_osc_module->osc_post(group, mpi_assert, win);
+    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/win_set_attr.c b/ompi/mpi/c/win_set_attr.c
deleted file mode 100644
index bdb41cb3e1e..00000000000
--- a/ompi/mpi/c/win_set_attr.c
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/attribute/attribute.h"
-#include "ompi/win/win.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_set_attr = PMPI_Win_set_attr
-#endif
-#define MPI_Win_set_attr PMPI_Win_set_attr
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_set_attr";
-
-
-int MPI_Win_set_attr(MPI_Win win, int win_keyval, void *attribute_val)
-{
-    int ret;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        }
-    }
-
-    ret = ompi_attr_set_c(WIN_ATTR, win, &win->w_keyhash,
-                          win_keyval, attribute_val, false);
-    OMPI_ERRHANDLER_RETURN(ret, win, MPI_ERR_OTHER, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/win_set_attr.c.in b/ompi/mpi/c/win_set_attr.c.in
new file mode 100644
index 00000000000..0776b8b1966
--- /dev/null
+++ b/ompi/mpi/c/win_set_attr.c.in
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/attribute/attribute.h"
+#include "ompi/win/win.h"
+
+PROTOTYPE ERROR_CLASS win_set_attr(WIN win, INT win_keyval, BUFFER_OUT attribute_val)
+{
+    int ret;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        }
+    }
+
+    ret = ompi_attr_set_c(WIN_ATTR, win, &win->w_keyhash,
+                          win_keyval, attribute_val, false);
+    OMPI_ERRHANDLER_RETURN(ret, win, MPI_ERR_OTHER, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/win_set_errhandler.c b/ompi/mpi/c/win_set_errhandler.c
deleted file mode 100644
index 820ba51b330..00000000000
--- a/ompi/mpi/c/win_set_errhandler.c
+++ /dev/null
@@ -1,73 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2008-2009 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015-2017 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_set_errhandler = PMPI_Win_set_errhandler
-#endif
-#define MPI_Win_set_errhandler PMPI_Win_set_errhandler
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_set_errhandler";
-
-
-int MPI_Win_set_errhandler(MPI_Win win, MPI_Errhandler errhandler)
-{
-    MPI_Errhandler tmp;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN,
-                                          FUNC_NAME);
-        } else if (NULL == errhandler ||
-                   MPI_ERRHANDLER_NULL == errhandler ||
-                   (OMPI_ERRHANDLER_TYPE_WIN != errhandler->eh_mpi_object_type &&
-                    OMPI_ERRHANDLER_TYPE_PREDEFINED != errhandler->eh_mpi_object_type) ) {
-            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ARG, FUNC_NAME);
-        }
-    }
-
-    /* Prepare the new error handler */
-    OBJ_RETAIN(errhandler);
-
-    OPAL_THREAD_LOCK(&win->w_lock);
-    /* Ditch the old errhandler, and decrement its refcount. */
-    tmp = win->error_handler;
-    win->error_handler = errhandler;
-    OBJ_RELEASE(tmp);
-    OPAL_THREAD_UNLOCK(&win->w_lock);
-
-    /* All done */
-    return MPI_SUCCESS;
-}
diff --git a/ompi/mpi/c/win_set_errhandler.c.in b/ompi/mpi/c/win_set_errhandler.c.in
new file mode 100644
index 00000000000..80a71e34aad
--- /dev/null
+++ b/ompi/mpi/c/win_set_errhandler.c.in
@@ -0,0 +1,65 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008-2009 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2017 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+
+PROTOTYPE ERROR_CLASS win_set_errhandler(WIN win, ERRHANDLER errhandler)
+{
+    MPI_Errhandler tmp;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN,
+                                          FUNC_NAME);
+        } else if (NULL == errhandler ||
+                   MPI_ERRHANDLER_NULL == errhandler ||
+                   (OMPI_ERRHANDLER_TYPE_WIN != errhandler->eh_mpi_object_type &&
+                    OMPI_ERRHANDLER_TYPE_PREDEFINED != errhandler->eh_mpi_object_type) ) {
+            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ARG, FUNC_NAME);
+        }
+    }
+
+    /* Prepare the new error handler */
+    OBJ_RETAIN(errhandler);
+
+    OPAL_THREAD_LOCK(&win->w_lock);
+    /* Ditch the old errhandler, and decrement its refcount. */
+    tmp = win->error_handler;
+    win->error_handler = errhandler;
+    OBJ_RELEASE(tmp);
+    OPAL_THREAD_UNLOCK(&win->w_lock);
+
+    /* All done */
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/c/win_set_info.c b/ompi/mpi/c/win_set_info.c
deleted file mode 100644
index bb9c6bb572b..00000000000
--- a/ompi/mpi/c/win_set_info.c
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright (c) 2013      Sandia National Laboratories.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-#include "ompi/communicator/communicator.h"
-#include "opal/util/info_subscriber.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_set_info = PMPI_Win_set_info
-#endif
-#define MPI_Win_set_info PMPI_Win_set_info
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_set_info";
-
-
-int MPI_Win_set_info(MPI_Win win, MPI_Info info)
-{
-    int ret;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        }
-
-        if (NULL == info || MPI_INFO_NULL == info ||
-            ompi_info_is_freed(info)) {
-            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_INFO, FUNC_NAME);
-        }
-    }
-
-    ret = opal_infosubscribe_change_info(&(win->super), &(info->super));
-
-    OMPI_ERRHANDLER_RETURN(ret, win, ret, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/win_set_info.c.in b/ompi/mpi/c/win_set_info.c.in
new file mode 100644
index 00000000000..d1b376ca305
--- /dev/null
+++ b/ompi/mpi/c/win_set_info.c.in
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2013      Sandia National Laboratories.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+#include "ompi/communicator/communicator.h"
+#include "opal/util/info_subscriber.h"
+
+PROTOTYPE ERROR_CLASS win_set_info(WIN win, INFO info)
+{
+    int ret;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        }
+
+        if (NULL == info || MPI_INFO_NULL == info ||
+            ompi_info_is_freed(info)) {
+            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_INFO, FUNC_NAME);
+        }
+    }
+
+    ret = opal_infosubscribe_change_info(&(win->super), &(info->super));
+
+    OMPI_ERRHANDLER_RETURN(ret, win, ret, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/win_set_name.c b/ompi/mpi/c/win_set_name.c
deleted file mode 100644
index 2d20c5a4d10..00000000000
--- a/ompi/mpi/c/win_set_name.c
+++ /dev/null
@@ -1,59 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_set_name = PMPI_Win_set_name
-#endif
-#define MPI_Win_set_name PMPI_Win_set_name
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_set_name";
-
-
-int MPI_Win_set_name(MPI_Win win, const char *win_name)
-{
-    int ret;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        } else if (NULL == win_name) {
-            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ARG, FUNC_NAME);
-        }
-    }
-
-    ret = ompi_win_set_name(win, win_name);
-    OMPI_ERRHANDLER_RETURN(ret, win, ret, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/win_set_name.c.in b/ompi/mpi/c/win_set_name.c.in
new file mode 100644
index 00000000000..fdbd842fe57
--- /dev/null
+++ b/ompi/mpi/c/win_set_name.c.in
@@ -0,0 +1,51 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+
+PROTOTYPE ERROR_CLASS win_set_name(WIN win, STRING win_name)
+{
+    int ret;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        } else if (NULL == win_name) {
+            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ARG, FUNC_NAME);
+        }
+    }
+
+    ret = ompi_win_set_name(win, win_name);
+    OMPI_ERRHANDLER_RETURN(ret, win, ret, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/win_shared_query.c b/ompi/mpi/c/win_shared_query.c
deleted file mode 100644
index 100d9bb599b..00000000000
--- a/ompi/mpi/c/win_shared_query.c
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved.
- * Copyright (c) 2015-2017 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-#include "ompi/mca/osc/osc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_shared_query = PMPI_Win_shared_query
-#endif
-#define MPI_Win_shared_query PMPI_Win_shared_query
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_shared_query";
-
-
-int MPI_Win_shared_query(MPI_Win win, int rank, MPI_Aint *size, int *disp_unit, void *baseptr)
-{
-    int rc;
-    size_t tsize;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-         } else if (MPI_PROC_NULL != rank && ompi_win_peer_invalid(win, rank)) {
-            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_RANK, FUNC_NAME);
-         }
-    }
-
-    if (NULL != win->w_osc_module->osc_win_shared_query) {
-        rc = win->w_osc_module->osc_win_shared_query(win, rank, &tsize, disp_unit, baseptr);
-        *size = tsize;
-    } else {
-        rc = MPI_ERR_RMA_FLAVOR;
-    }
-    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/win_shared_query.c.in b/ompi/mpi/c/win_shared_query.c.in
new file mode 100644
index 00000000000..ad88189428f
--- /dev/null
+++ b/ompi/mpi/c/win_shared_query.c.in
@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2015-2017 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c)      2024 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+#include "ompi/mca/osc/osc.h"
+
+PROTOTYPE ERROR_CLASS win_shared_query(WIN win, INT rank, AINT_OUT size, INT_AINT_OUT disp_unit, BUFFER_OUT baseptr)
+{
+    int rc;
+    size_t tsize;
+    ptrdiff_t du;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+         } else if (MPI_PROC_NULL != rank && ompi_win_peer_invalid(win, rank)) {
+            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_RANK, FUNC_NAME);
+         }
+    }
+
+    if (NULL != win->w_osc_module->osc_win_shared_query) {
+        rc = win->w_osc_module->osc_win_shared_query(win, rank, &tsize, &du, baseptr);
+        *size = tsize;
+        *disp_unit = du;
+    } else {
+        rc = MPI_ERR_RMA_FLAVOR;
+    }
+    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/win_start.c b/ompi/mpi/c/win_start.c
deleted file mode 100644
index 19f7658120e..00000000000
--- a/ompi/mpi/c/win_start.c
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-#include "ompi/mca/osc/osc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_start = PMPI_Win_start
-#endif
-#define MPI_Win_start PMPI_Win_start
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_start";
-
-
-int MPI_Win_start(MPI_Group group, int assert, MPI_Win win)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        } else if (0 != (assert & ~(MPI_MODE_NOCHECK))) {
-            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ASSERT, FUNC_NAME);
-        }
-    }
-
-    rc = win->w_osc_module->osc_start(group, assert, win);
-    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/win_start.c.in b/ompi/mpi/c/win_start.c.in
new file mode 100644
index 00000000000..ae7254c8f10
--- /dev/null
+++ b/ompi/mpi/c/win_start.c.in
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+#include "ompi/mca/osc/osc.h"
+
+PROTOTYPE ERROR_CLASS win_start(GROUP group, INT mpi_assert, WIN win)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        } else if (0 != (mpi_assert & ~(MPI_MODE_NOCHECK))) {
+            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_ASSERT, FUNC_NAME);
+        }
+    }
+
+    rc = win->w_osc_module->osc_start(group, mpi_assert, win);
+    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/win_sync.c b/ompi/mpi/c/win_sync.c
deleted file mode 100644
index 11f05e47c01..00000000000
--- a/ompi/mpi/c/win_sync.c
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/info/info.h"
-#include "ompi/win/win.h"
-#include "ompi/memchecker.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_sync = PMPI_Win_sync
-#endif
-#define MPI_Win_sync PMPI_Win_sync
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_sync";
-
-int MPI_Win_sync(MPI_Win win)
-{
-    int ret = MPI_SUCCESS;
-
-    /* argument checking */
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        }
-    }
-
-    ret = win->w_osc_module->osc_sync(win);
-    OMPI_ERRHANDLER_RETURN(ret, win, ret, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/win_sync.c.in b/ompi/mpi/c/win_sync.c.in
new file mode 100644
index 00000000000..ad28415a559
--- /dev/null
+++ b/ompi/mpi/c/win_sync.c.in
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/info/info.h"
+#include "ompi/win/win.h"
+#include "ompi/memchecker.h"
+
+PROTOTYPE ERROR_CLASS win_sync(WIN win)
+{
+    int ret = MPI_SUCCESS;
+
+    /* argument checking */
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        }
+    }
+
+    ret = win->w_osc_module->osc_sync(win);
+    OMPI_ERRHANDLER_RETURN(ret, win, ret, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/win_test.c b/ompi/mpi/c/win_test.c
deleted file mode 100644
index a2ef3c8b448..00000000000
--- a/ompi/mpi/c/win_test.c
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-#include "ompi/mca/osc/osc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_test = PMPI_Win_test
-#endif
-#define MPI_Win_test PMPI_Win_test
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_test";
-
-
-int MPI_Win_test(MPI_Win win, int *flag)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        }
-    }
-
-    rc = win->w_osc_module->osc_test(win, flag);
-    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/win_test.c.in b/ompi/mpi/c/win_test.c.in
new file mode 100644
index 00000000000..b3cb93a155b
--- /dev/null
+++ b/ompi/mpi/c/win_test.c.in
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+#include "ompi/mca/osc/osc.h"
+
+PROTOTYPE ERROR_CLASS win_test(WIN win, INT_OUT flag)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        }
+    }
+
+    rc = win->w_osc_module->osc_test(win, flag);
+    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/win_unlock.c b/ompi/mpi/c/win_unlock.c
deleted file mode 100644
index 3e08087b553..00000000000
--- a/ompi/mpi/c/win_unlock.c
+++ /dev/null
@@ -1,59 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2014      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2017 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-#include "ompi/mca/osc/osc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_unlock = PMPI_Win_unlock
-#endif
-#define MPI_Win_unlock PMPI_Win_unlock
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_unlock";
-
-
-int MPI_Win_unlock(int rank, MPI_Win win)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        } else if (ompi_win_peer_invalid(win, rank)) {
-            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_RANK, FUNC_NAME);
-        }
-    }
-
-    rc = win->w_osc_module->osc_unlock(rank, win);
-    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/win_unlock.c.in b/ompi/mpi/c/win_unlock.c.in
new file mode 100644
index 00000000000..7beaa304b00
--- /dev/null
+++ b/ompi/mpi/c/win_unlock.c.in
@@ -0,0 +1,51 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2014      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2017 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+#include "ompi/mca/osc/osc.h"
+
+PROTOTYPE ERROR_CLASS win_unlock(INT rank, WIN win)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        } else if (ompi_win_peer_invalid(win, rank)) {
+            return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_RANK, FUNC_NAME);
+        }
+    }
+
+    rc = win->w_osc_module->osc_unlock(rank, win);
+    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/win_unlock_all.c b/ompi/mpi/c/win_unlock_all.c
deleted file mode 100644
index cbf01986a78..00000000000
--- a/ompi/mpi/c/win_unlock_all.c
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-#include "ompi/mca/osc/osc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_unlock_all = PMPI_Win_unlock_all
-#endif
-#define MPI_Win_unlock_all PMPI_Win_unlock_all
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_unlock_all";
-
-
-int MPI_Win_unlock_all(MPI_Win win)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        }
-    }
-
-    rc = win->w_osc_module->osc_unlock_all(win);
-    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/win_unlock_all.c.in b/ompi/mpi/c/win_unlock_all.c.in
new file mode 100644
index 00000000000..a3d0b094f75
--- /dev/null
+++ b/ompi/mpi/c/win_unlock_all.c.in
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+#include "ompi/mca/osc/osc.h"
+
+PROTOTYPE ERROR_CLASS win_unlock_all(WIN win)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        }
+    }
+
+    rc = win->w_osc_module->osc_unlock_all(win);
+    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/win_wait.c b/ompi/mpi/c/win_wait.c
deleted file mode 100644
index d55ad56f999..00000000000
--- a/ompi/mpi/c/win_wait.c
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/win/win.h"
-#include "ompi/mca/osc/osc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Win_wait = PMPI_Win_wait
-#endif
-#define MPI_Win_wait PMPI_Win_wait
-#endif
-
-static const char FUNC_NAME[] = "MPI_Win_wait";
-
-
-int MPI_Win_wait(MPI_Win win)
-{
-    int rc;
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-
-        if (ompi_win_invalid(win)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
-        }
-    }
-
-    rc = win->w_osc_module->osc_wait(win);
-    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
-}
diff --git a/ompi/mpi/c/win_wait.c.in b/ompi/mpi/c/win_wait.c.in
new file mode 100644
index 00000000000..7ad43c67337
--- /dev/null
+++ b/ompi/mpi/c/win_wait.c.in
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/win/win.h"
+#include "ompi/mca/osc/osc.h"
+
+PROTOTYPE ERROR_CLASS win_wait(WIN win)
+{
+    int rc;
+
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_win_invalid(win)) {
+            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME);
+        }
+    }
+
+    rc = win->w_osc_module->osc_wait(win);
+    OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME);
+}
diff --git a/ompi/mpi/c/wtick.c b/ompi/mpi/c/wtick.c
index f6504dccee2..3136a7e1b08 100644
--- a/ompi/mpi/c/wtick.c
+++ b/ompi/mpi/c/wtick.c
@@ -9,7 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2007-2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2007-2022 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2015-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2017      IBM Corporation.  All rights reserved.
@@ -31,10 +31,11 @@
 #include <time.h>
 #endif
 
-#include MCA_timer_IMPLEMENTATION_HEADER
 #include "ompi/mpi/c/bindings.h"
 #include "ompi/runtime/mpiruntime.h"
 
+#include "opal/util/clock_gettime.h"
+
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_Wtick = PMPI_Wtick
@@ -44,38 +45,15 @@
 
 double MPI_Wtick(void)
 {
-    /*
-     * See https://github.com/open-mpi/ompi/issues/3003
-     * to get an idea what's going on here.
-     */
-#if 0
-#if OPAL_TIMER_CYCLE_NATIVE
-    {
-        opal_timer_t freq = opal_timer_base_get_freq();
-        if (0 == freq) {
-            /* That should never happen, but if it does, return a bogus value
-             * rather than crashing with a division by zero */
-            return (double)0.0;
-        }
-        return (double)1.0 / (double)freq;
-    }
-#elif OPAL_TIMER_USEC_NATIVE
-    return 0.000001;
-#endif
-#else
-#if defined(__linux__) && OPAL_HAVE_CLOCK_GETTIME
+    // We intentionally don't use the OPAL timer framework here.  See
+    // https://github.com/open-mpi/ompi/issues/3003 for more details.
     struct timespec spec;
     double wtick = 0.0;
-    if (0 == clock_getres(CLOCK_MONOTONIC, &spec)){
+    if (0 == opal_clock_getres(&spec)){
         wtick =  spec.tv_sec + spec.tv_nsec * 1.0e-09;
     } else {
         /* guess */
         wtick = 1.0e-09;
     }
     return wtick;
-#else
-    /* Otherwise, we already return usec precision. */
-    return 0.000001;
-#endif
-#endif
 }
diff --git a/ompi/mpi/c/wtime.c b/ompi/mpi/c/wtime.c
deleted file mode 100644
index e22d26d943c..00000000000
--- a/ompi/mpi/c/wtime.c
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2018 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2014 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation.  All rights reserved.
- * Copyright (c) 2017      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-
-#ifdef HAVE_SYS_TIME_H
-#include <sys/time.h>
-#endif
-#include <stdio.h>
-#ifdef HAVE_TIME_H
-#include <time.h>
-#endif  /* HAVE_TIME_H */
-
-#include MCA_timer_IMPLEMENTATION_HEADER
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/mpiruntime.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPI_Wtime = PMPI_Wtime
-#endif
-#define MPI_Wtime PMPI_Wtime
-/**
- * Have a base time set on the first call to wtime, to improve the range
- * and accuracy of the user visible timer.
- * More info: https://github.com/mpi-forum/mpi-issues/issues/77#issuecomment-369663119
- */
-#if defined(__linux__) && OPAL_HAVE_CLOCK_GETTIME
-struct timespec ompi_wtime_time_origin = {.tv_sec = 0};
-#else
-struct timeval ompi_wtime_time_origin = {.tv_sec = 0};
-#endif
-#else  /* OMPI_BUILD_MPI_PROFILING */
-#if defined(__linux__) && OPAL_HAVE_CLOCK_GETTIME
-extern struct timespec ompi_wtime_time_origin;
-#else
-extern struct timeval ompi_wtime_time_origin;
-#endif
-#endif
-
-double MPI_Wtime(void)
-{
-    double wtime;
-
-    SPC_RECORD(OMPI_SPC_WTIME, 1);
-
-    /*
-     * See https://github.com/open-mpi/ompi/issues/3003 to find out
-     * what's happening here.
-     */
-#if 0
-#if OPAL_TIMER_CYCLE_NATIVE
-    wtime = ((double) opal_timer_base_get_cycles()) / opal_timer_base_get_freq();
-#elif OPAL_TIMER_USEC_NATIVE
-    wtime = ((double) opal_timer_base_get_usec()) / 1000000.0;
-#endif
-#else
-#if defined(__linux__) && OPAL_HAVE_CLOCK_GETTIME
-    struct timespec tp;
-    (void) clock_gettime(CLOCK_MONOTONIC, &tp);
-    if( OPAL_UNLIKELY(0 == ompi_wtime_time_origin.tv_sec) ) {
-        ompi_wtime_time_origin = tp;
-    }
-    wtime  = (double)(tp.tv_nsec - ompi_wtime_time_origin.tv_nsec)/1.0e+9;
-    wtime += (tp.tv_sec - ompi_wtime_time_origin.tv_sec);
-#else
-    /* Fall back to gettimeofday() if we have nothing else */
-    struct timeval tv;
-    gettimeofday(&tv, NULL);
-    if( OPAL_UNLIKELY(0 == ompi_wtime_time_origin.tv_sec) ) {
-        ompi_wtime_time_origin = tv;
-    }
-    wtime  = (double)(tv.tv_usec - ompi_wtime_time_origin.tv_usec) / 1.0e+6;
-    wtime += (tv.tv_sec - ompi_wtime_time_origin.tv_sec);
-#endif
-#endif
-
-    return wtime;
-}
diff --git a/ompi/mpi/c/wtime.c.in b/ompi/mpi/c/wtime.c.in
new file mode 100644
index 00000000000..e31db4d630e
--- /dev/null
+++ b/ompi/mpi/c/wtime.c.in
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2022 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2017      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
+#include <stdio.h>
+#ifdef HAVE_TIME_H
+#include <time.h>
+#endif  /* HAVE_TIME_H */
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/mpiruntime.h"
+#include "ompi/runtime/ompi_spc.h"
+
+#include "opal/util/clock_gettime.h"
+
+/**
+ * Use this as a base time set early during MPI initialization to improve the range
+ * and accuracy of the user visible timer.
+ * More info: https://github.com/mpi-forum/mpi-issues/issues/77#issuecomment-369663119
+ */
+extern struct timespec ompi_wtime_time_origin;
+
+PROTOTYPE DOUBLE wtime()
+{
+    double wtime;
+
+    SPC_RECORD(OMPI_SPC_WTIME, 1);
+
+    // We intentionally don't use the OPAL timer framework here.  See
+    // https://github.com/open-mpi/ompi/issues/3003 for more details.
+    struct timespec tp;
+    (void) opal_clock_gettime(&tp);
+    wtime  = (double)(tp.tv_nsec - ompi_wtime_time_origin.tv_nsec)/1.0e+9;
+    wtime += (tp.tv_sec - ompi_wtime_time_origin.tv_sec);
+
+    return wtime;
+}
diff --git a/ompi/mpi/fortran/base/datarep.h b/ompi/mpi/fortran/base/datarep.h
index 2f94c10dee2..51f8392d962 100644
--- a/ompi/mpi/fortran/base/datarep.h
+++ b/ompi/mpi/fortran/base/datarep.h
@@ -59,7 +59,7 @@ typedef void (ompi_mpi2_fortran_datarep_extent_fn_t)
 
 /**
  * Macro for declaring each of the 5 back-end Fortran functions for
- * MPI_CONVERSION_FN_NULL.  We need the 4 fortran compiler convetions
+ * MPI_CONVERSION_FN_NULL.  We need the 4 fortran compiler conversions
  * and 1 for the "real" back-end function (even though these functions
  * are never invoked -- they're only used as sentinel values -- it's
  * simpler to use the same kind of code structure that we use for the
diff --git a/ompi/mpi/fortran/base/fint_2_int.h b/ompi/mpi/fortran/base/fint_2_int.h
index 44ce1289567..15b55eaa867 100644
--- a/ompi/mpi/fortran/base/fint_2_int.h
+++ b/ompi/mpi/fortran/base/fint_2_int.h
@@ -13,6 +13,7 @@
  * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
  * Copyright (c) 2014-2019 Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -47,9 +48,10 @@
   #define OMPI_SINGLE_FINT_2_INT(in)
   #define OMPI_SINGLE_INT_2_FINT(in)
   #define OMPI_ARRAY_INT_2_FINT(in, n)
+  #define OMPI_COND_STATEMENT(a)
 
 #elif OMPI_SIZEOF_FORTRAN_INTEGER > SIZEOF_INT
-  #define OMPI_ARRAY_NAME_DECL(a) int *c_##a
+  #define OMPI_ARRAY_NAME_DECL(a) int *c_##a = NULL
   #define OMPI_2_DIM_ARRAY_NAME_DECL(a, dim2) int (*c_##a)[dim2], dim2_index
   #define OMPI_SINGLE_NAME_DECL(a) int c_##a
   #define OMPI_ARRAY_NAME_CONVERT(a) c_##a
@@ -60,7 +62,7 @@
 
   /* This is for OUT parameters. Does only alloc */
   #define OMPI_ARRAY_FINT_2_INT_ALLOC(in, n) \
-    OMPI_ARRAY_NAME_CONVERT(in) = malloc(n * sizeof(int))
+    OMPI_ARRAY_NAME_CONVERT(in) = malloc((n) * sizeof(int))
 
   /* This is for IN/IN-OUT parameters. Does alloc and assignment */
   #define OMPI_ARRAY_FINT_2_INT(in, n) \
@@ -86,7 +88,8 @@
 
   /* This is for IN parameters. Does only free */
   #define OMPI_ARRAY_FINT_2_INT_CLEANUP(in) \
-    free(OMPI_ARRAY_NAME_CONVERT(in))
+    if (NULL != OMPI_ARRAY_NAME_CONVERT(in)) \
+      free(OMPI_ARRAY_NAME_CONVERT(in))
 
   /* This is for single IN parameter */
   #define OMPI_SINGLE_FINT_2_INT(in) \
@@ -96,7 +99,7 @@
   #define OMPI_SINGLE_INT_2_FINT(in) \
     *(in) = OMPI_ARRAY_NAME_CONVERT(in)
 
-  /* This is for OUT/IN-OUT parametes. Does back assignment and free */
+  /* This is for OUT/IN-OUT parameters. Does back assignment and free */
   #define OMPI_ARRAY_INT_2_FINT(in, n) \
     do { \
       int converted_n = (int)(n); \
@@ -105,8 +108,10 @@
       } \
       free(OMPI_ARRAY_NAME_CONVERT(in)); \
     } while (0)
+
+  #define OMPI_COND_STATEMENT(a) a
 #else /* int > MPI_Fint  */
-  #define OMPI_ARRAY_NAME_DECL(a) int *c_##a
+  #define OMPI_ARRAY_NAME_DECL(a) int *c_##a = NULL
   #define OMPI_2_DIM_ARRAY_NAME_DECL(a, dim2) int (*c_##a)[dim2], dim2_index
   #define OMPI_SINGLE_NAME_DECL(a) int c_##a
   #define OMPI_ARRAY_NAME_CONVERT(a) c_##a
@@ -117,7 +122,7 @@
 
   /* This is for OUT parameters. Does only alloc */
   #define OMPI_ARRAY_FINT_2_INT_ALLOC(in, n) \
-    OMPI_ARRAY_NAME_CONVERT(in) = malloc(n * sizeof(int))
+    OMPI_ARRAY_NAME_CONVERT(in) = malloc((n) * sizeof(int))
 
   #define OMPI_ARRAY_FINT_2_INT(in, n) \
     do { \
@@ -140,7 +145,8 @@
     } while (0)
 
   #define OMPI_ARRAY_FINT_2_INT_CLEANUP(in) \
-    free(OMPI_ARRAY_NAME_CONVERT(in))
+    if (NULL != OMPI_ARRAY_NAME_CONVERT(in)) \
+      free(OMPI_ARRAY_NAME_CONVERT(in))
 
   #define OMPI_SINGLE_FINT_2_INT(in) \
      OMPI_ARRAY_NAME_CONVERT(in) = *(in)
@@ -157,6 +163,7 @@
       free(OMPI_ARRAY_NAME_CONVERT(in)); \
     } while (0)
 
+  #define OMPI_COND_STATEMENT(a) a
 #endif
 
 /*
@@ -204,7 +211,7 @@
 #  define OMPI_LOGICAL_ARRAY_NAME_DECL(in)     int * c_##in
 #  define OMPI_LOGICAL_ARRAY_NAME_CONVERT(in)  c_##in
 #  define OMPI_ARRAY_LOGICAL_2_INT_ALLOC(in,n) \
-      OMPI_LOGICAL_ARRAY_NAME_CONVERT(in) = malloc(n * sizeof(int))
+      OMPI_LOGICAL_ARRAY_NAME_CONVERT(in) = malloc((n) * sizeof(int))
 #  define OMPI_ARRAY_LOGICAL_2_INT_CLEANUP(in) \
       free(OMPI_LOGICAL_ARRAY_NAME_CONVERT(in))
 
diff --git a/ompi/mpi/fortran/base/fortran_base_strings.h b/ompi/mpi/fortran/base/fortran_base_strings.h
index c1e4f7513e7..5ad59375ebb 100644
--- a/ompi/mpi/fortran/base/fortran_base_strings.h
+++ b/ompi/mpi/fortran/base/fortran_base_strings.h
@@ -55,7 +55,7 @@ BEGIN_C_DECLS
      * convert C strings to fortran strings.  It is assumed that the
      * fortran string is already allocated and has a length of len.
      */
-    OMPI_DECLSPEC int ompi_fortran_string_c2f(char *cstr, char *fstr, int len);
+    OMPI_DECLSPEC int ompi_fortran_string_c2f(const char* cstr, char* fstr, int len);
 
     /**
      * Convert an array of Fortran strings that are terminated with a
@@ -69,7 +69,7 @@ BEGIN_C_DECLS
      * @retval OMPI_SUCCESS upon success
      * @retval OMPI_ERROR upon error
      *
-     * This function is intented to be used in the MPI F77 bindings to
+     * This function is intended to be used in the MPI F77 bindings to
      * convert arrays of fortran strings to argv-style arrays of C
      * strings.  The argv array will be allocated and returned; it is
      * the caller's responsibility to invoke opal_argv_free() to free
@@ -115,9 +115,9 @@ BEGIN_C_DECLS
      * @param count Dimension of the array of argvs
      * @param array Array of fortran argv
      * @param len Length of Fortran array
-     * @param argv Returned C arrray of argvs
+     * @param argv Returned C array of argvs
      *
-     * This function is intented to be used in the MPI F77 bindings to
+     * This function is intended to be used in the MPI F77 bindings to
      * convert arrays of fortran strings to argv-style arrays of C
      * strings.  The argv array will be allocated and returned; it is
      * the caller's responsibility to invoke opal_argv_free() to free
diff --git a/ompi/mpi/fortran/base/gen-mpi-mangling.pl b/ompi/mpi/fortran/base/gen-mpi-mangling.pl
index a049c625228..5e4046178c3 100755
--- a/ompi/mpi/fortran/base/gen-mpi-mangling.pl
+++ b/ompi/mpi/fortran/base/gen-mpi-mangling.pl
@@ -1,6 +1,6 @@
 #!/usr/bin/env perl
 #
-# Copyright (c) 2015      Research Organization for Information Science
+# Copyright (c) 2015-2025 Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
 # Copyright (c) 2015-2020 Cisco Systems, Inc.  All rights reserved.
 # $COPYRIGHT$
@@ -16,6 +16,8 @@
 my $plain_arg;
 my $single_underscore_arg;
 my $double_underscore_arg;
+my $status_size_arg;
+my $align_arg;
 my $help_arg = 0;
 
 &Getopt::Long::Configure("bundling");
@@ -23,10 +25,12 @@
                                   "plain=i" => \$plain_arg,
                                   "single=i" => \$single_underscore_arg,
                                   "double=i" => \$double_underscore_arg,
+                                  "status-size=i" => \$status_size_arg,
+                                  "align=i" => \$align_arg,
                                   "help|h" => \$help_arg);
 
 if ($help_arg || !$ok) {
-    print "Usage: $0 [--caps|--plain|--single|--double] [--help]\n";
+    print "Usage: $0 [--caps|--plain|--single|--double] --status-size=<MPI_STATUS_SIZE> [--align=<alignment>] [--help]\n";
     exit(1 - $ok);
 }
 
@@ -50,25 +54,25 @@
 my $fortran;
 
 $fortran->{bottom} = {
-    c_type => "int",
+    c_type => "MPI_Fint",
     c_name => "mpi_fortran_bottom",
     f_type => "integer",
     f_name => "MPI_BOTTOM",
 };
 $fortran->{in_place} = {
-    c_type => "int",
+    c_type => "MPI_Fint",
     c_name => "mpi_fortran_in_place",
     f_type => "integer",
     f_name => "MPI_IN_PLACE",
 };
 $fortran->{unweighted} = {
-    c_type => "int",
+    c_type => "MPI_Fint",
     c_name => "mpi_fortran_unweighted",
     f_type => "integer, dimension(1)",
     f_name => "MPI_UNWEIGHTED",
 };
 $fortran->{weights_empty} = {
-    c_type => "int",
+    c_type => "MPI_Fint",
     c_name => "mpi_fortran_weights_empty",
     f_type => "integer, dimension(1)",
     f_name => "MPI_WEIGHTS_EMPTY",
@@ -88,19 +92,21 @@
 };
 
 $fortran->{errcodes_ignore} = {
-    c_type => "int",
+    c_type => "MPI_Fint",
     c_name => "mpi_fortran_errcodes_ignore",
     f_type => "integer, dimension(1)",
     f_name => "MPI_ERRCODES_IGNORE",
 };
 $fortran->{status_ignore} = {
-    c_type => "int *",
+    c_type => "MPI_Fint",
+    c_dim  => "[$status_size_arg]",
     c_name => "mpi_fortran_status_ignore",
     f_type => "type(MPI_STATUS)",
     f_name => "MPI_STATUS_IGNORE",
 };
 $fortran->{statuses_ignore} = {
-    c_type => "int *",
+    c_type => "MPI_Fint",
+    c_dim  => "[$status_size_arg]",
     c_name => "mpi_fortran_statuses_ignore",
     f_type => "type(MPI_STATUS)",
     f_name => "MPI_STATUSES_IGNORE(1)",
@@ -130,7 +136,7 @@ sub gen_c_constants_decl {
 
     print OUT "/* WARNING: This is a generated file!  Edits will be lost! */
 /*
- * Copyright (c) 2015      Research Organization for Information Science
+ * Copyright (c) 2015-2025 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2015      Cisco Systems, Inc.  All rights reserved.
  * \$COPYRIGHT\$
@@ -146,7 +152,7 @@ sub gen_c_constants_decl {
     foreach my $key (sort(keys(%{$fortran}))) {
         my $f = $fortran->{$key};
         my $m = mangle($f->{c_name});
-        print OUT "extern $f->{c_type} $m;
+        print OUT "extern $f->{c_type} $m $f->{c_dim};
 #define OMPI_IS_FORTRAN_" . uc($key) . "(addr) \\
         (addr == (void*) &$m)\n\n";
     }
@@ -160,7 +166,7 @@ sub gen_c_constants {
 
     print OUT "/* WARNING: This is a generated file!  Edits will be lost! */
 /*
- * Copyright (c) 2015      Research Organization for Information Science
+ * Copyright (c) 2015-2025 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2015      Cisco Systems, Inc.  All rights reserved.
  * \$COPYRIGHT\$
@@ -171,7 +177,9 @@ sub gen_c_constants {
     foreach my $key (sort(keys(%{$fortran}))) {
         my $f = $fortran->{$key};
         my $m = mangle($f->{c_name});
-        print OUT "$f->{c_type} $m;\n";
+	my $align = "";
+	$align = "__opal_attribute_aligned__($align_arg)" if ($align_arg > 0);
+        print OUT "$f->{c_type} $align $m $f->{c_dim};\n";
     }
 
     close (OUT);
diff --git a/ompi/mpi/fortran/base/gen-mpi-sizeof.pl b/ompi/mpi/fortran/base/gen-mpi-sizeof.pl
index 194e7b0e7f6..17c37f272c5 100755
--- a/ompi/mpi/fortran/base/gen-mpi-sizeof.pl
+++ b/ompi/mpi/fortran/base/gen-mpi-sizeof.pl
@@ -1,8 +1,9 @@
 #!/usr/bin/env perl
 #
 # Copyright (c) 2014-2015 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2015-2018 Research Organization for Information Science
+# Copyright (c) 2015-2021 Research Organization for Information Science
 #                         and Technology (RIST).  All rights reserved.
+# Copyright (c) 2022      IBM Corporation.  All rights reserved.
 # $COPYRIGHT$
 #
 # Script to generate the overloaded MPI_SIZEOF interfaces and
@@ -33,11 +34,14 @@
 my $generate_arg;
 my $mpi_arg;
 my $mpi_real2;
+my $mpi_iso_real16;
 my $mpi_real16;
 my $mpi_complex4;
 my $mpi_complex32;
 my $pmpi_arg;
 my $help_arg = 0;
+my $request_deprecate = 0;
+my $mpi_version = 0;
 
 &Getopt::Long::Configure("bundling");
 my $ok = Getopt::Long::GetOptions("complex32=i" => \$mpi_complex32,
@@ -51,6 +55,9 @@
                                   "pmpi" => \$pmpi_arg,
                                   "real16=i" => \$mpi_real16,
                                   "real2=i" => \$mpi_real2,
+                                  "iso_real16=i" => \$mpi_iso_real16,
+                                  "request_deprecate=i" => \$request_deprecate,
+                                  "mpi_version=i" => \$mpi_version,
                                   "help|h" => \$help_arg);
 
 die "Must specify header and/or impl filenames to output"
@@ -64,8 +71,8 @@
 die "Must specify --pmpi and/or --mpi if --impl is specified"
     if (defined($generate_arg) && $generate_arg &&
         (defined($impl_arg) && !defined($mpi_arg) && !defined($pmpi_arg)));
-die "Must specify real2, real16, complex4, and complex32"
-    if (!defined($mpi_real2) || !defined($mpi_real16) ||
+die "Must specify real2, iso_real16, real16, complex4, and complex32"
+    if (!defined($mpi_real2) || !defined($mpi_iso_real16) || !defined($mpi_real16) ||
         !defined($mpi_complex4) || !defined($mpi_complex32));
 
 #############################################################################
@@ -104,7 +111,13 @@ sub queue_sub {
     $subr->{start} = $start;
     $subr->{middle} = "${indent}  size = storage_size(x) / 8
 ${indent}  ${optional_ierror_statement}ierror = 0";
-    $subr->{end} = "${indent}END SUBROUTINE ^PREFIX^$sub_name^RANK^";
+    if (($mpi_version >= 4) && ($request_deprecate == 1)) {
+        $subr->{end} = "!GCC\$ ATTRIBUTES DEPRECATED :: ^PREFIX^$sub_name^RANK^\n";
+    }
+    else {
+        $subr->{end} = "";
+    }
+    $subr->{end} .= "${indent}END SUBROUTINE ^PREFIX^$sub_name^RANK^";
 
     # Save it in the overall hash
     $subs->{$sub_name} = $subr;
@@ -154,15 +167,19 @@ sub generate {
     queue_sub("integer(int${size})", "int${size}", "int${size}");
 }
 for my $size (qw/16 32 64 128/) {
-    if (!($size == 16 && $mpi_real2 == 0) &&
-        !($size == 128 &&$mpi_real16 == 0)) {
+    if (!($size == 16 && $mpi_real2 == 0 && $mpi_iso_real16 == 0) &&
+        !($size == 128 && $mpi_real16 == 0)) {
         queue_sub("real(real${size})", "real${size}", "real${size}");
     }
-    if (!($size == 16 && $mpi_complex4 == 0) &&
+    if (!($size == 16 && $mpi_complex4 == 0 && $mpi_iso_real16 == 0) &&
         !($size == 128 && $mpi_complex32 == 0)) {
         queue_sub("complex(real${size})", "complex${size}", "real${size}");
     }
 }
+if ($mpi_real2 == 1 && $mpi_iso_real16 == 0) {
+    queue_sub("real*2", "real16");
+    queue_sub("complex*4", "complex16");
+}
 queue_sub("character", "character");
 queue_sub("logical", "logical");
 
@@ -227,7 +244,7 @@ sub output_file {
 ! Specifically: we need support for the INTERFACE keyword,
 ! ISO_FORTRAN_ENV, and the STORAGE_SIZE() intrinsic on all types.
 ! Apparently, this compiler does not support both of those things, so
-! this file will be (effecitvely) blank (i.e., we didn't bother
+! this file will be (effectively) blank (i.e., we didn't bother
 ! generating the necessary stuff for MPI_SIZEOF because the compiler
 ! doesn't support
 ! it).
diff --git a/ompi/mpi/fortran/base/strings.c b/ompi/mpi/fortran/base/strings.c
index 95645a1d90f..5bbd96d5eea 100644
--- a/ompi/mpi/fortran/base/strings.c
+++ b/ompi/mpi/fortran/base/strings.c
@@ -90,7 +90,7 @@ int ompi_fortran_string_f2c(char *fstr, int len, char **cstr)
  * shorter string, or reading a shorter record, automatically pads the
  * rest of the string with blanks."
  */
-int ompi_fortran_string_c2f(char *cstr, char *fstr, int len)
+int ompi_fortran_string_c2f(const char *cstr, char *fstr, int len)
 {
     int i;
 
diff --git a/ompi/mpi/fortran/configure-fortran-output.h.in b/ompi/mpi/fortran/configure-fortran-output.h.in
index 2c96d83a2b6..ed239693b15 100644
--- a/ompi/mpi/fortran/configure-fortran-output.h.in
+++ b/ompi/mpi/fortran/configure-fortran-output.h.in
@@ -43,6 +43,9 @@
 ! Line 2 of the ignore TKR syntax
 #define OMPI_FORTRAN_IGNORE_TKR_TYPE @OMPI_FORTRAN_IGNORE_TKR_TYPE@
 
+! f08 TKR syntax (w/o TS 29113)
+#define OMPI_F08_IGNORE_TKR_PREDECL @OMPI_F08_IGNORE_TKR_PREDECL@
+#define OMPI_F08_IGNORE_TKR_TYPE @OMPI_F08_IGNORE_TKR_TYPE@
 
 #define OMPI_FORTRAN_BUILD_SIZEOF @OMPI_FORTRAN_BUILD_SIZEOF@
 ! Integers
diff --git a/ompi/mpi/fortran/mpiext-use-mpi-f08/Makefile.am b/ompi/mpi/fortran/mpiext-use-mpi-f08/Makefile.am
index 5e5312f6e2d..50d8445905b 100644
--- a/ompi/mpi/fortran/mpiext-use-mpi-f08/Makefile.am
+++ b/ompi/mpi/fortran/mpiext-use-mpi-f08/Makefile.am
@@ -73,14 +73,14 @@ CLEANFILES += *.i90
 #
 install-exec-hook:
 	@ for file in `ls *.mod`; do \
-	  echo $(INSTALL) $$file $(DESTDIR)$(libdir); \
-	  $(INSTALL) $$file $(DESTDIR)$(libdir); \
+	  echo $(INSTALL) $$file $(DESTDIR)$(OMPI_FORTRAN_MODULEDIR); \
+	  $(INSTALL) $$file $(DESTDIR)$(OMPI_FORTRAN_MODULEDIR); \
 	done
 
 uninstall-local:
 	@ for file in `ls *.mod`; do \
-	  echo rm -f $(DESTDIR)$(libdir)/$$file; \
-	  rm -f $(DESTDIR)$(libdir)/$$file; \
+	  echo rm -f $(DESTDIR)$(OMPI_FORTRAN_MODULEDIR)/$$file; \
+	  rm -f $(DESTDIR)$(OMPI_FORTRAN_MODULEDIR)/$$file; \
 	done
 
 else
diff --git a/ompi/mpi/fortran/mpiext-use-mpi/Makefile.am b/ompi/mpi/fortran/mpiext-use-mpi/Makefile.am
index 825da31f1a8..25d53171ffd 100644
--- a/ompi/mpi/fortran/mpiext-use-mpi/Makefile.am
+++ b/ompi/mpi/fortran/mpiext-use-mpi/Makefile.am
@@ -71,14 +71,14 @@ CLEANFILES += *.i90
 #
 install-exec-hook:
 	@ for file in `ls *.mod`; do \
-	  echo $(INSTALL) $$file $(DESTDIR)$(libdir); \
-	  $(INSTALL) $$file $(DESTDIR)$(libdir); \
+	  echo $(INSTALL) $$file $(DESTDIR)$(OMPI_FORTRAN_MODULEDIR); \
+	  $(INSTALL) $$file $(DESTDIR)$(OMPI_FORTRAN_MODULEDIR); \
 	done
 
 uninstall-local:
 	@ for file in `ls *.mod`; do \
-	  echo rm -f $(DESTDIR)$(libdir)/$$file; \
-	  rm -f $(DESTDIR)$(libdir)/$$file; \
+	  echo rm -f $(DESTDIR)$(OMPI_FORTRAN_MODULEDIR)/$$file; \
+	  rm -f $(DESTDIR)$(OMPI_FORTRAN_MODULEDIR)/$$file; \
 	done
 
 else
diff --git a/ompi/mpi/fortran/mpif-h/Makefile.am b/ompi/mpi/fortran/mpif-h/Makefile.am
index b8225bf889d..ecbf93a9d43 100644
--- a/ompi/mpi/fortran/mpif-h/Makefile.am
+++ b/ompi/mpi/fortran/mpif-h/Makefile.am
@@ -9,15 +9,18 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2006-2015 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2006-2021 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2011-2013 Inria.  All rights reserved.
 # Copyright (c) 2011-2013 Universite Bordeaux 1
 # Copyright (c) 2013-2018 Los Alamos National Security, LLC. All rights
 #                         reserved.
-# Copyright (c) 2015-2020 Research Organization for Information Science
+# Copyright (c) 2015-2021 Research Organization for Information Science
 #                         and Technology (RIST).  All rights reserved.
 # Copyright (c) 2016      IBM Corporation.  All rights reserved.
 # Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+# Copyright (c) 2021-2022 Triad National Security, LLC. All rights
+#                         reserved.
+#
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -55,7 +58,7 @@ CLEANFILES =
 lib@OMPI_LIBMPI_NAME@_mpifh_la_LIBADD = \
         $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
         $(OMPI_MPIEXT_MPIFH_LIBS) \
-        $(OMPI_TOP_BUILDDIR)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+        $(OMPI_TOP_BUILDDIR)/opal/lib@OPAL_LIB_NAME@.la
 
 lib@OMPI_LIBMPI_NAME@_mpifh_la_LDFLAGS = -version-info $(libmpi_mpifh_so_version)
 
@@ -122,6 +125,7 @@ sizeof_f.f90:
 	    --maxrank=$(OMPI_FORTRAN_MAX_ARRAY_RANK) \
 	    --generate=$(OMPI_FORTRAN_BUILD_SIZEOF) \
 	    --real2=$(OMPI_HAVE_FORTRAN_REAL2) \
+	    --iso_real16=$(OMPI_FORTRAN_HAVE_ISO_FORTRAN_ENV_REAL16) \
 	    --real16=$(OMPI_HAVE_FORTRAN_REAL16) \
 	    --complex4=$(OMPI_HAVE_FORTRAN_COMPLEX4) \
 	    --complex32=$(OMPI_HAVE_FORTRAN_COMPLEX32)
@@ -135,17 +139,25 @@ lib@OMPI_LIBMPI_NAME@_mpifh_la_SOURCES += \
         aint_add_f.c \
         aint_diff_f.c \
         allgather_f.c \
+        allgather_init_f.c \
         allgatherv_f.c \
+        allgatherv_init_f.c \
         alloc_mem_f.c \
         allreduce_f.c \
+        allreduce_init_f.c \
         alltoall_f.c \
+        alltoall_init_f.c \
         alltoallv_f.c \
+        alltoallv_init_f.c \
         alltoallw_f.c \
+        alltoallw_init_f.c \
         attr_delete_f.c \
         attr_get_f.c \
         attr_put_f.c \
         barrier_f.c \
+        barrier_init_f.c \
         bcast_f.c \
+        bcast_init_f.c \
         bsend_f.c \
         bsend_init_f.c \
         buffer_attach_f.c \
@@ -166,6 +178,7 @@ lib@OMPI_LIBMPI_NAME@_mpifh_la_SOURCES += \
         comm_connect_f.c \
         comm_create_errhandler_f.c \
         comm_create_f.c \
+        comm_create_from_group_f.c \
         comm_create_group_f.c \
         comm_create_keyval_f.c \
         comm_delete_attr_f.c \
@@ -173,6 +186,7 @@ lib@OMPI_LIBMPI_NAME@_mpifh_la_SOURCES += \
         comm_dup_f.c \
         comm_dup_with_info_f.c \
         comm_idup_f.c \
+        comm_idup_with_info_f.c \
         comm_free_f.c \
         comm_free_keyval_f.c \
         comm_get_attr_f.c \
@@ -204,6 +218,7 @@ lib@OMPI_LIBMPI_NAME@_mpifh_la_SOURCES += \
         error_class_f.c \
         error_string_f.c \
         exscan_f.c \
+        exscan_init_f.c \
         f_sync_reg_f.c \
         file_call_errhandler_f.c \
         file_close_f.c \
@@ -268,7 +283,9 @@ lib@OMPI_LIBMPI_NAME@_mpifh_la_SOURCES += \
         finalize_f.c \
         free_mem_f.c \
         gather_f.c \
+        gather_init_f.c \
         gatherv_f.c \
+        gatherv_init_f.c \
         get_address_f.c \
         get_count_f.c \
         get_elements_f.c \
@@ -287,6 +304,7 @@ lib@OMPI_LIBMPI_NAME@_mpifh_la_SOURCES += \
         group_compare_f.c \
         group_difference_f.c \
         group_excl_f.c \
+        group_from_session_pset_f.c \
         group_free_f.c \
         group_incl_f.c \
         group_intersection_f.c \
@@ -316,18 +334,21 @@ lib@OMPI_LIBMPI_NAME@_mpifh_la_SOURCES += \
         ineighbor_alltoallv_f.c \
         ineighbor_alltoallw_f.c \
         info_create_f.c \
+        info_create_env_f.c \
         info_delete_f.c \
         info_dup_f.c \
         info_free_f.c \
         info_get_f.c \
         info_get_nkeys_f.c \
         info_get_nthkey_f.c \
+        info_get_string_f.c \
         info_get_valuelen_f.c \
         info_set_f.c \
         init_f.c \
         initialized_f.c \
         init_thread_f.c \
         intercomm_create_f.c \
+        intercomm_create_from_groups_f.c \
         intercomm_merge_f.c \
         iprobe_f.c \
         irecv_f.c \
@@ -336,6 +357,8 @@ lib@OMPI_LIBMPI_NAME@_mpifh_la_SOURCES += \
         ireduce_scatter_block_f.c \
         irsend_f.c \
         isend_f.c \
+        isendrecv_f.c \
+        isendrecv_replace_f.c \
         iscan_f.c \
         iscatter_f.c \
         iscatterv_f.c \
@@ -347,10 +370,15 @@ lib@OMPI_LIBMPI_NAME@_mpifh_la_SOURCES += \
         mprobe_f.c \
         mrecv_f.c \
         neighbor_allgather_f.c \
+        neighbor_allgather_init_f.c \
         neighbor_allgatherv_f.c \
+        neighbor_allgatherv_init_f.c \
         neighbor_alltoall_f.c \
+        neighbor_alltoall_init_f.c \
         neighbor_alltoallv_f.c \
+        neighbor_alltoallv_init_f.c \
         neighbor_alltoallw_f.c \
+        neighbor_alltoallw_init_f.c \
         op_commutative_f.c \
         op_create_f.c \
         open_port_f.c \
@@ -359,28 +387,50 @@ lib@OMPI_LIBMPI_NAME@_mpifh_la_SOURCES += \
         pack_external_size_f.c \
         pack_f.c \
         pack_size_f.c \
+        parrived_f.c \
         pcontrol_f.c \
+        pready_f.c \
+        pready_list_f.c \
+        pready_range_f.c \
+        precv_init_f.c \
         probe_f.c \
+        psend_init_f.c \
         publish_name_f.c \
         query_thread_f.c \
         recv_f.c \
         recv_init_f.c \
         reduce_f.c \
+        reduce_init_f.c \
         reduce_local_f.c \
         reduce_scatter_f.c \
+        reduce_scatter_init_f.c \
         reduce_scatter_block_f.c \
+        reduce_scatter_block_init_f.c \
         register_datarep_f.c \
         request_free_f.c \
         request_get_status_f.c \
         rsend_f.c \
         rsend_init_f.c \
         scan_f.c \
+        scan_init_f.c \
         scatter_f.c \
+        scatter_init_f.c \
         scatterv_f.c \
+        scatterv_init_f.c \
         send_f.c \
         send_init_f.c \
         sendrecv_f.c \
         sendrecv_replace_f.c \
+        session_call_errhandler_f.c \
+        session_create_errhandler_f.c \
+        session_get_errhandler_f.c \
+        session_get_info_f.c \
+        session_get_nth_pset_f.c \
+        session_get_num_psets_f.c \
+        session_get_pset_info_f.c \
+        session_init_f.c \
+        session_finalize_f.c \
+        session_set_errhandler_f.c \
         ssend_f.c \
         ssend_init_f.c \
         startall_f.c \
@@ -486,8 +536,10 @@ lib@OMPI_LIBMPI_NAME@_mpifh_la_SOURCES += \
         win_flush_local_f.c \
         win_flush_local_all_f.c
 
-
-if OMPI_ENABLE_MPI1_COMPAT
+# The following functions were removed from the MPI standard, but are
+# retained for ABI compliance reasons.  They are listed independently
+# of the other MPI functions in case we one day change behavior around
+# ABI compliance.
 lib@OMPI_LIBMPI_NAME@_mpifh_la_SOURCES += \
         address_f.c \
         errhandler_create_f.c \
@@ -500,7 +552,6 @@ lib@OMPI_LIBMPI_NAME@_mpifh_la_SOURCES += \
         type_struct_f.c \
         type_ub_f.c
 endif
-endif
 
 #
 # Conditionally install the header files
diff --git a/ompi/mpi/fortran/mpif-h/allgather_init_f.c b/ompi/mpi/fortran/mpif-h/allgather_init_f.c
new file mode 100644
index 00000000000..784bad0628d
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/allgather_init_f.c
@@ -0,0 +1,99 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_ALLGATHER_INIT = ompi_allgather_init_f
+#pragma weak pmpi_allgather_init = ompi_allgather_init_f
+#pragma weak pmpi_allgather_init_ = ompi_allgather_init_f
+#pragma weak pmpi_allgather_init__ = ompi_allgather_init_f
+
+#pragma weak PMPI_Allgather_init_f = ompi_allgather_init_f
+#pragma weak PMPI_Allgather_init_f08 = ompi_allgather_init_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_ALLGATHER_INIT,
+                            pmpi_allgather_init,
+                            pmpi_allgather_init_,
+                            pmpi_allgather_init__,
+                            pompi_allgather_init_f,
+                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, info, request, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_ALLGATHER_INIT = ompi_allgather_init_f
+#pragma weak mpi_allgather_init = ompi_allgather_init_f
+#pragma weak mpi_allgather_init_ = ompi_allgather_init_f
+#pragma weak mpi_allgather_init__ = ompi_allgather_init_f
+
+#pragma weak MPI_Allgather_init_f = ompi_allgather_init_f
+#pragma weak MPI_Allgather_init_f08 = ompi_allgather_init_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_ALLGATHER_INIT,
+                            mpi_allgather_init,
+                            mpi_allgather_init_,
+                            mpi_allgather_init__,
+                            ompi_allgather_init_f,
+                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, info, request, ierr) )
+#else
+#define ompi_allgather_init_f pompi_allgather_init_f
+#endif
+#endif
+
+
+void ompi_allgather_init_f(char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype,
+                           char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype,
+                           MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
+{
+    int ierr_c;
+    MPI_Comm c_comm;
+    MPI_Request c_req;
+    MPI_Datatype c_sendtype, c_recvtype;
+    MPI_Info c_info;
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_sendtype = PMPI_Type_f2c(*sendtype);
+    c_recvtype = PMPI_Type_f2c(*recvtype);
+    c_info = PMPI_Info_f2c(*info);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    ierr_c = PMPI_Allgather_init(sendbuf,
+                                 OMPI_FINT_2_INT(*sendcount),
+                                 c_sendtype,
+                                 recvbuf,
+                                 OMPI_FINT_2_INT(*recvcount),
+                                 c_recvtype, c_comm, c_info, &c_req);
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(ierr_c);
+
+    if (MPI_SUCCESS == ierr_c) *request = PMPI_Request_c2f(c_req);
+}
diff --git a/ompi/mpi/fortran/mpif-h/allgatherv_init_f.c b/ompi/mpi/fortran/mpif-h/allgatherv_init_f.c
new file mode 100644
index 00000000000..0d98627c440
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/allgatherv_init_f.c
@@ -0,0 +1,118 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_ALLGATHERV_INIT = ompi_allgatherv_init_f
+#pragma weak pmpi_allgatherv_init = ompi_allgatherv_init_f
+#pragma weak pmpi_allgatherv_init_ = ompi_allgatherv_init_f
+#pragma weak pmpi_allgatherv_init__ = ompi_allgatherv_init_f
+
+#pragma weak PMPI_Allgatherv_init_f = ompi_allgatherv_init_f
+#pragma weak PMPI_Allgatherv_init_f08 = ompi_allgatherv_init_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_ALLGATHERV_INIT,
+                            pmpi_allgatherv_init,
+                            pmpi_allgatherv_init_,
+                            pmpi_allgatherv_init__,
+                            pompi_allgatherv_init_f,
+                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *displs, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, comm, info, request, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_ALLGATHERV_INIT = ompi_allgatherv_init_f
+#pragma weak mpi_allgatherv_init = ompi_allgatherv_init_f
+#pragma weak mpi_allgatherv_init_ = ompi_allgatherv_init_f
+#pragma weak mpi_allgatherv_init__ = ompi_allgatherv_init_f
+
+#pragma weak MPI_Allgatherv_init_f = ompi_allgatherv_init_f
+#pragma weak MPI_Allgatherv_init_f08 = ompi_allgatherv_init_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_ALLGATHERV_INIT,
+                            mpi_allgatherv_init,
+                            mpi_allgatherv_init_,
+                            mpi_allgatherv_init__,
+                            ompi_allgatherv_init_f,
+                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *displs, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, comm, info, request, ierr) )
+#else
+#define ompi_allgatherv_init_f pompi_allgatherv_init_f
+#endif
+#endif
+
+
+void ompi_allgatherv_init_f(char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype,
+                            char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *displs,
+                            MPI_Fint *recvtype, MPI_Fint *comm,
+                            MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
+{
+    MPI_Comm c_comm;
+    MPI_Datatype c_sendtype, c_recvtype;
+    MPI_Request c_request;
+    MPI_Info c_info;
+    int size, idx = 0, ierr_c;
+    OMPI_ARRAY_NAME_DECL(recvcounts);
+    OMPI_ARRAY_NAME_DECL(displs);
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_sendtype = PMPI_Type_f2c(*sendtype);
+    c_recvtype = PMPI_Type_f2c(*recvtype);
+    c_info = PMPI_Info_f2c(*info);
+
+    PMPI_Comm_size(c_comm, &size);
+    OMPI_ARRAY_FINT_2_INT(recvcounts, size);
+    OMPI_ARRAY_FINT_2_INT(displs, size);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    ierr_c = PMPI_Allgatherv_init(sendbuf,
+                                  OMPI_FINT_2_INT(*sendcount),
+                                  c_sendtype,
+                                  recvbuf,
+                                  OMPI_ARRAY_NAME_CONVERT(recvcounts),
+                                  OMPI_ARRAY_NAME_CONVERT(displs),
+                                  c_recvtype, c_comm, c_info, &c_request);
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(ierr_c);
+    if (MPI_SUCCESS == ierr_c) {
+        *request = PMPI_Request_c2f(c_request);
+        ompi_coll_base_nbc_request_t* nb_request = (ompi_coll_base_nbc_request_t*)c_request;
+        if (recvcounts != OMPI_ARRAY_NAME_CONVERT(recvcounts)) {
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(recvcounts);
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(displs);
+        }
+        nb_request->data.release_arrays[idx]   = NULL;
+     } else {
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(recvcounts);
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(displs);
+    }
+}
diff --git a/ompi/mpi/fortran/mpif-h/allreduce_init_f.c b/ompi/mpi/fortran/mpif-h/allreduce_init_f.c
new file mode 100644
index 00000000000..48c250d8078
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/allreduce_init_f.c
@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_ALLREDUCE_INIT = ompi_allreduce_init_f
+#pragma weak pmpi_allreduce_init = ompi_allreduce_init_f
+#pragma weak pmpi_allreduce_init_ = ompi_allreduce_init_f
+#pragma weak pmpi_allreduce_init__ = ompi_allreduce_init_f
+
+#pragma weak PMPI_Allreduce_init_f = ompi_allreduce_init_f
+#pragma weak PMPI_Allreduce_init_f08 = ompi_allreduce_init_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_ALLREDUCE_INIT,
+                            pmpi_allreduce_init,
+                            pmpi_allreduce_init_,
+                            pmpi_allreduce_init__,
+                            pompi_allreduce_init_f,
+                            (char *sendbuf, char *recvbuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, recvbuf, count, datatype, op, comm, info, request, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_ALLREDUCE_INIT = ompi_allreduce_init_f
+#pragma weak mpi_allreduce_init = ompi_allreduce_init_f
+#pragma weak mpi_allreduce_init_ = ompi_allreduce_init_f
+#pragma weak mpi_allreduce_init__ = ompi_allreduce_init_f
+
+#pragma weak MPI_Allreduce_init_f = ompi_allreduce_init_f
+#pragma weak MPI_Allreduce_init_f08 = ompi_allreduce_init_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_ALLREDUCE_INIT,
+                            mpi_allreduce_init,
+                            mpi_allreduce_init_,
+                            mpi_allreduce_init__,
+                            ompi_allreduce_init_f,
+                            (char *sendbuf, char *recvbuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, recvbuf, count, datatype, op, comm, info, request, ierr) )
+#else
+#define ompi_allreduce_init_f pompi_allreduce_init_f
+#endif
+#endif
+
+
+void ompi_allreduce_init_f(char *sendbuf, char *recvbuf, MPI_Fint *count,
+                           MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm,
+                           MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
+{
+    int ierr_c;
+    MPI_Comm c_comm;
+    MPI_Datatype c_type;
+    MPI_Info c_info;
+    MPI_Request c_request;
+    MPI_Op c_op;
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_type = PMPI_Type_f2c(*datatype);
+    c_op = PMPI_Op_f2c(*op);
+    c_info = PMPI_Info_f2c(*info);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    ierr_c = PMPI_Allreduce_init(sendbuf, recvbuf,
+                                 OMPI_FINT_2_INT(*count),
+                                 c_type, c_op, c_comm, c_info, &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(ierr_c);
+    if (MPI_SUCCESS == ierr_c) *request = PMPI_Request_c2f(c_request);
+}
diff --git a/ompi/mpi/fortran/mpif-h/alltoall_init_f.c b/ompi/mpi/fortran/mpif-h/alltoall_init_f.c
new file mode 100644
index 00000000000..baa268533e2
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/alltoall_init_f.c
@@ -0,0 +1,98 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_ALLTOALL_INIT = ompi_alltoall_init_f
+#pragma weak pmpi_alltoall_init = ompi_alltoall_init_f
+#pragma weak pmpi_alltoall_init_ = ompi_alltoall_init_f
+#pragma weak pmpi_alltoall_init__ = ompi_alltoall_init_f
+
+#pragma weak PMPI_Alltoall_init_f = ompi_alltoall_init_f
+#pragma weak PMPI_Alltoall_init_f08 = ompi_alltoall_init_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_ALLTOALL_INIT,
+                            pmpi_alltoall_init,
+                            pmpi_alltoall_init_,
+                            pmpi_alltoall_init__,
+                            pompi_alltoall_init_f,
+                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, info, request, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_ALLTOALL_INIT = ompi_alltoall_init_f
+#pragma weak mpi_alltoall_init = ompi_alltoall_init_f
+#pragma weak mpi_alltoall_init_ = ompi_alltoall_init_f
+#pragma weak mpi_alltoall_init__ = ompi_alltoall_init_f
+
+#pragma weak MPI_Alltoall_init_f = ompi_alltoall_init_f
+#pragma weak MPI_Alltoall_init_f08 = ompi_alltoall_init_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_ALLTOALL_INIT,
+                            mpi_alltoall_init,
+                            mpi_alltoall_init_,
+                            mpi_alltoall_init__,
+                            ompi_alltoall_init_f,
+                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, info, request, ierr) )
+#else
+#define ompi_alltoall_init_f pompi_alltoall_init_f
+#endif
+#endif
+
+
+void ompi_alltoall_init_f(char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype,
+                          char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype,
+                          MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
+{
+    int c_ierr;
+    MPI_Comm c_comm;
+    MPI_Request c_req;
+    MPI_Datatype c_sendtype, c_recvtype;
+    MPI_Info c_info;
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_sendtype = PMPI_Type_f2c(*sendtype);
+    c_recvtype = PMPI_Type_f2c(*recvtype);
+    c_info = PMPI_Info_f2c(*info);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = PMPI_Alltoall_init(sendbuf,
+                                OMPI_FINT_2_INT(*sendcount),
+                                c_sendtype,
+                                recvbuf,
+                                OMPI_FINT_2_INT(*recvcount),
+                                c_recvtype, c_comm, c_info, &c_req);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_req);
+}
diff --git a/ompi/mpi/fortran/mpif-h/alltoallv_init_f.c b/ompi/mpi/fortran/mpif-h/alltoallv_init_f.c
new file mode 100644
index 00000000000..1a10b0eadad
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/alltoallv_init_f.c
@@ -0,0 +1,126 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_ALLTOALLV_INIT = ompi_alltoallv_init_f
+#pragma weak pmpi_alltoallv_init = ompi_alltoallv_init_f
+#pragma weak pmpi_alltoallv_init_ = ompi_alltoallv_init_f
+#pragma weak pmpi_alltoallv_init__ = ompi_alltoallv_init_f
+
+#pragma weak PMPI_Alltoallv_init_f = ompi_alltoallv_init_f
+#pragma weak PMPI_Alltoallv_init_f08 = ompi_alltoallv_init_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_ALLTOALLV_INIT,
+                            pmpi_alltoallv_init,
+                            pmpi_alltoallv_init_,
+                            pmpi_alltoallv_init__,
+                            pompi_alltoallv_init_f,
+                            (char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *sdispls, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *rdispls, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, sendcounts, sdispls, sendtype, recvbuf, recvcounts, rdispls, recvtype, comm, info, request, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_ALLTOALLV_INIT = ompi_alltoallv_init_f
+#pragma weak mpi_alltoallv_init = ompi_alltoallv_init_f
+#pragma weak mpi_alltoallv_init_ = ompi_alltoallv_init_f
+#pragma weak mpi_alltoallv_init__ = ompi_alltoallv_init_f
+
+#pragma weak MPI_Alltoallv_init_f = ompi_alltoallv_init_f
+#pragma weak MPI_Alltoallv_init_f08 = ompi_alltoallv_init_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_ALLTOALLV_INIT,
+                            mpi_alltoallv_init,
+                            mpi_alltoallv_init_,
+                            mpi_alltoallv_init__,
+                            ompi_alltoallv_init_f,
+                            (char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *sdispls, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *rdispls, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, sendcounts, sdispls, sendtype, recvbuf, recvcounts, rdispls, recvtype, comm, info, request, ierr) )
+#else
+#define ompi_alltoallv_init_f pompi_alltoallv_init_f
+#endif
+#endif
+
+
+void ompi_alltoallv_init_f(char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *sdispls,
+                           MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts,
+                           MPI_Fint *rdispls, MPI_Fint *recvtype,
+                           MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
+{
+    MPI_Comm c_comm;
+    MPI_Datatype c_sendtype, c_recvtype;
+    MPI_Info c_info;
+    MPI_Request c_request;
+    int size, idx = 0, c_ierr;
+    OMPI_ARRAY_NAME_DECL(sendcounts);
+    OMPI_ARRAY_NAME_DECL(sdispls);
+    OMPI_ARRAY_NAME_DECL(recvcounts);
+    OMPI_ARRAY_NAME_DECL(rdispls);
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_sendtype = PMPI_Type_f2c(*sendtype);
+    c_recvtype = PMPI_Type_f2c(*recvtype);
+    c_info = PMPI_Info_f2c(*info);
+
+    PMPI_Comm_size(c_comm, &size);
+    OMPI_ARRAY_FINT_2_INT(sendcounts, size);
+    OMPI_ARRAY_FINT_2_INT(sdispls, size);
+    OMPI_ARRAY_FINT_2_INT(recvcounts, size);
+    OMPI_ARRAY_FINT_2_INT(rdispls, size);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = PMPI_Alltoallv_init(sendbuf,
+                                 OMPI_ARRAY_NAME_CONVERT(sendcounts),
+                                 OMPI_ARRAY_NAME_CONVERT(sdispls),
+                                 c_sendtype,
+                                 recvbuf,
+                                 OMPI_ARRAY_NAME_CONVERT(recvcounts),
+                                 OMPI_ARRAY_NAME_CONVERT(rdispls),
+                                 c_recvtype, c_comm, c_info, &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_request);
+        ompi_coll_base_nbc_request_t* nb_request = (ompi_coll_base_nbc_request_t*)c_request;
+        if (sendcounts != OMPI_ARRAY_NAME_CONVERT(sendcounts)) {
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(sendcounts);
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(sdispls);
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(recvcounts);
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(rdispls);
+        }
+        nb_request->data.release_arrays[idx]   = NULL;
+    } else {
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(sendcounts);
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(sdispls);
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(recvcounts);
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(rdispls);
+    }
+}
diff --git a/ompi/mpi/fortran/mpif-h/alltoallw_init_f.c b/ompi/mpi/fortran/mpif-h/alltoallw_init_f.c
new file mode 100644
index 00000000000..24b0489ba43
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/alltoallw_init_f.c
@@ -0,0 +1,146 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_ALLTOALLW_INIT = ompi_alltoallw_init_f
+#pragma weak pmpi_alltoallw_init = ompi_alltoallw_init_f
+#pragma weak pmpi_alltoallw_init_ = ompi_alltoallw_init_f
+#pragma weak pmpi_alltoallw_init__ = ompi_alltoallw_init_f
+
+#pragma weak PMPI_Alltoallw_init_f = ompi_alltoallw_init_f
+#pragma weak PMPI_Alltoallw_init_f08 = ompi_alltoallw_init_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_ALLTOALLW_INIT,
+                            pmpi_alltoallw_init,
+                            pmpi_alltoallw_init_,
+                            pmpi_alltoallw_init__,
+                            pompi_alltoallw_init_f,
+                            (char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *sdispls, MPI_Fint *sendtypes, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *rdispls, MPI_Fint *recvtypes, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, sendcounts, sdispls, sendtypes, recvbuf, recvcounts, rdispls, recvtypes, comm, info, request, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_ALLTOALLW_INIT = ompi_alltoallw_init_f
+#pragma weak mpi_alltoallw_init = ompi_alltoallw_init_f
+#pragma weak mpi_alltoallw_init_ = ompi_alltoallw_init_f
+#pragma weak mpi_alltoallw_init__ = ompi_alltoallw_init_f
+
+#pragma weak MPI_Alltoallw_init_f = ompi_alltoallw_init_f
+#pragma weak MPI_Alltoallw_init_f08 = ompi_alltoallw_init_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_ALLTOALLW_INIT,
+                            mpi_alltoallw_init,
+                            mpi_alltoallw_init_,
+                            mpi_alltoallw_init__,
+                            ompi_alltoallw_init_f,
+                            (char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *sdispls, MPI_Fint *sendtypes, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *rdispls, MPI_Fint *recvtypes, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, sendcounts, sdispls, sendtypes, recvbuf, recvcounts, rdispls, recvtypes, comm, info, request, ierr) )
+#else
+#define ompi_alltoallw_init_f pompi_alltoallw_init_f
+#endif
+#endif
+
+
+void ompi_alltoallw_init_f(char *sendbuf, MPI_Fint *sendcounts,
+                           MPI_Fint *sdispls, MPI_Fint *sendtypes,
+                           char *recvbuf, MPI_Fint *recvcounts,
+                           MPI_Fint *rdispls, MPI_Fint *recvtypes,
+                           MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
+{
+    MPI_Comm c_comm;
+    MPI_Datatype *c_sendtypes = NULL, *c_recvtypes;
+    MPI_Info c_info;
+    MPI_Request c_request;
+    int size, idx = 0, c_ierr;
+    OMPI_ARRAY_NAME_DECL(sendcounts);
+    OMPI_ARRAY_NAME_DECL(sdispls);
+    OMPI_ARRAY_NAME_DECL(recvcounts);
+    OMPI_ARRAY_NAME_DECL(rdispls);
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_info = PMPI_Info_f2c(*info);
+    size = OMPI_COMM_IS_INTER(c_comm)?ompi_comm_remote_size(c_comm):ompi_comm_size(c_comm);
+
+    if (!OMPI_IS_FORTRAN_IN_PLACE(sendbuf)) {
+        c_sendtypes = (MPI_Datatype *) malloc(size * sizeof(MPI_Datatype));
+        OMPI_ARRAY_FINT_2_INT(sendcounts, size);
+        OMPI_ARRAY_FINT_2_INT(sdispls, size);
+        for (int i=0; i<size; i++) {
+            c_sendtypes[i] = PMPI_Type_f2c(sendtypes[i]);
+        }
+    }
+
+    c_recvtypes = (MPI_Datatype *) malloc(size * sizeof(MPI_Datatype));
+    OMPI_ARRAY_FINT_2_INT(recvcounts, size);
+    OMPI_ARRAY_FINT_2_INT(rdispls, size);
+    for (int i=0; i<size; i++) {
+        c_recvtypes[i] = PMPI_Type_f2c(recvtypes[i]);
+    }
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = PMPI_Alltoallw_init(sendbuf,
+                                 OMPI_ARRAY_NAME_CONVERT(sendcounts),
+                                 OMPI_ARRAY_NAME_CONVERT(sdispls),
+                                 c_sendtypes,
+                                 recvbuf,
+                                 OMPI_ARRAY_NAME_CONVERT(recvcounts),
+                                 OMPI_ARRAY_NAME_CONVERT(rdispls),
+                                 c_recvtypes, c_comm, c_info, &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_request);
+        ompi_coll_base_nbc_request_t* nb_request = (ompi_coll_base_nbc_request_t*)c_request;
+        nb_request->data.release_arrays[idx++] = c_recvtypes;
+        if (recvcounts != OMPI_ARRAY_NAME_CONVERT(recvcounts)) {
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(recvcounts);
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(rdispls);
+        }
+        if (NULL != c_sendtypes) {
+            nb_request->data.release_arrays[idx++] = c_sendtypes;
+            if (sendcounts != OMPI_ARRAY_NAME_CONVERT(sendcounts)) {
+                nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(sendcounts);
+                nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(sdispls);
+            }
+        }
+        nb_request->data.release_arrays[idx]   = NULL;
+    } else {
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(sendcounts);
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(sdispls);
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(recvcounts);
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(rdispls);
+        if (NULL != c_sendtypes) {
+            free(c_sendtypes);
+        }
+        free(c_recvtypes);
+    }
+}
diff --git a/ompi/mpi/fortran/mpif-h/barrier_init_f.c b/ompi/mpi/fortran/mpif-h/barrier_init_f.c
new file mode 100644
index 00000000000..f82ac1df1c8
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/barrier_init_f.c
@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_BARRIER_INIT = ompi_barrier_init_f
+#pragma weak pmpi_barrier_init = ompi_barrier_init_f
+#pragma weak pmpi_barrier_init_ = ompi_barrier_init_f
+#pragma weak pmpi_barrier_init__ = ompi_barrier_init_f
+
+#pragma weak PMPI_Barrier_init_f = ompi_barrier_init_f
+#pragma weak PMPI_Barrier_init_f08 = ompi_barrier_init_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_BARRIER_INIT,
+                            pmpi_barrier_init,
+                            pmpi_barrier_init_,
+                            pmpi_barrier_init__,
+                            pompi_barrier_init_f,
+                            (MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (comm, info, request, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_BARRIER_INIT = ompi_barrier_init_f
+#pragma weak mpi_barrier_init = ompi_barrier_init_f
+#pragma weak mpi_barrier_init_ = ompi_barrier_init_f
+#pragma weak mpi_barrier_init__ = ompi_barrier_init_f
+
+#pragma weak MPI_Barrier_init_f = ompi_barrier_init_f
+#pragma weak MPI_Barrier_init_f08 = ompi_barrier_init_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_BARRIER_INIT,
+                            mpi_barrier_init,
+                            mpi_barrier_init_,
+                            mpi_barrier_init__,
+                            ompi_barrier_init_f,
+                            (MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (comm, info, request, ierr) )
+#else
+#define ompi_barrier_init_f pompi_barrier_init_f
+#endif
+#endif
+
+
+void ompi_barrier_init_f(MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
+{
+    int ierr_c;
+    MPI_Comm c_comm;
+    MPI_Info c_info;
+    MPI_Request c_req;
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_info = PMPI_Info_f2c(*info);
+
+    ierr_c = PMPI_Barrier_init(c_comm, c_info, &c_req);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(ierr_c);
+
+    if (MPI_SUCCESS == ierr_c) *request = PMPI_Request_c2f(c_req);
+}
diff --git a/ompi/mpi/fortran/mpif-h/bcast_init_f.c b/ompi/mpi/fortran/mpif-h/bcast_init_f.c
new file mode 100644
index 00000000000..8cd12934870
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/bcast_init_f.c
@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_BCAST_INIT = ompi_bcast_init_f
+#pragma weak pmpi_bcast_init = ompi_bcast_init_f
+#pragma weak pmpi_bcast_init_ = ompi_bcast_init_f
+#pragma weak pmpi_bcast_init__ = ompi_bcast_init_f
+
+#pragma weak PMPI_Bcast_init_f = ompi_bcast_init_f
+#pragma weak PMPI_Bcast_init_f08 = ompi_bcast_init_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_BCAST_INIT,
+                            pmpi_bcast_init,
+                            pmpi_bcast_init_,
+                            pmpi_bcast_init__,
+                            pompi_bcast_init_f,
+                            (char *buffer, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (buffer, count, datatype, root, comm, info, request, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_BCAST_INIT = ompi_bcast_init_f
+#pragma weak mpi_bcast_init = ompi_bcast_init_f
+#pragma weak mpi_bcast_init_ = ompi_bcast_init_f
+#pragma weak mpi_bcast_init__ = ompi_bcast_init_f
+
+#pragma weak MPI_Bcast_init_f = ompi_bcast_init_f
+#pragma weak MPI_Bcast_init_f08 = ompi_bcast_init_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_BCAST_INIT,
+                            mpi_bcast_init,
+                            mpi_bcast_init_,
+                            mpi_bcast_init__,
+                            ompi_bcast_init_f,
+                            (char *buffer, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (buffer, count, datatype, root, comm, info, request, ierr) )
+#else
+#define ompi_bcast_init_f pompi_bcast_init_f
+#endif
+#endif
+
+
+void ompi_bcast_init_f(char *buffer, MPI_Fint *count, MPI_Fint *datatype,
+                       MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request,
+                       MPI_Fint *ierr)
+{
+    int c_ierr;
+    MPI_Comm c_comm;
+    MPI_Info c_info;
+    MPI_Request c_req;
+    MPI_Datatype c_type;
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_type = PMPI_Type_f2c(*datatype);
+    c_info = PMPI_Info_f2c(*info);
+
+    c_ierr = PMPI_Bcast_init(OMPI_F2C_BOTTOM(buffer),
+                             OMPI_FINT_2_INT(*count),
+                             c_type,
+                             OMPI_FINT_2_INT(*root),
+                             c_comm,
+                             c_info,
+                             &c_req);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_req);
+}
diff --git a/ompi/mpi/fortran/mpif-h/comm_create_errhandler_f.c b/ompi/mpi/fortran/mpif-h/comm_create_errhandler_f.c
index 480e832242f..9881590dd11 100644
--- a/ompi/mpi/fortran/mpif-h/comm_create_errhandler_f.c
+++ b/ompi/mpi/fortran/mpif-h/comm_create_errhandler_f.c
@@ -74,8 +74,9 @@ void ompi_comm_create_errhandler_f(ompi_errhandler_fortran_handler_fn_t *functio
 				  MPI_Fint *errhandler, MPI_Fint *ierr)
 {
     int c_ierr;
-    MPI_Errhandler c_errhandler =
-        ompi_errhandler_create(OMPI_ERRHANDLER_TYPE_COMM,
+    MPI_Errhandler c_errhandler;
+
+    c_errhandler = ompi_errhandler_create(OMPI_ERRHANDLER_TYPE_COMM,
                                (ompi_errhandler_generic_handler_fn_t*) function,
                                OMPI_ERRHANDLER_LANG_FORTRAN);
     if (MPI_ERRHANDLER_NULL != c_errhandler) {
diff --git a/ompi/mpi/fortran/mpif-h/comm_create_from_group_f.c b/ompi/mpi/fortran/mpif-h/comm_create_from_group_f.c
new file mode 100644
index 00000000000..5afba43fbf6
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/comm_create_from_group_f.c
@@ -0,0 +1,112 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2013 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2019-2021 Triad National Security, LLC.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/fortran_base_strings.h"
+#include "ompi/constants.h"
+#include "ompi/instance/instance.h"
+#include "ompi/group/group.h"
+
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_COMM_CREATE_FROM_GROUP = ompi_comm_create_from_group_f
+#pragma weak pmpi_comm_create_from_group = ompi_comm_create_from_group_f
+#pragma weak pmpi_comm_create_from_group_ = ompi_comm_create_from_group_f
+#pragma weak pmpi_comm_create_from_group__ = ompi_comm_create_from_group_f
+
+#pragma weak PMPI_Comm_create_from_group_f = ompi_comm_create_from_group_f
+#pragma weak PMPI_Comm_create_from_group_f08 = ompi_comm_create_from_group_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_COMM_CREATE_FROM_GROUP,
+                            pmpi_comm_create_from_group,
+                            pmpi_comm_create_from_group_,
+                            pmpi_comm_create_from_group__,
+                            pompi_comm_create_from_group_f,
+                            (MPI_Fint *group, char *stringtag, MPI_Fint *info, MPI_Fint *errhandler, MPI_Fint *newcomm, MPI_Fint *ierr, int name_len),
+                            (group, stringtag, info, errhandler, newcomm, ierr, name_len) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_COMM_CREATE_FROM_GROUP = ompi_comm_create_from_group_f
+#pragma weak mpi_comm_create_from_group = ompi_comm_create_from_group_f
+#pragma weak mpi_comm_create_from_group_ = ompi_comm_create_from_group_f
+#pragma weak mpi_comm_create_from_group__ = ompi_comm_create_from_group_f
+
+#pragma weak MPI_Comm_create_from_group_f = ompi_comm_create_from_group_f
+#pragma weak MPI_Comm_create_from_group_f08 = ompi_comm_create_from_group_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_COMM_CREATE_FROM_GROUP,
+                            mpi_comm_create_from_group,
+                            mpi_comm_create_from_group_,
+                            mpi_comm_create_from_group__,
+                            ompi_comm_create_from_group_f,
+                            (MPI_Fint *group, char *stringtag, MPI_Fint *info, MPI_Fint *errhandler, MPI_Fint *newcomm, MPI_Fint *ierr, int name_len),
+                            (group, stringtag, info, errhandler, newcomm, ierr, name_len) )
+#else
+#define ompi_comm_create_from_group_f pompi_comm_create_from_group_f
+#endif
+#endif
+
+void ompi_comm_create_from_group_f(MPI_Fint *group, char *stringtag, MPI_Fint *info, MPI_Fint *errhandler, MPI_Fint *newcomm, MPI_Fint *ierr, int name_len)
+{
+    int c_ierr, ret;
+    MPI_Group c_group;
+    char *c_tag;
+    MPI_Comm c_comm;
+    MPI_Info c_info;
+    MPI_Errhandler c_err;
+
+    c_group = PMPI_Group_f2c(*group);
+    c_info = PMPI_Info_f2c(*info);
+    c_err = PMPI_Errhandler_f2c(*errhandler);
+
+    /* Convert the fortran string */
+
+    /* Convert the fortran string */
+    if (OMPI_SUCCESS != (ret = ompi_fortran_string_f2c(stringtag, name_len,
+                                                       &c_tag))) {
+        c_ierr = OMPI_ERRHANDLER_INVOKE(ompi_group_get_instance(c_group), ret, "MPI_COMM_CREATE_FROM_GROUP");
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        return;
+    }
+
+    c_ierr = PMPI_Comm_create_from_group(c_group, c_tag, c_info, c_err, &c_comm);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *newcomm = PMPI_Comm_c2f (c_comm);
+    }
+
+    /* Free the C tag */
+
+    free(c_tag);
+}
+
diff --git a/ompi/mpi/fortran/mpif-h/comm_create_keyval_f.c b/ompi/mpi/fortran/mpif-h/comm_create_keyval_f.c
index 51bb77363ff..fb069b7a47e 100644
--- a/ompi/mpi/fortran/mpif-h/comm_create_keyval_f.c
+++ b/ompi/mpi/fortran/mpif-h/comm_create_keyval_f.c
@@ -34,13 +34,13 @@
 #pragma weak PMPI_Comm_create_keyval_f = ompi_comm_create_keyval_f
 #pragma weak PMPI_Comm_create_keyval_f08 = ompi_comm_create_keyval_f
 #else
-OMPI_GENERATE_F77_BINDINGS (PMPI_COMM_CREATE_KEYVAL,
-                           pmpi_comm_create_keyval,
-                           pmpi_comm_create_keyval_,
-                           pmpi_comm_create_keyval__,
+OMPI_GENERATE_F77_BINDINGS(PMPI_COMM_CREATE_KEYVAL, pmpi_comm_create_keyval,
+                           pmpi_comm_create_keyval_, pmpi_comm_create_keyval__,
                            pompi_comm_create_keyval_f,
-                           (ompi_aint_copy_attr_function* comm_copy_attr_fn, ompi_aint_delete_attr_function* comm_delete_attr_fn, MPI_Fint *comm_keyval, MPI_Aint *extra_state, MPI_Fint *ierr),
-                           (comm_copy_attr_fn, comm_delete_attr_fn, comm_keyval, extra_state, ierr) )
+                           (ompi_aint_copy_attr_function comm_copy_attr_fn,
+                            ompi_aint_delete_attr_function comm_delete_attr_fn,
+                            MPI_Fint *comm_keyval, MPI_Aint *extra_state, MPI_Fint *ierr),
+                           (comm_copy_attr_fn, comm_delete_attr_fn, comm_keyval, extra_state, ierr))
 #endif
 #endif
 
@@ -54,13 +54,12 @@ OMPI_GENERATE_F77_BINDINGS (PMPI_COMM_CREATE_KEYVAL,
 #pragma weak MPI_Comm_create_keyval_f08 = ompi_comm_create_keyval_f
 #else
 #if ! OMPI_BUILD_MPI_PROFILING
-OMPI_GENERATE_F77_BINDINGS (MPI_COMM_CREATE_KEYVAL,
-                           mpi_comm_create_keyval,
-                           mpi_comm_create_keyval_,
-                           mpi_comm_create_keyval__,
-                           ompi_comm_create_keyval_f,
-                           (ompi_aint_copy_attr_function* comm_copy_attr_fn, ompi_aint_delete_attr_function* comm_delete_attr_fn, MPI_Fint *comm_keyval, MPI_Aint *extra_state, MPI_Fint *ierr),
-                           (comm_copy_attr_fn, comm_delete_attr_fn, comm_keyval, extra_state, ierr) )
+OMPI_GENERATE_F77_BINDINGS(MPI_COMM_CREATE_KEYVAL, mpi_comm_create_keyval, mpi_comm_create_keyval_,
+                           mpi_comm_create_keyval__, ompi_comm_create_keyval_f,
+                           (ompi_aint_copy_attr_function comm_copy_attr_fn,
+                            ompi_aint_delete_attr_function comm_delete_attr_fn,
+                            MPI_Fint *comm_keyval, MPI_Aint *extra_state, MPI_Fint *ierr),
+                           (comm_copy_attr_fn, comm_delete_attr_fn, comm_keyval, extra_state, ierr))
 #else
 #define ompi_comm_create_keyval_f pompi_comm_create_keyval_f
 #endif
@@ -68,11 +67,9 @@ OMPI_GENERATE_F77_BINDINGS (MPI_COMM_CREATE_KEYVAL,
 
 static const char FUNC_NAME[] = "MPI_Comm_create_keyval_f";
 
-
-void ompi_comm_create_keyval_f(ompi_aint_copy_attr_function* comm_copy_attr_fn,
-                              ompi_aint_delete_attr_function* comm_delete_attr_fn,
-                              MPI_Fint *comm_keyval,
-                              MPI_Aint *extra_state, MPI_Fint *ierr)
+void ompi_comm_create_keyval_f(ompi_aint_copy_attr_function comm_copy_attr_fn,
+                               ompi_aint_delete_attr_function comm_delete_attr_fn,
+                               MPI_Fint *comm_keyval, MPI_Aint *extra_state, MPI_Fint *ierr)
 {
     int ret, c_ierr;
     OMPI_SINGLE_NAME_DECL(comm_keyval);
diff --git a/ompi/mpi/fortran/mpif-h/comm_idup_with_info_f.c b/ompi/mpi/fortran/mpif-h/comm_idup_with_info_f.c
new file mode 100644
index 00000000000..c021e502ffe
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/comm_idup_with_info_f.c
@@ -0,0 +1,92 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2013 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2021      Triad National Security, LLC. All rights
+ *                         reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_COMM_IDUP_WITH_INFO = ompi_comm_idup_with_info_f
+#pragma weak pmpi_comm_idup_with_info = ompi_comm_idup_with_info_f
+#pragma weak pmpi_comm_idup_with_info_ = ompi_comm_idup_with_info_f
+#pragma weak pmpi_comm_idup_with_info__ = ompi_comm_idup_with_info_f
+
+#pragma weak PMPI_Comm_idup_with_info_f = ompi_comm_idup_with_info_f
+#pragma weak PMPI_Comm_idup_with_info_f08 = ompi_comm_idup_with_info_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_COMM_IDUP_WITH_INFO,
+                            pmpi_comm_idup_with_info,
+                            pmpi_comm_idup_with_info_,
+                            pmpi_comm_idup_with_info__,
+                            pompi_comm_idup_with_info_f,
+                            (MPI_Fint *comm, MPI_Fint *info, MPI_Fint *newcomm, MPI_Fint *request, MPI_Fint *ierr),
+                            (comm, info, newcomm, request, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_COMM_IDUP_WITH_INFO = ompi_comm_idup_with_info_f
+#pragma weak mpi_comm_idup_with_info = ompi_comm_idup_with_info_f
+#pragma weak mpi_comm_idup_with_info_ = ompi_comm_idup_with_info_f
+#pragma weak mpi_comm_idup_with_info__ = ompi_comm_idup_with_info_f
+
+#pragma weak MPI_Comm_idup_with_info_f = ompi_comm_idup_with_info_f
+#pragma weak MPI_Comm_idup_with_info_f08 = ompi_comm_idup_with_info_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_COMM_IDUP_WITH_INFO,
+                            mpi_comm_idup_with_info,
+                            mpi_comm_idup_with_info_,
+                            mpi_comm_idup_with_info__,
+                            ompi_comm_idup_with_info_f,
+                            (MPI_Fint *comm, MPI_Fint *info, MPI_Fint *newcomm, MPI_Fint *request, MPI_Fint *ierr),
+                            (comm, info, newcomm, request, ierr) )
+#else
+#define ompi_comm_idup_with_info_f pompi_comm_idup_with_info_f
+#endif
+#endif
+
+
+void ompi_comm_idup_with_info_f(MPI_Fint *comm, MPI_Fint *info, MPI_Fint *newcomm, MPI_Fint *request, MPI_Fint *ierr)
+{
+    int c_ierr;
+    MPI_Comm c_newcomm;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Info c_info;
+    MPI_Request c_req;
+
+    c_info = PMPI_Info_f2c(*info);
+
+    c_ierr = PMPI_Comm_idup_with_info(c_comm, c_info, &c_newcomm, &c_req);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *newcomm = PMPI_Comm_c2f(c_newcomm);
+        *request = PMPI_Request_c2f(c_req);
+    }
+}
diff --git a/ompi/mpi/fortran/mpif-h/comm_spawn_multiple_f.c b/ompi/mpi/fortran/mpif-h/comm_spawn_multiple_f.c
index 93eda2f341a..fd676539ed3 100644
--- a/ompi/mpi/fortran/mpif-h/comm_spawn_multiple_f.c
+++ b/ompi/mpi/fortran/mpif-h/comm_spawn_multiple_f.c
@@ -14,6 +14,7 @@
  *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -86,7 +87,7 @@ void ompi_comm_spawn_multiple_f(MPI_Fint *count, char *array_commands,
     int *c_errs;
     char **c_array_commands;
     char ***c_array_argv;
-    int maxprocs;
+    int __opal_attribute_unused__ maxprocs;
     OMPI_ARRAY_NAME_DECL(array_maxprocs);
     OMPI_ARRAY_NAME_DECL(array_errcds);
 
diff --git a/ompi/mpi/fortran/mpif-h/dist_graph_create_f.c b/ompi/mpi/fortran/mpif-h/dist_graph_create_f.c
index 3380be8a537..d702bdaecf2 100644
--- a/ompi/mpi/fortran/mpif-h/dist_graph_create_f.c
+++ b/ompi/mpi/fortran/mpif-h/dist_graph_create_f.c
@@ -9,6 +9,7 @@
  *                         reserved.
  * Copyright (c) 2015-2019 Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -72,7 +73,8 @@ void ompi_dist_graph_create_f(MPI_Fint *comm_old, MPI_Fint *n, MPI_Fint *sources
                               MPI_Fint *ierr)
 {
     MPI_Comm c_comm_old, c_comm_graph;
-    int count = 0, i;
+    int __opal_attribute_unused__ count = 0;
+    int i;
     MPI_Info c_info;
     int *c_weights;
     int c_ierr;
diff --git a/ompi/mpi/fortran/mpif-h/exscan_init_f.c b/ompi/mpi/fortran/mpif-h/exscan_init_f.c
new file mode 100644
index 00000000000..142944aa5c5
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/exscan_init_f.c
@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_EXSCAN_INIT = ompi_exscan_init_f
+#pragma weak pmpi_exscan_init = ompi_exscan_init_f
+#pragma weak pmpi_exscan_init_ = ompi_exscan_init_f
+#pragma weak pmpi_exscan_init__ = ompi_exscan_init_f
+
+#pragma weak PMPI_Exscan_init_f = ompi_exscan_init_f
+#pragma weak PMPI_Exscan_init_f08 = ompi_exscan_init_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_EXSCAN_INIT,
+                            pmpi_exscan_init,
+                            pmpi_exscan_init_,
+                            pmpi_exscan_init__,
+                            pompi_exscan_init_f,
+                            (char *sendbuf, char *recvbuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, recvbuf, count, datatype, op, comm, info, request, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_EXSCAN_INIT = ompi_exscan_init_f
+#pragma weak mpi_exscan_init = ompi_exscan_init_f
+#pragma weak mpi_exscan_init_ = ompi_exscan_init_f
+#pragma weak mpi_exscan_init__ = ompi_exscan_init_f
+
+#pragma weak MPI_Exscan_init_f = ompi_exscan_init_f
+#pragma weak MPI_Exscan_init_f08 = ompi_exscan_init_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_EXSCAN_INIT,
+                            mpi_exscan_init,
+                            mpi_exscan_init_,
+                            mpi_exscan_init__,
+                            ompi_exscan_init_f,
+                            (char *sendbuf, char *recvbuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, recvbuf, count, datatype, op, comm, info, request, ierr) )
+#else
+#define ompi_exscan_init_f pompi_exscan_init_f
+#endif
+#endif
+
+
+void ompi_exscan_init_f(char *sendbuf, char *recvbuf, MPI_Fint *count,
+                        MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm,
+                        MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
+{
+    int c_ierr;
+    MPI_Comm c_comm;
+    MPI_Datatype c_type;
+    MPI_Info c_info;
+    MPI_Request c_request;
+    MPI_Op c_op;
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_type = PMPI_Type_f2c(*datatype);
+    c_op = PMPI_Op_f2c(*op);
+    c_info = PMPI_Info_f2c(*info);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM (sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM (recvbuf);
+
+    c_ierr = PMPI_Exscan_init(sendbuf, recvbuf,
+                              OMPI_FINT_2_INT(*count),
+                              c_type, c_op, c_comm, c_info, &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
+}
diff --git a/ompi/mpi/fortran/mpif-h/file_create_errhandler_f.c b/ompi/mpi/fortran/mpif-h/file_create_errhandler_f.c
index 29ac3ffe365..a20af467621 100644
--- a/ompi/mpi/fortran/mpif-h/file_create_errhandler_f.c
+++ b/ompi/mpi/fortran/mpif-h/file_create_errhandler_f.c
@@ -74,8 +74,8 @@ void ompi_file_create_errhandler_f(ompi_errhandler_fortran_handler_fn_t* functio
 				  MPI_Fint *errhandler, MPI_Fint *ierr)
 {
     int c_ierr;
-    MPI_Errhandler c_errhandler =
-        ompi_errhandler_create(OMPI_ERRHANDLER_TYPE_FILE,
+    MPI_Errhandler c_errhandler;
+    c_errhandler = ompi_errhandler_create(OMPI_ERRHANDLER_TYPE_FILE,
                                (ompi_errhandler_generic_handler_fn_t*) function,
                                OMPI_ERRHANDLER_LANG_FORTRAN);
     if (MPI_ERRHANDLER_NULL != c_errhandler) {
diff --git a/ompi/mpi/fortran/mpif-h/gather_init_f.c b/ompi/mpi/fortran/mpif-h/gather_init_f.c
new file mode 100644
index 00000000000..19e7197cc87
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/gather_init_f.c
@@ -0,0 +1,98 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_GATHER_INIT = ompi_gather_init_f
+#pragma weak pmpi_gather_init = ompi_gather_init_f
+#pragma weak pmpi_gather_init_ = ompi_gather_init_f
+#pragma weak pmpi_gather_init__ = ompi_gather_init_f
+
+#pragma weak PMPI_Gather_init_f = ompi_gather_init_f
+#pragma weak PMPI_Gather_init_f08 = ompi_gather_init_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_GATHER_INIT,
+                            pmpi_gather_init,
+                            pmpi_gather_init_,
+                            pmpi_gather_init__,
+                            pompi_gather_init_f,
+                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, info, request, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_GATHER_INIT = ompi_gather_init_f
+#pragma weak mpi_gather_init = ompi_gather_init_f
+#pragma weak mpi_gather_init_ = ompi_gather_init_f
+#pragma weak mpi_gather_init__ = ompi_gather_init_f
+
+#pragma weak MPI_Gather_init_f = ompi_gather_init_f
+#pragma weak MPI_Gather_init_f08 = ompi_gather_init_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_GATHER_INIT,
+                            mpi_gather_init,
+                            mpi_gather_init_,
+                            mpi_gather_init__,
+                            ompi_gather_init_f,
+                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, info, request, ierr) )
+#else
+#define ompi_gather_init_f pompi_gather_init_f
+#endif
+#endif
+
+
+void ompi_gather_init_f(char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype,
+                        char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype,
+                        MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request,
+                        MPI_Fint *ierr)
+{
+    int c_ierr;
+    MPI_Comm c_comm;
+    MPI_Datatype c_sendtype, c_recvtype;
+    MPI_Info c_info;
+    MPI_Request c_request;
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_sendtype = PMPI_Type_f2c(*sendtype);
+    c_recvtype = PMPI_Type_f2c(*recvtype);
+    c_info = PMPI_Info_f2c(*info);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = PMPI_Gather_init(sendbuf, OMPI_FINT_2_INT(*sendcount),
+                              c_sendtype, recvbuf,
+                              OMPI_FINT_2_INT(*recvcount),
+                              c_recvtype,
+                              OMPI_FINT_2_INT(*root),
+                              c_comm, c_info, &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
+}
diff --git a/ompi/mpi/fortran/mpif-h/gatherv_init_f.c b/ompi/mpi/fortran/mpif-h/gatherv_init_f.c
new file mode 100644
index 00000000000..a87a5c9ddcc
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/gatherv_init_f.c
@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_GATHERV_INIT = ompi_gatherv_init_f
+#pragma weak pmpi_gatherv_init = ompi_gatherv_init_f
+#pragma weak pmpi_gatherv_init_ = ompi_gatherv_init_f
+#pragma weak pmpi_gatherv_init__ = ompi_gatherv_init_f
+
+#pragma weak PMPI_Gatherv_init_f = ompi_gatherv_init_f
+#pragma weak PMPI_Gatherv_init_f08 = ompi_gatherv_init_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_GATHERV_INIT,
+                            pmpi_gatherv_init,
+                            pmpi_gatherv_init_,
+                            pmpi_gatherv_init__,
+                            pompi_gatherv_init_f,
+                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *displs, MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, root, comm, info, request, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_GATHERV_INIT = ompi_gatherv_init_f
+#pragma weak mpi_gatherv_init = ompi_gatherv_init_f
+#pragma weak mpi_gatherv_init_ = ompi_gatherv_init_f
+#pragma weak mpi_gatherv_init__ = ompi_gatherv_init_f
+
+#pragma weak MPI_Gatherv_init_f = ompi_gatherv_init_f
+#pragma weak MPI_Gatherv_init_f08 = ompi_gatherv_init_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_GATHERV_INIT,
+                            mpi_gatherv_init,
+                            mpi_gatherv_init_,
+                            mpi_gatherv_init__,
+                            ompi_gatherv_init_f,
+                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *displs, MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request,MPI_Fint *ierr),
+                            (sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, root, comm, info, request, ierr) )
+#else
+#define ompi_gatherv_init_f pompi_gatherv_init_f
+#endif
+#endif
+
+
+void ompi_gatherv_init_f(char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype,
+                         char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *displs,
+                         MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm,
+                         MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
+{
+    MPI_Comm c_comm;
+    MPI_Datatype c_sendtype, c_recvtype;
+    MPI_Info c_info;
+    MPI_Request c_request;
+    int size, idx = 0, c_ierr;
+    OMPI_ARRAY_NAME_DECL(recvcounts);
+    OMPI_ARRAY_NAME_DECL(displs);
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_sendtype = PMPI_Type_f2c(*sendtype);
+    c_recvtype = PMPI_Type_f2c(*recvtype);
+    c_info = PMPI_Info_f2c(*info);
+
+    PMPI_Comm_size(c_comm, &size);
+    OMPI_ARRAY_FINT_2_INT(recvcounts, size);
+    OMPI_ARRAY_FINT_2_INT(displs, size);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = PMPI_Gatherv_init(sendbuf, OMPI_FINT_2_INT(*sendcount),
+                               c_sendtype, recvbuf,
+                               OMPI_ARRAY_NAME_CONVERT(recvcounts),
+                               OMPI_ARRAY_NAME_CONVERT(displs),
+                               c_recvtype,
+                               OMPI_FINT_2_INT(*root),
+                                c_comm, c_info, &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_request);
+        ompi_coll_base_nbc_request_t* nb_request = (ompi_coll_base_nbc_request_t*)c_request;
+        if (recvcounts != OMPI_ARRAY_NAME_CONVERT(recvcounts)) {
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(recvcounts);
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(displs);
+        }
+        nb_request->data.release_arrays[idx]   = NULL;
+     } else {
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(recvcounts);
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(displs);
+     }
+}
diff --git a/ompi/mpi/fortran/mpif-h/group_from_session_pset_f.c b/ompi/mpi/fortran/mpif-h/group_from_session_pset_f.c
new file mode 100644
index 00000000000..061c2f3a42d
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/group_from_session_pset_f.c
@@ -0,0 +1,108 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2013 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2019      Triad National Security, LLC.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/fortran_base_strings.h"
+#include "ompi/constants.h"
+#include "ompi/instance/instance.h"
+
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_GROUP_FROM_SESSION_PSET = ompi_group_from_session_pset_f
+#pragma weak pmpi_group_from_session_pset = ompi_group_from_session_pset_f
+#pragma weak pmpi_group_from_session_pset_ = ompi_group_from_session_pset_f
+#pragma weak pmpi_group_from_session_pset__ = ompi_group_from_session_pset_f
+
+#pragma weak PMPI_Group_from_session_pset_f = ompi_group_from_session_pset_f
+#pragma weak PMPI_Group_from_session_pset_f08 = ompi_group_from_session_pset_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_GROUP_FROM_SESSION_PSET,
+                            pmpi_group_from_session_pset,
+                            pmpi_group_from_session_pset_,
+                            pmpi_group_from_session_pset__,
+                            pompi_group_from_session_pset_f,
+                            (MPI_Fint *session, char *pset_name, MPI_Fint *newgroup, MPI_Fint *ierr, int name_len),
+                            (session, pset_name, newgroup, ierr, name_len) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_GROUP_FROM_SESSION_PSET = ompi_group_from_session_pset_f
+#pragma weak mpi_group_from_session_pset = ompi_group_from_session_pset_f
+#pragma weak mpi_group_from_session_pset_ = ompi_group_from_session_pset_f
+#pragma weak mpi_group_from_session_pset__ = ompi_group_from_session_pset_f
+
+#pragma weak MPI_Group_from_session_pset_f = ompi_group_from_session_pset_f
+#pragma weak MPI_Group_from_session_pset_f08 = ompi_group_from_session_pset_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_GROUP_FROM_SESSION_PSET,
+                            mpi_group_from_session_pset,
+                            mpi_group_from_session_pset_,
+                            mpi_group_from_session_pset__,
+                            ompi_group_from_session_pset_f,
+                            (MPI_Fint *session, char *pset_name, MPI_Fint *newgroup, MPI_Fint *ierr, int name_len),
+                            (session, pset_name, newgroup, ierr, name_len) )
+#else
+#define ompi_group_from_session_pset_f pompi_group_from_session_pset_f
+#endif
+#endif
+
+void ompi_group_from_session_pset_f(MPI_Fint *session,char *pset_name, MPI_Fint *newgroup, MPI_Fint *ierr, int name_len)
+{
+    int c_ierr, ret;
+    MPI_Session c_session;
+    char *c_name;
+    MPI_Group c_newgroup;
+
+    c_session = PMPI_Session_f2c(*session);
+
+    /* Convert the fortran string */
+
+    if (OMPI_SUCCESS != (ret = ompi_fortran_string_f2c(pset_name, name_len,
+                                                       &c_name))) {
+        c_ierr = OMPI_ERRHANDLER_INVOKE((ompi_instance_t *)c_session, ret,
+                                        "MPI_GROUP_FROM_SESSION_PSET");
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        return;
+    }
+
+    c_ierr = PMPI_Group_from_session_pset(c_session, c_name, &c_newgroup);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *newgroup = PMPI_Group_c2f (c_newgroup);
+    }
+
+    /* Free the C name */
+
+    free(c_name);
+}
+
+
diff --git a/ompi/mpi/fortran/mpif-h/iallgatherv_f.c b/ompi/mpi/fortran/mpif-h/iallgatherv_f.c
index 19671fea103..37926247905 100644
--- a/ompi/mpi/fortran/mpif-h/iallgatherv_f.c
+++ b/ompi/mpi/fortran/mpif-h/iallgatherv_f.c
@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -23,6 +23,7 @@
 
 #include "ompi/mpi/fortran/mpif-h/bindings.h"
 #include "ompi/mpi/fortran/base/constants.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -75,7 +76,7 @@ void ompi_iallgatherv_f(char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype,
     MPI_Comm c_comm;
     MPI_Datatype c_sendtype, c_recvtype;
     MPI_Request c_request;
-    int size, ierr_c;
+    int size, idx = 0, ierr_c;
     OMPI_ARRAY_NAME_DECL(recvcounts);
     OMPI_ARRAY_NAME_DECL(displs);
 
@@ -102,6 +103,15 @@ void ompi_iallgatherv_f(char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype,
     if (NULL != ierr) *ierr = OMPI_INT_2_FINT(ierr_c);
     if (MPI_SUCCESS == ierr_c) *request = PMPI_Request_c2f(c_request);
 
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(recvcounts);
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(displs);
+    if ( REQUEST_COMPLETE(c_request)) {
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(recvcounts);
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(displs);
+    } else {
+        ompi_coll_base_nbc_request_t* nb_request = (ompi_coll_base_nbc_request_t*)c_request;
+        if (recvcounts != OMPI_ARRAY_NAME_CONVERT(recvcounts)) {
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(recvcounts);
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(displs);
+        }
+        nb_request->data.release_arrays[idx]   = NULL;
+    }
 }
diff --git a/ompi/mpi/fortran/mpif-h/ialltoallv_f.c b/ompi/mpi/fortran/mpif-h/ialltoallv_f.c
index 0a447f5d5b6..b519a3c82cb 100644
--- a/ompi/mpi/fortran/mpif-h/ialltoallv_f.c
+++ b/ompi/mpi/fortran/mpif-h/ialltoallv_f.c
@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -23,6 +23,7 @@
 
 #include "ompi/mpi/fortran/mpif-h/bindings.h"
 #include "ompi/mpi/fortran/base/constants.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -75,7 +76,7 @@ void ompi_ialltoallv_f(char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *sdispls,
     MPI_Comm c_comm;
     MPI_Datatype c_sendtype, c_recvtype;
     MPI_Request c_request;
-    int size, c_ierr;
+    int size, idx = 0, c_ierr;
     OMPI_ARRAY_NAME_DECL(sendcounts);
     OMPI_ARRAY_NAME_DECL(sdispls);
     OMPI_ARRAY_NAME_DECL(recvcounts);
@@ -106,8 +107,19 @@ void ompi_ialltoallv_f(char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *sdispls,
     if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
     if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
 
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(sendcounts);
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(sdispls);
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(recvcounts);
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(rdispls);
+    if ( REQUEST_COMPLETE(c_request)) {
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(sendcounts);
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(sdispls);
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(recvcounts);
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(rdispls);
+    } else {
+        ompi_coll_base_nbc_request_t* nb_request = (ompi_coll_base_nbc_request_t*)c_request;
+        if (sendcounts != OMPI_ARRAY_NAME_CONVERT(sendcounts)) {
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(sendcounts);
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(sdispls);
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(recvcounts);
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(rdispls);
+        }
+        nb_request->data.release_arrays[idx]   = NULL;
+    }
 }
diff --git a/ompi/mpi/fortran/mpif-h/ialltoallw_f.c b/ompi/mpi/fortran/mpif-h/ialltoallw_f.c
index 75f8262bef5..1c5dd1400f3 100644
--- a/ompi/mpi/fortran/mpif-h/ialltoallw_f.c
+++ b/ompi/mpi/fortran/mpif-h/ialltoallw_f.c
@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -24,6 +24,7 @@
 #include "ompi/mpi/fortran/mpif-h/bindings.h"
 #include "ompi/mpi/fortran/base/constants.h"
 #include "ompi/communicator/communicator.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -77,7 +78,7 @@ void ompi_ialltoallw_f(char *sendbuf, MPI_Fint *sendcounts,
     MPI_Comm c_comm;
     MPI_Datatype *c_sendtypes = NULL, *c_recvtypes;
     MPI_Request c_request;
-    int size, c_ierr;
+    int size, idx = 0, c_ierr;
     OMPI_ARRAY_NAME_DECL(sendcounts);
     OMPI_ARRAY_NAME_DECL(sdispls);
     OMPI_ARRAY_NAME_DECL(recvcounts);
@@ -107,22 +108,37 @@ void ompi_ialltoallw_f(char *sendbuf, MPI_Fint *sendcounts,
     recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
 
     c_ierr = PMPI_Ialltoallw(sendbuf,
-                            OMPI_ARRAY_NAME_CONVERT(sendcounts),
-                            OMPI_ARRAY_NAME_CONVERT(sdispls),
-                            c_sendtypes,
-                            recvbuf,
-                            OMPI_ARRAY_NAME_CONVERT(recvcounts),
-                            OMPI_ARRAY_NAME_CONVERT(rdispls),
-                            c_recvtypes, c_comm, &c_request);
+                             OMPI_ARRAY_NAME_CONVERT(sendcounts),
+                             OMPI_ARRAY_NAME_CONVERT(sdispls),
+                             c_sendtypes,
+                             recvbuf,
+                             OMPI_ARRAY_NAME_CONVERT(recvcounts),
+                             OMPI_ARRAY_NAME_CONVERT(rdispls),
+                             c_recvtypes, c_comm, &c_request);
     if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
     if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
 
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(sendcounts);
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(sdispls);
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(recvcounts);
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(rdispls);
-    if (NULL != c_sendtypes) {
-        free(c_sendtypes);
+    if ( REQUEST_COMPLETE(c_request)) {
+        if (NULL != c_sendtypes) {
+            free(c_sendtypes);
+        }
+        free(c_recvtypes);
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(sendcounts);
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(sdispls);
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(recvcounts);
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(rdispls);
+    } else {
+        ompi_coll_base_nbc_request_t* nb_request = (ompi_coll_base_nbc_request_t*)c_request;
+        if (sendcounts != OMPI_ARRAY_NAME_CONVERT(sendcounts)) {
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(sendcounts);
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(recvcounts);
+        }
+        if (sdispls != OMPI_ARRAY_NAME_CONVERT(sdispls)) {
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(sdispls);
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(rdispls);
+        }
+        nb_request->data.release_arrays[idx++] = c_recvtypes;
+        nb_request->data.release_arrays[idx++] = c_sendtypes;
+        nb_request->data.release_arrays[idx]   = NULL;
     }
-    free(c_recvtypes);
 }
diff --git a/ompi/mpi/fortran/mpif-h/igatherv_f.c b/ompi/mpi/fortran/mpif-h/igatherv_f.c
index c367c2f4abf..8af9e33b914 100644
--- a/ompi/mpi/fortran/mpif-h/igatherv_f.c
+++ b/ompi/mpi/fortran/mpif-h/igatherv_f.c
@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -23,6 +23,7 @@
 
 #include "ompi/mpi/fortran/mpif-h/bindings.h"
 #include "ompi/mpi/fortran/base/constants.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -75,7 +76,7 @@ void ompi_igatherv_f(char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype,
     MPI_Comm c_comm;
     MPI_Datatype c_sendtype, c_recvtype;
     MPI_Request c_request;
-    int size, c_ierr;
+    int size, idx = 0, c_ierr;
     OMPI_ARRAY_NAME_DECL(recvcounts);
     OMPI_ARRAY_NAME_DECL(displs);
 
@@ -100,4 +101,16 @@ void ompi_igatherv_f(char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype,
                           c_comm, &c_request);
     if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
     if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
+
+    if ( REQUEST_COMPLETE(c_request)) {
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(recvcounts);
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(displs);
+    } else {
+        ompi_coll_base_nbc_request_t* nb_request = (ompi_coll_base_nbc_request_t*)c_request;
+        if (recvcounts != OMPI_ARRAY_NAME_CONVERT(recvcounts)) {
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(recvcounts);
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(displs);
+        }
+        nb_request->data.release_arrays[idx]   = NULL;
+    }
 }
diff --git a/ompi/mpi/fortran/mpif-h/improbe_f.c b/ompi/mpi/fortran/mpif-h/improbe_f.c
index 936cc4e399c..af5fa2b01fe 100644
--- a/ompi/mpi/fortran/mpif-h/improbe_f.c
+++ b/ompi/mpi/fortran/mpif-h/improbe_f.c
@@ -95,7 +95,7 @@ void ompi_improbe_f(MPI_Fint *source, MPI_Fint *tag, MPI_Fint *comm,
 
     if (MPI_SUCCESS == c_ierr) {
         OMPI_SINGLE_INT_2_LOGICAL(flag);
-        if (OMPI_FORTRAN_VALUE_TRUE == *flag) {
+        if (1 == OMPI_LOGICAL_2_INT(*flag)) {
             OMPI_FORTRAN_STATUS_RETURN(c_status,c_status2,status,c_ierr)
             *message = PMPI_Message_c2f(c_message);
         }
diff --git a/ompi/mpi/fortran/mpif-h/ineighbor_allgatherv_f.c b/ompi/mpi/fortran/mpif-h/ineighbor_allgatherv_f.c
index db30c7451a6..390e3dd2a74 100644
--- a/ompi/mpi/fortran/mpif-h/ineighbor_allgatherv_f.c
+++ b/ompi/mpi/fortran/mpif-h/ineighbor_allgatherv_f.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -26,6 +26,7 @@
 
 #include "ompi/mpi/fortran/mpif-h/bindings.h"
 #include "ompi/mpi/fortran/base/constants.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -78,7 +79,7 @@ void ompi_ineighbor_allgatherv_f(char *sendbuf, MPI_Fint *sendcount, MPI_Fint *s
     MPI_Comm c_comm;
     MPI_Datatype c_sendtype, c_recvtype;
     MPI_Request c_request;
-    int size, ierr_c;
+    int size, idx = 0, ierr_c;
     OMPI_ARRAY_NAME_DECL(recvcounts);
     OMPI_ARRAY_NAME_DECL(displs);
 
@@ -103,8 +104,16 @@ void ompi_ineighbor_allgatherv_f(char *sendbuf, MPI_Fint *sendcount, MPI_Fint *s
                                        c_recvtype, c_comm, &c_request);
 
     if (NULL != ierr) *ierr = OMPI_INT_2_FINT(ierr_c);
-    if (MPI_SUCCESS == ierr_c) *request = PMPI_Request_c2f(c_request);
-
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(recvcounts);
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(displs);
+    if (MPI_SUCCESS == ierr_c) {
+        *request = PMPI_Request_c2f(c_request);
+        ompi_coll_base_nbc_request_t* nb_request = (ompi_coll_base_nbc_request_t*)c_request;
+        if (recvcounts != OMPI_ARRAY_NAME_CONVERT(recvcounts)) {
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(recvcounts);
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(displs);
+        }
+        nb_request->data.release_arrays[idx]   = NULL;
+    } else {
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(recvcounts);
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(displs);
+    }
 }
diff --git a/ompi/mpi/fortran/mpif-h/ineighbor_alltoallv_f.c b/ompi/mpi/fortran/mpif-h/ineighbor_alltoallv_f.c
index 89761ace66d..c7b9901cadf 100644
--- a/ompi/mpi/fortran/mpif-h/ineighbor_alltoallv_f.c
+++ b/ompi/mpi/fortran/mpif-h/ineighbor_alltoallv_f.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -26,6 +26,7 @@
 
 #include "ompi/mpi/fortran/mpif-h/bindings.h"
 #include "ompi/mpi/fortran/base/constants.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -78,7 +79,7 @@ void ompi_ineighbor_alltoallv_f(char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *s
     MPI_Comm c_comm;
     MPI_Datatype c_sendtype, c_recvtype;
     MPI_Request c_request;
-    int size, c_ierr;
+    int size, idx = 0, c_ierr;
     OMPI_ARRAY_NAME_DECL(sendcounts);
     OMPI_ARRAY_NAME_DECL(sdispls);
     OMPI_ARRAY_NAME_DECL(recvcounts);
@@ -109,8 +110,17 @@ void ompi_ineighbor_alltoallv_f(char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *s
     if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
     if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
 
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(sendcounts);
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(sdispls);
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(recvcounts);
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(rdispls);
+    if ( REQUEST_COMPLETE(c_request)) {
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(sendcounts);
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(sdispls);
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(recvcounts);
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(rdispls);
+    } else {
+        ompi_coll_base_nbc_request_t* nb_request = (ompi_coll_base_nbc_request_t*)c_request;
+        nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(sendcounts);
+        nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(sdispls);
+        nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(sendcounts);
+        nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(rdispls);
+        nb_request->data.release_arrays[idx]   = NULL;
+    }
 }
diff --git a/ompi/mpi/fortran/mpif-h/ineighbor_alltoallw_f.c b/ompi/mpi/fortran/mpif-h/ineighbor_alltoallw_f.c
index 42116da7a81..680d6a9d94a 100644
--- a/ompi/mpi/fortran/mpif-h/ineighbor_alltoallw_f.c
+++ b/ompi/mpi/fortran/mpif-h/ineighbor_alltoallw_f.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -26,6 +26,7 @@
 
 #include "ompi/mpi/fortran/mpif-h/bindings.h"
 #include "ompi/mpi/fortran/base/constants.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -79,7 +80,7 @@ void ompi_ineighbor_alltoallw_f(char *sendbuf, MPI_Fint *sendcounts,
     MPI_Comm c_comm;
     MPI_Datatype *c_sendtypes, *c_recvtypes;
     MPI_Request c_request;
-    int size, c_ierr;
+    int size, idx = 0, c_ierr;
     OMPI_ARRAY_NAME_DECL(sendcounts);
     OMPI_ARRAY_NAME_DECL(recvcounts);
 
@@ -113,8 +114,17 @@ void ompi_ineighbor_alltoallw_f(char *sendbuf, MPI_Fint *sendcounts,
     if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
     if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
 
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(sendcounts);
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(recvcounts);
-    free(c_sendtypes);
-    free(c_recvtypes);
+    if ( REQUEST_COMPLETE(c_request)) {
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(sendcounts);
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(recvcounts);
+        free(c_sendtypes);
+        free(c_recvtypes);
+    } else {
+        ompi_coll_base_nbc_request_t* nb_request = (ompi_coll_base_nbc_request_t*)c_request;
+        nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(sendcounts);
+        nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(recvcounts);
+        nb_request->data.release_arrays[idx++] = c_sendtypes;
+        nb_request->data.release_arrays[idx++] = c_recvtypes;
+        nb_request->data.release_arrays[idx]   = NULL;
+    }
 }
diff --git a/ompi/mpi/fortran/mpif-h/info_create_env_f.c b/ompi/mpi/fortran/mpif-h/info_create_env_f.c
new file mode 100644
index 00000000000..e138d73a28e
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/info_create_env_f.c
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_INFO_CREATE_ENV = ompi_info_create_env_f
+#pragma weak pmpi_info_create_env = ompi_info_create_env_f
+#pragma weak pmpi_info_create_env_ = ompi_info_create_env_f
+#pragma weak pmpi_info_create_env__ = ompi_info_create_env_f
+
+#pragma weak PMPI_Info_create_f = ompi_info_create_env_f
+#pragma weak PMPI_Info_create_f08 = ompi_info_create_env_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_INFO_CREATE_ENV,
+                           pmpi_info_create_env,
+                           pmpi_info_create_env_,
+                           pmpi_info_create_env__,
+                           pompi_info_create_env_f,
+                           (MPI_Fint *info, MPI_Fint *ierr),
+                           (info, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_INFO_CREATE_ENV = ompi_info_create_env_f
+#pragma weak mpi_info_create_env = ompi_info_create_env_f
+#pragma weak mpi_info_create_env_ = ompi_info_create_env_f
+#pragma weak mpi_info_create_env__ = ompi_info_create_env_f
+
+#pragma weak MPI_Info_create_f = ompi_info_create_env_f
+#pragma weak MPI_Info_create_f08 = ompi_info_create_env_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_INFO_CREATE_ENV,
+                           mpi_info_create_env,
+                           mpi_info_create_env_,
+                           mpi_info_create_env__,
+                           ompi_info_create_env_f,
+                           (MPI_Fint *info, MPI_Fint *ierr),
+                           (info, ierr) )
+#else
+#define ompi_info_create_env_f pompi_info_create_env_f
+#endif
+#endif
+
+
+void ompi_info_create_env_f(MPI_Fint *info, MPI_Fint *ierr)
+{
+    int c_ierr;
+    MPI_Info c_info;
+
+    c_ierr = PMPI_Info_create_env(0, NULL, &c_info);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *info = PMPI_Info_c2f(c_info);
+    }
+}
diff --git a/ompi/mpi/fortran/mpif-h/info_get_f.c b/ompi/mpi/fortran/mpif-h/info_get_f.c
index a1b479b08ef..d9d6b4ac859 100644
--- a/ompi/mpi/fortran/mpif-h/info_get_f.c
+++ b/ompi/mpi/fortran/mpif-h/info_get_f.c
@@ -82,8 +82,9 @@ void ompi_info_get_f(MPI_Fint *info, char *key, MPI_Fint *valuelen,
 {
     int c_ierr, ret;
     MPI_Info c_info;
-    char *c_key = NULL, c_value[MPI_MAX_INFO_VAL + 1];
+    char *c_key = NULL;
     OMPI_LOGICAL_NAME_DECL(flag);
+    opal_cstring_t *info_str;
 
     if (OMPI_SUCCESS != (ret = ompi_fortran_string_f2c(key, key_len, &c_key))) {
         c_ierr = OMPI_ERRHANDLER_NOHANDLE_INVOKE(ret, FUNC_NAME);
@@ -92,10 +93,8 @@ void ompi_info_get_f(MPI_Fint *info, char *key, MPI_Fint *valuelen,
     }
     c_info = PMPI_Info_f2c(*info);
 
-    c_ierr = PMPI_Info_get(c_info, c_key,
-                          OMPI_FINT_2_INT(*valuelen),
-                          c_value,
-                          OMPI_LOGICAL_SINGLE_NAME_CONVERT(flag));
+    c_ierr = ompi_info_get(c_info, c_key, &info_str,
+                           OMPI_LOGICAL_SINGLE_NAME_CONVERT(flag));
     if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
 
     if (MPI_SUCCESS == c_ierr) {
@@ -107,12 +106,13 @@ void ompi_info_get_f(MPI_Fint *info, char *key, MPI_Fint *valuelen,
            Fortran compilers have TRUE == 1).  Note: use the full
            length of the Fortran string, not *valuelen.  See comment
            in ompi/mpi/fortran/base/strings.c. */
-        if (*flag && OMPI_SUCCESS !=
-            (ret = ompi_fortran_string_c2f(c_value, value, value_len))) {
-            c_ierr = OMPI_ERRHANDLER_NOHANDLE_INVOKE(ret, FUNC_NAME);
-            if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
-            free(c_key);
-            return;
+        if (*flag) {
+            if (OMPI_SUCCESS !=
+                (ret = ompi_fortran_string_c2f(info_str->string, value, value_len))) {
+                c_ierr = OMPI_ERRHANDLER_NOHANDLE_INVOKE(ret, FUNC_NAME);
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+            }
+            OBJ_RELEASE(info_str);
         }
     }
 
diff --git a/ompi/mpi/fortran/mpif-h/info_get_nthkey_f.c b/ompi/mpi/fortran/mpif-h/info_get_nthkey_f.c
index 2e93f50afe3..db5223ae2d4 100644
--- a/ompi/mpi/fortran/mpif-h/info_get_nthkey_f.c
+++ b/ompi/mpi/fortran/mpif-h/info_get_nthkey_f.c
@@ -81,18 +81,19 @@ void ompi_info_get_nthkey_f(MPI_Fint *info, MPI_Fint *n, char *key,
 {
     int c_ierr, ret;
     MPI_Info c_info;
-    char c_key[MPI_MAX_INFO_KEY + 1];
+    opal_cstring_t *key_str;
 
     c_info = PMPI_Info_f2c(*info);
 
-    c_ierr = PMPI_Info_get_nthkey(c_info,
-                                 OMPI_FINT_2_INT(*n),
-                                 c_key);
+    c_ierr = ompi_info_get_nthkey(c_info, OMPI_FINT_2_INT(*n), &key_str);
     if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
 
-    if (OMPI_SUCCESS != (ret = ompi_fortran_string_c2f(c_key, key, key_len))) {
-        c_ierr = OMPI_ERRHANDLER_NOHANDLE_INVOKE(ret, FUNC_NAME);
-        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
-        return;
+    if (NULL != key_str) {
+        if (OMPI_SUCCESS != (ret = ompi_fortran_string_c2f(key_str->string, key, key_len))) {
+            c_ierr = OMPI_ERRHANDLER_NOHANDLE_INVOKE(ret, FUNC_NAME);
+            if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        }
+        OBJ_RELEASE(key_str);
     }
+
 }
diff --git a/ompi/mpi/fortran/mpif-h/info_get_string_f.c b/ompi/mpi/fortran/mpif-h/info_get_string_f.c
new file mode 100644
index 00000000000..76919ba3d85
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/info_get_string_f.c
@@ -0,0 +1,143 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2017 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/constants.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/mpi/fortran/base/fortran_base_strings.h"
+#include "ompi/info/info.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_INFO_GET_STRING = ompi_info_get_string_f
+#pragma weak pmpi_info_get_string = ompi_info_get_string_f
+#pragma weak pmpi_info_get_string_ = ompi_info_get_string_f
+#pragma weak pmpi_info_get_string__ = ompi_info_get_string_f
+
+#pragma weak PMPI_Info_get_string_f = ompi_info_get_string_f
+#pragma weak PMPI_Info_get_string_f08 = ompi_info_get_string_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_INFO_GET_STRING,
+                            pmpi_info_get_string,
+                            pmpi_info_get_string_,
+                            pmpi_info_get_string__,
+                            pompi_info_get_string_f,
+                            (MPI_Fint *info, char *key, MPI_Fint *buflen, char *value, ompi_fortran_logical_t *flag, MPI_Fint *ierr, int key_len, int value_len),
+                            (info, key, buflen, value, flag, ierr, key_len, value_len) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_INFO_GET_STRING = ompi_info_get_string_f
+#pragma weak mpi_info_get_string = ompi_info_get_string_f
+#pragma weak mpi_info_get_string_ = ompi_info_get_string_f
+#pragma weak mpi_info_get_string__ = ompi_info_get_string_f
+
+#pragma weak MPI_Info_get_string_f = ompi_info_get_string_f
+#pragma weak MPI_Info_get_string_f08 = ompi_info_get_string_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_INFO_GET_STRING,
+                            mpi_info_get_string,
+                            mpi_info_get_string_,
+                            mpi_info_get_string__,
+                            ompi_info_get_string_f,
+                            (MPI_Fint *info, char *key, MPI_Fint *buflen, char *value, ompi_fortran_logical_t *flag, MPI_Fint *ierr, int key_len, int value_len),
+                            (info, key, buflen, value, flag, ierr, key_len, value_len) )
+#else
+#define ompi_info_get_string_f pompi_info_get_string_f
+#endif
+#endif
+
+
+static const char FUNC_NAME[] = "MPI_INFO_GET_STRING";
+
+/* Note that the key_len and value_len parameters are silently added
+   by the Fortran compiler, and will be filled in with the actual
+   length of the character array from the caller.  Hence, it's the max
+   length of the string that we can use. */
+
+void ompi_info_get_string_f(MPI_Fint *info, char *key, MPI_Fint *buflen,
+                            char *value, ompi_fortran_logical_t *flag, MPI_Fint *ierr,
+                            int key_len, int value_len)
+{
+    int c_ierr, ret;
+    MPI_Info c_info;
+    char *c_key = NULL;
+    OMPI_SINGLE_NAME_DECL(buflen);
+    OMPI_LOGICAL_NAME_DECL(flag);
+    opal_cstring_t *info_str;
+
+    if (OMPI_SUCCESS != (ret = ompi_fortran_string_f2c(key, key_len, &c_key))) {
+        c_ierr = OMPI_ERRHANDLER_NOHANDLE_INVOKE(ret, FUNC_NAME);
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        return;
+    }
+
+    c_info = PMPI_Info_f2c(*info);
+
+    if (0 == *buflen) {
+        c_ierr = ompi_info_get_valuelen(c_info, c_key,
+                                       OMPI_SINGLE_NAME_CONVERT(buflen),
+                                       OMPI_LOGICAL_SINGLE_NAME_CONVERT(flag));
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+        if (MPI_SUCCESS == c_ierr) {
+            OMPI_SINGLE_INT_2_FINT(buflen);
+            OMPI_SINGLE_INT_2_LOGICAL(flag);
+        }
+    } else { 
+        c_ierr = ompi_info_get(c_info, c_key, &info_str,
+                               OMPI_LOGICAL_SINGLE_NAME_CONVERT(flag));
+        if (NULL != ierr) {
+            *ierr = OMPI_INT_2_FINT(c_ierr);
+        }
+        
+
+        if (MPI_SUCCESS == c_ierr) {
+            OMPI_SINGLE_INT_2_LOGICAL(flag);
+
+            /* If we found the info key, copy the value back to the
+               Fortran string (note: all Fortran compilers have FALSE ==
+               0, so just check for any nonzero value, because not all
+               Fortran compilers have TRUE == 1).  Note: use the full
+               length of the Fortran string, which means adding one to the 3rd arg 
+               to ompi_fortran_string_c2f */
+            if (*flag) {
+                if (OMPI_SUCCESS !=
+                  (ret = ompi_fortran_string_c2f(info_str->string, value, value_len + 1))) {
+                    c_ierr = OMPI_ERRHANDLER_NOHANDLE_INVOKE(ret, FUNC_NAME);
+                    if (NULL != ierr) {
+                        *ierr = OMPI_INT_2_FINT(c_ierr);
+                    }
+                }
+                *buflen = info_str->length;
+                OBJ_RELEASE(info_str);
+            }
+        }
+    }
+
+    if (NULL != c_key) {
+        free(c_key);
+    }
+}
diff --git a/ompi/mpi/fortran/mpif-h/init_f.c b/ompi/mpi/fortran/mpif-h/init_f.c
index 087d967f534..8ed309f6827 100644
--- a/ompi/mpi/fortran/mpif-h/init_f.c
+++ b/ompi/mpi/fortran/mpif-h/init_f.c
@@ -22,17 +22,6 @@
 
 #include "ompi_config.h"
 
-#if (OPAL_HAVE_WEAK_SYMBOLS || ! OMPI_BUILD_MPI_PROFILING)
-#if OPAL_CC_USE_PRAGMA_IDENT
-#pragma ident OMPI_IDENT_STRING
-#elif OPAL_CC_USE_IDENT
-#ident OMPI_IDENT_STRING
-#else
-const char ident[] = OMPI_IDENT_STRING;
-#endif
-#endif
-
-
 #include "ompi/mpi/fortran/mpif-h/bindings.h"
 
 #if OMPI_BUILD_MPI_PROFILING
diff --git a/ompi/mpi/fortran/mpif-h/intercomm_create_from_groups_f.c b/ompi/mpi/fortran/mpif-h/intercomm_create_from_groups_f.c
new file mode 100644
index 00000000000..489ec241a3a
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/intercomm_create_from_groups_f.c
@@ -0,0 +1,123 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2013 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2019      Triad National Security, LLC.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/fortran_base_strings.h"
+#include "ompi/constants.h"
+#include "ompi/instance/instance.h"
+#include "ompi/group/group.h"
+
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_INTERCOMM_CREATE_FROM_GROUPS = ompi_intercomm_create_from_groups_f
+#pragma weak pmpi_intercomm_create_from_groups = ompi_intercomm_create_from_groups_f
+#pragma weak pmpi_intercomm_create_from_groups_ = ompi_intercomm_create_from_groups_f
+#pragma weak pmpi_intercomm_create_from_groups__ = ompi_intercomm_create_from_groups_f
+
+#pragma weak PMPI_Intercomm_create_from_groups_f = ompi_intercomm_create_from_groups_f
+#pragma weak PMPI_Intercomm_create_from_groups_f08 = ompi_intercomm_create_from_groups_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_INTERCOMM_CREATE_FROM_GROUPS,
+                            pmpi_intercomm_create_from_groups,
+                            pmpi_intercomm_create_from_groups_,
+                            pmpi_intercomm_create_from_groups__,
+                            pompi_intercomm_create_from_groups_f,
+                            (MPI_Fint *local_group, MPI_Fint *local_leader, MPI_Fint *remote_group,
+                             MPI_Fint *remote_leader, char *stringtag, MPI_Fint *info, MPI_Fint *errhandler,
+                             MPI_Fint *internewcomm, MPI_Fint *ierr, int name_len),
+                            (local_group, local_leader, remote_group,
+                             remote_leader, stringtag, info, errhandler, internewcomm, ierr, name_len) )
+
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_INTERCOMM_CREATE_FROM_GROUPS = ompi_intercomm_create_from_groups_f
+#pragma weak mpi_intercomm_create_from_groups = ompi_intercomm_create_from_groups_f
+#pragma weak mpi_intercomm_create_from_groups_ = ompi_intercomm_create_from_groups_f
+#pragma weak mpi_intercomm_create_from_groups__ = ompi_intercomm_create_from_groups_f
+
+#pragma weak MPI_Intercomm_create_from_groups_f = ompi_intercomm_create_from_groups_f
+#pragma weak MPI_Intercomm_create_from_groups_f08 = ompi_intercomm_create_from_groups_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_INTERCOMM_CREATE_FROM_GROUPS,
+                            mpi_intercomm_create_from_groups,
+                            mpi_intercomm_create_from_groups_,
+                            mpi_intercomm_create_from_groups__,
+                            ompi_intercomm_create_from_groups_f,
+                            (MPI_Fint *local_group, MPI_Fint *local_leader, MPI_Fint *remote_group,
+                             MPI_Fint *remote_leader, char *stringtag, MPI_Fint *info, MPI_Fint *errhandler,
+                             MPI_Fint *internewcomm, MPI_Fint *ierr, int name_len),
+                            (local_group, local_leader, remote_group,
+                             remote_leader, stringtag, info, errhandler, internewcomm, ierr, name_len) )
+#else
+#define ompi_intercomm_create_from_groups_f pompi_intercomm_create_from_groups_f
+#endif
+#endif
+
+void ompi_intercomm_create_from_groups_f(MPI_Fint *local_group, MPI_Fint *local_leader, MPI_Fint *remote_group,
+                                         MPI_Fint *remote_leader, char *stringtag, MPI_Fint *info, MPI_Fint *errhandler, 
+                                         MPI_Fint *internewcomm, MPI_Fint *ierr, int name_len)
+{
+    int c_ierr, ret;
+    MPI_Group c_lgroup, c_rgroup;
+    char *c_tag;
+    MPI_Comm c_intercomm;
+    MPI_Info c_info;
+    MPI_Errhandler c_err;
+
+    c_lgroup = PMPI_Group_f2c(*local_group);
+    c_rgroup = PMPI_Group_f2c(*remote_group);
+    c_info = PMPI_Info_f2c(*info);
+    c_err = PMPI_Errhandler_f2c(*errhandler);
+
+    /* Convert the fortran string */
+
+    /* Convert the fortran string */
+    if (OMPI_SUCCESS != (ret = ompi_fortran_string_f2c(stringtag, name_len,
+                                                       &c_tag))) {
+        c_ierr = OMPI_ERRHANDLER_INVOKE(ompi_group_get_instance(c_lgroup), ret, "MPI_INTERCOMM_CREATE_FROM_GROUPS");
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        return;
+    }
+
+    c_ierr = PMPI_Intercomm_create_from_groups(c_lgroup,  OMPI_FINT_2_INT(*local_leader), 
+                                               c_rgroup,  OMPI_FINT_2_INT(*remote_leader), 
+                                               c_tag, c_info, c_err, &c_intercomm);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *internewcomm = PMPI_Comm_c2f (c_intercomm);
+    }
+
+    /* Free the C tag */
+
+    free(c_tag);
+}
diff --git a/ompi/mpi/fortran/mpif-h/intercomm_merge_f.c b/ompi/mpi/fortran/mpif-h/intercomm_merge_f.c
index 6c07c66580b..865350280e1 100644
--- a/ompi/mpi/fortran/mpif-h/intercomm_merge_f.c
+++ b/ompi/mpi/fortran/mpif-h/intercomm_merge_f.c
@@ -10,8 +10,8 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015-2023 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -38,8 +38,8 @@ OMPI_GENERATE_F77_BINDINGS (PMPI_INTERCOMM_MERGE,
                            pmpi_intercomm_merge_,
                            pmpi_intercomm_merge__,
                            pompi_intercomm_merge_f,
-                           (MPI_Fint *intercomm, ompi_fortran_logical_t *high, MPI_Fint *newintercomm, MPI_Fint *ierr),
-                           (intercomm, high, newintercomm, ierr) )
+                           (MPI_Fint *intercomm, ompi_fortran_logical_t *high, MPI_Fint *newintracomm, MPI_Fint *ierr),
+                           (intercomm, high, newintracomm, ierr) )
 #endif
 #endif
 
@@ -58,8 +58,8 @@ OMPI_GENERATE_F77_BINDINGS (MPI_INTERCOMM_MERGE,
                            mpi_intercomm_merge_,
                            mpi_intercomm_merge__,
                            ompi_intercomm_merge_f,
-                           (MPI_Fint *intercomm, ompi_fortran_logical_t *high, MPI_Fint *newintercomm, MPI_Fint *ierr),
-                           (intercomm, high, newintercomm, ierr) )
+                           (MPI_Fint *intercomm, ompi_fortran_logical_t *high, MPI_Fint *newintracomm, MPI_Fint *ierr),
+                           (intercomm, high, newintracomm, ierr) )
 #else
 #define ompi_intercomm_merge_f pompi_intercomm_merge_f
 #endif
diff --git a/ompi/mpi/fortran/mpif-h/ireduce_scatter_f.c b/ompi/mpi/fortran/mpif-h/ireduce_scatter_f.c
index a0910394dfa..d09a397cac6 100644
--- a/ompi/mpi/fortran/mpif-h/ireduce_scatter_f.c
+++ b/ompi/mpi/fortran/mpif-h/ireduce_scatter_f.c
@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -23,6 +23,7 @@
 
 #include "ompi/mpi/fortran/mpif-h/bindings.h"
 #include "ompi/mpi/fortran/base/constants.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -72,12 +73,11 @@ void ompi_ireduce_scatter_f(char *sendbuf, char *recvbuf,
                             MPI_Fint *op, MPI_Fint *comm, MPI_Fint *request,
                             MPI_Fint *ierr)
 {
-    int c_ierr;
+    int c_ierr, size;
     MPI_Comm c_comm;
     MPI_Datatype c_type;
     MPI_Request c_request;
     MPI_Op c_op;
-    int size;
     OMPI_ARRAY_NAME_DECL(recvcounts);
 
     c_comm = PMPI_Comm_f2c(*comm);
@@ -96,4 +96,12 @@ void ompi_ireduce_scatter_f(char *sendbuf, char *recvbuf,
                                  c_type, c_op, c_comm, &c_request);
     if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
     if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
+
+    if ( REQUEST_COMPLETE(c_request)) {
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(recvcounts);
+    } else {
+        ompi_coll_base_nbc_request_t* nb_request = (ompi_coll_base_nbc_request_t*)c_request;
+        nb_request->data.release_arrays[0] = recvcounts;
+        nb_request->data.release_arrays[1] = NULL;
+    }
 }
diff --git a/ompi/mpi/fortran/mpif-h/iscatterv_f.c b/ompi/mpi/fortran/mpif-h/iscatterv_f.c
index cceeb47ce30..01226a83ea8 100644
--- a/ompi/mpi/fortran/mpif-h/iscatterv_f.c
+++ b/ompi/mpi/fortran/mpif-h/iscatterv_f.c
@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -24,6 +24,7 @@
 
 #include "ompi/mpi/fortran/mpif-h/bindings.h"
 #include "ompi/mpi/fortran/base/constants.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -77,7 +78,7 @@ void ompi_iscatterv_f(char *sendbuf, MPI_Fint *sendcounts,
     MPI_Comm c_comm;
     MPI_Datatype c_sendtype, c_recvtype;
     MPI_Request c_request;
-    int size, c_ierr;
+    int size, idx = 0, c_ierr;
     OMPI_ARRAY_NAME_DECL(sendcounts);
     OMPI_ARRAY_NAME_DECL(displs);
 
@@ -103,6 +104,15 @@ void ompi_iscatterv_f(char *sendbuf, MPI_Fint *sendcounts,
     if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
     if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
 
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(sendcounts);
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(displs);
+    if ( REQUEST_COMPLETE(c_request)) {
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(sendcounts);
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(displs);
+    } else {
+        ompi_coll_base_nbc_request_t* nb_request = (ompi_coll_base_nbc_request_t*)c_request;
+        if (sendcounts != OMPI_ARRAY_NAME_CONVERT(sendcounts)) {
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(sendcounts);
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(displs);
+        }
+        nb_request->data.release_arrays[idx]   = NULL;
+    }
 }
diff --git a/ompi/mpi/fortran/mpif-h/isendrecv_f.c b/ompi/mpi/fortran/mpif-h/isendrecv_f.c
new file mode 100644
index 00000000000..cf010ad0803
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/isendrecv_f.c
@@ -0,0 +1,98 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2021      Triad National Security, LLC. All rights
+ *                         reserved.
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_ISENDRECV = ompi_isendrecv_f
+#pragma weak pmpi_isendrecv = ompi_isendrecv_f
+#pragma weak pmpi_isendrecv_ = ompi_isendrecv_f
+#pragma weak pmpi_isendrecv__ = ompi_isendrecv_f
+
+#pragma weak PMPI_Isendrecv_f = ompi_isendrecv_f
+#pragma weak PMPI_Isendrecv_f08 = ompi_isendrecv_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_ISENDRECV,
+                           pmpi_isendrecv,
+                           pmpi_isendrecv_,
+                           pmpi_isendrecv__,
+                           pompi_isendrecv_f,
+                           (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, MPI_Fint *dest, MPI_Fint *sendtag, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *source, MPI_Fint *recvtag, MPI_Fint *comm, MPI_Fint *request, MPI_Fint *ierr),
+                           (sendbuf, sendcount, sendtype, dest, sendtag, recvbuf, recvcount, recvtype, source, recvtag, comm, request, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_ISENDRECV = ompi_isendrecv_f
+#pragma weak mpi_isendrecv = ompi_isendrecv_f
+#pragma weak mpi_isendrecv_ = ompi_isendrecv_f
+#pragma weak mpi_isendrecv__ = ompi_isendrecv_f
+
+#pragma weak MPI_Isendrecv_f = ompi_isendrecv_f
+#pragma weak MPI_Isendrecv_f08 = ompi_isendrecv_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_ISENDRECV,
+                           mpi_isendrecv,
+                           mpi_isendrecv_,
+                           mpi_isendrecv__,
+                           ompi_isendrecv_f,
+                           (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, MPI_Fint *dest, MPI_Fint *sendtag, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *source, MPI_Fint *recvtag, MPI_Fint *comm, MPI_Fint *request, MPI_Fint *ierr),
+                           (sendbuf, sendcount, sendtype, dest, sendtag, recvbuf, recvcount, recvtype, source, recvtag, comm, request, ierr) )
+#else
+#define ompi_isendrecv_f pompi_isendrecv_f
+#endif
+#endif
+
+
+void ompi_isendrecv_f(char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype,
+		    MPI_Fint *dest, MPI_Fint *sendtag, char *recvbuf,
+		    MPI_Fint *recvcount, MPI_Fint *recvtype,
+		    MPI_Fint *source, MPI_Fint *recvtag, MPI_Fint *comm,
+		    MPI_Fint *request, MPI_Fint *ierr)
+{
+   int c_ierr;
+   MPI_Comm c_comm;
+   MPI_Datatype c_sendtype = PMPI_Type_f2c(*sendtype);
+   MPI_Datatype c_recvtype = PMPI_Type_f2c(*recvtype);
+   MPI_Request c_req;
+
+   c_comm = PMPI_Comm_f2c (*comm);
+
+   c_ierr = PMPI_Isendrecv(OMPI_F2C_BOTTOM(sendbuf), OMPI_FINT_2_INT(*sendcount),
+                         c_sendtype,
+                         OMPI_FINT_2_INT(*dest),
+                         OMPI_FINT_2_INT(*sendtag),
+                         OMPI_F2C_BOTTOM(recvbuf), OMPI_FINT_2_INT(*recvcount),
+                         c_recvtype, OMPI_FINT_2_INT(*source),
+                         OMPI_FINT_2_INT(*recvtag),
+                         c_comm, &c_req);
+   if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+   if (MPI_SUCCESS == c_ierr) {
+      *request = PMPI_Request_c2f(c_req);
+   }
+}
diff --git a/ompi/mpi/fortran/mpif-h/isendrecv_replace_f.c b/ompi/mpi/fortran/mpif-h/isendrecv_replace_f.c
new file mode 100644
index 00000000000..37d69588d2b
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/isendrecv_replace_f.c
@@ -0,0 +1,99 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2021      Triad National Security, LLC. All rights
+ *                         reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_ISENDRECV_REPLACE = ompi_isendrecv_replace_f
+#pragma weak pmpi_isendrecv_replace = ompi_isendrecv_replace_f
+#pragma weak pmpi_isendrecv_replace_ = ompi_isendrecv_replace_f
+#pragma weak pmpi_isendrecv_replace__ = ompi_isendrecv_replace_f
+
+#pragma weak PMPI_Isendrecv_replace_f = ompi_isendrecv_replace_f
+#pragma weak PMPI_Isendrecv_replace_f08 = ompi_isendrecv_replace_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_ISENDRECV_REPLACE,
+                           pmpi_isendrecv_replace,
+                           pmpi_isendrecv_replace_,
+                           pmpi_isendrecv_replace__,
+                           pompi_isendrecv_replace_f,
+                           (char *buf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *dest, MPI_Fint *sendtag, MPI_Fint *source, MPI_Fint *recvtag, MPI_Fint *comm, MPI_Fint *request, MPI_Fint *ierr),
+                           (buf, count, datatype, dest, sendtag, source, recvtag, comm, request, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_ISENDRECV_REPLACE = ompi_isendrecv_replace_f
+#pragma weak mpi_isendrecv_replace = ompi_isendrecv_replace_f
+#pragma weak mpi_isendrecv_replace_ = ompi_isendrecv_replace_f
+#pragma weak mpi_isendrecv_replace__ = ompi_isendrecv_replace_f
+
+#pragma weak MPI_Isendrecv_replace_f = ompi_isendrecv_replace_f
+#pragma weak MPI_Isendrecv_replace_f08 = ompi_isendrecv_replace_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_ISENDRECV_REPLACE,
+                           mpi_isendrecv_replace,
+                           mpi_isendrecv_replace_,
+                           mpi_isendrecv_replace__,
+                           ompi_isendrecv_replace_f,
+                           (char *buf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *dest, MPI_Fint *sendtag, MPI_Fint *source, MPI_Fint *recvtag, MPI_Fint *comm, MPI_Fint *status, MPI_Fint *ierr),
+                           (buf, count, datatype, dest, sendtag, source, recvtag, comm, status, ierr) )
+#else
+#define ompi_isendrecv_replace_f pompi_isendrecv_replace_f
+#endif
+#endif
+
+
+void ompi_isendrecv_replace_f(char *buf, MPI_Fint *count, MPI_Fint *datatype,
+			    MPI_Fint *dest, MPI_Fint *sendtag,
+			    MPI_Fint *source, MPI_Fint *recvtag,
+			    MPI_Fint *comm, MPI_Fint *request, MPI_Fint *ierr)
+{
+   int c_ierr;
+   MPI_Datatype c_type = PMPI_Type_f2c(*datatype);
+   MPI_Comm c_comm;
+   MPI_Request c_req;
+
+   c_comm = PMPI_Comm_f2c (*comm);
+
+   c_ierr = PMPI_Isendrecv_replace(OMPI_F2C_BOTTOM(buf),
+                                 OMPI_FINT_2_INT(*count),
+                                 c_type,
+                                 OMPI_FINT_2_INT(*dest),
+                                 OMPI_FINT_2_INT(*sendtag),
+                                 OMPI_FINT_2_INT(*source),
+                                 OMPI_FINT_2_INT(*recvtag),
+                                 c_comm, &c_req);
+   if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+   if (MPI_SUCCESS == c_ierr) {
+      *request = PMPI_Request_c2f(c_req);
+   }
+
+}
diff --git a/ompi/mpi/fortran/mpif-h/keyval_create_f.c b/ompi/mpi/fortran/mpif-h/keyval_create_f.c
index 039c777129c..4ad5c51ca67 100644
--- a/ompi/mpi/fortran/mpif-h/keyval_create_f.c
+++ b/ompi/mpi/fortran/mpif-h/keyval_create_f.c
@@ -34,13 +34,12 @@
 #pragma weak PMPI_Keyval_create_f = ompi_keyval_create_f
 #pragma weak PMPI_Keyval_create_f08 = ompi_keyval_create_f
 #else
-OMPI_GENERATE_F77_BINDINGS (PMPI_KEYVAL_CREATE,
-                           pmpi_keyval_create,
-                           pmpi_keyval_create_,
-                           pmpi_keyval_create__,
-                           pompi_keyval_create_f,
-                           (ompi_fint_copy_attr_function* copy_fn, ompi_fint_delete_attr_function* delete_fn, MPI_Fint *keyval, MPI_Fint *extra_state, MPI_Fint *ierr),
-                           (copy_fn, delete_fn, keyval, extra_state, ierr) )
+OMPI_GENERATE_F77_BINDINGS(PMPI_KEYVAL_CREATE, pmpi_keyval_create, pmpi_keyval_create_,
+                           pmpi_keyval_create__, pompi_keyval_create_f,
+                           (ompi_fint_copy_attr_function copy_fn,
+                            ompi_fint_delete_attr_function delete_fn, MPI_Fint *keyval,
+                            MPI_Fint *extra_state, MPI_Fint *ierr),
+                           (copy_fn, delete_fn, keyval, extra_state, ierr))
 #endif
 #endif
 
@@ -54,13 +53,12 @@ OMPI_GENERATE_F77_BINDINGS (PMPI_KEYVAL_CREATE,
 #pragma weak MPI_Keyval_create_f08 = ompi_keyval_create_f
 #else
 #if ! OMPI_BUILD_MPI_PROFILING
-OMPI_GENERATE_F77_BINDINGS (MPI_KEYVAL_CREATE,
-                           mpi_keyval_create,
-                           mpi_keyval_create_,
-                           mpi_keyval_create__,
-                           ompi_keyval_create_f,
-                           (ompi_fint_copy_attr_function* copy_fn, ompi_fint_delete_attr_function* delete_fn, MPI_Fint *keyval, MPI_Fint *extra_state, MPI_Fint *ierr),
-                           (copy_fn, delete_fn, keyval, extra_state, ierr) )
+OMPI_GENERATE_F77_BINDINGS(MPI_KEYVAL_CREATE, mpi_keyval_create, mpi_keyval_create_,
+                           mpi_keyval_create__, ompi_keyval_create_f,
+                           (ompi_fint_copy_attr_function copy_fn,
+                            ompi_fint_delete_attr_function delete_fn, MPI_Fint *keyval,
+                            MPI_Fint *extra_state, MPI_Fint *ierr),
+                           (copy_fn, delete_fn, keyval, extra_state, ierr))
 #else
 #define ompi_keyval_create_f pompi_keyval_create_f
 #endif
@@ -68,10 +66,9 @@ OMPI_GENERATE_F77_BINDINGS (MPI_KEYVAL_CREATE,
 
 static const char FUNC_NAME[] = "MPI_keyval_create_f";
 
-void ompi_keyval_create_f(ompi_fint_copy_attr_function* copy_attr_fn,
-                         ompi_fint_delete_attr_function* delete_attr_fn,
-                         MPI_Fint *keyval, MPI_Fint *extra_state,
-                         MPI_Fint *ierr)
+void ompi_keyval_create_f(ompi_fint_copy_attr_function copy_attr_fn,
+                          ompi_fint_delete_attr_function delete_attr_fn, MPI_Fint *keyval,
+                          MPI_Fint *extra_state, MPI_Fint *ierr)
 {
     int ret, c_ierr;
     OMPI_SINGLE_NAME_DECL(keyval);
diff --git a/ompi/mpi/fortran/mpif-h/neighbor_allgather_init_f.c b/ompi/mpi/fortran/mpif-h/neighbor_allgather_init_f.c
new file mode 100644
index 00000000000..ab6f9df6605
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/neighbor_allgather_init_f.c
@@ -0,0 +1,102 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_NEIGHBOR_ALLGATHER_INIT = ompi_neighbor_allgather_init_f
+#pragma weak pmpi_neighbor_allgather_init = ompi_neighbor_allgather_init_f
+#pragma weak pmpi_neighbor_allgather_init_ = ompi_neighbor_allgather_init_f
+#pragma weak pmpi_neighbor_allgather_init__ = ompi_neighbor_allgather_init_f
+
+#pragma weak PMPI_Neighbor_allgather_init_f = ompi_neighbor_allgather_init_f
+#pragma weak PMPI_Neighbor_allgather_init_f08 = ompi_neighbor_allgather_init_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_NEIGHBOR_ALLGATHER_INIT,
+                            pmpi_neighbor_allgather_init,
+                            pmpi_neighbor_allgather_init_,
+                            pmpi_neighbor_allgather_init__,
+                            pompi_neighbor_allgather_init_f,
+                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, info, request, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_NEIGHBOR_ALLGATHER_INIT = ompi_neighbor_allgather_init_f
+#pragma weak mpi_neighbor_allgather_init = ompi_neighbor_allgather_init_f
+#pragma weak mpi_neighbor_allgather_init_ = ompi_neighbor_allgather_init_f
+#pragma weak mpi_neighbor_allgather_init__ = ompi_neighbor_allgather_init_f
+
+#pragma weak MPI_Neighbor_allgather_init_f = ompi_neighbor_allgather_init_f
+#pragma weak MPI_Neighbor_allgather_init_f08 = ompi_neighbor_allgather_init_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_NEIGHBOR_ALLGATHER_INIT,
+                            mpi_neighbor_allgather_init,
+                            mpi_neighbor_allgather_init_,
+                            mpi_neighbor_allgather_init__,
+                            ompi_neighbor_allgather_init_f,
+                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, info, request, ierr) )
+#else
+#define ompi_neighbor_allgather_init_f pompi_neighbor_allgather_init_f
+#endif
+#endif
+
+
+void ompi_neighbor_allgather_init_f(char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype,
+                                    char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype,
+                                    MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
+{
+    int ierr_c;
+    MPI_Comm c_comm;
+    MPI_Info c_info;
+    MPI_Request c_req;
+    MPI_Datatype c_sendtype, c_recvtype;
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_sendtype = PMPI_Type_f2c(*sendtype);
+    c_recvtype = PMPI_Type_f2c(*recvtype);
+    c_info = PMPI_Info_f2c(*info);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    ierr_c = PMPI_Neighbor_allgather_init(sendbuf,
+                                          OMPI_FINT_2_INT(*sendcount),
+                                          c_sendtype,
+                                          recvbuf,
+                                          OMPI_FINT_2_INT(*recvcount),
+                                          c_recvtype, c_comm, c_info, &c_req);
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(ierr_c);
+
+    if (MPI_SUCCESS == ierr_c) *request = PMPI_Request_c2f(c_req);
+}
diff --git a/ompi/mpi/fortran/mpif-h/neighbor_allgatherv_init_f.c b/ompi/mpi/fortran/mpif-h/neighbor_allgatherv_init_f.c
new file mode 100644
index 00000000000..532365e2aa7
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/neighbor_allgatherv_init_f.c
@@ -0,0 +1,121 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_NEIGHBOR_ALLGATHERV_INIT = ompi_neighbor_allgatherv_init_f
+#pragma weak pmpi_neighbor_allgatherv_init = ompi_neighbor_allgatherv_init_f
+#pragma weak pmpi_neighbor_allgatherv_init_ = ompi_neighbor_allgatherv_init_f
+#pragma weak pmpi_neighbor_allgatherv_init__ = ompi_neighbor_allgatherv_init_f
+
+#pragma weak PMPI_Neighbor_allgatherv_init_f = ompi_neighbor_allgatherv_init_f
+#pragma weak PMPI_Neighbor_allgatherv_init_f08 = ompi_neighbor_allgatherv_init_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_NEIGHBOR_ALLGATHERV_INIT,
+                            pmpi_neighbor_allgatherv_init,
+                            pmpi_neighbor_allgatherv_init_,
+                            pmpi_neighbor_allgatherv_init__,
+                            pompi_neighbor_allgatherv_init_f,
+                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *displs, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, comm, info, request, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_NEIGHBOR_ALLGATHERV_INIT = ompi_neighbor_allgatherv_init_f
+#pragma weak mpi_neighbor_allgatherv_init = ompi_neighbor_allgatherv_init_f
+#pragma weak mpi_neighbor_allgatherv_init_ = ompi_neighbor_allgatherv_init_f
+#pragma weak mpi_neighbor_allgatherv_init__ = ompi_neighbor_allgatherv_init_f
+
+#pragma weak MPI_Neighbor_allgatherv_init_f = ompi_neighbor_allgatherv_init_f
+#pragma weak MPI_Neighbor_allgatherv_init_f08 = ompi_neighbor_allgatherv_init_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_NEIGHBOR_ALLGATHERV_INIT,
+                            mpi_neighbor_allgatherv_init,
+                            mpi_neighbor_allgatherv_init_,
+                            mpi_neighbor_allgatherv_init__,
+                            ompi_neighbor_allgatherv_init_f,
+                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *displs, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, comm, info, request, ierr) )
+#else
+#define ompi_neighbor_allgatherv_init_f pompi_neighbor_allgatherv_init_f
+#endif
+#endif
+
+
+void ompi_neighbor_allgatherv_init_f(char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype,
+                                     char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *displs,
+                                     MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request,
+                                     MPI_Fint *ierr)
+{
+    MPI_Comm c_comm;
+    MPI_Datatype c_sendtype, c_recvtype;
+    MPI_Info c_info;
+    MPI_Request c_request;
+    int size, ierr_c;
+    OMPI_ARRAY_NAME_DECL(recvcounts);
+    OMPI_ARRAY_NAME_DECL(displs);
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_sendtype = PMPI_Type_f2c(*sendtype);
+    c_recvtype = PMPI_Type_f2c(*recvtype);
+    c_info = PMPI_Info_f2c(*info);
+
+    PMPI_Comm_size(c_comm, &size);
+    OMPI_ARRAY_FINT_2_INT(recvcounts, size);
+    OMPI_ARRAY_FINT_2_INT(displs, size);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    ierr_c = PMPI_Neighbor_allgatherv_init(sendbuf,
+                                           OMPI_FINT_2_INT(*sendcount),
+                                           c_sendtype,
+                                           recvbuf,
+                                           OMPI_ARRAY_NAME_CONVERT(recvcounts),
+                                           OMPI_ARRAY_NAME_CONVERT(displs),
+                                           c_recvtype, c_comm, c_info, &c_request);
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(ierr_c);
+    if (MPI_SUCCESS == ierr_c) {
+        *request = PMPI_Request_c2f(c_request);
+        ompi_coll_base_nbc_request_t* nb_request = (ompi_coll_base_nbc_request_t*)c_request;
+        if (recvcounts != OMPI_ARRAY_NAME_CONVERT(recvcounts)) {
+            nb_request->data.release_arrays[0] = OMPI_ARRAY_NAME_CONVERT(recvcounts);
+            nb_request->data.release_arrays[1] = OMPI_ARRAY_NAME_CONVERT(displs);
+            nb_request->data.release_arrays[2] = NULL;
+        }
+    } else {
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(recvcounts);
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(displs);
+    }
+}
diff --git a/ompi/mpi/fortran/mpif-h/neighbor_alltoall_init_f.c b/ompi/mpi/fortran/mpif-h/neighbor_alltoall_init_f.c
new file mode 100644
index 00000000000..509cba401ca
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/neighbor_alltoall_init_f.c
@@ -0,0 +1,102 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_NEIGHBOR_ALLTOALL_INIT = ompi_neighbor_alltoall_init_f
+#pragma weak pmpi_neighbor_alltoall_init = ompi_neighbor_alltoall_init_f
+#pragma weak pmpi_neighbor_alltoall_init_ = ompi_neighbor_alltoall_init_f
+#pragma weak pmpi_neighbor_alltoall_init__ = ompi_neighbor_alltoall_init_f
+
+#pragma weak PMPI_Neighbor_alltoall_init_f = ompi_neighbor_alltoall_init_f
+#pragma weak PMPI_Neighbor_alltoall_init_f08 = ompi_neighbor_alltoall_init_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_NEIGHBOR_ALLTOALL_INIT,
+                            pmpi_neighbor_alltoall_init,
+                            pmpi_neighbor_alltoall_init_,
+                            pmpi_neighbor_alltoall_init__,
+                            pompi_neighbor_alltoall_init_f,
+                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, info, request, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_NEIGHBOR_ALLTOALL_INIT = ompi_neighbor_alltoall_init_f
+#pragma weak mpi_neighbor_alltoall_init = ompi_neighbor_alltoall_init_f
+#pragma weak mpi_neighbor_alltoall_init_ = ompi_neighbor_alltoall_init_f
+#pragma weak mpi_neighbor_alltoall_init__ = ompi_neighbor_alltoall_init_f
+
+#pragma weak MPI_Neighbor_alltoall_init_f = ompi_neighbor_alltoall_init_f
+#pragma weak MPI_Neighbor_alltoall_init_f08 = ompi_neighbor_alltoall_init_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_NEIGHBOR_ALLTOALL_INIT,
+                            mpi_neighbor_alltoall_init,
+                            mpi_neighbor_alltoall_init_,
+                            mpi_neighbor_alltoall_init__,
+                            ompi_neighbor_alltoall_init_f,
+                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, info, request, ierr) )
+#else
+#define ompi_neighbor_alltoall_init_f pompi_neighbor_alltoall_init_f
+#endif
+#endif
+
+
+void ompi_neighbor_alltoall_init_f(char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype,
+                                   char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype,
+                                   MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
+{
+    int c_ierr;
+    MPI_Comm c_comm;
+    MPI_Info c_info;
+    MPI_Request c_req;
+    MPI_Datatype c_sendtype, c_recvtype;
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_sendtype = PMPI_Type_f2c(*sendtype);
+    c_recvtype = PMPI_Type_f2c(*recvtype);
+    c_info = PMPI_Info_f2c(*info);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = PMPI_Neighbor_alltoall_init(sendbuf,
+                                         OMPI_FINT_2_INT(*sendcount),
+                                         c_sendtype,
+                                         recvbuf,
+                                         OMPI_FINT_2_INT(*recvcount),
+                                         c_recvtype, c_comm, c_info, &c_req);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_req);
+}
diff --git a/ompi/mpi/fortran/mpif-h/neighbor_alltoallv_init_f.c b/ompi/mpi/fortran/mpif-h/neighbor_alltoallv_init_f.c
new file mode 100644
index 00000000000..27ffadbc446
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/neighbor_alltoallv_init_f.c
@@ -0,0 +1,129 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_NEIGHBOR_ALLTOALLV_INIT = ompi_neighbor_alltoallv_init_f
+#pragma weak pmpi_neighbor_alltoallv_init = ompi_neighbor_alltoallv_init_f
+#pragma weak pmpi_neighbor_alltoallv_init_ = ompi_neighbor_alltoallv_init_f
+#pragma weak pmpi_neighbor_alltoallv_init__ = ompi_neighbor_alltoallv_init_f
+
+#pragma weak PMPI_Neighbor_alltoallv_init_f = ompi_neighbor_alltoallv_init_f
+#pragma weak PMPI_Neighbor_alltoallv_init_f08 = ompi_neighbor_alltoallv_init_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_NEIGHBOR_ALLTOALLV_INIT,
+                            pmpi_neighbor_alltoallv_init,
+                            pmpi_neighbor_alltoallv_init_,
+                            pmpi_neighbor_alltoallv_init__,
+                            pompi_neighbor_alltoallv_init_f,
+                            (char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *sdispls, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *rdispls, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, sendcounts, sdispls, sendtype, recvbuf, recvcounts, rdispls, recvtype, comm, info, request, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_NEIGHBOR_ALLTOALLV_INIT = ompi_neighbor_alltoallv_init_f
+#pragma weak mpi_neighbor_alltoallv_init = ompi_neighbor_alltoallv_init_f
+#pragma weak mpi_neighbor_alltoallv_init_ = ompi_neighbor_alltoallv_init_f
+#pragma weak mpi_neighbor_alltoallv_init__ = ompi_neighbor_alltoallv_init_f
+
+#pragma weak MPI_Neighbor_alltoallv_init_f = ompi_neighbor_alltoallv_init_f
+#pragma weak MPI_Neighbor_alltoallv_init_f08 = ompi_neighbor_alltoallv_init_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_NEIGHBOR_ALLTOALLV_INIT,
+                            mpi_neighbor_alltoallv_init,
+                            mpi_neighbor_alltoallv_init_,
+                            mpi_neighbor_alltoallv_init__,
+                            ompi_neighbor_alltoallv_init_f,
+                            (char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *sdispls, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *rdispls, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, sendcounts, sdispls, sendtype, recvbuf, recvcounts, rdispls, recvtype, comm, info, request, ierr) )
+#else
+#define ompi_neighbor_alltoallv_init_f pompi_neighbor_alltoallv_init_f
+#endif
+#endif
+
+
+void ompi_neighbor_alltoallv_init_f(char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *sdispls,
+                                    MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts,
+                                    MPI_Fint *rdispls, MPI_Fint *recvtype,
+                                    MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
+{
+    MPI_Comm c_comm;
+    MPI_Datatype c_sendtype, c_recvtype;
+    MPI_Info c_info;
+    MPI_Request c_request;
+    int size, idx = 0, c_ierr;
+    OMPI_ARRAY_NAME_DECL(sendcounts);
+    OMPI_ARRAY_NAME_DECL(sdispls);
+    OMPI_ARRAY_NAME_DECL(recvcounts);
+    OMPI_ARRAY_NAME_DECL(rdispls);
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_sendtype = PMPI_Type_f2c(*sendtype);
+    c_recvtype = PMPI_Type_f2c(*recvtype);
+    c_info = PMPI_Info_f2c(*info);
+
+    PMPI_Comm_size(c_comm, &size);
+    OMPI_ARRAY_FINT_2_INT(sendcounts, size);
+    OMPI_ARRAY_FINT_2_INT(sdispls, size);
+    OMPI_ARRAY_FINT_2_INT(recvcounts, size);
+    OMPI_ARRAY_FINT_2_INT(rdispls, size);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = PMPI_Neighbor_alltoallv_init(sendbuf,
+                                          OMPI_ARRAY_NAME_CONVERT(sendcounts),
+                                          OMPI_ARRAY_NAME_CONVERT(sdispls),
+                                          c_sendtype,
+                                          recvbuf,
+                                          OMPI_ARRAY_NAME_CONVERT(recvcounts),
+                                          OMPI_ARRAY_NAME_CONVERT(rdispls),
+                                          c_recvtype, c_comm, c_info, &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_request);
+        ompi_coll_base_nbc_request_t* nb_request = (ompi_coll_base_nbc_request_t*)c_request;
+        if (sendcounts != OMPI_ARRAY_NAME_CONVERT(sendcounts)) {
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(sendcounts);
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(sdispls);
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(recvcounts);
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(rdispls);
+        }
+        nb_request->data.release_arrays[idx] = NULL;
+    } else {
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(sendcounts);
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(sdispls);
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(recvcounts);
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(rdispls);
+    }
+}
diff --git a/ompi/mpi/fortran/mpif-h/neighbor_alltoallw_init_f.c b/ompi/mpi/fortran/mpif-h/neighbor_alltoallw_init_f.c
new file mode 100644
index 00000000000..59910c88e36
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/neighbor_alltoallw_init_f.c
@@ -0,0 +1,131 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_NEIGHBOR_ALLTOALLW_INIT = ompi_neighbor_alltoallw_init_f
+#pragma weak pmpi_neighbor_alltoallw_init = ompi_neighbor_alltoallw_init_f
+#pragma weak pmpi_neighbor_alltoallw_init_ = ompi_neighbor_alltoallw_init_f
+#pragma weak pmpi_neighbor_alltoallw_init__ = ompi_neighbor_alltoallw_init_f
+
+#pragma weak PMPI_Neighbor_alltoallw_init_f = ompi_neighbor_alltoallw_init_f
+#pragma weak PMPI_Neighbor_alltoallw_init_f08 = ompi_neighbor_alltoallw_init_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_NEIGHBOR_ALLTOALLW_INIT,
+                            pmpi_neighbor_alltoallw_init,
+                            pmpi_neighbor_alltoallw_init_,
+                            pmpi_neighbor_alltoallw_init__,
+                            pompi_neighbor_alltoallw_init_f,
+                            (char *sendbuf, MPI_Fint *sendcounts, MPI_Aint *sdispls, MPI_Fint *sendtypes, char *recvbuf, MPI_Fint *recvcounts, MPI_Aint *rdispls, MPI_Fint *recvtypes, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, sendcounts, sdispls, sendtypes, recvbuf, recvcounts, rdispls, recvtypes, comm, info, request, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_NEIGHBOR_ALLTOALLW_INIT = ompi_neighbor_alltoallw_init_f
+#pragma weak mpi_neighbor_alltoallw_init = ompi_neighbor_alltoallw_init_f
+#pragma weak mpi_neighbor_alltoallw_init_ = ompi_neighbor_alltoallw_init_f
+#pragma weak mpi_neighbor_alltoallw_init__ = ompi_neighbor_alltoallw_init_f
+
+#pragma weak MPI_Neighbor_alltoallw_init_f = ompi_neighbor_alltoallw_init_f
+#pragma weak MPI_Neighbor_alltoallw_init_f08 = ompi_neighbor_alltoallw_init_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_NEIGHBOR_ALLTOALLW_INIT,
+                            mpi_neighbor_alltoallw_init,
+                            mpi_neighbor_alltoallw_init_,
+                            mpi_neighbor_alltoallw_init__,
+                            ompi_neighbor_alltoallw_init_f,
+                            (char *sendbuf, MPI_Fint *sendcounts, MPI_Aint *sdispls, MPI_Fint *sendtypes, char *recvbuf, MPI_Fint *recvcounts, MPI_Aint *rdispls, MPI_Fint *recvtypes, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, sendcounts, sdispls, sendtypes, recvbuf, recvcounts, rdispls, recvtypes, comm, info, request, ierr) )
+#else
+#define ompi_neighbor_alltoallw_init_f pompi_neighbor_alltoallw_init_f
+#endif
+#endif
+
+
+void ompi_neighbor_alltoallw_init_f(char *sendbuf, MPI_Fint *sendcounts,
+                                    MPI_Aint *sdispls, MPI_Fint *sendtypes,
+                                    char *recvbuf, MPI_Fint *recvcounts,
+                                    MPI_Aint *rdispls, MPI_Fint *recvtypes,
+                                    MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
+{
+    MPI_Comm c_comm;
+    MPI_Datatype *c_sendtypes, *c_recvtypes;
+    MPI_Info c_info;
+    MPI_Request c_request;
+    int size, idx = 0, c_ierr;
+    OMPI_ARRAY_NAME_DECL(sendcounts);
+    OMPI_ARRAY_NAME_DECL(recvcounts);
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_info = PMPI_Info_f2c(*info);
+    PMPI_Comm_size(c_comm, &size);
+
+    c_sendtypes = (MPI_Datatype *) malloc(2* size * sizeof(MPI_Datatype));
+    c_recvtypes = c_sendtypes + size;
+
+    OMPI_ARRAY_FINT_2_INT(sendcounts, size);
+    OMPI_ARRAY_FINT_2_INT(recvcounts, size);
+
+    while (size > 0) {
+        c_sendtypes[size - 1] = PMPI_Type_f2c(sendtypes[size - 1]);
+        c_recvtypes[size - 1] = PMPI_Type_f2c(recvtypes[size - 1]);
+        --size;
+    }
+
+    /* Neighbor_alltoallw_init does not support MPI_IN_PLACE */
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = PMPI_Neighbor_alltoallw_init(sendbuf,
+                                          OMPI_ARRAY_NAME_CONVERT(sendcounts),
+                                          sdispls,
+                                          c_sendtypes,
+                                          recvbuf,
+                                          OMPI_ARRAY_NAME_CONVERT(recvcounts),
+                                          rdispls,
+                                          c_recvtypes, c_comm, c_info, &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_request);
+        ompi_coll_base_nbc_request_t* nb_request = (ompi_coll_base_nbc_request_t*)c_request;
+        nb_request->data.release_arrays[idx++] = c_sendtypes;
+        if (sendcounts != OMPI_ARRAY_NAME_CONVERT(sendcounts)) {
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(sendcounts);
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(recvcounts);
+        }
+        nb_request->data.release_arrays[idx]   = NULL;
+    } else {
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(sendcounts);
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(recvcounts);
+        free(c_sendtypes);
+    }
+}
diff --git a/ompi/mpi/fortran/mpif-h/parrived_f.c b/ompi/mpi/fortran/mpif-h/parrived_f.c
new file mode 100644
index 00000000000..2dda6cbd4c4
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/parrived_f.c
@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_PARRIVED = ompi_parrived_f
+#pragma weak pmpi_parrived = ompi_parrived_f
+#pragma weak pmpi_parrived_ = ompi_parrived_f
+#pragma weak pmpi_parrived__ = ompi_parrived_f
+
+#pragma weak PMPI_Parrived_f = ompi_parrived_f
+#pragma weak PMPI_Parrived_f08 = ompi_parrived_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_PARRIVED,
+                           pmpi_parrived,
+                           pmpi_parrived_,
+                           pmpi_parrived__,
+                           pompi_parrived_f,
+                           (MPI_Fint *request, MPI_Fint *partition, MPI_Fint *flag, MPI_Fint *ierr),
+                           (request, partition, flag, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_PARRIVED = ompi_parrived_f
+#pragma weak mpi_parrived = ompi_parrived_f
+#pragma weak mpi_parrived_ = ompi_parrived_f
+#pragma weak mpi_parrived__ = ompi_parrived_f
+
+#pragma weak MPI_Parrived_f = ompi_parrived_f
+#pragma weak MPI_Parrived_f08 = ompi_parrived_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_PARRIVED,
+                           mpi_parrived,
+                           mpi_parrived_,
+                           mpi_parrived__,
+                           ompi_parrived_f,
+                           (MPI_Fint *request, MPI_Fint *partition, MPI_Fint *flag, MPI_Fint *ierr),
+                           (request, partition, flag, ierr) )
+#else
+#define ompi_parrived_f pompi_parrived_f
+#endif
+#endif
+
+
+void ompi_parrived_f(MPI_Fint *request, MPI_Fint *partition, ompi_fortran_logical_t *flag, MPI_Fint *ierr)
+{
+    int c_ierr;
+    MPI_Request c_req = PMPI_Request_f2c(*request);
+    OMPI_LOGICAL_NAME_DECL(flag);
+
+    c_ierr = PMPI_Parrived(c_req, OMPI_FINT_2_INT(*partition), OMPI_LOGICAL_SINGLE_NAME_CONVERT(flag));
+
+    OMPI_SINGLE_INT_2_LOGICAL(flag);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr && *flag) {
+        *request = OMPI_INT_2_FINT(c_req->req_f_to_c_index);
+    }
+}
diff --git a/ompi/mpi/fortran/mpif-h/pready_f.c b/ompi/mpi/fortran/mpif-h/pready_f.c
new file mode 100644
index 00000000000..e4fa7c81c5f
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/pready_f.c
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_PREADY = ompi_pready_f
+#pragma weak pmpi_pready = ompi_pready_f
+#pragma weak pmpi_pready_ = ompi_pready_f
+#pragma weak pmpi_pready__ = ompi_pready_f
+
+#pragma weak PMPI_Pready_f = ompi_pready_f
+#pragma weak PMPI_Pready_f08 = ompi_pready_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_PREADY,
+                           pmpi_pready,
+                           pmpi_pready_,
+                           pmpi_pready__,
+                           pompi_pready_f,
+                           (MPI_Fint *partition, MPI_Fint *request, MPI_Fint *ierr),
+                           (partition, request, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_PREADY = ompi_pready_f
+#pragma weak mpi_pready = ompi_pready_f
+#pragma weak mpi_pready_ = ompi_pready_f
+#pragma weak mpi_pready__ = ompi_pready_f
+
+#pragma weak MPI_Pready_f = ompi_pready_f
+#pragma weak MPI_Pready_f08 = ompi_pready_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_PREADY,
+                           mpi_pready,
+                           mpi_pready_,
+                           mpi_pready__,
+                           ompi_pready_f,
+                           (MPI_Fint *partition, MPI_Fint *request, MPI_Fint *ierr),
+                           (partition, request, ierr) )
+#else
+#define ompi_pready_f pompi_pready_f
+#endif
+#endif
+
+
+void ompi_pready_f(MPI_Fint *partition, MPI_Fint *request, MPI_Fint *ierr)
+{
+   int c_ierr;
+   MPI_Request c_req = PMPI_Request_f2c(*request);
+
+   c_ierr = PMPI_Pready(OMPI_FINT_2_INT(*partition), c_req);
+   if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+
+   if (MPI_SUCCESS == c_ierr) 
+   {
+       *request = PMPI_Request_c2f(c_req);
+   }
+}
diff --git a/ompi/mpi/fortran/mpif-h/pready_list_f.c b/ompi/mpi/fortran/mpif-h/pready_list_f.c
new file mode 100644
index 00000000000..fdbf85d6d58
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/pready_list_f.c
@@ -0,0 +1,89 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_PREADY_LIST = ompi_pready_list_f
+#pragma weak pmpi_pready_list = ompi_pready_list_f
+#pragma weak pmpi_pready_list_ = ompi_pready_list_f
+#pragma weak pmpi_pready_list__ = ompi_pready_list_f
+
+#pragma weak PMPI_Pready_list_f = ompi_pready_list_f
+#pragma weak PMPI_Pready_list_f08 = ompi_pready_list_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_PREADY_LIST,
+                           pmpi_pready_list,
+                           pmpi_pready_list_,
+                           pmpi_pready_list__,
+                           pompi_pready_list_f,
+                           (MPI_Fint *length, MPI_Fint *partitions, MPI_Fint *request, MPI_Fint *ierr),
+                           (length, partitions, request, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_PREADY_LIST = ompi_pready_list_f
+#pragma weak mpi_pready_list = ompi_pready_list_f
+#pragma weak mpi_pready_list_ = ompi_pready_list_f
+#pragma weak mpi_pready_list__ = ompi_pready_list_f
+
+#pragma weak MPI_Pready_list_f = ompi_pready_list_f
+#pragma weak MPI_Pready_list_f08 = ompi_pready_list_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_PREADY_LIST,
+                           mpi_pready_list,
+                           mpi_pready_list_,
+                           mpi_pready_list__,
+                           ompi_pready_list_f,
+                           (MPI_Fint *length, MPI_Fint *partitions, MPI_Fint *request, MPI_Fint *ierr),
+                           (length, partitions, request, ierr) )
+#else
+#define ompi_pready_list_f pompi_pready_list_f
+#endif
+#endif
+
+
+void ompi_pready_list_f(MPI_Fint *length, MPI_Fint *partitions, MPI_Fint *request, MPI_Fint *ierr)
+{
+   int c_ierr;
+   MPI_Request c_req = PMPI_Request_f2c(*request);
+   int *c_partitions = (int*)malloc(sizeof(int)*OMPI_FINT_2_INT(*length));
+   for(int i = 0; i < OMPI_FINT_2_INT(*length); i++)
+   {
+       c_partitions[i] = OMPI_FINT_2_INT(partitions[i]);
+   }
+
+   c_ierr = PMPI_Pready_list(OMPI_FINT_2_INT(*length), c_partitions, c_req);
+  
+   free(c_partitions);
+
+   if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+   if (MPI_SUCCESS == c_ierr) {
+      *request = PMPI_Request_c2f(c_req);
+   }
+}
diff --git a/ompi/mpi/fortran/mpif-h/pready_range_f.c b/ompi/mpi/fortran/mpif-h/pready_range_f.c
new file mode 100644
index 00000000000..958a260dcba
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/pready_range_f.c
@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_PREADY_RANGE = ompi_pready_range_f
+#pragma weak pmpi_pready_range = ompi_pready_range_f
+#pragma weak pmpi_pready_range_ = ompi_pready_range_f
+#pragma weak pmpi_pready_range__ = ompi_pready_range_f
+
+#pragma weak PMPI_Pready_range_f = ompi_pready_range_f
+#pragma weak PMPI_Pready_range_f08 = ompi_pready_range_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_PREADY_RANGE,
+                           pmpi_pready_range,
+                           pmpi_pready_range_,
+                           pmpi_pready_range__,
+                           pompi_pready_range_f,
+                           (MPI_Fint *partition_low, MPI_Fint *partition_high, MPI_Fint *request, MPI_Fint *ierr),
+                           (partition_low, partition_high, request, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_PREADY_RANGE = ompi_pready_range_f
+#pragma weak mpi_pready_range = ompi_pready_range_f
+#pragma weak mpi_pready_range_ = ompi_pready_range_f
+#pragma weak mpi_pready_range__ = ompi_pready_range_f
+
+#pragma weak MPI_Pready_range_f = ompi_pready_range_f
+#pragma weak MPI_Pready_range_f08 = ompi_pready_range_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_PREADY_RANGE,
+                           mpi_pready_range,
+                           mpi_pready_range_,
+                           mpi_pready_range__,
+                           ompi_pready_range_f,
+                           (MPI_Fint *partition_low, MPI_Fint *partition_high, MPI_Fint *request, MPI_Fint *ierr),
+                           (partition_low, partition_high, request, ierr) )
+#else
+#define ompi_pready_range_f pompi_pready_range_f
+#endif
+#endif
+
+
+void ompi_pready_range_f(MPI_Fint *partition_low, MPI_Fint *partition_high, MPI_Fint *request, MPI_Fint *ierr)
+{
+   int c_ierr;
+   MPI_Request c_req = PMPI_Request_f2c(*request);
+
+   c_ierr = PMPI_Pready_range(OMPI_FINT_2_INT(*partition_low), OMPI_FINT_2_INT(*partition_high), c_req);
+   if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+   if (MPI_SUCCESS == c_ierr) {
+      *request = PMPI_Request_c2f(c_req);
+   }
+}
diff --git a/ompi/mpi/fortran/mpif-h/precv_init_f.c b/ompi/mpi/fortran/mpif-h/precv_init_f.c
new file mode 100644
index 00000000000..2fe64d55c55
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/precv_init_f.c
@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2021      Bull S.A.S. All rights reserved.
+ * Copyright (c) 2023      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_PRECV_INIT = ompi_precv_init_f
+#pragma weak pmpi_precv_init = ompi_precv_init_f
+#pragma weak pmpi_precv_init_ = ompi_precv_init_f
+#pragma weak pmpi_precv_init__ = ompi_precv_init_f
+
+#pragma weak PMPI_Precv_init_f = ompi_precv_init_f
+#pragma weak PMPI_Precv_init_f08 = ompi_precv_init_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_PRECV_INIT,
+                           pmpi_precv_init,
+                           pmpi_precv_init_,
+                           pmpi_precv_init__,
+                           pompi_precv_init_f,
+                           (char *buf, MPI_Fint *partitions, MPI_Count *count, MPI_Fint *datatype, MPI_Fint *dest, MPI_Fint *tag, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                           (buf, partitions, count, datatype, dest, tag, comm, info, request, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_PRECV_INIT = ompi_precv_init_f
+#pragma weak mpi_precv_init = ompi_precv_init_f
+#pragma weak mpi_precv_init_ = ompi_precv_init_f
+#pragma weak mpi_precv_init__ = ompi_precv_init_f
+
+#pragma weak MPI_Precv_init_f = ompi_precv_init_f
+#pragma weak MPI_Precv_init_f08 = ompi_precv_init_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_PRECV_INIT,
+                           mpi_precv_init,
+                           mpi_precv_init_,
+                           mpi_precv_init__,
+                           ompi_precv_init_f,
+                           (char *buf, MPI_Fint *partitions, MPI_Count *count, MPI_Fint *datatype, MPI_Fint *dest, MPI_Fint *tag, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                           (buf, partitions, count, datatype, dest, tag, comm, info, request, ierr) )
+#else
+#define ompi_precv_init_f pompi_precv_init_f
+#endif
+#endif
+
+
+void ompi_precv_init_f(char *buf, MPI_Fint *partitions, MPI_Count *count, MPI_Fint *datatype, MPI_Fint *dest, MPI_Fint *tag, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
+{
+   int c_ierr;
+   MPI_Info c_info;
+   MPI_Datatype c_type = PMPI_Type_f2c(*datatype);
+   MPI_Request c_req;
+   MPI_Comm c_comm;
+
+   c_info = PMPI_Info_f2c(*info);
+   c_comm = PMPI_Comm_f2c (*comm);
+
+   c_ierr = PMPI_Precv_init(OMPI_F2C_BOTTOM(buf), 
+                      OMPI_FINT_2_INT(*partitions),
+                      *count,
+                      c_type, OMPI_FINT_2_INT(*dest),
+                      OMPI_FINT_2_INT(*tag),
+                      c_comm, c_info, &c_req);
+   if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+   if (MPI_SUCCESS == c_ierr) {
+      *request = PMPI_Request_c2f(c_req);
+   }
+}
diff --git a/ompi/mpi/fortran/mpif-h/profile/Makefile.am b/ompi/mpi/fortran/mpif-h/profile/Makefile.am
index 9d183297df8..6957518a71c 100644
--- a/ompi/mpi/fortran/mpif-h/profile/Makefile.am
+++ b/ompi/mpi/fortran/mpif-h/profile/Makefile.am
@@ -10,14 +10,16 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2009-2015 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2009-2021 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2011-2013 Inria.  All rights reserved.
 # Copyright (c) 2011-2013 Universite Bordeaux 1
 # Copyright (c) 2013-2014 Los Alamos National Security, LLC. All rights
 #                         reserved.
-# Copyright (c) 2015-2020 Research Organization for Information Science
+# Copyright (c) 2015-2021 Research Organization for Information Science
 #                         and Technology (RIST).  All rights reserved.
 # Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+# Copyright (c) 2019-2022 Triad National Security, LLC. All rights
+#                         reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -49,17 +51,25 @@ linked_files = \
         paint_add_f.c \
         paint_diff_f.c \
         pallgather_f.c \
+        pallgather_init_f.c \
         pallgatherv_f.c \
+        pallgatherv_init_f.c \
         palloc_mem_f.c \
         pallreduce_f.c \
+        pallreduce_init_f.c \
         palltoall_f.c \
+        palltoall_init_f.c \
         palltoallv_f.c \
+        palltoallv_init_f.c \
         palltoallw_f.c \
+        palltoallw_init_f.c \
         pattr_delete_f.c \
         pattr_get_f.c \
         pattr_put_f.c \
         pbarrier_f.c \
+        pbarrier_init_f.c \
         pbcast_f.c \
+        pbcast_init_f.c \
         pbsend_f.c \
         pbsend_init_f.c \
         pbuffer_attach_f.c \
@@ -80,6 +90,7 @@ linked_files = \
         pcomm_connect_f.c \
         pcomm_create_errhandler_f.c \
         pcomm_create_f.c \
+        pcomm_create_from_group_f.c \
         pcomm_create_group_f.c \
         pcomm_create_keyval_f.c \
         pcomm_delete_attr_f.c \
@@ -87,6 +98,7 @@ linked_files = \
         pcomm_dup_f.c \
         pcomm_dup_with_info_f.c \
         pcomm_idup_f.c \
+        pcomm_idup_with_info_f.c \
         pcomm_free_f.c \
         pcomm_free_keyval_f.c \
         pcomm_get_attr_f.c \
@@ -118,6 +130,7 @@ linked_files = \
         perror_class_f.c \
         perror_string_f.c \
         pexscan_f.c \
+        pexscan_init_f.c \
         pf_sync_reg_f.c \
         pfile_call_errhandler_f.c \
         pfile_close_f.c \
@@ -182,7 +195,9 @@ linked_files = \
         pfinalize_f.c \
         pfree_mem_f.c \
         pgather_f.c \
+        pgather_init_f.c \
         pgatherv_f.c \
+        pgatherv_init_f.c \
         pget_address_f.c \
         pget_count_f.c \
         pget_elements_f.c \
@@ -202,6 +217,7 @@ linked_files = \
         pgroup_difference_f.c \
         pgroup_excl_f.c \
         pgroup_free_f.c \
+        pgroup_from_session_pset_f.c \
         pgroup_incl_f.c \
         pgroup_intersection_f.c \
         pgroup_range_excl_f.c \
@@ -230,18 +246,21 @@ linked_files = \
         pineighbor_alltoallv_f.c \
         pineighbor_alltoallw_f.c \
         pinfo_create_f.c \
+        pinfo_create_env_f.c \
         pinfo_delete_f.c \
         pinfo_dup_f.c \
         pinfo_free_f.c \
         pinfo_get_f.c \
         pinfo_get_nkeys_f.c \
         pinfo_get_nthkey_f.c \
+        pinfo_get_string_f.c \
         pinfo_get_valuelen_f.c \
         pinfo_set_f.c \
         pinit_f.c \
         pinitialized_f.c \
         pinit_thread_f.c \
         pintercomm_create_f.c \
+        pintercomm_create_from_groups_f.c \
         pintercomm_merge_f.c \
         piprobe_f.c \
         pirecv_f.c \
@@ -253,6 +272,8 @@ linked_files = \
         piscatter_f.c \
         piscatterv_f.c \
         pisend_f.c \
+        pisendrecv_f.c \
+        pisendrecv_replace_f.c \
         pissend_f.c \
         pis_thread_main_f.c \
         pkeyval_create_f.c \
@@ -261,10 +282,15 @@ linked_files = \
         pmprobe_f.c \
         pmrecv_f.c \
         pneighbor_allgather_f.c \
+        pneighbor_allgather_init_f.c \
         pneighbor_allgatherv_f.c \
+        pneighbor_allgatherv_init_f.c \
         pneighbor_alltoall_f.c \
+        pneighbor_alltoall_init_f.c \
         pneighbor_alltoallv_f.c \
+        pneighbor_alltoallv_init_f.c \
         pneighbor_alltoallw_f.c \
+        pneighbor_alltoallw_init_f.c \
         pop_commutative_f.c \
         pop_create_f.c \
         popen_port_f.c \
@@ -273,27 +299,49 @@ linked_files = \
         ppack_external_size_f.c \
         ppack_f.c \
         ppack_size_f.c \
+        pparrived_f.c \
         ppcontrol_f.c \
+        ppready_f.c \
+        ppready_list_f.c \
+        ppready_range_f.c \
+        pprecv_init_f.c \
         pprobe_f.c \
+        ppsend_init_f.c \
         ppublish_name_f.c \
         pquery_thread_f.c \
         precv_f.c \
         precv_init_f.c \
         preduce_f.c \
+        preduce_init_f.c \
         preduce_local_f.c \
         preduce_scatter_f.c \
+        preduce_scatter_init_f.c \
         preduce_scatter_block_f.c \
+        preduce_scatter_block_init_f.c \
         prequest_free_f.c \
         prequest_get_status_f.c \
         prsend_f.c \
         prsend_init_f.c \
         pscan_f.c \
+        pscan_init_f.c \
         pscatter_f.c \
+        pscatter_init_f.c \
         pscatterv_f.c \
+        pscatterv_init_f.c \
         psend_f.c \
         psend_init_f.c \
         psendrecv_f.c \
         psendrecv_replace_f.c \
+        psession_call_errhandler_f.c \
+        psession_create_errhandler_f.c \
+        psession_get_errhandler_f.c \
+        psession_get_info_f.c \
+        psession_get_nth_pset_f.c \
+        psession_get_num_psets_f.c \
+        psession_get_pset_info_f.c \
+        psession_init_f.c \
+        psession_finalize_f.c \
+        psession_set_errhandler_f.c \
         pssend_f.c \
         pssend_init_f.c \
         pstartall_f.c \
@@ -349,8 +397,8 @@ linked_files = \
         pwaitany_f.c \
         pwait_f.c \
         pwaitsome_f.c \
-	pwtick_f.c \
-	pwtime_f.c \
+        pwtick_f.c \
+        pwtime_f.c \
         paccumulate_f.c \
         praccumulate_f.c \
         pregister_datarep_f.c \
@@ -400,7 +448,10 @@ linked_files = \
         pwin_flush_local_f.c \
         pwin_flush_local_all_f.c
 
-if OMPI_ENABLE_MPI1_COMPAT
+# The following functions were removed from the MPI standard, but are
+# retained for ABI compliance reasons.  They are listed independently
+# of the other MPI functions in case we one day change behavior around
+# ABI compliance.
 linked_files += \
         paddress_f.c \
         perrhandler_create_f.c \
@@ -412,7 +463,6 @@ linked_files += \
         ptype_lb_f.c \
         ptype_struct_f.c \
         ptype_ub_f.c
-endif
 
 #
 # Sym link in the sources from the real MPI directory
@@ -446,6 +496,7 @@ psizeof_f.f90:
 	    --maxrank=$(OMPI_FORTRAN_MAX_ARRAY_RANK) \
 	    --generate=$(OMPI_FORTRAN_BUILD_SIZEOF) \
 	    --real2=$(OMPI_HAVE_FORTRAN_REAL2) \
+	    --iso_real16=$(OMPI_FORTRAN_HAVE_ISO_FORTRAN_ENV_REAL16) \
 	    --real16=$(OMPI_HAVE_FORTRAN_REAL16) \
 	    --complex4=$(OMPI_HAVE_FORTRAN_COMPLEX4) \
 	    --complex32=$(OMPI_HAVE_FORTRAN_COMPLEX32)
@@ -465,11 +516,3 @@ endif
 # These files were created by targets above
 
 MAINTAINERCLEANFILES = $(nodist_libmpi_mpifh_pmpi_la_SOURCES)
-
-# Don't want these targets in here
-
-tags-recursive:
-tags:
-TAGS:
-GTAGS:
-ID:
diff --git a/ompi/mpi/fortran/mpif-h/prototypes_mpi.h b/ompi/mpi/fortran/mpif-h/prototypes_mpi.h
index bd560a9dc26..6573f3cbd15 100644
--- a/ompi/mpi/fortran/mpif-h/prototypes_mpi.h
+++ b/ompi/mpi/fortran/mpif-h/prototypes_mpi.h
@@ -1,4 +1,4 @@
-/*
+/* clang-format off
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
@@ -14,8 +14,11 @@
  * Copyright (c) 2011-2013 Universite Bordeaux 1
  * Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights
  *                         reserved.
- * Copyright (c) 2016-2020 Research Organization for Information Science
+ * Copyright (c) 2016-2023 Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2019-2023 Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2021      Bull S.A.S. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -90,19 +93,27 @@ PN2(void, MPI_Address, mpi_address, MPI_ADDRESS, (char *location, MPI_Fint *addr
 PN2(MPI_Aint, MPI_Aint_add, mpi_aint_add, MPI_AINT_ADD, (MPI_Aint *base, MPI_Aint *diff));
 PN2(MPI_Aint, MPI_Aint_diff, mpi_aint_diff, MPI_AINT_DIFF, (MPI_Aint *addr1, MPI_Aint *addr2));
 PN2(void, MPI_Allgather, mpi_allgather, MPI_ALLGATHER, (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *ierr));
+PN2(void, MPI_Allgather_init, mpi_allgather_init, MPI_ALLGATHER_INIT, (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Allgatherv, mpi_allgatherv, MPI_ALLGATHERV, (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *displs, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *ierr));
+PN2(void, MPI_Allgatherv_init, mpi_allgatherv_init, MPI_ALLGATHERV_INIT, (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *displs, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Alloc_mem, mpi_alloc_mem, MPI_ALLOC_MEM, (MPI_Aint *size, MPI_Fint *info, char *baseptr, MPI_Fint *ierr));
 /* Extra Alloc_mem prototype for the _cptr variant added in MPI-3.0 errata */
 PN2(void, MPI_Alloc_mem_cptr, mpi_alloc_mem_cptr, MPI_ALLOC_MEM_CPTR, (MPI_Aint *size, MPI_Fint *info, char *baseptr, MPI_Fint *ierr));
 PN2(void, MPI_Allreduce, mpi_allreduce, MPI_ALLREDUCE, (char *sendbuf, char *recvbuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *ierr));
+PN2(void, MPI_Allreduce_init, mpi_allreduce_init, MPI_ALLREDUCE_INIT, (char *sendbuf, char *recvbuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Alltoall, mpi_alltoall, MPI_ALLTOALL, (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *ierr));
+PN2(void, MPI_Alltoall_init, mpi_alltoall_init, MPI_ALLTOALL_INIT, (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Alltoallv, mpi_alltoallv, MPI_ALLTOALLV, (char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *sdispls, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *rdispls, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *ierr));
+PN2(void, MPI_Alltoallv_init, mpi_alltoallv_init, MPI_ALLTOALLV_INIT, (char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *sdispls, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *rdispls, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Alltoallw, mpi_alltoallw, MPI_ALLTOALLW, (char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *sdispls, MPI_Fint *sendtypes, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *rdispls, MPI_Fint *recvtypes, MPI_Fint *comm, MPI_Fint *ierr));
+PN2(void, MPI_Alltoallw_init, mpi_alltoallw_init, MPI_ALLTOALLW_INIT, (char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *sdispls, MPI_Fint *sendtypes, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *rdispls, MPI_Fint *recvtypes, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Attr_delete, mpi_attr_delete, MPI_ATTR_DELETE, (MPI_Fint *comm, MPI_Fint *keyval, MPI_Fint *ierr));
 PN2(void, MPI_Attr_get, mpi_attr_get, MPI_ATTR_GET, (MPI_Fint *comm, MPI_Fint *keyval, MPI_Fint *attribute_val, ompi_fortran_logical_t *flag, MPI_Fint *ierr));
 PN2(void, MPI_Attr_put, mpi_attr_put, MPI_ATTR_PUT, (MPI_Fint *comm, MPI_Fint *keyval, MPI_Fint *attribute_val, MPI_Fint *ierr));
 PN2(void, MPI_Barrier, mpi_barrier, MPI_BARRIER, (MPI_Fint *comm, MPI_Fint *ierr));
+PN2(void, MPI_Barrier_init, mpi_barrier_init, MPI_BARRIER_INIT, (MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Bcast, mpi_bcast, MPI_BCAST, (char *buffer, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *ierr));
+PN2(void, MPI_Bcast_init, mpi_bcast_init, MPI_BCAST_INIT, (char *buffer, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Bsend, mpi_bsend, MPI_BSEND, (char *buf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *dest, MPI_Fint *tag, MPI_Fint *comm, MPI_Fint *ierr));
 PN2(void, MPI_Bsend_init, mpi_bsend_init, MPI_BSEND_INIT, (char *buf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *dest, MPI_Fint *tag, MPI_Fint *comm, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Buffer_attach, mpi_buffer_attach, MPI_BUFFER_ATTACH, (char *buffer, MPI_Fint *size, MPI_Fint *ierr));
@@ -122,14 +133,16 @@ PN2(void, MPI_Comm_call_errhandler, mpi_comm_call_errhandler, MPI_COMM_CALL_ERRH
 PN2(void, MPI_Comm_compare, mpi_comm_compare, MPI_COMM_COMPARE, (MPI_Fint *comm1, MPI_Fint *comm2, MPI_Fint *result, MPI_Fint *ierr));
 PN2(void, MPI_Comm_connect, mpi_comm_connect, MPI_COMM_CONNECT, (char *port_name, MPI_Fint *info, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *newcomm, MPI_Fint *ierr, int port_name_len));
 PN2(void, MPI_Comm_create_errhandler, mpi_comm_create_errhandler, MPI_COMM_CREATE_ERRHANDLER, (ompi_errhandler_fortran_handler_fn_t* function, MPI_Fint *errhandler, MPI_Fint *ierr));
-PN2(void, MPI_Comm_create_keyval, mpi_comm_create_keyval, MPI_COMM_CREATE_KEYVAL, (ompi_aint_copy_attr_function* comm_copy_attr_fn, ompi_aint_delete_attr_function* comm_delete_attr_fn, MPI_Fint *comm_keyval, MPI_Aint *extra_state, MPI_Fint *ierr));
+PN2(void, MPI_Comm_create_keyval, mpi_comm_create_keyval, MPI_COMM_CREATE_KEYVAL, (ompi_aint_copy_attr_function comm_copy_attr_fn, ompi_aint_delete_attr_function comm_delete_attr_fn, MPI_Fint *comm_keyval, MPI_Aint *extra_state, MPI_Fint *ierr));
 PN2(void, MPI_Comm_create, mpi_comm_create, MPI_COMM_CREATE, (MPI_Fint *comm, MPI_Fint *group, MPI_Fint *newcomm, MPI_Fint *ierr));
+PN2(void, MPI_Comm_create_from_group, mpi_comm_create_from_group, MPI_COMM_CREATE_FROM_GROUP, (MPI_Fint *group, char *stringtag, MPI_Fint *info, MPI_Fint *errhandler, MPI_Fint *newcomm, MPI_Fint *ierr, int name_len));
 PN2(void, MPI_Comm_create_group, mpi_comm_create_group, MPI_COMM_CREATE_GROUP, (MPI_Fint *comm, MPI_Fint *group, MPI_Fint *tag, MPI_Fint *newcomm, MPI_Fint *ierr));
 PN2(void, MPI_Comm_delete_attr, mpi_comm_delete_attr, MPI_COMM_DELETE_ATTR, (MPI_Fint *comm, MPI_Fint *comm_keyval, MPI_Fint *ierr));
 PN2(void, MPI_Comm_disconnect, mpi_comm_disconnect, MPI_COMM_DISCONNECT, (MPI_Fint *comm, MPI_Fint *ierr));
 PN2(void, MPI_Comm_dup, mpi_comm_dup, MPI_COMM_DUP, (MPI_Fint *comm, MPI_Fint *newcomm, MPI_Fint *ierr));
 PN2(void, MPI_Comm_dup_with_info, mpi_comm_dup_with_info, MPI_COMM_DUP_WITH_INFO, (MPI_Fint *comm, MPI_Fint *info, MPI_Fint *newcomm, MPI_Fint *ierr));
 PN2(void, MPI_Comm_idup, mpi_comm_idup, MPI_COMM_IDUP, (MPI_Fint *comm, MPI_Fint *newcomm, MPI_Fint *request, MPI_Fint *ierr));
+PN2(void, MPI_Comm_idup_with_info, mpi_comm_idup_with_info, MPI_COMM_IDUP_WITH_INFO, (MPI_Fint *comm, MPI_Fint *info, MPI_Fint *newcomm, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Comm_free_keyval, mpi_comm_free_keyval, MPI_COMM_FREE_KEYVAL, (MPI_Fint *comm_keyval, MPI_Fint *ierr));
 PN2(void, MPI_Comm_free, mpi_comm_free, MPI_COMM_FREE, (MPI_Fint *comm, MPI_Fint *ierr));
 PN2(void, MPI_Comm_get_attr, mpi_comm_get_attr, MPI_COMM_GET_ATTR, (MPI_Fint *comm, MPI_Fint *comm_keyval, MPI_Aint *attribute_val, ompi_fortran_logical_t *flag, MPI_Fint *ierr));
@@ -165,6 +178,7 @@ PN2(void, MPI_Errhandler_set, mpi_errhandler_set, MPI_ERRHANDLER_SET, (MPI_Fint
 PN2(void, MPI_Error_class, mpi_error_class, MPI_ERROR_CLASS, (MPI_Fint *errorcode, MPI_Fint *errorclass, MPI_Fint *ierr));
 PN2(void, MPI_Error_string, mpi_error_string, MPI_ERROR_STRING, (MPI_Fint *errorcode, char *string, MPI_Fint *resultlen, MPI_Fint *ierr, int string_len));
 PN2(void, MPI_Exscan, mpi_exscan, MPI_EXSCAN, (char *sendbuf, char *recvbuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *ierr));
+PN2(void, MPI_Exscan_init, mpi_exscan_init, MPI_EXSCAN_INIT, (char *sendbuf, char *recvbuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_F_sync_reg, mpi_f_sync_reg, MPI_F_SYNC_REG, (char *buf));
 PN2(void, MPI_Fetch_and_op, mpi_fetch_and_op, MPI_FETCH_AND_OP, (char *origin_addr, char *result_addr, MPI_Fint *datatype, MPI_Fint *target_rank, MPI_Aint *target_disp, MPI_Fint *op, MPI_Fint *win, MPI_Fint *ierr));
 PN2(void, MPI_File_call_errhandler, mpi_file_call_errhandler, MPI_FILE_CALL_ERRHANDLER, (MPI_Fint *fh, MPI_Fint *errorcode, MPI_Fint *ierr));
@@ -230,7 +244,9 @@ PN2(void, MPI_Finalize, mpi_finalize, MPI_FINALIZE, (MPI_Fint *ierr));
 PN2(void, MPI_Finalized, mpi_finalized, MPI_FINALIZED, (ompi_fortran_logical_t *flag, MPI_Fint *ierr));
 PN2(void, MPI_Free_mem, mpi_free_mem, MPI_FREE_MEM, (char *base, MPI_Fint *ierr));
 PN2(void, MPI_Gather, mpi_gather, MPI_GATHER, (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *ierr));
+PN2(void, MPI_Gather_init, mpi_gather_init, MPI_GATHER_INIT, (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Gatherv, mpi_gatherv, MPI_GATHERV, (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *displs, MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *ierr));
+PN2(void, MPI_Gatherv_init, mpi_gatherv_init, MPI_GATHERV_INIT, (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *displs, MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Get_accumulate, mpi_get_accumulate, MPI_GET_ACCUMULATE, (char *origin_addr, MPI_Fint *origin_count, MPI_Fint *origin_datatype, char *result_addr, MPI_Fint *result_count, MPI_Fint *result_datatype, MPI_Fint *target_rank, MPI_Aint *target_disp, MPI_Fint *target_count, MPI_Fint *target_datatype, MPI_Fint *op, MPI_Fint *win, MPI_Fint *ierr));
 PN2(void, MPI_Get_address, mpi_get_address, MPI_GET_ADDRESS, (char *location, MPI_Aint *address, MPI_Fint *ierr));
 PN2(void, MPI_Get_count, mpi_get_count, MPI_GET_COUNT, (MPI_Fint *status, MPI_Fint *datatype, MPI_Fint *count, MPI_Fint *ierr));
@@ -247,11 +263,12 @@ PN2(void, MPI_Graph_neighbors_count, mpi_graph_neighbors_count, MPI_GRAPH_NEIGHB
 PN2(void, MPI_Graph_neighbors, mpi_graph_neighbors, MPI_GRAPH_NEIGHBORS, (MPI_Fint *comm, MPI_Fint *rank, MPI_Fint *maxneighbors, MPI_Fint *neighbors, MPI_Fint *ierr));
 PN2(void, MPI_Graphdims_get, mpi_graphdims_get, MPI_GRAPHDIMS_GET, (MPI_Fint *comm, MPI_Fint *nnodes, MPI_Fint *nedges, MPI_Fint *ierr));
 PN2(void, MPI_Grequest_complete, mpi_grequest_complete, MPI_GREQUEST_COMPLETE, (MPI_Fint *request, MPI_Fint *ierr));
-PN2(void, MPI_Grequest_start, mpi_grequest_start, MPI_GREQUEST_START, (MPI_F_Grequest_query_function* query_fn, MPI_F_Grequest_free_function* free_fn, MPI_F_Grequest_cancel_function* cancel_fn, MPI_Aint *extra_state, MPI_Fint *request, MPI_Fint *ierr));
+PN2(void, MPI_Grequest_start, mpi_grequest_start, MPI_GREQUEST_START, (MPI_F_Grequest_query_function query_fn, MPI_F_Grequest_free_function free_fn, MPI_F_Grequest_cancel_function cancel_fn, MPI_Aint *extra_state, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Group_compare, mpi_group_compare, MPI_GROUP_COMPARE, (MPI_Fint *group1, MPI_Fint *group2, MPI_Fint *result, MPI_Fint *ierr));
 PN2(void, MPI_Group_difference, mpi_group_difference, MPI_GROUP_DIFFERENCE, (MPI_Fint *group1, MPI_Fint *group2, MPI_Fint *newgroup, MPI_Fint *ierr));
 PN2(void, MPI_Group_excl, mpi_group_excl, MPI_GROUP_EXCL, (MPI_Fint *group, MPI_Fint *n, MPI_Fint *ranks, MPI_Fint *newgroup, MPI_Fint *ierr));
 PN2(void, MPI_Group_free, mpi_group_free, MPI_GROUP_FREE, (MPI_Fint *group, MPI_Fint *ierr));
+PN2(void, MPI_Group_from_session_pset, mpi_group_from_session_pset, MPI_GROUP_FROM_SESSION_PSET, (MPI_Fint *group, char *pset_name, MPI_Fint *newgroup, MPI_Fint *ierr, int name_len));
 PN2(void, MPI_Group_incl, mpi_group_incl, MPI_GROUP_INCL, (MPI_Fint *group, MPI_Fint *n, MPI_Fint *ranks, MPI_Fint *newgroup, MPI_Fint *ierr));
 PN2(void, MPI_Group_intersection, mpi_group_intersection, MPI_GROUP_INTERSECTION, (MPI_Fint *group1, MPI_Fint *group2, MPI_Fint *newgroup, MPI_Fint *ierr));
 PN2(void, MPI_Group_range_excl, mpi_group_range_excl, MPI_GROUP_RANGE_EXCL, (MPI_Fint *group, MPI_Fint *n, MPI_Fint ranges[][3], MPI_Fint *newgroup, MPI_Fint *ierr));
@@ -286,35 +303,45 @@ PN2(void, MPI_Iscan, mpi_iscan, MPI_ISCAN, (char *sendbuf, char *recvbuf, MPI_Fi
 PN2(void, MPI_Iscatter, mpi_iscatter, MPI_ISCATTER, (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Iscatterv, mpi_iscatterv, MPI_ISCATTERV, (char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *displs, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Info_create, mpi_info_create, MPI_INFO_CREATE, (MPI_Fint *info, MPI_Fint *ierr));
+PN2(void, MPI_Info_create_env, mpi_info_create_env, MPI_INFO_CREATE_ENV, (MPI_Fint *info, MPI_Fint *ierr));
 PN2(void, MPI_Info_delete, mpi_info_delete, MPI_INFO_DELETE, (MPI_Fint *info, char *key, MPI_Fint *ierr, int key_len));
 PN2(void, MPI_Info_dup, mpi_info_dup, MPI_INFO_DUP, (MPI_Fint *info, MPI_Fint *newinfo, MPI_Fint *ierr));
 PN2(void, MPI_Info_free, mpi_info_free, MPI_INFO_FREE, (MPI_Fint *info, MPI_Fint *ierr));
 PN2(void, MPI_Info_get, mpi_info_get, MPI_INFO_GET, (MPI_Fint *info, char *key, MPI_Fint *valuelen, char *value, ompi_fortran_logical_t *flag, MPI_Fint *ierr, int key_len, int value_len));
 PN2(void, MPI_Info_get_nkeys, mpi_info_get_nkeys, MPI_INFO_GET_NKEYS, (MPI_Fint *info, MPI_Fint *nkeys, MPI_Fint *ierr));
 PN2(void, MPI_Info_get_nthkey, mpi_info_get_nthkey, MPI_INFO_GET_NTHKEY, (MPI_Fint *info, MPI_Fint *n, char *key, MPI_Fint *ierr, int key_len));
+PN2(void, MPI_Info_get_string, mpi_info_get_string, MPI_INFO_GET_STRING, (MPI_Fint *info, char *key, MPI_Fint *buflen, char *value, ompi_fortran_logical_t *flag, MPI_Fint *ierr, int key_len, int value_len));
 PN2(void, MPI_Info_get_valuelen, mpi_info_get_valuelen, MPI_INFO_GET_VALUELEN, (MPI_Fint *info, char *key, MPI_Fint *valuelen, ompi_fortran_logical_t *flag, MPI_Fint *ierr, int key_len));
 PN2(void, MPI_Info_set, mpi_info_set, MPI_INFO_SET, (MPI_Fint *info, char *key, char *value, MPI_Fint *ierr, int key_len, int value_len));
 PN2(void, MPI_Init, mpi_init, MPI_INIT, (MPI_Fint *ierr));
 PN2(void, MPI_Initialized, mpi_initialized, MPI_INITIALIZED, (ompi_fortran_logical_t *flag, MPI_Fint *ierr));
 PN2(void, MPI_Init_thread, mpi_init_thread, MPI_INIT_THREAD, (MPI_Fint *required, MPI_Fint *provided, MPI_Fint *ierr));
 PN2(void, MPI_Intercomm_create, mpi_intercomm_create, MPI_INTERCOMM_CREATE, (MPI_Fint *local_comm, MPI_Fint *local_leader, MPI_Fint *bridge_comm, MPI_Fint *remote_leader, MPI_Fint *tag, MPI_Fint *newintercomm, MPI_Fint *ierr));
-PN2(void, MPI_Intercomm_merge, mpi_intercomm_merge, MPI_INTERCOMM_MERGE, (MPI_Fint *intercomm, ompi_fortran_logical_t *high, MPI_Fint *newintercomm, MPI_Fint *ierr));
+PN2(void, MPI_Intercomm_create_from_groups, mpi_intercomm_create_from_groups, MPI_INTERCOMM_CREATE_FROM_GROUPS, (MPI_Fint *local_group, MPI_Fint *local_leader, MPI_Fint *remote_group,  MPI_Fint *remote_leader, char *stringtag, MPI_Fint *info, MPI_Fint *errhandler, MPI_Fint *newintercomm, MPI_Fint *ierr, int name_len));
+PN2(void, MPI_Intercomm_merge, mpi_intercomm_merge, MPI_INTERCOMM_MERGE, (MPI_Fint *intercomm, ompi_fortran_logical_t *high, MPI_Fint *newintracomm, MPI_Fint *ierr));
 PN2(void, MPI_Iprobe, mpi_iprobe, MPI_IPROBE, (MPI_Fint *source, MPI_Fint *tag, MPI_Fint *comm, ompi_fortran_logical_t *flag, MPI_Fint *status, MPI_Fint *ierr));
 PN2(void, MPI_Irecv, mpi_irecv, MPI_IRECV, (char *buf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *source, MPI_Fint *tag, MPI_Fint *comm, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Irsend, mpi_irsend, MPI_IRSEND, (char *buf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *dest, MPI_Fint *tag, MPI_Fint *comm, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Isend, mpi_isend, MPI_ISEND, (char *buf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *dest, MPI_Fint *tag, MPI_Fint *comm, MPI_Fint *request, MPI_Fint *ierr));
+PN2(void, MPI_Isendrecv, mpi_isendrecv, MPI_ISENDRECV, (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, MPI_Fint *dest, MPI_Fint *sendtag, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *source, MPI_Fint *recvtag, MPI_Fint *comm, MPI_Fint *request, MPI_Fint *ierr));
+PN2(void, MPI_Isendrecv_replace, mpi_isendrecv_replace, MPI_ISENDRECV_REPLACE, (char *buf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *dest, MPI_Fint *sendtag, MPI_Fint *source, MPI_Fint *recvtag, MPI_Fint *comm, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Issend, mpi_issend, MPI_ISSEND, (char *buf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *dest, MPI_Fint *tag, MPI_Fint *comm, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Is_thread_main, mpi_is_thread_main, MPI_IS_THREAD_MAIN, (ompi_fortran_logical_t *flag, MPI_Fint *ierr));
-PN2(void, MPI_Keyval_create, mpi_keyval_create, MPI_KEYVAL_CREATE, (ompi_fint_copy_attr_function* copy_fn, ompi_fint_delete_attr_function* delete_fn, MPI_Fint *keyval, MPI_Fint *extra_state, MPI_Fint *ierr));
+PN2(void, MPI_Keyval_create, mpi_keyval_create, MPI_KEYVAL_CREATE, (ompi_fint_copy_attr_function copy_fn, ompi_fint_delete_attr_function delete_fn, MPI_Fint *keyval, MPI_Fint *extra_state, MPI_Fint *ierr));
 PN2(void, MPI_Keyval_free, mpi_keyval_free, MPI_KEYVAL_FREE, (MPI_Fint *keyval, MPI_Fint *ierr));
 PN2(void, MPI_Lookup_name, mpi_lookup_name, MPI_LOOKUP_NAME, (char *service_name, MPI_Fint *info, char *port_name, MPI_Fint *ierr, int service_name_len, int port_name_len));
 PN2(void, MPI_Mprobe, mpi_mprobe, MPI_MPROBE, (MPI_Fint *source, MPI_Fint *tag, MPI_Fint *comm, MPI_Fint *message, MPI_Fint *status, MPI_Fint *ierr));
 PN2(void, MPI_Mrecv, mpi_mrecv, MPI_MRECV, (char *buf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *message, MPI_Fint *status, MPI_Fint *ierr));
 PN2(void, MPI_Neighbor_allgather, mpi_neighbor_allgather, MPI_NEIGHBOR_ALLGATHER, (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *ierr));
+PN2(void, MPI_Neighbor_allgather_init, mpi_neighbor_allgather_init, MPI_NEIGHBOR_ALLGATHER_INIT, (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Neighbor_allgatherv, mpi_neighbor_allgatherv, MPI_NEIGHBOR_ALLGATHERV, (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *displs, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *ierr));
+PN2(void, MPI_Neighbor_allgatherv_init, mpi_neighbor_allgatherv_init, MPI_NEIGHBOR_ALLGATHERV_INIT, (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *displs, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Neighbor_alltoall, mpi_neighbor_alltoall, MPI_NEIGHBOR_ALLTOALL, (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *ierr));
+PN2(void, MPI_Neighbor_alltoall_init, mpi_neighbor_alltoall_init, MPI_NEIGHBOR_ALLTOALL_INIT, (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Neighbor_alltoallv, mpi_neighbor_alltoallv, MPI_NEIGHBOR_ALLTOALLV, (char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *sdispls, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *rdispls, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *ierr));
+PN2(void, MPI_Neighbor_alltoallv_init, mpi_neighbor_alltoallv_init, MPI_NEIGHBOR_ALLTOALLV_INIT, (char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *sdispls, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *rdispls, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Neighbor_alltoallw, mpi_neighbor_alltoallw, MPI_NEIGHBOR_ALLTOALLW, (char *sendbuf, MPI_Fint *sendcounts, MPI_Aint *sdispls, MPI_Fint *sendtypes, char *recvbuf, MPI_Fint *recvcounts, MPI_Aint *rdispls, MPI_Fint *recvtypes, MPI_Fint *comm, MPI_Fint *ierr));
+PN2(void, MPI_Neighbor_alltoallw_init, mpi_neighbor_alltoallw_init, MPI_NEIGHBOR_ALLTOALLW_INIT, (char *sendbuf, MPI_Fint *sendcounts, MPI_Aint *sdispls, MPI_Fint *sendtypes, char *recvbuf, MPI_Fint *recvcounts, MPI_Aint *rdispls, MPI_Fint *recvtypes, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Op_commutative, mpi_op_commutative, MPI_OP_COMMUTATIVE, (MPI_Fint *op, MPI_Fint *commute, MPI_Fint *ierr));
 PN2(void, MPI_Op_create, mpi_op_create, MPI_OP_CREATE, (ompi_op_fortran_handler_fn_t* function, ompi_fortran_logical_t *commute, MPI_Fint *op, MPI_Fint *ierr));
 PN2(void, MPI_Open_port, mpi_open_port, MPI_OPEN_PORT, (MPI_Fint *info, char *port_name, MPI_Fint *ierr, int port_name_len));
@@ -323,7 +350,13 @@ PN2(void, MPI_Pack_external, mpi_pack_external, MPI_PACK_EXTERNAL, (char *datare
 PN2(void, MPI_Pack_external_size, mpi_pack_external_size, MPI_PACK_EXTERNAL_SIZE, (char *datarep, MPI_Fint *incount, MPI_Fint *datatype, MPI_Aint *size, MPI_Fint *ierr, int datarep_len));
 PN2(void, MPI_Pack, mpi_pack, MPI_PACK, (char *inbuf, MPI_Fint *incount, MPI_Fint *datatype, char *outbuf, MPI_Fint *outsize, MPI_Fint *position, MPI_Fint *comm, MPI_Fint *ierr));
 PN2(void, MPI_Pack_size, mpi_pack_size, MPI_PACK_SIZE, (MPI_Fint *incount, MPI_Fint *datatype, MPI_Fint *comm, MPI_Fint *size, MPI_Fint *ierr));
+PN2(void, MPI_Parrived, mpi_parrived, MPI_PARRIVED, (MPI_Fint *request, MPI_Fint *partition, ompi_fortran_logical_t *flag, MPI_Fint *ierr));
 PN2(void, MPI_Pcontrol, mpi_pcontrol, MPI_PCONTROL, (MPI_Fint *level));
+PN2(void, MPI_Pready, mpi_pready, MPI_PREADY, (MPI_Fint *partition, MPI_Fint *request, MPI_Fint *ierr));
+PN2(void, MPI_Pready_list, mpi_pready_list, MPI_PREADY_LIST, (MPI_Fint *length, MPI_Fint *partition, MPI_Fint *request, MPI_Fint *ierr));
+PN2(void, MPI_Pready_range, mpi_pready_range, MPI_PREADY_RANGE, (MPI_Fint *partition_low, MPI_Fint *partition_high, MPI_Fint *request, MPI_Fint *ierr));
+PN2(void, MPI_Precv_init, mpi_precv_init, MPI_PRECV_INIT, (char *buf, MPI_Fint *partitions, MPI_Count *count, MPI_Fint *datatype, MPI_Fint *src, MPI_Fint *tag, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
+PN2(void, MPI_Psend_init, mpi_psend_init, MPI_PSEND_INIT, (char *buf, MPI_Fint *partitions, MPI_Count *count, MPI_Fint *datatype, MPI_Fint *dest, MPI_Fint *tag, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Probe, mpi_probe, MPI_PROBE, (MPI_Fint *source, MPI_Fint *tag, MPI_Fint *comm, MPI_Fint *status, MPI_Fint *ierr));
 PN2(void, MPI_Publish_name, mpi_publish_name, MPI_PUBLISH_NAME, (char *service_name, MPI_Fint *info, char *port_name, MPI_Fint *ierr, int service_name_len, int port_name_len));
 PN2(void, MPI_Put, mpi_put, MPI_PUT, (char *origin_addr, MPI_Fint *origin_count, MPI_Fint *origin_datatype, MPI_Fint *target_rank, MPI_Aint *target_disp, MPI_Fint *target_count, MPI_Fint *target_datatype, MPI_Fint *win, MPI_Fint *ierr));
@@ -332,9 +365,12 @@ PN2(void, MPI_Raccumulate, mpi_raccumulate, MPI_RACCUMULATE, (char *origin_addr,
 PN2(void, MPI_Recv_init, mpi_recv_init, MPI_RECV_INIT, (char *buf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *source, MPI_Fint *tag, MPI_Fint *comm, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Recv, mpi_recv, MPI_RECV, (char *buf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *source, MPI_Fint *tag, MPI_Fint *comm, MPI_Fint *status, MPI_Fint *ierr));
 PN2(void, MPI_Reduce, mpi_reduce, MPI_REDUCE, (char *sendbuf, char *recvbuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *ierr));
+PN2(void, MPI_Reduce_init, mpi_reduce_init, MPI_REDUCE_INIT, (char *sendbuf, char *recvbuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Reduce_local, mpi_reduce_local, MPI_REDUCE_LOCAL, (char *inbuf, char *inoutbuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *ierr));
 PN2(void, MPI_Reduce_scatter, mpi_reduce_scatter, MPI_REDUCE_SCATTER, (char *sendbuf, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *ierr));
+PN2(void, MPI_Reduce_scatter_init, mpi_reduce_scatter_init, MPI_REDUCE_SCATTER_INIT, (char *sendbuf, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Reduce_scatter_block, mpi_reduce_scatter_block, MPI_REDUCE_SCATTER_BLOCK, (char *sendbuf, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *ierr));
+PN2(void, MPI_Reduce_scatter_block_init, mpi_reduce_scatter_block_init, MPI_REDUCE_SCATTER_BLOCK_INIT, (char *sendbuf, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Register_datarep, mpi_register_datarep, MPI_REGISTER_DATAREP, (char *datarep, ompi_mpi2_fortran_datarep_conversion_fn_t *read_conversion_fn, ompi_mpi2_fortran_datarep_conversion_fn_t *write_conversion_fn, ompi_mpi2_fortran_datarep_extent_fn_t *dtype_file_extent_fn, MPI_Aint *extra_state, MPI_Fint *ierr, int datarep_len));
 PN2(void, MPI_Request_free, mpi_request_free, MPI_REQUEST_FREE, (MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Request_get_status, mpi_request_get_status, MPI_REQUEST_GET_STATUS, (MPI_Fint *request, ompi_fortran_logical_t *flag, MPI_Fint *status, MPI_Fint *ierr));
@@ -344,12 +380,25 @@ PN2(void, MPI_Rput, mpi_rput, MPI_RPUT, (char *origin_addr, MPI_Fint *origin_cou
 PN2(void, MPI_Rsend, mpi_rsend, MPI_RSEND, (char *ibuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *dest, MPI_Fint *tag, MPI_Fint *comm, MPI_Fint *ierr));
 PN2(void, MPI_Rsend_init, mpi_rsend_init, MPI_RSEND_INIT, (char *buf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *dest, MPI_Fint *tag, MPI_Fint *comm, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Scan, mpi_scan, MPI_SCAN, (char *sendbuf, char *recvbuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *ierr));
+PN2(void, MPI_Scan_init, mpi_scan_init, MPI_SCAN_INIT, (char *sendbuf, char *recvbuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Scatter, mpi_scatter, MPI_SCATTER, (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *ierr));
+PN2(void, MPI_Scatter_init, mpi_scatter_init, MPI_SCATTER_INIT, (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Scatterv, mpi_scatterv, MPI_SCATTERV, (char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *displs, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *ierr));
+PN2(void, MPI_Scatterv_init, mpi_scatterv_init, MPI_SCATTERV_INIT, (char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *displs, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Send_init, mpi_send_init, MPI_SEND_INIT, (char *buf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *dest, MPI_Fint *tag, MPI_Fint *comm, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Send, mpi_send, MPI_SEND, (char *buf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *dest, MPI_Fint *tag, MPI_Fint *comm, MPI_Fint *ierr));
 PN2(void, MPI_Sendrecv, mpi_sendrecv, MPI_SENDRECV, (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, MPI_Fint *dest, MPI_Fint *sendtag, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *source, MPI_Fint *recvtag, MPI_Fint *comm, MPI_Fint *status, MPI_Fint *ierr));
 PN2(void, MPI_Sendrecv_replace, mpi_sendrecv_replace, MPI_SENDRECV_REPLACE, (char *buf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *dest, MPI_Fint *sendtag, MPI_Fint *source, MPI_Fint *recvtag, MPI_Fint *comm, MPI_Fint *status, MPI_Fint *ierr));
+PN2(void, MPI_Session_call_errhandler, mpi_session_call_errhandler, MPI_SESSION_CALL_ERRHANDLER, (MPI_Fint *session, MPI_Fint *errorcode, MPI_Fint *ierr));
+PN2(void, MPI_Session_create_errhandler, mpi_session_create_errhandler, MPI_SESSION_CREATE_ERRHANDLER, (ompi_errhandler_fortran_handler_fn_t* function, MPI_Fint *errhandler, MPI_Fint *ierr));
+PN2(void, MPI_Session_get_errhandler, mpi_session_get_errhandler, MPI_SESSION_GET_ERRHANDLER, (MPI_Fint *session, MPI_Fint *erhandler, MPI_Fint *ierr));
+PN2(void, MPI_Session_get_info, mpi_session_get_info, MPI_SESSION_GET_INFO, (MPI_Fint *session, MPI_Fint *info, MPI_Fint *ierr));
+PN2(void, MPI_Session_get_nth_pset, mpi_session_get_nth_pset, MPI_SESSION_GET_NTH_PSET, (MPI_Fint *session, MPI_Fint *info, MPI_Fint *n, MPI_Fint *pset_len, char *pset_name, MPI_Fint *ierr, int pset_name_len));
+PN2(void, MPI_Session_get_num_psets, mpi_session_get_num_psets, MPI_SESSION_GET_NUM_PSETS, (MPI_Fint *session, MPI_Fint *info, MPI_Fint *npset_names, MPI_Fint *ierr));
+PN2(void, MPI_Session_get_pset_info, mpi_session_get_pset_info, MPI_SESSION_GET_PSET_INFO, (MPI_Fint *session, char *pset_name, MPI_Fint *info, MPI_Fint *ierr, int name_len));
+PN2(void, MPI_Session_init, mpi_session_init, MPI_SESSION_INIT, (MPI_Fint *info, MPI_Fint *errhandler, MPI_Fint *session, MPI_Fint *ierr));
+PN2(void, MPI_Session_finalize, mpi_session_finalize, MPI_SESSION_FINALIZE, (MPI_Fint *session, MPI_Fint *ierr));
+PN2(void, MPI_Session_set_errhandler, mpi_session_set_errhandler, MPI_SESSION_SET_ERRHANDLER, (MPI_Fint *session, MPI_Fint *erhandler, MPI_Fint *ierr));
 PN2(void, MPI_Ssend_init, mpi_ssend_init, MPI_SSEND_INIT, (char *buf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *dest, MPI_Fint *tag, MPI_Fint *comm, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPI_Ssend, mpi_ssend, MPI_SSEND, (char *buf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *dest, MPI_Fint *tag, MPI_Fint *comm, MPI_Fint *ierr));
 PN2(void, MPI_Start, mpi_start, MPI_START, (MPI_Fint *request, MPI_Fint *ierr));
@@ -373,7 +422,7 @@ PN2(void, MPI_Type_create_f90_integer, mpi_type_create_f90_integer, MPI_TYPE_CRE
 PN2(void, MPI_Type_create_f90_real, mpi_type_create_f90_real, MPI_TYPE_CREATE_F90_REAL, (MPI_Fint *p, MPI_Fint *r, MPI_Fint *newtype, MPI_Fint *ierr));
 PN2(void, MPI_Type_create_hindexed, mpi_type_create_hindexed, MPI_TYPE_CREATE_HINDEXED, (MPI_Fint *count, MPI_Fint *array_of_blocklengths, MPI_Aint *array_of_displacements, MPI_Fint *oldtype, MPI_Fint *newtype, MPI_Fint *ierr));
 PN2(void, MPI_Type_create_hvector, mpi_type_create_hvector, MPI_TYPE_CREATE_HVECTOR, (MPI_Fint *count, MPI_Fint *blocklength, MPI_Aint *stride, MPI_Fint *oldtype, MPI_Fint *newtype, MPI_Fint *ierr));
-PN2(void, MPI_Type_create_keyval, mpi_type_create_keyval, MPI_TYPE_CREATE_KEYVAL, (ompi_aint_copy_attr_function* type_copy_attr_fn, ompi_aint_delete_attr_function* type_delete_attr_fn, MPI_Fint *type_keyval, MPI_Aint *extra_state, MPI_Fint *ierr));
+PN2(void, MPI_Type_create_keyval, mpi_type_create_keyval, MPI_TYPE_CREATE_KEYVAL, (ompi_aint_copy_attr_function type_copy_attr_fn, ompi_aint_delete_attr_function type_delete_attr_fn, MPI_Fint *type_keyval, MPI_Aint *extra_state, MPI_Fint *ierr));
 PN2(void, MPI_Type_create_indexed_block, mpi_type_create_indexed_block, MPI_TYPE_CREATE_INDEXED_BLOCK, (MPI_Fint *count, MPI_Fint *blocklength, MPI_Fint *array_of_displacements, MPI_Fint *oldtype, MPI_Fint *newtype, MPI_Fint *ierr));
 PN2(void, MPI_Type_create_hindexed_block, mpi_type_create_hindexed_block, MPI_TYPE_CREATE_HINDEXED_BLOCK, (MPI_Fint *count, MPI_Fint *blocklength, MPI_Aint *array_of_displacements, MPI_Fint *oldtype, MPI_Fint *newtype, MPI_Fint *ierr));
 PN2(void, MPI_Type_create_struct, mpi_type_create_struct, MPI_TYPE_CREATE_STRUCT, (MPI_Fint *count, MPI_Fint *array_of_block_lengths, MPI_Aint *array_of_displacements, MPI_Fint *array_of_types, MPI_Fint *newtype, MPI_Fint *ierr));
@@ -421,7 +470,7 @@ PN2(void, MPI_Win_complete, mpi_win_complete, MPI_WIN_COMPLETE, (MPI_Fint *win,
 PN2(void, MPI_Win_create, mpi_win_create, MPI_WIN_CREATE, (char *base, MPI_Aint *size, MPI_Fint *disp_unit, MPI_Fint *info, MPI_Fint *comm, MPI_Fint *win, MPI_Fint *ierr));
 PN2(void, MPI_Win_create_dynamic, mpi_win_create_dynamic, MPI_WIN_CREATE_DYNAMIC, (MPI_Fint *info, MPI_Fint *comm, MPI_Fint *win, MPI_Fint *ierr));
 PN2(void, MPI_Win_create_errhandler, mpi_win_create_errhandler, MPI_WIN_CREATE_ERRHANDLER, (ompi_errhandler_fortran_handler_fn_t* function, MPI_Fint *errhandler, MPI_Fint *ierr));
-PN2(void, MPI_Win_create_keyval, mpi_win_create_keyval, MPI_WIN_CREATE_KEYVAL, (ompi_aint_copy_attr_function* win_copy_attr_fn, ompi_aint_delete_attr_function* win_delete_attr_fn, MPI_Fint *win_keyval, MPI_Aint *extra_state, MPI_Fint *ierr));
+PN2(void, MPI_Win_create_keyval, mpi_win_create_keyval, MPI_WIN_CREATE_KEYVAL,(ompi_aint_copy_attr_function win_copy_attr_fn, ompi_aint_delete_attr_function win_delete_attr_fn, MPI_Fint *win_keyval, MPI_Aint *extra_state, MPI_Fint *ierr));
 PN2(void, MPI_Win_delete_attr, mpi_win_delete_attr, MPI_WIN_DELETE_ATTR, (MPI_Fint *win, MPI_Fint *win_keyval, MPI_Fint *ierr));
 PN2(void, MPI_Win_detach, mpi_win_detach, MPI_WIN_DETACH, (MPI_Fint *win, char *base, MPI_Fint *ierr));
 PN2(void, MPI_Win_fence, mpi_win_fence, MPI_WIN_FENCE, (MPI_Fint *assert, MPI_Fint *win, MPI_Fint *ierr));
diff --git a/ompi/mpi/fortran/mpif-h/psend_init_f.c b/ompi/mpi/fortran/mpif-h/psend_init_f.c
new file mode 100644
index 00000000000..5325b9eb7d0
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/psend_init_f.c
@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2021      Bull S.A.S. All rights reserved.
+ * Copyright (c) 2023      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_PSEND_INIT = ompi_psend_init_f
+#pragma weak pmpi_psend_init = ompi_psend_init_f
+#pragma weak pmpi_psend_init_ = ompi_psend_init_f
+#pragma weak pmpi_psend_init__ = ompi_psend_init_f
+
+#pragma weak PMPI_Psend_init_f = ompi_psend_init_f
+#pragma weak PMPI_Psend_init_f08 = ompi_psend_init_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_PSEND_INIT,
+                           pmpi_psend_init,
+                           pmpi_psend_init_,
+                           pmpi_psend_init__,
+                           pompi_psend_init_f,
+                           (char *buf, MPI_Fint *partitions, MPI_Count *count, MPI_Fint *datatype, MPI_Fint *dest, MPI_Fint *tag, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                           (buf, partitions, count, datatype, dest, tag, comm, info, request, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_PSEND_INIT = ompi_psend_init_f
+#pragma weak mpi_psend_init = ompi_psend_init_f
+#pragma weak mpi_psend_init_ = ompi_psend_init_f
+#pragma weak mpi_psend_init__ = ompi_psend_init_f
+
+#pragma weak MPI_Psend_init_f = ompi_psend_init_f
+#pragma weak MPI_Psend_init_f08 = ompi_psend_init_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_PSEND_INIT,
+                           mpi_psend_init,
+                           mpi_psend_init_,
+                           mpi_psend_init__,
+                           ompi_psend_init_f,
+                           (char *buf, MPI_Fint *partitions, MPI_Count *count, MPI_Fint *datatype, MPI_Fint *dest, MPI_Fint *tag, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                           (buf, partitions, count, datatype, dest, tag, comm, info, request, ierr) )
+#else
+#define ompi_psend_init_f pompi_psend_init_f
+#endif
+#endif
+
+
+void ompi_psend_init_f(char *buf, MPI_Fint *partitions, MPI_Count *count, MPI_Fint *datatype, MPI_Fint *dest, MPI_Fint *tag, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
+{
+   int c_ierr;
+   MPI_Info c_info;
+   MPI_Datatype c_type = PMPI_Type_f2c(*datatype);
+   MPI_Request c_req;
+   MPI_Comm c_comm;
+
+   c_info = PMPI_Info_f2c(*info);
+   c_comm = PMPI_Comm_f2c (*comm);
+
+   c_ierr = PMPI_Psend_init(OMPI_F2C_BOTTOM(buf), 
+                      OMPI_FINT_2_INT(*partitions),
+                      *count,
+                      c_type, OMPI_FINT_2_INT(*dest),
+                      OMPI_FINT_2_INT(*tag),
+                      c_comm, c_info, &c_req);
+   if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+   if (MPI_SUCCESS == c_ierr) {
+      *request = PMPI_Request_c2f(c_req);
+   }
+}
diff --git a/ompi/mpi/fortran/mpif-h/reduce_init_f.c b/ompi/mpi/fortran/mpif-h/reduce_init_f.c
new file mode 100644
index 00000000000..01437078cdc
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/reduce_init_f.c
@@ -0,0 +1,98 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_REDUCE_INIT = ompi_reduce_init_f
+#pragma weak pmpi_reduce_init = ompi_reduce_init_f
+#pragma weak pmpi_reduce_init_ = ompi_reduce_init_f
+#pragma weak pmpi_reduce_init__ = ompi_reduce_init_f
+
+#pragma weak PMPI_Reduce_init_f = ompi_reduce_init_f
+#pragma weak PMPI_Reduce_init_f08 = ompi_reduce_init_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_REDUCE_INIT,
+                            pmpi_reduce_init,
+                            pmpi_reduce_init_,
+                            pmpi_reduce_init__,
+                            pompi_reduce_init_f,
+                            (char *sendbuf, char *recvbuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, recvbuf, count, datatype, op, root, comm, info, request, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_REDUCE_INIT = ompi_reduce_init_f
+#pragma weak mpi_reduce_init = ompi_reduce_init_f
+#pragma weak mpi_reduce_init_ = ompi_reduce_init_f
+#pragma weak mpi_reduce_init__ = ompi_reduce_init_f
+
+#pragma weak MPI_Reduce_init_f = ompi_reduce_init_f
+#pragma weak MPI_Reduce_init_f08 = ompi_reduce_init_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_REDUCE_INIT,
+                            mpi_reduce_init,
+                            mpi_reduce_init_,
+                            mpi_reduce_init__,
+                            ompi_reduce_init_f,
+                            (char *sendbuf, char *recvbuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, recvbuf, count, datatype, op, root, comm, info, request, ierr) )
+#else
+#define ompi_reduce_init_f pompi_reduce_init_f
+#endif
+#endif
+
+
+void ompi_reduce_init_f(char *sendbuf, char *recvbuf, MPI_Fint *count,
+                        MPI_Fint *datatype, MPI_Fint *op,
+                        MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request,
+                        MPI_Fint *ierr)
+{
+    int c_ierr;
+    MPI_Datatype c_type;
+    MPI_Info c_info;
+    MPI_Request c_request;
+    MPI_Op c_op;
+    MPI_Comm c_comm;
+
+    c_type = PMPI_Type_f2c(*datatype);
+    c_op = PMPI_Op_f2c(*op);
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_info = PMPI_Info_f2c(*info);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = PMPI_Reduce_init(sendbuf, recvbuf,
+                              OMPI_FINT_2_INT(*count),
+                              c_type, c_op,
+                              OMPI_FINT_2_INT(*root),
+                              c_comm, c_info, &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
+}
diff --git a/ompi/mpi/fortran/mpif-h/reduce_scatter_block_init_f.c b/ompi/mpi/fortran/mpif-h/reduce_scatter_block_init_f.c
new file mode 100644
index 00000000000..1bc36af0f7f
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/reduce_scatter_block_init_f.c
@@ -0,0 +1,99 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_REDUCE_SCATTER_BLOCK_INIT = ompi_reduce_scatter_block_init_f
+#pragma weak pmpi_reduce_scatter_block_init = ompi_reduce_scatter_block_init_f
+#pragma weak pmpi_reduce_scatter_block_init_ = ompi_reduce_scatter_block_init_f
+#pragma weak pmpi_reduce_scatter_block_init__ = ompi_reduce_scatter_block_init_f
+
+#pragma weak PMPI_Reduce_scatter_block_init_f = ompi_reduce_scatter_block_init_f
+#pragma weak PMPI_Reduce_scatter_block_init_f08 = ompi_reduce_scatter_block_init_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_REDUCE_SCATTER_BLOCK_INIT,
+                            pmpi_reduce_scatter_block_init,
+                            pmpi_reduce_scatter_block_init_,
+                            pmpi_reduce_scatter_block_init__,
+                            pompi_reduce_scatter_block_init_f,
+                            (char *sendbuf, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, recvbuf, recvcounts, datatype, op, comm, info, request, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_REDUCE_SCATTER_BLOCK_INIT = ompi_reduce_scatter_block_init_f
+#pragma weak mpi_reduce_scatter_block_init = ompi_reduce_scatter_block_init_f
+#pragma weak mpi_reduce_scatter_block_init_ = ompi_reduce_scatter_block_init_f
+#pragma weak mpi_reduce_scatter_block_init__ = ompi_reduce_scatter_block_init_f
+
+#pragma weak MPI_Reduce_scatter_block_init_f = ompi_reduce_scatter_block_init_f
+#pragma weak MPI_Reduce_scatter_block_init_f08 = ompi_reduce_scatter_block_init_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_REDUCE_SCATTER_BLOCK_INIT,
+                            mpi_reduce_scatter_block_init,
+                            mpi_reduce_scatter_block_init_,
+                            mpi_reduce_scatter_block_init__,
+                            ompi_reduce_scatter_block_init_f,
+                            (char *sendbuf, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, recvbuf, recvcounts, datatype, op, comm, info, request, ierr) )
+#else
+#define ompi_reduce_scatter_block_init_f pompi_reduce_scatter_block_init_f
+#endif
+#endif
+
+
+void ompi_reduce_scatter_block_init_f(char *sendbuf, char *recvbuf,
+                                      MPI_Fint *recvcount, MPI_Fint *datatype,
+                                      MPI_Fint *op, MPI_Fint *comm,
+                                      MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
+{
+    int c_ierr;
+    MPI_Comm c_comm;
+    MPI_Datatype c_type;
+    MPI_Info c_info;
+    MPI_Request c_request;
+    MPI_Op c_op;
+    int size;
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_type = PMPI_Type_f2c(*datatype);
+    c_op = PMPI_Op_f2c(*op);
+    c_info = PMPI_Info_f2c(*info);
+
+    PMPI_Comm_size(c_comm, &size);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = PMPI_Reduce_scatter_block_init(sendbuf, recvbuf,
+                                            OMPI_FINT_2_INT(*recvcount),
+                                            c_type, c_op, c_comm, c_info, &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
+}
diff --git a/ompi/mpi/fortran/mpif-h/reduce_scatter_init_f.c b/ompi/mpi/fortran/mpif-h/reduce_scatter_init_f.c
new file mode 100644
index 00000000000..d60a69dbfef
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/reduce_scatter_init_f.c
@@ -0,0 +1,109 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_REDUCE_SCATTER_INIT = ompi_reduce_scatter_init_f
+#pragma weak pmpi_reduce_scatter_init = ompi_reduce_scatter_init_f
+#pragma weak pmpi_reduce_scatter_init_ = ompi_reduce_scatter_init_f
+#pragma weak pmpi_reduce_scatter_init__ = ompi_reduce_scatter_init_f
+
+#pragma weak PMPI_Reduce_scatter_init_f = ompi_reduce_scatter_init_f
+#pragma weak PMPI_Reduce_scatter_init_f08 = ompi_reduce_scatter_init_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_REDUCE_SCATTER_INIT,
+                            pmpi_reduce_scatter_init,
+                            pmpi_reduce_scatter_init_,
+                            pmpi_reduce_scatter_init__,
+                            pompi_reduce_scatter_init_f,
+                            (char *sendbuf, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, recvbuf, recvcounts, datatype, op, comm, info, request, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_REDUCE_SCATTER_INIT = ompi_reduce_scatter_init_f
+#pragma weak mpi_reduce_scatter_init = ompi_reduce_scatter_init_f
+#pragma weak mpi_reduce_scatter_init_ = ompi_reduce_scatter_init_f
+#pragma weak mpi_reduce_scatter_init__ = ompi_reduce_scatter_init_f
+
+#pragma weak MPI_Reduce_scatter_init_f = ompi_reduce_scatter_init_f
+#pragma weak MPI_Reduce_scatter_init_f08 = ompi_reduce_scatter_init_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_REDUCE_SCATTER_INIT,
+                            mpi_reduce_scatter_init,
+                            mpi_reduce_scatter_init_,
+                            mpi_reduce_scatter_init__,
+                            ompi_reduce_scatter_init_f,
+                            (char *sendbuf, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, recvbuf, recvcounts, datatype, op, comm, info, request, ierr) )
+#else
+#define ompi_reduce_scatter_init_f pompi_reduce_scatter_init_f
+#endif
+#endif
+
+
+void ompi_reduce_scatter_init_f(char *sendbuf, char *recvbuf,
+                                MPI_Fint *recvcounts, MPI_Fint *datatype,
+                                MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request,
+                                MPI_Fint *ierr)
+{
+    int c_ierr;
+    MPI_Comm c_comm;
+    MPI_Datatype c_type;
+    MPI_Info c_info;
+    MPI_Request c_request;
+    MPI_Op c_op;
+    int size;
+    OMPI_ARRAY_NAME_DECL(recvcounts);
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_type = PMPI_Type_f2c(*datatype);
+    c_op = PMPI_Op_f2c(*op);
+    c_info = PMPI_Info_f2c(*info);
+
+    PMPI_Comm_size(c_comm, &size);
+    OMPI_ARRAY_FINT_2_INT(recvcounts, size);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = PMPI_Reduce_scatter_init(sendbuf, recvbuf,
+                                      OMPI_ARRAY_NAME_CONVERT(recvcounts),
+                                      c_type, c_op, c_comm, c_info, &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_request);
+        ompi_coll_base_nbc_request_t* nb_request = (ompi_coll_base_nbc_request_t*)c_request;
+        nb_request->data.release_arrays[0] = OMPI_ARRAY_NAME_CONVERT(recvcounts);
+        nb_request->data.release_arrays[1] = NULL;
+    } else {
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(recvcounts);
+    }
+}
diff --git a/ompi/mpi/fortran/mpif-h/scan_init_f.c b/ompi/mpi/fortran/mpif-h/scan_init_f.c
new file mode 100644
index 00000000000..a58d237b942
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/scan_init_f.c
@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_SCAN_INIT = ompi_scan_init_f
+#pragma weak pmpi_scan_init = ompi_scan_init_f
+#pragma weak pmpi_scan_init_ = ompi_scan_init_f
+#pragma weak pmpi_scan_init__ = ompi_scan_init_f
+
+#pragma weak PMPI_Scan_init_f = ompi_scan_init_f
+#pragma weak PMPI_Scan_init_f08 = ompi_scan_init_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_SCAN_INIT,
+                            pmpi_scan_init,
+                            pmpi_scan_init_,
+                            pmpi_scan_init__,
+                            pompi_scan_init_f,
+                            (char *sendbuf, char *recvbuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, recvbuf, count, datatype, op, comm, info, request, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_SCAN_INIT = ompi_scan_init_f
+#pragma weak mpi_scan_init = ompi_scan_init_f
+#pragma weak mpi_scan_init_ = ompi_scan_init_f
+#pragma weak mpi_scan_init__ = ompi_scan_init_f
+
+#pragma weak MPI_Scan_init_f = ompi_scan_init_f
+#pragma weak MPI_Scan_init_f08 = ompi_scan_init_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_SCAN_INIT,
+                            mpi_scan_init,
+                            mpi_scan_init_,
+                            mpi_scan_init__,
+                            ompi_scan_init_f,
+                            (char *sendbuf, char *recvbuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, recvbuf, count, datatype, op, comm, info, request, ierr) )
+#else
+#define ompi_scan_init_f pompi_scan_init_f
+#endif
+#endif
+
+
+void ompi_scan_init_f(char *sendbuf, char *recvbuf, MPI_Fint *count,
+                      MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm,
+                      MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
+{
+    int c_ierr;
+    MPI_Comm c_comm;
+    MPI_Datatype c_type;
+    MPI_Info c_info;
+    MPI_Request c_request;
+    MPI_Op c_op;
+
+    c_type = PMPI_Type_f2c(*datatype);
+    c_op = PMPI_Op_f2c(*op);
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_info = PMPI_Info_f2c(*info);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = PMPI_Scan_init(sendbuf, recvbuf,
+                            OMPI_FINT_2_INT(*count),
+                            c_type, c_op,
+                            c_comm, c_info, &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
+}
diff --git a/ompi/mpi/fortran/mpif-h/scatter_init_f.c b/ompi/mpi/fortran/mpif-h/scatter_init_f.c
new file mode 100644
index 00000000000..2f6f3e7fdb9
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/scatter_init_f.c
@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_SCATTER_INIT = ompi_scatter_init_f
+#pragma weak pmpi_scatter_init = ompi_scatter_init_f
+#pragma weak pmpi_scatter_init_ = ompi_scatter_init_f
+#pragma weak pmpi_scatter_init__ = ompi_scatter_init_f
+
+#pragma weak PMPI_Scatter_init_f = ompi_scatter_init_f
+#pragma weak PMPI_Scatter_init_f08 = ompi_scatter_init_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_SCATTER_INIT,
+                            pmpi_scatter_init,
+                            pmpi_scatter_init_,
+                            pmpi_scatter_init__,
+                            pompi_scatter_init_f,
+                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, info, request, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_SCATTER_INIT = ompi_scatter_init_f
+#pragma weak mpi_scatter_init = ompi_scatter_init_f
+#pragma weak mpi_scatter_init_ = ompi_scatter_init_f
+#pragma weak mpi_scatter_init__ = ompi_scatter_init_f
+
+#pragma weak MPI_Scatter_init_f = ompi_scatter_init_f
+#pragma weak MPI_Scatter_init_f08 = ompi_scatter_init_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_SCATTER_INIT,
+                            mpi_scatter_init,
+                            mpi_scatter_init_,
+                            mpi_scatter_init__,
+                            ompi_scatter_init_f,
+                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, info, request, ierr) )
+#else
+#define ompi_scatter_init_f pompi_scatter_init_f
+#endif
+#endif
+
+
+void ompi_scatter_init_f(char *sendbuf, MPI_Fint *sendcount,
+                         MPI_Fint *sendtype, char *recvbuf,
+                         MPI_Fint *recvcount, MPI_Fint *recvtype,
+                         MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request,
+                         MPI_Fint *ierr)
+{
+    int c_ierr;
+    MPI_Datatype c_sendtype, c_recvtype;
+    MPI_Info c_info;
+    MPI_Request c_request;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+
+    c_sendtype = PMPI_Type_f2c(*sendtype);
+    c_recvtype = PMPI_Type_f2c(*recvtype);
+    c_info = PMPI_Info_f2c(*info);
+
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_IN_PLACE(recvbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = PMPI_Scatter_init(sendbuf,OMPI_FINT_2_INT(*sendcount),
+                               c_sendtype, recvbuf,
+                               OMPI_FINT_2_INT(*recvcount),
+                               c_recvtype,
+                               OMPI_FINT_2_INT(*root), c_comm, c_info, &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
+}
diff --git a/ompi/mpi/fortran/mpif-h/scatterv_init_f.c b/ompi/mpi/fortran/mpif-h/scatterv_init_f.c
new file mode 100644
index 00000000000..394984214dd
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/scatterv_init_f.c
@@ -0,0 +1,118 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+#include "ompi/mca/coll/base/coll_base_util.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_SCATTERV_INIT = ompi_scatterv_init_f
+#pragma weak pmpi_scatterv_init = ompi_scatterv_init_f
+#pragma weak pmpi_scatterv_init_ = ompi_scatterv_init_f
+#pragma weak pmpi_scatterv_init__ = ompi_scatterv_init_f
+
+#pragma weak PMPI_Scatterv_init_f = ompi_scatterv_init_f
+#pragma weak PMPI_Scatterv_init_f08 = ompi_scatterv_init_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_SCATTERV_INIT,
+                            pmpi_scatterv_init,
+                            pmpi_scatterv_init_,
+                            pmpi_scatterv_init__,
+                            pompi_scatterv_init_f,
+                            (char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *displs, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, sendcounts, displs, sendtype, recvbuf, recvcount, recvtype, root, comm, info, request, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_SCATTERV_INIT = ompi_scatterv_init_f
+#pragma weak mpi_scatterv_init = ompi_scatterv_init_f
+#pragma weak mpi_scatterv_init_ = ompi_scatterv_init_f
+#pragma weak mpi_scatterv_init__ = ompi_scatterv_init_f
+
+#pragma weak MPI_Scatterv_init_f = ompi_scatterv_init_f
+#pragma weak MPI_Scatterv_init_f08 = ompi_scatterv_init_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_SCATTERV_INIT,
+                            mpi_scatterv_init,
+                            mpi_scatterv_init_,
+                            mpi_scatterv_init__,
+                            ompi_scatterv_init_f,
+                            (char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *displs, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
+                            (sendbuf, sendcounts, displs, sendtype, recvbuf, recvcount, recvtype, root, comm, info, request, ierr) )
+#else
+#define ompi_scatterv_init_f pompi_scatterv_init_f
+#endif
+#endif
+
+
+void ompi_scatterv_init_f(char *sendbuf, MPI_Fint *sendcounts,
+                          MPI_Fint *displs, MPI_Fint *sendtype,
+                          char *recvbuf, MPI_Fint *recvcount,
+                          MPI_Fint *recvtype, MPI_Fint *root,
+                          MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
+{
+    MPI_Comm c_comm;
+    MPI_Datatype c_sendtype, c_recvtype;
+    MPI_Info c_info;
+    MPI_Request c_request;
+    int size, idx = 0, c_ierr;
+    OMPI_ARRAY_NAME_DECL(sendcounts);
+    OMPI_ARRAY_NAME_DECL(displs);
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_sendtype = PMPI_Type_f2c(*sendtype);
+    c_recvtype = PMPI_Type_f2c(*recvtype);
+    c_info = PMPI_Info_f2c(*info);
+
+    PMPI_Comm_size(c_comm, &size);
+    OMPI_ARRAY_FINT_2_INT(sendcounts, size);
+    OMPI_ARRAY_FINT_2_INT(displs, size);
+
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_IN_PLACE(recvbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = PMPI_Scatterv_init(sendbuf,
+                                OMPI_ARRAY_NAME_CONVERT(sendcounts),
+                                OMPI_ARRAY_NAME_CONVERT(displs),
+                                c_sendtype, recvbuf,
+                                OMPI_FINT_2_INT(*recvcount),
+                                c_recvtype,
+                                OMPI_FINT_2_INT(*root), c_comm, c_info, &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_request);
+        ompi_coll_base_nbc_request_t* nb_request = (ompi_coll_base_nbc_request_t*)c_request;
+        if (sendcounts != OMPI_ARRAY_NAME_CONVERT(sendcounts)) {
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(sendcounts);
+            nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(displs);
+        }
+        nb_request->data.release_arrays[idx]   = NULL;
+     } else {
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(sendcounts);
+        OMPI_ARRAY_FINT_2_INT_CLEANUP(displs);
+     }
+}
diff --git a/ompi/mpi/fortran/mpif-h/session_call_errhandler_f.c b/ompi/mpi/fortran/mpif-h/session_call_errhandler_f.c
new file mode 100644
index 00000000000..deb4360c752
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/session_call_errhandler_f.c
@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      Triad National Security, LLC.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_SESSION_CALL_ERRHANDLER = ompi_session_call_errhandler_f
+#pragma weak pmpi_session_call_errhandler = ompi_session_call_errhandler_f
+#pragma weak pmpi_session_call_errhandler_ = ompi_session_call_errhandler_f
+#pragma weak pmpi_session_call_errhandler__ = ompi_session_call_errhandler_f
+
+#pragma weak PMPI_Session_call_errhandler_f = ompi_session_call_errhandler_f
+#pragma weak PMPI_Session_call_errhandler_f08 = ompi_session_call_errhandler_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_SESSION_CALL_ERRHANDLER,
+                            pmpi_session_call_errhandler,
+                            pmpi_session_call_errhandler_,
+                            pmpi_session_call_errhandler__,
+                            pompi_session_call_errhandler_f,
+                            (MPI_Fint *session, MPI_Fint *errorcode, MPI_Fint *ierr),
+                            (session, errorcode, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_SESSION_CALL_ERRHANDLER = ompi_session_call_errhandler_f
+#pragma weak mpi_session_call_errhandler = ompi_session_call_errhandler_f
+#pragma weak mpi_session_call_errhandler_ = ompi_session_call_errhandler_f
+#pragma weak mpi_session_call_errhandler__ = ompi_session_call_errhandler_f
+
+#pragma weak MPI_Session_call_errhandler_f = ompi_session_call_errhandler_f
+#pragma weak MPI_Session_call_errhandler_f08 = ompi_session_call_errhandler_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+   OMPI_GENERATE_F77_BINDINGS (MPI_SESSION_CALL_ERRHANDLER,
+                               mpi_session_call_errhandler,
+                               mpi_session_call_errhandler_,
+                               mpi_session_call_errhandler__,
+                               ompi_session_call_errhandler_f,
+                               (MPI_Fint *session, MPI_Fint *errorcode, MPI_Fint *ierr),
+                               (session, errorcode, ierr) )
+#else
+#define ompi_session_call_errhandler_f pompi_session_call_errhandler_f
+#endif
+#endif
+
+
+void ompi_session_call_errhandler_f(MPI_Fint *session, MPI_Fint *errorcode,
+                                 MPI_Fint *ierr)
+{
+   int c_ierr;
+   MPI_Session c_session;
+
+   c_session = PMPI_Session_f2c(*session);
+
+   c_ierr = PMPI_Session_call_errhandler(c_session, OMPI_FINT_2_INT(*errorcode));
+   if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/mpif-h/session_create_errhandler_f.c b/ompi/mpi/fortran/mpif-h/session_create_errhandler_f.c
new file mode 100644
index 00000000000..bf97f332570
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/session_create_errhandler_f.c
@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/instance/instance.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_SESSION_CREATE_ERRHANDLER = ompi_session_create_errhandler_f
+#pragma weak pmpi_session_create_errhandler = ompi_session_create_errhandler_f
+#pragma weak pmpi_session_create_errhandler_ = ompi_session_create_errhandler_f
+#pragma weak pmpi_session_create_errhandler__ = ompi_session_create_errhandler_f
+
+#pragma weak PMPI_Session_create_errhandler_f = ompi_session_create_errhandler_f
+#pragma weak PMPI_Session_create_errhandler_f08 = ompi_session_create_errhandler_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_SESSION_CREATE_ERRHANDLER,
+                           pmpi_session_create_errhandler,
+                           pmpi_session_create_errhandler_,
+                           pmpi_session_create_errhandler__,
+                           pompi_session_create_errhandler_f,
+                           (ompi_errhandler_fortran_handler_fn_t* function, MPI_Fint *errhandler, MPI_Fint *ierr),
+                           (function, errhandler, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_SESSION_CREATE_ERRHANDLER = ompi_session_create_errhandler_f
+#pragma weak mpi_session_create_errhandler = ompi_session_create_errhandler_f
+#pragma weak mpi_session_create_errhandler_ = ompi_session_create_errhandler_f
+#pragma weak mpi_session_create_errhandler__ = ompi_session_create_errhandler_f
+
+#pragma weak MPI_Session_create_errhandler_f = ompi_session_create_errhandler_f
+#pragma weak MPI_Session_create_errhandler_f08 = ompi_session_create_errhandler_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_SESSION_CREATE_ERRHANDLER,
+                           mpi_session_create_errhandler,
+                           mpi_session_create_errhandler_,
+                           mpi_session_create_errhandler__,
+                           ompi_session_create_errhandler_f,
+                           (ompi_errhandler_fortran_handler_fn_t* function, MPI_Fint *errhandler, MPI_Fint *ierr),
+                           (function, errhandler, ierr) )
+#else
+#define ompi_session_create_errhandler_f pompi_session_create_errhandler_f
+#endif
+#endif
+
+static const char FUNC_NAME[] = "MPI_SESSION_CREATE_ERRHANDLER";
+
+
+void ompi_session_create_errhandler_f(ompi_errhandler_fortran_handler_fn_t *function,
+  				      MPI_Fint *errhandler, MPI_Fint *ierr)
+{
+    int c_ierr;
+    MPI_Errhandler c_errhandler;
+
+    c_errhandler = ompi_errhandler_create(OMPI_ERRHANDLER_TYPE_INSTANCE,
+                               (ompi_errhandler_generic_handler_fn_t*) function,
+                               OMPI_ERRHANDLER_LANG_FORTRAN);
+    if (MPI_ERRHANDLER_NULL != c_errhandler) {
+        *errhandler = PMPI_Errhandler_c2f(c_errhandler);
+        c_ierr = MPI_SUCCESS;
+    } else {
+        c_ierr = MPI_ERR_INTERN;
+        OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_INTERN, FUNC_NAME);
+    }
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/mpif-h/session_finalize_f.c b/ompi/mpi/fortran/mpif-h/session_finalize_f.c
new file mode 100644
index 00000000000..4a7d44127e9
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/session_finalize_f.c
@@ -0,0 +1,90 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2013 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2019      Triad National Security, LLC.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_SESSION_FINALIZE = ompi_session_finalize_f
+#pragma weak pmpi_session_finalize = ompi_session_finalize_f
+#pragma weak pmpi_session_finalize_ = ompi_session_finalize_f
+#pragma weak pmpi_session_finalize__ = ompi_session_finalize_f
+
+#pragma weak PMPI_Session_finalize_f = ompi_session_finalize_f
+#pragma weak PMPI_Session_finalize_f08 = ompi_session_finalize_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_SESSION_FINALIZE,
+                            pmpi_session_finalize,
+                            pmpi_session_finalize_,
+                            pmpi_session_finalize__,
+                            pompi_session_finalize_f,
+                            (MPI_Fint *session, MPI_Fint *ierr),
+                            (session, ierr) )
+#endif
+#endif
+
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_SESSION_FINALIZE = ompi_session_finalize_f
+#pragma weak mpi_session_finalize = ompi_session_finalize_f
+#pragma weak mpi_session_finalize_ = ompi_session_finalize_f
+#pragma weak mpi_session_finalize__ = ompi_session_finalize_f
+
+#pragma weak MPI_Session_finalize_f = ompi_session_finalize_f
+#pragma weak MPI_Session_finalize_f08 = ompi_session_finalize_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_SESSION_FINALIZE,
+                            mpi_session_finalize,
+                            mpi_session_finalize_,
+                            mpi_session_finalize__,
+                            ompi_session_finalize_f,
+                            (MPI_Fint *session, MPI_Fint *ierr),
+                            (session, ierr) )
+#else
+#define ompi_session_finalize_f pompi_session_finalize_f
+#endif
+#endif
+
+void ompi_session_finalize_f(MPI_Fint *session, MPI_Fint *ierr)
+{
+    int c_ierr;
+    MPI_Session c_session;
+
+    c_session = PMPI_Session_f2c(*session);
+
+    c_ierr = PMPI_Session_finalize(&c_session);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    /* This value comes from the MPI_SESSION_NULL value in mpif.h.  Do not
+       change without consulting mpif.h! */
+
+    if (MPI_SUCCESS == c_ierr) {
+        *session = 0;
+    }
+}
diff --git a/ompi/mpi/fortran/mpif-h/session_get_errhandler_f.c b/ompi/mpi/fortran/mpif-h/session_get_errhandler_f.c
new file mode 100644
index 00000000000..9ea6ce1941a
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/session_get_errhandler_f.c
@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      Triad National Security, LLC.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_SESSION_GET_ERRHANDLER = ompi_session_get_errhandler_f
+#pragma weak pmpi_session_get_errhandler = ompi_session_get_errhandler_f
+#pragma weak pmpi_session_get_errhandler_ = ompi_session_get_errhandler_f
+#pragma weak pmpi_session_get_errhandler__ = ompi_session_get_errhandler_f
+
+#pragma weak PMPI_Session_get_errhandler_f = ompi_session_get_errhandler_f
+#pragma weak PMPI_Session_get_errhandler_f08 = ompi_session_get_errhandler_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_SESSION_GET_ERRHANDLER,
+                           pmpi_session_get_errhandler,
+                           pmpi_session_get_errhandler_,
+                           pmpi_session_get_errhandler__,
+                           pompi_session_get_errhandler_f,
+                           (MPI_Fint *session, MPI_Fint *erhandler, MPI_Fint *ierr),
+                           (session, erhandler, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_SESSION_GET_ERRHANDLER = ompi_session_get_errhandler_f
+#pragma weak mpi_session_get_errhandler = ompi_session_get_errhandler_f
+#pragma weak mpi_session_get_errhandler_ = ompi_session_get_errhandler_f
+#pragma weak mpi_session_get_errhandler__ = ompi_session_get_errhandler_f
+
+#pragma weak MPI_Session_get_errhandler_f = ompi_session_get_errhandler_f
+#pragma weak MPI_Session_get_errhandler_f08 = ompi_session_get_errhandler_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_SESSION_GET_ERRHANDLER,
+                           mpi_session_get_errhandler,
+                           mpi_session_get_errhandler_,
+                           mpi_session_get_errhandler__,
+                           ompi_session_get_errhandler_f,
+                           (MPI_Fint *session, MPI_Fint *erhandler, MPI_Fint *ierr),
+                           (session, erhandler, ierr) )
+#else
+#define ompi_session_get_errhandler_f pompi_session_get_errhandler_f
+#endif
+#endif
+
+
+void ompi_session_get_errhandler_f(MPI_Fint *session, MPI_Fint *errhandler,
+                                MPI_Fint *ierr)
+{
+    int c_ierr;
+    MPI_Session c_session;
+    MPI_Errhandler c_errhandler;
+
+    c_session = PMPI_Session_f2c(*session);
+
+    c_ierr = PMPI_Session_get_errhandler(c_session, &c_errhandler);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *errhandler = PMPI_Errhandler_c2f(c_errhandler);
+    }
+}
diff --git a/ompi/mpi/fortran/mpif-h/session_get_info_f.c b/ompi/mpi/fortran/mpif-h/session_get_info_f.c
new file mode 100644
index 00000000000..2172c33b7ff
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/session_get_info_f.c
@@ -0,0 +1,88 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2013 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2019      Triad National Security, LLC.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_SESSION_GET_INFO = ompi_session_get_info_f
+#pragma weak pmpi_session_get_info = ompi_session_get_info_f
+#pragma weak pmpi_session_get_info_ = ompi_session_get_info_f
+#pragma weak pmpi_session_get_info__ = ompi_session_get_info_f
+
+#pragma weak PMPI_Session_get_info_f = ompi_session_get_info_f
+#pragma weak PMPI_Session_get_info_f08 = ompi_session_get_info_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_SESSION_GET_INFO,
+                            pmpi_session_get_info,
+                            pmpi_session_get_info_,
+                            pmpi_session_get_info__,
+                            pompi_session_get_info_f,
+                            (MPI_Fint *session, MPI_Fint *npset_names, MPI_Fint *ierr),
+                            (session, npset_names, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_SESSION_GET_INFO = ompi_session_get_info_f
+#pragma weak mpi_session_get_info = ompi_session_get_info_f
+#pragma weak mpi_session_get_info_ = ompi_session_get_info_f
+#pragma weak mpi_session_get_info__ = ompi_session_get_info_f
+
+#pragma weak MPI_Session_get_info_f = ompi_session_get_info_f
+#pragma weak MPI_Session_get_info_f08 = ompi_session_get_info_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_SESSION_GET_INFO,
+                            mpi_session_get_info,
+                            mpi_session_get_info_,
+                            mpi_session_get_info__,
+                            ompi_session_get_info_f,
+                            (MPI_Fint *session, MPI_Fint *npset_names, MPI_Fint *ierr),
+                            (session, npset_names, ierr) )
+#else
+#define ompi_session_get_info_f pompi_session_get_info_f
+#endif
+#endif
+
+void ompi_session_get_info_f(MPI_Fint *session, MPI_Fint *info, MPI_Fint *ierr)
+{
+    int c_ierr;
+    MPI_Session c_session;
+    MPI_Info c_info;
+
+    c_session = PMPI_Session_f2c(*session);
+
+    c_ierr = PMPI_Session_get_info(c_session, &c_info);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *info = PMPI_Info_c2f(c_info);
+    }
+}
diff --git a/ompi/mpi/fortran/mpif-h/session_get_nth_pset_f.c b/ompi/mpi/fortran/mpif-h/session_get_nth_pset_f.c
new file mode 100644
index 00000000000..f221274417f
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/session_get_nth_pset_f.c
@@ -0,0 +1,109 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2013 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2023 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2019      Triad National Security, LLC.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/fortran_base_strings.h"
+#include "ompi/constants.h"
+
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_SESSION_GET_NTH_PSET = ompi_session_get_nth_pset_f
+#pragma weak pmpi_session_get_nth_pset = ompi_session_get_nth_pset_f
+#pragma weak pmpi_session_get_nth_pset_ = ompi_session_get_nth_pset_f
+#pragma weak pmpi_session_get_nth_pset__ = ompi_session_get_nth_pset_f
+
+#pragma weak PMPI_Session_get_nth_pset_f = ompi_session_get_nth_pset_f
+#pragma weak PMPI_Session_get_nth_pset_f08 = ompi_session_get_nth_pset_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_SESSION_GET_NTH_PSET,
+                            pmpi_session_get_nth_pset,
+                            pmpi_session_get_nth_pset_,
+                            pmpi_session_get_nth_pset__,
+                            pompi_session_get_nth_pset_f,
+                            (MPI_Fint *session, MPI_Fint *info, MPI_Fint *n, MPI_Fint *pset_len, char *pset_name, MPI_Fint *ierr, int pset_name_len),
+                            (session, info, n, pset_len, pset_name, ierr, pset_name_len))
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_SESSION_GET_NTH_PSET = ompi_session_get_nth_pset_f
+#pragma weak mpi_session_get_nth_pset = ompi_session_get_nth_pset_f
+#pragma weak mpi_session_get_nth_pset_ = ompi_session_get_nth_pset_f
+#pragma weak mpi_session_get_nth_pset__ = ompi_session_get_nth_pset_f
+
+#pragma weak MPI_Session_get_nth_pset_f = ompi_session_get_nth_pset_f
+#pragma weak MPI_Session_get_nth_pset_f08 = ompi_session_get_nth_pset_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_SESSION_GET_NTH_PSET,
+                            mpi_session_get_nth_pset,
+                            mpi_session_get_nth_pset_,
+                            mpi_session_get_nth_pset__,
+                            ompi_session_get_nth_pset_f,
+                            (MPI_Fint *session, MPI_Fint *info, MPI_Fint *n, MPI_Fint *pset_len, char *pset_name, MPI_Fint *ierr, int pset_name_len),
+                            (session, info, n, pset_len, pset_name, ierr, pset_name_len))
+#else
+#define ompi_session_get_nth_pset_f pompi_session_get_nth_pset_f
+#endif
+#endif
+
+void ompi_session_get_nth_pset_f(MPI_Fint *session, MPI_Fint *info, MPI_Fint *n, MPI_Fint *pset_len, char *pset_name, MPI_Fint *ierr,
+                                 int pset_name_len)
+{
+    int c_ierr;
+    MPI_Session c_session;
+    char c_name[MPI_MAX_PSET_NAME_LEN];
+    OMPI_SINGLE_NAME_DECL(pset_len);
+
+    c_session = PMPI_Session_f2c(*session);
+
+    if (0 == *pset_len) {
+        c_ierr = PMPI_Session_get_nth_pset(c_session, MPI_INFO_NULL, *n,
+                                           OMPI_SINGLE_NAME_CONVERT(pset_len),
+                                           c_name);
+        if (MPI_SUCCESS == c_ierr) {
+            OMPI_SINGLE_INT_2_FINT(pset_len);
+        }
+
+    } else {
+        c_ierr = PMPI_Session_get_nth_pset(c_session, MPI_INFO_NULL, *n,
+                                           OMPI_SINGLE_NAME_CONVERT(pset_len),
+                                           c_name);
+        /*
+         * use pset_name_len as that is the length of the supplied pset_name 
+         * otherwise there may be gibberish characters past *pset_len
+         */
+        if (MPI_SUCCESS == c_ierr) {
+            ompi_fortran_string_c2f(c_name, pset_name, pset_name_len);
+        }
+    }
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+}
diff --git a/ompi/mpi/fortran/mpif-h/session_get_num_psets_f.c b/ompi/mpi/fortran/mpif-h/session_get_num_psets_f.c
new file mode 100644
index 00000000000..f1498395e78
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/session_get_num_psets_f.c
@@ -0,0 +1,87 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2013 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2019      Triad National Security, LLC.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_SESSION_GET_NUM_PSETS = ompi_session_get_num_psets_f
+#pragma weak pmpi_session_get_num_psets = ompi_session_get_num_psets_f
+#pragma weak pmpi_session_get_num_psets_ = ompi_session_get_num_psets_f
+#pragma weak pmpi_session_get_num_psets__ = ompi_session_get_num_psets_f
+
+#pragma weak PMPI_Session_get_num_psets_f = ompi_session_get_num_psets_f
+#pragma weak PMPI_Session_get_num_psets_f08 = ompi_session_get_num_psets_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_SESSION_GET_NUM_PSETS,
+                            pmpi_session_get_num_psets,
+                            pmpi_session_get_num_psets_,
+                            pmpi_session_get_num_psets__,
+                            pompi_session_get_num_psets_f,
+                            (MPI_Fint *session, MPI_Fint *info, MPI_Fint *npset_names, MPI_Fint *ierr),
+                            (session, info, npset_names, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_SESSION_GET_NUM_PSETS = ompi_session_get_num_psets_f
+#pragma weak mpi_session_get_num_psets = ompi_session_get_num_psets_f
+#pragma weak mpi_session_get_num_psets_ = ompi_session_get_num_psets_f
+#pragma weak mpi_session_get_num_psets__ = ompi_session_get_num_psets_f
+
+#pragma weak MPI_Session_get_num_psets_f = ompi_session_get_num_psets_f
+#pragma weak MPI_Session_get_num_psets_f08 = ompi_session_get_num_psets_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_SESSION_GET_NUM_PSETS,
+                            mpi_session_get_num_psets,
+                            mpi_session_get_num_psets_,
+                            mpi_session_get_num_psets__,
+                            ompi_session_get_num_psets_f,
+                            (MPI_Fint *session, MPI_Fint *info, MPI_Fint *npset_names, MPI_Fint *ierr),
+                            (session, info, npset_names, ierr) )
+#else
+#define ompi_session_get_num_psets_f pompi_session_get_num_psets_f
+#endif
+#endif
+
+void ompi_session_get_num_psets_f(MPI_Fint *session, MPI_Fint *info, MPI_Fint *npset_names, MPI_Fint *ierr)
+{
+    int c_ierr;
+    MPI_Session c_session;
+    OMPI_SINGLE_NAME_DECL(npset_names);
+
+    c_session = PMPI_Session_f2c(*session);
+
+    c_ierr = PMPI_Session_get_num_psets(c_session, MPI_INFO_NULL, OMPI_SINGLE_NAME_CONVERT(npset_names));
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        OMPI_SINGLE_INT_2_FINT(npset_names);
+    }
+}
diff --git a/ompi/mpi/fortran/mpif-h/session_get_pset_info_f.c b/ompi/mpi/fortran/mpif-h/session_get_pset_info_f.c
new file mode 100644
index 00000000000..fb3b4cbed57
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/session_get_pset_info_f.c
@@ -0,0 +1,108 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2013 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2019      Triad National Security, LLC.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/fortran_base_strings.h"
+#include "ompi/constants.h"
+#include "ompi/instance/instance.h"
+
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_SESSION_GET_PSET_INFO = ompi_session_get_pset_info_f
+#pragma weak pmpi_session_get_pset_info = ompi_session_get_pset_info_f
+#pragma weak pmpi_session_get_pset_info_ = ompi_session_get_pset_info_f
+#pragma weak pmpi_session_get_pset_info__ = ompi_session_get_pset_info_f
+
+#pragma weak PMPI_Session_get_pset_info_f = ompi_session_get_pset_info_f
+#pragma weak PMPI_Session_get_pset_info_f08 = ompi_session_get_pset_info_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_SESSION_GET_PSET_INFO,
+                            pmpi_session_get_pset_info,
+                            pmpi_session_get_pset_info_,
+                            pmpi_session_get_pset_info__,
+                            pompi_session_get_pset_info_f,
+                            (MPI_Fint *session, char *pset_name, MPI_Fint *info, MPI_Fint *ierr, int name_len),
+                            (session, pset_name, info, ierr, name_len) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_SESSION_GET_PSET_INFO = ompi_session_get_pset_info_f
+#pragma weak mpi_session_get_pset_info = ompi_session_get_pset_info_f
+#pragma weak mpi_session_get_pset_info_ = ompi_session_get_pset_info_f
+#pragma weak mpi_session_get_pset_info__ = ompi_session_get_pset_info_f
+
+#pragma weak MPI_Session_get_pset_info_f = ompi_session_get_pset_info_f
+#pragma weak MPI_Session_get_pset_info_f08 = ompi_session_get_pset_info_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_SESSION_GET_PSET_INFO,
+                            mpi_session_get_pset_info,
+                            mpi_session_get_pset_info_,
+                            mpi_session_get_pset_info__,
+                            ompi_session_get_pset_info_f,
+                            (MPI_Fint *session, char *pset_name, MPI_Fint *info, MPI_Fint *ierr, int name_len),
+                            (session, pset_name, info, ierr, name_len) )
+#else
+#define ompi_session_get_pset_info_f pompi_session_get_pset_info_f
+#endif
+#endif
+
+void ompi_session_get_pset_info_f(MPI_Fint *session,char *pset_name, MPI_Fint *info, MPI_Fint *ierr, int name_len)
+{
+    int c_ierr, ret;
+    MPI_Session c_session;
+    char *c_name = NULL;
+    MPI_Info c_info;
+
+    c_session = PMPI_Session_f2c(*session);
+
+    /* Convert the fortran string */
+
+    if (OMPI_SUCCESS != (ret = ompi_fortran_string_f2c(pset_name, name_len,
+                                                       &c_name))) {
+        c_ierr = OMPI_ERRHANDLER_INVOKE((ompi_instance_t *)c_session, ret,
+                                        "MPI_SESSION_GET_PSET_INFO");
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        return;
+    }
+
+    c_ierr = PMPI_Session_get_pset_info(c_session, c_name, &c_info);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (NULL != c_name) {
+        free (c_name);
+    }
+
+    if (MPI_SUCCESS == c_ierr) {
+        *info = PMPI_Info_c2f(c_info);
+    }
+}
+
+
diff --git a/ompi/mpi/fortran/mpif-h/session_init_f.c b/ompi/mpi/fortran/mpif-h/session_init_f.c
new file mode 100644
index 00000000000..b36a324f09d
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/session_init_f.c
@@ -0,0 +1,89 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2013 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2019      Triad National Security, LLC.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_SESSION_INIT = ompi_session_init_f
+#pragma weak pmpi_session_init = ompi_session_init_f
+#pragma weak pmpi_session_init_ = ompi_session_init_f
+#pragma weak pmpi_session_init__ = ompi_session_init_f
+
+#pragma weak PMPI_Session_init_f = ompi_session_init_f
+#pragma weak PMPI_Session_init_f08 = ompi_session_init_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_SESSION_INIT,
+                            pmpi_session_init,
+                            pmpi_session_init_,
+                            pmpi_session_init__,
+                            pompi_session_init_f,
+                            (MPI_Fint *info, MPI_Fint *errhandler, MPI_Fint *session, MPI_Fint *ierr),
+                            (info, errhandler, session, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_SESSION_INIT = ompi_session_init_f
+#pragma weak mpi_session_init = ompi_session_init_f
+#pragma weak mpi_session_init_ = ompi_session_init_f
+#pragma weak mpi_session_init__ = ompi_session_init_f
+
+#pragma weak MPI_Session_init_f = ompi_session_init_f
+#pragma weak MPI_Session_init_f08 = ompi_session_init_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_SESSION_INIT,
+                            mpi_session_init,
+                            mpi_session_init_,
+                            mpi_session_init__,
+                            ompi_session_init_f,
+                            (MPI_Fint *info, MPI_Fint *errhandler, MPI_Fint *session, MPI_Fint *ierr),
+                            (info, errhandler, session, ierr) )
+#else
+#define ompi_session_init_f pompi_session_init_f
+#endif
+#endif
+
+void ompi_session_init_f(MPI_Fint *info, MPI_Fint *errhandler, MPI_Fint *session, MPI_Fint *ierr)
+{
+    int c_ierr;
+    MPI_Session c_session;
+    MPI_Info c_info;
+    MPI_Errhandler c_errhandler;
+
+    c_info = PMPI_Info_f2c(*info);
+    c_errhandler = PMPI_Errhandler_f2c(*errhandler);
+
+    c_ierr = PMPI_Session_init(c_info, c_errhandler, &c_session);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *session = PMPI_Session_c2f(c_session);
+    }
+}
diff --git a/ompi/mpi/fortran/mpif-h/session_set_errhandler_f.c b/ompi/mpi/fortran/mpif-h/session_set_errhandler_f.c
new file mode 100644
index 00000000000..e90a2895063
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/session_set_errhandler_f.c
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      Triad National Security, LLC.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/errhandler/errhandler.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPI_SESSION_SET_ERRHANDLER = ompi_session_set_errhandler_f
+#pragma weak pmpi_session_set_errhandler = ompi_session_set_errhandler_f
+#pragma weak pmpi_session_set_errhandler_ = ompi_session_set_errhandler_f
+#pragma weak pmpi_session_set_errhandler__ = ompi_session_set_errhandler_f
+
+#pragma weak PMPI_Session_set_errhandler_f = ompi_session_set_errhandler_f
+#pragma weak PMPI_Session_set_errhandler_f08 = ompi_session_set_errhandler_f
+#else
+OMPI_GENERATE_F77_BINDINGS (PMPI_SESSION_SET_ERRHANDLER,
+                           pmpi_session_set_errhandler,
+                           pmpi_session_set_errhandler_,
+                           pmpi_session_set_errhandler__,
+                           pompi_session_set_errhandler_f,
+                           (MPI_Fint *session, MPI_Fint *errhandler, MPI_Fint *ierr),
+                           (session, errhandler, ierr) )
+#endif
+#endif
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_SESSION_SET_ERRHANDLER = ompi_session_set_errhandler_f
+#pragma weak mpi_session_set_errhandler = ompi_session_set_errhandler_f
+#pragma weak mpi_session_set_errhandler_ = ompi_session_set_errhandler_f
+#pragma weak mpi_session_set_errhandler__ = ompi_session_set_errhandler_f
+
+#pragma weak MPI_Session_set_errhandler_f = ompi_session_set_errhandler_f
+#pragma weak MPI_Session_set_errhandler_f08 = ompi_session_set_errhandler_f
+#else
+#if ! OMPI_BUILD_MPI_PROFILING
+OMPI_GENERATE_F77_BINDINGS (MPI_SESSION_SET_ERRHANDLER,
+                           mpi_session_set_errhandler,
+                           mpi_session_set_errhandler_,
+                           mpi_session_set_errhandler__,
+                           ompi_session_set_errhandler_f,
+                           (MPI_Fint *session, MPI_Fint *errhandler, MPI_Fint *ierr),
+                           (session, errhandler, ierr) )
+#else
+#define ompi_session_set_errhandler_f pompi_session_set_errhandler_f
+#endif
+#endif
+
+
+void ompi_session_set_errhandler_f(MPI_Fint *session, MPI_Fint *errhandler,
+			       MPI_Fint *ierr)
+{
+    int c_ierr;
+    MPI_Session c_session;
+    MPI_Errhandler c_errhandler;
+
+    c_session = PMPI_Session_f2c(*session);
+    c_errhandler = PMPI_Errhandler_f2c(*errhandler);
+
+    c_ierr = PMPI_Session_set_errhandler(c_session, c_errhandler);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/mpif-h/type_create_keyval_f.c b/ompi/mpi/fortran/mpif-h/type_create_keyval_f.c
index 30f365ea549..22074a8d93f 100644
--- a/ompi/mpi/fortran/mpif-h/type_create_keyval_f.c
+++ b/ompi/mpi/fortran/mpif-h/type_create_keyval_f.c
@@ -34,13 +34,13 @@
 #pragma weak PMPI_Type_create_keyval_f = ompi_type_create_keyval_f
 #pragma weak PMPI_Type_create_keyval_f08 = ompi_type_create_keyval_f
 #else
-OMPI_GENERATE_F77_BINDINGS (PMPI_TYPE_CREATE_KEYVAL,
-                           pmpi_type_create_keyval,
-                           pmpi_type_create_keyval_,
-                           pmpi_type_create_keyval__,
+OMPI_GENERATE_F77_BINDINGS(PMPI_TYPE_CREATE_KEYVAL, pmpi_type_create_keyval,
+                           pmpi_type_create_keyval_, pmpi_type_create_keyval__,
                            pompi_type_create_keyval_f,
-                           (ompi_aint_copy_attr_function* type_copy_attr_fn, ompi_aint_delete_attr_function* type_delete_attr_fn, MPI_Fint *type_keyval, MPI_Aint *extra_state, MPI_Fint *ierr),
-                           (type_copy_attr_fn, type_delete_attr_fn, type_keyval, extra_state, ierr) )
+                           (ompi_aint_copy_attr_function type_copy_attr_fn,
+                            ompi_aint_delete_attr_function type_delete_attr_fn,
+                            MPI_Fint *type_keyval, MPI_Aint *extra_state, MPI_Fint *ierr),
+                           (type_copy_attr_fn, type_delete_attr_fn, type_keyval, extra_state, ierr))
 #endif
 #endif
 
@@ -54,13 +54,12 @@ OMPI_GENERATE_F77_BINDINGS (PMPI_TYPE_CREATE_KEYVAL,
 #pragma weak MPI_Type_create_keyval_f08 = ompi_type_create_keyval_f
 #else
 #if ! OMPI_BUILD_MPI_PROFILING
-OMPI_GENERATE_F77_BINDINGS (MPI_TYPE_CREATE_KEYVAL,
-                           mpi_type_create_keyval,
-                           mpi_type_create_keyval_,
-                           mpi_type_create_keyval__,
-                           ompi_type_create_keyval_f,
-                           (ompi_aint_copy_attr_function* type_copy_attr_fn, ompi_aint_delete_attr_function* type_delete_attr_fn, MPI_Fint *type_keyval, MPI_Aint *extra_state, MPI_Fint *ierr),
-                           (type_copy_attr_fn, type_delete_attr_fn, type_keyval, extra_state, ierr) )
+OMPI_GENERATE_F77_BINDINGS(MPI_TYPE_CREATE_KEYVAL, mpi_type_create_keyval, mpi_type_create_keyval_,
+                           mpi_type_create_keyval__, ompi_type_create_keyval_f,
+                           (ompi_aint_copy_attr_function type_copy_attr_fn,
+                            ompi_aint_delete_attr_function type_delete_attr_fn,
+                            MPI_Fint *type_keyval, MPI_Aint *extra_state, MPI_Fint *ierr),
+                           (type_copy_attr_fn, type_delete_attr_fn, type_keyval, extra_state, ierr))
 #else
 #define ompi_type_create_keyval_f pompi_type_create_keyval_f
 #endif
@@ -68,8 +67,8 @@ OMPI_GENERATE_F77_BINDINGS (MPI_TYPE_CREATE_KEYVAL,
 
 static char FUNC_NAME[] = "MPI_Type_create_keyval_f";
 
-void ompi_type_create_keyval_f(ompi_aint_copy_attr_function* type_copy_attr_fn,
-                               ompi_aint_delete_attr_function* type_delete_attr_fn,
+void ompi_type_create_keyval_f(ompi_aint_copy_attr_function type_copy_attr_fn,
+                               ompi_aint_delete_attr_function type_delete_attr_fn,
                                MPI_Fint *type_keyval, MPI_Aint *extra_state, MPI_Fint *ierr)
 {
     int ret, c_ierr;
diff --git a/ompi/mpi/fortran/mpif-h/type_get_envelope_f_c.c b/ompi/mpi/fortran/mpif-h/type_get_envelope_f_c.c
new file mode 100644
index 00000000000..ba088345f57
--- /dev/null
+++ b/ompi/mpi/fortran/mpif-h/type_get_envelope_f_c.c
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+
+void ompi_type_get_envelope_f_c(MPI_Fint *type, MPI_Count *num_integers,
+			     MPI_Count *num_addresses,
+			     MPI_Count *num_large_counts,
+			     MPI_Count *num_datatypes, MPI_Fint *combiner,
+			     MPI_Fint *ierr)
+{
+    int c_ierr;
+    MPI_Datatype c_type = PMPI_Type_f2c(*type);
+    OMPI_SINGLE_NAME_DECL(num_integers);
+    OMPI_SINGLE_NAME_DECL(num_addresses);
+    OMPI_SINGLE_NAME_DECL(num_datatypes);
+    OMPI_SINGLE_NAME_DECL(combiner);
+
+    c_ierr = PMPI_Type_get_envelope_c(c_type,
+                                   OMPI_SINGLE_NAME_CONVERT(num_integers),
+                                   OMPI_SINGLE_NAME_CONVERT(num_addresses),
+                                   OMPI_SINGLE_NAME_CONVERT(num_large_counts),
+                                   OMPI_SINGLE_NAME_CONVERT(num_datatypes),
+                                   OMPI_SINGLE_NAME_CONVERT(combiner));
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        OMPI_SINGLE_INT_2_FINT(num_integers);
+        OMPI_SINGLE_INT_2_FINT(num_addresses);
+        OMPI_SINGLE_INT_2_FINT(num_datatypes);
+        OMPI_SINGLE_INT_2_FINT(combiner);
+    }
+}
diff --git a/ompi/mpi/fortran/mpif-h/win_create_errhandler_f.c b/ompi/mpi/fortran/mpif-h/win_create_errhandler_f.c
index aae4adc3bd7..c46bdcb9bc3 100644
--- a/ompi/mpi/fortran/mpif-h/win_create_errhandler_f.c
+++ b/ompi/mpi/fortran/mpif-h/win_create_errhandler_f.c
@@ -73,8 +73,9 @@ static const char FUNC_NAME[] = "MPI_WIN_CREATE_ERRHANDLER";
 void ompi_win_create_errhandler_f(ompi_errhandler_fortran_handler_fn_t* function,
 				 MPI_Fint *errhandler, MPI_Fint *ierr)
 {
-    MPI_Errhandler c_errhandler =
-        ompi_errhandler_create(OMPI_ERRHANDLER_TYPE_WIN,
+    MPI_Errhandler c_errhandler;
+
+    c_errhandler = ompi_errhandler_create(OMPI_ERRHANDLER_TYPE_WIN,
                                (ompi_errhandler_generic_handler_fn_t*) function,
                                OMPI_ERRHANDLER_LANG_FORTRAN);
     if (MPI_ERRHANDLER_NULL != c_errhandler) {
diff --git a/ompi/mpi/fortran/mpif-h/win_create_keyval_f.c b/ompi/mpi/fortran/mpif-h/win_create_keyval_f.c
index aefb8eb0641..8ce9842806a 100644
--- a/ompi/mpi/fortran/mpif-h/win_create_keyval_f.c
+++ b/ompi/mpi/fortran/mpif-h/win_create_keyval_f.c
@@ -34,13 +34,12 @@
 #pragma weak PMPI_Win_create_keyval_f = ompi_win_create_keyval_f
 #pragma weak PMPI_Win_create_keyval_f08 = ompi_win_create_keyval_f
 #else
-OMPI_GENERATE_F77_BINDINGS (PMPI_WIN_CREATE_KEYVAL,
-                           pmpi_win_create_keyval,
-                           pmpi_win_create_keyval_,
-                           pmpi_win_create_keyval__,
-                           pompi_win_create_keyval_f,
-                           (ompi_aint_copy_attr_function* win_copy_attr_fn, ompi_aint_delete_attr_function* win_delete_attr_fn, MPI_Fint *win_keyval, MPI_Aint *extra_state, MPI_Fint *ierr),
-                           (win_copy_attr_fn, win_delete_attr_fn, win_keyval, extra_state, ierr) )
+OMPI_GENERATE_F77_BINDINGS(PMPI_WIN_CREATE_KEYVAL, pmpi_win_create_keyval, pmpi_win_create_keyval_,
+                           pmpi_win_create_keyval__, pompi_win_create_keyval_f,
+                           (ompi_aint_copy_attr_function win_copy_attr_fn,
+                            ompi_aint_delete_attr_function win_delete_attr_fn, MPI_Fint *win_keyval,
+                            MPI_Aint *extra_state, MPI_Fint *ierr),
+                           (win_copy_attr_fn, win_delete_attr_fn, win_keyval, extra_state, ierr))
 #endif
 #endif
 
@@ -54,13 +53,12 @@ OMPI_GENERATE_F77_BINDINGS (PMPI_WIN_CREATE_KEYVAL,
 #pragma weak MPI_Win_create_keyval_f08 = ompi_win_create_keyval_f
 #else
 #if ! OMPI_BUILD_MPI_PROFILING
-OMPI_GENERATE_F77_BINDINGS (MPI_WIN_CREATE_KEYVAL,
-                           mpi_win_create_keyval,
-                           mpi_win_create_keyval_,
-                           mpi_win_create_keyval__,
-                           ompi_win_create_keyval_f,
-                           (ompi_aint_copy_attr_function* win_copy_attr_fn, ompi_aint_delete_attr_function* win_delete_attr_fn, MPI_Fint *win_keyval, MPI_Aint *extra_state, MPI_Fint *ierr),
-                           (win_copy_attr_fn, win_delete_attr_fn, win_keyval, extra_state, ierr) )
+OMPI_GENERATE_F77_BINDINGS(MPI_WIN_CREATE_KEYVAL, mpi_win_create_keyval, mpi_win_create_keyval_,
+                           mpi_win_create_keyval__, ompi_win_create_keyval_f,
+                           (ompi_aint_copy_attr_function win_copy_attr_fn,
+                            ompi_aint_delete_attr_function win_delete_attr_fn, MPI_Fint *win_keyval,
+                            MPI_Aint *extra_state, MPI_Fint *ierr),
+                           (win_copy_attr_fn, win_delete_attr_fn, win_keyval, extra_state, ierr))
 #else
 #define ompi_win_create_keyval_f pompi_win_create_keyval_f
 #endif
@@ -68,9 +66,9 @@ OMPI_GENERATE_F77_BINDINGS (MPI_WIN_CREATE_KEYVAL,
 
 static char FUNC_NAME[] = "MPI_Win_create_keyval";
 
-void ompi_win_create_keyval_f(ompi_aint_copy_attr_function* win_copy_attr_fn,
-                             ompi_aint_delete_attr_function* win_delete_attr_fn,
-                             MPI_Fint *win_keyval, MPI_Aint *extra_state, MPI_Fint *ierr)
+void ompi_win_create_keyval_f(ompi_aint_copy_attr_function win_copy_attr_fn,
+                              ompi_aint_delete_attr_function win_delete_attr_fn,
+                              MPI_Fint *win_keyval, MPI_Aint *extra_state, MPI_Fint *ierr)
 {
     int ret, c_ierr;
     OMPI_SINGLE_NAME_DECL(win_keyval);
diff --git a/ompi/mpi/fortran/use-mpi-f08/Makefile.am b/ompi/mpi/fortran/use-mpi-f08/Makefile.am
index 5ba1c4da486..98e1826a1f4 100644
--- a/ompi/mpi/fortran/use-mpi-f08/Makefile.am
+++ b/ompi/mpi/fortran/use-mpi-f08/Makefile.am
@@ -7,12 +7,14 @@
 # Copyright (c) 2012-2013 Inria.  All rights reserved.
 # Copyright (c) 2013-2018 Los Alamos National Security, LLC. All rights
 #                         reserved.
-# Copyright (c) 2015-2020 Research Organization for Information Science
+# Copyright (c) 2015-2021 Research Organization for Information Science
 #                         and Technology (RIST).  All rights reserved.
 # Copyright (c) 2016      IBM Corporation.  All rights reserved.
 # Copyright (c) 2017-2018 FUJITSU LIMITED.  All rights reserved.
-# Copyright (c) 2019      Triad National Security, LLC. All rights
+# Copyright (c) 2019-2022 Triad National Security, LLC. All rights
 #                         reserved.
+# Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+# Copyright (c) 2022      IBM Corporation.  All rights reserved.
 #
 # $COPYRIGHT$
 #
@@ -21,8 +23,6 @@
 # $HEADER$
 #
 
-SUBDIRS = profile
-
 include $(top_srcdir)/Makefile.ompi-rules
 
 # Note that Automake's Fortran-buidling rules uses CPPFLAGS and
@@ -34,23 +34,37 @@ AM_CPPFLAGS =
 
 # This Makefile is only relevant if we're building the "use mpi_f08"
 # MPI bindings.
+
+# Profiling entry points are generated using a combination of the
+# the mod/mpi-f08-rename.h file and the OMPI_BUILD_MPI_PROFILING
+# define flag.  This approach is used both for the generated
+# F08 entry points like MPI_SEND and ones that are still built
+# using *.F90 files like COMM_SET_ERRHANDLER.
+#
+# When adding a new function, if it involves big count arguments and/or
+# involves a user supplied buffer(s), and hence the need to generate
+# either assumed rank or assumed shape arrays with possible async
+# attributes, the *.c.in method needs to be used. Otherwise a *.F90
+# file can be used.   In addition, the mod/mpi-f08-rename.h file 
+# will need to be updated to include a rename line for the new method.
+#
 if OMPI_BUILD_FORTRAN_USEMPIF08_BINDINGS
 
 AM_FCFLAGS = -I$(top_srcdir)/ompi/mpi/fortran/use-mpi-f08/mod \
              -I$(top_builddir)/ompi/include \
              -I$(top_srcdir)/ompi/include \
              $(OMPI_FC_MODULE_FLAG)$(top_builddir)/ompi/mpi/fortran/use-mpi \
-             $(OMPI_FC_MODULE_FLAG)$(top_builddir)/ompi/$(OMPI_FORTRAN_USEMPI_DIR) \
+             $(OMPI_FC_MODULE_FLAG)$(top_builddir)/ompi/mpi/fortran/use-mpi-ignore-tkr \
              $(OMPI_FC_MODULE_FLAG)mod \
              $(OMPI_FC_MODULE_FLAG)bindings \
-             -I$(top_srcdir) -I$(top_builddir) $(FCFLAGS_f90) \
-             -DOMPI_BUILD_MPI_PROFILING=0
+             -I$(top_srcdir) -I$(top_builddir) $(FCFLAGS_f90)
 
 MOSTLYCLEANFILES = *.mod
 
 CLEANFILES += *.i90
 
 lib_LTLIBRARIES = lib@OMPI_LIBMPI_NAME@_usempif08.la
+noinst_LTLIBRARIES = lib@OMPI_LIBMPI_NAME@_usempif08_profile.la
 
 module_sentinel_files = \
         mod/libforce_usempif08_internal_modules_to_be_built.la \
@@ -75,9 +89,12 @@ sizeof_f08.h:
 	    --maxrank=$(OMPI_FORTRAN_MAX_ARRAY_RANK) \
 	    --generate=$(OMPI_FORTRAN_BUILD_SIZEOF) \
 	    --real2=$(OMPI_HAVE_FORTRAN_REAL2) \
+	    --iso_real16=$(OMPI_FORTRAN_HAVE_ISO_FORTRAN_ENV_REAL16) \
 	    --real16=$(OMPI_HAVE_FORTRAN_REAL16) \
 	    --complex4=$(OMPI_HAVE_FORTRAN_COMPLEX4) \
-	    --complex32=$(OMPI_HAVE_FORTRAN_COMPLEX32)
+	    --complex32=$(OMPI_HAVE_FORTRAN_COMPLEX32) \
+	    --mpi_version=$(MPI_VERSION) \
+	    --request_deprecate=$(OMPI_FORTRAN_HAVE_ATTR_DEPRECATED)
 
 sizeof_f08.f90: $(top_builddir)/config.status
 sizeof_f08.f90: $(sizeof_pl)
@@ -87,45 +104,36 @@ sizeof_f08.f90:
 	    --maxrank=$(OMPI_FORTRAN_MAX_ARRAY_RANK) \
 	    --generate=$(OMPI_FORTRAN_BUILD_SIZEOF) \
 	    --real2=$(OMPI_HAVE_FORTRAN_REAL2) \
+	    --iso_real16=$(OMPI_FORTRAN_HAVE_ISO_FORTRAN_ENV_REAL16) \
 	    --real16=$(OMPI_HAVE_FORTRAN_REAL16) \
 	    --complex4=$(OMPI_HAVE_FORTRAN_COMPLEX4) \
 	    --complex32=$(OMPI_HAVE_FORTRAN_COMPLEX32)
 
-profile/psizeof_f08.f90: $(top_builddir)/config.status
-profile/psizeof_f08.f90: $(sizeof_pl)
-profile/psizeof_f08.f90:
+psizeof_f08.f90: $(top_builddir)/config.status
+psizeof_f08.f90: $(sizeof_pl)
+psizeof_f08.f90:
 	$(OMPI_V_GEN) $(sizeof_pl) \
 	    --impl=$@ --ierror=optional --pmpi \
 	    --maxrank=$(OMPI_FORTRAN_MAX_ARRAY_RANK) \
 	    --generate=$(OMPI_FORTRAN_BUILD_SIZEOF) \
 	    --real2=$(OMPI_HAVE_FORTRAN_REAL2) \
+	    --iso_real16=$(OMPI_FORTRAN_HAVE_ISO_FORTRAN_ENV_REAL16) \
 	    --real16=$(OMPI_HAVE_FORTRAN_REAL16) \
 	    --complex4=$(OMPI_HAVE_FORTRAN_COMPLEX4) \
 	    --complex32=$(OMPI_HAVE_FORTRAN_COMPLEX32)
 
-CLEANFILES += sizeof_f08.h sizeof_f08.f90 profile/psizeof_f08.f90
+CLEANFILES += sizeof_f08.h sizeof_f08.f90 psizeof_f08.f90
 
 mpi_api_files = \
         abort_f08.F90 \
-        accumulate_f08.F90 \
         add_error_class_f08.F90 \
         add_error_code_f08.F90 \
         add_error_string_f08.F90 \
         aint_add_f08.F90 \
         aint_diff_f08.F90 \
-        allgather_f08.F90 \
-        allgatherv_f08.F90 \
         alloc_mem_f08.F90 \
-        allreduce_f08.F90 \
-        alltoall_f08.F90 \
-        alltoallv_f08.F90 \
-        alltoallw_f08.F90 \
         barrier_f08.F90 \
-        bcast_f08.F90 \
-        bsend_f08.F90 \
-        bsend_init_f08.F90 \
-        buffer_attach_f08.F90 \
-        buffer_detach_f08.F90 \
+        barrier_init_f08.F90 \
         cancel_f08.F90 \
         cart_coords_f08.F90 \
         cart_create_f08.F90 \
@@ -142,6 +150,7 @@ mpi_api_files = \
         comm_connect_f08.F90 \
         comm_create_errhandler_f08.F90 \
         comm_create_f08.F90 \
+        comm_create_from_group_f08.F90 \
         comm_create_group_f08.F90 \
         comm_create_keyval_f08.F90 \
         comm_delete_attr_f08.F90 \
@@ -149,6 +158,7 @@ mpi_api_files = \
         comm_dup_f08.F90 \
         comm_dup_with_info_f08.F90 \
         comm_idup_f08.F90 \
+        comm_idup_with_info_f08.F90 \
         comm_free_f08.F90 \
         comm_free_keyval_f08.F90 \
         comm_get_attr_f08.F90 \
@@ -171,7 +181,6 @@ mpi_api_files = \
         comm_split_f08.F90 \
         comm_split_type_f08.F90 \
         comm_test_inter_f08.F90 \
-        compare_and_swap_f08.F90 \
         dist_graph_create_adjacent_f08.F90 \
         dist_graph_create_f08.F90 \
         dist_graph_neighbors_count_f08.F90 \
@@ -180,9 +189,6 @@ mpi_api_files = \
         errhandler_free_f08.F90 \
         error_class_f08.F90 \
         error_string_f08.F90 \
-        exscan_f08.F90 \
-        f_sync_reg_f08.F90 \
-        fetch_and_op_f08.F90 \
         file_call_errhandler_f08.F90 \
         file_close_f08.F90 \
         file_create_errhandler_f08.F90 \
@@ -198,30 +204,8 @@ mpi_api_files = \
         file_get_size_f08.F90 \
         file_get_type_extent_f08.F90 \
         file_get_view_f08.F90 \
-        file_iread_at_f08.F90 \
-        file_iread_f08.F90 \
-        file_iread_at_all_f08.F90 \
-        file_iread_all_f08.F90 \
-        file_iread_shared_f08.F90 \
-        file_iwrite_at_f08.F90 \
-        file_iwrite_f08.F90 \
-        file_iwrite_at_all_f08.F90 \
-        file_iwrite_all_f08.F90 \
-        file_iwrite_shared_f08.F90 \
         file_open_f08.F90 \
         file_preallocate_f08.F90 \
-        file_read_all_begin_f08.F90 \
-        file_read_all_end_f08.F90 \
-        file_read_all_f08.F90 \
-        file_read_at_all_begin_f08.F90 \
-        file_read_at_all_end_f08.F90 \
-        file_read_at_all_f08.F90 \
-        file_read_at_f08.F90 \
-        file_read_f08.F90 \
-        file_read_ordered_begin_f08.F90 \
-        file_read_ordered_end_f08.F90 \
-        file_read_ordered_f08.F90 \
-        file_read_shared_f08.F90 \
         file_seek_f08.F90 \
         file_seek_shared_f08.F90 \
         file_set_atomicity_f08.F90 \
@@ -230,29 +214,9 @@ mpi_api_files = \
         file_set_size_f08.F90 \
         file_set_view_f08.F90 \
         file_sync_f08.F90 \
-        file_write_all_begin_f08.F90 \
-        file_write_all_end_f08.F90 \
-        file_write_all_f08.F90 \
-        file_write_at_all_begin_f08.F90 \
-        file_write_at_all_end_f08.F90 \
-        file_write_at_all_f08.F90 \
-        file_write_at_f08.F90 \
-        file_write_f08.F90 \
-        file_write_ordered_begin_f08.F90 \
-        file_write_ordered_end_f08.F90 \
-        file_write_ordered_f08.F90 \
-        file_write_shared_f08.F90 \
         finalized_f08.F90 \
         finalize_f08.F90 \
-        free_mem_f08.F90 \
-        gather_f08.F90 \
-        gatherv_f08.F90 \
-        get_accumulate_f08.F90 \
-        get_address_f08.F90 \
-        get_count_f08.F90 \
-        get_elements_f08.F90 \
         get_elements_x_f08.F90 \
-        get_f08.F90 \
         get_library_version_f08.F90 \
         get_processor_name_f08.F90 \
         get_version_f08.F90 \
@@ -268,6 +232,7 @@ mpi_api_files = \
         group_difference_f08.F90 \
         group_excl_f08.F90 \
         group_free_f08.F90 \
+        group_from_session_pset_f08.F90 \
         group_incl_f08.F90 \
         group_intersection_f08.F90 \
         group_range_excl_f08.F90 \
@@ -276,26 +241,10 @@ mpi_api_files = \
         group_size_f08.F90 \
         group_translate_ranks_f08.F90 \
         group_union_f08.F90 \
-        iallgather_f08.F90 \
-        iallgatherv_f08.F90 \
-        iallreduce_f08.F90 \
-        ialltoall_f08.F90 \
-        ialltoallv_f08.F90 \
-        ialltoallw_f08.F90 \
         ibarrier_f08.F90 \
-        ibcast_f08.F90 \
-        ibsend_f08.F90 \
-        iexscan_f08.F90 \
-        igather_f08.F90 \
-        igatherv_f08.F90 \
         improbe_f08.F90 \
-        imrecv_f08.F90 \
-        ineighbor_allgather_f08.F90 \
-        ineighbor_allgatherv_f08.F90 \
-        ineighbor_alltoall_f08.F90 \
-        ineighbor_alltoallv_f08.F90 \
-        ineighbor_alltoallw_f08.F90 \
         info_create_f08.F90 \
+        info_create_env_f08.F90 \
         info_delete_f08.F90 \
         info_dup_f08.F90 \
         info_free_f08.F90 \
@@ -303,133 +252,87 @@ mpi_api_files = \
         info_get_nkeys_f08.F90 \
         info_get_nthkey_f08.F90 \
         info_get_valuelen_f08.F90 \
+        info_get_string_f08.F90 \
         info_set_f08.F90 \
         init_f08.F90 \
         initialized_f08.F90 \
         init_thread_f08.F90 \
         intercomm_create_f08.F90 \
+        intercomm_create_from_groups_f08.F90 \
         intercomm_merge_f08.F90 \
         iprobe_f08.F90 \
-        irecv_f08.F90 \
-        ireduce_f08.F90 \
-        ireduce_scatter_f08.F90 \
-        ireduce_scatter_block_f08.F90 \
-        irsend_f08.F90 \
-        iscan_f08.F90 \
-        iscatter_f08.F90 \
-        iscatterv_f08.F90 \
-        isend_f08.F90 \
-        issend_f08.F90 \
         is_thread_main_f08.F90 \
         lookup_name_f08.F90 \
         mprobe_f08.F90 \
-        mrecv_f08.F90 \
-        neighbor_allgather_f08.F90 \
-        neighbor_allgatherv_f08.F90 \
-        neighbor_alltoall_f08.F90 \
-        neighbor_alltoallv_f08.F90 \
-        neighbor_alltoallw_f08.F90 \
         op_commutative_f08.F90 \
         op_create_f08.F90 \
         open_port_f08.F90 \
         op_free_f08.F90 \
-        pack_external_f08.F90 \
-        pack_external_size_f08.F90 \
-        pack_f08.F90 \
-        pack_size_f08.F90 \
+        parrived_f08.F90 \
         pcontrol_f08.F90 \
+        pready_f08.F90 \
+        pready_range_f08.F90 \
+        pready_list_f08.F90 \
         probe_f08.F90 \
         publish_name_f08.F90 \
-        put_f08.F90 \
         query_thread_f08.F90 \
-        raccumulate_f08.F90 \
-        recv_f08.F90 \
-        recv_init_f08.F90 \
-        reduce_f08.F90 \
-        reduce_local_f08.F90 \
-        reduce_scatter_f08.F90 \
-        reduce_scatter_block_f08.F90 \
         register_datarep_f08.F90 \
         request_free_f08.F90 \
         request_get_status_f08.F90 \
-        rget_f08.F90 \
-        rget_accumulate_f08.F90 \
-        rput_f08.F90 \
-        rsend_f08.F90 \
-        rsend_init_f08.F90 \
-        scan_f08.F90 \
-        scatter_f08.F90 \
-        scatterv_f08.F90 \
-        send_f08.F90 \
-        send_init_f08.F90 \
-        sendrecv_f08.F90 \
-        sendrecv_replace_f08.F90 \
-        ssend_f08.F90 \
-        ssend_init_f08.F90 \
+        session_call_errhandler_f08.F90\
+        session_create_errhandler_f08.F90\
+        session_get_errhandler_f08.F90\
+	session_get_info_f08.F90 \
+	session_get_nth_pset_f08.F90 \
+	session_get_num_psets_f08.F90 \
+	session_get_pset_info_f08.F90 \
+	session_init_f08.F90 \
+	session_finalize_f08.F90 \
+        session_set_errhandler_f08.F90\
         startall_f08.F90 \
         start_f08.F90 \
         status_f082f_f08.F90 \
         status_f2f08_f08.F90 \
         status_set_cancelled_f08.F90 \
-        status_set_elements_f08.F90 \
         status_set_elements_x_f08.F90 \
         testall_f08.F90 \
-        testany_f08.F90 \
         test_cancelled_f08.F90 \
         test_f08.F90 \
         testsome_f08.F90 \
         topo_test_f08.F90 \
         type_commit_f08.F90 \
-        type_contiguous_f08.F90 \
-        type_create_darray_f08.F90 \
         type_create_f90_complex_f08.F90 \
         type_create_f90_integer_f08.F90 \
         type_create_f90_real_f08.F90 \
-        type_create_hindexed_f08.F90 \
-        type_create_hvector_f08.F90 \
-        type_create_indexed_block_f08.F90 \
-        type_create_hindexed_block_f08.F90 \
         type_create_keyval_f08.F90 \
         type_create_resized_f08.F90 \
-        type_create_struct_f08.F90 \
-        type_create_subarray_f08.F90 \
         type_delete_attr_f08.F90 \
         type_dup_f08.F90 \
         type_free_f08.F90 \
         type_free_keyval_f08.F90 \
         type_get_attr_f08.F90 \
         type_get_contents_f08.F90 \
-        type_get_envelope_f08.F90 \
+        type_get_contents_f08_c.F90 \
         type_get_extent_f08.F90 \
         type_get_extent_x_f08.F90 \
         type_get_name_f08.F90 \
-        type_get_true_extent_f08.F90 \
         type_get_true_extent_x_f08.F90 \
-        type_indexed_f08.F90 \
+        type_get_envelope_f08.F90 \
+        type_get_envelope_f08_c.F90 \
         type_match_size_f08.F90 \
         type_set_attr_f08.F90 \
         type_set_name_f08.F90 \
-        type_size_f08.F90 \
         type_size_x_f08.F90 \
-        type_vector_f08.F90 \
-        unpack_external_f08.F90 \
-        unpack_f08.F90 \
         unpublish_name_f08.F90 \
-        waitall_f08.F90 \
         waitany_f08.F90 \
         wait_f08.F90 \
         waitsome_f08.F90 \
-        win_allocate_f08.F90 \
-        win_allocate_shared_f08.F90 \
-        win_attach_f08.F90 \
         win_call_errhandler_f08.F90 \
         win_complete_f08.F90 \
         win_create_dynamic_f08.F90 \
         win_create_errhandler_f08.F90 \
-        win_create_f08.F90 \
         win_create_keyval_f08.F90 \
         win_delete_attr_f08.F90 \
-        win_detach_f08.F90 \
         win_fence_f08.F90 \
         win_flush_f08.F90 \
         win_flush_all_f08.F90 \
@@ -449,34 +352,30 @@ mpi_api_files = \
         win_set_errhandler_f08.F90 \
         win_set_info_f08.F90 \
         win_set_name_f08.F90 \
-        win_shared_query_f08.F90 \
         win_start_f08.F90 \
         win_sync_f08.F90 \
         win_test_f08.F90 \
         win_unlock_f08.F90 \
         win_unlock_all_f08.F90 \
-        win_wait_f08.F90
-
-# JMS Somehow this variable substitution isn't quite working, and I
-# don't have time to figure it out.  So just wholesale copy the file
-# list.  :-(
-#pmpi_api_files = $(mpi_api_files:%=profile/p%)
+        win_wait_f08.F90 \
+        api_f08_generated.F90
 
 lib@OMPI_LIBMPI_NAME@_usempif08_la_SOURCES = \
         $(mpi_api_files) \
         mpi-f08.F90
 
-# These are generated; do not ship them
-nodist_lib@OMPI_LIBMPI_NAME@_usempif08_la_SOURCES =
-
 if BUILD_FORTRAN_SIZEOF
 SIZEOF_H = sizeof_f08.h
-nodist_lib@OMPI_LIBMPI_NAME@_usempif08_la_SOURCES += \
+nodist_lib@OMPI_LIBMPI_NAME@_usempif08_la_SOURCES = \
         sizeof_f08.h \
         sizeof_f08.f90 \
-        profile/psizeof_f08.f90
+        psizeof_f08.f90
 endif
 
+lib@OMPI_LIBMPI_NAME@_usempif08_la_FCFLAGS = \
+        $(AM_FCFLAGS) \
+        -DOMPI_BUILD_MPI_PROFILING=0
+
 #
 # Include the mpi_f08-based MPI extensions in libmpi_usempif08, too.
 #
@@ -486,13 +385,20 @@ endif
 #
 
 lib@OMPI_LIBMPI_NAME@_usempif08_la_LIBADD = \
-        profile/libmpi_usempif08_pmpi.la \
+        lib@OMPI_LIBMPI_NAME@_usempif08_profile.la \
         $(OMPI_MPIEXT_USEMPIF08_LIBS) \
         $(top_builddir)/ompi/mpi/fortran/mpif-h/lib@OMPI_LIBMPI_NAME@_mpifh.la \
         $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
         mod/libusempif08_internal_modules.la \
         base/libusempif08_ccode.la
-lib@OMPI_LIBMPI_NAME@_usempif08_la_DEPENDENCIES = $(module_sentinel_files)
+
+#
+# Make sure to build the profile library before this library, since adding it
+# to LIBADD doesn't enforce any ordering
+#
+lib@OMPI_LIBMPI_NAME@_usempif08_la_DEPENDENCIES = \
+        $(module_sentinel_files) \
+        lib@OMPI_LIBMPI_NAME@_usempif08_profile.la
 lib@OMPI_LIBMPI_NAME@_usempif08_la_LDFLAGS = -version-info $(libmpi_usempif08_so_version)
 
 #
@@ -505,6 +411,51 @@ mpi_api_lo_files = $(mpi_api_files:.F90=.lo)
 $(mpi_api_lo_files): bindings/libforce_usempif08_internal_bindings_to_be_built.la
 
 mpi-f08.lo: $(module_sentinel_files) $(SIZEOF_H)
+mpi-f08.F90: $(SIZEOF_H)
+
+#
+# Profiling interface
+#
+
+lib@OMPI_LIBMPI_NAME@_usempif08_profile_la_SOURCES = \
+        $(mpi_api_files)
+
+lib@OMPI_LIBMPI_NAME@_usempif08_profile_la_FCFLAGS = \
+        $(AM_FCFLAGS) \
+        -DOMPI_BUILD_MPI_PROFILING=1
+
+
+#
+# Generate the Fortran bindings and C wrapper functions for bindings with a
+# *.in template.
+#
+
+if OMPI_GENERATE_BINDINGS
+
+include Makefile.prototype_files
+template_files =${prototype_files:%=$(abs_top_srcdir)/ompi/mpi/fortran/use-mpi-f08/%}
+
+if OMPI_FORTRAN_HAVE_TS
+gen_ts = --generate-ts-suffix
+endif
+
+api_f08_generated.F90: $(template_files)
+	$(OMPI_V_GEN) $(PYTHON) $(top_srcdir)/ompi/mpi/bindings/bindings.py \
+	    --builddir $(abs_top_builddir) \
+	    --srcdir $(abs_top_srcdir) \
+	    --output $(abs_builddir)/$@ \
+	    fortran \
+	    $(gen_ts) \
+	    code \
+	    --lang fortran \
+	    --prototype-files $(template_files)
+
+EXTRA_DIST = $(prototype_files)
+
+endif
+
+# Delete generated file on maintainer-clean
+MAINTAINERCLEANFILES = api_f08_generated.F90
 
 ###########################################################################
 
@@ -513,14 +464,14 @@ mpi-f08.lo: $(module_sentinel_files) $(SIZEOF_H)
 
 install-exec-hook:
 	@ for file in `ls *.mod`; do \
-	  echo $(INSTALL) $$file $(DESTDIR)$(libdir); \
-	  $(INSTALL) $$file $(DESTDIR)$(libdir); \
+	  echo $(INSTALL) $$file $(DESTDIR)$(OMPI_FORTRAN_MODULEDIR); \
+	  $(INSTALL) $$file $(DESTDIR)$(OMPI_FORTRAN_MODULEDIR); \
 	done
 
 uninstall-local:
 	@ for file in `ls *.mod`; do \
-	  echo rm -f $(DESTDIR)$(libdir)/$$file; \
-	  rm -f $(DESTDIR)$(libdir)/$$file; \
+	  echo rm -f $(DESTDIR)$(OMPI_FORTRAN_MODULEDIR)/$$file; \
+	  rm -f $(DESTDIR)$(OMPI_FORTRAN_MODULEDIR)/$$file; \
 	done
 
 endif
diff --git a/ompi/mpi/fortran/use-mpi-f08/Makefile.prototype_files b/ompi/mpi/fortran/use-mpi-f08/Makefile.prototype_files
new file mode 100644
index 00000000000..f17f7edefa9
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/Makefile.prototype_files
@@ -0,0 +1,169 @@
+#
+# Shared list of prototype files to avoid listing dependencies multiple times.
+#
+
+prototype_files = \
+        accumulate_ts.c.in \
+        allgather_init_ts.c.in \
+        allgather_ts.c.in \
+        allgatherv_init_ts.c.in \
+        allgatherv_ts.c.in \
+        allreduce_init_ts.c.in \
+        allreduce_ts.c.in \
+        alltoall_init_ts.c.in \
+        alltoall_ts.c.in \
+        alltoallv_init_ts.c.in \
+        alltoallv_ts.c.in \
+        alltoallw_init_ts.c.in \
+        alltoallw_ts.c.in \
+        bcast_init_ts.c.in \
+        bcast_ts.c.in \
+        bsend_init_ts.c.in \
+        bsend_ts.c.in \
+        buffer_attach_ts.c.in \
+        buffer_detach.c.in \
+        compare_and_swap_ts.c.in \
+        exscan_init_ts.c.in \
+        exscan_ts.c.in \
+        fetch_and_op_ts.c.in \
+        file_iread_all_ts.c.in \
+        file_iread_at_all_ts.c.in \
+        file_iread_at_ts.c.in \
+        file_iread_shared_ts.c.in \
+        file_iread_ts.c.in \
+        file_iwrite_all_ts.c.in \
+        file_iwrite_at_all_ts.c.in \
+        file_iwrite_at_ts.c.in \
+        file_iwrite_shared_ts.c.in \
+        file_iwrite_ts.c.in \
+        file_read_all_begin_ts.c.in \
+        file_read_all_end_ts.c.in \
+        file_read_all_ts.c.in \
+        file_read_at_all_begin_ts.c.in \
+        file_read_at_all_end_ts.c.in \
+        file_read_at_all_ts.c.in \
+        file_read_at_ts.c.in \
+        file_read_ordered_begin_ts.c.in \
+        file_read_ordered_end_ts.c.in \
+        file_read_ordered_ts.c.in \
+        file_read_shared_ts.c.in \
+        file_read_ts.c.in \
+        file_write_all_begin_ts.c.in \
+        file_write_all_end_ts.c.in \
+        file_write_all_ts.c.in \
+        file_write_at_all_begin_ts.c.in \
+        file_write_at_all_end_ts.c.in \
+        file_write_at_all_ts.c.in \
+        file_write_at_ts.c.in \
+        file_write_ordered_begin_ts.c.in \
+        file_write_ordered_end_ts.c.in \
+        file_write_ordered_ts.c.in \
+        file_write_shared_ts.c.in \
+        file_write_ts.c.in \
+        free_mem_ts.c.in \
+	f_sync_reg_ts.c.in \
+        gather_init_ts.c.in \
+        gather_ts.c.in \
+        gatherv_init_ts.c.in \
+        gatherv_ts.c.in \
+        get_accumulate_ts.c.in \
+        get_address_ts.c.in \
+        get_count.c.in \
+        get_elements.c.in \
+        get_ts.c.in \
+        iallgather_ts.c.in \
+        iallgatherv_ts.c.in \
+        iallreduce_ts.c.in \
+        ialltoall_ts.c.in \
+        ialltoallv_ts.c.in \
+        ialltoallw_ts.c.in \
+        ibcast_ts.c.in \
+        ibsend_ts.c.in \
+        iexscan_ts.c.in \
+        igather_ts.c.in \
+        igatherv_ts.c.in \
+        imrecv_ts.c.in \
+        ineighbor_allgather_ts.c.in \
+        ineighbor_allgatherv_ts.c.in \
+        ineighbor_alltoall_ts.c.in \
+        ineighbor_alltoallv_ts.c.in \
+        ineighbor_alltoallw_ts.c.in \
+        irecv_ts.c.in \
+        ireduce_scatter_block_ts.c.in \
+        ireduce_scatter_ts.c.in \
+        ireduce_ts.c.in \
+        irsend_ts.c.in \
+        iscan_ts.c.in \
+        iscatter_ts.c.in \
+        iscatterv_ts.c.in \
+        isendrecv_replace_ts.c.in \
+        isendrecv_ts.c.in \
+        isend_ts.c.in \
+        issend_ts.c.in \
+        mrecv_ts.c.in \
+        neighbor_allgather_init_ts.c.in \
+        neighbor_allgather_ts.c.in \
+        neighbor_allgatherv_init_ts.c.in \
+        neighbor_allgatherv_ts.c.in \
+        neighbor_alltoall_init_ts.c.in \
+        neighbor_alltoall_ts.c.in \
+        neighbor_alltoallv_init_ts.c.in \
+        neighbor_alltoallv_ts.c.in \
+        neighbor_alltoallw_init_ts.c.in \
+        neighbor_alltoallw_ts.c.in \
+        pack_external_size.c.in \
+        pack_external_ts.c.in \
+        pack_size.c.in \
+        pack_ts.c.in \
+        precv_init_ts.c.in \
+        psend_init_ts.c.in \
+        put_ts.c.in \
+        raccumulate_ts.c.in \
+        recv_init_ts.c.in \
+        recv_ts.c.in \
+        reduce_init_ts.c.in \
+        reduce_local_ts.c.in \
+        reduce_scatter_block_init_ts.c.in \
+        reduce_scatter_block_ts.c.in \
+        reduce_scatter_init_ts.c.in \
+        reduce_scatter_ts.c.in \
+        reduce_ts.c.in \
+        rget_accumulate_ts.c.in \
+        rget_ts.c.in \
+        rput_ts.c.in \
+        rsend_init_ts.c.in \
+        rsend_ts.c.in \
+        scan_init_ts.c.in \
+        scan_ts.c.in \
+        scatter_init_ts.c.in \
+        scatter_ts.c.in \
+        scatterv_init_ts.c.in \
+        scatterv_ts.c.in \
+        send_init_ts.c.in \
+        sendrecv_replace_ts.c.in \
+        sendrecv_ts.c.in \
+        send_ts.c.in \
+        ssend_init_ts.c.in \
+        ssend_ts.c.in \
+        testany.c.in \
+        type_contiguous.c.in \
+        type_create_darray.c.in \
+        type_create_hindexed_block.c.in \
+        type_create_hindexed.c.in \
+        type_create_hvector.c.in \
+        type_create_indexed_block.c.in \
+        type_create_struct.c.in \
+        type_create_subarray.c.in \
+        type_get_true_extent.c.in \
+        type_indexed.c.in \
+        type_size.c.in \
+        type_vector.c.in \
+        unpack_external_ts.c.in \
+        unpack_ts.c.in \
+        waitall.c.in \
+        win_allocate.c.in \
+        win_allocate_shared.c.in \
+        win_attach_ts.c.in \
+        win_create_ts.c.in \
+        win_detach_ts.c.in \
+        win_shared_query.c.in
diff --git a/ompi/mpi/fortran/use-mpi-f08/accumulate_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/accumulate_f08.F90
deleted file mode 100644
index 4eacb2b48a8..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/accumulate_f08.F90
+++ /dev/null
@@ -1,35 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Accumulate_f08(origin_addr,origin_count,origin_datatype,&
-                              target_rank,target_disp,target_count,    &
-                              target_datatype,op,win,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Win, MPI_ADDRESS_KIND
-   use :: ompi_mpifh_bindings, only : ompi_accumulate_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: origin_addr
-   INTEGER, INTENT(IN) :: origin_count, target_rank, target_count
-   TYPE(MPI_Datatype), INTENT(IN) :: origin_datatype
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
-   TYPE(MPI_Datatype), INTENT(IN) :: target_datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Win), INTENT(IN) :: win
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_accumulate_f(origin_addr,origin_count,origin_datatype%MPI_VAL,target_rank,&
-                          target_disp,target_count,target_datatype%MPI_VAL,&
-                          op%MPI_VAL,win%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Accumulate_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/accumulate_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/accumulate_ts.c.in
new file mode 100644
index 00000000000..b2d9495b025
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/accumulate_ts.c.in
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID accumulate(BUFFER x, COUNT origin_count,
+                          DATATYPE origin_datatype, RANK target_rank,
+                          AINT target_disp, COUNT target_count,
+                          DATATYPE target_datatype, OP op, WIN win)
+{
+    int c_ierr;
+
+    MPI_Datatype c_origin_datatype, c_origin_type = PMPI_Type_f2c(*origin_datatype);
+    MPI_Datatype c_target_datatype = PMPI_Type_f2c(*target_datatype);
+    MPI_Win c_win = PMPI_Win_f2c(*win);
+    MPI_Op c_op = PMPI_Op_f2c(*op);
+    char *origin_addr = OMPI_CFI_BASE_ADDR(x);
+    int c_origin_count = OMPI_INT_2_FINT(*origin_count);
+
+    OMPI_CFI_2_C(x, c_origin_count, c_origin_type, c_origin_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_win, c_ierr, FUNC_NAME)
+        return;
+    }
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(origin_addr),
+                          c_origin_count,
+                          c_origin_datatype,
+                          OMPI_FINT_2_INT(*target_rank),
+                          *target_disp,
+                          OMPI_FINT_2_INT(*target_count),
+                          c_target_datatype, c_op, c_win);
+    if (c_origin_datatype != c_origin_type) {
+        ompi_datatype_destroy(&c_origin_datatype);
+    }
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/add_error_string_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/add_error_string_f08.F90
index 260d8b6313b..f6739346527 100644
--- a/ompi/mpi/fortran/use-mpi-f08/add_error_string_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/add_error_string_f08.F90
@@ -11,13 +11,14 @@
 
 subroutine MPI_Add_error_string_f08(errorcode,string,ierror)
    use :: ompi_mpifh_bindings, only : ompi_add_error_string_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
    implicit none
    integer, intent(in) :: errorcode
    character(len=*), intent(in) :: string
    integer, optional, intent(out) :: ierror
    integer :: c_ierror
 
-   call ompi_add_error_string_f(errorcode, string, c_ierror, len(string))
+   call ompi_add_error_string_f(errorcode, string, c_ierror, len(string,KIND=C_INT))
    if (present(ierror)) ierror = c_ierror
 
 end subroutine MPI_Add_error_string_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/allgather_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/allgather_f08.F90
deleted file mode 100644
index ed2aefbad59..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/allgather_f08.F90
+++ /dev/null
@@ -1,31 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Allgather_f08(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   use :: ompi_mpifh_bindings, only : ompi_allgather_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_allgather_f(sendbuf,sendcount,sendtype%MPI_VAL,&
-                         recvbuf,recvcount,recvtype%MPI_VAL,comm%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Allgather_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/allgather_init_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/allgather_init_ts.c.in
new file mode 100644
index 00000000000..b962d280136
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/allgather_init_ts.c.in
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+
+PROTOTYPE VOID allgather_init(BUFFER_ASYNC x1, COUNT sendcount, DATATYPE sendtype,
+                              BUFFER_ASYNC_OUT x2, COUNT recvcount, DATATYPE recvtype,
+                              COMM comm, INFO info, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Request c_req;
+    MPI_Datatype c_sendtype = NULL, c_recvtype = NULL, c_senddatatype = NULL;
+    MPI_Info c_info;
+    void *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_sendcount = (@COUNT_TYPE@) *sendcount;
+    @COUNT_TYPE@ c_recvcount = (@COUNT_TYPE@) *recvcount;
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_sendtype = PMPI_Type_f2c(*sendtype);
+    c_senddatatype = c_sendtype;
+    c_recvtype = PMPI_Type_f2c(*recvtype);
+    c_info = PMPI_Info_f2c(*info);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    if (OMPI_COMM_IS_INTER(c_comm) || !OMPI_IS_FORTRAN_IN_PLACE(sendbuf)) {
+        OMPI_CFI_2_C(x1, c_sendcount, c_sendtype, c_senddatatype, c_ierr);
+        if (MPI_SUCCESS != c_ierr) {
+            if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+            OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+            return;
+        }
+    } else {
+        sendbuf = MPI_IN_PLACE;
+    }
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          c_sendcount,
+                          c_senddatatype,
+                          recvbuf,
+                          c_recvcount,
+                          c_recvtype, c_comm, c_info, &c_req);
+
+    if (c_senddatatype != c_sendtype) {
+        ompi_datatype_destroy(&c_senddatatype);
+    }
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_req);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/allgather_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/allgather_ts.c.in
new file mode 100644
index 00000000000..6643849ee13
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/allgather_ts.c.in
@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+
+PROTOTYPE VOID allgather(BUFFER x1, COUNT sendcount, DATATYPE sendtype,
+                         BUFFER_OUT x2, COUNT recvcount, DATATYPE recvtype,
+                         COMM comm)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    @COUNT_TYPE@ c_sendcount = (@COUNT_TYPE@)*sendcount;
+    @COUNT_TYPE@ c_recvcount = (@COUNT_TYPE@)*recvcount;
+    MPI_Datatype c_sendtype = NULL, c_senddatatype = NULL;
+    MPI_Datatype c_recvtype = PMPI_Type_f2c(*recvtype);
+    void *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    if (OMPI_COMM_IS_INTER(c_comm) || !OMPI_IS_FORTRAN_IN_PLACE(sendbuf)) {
+        c_sendtype = PMPI_Type_f2c(*sendtype);
+        c_sendcount = OMPI_FINT_2_INT(*sendcount);
+        OMPI_CFI_2_C(x1, c_sendcount, c_sendtype, c_senddatatype, c_ierr);
+        if (MPI_SUCCESS != c_ierr) {
+            if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+            OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+            return;
+        }
+    } else {
+        sendbuf = MPI_IN_PLACE;
+    }
+
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          c_sendcount,
+                          c_senddatatype,
+                          recvbuf,
+                          c_recvcount,
+                          c_recvtype, c_comm);
+
+    if (c_senddatatype != c_sendtype) {
+        ompi_datatype_destroy(&c_senddatatype);
+    }
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/allgatherv_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/allgatherv_f08.F90
deleted file mode 100644
index 260b89a986b..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/allgatherv_f08.F90
+++ /dev/null
@@ -1,33 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Allgatherv_f08(sendbuf,sendcount,sendtype,recvbuf,recvcounts,&
-                              displs,recvtype,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   use :: ompi_mpifh_bindings, only : ompi_allgatherv_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount
-   INTEGER, INTENT(IN) :: recvcounts(*), displs(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_allgatherv_f(sendbuf,sendcount,sendtype%MPI_VAL,recvbuf,recvcounts,&
-                          displs,recvtype%MPI_VAL,comm%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Allgatherv_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/allgatherv_init_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/allgatherv_init_ts.c.in
new file mode 100644
index 00000000000..c559a5f107a
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/allgatherv_init_ts.c.in
@@ -0,0 +1,100 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID allgatherv_init(BUFFER_ASYNC x1, COUNT sendcount, DATATYPE sendtype,
+                               BUFFER_ASYNC_OUT x2, COUNT_ARRAY recvcounts, DISP_ARRAY displs,
+                               DATATYPE recvtype, COMM comm,
+                               INFO info, REQUEST_OUT request)
+{
+    MPI_Comm c_comm;
+    MPI_Datatype c_sendtype = NULL, c_senddatatype = NULL, c_recvtype = NULL;
+    MPI_Request c_request;
+    MPI_Info c_info;
+    int size, idx = 0, c_ierr;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_sendcount = (@COUNT_TYPE@) *sendcount;
+    @COUNT_TYPE@ *tmp_recvcounts = NULL;
+    @DISP_TYPE@ *tmp_displs = NULL;
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_sendtype = PMPI_Type_f2c(*sendtype);
+    c_recvtype = PMPI_Type_f2c(*recvtype);
+    c_info = PMPI_Info_f2c(*info);
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    if (OMPI_COMM_IS_INTER(c_comm)) {
+        size = ompi_comm_remote_size(c_comm);
+        c_sendtype = PMPI_Type_f2c(*sendtype);
+        OMPI_CFI_2_C(x1, c_sendcount, c_sendtype, c_senddatatype, c_ierr);
+        if (MPI_SUCCESS != c_ierr) {
+            if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+            OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+            return;
+        }
+    } else {
+        size = ompi_comm_size(c_comm);
+        if (OMPI_IS_FORTRAN_IN_PLACE(sendbuf)) {
+            sendbuf = MPI_IN_PLACE;
+        } else {
+            c_sendtype = PMPI_Type_f2c(*sendtype);
+            OMPI_CFI_2_C(x1, c_sendcount, c_sendtype, c_senddatatype, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+                return;
+            }
+        }
+    }
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(recvcounts, tmp_recvcounts, size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(displs, tmp_displs, size);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          c_sendcount,
+                          c_senddatatype,
+                          recvbuf,
+                          tmp_recvcounts,
+                          tmp_displs,
+                          c_recvtype, c_comm, c_info, &c_request);
+
+    if (c_senddatatype != c_sendtype) {
+        ompi_datatype_destroy(&c_senddatatype);
+    }
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_request);
+    }
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(recvcounts, tmp_recvcounts, c_request, c_ierr, idx);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(displs, tmp_displs, c_request, c_ierr, idx);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/allgatherv_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/allgatherv_ts.c.in
new file mode 100644
index 00000000000..bccc21b864e
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/allgatherv_ts.c.in
@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+
+PROTOTYPE VOID allgatherv(BUFFER x1, COUNT sendcount, DATATYPE sendtype,
+                          BUFFER_OUT x2, COUNT_ARRAY recvcounts, DISP_ARRAY displs,
+                          DATATYPE recvtype, COMM comm)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    int c_sendcount = 0;
+    MPI_Datatype c_sendtype = NULL, c_senddatatype = NULL, c_recvtype = PMPI_Type_f2c(*recvtype);
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    int size = 0;
+    @COUNT_TYPE@ *tmp_recvcounts = NULL;
+    @DISP_TYPE@ *tmp_displs = NULL;
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    if (OMPI_COMM_IS_INTER(c_comm)) {
+        size = ompi_comm_remote_size(c_comm);
+        c_sendtype = PMPI_Type_f2c(*sendtype);
+        c_sendcount = (@COUNT_TYPE@) *sendcount;
+        OMPI_CFI_2_C(x1, c_sendcount, c_sendtype, c_senddatatype, c_ierr);
+        if (MPI_SUCCESS != c_ierr) {
+            if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+            OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+            return;
+        }
+    } else {
+        size = ompi_comm_size(c_comm);
+        if (OMPI_IS_FORTRAN_IN_PLACE(sendbuf)) {
+            sendbuf = MPI_IN_PLACE;
+        } else {
+            c_sendtype = PMPI_Type_f2c(*sendtype);
+            c_sendcount = (@COUNT_TYPE@) *sendcount;
+            OMPI_CFI_2_C(x1, c_sendcount, c_sendtype, c_senddatatype, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+                return;
+            }
+        }
+    }
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(recvcounts, tmp_recvcounts, size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(displs, tmp_displs, size);
+
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          c_sendcount,
+                          c_senddatatype,
+                          recvbuf,
+                          tmp_recvcounts,
+                          tmp_displs,
+                          c_recvtype, c_comm);
+
+    if (c_senddatatype != c_sendtype) {
+        ompi_datatype_destroy(&c_senddatatype);
+    }
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(recvcounts, tmp_recvcounts);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(displs, tmp_displs);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/allreduce_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/allreduce_f08.F90
deleted file mode 100644
index 0e98b9e171a..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/allreduce_f08.F90
+++ /dev/null
@@ -1,31 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Allreduce_f08(sendbuf,recvbuf,count,datatype,op,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm
-   use :: ompi_mpifh_bindings, only : ompi_allreduce_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_allreduce_f(sendbuf,recvbuf,count,datatype%MPI_VAL,&
-                         op%MPI_VAL,comm%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Allreduce_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/allreduce_init_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/allreduce_init_ts.c.in
new file mode 100644
index 00000000000..7a12727f921
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/allreduce_init_ts.c.in
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID allreduce_init(BUFFER_ASYNC x1, BUFFER_ASYNC_OUT x2, COUNT count,
+                              DATATYPE datatype, OP op, COMM comm,
+                              INFO info, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Comm c_comm;
+    MPI_Datatype c_type;
+    MPI_Info c_info;
+    MPI_Request c_request;
+    MPI_Op c_op;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_type = PMPI_Type_f2c(*datatype);
+    c_op = PMPI_Op_f2c(*op);
+    c_info = PMPI_Info_f2c(*info);
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf, recvbuf,
+                          (@COUNT_TYPE@) *count,
+                          c_type, c_op, c_comm, c_info, &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/allreduce_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/allreduce_ts.c.in
new file mode 100644
index 00000000000..05039409c59
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/allreduce_ts.c.in
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+
+PROTOTYPE VOID allreduce(BUFFER x1, BUFFER_OUT x2, COUNT count,
+                         DATATYPE datatype, OP op, COMM comm)
+{
+    int c_ierr;
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype c_type;
+    MPI_Op c_op;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    c_type = PMPI_Type_f2c(*datatype);
+    c_op = PMPI_Op_f2c(*op);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf, recvbuf,
+                          c_count,
+                          c_type, c_op, c_comm);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/alltoall_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/alltoall_f08.F90
deleted file mode 100644
index f201dd2f769..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/alltoall_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Alltoall_f08(sendbuf,sendcount,sendtype,recvbuf,&
-                            recvcount,recvtype,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   use :: ompi_mpifh_bindings, only : ompi_alltoall_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_alltoall_f(sendbuf,sendcount,sendtype%MPI_VAL,recvbuf,&
-                        recvcount,recvtype%MPI_VAL,comm%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Alltoall_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/alltoall_init_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/alltoall_init_ts.c.in
new file mode 100644
index 00000000000..61e9957ec5b
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/alltoall_init_ts.c.in
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+
+PROTOTYPE VOID alltoall_init(BUFFER_ASYNC x1, COUNT sendcount, DATATYPE sendtype,
+                             BUFFER_ASYNC_OUT x2, COUNT recvcount, DATATYPE recvtype,
+                             COMM comm, INFO info, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Comm c_comm;
+    MPI_Request c_req;
+    MPI_Datatype c_sendtype, c_recvtype;
+    MPI_Info c_info;
+    void *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_sendcount = (@COUNT_TYPE@)*sendcount;
+    @COUNT_TYPE@ c_recvcount = (@COUNT_TYPE@)*recvcount;
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_sendtype = PMPI_Type_f2c(*sendtype);
+    c_recvtype = PMPI_Type_f2c(*recvtype);
+    c_info = PMPI_Info_f2c(*info);
+
+    if (OMPI_COMM_IS_INTER(c_comm) || !OMPI_IS_FORTRAN_IN_PLACE(sendbuf)) {
+        OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+        if (MPI_SUCCESS != c_ierr) {
+            if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+            OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+            return;
+        }
+    } else {
+        sendbuf = MPI_IN_PLACE;
+    }
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          c_sendcount,
+                          c_sendtype,
+                          recvbuf,
+                          c_recvcount,
+                          c_recvtype, c_comm, c_info, &c_req);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_req);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/alltoall_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/alltoall_ts.c.in
new file mode 100644
index 00000000000..87a7b66f207
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/alltoall_ts.c.in
@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+
+PROTOTYPE VOID alltoall(BUFFER x1, COUNT sendcount, DATATYPE sendtype,
+                        BUFFER_OUT x2, COUNT recvcount, DATATYPE recvtype,
+                        COMM comm)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype c_sendtype = NULL, c_recvtype = PMPI_Type_f2c(*recvtype);
+    void *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_sendcount = (@COUNT_TYPE@)(*sendcount);
+    @COUNT_TYPE@ c_recvcount =  (@COUNT_TYPE@)(*recvcount);
+
+    if (OMPI_COMM_IS_INTER(c_comm) || !OMPI_IS_FORTRAN_IN_PLACE(sendbuf)) {
+        c_sendtype = PMPI_Type_f2c(*sendtype);
+        OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+        if (MPI_SUCCESS != c_ierr) {
+            if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+            OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+            return;
+        }
+    } else {
+        sendbuf = MPI_IN_PLACE;
+    }
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          c_sendcount,
+                          c_sendtype,
+                          recvbuf,
+                          c_recvcount,
+                          c_recvtype, c_comm);
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/alltoallv_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/alltoallv_f08.F90
deleted file mode 100644
index 0acf0fd03a5..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/alltoallv_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Alltoallv_f08(sendbuf,sendcounts,sdispls,sendtype,recvbuf,&
-                             recvcounts,rdispls,recvtype,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   use :: ompi_mpifh_bindings, only : ompi_alltoallv_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_alltoallv_f(sendbuf,sendcounts,sdispls,sendtype%MPI_VAL,&
-                         recvbuf,recvcounts,rdispls,recvtype%MPI_VAL,comm%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Alltoallv_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/alltoallv_init_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/alltoallv_init_ts.c.in
new file mode 100644
index 00000000000..0b1608fd32f
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/alltoallv_init_ts.c.in
@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID alltoallv_init(BUFFER_ASYNC x1, COUNT_ARRAY sendcounts, DISP_ARRAY sdispls,
+                              DATATYPE sendtype, BUFFER_ASYNC_OUT x2, COUNT_ARRAY recvcounts,
+                              DISP_ARRAY rdispls, DATATYPE recvtype,
+                              COMM comm, INFO info, REQUEST_OUT request)
+{
+    MPI_Comm c_comm;
+    MPI_Datatype c_sendtype, c_recvtype;
+    MPI_Info c_info;
+    MPI_Request c_request;
+    int size, idx = 0, c_ierr;
+    @COUNT_TYPE@ *tmp_sendcounts = NULL;
+    @DISP_TYPE@ *tmp_sdispls = NULL;
+    @COUNT_TYPE@ *tmp_recvcounts = NULL;
+    @DISP_TYPE@ *tmp_rdispls = NULL;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_sendtype = PMPI_Type_f2c(*sendtype);
+    c_recvtype = PMPI_Type_f2c(*recvtype);
+    c_info = PMPI_Info_f2c(*info);
+
+    PMPI_Comm_size(c_comm, &size);
+    if (OMPI_COMM_IS_INTER(c_comm) || !OMPI_IS_FORTRAN_IN_PLACE(sendbuf)) {
+        OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+        if (MPI_SUCCESS != c_ierr) {
+            if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+            OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+            return;
+        }
+        OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(sendcounts, tmp_sendcounts, size);
+        OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(sdispls, tmp_sdispls, size);
+    } else {
+        sendbuf = MPI_IN_PLACE;
+    }
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(recvcounts, tmp_recvcounts, size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(rdispls, tmp_rdispls, size);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          tmp_sendcounts,
+                          tmp_sdispls,
+                          c_sendtype,
+                          recvbuf,
+                          tmp_recvcounts,
+                          tmp_rdispls,
+                          c_recvtype, c_comm, c_info, &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_request);
+    }
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(sendcounts, tmp_sendcounts, c_request, c_ierr, idx);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(sdispls, tmp_sdispls, c_request, c_ierr, idx);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(recvcounts, tmp_recvcounts, c_request, c_ierr, idx);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(rdispls, tmp_rdispls, c_request, c_ierr, idx);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/alltoallv_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/alltoallv_ts.c.in
new file mode 100644
index 00000000000..8af8ec4c8a9
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/alltoallv_ts.c.in
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID alltoallv(BUFFER x1, COUNT_ARRAY sendcounts, DISP_ARRAY sdispls,
+                         DATATYPE sendtype, BUFFER_OUT x2, COUNT_ARRAY recvcounts,
+                         DISP_ARRAY rdispls, DATATYPE recvtype, COMM comm)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    MPI_Datatype c_sendtype = NULL, c_recvtype = PMPI_Type_f2c(*recvtype);
+    int size = OMPI_COMM_IS_INTER(c_comm)?ompi_comm_remote_size(c_comm):ompi_comm_size(c_comm);
+    @COUNT_TYPE@ *tmp_sendcounts = NULL;
+    @DISP_TYPE@ *tmp_sdispls = NULL;
+    @COUNT_TYPE@ *tmp_recvcounts = NULL;
+    @DISP_TYPE@ *tmp_rdispls = NULL;
+
+    if (OMPI_COMM_IS_INTER(c_comm) || !OMPI_IS_FORTRAN_IN_PLACE(sendbuf)) {
+        c_sendtype = PMPI_Type_f2c(*sendtype);
+        OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+        if (MPI_SUCCESS != c_ierr) {
+            if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+            OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+            return;
+        }
+        OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(sendcounts, tmp_sendcounts, size);
+        OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(sdispls, tmp_sdispls, size);
+    } else {
+        sendbuf = MPI_IN_PLACE;
+    }
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(recvcounts, tmp_recvcounts, size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(rdispls, tmp_rdispls, size);
+
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          tmp_sendcounts,
+                          tmp_sdispls,
+                          c_sendtype,
+                          recvbuf,
+                          tmp_recvcounts,
+                          tmp_rdispls,
+                          c_recvtype, c_comm);
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(sendcounts, tmp_sendcounts);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(sdispls, tmp_sdispls);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(recvcounts, tmp_recvcounts);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(rdispls, tmp_rdispls);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/alltoallw_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/alltoallw_f08.F90
deleted file mode 100644
index f63ac4842f7..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/alltoallw_f08.F90
+++ /dev/null
@@ -1,42 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2013 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Alltoallw_f08(sendbuf,sendcounts,sdispls,sendtypes,&
-                             recvbuf,recvcounts,rdispls,recvtypes,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   use :: ompi_mpifh_bindings, only : ompi_alltoallw_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtypes(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtypes(*)
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   ! Note that we pass a scalar here for both the sendtypes and
-   ! recvtypes arguments, even though the real Alltoallw function
-   ! expects an array of integers.  This is a hack: we know that
-   ! [send|recv]types(1)%MPI_VAL will pass the address of the first
-   ! integer in the array of Type(MPI_Datatype) derived types.  And
-   ! since Type(MPI_Datatype) are exactly memory-equivalent to a
-   ! single INTEGER, passing the address of the first one is the same
-   ! as passing the address to an array of integers.  To be clear: the
-   ! back-end ompi_alltoallw_f is expecting a pointer to an array of
-   ! integers.  So it all works out (but is a hack :-\ ).
-   call ompi_alltoallw_f(sendbuf,sendcounts,sdispls,sendtypes(1)%MPI_VAL,&
-                         recvbuf,recvcounts,rdispls,recvtypes(1)%MPI_VAL,comm%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Alltoallw_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/alltoallw_init_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/alltoallw_init_ts.c.in
new file mode 100644
index 00000000000..34765cd1af1
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/alltoallw_init_ts.c.in
@@ -0,0 +1,99 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID alltoallw_init(BUFFER_ASYNC x1, COUNT_ARRAY sendcounts,
+                              DISP_ARRAY sdispls, DATATYPE_ARRAY sendtypes,
+                              BUFFER_ASYNC_OUT x2, COUNT_ARRAY recvcounts,
+                              DISP_ARRAY rdispls, DATATYPE_ARRAY recvtypes,
+                              COMM comm, INFO info, REQUEST_OUT request)
+{
+    MPI_Comm c_comm;
+    MPI_Datatype *c_sendtypes = NULL, *c_recvtypes;
+    MPI_Info c_info;
+    MPI_Request c_request;
+    int size, idx = 0, c_ierr;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ *tmp_sendcounts = NULL;
+    @DISP_TYPE@ *tmp_sdispls = NULL;
+    @COUNT_TYPE@ *tmp_recvcounts = NULL;
+    @DISP_TYPE@ *tmp_rdispls = NULL;
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_info = PMPI_Info_f2c(*info);
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    size = OMPI_COMM_IS_INTER(c_comm)?ompi_comm_remote_size(c_comm):ompi_comm_size(c_comm);
+
+    if (!OMPI_IS_FORTRAN_IN_PLACE(sendbuf)) {
+        c_sendtypes = (MPI_Datatype *) malloc(size * sizeof(MPI_Datatype));
+        OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(sendcounts, tmp_sendcounts, size);
+        OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(sdispls, tmp_sdispls, size);
+        for (int i=0; i<size; i++) {
+            c_sendtypes[i] = PMPI_Type_f2c(sendtypes[i]);
+        }
+    }
+
+    c_recvtypes = (MPI_Datatype *) malloc(size * sizeof(MPI_Datatype));
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(recvcounts, tmp_recvcounts, size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(rdispls, tmp_rdispls, size);
+    for (int i=0; i<size; i++) {
+        c_recvtypes[i] = PMPI_Type_f2c(recvtypes[i]);
+    }
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          tmp_sendcounts,
+                          tmp_sdispls,
+                          c_sendtypes,
+                          recvbuf,
+                          tmp_recvcounts,
+                          tmp_rdispls,
+                          c_recvtypes, c_comm, c_info, &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_request);
+        if (NULL != c_sendtypes) {
+            free(c_sendtypes);
+        }
+        free(c_recvtypes);
+    }
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(sendcounts, tmp_sendcounts, c_request, c_ierr, idx);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(sdispls, tmp_sdispls, c_request, c_ierr, idx);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(recvcounts, tmp_recvcounts, c_request, c_ierr, idx);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(rdispls, tmp_rdispls, c_request, c_ierr, idx);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/alltoallw_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/alltoallw_ts.c.in
new file mode 100644
index 00000000000..fdf800917b4
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/alltoallw_ts.c.in
@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID alltoallw(BUFFER x1, COUNT_ARRAY sendcounts,
+                         DISP_ARRAY sdispls, DATATYPE_ARRAY sendtypes,
+                         BUFFER_OUT x2, COUNT_ARRAY recvcounts,
+                         DISP_ARRAY rdispls, DATATYPE_ARRAY recvtypes,
+                         COMM comm)
+{
+    int size, c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    MPI_Datatype *c_sendtypes = NULL, *c_recvtypes;
+    @COUNT_TYPE@ *tmp_sendcounts = NULL;
+    @DISP_TYPE@ *tmp_sdispls = NULL;
+    @COUNT_TYPE@ *tmp_recvcounts = NULL;
+    @DISP_TYPE@ *tmp_rdispls = NULL;
+
+    size = OMPI_COMM_IS_INTER(c_comm)?ompi_comm_remote_size(c_comm):ompi_comm_size(c_comm);
+
+    if (OMPI_COMM_IS_INTER(c_comm) || !OMPI_IS_FORTRAN_IN_PLACE(sendbuf)) {
+        OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+        if (MPI_SUCCESS != c_ierr) {
+            if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+            OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+            return;
+        }
+        c_sendtypes = (MPI_Datatype *) malloc(size * sizeof(MPI_Datatype));
+        for (int i=0; i<size; i++) {
+            c_sendtypes[i] = PMPI_Type_f2c(sendtypes[i]);
+        }
+        OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(sendcounts, tmp_sendcounts, size);
+        OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(sdispls, tmp_sdispls, size);
+    } else {
+        sendbuf = MPI_IN_PLACE;
+    }
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    c_recvtypes = (MPI_Datatype *) malloc(size * sizeof(MPI_Datatype));
+    for (int i=0; i<size; i++) {
+        c_recvtypes[i] = PMPI_Type_f2c(recvtypes[i]);
+    }
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(recvcounts, tmp_recvcounts, size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(rdispls, tmp_rdispls, size);
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          tmp_sendcounts,
+                          tmp_sdispls,
+                          c_sendtypes,
+                          recvbuf,
+                          tmp_recvcounts,
+                          tmp_rdispls,
+                          c_recvtypes, c_comm);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(sendcounts, tmp_sendcounts);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(sdispls, tmp_sdispls);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(recvcounts, tmp_recvcounts);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(rdispls, tmp_rdispls);
+
+    if (NULL != c_sendtypes) {
+        free(c_sendtypes);
+    }
+    free(c_recvtypes);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/barrier_init_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/barrier_init_f08.F90
new file mode 100644
index 00000000000..03b40a41292
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/barrier_init_f08.F90
@@ -0,0 +1,25 @@
+! -*- f90 -*-
+!
+! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
+!                         All rights reserved.
+! Copyright (c) 2018-2021 Research Organization for Information Science
+!                         and Technology (RIST).  All rights reserved.
+! $COPYRIGHT$
+
+#include "mpi-f08-rename.h"
+
+subroutine MPI_Barrier_init_f08(comm,info,request,ierror)
+   use :: mpi_f08_types, only : MPI_Comm, MPI_Info, MPI_Request
+   use :: ompi_mpifh_bindings, only : ompi_barrier_init_f
+   implicit none
+   TYPE(MPI_Comm), INTENT(IN) :: comm
+   TYPE(MPI_Info), INTENT(IN) :: info
+   TYPE(MPI_Request), INTENT(OUT) :: request
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+   integer :: c_ierror
+
+   call ompi_barrier_init_f(comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
+   if (present(ierror)) ierror = c_ierror
+
+end subroutine MPI_Barrier_init_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/base/Makefile.am b/ompi/mpi/fortran/use-mpi-f08/base/Makefile.am
index 7b62c571a4a..0e5ef89e9fe 100644
--- a/ompi/mpi/fortran/use-mpi-f08/base/Makefile.am
+++ b/ompi/mpi/fortran/use-mpi-f08/base/Makefile.am
@@ -31,6 +31,39 @@ if OMPI_BUILD_FORTRAN_USEMPIF08_BINDINGS
 noinst_LTLIBRARIES = libusempif08_ccode.la
 
 libusempif08_ccode_la_SOURCES = \
-        buffer_detach.c
+        ts.h \
+        bigcount.h \
+        api_f08_generated.c \
+	ompi_type_get_extent_f_c.c \
+	ompi_type_get_envelope_f_c.c \
+        ompi_type_get_contents_f_c.c
+
+if OMPI_FORTRAN_HAVE_TS
+libusempif08_ccode_la_SOURCES += \
+        ts.c
+endif
+#
+# api_f08_generated.c should work with all compilers, supporting CFI or TS 29113 support
+# through macros in ts.h and ts.c.
+#
+if OMPI_GENERATE_BINDINGS
+
+include ../Makefile.prototype_files
+template_files =${prototype_files:%=$(abs_top_srcdir)/ompi/mpi/fortran/use-mpi-f08/%}
+
+api_f08_generated.c: $(template_files)
+	$(OMPI_V_GEN) $(PYTHON) $(top_srcdir)/ompi/mpi/bindings/bindings.py \
+	    --builddir $(abs_top_builddir) \
+	    --srcdir $(abs_top_srcdir) \
+	    --output $(abs_builddir)/$@ \
+	    fortran \
+	    code \
+	    --lang c \
+	    --prototype-files $(template_files)
+
+endif
+
+# Delete generated files on maintainer-clean
+MAINTAINERCLEANFILES = api_f08_generated.c
 
 endif
diff --git a/ompi/mpi/fortran/use-mpi-f08/base/bigcount.h b/ompi/mpi/fortran/use-mpi-f08/base/bigcount.h
new file mode 100644
index 00000000000..d7dd109005b
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/base/bigcount.h
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/*
+ * Bigcount array conversion macros for Fortran templates.
+ */
+
+#define OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(array, tmp_array, n) \
+    do { \
+        if (sizeof(*(array)) == sizeof(*(tmp_array))) { \
+            (tmp_array) = (void *)(array); \
+        } else { \
+            (tmp_array) = malloc(sizeof(*tmp_array) * n); \
+            for (int bigcount_array_i = 0; bigcount_array_i < n; ++bigcount_array_i) { \
+                (tmp_array)[bigcount_array_i] = (array)[bigcount_array_i]; \
+            } \
+        } \
+    } while (0)
+
+#define OMPI_FORTRAN_BIGCOUNT_ARRAY_COPYOUT(array, tmp_array, n) \
+    do { \
+        if ((array) != (tmp_array) && NULL != (tmp_array)) { \
+            for (int bigcount_array_i = 0; bigcount_array_i < n; ++bigcount_array_i) { \
+                (array)[bigcount_array_i] = (tmp_array)[bigcount_array_i]; \
+            } \
+        } \
+    } while (0)
+
+
+#define OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(array, tmp_array) \
+    do { \
+        if ((void *)(array) != (void *)(tmp_array) && NULL != (tmp_array)) { \
+            free(tmp_array); \
+            tmp_array = NULL; \
+        } \
+    } while (0)
+
+#define OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(array, tmp_array, c_request, c_ierr, idx) \
+    do { \
+        if (MPI_SUCCESS == (c_ierr)) { \
+            ompi_coll_base_nbc_request_t* nb_request = (ompi_coll_base_nbc_request_t*)c_request; \
+            if ((void *)(array) != (void *)(tmp_array) && (tmp_array) != NULL) { \
+                nb_request->data.release_arrays[(idx)++] = tmp_array; \
+            } \
+            nb_request->data.release_arrays[idx] = NULL; \
+        } else { \
+            OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP((array), (tmp_array)); \
+        } \
+    } while (0)
diff --git a/ompi/mpi/fortran/use-mpi-f08/base/buffer_detach.c b/ompi/mpi/fortran/use-mpi-f08/base/buffer_detach.c
deleted file mode 100644
index 3d24b3f9656..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/base/buffer_detach.c
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2011-2015 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "mpi.h"
-#include "ompi/mpi/fortran/base/fint_2_int.h"
-
-/*
- * This function implemented in this file is only called from Fortran,
- * so we never bothered to put a prototype for it in any C header
- * file.  To avoid compiler warnings about no protoype, we prototype
- * it here.
- */
-OMPI_DECLSPEC void ompi_buffer_detach_f08(char *buffer, MPI_Fint *size,
-                                          MPI_Fint *ierr);
-
-OMPI_DECLSPEC void pompi_buffer_detach_f08(char *buffer, MPI_Fint *size,
-                                          MPI_Fint *ierr);
-
-/* (this comment is repeated in ompi/mpi/fortran/mpif-h/buffer_detach_f.c)
- *
- * MPI-3.1 section 3.6, page 45, states that the mpif.h and mpi module
- * interfaces for MPI_BUFFER_DETACH ignore the buffer argument.
- * Therefore, for the mpif.h and mpi module interfaces, we use a dummy
- * variable and leave the value handed in alone.
- *
- * The mpi_f08 implementation for MPI_BUFFER_DETACH therefore is a
- * separate routine in the use-mpi-f08 directory (it's not built in
- * the mpif-h directory because of all the different combinations of
- * supporting weak symbols (or not), building the profiling layer (or
- * not), etc.).
- *
- * Note that we only need to build this function once -- the F08
- * interfaces for MPI_BUFFER_ATTACH and PMPI_BUFFER_ATTACH both
- * bind(C) to the name ompi_buffer_detach_f08.
- */
-void ompi_buffer_detach_f08(char *buffer, MPI_Fint *size, MPI_Fint *ierr)
-{
-    int c_ierr;
-    void *dummy;
-    OMPI_SINGLE_NAME_DECL(size);
-
-    c_ierr = PMPI_Buffer_detach(&dummy, OMPI_SINGLE_NAME_CONVERT(size));
-    if (NULL != ierr) {
-        *ierr = OMPI_INT_2_FINT(c_ierr);
-    }
-
-    if (MPI_SUCCESS == c_ierr) {
-        OMPI_SINGLE_INT_2_FINT(size);
-        *(void **)buffer = dummy;
-    }
-}
-
-void pompi_buffer_detach_f08(char *buffer, MPI_Fint *size, MPI_Fint *ierr)
-{
-    ompi_buffer_detach_f08(buffer, size, ierr);
-}
diff --git a/ompi/mpi/fortran/use-mpi-f08/base/ompi_type_get_contents_f_c.c b/ompi/mpi/fortran/use-mpi-f08/base/ompi_type_get_contents_f_c.c
new file mode 100644
index 00000000000..dac60b78cdc
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/base/ompi_type_get_contents_f_c.c
@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+#include "ompi/mpi/fortran/use-mpi-f08/base/bigcount.h"
+
+/*
+ * big count entry point, only needed by F08 bindings.
+ */
+
+static const char FUNC_NAME[] = "MPI_TYPE_GET_CONTENTS_C";
+
+void ompi_type_get_contents_f_c(MPI_Fint *mtype, MPI_Count *max_integers,
+                             MPI_Count *max_addresses, MPI_Count *max_large_counts,
+			     MPI_Count *max_datatypes,
+                             MPI_Fint *array_of_integers,
+                             MPI_Aint *array_of_addresses,
+                             MPI_Count *array_of_large_counts,
+                             MPI_Fint *array_of_datatypes, MPI_Fint *ierr);
+void ompi_type_get_contents_f_c(MPI_Fint *mtype, MPI_Count *max_integers,
+                             MPI_Count *max_addresses, MPI_Count *max_large_counts,
+			     MPI_Count *max_datatypes,
+                             MPI_Fint *array_of_integers,
+                             MPI_Aint *array_of_addresses,
+                             MPI_Count *array_of_large_counts,
+                             MPI_Fint *array_of_datatypes, MPI_Fint *ierr)
+{
+    MPI_Datatype *c_datatype_array = NULL;
+    int *tmp_array_of_integers = NULL;
+    MPI_Datatype c_mtype = PMPI_Type_f2c(*mtype);
+    int i, c_ierr;
+
+    if (*max_datatypes) {
+        c_datatype_array = (MPI_Datatype *) malloc(*max_datatypes * sizeof(MPI_Datatype));
+        if (NULL == c_datatype_array) {
+            c_ierr = OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_NO_MEM,
+                                            FUNC_NAME);
+            if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+            return;
+        }
+    }
+
+    if (*max_integers) {
+        OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(array_of_integers, tmp_array_of_integers, *max_integers);
+    }
+
+    c_ierr = PMPI_Type_get_contents_c(c_mtype,
+                                      *max_integers,
+                                      *max_addresses,
+                                      *max_large_counts,
+                                      *max_datatypes,
+                                      tmp_array_of_integers,
+                                      array_of_addresses,
+                                      array_of_large_counts,
+			              c_datatype_array);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        OMPI_FORTRAN_BIGCOUNT_ARRAY_COPYOUT(array_of_integers, tmp_array_of_integers, *max_integers);
+        for (i = 0; i < *max_datatypes; i++) {
+          array_of_datatypes[i] = PMPI_Type_c2f(c_datatype_array[i]);
+        }
+    }
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(array_of_integers, tmp_array_of_integers);
+
+    if (NULL != c_datatype_array) {
+        free(c_datatype_array);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/base/ompi_type_get_envelope_f_c.c b/ompi/mpi/fortran/use-mpi-f08/base/ompi_type_get_envelope_f_c.c
new file mode 100644
index 00000000000..0549d0cae9a
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/base/ompi_type_get_envelope_f_c.c
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+
+#include "ompi_config.h"
+
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+
+void ompi_type_get_envelope_f_c(MPI_Fint *type, MPI_Count *num_integers,
+			     MPI_Count *num_addresses,
+			     MPI_Count *num_large_counts,
+			     MPI_Count *num_datatypes, MPI_Fint *combiner,
+			     MPI_Fint *ierr);
+void ompi_type_get_envelope_f_c(MPI_Fint *type, MPI_Count *num_integers,
+			     MPI_Count *num_addresses,
+			     MPI_Count *num_large_counts,
+			     MPI_Count *num_datatypes, MPI_Fint *combiner,
+			     MPI_Fint *ierr)
+{
+    int c_ierr;
+    MPI_Datatype c_type = PMPI_Type_f2c(*type);
+    int c_combiner;
+
+    c_ierr = PMPI_Type_get_envelope_c(c_type,
+                                     num_integers,
+                                     num_addresses,
+                                     num_large_counts,
+                                     num_datatypes,
+                                     &c_combiner);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *combiner = (MPI_Fint)(c_combiner);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/base/ompi_type_get_extent_f_c.c b/ompi/mpi/fortran/use-mpi-f08/base/ompi_type_get_extent_f_c.c
new file mode 100644
index 00000000000..5eea206fc04
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/base/ompi_type_get_extent_f_c.c
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+
+/*
+ * big count entry point, only needed by F08 bindings.
+ */
+void ompi_type_get_extent_f_c(MPI_Fint *type, MPI_Count *lb,
+                           MPI_Count *extent, MPI_Fint *ierr);
+void ompi_type_get_extent_f_c(MPI_Fint *type, MPI_Count *lb,
+                           MPI_Count *extent, MPI_Fint *ierr)
+{
+    int c_ierr;
+    MPI_Datatype c_type = PMPI_Type_f2c(*type);
+
+    c_ierr = PMPI_Type_get_extent_c(c_type, lb, extent);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/base/ts.c b/ompi/mpi/fortran/use-mpi-f08/base/ts.c
new file mode 100644
index 00000000000..4ff011db7ef
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/base/ts.c
@@ -0,0 +1,138 @@
+/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/*
+ * Copyright (c) 2014      Argonne National Laboratory.
+ * Copyright (c) 2019      Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ts.h"
+
+#include <assert.h>
+
+int ompi_ts_create_datatype(CFI_cdesc_t *cdesc, int oldcount, MPI_Datatype oldtype, MPI_Datatype *newtype)
+{
+    const int MAX_RANK = 15; /* Fortran 2008 specifies a maximum rank of 15 */
+    MPI_Datatype types[MAX_RANK + 1]; /* Use a fixed size array to avoid malloc. + 1 for oldtype */
+    int mpi_errno = MPI_SUCCESS;
+    int accum_elems = 1;
+    int accum_sm = cdesc->elem_len;
+    int done = false; /* Have we created a datatype for oldcount of oldtype? */
+    int last; /* Index of the last successfully created datatype in types[] */
+    int extent;
+    int i, j;
+
+#if OPAL_ENABLE_DEBUG
+    {
+        size_t size;
+        assert(cdesc->rank <= MAX_RANK);
+        ompi_datatype_type_size(oldtype, &size);
+        /* When cdesc->elem_len != size, things suddenly become complicated. Generally, it is hard to create
+         * a composite datatype based on two datatypes. Currently we don't support it and doubt it is useful.
+         */
+        assert(cdesc->elem_len == size);
+    }
+#endif
+
+    types[0] = oldtype;
+    i = 0;
+    done = false;
+    while (i < cdesc->rank && !done) {
+        if (oldcount % accum_elems) {
+            /* oldcount should be a multiple of accum_elems, otherwise we might need an
+             * MPI indexed datatype to describle the irregular region, which is not supported yet.
+             */
+            mpi_errno = MPI_ERR_INTERN;
+            last = i;
+            goto fn_exit;
+        }
+
+        extent = oldcount / accum_elems;
+        if (extent > cdesc->dim[i].extent) {
+            extent = cdesc->dim[i].extent;
+        } else {
+            /* Up to now, we have accumlated enough elements */
+            done = true;
+        }
+
+        if (cdesc->dim[i].sm == accum_sm) {
+            mpi_errno = PMPI_Type_contiguous(extent, types[i], &types[i+1]);
+        } else {
+            mpi_errno = PMPI_Type_create_hvector(extent, 1, cdesc->dim[i].sm, types[i], &types[i+1]);
+        }
+        if (mpi_errno != MPI_SUCCESS) {
+            last = i;
+            goto fn_exit;
+        }
+
+        accum_sm = cdesc->dim[i].sm * cdesc->dim[i].extent;
+        accum_elems  *= cdesc->dim[i].extent;
+        i++;
+    }
+
+    if (done) {
+        *newtype = types[i];
+        MPI_Type_commit(newtype);
+        last = i - 1; /* To avoid freeing newtype */
+        mpi_errno = MPI_SUCCESS;
+    } else {
+        /* If # of elements given by "oldcount oldtype" is bigger than
+         * what cdesc describles, then we will reach here.
+         */
+        last = i;
+        mpi_errno = MPI_ERR_ARG;
+        goto fn_exit;
+    }
+
+fn_exit:
+    for (j = 1; j <= last; j++) {
+        PMPI_Type_free(&types[j]);
+    }
+    return mpi_errno;
+}
+
+static void copy(CFI_dim_t *dim, int rank, char * base, char **dest, size_t len) {
+    for (CFI_index_t i=0; i<dim->extent; i++) {
+        if (rank > 1) {
+            copy(dim-1, rank-1, base, dest, len);
+        } else {
+            memcpy(*dest, base, len);
+            *dest += len;
+        }
+        base += dim->sm;
+    }
+}
+
+int ompi_ts_copy(CFI_cdesc_t *cdesc, char *buffer) {
+    copy(&cdesc->dim[cdesc->rank - 1], cdesc->rank, cdesc->base_addr, &buffer, cdesc->elem_len);
+    return OMPI_SUCCESS;
+}
+
+static void copy_back(CFI_dim_t *dim, int rank, char * base, char **source, size_t len) {
+    for (CFI_index_t i=0; i<dim->extent; i++) {
+        if (rank > 1) {
+            copy_back(dim-1, rank-1, base, source, len);
+        } else {
+            memcpy(base, *source, len);
+            *source += len;
+        }
+        base += dim->sm;
+    }
+}
+
+int ompi_ts_copy_back(char *buffer, CFI_cdesc_t *cdesc) {
+    copy_back(&cdesc->dim[cdesc->rank - 1], cdesc->rank, cdesc->base_addr, &buffer, cdesc->elem_len);
+    return OMPI_SUCCESS;
+}
+
+size_t ompi_ts_size(CFI_cdesc_t *cdesc) {
+    size_t res = cdesc->elem_len;
+    for (int i=0; i<cdesc->rank; i++) {
+         res *= cdesc->dim[i].extent;
+    }
+    return res;
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/base/ts.h b/ompi/mpi/fortran/use-mpi-f08/base/ts.h
new file mode 100644
index 00000000000..b6cbd73d31e
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/base/ts.h
@@ -0,0 +1,158 @@
+/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/*
+ * Copyright (c) 2014      Argonne National Laboratory.
+ * Copyright (c) 2019      Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+
+#if OMPI_FORTRAN_HAVE_TS
+
+#include <ISO_Fortran_binding.h>
+
+#define OMPI_CFI_BUFFER CFI_cdesc_t
+
+extern int ompi_ts_create_datatype(CFI_cdesc_t *cdesc, int oldcount, MPI_Datatype oldtype, MPI_Datatype *newtype);
+
+extern size_t ompi_ts_size(CFI_cdesc_t *cdesc);
+
+extern int ompi_ts_copy_back(char *buffer, CFI_cdesc_t *cdesc);
+
+extern int ompi_ts_copy(CFI_cdesc_t *cdesc, char *buffer);
+
+#define OMPI_CFI_BASE_ADDR(x) (x)->base_addr
+
+#define OMPI_CFI_2_C(x, count, type, datatype, rc)                      \
+    do {                                                                \
+        datatype = type;                                                \
+        if (x->rank != 0 && !CFI_is_contiguous(x)) {                    \
+            rc = ompi_ts_create_datatype(x, count, type, &datatype);    \
+            if (OPAL_LIKELY(MPI_SUCCESS == rc)) {                       \
+                count = 1;                                              \
+            }                                                           \
+        } else {                                                        \
+            rc = MPI_SUCCESS;                                           \
+        }                                                               \
+    } while (0)
+
+#define OMPI_CFI_2_C_ALLOC(x, buffer, count, type, datatype, rc)        \
+    do {                                                                \
+        datatype = type;                                                \
+        if (x->rank != 0 && !CFI_is_contiguous(x)) {                    \
+            size_t size = ompi_ts_size(x);                              \
+            buffer = malloc(size);                                      \
+            if (OPAL_UNLIKELY(NULL == buffer)) {                        \
+                rc = MPI_ERR_NO_MEM;                                    \
+            } else {                                                    \
+                rc = MPI_SUCCESS;                                       \
+            }                                                           \
+        } else {                                                        \
+            buffer = x->base_addr;                                      \
+            rc = MPI_SUCCESS;                                           \
+        }                                                               \
+    } while (0)
+
+#define OMPI_CFI_2_C_COPY(x, buffer, count, type, datatype, rc)         \
+    do {                                                                \
+        datatype = type;                                                \
+        if (x->rank != 0 && !CFI_is_contiguous(x)) {                    \
+            size_t size = ompi_ts_size(x);                              \
+            buffer = malloc(size);                                      \
+            if (OPAL_UNLIKELY(NULL == buffer)) {                        \
+                rc = MPI_ERR_NO_MEM;                                    \
+            } else {                                                    \
+                rc = ompi_ts_copy(x, buffer);                           \
+            }                                                           \
+        } else {                                                        \
+            buffer = x->base_addr;                                      \
+            rc = MPI_SUCCESS;                                           \
+        }                                                               \
+    } while (0)
+
+#define OMPI_C_2_CFI_FREE(x, buffer, count, type, datatype, rc)         \
+    do {                                                                \
+        if (buffer != x->base_addr) {                                   \
+            free(buffer);                                               \
+        }                                                               \
+        if (type != datatype) {                                         \
+            rc = PMPI_Type_free(&datatype);                             \
+        }                                                               \
+    } while (0)
+
+#define OMPI_C_2_CFI_COPY(x, buffer, count, type, datatype, rc)         \
+    do {                                                                \
+        if (buffer != x->base_addr) {                                   \
+            rc = ompi_ts_copy_back(buffer, x);                          \
+            free(buffer);                                               \
+        }                                                               \
+        if (type != datatype) {                                         \
+            rc = PMPI_Type_free(&datatype);                             \
+        }                                                               \
+    } while (0)
+
+#define OMPI_CFI_IS_CONTIGUOUS(x)                                       \
+    (0 == x->rank || CFI_is_contiguous(x))
+
+#define OMPI_CFI_CHECK_CONTIGUOUS(x, rc)                                \
+    do {                                                                \
+        if (OMPI_CFI_IS_CONTIGUOUS(x)) {                                \
+            rc = MPI_SUCCESS;                                           \
+        } else {                                                        \
+            rc = MPI_ERR_INTERN;                                        \
+        }                                                               \
+    } while (0)
+
+#else
+
+/*
+ * Macros for compilers not supporting TS 29113.
+ */
+
+#define OMPI_CFI_BUFFER char
+
+#define OMPI_CFI_BASE_ADDR(x) (x)
+
+#define OMPI_CFI_2_C(x, count, type, datatype, rc)                      \
+    do {                                                                \
+        datatype = type;                                                \
+        rc = MPI_SUCCESS;                                               \
+    } while (0)
+
+#define OMPI_CFI_2_C_ALLOC(x, buffer, count, type, datatype, rc)        \
+    do {                                                                \
+        datatype = type;                                                \
+        buffer = x;                                                     \
+        rc = MPI_SUCCESS;                                               \
+    } while (0)
+
+#define OMPI_CFI_2_C_COPY(x, buffer, count, type, datatype, rc)         \
+    do {                                                                \
+        datatype = type;                                                \
+        buffer = x;                                                     \
+        rc = MPI_SUCCESS;                                               \
+    } while (0)
+
+#define OMPI_C_2_CFI_FREE(x, buffer, count, type, datatype, rc)         \
+    do {} while (0)
+
+#define OMPI_C_2_CFI_COPY(x, buffer, count, type, datatype, rc)         \
+    do {} while (0)
+
+#define OMPI_CFI_IS_CONTIGUOUS(x) 1
+
+#define OMPI_CFI_CHECK_CONTIGUOUS(x, rc)                                \
+    do {                                                                \
+        rc = MPI_SUCCESS;                                               \
+    } while (0)
+#endif /* OMPI_FORTRAN_HAVE_TS */
+
+#define OMPI_COUNT_CONVERT(fcount)
diff --git a/ompi/mpi/fortran/use-mpi-f08/bcast_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/bcast_f08.F90
deleted file mode 100644
index 1a5e5001411..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/bcast_f08.F90
+++ /dev/null
@@ -1,29 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Bcast_f08(buffer,count,datatype,root,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   use :: ompi_mpifh_bindings, only : ompi_bcast_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: buffer
-   INTEGER, INTENT(IN) :: count, root
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_bcast_f(buffer,count,datatype%MPI_VAL,root,comm%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Bcast_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/bcast_init_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/bcast_init_ts.c.in
new file mode 100644
index 00000000000..e872a72d8ec
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/bcast_init_ts.c.in
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID bcast_init(BUFFER_ASYNC x, COUNT count, DATATYPE datatype,
+                          RANK root, COMM comm, INFO info, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Comm c_comm;
+    MPI_Info c_info;
+    MPI_Request c_req;
+    MPI_Datatype c_type, c_datatype = MPI_DATATYPE_NULL;
+    void *buffer = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+    int c_root = OMPI_FINT_2_INT(*root);
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_type = PMPI_Type_f2c(*datatype);
+    c_info = PMPI_Info_f2c(*info);
+
+    if (OMPI_COMM_IS_INTRA(c_comm) || MPI_PROC_NULL != c_root) {
+        OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+        if (MPI_SUCCESS != c_ierr) {
+            if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+            OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+            return;
+        }
+    }
+
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(buffer),
+                          c_count,
+                          (c_datatype == MPI_DATATYPE_NULL) ? c_type : c_datatype,
+                          c_root,
+                          c_comm,
+                          c_info,
+                          &c_req);
+    if (MPI_DATATYPE_NULL != c_datatype && c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_req);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/bcast_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/bcast_ts.c.in
new file mode 100644
index 00000000000..8dedd051ae1
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/bcast_ts.c.in
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID bcast(BUFFER_OUT x, COUNT count, DATATYPE datatype,
+                     RANK root, COMM comm)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype c_datatype = NULL, c_type = NULL;
+    int c_root = OMPI_FINT_2_INT(*root);
+    void *buffer = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+
+    if (OMPI_COMM_IS_INTRA(c_comm) || MPI_PROC_NULL != c_root) {
+        c_type = PMPI_Type_f2c(*datatype);
+        OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+        if (MPI_SUCCESS != c_ierr) {
+            if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+            OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+            return;
+        }
+    }
+
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(buffer),
+                          c_count,
+                          c_datatype,
+                          c_root,
+                          c_comm);
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/bindings.h b/ompi/mpi/fortran/use-mpi-f08/bindings.h
new file mode 100644
index 00000000000..7452a5e9bd1
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/bindings.h
@@ -0,0 +1,219 @@
+/*
+ * Copyright (c) 2019      Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef OMPI_CDESC_BINDINGS_H
+#define OMPI_CDESC_BINDINGS_H
+
+#include "ompi_config.h"
+
+#include "ts.h"
+
+#include "mpi.h"
+
+void ompi_bsend_ts(CFI_cdesc_t* x, MPI_Fint *count, MPI_Fint *datatype,
+                   MPI_Fint *dest, MPI_Fint *tag, MPI_Fint *comm,
+                   MPI_Fint *ierr);
+
+void ompi_bsend_init_ts(CFI_cdesc_t* x, MPI_Fint *count, MPI_Fint *datatype,
+                        MPI_Fint *dest, MPI_Fint *tag, MPI_Fint *comm,
+                        MPI_Fint *request, MPI_Fint *ierr);
+
+void ompi_buffer_attach_ts(CFI_cdesc_t *x, MPI_Fint *size, MPI_Fint *ierr);
+
+void ompi_ibsend_ts(CFI_cdesc_t* x, MPI_Fint *count, MPI_Fint *datatype,
+                    MPI_Fint *dest, MPI_Fint *tag, MPI_Fint *comm,
+                    MPI_Fint *request, MPI_Fint *ierr);
+
+void ompi_irecv_ts(CFI_cdesc_t *x, MPI_Fint *count, MPI_Fint *datatype,
+                   MPI_Fint *source, MPI_Fint *tag, MPI_Fint *comm,
+                   MPI_Fint *request, MPI_Fint *ierr);
+
+void ompi_irsend_ts(CFI_cdesc_t* x, MPI_Fint *count, MPI_Fint *datatype,
+                    MPI_Fint *dest, MPI_Fint *tag, MPI_Fint *comm,
+                    MPI_Fint *request, MPI_Fint *ierr);
+
+void ompi_isend_ts(CFI_cdesc_t *x, MPI_Fint *count, MPI_Fint *datatype,
+                   MPI_Fint *dest, MPI_Fint *tag, MPI_Fint *comm,
+                   MPI_Fint *request, MPI_Fint *ierr);
+
+void ompi_issend_ts(CFI_cdesc_t* x, MPI_Fint *count, MPI_Fint *datatype,
+                    MPI_Fint *dest, MPI_Fint *tag, MPI_Fint *comm,
+                    MPI_Fint *request, MPI_Fint *ierr);
+
+void ompi_recv_ts(CFI_cdesc_t *x, MPI_Fint *count, MPI_Fint *datatype,
+                  MPI_Fint *source, MPI_Fint *tag, MPI_Fint *comm,
+                  MPI_Fint *status, MPI_Fint *ierr);
+
+void ompi_recv_init_ts(CFI_cdesc_t* x, MPI_Fint *count, MPI_Fint *datatype,
+                       MPI_Fint *source, MPI_Fint *tag, MPI_Fint *comm,
+                       MPI_Fint *request, MPI_Fint *ierr);
+
+void ompi_rsend_ts(CFI_cdesc_t* x, MPI_Fint *count, MPI_Fint *datatype,
+                   MPI_Fint *dest, MPI_Fint *tag, MPI_Fint *comm,
+                   MPI_Fint *ierr);
+
+void ompi_rsend_init_ts(CFI_cdesc_t* x, MPI_Fint *count,
+                        MPI_Fint *datatype, MPI_Fint *dest,
+                        MPI_Fint *tag, MPI_Fint *comm,
+                        MPI_Fint *request, MPI_Fint *ierr);
+
+void ompi_send_ts(CFI_cdesc_t* x, MPI_Fint *count, MPI_Fint *datatype,
+                  MPI_Fint *dest, MPI_Fint *tag, MPI_Fint *comm, MPI_Fint *ierr);
+
+void ompi_sendrecv_ts(CFI_cdesc_t* x1, MPI_Fint *sendcount, MPI_Fint *sendtype,
+                      MPI_Fint *dest, MPI_Fint *sendtag, CFI_cdesc_t* x2,
+                      MPI_Fint *recvcount, MPI_Fint *recvtype,
+                      MPI_Fint *source, MPI_Fint *recvtag, MPI_Fint *comm,
+                      MPI_Fint *status, MPI_Fint *ierr);
+
+void ompi_sendrecv_replace_ts(CFI_cdesc_t* x, MPI_Fint *count, MPI_Fint *datatype,
+                              MPI_Fint *dest, MPI_Fint *sendtag,
+                              MPI_Fint *source, MPI_Fint *recvtag,
+                              MPI_Fint *comm, MPI_Fint *status, MPI_Fint *ierr);
+
+void ompi_send_init_ts(CFI_cdesc_t *x, MPI_Fint *count, MPI_Fint *datatype,
+                       MPI_Fint *dest, MPI_Fint *tag, MPI_Fint *comm,
+                       MPI_Fint *request, MPI_Fint *ierr);
+
+void ompi_ssend_ts(CFI_cdesc_t* x, MPI_Fint *count, MPI_Fint *datatype,
+                   MPI_Fint *dest, MPI_Fint *tag,
+                   MPI_Fint *comm, MPI_Fint *ierr);
+
+void ompi_ssend_init_ts(CFI_cdesc_t* x, MPI_Fint *count, MPI_Fint *datatype,
+                        MPI_Fint *dest, MPI_Fint *tag,
+                        MPI_Fint *comm, MPI_Fint *request, MPI_Fint *ierr);
+
+void ompi_get_address_ts(CFI_cdesc_t *x, MPI_Aint *address, MPI_Fint *ierr);
+
+void ompi_pack_ts(CFI_cdesc_t* x1, MPI_Fint *incount, MPI_Fint *datatype,
+                  CFI_cdesc_t* x2, MPI_Fint *outsize, MPI_Fint *position,
+                  MPI_Fint *comm, MPI_Fint *ierr);
+
+void ompi_pack_external_ts(char *datarep, CFI_cdesc_t* x1, MPI_Fint *incount,
+                           MPI_Fint *datatype, CFI_cdesc_t* x2,
+                           MPI_Aint *outsize, MPI_Aint *position,
+                           MPI_Fint *ierr, int datarep_len);
+
+void ompi_unpack_ts(CFI_cdesc_t* x1, MPI_Fint *insize, MPI_Fint *position,
+                           CFI_cdesc_t* x2, MPI_Fint *outcount, MPI_Fint *datatype,
+                           MPI_Fint *comm, MPI_Fint *ierr);
+
+void ompi_unpack_external_ts(char *datarep, CFI_cdesc_t* x1, MPI_Aint *insize,
+                             MPI_Aint *position, CFI_cdesc_t* x2,
+                             MPI_Fint *outcount, MPI_Fint *datatype,
+                             MPI_Fint *ierr, int datarep_len);
+
+void ompi_allgather_ts(CFI_cdesc_t* x1, MPI_Fint *sendcount, MPI_Fint *sendtype,
+                       CFI_cdesc_t* x2, MPI_Fint *recvcount, MPI_Fint *recvtype,
+                       MPI_Fint *comm, MPI_Fint *ierr);
+
+void ompi_allgatherv_ts(CFI_cdesc_t *x1, MPI_Fint *sendcount, MPI_Fint *sendtype,
+                        CFI_cdesc_t *x2, MPI_Fint *recvcounts, MPI_Fint *displs,
+                        MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *ierr);
+
+void ompi_allreduce_ts(CFI_cdesc_t *x1, CFI_cdesc_t *x2, MPI_Fint *count,
+                       MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm,
+                       MPI_Fint *ierr);
+
+void ompi_alltoall_ts(CFI_cdesc_t* x1, MPI_Fint *sendcount, MPI_Fint *sendtype,
+                      CFI_cdesc_t* x2, MPI_Fint *recvcount, MPI_Fint *recvtype,
+                      MPI_Fint *comm, MPI_Fint *ierr);
+
+void ompi_alltoallv_ts(CFI_cdesc_t *x1, MPI_Fint *sendcounts, MPI_Fint *sdispls,
+                       MPI_Fint *sendtype, CFI_cdesc_t *x2, MPI_Fint *recvcounts,
+                       MPI_Fint *rdispls, MPI_Fint *recvtype,
+                       MPI_Fint *comm, MPI_Fint *ierr);
+
+void ompi_alltoallw_ts(CFI_cdesc_t *x1, MPI_Fint *sendcounts,
+                       MPI_Fint *sdispls, MPI_Fint *sendtypes,
+                       CFI_cdesc_t *x2, MPI_Fint *recvcounts,
+                       MPI_Fint *rdispls, MPI_Fint *recvtypes,
+                       MPI_Fint *comm, MPI_Fint *ierr);
+
+void ompi_bcast_ts(CFI_cdesc_t* x, MPI_Fint *count, MPI_Fint *datatype,
+                   MPI_Fint *root, MPI_Fint *comm, MPI_Fint *ierr);
+
+void ompi_exscan_ts(CFI_cdesc_t *x1, CFI_cdesc_t *x2, MPI_Fint *count,
+                    MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm,
+                    MPI_Fint *ierr);
+
+void ompi_gather_ts(CFI_cdesc_t *x1, MPI_Fint *sendcount, MPI_Fint *sendtype,
+                    CFI_cdesc_t *x2, MPI_Fint *recvcount, MPI_Fint *recvtype,
+                    MPI_Fint *root, MPI_Fint *comm, MPI_Fint *ierr);
+
+void ompi_gatherv_ts(CFI_cdesc_t *x1, MPI_Fint *sendcount, MPI_Fint *sendtype,
+                     CFI_cdesc_t *x2, MPI_Fint *recvcounts, MPI_Fint *displs,
+                     MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm,
+                     MPI_Fint *ierr);
+
+void ompi_reduce_ts(CFI_cdesc_t *x1, CFI_cdesc_t *x2, MPI_Fint *count,
+                    MPI_Fint *datatype, MPI_Fint *op,
+                    MPI_Fint *root, MPI_Fint *comm, MPI_Fint *ierr);
+
+void ompi_reduce_local_ts(CFI_cdesc_t *x1, CFI_cdesc_t *x2, MPI_Fint *count,
+                          MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *ierr);
+
+void ompi_reduce_scatter_ts(CFI_cdesc_t *x1, CFI_cdesc_t *x2,
+                            MPI_Fint *recvcounts, MPI_Fint *datatype,
+                            MPI_Fint *op, MPI_Fint *comm, MPI_Fint *ierr);
+
+void ompi_reduce_scatter_block_ts(CFI_cdesc_t *x1, CFI_cdesc_t *x2,
+                                  MPI_Fint *recvcount, MPI_Fint *datatype,
+                                  MPI_Fint *op, MPI_Fint *comm, MPI_Fint *ierr);
+
+void ompi_scan_ts(CFI_cdesc_t *x1, CFI_cdesc_t *x2, MPI_Fint *count,
+                  MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm,
+                  MPI_Fint *ierr);
+
+void ompi_scatter_ts(CFI_cdesc_t *x1, MPI_Fint *sendcount,
+                     MPI_Fint *sendtype, CFI_cdesc_t *x2,
+                     MPI_Fint *recvcount, MPI_Fint *recvtype,
+                     MPI_Fint *root, MPI_Fint *comm, MPI_Fint *ierr);
+
+void ompi_scatterv_ts(CFI_cdesc_t *x1, MPI_Fint *sendcounts,
+                      MPI_Fint *displs, MPI_Fint *sendtype,
+                      CFI_cdesc_t *x2, MPI_Fint *recvcount,
+                      MPI_Fint *recvtype, MPI_Fint *root,
+                      MPI_Fint *comm, MPI_Fint *ierr);
+
+void ompi_free_mem_ts(CFI_cdesc_t *x, MPI_Fint *ierr);
+
+void ompi_f_sync_reg_ts(CFI_cdesc_t *x);
+
+void ompi_imrecv_ts(CFI_cdesc_t* x, MPI_Fint *count, MPI_Fint *datatype,
+                    MPI_Fint *message, MPI_Fint *request, MPI_Fint *ierr);
+
+void ompi_mrecv_ts(CFI_cdesc_t* x, MPI_Fint *count, MPI_Fint *datatype,
+                   MPI_Fint *message, MPI_Fint *status, MPI_Fint *ierr);
+
+void ompi_neighbor_allgather_ts(CFI_cdesc_t *x1, MPI_Fint *sendcount, MPI_Fint *sendtype,
+                                CFI_cdesc_t *x2, MPI_Fint *recvcount, MPI_Fint *recvtype,
+                                MPI_Fint *comm, MPI_Fint *ierr);
+
+void ompi_neighbor_allgatherv_ts(CFI_cdesc_t *x1, MPI_Fint *sendcount, MPI_Fint *sendtype,
+                                 CFI_cdesc_t *x2, MPI_Fint *recvcounts, MPI_Fint *displs,
+                                 MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *ierr);
+
+void ompi_neighbor_alltoall_ts(CFI_cdesc_t *x1, MPI_Fint *sendcount, MPI_Fint *sendtype,
+                               CFI_cdesc_t *x2, MPI_Fint *recvcount, MPI_Fint *recvtype,
+                               MPI_Fint *comm, MPI_Fint *ierr);
+
+void ompi_neighbor_alltoallv_ts(CFI_cdesc_t *x1, MPI_Fint *sendcounts, MPI_Fint *sdispls,
+                                MPI_Fint *sendtype, CFI_cdesc_t *x2, MPI_Fint *recvcounts,
+                                MPI_Fint *rdispls, MPI_Fint *recvtype,
+                                MPI_Fint *comm, MPI_Fint *ierr);
+
+void ompi_neighbor_alltoallw_ts(CFI_cdesc_t *x1, MPI_Fint *sendcounts,
+                                MPI_Aint *sdispls, MPI_Fint *sendtypes,
+                                CFI_cdesc_t *x2, MPI_Fint *recvcounts,
+                                MPI_Aint *rdispls, MPI_Fint *recvtypes,
+                                MPI_Fint *comm, MPI_Fint *ierr);
+
+#endif /* OMPI_CDESC_BINDINGS_H */
diff --git a/ompi/mpi/fortran/use-mpi-f08/bindings/Makefile.am b/ompi/mpi/fortran/use-mpi-f08/bindings/Makefile.am
index 9e038f7e87f..0945ab72b54 100644
--- a/ompi/mpi/fortran/use-mpi-f08/bindings/Makefile.am
+++ b/ompi/mpi/fortran/use-mpi-f08/bindings/Makefile.am
@@ -34,7 +34,7 @@ if OMPI_BUILD_FORTRAN_USEMPIF08_BINDINGS
 AM_FCFLAGS = -I$(top_builddir)/ompi/include \
              -I$(top_srcdir)/ompi/include \
              $(OMPI_FC_MODULE_FLAG)$(top_builddir)/ompi/mpi/fortran/use-mpi \
-             $(OMPI_FC_MODULE_FLAG)$(top_builddir)/ompi/$(OMPI_FORTRAN_USEMPI_DIR) \
+             $(OMPI_FC_MODULE_FLAG)$(top_builddir)/ompi/mpi/fortran/use-mpi-ignore-tkr \
              $(OMPI_FC_MODULE_FLAG). \
              $(OMPI_FC_MODULE_FLAG)../mod \
              -I$(top_srcdir) -I$(top_builddir) $(FCFLAGS_f90)
diff --git a/ompi/mpi/fortran/use-mpi-f08/bindings/mpi-f-interfaces-bind.h b/ompi/mpi/fortran/use-mpi-f08/bindings/mpi-f-interfaces-bind.h
index 140ad0d01f2..3b1ac4ff827 100644
--- a/ompi/mpi/fortran/use-mpi-f08/bindings/mpi-f-interfaces-bind.h
+++ b/ompi/mpi/fortran/use-mpi-f08/bindings/mpi-f-interfaces-bind.h
@@ -1,6 +1,6 @@
 ! -*- f90 -*-
 !
-! Copyright (c) 2009-2015 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2009-2022 Cisco Systems, Inc.  All rights reserved
 ! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
 !                         All rights reserved.
 ! Copyright (c) 2012      The University of Tennessee and The University
@@ -9,6 +9,11 @@
 ! Copyright (c) 2012      Inria.  All rights reserved.
 ! Copyright (c) 2015-2020 Research Organization for Information Science
 !                         and Technology (RIST).  All rights reserved.
+! Copyright (c) 2019      Triad National Security, LLC. All rights
+!                         reserved.
+! Copyright (c) 2021      Bull S.A.S. All rights reserved.
+! Copyright (c) 2021-2022 Triad National Security, LLC. All rights
+!                         reserved.
 ! $COPYRIGHT$
 !
 ! This file provides the interface specifications for the MPI Fortran
@@ -145,6 +150,7 @@
 ! MPI_Is_thread_main
 ! MPI_Op_commutative
 ! MPI_Op_create
+! MPI_Parrived
 ! MPI_Type_get_attr
 ! MPI_Win_get_attr
 ! MPI_Win_test
@@ -176,7 +182,7 @@ end subroutine ompi_bsend_f
 subroutine ompi_bsend_init_f(buf,count,datatype,dest,tag,comm,request,ierror) &
    BIND(C, name="ompi_bsend_init_f")
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: buf
    INTEGER, INTENT(IN) :: count, dest, tag
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(IN) :: comm
@@ -187,7 +193,7 @@ end subroutine ompi_bsend_init_f
 subroutine ompi_buffer_attach_f(buffer,size,ierror) &
    BIND(C, name="ompi_buffer_attach_f")
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buffer
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: buffer
    INTEGER, INTENT(IN) :: size
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_buffer_attach_f
@@ -224,7 +230,7 @@ end subroutine ompi_get_count_f
 subroutine ompi_ibsend_f(buf,count,datatype,dest,tag,comm,request,ierror) &
    BIND(C, name="ompi_ibsend_f")
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: buf
    INTEGER, INTENT(IN) :: count, dest, tag
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(IN) :: comm
@@ -235,7 +241,7 @@ end subroutine ompi_ibsend_f
 subroutine ompi_irecv_f(buf,count,datatype,source,tag,comm,request,ierror) &
    BIND(C, name="ompi_irecv_f")
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: buf
    INTEGER, INTENT(IN) :: count, source, tag
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(IN) :: comm
@@ -246,7 +252,7 @@ end subroutine ompi_irecv_f
 subroutine ompi_irsend_f(buf,count,datatype,dest,tag,comm,request,ierror) &
    BIND(C, name="ompi_irsend_f")
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: buf
    INTEGER, INTENT(IN) :: count, dest, tag
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(IN) :: comm
@@ -257,7 +263,7 @@ end subroutine ompi_irsend_f
 subroutine ompi_isend_f(buf,count,datatype,dest,tag,comm,request,ierror) &
    BIND(C, name="ompi_isend_f")
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: buf
    INTEGER, INTENT(IN) :: count, dest, tag
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(IN) :: comm
@@ -265,10 +271,36 @@ subroutine ompi_isend_f(buf,count,datatype,dest,tag,comm,request,ierror) &
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_isend_f
 
+subroutine ompi_isendrecv_f(sendbuf,sendcount,sendtype,dest,sendtag,recvbuf, &
+                           recvcount,recvtype,source,recvtag,comm,request,ierror) &
+   BIND(C, name="ompi_isendrecv_f")
+   implicit none
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
+   INTEGER, INTENT(IN) :: sendcount, dest, sendtag, recvcount, source, recvtag
+   INTEGER, INTENT(IN) :: sendtype
+   INTEGER, INTENT(IN) :: recvtype
+   INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(OUT) :: request 
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_isendrecv_f
+
+subroutine ompi_isendrecv_replace_f(buf,count,datatype,dest,sendtag,source, &
+                                   recvtag,comm,request,ierror) &
+   BIND(C, name="ompi_isendrecv_replace_f")
+   implicit none
+   OMPI_FORTRAN_IGNORE_TKR_TYPE :: buf
+   INTEGER, INTENT(IN) :: count, dest, sendtag, source, recvtag
+   INTEGER, INTENT(IN) :: datatype
+   INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(OUT) :: request
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_isendrecv_replace_f
+
 subroutine ompi_issend_f(buf,count,datatype,dest,tag,comm,request,ierror) &
    BIND(C, name="ompi_issend_f")
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: buf
    INTEGER, INTENT(IN) :: count, dest, tag
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(IN) :: comm
@@ -276,13 +308,66 @@ subroutine ompi_issend_f(buf,count,datatype,dest,tag,comm,request,ierror) &
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_issend_f
 
+subroutine ompi_psend_init_f(buf,partitions,count,datatype,dest,tag,comm,info,request,ierror) &
+   BIND(C, name="ompi_psend_init_f")
+   use :: mpi_f08_types, only : MPI_COUNT_KIND
+   implicit none
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   INTEGER, INTENT(IN) :: partitions, dest, tag
+   INTEGER(KIND=MPI_COUNT_KIND), INTENT(IN):: count
+   INTEGER, INTENT(IN) :: datatype
+   INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(IN) :: info
+   INTEGER, INTENT(OUT) :: request
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_psend_init_f
+
+subroutine ompi_precv_init_f(buf,partitions,count,datatype,dest,tag,comm,info,request,ierror) &
+   BIND(C, name="ompi_precv_init_f")
+   use :: mpi_f08_types, only : MPI_COUNT_KIND
+   implicit none
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   INTEGER, INTENT(IN) :: partitions, dest, tag
+   INTEGER(KIND=MPI_COUNT_KIND), INTENT(IN):: count
+   INTEGER, INTENT(IN) :: datatype
+   INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(IN) :: info
+   INTEGER, INTENT(OUT) :: request
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_precv_init_f
+
+subroutine ompi_pready_f(partition,request,ierror) &
+   BIND(C, name="ompi_pready_f")
+   implicit none
+   INTEGER, INTENT(IN) :: partition
+   INTEGER, INTENT(IN) :: request
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_pready_f
+
+subroutine ompi_pready_list_f(length,partitions,request,ierror) &
+   BIND(C, name="ompi_pready_list_f")
+   implicit none
+   INTEGER, INTENT(IN) :: length
+   INTEGER, INTENT(IN) :: partitions(*)
+   INTEGER, INTENT(IN) :: request
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_pready_list_f
+
+subroutine ompi_pready_range_f(partition_low,partition_high,request,ierror) &
+   BIND(C, name="ompi_pready_range_f")
+   implicit none
+   INTEGER, INTENT(IN) :: partition_low, partition_high
+   INTEGER, INTENT(IN) :: request
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_pready_range_f
+
 subroutine ompi_probe_f(source,tag,comm,status,ierror) &
    BIND(C, name="ompi_probe_f")
    use :: mpi_f08_types, only : MPI_Status
    implicit none
    INTEGER, INTENT(IN) :: source, tag
    INTEGER, INTENT(IN) :: comm
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   TYPE(MPI_Status) :: status
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_probe_f
 
@@ -294,14 +379,14 @@ subroutine ompi_recv_f(buf,count,datatype,source,tag,comm,status,ierror) &
    INTEGER, INTENT(IN) :: count, source, tag
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(IN) :: comm
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   TYPE(MPI_Status) :: status
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_recv_f
 
 subroutine ompi_recv_init_f(buf,count,datatype,source,tag,comm,request,ierror) &
    BIND(C, name="ompi_recv_init_f")
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: buf
    INTEGER, INTENT(IN) :: count, source, tag
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(IN) :: comm
@@ -329,7 +414,7 @@ end subroutine ompi_rsend_f
 subroutine ompi_rsend_init_f(buf,count,datatype,dest,tag,comm,request,ierror) &
    BIND(C, name="ompi_rsend_init_f")
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: buf
    INTEGER, INTENT(IN) :: count, dest, tag
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(IN) :: comm
@@ -358,7 +443,7 @@ subroutine ompi_sendrecv_f(sendbuf,sendcount,sendtype,dest,sendtag,recvbuf, &
    INTEGER, INTENT(IN) :: sendtype
    INTEGER, INTENT(IN) :: recvtype
    INTEGER, INTENT(IN) :: comm
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   TYPE(MPI_Status) :: status
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_sendrecv_f
 
@@ -371,14 +456,14 @@ subroutine ompi_sendrecv_replace_f(buf,count,datatype,dest,sendtag,source, &
    INTEGER, INTENT(IN) :: count, dest, sendtag, source, recvtag
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(IN) :: comm
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   TYPE(MPI_Status) :: status
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_sendrecv_replace_f
 
 subroutine ompi_send_init_f(buf,count,datatype,dest,tag,comm,request,ierror) &
    BIND(C, name="ompi_send_init_f")
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: buf
    INTEGER, INTENT(IN) :: count, dest, tag
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(IN) :: comm
@@ -399,7 +484,7 @@ end subroutine ompi_ssend_f
 subroutine ompi_ssend_init_f(buf,count,datatype,dest,tag,comm,request,ierror) &
    BIND(C, name="ompi_ssend_init_f")
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: buf
    INTEGER, INTENT(IN) :: count, dest, tag
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(IN) :: comm
@@ -427,7 +512,7 @@ subroutine ompi_wait_f(request,status,ierror) &
    use :: mpi_f08_types, only : MPI_Status
    implicit none
    INTEGER, INTENT(INOUT) :: request
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   TYPE(MPI_Status) :: status
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_wait_f
 
@@ -437,7 +522,7 @@ subroutine ompi_waitall_f(count,array_of_requests,array_of_statuses,ierror) &
    implicit none
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(INOUT) :: array_of_requests(count)
-   TYPE(MPI_Status), INTENT(OUT) :: array_of_statuses(count)
+   TYPE(MPI_Status) :: array_of_statuses(*)
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_waitall_f
 
@@ -448,7 +533,7 @@ subroutine ompi_waitany_f(count,array_of_requests,index,status,ierror) &
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(INOUT) :: array_of_requests(count)
    INTEGER, INTENT(OUT) :: index
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   TYPE(MPI_Status) :: status
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_waitany_f
 
@@ -461,7 +546,7 @@ subroutine ompi_waitsome_f(incount,array_of_requests,outcount, &
    INTEGER, INTENT(INOUT) :: array_of_requests(incount)
    INTEGER, INTENT(OUT) :: outcount
    INTEGER, INTENT(OUT) :: array_of_indices(*)
-   TYPE(MPI_Status), INTENT(OUT) :: array_of_statuses(*)
+   TYPE(MPI_Status) :: array_of_statuses(*)
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_waitsome_f
 
@@ -469,7 +554,7 @@ subroutine ompi_get_address_f(location,address,ierror) &
    BIND(C, name="ompi_get_address_f")
    use :: mpi_f08_types, only : MPI_ADDRESS_KIND
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: location
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: location
    INTEGER(MPI_ADDRESS_KIND), INTENT(OUT) :: address
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_get_address_f
@@ -510,7 +595,7 @@ end subroutine ompi_pack_f
 subroutine ompi_pack_external_f(datarep,inbuf,incount,datatype, &
                                 outbuf,outsize,position,ierror,datarep_len) &
    BIND(C, name="ompi_pack_external_f")
-   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
    use :: mpi_f08_types, only : MPI_ADDRESS_KIND
    implicit none
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(IN) :: datarep
@@ -521,12 +606,12 @@ subroutine ompi_pack_external_f(datarep,inbuf,incount,datatype, &
    INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: outsize
    INTEGER(MPI_ADDRESS_KIND), INTENT(INOUT) :: position
    INTEGER, INTENT(OUT) :: ierror
-   INTEGER, VALUE, INTENT(IN) :: datarep_len
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: datarep_len
 end subroutine ompi_pack_external_f
 
 subroutine ompi_pack_external_size_f(datarep,incount,datatype,size,ierror,datarep_len) &
    BIND(C, name="ompi_pack_external_size_f")
-   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
    use :: mpi_f08_types, only : MPI_ADDRESS_KIND
    implicit none
    INTEGER, INTENT(IN) :: datatype
@@ -534,7 +619,7 @@ subroutine ompi_pack_external_size_f(datarep,incount,datatype,size,ierror,datare
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(IN) :: datarep
    INTEGER(MPI_ADDRESS_KIND), INTENT(OUT) :: size
    INTEGER, INTENT(OUT) :: ierror
-   INTEGER, VALUE, INTENT(IN) :: datarep_len
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: datarep_len
 end subroutine ompi_pack_external_size_f
 
 subroutine ompi_pack_size_f(incount,datatype,comm,size,ierror) &
@@ -651,7 +736,7 @@ subroutine ompi_type_create_subarray_f(ndims,array_of_sizes, &
    BIND(C, name="ompi_type_create_subarray_f")
    implicit none
    INTEGER, INTENT(IN) :: ndims, order
-   INTEGER, INTENT(IN) :: array_of_sizes(*), array_of_subsizes(*), array_of_starts(*)
+   INTEGER, INTENT(IN) :: array_of_sizes(ndims), array_of_subsizes(ndims), array_of_starts(ndims)
    INTEGER, INTENT(IN) :: oldtype
    INTEGER, INTENT(OUT) :: newtype
    INTEGER, INTENT(OUT) :: ierror
@@ -686,6 +771,21 @@ subroutine ompi_type_get_contents_f(datatype,max_integers,max_addresses, &
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_type_get_contents_f
 
+subroutine ompi_type_get_contents_f_c(datatype,max_integers,max_addresses, &
+                    max_large_counts, max_datatypes,array_of_integers,array_of_addresses, &
+                    array_of_large_counts, array_of_datatypes,ierror) &
+   BIND(C, name="ompi_type_get_contents_f_c")
+   use :: mpi_f08_types, only : MPI_ADDRESS_KIND, MPI_COUNT_KIND
+   implicit none
+   INTEGER, INTENT(IN) :: datatype
+   INTEGER(KIND=MPI_COUNT_KIND), INTENT(IN) :: max_integers, max_addresses, max_large_counts, max_datatypes
+   INTEGER, INTENT(OUT) :: array_of_integers(max_integers)
+   INTEGER(MPI_ADDRESS_KIND), INTENT(OUT) :: array_of_addresses(max_addresses)
+   INTEGER(KIND=MPI_COUNT_KIND), INTENT(OUT) :: array_of_large_counts(max_large_counts)
+   INTEGER, INTENT(OUT) :: array_of_datatypes(max_datatypes)
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_type_get_contents_f_c
+
 subroutine ompi_type_get_envelope_f(datatype,num_integers, &
                                     num_addresses,num_datatypes,combiner,ierror) &
    BIND(C, name="ompi_type_get_envelope_f")
@@ -695,6 +795,17 @@ subroutine ompi_type_get_envelope_f(datatype,num_integers, &
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_type_get_envelope_f
 
+subroutine ompi_type_get_envelope_f_c(datatype,num_integers, num_addresses, &
+                                      num_large_counts, num_datatypes,combiner,ierror) &
+   BIND(C, name="ompi_type_get_envelope_f_c")
+   use :: mpi_f08_types, only : MPI_COUNT_KIND
+   implicit none
+   INTEGER, INTENT(IN) :: datatype
+   INTEGER(KIND=MPI_COUNT_KIND), INTENT(OUT) :: num_integers, num_large_counts, num_addresses, &
+                                                num_datatypes, combiner
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_type_get_envelope_f_c
+
 subroutine ompi_type_get_extent_f(datatype,lb,extent,ierror) &
    BIND(C, name="ompi_type_get_extent_f")
    use :: mpi_f08_types, only : MPI_ADDRESS_KIND
@@ -784,7 +895,7 @@ end subroutine ompi_unpack_f
 subroutine ompi_unpack_external_f(datarep,inbuf,insize,position, &
                                   outbuf,outcount,datatype,ierror,datarep_len) &
    BIND(C, name="ompi_unpack_external_f")
-   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
    use :: mpi_f08_types, only : MPI_ADDRESS_KIND
    implicit none
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(IN) :: datarep
@@ -795,7 +906,7 @@ subroutine ompi_unpack_external_f(datarep,inbuf,insize,position, &
    INTEGER, INTENT(IN) :: outcount
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(OUT) :: ierror
-   INTEGER, VALUE, INTENT(IN) :: datarep_len
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: datarep_len
 end subroutine ompi_unpack_external_f
 
 subroutine ompi_allgather_f(sendbuf,sendcount,sendtype,recvbuf, &
@@ -812,11 +923,11 @@ subroutine ompi_allgather_f(sendbuf,sendcount,sendtype,recvbuf, &
 end subroutine ompi_allgather_f
 
 subroutine ompi_iallgather_f(sendbuf,sendcount,sendtype,recvbuf, &
-                            recvcount,recvtype,comm,request,ierror) &
+                             recvcount,recvtype,comm,request,ierror) &
    BIND(C, name="ompi_iallgather_f")
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
    INTEGER, INTENT(IN) :: sendcount, recvcount
    INTEGER, INTENT(IN) :: sendtype
    INTEGER, INTENT(IN) :: recvtype
@@ -825,6 +936,21 @@ subroutine ompi_iallgather_f(sendbuf,sendcount,sendtype,recvbuf, &
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_iallgather_f
 
+subroutine ompi_allgather_init_f(sendbuf,sendcount,sendtype,recvbuf, &
+                                 recvcount,recvtype,comm,info,request,ierror) &
+   BIND(C, name="ompi_allgather_init_f")
+   implicit none
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
+   INTEGER, INTENT(IN) :: sendcount, recvcount
+   INTEGER, INTENT(IN) :: sendtype
+   INTEGER, INTENT(IN) :: recvtype
+   INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(IN) :: info
+   INTEGER, INTENT(OUT) :: request
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_allgather_init_f
+
 subroutine ompi_allgatherv_f(sendbuf,sendcount,sendtype,recvbuf, &
                              recvcounts,displs,recvtype,comm,ierror) &
    BIND(C, name="ompi_allgatherv_f")
@@ -843,10 +969,10 @@ subroutine ompi_iallgatherv_f(sendbuf,sendcount,sendtype,recvbuf, &
                              recvcounts,displs,recvtype,comm,request,ierror) &
    BIND(C, name="ompi_iallgatherv_f")
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
    INTEGER, INTENT(IN) :: sendcount
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: recvcounts(*), displs(*)
+   INTEGER, INTENT(IN) OMPI_ASYNCHRONOUS :: recvcounts(*), displs(*)
    INTEGER, INTENT(IN) :: sendtype
    INTEGER, INTENT(IN) :: recvtype
    INTEGER, INTENT(IN) :: comm
@@ -854,6 +980,22 @@ subroutine ompi_iallgatherv_f(sendbuf,sendcount,sendtype,recvbuf, &
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_iallgatherv_f
 
+subroutine ompi_allgatherv_init_f(sendbuf,sendcount,sendtype,recvbuf, &
+                             recvcounts,displs,recvtype,comm,info,request,ierror) &
+   BIND(C, name="ompi_allgatherv_init_f")
+   implicit none
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
+   INTEGER, INTENT(IN) :: sendcount
+   INTEGER, INTENT(IN) OMPI_ASYNCHRONOUS :: recvcounts(*), displs(*)
+   INTEGER, INTENT(IN) :: sendtype
+   INTEGER, INTENT(IN) :: recvtype
+   INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(IN) :: info
+   INTEGER, INTENT(OUT) :: request
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_allgatherv_init_f
+
 subroutine ompi_allreduce_f(sendbuf,recvbuf,count,datatype,op,comm,ierror) &
    BIND(C, name="ompi_allreduce_f")
    implicit none
@@ -869,8 +1011,8 @@ end subroutine ompi_allreduce_f
 subroutine ompi_iallreduce_f(sendbuf,recvbuf,count,datatype,op,comm,request,ierror) &
    BIND(C, name="ompi_iallreduce_f")
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(IN) :: op
@@ -879,6 +1021,20 @@ subroutine ompi_iallreduce_f(sendbuf,recvbuf,count,datatype,op,comm,request,ierr
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_iallreduce_f
 
+subroutine ompi_allreduce_init_f(sendbuf,recvbuf,count,datatype,op,comm,info,request,ierror) &
+   BIND(C, name="ompi_allreduce_init_f")
+   implicit none
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
+   INTEGER, INTENT(IN) :: count
+   INTEGER, INTENT(IN) :: datatype
+   INTEGER, INTENT(IN) :: op
+   INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(IN) :: info
+   INTEGER, INTENT(OUT) :: request
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_allreduce_init_f
+
 subroutine ompi_alltoall_f(sendbuf,sendcount,sendtype,recvbuf, &
                            recvcount,recvtype,comm,ierror) &
    BIND(C, name="ompi_alltoall_f")
@@ -896,8 +1052,8 @@ subroutine ompi_ialltoall_f(sendbuf,sendcount,sendtype,recvbuf, &
                            recvcount,recvtype,comm,request,ierror) &
    BIND(C, name="ompi_ialltoall_f")
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
    INTEGER, INTENT(IN) :: sendcount, recvcount
    INTEGER, INTENT(IN) :: sendtype
    INTEGER, INTENT(IN) :: recvtype
@@ -906,6 +1062,21 @@ subroutine ompi_ialltoall_f(sendbuf,sendcount,sendtype,recvbuf, &
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_ialltoall_f
 
+subroutine ompi_alltoall_init_f(sendbuf,sendcount,sendtype,recvbuf, &
+                           recvcount,recvtype,comm,info,request,ierror) &
+   BIND(C, name="ompi_alltoall_init_f")
+   implicit none
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
+   INTEGER, INTENT(IN) :: sendcount, recvcount
+   INTEGER, INTENT(IN) :: sendtype
+   INTEGER, INTENT(IN) :: recvtype
+   INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(IN) :: info
+   INTEGER, INTENT(OUT) :: request
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_alltoall_init_f
+
 subroutine ompi_alltoallv_f(sendbuf,sendcounts,sdispls,sendtype, &
                             recvbuf,recvcounts,rdispls,recvtype,comm,ierror) &
    BIND(C, name="ompi_alltoallv_f")
@@ -923,9 +1094,9 @@ subroutine ompi_ialltoallv_f(sendbuf,sendcounts,sdispls,sendtype, &
                             recvbuf,recvcounts,rdispls,recvtype,comm,request,ierror) &
    BIND(C, name="ompi_ialltoallv_f")
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
+   INTEGER, INTENT(IN) OMPI_ASYNCHRONOUS :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
    INTEGER, INTENT(IN) :: sendtype
    INTEGER, INTENT(IN) :: recvtype
    INTEGER, INTENT(IN) :: comm
@@ -933,6 +1104,21 @@ subroutine ompi_ialltoallv_f(sendbuf,sendcounts,sdispls,sendtype, &
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_ialltoallv_f
 
+subroutine ompi_alltoallv_init_f(sendbuf,sendcounts,sdispls,sendtype, &
+                            recvbuf,recvcounts,rdispls,recvtype,comm,info,request,ierror) &
+   BIND(C, name="ompi_alltoallv_init_f")
+   implicit none
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
+   INTEGER, INTENT(IN) OMPI_ASYNCHRONOUS :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
+   INTEGER, INTENT(IN) :: sendtype
+   INTEGER, INTENT(IN) :: recvtype
+   INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(IN) :: info
+   INTEGER, INTENT(OUT) :: request
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_alltoallv_init_f
+
 subroutine ompi_alltoallw_f(sendbuf,sendcounts,sdispls,sendtypes, &
                             recvbuf,recvcounts,rdispls,recvtypes,comm,ierror) &
    BIND(C, name="ompi_alltoallw_f")
@@ -950,16 +1136,31 @@ subroutine ompi_ialltoallw_f(sendbuf,sendcounts,sdispls,sendtypes, &
                             recvbuf,recvcounts,rdispls,recvtypes,comm,request,ierror) &
    BIND(C, name="ompi_ialltoallw_f")
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: sendtypes
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: recvtypes
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
+   INTEGER, INTENT(IN) OMPI_ASYNCHRONOUS :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
+   INTEGER, INTENT(IN) OMPI_ASYNCHRONOUS :: sendtypes
+   INTEGER, INTENT(IN) OMPI_ASYNCHRONOUS :: recvtypes
    INTEGER, INTENT(IN) :: comm
    INTEGER, INTENT(OUT) :: request
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_ialltoallw_f
 
+subroutine ompi_alltoallw_init_f(sendbuf,sendcounts,sdispls,sendtypes, &
+                            recvbuf,recvcounts,rdispls,recvtypes,comm,info,request,ierror) &
+   BIND(C, name="ompi_alltoallw_init_f")
+   implicit none
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
+   INTEGER, INTENT(IN) OMPI_ASYNCHRONOUS :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
+   INTEGER, INTENT(IN) OMPI_ASYNCHRONOUS :: sendtypes
+   INTEGER, INTENT(IN) OMPI_ASYNCHRONOUS :: recvtypes
+   INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(IN) :: info
+   INTEGER, INTENT(OUT) :: request
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_alltoallw_init_f
+
 subroutine ompi_barrier_f(comm,ierror) &
    BIND(C, name="ompi_barrier_f")
    implicit none
@@ -975,6 +1176,15 @@ subroutine ompi_ibarrier_f(comm,request,ierror) &
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_ibarrier_f
 
+subroutine ompi_barrier_init_f(comm,info,request,ierror) &
+   BIND(C, name="ompi_barrier_init_f")
+   implicit none
+   INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(IN) :: info
+   INTEGER, INTENT(OUT) :: request
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_barrier_init_f
+
 subroutine ompi_bcast_f(buffer,count,datatype,root,comm,ierror) &
    BIND(C, name="ompi_bcast_f")
    implicit none
@@ -988,7 +1198,7 @@ end subroutine ompi_bcast_f
 subroutine ompi_ibcast_f(buffer,count,datatype,root,comm,request,ierror) &
    BIND(C, name="ompi_ibcast_f")
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: buffer
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: buffer
    INTEGER, INTENT(IN) :: count, root
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(IN) :: comm
@@ -996,6 +1206,18 @@ subroutine ompi_ibcast_f(buffer,count,datatype,root,comm,request,ierror) &
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_ibcast_f
 
+subroutine ompi_bcast_init_f(buffer,count,datatype,root,comm,info,request,ierror) &
+   BIND(C, name="ompi_bcast_init_f")
+   implicit none
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: buffer
+   INTEGER, INTENT(IN) :: count, root
+   INTEGER, INTENT(IN) :: datatype
+   INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(IN) :: info
+   INTEGER, INTENT(OUT) :: request
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_bcast_init_f
+
 subroutine ompi_exscan_f(sendbuf,recvbuf,count,datatype,op,comm,ierror) &
    BIND(C, name="ompi_exscan_f")
    implicit none
@@ -1011,8 +1233,8 @@ end subroutine ompi_exscan_f
 subroutine ompi_iexscan_f(sendbuf,recvbuf,count,datatype,op,comm,request,ierror) &
    BIND(C, name="ompi_iexscan_f")
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(IN) :: op
@@ -1021,6 +1243,20 @@ subroutine ompi_iexscan_f(sendbuf,recvbuf,count,datatype,op,comm,request,ierror)
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_iexscan_f
 
+subroutine ompi_exscan_init_f(sendbuf,recvbuf,count,datatype,op,comm,info,request,ierror) &
+   BIND(C, name="ompi_exscan_init_f")
+   implicit none
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
+   INTEGER, INTENT(IN) :: count
+   INTEGER, INTENT(IN) :: datatype
+   INTEGER, INTENT(IN) :: op
+   INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(IN) :: info
+   INTEGER, INTENT(OUT) :: request
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_exscan_init_f
+
 subroutine ompi_gather_f(sendbuf,sendcount,sendtype,recvbuf, &
                          recvcount,recvtype,root,comm,ierror) &
    BIND(C, name="ompi_gather_f")
@@ -1038,8 +1274,8 @@ subroutine ompi_igather_f(sendbuf,sendcount,sendtype,recvbuf, &
                          recvcount,recvtype,root,comm,request,ierror) &
    BIND(C, name="ompi_igather_f")
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
    INTEGER, INTENT(IN) :: sendcount, recvcount, root
    INTEGER, INTENT(IN) :: sendtype
    INTEGER, INTENT(IN) :: recvtype
@@ -1048,6 +1284,21 @@ subroutine ompi_igather_f(sendbuf,sendcount,sendtype,recvbuf, &
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_igather_f
 
+subroutine ompi_gather_init_f(sendbuf,sendcount,sendtype,recvbuf, &
+                         recvcount,recvtype,root,comm,info,request,ierror) &
+   BIND(C, name="ompi_gather_init_f")
+   implicit none
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
+   INTEGER, INTENT(IN) :: sendcount, recvcount, root
+   INTEGER, INTENT(IN) :: sendtype
+   INTEGER, INTENT(IN) :: recvtype
+   INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(IN) :: info
+   INTEGER, INTENT(OUT) :: request
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_gather_init_f
+
 subroutine ompi_gatherv_f(sendbuf,sendcount,sendtype,recvbuf, &
                           recvcounts,displs,recvtype,root,comm,ierror) &
    BIND(C, name="ompi_gatherv_f")
@@ -1066,10 +1317,10 @@ subroutine ompi_igatherv_f(sendbuf,sendcount,sendtype,recvbuf, &
                           recvcounts,displs,recvtype,root,comm,request,ierror) &
    BIND(C, name="ompi_igatherv_f")
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
    INTEGER, INTENT(IN) :: sendcount, root
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: recvcounts(*), displs(*)
+   INTEGER, INTENT(IN) OMPI_ASYNCHRONOUS :: recvcounts(*), displs(*)
    INTEGER, INTENT(IN) :: sendtype
    INTEGER, INTENT(IN) :: recvtype
    INTEGER, INTENT(IN) :: comm
@@ -1077,6 +1328,22 @@ subroutine ompi_igatherv_f(sendbuf,sendcount,sendtype,recvbuf, &
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_igatherv_f
 
+subroutine ompi_gatherv_init_f(sendbuf,sendcount,sendtype,recvbuf, &
+                          recvcounts,displs,recvtype,root,comm,info,request,ierror) &
+   BIND(C, name="ompi_gatherv_init_f")
+   implicit none
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
+   INTEGER, INTENT(IN) :: sendcount, root
+   INTEGER, INTENT(IN) OMPI_ASYNCHRONOUS :: recvcounts(*), displs(*)
+   INTEGER, INTENT(IN) :: sendtype
+   INTEGER, INTENT(IN) :: recvtype
+   INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(IN) :: info
+   INTEGER, INTENT(OUT) :: request
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_gatherv_init_f
+
 subroutine ompi_op_free_f(op,ierror) &
    BIND(C, name="ompi_op_free_f")
    implicit none
@@ -1099,8 +1366,8 @@ end subroutine ompi_reduce_f
 subroutine ompi_ireduce_f(sendbuf,recvbuf,count,datatype,op,root,comm,request,ierror) &
    BIND(C, name="ompi_ireduce_f")
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
    INTEGER, INTENT(IN) :: count, root
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(IN) :: op
@@ -1109,6 +1376,20 @@ subroutine ompi_ireduce_f(sendbuf,recvbuf,count,datatype,op,root,comm,request,ie
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_ireduce_f
 
+subroutine ompi_reduce_init_f(sendbuf,recvbuf,count,datatype,op,root,comm,info,request,ierror) &
+   BIND(C, name="ompi_reduce_init_f")
+   implicit none
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
+   INTEGER, INTENT(IN) :: count, root
+   INTEGER, INTENT(IN) :: datatype
+   INTEGER, INTENT(IN) :: op
+   INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(IN) :: info
+   INTEGER, INTENT(OUT) :: request
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_reduce_init_f
+
 subroutine ompi_reduce_local_f(inbuf,inoutbuf,count,datatype,op,ierror) &
    BIND(C, name="ompi_reduce_local_f")
    implicit none
@@ -1137,9 +1418,9 @@ subroutine ompi_ireduce_scatter_f(sendbuf,recvbuf,recvcounts, &
                                  datatype,op,comm,request,ierror) &
    BIND(C, name="ompi_ireduce_scatter_f")
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: recvcounts(*)
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
+   INTEGER, INTENT(IN) OMPI_ASYNCHRONOUS :: recvcounts(*)
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(IN) :: op
    INTEGER, INTENT(IN) :: comm
@@ -1147,6 +1428,21 @@ subroutine ompi_ireduce_scatter_f(sendbuf,recvbuf,recvcounts, &
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_ireduce_scatter_f
 
+subroutine ompi_reduce_scatter_init_f(sendbuf,recvbuf,recvcounts, &
+                                 datatype,op,comm,info,request,ierror) &
+   BIND(C, name="ompi_reduce_scatter_init_f")
+   implicit none
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
+   INTEGER, INTENT(IN) OMPI_ASYNCHRONOUS :: recvcounts(*)
+   INTEGER, INTENT(IN) :: datatype
+   INTEGER, INTENT(IN) :: op
+   INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(IN) :: info
+   INTEGER, INTENT(OUT) :: request
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_reduce_scatter_init_f
+
 subroutine ompi_reduce_scatter_block_f(sendbuf,recvbuf,recvcount, &
                                        datatype,op,comm,ierror) &
    BIND(C, name="ompi_reduce_scatter_block_f")
@@ -1164,8 +1460,8 @@ subroutine ompi_ireduce_scatter_block_f(sendbuf,recvbuf,recvcount, &
                                        datatype,op,comm,request,ierror) &
    BIND(C, name="ompi_ireduce_scatter_block_f")
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
    INTEGER, INTENT(IN) :: recvcount
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(IN) :: op
@@ -1174,6 +1470,21 @@ subroutine ompi_ireduce_scatter_block_f(sendbuf,recvbuf,recvcount, &
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_ireduce_scatter_block_f
 
+subroutine ompi_reduce_scatter_block_init_f(sendbuf,recvbuf,recvcount, &
+                                       datatype,op,comm,info,request,ierror) &
+   BIND(C, name="ompi_reduce_scatter_block_init_f")
+   implicit none
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
+   INTEGER, INTENT(IN) :: recvcount
+   INTEGER, INTENT(IN) :: datatype
+   INTEGER, INTENT(IN) :: op
+   INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(IN) :: info
+   INTEGER, INTENT(OUT) :: request
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_reduce_scatter_block_init_f
+
 subroutine ompi_scan_f(sendbuf,recvbuf,count,datatype,op,comm,ierror) &
    BIND(C, name="ompi_scan_f")
    implicit none
@@ -1189,8 +1500,8 @@ end subroutine ompi_scan_f
 subroutine ompi_iscan_f(sendbuf,recvbuf,count,datatype,op,comm,request,ierror) &
    BIND(C, name="ompi_iscan_f")
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(IN) :: op
@@ -1199,6 +1510,20 @@ subroutine ompi_iscan_f(sendbuf,recvbuf,count,datatype,op,comm,request,ierror) &
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_iscan_f
 
+subroutine ompi_scan_init_f(sendbuf,recvbuf,count,datatype,op,comm,info,request,ierror) &
+   BIND(C, name="ompi_scan_init_f")
+   implicit none
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
+   INTEGER, INTENT(IN) :: count
+   INTEGER, INTENT(IN) :: datatype
+   INTEGER, INTENT(IN) :: op
+   INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(IN) :: info
+   INTEGER, INTENT(OUT) :: request
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_scan_init_f
+
 subroutine ompi_scatter_f(sendbuf,sendcount,sendtype,recvbuf, &
                           recvcount,recvtype,root,comm,ierror) &
    BIND(C, name="ompi_scatter_f")
@@ -1216,8 +1541,8 @@ subroutine ompi_iscatter_f(sendbuf,sendcount,sendtype,recvbuf, &
                           recvcount,recvtype,root,comm,request,ierror) &
    BIND(C, name="ompi_iscatter_f")
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
    INTEGER, INTENT(IN) :: sendcount, recvcount, root
    INTEGER, INTENT(IN) :: sendtype
    INTEGER, INTENT(IN) :: recvtype
@@ -1226,6 +1551,21 @@ subroutine ompi_iscatter_f(sendbuf,sendcount,sendtype,recvbuf, &
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_iscatter_f
 
+subroutine ompi_scatter_init_f(sendbuf,sendcount,sendtype,recvbuf, &
+                          recvcount,recvtype,root,comm,info,request,ierror) &
+   BIND(C, name="ompi_scatter_init_f")
+   implicit none
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
+   INTEGER, INTENT(IN) :: sendcount, recvcount, root
+   INTEGER, INTENT(IN) :: sendtype
+   INTEGER, INTENT(IN) :: recvtype
+   INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(IN) :: info
+   INTEGER, INTENT(OUT) :: request
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_scatter_init_f
+
 subroutine ompi_scatterv_f(sendbuf,sendcounts,displs,sendtype, &
                            recvbuf,recvcount,recvtype,root,comm,ierror) &
    BIND(C, name="ompi_scatterv_f")
@@ -1244,10 +1584,11 @@ subroutine ompi_iscatterv_f(sendbuf,sendcounts,displs,sendtype, &
                            recvbuf,recvcount,recvtype,root,comm,request,ierror) &
    BIND(C, name="ompi_iscatterv_f")
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
    INTEGER, INTENT(IN) :: recvcount, root
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: sendcounts(*), displs(*)
+   INTEGER, INTENT(IN) OMPI_ASYNCHRONOUS :: sendcounts(*), displs(*)
+
    INTEGER, INTENT(IN) :: sendtype
    INTEGER, INTENT(IN) :: recvtype
    INTEGER, INTENT(IN) :: comm
@@ -1255,6 +1596,22 @@ subroutine ompi_iscatterv_f(sendbuf,sendcounts,displs,sendtype, &
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_iscatterv_f
 
+subroutine ompi_scatterv_init_f(sendbuf,sendcounts,displs,sendtype, &
+                           recvbuf,recvcount,recvtype,root,comm,info,request,ierror) &
+   BIND(C, name="ompi_scatterv_init_f")
+   implicit none
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
+   INTEGER, INTENT(IN) :: recvcount, root
+   INTEGER, INTENT(IN) OMPI_ASYNCHRONOUS :: sendcounts(*), displs(*)
+   INTEGER, INTENT(IN) :: sendtype
+   INTEGER, INTENT(IN) :: recvtype
+   INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(IN) :: info
+   INTEGER, INTENT(OUT) :: request
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_scatterv_init_f
+
 subroutine ompi_comm_compare_f(comm1,comm2,result,ierror) &
    BIND(C, name="ompi_comm_compare_f")
    implicit none
@@ -1273,6 +1630,19 @@ subroutine ompi_comm_create_f(comm,group,newcomm,ierror) &
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_comm_create_f
 
+subroutine ompi_comm_create_from_group_f(group, stringtag, info, errhandler, newcomm, ierror, name_len) &
+   BIND(C, name="ompi_comm_create_from_group_f")
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
+   implicit none
+   integer, intent(in) :: group
+   CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(IN) :: stringtag
+   integer, intent(in) :: info
+   integer, intent(in) :: errhandler
+   integer, intent(out) :: newcomm
+   integer, intent(out) :: ierror
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: name_len
+end subroutine ompi_comm_create_from_group_f
+
 subroutine ompi_comm_create_group_f(comm, group, tag, newcomm, ierror) &
    BIND(C, name="ompi_comm_create_group_f")
    implicit none
@@ -1321,6 +1691,16 @@ subroutine ompi_comm_dup_with_info_f(comm, info, newcomm, ierror) &
   integer, intent(out) :: ierror
 end subroutine ompi_comm_dup_with_info_f
 
+subroutine ompi_comm_idup_with_info_f(comm, info, newcomm, request, ierror) &
+   BIND(C, name="ompi_comm_idup_with_info_f")
+   implicit none
+  integer, intent(in) :: comm
+  integer, intent(in) :: info
+  integer, intent(out) :: newcomm
+  integer, intent(out) :: request
+  integer, intent(out) :: ierror
+end subroutine ompi_comm_idup_with_info_f
+
 subroutine ompi_comm_free_f(comm,ierror) &
    BIND(C, name="ompi_comm_free_f")
    implicit none
@@ -1345,15 +1725,28 @@ end subroutine ompi_comm_get_info_f
 
 subroutine ompi_comm_get_name_f(comm,comm_name,resultlen,ierror,comm_name_len) &
    BIND(C, name="ompi_comm_get_name_f")
-   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
    implicit none
    INTEGER, INTENT(IN) :: comm
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(OUT) :: comm_name
    INTEGER, INTENT(OUT) :: resultlen
    INTEGER, INTENT(OUT) :: ierror
-   INTEGER, VALUE, INTENT(IN) :: comm_name_len
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: comm_name_len
 end subroutine ompi_comm_get_name_f
 
+subroutine ompi_comm_from_group_f(group, stringtag, info, errhandler, newcomm, ierror, name_len) &
+   BIND(C, name="ompi_comm_from_group_f")
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   implicit none
+   INTEGER, INTENT(IN) :: group
+   CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(IN) :: stringtag
+   INTEGER, INTENT(IN) :: info
+   INTEGER, INTENT(IN) :: errhandler
+   INTEGER, INTENT(OUT) :: newcomm
+   INTEGER, INTENT(OUT) :: ierror
+   INTEGER, VALUE, INTENT(IN) :: name_len
+end subroutine ompi_comm_from_group_f
+
 subroutine ompi_comm_group_f(comm,group,ierror) &
    BIND(C, name="ompi_comm_group_f")
    implicit none
@@ -1366,7 +1759,7 @@ subroutine ompi_comm_idup_f(comm, newcomm, request, ierror) &
    BIND(C, name="ompi_comm_idup_f")
    implicit none
   integer, intent(in) :: comm
-  integer, intent(out) :: newcomm
+  integer, intent(out) OMPI_ASYNCHRONOUS :: newcomm
   integer, intent(out) :: request
   integer, intent(out) :: ierror
 end subroutine ompi_comm_idup_f
@@ -1415,12 +1808,12 @@ end subroutine ompi_comm_set_info_f
 
 subroutine ompi_comm_set_name_f(comm,comm_name,ierror,comm_name_len) &
    BIND(C, name="ompi_comm_set_name_f")
-   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
    implicit none
    INTEGER, INTENT(IN) :: comm
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(IN) :: comm_name
    INTEGER, INTENT(OUT) :: ierror
-   INTEGER, VALUE, INTENT(IN) :: comm_name_len
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: comm_name_len
 end subroutine ompi_comm_set_name_f
 
 subroutine ompi_comm_size_f(comm,size,ierror) &
@@ -1463,7 +1856,7 @@ subroutine ompi_group_excl_f(group,n,ranks,newgroup,ierror) &
    implicit none
    INTEGER, INTENT(IN) :: group
    INTEGER, INTENT(IN) :: n
-   INTEGER, INTENT(IN) :: ranks(*)
+   INTEGER, INTENT(IN) :: ranks(n)
    INTEGER, INTENT(OUT) :: newgroup
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_group_excl_f
@@ -1475,11 +1868,22 @@ subroutine ompi_group_free_f(group,ierror) &
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_group_free_f
 
+subroutine ompi_group_from_session_pset_f(session, pset_name, newgroup, ierror, name_len) &
+   BIND(C, name="ompi_group_from_session_pset_f")
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
+   implicit none
+   INTEGER, INTENT(IN) :: session
+   CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(IN) :: pset_name
+   INTEGER, INTENT(OUT) :: newgroup
+   integer, intent(out) :: ierror
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: name_len
+end subroutine ompi_group_from_session_pset_f
+
 subroutine ompi_group_incl_f(group,n,ranks,newgroup,ierror) &
    BIND(C, name="ompi_group_incl_f")
    implicit none
    INTEGER, INTENT(IN) :: n
-   INTEGER, INTENT(IN) :: ranks(*)
+   INTEGER, INTENT(IN) :: ranks(n)
    INTEGER, INTENT(IN) :: group
    INTEGER, INTENT(OUT) :: newgroup
    INTEGER, INTENT(OUT) :: ierror
@@ -1499,7 +1903,7 @@ subroutine ompi_group_range_excl_f(group,n,ranges,newgroup,ierror) &
    implicit none
    INTEGER, INTENT(IN) :: group
    INTEGER, INTENT(IN) :: n
-   INTEGER, INTENT(IN) :: ranges(*)
+   INTEGER, INTENT(IN) :: ranges(3, n)
    INTEGER, INTENT(OUT) :: newgroup
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_group_range_excl_f
@@ -1509,7 +1913,7 @@ subroutine ompi_group_range_incl_f(group,n,ranges,newgroup,ierror) &
    implicit none
    INTEGER, INTENT(IN) :: group
    INTEGER, INTENT(IN) :: n
-   INTEGER, INTENT(IN) :: ranges(*)
+   INTEGER, INTENT(IN) :: ranges(3, n)
    INTEGER, INTENT(OUT) :: newgroup
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_group_range_incl_f
@@ -1535,8 +1939,8 @@ subroutine ompi_group_translate_ranks_f(group1,n,ranks1,group2,ranks2,ierror) &
    implicit none
    INTEGER, INTENT(IN) :: group1, group2
    INTEGER, INTENT(IN) :: n
-   INTEGER, INTENT(IN) :: ranks1(*)
-   INTEGER, INTENT(OUT) :: ranks2(*)
+   INTEGER, INTENT(IN) :: ranks1(n)
+   INTEGER, INTENT(OUT) :: ranks2(n)
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_group_translate_ranks_f
 
@@ -1558,6 +1962,21 @@ subroutine ompi_intercomm_create_f(local_comm,local_leader,peer_comm, &
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_intercomm_create_f
 
+subroutine ompi_intercomm_create_from_groups_f(local_group, local_leader, remote_group,  &
+                                               remote_leader, stringtag, info, errhandler, &
+                                               newintercomm, ierror, name_len) &
+   BIND(C, name="ompi_intercomm_create_from_groups_f")
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
+   implicit none
+   INTEGER, INTENT(IN) :: local_group, remote_group
+   INTEGER, INTENT(IN) :: local_leader, remote_leader
+   CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(IN) :: stringtag
+   INTEGER, INTENT(IN) :: info, errhandler
+   INTEGER, INTENT(OUT) :: newintercomm
+   INTEGER, INTENT(OUT) :: ierror
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: name_len
+end subroutine ompi_intercomm_create_from_groups_f
+
 subroutine ompi_type_create_keyval_f(type_copy_attr_fn,type_delete_attr_fn, &
                                      type_keyval,extra_state,ierror) &
    BIND(C, name="ompi_type_create_keyval_f")
@@ -1588,13 +2007,13 @@ end subroutine ompi_type_free_keyval_f
 
 subroutine ompi_type_get_name_f(datatype,type_name,resultlen,ierror,type_name_len) &
    BIND(C, name="ompi_type_get_name_f")
-   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
    implicit none
    INTEGER, INTENT(IN) :: datatype
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(OUT) :: type_name
    INTEGER, INTENT(OUT) :: resultlen
    INTEGER, INTENT(OUT) :: ierror
-   INTEGER, VALUE, INTENT(IN) :: type_name_len
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: type_name_len
 end subroutine ompi_type_get_name_f
 
 subroutine ompi_type_set_attr_f(datatype,type_keyval,attribute_val,ierror) &
@@ -1609,12 +2028,12 @@ end subroutine ompi_type_set_attr_f
 
 subroutine ompi_type_set_name_f(datatype,type_name,ierror,type_name_len) &
    BIND(C, name="ompi_type_set_name_f")
-   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
    implicit none
    INTEGER, INTENT(IN) :: datatype
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(IN) :: type_name
    INTEGER, INTENT(OUT) :: ierror
-   INTEGER, VALUE, INTENT(IN) :: type_name_len
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: type_name_len
 end subroutine ompi_type_set_name_f
 
 subroutine ompi_win_allocate_f(size, disp_unit, info, comm, &
@@ -1673,13 +2092,13 @@ end subroutine ompi_win_free_keyval_f
 
 subroutine ompi_win_get_name_f(win,win_name,resultlen,ierror,win_name_len) &
    BIND(C, name="ompi_win_get_name_f")
-   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
    implicit none
    INTEGER, INTENT(IN) :: win
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(OUT) :: win_name
    INTEGER, INTENT(OUT) :: resultlen
    INTEGER, INTENT(OUT) :: ierror
-   INTEGER, VALUE, INTENT(IN) :: win_name_len
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: win_name_len
 end subroutine ompi_win_get_name_f
 
 subroutine ompi_win_set_attr_f(win,win_keyval,attribute_val,ierror) &
@@ -1694,12 +2113,12 @@ end subroutine ompi_win_set_attr_f
 
 subroutine ompi_win_set_name_f(win,win_name,ierror,win_name_len) &
    BIND(C, name="ompi_win_set_name_f")
-   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
    implicit none
    INTEGER, INTENT(IN) :: win
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(IN) :: win_name
    INTEGER, INTENT(OUT) :: ierror
-   INTEGER, VALUE, INTENT(IN) :: win_name_len
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: win_name_len
 end subroutine ompi_win_set_name_f
 
 subroutine ompi_cartdim_get_f(comm,ndims,ierror) &
@@ -1741,7 +2160,7 @@ subroutine ompi_dims_create_f(nnodes,ndims,dims,ierror) &
    BIND(C, name="ompi_dims_create_f")
    implicit none
    INTEGER, INTENT(IN) :: nnodes, ndims
-   INTEGER, INTENT(INOUT) :: dims(*)
+   INTEGER, INTENT(INOUT) :: dims(ndims)
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_dims_create_f
 
@@ -1769,7 +2188,7 @@ subroutine ompi_graph_get_f(comm,maxindex,maxedges,index,edges,ierror) &
    implicit none
    INTEGER, INTENT(IN) :: comm
    INTEGER, INTENT(IN) :: maxindex, maxedges
-   INTEGER, INTENT(OUT) :: index(*), edges(*)
+   INTEGER, INTENT(OUT) :: index(maxindex), edges(maxedges)
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_graph_get_f
 
@@ -1778,7 +2197,7 @@ subroutine ompi_graph_map_f(comm,nnodes,index,edges,newrank,ierror) &
    implicit none
    INTEGER, INTENT(IN) :: comm
    INTEGER, INTENT(IN) :: nnodes
-   INTEGER, INTENT(IN) :: index(*), edges(*)
+   INTEGER, INTENT(IN) :: index(nnodes), edges(*)
    INTEGER, INTENT(OUT) :: newrank
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_graph_map_f
@@ -1788,7 +2207,7 @@ subroutine ompi_graph_neighbors_f(comm,rank,maxneighbors,neighbors,ierror) &
    implicit none
    INTEGER, INTENT(IN) :: comm
    INTEGER, INTENT(IN) :: rank, maxneighbors
-   INTEGER, INTENT(OUT) :: neighbors(*)
+   INTEGER, INTENT(OUT) :: neighbors(maxneighbors)
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_graph_neighbors_f
 
@@ -1865,12 +2284,12 @@ end subroutine ompi_add_error_code_f
 
 subroutine ompi_add_error_string_f(errorcode,string,ierror,str_len) &
    BIND(C, name="ompi_add_error_string_f")
-   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
    implicit none
    INTEGER, INTENT(IN) :: errorcode
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(IN) :: string
    INTEGER, INTENT(OUT) :: ierror
-   INTEGER, VALUE, INTENT(IN) :: str_len
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: str_len
 end subroutine ompi_add_error_string_f
 
 subroutine ompi_alloc_mem_f(size,info,baseptr,ierror) &
@@ -1934,13 +2353,13 @@ end subroutine ompi_error_class_f
 
 subroutine ompi_error_string_f(errorcode,string,resultlen,ierror,str_len) &
    BIND(C, name="ompi_error_string_f")
-   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
    implicit none
    INTEGER, INTENT(IN) :: errorcode
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(OUT) :: string
    INTEGER, INTENT(OUT) :: resultlen
    INTEGER, INTENT(OUT) :: ierror
-   INTEGER, VALUE, INTENT(IN) :: str_len
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: str_len
 end subroutine ompi_error_string_f
 
 subroutine ompi_file_call_errhandler_f(fh,errorcode,ierror) &
@@ -1991,12 +2410,12 @@ end subroutine ompi_free_mem_f
 
 subroutine ompi_get_processor_name_f(name,resultlen,ierror,name_len) &
    BIND(C, name="ompi_get_processor_name_f")
-   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
    implicit none
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(OUT) :: name
    INTEGER, INTENT(OUT) :: resultlen
    INTEGER, INTENT(OUT) :: ierror
-   INTEGER, VALUE, INTENT(IN) :: name_len
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: name_len
 end subroutine ompi_get_processor_name_f
 
 subroutine ompi_get_version_f(version,subversion,ierror) &
@@ -2052,14 +2471,21 @@ subroutine ompi_info_create_f(info,ierror) &
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_info_create_f
 
+subroutine ompi_info_create_env_f(info,ierror) &
+   BIND(C, name="ompi_info_create_env_f")
+   implicit none
+   INTEGER, INTENT(OUT) :: info
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_info_create_env_f
+
 subroutine ompi_info_delete_f(info,key,ierror,key_len) &
    BIND(C, name="ompi_info_delete_f")
-   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
    implicit none
    INTEGER, INTENT(IN) :: info
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(IN) :: key
    INTEGER, INTENT(OUT) :: ierror
-   INTEGER, VALUE, INTENT(IN) :: key_len
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: key_len
 end subroutine ompi_info_delete_f
 
 subroutine ompi_info_dup_f(info,newinfo,ierror) &
@@ -2087,37 +2513,37 @@ end subroutine ompi_info_get_nkeys_f
 
 subroutine ompi_info_get_nthkey_f(info,n,key,ierror,key_len) &
    BIND(C, name="ompi_info_get_nthkey_f")
-   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
    implicit none
    INTEGER, INTENT(IN) :: info
    INTEGER, INTENT(IN) :: n
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(OUT) :: key
    INTEGER, INTENT(OUT) :: ierror
-   INTEGER, VALUE, INTENT(IN) :: key_len
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: key_len
 end subroutine ompi_info_get_nthkey_f
 
 subroutine ompi_info_set_f(info,key,value,ierror,key_len,value_len) &
    BIND(C, name="ompi_info_set_f")
-   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
    implicit none
    INTEGER, INTENT(IN) :: info
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(IN) :: key, value
    INTEGER, INTENT(OUT) :: ierror
-   INTEGER, VALUE, INTENT(IN) :: key_len, value_len
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: key_len, value_len
 end subroutine ompi_info_set_f
 
 subroutine ompi_close_port_f(port_name,ierror,port_name_len) &
    BIND(C, name="ompi_close_port_f")
-   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
    implicit none
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(IN) :: port_name
    INTEGER, INTENT(OUT) :: ierror
-   INTEGER, VALUE, INTENT(IN) :: port_name_len
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: port_name_len
 end subroutine ompi_close_port_f
 
 subroutine ompi_comm_accept_f(port_name,info,root,comm,newcomm,ierror,port_name_len) &
    BIND(C, name="ompi_comm_accept_f")
-   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
    implicit none
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(IN) :: port_name
    INTEGER, INTENT(IN) :: info
@@ -2125,12 +2551,12 @@ subroutine ompi_comm_accept_f(port_name,info,root,comm,newcomm,ierror,port_name_
    INTEGER, INTENT(IN) :: comm
    INTEGER, INTENT(OUT) :: newcomm
    INTEGER, INTENT(OUT) :: ierror
-   INTEGER, VALUE, INTENT(IN) :: port_name_len
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: port_name_len
 end subroutine ompi_comm_accept_f
 
 subroutine ompi_comm_connect_f(port_name,info,root,comm,newcomm,ierror,port_name_len) &
    BIND(C, name="ompi_comm_connect_f")
-   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
    implicit none
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(IN) :: port_name
    INTEGER, INTENT(IN) :: info
@@ -2138,7 +2564,7 @@ subroutine ompi_comm_connect_f(port_name,info,root,comm,newcomm,ierror,port_name
    INTEGER, INTENT(IN) :: comm
    INTEGER, INTENT(OUT) :: newcomm
    INTEGER, INTENT(OUT) :: ierror
-   INTEGER, VALUE, INTENT(IN) :: port_name_len
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: port_name_len
 end subroutine ompi_comm_connect_f
 
 subroutine ompi_comm_disconnect_f(comm,ierror) &
@@ -2166,7 +2592,7 @@ end subroutine ompi_comm_join_f
 subroutine ompi_comm_spawn_f(command,argv,maxprocs,info,root,comm, &
                              intercomm, array_of_errcodes,ierror,cmd_len,argv_len) &
    BIND(C, name="ompi_comm_spawn_f")
-   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
    implicit none
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(IN) :: command, argv
    INTEGER, INTENT(IN) :: maxprocs, root
@@ -2175,7 +2601,7 @@ subroutine ompi_comm_spawn_f(command,argv,maxprocs,info,root,comm, &
    INTEGER, INTENT(OUT) :: intercomm
    INTEGER, INTENT(OUT) :: array_of_errcodes(*)
    INTEGER, INTENT(OUT) :: ierror
-   INTEGER, VALUE, INTENT(IN) :: cmd_len, argv_len
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: cmd_len, argv_len
 end subroutine ompi_comm_spawn_f
 
 
@@ -2185,7 +2611,7 @@ subroutine ompi_comm_spawn_multiple_f(count,array_of_commands, &
                                       comm,intercomm,array_of_errcodes,ierror, &
                                       cmd_len, argv_len) &
    BIND(C, name="ompi_comm_spawn_multiple_f")
-   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
    implicit none
    INTEGER, INTENT(IN) :: count, root
    INTEGER, INTENT(IN) :: array_of_maxprocs(count)
@@ -2195,51 +2621,51 @@ subroutine ompi_comm_spawn_multiple_f(count,array_of_commands, &
    INTEGER, INTENT(OUT) :: intercomm
    INTEGER, INTENT(OUT) :: array_of_errcodes(*)
    INTEGER, INTENT(OUT) :: ierror
-   INTEGER, VALUE, INTENT(IN) :: cmd_len, argv_len
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: cmd_len, argv_len
 end subroutine ompi_comm_spawn_multiple_f
 
 subroutine ompi_lookup_name_f(service_name,info,port_name,ierror, &
                               service_name_len,port_name_len) &
    BIND(C, name="ompi_lookup_name_f")
-   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
    implicit none
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(IN) :: service_name
    INTEGER, INTENT(IN) :: info
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(OUT) :: port_name
    INTEGER, INTENT(OUT) :: ierror
-   INTEGER, VALUE, INTENT(IN) :: service_name_len, port_name_len
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: service_name_len, port_name_len
 end subroutine ompi_lookup_name_f
 
 subroutine ompi_open_port_f(info,port_name,ierror,port_name_len) &
    BIND(C, name="ompi_open_port_f")
-   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
    implicit none
    INTEGER, INTENT(IN) :: info
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(OUT) :: port_name
    INTEGER, INTENT(OUT) :: ierror
-   INTEGER, VALUE, INTENT(IN) :: port_name_len
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: port_name_len
 end subroutine ompi_open_port_f
 
 subroutine ompi_publish_name_f(service_name,info,port_name,ierror, &
                                service_name_len,port_name_len) &
    BIND(C, name="ompi_publish_name_f")
-   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
    implicit none
    INTEGER, INTENT(IN) :: info
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(IN) :: service_name, port_name
    INTEGER, INTENT(OUT) :: ierror
-   INTEGER, VALUE, INTENT(IN) :: service_name_len, port_name_len
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: service_name_len, port_name_len
 end subroutine ompi_publish_name_f
 
 subroutine ompi_unpublish_name_f(service_name,info,port_name, &
                                  ierror,service_name_len,port_name_len) &
    BIND(C, name="ompi_unpublish_name_f")
-   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
    implicit none
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(IN) :: service_name, port_name
    INTEGER, INTENT(IN) :: info
    INTEGER, INTENT(OUT) :: ierror
-   INTEGER, VALUE, INTENT(IN) :: service_name_len, port_name_len
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: service_name_len, port_name_len
 end subroutine ompi_unpublish_name_f
 
 subroutine ompi_accumulate_f(origin_addr,origin_count,origin_datatype, &
@@ -2248,7 +2674,7 @@ subroutine ompi_accumulate_f(origin_addr,origin_count,origin_datatype, &
    BIND(C, name="ompi_accumulate_f")
    use :: mpi_f08_types, only : MPI_ADDRESS_KIND
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: origin_addr
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: origin_addr
    INTEGER, INTENT(IN) :: origin_count, target_rank, target_count
    INTEGER, INTENT(IN) :: origin_datatype
    INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
@@ -2265,7 +2691,7 @@ subroutine ompi_raccumulate_f(origin_addr,origin_count,origin_datatype, &
    BIND(C, name="ompi_raccumulate_f")
    use :: mpi_f08_types, only : MPI_ADDRESS_KIND
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: origin_addr
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: origin_addr
    INTEGER, INTENT(IN) :: origin_count, target_rank, target_count
    INTEGER, INTENT(IN) :: origin_datatype
    INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
@@ -2281,7 +2707,7 @@ subroutine ompi_get_f(origin_addr,origin_count,origin_datatype,target_rank, &
    BIND(C, name="ompi_get_f")
    use :: mpi_f08_types, only : MPI_ADDRESS_KIND
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: origin_addr
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: origin_addr
    INTEGER, INTENT(IN) :: origin_count, target_rank, target_count
    INTEGER, INTENT(IN) :: origin_datatype
    INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
@@ -2296,7 +2722,7 @@ subroutine ompi_rget_f(origin_addr,origin_count,origin_datatype,target_rank, &
    BIND(C, name="ompi_rget_f")
    use :: mpi_f08_types, only : MPI_ADDRESS_KIND
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: origin_addr
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: origin_addr
    INTEGER, INTENT(IN) :: origin_count, target_rank, target_count
    INTEGER, INTENT(IN) :: origin_datatype
    INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
@@ -2314,10 +2740,10 @@ subroutine ompi_get_accumulate_f(origin_addr,origin_count,origin_datatype, &
    BIND(C, name="ompi_get_accumulate_f")
    use :: mpi_f08_types, only : MPI_ADDRESS_KIND
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: origin_addr
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: origin_addr
    INTEGER, INTENT(IN) :: origin_count, result_count, target_rank, target_count
    INTEGER, INTENT(IN) :: origin_datatype
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: result_addr
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: result_addr
    INTEGER, INTENT(IN) :: result_datatype
    INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
    INTEGER, INTENT(IN) :: target_datatype
@@ -2334,10 +2760,10 @@ subroutine ompi_rget_accumulate_f(origin_addr,origin_count,origin_datatype, &
    BIND(C, name="ompi_rget_accumulate_f")
    use :: mpi_f08_types, only : MPI_ADDRESS_KIND
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: origin_addr
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: origin_addr
    INTEGER, INTENT(IN) :: origin_count, result_count, target_rank, target_count
    INTEGER, INTENT(IN) :: origin_datatype
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: result_addr
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: result_addr
    INTEGER, INTENT(IN) :: result_datatype
    INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
    INTEGER, INTENT(IN) :: target_datatype
@@ -2352,7 +2778,7 @@ subroutine ompi_put_f(origin_addr,origin_count,origin_datatype,target_rank, &
    BIND(C, name="ompi_put_f")
    use :: mpi_f08_types, only : MPI_ADDRESS_KIND
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: origin_addr
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: origin_addr
    INTEGER, INTENT(IN) :: origin_count, target_rank, target_count
    INTEGER, INTENT(IN) :: origin_datatype
    INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
@@ -2367,7 +2793,7 @@ subroutine ompi_rput_f(origin_addr,origin_count,origin_datatype,target_rank, &
    BIND(C, name="ompi_rput_f")
    use :: mpi_f08_types, only : MPI_ADDRESS_KIND
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: origin_addr
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: origin_addr
    INTEGER, INTENT(IN) :: origin_count, target_rank, target_count
    INTEGER, INTENT(IN) :: origin_datatype
    INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
@@ -2390,8 +2816,8 @@ subroutine ompi_compare_and_swap_f(origin_addr,compare_addr,result_addr, &
    BIND(C, name="ompi_compare_and_swap_f")
    use :: mpi_f08_types, only : MPI_ADDRESS_KIND
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: origin_addr, compare_addr
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: result_addr
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: origin_addr, compare_addr
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: result_addr
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(IN) :: target_rank
    INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
@@ -2404,8 +2830,8 @@ subroutine ompi_fetch_and_op_f(origin_addr,result_addr,datatype,target_rank, &
    BIND(C, name="ompi_fetch_and_op_f")
    use :: mpi_f08_types, only : MPI_ADDRESS_KIND
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: origin_addr
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: result_addr
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: origin_addr
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: result_addr
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(IN) :: target_rank
    INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
@@ -2418,7 +2844,7 @@ subroutine ompi_win_create_f(base,size,disp_unit,info,comm,win,ierror) &
    BIND(C, name="ompi_win_create_f")
    use :: mpi_f08_types, only : MPI_ADDRESS_KIND
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: base
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: base
    INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: size
    INTEGER, INTENT(IN) :: disp_unit
    INTEGER, INTENT(IN) :: info
@@ -2440,7 +2866,7 @@ subroutine ompi_win_attach_f(win,base,size,ierror) &
    BIND(C, name="ompi_win_attach_f")
    use :: mpi_f08_types, only : MPI_ADDRESS_KIND
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: base
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: base
    INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: size
    INTEGER, INTENT(IN) :: win
    INTEGER, INTENT(OUT) :: ierror
@@ -2450,7 +2876,7 @@ subroutine ompi_win_detach_f(win,base,ierror) &
    BIND(C, name="ompi_win_detach_f")
    use :: mpi_f08_types, only : MPI_ADDRESS_KIND
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: base
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: base
    INTEGER, INTENT(IN) :: win
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_win_detach_f
@@ -2508,11 +2934,11 @@ subroutine ompi_win_get_group_f(win,group,ierror) &
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_win_get_group_f
 
-subroutine ompi_win_get_info_f(comm,info,ierror) &
+subroutine ompi_win_get_info_f(win,info_used,ierror) &
    BIND(C, name="ompi_win_get_info_f")
    implicit none
-   INTEGER, INTENT(IN) :: comm
-   INTEGER, INTENT(OUT) :: info
+   INTEGER, INTENT(IN) :: win
+   INTEGER, INTENT(OUT) :: info_used
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_win_get_info_f
 
@@ -2541,10 +2967,10 @@ subroutine ompi_win_post_f(group,assert,win,ierror) &
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_win_post_f
 
-subroutine ompi_win_set_info_f(comm,info,ierror) &
+subroutine ompi_win_set_info_f(win,info,ierror) &
    BIND(C, name="ompi_win_set_info_f")
    implicit none
-   INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(IN) :: win
    INTEGER, INTENT(IN) :: info
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_win_set_info_f
@@ -2636,7 +3062,7 @@ subroutine ompi_query_thread_f(provided,ierror) &
 end subroutine ompi_query_thread_f
 
 subroutine ompi_status_f082f_f(f08_status,f_status,ierror) &
-   BIND(C, name="ompi_status_f2f08_f")
+   BIND(C, name="ompi_status_f082f_f")
    use :: mpi_f08_types, only : MPI_Status, MPI_STATUS_SIZE
    implicit none
    TYPE(MPI_Status), INTENT(IN) :: f08_status
@@ -2645,7 +3071,7 @@ subroutine ompi_status_f082f_f(f08_status,f_status,ierror) &
 end subroutine ompi_status_f082f_f
 
 subroutine ompi_status_f2f08_f(f_status,f08_status,ierror) &
-   BIND(C, name="ompi_status_f082f_f")
+   BIND(C, name="ompi_status_f2f08_f")
    use :: mpi_f08_types, only : MPI_Status, MPI_STATUS_SIZE
    implicit none
    INTEGER, INTENT(IN) :: f_status(MPI_STATUS_SIZE)
@@ -2682,12 +3108,12 @@ end subroutine ompi_file_close_f
 
 subroutine ompi_file_delete_f(filename,info,ierror,filename_len) &
    BIND(C, name="ompi_file_delete_f")
-   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
    implicit none
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(IN) :: filename
    INTEGER, INTENT(IN) :: info
    INTEGER, INTENT(OUT) :: ierror
-   INTEGER, VALUE, INTENT(IN) :: filename_len
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: filename_len
 end subroutine ompi_file_delete_f
 
 subroutine ompi_file_get_amode_f(fh,amode,ierror) &
@@ -2763,7 +3189,7 @@ end subroutine ompi_file_get_type_extent_f
 
 subroutine ompi_file_get_view_f(fh,disp,etype,filetype,datarep,ierror,datarep_len) &
    BIND(C, name="ompi_file_get_view_f")
-   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
    use :: mpi_f08_types, only : MPI_OFFSET_KIND
    implicit none
    INTEGER, INTENT(IN) :: fh
@@ -2772,14 +3198,14 @@ subroutine ompi_file_get_view_f(fh,disp,etype,filetype,datarep,ierror,datarep_le
    INTEGER, INTENT(OUT) :: filetype
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(OUT) :: datarep
    INTEGER, INTENT(OUT) :: ierror
-   INTEGER, VALUE, INTENT(IN) :: datarep_len
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: datarep_len
 end subroutine ompi_file_get_view_f
 
 subroutine ompi_file_iread_f(fh,buf,count,datatype,request,ierror) &
    BIND(C, name="ompi_file_iread_f")
    implicit none
    INTEGER, INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: buf
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(OUT) :: request
@@ -2792,7 +3218,7 @@ subroutine ompi_file_iread_at_f(fh,offset,buf,count,datatype,request,ierror) &
    implicit none
    INTEGER, INTENT(IN) :: fh
    INTEGER(MPI_OFFSET_KIND), INTENT(IN) :: offset
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: buf
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(OUT) :: request
@@ -2803,7 +3229,7 @@ subroutine ompi_file_iread_all_f(fh,buf,count,datatype,request,ierror) &
    BIND(C, name="ompi_file_iread_all_f")
    implicit none
    INTEGER, INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: buf
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(OUT) :: request
@@ -2816,7 +3242,7 @@ subroutine ompi_file_iread_at_all_f(fh,offset,buf,count,datatype,request,ierror)
    implicit none
    INTEGER, INTENT(IN) :: fh
    INTEGER(MPI_OFFSET_KIND), INTENT(IN) :: offset
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: buf
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(OUT) :: request
@@ -2827,7 +3253,7 @@ subroutine ompi_file_iread_shared_f(fh,buf,count,datatype,request,ierror) &
    BIND(C, name="ompi_file_iread_shared_f")
    implicit none
    INTEGER, INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: buf
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(OUT) :: request
@@ -2838,7 +3264,7 @@ subroutine ompi_file_iwrite_f(fh,buf,count,datatype,request,ierror) &
    BIND(C, name="ompi_file_iwrite_f")
    implicit none
    INTEGER, INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: buf
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(OUT) :: request
@@ -2851,7 +3277,7 @@ subroutine ompi_file_iwrite_at_f(fh,offset,buf,count,datatype,request,ierror) &
    implicit none
    INTEGER, INTENT(IN) :: fh
    INTEGER(MPI_OFFSET_KIND), INTENT(IN) :: offset
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: buf
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(OUT) :: request
@@ -2862,7 +3288,7 @@ subroutine ompi_file_iwrite_all_f(fh,buf,count,datatype,request,ierror) &
    BIND(C, name="ompi_file_iwrite_all_f")
    implicit none
    INTEGER, INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: buf
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(OUT) :: request
@@ -2875,7 +3301,7 @@ subroutine ompi_file_iwrite_at_all_f(fh,offset,buf,count,datatype,request,ierror
    implicit none
    INTEGER, INTENT(IN) :: fh
    INTEGER(MPI_OFFSET_KIND), INTENT(IN) :: offset
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: buf
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(OUT) :: request
@@ -2885,7 +3311,7 @@ end subroutine ompi_file_iwrite_at_all_f
 subroutine ompi_file_iwrite_shared_f(fh,buf,count,datatype,request,ierror) &
    BIND(C, name="ompi_file_iwrite_shared_f")
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: buf
    INTEGER, INTENT(IN) :: fh
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
@@ -2895,7 +3321,7 @@ end subroutine ompi_file_iwrite_shared_f
 
 subroutine ompi_file_open_f(comm,filename,amode,info,fh,ierror,filename_len) &
    BIND(C, name="ompi_file_open_f")
-   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
    implicit none
    INTEGER, INTENT(IN) :: comm
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(IN) :: filename
@@ -2903,7 +3329,7 @@ subroutine ompi_file_open_f(comm,filename,amode,info,fh,ierror,filename_len) &
    INTEGER, INTENT(IN) :: info
    INTEGER, INTENT(OUT) :: fh
    INTEGER, INTENT(OUT) :: ierror
-   INTEGER, VALUE, INTENT(IN) :: filename_len
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: filename_len
 end subroutine ompi_file_open_f
 
 subroutine ompi_file_preallocate_f(fh,size,ierror) &
@@ -2920,10 +3346,10 @@ subroutine ompi_file_read_f(fh,buf,count,datatype,status,ierror) &
    use :: mpi_f08_types, only : MPI_Status
    implicit none
    INTEGER, INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE :: buf
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   TYPE(MPI_Status) :: status
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_file_read_f
 
@@ -2932,10 +3358,10 @@ subroutine ompi_file_read_all_f(fh,buf,count,datatype,status,ierror) &
    use :: mpi_f08_types, only : MPI_Status
    implicit none
    INTEGER, INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE :: buf
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   TYPE(MPI_Status) :: status
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_file_read_all_f
 
@@ -2943,7 +3369,7 @@ subroutine ompi_file_read_all_begin_f(fh,buf,count,datatype,ierror) &
    BIND(C, name="ompi_file_read_all_begin_f")
    implicit none
    INTEGER, INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: buf
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(OUT) :: ierror
@@ -2954,8 +3380,8 @@ subroutine ompi_file_read_all_end_f(fh,buf,status,ierror) &
    use :: mpi_f08_types, only : MPI_Status
    implicit none
    INTEGER, INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: buf
+   TYPE(MPI_Status) :: status
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_file_read_all_end_f
 
@@ -2965,10 +3391,10 @@ subroutine ompi_file_read_at_f(fh,offset,buf,count,datatype,status,ierror) &
    implicit none
    INTEGER, INTENT(IN) :: fh
    INTEGER(MPI_OFFSET_KIND), INTENT(IN) :: offset
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE :: buf
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   TYPE(MPI_Status) :: status
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_file_read_at_f
 
@@ -2978,10 +3404,10 @@ subroutine ompi_file_read_at_all_f(fh,offset,buf,count,datatype,status,ierror) &
    implicit none
    INTEGER, INTENT(IN) :: fh
    INTEGER(MPI_OFFSET_KIND), INTENT(IN) :: offset
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE :: buf
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   TYPE(MPI_Status) :: status
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_file_read_at_all_f
 
@@ -2991,7 +3417,7 @@ subroutine ompi_file_read_at_all_begin_f(fh,offset,buf,count,datatype,ierror) &
    implicit none
    INTEGER, INTENT(IN) :: fh
    INTEGER(MPI_OFFSET_KIND), INTENT(IN) :: offset
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: buf
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(OUT) :: ierror
@@ -3002,8 +3428,8 @@ subroutine ompi_file_read_at_all_end_f(fh,buf,status,ierror) &
    use :: mpi_f08_types, only : MPI_Status
    implicit none
    INTEGER, INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: buf
+   TYPE(MPI_Status) :: status
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_file_read_at_all_end_f
 
@@ -3012,10 +3438,10 @@ subroutine ompi_file_read_ordered_f(fh,buf,count,datatype,status,ierror) &
    use :: mpi_f08_types, only : MPI_Status
    implicit none
    INTEGER, INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE :: buf
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   TYPE(MPI_Status) :: status
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_file_read_ordered_f
 
@@ -3023,7 +3449,7 @@ subroutine ompi_file_read_ordered_begin_f(fh,buf,count,datatype,ierror) &
    BIND(C, name="ompi_file_read_ordered_begin_f")
    implicit none
    INTEGER, INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: buf
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(OUT) :: ierror
@@ -3034,8 +3460,8 @@ subroutine ompi_file_read_ordered_end_f(fh,buf,status,ierror) &
    use :: mpi_f08_types, only : MPI_Status
    implicit none
    INTEGER, INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: buf
+   TYPE(MPI_Status) :: status
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_file_read_ordered_end_f
 
@@ -3044,10 +3470,10 @@ subroutine ompi_file_read_shared_f(fh,buf,count,datatype,status,ierror) &
    use :: mpi_f08_types, only : MPI_Status
    implicit none
    INTEGER, INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE :: buf
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   TYPE(MPI_Status) :: status
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_file_read_shared_f
 
@@ -3090,7 +3516,7 @@ end subroutine ompi_file_set_size_f
 
 subroutine ompi_file_set_view_f(fh,disp,etype,filetype,datarep,info,ierror,datarep_len) &
    BIND(C, name="ompi_file_set_view_f")
-   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
    use :: mpi_f08_types, only : MPI_OFFSET_KIND
    implicit none
    INTEGER, INTENT(IN) :: fh
@@ -3100,7 +3526,7 @@ subroutine ompi_file_set_view_f(fh,disp,etype,filetype,datarep,info,ierror,datar
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(IN) :: datarep
    INTEGER, INTENT(IN) :: info
    INTEGER, INTENT(OUT) :: ierror
-   INTEGER, VALUE, INTENT(IN) :: datarep_len
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: datarep_len
 end subroutine ompi_file_set_view_f
 
 subroutine ompi_file_sync_f(fh,ierror) &
@@ -3118,7 +3544,7 @@ subroutine ompi_file_write_f(fh,buf,count,datatype,status,ierror) &
    OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   TYPE(MPI_Status) :: status
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_file_write_f
 
@@ -3130,7 +3556,7 @@ subroutine ompi_file_write_all_f(fh,buf,count,datatype,status,ierror) &
    OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   TYPE(MPI_Status) :: status
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_file_write_all_f
 
@@ -3138,7 +3564,7 @@ subroutine ompi_file_write_all_begin_f(fh,buf,count,datatype,ierror) &
    BIND(C, name="ompi_file_write_all_begin_f")
    implicit none
    INTEGER, INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: buf
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(OUT) :: ierror
@@ -3149,8 +3575,8 @@ subroutine ompi_file_write_all_end_f(fh,buf,status,ierror) &
    use :: mpi_f08_types, only : MPI_Status
    implicit none
    INTEGER, INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: buf
+   TYPE(MPI_Status) :: status
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_file_write_all_end_f
 
@@ -3163,7 +3589,7 @@ subroutine ompi_file_write_at_f(fh,offset,buf,count,datatype,status,ierror) &
    OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   TYPE(MPI_Status) :: status
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_file_write_at_f
 
@@ -3176,7 +3602,7 @@ subroutine ompi_file_write_at_all_f(fh,offset,buf,count,datatype,status,ierror)
    OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   TYPE(MPI_Status) :: status
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_file_write_at_all_f
 
@@ -3186,7 +3612,7 @@ subroutine ompi_file_write_at_all_begin_f(fh,offset,buf,count,datatype,ierror) &
    implicit none
    INTEGER, INTENT(IN) :: fh
    INTEGER(MPI_OFFSET_KIND), INTENT(IN) :: offset
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: buf
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(OUT) :: ierror
@@ -3197,8 +3623,8 @@ subroutine ompi_file_write_at_all_end_f(fh,buf,status,ierror) &
    use :: mpi_f08_types, only : MPI_Status
    implicit none
    INTEGER, INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: buf
+   TYPE(MPI_Status) :: status
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_file_write_at_all_end_f
 
@@ -3210,7 +3636,7 @@ subroutine ompi_file_write_ordered_f(fh,buf,count,datatype,status,ierror) &
    OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   TYPE(MPI_Status) :: status
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_file_write_ordered_f
 
@@ -3218,7 +3644,7 @@ subroutine ompi_file_write_ordered_begin_f(fh,buf,count,datatype,ierror) &
    BIND(C, name="ompi_file_write_ordered_begin_f")
    implicit none
    INTEGER, INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: buf
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
    INTEGER, INTENT(OUT) :: ierror
@@ -3229,8 +3655,8 @@ subroutine ompi_file_write_ordered_end_f(fh,buf,status,ierror) &
    use :: mpi_f08_types, only : MPI_Status
    implicit none
    INTEGER, INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: buf
+   TYPE(MPI_Status) :: status
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_file_write_ordered_end_f
 
@@ -3242,7 +3668,7 @@ subroutine ompi_file_write_shared_f(fh,buf,count,datatype,status,ierror) &
    OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
    INTEGER, INTENT(IN) :: count
    INTEGER, INTENT(IN) :: datatype
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   TYPE(MPI_Status) :: status
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_file_write_shared_f
 
@@ -3250,7 +3676,7 @@ subroutine ompi_register_datarep_f(datarep,read_conversion_fn, &
                                    write_conversion_fn,dtype_file_extent_fn, &
                                    extra_state,ierror,datarep_len) &
    BIND(C, name="ompi_register_datarep_f")
-   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
    use :: mpi_f08_types, only : MPI_ADDRESS_KIND
    use, intrinsic :: iso_c_binding, only: c_funptr
    implicit none
@@ -3260,7 +3686,7 @@ subroutine ompi_register_datarep_f(datarep,read_conversion_fn, &
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(IN) :: datarep
    INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: extra_state
    INTEGER, INTENT(OUT) :: ierror
-   INTEGER, VALUE, INTENT(IN) :: datarep_len
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: datarep_len
 end subroutine ompi_register_datarep_f
 
 !
@@ -3337,12 +3763,12 @@ end subroutine ompi_f_sync_reg_f
 
 subroutine ompi_get_library_version_f(name,resultlen,ierror,name_len) &
    BIND(C, name="ompi_get_library_version_f")
-   use, intrinsic :: ISO_C_BINDING, only : C_CHAR
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
    implicit none
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(OUT) :: name
    INTEGER, INTENT(OUT) :: resultlen
    INTEGER, INTENT(OUT) :: ierror
-   INTEGER, VALUE, INTENT(IN) :: name_len
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: name_len
 end subroutine ompi_get_library_version_f
 
 subroutine ompi_mprobe_f(source,tag,comm,message,status,ierror) &
@@ -3352,7 +3778,7 @@ subroutine ompi_mprobe_f(source,tag,comm,message,status,ierror) &
    INTEGER, INTENT(IN) :: source, tag
    INTEGER, INTENT(IN) :: comm
    INTEGER, INTENT(OUT) :: message
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   TYPE(MPI_Status) :: status
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_mprobe_f
 
@@ -3397,14 +3823,29 @@ subroutine ompi_ineighbor_allgather_f(sendbuf,sendcount,sendtype,recvbuf,recvcou
                              BIND(C, name="ompi_ineighbor_allgather_f")
    use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
    implicit none
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
+   INTEGER, INTENT(IN) :: sendcount, recvcount
+   INTEGER, INTENT(IN) :: sendtype, recvtype
+   INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(OUT) :: request
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_ineighbor_allgather_f
+
+subroutine ompi_neighbor_allgather_init_f(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype, &
+                             comm,info,request,ierror) &
+                             BIND(C, name="ompi_neighbor_allgather_init_f")
+   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
+   implicit none
    OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
    OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
    INTEGER, INTENT(IN) :: sendcount, recvcount
    INTEGER, INTENT(IN) :: sendtype, recvtype
    INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(IN) :: info
    INTEGER, INTENT(OUT) :: request
    INTEGER, INTENT(OUT) :: ierror
-end subroutine ompi_ineighbor_allgather_f
+end subroutine ompi_neighbor_allgather_init_f
 
 subroutine ompi_neighbor_allgatherv_f(sendbuf,sendcount,sendtype,recvbuf,recvcounts,displs, &
                               recvtype,comm,ierror) &
@@ -3425,15 +3866,31 @@ subroutine ompi_ineighbor_allgatherv_f(sendbuf,sendcount,sendtype,recvbuf,recvco
                               BIND(C, name="ompi_ineighbor_allgatherv_f")
    use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
    implicit none
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
+   INTEGER, INTENT(IN) :: sendcount
+   INTEGER, INTENT(IN) OMPI_ASYNCHRONOUS :: recvcounts(*), displs(*)
+   INTEGER, INTENT(IN) :: sendtype, recvtype
+   INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(OUT) :: request
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_ineighbor_allgatherv_f
+
+subroutine ompi_neighbor_allgatherv_init_f(sendbuf,sendcount,sendtype,recvbuf,recvcounts,displs, &
+                              recvtype,comm,info,request,ierror) &
+                              BIND(C, name="ompi_neighbor_allgatherv_init_f")
+   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
+   implicit none
    OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
    OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
    INTEGER, INTENT(IN) :: sendcount
    INTEGER, INTENT(IN) :: recvcounts(*), displs(*)
    INTEGER, INTENT(IN) :: sendtype, recvtype
    INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(IN) :: info
    INTEGER, INTENT(OUT) :: request
    INTEGER, INTENT(OUT) :: ierror
-end subroutine ompi_ineighbor_allgatherv_f
+end subroutine ompi_neighbor_allgatherv_init_f
 
 subroutine ompi_neighbor_alltoall_f(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype, &
                             comm,ierror) &
@@ -3453,14 +3910,29 @@ subroutine ompi_ineighbor_alltoall_f(sendbuf,sendcount,sendtype,recvbuf,recvcoun
                             BIND(C, name="ompi_ineighbor_alltoall_f")
    use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
    implicit none
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
+   INTEGER, INTENT(IN) :: sendcount, recvcount
+   INTEGER, INTENT(IN) :: sendtype, recvtype
+   INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(OUT) :: request
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_ineighbor_alltoall_f
+
+subroutine ompi_neighbor_alltoall_init_f(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype, &
+                            comm,info,request,ierror) &
+                            BIND(C, name="ompi_neighbor_alltoall_init_f")
+   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
+   implicit none
    OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
    OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
    INTEGER, INTENT(IN) :: sendcount, recvcount
    INTEGER, INTENT(IN) :: sendtype, recvtype
    INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(IN) :: info
    INTEGER, INTENT(OUT) :: request
    INTEGER, INTENT(OUT) :: ierror
-end subroutine ompi_ineighbor_alltoall_f
+end subroutine ompi_neighbor_alltoall_init_f
 
 subroutine ompi_neighbor_alltoallv_f(sendbuf,sendcounts,sdispls,sendtype,recvbuf,recvcounts, &
                              rdispls,recvtype,comm,ierror) &
@@ -3480,14 +3952,29 @@ subroutine ompi_ineighbor_alltoallv_f(sendbuf,sendcounts,sdispls,sendtype,recvbu
                              BIND(C, name="ompi_ineighbor_alltoallv_f")
    use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
    implicit none
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
+   INTEGER, INTENT(IN) OMPI_ASYNCHRONOUS :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
+   INTEGER, INTENT(IN) :: sendtype, recvtype
+   INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(OUT) :: request
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_ineighbor_alltoallv_f
+
+subroutine ompi_neighbor_alltoallv_init_f(sendbuf,sendcounts,sdispls,sendtype,recvbuf,recvcounts, &
+                             rdispls,recvtype,comm,info,request,ierror) &
+                             BIND(C, name="ompi_neighbor_alltoallv_init_f")
+   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
+   implicit none
    OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
    OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
    INTEGER, INTENT(IN) :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
    INTEGER, INTENT(IN) :: sendtype, recvtype
    INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(IN) :: info
    INTEGER, INTENT(OUT) :: request
    INTEGER, INTENT(OUT) :: ierror
-end subroutine ompi_ineighbor_alltoallv_f
+end subroutine ompi_neighbor_alltoallv_init_f
 
 subroutine ompi_neighbor_alltoallw_f(sendbuf,sendcounts,sdispls,sendtypes,recvbuf,recvcounts, &
                              rdispls,recvtypes,comm,ierror) &
@@ -3508,14 +3995,122 @@ subroutine ompi_ineighbor_alltoallw_f(sendbuf,sendcounts,sdispls,sendtypes,recvb
                              BIND(C, name="ompi_ineighbor_alltoallw_f")
    use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request, MPI_ADDRESS_KIND
    implicit none
+   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: sendbuf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: recvbuf
+   INTEGER, INTENT(IN) OMPI_ASYNCHRONOUS :: sendcounts(*), recvcounts(*)
+   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) OMPI_ASYNCHRONOUS :: sdispls(*), rdispls(*)
+   INTEGER, INTENT(IN) :: sendtypes, recvtypes
+   INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(OUT) :: request
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_ineighbor_alltoallw_f
+
+subroutine ompi_neighbor_alltoallw_init_f(sendbuf,sendcounts,sdispls,sendtypes,recvbuf,recvcounts, &
+                             rdispls,recvtypes,comm,info,request,ierror) &
+                             BIND(C, name="ompi_neighbor_alltoallw_init_f")
+   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request, MPI_ADDRESS_KIND
+   implicit none
    OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
    OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
    INTEGER, INTENT(IN) :: sendcounts(*), recvcounts(*)
    INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: sdispls(*), rdispls(*)
    INTEGER, INTENT(IN) :: sendtypes, recvtypes
    INTEGER, INTENT(IN) :: comm
+   INTEGER, INTENT(IN) :: info
    INTEGER, INTENT(OUT) :: request
    INTEGER, INTENT(OUT) :: ierror
-end subroutine ompi_ineighbor_alltoallw_f
+end subroutine ompi_neighbor_alltoallw_init_f
+
+subroutine ompi_session_get_info_f(session, info, ierror) &
+   BIND(C, name="ompi_session_get_info_f")
+   implicit none
+   INTEGER, INTENT(IN) :: session
+   INTEGER, INTENT(out) :: info
+   INTEGER, INTENT(out) :: ierror
+end subroutine ompi_session_get_info_f
+
+subroutine ompi_session_get_nth_pset_f(session, info, n, pset_len, pset_name, ierror, pset_name_len) &
+   BIND(C, name="ompi_session_get_nth_pset_f")
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
+   implicit none
+   INTEGER, INTENT(IN) :: session
+   INTEGER, INTENT(IN) :: info
+   INTEGER, INTENT(IN) :: n
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: pset_name_len
+   INTEGER, INTENT(INOUT) :: pset_len
+   CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(OUT) :: pset_name
+   INTEGER, INTENT(out) :: ierror
+end subroutine ompi_session_get_nth_pset_f
+
+subroutine ompi_session_get_num_psets_f(session, info, npset_names, ierror) &
+   BIND(C, name="ompi_session_get_num_psets_f")
+   implicit none
+   INTEGER, INTENT(IN) :: session
+   INTEGER, INTENT(IN) :: info
+   INTEGER, INTENT(out) :: npset_names
+   INTEGER, INTENT(out) :: ierror
+end subroutine ompi_session_get_num_psets_f
+
+subroutine ompi_session_get_pset_info_f(session, pset_name, info, ierror, name_len) &
+   BIND(C, name="ompi_session_get_pset_info_f")
+   use, intrinsic :: ISO_C_BINDING, only : C_CHAR, C_INT
+   implicit none
+   INTEGER, INTENT(IN) :: session
+   CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(IN) :: pset_name
+   INTEGER, INTENT(out) :: info
+   INTEGER, INTENT(out) :: ierror
+   INTEGER(KIND=C_INT), VALUE, INTENT(IN) :: name_len
+end subroutine ompi_session_get_pset_info_f
+
+subroutine ompi_session_init_f(info, errhandler, session, ierror) &
+   BIND(C, name="ompi_session_init_f")
+   implicit none
+   INTEGER, INTENT(IN) :: info
+   INTEGER, INTENT(IN) :: errhandler
+   INTEGER, INTENT(out) :: session
+   INTEGER, INTENT(out) :: ierror
+end subroutine ompi_session_init_f
+
+subroutine ompi_session_finalize_f(session, ierror) &
+  BIND(C, name="ompi_session_finalize_f")
+  implicit none
+  INTEGER, INTENT(out) :: session
+  INTEGER, INTENT(out) :: ierror
+end subroutine ompi_session_finalize_f
+
+subroutine ompi_session_call_errhandler_f(session,errorcode,ierror) &
+   BIND(C, name="ompi_session_call_errhandler_f")
+   implicit none
+   INTEGER, INTENT(IN) :: session
+   INTEGER, INTENT(IN) :: errorcode
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_session_call_errhandler_f
+
+
+subroutine ompi_session_create_errhandler_f(session_errhandler_fn,errhandler,ierror) &
+   BIND(C, name="ompi_session_create_errhandler_f")
+   use, intrinsic :: iso_c_binding, only: c_funptr
+   implicit none
+   type(c_funptr), value :: session_errhandler_fn
+   INTEGER, INTENT(OUT) :: errhandler
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_session_create_errhandler_f
+
+subroutine ompi_session_get_errhandler_f(session,errhandler,ierror) &
+   BIND(C, name="ompi_session_get_errhandler_f")
+   implicit none
+   INTEGER, INTENT(IN) :: session
+   INTEGER, INTENT(OUT) :: errhandler
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_session_get_errhandler_f
+
+subroutine ompi_session_set_errhandler_f(session,errhandler,ierror) &
+   BIND(C, name="ompi_session_set_errhandler_f")
+   implicit none
+   INTEGER, INTENT(IN) :: session
+   INTEGER, INTENT(IN) :: errhandler
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_session_set_errhandler_f
+
 
 end interface
diff --git a/ompi/mpi/fortran/use-mpi-f08/bsend_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/bsend_f08.F90
deleted file mode 100644
index d35900bacc1..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/bsend_f08.F90
+++ /dev/null
@@ -1,28 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Bsend_f08(buf,count,datatype,dest,tag,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   use :: ompi_mpifh_bindings, only : ompi_bsend_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count, dest, tag
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_bsend_f(buf,count,datatype%MPI_VAL,dest,tag,comm%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Bsend_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/bsend_init_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/bsend_init_f08.F90
deleted file mode 100644
index dfaafdc8dcb..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/bsend_init_f08.F90
+++ /dev/null
@@ -1,30 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Bsend_init_f08(buf,count,datatype,dest,tag,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_bsend_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count, dest, tag
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_bsend_init_f(buf,count,datatype%MPI_VAL,dest,tag,comm%MPI_VAL, &
-                          request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Bsend_init_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/bsend_init_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/bsend_init_ts.c.in
new file mode 100644
index 00000000000..fd25c5e2315
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/bsend_init_ts.c.in
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID bsend_init(BUFFER x, COUNT count, DATATYPE datatype, RANK dest,
+                          TAG tag, COMM comm, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    MPI_Request c_req;
+    MPI_Comm c_comm = PMPI_Comm_f2c (*comm);
+
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(buf), c_count,
+                          c_datatype,
+                          OMPI_FINT_2_INT(*dest),
+                          OMPI_FINT_2_INT(*tag),
+                          c_comm, &c_req);
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_req);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/bsend_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/bsend_ts.c.in
new file mode 100644
index 00000000000..4b9812d8e80
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/bsend_ts.c.in
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID bsend(BUFFER x, COUNT count, DATATYPE datatype, RANK dest,
+                     TAG tag, COMM comm)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c (*comm);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(buf), c_count,
+                          c_datatype, OMPI_FINT_2_INT(*dest),
+                          OMPI_FINT_2_INT(*tag), c_comm);
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/buffer_attach_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/buffer_attach_f08.F90
deleted file mode 100644
index 5e3e907bd58..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/buffer_attach_f08.F90
+++ /dev/null
@@ -1,26 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Buffer_attach_f08(buffer,size,ierror)
-   use :: ompi_mpifh_bindings, only : ompi_buffer_attach_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: buffer
-   INTEGER, INTENT(IN) :: size
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_buffer_attach_f(buffer,size,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Buffer_attach_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/buffer_attach_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/buffer_attach_ts.c.in
new file mode 100644
index 00000000000..93e3b956d3b
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/buffer_attach_ts.c.in
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID buffer_attach(BUFFER_ASYNC x, COUNT size)
+{
+    int c_ierr;
+    @COUNT_TYPE@ c_size = (@COUNT_TYPE@)*size;
+
+    if (OMPI_CFI_IS_CONTIGUOUS(x)) {
+        c_ierr = PMPI_Buffer_attach(OMPI_CFI_BASE_ADDR(x), c_size);
+    } else {
+        c_ierr = MPI_ERR_BUFFER;
+        OMPI_ERRHANDLER_INVOKE(MPI_COMM_SELF, c_ierr, FUNC_NAME);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/buffer_detach.c.in b/ompi/mpi/fortran/use-mpi-f08/buffer_detach.c.in
new file mode 100644
index 00000000000..975e357dc82
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/buffer_detach.c.in
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2011-2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID buffer_detach(C_PTR_OUT buffer, COUNT size)
+{
+    int c_ierr;
+    void *dummy;
+    @COUNT_TYPE@ c_size;
+
+    c_ierr = @INNER_CALL@(&dummy, &c_size);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *size = (@COUNT_FINT_TYPE@)(c_size);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/buffer_detach_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/buffer_detach_f08.F90
deleted file mode 100644
index 53cb423b4b7..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/buffer_detach_f08.F90
+++ /dev/null
@@ -1,27 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2015 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Buffer_detach_f08(buffer_addr,size,ierror)
-   USE, INTRINSIC ::  ISO_C_BINDING, ONLY : C_PTR
-   use :: ompi_mpifh_bindings, only : ompi_buffer_detach_f
-   implicit none
-   TYPE(C_PTR), INTENT(OUT) ::  buffer_addr
-   INTEGER, INTENT(OUT) :: size
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_buffer_detach_f(buffer_addr,size,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Buffer_detach_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/close_port_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/close_port_f08.F90
index 53d255ee17e..c2b799fa427 100644
--- a/ompi/mpi/fortran/use-mpi-f08/close_port_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/close_port_f08.F90
@@ -11,12 +11,13 @@
 
 subroutine MPI_Close_port_f08(port_name,ierror)
    use :: ompi_mpifh_bindings, only : ompi_close_port_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
    implicit none
    CHARACTER(LEN=*), INTENT(IN) :: port_name
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
    integer :: c_ierror
 
-   call ompi_close_port_f(port_name,c_ierror,len(port_name))
+   call ompi_close_port_f(port_name,c_ierror,len(port_name,KIND=C_INT))
    if (present(ierror)) ierror = c_ierror
 
 end subroutine MPI_Close_port_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/comm_accept_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/comm_accept_f08.F90
index a900b11dd4e..27d469642af 100644
--- a/ompi/mpi/fortran/use-mpi-f08/comm_accept_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/comm_accept_f08.F90
@@ -12,6 +12,7 @@
 subroutine MPI_Comm_accept_f08(port_name,info,root,comm,newcomm,ierror)
    use :: mpi_f08_types, only : MPI_Info, MPI_Comm
    use :: ompi_mpifh_bindings, only : ompi_comm_accept_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
    implicit none
    CHARACTER(LEN=*), INTENT(IN) :: port_name
    TYPE(MPI_Info), INTENT(IN) :: info
@@ -22,7 +23,7 @@ subroutine MPI_Comm_accept_f08(port_name,info,root,comm,newcomm,ierror)
    integer :: c_ierror
 
    call ompi_comm_accept_f(port_name,info%MPI_VAL,root,comm%MPI_VAL,newcomm%MPI_VAL, &
-                           c_ierror,len(port_name))
+                           c_ierror,len(port_name,KIND=C_INT))
    if (present(ierror)) ierror = c_ierror
 
 end subroutine MPI_Comm_accept_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/comm_connect_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/comm_connect_f08.F90
index cbe71f42d72..69e64204444 100644
--- a/ompi/mpi/fortran/use-mpi-f08/comm_connect_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/comm_connect_f08.F90
@@ -12,6 +12,7 @@
 subroutine MPI_Comm_connect_f08(port_name,info,root,comm,newcomm,ierror)
    use :: mpi_f08_types, only : MPI_Info, MPI_Comm
    use :: ompi_mpifh_bindings, only : ompi_comm_connect_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
    implicit none
    CHARACTER(LEN=*), INTENT(IN) :: port_name
    TYPE(MPI_Info), INTENT(IN) :: info
@@ -22,7 +23,7 @@ subroutine MPI_Comm_connect_f08(port_name,info,root,comm,newcomm,ierror)
    integer :: c_ierror
 
    call ompi_comm_connect_f(port_name,info%MPI_VAL,root,comm%MPI_VAL,newcomm%MPI_VAL, &
-                            c_ierror,len(port_name))
+                            c_ierror,len(port_name,KIND=C_INT))
    if (present(ierror)) ierror = c_ierror
 
 end subroutine MPI_Comm_connect_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/comm_create_from_group_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/comm_create_from_group_f08.F90
new file mode 100644
index 00000000000..63d1e2bdbf2
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/comm_create_from_group_f08.F90
@@ -0,0 +1,32 @@
+! -*- f90 -*-
+!
+! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2009-2013 Los Alamos National Security, LLC.
+!                         All rights reserved.
+! Copyright (c) 2018      Research Organization for Information Science
+!                         and Technology (RIST).  All rights reserved.
+! Copyright (c) 2019      Triad National Security, LLC. All rights
+!                         reserved.
+! $COPYRIGHT$
+
+#include "mpi-f08-rename.h"
+
+subroutine MPI_Comm_create_from_group_f08(group, stringtag, info, errhandler, newcomm, ierror)
+   use :: mpi_f08_types, only : MPI_Comm, MPI_Group, MPI_Errhandler, MPI_Info
+   use :: ompi_mpifh_bindings, only : ompi_comm_create_from_group_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
+   implicit none
+   TYPE(MPI_Group), INTENT(IN) :: group
+   CHARACTER(LEN=*), INTENT(IN) :: stringtag
+   TYPE(MPI_Info), INTENT(IN) :: info
+   TYPE(MPI_Errhandler), INTENT(IN) :: errhandler
+   TYPE(MPI_Comm), INTENT(OUT) :: newcomm
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+   integer :: c_ierror
+
+   call ompi_comm_create_from_group_f(group%MPI_VAL, stringtag, info%MPI_VAL, errhandler%MPI_VAL, &
+                                      newcomm%MPI_VAL, c_ierror, len(stringtag,KIND=C_INT))
+   if (present(ierror)) ierror = c_ierror
+
+end subroutine MPI_Comm_create_from_group_f08
+
diff --git a/ompi/mpi/fortran/use-mpi-f08/comm_get_name_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/comm_get_name_f08.F90
index 3b7dad61b75..30f18a777fc 100644
--- a/ompi/mpi/fortran/use-mpi-f08/comm_get_name_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/comm_get_name_f08.F90
@@ -12,6 +12,7 @@
 subroutine MPI_Comm_get_name_f08(comm,comm_name,resultlen,ierror)
    use :: mpi_f08_types, only : MPI_Comm, MPI_MAX_OBJECT_NAME
    use :: ompi_mpifh_bindings, only : ompi_comm_get_name_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
    implicit none
    TYPE(MPI_Comm), INTENT(IN) :: comm
    CHARACTER(LEN=*), INTENT(OUT) :: comm_name
@@ -19,7 +20,8 @@ subroutine MPI_Comm_get_name_f08(comm,comm_name,resultlen,ierror)
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
    integer :: c_ierror
 
-   call ompi_comm_get_name_f(comm%MPI_VAL,comm_name,resultlen,c_ierror,len(comm_name))
+   call ompi_comm_get_name_f(comm%MPI_VAL,comm_name,resultlen,c_ierror, &
+                             len(comm_name,KIND=C_INT))
    if (present(ierror)) ierror = c_ierror
 
 end subroutine MPI_Comm_get_name_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/comm_idup_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/comm_idup_f08.F90
index a0eb02bd672..5f48967dd4a 100644
--- a/ompi/mpi/fortran/use-mpi-f08/comm_idup_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/comm_idup_f08.F90
@@ -7,6 +7,8 @@
 !                         and Technology (RIST).  All rights reserved.
 ! $COPYRIGHT$
 
+#include "ompi/mpi/fortran/configure-fortran-output.h"
+
 #include "mpi-f08-rename.h"
 
 subroutine MPI_Comm_idup_f08(comm,newcomm,request,ierror)
@@ -14,7 +16,7 @@ subroutine MPI_Comm_idup_f08(comm,newcomm,request,ierror)
    use :: ompi_mpifh_bindings, only : ompi_comm_idup_f
    implicit none
    TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Comm), INTENT(OUT) :: newcomm
+   TYPE(MPI_Comm), INTENT(OUT) OMPI_ASYNCHRONOUS :: newcomm
    TYPE(MPI_Request), INTENT(OUT) :: request
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
    integer :: c_ierror
diff --git a/ompi/mpi/fortran/use-mpi-f08/comm_idup_with_info_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/comm_idup_with_info_f08.F90
new file mode 100644
index 00000000000..c402a16709f
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/comm_idup_with_info_f08.F90
@@ -0,0 +1,30 @@
+! -*- f90 -*-
+!
+! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2009-2013 Los Alamos National Security, LLC.
+!                         All rights reserved.
+! Copyright (c) 2018-2020 Research Organization for Information Science
+!                         and Technology (RIST).  All rights reserved.
+! Copyright (c) 2021      Triad National Security, LLC. All rights
+!                         reserved.
+! $COPYRIGHT$
+
+#include "mpi-f08-rename.h"
+
+subroutine MPI_Comm_idup_with_info_f08(comm,info,newcomm,request,ierror)
+   use :: mpi_f08_types, only : MPI_Comm, MPI_Info, MPI_Request
+   use :: ompi_mpifh_bindings, only : ompi_comm_idup_with_info_f
+   implicit none
+   TYPE(MPI_Comm), INTENT(IN) :: comm
+   TYPE(MPI_Info), INTENT(IN) :: info
+   TYPE(MPI_Comm), INTENT(OUT) :: newcomm
+   TYPE(MPI_Request), INTENT(OUT) :: request
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+   integer :: c_ierror
+
+   call ompi_comm_idup_with_info_f(comm%MPI_VAL,info%MPI_VAL,newcomm%MPI_VAL,request%MPI_VAL,c_ierror)
+   if (present(ierror)) ierror = c_ierror
+
+end subroutine MPI_Comm_idup_with_info_f08
+
+
diff --git a/ompi/mpi/fortran/use-mpi-f08/comm_set_name_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/comm_set_name_f08.F90
index 958d9e267fe..e8f1b706f64 100644
--- a/ompi/mpi/fortran/use-mpi-f08/comm_set_name_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/comm_set_name_f08.F90
@@ -12,13 +12,14 @@
 subroutine MPI_Comm_set_name_f08(comm,comm_name,ierror)
    use :: mpi_f08_types, only : MPI_Comm
    use :: ompi_mpifh_bindings, only : ompi_comm_set_name_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
    implicit none
    TYPE(MPI_Comm), INTENT(IN) :: comm
    CHARACTER(LEN=*), INTENT(IN) :: comm_name
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
    integer :: c_ierror
 
-   call ompi_comm_set_name_f(comm%MPI_VAL,comm_name,c_ierror,len(comm_name))
+   call ompi_comm_set_name_f(comm%MPI_VAL,comm_name,c_ierror,len(comm_name,KIND=C_INT))
    if (present(ierror)) ierror = c_ierror
 
 end subroutine MPI_Comm_set_name_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/comm_spawn_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/comm_spawn_f08.F90
index 39514406e86..4a2b9c5d9af 100644
--- a/ompi/mpi/fortran/use-mpi-f08/comm_spawn_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/comm_spawn_f08.F90
@@ -14,6 +14,7 @@ subroutine MPI_Comm_spawn_f08(command,argv,maxprocs,info,root,comm,intercomm, &
                               array_of_errcodes,ierror)
    use :: mpi_f08_types, only : MPI_Info, MPI_Comm
    use :: ompi_mpifh_bindings, only : ompi_comm_spawn_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
    implicit none
    CHARACTER(LEN=*), INTENT(IN) :: command, argv(*)
    INTEGER, INTENT(IN) :: maxprocs, root
@@ -27,7 +28,7 @@ subroutine MPI_Comm_spawn_f08(command,argv,maxprocs,info,root,comm,intercomm, &
    call ompi_comm_spawn_f(command,argv,maxprocs,                            &
                           info%MPI_VAL,root,comm%MPI_VAL,intercomm%MPI_VAL, &
                           array_of_errcodes,c_ierror,                       &
-                          len(command), len(argv))
+                          len(command,KIND=C_INT), len(argv,KIND=C_INT))
    if (present(ierror)) ierror = c_ierror
 
 end subroutine MPI_Comm_spawn_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/comm_spawn_multiple_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/comm_spawn_multiple_f08.F90
index 300352aed71..3d668a6c0e9 100644
--- a/ompi/mpi/fortran/use-mpi-f08/comm_spawn_multiple_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/comm_spawn_multiple_f08.F90
@@ -15,6 +15,7 @@ subroutine MPI_Comm_spawn_multiple_f08(count,array_of_commands,array_of_argv, &
                                        comm,intercomm,array_of_errcodes,ierror)
    use :: mpi_f08_types, only : MPI_Info, MPI_Comm
    use :: ompi_mpifh_bindings, only : ompi_comm_spawn_multiple_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
    implicit none
    INTEGER, INTENT(IN) :: count, root
    INTEGER, INTENT(IN) :: array_of_maxprocs(count)
@@ -33,7 +34,7 @@ subroutine MPI_Comm_spawn_multiple_f08(count,array_of_commands,array_of_argv, &
    call ompi_comm_spawn_multiple_f(count,array_of_commands,array_of_argv, &
                                    array_of_maxprocs,array_of_info(:)%MPI_VAL,root, &
                                    comm%MPI_VAL,intercomm%MPI_VAL,array_of_errcodes,c_ierror, &
-                                   len(array_of_commands), len(array_of_argv))
+                                   len(array_of_commands,KIND=C_INT), len(array_of_argv,KIND=C_INT))
    if (present(ierror)) ierror = c_ierror
 
 end subroutine MPI_Comm_spawn_multiple_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/compare_and_swap_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/compare_and_swap_f08.F90
deleted file mode 100644
index d62dc914724..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/compare_and_swap_f08.F90
+++ /dev/null
@@ -1,33 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2014 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Compare_and_swap_f08(origin_addr,compare_addr,result_addr,&
-                                    datatype,target_rank,target_disp,win,&
-                                    ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Win, MPI_ADDRESS_KIND
-   use :: ompi_mpifh_bindings, only : ompi_compare_and_swap_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: origin_addr, compare_addr
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: result_addr
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER, INTENT(IN) :: target_rank
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
-   TYPE(MPI_Win), INTENT(IN) :: win
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_compare_and_swap_f(origin_addr,compare_addr,result_addr,datatype%MPI_VAL,&
-                                target_rank,target_disp,win%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Compare_and_swap_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/compare_and_swap_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/compare_and_swap_ts.c.in
new file mode 100644
index 00000000000..62308b358ca
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/compare_and_swap_ts.c.in
@@ -0,0 +1,60 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2014      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID compare_and_swap(BUFFER x1, BUFFER x2, BUFFER_ASYNC x3,
+                                DATATYPE datatype, RANK target_rank, AINT target_disp,
+                                WIN win)
+{
+    int c_ierr;
+    MPI_Datatype c_datatype = PMPI_Type_f2c(*datatype);
+    MPI_Win c_win = PMPI_Win_f2c(*win);
+    char *origin_addr = OMPI_CFI_BASE_ADDR(x1), *compare_addr = OMPI_CFI_BASE_ADDR(x2), *result_addr = OMPI_CFI_BASE_ADDR(x3);
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_win, c_ierr, FUNC_NAME)
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_win, c_ierr, FUNC_NAME)
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x3, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        return;
+    }
+    c_ierr = PMPI_Compare_and_swap(OMPI_F2C_BOTTOM(origin_addr),
+                                   OMPI_F2C_BOTTOM(compare_addr),
+                                   OMPI_F2C_BOTTOM(result_addr),
+                                   c_datatype,
+                                   OMPI_FINT_2_INT(*target_rank),
+                                   *target_disp, c_win);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/error_string_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/error_string_f08.F90
index 518cc092423..099bbb33ab1 100644
--- a/ompi/mpi/fortran/use-mpi-f08/error_string_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/error_string_f08.F90
@@ -11,6 +11,7 @@
 
 subroutine MPI_Error_string_f08(errorcode,string,resultlen,ierror)
    use :: ompi_mpifh_bindings, only : ompi_error_string_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
    implicit none
    integer, intent(in) :: errorcode
    character(len=*), intent(out) :: string
@@ -18,7 +19,7 @@ subroutine MPI_Error_string_f08(errorcode,string,resultlen,ierror)
    integer, optional, intent(out) :: ierror
    integer :: c_ierror
 
-   call ompi_error_string_f(errorcode,string,resultlen,c_ierror,len(string))
+   call ompi_error_string_f(errorcode,string,resultlen,c_ierror,len(string,KIND=C_INT))
    if (present(ierror)) ierror = c_ierror
 
 end subroutine MPI_Error_string_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/exscan_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/exscan_f08.F90
deleted file mode 100644
index cf78dc016c5..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/exscan_f08.F90
+++ /dev/null
@@ -1,31 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Exscan_f08(sendbuf,recvbuf,count,datatype,op,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm
-   use :: ompi_mpifh_bindings, only : ompi_exscan_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_exscan_f(sendbuf,recvbuf,count,datatype%MPI_VAL,&
-                      op%MPI_VAL,comm%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Exscan_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/exscan_init_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/exscan_init_ts.c.in
new file mode 100644
index 00000000000..af0d924d304
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/exscan_init_ts.c.in
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID exscan_init(BUFFER_ASYNC x1, BUFFER_ASYNC_OUT x2, COUNT count,
+                           DATATYPE datatype, OP op, COMM comm,
+                           INFO info, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Comm c_comm;
+    MPI_Datatype c_type;
+    MPI_Info c_info;
+    MPI_Request c_request;
+    MPI_Op c_op;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_type = PMPI_Type_f2c(*datatype);
+    c_op = PMPI_Op_f2c(*op);
+    c_info = PMPI_Info_f2c(*info);
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM (sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM (recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf, recvbuf,
+                          (@COUNT_TYPE@) *count,
+                          c_type, c_op, c_comm, c_info, &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/exscan_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/exscan_ts.c.in
new file mode 100644
index 00000000000..234037053ad
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/exscan_ts.c.in
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID exscan(BUFFER x1, BUFFER_OUT x2, COUNT count,
+                      DATATYPE datatype, OP op, COMM comm)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype c_type;
+    MPI_Op c_op;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    c_type = PMPI_Type_f2c(*datatype);
+    c_op = PMPI_Op_f2c(*op);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM (sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM (recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf, recvbuf,
+                          (@COUNT_TYPE@) *count,
+                          c_type, c_op, c_comm);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/f_sync_reg_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/f_sync_reg_f08.F90
index 6b100045f82..ec8d86c5cb6 100644
--- a/ompi/mpi/fortran/use-mpi-f08/f_sync_reg_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/f_sync_reg_f08.F90
@@ -15,7 +15,7 @@
 subroutine MPI_F_sync_reg_f08(buf)
    use :: ompi_mpifh_bindings, only : ompi_f_sync_reg_f
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS:: buf
 
    call ompi_f_sync_reg_f(buf)
 
diff --git a/ompi/mpi/fortran/use-mpi-f08/f_sync_reg_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/f_sync_reg_ts.c.in
new file mode 100644
index 00000000000..08d72b85a82
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/f_sync_reg_ts.c.in
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      University of Oregon.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID f_sync_reg(BUFFER x)
+{
+    /* This is a noop in C to disable potential Fortran optimizations. */
+    return;
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/fetch_and_op_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/fetch_and_op_f08.F90
deleted file mode 100644
index e618a861bf0..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/fetch_and_op_f08.F90
+++ /dev/null
@@ -1,33 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2014 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Fetch_and_op_f08(origin_addr,result_addr,datatype,target_rank, &
-                                target_disp,op,win,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Win, MPI_ADDRESS_KIND
-   use :: ompi_mpifh_bindings, only : ompi_fetch_and_op_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: origin_addr
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: result_addr
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER, INTENT(IN) :: target_rank
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Win), INTENT(IN) :: win
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_fetch_and_op_f(origin_addr,result_addr,datatype%MPI_VAL,target_rank,&
-                            target_disp,op%MPI_VAL,win%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Fetch_and_op_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/fetch_and_op_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/fetch_and_op_ts.c.in
new file mode 100644
index 00000000000..c6ca7e55415
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/fetch_and_op_ts.c.in
@@ -0,0 +1,55 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2014      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID fetch_and_op(BUFFER x1, BUFFER_OUT x2, DATATYPE datatype,
+                            RANK target_rank, AINT target_disp,
+                            OP op, WIN win)
+{
+    int c_ierr;
+    MPI_Datatype c_datatype = PMPI_Type_f2c(*datatype);
+    MPI_Win c_win = PMPI_Win_f2c(*win);
+    MPI_Op c_op = PMPI_Op_f2c(*op);
+    char *origin_addr = OMPI_CFI_BASE_ADDR(x1), *result_addr = OMPI_CFI_BASE_ADDR(x2);
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_win, c_ierr, FUNC_NAME)
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_win, c_ierr, FUNC_NAME)
+        return;
+    }
+    c_ierr = PMPI_Fetch_and_op(OMPI_F2C_BOTTOM(origin_addr),
+                               OMPI_F2C_BOTTOM(result_addr),
+                               c_datatype,
+                               OMPI_FINT_2_INT(*target_rank),
+                               *target_disp, c_op, c_win);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_delete_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_delete_f08.F90
index c422403e348..e34374140ac 100644
--- a/ompi/mpi/fortran/use-mpi-f08/file_delete_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/file_delete_f08.F90
@@ -12,13 +12,14 @@
 subroutine MPI_File_delete_f08(filename,info,ierror)
    use :: mpi_f08_types, only : MPI_Info
    use :: ompi_mpifh_bindings, only : ompi_file_delete_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
    implicit none
    CHARACTER(LEN=*), INTENT(IN) :: filename
    TYPE(MPI_Info), INTENT(IN) :: info
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
    integer :: c_ierror
 
-   call ompi_file_delete_f(filename,info%MPI_VAL,c_ierror,len(filename))
+   call ompi_file_delete_f(filename,info%MPI_VAL,c_ierror,len(filename,KIND=C_INT))
    if (present(ierror)) ierror = c_ierror
 
 end subroutine MPI_File_delete_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_get_view_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_get_view_f08.F90
index e56ddc99576..e1c541b178e 100644
--- a/ompi/mpi/fortran/use-mpi-f08/file_get_view_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/file_get_view_f08.F90
@@ -12,6 +12,7 @@
 subroutine MPI_File_get_view_f08(fh,disp,etype,filetype,datarep,ierror)
    use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_OFFSET_KIND
    use :: ompi_mpifh_bindings, only : ompi_file_get_view_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
    implicit none
    TYPE(MPI_File), INTENT(IN) :: fh
    INTEGER(MPI_OFFSET_KIND), INTENT(OUT) :: disp
@@ -22,7 +23,7 @@ subroutine MPI_File_get_view_f08(fh,disp,etype,filetype,datarep,ierror)
    integer :: c_ierror
 
    call ompi_file_get_view_f(fh%MPI_VAL,disp,etype%MPI_VAL,filetype%MPI_VAL, &
-                             datarep,c_ierror,len(datarep))
+                             datarep,c_ierror,len(datarep,KIND=C_INT))
    if (present(ierror)) ierror = c_ierror
 
 end subroutine MPI_File_get_view_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_iread_all_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_iread_all_f08.F90
deleted file mode 100644
index 6f28c453492..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_iread_all_f08.F90
+++ /dev/null
@@ -1,29 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_iread_all_f08(fh,buf,count,datatype,request,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_file_iread_all_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_iread_all_f(fh%MPI_VAL,buf,count,datatype%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_iread_all_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_iread_all_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_iread_all_ts.c.in
new file mode 100644
index 00000000000..23c828f3a6c
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_iread_all_ts.c.in
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_iread_all(FILE fh, BUFFER_ASYNC x, COUNT count,
+                              DATATYPE datatype, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+    MPI_Request c_request;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_fh, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(c_fh, OMPI_F2C_BOTTOM(buf),
+                          c_count,
+                          c_datatype, &c_request);
+
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_request);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_iread_at_all_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_iread_at_all_f08.F90
deleted file mode 100644
index ce12f53ba16..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_iread_at_all_f08.F90
+++ /dev/null
@@ -1,31 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_iread_at_all_f08(fh,offset,buf,count,datatype,request,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Request, MPI_OFFSET_KIND
-   use :: ompi_mpifh_bindings, only : ompi_file_iread_at_all_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   INTEGER(MPI_OFFSET_KIND), INTENT(IN) :: offset
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_iread_at_all_f(fh%MPI_VAL,offset,buf,count,&
-                             datatype%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_iread_at_all_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_iread_at_all_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_iread_at_all_ts.c.in
new file mode 100644
index 00000000000..b5fcb9ce8a3
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_iread_at_all_ts.c.in
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_iread_at_all(FILE fh, OFFSET offset,
+                                 BUFFER_ASYNC x, COUNT count,
+                                 DATATYPE datatype, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+    MPI_Request c_request;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_fh, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(c_fh, (MPI_Offset) *offset,
+                          OMPI_F2C_BOTTOM(buf),
+                          c_count,
+                          c_datatype,
+                          &c_request);
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_request);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_iread_at_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_iread_at_f08.F90
index 945fafd844b..177a855ba35 100644
--- a/ompi/mpi/fortran/use-mpi-f08/file_iread_at_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/file_iread_at_f08.F90
@@ -17,7 +17,7 @@ subroutine MPI_File_iread_at_f08(fh,offset,buf,count,datatype,request,ierror)
    implicit none
    TYPE(MPI_File), INTENT(IN) :: fh
    INTEGER(MPI_OFFSET_KIND), INTENT(IN) :: offset
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: buf
    INTEGER, INTENT(IN) :: count
    TYPE(MPI_Datatype), INTENT(IN) :: datatype
    TYPE(MPI_Request), INTENT(OUT) :: request
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_iread_at_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_iread_at_ts.c.in
new file mode 100644
index 00000000000..c89829cac80
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_iread_at_ts.c.in
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_iread_at(FILE fh, OFFSET offset,
+                             BUFFER_ASYNC x, COUNT count,
+                             DATATYPE datatype, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+    MPI_Request c_request;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_fh, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(c_fh, (MPI_Offset) *offset,
+                          OMPI_F2C_BOTTOM(buf),
+                          c_count,
+                          c_datatype,
+                          &c_request);
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_request);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_iread_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_iread_f08.F90
deleted file mode 100644
index a370b062ad5..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_iread_f08.F90
+++ /dev/null
@@ -1,29 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_iread_f08(fh,buf,count,datatype,request,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_file_iread_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_iread_f(fh%MPI_VAL,buf,count,datatype%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_iread_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_iread_shared_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_iread_shared_f08.F90
deleted file mode 100644
index bd9bb3aa093..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_iread_shared_f08.F90
+++ /dev/null
@@ -1,30 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_iread_shared_f08(fh,buf,count,datatype,request,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_file_iread_shared_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_iread_shared_f(fh%MPI_VAL,buf,count,&
-                                 datatype%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_iread_shared_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_iread_shared_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_iread_shared_ts.c.in
new file mode 100644
index 00000000000..6decb21234d
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_iread_shared_ts.c.in
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_iread_shared(FILE fh, BUFFER_ASYNC x, COUNT count,
+                                 DATATYPE datatype, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+    MPI_Request c_request;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_fh, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(c_fh,
+                          OMPI_F2C_BOTTOM(buf),
+                          c_count,
+                          c_datatype,
+                          &c_request);
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_request);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_iread_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_iread_ts.c.in
new file mode 100644
index 00000000000..b1c4aabad12
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_iread_ts.c.in
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_iread(FILE fh, BUFFER_ASYNC x, COUNT count,
+                          DATATYPE datatype, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+    MPI_Request c_request;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_fh, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(c_fh, OMPI_F2C_BOTTOM(buf),
+                          c_count,
+                          c_datatype, &c_request);
+
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_request);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_iwrite_all_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_iwrite_all_f08.F90
deleted file mode 100644
index 49eda45013c..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_iwrite_all_f08.F90
+++ /dev/null
@@ -1,30 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_iwrite_all_f08(fh,buf,count,datatype,request,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_file_iwrite_all_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_iwrite_all_f(fh%MPI_VAL,buf,count,&
-                           datatype%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_iwrite_all_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_iwrite_all_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_iwrite_all_ts.c.in
new file mode 100644
index 00000000000..3f11d68db89
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_iwrite_all_ts.c.in
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_iwrite_all(FILE fh, BUFFER x, COUNT count,
+                               DATATYPE datatype, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+    MPI_Request c_request;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_fh, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(c_fh, OMPI_F2C_BOTTOM(buf),
+                          c_count,
+                          c_datatype,
+                          &c_request);
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_request);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_iwrite_at_all_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_iwrite_at_all_f08.F90
deleted file mode 100644
index 1d2d98fe5da..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_iwrite_at_all_f08.F90
+++ /dev/null
@@ -1,31 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_iwrite_at_all_f08(fh,offset,buf,count,datatype,request,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Request, MPI_OFFSET_KIND
-   use :: ompi_mpifh_bindings, only : ompi_file_iwrite_at_all_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   INTEGER(MPI_OFFSET_KIND), INTENT(IN) :: offset
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_iwrite_at_all_f(fh%MPI_VAL,offset,buf,count,&
-                              datatype%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_iwrite_at_all_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_iwrite_at_all_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_iwrite_at_all_ts.c.in
new file mode 100644
index 00000000000..09a8bb39598
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_iwrite_at_all_ts.c.in
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_iwrite_at_all(FILE fh, OFFSET offset, BUFFER x,
+                                  COUNT count, DATATYPE datatype,
+                                  REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+    MPI_Request c_request;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_fh, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(c_fh, (MPI_Offset) *offset,
+                          OMPI_F2C_BOTTOM(buf),
+                          c_count,
+                          c_datatype,
+                          &c_request);
+
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_request);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_iwrite_at_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_iwrite_at_f08.F90
deleted file mode 100644
index 8f7a3f55697..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_iwrite_at_f08.F90
+++ /dev/null
@@ -1,31 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_iwrite_at_f08(fh,offset,buf,count,datatype,request,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Request, MPI_OFFSET_KIND
-   use :: ompi_mpifh_bindings, only : ompi_file_iwrite_at_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   INTEGER(MPI_OFFSET_KIND), INTENT(IN) :: offset
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_iwrite_at_f(fh%MPI_VAL,offset,buf,count,&
-                              datatype%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_iwrite_at_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_iwrite_at_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_iwrite_at_ts.c.in
new file mode 100644
index 00000000000..e6e09c1c560
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_iwrite_at_ts.c.in
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_iwrite_at(FILE fh, OFFSET offset, BUFFER x,
+                              COUNT count, DATATYPE datatype,
+                              REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+    MPI_Request c_request;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_fh, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(c_fh, (MPI_Offset) *offset,
+                          OMPI_F2C_BOTTOM(buf),
+                          c_count,
+                          c_datatype,
+                          &c_request);
+
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+       *request = PMPI_Request_c2f(c_request);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_iwrite_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_iwrite_f08.F90
deleted file mode 100644
index 72d7b499730..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_iwrite_f08.F90
+++ /dev/null
@@ -1,30 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_iwrite_f08(fh,buf,count,datatype,request,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_file_iwrite_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_iwrite_f(fh%MPI_VAL,buf,count,&
-                           datatype%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_iwrite_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_iwrite_shared_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_iwrite_shared_f08.F90
deleted file mode 100644
index 8f5500c664f..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_iwrite_shared_f08.F90
+++ /dev/null
@@ -1,30 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_iwrite_shared_f08(fh,buf,count,datatype,request,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_file_iwrite_shared_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   TYPE(MPI_File), INTENT(IN) :: fh
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_iwrite_shared_f(fh%MPI_VAL,buf,count,&
-                                  datatype%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_iwrite_shared_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_iwrite_shared_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_iwrite_shared_ts.c.in
new file mode 100644
index 00000000000..0e5a236cbef
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_iwrite_shared_ts.c.in
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_iwrite_shared(FILE fh, BUFFER x, COUNT count,
+                                  DATATYPE datatype, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+    MPI_Request c_request;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_fh, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(c_fh, OMPI_F2C_BOTTOM(buf), c_count,
+                          c_datatype, &c_request);
+
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_request);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_iwrite_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_iwrite_ts.c.in
new file mode 100644
index 00000000000..c994d7cad3c
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_iwrite_ts.c.in
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_iwrite(FILE fh, BUFFER x, COUNT count, DATATYPE datatype,
+                           REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+    MPI_Request c_request;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_fh, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(c_fh, OMPI_F2C_BOTTOM(buf), c_count,
+                          c_datatype, &c_request);
+
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_request);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_open_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_open_f08.F90
index f3df1e86424..fe551880f4b 100644
--- a/ompi/mpi/fortran/use-mpi-f08/file_open_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/file_open_f08.F90
@@ -12,6 +12,7 @@
 subroutine MPI_File_open_f08(comm,filename,amode,info,fh,ierror)
    use :: mpi_f08_types, only : MPI_Comm, MPI_Info, MPI_File
    use :: ompi_mpifh_bindings, only : ompi_file_open_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
    implicit none
    TYPE(MPI_Comm), INTENT(IN) :: comm
    CHARACTER(LEN=*), INTENT(IN) :: filename
@@ -22,7 +23,7 @@ subroutine MPI_File_open_f08(comm,filename,amode,info,fh,ierror)
    integer :: c_ierror
 
    call ompi_file_open_f(comm%MPI_VAL,filename,amode,info%MPI_VAL,fh%MPI_VAL, &
-                         c_ierror, len(filename))
+                         c_ierror, len(filename,KIND=C_INT))
    if (present(ierror)) ierror = c_ierror
 
 end subroutine MPI_File_open_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_read_all_begin_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_read_all_begin_f08.F90
deleted file mode 100644
index f4a8cb36842..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_read_all_begin_f08.F90
+++ /dev/null
@@ -1,28 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_read_all_begin_f08(fh,buf,count,datatype,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype
-   use :: ompi_mpifh_bindings, only : ompi_file_read_all_begin_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_read_all_begin_f(fh%MPI_VAL,buf,count,datatype%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_read_all_begin_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_read_all_begin_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_read_all_begin_ts.c.in
new file mode 100644
index 00000000000..f962e2e3566
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_read_all_begin_ts.c.in
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_read_all_begin(FILE fh, BUFFER_ASYNC x,
+                                   COUNT count, DATATYPE datatype)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_fh, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(c_fh, OMPI_F2C_BOTTOM(buf), c_count, c_datatype);
+
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_read_all_end_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_read_all_end_f08.F90
deleted file mode 100644
index 051ee558012..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_read_all_end_f08.F90
+++ /dev/null
@@ -1,27 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_read_all_end_f08(fh,buf,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Status
-   use :: ompi_mpifh_bindings, only : ompi_file_read_all_end_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   TYPE(MPI_Status), INTENT(OUT) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_read_all_end_f(fh%MPI_VAL,buf,status,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_read_all_end_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_read_all_end_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_read_all_end_ts.c.in
new file mode 100644
index 00000000000..40aedf00e4c
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_read_all_end_ts.c.in
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_read_all_end(FILE fh, BUFFER_ASYNC x, STATUS_OUT status)
+{
+    int c_ierr;
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    OMPI_FORTRAN_STATUS_DECLARATION(c_status,c_status2)
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+
+    OMPI_FORTRAN_STATUS_SET_POINTER(c_status,c_status2,status)
+
+    c_ierr = PMPI_File_read_all_end(c_fh, buf, c_status);
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    OMPI_FORTRAN_STATUS_RETURN(c_status,c_status2,status,c_ierr)
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_read_all_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_read_all_f08.F90
deleted file mode 100644
index 4a917a50265..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_read_all_f08.F90
+++ /dev/null
@@ -1,29 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_read_all_f08(fh,buf,count,datatype,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Status
-   use :: ompi_mpifh_bindings, only : ompi_file_read_all_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Status), INTENT(OUT) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_read_all_f(fh%MPI_VAL,buf,count,datatype%MPI_VAL,status,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_read_all_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_read_all_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_read_all_ts.c.in
new file mode 100644
index 00000000000..4116d175198
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_read_all_ts.c.in
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_read_all(FILE fh, BUFFER x, COUNT count,
+                             DATATYPE datatype, STATUS_OUT status)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+    OMPI_FORTRAN_STATUS_DECLARATION(c_status,c_status2)
+
+    OMPI_FORTRAN_STATUS_SET_POINTER(c_status,c_status2,status)
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_fh, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(c_fh, OMPI_F2C_BOTTOM(buf), c_count, c_datatype,
+                          c_status);
+
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    OMPI_FORTRAN_STATUS_RETURN(c_status,c_status2,status,c_ierr)
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_read_at_all_begin_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_read_at_all_begin_f08.F90
deleted file mode 100644
index 8e79583995f..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_read_at_all_begin_f08.F90
+++ /dev/null
@@ -1,30 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_read_at_all_begin_f08(fh,offset,buf,count,datatype,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_OFFSET_KIND
-   use :: ompi_mpifh_bindings, only : ompi_file_read_at_all_begin_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   INTEGER(MPI_OFFSET_KIND), INTENT(IN) :: offset
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_read_at_all_begin_f(fh%MPI_VAL,offset,buf,count,&
-                                      datatype%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_read_at_all_begin_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_read_at_all_begin_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_read_at_all_begin_ts.c.in
new file mode 100644
index 00000000000..23b60e81f5e
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_read_at_all_begin_ts.c.in
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_read_at_all_begin(FILE fh, OFFSET offset,
+                                      BUFFER_ASYNC x, COUNT count,
+                                      DATATYPE datatype)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_fh, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(c_fh, (MPI_Offset) *offset, OMPI_F2C_BOTTOM(buf),
+                          c_count, c_datatype);
+
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_read_at_all_end_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_read_at_all_end_f08.F90
deleted file mode 100644
index 3fc8c90d258..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_read_at_all_end_f08.F90
+++ /dev/null
@@ -1,27 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_read_at_all_end_f08(fh,buf,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Status
-   use :: ompi_mpifh_bindings, only : ompi_file_read_at_all_end_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   TYPE(MPI_Status), INTENT(OUT) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_read_at_all_end_f(fh%MPI_VAL,buf,status,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_read_at_all_end_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_read_at_all_end_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_read_at_all_end_ts.c.in
new file mode 100644
index 00000000000..5193f23e8b9
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_read_at_all_end_ts.c.in
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_read_at_all_end(FILE fh, BUFFER_ASYNC x,
+                                    STATUS_OUT status)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    OMPI_FORTRAN_STATUS_DECLARATION(c_status,c_status2)
+
+    OMPI_FORTRAN_STATUS_SET_POINTER(c_status,c_status2,status)
+
+    c_ierr = PMPI_File_read_at_all_end(c_fh, buf, c_status);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    OMPI_FORTRAN_STATUS_RETURN(c_status,c_status2,status,c_ierr)
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_read_at_all_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_read_at_all_f08.F90
deleted file mode 100644
index e358b7dbbaf..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_read_at_all_f08.F90
+++ /dev/null
@@ -1,31 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_read_at_all_f08(fh,offset,buf,count,datatype,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Status, MPI_OFFSET_KIND
-   use :: ompi_mpifh_bindings, only : ompi_file_read_at_all_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   INTEGER(MPI_OFFSET_KIND), INTENT(IN) :: offset
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Status), INTENT(OUT) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_read_at_all_f(fh%MPI_VAL,offset,buf,count,&
-                                datatype%MPI_VAL,status,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_read_at_all_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_read_at_all_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_read_at_all_ts.c.in
new file mode 100644
index 00000000000..d9ec1042113
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_read_at_all_ts.c.in
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_read_at_all(FILE fh, OFFSET offset,
+                                BUFFER x, COUNT count,
+                                DATATYPE datatype, STATUS_OUT status)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+    OMPI_FORTRAN_STATUS_DECLARATION(c_status,c_status2)
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_fh, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    OMPI_FORTRAN_STATUS_SET_POINTER(c_status,c_status2,status)
+
+    c_ierr = @INNER_CALL@(c_fh, (MPI_Offset) *offset, OMPI_F2C_BOTTOM(buf),
+                          c_count, c_datatype, c_status);
+
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    OMPI_FORTRAN_STATUS_RETURN(c_status,c_status2,status,c_ierr)
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_read_at_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_read_at_f08.F90
deleted file mode 100644
index e7004c759e0..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_read_at_f08.F90
+++ /dev/null
@@ -1,30 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_read_at_f08(fh,offset,buf,count,datatype,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Status, MPI_OFFSET_KIND
-   use :: ompi_mpifh_bindings, only : ompi_file_read_at_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   INTEGER(MPI_OFFSET_KIND), INTENT(IN) :: offset
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Status), INTENT(OUT) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_read_at_f(fh%MPI_VAL,offset,buf,count,datatype%MPI_VAL,status,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_read_at_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_read_at_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_read_at_ts.c.in
new file mode 100644
index 00000000000..054ae0cba1f
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_read_at_ts.c.in
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_read_at(FILE fh, OFFSET offset, BUFFER x,
+                            COUNT count, DATATYPE datatype,
+                            STATUS_OUT status)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+    OMPI_FORTRAN_STATUS_DECLARATION(c_status,c_status2)
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_fh, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    OMPI_FORTRAN_STATUS_SET_POINTER(c_status,c_status2,status)
+
+    c_ierr = @INNER_CALL@(c_fh, (MPI_Offset) *offset, buf, c_count,
+                          c_datatype, c_status);
+
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    OMPI_FORTRAN_STATUS_RETURN(c_status,c_status2,status,c_ierr)
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_read_ordered_begin_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_read_ordered_begin_f08.F90
deleted file mode 100644
index 816ea90b39c..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_read_ordered_begin_f08.F90
+++ /dev/null
@@ -1,28 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_read_ordered_begin_f08(fh,buf,count,datatype,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype
-   use :: ompi_mpifh_bindings, only : ompi_file_read_ordered_begin_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_read_ordered_begin_f(fh%MPI_VAL,buf,count,datatype%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_read_ordered_begin_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_read_ordered_begin_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_read_ordered_begin_ts.c.in
new file mode 100644
index 00000000000..c48b13cbe15
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_read_ordered_begin_ts.c.in
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_read_ordered_begin(FILE fh, BUFFER_ASYNC x, COUNT count,
+                                       DATATYPE datatype)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_fh, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(c_fh, OMPI_F2C_BOTTOM(buf), c_count, c_datatype);
+
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_read_ordered_end_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_read_ordered_end_f08.F90
deleted file mode 100644
index 9dac5eaa06b..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_read_ordered_end_f08.F90
+++ /dev/null
@@ -1,27 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_read_ordered_end_f08(fh,buf,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Status
-   use :: ompi_mpifh_bindings, only : ompi_file_read_ordered_end_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   TYPE(MPI_Status), INTENT(OUT) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_read_ordered_end_f(fh%MPI_VAL,buf,status,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_read_ordered_end_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_read_ordered_end_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_read_ordered_end_ts.c.in
new file mode 100644
index 00000000000..0c7d150121f
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_read_ordered_end_ts.c.in
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_read_ordered_end(FILE fh, BUFFER_ASYNC x, STATUS_OUT status)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    OMPI_FORTRAN_STATUS_DECLARATION(c_status,c_status2)
+
+    OMPI_FORTRAN_STATUS_SET_POINTER(c_status,c_status2,status)
+
+    c_ierr = @INNER_CALL@(c_fh, buf, c_status);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    OMPI_FORTRAN_STATUS_RETURN(c_status,c_status2,status,c_ierr)
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_read_ordered_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_read_ordered_f08.F90
deleted file mode 100644
index c9947619c16..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_read_ordered_f08.F90
+++ /dev/null
@@ -1,30 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_read_ordered_f08(fh,buf,count,datatype,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Status
-   use :: ompi_mpifh_bindings, only : ompi_file_read_ordered_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Status), INTENT(OUT) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_read_ordered_f(fh%MPI_VAL,buf,count,&
-                                 datatype%MPI_VAL,status,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_read_ordered_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_read_ordered_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_read_ordered_ts.c.in
new file mode 100644
index 00000000000..c59e581b627
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_read_ordered_ts.c.in
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_read_ordered(FILE fh, BUFFER x, COUNT count,
+                                 DATATYPE datatype, STATUS_OUT status)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+    OMPI_FORTRAN_STATUS_DECLARATION(c_status,c_status2)
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_fh, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    OMPI_FORTRAN_STATUS_SET_POINTER(c_status,c_status2,status)
+
+    c_ierr = @INNER_CALL@(c_fh, OMPI_F2C_BOTTOM(buf), c_count, c_datatype,
+                          c_status);
+
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    OMPI_FORTRAN_STATUS_RETURN(c_status,c_status2,status,c_ierr)
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_read_shared_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_read_shared_f08.F90
deleted file mode 100644
index a6bd046f497..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_read_shared_f08.F90
+++ /dev/null
@@ -1,29 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_read_shared_f08(fh,buf,count,datatype,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Status
-   use :: ompi_mpifh_bindings, only : ompi_file_read_shared_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Status), INTENT(OUT) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_read_shared_f(fh%MPI_VAL,buf,count,datatype%MPI_VAL,status,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_read_shared_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_read_shared_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_read_shared_ts.c.in
new file mode 100644
index 00000000000..b47b0a46a46
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_read_shared_ts.c.in
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_read_shared(FILE fh, BUFFER x, COUNT count,
+                                DATATYPE datatype, STATUS_OUT status)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+    OMPI_FORTRAN_STATUS_DECLARATION(c_status,c_status2)
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_fh, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    OMPI_FORTRAN_STATUS_SET_POINTER(c_status,c_status2,status)
+
+    c_ierr = @INNER_CALL@(c_fh, OMPI_F2C_BOTTOM(buf), c_count, c_datatype,
+                          c_status);
+
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    OMPI_FORTRAN_STATUS_RETURN(c_status,c_status2,status,c_ierr)
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_read_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_read_ts.c.in
new file mode 100644
index 00000000000..3c36266af7b
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_read_ts.c.in
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_read(FILE fh, BUFFER x, COUNT count,
+                         DATATYPE datatype, STATUS_OUT status)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+    OMPI_FORTRAN_STATUS_DECLARATION(c_status,c_status2)
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_fh, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    OMPI_FORTRAN_STATUS_SET_POINTER(c_status,c_status2,status)
+
+    c_ierr = @INNER_CALL@(c_fh, OMPI_F2C_BOTTOM(buf), c_count, c_datatype,
+                          c_status);
+
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    OMPI_FORTRAN_STATUS_RETURN(c_status,c_status2,status,c_ierr)
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_set_view_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_set_view_f08.F90
index b0ad495eec7..ff2533c7c4e 100644
--- a/ompi/mpi/fortran/use-mpi-f08/file_set_view_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/file_set_view_f08.F90
@@ -12,6 +12,7 @@
 subroutine MPI_File_set_view_f08(fh,disp,etype,filetype,datarep,info,ierror)
    use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Info, MPI_OFFSET_KIND
    use :: ompi_mpifh_bindings, only : ompi_file_set_view_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
    implicit none
    TYPE(MPI_File), INTENT(IN) :: fh
    INTEGER(MPI_OFFSET_KIND), INTENT(IN) :: disp
@@ -23,7 +24,7 @@ subroutine MPI_File_set_view_f08(fh,disp,etype,filetype,datarep,info,ierror)
    integer :: c_ierror
 
    call ompi_file_set_view_f(fh%MPI_VAL,disp,etype%MPI_VAL,filetype%MPI_VAL, &
-                             datarep,info%MPI_VAL,c_ierror,len(datarep))
+                             datarep,info%MPI_VAL,c_ierror,len(datarep,KIND=C_INT))
    if (present(ierror)) ierror = c_ierror
 
 end subroutine MPI_File_set_view_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_write_all_begin_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_write_all_begin_f08.F90
deleted file mode 100644
index 7e3f2567dde..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_write_all_begin_f08.F90
+++ /dev/null
@@ -1,28 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_write_all_begin_f08(fh,buf,count,datatype,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype
-   use :: ompi_mpifh_bindings, only : ompi_file_write_all_begin_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_write_all_begin_f(fh%MPI_VAL,buf,count,datatype%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_write_all_begin_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_write_all_begin_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_write_all_begin_ts.c.in
new file mode 100644
index 00000000000..1f79c0c43b9
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_write_all_begin_ts.c.in
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_write_all_begin(FILE fh, BUFFER_ASYNC x,
+                                    COUNT count, DATATYPE datatype)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_fh, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(c_fh, OMPI_F2C_BOTTOM(buf), c_count, c_datatype);
+
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_write_all_end_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_write_all_end_f08.F90
deleted file mode 100644
index 530422099f8..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_write_all_end_f08.F90
+++ /dev/null
@@ -1,27 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_write_all_end_f08(fh,buf,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Status
-   use :: ompi_mpifh_bindings, only : ompi_file_write_all_end_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   TYPE(MPI_Status), INTENT(OUT) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_write_all_end_f(fh%MPI_VAL,buf,status,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_write_all_end_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_write_all_end_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_write_all_end_ts.c.in
new file mode 100644
index 00000000000..420224432d8
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_write_all_end_ts.c.in
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_write_all_end(FILE fh, BUFFER_ASYNC x, STATUS_OUT status)
+{
+    int c_ierr;
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    OMPI_FORTRAN_STATUS_DECLARATION(c_status,c_status2)
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+
+    OMPI_FORTRAN_STATUS_SET_POINTER(c_status,c_status2,status)
+
+    c_ierr = @INNER_CALL@(c_fh, buf, c_status);
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    OMPI_FORTRAN_STATUS_RETURN(c_status,c_status2,status,c_ierr)
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_write_all_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_write_all_f08.F90
deleted file mode 100644
index af975304c94..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_write_all_f08.F90
+++ /dev/null
@@ -1,30 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_write_all_f08(fh,buf,count,datatype,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Status
-   use :: ompi_mpifh_bindings, only : ompi_file_write_all_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Status), INTENT(OUT) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_write_all_f(fh%MPI_VAL,buf,count,&
-                             datatype%MPI_VAL,status,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_write_all_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_write_all_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_write_all_ts.c.in
new file mode 100644
index 00000000000..9c07a808bef
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_write_all_ts.c.in
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_write_all(FILE fh, BUFFER x, COUNT count,
+                              DATATYPE datatype, STATUS_OUT status)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+    OMPI_FORTRAN_STATUS_DECLARATION(c_status,c_status2)
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_fh, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    OMPI_FORTRAN_STATUS_SET_POINTER(c_status,c_status2,status)
+
+    c_ierr = @INNER_CALL@(c_fh, OMPI_F2C_BOTTOM(buf), c_count,
+                          c_datatype, c_status);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+
+    OMPI_FORTRAN_STATUS_RETURN(c_status,c_status2,status,c_ierr)
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_write_at_all_begin_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_write_at_all_begin_f08.F90
deleted file mode 100644
index 13a2677d8d3..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_write_at_all_begin_f08.F90
+++ /dev/null
@@ -1,30 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_write_at_all_begin_f08(fh,offset,buf,count,datatype,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_OFFSET_KIND
-   use :: ompi_mpifh_bindings, only : ompi_file_write_at_all_begin_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   INTEGER(MPI_OFFSET_KIND), INTENT(IN) :: offset
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_write_at_all_begin_f(fh%MPI_VAL,offset,buf,count,&
-                                      datatype%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_write_at_all_begin_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_write_at_all_begin_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_write_at_all_begin_ts.c.in
new file mode 100644
index 00000000000..242f066e0d9
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_write_at_all_begin_ts.c.in
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_write_at_all_begin(FILE fh, OFFSET offset,
+                                       BUFFER_ASYNC x, COUNT count,
+                                       DATATYPE datatype)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_fh, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(c_fh, (MPI_Offset) *offset, OMPI_F2C_BOTTOM(buf),
+                          c_count, c_datatype);
+
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_write_at_all_end_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_write_at_all_end_f08.F90
deleted file mode 100644
index 087a03ea5fd..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_write_at_all_end_f08.F90
+++ /dev/null
@@ -1,27 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_write_at_all_end_f08(fh,buf,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Status
-   use :: ompi_mpifh_bindings, only : ompi_file_write_at_all_end_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   TYPE(MPI_Status), INTENT(OUT) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_write_at_all_end_f(fh%MPI_VAL,buf,status,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_write_at_all_end_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_write_at_all_end_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_write_at_all_end_ts.c.in
new file mode 100644
index 00000000000..3883cc5634b
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_write_at_all_end_ts.c.in
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_write_at_all_end(FILE fh, BUFFER_ASYNC x, STATUS_OUT status)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    OMPI_FORTRAN_STATUS_DECLARATION(c_status,c_status2)
+
+    OMPI_FORTRAN_STATUS_SET_POINTER(c_status,c_status2,status)
+
+    c_ierr = PMPI_File_write_at_all_end(c_fh, buf, c_status);
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    OMPI_FORTRAN_STATUS_RETURN(c_status,c_status2,status,c_ierr)
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_write_at_all_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_write_at_all_f08.F90
deleted file mode 100644
index 6973a334bae..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_write_at_all_f08.F90
+++ /dev/null
@@ -1,31 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_write_at_all_f08(fh,offset,buf,count,datatype,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Status, MPI_OFFSET_KIND
-   use :: ompi_mpifh_bindings, only : ompi_file_write_at_all_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   INTEGER(MPI_OFFSET_KIND), INTENT(IN) :: offset
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Status), INTENT(OUT) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_write_at_all_f(fh%MPI_VAL,offset,buf,count,&
-                                 datatype%MPI_VAL,status,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_write_at_all_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_write_at_all_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_write_at_all_ts.c.in
new file mode 100644
index 00000000000..4ecd1486336
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_write_at_all_ts.c.in
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_write_at_all(FILE fh, OFFSET offset,
+                                 BUFFER x, COUNT count,
+                                 DATATYPE datatype, STATUS_OUT status)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+    OMPI_FORTRAN_STATUS_DECLARATION(c_status,c_status2)
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_fh, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    OMPI_FORTRAN_STATUS_SET_POINTER(c_status,c_status2,status)
+
+    c_ierr = @INNER_CALL@(c_fh, (MPI_Offset) *offset, OMPI_F2C_BOTTOM(buf),
+                          c_count, c_datatype, c_status);
+
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    OMPI_FORTRAN_STATUS_RETURN(c_status,c_status2,status,c_ierr)
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_write_at_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_write_at_f08.F90
deleted file mode 100644
index 1ad8f65703e..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_write_at_f08.F90
+++ /dev/null
@@ -1,31 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_write_at_f08(fh,offset,buf,count,datatype,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Status, MPI_OFFSET_KIND
-   use :: ompi_mpifh_bindings, only : ompi_file_write_at_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   INTEGER(MPI_OFFSET_KIND), INTENT(IN) :: offset
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Status), INTENT(OUT) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_write_at_f(fh%MPI_VAL,offset,buf,count,&
-                             datatype%MPI_VAL,status,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_write_at_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_write_at_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_write_at_ts.c.in
new file mode 100644
index 00000000000..771cb58392f
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_write_at_ts.c.in
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_write_at(FILE fh, OFFSET offset,
+                             BUFFER x, COUNT count,
+                             DATATYPE datatype, STATUS_OUT status)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+    OMPI_FORTRAN_STATUS_DECLARATION(c_status,c_status2)
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_fh, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    OMPI_FORTRAN_STATUS_SET_POINTER(c_status,c_status2,status)
+
+    c_ierr = @INNER_CALL@(c_fh, (MPI_Offset) *offset, OMPI_F2C_BOTTOM(buf),
+                          c_count, c_datatype, c_status);
+
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    OMPI_FORTRAN_STATUS_RETURN(c_status,c_status2,status,c_ierr)
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_write_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_write_f08.F90
deleted file mode 100644
index 8dd3c49496d..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_write_f08.F90
+++ /dev/null
@@ -1,30 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_write_f08(fh,buf,count,datatype,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Status
-   use :: ompi_mpifh_bindings, only : ompi_file_write_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Status), INTENT(OUT) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_write_f(fh%MPI_VAL,buf,count,&
-                          datatype%MPI_VAL,status,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_write_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_write_ordered_begin_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_write_ordered_begin_f08.F90
deleted file mode 100644
index 0e1aab6f447..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_write_ordered_begin_f08.F90
+++ /dev/null
@@ -1,28 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_write_ordered_begin_f08(fh,buf,count,datatype,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype
-   use :: ompi_mpifh_bindings, only : ompi_file_write_ordered_begin_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_write_ordered_begin_f(fh%MPI_VAL,buf,count,datatype%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_write_ordered_begin_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_write_ordered_begin_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_write_ordered_begin_ts.c.in
new file mode 100644
index 00000000000..2c8483667f1
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_write_ordered_begin_ts.c.in
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_write_ordered_begin(FILE fh, BUFFER_ASYNC x,
+                                        COUNT count, DATATYPE datatype)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_fh, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(c_fh, OMPI_F2C_BOTTOM(buf), c_count, c_datatype);
+
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_write_ordered_end_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_write_ordered_end_f08.F90
deleted file mode 100644
index 7a531313270..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_write_ordered_end_f08.F90
+++ /dev/null
@@ -1,27 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_write_ordered_end_f08(fh,buf,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Status
-   use :: ompi_mpifh_bindings, only : ompi_file_write_ordered_end_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   TYPE(MPI_Status), INTENT(OUT) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_write_ordered_end_f(fh%MPI_VAL,buf,status,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_write_ordered_end_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_write_ordered_end_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_write_ordered_end_ts.c.in
new file mode 100644
index 00000000000..c1fe3feffda
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_write_ordered_end_ts.c.in
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_write_ordered_end(FILE fh, BUFFER_ASYNC x, STATUS_OUT status)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    OMPI_FORTRAN_STATUS_DECLARATION(c_status,c_status2)
+
+    OMPI_FORTRAN_STATUS_SET_POINTER(c_status,c_status2,status)
+
+    c_ierr = PMPI_File_write_ordered_end(c_fh, buf, c_status);
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    OMPI_FORTRAN_STATUS_RETURN(c_status,c_status2,status,c_ierr)
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_write_ordered_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_write_ordered_f08.F90
deleted file mode 100644
index 8087420e5ff..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_write_ordered_f08.F90
+++ /dev/null
@@ -1,30 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_write_ordered_f08(fh,buf,count,datatype,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Status
-   use :: ompi_mpifh_bindings, only : ompi_file_write_ordered_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Status), INTENT(OUT) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_write_ordered_f(fh%MPI_VAL,buf,count,&
-                                  datatype%MPI_VAL,status,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_write_ordered_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_write_ordered_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_write_ordered_ts.c.in
new file mode 100644
index 00000000000..2e67e4b142f
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_write_ordered_ts.c.in
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_write_ordered(FILE fh, BUFFER x, COUNT count,
+                                  DATATYPE datatype, STATUS_OUT status)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+    OMPI_FORTRAN_STATUS_DECLARATION(c_status,c_status2)
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_fh, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    OMPI_FORTRAN_STATUS_SET_POINTER(c_status,c_status2,status)
+
+    c_ierr = @INNER_CALL@(c_fh, OMPI_F2C_BOTTOM(buf), c_count, c_datatype,
+                          c_status);
+
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    OMPI_FORTRAN_STATUS_RETURN(c_status,c_status2,status,c_ierr)
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_write_shared_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/file_write_shared_f08.F90
deleted file mode 100644
index f2e4400dda9..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/file_write_shared_f08.F90
+++ /dev/null
@@ -1,30 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_File_write_shared_f08(fh,buf,count,datatype,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Status
-   use :: ompi_mpifh_bindings, only : ompi_file_write_shared_f
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Status), INTENT(OUT) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_file_write_shared_f(fh%MPI_VAL,buf,count,&
-                                 datatype%MPI_VAL,status,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_File_write_shared_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_write_shared_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_write_shared_ts.c.in
new file mode 100644
index 00000000000..62b5c80b456
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_write_shared_ts.c.in
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_write_shared(FILE fh, BUFFER x, COUNT count,
+                                 DATATYPE datatype, STATUS_OUT status)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+    OMPI_FORTRAN_STATUS_DECLARATION(c_status,c_status2)
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_fh, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    OMPI_FORTRAN_STATUS_SET_POINTER(c_status,c_status2,status)
+
+    c_ierr = @INNER_CALL@(c_fh, OMPI_F2C_BOTTOM(buf), c_count, c_datatype,
+                          c_status);
+
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    OMPI_FORTRAN_STATUS_RETURN(c_status,c_status2,status,c_ierr)
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/file_write_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/file_write_ts.c.in
new file mode 100644
index 00000000000..f2b97e4a55b
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/file_write_ts.c.in
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID file_write(FILE fh, BUFFER x, COUNT count,
+                          DATATYPE datatype, STATUS_OUT status)
+{
+    int c_ierr;
+    MPI_File c_fh = PMPI_File_f2c(*fh);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ __opal_attribute_unused__ c_count =  (@COUNT_TYPE@) *count;
+    OMPI_FORTRAN_STATUS_DECLARATION(c_status,c_status2)
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_fh, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    OMPI_FORTRAN_STATUS_SET_POINTER(c_status,c_status2,status)
+
+    c_ierr = @INNER_CALL@(c_fh, OMPI_F2C_BOTTOM(buf),
+                          OMPI_FINT_2_INT(*count),
+                          c_type, c_status);
+
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    OMPI_FORTRAN_STATUS_RETURN(c_status,c_status2,status,c_ierr)
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/free_mem_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/free_mem_f08.F90
deleted file mode 100644
index 02a4006e5d2..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/free_mem_f08.F90
+++ /dev/null
@@ -1,22 +0,0 @@
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All Rights reserved.
-! Copyright (c) 2015-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Free_mem_f08(base,ierror)
-   use :: ompi_mpifh_bindings, only : ompi_free_mem_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: base
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_free_mem_f(base,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Free_mem_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/free_mem_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/free_mem_ts.c.in
new file mode 100644
index 00000000000..deced5f5920
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/free_mem_ts.c.in
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID free_mem(BUFFER x)
+{
+    int c_ierr;
+    if (OMPI_CFI_IS_CONTIGUOUS(x)) {
+        c_ierr = PMPI_Free_mem(OMPI_CFI_BASE_ADDR(x));
+    } else {
+        c_ierr = MPI_ERR_BUFFER;
+        OMPI_ERRHANDLER_INVOKE(MPI_COMM_SELF, c_ierr, FUNC_NAME);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/gather_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/gather_f08.F90
deleted file mode 100644
index f238c103fbb..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/gather_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Gather_f08(sendbuf,sendcount,sendtype,recvbuf,recvcount,&
-                          recvtype,root,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   use :: ompi_mpifh_bindings, only : ompi_gather_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount, root
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_gather_f(sendbuf,sendcount,sendtype%MPI_VAL,recvbuf,recvcount,&
-                      recvtype%MPI_VAL,root,comm%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Gather_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/gather_init_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/gather_init_ts.c.in
new file mode 100644
index 00000000000..85ad3eb9bc4
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/gather_init_ts.c.in
@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID gather_init(BUFFER_ASYNC x1, COUNT sendcount, DATATYPE sendtype,
+                           BUFFER_ASYNC_OUT x2, COUNT recvcount, DATATYPE recvtype,
+                           RANK root, COMM comm, INFO info, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Comm c_comm;
+    MPI_Datatype c_sendtype, c_recvtype;
+    MPI_Datatype c_senddatatype = MPI_DATATYPE_NULL;
+    MPI_Info c_info;
+    MPI_Request c_request;
+    int c_root = OMPI_FINT_2_INT(*root);
+    void *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_sendcount = (@COUNT_TYPE@) *sendcount;
+    @COUNT_TYPE@ c_recvcount = (@COUNT_TYPE@) *recvcount;
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_sendtype = PMPI_Type_f2c(*sendtype);
+    c_recvtype = PMPI_Type_f2c(*recvtype);
+    c_info = PMPI_Info_f2c(*info);
+    if (OMPI_COMM_IS_INTER(c_comm)) {
+        if (MPI_ROOT == c_root) {
+            OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+                return;
+            }
+        } else if (MPI_PROC_NULL != c_root) {
+            OMPI_CFI_2_C(x1, c_sendcount, c_sendtype, c_senddatatype, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+                return;
+            }
+        }
+    } else {
+        if (ompi_comm_rank(c_comm) == c_root) {
+            OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+                return;
+            }
+        }
+        if (OMPI_IS_FORTRAN_IN_PLACE(sendbuf)) {
+            sendbuf = MPI_IN_PLACE;
+        } else {
+            OMPI_CFI_2_C(x1, c_sendcount, c_sendtype, c_senddatatype, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+                return;
+            }
+        }
+    }
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf, c_sendcount,
+                          c_senddatatype,
+                          recvbuf,
+                          c_recvcount,
+                          c_recvtype,
+                          c_root,
+                          c_comm, c_info, &c_request);
+    if (MPI_DATATYPE_NULL != c_senddatatype && c_senddatatype != c_sendtype) {
+        ompi_datatype_destroy(&c_senddatatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/gather_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/gather_ts.c.in
new file mode 100644
index 00000000000..c20b368ae53
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/gather_ts.c.in
@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID gather(BUFFER x1, COUNT sendcount, DATATYPE sendtype,
+                      BUFFER x2, COUNT recvcount, DATATYPE recvtype,
+                      RANK root, COMM comm)
+{
+    int c_root, c_ierr;
+    MPI_Comm c_comm;
+    MPI_Datatype c_senddatatype = NULL, c_sendtype = NULL;
+    void *sendbuf = OMPI_CFI_BASE_ADDR(x1);
+    @COUNT_TYPE@ c_sendcount =  (@COUNT_TYPE@)*sendcount;
+    @COUNT_TYPE@ c_recvcount =  (@COUNT_TYPE@)*recvcount;
+    MPI_Datatype c_recvtype = NULL;
+    void *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_root = OMPI_FINT_2_INT(*root);
+
+    if (OMPI_COMM_IS_INTER(c_comm)) {
+        if (MPI_ROOT == c_root) {
+            c_recvtype = PMPI_Type_f2c(*recvtype);
+            OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+                return;
+            }
+        } else if (MPI_PROC_NULL != c_root) {
+            c_sendtype = PMPI_Type_f2c(*sendtype);
+            OMPI_CFI_2_C(x1, c_sendcount, c_sendtype, c_senddatatype, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+                return;
+            }
+        }
+    } else {
+        if (ompi_comm_rank(c_comm) == c_root) {
+            c_recvtype = PMPI_Type_f2c(*recvtype);
+            OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+                return;
+            }
+        }
+        if (OMPI_IS_FORTRAN_IN_PLACE(sendbuf)) {
+            sendbuf = MPI_IN_PLACE;
+        } else {
+            c_sendtype = PMPI_Type_f2c(*sendtype);
+            OMPI_CFI_2_C(x1, c_sendcount, c_sendtype, c_senddatatype, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+                return;
+            }
+        }
+    }
+
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+
+    c_ierr = PMPI_Gather(sendbuf, c_sendcount,
+                        c_senddatatype, recvbuf,
+                        c_recvcount,
+                        c_recvtype,
+                        c_root,
+                        c_comm);
+    if (c_senddatatype != c_sendtype) {
+        ompi_datatype_destroy(&c_senddatatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/gatherv_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/gatherv_f08.F90
deleted file mode 100644
index b53fa2e00e7..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/gatherv_f08.F90
+++ /dev/null
@@ -1,33 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Gatherv_f08(sendbuf,sendcount,sendtype,recvbuf,recvcounts,&
-                           displs,recvtype,root,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   use :: ompi_mpifh_bindings, only : ompi_gatherv_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, root
-   INTEGER, INTENT(IN) :: recvcounts(*), displs(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_gatherv_f(sendbuf,sendcount,sendtype%MPI_VAL,recvbuf,recvcounts,&
-                       displs,recvtype%MPI_VAL,root,comm%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Gatherv_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/gatherv_init_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/gatherv_init_ts.c.in
new file mode 100644
index 00000000000..da4c954cebb
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/gatherv_init_ts.c.in
@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID gatherv_init(BUFFER_ASYNC x1, COUNT sendcount, DATATYPE sendtype,
+                            BUFFER_ASYNC_OUT x2, COUNT_ARRAY recvcounts, DISP_ARRAY displs,
+                            DATATYPE recvtype, RANK root, COMM comm,
+                            INFO info, REQUEST_OUT request)
+{
+    MPI_Comm c_comm;
+    MPI_Datatype c_sendtype, c_recvtype;
+    MPI_Datatype c_senddatatype = MPI_DATATYPE_NULL;
+    MPI_Info c_info;
+    MPI_Request c_request;
+    int idx = 0, c_ierr;
+    int c_root = OMPI_FINT_2_INT(*root);
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ *tmp_recvcounts = NULL;
+    @DISP_TYPE@ *tmp_displs = NULL;
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_sendtype = PMPI_Type_f2c(*sendtype);
+    c_recvtype = PMPI_Type_f2c(*recvtype);
+    c_info = PMPI_Info_f2c(*info);
+    @COUNT_TYPE@ c_sendcount = (@COUNT_TYPE@) *sendcount;
+
+    if (OMPI_COMM_IS_INTER(c_comm)) {
+        if (MPI_ROOT == c_root) {
+            int size = ompi_comm_remote_size(c_comm);
+            c_recvtype = PMPI_Type_f2c(*recvtype);
+            OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+                return;
+            }
+            OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(recvcounts, tmp_recvcounts, size);
+            OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(displs, tmp_displs, size);
+        } else if (MPI_PROC_NULL != c_root) {
+            c_sendtype = PMPI_Type_f2c(*sendtype);
+            OMPI_CFI_2_C(x1, c_sendcount, c_sendtype, c_senddatatype, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+                return;
+            }
+        }
+    } else {
+        if (ompi_comm_rank(c_comm) == c_root) {
+            int size = ompi_comm_size(c_comm);
+            c_recvtype = PMPI_Type_f2c(*recvtype);
+            OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+                return;
+            }
+            OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(recvcounts, tmp_recvcounts, size);
+            OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(displs, tmp_displs, size);
+        }
+        if (OMPI_IS_FORTRAN_IN_PLACE(sendbuf)) {
+            sendbuf = MPI_IN_PLACE;
+        } else {
+            c_sendtype = PMPI_Type_f2c(*sendtype);
+            OMPI_CFI_2_C(x1, c_sendcount, c_sendtype, c_senddatatype, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+                return;
+            }
+        }
+    }
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          c_sendcount,
+                          c_senddatatype,
+                          recvbuf,
+                          tmp_recvcounts,
+                          tmp_displs,
+                          c_recvtype,
+                          c_root,
+                          c_comm, c_info, &c_request);
+    if (MPI_DATATYPE_NULL != c_senddatatype && c_senddatatype != c_sendtype) {
+        ompi_datatype_destroy(&c_senddatatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(recvcounts, tmp_recvcounts, c_request, c_ierr, idx);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(displs, tmp_displs, c_request, c_ierr, idx);
+}
+
diff --git a/ompi/mpi/fortran/use-mpi-f08/gatherv_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/gatherv_ts.c.in
new file mode 100644
index 00000000000..bb4c827c4d5
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/gatherv_ts.c.in
@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID gatherv(BUFFER x1, COUNT sendcount, DATATYPE sendtype,
+                       BUFFER x2, COUNT_ARRAY recvcounts, DISP_ARRAY displs,
+                       DATATYPE recvtype, RANK root, COMM comm)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    int c_root = OMPI_FINT_2_INT(*root);
+    MPI_Datatype c_senddatatype = NULL, c_sendtype = NULL, c_recvtype = NULL;
+    @COUNT_TYPE@ c_sendcount = (@COUNT_TYPE@) *sendcount;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ *tmp_recvcounts = NULL;
+    @DISP_TYPE@ *tmp_displs = NULL;
+
+    if (OMPI_COMM_IS_INTER(c_comm)) {
+        if (MPI_ROOT == c_root) {
+            int size = ompi_comm_remote_size(c_comm);
+            c_recvtype = PMPI_Type_f2c(*recvtype);
+            OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+                return;
+            }
+            OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(recvcounts, tmp_recvcounts, size);
+            OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(displs, tmp_displs, size);
+        } else if (MPI_PROC_NULL != c_root) {
+            c_sendtype = PMPI_Type_f2c(*sendtype);
+            OMPI_CFI_2_C(x1, c_sendcount, c_sendtype, c_senddatatype, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+                return;
+            }
+        }
+    } else {
+        if (ompi_comm_rank(c_comm) == c_root) {
+            int size = ompi_comm_size(c_comm);
+            c_recvtype = PMPI_Type_f2c(*recvtype);
+            OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+                return;
+            }
+            OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(recvcounts, tmp_recvcounts, size);
+            OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(displs, tmp_displs, size);
+        }
+        if (OMPI_IS_FORTRAN_IN_PLACE(sendbuf)) {
+            sendbuf = MPI_IN_PLACE;
+        } else {
+            c_sendtype = PMPI_Type_f2c(*sendtype);
+            OMPI_CFI_2_C(x1, c_sendcount, c_sendtype, c_senddatatype, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+                return;
+            }
+        }
+    }
+
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf, c_sendcount,
+                          c_senddatatype, recvbuf,
+                          tmp_recvcounts,
+                          tmp_displs,
+                          c_recvtype,
+                          c_root,
+                          c_comm);
+
+    if (c_senddatatype != c_sendtype) {
+        ompi_datatype_destroy(&c_senddatatype);
+    }
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(recvcounts, tmp_recvcounts);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(displs, tmp_displs);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/get_accumulate_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/get_accumulate_f08.F90
deleted file mode 100644
index 583e4e00008..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/get_accumulate_f08.F90
+++ /dev/null
@@ -1,39 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2014 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Get_accumulate_f08(origin_addr,origin_count,origin_datatype,&
-                                  result_addr,result_count,result_datatype,&
-                                  target_rank,target_disp,target_count,    &
-                                  target_datatype,op,win,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Win, MPI_ADDRESS_KIND
-   use :: ompi_mpifh_bindings, only : ompi_get_accumulate_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: origin_addr
-   INTEGER, INTENT(IN) :: origin_count, result_count, target_rank, target_count
-   TYPE(MPI_Datatype), INTENT(IN) :: origin_datatype
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: result_addr
-   TYPE(MPI_Datatype), INTENT(IN) :: result_datatype
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
-   TYPE(MPI_Datatype), INTENT(IN) :: target_datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Win), INTENT(IN) :: win
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_get_accumulate_f(origin_addr,origin_count,origin_datatype%MPI_VAL,&
-                              result_addr,result_count,result_datatype%MPI_VAL,&
-                              target_rank,target_disp,target_count,target_datatype%MPI_VAL,&
-                              op%MPI_VAL,win%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Get_accumulate_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/get_accumulate_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/get_accumulate_ts.c.in
new file mode 100644
index 00000000000..3b3b00d39f0
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/get_accumulate_ts.c.in
@@ -0,0 +1,77 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2014      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID get_accumulate(BUFFER_ASYNC x1, COUNT origin_count,
+                              DATATYPE origin_datatype, BUFFER_ASYNC_OUT x2,
+                              COUNT result_count, DATATYPE result_datatype,
+                              RANK target_rank, DISP target_disp,
+                              COUNT target_count, DATATYPE target_datatype,
+                              OP op, WIN win)
+{
+    int c_ierr;
+    MPI_Datatype c_origin_datatype, c_origin_type = PMPI_Type_f2c(*origin_datatype);
+    MPI_Datatype c_result_datatype, c_result_type = PMPI_Type_f2c(*result_datatype);
+    MPI_Datatype c_target_datatype = PMPI_Type_f2c(*target_datatype);
+    MPI_Win c_win = PMPI_Win_f2c(*win);
+    MPI_Op c_op = PMPI_Op_f2c(*op);
+    char *origin_addr = OMPI_CFI_BASE_ADDR(x1);
+    @COUNT_TYPE@ c_origin_count = (@COUNT_TYPE@) *origin_count;
+    char *result_addr = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_result_count = (@COUNT_TYPE@) *result_count;
+
+    OMPI_CFI_2_C(x1, c_origin_count, c_origin_type, c_origin_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_win, c_ierr, FUNC_NAME)
+        return;
+    }
+    OMPI_CFI_2_C(x2, c_result_count, c_result_type, c_result_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        if (c_origin_datatype != c_origin_type) {
+            ompi_datatype_destroy(&c_origin_datatype);
+        }
+        OMPI_ERRHANDLER_INVOKE(c_win, c_ierr, FUNC_NAME)
+        return;
+    }
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(origin_addr),
+                          c_origin_count,
+                          c_origin_datatype,
+                          OMPI_F2C_BOTTOM(result_addr),
+                          c_result_count,
+                          c_result_datatype,
+                          OMPI_FINT_2_INT(*target_rank),
+                          *target_disp,
+                          OMPI_FINT_2_INT(*target_count),
+                          c_target_datatype, c_op, c_win);
+    if (c_origin_datatype != c_origin_type) {
+        ompi_datatype_destroy(&c_origin_datatype);
+    }
+    if (c_result_datatype != c_result_type) {
+        ompi_datatype_destroy(&c_result_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/get_address_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/get_address_f08.F90
deleted file mode 100644
index e17d200c1ed..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/get_address_f08.F90
+++ /dev/null
@@ -1,26 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Get_address_f08(location,address,ierror)
-   use :: mpi_f08_types, only : MPI_ADDRESS_KIND
-   use :: ompi_mpifh_bindings, only : ompi_get_address_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: location
-   INTEGER(MPI_ADDRESS_KIND), INTENT(OUT) :: address
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_get_address_f(location,address,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Get_address_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/get_address_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/get_address_ts.c.in
new file mode 100644
index 00000000000..d672e8c1139
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/get_address_ts.c.in
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID get_address(BUFFER_ASYNC x, AINT address)
+{
+    int c_ierr;
+    MPI_Aint c_address;
+
+    c_ierr = PMPI_Get_address(OMPI_F2C_BOTTOM(OMPI_CFI_BASE_ADDR(x)), &c_address);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *address = (MPI_Aint) c_address;
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/get_count.c.in b/ompi/mpi/fortran/use-mpi-f08/get_count.c.in
new file mode 100644
index 00000000000..ca1a3a60fe9
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/get_count.c.in
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID get_count(STATUS status, DATATYPE datatype, COUNT count)
+{
+    int c_ierr;
+    MPI_Datatype c_type = PMPI_Type_f2c(*datatype);
+    MPI_Status   c_status;
+    @COUNT_TYPE@ c_count;
+
+    if (OMPI_IS_FORTRAN_STATUS_IGNORE(status)) {
+        *count = OMPI_INT_2_FINT(0);
+        c_ierr = MPI_SUCCESS;
+    } else {
+        c_ierr = PMPI_Status_f2c(status, &c_status);
+
+        if (MPI_SUCCESS == c_ierr) {
+            c_ierr = @INNER_CALL@(&c_status, c_type,
+                                   &c_count);
+        }
+    }
+
+    if (MPI_SUCCESS == c_ierr) {
+        *count = (@COUNT_FINT_TYPE@)(c_count);
+    }
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/get_count_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/get_count_f08.F90
deleted file mode 100644
index 63333becea8..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/get_count_f08.F90
+++ /dev/null
@@ -1,25 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Get_count_f08(status,datatype,count,ierror)
-   use :: mpi_f08_types, only : MPI_Status, MPI_Datatype
-   use :: ompi_mpifh_bindings, only : ompi_get_count_f
-   implicit none
-   TYPE(MPI_Status), INTENT(IN) :: status
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER, INTENT(OUT) :: count
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_get_count_f(status,datatype%MPI_VAL,count,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Get_count_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/get_elements.c.in b/ompi/mpi/fortran/use-mpi-f08/get_elements.c.in
new file mode 100644
index 00000000000..174aa31798e
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/get_elements.c.in
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID get_elements(STATUS status, DATATYPE datatype, COUNT_OUT count)
+{
+    int c_ierr = MPI_SUCCESS;
+    MPI_Datatype c_type = PMPI_Type_f2c(*datatype);
+    MPI_Status   c_status;
+    @COUNT_TYPE@ c_count = 0;
+
+    if (!OMPI_IS_FORTRAN_STATUS_IGNORE(status)) {
+        c_ierr = PMPI_Status_f2c(status, &c_status);
+        if (MPI_SUCCESS == c_ierr) {
+            c_ierr = @INNER_CALL@(&c_status, c_type,
+                                      &c_count);
+        }
+    }
+
+    if (MPI_SUCCESS == c_ierr) {
+        *count = (@COUNT_FINT_TYPE@)(c_count);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/get_elements_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/get_elements_f08.F90
deleted file mode 100644
index 472e3089183..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/get_elements_f08.F90
+++ /dev/null
@@ -1,25 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Get_elements_f08(status,datatype,count,ierror)
-   use :: mpi_f08_types, only : MPI_Status, MPI_Datatype
-   use :: ompi_mpifh_bindings, only : ompi_get_elements_f
-   implicit none
-   TYPE(MPI_Status), INTENT(IN) :: status
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER, INTENT(OUT) :: count
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_get_elements_f(status,datatype%MPI_VAL,count,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Get_elements_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/get_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/get_f08.F90
deleted file mode 100644
index 38fa5337c35..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/get_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2018 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Get_f08(origin_addr,origin_count,origin_datatype,target_rank,&
-                       target_disp,target_count,target_datatype,win,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Win, MPI_ADDRESS_KIND
-   use :: ompi_mpifh_bindings, only : ompi_get_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: origin_addr
-   INTEGER, INTENT(IN) :: origin_count, target_rank, target_count
-   TYPE(MPI_Datatype), INTENT(IN) :: origin_datatype
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
-   TYPE(MPI_Datatype), INTENT(IN) :: target_datatype
-   TYPE(MPI_Win), INTENT(IN) :: win
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_get_f(origin_addr,origin_count,origin_datatype%MPI_VAL,target_rank,&
-                   target_disp,target_count,target_datatype%MPI_VAL,win%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Get_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/get_library_version_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/get_library_version_f08.F90
index 6b46062d1d4..925d39e44e2 100644
--- a/ompi/mpi/fortran/use-mpi-f08/get_library_version_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/get_library_version_f08.F90
@@ -12,13 +12,14 @@
 subroutine MPI_Get_library_version_f08(version,resultlen,ierror)
    use :: mpi_f08_types, only : MPI_MAX_LIBRARY_VERSION_STRING
    use :: ompi_mpifh_bindings, only : ompi_get_library_version_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
    implicit none
    character(len=MPI_MAX_LIBRARY_VERSION_STRING), intent(out) :: version
    integer, intent(out) :: resultlen
    integer, optional, intent(out) :: ierror
    integer :: c_ierror
 
-   call ompi_get_library_version_f(version,resultlen,c_ierror,len(version))
+   call ompi_get_library_version_f(version,resultlen,c_ierror,len(version,KIND=C_INT))
    if (present(ierror)) ierror = c_ierror
 
 end subroutine MPI_Get_library_version_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/get_processor_name_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/get_processor_name_f08.F90
index ec8d2141b35..395aff9ed49 100644
--- a/ompi/mpi/fortran/use-mpi-f08/get_processor_name_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/get_processor_name_f08.F90
@@ -11,13 +11,14 @@
 
 subroutine MPI_Get_processor_name_f08(name,resultlen,ierror)
    use :: ompi_mpifh_bindings, only : ompi_get_processor_name_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
    implicit none
    character(len=*), intent(out) :: name
    integer, intent(out) :: resultlen
    integer, optional, intent(out) :: ierror
    integer :: c_ierror
 
-   call ompi_get_processor_name_f(name,resultlen,c_ierror,len(name))
+   call ompi_get_processor_name_f(name,resultlen,c_ierror,len(name,KIND=C_INT))
    if (present(ierror)) ierror = c_ierror
 
 end subroutine MPI_Get_processor_name_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/get_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/get_ts.c.in
new file mode 100644
index 00000000000..c47dc0622b6
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/get_ts.c.in
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID get(BUFFER_ASYNC x, COUNT origin_count,
+                   DATATYPE origin_datatype, RANK target_rank,
+                   AINT target_disp, COUNT target_count,
+                   DATATYPE target_datatype, WIN win)
+{
+    int c_ierr;
+    MPI_Datatype c_origin_datatype, c_origin_type = PMPI_Type_f2c(*origin_datatype);
+    MPI_Datatype c_target_datatype = PMPI_Type_f2c(*target_datatype);
+    MPI_Win c_win = PMPI_Win_f2c(*win);
+    char *origin_addr = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_origin_count = (@COUNT_TYPE@) *origin_count;
+
+    OMPI_CFI_2_C(x, c_origin_count, c_origin_type, c_origin_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_win, c_ierr, FUNC_NAME)
+        return;
+    }
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(origin_addr),
+                          c_origin_count,
+                          c_origin_datatype,
+                          OMPI_FINT_2_INT(*target_rank),
+                          *target_disp,
+                          OMPI_FINT_2_INT(*target_count),
+                          c_target_datatype, c_win);
+    if (c_origin_datatype != c_origin_type) {
+        ompi_datatype_destroy(&c_origin_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/group_from_session_pset_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/group_from_session_pset_f08.F90
new file mode 100644
index 00000000000..ce4ff407da9
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/group_from_session_pset_f08.F90
@@ -0,0 +1,30 @@
+! -*- f90 -*-
+!
+! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2009-2013 Los Alamos National Security, LLC.
+!                         All rights reserved.
+! Copyright (c) 2018      Research Organization for Information Science
+!                         and Technology (RIST).  All rights reserved.
+! Copyright (c) 2019-2021 Triad National Security, LLC. All rights
+!                         reserved.
+! $COPYRIGHT$
+
+#include "ompi/mpi/fortran/configure-fortran-output.h"
+#include "mpi-f08-rename.h"
+
+subroutine MPI_Group_from_session_pset_f08(session, pset_name, newgroup, ierror)
+   use :: mpi_f08_types, only : MPI_Session, MPI_Group
+   use :: ompi_mpifh_bindings, only : ompi_group_from_session_pset_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
+   implicit none
+   TYPE(MPI_Session), INTENT(IN) :: session
+   CHARACTER(LEN=*), INTENT(IN) :: pset_name
+   TYPE(MPI_Group), INTENT(OUT) :: newgroup
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+   integer :: c_ierror
+
+   call ompi_group_from_session_pset_f(session%MPI_VAL, pset_name, newgroup%MPI_VAL, c_ierror, len(pset_name,KIND=C_INT))
+   if (present(ierror)) ierror = c_ierror
+
+end subroutine MPI_Group_from_session_pset_f08
+
diff --git a/ompi/mpi/fortran/use-mpi-f08/iallgather_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/iallgather_f08.F90
deleted file mode 100644
index e6eb5d07ecc..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/iallgather_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Iallgather_f08(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_iallgather_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_iallgather_f(sendbuf,sendcount,sendtype%MPI_VAL,&
-                          recvbuf,recvcount,recvtype%MPI_VAL,comm%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Iallgather_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/iallgather_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/iallgather_ts.c.in
new file mode 100644
index 00000000000..464d0651328
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/iallgather_ts.c.in
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID iallgather(BUFFER_ASYNC x1, COUNT sendcount, DATATYPE sendtype,
+                          BUFFER_ASYNC_OUT x2, COUNT recvcount, DATATYPE recvtype,
+                          COMM comm, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Request c_req;
+    MPI_Datatype c_sendtype = NULL, c_senddatatype = NULL;
+    void *sendbuf = OMPI_CFI_BASE_ADDR(x1);
+    @COUNT_TYPE@ c_sendcount = (@COUNT_TYPE@) *sendcount;
+    MPI_Datatype c_recvtype = PMPI_Type_f2c(*recvtype);
+    void *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_recvcount = (@COUNT_TYPE@) *recvcount;
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    if (OMPI_COMM_IS_INTER(c_comm) || !OMPI_IS_FORTRAN_IN_PLACE(sendbuf)) {
+        c_sendtype = PMPI_Type_f2c(*sendtype);
+        OMPI_CFI_2_C(x1, c_sendcount, c_sendtype, c_senddatatype, c_ierr);
+        if (MPI_SUCCESS != c_ierr) {
+            if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+            OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+            return;
+        }
+    } else {
+        sendbuf = MPI_IN_PLACE;
+    }
+
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf, c_sendcount, c_sendtype,
+                          recvbuf, c_recvcount, c_recvtype,
+                          c_comm, &c_req);
+
+    if (c_senddatatype != c_sendtype && c_senddatatype != NULL) {
+        ompi_datatype_destroy(&c_senddatatype);
+    }
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_req);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/iallgatherv_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/iallgatherv_f08.F90
deleted file mode 100644
index e9b4b2af7de..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/iallgatherv_f08.F90
+++ /dev/null
@@ -1,34 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Iallgatherv_f08(sendbuf,sendcount,sendtype,recvbuf,recvcounts,&
-                              displs,recvtype,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_iallgatherv_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: recvcounts(*), displs(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_iallgatherv_f(sendbuf,sendcount,sendtype%MPI_VAL,recvbuf,recvcounts,&
-                          displs,recvtype%MPI_VAL,comm%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Iallgatherv_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/iallgatherv_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/iallgatherv_ts.c.in
new file mode 100644
index 00000000000..0323d69bb33
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/iallgatherv_ts.c.in
@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID iallgatherv(BUFFER_ASYNC x1, COUNT sendcount, DATATYPE sendtype,
+                           BUFFER_ASYNC_OUT x2, COUNT_ARRAY recvcounts, DISP_ARRAY displs,
+                           DATATYPE recvtype, COMM comm, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    @COUNT_TYPE@ c_sendcount = (@COUNT_TYPE@)*sendcount;
+    MPI_Datatype c_sendtype = NULL, c_senddatatype = NULL;
+    MPI_Datatype c_recvtype = PMPI_Type_f2c(*recvtype);
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    MPI_Request c_request;
+    int size;
+    @COUNT_TYPE@ *tmp_recvcounts = NULL;
+    @DISP_TYPE@ *tmp_displs = NULL;
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+    if (OMPI_COMM_IS_INTER(c_comm)) {
+        size = ompi_comm_remote_size(c_comm);
+        c_sendtype = PMPI_Type_f2c(*sendtype);
+        OMPI_CFI_2_C(x1, c_sendcount, c_sendtype, c_senddatatype, c_ierr);
+        if (MPI_SUCCESS != c_ierr) {
+            if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+            OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+            return;
+        }
+    } else {
+        size = ompi_comm_size(c_comm);
+        if (OMPI_IS_FORTRAN_IN_PLACE(sendbuf)) {
+            sendbuf = MPI_IN_PLACE;
+        } else {
+            c_sendtype = PMPI_Type_f2c(*sendtype);
+            OMPI_CFI_2_C(x1, c_sendcount, c_sendtype, c_senddatatype, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+                return;
+            }
+        }
+    }
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(recvcounts, tmp_recvcounts, size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(displs, tmp_displs, size);
+
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf, c_sendcount, c_sendtype,
+                          recvbuf,
+                          tmp_recvcounts,
+                          tmp_displs,
+                          c_recvtype,
+                          c_comm, &c_request);
+
+    if (c_senddatatype != c_sendtype && NULL != c_senddatatype) {
+        ompi_datatype_destroy(&c_senddatatype);
+    }
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(recvcounts, tmp_recvcounts);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(displs, tmp_displs);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/iallreduce_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/iallreduce_f08.F90
deleted file mode 100644
index e1319a59aef..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/iallreduce_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Iallreduce_f08(sendbuf,recvbuf,count,datatype,op,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_iallreduce_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_iallreduce_f(sendbuf,recvbuf,count,datatype%MPI_VAL,&
-                         op%MPI_VAL,comm%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Iallreduce_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/iallreduce_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/iallreduce_ts.c.in
new file mode 100644
index 00000000000..5f803ada126
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/iallreduce_ts.c.in
@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID iallreduce(BUFFER_ASYNC x1, BUFFER_ASYNC_OUT x2, COUNT count,
+                          DATATYPE datatype, OP op, COMM comm,
+                          REQUEST_OUT request)
+{
+    int c_ierr;
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@)*count;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype c_type;
+    MPI_Request c_request;
+    MPI_Op c_op;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+    c_type = PMPI_Type_f2c(*datatype);
+    c_op = PMPI_Op_f2c(*op);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf, recvbuf,
+                          c_count,
+                          c_type, c_op, c_comm, &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/ialltoall_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/ialltoall_f08.F90
deleted file mode 100644
index 7a4963ae108..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/ialltoall_f08.F90
+++ /dev/null
@@ -1,33 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Ialltoall_f08(sendbuf,sendcount,sendtype,recvbuf,&
-                             recvcount,recvtype,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_ialltoall_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_ialltoall_f(sendbuf,sendcount,sendtype%MPI_VAL,recvbuf,&
-                         recvcount,recvtype%MPI_VAL,comm%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Ialltoall_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/ialltoall_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/ialltoall_ts.c.in
new file mode 100644
index 00000000000..1a9c39a8725
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/ialltoall_ts.c.in
@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID ialltoall(BUFFER_ASYNC x1, COUNT sendcount, DATATYPE sendtype,
+                         BUFFER_ASYNC_OUT x2, COUNT recvcount, DATATYPE recvtype,
+                         COMM comm, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype c_sendtype = NULL, c_recvtype = PMPI_Type_f2c(*recvtype);
+    void *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_sendcount = 0, c_recvcount = (@COUNT_TYPE@)*recvcount;
+    MPI_Request c_request;
+
+    if (OMPI_COMM_IS_INTER(c_comm) || !OMPI_IS_FORTRAN_IN_PLACE(sendbuf)) {
+        c_sendtype = PMPI_Type_f2c(*sendtype);
+        c_sendcount = (@COUNT_TYPE@)*sendcount;
+        OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+        if (MPI_SUCCESS != c_ierr) {
+            if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+            OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+            return;
+        }
+    } else {
+        sendbuf = MPI_IN_PLACE;
+    }
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          c_sendcount,
+                          c_sendtype,
+                          recvbuf,
+                          c_recvcount,
+                          c_recvtype, c_comm, &c_request);
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/ialltoallv_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/ialltoallv_f08.F90
deleted file mode 100644
index c5b91bc499b..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/ialltoallv_f08.F90
+++ /dev/null
@@ -1,33 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Ialltoallv_f08(sendbuf,sendcounts,sdispls,sendtype,recvbuf,&
-                             recvcounts,rdispls,recvtype,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_ialltoallv_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_ialltoallv_f(sendbuf,sendcounts,sdispls,sendtype%MPI_VAL,&
-                         recvbuf,recvcounts,rdispls,recvtype%MPI_VAL,comm%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Ialltoallv_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/ialltoallv_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/ialltoallv_ts.c.in
new file mode 100644
index 00000000000..1e6b401f934
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/ialltoallv_ts.c.in
@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID ialltoallv(BUFFER_ASYNC x1, COUNT_ARRAY sendcounts, DISP_ARRAY sdispls,
+                          DATATYPE sendtype, BUFFER_ASYNC_OUT x2, COUNT_ARRAY recvcounts,
+                          DISP_ARRAY rdispls, DATATYPE recvtype,
+                          COMM comm, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    MPI_Datatype c_sendtype = NULL, c_recvtype = PMPI_Type_f2c(*recvtype);
+    MPI_Request c_request;
+    int size = OMPI_COMM_IS_INTER(c_comm)?ompi_comm_remote_size(c_comm):ompi_comm_size(c_comm);
+    @COUNT_TYPE@ *tmp_sendcounts = NULL;
+    @DISP_TYPE@ *tmp_sdispls = NULL;
+    @COUNT_TYPE@ *tmp_recvcounts = NULL;
+    @DISP_TYPE@ *tmp_rdispls = NULL;
+
+    if (OMPI_COMM_IS_INTER(c_comm) || !OMPI_IS_FORTRAN_IN_PLACE(sendbuf)) {
+        c_sendtype = PMPI_Type_f2c(*sendtype);
+        OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+        if (MPI_SUCCESS != c_ierr) {
+            if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+            OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+            return;
+        }
+        OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(sendcounts, tmp_sendcounts, size);
+        OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(sdispls, tmp_sdispls, size);
+    } else {
+        sendbuf = MPI_IN_PLACE;
+    }
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(recvcounts, tmp_recvcounts, size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(rdispls, tmp_rdispls, size);
+
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          tmp_sendcounts,
+                          tmp_sdispls,
+                          c_sendtype,
+                          recvbuf,
+                          tmp_recvcounts,
+                          tmp_rdispls,
+                          c_recvtype, c_comm, &c_request);
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(sendcounts, tmp_sendcounts);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(sdispls, tmp_sdispls);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(recvcounts, tmp_recvcounts);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(rdispls, tmp_rdispls);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/ialltoallw_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/ialltoallw_f08.F90
deleted file mode 100644
index 696a828e24e..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/ialltoallw_f08.F90
+++ /dev/null
@@ -1,43 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2013 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Ialltoallw_f08(sendbuf,sendcounts,sdispls,sendtypes,&
-                             recvbuf,recvcounts,rdispls,recvtypes,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_ialltoallw_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
-   TYPE(MPI_Datatype), INTENT(IN), ASYNCHRONOUS :: sendtypes(*), recvtypes(*)
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   ! Note that we pass a scalar here for both the sendtypes and
-   ! recvtypes arguments, even though the real Alltoallw function
-   ! expects an array of integers.  This is a hack: we know that
-   ! [send|recv]types(1)%MPI_VAL will pass the address of the first
-   ! integer in the array of Type(MPI_Datatype) derived types.  And
-   ! since Type(MPI_Datatype) are exactly memory-equivalent to a
-   ! single INTEGER, passing the address of the first one is the same
-   ! as passing the address to an array of integers.  To be clear: the
-   ! back-end ompi_alltoallw_f is expecting a pointer to an array of
-   ! integers.  So it all works out (but is a hack :-\ ).
-   call ompi_ialltoallw_f(sendbuf,sendcounts,sdispls,sendtypes(1)%MPI_VAL,&
-                         recvbuf,recvcounts,rdispls,recvtypes(1)%MPI_VAL,comm%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Ialltoallw_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/ialltoallw_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/ialltoallw_ts.c.in
new file mode 100644
index 00000000000..dbeebf41910
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/ialltoallw_ts.c.in
@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID ialltoallw(BUFFER_ASYNC x1, COUNT_ARRAY sendcounts,
+                          DISP_ARRAY sdispls, DATATYPE_ARRAY sendtypes,
+                          BUFFER_ASYNC_OUT x2, COUNT_ARRAY recvcounts,
+                          DISP_ARRAY rdispls, DATATYPE_ARRAY recvtypes,
+                          COMM comm, REQUEST_OUT request)
+{
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype *c_sendtypes = NULL, *c_recvtypes;
+    MPI_Request c_request;
+    int size, c_ierr;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ *tmp_sendcounts = NULL;
+    @DISP_TYPE@ *tmp_sdispls = NULL;
+    @COUNT_TYPE@ *tmp_recvcounts = NULL;
+    @DISP_TYPE@ *tmp_rdispls = NULL;
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+    size = OMPI_COMM_IS_INTER(c_comm)?ompi_comm_remote_size(c_comm):ompi_comm_size(c_comm);
+
+    if (!OMPI_IS_FORTRAN_IN_PLACE(sendbuf)) {
+        c_sendtypes = (MPI_Datatype *) malloc(size * sizeof(MPI_Datatype));
+        OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(sendcounts, tmp_sendcounts, size);
+        OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(sdispls, tmp_sdispls, size);
+        for (int i=0; i<size; i++) {
+            c_sendtypes[i] = PMPI_Type_f2c(sendtypes[i]);
+        }
+    }
+
+    c_recvtypes = (MPI_Datatype *) malloc(size * sizeof(MPI_Datatype));
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(recvcounts, tmp_recvcounts, size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(rdispls, tmp_rdispls, size);
+    for (int i=0; i<size; i++) {
+        c_recvtypes[i] = PMPI_Type_f2c(recvtypes[i]);
+    }
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          tmp_sendcounts,
+                          tmp_sdispls,
+                          c_sendtypes,
+                          recvbuf,
+                          tmp_recvcounts,
+                          tmp_rdispls,
+                          c_recvtypes, c_comm, &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(sendcounts, tmp_sendcounts);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(sdispls, tmp_sdispls);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(recvcounts, tmp_recvcounts);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(rdispls, tmp_rdispls);
+    if (NULL != c_sendtypes) {
+        free(c_sendtypes);
+    }
+    free(c_recvtypes);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/ibcast_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/ibcast_f08.F90
deleted file mode 100644
index 9d8492e69a7..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/ibcast_f08.F90
+++ /dev/null
@@ -1,30 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Ibcast_f08(buffer,count,datatype,root,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_ibcast_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: buffer
-   INTEGER, INTENT(IN) :: count, root
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_ibcast_f(buffer,count,datatype%MPI_VAL,root,comm%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Ibcast_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/ibcast_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/ibcast_ts.c.in
new file mode 100644
index 00000000000..abd5717f8b5
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/ibcast_ts.c.in
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID ibcast(BUFFER_ASYNC_OUT x, COUNT count, DATATYPE datatype,
+                      RANK root, COMM comm, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype c_datatype = NULL, c_type = NULL;
+    int c_root = OMPI_FINT_2_INT(*root);
+    void *buffer = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = 0;
+    MPI_Request c_req;
+
+    if (OMPI_COMM_IS_INTRA(c_comm) || MPI_PROC_NULL != c_root) {
+        c_type = PMPI_Type_f2c(*datatype);
+        c_count = (@COUNT_TYPE@) *count;
+        OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+        if (MPI_SUCCESS != c_ierr) {
+            if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+            OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+            return;
+        }
+    }
+
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(buffer),
+                          c_count,
+                          c_datatype,
+                          c_root,
+                          c_comm,
+                          &c_req);
+
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_req);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/ibsend_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/ibsend_f08.F90
deleted file mode 100644
index f89926bed6e..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/ibsend_f08.F90
+++ /dev/null
@@ -1,29 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Ibsend_f08(buf,count,datatype,dest,tag,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_ibsend_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count, dest, tag
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_ibsend_f(buf,count,datatype%MPI_VAL,dest,tag,comm%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Ibsend_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/ibsend_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/ibsend_ts.c.in
new file mode 100644
index 00000000000..4a03691e5c9
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/ibsend_ts.c.in
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID ibsend(BUFFER_ASYNC x, COUNT count, DATATYPE datatype,
+                      RANK dest, TAG tag, COMM comm,
+                      REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    MPI_Request c_req;
+    MPI_Comm c_comm = PMPI_Comm_f2c (*comm);
+
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(buf), c_count,
+                          c_datatype, OMPI_FINT_2_INT(*dest),
+                          OMPI_FINT_2_INT(*tag),
+                          c_comm, &c_req);
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_req);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/iexscan_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/iexscan_f08.F90
deleted file mode 100644
index aee9b90b8cf..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/iexscan_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Iexscan_f08(sendbuf,recvbuf,count,datatype,op,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_iexscan_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_iexscan_f(sendbuf,recvbuf,count,datatype%MPI_VAL,&
-                      op%MPI_VAL,comm%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Iexscan_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/iexscan_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/iexscan_ts.c.in
new file mode 100644
index 00000000000..aa87c6f080d
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/iexscan_ts.c.in
@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID iexscan(BUFFER_ASYNC x1, BUFFER_ASYNC_OUT x2, COUNT count,
+                       DATATYPE datatype, OP op, COMM comm,
+                       REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype c_type;
+    MPI_Request c_request;
+    MPI_Op c_op;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@)*count;
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+    c_type = PMPI_Type_f2c(*datatype);
+    c_op = PMPI_Op_f2c(*op);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM (sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM (recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf, recvbuf,
+                          c_count,
+                          c_type, c_op, c_comm, &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/igather_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/igather_f08.F90
deleted file mode 100644
index e86dbe4e699..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/igather_f08.F90
+++ /dev/null
@@ -1,33 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Igather_f08(sendbuf,sendcount,sendtype,recvbuf,recvcount,&
-                          recvtype,root,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_igather_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount, root
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_igather_f(sendbuf,sendcount,sendtype%MPI_VAL,recvbuf,recvcount,&
-                      recvtype%MPI_VAL,root,comm%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Igather_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/igather_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/igather_ts.c.in
new file mode 100644
index 00000000000..d5df09345cf
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/igather_ts.c.in
@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID igather(BUFFER_ASYNC x1, COUNT sendcount, DATATYPE sendtype,
+                       BUFFER_ASYNC_OUT x2, COUNT recvcount, DATATYPE recvtype,
+                       RANK root, COMM comm, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    int c_root = OMPI_FINT_2_INT(*root);
+    MPI_Datatype c_senddatatype = NULL, c_sendtype = NULL, c_recvtype = NULL;
+    @COUNT_TYPE@ c_sendcount = 0, c_recvcount = 0;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    MPI_Request c_request;
+
+    if (OMPI_COMM_IS_INTER(c_comm)) {
+        if (MPI_ROOT == c_root) {
+            c_recvtype = PMPI_Type_f2c(*recvtype);
+            c_recvcount = (@COUNT_TYPE@) *recvcount;
+            OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+                return;
+            }
+        } else if (MPI_PROC_NULL != c_root) {
+            c_sendtype = PMPI_Type_f2c(*sendtype);
+            c_sendcount = (@COUNT_TYPE@) *sendcount;
+            OMPI_CFI_2_C(x1, c_sendcount, c_sendtype, c_senddatatype, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+                return;
+            }
+        }
+    } else {
+        if (ompi_comm_rank(c_comm) == c_root) {
+            c_recvtype = PMPI_Type_f2c(*recvtype);
+            c_recvcount = (@COUNT_TYPE@) *recvcount;
+            OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+                return;
+            }
+        }
+        if (OMPI_IS_FORTRAN_IN_PLACE(sendbuf)) {
+            sendbuf = MPI_IN_PLACE;
+        } else {
+            c_sendtype = PMPI_Type_f2c(*sendtype);
+            c_sendcount = (@COUNT_TYPE@) *sendcount;
+            OMPI_CFI_2_C(x1, c_sendcount, c_sendtype, c_senddatatype, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+                return;
+            }
+        }
+    }
+
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf, c_sendcount,
+                          c_senddatatype, recvbuf,
+                          c_recvcount,
+                          c_recvtype,
+                          c_root,
+                          c_comm, &c_request);
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/igatherv_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/igatherv_f08.F90
deleted file mode 100644
index 65577d4559c..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/igatherv_f08.F90
+++ /dev/null
@@ -1,34 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Igatherv_f08(sendbuf,sendcount,sendtype,recvbuf,recvcounts,&
-                           displs,recvtype,root,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_igatherv_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, root
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: recvcounts(*), displs(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_igatherv_f(sendbuf,sendcount,sendtype%MPI_VAL,recvbuf,recvcounts,&
-                       displs,recvtype%MPI_VAL,root,comm%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Igatherv_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/igatherv_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/igatherv_ts.c.in
new file mode 100644
index 00000000000..f5f51bbd023
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/igatherv_ts.c.in
@@ -0,0 +1,105 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID igatherv(BUFFER_ASYNC x1, COUNT sendcount, DATATYPE sendtype,
+                        BUFFER_ASYNC_OUT x2, COUNT_ARRAY recvcounts, DISP_ARRAY displs,
+                        DATATYPE recvtype, RANK root, COMM comm,
+                        REQUEST_OUT request)
+{
+
+    int c_ierr, idx=0;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    int c_root = OMPI_FINT_2_INT(*root);
+    MPI_Datatype c_senddatatype = NULL, c_sendtype = NULL, c_recvtype = NULL;
+    @COUNT_TYPE@ c_sendcount = 0;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    MPI_Request c_request;
+    @COUNT_TYPE@ *tmp_recvcounts = NULL;
+    @DISP_TYPE@ *tmp_displs = NULL;
+
+    if (OMPI_COMM_IS_INTER(c_comm)) {
+        if (MPI_ROOT == c_root) {
+            int size = ompi_comm_remote_size(c_comm);
+            c_recvtype = PMPI_Type_f2c(*recvtype);
+            OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+                return;
+            }
+            OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(recvcounts, tmp_recvcounts, size);
+            OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(displs, tmp_displs, size);
+        } else if (MPI_PROC_NULL != c_root) {
+            c_sendtype = PMPI_Type_f2c(*sendtype);
+            c_sendcount = (@COUNT_TYPE@) *sendcount;
+            OMPI_CFI_2_C(x1, c_sendcount, c_sendtype, c_senddatatype, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+                return;
+            }
+        }
+    } else {
+        if (ompi_comm_rank(c_comm) == c_root) {
+            int size = ompi_comm_size(c_comm);
+            c_recvtype = PMPI_Type_f2c(*recvtype);
+            OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+                return;
+            }
+            OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(recvcounts, tmp_recvcounts, size);
+            OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(displs, tmp_displs, size);
+        }
+        if (OMPI_IS_FORTRAN_IN_PLACE(sendbuf)) {
+            sendbuf = MPI_IN_PLACE;
+        } else {
+            c_sendtype = PMPI_Type_f2c(*sendtype);
+            c_sendcount = (@COUNT_TYPE@) *sendcount;
+            OMPI_CFI_2_C(x1, c_sendcount, c_sendtype, c_senddatatype, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+                return;
+            }
+        }
+    }
+
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf, c_sendcount,
+                          c_senddatatype, recvbuf,
+                          tmp_recvcounts,
+                          tmp_displs,
+                          c_recvtype,
+                          c_root,
+                          c_comm, &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(recvcounts, tmp_recvcounts, c_request, c_ierr, idx);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(displs, tmp_displs, c_request, c_ierr, idx);
+
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/improbe_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/improbe_f08.F90
index 4a5b870464f..e1039819975 100644
--- a/ompi/mpi/fortran/use-mpi-f08/improbe_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/improbe_f08.F90
@@ -17,7 +17,7 @@ subroutine MPI_Improbe_f08(source,tag,comm,flag,message,status,ierror)
    TYPE(MPI_Comm), INTENT(IN) :: comm
    LOGICAL, INTENT(OUT) :: flag
    TYPE(MPI_Message), INTENT(OUT) :: message
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   TYPE(MPI_Status) :: status
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
    integer :: c_ierror
 
diff --git a/ompi/mpi/fortran/use-mpi-f08/imrecv_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/imrecv_f08.F90
deleted file mode 100644
index 8242465a83f..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/imrecv_f08.F90
+++ /dev/null
@@ -1,29 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Imrecv_f08(buf,count,datatype,message,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Message, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_imrecv_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Message), INTENT(INOUT) :: message
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_imrecv_f(buf,count,datatype%MPI_VAL,message%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Imrecv_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/imrecv_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/imrecv_ts.c.in
new file mode 100644
index 00000000000..408f09aface
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/imrecv_ts.c.in
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID imrecv(BUFFER_ASYNC_OUT x, COUNT count, DATATYPE datatype,
+                      MESSAGE_INOUT message, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    MPI_Request c_req;
+    MPI_Message c_message;
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+
+    c_message = PMPI_Message_f2c(*message);
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(MPI_COMM_SELF, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(buf), c_count,
+                          c_datatype, &c_message, &c_req);
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+       *request = PMPI_Request_c2f(c_req);
+       /* message is an INOUT, and may be updated by the recv */
+       *message = PMPI_Message_c2f(c_message);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/ineighbor_allgather_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/ineighbor_allgather_f08.F90
deleted file mode 100644
index 2db49aff9f4..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/ineighbor_allgather_f08.F90
+++ /dev/null
@@ -1,33 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2013 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Ineighbor_allgather_f08(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_ineighbor_allgather_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_ineighbor_allgather_f(sendbuf,sendcount,sendtype%MPI_VAL,&
-                                   recvbuf,recvcount,recvtype%MPI_VAL,comm%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Ineighbor_allgather_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/ineighbor_allgather_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/ineighbor_allgather_ts.c.in
new file mode 100644
index 00000000000..a8f0584b659
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/ineighbor_allgather_ts.c.in
@@ -0,0 +1,71 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID ineighbor_allgather(BUFFER_ASYNC x1, COUNT sendcount, DATATYPE sendtype,
+                                   BUFFER_ASYNC_OUT x2, COUNT recvcount, DATATYPE recvtype,
+                                   COMM comm, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    MPI_Datatype __opal_attribute_unused__  c_senddatatype;
+    MPI_Datatype c_sendtype = PMPI_Type_f2c(*sendtype);
+    @COUNT_TYPE@ c_sendcount = (@COUNT_TYPE@) *sendcount;
+    @COUNT_TYPE@ c_recvcount = (@COUNT_TYPE@) *recvcount;
+    MPI_Datatype c_recvtype = PMPI_Type_f2c(*recvtype);
+    MPI_Request c_req;
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+    OMPI_CFI_2_C(x1, c_sendcount, c_sendtype, c_senddatatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    c_sendtype = PMPI_Type_f2c(*sendtype);
+    c_recvtype = PMPI_Type_f2c(*recvtype);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          c_sendcount,
+                          c_sendtype,
+                          recvbuf,
+                          c_recvcount,
+                          c_recvtype, c_comm, &c_req);
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_req);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/ineighbor_allgatherv_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/ineighbor_allgatherv_f08.F90
deleted file mode 100644
index f87252202bb..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/ineighbor_allgatherv_f08.F90
+++ /dev/null
@@ -1,35 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2013 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Ineighbor_allgatherv_f08(sendbuf,sendcount,sendtype,recvbuf,recvcounts,&
-                                        displs,recvtype,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_ineighbor_allgatherv_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount
-   INTEGER, INTENT(IN) :: recvcounts(*), displs(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_ineighbor_allgatherv_f(sendbuf,sendcount,sendtype%MPI_VAL,recvbuf,recvcounts,&
-                                    displs,recvtype%MPI_VAL,comm%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Ineighbor_allgatherv_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/ineighbor_allgatherv_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/ineighbor_allgatherv_ts.c.in
new file mode 100644
index 00000000000..6cffe2c3007
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/ineighbor_allgatherv_ts.c.in
@@ -0,0 +1,78 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID ineighbor_allgatherv(BUFFER_ASYNC x1, COUNT sendcount, DATATYPE sendtype,
+                                    BUFFER_ASYNC_OUT x2, COUNT_ARRAY recvcounts, DISP_ARRAY displs,
+                                    DATATYPE recvtype, COMM comm, REQUEST_OUT request)
+{
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype  __opal_attribute_unused__ c_senddatatype;
+    MPI_Datatype c_sendtype = PMPI_Type_f2c(*sendtype);
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_sendcount = (@COUNT_TYPE@) *sendcount;
+    MPI_Datatype c_recvtype = PMPI_Type_f2c(*recvtype);
+    MPI_Request c_request;
+    int size, c_ierr;
+    @COUNT_TYPE@ *tmp_recvcounts = NULL;
+    @DISP_TYPE@ *tmp_displs = NULL;
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+    OMPI_CFI_2_C(x1, c_sendcount, c_sendtype, c_senddatatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+    c_sendtype = PMPI_Type_f2c(*sendtype);
+    c_recvtype = PMPI_Type_f2c(*recvtype);
+
+    PMPI_Comm_size(c_comm, &size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(recvcounts, tmp_recvcounts, size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(displs, tmp_displs, size);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          c_sendcount,
+                          c_sendtype,
+                          recvbuf,
+                          tmp_recvcounts,
+                          tmp_displs,
+                          c_recvtype, c_comm, &c_request);
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(recvcounts, tmp_recvcounts);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(displs, tmp_displs);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/ineighbor_alltoall_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/ineighbor_alltoall_f08.F90
deleted file mode 100644
index 71e59019e90..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/ineighbor_alltoall_f08.F90
+++ /dev/null
@@ -1,34 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2013 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Ineighbor_alltoall_f08(sendbuf,sendcount,sendtype,recvbuf,&
-                                      recvcount,recvtype,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_ineighbor_alltoall_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_ineighbor_alltoall_f(sendbuf,sendcount,sendtype%MPI_VAL,recvbuf,&
-                                  recvcount,recvtype%MPI_VAL,comm%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Ineighbor_alltoall_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/ineighbor_alltoall_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/ineighbor_alltoall_ts.c.in
new file mode 100644
index 00000000000..045e9756b57
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/ineighbor_alltoall_ts.c.in
@@ -0,0 +1,68 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID ineighbor_alltoall(BUFFER_ASYNC x1, COUNT sendcount, DATATYPE sendtype,
+                                  BUFFER_ASYNC_OUT x2, COUNT recvcount, DATATYPE recvtype,
+                                  COMM comm, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Request c_req;
+    MPI_Datatype c_sendtype, c_recvtype;
+    @COUNT_TYPE@ c_sendcount = (@COUNT_TYPE@)*sendcount;
+    @COUNT_TYPE@ c_recvcount = (@COUNT_TYPE@)*recvcount;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    c_sendtype = PMPI_Type_f2c(*sendtype);
+    c_recvtype = PMPI_Type_f2c(*recvtype);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          c_sendcount,
+                          c_sendtype,
+                          recvbuf,
+                          c_recvcount,
+                          c_recvtype, c_comm, &c_req);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_req);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/ineighbor_alltoallv_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/ineighbor_alltoallv_f08.F90
deleted file mode 100644
index fff34a9a416..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/ineighbor_alltoallv_f08.F90
+++ /dev/null
@@ -1,35 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2013 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Ineighbor_alltoallv_f08(sendbuf,sendcounts,sdispls,sendtype,recvbuf,&
-                                       recvcounts,rdispls,recvtype,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_ineighbor_alltoallv_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_ineighbor_alltoallv_f(sendbuf,sendcounts,sdispls,sendtype%MPI_VAL,&
-                                   recvbuf,recvcounts,rdispls,recvtype%MPI_VAL,&
-                                   comm%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Ineighbor_alltoallv_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/ineighbor_alltoallv_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/ineighbor_alltoallv_ts.c.in
new file mode 100644
index 00000000000..1de86c01d45
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/ineighbor_alltoallv_ts.c.in
@@ -0,0 +1,83 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID ineighbor_alltoallv(BUFFER_ASYNC x1, COUNT_ARRAY sendcounts, DISP_ARRAY sdispls,
+                                   DATATYPE sendtype, BUFFER_ASYNC_OUT x2, COUNT_ARRAY recvcounts,
+                                   DISP_ARRAY rdispls, DATATYPE recvtype,
+                                   COMM comm, REQUEST_OUT request)
+{
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype c_sendtype, c_recvtype;
+    MPI_Request c_request;
+    int size, c_ierr;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ *tmp_sendcounts = NULL;
+    @DISP_TYPE@ *tmp_sdispls = NULL;
+    @COUNT_TYPE@ *tmp_recvcounts = NULL;
+    @DISP_TYPE@ *tmp_rdispls = NULL;
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    c_sendtype = PMPI_Type_f2c(*sendtype);
+    c_recvtype = PMPI_Type_f2c(*recvtype);
+
+    PMPI_Comm_size(c_comm, &size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(sendcounts, tmp_sendcounts, size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(sdispls, tmp_sdispls, size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(recvcounts, tmp_recvcounts, size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(rdispls, tmp_rdispls, size);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          tmp_sendcounts,
+                          tmp_sdispls,
+                          c_sendtype,
+                          recvbuf,
+                          tmp_recvcounts,
+                          tmp_rdispls,
+                          c_recvtype, c_comm, &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(sendcounts, tmp_sendcounts);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(sdispls, tmp_sdispls);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(recvcounts, tmp_recvcounts);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(rdispls, tmp_rdispls);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/ineighbor_alltoallw_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/ineighbor_alltoallw_f08.F90
deleted file mode 100644
index a017ae3d255..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/ineighbor_alltoallw_f08.F90
+++ /dev/null
@@ -1,35 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2013 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2013 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Ineighbor_alltoallw_f08(sendbuf,sendcounts,sdispls,sendtypes,&
-                                       recvbuf,recvcounts,rdispls,recvtypes,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request, MPI_ADDRESS_KIND
-   use :: ompi_mpifh_bindings, only : ompi_ineighbor_alltoallw_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: sendcounts(*), recvcounts(*)
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: sdispls(*), rdispls(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtypes(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtypes(*)
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_ineighbor_alltoallw_f(sendbuf,sendcounts,sdispls,sendtypes(1)%MPI_VAL,&
-                                   recvbuf,recvcounts,rdispls,recvtypes(1)%MPI_VAL,&
-                                   comm%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Ineighbor_alltoallw_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/ineighbor_alltoallw_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/ineighbor_alltoallw_ts.c.in
new file mode 100644
index 00000000000..4c387b3dcde
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/ineighbor_alltoallw_ts.c.in
@@ -0,0 +1,86 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID ineighbor_alltoallw(BUFFER_ASYNC x1, COUNT_ARRAY sendcounts,
+                                   AINT_ARRAY sdispls, DATATYPE_ARRAY sendtypes,
+                                   BUFFER_ASYNC_OUT x2, COUNT_ARRAY recvcounts,
+                                   AINT_ARRAY rdispls, DATATYPE_ARRAY recvtypes,
+                                   COMM comm, REQUEST_OUT request)
+{
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype *c_sendtypes, *c_recvtypes;
+    MPI_Request c_request;
+    int size, c_ierr;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ *tmp_sendcounts = NULL;
+    @COUNT_TYPE@ *tmp_recvcounts = NULL;
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+    PMPI_Comm_size(c_comm, &size);
+
+    c_sendtypes = (MPI_Datatype *) malloc(size * sizeof(MPI_Datatype));
+    c_recvtypes = (MPI_Datatype *) malloc(size * sizeof(MPI_Datatype));
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(sendcounts, tmp_sendcounts, size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(recvcounts, tmp_recvcounts, size);
+
+    while (size > 0) {
+        c_sendtypes[size - 1] = PMPI_Type_f2c(sendtypes[size - 1]);
+        c_recvtypes[size - 1] = PMPI_Type_f2c(recvtypes[size - 1]);
+        --size;
+    }
+
+    /* Ineighbor_alltoallw does not support MPI_IN_PLACE */
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          tmp_sendcounts,
+                          sdispls,
+                          c_sendtypes,
+                          recvbuf,
+                          tmp_recvcounts,
+                          rdispls,
+                          c_recvtypes, c_comm, &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(sendcounts, tmp_sendcounts);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(recvcounts, tmp_recvcounts);
+    free(c_sendtypes);
+    free(c_recvtypes);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/info_create_env_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/info_create_env_f08.F90
new file mode 100644
index 00000000000..64716269061
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/info_create_env_f08.F90
@@ -0,0 +1,23 @@
+! -*- f90 -*-
+!
+! Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
+!               All Rights reserved.
+! Copyright (c) 2018-2020 Research Organization for Information Science
+!                         and Technology (RIST).  All rights reserved.
+! $COPYRIGHT$
+
+#include "mpi-f08-rename.h"
+
+subroutine MPI_Info_create_env_f08(info,ierror)
+   use :: mpi_f08_types, only : MPI_Info
+   use :: ompi_mpifh_bindings, only : ompi_info_create_env_f
+   implicit none
+   TYPE(MPI_Info), INTENT(OUT) :: info
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+   integer :: c_ierror
+
+   call ompi_info_create_env_f(info%MPI_VAL,c_ierror)
+   if (present(ierror)) ierror = c_ierror
+
+end subroutine MPI_Info_create_env_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/info_delete_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/info_delete_f08.F90
index f5fdd946dd7..98f7b501103 100644
--- a/ompi/mpi/fortran/use-mpi-f08/info_delete_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/info_delete_f08.F90
@@ -12,13 +12,14 @@
 subroutine MPI_Info_delete_f08(info,key,ierror)
    use :: mpi_f08_types, only : MPI_Info
    use :: ompi_mpifh_bindings, only : ompi_info_delete_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
    implicit none
    TYPE(MPI_Info), INTENT(IN) :: info
    CHARACTER(LEN=*), INTENT(IN) :: key
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
    integer :: c_ierror
 
-   call ompi_info_delete_f(info%MPI_VAL,key,c_ierror,len(key))
+   call ompi_info_delete_f(info%MPI_VAL,key,c_ierror,len(key,KIND=C_INT))
    if (present(ierror)) ierror = c_ierror
 
 end subroutine MPI_Info_delete_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/info_get_nthkey_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/info_get_nthkey_f08.F90
index 40a53633c8d..461ef3de062 100644
--- a/ompi/mpi/fortran/use-mpi-f08/info_get_nthkey_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/info_get_nthkey_f08.F90
@@ -12,6 +12,7 @@
 subroutine MPI_Info_get_nthkey_f08(info,n,key,ierror)
    use :: mpi_f08_types, only : MPI_Info
    use :: ompi_mpifh_bindings, only : ompi_info_get_nthkey_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
    implicit none
    TYPE(MPI_Info), INTENT(IN) :: info
    INTEGER, INTENT(IN) :: n
@@ -19,7 +20,7 @@ subroutine MPI_Info_get_nthkey_f08(info,n,key,ierror)
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
    integer :: c_ierror
 
-   call ompi_info_get_nthkey_f(info%MPI_VAL,n,key,c_ierror,len(key))
+   call ompi_info_get_nthkey_f(info%MPI_VAL,n,key,c_ierror,len(key,KIND=C_INT))
    if (present(ierror)) ierror = c_ierror
 
 end subroutine MPI_Info_get_nthkey_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/info_get_string_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/info_get_string_f08.F90
new file mode 100644
index 00000000000..19e97056b05
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/info_get_string_f08.F90
@@ -0,0 +1,28 @@
+! -*- f90 -*-
+!
+! Copyright (c) 2010-2013 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
+!               All Rights reserved.
+! Copyright (c) 2019-2020 Research Organization for Information Science
+!                         and Technology (RIST).  All rights reserved.
+! $COPYRIGHT$
+
+#include "mpi-f08-rename.h"
+
+subroutine MPI_Info_get_string_f08(info,key,buflen,value,flag,ierror)
+   use :: mpi_f08_types, only : MPI_Info
+   ! See note in mpi-f-interfaces-bind.h for why we "use mpi" here and
+   ! call a PMPI_* subroutine below.
+   use :: mpi, only : PMPI_Info_get_string
+   implicit none
+   TYPE(MPI_Info), INTENT(IN) :: info
+   CHARACTER(LEN=*), INTENT(IN) :: key
+   INTEGER, INTENT(INOUT) :: buflen
+   CHARACTER(LEN=*), INTENT(OUT) :: value
+   LOGICAL, INTENT(OUT) :: flag
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+   integer :: c_ierror
+
+   call PMPI_Info_get_string(info%MPI_VAL,key,buflen,value,flag,c_ierror)
+   if (present(ierror)) ierror = c_ierror
+end subroutine MPI_Info_get_string_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/info_set_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/info_set_f08.F90
index f33f90e840d..b7868ae2e4b 100644
--- a/ompi/mpi/fortran/use-mpi-f08/info_set_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/info_set_f08.F90
@@ -12,13 +12,15 @@
 subroutine MPI_Info_set_f08(info,key,value,ierror)
    use :: mpi_f08_types, only : MPI_Info
    use :: ompi_mpifh_bindings, only : ompi_info_set_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
    implicit none
    TYPE(MPI_Info), INTENT(IN) :: info
    CHARACTER(LEN=*), INTENT(IN) :: key, value
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
    integer :: c_ierror
 
-   call ompi_info_set_f(info%MPI_VAL,key,value,c_ierror,len(key),len(value))
+   call ompi_info_set_f(info%MPI_VAL,key,value,c_ierror, &
+                        len(key,KIND=C_INT),len(value,KIND=C_INT))
    if (present(ierror)) ierror = c_ierror
 
 end subroutine MPI_Info_set_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/intercomm_create_from_groups_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/intercomm_create_from_groups_f08.F90
new file mode 100644
index 00000000000..809dabf0846
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/intercomm_create_from_groups_f08.F90
@@ -0,0 +1,38 @@
+! -*- f90 -*-
+!
+! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2009-2013 Los Alamos National Security, LLC.
+!                         All rights reserved.
+! Copyright (c) 2018      Research Organization for Information Science
+!                         and Technology (RIST).  All rights reserved.
+! Copyright (c) 2019      Triad National Security, LLC. All rights
+!                         reserved.
+! $COPYRIGHT$
+
+#include "mpi-f08-rename.h"
+
+subroutine MPI_Intercomm_create_from_groups_f08(local_group, local_leader, remote_group, &
+                                                remote_leader, stringtag, info, errhandler, &
+                                                newintercomm, ierror)
+   use :: mpi_f08_types, only : MPI_Comm, MPI_Group, MPI_Errhandler, MPI_Info
+   use :: ompi_mpifh_bindings, only : ompi_intercomm_create_from_groups_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
+   implicit none
+   TYPE(MPI_Group), INTENT(IN) :: local_group, remote_group
+   INTEGER, INTENT(IN):: local_leader, remote_leader
+   CHARACTER(LEN=*), INTENT(IN) :: stringtag
+   TYPE(MPI_Info), INTENT(IN) :: info
+   TYPE(MPI_Errhandler), INTENT(IN) :: errhandler
+   TYPE(MPI_Comm), INTENT(OUT) :: newintercomm
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+   integer :: c_ierror
+
+   call ompi_intercomm_create_from_groups_f(local_group%MPI_VAL, local_leader, &
+                                            remote_group%MPI_VAL,  &
+                                            remote_leader, stringtag, info%MPI_VAL, &
+                                            errhandler%MPI_VAL, &
+                                            newintercomm%MPI_VAL, c_ierror, len(stringtag,KIND=C_INT))
+   if (present(ierror)) ierror = c_ierror
+
+end subroutine MPI_Intercomm_create_from_groups_f08
+
diff --git a/ompi/mpi/fortran/use-mpi-f08/iprobe_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/iprobe_f08.F90
index 9d44c3c0673..b53af9a8efd 100644
--- a/ompi/mpi/fortran/use-mpi-f08/iprobe_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/iprobe_f08.F90
@@ -16,7 +16,7 @@ subroutine MPI_Iprobe_f08(source,tag,comm,flag,status,ierror)
    INTEGER, INTENT(IN) :: source, tag
    TYPE(MPI_Comm), INTENT(IN) :: comm
    LOGICAL, INTENT(OUT) :: flag
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   TYPE(MPI_Status) :: status
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
    integer :: c_ierror
 
@@ -29,7 +29,7 @@ subroutine PMPI_Iprobe(source, tag, comm, flag, status, ierror)
         integer, intent(in) :: tag
         integer, intent(in) :: comm
         logical, intent(out) :: flag
-        TYPE(MPI_Status), intent(out) :: status
+        TYPE(MPI_Status) :: status
         integer, intent(out) :: ierror
       end subroutine PMPI_Iprobe
    end interface
diff --git a/ompi/mpi/fortran/use-mpi-f08/irecv_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/irecv_ts.c.in
new file mode 100644
index 00000000000..19048b01ae5
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/irecv_ts.c.in
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE void irecv(BUFFER_OUT x, COUNT count, DATATYPE datatype,
+                     RANK source, TAG tag, COMM comm,
+                     REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    MPI_Request c_req;
+    MPI_Comm c_comm = PMPI_Comm_f2c (*comm);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(buf), c_count,
+                          c_datatype, OMPI_FINT_2_INT(*source),
+                          OMPI_FINT_2_INT(*tag), c_comm, &c_req);
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+       *request = PMPI_Request_c2f(c_req);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/ireduce_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/ireduce_f08.F90
deleted file mode 100644
index 818efd186a0..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/ireduce_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Ireduce_f08(sendbuf,recvbuf,count,datatype,op,root,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_ireduce_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: count, root
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_ireduce_f(sendbuf,recvbuf,count,datatype%MPI_VAL,&
-                      op%MPI_VAL,root,comm%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Ireduce_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/ireduce_scatter_block_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/ireduce_scatter_block_f08.F90
deleted file mode 100644
index a44d2cf54ba..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/ireduce_scatter_block_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Ireduce_scatter_block_f08(sendbuf,recvbuf,recvcount,datatype,op,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_ireduce_scatter_block_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: recvcount
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_ireduce_scatter_block_f(sendbuf,recvbuf,recvcount,&
-                                    datatype%MPI_VAL,op%MPI_VAL,comm%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Ireduce_scatter_block_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/ireduce_scatter_block_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/ireduce_scatter_block_ts.c.in
new file mode 100644
index 00000000000..e17a6f4b8cb
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/ireduce_scatter_block_ts.c.in
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID ireduce_scatter_block(BUFFER_ASYNC x1, BUFFER_ASYNC_OUT x2,
+                                     COUNT recvcount, DATATYPE datatype,
+                                     OP op, COMM comm,
+                                     REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype c_type;
+    MPI_Request c_request;
+    MPI_Op c_op;
+    int size;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_recvcount = (@COUNT_TYPE@) *recvcount;
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    c_type = PMPI_Type_f2c(*datatype);
+    c_op = PMPI_Op_f2c(*op);
+
+    PMPI_Comm_size(c_comm, &size);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf, recvbuf, c_recvcount,
+                          c_type, c_op, c_comm, &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/ireduce_scatter_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/ireduce_scatter_f08.F90
deleted file mode 100644
index 87bb26eeb25..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/ireduce_scatter_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Ireduce_scatter_f08(sendbuf,recvbuf,recvcounts,datatype,op,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_ireduce_scatter_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: recvcounts(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_ireduce_scatter_f(sendbuf,recvbuf,recvcounts,datatype%MPI_VAL,&
-                              op%MPI_VAL,comm%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Ireduce_scatter_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/ireduce_scatter_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/ireduce_scatter_ts.c.in
new file mode 100644
index 00000000000..e6c319b03e3
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/ireduce_scatter_ts.c.in
@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID ireduce_scatter(BUFFER_ASYNC x1, BUFFER_ASYNC_OUT x2,
+                               COUNT_ARRAY recvcounts, DATATYPE datatype,
+                               OP op, COMM comm, REQUEST_OUT request)
+{
+    int c_ierr, idx = 0;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype c_type;
+    MPI_Request c_request;
+    MPI_Op c_op;
+    int size;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ *tmp_recvcounts = NULL;
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+    c_type = PMPI_Type_f2c(*datatype);
+    c_op = PMPI_Op_f2c(*op);
+
+    PMPI_Comm_size(c_comm, &size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(recvcounts, tmp_recvcounts, size);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf, recvbuf, tmp_recvcounts,
+                          c_type, c_op, c_comm, &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(recvcounts, tmp_recvcounts, c_request, c_ierr, idx);
+
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/ireduce_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/ireduce_ts.c.in
new file mode 100644
index 00000000000..59ee1e2123c
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/ireduce_ts.c.in
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID ireduce(BUFFER_ASYNC x1, BUFFER_ASYNC_OUT x2, COUNT count,
+                       DATATYPE datatype, OP op,
+                       RANK root, COMM comm, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Datatype c_type;
+    MPI_Request c_request;
+    MPI_Op c_op;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    c_type = PMPI_Type_f2c(*datatype);
+    c_op = PMPI_Op_f2c(*op);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf, recvbuf,
+                          c_count,
+                          c_type, c_op,
+                          OMPI_FINT_2_INT(*root),
+                          c_comm, &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/irsend_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/irsend_f08.F90
deleted file mode 100644
index b87f025b284..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/irsend_f08.F90
+++ /dev/null
@@ -1,29 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Irsend_f08(buf,count,datatype,dest,tag,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_irsend_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) OMPI_ASYNCHRONOUS :: buf
-   INTEGER, INTENT(IN) :: count, dest, tag
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_irsend_f(buf,count,datatype%MPI_VAL,dest,tag,comm%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Irsend_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/irsend_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/irsend_ts.c.in
new file mode 100644
index 00000000000..f21a428af52
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/irsend_ts.c.in
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID irsend(BUFFER_ASYNC x, COUNT count, DATATYPE datatype, RANK dest,
+                      TAG tag, COMM comm, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    MPI_Request c_req;
+    MPI_Comm c_comm = PMPI_Comm_f2c (*comm);
+
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(buf), c_count,
+                          c_datatype, OMPI_FINT_2_INT(*dest),
+                          OMPI_FINT_2_INT(*tag), c_comm,
+                          &c_req);
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_req);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/iscan_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/iscan_f08.F90
deleted file mode 100644
index bf8a8524180..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/iscan_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Iscan_f08(sendbuf,recvbuf,count,datatype,op,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_iscan_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_iscan_f(sendbuf,recvbuf,count,datatype%MPI_VAL,&
-                    op%MPI_VAL,comm%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Iscan_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/iscan_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/iscan_ts.c.in
new file mode 100644
index 00000000000..2e67a22133d
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/iscan_ts.c.in
@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID iscan(BUFFER_ASYNC x1, BUFFER_ASYNC_OUT x2, COUNT count,
+                     DATATYPE datatype, OP op, COMM comm,
+                     REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype c_type;
+    MPI_Request c_request;
+    MPI_Op c_op;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    c_type = PMPI_Type_f2c(*datatype);
+    c_op = PMPI_Op_f2c(*op);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf, recvbuf,
+                          c_count,
+                          c_type, c_op,
+                          c_comm, &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/iscatter_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/iscatter_f08.F90
deleted file mode 100644
index 63c6ded5a47..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/iscatter_f08.F90
+++ /dev/null
@@ -1,33 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Iscatter_f08(sendbuf,sendcount,sendtype,recvbuf,&
-                           recvcount,recvtype,root,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_iscatter_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount, root
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_iscatter_f(sendbuf,sendcount,sendtype%MPI_VAL,recvbuf,recvcount,&
-                       recvtype%MPI_VAL,root,comm%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Iscatter_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/iscatter_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/iscatter_ts.c.in
new file mode 100644
index 00000000000..42f7870374b
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/iscatter_ts.c.in
@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID iscatter(BUFFER_ASYNC x1, COUNT sendcount,
+                        DATATYPE sendtype, BUFFER_ASYNC_OUT x2,
+                        COUNT recvcount, DATATYPE recvtype,
+                        RANK root, COMM comm, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    int c_root = OMPI_FINT_2_INT(*root);
+    MPI_Datatype c_sendtype = NULL, c_recvtype = NULL, c_recvdatatype = NULL;
+    @COUNT_TYPE@ c_sendcount = (@COUNT_TYPE@) *sendcount;
+    @COUNT_TYPE@ c_recvcount = (@COUNT_TYPE@) *recvcount;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    MPI_Request c_request;
+
+    if (OMPI_COMM_IS_INTER(c_comm)) {
+        if (MPI_ROOT == c_root) {
+            c_sendtype = PMPI_Type_f2c(*sendtype);
+            OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+                return;
+            }
+        } else if (MPI_PROC_NULL != c_root) {
+            c_recvtype = PMPI_Type_f2c(*recvtype);
+            OMPI_CFI_2_C(x2, c_recvcount, c_recvtype, c_recvdatatype, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+                return;
+            }
+        }
+    } else {
+        if (ompi_comm_rank(c_comm) == c_root) {
+            c_sendtype = PMPI_Type_f2c(*sendtype);
+            OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+                return;
+            }
+        }
+        if (OMPI_IS_FORTRAN_IN_PLACE(recvbuf)) {
+            recvbuf = MPI_IN_PLACE;
+        } else {
+            c_recvtype = PMPI_Type_f2c(*recvtype);
+            OMPI_CFI_2_C(x2, c_recvcount, c_recvtype, c_recvdatatype, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+                return;
+            }
+        }
+    }
+
+    recvbuf = (char *) OMPI_F2C_IN_PLACE(recvbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+
+    c_ierr = @INNER_CALL@(sendbuf, c_sendcount, c_sendtype,
+                          recvbuf, c_recvcount, c_recvdatatype,
+                          c_root, c_comm, &c_request);
+
+    if (c_recvdatatype != c_recvtype) {
+        ompi_datatype_destroy(&c_recvdatatype);
+    }
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/iscatterv_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/iscatterv_f08.F90
deleted file mode 100644
index 1d7e81447e9..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/iscatterv_f08.F90
+++ /dev/null
@@ -1,34 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Iscatterv_f08(sendbuf,sendcounts,displs,sendtype,recvbuf,&
-                            recvcount,recvtype,root,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_iscatterv_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: recvcount, root
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: sendcounts(*), displs(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_iscatterv_f(sendbuf,sendcounts,displs,sendtype%MPI_VAL,recvbuf,&
-                        recvcount,recvtype%MPI_VAL,root,comm%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Iscatterv_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/iscatterv_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/iscatterv_ts.c.in
new file mode 100644
index 00000000000..7f983faed51
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/iscatterv_ts.c.in
@@ -0,0 +1,109 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID iscatterv(BUFFER_ASYNC x1, COUNT_ARRAY sendcounts,
+                         DISP_ARRAY displs, DATATYPE sendtype,
+                         BUFFER_ASYNC_OUT x2, COUNT recvcount,
+                         DATATYPE recvtype, RANK root,
+                         COMM comm, REQUEST_OUT request)
+{
+    int c_ierr, idx = 0;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    int c_root = OMPI_FINT_2_INT(*root);
+    MPI_Datatype  __opal_attribute_unused__ c_recvdatatype = MPI_DATATYPE_NULL;
+    MPI_Datatype c_sendtype = MPI_DATATYPE_NULL, c_recvtype = MPI_DATATYPE_NULL;
+    @COUNT_TYPE@ c_recvcount = (@COUNT_TYPE@)*recvcount;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    MPI_Request c_request;
+    @COUNT_TYPE@ *tmp_sendcounts = NULL;
+    @DISP_TYPE@ *tmp_displs = NULL;
+
+    if (OMPI_COMM_IS_INTER(c_comm)) {
+        if (MPI_ROOT == c_root) {
+            int size = ompi_comm_size(c_comm);
+            c_sendtype = PMPI_Type_f2c(*sendtype);
+            OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+                return;
+            }
+            OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(sendcounts, tmp_sendcounts, size);
+            OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(displs, tmp_displs, size);
+        } else if (MPI_PROC_NULL != c_root) {
+            c_recvtype = PMPI_Type_f2c(*recvtype);
+            OMPI_CFI_2_C(x2, c_recvcount, c_recvtype, c_recvdatatype, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+                return;
+            }
+        }
+    } else {
+        if (ompi_comm_rank(c_comm) == c_root) {
+            int size = ompi_comm_size(c_comm);
+            c_sendtype = PMPI_Type_f2c(*sendtype);
+            OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(sendcounts, tmp_sendcounts, size);
+            OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(displs, tmp_displs, size);
+            OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+                return;
+            }
+        }
+        if (OMPI_IS_FORTRAN_IN_PLACE(recvbuf)) {
+            recvbuf = MPI_IN_PLACE;
+        } else {
+            c_recvtype = PMPI_Type_f2c(*recvtype);
+            OMPI_CFI_2_C(x2, c_recvcount, c_recvtype, c_recvdatatype, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+                return;
+            }
+        }
+    }
+
+    recvbuf = (char *) OMPI_F2C_IN_PLACE(recvbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          tmp_sendcounts,
+                          tmp_displs,
+                          c_sendtype,
+                          recvbuf,
+                          c_recvcount,
+                          (c_recvdatatype == MPI_DATATYPE_NULL) ? c_recvtype : c_recvdatatype,
+                          c_root, c_comm, &c_request);
+    if (c_recvdatatype != MPI_DATATYPE_NULL && c_recvdatatype != c_recvtype) {
+        ompi_datatype_destroy(&c_recvdatatype);
+    }
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(sendcounts, tmp_sendcounts, c_request, c_ierr, idx);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(displs, tmp_displs, c_request, c_ierr, idx);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/isend_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/isend_ts.c.in
new file mode 100644
index 00000000000..fb9844eb2d4
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/isend_ts.c.in
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID isend(BUFFER x, COUNT count, DATATYPE datatype, RANK dest,
+                     TAG tag, COMM comm, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    MPI_Request c_req;
+    MPI_Comm c_comm = PMPI_Comm_f2c (*comm);
+
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count =  (@COUNT_TYPE@) *count;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(buf), c_count,
+                          c_datatype, OMPI_FINT_2_INT(*dest),
+                          OMPI_FINT_2_INT(*tag),
+                          c_comm, &c_req);
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+       *request = PMPI_Request_c2f(c_req);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/isendrecv_replace_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/isendrecv_replace_ts.c.in
new file mode 100644
index 00000000000..20c53252166
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/isendrecv_replace_ts.c.in
@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID isendrecv_replace(BUFFER_ASYNC_OUT x, COUNT count, DATATYPE datatype,
+                                 RANK dest, TAG sendtag,
+                                 RANK source, TAG recvtag,
+                                 COMM comm, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Request c_req;
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    MPI_Comm c_comm = PMPI_Comm_f2c (*comm);
+
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(buf),
+                          c_count,
+                          c_datatype,
+                          OMPI_FINT_2_INT(*dest),
+                          OMPI_FINT_2_INT(*sendtag),
+                          OMPI_FINT_2_INT(*source),
+                          OMPI_FINT_2_INT(*recvtag),
+                          c_comm, &c_req);
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+       *request = PMPI_Request_c2f(c_req);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/isendrecv_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/isendrecv_ts.c.in
new file mode 100644
index 00000000000..cbf9b4aca9c
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/isendrecv_ts.c.in
@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID isendrecv(BUFFER_ASYNC x1, COUNT sendcount, DATATYPE sendtype,
+                         RANK dest, TAG sendtag, BUFFER_ASYNC_OUT x2,
+                         COUNT recvcount, DATATYPE recvtype,
+                         RANK source, TAG recvtag, COMM comm,
+                         REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c (*comm);
+    MPI_Request c_req;
+    MPI_Datatype c_senddatatype, c_sendtype = PMPI_Type_f2c(*sendtype);
+    MPI_Datatype c_recvdatatype, c_recvtype = PMPI_Type_f2c(*recvtype);
+    void *sendbuf = OMPI_CFI_BASE_ADDR(x1);
+    @COUNT_TYPE@ c_sendcount = (@COUNT_TYPE@) *sendcount;
+    void *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_recvcount = (@COUNT_TYPE@) *recvcount;
+
+    OMPI_CFI_2_C(x1, c_sendcount, c_sendtype, c_senddatatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+    OMPI_CFI_2_C(x2, c_recvcount, c_recvtype, c_recvdatatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        if (c_senddatatype != c_sendtype) {
+            ompi_datatype_destroy(&c_senddatatype);
+        }
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(sendbuf), c_sendcount,
+                          c_senddatatype,
+                          OMPI_FINT_2_INT(*dest),
+                          OMPI_FINT_2_INT(*sendtag),
+                          OMPI_F2C_BOTTOM(recvbuf), c_recvcount,
+                          c_recvdatatype, OMPI_FINT_2_INT(*source),
+                          OMPI_FINT_2_INT(*recvtag),
+                          c_comm, &c_req);
+    if (c_senddatatype != c_sendtype) {
+        ompi_datatype_destroy(&c_senddatatype);
+    }
+    if (c_recvdatatype != c_recvtype) {
+        ompi_datatype_destroy(&c_recvdatatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+       *request = PMPI_Request_c2f(c_req);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/issend_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/issend_f08.F90
deleted file mode 100644
index f162a4e5839..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/issend_f08.F90
+++ /dev/null
@@ -1,29 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Issend_f08(buf,count,datatype,dest,tag,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_issend_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count, dest, tag
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_issend_f(buf,count,datatype%MPI_VAL,dest,tag,comm%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Issend_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/issend_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/issend_ts.c.in
new file mode 100644
index 00000000000..56ea0d70652
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/issend_ts.c.in
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID issend(BUFFER_ASYNC x, COUNT count, DATATYPE datatype, RANK dest,
+                      TAG tag, COMM comm, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    MPI_Request c_req;
+    MPI_Comm c_comm = PMPI_Comm_f2c (*comm);
+
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(buf), c_count,
+                          c_datatype, OMPI_FINT_2_INT(*dest),
+                          OMPI_FINT_2_INT(*tag),
+                          c_comm, &c_req);
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_req);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/lookup_name_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/lookup_name_f08.F90
index fa200f361b9..434b96ad53c 100644
--- a/ompi/mpi/fortran/use-mpi-f08/lookup_name_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/lookup_name_f08.F90
@@ -12,6 +12,7 @@
 subroutine MPI_Lookup_name_f08(service_name,info,port_name,ierror)
    use :: mpi_f08_types, only : MPI_Info
    use :: ompi_mpifh_bindings, only : ompi_lookup_name_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
    implicit none
    CHARACTER(LEN=*), INTENT(IN) :: service_name
    TYPE(MPI_Info), INTENT(IN) :: info
@@ -20,7 +21,7 @@ subroutine MPI_Lookup_name_f08(service_name,info,port_name,ierror)
    integer :: c_ierror
 
    call ompi_lookup_name_f(service_name,info%MPI_VAL,port_name,c_ierror, &
-                           len(service_name),len(port_name))
+                           len(service_name,KIND=C_INT),len(port_name,KIND=C_INT))
    if (present(ierror)) ierror = c_ierror
 
 end subroutine MPI_Lookup_name_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/mod/Makefile.am b/ompi/mpi/fortran/use-mpi-f08/mod/Makefile.am
index 87f1c2db8ba..d851dd2301a 100644
--- a/ompi/mpi/fortran/use-mpi-f08/mod/Makefile.am
+++ b/ompi/mpi/fortran/use-mpi-f08/mod/Makefile.am
@@ -10,6 +10,8 @@
 # Copyright (c) 2015-2020 Research Organization for Information Science
 #                         and Technology (RIST).  All rights reserved.
 # Copyright (c) 2016      IBM Corporation.  All rights reserved.
+# Copyright (C) 2024      Triad National Security, LLC. All rights
+#                         reserved.
 #
 # $COPYRIGHT$
 #
@@ -34,7 +36,7 @@ if OMPI_BUILD_FORTRAN_USEMPIF08_BINDINGS
 AM_FCFLAGS = -I$(top_builddir)/ompi/include \
              -I$(top_srcdir)/ompi/include \
              $(OMPI_FC_MODULE_FLAG)$(top_builddir)/ompi/mpi/fortran/use-mpi \
-             $(OMPI_FC_MODULE_FLAG)$(top_builddir)/ompi/$(OMPI_FORTRAN_USEMPI_DIR) \
+             $(OMPI_FC_MODULE_FLAG)$(top_builddir)/ompi/mpi/fortran/use-mpi-ignore-tkr \
              $(OMPI_FC_MODULE_FLAG). \
              -I$(top_srcdir) -I$(top_builddir) $(FCFLAGS_f90)
 
@@ -64,7 +66,9 @@ libforce_usempif08_internal_modules_to_be_built_la_SOURCES = \
 
 nodist_noinst_HEADERS = mpi-f08-interfaces.h
 
-noinst_HEADERS = mpi-f08-rename.h
+noinst_HEADERS = \
+	mpi-f08-rename.h \
+        mpi-f08-interfaces-generated.h
 
 libforce_usempi_internal_modules_to_be_built.la: libusempif08_internal_modules.la
 
@@ -72,6 +76,32 @@ config_h = \
     $(top_builddir)/ompi/mpi/fortran/configure-fortran-output.h \
     $(top_srcdir)/ompi/mpi/fortran/configure-fortran-output-bottom.h
 
+# Generate the Fortran interfaces
+if OMPI_GENERATE_BINDINGS
+
+include ../Makefile.prototype_files
+template_files =${prototype_files:%=$(abs_top_srcdir)/ompi/mpi/fortran/use-mpi-f08/%}
+
+if OMPI_FORTRAN_HAVE_TS
+gen_ts = --generate-ts-suffix
+endif
+
+
+mpi-f08-interfaces-generated.h: $(template_files)
+	$(OMPI_V_GEN) $(PYTHON) $(top_srcdir)/ompi/mpi/bindings/bindings.py \
+	    --builddir $(abs_top_builddir) \
+	    --srcdir $(abs_top_srcdir) \
+	    --output $(abs_builddir)/$@ \
+	    fortran \
+	    $(gen_ts) \
+	    interface \
+	    --prototype-files $(template_files)
+
+endif
+
+# Delete generated file on maintainer-clean
+MAINTAINERCLEANFILES = mpi-f08-interfaces-generated.h
+
 #
 # Automake doesn't do Fortran dependency analysis, so must list them
 # manually here.  Bummer!
@@ -83,6 +113,7 @@ mpi-f08-interfaces.lo: $(config_h)
 mpi-f08-interfaces.lo: mpi-f08-interfaces.F90
 mpi-f08-interfaces.lo: mpi-f08-interfaces-callbacks.lo
 mpi-f08-interfaces.lo: mpi-f08-interfaces.h
+mpi-f08-interfaces.lo: mpi-f08-interfaces-generated.h
 mpi-f08-interfaces-callbacks.lo: $(config_h)
 mpi-f08-interfaces-callbacks.lo: mpi-f08-interfaces-callbacks.F90
 mpi-f08-interfaces-callbacks.lo: mpi-f08-types.lo
@@ -94,6 +125,7 @@ pmpi-f08-interfaces.lo: pmpi-f08-interfaces.F90
 pmpi-f08-interfaces.lo: mpi-f08-interfaces-callbacks.lo
 pmpi-f08-interfaces.lo: mpi-f08-interfaces.h
 pmpi-f08-interfaces.lo: mpi-f08-rename.h
+pmpi-f08-interfaces.lo: mpi-f08-interfaces-generated.h
 
 ###########################################################################
 
@@ -102,14 +134,14 @@ pmpi-f08-interfaces.lo: mpi-f08-rename.h
 
 install-exec-hook:
 	@ for file in `ls *.mod`; do \
-	  echo $(INSTALL) $$file $(DESTDIR)$(libdir); \
-	  $(INSTALL) $$file $(DESTDIR)$(libdir); \
+	  echo $(INSTALL) $$file $(DESTDIR)$(OMPI_FORTRAN_MODULEDIR); \
+	  $(INSTALL) $$file $(DESTDIR)$(OMPI_FORTRAN_MODULEDIR); \
 	done
 
 uninstall-local:
 	@ for file in `ls *.mod`; do \
-	  echo rm -f $(DESTDIR)$(libdir)/$$file; \
-	  rm -f $(DESTDIR)$(libdir)/$$file; \
+	  echo rm -f $(DESTDIR)$(OMPI_FORTRAN_MODULEDIR)/$$file; \
+	  rm -f $(DESTDIR)$(OMPI_FORTRAN_MODULEDIR)/$$file; \
 	done
 
 endif
diff --git a/ompi/mpi/fortran/use-mpi-f08/mod/mpi-f08-interfaces-callbacks.F90 b/ompi/mpi/fortran/use-mpi-f08/mod/mpi-f08-interfaces-callbacks.F90
index d72ce1b9e2f..efe1f7a8422 100644
--- a/ompi/mpi/fortran/use-mpi-f08/mod/mpi-f08-interfaces-callbacks.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/mod/mpi-f08-interfaces-callbacks.F90
@@ -160,6 +160,15 @@ SUBROUTINE MPI_File_errhandler_function(file, error_code)
 END SUBROUTINE
 END INTERFACE
 
+OMPI_ABSTRACT INTERFACE
+SUBROUTINE MPI_Session_errhandler_function(session,error_code)
+   USE mpi_f08_types
+   IMPLICIT NONE
+   TYPE(MPI_Session) :: session
+   INTEGER :: error_code
+END SUBROUTINE
+END INTERFACE
+
 OMPI_ABSTRACT INTERFACE
 SUBROUTINE MPI_Grequest_query_function(extra_state,status,ierror)
    USE mpi_f08_types
diff --git a/ompi/mpi/fortran/use-mpi-f08/mod/mpi-f08-interfaces.F90 b/ompi/mpi/fortran/use-mpi-f08/mod/mpi-f08-interfaces.F90
index 2bbd07eb5f9..ad4a92223b0 100644
--- a/ompi/mpi/fortran/use-mpi-f08/mod/mpi-f08-interfaces.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/mod/mpi-f08-interfaces.F90
@@ -10,6 +10,8 @@
 ! Copyright (c) 2015-2020 Research Organization for Information Science
 !                         and Technology (RIST).  All rights reserved.
 ! Copyright (c) 2017-2018 FUJITSU LIMITED.  All rights reserved.
+! Copyright (c) 2019      Triad National Security, LLC. All rights
+!                         reserved.
 ! $COPYRIGHT$
 !
 ! This file provides the interface specifications for the MPI Fortran
@@ -22,6 +24,7 @@
 module mpi_f08_interfaces
 
 #include "mpi-f08-interfaces.h"
+#include "mpi-f08-interfaces-generated.h"
 
 ! MPI_Wtick is not a wrapper function
 !
diff --git a/ompi/mpi/fortran/use-mpi-f08/mod/mpi-f08-interfaces.h.in b/ompi/mpi/fortran/use-mpi-f08/mod/mpi-f08-interfaces.h.in
index 559c3bcfc0d..9b462537a25 100644
--- a/ompi/mpi/fortran/use-mpi-f08/mod/mpi-f08-interfaces.h.in
+++ b/ompi/mpi/fortran/use-mpi-f08/mod/mpi-f08-interfaces.h.in
@@ -1,6 +1,6 @@
 ! -*- f90 -*-
 !
-! Copyright (c) 2009-2018 Cisco Systems, Inc.  All rights reserved
+! Copyright (c) 2009-2022 Cisco Systems, Inc.  All rights reserved
 ! Copyright (c) 2009-2015 Los Alamos National Security, LLC.
 !                         All rights reserved.
 ! Copyright (c) 2012      The University of Tennessee and The University
@@ -10,6 +10,8 @@
 ! Copyright (c) 2015-2020 Research Organization for Information Science
 !                         and Technology (RIST). All rights reserved.
 ! Copyright (c) 2017-2018 FUJITSU LIMITED.  All rights reserved.
+! Copyright (c) 2021-2023 Triad National Security, LLC. All rights
+!                         reserved.
 ! $COPYRIGHT$
 !
 ! This file provides the interface specifications for the MPI Fortran
@@ -17,53 +19,6 @@
 ! and the name for tools ("MPI_Init_f08") and the back-end implementation
 ! name (e.g., "MPI_Init_f08").
 
-interface  MPI_Bsend
-subroutine MPI_Bsend_f08(buf,count,datatype,dest,tag,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count, dest, tag
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Bsend_f08
-end interface  MPI_Bsend
-
-interface  MPI_Bsend_init
-subroutine MPI_Bsend_init_f08(buf,count,datatype,dest,tag,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buf
-   INTEGER, INTENT(IN) :: count, dest, tag
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Bsend_init_f08
-end interface  MPI_Bsend_init
-
-interface  MPI_Buffer_attach
-subroutine MPI_Buffer_attach_f08(buffer,size,ierror)
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buffer
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buffer
-   INTEGER, INTENT(IN) :: size
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Buffer_attach_f08
-end interface  MPI_Buffer_attach
-
-interface  MPI_Buffer_detach
-subroutine MPI_Buffer_detach_f08(buffer_addr,size,ierror)
-   USE, INTRINSIC ::  ISO_C_BINDING, ONLY : C_PTR
-   implicit none
-   TYPE(C_PTR), INTENT(OUT) ::  buffer_addr
-   INTEGER, INTENT(OUT) :: size
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Buffer_detach_f08
-end interface  MPI_Buffer_detach
-
 interface  MPI_Cancel
 subroutine MPI_Cancel_f08(request,ierror)
    use :: mpi_f08_types, only : MPI_Request
@@ -73,31 +28,6 @@ subroutine MPI_Cancel_f08(request,ierror)
 end subroutine MPI_Cancel_f08
 end interface  MPI_Cancel
 
-interface  MPI_Get_count
-subroutine MPI_Get_count_f08(status,datatype,count,ierror)
-   use :: mpi_f08_types, only : MPI_Status, MPI_Datatype
-   implicit none
-   TYPE(MPI_Status), INTENT(IN) :: status
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER, INTENT(OUT) :: count
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Get_count_f08
-end interface  MPI_Get_count
-
-interface  MPI_Ibsend
-subroutine MPI_Ibsend_f08(buf,count,datatype,dest,tag,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count, dest, tag
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Ibsend_f08
-end interface  MPI_Ibsend
-
 interface  MPI_Iprobe
 subroutine MPI_Iprobe_f08(source,tag,comm,flag,status,ierror)
    use :: mpi_f08_types, only : MPI_Comm, MPI_Status
@@ -105,66 +35,52 @@ subroutine MPI_Iprobe_f08(source,tag,comm,flag,status,ierror)
    INTEGER, INTENT(IN) :: source, tag
    TYPE(MPI_Comm), INTENT(IN) :: comm
    LOGICAL, INTENT(OUT) :: flag
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   TYPE(MPI_Status) :: status
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
 end subroutine MPI_Iprobe_f08
 end interface  MPI_Iprobe
 
-interface  MPI_Irecv
-subroutine MPI_Irecv_f08(buf,count,datatype,source,tag,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
+interface  MPI_Pready
+subroutine MPI_Pready_f08(partition,request,ierror)
+   use :: mpi_f08_types, only : MPI_Request
    implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buf
-   INTEGER, INTENT(IN) :: count, source, tag
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
+   INTEGER, INTENT(IN) :: partition
+   TYPE(MPI_Request), INTENT(IN) :: request
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Irecv_f08
-end interface  MPI_Irecv
+end subroutine MPI_Pready_f08
+end interface  MPI_Pready
 
-interface  MPI_Irsend
-subroutine MPI_Irsend_f08(buf,count,datatype,dest,tag,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
+interface  MPI_Pready_list
+subroutine MPI_Pready_list_f08(length,partitions,request,ierror)
+   use :: mpi_f08_types, only : MPI_Request
    implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count, dest, tag
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
+   INTEGER, INTENT(IN) :: length
+   INTEGER, DIMENSION(*), INTENT(IN) :: partitions
+   TYPE(MPI_Request), INTENT(IN) :: request
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Irsend_f08
-end interface  MPI_Irsend
+end subroutine MPI_Pready_list_f08
+end interface  MPI_Pready_list
 
-interface  MPI_Isend
-subroutine MPI_Isend_f08(buf,count,datatype,dest,tag,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
+interface  MPI_Pready_range
+subroutine MPI_Pready_range_f08(partition_low,partition_high,request,ierror)
+   use :: mpi_f08_types, only : MPI_Request
    implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count, dest, tag
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
+   INTEGER, INTENT(IN) :: partition_low, partition_high
+   TYPE(MPI_Request), INTENT(IN) :: request
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Isend_f08
-end interface  MPI_Isend
+end subroutine MPI_Pready_range_f08
+end interface  MPI_Pready_range
 
-interface  MPI_Issend
-subroutine MPI_Issend_f08(buf,count,datatype,dest,tag,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
+interface  MPI_Parrived
+subroutine MPI_Parrived_f08(request,partition,flag,ierror)
+   use :: mpi_f08_types, only : MPI_Request
    implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count, dest, tag
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
+   TYPE(MPI_Request), INTENT(IN) :: request
+   INTEGER, INTENT(IN) :: partition
+   LOGICAL, INTENT(OUT) :: flag
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Issend_f08
-end interface  MPI_Issend
+end subroutine MPI_Parrived_f08
+end interface  MPI_Parrived
 
 interface  MPI_Probe
 subroutine MPI_Probe_f08(source,tag,comm,status,ierror)
@@ -172,39 +88,11 @@ subroutine MPI_Probe_f08(source,tag,comm,status,ierror)
    implicit none
    INTEGER, INTENT(IN) :: source, tag
    TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   TYPE(MPI_Status) :: status
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
 end subroutine MPI_Probe_f08
 end interface  MPI_Probe
 
-interface  MPI_Recv
-subroutine MPI_Recv_f08(buf,count,datatype,source,tag,comm,status,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Status
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buf
-   INTEGER, INTENT(IN) :: count, source, tag
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Status) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Recv_f08
-end interface  MPI_Recv
-
-interface  MPI_Recv_init
-subroutine MPI_Recv_init_f08(buf,count,datatype,source,tag,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buf
-   INTEGER, INTENT(IN) :: count, source, tag
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Recv_init_f08
-end interface  MPI_Recv_init
-
 interface  MPI_Request_free
 subroutine MPI_Request_free_f08(request,ierror)
    use :: mpi_f08_types, only : MPI_Request
@@ -225,117 +113,111 @@ subroutine MPI_Request_get_status_f08(request,flag,status,ierror)
 end subroutine MPI_Request_get_status_f08
 end interface  MPI_Request_get_status
 
-interface  MPI_Rsend
-subroutine MPI_Rsend_f08(buf,count,datatype,dest,tag,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
+interface MPI_Session_call_errhandler
+subroutine MPI_Session_call_errhandler_f08(session,errorcode,ierror)
+   use :: mpi_f08_types, only : MPI_Session
    implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count, dest, tag
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
+   TYPE(MPI_Session), INTENT(IN) :: session
+   INTEGER, INTENT(IN) :: errorcode
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Rsend_f08
-end interface  MPI_Rsend
+end subroutine MPI_Session_call_errhandler_f08
+end interface MPI_Session_call_errhandler
 
-interface  MPI_Rsend_init
-subroutine MPI_Rsend_init_f08(buf,count,datatype,dest,tag,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
+interface  MPI_Session_create_errhandler
+subroutine MPI_Session_create_errhandler_f08(session_errhandler_fn,errhandler,ierror)
+   use :: mpi_f08_types, only : MPI_Errhandler
+   use :: mpi_f08_interfaces_callbacks, only : MPI_Session_errhandler_function
    implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buf
-   INTEGER, INTENT(IN) :: count, dest, tag
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
+   PROCEDURE(MPI_Session_errhandler_function) :: session_errhandler_fn
+   TYPE(MPI_Errhandler), INTENT(OUT) :: errhandler
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Rsend_init_f08
-end interface  MPI_Rsend_init
+end subroutine MPI_Session_create_errhandler_f08
+end interface  MPI_Session_create_errhandler
 
-interface  MPI_Send
-subroutine MPI_Send_f08(buf,count,datatype,dest,tag,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
+interface MPI_Session_get_errhandler
+subroutine MPI_Session_get_errhandler_f08(session,errhandler,ierror)
+   use :: mpi_f08_types, only : MPI_Session, MPI_Errhandler
    implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count, dest, tag
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
+   TYPE(MPI_Session), INTENT(IN) :: session
+   TYPE(MPI_Errhandler), INTENT(OUT) :: errhandler
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Send_f08
-end interface  MPI_Send
+end subroutine MPI_Session_get_errhandler_f08
+end interface MPI_Session_get_errhandler
 
-interface  MPI_Sendrecv
-subroutine MPI_Sendrecv_f08(sendbuf,sendcount,sendtype,dest,sendtag,recvbuf, &
-                            recvcount,recvtype,source,recvtag,comm,status,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Status
+interface MPI_Session_get_info
+subroutine MPI_Session_get_info_f08(session, info, ierror)
+   use :: mpi_f08_types, only : MPI_Session, MPI_Info
    implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, dest, sendtag, recvcount, source, recvtag
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype, recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Status) :: status
+   TYPE(MPI_Session), INTENT(IN) :: session
+   TYPE(MPI_Info), INTENT(OUT) :: info
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Sendrecv_f08
-end interface  MPI_Sendrecv
+end subroutine MPI_Session_get_info_f08
+end interface MPI_Session_get_info
 
-interface  MPI_Sendrecv_replace
-subroutine MPI_Sendrecv_replace_f08(buf,count,datatype,dest,sendtag,source,recvtag, &
-                                    comm,status,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Status
+interface MPI_Session_get_nth_pset
+subroutine MPI_Session_get_nth_pset_f08(session, info, n, pset_len, pset_name, ierror)
+   use :: mpi_f08_types, only : MPI_Session, MPI_Info
    implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buf
-   INTEGER, INTENT(IN) :: count, dest, sendtag, source, recvtag
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Status) :: status
+   TYPE(MPI_Session), INTENT(IN) :: session
+   TYPE(MPI_Info), INTENT(IN) :: info
+   INTEGER, INTENT(IN) :: n
+   INTEGER, INTENT(INOUT) :: pset_len
+   CHARACTER(LEN=*), INTENT(OUT) :: pset_name
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Sendrecv_replace_f08
-end interface  MPI_Sendrecv_replace
+end subroutine MPI_Session_get_nth_pset_f08
+end interface MPI_Session_get_nth_pset
 
-interface  MPI_Send_init
-subroutine MPI_Send_init_f08(buf,count,datatype,dest,tag,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
+interface MPI_Session_get_num_psets
+subroutine MPI_Session_get_num_psets_f08(session, info, npset_names, ierror)
+   use :: mpi_f08_types, only : MPI_Session, MPI_Info
    implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buf
-   INTEGER, INTENT(IN) :: count, dest, tag
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
+   TYPE(MPI_Session), INTENT(IN) :: session
+   TYPE(MPI_Info), INTENT(IN) :: info
+   INTEGER, INTENT(OUT) :: npset_names
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Send_init_f08
-end interface  MPI_Send_init
+end subroutine MPI_Session_get_num_psets_f08
+end interface  MPI_Session_get_num_psets
 
-interface  MPI_Ssend
-subroutine MPI_Ssend_f08(buf,count,datatype,dest,tag,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
+interface MPI_Session_get_pset_info
+subroutine MPI_Session_get_pset_info_f08(session, pset_name, info, ierror)
+   use :: mpi_f08_types, only : MPI_Session, MPI_Info
    implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count, dest, tag
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
+   TYPE(MPI_Session), INTENT(IN) :: session
+   CHARACTER(LEN=*), INTENT(IN) :: pset_name
+   TYPE(MPI_Info), INTENT(OUT) :: info
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Ssend_f08
-end interface  MPI_Ssend
+end subroutine MPI_Session_get_pset_info_f08
+end interface MPI_Session_get_pset_info
 
-interface  MPI_Ssend_init
-subroutine MPI_Ssend_init_f08(buf,count,datatype,dest,tag,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
+interface  MPI_Session_init
+subroutine MPI_Session_init_f08(info,errhandler,session,ierror)
+   use :: mpi_f08_types, only : MPI_Session, MPI_Info, MPI_Errhandler
    implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buf
-   INTEGER, INTENT(IN) :: count, dest, tag
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
+   TYPE(MPI_Info), INTENT(IN) :: info
+   TYPE(MPI_Errhandler), INTENT(IN) :: errhandler
+   TYPE(MPI_Session), INTENT(OUT) :: session
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Ssend_init_f08
-end interface  MPI_Ssend_init
+end subroutine MPI_Session_init_f08
+end interface  MPI_Session_init
+
+interface  MPI_Session_finalize
+subroutine MPI_Session_finalize_f08(session,ierror)
+   use :: mpi_f08_types, only : MPI_Session
+   implicit none
+   TYPE(MPI_Session), INTENT(INOUT) :: session
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+end subroutine MPI_Session_finalize_f08
+end interface  MPI_Session_finalize
+
+interface MPI_Session_set_errhandler
+subroutine MPI_Session_set_errhandler_f08(session,errhandler,ierror)
+   use :: mpi_f08_types, only : MPI_Session, MPI_Errhandler
+   implicit none
+   TYPE(MPI_Session), INTENT(IN) :: session
+   TYPE(MPI_Errhandler), INTENT(IN) :: errhandler
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+end subroutine MPI_Session_set_errhandler_f08
+end interface MPI_Session_set_errhandler
 
 interface  MPI_Start
 subroutine MPI_Start_f08(request,ierror)
@@ -379,19 +261,6 @@ subroutine MPI_Testall_f08(count,array_of_requests,flag,array_of_statuses,ierror
 end subroutine MPI_Testall_f08
 end interface  MPI_Testall
 
-interface  MPI_Testany
-subroutine MPI_Testany_f08(count,array_of_requests,index,flag,status,ierror)
-   use :: mpi_f08_types, only : MPI_Request, MPI_Status
-   implicit none
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Request), INTENT(INOUT) :: array_of_requests(count)
-   INTEGER, INTENT(OUT) :: index
-   LOGICAL, INTENT(OUT) :: flag
-   TYPE(MPI_Status) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Testany_f08
-end interface  MPI_Testany
-
 interface  MPI_Testsome
 subroutine MPI_Testsome_f08(incount,array_of_requests,outcount, &
                         array_of_indices,array_of_statuses,ierror)
@@ -425,17 +294,6 @@ subroutine MPI_Wait_f08(request,status,ierror)
 end subroutine MPI_Wait_f08
 end interface  MPI_Wait
 
-interface  MPI_Waitall
-subroutine MPI_Waitall_f08(count,array_of_requests,array_of_statuses,ierror)
-   use :: mpi_f08_types, only : MPI_Request, MPI_Status
-   implicit none
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Request), INTENT(INOUT) :: array_of_requests(count)
-   TYPE(MPI_Status) :: array_of_statuses(*)
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Waitall_f08
-end interface  MPI_Waitall
-
 interface  MPI_Waitany
 subroutine MPI_Waitany_f08(count,array_of_requests,index,status,ierror)
    use :: mpi_f08_types, only : MPI_Request, MPI_Status
@@ -461,28 +319,6 @@ subroutine MPI_Waitsome_f08(incount,array_of_requests,outcount, &
 end subroutine MPI_Waitsome_f08
 end interface  MPI_Waitsome
 
-interface  MPI_Get_address
-subroutine MPI_Get_address_f08(location,address,ierror)
-   use :: mpi_f08_types, only : MPI_ADDRESS_KIND
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ location
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: location
-   INTEGER(MPI_ADDRESS_KIND), INTENT(OUT) :: address
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Get_address_f08
-end interface  MPI_Get_address
-
-interface  MPI_Get_elements
-subroutine MPI_Get_elements_f08(status,datatype,count,ierror)
-   use :: mpi_f08_types, only : MPI_Status, MPI_Datatype
-   implicit none
-   TYPE(MPI_Status), INTENT(IN) :: status
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER, INTENT(OUT) :: count
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Get_elements_f08
-end interface  MPI_Get_elements
-
 interface  MPI_Get_elements_x
 subroutine MPI_Get_elements_x_f08(status,datatype,count,ierror)
    use :: mpi_f08_types, only : MPI_Status, MPI_Datatype, MPI_COUNT_KIND
@@ -494,63 +330,6 @@ subroutine MPI_Get_elements_x_f08(status,datatype,count,ierror)
 end subroutine MPI_Get_elements_x_f08
 end interface  MPI_Get_elements_x
 
-interface  MPI_Pack
-subroutine MPI_Pack_f08(inbuf,incount,datatype,outbuf,outsize,position,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ inbuf, outbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: inbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: outbuf
-   INTEGER, INTENT(IN) :: incount, outsize
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER, INTENT(INOUT) :: position
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Pack_f08
-end interface  MPI_Pack
-
-interface  MPI_Pack_external
-subroutine MPI_Pack_external_f08(datarep,inbuf,incount,datatype,outbuf,outsize, &
-                                 position,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_ADDRESS_KIND
-   implicit none
-   CHARACTER(LEN=*), INTENT(IN) :: datarep
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ inbuf, outbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: inbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: outbuf
-   INTEGER, INTENT(IN) :: incount
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: outsize
-   INTEGER(MPI_ADDRESS_KIND), INTENT(INOUT) :: position
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Pack_external_f08
-end interface  MPI_Pack_external
-
-interface  MPI_Pack_external_size
-subroutine MPI_Pack_external_size_f08(datarep,incount,datatype,size,ierror &
-           )
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_ADDRESS_KIND
-   implicit none
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER, INTENT(IN) :: incount
-   CHARACTER(LEN=*), INTENT(IN) :: datarep
-   INTEGER(MPI_ADDRESS_KIND), INTENT(OUT) :: size
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Pack_external_size_f08
-end interface  MPI_Pack_external_size
-
-interface  MPI_Pack_size
-subroutine MPI_Pack_size_f08(incount,datatype,comm,size,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   implicit none
-   INTEGER, INTENT(IN) :: incount
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, INTENT(OUT) :: size
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Pack_size_f08
-end interface  MPI_Pack_size
-
 interface  MPI_Type_commit
 subroutine MPI_Type_commit_f08(datatype,ierror)
    use :: mpi_f08_types, only : MPI_Datatype
@@ -560,84 +339,6 @@ subroutine MPI_Type_commit_f08(datatype,ierror)
 end subroutine MPI_Type_commit_f08
 end interface  MPI_Type_commit
 
-interface  MPI_Type_contiguous
-subroutine MPI_Type_contiguous_f08(count,oldtype,newtype,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype
-   implicit none
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: oldtype
-   TYPE(MPI_Datatype), INTENT(OUT) :: newtype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Type_contiguous_f08
-end interface  MPI_Type_contiguous
-
-interface  MPI_Type_create_darray
-subroutine MPI_Type_create_darray_f08(size,rank,ndims,array_of_gsizes, &
-                    array_of_distribs,array_of_dargs,array_of_psizes,order, &
-                    oldtype,newtype,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype
-   implicit none
-   INTEGER, INTENT(IN) :: size, rank, ndims, order
-   INTEGER, INTENT(IN) :: array_of_gsizes(ndims), array_of_distribs(ndims)
-   INTEGER, INTENT(IN) :: array_of_dargs(ndims), array_of_psizes(ndims)
-   TYPE(MPI_Datatype), INTENT(IN) :: oldtype
-   TYPE(MPI_Datatype), INTENT(OUT) :: newtype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Type_create_darray_f08
-end interface  MPI_Type_create_darray
-
-interface  MPI_Type_create_hindexed
-subroutine MPI_Type_create_hindexed_f08(count,array_of_blocklengths, &
-                                        array_of_displacements,oldtype,newtype,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_ADDRESS_KIND
-   implicit none
-   INTEGER, INTENT(IN) :: count
-   INTEGER, INTENT(IN) :: array_of_blocklengths(count)
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: array_of_displacements(count)
-   TYPE(MPI_Datatype), INTENT(IN) :: oldtype
-   TYPE(MPI_Datatype), INTENT(OUT) :: newtype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Type_create_hindexed_f08
-end interface  MPI_Type_create_hindexed
-
-interface  MPI_Type_create_hvector
-subroutine MPI_Type_create_hvector_f08(count,blocklength,stride,oldtype,newtype,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_ADDRESS_KIND
-   implicit none
-   INTEGER, INTENT(IN) :: count, blocklength
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: stride
-   TYPE(MPI_Datatype), INTENT(IN) :: oldtype
-   TYPE(MPI_Datatype), INTENT(OUT) :: newtype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Type_create_hvector_f08
-end interface  MPI_Type_create_hvector
-
-interface  MPI_Type_create_indexed_block
-subroutine MPI_Type_create_indexed_block_f08(count,blocklength, &
-                           array_of_displacements,oldtype,newtype,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype
-   implicit none
-   INTEGER, INTENT(IN) :: count, blocklength
-   INTEGER, INTENT(IN) :: array_of_displacements(count)
-   TYPE(MPI_Datatype), INTENT(IN) :: oldtype
-   TYPE(MPI_Datatype), INTENT(OUT) :: newtype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Type_create_indexed_block_f08
-end interface  MPI_Type_create_indexed_block
-
-interface  MPI_Type_create_hindexed_block
-subroutine MPI_Type_create_hindexed_block_f08(count,blocklength, &
-                           array_of_displacements,oldtype,newtype,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_ADDRESS_KIND
-   implicit none
-   INTEGER, INTENT(IN) :: count, blocklength
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: array_of_displacements(count)
-   TYPE(MPI_Datatype), INTENT(IN) :: oldtype
-   TYPE(MPI_Datatype), INTENT(OUT) :: newtype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Type_create_hindexed_block_f08
-end interface  MPI_Type_create_hindexed_block
-
 interface  MPI_Type_create_resized
 subroutine MPI_Type_create_resized_f08(oldtype,lb,extent,newtype,ierror)
    use :: mpi_f08_types, only : MPI_Datatype, MPI_ADDRESS_KIND
@@ -649,34 +350,6 @@ subroutine MPI_Type_create_resized_f08(oldtype,lb,extent,newtype,ierror)
 end subroutine MPI_Type_create_resized_f08
 end interface  MPI_Type_create_resized
 
-interface  MPI_Type_create_struct
-subroutine MPI_Type_create_struct_f08(count,array_of_blocklengths, &
-                           array_of_displacements,array_of_types,newtype,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_ADDRESS_KIND
-   implicit none
-   INTEGER, INTENT(IN) :: count
-   INTEGER, INTENT(IN) :: array_of_blocklengths(count)
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: array_of_displacements(count)
-   TYPE(MPI_Datatype), INTENT(IN) :: array_of_types(count)
-   TYPE(MPI_Datatype), INTENT(OUT) :: newtype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Type_create_struct_f08
-end interface  MPI_Type_create_struct
-
-interface  MPI_Type_create_subarray
-subroutine MPI_Type_create_subarray_f08(ndims,array_of_sizes,array_of_subsizes, &
-                    array_of_starts,order,oldtype,newtype,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype
-   implicit none
-   INTEGER, INTENT(IN) :: ndims, order
-   INTEGER, INTENT(IN) :: array_of_sizes(ndims), array_of_subsizes(ndims)
-   INTEGER, INTENT(IN) :: array_of_starts(ndims)
-   TYPE(MPI_Datatype), INTENT(IN) :: oldtype
-   TYPE(MPI_Datatype), INTENT(OUT) :: newtype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Type_create_subarray_f08
-end interface  MPI_Type_create_subarray
-
 interface  MPI_Type_dup
 subroutine MPI_Type_dup_f08(oldtype,newtype,ierror)
    use :: mpi_f08_types, only : MPI_Datatype
@@ -709,6 +382,20 @@ subroutine MPI_Type_get_contents_f08(datatype,max_integers,max_addresses,max_dat
    TYPE(MPI_Datatype), INTENT(OUT) :: array_of_datatypes(max_datatypes)
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
 end subroutine MPI_Type_get_contents_f08
+subroutine MPI_Type_get_contents_f08_c(datatype, max_integers, max_addresses, max_large_counts, &
+                                       max_datatypes, array_of_integers, array_of_addresses, &
+                                       array_of_large_counts, array_of_datatypes, ierror)
+   use :: mpi_f08_types, only : MPI_Datatype, MPI_ADDRESS_KIND, MPI_COUNT_KIND
+   implicit none
+   TYPE(MPI_Datatype), INTENT(IN) :: datatype
+   INTEGER(KIND=MPI_COUNT_KIND), INTENT(IN) :: max_integers, max_addresses, &
+                                               max_large_counts, max_datatypes
+   INTEGER, INTENT(OUT) :: array_of_integers(max_integers)
+   INTEGER(MPI_ADDRESS_KIND), INTENT(OUT) :: array_of_addresses(max_addresses)
+   INTEGER(KIND=MPI_COUNT_KIND), INTENT(OUT) :: array_of_large_counts(max_large_counts)
+   TYPE(MPI_Datatype), INTENT(OUT) :: array_of_datatypes(max_datatypes)
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+end subroutine MPI_Type_get_contents_f08_c
 end interface  MPI_Type_get_contents
 
 interface  MPI_Type_get_envelope
@@ -720,6 +407,15 @@ subroutine MPI_Type_get_envelope_f08(datatype,num_integers,num_addresses,num_dat
    INTEGER, INTENT(OUT) :: num_integers, num_addresses, num_datatypes, combiner
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
 end subroutine MPI_Type_get_envelope_f08
+subroutine MPI_Type_get_envelope_f08_c(datatype,num_integers,num_addresses,num_large_counts,num_datatypes, &
+                                     combiner,ierror)
+   use :: mpi_f08_types, only : MPI_Datatype, MPI_COUNT_KIND
+   implicit none
+   TYPE(MPI_Datatype), INTENT(IN) :: datatype
+   INTEGER(MPI_COUNT_KIND), INTENT(OUT) :: num_integers, num_addresses, num_large_counts, num_datatypes
+   INTEGER, INTENT(OUT) :: combiner
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+end subroutine MPI_Type_get_envelope_f08_c
 end interface  MPI_Type_get_envelope
 
 interface  MPI_Type_get_extent
@@ -742,16 +438,6 @@ subroutine MPI_Type_get_extent_x_f08(datatype,lb,extent,ierror)
 end subroutine MPI_Type_get_extent_x_f08
 end interface  MPI_Type_get_extent_x
 
-interface  MPI_Type_get_true_extent
-subroutine MPI_Type_get_true_extent_f08(datatype,true_lb,true_extent,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_ADDRESS_KIND
-   implicit none
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER(MPI_ADDRESS_KIND), INTENT(OUT) :: true_lb, true_extent
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Type_get_true_extent_f08
-end interface  MPI_Type_get_true_extent
-
 interface  MPI_Type_get_true_extent_x
 subroutine MPI_Type_get_true_extent_x_f08(datatype,true_lb,true_extent,ierror)
    use :: mpi_f08_types, only : MPI_Datatype, MPI_ADDRESS_KIND, MPI_COUNT_KIND
@@ -762,29 +448,6 @@ subroutine MPI_Type_get_true_extent_x_f08(datatype,true_lb,true_extent,ierror)
 end subroutine MPI_Type_get_true_extent_x_f08
 end interface  MPI_Type_get_true_extent_x
 
-interface  MPI_Type_indexed
-subroutine MPI_Type_indexed_f08(count,array_of_blocklengths, &
-                                array_of_displacements,oldtype,newtype,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype
-   implicit none
-   INTEGER, INTENT(IN) :: count
-   INTEGER, INTENT(IN) :: array_of_blocklengths(count), array_of_displacements(count)
-   TYPE(MPI_Datatype), INTENT(IN) :: oldtype
-   TYPE(MPI_Datatype), INTENT(OUT) :: newtype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Type_indexed_f08
-end interface  MPI_Type_indexed
-
-interface  MPI_Type_size
-subroutine MPI_Type_size_f08(datatype,size,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype
-   implicit none
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER, INTENT(OUT) :: size
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Type_size_f08
-end interface  MPI_Type_size
-
 interface  MPI_Type_size_x
 subroutine MPI_Type_size_x_f08(datatype,size,ierror)
    use :: mpi_f08_types, only : MPI_Datatype, MPI_COUNT_KIND
@@ -795,379 +458,35 @@ subroutine MPI_Type_size_x_f08(datatype,size,ierror)
 end subroutine MPI_Type_size_x_f08
 end interface  MPI_Type_size_x
 
-interface  MPI_Type_vector
-subroutine MPI_Type_vector_f08(count,blocklength,stride,oldtype,newtype,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype
+interface  MPI_Barrier
+subroutine MPI_Barrier_f08(comm,ierror)
+   use :: mpi_f08_types, only : MPI_Comm
    implicit none
-   INTEGER, INTENT(IN) :: count, blocklength, stride
-   TYPE(MPI_Datatype), INTENT(IN) :: oldtype
-   TYPE(MPI_Datatype), INTENT(OUT) :: newtype
+   TYPE(MPI_Comm), INTENT(IN) :: comm
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Type_vector_f08
-end interface  MPI_Type_vector
+end subroutine MPI_Barrier_f08
+end interface  MPI_Barrier
 
-interface  MPI_Unpack
-subroutine MPI_Unpack_f08(inbuf,insize,position,outbuf,outcount,datatype,comm, &
-                          ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
+interface  MPI_Ibarrier
+subroutine MPI_Ibarrier_f08(comm,request,ierror)
+   use :: mpi_f08_types, only : MPI_Comm, MPI_Request
    implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ inbuf, outbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: inbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: outbuf
-   INTEGER, INTENT(IN) :: insize, outcount
-   INTEGER, INTENT(INOUT) :: position
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
    TYPE(MPI_Comm), INTENT(IN) :: comm
+   TYPE(MPI_Request), INTENT(OUT) :: request
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Unpack_f08
-end interface  MPI_Unpack
+end subroutine MPI_Ibarrier_f08
+end interface  MPI_Ibarrier
 
-interface  MPI_Unpack_external
-subroutine MPI_Unpack_external_f08(datarep,inbuf,insize,position,outbuf,outcount, &
-                                   datatype,ierror &
-           )
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_ADDRESS_KIND
+interface  MPI_Barrier_init
+subroutine MPI_Barrier_init_f08(comm,info,request,ierror)
+   use :: mpi_f08_types, only : MPI_Comm, MPI_Info, MPI_Request
    implicit none
-   CHARACTER(LEN=*), INTENT(IN) :: datarep
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ inbuf, outbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: inbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: outbuf
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: insize
-   INTEGER(MPI_ADDRESS_KIND), INTENT(INOUT) :: position
-   INTEGER, INTENT(IN) :: outcount
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
+   TYPE(MPI_Comm), INTENT(IN) :: comm
+   TYPE(MPI_Info), INTENT(IN) :: info
+   TYPE(MPI_Request), INTENT(OUT) :: request
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Unpack_external_f08
-end interface  MPI_Unpack_external
-
-interface  MPI_Allgather
-subroutine MPI_Allgather_f08(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype, &
-                             comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype, recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Allgather_f08
-end interface  MPI_Allgather
-
-interface  MPI_Iallgather
-subroutine MPI_Iallgather_f08(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype, &
-                             comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype, recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Iallgather_f08
-end interface  MPI_Iallgather
-
-interface  MPI_Allgatherv
-subroutine MPI_Allgatherv_f08(sendbuf,sendcount,sendtype,recvbuf,recvcounts,displs, &
-                              recvtype,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount
-   INTEGER, INTENT(IN) :: recvcounts(*), displs(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype, recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Allgatherv_f08
-end interface  MPI_Allgatherv
-
-interface  MPI_Iallgatherv
-subroutine MPI_Iallgatherv_f08(sendbuf,sendcount,sendtype,recvbuf,recvcounts,displs, &
-                              recvtype,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: recvcounts(*), displs(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype, recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Iallgatherv_f08
-end interface  MPI_Iallgatherv
-
-interface  MPI_Allreduce
-subroutine MPI_Allreduce_f08(sendbuf,recvbuf,count,datatype,op,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Allreduce_f08
-end interface  MPI_Allreduce
-
-interface  MPI_Iallreduce
-subroutine MPI_Iallreduce_f08(sendbuf,recvbuf,count,datatype,op,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm, MPI_Request
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Iallreduce_f08
-end interface  MPI_Iallreduce
-
-interface  MPI_Alltoall
-subroutine MPI_Alltoall_f08(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype, &
-                            comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype, recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Alltoall_f08
-end interface  MPI_Alltoall
-
-interface  MPI_Ialltoall
-subroutine MPI_Ialltoall_f08(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype, &
-                            comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: sendcount, recvcount
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype, recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Ialltoall_f08
-end interface  MPI_Ialltoall
-
-interface  MPI_Alltoallv
-subroutine MPI_Alltoallv_f08(sendbuf,sendcounts,sdispls,sendtype,recvbuf,recvcounts, &
-                             rdispls,recvtype,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
-   INTEGER, INTENT(IN) :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype, recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Alltoallv_f08
-end interface  MPI_Alltoallv
-
-interface  MPI_Ialltoallv
-subroutine MPI_Ialltoallv_f08(sendbuf,sendcounts,sdispls,sendtype,recvbuf,recvcounts, &
-                             rdispls,recvtype,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype, recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Ialltoallv_f08
-end interface  MPI_Ialltoallv
-
-interface  MPI_Alltoallw
-subroutine MPI_Alltoallw_f08(sendbuf,sendcounts,sdispls,sendtypes,recvbuf,recvcounts, &
-                             rdispls,recvtypes,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
-   INTEGER, INTENT(IN) :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtypes(*), recvtypes(*)
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Alltoallw_f08
-end interface  MPI_Alltoallw
-
-interface  MPI_Ialltoallw
-subroutine MPI_Ialltoallw_f08(sendbuf,sendcounts,sdispls,sendtypes,recvbuf,recvcounts, &
-                             rdispls,recvtypes,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
-   TYPE(MPI_Datatype), INTENT(IN), ASYNCHRONOUS :: sendtypes(*), recvtypes(*)
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Ialltoallw_f08
-end interface  MPI_Ialltoallw
-
-interface  MPI_Barrier
-subroutine MPI_Barrier_f08(comm,ierror)
-   use :: mpi_f08_types, only : MPI_Comm
-   implicit none
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Barrier_f08
-end interface  MPI_Barrier
-
-interface  MPI_Ibarrier
-subroutine MPI_Ibarrier_f08(comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Comm, MPI_Request
-   implicit none
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Ibarrier_f08
-end interface  MPI_Ibarrier
-
-interface  MPI_Bcast
-subroutine MPI_Bcast_f08(buffer,count,datatype,root,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buffer
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buffer
-   INTEGER, INTENT(IN) :: count, root
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Bcast_f08
-end interface  MPI_Bcast
-
-interface  MPI_Ibcast
-subroutine MPI_Ibcast_f08(buffer,count,datatype,root,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buffer
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, ASYNCHRONOUS :: buffer
-   INTEGER, INTENT(IN) :: count, root
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Ibcast_f08
-end interface  MPI_Ibcast
-
-interface  MPI_Exscan
-subroutine MPI_Exscan_f08(sendbuf,recvbuf,count,datatype,op,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Exscan_f08
-end interface  MPI_Exscan
-
-interface  MPI_Iexscan
-subroutine MPI_Iexscan_f08(sendbuf,recvbuf,count,datatype,op,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm, MPI_Request
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Iexscan_f08
-end interface  MPI_Iexscan
-
-interface  MPI_Gather
-subroutine MPI_Gather_f08(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype, &
-                          root,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount, root
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype, recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Gather_f08
-end interface  MPI_Gather
-
-interface  MPI_Igather
-subroutine MPI_Igather_f08(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype, &
-                          root,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount, root
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype, recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Igather_f08
-end interface  MPI_Igather
-
-interface  MPI_Gatherv
-subroutine MPI_Gatherv_f08(sendbuf,sendcount,sendtype,recvbuf,recvcounts,displs, &
-                           recvtype,root,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, root
-   INTEGER, INTENT(IN) :: recvcounts(*), displs(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype, recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Gatherv_f08
-end interface  MPI_Gatherv
-
-interface  MPI_Igatherv
-subroutine MPI_Igatherv_f08(sendbuf,sendcount,sendtype,recvbuf,recvcounts,displs, &
-                           recvtype,root,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, root
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: recvcounts(*), displs(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype, recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Igatherv_f08
-end interface  MPI_Igatherv
+end subroutine MPI_Barrier_init_f08
+end interface  MPI_Barrier_init
 
 interface  MPI_Op_commutative
 subroutine MPI_Op_commutative_f08(op,commute,ierror)
@@ -1200,212 +519,6 @@ subroutine MPI_Op_free_f08(op,ierror)
 end subroutine MPI_Op_free_f08
 end interface  MPI_Op_free
 
-interface  MPI_Reduce
-subroutine MPI_Reduce_f08(sendbuf,recvbuf,count,datatype,op,root,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
-   INTEGER, INTENT(IN) :: count, root
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Reduce_f08
-end interface  MPI_Reduce
-
-interface  MPI_Ireduce
-subroutine MPI_Ireduce_f08(sendbuf,recvbuf,count,datatype,op,root,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm, MPI_Request
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: count, root
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Ireduce_f08
-end interface  MPI_Ireduce
-
-interface  MPI_Reduce_local
-subroutine MPI_Reduce_local_f08(inbuf,inoutbuf,count,datatype,op,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ inbuf, inoutbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: inbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: inoutbuf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Reduce_local_f08
-end interface  MPI_Reduce_local
-
-interface  MPI_Reduce_scatter
-subroutine MPI_Reduce_scatter_f08(sendbuf,recvbuf,recvcounts,datatype,op,comm, &
-                                  ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
-   INTEGER, INTENT(IN) :: recvcounts(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Reduce_scatter_f08
-end interface  MPI_Reduce_scatter
-
-interface  MPI_Ireduce_scatter
-subroutine MPI_Ireduce_scatter_f08(sendbuf,recvbuf,recvcounts,datatype,op,comm, &
-                                  request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm, MPI_Request
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: recvcounts(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Ireduce_scatter_f08
-end interface  MPI_Ireduce_scatter
-
-interface  MPI_Reduce_scatter_block
-subroutine MPI_Reduce_scatter_block_f08(sendbuf,recvbuf,recvcount,datatype,op,comm, &
-                                        ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
-   INTEGER, INTENT(IN) :: recvcount
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Reduce_scatter_block_f08
-end interface  MPI_Reduce_scatter_block
-
-interface  MPI_Ireduce_scatter_block
-subroutine MPI_Ireduce_scatter_block_f08(sendbuf,recvbuf,recvcount,datatype,op,comm, &
-                                        request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm, MPI_Request
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: recvcount
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Ireduce_scatter_block_f08
-end interface  MPI_Ireduce_scatter_block
-
-interface  MPI_Scan
-subroutine MPI_Scan_f08(sendbuf,recvbuf,count,datatype,op,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Scan_f08
-end interface  MPI_Scan
-
-interface  MPI_Iscan
-subroutine MPI_Iscan_f08(sendbuf,recvbuf,count,datatype,op,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm, MPI_Request
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Iscan_f08
-end interface  MPI_Iscan
-
-interface  MPI_Scatter
-subroutine MPI_Scatter_f08(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype, &
-                           root,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount, root
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype, recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Scatter_f08
-end interface  MPI_Scatter
-
-interface  MPI_Iscatter
-subroutine MPI_Iscatter_f08(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype, &
-                           root,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount, root
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype, recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Iscatter_f08
-end interface  MPI_Iscatter
-
-interface  MPI_Scatterv
-subroutine MPI_Scatterv_f08(sendbuf,sendcounts,displs,sendtype,recvbuf,recvcount, &
-                            recvtype,root,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
-   INTEGER, INTENT(IN) :: recvcount, root
-   INTEGER, INTENT(IN) :: sendcounts(*), displs(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype, recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Scatterv_f08
-end interface  MPI_Scatterv
-
-interface  MPI_Iscatterv
-subroutine MPI_Iscatterv_f08(sendbuf,sendcounts,displs,sendtype,recvbuf,recvcount, &
-                            recvtype,root,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: recvcount, root
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: sendcounts(*), displs(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype, recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Iscatterv_f08
-end interface  MPI_Iscatterv
-
 interface  MPI_Comm_compare
 subroutine MPI_Comm_compare_f08(comm1,comm2,result,ierror)
    use :: mpi_f08_types, only : MPI_Comm
@@ -1428,6 +541,20 @@ subroutine MPI_Comm_create_f08(comm,group,newcomm,ierror)
 end subroutine MPI_Comm_create_f08
 end interface  MPI_Comm_create
 
+interface  MPI_Comm_create_from_group
+subroutine MPI_Comm_create_from_group_f08(group, stringtag, info, errhandler, newcomm, ierror)
+   use :: mpi_f08_types, only : MPI_Comm, MPI_Group, MPI_Info, MPI_Errhandler
+   implicit none
+   TYPE(MPI_Group), INTENT(IN) :: group
+   CHARACTER(LEN=*), INTENT(IN) :: stringtag
+   TYPE(MPI_Info), INTENT(IN) :: info
+   TYPE(MPI_Errhandler), INTENT(IN) :: errhandler
+   TYPE(MPI_Comm), INTENT(OUT) :: newcomm
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+
+end subroutine MPI_Comm_create_from_group_f08
+end interface  MPI_Comm_create_from_group
+
 interface  MPI_Comm_create_group
 subroutine MPI_Comm_create_group_f08(comm,group,tag,newcomm,ierror)
    use :: mpi_f08_types, only : MPI_Comm, MPI_Group
@@ -1491,12 +618,25 @@ subroutine MPI_Comm_idup_f08(comm,newcomm,request,ierror)
    use :: mpi_f08_types, only : MPI_Comm, MPI_Request
    implicit none
    TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Comm), INTENT(OUT) :: newcomm
+   TYPE(MPI_Comm), INTENT(OUT) OMPI_ASYNCHRONOUS :: newcomm
    TYPE(MPI_Request), INTENT(OUT) :: request
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
 end subroutine MPI_Comm_idup_f08
 end interface  MPI_Comm_idup
 
+interface  MPI_Comm_idup_with_info
+subroutine MPI_Comm_idup_with_info_f08(comm,info,newcomm,request,ierror)
+   use :: mpi_f08_types, only : MPI_Comm, MPI_Request, MPI_info
+   implicit none
+   TYPE(MPI_Comm), INTENT(IN) :: comm
+   TYPE(MPI_Info), INTENT(IN) :: info
+   TYPE(MPI_Comm), INTENT(OUT) :: newcomm
+   TYPE(MPI_Request), INTENT(OUT) :: request
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+end subroutine MPI_Comm_idup_with_info_f08
+end interface  MPI_Comm_idup_with_info
+
+
 interface  MPI_Comm_free
 subroutine MPI_Comm_free_f08(comm,ierror)
    use :: mpi_f08_types, only : MPI_Comm
@@ -1689,6 +829,17 @@ subroutine MPI_Group_free_f08(group,ierror)
 end subroutine MPI_Group_free_f08
 end interface  MPI_Group_free
 
+interface MPI_Group_from_session_pset
+subroutine MPI_Group_from_session_pset_f08(session, pset_name, newgroup, ierror)
+   use :: mpi_f08_types, only : MPI_Session, MPI_Group
+   implicit none
+   TYPE(MPI_Session), INTENT(IN) :: session
+   CHARACTER(LEN=*), INTENT(IN) :: pset_name
+   TYPE(MPI_Group), INTENT(OUT) :: newgroup
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+end subroutine MPI_Group_from_session_pset_f08
+end interface MPI_Group_from_session_pset
+
 interface  MPI_Group_incl
 subroutine MPI_Group_incl_f08(group,n,ranks,newgroup,ierror)
    use :: mpi_f08_types, only : MPI_Group
@@ -1874,36 +1025,6 @@ subroutine MPI_Type_set_name_f08(datatype,type_name,ierror)
 end subroutine MPI_Type_set_name_f08
 end interface  MPI_Type_set_name
 
-interface  MPI_Win_allocate
-subroutine MPI_Win_allocate_f08(size, disp_unit, info, comm, &
-      baseptr, win, ierror)
-  USE, INTRINSIC ::  ISO_C_BINDING, ONLY : C_PTR
-  use :: mpi_f08_types, only : MPI_Info, MPI_Comm, MPI_Win, MPI_ADDRESS_KIND
-  INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) ::  size
-  INTEGER, INTENT(IN) ::  disp_unit
-  TYPE(MPI_Info), INTENT(IN) ::  info
-  TYPE(MPI_Comm), INTENT(IN) ::  comm
-  TYPE(C_PTR), INTENT(OUT) ::  baseptr
-  TYPE(MPI_Win), INTENT(OUT) ::  win
-  INTEGER, OPTIONAL, INTENT(OUT) ::  ierror
-end subroutine MPI_Win_allocate_f08
-end interface  MPI_Win_allocate
-
-interface  MPI_Win_allocate_shared
-subroutine MPI_Win_allocate_shared_f08(size, disp_unit, info, comm, &
-      baseptr, win, ierror)
-  USE, INTRINSIC ::  ISO_C_BINDING, ONLY : C_PTR
-  use :: mpi_f08_types, only : MPI_Info, MPI_Comm, MPI_Win, MPI_ADDRESS_KIND
-  INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) ::  size
-  INTEGER, INTENT(IN) ::  disp_unit
-  TYPE(MPI_Info), INTENT(IN) ::  info
-  TYPE(MPI_Comm), INTENT(IN) ::  comm
-  TYPE(C_PTR), INTENT(OUT) ::  baseptr
-  TYPE(MPI_Win), INTENT(OUT) ::  win
-  INTEGER, OPTIONAL, INTENT(OUT) ::  ierror
-end subroutine MPI_Win_allocate_shared_f08
-end interface  MPI_Win_allocate_shared
-
 interface  MPI_Win_create_keyval
 subroutine MPI_Win_create_keyval_f08(win_copy_attr_fn,win_delete_attr_fn,win_keyval, &
                                      extra_state,ierror)
@@ -1950,11 +1071,11 @@ end subroutine MPI_Win_get_attr_f08
 end interface  MPI_Win_get_attr
 
 interface  MPI_Win_get_info
-subroutine MPI_Win_get_info_f08(win,info,ierror)
+subroutine MPI_Win_get_info_f08(win,info_used,ierror)
    use :: mpi_f08_types, only : MPI_Win, MPI_Info
    implicit none
    TYPE(MPI_Win), INTENT(IN) :: win
-   TYPE(MPI_Info), INTENT(OUT) :: info
+   TYPE(MPI_Info), INTENT(OUT) :: info_used
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
 end subroutine MPI_Win_get_info_f08
 end interface  MPI_Win_get_info
@@ -2232,11 +1353,11 @@ end subroutine MPI_Topo_test_f08
 end interface  MPI_Topo_test
 
 interface MPI_Aint_add
-function  MPI_Aint_add_f08(base,diff)
+function  MPI_Aint_add_f08(base,disp)
    use :: mpi_f08_types, only : MPI_ADDRESS_KIND
    implicit none
-   INTEGER(MPI_ADDRESS_KIND) :: base
-   INTEGER(MPI_ADDRESS_KIND) :: diff
+   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: base
+   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: disp
    INTEGER(MPI_ADDRESS_KIND) :: MPI_Aint_add_f08
 end function  MPI_Aint_add_f08
 end interface MPI_Aint_add
@@ -2245,8 +1366,8 @@ interface MPI_Aint_diff
 function  MPI_Aint_diff_f08(addr1,addr2)
    use :: mpi_f08_types, only : MPI_ADDRESS_KIND
    implicit none
-   INTEGER(MPI_ADDRESS_KIND) :: addr1
-   INTEGER(MPI_ADDRESS_KIND) :: addr2
+   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: addr1
+   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: addr2
    INTEGER(MPI_ADDRESS_KIND) :: MPI_Aint_diff_f08
 end function  MPI_Aint_diff_f08
 end interface MPI_Aint_diff
@@ -2425,18 +1546,6 @@ subroutine MPI_Finalized_f08(flag,ierror)
 end subroutine MPI_Finalized_f08
 end interface  MPI_Finalized
 
-! ASYNCHRONOUS had to removed from the base argument because
-! the dummy argument is not an assumed-shape array.  This will
-! be okay once the Interop TR is implemented.
-interface  MPI_Free_mem
-subroutine MPI_Free_mem_f08(base,ierror)
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ base
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: base
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Free_mem_f08
-end interface  MPI_Free_mem
-
 interface  MPI_Get_processor_name
 subroutine MPI_Get_processor_name_f08(name,resultlen,ierror)
    use :: mpi_f08_types, only : MPI_MAX_PROCESSOR_NAME
@@ -2520,6 +1629,15 @@ subroutine MPI_Info_create_f08(info,ierror)
 end subroutine MPI_Info_create_f08
 end interface  MPI_Info_create
 
+interface  MPI_Info_create_env
+subroutine MPI_Info_create_env_f08(info,ierror)
+   use :: mpi_f08_types, only : MPI_Info
+   implicit none
+   TYPE(MPI_Info), INTENT(OUT) :: info
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+end subroutine MPI_Info_create_env_f08
+end interface  MPI_Info_create_env
+
 interface  MPI_Info_delete
 subroutine MPI_Info_delete_f08(info,key,ierror)
    use :: mpi_f08_types, only : MPI_Info
@@ -2605,6 +1723,19 @@ subroutine MPI_Info_set_f08(info,key,value,ierror)
 end subroutine MPI_Info_set_f08
 end interface  MPI_Info_set
 
+interface  MPI_Info_get_string
+subroutine MPI_Info_get_string_f08(info,key,buflen,value,flag,ierror)
+   use :: mpi_f08_types, only : MPI_Info
+   implicit none
+   TYPE(MPI_Info), INTENT(IN) :: info
+   CHARACTER(LEN=*), INTENT(IN) :: key
+   INTEGER, INTENT(INOUT) :: buflen
+   CHARACTER(LEN=*), INTENT(OUT) :: value
+   LOGICAL, INTENT(OUT) :: flag
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+end subroutine MPI_Info_get_string_f08
+end interface  MPI_Info_get_string
+
 interface  MPI_Close_port
 subroutine MPI_Close_port_f08(port_name,ierror)
    implicit none
@@ -2739,205 +1870,14 @@ subroutine MPI_Unpublish_name_f08(service_name,info,port_name,ierror)
 end subroutine MPI_Unpublish_name_f08
 end interface  MPI_Unpublish_name
 
-interface  MPI_Accumulate
-subroutine MPI_Accumulate_f08(origin_addr,origin_count,origin_datatype,target_rank, &
-                              target_disp,target_count,target_datatype,op,win,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Win, MPI_ADDRESS_KIND
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ origin_addr
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: origin_addr
-   INTEGER, INTENT(IN) :: origin_count, target_rank, target_count
-   TYPE(MPI_Datatype), INTENT(IN) :: origin_datatype
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
-   TYPE(MPI_Datatype), INTENT(IN) :: target_datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Win), INTENT(IN) :: win
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Accumulate_f08
-end interface  MPI_Accumulate
-
-interface  MPI_Raccumulate
-subroutine MPI_Raccumulate_f08(origin_addr,origin_count,origin_datatype,target_rank, &
-                               target_disp,target_count,target_datatype,op,win,request, &
-                               ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Win, MPI_Request, MPI_ADDRESS_KIND
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ origin_addr
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: origin_addr
-   INTEGER, INTENT(IN) :: origin_count, target_rank, target_count
-   TYPE(MPI_Datatype), INTENT(IN) :: origin_datatype
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
-   TYPE(MPI_Datatype), INTENT(IN) :: target_datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Win), INTENT(IN) :: win
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Raccumulate_f08
-end interface  MPI_Raccumulate
-
-interface  MPI_Get
-subroutine MPI_Get_f08(origin_addr,origin_count,origin_datatype,target_rank, &
-                               target_disp,target_count,target_datatype,win,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Win, MPI_ADDRESS_KIND
+interface  MPI_Win_complete
+subroutine MPI_Win_complete_f08(win,ierror)
+   use :: mpi_f08_types, only : MPI_Info, MPI_Comm, MPI_Win, MPI_ADDRESS_KIND
    implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ origin_addr
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: origin_addr
-   INTEGER, INTENT(IN) :: origin_count, target_rank, target_count
-   TYPE(MPI_Datatype), INTENT(IN) :: origin_datatype
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
-   TYPE(MPI_Datatype), INTENT(IN) :: target_datatype
    TYPE(MPI_Win), INTENT(IN) :: win
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Get_f08
-end interface  MPI_Get
-
-interface  MPI_Rget
-subroutine MPI_Rget_f08(origin_addr,origin_count,origin_datatype,target_rank, &
-                        target_disp,target_count,target_datatype,win,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Request, MPI_Win, MPI_ADDRESS_KIND
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ origin_addr
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: origin_addr
-   INTEGER, INTENT(IN) :: origin_count, target_rank, target_count
-   TYPE(MPI_Datatype), INTENT(IN) :: origin_datatype
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
-   TYPE(MPI_Datatype), INTENT(IN) :: target_datatype
-   TYPE(MPI_Win), INTENT(IN) :: win
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Rget_f08
-end interface  MPI_Rget
-
-interface  MPI_Get_accumulate
-subroutine MPI_Get_accumulate_f08(origin_addr,origin_count,origin_datatype,result_addr, &
-                                  result_count,result_datatype,target_rank,target_disp, &
-                                  target_count,target_datatype,op,win,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Win, MPI_ADDRESS_KIND
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ origin_addr,result_addr
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: origin_addr
-   INTEGER, INTENT(IN) :: origin_count, result_count, target_rank, target_count
-   TYPE(MPI_Datatype), INTENT(IN) :: origin_datatype
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: result_addr
-   TYPE(MPI_Datatype), INTENT(IN) :: result_datatype
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
-   TYPE(MPI_Datatype), INTENT(IN) :: target_datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Win), INTENT(IN) :: win
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Get_accumulate_f08
-end interface  MPI_Get_accumulate
-
-interface  MPI_Rget_accumulate
-subroutine MPI_Rget_accumulate_f08(origin_addr,origin_count,origin_datatype,result_addr, &
-                                   result_count,result_datatype,target_rank,target_disp, &
-                                   target_count,target_datatype,op,win,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Request, MPI_Win, MPI_ADDRESS_KIND
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ origin_addr,result_addr
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: origin_addr
-   INTEGER, INTENT(IN) :: origin_count, result_count, target_rank, target_count
-   TYPE(MPI_Datatype), INTENT(IN) :: origin_datatype
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: result_addr
-   TYPE(MPI_Datatype), INTENT(IN) :: result_datatype
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
-   TYPE(MPI_Datatype), INTENT(IN) :: target_datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Win), INTENT(IN) :: win
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Rget_accumulate_f08
-end interface  MPI_Rget_accumulate
-
-interface  MPI_Put
-subroutine MPI_Put_f08(origin_addr,origin_count,origin_datatype,target_rank, &
-                               target_disp,target_count,target_datatype,win,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Win, MPI_ADDRESS_KIND
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ origin_addr
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: origin_addr
-   INTEGER, INTENT(IN) :: origin_count, target_rank, target_count
-   TYPE(MPI_Datatype), INTENT(IN) :: origin_datatype
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
-   TYPE(MPI_Datatype), INTENT(IN) :: target_datatype
-   TYPE(MPI_Win), INTENT(IN) :: win
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Put_f08
-end interface  MPI_Put
-
-interface  MPI_Rput
-subroutine MPI_Rput_f08(origin_addr,origin_count,origin_datatype,target_rank, &
-                        target_disp,target_count,target_datatype,win,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Win, MPI_Request, MPI_ADDRESS_KIND
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ origin_addr
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: origin_addr
-   INTEGER, INTENT(IN) :: origin_count, target_rank, target_count
-   TYPE(MPI_Datatype), INTENT(IN) :: origin_datatype
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
-   TYPE(MPI_Datatype), INTENT(IN) :: target_datatype
-   TYPE(MPI_Win), INTENT(IN) :: win
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Rput_f08
-end interface  MPI_Rput
-
-interface  MPI_Fetch_and_op
-subroutine MPI_Fetch_and_op_f08(origin_addr,result_addr,datatype,target_rank, &
-                                target_disp,op,win,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Win, MPI_ADDRESS_KIND
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ origin_addr,result_addr
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: origin_addr
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: result_addr
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER, INTENT(IN) :: target_rank
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Win), INTENT(IN) :: win
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Fetch_and_op_f08
-end interface  MPI_Fetch_and_op
-
-interface  MPI_Compare_and_swap
-subroutine MPI_Compare_and_swap_f08(origin_addr,compare_addr,result_addr,datatype, &
-                                    target_rank,target_disp,win,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Win, MPI_ADDRESS_KIND
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ origin_addr,compare_addr,result_addr
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: origin_addr,compare_addr
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: result_addr
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER, INTENT(IN) :: target_rank
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
-   TYPE(MPI_Win), INTENT(IN) :: win
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Compare_and_swap_f08
-end interface  MPI_Compare_and_swap
-
-interface  MPI_Win_complete
-subroutine MPI_Win_complete_f08(win,ierror)
-   use :: mpi_f08_types, only : MPI_Info, MPI_Comm, MPI_Win, MPI_ADDRESS_KIND
-   implicit none
-   TYPE(MPI_Win), INTENT(IN) :: win
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Win_complete_f08
-end interface  MPI_Win_complete
-
-interface  MPI_Win_create
-subroutine MPI_Win_create_f08(base,size,disp_unit,info,comm,win,ierror)
-   use :: mpi_f08_types, only : MPI_Info, MPI_Comm, MPI_Win, MPI_ADDRESS_KIND
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ base
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: base
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: size
-   INTEGER, INTENT(IN) :: disp_unit
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Win), INTENT(OUT) :: win
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Win_create_f08
-end interface  MPI_Win_create
+end subroutine MPI_Win_complete_f08
+end interface  MPI_Win_complete
 
 interface  MPI_Win_create_dynamic
 subroutine MPI_Win_create_dynamic_f08(info,comm,win,ierror)
@@ -2950,29 +1890,6 @@ subroutine MPI_Win_create_dynamic_f08(info,comm,win,ierror)
 end subroutine MPI_Win_create_dynamic_f08
 end interface  MPI_Win_create_dynamic
 
-interface  MPI_Win_attach
-subroutine MPI_Win_attach_f08(win,base,size,ierror)
-   use :: mpi_f08_types, only : MPI_Win, MPI_ADDRESS_KIND
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ base
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: base
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: size
-   TYPE(MPI_Win), INTENT(IN) :: win
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Win_attach_f08
-end interface  MPI_Win_attach
-
-interface  MPI_Win_detach
-subroutine MPI_Win_detach_f08(win,base,ierror)
-   use :: mpi_f08_types, only : MPI_Win, MPI_ADDRESS_KIND
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ base
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: base
-   TYPE(MPI_Win), INTENT(IN) :: win
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Win_detach_f08
-end interface  MPI_Win_detach
-
 interface  MPI_Win_fence
 subroutine MPI_Win_fence_f08(assert,win,ierror)
    use :: mpi_f08_types, only : MPI_Win
@@ -3033,20 +1950,6 @@ subroutine MPI_Win_post_f08(group,assert,win,ierror)
 end subroutine MPI_Win_post_f08
 end interface  MPI_Win_post
 
-interface  MPI_Win_shared_query
-subroutine MPI_Win_shared_query_f08(win, rank, size, disp_unit, baseptr,&
-      ierror)
-  USE, INTRINSIC ::  ISO_C_BINDING, ONLY : C_PTR
-  use :: mpi_f08_types, only : MPI_Win, MPI_ADDRESS_KIND
-  TYPE(MPI_Win), INTENT(IN) ::  win
-  INTEGER, INTENT(IN) ::  rank
-  INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(OUT) ::  size
-  INTEGER, INTENT(OUT) ::  disp_unit
-  TYPE(C_PTR), INTENT(OUT) ::  baseptr
-  INTEGER, OPTIONAL, INTENT(OUT) ::  ierror
-end subroutine MPI_Win_shared_query_f08
-end interface  MPI_Win_shared_query
-
 interface  MPI_Win_start
 subroutine MPI_Win_start_f08(group,assert,win,ierror)
    use :: mpi_f08_types, only : MPI_Group, MPI_Win
@@ -3204,17 +2107,6 @@ subroutine MPI_Status_set_cancelled_f08(status,flag,ierror)
 end subroutine MPI_Status_set_cancelled_f08
 end interface  MPI_Status_set_cancelled
 
-interface  MPI_Status_set_elements
-subroutine MPI_Status_set_elements_f08(status,datatype,count,ierror)
-   use :: mpi_f08_types, only : MPI_Status, MPI_Datatype
-   implicit none
-   TYPE(MPI_Status), INTENT(INOUT) :: status
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER, INTENT(IN) :: count
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Status_set_elements_f08
-end interface  MPI_Status_set_elements
-
 interface  MPI_Status_set_elements_x
 subroutine MPI_Status_set_elements_x_f08(status,datatype,count,ierror)
    use :: mpi_f08_types, only : MPI_Status, MPI_Datatype, MPI_COUNT_KIND
@@ -3350,150 +2242,6 @@ subroutine MPI_File_get_view_f08(fh,disp,etype,filetype,datarep,ierror)
 end subroutine MPI_File_get_view_f08
 end interface  MPI_File_get_view
 
-interface  MPI_File_iread
-subroutine MPI_File_iread_f08(fh,buf,count,datatype,request,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Request
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_iread_f08
-end interface  MPI_File_iread
-
-interface  MPI_File_iread_at
-subroutine MPI_File_iread_at_f08(fh,offset,buf,count,datatype,request,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Request, MPI_OFFSET_KIND
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   INTEGER(MPI_OFFSET_KIND), INTENT(IN) :: offset
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_iread_at_f08
-end interface  MPI_File_iread_at
-
-interface  MPI_File_iread_all
-subroutine MPI_File_iread_all_f08(fh,buf,count,datatype,request,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Request
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_iread_all_f08
-end interface  MPI_File_iread_all
-
-interface  MPI_File_iread_at_all
-subroutine MPI_File_iread_at_all_f08(fh,offset,buf,count,datatype,request,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Request, MPI_OFFSET_KIND
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   INTEGER(MPI_OFFSET_KIND), INTENT(IN) :: offset
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_iread_at_all_f08
-end interface  MPI_File_iread_at_all
-
-interface  MPI_File_iread_shared
-subroutine MPI_File_iread_shared_f08(fh,buf,count,datatype,request,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Request
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_iread_shared_f08
-end interface  MPI_File_iread_shared
-
-interface  MPI_File_iwrite
-subroutine MPI_File_iwrite_f08(fh,buf,count,datatype,request,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Request
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_iwrite_f08
-end interface  MPI_File_iwrite
-
-interface  MPI_File_iwrite_at
-subroutine MPI_File_iwrite_at_f08(fh,offset,buf,count,datatype,request,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Request, MPI_OFFSET_KIND
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   INTEGER(MPI_OFFSET_KIND), INTENT(IN) :: offset
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_iwrite_at_f08
-end interface  MPI_File_iwrite_at
-
-interface  MPI_File_iwrite_all
-subroutine MPI_File_iwrite_all_f08(fh,buf,count,datatype,request,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Request
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_iwrite_all_f08
-end interface  MPI_File_iwrite_all
-
-interface  MPI_File_iwrite_at_all
-subroutine MPI_File_iwrite_at_all_f08(fh,offset,buf,count,datatype,request,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Request, MPI_OFFSET_KIND
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   INTEGER(MPI_OFFSET_KIND), INTENT(IN) :: offset
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_iwrite_at_all_f08
-end interface  MPI_File_iwrite_at_all
-
-interface  MPI_File_iwrite_shared
-subroutine MPI_File_iwrite_shared_f08(fh,buf,count,datatype,request,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Request
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: buf
-   TYPE(MPI_File), INTENT(IN) :: fh
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_iwrite_shared_f08
-end interface  MPI_File_iwrite_shared
-
 interface  MPI_File_open
 subroutine MPI_File_open_f08(comm,filename,amode,info,fh,ierror)
    use :: mpi_f08_types, only : MPI_Comm, MPI_Info, MPI_File
@@ -3517,168 +2265,6 @@ subroutine MPI_File_preallocate_f08(fh,size,ierror)
 end subroutine MPI_File_preallocate_f08
 end interface  MPI_File_preallocate
 
-interface  MPI_File_read
-subroutine MPI_File_read_f08(fh,buf,count,datatype,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Status
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Status) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_read_f08
-end interface  MPI_File_read
-
-interface  MPI_File_read_all
-subroutine MPI_File_read_all_f08(fh,buf,count,datatype,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Status
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Status) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_read_all_f08
-end interface  MPI_File_read_all
-
-interface  MPI_File_read_all_begin
-subroutine MPI_File_read_all_begin_f08(fh,buf,count,datatype,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_read_all_begin_f08
-end interface  MPI_File_read_all_begin
-
-interface  MPI_File_read_all_end
-subroutine MPI_File_read_all_end_f08(fh,buf,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Status
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buf
-   TYPE(MPI_Status) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_read_all_end_f08
-end interface  MPI_File_read_all_end
-
-interface  MPI_File_read_at
-subroutine MPI_File_read_at_f08(fh,offset,buf,count,datatype,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Status, MPI_OFFSET_KIND
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   INTEGER(MPI_OFFSET_KIND), INTENT(IN) :: offset
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Status) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_read_at_f08
-end interface  MPI_File_read_at
-
-interface  MPI_File_read_at_all
-subroutine MPI_File_read_at_all_f08(fh,offset,buf,count,datatype,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Status, MPI_OFFSET_KIND
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   INTEGER(MPI_OFFSET_KIND), INTENT(IN) :: offset
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Status) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_read_at_all_f08
-end interface  MPI_File_read_at_all
-
-interface  MPI_File_read_at_all_begin
-subroutine MPI_File_read_at_all_begin_f08(fh,offset,buf,count,datatype,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_OFFSET_KIND
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   INTEGER(MPI_OFFSET_KIND), INTENT(IN) :: offset
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_read_at_all_begin_f08
-end interface  MPI_File_read_at_all_begin
-
-interface  MPI_File_read_at_all_end
-subroutine MPI_File_read_at_all_end_f08(fh,buf,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Status
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buf
-   TYPE(MPI_Status) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_read_at_all_end_f08
-end interface  MPI_File_read_at_all_end
-
-interface  MPI_File_read_ordered
-subroutine MPI_File_read_ordered_f08(fh,buf,count,datatype,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Status
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Status) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_read_ordered_f08
-end interface  MPI_File_read_ordered
-
-interface  MPI_File_read_ordered_begin
-subroutine MPI_File_read_ordered_begin_f08(fh,buf,count,datatype,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_read_ordered_begin_f08
-end interface  MPI_File_read_ordered_begin
-
-interface  MPI_File_read_ordered_end
-subroutine MPI_File_read_ordered_end_f08(fh,buf,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Status
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buf
-   TYPE(MPI_Status) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_read_ordered_end_f08
-end interface  MPI_File_read_ordered_end
-
-interface  MPI_File_read_shared
-subroutine MPI_File_read_shared_f08(fh,buf,count,datatype,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Status
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Status) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_read_shared_f08
-end interface  MPI_File_read_shared
-
 interface  MPI_File_seek
 subroutine MPI_File_seek_f08(fh,offset,whence,ierror)
    use :: mpi_f08_types, only : MPI_File, MPI_OFFSET_KIND
@@ -3754,168 +2340,6 @@ subroutine MPI_File_sync_f08(fh,ierror)
 end subroutine MPI_File_sync_f08
 end interface  MPI_File_sync
 
-interface  MPI_File_write
-subroutine MPI_File_write_f08(fh,buf,count,datatype,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Status
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Status) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_write_f08
-end interface  MPI_File_write
-
-interface  MPI_File_write_all
-subroutine MPI_File_write_all_f08(fh,buf,count,datatype,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Status
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Status) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_write_all_f08
-end interface  MPI_File_write_all
-
-interface  MPI_File_write_all_begin
-subroutine MPI_File_write_all_begin_f08(fh,buf,count,datatype,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_write_all_begin_f08
-end interface  MPI_File_write_all_begin
-
-interface  MPI_File_write_all_end
-subroutine MPI_File_write_all_end_f08(fh,buf,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Status
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: buf
-   TYPE(MPI_Status) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_write_all_end_f08
-end interface  MPI_File_write_all_end
-
-interface  MPI_File_write_at
-subroutine MPI_File_write_at_f08(fh,offset,buf,count,datatype,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Status, MPI_OFFSET_KIND
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   INTEGER(MPI_OFFSET_KIND), INTENT(IN) :: offset
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Status) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_write_at_f08
-end interface  MPI_File_write_at
-
-interface  MPI_File_write_at_all
-subroutine MPI_File_write_at_all_f08(fh,offset,buf,count,datatype,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Status, MPI_OFFSET_KIND
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   INTEGER(MPI_OFFSET_KIND), INTENT(IN) :: offset
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Status) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_write_at_all_f08
-end interface  MPI_File_write_at_all
-
-interface  MPI_File_write_at_all_begin
-subroutine MPI_File_write_at_all_begin_f08(fh,offset,buf,count,datatype,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_OFFSET_KIND
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   INTEGER(MPI_OFFSET_KIND), INTENT(IN) :: offset
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_write_at_all_begin_f08
-end interface  MPI_File_write_at_all_begin
-
-interface  MPI_File_write_at_all_end
-subroutine MPI_File_write_at_all_end_f08(fh,buf,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Status
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: buf
-   TYPE(MPI_Status) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_write_at_all_end_f08
-end interface  MPI_File_write_at_all_end
-
-interface  MPI_File_write_ordered
-subroutine MPI_File_write_ordered_f08(fh,buf,count,datatype,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Status
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Status) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_write_ordered_f08
-end interface  MPI_File_write_ordered
-
-interface  MPI_File_write_ordered_begin
-subroutine MPI_File_write_ordered_begin_f08(fh,buf,count,datatype,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_write_ordered_begin_f08
-end interface  MPI_File_write_ordered_begin
-
-interface  MPI_File_write_ordered_end
-subroutine MPI_File_write_ordered_end_f08(fh,buf,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Status
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: buf
-   TYPE(MPI_Status) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_write_ordered_end_f08
-end interface  MPI_File_write_ordered_end
-
-interface  MPI_File_write_shared
-subroutine MPI_File_write_shared_f08(fh,buf,count,datatype,status,ierror)
-   use :: mpi_f08_types, only : MPI_File, MPI_Datatype, MPI_Status
-   implicit none
-   TYPE(MPI_File), INTENT(IN) :: fh
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Status) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_File_write_shared_f08
-end interface  MPI_File_write_shared
-
 interface  MPI_Register_datarep
 subroutine MPI_Register_datarep_f08(datarep,read_conversion_fn,write_conversion_fn, &
                                             dtype_file_extent_fn,extra_state,ierror)
@@ -4010,14 +2434,6 @@ subroutine MPI_Comm_split_type_f08(comm,split_type,key,info,newcomm,ierror)
 end subroutine MPI_Comm_split_type_f08
 end interface  MPI_Comm_split_type
 
-interface  MPI_F_sync_reg
-subroutine MPI_F_sync_reg_f08(buf)
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buf
-end subroutine MPI_F_sync_reg_f08
-end interface  MPI_F_sync_reg
-
 interface  MPI_Get_library_version
 subroutine MPI_Get_library_version_f08(version,resultlen,ierror)
    use :: mpi_f08_types, only : MPI_MAX_LIBRARY_VERSION_STRING
@@ -4035,7 +2451,7 @@ subroutine MPI_Mprobe_f08(source,tag,comm,message,status,ierror)
    INTEGER, INTENT(IN) :: source, tag
    TYPE(MPI_Comm), INTENT(IN) :: comm
    TYPE(MPI_Message), INTENT(OUT) :: message
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   TYPE(MPI_Status) :: status
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
 end subroutine MPI_Mprobe_f08
 end interface  MPI_Mprobe
@@ -4048,194 +2464,27 @@ subroutine MPI_Improbe_f08(source,tag,comm,flag,message,status,ierror)
    TYPE(MPI_Comm), INTENT(IN) :: comm
    LOGICAL, INTENT(OUT) :: flag
    TYPE(MPI_Message), INTENT(OUT) :: message
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   TYPE(MPI_Status) :: status
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
 end subroutine MPI_Improbe_f08
 end interface  MPI_Improbe
 
-interface  MPI_Imrecv
-subroutine MPI_Imrecv_f08(buf,count,datatype,message,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Message, MPI_Request
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Message), INTENT(INOUT) :: message
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Imrecv_f08
-end interface  MPI_Imrecv
-
-interface  MPI_Mrecv
-subroutine MPI_Mrecv_f08(buf,count,datatype,message,status,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Message, MPI_Status
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Message), INTENT(INOUT) :: message
-   TYPE(MPI_Status) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Mrecv_f08
-end interface  MPI_Mrecv
-
-interface  MPI_Neighbor_allgather
-subroutine MPI_Neighbor_allgather_f08(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype, &
-                             comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype, recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Neighbor_allgather_f08
-end interface  MPI_Neighbor_allgather
-
-interface  MPI_Ineighbor_allgather
-subroutine MPI_Ineighbor_allgather_f08(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype, &
-                             comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype, recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Ineighbor_allgather_f08
-end interface  MPI_Ineighbor_allgather
-
-interface  MPI_Neighbor_allgatherv
-subroutine MPI_Neighbor_allgatherv_f08(sendbuf,sendcount,sendtype,recvbuf,recvcounts,displs, &
-                              recvtype,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount
-   INTEGER, INTENT(IN) :: recvcounts(*), displs(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype, recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Neighbor_allgatherv_f08
-end interface  MPI_Neighbor_allgatherv
-
-interface  MPI_Ineighbor_allgatherv
-subroutine MPI_Ineighbor_allgatherv_f08(sendbuf,sendcount,sendtype,recvbuf,recvcounts,displs, &
-                              recvtype,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount
-   INTEGER, INTENT(IN) :: recvcounts(*), displs(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype, recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Ineighbor_allgatherv_f08
-end interface  MPI_Ineighbor_allgatherv
-
-interface  MPI_Neighbor_alltoall
-subroutine MPI_Neighbor_alltoall_f08(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype, &
-                            comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype, recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Neighbor_alltoall_f08
-end interface  MPI_Neighbor_alltoall
-
-interface  MPI_Ineighbor_alltoall
-subroutine MPI_Ineighbor_alltoall_f08(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype, &
-                            comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype, recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Ineighbor_alltoall_f08
-end interface  MPI_Ineighbor_alltoall
-
-interface  MPI_Neighbor_alltoallv
-subroutine MPI_Neighbor_alltoallv_f08(sendbuf,sendcounts,sdispls,sendtype,recvbuf,recvcounts, &
-                             rdispls,recvtype,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
-   INTEGER, INTENT(IN) :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype, recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Neighbor_alltoallv_f08
-end interface  MPI_Neighbor_alltoallv
-
-interface  MPI_Ineighbor_alltoallv
-subroutine MPI_Ineighbor_alltoallv_f08(sendbuf,sendcounts,sdispls,sendtype,recvbuf,recvcounts, &
-                             rdispls,recvtype,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
+interface MPI_Status_f2f08
+subroutine MPI_Status_f2f08_f08(f_status,f08_status,ierror)
+   use :: mpi_f08_types, only : MPI_Status, MPI_STATUS_SIZE
    implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
-   INTEGER, INTENT(IN) :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype, recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
+   INTEGER, INTENT(IN) :: f_status(MPI_STATUS_SIZE)
+   TYPE(MPI_Status), INTENT(OUT) :: f08_status
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Ineighbor_alltoallv_f08
-end interface  MPI_Ineighbor_alltoallv
+end subroutine MPI_Status_f2f08_f08
+end interface MPI_Status_f2f08
 
-interface  MPI_Neighbor_alltoallw
-subroutine MPI_Neighbor_alltoallw_f08(sendbuf,sendcounts,sdispls,sendtypes,recvbuf,recvcounts, &
-                             rdispls,recvtypes,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_ADDRESS_KIND
+interface MPI_Status_f082f
+subroutine MPI_Status_f082f_f08(f08_status,f_status,ierror)
+   use :: mpi_f08_types, only : MPI_Status, MPI_STATUS_SIZE
    implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
-   INTEGER, INTENT(IN) :: sendcounts(*), recvcounts(*)
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: sdispls(*), rdispls(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtypes(*), recvtypes(*)
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Neighbor_alltoallw_f08
-end interface  MPI_Neighbor_alltoallw
-
-interface  MPI_Ineighbor_alltoallw
-subroutine MPI_Ineighbor_alltoallw_f08(sendbuf,sendcounts,sdispls,sendtypes,recvbuf,recvcounts, &
-                             rdispls,recvtypes,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request, MPI_ADDRESS_KIND
-   implicit none
-   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf, recvbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@, INTENT(IN) :: sendbuf
-   @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
-   INTEGER, INTENT(IN) :: sendcounts(*), recvcounts(*)
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: sdispls(*), rdispls(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtypes(*), recvtypes(*)
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
+   TYPE(MPI_Status), INTENT(IN) :: f08_status
+   INTEGER, INTENT(OUT) :: f_status(MPI_STATUS_SIZE)
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-end subroutine MPI_Ineighbor_alltoallw_f08
-end interface  MPI_Ineighbor_alltoallw
+end subroutine MPI_Status_f082f_f08
+end interface MPI_Status_f082f
diff --git a/ompi/mpi/fortran/use-mpi-f08/mod/mpi-f08-rename.h b/ompi/mpi/fortran/use-mpi-f08/mod/mpi-f08-rename.h
index 8e948eb93c3..ea8370f4b54 100644
--- a/ompi/mpi/fortran/use-mpi-f08/mod/mpi-f08-rename.h
+++ b/ompi/mpi/fortran/use-mpi-f08/mod/mpi-f08-rename.h
@@ -5,703 +5,804 @@
 
 #if OMPI_BUILD_MPI_PROFILING
 
-#define MPI_Bsend PMPI_Bsend
-#define MPI_Bsend_f08 PMPI_Bsend_f08
-#define MPI_Bsend_init PMPI_Bsend_init
-#define MPI_Bsend_init_f08 PMPI_Bsend_init_f08
-#define MPI_Buffer_attach PMPI_Buffer_attach
-#define MPI_Buffer_attach_f08 PMPI_Buffer_attach_f08
-#define MPI_Buffer_detach PMPI_Buffer_detach
-#define MPI_Buffer_detach_f08 PMPI_Buffer_detach_f08
-#define MPI_Cancel PMPI_Cancel
-#define MPI_Cancel_f08 PMPI_Cancel_f08
-#define MPI_Get_count PMPI_Get_count
-#define MPI_Get_count_f08 PMPI_Get_count_f08
-#define MPI_Ibsend PMPI_Ibsend
-#define MPI_Ibsend_f08 PMPI_Ibsend_f08
-#define MPI_Iprobe PMPI_Iprobe
-#define MPI_Iprobe_f08 PMPI_Iprobe_f08
-#define MPI_Irecv PMPI_Irecv
-#define MPI_Irecv_f08 PMPI_Irecv_f08
-#define MPI_Irsend PMPI_Irsend
-#define MPI_Irsend_f08 PMPI_Irsend_f08
-#define MPI_Isend PMPI_Isend
-#define MPI_Isend_f08 PMPI_Isend_f08
-#define MPI_Issend PMPI_Issend
-#define MPI_Issend_f08 PMPI_Issend_f08
-#define MPI_Probe PMPI_Probe
-#define MPI_Probe_f08 PMPI_Probe_f08
-#define MPI_Recv PMPI_Recv
-#define MPI_Recv_f08 PMPI_Recv_f08
-#define MPI_Recv_init PMPI_Recv_init
-#define MPI_Recv_init_f08 PMPI_Recv_init_f08
-#define MPI_Request_free PMPI_Request_free
-#define MPI_Request_free_f08 PMPI_Request_free_f08
-#define MPI_Request_get_status PMPI_Request_get_status
-#define MPI_Request_get_status_f08 PMPI_Request_get_status_f08
-#define MPI_Rsend PMPI_Rsend
-#define MPI_Rsend_f08 PMPI_Rsend_f08
-#define MPI_Rsend_init PMPI_Rsend_init
-#define MPI_Rsend_init_f08 PMPI_Rsend_init_f08
-#define MPI_Send PMPI_Send
-#define MPI_Send_f08 PMPI_Send_f08
-#define MPI_Sendrecv PMPI_Sendrecv
-#define MPI_Sendrecv_f08 PMPI_Sendrecv_f08
-#define MPI_Sendrecv_replace PMPI_Sendrecv_replace
-#define MPI_Sendrecv_replace_f08 PMPI_Sendrecv_replace_f08
-#define MPI_Send_init PMPI_Send_init
-#define MPI_Send_init_f08 PMPI_Send_init_f08
-#define MPI_Ssend PMPI_Ssend
-#define MPI_Ssend_f08 PMPI_Ssend_f08
-#define MPI_Ssend_init PMPI_Ssend_init
-#define MPI_Ssend_init_f08 PMPI_Ssend_init_f08
-#define MPI_Start PMPI_Start
-#define MPI_Start_f08 PMPI_Start_f08
-#define MPI_Startall PMPI_Startall
-#define MPI_Startall_f08 PMPI_Startall_f08
-#define MPI_Test PMPI_Test
-#define MPI_Test_f08 PMPI_Test_f08
-#define MPI_Testall PMPI_Testall
-#define MPI_Testall_f08 PMPI_Testall_f08
-#define MPI_Testany PMPI_Testany
-#define MPI_Testany_f08 PMPI_Testany_f08
-#define MPI_Testsome PMPI_Testsome
-#define MPI_Testsome_f08 PMPI_Testsome_f08
-#define MPI_Test_cancelled PMPI_Test_cancelled
-#define MPI_Test_cancelled_f08 PMPI_Test_cancelled_f08
-#define MPI_Wait PMPI_Wait
-#define MPI_Wait_f08 PMPI_Wait_f08
-#define MPI_Waitall PMPI_Waitall
-#define MPI_Waitall_f08 PMPI_Waitall_f08
-#define MPI_Waitany PMPI_Waitany
-#define MPI_Waitany_f08 PMPI_Waitany_f08
-#define MPI_Waitsome PMPI_Waitsome
-#define MPI_Waitsome_f08 PMPI_Waitsome_f08
-#define MPI_Get_address PMPI_Get_address
-#define MPI_Get_address_f08 PMPI_Get_address_f08
-#define MPI_Get_elements PMPI_Get_elements
-#define MPI_Get_elements_f08 PMPI_Get_elements_f08
-#define MPI_Get_elements_x PMPI_Get_elements_x
-#define MPI_Get_elements_x_f08 PMPI_Get_elements_x_f08
-#define MPI_Pack PMPI_Pack
-#define MPI_Pack_f08 PMPI_Pack_f08
-#define MPI_Pack_external PMPI_Pack_external
-#define MPI_Pack_external_f08 PMPI_Pack_external_f08
-#define MPI_Pack_external_size PMPI_Pack_external_size
-#define MPI_Pack_external_size_f08 PMPI_Pack_external_size_f08
-#define MPI_Pack_size PMPI_Pack_size
-#define MPI_Pack_size_f08 PMPI_Pack_size_f08
-#define MPI_Type_commit PMPI_Type_commit
-#define MPI_Type_commit_f08 PMPI_Type_commit_f08
-#define MPI_Type_contiguous PMPI_Type_contiguous
-#define MPI_Type_contiguous_f08 PMPI_Type_contiguous_f08
-#define MPI_Type_create_darray PMPI_Type_create_darray
-#define MPI_Type_create_darray_f08 PMPI_Type_create_darray_f08
-#define MPI_Type_create_hindexed PMPI_Type_create_hindexed
-#define MPI_Type_create_hindexed_f08 PMPI_Type_create_hindexed_f08
-#define MPI_Type_create_hvector PMPI_Type_create_hvector
-#define MPI_Type_create_hvector_f08 PMPI_Type_create_hvector_f08
-#define MPI_Type_create_indexed_block PMPI_Type_create_indexed_block
-#define MPI_Type_create_indexed_block_f08 PMPI_Type_create_indexed_block_f08
-#define MPI_Type_create_hindexed_block PMPI_Type_create_hindexed_block
-#define MPI_Type_create_hindexed_block_f08 PMPI_Type_create_hindexed_block_f08
-#define MPI_Type_create_resized PMPI_Type_create_resized
-#define MPI_Type_create_resized_f08 PMPI_Type_create_resized_f08
-#define MPI_Type_create_struct PMPI_Type_create_struct
-#define MPI_Type_create_struct_f08 PMPI_Type_create_struct_f08
-#define MPI_Type_create_subarray PMPI_Type_create_subarray
-#define MPI_Type_create_subarray_f08 PMPI_Type_create_subarray_f08
-#define MPI_Type_dup PMPI_Type_dup
-#define MPI_Type_dup_f08 PMPI_Type_dup_f08
-#define MPI_Type_free PMPI_Type_free
-#define MPI_Type_free_f08 PMPI_Type_free_f08
-#define MPI_Type_get_contents PMPI_Type_get_contents
-#define MPI_Type_get_contents_f08 PMPI_Type_get_contents_f08
-#define MPI_Type_get_envelope PMPI_Type_get_envelope
-#define MPI_Type_get_envelope_f08 PMPI_Type_get_envelope_f08
-#define MPI_Type_get_extent PMPI_Type_get_extent
-#define MPI_Type_get_extent_f08 PMPI_Type_get_extent_f08
-#define MPI_Type_get_extent_x PMPI_Type_get_extent_x
-#define MPI_Type_get_extent_x_f08 PMPI_Type_get_extent_x_f08
-#define MPI_Type_get_true_extent PMPI_Type_get_true_extent
-#define MPI_Type_get_true_extent_f08 PMPI_Type_get_true_extent_f08
-#define MPI_Type_get_true_extent_x PMPI_Type_get_true_extent_x
-#define MPI_Type_get_true_extent_x_f08 PMPI_Type_get_true_extent_x_f08
-#define MPI_Type_indexed PMPI_Type_indexed
-#define MPI_Type_indexed_f08 PMPI_Type_indexed_f08
-#define MPI_Type_size PMPI_Type_size
-#define MPI_Type_size_f08 PMPI_Type_size_f08
-#define MPI_Type_size_x PMPI_Type_size_x
-#define MPI_Type_size_x_f08 PMPI_Type_size_x_f08
-#define MPI_Type_vector PMPI_Type_vector
-#define MPI_Type_vector_f08 PMPI_Type_vector_f08
-#define MPI_Unpack PMPI_Unpack
-#define MPI_Unpack_f08 PMPI_Unpack_f08
-#define MPI_Unpack_external PMPI_Unpack_external
-#define MPI_Unpack_external_f08 PMPI_Unpack_external_f08
-#define MPI_Allgather PMPI_Allgather
+#define MPI_Abort_f08 PMPI_Abort_f08
+#define MPI_Abort PMPI_Abort
+#define MPI_Accumulate_f08 PMPI_Accumulate_f08
+#define MPI_Accumulate PMPI_Accumulate
+#define MPI_Add_error_class_f08 PMPI_Add_error_class_f08
+#define MPI_Add_error_class PMPI_Add_error_class
+#define MPI_Add_error_code_f08 PMPI_Add_error_code_f08
+#define MPI_Add_error_code PMPI_Add_error_code
+#define MPI_Add_error_string_f08 PMPI_Add_error_string_f08
+#define MPI_Add_error_string PMPI_Add_error_string
+#define MPI_Aint_add_f08 PMPI_Aint_add_f08
+#define MPI_Aint_add PMPI_Aint_add
+#define MPI_Aint_diff_f08 PMPI_Aint_diff_f08
+#define MPI_Aint_diff PMPI_Aint_diff
 #define MPI_Allgather_f08 PMPI_Allgather_f08
-#define MPI_Iallgather PMPI_Iallgather
-#define MPI_Iallgather_f08 PMPI_Iallgather_f08
-#define MPI_Allgatherv PMPI_Allgatherv
+#define MPI_Allgather_init_f08 PMPI_Allgather_init_f08
+#define MPI_Allgather_init PMPI_Allgather_init
+#define MPI_Allgather PMPI_Allgather
 #define MPI_Allgatherv_f08 PMPI_Allgatherv_f08
-#define MPI_Iallgatherv PMPI_Iallgatherv
-#define MPI_Iallgatherv_f08 PMPI_Iallgatherv_f08
-#define MPI_Allreduce PMPI_Allreduce
+#define MPI_Allgatherv_init_f08 PMPI_Allgatherv_init_f08
+#define MPI_Allgatherv_init PMPI_Allgatherv_init
+#define MPI_Allgatherv PMPI_Allgatherv
+#define MPI_Alloc_mem_f08 PMPI_Alloc_mem_f08
+#define MPI_Alloc_mem PMPI_Alloc_mem
 #define MPI_Allreduce_f08 PMPI_Allreduce_f08
-#define MPI_Iallreduce PMPI_Iallreduce
-#define MPI_Iallreduce_f08 PMPI_Iallreduce_f08
-#define MPI_Alltoall PMPI_Alltoall
+#define MPI_Allreduce_init_f08 PMPI_Allreduce_init_f08
+#define MPI_Allreduce_init PMPI_Allreduce_init
+#define MPI_Allreduce PMPI_Allreduce
 #define MPI_Alltoall_f08 PMPI_Alltoall_f08
-#define MPI_Ialltoall PMPI_Ialltoall
-#define MPI_Ialltoall_f08 PMPI_Ialltoall_f08
-#define MPI_Alltoallv PMPI_Alltoallv
+#define MPI_Alltoall_init_f08 PMPI_Alltoall_init_f08
+#define MPI_Alltoall_init PMPI_Alltoall_init
+#define MPI_Alltoall PMPI_Alltoall
 #define MPI_Alltoallv_f08 PMPI_Alltoallv_f08
-#define MPI_Ialltoallv PMPI_Ialltoallv
-#define MPI_Ialltoallv_f08 PMPI_Ialltoallv_f08
-#define MPI_Alltoallw PMPI_Alltoallw
+#define MPI_Alltoallv_init_f08 PMPI_Alltoallv_init_f08
+#define MPI_Alltoallv_init PMPI_Alltoallv_init
+#define MPI_Alltoallv PMPI_Alltoallv
 #define MPI_Alltoallw_f08 PMPI_Alltoallw_f08
-#define MPI_Ialltoallw PMPI_Ialltoallw
-#define MPI_Ialltoallw_f08 PMPI_Ialltoallw_f08
-#define MPI_Barrier PMPI_Barrier
+#define MPI_Alltoallw_init_f08 PMPI_Alltoallw_init_f08
+#define MPI_Alltoallw_init PMPI_Alltoallw_init
+#define MPI_Alltoallw PMPI_Alltoallw
 #define MPI_Barrier_f08 PMPI_Barrier_f08
-#define MPI_Ibarrier PMPI_Ibarrier
-#define MPI_Ibarrier_f08 PMPI_Ibarrier_f08
-#define MPI_Bcast PMPI_Bcast
+#define MPI_Barrier_init_f08 PMPI_Barrier_init_f08
+#define MPI_Barrier_init PMPI_Barrier_init
+#define MPI_Barrier PMPI_Barrier
 #define MPI_Bcast_f08 PMPI_Bcast_f08
-#define MPI_Ibcast PMPI_Ibcast
-#define MPI_Ibcast_f08 PMPI_Ibcast_f08
-#define MPI_Exscan PMPI_Exscan
-#define MPI_Exscan_f08 PMPI_Exscan_f08
-#define MPI_Iexscan PMPI_Iexscan
-#define MPI_Iexscan_f08 PMPI_Iexscan_f08
-#define MPI_Gather PMPI_Gather
-#define MPI_Gather_f08 PMPI_Gather_f08
-#define MPI_Igather PMPI_Igather
-#define MPI_Igather_f08 PMPI_Igather_f08
-#define MPI_Gatherv PMPI_Gatherv
-#define MPI_Gatherv_f08 PMPI_Gatherv_f08
-#define MPI_Igatherv PMPI_Igatherv
-#define MPI_Igatherv_f08 PMPI_Igatherv_f08
-#define MPI_Op_commutative PMPI_Op_commutative
-#define MPI_Op_commutative_f08 PMPI_Op_commutative_f08
-#define MPI_Op_create PMPI_Op_create
-#define MPI_Op_create_f08 PMPI_Op_create_f08
-#define MPI_Op_free PMPI_Op_free
-#define MPI_Op_free_f08 PMPI_Op_free_f08
-#define MPI_Reduce PMPI_Reduce
-#define MPI_Reduce_f08 PMPI_Reduce_f08
-#define MPI_Ireduce PMPI_Ireduce
-#define MPI_Ireduce_f08 PMPI_Ireduce_f08
-#define MPI_Reduce_local PMPI_Reduce_local
-#define MPI_Reduce_local_f08 PMPI_Reduce_local_f08
-#define MPI_Reduce_scatter PMPI_Reduce_scatter
-#define MPI_Reduce_scatter_f08 PMPI_Reduce_scatter_f08
-#define MPI_Ireduce_scatter PMPI_Ireduce_scatter
-#define MPI_Ireduce_scatter_f08 PMPI_Ireduce_scatter_f08
-#define MPI_Reduce_scatter_block PMPI_Reduce_scatter_block
-#define MPI_Reduce_scatter_block_f08 PMPI_Reduce_scatter_block_f08
-#define MPI_Ireduce_scatter_block PMPI_Ireduce_scatter_block
-#define MPI_Ireduce_scatter_block_f08 PMPI_Ireduce_scatter_block_f08
-#define MPI_Scan PMPI_Scan
-#define MPI_Scan_f08 PMPI_Scan_f08
-#define MPI_Iscan PMPI_Iscan
-#define MPI_Iscan_f08 PMPI_Iscan_f08
-#define MPI_Scatter PMPI_Scatter
-#define MPI_Scatter_f08 PMPI_Scatter_f08
-#define MPI_Iscatter PMPI_Iscatter
-#define MPI_Iscatter_f08 PMPI_Iscatter_f08
-#define MPI_Scatterv PMPI_Scatterv
-#define MPI_Scatterv_f08 PMPI_Scatterv_f08
-#define MPI_Iscatterv PMPI_Iscatterv
-#define MPI_Iscatterv_f08 PMPI_Iscatterv_f08
-#define MPI_Comm_compare PMPI_Comm_compare
+#define MPI_Bcast_init_f08 PMPI_Bcast_init_f08
+#define MPI_Bcast_init PMPI_Bcast_init
+#define MPI_Bcast PMPI_Bcast
+#define MPI_Bsend_f08 PMPI_Bsend_f08
+#define MPI_Bsend_init_f08 PMPI_Bsend_init_f08
+#define MPI_Bsend_init PMPI_Bsend_init
+#define MPI_Bsend PMPI_Bsend
+#define MPI_Buffer_attach_f08 PMPI_Buffer_attach_f08
+#define MPI_Buffer_attach PMPI_Buffer_attach
+#define MPI_Buffer_detach_f08 PMPI_Buffer_detach_f08
+#define MPI_Buffer_detach PMPI_Buffer_detach
+#define MPI_Cancel_f08 PMPI_Cancel_f08
+#define MPI_Cancel PMPI_Cancel
+#define MPI_Cart_coords_f08 PMPI_Cart_coords_f08
+#define MPI_Cart_coords PMPI_Cart_coords
+#define MPI_Cart_create_f08 PMPI_Cart_create_f08
+#define MPI_Cart_create PMPI_Cart_create
+#define MPI_Cartdim_get_f08 PMPI_Cartdim_get_f08
+#define MPI_Cartdim_get PMPI_Cartdim_get
+#define MPI_Cart_get_f08 PMPI_Cart_get_f08
+#define MPI_Cart_get PMPI_Cart_get
+#define MPI_Cart_map_f08 PMPI_Cart_map_f08
+#define MPI_Cart_map PMPI_Cart_map
+#define MPI_Cart_rank_f08 PMPI_Cart_rank_f08
+#define MPI_Cart_rank PMPI_Cart_rank
+#define MPI_Cart_shift_f08 PMPI_Cart_shift_f08
+#define MPI_Cart_shift PMPI_Cart_shift
+#define MPI_Cart_sub_f08 PMPI_Cart_sub_f08
+#define MPI_Cart_sub PMPI_Cart_sub
+#define MPI_Close_port_f08 PMPI_Close_port_f08
+#define MPI_Close_port PMPI_Close_port
+#define MPI_Comm_accept_f08 PMPI_Comm_accept_f08
+#define MPI_Comm_accept PMPI_Comm_accept
+#define MPI_Comm_call_errhandler_f08 PMPI_Comm_call_errhandler_f08
+#define MPI_Comm_call_errhandler PMPI_Comm_call_errhandler
 #define MPI_Comm_compare_f08 PMPI_Comm_compare_f08
-#define MPI_Comm_create PMPI_Comm_create
+#define MPI_Comm_compare PMPI_Comm_compare
+#define MPI_Comm_connect_f08 PMPI_Comm_connect_f08
+#define MPI_Comm_connect PMPI_Comm_connect
+#define MPI_Comm_create_errhandler_f08 PMPI_Comm_create_errhandler_f08
+#define MPI_Comm_create_errhandler PMPI_Comm_create_errhandler
 #define MPI_Comm_create_f08 PMPI_Comm_create_f08
-#define MPI_Comm_create_group PMPI_Comm_create_group
+#define MPI_Comm_create_from_group_f08 PMPI_Comm_create_from_group_f08
+#define MPI_Comm_create_from_group PMPI_Comm_create_from_group
 #define MPI_Comm_create_group_f08 PMPI_Comm_create_group_f08
-#define MPI_Comm_create_keyval PMPI_Comm_create_keyval
+#define MPI_Comm_create_group PMPI_Comm_create_group
 #define MPI_Comm_create_keyval_f08 PMPI_Comm_create_keyval_f08
-#define MPI_Comm_delete_attr PMPI_Comm_delete_attr
+#define MPI_Comm_create_keyval PMPI_Comm_create_keyval
+#define MPI_Comm_create PMPI_Comm_create
 #define MPI_Comm_delete_attr_f08 PMPI_Comm_delete_attr_f08
-#define MPI_Comm_dup PMPI_Comm_dup
+#define MPI_Comm_delete_attr PMPI_Comm_delete_attr
+#define MPI_Comm_disconnect_f08 PMPI_Comm_disconnect_f08
+#define MPI_Comm_disconnect PMPI_Comm_disconnect
 #define MPI_Comm_dup_f08 PMPI_Comm_dup_f08
-#define MPI_Comm_dup_with_info PMPI_Comm_dup_with_info
+#define MPI_Comm_dup PMPI_Comm_dup
 #define MPI_Comm_dup_with_info_f08 PMPI_Comm_dup_with_info_f08
-#define MPI_Comm_idup PMPI_Comm_idup
-#define MPI_Comm_idup_f08 PMPI_Comm_idup_f08
-#define MPI_Comm_free PMPI_Comm_free
+#define MPI_Comm_dup_with_info PMPI_Comm_dup_with_info
 #define MPI_Comm_free_f08 PMPI_Comm_free_f08
-#define MPI_Comm_free_keyval PMPI_Comm_free_keyval
 #define MPI_Comm_free_keyval_f08 PMPI_Comm_free_keyval_f08
-#define MPI_Comm_get_attr PMPI_Comm_get_attr
+#define MPI_Comm_free_keyval PMPI_Comm_free_keyval
+#define MPI_Comm_free PMPI_Comm_free
 #define MPI_Comm_get_attr_f08 PMPI_Comm_get_attr_f08
-#define MPI_Comm_get_info PMPI_Comm_get_info
+#define MPI_Comm_get_attr PMPI_Comm_get_attr
+#define MPI_Comm_get_errhandler_f08 PMPI_Comm_get_errhandler_f08
+#define MPI_Comm_get_errhandler PMPI_Comm_get_errhandler
 #define MPI_Comm_get_info_f08 PMPI_Comm_get_info_f08
-#define MPI_Comm_get_name PMPI_Comm_get_name
+#define MPI_Comm_get_info PMPI_Comm_get_info
 #define MPI_Comm_get_name_f08 PMPI_Comm_get_name_f08
-#define MPI_Comm_group PMPI_Comm_group
+#define MPI_Comm_get_name PMPI_Comm_get_name
+#define MPI_Comm_get_parent_f08 PMPI_Comm_get_parent_f08
+#define MPI_Comm_get_parent PMPI_Comm_get_parent
 #define MPI_Comm_group_f08 PMPI_Comm_group_f08
-#define MPI_Comm_rank PMPI_Comm_rank
+#define MPI_Comm_group PMPI_Comm_group
+#define MPI_Comm_idup_f08 PMPI_Comm_idup_f08
+#define MPI_Comm_idup PMPI_Comm_idup
+#define MPI_Comm_idup_with_info_f08 PMPI_Comm_idup_with_info_f08
+#define MPI_Comm_idup_with_info PMPI_Comm_idup_with_info
+#define MPI_Comm_join_f08 PMPI_Comm_join_f08
+#define MPI_Comm_join PMPI_Comm_join
 #define MPI_Comm_rank_f08 PMPI_Comm_rank_f08
-#define MPI_Comm_remote_group PMPI_Comm_remote_group
+#define MPI_Comm_rank PMPI_Comm_rank
 #define MPI_Comm_remote_group_f08 PMPI_Comm_remote_group_f08
-#define MPI_Comm_remote_size PMPI_Comm_remote_size
+#define MPI_Comm_remote_group PMPI_Comm_remote_group
 #define MPI_Comm_remote_size_f08 PMPI_Comm_remote_size_f08
-#define MPI_Comm_set_attr PMPI_Comm_set_attr
+#define MPI_Comm_remote_size PMPI_Comm_remote_size
 #define MPI_Comm_set_attr_f08 PMPI_Comm_set_attr_f08
-#define MPI_Comm_set_info PMPI_Comm_set_info
+#define MPI_Comm_set_attr PMPI_Comm_set_attr
+#define MPI_Comm_set_errhandler_f08 PMPI_Comm_set_errhandler_f08
+#define MPI_Comm_set_errhandler PMPI_Comm_set_errhandler
 #define MPI_Comm_set_info_f08 PMPI_Comm_set_info_f08
-#define MPI_Comm_set_name PMPI_Comm_set_name
+#define MPI_Comm_set_info PMPI_Comm_set_info
 #define MPI_Comm_set_name_f08 PMPI_Comm_set_name_f08
-#define MPI_Comm_size PMPI_Comm_size
+#define MPI_Comm_set_name PMPI_Comm_set_name
 #define MPI_Comm_size_f08 PMPI_Comm_size_f08
-#define MPI_Comm_split PMPI_Comm_split
+#define MPI_Comm_size PMPI_Comm_size
+#define MPI_Comm_spawn_f08 PMPI_Comm_spawn_f08
+#define MPI_Comm_spawn_multiple_f08 PMPI_Comm_spawn_multiple_f08
+#define MPI_Comm_spawn_multiple PMPI_Comm_spawn_multiple
+#define MPI_Comm_spawn PMPI_Comm_spawn
 #define MPI_Comm_split_f08 PMPI_Comm_split_f08
-#define MPI_Comm_test_inter PMPI_Comm_test_inter
+#define MPI_Comm_split PMPI_Comm_split
+#define MPI_Comm_split_type_f08 PMPI_Comm_split_type_f08
+#define MPI_Comm_split_type PMPI_Comm_split_type
 #define MPI_Comm_test_inter_f08 PMPI_Comm_test_inter_f08
-#define MPI_Group_compare PMPI_Group_compare
+#define MPI_Comm_test_inter PMPI_Comm_test_inter
+#define MPI_Compare_and_swap_f08 PMPI_Compare_and_swap_f08
+#define MPI_Compare_and_swap PMPI_Compare_and_swap
+#define MPI_Dims_create_f08 PMPI_Dims_create_f08
+#define MPI_Dims_create PMPI_Dims_create
+#define MPI_Dist_graph_create_adjacent_f08 PMPI_Dist_graph_create_adjacent_f08
+#define MPI_Dist_graph_create_adjacent PMPI_Dist_graph_create_adjacent
+#define MPI_Dist_graph_create_f08 PMPI_Dist_graph_create_f08
+#define MPI_Dist_graph_create PMPI_Dist_graph_create
+#define MPI_Dist_graph_neighbors_count_f08 PMPI_Dist_graph_neighbors_count_f08
+#define MPI_Dist_graph_neighbors_count PMPI_Dist_graph_neighbors_count
+#define MPI_Dist_graph_neighbors_f08 PMPI_Dist_graph_neighbors_f08
+#define MPI_Dist_graph_neighbors PMPI_Dist_graph_neighbors
+#define MPI_Errhandler_free_f08 PMPI_Errhandler_free_f08
+#define MPI_Errhandler_free PMPI_Errhandler_free
+#define MPI_Error_class_f08 PMPI_Error_class_f08
+#define MPI_Error_class PMPI_Error_class
+#define MPI_Error_string_f08 PMPI_Error_string_f08
+#define MPI_Error_string PMPI_Error_string
+#define MPI_Exscan_f08 PMPI_Exscan_f08
+#define MPI_Exscan_init_f08 PMPI_Exscan_init_f08
+#define MPI_Exscan_init PMPI_Exscan_init
+#define MPI_Exscan PMPI_Exscan
+#define MPI_Fetch_and_op_f08 PMPI_Fetch_and_op_f08
+#define MPI_Fetch_and_op PMPI_Fetch_and_op
+#define MPI_File_call_errhandler_f08 PMPI_File_call_errhandler_f08
+#define MPI_File_call_errhandler PMPI_File_call_errhandler
+#define MPI_File_close_f08 PMPI_File_close_f08
+#define MPI_File_close PMPI_File_close
+#define MPI_File_create_errhandler_f08 PMPI_File_create_errhandler_f08
+#define MPI_File_create_errhandler PMPI_File_create_errhandler
+#define MPI_File_delete_f08 PMPI_File_delete_f08
+#define MPI_File_delete PMPI_File_delete
+#define MPI_File_get_amode_f08 PMPI_File_get_amode_f08
+#define MPI_File_get_amode PMPI_File_get_amode
+#define MPI_File_get_atomicity_f08 PMPI_File_get_atomicity_f08
+#define MPI_File_get_atomicity PMPI_File_get_atomicity
+#define MPI_File_get_byte_offset_f08 PMPI_File_get_byte_offset_f08
+#define MPI_File_get_byte_offset PMPI_File_get_byte_offset
+#define MPI_File_get_errhandler_f08 PMPI_File_get_errhandler_f08
+#define MPI_File_get_errhandler PMPI_File_get_errhandler
+#define MPI_File_get_group_f08 PMPI_File_get_group_f08
+#define MPI_File_get_group PMPI_File_get_group
+#define MPI_File_get_info_f08 PMPI_File_get_info_f08
+#define MPI_File_get_info PMPI_File_get_info
+#define MPI_File_get_position_f08 PMPI_File_get_position_f08
+#define MPI_File_get_position PMPI_File_get_position
+#define MPI_File_get_position_shared_f08 PMPI_File_get_position_shared_f08
+#define MPI_File_get_position_shared PMPI_File_get_position_shared
+#define MPI_File_get_size_f08 PMPI_File_get_size_f08
+#define MPI_File_get_size PMPI_File_get_size
+#define MPI_File_get_type_extent_f08 PMPI_File_get_type_extent_f08
+#define MPI_File_get_type_extent PMPI_File_get_type_extent
+#define MPI_File_get_view_f08 PMPI_File_get_view_f08
+#define MPI_File_get_view PMPI_File_get_view
+#define MPI_File_iread_all_f08 PMPI_File_iread_all_f08
+#define MPI_File_iread_all PMPI_File_iread_all
+#define MPI_File_iread_at_all_f08 PMPI_File_iread_at_all_f08
+#define MPI_File_iread_at_all PMPI_File_iread_at_all
+#define MPI_File_iread_at_f08 PMPI_File_iread_at_f08
+#define MPI_File_iread_at PMPI_File_iread_at
+#define MPI_File_iread_f08 PMPI_File_iread_f08
+#define MPI_File_iread PMPI_File_iread
+#define MPI_File_iread_shared_f08 PMPI_File_iread_shared_f08
+#define MPI_File_iread_shared PMPI_File_iread_shared
+#define MPI_File_iwrite_all_f08 PMPI_File_iwrite_all_f08
+#define MPI_File_iwrite_all PMPI_File_iwrite_all
+#define MPI_File_iwrite_at_all_f08 PMPI_File_iwrite_at_all_f08
+#define MPI_File_iwrite_at_all PMPI_File_iwrite_at_all
+#define MPI_File_iwrite_at_f08 PMPI_File_iwrite_at_f08
+#define MPI_File_iwrite_at PMPI_File_iwrite_at
+#define MPI_File_iwrite_f08 PMPI_File_iwrite_f08
+#define MPI_File_iwrite PMPI_File_iwrite
+#define MPI_File_iwrite_shared_f08 PMPI_File_iwrite_shared_f08
+#define MPI_File_iwrite_shared PMPI_File_iwrite_shared
+#define MPI_File_open_f08 PMPI_File_open_f08
+#define MPI_File_open PMPI_File_open
+#define MPI_File_preallocate_f08 PMPI_File_preallocate_f08
+#define MPI_File_preallocate PMPI_File_preallocate
+#define MPI_File_read_all_begin_f08 PMPI_File_read_all_begin_f08
+#define MPI_File_read_all_begin PMPI_File_read_all_begin
+#define MPI_File_read_all_end_f08 PMPI_File_read_all_end_f08
+#define MPI_File_read_all_end PMPI_File_read_all_end
+#define MPI_File_read_all_f08 PMPI_File_read_all_f08
+#define MPI_File_read_all PMPI_File_read_all
+#define MPI_File_read_at_all_begin_f08 PMPI_File_read_at_all_begin_f08
+#define MPI_File_read_at_all_begin PMPI_File_read_at_all_begin
+#define MPI_File_read_at_all_end_f08 PMPI_File_read_at_all_end_f08
+#define MPI_File_read_at_all_end PMPI_File_read_at_all_end
+#define MPI_File_read_at_all_f08 PMPI_File_read_at_all_f08
+#define MPI_File_read_at_all PMPI_File_read_at_all
+#define MPI_File_read_at_f08 PMPI_File_read_at_f08
+#define MPI_File_read_at PMPI_File_read_at
+#define MPI_File_read_f08 PMPI_File_read_f08
+#define MPI_File_read_ordered_begin_f08 PMPI_File_read_ordered_begin_f08
+#define MPI_File_read_ordered_begin PMPI_File_read_ordered_begin
+#define MPI_File_read_ordered_end_f08 PMPI_File_read_ordered_end_f08
+#define MPI_File_read_ordered_end PMPI_File_read_ordered_end
+#define MPI_File_read_ordered_f08 PMPI_File_read_ordered_f08
+#define MPI_File_read_ordered PMPI_File_read_ordered
+#define MPI_File_read PMPI_File_read
+#define MPI_File_read_shared_f08 PMPI_File_read_shared_f08
+#define MPI_File_read_shared PMPI_File_read_shared
+#define MPI_File_seek_f08 PMPI_File_seek_f08
+#define MPI_File_seek PMPI_File_seek
+#define MPI_File_seek_shared_f08 PMPI_File_seek_shared_f08
+#define MPI_File_seek_shared PMPI_File_seek_shared
+#define MPI_File_set_atomicity_f08 PMPI_File_set_atomicity_f08
+#define MPI_File_set_atomicity PMPI_File_set_atomicity
+#define MPI_File_set_errhandler_f08 PMPI_File_set_errhandler_f08
+#define MPI_File_set_errhandler PMPI_File_set_errhandler
+#define MPI_File_set_info_f08 PMPI_File_set_info_f08
+#define MPI_File_set_info PMPI_File_set_info
+#define MPI_File_set_size_f08 PMPI_File_set_size_f08
+#define MPI_File_set_size PMPI_File_set_size
+#define MPI_File_set_view_f08 PMPI_File_set_view_f08
+#define MPI_File_set_view PMPI_File_set_view
+#define MPI_File_sync_f08 PMPI_File_sync_f08
+#define MPI_File_sync PMPI_File_sync
+#define MPI_File_write_all_begin_f08 PMPI_File_write_all_begin_f08
+#define MPI_File_write_all_begin PMPI_File_write_all_begin
+#define MPI_File_write_all_end_f08 PMPI_File_write_all_end_f08
+#define MPI_File_write_all_end PMPI_File_write_all_end
+#define MPI_File_write_all_f08 PMPI_File_write_all_f08
+#define MPI_File_write_all PMPI_File_write_all
+#define MPI_File_write_at_all_begin_f08 PMPI_File_write_at_all_begin_f08
+#define MPI_File_write_at_all_begin PMPI_File_write_at_all_begin
+#define MPI_File_write_at_all_end_f08 PMPI_File_write_at_all_end_f08
+#define MPI_File_write_at_all_end PMPI_File_write_at_all_end
+#define MPI_File_write_at_all_f08 PMPI_File_write_at_all_f08
+#define MPI_File_write_at_all PMPI_File_write_at_all
+#define MPI_File_write_at_f08 PMPI_File_write_at_f08
+#define MPI_File_write_at PMPI_File_write_at
+#define MPI_File_write_f08 PMPI_File_write_f08
+#define MPI_File_write_ordered_begin_f08 PMPI_File_write_ordered_begin_f08
+#define MPI_File_write_ordered_begin PMPI_File_write_ordered_begin
+#define MPI_File_write_ordered_end_f08 PMPI_File_write_ordered_end_f08
+#define MPI_File_write_ordered_end PMPI_File_write_ordered_end
+#define MPI_File_write_ordered_f08 PMPI_File_write_ordered_f08
+#define MPI_File_write_ordered PMPI_File_write_ordered
+#define MPI_File_write PMPI_File_write
+#define MPI_File_write_shared_f08 PMPI_File_write_shared_f08
+#define MPI_File_write_shared PMPI_File_write_shared
+#define MPI_Finalized_f08 PMPI_Finalized_f08
+#define MPI_Finalized PMPI_Finalized
+#define MPI_Finalize_f08 PMPI_Finalize_f08
+#define MPI_Finalize PMPI_Finalize
+#define MPI_Free_mem_f08 PMPI_Free_mem_f08
+#define MPI_Free_mem PMPI_Free_mem
+#define MPI_F_sync_reg_f08 PMPI_F_sync_reg_f08
+#define MPI_F_sync_reg PMPI_F_sync_reg
+#define MPI_Gather_f08 PMPI_Gather_f08
+#define MPI_Gather_init_f08 PMPI_Gather_init_f08
+#define MPI_Gather_init PMPI_Gather_init
+#define MPI_Gather PMPI_Gather
+#define MPI_Gatherv_f08 PMPI_Gatherv_f08
+#define MPI_Gatherv_init_f08 PMPI_Gatherv_init_f08
+#define MPI_Gatherv_init PMPI_Gatherv_init
+#define MPI_Gatherv PMPI_Gatherv
+#define MPI_Get_accumulate_f08 PMPI_Get_accumulate_f08
+#define MPI_Get_accumulate PMPI_Get_accumulate
+#define MPI_Get_address_f08 PMPI_Get_address_f08
+#define MPI_Get_address PMPI_Get_address
+#define MPI_Get_count_f08 PMPI_Get_count_f08
+#define MPI_Get_count PMPI_Get_count
+#define MPI_Get_elements_f08 PMPI_Get_elements_f08
+#define MPI_Get_elements PMPI_Get_elements
+#define MPI_Get_elements_x_f08 PMPI_Get_elements_x_f08
+#define MPI_Get_elements_x PMPI_Get_elements_x
+#define MPI_Get_f08 PMPI_Get_f08
+#define MPI_Get_library_version_f08 PMPI_Get_library_version_f08
+#define MPI_Get_library_version PMPI_Get_library_version
+#define MPI_Get PMPI_Get
+#define MPI_Get_processor_name_f08 PMPI_Get_processor_name_f08
+#define MPI_Get_processor_name PMPI_Get_processor_name
+#define MPI_Get_version_f08 PMPI_Get_version_f08
+#define MPI_Get_version PMPI_Get_version
+#define MPI_Graph_create_f08 PMPI_Graph_create_f08
+#define MPI_Graph_create PMPI_Graph_create
+#define MPI_Graphdims_get_f08 PMPI_Graphdims_get_f08
+#define MPI_Graphdims_get PMPI_Graphdims_get
+#define MPI_Graph_get_f08 PMPI_Graph_get_f08
+#define MPI_Graph_get PMPI_Graph_get
+#define MPI_Graph_map_f08 PMPI_Graph_map_f08
+#define MPI_Graph_map PMPI_Graph_map
+#define MPI_Graph_neighbors_count_f08 PMPI_Graph_neighbors_count_f08
+#define MPI_Graph_neighbors_count PMPI_Graph_neighbors_count
+#define MPI_Graph_neighbors_f08 PMPI_Graph_neighbors_f08
+#define MPI_Graph_neighbors PMPI_Graph_neighbors
+#define MPI_Grequest_complete_f08 PMPI_Grequest_complete_f08
+#define MPI_Grequest_complete PMPI_Grequest_complete
+#define MPI_Grequest_start_f08 PMPI_Grequest_start_f08
+#define MPI_Grequest_start PMPI_Grequest_start
 #define MPI_Group_compare_f08 PMPI_Group_compare_f08
-#define MPI_Group_difference PMPI_Group_difference
+#define MPI_Group_compare PMPI_Group_compare
 #define MPI_Group_difference_f08 PMPI_Group_difference_f08
-#define MPI_Group_excl PMPI_Group_excl
+#define MPI_Group_difference PMPI_Group_difference
 #define MPI_Group_excl_f08 PMPI_Group_excl_f08
-#define MPI_Group_free PMPI_Group_free
+#define MPI_Group_excl PMPI_Group_excl
 #define MPI_Group_free_f08 PMPI_Group_free_f08
-#define MPI_Group_incl PMPI_Group_incl
+#define MPI_Group_free PMPI_Group_free
+#define MPI_Group_from_session_pset_f08 PMPI_Group_from_session_pset_f08
+#define MPI_Group_from_session_pset PMPI_Group_from_session_pset
 #define MPI_Group_incl_f08 PMPI_Group_incl_f08
-#define MPI_Group_intersection PMPI_Group_intersection
+#define MPI_Group_incl PMPI_Group_incl
 #define MPI_Group_intersection_f08 PMPI_Group_intersection_f08
-#define MPI_Group_range_excl PMPI_Group_range_excl
+#define MPI_Group_intersection PMPI_Group_intersection
 #define MPI_Group_range_excl_f08 PMPI_Group_range_excl_f08
-#define MPI_Group_range_incl PMPI_Group_range_incl
+#define MPI_Group_range_excl PMPI_Group_range_excl
 #define MPI_Group_range_incl_f08 PMPI_Group_range_incl_f08
-#define MPI_Group_rank PMPI_Group_rank
+#define MPI_Group_range_incl PMPI_Group_range_incl
 #define MPI_Group_rank_f08 PMPI_Group_rank_f08
-#define MPI_Group_size PMPI_Group_size
+#define MPI_Group_rank PMPI_Group_rank
 #define MPI_Group_size_f08 PMPI_Group_size_f08
-#define MPI_Group_translate_ranks PMPI_Group_translate_ranks
+#define MPI_Group_size PMPI_Group_size
 #define MPI_Group_translate_ranks_f08 PMPI_Group_translate_ranks_f08
-#define MPI_Group_union PMPI_Group_union
+#define MPI_Group_translate_ranks PMPI_Group_translate_ranks
 #define MPI_Group_union_f08 PMPI_Group_union_f08
-#define MPI_Intercomm_create PMPI_Intercomm_create
+#define MPI_Group_union PMPI_Group_union
+#define MPI_Iallgather_f08 PMPI_Iallgather_f08
+#define MPI_Iallgather PMPI_Iallgather
+#define MPI_Iallgatherv_f08 PMPI_Iallgatherv_f08
+#define MPI_Iallgatherv PMPI_Iallgatherv
+#define MPI_Iallreduce_f08 PMPI_Iallreduce_f08
+#define MPI_Iallreduce PMPI_Iallreduce
+#define MPI_Ialltoall_f08 PMPI_Ialltoall_f08
+#define MPI_Ialltoall PMPI_Ialltoall
+#define MPI_Ialltoallv_f08 PMPI_Ialltoallv_f08
+#define MPI_Ialltoallv PMPI_Ialltoallv
+#define MPI_Ialltoallw_f08 PMPI_Ialltoallw_f08
+#define MPI_Ialltoallw PMPI_Ialltoallw
+#define MPI_Ibarrier_f08 PMPI_Ibarrier_f08
+#define MPI_Ibarrier PMPI_Ibarrier
+#define MPI_Ibcast_f08 PMPI_Ibcast_f08
+#define MPI_Ibcast PMPI_Ibcast
+#define MPI_Ibsend_f08 PMPI_Ibsend_f08
+#define MPI_Ibsend PMPI_Ibsend
+#define MPI_Iexscan_f08 PMPI_Iexscan_f08
+#define MPI_Iexscan PMPI_Iexscan
+#define MPI_Igather_f08 PMPI_Igather_f08
+#define MPI_Igather PMPI_Igather
+#define MPI_Igatherv_f08 PMPI_Igatherv_f08
+#define MPI_Igatherv PMPI_Igatherv
+#define MPI_Improbe_f08 PMPI_Improbe_f08
+#define MPI_Improbe PMPI_Improbe
+#define MPI_Imrecv_f08 PMPI_Imrecv_f08
+#define MPI_Imrecv PMPI_Imrecv
+#define MPI_Ineighbor_allgather_f08 PMPI_Ineighbor_allgather_f08
+#define MPI_Ineighbor_allgather PMPI_Ineighbor_allgather
+#define MPI_Ineighbor_allgatherv_f08 PMPI_Ineighbor_allgatherv_f08
+#define MPI_Ineighbor_allgatherv PMPI_Ineighbor_allgatherv
+#define MPI_Ineighbor_alltoall_f08 PMPI_Ineighbor_alltoall_f08
+#define MPI_Ineighbor_alltoall PMPI_Ineighbor_alltoall
+#define MPI_Ineighbor_alltoallv_f08 PMPI_Ineighbor_alltoallv_f08
+#define MPI_Ineighbor_alltoallv PMPI_Ineighbor_alltoallv
+#define MPI_Ineighbor_alltoallw_f08 PMPI_Ineighbor_alltoallw_f08
+#define MPI_Ineighbor_alltoallw PMPI_Ineighbor_alltoallw
+#define MPI_Info_create_env_f08 PMPI_Info_create_env_f08
+#define MPI_Info_create_env PMPI_Info_create_env
+#define MPI_Info_create_f08 PMPI_Info_create_f08
+#define MPI_Info_create PMPI_Info_create
+#define MPI_Info_delete_f08 PMPI_Info_delete_f08
+#define MPI_Info_delete PMPI_Info_delete
+#define MPI_Info_dup_f08 PMPI_Info_dup_f08
+#define MPI_Info_dup PMPI_Info_dup
+#define MPI_Info_free_f08 PMPI_Info_free_f08
+#define MPI_Info_free PMPI_Info_free
+#define MPI_Info_get_f08 PMPI_Info_get_f08
+#define MPI_Info_get_nkeys_f08 PMPI_Info_get_nkeys_f08
+#define MPI_Info_get_nkeys PMPI_Info_get_nkeys
+#define MPI_Info_get_nthkey_f08 PMPI_Info_get_nthkey_f08
+#define MPI_Info_get_nthkey PMPI_Info_get_nthkey
+#define MPI_Info_get PMPI_Info_get
+#define MPI_Info_get_string_f08 PMPI_Info_get_string_f08
+#define MPI_Info_get_string PMPI_Info_get_string
+#define MPI_Info_get_valuelen_f08 PMPI_Info_get_valuelen_f08
+#define MPI_Info_get_valuelen PMPI_Info_get_valuelen
+#define MPI_Info_set_f08 PMPI_Info_set_f08
+#define MPI_Info_set PMPI_Info_set
+#define MPI_Init_f08 PMPI_Init_f08
+#define MPI_Initialized_f08 PMPI_Initialized_f08
+#define MPI_Initialized PMPI_Initialized
+#define MPI_Init PMPI_Init
+#define MPI_Init_thread_f08 PMPI_Init_thread_f08
+#define MPI_Init_thread PMPI_Init_thread
 #define MPI_Intercomm_create_f08 PMPI_Intercomm_create_f08
-#define MPI_Intercomm_merge PMPI_Intercomm_merge
+#define MPI_Intercomm_create_from_groups_f08 PMPI_Intercomm_create_from_groups_f08
+#define MPI_Intercomm_create_from_groups PMPI_Intercomm_create_from_groups
+#define MPI_Intercomm_create PMPI_Intercomm_create
 #define MPI_Intercomm_merge_f08 PMPI_Intercomm_merge_f08
-#define MPI_Type_create_keyval PMPI_Type_create_keyval
+#define MPI_Intercomm_merge PMPI_Intercomm_merge
+#define MPI_Iprobe_f08 PMPI_Iprobe_f08
+#define MPI_Iprobe PMPI_Iprobe
+#define MPI_Irecv_f08 PMPI_Irecv_f08
+#define MPI_Irecv PMPI_Irecv
+#define MPI_Ireduce_f08 PMPI_Ireduce_f08
+#define MPI_Ireduce PMPI_Ireduce
+#define MPI_Ireduce_scatter_block_f08 PMPI_Ireduce_scatter_block_f08
+#define MPI_Ireduce_scatter_block PMPI_Ireduce_scatter_block
+#define MPI_Ireduce_scatter_f08 PMPI_Ireduce_scatter_f08
+#define MPI_Ireduce_scatter PMPI_Ireduce_scatter
+#define MPI_Irsend_f08 PMPI_Irsend_f08
+#define MPI_Irsend PMPI_Irsend
+#define MPI_Iscan_f08 PMPI_Iscan_f08
+#define MPI_Iscan PMPI_Iscan
+#define MPI_Iscatter_f08 PMPI_Iscatter_f08
+#define MPI_Iscatter PMPI_Iscatter
+#define MPI_Iscatterv_f08 PMPI_Iscatterv_f08
+#define MPI_Iscatterv PMPI_Iscatterv
+#define MPI_Isend_f08 PMPI_Isend_f08
+#define MPI_Isend PMPI_Isend
+#define MPI_Isendrecv_f08 PMPI_Isendrecv_f08
+#define MPI_Isendrecv PMPI_Isendrecv
+#define MPI_Isendrecv_replace_f08 PMPI_Isendrecv_replace_f08
+#define MPI_Isendrecv_replace PMPI_Isendrecv_replace
+#define MPI_Issend_f08 PMPI_Issend_f08
+#define MPI_Issend PMPI_Issend
+#define MPI_Is_thread_main_f08 PMPI_Is_thread_main_f08
+#define MPI_Is_thread_main PMPI_Is_thread_main
+#define MPI_Lookup_name_f08 PMPI_Lookup_name_f08
+#define MPI_Lookup_name PMPI_Lookup_name
+#define MPI_Mprobe_f08 PMPI_Mprobe_f08
+#define MPI_Mprobe PMPI_Mprobe
+#define MPI_Mrecv_f08 PMPI_Mrecv_f08
+#define MPI_Mrecv PMPI_Mrecv
+#define MPI_Neighbor_allgather_f08 PMPI_Neighbor_allgather_f08
+#define MPI_Neighbor_allgather_init_f08 PMPI_Neighbor_allgather_init_f08
+#define MPI_Neighbor_allgather_init PMPI_Neighbor_allgather_init
+#define MPI_Neighbor_allgather PMPI_Neighbor_allgather
+#define MPI_Neighbor_allgatherv_f08 PMPI_Neighbor_allgatherv_f08
+#define MPI_Neighbor_allgatherv_init_f08 PMPI_Neighbor_allgatherv_init_f08
+#define MPI_Neighbor_allgatherv_init PMPI_Neighbor_allgatherv_init
+#define MPI_Neighbor_allgatherv PMPI_Neighbor_allgatherv
+#define MPI_Neighbor_alltoall_f08 PMPI_Neighbor_alltoall_f08
+#define MPI_Neighbor_alltoall_init_f08 PMPI_Neighbor_alltoall_init_f08
+#define MPI_Neighbor_alltoall_init PMPI_Neighbor_alltoall_init
+#define MPI_Neighbor_alltoall PMPI_Neighbor_alltoall
+#define MPI_Neighbor_alltoallv_f08 PMPI_Neighbor_alltoallv_f08
+#define MPI_Neighbor_alltoallv_init_f08 PMPI_Neighbor_alltoallv_init_f08
+#define MPI_Neighbor_alltoallv_init PMPI_Neighbor_alltoallv_init
+#define MPI_Neighbor_alltoallv PMPI_Neighbor_alltoallv
+#define MPI_Neighbor_alltoallw_f08 PMPI_Neighbor_alltoallw_f08
+#define MPI_Neighbor_alltoallw_init_f08 PMPI_Neighbor_alltoallw_init_f08
+#define MPI_Neighbor_alltoallw_init PMPI_Neighbor_alltoallw_init
+#define MPI_Neighbor_alltoallw PMPI_Neighbor_alltoallw
+#define MPI_Op_commutative_f08 PMPI_Op_commutative_f08
+#define MPI_Op_commutative PMPI_Op_commutative
+#define MPI_Op_create_f08 PMPI_Op_create_f08
+#define MPI_Op_create PMPI_Op_create
+#define MPI_Open_port_f08 PMPI_Open_port_f08
+#define MPI_Open_port PMPI_Open_port
+#define MPI_Op_free_f08 PMPI_Op_free_f08
+#define MPI_Op_free PMPI_Op_free
+#define MPI_Pack_external_f08 PMPI_Pack_external_f08
+#define MPI_Pack_external PMPI_Pack_external
+#define MPI_Pack_external_size_f08 PMPI_Pack_external_size_f08
+#define MPI_Pack_external_size PMPI_Pack_external_size
+#define MPI_Pack_f08 PMPI_Pack_f08
+#define MPI_Pack PMPI_Pack
+#define MPI_Pack_size_f08 PMPI_Pack_size_f08
+#define MPI_Pack_size PMPI_Pack_size
+#define MPI_Parrived_f08 PMPI_Parrived_f08
+#define MPI_Parrived PMPI_Parrived
+#define MPI_Pcontrol_f08 PMPI_Pcontrol_f08
+#define MPI_Pcontrol PMPI_Pcontrol
+#define MPI_Pready_f08 PMPI_Pready_f08
+#define MPI_Pready_list_f08 PMPI_Pready_list_f08
+#define MPI_Pready_list PMPI_Pready_list
+#define MPI_Pready PMPI_Pready
+#define MPI_Pready_range_f08 PMPI_Pready_range_f08
+#define MPI_Pready_range PMPI_Pready_range
+#define MPI_Precv_init_f08 PMPI_Precv_init_f08
+#define MPI_Precv_init PMPI_Precv_init
+#define MPI_Probe_f08 PMPI_Probe_f08
+#define MPI_Probe PMPI_Probe
+#define MPI_Psend_init_f08 PMPI_Psend_init_f08
+#define MPI_Psend_init PMPI_Psend_init
+#define MPI_Publish_name_f08 PMPI_Publish_name_f08
+#define MPI_Publish_name PMPI_Publish_name
+#define MPI_Put_f08 PMPI_Put_f08
+#define MPI_Put PMPI_Put
+#define MPI_Query_thread_f08 PMPI_Query_thread_f08
+#define MPI_Query_thread PMPI_Query_thread
+#define MPI_Raccumulate_f08 PMPI_Raccumulate_f08
+#define MPI_Raccumulate PMPI_Raccumulate
+#define MPI_Recv_f08_c PMPI_Recv_f08_c
+#define MPI_Recv_f08 PMPI_Recv_f08
+#define MPI_Recv_init_f08 PMPI_Recv_init_f08
+#define MPI_Recv_init PMPI_Recv_init
+#define MPI_Recv PMPI_Recv
+#define MPI_Reduce_f08 PMPI_Reduce_f08
+#define MPI_Reduce_init_f08 PMPI_Reduce_init_f08
+#define MPI_Reduce_init PMPI_Reduce_init
+#define MPI_Reduce_local_f08 PMPI_Reduce_local_f08
+#define MPI_Reduce_local PMPI_Reduce_local
+#define MPI_Reduce PMPI_Reduce
+#define MPI_Reduce_scatter_block_f08 PMPI_Reduce_scatter_block_f08
+#define MPI_Reduce_scatter_block_init_f08 PMPI_Reduce_scatter_block_init_f08
+#define MPI_Reduce_scatter_block_init PMPI_Reduce_scatter_block_init
+#define MPI_Reduce_scatter_block PMPI_Reduce_scatter_block
+#define MPI_Reduce_scatter_f08 PMPI_Reduce_scatter_f08
+#define MPI_Reduce_scatter_init_f08 PMPI_Reduce_scatter_init_f08
+#define MPI_Reduce_scatter_init PMPI_Reduce_scatter_init
+#define MPI_Reduce_scatter PMPI_Reduce_scatter
+#define MPI_Register_datarep_f08 PMPI_Register_datarep_f08
+#define MPI_Register_datarep PMPI_Register_datarep
+#define MPI_Request_free_f08 PMPI_Request_free_f08
+#define MPI_Request_free PMPI_Request_free
+#define MPI_Request_get_status_f08 PMPI_Request_get_status_f08
+#define MPI_Request_get_status PMPI_Request_get_status
+#define MPI_Rget_accumulate_f08 PMPI_Rget_accumulate_f08
+#define MPI_Rget_accumulate PMPI_Rget_accumulate
+#define MPI_Rget_f08 PMPI_Rget_f08
+#define MPI_Rget PMPI_Rget
+#define MPI_Rput_f08 PMPI_Rput_f08
+#define MPI_Rput PMPI_Rput
+#define MPI_Rsend_f08 PMPI_Rsend_f08
+#define MPI_Rsend_init_f08 PMPI_Rsend_init_f08
+#define MPI_Rsend_init PMPI_Rsend_init
+#define MPI_Rsend PMPI_Rsend
+#define MPI_Scan_f08 PMPI_Scan_f08
+#define MPI_Scan_init_f08 PMPI_Scan_init_f08
+#define MPI_Scan_init PMPI_Scan_init
+#define MPI_Scan PMPI_Scan
+#define MPI_Scatter_f08 PMPI_Scatter_f08
+#define MPI_Scatter_init_f08 PMPI_Scatter_init_f08
+#define MPI_Scatter_init PMPI_Scatter_init
+#define MPI_Scatter PMPI_Scatter
+#define MPI_Scatterv_f08 PMPI_Scatterv_f08
+#define MPI_Scatterv_init_f08 PMPI_Scatterv_init_f08
+#define MPI_Scatterv_init PMPI_Scatterv_init
+#define MPI_Scatterv PMPI_Scatterv
+#define MPI_Send_f08_c PMPI_Send_f08_c
+#define MPI_Send_f08 PMPI_Send_f08
+#define MPI_Send_init_f08 PMPI_Send_init_f08
+#define MPI_Send_init PMPI_Send_init
+#define MPI_Send PMPI_Send
+#define MPI_Sendrecv_f08 PMPI_Sendrecv_f08
+#define MPI_Sendrecv PMPI_Sendrecv
+#define MPI_Sendrecv_replace_f08 PMPI_Sendrecv_replace_f08
+#define MPI_Sendrecv_replace PMPI_Sendrecv_replace
+#define MPI_Session_call_errhandler_f08 PMPI_Session_call_errhandler_f08
+#define MPI_Session_call_errhandler PMPI_Session_call_errhandler
+#define MPI_Session_create_errhandler_f08 PMPI_Session_create_errhandler_f08
+#define MPI_Session_create_errhandler PMPI_Session_create_errhandler
+#define MPI_Session_finalize_f08 PMPI_Session_finalize_f08
+#define MPI_Session_finalize PMPI_Session_finalize
+#define MPI_Session_get_errhandler_f08 PMPI_Session_get_errhandler_f08
+#define MPI_Session_get_errhandler PMPI_Session_get_errhandler
+#define MPI_Session_get_info_f08 PMPI_Session_get_info_f08
+#define MPI_Session_get_info PMPI_Session_get_info
+#define MPI_Session_get_info PMPI_Session_get_info
+#define MPI_Session_get_nth_pset_f08 PMPI_Session_get_nth_pset_f08
+#define MPI_Session_get_nth_psetlen_f08 PMPI_Session_get_nth_psetlen_f08
+#define MPI_Session_get_nth_psetlen PMPI_Session_get_nth_psetlen
+#define MPI_Session_get_nth_pset PMPI_Session_get_nth_pset
+#define MPI_Session_get_num_psets_f08 PMPI_Session_get_num_psets_f08
+#define MPI_Session_get_num_psets PMPI_Session_get_num_psets
+#define MPI_Session_get_pset_info_f08 PMPI_Session_get_pset_info_f08
+#define MPI_Session_get_pset_info PMPI_Session_get_pset_info
+#define MPI_Session_init_f08 PMPI_Session_init_f08
+#define MPI_Session_init PMPI_Session_init
+#define MPI_Session_set_errhandler_f08 PMPI_Session_set_errhandler_f08
+#define MPI_Session_set_errhandler PMPI_Session_set_errhandler
+#define MPI_Ssend_f08 PMPI_Ssend_f08
+#define MPI_Ssend_init_f08 PMPI_Ssend_init_f08
+#define MPI_Ssend_init PMPI_Ssend_init
+#define MPI_Ssend PMPI_Ssend
+#define MPI_Startall_f08 PMPI_Startall_f08
+#define MPI_Startall PMPI_Startall
+#define MPI_Start_f08 PMPI_Start_f08
+#define MPI_Start PMPI_Start
+#define MPI_Status_f082f_f08 PMPI_Status_f082f_f08
+#define MPI_Status_f082f PMPI_Status_f082f
+#define MPI_Status_f2f08_f08 PMPI_Status_f2f08_f08
+#define MPI_Status_f2f08 PMPI_Status_f2f08
+#define MPI_Status_set_cancelled_f08 PMPI_Status_set_cancelled_f08
+#define MPI_Status_set_cancelled PMPI_Status_set_cancelled
+#define MPI_Status_set_elements_f08 PMPI_Status_set_elements_f08
+#define MPI_Status_set_elements PMPI_Status_set_elements
+#define MPI_Status_set_elements_x_f08 PMPI_Status_set_elements_x_f08
+#define MPI_Status_set_elements_x PMPI_Status_set_elements_x
+#define MPI_Testall_f08 PMPI_Testall_f08
+#define MPI_Testall PMPI_Testall
+#define MPI_Testany_f08 PMPI_Testany_f08
+#define MPI_Testany PMPI_Testany
+#define MPI_Test_cancelled_f08 PMPI_Test_cancelled_f08
+#define MPI_Test_cancelled PMPI_Test_cancelled
+#define MPI_Test_f08 PMPI_Test_f08
+#define MPI_Test PMPI_Test
+#define MPI_Testsome_f08 PMPI_Testsome_f08
+#define MPI_Testsome PMPI_Testsome
+#define MPI_Topo_test_f08 PMPI_Topo_test_f08
+#define MPI_Topo_test PMPI_Topo_test
+#define MPI_Type_commit_f08 PMPI_Type_commit_f08
+#define MPI_Type_commit PMPI_Type_commit
+#define MPI_Type_contiguous_f08 PMPI_Type_contiguous_f08
+#define MPI_Type_contiguous PMPI_Type_contiguous
+#define MPI_Type_create_darray_f08 PMPI_Type_create_darray_f08
+#define MPI_Type_create_darray PMPI_Type_create_darray
+#define MPI_Type_create_f90_complex_f08 PMPI_Type_create_f90_complex_f08
+#define MPI_Type_create_f90_complex PMPI_Type_create_f90_complex
+#define MPI_Type_create_f90_integer_f08 PMPI_Type_create_f90_integer_f08
+#define MPI_Type_create_f90_integer PMPI_Type_create_f90_integer
+#define MPI_Type_create_f90_real_f08 PMPI_Type_create_f90_real_f08
+#define MPI_Type_create_f90_real PMPI_Type_create_f90_real
+#define MPI_Type_create_hindexed_block_f08 PMPI_Type_create_hindexed_block_f08
+#define MPI_Type_create_hindexed_block PMPI_Type_create_hindexed_block
+#define MPI_Type_create_hindexed_f08 PMPI_Type_create_hindexed_f08
+#define MPI_Type_create_hindexed PMPI_Type_create_hindexed
+#define MPI_Type_create_hvector_f08 PMPI_Type_create_hvector_f08
+#define MPI_Type_create_hvector PMPI_Type_create_hvector
+#define MPI_Type_create_indexed_block_f08 PMPI_Type_create_indexed_block_f08
+#define MPI_Type_create_indexed_block PMPI_Type_create_indexed_block
 #define MPI_Type_create_keyval_f08 PMPI_Type_create_keyval_f08
-#define MPI_Type_delete_attr PMPI_Type_delete_attr
+#define MPI_Type_create_keyval PMPI_Type_create_keyval
+#define MPI_Type_create_resized_f08 PMPI_Type_create_resized_f08
+#define MPI_Type_create_resized PMPI_Type_create_resized
+#define MPI_Type_create_struct_f08 PMPI_Type_create_struct_f08
+#define MPI_Type_create_struct PMPI_Type_create_struct
+#define MPI_Type_create_subarray_f08 PMPI_Type_create_subarray_f08
+#define MPI_Type_create_subarray PMPI_Type_create_subarray
 #define MPI_Type_delete_attr_f08 PMPI_Type_delete_attr_f08
-#define MPI_Type_free_keyval PMPI_Type_free_keyval
+#define MPI_Type_delete_attr PMPI_Type_delete_attr
+#define MPI_Type_dup_f08 PMPI_Type_dup_f08
+#define MPI_Type_dup PMPI_Type_dup
+#define MPI_Type_free_f08 PMPI_Type_free_f08
 #define MPI_Type_free_keyval_f08 PMPI_Type_free_keyval_f08
-#define MPI_Type_get_attr PMPI_Type_get_attr
+#define MPI_Type_free_keyval PMPI_Type_free_keyval
+#define MPI_Type_free PMPI_Type_free
 #define MPI_Type_get_attr_f08 PMPI_Type_get_attr_f08
-#define MPI_Type_get_name PMPI_Type_get_name
+#define MPI_Type_get_attr PMPI_Type_get_attr
+#define MPI_Type_get_contents_f08_c PMPI_Type_get_contents_f08_c
+#define MPI_Type_get_contents_f08 PMPI_Type_get_contents_f08
+#define MPI_Type_get_contents PMPI_Type_get_contents
+#define MPI_Type_get_envelope_f08_c PMPI_Type_get_envelope_f08_c
+#define MPI_Type_get_envelope_f08 PMPI_Type_get_envelope_f08
+#define MPI_Type_get_envelope PMPI_Type_get_envelope
+#define MPI_Type_get_extent_f08 PMPI_Type_get_extent_f08
+#define MPI_Type_get_extent PMPI_Type_get_extent
+#define MPI_Type_get_extent_x_f08 PMPI_Type_get_extent_x_f08
+#define MPI_Type_get_extent_x PMPI_Type_get_extent_x
 #define MPI_Type_get_name_f08 PMPI_Type_get_name_f08
-#define MPI_Type_set_attr PMPI_Type_set_attr
+#define MPI_Type_get_name PMPI_Type_get_name
+#define MPI_Type_get_true_extent_f08 PMPI_Type_get_true_extent_f08
+#define MPI_Type_get_true_extent PMPI_Type_get_true_extent
+#define MPI_Type_get_true_extent_x_f08 PMPI_Type_get_true_extent_x_f08
+#define MPI_Type_get_true_extent_x PMPI_Type_get_true_extent_x
+#define MPI_Type_indexed_f08 PMPI_Type_indexed_f08
+#define MPI_Type_indexed PMPI_Type_indexed
+#define MPI_Type_match_size_f08 PMPI_Type_match_size_f08
+#define MPI_Type_match_size PMPI_Type_match_size
 #define MPI_Type_set_attr_f08 PMPI_Type_set_attr_f08
-#define MPI_Type_set_name PMPI_Type_set_name
+#define MPI_Type_set_attr PMPI_Type_set_attr
 #define MPI_Type_set_name_f08 PMPI_Type_set_name_f08
-#define MPI_Win_allocate PMPI_Win_allocate
+#define MPI_Type_set_name PMPI_Type_set_name
+#define MPI_Type_size_f08 PMPI_Type_size_f08
+#define MPI_Type_size PMPI_Type_size
+#define MPI_Type_size_x_f08 PMPI_Type_size_x_f08
+#define MPI_Type_size_x PMPI_Type_size_x
+#define MPI_Type_vector_f08 PMPI_Type_vector_f08
+#define MPI_Type_vector PMPI_Type_vector
+#define MPI_Unpack_external_f08 PMPI_Unpack_external_f08
+#define MPI_Unpack_external PMPI_Unpack_external
+#define MPI_Unpack_f08 PMPI_Unpack_f08
+#define MPI_Unpack PMPI_Unpack
+#define MPI_Unpublish_name_f08 PMPI_Unpublish_name_f08
+#define MPI_Unpublish_name PMPI_Unpublish_name
+#define MPI_Waitall_f08 PMPI_Waitall_f08
+#define MPI_Waitall PMPI_Waitall
+#define MPI_Waitany_f08 PMPI_Waitany_f08
+#define MPI_Waitany PMPI_Waitany
+#define MPI_Wait_f08 PMPI_Wait_f08
+#define MPI_Wait PMPI_Wait
+#define MPI_Waitsome_f08 PMPI_Waitsome_f08
+#define MPI_Waitsome PMPI_Waitsome
 #define MPI_Win_allocate_f08 PMPI_Win_allocate_f08
-#define MPI_Win_allocate_shared PMPI_Win_allocate_shared
+#define MPI_Win_allocate PMPI_Win_allocate
 #define MPI_Win_allocate_shared_f08 PMPI_Win_allocate_shared_f08
-#define MPI_Win_create_keyval PMPI_Win_create_keyval
-#define MPI_Win_create_keyval_f08 PMPI_Win_create_keyval_f08
-#define MPI_Win_delete_attr PMPI_Win_delete_attr
-#define MPI_Win_delete_attr_f08 PMPI_Win_delete_attr_f08
-#define MPI_Win_free_keyval PMPI_Win_free_keyval
-#define MPI_Win_free_keyval_f08 PMPI_Win_free_keyval_f08
-#define MPI_Win_get_attr PMPI_Win_get_attr
-#define MPI_Win_get_attr_f08 PMPI_Win_get_attr_f08
-#define MPI_Win_get_info PMPI_Win_get_info
-#define MPI_Win_get_info_f08 PMPI_Win_get_info_f08
-#define MPI_Win_get_name PMPI_Win_get_name
-#define MPI_Win_get_name_f08 PMPI_Win_get_name_f08
-#define MPI_Win_set_attr PMPI_Win_set_attr
-#define MPI_Win_set_attr_f08 PMPI_Win_set_attr_f08
-#define MPI_Win_set_info PMPI_Win_set_info
-#define MPI_Win_set_info_f08 PMPI_Win_set_info_f08
-#define MPI_Win_set_name PMPI_Win_set_name
-#define MPI_Win_set_name_f08 PMPI_Win_set_name_f08
-#define MPI_Cartdim_get PMPI_Cartdim_get
-#define MPI_Cartdim_get_f08 PMPI_Cartdim_get_f08
-#define MPI_Cart_coords PMPI_Cart_coords
-#define MPI_Cart_coords_f08 PMPI_Cart_coords_f08
-#define MPI_Cart_create PMPI_Cart_create
-#define MPI_Cart_create_f08 PMPI_Cart_create_f08
-#define MPI_Cart_get PMPI_Cart_get
-#define MPI_Cart_get_f08 PMPI_Cart_get_f08
-#define MPI_Cart_map PMPI_Cart_map
-#define MPI_Cart_map_f08 PMPI_Cart_map_f08
-#define MPI_Cart_rank PMPI_Cart_rank
-#define MPI_Cart_rank_f08 PMPI_Cart_rank_f08
-#define MPI_Cart_shift PMPI_Cart_shift
-#define MPI_Cart_shift_f08 PMPI_Cart_shift_f08
-#define MPI_Cart_sub PMPI_Cart_sub
-#define MPI_Cart_sub_f08 PMPI_Cart_sub_f08
-#define MPI_Dims_create PMPI_Dims_create
-#define MPI_Dims_create_f08 PMPI_Dims_create_f08
-#define MPI_Dist_graph_create PMPI_Dist_graph_create
-#define MPI_Dist_graph_create_f08 PMPI_Dist_graph_create_f08
-#define MPI_Dist_graph_create_adjacent PMPI_Dist_graph_create_adjacent
-#define MPI_Dist_graph_create_adjacent_f08 PMPI_Dist_graph_create_adjacent_f08
-#define MPI_Dist_graph_neighbors PMPI_Dist_graph_neighbors
-#define MPI_Dist_graph_neighbors_f08 PMPI_Dist_graph_neighbors_f08
-#define MPI_Dist_graph_neighbors_count PMPI_Dist_graph_neighbors_count
-#define MPI_Dist_graph_neighbors_count_f08 PMPI_Dist_graph_neighbors_count_f08
-#define MPI_Graphdims_get PMPI_Graphdims_get
-#define MPI_Graphdims_get_f08 PMPI_Graphdims_get_f08
-#define MPI_Graph_create PMPI_Graph_create
-#define MPI_Graph_create_f08 PMPI_Graph_create_f08
-#define MPI_Graph_get PMPI_Graph_get
-#define MPI_Graph_get_f08 PMPI_Graph_get_f08
-#define MPI_Graph_map PMPI_Graph_map
-#define MPI_Graph_map_f08 PMPI_Graph_map_f08
-#define MPI_Graph_neighbors PMPI_Graph_neighbors
-#define MPI_Graph_neighbors_f08 PMPI_Graph_neighbors_f08
-#define MPI_Graph_neighbors_count PMPI_Graph_neighbors_count
-#define MPI_Graph_neighbors_count_f08 PMPI_Graph_neighbors_count_f08
-#define MPI_Topo_test PMPI_Topo_test
-#define MPI_Topo_test_f08 PMPI_Topo_test_f08
-#define MPI_Aint_add PMPI_Aint_add
-#define MPI_Aint_add_f08 PMPI_Aint_add_f08
-#define MPI_Aint_diff PMPI_Aint_diff
-#define MPI_Aint_diff_f08 PMPI_Aint_diff_f08
-#define MPI_Abort PMPI_Abort
-#define MPI_Abort_f08 PMPI_Abort_f08
-#define MPI_Add_error_class PMPI_Add_error_class
-#define MPI_Add_error_class_f08 PMPI_Add_error_class_f08
-#define MPI_Add_error_code PMPI_Add_error_code
-#define MPI_Add_error_code_f08 PMPI_Add_error_code_f08
-#define MPI_Add_error_string PMPI_Add_error_string
-#define MPI_Add_error_string_f08 PMPI_Add_error_string_f08
-#define MPI_Alloc_mem PMPI_Alloc_mem
-#define MPI_Alloc_mem_f08 PMPI_Alloc_mem_f08
-#define MPI_Comm_call_errhandler PMPI_Comm_call_errhandler
-#define MPI_Comm_call_errhandler_f08 PMPI_Comm_call_errhandler_f08
-#define MPI_Comm_create_errhandler PMPI_Comm_create_errhandler
-#define MPI_Comm_create_errhandler_f08 PMPI_Comm_create_errhandler_f08
-#define MPI_Comm_get_errhandler PMPI_Comm_get_errhandler
-#define MPI_Comm_get_errhandler_f08 PMPI_Comm_get_errhandler_f08
-#define MPI_Comm_set_errhandler PMPI_Comm_set_errhandler
-#define MPI_Comm_set_errhandler_f08 PMPI_Comm_set_errhandler_f08
-#define MPI_Errhandler_free PMPI_Errhandler_free
-#define MPI_Errhandler_free_f08 PMPI_Errhandler_free_f08
-#define MPI_Error_class PMPI_Error_class
-#define MPI_Error_class_f08 PMPI_Error_class_f08
-#define MPI_Error_string PMPI_Error_string
-#define MPI_Error_string_f08 PMPI_Error_string_f08
-#define MPI_File_call_errhandler PMPI_File_call_errhandler
-#define MPI_File_call_errhandler_f08 PMPI_File_call_errhandler_f08
-#define MPI_File_create_errhandler PMPI_File_create_errhandler
-#define MPI_File_create_errhandler_f08 PMPI_File_create_errhandler_f08
-#define MPI_File_get_errhandler PMPI_File_get_errhandler
-#define MPI_File_get_errhandler_f08 PMPI_File_get_errhandler_f08
-#define MPI_File_set_errhandler PMPI_File_set_errhandler
-#define MPI_File_set_errhandler_f08 PMPI_File_set_errhandler_f08
-#define MPI_Finalize PMPI_Finalize
-#define MPI_Finalize_f08 PMPI_Finalize_f08
-#define MPI_Finalized PMPI_Finalized
-#define MPI_Finalized_f08 PMPI_Finalized_f08
-#define MPI_Free_mem PMPI_Free_mem
-#define MPI_Free_mem_f08 PMPI_Free_mem_f08
-#define MPI_Get_processor_name PMPI_Get_processor_name
-#define MPI_Get_processor_name_f08 PMPI_Get_processor_name_f08
-#define MPI_Get_version PMPI_Get_version
-#define MPI_Get_version_f08 PMPI_Get_version_f08
-#define MPI_Init PMPI_Init
-#define MPI_Init_f08 PMPI_Init_f08
-#define MPI_Initialized PMPI_Initialized
-#define MPI_Initialized_f08 PMPI_Initialized_f08
-#define MPI_Win_call_errhandler PMPI_Win_call_errhandler
+#define MPI_Win_allocate_shared PMPI_Win_allocate_shared
+#define MPI_Win_attach_f08 PMPI_Win_attach_f08
+#define MPI_Win_attach PMPI_Win_attach
 #define MPI_Win_call_errhandler_f08 PMPI_Win_call_errhandler_f08
-#define MPI_Win_create_errhandler PMPI_Win_create_errhandler
-#define MPI_Win_create_errhandler_f08 PMPI_Win_create_errhandler_f08
-#define MPI_Win_get_errhandler PMPI_Win_get_errhandler
-#define MPI_Win_get_errhandler_f08 PMPI_Win_get_errhandler_f08
-#define MPI_Win_set_errhandler PMPI_Win_set_errhandler
-#define MPI_Win_set_errhandler_f08 PMPI_Win_set_errhandler_f08
-#define MPI_Info_create PMPI_Info_create
-#define MPI_Info_create_f08 PMPI_Info_create_f08
-#define MPI_Info_delete PMPI_Info_delete
-#define MPI_Info_delete_f08 PMPI_Info_delete_f08
-#define MPI_Info_dup PMPI_Info_dup
-#define MPI_Info_dup_f08 PMPI_Info_dup_f08
-#define MPI_Info_free PMPI_Info_free
-#define MPI_Info_free_f08 PMPI_Info_free_f08
-#define MPI_Info_get PMPI_Info_get
-#define MPI_Info_get_f08 PMPI_Info_get_f08
-#define MPI_Info_get_nkeys PMPI_Info_get_nkeys
-#define MPI_Info_get_nkeys_f08 PMPI_Info_get_nkeys_f08
-#define MPI_Info_get_nthkey PMPI_Info_get_nthkey
-#define MPI_Info_get_nthkey_f08 PMPI_Info_get_nthkey_f08
-#define MPI_Info_get_valuelen PMPI_Info_get_valuelen
-#define MPI_Info_get_valuelen_f08 PMPI_Info_get_valuelen_f08
-#define MPI_Info_set PMPI_Info_set
-#define MPI_Info_set_f08 PMPI_Info_set_f08
-#define MPI_Close_port PMPI_Close_port
-#define MPI_Close_port_f08 PMPI_Close_port_f08
-#define MPI_Comm_accept PMPI_Comm_accept
-#define MPI_Comm_accept_f08 PMPI_Comm_accept_f08
-#define MPI_Comm_connect PMPI_Comm_connect
-#define MPI_Comm_connect_f08 PMPI_Comm_connect_f08
-#define MPI_Comm_disconnect PMPI_Comm_disconnect
-#define MPI_Comm_disconnect_f08 PMPI_Comm_disconnect_f08
-#define MPI_Comm_get_parent PMPI_Comm_get_parent
-#define MPI_Comm_get_parent_f08 PMPI_Comm_get_parent_f08
-#define MPI_Comm_join PMPI_Comm_join
-#define MPI_Comm_join_f08 PMPI_Comm_join_f08
-#define MPI_Comm_spawn PMPI_Comm_spawn
-#define MPI_Comm_spawn_f08 PMPI_Comm_spawn_f08
-#define MPI_Comm_spawn_multiple PMPI_Comm_spawn_multiple
-#define MPI_Comm_spawn_multiple_f08 PMPI_Comm_spawn_multiple_f08
-#define MPI_Lookup_name PMPI_Lookup_name
-#define MPI_Lookup_name_f08 PMPI_Lookup_name_f08
-#define MPI_Open_port PMPI_Open_port
-#define MPI_Open_port_f08 PMPI_Open_port_f08
-#define MPI_Publish_name PMPI_Publish_name
-#define MPI_Publish_name_f08 PMPI_Publish_name_f08
-#define MPI_Unpublish_name PMPI_Unpublish_name
-#define MPI_Unpublish_name_f08 PMPI_Unpublish_name_f08
-#define MPI_Accumulate PMPI_Accumulate
-#define MPI_Accumulate_f08 PMPI_Accumulate_f08
-#define MPI_Raccumulate PMPI_Raccumulate
-#define MPI_Raccumulate_f08 PMPI_Raccumulate_f08
-#define MPI_Get PMPI_Get
-#define MPI_Get_f08 PMPI_Get_f08
-#define MPI_Rget PMPI_Rget
-#define MPI_Rget_f08 PMPI_Rget_f08
-#define MPI_Get_accumulate PMPI_Get_accumulate
-#define MPI_Get_accumulate_f08 PMPI_Get_accumulate_f08
-#define MPI_Rget_accumulate PMPI_Rget_accumulate
-#define MPI_Rget_accumulate_f08 PMPI_Rget_accumulate_f08
-#define MPI_Put PMPI_Put
-#define MPI_Put_f08 PMPI_Put_f08
-#define MPI_Rput PMPI_Rput
-#define MPI_Rput_f08 PMPI_Rput_f08
-#define MPI_Fetch_and_op PMPI_Fetch_and_op
-#define MPI_Fetch_and_op_f08 PMPI_Fetch_and_op_f08
-#define MPI_Compare_and_swap PMPI_Compare_and_swap
-#define MPI_Compare_and_swap_f08 PMPI_Compare_and_swap_f08
-#define MPI_Win_complete PMPI_Win_complete
+#define MPI_Win_call_errhandler PMPI_Win_call_errhandler
 #define MPI_Win_complete_f08 PMPI_Win_complete_f08
-#define MPI_Win_create PMPI_Win_create
-#define MPI_Win_create_f08 PMPI_Win_create_f08
-#define MPI_Win_create_dynamic PMPI_Win_create_dynamic
+#define MPI_Win_complete PMPI_Win_complete
 #define MPI_Win_create_dynamic_f08 PMPI_Win_create_dynamic_f08
-#define MPI_Win_attach PMPI_Win_attach
-#define MPI_Win_attach_f08 PMPI_Win_attach_f08
-#define MPI_Win_detach PMPI_Win_detach
+#define MPI_Win_create_dynamic PMPI_Win_create_dynamic
+#define MPI_Win_create_errhandler_f08 PMPI_Win_create_errhandler_f08
+#define MPI_Win_create_errhandler PMPI_Win_create_errhandler
+#define MPI_Win_create_f08 PMPI_Win_create_f08
+#define MPI_Win_create_keyval_f08 PMPI_Win_create_keyval_f08
+#define MPI_Win_create_keyval PMPI_Win_create_keyval
+#define MPI_Win_create PMPI_Win_create
+#define MPI_Win_delete_attr_f08 PMPI_Win_delete_attr_f08
+#define MPI_Win_delete_attr PMPI_Win_delete_attr
 #define MPI_Win_detach_f08 PMPI_Win_detach_f08
-#define MPI_Win_fence PMPI_Win_fence
+#define MPI_Win_detach PMPI_Win_detach
 #define MPI_Win_fence_f08 PMPI_Win_fence_f08
-#define MPI_Win_free PMPI_Win_free
+#define MPI_Win_fence PMPI_Win_fence
+#define MPI_Win_flush_all_f08 PMPI_Win_flush_all_f08
+#define MPI_Win_flush_all PMPI_Win_flush_all
+#define MPI_Win_flush_f08 PMPI_Win_flush_f08
+#define MPI_Win_flush_local_all_f08 PMPI_Win_flush_local_all_f08
+#define MPI_Win_flush_local_all PMPI_Win_flush_local_all
+#define MPI_Win_flush_local_f08 PMPI_Win_flush_local_f08
+#define MPI_Win_flush_local PMPI_Win_flush_local
+#define MPI_Win_flush PMPI_Win_flush
 #define MPI_Win_free_f08 PMPI_Win_free_f08
-#define MPI_Win_get_group PMPI_Win_get_group
+#define MPI_Win_free_keyval_f08 PMPI_Win_free_keyval_f08
+#define MPI_Win_free_keyval PMPI_Win_free_keyval
+#define MPI_Win_free PMPI_Win_free
+#define MPI_Win_get_attr_f08 PMPI_Win_get_attr_f08
+#define MPI_Win_get_attr PMPI_Win_get_attr
+#define MPI_Win_get_errhandler_f08 PMPI_Win_get_errhandler_f08
+#define MPI_Win_get_errhandler PMPI_Win_get_errhandler
 #define MPI_Win_get_group_f08 PMPI_Win_get_group_f08
-#define MPI_Win_lock PMPI_Win_lock
-#define MPI_Win_lock_f08 PMPI_Win_lock_f08
-#define MPI_Win_lock_all PMPI_Win_lock_all
+#define MPI_Win_get_group PMPI_Win_get_group
+#define MPI_Win_get_info_f08 PMPI_Win_get_info_f08
+#define MPI_Win_get_info PMPI_Win_get_info
+#define MPI_Win_get_name_f08 PMPI_Win_get_name_f08
+#define MPI_Win_get_name PMPI_Win_get_name
 #define MPI_Win_lock_all_f08 PMPI_Win_lock_all_f08
-#define MPI_Win_post PMPI_Win_post
+#define MPI_Win_lock_all PMPI_Win_lock_all
+#define MPI_Win_lock_f08 PMPI_Win_lock_f08
+#define MPI_Win_lock PMPI_Win_lock
 #define MPI_Win_post_f08 PMPI_Win_post_f08
-#define MPI_Win_shared_query PMPI_Win_shared_query
+#define MPI_Win_post PMPI_Win_post
+#define MPI_Win_set_attr_f08 PMPI_Win_set_attr_f08
+#define MPI_Win_set_attr PMPI_Win_set_attr
+#define MPI_Win_set_errhandler_f08 PMPI_Win_set_errhandler_f08
+#define MPI_Win_set_errhandler PMPI_Win_set_errhandler
+#define MPI_Win_set_info_f08 PMPI_Win_set_info_f08
+#define MPI_Win_set_info PMPI_Win_set_info
+#define MPI_Win_set_name_f08 PMPI_Win_set_name_f08
+#define MPI_Win_set_name PMPI_Win_set_name
 #define MPI_Win_shared_query_f08 PMPI_Win_shared_query_f08
-#define MPI_Win_start PMPI_Win_start
+#define MPI_Win_shared_query PMPI_Win_shared_query
 #define MPI_Win_start_f08 PMPI_Win_start_f08
-#define MPI_Win_sync PMPI_Win_sync
+#define MPI_Win_start PMPI_Win_start
 #define MPI_Win_sync_f08 PMPI_Win_sync_f08
-#define MPI_Win_test PMPI_Win_test
+#define MPI_Win_sync PMPI_Win_sync
 #define MPI_Win_test_f08 PMPI_Win_test_f08
-#define MPI_Win_unlock PMPI_Win_unlock
-#define MPI_Win_unlock_f08 PMPI_Win_unlock_f08
-#define MPI_Win_unlock_all PMPI_Win_unlock_all
+#define MPI_Win_test PMPI_Win_test
 #define MPI_Win_unlock_all_f08 PMPI_Win_unlock_all_f08
-#define MPI_Win_wait PMPI_Win_wait
+#define MPI_Win_unlock_all PMPI_Win_unlock_all
+#define MPI_Win_unlock_f08 PMPI_Win_unlock_f08
+#define MPI_Win_unlock PMPI_Win_unlock
 #define MPI_Win_wait_f08 PMPI_Win_wait_f08
-#define MPI_Win_flush PMPI_Win_flush
-#define MPI_Win_flush_f08 PMPI_Win_flush_f08
-#define MPI_Win_flush_local PMPI_Win_flush_local
-#define MPI_Win_flush_local_f08 PMPI_Win_flush_local_f08
-#define MPI_Win_flush_local_all PMPI_Win_flush_local_all
-#define MPI_Win_flush_local_all_f08 PMPI_Win_flush_local_all_f08
-#define MPI_Win_flush_all PMPI_Win_flush_all
-#define MPI_Win_flush_all_f08 PMPI_Win_flush_all_f08
-#define MPI_Grequest_complete PMPI_Grequest_complete
-#define MPI_Grequest_complete_f08 PMPI_Grequest_complete_f08
-#define MPI_Grequest_start PMPI_Grequest_start
-#define MPI_Grequest_start_f08 PMPI_Grequest_start_f08
-#define MPI_Init_thread PMPI_Init_thread
-#define MPI_Init_thread_f08 PMPI_Init_thread_f08
-#define MPI_Is_thread_main PMPI_Is_thread_main
-#define MPI_Is_thread_main_f08 PMPI_Is_thread_main_f08
-#define MPI_Query_thread PMPI_Query_thread
-#define MPI_Query_thread_f08 PMPI_Query_thread_f08
-#define MPI_Status_f082f_f08 PMPI_Status_f082f_f08
-#define MPI_Status_f2f08_f08 PMPI_Status_f2f08_f08
-#define MPI_Status_set_cancelled PMPI_Status_set_cancelled
-#define MPI_Status_set_cancelled_f08 PMPI_Status_set_cancelled_f08
-#define MPI_Status_set_elements PMPI_Status_set_elements
-#define MPI_Status_set_elements_f08 PMPI_Status_set_elements_f08
-#define MPI_Status_set_elements_x PMPI_Status_set_elements_x
-#define MPI_Status_set_elements_x_f08 PMPI_Status_set_elements_x_f08
-#define MPI_File_close PMPI_File_close
-#define MPI_File_close_f08 PMPI_File_close_f08
-#define MPI_File_delete PMPI_File_delete
-#define MPI_File_delete_f08 PMPI_File_delete_f08
-#define MPI_File_get_amode PMPI_File_get_amode
-#define MPI_File_get_amode_f08 PMPI_File_get_amode_f08
-#define MPI_File_get_atomicity PMPI_File_get_atomicity
-#define MPI_File_get_atomicity_f08 PMPI_File_get_atomicity_f08
-#define MPI_File_get_byte_offset PMPI_File_get_byte_offset
-#define MPI_File_get_byte_offset_f08 PMPI_File_get_byte_offset_f08
-#define MPI_File_get_group PMPI_File_get_group
-#define MPI_File_get_group_f08 PMPI_File_get_group_f08
-#define MPI_File_get_info PMPI_File_get_info
-#define MPI_File_get_info_f08 PMPI_File_get_info_f08
-#define MPI_File_get_position PMPI_File_get_position
-#define MPI_File_get_position_f08 PMPI_File_get_position_f08
-#define MPI_File_get_position_shared PMPI_File_get_position_shared
-#define MPI_File_get_position_shared_f08 PMPI_File_get_position_shared_f08
-#define MPI_File_get_size PMPI_File_get_size
-#define MPI_File_get_size_f08 PMPI_File_get_size_f08
-#define MPI_File_get_type_extent PMPI_File_get_type_extent
-#define MPI_File_get_type_extent_f08 PMPI_File_get_type_extent_f08
-#define MPI_File_get_view PMPI_File_get_view
-#define MPI_File_get_view_f08 PMPI_File_get_view_f08
-#define MPI_File_iread PMPI_File_iread
-#define MPI_File_iread_f08 PMPI_File_iread_f08
-#define MPI_File_iread_at PMPI_File_iread_at
-#define MPI_File_iread_at_f08 PMPI_File_iread_at_f08
-#define MPI_File_iread_all PMPI_File_iread_all
-#define MPI_File_iread_all_f08 PMPI_File_iread_all_f08
-#define MPI_File_iread_at_all PMPI_File_iread_at_all
-#define MPI_File_iread_at_all_f08 PMPI_File_iread_at_all_f08
-#define MPI_File_iread_shared PMPI_File_iread_shared
-#define MPI_File_iread_shared_f08 PMPI_File_iread_shared_f08
-#define MPI_File_iwrite PMPI_File_iwrite
-#define MPI_File_iwrite_f08 PMPI_File_iwrite_f08
-#define MPI_File_iwrite_at PMPI_File_iwrite_at
-#define MPI_File_iwrite_at_f08 PMPI_File_iwrite_at_f08
-#define MPI_File_iwrite_all PMPI_File_iwrite_all
-#define MPI_File_iwrite_all_f08 PMPI_File_iwrite_all_f08
-#define MPI_File_iwrite_at_all PMPI_File_iwrite_at_all
-#define MPI_File_iwrite_at_all_f08 PMPI_File_iwrite_at_all_f08
-#define MPI_File_iwrite_shared PMPI_File_iwrite_shared
-#define MPI_File_iwrite_shared_f08 PMPI_File_iwrite_shared_f08
-#define MPI_File_open PMPI_File_open
-#define MPI_File_open_f08 PMPI_File_open_f08
-#define MPI_File_preallocate PMPI_File_preallocate
-#define MPI_File_preallocate_f08 PMPI_File_preallocate_f08
-#define MPI_File_read PMPI_File_read
-#define MPI_File_read_f08 PMPI_File_read_f08
-#define MPI_File_read_all PMPI_File_read_all
-#define MPI_File_read_all_f08 PMPI_File_read_all_f08
-#define MPI_File_read_all_begin PMPI_File_read_all_begin
-#define MPI_File_read_all_begin_f08 PMPI_File_read_all_begin_f08
-#define MPI_File_read_all_end PMPI_File_read_all_end
-#define MPI_File_read_all_end_f08 PMPI_File_read_all_end_f08
-#define MPI_File_read_at PMPI_File_read_at
-#define MPI_File_read_at_f08 PMPI_File_read_at_f08
-#define MPI_File_read_at_all PMPI_File_read_at_all
-#define MPI_File_read_at_all_f08 PMPI_File_read_at_all_f08
-#define MPI_File_read_at_all_begin PMPI_File_read_at_all_begin
-#define MPI_File_read_at_all_begin_f08 PMPI_File_read_at_all_begin_f08
-#define MPI_File_read_at_all_end PMPI_File_read_at_all_end
-#define MPI_File_read_at_all_end_f08 PMPI_File_read_at_all_end_f08
-#define MPI_File_read_ordered PMPI_File_read_ordered
-#define MPI_File_read_ordered_f08 PMPI_File_read_ordered_f08
-#define MPI_File_read_ordered_begin PMPI_File_read_ordered_begin
-#define MPI_File_read_ordered_begin_f08 PMPI_File_read_ordered_begin_f08
-#define MPI_File_read_ordered_end PMPI_File_read_ordered_end
-#define MPI_File_read_ordered_end_f08 PMPI_File_read_ordered_end_f08
-#define MPI_File_read_shared PMPI_File_read_shared
-#define MPI_File_read_shared_f08 PMPI_File_read_shared_f08
-#define MPI_File_seek PMPI_File_seek
-#define MPI_File_seek_f08 PMPI_File_seek_f08
-#define MPI_File_seek_shared PMPI_File_seek_shared
-#define MPI_File_seek_shared_f08 PMPI_File_seek_shared_f08
-#define MPI_File_set_atomicity PMPI_File_set_atomicity
-#define MPI_File_set_atomicity_f08 PMPI_File_set_atomicity_f08
-#define MPI_File_set_info PMPI_File_set_info
-#define MPI_File_set_info_f08 PMPI_File_set_info_f08
-#define MPI_File_set_size PMPI_File_set_size
-#define MPI_File_set_size_f08 PMPI_File_set_size_f08
-#define MPI_File_set_view PMPI_File_set_view
-#define MPI_File_set_view_f08 PMPI_File_set_view_f08
-#define MPI_File_sync PMPI_File_sync
-#define MPI_File_sync_f08 PMPI_File_sync_f08
-#define MPI_File_write PMPI_File_write
-#define MPI_File_write_f08 PMPI_File_write_f08
-#define MPI_File_write_all PMPI_File_write_all
-#define MPI_File_write_all_f08 PMPI_File_write_all_f08
-#define MPI_File_write_all_begin PMPI_File_write_all_begin
-#define MPI_File_write_all_begin_f08 PMPI_File_write_all_begin_f08
-#define MPI_File_write_all_end PMPI_File_write_all_end
-#define MPI_File_write_all_end_f08 PMPI_File_write_all_end_f08
-#define MPI_File_write_at PMPI_File_write_at
-#define MPI_File_write_at_f08 PMPI_File_write_at_f08
-#define MPI_File_write_at_all PMPI_File_write_at_all
-#define MPI_File_write_at_all_f08 PMPI_File_write_at_all_f08
-#define MPI_File_write_at_all_begin PMPI_File_write_at_all_begin
-#define MPI_File_write_at_all_begin_f08 PMPI_File_write_at_all_begin_f08
-#define MPI_File_write_at_all_end PMPI_File_write_at_all_end
-#define MPI_File_write_at_all_end_f08 PMPI_File_write_at_all_end_f08
-#define MPI_File_write_ordered PMPI_File_write_ordered
-#define MPI_File_write_ordered_f08 PMPI_File_write_ordered_f08
-#define MPI_File_write_ordered_begin PMPI_File_write_ordered_begin
-#define MPI_File_write_ordered_begin_f08 PMPI_File_write_ordered_begin_f08
-#define MPI_File_write_ordered_end PMPI_File_write_ordered_end
-#define MPI_File_write_ordered_end_f08 PMPI_File_write_ordered_end_f08
-#define MPI_File_write_shared PMPI_File_write_shared
-#define MPI_File_write_shared_f08 PMPI_File_write_shared_f08
-#define MPI_Register_datarep PMPI_Register_datarep
-#define MPI_Register_datarep_f08 PMPI_Register_datarep_f08
-#define MPI_Type_create_f90_complex PMPI_Type_create_f90_complex
-#define MPI_Type_create_f90_complex_f08 PMPI_Type_create_f90_complex_f08
-#define MPI_Type_create_f90_integer PMPI_Type_create_f90_integer
-#define MPI_Type_create_f90_integer_f08 PMPI_Type_create_f90_integer_f08
-#define MPI_Type_create_f90_real PMPI_Type_create_f90_real
-#define MPI_Type_create_f90_real_f08 PMPI_Type_create_f90_real_f08
-#define MPI_Type_match_size PMPI_Type_match_size
-#define MPI_Type_match_size_f08 PMPI_Type_match_size_f08
-#define MPI_Pcontrol PMPI_Pcontrol
-#define MPI_Pcontrol_f08 PMPI_Pcontrol_f08
-#define MPI_Comm_split_type PMPI_Comm_split_type
-#define MPI_Comm_split_type_f08 PMPI_Comm_split_type_f08
-#define MPI_F_sync_reg PMPI_F_sync_reg
-#define MPI_F_sync_reg_f08 PMPI_F_sync_reg_f08
-#define MPI_Get_library_version PMPI_Get_library_version
-#define MPI_Get_library_version_f08 PMPI_Get_library_version_f08
-#define MPI_Mprobe PMPI_Mprobe
-#define MPI_Mprobe_f08 PMPI_Mprobe_f08
-#define MPI_Improbe PMPI_Improbe
-#define MPI_Improbe_f08 PMPI_Improbe_f08
-#define MPI_Imrecv PMPI_Imrecv
-#define MPI_Imrecv_f08 PMPI_Imrecv_f08
-#define MPI_Mrecv PMPI_Mrecv
-#define MPI_Mrecv_f08 PMPI_Mrecv_f08
-#define MPI_Neighbor_allgather PMPI_Neighbor_allgather
-#define MPI_Neighbor_allgather_f08 PMPI_Neighbor_allgather_f08
-#define MPI_Ineighbor_allgather PMPI_Ineighbor_allgather
-#define MPI_Ineighbor_allgather_f08 PMPI_Ineighbor_allgather_f08
-#define MPI_Neighbor_allgatherv PMPI_Neighbor_allgatherv
-#define MPI_Neighbor_allgatherv_f08 PMPI_Neighbor_allgatherv_f08
-#define MPI_Ineighbor_allgatherv PMPI_Ineighbor_allgatherv
-#define MPI_Ineighbor_allgatherv_f08 PMPI_Ineighbor_allgatherv_f08
-#define MPI_Neighbor_alltoall PMPI_Neighbor_alltoall
-#define MPI_Neighbor_alltoall_f08 PMPI_Neighbor_alltoall_f08
-#define MPI_Ineighbor_alltoall PMPI_Ineighbor_alltoall
-#define MPI_Ineighbor_alltoall_f08 PMPI_Ineighbor_alltoall_f08
-#define MPI_Neighbor_alltoallv PMPI_Neighbor_alltoallv
-#define MPI_Neighbor_alltoallv_f08 PMPI_Neighbor_alltoallv_f08
-#define MPI_Ineighbor_alltoallv PMPI_Ineighbor_alltoallv
-#define MPI_Ineighbor_alltoallv_f08 PMPI_Ineighbor_alltoallv_f08
-#define MPI_Neighbor_alltoallw PMPI_Neighbor_alltoallw
-#define MPI_Neighbor_alltoallw_f08 PMPI_Neighbor_alltoallw_f08
-#define MPI_Ineighbor_alltoallw PMPI_Ineighbor_alltoallw
-#define MPI_Ineighbor_alltoallw_f08 PMPI_Ineighbor_alltoallw_f08
+#define MPI_Win_wait PMPI_Win_wait
 
 #endif
diff --git a/ompi/mpi/fortran/use-mpi-f08/mod/mpi-f08-types.F90 b/ompi/mpi/fortran/use-mpi-f08/mod/mpi-f08-types.F90
index c1abace16b3..db0cb3e323c 100644
--- a/ompi/mpi/fortran/use-mpi-f08/mod/mpi-f08-types.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/mod/mpi-f08-types.F90
@@ -1,6 +1,6 @@
 ! -*- f90 -*-
 !
-! Copyright (c) 2009-2015 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2009-2022 Cisco Systems, Inc.  All rights reserved
 ! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
 !                         All rights reserved.
 ! Copyright (c) 2015-2020 Research Organization for Information Science
@@ -8,6 +8,7 @@
 ! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
 ! Copyright (c) 2020      The University of Tennessee and The University
 !                         of Tennessee Research Foundation.  All rights
+! Copyright (c) 2019-2021 Triad National Security, LLC. All rights
 !                         reserved.
 ! $COPYRIGHT$
 !
@@ -27,50 +28,6 @@ module mpi_f08_types
    include "mpif-constants.h"
    include "mpif-io-constants.h"
 
-   !
-   ! derived types
-   !
-
-   type, BIND(C) :: MPI_Comm
-      integer :: MPI_VAL
-   end type MPI_Comm
-
-   type, BIND(C) :: MPI_Datatype
-      integer :: MPI_VAL
-   end type MPI_Datatype
-
-   type, BIND(C) :: MPI_Errhandler
-      integer :: MPI_VAL
-   end type MPI_Errhandler
-
-   type, BIND(C) :: MPI_File
-      integer :: MPI_VAL
-   end type MPI_File
-
-   type, BIND(C) :: MPI_Group
-      integer :: MPI_VAL
-   end type MPI_Group
-
-   type, BIND(C) :: MPI_Info
-      integer :: MPI_VAL
-   end type MPI_Info
-
-   type, BIND(C) :: MPI_Message
-      integer :: MPI_VAL
-   end type MPI_Message
-
-   type, BIND(C) :: MPI_Op
-      integer :: MPI_VAL
-   end type MPI_Op
-
-   type, BIND(C) :: MPI_Request
-      integer :: MPI_VAL
-   end type MPI_Request
-
-   type, BIND(C) :: MPI_Win
-      integer :: MPI_VAL
-   end type MPI_Win
-
   !
   ! Pre-defined handles
   !
@@ -117,6 +74,7 @@ module mpi_f08_types
   type(MPI_Request), parameter   :: MPI_REQUEST_NULL             = MPI_Request(OMPI_MPI_REQUEST_NULL)
   type(MPI_Win), parameter       :: MPI_WIN_NULL                 = MPI_Win(OMPI_MPI_WIN_NULL)
   type(MPI_File), parameter      :: MPI_FILE_NULL                = MPI_File(OMPI_MPI_FILE_NULL)
+  type(MPI_Session), parameter   :: MPI_SESSION_NULL             = MPI_Session(OMPI_MPI_SESSION_NULL)
 
   !
   ! Pre-defined datatype bindings
@@ -206,138 +164,4 @@ module mpi_f08_types
 !------------------------------
 #include "mpif-f08-types.h"
 
-!... Interfaces for operators with handles
-!-----------------------------------------
-interface operator (.EQ.)
-  module procedure ompi_comm_op_eq
-  module procedure ompi_datatype_op_eq
-  module procedure ompi_errhandler_op_eq
-  module procedure ompi_file_op_eq
-  module procedure ompi_group_op_eq
-  module procedure ompi_info_op_eq
-  module procedure ompi_message_op_eq
-  module procedure ompi_op_op_eq
-  module procedure ompi_request_op_eq
-  module procedure ompi_win_op_eq
-end interface
-
-interface operator (.NE.)
-  module procedure ompi_comm_op_ne
-  module procedure ompi_datatype_op_ne
-  module procedure ompi_errhandler_op_ne
-  module procedure ompi_file_op_ne
-  module procedure ompi_group_op_ne
-  module procedure ompi_info_op_ne
-  module procedure ompi_message_op_ne
-  module procedure ompi_op_op_ne
-  module procedure ompi_request_op_ne
-  module procedure ompi_win_op_ne
-end interface
-
-contains
-
-!... .EQ. operator
-!-----------------
-  logical function ompi_comm_op_eq(a, b)
-    type(MPI_Comm), intent(in) :: a, b
-    ompi_comm_op_eq = (a%MPI_VAL .EQ. b%MPI_VAL)
-  end function ompi_comm_op_eq
-
-  logical function ompi_datatype_op_eq(a, b)
-    type(MPI_Datatype), intent(in) :: a, b
-    ompi_datatype_op_eq = (a%MPI_VAL .EQ. b%MPI_VAL)
-  end function ompi_datatype_op_eq
-
-  logical function ompi_errhandler_op_eq(a, b)
-    type(MPI_Errhandler), intent(in) :: a, b
-    ompi_errhandler_op_eq = (a%MPI_VAL .EQ. b%MPI_VAL)
-  end function ompi_errhandler_op_eq
-
-  logical function ompi_file_op_eq(a, b)
-    type(MPI_File), intent(in) :: a, b
-    ompi_file_op_eq = (a%MPI_VAL .EQ. b%MPI_VAL)
-  end function ompi_file_op_eq
-
-  logical function ompi_group_op_eq(a, b)
-    type(MPI_Group), intent(in) :: a, b
-    ompi_group_op_eq = (a%MPI_VAL .EQ. b%MPI_VAL)
-  end function ompi_group_op_eq
-
-  logical function ompi_info_op_eq(a, b)
-    type(MPI_Info), intent(in) :: a, b
-    ompi_info_op_eq = (a%MPI_VAL .EQ. b%MPI_VAL)
-  end function ompi_info_op_eq
-
-  logical function ompi_message_op_eq(a, b)
-    type(MPI_Message), intent(in) :: a, b
-    ompi_message_op_eq = (a%MPI_VAL .EQ. b%MPI_VAL)
-  end function ompi_message_op_eq
-
-  logical function ompi_op_op_eq(a, b)
-    type(MPI_Op), intent(in) :: a, b
-    ompi_op_op_eq = (a%MPI_VAL .EQ. b%MPI_VAL)
-  end function ompi_op_op_eq
-
-  logical function ompi_request_op_eq(a, b)
-    type(MPI_Request), intent(in) :: a, b
-    ompi_request_op_eq = (a%MPI_VAL .EQ. b%MPI_VAL)
-  end function ompi_request_op_eq
-
-  logical function ompi_win_op_eq(a, b)
-    type(MPI_Win), intent(in) :: a, b
-    ompi_win_op_eq = (a%MPI_VAL .EQ. b%MPI_VAL)
-  end function ompi_win_op_eq
-
-!... .NE. operator
-!-----------------
-  logical function ompi_comm_op_ne(a, b)
-    type(MPI_Comm), intent(in) :: a, b
-    ompi_comm_op_ne = (a%MPI_VAL .NE. b%MPI_VAL)
-  end function ompi_comm_op_ne
-
-  logical function ompi_datatype_op_ne(a, b)
-    type(MPI_Datatype), intent(in) :: a, b
-    ompi_datatype_op_ne = (a%MPI_VAL .NE. b%MPI_VAL)
-  end function ompi_datatype_op_ne
-
-  logical function ompi_errhandler_op_ne(a, b)
-    type(MPI_Errhandler), intent(in) :: a, b
-    ompi_errhandler_op_ne = (a%MPI_VAL .NE. b%MPI_VAL)
-  end function ompi_errhandler_op_ne
-
-  logical function ompi_file_op_ne(a, b)
-    type(MPI_File), intent(in) :: a, b
-    ompi_file_op_ne = (a%MPI_VAL .NE. b%MPI_VAL)
-  end function ompi_file_op_ne
-
-  logical function ompi_group_op_ne(a, b)
-    type(MPI_Group), intent(in) :: a, b
-    ompi_group_op_ne = (a%MPI_VAL .NE. b%MPI_VAL)
-  end function ompi_group_op_ne
-
-  logical function ompi_info_op_ne(a, b)
-    type(MPI_Info), intent(in) :: a, b
-    ompi_info_op_ne = (a%MPI_VAL .NE. b%MPI_VAL)
-  end function ompi_info_op_ne
-
-  logical function ompi_message_op_ne(a, b)
-    type(MPI_Message), intent(in) :: a, b
-    ompi_message_op_ne = (a%MPI_VAL .NE. b%MPI_VAL)
-  end function ompi_message_op_ne
-
-  logical function ompi_op_op_ne(a, b)
-    type(MPI_Op), intent(in) :: a, b
-    ompi_op_op_ne = (a%MPI_VAL .NE. b%MPI_VAL)
-  end function ompi_op_op_ne
-
-  logical function ompi_request_op_ne(a, b)
-    type(MPI_Request), intent(in) :: a, b
-    ompi_request_op_ne = (a%MPI_VAL .NE. b%MPI_VAL)
-  end function ompi_request_op_ne
-
-  logical function ompi_win_op_ne(a, b)
-    type(MPI_Win), intent(in) :: a, b
-    ompi_win_op_ne = (a%MPI_VAL .NE. b%MPI_VAL)
-  end function ompi_win_op_ne
-
 end module mpi_f08_types
diff --git a/ompi/mpi/fortran/use-mpi-f08/mod/pmpi-f08-interfaces.F90 b/ompi/mpi/fortran/use-mpi-f08/mod/pmpi-f08-interfaces.F90
index ddd81d17f74..72a45e57d61 100644
--- a/ompi/mpi/fortran/use-mpi-f08/mod/pmpi-f08-interfaces.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/mod/pmpi-f08-interfaces.F90
@@ -10,6 +10,8 @@
 ! Copyright (c) 2015-2020 Research Organization for Information Science
 !                         and Technology (RIST).  All rights reserved.
 ! Copyright (c) 2017-2018 FUJITSU LIMITED.  All rights reserved.
+! Copyright (c) 2019-2025 Triad National Security, LLC. All rights
+!                         reserved.
 ! $COPYRIGHT$
 !
 ! This file provides the interface specifications for the MPI Fortran
@@ -26,6 +28,7 @@
 module pmpi_f08_interfaces
 
 #include "mpi-f08-interfaces.h"
+#include "mpi-f08-interfaces-generated.h"
 
 ! MPI_Wtick is not a wrapper function
 !
diff --git a/ompi/mpi/fortran/use-mpi-f08/mprobe_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/mprobe_f08.F90
index adb54ba98eb..bcde2f5ea49 100644
--- a/ompi/mpi/fortran/use-mpi-f08/mprobe_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/mprobe_f08.F90
@@ -16,7 +16,7 @@ subroutine MPI_Mprobe_f08(source,tag,comm,message,status,ierror)
    INTEGER, INTENT(IN) :: source, tag
    TYPE(MPI_Comm), INTENT(IN) :: comm
    TYPE(MPI_Message), INTENT(OUT) :: message
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   TYPE(MPI_Status) :: status
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
    integer :: c_ierror
 
diff --git a/ompi/mpi/fortran/use-mpi-f08/mrecv_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/mrecv_f08.F90
deleted file mode 100644
index 3eb9651cb31..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/mrecv_f08.F90
+++ /dev/null
@@ -1,29 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Mrecv_f08(buf,count,datatype,message,status,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Message, MPI_Status
-   use :: ompi_mpifh_bindings, only : ompi_mrecv_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: buf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Message), INTENT(INOUT) :: message
-   TYPE(MPI_Status) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_mrecv_f(buf,count,datatype%MPI_VAL,message%MPI_VAL,status,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Mrecv_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/mrecv_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/mrecv_ts.c.in
new file mode 100644
index 00000000000..eefce291914
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/mrecv_ts.c.in
@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID mrecv(BUFFER x, COUNT count, DATATYPE datatype,
+                     MESSAGE_INOUT message, STATUS status)
+{
+    int c_ierr;
+    OMPI_FORTRAN_STATUS_DECLARATION(c_status,c_status2)
+    MPI_Message c_message = PMPI_Message_f2c(*message);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+
+    OMPI_FORTRAN_STATUS_SET_POINTER(c_status,c_status2,status)
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(MPI_COMM_SELF, c_ierr, FUNC_NAME);
+        return;
+    }
+
+
+    /* Call the C function */
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(buf), c_count,
+                          c_datatype, &c_message,
+                          c_status);
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+       OMPI_FORTRAN_STATUS_RETURN(c_status,c_status2,status,c_ierr)
+       /* message is an INOUT, and may be updated by the recv */
+       *message = PMPI_Message_c2f(c_message);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/neighbor_allgather_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/neighbor_allgather_f08.F90
deleted file mode 100644
index 084ab11076b..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/neighbor_allgather_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2013 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Neighbor_allgather_f08(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   use :: ompi_mpifh_bindings, only : ompi_neighbor_allgather_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_neighbor_allgather_f(sendbuf,sendcount,sendtype%MPI_VAL,&
-                                  recvbuf,recvcount,recvtype%MPI_VAL,comm%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Neighbor_allgather_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/neighbor_allgather_init_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/neighbor_allgather_init_ts.c.in
new file mode 100644
index 00000000000..652eac1b4c8
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/neighbor_allgather_init_ts.c.in
@@ -0,0 +1,82 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID neighbor_allgather_init(BUFFER x1, COUNT sendcount, DATATYPE sendtype,
+                                  BUFFER_OUT x2, COUNT recvcount, DATATYPE recvtype,
+                                  COMM comm, INFO info, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype c_senddatatype, c_sendtype = PMPI_Type_f2c(*sendtype);
+    MPI_Request c_request;
+    MPI_Info c_info;
+    void *sendbuf = OMPI_CFI_BASE_ADDR(x1);
+    @COUNT_TYPE@ c_sendcount = (@COUNT_TYPE@) *sendcount;
+    MPI_Datatype c_recvtype = PMPI_Type_f2c(*recvtype);
+    void *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_recvcount = (@COUNT_TYPE@) *recvcount;
+
+    c_info = PMPI_Info_f2c(*info);
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    OMPI_CFI_2_C(x1, c_sendcount, c_sendtype, c_senddatatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    c_sendtype = PMPI_Type_f2c(*sendtype);
+    c_recvtype = PMPI_Type_f2c(*recvtype);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          c_sendcount,
+                          c_sendtype,
+                          recvbuf,
+                          c_recvcount,
+                          c_recvtype, 
+			  c_comm,
+			  c_info,
+			  &c_request);
+
+    if (c_senddatatype != c_sendtype) {
+        ompi_datatype_destroy(&c_senddatatype);
+    }
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_request);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/neighbor_allgather_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/neighbor_allgather_ts.c.in
new file mode 100644
index 00000000000..2936ec736ca
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/neighbor_allgather_ts.c.in
@@ -0,0 +1,70 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID neighbor_allgather(BUFFER x1, COUNT sendcount, DATATYPE sendtype,
+                                  BUFFER_OUT x2, COUNT recvcount, DATATYPE recvtype,
+                                  COMM comm)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype __opal_attribute_unused__ c_senddatatype;
+    MPI_Datatype c_sendtype = PMPI_Type_f2c(*sendtype);
+    void *sendbuf = OMPI_CFI_BASE_ADDR(x1);
+    @COUNT_TYPE@ c_sendcount = (@COUNT_TYPE@) *sendcount;
+    MPI_Datatype c_recvtype = PMPI_Type_f2c(*recvtype);
+    void *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_recvcount = (@COUNT_TYPE@) *recvcount;
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    OMPI_CFI_2_C(x1, c_sendcount, c_sendtype, c_senddatatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          c_sendcount,
+                          c_senddatatype,
+                          recvbuf,
+                          c_recvcount,
+                          c_recvtype, c_comm);
+
+    if (c_senddatatype != c_sendtype) {
+        ompi_datatype_destroy(&c_senddatatype);
+    }
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/neighbor_allgatherv_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/neighbor_allgatherv_f08.F90
deleted file mode 100644
index 7335c3f8b39..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/neighbor_allgatherv_f08.F90
+++ /dev/null
@@ -1,34 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2013 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Neighbor_allgatherv_f08(sendbuf,sendcount,sendtype,recvbuf,recvcounts,&
-                                       displs,recvtype,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   use :: ompi_mpifh_bindings, only : ompi_neighbor_allgatherv_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount
-   INTEGER, INTENT(IN) :: recvcounts(*), displs(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_neighbor_allgatherv_f(sendbuf,sendcount,sendtype%MPI_VAL,recvbuf,recvcounts,&
-                                   displs,recvtype%MPI_VAL,comm%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Neighbor_allgatherv_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/neighbor_allgatherv_init_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/neighbor_allgatherv_init_ts.c.in
new file mode 100644
index 00000000000..394debf1c27
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/neighbor_allgatherv_init_ts.c.in
@@ -0,0 +1,87 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID neighbor_allgatherv_init(BUFFER x1, COUNT sendcount, DATATYPE sendtype,
+                                        BUFFER_OUT x2, COUNT_ARRAY recvcounts, DISP_ARRAY displs,
+                                        DATATYPE recvtype, COMM comm,
+					INFO info, REQUEST_OUT request)
+{
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype c_senddatatype, c_sendtype = PMPI_Type_f2c(*sendtype);
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_sendcount = (@COUNT_TYPE@) *sendcount;
+    MPI_Datatype c_recvtype = PMPI_Type_f2c(*recvtype);
+    MPI_Request c_request;
+    MPI_Info c_info;
+    int size, c_ierr, idx=0;
+    @COUNT_TYPE@ *tmp_recvcounts = NULL;
+    @DISP_TYPE@ *tmp_displs = NULL;
+
+    c_info = PMPI_Info_f2c(*info);
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    OMPI_CFI_2_C(x1, c_sendcount, c_sendtype, c_senddatatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    PMPI_Comm_size(c_comm, &size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(recvcounts, tmp_recvcounts, size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(displs, tmp_displs, size);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          c_sendcount,
+                          c_senddatatype,
+                          recvbuf,
+                          tmp_recvcounts,
+                          tmp_displs,
+                          c_recvtype, 
+			  c_comm,
+			  c_info,
+			  &c_request);
+    if (c_senddatatype != c_sendtype) {
+        ompi_datatype_destroy(&c_senddatatype);
+    }
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_request);
+    }
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(recvcounts, tmp_recvcounts, c_request, c_ierr, idx);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(displs, tmp_displs, c_request, c_ierr, idx);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/neighbor_allgatherv_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/neighbor_allgatherv_ts.c.in
new file mode 100644
index 00000000000..dc4e9f7eb5c
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/neighbor_allgatherv_ts.c.in
@@ -0,0 +1,78 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID neighbor_allgatherv(BUFFER x1, COUNT sendcount, DATATYPE sendtype,
+                                   BUFFER_OUT x2, COUNT_ARRAY recvcounts, DISP_ARRAY displs,
+                                   DATATYPE recvtype, COMM comm)
+{
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype  __opal_attribute_unused__ c_senddatatype;
+    MPI_Datatype c_sendtype = PMPI_Type_f2c(*sendtype);
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_sendcount = (@COUNT_TYPE@) *sendcount;
+    MPI_Datatype c_recvtype = PMPI_Type_f2c(*recvtype);
+    int size, c_ierr;
+    @COUNT_TYPE@ *tmp_recvcounts = NULL;
+    @DISP_TYPE@ *tmp_displs = NULL;
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    OMPI_CFI_2_C(x1, c_sendcount, c_sendtype, c_senddatatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    PMPI_Comm_size(c_comm, &size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(recvcounts, tmp_recvcounts, size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(displs, tmp_displs, size);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          c_sendcount,
+                          c_senddatatype,
+                          recvbuf,
+                          tmp_recvcounts,
+                          tmp_displs,
+                          c_recvtype, c_comm);
+
+    if (c_senddatatype != c_sendtype) {
+        ompi_datatype_destroy(&c_senddatatype);
+    }
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(recvcounts, tmp_recvcounts);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(displs, tmp_displs);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/neighbor_alltoall_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/neighbor_alltoall_f08.F90
deleted file mode 100644
index 8bfb42dd83b..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/neighbor_alltoall_f08.F90
+++ /dev/null
@@ -1,33 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2013 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Neighbor_alltoall_f08(sendbuf,sendcount,sendtype,recvbuf,&
-                                     recvcount,recvtype,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   use :: ompi_mpifh_bindings, only : ompi_neighbor_alltoall_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_neighbor_alltoall_f(sendbuf,sendcount,sendtype%MPI_VAL,recvbuf,&
-                                 recvcount,recvtype%MPI_VAL,comm%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Neighbor_alltoall_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/neighbor_alltoall_init_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/neighbor_alltoall_init_ts.c.in
new file mode 100644
index 00000000000..f2f6246584f
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/neighbor_alltoall_init_ts.c.in
@@ -0,0 +1,73 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID neighbor_alltoall_init(BUFFER x1, COUNT sendcount, DATATYPE sendtype,
+                                      BUFFER_OUT x2, COUNT recvcount, DATATYPE recvtype,
+                                      COMM comm, INFO info, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype c_sendtype = PMPI_Type_f2c(*sendtype);
+    MPI_Datatype c_recvtype = PMPI_Type_f2c(*recvtype);
+    MPI_Request c_request;
+    MPI_Info c_info;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_sendcount = (@COUNT_TYPE@)*sendcount;
+    @COUNT_TYPE@ c_recvcount = (@COUNT_TYPE@)*recvcount;
+
+    c_info = PMPI_Info_f2c(*info);
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          c_sendcount,
+                          c_sendtype,
+                          recvbuf,
+                          c_recvcount,
+                          c_recvtype, 
+			  c_comm,
+			  c_info,
+			  &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_request);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/neighbor_alltoall_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/neighbor_alltoall_ts.c.in
new file mode 100644
index 00000000000..5ffde86961f
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/neighbor_alltoall_ts.c.in
@@ -0,0 +1,63 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID neighbor_alltoall(BUFFER x1, COUNT sendcount, DATATYPE sendtype,
+                                 BUFFER_OUT x2, COUNT recvcount, DATATYPE recvtype,
+                                 COMM comm)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype c_sendtype = PMPI_Type_f2c(*sendtype);
+    MPI_Datatype c_recvtype = PMPI_Type_f2c(*recvtype);
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_sendcount = (@COUNT_TYPE@)*sendcount;
+    @COUNT_TYPE@ c_recvcount = (@COUNT_TYPE@)*recvcount;
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          c_sendcount,
+                          c_sendtype,
+                          recvbuf,
+                          c_recvcount,
+                          c_recvtype, c_comm);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/neighbor_alltoallv_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/neighbor_alltoallv_f08.F90
deleted file mode 100644
index 377ac5b6f1f..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/neighbor_alltoallv_f08.F90
+++ /dev/null
@@ -1,34 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2013 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Neighbor_alltoallv_f08(sendbuf,sendcounts,sdispls,sendtype,recvbuf,&
-                                      recvcounts,rdispls,recvtype,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   use :: ompi_mpifh_bindings, only : ompi_neighbor_alltoallv_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_neighbor_alltoallv_f(sendbuf,sendcounts,sdispls,sendtype%MPI_VAL,&
-                                  recvbuf,recvcounts,rdispls,recvtype%MPI_VAL,&
-                                  comm%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Neighbor_alltoallv_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/neighbor_alltoallv_init_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/neighbor_alltoallv_init_ts.c.in
new file mode 100644
index 00000000000..0a5729ccb03
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/neighbor_alltoallv_init_ts.c.in
@@ -0,0 +1,92 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID neighbor_alltoallv_init(BUFFER x1, COUNT_ARRAY sendcounts, DISP_ARRAY sdispls,
+                                       DATATYPE sendtype, BUFFER_OUT x2, COUNT_ARRAY recvcounts,
+                                       DISP_ARRAY rdispls, DATATYPE recvtype,
+                                       COMM comm,
+				       INFO info,
+				       REQUEST_OUT request)
+{
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype c_sendtype = PMPI_Type_f2c(*sendtype);
+    MPI_Datatype c_recvtype = PMPI_Type_f2c(*recvtype);;
+    MPI_Info c_info;
+    MPI_Request c_request;
+    int size, c_ierr, idx = 0;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ *tmp_sendcounts = NULL;
+    @DISP_TYPE@ *tmp_sdispls = NULL;
+    @COUNT_TYPE@ *tmp_recvcounts = NULL;
+    @DISP_TYPE@ *tmp_rdispls = NULL;
+
+    c_info = PMPI_Info_f2c(*info);
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    PMPI_Comm_size(c_comm, &size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(sendcounts, tmp_sendcounts, size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(sdispls, tmp_sdispls, size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(recvcounts, tmp_recvcounts, size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(rdispls, tmp_rdispls, size);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          tmp_sendcounts,
+                          tmp_sdispls,
+                          c_sendtype,
+                          recvbuf,
+                          tmp_recvcounts,
+                          tmp_rdispls,
+                          c_recvtype,
+			  c_comm,
+			  c_info,
+			  &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_request);
+    }
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(recvcounts, tmp_recvcounts, c_request, c_ierr, idx);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(sendcounts, tmp_sendcounts, c_request, c_ierr, idx);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(rdispls, tmp_rdispls, c_request, c_ierr, idx);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(sdispls, tmp_sdispls, c_request, c_ierr, idx);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/neighbor_alltoallv_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/neighbor_alltoallv_ts.c.in
new file mode 100644
index 00000000000..8610e0221d3
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/neighbor_alltoallv_ts.c.in
@@ -0,0 +1,79 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID neighbor_alltoallv(BUFFER x1, COUNT_ARRAY sendcounts, DISP_ARRAY sdispls,
+                                  DATATYPE sendtype, BUFFER_OUT x2, COUNT_ARRAY recvcounts,
+                                  DISP_ARRAY rdispls, DATATYPE recvtype,
+                                  COMM comm)
+{
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype c_sendtype = PMPI_Type_f2c(*sendtype);
+    MPI_Datatype c_recvtype = PMPI_Type_f2c(*recvtype);
+    int size, c_ierr;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ *tmp_sendcounts = NULL;
+    @DISP_TYPE@ *tmp_sdispls = NULL;
+    @COUNT_TYPE@ *tmp_recvcounts = NULL;
+    @DISP_TYPE@ *tmp_rdispls = NULL;
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    PMPI_Comm_size(c_comm, &size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(sendcounts, tmp_sendcounts, size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(sdispls, tmp_sdispls, size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(recvcounts, tmp_recvcounts, size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(rdispls, tmp_rdispls, size);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          tmp_sendcounts,
+                          tmp_sdispls,
+                          c_sendtype,
+                          recvbuf,
+                          tmp_recvcounts,
+                          tmp_rdispls,
+                          c_recvtype, c_comm);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(sendcounts, tmp_sendcounts);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(sdispls, tmp_sdispls);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(recvcounts, tmp_recvcounts);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(rdispls, tmp_rdispls);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/neighbor_alltoallw_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/neighbor_alltoallw_f08.F90
deleted file mode 100644
index e0fd6a68b8e..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/neighbor_alltoallw_f08.F90
+++ /dev/null
@@ -1,34 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2013 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2013 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Neighbor_alltoallw_f08(sendbuf,sendcounts,sdispls,sendtypes,&
-                                      recvbuf,recvcounts,rdispls,recvtypes,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_ADDRESS_KIND
-   use :: ompi_mpifh_bindings, only : ompi_neighbor_alltoallw_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: sendcounts(*), recvcounts(*)
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: sdispls(*), rdispls(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtypes(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtypes(*)
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_neighbor_alltoallw_f(sendbuf,sendcounts,sdispls,sendtypes(1)%MPI_VAL,&
-                                  recvbuf,recvcounts,rdispls,recvtypes(1)%MPI_VAL,&
-                                  comm%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Neighbor_alltoallw_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/neighbor_alltoallw_init_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/neighbor_alltoallw_init_ts.c.in
new file mode 100644
index 00000000000..8a514759b41
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/neighbor_alltoallw_init_ts.c.in
@@ -0,0 +1,102 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID neighbor_alltoallw_init(BUFFER x1, COUNT_ARRAY sendcounts,
+                                      AINT_ARRAY sdispls, DATATYPE_ARRAY sendtypes,
+                                      BUFFER_OUT x2, COUNT_ARRAY recvcounts,
+                                      AINT_ARRAY rdispls, DATATYPE_ARRAY recvtypes,
+                                      COMM comm,
+				      INFO info,
+				      REQUEST_OUT request)
+{
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype *c_sendtypes, *c_recvtypes;
+    MPI_Request c_request;
+    MPI_Info c_info;
+    int size, c_ierr, idx = 0;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ *tmp_sendcounts = NULL;
+    @COUNT_TYPE@ *tmp_recvcounts = NULL;
+    MPI_Aint *tmp_sdispls = NULL, *tmp_rdispls;
+
+    c_info = PMPI_Info_f2c(*info);
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    PMPI_Comm_size(c_comm, &size);
+
+    c_sendtypes = (MPI_Datatype *) malloc(size * sizeof(MPI_Datatype));
+    c_recvtypes = (MPI_Datatype *) malloc(size * sizeof(MPI_Datatype));
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(sendcounts, tmp_sendcounts, size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(recvcounts, tmp_recvcounts, size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(sdispls, tmp_sdispls, size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(rdispls, tmp_rdispls, size);
+
+    /* Alltoallw does not support MPI_IN_PLACE */
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    while (size > 0) {
+        c_sendtypes[size - 1] = PMPI_Type_f2c(sendtypes[size - 1]);
+        c_recvtypes[size - 1] = PMPI_Type_f2c(recvtypes[size - 1]);
+        --size;
+    }
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          tmp_sendcounts,
+                          tmp_sdispls,
+                          c_sendtypes,
+                          recvbuf,
+                          tmp_recvcounts,
+                          tmp_rdispls,
+                          c_recvtypes, 
+			  c_comm,
+			  c_info,
+			  &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_request);
+    }
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(recvcounts, tmp_recvcounts, c_request, c_ierr, idx);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(sendcounts, tmp_sendcounts, c_request, c_ierr, idx);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(rdispls, tmp_rdispls, c_request, c_ierr, idx);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(sdispls, tmp_sdispls, c_request, c_ierr, idx);
+
+    free(c_sendtypes);
+    free(c_recvtypes);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/neighbor_alltoallw_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/neighbor_alltoallw_ts.c.in
new file mode 100644
index 00000000000..e4dc0181dc7
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/neighbor_alltoallw_ts.c.in
@@ -0,0 +1,84 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID neighbor_alltoallw(BUFFER x1, COUNT_ARRAY sendcounts,
+                                  AINT_ARRAY sdispls, DATATYPE_ARRAY sendtypes,
+                                  BUFFER_OUT x2, COUNT_ARRAY recvcounts,
+                                  AINT_ARRAY rdispls, DATATYPE_ARRAY recvtypes,
+                                  COMM comm)
+{
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype *c_sendtypes, *c_recvtypes;
+    int size, c_ierr;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ *tmp_sendcounts = NULL;
+    @COUNT_TYPE@ *tmp_recvcounts = NULL;
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    PMPI_Comm_size(c_comm, &size);
+
+    c_sendtypes = (MPI_Datatype *) malloc(size * sizeof(MPI_Datatype));
+    c_recvtypes = (MPI_Datatype *) malloc(size * sizeof(MPI_Datatype));
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(sendcounts, tmp_sendcounts, size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(recvcounts, tmp_recvcounts, size);
+
+    while (size > 0) {
+        c_sendtypes[size - 1] = PMPI_Type_f2c(sendtypes[size - 1]);
+        c_recvtypes[size - 1] = PMPI_Type_f2c(recvtypes[size - 1]);
+        --size;
+    }
+
+    /* Alltoallw does not support MPI_IN_PLACE */
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          tmp_sendcounts,
+                          sdispls,
+                          c_sendtypes,
+                          recvbuf,
+                          tmp_recvcounts,
+                          rdispls,
+                          c_recvtypes, c_comm);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(sendcounts, tmp_sendcounts);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(recvcounts, tmp_recvcounts);
+    free(c_sendtypes);
+    free(c_recvtypes);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/open_port_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/open_port_f08.F90
index 9ee276fe2f6..0e3af2c5ae3 100644
--- a/ompi/mpi/fortran/use-mpi-f08/open_port_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/open_port_f08.F90
@@ -12,13 +12,14 @@
 subroutine MPI_Open_port_f08(info,port_name,ierror)
    use :: mpi_f08_types, only : MPI_Info
    use :: ompi_mpifh_bindings, only : ompi_open_port_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
    implicit none
    TYPE(MPI_Info), INTENT(IN) :: info
    CHARACTER(LEN=*), INTENT(OUT) :: port_name
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
    integer :: c_ierror
 
-   call ompi_open_port_f(info%MPI_VAL,port_name,c_ierror,len(port_name))
+   call ompi_open_port_f(info%MPI_VAL,port_name,c_ierror,len(port_name,KIND=C_INT))
    if (present(ierror)) ierror = c_ierror
 
 end subroutine MPI_Open_port_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/pack_external_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/pack_external_f08.F90
deleted file mode 100644
index 9f4ae490051..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/pack_external_f08.F90
+++ /dev/null
@@ -1,34 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Pack_external_f08(datarep,inbuf,incount,datatype,outbuf,outsize, &
-                                 position,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_ADDRESS_KIND
-   use :: ompi_mpifh_bindings, only : ompi_pack_external_f
-   implicit none
-   CHARACTER(LEN=*), INTENT(IN) :: datarep
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: inbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: outbuf
-   INTEGER, INTENT(IN) :: incount
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: outsize
-   INTEGER(MPI_ADDRESS_KIND), INTENT(INOUT) :: position
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_pack_external_f(datarep,inbuf,incount,datatype%MPI_VAL,outbuf, &
-                             outsize,position,c_ierror,len(datarep))
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Pack_external_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/pack_external_size.c.in b/ompi/mpi/fortran/use-mpi-f08/pack_external_size.c.in
new file mode 100644
index 00000000000..d3fcabdf9a2
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/pack_external_size.c.in
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2017 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID pack_external_size(CHAR_ARRAY datarep, COUNT incount,
+                             DATATYPE datatype, AINT_COUNT_OUT size)
+{
+    int c_ierr;
+    MPI_Datatype type = PMPI_Type_f2c(*datatype);
+    @COUNT_TYPE@ c_incount = (@COUNT_TYPE@)*incount;
+
+    c_ierr = @INNER_CALL@(datarep,
+                          c_incount,
+                          type, size);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/pack_external_size_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/pack_external_size_f08.F90
deleted file mode 100644
index 477487a617d..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/pack_external_size_f08.F90
+++ /dev/null
@@ -1,26 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Pack_external_size_f08(datarep,incount,datatype,size,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_ADDRESS_KIND
-   use :: ompi_mpifh_bindings, only : ompi_pack_external_size_f
-   implicit none
-   CHARACTER(LEN=*), INTENT(IN) :: datarep
-   INTEGER, INTENT(IN) :: incount
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER(MPI_ADDRESS_KIND), INTENT(OUT) :: size
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_pack_external_size_f(datarep,incount,datatype%MPI_VAL,size,c_ierror,len(datarep))
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Pack_external_size_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/pack_external_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/pack_external_ts.c.in
new file mode 100644
index 00000000000..8168c2d16a9
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/pack_external_ts.c.in
@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID pack_external(CHAR_ARRAY datarep, BUFFER x1, COUNT incount,
+                             DATATYPE datatype, BUFFER x2,
+                             AINT_COUNT outsize, AINT_COUNT_INOUT position)
+{
+    int c_ierr;
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *inbuf = OMPI_CFI_BASE_ADDR(x1);
+    char *outbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_incount = (@COUNT_TYPE@)(*incount);
+
+    OMPI_CFI_2_C(x1, c_incount, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(MPI_COMM_SELF, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (c_datatype != c_type) {
+            ompi_datatype_destroy(&c_datatype);
+        }
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(MPI_COMM_SELF, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(datarep, OMPI_F2C_BOTTOM(inbuf),
+                               c_incount,
+                               c_datatype, outbuf,
+                               *outsize,
+                               position);
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/pack_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/pack_f08.F90
deleted file mode 100644
index 5b0da89c213..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/pack_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Pack_f08(inbuf,incount,datatype,outbuf,outsize,position,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   use :: ompi_mpifh_bindings, only : ompi_pack_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: inbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: outbuf
-   INTEGER, INTENT(IN) :: incount, outsize
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER, INTENT(INOUT) :: position
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_pack_f(inbuf,incount,datatype%MPI_VAL,outbuf,outsize, &
-                    position,comm%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Pack_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/pack_size.c.in b/ompi/mpi/fortran/use-mpi-f08/pack_size.c.in
new file mode 100644
index 00000000000..26926054f3b
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/pack_size.c.in
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID pack_size(COUNT incount, DATATYPE datatype,
+   		         COMM comm, COUNT_OUT size)
+{
+    int c_ierr;
+    MPI_Comm c_comm;
+    MPI_Datatype c_type;
+    @COUNT_TYPE@ c_size;
+    @COUNT_TYPE@ c_incount = (@COUNT_TYPE@)*incount;
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_type = PMPI_Type_f2c(*datatype);
+
+    c_ierr = @INNER_CALL@(c_incount,
+                          c_type, c_comm,
+                          &c_size);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *size = (@COUNT_FINT_TYPE@)(c_size);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/pack_size_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/pack_size_f08.F90
deleted file mode 100644
index 57c1f54dbbb..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/pack_size_f08.F90
+++ /dev/null
@@ -1,26 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Pack_size_f08(incount,datatype,comm,size,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   use :: ompi_mpifh_bindings, only : ompi_pack_size_f
-   implicit none
-   INTEGER, INTENT(IN) :: incount
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, INTENT(OUT) :: size
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_pack_size_f(incount,datatype%MPI_VAL,comm%MPI_VAL,size,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Pack_size_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/pack_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/pack_ts.c.in
new file mode 100644
index 00000000000..227426f0670
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/pack_ts.c.in
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID pack(BUFFER x1, COUNT incount, DATATYPE datatype,
+                    BUFFER x2, COUNT outsize, COUNT_INOUT position,
+                    COMM comm)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *inbuf = OMPI_CFI_BASE_ADDR(x1);
+    char *outbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_position = (@COUNT_TYPE@)(*position);
+    @COUNT_TYPE@ c_incount = (@COUNT_TYPE@)(*incount);
+    @COUNT_TYPE@ c_outsize = (@COUNT_TYPE@)(*outsize);
+
+    OMPI_CFI_2_C(x1, c_incount, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (c_datatype != c_type) {
+            ompi_datatype_destroy(&c_datatype);
+        }
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(inbuf),
+                       c_incount,
+                       c_datatype,
+                       outbuf,
+                       c_outsize,
+                       &c_position,
+                       c_comm);
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *position = (@COUNT_FINT_TYPE@)(c_position);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/parrived_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/parrived_f08.F90
new file mode 100644
index 00000000000..f79523993b2
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/parrived_f08.F90
@@ -0,0 +1,29 @@
+! -*- f90 -*-
+!
+! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
+!                         All rights reserved.
+! Copyright (c) 2018      Research Organization for Information Science
+!                         and Technology (RIST).  All rights reserved.
+! Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+! $COPYRIGHT$
+
+#include "mpi-f08-rename.h"
+#include "ompi/mpi/fortran/configure-fortran-output.h"
+
+subroutine MPI_Parrived_f08(request,partition,flag,ierror)
+   ! See note in mpi-f-interfaces-bind.h for why we "use mpi" here and
+   ! call a PMPI_* subroutine below.
+   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
+   use :: mpi, only : PMPI_Parrived
+   implicit none
+   TYPE(MPI_Request), INTENT(IN) :: request
+   INTEGER, INTENT(IN) :: partition
+   LOGICAL, INTENT(OUT) :: flag
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+   integer :: c_ierror
+
+   call PMPI_Parrived(request%MPI_VAL,partition,flag,c_ierror)
+   if (present(ierror)) ierror = c_ierror
+
+end subroutine MPI_Parrived_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/pready_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/pready_f08.F90
new file mode 100644
index 00000000000..a21f4f4b5e9
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/pready_f08.F90
@@ -0,0 +1,26 @@
+! -*- f90 -*-
+!
+! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
+!                         All rights reserved.
+! Copyright (c) 2018      Research Organization for Information Science
+!                         and Technology (RIST).  All rights reserved.
+! Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+! $COPYRIGHT$
+
+#include "mpi-f08-rename.h"
+#include "ompi/mpi/fortran/configure-fortran-output.h"
+
+subroutine MPI_Pready_f08(partition,request,ierror)
+   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
+   use :: ompi_mpifh_bindings, only : ompi_pready_f
+   implicit none
+   INTEGER, INTENT(IN) :: partition
+   TYPE(MPI_Request), INTENT(IN) :: request
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+   integer :: c_ierror
+
+   call ompi_pready_f(partition,request%MPI_VAL,c_ierror)
+   if (present(ierror)) ierror = c_ierror
+
+end subroutine MPI_Pready_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/pready_list_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/pready_list_f08.F90
new file mode 100644
index 00000000000..f021e107cf2
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/pready_list_f08.F90
@@ -0,0 +1,27 @@
+! -*- f90 -*-
+!
+! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
+!                         All rights reserved.
+! Copyright (c) 2018      Research Organization for Information Science
+!                         and Technology (RIST).  All rights reserved.
+! Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+! $COPYRIGHT$
+
+#include "mpi-f08-rename.h"
+#include "ompi/mpi/fortran/configure-fortran-output.h"
+
+subroutine MPI_Pready_list_f08(length,partitions,request,ierror)
+   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
+   use :: ompi_mpifh_bindings, only : ompi_pready_list_f
+   implicit none
+   INTEGER, INTENT(IN) :: length
+   INTEGER, dimension(*), INTENT(IN) :: partitions
+   TYPE(MPI_Request), INTENT(IN) :: request
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+   integer :: c_ierror
+
+   call ompi_pready_list_f(length,partitions,request%MPI_VAL,c_ierror)
+   if (present(ierror)) ierror = c_ierror
+
+end subroutine MPI_Pready_list_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/pready_range_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/pready_range_f08.F90
new file mode 100644
index 00000000000..5f6290c4eac
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/pready_range_f08.F90
@@ -0,0 +1,26 @@
+! -*- f90 -*-
+!
+! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
+!                         All rights reserved.
+! Copyright (c) 2018      Research Organization for Information Science
+!                         and Technology (RIST).  All rights reserved.
+! Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+! $COPYRIGHT$
+
+#include "mpi-f08-rename.h"
+#include "ompi/mpi/fortran/configure-fortran-output.h"
+
+subroutine MPI_Pready_range_f08(partition_low,partition_high,request,ierror)
+   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
+   use :: ompi_mpifh_bindings, only : ompi_pready_range_f
+   implicit none
+   INTEGER, INTENT(IN) :: partition_low,partition_high
+   TYPE(MPI_Request), INTENT(IN) :: request
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+   integer :: c_ierror
+
+   call ompi_pready_range_f(partition_low,partition_high,request%MPI_VAL,c_ierror)
+   if (present(ierror)) ierror = c_ierror
+
+end subroutine MPI_Pready_range_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/precv_init_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/precv_init_ts.c.in
new file mode 100644
index 00000000000..53a5b30f526
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/precv_init_ts.c.in
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2021      Bull S.A.S. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE void precv_init(BUFFER_OUT x, INT partitions, PARTITIONED_COUNT count, DATATYPE datatype, 
+		          RANK dest, TAG tag, COMM comm, INFO info, REQUEST_OUT request)
+{
+   int c_ierr;
+   MPI_Info c_info;
+   MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+   MPI_Request c_req;
+   MPI_Comm c_comm;
+   void *buf = OMPI_CFI_BASE_ADDR(x);
+   @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+
+   c_info = PMPI_Info_f2c(*info);
+   c_comm = PMPI_Comm_f2c (*comm);
+
+   OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+   c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(buf), 
+                      OMPI_FINT_2_INT(*partitions),
+                      c_count,
+                      c_type, OMPI_FINT_2_INT(*dest),
+                      OMPI_FINT_2_INT(*tag),
+                      c_comm, c_info, &c_req);
+   if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+
+   if (MPI_SUCCESS == c_ierr) {
+      *request = PMPI_Request_c2f(c_req);
+   }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/probe_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/probe_f08.F90
index 18459ba5ad4..e6be94ca868 100644
--- a/ompi/mpi/fortran/use-mpi-f08/probe_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/probe_f08.F90
@@ -15,7 +15,7 @@ subroutine MPI_Probe_f08(source,tag,comm,status,ierror)
    implicit none
    INTEGER, INTENT(IN) :: source, tag
    TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Status), INTENT(OUT) :: status
+   TYPE(MPI_Status) :: status
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
    integer :: c_ierror
 
diff --git a/ompi/mpi/fortran/use-mpi-f08/profile/Makefile.am b/ompi/mpi/fortran/use-mpi-f08/profile/Makefile.am
deleted file mode 100644
index 5a52d6c151d..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/profile/Makefile.am
+++ /dev/null
@@ -1,434 +0,0 @@
-
-# -*- makefile.am	 -*-
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2013 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2009-2014 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
-# Copyright (c) 2012      Oak Ridge National Laboratory. All rights reserved.
-# Copyright (c) 2012-2013 Inria.  All rights reserved.
-# Copyright (c) 2013      Los Alamos National Security, LLC. All rights
-#                         reserved.
-# Copyright (c) 2015-2020 Research Organization for Information Science
-#                         and Technology (RIST). All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-include $(top_srcdir)/Makefile.ompi-rules
-
-# This Makefile is only relevant if we're building the "use mpi_f08"
-# MPI bindings.
-if OMPI_BUILD_FORTRAN_USEMPIF08_BINDINGS
-
-AM_FCFLAGS = -I$(top_srcdir)/ompi/mpi/fortran/use-mpi-f08/mod \
-             -I$(top_builddir)/ompi/include \
-             -I$(top_srcdir)/ompi/include \
-             $(OMPI_FC_MODULE_FLAG)$(top_builddir)/ompi/mpi/fortran/use-mpi \
-             $(OMPI_FC_MODULE_FLAG)$(top_builddir)/ompi/$(OMPI_FORTRAN_USEMPI_DIR) \
-             $(OMPI_FC_MODULE_FLAG)../mod \
-             $(OMPI_FC_MODULE_FLAG)../bindings \
-             -I$(top_srcdir) -I$(top_builddir) $(FCFLAGS_f90) \
-             -DOMPI_BUILD_MPI_PROFILING=1
-
-CLEANFILES += *.i90
-
-noinst_LTLIBRARIES = libmpi_usempif08_pmpi.la
-
-pmpi_api_files = \
-        pabort_f08.F90 \
-        paccumulate_f08.F90 \
-        padd_error_class_f08.F90 \
-        padd_error_code_f08.F90 \
-        padd_error_string_f08.F90 \
-        paint_add_f08.F90 \
-        paint_diff_f08.F90 \
-        pallgather_f08.F90 \
-        pallgatherv_f08.F90 \
-        palloc_mem_f08.F90 \
-        pallreduce_f08.F90 \
-        palltoall_f08.F90 \
-        palltoallv_f08.F90 \
-        palltoallw_f08.F90 \
-        pbarrier_f08.F90 \
-        pbcast_f08.F90 \
-        pbsend_f08.F90 \
-        pbsend_init_f08.F90 \
-        pbuffer_attach_f08.F90 \
-        pbuffer_detach_f08.F90 \
-        pcancel_f08.F90 \
-        pcart_coords_f08.F90 \
-        pcart_create_f08.F90 \
-        pcartdim_get_f08.F90 \
-        pcart_get_f08.F90 \
-        pcart_map_f08.F90 \
-        pcart_rank_f08.F90 \
-        pcart_shift_f08.F90 \
-        pcart_sub_f08.F90 \
-        pclose_port_f08.F90 \
-        pcomm_accept_f08.F90 \
-        pcomm_call_errhandler_f08.F90 \
-        pcomm_compare_f08.F90 \
-        pcomm_connect_f08.F90 \
-        pcomm_create_errhandler_f08.F90 \
-        pcomm_create_f08.F90 \
-        pcomm_create_group_f08.F90 \
-        pcomm_create_keyval_f08.F90 \
-        pcomm_delete_attr_f08.F90 \
-        pcomm_disconnect_f08.F90 \
-        pcomm_dup_f08.F90 \
-        pcomm_dup_with_info_f08.F90 \
-        pcomm_idup_f08.F90 \
-        pcomm_free_f08.F90 \
-        pcomm_free_keyval_f08.F90 \
-        pcomm_get_attr_f08.F90 \
-        pcomm_get_errhandler_f08.F90 \
-        pcomm_get_info_f08.F90 \
-        pcomm_get_name_f08.F90 \
-        pcomm_get_parent_f08.F90 \
-        pcomm_group_f08.F90 \
-        pcomm_join_f08.F90 \
-        pcomm_rank_f08.F90 \
-        pcomm_remote_group_f08.F90 \
-        pcomm_remote_size_f08.F90 \
-        pcomm_set_attr_f08.F90 \
-        pcomm_set_errhandler_f08.F90 \
-        pcomm_set_info_f08.F90 \
-        pcomm_set_name_f08.F90 \
-        pcomm_size_f08.F90 \
-        pcomm_spawn_f08.F90 \
-        pcomm_spawn_multiple_f08.F90 \
-        pcomm_split_f08.F90 \
-        pcomm_split_type_f08.F90 \
-        pcomm_test_inter_f08.F90 \
-        pcompare_and_swap_f08.F90 \
-        pdims_create_f08.F90 \
-        pdist_graph_create_adjacent_f08.F90 \
-        pdist_graph_create_f08.F90 \
-        pdist_graph_neighbors_count_f08.F90 \
-        pdist_graph_neighbors_f08.F90 \
-        perrhandler_free_f08.F90 \
-        perror_class_f08.F90 \
-        perror_string_f08.F90 \
-        pexscan_f08.F90 \
-        pf_sync_reg_f08.F90 \
-        pfetch_and_op_f08.F90 \
-        pfile_call_errhandler_f08.F90 \
-        pfile_close_f08.F90 \
-        pfile_create_errhandler_f08.F90 \
-        pfile_delete_f08.F90 \
-        pfile_get_amode_f08.F90 \
-        pfile_get_atomicity_f08.F90 \
-        pfile_get_byte_offset_f08.F90 \
-        pfile_get_errhandler_f08.F90 \
-        pfile_get_group_f08.F90 \
-        pfile_get_info_f08.F90 \
-        pfile_get_position_f08.F90 \
-        pfile_get_position_shared_f08.F90 \
-        pfile_get_size_f08.F90 \
-        pfile_get_type_extent_f08.F90 \
-        pfile_get_view_f08.F90 \
-        pfile_iread_at_f08.F90 \
-        pfile_iread_f08.F90 \
-        pfile_iread_at_all_f08.F90 \
-        pfile_iread_all_f08.F90 \
-        pfile_iread_shared_f08.F90 \
-        pfile_iwrite_at_f08.F90 \
-        pfile_iwrite_f08.F90 \
-        pfile_iwrite_at_all_f08.F90 \
-        pfile_iwrite_all_f08.F90 \
-        pfile_iwrite_shared_f08.F90 \
-        pfile_open_f08.F90 \
-        pfile_preallocate_f08.F90 \
-        pfile_read_all_begin_f08.F90 \
-        pfile_read_all_end_f08.F90 \
-        pfile_read_all_f08.F90 \
-        pfile_read_at_all_begin_f08.F90 \
-        pfile_read_at_all_end_f08.F90 \
-        pfile_read_at_all_f08.F90 \
-        pfile_read_at_f08.F90 \
-        pfile_read_f08.F90 \
-        pfile_read_ordered_begin_f08.F90 \
-        pfile_read_ordered_end_f08.F90 \
-        pfile_read_ordered_f08.F90 \
-        pfile_read_shared_f08.F90 \
-        pfile_seek_f08.F90 \
-        pfile_seek_shared_f08.F90 \
-        pfile_set_atomicity_f08.F90 \
-        pfile_set_errhandler_f08.F90 \
-        pfile_set_info_f08.F90 \
-        pfile_set_size_f08.F90 \
-        pfile_set_view_f08.F90 \
-        pfile_sync_f08.F90 \
-        pfile_write_all_begin_f08.F90 \
-        pfile_write_all_end_f08.F90 \
-        pfile_write_all_f08.F90 \
-        pfile_write_at_all_begin_f08.F90 \
-        pfile_write_at_all_end_f08.F90 \
-        pfile_write_at_all_f08.F90 \
-        pfile_write_at_f08.F90 \
-        pfile_write_f08.F90 \
-        pfile_write_ordered_begin_f08.F90 \
-        pfile_write_ordered_end_f08.F90 \
-        pfile_write_ordered_f08.F90 \
-        pfile_write_shared_f08.F90 \
-        pfinalized_f08.F90 \
-        pfinalize_f08.F90 \
-        pfree_mem_f08.F90 \
-        pgather_f08.F90 \
-        pgatherv_f08.F90 \
-        pget_accumulate_f08.F90 \
-        pget_address_f08.F90 \
-        pget_count_f08.F90 \
-        pget_elements_f08.F90 \
-        pget_elements_x_f08.F90 \
-        pget_f08.F90 \
-        pget_library_version_f08.F90 \
-        pget_processor_name_f08.F90 \
-        pget_version_f08.F90 \
-        pgraph_create_f08.F90 \
-        pgraphdims_get_f08.F90 \
-        pgraph_get_f08.F90 \
-        pgraph_map_f08.F90 \
-        pgraph_neighbors_count_f08.F90 \
-        pgraph_neighbors_f08.F90 \
-        pgrequest_complete_f08.F90 \
-        pgrequest_start_f08.F90 \
-        pgroup_compare_f08.F90 \
-        pgroup_difference_f08.F90 \
-        pgroup_excl_f08.F90 \
-        pgroup_free_f08.F90 \
-        pgroup_incl_f08.F90 \
-        pgroup_intersection_f08.F90 \
-        pgroup_range_excl_f08.F90 \
-        pgroup_range_incl_f08.F90 \
-        pgroup_rank_f08.F90 \
-        pgroup_size_f08.F90 \
-        pgroup_translate_ranks_f08.F90 \
-        pgroup_union_f08.F90 \
-        piallgather_f08.F90 \
-        piallgatherv_f08.F90 \
-        piallreduce_f08.F90 \
-        pialltoall_f08.F90 \
-        pialltoallv_f08.F90 \
-        pialltoallw_f08.F90 \
-        pibarrier_f08.F90 \
-        pibcast_f08.F90 \
-        pibsend_f08.F90 \
-        pigather_f08.F90 \
-        pigatherv_f08.F90 \
-        piexscan_f08.F90 \
-        pimprobe_f08.F90 \
-        pimrecv_f08.F90 \
-        pineighbor_allgather_f08.F90 \
-        pineighbor_allgatherv_f08.F90 \
-        pineighbor_alltoall_f08.F90 \
-        pineighbor_alltoallv_f08.F90 \
-        pineighbor_alltoallw_f08.F90 \
-        pinfo_create_f08.F90 \
-        pinfo_delete_f08.F90 \
-        pinfo_dup_f08.F90 \
-        pinfo_free_f08.F90 \
-        pinfo_get_f08.F90 \
-        pinfo_get_nkeys_f08.F90 \
-        pinfo_get_nthkey_f08.F90 \
-        pinfo_get_valuelen_f08.F90 \
-        pinfo_set_f08.F90 \
-        pinit_f08.F90 \
-        pinitialized_f08.F90 \
-        pinit_thread_f08.F90 \
-        pintercomm_create_f08.F90 \
-        pintercomm_merge_f08.F90 \
-        piprobe_f08.F90 \
-        pirecv_f08.F90 \
-        pireduce_f08.F90 \
-        pireduce_scatter_f08.F90 \
-        pireduce_scatter_block_f08.F90 \
-        pirsend_f08.F90 \
-        piscan_f08.F90 \
-        piscatter_f08.F90 \
-        piscatterv_f08.F90 \
-        pisend_f08.F90 \
-        pissend_f08.F90 \
-        pis_thread_main_f08.F90 \
-        plookup_name_f08.F90 \
-        pmprobe_f08.F90 \
-        pmrecv_f08.F90 \
-        pneighbor_allgather_f08.F90 \
-        pneighbor_allgatherv_f08.F90 \
-        pneighbor_alltoall_f08.F90 \
-        pneighbor_alltoallv_f08.F90 \
-        pneighbor_alltoallw_f08.F90 \
-        pop_commutative_f08.F90 \
-        pop_create_f08.F90 \
-        popen_port_f08.F90 \
-        pop_free_f08.F90 \
-        ppack_external_f08.F90 \
-        ppack_external_size_f08.F90 \
-        ppack_f08.F90 \
-        ppack_size_f08.F90 \
-        ppcontrol_f08.F90 \
-        pprobe_f08.F90 \
-        ppublish_name_f08.F90 \
-        pput_f08.F90 \
-        pquery_thread_f08.F90 \
-        praccumulate_f08.F90 \
-        precv_f08.F90 \
-        precv_init_f08.F90 \
-        preduce_f08.F90 \
-        preduce_local_f08.F90 \
-        preduce_scatter_f08.F90 \
-        preduce_scatter_block_f08.F90 \
-        pregister_datarep_f08.F90 \
-        prequest_free_f08.F90 \
-        prequest_get_status_f08.F90 \
-        prget_f08.F90 \
-        prget_accumulate_f08.F90 \
-        prput_f08.F90 \
-        prsend_f08.F90 \
-        prsend_init_f08.F90 \
-        pscan_f08.F90 \
-        pscatter_f08.F90 \
-        pscatterv_f08.F90 \
-        psend_f08.F90 \
-        psend_init_f08.F90 \
-        psendrecv_f08.F90 \
-        psendrecv_replace_f08.F90 \
-        pssend_f08.F90 \
-        pssend_init_f08.F90 \
-        pstartall_f08.F90 \
-        pstart_f08.F90 \
-        pstatus_f082f_f08.F90 \
-        pstatus_f2f08_f08.F90 \
-        pstatus_set_cancelled_f08.F90 \
-        pstatus_set_elements_f08.F90 \
-        pstatus_set_elements_x_f08.F90 \
-        ptestall_f08.F90 \
-        ptestany_f08.F90 \
-        ptest_cancelled_f08.F90 \
-        ptest_f08.F90 \
-        ptestsome_f08.F90 \
-        ptopo_test_f08.F90 \
-        ptype_commit_f08.F90 \
-        ptype_contiguous_f08.F90 \
-        ptype_create_darray_f08.F90 \
-        ptype_create_f90_complex_f08.F90 \
-        ptype_create_f90_integer_f08.F90 \
-        ptype_create_f90_real_f08.F90 \
-        ptype_create_hindexed_f08.F90 \
-        ptype_create_hvector_f08.F90 \
-        ptype_create_indexed_block_f08.F90 \
-        ptype_create_hindexed_block_f08.F90 \
-        ptype_create_keyval_f08.F90 \
-        ptype_create_resized_f08.F90 \
-        ptype_create_struct_f08.F90 \
-        ptype_create_subarray_f08.F90 \
-        ptype_delete_attr_f08.F90 \
-        ptype_dup_f08.F90 \
-        ptype_free_f08.F90 \
-        ptype_free_keyval_f08.F90 \
-        ptype_get_attr_f08.F90 \
-        ptype_get_contents_f08.F90 \
-        ptype_get_envelope_f08.F90 \
-        ptype_get_extent_f08.F90 \
-        ptype_get_extent_x_f08.F90 \
-        ptype_get_name_f08.F90 \
-        ptype_get_true_extent_f08.F90 \
-        ptype_get_true_extent_x_f08.F90 \
-        ptype_indexed_f08.F90 \
-        ptype_match_size_f08.F90 \
-        ptype_set_attr_f08.F90 \
-        ptype_set_name_f08.F90 \
-        ptype_size_f08.F90 \
-        ptype_size_x_f08.F90 \
-        ptype_vector_f08.F90 \
-        punpack_external_f08.F90 \
-        punpack_f08.F90 \
-        punpublish_name_f08.F90 \
-        pwaitall_f08.F90 \
-        pwaitany_f08.F90 \
-        pwait_f08.F90 \
-        pwaitsome_f08.F90 \
-        pwin_allocate_f08.F90 \
-        pwin_allocate_shared_f08.F90 \
-        pwin_attach_f08.F90 \
-        pwin_call_errhandler_f08.F90 \
-        pwin_complete_f08.F90 \
-        pwin_create_dynamic_f08.F90 \
-        pwin_create_errhandler_f08.F90 \
-        pwin_create_f08.F90 \
-        pwin_create_keyval_f08.F90 \
-        pwin_delete_attr_f08.F90 \
-        pwin_detach_f08.F90 \
-        pwin_fence_f08.F90 \
-        pwin_flush_f08.F90 \
-        pwin_flush_all_f08.F90 \
-        pwin_flush_local_f08.F90 \
-        pwin_flush_local_all_f08.F90 \
-        pwin_free_f08.F90 \
-        pwin_free_keyval_f08.F90 \
-        pwin_get_attr_f08.F90 \
-        pwin_get_errhandler_f08.F90 \
-        pwin_get_group_f08.F90 \
-        pwin_get_info_f08.F90 \
-        pwin_get_name_f08.F90 \
-        pwin_lock_f08.F90 \
-        pwin_lock_all_f08.F90 \
-        pwin_post_f08.F90 \
-        pwin_set_attr_f08.F90 \
-        pwin_set_errhandler_f08.F90 \
-        pwin_set_info_f08.F90 \
-        pwin_set_name_f08.F90 \
-        pwin_shared_query_f08.F90 \
-        pwin_start_f08.F90 \
-        pwin_sync_f08.F90 \
-        pwin_test_f08.F90 \
-        pwin_unlock_f08.F90 \
-        pwin_unlock_all_f08.F90 \
-        pwin_wait_f08.F90
-
-#
-# Automake doesn't do Fortran dependency analysis, so must list them
-# manually here.  Bummer!
-#
-
-pmpi_api_lo_files = $(pmpi_api_files:.F90=.lo)
-
-$(pmpi_api_lo_files): ../bindings/libforce_usempif08_internal_bindings_to_be_built.la
-
-nodist_libmpi_usempif08_pmpi_la_SOURCES = \
-        $(pmpi_api_files)
-
-#
-# Sym link in the sources from the real MPI directory
-#
-$(nodist_libmpi_usempif08_pmpi_la_SOURCES):
-	$(OMPI_V_LN_S) if test ! -r $@ ; then \
-		pname=`echo $@ | cut -b '2-'` ; \
-		$(LN_S) $(top_srcdir)/ompi/mpi/fortran/use-mpi-f08/$$pname $@ ; \
-	fi
-
-# These files were created by targets above
-
-MAINTAINERCLEANFILES = $(nodist_libmpi_usempif08_pmpi_la_SOURCES)
-
-endif
-
-# Don't want these targets in here
-
-tags-recursive:
-tags:
-TAGS:
-GTAGS:
-ID:
diff --git a/ompi/mpi/fortran/use-mpi-f08/psend_init_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/psend_init_ts.c.in
new file mode 100644
index 00000000000..9d7ad29b22c
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/psend_init_ts.c.in
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2021      Bull S.A.S. All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID psend_init(BUFFER_ASYNC x, INT partitions, PARTITIONED_COUNT count, DATATYPE datatype, 
+		          RANK dest, TAG tag, COMM comm, INFO info, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Info c_info;
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    MPI_Request c_req;
+    MPI_Comm c_comm;
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+
+    c_info = PMPI_Info_f2c(*info);
+    c_comm = PMPI_Comm_f2c (*comm);
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(buf), 
+                          OMPI_FINT_2_INT(*partitions),
+                          c_count,
+                          c_type, OMPI_FINT_2_INT(*dest),
+                          OMPI_FINT_2_INT(*tag),
+                          c_comm, c_info, &c_req);
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+   if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+   if (MPI_SUCCESS == c_ierr) {
+      *request = PMPI_Request_c2f(c_req);
+   }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/publish_name_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/publish_name_f08.F90
index e02e21d22c0..e96dccebbe6 100644
--- a/ompi/mpi/fortran/use-mpi-f08/publish_name_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/publish_name_f08.F90
@@ -12,6 +12,7 @@
 subroutine MPI_Publish_name_f08(service_name,info,port_name,ierror)
    use :: mpi_f08_types, only : MPI_Info
    use :: ompi_mpifh_bindings, only : ompi_publish_name_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
    implicit none
    TYPE(MPI_Info), INTENT(IN) :: info
    CHARACTER(LEN=*), INTENT(IN) :: service_name, port_name
@@ -19,7 +20,7 @@ subroutine MPI_Publish_name_f08(service_name,info,port_name,ierror)
    integer :: c_ierror
 
    call ompi_publish_name_f(service_name,info%MPI_VAL,port_name,c_ierror, &
-                            len(service_name), len(port_name))
+                            len(service_name,KIND=C_INT), len(port_name,KIND=C_INT))
    if (present(ierror)) ierror = c_ierror
 
 end subroutine MPI_Publish_name_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/put_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/put_f08.F90
deleted file mode 100644
index efc0e86213c..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/put_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Put_f08(origin_addr,origin_count,origin_datatype,target_rank,&
-                       target_disp,target_count,target_datatype,win,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Win, MPI_ADDRESS_KIND
-   use :: ompi_mpifh_bindings, only : ompi_put_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: origin_addr
-   INTEGER, INTENT(IN) :: origin_count, target_rank, target_count
-   TYPE(MPI_Datatype), INTENT(IN) :: origin_datatype
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
-   TYPE(MPI_Datatype), INTENT(IN) :: target_datatype
-   TYPE(MPI_Win), INTENT(IN) :: win
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_put_f(origin_addr,origin_count,origin_datatype%MPI_VAL,target_rank,&
-                   target_disp,target_count,target_datatype%MPI_VAL,win%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Put_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/put_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/put_ts.c.in
new file mode 100644
index 00000000000..89ef56a2552
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/put_ts.c.in
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID put(BUFFER x, COUNT origin_count,
+                   DATATYPE origin_datatype, RANK target_rank,
+                   AINT target_disp, COUNT target_count,
+                   DATATYPE target_datatype, WIN win)
+{
+    int c_ierr;
+    MPI_Datatype c_origin_datatype, c_origin_type = PMPI_Type_f2c(*origin_datatype);
+    MPI_Datatype c_target_datatype = PMPI_Type_f2c(*target_datatype);
+    MPI_Win c_win = PMPI_Win_f2c(*win);
+    char *origin_addr = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_origin_count = (@COUNT_TYPE@)*origin_count;
+    @COUNT_TYPE@ c_target_count = (@COUNT_TYPE@)*target_count;
+
+    OMPI_CFI_2_C(x, c_origin_count, c_origin_type, c_origin_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_win, c_ierr, FUNC_NAME)
+        return;
+    }
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(origin_addr),
+                          c_origin_count,
+                          c_origin_datatype,
+                          OMPI_FINT_2_INT(*target_rank),
+                          *target_disp,
+                          c_target_count,
+                          c_target_datatype, c_win);
+    if (c_origin_datatype != c_origin_type) {
+        ompi_datatype_destroy(&c_origin_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/raccumulate_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/raccumulate_f08.F90
deleted file mode 100644
index f612d3700e2..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/raccumulate_f08.F90
+++ /dev/null
@@ -1,36 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2014 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Raccumulate_f08(origin_addr,origin_count,origin_datatype,&
-                               target_rank,target_disp,target_count,    &
-                               target_datatype,op,win,request, ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Win, MPI_Request, MPI_ADDRESS_KIND
-   use :: ompi_mpifh_bindings, only : ompi_raccumulate_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: origin_addr
-   INTEGER, INTENT(IN) :: origin_count, target_rank, target_count
-   TYPE(MPI_Datatype), INTENT(IN) :: origin_datatype
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
-   TYPE(MPI_Datatype), INTENT(IN) :: target_datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Win), INTENT(IN) :: win
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_raccumulate_f(origin_addr,origin_count,origin_datatype%MPI_VAL,target_rank,&
-                           target_disp,target_count,target_datatype%MPI_VAL,&
-                           op%MPI_VAL,win%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Raccumulate_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/raccumulate_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/raccumulate_ts.c.in
new file mode 100644
index 00000000000..16d6ea56dba
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/raccumulate_ts.c.in
@@ -0,0 +1,67 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2014-2016 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID raccumulate(BUFFER x, COUNT origin_count,
+                           DATATYPE origin_datatype, RANK target_rank,
+                           AINT target_disp, COUNT target_count,
+                           DATATYPE target_datatype, OP op, WIN win,
+                           REQUEST_OUT request)
+{
+    int c_ierr;
+
+    MPI_Datatype c_origin_datatype, c_origin_type = PMPI_Type_f2c(*origin_datatype);
+    MPI_Datatype c_target_datatype = PMPI_Type_f2c(*target_datatype);
+    MPI_Win c_win = PMPI_Win_f2c(*win);
+    MPI_Op c_op = PMPI_Op_f2c(*op);
+    MPI_Request c_req;
+    char *origin_addr = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_origin_count = (@COUNT_TYPE@) *origin_count;
+    @COUNT_TYPE@ c_target_count = (@COUNT_TYPE@) *target_count;
+
+    OMPI_CFI_2_C(x, c_origin_count, c_origin_type, c_origin_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_win, c_ierr, FUNC_NAME)
+        return;
+    }
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(origin_addr),
+                          c_origin_count,
+                          c_origin_datatype,
+                          OMPI_FINT_2_INT(*target_rank),
+                          *target_disp,
+                          c_target_count,
+                          c_target_datatype, c_op, c_win,
+                          &c_req);
+    if (c_origin_datatype != c_origin_type) {
+        ompi_datatype_destroy(&c_origin_datatype);
+    }
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_req);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/recv_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/recv_f08.F90
deleted file mode 100644
index 034fe0fdc19..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/recv_f08.F90
+++ /dev/null
@@ -1,29 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Recv_f08(buf,count,datatype,source,tag,comm,status,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Status
-   use :: ompi_mpifh_bindings, only : ompi_recv_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: buf
-   INTEGER, INTENT(IN) :: count, source, tag
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Status), INTENT(OUT) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_recv_f(buf,count,datatype%MPI_VAL,source,tag,comm%MPI_VAL,status,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Recv_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/recv_init_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/recv_init_f08.F90
deleted file mode 100644
index 121e97f097f..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/recv_init_f08.F90
+++ /dev/null
@@ -1,30 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Recv_init_f08(buf,count,datatype,source,tag,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_recv_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: buf
-   INTEGER, INTENT(IN) :: count, source, tag
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_recv_init_f(buf,count,datatype%MPI_VAL,source,tag,comm%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Recv_init_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/recv_init_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/recv_init_ts.c.in
new file mode 100644
index 00000000000..d5653252687
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/recv_init_ts.c.in
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID recv_init(BUFFER_ASYNC x, COUNT count, DATATYPE datatype,
+                         RANK source, TAG tag, COMM comm,
+                         REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    MPI_Request c_req;
+    MPI_Comm c_comm = PMPI_Comm_f2c (*comm);
+
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(buf), c_count,
+                          c_datatype, OMPI_FINT_2_INT(*source),
+                          OMPI_INT_2_FINT(*tag), c_comm,
+                          &c_req);
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+       *request = PMPI_Request_c2f(c_req);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/recv_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/recv_ts.c.in
new file mode 100644
index 00000000000..5cada770821
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/recv_ts.c.in
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID recv(BUFFER_OUT x, COUNT count, DATATYPE datatype,
+                    RANK source, TAG tag, COMM comm, STATUS_OUT status)
+{
+    OMPI_FORTRAN_STATUS_DECLARATION(c_status,c_status2)
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    int c_ierr;
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    OMPI_FORTRAN_STATUS_SET_POINTER(c_status,c_status2,status)
+
+    /* Call the C function */
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(buf), c_count,
+                          c_datatype, OMPI_FINT_2_INT(*source),
+                          OMPI_FINT_2_INT(*tag), c_comm,
+                          c_status);
+
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    OMPI_FORTRAN_STATUS_RETURN(c_status,c_status2,status,c_ierr)
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/reduce_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/reduce_f08.F90
deleted file mode 100644
index 1f9baea4005..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/reduce_f08.F90
+++ /dev/null
@@ -1,31 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Reduce_f08(sendbuf,recvbuf,count,datatype,op,root,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm
-   use :: ompi_mpifh_bindings, only : ompi_reduce_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: count, root
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_reduce_f(sendbuf,recvbuf,count,datatype%MPI_VAL,&
-                      op%MPI_VAL,root,comm%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Reduce_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/reduce_init_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/reduce_init_ts.c.in
new file mode 100644
index 00000000000..90a4c2b3077
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/reduce_init_ts.c.in
@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID reduce_init(BUFFER x1, BUFFER_OUT x2, COUNT count,
+                           DATATYPE datatype, OP op,
+                           RANK root, COMM comm,
+	   	           INFO info, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Datatype c_type;
+    MPI_Op c_op;
+    MPI_Request c_request;
+    MPI_Info c_info;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+
+    c_info = PMPI_Info_f2c(*info);
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    c_type = PMPI_Type_f2c(*datatype);
+    c_op = PMPI_Op_f2c(*op);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf, recvbuf,
+                          OMPI_FINT_2_INT(*count),
+                          c_type, c_op,
+                          OMPI_FINT_2_INT(*root),
+                          c_comm,
+			  c_info,
+			  &c_request);
+   if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+   if (MPI_SUCCESS == c_ierr) {
+       *request = PMPI_Request_c2f(c_request);
+   }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/reduce_local_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/reduce_local_f08.F90
deleted file mode 100644
index ad9c10106b0..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/reduce_local_f08.F90
+++ /dev/null
@@ -1,29 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Reduce_local_f08(inbuf,inoutbuf,count,datatype,op,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op
-   use :: ompi_mpifh_bindings, only : ompi_reduce_local_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: inbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: inoutbuf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_reduce_local_f(inbuf,inoutbuf,count,datatype%MPI_VAL,op%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Reduce_local_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/reduce_local_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/reduce_local_ts.c.in
new file mode 100644
index 00000000000..d4d146db1f7
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/reduce_local_ts.c.in
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID reduce_local(BUFFER x1, BUFFER_OUT x2, COUNT count,
+                            DATATYPE datatype, OP op)
+{
+    int c_ierr;
+    MPI_Datatype c_type;
+    MPI_Op c_op;
+    char *inbuf = OMPI_CFI_BASE_ADDR(x1), *inoutbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@)*count;
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(MPI_COMM_SELF, c_ierr, FUNC_NAME)
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(MPI_COMM_SELF, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    c_type = PMPI_Type_f2c(*datatype);
+    c_op = PMPI_Op_f2c(*op);
+
+    inbuf = (char *) OMPI_F2C_BOTTOM(inbuf);
+    inoutbuf = (char *) OMPI_F2C_BOTTOM(inoutbuf);
+
+    c_ierr = @INNER_CALL@(inbuf, inoutbuf,
+                          c_count,
+                          c_type, c_op);
+   if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/reduce_scatter_block_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/reduce_scatter_block_f08.F90
deleted file mode 100644
index 6bd4b38bbce..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/reduce_scatter_block_f08.F90
+++ /dev/null
@@ -1,31 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Reduce_scatter_block_f08(sendbuf,recvbuf,recvcount,datatype,op,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm
-   use :: ompi_mpifh_bindings, only : ompi_reduce_scatter_block_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: recvcount
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_reduce_scatter_block_f(sendbuf,recvbuf,recvcount,&
-                                    datatype%MPI_VAL,op%MPI_VAL,comm%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Reduce_scatter_block_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/reduce_scatter_block_init_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/reduce_scatter_block_init_ts.c.in
new file mode 100644
index 00000000000..ed1e2a78401
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/reduce_scatter_block_init_ts.c.in
@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID reduce_scatter_block_init(BUFFER x1, BUFFER_OUT x2,
+                                         COUNT recvcount, DATATYPE datatype,
+                                         OP op, COMM comm, INFO info,
+					 REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype c_type;
+    MPI_Op c_op;
+    MPI_Info c_info;
+    MPI_Request c_request;
+    int size;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_recvcount = (@COUNT_TYPE@)*recvcount;
+
+    c_info = PMPI_Info_f2c(*info);
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    c_type = PMPI_Type_f2c(*datatype);
+    c_op = PMPI_Op_f2c(*op);
+
+    PMPI_Comm_size(c_comm, &size);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf, recvbuf,
+                          c_recvcount,
+                          c_type, c_op, c_comm,
+			  c_info, &c_request);
+   if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+   if (MPI_SUCCESS == c_ierr) {
+       *request = PMPI_Request_c2f(c_request);
+   }
+
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/reduce_scatter_block_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/reduce_scatter_block_ts.c.in
new file mode 100644
index 00000000000..2603074a872
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/reduce_scatter_block_ts.c.in
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID reduce_scatter_block(BUFFER x1, BUFFER_OUT x2,
+                                    COUNT recvcount, DATATYPE datatype,
+                                    OP op, COMM comm)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype c_type;
+    MPI_Op c_op;
+    int size;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_recvcount = (@COUNT_TYPE@)*recvcount;
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    c_type = PMPI_Type_f2c(*datatype);
+    c_op = PMPI_Op_f2c(*op);
+
+    PMPI_Comm_size(c_comm, &size);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf, recvbuf,
+                          c_recvcount,
+                          c_type, c_op, c_comm);
+   if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/reduce_scatter_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/reduce_scatter_f08.F90
deleted file mode 100644
index a8587ea033b..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/reduce_scatter_f08.F90
+++ /dev/null
@@ -1,31 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Reduce_scatter_f08(sendbuf,recvbuf,recvcounts,datatype,op,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm
-   use :: ompi_mpifh_bindings, only : ompi_reduce_scatter_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: recvcounts(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_reduce_scatter_f(sendbuf,recvbuf,recvcounts,datatype%MPI_VAL,&
-                              op%MPI_VAL,comm%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Reduce_scatter_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/reduce_scatter_init_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/reduce_scatter_init_ts.c.in
new file mode 100644
index 00000000000..a9a9d02a64b
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/reduce_scatter_init_ts.c.in
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID reduce_scatter_init(BUFFER x1, BUFFER_OUT x2,
+                                   COUNT_ARRAY recvcounts, DATATYPE datatype,
+                                   OP op, COMM comm, INFO info, 
+				   REQUEST_OUT request)
+{
+    int c_ierr, idx = 0;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype c_type;
+    MPI_Op c_op;
+    MPI_Request c_request;
+    MPI_Info c_info;
+    int size;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ *tmp_recvcounts = NULL;
+
+    c_info = PMPI_Info_f2c(*info);
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    c_type = PMPI_Type_f2c(*datatype);
+    c_op = PMPI_Op_f2c(*op);
+
+    PMPI_Comm_size(c_comm, &size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(recvcounts, tmp_recvcounts, size);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf, 
+		          recvbuf,
+                          tmp_recvcounts,
+                          c_type, 
+			  c_op, 
+			  c_comm,
+			  c_info,
+			  &c_request);
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_request);
+    }
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(recvcounts, tmp_recvcounts, c_request, c_ierr, idx);
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/reduce_scatter_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/reduce_scatter_ts.c.in
new file mode 100644
index 00000000000..3203c959999
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/reduce_scatter_ts.c.in
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID reduce_scatter(BUFFER x1, BUFFER_OUT x2,
+                              COUNT_ARRAY recvcounts, DATATYPE datatype,
+                              OP op, COMM comm)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype c_type;
+    MPI_Op c_op;
+    int size;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ *tmp_recvcounts = NULL;
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    c_type = PMPI_Type_f2c(*datatype);
+    c_op = PMPI_Op_f2c(*op);
+
+    PMPI_Comm_size(c_comm, &size);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(recvcounts, tmp_recvcounts, size);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf, recvbuf,
+                          tmp_recvcounts,
+                          c_type, c_op, c_comm);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(recvcounts, tmp_recvcounts);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/reduce_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/reduce_ts.c.in
new file mode 100644
index 00000000000..85a6d614213
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/reduce_ts.c.in
@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID reduce(BUFFER x1, BUFFER_OUT x2, COUNT count,
+                      DATATYPE datatype, OP op,
+                      RANK root, COMM comm)
+{
+    int c_ierr;
+    MPI_Datatype c_type;
+    MPI_Op c_op;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    c_type = PMPI_Type_f2c(*datatype);
+    c_op = PMPI_Op_f2c(*op);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf, recvbuf,
+                          OMPI_FINT_2_INT(*count),
+                          c_type, c_op,
+                          OMPI_FINT_2_INT(*root),
+                          c_comm);
+   if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/register_datarep_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/register_datarep_f08.F90
index e1bddfacea7..b5b5850e00c 100644
--- a/ompi/mpi/fortran/use-mpi-f08/register_datarep_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/register_datarep_f08.F90
@@ -13,7 +13,7 @@
 
 subroutine MPI_Register_datarep_f08(datarep,read_conversion_fn,write_conversion_fn, &
                                     dtype_file_extent_fn,extra_state,ierror)
-   use, intrinsic :: iso_c_binding, only: c_funptr, c_funloc
+   use, intrinsic :: iso_c_binding, only: c_funptr, c_funloc, C_INT
    use :: mpi_f08_types, only : MPI_ADDRESS_KIND
    use :: mpi_f08_interfaces_callbacks, only : MPI_Datarep_conversion_function
    use :: mpi_f08_interfaces_callbacks, only : MPI_Datarep_extent_function
@@ -32,7 +32,7 @@ subroutine MPI_Register_datarep_f08(datarep,read_conversion_fn,write_conversion_
    fwrite_fn = c_funloc(write_conversion_fn)
    fdtype_fn = c_funloc(dtype_file_extent_fn)
    call ompi_register_datarep_f(datarep,fread_fn,fwrite_fn, &
-                                fdtype_fn,extra_state,c_ierror,len(datarep))
+                                fdtype_fn,extra_state,c_ierror,len(datarep,KIND=C_INT))
    if (present(ierror)) ierror = c_ierror
 
 end subroutine MPI_Register_datarep_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/rget_accumulate_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/rget_accumulate_f08.F90
deleted file mode 100644
index 831da177e1c..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/rget_accumulate_f08.F90
+++ /dev/null
@@ -1,40 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2014 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Rget_accumulate_f08(origin_addr,origin_count,origin_datatype,&
-                                   result_addr,result_count,result_datatype,&
-                                   target_rank,target_disp,target_count,    &
-                                   target_datatype,op,win,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Win, MPI_Request, MPI_ADDRESS_KIND
-   use :: ompi_mpifh_bindings, only : ompi_rget_accumulate_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: origin_addr
-   INTEGER, INTENT(IN) :: origin_count, result_count, target_rank, target_count
-   TYPE(MPI_Datatype), INTENT(IN) :: origin_datatype
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: result_addr
-   TYPE(MPI_Datatype), INTENT(IN) :: result_datatype
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
-   TYPE(MPI_Datatype), INTENT(IN) :: target_datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Win), INTENT(IN) :: win
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_rget_accumulate_f(origin_addr,origin_count,origin_datatype%MPI_VAL,&
-                               result_addr,result_count,result_datatype%MPI_VAL,&
-                               target_rank,target_disp,target_count,target_datatype%MPI_VAL,&
-                               op%MPI_VAL,win%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Rget_accumulate_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/rget_accumulate_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/rget_accumulate_ts.c.in
new file mode 100644
index 00000000000..a9aedc9197d
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/rget_accumulate_ts.c.in
@@ -0,0 +1,84 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2014-2016 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2015      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID rget_accumulate(BUFFER_ASYNC x1, COUNT origin_count,
+                               DATATYPE origin_datatype, BUFFER_ASYNC_OUT x2,
+                               COUNT result_count, DATATYPE result_datatype,
+                               RANK target_rank, AINT target_disp,
+                               COUNT target_count, DATATYPE target_datatype,
+                               OP op, WIN win, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Datatype c_origin_datatype, c_origin_type = PMPI_Type_f2c(*origin_datatype);
+    MPI_Datatype c_result_datatype, c_result_type = PMPI_Type_f2c(*result_datatype);
+    MPI_Datatype c_target_datatype = PMPI_Type_f2c(*target_datatype);
+    MPI_Win c_win = PMPI_Win_f2c(*win);
+    MPI_Op c_op = PMPI_Op_f2c(*op);
+    char *origin_addr = OMPI_CFI_BASE_ADDR(x1);
+    @COUNT_TYPE@ c_origin_count = (@COUNT_TYPE@) *origin_count;
+    @COUNT_TYPE@ c_target_count = (@COUNT_TYPE@) *target_count;
+    char *result_addr = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_result_count = (@COUNT_TYPE@) *result_count;
+    MPI_Request c_req;
+
+    OMPI_CFI_2_C(x1, c_origin_count, c_origin_type, c_origin_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_win, c_ierr, FUNC_NAME)
+        return;
+    }
+    OMPI_CFI_2_C(x2, c_result_count, c_result_type, c_result_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        if (c_origin_datatype != c_origin_type) {
+            ompi_datatype_destroy(&c_origin_datatype);
+        }
+        OMPI_ERRHANDLER_INVOKE(c_win, c_ierr, FUNC_NAME);
+        return;
+    }
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(origin_addr),
+                          c_origin_count,
+                          c_origin_datatype,
+                          OMPI_F2C_BOTTOM(result_addr),
+                          c_result_count,
+                          c_result_datatype,
+                          OMPI_FINT_2_INT(*target_rank),
+                          *target_disp,
+                          c_target_count,
+                          c_target_datatype, c_op, c_win, &c_req);
+    if (c_origin_datatype != c_origin_type) {
+        ompi_datatype_destroy(&c_origin_datatype);
+    }
+    if (c_result_datatype != c_result_type) {
+        ompi_datatype_destroy(&c_result_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_req);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/rget_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/rget_f08.F90
deleted file mode 100644
index fcc81c3d8d0..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/rget_f08.F90
+++ /dev/null
@@ -1,34 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2014 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Rget_f08(origin_addr,origin_count,origin_datatype,target_rank,&
-                        target_disp,target_count,target_datatype,win,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Win, MPI_Request, MPI_ADDRESS_KIND
-   use :: ompi_mpifh_bindings, only : ompi_rget_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: origin_addr
-   INTEGER, INTENT(IN) :: origin_count, target_rank, target_count
-   TYPE(MPI_Datatype), INTENT(IN) :: origin_datatype
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
-   TYPE(MPI_Datatype), INTENT(IN) :: target_datatype
-   TYPE(MPI_Win), INTENT(IN) :: win
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_rget_f(origin_addr,origin_count,origin_datatype%MPI_VAL,target_rank,&
-                    target_disp,target_count,target_datatype%MPI_VAL,win%MPI_VAL,&
-                    request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Rget_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/rget_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/rget_ts.c.in
new file mode 100644
index 00000000000..734f34033c1
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/rget_ts.c.in
@@ -0,0 +1,62 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2014-2016 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID rget(BUFFER_ASYNC_OUT x, COUNT origin_count,
+                    DATATYPE origin_datatype, RANK target_rank,
+                    AINT target_disp, COUNT target_count,
+                    DATATYPE target_datatype, WIN win, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Datatype c_origin_datatype, c_origin_type = PMPI_Type_f2c(*origin_datatype);
+    MPI_Datatype c_target_datatype = PMPI_Type_f2c(*target_datatype);
+    MPI_Win c_win = PMPI_Win_f2c(*win);
+    char *origin_addr = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_origin_count = (@COUNT_TYPE@) *origin_count;
+    @COUNT_TYPE@ c_target_count = (@COUNT_TYPE@) *target_count;
+    MPI_Request c_req;
+
+    OMPI_CFI_2_C(x, c_origin_count, c_origin_type, c_origin_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_win, c_ierr, FUNC_NAME);
+        return;
+    }
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(origin_addr),
+                          c_origin_count,
+                          c_origin_datatype,
+                          OMPI_FINT_2_INT(*target_rank),
+                          *target_disp,
+                          c_target_count,
+                          c_target_datatype, c_win, &c_req);
+    if (c_origin_datatype != c_origin_type) {
+        ompi_datatype_destroy(&c_origin_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_req);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/rput_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/rput_f08.F90
deleted file mode 100644
index 9372e713388..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/rput_f08.F90
+++ /dev/null
@@ -1,34 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2014 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Rput_f08(origin_addr,origin_count,origin_datatype,target_rank,&
-                        target_disp,target_count,target_datatype,win,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Win, MPI_Request, MPI_ADDRESS_KIND
-   use :: ompi_mpifh_bindings, only : ompi_rput_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: origin_addr
-   INTEGER, INTENT(IN) :: origin_count, target_rank, target_count
-   TYPE(MPI_Datatype), INTENT(IN) :: origin_datatype
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
-   TYPE(MPI_Datatype), INTENT(IN) :: target_datatype
-   TYPE(MPI_Win), INTENT(IN) :: win
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_rput_f(origin_addr,origin_count,origin_datatype%MPI_VAL,target_rank,&
-                    target_disp,target_count,target_datatype%MPI_VAL,win%MPI_VAL,&
-                    request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Rput_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/rput_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/rput_ts.c.in
new file mode 100644
index 00000000000..e74aeafa456
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/rput_ts.c.in
@@ -0,0 +1,62 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2014-2016 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID rput(BUFFER_ASYNC x, COUNT origin_count,
+                    DATATYPE origin_datatype, RANK target_rank,
+                    AINT target_disp, COUNT target_count,
+                    DATATYPE target_datatype, WIN win, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Datatype c_origin_datatype, c_origin_type = PMPI_Type_f2c(*origin_datatype);
+    MPI_Datatype c_target_datatype = PMPI_Type_f2c(*target_datatype);
+    MPI_Win c_win = PMPI_Win_f2c(*win);
+    char *origin_addr = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_origin_count = (@COUNT_TYPE@) *origin_count;
+    @COUNT_TYPE@ c_target_count = (@COUNT_TYPE@) *target_count;
+    MPI_Request c_req;
+
+    OMPI_CFI_2_C(x, c_origin_count, c_origin_type, c_origin_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_win, c_ierr, FUNC_NAME);
+        return;
+    }
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(origin_addr),
+                          c_origin_count,
+                          c_origin_datatype,
+                          OMPI_FINT_2_INT(*target_rank),
+                          *target_disp,
+                          c_target_count,
+                          c_target_datatype, c_win, &c_req);
+    if (c_origin_datatype != c_origin_type) {
+        ompi_datatype_destroy(&c_origin_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_req);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/rsend_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/rsend_f08.F90
deleted file mode 100644
index e00ebee63e9..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/rsend_f08.F90
+++ /dev/null
@@ -1,28 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Rsend_f08(buf,count,datatype,dest,tag,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   use :: ompi_mpifh_bindings, only : ompi_rsend_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count, dest, tag
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_rsend_f(buf,count,datatype%MPI_VAL,dest,tag,comm%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Rsend_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/rsend_init_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/rsend_init_f08.F90
deleted file mode 100644
index ca87afc5cef..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/rsend_init_f08.F90
+++ /dev/null
@@ -1,29 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Rsend_init_f08(buf,count,datatype,dest,tag,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_rsend_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count, dest, tag
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_rsend_init_f(buf,count,datatype%MPI_VAL,dest,tag,comm%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Rsend_init_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/rsend_init_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/rsend_init_ts.c.in
new file mode 100644
index 00000000000..7c4948edac7
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/rsend_init_ts.c.in
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID rsend_init(BUFFER_ASYNC x, COUNT count,
+                          DATATYPE datatype, RANK dest,
+                          TAG tag, COMM comm,
+                          REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    MPI_Request c_req;
+    MPI_Comm c_comm = PMPI_Comm_f2c (*comm);
+
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(buf), c_count,
+                          c_datatype, OMPI_FINT_2_INT(*dest),
+                          OMPI_FINT_2_INT(*tag),
+                          c_comm, &c_req);
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_req);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/rsend_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/rsend_ts.c.in
new file mode 100644
index 00000000000..b46fbd78715
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/rsend_ts.c.in
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID rsend(BUFFER x, COUNT count, DATATYPE datatype,
+                     RANK dest, TAG tag, COMM comm)
+{
+    int c_ierr;
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    MPI_Comm c_comm = PMPI_Comm_f2c (*comm);
+
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(buf), c_count,
+                          c_datatype, OMPI_FINT_2_INT(*dest),
+                          OMPI_FINT_2_INT(*tag), c_comm);
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/scan_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/scan_f08.F90
deleted file mode 100644
index 78a09fa6dad..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/scan_f08.F90
+++ /dev/null
@@ -1,31 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Scan_f08(sendbuf,recvbuf,count,datatype,op,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm
-   use :: ompi_mpifh_bindings, only : ompi_scan_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_scan_f(sendbuf,recvbuf,count,datatype%MPI_VAL,&
-                    op%MPI_VAL,comm%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Scan_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/scan_init_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/scan_init_ts.c.in
new file mode 100644
index 00000000000..82df836470c
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/scan_init_ts.c.in
@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID scan_init(BUFFER x1, BUFFER_OUT x2, COUNT count,
+                        DATATYPE datatype, OP op, COMM comm,
+                        INFO info, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype c_type;
+    MPI_Op c_op;
+    MPI_Info c_info;
+    MPI_Request c_request;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@)*count;
+
+    c_info = PMPI_Info_f2c(*info);
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    c_type = PMPI_Type_f2c(*datatype);
+    c_op = PMPI_Op_f2c(*op);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf, recvbuf,
+                          c_count,
+                          c_type, c_op,
+                          c_comm,
+			  c_info,
+			  &c_request);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_request);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/scan_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/scan_ts.c.in
new file mode 100644
index 00000000000..8aface86de8
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/scan_ts.c.in
@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID scan(BUFFER x1, BUFFER_OUT x2, COUNT count,
+                    DATATYPE datatype, OP op, COMM comm)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype c_type;
+    MPI_Op c_op;
+    char *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@)*count;
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+    OMPI_CFI_CHECK_CONTIGUOUS(x2, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+        return;
+    }
+
+    c_type = PMPI_Type_f2c(*datatype);
+    c_op = PMPI_Op_f2c(*op);
+
+    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf, recvbuf,
+                          c_count,
+                          c_type, c_op,
+                          c_comm);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/scatter_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/scatter_f08.F90
deleted file mode 100644
index 26ee9507e6f..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/scatter_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Scatter_f08(sendbuf,sendcount,sendtype,recvbuf,&
-                           recvcount,recvtype,root,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   use :: ompi_mpifh_bindings, only : ompi_scatter_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount, root
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_scatter_f(sendbuf,sendcount,sendtype%MPI_VAL,recvbuf,recvcount,&
-                       recvtype%MPI_VAL,root,comm%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Scatter_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/scatter_init_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/scatter_init_ts.c.in
new file mode 100644
index 00000000000..94f94eb5c85
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/scatter_init_ts.c.in
@@ -0,0 +1,101 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2013 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID scatter_init(BUFFER x1, COUNT sendcount,
+                            DATATYPE sendtype, BUFFER_OUT x2,
+                            COUNT recvcount, DATATYPE recvtype,
+                            RANK root, COMM comm, INFO info, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    int c_root = OMPI_FINT_2_INT(*root);
+    MPI_Datatype c_sendtype = NULL, c_recvtype = NULL, c_recvdatatype = NULL;
+    void *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    MPI_Info c_info;
+    MPI_Request c_request;
+    @COUNT_TYPE@ c_sendcount = 0, c_recvcount = 0;
+
+    c_info = PMPI_Info_f2c(*info);
+
+    if (OMPI_COMM_IS_INTER(c_comm)) {
+        if (MPI_ROOT == c_root) {
+            c_sendtype = PMPI_Type_f2c(*sendtype);
+            c_sendcount = (@COUNT_TYPE@) *sendcount;
+            OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+                return;
+            }
+        } else if (MPI_PROC_NULL != c_root) {
+            c_recvtype = PMPI_Type_f2c(*recvtype);
+            c_recvcount = (@COUNT_TYPE@) *recvcount;
+            OMPI_CFI_2_C(x2, c_recvcount, c_recvtype, c_recvdatatype, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+                return;
+            }
+        }
+    } else {
+        if (ompi_comm_rank(c_comm) == c_root) {
+            c_sendtype = PMPI_Type_f2c(*sendtype);
+            c_sendcount = (@COUNT_TYPE@) *sendcount;
+            OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+                return;
+            }
+        }
+        if (OMPI_IS_FORTRAN_IN_PLACE(recvbuf)) {
+            recvbuf = MPI_IN_PLACE;
+        } else {
+            c_recvtype = PMPI_Type_f2c(*recvtype);
+            c_recvcount = (@COUNT_TYPE@) *recvcount;
+            OMPI_CFI_2_C(x2, c_recvcount, c_recvtype, c_recvdatatype, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+                return;
+            }
+        }
+    }
+
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf,c_sendcount, c_sendtype,
+                          recvbuf, c_recvcount, c_recvdatatype,
+                          c_root, c_comm, c_info, &c_request);
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_request);
+    }
+
+    if ((c_recvdatatype != NULL ) && (c_recvdatatype != c_recvtype)){
+        ompi_datatype_destroy(&c_recvdatatype);
+    }
+
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/scatter_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/scatter_ts.c.in
new file mode 100644
index 00000000000..7eb27a09fda
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/scatter_ts.c.in
@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2013 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID scatter(BUFFER x1, COUNT sendcount,
+                       DATATYPE sendtype, BUFFER_OUT x2,
+                       COUNT recvcount, DATATYPE recvtype,
+                       RANK root, COMM comm)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    int c_root = OMPI_FINT_2_INT(*root);
+    MPI_Datatype c_sendtype = NULL, c_recvtype = NULL, c_recvdatatype = NULL;
+    void *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_sendcount = 0, c_recvcount = 0;
+
+    if (OMPI_COMM_IS_INTER(c_comm)) {
+        if (MPI_ROOT == c_root) {
+            c_sendtype = PMPI_Type_f2c(*sendtype);
+            c_sendcount = (@COUNT_TYPE@) *sendcount;
+            OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+                return;
+            }
+        } else if (MPI_PROC_NULL != c_root) {
+            c_recvtype = PMPI_Type_f2c(*recvtype);
+            c_recvcount = (@COUNT_TYPE@) *recvcount;
+            OMPI_CFI_2_C(x2, c_recvcount, c_recvtype, c_recvdatatype, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+                return;
+            }
+        }
+    } else {
+        if (ompi_comm_rank(c_comm) == c_root) {
+            c_sendtype = PMPI_Type_f2c(*sendtype);
+            c_sendcount = (@COUNT_TYPE@) *sendcount;
+            OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+                return;
+            }
+        }
+        if (OMPI_IS_FORTRAN_IN_PLACE(recvbuf)) {
+            recvbuf = MPI_IN_PLACE;
+        } else {
+            c_recvtype = PMPI_Type_f2c(*recvtype);
+            c_recvcount = (@COUNT_TYPE@) *recvcount;
+            OMPI_CFI_2_C(x2, c_recvcount, c_recvtype, c_recvdatatype, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+                return;
+            }
+        }
+    }
+
+    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
+    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
+
+    c_ierr = @INNER_CALL@(sendbuf,c_sendcount, c_sendtype,
+                          recvbuf, c_recvcount, c_recvdatatype,
+                          c_root, c_comm);
+
+    if (c_recvdatatype != c_recvtype) {
+        ompi_datatype_destroy(&c_recvdatatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/scatterv_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/scatterv_f08.F90
deleted file mode 100644
index 931d264748b..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/scatterv_f08.F90
+++ /dev/null
@@ -1,33 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Scatterv_f08(sendbuf,sendcounts,displs,sendtype,recvbuf,&
-                            recvcount,recvtype,root,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   use :: ompi_mpifh_bindings, only : ompi_scatterv_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: recvcount, root
-   INTEGER, INTENT(IN) :: sendcounts(*), displs(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_scatterv_f(sendbuf,sendcounts,displs,sendtype%MPI_VAL,recvbuf,&
-                        recvcount,recvtype%MPI_VAL,root,comm%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Scatterv_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/scatterv_init_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/scatterv_init_ts.c.in
new file mode 100644
index 00000000000..046f0669336
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/scatterv_init_ts.c.in
@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID scatterv_init(BUFFER x1, COUNT_ARRAY sendcounts,
+                             DISP_ARRAY displs, DATATYPE sendtype,
+                             BUFFER_OUT x2, COUNT recvcount,
+                             DATATYPE recvtype, RANK root,
+                             COMM comm, INFO info, REQUEST_OUT request)
+{
+    int c_ierr;
+    int idx = 0;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    int c_root = OMPI_FINT_2_INT(*root);
+    MPI_Datatype c_sendtype = NULL, c_recvtype = NULL, c_recvdatatype = NULL;
+    void *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    MPI_Info c_info;
+    MPI_Request c_request;
+    @COUNT_TYPE@ c_recvcount = 0;
+    @COUNT_TYPE@ *tmp_sendcounts = NULL;
+    @DISP_TYPE@ *tmp_displs = NULL;
+
+    c_info = PMPI_Info_f2c(*info);
+
+    if (OMPI_COMM_IS_INTER(c_comm)) {
+        if (MPI_ROOT == c_root) {
+            int size = ompi_comm_size(c_comm);
+            c_sendtype = PMPI_Type_f2c(*sendtype);
+            OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(sendcounts, tmp_sendcounts, size);
+            OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(displs, tmp_displs, size);
+            OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+                return;
+            }
+        } else if (MPI_PROC_NULL != c_root) {
+            c_recvtype = PMPI_Type_f2c(*recvtype);
+            c_recvcount = (@COUNT_TYPE@) *recvcount;
+            OMPI_CFI_2_C(x2, c_recvcount, c_recvtype, c_recvdatatype, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+               OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+                return;
+            }
+        }
+    } else {
+        if (ompi_comm_rank(c_comm) == c_root) {
+            int size = ompi_comm_size(c_comm);
+            c_sendtype = PMPI_Type_f2c(*sendtype);
+            OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+                return;
+            }
+            OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(sendcounts, tmp_sendcounts, size);
+            OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(displs, tmp_displs, size);
+        }
+        if (OMPI_IS_FORTRAN_IN_PLACE(recvbuf)) {
+            recvbuf = MPI_IN_PLACE;
+        } else {
+            c_recvtype = PMPI_Type_f2c(*recvtype);
+            c_recvcount = (@COUNT_TYPE@) *recvcount;
+            OMPI_CFI_2_C(x2, c_recvcount, c_recvtype, c_recvdatatype, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+                return;
+            }
+        }
+    }
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          tmp_sendcounts,
+                          tmp_displs,
+                          c_sendtype,
+                          recvbuf,
+			  c_recvcount,
+			  c_recvdatatype,
+                          c_root,
+			  c_comm, 
+			  c_info,
+			  &c_request);
+    if (NULL != ierr) {
+        *ierr = OMPI_INT_2_FINT(c_ierr);
+    }
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_request);
+    }
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(sendcounts, tmp_sendcounts, c_request, c_ierr, idx);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP_NONBLOCKING(displs, tmp_displs, c_request, c_ierr, idx);
+
+     if ((c_recvdatatype != NULL ) && (c_recvdatatype != c_recvtype)){
+        ompi_datatype_destroy(&c_recvdatatype);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/scatterv_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/scatterv_ts.c.in
new file mode 100644
index 00000000000..23fad728bfe
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/scatterv_ts.c.in
@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2013 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID scatterv(BUFFER x1, COUNT_ARRAY sendcounts,
+                        DISP_ARRAY displs, DATATYPE sendtype,
+                        BUFFER_OUT x2, COUNT recvcount,
+                        DATATYPE recvtype, RANK root,
+                        COMM comm)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    int c_root = OMPI_FINT_2_INT(*root);
+    MPI_Datatype c_sendtype = NULL, c_recvtype = NULL, c_recvdatatype = NULL;
+    void *sendbuf = OMPI_CFI_BASE_ADDR(x1), *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_recvcount = 0;
+    @COUNT_TYPE@ *tmp_sendcounts = NULL;
+    @DISP_TYPE@ *tmp_displs = NULL;
+
+    if (OMPI_COMM_IS_INTER(c_comm)) {
+        if (MPI_ROOT == c_root) {
+            int size = ompi_comm_size(c_comm);
+            c_sendtype = PMPI_Type_f2c(*sendtype);
+            OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(sendcounts, tmp_sendcounts, size);
+            OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(displs, tmp_displs, size);
+            OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+                return;
+            }
+        } else if (MPI_PROC_NULL != c_root) {
+            c_recvtype = PMPI_Type_f2c(*recvtype);
+            c_recvcount = (@COUNT_TYPE@) *recvcount;
+            OMPI_CFI_2_C(x2, c_recvcount, c_recvtype, c_recvdatatype, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+                return;
+            }
+        }
+    } else {
+        if (ompi_comm_rank(c_comm) == c_root) {
+            int size = ompi_comm_size(c_comm);
+            c_sendtype = PMPI_Type_f2c(*sendtype);
+            OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+                return;
+            }
+            OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(sendcounts, tmp_sendcounts, size);
+            OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(displs, tmp_displs, size);
+        }
+        if (OMPI_IS_FORTRAN_IN_PLACE(recvbuf)) {
+            recvbuf = MPI_IN_PLACE;
+        } else {
+            c_recvtype = PMPI_Type_f2c(*recvtype);
+            c_recvcount = (@COUNT_TYPE@) *recvcount;
+            OMPI_CFI_2_C(x2, c_recvcount, c_recvtype, c_recvdatatype, c_ierr);
+            if (MPI_SUCCESS != c_ierr) {
+                if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+                OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME)
+                return;
+            }
+        }
+    }
+
+    c_ierr = @INNER_CALL@(sendbuf,
+                          tmp_sendcounts,
+                          tmp_displs,
+                          c_sendtype,
+                          recvbuf, c_recvcount, c_recvdatatype,
+                          c_root, c_comm);
+
+    if (c_recvdatatype != c_recvtype) {
+        ompi_datatype_destroy(&c_recvdatatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(sendcounts, tmp_sendcounts);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(displs, tmp_displs);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/send_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/send_f08.F90
deleted file mode 100644
index 25fecbffb7a..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/send_f08.F90
+++ /dev/null
@@ -1,28 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Send_f08(buf,count,datatype,dest,tag,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   use :: ompi_mpifh_bindings, only : ompi_send_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count, dest, tag
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_send_f(buf,count,datatype%MPI_VAL,dest,tag,comm%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Send_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/send_init_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/send_init_f08.F90
deleted file mode 100644
index bf7e9e1025f..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/send_init_f08.F90
+++ /dev/null
@@ -1,30 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Send_init_f08(buf,count,datatype,dest,tag,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_send_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count, dest, tag
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_send_init_f(buf,count,datatype%MPI_VAL,dest,tag,comm%MPI_VAL, &
-                         request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Send_init_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/send_init_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/send_init_ts.c.in
new file mode 100644
index 00000000000..edb96d73536
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/send_init_ts.c.in
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID send_init(BUFFER_ASYNC x, COUNT count, DATATYPE datatype,
+                         RANK dest, TAG tag, COMM comm,
+                         REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    MPI_Request c_req;
+    MPI_Comm c_comm = PMPI_Comm_f2c (*comm);
+
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(buf), c_count,
+                          c_datatype, OMPI_FINT_2_INT(*dest),
+                          OMPI_FINT_2_INT(*tag),
+                          c_comm, &c_req);
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_req);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/send_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/send_ts.c.in
new file mode 100644
index 00000000000..b35715f5d55
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/send_ts.c.in
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID send(BUFFER x, COUNT count, DATATYPE datatype, RANK dest,
+                    TAG tag, COMM comm)
+{
+    int c_ierr;
+
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(buf), c_count,
+                          c_datatype, OMPI_FINT_2_INT(*dest),
+                          OMPI_FINT_2_INT(*tag), c_comm);
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/sendrecv_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/sendrecv_f08.F90
deleted file mode 100644
index 9cbef5c81e8..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/sendrecv_f08.F90
+++ /dev/null
@@ -1,34 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Sendrecv_f08(sendbuf,sendcount,sendtype,dest,sendtag,recvbuf, &
-                        recvcount,recvtype,source,recvtag,comm,status,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Status
-   use :: ompi_mpifh_bindings, only : ompi_sendrecv_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, dest, sendtag, recvcount, source, recvtag
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Status), INTENT(OUT) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_sendrecv_f(sendbuf,sendcount,sendtype%MPI_VAL,dest,sendtag,recvbuf, &
-                        recvcount,recvtype%MPI_VAL,source,recvtag,comm%MPI_VAL,status,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Sendrecv_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/sendrecv_replace_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/sendrecv_replace_f08.F90
deleted file mode 100644
index 4ed700efa61..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/sendrecv_replace_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Sendrecv_replace_f08(buf,count,datatype,dest,sendtag,source, &
-                                    recvtag,comm,status,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Status
-   use :: ompi_mpifh_bindings, only : ompi_sendrecv_replace_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: buf
-   INTEGER, INTENT(IN) :: count, dest, sendtag, source, recvtag
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Status), INTENT(OUT) :: status
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_sendrecv_replace_f(buf,count,datatype%MPI_VAL,dest,sendtag,source, &
-                                recvtag,comm%MPI_VAL,status,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Sendrecv_replace_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/sendrecv_replace_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/sendrecv_replace_ts.c.in
new file mode 100644
index 00000000000..c7dac8f2944
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/sendrecv_replace_ts.c.in
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID sendrecv_replace(BUFFER_OUT x, COUNT count, DATATYPE datatype,
+                                RANK dest, TAG sendtag,
+                                RANK source, TAG recvtag,
+                                COMM comm, STATUS_OUT status)
+{
+    int c_ierr;
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    MPI_Comm c_comm = PMPI_Comm_f2c (*comm);
+
+    MPI_Status c_status;
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(buf),
+                          c_count,
+                          c_datatype,
+                          OMPI_FINT_2_INT(*dest),
+                          OMPI_FINT_2_INT(*sendtag),
+                          OMPI_FINT_2_INT(*source),
+                          OMPI_FINT_2_INT(*recvtag),
+                          c_comm, &c_status);
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr &&
+        !OMPI_IS_FORTRAN_STATUS_IGNORE(status)) {
+        PMPI_Status_c2f(&c_status, status);
+   }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/sendrecv_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/sendrecv_ts.c.in
new file mode 100644
index 00000000000..58619186a4c
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/sendrecv_ts.c.in
@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID sendrecv(BUFFER x1, COUNT sendcount, DATATYPE sendtype,
+                        RANK dest, TAG sendtag, BUFFER_OUT x2,
+                        COUNT recvcount, DATATYPE recvtype,
+                        RANK source, TAG recvtag, COMM comm,
+                        STATUS_OUT status)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c (*comm);
+
+    MPI_Datatype c_senddatatype, c_sendtype = PMPI_Type_f2c(*sendtype);
+    MPI_Datatype c_recvdatatype, c_recvtype = PMPI_Type_f2c(*recvtype);
+    MPI_Status c_status;
+    void *sendbuf = OMPI_CFI_BASE_ADDR(x1);
+    @COUNT_TYPE@ c_sendcount = (@COUNT_TYPE@) *sendcount;
+    void *recvbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_recvcount = (@COUNT_TYPE@) *recvcount;
+
+    OMPI_CFI_2_C(x1, c_sendcount, c_sendtype, c_senddatatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+    OMPI_CFI_2_C(x2, c_recvcount, c_recvtype, c_recvdatatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        if (c_senddatatype != c_sendtype) {
+            ompi_datatype_destroy(&c_senddatatype);
+        }
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(sendbuf), c_sendcount,
+                          c_senddatatype,
+                          OMPI_FINT_2_INT(*dest),
+                          OMPI_FINT_2_INT(*sendtag),
+                          OMPI_F2C_BOTTOM(recvbuf), c_recvcount,
+                          c_recvdatatype, OMPI_FINT_2_INT(*source),
+                          OMPI_FINT_2_INT(*recvtag),
+                          c_comm, &c_status);
+    if (c_senddatatype != c_sendtype) {
+        ompi_datatype_destroy(&c_senddatatype);
+    }
+    if (c_recvdatatype != c_recvtype) {
+        ompi_datatype_destroy(&c_recvdatatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr &&
+        !OMPI_IS_FORTRAN_STATUS_IGNORE(status)) {
+        PMPI_Status_c2f(&c_status, status);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/session_call_errhandler_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/session_call_errhandler_f08.F90
new file mode 100644
index 00000000000..bbd849dd624
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/session_call_errhandler_f08.F90
@@ -0,0 +1,26 @@
+! -*- f90 -*-
+!
+! Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
+!                         All rights reserved.
+! Copyright (c) 2018-2020 Research Organization for Information Science
+!                         and Technology (RIST).  All rights reserved.
+! Copyright (c) 2022      Triad National Security, LLC. All rights
+!                         reserved.
+! $COPYRIGHT$
+
+#include "mpi-f08-rename.h"
+
+subroutine MPI_Session_call_errhandler_f08(session,errorcode,ierror)
+   use :: mpi_f08_types, only : MPI_Session
+   use :: ompi_mpifh_bindings, only : ompi_session_call_errhandler_f
+   implicit none
+   TYPE(MPI_Session), INTENT(IN) :: session
+   INTEGER, INTENT(IN) :: errorcode
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+   integer :: c_ierror
+
+   call ompi_session_call_errhandler_f(session%MPI_VAL,errorcode,c_ierror)
+   if (present(ierror)) ierror = c_ierror
+
+end subroutine MPI_Session_call_errhandler_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/session_create_errhandler_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/session_create_errhandler_f08.F90
new file mode 100644
index 00000000000..497e1e19d57
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/session_create_errhandler_f08.F90
@@ -0,0 +1,32 @@
+! -*- f90 -*-
+!
+! Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
+!                         All rights reserved.
+! Copyright (c) 2018-2020 Research Organization for Information Science
+!                         and Technology (RIST).  All rights reserved.
+! Copyright (c) 2020-2022 Triad National Security, LLC. All rights
+!                         reserved.
+! $COPYRIGHT$
+
+#include "ompi/mpi/fortran/configure-fortran-output.h"
+
+#include "mpi-f08-rename.h"
+
+subroutine MPI_Session_create_errhandler_f08(session_errhandler_fn,errhandler,ierror)
+   use, intrinsic :: iso_c_binding, only: c_funptr, c_funloc
+   use :: mpi_f08_types, only : MPI_Errhandler
+   use :: mpi_f08_interfaces_callbacks, only : MPI_Session_errhandler_function
+   use :: ompi_mpifh_bindings, only : ompi_session_create_errhandler_f
+   implicit none
+   PROCEDURE(MPI_Session_errhandler_function) :: session_errhandler_fn
+   TYPE(MPI_Errhandler), INTENT(OUT) :: errhandler
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+   integer :: c_ierror
+   type(c_funptr) :: fn
+
+   fn = c_funloc(session_errhandler_fn)
+   call ompi_session_create_errhandler_f(fn,errhandler%MPI_VAL,c_ierror)
+   if (present(ierror)) ierror = c_ierror
+
+end subroutine MPI_Session_create_errhandler_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/session_finalize_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/session_finalize_f08.F90
new file mode 100644
index 00000000000..ef626258f0d
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/session_finalize_f08.F90
@@ -0,0 +1,26 @@
+! -*- f90 -*-
+!
+! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2009-2013 Los Alamos National Security, LLC.
+!                         All rights reserved.
+! Copyright (c) 2018      Research Organization for Information Science
+!                         and Technology (RIST).  All rights reserved.
+! Copyright (c) 2019      Triad National Security, LLC. All rights
+!                         reserved.
+! $COPYRIGHT$
+
+#include "mpi-f08-rename.h"
+
+subroutine MPI_Session_finalize_f08(session,ierror)
+   use :: mpi_f08_types, only : MPI_Session
+   use :: ompi_mpifh_bindings, only : ompi_session_finalize_f
+   implicit none
+   TYPE(MPI_Session), INTENT(OUT) :: session
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+   integer :: c_ierror
+
+   call ompi_session_finalize_f(session%MPI_VAL,c_ierror)
+   if (present(ierror)) ierror = c_ierror
+
+end subroutine MPI_Session_finalize_f08
+
diff --git a/ompi/mpi/fortran/use-mpi-f08/session_get_errhandler_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/session_get_errhandler_f08.F90
new file mode 100644
index 00000000000..8867d35db03
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/session_get_errhandler_f08.F90
@@ -0,0 +1,26 @@
+! -*- f90 -*-
+!
+! Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
+!                         All rights reserved.
+! Copyright (c) 2018-2020 Research Organization for Information Science
+!                         and Technology (RIST).  All rights reserved.
+! Copyright (c) 2022      Triad National Security, LLC. All rights
+!                         reserved.
+! $COPYRIGHT$
+
+#include "mpi-f08-rename.h"
+
+subroutine MPI_Session_get_errhandler_f08(session,errhandler,ierror)
+   use :: mpi_f08_types, only : MPI_Session, MPI_Errhandler
+   use :: ompi_mpifh_bindings, only : ompi_session_get_errhandler_f
+   implicit none
+   TYPE(MPI_Session), INTENT(IN) :: session
+   TYPE(MPI_Errhandler), INTENT(OUT) :: errhandler
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+   integer :: c_ierror
+
+   call ompi_session_get_errhandler_f(session%MPI_VAL,errhandler%MPI_VAL,c_ierror)
+   if (present(ierror)) ierror = c_ierror
+
+end subroutine MPI_Session_get_errhandler_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/session_get_info_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/session_get_info_f08.F90
new file mode 100644
index 00000000000..a1abe6dfdb5
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/session_get_info_f08.F90
@@ -0,0 +1,27 @@
+! -*- f90 -*-
+!
+! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2009-2013 Los Alamos National Security, LLC.
+!                         All rights reserved.
+! Copyright (c) 2018      Research Organization for Information Science
+!                         and Technology (RIST).  All rights reserved.
+! Copyright (c) 2019-2022 Triad National Security, LLC. All rights
+!                         reserved.
+! $COPYRIGHT$
+
+#include "mpi-f08-rename.h"
+
+subroutine MPI_Session_get_info_f08(session, info, ierror)
+   use :: mpi_f08_types, only : MPI_Session, MPI_Info
+   use :: ompi_mpifh_bindings, only : ompi_session_get_info_f
+   implicit none
+   TYPE(MPI_Session), INTENT(IN) :: session
+   TYPE(MPI_Info), INTENT(OUT) :: info
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+   integer :: c_ierror
+
+   call ompi_session_get_info_f(session%MPI_VAL, info%MPI_VAL, c_ierror)
+   if (present(ierror)) ierror = c_ierror
+
+end subroutine MPI_Session_get_info_f08
+
diff --git a/ompi/mpi/fortran/use-mpi-f08/session_get_nth_pset_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/session_get_nth_pset_f08.F90
new file mode 100644
index 00000000000..1c355978abc
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/session_get_nth_pset_f08.F90
@@ -0,0 +1,32 @@
+! -*- f90 -*-
+!
+! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2009-2013 Los Alamos National Security, LLC.
+!                         All rights reserved.
+! Copyright (c) 2018      Research Organization for Information Science
+!                         and Technology (RIST).  All rights reserved.
+! Copyright (c) 2019-2022 Triad National Security, LLC. All rights
+!                         reserved.
+! $COPYRIGHT$
+
+#include "mpi-f08-rename.h"
+
+subroutine MPI_Session_get_nth_pset_f08(session, info, n, pset_len, pset_name, ierror)
+   use :: mpi_f08_types, only : MPI_Session, MPI_Info, MPI_INFO_NULL
+   use :: ompi_mpifh_bindings, only : ompi_session_get_nth_pset_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
+   implicit none
+   TYPE(MPI_Session), INTENT(IN) :: session
+   TYPE(MPI_Info), INTENT(IN) :: info
+   INTEGER, OPTIONAL, INTENT(IN) :: n
+   INTEGER, OPTIONAL, INTENT(INOUT) :: pset_len
+   CHARACTER(LEN=*), INTENT(OUT) :: pset_name
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+   integer :: c_ierror
+ 
+   call ompi_session_get_nth_pset_f(session%MPI_VAL, MPI_INFO_NULL%MPI_VAL, n,     &
+                                    pset_len, pset_name, c_ierror,                 &
+                                    len(pset_name,KIND=C_INT))
+   if (present(ierror)) ierror = c_ierror
+
+end subroutine MPI_Session_get_nth_pset_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/session_get_num_psets_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/session_get_num_psets_f08.F90
new file mode 100644
index 00000000000..51a6e4a0a75
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/session_get_num_psets_f08.F90
@@ -0,0 +1,27 @@
+! -*- f90 -*-
+!
+! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2009-2013 Los Alamos National Security, LLC.
+!                         All rights reserved.
+! Copyright (c) 2018      Research Organization for Information Science
+!                         and Technology (RIST).  All rights reserved.
+! Copyright (c) 2019-2022 Triad National Security, LLC. All rights
+!                         reserved.
+! $COPYRIGHT$
+
+#include "mpi-f08-rename.h"
+
+subroutine MPI_Session_get_num_psets_f08(session, info, npset_names, ierror)
+   use :: mpi_f08_types, only : MPI_Session, MPI_Info, MPI_INFO_NULL
+   use :: ompi_mpifh_bindings, only : ompi_session_get_num_psets_f
+   implicit none
+   TYPE(MPI_Session), INTENT(IN) :: session
+   TYPE(MPI_Info), INTENT(IN) :: info
+   INTEGER, OPTIONAL, INTENT(OUT) :: npset_names
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+   integer :: c_ierror
+
+   call ompi_session_get_num_psets_f(session%MPI_VAL, MPI_INFO_NULL%MPI_VAL, npset_names, c_ierror)
+   if (present(ierror)) ierror = c_ierror
+
+end subroutine MPI_Session_get_num_psets_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/session_get_pset_info_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/session_get_pset_info_f08.F90
new file mode 100644
index 00000000000..2aebfee9a14
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/session_get_pset_info_f08.F90
@@ -0,0 +1,30 @@
+! -*- f90 -*-
+!
+! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2009-2013 Los Alamos National Security, LLC.
+!                         All rights reserved.
+! Copyright (c) 2018      Research Organization for Information Science
+!                         and Technology (RIST).  All rights reserved.
+! Copyright (c) 2019-2022 Triad National Security, LLC. All rights
+!                         reserved.
+! $COPYRIGHT$
+
+#include "mpi-f08-rename.h"
+
+subroutine MPI_Session_get_pset_info_f08(session, pset_name, info, ierror)
+   use :: mpi_f08_types, only : MPI_Session, MPI_Info
+   use :: ompi_mpifh_bindings, only : ompi_session_get_pset_info_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
+   implicit none
+   TYPE(MPI_Session), INTENT(IN) :: session
+   CHARACTER(LEN=*), INTENT(IN) :: pset_name
+   TYPE(MPI_Info), INTENT(OUT) :: info
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+   integer :: c_ierror
+
+   call ompi_session_get_pset_info_f(session%MPI_VAL, pset_name, info%MPI_VAL, c_ierror, &
+                                     len(pset_name,KIND=C_INT))
+   if (present(ierror)) ierror = c_ierror
+
+end subroutine MPI_Session_get_pset_info_f08
+
diff --git a/ompi/mpi/fortran/use-mpi-f08/session_init_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/session_init_f08.F90
new file mode 100644
index 00000000000..b9eee1338b1
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/session_init_f08.F90
@@ -0,0 +1,30 @@
+! -*- f90 -*-
+!
+! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2009-2013 Los Alamos National Security, LLC.
+!                         All rights reserved.
+! Copyright (c) 2018      Research Organization for Information Science
+!                         and Technology (RIST).  All rights reserved.
+! Copyright (c) 2019-2021 Triad National Security, LLC. All rights
+!                         reserved.
+! $COPYRIGHT$
+
+#include "ompi/mpi/fortran/configure-fortran-output.h"
+
+#include "mpi-f08-rename.h"
+
+subroutine MPI_Session_init_f08(info,errhandler,session,ierror)
+   use :: mpi_f08_types, only : MPI_Session, MPI_Info, MPI_Errhandler
+   use :: ompi_mpifh_bindings, only : ompi_session_init_f
+   implicit none
+   TYPE(MPI_Info), INTENT(IN) :: info
+   TYPE(MPI_Errhandler), INTENT(OUT) :: errhandler
+   TYPE(MPI_Session), INTENT(OUT) :: session
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+   integer :: c_ierror
+
+   call ompi_session_init_f(info%MPI_VAL,errhandler%MPI_VAL,session%MPI_VAL,c_ierror)
+   if (present(ierror)) ierror = c_ierror
+
+end subroutine MPI_Session_init_f08
+
diff --git a/ompi/mpi/fortran/use-mpi-f08/session_set_errhandler_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/session_set_errhandler_f08.F90
new file mode 100644
index 00000000000..68864a070ae
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/session_set_errhandler_f08.F90
@@ -0,0 +1,26 @@
+! -*- f90 -*-
+!
+! Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
+!               All Rights reserved.
+! Copyright (c) 2018-2020 Research Organization for Information Science
+!                         and Technology (RIST).  All rights reserved.
+! Copyright (c) 2019-2022 Triad National Security, LLC. All rights
+!                         reserved.
+! $COPYRIGHT$
+
+#include "mpi-f08-rename.h"
+
+subroutine MPI_Session_set_errhandler_f08(session,errhandler,ierror)
+   use :: mpi_f08_types, only : MPI_Session, MPI_Errhandler
+   use :: ompi_mpifh_bindings, only : ompi_session_set_errhandler_f
+   implicit none
+   TYPE(MPI_Session), INTENT(IN) :: session
+   TYPE(MPI_Errhandler), INTENT(IN) :: errhandler
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+   integer :: c_ierror
+
+   call ompi_session_set_errhandler_f(session%MPI_VAL,errhandler%MPI_VAL,c_ierror)
+   if (present(ierror)) ierror = c_ierror
+
+end subroutine MPI_Session_set_errhandler_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/ssend_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/ssend_f08.F90
deleted file mode 100644
index b4221f659bb..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/ssend_f08.F90
+++ /dev/null
@@ -1,28 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Ssend_f08(buf,count,datatype,dest,tag,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   use :: ompi_mpifh_bindings, only : ompi_ssend_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count, dest, tag
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_ssend_f(buf,count,datatype%MPI_VAL,dest,tag,comm%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Ssend_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/ssend_init_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/ssend_init_f08.F90
deleted file mode 100644
index f2ea586fc26..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/ssend_init_f08.F90
+++ /dev/null
@@ -1,30 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Ssend_init_f08(buf,count,datatype,dest,tag,comm,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   use :: ompi_mpifh_bindings, only : ompi_ssend_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buf
-   INTEGER, INTENT(IN) :: count, dest, tag
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_ssend_init_f(buf,count,datatype%MPI_VAL,dest,tag,comm%MPI_VAL, &
-                          request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Ssend_init_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/ssend_init_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/ssend_init_ts.c.in
new file mode 100644
index 00000000000..e5a58d0f8f5
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/ssend_init_ts.c.in
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID ssend_init(BUFFER_ASYNC x, COUNT count, DATATYPE datatype,
+                          RANK dest, TAG tag,
+                          COMM comm, REQUEST_OUT request)
+{
+    int c_ierr;
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    MPI_Request c_req;
+    MPI_Comm c_comm = PMPI_Comm_f2c (*comm);
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(buf), c_count,
+                          c_datatype, OMPI_FINT_2_INT(*dest),
+                          OMPI_FINT_2_INT(*tag),
+                          c_comm, &c_req);
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *request = PMPI_Request_c2f(c_req);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/ssend_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/ssend_ts.c.in
new file mode 100644
index 00000000000..ea82a0f75c0
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/ssend_ts.c.in
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID ssend(BUFFER x, COUNT count, DATATYPE datatype,
+                     RANK dest, TAG tag,
+                     COMM comm)
+{
+    int c_ierr;
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    MPI_Comm c_comm = PMPI_Comm_f2c (*comm);
+
+    void *buf = OMPI_CFI_BASE_ADDR(x);
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@) *count;
+
+    OMPI_CFI_2_C(x, c_count, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(OMPI_F2C_BOTTOM(buf), c_count,
+                          c_datatype, OMPI_FINT_2_INT(*dest),
+                          OMPI_FINT_2_INT(*tag), c_comm);
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/status_set_elements_c.in b/ompi/mpi/fortran/use-mpi-f08/status_set_elements_c.in
new file mode 100644
index 00000000000..0001700d391
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/status_set_elements_c.in
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID status_set_elements(STATUS status, DATATYPE datatype,
+			           COUNT count)
+{
+    int c_ierr;
+    MPI_Datatype c_type = PMPI_Type_f2c(*datatype);
+    MPI_Status c_status;
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@)*count;
+
+    /* This seems silly, but someone will do it */
+
+    if (OMPI_IS_FORTRAN_STATUS_IGNORE(status)) {
+        c_ierr = MPI_SUCCESS;
+    } else {
+        PMPI_Status_f2c( status, &c_status );
+
+        c_ierr = @INNER_CALL@(&c_status, c_type, c_count);
+
+        /* If datatype is really being set, then that needs to be
+           converted.... */
+        if (MPI_SUCCESS == c_ierr) {
+            PMPI_Status_c2f(&c_status, status);
+        }
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/status_set_elements_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/status_set_elements_f08.F90
deleted file mode 100644
index 25f06ef3074..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/status_set_elements_f08.F90
+++ /dev/null
@@ -1,25 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Status_set_elements_f08(status,datatype,count,ierror)
-   use :: mpi_f08_types, only : MPI_Status, MPI_Datatype
-   use :: ompi_mpifh_bindings, only : ompi_status_set_elements_f
-   implicit none
-   TYPE(MPI_Status), INTENT(INOUT) :: status
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER, INTENT(IN) :: count
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_status_set_elements_f(status,datatype%MPI_VAL,count,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Status_set_elements_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/testany.c.in b/ompi/mpi/fortran/use-mpi-f08/testany.c.in
new file mode 100644
index 00000000000..7ae3bcb343b
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/testany.c.in
@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID testany(INT count, REQUEST_ARRAY array_of_requests:count, INT indx,
+                       LOGICAL_OUT flag, STATUS_OUT status)
+{
+    MPI_Request *c_req;
+    MPI_Status c_status;
+    int i, c_ierr, c_flag;
+    int tmp_indx;
+
+    /* Shortcut to avoid malloc(0) if *count==0.  We're intentionally
+       skipping other parameter error checks. */
+    if (OPAL_UNLIKELY(0 == OMPI_FINT_2_INT(*count))) {
+        *flag = OMPI_FORTRAN_VALUE_TRUE;
+        *indx = OMPI_INT_2_FINT(MPI_UNDEFINED);
+        PMPI_Status_c2f(&ompi_status_empty, status);
+        *ierr = OMPI_INT_2_FINT(MPI_SUCCESS);
+        return;
+    }
+
+    c_req = (MPI_Request *) malloc(OMPI_FINT_2_INT(*count) * sizeof(MPI_Request));
+    if (c_req == NULL) {
+        c_ierr = OMPI_ERRHANDLER_NOHANDLE_INVOKE(
+                                        MPI_ERR_NO_MEM,
+                                        FUNC_NAME);
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        return;
+    }
+
+    for (i = 0; i < OMPI_FINT_2_INT(*count); ++i) {
+        c_req[i] = PMPI_Request_f2c(array_of_requests[i]);
+    }
+
+    c_ierr = PMPI_Testany(OMPI_FINT_2_INT(*count), c_req,
+                          &tmp_indx,
+                          &c_flag,
+                         &c_status);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+
+        /* Increment index by one for fortran conventions.  Note that
+           all Fortran compilers have FALSE==0; we just need to check
+           for any nonzero value (because TRUE is not always 1) */
+
+        if (c_flag &&
+            MPI_UNDEFINED != tmp_indx) {
+            array_of_requests[tmp_indx] =
+                c_req[tmp_indx]->req_f_to_c_index;
+            *indx = OMPI_INT_2_FINT(tmp_indx+1);
+            *flag = OMPI_INT_2_FINT(c_flag);
+        }
+        if (!OMPI_IS_FORTRAN_STATUS_IGNORE(status)) {
+            PMPI_Status_c2f(&c_status, status);
+        }
+    }
+    free(c_req);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/type_contiguous.c.in b/ompi/mpi/fortran/use-mpi-f08/type_contiguous.c.in
new file mode 100644
index 00000000000..2209ffb5773
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/type_contiguous.c.in
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID type_contiguous(COUNT count, DATATYPE oldtype,
+			       DATATYPE_OUT newtype)
+{
+    int c_ierr;
+    MPI_Datatype c_old = PMPI_Type_f2c(*oldtype);
+    MPI_Datatype c_new;
+
+    c_ierr = @INNER_CALL@(OMPI_FINT_2_INT(*count), c_old, &c_new);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *newtype = PMPI_Type_c2f(c_new);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/type_contiguous_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/type_contiguous_f08.F90
deleted file mode 100644
index cd00a5a0fd4..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/type_contiguous_f08.F90
+++ /dev/null
@@ -1,25 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Type_contiguous_f08(count,oldtype,newtype,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype
-   use :: ompi_mpifh_bindings, only : ompi_type_contiguous_f
-   implicit none
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: oldtype
-   TYPE(MPI_Datatype), INTENT(OUT) :: newtype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_type_contiguous_f(count,oldtype%MPI_VAL,newtype%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Type_contiguous_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/type_create_darray.c.in b/ompi/mpi/fortran/use-mpi-f08/type_create_darray.c.in
new file mode 100644
index 00000000000..7e7b39f34c5
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/type_create_darray.c.in
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID  type_create_darray(INT size, RANK rank,
+			      INT ndims, COUNT_ARRAY gsize_array,
+			      INT_ARRAY distrib_array, INT_ARRAY darg_array,
+			      INT_ARRAY psize_array, INT order,
+			      DATATYPE oldtype, DATATYPE_OUT newtype)
+{
+    int c_ierr;
+    MPI_Datatype c_old = PMPI_Type_f2c(*oldtype);
+    MPI_Datatype c_new;
+    @COUNT_TYPE@ *tmp_gsize_array = NULL;
+    int *tmp_distrib_array = NULL;
+    int *tmp_darg_array = NULL;
+    int *tmp_psize_array = NULL;
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(gsize_array, tmp_gsize_array, *ndims);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(distrib_array, tmp_distrib_array, *ndims);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(darg_array, tmp_darg_array, *ndims);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(psize_array, tmp_psize_array, *ndims);
+
+    c_ierr = @INNER_CALL@(OMPI_FINT_2_INT(*size),
+                                    OMPI_FINT_2_INT(*rank),
+                                    OMPI_FINT_2_INT(*ndims),
+                                    tmp_gsize_array,
+                                    tmp_distrib_array,
+                                    tmp_darg_array,
+                                    tmp_psize_array,
+                                    OMPI_FINT_2_INT(*order), c_old, &c_new);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(gsize_array, tmp_gsize_array);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(distrib_array, tmp_distrib_array);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(darg_array, tmp_darg_array);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(psize_array, tmp_psize_array);
+
+    if (MPI_SUCCESS == c_ierr) {
+      *newtype = PMPI_Type_c2f(c_new);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/type_create_darray_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/type_create_darray_f08.F90
deleted file mode 100644
index 3f55f301c41..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/type_create_darray_f08.F90
+++ /dev/null
@@ -1,30 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Type_create_darray_f08(size,rank,ndims,&
-                    array_of_gsizes,array_of_distribs,array_of_dargs,array_of_psizes,&
-                    order,oldtype,newtype,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype
-   use :: ompi_mpifh_bindings, only : ompi_type_create_darray_f
-   implicit none
-   INTEGER, INTENT(IN) :: size, rank, ndims, order
-   INTEGER, INTENT(IN) :: array_of_gsizes(*), array_of_distribs(*), array_of_dargs(*), array_of_psizes(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: oldtype
-   TYPE(MPI_Datatype), INTENT(OUT) :: newtype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_type_create_darray_f(size,rank,ndims, &
-                    array_of_gsizes,array_of_distribs,array_of_dargs,array_of_psizes, &
-                    order,oldtype%MPI_VAL,newtype%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Type_create_darray_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/type_create_hindexed.c.in b/ompi/mpi/fortran/use-mpi-f08/type_create_hindexed.c.in
new file mode 100644
index 00000000000..71ee257b4ee
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/type_create_hindexed.c.in
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID type_create_hindexed(COUNT count,
+				    COUNT_ARRAY array_of_blocklengths,
+				    AINT_COUNT_ARRAY array_of_displacements,
+				    DATATYPE oldtype, 
+				    DATATYPE_OUT newtype)
+{
+    int c_ierr;
+    MPI_Datatype c_old = PMPI_Type_f2c(*oldtype);
+    MPI_Datatype c_new;
+    @COUNT_TYPE@ *tmp_array_of_blocklengths = NULL;
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(array_of_blocklengths, tmp_array_of_blocklengths, *count);
+
+    c_ierr = @INNER_CALL@(OMPI_FINT_2_INT(*count),
+                                tmp_array_of_blocklengths,
+                                array_of_displacements, c_old,
+                                &c_new);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *newtype = PMPI_Type_c2f(c_new);
+    }
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(array_of_blocklengths, tmp_array_of_blocklengths);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/type_create_hindexed_block.c.in b/ompi/mpi/fortran/use-mpi-f08/type_create_hindexed_block.c.in
new file mode 100644
index 00000000000..3f58e681490
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/type_create_hindexed_block.c.in
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2012      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2012      Inria.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID type_create_hindexed_block(COUNT count, COUNT blocklength,
+				          AINT_COUNT_ARRAY array_of_displacements,
+				          DATATYPE oldtype, DATATYPE_OUT newtype)
+{
+    int c_ierr;
+    MPI_Datatype c_old = PMPI_Type_f2c(*oldtype);
+    MPI_Datatype c_new;
+
+    c_ierr = @INNER_CALL@(OMPI_FINT_2_INT(*count),
+			OMPI_FINT_2_INT(*blocklength),
+			array_of_displacements,
+                        c_old, &c_new);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *newtype = PMPI_Type_c2f(c_new);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/type_create_hindexed_block_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/type_create_hindexed_block_f08.F90
deleted file mode 100644
index f85fea57629..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/type_create_hindexed_block_f08.F90
+++ /dev/null
@@ -1,30 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2012      The University of Tennessee and The University
-!                         of Tennessee Research Foundation.  All rights
-!                         reserved.
-! Copyright (c) 2012      Inria.  All rights reserved.
-! Copyright (c) 2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Type_create_hindexed_block_f08(count,blocklength, &
-                           array_of_displacements,oldtype,newtype,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_ADDRESS_KIND
-   use :: ompi_mpifh_bindings, only : ompi_type_create_hindexed_block_f
-   implicit none
-   INTEGER, INTENT(IN) :: count, blocklength
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: array_of_displacements(count)
-   TYPE(MPI_Datatype), INTENT(IN) :: oldtype
-   TYPE(MPI_Datatype), INTENT(OUT) :: newtype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_type_create_hindexed_block_f(count,blocklength,array_of_displacements, &
-                                         oldtype%MPI_VAL,newtype%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Type_create_hindexed_block_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/type_create_hindexed_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/type_create_hindexed_f08.F90
deleted file mode 100644
index 66443dd81cd..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/type_create_hindexed_f08.F90
+++ /dev/null
@@ -1,30 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Type_create_hindexed_f08(count,array_of_blocklengths, &
-                                        array_of_displacements,oldtype,newtype,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_ADDRESS_KIND
-   use :: ompi_mpifh_bindings, only : ompi_type_create_hindexed_f
-   implicit none
-   INTEGER, INTENT(IN) :: count
-   INTEGER, INTENT(IN) :: array_of_blocklengths(count)
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: array_of_displacements(count)
-   TYPE(MPI_Datatype), INTENT(IN) :: oldtype
-   TYPE(MPI_Datatype), INTENT(OUT) :: newtype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_type_create_hindexed_f(count,array_of_blocklengths, &
-                                    array_of_displacements,oldtype%MPI_VAL, &
-                                    newtype%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Type_create_hindexed_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/type_create_hvector.c.in b/ompi/mpi/fortran/use-mpi-f08/type_create_hvector.c.in
new file mode 100644
index 00000000000..3588a96b550
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/type_create_hvector.c.in
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID type_create_hvector(COUNT count, COUNT blocklength,
+			           AINT_COUNT stride, DATATYPE oldtype,
+			           DATATYPE_OUT newtype)
+{
+    int c_ierr;
+    MPI_Datatype c_old = PMPI_Type_f2c(*oldtype);
+    MPI_Datatype c_new;
+    @COUNT_TYPE@ c_blocklength = (@COUNT_TYPE@)*blocklength;
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@)*count;
+
+    c_ierr = @INNER_CALL@(c_count,
+                              c_blocklength,
+                              *stride,
+                              c_old, &c_new);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *newtype = PMPI_Type_c2f(c_new);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/type_create_hvector_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/type_create_hvector_f08.F90
deleted file mode 100644
index 87b93a7e675..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/type_create_hvector_f08.F90
+++ /dev/null
@@ -1,27 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Type_create_hvector_f08(count,blocklength,stride,oldtype,newtype,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_ADDRESS_KIND
-   use :: ompi_mpifh_bindings, only : ompi_type_create_hvector_f
-   implicit none
-   INTEGER, INTENT(IN) :: count, blocklength
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: stride
-   TYPE(MPI_Datatype), INTENT(IN) :: oldtype
-   TYPE(MPI_Datatype), INTENT(OUT) :: newtype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_type_create_hvector_f(count,blocklength,stride, &
-                                   oldtype%MPI_VAL,newtype%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Type_create_hvector_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/type_create_indexed_block.c.in b/ompi/mpi/fortran/use-mpi-f08/type_create_indexed_block.c.in
new file mode 100644
index 00000000000..a9c8e18b274
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/type_create_indexed_block.c.in
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID type_create_indexed_block(COUNT count, COUNT blocklength,
+				         COUNT_ARRAY array_of_displacements,
+				         DATATYPE oldtype, DATATYPE_OUT newtype)
+{
+    int c_ierr;
+    MPI_Datatype c_old = PMPI_Type_f2c(*oldtype);
+    MPI_Datatype c_new;
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@)*count;
+    @COUNT_TYPE@ c_blocklength = (@COUNT_TYPE@)*blocklength;
+    @COUNT_TYPE@ *tmp_array_of_displacements = NULL;
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(array_of_displacements, tmp_array_of_displacements, *count);
+
+    c_ierr = @INNER_CALL@(c_count,
+			c_blocklength,
+			tmp_array_of_displacements,
+                        c_old, &c_new);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *newtype = PMPI_Type_c2f(c_new);
+    }
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(array_of_displacements, tmp_array_of_displacements);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/type_create_indexed_block_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/type_create_indexed_block_f08.F90
deleted file mode 100644
index 956acbd0cc3..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/type_create_indexed_block_f08.F90
+++ /dev/null
@@ -1,28 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Type_create_indexed_block_f08(count,blocklength, &
-                           array_of_displacements,oldtype,newtype,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype
-   use :: ompi_mpifh_bindings, only : ompi_type_create_indexed_block_f
-   implicit none
-   INTEGER, INTENT(IN) :: count, blocklength
-   INTEGER, INTENT(IN) :: array_of_displacements(count)
-   TYPE(MPI_Datatype), INTENT(IN) :: oldtype
-   TYPE(MPI_Datatype), INTENT(OUT) :: newtype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_type_create_indexed_block_f(count,blocklength,array_of_displacements, &
-                                         oldtype%MPI_VAL,newtype%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Type_create_indexed_block_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/type_create_struct.c.in b/ompi/mpi/fortran/use-mpi-f08/type_create_struct.c.in
new file mode 100644
index 00000000000..95774d33cba
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/type_create_struct.c.in
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID  type_create_struct(COUNT count,
+   			           COUNT_ARRAY array_of_block_lengths,
+			           AINT_COUNT_ARRAY array_of_displacements,
+   			           DATATYPE_ARRAY array_of_types, 
+			           DATATYPE_OUT newtype)
+{
+    MPI_Datatype c_new;
+    MPI_Datatype *c_type_old_array = NULL;
+    int i, c_ierr;
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@)*count;
+    @COUNT_TYPE@ *tmp_array_of_block_lengths = NULL;
+
+    c_type_old_array = (MPI_Datatype *) malloc(*count * sizeof(MPI_Datatype));
+    if (NULL == c_type_old_array) {
+        c_ierr = OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_NO_MEM,
+                                        FUNC_NAME);
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        return;
+    }
+
+    for (i = 0; i < *count; i++) {
+        c_type_old_array[i] = PMPI_Type_f2c(array_of_types[i]);
+    }
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(array_of_block_lengths, tmp_array_of_block_lengths, *count);
+
+    c_ierr = @INNER_CALL@(c_count,
+			   tmp_array_of_block_lengths,
+                           array_of_displacements,
+                           c_type_old_array, &c_new);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(array_of_block_lengths, tmp_array_of_block_lengths);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *newtype = PMPI_Type_c2f(c_new);
+    }
+
+    free(c_type_old_array);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/type_create_struct_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/type_create_struct_f08.F90
deleted file mode 100644
index 92a1de45dbe..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/type_create_struct_f08.F90
+++ /dev/null
@@ -1,29 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Type_create_struct_f08(count,array_of_blocklengths,array_of_displacements, &
-                                      array_of_types,newtype,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_ADDRESS_KIND
-   use :: ompi_mpifh_bindings, only : ompi_type_create_struct_f
-   implicit none
-   INTEGER, INTENT(IN) :: count
-   INTEGER, INTENT(IN) :: array_of_blocklengths(count)
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: array_of_displacements(count)
-   TYPE(MPI_Datatype), INTENT(IN) :: array_of_types(count)
-   TYPE(MPI_Datatype), INTENT(OUT) :: newtype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_type_create_struct_f(count,array_of_blocklengths,array_of_displacements, &
-                                  array_of_types(:)%MPI_VAL,newtype%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Type_create_struct_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/type_create_subarray.c.in b/ompi/mpi/fortran/use-mpi-f08/type_create_subarray.c.in
new file mode 100644
index 00000000000..eed01629ad8
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/type_create_subarray.c.in
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID type_create_subarray(INT ndims,  COUNT_ARRAY size_array,
+				    COUNT_ARRAY subsize_array,
+				    COUNT_ARRAY start_array, INT order,
+				    DATATYPE oldtype, DATATYPE_OUT newtype)
+{
+    int c_ierr;
+    MPI_Datatype c_old;
+    MPI_Datatype c_new;
+    @COUNT_TYPE@ *tmp_size_array = NULL;
+    @COUNT_TYPE@ *tmp_subsize_array = NULL;
+    @COUNT_TYPE@ *tmp_start_array = NULL;
+
+    c_old = PMPI_Type_f2c(*oldtype);
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(size_array, tmp_size_array, *ndims);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(subsize_array, tmp_subsize_array, *ndims);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(start_array, tmp_start_array, *ndims);
+
+    c_ierr = @INNER_CALL@(OMPI_FINT_2_INT(*ndims),
+                          tmp_size_array,
+                          tmp_subsize_array,
+                          tmp_start_array,
+                          *order, c_old, &c_new);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *newtype = PMPI_Type_c2f(c_new);
+    }
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(size_array, tmp_size_array);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(subsize_array, tmp_subsize_array);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(start_array, tmp_start_array);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/type_create_subarray_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/type_create_subarray_f08.F90
deleted file mode 100644
index a870231b922..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/type_create_subarray_f08.F90
+++ /dev/null
@@ -1,29 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Type_create_subarray_f08(ndims,array_of_sizes,array_of_subsizes, &
-                                        array_of_starts,order,oldtype,newtype,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype
-   use :: ompi_mpifh_bindings, only : ompi_type_create_subarray_f
-   implicit none
-   INTEGER, INTENT(IN) :: ndims, order
-   INTEGER, INTENT(IN) :: array_of_sizes(*), array_of_subsizes(*), array_of_starts(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: oldtype
-   TYPE(MPI_Datatype), INTENT(OUT) :: newtype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_type_create_subarray_f(ndims,array_of_sizes, &
-                                    array_of_subsizes,array_of_starts, &
-                                    order,oldtype%MPI_VAL,newtype%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Type_create_subarray_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/type_get_contents_f08_c.F90 b/ompi/mpi/fortran/use-mpi-f08/type_get_contents_f08_c.F90
new file mode 100644
index 00000000000..5b089fb9ca9
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/type_get_contents_f08_c.F90
@@ -0,0 +1,33 @@
+! -*- f90 -*-
+!
+! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
+!                         All rights reserved.
+! Copyright (c) 2018-2020 Research Organization for Information Science
+!                         and Technology (RIST).  All rights reserved.
+! $COPYRIGHT$
+
+#include "mpi-f08-rename.h"
+
+subroutine MPI_Type_get_contents_f08_c(datatype, max_integers, max_addresses, max_large_counts, &
+                                       max_datatypes, array_of_integers, array_of_addresses, &
+                                       array_of_large_counts, array_of_datatypes, ierror)
+   use :: mpi_f08_types, only : MPI_Datatype, MPI_ADDRESS_KIND, MPI_COUNT_KIND
+   use :: ompi_mpifh_bindings, only : ompi_type_get_contents_f_c
+   implicit none
+   TYPE(MPI_Datatype), INTENT(IN) :: datatype
+   INTEGER(KIND=MPI_COUNT_KIND), INTENT(IN) :: max_integers, max_addresses, &
+                                               max_large_counts, max_datatypes
+   INTEGER, INTENT(OUT) :: array_of_integers(max_integers)
+   INTEGER(MPI_ADDRESS_KIND), INTENT(OUT) :: array_of_addresses(max_addresses)
+   INTEGER(KIND=MPI_COUNT_KIND), INTENT(OUT) :: array_of_large_counts(max_large_counts)
+   TYPE(MPI_Datatype), INTENT(OUT) :: array_of_datatypes(max_datatypes)
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+   integer :: c_ierror
+
+   call ompi_type_get_contents_f_c(datatype%MPI_VAL,max_integers,max_addresses, &
+                    max_large_counts, max_datatypes,array_of_integers,array_of_addresses, &
+                    array_of_large_counts, array_of_datatypes(:)%MPI_VAL,c_ierror)
+   if (present(ierror)) ierror = c_ierror
+
+end subroutine MPI_Type_get_contents_f08_c
diff --git a/ompi/mpi/fortran/use-mpi-f08/type_get_envelope_f08_c.F90 b/ompi/mpi/fortran/use-mpi-f08/type_get_envelope_f08_c.F90
new file mode 100644
index 00000000000..cefa541ba8b
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/type_get_envelope_f08_c.F90
@@ -0,0 +1,28 @@
+! -*- f90 -*-
+!
+! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
+!                         All rights reserved.
+! Copyright (c) 2018-2020 Research Organization for Information Science
+!                         and Technology (RIST).  All rights reserved.
+! $COPYRIGHT$
+
+#include "mpi-f08-rename.h"
+
+subroutine MPI_Type_get_envelope_f08_c(datatype,num_integers,num_addresses, &
+                                     num_large_counts, num_datatypes,combiner,ierror)
+   use :: mpi_f08_types, only : MPI_Datatype, MPI_ADDRESS_KIND, MPI_COUNT_KIND
+   use :: ompi_mpifh_bindings, only : ompi_type_get_envelope_f_c
+   implicit none
+   TYPE(MPI_Datatype), INTENT(IN) :: datatype
+   INTEGER(KIND=MPI_COUNT_KIND), INTENT(OUT) :: num_integers, num_addresses, num_large_counts, &
+                                                num_datatypes, combiner
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+   integer :: c_ierror
+
+   call ompi_type_get_envelope_f_c(datatype%MPI_VAL,num_integers,num_addresses, &
+                                 num_large_counts, num_datatypes,combiner,c_ierror)
+   if (present(ierror)) ierror = c_ierror
+
+end subroutine MPI_Type_get_envelope_f08_c
+
diff --git a/ompi/mpi/fortran/use-mpi-f08/type_get_name_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/type_get_name_f08.F90
index 5828adca60a..3d3341fda3f 100644
--- a/ompi/mpi/fortran/use-mpi-f08/type_get_name_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/type_get_name_f08.F90
@@ -12,6 +12,7 @@
 subroutine MPI_Type_get_name_f08(datatype,type_name,resultlen,ierror)
    use :: mpi_f08_types, only : MPI_Datatype, MPI_MAX_OBJECT_NAME
    use :: ompi_mpifh_bindings, only : ompi_type_get_name_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
    implicit none
    TYPE(MPI_Datatype), INTENT(IN) :: datatype
    CHARACTER(LEN=*), INTENT(OUT) :: type_name
@@ -19,7 +20,8 @@ subroutine MPI_Type_get_name_f08(datatype,type_name,resultlen,ierror)
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
    integer :: c_ierror
 
-   call ompi_type_get_name_f(datatype%MPI_VAL,type_name,resultlen,c_ierror,len(type_name))
+   call ompi_type_get_name_f(datatype%MPI_VAL,type_name,resultlen,c_ierror, &
+                             len(type_name,KIND=C_INT))
    if (present(ierror)) ierror = c_ierror
 
 end subroutine MPI_Type_get_name_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/type_get_true_extent.c.in b/ompi/mpi/fortran/use-mpi-f08/type_get_true_extent.c.in
new file mode 100644
index 00000000000..32188707e20
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/type_get_true_extent.c.in
@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID type_get_true_extent(DATATYPE datatype, AINT_OUT true_lb, AINT_OUT true_extent)
+{
+    int c_ierr;
+    MPI_Datatype c_type = PMPI_Type_f2c(*datatype);
+
+    c_ierr = @INNER_CALL@(c_type, true_lb, true_extent);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/type_get_true_extent_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/type_get_true_extent_f08.F90
deleted file mode 100644
index de477a00bbe..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/type_get_true_extent_f08.F90
+++ /dev/null
@@ -1,24 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Type_get_true_extent_f08(datatype,true_lb,true_extent,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_ADDRESS_KIND
-   use :: ompi_mpifh_bindings, only : ompi_type_get_true_extent_f
-   implicit none
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER(MPI_ADDRESS_KIND), INTENT(OUT) :: true_lb, true_extent
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_type_get_true_extent_f(datatype%MPI_VAL,true_lb,true_extent,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Type_get_true_extent_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/type_indexed.c.in b/ompi/mpi/fortran/use-mpi-f08/type_indexed.c.in
new file mode 100644
index 00000000000..5dd1164fc72
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/type_indexed.c.in
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID type_indexed(COUNT count, COUNT_ARRAY array_of_blocklengths,
+			COUNT_ARRAY array_of_displacements, DATATYPE oldtype,
+			DATATYPE_OUT newtype)
+{
+    int c_ierr;
+    MPI_Datatype c_old = PMPI_Type_f2c(*oldtype);
+    MPI_Datatype c_new;
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@)*count;
+    @COUNT_TYPE@ *tmp_array_of_blocklengths = NULL;
+    @COUNT_TYPE@ *tmp_array_of_displacements = NULL;
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(array_of_blocklengths, tmp_array_of_blocklengths, *count);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_SET(array_of_displacements, tmp_array_of_displacements, *count);
+
+    c_ierr = @INNER_CALL@(c_count,
+                          tmp_array_of_blocklengths,
+                          tmp_array_of_displacements,
+                          c_old, &c_new);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(array_of_blocklengths, tmp_array_of_blocklengths);
+    OMPI_FORTRAN_BIGCOUNT_ARRAY_CLEANUP(array_of_displacements, tmp_array_of_displacements);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *newtype = PMPI_Type_c2f(c_new);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/type_indexed_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/type_indexed_f08.F90
deleted file mode 100644
index 813665ce6c1..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/type_indexed_f08.F90
+++ /dev/null
@@ -1,28 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Type_indexed_f08(count,array_of_blocklengths, &
-                                array_of_displacements,oldtype,newtype,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_ADDRESS_KIND
-   use :: ompi_mpifh_bindings, only : ompi_type_indexed_f
-   implicit none
-   INTEGER, INTENT(IN) :: count
-   INTEGER, INTENT(IN) :: array_of_blocklengths(count), array_of_displacements(count)
-   TYPE(MPI_Datatype), INTENT(IN) :: oldtype
-   TYPE(MPI_Datatype), INTENT(OUT) :: newtype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_type_indexed_f(count,array_of_blocklengths,array_of_displacements, &
-                            oldtype%MPI_VAL,newtype%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Type_indexed_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/type_set_name_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/type_set_name_f08.F90
index 9cefe0cc77f..e4a2d809186 100644
--- a/ompi/mpi/fortran/use-mpi-f08/type_set_name_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/type_set_name_f08.F90
@@ -12,13 +12,14 @@
 subroutine MPI_Type_set_name_f08(datatype,type_name,ierror)
    use :: mpi_f08_types, only : MPI_Datatype
    use :: ompi_mpifh_bindings, only : ompi_type_set_name_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
    implicit none
    TYPE(MPI_Datatype), INTENT(IN) :: datatype
    CHARACTER(LEN=*), INTENT(IN) :: type_name
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
    integer :: c_ierror
 
-   call ompi_type_set_name_f(datatype%MPI_VAL,type_name,c_ierror,len(type_name))
+   call ompi_type_set_name_f(datatype%MPI_VAL,type_name,c_ierror,len(type_name,KIND=C_INT))
    if (present(ierror)) ierror = c_ierror
 
 end subroutine MPI_Type_set_name_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/type_size.c.in b/ompi/mpi/fortran/use-mpi-f08/type_size.c.in
new file mode 100644
index 00000000000..c71e3f9294d
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/type_size.c.in
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID type_size(DATATYPE type, COUNT size)
+{
+    int c_ierr;
+    MPI_Datatype c_type = PMPI_Type_f2c(*type);
+    @COUNT_TYPE@ c_size;
+
+    c_ierr = @INNER_CALL@(c_type, &c_size);
+    if (MPI_SUCCESS == c_ierr)  {
+        *size = (@COUNT_FINT_TYPE@)(c_size);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/type_size_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/type_size_f08.F90
deleted file mode 100644
index 244a6e2378b..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/type_size_f08.F90
+++ /dev/null
@@ -1,24 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Type_size_f08(datatype,size,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_ADDRESS_KIND
-   use :: ompi_mpifh_bindings, only : ompi_type_size_f
-   implicit none
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER, INTENT(OUT) :: size
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_type_size_f(datatype%MPI_VAL,size,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Type_size_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/type_vector.c.in b/ompi/mpi/fortran/use-mpi-f08/type_vector.c.in
new file mode 100644
index 00000000000..b458e31eded
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/type_vector.c.in
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID  type_vector(COUNT count, COUNT blocklength,
+		       COUNT stride, DATATYPE oldtype,
+		       DATATYPE_OUT newtype)
+{
+    int c_ierr;
+    MPI_Datatype c_old;
+    MPI_Datatype c_new;
+    @COUNT_TYPE@ c_count = (@COUNT_TYPE@)*count;
+    @COUNT_TYPE@ c_blocklength = (@COUNT_TYPE@)*blocklength;
+    @COUNT_TYPE@ c_stride = (@COUNT_TYPE@)*stride;
+
+    c_old = PMPI_Type_f2c(*oldtype);
+
+    c_ierr = @INNER_CALL@(c_count,
+                          c_blocklength,
+                          c_stride,
+                          c_old, &c_new);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        *newtype = PMPI_Type_c2f(c_new);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/type_vector_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/type_vector_f08.F90
deleted file mode 100644
index 66791440104..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/type_vector_f08.F90
+++ /dev/null
@@ -1,26 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Type_vector_f08(count,blocklength,stride,oldtype,newtype,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype
-   use :: ompi_mpifh_bindings, only : ompi_type_vector_f
-   implicit none
-   INTEGER, INTENT(IN) :: count, blocklength, stride
-   TYPE(MPI_Datatype), INTENT(IN) :: oldtype
-   TYPE(MPI_Datatype), INTENT(OUT) :: newtype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_type_vector_f(count,blocklength,stride, &
-                           oldtype%MPI_VAL,newtype%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Type_vector_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/unpack_external_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/unpack_external_f08.F90
deleted file mode 100644
index 8cf2e637fe8..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/unpack_external_f08.F90
+++ /dev/null
@@ -1,33 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Unpack_external_f08(datarep,inbuf,insize,position,outbuf,outcount,datatype,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_ADDRESS_KIND
-   use :: ompi_mpifh_bindings, only : ompi_unpack_external_f
-   implicit none
-   CHARACTER(LEN=*), INTENT(IN) :: datarep
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: inbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: outbuf
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: insize
-   INTEGER(MPI_ADDRESS_KIND), INTENT(INOUT) :: position
-   INTEGER, INTENT(IN) :: outcount
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_unpack_external_f(datarep,inbuf,insize,position,outbuf,&
-                               outcount,datatype%MPI_VAL,c_ierror,len(datarep))
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Unpack_external_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/unpack_external_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/unpack_external_ts.c.in
new file mode 100644
index 00000000000..4852a43bada
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/unpack_external_ts.c.in
@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID unpack_external(CHAR_ARRAY datarep, BUFFER x1, AINT_COUNT insize,
+                               AINT_COUNT_INOUT position, BUFFER x2,
+                               COUNT_OUT outcount,  DATATYPE datatype)
+{
+    int c_ierr;
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    char *inbuf = OMPI_CFI_BASE_ADDR(x1);
+    void *outbuf = OMPI_CFI_BASE_ADDR(x2);
+
+    c_type = PMPI_Type_f2c(*datatype);
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(MPI_COMM_SELF, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    OMPI_CFI_2_C(x2, *outcount, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(MPI_COMM_SELF, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(datarep, inbuf,
+                                  *insize,
+                                  position,
+                                  OMPI_F2C_BOTTOM(outbuf),
+                                  *outcount,
+                                  c_datatype);
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/unpack_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/unpack_f08.F90
deleted file mode 100644
index 0f91d7859bc..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/unpack_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Unpack_f08(inbuf,insize,position,outbuf,outcount,datatype,comm,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm
-   use :: ompi_mpifh_bindings, only : ompi_unpack_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: inbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: outbuf
-   INTEGER, INTENT(IN) :: insize, outcount
-   INTEGER, INTENT(INOUT) :: position
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_unpack_f(inbuf,insize,position,outbuf,outcount, &
-                      datatype%MPI_VAL,comm%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Unpack_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/unpack_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/unpack_ts.c.in
new file mode 100644
index 00000000000..06da84b4529
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/unpack_ts.c.in
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID unpack(BUFFER x1, COUNT insize, COUNT_INOUT position,
+                      BUFFER_OUT x2, COUNT outcount, DATATYPE datatype,
+                      COMM comm)
+{
+    int c_ierr;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Datatype c_datatype, c_type = PMPI_Type_f2c(*datatype);
+    char *inbuf = OMPI_CFI_BASE_ADDR(x1);
+    void *outbuf = OMPI_CFI_BASE_ADDR(x2);
+    @COUNT_TYPE@ c_outcount = (@COUNT_TYPE@)(*outcount);
+    @COUNT_TYPE@ c_position = (@COUNT_TYPE@)(*position);
+    @COUNT_TYPE@ c_insize = (@COUNT_TYPE@)(*insize);
+
+    OMPI_CFI_CHECK_CONTIGUOUS(x1, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    OMPI_CFI_2_C(x2, c_outcount, c_type, c_datatype, c_ierr);
+    if (MPI_SUCCESS != c_ierr) {
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+        return;
+    }
+
+    c_ierr = @INNER_CALL@(inbuf, 
+                         c_insize,
+                         &c_position,
+                         OMPI_F2C_BOTTOM(outbuf), c_outcount,
+                         c_datatype, c_comm);
+    if (c_datatype != c_type) {
+        ompi_datatype_destroy(&c_datatype);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+         *position = (@COUNT_FINT_TYPE@)(c_position);
+     }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/unpublish_name_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/unpublish_name_f08.F90
index 779a856a6de..6757c85f4de 100644
--- a/ompi/mpi/fortran/use-mpi-f08/unpublish_name_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/unpublish_name_f08.F90
@@ -12,6 +12,7 @@
 subroutine MPI_Unpublish_name_f08(service_name,info,port_name,ierror)
    use :: mpi_f08_types, only : MPI_Info
    use :: ompi_mpifh_bindings, only : ompi_unpublish_name_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
    implicit none
    CHARACTER(LEN=*), INTENT(IN) :: service_name, port_name
    TYPE(MPI_Info), INTENT(IN) :: info
@@ -19,7 +20,8 @@ subroutine MPI_Unpublish_name_f08(service_name,info,port_name,ierror)
    integer :: c_ierror
 
    call ompi_unpublish_name_f(service_name,info%MPI_VAL,port_name,c_ierror, &
-                              len(service_name), len(port_name))
+                              len(service_name,KIND=C_INT), &
+                              len(port_name, KIND=C_INT))
    if (present(ierror)) ierror = c_ierror
 
 end subroutine MPI_Unpublish_name_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/waitall.c.in b/ompi/mpi/fortran/use-mpi-f08/waitall.c.in
new file mode 100644
index 00000000000..38cd9389722
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/waitall.c.in
@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID waitall(INT count, REQUEST_ARRAY array_of_requests:count,
+                       STATUS_ARRAY array_of_statuses)
+{
+    MPI_Request *c_req;
+    MPI_Status *c_status;
+    int i, c_ierr;
+
+    /* Shortcut to avoid malloc(0) if *count==0.  We're intentionally
+       skipping other parameter error checks. */
+    if (OPAL_UNLIKELY(0 == OMPI_FINT_2_INT(*count))) {
+        *ierr = OMPI_INT_2_FINT(MPI_SUCCESS);
+        return;
+    }
+
+    c_req = (MPI_Request *) malloc(OMPI_FINT_2_INT(*count) *
+                   (sizeof(MPI_Request) + sizeof(MPI_Status)));
+    if (NULL == c_req) {
+        c_ierr = OMPI_ERRHANDLER_NOHANDLE_INVOKE(
+                                        MPI_ERR_NO_MEM,
+                                        FUNC_NAME);
+        if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+        return;
+    }
+    c_status = (MPI_Status*) (c_req + OMPI_FINT_2_INT(*count));
+
+    for (i = 0; i < OMPI_FINT_2_INT(*count); ++i) {
+        c_req[i] = PMPI_Request_f2c(array_of_requests[i]);
+    }
+
+    c_ierr = PMPI_Waitall(OMPI_FINT_2_INT(*count), c_req, c_status);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+        for (i = 0; i < OMPI_FINT_2_INT(*count); ++i) {
+            array_of_requests[i] = c_req[i]->req_f_to_c_index;
+            if (!OMPI_IS_FORTRAN_STATUSES_IGNORE(array_of_statuses) &&
+                !OMPI_IS_FORTRAN_STATUS_IGNORE(&array_of_statuses[i])) {
+                PMPI_Status_c2f( &c_status[i], &array_of_statuses[i * (sizeof(MPI_Status) / sizeof(int))]);
+            }
+        }
+    }
+    free(c_req);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/waitall_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/waitall_f08.F90
deleted file mode 100644
index f07551d4c45..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/waitall_f08.F90
+++ /dev/null
@@ -1,25 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Waitall_f08(count,array_of_requests,array_of_statuses,ierror)
-   use :: mpi_f08_types, only : MPI_Request, MPI_Status
-   use :: ompi_mpifh_bindings, only : ompi_waitall_f
-   implicit none
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Request), INTENT(INOUT) :: array_of_requests(count)
-   TYPE(MPI_Status), INTENT(OUT) :: array_of_statuses(*)
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_waitall_f(count,array_of_requests(:)%MPI_VAL,array_of_statuses,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Waitall_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/win_allocate.c.in b/ompi/mpi/fortran/use-mpi-f08/win_allocate.c.in
new file mode 100644
index 00000000000..b52e758fc5b
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/win_allocate.c.in
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2014 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID win_allocate(AINT size, DISP disp_unit,
+                            INFO info, COMM comm, C_PTR_OUT baseptr,
+                            WIN_OUT win)
+{
+    int c_ierr;
+    MPI_Info c_info;
+    MPI_Comm c_comm;
+    MPI_Win c_win;
+    @DISP_TYPE@ c_disp_unit = (@DISP_TYPE@)*disp_unit;
+
+    c_info = PMPI_Info_f2c(*info);
+    c_comm = PMPI_Comm_f2c(*comm);
+
+    c_ierr = @INNER_CALL@(*size, c_disp_unit,
+                                     c_info, c_comm,
+                                     baseptr, &c_win);
+    *win = PMPI_Win_c2f(c_win);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/win_allocate_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/win_allocate_f08.F90
deleted file mode 100644
index a0f4f06562d..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/win_allocate_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2004-2014 High Performance Computing Center Stuttgart,
-!                         University of Stuttgart.  All rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Win_allocate_f08(size, disp_unit, info, comm, &
-      baseptr, win, ierror)
-   USE, INTRINSIC ::  ISO_C_BINDING, ONLY : C_PTR
-   use :: mpi_f08_types, only : MPI_Info, MPI_Comm, MPI_Win, MPI_ADDRESS_KIND
-   use :: ompi_mpifh_bindings, only : ompi_win_allocate_f
-   implicit none
-   INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) ::  size
-   INTEGER, INTENT(IN) ::  disp_unit
-   TYPE(MPI_Info), INTENT(IN) ::  info
-   TYPE(MPI_Comm), INTENT(IN) ::  comm
-   TYPE(C_PTR), INTENT(OUT) ::  baseptr
-   TYPE(MPI_Win), INTENT(OUT) ::  win
-   INTEGER, OPTIONAL, INTENT(OUT) ::  ierror
-   integer :: c_ierror
-
-  call ompi_win_allocate_f(size, disp_unit, info%MPI_VAL, comm%MPI_VAL, baseptr, win%MPI_VAL, c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Win_allocate_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/win_allocate_shared.c.in b/ompi/mpi/fortran/use-mpi-f08/win_allocate_shared.c.in
new file mode 100644
index 00000000000..ff13c22d178
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/win_allocate_shared.c.in
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2014 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID win_allocate_shared(AINT size, DISP disp_unit,
+                                   INFO info, COMM comm, C_PTR_OUT baseptr,
+                                   WIN win)
+{
+    int c_ierr;
+    MPI_Info c_info;
+    MPI_Comm c_comm;
+    MPI_Win c_win;
+    @DISP_TYPE@ c_disp_unit = (@DISP_TYPE@)*disp_unit;
+
+    c_info = PMPI_Info_f2c(*info);
+    c_comm = PMPI_Comm_f2c(*comm);
+
+    c_ierr = @INNER_CALL@(*size, c_disp_unit,
+                                     c_info, c_comm,
+                                     baseptr, &c_win);
+    *win = PMPI_Win_c2f(c_win);
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/win_allocate_shared_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/win_allocate_shared_f08.F90
deleted file mode 100644
index ec692f900fb..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/win_allocate_shared_f08.F90
+++ /dev/null
@@ -1,30 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Win_allocate_shared_f08(size, disp_unit, info, comm, &
-      baseptr, win, ierror)
-   USE, INTRINSIC ::  ISO_C_BINDING, ONLY : C_PTR
-   use :: mpi_f08_types, only : MPI_Info, MPI_Comm, MPI_Win, MPI_ADDRESS_KIND
-   use :: ompi_mpifh_bindings, only : ompi_win_allocate_shared_f
-   implicit none
-   INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) ::  size
-   INTEGER, INTENT(IN) ::  disp_unit
-   TYPE(MPI_Info), INTENT(IN) ::  info
-   TYPE(MPI_Comm), INTENT(IN) ::  comm
-   TYPE(C_PTR), INTENT(OUT) ::  baseptr
-   TYPE(MPI_Win), INTENT(OUT) ::  win
-   INTEGER, OPTIONAL, INTENT(OUT) ::  ierror
-   integer :: c_ierror
-
-  call ompi_win_allocate_shared_f(size, disp_unit, info%MPI_VAL, comm%MPI_VAL, baseptr, win%MPI_VAL, c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Win_allocate_shared_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/win_attach_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/win_attach_f08.F90
deleted file mode 100644
index 43210f25efd..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/win_attach_f08.F90
+++ /dev/null
@@ -1,25 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2015-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Win_attach_f08(win,base,size,ierror)
-   use :: mpi_f08_types, only : MPI_Win, MPI_ADDRESS_KIND
-   use :: ompi_mpifh_bindings, only : ompi_win_attach_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: base
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: size
-   TYPE(MPI_Win), INTENT(IN) :: win
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_win_attach_f(win%MPI_VAL,base,size,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Win_attach_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/win_attach_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/win_attach_ts.c.in
new file mode 100644
index 00000000000..bfb4b3accb2
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/win_attach_ts.c.in
@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID win_attach(WIN win, BUFFER x, AINT size)
+{
+    int c_ierr;
+    MPI_Win c_win;
+
+    c_win = PMPI_Win_f2c(*win);
+    if (OMPI_CFI_IS_CONTIGUOUS(x)) {
+        c_ierr = PMPI_Win_attach(c_win, OMPI_CFI_BASE_ADDR(x), *size);
+    } else {
+        c_ierr = MPI_ERR_BUFFER;
+        OMPI_ERRHANDLER_INVOKE(c_win, c_ierr, FUNC_NAME);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/win_create_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/win_create_f08.F90
deleted file mode 100644
index 3e452edd539..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/win_create_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Win_create_f08(base,size,disp_unit,info,comm,win,ierror)
-   use :: mpi_f08_types, only : MPI_Info, MPI_Comm, MPI_Win, MPI_ADDRESS_KIND
-   use :: ompi_mpifh_bindings, only : ompi_win_create_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: base
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: size
-   INTEGER, INTENT(IN) :: disp_unit
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Win), INTENT(OUT) :: win
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompi_win_create_f(base,size,disp_unit,info%MPI_VAL,&
-                          comm%MPI_VAL,win%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Win_create_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/win_create_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/win_create_ts.c.in
new file mode 100644
index 00000000000..e2dcdd59630
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/win_create_ts.c.in
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID win_create(BUFFER x, AINT size, DISP disp_unit,
+                          INFO info, COMM comm, WIN_OUT win)
+{
+    int c_ierr;
+    MPI_Win c_win;
+    MPI_Info c_info;
+    MPI_Comm c_comm;
+    @DISP_TYPE@ c_disp_unit = (@DISP_TYPE@)*disp_unit;
+
+    c_comm = PMPI_Comm_f2c(*comm);
+    c_info = PMPI_Info_f2c(*info);
+
+    if (OMPI_CFI_IS_CONTIGUOUS(x)) {
+        c_ierr = @INNER_CALL@(OMPI_CFI_BASE_ADDR(x), *size,
+                                 c_disp_unit,
+                                 c_info, c_comm, &c_win);
+    } else {
+        c_ierr = MPI_ERR_BUFFER;
+        OMPI_ERRHANDLER_INVOKE(c_comm, c_ierr, FUNC_NAME);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+    if (MPI_SUCCESS == c_ierr) {
+       *win = PMPI_Win_c2f(c_win);
+    }
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/win_detach_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/win_detach_f08.F90
index 54d09cb2ea4..a85f5b62523 100644
--- a/ompi/mpi/fortran/use-mpi-f08/win_detach_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/win_detach_f08.F90
@@ -13,7 +13,7 @@ subroutine MPI_Win_detach_f08(win,base,ierror)
    use :: mpi_f08_types, only : MPI_Win, MPI_ADDRESS_KIND
    use :: ompi_mpifh_bindings, only : ompi_win_detach_f
    implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: base
+   OMPI_FORTRAN_IGNORE_TKR_TYPE OMPI_ASYNCHRONOUS :: base
    TYPE(MPI_Win), INTENT(IN) :: win
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
    integer :: c_ierror
diff --git a/ompi/mpi/fortran/use-mpi-f08/win_detach_ts.c.in b/ompi/mpi/fortran/use-mpi-f08/win_detach_ts.c.in
new file mode 100644
index 00000000000..e89f9f4396f
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/win_detach_ts.c.in
@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID win_detach(WIN win, BUFFER x)
+{
+    int c_ierr;
+    MPI_Win c_win;
+
+    c_win = PMPI_Win_f2c(*win);
+    if (OMPI_CFI_IS_CONTIGUOUS(x)) {
+         c_ierr = @INNER_CALL@(c_win, OMPI_CFI_BASE_ADDR(x));
+    } else {
+        c_ierr = MPI_ERR_BUFFER;
+        OMPI_ERRHANDLER_INVOKE(c_win, c_ierr, FUNC_NAME);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/win_get_name_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/win_get_name_f08.F90
index 99088f8b277..ddce2e54aa4 100644
--- a/ompi/mpi/fortran/use-mpi-f08/win_get_name_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/win_get_name_f08.F90
@@ -13,6 +13,7 @@ subroutine MPI_Win_get_name_f08(win,win_name,resultlen,ierror)
    use, intrinsic :: ISO_C_BINDING, only : C_CHAR
    use :: mpi_f08_types, only : MPI_Win, MPI_MAX_OBJECT_NAME
    use :: ompi_mpifh_bindings, only : ompi_win_get_name_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
    implicit none
    TYPE(MPI_Win), INTENT(IN) :: win
    CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(OUT) :: win_name
@@ -20,7 +21,8 @@ subroutine MPI_Win_get_name_f08(win,win_name,resultlen,ierror)
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
    integer :: c_ierror
 
-   call ompi_win_get_name_f(win%MPI_VAL,win_name,resultlen,c_ierror,len(win_name))
+   call ompi_win_get_name_f(win%MPI_VAL,win_name,resultlen,c_ierror, &
+                            len(win_name,KIND=C_INT))
    if (present(ierror)) ierror = c_ierror
 
 end subroutine MPI_Win_get_name_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/win_set_name_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/win_set_name_f08.F90
index 8cb15884432..b81868bbe70 100644
--- a/ompi/mpi/fortran/use-mpi-f08/win_set_name_f08.F90
+++ b/ompi/mpi/fortran/use-mpi-f08/win_set_name_f08.F90
@@ -12,13 +12,14 @@
 subroutine MPI_Win_set_name_f08(win,win_name,ierror)
    use :: mpi_f08_types, only : MPI_Win
    use :: ompi_mpifh_bindings, only : ompi_win_set_name_f
+   use, intrinsic :: ISO_C_BINDING, only : C_INT
    implicit none
    TYPE(MPI_Win), INTENT(IN) :: win
    CHARACTER(LEN=*), INTENT(IN) :: win_name
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
    integer :: c_ierror
 
-   call ompi_win_set_name_f(win%MPI_VAL,win_name,c_ierror,len(win_name))
+   call ompi_win_set_name_f(win%MPI_VAL,win_name,c_ierror,len(win_name,KIND=C_INT))
    if (present(ierror)) ierror = c_ierror
 
 end subroutine MPI_Win_set_name_f08
diff --git a/ompi/mpi/fortran/use-mpi-f08/win_shared_query.c.in b/ompi/mpi/fortran/use-mpi-f08/win_shared_query.c.in
new file mode 100644
index 00000000000..e8da907b729
--- /dev/null
+++ b/ompi/mpi/fortran/use-mpi-f08/win_shared_query.c.in
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2017 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+PROTOTYPE VOID win_shared_query(WIN win, INT rank, AINT_OUT size,
+                                DISP_OUT disp_unit, C_PTR_OUT baseptr)
+{
+    int c_ierr;
+    MPI_Win c_win;
+    @DISP_TYPE@ c_disp_unit;
+
+    c_win = PMPI_Win_f2c(*win);
+
+    c_ierr = @INNER_CALL@(c_win, OMPI_FINT_2_INT(*rank), size,
+                          &c_disp_unit, baseptr);
+    if (MPI_SUCCESS == c_ierr) {
+        *disp_unit =  (@COUNT_FINT_TYPE@)(c_disp_unit);
+    }
+    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
+
+}
diff --git a/ompi/mpi/fortran/use-mpi-f08/win_shared_query_f08.F90 b/ompi/mpi/fortran/use-mpi-f08/win_shared_query_f08.F90
deleted file mode 100644
index e69f3028212..00000000000
--- a/ompi/mpi/fortran/use-mpi-f08/win_shared_query_f08.F90
+++ /dev/null
@@ -1,29 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!               All Rights reserved.
-! Copyright (c) 2018-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "mpi-f08-rename.h"
-
-subroutine MPI_Win_shared_query_f08(win, rank, size, disp_unit, baseptr,&
-      ierror)
-   USE, INTRINSIC ::  ISO_C_BINDING, ONLY : C_PTR
-   use :: mpi_f08_types, only : MPI_Win, MPI_ADDRESS_KIND
-   use :: ompi_mpifh_bindings, only : ompi_win_shared_query_f
-   implicit none
-   TYPE(MPI_Win), INTENT(IN) ::  win
-   INTEGER, INTENT(IN) ::  rank
-   INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(OUT) ::  size
-   INTEGER, INTENT(OUT) ::  disp_unit
-   TYPE(C_PTR), INTENT(OUT) ::  baseptr
-   INTEGER, OPTIONAL, INTENT(OUT) ::  ierror
-   integer :: c_ierror
-
-   call ompi_win_shared_query_f(win%MPI_VAL, rank, size, disp_unit, baseptr, c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPI_Win_shared_query_f08
diff --git a/ompi/mpi/fortran/use-mpi-ignore-tkr/Makefile.am b/ompi/mpi/fortran/use-mpi-ignore-tkr/Makefile.am
index 3916b159949..ed23a0fdea7 100644
--- a/ompi/mpi/fortran/use-mpi-ignore-tkr/Makefile.am
+++ b/ompi/mpi/fortran/use-mpi-ignore-tkr/Makefile.am
@@ -1,11 +1,12 @@
 # -*- makefile -*-
 #
 # Copyright (c) 2006-2019 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2015-2020 Research Organization for Information Science
+# Copyright (c) 2015-2021 Research Organization for Information Science
 #                         and Technology (RIST).  All rights reserved.
 # Copyright (c) 2016      IBM Corporation.  All rights reserved.
 #
 # Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+# Copyright (c) 2022      IBM Corporation.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -82,7 +83,7 @@ endif
 # all platforms).
 lib@OMPI_LIBMPI_NAME@_usempi_ignore_tkr_la_LIBADD = \
         $(libusempi_internal_modules) \
-        $(OMPI_TOP_BUILDDIR)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+        $(OMPI_TOP_BUILDDIR)/opal/lib@OPAL_LIB_NAME@.la
 lib@OMPI_LIBMPI_NAME@_usempi_ignore_tkr_la_LDFLAGS = \
         -version-info $(libmpi_usempi_ignore_tkr_so_version) \
         $(OMPI_FORTRAN_EXTRA_SHARED_LIBRARY_FLAGS)
@@ -102,9 +103,12 @@ mpi-ignore-tkr-sizeof.h:
 	    --maxrank=$(OMPI_FORTRAN_MAX_ARRAY_RANK) \
 	    --generate=$(OMPI_FORTRAN_BUILD_SIZEOF) \
 	    --real2=$(OMPI_HAVE_FORTRAN_REAL2) \
+	    --iso_real16=$(OMPI_FORTRAN_HAVE_ISO_FORTRAN_ENV_REAL16) \
 	    --real16=$(OMPI_HAVE_FORTRAN_REAL16) \
 	    --complex4=$(OMPI_HAVE_FORTRAN_COMPLEX4) \
-	    --complex32=$(OMPI_HAVE_FORTRAN_COMPLEX32)
+	    --complex32=$(OMPI_HAVE_FORTRAN_COMPLEX32) \
+	    --mpi_version=$(MPI_VERSION) \
+	    --request_deprecate=$(OMPI_FORTRAN_HAVE_ATTR_DEPRECATED)
 
 mpi-ignore-tkr-sizeof.f90: $(top_builddir)/config.status
 mpi-ignore-tkr-sizeof.f90: $(sizeof_pl)
@@ -114,6 +118,7 @@ mpi-ignore-tkr-sizeof.f90:
 	    --maxrank=$(OMPI_FORTRAN_MAX_ARRAY_RANK) \
 	    --generate=$(OMPI_FORTRAN_BUILD_SIZEOF) \
 	    --real2=$(OMPI_HAVE_FORTRAN_REAL2) \
+	    --iso_real16=$(OMPI_FORTRAN_HAVE_ISO_FORTRAN_ENV_REAL16) \
 	    --real16=$(OMPI_HAVE_FORTRAN_REAL16) \
 	    --complex4=$(OMPI_HAVE_FORTRAN_COMPLEX4) \
 	    --complex32=$(OMPI_HAVE_FORTRAN_COMPLEX32)
@@ -131,14 +136,14 @@ CLEANFILES += *.i90
 
 install-exec-hook:
 	@ for file in `ls *.mod`; do \
-	  echo $(INSTALL) $$file $(DESTDIR)$(libdir); \
-	  $(INSTALL) $$file $(DESTDIR)$(libdir); \
+	  echo $(INSTALL) $$file $(DESTDIR)$(OMPI_FORTRAN_MODULEDIR); \
+	  $(INSTALL) $$file $(DESTDIR)$(OMPI_FORTRAN_MODULEDIR); \
 	done
 
 uninstall-local:
 	@ for file in `ls *.mod`; do \
-	  echo rm -f $(DESTDIR)$(libdir)/$$file; \
-	  rm -f $(DESTDIR)$(libdir)/$$file; \
+	  echo rm -f $(DESTDIR)$(OMPI_FORTRAN_MODULEDIR)/$$file; \
+	  rm -f $(DESTDIR)$(OMPI_FORTRAN_MODULEDIR)/$$file; \
 	done
 
 endif
diff --git a/ompi/mpi/fortran/use-mpi-ignore-tkr/mpi-ignore-tkr-file-interfaces.h.in b/ompi/mpi/fortran/use-mpi-ignore-tkr/mpi-ignore-tkr-file-interfaces.h.in
index c00ebc5d8c7..a42973736ea 100644
--- a/ompi/mpi/fortran/use-mpi-ignore-tkr/mpi-ignore-tkr-file-interfaces.h.in
+++ b/ompi/mpi/fortran/use-mpi-ignore-tkr/mpi-ignore-tkr-file-interfaces.h.in
@@ -1,6 +1,6 @@
 ! -*- fortran -*-
 !
-! Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2006-2021 Cisco Systems, Inc.  All rights reserved
 ! Copyright (c) 2007      Los Alamos National Security, LLC.  All rights
 !                         reserved.
 ! Copyright (c) 2019-2020 Research Organization for Information Science
@@ -12,7 +12,7 @@
 ! $HEADER$
 
 
-interface MPI_File_call_errhandler
+interface
 
 subroutine MPI_File_call_errhandler(fh, errorcode, ierror)
   integer, intent(in) :: fh
@@ -22,7 +22,7 @@ end subroutine MPI_File_call_errhandler
 
 end interface
 
-interface MPI_File_close
+interface
 
 subroutine MPI_File_close(fh, ierror)
   integer, intent(inout) :: fh
@@ -31,7 +31,7 @@ end subroutine MPI_File_close
 
 end interface
 
-interface MPI_File_create_errhandler
+interface
 
 subroutine MPI_File_create_errhandler(function, errhandler, ierror)
   external :: function
@@ -41,7 +41,7 @@ end subroutine MPI_File_create_errhandler
 
 end interface
 
-interface MPI_File_delete
+interface
 
 subroutine MPI_File_delete(filename, info, ierror)
   character(len=*), intent(in) :: filename
@@ -51,7 +51,7 @@ end subroutine MPI_File_delete
 
 end interface
 
-interface MPI_File_get_amode
+interface
 
 subroutine MPI_File_get_amode(fh, amode, ierror)
   integer, intent(in) :: fh
@@ -61,7 +61,7 @@ end subroutine MPI_File_get_amode
 
 end interface
 
-interface MPI_File_get_atomicity
+interface
 
 subroutine MPI_File_get_atomicity(fh, flag, ierror)
   integer, intent(in) :: fh
@@ -71,7 +71,7 @@ end subroutine MPI_File_get_atomicity
 
 end interface
 
-interface MPI_File_get_byte_offset
+interface
 
 subroutine MPI_File_get_byte_offset(fh, offset, disp, ierror)
   include 'mpif-config.h'
@@ -83,7 +83,7 @@ end subroutine MPI_File_get_byte_offset
 
 end interface
 
-interface MPI_File_get_errhandler
+interface
 
 subroutine MPI_File_get_errhandler(file, errhandler, ierror)
   integer, intent(in) :: file
@@ -93,7 +93,7 @@ end subroutine MPI_File_get_errhandler
 
 end interface
 
-interface MPI_File_get_group
+interface
 
 subroutine MPI_File_get_group(fh, group, ierror)
   integer, intent(in) :: fh
@@ -103,7 +103,7 @@ end subroutine MPI_File_get_group
 
 end interface
 
-interface MPI_File_get_info
+interface
 
 subroutine MPI_File_get_info(fh, info_used, ierror)
   integer, intent(in) :: fh
@@ -113,7 +113,7 @@ end subroutine MPI_File_get_info
 
 end interface
 
-interface MPI_File_get_position
+interface
 
 subroutine MPI_File_get_position(fh, offset, ierror)
   include 'mpif-config.h'
@@ -124,7 +124,7 @@ end subroutine MPI_File_get_position
 
 end interface
 
-interface MPI_File_get_position_shared
+interface
 
 subroutine MPI_File_get_position_shared(fh, offset, ierror)
   include 'mpif-config.h'
@@ -135,7 +135,7 @@ end subroutine MPI_File_get_position_shared
 
 end interface
 
-interface MPI_File_get_size
+interface
 
 subroutine MPI_File_get_size(fh, size, ierror)
   include 'mpif-config.h'
@@ -146,7 +146,7 @@ end subroutine MPI_File_get_size
 
 end interface
 
-interface MPI_File_get_type_extent
+interface
 
 subroutine MPI_File_get_type_extent(fh, datatype, extent, ierror)
   include 'mpif-config.h'
@@ -158,7 +158,7 @@ end subroutine MPI_File_get_type_extent
 
 end interface
 
-interface MPI_File_get_view
+interface
 
 subroutine MPI_File_get_view(fh, disp, etype, filetype, datarep&
         , ierror)
@@ -173,7 +173,7 @@ end subroutine MPI_File_get_view
 
 end interface
 
-interface MPI_File_iread
+interface
 
 subroutine MPI_File_iread(fh, buf, count, datatype, request&
         , ierror)
@@ -188,7 +188,7 @@ end subroutine MPI_File_iread
 
 end interface
 
-interface MPI_File_iread_all
+interface
 
 subroutine MPI_File_iread_all(fh, buf, count, datatype, request&
         , ierror)
@@ -203,7 +203,7 @@ end subroutine MPI_File_iread_all
 
 end interface
 
-interface MPI_File_iread_at
+interface
 
 subroutine MPI_File_iread_at(fh, offset, buf, count, datatype, &
         request, ierror)
@@ -220,7 +220,7 @@ end subroutine MPI_File_iread_at
 
 end interface
 
-interface MPI_File_iread_at_all
+interface
 
 subroutine MPI_File_iread_at_all(fh, offset, buf, count, datatype, &
         request, ierror)
@@ -237,7 +237,7 @@ end subroutine MPI_File_iread_at_all
 
 end interface
 
-interface MPI_File_iread_shared
+interface
 
 subroutine MPI_File_iread_shared(fh, buf, count, datatype, request&
         , ierror)
@@ -252,7 +252,7 @@ end subroutine MPI_File_iread_shared
 
 end interface
 
-interface MPI_File_iwrite
+interface
 
 subroutine MPI_File_iwrite(fh, buf, count, datatype, request&
         , ierror)
@@ -267,7 +267,7 @@ end subroutine MPI_File_iwrite
 
 end interface
 
-interface MPI_File_iwrite_all
+interface
 
 subroutine MPI_File_iwrite_all(fh, buf, count, datatype, request&
         , ierror)
@@ -282,7 +282,7 @@ end subroutine MPI_File_iwrite_all
 
 end interface
 
-interface MPI_File_iwrite_at
+interface
 
 subroutine MPI_File_iwrite_at(fh, offset, buf, count, datatype, &
         request, ierror)
@@ -299,7 +299,7 @@ end subroutine MPI_File_iwrite_at
 
 end interface
 
-interface MPI_File_iwrite_at_all
+interface
 
 subroutine MPI_File_iwrite_at_all(fh, offset, buf, count, datatype, &
         request, ierror)
@@ -316,7 +316,7 @@ end subroutine MPI_File_iwrite_at_all
 
 end interface
 
-interface MPI_File_iwrite_shared
+interface
 
 subroutine MPI_File_iwrite_shared(fh, buf, count, datatype, request&
         , ierror)
@@ -331,7 +331,7 @@ end subroutine MPI_File_iwrite_shared
 
 end interface
 
-interface MPI_File_open
+interface
 
 subroutine MPI_File_open(comm, filename, amode, info, fh&
         , ierror)
@@ -345,7 +345,7 @@ end subroutine MPI_File_open
 
 end interface
 
-interface MPI_File_preallocate
+interface
 
 subroutine MPI_File_preallocate(fh, size, ierror)
   include 'mpif-config.h'
@@ -356,7 +356,7 @@ end subroutine MPI_File_preallocate
 
 end interface
 
-interface MPI_File_read
+interface
 
 subroutine MPI_File_read(fh, buf, count, datatype, status&
         , ierror)
@@ -372,7 +372,7 @@ end subroutine MPI_File_read
 
 end interface
 
-interface MPI_File_read_all
+interface
 
 subroutine MPI_File_read_all(fh, buf, count, datatype, status&
         , ierror)
@@ -388,7 +388,7 @@ end subroutine MPI_File_read_all
 
 end interface
 
-interface MPI_File_read_all_begin
+interface
 
 subroutine MPI_File_read_all_begin(fh, buf, count, datatype, ierror)
   integer, intent(in) :: fh
@@ -401,7 +401,7 @@ end subroutine MPI_File_read_all_begin
 
 end interface
 
-interface MPI_File_read_all_end
+interface
 
 subroutine MPI_File_read_all_end(fh, buf, status, ierror)
   include 'mpif-config.h'
@@ -414,7 +414,7 @@ end subroutine MPI_File_read_all_end
 
 end interface
 
-interface MPI_File_read_at
+interface
 
 subroutine MPI_File_read_at(fh, offset, buf, count, datatype, &
         status, ierror)
@@ -431,7 +431,7 @@ end subroutine MPI_File_read_at
 
 end interface
 
-interface MPI_File_read_at_all
+interface
 
 subroutine MPI_File_read_at_all(fh, offset, buf, count, datatype, &
         status, ierror)
@@ -448,7 +448,7 @@ end subroutine MPI_File_read_at_all
 
 end interface
 
-interface MPI_File_read_at_all_begin
+interface
 
 subroutine MPI_File_read_at_all_begin(fh, offset, buf, count, datatype&
         , ierror)
@@ -464,7 +464,7 @@ end subroutine MPI_File_read_at_all_begin
 
 end interface
 
-interface MPI_File_read_at_all_end
+interface
 
 subroutine MPI_File_read_at_all_end(fh, buf, status, ierror)
   include 'mpif-config.h'
@@ -477,7 +477,7 @@ end subroutine MPI_File_read_at_all_end
 
 end interface
 
-interface MPI_File_read_ordered
+interface
 
 subroutine MPI_File_read_ordered(fh, buf, count, datatype, status&
         , ierror)
@@ -493,7 +493,7 @@ end subroutine MPI_File_read_ordered
 
 end interface
 
-interface MPI_File_read_ordered_begin
+interface
 
 subroutine MPI_File_read_ordered_begin(fh, buf, count, datatype, ierror)
   integer, intent(in) :: fh
@@ -506,7 +506,7 @@ end subroutine MPI_File_read_ordered_begin
 
 end interface
 
-interface MPI_File_read_ordered_end
+interface
 
 subroutine MPI_File_read_ordered_end(fh, buf, status, ierror)
   include 'mpif-config.h'
@@ -519,7 +519,7 @@ end subroutine MPI_File_read_ordered_end
 
 end interface
 
-interface MPI_File_read_shared
+interface
 
 subroutine MPI_File_read_shared(fh, buf, count, datatype, status&
         , ierror)
@@ -535,7 +535,7 @@ end subroutine MPI_File_read_shared
 
 end interface
 
-interface MPI_File_seek
+interface
 
 subroutine MPI_File_seek(fh, offset, whence, ierror)
   include 'mpif-config.h'
@@ -547,7 +547,7 @@ end subroutine MPI_File_seek
 
 end interface
 
-interface MPI_File_seek_shared
+interface
 
 subroutine MPI_File_seek_shared(fh, offset, whence, ierror)
   include 'mpif-config.h'
@@ -559,7 +559,7 @@ end subroutine MPI_File_seek_shared
 
 end interface
 
-interface MPI_File_set_atomicity
+interface
 
 subroutine MPI_File_set_atomicity(fh, flag, ierror)
   integer, intent(in) :: fh
@@ -569,7 +569,7 @@ end subroutine MPI_File_set_atomicity
 
 end interface
 
-interface MPI_File_set_errhandler
+interface
 
 subroutine MPI_File_set_errhandler(file, errhandler, ierror)
   integer, intent(in) :: file
@@ -579,7 +579,7 @@ end subroutine MPI_File_set_errhandler
 
 end interface
 
-interface MPI_File_set_info
+interface
 
 subroutine MPI_File_set_info(fh, info, ierror)
   integer, intent(in) :: fh
@@ -589,7 +589,7 @@ end subroutine MPI_File_set_info
 
 end interface
 
-interface MPI_File_set_size
+interface
 
 subroutine MPI_File_set_size(fh, size, ierror)
   include 'mpif-config.h'
@@ -600,7 +600,7 @@ end subroutine MPI_File_set_size
 
 end interface
 
-interface MPI_File_set_view
+interface
 
 subroutine MPI_File_set_view(fh, disp, etype, filetype, datarep, &
         info, ierror)
@@ -616,7 +616,7 @@ end subroutine MPI_File_set_view
 
 end interface
 
-interface MPI_File_sync
+interface
 
 subroutine MPI_File_sync(fh, ierror)
   integer, intent(in) :: fh
@@ -625,7 +625,7 @@ end subroutine MPI_File_sync
 
 end interface
 
-interface MPI_File_write
+interface
 
 subroutine MPI_File_write(fh, buf, count, datatype, status&
         , ierror)
@@ -641,7 +641,7 @@ end subroutine MPI_File_write
 
 end interface
 
-interface MPI_File_write_all
+interface
 
 subroutine MPI_File_write_all(fh, buf, count, datatype, status&
         , ierror)
@@ -657,7 +657,7 @@ end subroutine MPI_File_write_all
 
 end interface
 
-interface MPI_File_write_all_begin
+interface
 
 subroutine MPI_File_write_all_begin(fh, buf, count, datatype, ierror)
   integer, intent(in) :: fh
@@ -670,7 +670,7 @@ end subroutine MPI_File_write_all_begin
 
 end interface
 
-interface MPI_File_write_all_end
+interface
 
 subroutine MPI_File_write_all_end(fh, buf, status, ierror)
   include 'mpif-config.h'
@@ -683,7 +683,7 @@ end subroutine MPI_File_write_all_end
 
 end interface
 
-interface MPI_File_write_at
+interface
 
 subroutine MPI_File_write_at(fh, offset, buf, count, datatype, &
         status, ierror)
@@ -700,7 +700,7 @@ end subroutine MPI_File_write_at
 
 end interface
 
-interface MPI_File_write_at_all
+interface
 
 subroutine MPI_File_write_at_all(fh, offset, buf, count, datatype, &
         status, ierror)
@@ -717,7 +717,7 @@ end subroutine MPI_File_write_at_all
 
 end interface
 
-interface MPI_File_write_at_all_begin
+interface
 
 subroutine MPI_File_write_at_all_begin(fh, offset, buf, count, datatype&
         , ierror)
@@ -733,7 +733,7 @@ end subroutine MPI_File_write_at_all_begin
 
 end interface
 
-interface MPI_File_write_at_all_end
+interface
 
 subroutine MPI_File_write_at_all_end(fh, buf, status, ierror)
   include 'mpif-config.h'
@@ -746,7 +746,7 @@ end subroutine MPI_File_write_at_all_end
 
 end interface
 
-interface MPI_File_write_ordered
+interface
 
 subroutine MPI_File_write_ordered(fh, buf, count, datatype, status&
         , ierror)
@@ -762,7 +762,7 @@ end subroutine MPI_File_write_ordered
 
 end interface
 
-interface MPI_File_write_ordered_begin
+interface
 
 subroutine MPI_File_write_ordered_begin(fh, buf, count, datatype, ierror)
   integer, intent(in) :: fh
@@ -775,7 +775,7 @@ end subroutine MPI_File_write_ordered_begin
 
 end interface
 
-interface MPI_File_write_ordered_end
+interface
 
 subroutine MPI_File_write_ordered_end(fh, buf, status, ierror)
   include 'mpif-config.h'
@@ -788,7 +788,7 @@ end subroutine MPI_File_write_ordered_end
 
 end interface
 
-interface MPI_File_write_shared
+interface
 
 subroutine MPI_File_write_shared(fh, buf, count, datatype, status&
         , ierror)
diff --git a/ompi/mpi/fortran/use-mpi-ignore-tkr/mpi-ignore-tkr-interfaces.h.in b/ompi/mpi/fortran/use-mpi-ignore-tkr/mpi-ignore-tkr-interfaces.h.in
index ebaa2b6a561..6253d378bcc 100644
--- a/ompi/mpi/fortran/use-mpi-ignore-tkr/mpi-ignore-tkr-interfaces.h.in
+++ b/ompi/mpi/fortran/use-mpi-ignore-tkr/mpi-ignore-tkr-interfaces.h.in
@@ -1,6 +1,6 @@
 ! -*- fortran -*-
 !
-! Copyright (c) 2006-2018 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2006-2021 Cisco Systems, Inc.  All rights reserved
 ! Copyright (c) 2007      Los Alamos National Security, LLC.  All rights
 !                         reserved.
 ! Copyright (c) 2012      The University of Tennessee and The University
@@ -9,8 +9,12 @@
 ! Copyright (c) 2012      Inria.  All rights reserved.
 ! Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights
 !                         reserved.
-! Copyright (c) 2015-2020 Research Organization for Information Science
+! Copyright (c) 2015-2023 Research Organization for Information Science
 !                         and Technology (RIST).  All rights reserved.
+! Copyright (c) 2019-2023 Triad National Security, LLC. All rights
+!                         reserved.
+! Copyright (c) 2021      Bull S.A.S. All rights reserved.
+! Copyright (c) 2021      IBM Corporation.  All rights reserved.
 ! $COPYRIGHT$
 !
 ! Additional copyrights may follow
@@ -18,7 +22,7 @@
 ! $HEADER$
 
 
-interface MPI_Abort
+interface
 
 subroutine MPI_Abort(comm, errorcode, ierror)
   integer, intent(in) :: comm
@@ -29,7 +33,7 @@ end subroutine MPI_Abort
 end interface
 
 
-interface MPI_Accumulate
+interface
 
 subroutine MPI_Accumulate(origin_addr, origin_count, origin_datatype, target_rank, target_disp, &
         target_count, target_datatype, op, win, ierror)
@@ -50,7 +54,7 @@ end subroutine MPI_Accumulate
 end interface
 
 
-interface MPI_Add_error_class
+interface
 
 subroutine MPI_Add_error_class(errorclass, ierror)
   integer, intent(out) :: errorclass
@@ -60,7 +64,7 @@ end subroutine MPI_Add_error_class
 end interface
 
 
-interface MPI_Add_error_code
+interface
 
 subroutine MPI_Add_error_code(errorclass, errorcode, ierror)
   integer, intent(in) :: errorclass
@@ -71,7 +75,7 @@ end subroutine MPI_Add_error_code
 end interface
 
 
-interface MPI_Add_error_string
+interface
 
 subroutine MPI_Add_error_string(errorcode, string, ierror)
   integer, intent(in) :: errorcode
@@ -82,7 +86,7 @@ end subroutine MPI_Add_error_string
 end interface
 
 
-interface MPI_Aint_add
+interface
 
 function MPI_Aint_add(base, diff)
   include 'mpif-config.h'
@@ -94,7 +98,7 @@ end function MPI_Aint_add
 end interface
 
 
-interface MPI_Aint_diff
+interface
 
 function MPI_Aint_diff(addr1, addr2)
   include 'mpif-config.h'
@@ -106,7 +110,7 @@ end function MPI_Aint_diff
 end interface
 
 
-interface MPI_Allgather
+interface
 
 subroutine MPI_Allgather(sendbuf, sendcount, sendtype, recvbuf, recvcount, &
         recvtype, comm, ierror)
@@ -125,7 +129,29 @@ end subroutine MPI_Allgather
 end interface
 
 
-interface MPI_Allgatherv
+interface
+
+subroutine MPI_Allgather_init(sendbuf, sendcount, sendtype, &
+                              recvbuf, recvcount, recvtype, &
+                              comm, info, request, ierror)
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@, intent(in) :: sendbuf
+  integer, intent(in) :: sendcount
+  integer, intent(in) :: sendtype
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ recvbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@, intent(in) :: recvbuf
+  integer, intent(in) :: recvcount
+  integer, intent(in) :: recvtype
+  integer, intent(in) :: comm
+  integer, intent(in) :: info
+  integer, intent(out) :: request
+  integer, intent(out) :: ierror
+end subroutine MPI_Allgather_init
+
+end interface
+
+
+interface
 
 subroutine MPI_Allgatherv(sendbuf, sendcount, sendtype, recvbuf, recvcounts, &
         displs, recvtype, comm, ierror)
@@ -145,6 +171,29 @@ end subroutine MPI_Allgatherv
 end interface
 
 
+interface
+
+subroutine MPI_Allgatherv_init(sendbuf, sendcount, sendtype, recvbuf, recvcounts, &
+        displs, recvtype, comm, info, request, ierror)
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@, intent(in) :: sendbuf
+  integer, intent(in) :: sendcount
+  integer, intent(in) :: sendtype
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ recvbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
+  integer, dimension(*), intent(in) :: recvcounts
+  integer, dimension(*), intent(in) :: displs
+  integer, intent(in) :: recvtype
+  integer, intent(in) :: comm
+  integer, intent(in) :: info
+  integer, intent(out) :: request
+  integer, intent(out) :: ierror
+end subroutine MPI_Allgatherv_init
+
+end interface
+
+
+! This interface requires a name because there are multiple subroutines.
 interface MPI_Alloc_mem
 
 subroutine MPI_Alloc_mem(size, info, baseptr, ierror)
@@ -167,7 +216,7 @@ end subroutine MPI_Alloc_mem_cptr
 end interface
 
 
-interface MPI_Allreduce
+interface
 
 subroutine MPI_Allreduce(sendbuf, recvbuf, count, datatype, op, &
         comm, ierror)
@@ -185,7 +234,27 @@ end subroutine MPI_Allreduce
 end interface
 
 
-interface MPI_Alltoall
+interface
+
+subroutine MPI_Allreduce_init(sendbuf, recvbuf, count, datatype, op, &
+        comm, info, request, ierror)
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@, intent(in) :: sendbuf
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ recvbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
+  integer, intent(in) :: count
+  integer, intent(in) :: datatype
+  integer, intent(in) :: op
+  integer, intent(in) :: comm
+  integer, intent(in) :: info
+  integer, intent(out) :: request
+  integer, intent(out) :: ierror
+end subroutine MPI_Allreduce_init
+
+end interface
+
+
+interface
 
 subroutine MPI_Alltoall(sendbuf, sendcount, sendtype, recvbuf, recvcount, &
         recvtype, comm, ierror)
@@ -204,7 +273,28 @@ end subroutine MPI_Alltoall
 end interface
 
 
-interface MPI_Alltoallv
+interface
+
+subroutine MPI_Alltoall_init(sendbuf, sendcount, sendtype, recvbuf, recvcount, &
+        recvtype, comm, info, request, ierror)
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@, intent(in) :: sendbuf
+  integer, intent(in) :: sendcount
+  integer, intent(in) :: sendtype
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ recvbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
+  integer, intent(in) :: recvcount
+  integer, intent(in) :: recvtype
+  integer, intent(in) :: comm
+  integer, intent(in) :: info
+  integer, intent(out) :: request
+  integer, intent(out) :: ierror
+end subroutine MPI_Alltoall_init
+
+end interface
+
+
+interface
 
 subroutine MPI_Alltoallv(sendbuf, sendcounts, sdispls, sendtype, recvbuf, &
         recvcounts, rdispls, recvtype, comm, ierror)
@@ -225,7 +315,30 @@ end subroutine MPI_Alltoallv
 end interface
 
 
-interface MPI_Alltoallw
+interface
+
+subroutine MPI_Alltoallv_init(sendbuf, sendcounts, sdispls, sendtype, recvbuf, &
+        recvcounts, rdispls, recvtype, comm, info, request, ierror)
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@, intent(in) :: sendbuf
+  integer, dimension(*), intent(in) :: sendcounts
+  integer, dimension(*), intent(in) :: sdispls
+  integer, intent(in) :: sendtype
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ recvbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
+  integer, dimension(*), intent(in) :: recvcounts
+  integer, dimension(*), intent(in) :: rdispls
+  integer, intent(in) :: recvtype
+  integer, intent(in) :: comm
+  integer, intent(in) :: info
+  integer, intent(out) :: request
+  integer, intent(out) :: ierror
+end subroutine MPI_Alltoallv_init
+
+end interface
+
+
+interface
 
 subroutine MPI_Alltoallw(sendbuf, sendcounts, sdispls, sendtypes, recvbuf, &
         recvcounts, rdispls, recvtypes, comm, ierror)
@@ -246,7 +359,30 @@ end subroutine MPI_Alltoallw
 end interface
 
 
-interface MPI_Barrier
+interface
+
+subroutine MPI_Alltoallw_init(sendbuf, sendcounts, sdispls, sendtypes, recvbuf, &
+        recvcounts, rdispls, recvtypes, comm, info, request, ierror)
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@, intent(in) :: sendbuf
+  integer, dimension(*), intent(in) :: sendcounts
+  integer, dimension(*), intent(in) :: sdispls
+  integer, dimension(*), intent(in) :: sendtypes
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ recvbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
+  integer, dimension(*), intent(in) :: recvcounts
+  integer, dimension(*), intent(in) :: rdispls
+  integer, dimension(*), intent(in) :: recvtypes
+  integer, intent(in) :: comm
+  integer, intent(in) :: info
+  integer, intent(out) :: request
+  integer, intent(out) :: ierror
+end subroutine MPI_Alltoallw_init
+
+end interface
+
+
+interface
 
 subroutine MPI_Barrier(comm, ierror)
   integer, intent(in) :: comm
@@ -256,7 +392,19 @@ end subroutine MPI_Barrier
 end interface
 
 
-interface MPI_Bcast
+interface
+
+subroutine MPI_Barrier_init(comm, info, request, ierror)
+  integer, intent(in) :: comm
+  integer, intent(in) :: info
+  integer, intent(out) :: request
+  integer, intent(out) :: ierror
+end subroutine MPI_Barrier_init
+
+end interface
+
+
+interface
 
 subroutine MPI_Bcast(buffer, count, datatype, root, comm&
         , ierror)
@@ -272,7 +420,25 @@ end subroutine MPI_Bcast
 end interface
 
 
-interface MPI_Bsend
+interface
+
+subroutine MPI_Bcast_init(buffer, count, datatype, root, comm, &
+        info, request, ierror)
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buffer
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buffer
+  integer, intent(in) :: count
+  integer, intent(in) :: datatype
+  integer, intent(in) :: root
+  integer, intent(in) :: comm
+  integer, intent(in) :: info
+  integer, intent(out) :: request
+  integer, intent(out) :: ierror
+end subroutine MPI_Bcast_init
+
+end interface
+
+
+interface
 
 subroutine MPI_Bsend(buf, count, datatype, dest, tag, &
         comm, ierror)
@@ -289,7 +455,7 @@ end subroutine MPI_Bsend
 end interface
 
 
-interface MPI_Bsend_init
+interface
 
 subroutine MPI_Bsend_init(buf, count, datatype, dest, tag, &
         comm, request, ierror)
@@ -307,7 +473,7 @@ end subroutine MPI_Bsend_init
 end interface
 
 
-interface MPI_Buffer_attach
+interface
 
 subroutine MPI_Buffer_attach(buffer, size, ierror)
   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buffer
@@ -319,7 +485,7 @@ end subroutine MPI_Buffer_attach
 end interface
 
 
-interface MPI_Buffer_detach
+interface
 
 subroutine MPI_Buffer_detach(buffer, size, ierror)
   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buffer
@@ -331,7 +497,7 @@ end subroutine MPI_Buffer_detach
 end interface
 
 
-interface MPI_Cancel
+interface
 
 subroutine MPI_Cancel(request, ierror)
   integer, intent(in) :: request
@@ -341,7 +507,7 @@ end subroutine MPI_Cancel
 end interface
 
 
-interface MPI_Cart_coords
+interface
 
 subroutine MPI_Cart_coords(comm, rank, maxdims, coords, ierror)
   integer, intent(in) :: comm
@@ -354,7 +520,7 @@ end subroutine MPI_Cart_coords
 end interface
 
 
-interface MPI_Cart_create
+interface
 
 subroutine MPI_Cart_create(old_comm, ndims, dims, periods, reorder, &
         comm_cart, ierror)
@@ -370,7 +536,7 @@ end subroutine MPI_Cart_create
 end interface
 
 
-interface MPI_Cart_get
+interface
 
 subroutine MPI_Cart_get(comm, maxdims, dims, periods, coords&
         , ierror)
@@ -385,7 +551,7 @@ end subroutine MPI_Cart_get
 end interface
 
 
-interface MPI_Cart_map
+interface
 
 subroutine MPI_Cart_map(comm, ndims, dims, periods, newrank&
         , ierror)
@@ -400,7 +566,7 @@ end subroutine MPI_Cart_map
 end interface
 
 
-interface MPI_Cart_rank
+interface
 
 subroutine MPI_Cart_rank(comm, coords, rank, ierror)
   integer, intent(in) :: comm
@@ -412,7 +578,7 @@ end subroutine MPI_Cart_rank
 end interface
 
 
-interface MPI_Cart_shift
+interface
 
 subroutine MPI_Cart_shift(comm, direction, disp, rank_source, rank_dest&
         , ierror)
@@ -427,7 +593,7 @@ end subroutine MPI_Cart_shift
 end interface
 
 
-interface MPI_Cart_sub
+interface
 
 subroutine MPI_Cart_sub(comm, remain_dims, new_comm, ierror)
   integer, intent(in) :: comm
@@ -439,7 +605,7 @@ end subroutine MPI_Cart_sub
 end interface
 
 
-interface MPI_Cartdim_get
+interface
 
 subroutine MPI_Cartdim_get(comm, ndims, ierror)
   integer, intent(in) :: comm
@@ -450,7 +616,7 @@ end subroutine MPI_Cartdim_get
 end interface
 
 
-interface MPI_Close_port
+interface
 
 subroutine MPI_Close_port(port_name, ierror)
   character(len=*), intent(in) :: port_name
@@ -460,7 +626,7 @@ end subroutine MPI_Close_port
 end interface
 
 
-interface MPI_Comm_accept
+interface
 
 subroutine MPI_Comm_accept(port_name, info, root, comm, newcomm&
         , ierror)
@@ -475,7 +641,7 @@ end subroutine MPI_Comm_accept
 end interface
 
 
-interface MPI_Comm_call_errhandler
+interface
 
 subroutine MPI_Comm_call_errhandler(comm, errorcode, ierror)
   integer, intent(in) :: comm
@@ -486,7 +652,7 @@ end subroutine MPI_Comm_call_errhandler
 end interface
 
 
-interface MPI_Comm_compare
+interface
 
 subroutine MPI_Comm_compare(comm1, comm2, result, ierror)
   integer, intent(in) :: comm1
@@ -498,7 +664,7 @@ end subroutine MPI_Comm_compare
 end interface
 
 
-interface MPI_Comm_connect
+interface
 
 subroutine MPI_Comm_connect(port_name, info, root, comm, newcomm&
         , ierror)
@@ -513,7 +679,7 @@ end subroutine MPI_Comm_connect
 end interface
 
 
-interface MPI_Comm_create
+interface
 
 subroutine MPI_Comm_create(comm, group, newcomm, ierror)
   integer, intent(in) :: comm
@@ -525,7 +691,7 @@ end subroutine MPI_Comm_create
 end interface
 
 
-interface MPI_Comm_create_errhandler
+interface
 
 subroutine MPI_Comm_create_errhandler(function, errhandler, ierror)
   external :: function
@@ -535,8 +701,20 @@ end subroutine MPI_Comm_create_errhandler
 
 end interface
 
+interface  MPI_Comm_create_from_group
+
+subroutine MPI_Comm_create_from_group(group, stringtag, info, errhandler, newcomm, ierror)
+   integer, INTENT(IN) :: group
+   CHARACTER(LEN=*), INTENT(IN) :: stringtag
+   integer, INTENT(IN) :: info
+   integer, INTENT(IN) :: errhandler
+   integer, INTENT(OUT) :: newcomm
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine MPI_Comm_create_from_group
+
+end interface
 
-interface MPI_Comm_create_group
+interface
 
 subroutine MPI_Comm_create_group(comm, group, tag, newcomm, ierror)
   integer, intent(in) :: comm
@@ -549,7 +727,7 @@ end subroutine MPI_Comm_create_group
 end interface
 
 
-interface MPI_Comm_create_keyval
+interface
 
 subroutine MPI_Comm_create_keyval(comm_copy_attr_fn, comm_delete_attr_fn, comm_keyval, extra_state, ierror)
   include 'mpif-config.h'
@@ -563,7 +741,7 @@ end subroutine MPI_Comm_create_keyval
 end interface
 
 
-interface MPI_Comm_delete_attr
+interface
 
 subroutine MPI_Comm_delete_attr(comm, comm_keyval, ierror)
   integer, intent(in) :: comm
@@ -574,7 +752,7 @@ end subroutine MPI_Comm_delete_attr
 end interface
 
 
-interface MPI_Comm_disconnect
+interface
 
 subroutine MPI_Comm_disconnect(comm, ierror)
   integer, intent(inout) :: comm
@@ -584,7 +762,7 @@ end subroutine MPI_Comm_disconnect
 end interface
 
 
-interface MPI_Comm_dup
+interface
 
 subroutine MPI_Comm_dup(comm, newcomm, ierror)
   integer, intent(in) :: comm
@@ -595,7 +773,7 @@ end subroutine MPI_Comm_dup
 end interface
 
 
-interface MPI_Comm_dup_with_info
+interface
 
 subroutine MPI_Comm_dup_with_info(comm, info, newcomm, ierror)
   integer, intent(in) :: comm
@@ -607,7 +785,7 @@ end subroutine MPI_Comm_dup_with_info
 end interface
 
 
-interface MPI_Comm_free
+interface
 
 subroutine MPI_Comm_free(comm, ierror)
   integer, intent(inout) :: comm
@@ -617,7 +795,7 @@ end subroutine MPI_Comm_free
 end interface
 
 
-interface MPI_Comm_free_keyval
+interface
 
 subroutine MPI_Comm_free_keyval(comm_keyval, ierror)
   integer, intent(inout) :: comm_keyval
@@ -627,7 +805,7 @@ end subroutine MPI_Comm_free_keyval
 end interface
 
 
-interface MPI_Comm_get_attr
+interface
 
 subroutine MPI_Comm_get_attr(comm, comm_keyval, attribute_val, flag, ierror)
   include 'mpif-config.h'
@@ -641,7 +819,7 @@ end subroutine MPI_Comm_get_attr
 end interface
 
 
-interface MPI_Comm_get_errhandler
+interface
 
 subroutine MPI_Comm_get_errhandler(comm, erhandler, ierror)
   integer, intent(in) :: comm
@@ -652,7 +830,7 @@ end subroutine MPI_Comm_get_errhandler
 end interface
 
 
-interface MPI_Comm_get_info
+interface
 
 subroutine MPI_Comm_get_info(comm, info_used, ierror)
   integer, intent(in) :: comm
@@ -663,7 +841,7 @@ end subroutine MPI_Comm_get_info
 end interface
 
 
-interface MPI_Comm_get_name
+interface
 
 subroutine MPI_Comm_get_name(comm, comm_name, resultlen, ierror)
   integer, intent(in) :: comm
@@ -675,7 +853,7 @@ end subroutine MPI_Comm_get_name
 end interface
 
 
-interface MPI_Comm_get_parent
+interface
 
 subroutine MPI_Comm_get_parent(parent, ierror)
   integer, intent(out) :: parent
@@ -685,7 +863,7 @@ end subroutine MPI_Comm_get_parent
 end interface
 
 
-interface MPI_Comm_group
+interface
 
 subroutine MPI_Comm_group(comm, group, ierror)
   integer, intent(in) :: comm
@@ -696,7 +874,7 @@ end subroutine MPI_Comm_group
 end interface
 
 
-interface MPI_Comm_idup
+interface
 
 subroutine MPI_Comm_idup(comm, newcomm, request, ierror)
   integer, intent(in) :: comm
@@ -707,8 +885,19 @@ end subroutine MPI_Comm_idup
 
 end interface
 
+interface
+
+subroutine MPI_Comm_idup_with_info(comm, info, newcomm, request, ierror)
+  integer, intent(in) :: comm
+  integer, intent(in) :: info
+  integer, intent(out) :: newcomm
+  integer, intent(out) :: request
+  integer, intent(out) :: ierror
+end subroutine MPI_Comm_idup_with_info
+
+end interface
 
-interface MPI_Comm_join
+interface
 
 subroutine MPI_Comm_join(fd, intercomm, ierror)
   integer, intent(in) :: fd
@@ -719,7 +908,7 @@ end subroutine MPI_Comm_join
 end interface
 
 
-interface MPI_Comm_rank
+interface
 
 subroutine MPI_Comm_rank(comm, rank, ierror)
   integer, intent(in) :: comm
@@ -730,7 +919,7 @@ end subroutine MPI_Comm_rank
 end interface
 
 
-interface MPI_Comm_remote_group
+interface
 
 subroutine MPI_Comm_remote_group(comm, group, ierror)
   integer, intent(in) :: comm
@@ -741,7 +930,7 @@ end subroutine MPI_Comm_remote_group
 end interface
 
 
-interface MPI_Comm_remote_size
+interface
 
 subroutine MPI_Comm_remote_size(comm, size, ierror)
   integer, intent(in) :: comm
@@ -752,7 +941,7 @@ end subroutine MPI_Comm_remote_size
 end interface
 
 
-interface MPI_Comm_set_attr
+interface
 
 subroutine MPI_Comm_set_attr(comm, comm_keyval, attribute_val, ierror)
   include 'mpif-config.h'
@@ -765,7 +954,7 @@ end subroutine MPI_Comm_set_attr
 end interface
 
 
-interface MPI_Comm_set_errhandler
+interface
 
 subroutine MPI_Comm_set_errhandler(comm, errhandler, ierror)
   integer, intent(in) :: comm
@@ -776,7 +965,7 @@ end subroutine MPI_Comm_set_errhandler
 end interface
 
 
-interface MPI_Comm_set_info
+interface
 
 subroutine MPI_Comm_set_info(comm, info, ierror)
   include 'mpif-config.h'
@@ -788,7 +977,7 @@ end subroutine MPI_Comm_set_info
 end interface
 
 
-interface MPI_Comm_set_name
+interface
 
 subroutine MPI_Comm_set_name(comm, comm_name, ierror)
   integer, intent(in) :: comm
@@ -799,7 +988,7 @@ end subroutine MPI_Comm_set_name
 end interface
 
 
-interface MPI_Comm_size
+interface
 
 subroutine MPI_Comm_size(comm, size, ierror)
   integer, intent(in) :: comm
@@ -810,7 +999,7 @@ end subroutine MPI_Comm_size
 end interface
 
 
-interface MPI_Comm_spawn
+interface
 
 subroutine MPI_Comm_spawn(command, argv, maxprocs, info, root, &
         comm, intercomm, array_of_errcodes, ierror)
@@ -828,7 +1017,7 @@ end subroutine MPI_Comm_spawn
 end interface
 
 
-interface MPI_Comm_spawn_multiple
+interface
 
 subroutine MPI_Comm_spawn_multiple(count, array_of_commands, array_of_argv, array_of_maxprocs, array_of_info, &
         root, comm, intercomm, array_of_errcodes, ierror)
@@ -847,7 +1036,7 @@ end subroutine MPI_Comm_spawn_multiple
 end interface
 
 
-interface MPI_Comm_split
+interface
 
 subroutine MPI_Comm_split(comm, color, key, newcomm, ierror)
   integer, intent(in) :: comm
@@ -860,7 +1049,7 @@ end subroutine MPI_Comm_split
 end interface
 
 
-interface MPI_Comm_split_type
+interface
 
 subroutine MPI_Comm_split_type(comm, split_type, key, info, newcomm, ierror)
   integer, intent(in) :: comm
@@ -874,7 +1063,7 @@ end subroutine MPI_Comm_split_type
 end interface
 
 
-interface MPI_Comm_test_inter
+interface
 
 subroutine MPI_Comm_test_inter(comm, flag, ierror)
   integer, intent(in) :: comm
@@ -885,7 +1074,7 @@ end subroutine MPI_Comm_test_inter
 end interface
 
 
-interface MPI_Compare_and_swap
+interface
 
 subroutine MPI_Compare_and_swap(origin_addr, compare_addr, result_addr, &
                                 datatype, target_rank, target_disp, win, ierror)
@@ -904,7 +1093,7 @@ end subroutine MPI_Compare_and_swap
 end interface
 
 
-interface MPI_Dims_create
+interface
 
 subroutine MPI_Dims_create(nnodes, ndims, dims, ierror)
   integer, intent(in) :: nnodes
@@ -916,7 +1105,7 @@ end subroutine MPI_Dims_create
 end interface
 
 
-interface MPI_Dist_graph_create
+interface
 
 subroutine MPI_Dist_graph_create(comm_old,n,sources,degrees,destinations,weights, &
       info,reorder,comm_dist_graph,ierror)
@@ -932,7 +1121,7 @@ end subroutine MPI_Dist_graph_create
 end interface
 
 
-interface MPI_Dist_graph_create_adjacent
+interface
 
 subroutine MPI_Dist_graph_create_adjacent(comm_old,indegree,sources,sourceweights, &
       outdegree,destinations,destweights,info,reorder, &
@@ -950,7 +1139,7 @@ end subroutine MPI_Dist_graph_create_adjacent
 end interface
 
 
-interface MPI_Dist_graph_neighbors
+interface
 
 subroutine MPI_Dist_graph_neighbors(comm,maxindegree,sources,sourceweights, &
       maxoutdegree,destinations,destweights,ierror)
@@ -965,7 +1154,7 @@ end subroutine MPI_Dist_graph_neighbors
 end interface
 
 
-interface MPI_Dist_graph_neighbors_count
+interface
 
 subroutine MPI_Dist_graph_neighbors_count(comm,indegree,outdegree,weighted,ierror)
    implicit none
@@ -978,7 +1167,7 @@ end subroutine MPI_Dist_graph_neighbors_count
 end interface
 
 
-interface MPI_Errhandler_free
+interface
 
 subroutine MPI_Errhandler_free(errhandler, ierror)
   integer, intent(inout) :: errhandler
@@ -988,7 +1177,7 @@ end subroutine MPI_Errhandler_free
 end interface
 
 
-interface MPI_Error_class
+interface
 
 subroutine MPI_Error_class(errorcode, errorclass, ierror)
   integer, intent(in) :: errorcode
@@ -999,7 +1188,7 @@ end subroutine MPI_Error_class
 end interface
 
 
-interface MPI_Error_string
+interface
 
 subroutine MPI_Error_string(errorcode, string, resultlen, ierror)
   integer, intent(in) :: errorcode
@@ -1011,7 +1200,7 @@ end subroutine MPI_Error_string
 end interface
 
 
-interface MPI_Exscan
+interface
 
 subroutine MPI_Exscan(sendbuf, recvbuf, count, datatype, op, &
         comm, ierror)
@@ -1029,7 +1218,27 @@ end subroutine MPI_Exscan
 end interface
 
 
-interface MPI_F_sync_reg
+interface
+
+subroutine MPI_Exscan_init(sendbuf, recvbuf, count, datatype, op, &
+        comm, info, request, ierror)
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@, intent(in) :: sendbuf
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ recvbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
+  integer, intent(in) :: count
+  integer, intent(in) :: datatype
+  integer, intent(in) :: op
+  integer, intent(in) :: comm
+  integer, intent(in) :: info
+  integer, intent(out) :: request
+  integer, intent(out) :: ierror
+end subroutine MPI_Exscan_init
+
+end interface
+
+
+interface
 
 subroutine MPI_F_sync_reg(buf)
   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
@@ -1039,7 +1248,7 @@ end subroutine MPI_F_sync_reg
 end interface
 
 
-interface MPI_Fetch_and_op
+interface
 
 subroutine MPI_Fetch_and_op(origin_addr, result_addr, datatype, target_rank,&
                             target_disp, op, win, ierror)
@@ -1059,7 +1268,7 @@ end subroutine MPI_Fetch_and_op
 end interface
 
 
-interface MPI_Finalize
+interface
 
 subroutine MPI_Finalize(ierror)
   integer, intent(out) :: ierror
@@ -1068,7 +1277,7 @@ end subroutine MPI_Finalize
 end interface
 
 
-interface MPI_Finalized
+interface
 
 subroutine MPI_Finalized(flag, ierror)
   logical, intent(out) :: flag
@@ -1078,7 +1287,7 @@ end subroutine MPI_Finalized
 end interface
 
 
-interface MPI_Free_mem
+interface
 
 subroutine MPI_Free_mem(base, ierror)
   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ base
@@ -1089,7 +1298,7 @@ end subroutine MPI_Free_mem
 end interface
 
 
-interface MPI_Gather
+interface
 
 subroutine MPI_Gather(sendbuf, sendcount, sendtype, recvbuf, recvcount, &
         recvtype, root, comm, ierror)
@@ -1109,7 +1318,29 @@ end subroutine MPI_Gather
 end interface
 
 
-interface MPI_Gatherv
+interface
+
+subroutine MPI_Gather_init(sendbuf, sendcount, sendtype, recvbuf, recvcount, &
+        recvtype, root, comm, info, request, ierror)
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@, intent(in) :: sendbuf
+  integer, intent(in) :: sendcount
+  integer, intent(in) :: sendtype
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ recvbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
+  integer, intent(in) :: recvcount
+  integer, intent(in) :: recvtype
+  integer, intent(in) :: root
+  integer, intent(in) :: comm
+  integer, intent(in) :: info
+  integer, intent(out) :: request
+  integer, intent(out) :: ierror
+end subroutine MPI_Gather_init
+
+end interface
+
+
+interface
 
 subroutine MPI_Gatherv(sendbuf, sendcount, sendtype, recvbuf, recvcounts, &
         displs, recvtype, root, comm, ierror)
@@ -1130,7 +1361,30 @@ end subroutine MPI_Gatherv
 end interface
 
 
-interface MPI_Get
+interface
+
+subroutine MPI_Gatherv_init(sendbuf, sendcount, sendtype, recvbuf, recvcounts, &
+        displs, recvtype, root, comm, info, request, ierror)
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@, intent(in) :: sendbuf
+  integer, intent(in) :: sendcount
+  integer, intent(in) :: sendtype
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ recvbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
+  integer, dimension(*), intent(in) :: recvcounts
+  integer, dimension(*), intent(in) :: displs
+  integer, intent(in) :: recvtype
+  integer, intent(in) :: root
+  integer, intent(in) :: comm
+  integer, intent(in) :: info
+  integer, intent(out) :: request
+  integer, intent(out) :: ierror
+end subroutine MPI_Gatherv_init
+
+end interface
+
+
+interface
 
 subroutine MPI_Get(origin_addr, origin_count, origin_datatype, target_rank, target_disp, &
         target_count, target_datatype, win, ierror)
@@ -1150,7 +1404,7 @@ end subroutine MPI_Get
 end interface
 
 
-interface MPI_Get_accumulate
+interface
 
 subroutine MPI_Get_accumulate(origin_addr, origin_count, origin_datatype,&
                               result_addr, result_count, result_datatype,&
@@ -1177,7 +1431,7 @@ end subroutine MPI_Get_accumulate
 end interface
 
 
-interface MPI_Get_address
+interface
 
 subroutine MPI_Get_address(location, address, ierror)
   include 'mpif-config.h'
@@ -1190,7 +1444,7 @@ end subroutine MPI_Get_address
 end interface
 
 
-interface MPI_Get_count
+interface
 
 subroutine MPI_Get_count(status, datatype, count, ierror)
   include 'mpif-config.h'
@@ -1203,7 +1457,7 @@ end subroutine MPI_Get_count
 end interface
 
 
-interface MPI_Get_elements
+interface
 
 subroutine MPI_Get_elements(status, datatype, count, ierror)
   include 'mpif-config.h'
@@ -1216,7 +1470,7 @@ end subroutine MPI_Get_elements
 end interface
 
 
-interface MPI_Get_elements_x
+interface
 
 subroutine MPI_Get_elements_x(status, datatype, count, ierror)
   include 'mpif-config.h'
@@ -1229,7 +1483,7 @@ end subroutine MPI_Get_elements_x
 end interface
 
 
-interface MPI_Get_library_version
+interface
 
 subroutine MPI_Get_library_version(version, resultlen, ierror)
   character(len=*), intent(out) :: version
@@ -1240,7 +1494,7 @@ end subroutine MPI_Get_library_version
 end interface
 
 
-interface MPI_Get_processor_name
+interface
 
 subroutine MPI_Get_processor_name(name, resultlen, ierror)
   character(len=*), intent(out) :: name
@@ -1251,7 +1505,7 @@ end subroutine MPI_Get_processor_name
 end interface
 
 
-interface MPI_Get_version
+interface
 
 subroutine MPI_Get_version(version, subversion, ierror)
   integer, intent(out) :: version
@@ -1262,7 +1516,7 @@ end subroutine MPI_Get_version
 end interface
 
 
-interface MPI_Graph_create
+interface
 
 subroutine MPI_Graph_create(comm_old, nnodes, index, edges, reorder, &
         comm_graph, ierror)
@@ -1278,7 +1532,7 @@ end subroutine MPI_Graph_create
 end interface
 
 
-interface MPI_Graph_get
+interface
 
 subroutine MPI_Graph_get(comm, maxindex, maxedges, index, edges&
         , ierror)
@@ -1293,7 +1547,7 @@ end subroutine MPI_Graph_get
 end interface
 
 
-interface MPI_Graph_map
+interface
 
 subroutine MPI_Graph_map(comm, nnodes, index, edges, newrank&
         , ierror)
@@ -1308,7 +1562,7 @@ end subroutine MPI_Graph_map
 end interface
 
 
-interface MPI_Graph_neighbors
+interface
 
 subroutine MPI_Graph_neighbors(comm, rank, maxneighbors, neighbors, ierror)
   integer, intent(in) :: comm
@@ -1321,7 +1575,7 @@ end subroutine MPI_Graph_neighbors
 end interface
 
 
-interface MPI_Graph_neighbors_count
+interface
 
 subroutine MPI_Graph_neighbors_count(comm, rank, nneighbors, ierror)
   integer, intent(in) :: comm
@@ -1333,7 +1587,7 @@ end subroutine MPI_Graph_neighbors_count
 end interface
 
 
-interface MPI_Graphdims_get
+interface
 
 subroutine MPI_Graphdims_get(comm, nnodes, nedges, ierror)
   integer, intent(in) :: comm
@@ -1345,7 +1599,7 @@ end subroutine MPI_Graphdims_get
 end interface
 
 
-interface MPI_Grequest_complete
+interface
 
 subroutine MPI_Grequest_complete(request, ierror)
   integer, intent(in) :: request
@@ -1355,7 +1609,7 @@ end subroutine MPI_Grequest_complete
 end interface
 
 
-interface MPI_Grequest_start
+interface
 
 subroutine MPI_Grequest_start(query_fn, free_fn, cancel_fn, extra_state, request&
         , ierror)
@@ -1371,7 +1625,7 @@ end subroutine MPI_Grequest_start
 end interface
 
 
-interface MPI_Group_compare
+interface
 
 subroutine MPI_Group_compare(group1, group2, result, ierror)
   integer, intent(in) :: group1
@@ -1383,7 +1637,7 @@ end subroutine MPI_Group_compare
 end interface
 
 
-interface MPI_Group_difference
+interface
 
 subroutine MPI_Group_difference(group1, group2, newgroup, ierror)
   integer, intent(in) :: group1
@@ -1395,7 +1649,7 @@ end subroutine MPI_Group_difference
 end interface
 
 
-interface MPI_Group_excl
+interface
 
 subroutine MPI_Group_excl(group, n, ranks, newgroup, ierror)
   integer, intent(in) :: group
@@ -1408,7 +1662,7 @@ end subroutine MPI_Group_excl
 end interface
 
 
-interface MPI_Group_free
+interface
 
 subroutine MPI_Group_free(group, ierror)
   integer, intent(inout) :: group
@@ -1417,8 +1671,20 @@ end subroutine MPI_Group_free
 
 end interface
 
+interface MPI_Group_from_session_pset
+
+subroutine MPI_Group_from_session_pset(session, pset_name, newgroup, ierror)
+   implicit none
+   integer, INTENT(IN) :: session
+   CHARACTER(LEN=*), INTENT(IN) :: pset_name
+   integer, INTENT(OUT) :: newgroup
+   INTEGER, INTENT(OUT) :: ierror
+   integer :: c_ierror
+end subroutine MPI_Group_from_session_pset
+
+end interface
 
-interface MPI_Group_incl
+interface
 
 subroutine MPI_Group_incl(group, n, ranks, newgroup, ierror)
   integer, intent(in) :: group
@@ -1431,7 +1697,7 @@ end subroutine MPI_Group_incl
 end interface
 
 
-interface MPI_Group_intersection
+interface
 
 subroutine MPI_Group_intersection(group1, group2, newgroup, ierror)
   integer, intent(in) :: group1
@@ -1443,7 +1709,7 @@ end subroutine MPI_Group_intersection
 end interface
 
 
-interface MPI_Group_range_excl
+interface
 
 subroutine MPI_Group_range_excl(group, n, ranges, newgroup, ierror)
   integer, intent(in) :: group
@@ -1456,7 +1722,7 @@ end subroutine MPI_Group_range_excl
 end interface
 
 
-interface MPI_Group_range_incl
+interface
 
 subroutine MPI_Group_range_incl(group, n, ranges, newgroup, ierror)
   integer, intent(in) :: group
@@ -1469,7 +1735,7 @@ end subroutine MPI_Group_range_incl
 end interface
 
 
-interface MPI_Group_rank
+interface
 
 subroutine MPI_Group_rank(group, rank, ierror)
   integer, intent(in) :: group
@@ -1480,7 +1746,7 @@ end subroutine MPI_Group_rank
 end interface
 
 
-interface MPI_Group_size
+interface
 
 subroutine MPI_Group_size(group, size, ierror)
   integer, intent(in) :: group
@@ -1491,7 +1757,7 @@ end subroutine MPI_Group_size
 end interface
 
 
-interface MPI_Group_translate_ranks
+interface
 
 subroutine MPI_Group_translate_ranks(group1, n, ranks1, group2, ranks2&
         , ierror)
@@ -1506,7 +1772,7 @@ end subroutine MPI_Group_translate_ranks
 end interface
 
 
-interface MPI_Group_union
+interface
 
 subroutine MPI_Group_union(group1, group2, newgroup, ierror)
   integer, intent(in) :: group1
@@ -1518,7 +1784,7 @@ end subroutine MPI_Group_union
 end interface
 
 
-interface MPI_Iallgather
+interface
 
 subroutine MPI_Iallgather(sendbuf, sendcount, sendtype, recvbuf, recvcount, &
         recvtype, comm, request, ierror)
@@ -1538,7 +1804,7 @@ end subroutine MPI_Iallgather
 end interface
 
 
-interface MPI_Iallgatherv
+interface
 
 subroutine MPI_Iallgatherv(sendbuf, sendcount, sendtype, recvbuf, recvcounts, &
         displs, recvtype, comm, request, ierror)
@@ -1559,7 +1825,7 @@ end subroutine MPI_Iallgatherv
 end interface
 
 
-interface MPI_Iallreduce
+interface
 
 subroutine MPI_Iallreduce(sendbuf, recvbuf, count, datatype, op, &
         comm, request, ierror)
@@ -1578,7 +1844,7 @@ end subroutine MPI_Iallreduce
 end interface
 
 
-interface MPI_Ialltoall
+interface
 
 subroutine MPI_Ialltoall(sendbuf, sendcount, sendtype, recvbuf, recvcount, &
         recvtype, comm, request, ierror)
@@ -1598,7 +1864,7 @@ end subroutine MPI_Ialltoall
 end interface
 
 
-interface MPI_Ialltoallv
+interface
 
 subroutine MPI_Ialltoallv(sendbuf, sendcounts, sdispls, sendtype, recvbuf, &
         recvcounts, rdispls, recvtype, comm, request, ierror)
@@ -1620,7 +1886,7 @@ end subroutine MPI_Ialltoallv
 end interface
 
 
-interface MPI_Ialltoallw
+interface
 
 subroutine MPI_Ialltoallw(sendbuf, sendcounts, sdispls, sendtypes, recvbuf, &
         recvcounts, rdispls, recvtypes, comm, request, ierror)
@@ -1642,7 +1908,7 @@ end subroutine MPI_Ialltoallw
 end interface
 
 
-interface MPI_Ibarrier
+interface
 
 subroutine MPI_Ibarrier(comm, request, ierror)
   integer, intent(in) :: comm
@@ -1653,7 +1919,7 @@ end subroutine MPI_Ibarrier
 end interface
 
 
-interface MPI_Ibcast
+interface
 
 subroutine MPI_Ibcast(buffer, count, datatype, root, comm&
         , request, ierror)
@@ -1670,7 +1936,7 @@ end subroutine MPI_Ibcast
 end interface
 
 
-interface MPI_Ibsend
+interface
 
 subroutine MPI_Ibsend(buf, count, datatype, dest, tag, &
         comm, request, ierror)
@@ -1688,7 +1954,7 @@ end subroutine MPI_Ibsend
 end interface
 
 
-interface MPI_Iexscan
+interface
 
 subroutine MPI_Iexscan(sendbuf, recvbuf, count, datatype, op, &
         comm, request, ierror)
@@ -1707,7 +1973,7 @@ end subroutine MPI_Iexscan
 end interface
 
 
-interface MPI_Igather
+interface
 
 subroutine MPI_Igather(sendbuf, sendcount, sendtype, recvbuf, recvcount, &
         recvtype, root, comm, request, ierror)
@@ -1728,7 +1994,7 @@ end subroutine MPI_Igather
 end interface
 
 
-interface MPI_Igatherv
+interface
 
 subroutine MPI_Igatherv(sendbuf, sendcount, sendtype, recvbuf, recvcounts, &
         displs, recvtype, root, comm, request, ierror)
@@ -1750,7 +2016,7 @@ end subroutine MPI_Igatherv
 end interface
 
 
-interface MPI_Improbe
+interface
 
 subroutine MPI_Improbe(source, tag, comm, flag, message, status, ierror)
   include 'mpif-config.h'
@@ -1766,7 +2032,7 @@ end subroutine MPI_Improbe
 end interface
 
 
-interface MPI_Imrecv
+interface
 
 subroutine MPI_Imrecv(buf, count, datatype, message, request, ierror)
   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
@@ -1781,7 +2047,7 @@ end subroutine MPI_Imrecv
 end interface
 
 
-interface MPI_Ineighbor_allgather
+interface
 
 subroutine MPI_Ineighbor_allgather(sendbuf, sendcount, sendtype, recvbuf, recvcount, &
         recvtype, comm, request, ierror)
@@ -1801,7 +2067,7 @@ end subroutine MPI_Ineighbor_allgather
 end interface
 
 
-interface MPI_Ineighbor_allgatherv
+interface
 
 subroutine MPI_Ineighbor_allgatherv(sendbuf, sendcount, sendtype, recvbuf, recvcounts, &
         displs, recvtype, comm, request, ierror)
@@ -1822,7 +2088,7 @@ end subroutine MPI_Ineighbor_allgatherv
 end interface
 
 
-interface MPI_Ineighbor_alltoall
+interface
 
 subroutine MPI_Ineighbor_alltoall(sendbuf, sendcount, sendtype, recvbuf, recvcount, &
         recvtype, comm, request, ierror)
@@ -1842,7 +2108,7 @@ end subroutine MPI_Ineighbor_alltoall
 end interface
 
 
-interface MPI_Ineighbor_alltoallv
+interface
 
 subroutine MPI_Ineighbor_alltoallv(sendbuf, sendcounts, sdispls, sendtype, recvbuf, &
         recvcounts, rdispls, recvtype, comm, request, ierror)
@@ -1864,7 +2130,7 @@ end subroutine MPI_Ineighbor_alltoallv
 end interface
 
 
-interface MPI_Ineighbor_alltoallw
+interface
 
 subroutine MPI_Ineighbor_alltoallw(sendbuf, sendcounts, sdispls, sendtypes, recvbuf, &
         recvcounts, rdispls, recvtypes, comm, request, ierror)
@@ -1887,7 +2153,7 @@ end subroutine MPI_Ineighbor_alltoallw
 end interface
 
 
-interface MPI_Info_create
+interface
 
 subroutine MPI_Info_create(info, ierror)
   integer, intent(out) :: info
@@ -1896,8 +2162,16 @@ end subroutine MPI_Info_create
 
 end interface
 
+interface
+
+subroutine MPI_Info_create_env(info, ierror)
+  integer, intent(out) :: info
+  integer, intent(out) :: ierror
+end subroutine MPI_Info_create_env
+
+end interface
 
-interface MPI_Info_delete
+interface
 
 subroutine MPI_Info_delete(info, key, ierror)
   integer, intent(in) :: info
@@ -1908,7 +2182,7 @@ end subroutine MPI_Info_delete
 end interface
 
 
-interface MPI_Info_dup
+interface
 
 subroutine MPI_Info_dup(info, newinfo, ierror)
   integer, intent(in) :: info
@@ -1919,7 +2193,7 @@ end subroutine MPI_Info_dup
 end interface
 
 
-interface MPI_Info_free
+interface
 
 subroutine MPI_Info_free(info, ierror)
   integer, intent(inout) :: info
@@ -1929,7 +2203,7 @@ end subroutine MPI_Info_free
 end interface
 
 
-interface MPI_Info_get
+interface
 
 subroutine MPI_Info_get(info, key, valuelen, value, flag&
         , ierror)
@@ -1944,7 +2218,7 @@ end subroutine MPI_Info_get
 end interface
 
 
-interface MPI_Info_get_nkeys
+interface
 
 subroutine MPI_Info_get_nkeys(info, nkeys, ierror)
   integer, intent(in) :: info
@@ -1955,7 +2229,7 @@ end subroutine MPI_Info_get_nkeys
 end interface
 
 
-interface MPI_Info_get_nthkey
+interface
 
 subroutine MPI_Info_get_nthkey(info, n, key, ierror)
   integer, intent(in) :: info
@@ -1966,8 +2240,21 @@ end subroutine MPI_Info_get_nthkey
 
 end interface
 
+interface
+
+subroutine MPI_Info_get_string(info, key, buflen, value, flag, ierror)
+  integer, intent(in) :: info
+  character(len=*), intent(in) :: key
+  integer, intent(inout) :: buflen
+  character(len=*), intent(out) :: value
+  logical, intent(out) :: flag
+  integer, intent(out) :: ierror
+end subroutine MPI_Info_get_string
+
+end interface
+
 
-interface MPI_Info_get_valuelen
+interface
 
 subroutine MPI_Info_get_valuelen(info, key, valuelen, flag, ierror)
   integer, intent(in) :: info
@@ -1980,7 +2267,7 @@ end subroutine MPI_Info_get_valuelen
 end interface
 
 
-interface MPI_Info_set
+interface
 
 subroutine MPI_Info_set(info, key, value, ierror)
   integer, intent(in) :: info
@@ -1992,7 +2279,7 @@ end subroutine MPI_Info_set
 end interface
 
 
-interface MPI_Init
+interface
 
 subroutine MPI_Init(ierror)
   integer, intent(out) :: ierror
@@ -2001,7 +2288,7 @@ end subroutine MPI_Init
 end interface
 
 
-interface MPI_Init_thread
+interface
 
 subroutine MPI_Init_thread(required, provided, ierror)
   integer, intent(in) :: required
@@ -2012,7 +2299,7 @@ end subroutine MPI_Init_thread
 end interface
 
 
-interface MPI_Initialized
+interface
 
 subroutine MPI_Initialized(flag, ierror)
   logical, intent(out) :: flag
@@ -2022,7 +2309,7 @@ end subroutine MPI_Initialized
 end interface
 
 
-interface MPI_Intercomm_create
+interface
 
 subroutine MPI_Intercomm_create(local_comm, local_leader, bridge_comm, remote_leader, tag, &
         newintercomm, ierror)
@@ -2037,20 +2324,36 @@ end subroutine MPI_Intercomm_create
 
 end interface
 
+interface MPI_Intercomm_create_from_groups
+
+subroutine MPI_Intercomm_create_from_groups(local_group, local_leader, remote_group, &
+                                            remote_leader, stringtag, info, errhandler, &
+                                            newintercomm, ierror)
+   implicit none
+   integer, INTENT(IN) :: local_group, remote_group
+   integer, INTENT(IN):: local_leader, remote_leader
+   CHARACTER(LEN=*), INTENT(IN) :: stringtag
+   integer, INTENT(IN) :: info
+   integer, INTENT(IN) :: errhandler
+   integer, INTENT(OUT) :: newintercomm
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine MPI_Intercomm_create_from_groups
+
+end interface
 
-interface MPI_Intercomm_merge
+interface
 
-subroutine MPI_Intercomm_merge(intercomm, high, newintercomm, ierror)
+subroutine MPI_Intercomm_merge(intercomm, high, newintracomm, ierror)
   integer, intent(in) :: intercomm
   logical, intent(in) :: high
-  integer, intent(out) :: newintercomm
+  integer, intent(out) :: newintracomm
   integer, intent(out) :: ierror
 end subroutine MPI_Intercomm_merge
 
 end interface
 
 
-interface MPI_Iprobe
+interface
 
 subroutine MPI_Iprobe(source, tag, comm, flag, status&
         , ierror)
@@ -2066,7 +2369,7 @@ end subroutine MPI_Iprobe
 end interface
 
 
-interface MPI_Irecv
+interface
 
 subroutine MPI_Irecv(buf, count, datatype, source, tag, &
         comm, request, ierror)
@@ -2084,7 +2387,7 @@ end subroutine MPI_Irecv
 end interface
 
 
-interface MPI_Ireduce
+interface
 
 subroutine MPI_Ireduce(sendbuf, recvbuf, count, datatype, op, &
         root, comm, request, ierror)
@@ -2104,7 +2407,7 @@ end subroutine MPI_Ireduce
 end interface
 
 
-interface MPI_Ireduce_scatter
+interface
 
 subroutine MPI_Ireduce_scatter(sendbuf, recvbuf, recvcounts, datatype, op, &
         comm, request, ierror)
@@ -2123,7 +2426,7 @@ end subroutine MPI_Ireduce_scatter
 end interface
 
 
-interface MPI_Ireduce_scatter_block
+interface
 
 subroutine MPI_Ireduce_scatter_block(sendbuf, recvbuf, recvcount, datatype, op, &
         comm, request, ierror)
@@ -2142,7 +2445,7 @@ end subroutine MPI_Ireduce_scatter_block
 end interface
 
 
-interface MPI_Irsend
+interface
 
 subroutine MPI_Irsend(buf, count, datatype, dest, tag, &
         comm, request, ierror)
@@ -2160,7 +2463,7 @@ end subroutine MPI_Irsend
 end interface
 
 
-interface MPI_Is_thread_main
+interface
 
 subroutine MPI_Is_thread_main(flag, ierror)
   logical, intent(out) :: flag
@@ -2170,7 +2473,7 @@ end subroutine MPI_Is_thread_main
 end interface
 
 
-interface MPI_Iscan
+interface
 
 subroutine MPI_Iscan(sendbuf, recvbuf, count, datatype, op, &
         comm, request, ierror)
@@ -2189,7 +2492,7 @@ end subroutine MPI_Iscan
 end interface
 
 
-interface MPI_Iscatter
+interface
 
 subroutine MPI_Iscatter(sendbuf, sendcount, sendtype, recvbuf, recvcount, &
         recvtype, root, comm, request, ierror)
@@ -2210,7 +2513,7 @@ end subroutine MPI_Iscatter
 end interface
 
 
-interface MPI_Iscatterv
+interface
 
 subroutine MPI_Iscatterv(sendbuf, sendcounts, displs, sendtype, recvbuf, &
         recvcount, recvtype, root, comm, request, ierror)
@@ -2232,7 +2535,7 @@ end subroutine MPI_Iscatterv
 end interface
 
 
-interface MPI_Isend
+interface
 
 subroutine MPI_Isend(buf, count, datatype, dest, tag, &
         comm, request, ierror)
@@ -2249,8 +2552,52 @@ end subroutine MPI_Isend
 
 end interface
 
+interface
+
+subroutine MPI_Isendrecv(sendbuf, sendcount, sendtype, dest, sendtag, &
+        recvbuf, recvcount, recvtype, source, recvtag, comm, request, ierror)
+  include 'mpif-config.h'
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@, intent(in) :: sendbuf
+  integer, intent(in) :: sendcount
+  integer, intent(in) :: sendtype
+  integer, intent(in) :: dest
+  integer, intent(in) :: sendtag
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ recvbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
+  integer, intent(in) :: recvcount
+  integer, intent(in) :: recvtype
+  integer, intent(in) :: source
+  integer, intent(in) :: recvtag
+  integer, intent(in) :: comm
+  integer, intent(out) :: request
+  integer, intent(out) :: ierror
+end subroutine MPI_Isendrecv
+
+end interface
+
+interface
+
+subroutine MPI_Isendrecv_replace(buf, count, datatype, dest, sendtag, &
+        source, recvtag, comm, request, ierror)
+  include 'mpif-config.h'
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: buf
+  integer, intent(in) :: count
+  integer, intent(in) :: datatype
+  integer, intent(in) :: dest
+  integer, intent(in) :: sendtag
+  integer, intent(in) :: source
+  integer, intent(in) :: recvtag
+  integer, intent(in) :: comm
+  integer, intent(out) :: request
+  integer, intent(out) :: ierror
+end subroutine MPI_Isendrecv_replace
+
+end interface
+
 
-interface MPI_Issend
+interface
 
 subroutine MPI_Issend(buf, count, datatype, dest, tag, &
         comm, request, ierror)
@@ -2268,19 +2615,108 @@ end subroutine MPI_Issend
 end interface
 
 
-interface MPI_Lookup_name
+interface
 
-subroutine MPI_Lookup_name(service_name, info, port_name, ierror)
-  character(len=*), intent(in) :: service_name
-  integer, intent(in) :: info
-  character(len=*), intent(out) :: port_name
+subroutine MPI_Psend_init(buf, partitions, count, datatype, dest, tag, &
+        comm, info, request, ierror)
+  include 'mpif-config.h'
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@, intent(in) :: buf
+  integer, intent(in) :: partitions 
+  integer(KIND=MPI_COUNT_KIND), intent(in) :: count
+  integer, intent(in) :: datatype
+  integer, intent(in) :: dest
+  integer, intent(in) :: tag
+  integer, intent(in) :: comm
+  integer, intent(in) :: info
+  integer, intent(out) :: request
+  integer, intent(out) :: ierror
+end subroutine MPI_Psend_init
+
+end interface
+
+
+interface
+
+subroutine MPI_Precv_init(buf, partitions, count, datatype, dest, tag, &
+        comm, info, request, ierror)
+  include 'mpif-config.h'
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ buf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@, intent(in) :: buf
+  integer, intent(in) :: partitions
+  integer(KIND=MPI_COUNT_KIND), intent(in) :: count
+  integer, intent(in) :: datatype
+  integer, intent(in) :: dest
+  integer, intent(in) :: tag
+  integer, intent(in) :: comm
+  integer, intent(in) :: info
+  integer, intent(out) :: request
+  integer, intent(out) :: ierror
+end subroutine MPI_Precv_init
+
+end interface
+
+
+interface
+
+subroutine MPI_Pready(partition, request, ierror)
+  integer, intent(in) :: partition
+  integer, intent(in) :: request
+  integer, intent(out) :: ierror
+end subroutine MPI_Pready
+
+end interface
+
+
+interface
+
+subroutine MPI_Pready_list(length, partitions, request, ierror)
+  integer, intent(in) :: length
+  integer, dimension(*), intent(in) :: partitions
+  integer, intent(in) :: request
+  integer, intent(out) :: ierror
+end subroutine MPI_Pready_list
+
+end interface
+
+
+interface
+
+subroutine MPI_Pready_range(partition_low, partition_high, request, ierror)
+  integer, intent(in) :: partition_low
+  integer, intent(in) :: partition_high
+  integer, intent(in) :: request
+  integer, intent(out) :: ierror
+end subroutine MPI_Pready_range
+
+end interface
+
+
+interface
+
+subroutine MPI_Parrived(request, partition, flag, ierror)
+  integer, intent(in) :: request
+  integer, intent(in) :: partition
+  logical, intent(in) :: flag
+  integer, intent(out) :: ierror
+end subroutine MPI_Parrived
+
+end interface
+
+
+interface
+
+subroutine MPI_Lookup_name(service_name, info, port_name, ierror)
+  character(len=*), intent(in) :: service_name
+  integer, intent(in) :: info
+  character(len=*), intent(out) :: port_name
   integer, intent(out) :: ierror
 end subroutine MPI_Lookup_name
 
 end interface
 
 
-interface MPI_Mprobe
+interface
 
 subroutine MPI_Mprobe(source, tag, comm, message, status, ierror)
   include 'mpif-config.h'
@@ -2295,7 +2731,7 @@ end subroutine MPI_Mprobe
 end interface
 
 
-interface MPI_Mrecv
+interface
 
 subroutine MPI_Mrecv(buf, count, datatype, message, status, ierror)
   include 'mpif-config.h'
@@ -2311,7 +2747,7 @@ end subroutine MPI_Mrecv
 end interface
 
 
-interface MPI_Neighbor_allgather
+interface
 
 subroutine MPI_Neighbor_allgather(sendbuf, sendcount, sendtype, recvbuf, recvcount, &
         recvtype, comm, ierror)
@@ -2330,7 +2766,28 @@ end subroutine MPI_Neighbor_allgather
 end interface
 
 
-interface MPI_Neighbor_allgatherv
+interface
+
+subroutine MPI_Neighbor_allgather_init(sendbuf, sendcount, sendtype, recvbuf, recvcount, &
+        recvtype, comm, info, request, ierror)
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@, intent(in) :: sendbuf
+  integer, intent(in) :: sendcount
+  integer, intent(in) :: sendtype
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ recvbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
+  integer, intent(in) :: recvcount
+  integer, intent(in) :: recvtype
+  integer, intent(in) :: comm
+  integer, intent(in) :: info
+  integer, intent(out) :: request
+  integer, intent(out) :: ierror
+end subroutine MPI_Neighbor_allgather_init
+
+end interface
+
+
+interface
 
 subroutine MPI_Neighbor_allgatherv(sendbuf, sendcount, sendtype, recvbuf, recvcounts, &
         displs, recvtype, comm, ierror)
@@ -2350,7 +2807,29 @@ end subroutine MPI_Neighbor_allgatherv
 end interface
 
 
-interface MPI_Neighbor_alltoall
+interface
+
+subroutine MPI_Neighbor_allgatherv_init(sendbuf, sendcount, sendtype, recvbuf, recvcounts, &
+        displs, recvtype, comm, info, request, ierror)
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@, intent(in) :: sendbuf
+  integer, intent(in) :: sendcount
+  integer, intent(in) :: sendtype
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ recvbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
+  integer, dimension(*), intent(in) :: recvcounts
+  integer, dimension(*), intent(in) :: displs
+  integer, intent(in) :: recvtype
+  integer, intent(in) :: comm
+  integer, intent(in) :: info
+  integer, intent(out) :: request
+  integer, intent(out) :: ierror
+end subroutine MPI_Neighbor_allgatherv_init
+
+end interface
+
+
+interface
 
 subroutine MPI_Neighbor_alltoall(sendbuf, sendcount, sendtype, recvbuf, recvcount, &
         recvtype, comm, ierror)
@@ -2369,7 +2848,28 @@ end subroutine MPI_Neighbor_alltoall
 end interface
 
 
-interface MPI_Neighbor_alltoallv
+interface
+
+subroutine MPI_Neighbor_alltoall_init(sendbuf, sendcount, sendtype, recvbuf, recvcount, &
+        recvtype, comm, info, request, ierror)
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@, intent(in) :: sendbuf
+  integer, intent(in) :: sendcount
+  integer, intent(in) :: sendtype
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ recvbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
+  integer, intent(in) :: recvcount
+  integer, intent(in) :: recvtype
+  integer, intent(in) :: comm
+  integer, intent(in) :: info
+  integer, intent(out) :: request
+  integer, intent(out) :: ierror
+end subroutine MPI_Neighbor_alltoall_init
+
+end interface
+
+
+interface
 
 subroutine MPI_Neighbor_alltoallv(sendbuf, sendcounts, sdispls, sendtype, recvbuf, &
         recvcounts, rdispls, recvtype, comm, ierror)
@@ -2390,7 +2890,30 @@ end subroutine MPI_Neighbor_alltoallv
 end interface
 
 
-interface MPI_Neighbor_alltoallw
+interface
+
+subroutine MPI_Neighbor_alltoallv_init(sendbuf, sendcounts, sdispls, sendtype, recvbuf, &
+        recvcounts, rdispls, recvtype, comm, info, request, ierror)
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@, intent(in) :: sendbuf
+  integer, dimension(*), intent(in) :: sendcounts
+  integer, dimension(*), intent(in) :: sdispls
+  integer, intent(in) :: sendtype
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ recvbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
+  integer, dimension(*), intent(in) :: recvcounts
+  integer, dimension(*), intent(in) :: rdispls
+  integer, intent(in) :: recvtype
+  integer, intent(in) :: comm
+  integer, intent(in) :: info
+  integer, intent(out) :: request
+  integer, intent(out) :: ierror
+end subroutine MPI_Neighbor_alltoallv_init
+
+end interface
+
+
+interface
 
 subroutine MPI_Neighbor_alltoallw(sendbuf, sendcounts, sdispls, sendtypes, recvbuf, &
         recvcounts, rdispls, recvtypes, comm, ierror)
@@ -2412,7 +2935,31 @@ end subroutine MPI_Neighbor_alltoallw
 end interface
 
 
-interface MPI_Op_commutative
+interface
+
+subroutine MPI_Neighbor_alltoallw_init(sendbuf, sendcounts, sdispls, sendtypes, recvbuf, &
+        recvcounts, rdispls, recvtypes, comm, info, request, ierror)
+  include 'mpif-config.h'
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@, intent(in) :: sendbuf
+  integer, dimension(*), intent(in) :: sendcounts
+  integer(kind=MPI_ADDRESS_KIND), dimension(*), intent(in) :: sdispls
+  integer, dimension(*), intent(in) :: sendtypes
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ recvbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
+  integer, dimension(*), intent(in) :: recvcounts
+  integer(kind=MPI_ADDRESS_KIND), dimension(*), intent(in) :: rdispls
+  integer, dimension(*), intent(in) :: recvtypes
+  integer, intent(in) :: comm
+  integer, intent(in) :: info
+  integer, intent(out) :: request
+  integer, intent(out) :: ierror
+end subroutine MPI_Neighbor_alltoallw_init
+
+end interface
+
+
+interface
 
 subroutine MPI_Op_commutative(op, commute, ierror)
   integer, intent(in) :: op
@@ -2423,7 +2970,7 @@ end subroutine MPI_Op_commutative
 end interface
 
 
-interface MPI_Op_create
+interface
 
 subroutine MPI_Op_create(function, commute, op, ierror)
   external :: function
@@ -2435,7 +2982,7 @@ end subroutine MPI_Op_create
 end interface
 
 
-interface MPI_Op_free
+interface
 
 subroutine MPI_Op_free(op, ierror)
   integer, intent(inout) :: op
@@ -2445,7 +2992,7 @@ end subroutine MPI_Op_free
 end interface
 
 
-interface MPI_Open_port
+interface
 
 subroutine MPI_Open_port(info, port_name, ierror)
   integer, intent(in) :: info
@@ -2456,7 +3003,7 @@ end subroutine MPI_Open_port
 end interface
 
 
-interface MPI_Pack
+interface
 
 subroutine MPI_Pack(inbuf, incount, datatype, outbuf, outsize, &
         position, comm, ierror)
@@ -2475,7 +3022,7 @@ end subroutine MPI_Pack
 end interface
 
 
-interface MPI_Pack_external
+interface
 
 subroutine MPI_Pack_external(datarep, inbuf, incount, datatype, outbuf, &
         outsize, position, ierror)
@@ -2495,7 +3042,7 @@ end subroutine MPI_Pack_external
 end interface
 
 
-interface MPI_Pack_external_size
+interface
 
 subroutine MPI_Pack_external_size(datarep, incount, datatype, size, ierror)
   include 'mpif-config.h'
@@ -2509,7 +3056,7 @@ end subroutine MPI_Pack_external_size
 end interface
 
 
-interface MPI_Pack_size
+interface
 
 subroutine MPI_Pack_size(incount, datatype, comm, size, ierror)
   integer, intent(in) :: incount
@@ -2522,7 +3069,7 @@ end subroutine MPI_Pack_size
 end interface
 
 
-interface MPI_Pcontrol
+interface
 
 subroutine MPI_Pcontrol(level)
   integer, intent(in) :: level
@@ -2532,7 +3079,7 @@ end subroutine MPI_Pcontrol
 end interface
 
 
-interface MPI_Probe
+interface
 
 subroutine MPI_Probe(source, tag, comm, status, ierror)
   include 'mpif-config.h'
@@ -2546,7 +3093,7 @@ end subroutine MPI_Probe
 end interface
 
 
-interface MPI_Publish_name
+interface
 
 subroutine MPI_Publish_name(service_name, info, port_name, ierror)
   character(len=*), intent(in) :: service_name
@@ -2558,7 +3105,7 @@ end subroutine MPI_Publish_name
 end interface
 
 
-interface MPI_Put
+interface
 
 subroutine MPI_Put(origin_addr, origin_count, origin_datatype, target_rank, target_disp, &
         target_count, target_datatype, win, ierror)
@@ -2578,7 +3125,7 @@ end subroutine MPI_Put
 end interface
 
 
-interface MPI_Query_thread
+interface
 
 subroutine MPI_Query_thread(provided, ierror)
   integer, intent(out) :: provided
@@ -2588,7 +3135,7 @@ end subroutine MPI_Query_thread
 end interface
 
 
-interface MPI_Raccumulate
+interface
 
 subroutine MPI_Raccumulate(origin_addr, origin_count, origin_datatype,&
                            target_rank, target_disp, target_count, &
@@ -2611,7 +3158,7 @@ end subroutine MPI_Raccumulate
 end interface
 
 
-interface MPI_Recv
+interface
 
 subroutine MPI_Recv(buf, count, datatype, source, tag, &
         comm, status, ierror)
@@ -2630,7 +3177,7 @@ end subroutine MPI_Recv
 end interface
 
 
-interface MPI_Recv_init
+interface
 
 subroutine MPI_Recv_init(buf, count, datatype, source, tag, &
         comm, request, ierror)
@@ -2648,7 +3195,7 @@ end subroutine MPI_Recv_init
 end interface
 
 
-interface MPI_Reduce
+interface
 
 subroutine MPI_Reduce(sendbuf, recvbuf, count, datatype, op, &
         root, comm, ierror)
@@ -2667,7 +3214,28 @@ end subroutine MPI_Reduce
 end interface
 
 
-interface MPI_Reduce_local
+interface
+
+subroutine MPI_Reduce_init(sendbuf, recvbuf, count, datatype, op, &
+        root, comm, info, request, ierror)
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@, intent(in) :: sendbuf
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ recvbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
+  integer, intent(in) :: count
+  integer, intent(in) :: datatype
+  integer, intent(in) :: op
+  integer, intent(in) :: root
+  integer, intent(in) :: comm
+  integer, intent(in) :: info
+  integer, intent(out) :: request
+  integer, intent(out) :: ierror
+end subroutine MPI_Reduce_init
+
+end interface
+
+
+interface
 
 subroutine MPI_Reduce_local(inbuf, inout, count, datatype, op, &
         ierror)
@@ -2684,7 +3252,7 @@ end subroutine MPI_Reduce_local
 end interface
 
 
-interface MPI_Reduce_scatter
+interface
 
 subroutine MPI_Reduce_scatter(sendbuf, recvbuf, recvcounts, datatype, op, &
         comm, ierror)
@@ -2702,7 +3270,27 @@ end subroutine MPI_Reduce_scatter
 end interface
 
 
-interface MPI_Reduce_scatter_block
+interface
+
+subroutine MPI_Reduce_scatter_init(sendbuf, recvbuf, recvcounts, datatype, op, &
+        comm, info, request, ierror)
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@, intent(in) :: sendbuf
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ recvbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
+  integer, dimension(*), intent(in) :: recvcounts
+  integer, intent(in) :: datatype
+  integer, intent(in) :: op
+  integer, intent(in) :: comm
+  integer, intent(in) :: info
+  integer, intent(out) :: request
+  integer, intent(out) :: ierror
+end subroutine MPI_Reduce_scatter_init
+
+end interface
+
+
+interface
 
 subroutine MPI_Reduce_scatter_block(sendbuf, recvbuf, recvcount, datatype, op, &
         comm, ierror)
@@ -2720,7 +3308,27 @@ end subroutine MPI_Reduce_scatter_block
 end interface
 
 
-interface MPI_Register_datarep
+interface
+
+subroutine MPI_Reduce_scatter_block_init(sendbuf, recvbuf, recvcount, datatype, op, &
+        comm, info, request, ierror)
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@, intent(in) :: sendbuf
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ recvbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
+  integer, intent(in) :: recvcount
+  integer, intent(in) :: datatype
+  integer, intent(in) :: op
+  integer, intent(in) :: comm
+  integer, intent(in) :: info
+  integer, intent(out) :: request
+  integer, intent(out) :: ierror
+end subroutine MPI_Reduce_scatter_block_init
+
+end interface
+
+
+interface
 
 subroutine MPI_Register_datarep(datarep, read_conversion_fn, write_conversion_fn, dtype_file_extent_fn, extra_state&
         , ierror)
@@ -2736,7 +3344,7 @@ end subroutine MPI_Register_datarep
 end interface
 
 
-interface MPI_Request_free
+interface
 
 subroutine MPI_Request_free(request, ierror)
   integer, intent(inout) :: request
@@ -2746,7 +3354,7 @@ end subroutine MPI_Request_free
 end interface
 
 
-interface MPI_Request_get_status
+interface
 
 subroutine MPI_Request_get_status(request, flag, status, ierror)
   include 'mpif-config.h'
@@ -2759,7 +3367,7 @@ end subroutine MPI_Request_get_status
 end interface
 
 
-interface MPI_Rget
+interface
 
 subroutine MPI_Rget(origin_addr, origin_count, origin_datatype, &
                     target_rank, target_disp, target_count, &
@@ -2781,7 +3389,7 @@ end subroutine MPI_Rget
 end interface
 
 
-interface MPI_Rget_accumulate
+interface
 
 subroutine MPI_Rget_accumulate(origin_addr, origin_count, origin_datatype,&
                                result_addr, result_count, result_datatype,&
@@ -2809,7 +3417,7 @@ end subroutine MPI_Rget_accumulate
 end interface
 
 
-interface MPI_Rput
+interface
 
 subroutine MPI_Rput(origin_addr, origin_count, origin_datatype, &
                     target_rank, target_disp, target_count, &
@@ -2831,7 +3439,7 @@ end subroutine MPI_Rput
 end interface
 
 
-interface MPI_Rsend
+interface
 
 subroutine MPI_Rsend(ibuf, count, datatype, dest, tag, &
         comm, ierror)
@@ -2848,7 +3456,7 @@ end subroutine MPI_Rsend
 end interface
 
 
-interface MPI_Rsend_init
+interface
 
 subroutine MPI_Rsend_init(buf, count, datatype, dest, tag, &
         comm, request, ierror)
@@ -2866,7 +3474,7 @@ end subroutine MPI_Rsend_init
 end interface
 
 
-interface MPI_Scan
+interface
 
 subroutine MPI_Scan(sendbuf, recvbuf, count, datatype, op, &
         comm, ierror)
@@ -2884,7 +3492,27 @@ end subroutine MPI_Scan
 end interface
 
 
-interface MPI_Scatter
+interface
+
+subroutine MPI_Scan_init(sendbuf, recvbuf, count, datatype, op, &
+        comm, info, request, ierror)
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@, intent(in) :: sendbuf
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ recvbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
+  integer, intent(in) :: count
+  integer, intent(in) :: datatype
+  integer, intent(in) :: op
+  integer, intent(in) :: comm
+  integer, intent(in) :: info
+  integer, intent(out) :: request
+  integer, intent(out) :: ierror
+end subroutine MPI_Scan_init
+
+end interface
+
+
+interface
 
 subroutine MPI_Scatter(sendbuf, sendcount, sendtype, recvbuf, recvcount, &
         recvtype, root, comm, ierror)
@@ -2904,7 +3532,29 @@ end subroutine MPI_Scatter
 end interface
 
 
-interface MPI_Scatterv
+interface
+
+subroutine MPI_Scatter_init(sendbuf, sendcount, sendtype, recvbuf, recvcount, &
+        recvtype, root, comm, info, request, ierror)
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@, intent(in) :: sendbuf
+  integer, intent(in) :: sendcount
+  integer, intent(in) :: sendtype
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ recvbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
+  integer, intent(in) :: recvcount
+  integer, intent(in) :: recvtype
+  integer, intent(in) :: root
+  integer, intent(in) :: comm
+  integer, intent(in) :: info
+  integer, intent(out) :: request
+  integer, intent(out) :: ierror
+end subroutine MPI_Scatter_init
+
+end interface
+
+
+interface
 
 subroutine MPI_Scatterv(sendbuf, sendcounts, displs, sendtype, recvbuf, &
         recvcount, recvtype, root, comm, ierror)
@@ -2925,7 +3575,30 @@ end subroutine MPI_Scatterv
 end interface
 
 
-interface MPI_Send
+interface
+
+subroutine MPI_Scatterv_init(sendbuf, sendcounts, displs, sendtype, recvbuf, &
+        recvcount, recvtype, root, comm, info, request, ierror)
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ sendbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@, intent(in) :: sendbuf
+  integer, dimension(*), intent(in) :: sendcounts
+  integer, dimension(*), intent(in) :: displs
+  integer, intent(in) :: sendtype
+  @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ recvbuf
+  @OMPI_FORTRAN_IGNORE_TKR_TYPE@ :: recvbuf
+  integer, intent(in) :: recvcount
+  integer, intent(in) :: recvtype
+  integer, intent(in) :: root
+  integer, intent(in) :: comm
+  integer, intent(in) :: info
+  integer, intent(out) :: request
+  integer, intent(out) :: ierror
+end subroutine MPI_Scatterv_init
+
+end interface
+
+
+interface
 
 subroutine MPI_Send(buf, count, datatype, dest, tag, &
         comm, ierror)
@@ -2942,7 +3615,7 @@ end subroutine MPI_Send
 end interface
 
 
-interface MPI_Send_init
+interface
 
 subroutine MPI_Send_init(buf, count, datatype, dest, tag, &
         comm, request, ierror)
@@ -2960,7 +3633,7 @@ end subroutine MPI_Send_init
 end interface
 
 
-interface MPI_Sendrecv
+interface
 
 subroutine MPI_Sendrecv(sendbuf, sendcount, sendtype, dest, sendtag, &
         recvbuf, recvcount, recvtype, source, recvtag, comm, status, ierror)
@@ -2985,7 +3658,7 @@ end subroutine MPI_Sendrecv
 end interface
 
 
-interface MPI_Sendrecv_replace
+interface
 
 subroutine MPI_Sendrecv_replace(buf, count, datatype, dest, sendtag, &
         source, recvtag, comm, status, ierror)
@@ -3005,8 +3678,105 @@ end subroutine MPI_Sendrecv_replace
 
 end interface
 
+interface
+
+subroutine MPI_Session_call_errhandler(session, errorcode, ierror)
+  integer, intent(in) :: session
+  integer, intent(in) :: errorcode
+  integer, intent(out) :: ierror
+end subroutine MPI_Session_call_errhandler
+
+end interface
 
-interface MPI_Ssend
+
+interface
+
+subroutine MPI_Session_create_errhandler(function, errhandler, ierror)
+  external :: function
+  integer, intent(out) :: errhandler
+  integer, intent(out) :: ierror
+end subroutine MPI_Session_create_errhandler
+
+end interface
+
+interface
+
+subroutine MPI_Session_get_errhandler(session, erhandler, ierror)
+  integer, intent(in) :: session
+  integer, intent(out) :: erhandler
+  integer, intent(out) :: ierror
+end subroutine MPI_Session_get_errhandler
+
+end interface
+
+
+interface MPI_Session_get_info
+subroutine MPI_Session_get_info(session, info, ierror)
+   integer, INTENT(IN) :: session
+   integer, INTENT(OUT) :: info
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine MPI_Session_get_info
+end interface
+
+interface MPI_Session_get_nth_pset
+subroutine MPI_Session_get_nth_pset(session, info, n, pset_len, pset_name, ierror)
+   integer, INTENT(IN) :: session
+   integer, INTENT(IN) :: info
+   INTEGER, INTENT(IN) :: n
+   INTEGER, INTENT(INOUT) :: pset_len
+   CHARACTER(LEN=*), INTENT(OUT) :: pset_name
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine MPI_Session_get_nth_pset
+end interface
+
+interface MPI_Session_get_nth_psetlen
+subroutine MPI_Session_get_nth_psetlen(session, n, pset_len, ierror)
+   implicit none
+   integer, INTENT(IN) :: session
+   INTEGER, INTENT(IN) :: n
+   INTEGER, INTENT(OUT) :: pset_len
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine MPI_Session_get_nth_psetlen
+end interface 
+
+interface MPI_Session_get_pset_info
+subroutine MPI_Session_get_pset_info(session, pset_name, info, ierror)
+   integer, INTENT(IN) :: session
+   CHARACTER(LEN=*), INTENT(IN) :: pset_name
+   integer, INTENT(OUT) :: info
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine MPI_Session_get_pset_info
+end interface
+
+interface  MPI_Session_init
+subroutine MPI_Session_init(info,errhandler,session,ierror)
+   integer, intent(IN) :: info
+   integer, intent(IN) :: errhandler
+   integer, intent(OUT) :: session
+   integer, intent(OUT) :: ierror
+end subroutine MPI_Session_init
+end interface
+
+interface
+
+subroutine MPI_Session_finalize(session,ierror)
+   integer, intent(inout) :: session
+   integer, intent(OUT) :: ierror
+end subroutine MPI_Session_finalize
+
+end interface
+
+interface
+
+subroutine MPI_Session_set_errhandler(session, erhandler, ierror)
+  integer, intent(in) :: session
+  integer, intent(out) :: erhandler
+  integer, intent(out) :: ierror
+end subroutine MPI_Session_set_errhandler
+
+end interface
+
+interface
 
 subroutine MPI_Ssend(buf, count, datatype, dest, tag, &
         comm, ierror)
@@ -3023,7 +3793,7 @@ end subroutine MPI_Ssend
 end interface
 
 
-interface MPI_Ssend_init
+interface
 
 subroutine MPI_Ssend_init(buf, count, datatype, dest, tag, &
         comm, request, ierror)
@@ -3041,7 +3811,7 @@ end subroutine MPI_Ssend_init
 end interface
 
 
-interface MPI_Start
+interface
 
 subroutine MPI_Start(request, ierror)
   integer, intent(inout) :: request
@@ -3051,7 +3821,7 @@ end subroutine MPI_Start
 end interface
 
 
-interface MPI_Startall
+interface
 
 subroutine MPI_Startall(count, array_of_requests, ierror)
   integer, intent(in) :: count
@@ -3062,7 +3832,7 @@ end subroutine MPI_Startall
 end interface
 
 
-interface MPI_Status_set_cancelled
+interface
 
 subroutine MPI_Status_set_cancelled(status, flag, ierror)
   include 'mpif-config.h'
@@ -3074,7 +3844,7 @@ end subroutine MPI_Status_set_cancelled
 end interface
 
 
-interface MPI_Status_set_elements
+interface
 
 subroutine MPI_Status_set_elements(status, datatype, count, ierror)
   include 'mpif-config.h'
@@ -3087,7 +3857,7 @@ end subroutine MPI_Status_set_elements
 end interface
 
 
-interface MPI_Status_set_elements_x
+interface
 
 subroutine MPI_Status_set_elements_x(status, datatype, count, ierror)
   include 'mpif-config.h'
@@ -3100,7 +3870,7 @@ end subroutine MPI_Status_set_elements_x
 end interface
 
 
-interface MPI_Test
+interface
 
 subroutine MPI_Test(request, flag, status, ierror)
   include 'mpif-config.h'
@@ -3113,7 +3883,7 @@ end subroutine MPI_Test
 end interface
 
 
-interface MPI_Test_cancelled
+interface
 
 subroutine MPI_Test_cancelled(status, flag, ierror)
   include 'mpif-config.h'
@@ -3125,12 +3895,12 @@ end subroutine MPI_Test_cancelled
 end interface
 
 
-interface MPI_Testall
+interface
 
 subroutine MPI_Testall(count, array_of_requests, flag, array_of_statuses, ierror)
   include 'mpif-config.h'
   integer, intent(in) :: count
-  integer, dimension(count), intent(inout) :: array_of_requests
+  integer, dimension(*), intent(inout) :: array_of_requests
   logical, intent(out) :: flag
   integer, dimension(MPI_STATUS_SIZE, *), intent(out) :: array_of_statuses
   integer, intent(out) :: ierror
@@ -3139,13 +3909,13 @@ end subroutine MPI_Testall
 end interface
 
 
-interface MPI_Testany
+interface
 
 subroutine MPI_Testany(count, array_of_requests, index, flag, status&
         , ierror)
   include 'mpif-config.h'
   integer, intent(in) :: count
-  integer, dimension(count), intent(inout) :: array_of_requests
+  integer, dimension(*), intent(inout) :: array_of_requests
   integer, intent(out) :: index
   logical, intent(out) :: flag
   integer, dimension(MPI_STATUS_SIZE), intent(out) :: status
@@ -3155,13 +3925,13 @@ end subroutine MPI_Testany
 end interface
 
 
-interface MPI_Testsome
+interface
 
 subroutine MPI_Testsome(incount, array_of_requests, outcount, array_of_indices, array_of_statuses&
         , ierror)
   include 'mpif-config.h'
   integer, intent(in) :: incount
-  integer, dimension(incount), intent(inout) :: array_of_requests
+  integer, dimension(*), intent(inout) :: array_of_requests
   integer, intent(out) :: outcount
   integer, dimension(*), intent(out) :: array_of_indices
   integer, dimension(MPI_STATUS_SIZE, *), intent(out) :: array_of_statuses
@@ -3171,7 +3941,7 @@ end subroutine MPI_Testsome
 end interface
 
 
-interface MPI_Topo_test
+interface
 
 subroutine MPI_Topo_test(comm, status, ierror)
   integer, intent(in) :: comm
@@ -3182,7 +3952,7 @@ end subroutine MPI_Topo_test
 end interface
 
 
-interface MPI_Type_commit
+interface
 
 subroutine MPI_Type_commit(datatype, ierror)
   integer, intent(inout) :: datatype
@@ -3192,7 +3962,7 @@ end subroutine MPI_Type_commit
 end interface
 
 
-interface MPI_Type_contiguous
+interface
 
 subroutine MPI_Type_contiguous(count, oldtype, newtype, ierror)
   integer, intent(in) :: count
@@ -3204,7 +3974,7 @@ end subroutine MPI_Type_contiguous
 end interface
 
 
-interface MPI_Type_create_darray
+interface
 
 subroutine MPI_Type_create_darray(size, rank, ndims, gsize_array, distrib_array, &
         darg_array, psize_array, order, oldtype, newtype, ierror)
@@ -3224,7 +3994,7 @@ end subroutine MPI_Type_create_darray
 end interface
 
 
-interface MPI_Type_create_f90_complex
+interface
 
 subroutine MPI_Type_create_f90_complex(p, r, newtype, ierror)
   integer, intent(in) :: p
@@ -3236,7 +4006,7 @@ end subroutine MPI_Type_create_f90_complex
 end interface
 
 
-interface MPI_Type_create_f90_integer
+interface
 
 subroutine MPI_Type_create_f90_integer(r, newtype, ierror)
   integer, intent(in) :: r
@@ -3247,7 +4017,7 @@ end subroutine MPI_Type_create_f90_integer
 end interface
 
 
-interface MPI_Type_create_f90_real
+interface
 
 subroutine MPI_Type_create_f90_real(p, r, newtype, ierror)
   integer, intent(in) :: p
@@ -3259,7 +4029,7 @@ end subroutine MPI_Type_create_f90_real
 end interface
 
 
-interface MPI_Type_create_hindexed
+interface
 
 subroutine MPI_Type_create_hindexed(count, array_of_blocklengths, array_of_displacements, oldtype, newtype&
         , ierror)
@@ -3275,7 +4045,7 @@ end subroutine MPI_Type_create_hindexed
 end interface
 
 
-interface MPI_Type_create_hindexed_block
+interface
 
 subroutine MPI_Type_create_hindexed_block(count, blocklength, array_of_displacements, oldtype, newtype&
         , ierror)
@@ -3291,7 +4061,7 @@ end subroutine MPI_Type_create_hindexed_block
 end interface
 
 
-interface MPI_Type_create_hvector
+interface
 
 subroutine MPI_Type_create_hvector(count, blocklength, stride, oldtype, newtype&
         , ierror)
@@ -3307,7 +4077,7 @@ end subroutine MPI_Type_create_hvector
 end interface
 
 
-interface MPI_Type_create_indexed_block
+interface
 
 subroutine MPI_Type_create_indexed_block(count, blocklength, array_of_displacements, oldtype, newtype&
         , ierror)
@@ -3322,7 +4092,7 @@ end subroutine MPI_Type_create_indexed_block
 end interface
 
 
-interface MPI_Type_create_keyval
+interface
 
 subroutine MPI_Type_create_keyval(type_copy_attr_fn, type_delete_attr_fn, type_keyval, extra_state, ierror)
   include 'mpif-config.h'
@@ -3336,7 +4106,7 @@ end subroutine MPI_Type_create_keyval
 end interface
 
 
-interface MPI_Type_create_resized
+interface
 
 subroutine MPI_Type_create_resized(oldtype, lb, extent, newtype, ierror)
   include 'mpif-config.h'
@@ -3350,7 +4120,7 @@ end subroutine MPI_Type_create_resized
 end interface
 
 
-interface MPI_Type_create_struct
+interface
 
 subroutine MPI_Type_create_struct(count, array_of_block_lengths, array_of_displacements, array_of_types, newtype&
         , ierror)
@@ -3366,7 +4136,7 @@ end subroutine MPI_Type_create_struct
 end interface
 
 
-interface MPI_Type_create_subarray
+interface
 
 subroutine MPI_Type_create_subarray(ndims, size_array, subsize_array, start_array, order, &
         oldtype, newtype, ierror)
@@ -3383,7 +4153,7 @@ end subroutine MPI_Type_create_subarray
 end interface
 
 
-interface MPI_Type_delete_attr
+interface
 
 subroutine MPI_Type_delete_attr(datatype, type_keyval, ierror)
   integer, intent(in) :: datatype
@@ -3394,7 +4164,7 @@ end subroutine MPI_Type_delete_attr
 end interface
 
 
-interface MPI_Type_dup
+interface
 
 subroutine MPI_Type_dup(datatype, newtype, ierror)
   integer, intent(in) :: datatype
@@ -3405,7 +4175,7 @@ end subroutine MPI_Type_dup
 end interface
 
 
-interface MPI_Type_free
+interface
 
 subroutine MPI_Type_free(datatype, ierror)
   integer, intent(inout) :: datatype
@@ -3415,7 +4185,7 @@ end subroutine MPI_Type_free
 end interface
 
 
-interface MPI_Type_free_keyval
+interface
 
 subroutine MPI_Type_free_keyval(type_keyval, ierror)
   integer, intent(inout) :: type_keyval
@@ -3425,7 +4195,7 @@ end subroutine MPI_Type_free_keyval
 end interface
 
 
-interface MPI_Type_get_attr
+interface
 
 subroutine MPI_Type_get_attr(datatype, type_keyval, attribute_val, flag, ierror)
   include 'mpif-config.h'
@@ -3439,7 +4209,7 @@ end subroutine MPI_Type_get_attr
 end interface
 
 
-interface MPI_Type_get_contents
+interface
 
 subroutine MPI_Type_get_contents(datatype, max_integers, max_addresses, max_datatypes, array_of_integers, &
         array_of_addresses, array_of_datatypes, ierror)
@@ -3457,7 +4227,7 @@ end subroutine MPI_Type_get_contents
 end interface
 
 
-interface MPI_Type_get_envelope
+interface
 
 subroutine MPI_Type_get_envelope(datatype, num_integers, num_addresses, num_datatypes, combiner&
         , ierror)
@@ -3472,7 +4242,7 @@ end subroutine MPI_Type_get_envelope
 end interface
 
 
-interface MPI_Type_get_extent
+interface
 
 subroutine MPI_Type_get_extent(datatype, lb, extent, ierror)
   include 'mpif-config.h'
@@ -3485,7 +4255,7 @@ end subroutine MPI_Type_get_extent
 end interface
 
 
-interface MPI_Type_get_extent_x
+interface
 
 subroutine MPI_Type_get_extent_x(datatype, lb, extent, ierror)
   include 'mpif-config.h'
@@ -3498,7 +4268,7 @@ end subroutine MPI_Type_get_extent_x
 end interface
 
 
-interface MPI_Type_get_name
+interface
 
 subroutine MPI_Type_get_name(datatype, type_name, resultlen, ierror)
   integer, intent(in) :: datatype
@@ -3510,7 +4280,7 @@ end subroutine MPI_Type_get_name
 end interface
 
 
-interface MPI_Type_get_true_extent
+interface
 
 subroutine MPI_Type_get_true_extent(datatype, true_lb, true_extent, ierror)
   include 'mpif-config.h'
@@ -3523,7 +4293,7 @@ end subroutine MPI_Type_get_true_extent
 end interface
 
 
-interface MPI_Type_get_true_extent_x
+interface
 
 subroutine MPI_Type_get_true_extent_x(datatype, true_lb, true_extent, ierror)
   include 'mpif-config.h'
@@ -3536,7 +4306,7 @@ end subroutine MPI_Type_get_true_extent_x
 end interface
 
 
-interface MPI_Type_indexed
+interface
 
 subroutine MPI_Type_indexed(count, array_of_blocklengths, array_of_displacements, oldtype, newtype&
         , ierror)
@@ -3551,7 +4321,7 @@ end subroutine MPI_Type_indexed
 end interface
 
 
-interface MPI_Type_match_size
+interface
 
 subroutine MPI_Type_match_size(typeclass, size, datatype, ierror)
   integer, intent(in) :: typeclass
@@ -3563,7 +4333,7 @@ end subroutine MPI_Type_match_size
 end interface
 
 
-interface MPI_Type_set_attr
+interface
 
 subroutine MPI_Type_set_attr(datatype, type_keyval, attr_val, ierror)
   include 'mpif-config.h'
@@ -3576,7 +4346,7 @@ end subroutine MPI_Type_set_attr
 end interface
 
 
-interface MPI_Type_set_name
+interface
 
 subroutine MPI_Type_set_name(datatype, type_name, ierror)
   integer, intent(in) :: datatype
@@ -3587,7 +4357,7 @@ end subroutine MPI_Type_set_name
 end interface
 
 
-interface MPI_Type_size
+interface
 
 subroutine MPI_Type_size(datatype, size, ierror)
   integer, intent(in) :: datatype
@@ -3598,7 +4368,7 @@ end subroutine MPI_Type_size
 end interface
 
 
-interface MPI_Type_size_x
+interface
 
 subroutine MPI_Type_size_x(datatype, size, ierror)
   include 'mpif-config.h'
@@ -3610,7 +4380,7 @@ end subroutine MPI_Type_size_x
 end interface
 
 
-interface MPI_Type_vector
+interface
 
 subroutine MPI_Type_vector(count, blocklength, stride, oldtype, newtype&
         , ierror)
@@ -3625,7 +4395,7 @@ end subroutine MPI_Type_vector
 end interface
 
 
-interface MPI_Unpack
+interface
 
 subroutine MPI_Unpack(inbuf, insize, position, outbuf, outcount, &
         datatype, comm, ierror)
@@ -3644,7 +4414,7 @@ end subroutine MPI_Unpack
 end interface
 
 
-interface MPI_Unpack_external
+interface
 
 subroutine MPI_Unpack_external(datarep, inbuf, insize, position, outbuf, &
         outcount, datatype, ierror)
@@ -3664,7 +4434,7 @@ end subroutine MPI_Unpack_external
 end interface
 
 
-interface MPI_Unpublish_name
+interface
 
 subroutine MPI_Unpublish_name(service_name, info, port_name, ierror)
   character(len=*), intent(in) :: service_name
@@ -3676,7 +4446,7 @@ end subroutine MPI_Unpublish_name
 end interface
 
 
-interface MPI_Wait
+interface
 
 subroutine MPI_Wait(request, status, ierror)
   include 'mpif-config.h'
@@ -3688,12 +4458,12 @@ end subroutine MPI_Wait
 end interface
 
 
-interface MPI_Waitall
+interface
 
 subroutine MPI_Waitall(count, array_of_requests, array_of_statuses, ierror)
   include 'mpif-config.h'
   integer, intent(in) :: count
-  integer, dimension(count), intent(inout) :: array_of_requests
+  integer, dimension(*), intent(inout) :: array_of_requests
   integer, dimension(MPI_STATUS_SIZE, *), intent(out) :: array_of_statuses
   integer, intent(out) :: ierror
 end subroutine MPI_Waitall
@@ -3701,12 +4471,12 @@ end subroutine MPI_Waitall
 end interface
 
 
-interface MPI_Waitany
+interface
 
 subroutine MPI_Waitany(count, array_of_requests, index, status, ierror)
   include 'mpif-config.h'
   integer, intent(in) :: count
-  integer, dimension(count), intent(inout) :: array_of_requests
+  integer, dimension(*), intent(inout) :: array_of_requests
   integer, intent(out) :: index
   integer, dimension(MPI_STATUS_SIZE), intent(out) :: status
   integer, intent(out) :: ierror
@@ -3715,13 +4485,13 @@ end subroutine MPI_Waitany
 end interface
 
 
-interface MPI_Waitsome
+interface
 
 subroutine MPI_Waitsome(incount, array_of_requests, outcount, array_of_indices, array_of_statuses&
         , ierror)
   include 'mpif-config.h'
   integer, intent(in) :: incount
-  integer, dimension(incount), intent(inout) :: array_of_requests
+  integer, dimension(*), intent(inout) :: array_of_requests
   integer, intent(out) :: outcount
   integer, dimension(*), intent(out) :: array_of_indices
   integer, dimension(MPI_STATUS_SIZE, *), intent(out) :: array_of_statuses
@@ -3731,6 +4501,7 @@ end subroutine MPI_Waitsome
 end interface
 
 
+! This interface requires a name because there are multiple subroutines.
 interface MPI_Win_allocate
 
 subroutine MPI_Win_allocate(size, disp_unit, info, comm, &
@@ -3757,6 +4528,7 @@ end subroutine MPI_Win_allocate_cptr
 end interface
 
 
+! This interface requires a name because there are multiple subroutines.
 interface MPI_Win_allocate_shared
 
 subroutine MPI_Win_allocate_shared(size, disp_unit, info, comm, &
@@ -3783,7 +4555,7 @@ end subroutine MPI_Win_allocate_shared_cptr
 end interface
 
 
-interface MPI_Win_attach
+interface
 
 subroutine MPI_Win_attach(win, base, size, ierror)
   include 'mpif-config.h'
@@ -3797,7 +4569,7 @@ end subroutine MPI_Win_attach
 end interface
 
 
-interface MPI_Win_call_errhandler
+interface
 
 subroutine MPI_Win_call_errhandler(win, errorcode, ierror)
   integer, intent(in) :: win
@@ -3808,7 +4580,7 @@ end subroutine MPI_Win_call_errhandler
 end interface
 
 
-interface MPI_Win_complete
+interface
 
 subroutine MPI_Win_complete(win, ierror)
   integer, intent(in) :: win
@@ -3818,7 +4590,7 @@ end subroutine MPI_Win_complete
 end interface
 
 
-interface MPI_Win_create
+interface
 
 subroutine MPI_Win_create(base, size, disp_unit, info, comm, &
         win, ierror)
@@ -3836,7 +4608,7 @@ end subroutine MPI_Win_create
 end interface
 
 
-interface MPI_Win_create_dynamic
+interface
 
 subroutine MPI_Win_create_dynamic(info, comm, &
         win, ierror)
@@ -3850,7 +4622,7 @@ end subroutine MPI_Win_create_dynamic
 end interface
 
 
-interface MPI_Win_create_errhandler
+interface
 
 subroutine MPI_Win_create_errhandler(function, errhandler, ierror)
   external :: function
@@ -3861,7 +4633,7 @@ end subroutine MPI_Win_create_errhandler
 end interface
 
 
-interface MPI_Win_create_keyval
+interface
 
 subroutine MPI_Win_create_keyval(win_copy_attr_fn, win_delete_attr_fn, win_keyval, extra_state, ierror)
   include 'mpif-config.h'
@@ -3875,7 +4647,7 @@ end subroutine MPI_Win_create_keyval
 end interface
 
 
-interface MPI_Win_delete_attr
+interface
 
 subroutine MPI_Win_delete_attr(win, win_keyval, ierror)
   integer, intent(in) :: win
@@ -3886,7 +4658,7 @@ end subroutine MPI_Win_delete_attr
 end interface
 
 
-interface MPI_Win_detach
+interface
 
 subroutine MPI_Win_detach(win, base, ierror)
   include 'mpif-config.h'
@@ -3899,7 +4671,7 @@ end subroutine MPI_Win_detach
 end interface
 
 
-interface MPI_Win_fence
+interface
 
 subroutine MPI_Win_fence(assert, win, ierror)
   integer, intent(in) :: assert
@@ -3910,7 +4682,7 @@ end subroutine MPI_Win_fence
 end interface
 
 
-interface MPI_Win_flush
+interface
 
 subroutine MPI_Win_flush(rank, win, ierror)
   integer, intent(in) :: rank
@@ -3921,7 +4693,7 @@ end subroutine MPI_Win_flush
 end interface
 
 
-interface MPI_Win_flush_all
+interface
 
 subroutine MPI_Win_flush_all(win, ierror)
   integer, intent(in) :: win
@@ -3931,7 +4703,7 @@ end subroutine MPI_Win_flush_all
 end interface
 
 
-interface MPI_Win_flush_local
+interface
 
 subroutine MPI_Win_flush_local(rank, win, ierror)
   integer, intent(in) :: rank
@@ -3942,7 +4714,7 @@ end subroutine MPI_Win_flush_local
 end interface
 
 
-interface MPI_Win_flush_local_all
+interface
 
 subroutine MPI_Win_flush_local_all(win, ierror)
   integer, intent(in) :: win
@@ -3952,7 +4724,7 @@ end subroutine MPI_Win_flush_local_all
 end interface
 
 
-interface MPI_Win_free
+interface
 
 subroutine MPI_Win_free(win, ierror)
   integer, intent(inout) :: win
@@ -3962,7 +4734,7 @@ end subroutine MPI_Win_free
 end interface
 
 
-interface MPI_Win_free_keyval
+interface
 
 subroutine MPI_Win_free_keyval(win_keyval, ierror)
   integer, intent(inout) :: win_keyval
@@ -3972,7 +4744,7 @@ end subroutine MPI_Win_free_keyval
 end interface
 
 
-interface MPI_Win_get_attr
+interface
 
 subroutine MPI_Win_get_attr(win, win_keyval, attribute_val, flag, ierror)
   include 'mpif-config.h'
@@ -3986,7 +4758,7 @@ end subroutine MPI_Win_get_attr
 end interface
 
 
-interface MPI_Win_get_errhandler
+interface
 
 subroutine MPI_Win_get_errhandler(win, errhandler, ierror)
   integer, intent(in) :: win
@@ -3997,7 +4769,7 @@ end subroutine MPI_Win_get_errhandler
 end interface
 
 
-interface MPI_Win_get_group
+interface
 
 subroutine MPI_Win_get_group(win, group, ierror)
   integer, intent(in) :: win
@@ -4008,7 +4780,7 @@ end subroutine MPI_Win_get_group
 end interface
 
 
-interface MPI_Win_get_info
+interface
 
 subroutine MPI_Win_get_info(comm, info, ierror)
   include 'mpif-config.h'
@@ -4020,7 +4792,7 @@ end subroutine MPI_Win_get_info
 end interface
 
 
-interface MPI_Win_get_name
+interface
 
 subroutine MPI_Win_get_name(win, win_name, resultlen, ierror)
   integer, intent(in) :: win
@@ -4032,7 +4804,7 @@ end subroutine MPI_Win_get_name
 end interface
 
 
-interface MPI_Win_lock
+interface
 
 subroutine MPI_Win_lock(lock_type, rank, assert, win, ierror)
   integer, intent(in) :: lock_type
@@ -4045,7 +4817,7 @@ end subroutine MPI_Win_lock
 end interface
 
 
-interface MPI_Win_lock_all
+interface
 
 subroutine MPI_Win_lock_all(assert, win, ierror)
   integer, intent(in) :: assert
@@ -4056,7 +4828,7 @@ end subroutine MPI_Win_lock_all
 end interface
 
 
-interface MPI_Win_post
+interface
 
 subroutine MPI_Win_post(group, assert, win, ierror)
   integer, intent(in) :: group
@@ -4068,7 +4840,7 @@ end subroutine MPI_Win_post
 end interface
 
 
-interface MPI_Win_set_attr
+interface
 
 subroutine MPI_Win_set_attr(win, win_keyval, attribute_val, ierror)
   include 'mpif-config.h'
@@ -4081,7 +4853,7 @@ end subroutine MPI_Win_set_attr
 end interface
 
 
-interface MPI_Win_set_errhandler
+interface
 
 subroutine MPI_Win_set_errhandler(win, errhandler, ierror)
   integer, intent(in) :: win
@@ -4092,7 +4864,7 @@ end subroutine MPI_Win_set_errhandler
 end interface
 
 
-interface MPI_Win_set_info
+interface
 
 subroutine MPI_Win_set_info(comm, info, ierror)
   include 'mpif-config.h'
@@ -4104,7 +4876,7 @@ end subroutine MPI_Win_set_info
 end interface
 
 
-interface MPI_Win_set_name
+interface
 
 subroutine MPI_Win_set_name(win, win_name, ierror)
   integer, intent(in) :: win
@@ -4115,6 +4887,7 @@ end subroutine MPI_Win_set_name
 end interface
 
 
+! This interface requires a name because there are multiple subroutines.
 interface MPI_Win_shared_query
 
 subroutine MPI_Win_shared_query(win, rank, size, disp_unit, baseptr,&
@@ -4143,7 +4916,7 @@ end subroutine MPI_Win_shared_query_cptr
 end interface
 
 
-interface MPI_Win_start
+interface
 
 subroutine MPI_Win_start(group, assert, win, ierror)
   integer, intent(in) :: group
@@ -4155,7 +4928,7 @@ end subroutine MPI_Win_start
 end interface
 
 
-interface MPI_Win_sync
+interface
 
 subroutine MPI_Win_sync(win, ierror)
   integer, intent(in) :: win
@@ -4165,7 +4938,7 @@ end subroutine MPI_Win_sync
 end interface
 
 
-interface MPI_Win_test
+interface
 
 subroutine MPI_Win_test(win, flag, ierror)
   integer, intent(in) :: win
@@ -4176,7 +4949,7 @@ end subroutine MPI_Win_test
 end interface
 
 
-interface MPI_Win_unlock
+interface
 
 subroutine MPI_Win_unlock(rank, win, ierror)
   integer, intent(in) :: rank
@@ -4187,7 +4960,7 @@ end subroutine MPI_Win_unlock
 end interface
 
 
-interface MPI_Win_unlock_all
+interface
 
 subroutine MPI_Win_unlock_all(win, ierror)
   integer, intent(in) :: win
@@ -4197,7 +4970,7 @@ end subroutine MPI_Win_unlock_all
 end interface
 
 
-interface MPI_Win_wait
+interface
 
 subroutine MPI_Win_wait(win, ierror)
   integer, intent(in) :: win
@@ -4207,7 +4980,7 @@ end subroutine MPI_Win_wait
 end interface
 
 
-interface MPI_Wtick
+interface
 
 function MPI_Wtick()
     double precision MPI_Wtick
@@ -4216,7 +4989,7 @@ end function MPI_Wtick
 end interface
 
 
-interface MPI_Wtime
+interface
 
 function MPI_Wtime()
     double precision MPI_Wtime
diff --git a/ompi/mpi/fortran/use-mpi-ignore-tkr/mpi-ignore-tkr-removed-interfaces.h.in b/ompi/mpi/fortran/use-mpi-ignore-tkr/mpi-ignore-tkr-removed-interfaces.h.in
index e8b2c2e061e..4085afe222b 100644
--- a/ompi/mpi/fortran/use-mpi-ignore-tkr/mpi-ignore-tkr-removed-interfaces.h.in
+++ b/ompi/mpi/fortran/use-mpi-ignore-tkr/mpi-ignore-tkr-removed-interfaces.h.in
@@ -1,6 +1,6 @@
 ! -*- fortran -*-
 !
-! Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2006-2021 Cisco Systems, Inc.  All rights reserved
 ! Copyright (c) 2007      Los Alamos National Security, LLC.  All rights
 !                         reserved.
 ! Copyright (c) 2012      The University of Tennessee and The University
@@ -17,7 +17,7 @@
 !
 ! $HEADER$
 
-interface MPI_Address
+interface
 
 subroutine MPI_Address(location, address, ierror)
   @OMPI_FORTRAN_IGNORE_TKR_PREDECL@ location
@@ -28,7 +28,7 @@ end subroutine MPI_Address
 
 end interface
 
-interface MPI_Attr_delete
+interface
 
 subroutine MPI_Attr_delete(comm, keyval, ierror)
   integer, intent(in) :: comm
@@ -38,7 +38,7 @@ end subroutine MPI_Attr_delete
 
 end interface
 
-interface MPI_Attr_get
+interface
 
 subroutine MPI_Attr_get(comm, keyval, attribute_val, flag, ierror)
   integer, intent(in) :: comm
@@ -50,7 +50,7 @@ end subroutine MPI_Attr_get
 
 end interface
 
-interface MPI_Attr_put
+interface
 
 subroutine MPI_Attr_put(comm, keyval, attribute_val, ierror)
   integer, intent(in) :: comm
@@ -61,7 +61,7 @@ end subroutine MPI_Attr_put
 
 end interface
 
-interface MPI_Errhandler_create
+interface
 
 subroutine MPI_Errhandler_create(function, errhandler, ierror)
   external :: function
@@ -71,7 +71,7 @@ end subroutine MPI_Errhandler_create
 
 end interface
 
-interface MPI_Errhandler_get
+interface
 
 subroutine MPI_Errhandler_get(comm, errhandler, ierror)
   integer, intent(in) :: comm
@@ -81,7 +81,7 @@ end subroutine MPI_Errhandler_get
 
 end interface
 
-interface MPI_Errhandler_set
+interface
 
 subroutine MPI_Errhandler_set(comm, errhandler, ierror)
   integer, intent(in) :: comm
@@ -91,7 +91,7 @@ end subroutine MPI_Errhandler_set
 
 end interface
 
-interface MPI_Keyval_create
+interface
 
 subroutine MPI_Keyval_create(copy_fn, delete_fn, keyval, extra_state, ierror)
   external :: copy_fn
@@ -103,7 +103,7 @@ end subroutine MPI_Keyval_create
 
 end interface
 
-interface MPI_Keyval_free
+interface
 
 subroutine MPI_Keyval_free(keyval, ierror)
   integer, intent(inout) :: keyval
@@ -112,7 +112,7 @@ end subroutine MPI_Keyval_free
 
 end interface
 
-interface MPI_Type_extent
+interface
 
 subroutine MPI_Type_extent(datatype, extent, ierror)
   integer, intent(in) :: datatype
@@ -122,7 +122,7 @@ end subroutine MPI_Type_extent
 
 end interface
 
-interface MPI_Type_hindexed
+interface
 
 subroutine MPI_Type_hindexed(count, array_of_blocklengths, array_of_displacements, oldtype, newtype&
         , ierror)
@@ -136,7 +136,7 @@ end subroutine MPI_Type_hindexed
 
 end interface
 
-interface MPI_Type_hvector
+interface
 
 subroutine MPI_Type_hvector(count, blocklength, stride, oldtype, newtype&
         , ierror)
@@ -150,7 +150,7 @@ end subroutine MPI_Type_hvector
 
 end interface
 
-interface MPI_Type_lb
+interface
 
 subroutine MPI_Type_lb(datatype, lb, ierror)
   integer, intent(in) :: datatype
@@ -160,7 +160,7 @@ end subroutine MPI_Type_lb
 
 end interface
 
-interface MPI_Type_struct
+interface
 
 subroutine MPI_Type_struct(count, array_of_blocklengths, array_of_displacements, array_of_types, newtype&
         , ierror)
@@ -174,7 +174,7 @@ end subroutine MPI_Type_struct
 
 end interface
 
-interface MPI_Type_ub
+interface
 
 subroutine MPI_Type_ub(datatype, ub, ierror)
   integer, intent(in) :: datatype
diff --git a/ompi/mpi/fortran/use-mpi-ignore-tkr/mpi-ignore-tkr-status.h b/ompi/mpi/fortran/use-mpi-ignore-tkr/mpi-ignore-tkr-status.h
index bd5635700fe..783302c363c 100644
--- a/ompi/mpi/fortran/use-mpi-ignore-tkr/mpi-ignore-tkr-status.h
+++ b/ompi/mpi/fortran/use-mpi-ignore-tkr/mpi-ignore-tkr-status.h
@@ -2,6 +2,7 @@
 !
 ! Copyright (c) 2020      Research Organization for Information Science
 !                         and Technology (RIST).  All rights reserved.
+! Copyright (c) 2021      Cisco Systems, Inc.  All rights reserved
 ! $COPYRIGHT$
 !
 ! Additional copyrights may follow
@@ -9,7 +10,7 @@
 ! $HEADER$
 
 
-interface MPI_Status_f082f
+interface
 
 subroutine MPI_Status_f082f(f08_status, f_status, ierror)
   use mpi_types
@@ -22,7 +23,7 @@ end subroutine MPI_Status_f082f
 end interface
 
 
-interface MPI_Status_f2f08
+interface
 
 subroutine MPI_Status_f2f08(f_status, f08_status, ierror)
   use mpi_types
@@ -35,7 +36,7 @@ end subroutine MPI_Status_f2f08
 end interface
 
 
-interface PMPI_Status_f082f
+interface
 
 subroutine PMPI_Status_f082f(f08_status, f_status, ierror)
   use mpi_types
@@ -48,7 +49,7 @@ end subroutine PMPI_Status_f082f
 end interface
 
 
-interface PMPI_Status_f2f08
+interface
 
 subroutine PMPI_Status_f2f08(f_status, f08_status, ierror)
   use mpi_types
diff --git a/ompi/mpi/fortran/use-mpi-ignore-tkr/pmpi-ignore-tkr-interfaces.h b/ompi/mpi/fortran/use-mpi-ignore-tkr/pmpi-ignore-tkr-interfaces.h
index b6bb4821d86..f27471b9580 100644
--- a/ompi/mpi/fortran/use-mpi-ignore-tkr/pmpi-ignore-tkr-interfaces.h
+++ b/ompi/mpi/fortran/use-mpi-ignore-tkr/pmpi-ignore-tkr-interfaces.h
@@ -1,6 +1,6 @@
 ! -*- fortran -*-
 !
-! Copyright (c) 2020      Research Organization for Information Science
+! Copyright (c) 2020-2022 Research Organization for Information Science
 !                         and Technology (RIST).  All rights reserved.
 ! $COPYRIGHT$
 !
@@ -16,15 +16,23 @@
 #define MPI_Aint_add PMPI_Aint_add
 #define MPI_Aint_diff PMPI_Aint_diff
 #define MPI_Allgather PMPI_Allgather
+#define MPI_Allgather_init PMPI_Allgather_init
 #define MPI_Allgatherv PMPI_Allgatherv
+#define MPI_Allgatherv_init PMPI_Allgatherv_init
 #define MPI_Alloc_mem PMPI_Alloc_mem
 #define MPI_Alloc_mem_cptr PMPI_Alloc_mem_cptr
 #define MPI_Allreduce PMPI_Allreduce
+#define MPI_Allreduce_init PMPI_Allreduce_init
 #define MPI_Alltoall PMPI_Alltoall
+#define MPI_Alltoall_init PMPI_Alltoall_init
 #define MPI_Alltoallv PMPI_Alltoallv
+#define MPI_Alltoallv_init PMPI_Alltoallv_init
 #define MPI_Alltoallw PMPI_Alltoallw
+#define MPI_Alltoallw_init PMPI_Alltoallw_init
 #define MPI_Barrier PMPI_Barrier
-#define MPI_Bcast PPMPI_Bcast
+#define MPI_Barrier_init PMPI_Barrier_init
+#define MPI_Bcast PMPI_Bcast
+#define MPI_Bcast_init PMPI_Bcast_init
 #define MPI_Bsend PMPI_Bsend
 #define MPI_Bsend_init PMPI_Bsend_init
 #define MPI_Buffer_attach PMPI_Buffer_attach
@@ -45,6 +53,7 @@
 #define MPI_Comm_connect PMPI_Comm_connect
 #define MPI_Comm_create PMPI_Comm_create
 #define MPI_Comm_create_errhandler PMPI_Comm_create_errhandler
+#define MPI_Comm_create_from_group PMPI_Comm_create_from_group
 #define MPI_Comm_create_group PMPI_Comm_create_group
 #define MPI_Comm_create_keyval PMPI_Comm_create_keyval
 #define MPI_Comm_delete_attr PMPI_Comm_delete_attr
@@ -60,6 +69,7 @@
 #define MPI_Comm_get_parent PMPI_Comm_get_parent
 #define MPI_Comm_group PMPI_Comm_group
 #define MPI_Comm_idup PMPI_Comm_idup
+#define MPI_Comm_idup_with_info PMPI_Comm_idup_with_info
 #define MPI_Comm_join PMPI_Comm_join
 #define MPI_Comm_rank PMPI_Comm_rank
 #define MPI_Comm_remote_group PMPI_Comm_remote_group
@@ -84,13 +94,16 @@
 #define MPI_Error_class PMPI_Error_class
 #define MPI_Error_string PMPI_Error_string
 #define MPI_Exscan PMPI_Exscan
+#define MPI_Exscan_init PMPI_Exscan_init
 #define MPI_F_sync_reg PMPI_F_sync_reg
 #define MPI_Fetch_and_op PMPI_Fetch_and_op
 #define MPI_Finalize PMPI_Finalize
 #define MPI_Finalized PMPI_Finalized
 #define MPI_Free_mem PMPI_Free_mem
 #define MPI_Gather PMPI_Gather
+#define MPI_Gather_init PMPI_Gather_init
 #define MPI_Gatherv PMPI_Gatherv
+#define MPI_Gatherv_init PMPI_Gatherv_init
 #define MPI_Get PMPI_Get
 #define MPI_Get_accumulate PMPI_Get_accumulate
 #define MPI_Get_address PMPI_Get_address
@@ -111,6 +124,7 @@
 #define MPI_Group_compare PMPI_Group_compare
 #define MPI_Group_difference PMPI_Group_difference
 #define MPI_Group_excl PMPI_Group_excl
+#define MPI_Group_from_session_pset PMPI_Group_from_session_pset
 #define MPI_Group_free PMPI_Group_free
 #define MPI_Group_incl PMPI_Group_incl
 #define MPI_Group_intersection PMPI_Group_intersection
@@ -140,18 +154,21 @@
 #define MPI_Ineighbor_alltoallv PMPI_Ineighbor_alltoallv
 #define MPI_Ineighbor_alltoallw PMPI_Ineighbor_alltoallw
 #define MPI_Info_create PMPI_Info_create
+#define MPI_Info_create_env PMPI_Info_create_env
 #define MPI_Info_delete PMPI_Info_delete
 #define MPI_Info_dup PMPI_Info_dup
 #define MPI_Info_free PMPI_Info_free
 #define MPI_Info_get PMPI_Info_get
 #define MPI_Info_get_nkeys PMPI_Info_get_nkeys
 #define MPI_Info_get_nthkey PMPI_Info_get_nthkey
+#define MPI_Info_get_string PMPI_Info_get_string
 #define MPI_Info_get_valuelen PMPI_Info_get_valuelen
 #define MPI_Info_set PMPI_Info_set
 #define MPI_Init PMPI_Init
 #define MPI_Init_thread PMPI_Init_thread
 #define MPI_Initialized PMPI_Initialized
 #define MPI_Intercomm_create PMPI_Intercomm_create
+#define MPI_Intercomm_create_from_groups PMPI_Intercomm_create_from_groups
 #define MPI_Intercomm_merge PMPI_Intercomm_merge
 #define MPI_Iprobe PMPI_Iprobe
 #define MPI_Irecv PMPI_Irecv
@@ -164,15 +181,22 @@
 #define MPI_Iscatter PMPI_Iscatter
 #define MPI_Iscatterv PMPI_Iscatterv
 #define MPI_Isend PMPI_Isend
+#define MPI_Isendrecv PMPI_Isendrecv
+#define MPI_Isendrecv_replace PMPI_Isendrecv_replace
 #define MPI_Issend PMPI_Issend
 #define MPI_Lookup_name PMPI_Lookup_name
 #define MPI_Mprobe PMPI_Mprobe
 #define MPI_Mrecv PMPI_Mrecv
 #define MPI_Neighbor_allgather PMPI_Neighbor_allgather
+#define MPI_Neighbor_allgather_init PMPI_Neighbor_allgather_init
 #define MPI_Neighbor_allgatherv PMPI_Neighbor_allgatherv
+#define MPI_Neighbor_allgatherv_init PMPI_Neighbor_allgatherv_init
 #define MPI_Neighbor_alltoall PMPI_Neighbor_alltoall
+#define MPI_Neighbor_alltoall_init PMPI_Neighbor_alltoall_init
 #define MPI_Neighbor_alltoallv PMPI_Neighbor_alltoallv
+#define MPI_Neighbor_alltoallv_init PMPI_Neighbor_alltoallv_init
 #define MPI_Neighbor_alltoallw PMPI_Neighbor_alltoallw
+#define MPI_Neighbor_alltoallw_init PMPI_Neighbor_alltoallw_init
 #define MPI_Op_commutative PMPI_Op_commutative
 #define MPI_Op_create PMPI_Op_create
 #define MPI_Op_free PMPI_Op_free
@@ -181,8 +205,14 @@
 #define MPI_Pack_external PMPI_Pack_external
 #define MPI_Pack_external_size PMPI_Pack_external_size
 #define MPI_Pack_size PMPI_Pack_size
+#define MPI_Parrived PMPI_Parrived
 #define MPI_Pcontrol PMPI_Pcontrol
+#define MPI_Pready PMPI_Pready
+#define MPI_Pready_list PMPI_Pready_list
+#define MPI_Pready_range PMPI_Pready_range
+#define MPI_Precv_init PMPI_Precv_init
 #define MPI_Probe PMPI_Probe
+#define MPI_Psend_init PMPI_Psend_init
 #define MPI_Publish_name PMPI_Publish_name
 #define MPI_Put PMPI_Put
 #define MPI_Query_thread PMPI_Query_thread
@@ -190,9 +220,12 @@
 #define MPI_Recv PMPI_Recv
 #define MPI_Recv_init PMPI_Recv_init
 #define MPI_Reduce PMPI_Reduce
+#define MPI_Reduce_init PMPI_Reduce_init
 #define MPI_Reduce_local PMPI_Reduce_local
 #define MPI_Reduce_scatter PMPI_Reduce_scatter
+#define MPI_Reduce_scatter_init PMPI_Reduce_scatter_init
 #define MPI_Reduce_scatter_block PMPI_Reduce_scatter_block
+#define MPI_Reduce_scatter_block_init PMPI_Reduce_scatter_block_init
 #define MPI_Register_datarep PMPI_Register_datarep
 #define MPI_Request_free PMPI_Request_free
 #define MPI_Request_get_status PMPI_Request_get_status
@@ -202,12 +235,25 @@
 #define MPI_Rsend PMPI_Rsend
 #define MPI_Rsend_init PMPI_Rsend_init
 #define MPI_Scan PMPI_Scan
+#define MPI_Scan_init PMPI_Scan_init
 #define MPI_Scatter PMPI_Scatter
+#define MPI_Scatter_init PMPI_Scatter_init
 #define MPI_Scatterv PMPI_Scatterv
+#define MPI_Scatterv_init PMPI_Scatterv_init
 #define MPI_Send PMPI_Send
 #define MPI_Send_init PMPI_Send_init
 #define MPI_Sendrecv PMPI_Sendrecv
 #define MPI_Sendrecv_replace PMPI_Sendrecv_replace
+#define MPI_Session_call_errhandler PMPI_Session_call_errhandler
+#define MPI_Session_create_errhandler PMPI_Session_create_errhandler
+#define MPI_Session_get_errhandler PMPI_Session_get_errhandler
+#define MPI_Session_get_info PMPI_Session_get_info
+#define MPI_Session_get_nth_pset PMPI_Session_get_nth_pset
+#define MPI_Session_get_nth_psetlen PMPI_Session_get_nth_psetlen
+#define MPI_Session_get_pset_info PMPI_Session_get_pset_info
+#define MPI_Session_init PMPI_Session_init
+#define MPI_Session_finalize PMPI_Session_finalize
+#define MPI_Session_set_errhandler PMPI_Session_set_errhandler
 #define MPI_Ssend PMPI_Ssend
 #define MPI_Ssend_init PMPI_Ssend_init
 #define MPI_Start PMPI_Start
diff --git a/ompi/mpi/fortran/use-mpi-tkr/Makefile.am b/ompi/mpi/fortran/use-mpi-tkr/Makefile.am
deleted file mode 100644
index ce5c8a295d0..00000000000
--- a/ompi/mpi/fortran/use-mpi-tkr/Makefile.am
+++ /dev/null
@@ -1,202 +0,0 @@
-# -*- makefile -*-
-#
-# Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2006-2019 Cisco Systems, Inc.  All rights reserved
-# Copyright (c) 2007      Los Alamos National Security, LLC.  All rights
-#                         reserved.
-# Copyright (c) 2014-2020 Research Organization for Information Science
-#                         and Technology (RIST).  All rights reserved.
-# Copyright (c) 2016      IBM Corporation.  All rights reserved.
-# Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-include $(top_srcdir)/Makefile.ompi-rules
-
-# THIS MAKEFILE IS OLD AND ONLY TO SUPPORT FORTRAN COMPILERS THAT DO
-# NOT SUPPORT "IGNORE TKR" FUNCTIONALITY (i.e., gfortran before v4.9).
-# All other Fortran compilers support ignore TKR and don't compile
-# this directory -- instead, they compile
-# ompi/fortran/use-mpi-ignore-tkr.
-
-# Note that Automake's Fortran-buidling rules uses CPPFLAGS and
-# AM_CPPFLAGS.  This can cause weirdness (e.g.,
-# https://github.com/open-mpi/ompi/issues/7253).  Let's just zero
-# those out and rely on AM_FCFLAGS.
-CPPFLAGS =
-AM_CPPFLAGS =
-
-if OMPI_BUILD_FORTRAN_USEMPI_TKR_BINDINGS
-
-########################################################################
-
-# Note the "-I." in the flags below.  This is because mpi.F90 includes
-# mpi-f90-interfaces.h, which will be in the build tree (i.e., the
-# current directory) because it is generated.
-
-AM_FCFLAGS = -I$(top_builddir)/ompi/include -I$(top_srcdir)/ompi/include \
-             $(OMPI_FC_MODULE_FLAG). -I$(top_srcdir) -I$(top_builddir) -I. \
-             -I$(top_builddir)/ompi/mpi/fortran/use-mpi \
-             -I$(top_builddir)/ompi/mpi/fortran/use-mpi-tkr $(FCFLAGS_f90)
-
-# Do different things if the top-level configure decided that we're
-# going to build F90 bindings or not.
-
-lib_LTLIBRARIES =
-
-# Add the f90 library to the list of libraries to build
-
-lib_LTLIBRARIES += lib@OMPI_LIBMPI_NAME@_usempi.la
-
-# Automake doesn't know how to do F90 dependency analysis, so manually
-# list this here (i.e., "mpi-f90-interfaces.h" is included in
-# mpi.F90).
-
-mpi.lo: mpi.F90
-mpi.lo: mpi-f90-interfaces.h
-mpi.lo: mpi-f90-status.h
-mpi.lo: mpi-f90-file-interfaces.h
-mpi.lo: mpi-f90-removed-interfaces.h
-mpi.lo: $(top_builddir)/ompi/mpi/fortran/configure-fortran-output.h
-mpi.lo: mpi-f90-cptr-interfaces.h
-mpi.lo: pmpi-f90-interfaces.h
-mpi.lo: pmpi-f90-status.h
-mpi.lo: pmpi-f90-file-interfaces.h
-mpi.lo: pmpi-f90-removed-interfaces.h
-mpi.lo: pmpi-f90-cptr-interfaces.h
-
-# Per MPI-3 p610:34-41, if we're building a TKR mpi module, we should
-# not build any interfaces for MPI subroutines that take a choice
-# buffer dummy argument.  We therefore really only need to build a
-# handful of subroutines.
-
-lib@OMPI_LIBMPI_NAME@_usempi_la_SOURCES = \
-     mpi.F90 \
-     mpi_aint_add_f90.f90 \
-     mpi_aint_diff_f90.f90 \
-     mpi_comm_spawn_multiple_f90.f90 \
-     mpi_testall_f90.f90 \
-     mpi_testsome_f90.f90 \
-     mpi_waitall_f90.f90 \
-     mpi_waitsome_f90.f90 \
-     mpi_wtick_f90.f90 \
-     mpi_wtime_f90.f90
-
-# Don't distribute mpi-tkr-sizeof-*; they're generated.
-
-nodist_lib@OMPI_LIBMPI_NAME@_usempi_la_SOURCES =
-if BUILD_FORTRAN_SIZEOF
-nodist_lib@OMPI_LIBMPI_NAME@_usempi_la_SOURCES += \
-     mpi-tkr-sizeof.h \
-     mpi-tkr-sizeof.f90
-endif
-mpi.lo: $(nodist_lib@OMPI_LIBMPI_NAME@_usempi_la_SOURCES)
-
-libusempi_internal_modules =
-if OMPI_FORTRAN_HAVE_TYPE_MPI_STATUS
-libusempi_internal_modules += $(top_builddir)/ompi/mpi/fortran/use-mpi/libusempi_internal_modules.la
-endif
-
-# Note that we invoke some OPAL functions directly in
-# libmpi_usempi.la, so we need to link in the OPAL library directly
-# (pulling it in indirectly via libmpi.la does not work on all
-# platforms).
-lib@OMPI_LIBMPI_NAME@_usempi_la_LIBADD = \
-        $(top_builddir)/ompi/mpi/fortran/mpif-h/lib@OMPI_LIBMPI_NAME@_mpifh.la \
-        $(libusempi_internal_modules) \
-        $(OMPI_TOP_BUILDDIR)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
-# Set the library version
-lib@OMPI_LIBMPI_NAME@_usempi_la_LDFLAGS = \
-        -version-info $(libmpi_usempi_tkr_so_version) \
-        $(OMPI_FORTRAN_EXTRA_SHARED_LIBRARY_FLAGS)
-
-#
-# mpi-ignore-tkr-sizeof.* are generated based on some results from
-# configure tests.
-#
-
-sizeof_pl=$(top_srcdir)/ompi/mpi/fortran/base/gen-mpi-sizeof.pl
-
-mpi-tkr-sizeof.h: $(top_builddir)/config.status
-mpi-tkr-sizeof.h: $(sizeof_pl)
-mpi-tkr-sizeof.h:
-	$(OMPI_V_GEN) $(sizeof_pl) \
-	    --header=$@ --ierror=mandatory \
-	    --maxrank=$(OMPI_FORTRAN_MAX_ARRAY_RANK) \
-	    --generate=$(OMPI_FORTRAN_BUILD_SIZEOF) \
-	    --real2=$(OMPI_HAVE_FORTRAN_REAL2) \
-	    --real16=$(OMPI_HAVE_FORTRAN_REAL16) \
-	    --complex4=$(OMPI_HAVE_FORTRAN_COMPLEX4) \
-	    --complex32=$(OMPI_HAVE_FORTRAN_COMPLEX32)
-
-mpi-tkr-sizeof.f90: $(top_builddir)/config.status
-mpi-tkr-sizeof.f90: $(sizeof_pl)
-mpi-tkr-sizeof.f90:
-	$(OMPI_V_GEN) $(sizeof_pl) \
-	    --impl=$@ --ierror=mandatory --mpi --pmpi \
-	    --maxrank=$(OMPI_FORTRAN_MAX_ARRAY_RANK) \
-	    --generate=$(OMPI_FORTRAN_BUILD_SIZEOF) \
-	    --real2=$(OMPI_HAVE_FORTRAN_REAL2) \
-	    --real16=$(OMPI_HAVE_FORTRAN_REAL16) \
-	    --complex4=$(OMPI_HAVE_FORTRAN_COMPLEX4) \
-	    --complex32=$(OMPI_HAVE_FORTRAN_COMPLEX32)
-
-#
-# Clean up all F90 module files and all generated files
-#
-
-CLEANFILES += mpi-tkr-sizeof.h mpi-tkr-sizeof.f90
-MOSTLYCLEANFILES = *.mod
-DISTCLEANFILES = $(nodist_lib@OMPI_LIBMPI_NAME@_usempi_la_SOURCES)
-
-#
-# Install the generated .mod files.  Unfortunately, each F90 compiler
-# may generate different filenames, so we have to use a glob.  :-(
-#
-
-install-exec-hook:
-	@ for file in `ls *.mod`; do \
-	  echo $(INSTALL) $$file $(DESTDIR)$(libdir); \
-	  $(INSTALL) $$file $(DESTDIR)$(libdir); \
-	done
-
-uninstall-local:
-	@ for file in `ls *.mod`; do \
-	  echo rm -f $(DESTDIR)$(libdir)/$$file; \
-	  rm -f $(DESTDIR)$(libdir)/$$file; \
-	done
-
-# if OMPI_BUILD_FORTRAN_USEMPI_TKR_BINDINGS
-endif
-
-# These files must be EXTRA_DIST'ed because they are included; they
-# are not compiled directly (and we *can't* list the .F90 file in
-# _SOURCES, because it would be compiled -- but it *must* have a .F90
-# suffix so that the Fortran compiler will interpret the #if
-# directives properly).
-
-EXTRA_DIST = \
-     mpi-f90-interfaces.h \
-     mpi-f90-status.h \
-     mpi-f90-file-interfaces.h \
-     mpi-f90-removed-interfaces.h \
-     mpi-f90-cptr-interfaces.h \
-     pmpi-f90-interfaces.h \
-     pmpi-f90-status.h \
-     pmpi-f90-file-interfaces.h \
-     pmpi-f90-removed-interfaces.h \
-     pmpi-f90-cptr-interfaces.h
-
diff --git a/ompi/mpi/fortran/use-mpi-tkr/fortran_kinds.sh.in b/ompi/mpi/fortran/use-mpi-tkr/fortran_kinds.sh.in
deleted file mode 100644
index bc81b890427..00000000000
--- a/ompi/mpi/fortran/use-mpi-tkr/fortran_kinds.sh.in
+++ /dev/null
@@ -1,84 +0,0 @@
-#! /bin/sh
-#
-# Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2006-2014 Cisco Systems, Inc.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-max_array_rank=7
-
-ikinds="@OMPI_FORTRAN_IKINDS@"
-rkinds="@OMPI_FORTRAN_RKINDS@"
-ckinds="@OMPI_FORTRAN_CKINDS@"
-
-# In prior versions of Open MPI, we used to allow the user to choose
-# which "size" of "use mpi" module to build: tiny, small, medium, and
-# large.  Large never really worked.
-#
-# According to MPI-3 p610:34-41, if ignore TKR is not supported, we
-# should not provide interfaces for MPI subroutines with choice
-# buffers.  So we default to "small" (in 1.7-1.7.3, we defaulted to
-# "medium", but then MPI-3 p610 was pointed out to us in Jan 2014, so
-# we changed the default back to "small").
-size=small
-
-# fix up for broken generation scripts that don't properly compute the
-# ranks variable before using it
-ranks="1"
-rank="2"
-while test "`expr $rank \\<= $max_array_rank`" = "1"; do
-    ranks="$ranks $rank"
-    rank="`expr $rank + 1`"
-done
-unset rank
-
-#
-# Helper function
-#
-check_size() {
-    # Sanity check
-    if test "$1" != "trivial" -a "$1" != "small" -a "$1" != "medium" -a \
-        "$1" != "large"; then
-        echo "WARNING: unrecognized size: $1" >&2
-    fi
-
-    # Check the value
-    output=0
-    if test "$1" = "trivial"; then
-        # Since we've got a trivial source, always build it (since
-        # trivial is the smallest allowable size to build, and we
-        # already verified above that the size is valid).
-        output=1
-    elif test "$1" = "small"; then
-        # If we've got a small source, build it unless the user only
-        # requested trivial.
-        if test "$size" != "trivial"; then
-            output=1
-        fi
-    elif test "$1" = "medium"; then
-        # If we've got a medium source, build it only if the user
-        # requested medium or large.
-        if test "$size" = "medium" -o "$size" = "large"; then
-            output=1
-        fi
-    elif test "$1" = "large"; then
-        # If we've got a large source, build it only if the user
-        # requested large.
-        if test "$size" = "large"; then
-            output=1
-        fi
-    fi
-}
diff --git a/ompi/mpi/fortran/use-mpi-tkr/fortran_sizes.h.in b/ompi/mpi/fortran/use-mpi-tkr/fortran_sizes.h.in
deleted file mode 100644
index 1f218f6c9d8..00000000000
--- a/ompi/mpi/fortran/use-mpi-tkr/fortran_sizes.h.in
+++ /dev/null
@@ -1,59 +0,0 @@
-!
-! Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-!                         University Research and Technology
-!                         Corporation.  All rights reserved.
-! Copyright (c) 2004-2005 The University of Tennessee and The University
-!                         of Tennessee Research Foundation.  All rights
-!                         reserved.
-! Copyright (c) 2004-2007 High Performance Computing Center Stuttgart,
-!                         University of Stuttgart.  All rights reserved.
-! Copyright (c) 2004-2005 The Regents of the University of California.
-!                         All rights reserved.
-! Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-!
-! Additional copyrights may follow
-!
-! $HEADER$
-!
-
-! This file is used by scripts/mpi_sizeof.f90.sh (and therefore mpi_sizeof.f90)
-
-integer :: OMPI_SIZEOF_F90_CHARACTER
-integer :: OMPI_SIZEOF_F90_LOGICAL
-
-integer :: OMPI_SIZEOF_F90_INT1
-integer :: OMPI_SIZEOF_F90_INT2
-integer :: OMPI_SIZEOF_F90_INT4
-integer :: OMPI_SIZEOF_F90_INT8
-integer :: OMPI_SIZEOF_F90_INT16
-
-integer :: OMPI_SIZEOF_F90_REAL2
-integer :: OMPI_SIZEOF_F90_REAL4
-integer :: OMPI_SIZEOF_F90_REAL8
-integer :: OMPI_SIZEOF_F90_REAL16
-
-integer :: OMPI_SIZEOF_F90_COMPLEX4
-integer :: OMPI_SIZEOF_F90_COMPLEX8
-integer :: OMPI_SIZEOF_F90_COMPLEX16
-integer :: OMPI_SIZEOF_F90_COMPLEX32
-
-parameter(OMPI_SIZEOF_F90_CHARACTER = @OMPI_SIZEOF_FORTRAN_CHARACTER@)
-parameter(OMPI_SIZEOF_F90_LOGICAL = @OMPI_SIZEOF_FORTRAN_LOGICAL@)
-
-parameter(OMPI_SIZEOF_F90_INT1 = @OMPI_SIZEOF_FORTRAN_INTEGER1@)
-parameter(OMPI_SIZEOF_F90_INT2 = @OMPI_SIZEOF_FORTRAN_INTEGER2@)
-parameter(OMPI_SIZEOF_F90_INT4 = @OMPI_SIZEOF_FORTRAN_INTEGER4@)
-parameter(OMPI_SIZEOF_F90_INT8 = @OMPI_SIZEOF_FORTRAN_INTEGER8@)
-parameter(OMPI_SIZEOF_F90_INT16 = @OMPI_SIZEOF_FORTRAN_INTEGER16@)
-
-parameter(OMPI_SIZEOF_F90_REAL2 = @OMPI_SIZEOF_FORTRAN_REAL2@)
-parameter(OMPI_SIZEOF_F90_REAL4 = @OMPI_SIZEOF_FORTRAN_REAL4@)
-parameter(OMPI_SIZEOF_F90_REAL8 = @OMPI_SIZEOF_FORTRAN_REAL8@)
-parameter(OMPI_SIZEOF_F90_REAL16 = @OMPI_SIZEOF_FORTRAN_REAL16@)
-
-parameter(OMPI_SIZEOF_F90_COMPLEX4 = @OMPI_SIZEOF_FORTRAN_COMPLEX4@)
-parameter(OMPI_SIZEOF_F90_COMPLEX8 = @OMPI_SIZEOF_FORTRAN_COMPLEX8@)
-parameter(OMPI_SIZEOF_F90_COMPLEX16 = @OMPI_SIZEOF_FORTRAN_COMPLEX16@)
-parameter(OMPI_SIZEOF_F90_COMPLEX32 = @OMPI_SIZEOF_FORTRAN_COMPLEX32@)
diff --git a/ompi/mpi/fortran/use-mpi-tkr/mpi-f90-cptr-interfaces.h b/ompi/mpi/fortran/use-mpi-tkr/mpi-f90-cptr-interfaces.h
deleted file mode 100644
index d4a51f4bb48..00000000000
--- a/ompi/mpi/fortran/use-mpi-tkr/mpi-f90-cptr-interfaces.h
+++ /dev/null
@@ -1,106 +0,0 @@
-! -*- fortran -*-
-!
-! Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
-! $COPYRIGHT$
-!
-! Additional copyrights may follow
-!
-! $HEADER$
-!
-! This file contains interfaces that use the ISO_C_BINDING module and
-! the TYPE(C_PTR) type, which not all Fortran compilers support (e.g.,
-! gfortran on RHEL 5 does not support this module/type).  So we use a
-! preprocessor macro to protect the problematic declarations.
-!
-! This file is included via a preprocessor include directorive in
-! mpi.F90, which allows us to use the preprocessor "if" directive,
-! below.
-!
-
-interface MPI_Win_allocate
-
-subroutine MPI_Win_allocate(size, disp_unit, info, comm, &
-      baseptr, win, ierror)
-  include 'mpif-config.h'
-  integer(KIND=MPI_ADDRESS_KIND), intent(in) :: size
-  integer, intent(in) :: disp_unit
-  integer, intent(in) :: info
-  integer, intent(in) :: comm
-  integer(KIND=MPI_ADDRESS_KIND), intent(out) :: baseptr
-  integer, intent(out) :: win
-  integer, intent(out) :: ierror
-end subroutine MPI_Win_allocate
-
-! Only include the 2nd interface if we have ISO_C_BINDING / TYPE(C_PTR)
-#if OMPI_FORTRAN_HAVE_ISO_C_BINDING
-subroutine MPI_Win_allocate_cptr(size, disp_unit, info, comm, &
-     baseptr, win, ierror)
-  use, intrinsic :: iso_c_binding, only : c_ptr
-  include 'mpif-config.h'
-  integer :: disp_unit, info, comm, win, ierror
-  integer(KIND=MPI_ADDRESS_KIND) :: size
-  type(C_PTR) :: baseptr
-end subroutine MPI_Win_allocate_cptr
-#endif
-
-end interface
-
-
-interface MPI_Win_allocate_shared
-
-subroutine MPI_Win_allocate_shared(size, disp_unit, info, comm, &
-      baseptr, win, ierror)
-  include 'mpif-config.h'
-  integer(KIND=MPI_ADDRESS_KIND), intent(in) :: size
-  integer, intent(in) :: disp_unit
-  integer, intent(in) :: info
-  integer, intent(in) :: comm
-  integer(KIND=MPI_ADDRESS_KIND), intent(out) :: baseptr
-  integer, intent(out) :: win
-  integer, intent(out) :: ierror
-end subroutine MPI_Win_allocate_shared
-
-! Only include the 2nd interface if we have ISO_C_BINDING / TYPE(C_PTR)
-#if OMPI_FORTRAN_HAVE_ISO_C_BINDING
-subroutine MPI_Win_allocate_shared_cptr(size, disp_unit, info, comm, &
-     baseptr, win, ierror)
-  use, intrinsic :: iso_c_binding, only : c_ptr
-  include 'mpif-config.h'
-  integer :: disp_unit, info, comm, win, ierror
-  integer(KIND=MPI_ADDRESS_KIND) :: size
-  type(C_PTR) :: baseptr
-end subroutine MPI_Win_allocate_shared_cptr
-#endif
-
-end interface
-
-
-interface MPI_Win_shared_query
-
-subroutine MPI_Win_shared_query(win, rank, size, disp_unit, baseptr,&
-      ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: win
-  integer, intent(in) :: rank
-  integer(KIND=MPI_ADDRESS_KIND), intent(out) :: size
-  integer, intent(out) :: disp_unit
-  integer(KIND=MPI_ADDRESS_KIND), intent(out) :: baseptr
-  integer, intent(out) :: ierror
-end subroutine MPI_Win_shared_query
-
-! Only include the 2nd interface if we have ISO_C_BINDING / TYPE(C_PTR)
-#if OMPI_FORTRAN_HAVE_ISO_C_BINDING
-subroutine MPI_Win_shared_query_cptr(win, rank, size, disp_unit, baseptr,&
-      ierror)
-  use, intrinsic :: iso_c_binding, only : c_ptr
-  include 'mpif-config.h'
-  integer, intent(in) :: win
-  integer, intent(in) :: rank
-  integer(KIND=MPI_ADDRESS_KIND), intent(out) :: size
-  integer, intent(out) :: disp_unit
-  type(C_PTR), intent(out) :: baseptr
-  integer, intent(out) :: ierror
-end subroutine MPI_Win_shared_query_cptr
-#endif
-
-end interface
diff --git a/ompi/mpi/fortran/use-mpi-tkr/mpi-f90-file-interfaces.h b/ompi/mpi/fortran/use-mpi-tkr/mpi-f90-file-interfaces.h
deleted file mode 100644
index eb268735efa..00000000000
--- a/ompi/mpi/fortran/use-mpi-tkr/mpi-f90-file-interfaces.h
+++ /dev/null
@@ -1,321 +0,0 @@
-! -*- fortran -*-
-!
-! Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
-!                         University Research and Technology
-!                         Corporation.  All rights reserved.
-! Copyright (c) 2004-2005 The University of Tennessee and The University
-!                         of Tennessee Research Foundation.  All rights
-!                         reserved.
-! Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-!                         University of Stuttgart.  All rights reserved.
-! Copyright (c) 2004-2005 The Regents of the University of California.
-!                         All rights reserved.
-! Copyright (c) 2006-2014 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2016      Research Organization for Information Science
-!                         and Technology (RIST). All rights reserved.
-! $COPYRIGHT$
-!
-! Additional copyrights may follow
-!
-! $HEADER$
-!
-
-interface MPI_File_call_errhandler
-
-subroutine MPI_File_call_errhandler(fh, errorcode, ierror)
-  integer, intent(in) :: fh
-  integer, intent(in) :: errorcode
-  integer, intent(out) :: ierror
-end subroutine MPI_File_call_errhandler
-
-end interface
-
-
-interface MPI_File_close
-
-subroutine MPI_File_close(fh, ierror)
-  integer, intent(inout) :: fh
-  integer, intent(out) :: ierror
-end subroutine MPI_File_close
-
-end interface
-
-
-interface MPI_File_create_errhandler
-
-subroutine MPI_File_create_errhandler(function, errhandler, ierror)
-  external :: function
-  integer, intent(out) :: errhandler
-  integer, intent(out) :: ierror
-end subroutine MPI_File_create_errhandler
-
-end interface
-
-
-interface MPI_File_delete
-
-subroutine MPI_File_delete(filename, info, ierror)
-  character(len=*), intent(in) :: filename
-  integer, intent(in) :: info
-  integer, intent(out) :: ierror
-end subroutine MPI_File_delete
-
-end interface
-
-
-interface MPI_File_get_amode
-
-subroutine MPI_File_get_amode(fh, amode, ierror)
-  integer, intent(in) :: fh
-  integer, intent(out) :: amode
-  integer, intent(out) :: ierror
-end subroutine MPI_File_get_amode
-
-end interface
-
-
-interface MPI_File_get_atomicity
-
-subroutine MPI_File_get_atomicity(fh, flag, ierror)
-  integer, intent(in) :: fh
-  logical, intent(out) :: flag
-  integer, intent(out) :: ierror
-end subroutine MPI_File_get_atomicity
-
-end interface
-
-
-interface MPI_File_get_byte_offset
-
-subroutine MPI_File_get_byte_offset(fh, offset, disp, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: fh
-  integer(kind=MPI_OFFSET_KIND), intent(in) :: offset
-  integer(kind=MPI_OFFSET_KIND), intent(out) :: disp
-  integer, intent(out) :: ierror
-end subroutine MPI_File_get_byte_offset
-
-end interface
-
-
-interface MPI_File_get_errhandler
-
-subroutine MPI_File_get_errhandler(file, errhandler, ierror)
-  integer, intent(in) :: file
-  integer, intent(out) :: errhandler
-  integer, intent(out) :: ierror
-end subroutine MPI_File_get_errhandler
-
-end interface
-
-
-interface MPI_File_get_group
-
-subroutine MPI_File_get_group(fh, group, ierror)
-  integer, intent(in) :: fh
-  integer, intent(out) :: group
-  integer, intent(out) :: ierror
-end subroutine MPI_File_get_group
-
-end interface
-
-
-interface MPI_File_get_info
-
-subroutine MPI_File_get_info(fh, info_used, ierror)
-  integer, intent(in) :: fh
-  integer, intent(out) :: info_used
-  integer, intent(out) :: ierror
-end subroutine MPI_File_get_info
-
-end interface
-
-
-interface MPI_File_get_position
-
-subroutine MPI_File_get_position(fh, offset, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: fh
-  integer(kind=MPI_OFFSET_KIND), intent(out) :: offset
-  integer, intent(out) :: ierror
-end subroutine MPI_File_get_position
-
-end interface
-
-
-interface MPI_File_get_position_shared
-
-subroutine MPI_File_get_position_shared(fh, offset, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: fh
-  integer(kind=MPI_OFFSET_KIND), intent(out) :: offset
-  integer, intent(out) :: ierror
-end subroutine MPI_File_get_position_shared
-
-end interface
-
-
-interface MPI_File_get_size
-
-subroutine MPI_File_get_size(fh, size, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: fh
-  integer(kind=MPI_OFFSET_KIND), intent(out) :: size
-  integer, intent(out) :: ierror
-end subroutine MPI_File_get_size
-
-end interface
-
-
-interface MPI_File_get_type_extent
-
-subroutine MPI_File_get_type_extent(fh, datatype, extent, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: fh
-  integer, intent(in) :: datatype
-  integer(kind=MPI_ADDRESS_KIND), intent(out) :: extent
-  integer, intent(out) :: ierror
-end subroutine MPI_File_get_type_extent
-
-end interface
-
-
-interface MPI_File_get_view
-
-subroutine MPI_File_get_view(fh, disp, etype, filetype, datarep&
-        , ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: fh
-  integer(kind=MPI_OFFSET_KIND), intent(out) :: disp
-  integer, intent(out) :: etype
-  integer, intent(out) :: filetype
-  character(len=*), intent(out) :: datarep
-  integer, intent(out) :: ierror
-end subroutine MPI_File_get_view
-
-end interface
-
-
-interface MPI_File_open
-
-subroutine MPI_File_open(comm, filename, amode, info, fh&
-        , ierror)
-  integer, intent(in) :: comm
-  character(len=*), intent(in) :: filename
-  integer, intent(in) :: amode
-  integer, intent(in) :: info
-  integer, intent(out) :: fh
-  integer, intent(out) :: ierror
-end subroutine MPI_File_open
-
-end interface
-
-
-interface MPI_File_preallocate
-
-subroutine MPI_File_preallocate(fh, size, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: fh
-  integer(kind=MPI_OFFSET_KIND), intent(in) :: size
-  integer, intent(out) :: ierror
-end subroutine MPI_File_preallocate
-
-end interface
-
-
-interface MPI_File_seek
-
-subroutine MPI_File_seek(fh, offset, whence, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: fh
-  integer(kind=MPI_OFFSET_KIND), intent(in) :: offset
-  integer, intent(in) :: whence
-  integer, intent(out) :: ierror
-end subroutine MPI_File_seek
-
-end interface
-
-
-interface MPI_File_seek_shared
-
-subroutine MPI_File_seek_shared(fh, offset, whence, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: fh
-  integer(kind=MPI_OFFSET_KIND), intent(in) :: offset
-  integer, intent(in) :: whence
-  integer, intent(out) :: ierror
-end subroutine MPI_File_seek_shared
-
-end interface
-
-
-interface MPI_File_set_atomicity
-
-subroutine MPI_File_set_atomicity(fh, flag, ierror)
-  integer, intent(in) :: fh
-  logical, intent(in) :: flag
-  integer, intent(out) :: ierror
-end subroutine MPI_File_set_atomicity
-
-end interface
-
-
-interface MPI_File_set_errhandler
-
-subroutine MPI_File_set_errhandler(file, errhandler, ierror)
-  integer, intent(in) :: file
-  integer, intent(in) :: errhandler
-  integer, intent(out) :: ierror
-end subroutine MPI_File_set_errhandler
-
-end interface
-
-
-interface MPI_File_set_info
-
-subroutine MPI_File_set_info(fh, info, ierror)
-  integer, intent(in) :: fh
-  integer, intent(in) :: info
-  integer, intent(out) :: ierror
-end subroutine MPI_File_set_info
-
-end interface
-
-
-interface MPI_File_set_size
-
-subroutine MPI_File_set_size(fh, size, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: fh
-  integer(kind=MPI_OFFSET_KIND), intent(in) :: size
-  integer, intent(out) :: ierror
-end subroutine MPI_File_set_size
-
-end interface
-
-
-interface MPI_File_set_view
-
-subroutine MPI_File_set_view(fh, disp, etype, filetype, datarep, &
-        info, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: fh
-  integer(kind=MPI_OFFSET_KIND), intent(in) :: disp
-  integer, intent(in) :: etype
-  integer, intent(in) :: filetype
-  character(len=*), intent(in) :: datarep
-  integer, intent(in) :: info
-  integer, intent(out) :: ierror
-end subroutine MPI_File_set_view
-
-end interface
-
-
-interface MPI_File_sync
-
-subroutine MPI_File_sync(fh, ierror)
-  integer, intent(in) :: fh
-  integer, intent(out) :: ierror
-end subroutine MPI_File_sync
-
-end interface
diff --git a/ompi/mpi/fortran/use-mpi-tkr/mpi-f90-interfaces.h b/ompi/mpi/fortran/use-mpi-tkr/mpi-f90-interfaces.h
deleted file mode 100644
index b6a5c08e71d..00000000000
--- a/ompi/mpi/fortran/use-mpi-tkr/mpi-f90-interfaces.h
+++ /dev/null
@@ -1,2447 +0,0 @@
-! -*- fortran -*-
-!
-! Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
-!                         University Research and Technology
-!                         Corporation.  All rights reserved.
-! Copyright (c) 2004-2005 The University of Tennessee and The University
-!                         of Tennessee Research Foundation.  All rights
-!                         reserved.
-! Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-!                         University of Stuttgart.  All rights reserved.
-! Copyright (c) 2004-2005 The Regents of the University of California.
-!                         All rights reserved.
-! Copyright (c) 2006-2014 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2016-2018 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-!
-! Additional copyrights may follow
-!
-! $HEADER$
-!
-
-interface MPI_Wtick
-
-function MPI_Wtick()
-    double precision MPI_Wtick
-end function MPI_Wtick
-
-end interface
-
-
-interface MPI_Wtime
-
-function MPI_Wtime()
-    double precision MPI_Wtime
-end function MPI_Wtime
-
-end interface
-
-
-interface MPI_Abort
-
-subroutine MPI_Abort(comm, errorcode, ierror)
-  integer, intent(in) :: comm
-  integer, intent(in) :: errorcode
-  integer, intent(out) :: ierror
-end subroutine MPI_Abort
-
-end interface
-
-
-interface MPI_Add_error_class
-
-subroutine MPI_Add_error_class(errorclass, ierror)
-  integer, intent(out) :: errorclass
-  integer, intent(out) :: ierror
-end subroutine MPI_Add_error_class
-
-end interface
-
-
-interface MPI_Add_error_code
-
-subroutine MPI_Add_error_code(errorclass, errorcode, ierror)
-  integer, intent(in) :: errorclass
-  integer, intent(out) :: errorcode
-  integer, intent(out) :: ierror
-end subroutine MPI_Add_error_code
-
-end interface
-
-
-interface MPI_Add_error_string
-
-subroutine MPI_Add_error_string(errorcode, string, ierror)
-  integer, intent(in) :: errorcode
-  character(len=*), intent(in) :: string
-  integer, intent(out) :: ierror
-end subroutine MPI_Add_error_string
-
-end interface
-
-interface MPI_Aint_add
-
-function MPI_Aint_add(base, diff)
-  include 'mpif-config.h'
-  integer(kind=MPI_ADDRESS_KIND), intent(in) :: base
-  integer(kind=MPI_ADDRESS_KIND), intent(in) :: diff
-  integer(kind=MPI_ADDRESS_KIND) MPI_Aint_add
-end function MPI_Aint_add
-
-end interface
-
-interface MPI_Aint_diff
-
-function MPI_Aint_diff(addr1, addr2)
-  include 'mpif-config.h'
-  integer(kind=MPI_ADDRESS_KIND), intent(in) :: addr1
-  integer(kind=MPI_ADDRESS_KIND), intent(in) :: addr2
-  integer(kind=MPI_ADDRESS_KIND) MPI_Aint_diff
-end function MPI_Aint_diff
-
-end interface
-
-
-interface MPI_Barrier
-
-subroutine MPI_Barrier(comm, ierror)
-  integer, intent(in) :: comm
-  integer, intent(out) :: ierror
-end subroutine MPI_Barrier
-
-end interface
-
-
-interface MPI_Ibarrier
-
-subroutine MPI_Ibarrier(comm, request, ierror)
-  integer, intent(in) :: comm
-  integer, intent(out) :: request
-  integer, intent(out) :: ierror
-end subroutine MPI_Ibarrier
-
-end interface
-
-
-interface MPI_Cancel
-
-subroutine MPI_Cancel(request, ierror)
-  integer, intent(in) :: request
-  integer, intent(out) :: ierror
-end subroutine MPI_Cancel
-
-end interface
-
-
-interface MPI_Cart_coords
-
-subroutine MPI_Cart_coords(comm, rank, maxdims, coords, ierror)
-  integer, intent(in) :: comm
-  integer, intent(in) :: rank
-  integer, intent(in) :: maxdims
-  integer, dimension(*), intent(out) :: coords
-  integer, intent(out) :: ierror
-end subroutine MPI_Cart_coords
-
-end interface
-
-
-interface MPI_Cart_create
-
-subroutine MPI_Cart_create(old_comm, ndims, dims, periods, reorder, &
-        comm_cart, ierror)
-  integer, intent(in) :: old_comm
-  integer, intent(in) :: ndims
-  integer, dimension(*), intent(in) :: dims
-  logical, dimension(*), intent(in) :: periods
-  logical, intent(in) :: reorder
-  integer, intent(out) :: comm_cart
-  integer, intent(out) :: ierror
-end subroutine MPI_Cart_create
-
-end interface
-
-
-interface MPI_Cart_get
-
-subroutine MPI_Cart_get(comm, maxdims, dims, periods, coords&
-        , ierror)
-  integer, intent(in) :: comm
-  integer, intent(in) :: maxdims
-  integer, dimension(*), intent(out) :: dims
-  logical, dimension(*), intent(out) :: periods
-  integer, dimension(*), intent(out) :: coords
-  integer, intent(out) :: ierror
-end subroutine MPI_Cart_get
-
-end interface
-
-
-interface MPI_Cart_map
-
-subroutine MPI_Cart_map(comm, ndims, dims, periods, newrank&
-        , ierror)
-  integer, intent(in) :: comm
-  integer, intent(in) :: ndims
-  integer, dimension(*), intent(in) :: dims
-  logical, dimension(*), intent(in) :: periods
-  integer, intent(out) :: newrank
-  integer, intent(out) :: ierror
-end subroutine MPI_Cart_map
-
-end interface
-
-
-interface MPI_Cart_rank
-
-subroutine MPI_Cart_rank(comm, coords, rank, ierror)
-  integer, intent(in) :: comm
-  integer, dimension(*), intent(in) :: coords
-  integer, intent(out) :: rank
-  integer, intent(out) :: ierror
-end subroutine MPI_Cart_rank
-
-end interface
-
-
-interface MPI_Cart_shift
-
-subroutine MPI_Cart_shift(comm, direction, disp, rank_source, rank_dest&
-        , ierror)
-  integer, intent(in) :: comm
-  integer, intent(in) :: direction
-  integer, intent(in) :: disp
-  integer, intent(out) :: rank_source
-  integer, intent(out) :: rank_dest
-  integer, intent(out) :: ierror
-end subroutine MPI_Cart_shift
-
-end interface
-
-
-interface MPI_Cart_sub
-
-subroutine MPI_Cart_sub(comm, remain_dims, new_comm, ierror)
-  integer, intent(in) :: comm
-  logical, dimension(*), intent(in) :: remain_dims
-  integer, intent(out) :: new_comm
-  integer, intent(out) :: ierror
-end subroutine MPI_Cart_sub
-
-end interface
-
-
-interface MPI_Cartdim_get
-
-subroutine MPI_Cartdim_get(comm, ndims, ierror)
-  integer, intent(in) :: comm
-  integer, intent(out) :: ndims
-  integer, intent(out) :: ierror
-end subroutine MPI_Cartdim_get
-
-end interface
-
-
-interface MPI_Comm_call_errhandler
-
-subroutine MPI_Comm_call_errhandler(comm, errorcode, ierror)
-  integer, intent(in) :: comm
-  integer, intent(in) :: errorcode
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_call_errhandler
-
-end interface
-
-
-interface MPI_Comm_compare
-
-subroutine MPI_Comm_compare(comm1, comm2, result, ierror)
-  integer, intent(in) :: comm1
-  integer, intent(in) :: comm2
-  integer, intent(out) :: result
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_compare
-
-end interface
-
-
-interface MPI_Comm_create
-
-subroutine MPI_Comm_create(comm, group, newcomm, ierror)
-  integer, intent(in) :: comm
-  integer, intent(in) :: group
-  integer, intent(out) :: newcomm
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_create
-
-end interface
-
-
-interface MPI_Comm_create_group
-
-subroutine MPI_Comm_create_group(comm, group, tag, newcomm, ierror)
-  integer, intent(in) :: comm
-  integer, intent(in) :: group
-  integer, intent(in) :: tag
-  integer, intent(out) :: newcomm
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_create_group
-
-end interface
-
-
-interface MPI_Comm_create_errhandler
-
-subroutine MPI_Comm_create_errhandler(function, errhandler, ierror)
-  external :: function
-  integer, intent(out) :: errhandler
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_create_errhandler
-
-end interface
-
-
-interface MPI_Comm_create_keyval
-
-subroutine MPI_Comm_create_keyval(comm_copy_attr_fn, comm_delete_attr_fn, comm_keyval, extra_state, ierror)
-  include 'mpif-config.h'
-  external :: comm_copy_attr_fn
-  external :: comm_delete_attr_fn
-  integer, intent(out) :: comm_keyval
-  integer(kind=MPI_ADDRESS_KIND), intent(in) :: extra_state
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_create_keyval
-
-end interface
-
-
-interface MPI_Comm_delete_attr
-
-subroutine MPI_Comm_delete_attr(comm, comm_keyval, ierror)
-  integer, intent(in) :: comm
-  integer, intent(in) :: comm_keyval
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_delete_attr
-
-end interface
-
-
-interface MPI_Comm_dup
-
-subroutine MPI_Comm_dup(comm, newcomm, ierror)
-  integer, intent(in) :: comm
-  integer, intent(out) :: newcomm
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_dup
-
-end interface
-
-
-interface MPI_Comm_dup_with_info
-
-subroutine MPI_Comm_dup_with_info(comm, info, newcomm, ierror)
-  integer, intent(in) :: comm
-  integer, intent(in) :: info
-  integer, intent(out) :: newcomm
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_dup_with_info
-
-end interface
-
-
-interface MPI_Comm_idup
-
-subroutine MPI_Comm_idup(comm, newcomm, request, ierror)
-  integer, intent(in) :: comm
-  integer, intent(out) :: newcomm
-  integer, intent(out) :: request
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_idup
-
-end interface
-
-
-interface MPI_Comm_free
-
-subroutine MPI_Comm_free(comm, ierror)
-  integer, intent(inout) :: comm
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_free
-
-end interface
-
-
-interface MPI_Comm_free_keyval
-
-subroutine MPI_Comm_free_keyval(comm_keyval, ierror)
-  integer, intent(inout) :: comm_keyval
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_free_keyval
-
-end interface
-
-
-interface MPI_Comm_get_info
-
-subroutine MPI_Comm_get_info(comm, info_used, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: comm
-  integer, intent(out) :: info_used
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_get_info
-
-end interface
-
-
-interface MPI_Comm_get_attr
-
-subroutine MPI_Comm_get_attr(comm, comm_keyval, attribute_val, flag, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: comm
-  integer, intent(in) :: comm_keyval
-  integer(kind=MPI_ADDRESS_KIND), intent(out) :: attribute_val
-  logical, intent(out) :: flag
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_get_attr
-
-end interface
-
-
-interface MPI_Comm_get_errhandler
-
-subroutine MPI_Comm_get_errhandler(comm, erhandler, ierror)
-  integer, intent(in) :: comm
-  integer, intent(out) :: erhandler
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_get_errhandler
-
-end interface
-
-
-interface MPI_Comm_get_name
-
-subroutine MPI_Comm_get_name(comm, comm_name, resultlen, ierror)
-  integer, intent(in) :: comm
-  character(len=*), intent(out) :: comm_name
-  integer, intent(out) :: resultlen
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_get_name
-
-end interface
-
-
-interface MPI_Comm_group
-
-subroutine MPI_Comm_group(comm, group, ierror)
-  integer, intent(in) :: comm
-  integer, intent(out) :: group
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_group
-
-end interface
-
-
-interface MPI_Comm_rank
-
-subroutine MPI_Comm_rank(comm, rank, ierror)
-  integer, intent(in) :: comm
-  integer, intent(out) :: rank
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_rank
-
-end interface
-
-
-interface MPI_Comm_remote_group
-
-subroutine MPI_Comm_remote_group(comm, group, ierror)
-  integer, intent(in) :: comm
-  integer, intent(out) :: group
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_remote_group
-
-end interface
-
-
-interface MPI_Comm_remote_size
-
-subroutine MPI_Comm_remote_size(comm, size, ierror)
-  integer, intent(in) :: comm
-  integer, intent(out) :: size
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_remote_size
-
-end interface
-
-
-interface MPI_Comm_set_info
-
-subroutine MPI_Comm_set_info(comm, info, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: comm
-  integer, intent(in) :: info
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_set_info
-
-end interface
-
-
-interface MPI_Comm_set_attr
-
-subroutine MPI_Comm_set_attr(comm, comm_keyval, attribute_val, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: comm
-  integer, intent(in) :: comm_keyval
-  integer(kind=MPI_ADDRESS_KIND), intent(in) :: attribute_val
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_set_attr
-
-end interface
-
-
-interface MPI_Comm_set_errhandler
-
-subroutine MPI_Comm_set_errhandler(comm, errhandler, ierror)
-  integer, intent(in) :: comm
-  integer, intent(in) :: errhandler
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_set_errhandler
-
-end interface
-
-
-interface MPI_Comm_set_name
-
-subroutine MPI_Comm_set_name(comm, comm_name, ierror)
-  integer, intent(in) :: comm
-  character(len=*), intent(in) :: comm_name
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_set_name
-
-end interface
-
-
-interface MPI_Comm_size
-
-subroutine MPI_Comm_size(comm, size, ierror)
-  integer, intent(in) :: comm
-  integer, intent(out) :: size
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_size
-
-end interface
-
-
-interface MPI_Comm_split
-
-subroutine MPI_Comm_split(comm, color, key, newcomm, ierror)
-  integer, intent(in) :: comm
-  integer, intent(in) :: color
-  integer, intent(in) :: key
-  integer, intent(out) :: newcomm
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_split
-
-end interface
-
-
-interface MPI_Comm_test_inter
-
-subroutine MPI_Comm_test_inter(comm, flag, ierror)
-  integer, intent(in) :: comm
-  logical, intent(out) :: flag
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_test_inter
-
-end interface
-
-
-interface MPI_Dims_create
-
-subroutine MPI_Dims_create(nnodes, ndims, dims, ierror)
-  integer, intent(in) :: nnodes
-  integer, intent(in) :: ndims
-  integer, dimension(*), intent(inout) :: dims
-  integer, intent(out) :: ierror
-end subroutine MPI_Dims_create
-
-end interface
-
-
-interface MPI_Errhandler_free
-
-subroutine MPI_Errhandler_free(errhandler, ierror)
-  integer, intent(inout) :: errhandler
-  integer, intent(out) :: ierror
-end subroutine MPI_Errhandler_free
-
-end interface
-
-
-interface MPI_Error_class
-
-subroutine MPI_Error_class(errorcode, errorclass, ierror)
-  integer, intent(in) :: errorcode
-  integer, intent(out) :: errorclass
-  integer, intent(out) :: ierror
-end subroutine MPI_Error_class
-
-end interface
-
-
-interface MPI_Error_string
-
-subroutine MPI_Error_string(errorcode, string, resultlen, ierror)
-  integer, intent(in) :: errorcode
-  character(len=*), intent(out) :: string
-  integer, intent(out) :: resultlen
-  integer, intent(out) :: ierror
-end subroutine MPI_Error_string
-
-end interface
-
-
-interface MPI_Finalize
-
-subroutine MPI_Finalize(ierror)
-  integer, intent(out) :: ierror
-end subroutine MPI_Finalize
-
-end interface
-
-
-interface MPI_Finalized
-
-subroutine MPI_Finalized(flag, ierror)
-  logical, intent(out) :: flag
-  integer, intent(out) :: ierror
-end subroutine MPI_Finalized
-
-end interface
-
-
-interface MPI_Get_count
-
-subroutine MPI_Get_count(status, datatype, count, ierror)
-  include 'mpif-config.h'
-  integer, dimension(MPI_STATUS_SIZE), intent(in) :: status
-  integer, intent(in) :: datatype
-  integer, intent(out) :: count
-  integer, intent(out) :: ierror
-end subroutine MPI_Get_count
-
-end interface
-
-
-interface MPI_Get_elements
-
-subroutine MPI_Get_elements(status, datatype, count, ierror)
-  include 'mpif-config.h'
-  integer, dimension(MPI_STATUS_SIZE), intent(in) :: status
-  integer, intent(in) :: datatype
-  integer, intent(out) :: count
-  integer, intent(out) :: ierror
-end subroutine MPI_Get_elements
-
-end interface
-
-
-interface MPI_Get_elements_x
-
-subroutine MPI_Get_elements_x(status, datatype, count, ierror)
-  include 'mpif-config.h'
-  integer, dimension(MPI_STATUS_SIZE), intent(in) :: status
-  integer, intent(in) :: datatype
-  integer(kind=MPI_COUNT_KIND), intent(out) :: count
-  integer, intent(out) :: ierror
-end subroutine MPI_Get_elements_x
-
-end interface
-
-
-interface MPI_Get_processor_name
-
-subroutine MPI_Get_processor_name(name, resultlen, ierror)
-  character(len=*), intent(out) :: name
-  integer, intent(out) :: resultlen
-  integer, intent(out) :: ierror
-end subroutine MPI_Get_processor_name
-
-end interface
-
-
-interface MPI_Get_version
-
-subroutine MPI_Get_version(version, subversion, ierror)
-  integer, intent(out) :: version
-  integer, intent(out) :: subversion
-  integer, intent(out) :: ierror
-end subroutine MPI_Get_version
-
-end interface
-
-
-interface MPI_Graph_create
-
-subroutine MPI_Graph_create(comm_old, nnodes, index, edges, reorder, &
-        comm_graph, ierror)
-  integer, intent(in) :: comm_old
-  integer, intent(in) :: nnodes
-  integer, dimension(*), intent(in) :: index
-  integer, dimension(*), intent(in) :: edges
-  logical, intent(in) :: reorder
-  integer, intent(out) :: comm_graph
-  integer, intent(out) :: ierror
-end subroutine MPI_Graph_create
-
-end interface
-
-
-interface MPI_Graph_get
-
-subroutine MPI_Graph_get(comm, maxindex, maxedges, index, edges&
-        , ierror)
-  integer, intent(in) :: comm
-  integer, intent(in) :: maxindex
-  integer, intent(in) :: maxedges
-  integer, dimension(*), intent(out) :: index
-  integer, dimension(*), intent(out) :: edges
-  integer, intent(out) :: ierror
-end subroutine MPI_Graph_get
-
-end interface
-
-
-interface MPI_Graph_map
-
-subroutine MPI_Graph_map(comm, nnodes, index, edges, newrank&
-        , ierror)
-  integer, intent(in) :: comm
-  integer, intent(in) :: nnodes
-  integer, dimension(*), intent(in) :: index
-  integer, dimension(*), intent(in) :: edges
-  integer, intent(out) :: newrank
-  integer, intent(out) :: ierror
-end subroutine MPI_Graph_map
-
-end interface
-
-
-interface MPI_Graph_neighbors
-
-subroutine MPI_Graph_neighbors(comm, rank, maxneighbors, neighbors, ierror)
-  integer, intent(in) :: comm
-  integer, intent(in) :: rank
-  integer, intent(in) :: maxneighbors
-  integer, dimension(*), intent(out) :: neighbors
-  integer, intent(out) :: ierror
-end subroutine MPI_Graph_neighbors
-
-end interface
-
-
-interface MPI_Graph_neighbors_count
-
-subroutine MPI_Graph_neighbors_count(comm, rank, nneighbors, ierror)
-  integer, intent(in) :: comm
-  integer, intent(in) :: rank
-  integer, intent(out) :: nneighbors
-  integer, intent(out) :: ierror
-end subroutine MPI_Graph_neighbors_count
-
-end interface
-
-
-interface MPI_Graphdims_get
-
-subroutine MPI_Graphdims_get(comm, nnodes, nedges, ierror)
-  integer, intent(in) :: comm
-  integer, intent(out) :: nnodes
-  integer, intent(out) :: nedges
-  integer, intent(out) :: ierror
-end subroutine MPI_Graphdims_get
-
-end interface
-
-
-interface MPI_Grequest_complete
-
-subroutine MPI_Grequest_complete(request, ierror)
-  integer, intent(in) :: request
-  integer, intent(out) :: ierror
-end subroutine MPI_Grequest_complete
-
-end interface
-
-
-interface MPI_Grequest_start
-
-subroutine MPI_Grequest_start(query_fn, free_fn, cancel_fn, extra_state, request&
-        , ierror)
-  include 'mpif-config.h'
-  external :: query_fn
-  external :: free_fn
-  external :: cancel_fn
-  integer(kind=MPI_ADDRESS_KIND), intent(in) :: extra_state
-  integer, intent(out) :: request
-  integer, intent(out) :: ierror
-end subroutine MPI_Grequest_start
-
-end interface
-
-
-interface MPI_Group_compare
-
-subroutine MPI_Group_compare(group1, group2, result, ierror)
-  integer, intent(in) :: group1
-  integer, intent(in) :: group2
-  integer, intent(out) :: result
-  integer, intent(out) :: ierror
-end subroutine MPI_Group_compare
-
-end interface
-
-
-interface MPI_Group_difference
-
-subroutine MPI_Group_difference(group1, group2, newgroup, ierror)
-  integer, intent(in) :: group1
-  integer, intent(in) :: group2
-  integer, intent(out) :: newgroup
-  integer, intent(out) :: ierror
-end subroutine MPI_Group_difference
-
-end interface
-
-
-interface MPI_Group_excl
-
-subroutine MPI_Group_excl(group, n, ranks, newgroup, ierror)
-  integer, intent(in) :: group
-  integer, intent(in) :: n
-  integer, dimension(*), intent(in) :: ranks
-  integer, intent(out) :: newgroup
-  integer, intent(out) :: ierror
-end subroutine MPI_Group_excl
-
-end interface
-
-
-interface MPI_Group_free
-
-subroutine MPI_Group_free(group, ierror)
-  integer, intent(inout) :: group
-  integer, intent(out) :: ierror
-end subroutine MPI_Group_free
-
-end interface
-
-
-interface MPI_Group_incl
-
-subroutine MPI_Group_incl(group, n, ranks, newgroup, ierror)
-  integer, intent(in) :: group
-  integer, intent(in) :: n
-  integer, dimension(*), intent(in) :: ranks
-  integer, intent(out) :: newgroup
-  integer, intent(out) :: ierror
-end subroutine MPI_Group_incl
-
-end interface
-
-
-interface MPI_Group_intersection
-
-subroutine MPI_Group_intersection(group1, group2, newgroup, ierror)
-  integer, intent(in) :: group1
-  integer, intent(in) :: group2
-  integer, intent(out) :: newgroup
-  integer, intent(out) :: ierror
-end subroutine MPI_Group_intersection
-
-end interface
-
-
-interface MPI_Group_range_excl
-
-subroutine MPI_Group_range_excl(group, n, ranges, newgroup, ierror)
-  integer, intent(in) :: group
-  integer, intent(in) :: n
-  integer, dimension(3, *), intent(in) :: ranges
-  integer, intent(out) :: newgroup
-  integer, intent(out) :: ierror
-end subroutine MPI_Group_range_excl
-
-end interface
-
-
-interface MPI_Group_range_incl
-
-subroutine MPI_Group_range_incl(group, n, ranges, newgroup, ierror)
-  integer, intent(in) :: group
-  integer, intent(in) :: n
-  integer, dimension(3, *), intent(in) :: ranges
-  integer, intent(out) :: newgroup
-  integer, intent(out) :: ierror
-end subroutine MPI_Group_range_incl
-
-end interface
-
-
-interface MPI_Group_rank
-
-subroutine MPI_Group_rank(group, rank, ierror)
-  integer, intent(in) :: group
-  integer, intent(out) :: rank
-  integer, intent(out) :: ierror
-end subroutine MPI_Group_rank
-
-end interface
-
-
-interface MPI_Group_size
-
-subroutine MPI_Group_size(group, size, ierror)
-  integer, intent(in) :: group
-  integer, intent(out) :: size
-  integer, intent(out) :: ierror
-end subroutine MPI_Group_size
-
-end interface
-
-
-interface MPI_Group_translate_ranks
-
-subroutine MPI_Group_translate_ranks(group1, n, ranks1, group2, ranks2&
-        , ierror)
-  integer, intent(in) :: group1
-  integer, intent(in) :: n
-  integer, dimension(*), intent(in) :: ranks1
-  integer, intent(in) :: group2
-  integer, dimension(*), intent(out) :: ranks2
-  integer, intent(out) :: ierror
-end subroutine MPI_Group_translate_ranks
-
-end interface
-
-
-interface MPI_Group_union
-
-subroutine MPI_Group_union(group1, group2, newgroup, ierror)
-  integer, intent(in) :: group1
-  integer, intent(in) :: group2
-  integer, intent(out) :: newgroup
-  integer, intent(out) :: ierror
-end subroutine MPI_Group_union
-
-end interface
-
-
-interface MPI_Info_create
-
-subroutine MPI_Info_create(info, ierror)
-  integer, intent(out) :: info
-  integer, intent(out) :: ierror
-end subroutine MPI_Info_create
-
-end interface
-
-
-interface MPI_Info_delete
-
-subroutine MPI_Info_delete(info, key, ierror)
-  integer, intent(in) :: info
-  character(len=*), intent(in) :: key
-  integer, intent(out) :: ierror
-end subroutine MPI_Info_delete
-
-end interface
-
-
-interface MPI_Info_dup
-
-subroutine MPI_Info_dup(info, newinfo, ierror)
-  integer, intent(in) :: info
-  integer, intent(out) :: newinfo
-  integer, intent(out) :: ierror
-end subroutine MPI_Info_dup
-
-end interface
-
-
-interface MPI_Info_free
-
-subroutine MPI_Info_free(info, ierror)
-  integer, intent(inout) :: info
-  integer, intent(out) :: ierror
-end subroutine MPI_Info_free
-
-end interface
-
-
-interface MPI_Info_get
-
-subroutine MPI_Info_get(info, key, valuelen, value, flag&
-        , ierror)
-  integer, intent(in) :: info
-  character(len=*), intent(in) :: key
-  integer, intent(in) :: valuelen
-  character(len=*), intent(out) :: value
-  logical, intent(out) :: flag
-  integer, intent(out) :: ierror
-end subroutine MPI_Info_get
-
-end interface
-
-
-interface MPI_Info_get_nkeys
-
-subroutine MPI_Info_get_nkeys(info, nkeys, ierror)
-  integer, intent(in) :: info
-  integer, intent(out) :: nkeys
-  integer, intent(out) :: ierror
-end subroutine MPI_Info_get_nkeys
-
-end interface
-
-
-interface MPI_Info_get_nthkey
-
-subroutine MPI_Info_get_nthkey(info, n, key, ierror)
-  integer, intent(in) :: info
-  integer, intent(in) :: n
-  character(len=*), intent(out) :: key
-  integer, intent(out) :: ierror
-end subroutine MPI_Info_get_nthkey
-
-end interface
-
-
-interface MPI_Info_get_valuelen
-
-subroutine MPI_Info_get_valuelen(info, key, valuelen, flag, ierror)
-  integer, intent(in) :: info
-  character(len=*), intent(in) :: key
-  integer, intent(out) :: valuelen
-  logical, intent(out) :: flag
-  integer, intent(out) :: ierror
-end subroutine MPI_Info_get_valuelen
-
-end interface
-
-
-interface MPI_Info_set
-
-subroutine MPI_Info_set(info, key, value, ierror)
-  integer, intent(in) :: info
-  character(len=*), intent(in) :: key
-  character(len=*), intent(in) :: value
-  integer, intent(out) :: ierror
-end subroutine MPI_Info_set
-
-end interface
-
-
-interface MPI_Init
-
-subroutine MPI_Init(ierror)
-  integer, intent(out) :: ierror
-end subroutine MPI_Init
-
-end interface
-
-
-interface MPI_Init_thread
-
-subroutine MPI_Init_thread(required, provided, ierror)
-  integer, intent(in) :: required
-  integer, intent(out) :: provided
-  integer, intent(out) :: ierror
-end subroutine MPI_Init_thread
-
-end interface
-
-
-interface MPI_Initialized
-
-subroutine MPI_Initialized(flag, ierror)
-  logical, intent(out) :: flag
-  integer, intent(out) :: ierror
-end subroutine MPI_Initialized
-
-end interface
-
-
-interface MPI_Intercomm_create
-
-subroutine MPI_Intercomm_create(local_comm, local_leader, bridge_comm, remote_leader, tag, &
-        newintercomm, ierror)
-  integer, intent(in) :: local_comm
-  integer, intent(in) :: local_leader
-  integer, intent(in) :: bridge_comm
-  integer, intent(in) :: remote_leader
-  integer, intent(in) :: tag
-  integer, intent(out) :: newintercomm
-  integer, intent(out) :: ierror
-end subroutine MPI_Intercomm_create
-
-end interface
-
-
-interface MPI_Intercomm_merge
-
-subroutine MPI_Intercomm_merge(intercomm, high, newintercomm, ierror)
-  integer, intent(in) :: intercomm
-  logical, intent(in) :: high
-  integer, intent(out) :: newintercomm
-  integer, intent(out) :: ierror
-end subroutine MPI_Intercomm_merge
-
-end interface
-
-
-interface MPI_Iprobe
-
-subroutine MPI_Iprobe(source, tag, comm, flag, status&
-        , ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: source
-  integer, intent(in) :: tag
-  integer, intent(in) :: comm
-  logical, intent(out) :: flag
-  integer, dimension(MPI_STATUS_SIZE), intent(out) :: status
-  integer, intent(out) :: ierror
-end subroutine MPI_Iprobe
-
-end interface
-
-
-interface MPI_Is_thread_main
-
-subroutine MPI_Is_thread_main(flag, ierror)
-  logical, intent(out) :: flag
-  integer, intent(out) :: ierror
-end subroutine MPI_Is_thread_main
-
-end interface
-
-
-interface MPI_Op_commutative
-
-subroutine MPI_Op_commutative(op, commute, ierror)
-  integer, intent(in) :: op
-  logical, intent(out) :: commute
-  integer, intent(out) :: ierror
-end subroutine MPI_Op_commutative
-
-end interface
-
-
-interface MPI_Op_create
-
-subroutine MPI_Op_create(function, commute, op, ierror)
-  external :: function
-  logical, intent(in) :: commute
-  integer, intent(out) :: op
-  integer, intent(out) :: ierror
-end subroutine MPI_Op_create
-
-end interface
-
-
-interface MPI_Op_free
-
-subroutine MPI_Op_free(op, ierror)
-  integer, intent(inout) :: op
-  integer, intent(out) :: ierror
-end subroutine MPI_Op_free
-
-end interface
-
-
-interface MPI_Pack_external_size
-
-subroutine MPI_Pack_external_size(datarep, incount, datatype, size, ierror)
-  include 'mpif-config.h'
-  character(len=*), intent(in) :: datarep
-  integer, intent(in) :: incount
-  integer, intent(in) :: datatype
-  integer(kind=MPI_ADDRESS_KIND), intent(out) :: size
-  integer, intent(out) :: ierror
-end subroutine MPI_Pack_external_size
-
-end interface
-
-
-interface MPI_Pack_size
-
-subroutine MPI_Pack_size(incount, datatype, comm, size, ierror)
-  integer, intent(in) :: incount
-  integer, intent(in) :: datatype
-  integer, intent(in) :: comm
-  integer, intent(out) :: size
-  integer, intent(out) :: ierror
-end subroutine MPI_Pack_size
-
-end interface
-
-
-interface MPI_Pcontrol
-
-subroutine MPI_Pcontrol(level)
-  integer, intent(in) :: level
-
-end subroutine MPI_Pcontrol
-
-end interface
-
-
-interface MPI_Probe
-
-subroutine MPI_Probe(source, tag, comm, status, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: source
-  integer, intent(in) :: tag
-  integer, intent(in) :: comm
-  integer, dimension(MPI_STATUS_SIZE), intent(out) :: status
-  integer, intent(out) :: ierror
-end subroutine MPI_Probe
-
-end interface
-
-
-interface MPI_Query_thread
-
-subroutine MPI_Query_thread(provided, ierror)
-  integer, intent(out) :: provided
-  integer, intent(out) :: ierror
-end subroutine MPI_Query_thread
-
-end interface
-
-
-interface MPI_Register_datarep
-
-subroutine MPI_Register_datarep(datarep, read_conversion_fn, write_conversion_fn, dtype_file_extent_fn, extra_state&
-        , ierror)
-  include 'mpif-config.h'
-  character(len=*), intent(in) :: datarep
-  external :: read_conversion_fn
-  external :: write_conversion_fn
-  external :: dtype_file_extent_fn
-  integer(kind=MPI_ADDRESS_KIND), intent(in) :: extra_state
-  integer, intent(out) :: ierror
-end subroutine MPI_Register_datarep
-
-end interface
-
-
-interface MPI_Request_free
-
-subroutine MPI_Request_free(request, ierror)
-  integer, intent(inout) :: request
-  integer, intent(out) :: ierror
-end subroutine MPI_Request_free
-
-end interface
-
-
-interface MPI_Request_get_status
-
-subroutine MPI_Request_get_status(request, flag, status, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: request
-  logical, intent(out) :: flag
-  integer, dimension(MPI_STATUS_SIZE), intent(out) :: status
-  integer, intent(out) :: ierror
-end subroutine MPI_Request_get_status
-
-end interface
-
-
-interface MPI_Start
-
-subroutine MPI_Start(request, ierror)
-  integer, intent(inout) :: request
-  integer, intent(out) :: ierror
-end subroutine MPI_Start
-
-end interface
-
-
-interface MPI_Startall
-
-subroutine MPI_Startall(count, array_of_requests, ierror)
-  integer, intent(in) :: count
-  integer, dimension(*), intent(inout) :: array_of_requests
-  integer, intent(out) :: ierror
-end subroutine MPI_Startall
-
-end interface
-
-
-interface MPI_Status_set_cancelled
-
-subroutine MPI_Status_set_cancelled(status, flag, ierror)
-  include 'mpif-config.h'
-  integer, dimension(MPI_STATUS_SIZE), intent(inout) :: status
-  logical, intent(in) :: flag
-  integer, intent(out) :: ierror
-end subroutine MPI_Status_set_cancelled
-
-end interface
-
-
-interface MPI_Status_set_elements
-
-subroutine MPI_Status_set_elements(status, datatype, count, ierror)
-  include 'mpif-config.h'
-  integer, dimension(MPI_STATUS_SIZE), intent(inout) :: status
-  integer, intent(in) :: datatype
-  integer, intent(in) :: count
-  integer, intent(out) :: ierror
-end subroutine MPI_Status_set_elements
-
-end interface
-
-
-interface MPI_Test
-
-subroutine MPI_Test(request, flag, status, ierror)
-  include 'mpif-config.h'
-  integer, intent(inout) :: request
-  logical, intent(out) :: flag
-  integer, dimension(MPI_STATUS_SIZE), intent(out) :: status
-  integer, intent(out) :: ierror
-end subroutine MPI_Test
-
-end interface
-
-
-interface MPI_Test_cancelled
-
-subroutine MPI_Test_cancelled(status, flag, ierror)
-  include 'mpif-config.h'
-  integer, dimension(MPI_STATUS_SIZE), intent(in) :: status
-  logical, intent(out) :: flag
-  integer, intent(out) :: ierror
-end subroutine MPI_Test_cancelled
-
-end interface
-
-
-interface MPI_Testall
-
-subroutine MPI_Testall(count, array_of_requests, flag, array_of_statuses, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: count
-  integer, dimension(count), intent(inout) :: array_of_requests
-  logical, intent(out) :: flag
-  integer, dimension(MPI_STATUS_SIZE, count), intent(out) :: array_of_statuses
-  integer, intent(out) :: ierror
-end subroutine MPI_Testall
-
-end interface
-
-
-interface MPI_Testany
-
-subroutine MPI_Testany(count, array_of_requests, index, flag, status&
-        , ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: count
-  integer, dimension(count), intent(inout) :: array_of_requests
-  integer, intent(out) :: index
-  logical, intent(out) :: flag
-  integer, dimension(MPI_STATUS_SIZE), intent(out) :: status
-  integer, intent(out) :: ierror
-end subroutine MPI_Testany
-
-end interface
-
-
-interface MPI_Testsome
-
-subroutine MPI_Testsome(incount, array_of_requests, outcount, array_of_indices, array_of_statuses&
-        , ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: incount
-  integer, dimension(incount), intent(inout) :: array_of_requests
-  integer, intent(out) :: outcount
-  integer, dimension(*), intent(out) :: array_of_indices
-  integer, dimension(MPI_STATUS_SIZE, *), intent(out) :: array_of_statuses
-  integer, intent(out) :: ierror
-end subroutine MPI_Testsome
-
-end interface
-
-
-interface MPI_Topo_test
-
-subroutine MPI_Topo_test(comm, status, ierror)
-  integer, intent(in) :: comm
-  integer, intent(out) :: status
-  integer, intent(out) :: ierror
-end subroutine MPI_Topo_test
-
-end interface
-
-
-interface MPI_Type_commit
-
-subroutine MPI_Type_commit(datatype, ierror)
-  integer, intent(inout) :: datatype
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_commit
-
-end interface
-
-
-interface MPI_Type_contiguous
-
-subroutine MPI_Type_contiguous(count, oldtype, newtype, ierror)
-  integer, intent(in) :: count
-  integer, intent(in) :: oldtype
-  integer, intent(out) :: newtype
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_contiguous
-
-end interface
-
-
-interface MPI_Type_create_darray
-
-subroutine MPI_Type_create_darray(size, rank, ndims, gsize_array, distrib_array, &
-        darg_array, psize_array, order, oldtype, newtype, ierror)
-  integer, intent(in) :: size
-  integer, intent(in) :: rank
-  integer, intent(in) :: ndims
-  integer, dimension(*), intent(in) :: gsize_array
-  integer, dimension(*), intent(in) :: distrib_array
-  integer, dimension(*), intent(in) :: darg_array
-  integer, dimension(*), intent(in) :: psize_array
-  integer, intent(in) :: order
-  integer, intent(in) :: oldtype
-  integer, intent(out) :: newtype
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_create_darray
-
-end interface
-
-
-interface MPI_Type_create_f90_complex
-
-subroutine MPI_Type_create_f90_complex(p, r, newtype, ierror)
-  integer, intent(in) :: p
-  integer, intent(in) :: r
-  integer, intent(out) :: newtype
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_create_f90_complex
-
-end interface
-
-
-interface MPI_Type_create_f90_integer
-
-subroutine MPI_Type_create_f90_integer(r, newtype, ierror)
-  integer, intent(in) :: r
-  integer, intent(out) :: newtype
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_create_f90_integer
-
-end interface
-
-
-interface MPI_Type_create_f90_real
-
-subroutine MPI_Type_create_f90_real(p, r, newtype, ierror)
-  integer, intent(in) :: p
-  integer, intent(in) :: r
-  integer, intent(out) :: newtype
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_create_f90_real
-
-end interface
-
-
-interface MPI_Type_create_hindexed
-
-subroutine MPI_Type_create_hindexed(count, array_of_blocklengths, array_of_displacements, oldtype, newtype&
-        , ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: count
-  integer, dimension(*), intent(in) :: array_of_blocklengths
-  integer(kind=MPI_ADDRESS_KIND), dimension(*), intent(in) :: array_of_displacements
-  integer, intent(in) :: oldtype
-  integer, intent(out) :: newtype
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_create_hindexed
-
-end interface
-
-
-interface MPI_Type_create_hvector
-
-subroutine MPI_Type_create_hvector(count, blocklength, stride, oldtype, newtype&
-        , ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: count
-  integer, intent(in) :: blocklength
-  integer(kind=MPI_ADDRESS_KIND), intent(in) :: stride
-  integer, intent(in) :: oldtype
-  integer, intent(out) :: newtype
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_create_hvector
-
-end interface
-
-
-interface MPI_Type_create_indexed_block
-
-subroutine MPI_Type_create_indexed_block(count, blocklength, array_of_displacements, oldtype, newtype&
-        , ierror)
-  integer, intent(in) :: count
-  integer, intent(in) :: blocklength
-  integer, dimension(*), intent(in) :: array_of_displacements
-  integer, intent(in) :: oldtype
-  integer, intent(out) :: newtype
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_create_indexed_block
-
-end interface
-
-
-interface MPI_Type_create_keyval
-
-subroutine MPI_Type_create_keyval(type_copy_attr_fn, type_delete_attr_fn, type_keyval, extra_state, ierror)
-  include 'mpif-config.h'
-  external :: type_copy_attr_fn
-  external :: type_delete_attr_fn
-  integer, intent(out) :: type_keyval
-  integer(kind=MPI_ADDRESS_KIND), intent(in) :: extra_state
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_create_keyval
-
-end interface
-
-
-interface MPI_Type_create_resized
-
-subroutine MPI_Type_create_resized(oldtype, lb, extent, newtype, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: oldtype
-  integer(kind=MPI_ADDRESS_KIND), intent(in) :: lb
-  integer(kind=MPI_ADDRESS_KIND), intent(in) :: extent
-  integer, intent(out) :: newtype
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_create_resized
-
-end interface
-
-
-interface MPI_Type_create_struct
-
-subroutine MPI_Type_create_struct(count, array_of_block_lengths, array_of_displacements, array_of_types, newtype&
-        , ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: count
-  integer, dimension(*), intent(in) :: array_of_block_lengths
-  integer(kind=MPI_ADDRESS_KIND), dimension(*), intent(in) :: array_of_displacements
-  integer, dimension(*), intent(in) :: array_of_types
-  integer, intent(out) :: newtype
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_create_struct
-
-end interface
-
-
-interface MPI_Type_create_subarray
-
-subroutine MPI_Type_create_subarray(ndims, size_array, subsize_array, start_array, order, &
-        oldtype, newtype, ierror)
-  integer, intent(in) :: ndims
-  integer, dimension(*), intent(in) :: size_array
-  integer, dimension(*), intent(in) :: subsize_array
-  integer, dimension(*), intent(in) :: start_array
-  integer, intent(in) :: order
-  integer, intent(in) :: oldtype
-  integer, intent(out) :: newtype
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_create_subarray
-
-end interface
-
-
-interface MPI_Type_delete_attr
-
-subroutine MPI_Type_delete_attr(datatype, type_keyval, ierror)
-  integer, intent(in) :: datatype
-  integer, intent(in) :: type_keyval
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_delete_attr
-
-end interface
-
-
-interface MPI_Type_dup
-
-subroutine MPI_Type_dup(oldtype, newtype, ierror)
-  integer, intent(in) :: oldtype
-  integer, intent(out) :: newtype
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_dup
-
-end interface
-
-
-interface MPI_Type_free
-
-subroutine MPI_Type_free(datatype, ierror)
-  integer, intent(inout) :: datatype
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_free
-
-end interface
-
-
-interface MPI_Type_free_keyval
-
-subroutine MPI_Type_free_keyval(type_keyval, ierror)
-  integer, intent(inout) :: type_keyval
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_free_keyval
-
-end interface
-
-
-interface MPI_Type_get_attr
-
-subroutine MPI_Type_get_attr(datatype, type_keyval, attribute_val, flag, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: datatype
-  integer, intent(in) :: type_keyval
-  integer(kind=MPI_ADDRESS_KIND), intent(out) :: attribute_val
-  logical, intent(out) :: flag
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_get_attr
-
-end interface
-
-
-interface MPI_Type_get_contents
-
-subroutine MPI_Type_get_contents(datatype, max_integers, max_addresses, max_datatypes, array_of_integers, &
-        array_of_addresses, array_of_datatypes, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: datatype
-  integer, intent(in) :: max_integers
-  integer, intent(in) :: max_addresses
-  integer, intent(in) :: max_datatypes
-  integer, dimension(*), intent(out) :: array_of_integers
-  integer(kind=MPI_ADDRESS_KIND), dimension(*), intent(out) :: array_of_addresses
-  integer, dimension(*), intent(out) :: array_of_datatypes
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_get_contents
-
-end interface
-
-
-interface MPI_Type_get_envelope
-
-subroutine MPI_Type_get_envelope(datatype, num_integers, num_addresses, num_datatypes, combiner&
-        , ierror)
-  integer, intent(in) :: datatype
-  integer, intent(out) :: num_integers
-  integer, intent(out) :: num_addresses
-  integer, intent(out) :: num_datatypes
-  integer, intent(out) :: combiner
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_get_envelope
-
-end interface
-
-
-interface MPI_Type_get_extent
-
-subroutine MPI_Type_get_extent(datatype, lb, extent, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: datatype
-  integer(kind=MPI_ADDRESS_KIND), intent(out) :: lb
-  integer(kind=MPI_ADDRESS_KIND), intent(out) :: extent
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_get_extent
-
-end interface
-
-
-interface MPI_Type_get_extent_x
-
-subroutine MPI_Type_get_extent_x(datatype, lb, extent, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: datatype
-  integer(kind=MPI_COUNT_KIND), intent(out) :: lb
-  integer(kind=MPI_COUNT_KIND), intent(out) :: extent
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_get_extent_x
-
-end interface
-
-
-interface MPI_Type_get_name
-
-subroutine MPI_Type_get_name(datatype, type_name, resultlen, ierror)
-  integer, intent(in) :: datatype
-  character(len=*), intent(out) :: type_name
-  integer, intent(out) :: resultlen
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_get_name
-
-end interface
-
-
-interface MPI_Type_get_true_extent
-
-subroutine MPI_Type_get_true_extent(datatype, true_lb, true_extent, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: datatype
-  integer(kind=MPI_ADDRESS_KIND), intent(out) :: true_lb
-  integer(kind=MPI_ADDRESS_KIND), intent(out) :: true_extent
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_get_true_extent
-
-end interface
-
-
-interface MPI_Type_get_true_extent_x
-
-subroutine MPI_Type_get_true_extent_x(datatype, true_lb, true_extent, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: datatype
-  integer(kind=MPI_COUNT_KIND), intent(out) :: true_lb
-  integer(kind=MPI_COUNT_KIND), intent(out) :: true_extent
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_get_true_extent_x
-
-end interface
-
-
-interface MPI_Type_indexed
-
-subroutine MPI_Type_indexed(count, array_of_blocklengths, array_of_displacements, oldtype, newtype&
-        , ierror)
-  integer, intent(in) :: count
-  integer, dimension(*), intent(in) :: array_of_blocklengths
-  integer, dimension(*), intent(in) :: array_of_displacements
-  integer, intent(in) :: oldtype
-  integer, intent(out) :: newtype
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_indexed
-
-end interface
-interface MPI_Type_match_size
-
-subroutine MPI_Type_match_size(typeclass, size, datatype, ierror)
-  integer, intent(in) :: typeclass
-  integer, intent(in) :: size
-  integer, intent(out) :: datatype
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_match_size
-
-end interface
-
-
-interface MPI_Type_set_attr
-
-subroutine MPI_Type_set_attr(datatype, type_keyval, attr_val, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: datatype
-  integer, intent(in) :: type_keyval
-  integer(kind=MPI_ADDRESS_KIND), intent(in) :: attr_val
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_set_attr
-
-end interface
-
-
-interface MPI_Type_set_name
-
-subroutine MPI_Type_set_name(datatype, type_name, ierror)
-  integer, intent(in) :: datatype
-  character(len=*), intent(in) :: type_name
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_set_name
-
-end interface
-
-
-interface MPI_Type_size
-
-subroutine MPI_Type_size(datatype, size, ierror)
-  integer, intent(in) :: datatype
-  integer, intent(out) :: size
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_size
-
-end interface
-
-
-interface MPI_Type_size_x
-
-subroutine MPI_Type_size_x(datatype, size, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: datatype
-  integer(kind=MPI_COUNT_KIND), intent(out) :: size
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_size_x
-
-end interface
-
-
-interface MPI_Type_vector
-
-subroutine MPI_Type_vector(count, blocklength, stride, oldtype, newtype&
-        , ierror)
-  integer, intent(in) :: count
-  integer, intent(in) :: blocklength
-  integer, intent(in) :: stride
-  integer, intent(in) :: oldtype
-  integer, intent(out) :: newtype
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_vector
-
-end interface
-
-
-interface MPI_Wait
-
-subroutine MPI_Wait(request, status, ierror)
-  include 'mpif-config.h'
-  integer, intent(inout) :: request
-  integer, dimension(MPI_STATUS_SIZE), intent(out) :: status
-  integer, intent(out) :: ierror
-end subroutine MPI_Wait
-
-end interface
-
-
-interface MPI_Waitall
-
-subroutine MPI_Waitall(count, array_of_requests, array_of_statuses, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: count
-  integer, dimension(count), intent(inout) :: array_of_requests
-  integer, dimension(MPI_STATUS_SIZE, *), intent(out) :: array_of_statuses
-  integer, intent(out) :: ierror
-end subroutine MPI_Waitall
-
-end interface
-
-
-interface MPI_Waitany
-
-subroutine MPI_Waitany(count, array_of_requests, index, status, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: count
-  integer, dimension(count), intent(inout) :: array_of_requests
-  integer, intent(out) :: index
-  integer, dimension(MPI_STATUS_SIZE), intent(out) :: status
-  integer, intent(out) :: ierror
-end subroutine MPI_Waitany
-
-end interface
-
-
-interface MPI_Waitsome
-
-subroutine MPI_Waitsome(incount, array_of_requests, outcount, array_of_indices, array_of_statuses&
-        , ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: incount
-  integer, dimension(incount), intent(inout) :: array_of_requests
-  integer, intent(out) :: outcount
-  integer, dimension(*), intent(out) :: array_of_indices
-  integer, dimension(MPI_STATUS_SIZE, *), intent(out) :: array_of_statuses
-  integer, intent(out) :: ierror
-end subroutine MPI_Waitsome
-
-end interface
-
-
-interface MPI_Win_call_errhandler
-
-subroutine MPI_Win_call_errhandler(win, errorcode, ierror)
-  integer, intent(in) :: win
-  integer, intent(in) :: errorcode
-  integer, intent(out) :: ierror
-end subroutine MPI_Win_call_errhandler
-
-end interface
-
-
-interface MPI_Win_complete
-
-subroutine MPI_Win_complete(win, ierror)
-  integer, intent(in) :: win
-  integer, intent(out) :: ierror
-end subroutine MPI_Win_complete
-
-end interface
-
-
-interface MPI_Win_create_errhandler
-
-subroutine MPI_Win_create_errhandler(function, errhandler, ierror)
-  external :: function
-  integer, intent(out) :: errhandler
-  integer, intent(out) :: ierror
-end subroutine MPI_Win_create_errhandler
-
-end interface
-
-
-interface MPI_Win_create_keyval
-
-subroutine MPI_Win_create_keyval(win_copy_attr_fn, win_delete_attr_fn, win_keyval, extra_state, ierror)
-  include 'mpif-config.h'
-  external :: win_copy_attr_fn
-  external :: win_delete_attr_fn
-  integer, intent(out) :: win_keyval
-  integer(kind=MPI_ADDRESS_KIND), intent(in) :: extra_state
-  integer, intent(out) :: ierror
-end subroutine MPI_Win_create_keyval
-
-end interface
-
-
-interface MPI_Win_delete_attr
-
-subroutine MPI_Win_delete_attr(win, win_keyval, ierror)
-  integer, intent(in) :: win
-  integer, intent(in) :: win_keyval
-  integer, intent(out) :: ierror
-end subroutine MPI_Win_delete_attr
-
-end interface
-
-
-interface MPI_Win_fence
-
-subroutine MPI_Win_fence(assert, win, ierror)
-  integer, intent(in) :: assert
-  integer, intent(in) :: win
-  integer, intent(out) :: ierror
-end subroutine MPI_Win_fence
-
-end interface
-
-
-interface MPI_Win_free
-
-subroutine MPI_Win_free(win, ierror)
-  integer, intent(inout) :: win
-  integer, intent(out) :: ierror
-end subroutine MPI_Win_free
-
-end interface
-
-
-interface MPI_Win_free_keyval
-
-subroutine MPI_Win_free_keyval(win_keyval, ierror)
-  integer, intent(inout) :: win_keyval
-  integer, intent(out) :: ierror
-end subroutine MPI_Win_free_keyval
-
-end interface
-
-
-interface MPI_Win_get_attr
-
-subroutine MPI_Win_get_attr(win, win_keyval, attribute_val, flag, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: win
-  integer, intent(in) :: win_keyval
-  integer(kind=MPI_ADDRESS_KIND), intent(out) :: attribute_val
-  logical, intent(out) :: flag
-  integer, intent(out) :: ierror
-end subroutine MPI_Win_get_attr
-
-end interface
-
-
-interface MPI_Win_get_errhandler
-
-subroutine MPI_Win_get_errhandler(win, errhandler, ierror)
-  integer, intent(in) :: win
-  integer, intent(out) :: errhandler
-  integer, intent(out) :: ierror
-end subroutine MPI_Win_get_errhandler
-
-end interface
-
-
-interface MPI_Win_get_group
-
-subroutine MPI_Win_get_group(win, group, ierror)
-  integer, intent(in) :: win
-  integer, intent(out) :: group
-  integer, intent(out) :: ierror
-end subroutine MPI_Win_get_group
-
-end interface
-
-
-interface MPI_Win_get_name
-
-subroutine MPI_Win_get_name(win, win_name, resultlen, ierror)
-  integer, intent(in) :: win
-  character(len=*), intent(out) :: win_name
-  integer, intent(out) :: resultlen
-  integer, intent(out) :: ierror
-end subroutine MPI_Win_get_name
-
-end interface
-
-
-interface MPI_Win_lock
-
-subroutine MPI_Win_lock(lock_type, rank, assert, win, ierror)
-  integer, intent(in) :: lock_type
-  integer, intent(in) :: rank
-  integer, intent(in) :: assert
-  integer, intent(in) :: win
-  integer, intent(out) :: ierror
-end subroutine MPI_Win_lock
-
-end interface
-
-
-interface MPI_Win_post
-
-subroutine MPI_Win_post(group, assert, win, ierror)
-  integer, intent(in) :: group
-  integer, intent(in) :: assert
-  integer, intent(in) :: win
-  integer, intent(out) :: ierror
-end subroutine MPI_Win_post
-
-end interface
-
-
-interface MPI_Win_set_attr
-
-subroutine MPI_Win_set_attr(win, win_keyval, attribute_val, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: win
-  integer, intent(in) :: win_keyval
-  integer(kind=MPI_ADDRESS_KIND), intent(in) :: attribute_val
-  integer, intent(out) :: ierror
-end subroutine MPI_Win_set_attr
-
-end interface
-
-
-interface MPI_Win_set_errhandler
-
-subroutine MPI_Win_set_errhandler(win, errhandler, ierror)
-  integer, intent(in) :: win
-  integer, intent(in) :: errhandler
-  integer, intent(out) :: ierror
-end subroutine MPI_Win_set_errhandler
-
-end interface
-
-
-interface MPI_Win_set_name
-
-subroutine MPI_Win_set_name(win, win_name, ierror)
-  integer, intent(in) :: win
-  character(len=*), intent(in) :: win_name
-  integer, intent(out) :: ierror
-end subroutine MPI_Win_set_name
-
-end interface
-
-
-interface MPI_Win_start
-
-subroutine MPI_Win_start(group, assert, win, ierror)
-  integer, intent(in) :: group
-  integer, intent(in) :: assert
-  integer, intent(in) :: win
-  integer, intent(out) :: ierror
-end subroutine MPI_Win_start
-
-end interface
-
-
-interface MPI_Win_test
-
-subroutine MPI_Win_test(win, flag, ierror)
-  integer, intent(in) :: win
-  logical, intent(out) :: flag
-  integer, intent(out) :: ierror
-end subroutine MPI_Win_test
-
-end interface
-
-
-interface MPI_Win_unlock
-
-subroutine MPI_Win_unlock(rank, win, ierror)
-  integer, intent(in) :: rank
-  integer, intent(in) :: win
-  integer, intent(out) :: ierror
-end subroutine MPI_Win_unlock
-
-end interface
-
-
-interface MPI_Win_wait
-
-subroutine MPI_Win_wait(win, ierror)
-  integer, intent(in) :: win
-  integer, intent(out) :: ierror
-end subroutine MPI_Win_wait
-
-end interface
-
-
-interface MPI_Close_port
-
-subroutine MPI_Close_port(port_name, ierror)
-  character(len=*), intent(in) :: port_name
-  integer, intent(out) :: ierror
-end subroutine MPI_Close_port
-
-end interface
-
-
-interface MPI_Lookup_name
-
-subroutine MPI_Lookup_name(service_name, info, port_name, ierror)
-  character(len=*), intent(in) :: service_name
-  integer, intent(in) :: info
-  character(len=*), intent(out) :: port_name
-  integer, intent(out) :: ierror
-end subroutine MPI_Lookup_name
-
-end interface
-
-
-interface MPI_Open_port
-
-subroutine MPI_Open_port(info, port_name, ierror)
-  integer, intent(in) :: info
-  character(len=*), intent(out) :: port_name
-  integer, intent(out) :: ierror
-end subroutine MPI_Open_port
-
-end interface
-
-
-interface MPI_Publish_name
-
-subroutine MPI_Publish_name(service_name, info, port_name, ierror)
-  character(len=*), intent(in) :: service_name
-  integer, intent(in) :: info
-  character(len=*), intent(in) :: port_name
-  integer, intent(out) :: ierror
-end subroutine MPI_Publish_name
-
-end interface
-
-
-interface MPI_Unpublish_name
-
-subroutine MPI_Unpublish_name(service_name, info, port_name, ierror)
-  character(len=*), intent(in) :: service_name
-  integer, intent(in) :: info
-  character(len=*), intent(in) :: port_name
-  integer, intent(out) :: ierror
-end subroutine MPI_Unpublish_name
-
-end interface
-
-
-interface MPI_Comm_disconnect
-
-subroutine MPI_Comm_disconnect(comm, ierror)
-  integer, intent(inout) :: comm
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_disconnect
-
-end interface
-
-
-interface MPI_Comm_get_parent
-
-subroutine MPI_Comm_get_parent(parent, ierror)
-  integer, intent(out) :: parent
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_get_parent
-
-end interface
-
-
-interface MPI_Comm_join
-
-subroutine MPI_Comm_join(fd, intercomm, ierror)
-  integer, intent(in) :: fd
-  integer, intent(out) :: intercomm
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_join
-
-end interface
-
-
-interface MPI_Comm_accept
-
-subroutine MPI_Comm_accept(port_name, info, root, comm, newcomm&
-        , ierror)
-  character(len=*), intent(in) :: port_name
-  integer, intent(in) :: info
-  integer, intent(in) :: root
-  integer, intent(in) :: comm
-  integer, intent(out) :: newcomm
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_accept
-
-end interface
-
-
-interface MPI_Comm_connect
-
-subroutine MPI_Comm_connect(port_name, info, root, comm, newcomm&
-        , ierror)
-  character(len=*), intent(in) :: port_name
-  integer, intent(in) :: info
-  integer, intent(in) :: root
-  integer, intent(in) :: comm
-  integer, intent(out) :: newcomm
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_connect
-
-end interface
-
-
-interface MPI_Comm_spawn
-
-subroutine MPI_Comm_spawn(command, argv, maxprocs, info, root, &
-        comm, intercomm, array_of_errcodes, ierror)
-  character(len=*), intent(in) :: command
-  character(len=*), dimension(*), intent(in) :: argv
-  integer, intent(in) :: maxprocs
-  integer, intent(in) :: info
-  integer, intent(in) :: root
-  integer, intent(in) :: comm
-  integer, intent(out) :: intercomm
-  integer, dimension(*), intent(out) :: array_of_errcodes
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_spawn
-
-end interface
-
-
-interface MPI_Comm_spawn_multiple
-
-subroutine MPI_Comm_spawn_multiple(count, array_of_commands, array_of_argv, array_of_maxprocs, array_of_info, &
-        root, comm, intercomm, array_of_errcodes, ierror)
-  integer, intent(in) :: count
-  character(len=*), dimension(*), intent(in) :: array_of_commands
-  character(len=*), dimension(count,*), intent(in) :: array_of_argv
-  integer, dimension(*), intent(in) :: array_of_maxprocs
-  integer, dimension(*), intent(in) :: array_of_info
-  integer, intent(in) :: root
-  integer, intent(in) :: comm
-  integer, intent(out) :: intercomm
-  integer, dimension(*), intent(out) :: array_of_errcodes
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_spawn_multiple
-
-end interface
-
-
-interface MPI_Mprobe
-
-subroutine MPI_Mprobe(source, tag, comm, message, status, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: source
-  integer, intent(in) :: tag
-  integer, intent(in) :: comm
-  integer, intent(out) :: message
-  integer, dimension(MPI_STATUS_SIZE), intent(out) :: status
-  integer, intent(out) :: ierror
-end subroutine MPI_Mprobe
-
-end interface
-
-
-interface MPI_Improbe
-
-subroutine MPI_Improbe(source, tag, comm, flag, message, status, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: source
-  integer, intent(in) :: tag
-  integer, intent(in) :: comm
-  logical, intent(out) :: flag
-  integer, intent(out) :: message
-  integer, dimension(MPI_STATUS_SIZE), intent(out) :: status
-  integer, intent(out) :: ierror
-end subroutine MPI_Improbe
-
-end interface
-
-
-interface MPI_Get_library_version
-
-subroutine MPI_Get_library_version(version, resultlen, ierror)
-  character(len=*), intent(out) :: version
-  integer, intent(out) :: resultlen
-  integer, intent(out) :: ierror
-end subroutine MPI_Get_library_version
-
-end interface
-
-
-interface MPI_Comm_split_type
-
-subroutine MPI_Comm_split_type(comm, split_type, key, info, newcomm, ierror)
-  integer, intent(in) :: comm
-  integer, intent(in) :: split_type
-  integer, intent(in) :: key
-  integer, intent(in) :: info
-  integer, intent(out) :: newcomm
-  integer, intent(out) :: ierror
-end subroutine MPI_Comm_split_type
-
-end interface
-
-
-interface MPI_Type_create_hindexed_block
-
-subroutine MPI_Type_create_hindexed_block(count, blocklength, array_of_displacements, oldtype, newtype&
-        , ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: count
-  integer, intent(in) :: blocklength
-  integer(kind=MPI_ADDRESS_KIND), dimension(*), intent(in) :: array_of_displacements
-  integer, intent(in) :: oldtype
-  integer, intent(out) :: newtype
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_create_hindexed_block
-
-end interface
-
-
-interface MPI_Dist_graph_create
-
-subroutine MPI_Dist_graph_create(comm_old, n, sources, degrees, destinations, &
-        weights, info, reorder, comm_dist_graph, ierror)
-  integer, intent(in) :: comm_old
-  integer, intent(in) :: n
-  integer, dimension(n), intent(in) :: sources
-  integer, dimension(n), intent(in) :: degrees
-  integer, dimension(n), intent(in) :: destinations
-  integer, dimension(n), intent(in) :: weights
-  integer, intent(in) :: info
-  logical, intent(in) :: reorder
-  integer, intent(out) :: comm_dist_graph
-  integer, intent(out) :: ierror
-end subroutine MPI_Dist_graph_create
-
-end interface
-
-
-interface MPI_Dist_graph_create_adjacent
-
-subroutine MPI_Dist_graph_create_adjacent(comm_old, indegree, sources, sourceweights, &
-       outdegree, destinations, destweights, info, reorder, &
-       comm_dist_graph, ierror)
-  integer, intent(in) :: comm_old
-  integer, intent(in) :: indegree
-  integer, dimension(indegree), intent(in) :: sources
-  integer, dimension(indegree), intent(in) :: sourceweights
-  integer, intent(in) :: outdegree
-  integer, dimension(outdegree), intent(in) :: destinations
-  integer, dimension(outdegree), intent(in) :: destweights
-  integer, intent(in) :: info
-  logical, intent(in) :: reorder
-  integer, intent(out) :: comm_dist_graph
-  integer, intent(out) :: ierror
-end subroutine MPI_Dist_graph_create_adjacent
-
-end interface
-
-
-interface MPI_Dist_graph_neighbors_count
-
-subroutine MPI_Dist_graph_neighbors_count(comm, indegree, outdegree, weighted, ierror)
-  integer, intent(in) :: comm
-  integer, intent(out) :: indegree
-  integer, intent(out) :: outdegree
-  logical, intent(out) :: weighted
-  integer, intent(out) :: ierror
-end subroutine MPI_Dist_graph_neighbors_count
-
-end interface
-
-
-interface MPI_Dist_graph_neighbors
-
-subroutine MPI_Dist_graph_neighbors(comm, maxindegree, sources, sourceweights, &
-       maxoutdegree, destinations, destweights, ierror)
-  integer, intent(in) :: comm
-  integer, intent(in) :: maxindegree
-  integer, dimension(maxindegree), intent(out) :: sources
-  integer, dimension(maxindegree), intent(out) :: sourceweights
-  integer, intent(in) :: maxoutdegree
-  integer, dimension(maxoutdegree), intent(out) :: destinations
-  integer, dimension(maxoutdegree), intent(out) :: destweights
-  integer, intent(out) :: ierror
-end subroutine MPI_Dist_graph_neighbors
-
-end interface
-
-
-interface MPI_Win_flush
-
-subroutine MPI_Win_flush(rank, win, ierror)
-  integer, intent(in) :: rank
-  integer, intent(in) :: win
-  integer, intent(out) :: ierror
-end subroutine MPI_Win_flush
-
-end interface
-
-
-interface MPI_Win_flush_all
-
-subroutine MPI_Win_flush_all(win, ierror)
-  integer, intent(in) :: win
-  integer, intent(out) :: ierror
-end subroutine MPI_Win_flush_all
-
-end interface
-
-
-interface MPI_Win_flush_local
-
-subroutine MPI_Win_flush_local(rank, win, ierror)
-  integer, intent(in) :: rank
-  integer, intent(in) :: win
-  integer, intent(out) :: ierror
-end subroutine MPI_Win_flush_local
-
-end interface
-
-
-interface MPI_Win_flush_local_all
-
-subroutine MPI_Win_flush_local_all(win, ierror)
-  integer, intent(in) :: win
-  integer, intent(out) :: ierror
-end subroutine MPI_Win_flush_local_all
-
-end interface
diff --git a/ompi/mpi/fortran/use-mpi-tkr/mpi-f90-removed-interfaces.h b/ompi/mpi/fortran/use-mpi-tkr/mpi-f90-removed-interfaces.h
deleted file mode 100644
index 48ede9d15e8..00000000000
--- a/ompi/mpi/fortran/use-mpi-tkr/mpi-f90-removed-interfaces.h
+++ /dev/null
@@ -1,185 +0,0 @@
-! -*- fortran -*-
-!
-! Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
-!                         University Research and Technology
-!                         Corporation.  All rights reserved.
-! Copyright (c) 2004-2005 The University of Tennessee and The University
-!                         of Tennessee Research Foundation.  All rights
-!                         reserved.
-! Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-!                         University of Stuttgart.  All rights reserved.
-! Copyright (c) 2004-2005 The Regents of the University of California.
-!                         All rights reserved.
-! Copyright (c) 2006-2018 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2016      Research Organization for Information Science
-!                         and Technology (RIST). All rights reserved.
-! Copyright (c) 2018      Los Alamos National Security, LLC. All rights
-!                         reserved.
-! $COPYRIGHT$
-!
-! Additional copyrights may follow
-!
-! $HEADER$
-!
-
-interface MPI_Attr_delete
-
-subroutine MPI_Attr_delete(comm, keyval, ierror)
-  integer, intent(in) :: comm
-  integer, intent(in) :: keyval
-  integer, intent(out) :: ierror
-end subroutine MPI_Attr_delete
-
-end interface
-
-
-interface MPI_Attr_get
-
-subroutine MPI_Attr_get(comm, keyval, attribute_val, flag, ierror)
-  integer, intent(in) :: comm
-  integer, intent(in) :: keyval
-  integer, intent(out) :: attribute_val
-  logical, intent(out) :: flag
-  integer, intent(out) :: ierror
-end subroutine MPI_Attr_get
-
-end interface
-
-
-interface MPI_Attr_put
-
-subroutine MPI_Attr_put(comm, keyval, attribute_val, ierror)
-  integer, intent(in) :: comm
-  integer, intent(in) :: keyval
-  integer, intent(in) :: attribute_val
-  integer, intent(out) :: ierror
-end subroutine MPI_Attr_put
-
-end interface
-
-interface MPI_Errhandler_create
-
-subroutine MPI_Errhandler_create(function, errhandler, ierror)
-  external :: function
-  integer, intent(out) :: errhandler
-  integer, intent(out) :: ierror
-end subroutine MPI_Errhandler_create
-
-end interface
-
-interface MPI_Errhandler_get
-
-subroutine MPI_Errhandler_get(comm, errhandler, ierror)
-  integer, intent(in) :: comm
-  integer, intent(out) :: errhandler
-  integer, intent(out) :: ierror
-end subroutine MPI_Errhandler_get
-
-end interface
-
-
-interface MPI_Errhandler_set
-
-subroutine MPI_Errhandler_set(comm, errhandler, ierror)
-  integer, intent(in) :: comm
-  integer, intent(in) :: errhandler
-  integer, intent(out) :: ierror
-end subroutine MPI_Errhandler_set
-
-end interface
-
-interface MPI_Keyval_create
-
-subroutine MPI_Keyval_create(copy_fn, delete_fn, keyval, extra_state, ierror)
-  external :: copy_fn
-  external :: delete_fn
-  integer, intent(out) :: keyval
-  integer, intent(in) :: extra_state
-  integer, intent(out) :: ierror
-end subroutine MPI_Keyval_create
-
-end interface
-
-
-interface MPI_Keyval_free
-
-subroutine MPI_Keyval_free(keyval, ierror)
-  integer, intent(inout) :: keyval
-  integer, intent(out) :: ierror
-end subroutine MPI_Keyval_free
-
-end interface
-
-interface MPI_Type_extent
-
-subroutine MPI_Type_extent(datatype, extent, ierror)
-  integer, intent(in) :: datatype
-  integer, intent(out) :: extent
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_extent
-
-end interface
-
-interface MPI_Type_hindexed
-
-subroutine MPI_Type_hindexed(count, array_of_blocklengths, array_of_displacements, oldtype, newtype&
-        , ierror)
-  integer, intent(in) :: count
-  integer, dimension(*), intent(in) :: array_of_blocklengths
-  integer, dimension(*), intent(in) :: array_of_displacements
-  integer, intent(in) :: oldtype
-  integer, intent(out) :: newtype
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_hindexed
-
-end interface
-
-
-interface MPI_Type_hvector
-
-subroutine MPI_Type_hvector(count, blocklength, stride, oldtype, newtype&
-        , ierror)
-  integer, intent(in) :: count
-  integer, intent(in) :: blocklength
-  integer, intent(in) :: stride
-  integer, intent(in) :: oldtype
-  integer, intent(out) :: newtype
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_hvector
-
-end interface
-
-interface MPI_Type_lb
-
-subroutine MPI_Type_lb(datatype, lb, ierror)
-  integer, intent(in) :: datatype
-  integer, intent(out) :: lb
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_lb
-
-end interface
-
-interface MPI_Type_struct
-
-subroutine MPI_Type_struct(count, array_of_blocklengths, array_of_displacements, array_of_types, newtype&
-        , ierror)
-  integer, intent(in) :: count
-  integer, dimension(*), intent(in) :: array_of_blocklengths
-  integer, dimension(*), intent(in) :: array_of_displacements
-  integer, dimension(*), intent(in) :: array_of_types
-  integer, intent(out) :: newtype
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_struct
-
-end interface
-
-
-interface MPI_Type_ub
-
-subroutine MPI_Type_ub(datatype, ub, ierror)
-  integer, intent(in) :: datatype
-  integer, intent(out) :: ub
-  integer, intent(out) :: ierror
-end subroutine MPI_Type_ub
-
-end interface
diff --git a/ompi/mpi/fortran/use-mpi-tkr/mpi-f90-status.h b/ompi/mpi/fortran/use-mpi-tkr/mpi-f90-status.h
deleted file mode 100644
index 1f75bb46d31..00000000000
--- a/ompi/mpi/fortran/use-mpi-tkr/mpi-f90-status.h
+++ /dev/null
@@ -1,35 +0,0 @@
-! -*- fortran -*-
-!
-! Copyright (c) 2020      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-!
-! Additional copyrights may follow
-!
-! $HEADER$
-
-
-interface MPI_Status_f082f
-
-subroutine MPI_Status_f082f(f08_status, f_status, ierror)
-  use mpi_types
-  include 'mpif-config.h'
-  type(MPI_Status), intent(in) :: f08_status
-  integer, intent(out) :: f_status(MPI_STATUS_SIZE)
-  integer, intent(out) :: ierror
-end subroutine MPI_Status_f082f
-
-end interface
-
-
-interface MPI_Status_f2f08
-
-subroutine MPI_Status_f2f08(f_status, f08_status, ierror)
-  use mpi_types
-  include 'mpif-config.h'
-  integer, intent(in) :: f_status(MPI_STATUS_SIZE)
-  type(MPI_Status), intent(out) :: f08_status
-  integer, intent(out) :: ierror
-end subroutine MPI_Status_f2f08
-
-end interface
diff --git a/ompi/mpi/fortran/use-mpi-tkr/mpi.F90 b/ompi/mpi/fortran/use-mpi-tkr/mpi.F90
deleted file mode 100644
index b8b8942fdf3..00000000000
--- a/ompi/mpi/fortran/use-mpi-tkr/mpi.F90
+++ /dev/null
@@ -1,81 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-!                         University Research and Technology
-!                         Corporation.  All rights reserved.
-! Copyright (c) 2004-2005 The University of Tennessee and The University
-!                         of Tennessee Research Foundation.  All rights
-!                         reserved.
-! Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-!                         University of Stuttgart.  All rights reserved.
-! Copyright (c) 2004-2005 The Regents of the University of California.
-!                         All rights reserved.
-! Copyright (c) 2006-2014 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2016-2020 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-!
-! Additional copyrights may follow
-!
-! $HEADER$
-!
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-module mpi
-
-#if OMPI_FORTRAN_HAVE_TYPE_MPI_STATUS
-  use mpi_types
-#endif
-
-  include "mpif-config.h"
-  include "mpif-constants.h"
-  include "mpif-handles.h"
-  include "mpif-io-constants.h"
-  include "mpif-io-handles.h"
-  include "mpif-sentinels.h"
-
-! The MPI attribute callback functions
-
-  include "../base/attr-fn-int-callback-interfaces.h"
-
-! The MPI_CONVERSION_FN_NULL function
-
-  include "../base/conversion-fn-null-int-interface.h"
-
-! Functions that have overloaded interfaces with TYPE(C_PTR) (which
-! this compiler may or may not support).  We use an "if" preprocessor
-! macro in this file, so we need to use the preprocessor include
-! directive, not the Fortran include.
-#include "mpi-f90-cptr-interfaces.h"
-#include "pmpi-f90-cptr-interfaces.h"
-
-#if OMPI_FORTRAN_HAVE_TYPE_MPI_STATUS
-  include "mpi-f90-status.h"
-  include "pmpi-f90-status.h"
-#endif
-
-! This file is generated, and is *huge*.  Its size is directly related
-! to the --with-f90-max-array-dim configure parameter.
-
-# include "mpi-f90-interfaces.h"
-# include "pmpi-f90-interfaces.h"
-#if OMPI_PROVIDE_MPI_FILE_INTEFACE
-# include "mpi-f90-file-interfaces.h"
-# include "pmpi-f90-file-interfaces.h"
-#endif
-
-#if OMPI_FORTRAN_BUILD_SIZEOF
-  include "mpi-tkr-sizeof.h"
-#endif
-
-#if !defined(OMPI_ENABLE_MPI1_COMPAT)
-
-#error "Remove MPI-1 compat code"
-
-#elif OMPI_ENABLE_MPI1_COMPAT
-# include "mpi-f90-removed-interfaces.h"
-# include "pmpi-f90-removed-interfaces.h"
-#endif
-
-end module mpi
diff --git a/ompi/mpi/fortran/use-mpi-tkr/mpi_aint_add_f90.f90 b/ompi/mpi/fortran/use-mpi-tkr/mpi_aint_add_f90.f90
deleted file mode 100644
index e1e7140cd1e..00000000000
--- a/ompi/mpi/fortran/use-mpi-tkr/mpi_aint_add_f90.f90
+++ /dev/null
@@ -1,31 +0,0 @@
-! -*- fortran -*-
-!
-! Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
-!                         University Research and Technology
-!                         Corporation.  All rights reserved.
-! Copyright (c) 2004-2005 The University of Tennessee and The University
-!                         of Tennessee Research Foundation.  All rights
-!                         reserved.
-! Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-!                         University of Stuttgart.  All rights reserved.
-! Copyright (c) 2004-2005 The Regents of the University of California.
-!                         All rights reserved.
-! Copyright (c) 2006-2014 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2015      Los Alamos National Security, LLC. All rights
-!                         reserved.
-! $COPYRIGHT$
-!
-! Additional copyrights may follow
-!
-! $HEADER$
-!
-
-function MPI_Aint_add(base, diff)
-  include 'mpif-config.h'
-  integer(kind=MPI_ADDRESS_KIND), intent(in) :: base
-  integer(kind=MPI_ADDRESS_KIND), intent(in) :: diff
-  integer(kind=MPI_ADDRESS_KIND) :: MPI_Aint_add,foo
-  call MPI_Aint_add_f90(base,diff,foo)
-  MPI_Aint_add = foo
-end function MPI_Aint_add
-
diff --git a/ompi/mpi/fortran/use-mpi-tkr/mpi_aint_diff_f90.f90 b/ompi/mpi/fortran/use-mpi-tkr/mpi_aint_diff_f90.f90
deleted file mode 100644
index 2bc6e485c87..00000000000
--- a/ompi/mpi/fortran/use-mpi-tkr/mpi_aint_diff_f90.f90
+++ /dev/null
@@ -1,31 +0,0 @@
-! -*- fortran -*-
-!
-! Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
-!                         University Research and Technology
-!                         Corporation.  All rights reserved.
-! Copyright (c) 2004-2005 The University of Tennessee and The University
-!                         of Tennessee Research Foundation.  All rights
-!                         reserved.
-! Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-!                         University of Stuttgart.  All rights reserved.
-! Copyright (c) 2004-2005 The Regents of the University of California.
-!                         All rights reserved.
-! Copyright (c) 2006-2014 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2015      Los Alamos National Security, LLC. All rights
-!                         reserved.
-! $COPYRIGHT$
-!
-! Additional copyrights may follow
-!
-! $HEADER$
-!
-
-function MPI_Aint_diff(addr1, addr2)
-  include 'mpif-config.h'
-  integer(kind=MPI_ADDRESS_KIND), intent(in) :: addr1
-  integer(kind=MPI_ADDRESS_KIND), intent(in) :: addr2
-  integer(kind=MPI_ADDRESS_KIND) :: MPI_Aint_diff,foo
-  call MPI_Aint_diff_f90(addr1,addr2,foo)
-  MPI_Aint_diff = foo
-end function MPI_Aint_diff
-
diff --git a/ompi/mpi/fortran/use-mpi-tkr/mpi_comm_spawn_multiple_f90.f90 b/ompi/mpi/fortran/use-mpi-tkr/mpi_comm_spawn_multiple_f90.f90
deleted file mode 100644
index 64007d26898..00000000000
--- a/ompi/mpi/fortran/use-mpi-tkr/mpi_comm_spawn_multiple_f90.f90
+++ /dev/null
@@ -1,60 +0,0 @@
-! -*- fortran -*-
-!
-! Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
-!                         University Research and Technology
-!                         Corporation.  All rights reserved.
-! Copyright (c) 2004-2005 The University of Tennessee and The University
-!                         of Tennessee Research Foundation.  All rights
-!                         reserved.
-! Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-!                         University of Stuttgart.  All rights reserved.
-! Copyright (c) 2004-2005 The Regents of the University of California.
-!                         All rights reserved.
-! Copyright (c) 2006-2014 Cisco Systems, Inc.  All rights reserved.
-! $COPYRIGHT$
-!
-! Additional copyrights may follow
-!
-! $HEADER$
-!
-
-subroutine MPI_Comm_spawn_multipleA(count, array_of_commands, array_of_argv, &
-        array_of_maxprocs, array_of_info, &
-        root, comm, intercomm, array_of_errcodes, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: count
-  character(len=*), dimension(*), intent(in) :: array_of_commands
-  character(len=*), dimension(count,*), intent(in) :: array_of_argv
-  integer, dimension(*), intent(in) :: array_of_maxprocs
-  integer, dimension(*), intent(in) :: array_of_info
-  integer, intent(in) :: root
-  integer, intent(in) :: comm
-  integer, intent(out) :: intercomm
-  integer, dimension(*), intent(out) :: array_of_errcodes
-  integer, intent(out) :: ierror
-
-  call MPI_Comm_spawn_multiple(count, array_of_commands, array_of_argv, &
-      array_of_maxprocs, array_of_info, root, comm, intercomm, &
-      array_of_errcodes, ierror)
-end subroutine MPI_Comm_spawn_multipleA
-
-subroutine MPI_Comm_spawn_multipleN(count, array_of_commands, array_of_argv, &
-        array_of_maxprocs, array_of_info, &
-        root, comm, intercomm, array_of_errcodes, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: count
-  character(len=*), dimension(*), intent(in) :: array_of_commands
-  double precision, intent(in) :: array_of_argv
-  integer, dimension(*), intent(in) :: array_of_maxprocs
-  integer, dimension(*), intent(in) :: array_of_info
-  integer, intent(in) :: root
-  integer, intent(in) :: comm
-  integer, intent(out) :: intercomm
-  integer, dimension(*), intent(out) :: array_of_errcodes
-  integer, intent(out) :: ierror
-
-  call MPI_Comm_spawn_multiple(count, array_of_commands, array_of_argv, &
-      array_of_maxprocs, array_of_info, root, comm, intercomm, &
-      array_of_errcodes, ierror)
-end subroutine MPI_Comm_spawn_multipleN
-
diff --git a/ompi/mpi/fortran/use-mpi-tkr/mpi_testall_f90.f90 b/ompi/mpi/fortran/use-mpi-tkr/mpi_testall_f90.f90
deleted file mode 100644
index c54478f95c6..00000000000
--- a/ompi/mpi/fortran/use-mpi-tkr/mpi_testall_f90.f90
+++ /dev/null
@@ -1,43 +0,0 @@
-! -*- fortran -*-
-!
-! Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
-!                         University Research and Technology
-!                         Corporation.  All rights reserved.
-! Copyright (c) 2004-2005 The University of Tennessee and The University
-!                         of Tennessee Research Foundation.  All rights
-!                         reserved.
-! Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-!                         University of Stuttgart.  All rights reserved.
-! Copyright (c) 2004-2005 The Regents of the University of California.
-!                         All rights reserved.
-! Copyright (c) 2006-2014 Cisco Systems, Inc.  All rights reserved.
-! $COPYRIGHT$
-!
-! Additional copyrights may follow
-!
-! $HEADER$
-!
-
-subroutine MPI_TestallS(count, array_of_requests, flag, array_of_statuses, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: count
-  integer, dimension(count), intent(inout) :: array_of_requests
-  logical, intent(out) :: flag
-  integer, dimension(MPI_STATUS_SIZE, count), intent(out) :: array_of_statuses
-  integer, intent(out) :: ierror
-
-  call MPI_Testall(count, array_of_requests, flag, array_of_statuses, ierror)
-end subroutine MPI_TestallS
-
-
-subroutine MPI_TestallI(count, array_of_requests, flag, array_of_statuses, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: count
-  integer, dimension(count), intent(inout) :: array_of_requests
-  logical, intent(out) :: flag
-  double precision, intent(out) :: array_of_statuses
-  integer, intent(out) :: ierror
-
-  call MPI_Testall(count, array_of_requests, flag, array_of_statuses, ierror)
-end subroutine MPI_TestallI
-
diff --git a/ompi/mpi/fortran/use-mpi-tkr/mpi_testsome_f90.f90 b/ompi/mpi/fortran/use-mpi-tkr/mpi_testsome_f90.f90
deleted file mode 100644
index bd54a27fed4..00000000000
--- a/ompi/mpi/fortran/use-mpi-tkr/mpi_testsome_f90.f90
+++ /dev/null
@@ -1,47 +0,0 @@
-! -*- fortran -*-
-!
-! Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
-!                         University Research and Technology
-!                         Corporation.  All rights reserved.
-! Copyright (c) 2004-2005 The University of Tennessee and The University
-!                         of Tennessee Research Foundation.  All rights
-!                         reserved.
-! Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-!                         University of Stuttgart.  All rights reserved.
-! Copyright (c) 2004-2005 The Regents of the University of California.
-!                         All rights reserved.
-! Copyright (c) 2006-2014 Cisco Systems, Inc.  All rights reserved.
-! $COPYRIGHT$
-!
-! Additional copyrights may follow
-!
-! $HEADER$
-!
-
-subroutine MPI_TestsomeS(incount, array_of_requests, outcount, array_of_indices, array_of_statuses&
-        , ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: incount
-  integer, dimension(incount), intent(inout) :: array_of_requests
-  integer, intent(out) :: outcount
-  integer, dimension(*), intent(out) :: array_of_indices
-  integer, dimension(MPI_STATUS_SIZE, incount), intent(out) :: array_of_statuses
-  integer, intent(out) :: ierror
-
-  call MPI_Testsome(incount, array_of_requests, outcount, array_of_indices, array_of_statuses, ierror)
-end subroutine MPI_TestsomeS
-
-
-subroutine MPI_TestsomeI(incount, array_of_requests, outcount, array_of_indices, array_of_statuses&
-        , ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: incount
-  integer, dimension(incount), intent(inout) :: array_of_requests
-  integer, intent(out) :: outcount
-  integer, dimension(*), intent(out) :: array_of_indices
-  double precision, intent(out) :: array_of_statuses
-  integer, intent(out) :: ierror
-
-  call MPI_Testsome(incount, array_of_requests, outcount, array_of_indices, array_of_statuses, ierror)
-end subroutine MPI_TestsomeI
-
diff --git a/ompi/mpi/fortran/use-mpi-tkr/mpi_waitall_f90.f90 b/ompi/mpi/fortran/use-mpi-tkr/mpi_waitall_f90.f90
deleted file mode 100644
index 7a093092ae8..00000000000
--- a/ompi/mpi/fortran/use-mpi-tkr/mpi_waitall_f90.f90
+++ /dev/null
@@ -1,39 +0,0 @@
-! -*- fortran -*-
-!
-! Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
-!                         University Research and Technology
-!                         Corporation.  All rights reserved.
-! Copyright (c) 2004-2005 The University of Tennessee and The University
-!                         of Tennessee Research Foundation.  All rights
-!                         reserved.
-! Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-!                         University of Stuttgart.  All rights reserved.
-! Copyright (c) 2004-2005 The Regents of the University of California.
-!                         All rights reserved.
-! Copyright (c) 2006-2014 Cisco Systems, Inc.  All rights reserved.
-! $COPYRIGHT$
-!
-! Additional copyrights may follow
-!
-! $HEADER$
-!
-
-subroutine MPI_WaitallS(count, array_of_requests, array_of_statuses, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: count
-  integer, dimension(count), intent(inout) :: array_of_requests
-  integer, dimension(MPI_STATUS_SIZE, count), intent(out) :: array_of_statuses
-  integer, intent(out) :: ierror
-  call MPI_Waitall(count, array_of_requests, array_of_statuses, ierror)
-end subroutine MPI_WaitallS
-
-
-subroutine MPI_WaitallI(count, array_of_requests, array_of_statuses, ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: count
-  integer, dimension(count), intent(inout) :: array_of_requests
-  double precision, intent(out) :: array_of_statuses
-  integer, intent(out) :: ierror
-  call MPI_Waitall(count, array_of_requests, array_of_statuses, ierror)
-end subroutine MPI_WaitallI
-
diff --git a/ompi/mpi/fortran/use-mpi-tkr/mpi_waitsome_f90.f90 b/ompi/mpi/fortran/use-mpi-tkr/mpi_waitsome_f90.f90
deleted file mode 100644
index cecd5ce4e60..00000000000
--- a/ompi/mpi/fortran/use-mpi-tkr/mpi_waitsome_f90.f90
+++ /dev/null
@@ -1,47 +0,0 @@
-! -*- fortran -*-
-!
-! Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
-!                         University Research and Technology
-!                         Corporation.  All rights reserved.
-! Copyright (c) 2004-2005 The University of Tennessee and The University
-!                         of Tennessee Research Foundation.  All rights
-!                         reserved.
-! Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-!                         University of Stuttgart.  All rights reserved.
-! Copyright (c) 2004-2005 The Regents of the University of California.
-!                         All rights reserved.
-! Copyright (c) 2006-2014 Cisco Systems, Inc.  All rights reserved.
-! $COPYRIGHT$
-!
-! Additional copyrights may follow
-!
-! $HEADER$
-!
-
-subroutine MPI_WaitsomeS(incount, array_of_requests, outcount, array_of_indices, array_of_statuses&
-        , ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: incount
-  integer, dimension(incount), intent(inout) :: array_of_requests
-  integer, intent(out) :: outcount
-  integer, dimension(*), intent(out) :: array_of_indices
-  integer, dimension(MPI_STATUS_SIZE, incount), intent(out) :: array_of_statuses
-  integer, intent(out) :: ierror
-
-  call MPI_Waitsome(incount, array_of_requests, outcount, array_of_indices, array_of_statuses, ierror)
-end subroutine MPI_WaitsomeS
-
-
-subroutine MPI_WaitsomeI(incount, array_of_requests, outcount, array_of_indices, array_of_statuses&
-        , ierror)
-  include 'mpif-config.h'
-  integer, intent(in) :: incount
-  integer, dimension(incount), intent(inout) :: array_of_requests
-  integer, intent(out) :: outcount
-  integer, dimension(*), intent(out) :: array_of_indices
-  double precision, intent(out) :: array_of_statuses
-  integer, intent(out) :: ierror
-
-  call MPI_Waitsome(incount, array_of_requests, outcount, array_of_indices, array_of_statuses, ierror)
-end subroutine MPI_WaitsomeI
-
diff --git a/ompi/mpi/fortran/use-mpi-tkr/mpi_wtick_f90.f90 b/ompi/mpi/fortran/use-mpi-tkr/mpi_wtick_f90.f90
deleted file mode 100644
index dcd4e0d5ff2..00000000000
--- a/ompi/mpi/fortran/use-mpi-tkr/mpi_wtick_f90.f90
+++ /dev/null
@@ -1,27 +0,0 @@
-! -*- fortran -*-
-!
-! Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
-!                         University Research and Technology
-!                         Corporation.  All rights reserved.
-! Copyright (c) 2004-2005 The University of Tennessee and The University
-!                         of Tennessee Research Foundation.  All rights
-!                         reserved.
-! Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-!                         University of Stuttgart.  All rights reserved.
-! Copyright (c) 2004-2005 The Regents of the University of California.
-!                         All rights reserved.
-! Copyright (c) 2006-2014 Cisco Systems, Inc.  All rights reserved.
-! $COPYRIGHT$
-!
-! Additional copyrights may follow
-!
-! $HEADER$
-!
-
-function MPI_Wtick()
-  implicit none
-  double precision :: MPI_Wtick, foo
-  call MPI_Wtick_f90(foo)
-  MPI_Wtick = foo
-end function MPI_Wtick
-
diff --git a/ompi/mpi/fortran/use-mpi-tkr/mpi_wtime_f90.f90 b/ompi/mpi/fortran/use-mpi-tkr/mpi_wtime_f90.f90
deleted file mode 100644
index 18880e21e30..00000000000
--- a/ompi/mpi/fortran/use-mpi-tkr/mpi_wtime_f90.f90
+++ /dev/null
@@ -1,27 +0,0 @@
-! -*- fortran -*-
-!
-! Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
-!                         University Research and Technology
-!                         Corporation.  All rights reserved.
-! Copyright (c) 2004-2005 The University of Tennessee and The University
-!                         of Tennessee Research Foundation.  All rights
-!                         reserved.
-! Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-!                         University of Stuttgart.  All rights reserved.
-! Copyright (c) 2004-2005 The Regents of the University of California.
-!                         All rights reserved.
-! Copyright (c) 2006-2014 Cisco Systems, Inc.  All rights reserved.
-! $COPYRIGHT$
-!
-! Additional copyrights may follow
-!
-! $HEADER$
-!
-
-function MPI_Wtime()
-  implicit none
-  double precision :: MPI_Wtime, foo
-  call MPI_Wtime_f90(foo)
-  MPI_Wtime = foo
-end function MPI_Wtime
-
diff --git a/ompi/mpi/fortran/use-mpi-tkr/pmpi-f90-cptr-interfaces.h b/ompi/mpi/fortran/use-mpi-tkr/pmpi-f90-cptr-interfaces.h
deleted file mode 100644
index 5cf5a778578..00000000000
--- a/ompi/mpi/fortran/use-mpi-tkr/pmpi-f90-cptr-interfaces.h
+++ /dev/null
@@ -1,18 +0,0 @@
-! -*- fortran -*-
-!
-! Copyright (c) 2020      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-!
-! Additional copyrights may follow
-!
-! $HEADER$
-
-#define MPI_Win_allocate PMPI_Win_allocate
-#define MPI_Win_allocate_cptr PMPI_Win_allocate_cptr
-#define MPI_Win_allocate_shared PMPI_Win_allocate_shared
-#define MPI_Win_allocate_shared_cptr PMPI_Win_allocate_shared_cptr
-#define MPI_Win_shared_query PMPI_Win_shared_query
-#define MPI_Win_shared_query_cptr PMPI_Win_shared_query_cptr
-
-#include "ompi/mpi/fortran/use-mpi-tkr/mpi-f90-cptr-interfaces.h"
diff --git a/ompi/mpi/fortran/use-mpi-tkr/pmpi-f90-file-interfaces.h b/ompi/mpi/fortran/use-mpi-tkr/pmpi-f90-file-interfaces.h
deleted file mode 100644
index 783e16642dd..00000000000
--- a/ompi/mpi/fortran/use-mpi-tkr/pmpi-f90-file-interfaces.h
+++ /dev/null
@@ -1,37 +0,0 @@
-! -*- fortran -*-
-!
-! Copyright (c) 2020      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-!
-! Additional copyrights may follow
-!
-! $HEADER$
-
-#define MPI_File_call_errhandler PMPI_File_call_errhandler
-#define MPI_File_close PMPI_File_close
-#define MPI_File_create_errhandler PMPI_File_create_errhandler
-#define MPI_File_delete PMPI_File_delete
-#define MPI_File_get_amode PMPI_File_get_amode
-#define MPI_File_get_atomicity PMPI_File_get_atomicity
-#define MPI_File_get_byte_offset PMPI_File_get_byte_offset
-#define MPI_File_get_errhandler PMPI_File_get_errhandler
-#define MPI_File_get_group PMPI_File_get_group
-#define MPI_File_get_info PMPI_File_get_info
-#define MPI_File_get_position PMPI_File_get_position
-#define MPI_File_get_position_shared PMPI_File_get_position_shared
-#define MPI_File_get_size PMPI_File_get_size
-#define MPI_File_get_type_extent PMPI_File_get_type_extent
-#define MPI_File_get_view PMPI_File_get_view
-#define MPI_File_open PMPI_File_open
-#define MPI_File_preallocate PMPI_File_preallocate
-#define MPI_File_seek PMPI_File_seek
-#define MPI_File_seek_shared PMPI_File_seek_shared
-#define MPI_File_set_atomicity PMPI_File_set_atomicity
-#define MPI_File_set_errhandler PMPI_File_set_errhandler
-#define MPI_File_set_info PMPI_File_set_info
-#define MPI_File_set_size PMPI_File_set_size
-#define MPI_File_set_view PMPI_File_set_view
-#define MPI_File_sync PMPI_File_sync
-
-#include "ompi/mpi/fortran/use-mpi-tkr/mpi-f90-file-interfaces.h"
diff --git a/ompi/mpi/fortran/use-mpi-tkr/pmpi-f90-interfaces.h b/ompi/mpi/fortran/use-mpi-tkr/pmpi-f90-interfaces.h
deleted file mode 100644
index e8e5d6d9582..00000000000
--- a/ompi/mpi/fortran/use-mpi-tkr/pmpi-f90-interfaces.h
+++ /dev/null
@@ -1,210 +0,0 @@
-! -*- fortran -*-
-!
-! Copyright (c) 2020      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-!
-! Additional copyrights may follow
-!
-! $HEADER$
-
-#define MPI_Wtick PMPI_Wtick
-#define MPI_Wtime PMPI_Wtime
-#define MPI_Abort PMPI_Abort
-#define MPI_Add_error_class PMPI_Add_error_class
-#define MPI_Add_error_code PMPI_Add_error_code
-#define MPI_Add_error_string PMPI_Add_error_string
-#define MPI_Aint_add PMPI_Aint_add
-#define MPI_Aint_diff PMPI_Aint_diff
-#define MPI_Barrier PMPI_Barrier
-#define MPI_Ibarrier PMPI_Ibarrier
-#define MPI_Cancel PMPI_Cancel
-#define MPI_Cart_coords PMPI_Cart_coords
-#define MPI_Cart_create PMPI_Cart_create
-#define MPI_Cart_get PMPI_Cart_get
-#define MPI_Cart_map PMPI_Cart_map
-#define MPI_Cart_rank PMPI_Cart_rank
-#define MPI_Cart_shift PMPI_Cart_shift
-#define MPI_Cart_sub PMPI_Cart_sub
-#define MPI_Cartdim_get PMPI_Cartdim_get
-#define MPI_Comm_call_errhandler PMPI_Comm_call_errhandler
-#define MPI_Comm_compare PMPI_Comm_compare
-#define MPI_Comm_create PMPI_Comm_create
-#define MPI_Comm_create_group PMPI_Comm_create_group
-#define MPI_Comm_create_errhandler PMPI_Comm_create_errhandler
-#define MPI_Comm_create_keyval PMPI_Comm_create_keyval
-#define MPI_Comm_delete_attr PMPI_Comm_delete_attr
-#define MPI_Comm_dup PMPI_Comm_dup
-#define MPI_Comm_dup_with_info PMPI_Comm_dup_with_info
-#define MPI_Comm_idup PMPI_Comm_idup
-#define MPI_Comm_free PMPI_Comm_free
-#define MPI_Comm_free_keyval PMPI_Comm_free_keyval
-#define MPI_Comm_get_info PMPI_Comm_get_info
-#define MPI_Comm_get_attr PMPI_Comm_get_attr
-#define MPI_Comm_get_errhandler PMPI_Comm_get_errhandler
-#define MPI_Comm_get_name PMPI_Comm_get_name
-#define MPI_Comm_group PMPI_Comm_group
-#define MPI_Comm_rank PMPI_Comm_rank
-#define MPI_Comm_remote_group PMPI_Comm_remote_group
-#define MPI_Comm_remote_size PMPI_Comm_remote_size
-#define MPI_Comm_set_info PMPI_Comm_set_info
-#define MPI_Comm_set_attr PMPI_Comm_set_attr
-#define MPI_Comm_set_errhandler PMPI_Comm_set_errhandler
-#define MPI_Comm_set_name PMPI_Comm_set_name
-#define MPI_Comm_size PMPI_Comm_size
-#define MPI_Comm_split PMPI_Comm_split
-#define MPI_Comm_test_inter PMPI_Comm_test_inter
-#define MPI_Dims_create PMPI_Dims_create
-#define MPI_Errhandler_free PMPI_Errhandler_free
-#define MPI_Error_class PMPI_Error_class
-#define MPI_Error_string PMPI_Error_string
-#define MPI_Finalize PMPI_Finalize
-#define MPI_Finalized PMPI_Finalized
-#define MPI_Get_count PMPI_Get_count
-#define MPI_Get_elements PMPI_Get_elements
-#define MPI_Get_elements_x PMPI_Get_elements_x
-#define MPI_Get_processor_name PMPI_Get_processor_name
-#define MPI_Get_version PMPI_Get_version
-#define MPI_Graph_create PMPI_Graph_create
-#define MPI_Graph_get PMPI_Graph_get
-#define MPI_Graph_map PMPI_Graph_map
-#define MPI_Graph_neighbors PMPI_Graph_neighbors
-#define MPI_Graph_neighbors_count PMPI_Graph_neighbors_count
-#define MPI_Graphdims_get PMPI_Graphdims_get
-#define MPI_Grequest_complete PMPI_Grequest_complete
-#define MPI_Grequest_start PMPI_Grequest_start
-#define MPI_Group_compare PMPI_Group_compare
-#define MPI_Group_difference PMPI_Group_difference
-#define MPI_Group_excl PMPI_Group_excl
-#define MPI_Group_free PMPI_Group_free
-#define MPI_Group_incl PMPI_Group_incl
-#define MPI_Group_intersection PMPI_Group_intersection
-#define MPI_Group_range_excl PMPI_Group_range_excl
-#define MPI_Group_range_incl PMPI_Group_range_incl
-#define MPI_Group_rank PMPI_Group_rank
-#define MPI_Group_size PMPI_Group_size
-#define MPI_Group_translate_ranks PMPI_Group_translate_ranks
-#define MPI_Group_union PMPI_Group_union
-#define MPI_Info_create PMPI_Info_create
-#define MPI_Info_delete PMPI_Info_delete
-#define MPI_Info_dup PMPI_Info_dup
-#define MPI_Info_free PMPI_Info_free
-#define MPI_Info_get PMPI_Info_get
-#define MPI_Info_get_nkeys PMPI_Info_get_nkeys
-#define MPI_Info_get_nthkey PMPI_Info_get_nthkey
-#define MPI_Info_get_valuelen PMPI_Info_get_valuelen
-#define MPI_Info_set PMPI_Info_set
-#define MPI_Init PMPI_Init
-#define MPI_Init_thread PMPI_Init_thread
-#define MPI_Initialized PMPI_Initialized
-#define MPI_Intercomm_create PMPI_Intercomm_create
-#define MPI_Intercomm_merge PMPI_Intercomm_merge
-#define MPI_Iprobe PMPI_Iprobe
-#define MPI_Is_thread_main PMPI_Is_thread_main
-#define MPI_Op_commutative PMPI_Op_commutative
-#define MPI_Op_create PMPI_Op_create
-#define MPI_Op_free PMPI_Op_free
-#define MPI_Pack_external_size PMPI_Pack_external_size
-#define MPI_Pack_size PMPI_Pack_size
-#define MPI_Pcontrol PMPI_Pcontrol
-#define MPI_Probe PMPI_Probe
-#define MPI_Query_thread PMPI_Query_thread
-#define MPI_Register_datarep PMPI_Register_datarep
-#define MPI_Request_free PMPI_Request_free
-#define MPI_Request_get_status PMPI_Request_get_status
-#define MPI_Start PMPI_Start
-#define MPI_Startall PMPI_Startall
-#define MPI_Status_f2f08 PMPI_Status_f2f08
-#define MPI_Status_f082f PMPI_Status_f082f
-#define MPI_Status_set_cancelled PMPI_Status_set_cancelled
-#define MPI_Status_set_elements PMPI_Status_set_elements
-#define MPI_Test PMPI_Test
-#define MPI_Test_cancelled PMPI_Test_cancelled
-#define MPI_Testall PMPI_Testall
-#define MPI_Testany PMPI_Testany
-#define MPI_Testsome PMPI_Testsome
-#define MPI_Topo_test PMPI_Topo_test
-#define MPI_Type_commit PMPI_Type_commit
-#define MPI_Type_contiguous PMPI_Type_contiguous
-#define MPI_Type_create_darray PMPI_Type_create_darray
-#define MPI_Type_create_f90_complex PMPI_Type_create_f90_complex
-#define MPI_Type_create_f90_integer PMPI_Type_create_f90_integer
-#define MPI_Type_create_f90_real PMPI_Type_create_f90_real
-#define MPI_Type_create_hindexed PMPI_Type_create_hindexed
-#define MPI_Type_create_hvector PMPI_Type_create_hvector
-#define MPI_Type_create_indexed_block PMPI_Type_create_indexed_block
-#define MPI_Type_create_keyval PMPI_Type_create_keyval
-#define MPI_Type_create_resized PMPI_Type_create_resized
-#define MPI_Type_create_struct PMPI_Type_create_struct
-#define MPI_Type_create_subarray PMPI_Type_create_subarray
-#define MPI_Type_delete_attr PMPI_Type_delete_attr
-#define MPI_Type_dup PMPI_Type_dup
-#define MPI_Type_free PMPI_Type_free
-#define MPI_Type_free_keyval PMPI_Type_free_keyval
-#define MPI_Type_get_attr PMPI_Type_get_attr
-#define MPI_Type_get_contents PMPI_Type_get_contents
-#define MPI_Type_get_envelope PMPI_Type_get_envelope
-#define MPI_Type_get_extent PMPI_Type_get_extent
-#define MPI_Type_get_extent_x PMPI_Type_get_extent_x
-#define MPI_Type_get_name PMPI_Type_get_name
-#define MPI_Type_get_true_extent PMPI_Type_get_true_extent
-#define MPI_Type_get_true_extent_x PMPI_Type_get_true_extent_x
-#define MPI_Type_indexed PMPI_Type_indexed
-#define MPI_Type_match_size PMPI_Type_match_size
-#define MPI_Type_set_attr PMPI_Type_set_attr
-#define MPI_Type_set_name PMPI_Type_set_name
-#define MPI_Type_size PMPI_Type_size
-#define MPI_Type_size_x PMPI_Type_size_x
-#define MPI_Type_vector PMPI_Type_vector
-#define MPI_Wait PMPI_Wait
-#define MPI_Waitall PMPI_Waitall
-#define MPI_Waitany PMPI_Waitany
-#define MPI_Waitsome PMPI_Waitsome
-#define MPI_Win_call_errhandler PMPI_Win_call_errhandler
-#define MPI_Win_complete PMPI_Win_complete
-#define MPI_Win_create_errhandler PMPI_Win_create_errhandler
-#define MPI_Win_create_keyval PMPI_Win_create_keyval
-#define MPI_Win_delete_attr PMPI_Win_delete_attr
-#define MPI_Win_fence PMPI_Win_fence
-#define MPI_Win_free PMPI_Win_free
-#define MPI_Win_free_keyval PMPI_Win_free_keyval
-#define MPI_Win_get_attr PMPI_Win_get_attr
-#define MPI_Win_get_errhandler PMPI_Win_get_errhandler
-#define MPI_Win_get_group PMPI_Win_get_group
-#define MPI_Win_get_name PMPI_Win_get_name
-#define MPI_Win_lock PMPI_Win_lock
-#define MPI_Win_post PMPI_Win_post
-#define MPI_Win_set_attr PMPI_Win_set_attr
-#define MPI_Win_set_errhandler PMPI_Win_set_errhandler
-#define MPI_Win_set_name PMPI_Win_set_name
-#define MPI_Win_start PMPI_Win_start
-#define MPI_Win_test PMPI_Win_test
-#define MPI_Win_unlock PMPI_Win_unlock
-#define MPI_Win_wait PMPI_Win_wait
-#define MPI_Close_port PMPI_Close_port
-#define MPI_Lookup_name PMPI_Lookup_name
-#define MPI_Open_port PMPI_Open_port
-#define MPI_Publish_name PMPI_Publish_name
-#define MPI_Unpublish_name PMPI_Unpublish_name
-#define MPI_Comm_disconnect PMPI_Comm_disconnect
-#define MPI_Comm_get_parent PMPI_Comm_get_parent
-#define MPI_Comm_join PMPI_Comm_join
-#define MPI_Comm_accept PMPI_Comm_accept
-#define MPI_Comm_connect PMPI_Comm_connect
-#define MPI_Comm_spawn PMPI_Comm_spawn
-#define MPI_Comm_spawn_multiple PMPI_Comm_spawn_multiple
-#define MPI_Mprobe PMPI_Mprobe
-#define MPI_Improbe PMPI_Improbe
-#define MPI_Get_library_version PMPI_Get_library_version
-#define MPI_Comm_split_type PMPI_Comm_split_type
-#define MPI_Type_create_hindexed_block PMPI_Type_create_hindexed_block
-#define MPI_Dist_graph_create PMPI_Dist_graph_create
-#define MPI_Dist_graph_create_adjacent PMPI_Dist_graph_create_adjacent
-#define MPI_Dist_graph_neighbors_count PMPI_Dist_graph_neighbors_count
-#define MPI_Dist_graph_neighbors PMPI_Dist_graph_neighbors
-#define MPI_Win_flush PMPI_Win_flush
-#define MPI_Win_flush_all PMPI_Win_flush_all
-#define MPI_Win_flush_local PMPI_Win_flush_local
-#define MPI_Win_flush_local_all PMPI_Win_flush_local_all
-
-#include "ompi/mpi/fortran/use-mpi-tkr/mpi-f90-interfaces.h"
diff --git a/ompi/mpi/fortran/use-mpi-tkr/pmpi-f90-removed-interfaces.h b/ompi/mpi/fortran/use-mpi-tkr/pmpi-f90-removed-interfaces.h
deleted file mode 100644
index 39a51e87dc7..00000000000
--- a/ompi/mpi/fortran/use-mpi-tkr/pmpi-f90-removed-interfaces.h
+++ /dev/null
@@ -1,26 +0,0 @@
-! -*- fortran -*-
-!
-! Copyright (c) 2020      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-!
-! Additional copyrights may follow
-!
-! $HEADER$
-
-#define MPI_Attr_delete PMPI_Attr_delete
-#define MPI_Attr_get PMPI_Attr_get
-#define MPI_Attr_put PMPI_Attr_put
-#define MPI_Errhandler_create PMPI_Errhandler_create
-#define MPI_Errhandler_get PMPI_Errhandler_get
-#define MPI_Errhandler_set PMPI_Errhandler_set
-#define MPI_Keyval_create PMPI_Keyval_create
-#define MPI_Keyval_free PMPI_Keyval_free
-#define MPI_Type_extent PMPI_Type_extent
-#define MPI_Type_hindexed PMPI_Type_hindexed
-#define MPI_Type_hvector PMPI_Type_hvector
-#define MPI_Type_lb PMPI_Type_lb
-#define MPI_Type_struct PMPI_Type_struct
-#define MPI_Type_ub PMPI_Type_ub
-
-#include "ompi/mpi/fortran/use-mpi-tkr/mpi-f90-removed-interfaces.h"
diff --git a/ompi/mpi/fortran/use-mpi-tkr/pmpi-f90-status.h b/ompi/mpi/fortran/use-mpi-tkr/pmpi-f90-status.h
deleted file mode 100644
index d20bd24829e..00000000000
--- a/ompi/mpi/fortran/use-mpi-tkr/pmpi-f90-status.h
+++ /dev/null
@@ -1,35 +0,0 @@
-! -*- fortran -*-
-!
-! Copyright (c) 2020      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-!
-! Additional copyrights may follow
-!
-! $HEADER$
-
-
-interface PMPI_Status_f082f
-
-subroutine PMPI_Status_f082f(f08_status, f_status, ierror)
-  use mpi_types
-  include 'mpif-config.h'
-  type(MPI_Status), intent(in) :: f08_status
-  integer, intent(out) :: f_status(MPI_STATUS_SIZE)
-  integer, intent(out) :: ierror
-end subroutine PMPI_Status_f082f
-
-end interface
-
-
-interface PMPI_Status_f2f08
-
-subroutine PMPI_Status_f2f08(f_status, f08_status, ierror)
-  use mpi_types
-  include 'mpif-config.h'
-  integer, intent(in) :: f_status(MPI_STATUS_SIZE)
-  type(MPI_Status), intent(out) :: f08_status
-  integer, intent(out) :: ierror
-end subroutine PMPI_Status_f2f08
-
-end interface
diff --git a/ompi/mpi/fortran/use-mpi-tkr/test/align_c.c b/ompi/mpi/fortran/use-mpi-tkr/test/align_c.c
deleted file mode 100644
index eb46082eb05..00000000000
--- a/ompi/mpi/fortran/use-mpi-tkr/test/align_c.c
+++ /dev/null
@@ -1,73 +0,0 @@
-#include "ompi_config.h"
-#include <assert.h>
-
-BEGIN_C_DECLS
-
-void PRINT_INT(int* i);
-
-void align_c(char *a, char *w, char *x, char *y, char *z)
-{
-    unsigned long aw, ax, ay, az;
-    int diff, diff4;
-
-    aw = (unsigned long) w;
-    ax = (unsigned long) x;
-    ay = (unsigned long) y;
-    az = (unsigned long) z;
-
-    if (! ((aw%16)||(ax%16)||(ay%16)||(az%16))) {
-        diff4 = 16;
-    }
-    else if (! ((aw%12)||(ax%12)||(ay%12)||(az%12))) {
-        diff4 = 12;
-    }
-    else if (! ((aw%8)||(ax%8)||(ay%8)||(az%8))) {
-        diff4 = 8;
-    }
-    else if (! ((aw%4)||(ax%4)||(ay%4)||(az%4))) {
-        diff4 = 4;
-    }
-    else if (! ((aw%2)||(ax%2)||(ay%2)||(az%2))) {
-        diff4 = 2;
-    }
-    else {
-        diff4 = 1;
-    }
-    diff = a - w;
-    diff = (diff >= 0) ? diff : -diff;
-    *a = diff;
-    assert(diff4 == diff);
-}
-
-
-void align_c_(char *a, char *w, char *x, char *y, char *z)
-{
-  align_c(a, w, x, y, z);
-}
-
-
-void align_c__(char *a, char *w, char *x, char *y, char *z)
-{
-  align_c(a, w, x, y, z);
-}
-
-
-void ALIGN_C(char *a, char *w, char *x, char *y, char *z)
-{
-  align_c(a, w, x, y, z);
-}
-
-
-void ALIGN_C_(char *a, char *w, char *x, char *y, char *z)
-{
-  align_c(a, w, x, y, z);
-}
-
-
-void ALIGN_C__(char *a, char *w, char *x, char *y, char *z)
-{
-  align_c(a, w, x, y, z);
-}
-
-END_C_DECLS
-
diff --git a/ompi/mpi/fortran/use-mpi-tkr/test/print_align.f90 b/ompi/mpi/fortran/use-mpi-tkr/test/print_align.f90
deleted file mode 100644
index 126ca6f3b6a..00000000000
--- a/ompi/mpi/fortran/use-mpi-tkr/test/print_align.f90
+++ /dev/null
@@ -1,108 +0,0 @@
-!
-! Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-!                         University Research and Technology
-!                         Corporation.  All rights reserved.
-! Copyright (c) 2004-2005 The University of Tennessee and The University
-!                         of Tennessee Research Foundation.  All rights
-!                         reserved.
-! Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-!                         University of Stuttgart.  All rights reserved.
-! Copyright (c) 2004-2005 The Regents of the University of California.
-!                         All rights reserved.
-! Copyright (c) 2012 Cisco Systems, Inc.  All rights reserved.
-! $COPYRIGHT$
-!
-! Additional copyrights may follow
-!
-! $HEADER$
-!
-! print_align.f90
-!   - prints out alignment of real types
-!
-
-program main
-
-  type AlignReal
-    integer(selected_int_kind(1)) :: a
-    real(selected_real_kind(6)) :: w
-    character b
-    real(selected_real_kind(6)) :: x
-    real(selected_real_kind(6)) :: y
-    character c
-    real(selected_real_kind(6)) :: z
-  end type
-
-  type AlignDouble
-    integer(selected_int_kind(1)) :: a
-    real(selected_real_kind(15)) :: w
-    character b
-    real(selected_real_kind(15)) :: x
-    real(selected_real_kind(15)) :: y
-
-    character c
-    real(selected_real_kind(15)) :: z
-  end type
-
-  type AlignQuad
-    integer(selected_int_kind(1)) :: a
-    real(selected_real_kind(31)) :: w
-    character b
-    real(selected_real_kind(31)) :: x
-    real(selected_real_kind(31)) :: y
-    character c
-    real(selected_real_kind(31)) :: z
-  end type
-
-  type AlignComplex
-    integer(selected_int_kind(1)) :: a
-    complex(selected_real_kind(6)) :: w
-    character b
-    complex(selected_real_kind(6)) :: x
-    complex(selected_real_kind(6)) :: y
-    character c
-    complex(selected_real_kind(6)) :: z
-  end type
-
-  type AlignDoubleComplex
-    integer(selected_int_kind(1)) :: a
-    complex(selected_real_kind(15)) :: w
-    character b
-    complex(selected_real_kind(15)) :: x
-    complex(selected_real_kind(15)) :: y
-    character c
-    complex(selected_real_kind(15)) :: z
-  end type
-
-  type AlignQuadComplex
-    integer(selected_int_kind(1)) :: a
-    complex(selected_real_kind(31)) :: w
-    character b
-    complex(selected_real_kind(31)) :: x
-    complex(selected_real_kind(31)) :: y
-    character c
-    complex(selected_real_kind(31)) :: z
-  end type
-
-  external align_c
-
-  type(AlignReal) :: ar
-  type(AlignDouble) :: ad
-  type(AlignQuad) :: aq
-  type(AlignComplex) :: ac
-  type(AlignDoubleComplex) :: adc
-  type(AlignQuadComplex) :: aqc
-
-  call align_c(ar%a, ar%w, ar%x, ar%y, ar%z)
-  print *, "alignment of real: ", ar%a
-  call align_c(ad%a, ad%w, ad%x, ad%y, ad%z)
-  print *, "alignment of double: ", ad%a
-  call align_c(aq%a, aq%w, aq%x, aq%y, aq%z)
-  print *, "alignment of quad: ", aq%a
-  call align_c(ac%a, ac%w, ac%x, ac%y, ac%z)
-  print *, "alignment of complex: ", ac%a
-  call align_c(adc%a, adc%w, adc%x, adc%y, adc%z)
-  print *, "alignment of double complex: ", adc%a
-  call align_c(aqc%a, aqc%w, aqc%x, aqc%y, aqc%z)
-  print *, "alignment of quad complex: ", aqc%a
-
-end program main
diff --git a/ompi/mpi/fortran/use-mpi-tkr/test/print_ikinds.f90 b/ompi/mpi/fortran/use-mpi-tkr/test/print_ikinds.f90
deleted file mode 100644
index d2b5e6aea9b..00000000000
--- a/ompi/mpi/fortran/use-mpi-tkr/test/print_ikinds.f90
+++ /dev/null
@@ -1,37 +0,0 @@
-!
-! Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-!                         University Research and Technology
-!                         Corporation.  All rights reserved.
-! Copyright (c) 2004-2005 The University of Tennessee and The University
-!                         of Tennessee Research Foundation.  All rights
-!                         reserved.
-! Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-!                         University of Stuttgart.  All rights reserved.
-! Copyright (c) 2004-2005 The Regents of the University of California.
-!                         All rights reserved.
-! Copyright (c) 2012 Cisco Systems, Inc.  All rights reserved.
-! $COPYRIGHT$
-!
-! Additional copyrights may follow
-!
-! $HEADER$
-!
-! print_ikinds.f90
-!   - prints out values of selected_int_kind to determine Fortran compiler
-!     dependent values
-!
-program main
-  integer :: i
-  !
-  !  integer(kind(1_LONG)) :: l = 1_LONG
-  !
-
-  do i = 1, 24
-    print*, "i = ", i, "selected_int_kind(i) = ", selected_int_kind(i)
-  end do
-
-  print*, "kind(1) = ", kind(1)
-  !
-  !  print*, "kind(1_LONG) = ", kind(l)
-  !
-end program main
diff --git a/ompi/mpi/fortran/use-mpi-tkr/test/print_prec_range.f90 b/ompi/mpi/fortran/use-mpi-tkr/test/print_prec_range.f90
deleted file mode 100644
index 1abc5526875..00000000000
--- a/ompi/mpi/fortran/use-mpi-tkr/test/print_prec_range.f90
+++ /dev/null
@@ -1,48 +0,0 @@
-!
-! Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-!                         University Research and Technology
-!                         Corporation.  All rights reserved.
-! Copyright (c) 2004-2005 The University of Tennessee and The University
-!                         of Tennessee Research Foundation.  All rights
-!                         reserved.
-! Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-!                         University of Stuttgart.  All rights reserved.
-! Copyright (c) 2004-2005 The Regents of the University of California.
-!                         All rights reserved.
-! Copyright (c) 2012 Cisco Systems, Inc.  All rights reserved.
-! $COPYRIGHT$
-!
-! Additional copyrights may follow
-!
-! $HEADER$
-!
-! print_prec_range.f90
-!   - prints out values of precision and range for real types
-!
-program main
-  real :: xr
-  real(kind(1.0D0)) :: xd
-  real(selected_real_kind(5)) :: x5
-  real(selected_real_kind(6)) :: x6
-  real(selected_real_kind(7)) :: x7
-  real(selected_real_kind(14)) :: x14
-  real(selected_real_kind(15)) :: x15
-  real(selected_real_kind(16)) :: x16
-  real(selected_real_kind(30)) :: x30
-  real(selected_real_kind(31)) :: x31
-  real(selected_real_kind(32)) :: x32
-
-  print *, "precision(p=5) = ", precision(x5), " range(p=5) = ", range(x5)
-  print *, "precision(p=6) = ", precision(x6), " range(p=6) = ", range(x6)
-  print *, "precision(p=7) = ", precision(x7), " range(p=7) = ", range(x7)
-  print *, "precision(p=14) = ", precision(x14), " range(p=14) = ", range(x14)
-  print *, "precision(p=15) = ", precision(x15), " range(p=15) = ", range(x15)
-  print *, "precision(p=16) = ", precision(x16), " range(p=16) = ", range(x16)
-  print *, "precision(p=30) = ", precision(x30), " range(p=30) = ", range(x30)
-  print *, "precision(p=31) = ", precision(x31), " range(p=31) = ", range(x31)
-  print *, "precision(p=32) = ", precision(x32), " range(p=32) = ", range(x32)
-  print *
-  print *, "precision(r) = ", precision(xr), " range(r) = ", range(r)
-  print *, "precision(d) = ", precision(xd), " range(d) = ", range(d)
-
-end program main
diff --git a/ompi/mpi/fortran/use-mpi-tkr/test/print_rkinds.f90 b/ompi/mpi/fortran/use-mpi-tkr/test/print_rkinds.f90
deleted file mode 100644
index 3eaa5466910..00000000000
--- a/ompi/mpi/fortran/use-mpi-tkr/test/print_rkinds.f90
+++ /dev/null
@@ -1,41 +0,0 @@
-!
-! Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-!                         University Research and Technology
-!                         Corporation.  All rights reserved.
-! Copyright (c) 2004-2005 The University of Tennessee and The University
-!                         of Tennessee Research Foundation.  All rights
-!                         reserved.
-! Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-!                         University of Stuttgart.  All rights reserved.
-! Copyright (c) 2004-2005 The Regents of the University of California.
-!                         All rights reserved.
-! Copyright (c) 2012 Cisco Systems, Inc.  All rights reserved.
-! $COPYRIGHT$
-!
-! Additional copyrights may follow
-!
-! $HEADER$
-!
-! print_rkinds.f90
-!   - prints out values of selected_real_kind to determine Fortran compiler
-!     dependent values
-!
-program main
-  integer :: i
-  !
-  !  real :: x = 2.2_QUAD
-  !
-  complex(kind(1.D0)) :: z
-
-  do i = 1, 32
-    print*, "i = ", i, "selected_real_kind(i) = ", selected_real_kind(i)
-  end do
-
-  print*, "kind(1.E0) = ", kind(1.E0)
-  print*, "kind(1.D0) = ", kind(1.D0)
-  !
-  !  print*, "kind( real :: x = 2.2_QUAD ) = ", kind(x)
-  !
-  print*, "kind( complex(kind(1.D0) :: z ) = ", kind(z)
-
-end program main
diff --git a/ompi/mpi/fortran/use-mpi-tkr/test/send_t.c b/ompi/mpi/fortran/use-mpi-tkr/test/send_t.c
deleted file mode 100644
index e39a02f25eb..00000000000
--- a/ompi/mpi/fortran/use-mpi-tkr/test/send_t.c
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2012 Cisco Systems, Inc.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- *
- * send_t.c - tests mpi_send variants
- *
- * Actual tests do not use MPI, they just test parameters and types.
- */
-
-#include <stdio.h>
-
-#define MPI_COMM_WORLD 0
-#define MPI_COMM_SELF  1
-
-#define MPI_INTEGER   7
-#define MPI_INTEGER1  8
-
-
-void mpi_send_1di(int* a, int* count, int* datatype,
-		  int* dest, int* tag, int* comm, int* ierr)
-{
-  int i;
-
-  *ierr = 0;
-  for (i = 0; i < *count; i++) {
-    if (a[i] != 10 - i) {
-      printf("a[%d] = %d\n", i, a[i]);
-      *ierr = 1;
-    }
-  }
-  if (*count != 10) *ierr = 1;
-  if (*datatype != MPI_INTEGER) *ierr = 1;
-  if (*dest != 59) *ierr = 1;
-  if (*tag != 999) *ierr = 1;
-  if (*comm != MPI_COMM_WORLD) *ierr = 1;
-}
-
-
-void mpi_send_1di1(char* a, int* count, int* datatype,
-		   int* dest, int* tag, int* comm, int* ierr)
-{
-  int i;
-
-  *ierr = 0;
-  for (i = 0; i < *count; i++) {
-    if (a[i] != i+1) {
-      printf("a[%d] = %d\n", i, a[i]);
-      *ierr = 1;
-    }
-  }
-  if (*count != 10) *ierr = 1;
-  if (*datatype != MPI_INTEGER1) *ierr = 1;
-  if (*dest != 59) *ierr = 1;
-  if (*tag != 999) *ierr = 1;
-  if (*comm != MPI_COMM_SELF) *ierr = 1;
-}
-
diff --git a/ompi/mpi/fortran/use-mpi-tkr/test/test_send.f90 b/ompi/mpi/fortran/use-mpi-tkr/test/test_send.f90
deleted file mode 100644
index 33c24ec845a..00000000000
--- a/ompi/mpi/fortran/use-mpi-tkr/test/test_send.f90
+++ /dev/null
@@ -1,54 +0,0 @@
-!
-! Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-!                         University Research and Technology
-!                         Corporation.  All rights reserved.
-! Copyright (c) 2004-2005 The University of Tennessee and The University
-!                         of Tennessee Research Foundation.  All rights
-!                         reserved.
-! Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-!                         University of Stuttgart.  All rights reserved.
-! Copyright (c) 2004-2005 The Regents of the University of California.
-!                         All rights reserved.
-! Copyright (c) 2012 Cisco Systems, Inc.  All rights reserved.
-! $COPYRIGHT$
-!
-! Additional copyrights may follow
-!
-! $HEADER$
-!
-! test_send.f90 - tests mpi_send variants (calls functions in send_t.c)
-!
-! Actual tests do not use MPI, they just test parameters and types.
-!
-
-program main
-  use mpi
-
-  implicit none
-
-  integer(kind=MPI_INTEGER1_KIND) :: ai1(10) = (/1,2,3,4,5,6,7,8,9,10/)
-  integer, target  :: ai(10) = (/10,9,8,7,6,5,4,3,2,1/)
-  integer, pointer :: pai(:)
-  integer :: count, dest, tag, ierr
-
-  count = 10
-  dest = 59
-  tag = 999
-
-  call mpi_send(ai, count, MPI_INTEGER, dest, tag, MPI_COMM_WORLD, ierr)
-  if (ierr /= MPI_SUCCESS) then
-    print *, "FAILURE test_send: ierr = ", ierr
-  end if
-
-  pai => ai
-  call mpi_send(pai, count, MPI_INTEGER, dest, tag, MPI_COMM_WORLD, ierr)
-  if (ierr /= MPI_SUCCESS) then
-    print *, "FAILURE test_send: ierr = ", ierr
-  end if
-
-  call mpi_send(ai1, count, MPI_INTEGER1, dest, tag, MPI_COMM_SELF, ierr)
-  if (ierr /= MPI_SUCCESS) then
-    print *, "FAILURE test_send: ierr = ", ierr
-  end if
-
-end program
diff --git a/ompi/mpi/fortran/use-mpi/Makefile.am b/ompi/mpi/fortran/use-mpi/Makefile.am
index 3f1bc6b1760..edc2b088ddc 100644
--- a/ompi/mpi/fortran/use-mpi/Makefile.am
+++ b/ompi/mpi/fortran/use-mpi/Makefile.am
@@ -55,13 +55,13 @@ mpi-types.lo: mpi-types.F90
 
 install-exec-hook:
 	@ for file in `ls *.mod`; do \
-	  echo $(INSTALL) $$file $(DESTDIR)$(libdir); \
-	  $(INSTALL) $$file $(DESTDIR)$(libdir); \
+	  echo $(INSTALL) $$file $(DESTDIR)$(OMPI_FORTRAN_MODULEDIR); \
+	  $(INSTALL) $$file $(DESTDIR)$(OMPI_FORTRAN_MODULEDIR); \
 	done
 
 uninstall-local:
 	@ for file in `ls *.mod`; do \
-	  echo rm -f $(DESTDIR)$(libdir)/$$file; \
-	  rm -f $(DESTDIR)$(libdir)/$$file; \
+	  echo rm -f $(DESTDIR)$(OMPI_FORTRAN_MODULEDIR)/$$file; \
+	  rm -f $(DESTDIR)$(OMPI_FORTRAN_MODULEDIR)/$$file; \
 	done
 endif
diff --git a/ompi/mpi/fortran/use-mpi/mpi-types.F90.in b/ompi/mpi/fortran/use-mpi/mpi-types.F90.in
index 3b2ac85e2e8..2b204eafdd7 100644
--- a/ompi/mpi/fortran/use-mpi/mpi-types.F90.in
+++ b/ompi/mpi/fortran/use-mpi/mpi-types.F90.in
@@ -2,6 +2,9 @@
 !
 ! Copyright (c) 2020      Research Organization for Information Science
 !                         and Technology (RIST).  All rights reserved.
+! Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved
+! Copyright (c) 2022      Triad National Security, LLC. All rights
+!                         reserved.
 ! $COPYRIGHT$
 !
 ! Additional copyrights may follow
@@ -16,9 +19,12 @@
 #define OMPI_PRIVATE
 #endif
 
+! These types appear in both the "mpi" module and the "mpi_f08" module
+! (yes, the MPI spec requires that the TYPE(MPI_Blah) types all show
+! up in both modules).
 module mpi_types
 
-type, BIND(C) :: MPI_Status
+  type, BIND(C) :: MPI_Status
       integer :: MPI_SOURCE
       integer :: MPI_TAG
       integer :: MPI_ERROR
@@ -36,4 +42,184 @@ type, BIND(C) :: MPI_Status
       integer OMPI_PRIVATE :: internal(@OMPI_FORTRAN_STATUS_SIZE@ - 3)
    end type MPI_Status
 
+   type, BIND(C) :: MPI_Comm
+      integer :: MPI_VAL
+   end type MPI_Comm
+
+   type, BIND(C) :: MPI_Datatype
+      integer :: MPI_VAL
+   end type MPI_Datatype
+
+   type, BIND(C) :: MPI_Errhandler
+      integer :: MPI_VAL
+   end type MPI_Errhandler
+
+   type, BIND(C) :: MPI_File
+      integer :: MPI_VAL
+   end type MPI_File
+
+   type, BIND(C) :: MPI_Group
+      integer :: MPI_VAL
+   end type MPI_Group
+
+   type, BIND(C) :: MPI_Info
+      integer :: MPI_VAL
+   end type MPI_Info
+
+   type, BIND(C) :: MPI_Message
+      integer :: MPI_VAL
+   end type MPI_Message
+
+   type, BIND(C) :: MPI_Op
+      integer :: MPI_VAL
+   end type MPI_Op
+
+   type, BIND(C) :: MPI_Request
+      integer :: MPI_VAL
+   end type MPI_Request
+
+   type, BIND(C) :: MPI_Session
+      integer :: MPI_VAL
+   end type MPI_Session
+
+   type, BIND(C) :: MPI_Win
+      integer :: MPI_VAL
+   end type MPI_Win
+
+   ! Interfaces for operators with handles.
+   ! There are no comparisons for MPI_Status because it is not a handle.
+   ! This covers both .EQ. and ==
+   interface operator (.EQ.)
+      module procedure ompi_comm_op_eq
+      module procedure ompi_datatype_op_eq
+      module procedure ompi_errhandler_op_eq
+      module procedure ompi_file_op_eq
+      module procedure ompi_group_op_eq
+      module procedure ompi_info_op_eq
+      module procedure ompi_message_op_eq
+      module procedure ompi_op_op_eq
+      module procedure ompi_request_op_eq
+      module procedure ompi_win_op_eq
+   end interface operator (.EQ.)
+
+   ! This covers both .NE. and /=
+   interface operator (.NE.)
+      module procedure ompi_comm_op_ne
+      module procedure ompi_datatype_op_ne
+      module procedure ompi_errhandler_op_ne
+      module procedure ompi_file_op_ne
+      module procedure ompi_group_op_ne
+      module procedure ompi_info_op_ne
+      module procedure ompi_message_op_ne
+      module procedure ompi_op_op_ne
+      module procedure ompi_request_op_ne
+      module procedure ompi_win_op_ne
+   end interface operator (.NE.)
+
+contains
+
+  ! .EQ. operator
+  !-----------------
+  @OMPI_FORTRAN_ELEMENTAL_TYPE@ logical function ompi_comm_op_eq(a, b)
+    type(MPI_Comm), intent(in) :: a, b
+    ompi_comm_op_eq = (a%MPI_VAL .EQ. b%MPI_VAL)
+  end function ompi_comm_op_eq
+
+  @OMPI_FORTRAN_ELEMENTAL_TYPE@ logical function ompi_datatype_op_eq(a, b)
+    type(MPI_Datatype), intent(in) :: a, b
+    ompi_datatype_op_eq = (a%MPI_VAL .EQ. b%MPI_VAL)
+  end function ompi_datatype_op_eq
+
+  @OMPI_FORTRAN_ELEMENTAL_TYPE@ logical function ompi_errhandler_op_eq(a, b)
+    type(MPI_Errhandler), intent(in) :: a, b
+    ompi_errhandler_op_eq = (a%MPI_VAL .EQ. b%MPI_VAL)
+  end function ompi_errhandler_op_eq
+
+  @OMPI_FORTRAN_ELEMENTAL_TYPE@ logical function ompi_file_op_eq(a, b)
+    type(MPI_File), intent(in) :: a, b
+    ompi_file_op_eq = (a%MPI_VAL .EQ. b%MPI_VAL)
+  end function ompi_file_op_eq
+
+  @OMPI_FORTRAN_ELEMENTAL_TYPE@ logical function ompi_group_op_eq(a, b)
+    type(MPI_Group), intent(in) :: a, b
+    ompi_group_op_eq = (a%MPI_VAL .EQ. b%MPI_VAL)
+  end function ompi_group_op_eq
+
+  @OMPI_FORTRAN_ELEMENTAL_TYPE@ logical function ompi_info_op_eq(a, b)
+    type(MPI_Info), intent(in) :: a, b
+    ompi_info_op_eq = (a%MPI_VAL .EQ. b%MPI_VAL)
+  end function ompi_info_op_eq
+
+  @OMPI_FORTRAN_ELEMENTAL_TYPE@ logical function ompi_message_op_eq(a, b)
+    type(MPI_Message), intent(in) :: a, b
+    ompi_message_op_eq = (a%MPI_VAL .EQ. b%MPI_VAL)
+  end function ompi_message_op_eq
+
+  @OMPI_FORTRAN_ELEMENTAL_TYPE@ logical function ompi_op_op_eq(a, b)
+    type(MPI_Op), intent(in) :: a, b
+    ompi_op_op_eq = (a%MPI_VAL .EQ. b%MPI_VAL)
+  end function ompi_op_op_eq
+
+  @OMPI_FORTRAN_ELEMENTAL_TYPE@ logical function ompi_request_op_eq(a, b)
+    type(MPI_Request), intent(in) :: a, b
+    ompi_request_op_eq = (a%MPI_VAL .EQ. b%MPI_VAL)
+  end function ompi_request_op_eq
+
+  @OMPI_FORTRAN_ELEMENTAL_TYPE@ logical function ompi_win_op_eq(a, b)
+    type(MPI_Win), intent(in) :: a, b
+    ompi_win_op_eq = (a%MPI_VAL .EQ. b%MPI_VAL)
+  end function ompi_win_op_eq
+
+  ! .NE. operator
+  !-----------------
+  @OMPI_FORTRAN_ELEMENTAL_TYPE@ logical function ompi_comm_op_ne(a, b)
+    type(MPI_Comm), intent(in) :: a, b
+    ompi_comm_op_ne = (a%MPI_VAL .NE. b%MPI_VAL)
+  end function ompi_comm_op_ne
+
+  @OMPI_FORTRAN_ELEMENTAL_TYPE@ logical function ompi_datatype_op_ne(a, b)
+    type(MPI_Datatype), intent(in) :: a, b
+    ompi_datatype_op_ne = (a%MPI_VAL .NE. b%MPI_VAL)
+  end function ompi_datatype_op_ne
+
+  @OMPI_FORTRAN_ELEMENTAL_TYPE@ logical function ompi_errhandler_op_ne(a, b)
+    type(MPI_Errhandler), intent(in) :: a, b
+    ompi_errhandler_op_ne = (a%MPI_VAL .NE. b%MPI_VAL)
+  end function ompi_errhandler_op_ne
+
+  @OMPI_FORTRAN_ELEMENTAL_TYPE@ logical function ompi_file_op_ne(a, b)
+    type(MPI_File), intent(in) :: a, b
+    ompi_file_op_ne = (a%MPI_VAL .NE. b%MPI_VAL)
+  end function ompi_file_op_ne
+
+  @OMPI_FORTRAN_ELEMENTAL_TYPE@ logical function ompi_group_op_ne(a, b)
+    type(MPI_Group), intent(in) :: a, b
+    ompi_group_op_ne = (a%MPI_VAL .NE. b%MPI_VAL)
+  end function ompi_group_op_ne
+
+  @OMPI_FORTRAN_ELEMENTAL_TYPE@ logical function ompi_info_op_ne(a, b)
+    type(MPI_Info), intent(in) :: a, b
+    ompi_info_op_ne = (a%MPI_VAL .NE. b%MPI_VAL)
+  end function ompi_info_op_ne
+
+  @OMPI_FORTRAN_ELEMENTAL_TYPE@ logical function ompi_message_op_ne(a, b)
+    type(MPI_Message), intent(in) :: a, b
+    ompi_message_op_ne = (a%MPI_VAL .NE. b%MPI_VAL)
+  end function ompi_message_op_ne
+
+  @OMPI_FORTRAN_ELEMENTAL_TYPE@ logical function ompi_op_op_ne(a, b)
+    type(MPI_Op), intent(in) :: a, b
+    ompi_op_op_ne = (a%MPI_VAL .NE. b%MPI_VAL)
+  end function ompi_op_op_ne
+
+  @OMPI_FORTRAN_ELEMENTAL_TYPE@ logical function ompi_request_op_ne(a, b)
+    type(MPI_Request), intent(in) :: a, b
+    ompi_request_op_ne  = (a%MPI_VAL .NE. b%MPI_VAL)
+  end function ompi_request_op_ne
+
+  @OMPI_FORTRAN_ELEMENTAL_TYPE@ logical function ompi_win_op_ne(a, b)
+    type(MPI_Win), intent(in) :: a, b
+    ompi_win_op_ne = (a%MPI_VAL .NE. b%MPI_VAL)
+  end function ompi_win_op_ne
+
 end module
diff --git a/ompi/mpi/help-mpi-api.txt b/ompi/mpi/help-mpi-api.txt
index 830b2b4e7e2..e7e29e5ab78 100644
--- a/ompi/mpi/help-mpi-api.txt
+++ b/ompi/mpi/help-mpi-api.txt
@@ -2,7 +2,7 @@
 #
 # Copyright (c) 2006      High Performance Computing Center Stuttgart,
 #                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2006-2015 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2006-2022 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2018 IBM Corporation. All rights reserved.
 # Copyright (c) 2020      The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
@@ -30,8 +30,3 @@ this environment.
 
   MPI function: %s
   Reason:       %s
-[info-set-with-reserved-prefix]
-Comments
-MPI_Info_set warning, key is using a reserved prefix.
-  Key:             %s
-  Reserved prefix: %s
diff --git a/ompi/mpi/java/c/mpi_Comm.c b/ompi/mpi/java/c/mpi_Comm.c
index 8dd0f236638..f31f210037a 100644
--- a/ompi/mpi/java/c/mpi_Comm.c
+++ b/ompi/mpi/java/c/mpi_Comm.c
@@ -933,7 +933,7 @@ JNIEXPORT jlong JNICALL Java_mpi_Comm_iGather(
     if(rootOrInter || sPtr == MPI_IN_PLACE)
     {
         /*
-         * In principle need the "id == root" check here and elsewere for
+         * In principle need the "id == root" check here and elsewhere for
          * correctness, in case arguments that are not supposed to be
          * significant except on root are legitimately passed in as `null',
          * say.  Shouldn't produce null pointer exception.
diff --git a/ompi/mpi/java/c/mpi_Info.c b/ompi/mpi/java/c/mpi_Info.c
index 93545c757e8..2bfdc5e8597 100644
--- a/ompi/mpi/java/c/mpi_Info.c
+++ b/ompi/mpi/java/c/mpi_Info.c
@@ -45,17 +45,20 @@ JNIEXPORT jlong JNICALL Java_mpi_Info_getNull(JNIEnv *env, jclass clazz)
     return (jlong)MPI_INFO_NULL;
 }
 
+// At least some versions of jni.h have a global named "jvalue", and
+// we get a compiler warning if we have a parameter or variable of the
+// same name.  So use "ljvalue" instead.
 JNIEXPORT void JNICALL Java_mpi_Info_set(
-        JNIEnv *env, jobject jthis, jlong handle, jstring jkey, jstring jvalue)
+        JNIEnv *env, jobject jthis, jlong handle, jstring jkey, jstring ljvalue)
 {
-    const char *key   = (*env)->GetStringUTFChars(env, jkey,   NULL),
-               *value = (*env)->GetStringUTFChars(env, jvalue, NULL);
+    const char *key   = (*env)->GetStringUTFChars(env, jkey,    NULL),
+               *value = (*env)->GetStringUTFChars(env, ljvalue, NULL);
 
     int rc = MPI_Info_set((MPI_Info)handle, (char*)key, (char*)value);
     ompi_java_exceptionCheck(env, rc);
 
-    (*env)->ReleaseStringUTFChars(env, jkey,   key);
-    (*env)->ReleaseStringUTFChars(env, jvalue, value);
+    (*env)->ReleaseStringUTFChars(env, jkey,    key);
+    (*env)->ReleaseStringUTFChars(env, ljvalue, value);
 }
 
 JNIEXPORT jstring JNICALL Java_mpi_Info_get(
@@ -83,9 +86,12 @@ JNIEXPORT jstring JNICALL Java_mpi_Info_get(
         return NULL;
     }
 
-    jstring jvalue = (*env)->NewStringUTF(env, value);
+    // At least some versions of jni.h have a global named "jvalue",
+    // and we get a compiler warning if we have a parameter or
+    // variable of the same name.  So use "ljvalue" instead.
+    jstring ljvalue = (*env)->NewStringUTF(env, value);
     free(value);
-    return jvalue;
+    return ljvalue;
 }
 
 JNIEXPORT void JNICALL Java_mpi_Info_delete(
diff --git a/ompi/mpi/java/c/mpi_MPI.c b/ompi/mpi/java/c/mpi_MPI.c
index d4c20ae0442..f596411c05d 100644
--- a/ompi/mpi/java/c/mpi_MPI.c
+++ b/ompi/mpi/java/c/mpi_MPI.c
@@ -1271,7 +1271,7 @@ void ompi_java_releasePtrArray(JNIEnv *env, jlongArray array,
  * code execution returns to Java code, an exception is immediately
  * thrown.  Since an exception has occurred somewhere in the C code,
  * the object that is returned from C may not be valid.  This is not
- * an issue, however, as the assignment opperation will not be
+ * an issue, however, as the assignment operation will not be
  * executed.  The results of this method need not be checked if the
  * only following code cleans up memory and then returns to Java.
  * If existing objects are changed after a call to this method, the
diff --git a/ompi/mpi/java/java/CartComm.java b/ompi/mpi/java/java/CartComm.java
index 87df031d151..4c49262e0e8 100644
--- a/ompi/mpi/java/java/CartComm.java
+++ b/ompi/mpi/java/java/CartComm.java
@@ -11,6 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -27,7 +28,7 @@
  *  you may not use this file except in compliance with the License.
  *  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *     https://www.apache.org/licenses/LICENSE-2.0
  *
  *  Unless required by applicable law or agreed to in writing, software
  *  distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/ompi/mpi/java/java/CartParms.java b/ompi/mpi/java/java/CartParms.java
index 2ea7eca7938..aefd2d8fe5b 100644
--- a/ompi/mpi/java/java/CartParms.java
+++ b/ompi/mpi/java/java/CartParms.java
@@ -11,6 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -27,7 +28,7 @@
  *  you may not use this file except in compliance with the License.
  *  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *     https://www.apache.org/licenses/LICENSE-2.0
  *
  *  Unless required by applicable law or agreed to in writing, software
  *  distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/ompi/mpi/java/java/Comm.java b/ompi/mpi/java/java/Comm.java
index 90f8b9102df..7483e8987f3 100644
--- a/ompi/mpi/java/java/Comm.java
+++ b/ompi/mpi/java/java/Comm.java
@@ -14,6 +14,7 @@
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2017-2019 FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -30,7 +31,7 @@
  *  you may not use this file except in compliance with the License.
  *  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *     https://www.apache.org/licenses/LICENSE-2.0
  *
  *  Unless required by applicable law or agreed to in writing, software
  *  distributed under the License is distributed on an "AS IS" BASIS,
@@ -906,7 +907,7 @@ private native long recvInit(
 	 * Java binding of the MPI operation {@code MPI_PACK}.
 	 * <p>
 	 * The return value is the output value of {@code position} - the
-	 * inital value incremented by the number of bytes written.
+	 * initial value incremented by the number of bytes written.
 	 * @param inbuf    input buffer
 	 * @param incount  number of items in input buffer
 	 * @param type     datatype of each item in input buffer
@@ -944,7 +945,7 @@ private native int pack(
 	 * Java binding of the MPI operation {@code MPI_UNPACK}.
 	 * <p>
 	 * The return value is the output value of {@code position} - the
-	 * inital value incremented by the number of bytes read.
+	 * initial value incremented by the number of bytes read.
 	 * @param inbuf    input buffer
 	 * @param position initial position in input buffer
 	 * @param outbuf   output buffer
@@ -1150,7 +1151,7 @@ public final int getTopology() throws MPIException
 
 	private native int getTopology(long comm) throws MPIException;
 
-	// Enviromental Management
+	// Environmental Management
 
 	/**
 	 * Abort MPI.
diff --git a/ompi/mpi/java/java/Count.java b/ompi/mpi/java/java/Count.java
index f0ccc9b552b..7c175c4549f 100644
--- a/ompi/mpi/java/java/Count.java
+++ b/ompi/mpi/java/java/Count.java
@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -17,7 +18,7 @@
  *  you may not use this file except in compliance with the License.
  *  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *     https://www.apache.org/licenses/LICENSE-2.0
  *
  *  Unless required by applicable law or agreed to in writing, software
  *  distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/ompi/mpi/java/java/Datatype.java b/ompi/mpi/java/java/Datatype.java
index 379343fc5ba..4b9e8295f43 100644
--- a/ompi/mpi/java/java/Datatype.java
+++ b/ompi/mpi/java/java/Datatype.java
@@ -12,6 +12,7 @@
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -28,7 +29,7 @@
  *  you may not use this file except in compliance with the License.
  *  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *     https://www.apache.org/licenses/LICENSE-2.0
  *
  *  Unless required by applicable law or agreed to in writing, software
  *  distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/ompi/mpi/java/java/Errhandler.java b/ompi/mpi/java/java/Errhandler.java
index 32e63867d75..cc7fa6e73e1 100644
--- a/ompi/mpi/java/java/Errhandler.java
+++ b/ompi/mpi/java/java/Errhandler.java
@@ -11,6 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -27,7 +28,7 @@
  *  you may not use this file except in compliance with the License.
  *  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *     https://www.apache.org/licenses/LICENSE-2.0
  *
  *  Unless required by applicable law or agreed to in writing, software
  *  distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/ompi/mpi/java/java/Freeable.java b/ompi/mpi/java/java/Freeable.java
index bca1dbb2db8..736a729fee1 100644
--- a/ompi/mpi/java/java/Freeable.java
+++ b/ompi/mpi/java/java/Freeable.java
@@ -11,6 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -27,7 +28,7 @@
  *  you may not use this file except in compliance with the License.
  *  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *     https://www.apache.org/licenses/LICENSE-2.0
  *
  *  Unless required by applicable law or agreed to in writing, software
  *  distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/ompi/mpi/java/java/GraphComm.java b/ompi/mpi/java/java/GraphComm.java
index 29adbee930e..8ee75c2459a 100644
--- a/ompi/mpi/java/java/GraphComm.java
+++ b/ompi/mpi/java/java/GraphComm.java
@@ -12,6 +12,7 @@
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -28,7 +29,7 @@
  *  you may not use this file except in compliance with the License.
  *  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *     https://www.apache.org/licenses/LICENSE-2.0
  *
  *  Unless required by applicable law or agreed to in writing, software
  *  distributed under the License is distributed on an "AS IS" BASIS,
@@ -135,7 +136,7 @@ protected GraphComm(long[] commRequest)
 	 * <p>The number of nodes and number of edges can be extracted
 	 * from the sizes of the {@code index} and {@code edges} fields
 	 * of the returned object.
-	 * @return object defining node degress and edges of graph
+	 * @return object defining node degrees and edges of graph
 	 * @throws MPIException Signals that an MPI error of some sort has occurred.
 	 */
 	public GraphParms getDims() throws MPIException
diff --git a/ompi/mpi/java/java/GraphParms.java b/ompi/mpi/java/java/GraphParms.java
index 357c94f2c0a..49dc5b1a789 100644
--- a/ompi/mpi/java/java/GraphParms.java
+++ b/ompi/mpi/java/java/GraphParms.java
@@ -11,6 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -27,7 +28,7 @@
  *  you may not use this file except in compliance with the License.
  *  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *     https://www.apache.org/licenses/LICENSE-2.0
  *
  *  Unless required by applicable law or agreed to in writing, software
  *  distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/ompi/mpi/java/java/Group.java b/ompi/mpi/java/java/Group.java
index 3af4cec3299..254af2100fb 100644
--- a/ompi/mpi/java/java/Group.java
+++ b/ompi/mpi/java/java/Group.java
@@ -11,6 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -27,7 +28,7 @@
  *  you may not use this file except in compliance with the License.
  *  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *     https://www.apache.org/licenses/LICENSE-2.0
  *
  *  Unless required by applicable law or agreed to in writing, software
  *  distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/ompi/mpi/java/java/Intercomm.java b/ompi/mpi/java/java/Intercomm.java
index 17ee1bf08ed..1cce85e2a80 100644
--- a/ompi/mpi/java/java/Intercomm.java
+++ b/ompi/mpi/java/java/Intercomm.java
@@ -11,6 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -27,7 +28,7 @@
  *  you may not use this file except in compliance with the License.
  *  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *     https://www.apache.org/licenses/LICENSE-2.0
  *
  *  Unless required by applicable law or agreed to in writing, software
  *  distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/ompi/mpi/java/java/Intracomm.java b/ompi/mpi/java/java/Intracomm.java
index b19ca5415f5..95decb25ef0 100644
--- a/ompi/mpi/java/java/Intracomm.java
+++ b/ompi/mpi/java/java/Intracomm.java
@@ -12,6 +12,7 @@
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -28,7 +29,7 @@
  *  you may not use this file except in compliance with the License.
  *  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *     https://www.apache.org/licenses/LICENSE-2.0
  *
  *  Unless required by applicable law or agreed to in writing, software
  *  distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/ompi/mpi/java/java/MPI.java b/ompi/mpi/java/java/MPI.java
index 9b253d633a6..e8887faede0 100644
--- a/ompi/mpi/java/java/MPI.java
+++ b/ompi/mpi/java/java/MPI.java
@@ -12,6 +12,7 @@
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -28,7 +29,7 @@
  *  you may not use this file except in compliance with the License.
  *  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *     https://www.apache.org/licenses/LICENSE-2.0
  *
  *  Unless required by applicable law or agreed to in writing, software
  *  distributed under the License is distributed on an "AS IS" BASIS,
@@ -204,7 +205,15 @@ public final class MPI
 
 	static
 	{
-		System.loadLibrary("mpi_java");
+		try
+		{
+			System.loadLibrary("mpi_java") ;
+		}
+		catch (UnsatisfiedLinkError e)
+		{
+			System.err.println("mpi java lib failed to load: " + e  + "\n") ;
+			System.exit(1) ;
+		}
 
 		DATATYPE_NULL = new Datatype();
 
diff --git a/ompi/mpi/java/java/MPIException.java b/ompi/mpi/java/java/MPIException.java
index 01cba6e90a2..fb6c7744f0c 100644
--- a/ompi/mpi/java/java/MPIException.java
+++ b/ompi/mpi/java/java/MPIException.java
@@ -11,6 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -27,7 +28,7 @@
  *  you may not use this file except in compliance with the License.
  *  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *     https://www.apache.org/licenses/LICENSE-2.0
  *
  *  Unless required by applicable law or agreed to in writing, software
  *  distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/ompi/mpi/java/java/Message.java b/ompi/mpi/java/java/Message.java
index 1414185cc99..a88974a6267 100644
--- a/ompi/mpi/java/java/Message.java
+++ b/ompi/mpi/java/java/Message.java
@@ -111,7 +111,7 @@ private native Status imProbe(int source, int tag, long comm)
 	/**
 	 * Java binding of {@code MPI_MRECV}.
 	 * @param buf   receive buffer
-	 * @param count number of elements in receve buffer
+	 * @param count number of elements in receive buffer
 	 * @param type  datatype of each receive buffer element
 	 * @return status object
 	 * @throws MPIException Signals that an MPI error of some sort has occurred.
@@ -142,7 +142,7 @@ private native long mRecv(
 	/**
 	 * Java binding of {@code MPI_IMRECV}.
 	 * @param buf   receive buffer
-	 * @param count number of elements in receve buffer
+	 * @param count number of elements in receive buffer
 	 * @param type  datatype of each receive buffer element
 	 * @return request object
 	 * @throws MPIException Signals that an MPI error of some sort has occurred.
diff --git a/ompi/mpi/java/java/Op.java b/ompi/mpi/java/java/Op.java
index fb4fc032b69..f7dd25a25b0 100644
--- a/ompi/mpi/java/java/Op.java
+++ b/ompi/mpi/java/java/Op.java
@@ -12,6 +12,7 @@
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -28,7 +29,7 @@
  *  you may not use this file except in compliance with the License.
  *  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *     https://www.apache.org/licenses/LICENSE-2.0
  *
  *  Unless required by applicable law or agreed to in writing, software
  *  distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/ompi/mpi/java/java/Prequest.java b/ompi/mpi/java/java/Prequest.java
index 4cbc9decf01..205e0d489a0 100644
--- a/ompi/mpi/java/java/Prequest.java
+++ b/ompi/mpi/java/java/Prequest.java
@@ -11,6 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -27,7 +28,7 @@
  *  you may not use this file except in compliance with the License.
  *  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *     https://www.apache.org/licenses/LICENSE-2.0
  *
  *  Unless required by applicable law or agreed to in writing, software
  *  distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/ompi/mpi/java/java/Request.java b/ompi/mpi/java/java/Request.java
index 962e12f7912..eb841731af3 100644
--- a/ompi/mpi/java/java/Request.java
+++ b/ompi/mpi/java/java/Request.java
@@ -12,6 +12,7 @@
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -28,7 +29,7 @@
  *  you may not use this file except in compliance with the License.
  *  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *     https://www.apache.org/licenses/LICENSE-2.0
  *
  *  Unless required by applicable law or agreed to in writing, software
  *  distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/ompi/mpi/java/java/ShiftParms.java b/ompi/mpi/java/java/ShiftParms.java
index 37f26572e9e..49ddc72014f 100644
--- a/ompi/mpi/java/java/ShiftParms.java
+++ b/ompi/mpi/java/java/ShiftParms.java
@@ -11,6 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -27,7 +28,7 @@
  *  you may not use this file except in compliance with the License.
  *  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *     https://www.apache.org/licenses/LICENSE-2.0
  *
  *  Unless required by applicable law or agreed to in writing, software
  *  distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/ompi/mpi/java/java/Status.java b/ompi/mpi/java/java/Status.java
index 6506b132413..eaeb467cf58 100644
--- a/ompi/mpi/java/java/Status.java
+++ b/ompi/mpi/java/java/Status.java
@@ -11,6 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -27,7 +28,7 @@
  *  you may not use this file except in compliance with the License.
  *  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *     https://www.apache.org/licenses/LICENSE-2.0
  *
  *  Unless required by applicable law or agreed to in writing, software
  *  distributed under the License is distributed on an "AS IS" BASIS,
@@ -95,7 +96,7 @@ private native int getCount(
 	/**
 	 * Tests if the communication was cancelled.
 	 * <p>Java binding of the MPI operation {@code MPI_TEST_CANCELLED}.
-	 * @return true if the operation was succesfully cancelled, false otherwise
+	 * @return true if the operation was successfully cancelled, false otherwise
 	 * @throws MPIException Signals that an MPI error of some sort has occurred.
 	 */
 	public boolean isCancelled() throws MPIException
diff --git a/ompi/mpi/java/java/UserFunction.java b/ompi/mpi/java/java/UserFunction.java
index 785fabc98a2..2618b8b2040 100644
--- a/ompi/mpi/java/java/UserFunction.java
+++ b/ompi/mpi/java/java/UserFunction.java
@@ -11,6 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -27,7 +28,7 @@
     you may not use this file except in compliance with the License.
     You may obtain a copy of the License at
 
-       http://www.apache.org/licenses/LICENSE-2.0
+       https://www.apache.org/licenses/LICENSE-2.0
 
     Unless required by applicable law or agreed to in writing, software
     distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/ompi/mpi/java/java/Version.java b/ompi/mpi/java/java/Version.java
index e63911210d4..2194ce35a54 100644
--- a/ompi/mpi/java/java/Version.java
+++ b/ompi/mpi/java/java/Version.java
@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -17,7 +18,7 @@
  *  you may not use this file except in compliance with the License.
  *  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *     https://www.apache.org/licenses/LICENSE-2.0
  *
  *  Unless required by applicable law or agreed to in writing, software
  *  distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/ompi/mpi/man/man3/MPI_Abort.3in b/ompi/mpi/man/man3/MPI_Abort.3in
deleted file mode 100644
index 112b3ebb54d..00000000000
--- a/ompi/mpi/man/man3/MPI_Abort.3in
+++ /dev/null
@@ -1,70 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Abort 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Abort\fP \- Terminates MPI execution environment.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Abort(MPI_Comm \fIcomm\fP, int\fI errorcode\fP)
-
-.fi
-.SH Fortran Syntax
-.ft R
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_ABORT(\fICOMM\fP, \fIERRORCODE\fP, \fIIERROR\fP)
-	INTEGER		\fICOMM\fP,\fI ERRORCODE\fP,\fI IERROR
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Abort(\fIcomm\fP, \fIerrorcode\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, INTENT(IN) :: \fIerrorcode\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-comm
-Communicator of tasks to abort.
-.TP 1i
-errorcode
-Error code to return to invoking environment.
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-This routine makes a "best attempt" to abort all tasks in the group of
-comm. This function does not require that the invoking environment take any
-action with the error code. However, a UNIX or POSIX
-environment should handle this as a return errorcode from the main program
-or an abort (errorcode).
-.sp
-The long-term goal of the Open MPI implementation is to terminate all processes in all tasks that contain a process in \fIcomm\fP, and the error code is not returned to the invoking environment. At the moment, this isn't fully implemented and MPI_Abort will terminate the entire job.
-.sp
-Note: All associated processes are sent a SIGTERM.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler
-may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_Accumulate.3in b/ompi/mpi/man/man3/MPI_Accumulate.3in
deleted file mode 100644
index baa1110714b..00000000000
--- a/ompi/mpi/man/man3/MPI_Accumulate.3in
+++ /dev/null
@@ -1,165 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013-2014 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Accumulate 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Accumulate\fP, \fBMPI_Raccumulate\fP \- Combines the contents of the origin buffer with that of a target buffer.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Accumulate(const void *\fIorigin_addr\fP, int \fIorigin_count\fP,
-	MPI_Datatype \fIorigin_datatype\fP, int \fItarget_rank\fP,
-	MPI_Aint \fItarget_disp\fP, int \fItarget_count\fP,
-	MPI_Datatype \fItarget_datatype\fP, MPI_Op \fIop\fP, MPI_Win \fIwin\fP)
-
-int MPI_Raccumulate(const void *\fIorigin_addr\fP, int \fIorigin_count\fP,
-	MPI_Datatype \fIorigin_datatype\fP, int \fItarget_rank\fP,
-	MPI_Aint \fItarget_disp\fP, int \fItarget_count\fP,
-	MPI_Datatype \fItarget_datatype\fP, MPI_Op \fIop\fP, MPI_Win \fIwin\fP,
-	MPI_Request *\fIrequest\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_ACCUMULATE(\fIORIGIN_ADDR, ORIGIN_COUNT, ORIGIN_DATATYPE, TARGET_RANK,
-	TARGET_DISP, TARGET_COUNT, TARGET_DATATYPE, OP, WIN, IERROR\fP)
-	<type> \fIORIGIN_ADDR\fP(*)
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fITARGET_DISP\fP
-	INTEGER \fIORIGIN_COUNT, ORIGIN_DATATYPE, TARGET_RANK, TARGET_COUNT,
-	TARGET_DATATYPE, OP, WIN, IERROR \fP
-
-MPI_RACCUMULATE(\fIORIGIN_ADDR, ORIGIN_COUNT, ORIGIN_DATATYPE, TARGET_RANK,
-	TARGET_DISP, TARGET_COUNT, TARGET_DATATYPE, OP, WIN, REQUEST, IERROR\fP)
-	<type> \fIORIGIN_ADDR\fP(*)
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fITARGET_DISP\fP
-	INTEGER \fIORIGIN_COUNT, ORIGIN_DATATYPE, TARGET_RANK, TARGET_COUNT,
-	TARGET_DATATYPE, OP, WIN, REQUEST, IERROR \fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Accumulate(\fIorigin_addr\fP, \fIorigin_count\fP, \fIorigin_datatype\fP, \fItarget_rank\fP,
-		\fItarget_disp\fP, \fItarget_count\fP, \fItarget_datatype\fP, \fIop\fP, \fIwin\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIorigin_addr\fP
-	INTEGER, INTENT(IN) :: \fIorigin_count\fP, \fItarget_rank\fP, \fItarget_count\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIorigin_datatype\fP, \fItarget_datatype\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) :: \fItarget_disp\fP
-	TYPE(MPI_Op), INTENT(IN) :: \fIop\fP
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPI_Raccumulate(\fIorigin_addr\fP, \fIorigin_count\fP, \fIorigin_datatype\fP, \fItarget_rank\fP,
-	\fItarget_disp\fP, \fItarget_count\fP, \fItarget_datatype\fP, \fIop\fP, \fIwin\fP, \fIrequest,\fP
-		\fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIorigin_addr\fP
-	INTEGER, INTENT(IN) :: \fIorigin_count\fP, \fItarget_rank\fP, \fItarget_count\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIorigin_datatype\fP, \fItarget_datatype\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) :: \fItarget_disp\fP
-	TYPE(MPI_Op), INTENT(IN) :: \fIop\fP
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-origin_addr
-Initial address of buffer (choice).
-.ft R
-.TP 1i
-origin_count
-Number of entries in buffer (nonnegative integer).
-.ft R
-.TP 1i
-origin_datatype
-Data type of each buffer entry (handle).
-.ft R
-.TP 1i
-target_rank
-Rank of target (nonnegative integer).
-.ft R
-.TP 1i
-target_disp
-Displacement from start of window to beginning of target buffer (nonnegative integer).
-.ft R
-.TP 1i
-target_count
-Number of entries in target buffer (nonnegative integer).
-.ft R
-.TP 1i
-target_datatype
-Data type of each entry in target buffer (handle).
-.ft R
-.TP 1i
-op
-Reduce operation (handle).
-.ft R
-.TP 1i
-win
-Window object (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-MPI_Raccumulate: RMA request
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-\fBMPI_Accumulate\fP is a function used for one-sided MPI communication that adds the contents of the origin buffer (as defined by \fIorigin_addr\fP, \fIorigin_count\fP, and \fIorigin_datatype\fP) to the buffer specified by the arguments \fItarget_count\fP and \fItarget_datatype\fP, at offset \fItarget_disp\fP, in the target window specified by \fItarget_rank\fP and \fIwin\fP, using the operation \fIop\fP. The target window can only be accessed by processes within the same node. This is similar to MPI_Put, except that data is combined into the target area instead of overwriting it.
-.sp
-Any of the predefined operations for MPI_Reduce can be used. User-defined functions cannot be used. For example, if \fIop\fP is MPI_SUM, each element of the origin buffer is added to the corresponding element in the target, replacing the former value in the target.
-.sp
-Each datatype argument must be a predefined data type or a derived data type, where all basic components are of the same predefined data type. Both datatype arguments must be constructed from the same predefined data type. The operation \fIop\fP applies to elements of that predefined type. The \fItarget_datatype\fP argument must not specify overlapping entries, and the target buffer must fit in the target window.
-.sp
-A new predefined operation, MPI_REPLACE, is defined. It corresponds to the associative function f(a, b) =b; that is, the current value in the target memory is replaced by the value supplied by the origin.
-.sp
-\fBMPI_Raccumulate\fP is similar to \fBMPI_Accumulate\fP, except that it allocates a communication request object and associates it with the request handle (the argument \fIrequest\fP) that can be used to wait or test for completion. The completion of an \fBMPI_Raccumulate\fP operation indicates that the \fIorigin_addr\fP buffer is free to be updated. It does not indicate that the operation has completed at the target window.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fITARGET_DISP\fP argument only for Fortran 90.  FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_ADDRESS_KIND \fITARGET_DISP\fP
-.fi
-.sp
-where MPI_ADDRESS_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH NOTES
-MPI_Put is a special case of \fBMPI_Accumulate\fP, with the operation MPI_REPLACE. Note, however, that MPI_Put and \fBMPI_Accumulate\fP have different constraints on concurrent updates.
-.sp
-It is the user's responsibility to guarantee that, when
-using the accumulate functions, the target displacement argument is such
-that accesses to the window are properly aligned according to the data
-type arguments in the call to the \fBMPI_Accumulate\fP function.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler
-may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Put
-MPI_Get_accumulate
-MPI_Reduce
diff --git a/ompi/mpi/man/man3/MPI_Add_error_class.3in b/ompi/mpi/man/man3/MPI_Add_error_class.3in
deleted file mode 100644
index 1d150f96de1..00000000000
--- a/ompi/mpi/man/man3/MPI_Add_error_class.3in
+++ /dev/null
@@ -1,91 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Add_error_class 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-
-.SH NAME
-.nf
-\fBMPI_Add_error_class\fP \- Creates a new error class and returns its value
-
-.fi
-.SH SYNTAX
-.ft R
-
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Add_error_class(int *\fIerrorclass\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_ADD_ERROR_CLASS(\fIERRORCLASS, IERROR\fP)
-	INTEGER	\fIERRORCLASS, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Add_error_class(\fIerrorclass\fP, \fIierror\fP)
-	INTEGER, INTENT(OUT) :: \fIerrorclass\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1.4i
-errorclass
-New error class (integer).
-.ft R
-.TP 1.4i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-The function MPI_Add_error_class creates a new, local error
-class.
-
-.SH NOTES
-.ft R
-Because this function is local, the same value of \fIerrorclass\fP may
-not be returned on all processes that make this call, even if they
-call the function concurrently. Thus, same error on different
-processes may not cause the same value of \fIerrorclass\fP to be
-returned. To reduce the potential for confusion, MPI_Add_error_string
-may be used on multiple processes to associate the same error string
-with the newly created \fIerrorclass\fP. Even though \fIerrorclass\fP
-may not be consistent across processes, using MPI_Add_error_string
-will ensure the error string associated with it will be the same
-everywhere.
-.sp
-No function is provided to free error classes, as it is not expected
-that an application will create them in significant numbers.
-.sp
-The value returned is always greater than or equal to MPI_ERR_LASTCODE.
-
-.SH ERRORS
-.ft R
-Almost all MPI routines return an error value; C routines as
-the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Add_error_code
-MPI_Add_error_string
-MPI_Error_class
-MPI_Error_string
-
diff --git a/ompi/mpi/man/man3/MPI_Add_error_code.3in b/ompi/mpi/man/man3/MPI_Add_error_code.3in
deleted file mode 100644
index 6ddbbf671de..00000000000
--- a/ompi/mpi/man/man3/MPI_Add_error_code.3in
+++ /dev/null
@@ -1,84 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Add_error_code 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-
-.SH NAME
-\fBMPI_Add_error_code\fP \- Creates a new error code associated
-with \fIerrorclass\fP
-
-.SH SYNTAX
-.ft R
-
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Add_error_code(int \fIerrorclass\fP, int *\fIerrorcode\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_ADD_ERROR_CODE(\fIERRORCLASS, ERRORCODE, IERROR\fP)
-	INTEGER \fI ERRORCLASS, ERRORCODE, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Add_error_code(\fIerrorclass\fP, \fIerrorcode\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIerrorclass\fP
-	INTEGER, INTENT(OUT) :: \fIerrorcode\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1.4i
-errorclass
-MPI error class (integer).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1.4i
-errorcode
-Error code returned by an MPI routine or an MPI error class (integer).
-.ft R
-.TP 1.4i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-Creates a new error code associated with \fIerrorclass\fP and returns
-its value in \fIerrorcode\fP.
-
-.SH NOTES
-.ft R
-No function is provided to free error codes, as it is not expected
-that an application will create them in significant numbers.
-.sp
-The value returned is always greater than or equal to MPI_ERR_LASTCODE.
-
-.SH ERRORS
-.ft R
-Almost all MPI routines return an error value; C routines as
-the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Add_error_class
-MPI_Error_class
-
-
diff --git a/ompi/mpi/man/man3/MPI_Add_error_string.3in b/ompi/mpi/man/man3/MPI_Add_error_string.3in
deleted file mode 100644
index 9955732a989..00000000000
--- a/ompi/mpi/man/man3/MPI_Add_error_string.3in
+++ /dev/null
@@ -1,88 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Add_error_string 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-
-.SH NAME
-.nf
-\fBMPI_Add_error_string\fP \- Associates a string with an error code or class
-
-.fi
-.SH SYNTAX
-.ft R
-
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Add_error_string(int \fIerrorcode\fP, const char *\fIstring\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_ADD_ERROR_STRING(\fIERRORCODE, STRING, IERROR\fP)
-	INTEGER		\fIERRORCODE, IERROR\fP
-	CHARACTER*(*)	\fISTRING\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Add_error_string(\fIerrorcode\fP, \fIstring\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIerrorcode\fP
-	CHARACTER(LEN=*), INTENT(IN) :: \fIstring\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1.4i
-errorcode
-MPI error class, or an error code returned by an MPI routine (integer).
-.ft R
-.TP 1.4i
-string
-Text that corresponds to the error code or class (string).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1.4i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-This routine associates an error string with an error code or
-class. Calling MPI_Add_error_string for an error code or class that
-already has an associated error string will replace the old string
-with the new one. It is erroneous to call MPI_Add_error_string for an
-error value not generated via MPI_Add_error_class or
-MPI_Add_error_code (e.g., an error code or class with a value not
-greater than MPI_LAST_ERRCODE).
-
-.SH ERRORS
-.ft R
-Almost all MPI routines return an error value; C routines as
-the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Add_error_class
-MPI_Add_error_code
-MPI_Error_class
-MPI_Error_string
-
-
diff --git a/ompi/mpi/man/man3/MPI_Address.3in b/ompi/mpi/man/man3/MPI_Address.3in
deleted file mode 100644
index bd554053902..00000000000
--- a/ompi/mpi/man/man3/MPI_Address.3in
+++ /dev/null
@@ -1,92 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Address 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Address\fP \- Gets the address of a location in memory -- use of this routine is deprecated.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Address(void *\fIlocation\fP, MPI_Aint\fP *address\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-INCLUDE 'mpif.h'
-MPI_ADDRESS(\fILOCATION\fP,\fI ADDRESS\fP,\fI IERROR\fP)
-	<type>	\fILOCATION\fP (*)
-	INTEGER	\fIADDRESS\fP,\fI IERROR\fP
-
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-location
-Location in caller memory (choice).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-address
-Address of location (integer).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Note that use of this routine is \fIdeprecated\fP as of MPI-2. Please use MPI_Get_address instead.
-.sp
-The address of a location in memory can be found by invoking this function. Returns the (byte) address of location.
-.sp
-Example: Using MPI_Address for an array.
-.sp
-.nf
-REAL A(100,100)
-.fi
-.br
-   INTEGER I1, I2, DIFF
-.br
-   CALL MPI_ADDRESS(A(1,1), I1, IERROR)
-.br
-   CALL MPI_ADDRESS(A(10,10), I2, IERROR)
-.br
-   DIFF = I2 - I1
-.br
-! The value of DIFF is 909*sizeofreal; the values of I1 and I2 are
-.br
-! implementation dependent.
-.fi
-
-.SH NOTES
-.ft R
-This routine is provided for both Fortran and C programmers and may be useful when writing portable code. In the current release, the address returned by this routine will be the same as that produced by the C & operator.
-.sp
-C users may be tempted to avoid using
-MPI_Address and rely on the availability of the address operator &.
-Note, however, that & cast-expression is a pointer, not an address.
-ANSI C does not require that the value of a pointer (or the pointer
-cast to int) be the absolute address of the object pointed at although
-this is commonly the case. Furthermore, referencing may not have a unique
-definition on machines with a segmented address space. The use of
-MPI_Address to "reference" C variables guarantees portability to
-such machines as well.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler
-may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-MPI_Get_address
-.br
diff --git a/ompi/mpi/man/man3/MPI_Aint_add.3in b/ompi/mpi/man/man3/MPI_Aint_add.3in
deleted file mode 100644
index 800e2a1b975..00000000000
--- a/ompi/mpi/man/man3/MPI_Aint_add.3in
+++ /dev/null
@@ -1,94 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013-2015 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" $COPYRIGHT$
-.TH MPI_Aint_add 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Aint_add\fP, \fBMPI_Aint_diff\fP \- Portable functions for
-arithmetic on MPI_Aint values.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-MPI_Aint MPI_Aint_add(MPI_Aint \fIbase\fP, MPI_Aint \fIdisp\fP)
-
-MPI_Aint MPI_Aint_diff(MPI_Aint \fIaddr1\fP, MPI_Aint \fIaddr2\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-INTEGER(KIND=MPI_ADDRESS_KIND) MPI_AINT_ADD(\fIBASE, DISP\fP)
-        INTEGER(KIND=MPI_ADDRESS_KIND) \fIBASE, DISP\fP
-
-INTEGER(KIND=MPI_ADDRESS_KIND) MPI_AINT_DIFF(\fIADDR1, ADDR2\fP)
-        INTEGER(KIND=MPI_ADDRESS_KIND) \fIADDR1, ADDR2\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-INTEGER(KIND=MPI_ADDRESS_KIND) MPI_AINT_ADD(\fIBASE, DISP\fP)
-        INTEGER(KIND=MPI_ADDRESS_KIND) \fIBASE, DISP\fP
-
-INTEGER(KIND=MPI_ADDRESS_KIND) MPI_AINT_DIFF(\fIADDR1, ADDR2\fP)
-        INTEGER(KIND=MPI_ADDRESS_KIND) \fIADDR1, ADDR2\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-base
-Base address (integer).
-.ft R
-.TP 1i
-disp
-Displacement (integer).
-.ft R
-.TP 1i
-addr1
-Minuend address (integer).
-.ft R
-.TP
-addr2
-Subtrahend address (integer).
-
-.SH DESCRIPTION
-.ft R
-\fBMPI_Aint_add\fP produces a new MPI_Aint value that is equivalent to the sum of
-the \fIbase\fP and \fIdisp\fP arguments, where \fIbase\fP represents
-a base address returned by a call to \fBMPI_Get_address\fP and
-\fIdisp\fP represents a signed integer displacement. The resulting
-address is valid only at the process that generated \fIbase\fP, and it
-must correspond to a location in the same object referenced by
-\fIbase\fP, as described in MPI-3.1 \[char167] 4.1.12. The addition is
-performed in a manner that results in the correct MPI_Aint
-representation of the output address, as if the process that
-originally produced \fIbase\fP had called:
-
-.nf
-        \fBMPI_Get_address\fP ((char *) \fIbase\fP + \fIdisp\fP, &\fIresult\fP);
-.fi
-.sp
-.ft R
-\fBMPI_Aint_diff\fP produces a new MPI_Aint value that is equivalent
-to the difference between \fIaddr1\fP and \fIaddr2\fP arguments, where
-\fIaddr1\fP and \fIaddr2\fP represent addresses returned by calls to
-\fBMPI_Get_address\fP. The resulting address is valid only at the
-process that generated \fIaddr1\fP and \fIaddr2\fP, and \fIaddr1\fP
-and \fIaddr2\fP must correspond to locations in the same object in the
-same process, as described in MPI-3.1 \[char167] 4.1.12. The difference is
-calculated in a manner that results in the signed difference from
-\fIaddr1\fP to \fIaddr2\fP, as if the process that originally produced
-the addresses had called (char *) \fIaddr1\fP - (char *) \fIaddr2\fP
-on the addresses initially passed to \fBMPI_Get_address\fP.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Get_address
diff --git a/ompi/mpi/man/man3/MPI_Aint_diff.3in b/ompi/mpi/man/man3/MPI_Aint_diff.3in
deleted file mode 100644
index 5fb829b5fb7..00000000000
--- a/ompi/mpi/man/man3/MPI_Aint_diff.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Aint_add.3
diff --git a/ompi/mpi/man/man3/MPI_Allgather.3in b/ompi/mpi/man/man3/MPI_Allgather.3in
deleted file mode 100644
index 5a0bc50375e..00000000000
--- a/ompi/mpi/man/man3/MPI_Allgather.3in
+++ /dev/null
@@ -1,165 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Allgather 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Allgather, MPI_Iallgather\fP \- Gathers data from all processes and distributes it to all processes
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Allgather(const void\fI *sendbuf\fP, int \fI sendcount\fP,
-	 MPI_Datatype\fI sendtype\fP, void\fI *recvbuf\fP, int\fI recvcount\fP,
-	 MPI_Datatype\fI recvtype\fP, MPI_Comm\fI comm\fP)
-
-int MPI_Iallgather(const void\fI *sendbuf\fP, int \fI sendcount\fP,
-	 MPI_Datatype\fI sendtype\fP, void\fI *recvbuf\fP, int\fI recvcount\fP,
-	 MPI_Datatype\fI recvtype\fP, MPI_Comm\fI comm\fP, MPI_Request \fI*request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_ALLGATHER(\fISENDBUF\fP,\fI SENDCOUNT\fP,\fI SENDTYPE\fP,\fI RECVBUF\fP,\fI RECVCOUNT\fP,\fI
-		RECVTYPE\fP,\fI COMM\fP,\fI IERROR\fP)
-	<type>	\fISENDBUF\fP (*), \fIRECVBUF\fP (*)
-	INTEGER	\fISENDCOUNT\fP,\fI SENDTYPE\fP,\fI RECVCOUNT\fP,\fI RECVTYPE\fP,\fI COMM\fP,
-	INTEGER	\fIIERROR\fP
-
-MPI_IALLGATHER(\fISENDBUF, SENDCOUNT, SENDTYPE, RECVBUF, RECVCOUNT,
-		RECVTYPE, COMM, REQUEST, IERROR\fP)
-	<type>	\fISENDBUF\fP(*)\fI, RECVBUF\fP (*)
-	INTEGER	\fISENDCOUNT, SENDTYPE, RECVCOUNT, RECVTYPE, COMM\fP
-	INTEGER	\fIREQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Allgather(\fIsendbuf\fP, \fIsendcount\fP, \fIsendtype\fP, \fIrecvbuf\fP, \fIrecvcount\fP, \fIrecvtype\fP,
-		\fIcomm\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..) :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIsendcount\fP, \fIrecvcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPI_Iallgather(\fIsendbuf\fP, \fIsendcount\fP, \fIsendtype\fP, \fIrecvbuf\fP, \fIrecvcount\fP, \fIrecvtype\fP,
-		\fIcomm\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIsendcount\fP, \fIrecvcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-sendbuf
-Starting address of send buffer (choice).
-.TP 1i
-sendcount
-Number of elements in send buffer (integer).
-.TP 1i
-sendtype
-Datatype of send buffer elements (handle).
-.TP 1i
-recvbuf
-Starting address of recv buffer (choice).
-.TP 1i
-recvcount
-Number of elements received from any process (integer).
-.TP 1i
-recvtype
-Datatype of receive buffer elements (handle).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-recvbuf
-Address of receive buffer (choice).
-.ft R
-.TP 1i
-request
-Request (handle, non-blocking only).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Allgather is similar to MPI_Gather, except that all processes receive the result, instead of just the root. In other words, all processes contribute to the result, and all processes receive the result.
-.sp
-The type signature associated with sendcount, sendtype at a process must be equal to the type signature associated with recvcount, recvtype at any other process.
-.sp
-The outcome of a call to MPI_Allgather(\&...) is as if all processes executed n calls to
-.sp
-.nf
-  MPI_Gather(sendbuf,sendcount,sendtype,recvbuf,recvcount,
-             recvtype,root,comm),
-.fi
-.sp
-.fi
-for root = 0 , ..., n-1. The rules for correct usage of MPI_Allgather are easily found from the corresponding rules for MPI_Gather.
-.sp
-\fBExample:\fR The all-gather version of Example 1 in MPI_Gather. Using  MPI_Allgather, we will gather 100 ints from every process in the group to every process.
-.sp
-.nf
-MPI_Comm comm;
-    int gsize,sendarray[100];
-    int *rbuf;
-    \&...
-    MPI_Comm_size( comm, &gsize);
-    rbuf = (int *)malloc(gsize*100*sizeof(int));
-    MPI_Allgather( sendarray, 100, MPI_INT, rbuf, 100, MPI_INT, comm);
-.fi
-.sp
-After the call, every process has the group-wide concatenation of the sets of data.
-
-.SH USE OF IN-PLACE OPTION
-When the communicator is an intracommunicator, you can perform an all-gather operation in-place (the output buffer is used as the input buffer).  Use the variable MPI_IN_PLACE as the value of \fIsendbuf\fR.  In this case, \fIsendcount\fR and \fIsendtype\fR are ignored.  The input data of each process is assumed to be in the area where that process would receive its own contribution to the receive buffer.  Specifically, the outcome of a call to MPI_Allgather that used the in-place option is identical to the case in which all processes executed \fIn\fR calls to
-.sp
-.nf
-   MPI_ALLGATHER ( MPI_IN_PLACE, 0, MPI_DATATYPE_NULL, recvbuf,
-   recvcount, recvtype, root, comm )
-
-for root =0, ... , n-1.
-.fi
-.sp
-Note that MPI_IN_PLACE is a special kind of value; it has the same restrictions on its use as MPI_BOTTOM.
-.sp
-Because the in-place option converts the receive buffer into a send-and-receive buffer, a Fortran binding that includes INTENT must mark these as INOUT, not OUT.
-.sp
-.SH WHEN COMMUNICATOR IS AN INTER-COMMUNICATOR
-.sp
-When the communicator is an inter-communicator, the gather operation occurs in two phases.  The data is gathered from all the members of the first group and received by all the members of the second group.  Then the data is gathered from all the members of the second group and received by all the members of the first.  The operation, however, need not be symmetric.  The number of items sent by the processes in first group need not be equal to the number of items sent by the the processes in the second group.  You can move data in only one direction by giving \fIsendcount\fR a value of 0 for communication in the reverse direction.
-.sp
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler
-may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Allgatherv
-.br
-MPI_Gather
-
diff --git a/ompi/mpi/man/man3/MPI_Allgatherv.3in b/ompi/mpi/man/man3/MPI_Allgatherv.3in
deleted file mode 100644
index 8ebf6fae27b..00000000000
--- a/ompi/mpi/man/man3/MPI_Allgatherv.3in
+++ /dev/null
@@ -1,151 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2007-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Allgatherv 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Allgatherv, MPI_Iallgatherv\fP \- Gathers data from all processes and delivers it to all. Each process may contribute a different amount of data.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Allgatherv(const void\fI *sendbuf\fP, int\fI sendcount\fP,
-	MPI_Datatype\fI sendtype\fP, void\fI *recvbuf\fP, const int\fI recvcounts[]\fP,
-	const int\fI displs[]\fP, MPI_Datatype\fI recvtype\fP, MPI_Comm\fI comm\fP)
-
-int MPI_Iallgatherv(const void\fI *sendbuf\fP, int\fI sendcount\fP,
-	MPI_Datatype\fI sendtype\fP, void\fI *recvbuf\fP, const int\fI recvcounts[]\fP,
-	const int\fI displs[]\fP, MPI_Datatype\fI recvtype\fP, MPI_Comm\fI comm\fP,
-        MPI_Request \fI*request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_ALLGATHERV(\fISENDBUF\fP,\fI SENDCOUNT\fP, \fISENDTYPE\fP,\fI RECVBUF\fP,
-		\fIRECVCOUNT\fP,\fI DISPLS\fP, \fIRECVTYPE\fP,\fI COMM\fP,\fI IERROR\fP)
-	<type>	\fISENDBUF\fP(*), \fIRECVBUF\fP(*)
-	INTEGER	\fISENDCOUNT\fP,\fI SENDTYPE\fP, \fIRECVCOUNT\fP(*)
-	INTEGER	\fIDISPLS\fP(*),\fI RECVTYPE\fP,\fI COMM\fP,\fI IERROR\fP
-
-MPI_IALLGATHERV(\fISENDBUF, SENDCOUNT, SENDTYPE, RECVBUF,
-		RECVCOUNT, DISPLS, RECVTYPE, COMM, REQUEST, IERROR\fP)
-	<type>	\fISENDBUF\fP(*)\fI, RECVBUF\fP(*)
-	INTEGER	\fISENDCOUNT, SENDTYPE, RECVCOUNT\fP(*),
-	INTEGER	\fIDISPLS\fP(*)\fI, RECVTYPE, COMM, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Allgatherv(\fIsendbuf\fP, \fIsendcount\fP, \fIsendtype\fP, \fIrecvbuf\fP, \fIrecvcounts\fP, \fIdispls\fP,
-		\fIrecvtype\fP, \fIcomm\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..) :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIsendcount\fP, \fIrecvcounts(*)\fP, \fIdispls(*)\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPI_Iallgatherv(\fIsendbuf\fP, \fIsendcount\fP, \fIsendtype\fP, \fIrecvbuf\fP, \fIrecvcounts\fP, \fIdispls\fP,
-		\fIrecvtype\fP, \fIcomm\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIsendcount\fP
-	INTEGER, INTENT(IN), ASYNCHRONOUS :: \fIrecvcounts(*)\fP, \fIdispls(*)\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-sendbuf
-Starting address of send buffer (choice).
-.TP 1i
-sendcount
-Number of elements in send buffer (integer).
-.TP 1i
-sendtype
-Datatype of send buffer elements (handle).
-.TP 1i
-recvcount
-Integer array (of length group size) containing the number of elements that are received from each process.
-.TP 1i
-displs
-Integer array (of length group size). Entry i specifies the displacement (relative to recvbuf) at which to place the incoming data from process i.
-.TP 1i
-recvtype
-Datatype of receive buffer elements (handle).
-.TP 1i
-comm
-Communicator (handle).
-.sp
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-recvbuf
-Address of receive buffer (choice).
-.TP 1i
-request
-Request (handle, non-blocking only).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Allgatherv is similar to MPI_Allgather in that all processes gather data from all other processes, except that each process can send a different amount of data. The block of data sent from the jth process is received by every process and placed in the jth block of the buffer
-.I recvbuf.
-.sp
-The type signature associated with sendcount, sendtype, at process j must be equal to the type signature associated with recvcounts[j], recvtype at any other process.
-.sp
-The outcome is as if all processes executed calls to
-.nf
-MPI_Allgatherv(sendbuf,sendcount,sendtype,recvbuf,recvcount,
-            displs,recvtype,root,comm)
-.fi
-.sp
-for root = 0 , ..., n-1. The rules for correct usage of MPI_Allgatherv are easily found from the corresponding rules for MPI_Gatherv.
-
-.SH USE OF IN-PLACE OPTION
-When the communicator is an intracommunicator, you can perform an all-gather operation in-place (the output buffer is used as the input buffer).  Use the variable MPI_IN_PLACE as the value of \fIsendbuf\fR.  In this case, \fIsendcount\fR and \fIsendtype\fR are ignored.  The input data of each process is assumed to be in the area where that process would receive its own contribution to the receive buffer.  Specifically, the outcome of a call to MPI_Allgather that used the in-place option is identical to the case in which all processes executed \fIn\fR calls to
-.sp
-.nf
-   MPI_ALLGATHERV ( MPI_IN_PLACE, 0, MPI_DATATYPE_NULL, recvbuf,
-   recvcounts, displs, recvtype, root, comm )
-
-for root =0, ... , n-1.
-.fi
-.sp
-Note that MPI_IN_PLACE is a special kind of value; it has the same restrictions on its use as MPI_BOTTOM.
-.sp
-Because the in-place option converts the receive buffer into a send-and-receive buffer, a Fortran binding that includes INTENT must mark these as INOUT, not OUT.
-.sp
-.SH WHEN COMMUNICATOR IS AN INTER-COMMUNICATOR
-.sp
-When the communicator is an inter-communicator, the gather operation occurs in two phases.  The data is gathered from all the members of the first group, concatenated, and received by all the members of the second group.  Then the data is gathered from all the members of the second group, concatenated, and received by all the members of the first.  The send buffer arguments in the one group must be consistent with the receive buffer arguments in the other group, and vice versa.  The operation must exhibit symmetric, full-duplex behavior.
-.sp
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler
-may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Gatherv
-.br
-MPI_Allgather
diff --git a/ompi/mpi/man/man3/MPI_Alloc_mem.3in b/ompi/mpi/man/man3/MPI_Alloc_mem.3in
deleted file mode 100644
index 6f2daa1719e..00000000000
--- a/ompi/mpi/man/man3/MPI_Alloc_mem.3in
+++ /dev/null
@@ -1,108 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Alloc_mem 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Alloc_mem \fP \- Allocates a specified memory segment.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Alloc_mem(MPI_Aint \fIsize\fP, MPI_Info \fIinfo\fP, void *\fIbaseptr\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_ALLOC_MEM(\fISIZE, INFO, BASEPTR, IERROR\fP)
-	INTEGER \fIINFO, IERROR\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fISIZE, BASEPTR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Alloc_mem(\fIsize\fP, \fIinfo\fP, \fIbaseptr\fP, \fIierror\fP)
-	USE, INTRINSIC :: \fIISO_C_BINDING\fP, \fIONLY \fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) :: \fIsize\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(C_PTR), INTENT(OUT) :: \fIbaseptr\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-size
-Size of memory segment in bytes (nonnegative integer).
-.ft R
-.TP 1i
-info
-Info argument (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-baseptr
-Pointer to beginning of memory segment allocated.
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Alloc_mem allocates \fIsize\fP bytes of memory. The starting address
-of this memory is returned in the variable \fIbase\fP.
-.sp
-
-.SH FORTRAN NOTES
-.ft R
-There is no portable FORTRAN 77 syntax for using MPI_Alloc_mem.
-There is no portable Fortran syntax for using pointers returned
-from MPI_Alloc_mem. However, MPI_Alloc_mem can be used with Sun
-Fortran compilers.
-.sp
-From FORTRAN 77, you can use the following non-standard
-declarations for the SIZE and BASEPTR arguments:
-.nf
-           INCLUDE "mpif.h"
-           INTEGER*MPI_ADDRESS_KIND SIZE, BASEPTR
-.fi
-.sp
-From either FORTRAN 77 or Fortran 90, you can use "Cray pointers"
-for the BASEPTR argument. Cray pointers are described further in
-the Fortran User's Guide and are supported by many Fortran compilers.
-For example,
-.sp
-.nf
-           INCLUDE "mpif.h"
-           REAL*4 A(100,100)
-           POINTER (BASEPTR, A)
-           INTEGER*MPI_ADDRESS_KIND SIZE
-
-           SIZE = 4 * 100 * 100
-           CALL MPI_ALLOC_MEM(SIZE,MPI_INFO_NULL,BASEPTR,IERR)
-
-           ! use A
-
-           CALL MPI_FREE_MEM(A, IERR)
-.fi
-.ft R
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler
-may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Free_mem
diff --git a/ompi/mpi/man/man3/MPI_Allreduce.3in b/ompi/mpi/man/man3/MPI_Allreduce.3in
deleted file mode 100644
index 380b6f4c26e..00000000000
--- a/ompi/mpi/man/man3/MPI_Allreduce.3in
+++ /dev/null
@@ -1,176 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2007-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Allreduce 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Allreduce, MPI_Iallreduce\fP \- Combines values from all processes and distributes the result back to all processes.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Allreduce(const void \fI*sendbuf\fP, void \fI*recvbuf\fP, int\fI count\fP,
-                  MPI_Datatype\fI datatype\fP, MPI_Op\fI op\fP, MPI_Comm\fI comm\fP)
-
-int MPI_Iallreduce(const void \fI*sendbuf\fP, void \fI*recvbuf\fP, int\fI count\fP,
-                   MPI_Datatype\fI datatype\fP, MPI_Op\fI op\fP, MPI_Comm\fI comm\fP,
-                   MPI_Request \fI*request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_ALLREDUCE(\fISENDBUF\fP,\fI RECVBUF\fP, \fICOUNT\fP,\fI DATATYPE\fP,\fI OP\fP, \fICOMM\fP, \fIIERROR\fP)
-	<type>	\fISENDBUF\fP(*), \fIRECVBUF\fP(*)
-	INTEGER	\fICOUNT\fP,\fI DATATYPE\fP,\fI OP\fP,\fI COMM\fP,\fI IERROR\fP
-
-MPI_IALLREDUCE(\fISENDBUF, RECVBUF, COUNT, DATATYPE, OP, COMM, REQUEST, IERROR\fP)
-	<type>	\fISENDBUF\fP(*)\fI, RECVBUF\fP(*)
-	INTEGER	\fICOUNT, DATATYPE, OP, COMM, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Allreduce(\fIsendbuf\fP, \fIrecvbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIop\fP, \fIcomm\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..) :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Op), INTENT(IN) :: \fIop\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPI_Iallreduce(\fIsendbuf\fP, \fIrecvbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIop\fP, \fIcomm\fP, \fIrequest\fP,
-		\fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Op), INTENT(IN) :: \fIop\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-sendbuf
-Starting address of send buffer (choice).
-.TP 1i
-count
-Number of elements in send buffer (integer).
-.TP 1i
-datatype
-Datatype of elements of send buffer (handle).
-.TP 1i
-op
-Operation (handle).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-recvbuf
-Starting address of receive buffer (choice).
-.TP 1i
-request
-Request (handle, non-blocking only).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Same as MPI_Reduce except that the result appears in the receive buffer of all the group members.
-.sp
-\fBExample 1:\fR A routine that computes the product of a vector and an array that are distributed across a group of processes and returns the answer at all nodes (compare with Example 2, with MPI_Reduce, below).
-.sp
-.nf
-SUBROUTINE PAR_BLAS2(m, n, a, b, c, comm)
-REAL a(m), b(m,n)    ! local slice of array
-REAL c(n)            ! result
-REAL sum(n)
-INTEGER n, comm, i, j, ierr
-
-! local sum
-DO j= 1, n
-  sum(j) = 0.0
-  DO i = 1, m
-    sum(j) = sum(j) + a(i)*b(i,j)
-  END DO
-END DO
-
-! global sum
-CALL MPI_ALLREDUCE(sum, c, n, MPI_REAL, MPI_SUM, comm, ierr)
-
-! return result at all nodes
-RETURN
-.fi
-.sp
-\fBExample 2:\fR A routine that computes the product of a vector and an array that are distributed across a group of processes and returns the answer at node zero.
-.sp
-.nf
-SUBROUTINE PAR_BLAS2(m, n, a, b, c, comm)
-REAL a(m), b(m,n)    ! local slice of array
-REAL c(n)            ! result
-REAL sum(n)
-INTEGER n, comm, i, j, ierr
-
-! local sum
-DO j= 1, n
-  sum(j) = 0.0
-  DO i = 1, m
-    sum(j) = sum(j) + a(i)*b(i,j)
-  END DO
-END DO
-
-! global sum
-CALL MPI_REDUCE(sum, c, n, MPI_REAL, MPI_SUM, 0, comm, ierr)
-
-! return result at node zero (and garbage at the other nodes)
-RETURN
-.fi
-.SH USE OF IN-PLACE OPTION
-When the communicator is an intracommunicator, you can perform an all-reduce operation in-place (the output buffer is used as the input buffer).  Use the variable MPI_IN_PLACE as the value of \fIsendbuf\fR at all processes.
-.sp
-Note that MPI_IN_PLACE is a special kind of value; it has the same restrictions on its use as MPI_BOTTOM.
-.sp
-Because the in-place option converts the receive buffer into a send-and-receive buffer, a Fortran binding that includes INTENT must mark these as INOUT, not OUT.
-.sp
-.SH WHEN COMMUNICATOR IS AN INTER-COMMUNICATOR
-When the communicator is an inter-communicator, the reduce operation occurs in two phases.  The data is reduced from all the members of the first group and received by all the members of the second group.  Then the data is reduced from all the members of the second group and received by all the members of the first.  The operation exhibits a symmetric, full-duplex behavior.
-.sp
-When the communicator is an intra-communicator, these groups are the same, and the operation occurs in a single phase.
-.SH NOTES ON COLLECTIVE OPERATIONS
-
-The reduction functions (
-.I MPI_Op
-) do not return an error value.  As a result,
-if the functions detect an error, all they can do is either call
-.I MPI_Abort
-or silently skip the problem.  Thus, if you change the error handler from
-.I MPI_ERRORS_ARE_FATAL
-to something else, for example,
-.I MPI_ERRORS_RETURN
-,
-then no error may be indicated.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler
-may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-
diff --git a/ompi/mpi/man/man3/MPI_Alltoall.3in b/ompi/mpi/man/man3/MPI_Alltoall.3in
deleted file mode 100644
index 2a9babbd99b..00000000000
--- a/ompi/mpi/man/man3/MPI_Alltoall.3in
+++ /dev/null
@@ -1,169 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Alltoall 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-
-.SH NAME
-\fBMPI_Alltoall, MPI_Ialltoall\fP \- All processes send data to all processes
-
-.SH SYNTAX
-.ft R
-
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Alltoall(const void *\fIsendbuf\fP, int \fIsendcount\fP,
-	MPI_Datatype \fIsendtype\fP, void *\fIrecvbuf\fP, int \fIrecvcount\fP,
-	MPI_Datatype \fIrecvtype\fP, MPI_Comm \fIcomm\fP)
-
-int MPI_Ialltoall(const void *\fIsendbuf\fP, int \fIsendcount\fP,
-	MPI_Datatype \fIsendtype\fP, void *\fIrecvbuf\fP, int \fIrecvcount\fP,
-	MPI_Datatype \fIrecvtype\fP, MPI_Comm \fIcomm\fP, MPI_Request \fI*request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_ALLTOALL(\fISENDBUF, SENDCOUNT, SENDTYPE, RECVBUF, RECVCOUNT,
-	RECVTYPE, COMM, IERROR\fP)
-
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fISENDCOUNT, SENDTYPE, RECVCOUNT, RECVTYPE\fP
-	INTEGER	\fICOMM, IERROR\fP
-
-MPI_IALLTOALL(\fISENDBUF, SENDCOUNT, SENDTYPE, RECVBUF, RECVCOUNT,
-	RECVTYPE, COMM, REQUEST, IERROR\fP)
-
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fISENDCOUNT, SENDTYPE, RECVCOUNT, RECVTYPE\fP
-	INTEGER	\fICOMM, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Alltoall(\fIsendbuf\fP, \fIsendcount\fP, \fIsendtype\fP, \fIrecvbuf\fP, \fIrecvcount\fP, \fIrecvtype\fP,
-		\fIcomm\fP, \fIierror\fP)
-
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..) :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIsendcount\fP, \fIrecvcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPI_Ialltoall(\fIsendbuf\fP, \fIsendcount\fP, \fIsendtype\fP, \fIrecvbuf\fP, \fIrecvcount\fP, \fIrecvtype\fP,
-		\fIcomm\fP, \fIrequest\fP, \fIierror\fP)
-
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIsendcount\fP, \fIrecvcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1.2i
-sendbuf
-Starting address of send buffer (choice).
-.TP 1.2i
-sendcount
-Number of elements to send to each process (integer).
-.TP 1.2i
-sendtype
-Datatype of send buffer elements (handle).
-.TP 1.2i
-recvcount
-Number of elements to receive from each process (integer).
-.TP 1.2i
-recvtype
-Datatype of receive buffer elements (handle).
-.TP 1.2i
-comm
-Communicator over which data is to be exchanged (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1.2i
-recvbuf
-Starting address of receive buffer (choice).
-.TP 1.2i
-request
-Request (handle, non-blocking only).
-.ft R
-.TP 1.2i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Alltoall is a collective operation in which all processes send the same amount of data to each other, and receive the same amount of data from each other. The operation of this routine can be represented as follows, where each process performs 2n (n being the number of processes in communicator \fIcomm\fP) independent point-to-point communications (including communication with itself).
-.sp
-.nf
-	MPI_Comm_size(\fIcomm\fP, &n);
-	for (i = 0, i < n; i++)
-	    MPI_Send(\fIsendbuf\fP + i * \fIsendcount\fP * extent(\fIsendtype\fP),
-	        \fIsendcount\fP, \fIsendtype\fP, i, ..., \fIcomm\fP);
-	for (i = 0, i < n; i++)
-	    MPI_Recv(\fIrecvbuf\fP + i * \fIrecvcount\fP * extent(\fIrecvtype\fP),
-	        \fIrecvcount\fP, \fIrecvtype\fP, i, ..., \fIcomm\fP);
-.fi
-.sp
-Each process breaks up its local \fIsendbuf\fP into n blocks \- each
-containing \fIsendcount\fP elements of type \fIsendtype\fP \- and
-divides its \fIrecvbuf\fP similarly according to \fIrecvcount\fP and
-\fIrecvtype\fP. Process j sends the k-th block of its local
-\fIsendbuf\fP to process k, which places the data in the j-th block of
-its local \fIrecvbuf\fP. The amount of data sent must be equal to the
-amount of data received, pairwise, between every pair of processes.
-
-WHEN COMMUNICATOR IS AN INTER-COMMUNICATOR
-.sp
-When the communicator is an inter-communicator, the gather operation occurs in two phases.  The data is gathered from all the members of the first group and received by all the members of the second group.  Then the data is gathered from all the members of the second group and received by all the members of the first.  The operation exhibits a symmetric, full-duplex behavior.
-.sp
-The first group defines the root process.  The root process uses MPI_ROOT as the value of \fIroot\fR.  All other processes in the first group use MPI_PROC_NULL as the value of \fIroot\fR.  All processes in the second group use the rank of the root process in the first group as the value of \fIroot\fR.
-.sp
-When the communicator is an intra-communicator, these groups are the same, and the operation occurs in a single phase.
-
-.SH USE OF IN-PLACE OPTION
-When the communicator is an intracommunicator, you can perform an all-to-all operation in-place (the output buffer is used as the input buffer).  Use the variable MPI_IN_PLACE as the value of \fIsendbuf\fR.  In this case, \fIsendcount\fR and \fIsendtype\fR are ignored.  The input data of each process is assumed to be in the area where that process would receive its own contribution to the receive buffer.
-
-.sp
-.SH NOTES
-.sp
-All arguments on all processes are significant. The \fIcomm\fP argument,
-in particular, must describe the same communicator on all processes.
-.sp
-There are two MPI library functions that are more general than
-MPI_Alltoall. MPI_Alltoallv allows all-to-all communication to and
-from buffers that need not be contiguous; different processes may
-send and receive different amounts of data. MPI_Alltoallw expands
-MPI_Alltoallv's functionality to allow the exchange of data with
-different datatypes.
-
-.SH ERRORS
-.ft R
-Almost all MPI routines return an error value; C routines as
-the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Alltoallv
-MPI_Alltoallw
-
diff --git a/ompi/mpi/man/man3/MPI_Alltoallv.3in b/ompi/mpi/man/man3/MPI_Alltoallv.3in
deleted file mode 100644
index c3d70d71188..00000000000
--- a/ompi/mpi/man/man3/MPI_Alltoallv.3in
+++ /dev/null
@@ -1,200 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Alltoallv 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-
-.SH NAME
-\fBMPI_Alltoallv, MPI_Ialltoallv\fP \- All processes send different amount of data to, and receive different amount of data from, all processes
-.SH SYNTAX
-.ft R
-
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Alltoallv(const void *\fIsendbuf\fP, const int \fIsendcounts\fP[],
-	const int \fIsdispls\fP[], MPI_Datatype \fIsendtype\fP,
-	void *\fIrecvbuf\fP, const int\fI recvcounts\fP[],
-	const int \fIrdispls\fP[], MPI_Datatype \fIrecvtype\fP, MPI_Comm \fIcomm\fP)
-
-int MPI_Ialltoallv(const void *\fIsendbuf\fP, const int \fIsendcounts\fP[],
-	const int \fIsdispls\fP[], MPI_Datatype \fIsendtype\fP,
-	void *\fIrecvbuf\fP, const int\fI recvcounts\fP[],
-	const int \fIrdispls\fP[], MPI_Datatype \fIrecvtype\fP, MPI_Comm \fIcomm\fP,
-	MPI_Request \fI*request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_ALLTOALLV(\fISENDBUF, SENDCOUNTS, SDISPLS, SENDTYPE,
-	RECVBUF, RECVCOUNTS, RDISPLS, RECVTYPE, COMM, IERROR\fP)
-
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fISENDCOUNTS(*), SDISPLS(*), SENDTYPE\fP
-	INTEGER	\fIRECVCOUNTS(*), RDISPLS(*), RECVTYPE\fP
-	INTEGER	\fICOMM, IERROR\fP
-
-MPI_IALLTOALLV(\fISENDBUF, SENDCOUNTS, SDISPLS, SENDTYPE,
-	RECVBUF, RECVCOUNTS, RDISPLS, RECVTYPE, COMM, REQUEST, IERROR\fP)
-
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fISENDCOUNTS(*), SDISPLS(*), SENDTYPE\fP
-	INTEGER	\fIRECVCOUNTS(*), RDISPLS(*), RECVTYPE\fP
-	INTEGER	\fICOMM, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Alltoallv(\fIsendbuf\fP, \fIsendcounts\fP, \fIsdispls\fP, \fIsendtype\fP, \fIrecvbuf\fP, \fIrecvcounts\fP,
-		\fIrdispls\fP, \fIrecvtype\fP, \fIcomm\fP, \fIierror\fP)
-
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..) :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIsendcounts(*)\fP, \fIsdispls(*)\fP, \fIrecvcounts(*),\fP
-	\fIrdispls(*)\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPI_Ialltoallv(\fIsendbuf\fP, \fIsendcounts\fP, \fIsdispls\fP, \fIsendtype\fP, \fIrecvbuf\fP, \fIrecvcounts\fP,
-		\fIrdispls\fP, \fIrecvtype\fP, \fIcomm\fP, \fIrequest\fP, \fIierror\fP)
-
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN), ASYNCHRONOUS :: \fIsendcounts(*)\fP, \fIsdispls(*),\fP
-	\fIrecvcounts(*)\fP, \fIrdispls(*)\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1.2i
-sendbuf
-Starting address of send buffer.
-.TP 1.2i
-sendcounts
-Integer array, where entry i specifies the number of elements to send
-to rank i.
-.TP 1.2i
-sdispls
-Integer array, where entry i specifies the displacement (offset from
-\fIsendbuf\fP, in units of \fIsendtype\fP) from which to send data to
-rank i.
-.TP 1.2i
-sendtype
-Datatype of send buffer elements.
-.TP 1.2i
-recvcounts
-Integer array, where entry j specifies the number of elements to
-receive from rank j.
-.TP 1.2i
-rdispls
-Integer array, where entry j specifies the displacement (offset from
-\fIrecvbuf\fP, in units of \fIrecvtype\fP) to which data from rank j
-should be written.
-.TP 1.2i
-recvtype
-Datatype of receive buffer elements.
-.TP 1.2i
-comm
-Communicator over which data is to be exchanged.
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1.2i
-recvbuf
-Address of receive buffer.
-.TP 1.2i
-request
-Request (handle, non-blocking only).
-.ft R
-.TP 1.2i
-IERROR
-Fortran only: Error status.
-
-.SH DESCRIPTION
-.ft R
-MPI_Alltoallv is a generalized collective operation in which all
-processes send data to and receive data from all other processes. It
-adds flexibility to MPI_Alltoall by allowing the user to specify data
-to send and receive vector-style (via a displacement and element
-count). The operation of this routine can be thought of as follows,
-where each process performs 2n (n being the number of processes in
-communicator \fIcomm\fP) independent point-to-point communications
-(including communication with itself).
-.sp
-.nf
-	MPI_Comm_size(\fIcomm\fP, &n);
-	for (i = 0, i < n; i++)
-	    MPI_Send(\fIsendbuf\fP + \fIsdispls\fP[i] * extent(\fIsendtype\fP),
-	        \fIsendcounts\fP[i], \fIsendtype\fP, i, ..., \fIcomm\fP);
-	for (i = 0, i < n; i++)
-	    MPI_Recv(\fIrecvbuf\fP + \fIrdispls\fP[i] * extent(\fIrecvtype\fP),
-	        \fIrecvcounts\fP[i], \fIrecvtype\fP, i, ..., \fIcomm\fP);
-.fi
-.sp
-Process j sends the k-th block of its local \fIsendbuf\fP to process
-k, which places the data in the j-th block of its local
-\fIrecvbuf\fP.
-.sp
-When a pair of processes exchanges data, each may pass different
-element count and datatype arguments so long as the sender specifies
-the same amount of data to send (in bytes) as the receiver expects
-to receive.
-.sp
-Note that process i may send a different amount of data to process j
-than it receives from process j. Also, a process may send entirely
-different amounts of data to different processes in the communicator.
-
-.sp
-WHEN COMMUNICATOR IS AN INTER-COMMUNICATOR
-.sp
-When the communicator is an inter-communicator, the gather operation occurs in two phases.  The data is gathered from all the members of the first group and received by all the members of the second group.  Then the data is gathered from all the members of the second group and received by all the members of the first.  The operation exhibits a symmetric, full-duplex behavior.
-.sp
-The first group defines the root process.  The root process uses MPI_ROOT as the value of \fIroot\fR.  All other processes in the first group use MPI_PROC_NULL as the value of \fIroot\fR.  All processes in the second group use the rank of the root process in the first group as the value of \fIroot\fR.
-.sp
-When the communicator is an intra-communicator, these groups are the same, and the operation occurs in a single phase.
-.sp
-
-.SH USE OF IN-PLACE OPTION
-When the communicator is an intracommunicator, you can perform an all-to-all operation in-place (the output buffer is used as the input buffer).  Use the variable MPI_IN_PLACE as the value of \fIsendbuf\fR.  In this case, \fIsendcounts\fR, \fIsdispls\fP, and \fIsendtype\fR are ignored.  The input data of each process is assumed to be in the area where that process would receive its own contribution to the receive buffer.
-
-.SH NOTES
-.sp
-The specification of counts and displacements should not cause
-any location to be written more than once.
-.sp
-All arguments on all processes are significant. The \fIcomm\fP argument,
-in particular, must describe the same communicator on all processes.
-.sp
-The offsets of \fIsdispls\fP and \fIrdispls\fP are measured in units
-of \fIsendtype\fP and \fIrecvtype\fP, respectively. Compare this to
-MPI_Alltoallw, where these offsets are measured in bytes.
-
-.SH ERRORS
-.ft R
-Almost all MPI routines return an error value; C routines as
-the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Alltoall
-MPI_Alltoallw
-
diff --git a/ompi/mpi/man/man3/MPI_Alltoallw.3in b/ompi/mpi/man/man3/MPI_Alltoallw.3in
deleted file mode 100644
index 73a16c73041..00000000000
--- a/ompi/mpi/man/man3/MPI_Alltoallw.3in
+++ /dev/null
@@ -1,204 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Alltoallw 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-
-.SH NAME
-\fBMPI_Alltoallw, MPI_Ialltoallw\fP \- All processes send data of different types to, and receive data of different types from, all processes
-
-.SH SYNTAX
-.ft R
-
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Alltoallw(const void *\fIsendbuf\fP, const int \fIsendcounts\fP[],
-	const int \fIsdispls\fP[], const MPI_Datatype \fIsendtypes\fP[],
-	void *\fIrecvbuf\fP, const int \fIrecvcounts\fP[], const int \fIrdispls\fP[],
-	const MPI_Datatype \fIrecvtypes\fP[], MPI_Comm \fIcomm\fP)
-
-int MPI_Ialltoallw(const void *\fIsendbuf\fP, const int \fIsendcounts\fP[],
-	const int \fIsdispls\fP[], const MPI_Datatype \fIsendtypes\fP[],
-	void *\fIrecvbuf\fP, const int \fIrecvcounts\fP[], const int \fIrdispls\fP[],
-	const MPI_Datatype \fIrecvtypes\fP[], MPI_Comm \fIcomm\fP,
-	MPI_Request \fI*request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_ALLTOALLW(\fISENDBUF, SENDCOUNTS, SDISPLS, SENDTYPES,
-	RECVBUF, RECVCOUNTS, RDISPLS, RECVTYPES, COMM, IERROR\fP)
-
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fISENDCOUNTS(*), SDISPLS(*), SENDTYPES(*)\fP
-	INTEGER	\fIRECVCOUNTS(*), RDISPLS(*), RECVTYPES(*)\fP
-	INTEGER	\fICOMM, IERROR\fP
-
-MPI_IALLTOALLW(\fISENDBUF, SENDCOUNTS, SDISPLS, SENDTYPES,
-	RECVBUF, RECVCOUNTS, RDISPLS, RECVTYPES, COMM, REQUEST, IERROR\fP)
-
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fISENDCOUNTS(*), SDISPLS(*), SENDTYPES(*)\fP
-	INTEGER	\fIRECVCOUNTS(*), RDISPLS(*), RECVTYPES(*)\fP
-	INTEGER	\fICOMM, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Alltoallw(\fIsendbuf\fP, \fIsendcounts\fP, \fIsdispls\fP, \fIsendtypes\fP, \fIrecvbuf\fP, \fIrecvcounts\fP,
-		\fIrdispls\fP, \fIrecvtypes\fP, \fIcomm\fP, \fIierror\fP)
-
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..) :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIsendcounts(*)\fP, \fIsdispls(*)\fP, \fIrecvcounts(*),\fP
-	\fIrdispls(*)\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtypes(*)\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIrecvtypes(*)\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPI_Ialltoallw(\fIsendbuf\fP, \fIsendcounts\fP, \fIsdispls\fP, \fIsendtypes\fP, \fIrecvbuf\fP,
-		\fIrecvcounts\fP, \fIrdispls\fP, \fIrecvtypes\fP, \fIcomm\fP, \fIrequest\fP, \fIierror\fP)
-
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN), ASYNCHRONOUS :: \fIsendcounts(*)\fP, \fIsdispls(*),\fP
-	\fIrecvcounts(*)\fP, \fIrdispls(*)\fP
-	TYPE(MPI_Datatype), INTENT(IN), ASYNCHRONOUS :: \fIsendtypes(*),\fP
-	\fIrecvtypes(*)\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1.2i
-sendbuf
-Starting address of send buffer.
-.TP 1.2i
-sendcounts
-Integer array, where entry i specifies the number of elements to send
-to rank i.
-.TP 1.2i
-sdispls
-Integer array, where entry i specifies the displacement (in bytes,
-offset from \fIsendbuf\fP) from which to send data to rank i.
-.TP 1.2i
-sendtypes
-Datatype array, where entry i specifies the datatype to use when
-sending data to rank i.
-.TP 1.2i
-recvcounts
-Integer array, where entry j specifies the number of elements to
-receive from rank j.
-.TP 1.2i
-rdispls
-Integer array, where entry j specifies the displacement (in bytes,
-offset from \fIrecvbuf\fP) to which data from rank j should
-be written.
-.TP 1.2i
-recvtypes
-Datatype array, where entry j specifies the datatype to use when
-receiving data from rank j.
-.TP 1.2i
-comm
-Communicator over which data is to be exchanged.
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1.2i
-recvbuf
-Address of receive buffer.
-.TP 1.2i
-request
-Request (handle, non-blocking only).
-.ft R
-.TP 1.2i
-IERROR
-Fortran only: Error status.
-
-.SH DESCRIPTION
-.ft R
-MPI_Alltoallw is a generalized collective operation in which all
-processes send data to and receive data from all other processes. It
-adds flexibility to MPI_Alltoallv by allowing the user to specify the
-datatype of individual data blocks (in addition to displacement and
-element count). Its operation can be thought of in the following way,
-where each process performs 2n (n being the number of processes in
-communicator \fIcomm\fP) independent point-to-point communications
-(including communication with itself).
-.sp
-.nf
-	MPI_Comm_size(\fIcomm\fP, &n);
-	for (i = 0, i < n; i++)
-	    MPI_Send(\fIsendbuf\fP + \fIsdispls\fP[i], \fIsendcounts\fP[i],
-	        \fIsendtypes\fP[i], i, ..., \fIcomm\fP);
-	for (i = 0, i < n; i++)
-	    MPI_Recv(\fIrecvbuf\fP + \fIrdispls\fP[i], \fIrecvcounts\fP[i],
-	        \fIrecvtypes\fP[i], i, ..., \fIcomm\fP);
-.fi
-.sp
-Process j sends the k-th block of its local \fIsendbuf\fP to process
-k, which places the data in the j-th block of its local
-\fIrecvbuf\fP.
-.sp
-When a pair of processes exchanges data, each may pass different
-element count and datatype arguments so long as the sender specifies
-the same amount of data to send (in bytes) as the receiver expects
-to receive.
-.sp
-Note that process i may send a different amount of data to process j
-than it receives from process j. Also, a process may send entirely
-different amounts and types of data to different processes in the
-communicator.
-
-WHEN COMMUNICATOR IS AN INTER-COMMUNICATOR
-.sp
-When the communicator is an inter-communicator, the gather operation occurs in two phases.  The data is gathered from all the members of the first group and received by all the members of the second group.  Then the data is gathered from all the members of the second group and received by all the members of the first.  The operation exhibits a symmetric, full-duplex behavior.
-.sp
-The first group defines the root process.  The root process uses MPI_ROOT as the value of \fIroot\fR.  All other processes in the first group use MPI_PROC_NULL as the value of \fIroot\fR.  All processes in the second group use the rank of the root process in the first group as the value of \fIroot\fR.
-.sp
-When the communicator is an intra-communicator, these groups are the same, and the operation occurs in a single phase.
-.sp
-
-.SH USE OF IN-PLACE OPTION
-When the communicator is an intracommunicator, you can perform an all-to-all operation in-place (the output buffer is used as the input buffer).  Use the variable MPI_IN_PLACE as the value of \fIsendbuf\fR.  In this case, \fIsendcounts\fR, \fIsdispls\fP, and \fIsendtypes\fR are ignored.  The input data of each process is assumed to be in the area where that process would receive its own contribution to the receive buffer.
-
-.SH NOTES
-.sp
-The specification of counts, types, and displacements should not cause
-any location to be written more than once.
-.sp
-All arguments on all processes are significant. The \fIcomm\fP argument,
-in particular, must describe the same communicator on all processes.
-.sp
-The offsets of \fIsdispls\fP and \fIrdispls\fP are measured in bytes.
-Compare this to MPI_Alltoallv, where these offsets are measured in units
-of \fIsendtype\fP and \fIrecvtype\fP, respectively.
-
-.SH ERRORS
-.ft R
-Almost all MPI routines return an error value; C routines as
-the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Alltoall
-MPI_Alltoallv
-
diff --git a/ompi/mpi/man/man3/MPI_Attr_delete.3in b/ompi/mpi/man/man3/MPI_Attr_delete.3in
deleted file mode 100644
index 2ea3485e04c..00000000000
--- a/ompi/mpi/man/man3/MPI_Attr_delete.3in
+++ /dev/null
@@ -1,70 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Attr_delete 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Attr_delete\fP \- Deletes attribute value associated with a key -- use of this routine is deprecated.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Attr_delete(MPI_Comm\fI comm\fP, int\fI keyval\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-INCLUDE 'mpif.h'
-MPI_ATTR_DELETE(\fICOMM\fP,\fI KEYVAL\fP, \fIIERROR\fP)
-	INTEGER	\fICOMM\fP,\fI KEYVAL\fP,\fI IERROR\fP
-
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-comm
-Communicator to which attribute is attached (handle).
-.TP 1i
-keyval
-The key value of the deleted attribute (integer).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-Note that use of this routine is \fIdeprecated\fP as of MPI-2, and
-was \fIdeleted\fP in MPI-3. Please use MPI_Comm_delete_attr.  This
-function does not have a mpi_f08 binding.
-.sp
-Delete attribute from cache by key. This function invokes the attribute delete function delete_fn specified when the keyval was created. The call will fail if the delete_fn function returns an error code other than MPI_SUCCESS.
-
-Whenever a communicator is replicated using the function MPI_Comm_dup, all callback copy functions for attributes that are currently set are invoked (in arbitrary order). Whenever a communicator is deleted using the function MPI_Comm_free, all callback delete functions for attributes that are currently set are invoked.
-
-
-.SH NOTES
-Note that it is not defined by the MPI standard what happens if the
-delete_fn callback invokes other MPI functions.  In Open MPI, it is
-not valid for delete_fn callbacks (or any of their children) to add or
-delete attributes on the same object on which the delete_fn callback
-is being invoked.
-
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler
-may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-MPI_Comm_delete_attr
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Attr_get.3in b/ompi/mpi/man/man3/MPI_Attr_get.3in
deleted file mode 100644
index bcfb2819232..00000000000
--- a/ompi/mpi/man/man3/MPI_Attr_get.3in
+++ /dev/null
@@ -1,71 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Attr_get 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Attr_get\fP \- Retrieves attribute value by key -- use of this routine is deprecated.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Attr_get(MPI_Comm \fIcomm\fP, int\fI keyval\fP,void\fI *attribute_val\fP,
-	int\fI *flag \fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-INCLUDE 'mpif.h'
-MPI_ATTR_GET(\fICOMM\fP,\fI KEYVAL\fP, \fIATTRIBUTE_VAL\fP,\fI FLAG\fP,\fI IERROR\fP)
-	INTEGER	\fICOMM\fP,\fI KEYVAL\fP, \fIATTRIBUTE_VAL\fP,\fI IERROR\fP
-	LOGICAL	\fIFLAG\fP
-
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-comm
-Communicator to which attribute is attached (handle).
-.TP 1i
-keyval
- Key value (integer).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-attribute_val
-Attribute value, unless flag = false.
-.TP 1i
-flag
-True if an attribute value was extracted; false if no attribute is associated with the key.
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Note that use of this routine is \fIdeprecated\fP as of MPI-2, and
-was \fIdeleted\fP in MPI-3. Please use MPI_Comm_get_attr.  This
-function does not have a mpi_f08 binding.
-.sp
-Retrieves attribute value by key. The call is erroneous if there is no key
-with value keyval. On the other hand, the call is correct if the key value exists, but no attribute is attached on comm for that key; in such case, the call returns flag = false. In particular MPI_KEYVAL_INVALID is an erroneous key value.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler
-may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-
-.SH SEE ALSO
-
-MPI_Comm_get_attr
-.br
diff --git a/ompi/mpi/man/man3/MPI_Attr_put.3in b/ompi/mpi/man/man3/MPI_Attr_put.3in
deleted file mode 100644
index c4553a97bc9..00000000000
--- a/ompi/mpi/man/man3/MPI_Attr_put.3in
+++ /dev/null
@@ -1,73 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Attr_put 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Attr_put\fP \- Stores attribute value associated with a key -- use of this routine is deprecated.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Attr_put(MPI_Comm \fIcomm\fP, int\fI keyval\fP, void\fI *attribute_val\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-INCLUDE 'mpif.h'
-MPI_ATTR_PUT(\fICOMM\fP,\fI KEYVAL\fP, \fIATTRIBUTE_VAL\fP,\fI IERROR\fP)
-	INTEGER	\fICOMM\fP,\fI KEYVAL\fP,\fI ATTRIBUTE_VAL\fP,\fI IERROR
-
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-comm
-Communicator to which attribute will be attached (handle).
-.TP 1i
-keyval
-Key value, as returned by MPI_KEYVAL_CREATE (integer).
-.TP 1i
-attribute_val
-Attribute value.
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Note that use of this routine is \fIdeprecated\fP as of MPI-2, and
-was \fIdeleted\fP in MPI-3. Please use MPI_Comm_set_attr.  This
-function does not have a mpi_f08 binding.
-.sp
-MPI_Attr_put stores the stipulated attribute value attribute_val for subsequent retrieval by MPI_Attr_get. If the value is already present, then the outcome is as if MPI_Attr_delete was first called to delete the previous value (and the callback function delete_fn was executed), and a new value was next stored. The call is erroneous if there is no key with value keyval; in particular MPI_KEYVAL_INVALID is an erroneous key value. The call will fail if the delete_fn function returned an error code other than MPI_SUCCESS.
-
-.SH NOTES
-.ft R
-Values of the permanent attributes MPI_TAG_UB, MPI_HOST,
-MPI_IO, and MPI_WTIME_IS_GLOBAL may not be changed.
-.sp
-The type of the attribute value depends on whether C or Fortran is being used. In C, an attribute value is a pointer (void *); in Fortran, it is a single integer (not a pointer, since Fortran has no pointers and there are systems for which a pointer does not fit in an integer, e.g., any  32-bit address system that uses 64 bits for Fortran DOUBLE PRECISION).
-.sp
-If an attribute is already present, the delete function (specified when the corresponding keyval was created) will be called.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler
-may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Comm_set_attr
-.br
diff --git a/ompi/mpi/man/man3/MPI_Barrier.md b/ompi/mpi/man/man3/MPI_Barrier.md
deleted file mode 100644
index 63ebbbccc4b..00000000000
--- a/ompi/mpi/man/man3/MPI_Barrier.md
+++ /dev/null
@@ -1,46 +0,0 @@
-# NAME
-
-MPI_Barrier, MPI_Ibarrier - Synchronization between MPI processes in a group
-
-# Syntax
-
-## C Syntax
-```c
-#include <mpi.h>
-int MPI_Barrier(MPI_Comm)
-int MPI_Ibarrier(MPI_Comm comm, MPI_Request *request)
-```
-## Fortran Syntax
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_BARRIER(COMM, IERROR)
-    INTEGER COMM, IERROR
-MPI_IBARRIER(COMM, REQUEST, IERROR)
-    INTEGER COMM, REQUEST, IERROR
-```
-## Fortran 2008 Syntax
-```fortran
-USE mpi_f08
-MPI_Barrier(comm, ierror)
-    TYPE(MPI_Comm), INTENT(IN) :: comm
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-MPI_Ibarrier(comm, request, ierror)
-    TYPE(MPI_Comm), INTENT(IN) :: comm
-    TYPE(MPI_Request), INTENT (OUT) :: request
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-# Input Parameter
-* `comm` : Communicator (handle).
-# Output Parameters
-* `request` : Request (handle, non-blocking only).
-* `IERROR` : Fortran only: Error status (integer).
-# Description
-An MPI barrier completes after all groups members have entered the barrier.
-# When Communicator is an Inter-Communicator
-When the communicator is an inter-communicator, the barrier operation is performed across all processes in both groups.  All processes in the first group may exit the barrier when all processes in the second group have entered the barrier.
-# Errors
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-Before the error value is returned, the current MPI error handler is called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with `MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN` may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-# See Also
-[`MPI_Bcast`(3)](MPI_Bcast.html)
diff --git a/ompi/mpi/man/man3/MPI_Bcast.md b/ompi/mpi/man/man3/MPI_Bcast.md
deleted file mode 100644
index 1212c9ef47c..00000000000
--- a/ompi/mpi/man/man3/MPI_Bcast.md
+++ /dev/null
@@ -1,122 +0,0 @@
-# NAME
-
-`MPI_Bcast`, `MPI_Ibcast` - Broadcasts a message from the process with
-rank *root* to all other processes of the group.
-
-# SYNTAX
-
-## C Syntax
-
-```C
-#include <mpi.h>
-int MPI_Bcast(void *buffer, int count, MPI_Datatype datatype,
-    int root, MPI_Comm comm)
-
-int MPI_Ibcast(void *buffer, int count, MPI_Datatype datatype,
-    int root, MPI_Comm comm, MPI_Request *request)
-```
-
-## Fortran Syntax
-
-```Fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_BCAST(BUFFER, COUNT, DATATYPE, ROOT, COMM, IERROR)
-    <type>	BUFFER(*)
-    INTEGER	COUNT, DATATYPE, ROOT, COMM, IERROR
-
-MPI_IBCAST(BUFFER, COUNT, DATATYPE, ROOT, COMM, REQUEST, IERROR)
-    <type>	BUFFER(*)
-    INTEGER	COUNT, DATATYPE, ROOT, COMM, REQUEST, IERROR
-```
-
-## Fortran 2008 Syntax
-
-``` Fortran
-USE mpi_f08
-MPI_Bcast(buffer, count, datatype, root, comm, ierror)
-    TYPE(*), DIMENSION(..) :: buffer
-    INTEGER, INTENT(IN) :: count, root
-    TYPE(MPI_Datatype), INTENT(IN) :: datatype
-    TYPE(MPI_Comm), INTENT(IN) :: comm
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-
-MPI_Ibcast(buffer, count, datatype, root, comm, request, ierror)
-    TYPE(*), DIMENSION(..), ASYNCHRONOUS :: buffer
-    INTEGER, INTENT(IN) :: count, root
-    TYPE(MPI_Datatype), INTENT(IN) :: datatype
-    TYPE(MPI_Comm), INTENT(IN) :: comm
-    TYPE(MPI_Request), INTENT(OUT) :: request
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# INPUT/OUTPUT PARAMETERS
-
-* `buffer`: Starting address of buffer (choice).
-* `count`: Number of entries in buffer (integer).
-* `datatype`: Data type of buffer (handle).
-* `root`: Rank of broadcast root (integer).
-* `comm`: Communicator (handle).
-
-# OUTPUT PARAMETERS
-
-* `request`: Request (handle, non-blocking only).
-* `IERROR`: Fortran only: Error status (integer).
-
-# DESCRIPTION
-
-`MPI_Bcast` broadcasts a message from the process with rank root to all
-processes of the group, itself included. It is called by all members of
-group using the same arguments for `comm`, `root`. On return, the contents
-of root's communication buffer has been copied to all processes.
-
-General, derived datatypes are allowed for datatype. The type signature
-of count, datatype on any process must be equal to the type signature o	f
-count, datatype at the root. This implies that the amount of data sent
-must be equal to the amount received, pairwise between each process and
-the root. `MPI_Bcast` and all other data-movement collective routines make
-this restriction. Distinct type maps between sender and receiver are
-still allowed.
-
-**Example:** Broadcast 100 ints from process 0 to every process in the
-group.
-
-```C
-MPI_Comm comm;
-int array[100];
-int root=0;
-//...
-MPI_Bcast( array, 100, MPI_INT, root, comm);
-```
-
-As in many of our sample code fragments, we assume that some of the
-variables (such as comm in the example above) have been assigned
-appropriate values.
-
-# WHEN COMMUNICATOR IS AN INTER-COMMUNICATOR
-
-When the communicator is an inter-communicator, the root process in the
-first group broadcasts data to all the processes in the second group.
-The first group defines the root process. That process uses `MPI_ROOT` as
-the value of its `root` argument. The remaining processes use
-`MPI_PROC_NULL` as the value of their `root` argument. All processes in
-the second group use the rank of that root process in the first group as
-the value of their `root` argument. The receive buffer arguments of the
-processes in the second group must be consistent with the send buffer
-argument of the root process in the first group.
-
-# NOTES
-
-This function does not support the in-place option.
-
-# ERRORS
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
diff --git a/ompi/mpi/man/man3/MPI_Bsend.md b/ompi/mpi/man/man3/MPI_Bsend.md
deleted file mode 100644
index 4f8aeb34764..00000000000
--- a/ompi/mpi/man/man3/MPI_Bsend.md
+++ /dev/null
@@ -1,91 +0,0 @@
-# Name
-
-`MPI_Bsend` - Basic send with user-specified buffering.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-int MPI_Bsend(const void *buf, int count, MPI_Datatype datatype,
-   int dest, int tag, MPI_Comm comm)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_BSEND(BUF, COUNT,DATATYPE, DEST, TAG, COMM, IERROR)
-    <type>  BUF(*)
-    INTEGER COUNT, DATATYPE, DEST, TAG, COMM, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-MPI_Bsend(buf, count, datatype, dest, tag, comm, ierror)
-    TYPE(*), DIMENSION(..), INTENT(IN) :: buf
-    INTEGER, INTENT(IN) :: count, dest, tag
-    TYPE(MPI_Datatype), INTENT(IN) :: datatype
-    TYPE(MPI_Comm), INTENT(IN) :: comm
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameters
-
-* `buf` : Initial address of send buffer (choice).
-* `count` : Number of entries in send buffer (nonnegative integer).
-* `datatype` : Datatype of each send buffer element (handle).
-* `dest` : Rank of destination (integer).
-* `tag` : Message tag (integer).
-* `comm` : Communicator (handle).
-
-# Output Parameters
-
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-`MPI_Bsend` performs a buffered-mode, blocking send.
-
-# Notes
-
-This send is provided as a convenience function; it allows the user to
-send messages without worrying about where they are buffered (because
-the user must have provided buffer space with `MPI_Buffer_attach`).
-
-In deciding how much buffer space to allocate, remember that the buffer
-space is not available for reuse by subsequent `MPI_Bsend`s unless you
-are certain that the message has been received (not just that it should
-have been received). For example, this code does not allocate enough
-buffer space:
-```c
-MPI_Buffer_attach( b, n*sizeof(double) + MPI_BSEND_OVERHEAD );
-for (i=0; i<m; i++) {
-    MPI_Bsend( buf, n, MPI_DOUBLE, ... );
-}
-```
-because only enough buffer space is provided for a single send, and the
-loop may start a second `MPI_Bsend` before the first is done making use
-of the buffer.
-
-In C, you can force the messages to be delivered by `MPI_Buffer_detach(
-&b, &n );` `MPI_Buffer_attach( b, n );` (The `MPI_Buffer_detach` will not
-complete until all buffered messages are delivered.)
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value;
-C routines as the value of the function and Fortran routines in the last
-argument.
-
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O
-function errors. The error handler may be changed with `MPI_Comm_set_errhandler`;
-the predefined error handler `MPI_ERRORS_RETURN` may be used to cause error
-values to be returned. Note that MPI does not guarantee that an MPI program
-can continue past an error.
diff --git a/ompi/mpi/man/man3/MPI_Bsend_init.md b/ompi/mpi/man/man3/MPI_Bsend_init.md
deleted file mode 100644
index 50c2431a84c..00000000000
--- a/ompi/mpi/man/man3/MPI_Bsend_init.md
+++ /dev/null
@@ -1,85 +0,0 @@
-# Name
-
-`MPI_Bsend_init` - Builds a handle for a buffered send.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Bsend_init(const void *buf, int count, MPI_Datatype datatype,
-	int dest, int tag, MPI_Comm comm, MPI_Request *request)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_BSEND_INIT(BUF, COUNT, DATATYPE, DEST, TAG, COMM, REQUEST, IERROR)
-    <type>	BUF(*)
-    INTEGER	COUNT, DATATYPE, DEST, TAG,
-    INTEGER	COMM, REQUEST, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Bsend_init(buf, count, datatype, dest, tag, comm, request, ierror)
-    TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: buf
-    INTEGER, INTENT(IN) :: count, dest, tag
-    TYPE(MPI_Datatype), INTENT(IN) :: datatype
-    TYPE(MPI_Comm), INTENT(IN) :: comm
-    TYPE(MPI_Request), INTENT(OUT) :: request
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameters
-
-* `buf` : Initial address of send buffer (choice).
-* `count` : Number of elements sent (integer).
-* `datatype` : Type of each element (handle).
-* `dest` : Rank of destination (integer).
-* `tag` : Message tag (integer).
-* `comm` : Communicator (handle).
-
-# Output Parameters
-
-* `request` : Communication request (handle).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-Creates a persistent communication `request` for a buffered mode send, and
-binds to it all the arguments of a send operation.
-
-A communication (send or receive) that uses a persistent `request` is
-initiated by the function `MPI_Start`.
-
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-
-# See Also
-
-[MPI_Send_init(3)](MPI_Send_init.html)
-[MPI_Ssend_init(3)](MPI_Ssend_init.html)
-[MPI_Rsend_init(3)](MPI_Rsend_init.html)
-[MPI_Recv_init(3)](MPI_Recv_init.html)
-[MPI_Start(3)](MPI_Start.html)
-[MPI_Startall(3)](MPI_Startall.html)
diff --git a/ompi/mpi/man/man3/MPI_Buffer_attach.md b/ompi/mpi/man/man3/MPI_Buffer_attach.md
deleted file mode 100644
index 2ea01bf319f..00000000000
--- a/ompi/mpi/man/man3/MPI_Buffer_attach.md
+++ /dev/null
@@ -1,92 +0,0 @@
-# Name
-
-`MPI_Buffer_attach` - Attaches a user-defined buffer for sending.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Buffer_attach(void *buf, int size)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_BUFFER_ATTACH(BUF, SIZE, IERROR)
-	<type>	BUF(*)
-	INTEGER	SIZE, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Buffer_attach(buffer, size, ierror)
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: buffer
-	INTEGER, INTENT(IN) :: size
-	INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameters
-
-* `buf` : Initial buffer address (choice).
-* `size` : Buffer size, in bytes (integer).
-
-# Output Parameter
-
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-Provides to MPI a buffer in the user's memory to be used for buffering
-outgoing messages. The buffer is used only by messages sent in buffered
-mode. Only one buffer can be attached to a process at a time.
-
-# Notes
-
-The size given should be the sum of the sizes of all outstanding Bsends
-that you intend to have, plus `MPI_BSEND_OVERHEAD` bytes for each Bsend
-that you do. For the purposes of calculating size, you should use
-`MPI_Pack_size`. In other words, in the code
-
-```c
-MPI_Buffer_attach( buf, size );
-MPI_Bsend( ..., count=20, datatype=type1, ... );
-//...
-MPI_Bsend( ..., count=40, datatype=type2, ... );
-```
-the value of size in the `MPI_Buffer_attach` call should be greater than the value computed by
-
-```c
-MPI_Pack_size( 20, type1, comm, &s1 );
-MPI_Pack_size( 40, type2, comm, &s2 );
-size = s1 + s2 + 2 * MPI_BSEND_OVERHEAD;
-```
-
-`MPI_BSEND_OVERHEAD` gives the maximum amount of buffer space that may be
-used by the  Bsend  routines.  This value is in mpi.h for C and mpif.h
-for Fortran.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-
-# See Also
-
-[MPI_Buffer_detach(3)](MPI_Buffer_detach.html)
diff --git a/ompi/mpi/man/man3/MPI_Buffer_detach.md b/ompi/mpi/man/man3/MPI_Buffer_detach.md
deleted file mode 100644
index 3122d034e56..00000000000
--- a/ompi/mpi/man/man3/MPI_Buffer_detach.md
+++ /dev/null
@@ -1,116 +0,0 @@
-# Name
-
-`MPI_Buffer_detach` - Removes an existing buffer (for use in
-in `MPI_Bsend`, etc.)
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Buffer_detach(void *buf, int *size)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_BUFFER_DETACH(BUF, SIZE, IERROR)
-	<type>	BUF(*)
-	INTEGER	SIZE, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Buffer_detach(buffer_addr, size, ierror)
-	USE, INTRINSIC :: ISO_C_BINDING, ONLY
-	TYPE(C_PTR), INTENT(OUT) :: buffer_addr
-	INTEGER, INTENT(OUT) :: size
-	INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Output Parameters
-
-* `buf` : Initial buffer address (choice).
-* `size` : Buffer size, in bytes (integer).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-Detach the buffer currently associated with MPI. The call returns the
-address and the size of the detached buffer. This operation will block
-until all messages currently in the buffer have been transmitted. Upon
-return of this function, the user may reuse or deallocate the space
-taken by the buffer.
-
-Example: Calls to attach and detach buffers.
-```c
-#define BUFFSIZE 10000
-
-int size
-char *buff;
-MPI_Buffer_attach( malloc(BUFFSIZE), BUFFSIZE);
-/* a buffer of 10000 bytes can now be used by MPI_Bsend */
-MPI_Buffer_detach( &buff, &size);
-/* Buffer size reduced to zero */
-MPI_Buffer_attach( buff, size);
-/* Buffer of 10000 bytes available again */
-```
-
-# Notes
-The  reason  that  `MPI_Buffer_detach`  returns the address and size of
-the buffer being detached is to allow nested libraries to replace and
-restore the buffer. For example, consider
-```c
-int size, mysize, idummy;
-void *ptr, *myptr, *dummy;
-MPI_Buffer_detach( &ptr, &size );
-MPI_Buffer_attach( myptr, mysize );
-/*
-... library code ...
-*/
-MPI_Buffer_detach( &dummy, &idummy );
-MPI_Buffer_attach( ptr, size );
-```
-This is much like the action of the UNIX signal routine and has the
-same strengths (it's simple) and  weak‐nesses (it only works for
-nested usages).
-
-For Fortran: The Fortran binding for this routine is different.
-Because Fortran does not have pointers, it is impossible to provide
-a way to use the output of this routine to exchange buffers. In this
-case, only the size field is set.
-
-For C: Even though the buf argument is declared as void, it is really
-the address of a void pointer. See Rationale, below, for more details.
-
-Even though the C functions `MPI_Buffer_attach` and `MPI_Buffer_detach`
-both have  a  first  argument  of  type void*,  these  arguments are
-used differently: A pointer to the buffer is passed to MPI_Buffer_attach;
-the address of the pointer is passed to MPI_Buffer_detach, so that this
-call can return the pointer value.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-
-# See Also
-
-[MPI_Buffer_attach(3)](MPI_Buffer_attach.html)
-[MPI_Bsend(3)](MPI_Bsend.html)
diff --git a/ompi/mpi/man/man3/MPI_Cancel.md b/ompi/mpi/man/man3/MPI_Cancel.md
deleted file mode 100755
index d95f7348fcd..00000000000
--- a/ompi/mpi/man/man3/MPI_Cancel.md
+++ /dev/null
@@ -1,114 +0,0 @@
-# Name
-
-`MPI_Cancel` - Cancels a communication request.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Cancel(MPI_Request *request)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_CANCEL(REQUEST, IERROR)
-    INTEGER	REQUEST, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Cancel(request, ierror)
-    TYPE(MPI_Request), INTENT(IN) :: request
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-
-# Input Parameter
-
-* `request` : Communication request (handle).
-
-# Output Parameter
-
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-The `MPI_Cancel` operation allows pending communications to be canceled.
-This is required for cleanup. Posting a send or a receive ties up user
-resources (send or receive buffers), and a cancel may be needed to free
-these resources gracefully.
-
-A call to `MPI_Cancel` marks for cancellation a pending, nonblocking
-communication operation (send or receive). The cancel call is local. It
-returns immediately, possibly before the communication is actually
-canceled. It is still necessary to complete a communication that has
-been marked for cancellation, using a call to `MPI_Request_free,`
-`MPI_Wait`, or `MPI_Test` (or any of the derived operations).
-
-If a communication is marked for cancellation, then an `MPI_Wait` call for
-that communication is guaranteed to return, irrespective of the
-activities of other processes (i.e., `MPI_Wait` behaves as a local
-function); similarly if `MPI_Test` is repeatedly called in a busy wait
-loop for a canceled communication, then `MPI_Test` will eventually be
-successful.
-
-`MPI_Cancel` can be used to cancel a communication that uses a persistent
-`request` (see Section 3.9 in the MPI-1 Standard, "Persistent
-Communication Requests") in the same way it is used for nonpersistent
-`request`s. A successful cancellation cancels the active communication,
-but not the `request` itself. After the call to `MPI_Cancel` and the
-subsequent call to `MPI_Wait` or `MPI_Test`, the `request` becomes inactive
-and can be activated for a new communication.
-
-The successful cancellation of a buffered send frees the buffer space
-occupied by the pending message.
-
-Either the cancellation succeeds or the communication succeeds, but not
-both. If a send is marked for cancellation, then it must be the case
-that either the send completes normally, in which case the message sent
-is received at the destination process, or that the send is successfully
-canceled, in which case no part of the message is received at the
-destination. Then, any matching receive has to be satisfied by another
-send. If a receive is marked for cancellation, then it must be the case
-that either the receive completes normally, or that the receive is
-successfully canceled, in which case no part of the receive buffer is
-altered. Then, any matching send has to be satisfied by another receive.
-
-If the operation has been canceled, then information to that effect will
-be returned in the status argument of the operation that completes the
-communication.
-
-# Notes
-
-The primary expected use of `MPI_Cancel` is in multi-buffering schemes,
-where speculative `MPI_Irecvs` are made. When the computation completes,
-some of these `request`s may remain; using `MPI_Cancel` allows the user to
-cancel these unsatisfied `request`s.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-# See Also
-
-[`MPI_Probe`(3)](MPI_Probe.html)
-[`MPI_Iprobe`(3)](MPI_Iprobe.html)
-[`MPI_Test_cancelled`(3)](MPI_Test_cancelled.html)
-[`MPI_Cart_coords`(3)](MPI_Cart_coords.html)
diff --git a/ompi/mpi/man/man3/MPI_Cart_coords.md b/ompi/mpi/man/man3/MPI_Cart_coords.md
deleted file mode 100755
index aec947104d2..00000000000
--- a/ompi/mpi/man/man3/MPI_Cart_coords.md
+++ /dev/null
@@ -1,66 +0,0 @@
-# Name
-
-`MPI_Cart_coords` - Determines process coords in Cartesian topology
-given rank in group.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Cart_coords(MPI_Comm comm, int rank, int maxdims,
-    int coords[])
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_CART_COORDS(COMM, RANK, MAXDIMS, COORDS, IERROR)
-    INTEGER	COMM, RANK, MAXDIMS, COORDS(*), IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Cart_coords(comm, rank, maxdims, coords, ierror)
-    TYPE(MPI_Comm), INTENT(IN) :: comm
-    INTEGER, INTENT(IN) :: rank, maxdims
-    INTEGER, INTENT(OUT) :: coords(maxdims)
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameters
-
-* `comm` : Communicator with Cartesian structure (handle).
-* `rank` : Rank of a process within group of comm (integer).
-* `maxdims` :  Length of vector coords in the calling program (integer).
- Length of vector coords in the calling program (integer).
-
-# Output Parameters
-
-* `coords` : Integer array (of size ndims,which was defined by MPI_Cart_create
-call) containing the Cartesian coordinates of specified process
-(integer).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-`MPI_Cart_coords` provies a mapping of `rank`s to Cartesian coordinates.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
diff --git a/ompi/mpi/man/man3/MPI_Cart_create.md b/ompi/mpi/man/man3/MPI_Cart_create.md
deleted file mode 100755
index c3188445d0c..00000000000
--- a/ompi/mpi/man/man3/MPI_Cart_create.md
+++ /dev/null
@@ -1,80 +0,0 @@
-# Name
-
-`MPI_Cart_create` - Makes a new communicator to which Cartesian
-topology information has been attached.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Cart_create(MPI_Comm comm_old, int ndims, const int dims[],
-
-	const int periods[], int reorder, MPI_Comm *comm_cart)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_CART_CREATE(COMM_OLD, NDIMS, DIMS, PERIODS, REORDER,
-    	COMM_CART, IERROR)
-    INTEGER	COMM_OLD, NDIMS, DIMS(*), COMM_CART, IERROR
-    LOGICAL	PERIODS(*), REORDER
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Cart_create(comm_old, ndims, dims, periods, reorder, comm_cart, ierror)
-    TYPE(MPI_Comm), INTENT(IN) :: comm_old
-    INTEGER, INTENT(IN) :: ndims, dims(ndims)
-    LOGICAL, INTENT(IN) :: periods(ndims), reorder
-    TYPE(MPI_Comm), INTENT(OUT) :: comm_cart
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-
-# Input Parameters
-
-* `comm_old` : Input communicator (handle).
-* `ndims` : Number of dimensions of Cartesian grid (integer).
-* `dims` : Integer array of size ndims specifying the number of processes in
-each dimension.
-* `periods` : Logical array of size ndims specifying whether the grid is periodic
-(true) or not (false) in each dimension.
-* `reorder` : Ranking may be reordered (true) or not (false) (logical).
-
-# Output Parameters
-
-* `comm_cart` : Communicator with new Cartesian topology (handle).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-`MPI_Cart_create` returns a handle to a new communicator to which the
-Cartesian topology information is attached. If `reorder` = false then the
-rank of each process in the new group is identical to its rank in the
-old group. Otherwise, the function may `reorder` the processes (possibly
-so as to choose a good embedding of the virtual topology onto the
-physical machine). If the total size of the Cartesian grid is smaller
-than the size of the group of comm, then some processes are returned
-`MPI_COMM_NULL`, in analogy to `MPI_Comm_split`. The call is erroneous if it
-specifies a grid that is larger than the group size.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
diff --git a/ompi/mpi/man/man3/MPI_Cart_get.md b/ompi/mpi/man/man3/MPI_Cart_get.md
deleted file mode 100755
index d4dd0a5f030..00000000000
--- a/ompi/mpi/man/man3/MPI_Cart_get.md
+++ /dev/null
@@ -1,77 +0,0 @@
-# Name
-
-`MPI_Cart_get` - Retrieves Cartesian topology information associated
-with a communicator.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Cart_get(MPI_Comm comm, int maxdims, int dims[], int periods[],
-    int coords[])
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_CART_GET(COMM, MAXDIMS, DIMS, PERIODS, COORDS, IERROR)
-    INTEGER	COMM, MAXDIMS, DIMS(*), COORDS(*), IERROR
-    LOGICAL	PERIODS(*)
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Cart_get(comm, maxdims, dims, periods, coords, ierror)
-    TYPE(MPI_Comm), INTENT(IN) :: comm
-    INTEGER, INTENT(IN) :: maxdims
-    INTEGER, INTENT(OUT) :: dims(maxdims), coords(maxdims)
-    LOGICAL, INTENT(OUT) :: periods(maxdims)
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameters
-
-* `comm` : Communicator with Cartesian structure (handle).
-* `maxdims` : Length of vectors dims, periods, and coords in the calling program
-(integer).
-
-# Output Parameters
-
-* `dims` : Number of processes for each Cartesian dimension (array of
-integers).
-* `periods` : Periodicity (true/false) for each Cartesian dimension (array of
-logicals).
-* `coords` : Coordinates of calling process in Cartesian structure (array of
-integers).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-The functions `MPI_Cartdim_get` and `MPI_Cart_get` return the Cartesian
-topology information that was associated with a `comm`unicator by
-`MPI_Cart_create.`
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-# See Also
-
-[`MPI_Cartdim_get`(3)](MPI_Cartdim_get.html)
-[`MPI_Cart_create`(3)](MPI_Cart_create.html)
diff --git a/ompi/mpi/man/man3/MPI_Cart_map.md b/ompi/mpi/man/man3/MPI_Cart_map.md
deleted file mode 100755
index fee5af3aea1..00000000000
--- a/ompi/mpi/man/man3/MPI_Cart_map.md
+++ /dev/null
@@ -1,79 +0,0 @@
-# Name
-
-`MPI_Cart_map` - Maps process to Cartesian topology information.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Cart_map(MPI_Comm comm, int ndims, const int dims[],
-    const int periods[], int *newrank)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_CART_MAP(COMM, NDIMS, DIMS, PERIODS, NEWRANK, IERROR)
-    INTEGER	COMM, NDIMS, DIMS(*), NEWRANK, IERROR
-    LOGICAL	PERIODS(*)
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Cart_map(comm, ndims, dims, periods, newrank, ierror)
-    TYPE(MPI_Comm), INTENT(IN) :: comm
-    INTEGER, INTENT(IN) :: ndims, dims(ndims)
-    LOGICAL, INTENT(IN) :: periods(ndims)
-    INTEGER, INTENT(OUT) :: newrank
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameters
-
-* `comm` : Input communicator (handle).
-* `ndims` : Number of dimensions of Cartesian structure (integer).
-* `dims` : Integer array of size ndims specifying the number of processes in
-each coordinate direction.
-* `periods` : Logical array of size ndims specifying the periodicity specification
-in each coordinate direction.
-
-# Output Parameters
-
-* `newrank` : Reordered rank of the calling process; `MPI_UNDEFINED` if calling
-process does not belong to grid (integer).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-`MPI_Cart_map` and `MPI_Graph_map` can be used to implement all other
-topology functions. In general they will not be called by the user
-directly, unless he or she is creating additional virtual topology
-capability other than that provided by MPI.
-`MPI_Cart_map` computes an "optimal" placement for the calling process
-on the physical machine. A possible implementation of this function is
-to always return the rank of the calling process, that is, not to
-perform any reordering.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-# See Also
-
-[`MPI_Graph_map`(3)](MPI_Graph_map.html)
diff --git a/ompi/mpi/man/man3/MPI_Cart_rank.md b/ompi/mpi/man/man3/MPI_Cart_rank.md
deleted file mode 100755
index 1ae4e6a33fc..00000000000
--- a/ompi/mpi/man/man3/MPI_Cart_rank.md
+++ /dev/null
@@ -1,72 +0,0 @@
-# Name
-
-`MPI_Cart_rank` - Determines process rank in communicator given
-Cartesian location.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Cart_rank(MPI_Comm comm, int coords[], int *rank)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_CART_RANK(COMM, COORDS, RANK, IERROR)
-    INTEGER	COMM, COORDS(*), RANK, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Cart_rank(comm, coords, rank, ierror)
-    TYPE(MPI_Comm), INTENT(IN) :: comm
-    INTEGER, INTENT(IN) :: coords(*)
-    INTEGER, INTENT(OUT) :: rank
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameters
-
-* `comm` : Communicator with Cartesian structure (handle).
-* `coords` : Integer array (of size ndims, which was defined by `MPI_Cart_create`
-call) specifying the Cartesian coordinates of a process.
-
-# Output Parameter
-
-* `rank` : Rank of specified process (integer).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-For a process group with Cartesian structure, the function `MPI_Cart_rank`
-translates the logical process coordinates to process `rank`s as they are
-used by the point-to-point routines. For dimension i with periods(i) =
-true, if the coordinate, `coords(i)`, is out of range, that is, `coords(i)`
-< 0 or `coords(i)` >= `dims(i)`, it is shifted back to the interval 0 =<
-`coords(i)` < `dims(i)` automatically. Out-of-range coordinates are
-erroneous for nonperiodic dimensions.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-# See Also
-
-[`MPI_Cart_create`(3)](MPI_Cart_create.html)
diff --git a/ompi/mpi/man/man3/MPI_Cart_shift.md b/ompi/mpi/man/man3/MPI_Cart_shift.md
deleted file mode 100755
index 2a6a7bc5c5d..00000000000
--- a/ompi/mpi/man/man3/MPI_Cart_shift.md
+++ /dev/null
@@ -1,112 +0,0 @@
-# Name
-
-`MPI_Cart_shift` - Returns the shifted source and destination ranks,
-given a shift direction and amount.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Cart_shift(MPI_Comm comm, int direction, int disp,
-    int *rank_source, int *rank_dest)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_CART_SHIFT(COMM, DIRECTION, DISP, RANK_SOURCE,
-    	RANK_DEST, IERROR)
-    INTEGER	COMM, DIRECTION, DISP, RANK_SOURCE
-    INTEGER	RANK_DEST, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Cart_shift(comm, direction, disp, rank_source, rank_dest, ierror)
-    TYPE(MPI_Comm), INTENT(IN) :: comm
-    INTEGER, INTENT(IN) :: direction, disp
-    INTEGER, INTENT(OUT) :: rank_source, rank_dest
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameters
-
-* `comm` : Communicator with Cartesian structure (handle).
-* `direction` : Coordinate dimension of shift (integer).
-* `disp` : Displacement ( > 0: upward shift, < 0: downward shift) (integer).
-
-# Output Parameters
-
-* `rank_source` : Rank of source process (integer).
-* `rank_dest` : Rank of destination process (integer).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-If the process topology is a Cartesian structure, an `MPI_Sendrecv`
-operation is likely to be used along a coordinate `direction` to perform a
-shift of data. As input, `MPI_Sendrecv` takes the rank of a source process
-for the receive, and the rank of a destination process for the send. If
-the function `MPI_Cart_shift` is called for a Cartesian process group, it
-provides the calling process with the above identifiers, which then can
-be passed to `MPI_Sendrecv`. The user specifies the coordinate `direction`
-and the size of the step (positive or negative). The function is local.
-
-The `direction` argument indicates the dimension of the shift, i.e., the
-coordinate whose value is modified by the shift. The coordinates are
-numbered from 0 to ndims-1, where ndims is the number of dimensions.
-
-Note: The `direction` argument is in the range [0, n-1] for an
-n-dimensional Cartesian mesh.
-
-Depending on the periodicity of the Cartesian group in the specified
-coordinate `direction`, `MPI_Cart_shift` provides the identifiers for a
-circular or an end-off shift. In the case of an end-off shift, the value
-`MPI_PROC_NULL` may be returned in ``rank_source`` or ``rank_dest``, indicating
-that the source or the destination for the shift is out of range.
-
-Example: The `comm`unicator, `comm`, has a two-dimensional, periodic,
-Cartesian topology associated with it. A two-dimensional array of REALs
-is stored one element per process, in variable A. One wishes to skew
-this array, by shifting column i (vertically, i.e., along the column) by
-i steps.
-  
-```fortran
-! find process rank
-    CALL MPI_COMM_RANK(comm, rank, ierr)
-! find Cartesian coordinates
-    CALL MPI_CART_COORDS(comm, rank, maxdims, coords, ierr)
-! compute shift source and destination
-    CALL MPI_CART_SHIFT(comm, 0, coords(2), source, dest, ierr)
-! skew array
-    CALL MPI_SENDRECV_REPLACE(A, 1, MPI_REAL, dest, 0, source, 0, comm, status,
-                              ierr)
-```
-
-# Note
-
-In Fortran, the dimension indicated by DIRECTION = i has DIMS(i+1) nodes,
-where DIMS is the array that was used to create the grid. In C, the
-dimension indicated by direction = i is the dimension specified by
-dims[i].
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
diff --git a/ompi/mpi/man/man3/MPI_Cart_sub.3in b/ompi/mpi/man/man3/MPI_Cart_sub.3in
deleted file mode 100644
index a895b038bee..00000000000
--- a/ompi/mpi/man/man3/MPI_Cart_sub.3in
+++ /dev/null
@@ -1,82 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Cart_sub 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Cart_sub \fP \- Partitions a communicator into subgroups, which form lower-dimensional Cartesian subgrids.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Cart_sub(MPI_Comm \fIcomm\fP, const int\fI remain_dims\fP[], MPI_Comm\fI *comm_new\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_CART_SUB(\fICOMM, REMAIN_DIMS, COMM_NEW, IERROR\fP)
-	INTEGER	\fICOMM, COMM_NEW, IERROR\fP
-	LOGICAL	\fIREMAIN_DIMS\fP(*)
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Cart_sub(\fIcomm\fP, \fIremain_dims\fP, \fInewcomm\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	LOGICAL, INTENT(IN) :: \fIremain_dims(*)\fP
-	TYPE(MPI_Comm), INTENT(OUT) :: \fInewcomm\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-comm
-Communicator with Cartesian structure (handle).
-.TP 1i
-remain_dims
-The ith entry of remain_dims specifies whether the ith dimension is kept in the subgrid (true) or is dropped (false) (logical vector).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-comm_new
-Communicator containing the subgrid that includes the calling process (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-If a Cartesian topology has been created with MPI_Cart_create, the function  MPI_Cart_sub can be used to partition the communicator group into subgroups that form lower-dimensional Cartesian subgrids, and to build for each subgroup a communicator with the associated subgrid Cartesian topology. (This function is closely related to MPI_Comm_split.)
-.sp
-\fBExample:\fP Assume that  MPI_Cart_create( \&..., comm) has defined a (2 x 3 x 4)   grid. Let  remain_dims = (true, false, true). Then a call to
-.sp
-.nf
-    MPI_Cart_sub(comm, remain_dims, comm_new)
-.fi
-.sp
-will create three communicators, each with eight processes in a 2 x 4 Cartesian topology. If remain_dims = (false, false, true) then the call to MPI_Cart_sub(comm, remain_dims, comm_new) will create six nonoverlapping communicators, each with four processes, in a one-dimensional Cartesian topology.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-.nf
-MPI_Cart_create
-MPI_Comm_split
-
diff --git a/ompi/mpi/man/man3/MPI_Cart_sub.md b/ompi/mpi/man/man3/MPI_Cart_sub.md
deleted file mode 100644
index 408e58c6373..00000000000
--- a/ompi/mpi/man/man3/MPI_Cart_sub.md
+++ /dev/null
@@ -1,86 +0,0 @@
-# Name
-
-`MPI_Cart_sub` - Partitions a communicator into subgroups, which form
-lower-dimensional Cartesian subgrids.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Cart_sub(MPI_Comm comm, const int remain_dims[], MPI_Comm *comm_new)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_CART_SUB(COMM, REMAIN_DIMS, COMM_NEW, IERROR)
-    INTEGER COMM, COMM_NEW, IERROR
-    LOGICAL REMAIN_DIMS(*)
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Cart_sub(comm, remain_dims, newcomm, ierror)
-    TYPE(MPI_Comm), INTENT(IN) :: comm
-    LOGICAL, INTENT(IN) :: remain_dims(*)
-    TYPE(MPI_Comm), INTENT(OUT) :: newcomm
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameters
-
-* `comm` : Communicator with Cartesian structure (handle).
-* `remain_dims` : The ith entry of `remain_dims` specifies whether the ith dimension is
-kept in the subgrid (true) or is dropped (false) (logical vector).
-
-# Output Parameters
-
-* `comm_new` : Communicator containing the subgrid that includes the calling
-process (handle).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-If a Cartesian topology has been created with `MPI_Cart_create`, the
-function `MPI_Cart_sub` can be used to partition the communicator group
-into subgroups that form lower-dimensional Cartesian subgrids, and to
-build for each subgroup a communicator with the associated subgrid
-Cartesian topology. (This function is closely related to
-`MPI_Comm_split`.)
-
-Example: Assume that `MPI_Cart_create( ..., comm)` has defined a (2 x
-3 x 4) grid. Let `remain_dims` = (true, false, true). Then a call to
-
-    MPI_Cart_sub(comm, remain_dims, comm_new)
-
-will create three communicators, each with eight processes in a 2 x 4
-Cartesian topology. If `remain_dims = (false, false, true)` then the call
-to `MPI_Cart_sub(comm, remain_dims, comm_new)` will create six
-nonoverlapping communicators, each with four processes, in a
-one-dimensional Cartesian topology.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-# See Also
-
-[`MPI_Cart_create`(3)](MPI_Cart_create.html)
-[`MPI_Comm_split`(3)](MPI_Comm_split.html)
-
diff --git a/ompi/mpi/man/man3/MPI_Cartdim_get.md b/ompi/mpi/man/man3/MPI_Cartdim_get.md
deleted file mode 100755
index 5b8f3b12f28..00000000000
--- a/ompi/mpi/man/man3/MPI_Cartdim_get.md
+++ /dev/null
@@ -1,66 +0,0 @@
-# Name
-
-`MPI_Cartdim_get` - Retrieves Cartesian topology information
-associated with a communicator.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Cartdim_get(MPI_Comm comm, int *ndims)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_CARTDIM_GET(COMM, NDIMS, IERROR)
-    INTEGER	COMM, NDIMS, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Cartdim_get(comm, ndims, ierror)
-    TYPE(MPI_Comm), INTENT(IN) :: comm
-    INTEGER, INTENT(OUT) :: ndims
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-
-# Input Parameter
-
-* `comm` : Communicator with Cartesian structure (handle).
-
-# Output Parameters
-
-* `ndims` : Number of dimensions of the Cartesian structure (integer).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-`MPI_Cartdim_get` returns the number of dimensions of the Cartesian
-structure.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-# See Also
-
-[`MPI_Cart_get`(3)](MPI_Cart_get.html)
-[`MPI_Cart_create`(3)](MPI_Cart_create.html)
diff --git a/ompi/mpi/man/man3/MPI_Close_port.md b/ompi/mpi/man/man3/MPI_Close_port.md
deleted file mode 100755
index a713e6fc8b8..00000000000
--- a/ompi/mpi/man/man3/MPI_Close_port.md
+++ /dev/null
@@ -1,58 +0,0 @@
-# Name
-
-`MPI_Close_port` - Releases the specified network address.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Close_port(const char *port_name)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_CLOSE_PORT(PORT_NAME, IERROR)
-    CHARACTER*(*)	PORT_NAME
-    INTEGER		IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Close_port(port_name, ierror)
-    CHARACTER(LEN=*), INTENT(IN) :: port_name
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-
-# Input Parameter
-
-* `port_name` : A port (string).
-
-# Output Parameter
-
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-`MPI_Close_port` releases the network address represented by `port_name`.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
diff --git a/ompi/mpi/man/man3/MPI_Comm_accept.md b/ompi/mpi/man/man3/MPI_Comm_accept.md
deleted file mode 100755
index c091c597235..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_accept.md
+++ /dev/null
@@ -1,78 +0,0 @@
-# Name
-
-`MPI_Comm_accept` - Establishes communication with a client.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Comm_accept(const char *port_name, MPI_Info info, int root, MPI_Comm comm, MPI_Comm *newcomm)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_COMM_ACCEPT(PORT_NAME, INFO, ROOT, COMM, NEWCOMM, IERROR)
-    CHARACTER*(*)	PORT_NAME
-    INTEGER		INFO, ROOT, COMM, NEWCOMM, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Comm_accept(port_name, info, root, comm, newcomm, ierror)
-    CHARACTER(LEN=*), INTENT(IN) :: port_name
-    TYPE(MPI_Info), INTENT(IN) :: info
-    INTEGER, INTENT(IN) :: root
-    TYPE(MPI_Comm), INTENT(IN) :: comm
-    TYPE(MPI_Comm), INTENT(OUT) :: newcomm
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-
-# Input Parameters
-
-* `port_name` : Port name (string, used only on *root*).
-* `info` : Options given by root for the accept (handle, used only on root). No
-options currently supported.
-* `root` : Rank in *comm* of root node (integer).
-* `comm` : Intracommunicator over which call is collective (handle).
-
-# Output Parameters
-
-* `newcomm` : Intercommunicator with client as remote group (handle)
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-`MPI_Comm_accept` establishes communication with a client. It is
-collective over the calling communicator. It returns an
-intercommunicator that allows communication with the client, after the
-client has connected with the `MPI_Comm_accept` function using the
-`MPI_Comm_connect` function.
-The `port_name` must have been established through a call to
-`MPI_Open_port` on the `root`.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned.
-See the MPI man page for a full list of MPI error codes.
-
-# See Also
-
-[`MPI_Comm_connect`(3)](MPI_Comm_connect.html)
-[`MPI_Open_port`(3)](MPI_Open_port.html)
diff --git a/ompi/mpi/man/man3/MPI_Comm_c2f.3in b/ompi/mpi/man/man3/MPI_Comm_c2f.3in
deleted file mode 100644
index a13fce697dd..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_c2f.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Comm_f2c.3
diff --git a/ompi/mpi/man/man3/MPI_Comm_call_errhandler.md b/ompi/mpi/man/man3/MPI_Comm_call_errhandler.md
deleted file mode 100755
index 64bf08a93a9..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_call_errhandler.md
+++ /dev/null
@@ -1,69 +0,0 @@
-# Name
-
-`MPI_Comm_call_errhandler` - Passes the supplied error code to the
-error handler assigned to a communicator
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Comm_call_errhandler(MPI_Comm comm, int errorcode)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_COMM_CALL_ERRHANDLER(COMM, ERRORCODE, IERROR)
-    INTEGER	COMM, ERRORCODE, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Comm_call_errhandler(comm, errorcode, ierror)
-    TYPE(MPI_Comm), INTENT(IN) :: comm
-    INTEGER, INTENT(IN) :: errorcode
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-
-# Input Parameter
-
-* `comm` : communicator with error handler (handle).
-* `errorcode` : error code (integer).
-
-# Output Parameters
-
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-This function invokes the error handler assigned to the `comm`unicator
-`comm` with the supplied error code `errorcode`. If the error handler
-was successfully called, the process is not aborted, and the error
-handler returns, this function returns `MPI_SUCCESS.`
-
-# Notes
-
-Users should note that the default error handler is
-`MPI_ERRORS_ARE_FATAL`. Thus, calling this function will abort the
-processes in `comm` if the default error handler has not been changed.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-See the MPI man page for a full list of MPI error codes.
-
-# See Also
-
-[`MPI_Comm_create_errhandler`(3)](MPI_Comm_create_errhandler.html)
-[`MPI_Comm_set_errhandler`(3)](MPI_Comm_set_errhandler.html)
diff --git a/ompi/mpi/man/man3/MPI_Comm_compare.md b/ompi/mpi/man/man3/MPI_Comm_compare.md
deleted file mode 100755
index 22c0637b5f9..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_compare.md
+++ /dev/null
@@ -1,65 +0,0 @@
-# Name
-
-`MPI_Comm_compare` - Compares two communicators.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Comm_compare(MPI_Comm comm1, MPI_Comm comm2, int *result)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_COMM_COMPARE(COMM1, COMM2, RESULT, IERROR)
-    INTEGER	COMM1, COMM2, RESULT, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Comm_compare(comm1, comm2, result, ierror)
-    TYPE(MPI_Comm), INTENT(IN) :: comm1, comm2
-    INTEGER, INTENT(OUT) :: result
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-
-# Input Parameters
-
-* `comm1` : Comm1 (handle).
-* `comm2` : Comm2 (handle).
-
-# Output Parameters
-
-* `result` : Result of comparison (integer).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-`MPI_IDENT` `result`s if and only if `comm1` and `comm2` are handles for the
-same object (identical groups and same contexts). `MPI_CONGRUENT` results
-if the underlying groups are identical in constituents and rank order;
-these communicators differ only by context. `MPI_SIMILAR` results of the
-group members of both communicators are the same but the rank order
-differs. `MPI_UNEQUAL` results otherwise.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
diff --git a/ompi/mpi/man/man3/MPI_Comm_connect.md b/ompi/mpi/man/man3/MPI_Comm_connect.md
deleted file mode 100755
index 1d38e3d9353..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_connect.md
+++ /dev/null
@@ -1,86 +0,0 @@
-# Name
-
-`MPI_Comm_connect` - Establishes communication with a server.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Comm_connect(const char *port_name, MPI_Info info, int root,
-	MPI_Comm comm, MPI_Comm *newcomm)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_COMM_CONNECT(PORT_NAME, INFO, ROOT, COMM, NEWCOMM, IERROR)
-    CHARACTER*(*)	PORT_NAME
-    INTEGER		INFO, ROOT, COMM, NEWCOMM, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Comm_connect(port_name, info, root, comm, newcomm, ierror)
-    CHARACTER(LEN=*), INTENT(IN) :: port_name
-    TYPE(MPI_Info), INTENT(IN) :: info
-    INTEGER, INTENT(IN) :: root
-    TYPE(MPI_Comm), INTENT(IN) :: comm
-    TYPE(MPI_Comm), INTENT(OUT) :: newcomm
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameters
-
-* `port_name` : Port name (string, used only on *root*).
-* `info` : Options given by root for the connect (handle, used only on root).
-No options currently supported.
-* `root` : Rank in *comm* of root node (integer).
-* `comm` : Intracommunicator over which call is collective (handle).
-
-# Output Parameters
-
-* `newcomm` : Intercommunicator with client as remote group (handle)
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-`MPI_Comm_connect` establishes communication with a server specified by
-`port_name`. It is collective over the calling communicator and returns
-an intercommunicator in which the remote group participated in an
-`MPI_Comm_accept`. The `MPI_Comm_connect` call must only be called after the
-`MPI_Comm_accept` call has been made by the MPI job acting as the server.
-If the named port does not exist (or has been closed), `MPI_Comm_connect`
-raises an error of class `MPI_ERR_PORT`.
-MPI provides no guarantee of fairness in servicing connection attempts.
-That is, connection attempts are not necessarily satisfied in the order
-in which they were initiated, and competition from other connection
-attempts may prevent a particular connection attempt from being
-satisfied.
-The `port_name` parameter is the address of the server. It must be the
-same as the name returned by `MPI_Open_port` on the server.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-See the MPI man page for a full list of MPI error codes.
-
-# See Also
-
-[`MPI_Comm_accept`(3)](MPI_Comm_accept.html) 
-[`MPI_Open_port`(3)](MPI_Open_port.html)
diff --git a/ompi/mpi/man/man3/MPI_Comm_create.md b/ompi/mpi/man/man3/MPI_Comm_create.md
deleted file mode 100755
index 42daef5bf9d..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_create.md
+++ /dev/null
@@ -1,85 +0,0 @@
-# Name
-
-`MPI_Comm_create` - Creates a new communicator.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Comm_create(MPI_Comm comm, MPI_Group group, MPI_Comm *newcomm)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_COMM_CREATE(COMM, GROUP, NEWCOMM, IERROR)
-    INTEGER	COMM, GROUP, NEWCOMM, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Comm_create(comm, group, newcomm, ierror)
-    TYPE(MPI_Comm), INTENT(IN) :: comm
-    TYPE(MPI_Group), INTENT(IN) :: group
-    TYPE(MPI_Comm), INTENT(OUT) :: newcomm
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameter
-
-* `comm` : Communicator (handle).
-* `group` : Group, which is a subset of the group of comm (handle).
-
-# Output Parameters
-
-* `newcomm` : New communicator (handle).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-This function creates a new communicator `newcomm` with communication
-`group` defined by `group` and a new context. The function sets `newcomm` to
-a new communicator that spans all the processes that are in the `group`.
-It sets `newcomm` to `MPI_COMM_NULL` for processes that are not in the
-`group`.
-Each process must call with a `group` argument that is a subgroup of the
-`group` associated with `comm`; this could be `MPI_GROUP_EMPTY`. The
-processes may specify different values for the `group` argument. If a
-process calls with a non-empty `group`, then all processes in that `group`
-must call the function with the same `group` as argument, that is: the
-same processes in the same order. Otherwise the call is erroneous.
-
-# Notes
-
-`MPI_Comm_create` provides a means of making a subset of processes for the
-purpose of separate MIMD computation, with separate communication space.
-`newcomm`, which is created by `MPI_Comm_create`, can be used in
-subsequent calls to `MPI_Comm_create` (or other communicator constructors)
-to further subdivide a computation into parallel sub-computations. A
-more general service is provided by `MPI_Comm_split`.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-# See Also
-
-[`MPI_Comm_split`(3)](MPI_Comm_split.html)
-[`MPI_Intercomm_create`(3)](MPI_Intercomm_create.html)
-[`MPI_Comm_create_group`(3)](MPI_Comm_create_group.html)
diff --git a/ompi/mpi/man/man3/MPI_Comm_create_errhandler.md b/ompi/mpi/man/man3/MPI_Comm_create_errhandler.md
deleted file mode 100755
index 8fee33561a4..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_create_errhandler.md
+++ /dev/null
@@ -1,83 +0,0 @@
-# Name
-
-`MPI_Comm_create_errhandler` - Creates an error handler that can be
-attached to communicators.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Comm_create_errhandler(MPI_Comm_errhandler_function *function,
-	MPI_Errhandler *errhandler)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_COMM_CREATE_ERRHANDLER(FUNCTION, ERRHANDLER, IERROR)
-    EXTERNAL	FUNCTION
-    INTEGER	ERRHANDLER, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Comm_create_errhandler(comm_errhandler_fn, errhandler, ierror)
-    PROCEDURE(MPI_Comm_errhandler_function) :: comm_errhandler_fn
-    TYPE(MPI_Errhandler), INTENT(OUT) :: errhandler
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Deprecated Type Name Note
-
-MPI-2.2 deprecated the `MPI_Comm_errhandler_fn` and `MPI::Comm::Errhandler_fn`
-types in favor of `MPI_Comm_errhandler_function` and `MPI::Comm::Errhandler_function`,
-respectively. Open MPI supports both names (indeed, the \_fn names are typedefs 
-to the \_function names).
-
-# Input Parameter
-
-* `function` : User-defined error handling procedure (function).
-
-# Output Parameters
-
-* `errhandler` : MPI error handler (handle).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-`MPI_Comm_create_errhandler` creates an error handler that can be attached
-to communicators. This `function` is identical to `MPI_Errhandler_create`,
-the use of which is deprecated.
-In C, the user routine should be a `function` of type
-`MPI_Comm_errhandler_function`, which is defined as
-```c
-typedef void MPI_Comm_errhandler_function(MPI_Comm *, int *, ...);
-```
-The first argument is the communicator in use. The second is the error
-code to be returned by the MPI routine that raised the error. This
-typedef replaces `MPI_Handler_function`, the use of which is deprecated.
-In Fortran, the user routine should be of this form:
-```fortran
-SUBROUTINE COMM_ERRHANDLER_FUNCTION(COMM, ERROR_CODE, ...)
-   INTEGER COMM, ERROR_CODE
-```
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the `function` and Fortran routines in the last argument.
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O `function` errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
diff --git a/ompi/mpi/man/man3/MPI_Comm_create_group.md b/ompi/mpi/man/man3/MPI_Comm_create_group.md
deleted file mode 100755
index dcfa8699290..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_create_group.md
+++ /dev/null
@@ -1,94 +0,0 @@
-# Name
-
-`MPI_Comm_create_group` - Creates a new communicator.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Comm_create_group(MPI_Comm comm, MPI_Group group, int tag, MPI_Comm *newcomm)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_COMM_CREATE_GROUP(COMM, GROUP, TAG, NEWCOMM, IERROR)
-    INTEGER	COMM, GROUP, TAG, NEWCOMM, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Comm_create_group(comm, group, tag, newcomm, ierror)
-    TYPE(MPI_Comm), INTENT(IN) :: comm
-    TYPE(MPI_Group), INTENT(IN) :: group
-    INTEGER, INTENT(IN) :: tag
-    TYPE(MPI_Comm), INTENT(OUT) :: newcomm
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameters
-
-* `comm` : Communicator (handle).
-* `group` : Group, which is a subset of the group of comm (handle).
-* `tag` : Tag (integer).
-
-# Output Parameters
-
-* `newcomm` : New communicator (handle).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-`MPI_Comm_create_group` is similar to `MPI_Comm_create`; however,
-`MPI_Comm_create` must be called by all processes in the `group` of `comm`,
-whereas `MPI_Comm_create_group` must be called by all processes in `group`,
-which is a subgroup of the `group` of `comm`. In addition,
-`MPI_Comm_create_group` requires that `comm` is an intracommunicator.
-`MPI_Comm_create_group` returns a new intracommunicator, `newcomm`, for
-which the `group` argument defines the communication `group`. No cached
-information propagates from `comm` to `newcomm`.
-Each process must provide a `group` argument that is a sub`group` of the
-`group` associated with `comm`; this could be `MPI_GROUP_EMPTY`. If a
-non-empty `group` is specified, then all processes in that `group` must call
-the function, and each of these processes must provide the same
-arguments, including a `group` that contains the same members with the
-same ordering. Otherwise the call is erroneous. If the calling process
-is a member of the `group` given as the `group` argument, then `newcomm` is
-a communicator with `group` as its associated `group`. If the calling
-process is not a member of `group`, e.g., `group` is `MPI_GROUP_EMPTY`, then
-the call is a local operation and `MPI_COMM_NULL` is returned as
-`newcomm`.
-
-# Notes
-
-`MPI_Comm_create_group` provides a means of making a subset of processes
-for the purpose of separate MIMD computation, with separate
-communication space. `newcomm`, which is created by
-`MPI_Comm_create_group`, can be used in subsequent calls to
-`MPI_Comm_create_group` (or other communicator constructors) to further
-subdivide a computation into parallel sub-computations. A more general
-service is provided by `MPI_Comm_split`.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-# See Also
-
-[`MPI_Comm_create`(3)](MPI_Comm_create.html)
diff --git a/ompi/mpi/man/man3/MPI_Comm_create_keyval.md b/ompi/mpi/man/man3/MPI_Comm_create_keyval.md
deleted file mode 100755
index 260f9e3b632..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_create_keyval.md
+++ /dev/null
@@ -1,124 +0,0 @@
-# Name
-
-`MPI_Comm_create_keyval` - Generates a new attribute key.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Comm_create_keyval(MPI_Comm_copy_attr_function
-    *comm_copy_attr_fn, MPI_Comm_delete_attr_function
-    *comm_delete_attr_fn, int *comm_keyval,
-    void *extra_state)
-```
-
-## Fortran Syntax (See Fortran 77 Notes)
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_COMM_CREATE_KEYVAL(COMM_COPY_ATTR_FN, COMM_DELETE_ATTR_FN,
-    COMM_KEYVAL, EXTRA_STATE, IERROR)
-    EXTERNAL COMM_COPY_ATTR_FN, COMM_DELETE_ATTR_FN
-    INTEGER COMM_KEYVAL, IERROR 
-
-    INTEGER(KIND=MPI_ADDRESS_KIND) EXTRA_STATE
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Comm_create_keyval(comm_copy_attr_fn, comm_delete_attr_fn, comm_keyval,
-    	extra_state, ierror)
-    PROCEDURE(MPI_Comm_copy_attr_function) :: comm_copy_attr_fn
-    PROCEDURE(MPI_Comm_delete_attr_function) :: comm_delete_attr_fn
-    INTEGER, INTENT(OUT) :: comm_keyval
-    INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) :: extra_state
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameters
-
-* `comm_copy_attr_fn` : Copy callback function for `comm_keyval` (function).
-* `comm_delete_attr_fn` : Delete callback function for `comm_keyval` (function).
-* `extra_state` : Extra state for callback functions.
-
-# Output Parameter
-
-* `comm_keyval` : Key value for future access (integer).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-This function replaces `MPI_Keyval_create`, the use of which is
-deprecated. The C binding is identical. The Fortran binding differs in
-that `extra_state` is an address-sized integer. Also, the copy and
-delete callback functions have Fortran bindings that are consistent with
-address-sized attributes.
-The argument `comm_copy_attr_fn` may be specified as
-`MPI_COMM_NULL_COPY_FN` or `MPI_COMM_DUP_FN` from C or Fortran.
-`MPI_COMM_NULL_COPY_FN` is a function that does nothing more than
-returning `flag = 0` and `MPI_SUCCESS`. `MPI_COMM_DUP_FN` is a simple-minded
-copy function that sets `flag = 1`, returns the value of
-`attribute_val_in` in `attribute_val_out`, and returns `MPI_SUCCESS`.
-These replace the MPI-1 predefined callbacks `MPI_NULL_COPY_FN` and
-`MPI_DUP_FN`, the use of which is deprecated.
-The two C callback functions are:
-```c
-typedef int MPI_Comm_copy_attr_function(MPI_Comm oldcomm, int comm_keyval,
-             void *extra_state, void *attribute_val_in,
-             void *attribute_val_out, int *flag);
-
-typedef int MPI_Comm_delete_attr_function(MPI_Comm comm, int comm_keyval,
-             void *attribute_val, void *extra_state);
-```
-which are the same as the MPI-1.1 calls but with a new name. The old
-names are deprecated.
-The two Fortran callback functions are:
-```fortran
-SUBROUTINE COMM_COPY_ATTR_FN(OLDCOMM, COMM_KEYVAL, EXTRA_STATE,
-             ATTRIBUTE_VAL_IN, ATTRIBUTE_VAL_OUT, FLAG, IERROR)
-    INTEGER OLDCOMM, COMM_KEYVAL, IERROR
-
-    INTEGER(KIND=MPI_ADDRESS_KIND) EXTRA_STATE, ATTRIBUTE_VAL_IN,
-        ATTRIBUTE_VAL_OUT
-    LOGICAL FLAG
-
-SUBROUTINE COMM_DELETE_ATTR_FN(COMM, COMM_KEYVAL, ATTRIBUTE_VAL, EXTRA_STATE,
-             IERROR)
-    INTEGER COMM, COMM_KEYVAL, IERROR
-
-    INTEGER(KIND=MPI_ADDRESS_KIND) ATTRIBUTE_VAL, EXTRA_STATE
-```
-
-## Fortran 77 Notes
-
-The MPI standard prescribes portable Fortran syntax for the
-`EXTRA_STATE` argument only for Fortran 90. FORTRAN 77 users may use the
-non-portable syntax
-```fortran
-INTEGER*MPI_ADDRESS_KIND EXTRA_STATE
-```
-where `MPI_ADDRESS_KIND` is a constant defined in mpif.h and gives the
-length of the declared integer in bytes.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-See the MPI man page for a full list of MPI error codes.
-SEE ALSO
-
-# See Also
diff --git a/ompi/mpi/man/man3/MPI_Comm_delete_attr.md b/ompi/mpi/man/man3/MPI_Comm_delete_attr.md
deleted file mode 100755
index 2b10bb1f2f6..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_delete_attr.md
+++ /dev/null
@@ -1,81 +0,0 @@
-# Name
-
-`MPI_Comm_delete_attr` - Deletes attribute value associated with a
-key.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Comm_delete_attr(MPI_Comm comm, int comm_keyval)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_COMM_DELETE_ATTR(COMM, COMM_KEYVAL, IERROR)
-    INTEGER	COMM, COMM_KEYVAL, IERROR 
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Comm_delete_attr(comm, comm_keyval, ierror)
-    TYPE(MPI_Comm), INTENT(IN) :: comm
-    INTEGER, INTENT(IN) :: comm_keyval
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-## Input/Output Parameter
-
-* `comm` : Communicator from which the attribute is deleted (handle).
-
-# Input Parameter
-
-* `comm_keyval` : Key value (integer).
-
-# Output Parameter
-
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-`MPI_Comm_delete_attr` deletes an attribute from cache by key. This
-function invokes the attribute delete function `delete_fn` specified when
-the `comm_keyval` was created. The call will fail if the `delete_fn`
-function returns an error code other than `MPI_SUCCESS.`
-Whenever a communicator is replicated using the function `MPI_Comm_dup`,
-all callback copy functions for attributes that are currently set are
-invoked (in arbitrary order). Whenever a communicator is deleted using
-the function `MPI_Comm_free`, all callback delete functions for attributes
-that are currently set are invoked.
-This function is the same as `MPI_Attr_delete` but is needed to match the
-`comm`unicator-specific functions introduced in the MPI-2 standard. The
-use of `MPI_Attr_delete` is deprecated.
-
-# Notes
-
-Note that it is not defined by the MPI standard what happens if the
-`delete_fn` callback invokes other MPI functions. In Open MPI, it is not
-valid for `delete_fn` callbacks (or any of their children) to add or
-delete attributes on the same object on which the `delete_fn` callback is
-being invoked.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
diff --git a/ompi/mpi/man/man3/MPI_Comm_disconnect.md b/ompi/mpi/man/man3/MPI_Comm_disconnect.md
deleted file mode 100755
index a687499ebd1..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_disconnect.md
+++ /dev/null
@@ -1,80 +0,0 @@
-# Name
-
-`MPI_Comm_disconnect` - Deallocates communicator object and sets
-handle to `MPI_COMM_NULL.`
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Comm_disconnect(MPI_Comm *comm)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_COMM_DISCONNECT(COMM, IERROR)
-    INTEGER	COMM, IERROR 
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Comm_disconnect(comm, ierror)
-    TYPE(MPI_Comm), INTENT(INOUT) :: comm
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-## Input/Output Parameter
-
-* `comm` : Communicator (handle).
-
-# Output Parameter
-
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-`MPI_Comm_disconnect` waits for all pending communication on `comm` to
-complete internally, deallocates the communicator object, and sets the
-handle to `MPI_COMM_NULL`. It is a collective operation.
-It may not be called with the communicator `MPI_COMM_WORLD` or
-`MPI_COMM_SELF.`
-`MPI_Comm_disconnect` may be called only if all communication is complete
-and matched, so that buffered data can be delivered to its destination.
-This requirement is the same as for `MPI_Finalize.`
-`MPI_Comm_disconnect` has the same action as `MPI_Comm_free`, except that it
-waits for pending communication to finish internally and enables the
-guarantee about the behavior of disconnected processes.
-
-# Notes
-
-To disconnect two processes you may need to call `MPI_Comm_disconnect,`
-`MPI_Win_free`, and `MPI_File_close` to remove all communication paths
-between the two processes. Note that it may be necessary to disconnect
-several communicators (or to free several windows or files) before two
-processes are completely independent.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-# See Also
-
-[`MPI_Comm_connect`(3)](MPI_Comm_connect.html)
-[`MPI_Comm_accept`(3)](MPI_Comm_accept.html)
diff --git a/ompi/mpi/man/man3/MPI_Comm_dup.md b/ompi/mpi/man/man3/MPI_Comm_dup.md
deleted file mode 100755
index 8c0938c0e03..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_dup.md
+++ /dev/null
@@ -1,89 +0,0 @@
-# Name
-
-`MPI_Comm_dup` - Duplicates an existing communicator with all its
-cached information.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_COMM_DUP(COMM, NEWCOMM, IERROR)
-    INTEGER	COMM, NEWCOMM, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Comm_dup(comm, newcomm, ierror)
-    TYPE(MPI_Comm), INTENT(IN) :: comm
-    TYPE(MPI_Comm), INTENT(OUT) :: newcomm
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameter
-
-* `comm` : Communicator (handle).
-
-# Output Parameters
-
-* `newcomm` : Copy of comm (handle).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-`MPI_Comm_dup` duplicates the existing communicator `comm` with associated
-key values. For each key value, the respective copy callback function
-determines the attribute value associated with this key in the new
-communicator; one particular action that a copy callback may take is to
-delete the attribute from the new communicator. Returns in newcomm a new
-communicator with the same group, any copied cached information, but a
-new context (see Section 5.7.1 of the MPI-1 Standard,
-"Functionality").
-
-# Notes
-
-This operation is used to provide a parallel library call with a
-duplicate communication space that has the same properties as the
-original communicator. This includes any attributes (see below) and
-topologies (see Chapter 6, "Process Topologies," in the MPI-1
-Standard). This call is valid even if there are pending point-to-point
-communications involving the communicator `comm`. A typical call might
-involve an `MPI_Comm_dup` at the beginning of the parallel call, and an
-`MPI_Comm_free` of that duplicated communicator at the end of the call.
-Other models of communicator management are also possible.
-This call applies to both intra- and intercommunicators.
-Note that it is not defined by the MPI standard what happens if the
-attribute copy callback invokes other MPI functions. In Open MPI, it is
-not valid for attribute copy callbacks (or any of their children) to add
-or delete attributes on the same object on which the attribute copy
-callback is being invoked.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-# See Also
-
-[`MPI_Comm_dup_with_info`(3)](MPI_Comm_dup_with_info.html)
-[`MPI_Comm_idup`(3)](MPI_Comm_idup.html)
diff --git a/ompi/mpi/man/man3/MPI_Comm_dup_with_info.md b/ompi/mpi/man/man3/MPI_Comm_dup_with_info.md
deleted file mode 100755
index 63102673ec5..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_dup_with_info.md
+++ /dev/null
@@ -1,89 +0,0 @@
-# Name
-
-`MPI_Comm_dup_with_info` - Duplicates an existing communicator using
-provided info.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Comm_dup_with_info(MPI_Comm comm, MPI_Info info, MPI_Comm *newcomm)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_COMM_DUP_WITH_INFO(COMM, INFO, NEWCOMM, IERROR)
-    INTEGER	COMM, INFO, NEWCOMM, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Comm_dup_with_info(comm, info, newcomm, ierror)
-    TYPE(MPI_Comm), INTENT(IN) :: comm
-    TYPE(MPI_Info), INTENT(IN) :: info
-    TYPE(MPI_Comm), INTENT(OUT) :: newcomm
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameter
-
-* `comm` : Communicator (handle).
-* `info` : Info argument (handle).
-
-# Output Parameters
-
-* `newcomm` : Copy of comm (handle).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-`MPI_Comm_dup_with_info` acts exactly like `MPI_Comm_dup` except that the
-`info` hints associated with the communicator `comm` are not duplicated in
-`newcomm`. The hints provided by the argument `info` are associated with
-the output communicator `newcomm` instead.
-See `MPI_Comm_set_info(3)` for the list of recognized `info` keys.
-
-# Notes
-
-This operation is used to provide a parallel library call with a
-duplicate communication space that has the same properties as the
-original communicator. This includes any attributes (see below) and
-topologies (see Chapter 6, "Process Topologies," in the MPI-1
-Standard). This call is valid even if there are pending point-to-point
-communications involving the communicator `comm`. A typical call might
-involve an `MPI_Comm_dup_with_info` at the beginning of the parallel call,
-and an `MPI_Comm_free` of that duplicated communicator at the end of the
-call. Other models of communicator management are also possible.
-This call applies to both intra- and intercommunicators.
-Note that it is not defined by the MPI standard what happens if the
-attribute copy callback invokes other MPI functions. In Open MPI, it is
-not valid for attribute copy callbacks (or any of their children) to add
-or delete attributes on the same object on which the attribute copy
-callback is being invoked.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-# See Also
-
-[`MPI_Comm_dup`(3)](MPI_Comm_dup.html)
-[`MPI_Comm_idup`(3)](MPI_Comm_idup.html)
-[`MPI_Comm_set_info`(3)](MPI_Comm_set_info.html)
diff --git a/ompi/mpi/man/man3/MPI_Comm_f2c.3in b/ompi/mpi/man/man3/MPI_Comm_f2c.3in
deleted file mode 100644
index 687990fb9f2..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_f2c.3in
+++ /dev/null
@@ -1,49 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Comm_f2c 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Comm_f2c, MPI_Comm_c2f, MPI_File_f2c, MPI_File_c2f, MPI_Info_f2c, MPI_Info_c2f, MPI_Message_f2c, MPI_Message_c2f, MPI_Op_f2c, MPI_Op_c2f, MPI_Request_f2c, MPI_Request_c2f, MPI_Type_f2c, MPI_Type_c2f, MPI_Win_f2c, MPI_Win_c2f \fP \- Translates a C handle into a Fortran handle, or vice versa.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-MPI_Comm MPI_Comm_f2c(MPI_Fint \fIcomm\fP)
-MPI_Fint MPI_Comm_c2f(MPI_Comm \fIcomm\fP)
-
-MPI_File MPI_File_f2c(MPI_Fint \fIfile\fP)
-MPI_Fint MPI_File_c2f(MPI_File \fIfile\fP)
-
-MPI_Group MPI_Group_f2c(MPI Fint \fIgroup\fP)
-MPI_Fint MPI_Group_c2f(MPI Group \fIgroup\fP)
-
-MPI_Info MPI_Info_f2c(MPI_Fint \fIinfo\fP)
-MPI_Fint MPI_Info_c2f(MPI_Info \fIinfo\fP)
-
-MPI_Message MPI_Message_f2c(MPI_Fint \fImessage\fP)
-MPI_Fint MPI_Message_c2f(MPI_Message \fImessage\fP)
-
-MPI_Op MPI_Op_f2c(MPI_Fint \fIop\fP)
-MPI_Fint MPI_Op_c2f(MPI_Op \fIop\fP)
-
-MPI_Request MPI_Request_f2c(MPI_Fint \fIrequest\fP)
-MPI_Fint MPI_Request_c2f(MPI_Request \fIrequest\fP)
-
-MPI_Datatype MPI_Type_f2c(MPI_Fint \fIdatatype\fP)
-MPI_Fint MPI_Type_c2f(MPI_Datatype \fIdatatype\fP)
-
-MPI_Win MPI_Win_f2c(MPI_Fint \fIwin\fP)
-MPI_Fint MPI_Win_c2f(MPI_Win \fIwin\fP)
-
-.fi
-.SH DESCRIPTION
-.ft R
-Handles are passed between Fortran and C by using an explicit C wrapper to convert Fortran handles to C handles. There is no direct access to C handles in Fortran. The type definition \fIMPI_Fint\fP is provided in C for an integer of the size that matches a Fortran \fIINTEGER\fP; usually, \fIMPI_Fint\fP will be equivalent to \fIint\fP. The handle translation functions are provided in C to convert from a Fortran handle (which is an integer) to a C handle, and vice versa.
-.PP
-For example, if \fIcomm\fP is a valid Fortran handle to a communicator, then MPI_Comm_f2c returns a valid C handle to that same communicator; if \fIcomm\fP = MPI_COMM_NULL (Fortran value), then MPI_Comm_f2c returns a null C handle; if \fIcomm\fP is an invalid Fortran handle, then MPI_Comm_f2c returns an invalid C handle.
-.SH NOTE
-This function does not return an error value. Consequently, the result of calling it before MPI_Init or after MPI_Finalize is undefined.
diff --git a/ompi/mpi/man/man3/MPI_Comm_free.3in b/ompi/mpi/man/man3/MPI_Comm_free.3in
deleted file mode 100644
index 9fa51cd0a7b..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_free.3in
+++ /dev/null
@@ -1,69 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Comm_free 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Comm_free \fP \- Mark a communicator object for deallocation.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Comm_free(MPI_Comm *\fIcomm\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_COMM_FREE(\fICOMM, IERROR\fP)
-	INTEGER	\fICOMM, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Comm_free(\fIcomm\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(INOUT) :: \fIcomm\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-comm
-Communicator to be destroyed (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-This operation marks the communicator object for deallocation. The handle is set to MPI_COMM_NULL. Any pending operations that use this communicator will complete normally; the object is actually deallocated only if there are no other active references to it. This call applies to intracommunicators and intercommunicators. Upon actual deallocation, the delete callback functions for all cached attributes (see Section 5.7 in the MPI-1 Standard, "Caching") are called in arbitrary order.
-
-
-.SH NOTES
-Note that it is not defined by the MPI standard what happens if the
-delete_fn callback invokes other MPI functions.  In Open MPI, it is
-not valid for delete_fn callbacks (or any of their children) to add or
-delete attributes on the same object on which the delete_fn callback
-is being invoked.
-
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Comm_delete_attr
diff --git a/ompi/mpi/man/man3/MPI_Comm_free_keyval.3in b/ompi/mpi/man/man3/MPI_Comm_free_keyval.3in
deleted file mode 100644
index 38abe527116..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_free_keyval.3in
+++ /dev/null
@@ -1,63 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Comm_free_keyval 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Comm_free_keyval\fP \- Frees attribute key for communicator cache attribute.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Comm_free_keyval(int *\fIcomm_keyval\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_COMM_FREE_KEYVAL(\fICOMM_KEYVAL, IERROR\fP)
-	INTEGER	\fICOMM_KEYVAL, IERROR \fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Comm_free_keyval(\fIcomm_keyval\fP, \fIierror\fP)
-	INTEGER, INTENT(INOUT) :: \fIcomm_keyval\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-comm_keyval
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-
-.SH DESCRIPTION
-.ft R
-MPI_Comm_free_keyval frees an extant attribute key. This function sets the value of \fIkeyval\fP to  MPI_KEYVAL_INVALID. Note that it is not erroneous to free an attribute key that is in use, because the actual free does not transpire until after all references (in other communicators on the process) to the key have been freed. These references need to be explicitly freed by the program, either via calls to MPI_Comm_delete_attr that free one attribute instance, or by calls to MPI_Comm_free that free all attribute instances associated with the freed communicator.
-.sp
-This call is identical to the call MPI_Keyval_free but is needed to match the  communicator-specific creation function introduced in the MPI-2 standard. The use of MPI_Keyval_free is deprecated.
-
-
-.SH NOTES
-.ft R
-Key values are global (they can be used with any and all communicators).
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_Comm_get_attr.3in b/ompi/mpi/man/man3/MPI_Comm_get_attr.3in
deleted file mode 100644
index 50766d01f9e..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_get_attr.3in
+++ /dev/null
@@ -1,86 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Comm_get_attr 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Comm_get_attr\fP \- Retrieves attribute value by key.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Comm_get_attr(MPI_Comm \fIcomm\fP, int \fIcomm_keyval\fP,
-	void *\fIattribute_val\fP, int *\fIflag\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_COMM_GET_ATTR(\fICOMM, COMM_KEYVAL, ATTRIBUTE_VAL, FLAG, IERROR\fP)
-	INTEGER	\fICOMM, COMM_KEYVAL, IERROR \fP
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fIATTRIBUTE_VAL\fP
-	LOGICAL \fIFLAG\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Comm_get_attr(\fIcomm\fP, \fIcomm_keyval\fP, \fIattribute_val\fP, \fIflag\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, INTENT(IN) :: \fIcomm_keyval\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(OUT) :: \fIattribute_val\fP
-	LOGICAL, INTENT(OUT) :: \fIflag\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-comm
-Communicator to which the attribute is attached (handle).
-.TP 1i
-comm_keyval
-Key value (integer).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-attribute_val
-Attribute value, unless f\fIlag\fP = false.
-.TP 1i
-flag
-False if no attribute is associated with the key (logical).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Comm_get_attr retrieves an attribute value by key. The call is erroneous if there is no key with value \fIkeyval\fP. On the other hand, the call is correct if the key value exists, but no attribute is attached on \fIcomm\fP for that key; in that case, the call returns \fIflag\fP = false. In particular, MPI_KEYVAL_INVALID is an erroneous key value.
-.sp
-This function replaces MPI_Attr_get, the use of which is deprecated. The C binding is identical. The Fortran binding differs in that \fIattribute_val\fP is an address-sized integer.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fIATTRIBUTE_VAL\fP argument only for Fortran 90. Sun FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_ADDRESS_KIND \fIATTRIBUTE_VAL\fP
-.fi
-.sp
-where MPI_ADDRESS_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_Comm_get_errhandler.3in b/ompi/mpi/man/man3/MPI_Comm_get_errhandler.3in
deleted file mode 100644
index ec5894569c2..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_get_errhandler.3in
+++ /dev/null
@@ -1,65 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Comm_get_errhandler 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Comm_get_errhandler \fP \- Retrieves error handler associated with a communicator.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Comm_get_errhandler(MPI_Comm \fIcomm\fP,
-	MPI_Errhandler *\fIerrhandler\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_COMM_GET_ERRHANDLER(\fICOMM, ERRHANDLER, IERROR\fP)
-	INTEGER	\fICOMM, ERRHANDLER, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Comm_get_errhandler(\fIcomm\fP, \fIerrhandler\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Errhandler), INTENT(OUT) :: \fIerrhandler\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-errhandler
-New error handler for communicator (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Comm_get_errhandler retrieves the error handler currently associated with a communicator. This call is identical to MPI_Errhandler_get, the use of which is deprecated.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-.sp
-See the MPI man page for a full list of MPI error codes.
-
-
diff --git a/ompi/mpi/man/man3/MPI_Comm_get_info.3in b/ompi/mpi/man/man3/MPI_Comm_get_info.3in
deleted file mode 100644
index 09067e00126..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_get_info.3in
+++ /dev/null
@@ -1,76 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines
-.\" $COPYRIGHT$
-.TH MPI_Comm_get_info 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Comm_get_info\fP \- Retrieves active communicator info hints
-.
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Comm_get_info(MPI_Comm \fIcomm\fP, MPI_Info \fI*info_used\fP)
-.
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_COMM_GET_INFO(\fICOMM, INFO_USED, IERROR\fP)
-	INTEGER	\fICOMM, INFO_USED, IERROR \fP
-.
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Comm_get_info(\fIcomm\fP, \fIinfo_used\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Info), INTENT(OUT) :: \fIinfo_used\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-comm
-Communicator from which to receive active info hints
-.
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-info_used
-New info object returned with all active hints on this communicator.
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-.
-.SH DESCRIPTION
-.ft R
-MPI_Comm_get_info returns a new info object containing the hints of
-the communicator associated with
-.IR comm .
-The current setting of all hints actually used by the system related
-to this communicator is returned in
-.IR info_used .
-If no such hints exist, a handle to a newly created info object is
-returned that contains no key/value pair. The user is responsible for
-freeing info_used via MPI_Info_free.
-.
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler
-MPI_ERRORS_RETURN may be used to cause error values to be
-returned. Note that MPI does not guarantee that an MPI program can
-continue past an error.
-.
-.SH SEE ALSO
-MPI_Comm_get_info,
-MPI_Info_free
diff --git a/ompi/mpi/man/man3/MPI_Comm_get_name.3in b/ompi/mpi/man/man3/MPI_Comm_get_name.3in
deleted file mode 100644
index 3ff46aab501..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_get_name.3in
+++ /dev/null
@@ -1,74 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Comm_get_name 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Comm_get_name\fP \- Returns the name that was most recently associated with a communicator.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Comm_get_name(MPI_Comm \fIcomm\fP, char *\fIcomm_name\fP, int *\fIresultlen\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_COMM_GET_NAME(\fICOMM, COMM_NAME, RESULTLEN, IERROR\fP)
-	INTEGER	\fICOMM, RESULTLEN, IERROR \fP
-	CHARACTER*(*) \fICOMM_NAME\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Comm_get_name(\fIcomm\fP, \fIcomm_name\fP, \fIresultlen\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	CHARACTER(LEN=MPI_MAX_OBJECT_NAME), INTENT(OUT) :: \fIcomm_name\fP
-	INTEGER, INTENT(OUT) :: \fIresultlen\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-comm
-Communicator the name of which is to be returned (handle).
-.TP 1i
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-comm_name
-Name previously stored on the communicator, or an empty string if no such name exists (string).
-.TP 1i
-resultlen
-Length of returned name (integer).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Comm_get_name returns the last name that was previously associated with the given communicator. The name may be set and retrieved from any language. The same name will be returned independent of the language used. \fIcomm_name\fP should be allocated so that it can hold a resulting string of length MPI_MAX_OBJECT_NAME characters. MPI_Comm_get_name returns a copy of the set name in \fIcomm_name\fP.
-.sp
-If the user has not associated a name with a communicator, or an error occurs, MPI_Comm_get_name will return an empty string (all spaces in Fortran, "" in C). The three predefined communicators will have predefined names associated with them. Thus, the names of MPI_COMM_WORLD, MPI_COMM_SELF, and MPI_COMM_PARENT will have the default of MPI_COMM_WORLD, MPI_COMM_SELF, and MPI_COMM_PARENT. The fact that the system may have chosen to give a default name to a communicator does not prevent the user from setting a name on the same communicator; doing this removes the old name and assigns the new one.
-
-.SH NOTES
-.ft R
-It is safe simply to print the string returned by MPI_Comm_get_name, as it is always a valid string even if there was no name.
-.sp
-Note that associating a name with a communicator has no effect on the semantics of an MPI program, and will (necessarily) increase the store requirement of the program, since the names must be saved. Therefore, there is no requirement that users use these functions to associate names with communicators. However debugging and profiling MPI applications may be made easier if names are associated with communicators, since the debugger or profiler should then be able to present information in a less cryptic manner.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_Comm_get_parent.3in b/ompi/mpi/man/man3/MPI_Comm_get_parent.3in
deleted file mode 100644
index faf3134e378..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_get_parent.3in
+++ /dev/null
@@ -1,68 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Comm_get_parent 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Comm_get_parent\fP \- Returns the parent intercommunicator of current spawned process.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Comm_get_parent(MPI_Comm *\fIparent\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_COMM_GET_PARENT(\fIPARENT, IERROR\fP)
-	INTEGER	\fIPARENT, IERROR \fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Comm_get_parent(\fIparent\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(OUT) :: \fIparent\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-parent
-The parent communicator (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-If a process was started with MPI_Comm_spawn or MPI_Comm_spawn_multiple, MPI_Comm_get_parent returns the "parent" intercommunicator of the current process. This parent intercommunicator is created implicitly inside of MPI_Init and is the same intercommunicator returned by the spawn call made in the parents.
-.sp
-If the process was not spawned, MPI_Comm_get_parent returns MPI_COMM_NULL.
-.sp
-After the parent communicator is freed or disconnected, MPI_Comm_get_parent returns MPI_COMM_NULL.
-
-.SH NOTES
-.ft R
-MPI_Comm_get_parent returns a handle to a single intercommunicator. Calling MPI_Comm_get_parent a second time returns a handle to the same intercommunicator. Freeing the handle with MPI_Comm_disconnect or MPI_Comm_free will cause other references to the intercommunicator to become invalid (dangling). Note that calling MPI_Comm_free on the parent communicator is not useful.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-.nf
-MPI_Comm_spawn
-MPI_Comm_spawn_multiple
-
diff --git a/ompi/mpi/man/man3/MPI_Comm_group.3in b/ompi/mpi/man/man3/MPI_Comm_group.3in
deleted file mode 100644
index 421b83d2c21..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_group.3in
+++ /dev/null
@@ -1,61 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Comm_group 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Comm_group \fP \- Returns the group associated with a communicator.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Comm_group(MPI_Comm \fIcomm\fP, MPI_Group *\fIgroup\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_COMM_GROUP(\fICOMM, GROUP, IERROR\fP)
-  	INTEGER	\fICOMM, GROUP, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Comm_group(\fIcomm\fP, \fIgroup\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Group), INTENT(OUT) :: \fIgroup\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-comm
-Communicator.
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-group
-Group in communicator (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-If the communicator is an intercommunicator (enables communication between two groups of processes), this function returns the local group.  To return the remote group, use the MPI_Comm_remote_group function.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_Comm_idup.3in b/ompi/mpi/man/man3/MPI_Comm_idup.3in
deleted file mode 100644
index b13dea4066b..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_idup.3in
+++ /dev/null
@@ -1,85 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" $COPYRIGHT$
-.TH MPI_Comm_idup 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Comm_idup \fP \-  Start the nonblocking duplication of an existing communicator with all its cached information.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Comm_idup(MPI_Comm \fIcomm\fP, MPI_Comm\fI *newcomm\fP, MPI_Request\fI *request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_COMM_IDUP(\fICOMM, NEWCOMM, REQUEST, IERROR\fP)
-	INTEGER	\fICOMM, NEWCOMM, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Comm_idup(\fIcomm\fP, \fInewcomm\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Comm), INTENT(OUT) :: \fInewcomm\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-newcomm
-Copy of comm (handle).
-.ft R
-.TP 1i
-request
-Communication request (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Comm_idup starts the nonblocking duplication of an existing communicator comm with associated key
-values. For each key value, the respective copy callback function determines the attribute value associated with this key in the new communicator; one particular action that a copy callback may take is to delete the attribute from the new communicator. Returns in newcomm a new communicator with the same group, any copied cached information, but a new context (see Section 5.7.1 of the MPI-1 Standard, "Functionality"). The communicator returned in \fInewcomm\fP will not be available until the request is complete.
-.sp
-The completion of a communicator duplication request can be determined by calling any of MPI_Wait, MPI_Waitany, MPI_Test, or MPI_Testany with the request returned by this function.
-
-.SH NOTES
-This operation is used to provide a parallel
-library call with a duplicate communication space that has the same properties as the original communicator. This includes any attributes (see below) and topologies (see Chapter 6, "Process Topologies," in the MPI-1 Standard). This call is valid even if there are pending point-to-point communications involving the communicator comm. A typical call might involve an MPI_Comm_idup at the beginning of the parallel call, and an MPI_Comm_free of that duplicated communicator at the end of the call. Other models of communicator management are also possible.
-.sp
-This call applies to both intra- and intercommunicators.
-
-Note that it is not defined by the MPI standard what happens if the
-attribute copy callback invokes other MPI functions.  In Open MPI, it
-is not valid for attribute copy callbacks (or any of their children)
-to add or delete attributes on the same object on which the attribute
-copy callback is being invoked.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-
-.SH SEE ALSO
-MPI_Comm_dup
-MPI_Comm_dup_with_info
diff --git a/ompi/mpi/man/man3/MPI_Comm_join.3in b/ompi/mpi/man/man3/MPI_Comm_join.3in
deleted file mode 100644
index ae493c9c783..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_join.3in
+++ /dev/null
@@ -1,104 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Comm_join 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-
-.SH NAME
-\fBMPI_Comm_join\fP \- Establishes communication between MPI jobs
-
-.SH SYNTAX
-.ft R
-
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Comm_join(int \fIfd\fP, MPI_Comm *\fIintercomm\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_COMM_JOIN(\fIFD, INTERCOMM, IERROR\fP)
-	INTEGER	\fIFD, INTERCOMM, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Comm_join(\fIfd\fP, \fIintercomm\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIfd\fP
-	TYPE(MPI_Comm), INTENT(OUT) :: \fIintercomm\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-fd
-socket file descriptor (socket).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-intercomm
-Intercommunicator between processes (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Comm_join creates an intercommunicator from the union of two MPI
-processes that are connected by a socket. \fIfd\fP is a file
-descriptor representing a socket of type SOCK_STREAM (a two-way
-reliable byte-stream connection). Nonblocking I/O and asynchronous
-notification via SIGIO must not be enabled for the socket. The socket
-must be in a connected state, and must be quiescent when MPI_Comm_join
-is called.
-.sp
-MPI_Comm_join must be called by the process at each end of the
-socket. It does not return until both processes have called
-MPI_Comm_join.
-
-.SH NOTES
-.ft R
-There are no MPI library calls for opening and manipulating a socket.
-The socket \fIfd\fP can be opened using standard socket API calls.
-MPI uses the socket to bootstrap creation of the intercommunicator,
-and for nothing else. Upon return, the file descriptor will be open
-and quiescent.
-.sp
-In a multithreaded process, the application must ensure that other
-threads do not access the socket while one is in the midst of
-calling MPI_Comm_join.
-.sp
-The returned communicator will contain the two processes connected by
-the socket, and may be used to establish MPI communication with
-additional processes, through the usual MPI communicator-creation
-mechanisms.
-
-.SH ERRORS
-.ft R
-Almost all MPI routines return an error value; C routines as
-the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-.sp
-See the MPI man page for a full list of MPI error codes.
-
-.SH SEE ALSO
-.ft R
-.nf
-socket(3SOCKET)
-MPI_Comm_create
-MPI_Comm_group
-
diff --git a/ompi/mpi/man/man3/MPI_Comm_rank.3in b/ompi/mpi/man/man3/MPI_Comm_rank.3in
deleted file mode 100644
index 7c8d0c0386b..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_rank.3in
+++ /dev/null
@@ -1,73 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Comm_rank 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Comm_rank\fP \- Determines the rank of the calling process in the communicator.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Comm_rank(MPI_Comm \fIcomm\fP, int\fI *rank\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_COMM_RANK(\fICOMM, RANK, IERROR\fP)
-	INTEGER	\fICOMM, RANK, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Comm_rank(\fIcomm\fP, \fIrank\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, INTENT(OUT) :: \fIrank\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-rank
-Rank of the calling process in group of comm (integer).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-This function gives the rank of the process in the
-particular communicator's group. It is equivalent to accessing the
-communicator's group with MPI_Comm_group, computing the rank using MPI_Group_rank, and then freeing the temporary group via MPI_Group_free.
-.sp
-Many programs will be written with the master-slave model, where one process (such as the rank-zero process) will play a supervisory role, and the other processes will serve as compute nodes. In this framework, MPI_Comm_size and MPI_Comm_rank are useful for determining the roles of the various processes of a communicator.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-.nf
-MPI_Comm_group
-MPI_Comm_size
-MPI_Comm_compare
-
diff --git a/ompi/mpi/man/man3/MPI_Comm_remote_group.3in b/ompi/mpi/man/man3/MPI_Comm_remote_group.3in
deleted file mode 100644
index 6a81efc8c0a..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_remote_group.3in
+++ /dev/null
@@ -1,72 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Comm_remote_group 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Comm_remote_group \fP \- Accesses the remote group associated with an intercommunicator.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Comm_remote_group(MPI_Comm \fIcomm\fP, MPI_Group\fI *group\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_COMM_REMOTE_GROUP(\fICOMM, GROUP, IERROR\fP)
-	INTEGER	\fICOMM, GROUP, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Comm_remote_group(\fIcomm\fP, \fIgroup\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Group), INTENT(OUT) :: \fIgroup\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-comm
-Communicator.
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-group
-Remote group of communicator.
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Comm_remote_group accesses the remote group associated with an intercommunicator.
-.sp
-The intercommunicator accessors (MPI_Comm_test_inter, MPI_Comm_remote_size,
-MPI_Comm_remote_group) are all local operations.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.sp
-.nf
-MPI_Comm_test_inter
-MPI_Comm_remote_size
-MPI_Intercomm_create
-MPI_Intercomm_merge
-
diff --git a/ompi/mpi/man/man3/MPI_Comm_remote_size.3in b/ompi/mpi/man/man3/MPI_Comm_remote_size.3in
deleted file mode 100644
index 37b8e3b818f..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_remote_size.3in
+++ /dev/null
@@ -1,72 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Comm_remote_size 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Comm_remote_size \fP \- Determines the size of the remote group associated with an intercommunicator.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Comm_remote_size(MPI_Comm \fIcomm\fP, int\fI *size\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_COMM_REMOTE_SIZE(\fICOMM, SIZE, IERROR\fP)
-	INTEGER	\fICOMM, SIZE, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Comm_remote_size(\fIcomm\fP, \fIsize\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, INTENT(OUT) :: \fIsize\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-size
-Number of processes in the remote group of comm (integer).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Comm_remote_size determines the size of the remote group associated with an intercommunicator.
-.sp
-The  intercommunicator accessors (MPI_Comm_test_inter, MPI_Comm_remote_size, MPI_Comm_remote_group) are all local operations.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-.nf
-MPI_Comm_test_inter
-MPI_Comm_remote_group
-MPI_Intercomm_create
-MPI_Intercomm_merge
-
diff --git a/ompi/mpi/man/man3/MPI_Comm_set_attr.3in b/ompi/mpi/man/man3/MPI_Comm_set_attr.3in
deleted file mode 100644
index ec79d39e7e6..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_set_attr.3in
+++ /dev/null
@@ -1,92 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Comm_set_attr 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Comm_set_attr\fP \- Stores attribute value associated with a key.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Comm_set_attr(MPI_Comm \fIcomm\fP, int \fIcomm_keyval\fP, void *\fIattribute_val\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_COMM_SET_ATTR(\fICOMM, COMM_KEYVAL, ATTRIBUTE_VAL, IERROR\fP)
-	INTEGER	\fICOMM, COMM_KEYVAL, IERROR \fP
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fIATTRIBUTE_VAL\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Comm_set_attr(\fIcomm\fP, \fIcomm_keyval\fP, \fIattribute_val\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, INTENT(IN) :: \fIcomm_keyval\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) :: \fIattribute_val\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-comm
-Communicator from which attribute will be attached (handle).
-
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-comm_keyval
-Key value (integer).
-.TP 1i
-attribute_val
-Attribute value.
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Comm_set_attr stores the stipulated attribute value \fIattribute_val\fP for subsequent retrieval by MPI_Comm_get_attr. If the value is already present, then the outcome is as if MPI_Comm_delete_attr was first called to delete the previous value (and the callback function delete_fn was executed), and a new value was next stored. The call is erroneous if there is no key with value \fIcomm_keyval\fP; in particular MPI_KEYVAL_INVALID is an erroneous key value. The call will fail if the delete_fn function returned an error code other than MPI_SUCCESS.
-.sp
-This function replaces MPI_Attr_put, the use of which is deprecated. The C binding is identical. The Fortran binding differs in that \fIattribute_val\fP is an address-sized integer.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fIATTRIBUTE_VAL\fP argument only for Fortran 90. Sun FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_ADDRESS_KIND \fIATTRIBUTE_VAL\fP
-.fi
-.sp
-where MPI_ADDRESS_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH NOTES
-.ft R
-Values of the permanent attributes MPI_TAG_UB, MPI_HOST,
-MPI_IO, and MPI_WTIME_IS_GLOBAL may not be changed.
-.sp
-The type of the attribute value depends on whether C or Fortran is being used. In C, an attribute value is a pointer (void *); in Fortran, it is a single, address-size integer system for which a pointer does not fit in an integer.
-.sp
-If an attribute is already present, the delete function (specified when the corresponding keyval was created) will be called.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_Comm_set_errhandler.3in b/ompi/mpi/man/man3/MPI_Comm_set_errhandler.3in
deleted file mode 100644
index 63da5416b78..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_set_errhandler.3in
+++ /dev/null
@@ -1,62 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Comm_set_errhandler 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Comm_set_errhandler \fP \- Attaches a new error handler to a communicator.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Comm_set_errhandler(MPI_Comm \fIcomm\fP,
-	MPI_Errhandler \fIerrhandler\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_COMM_SET_ERRHANDLER(\fICOMM, ERRHANDLER, IERROR\fP)
-	INTEGER	\fICOMM, ERRHANDLER, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Comm_set_errhandler(\fIcomm\fP, \fIerrhandler\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Errhandler), INTENT(IN) :: \fIerrhandler\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-errhandler
-New error handler for communicator (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Comm_set_errhandler attaches a new error handler to a communicator. The error handler must be either a predefined error handler or an error handler created by a call to MPI_Comm_create_errhandler. This call is identical to MPI_Errhandler_set, the use of which is deprecated.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_Comm_set_info.3in b/ompi/mpi/man/man3/MPI_Comm_set_info.3in
deleted file mode 100644
index 38bee95c823..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_set_info.3in
+++ /dev/null
@@ -1,103 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines
-.\" $COPYRIGHT$
-.TH MPI_Comm_set_info 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Comm_set_info\fP \- Set communicator info hints
-.
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Comm_set_info(MPI_Comm \fIcomm\fP, MPI_Info \fIinfo\fP)
-.
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_COMM_SET_INFO(\fICOMM, INFO, IERROR\fP)
-	INTEGER	\fICOMM, INFO, IERROR \fP
-.
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Comm_set_info(\fIcomm\fP, \fIinfo\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-comm
-Communicator on which to set info hints
-.TP 1i
-info
-Info object containing hints to be set on
-.I comm
-.
-.SH OUTPUT PARAMETERS
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-.
-.SH DESCRIPTION
-.ft R
-MPI_COMM_SET_INFO sets new values for the hints of the communicator
-associated with
-.IR comm .
-MPI_COMM_SET_INFO is a collective routine. The info object may be
-different on each process, but any info entries that an implementation
-requires to be the same on all processes must appear with the same
-value in each process's
-.I info
-object.
-.sp
-The following info key assertions may be accepted by Open MPI:
-.sp
-\fImpi_assert_no_any_tag\fP (boolean): If set to true, then the
-implementation may assume that the process will not use the
-MPI_ANY_TAG wildcard on the given
-communicator.
-.sp
-\fImpi_assert_no_any_source\fP (boolean): If set to true, then
-the implementation may assume that the process will not use the
-MPI_ANY_SOURCE wildcard on the given communicator.
-.sp
-\fImpi_assert_exact_length\fP (boolean): If set to true, then the
-implementation may assume that the lengths of messages received by the
-process are equal to the lengths of the corresponding receive buffers,
-for point-to-point communication operations on the given communicator.
-.sp
-\fImpi_assert_allow_overtaking\fP (boolean): If set to true, then the
-implementation may assume that point-to-point communications on the
-given communicator do not rely on the non-overtaking rule specified in
-MPI-3.1 Section 3.5. In other words, the application asserts that send
-operations are not required to be matched at the receiver in the order
-in which the send operations were performed by the sender, and receive
-operations are not required to be matched in the order in which they
-were performed by the receiver.
-.
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler
-MPI_ERRORS_RETURN may be used to cause error values to be
-returned. Note that MPI does not guarantee that an MPI program can
-continue past an error.
-.
-.SH SEE ALSO
-MPI_Comm_get_info,
-MPI_Info_create,
-MPI_Info_set,
-MPI_Info_free
diff --git a/ompi/mpi/man/man3/MPI_Comm_set_name.3in b/ompi/mpi/man/man3/MPI_Comm_set_name.3in
deleted file mode 100644
index 0605f7bf2c8..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_set_name.3in
+++ /dev/null
@@ -1,83 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Comm_set_name 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Comm_set_name\fP \- Associates a name with a communicator.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Comm_set_name(MPI_Comm \fIcomm\fP, const char *\fIcomm_name\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_COMM_SET_NAME(\fICOMM, COMM_NAME, IERROR\fP)
-	INTEGER	\fICOMM, IERROR \fP
-	CHARACTER*(*) \fICOMM_NAME\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Comm_set_name(\fIcomm\fP, \fIcomm_name\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	CHARACTER(LEN=*), INTENT(IN) :: \fIcomm_name\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-comm
-Communicator whose identifier is to be set (handle).
-
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-comm_name
-Character string to be used as the identifier for the communicator (string).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-
-.SH DESCRIPTION
-.ft R
-MPI_Comm_set_name allows a user to associate a name string with a communicator. The character string that is passed to MPI_Comm_set_name is saved inside the MPI library (so it can be freed by the caller immediately after the call, or allocated on the stack). Leading spaces in \fIname\fP are significant, but trailing ones are not.
-.sp
-MPI_Comm_set_name is a local (noncollective) operation, which affects only the name of the communicator as seen in the process that made the MPI_Comm_set_name call. There is no requirement that the same (or any) name be assigned to a communicator in every process where it exists.
-.sp
-The length of the name that can be stored is limited to the value of MPI_MAX_OBJECT_NAME in Fortran and MPI_MAX_OBJECT_NAME-1 in C (to allow for the null terminator). Attempts to set names longer than this will result in truncation of the name. MPI_MAX_OBJECT_NAME must have a value of at least 64.
-
-
-.SH NOTES
-.ft R
-Since MPI_Comm_set_name is provided to help debug code, it is sensible to give the same name to a communicator in all of the processes where it exists, to avoid confusion.
-.sp
-Regarding name length, under circumstances of store exhaustion, an attempt to set a name of any length could fail; therefore, the value of MPI_MAX_OBJECT_NAME should be viewed only as a strict upper bound on the name length, not a guarantee that setting names of less than this length will always succeed.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-MPI_Comm_get_name
-.sp
-
-
diff --git a/ompi/mpi/man/man3/MPI_Comm_size.3in b/ompi/mpi/man/man3/MPI_Comm_size.3in
deleted file mode 100644
index b0fc031a6a8..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_size.3in
+++ /dev/null
@@ -1,83 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Comm_size 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Comm_size \fP \- Returns the size of the group associated with a communicator.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Comm_size(MPI_Comm \fIcomm\fP, int *\fIsize\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_COMM_SIZE(\fICOMM, SIZE, IERROR\fP)
-	INTEGER	\fICOMM, SIZE, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Comm_size(\fIcomm\fP, \fIsize\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, INTENT(OUT) :: \fIsize\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-size
-Number of processes in the group of comm (integer).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-This function indicates the number of processes involved in a
-communicator. For MPI_COMM_WORLD, it indicates the total number of
-processes available. This function is equivalent to accessing the
-communicator's group with MPI_Comm_group, computing the size using
-MPI_Group_size, and then freeing the temporary group via
-MPI_Group_free. If the communicator is an inter-communicator (enables
-communication between two groups), this function returns the size of
-the local group.  To return the size of the remote group, use the
-MPI_Comm_remote_size function.
-.sp
-This call is often used with MPI_Comm_rank to determine the amount of concurrency available for a specific library or program. MPI_Comm_rank indicates the rank of the process that calls it in the range from  0 . . . size-1, where size is the return value of MPI_Comm_size.
-
-.SH NOTE
-.ft R
-MPI_COMM_NULL is not considered a valid argument to this function.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-.nf
-MPI_Comm_group
-MPI_Comm_rank
-MPI_Comm_compare
-
diff --git a/ompi/mpi/man/man3/MPI_Comm_spawn.3in b/ompi/mpi/man/man3/MPI_Comm_spawn.3in
deleted file mode 100644
index 8e0ced5b6e4..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_spawn.3in
+++ /dev/null
@@ -1,234 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright (c) 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Comm_spawn 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Comm_spawn\fP \- Spawns a number of identical binaries.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Comm_spawn(const char *\fIcommand\fP, char *\fIargv\fP[], int \fImaxprocs\fP,
-	MPI_Info \fIinfo\fP, int \fIroot\fP, MPI_Comm \fIcomm\fP,
-	MPI_Comm *\fIintercomm\fP, int \fIarray_of_errcodes\fP[])
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_COMM_SPAWN(\fICOMMAND, ARGV, MAXPROCS, INFO, ROOT, COMM,
-	INTERCOMM, ARRAY_OF_ERRCODES, IERROR\fP)
-
-	CHARACTER*(*) \fICOMMAND, ARGV(*)\fP
-	INTEGER	\fIINFO, MAXPROCS, ROOT, COMM, INTERCOMM,
-	ARRAY_OF_ERRCODES(*), IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Comm_spawn(\fIcommand\fP, \fIargv\fP, \fImaxprocs\fP, \fIinfo\fP, \fIroot\fP, \fIcomm\fP, \fIintercomm\fP,
-		\fIarray_of_errcodes\fP, \fIierror\fP)
-	CHARACTER(LEN=*), INTENT(IN) :: \fIcommand\fP, \fIargv(*)\fP
-	INTEGER, INTENT(IN) :: \fImaxprocs\fP, \fIroot\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Comm), INTENT(OUT) :: \fIintercomm\fP
-	INTEGER :: \fIarray_of_errcodes(*)\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-command
-Name of program to be spawned (string, significant only at \fIroot\fP).
-.TP 1i
-argv
-Arguments to \fIcommand\fP (array of strings, significant only at \fIroot\fP).
-.TP 1i
-maxprocs
-Maximum number of processes to start (integer, significant only at \fIroot\fP).
-.TP 1i
-info
-A set of key-value pairs telling the runtime system where and how to start the processes (handle, significant only at \fIroot\fP).
-.TP 1i
-root
-Rank of process in which previous arguments are examined (integer).
-.TP 1i
-comm
-Intracommunicator containing group of spawning processes (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-intercomm
-Intercommunicator between original group and the newly spawned group (handle).
-.TP 1i
-array_of_errcodes
-One code per process (array of integers).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Comm_spawn tries to start \fImaxprocs\fP identical copies of the MPI program specified by \fIcommand\fP, establishing communication with them and returning an intercommunicator. The spawned processes are referred to as children. The children have their own MPI_COMM_WORLD, which is separate from that of the parents. MPI_Comm_spawn is collective over \fIcomm\fP, and also may not return until MPI_Init has been called in the children. Similarly, MPI_Init in the children may not return until all parents have called MPI_Comm_spawn. In this sense, MPI_Comm_spawn in the parents and MPI_Init in the children form a collective operation over the union of parent and child processes. The intercommunicator returned by MPI_Comm_spawn contains the parent processes in the local group and the child processes in the remote group. The ordering of processes in the local and remote groups is the same as the as the ordering of the group of the \fIcomm\fP in the parents and of MPI_COMM_WORLD of the children, respectively. This intercommunicator can be obtained in the children through the function MPI_Comm_get_parent.
-.sp
-The MPI standard allows an implementation to use the MPI_UNIVERSE_SIZE attribute of MPI_COMM_WORLD to specify the number of processes that will be active in a program.  Although this implementation of the MPI standard defines MPI_UNIVERSE_SIZE, it does not allow the user to set its value.  If you try to set the value of MPI_UNIVERSE_SIZE, you will get an error message.
-.sp
-The \fIcommand\fP Argument
-.sp
-The \fIcommand\fP argument is a string containing the name of a program to be spawned. The string is null-terminated in C. In Fortran, leading and trailing spaces are stripped. MPI looks for the file first in the working directory of the spawning process.
-.sp
-The \fIargv\fP Argument
-.sp
-\fIargv\fP is an array of strings containing arguments that are passed
-to the program. The first element of \fIargv\fP is the first argument
-passed to \fIcommand\fP, not, as is conventional in some contexts, the
-command itself. The argument list is terminated by NULL in C
-and an empty string in Fortran (note that it is the MPI application's
-responsibility to ensure that the last entry of the
-.I argv
-array is an empty string; the compiler will not automatically insert
-it). In Fortran, leading and trailing spaces are always stripped, so
-that a string consisting of all spaces is considered an empty
-string. The constant MPI_ARGV_NULL may be used in C and Fortran
-to indicate an empty argument list. In C, this constant is the
-same as NULL.
-.sp
-In C, the MPI_Comm_spawn argument \fIargv\fP differs from the \fIargv\fP argument of \fImain\fP in two respects. First, it is shifted by one element. Specifically, \fIargv\fP[0] of \fImain\fP  contains the name of the program (given by \fIcommand\fP). \fIargv\fP[1] of \fImain\fP corresponds to \fIargv\fP[0] in MPI_Comm_spawn, \fIargv\fP[2] of \fImain\fP to \fIargv\fP[1] of MPI_Comm_spawn, and so on. Second, \fIargv\fP of MPI_Comm_spawn must be null-terminated, so that its length can be determined. Passing an \fIargv\fP of MPI_ARGV_NULL to MPI_Comm_spawn results in \fImain\fP receiving \fIargc\fP of 1 and an \fIargv\fP whose element 0 is the name of the program.
-.sp
-The \fImaxprocs\fP Argument
-.sp
-Open MPI tries to spawn \fImaxprocs\fP processes. If it is unable to spawn \fImaxprocs\fP processes, it raises an error of class MPI_ERR_SPAWN. If MPI is able to spawn the specified number of processes, MPI_Comm_spawn returns successfully and the number of spawned processes, \fIm\fP, is given by the size of the remote group of \fIintercomm\fP.
-.sp
-A spawn call with the default behavior is called hard. A spawn call for which fewer than \fImaxprocs\fP processes may be returned is called soft.
-.sp
-The \fIinfo\fP Argument
-.sp
-The \fIinfo\fP argument is an opaque handle of type MPI_Info in C and INTEGER in Fortran. It is a container for a number of user-specified (\fIkey,value\fP) pairs. \fIkey\fP and \fIvalue\fP are strings (null-terminated char* in C, character*(*) in Fortran). Routines to create and manipulate the \fIinfo\fP argument are described in Section 4.10 of the MPI-2 standard.
-.sp
-For the SPAWN calls, \fIinfo\fP provides additional, implementation-dependent instructions to MPI and the runtime system on how to start processes. An application may pass MPI_INFO_NULL in C or Fortran. Portable programs not requiring detailed control over process locations should use MPI_INFO_NULL.
-.sp
-The following keys for \fIinfo\fP are recognized in Open MPI. (The reserved values mentioned in Section 5.3.4 of the MPI-2 standard are not implemented.)
-.sp
-.nf
-Key                    Type     Description
----                    ----     -----------
-
-host                   char *   Host on which the process should be
-                                spawned.  See the \fIorte_host\fP man
-                                page for an explanation of how this
-                                will be used.
-hostfile               char *   Hostfile containing the hosts on which
-                                the processes are to be spawned. See
-                                the \fIorte_hostfile\fP man page for
-                                an explanation of how this will be
-                                used.
-add-host               char *   Add the specified host to the list of
-                                hosts known to this job and use it for
-                                the associated process. This will be
-                                used similarly to the -host option.
-add-hostfile           char *   Hostfile containing hosts to be added
-                                to the list of hosts known to this job
-                                and use it for the associated
-                                process. This will be used similarly
-                                to the -hostfile option.
-wdir                   char *   Directory where the executable is
-                                located. If files are to be
-                                pre-positioned, then this location is
-                                the desired working directory at time
-                                of execution - if not specified, then
-                                it will automatically be set to
-                                \fIompi_preload_files_dest_dir\fP.
-ompi_prefix            char *   Same as the --prefix command line
-                                argument to mpirun.
-ompi_preload_binary    bool     If set to true, pre-position the
-                                specified executable onto the remote
-                                host. A destination directory must
-                                also be provided.
-ompi_preload_files     char *   A comma-separated list of files that
-                                are to be pre-positioned in addition
-                                to the executable.  Note that this
-                                option does not depend upon
-                                \fIompi_preload_binary\fP - files can
-                                be moved to the target even if an
-                                executable is not moved.
-ompi_stdin_target      char *   Comma-delimited list of ranks to
-                                receive stdin when forwarded.
-ompi_non_mpi           bool     If set to true, launching a non-MPI
-                                application; the returned communicator
-                                will be MPI_COMM_NULL. Failure to set
-                                this flag when launching a non-MPI
-                                application will cause both the child
-                                and parent jobs to "hang".
-ompi_param             char *   Pass an OMPI MCA parameter to the
-                                child job.  If that parameter already
-                                exists in the environment, the value
-                                will be overwritten by the provided
-                                value.
-mapper                 char *   Mapper to be used for this job
-map_by                 char *   Mapping directive indicating how
-                                processes are to be mapped (slot,
-                                node, socket, etc.).
-rank_by                char *   Ranking directive indicating how
-                                processes are to be ranked (slot,
-                                node, socket, etc.).
-bind_to                char *   Binding directive indicating how
-                                processes are to be bound (core, slot,
-                                node, socket, etc.).
-path                   char *   List of directories to search for
-                                the executable
-npernode               char *   Number of processes to spawn on
-                                each node of the allocation
-pernode                bool     Equivalent to npernode of 1
-ppr                    char *   Spawn specified number of processes
-                                on each of the identified object type
-env                    char *   Newline-delimited list of envars to
-                                be passed to the spawned procs
-.fi
-
-\fIbool\fP info keys are actually strings but are evaluated as
-follows: if the string value is a number, it is converted to an
-integer and cast to a boolean (meaning that zero integers are false
-and non-zero values are true).  If the string value is
-(case-insensitive) "yes" or "true", the boolean is true.  If the
-string value is (case-insensitive) "no" or "false", the boolean is
-false.  All other string values are unrecognized, and therefore false.
-
-.sp
-The \fIroot\fP Argument
-.sp
-All arguments before the \fIroot\fP argument are examined only on the process whose rank in \fIcomm\fP is equal to \fIroot\fP. The value of these arguments on other processes is ignored.
-.sp
-The \fIarray_of_errcodes\fP Argument
-.sp
-The \fIarray_of_errcodes\fP is an array of length \fImaxprocs\fP in which MPI reports the status of the processes that MPI was requested to start. If all \fImaxprocs\fP processes were spawned, \fIarray_of_errcodes\fP is filled in with the value MPI_SUCCESS. If anyof the processes are \fInot\fP spawned, \fIarray_of_errcodes\fP is filled in with the value MPI_ERR_SPAWN. In C or Fortran, an application may pass MPI_ERRCODES_IGNORE if it is not interested in the error codes.
-
-.SH NOTES
-.ft R
-Completion of MPI_Comm_spawn in the parent does not necessarily mean that MPI_Init has been called in the children (although the returned intercommunicator can be used immediately).
-
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-.nf
-MPI_Comm_spawn_multiple(3)
-MPI_Comm_get_parent(3)
-mpirun(1)
-
diff --git a/ompi/mpi/man/man3/MPI_Comm_spawn_multiple.3in b/ompi/mpi/man/man3/MPI_Comm_spawn_multiple.3in
deleted file mode 100644
index 21414043bde..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_spawn_multiple.3in
+++ /dev/null
@@ -1,268 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright (c) 2010-2018 Cisco Systems, Inc.  All rights reserved
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Comm_spawn_multiple 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Comm_spawn_multiple\fP \- Spawns multiple binaries, or the same binary with multiple sets of arguments.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Comm_spawn_multiple(int \fIcount\fP, char *\fIarray_of_commands\fP[],
-	char **\fIarray_of_argv\fP[], const int \fIarray_of_maxprocs\fP[], const MPI_Info
-	\fIarray_of_info\fP[], int \fIroot\fP, MPI_Comm \fIcomm\fP, MPI_Comm *\fIintercomm\fP,
-	int \fIarray_of_errcodes\fP[])
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_COMM_SPAWN_MULTIPLE(\fICOUNT, ARRAY_OF_COMMANDS, ARRAY_OF_ARGV,
-	ARRAY_OF_MAXPROCS, ARRAY_OF_INFO, ROOT, COMM, INTERCOMM,
-	ARRAY_OF_ERRCODES, IERROR\fP)
-	INTEGER	\fICOUNT, ARRAY_OF_INFO(*), ARRAY_OF_MAXPROCS(*), ROOT,
-		COMM, INTERCOMM, ARRAY_OF_ERRCODES(*), IERROR\fP
-	CHARACTER*(*) \fIARRAY_OF_COMMANDS\fP(*), \fIARRAY_OF_ARGV\fP(\fICOUNT\fP, *)
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Comm_spawn_multiple(\fIcount\fP, \fIarray_of_commands\fP, \fIarray_of_argv\fP,
-	\fIarray_of_maxprocs\fP, \fIarray_of_info\fP, \fIroot\fP, \fIcomm\fP, \fIintercomm,\fP
-		\fIarray_of_errcodes\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIcount\fP, \fIarray_of_maxprocs(*)\fP, \fIroot\fP
-	CHARACTER(LEN=*), INTENT(IN) :: \fIarray_of_commands(*)\fP
-	CHARACTER(LEN=*), INTENT(IN) :: \fIarray_of_argv(count\fP, \fI*)\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIarray_of_info(*)\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Comm), INTENT(OUT) :: \fIintercomm\fP
-	INTEGER :: \fIarray_of_errcodes(*)\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-count
-Number of commands (positive integer, significant to MPI only at \fIroot\fP -- see NOTES).
-.TP 1i
-array_of_commands
-Programs to be executed (array of strings, significant only at \fIroot\fP).
-.TP 1i
-array_of_argv
-Arguments for \fIcommands\fP (array of array of strings,  significant only at \fIroot\fP).
-.TP 1i
-array_of_maxprocs
-Maximum number of processes to start for each command (array of integers, significant only at \fIroot\fP).
-.TP 1i
-array_of_info
-Info objects telling the runtime system where and how to start processes (array of handles, significant only at \fIroot\fP).
-.TP 1i
-root
-Rank of process in which previous arguments are examined (integer).
-.TP 1i
-comm
-Intracommunicator containing group of spawning processes (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-intercomm
-Intercommunicator between original group and the newly spawned group (handle).
-.TP 1i
-array_of_errcodes
-One code per process (array of integers).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Comm_spawn_multiple is identical to MPI_Comm_spawn(3) except that
-it can specify multiple executables. The first argument, \fIcount\fP,
-indicates the number of executables. The next three arguments are
-arrays of the corresponding arguments in MPI_Comm_spawn(3). The next
-argument, \fIarray_of_info\fP, is an array of \fIinfo\fP arguments, one
-for each executable. See the INFO ARGUMENTS section for more information.
-.sp
-For the Fortran version of \fIarray_of_argv\fP, the element \fIarray_of_argv\fP(i,j) is the jth argument to command number i.
-.sp
-In any language, an application may use the constant MPI_ARGVS_NULL (which is likely to be (char ***)0 in C) to specify that no arguments should be passed to any commands. The effect of setting individual elements of \fIarray_of_argv\fP to MPI_ARGV_NULL is not defined. To specify arguments for some commands but not others, the commands without arguments should have a corresponding \fIargv\fP whose first element is null ((char *)0 in C and empty string in Fortran).
-.sp
-All of the spawned processes have the same MPI_COMM_WORLD. Their ranks in MPI_COMM_WORLD correspond directly to the order in which the commands are specified in MPI_Comm_spawn_multiple. Assume that m1 processes are generated by the first command, m2 by the second, etc. The processes corresponding to the first command have ranks 0, 1,..., m1-1. The processes in the second command have ranks m1, m1+1, ..., m1+m2-1. The processes in the third have ranks m1+m2, m1+m2+1, ..., m1+m2+m3-1, etc.
-.sp
-The \fIarray_of_errcodes\fP argument is 1-dimensional array of size
-.sp
-.nf
-	 _ count
-	\\       n ,
-	/_ i=1   i
-.fi
-.sp
-where i is the ith element of \fIarray_of_maxprocs\fP. Command number \fIi\fP corresponds to the i contiguous slots in this array from element
-.sp
-.nf
-                      _              _
-	 _ \fIi\fP-1          |   _ \fIi\fP          |
-	\\       n ,  to |  \\      n      | -1
-	/_ \fIj\fP=1   i      |  /_ \fIj\fP=1  j     |
-                     |_              _|
-.fi
-.sp
-Error codes are treated as for MPI_Comm_spawn(3).
-
-
-.SH INFO ARGUMENTS
-The following keys for \fIinfo\fP are recognized in "#PACKAGE_NAME#". (The reserved values mentioned in Section 5.3.4 of the MPI-2 standard are not implemented.)
-.sp
-.sp
-.nf
-Key                    Type     Description
----                    ----     -----------
-
-host                   char *   Comma-separated list of hosts on which
-                                the processes should be spawned.  See
-                                the \fIorte_host\fP man page for an
-                                explanation of how this will be used.
-hostfile               char *   Hostfile containing the hosts on which
-                                the processes are to be spawned. See
-                                the \fIorte_hostfile\fP man page for
-                                an explanation of how this will be
-                                used.
-add-host               char *   Add the specified hosts to the list of
-                                hosts known to this job and use it for
-                                the associated processes. This will be
-                                used similarly to the -host option.
-add-hostfile           char *   Hostfile containing hosts to be added
-                                to the list of hosts known to this job
-                                and use it for the associated
-                                process. This will be used similarly
-                                to the -hostfile option.
-wdir                   char *   Directory where the executable is
-                                located. If files are to be
-                                pre-positioned, then this location is
-                                the desired working directory at time
-                                of execution - if not specified, then
-                                it will automatically be set to
-                                \fIompi_preload_files_dest_dir\fP.
-ompi_prefix            char *   Same as the --prefix command line
-                                argument to mpirun.
-ompi_preload_binary    bool     If set to true, pre-position the
-                                specified executable onto the remote
-                                host. A destination directory must
-                                also be provided.
-ompi_preload_files     char *   A comma-separated list of files that
-                                are to be pre-positioned in addition
-                                to the executable.  Note that this
-                                option does not depend upon
-                                \fIompi_preload_binary\fP - files can
-                                be moved to the target even if an
-                                executable is not moved.
-ompi_stdin_target      char *   Comma-delimited list of ranks to
-                                receive stdin when forwarded.
-ompi_non_mpi           bool     If set to true, launching a non-MPI
-                                application; the returned communicator
-                                will be MPI_COMM_NULL. Failure to set
-                                this flag when launching a non-MPI
-                                application will cause both the child
-                                and parent jobs to "hang".
-ompi_param             char *   Pass an OMPI MCA parameter to the
-                                child job.  If that parameter already
-                                exists in the environment, the value
-                                will be overwritten by the provided
-                                value.
-mapper                 char *   Mapper to be used for this job
-map_by                 char *   Mapping directive indicating how
-                                processes are to be mapped (slot,
-                                node, socket, etc.).
-rank_by                char *   Ranking directive indicating how
-                                processes are to be ranked (slot,
-                                node, socket, etc.).
-bind_to                char *   Binding directive indicating how
-                                processes are to be bound (core, slot,
-                                node, socket, etc.).
-path                   char *   List of directories to search for
-                                the executable
-npernode               char *   Number of processes to spawn on
-                                each node of the allocation
-pernode                bool     Equivalent to npernode of 1
-ppr                    char *   Spawn specified number of processes
-                                on each of the identified object type
-env                    char *   Newline-delimited list of envars to
-                                be passed to the spawned procs
-.fi
-
-.sp
-\fIbool\fP info keys are actually strings but are evaluated as
-follows: if the string value is a number, it is converted to an
-integer and cast to a boolean (meaning that zero integers are false
-and non-zero values are true).  If the string value is
-(case-insensitive) "yes" or "true", the boolean is true.  If the
-string value is (case-insensitive) "no" or "false", the boolean is
-false.  All other string values are unrecognized, and therefore false.
-
-.sp
-Note that if any of the info handles have \fIompi_non_mpi\fP set to
-true, then all info handles must have it set to true.  If some are set
-to true, but others are set to false (or are unset), MPI_ERR_INFO will
-be returned.
-
-.sp
-Note that in "#PACKAGE_NAME#", the first array location in \fIarray_of_info\fP is applied to all the commands in \fIarray_of_commands\fP.
-
-.SH NOTES
-The argument \fIcount\fP is interpreted by MPI only at the root, as is \fIarray_of_argv\fP. Since the leading dimension of \fIarray_of_argv\fP is \fIcount\fP, a nonpositive value of \fIcount\fP at a nonroot node could theoretically cause a runtime bounds check error, even though \fIarray_of_argv\fP should be ignored by the subroutine. If this happens, you should explicitly supply a reasonable value of \fIcount\fP on the nonroot nodes.
-.sp
-Similar to MPI_Comm_spawn(3), it is the application's responsibility
-to terminate each individual set of argv in the
-.I array_of_argv
-argument.  In C, each argv array is terminated by a NULL pointer.  In
-Fortran, each argv array is terminated by an empty string (note that
-compilers will not automatically insert this blank string; the
-application must ensure to have enough space for an empty string entry
-as the last element of the array).
-.sp
-Other restrictions apply to the
-.I array_of_argv
-parameter; see MPI_Comm_spawn(3)'s description of the
-.I argv
-parameter for more details.
-.sp
-MPI-3.1 implies (but does not directly state) that the argument
-\fIarray_of_commands\fP must be an array of strings of length
-\fIcount\fP.  Unlike the \fIarray_of_argv\fP parameter,
-\fIarray_of_commands\fP does not need to be terminated with a NULL
-pointer in C or a blank string in Fortran.  Older versions of Open MPI
-required that \fIarray_of_commands\fP be terminated with a blank
-string in Fortran; that is no longer required in this version of Open
-MPI.
-.sp
-Calling MPI_Comm_spawn(3) many times would create many sets of
-children with different MPI_COMM_WORLDs, whereas
-MPI_Comm_spawn_multiple creates children with a single MPI_COMM_WORLD,
-so the two methods are not completely equivalent. Also if you need to
-spawn multiple executables, you may get better performance by using
-MPI_Comm_spawn_multiple instead of calling MPI_Comm_spawn(3) several
-times.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-.nf
-MPI_Comm_spawn(3)
-MPI_Comm_get_parent(3)
-mpirun(1)
diff --git a/ompi/mpi/man/man3/MPI_Comm_split.3in b/ompi/mpi/man/man3/MPI_Comm_split.3in
deleted file mode 100644
index 3dd360e680a..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_split.3in
+++ /dev/null
@@ -1,101 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Comm_split 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Comm_split \fP \- Creates new communicators based on colors and keys.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Comm_split(MPI_Comm \fIcomm\fP, int\fI color\fP, int\fI key\fP,
-	MPI_Comm *\fInewcomm\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_COMM_SPLIT(\fICOMM, COLOR, KEY, NEWCOMM, IERROR\fP)
-	INTEGER	\fICOMM, COLOR, KEY, NEWCOMM, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Comm_split(\fIcomm\fP, \fIcolor\fP, \fIkey\fP, \fInewcomm\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, INTENT(IN) :: \fIcolor\fP, \fIkey\fP
-	TYPE(MPI_Comm), INTENT(OUT) :: \fInewcomm\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-comm
-Communicator (handle).
-.TP 1i
-color
-Control of subset assignment (nonnegative integer).
-.TP 1i
-key
-Control of rank assignment (integer).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-newcomm
-New communicator (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-This function partitions the group associated with comm into disjoint subgroups, one for each value of color. Each subgroup contains all processes of the same color. Within each subgroup, the processes are ranked in the order defined by the value of the argument key, with ties broken according to their rank in the old group. A new communicator is created for each subgroup and returned in newcomm. A process may supply the color value MPI_UNDEFINED, in which case newcomm returns MPI_COMM_NULL. This is a collective call, but each process is permitted to provide different values for color and key.
-.sp
-When you call MPI_Comm_split on an inter-communicator, the processes on the left with the same color as those on the right combine to create a new inter-communicator.  The key argument describes the relative rank of processes on each side of the inter-communicator.  The function returns MPI_COMM_NULL for  those colors that are specified on only one side of the inter-communicator, or for those that specify MPI_UNDEFINED as the color.
-.sp
-A call to MPI_Comm_create(\fIcomm\fP, \fIgroup\fP, \fInewcomm\fP) is equivalent to a call to MPI_Comm_split(\fIcomm\fP, \fIcolor\fP,\fI key\fP, \fInewcomm\fP), where all members of \fIgroup\fP provide \fIcolor\fP = 0 and \fIkey\fP = rank in group, and all processes that are not members of \fIgroup\fP provide \fIcolor\fP = MPI_UNDEFINED. The function MPI_Comm_split allows more general partitioning of a group into one or more subgroups with optional reordering.
-.sp
-The value of \fIcolor\fP must be nonnegative or MPI_UNDEFINED.
-
-.SH NOTES
-.ft R
-This is an extremely powerful mechanism for
-dividing a single communicating group of processes into k subgroups, with k
-chosen implicitly by the user (by the number of colors asserted over all
-the processes). Each resulting communicator will be nonoverlapping. Such a division could be useful for defining a hierarchy of computations, such as for multigrid or linear algebra.
-.sp
-Multiple calls to MPI_Comm_split can be used to overcome the requirement that any call have no overlap of the resulting communicators (each process is of only one color per call). In this way, multiple overlapping communication structures can be created. Creative use of the color and key in such splitting operations is encouraged.
-.sp
-Note that, for a fixed color, the keys need not be unique. It is MPI_Comm_split's responsibility to sort processes in ascending order according to this key, and to break ties in a consistent way. If all the keys are specified in the same way, then all the processes in a given color will have the relative rank order as they did in their parent group. (In general, they will have different ranks.)
-.sp
-Essentially, making the key value zero for all processes of a given color
-means that one needn't really pay attention to the rank-order of the processes in the new communicator.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Comm_create
-.br
-MPI_Intercomm_create
-.br
-MPI_Comm_dup
-.br
-MPI_Comm_free
-
-
diff --git a/ompi/mpi/man/man3/MPI_Comm_split_type.3in b/ompi/mpi/man/man3/MPI_Comm_split_type.3in
deleted file mode 100644
index 09df426faea..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_split_type.3in
+++ /dev/null
@@ -1,165 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" $COPYRIGHT$
-.TH MPI_Comm_split_type 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Comm_split_type \fP \- Creates new communicators based on colors and keys.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Comm_split_type(MPI_Comm \fIcomm\fP, int\fI split_type\fP, int\fI key\fP,
-	MPI_Info info, MPI_Comm *\fInewcomm\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_COMM_SPLIT_TYPE(\fICOMM, SPLIT_TYPE, KEY, INFO, NEWCOMM, IERROR\fP)
-	INTEGER	\fICOMM, SPLIT_TYPE, KEY, INFO, NEWCOMM, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Comm_split_type(\fIcomm\fP, \fIsplit_type\fP, \fIkey\fP, \fIinfo\fP, \fInewcomm\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, INTENT(IN) :: \fIsplit_type\fP, \fIkey\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(MPI_Comm), INTENT(OUT) :: \fInewcomm\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-comm
-Communicator (handle).
-.TP 1i
-split_type
-Type of processes to be grouped together (integer).
-.TP 1i
-key
-Control of rank assignment (integer).
-.TP 1i
-info
-Info argument (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-newcomm
-New communicator (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-This function partitions the group associated with \fIcomm\fP into disjoint subgroups, based on
-the type specied by \fIsplit_type\fP. Each subgroup contains all processes of the same type.
-Within each subgroup, the processes are ranked in the order defined by the value of the
-argument \fIkey\fP, with ties broken according to their rank in the old group. A new communicator
-is created for each subgroup and returned in newcomm. This is a collective call;
-all processes must provide the same \fIsplit_type\fP, but each process is permitted to provide
-different values for key. An exception to this rule is that a process may supply the type
-value MPI_UNDEFINED, in which case newcomm returns MPI_COMM_NULL.
-
-.SH SPLIT TYPES
-.ft R
-.TP 1i
-MPI_COMM_TYPE_SHARED
-This type splits the communicator into subcommunicators, each of which can create a shared memory region.
-
-.ft R
-.TP 1i
-OMPI_COMM_TYPE_NODE
-Synonym for MPI_COMM_TYPE_SHARED.
-.ft R
-.TP 1i
-OMPI_COMM_TYPE_HWTHREAD
-This type splits the communicator into subcommunicators, each of which belongs to the same hardware thread.
-.ft R
-.TP 1i
-OMPI_COMM_TYPE_CORE
-This type splits the communicator into subcommunicators, each of which belongs to the same core/processing unit.
-.ft R
-.TP 1i
-OMPI_COMM_TYPE_L1CACHE
-This type splits the communicator into subcommunicators, each of which belongs to the same L1 cache.
-.ft R
-.TP 1i
-OMPI_COMM_TYPE_L2CACHE
-This type splits the communicator into subcommunicators, each of which belongs to the same L2 cache.
-.ft R
-.TP 1i
-OMPI_COMM_TYPE_L3CACHE
-This type splits the communicator into subcommunicators, each of which belongs to the same L3 cache.
-.ft R
-.TP 1i
-OMPI_COMM_TYPE_SOCKET
-This type splits the communicator into subcommunicators, each of which belongs to the same socket.
-.ft R
-.TP 1i
-OMPI_COMM_TYPE_NUMA
-This type splits the communicator into subcommunicators, each of which belongs to the same NUMA-node.
-.ft R
-.TP 1i
-OMPI_COMM_TYPE_BOARD
-This type splits the communicator into subcommunicators, each of which belongs to the same board.
-.ft R
-.TP 1i
-OMPI_COMM_TYPE_HOST
-This type splits the communicator into subcommunicators, each of which belongs to the same host.
-.ft R
-.TP 1i
-OMPI_COMM_TYPE_CU
-This type splits the communicator into subcommunicators, each of which belongs to the same computational unit.
-.ft R
-.TP 1i
-OMPI_COMM_TYPE_CLUSTER
-This type splits the communicator into subcommunicators, each of which belongs to the same cluster.
-
-.SH NOTES
-.sp
-The communicator keys denoted with an
-.I OMPI_
-prefix instead of an
-.I MPI_
-prefix are specific to Open MPI, and are not part of the MPI
-standard.  Their use should be protected by the
-.I OPEN_MPI
-C preprocessor macro.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler
-MPI_ERRORS_RETURN may be used to cause error values to be
-returned. Note that MPI does not guarantee that an MPI program can
-continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Comm_create
-.br
-MPI_Intercomm_create
-.br
-MPI_Comm_dup
-.br
-MPI_Comm_free
-.br
-MPI_Comm_split
-
diff --git a/ompi/mpi/man/man3/MPI_Comm_test_inter.3in b/ompi/mpi/man/man3/MPI_Comm_test_inter.3in
deleted file mode 100644
index 69ce53ef97c..00000000000
--- a/ompi/mpi/man/man3/MPI_Comm_test_inter.3in
+++ /dev/null
@@ -1,87 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Comm_test_inter 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Comm_test_inter \fP \- Tests to see if a comm is an intercommunicator.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Comm_test_inter(MPI_Comm \fIcomm\fP, int\fI *flag\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_COMM_TEST_INTER(\fICOMM, FLAG, IERROR\fP)
-	INTEGER	\fICOMM, IERROR\fP
-	LOGICAL	\fIFLAG \fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Comm_test_inter(\fIcomm\fP, \fIflag\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	LOGICAL, INTENT(OUT) :: \fIflag\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-flag     (Logical.)
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-This local routine allows the calling process to determine the type of a communicator. It returns true for an intercommunicator, false for an intracommunicator.
-.sp
-The type of communicator also affects the value returned by three other functions.  When dealing with an intracommunicator (enables communication within a single group), the functions listed below return the expected values, group size, group, and rank.  When dealing with an inter-communicator, however, they return the following values:
-.sp
-.nf
-MPI_Comm_size	Returns the size of the local group.
-MPI_Comm_group	Returns the local group.
-MPI_Comm_rank	Returns the rank in the local group.
-.fi
-.sp
-To return the remote group and remote group size of an inter-communicator, use the MPI_Comm_remote_group and MPI_Comm_remote_size functions.
-.sp
-The operation MPI_Comm_compare is valid for intercommunicators. Both communicators must be either intra- or intercommunicators, or else MPI_UNEQUAL results. Both corresponding local and remote groups must compare correctly to get the results MPI_CONGRUENT and MPI_SIMILAR. In particular, it is possible for MPI_SIMILAR to result because either the local or remote groups were similar but not identical.
-.sp
-The following accessors provide consistent access to the remote group of an
-intercommunicator: MPI_Comm_remote_size, MPI_Comm_remote_group.
-.sp
-The intercommunicator accessors (MPI_Comm_test_inter, MPI_Comm_remote_size, MPI_Comm_remote_group) are all local operations.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-.nf
-MPI_Comm_remote_group
-MPI_Comm_remote_size
-MPI_Intercomm_create
-MPI_Intercomm_merge
-
diff --git a/ompi/mpi/man/man3/MPI_Compare_and_swap.3in b/ompi/mpi/man/man3/MPI_Compare_and_swap.3in
deleted file mode 100644
index 0a28b12420a..00000000000
--- a/ompi/mpi/man/man3/MPI_Compare_and_swap.3in
+++ /dev/null
@@ -1,112 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013-2015 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" $COPYRIGHT$
-.TH MPI_Compare_and_swap 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Compare_and_swap\fP \- Perform RMA compare-and-swap
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Compare_and_swap(const void *\fIorigin_addr\fP, const void *\fIcompare_addr\fP,
-	void *\fIresult_addr\fP, MPI_Datatype \fIdatatype\fP, int \fItarget_rank\fP,
-	MPI_Aint \fItarget_disp\fP, MPI_Win \fIwin\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_COMPARE_AND_SWAP(\fIORIGIN_ADDR, COMPARE_ADDR, RESULT_ADDR, DATATYPE, TARGET_RANK,
-	TARGET_DISP, WIN, IERROR\fP)
-	<type> \fIORIGIN_ADDR\fP, \fICOMPARE_ADDR\fP, \fIRESULT_ADDR\fP(*)
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fITARGET_DISP\fP
-	INTEGER \fIDATATYPE, TARGET_RANK, WIN, IERROR \fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Compare_and_swap(\fIorigin_addr\fP, \fIcompare_addr\fP, \fIresult_addr\fP, \fIdatatype\fP,
-		\fItarget_rank\fP, \fItarget_disp\fP, \fIwin\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIorigin_addr\fP, \fIcompare_addr\fP
-	TYPE(*), DIMENSION(..) :: \fIresult_addr\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	INTEGER, INTENT(IN) :: \fItarget_rank\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) :: \fItarget_disp\fP
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-origin_addr
-Initial address of buffer (choice).
-.ft R
-.TP
-compare_addr
-Initial address of compare buffer (choice).
-.ft R
-.TP
-result_addr
-Initial address of result buffer (choice).
-.ft R
-.TP
-datatype
-Data type of the entry in origin, result, and target buffers (handle).
-.ft R
-.TP 1i
-target_rank
-Rank of target (nonnegative integer).
-.ft R
-.TP 1i
-target_disp
-Displacement from start of window to beginning of target buffer (nonnegative integer).
-.ft R
-.TP 1i
-win
-Window object (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-This function compares one element of type \fIdatatype\fP in the compare buffer \fIcompare_addr\fP with the buffer at offset \fItarget_disp\fP in the target window specified by \fItarget_rank\fP and \fIwin\fP and replaces the value at the target with the value in the origin buffer \fIorigin_addr\fP if the compare buffer and the target buffer are identical. The original value at the target is returned in the buffer \fIresult_addr\fP. The parameter \fIdatatype\fP must belong to one of the following categories of predefined datatypes: C integer, Fortran integer, Logical, Multi-language types, or Byte as specified in MPI-3 § 5.9.2 on page 176.
-.sp
-The origin and result buffers (\fIorigin_addr\fP and \fIresult_addr\fP) must be disjoint.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fITARGET_DISP\fP argument only for Fortran 90.  FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_ADDRESS_KIND \fITARGET_DISP\fP
-.fi
-.sp
-where MPI_ADDRESS_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH NOTES
-It is the user's responsibility to guarantee that, when
-using the accumulate functions, the target displacement argument is such
-that accesses to the window are properly aligned according to the data
-type arguments in the call to the \fBMPI_Compare_and_swap\fP function.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler
-may be changed with \fBMPI_Comm_set_errhandler\fP; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
diff --git a/ompi/mpi/man/man3/MPI_Dims_create.3in b/ompi/mpi/man/man3/MPI_Dims_create.3in
deleted file mode 100644
index 3d2b02be2df..00000000000
--- a/ompi/mpi/man/man3/MPI_Dims_create.3in
+++ /dev/null
@@ -1,86 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Dims_create 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Dims_create \fP \- Creates a division of processors in a Cartesian grid.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Dims_create(int \fInnodes\fP, int\fI ndims\fP, int\fI dims\fP[])
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_DIMS_CREATE(\fINNODES, NDIMS, DIMS, IERROR\fP)
-	INTEGER	\fINNODES, NDIMS, DIMS(*), IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Dims_create(\fInnodes\fP, \fIndims\fP, \fIdims\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fInnodes\fP, \fIndims\fP
-	INTEGER, INTENT(INOUT) :: \fIdims(ndims)\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-nnodes
-Number of nodes in a grid (integer).
-.TP 1i
-ndims
-Number of Cartesian dimensions (integer).
-
-.SH IN/OUT PARAMETER
-.TP 1i
-dims
-Integer array of size ndims specifying the number of nodes in each dimension.
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-For Cartesian topologies, the function MPI_Dims_create helps the user select a balanced distribution of processes per coordinate direction, depending on the number of processes in the group to be balanced and optional constraints that can be specified by the user. One use is to partition all the processes (the size of MPI_COMM_WORLD's group) into an n-dimensional topology.
-.sp
-The entries in the array \fIdims\fP are set to describe a Cartesian grid with \fIndims\fP dimensions and a total of \fInnodes\fP nodes. The dimensions are set to be as close to each other as possible, using an appropriate divisibility algorithm. The caller may further constrain the operation of this routine by specifying elements of array dims. If dims[i] is set to a positive number, the routine will not modify the number of nodes in dimension i; only those entries where  dims[i] = 0 are modified by the call.
-.sp
-Negative input values of dims[i] are erroneous. An error will occur if
-nnodes is not a multiple of ((pi) over (i, dims[i] != 0)) dims[i].
-.sp
-For dims[i] set by the call, dims[i] will be ordered in nonincreasing order. Array dims is suitable for use as input to routine MPI_Cart_create. MPI_Dims_create is local.
-.sp
-\fBExample:\fP
-.nf
-
-dims
-before					dims
-call		function call		on return
------------------------------------------------------
-(0,0)	MPI_Dims_create(6, 2, dims)	(3,2)
-(0,0)	MPI_Dims_create(7, 2, dims) 	(7,1)
-(0,3,0)	MPI_Dims_create(6, 3, dims)	(2,3,1)
-(0,3,0)	MPI_Dims_create(7, 3, dims)	erroneous call
-------------------------------------------------------
-
-.fi
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_Dist_graph_create.3in b/ompi/mpi/man/man3/MPI_Dist_graph_create.3in
deleted file mode 100644
index 3f48eedf901..00000000000
--- a/ompi/mpi/man/man3/MPI_Dist_graph_create.3in
+++ /dev/null
@@ -1,139 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      FUJITSU LIMITED.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Dist_graph_create 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Dist_graph_create \fP \- Makes a new communicator to which topology information has been attached.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Dist_graph_create(MPI_Comm \fIcomm_old\fP, int\fI n\fP, const int\fI sources[]\fP,
-	const int\fI degrees[]\fP, const int\fI destinations\fP[], const int\fI weights\fP[],
-        MPI_Info info, int\fI reorder\fP, MPI_Comm\fI *comm_dist_graph\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_DIST_GRAPH_CREATE(\fICOMM_OLD, N, SOURCES, DEGREES, DESTINATIONS, WEIGHTS,
-                INFO, REORDER, COMM_DIST_GRAPH, IERROR\fP)
-	INTEGER	\fICOMM_OLD, N, SOURCES(*), DEGRES(*), WEIGHTS(*), INFO\fP
-	INTEGER	\fICOMM_DIST_GRAPH, IERROR\fP
-	LOGICAL   \fIREORDER\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Dist_Graph_create(\fIcomm_old\fP, \fIn\fP, \fIsources\fP, \fIdegrees\fP, \fIdestinations\fP, \fIweights\fP,
-		\fIinfo\fP, \fIreorder\fP, \fIcomm_dist_graph\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm_old\fP
-	INTEGER, INTENT(IN) :: \fIn\fP, \fIsources(n)\fP, \fIdegrees(n)\fP, \fIdestinations(*)\fP
-	INTEGER, INTENT(IN) :: \fIweights(*)\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	LOGICAL, INTENT(IN) :: reorder
-	TYPE(MPI_Comm), INTENT(OUT) :: \fIcomm_dist_graph\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-comm_old
-Input communicator without topology (handle).
-.TP 1i
-n
-Number of source nodes for which this process specifies edges (non-negative integer).
-.TP 1i
-sources
-Array containing the \fIn\fP source nodes for which this process specifies edges (array of non-negative integers).
-.TP 1i
-degrees
-Array specifying the number of destinations for each source node in the source node array (array of non-negative integers).
-.TP 1i
-destinations
-Destination nodes for the source nodes in the source node array (array of non-negative integers).
-.TP 1i
-weights
-Weights for source to destination edges (array of non-negative integers).
-.TP 1i
-info
-Hints on optimization and interpretation of weights (handle).
-.TP 1i
-reorder
-Ranking may be reordered (true) or not (false) (logical).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-comm_dist_graph
-Communicator with distributed graph topology added (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Dist_graph_create creates a new communicator \fIcomm_dist_graph\fP with distrubuted
-graph topology and returns a handle to the new communicator. The number of processes in
-\fIcomm_dist_graph\fP is identical to the number of processes in \fIcomm_old\fP. Concretely, each process calls the
-constructor with a set of directed (source,destination) communication edges as described below.
-Every process passes an array of \fIn\fP source nodes in the \fIsources\fP array. For each source node, a
-non-negative number of destination nodes is specied in the \fIdegrees\fP array. The destination
-nodes are stored in the corresponding consecutive segment of the \fIdestinations\fP array. More
-precisely, if the i-th node in sources is s, this specifies \fIdegrees\fP[i] \fIedges\fP (s,d) with d of the j-th
-such edge stored in \fIdestinations\fP[\fIdegrees\fP[0]+...+\fIdegrees\fP[i-1]+j]. The weight of this edge is
-stored in \fIweights\fP[\fIdegrees\fP[0]+...+\fIdegrees\fP[i-1]+j]. Both the \fIsources\fP and the \fIdestinations\fP arrays
-may contain the same node more than once, and the order in which nodes are listed as
-destinations or sources is not signicant. Similarly, different processes may specify edges
-with the same source and destination nodes. Source and destination nodes must be process
-ranks of comm_old. Different processes may specify different numbers of source and
-destination nodes, as well as different source to destination edges. This allows a fully distributed
-specification of the communication graph. Isolated processes (i.e., processes with
-no outgoing or incoming edges, that is, processes that do not occur as source or destination
-node in the graph specication) are allowed. The call to MPI_Dist_graph_create is collective.
-
-If reorder = false, all processes will have the same rank in comm_dist_graph as in
-comm_old. If reorder = true then the MPI library is free to remap to other processes (of
-comm_old) in order to improve communication on the edges of the communication graph.
-The weight associated with each edge is a hint to the MPI library about the amount or
-intensity of communication on that edge, and may be used to compute a \"best\" reordering.
-
-.SH WEIGHTS
-.ft R
-Weights are specied as non-negative integers and can be used to influence the process
-remapping strategy and other internal MPI optimizations. For instance, approximate count
-arguments of later communication calls along specic edges could be used as their edge
-weights. Multiplicity of edges can likewise indicate more intense communication between
-pairs of processes. However, the exact meaning of edge weights is not specied by the MPI
-standard and is left to the implementation. An application can supply the special value
-MPI_UNWEIGHTED for the weight array to indicate that all edges have the same (effectively no)
-weight. It is erroneous to supply MPI_UNWEIGHTED for some but not
-all processes of comm_old. If the graph is weighted but \fIn\fP = 0, then MPI_WEIGHTS_EMPTY
-or any arbitrary array may be passed to weights. Note that MPI_UNWEIGHTED and
-MPI_WEIGHTS_EMPTY are not special weight values; rather they are special values for the
-total array argument. In Fortran, MPI_UNWEIGHTED and MPI_WEIGHTS_EMPTY are objects
-like MPI_BOTTOM (not usable for initialization or assignment). See MPI-3 § 2.5.4.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Dist_graph_create_adjacent
-MPI_Dist_graph_neighbors
-MPI_Dist_graph_neighbors_count
-
diff --git a/ompi/mpi/man/man3/MPI_Dist_graph_create_adjacent.3in b/ompi/mpi/man/man3/MPI_Dist_graph_create_adjacent.3in
deleted file mode 100644
index 91da9628dbe..00000000000
--- a/ompi/mpi/man/man3/MPI_Dist_graph_create_adjacent.3in
+++ /dev/null
@@ -1,134 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      FUJITSU LIMITED.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Dist_graph_create_adjacent 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Dist_graph_create_adjacent \fP \- Makes a new communicator to which topology information has been attached.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Dist_graph_create_adjacent(MPI_Comm \fIcomm_old\fP, int\fI indegree\fP, const int\fI sources[]\fP,
-	const int\fI sourceweights[]\fP, int\fI outdegree\fP, const int\fI destinations\fP[], const int\fI destweights\fP[],
-        MPI_Info info, int\fI reorder\fP, MPI_Comm\fI *comm_dist_graph\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_DIST_GRAPH_CREATE_ADJACENT(\fICOMM_OLD, INDEGREE, SOURCES, SOURCEWEIGHTS, OUTDEGREE,
-                DESTINATIONS, DESTWEIGHTS, INFO, REORDER, COMM_DIST_GRAPH, IERROR\fP)
-	INTEGER	\fICOMM_OLD, INDEGREE, SOURCES(*), SOURCEWEIGHTS(*), OUTDEGREE, DESTINATIONS(*), DESTWEIGHTS(*), INFO\fP
-	INTEGER	\fICOMM_DIST_GRAPH, IERROR\fP
-	LOGICAL \fIREORDER\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Dist_Graph_create_adjacent(\fIcomm_old\fP, \fIndegree\fP, \fIsources\fP, \fIsourceweights\fP,
-		\fIoutdegree\fP, \fIdestinations\fP, \fIdestweights\fP, \fIinfo\fP, \fIreorder\fP,
-		\fIcomm_dist_graph\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm_old\fP
-	INTEGER, INTENT(IN) :: \fIindegree\fP, \fIsources(indegree)\fP, \fIoutdegree\fP, \fIdestinations(outdegree)\fP
-	INTEGER, INTENT(IN) :: \fIsourceweights(*)\fP, \fIdestweights(*)\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	LOGICAL, INTENT(IN) :: reorder
-	TYPE(MPI_Comm), INTENT(OUT) :: \fIcomm_dist_graph\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-comm_old
-Input communicator without topology (handle).
-.TP 1i
-indegree
-Size of \fIsources\fP and \fIsourceweights\fP arrays (non-negative integer).
-.TP 1i
-sources
-Ranks of processes for which the calling process is a destination (array of non-negative integers).
-.TP 1i
-sourceweights
-Weights of the edges into the calling process (array of non-negative integers).
-.TP 1i
-outdegree
-Size of \fIdestinations\fP and \fIdestweights\fP arrays (non-negative integer).
-.TP 1i
-destinations
-Ranks of processes for which the calling process is a source (array of non-negative integers).
-.TP 1i
-destweights
-Weights of the edges out of the calling process (array of non-negative integers).
-.TP 1i
-info
-Hints on optimization and interpretation of weights (handle).
-.TP 1i
-reorder
-Ranking may be reordered (true) or not (false) (logical).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-comm_dist_graph
-Communicator with distributed graph topology added (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Dist_graph_create_adjacent creats a new communicator \fIcomm_dist_graph\fP with distrubuted
-graph topology and returns a handle to the new communicator. The number of processes in
-\fIcomm_dist_graph\fP is identical to the number of processes in \fIcomm_old\fP. Each process passes all
-information about its incoming and outgoing edges in the virtual distributed graph topology.
-The calling processes must ensure that each edge of the graph is described in the source
-and in the destination process with the same weights. If there are multiple edges for a given
-(source,dest) pair, then the sequence of the weights of these edges does not matter. The
-complete communication topology is the combination of all edges shown in the \fIsources\fP arrays
-of all processes in comm_old, which must be identical to the combination of all edges shown
-in the \fIdestinations\fP arrays. Source and destination ranks must be process ranks of comm_old.
-This allows a fully distributed specication of the communication graph. Isolated processes
-(i.e., processes with no outgoing or incoming edges, that is, processes that have specied
-indegree and outdegree as zero and thus do not occur as source or destination rank in the
-graph specication) are allowed. The call to MPI_Dist_graph_create_adjacent is collective.
-
-.SH WEIGHTS
-.ft R
-Weights are specied as non-negative integers and can be used to influence the process
-remapping strategy and other internal MPI optimizations. For instance, approximate count
-arguments of later communication calls along specic edges could be used as their edge
-weights. Multiplicity of edges can likewise indicate more intense communication between
-pairs of processes. However, the exact meaning of edge weights is not specied by the MPI
-standard and is left to the implementation. An application can supply the special value
-MPI_UNWEIGHTED for the weight array to indicate that all edges have the same (effectively
-no) weight. It is erroneous to supply MPI_UNWEIGHTED for some but not all processes of
-comm_old. If the graph is weighted but \fIindegree\fP or \fIoutdegree\fP is zero, then
-MPI_WEIGHTS_EMPTY or any arbitrary array may be passed to sourceweights or destweights
-respectively. Note that MPI_UNWEIGHTED and MPI_WEIGHTS_EMPTY are not special weight values;
-rather they are special values for the total array argument. In Fortran, MPI_UNWEIGHTED
-and MPI_WEIGHTS_EMPTY are objects like MPI_BOTTOM (not usable for initialization or
-assignment). See MPI-3 § 2.5.4.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Dist_graph_create
-MPI_Dist_graph_neighbors
-MPI_Dist_graph_neighbors_count
-
diff --git a/ompi/mpi/man/man3/MPI_Dist_graph_neighbors.3in b/ompi/mpi/man/man3/MPI_Dist_graph_neighbors.3in
deleted file mode 100644
index 2ad96f625e2..00000000000
--- a/ompi/mpi/man/man3/MPI_Dist_graph_neighbors.3in
+++ /dev/null
@@ -1,95 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" $COPYRIGHT$
-.TH MPI_Dist_graph_neighbors 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Dist_graph_neighbors \fP \- Returns the neighbors of the calling process in a distributed graph topology.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Dist_graph_neighbors(MPI_Comm \fIcomm\fP, int \fImaxindegree\fP, int \fIsources\fP[], int \fIsourceweights\fP[],
-                             int \fImaxoutdegree\fP, int \fIdestinations\fP[], int \fIdestweights\fP[])
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_DIST_GRAPH_NEIGHBORS(COMM, MAXINDEGREE, SOURCES, SOURCEWEIGHTS,
-        MAXOUTDEGREE, DESTINATIONS, DESTWEIGHTS, IERROR)
-        INTEGER COMM, MAXINDEGREE, SOURCES(*), SOURCEWEIGHTS(*), MAXOUTDEGREE,
-                DESTINATIONS(*), DESTWEIGHTS(*), IERROR
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Dist_Graph_neighbors(\fIcomm\fP, \fImaxindegree\fP, \fIsources\fP, \fIsourceweights\fP,
-		\fImaxoutdegree\fP, \fIdestinations\fP, \fIdestweights\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, INTENT(IN) :: \fImaxindegree\fP, \fImaxoutdegree\fP
-	INTEGER, INTENT(OUT) :: \fIsources(maxindegree)\fP, \fIdestinations(maxoutdegree)\fP
-	INTEGER :: sourceweights(*), destweights(*)
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-comm
-Communicator with distributed graph topology (handle).
-.TP 1i
-maxindegree
-Size of \fIsources\fP and \fIsourceweights\fP arrays (non-negative integer).
-.TP 1i
-maxoutdegree
-Size of \fIdestinations\fP and \fIdestweights\fP arrays (non-negative integer).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-sources
-Processes for which the calling process is a destination (array of non-negative integers).
-.TP 1i
-sourceweights
-Weights of the edges into the calling process (array of non-negative integers).
-.TP 1i
-destinations
-Processes for which the calling process is a source (array of non-negative integers).
-.TP 1i
-destweights
-Weights of the edges out of the calling process (array of non-negative integers).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Dist_graph_neighbors returns the source and destination ranks in a distributed graph topology
-for the calling process. This call will return up to \fImaxindegree\fP source ranks in the \fIsources\fP array
-and up to \fImaxoutdegree\fP destination ranks in the \fIdestinations\fP array. If weights were
-specified at the time of the communicator's creation then the associated weights
-are returned in the \fIsourceweights\fP and \fI destweights\fP arrays. If the communicator
-was created with MPI_Dist_graph_create_adjacent then the order of the values in \fIsources\fP and
-\fIdestinations\fP is identical to the input that was used by the process with the same rank in
-comm_old in the creation call.
-
-.fi
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Dist_graph_neighbors_count
-
diff --git a/ompi/mpi/man/man3/MPI_Dist_graph_neighbors_count.3in b/ompi/mpi/man/man3/MPI_Dist_graph_neighbors_count.3in
deleted file mode 100644
index 7a24fe2c04a..00000000000
--- a/ompi/mpi/man/man3/MPI_Dist_graph_neighbors_count.3in
+++ /dev/null
@@ -1,74 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" $COPYRIGHT$
-.TH MPI_Dist_graph_neighbors_count 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Dist_graph_neighbors_count \fP \- Returns the number of in and out edges for the calling processes in a distributed graph topology and a flag indicating whether the distributed graph is weighted.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Dist_graph_neighbors_count(MPI_Comm \fIcomm\fP, int\fI *indegree\fP,
-	int\fI *outdegree\fP, int\fI *weighted\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_DIST_GRAPH_NEIGHBORS_COUNT(\fICOMM, INDEGREE, OUTDEGREE, WEIGHTED, IERROR\fP)
-	INTEGER	\fICOMM, INDEGREE, OUTDEGREE, IERROR\fP
-        LOGICAL \fIWEIGHTED\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Dist_graph_neighbors_count(\fIcomm\fP, \fIindegree\fP, \fIoutdegree\fP, \fIweighted\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, INTENT(IN) :: \fIindegree\fP, \fIoutdegree\fP
-	INTEGER, INTENT(OUT) :: \fIweighted\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-comm
-Communicator with distributed graph topology (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-indegree
-Number of edges into this process (non-negative integer).
-.TP 1i
-outdegree
-Number of edges out of this process (non-negative integer).
-.TP 1i
-weighted
-False if MPI_UNWEIGHTED was supplied during creation, true otherwise (logical).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Dist_graph_neighbors_count and MPI_Graph_neighbors provide adjacency information for a distributed graph topology. MPI_Dist_graph_neighbors_count returns the number of sources and destinations for the calling process.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Dist_graph_neighbors
diff --git a/ompi/mpi/man/man3/MPI_Errhandler_create.3in b/ompi/mpi/man/man3/MPI_Errhandler_create.3in
deleted file mode 100644
index 806c4f3999b..00000000000
--- a/ompi/mpi/man/man3/MPI_Errhandler_create.3in
+++ /dev/null
@@ -1,79 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Errhandler_create 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Errhandler_create \fP \- Creates an MPI-style error handler -- use of this routine is deprecated.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Errhandler_create(MPI_Handler_function *\fIfunction\fP,
-	MPI_Errhandler *\fIerrhandler\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-INCLUDE 'mpif.h'
-MPI_ERRHANDLER_CREATE(\fIFUNCTION, ERRHANDLER, IERROR\fP)
-	EXTERNAL	\fIFUNCTION\fP
-	INTEGER	\fIERRHANDLER, IERROR\fP
-
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-function
-User-defined error handling procedure.
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-errhandler
-MPI error handler (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Note that use of this routine is \fIdeprecated\fP as of MPI-2. Please use MPI_Comm_create_errhandler instead.
-.sp
-Registers the user routine function for use as an MPI error handler. Returns in errhandler a handle to the registered error handler.
-.sp
-In the C language, the user routine should be a C function of type MPI_Handler_function, which is defined as
-.sp
-.nf
-    typedef void (MPI_Handler_function)(MPI_Comm *, int *, \&...);
-.fi
-.sp
-The first argument is the communicator in use. The second is the error code
-to be returned by the MPI routine that raised the error. If the routine would have returned MPI_ERR_IN_STATUS, it is the error code returned in the status for the request that caused the error handler to be invoked. The remaining arguments are stdargs arguments whose number and meaning is implementation-dependent. An implementation should clearly document these arguments. Addresses are used so that the handler may be written in Fortran.
-
-.SH NOTE
-.ft R
-The MPI-1 Standard states that an implementation may make the output value (errhandler) simply the address of the function. However, the action of MPI_Errhandler_ free makes this impossible, since it is required to set the value of the argument to MPI_ERRHANDLER_NULL. In addition, the actual error handler must remain until all communicators that use it are freed.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.br
-MPI_Comm_create_errhandler
-.br
-MPI_Comm_get_errhandler
-.br
-MPI_Comm_set_errhandler
-
-
-
diff --git a/ompi/mpi/man/man3/MPI_Errhandler_free.3in b/ompi/mpi/man/man3/MPI_Errhandler_free.3in
deleted file mode 100644
index b7accb78894..00000000000
--- a/ompi/mpi/man/man3/MPI_Errhandler_free.3in
+++ /dev/null
@@ -1,68 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Errhandler_free 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Errhandler_free \fP \- Frees an MPI-style error handler.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Errhandler_free(MPI_Errhandler *\fIerrhandler\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_ERRHANDLER_FREE(\fIERRHANDLER, IERROR\fP)
-	INTEGER	\fIERRHANDLER, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Errhandler_free(\fIerrhandler\fP, \fIierror\fP)
-	TYPE(MPI_Errhandler), INTENT(INOUT) :: \fIerrhandler\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-errhandler
-MPI error handler (handle). Set to MPI_ERRHANDLER_NULL on exit.
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Marks the error handler associated with errhandler for deallocation and sets errhandler to MPI_ERRHANDLER_NULL. The error handler will be deallocated after all communicators associated with it have been deallocated.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Comm_create_errhandler
-.br
-MPI_Comm_get_errhandler
-.br
-MPI_Comm_set_errhandler
-
-
-
-
diff --git a/ompi/mpi/man/man3/MPI_Errhandler_get.3in b/ompi/mpi/man/man3/MPI_Errhandler_get.3in
deleted file mode 100644
index 8a23805dce9..00000000000
--- a/ompi/mpi/man/man3/MPI_Errhandler_get.3in
+++ /dev/null
@@ -1,66 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Errhandler_get 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Errhandler_get \fP \- Gets the error handler for a communicator -- use of this routine is deprecated.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Errhandler_get(MPI_Comm \fIcomm\fP, MPI_Errhandler\fI *errhandler\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-INCLUDE 'mpif.h'
-MPI_ERRHANDLER_GET(\fICOMM, ERRHANDLER, IERROR\fP)
-	INTEGER	\fICOMM, ERRHANDLER, IERROR\fP
-
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-comm
-Communicator to get the error handler from (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-errhandler
-MPI error handler currently associated with communicator (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Note that use of this routine is \fIdeprecated\fP as of MPI-2. Please use MPI_Comm_get_errhandler instead.
-.sp
-Returns in errhandler (a handle to) the error handler that is currently associated with communicator comm.
-.sp
-\fBExample:\fP A library function may register at its entry point the current error handler for a communicator, set its own private error handler for this communicator, and restore before exiting the previous error handler.
-
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Comm_create_errhandler
-.br
-MPI_Comm_get_errhandler
-.br
-MPI_Comm_set_errhandler
-
diff --git a/ompi/mpi/man/man3/MPI_Errhandler_set.3in b/ompi/mpi/man/man3/MPI_Errhandler_set.3in
deleted file mode 100644
index 928cae994e1..00000000000
--- a/ompi/mpi/man/man3/MPI_Errhandler_set.3in
+++ /dev/null
@@ -1,62 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Errhandler_set 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Errhandler_set \fP \- Sets the error handler for a communicator -- use of this routine is deprecated.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Errhandler_set(MPI_Comm \fIcomm\fP, MPI_Errhandler \fIerrhandler\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-INCLUDE 'mpif.h'
-MPI_ERRHANDLER_SET(\fICOMM, ERRHANDLER, IERROR\fP)
-	INTEGER	\fICOMM, ERRHANDLER, IERROR\fP
-
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-comm
-Communicator to set the error handler for (handle).
-.TP 1i
-errhandler
-New MPI error handler for communicator (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Note that use of this routine is \fIdeprecated\fP as of MPI-2. Please use MPI_Comm_set_errhandler instead.
-.sp
-Associates the new error handler errhandler with communicator comm at the calling process. Note that an error handler is always associated with the communicator.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Comm_create_errhandler
-.br
-MPI_Comm_get_errhandler
-.br
-MPI_Comm_set_errhandler
-
-
diff --git a/ompi/mpi/man/man3/MPI_Error_class.3in b/ompi/mpi/man/man3/MPI_Error_class.3in
deleted file mode 100644
index 21397da88c1..00000000000
--- a/ompi/mpi/man/man3/MPI_Error_class.3in
+++ /dev/null
@@ -1,67 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Error_class 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Error_class \fP \- Converts an error code into an error class.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Error_class(int \fIerrorcode\fP, int\fI *errorclass\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_ERROR_CLASS(\fIERRORCODE, ERRORCLASS, IERROR\fP)
-	INTEGER	\fIERRORCODE, ERRORCLASS, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Error_class(\fIerrorcode\fP, \fIerrorclass\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIerrorcode\fP
-	INTEGER, INTENT(OUT) :: \fIerrorclass\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-errorcode
-Error code returned by an MPI routine.
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-errorclass
-Error class associated with errorcode.
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-The function MPI_Error_class maps each standard error code (error class) onto itself.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Error_string
-
-
diff --git a/ompi/mpi/man/man3/MPI_Error_string.3in b/ompi/mpi/man/man3/MPI_Error_string.3in
deleted file mode 100644
index 4bf346eea31..00000000000
--- a/ompi/mpi/man/man3/MPI_Error_string.3in
+++ /dev/null
@@ -1,75 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Error_string 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Error_string \fP \- Returns a string for a given error code.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Error_string(int \fIerrorcode\fP, char\fI *string\fP, int\fI *resultlen\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_ERROR_STRING(\fIERRORCODE, STRING, RESULTLEN, IERROR\fP)
-	INTEGER		\fIERRORCODE, RESULTLEN, IERROR\fP
-	CHARACTER*(*)	\fISTRING\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Error_string(\fIerrorcode\fP, \fIstring\fP, \fIresultlen\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIerrorcode\fP
-	CHARACTER(LEN=MPI_MAX_ERROR_STRING), INTENT(OUT) :: \fIstring\fP
-	INTEGER, INTENT(OUT) :: \fIresultlen\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-errorcode
-Error code returned by an MPI routine or an MPI error class.
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-string
-Text that corresponds to the errorcode.
-.TP 1i
-resultlen
-Length of string.
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Returns the error string associated with an error code or class. The argument string must represent storage that is at least MPI_MAX_ERROR_STRING characters long.
-.sp
-The number of characters actually written is returned in the output
-argument, resultlen.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Error_class
-
-
diff --git a/ompi/mpi/man/man3/MPI_Exscan.3in b/ompi/mpi/man/man3/MPI_Exscan.3in
deleted file mode 100644
index 8741eff6435..00000000000
--- a/ompi/mpi/man/man3/MPI_Exscan.3in
+++ /dev/null
@@ -1,165 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Exscan 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-
-.SH NAME
-\fBMPI_Exscan, MPI_Iexscan\fP \- Computes an exclusive scan (partial reduction)
-
-.SH SYNTAX
-.ft R
-
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Exscan(const void *\fIsendbuf\fP, void *\fIrecvbuf\fP, int \fIcount\fP,
-	MPI_Datatype \fIdatatype\fP, MPI_Op \fIop\fP, MPI_Comm \fIcomm\fP)
-
-int MPI_Iexscan(const void *\fIsendbuf\fP, void *\fIrecvbuf\fP, int \fIcount\fP,
-	MPI_Datatype \fIdatatype\fP, MPI_Op \fIop\fP, MPI_Comm \fIcomm\fP,
-	MPI_Request \fI*request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_EXSCAN(\fISENDBUF, RECVBUF, COUNT, DATATYPE, OP, COMM, IERROR\fP)
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fICOUNT, DATATYPE, OP, COMM, IERROR\fP
-
-MPI_IEXSCAN(\fISENDBUF, RECVBUF, COUNT, DATATYPE, OP, COMM, REQUEST, IERROR\fP)
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fICOUNT, DATATYPE, OP, COMM, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Exscan(\fIsendbuf\fP, \fIrecvbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIop\fP, \fIcomm\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..) :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Op), INTENT(IN) :: \fIop\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPI_Iexscan(\fIsendbuf\fP, \fIrecvbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIop\fP, \fIcomm\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Op), INTENT(IN) :: \fIop\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-sendbuf
-Send buffer (choice).
-.TP 1i
-count
-Number of elements in input buffer (integer).
-.TP 1i
-datatype
-Data type of elements of input buffer (handle).
-.TP 1i
-op
-Operation (handle).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-recvbuf
-Receive buffer (choice).
-.TP 1i
-request
-Request (handle, non-blocking only).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Exscan is used to perform an exclusive prefix reduction on data
-distributed across the calling processes. The operation returns, in
-the \fIrecvbuf\fP of the process with rank i, the reduction
-(calculated according to the function \fIop\fP) of the values in the
-\fIsendbuf\fPs of processes with ranks 0, ..., i-1. Compare this with
-the functionality of MPI_Scan, which calculates over the range 0, ...,
-i (inclusive). The type of operations supported, their semantics, and
-the constraints on send and receive buffers are as for MPI_Reduce.
-.sp
-The value in \fIrecvbuf\fP on process 0 is undefined and unreliable
-as \fIrecvbuf\fP is not significant for process 0. The value of
-\fIrecvbuf\fP on process 1 is always the value in \fIsendbuf\fP on
-process 0.
-.sp
-.SH USE OF IN-PLACE OPTION
-The `in place' option for intracommunicators is specified by passing MPI_IN_PLACE in the \fIsendbuf\fP argument. In this case, the input data is taken from the receive buffer, and replaced by the output data.
-.sp
-Note that MPI_IN_PLACE is a special kind of value; it has the same restrictions on its use as MPI_BOTTOM.
-.sp
-Because the in-place option converts the receive buffer into a send-and-receive buffer, a Fortran binding that includes INTENT must mark these as INOUT, not OUT.
-.sp
-
-.SH NOTES
-.ft R
-MPI does not specify which process computes which operation. In
-particular, both processes 0 and 1 may participate in the computation
-even though the results for both processes' \fIrecvbuf\fP are
-degenerate. Therefore, all processes, including 0 and 1, must provide
-the same \fIop\fP.
-.sp
-It can be argued, from a mathematical perspective, that the definition
-of MPI_Exscan is unsatisfactory because the output at process 0 is
-undefined.  The "mathematically correct" output for process 0 would be
-the unit element of the reduction operation. However, such a
-definition of an exclusive scan would not work with user-defined
-\fIop\fP functions as there is no way for MPI to "know" the unit value
-for these custom operations.
-
-.SH NOTES ON COLLECTIVE OPERATIONS
-.ft R
-The reduction functions of type MPI_Op do not return an error value.
-As a result, if the functions detect an error, all they can do is
-either call MPI_Abort or silently skip the problem. Thus, if the
-error handler is changed from MPI_ERRORS_ARE_FATAL to something else
-(e.g., MPI_ERRORS_RETURN), then no error may be indicated.
-.sp
-The reason for this is the performance problems in ensuring that
-all collective routines return the same error value.
-
-.SH ERRORS
-.ft R
-Almost all MPI routines return an error value; C routines as
-the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-.sp
-See the MPI man page for a full list of MPI error codes.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Op_create
-MPI_Reduce
-MPI_Scan
-
diff --git a/ompi/mpi/man/man3/MPI_Fetch_and_op.3in b/ompi/mpi/man/man3/MPI_Fetch_and_op.3in
deleted file mode 100644
index fc380a3aef7..00000000000
--- a/ompi/mpi/man/man3/MPI_Fetch_and_op.3in
+++ /dev/null
@@ -1,124 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013-2015 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" $COPYRIGHT$
-.TH MPI_Fetch_and_op 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Fetch_and_op\fP \- Combines the contents of the origin buffer with that of a target buffer and returns the target buffer value.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Fetch_and_op(const void *\fIorigin_addr\fP, void *\fIresult_addr\fP,
-	MPI_Datatype \fIdatatype\fP, int \fItarget_rank\fP, MPI_Aint \fItarget_disp\fP,
-	MPI_Op \fIop\fP, MPI_Win \fIwin\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FETCH_AND_OP(\fIORIGIN_ADDR, RESULT_ADDR, DATATYPE, TARGET_RANK,
-                 TARGET_DISP, OP, WIN, IERROR\fP)
-	<type> \fIORIGIN_ADDR\fP, \fIRESULT_ADDR\fP(*)
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fITARGET_DISP\fP
-	INTEGER \fIDATATYPE, TARGET_RANK, OP, WIN, IERROR \fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Fetch_and_op(\fIorigin_addr\fP, \fIresult_addr\fP, \fIdatatype\fP, \fItarget_rank\fP,
-		\fItarget_disp\fP, \fIop\fP, \fIwin\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIorigin_addr\fP
-	TYPE(*), DIMENSION(..) :: \fIresult_addr\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	INTEGER, INTENT(IN) :: \fItarget_rank\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) :: \fItarget_disp\fP
-	TYPE(MPI_Op), INTENET(IN) :: \fIop\fP
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-origin_addr
-Initial address of buffer (choice).
-.ft R
-.TP
-result_addr
-Initial address of result buffer (choice).
-.ft R
-.TP
-datatype
-Data type of the entry in origin, result, and target buffers (handle).
-.ft R
-.TP 1i
-target_rank
-Rank of target (nonnegative integer).
-.ft R
-.TP 1i
-target_disp
-Displacement from start of window to beginning of target buffer (nonnegative integer).
-.ft R
-.TP 1i
-op
-Reduce operation (handle).
-.ft R
-.TP 1i
-win
-Window object (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Accumulate one element of type \fIdatatype\fP from the origin buffer (\fIorigin_addr\fP) to the buffer at offset \fItarget_disp\fP, in the target window specified by \fItarget_rank\fP and \fIwin\fP, using the operation \fIop\fP and return in the result buffer \fIresult_addr\fP the contents of the target buffer before the accumulation.
-.sp
-The origin and result buffers (\fIorigin_addr\fP and \fIresult_addr\fP) must be disjoint. Any of the predefined operations for \fBMPI_Rreduce\fP, as well as MPI_NO_OP or MPI_REPLACE, can be specified as \fIop\fP; user-defined functions cannot be used. The \fIdatatype\fP argument must be a predefined datatype. The operation is executed atomically.
-.sp
-A new predefined operation, MPI_REPLACE, is defined. It corresponds to the associative function f(a, b) =b; that is, the current value in the target memory is replaced by the value supplied by the origin.
-.sp
-A new predefined operation, MPI_NO_OP, is defined. It corresponds to the assiciative function f(a, b) = a; that is the current value in the target memory is returned in the result buffer at the origin and no operation is performed on the target buffer.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fITARGET_DISP\fP argument only for Fortran 90.  FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_ADDRESS_KIND \fITARGET_DISP\fP
-.fi
-.sp
-where MPI_ADDRESS_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH NOTES
-It is the user's responsibility to guarantee that, when
-using the accumulate functions, the target displacement argument is such
-that accesses to the window are properly aligned according to the data
-type arguments in the call to the MPI_Fetch_and_op function.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler
-may be changed with \fBMPI_Comm_set_errhandler\fP; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Get_accumulate
-.br
-MPI_Reduce
diff --git a/ompi/mpi/man/man3/MPI_File_c2f.3in b/ompi/mpi/man/man3/MPI_File_c2f.3in
deleted file mode 100644
index a13fce697dd..00000000000
--- a/ompi/mpi/man/man3/MPI_File_c2f.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Comm_f2c.3
diff --git a/ompi/mpi/man/man3/MPI_File_call_errhandler.3in b/ompi/mpi/man/man3/MPI_File_call_errhandler.3in
deleted file mode 100644
index ecf27696f5b..00000000000
--- a/ompi/mpi/man/man3/MPI_File_call_errhandler.3in
+++ /dev/null
@@ -1,77 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_call_errhandler 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-
-.SH NAME
-\fBMPI_File_call_errhandler\fP \- Passes the supplied error code to the
-error handler assigned to a file
-
-.SH SYNTAX
-.ft R
-
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_call_errhandler(MPI_File \fIfh\fP, int \fIerrorcode\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_CALL_ERRHANDLER(\fIFH, ERRORCODE, IERROR\fP)
-	INTEGER	\fIFH, IERRORCODE, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_call_errhandler(\fIfh\fP, \fIerrorcode\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	INTEGER, INTENT(IN) :: \fIerrorcode\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1.4i
-fh
-file with error handler (handle).
-.ft R
-.TP 1.4i
-errorcode
-MPI error code (integer).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1.4i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-This function invokes the error handler assigned to the file handle
-\fIfh\fP with the supplied error code \fIerrorcode\fP. If the error
-handler was successfully called, the process is not aborted, and the
-error handler returns, this function returns MPI_SUCCESS.
-.sp
-Unlike errors on communicators and windows, the default errorhandler
-for files is MPI_ERRORS_RETURN.
-
-.SH ERRORS
-.ft R
-Almost all MPI routines return an error value; C routines as
-the value of the function and Fortran routines in the last argument.
-.sp
-See the MPI man page for a full list of MPI error codes.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_File_create_errhandler
-MPI_File_set_errhandler
-
diff --git a/ompi/mpi/man/man3/MPI_File_close.3in b/ompi/mpi/man/man3/MPI_File_close.3in
deleted file mode 100644
index 0f1b2a9f9d1..00000000000
--- a/ompi/mpi/man/man3/MPI_File_close.3in
+++ /dev/null
@@ -1,66 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_close 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_close\fP \- Closes a file (collective).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_close(MPI_File \fI*fh\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_CLOSE(\fIFH\fP, \fIIERROR\fP)
-	INTEGER	\fIFH, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_close(\fIfh\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(INOUT) :: \fIfh\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_close first synchronizes file state, then closes the file
-associated with
-.I fh.
-MPI_File_close is a collective routine. The user is responsible for
-ensuring that all outstanding requests associated with
-.I fh
-have completed before calling MPI_File_close.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-
diff --git a/ompi/mpi/man/man3/MPI_File_create_errhandler.3in b/ompi/mpi/man/man3/MPI_File_create_errhandler.3in
deleted file mode 100644
index c5cabc644c4..00000000000
--- a/ompi/mpi/man/man3/MPI_File_create_errhandler.3in
+++ /dev/null
@@ -1,89 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright 2009-2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_create_errhandler 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_create_errhandler \fP \- Creates an MPI-style error handler that can be attached to a file.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_create_errhandler(MPI_File_errhandler_function \fI*function\fP,
-	MPI_Errhandler \fI*errhandler\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_CREATE_ERRHANDLER(\fIFUNCTION, ERRHANDLER, IERROR\fP)
-	EXTERNAL	\fIFUNCTION\fP
-	INTEGER	\fIERRHANDLER, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_create_errhandler(\fIfile_errhandler_fn\fP, \fIerrhandler\fP, \fIierror\fP)
-	PROCEDURE(MPI_File_errhandler_function) :: \fIfile_errhandler_fn\fP
-	TYPE(MPI_Errhandler), INTENT(OUT) :: \fIerrhandler\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH DEPRECATED TYPE NAME NOTE
-.ft R
-MPI-2.2 deprecated the MPI_File_errhandler_fn and
-MPI::file::Errhandler_fn types in favor of
-MPI_File_errhandler_function and MPI::File::Errhandler_function,
-respectively.  Open MPI supports both names (indeed, the _fn names are
-typedefs to the _function names).
-
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-function
-User-defined error handling procedure (function).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-errhandler
-MPI error handler (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Registers the user routine \fIfunction\fP for use as an MPI error handler. Returns in errhandler a handle to the registered error handler.
-.sp
-In the C language, the user routine \fIfunction\fP should be a C function of type MPI_File_errhandler_function, which is defined as
-.sp
-.nf
-    typedef void (MPI_File_errhandler_function)(MPI_File *, int *,
-    \&...);
-.fi
-.sp
-The first argument to \fIfunction\fP is the file in use. The second is the error code
-to be returned by the MPI routine that raised the error.
-.sp
-In the Fortran language, the user routine should be of the form:
-.sp
-.nf
-    SUBROUTINE FILE_ERRHANDLER_FUNCTION(FILE, ERROR_CODE, ...)
-        INTEGER FILE, ERROR_CODE
-.fi
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-
diff --git a/ompi/mpi/man/man3/MPI_File_delete.3in b/ompi/mpi/man/man3/MPI_File_delete.3in
deleted file mode 100644
index 72d9cee190f..00000000000
--- a/ompi/mpi/man/man3/MPI_File_delete.3in
+++ /dev/null
@@ -1,68 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_delete 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_delete\fP \- Deletes a file.
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_delete(const char \fI*filename\fP, MPI_Info \fIinfo\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_DELETE(\fIFILENAME\fP, \fIINFO\fP, \fIIERROR\fP)
-	CHARACTER*(*)	\fIFILENAME\fP
-	INTEGER	\fIINFO, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_delete(\fIfilename\fP, \fIinfo\fP, \fIierror\fP)
-	CHARACTER(LEN=*), INTENT(IN) :: \fIfilename\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-filename
-Name of file to delete (string).
-.TP 1i
-info
-Info object (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_delete deletes the file identified by the file name
-\fIfilename\fP, provided it is not currently open by any process. It is an error to delete the file with MPI_File_delete if some process has it open, but MPI_File_delete does not check this. If the file does not exist, MPI_File_delete returns an error in the class MPI_ERR_NO_SUCH_FILE.
-.sp
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-
diff --git a/ompi/mpi/man/man3/MPI_File_f2c.3in b/ompi/mpi/man/man3/MPI_File_f2c.3in
deleted file mode 100644
index f2a362310d8..00000000000
--- a/ompi/mpi/man/man3/MPI_File_f2c.3in
+++ /dev/null
@@ -1,2 +0,0 @@
-.\" -*- nroff -*-
-.so man3/MPI_Comm_f2c.3
diff --git a/ompi/mpi/man/man3/MPI_File_get_amode.3in b/ompi/mpi/man/man3/MPI_File_get_amode.3in
deleted file mode 100644
index 8100dadba93..00000000000
--- a/ompi/mpi/man/man3/MPI_File_get_amode.3in
+++ /dev/null
@@ -1,67 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_get_amode 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_get_amode\fP \- Returns access mode associated with an open file.
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_get_amode(MPI_File \fIfh\fP, int \fI*amode\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_GET_AMODE(\fIFH\fP, \fIAMODE\fP, \fIIERROR\fP)
-	INTEGER	\fIFH, AMODE, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_get_amode(\fIfh\fP, \fIamode\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	INTEGER, INTENT(OUT) :: \fIamode\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-amode
-File access mode used to open the file (integer).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-
-MPI_File_get_amode returns, in
-.I amode,
-the access mode associated with the open file
-.I fh.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_get_atomicity.3in b/ompi/mpi/man/man3/MPI_File_get_atomicity.3in
deleted file mode 100644
index 3d9631cd740..00000000000
--- a/ompi/mpi/man/man3/MPI_File_get_atomicity.3in
+++ /dev/null
@@ -1,72 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_get_atomicity 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_get_atomicity\fP \- Returns current consistency semantics for data-access operations.
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_get_atomicity(MPI_File \fIfh\fP, int \fI*flag\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_GET_ATOMICITY(\fIFH\fP, \fIFLAG\fP, \fIIERROR\fP)
-	INTEGER	\fIFH, IERROR\fP
-	LOGICAL	\fIFLAG\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_get_atomicity(\fIfh\fP, \fIflag\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	LOGICAL, INTENT(OUT) :: \fIflag\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-flag
-true if atomic mode is enabled, false if nonatomic mode is enabled (boolean).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_get_atomicity returns the current consistency semantics for
-data access operations on the set of file handles created by one
-collective MPI_File_open. If \fIflag\fP is
-.I true,
-atomic mode is currently enabled; if
-.I flag
-is
-.I false,
-nonatomic mode is currently enabled.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_get_byte_offset.3in b/ompi/mpi/man/man3/MPI_File_get_byte_offset.3in
deleted file mode 100644
index 67bd05731a3..00000000000
--- a/ompi/mpi/man/man3/MPI_File_get_byte_offset.3in
+++ /dev/null
@@ -1,86 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_get_byte_offset 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_get_byte_offset\fP \- Converts a view-relative offset into an absolute byte position.
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_get_byte_offset(MPI_File \fIfh\fP, MPI_Offset \fIoffset\fP,
-	MPI_Offset \fI*disp\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_GET_BYTE_OFFSET(\fIFH\fP, \fIOFFSET\fP, \fIDISP\fP, \fIIERROR\fP)
-	INTEGER	\fIFH, IERROR\fP
-	INTEGER(KIND=MPI_OFFSET_KIND)	\fIOFFSET, DISP\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_get_byte_offset(\fIfh\fP, \fIoffset\fP, \fIdisp\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	INTEGER(KIND=MPI_OFFSET_KIND), INTENT(IN) :: \fIoffset\fP
-	INTEGER(KIND=MPI_OFFSET_KIND), INTENT(OUT) :: \fIdisp\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-fh
-File handle (handle).
-.ft R
-.TP 1i
-offset
-Offset (integer).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-disp
-Absolute byte position of offset (integer).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_get_byte_offset converts an offset specified for the current view to its corresponding displacement value, or absolute byte position, from the beginning of the file. The absolute byte position of \fIoffset\fP relative to the current view of \fIfh\fP is returned in \fIdisp\fP.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fIOFFSET\fP and \fIDISP\fP arguments only for Fortran 90. Sun FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_OFFSET_KIND \fIOFFSET\fP
-or
-     INTEGER*MPI_OFFSET_KIND \fIDISP\fP
-.fi
-.sp
-where MPI_OFFSET_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-
diff --git a/ompi/mpi/man/man3/MPI_File_get_errhandler.3in b/ompi/mpi/man/man3/MPI_File_get_errhandler.3in
deleted file mode 100644
index 1f26f9e5b47..00000000000
--- a/ompi/mpi/man/man3/MPI_File_get_errhandler.3in
+++ /dev/null
@@ -1,64 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2016      Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_get_errhandler 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_get_errhandler \fP \- Gets the error handler for a file.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_get_errhandler(MPI_File \fIfile\fP, MPI_Errhandler\fI
-	*errhandler\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_GET_ERRHANDLER(\fIFILE, ERRHANDLER, IERROR\fP)
-	INTEGER	\fIFILE, ERRHANDLER, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_get_errhandler(\fIfile\fP, \fIerrhandler\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfile\fP
-	TYPE(MPI_Errhandler), INTENT(OUT) :: \fIerrhandler\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-file
-File (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-errhandler
-MPI error handler currently associated with file (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Returns in \fIerrhandler\fP (a handle to) the error handler that is currently associated with file \fIfile\fP.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_get_group.3in b/ompi/mpi/man/man3/MPI_File_get_group.3in
deleted file mode 100644
index cd945151d98..00000000000
--- a/ompi/mpi/man/man3/MPI_File_get_group.3in
+++ /dev/null
@@ -1,69 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_get_group 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_get_group\fP \- Returns a duplicate of the process group of a file.
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_get_group(MPI_File \fIfh\fP, MPI_Group \fI*group\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_GET_GROUP(\fIFH\fP, \fIGROUP\fP, \fIIERROR\fP)
-	INTEGER	\fIFH, GROUP, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_get_group(\fIfh\fP, \fIgroup\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	TYPE(MPI_Group), INTENT(OUT) :: \fIgroup\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH OUTPUT PARAMETERS
-.TP 1i
-group
-Group that opened the file (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_get_group returns a duplicate of the group of the communicator
-used to open the file associated with
-.I fh.
-The group is returned in
-.I group.
-The user is responsible for freeing
-.I group,
-using MPI_Group_free.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_get_info.3in b/ompi/mpi/man/man3/MPI_File_get_info.3in
deleted file mode 100644
index d97cefa4c7c..00000000000
--- a/ompi/mpi/man/man3/MPI_File_get_info.3in
+++ /dev/null
@@ -1,105 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_get_info 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_get_info\fP \- Returns a new info object containing values for current hints associated with a file.
-
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_get_info(MPI_File \fIfh\fP, MPI_Info \fI*info_used\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_GET_INFO(\fIFH\fP, \fIINFO_USED\fP, \fIIERROR\fP)
-	INTEGER	\fIFH, INFO_USED, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_get_info(\fIfh\fP, \fIinfo_used\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	TYPE(MPI_Info), INTENT(OUT) :: \fIinfo_used\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-info_used
-New info object (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_get_info returns a new info object containing all the hints that the system currently associates with the file \fIfh\fP. The current setting of all hints actually used by the system related to this open file is returned in \fIinfo_used\fP. The user is responsible for freeing \fIinfo_used\fP via MPI_Info_free.
-
-Note that the set of hints returned in \fIinfo_used\fP may be greater or smaller than the set of hints passed in to MPI_File_open, MPI_File_set_view, and MPI_File_set_info, as the system may not recognize some hints set by the user, and may automatically set other hints that the user has not requested to be set. See the HINTS section for a list of hints that can be set.
-
-.SH HINTS
-.ft R
-The following hints can be used as values for the \fIinfo_used\fP argument.
-.sp
-SETTABLE HINTS:
-.sp
-- shared_file_timeout: Amount of time (in seconds) to wait for access to the
-shared file pointer before exiting with MPI_ERR_TIMEDOUT.
-.sp
-- rwlock_timeout: Amount of time (in seconds) to wait for obtaining a read or
-write lock on a contiguous chunk of a UNIX file before exiting with MPI_ERR_TIMEDOUT.
-.sp
-- noncoll_read_bufsize:  Maximum size of the buffer used by
-MPI I/O to satisfy  read requests in
-the noncollective data-access routines. (See NOTE, below.)
-.sp
-- noncoll_write_bufsize: Maximum size of the buffer used by
-MPI I/O to satisfy write requests in
-the noncollective data-access routines. (See NOTE, below.)
-.sp
-- coll_read_bufsize:  Maximum size of the buffer used by MPI
-I/O to satisfy read requests in the
-collective data-access routines. (See NOTE, below.)
-.sp
-- coll_write_bufsize:  Maximum size of the buffer used by MPI
-I/O to satisfy write requests in the
-collective data-access routines. (See NOTE, below.)
-.sp
-NOTE: A buffer size smaller than the distance (in bytes) in a UNIX file between the first byte and the last byte of the access request causes MPI I/O to iterate and perform multiple UNIX read() or write() calls. If the request includes multiple noncontiguous chunks of data, and the buffer size is greater than the size of those chunks, then the UNIX read() or write() (made at the MPI I/O level) will access data not requested by this process in order to reduce the total number of write() calls made. If this is not desirable behavior, you should reduce this buffer size to equal the size of the contiguous chunks within the aggregate request.
-.sp
-- mpiio_concurrency: (boolean) controls whether nonblocking
-I/O routines can bind an extra thread to an LWP.
-.sp
-- mpiio_coll_contiguous: (boolean) controls whether subsequent collective data accesses will request collectively contiguous regions of the file.
-.sp
-NON-SETTABLE HINTS:
-.sp
-- filename: Access this hint to get the name of the file.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_get_position.3in b/ompi/mpi/man/man3/MPI_File_get_position.3in
deleted file mode 100644
index 6405a62baf4..00000000000
--- a/ompi/mpi/man/man3/MPI_File_get_position.3in
+++ /dev/null
@@ -1,81 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_get_position 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_get_position\fP \- Returns the current position of the individual file pointer.
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_get_position(MPI_File \fIfh\fP, MPI_Offset \fI*offset\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_GET_POSITION(\fIFH\fP, \fIOFFSET\fP, \fIIERROR\fP)
-	INTEGER	\fIFH, IERROR\fP
-	INTEGER(KIND=MPI_OFFSET_KIND)	\fIOFFSET\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_get_position(\fIfh\fP, \fIoffset\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	INTEGER(KIND=MPI_OFFSET_KIND), INTENT(OUT) :: \fIoffset\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-offset
-Offset of the individual file pointer (integer).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_get_position returns, in
-.I offset,
-the current position of the individual file pointer in
-.I etype
-units relative to the current displacement and file type.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fIOFFSET\fP argument only for Fortran 90. Sun FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_OFFSET_KIND \fIOFFSET\fP
-.fi
-.sp
-where MPI_ADDRESS_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_get_position_shared.3in b/ompi/mpi/man/man3/MPI_File_get_position_shared.3in
deleted file mode 100644
index 568ee4b3420..00000000000
--- a/ompi/mpi/man/man3/MPI_File_get_position_shared.3in
+++ /dev/null
@@ -1,82 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015      Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" Copyright (c) 2020      FUJITSU LIMITED.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_get_position_shared 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_get_position_shared\fP \- Returns the current position of the shared file pointer.
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_get_position_shared(MPI_File \fIfh\fP, MPI_Offset \fI*offset\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_GET_POSITION_SHARED(\fIFH\fP, \fIOFFSET\fP, \fIIERROR\fP)
-	INTEGER	\fIFH, IERROR\fP
-	INTEGER(KIND=MPI_OFFSET_KIND)	\fIOFFSET\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_get_position_shared(\fIfh\fP, \fIoffset\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	INTEGER(KIND=MPI_OFFSET_KIND), INTENT(OUT) :: \fIoffset\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-offset
-Offset of the shared file pointer (integer).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_get_position_shared returns, in
-.I offset,
-the current position of the shared file pointer in
-.I etype
-units relative to the current displacement and file type.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fIOFFSET\fP argument only for Fortran 90. Sun FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_OFFSET_KIND \fIOFFSET\fP
-.fi
-.sp
-where MPI_ADDRESS_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_get_size.3in b/ompi/mpi/man/man3/MPI_File_get_size.3in
deleted file mode 100644
index e13aae9823e..00000000000
--- a/ompi/mpi/man/man3/MPI_File_get_size.3in
+++ /dev/null
@@ -1,85 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_get_size 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_get_size\fP \- Returns the current size of the file.
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_get_size(MPI_File \fIfh\fP, MPI_Offset \fI*size\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_GET_SIZE(\fIFH\fP, \fISIZE\fP, \fIIERROR\fP)
-	INTEGER	\fIFH, ERROR\fP
-	INTEGER(KIND=MPI_OFFSET_KIND)	\fISIZE\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_get_size(\fIfh\fP, \fIsize\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	INTEGER(KIND=MPI_OFFSET_KIND), INTENT(OUT) :: \fIsize\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-fh
-File handle (handle).
-.TP 1i
-size
-Size of the file in bytes (integer).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_get_size returns, in
-.I size
-, the current size in bytes of the file associated with the file handle
-\fIfh\fP. Note that the file size returned by Solaris may not represent the number of bytes physically allocated for the file in those cases where all bytes in this file have not been written at least once.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fISIZE\fP argument only for Fortran 90. Sun FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_OFFSET_KIND \fISIZE\fP
-.fi
-.sp
-where MPI_ADDRESS_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.br
-MPI_File_preallocate
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_File_get_type_extent.3in b/ompi/mpi/man/man3/MPI_File_get_type_extent.3in
deleted file mode 100644
index 2199129ed25..00000000000
--- a/ompi/mpi/man/man3/MPI_File_get_type_extent.3in
+++ /dev/null
@@ -1,88 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_get_type_extent 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_get_type_extent\fP \- Returns the extent of the data type in a file.
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_get_type_extent(MPI_File \fIfh\fP, MPI_Datatype
-	\fIdatatype\fP, MPI_Aint \fI*extent\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_GET_TYPE_EXTENT(\fIFH\fP, \fIDATATYPE\fP, \fIEXTENT\fP, \fIIERROR\fP)
-	INTEGER	\fIFH, DATATYPE, IERROR\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND)	\fIEXTENT\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_get_type_extent(\fIfh\fP, \fIdatatype\fP, \fIextent\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(OUT) :: \fIextent\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-fh
-File handle (handle).
-.ft R
-.TP 1i
-datatype
-Data type (handle).
-
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-extent
-Data type extent (integer).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_get_type_extent can be used to calculate \fIextent\fP for \fIdatatype\fP in the file. The extent is the same for all processes accessing the file associated with \fIfh\fP. If the current view uses a user-defined data representation, MPI_File_get_type_extent uses the \fIdtype_file_extent_fn\fP callback to calculate the extent.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fIEXTENT\fP argument only for Fortran 90. FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_ADDRESS_KIND \fIEXTENT\fP
-.fi
-.sp
-where MPI_ADDRESS_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH NOTES
-.ft R
-If the file data representation is other than "native," care must be taken in constructing etypes and file types. Any of the data-type constructor functions may be used; however, for those functions that accept displacements in bytes, the displacements must be specified in terms of their values in the file for the file data representation being used. MPI will interpret these byte displacements as is; no scaling will be done. The function MPI_File_get_type_extent can be used to calculate the extents of data types in the file. For etypes and  file types that are portable data types, MPI will scale any displacements in the data types to match the file data representation. Data types passed as arguments to read/write routines specify the data layout in memory; therefore, they must always be constructed using displacements corresponding to displacements in memory.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_get_view.3in b/ompi/mpi/man/man3/MPI_File_get_view.3in
deleted file mode 100644
index aee619c2119..00000000000
--- a/ompi/mpi/man/man3/MPI_File_get_view.3in
+++ /dev/null
@@ -1,101 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_get_view 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_get_view\fP \- Returns the process's view of data in the file.
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_get_view(MPI_File \fIfh\fP, MPI_Offset \fI*disp\fP,
-	MPI_Datatype \fI*etype\fP, MPI_Datatype \fI*filetype\fP,
-	char \fI*datarep\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_GET_VIEW(\fIFH\fP, \fIDISP\fP, \fIETYPE\fP,
-	\fIFILETYPE\fP, \fIDATAREP\fP, \fIIERROR\fP)
-	INTEGER	\fIFH, ETYPE, FILETYPE, IERROR\fP
-	CHARACTER*(*)	\fIDATAREP\fP
-	INTEGER(KIND=MPI_OFFSET_KIND)	\fIDISP\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_get_view(\fIfh\fP, \fIdisp\fP, \fIetype\fP, \fIfiletype\fP, \fIdatarep\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	INTEGER(KIND=MPI_OFFSET_KIND), INTENT(OUT) :: \fIdisp\fP
-	TYPE(MPI_Datatype), INTENT(OUT) :: \fIetype\fP, \fIfiletype\fP
-	CHARACTER(LEN=*), INTENT(OUT) :: \fIdatarep\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-disp
-Displacement (integer).
-.TP 1i
-etype
-Elementary data type (handle).
-.TP 1i
-filetype
-File type (handle). See Restrictions, below.
-.TP 1i
-datarep
-Data representation (string).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-The MPI_File_get_view routine returns the process's view of the data
-in the file. The current values of the displacement, etype, and
-filetype are returned in
-.I disp,
-.I etype,
-and
-.I filetype,
-respectively.
-.sp
-The MPI_File_get_view interface allows the user to pass a data-representation string via the \fIdatarep\fP argument.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fIDISP\fP argument only for Fortran 90.  FORTRAN 77
-users may use the non-portable syntax.
-.sp
-.nf
-     INTEGER*MPI_OFFSET_KIND \fIDISP\fP
-.fi
-.sp
-where MPI_OFFSET_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_iread.3in b/ompi/mpi/man/man3/MPI_File_iread.3in
deleted file mode 100644
index df4ca42536e..00000000000
--- a/ompi/mpi/man/man3/MPI_File_iread.3in
+++ /dev/null
@@ -1,96 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_iread 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_iread\fP \- Reads a file starting at the location specified by the individual file pointer (nonblocking, noncollective).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_iread(MPI_File \fIfh\fP, void  \fI*buf\fP, int  \fIcount\fP,
-	MPI_Datatype  \fIdatatype\fP, MPI_Request  \fI*request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_IREAD(\fIFH\fP, \fIBUF\fP, \fICOUNT\fP, \fIDATATYPE\fP, \fIREQUEST\fP, \fIIERROR\fP)
-	<type>	\fIBUF(*)\fP
-	INTEGER	\fIFH, COUNT, DATATYPE, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_iread(\fIfh\fP, \fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-count
-Number of elements in the buffer (integer).
-.ft R
-.TP 1i
-datatype
-Data type of each buffer element (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of buffer (choice).
-.ft R
-.TP 1i
-request
-Request object (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_iread is a nonblocking version of MPI_File_read. It attempts to read from the file associated with
-.I fh
-at the current individual file pointer position maintained by the system in which a total number of
-.I count
-data items having
-.I datatype
-type  are read into the user's buffer
-.I buf.
-The data is taken out of those parts of the
-file specified by the current view. MPI_File_iread stores the
-number of data-type elements actually read in
-.I status.
-All other fields of
-.I status
-are undefined. It is erroneous to call this function if MPI_MODE_SEQUENTIAL mode was specified when the file was opened.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_iread_all.3in b/ompi/mpi/man/man3/MPI_File_iread_all.3in
deleted file mode 100644
index 374604761a9..00000000000
--- a/ompi/mpi/man/man3/MPI_File_iread_all.3in
+++ /dev/null
@@ -1,96 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_iread_all 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_iread_all\fP \- Reads a file starting at the location specified by the individual file pointer (nonblocking, collective).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_iread_all(MPI_File \fIfh\fP, void  \fI*buf\fP, int  \fIcount\fP,
-	MPI_Datatype  \fIdatatype\fP, MPI_Request  \fI*request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_IREAD_ALL(\fIFH\fP, \fIBUF\fP, \fICOUNT\fP, \fIDATATYPE\fP, \fIREQUEST\fP, \fIIERROR\fP)
-	<type>	\fIBUF(*)\fP
-	INTEGER	\fIFH, COUNT, DATATYPE, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_iread_all(\fIfh\fP, \fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	TYPE(*), DIMENSION(..) :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-count
-Number of elements in the buffer (integer).
-.ft R
-.TP 1i
-datatype
-Data type of each buffer element (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of buffer (choice).
-.ft R
-.TP 1i
-request
-Request object (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_iread_all is a nonblocking version of MPI_File_read_all. It attempts to read from the file associated with
-.I fh
-at the current individual file pointer position maintained by the system in which a total number of
-.I count
-data items having
-.I datatype
-type  are read into the user's buffer
-.I buf.
-The data is taken out of those parts of the
-file specified by the current view. MPI_File_iread_all stores the
-number of data-type elements actually read in
-.I status.
-All other fields of
-.I status
-are undefined. It is erroneous to call this function if MPI_MODE_SEQUENTIAL mode was specified when the file was opened.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_iread_at.3in b/ompi/mpi/man/man3/MPI_File_iread_at.3in
deleted file mode 100644
index cc65fa0c679..00000000000
--- a/ompi/mpi/man/man3/MPI_File_iread_at.3in
+++ /dev/null
@@ -1,123 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_iread_at 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_iread_at\fP \- Reads a file at an explicitly specified offset (nonblocking, noncollective).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_iread_at(MPI_File \fIfh\fP, MPI_Offset \fIoffset\fP,
-	void \fI*buf\fP, int \fIcount\fP, MPI_Datatype \fIdatatype\fP,
-	MPI_Request \fI*request\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_IREAD_AT(\fIFH\fP, \fIOFFSET\fP, \fIBUF\fP, \fICOUNT\fP, \fIDATATYPE\fP, \fIREQUEST\fP, \fIIERROR\fP)
-	<type>	\fIBUF\fP(*)
-	INTEGER	\fIFH, COUNT, DATATYPE, REQUEST, IERROR\fP
-	INTEGER(KIND=MPI_OFFSET_KIND)	\fIOFFSET\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_iread_at(\fIfh\fP, \fIoffset\fP, \fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	INTEGER(KIND=MPI_OFFSET_KIND), INTENT(IN) :: \fIoffset\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-fh
-File handle (handle).
-.ft R
-.TP 1i
-offset
-File offset (integer).
-.ft R
-.TP 1i
-count
-Number of elements in the buffer (integer).
-.ft R
-.TP 1i
-datatype
-Data type of each buffer element (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of the buffer (choice).
-.ft R
-.TP 1i
-request
-Request object (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_iread_at is the nonblocking version of MPI_File_read_at.
-
-MPI_File_iread_at is a nonblocking routine that attempts to read from the file associated with
-.I fh
-at the
-.I offset
-position a total number of
-.I count
-data items having
-.I datatype
-type into the user's buffer
-.I buf.
-The
-.I offset
-is in etype units relative to the current view. That is, holes are not counted
-when locating an offset. The data is taken out of those parts of the
-file specified by the current view. MPI_File_iread_at stores the
-number of
-.I datatype
-elements actually read in
-.I status.
-All other fields of
-.I status
-are undefined.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fIOFFSET\fP argument only for Fortran 90.  Sun FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_OFFSET_KIND \fIOFFSET\fP
-.fi
-.sp
-where MPI_OFFSET_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_iread_at_all.3in b/ompi/mpi/man/man3/MPI_File_iread_at_all.3in
deleted file mode 100644
index c0c93bd67f8..00000000000
--- a/ompi/mpi/man/man3/MPI_File_iread_at_all.3in
+++ /dev/null
@@ -1,123 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_iread_at_all 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_iread_at_all\fP \- Reads a file at an explicitly specified offset (nonblocking, collective).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_iread_at_all(MPI_File \fIfh\fP, MPI_Offset \fIoffset\fP,
-	void \fI*buf\fP, int \fIcount\fP, MPI_Datatype \fIdatatype\fP,
-	MPI_Request \fI*request\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_IREAD_AT_ALL(\fIFH\fP, \fIOFFSET\fP, \fIBUF\fP, \fICOUNT\fP, \fIDATATYPE\fP, \fIREQUEST\fP, \fIIERROR\fP)
-	<type>	\fIBUF\fP(*)
-	INTEGER	\fIFH, COUNT, DATATYPE, REQUEST, IERROR\fP
-	INTEGER(KIND=MPI_OFFSET_KIND)	\fIOFFSET\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_iread_at_all(\fIfh\fP, \fIoffset\fP, \fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	INTEGER(KIND=MPI_OFFSET_KIND), INTENT(IN) :: offset
-	TYPE(*), DIMENSION(..) :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-fh
-File handle (handle).
-.ft R
-.TP 1i
-offset
-File offset (integer).
-.ft R
-.TP 1i
-count
-Number of elements in the buffer (integer).
-.ft R
-.TP 1i
-datatype
-Data type of each buffer element (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of the buffer (choice).
-.ft R
-.TP 1i
-request
-Request object (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_iread_at_all is the nonblocking version of MPI_File_read_at_all.
-
-MPI_File_iread_at_all is a nonblocking routine that attempts to read from the file associated with
-.I fh
-at the
-.I offset
-position a total number of
-.I count
-data items having
-.I datatype
-type into the user's buffer
-.I buf.
-The
-.I offset
-is in etype units relative to the current view. That is, holes are not counted
-when locating an offset. The data is taken out of those parts of the
-file specified by the current view. MPI_File_iread_at_all stores the
-number of
-.I datatype
-elements actually read in
-.I status.
-All other fields of
-.I status
-are undefined.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fIOFFSET\fP argument only for Fortran 90.  Sun FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_OFFSET_KIND \fIOFFSET\fP
-.fi
-.sp
-where MPI_OFFSET_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_iread_shared.3in b/ompi/mpi/man/man3/MPI_File_iread_shared.3in
deleted file mode 100644
index 0fbbdf39492..00000000000
--- a/ompi/mpi/man/man3/MPI_File_iread_shared.3in
+++ /dev/null
@@ -1,82 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_iread_shared 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_iread_shared\fP \- Reads a file using the shared file pointer (nonblocking, noncollective).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_iread_shared(MPI_File \fIfh\fP, void \fI*buf\fP, int \fIcount\fP,
-	MPI_Datatype \fIdatatype\fP, MPI_Request \fI*request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_IREAD_SHARED(\fIFH\fP, \fIBUF\fP, \fICOUNT\fP, \fIDATATYPE\fP, \fIREQUEST\fP, \fIIERROR\fP)
-	<type>	\fIBUF(*)\fP
-	INTEGER	\fIFH, COUNT, DATATYPE, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_iread_shared(\fIfh\fP, \fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-count
-Number of elements in buffer (integer).
-.ft R
-.TP 1i
-datatype
-Data type of each buffer element (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of buffer (choice).
-.ft R
-.TP 1i
-request
-Request object (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_iread_shared is a nonblocking version of the MPI_File_read_shared interface. It uses the shared file pointer to read files. The order of serialization among the processors is not deterministic for this noncollective routine, so you need to use other methods of synchronization to impose a particular order among processors.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_iwrite.3in b/ompi/mpi/man/man3/MPI_File_iwrite.3in
deleted file mode 100644
index f0850bdd6ec..00000000000
--- a/ompi/mpi/man/man3/MPI_File_iwrite.3in
+++ /dev/null
@@ -1,101 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_iwrite 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_iwrite\fP \- Writes a file starting at the location specified by the individual file pointer (nonblocking, noncollective).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_iwrite(MPI_File \fIfh\fP, const void \fI*buf\fP, int \fIcount\fP,
-	MPI_Datatype \fIdatatype\fP, MPI_Request \fI*request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_IWRITE(\fIFH\fP, \fIBUF\fP, \fICOUNT\fP, \fIDATATYPE\fP, \fIREQUEST\fP, \fIIERROR\fP)
-	<type>	\fIBUF(*)\fP
-	INTEGER	\fIFH, COUNT, DATATYPE, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_iwrite(\fIfh\fP, \fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of buffer (choice).
-.ft R
-.TP 1i
-count
-Number of elements in buffer (integer).
-.ft R
-.TP 1i
-datatype
-Data type of each buffer element (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-request
-Request object (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_iwrite is a nonblocking version of the MPI_File_write interface. It attempts to write into the file associated with
-.I fh
-(at the current individual file pointer position maintained by the system) a total number of
-.I count
-data items having
-.I datatype
-type from the user's buffer
-.I buf.
-The data is written into those parts of the
-file specified by the current view. MPI_File_iwrite stores the
-number of
-.I datatype
-elements actually written in
-.I status.
-All other fields of
-.I status
-are undefined.
-.sp
-It is erroneous to call this function if MPI_MODE_SEQUENTIAL mode was specified when the file was open.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_iwrite_all.3in b/ompi/mpi/man/man3/MPI_File_iwrite_all.3in
deleted file mode 100644
index 1440a502b08..00000000000
--- a/ompi/mpi/man/man3/MPI_File_iwrite_all.3in
+++ /dev/null
@@ -1,101 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_iwrite_all 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_iwrite_all\fP \- Writes a file starting at the location specified by the individual file pointer (nonblocking, collective).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_iwrite_all(MPI_File \fIfh\fP, const void \fI*buf\fP, int \fIcount\fP,
-	MPI_Datatype \fIdatatype\fP, MPI_Request \fI*request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_IWRITE_ALL(\fIFH\fP, \fIBUF\fP, \fICOUNT\fP, \fIDATATYPE\fP, \fIREQUEST\fP, \fIIERROR\fP)
-	<type>	\fIBUF(*)\fP
-	INTEGER	\fIFH, COUNT, DATATYPE, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_iwrite_all(\fIfh\fP, \fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	TYPE(*), DIMENSION(..) :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of buffer (choice).
-.ft R
-.TP 1i
-count
-Number of elements in buffer (integer).
-.ft R
-.TP 1i
-datatype
-Data type of each buffer element (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-request
-Request object (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_iwrite_all is a nonblocking version of the MPI_File_write_all interface. It attempts to write into the file associated with
-.I fh
-(at the current individual file pointer position maintained by the system) a total number of
-.I count
-data items having
-.I datatype
-type from the user's buffer
-.I buf.
-The data is written into those parts of the
-file specified by the current view. MPI_File_iwrite_all stores the
-number of
-.I datatype
-elements actually written in
-.I status.
-All other fields of
-.I status
-are undefined.
-.sp
-It is erroneous to call this function if MPI_MODE_SEQUENTIAL mode was specified when the file was open.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_iwrite_at.3in b/ompi/mpi/man/man3/MPI_File_iwrite_at.3in
deleted file mode 100644
index 4adb62d77f0..00000000000
--- a/ompi/mpi/man/man3/MPI_File_iwrite_at.3in
+++ /dev/null
@@ -1,125 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_iwrite_at 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_iwrite_at\fP \- Writes a file at an explicitly specified offset (nonblocking, noncollective).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_iwrite_at(MPI_File \fIfh\fP, MPI_Offset \fIoffset\fP,
-	const void \fI*buf\fP, int \fIcount\fP, MPI_Datatype \fIdatatype\fP, MPI_Request \fI*request\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_IWRITE_AT(\fIFH\fP, \fIOFFSET\fP, \fIBUF\fP, \fICOUNT\fP, \fIDATATYPE\fP, \fIREQUEST\fP, \fIIERROR\fP)
-	<type>	\fIBUF\fP(*)
-	INTEGER	\fIFH, COUNT, DATATYPE, REQUEST, IERROR\fP
-	INTEGER(KIND=MPI_OFFSET_KIND)	\fIOFFSET\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_iwrite_at(\fIfh\fP, \fIoffset\fP, \fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	INTEGER(KIND=MPI_OFFSET_KIND), INTENT(IN) :: \fIoffset\fP
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-offset
-File offset (integer).
-.ft R
-.TP 1i
-buf
-Initial address of buffer (choice).
-.ft R
-.TP 1i
-count
-Number of elements in buffer (integer).
-.ft R
-.TP 1i
-datatype
-Data type of each buffer element (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-request
-Request object (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_iwrite_at is a nonblocking version of MPI_File_write_at. It attempts to write into the file associated with
-.I fh
-(at the
-.I offset
-position) a total number of
-.I count
-data items having
-.I datatype
-type from the user's buffer
-.I buf.
-The offset is in
-.I etype
-units relative to the current view. That is, holes are not counted
-when locating an offset. The data is written into those parts of the
-file specified by the current view. MPI_File_iwrite_at stores the
-number of
-.I datatype
-elements actually written in
-.I status.
-All other fields of
-.I status
-are undefined. The request structure can be passed to MPI_Wait or MPI_Test, which will return a status with the number of bytes actually accessed.
-.sp
-It is erroneous to call this function if MPI_MODE_SEQUENTIAL mode was specified when the file was open.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fIOFFSET\fP argument only for Fortran 90.  FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_OFFSET_KIND \fIOFFSET\fP
-.fi
-.sp
-where MPI_OFFSET_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_iwrite_at_all.3in b/ompi/mpi/man/man3/MPI_File_iwrite_at_all.3in
deleted file mode 100644
index 3b9db67c415..00000000000
--- a/ompi/mpi/man/man3/MPI_File_iwrite_at_all.3in
+++ /dev/null
@@ -1,125 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_iwrite_at_all 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_iwrite_at_all\fP \- Writes a file at an explicitly specified offset (nonblocking, collective).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_iwrite_at_all(MPI_File \fIfh\fP, MPI_Offset \fIoffset\fP,
-	const void \fI*buf\fP, int \fIcount\fP, MPI_Datatype \fIdatatype\fP, MPI_Request \fI*request\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_IWRITE_AT_ALL(\fIFH\fP, \fIOFFSET\fP, \fIBUF\fP, \fICOUNT\fP, \fIDATATYPE\fP, \fIREQUEST\fP, \fIIERROR\fP)
-	<type>	\fIBUF\fP(*)
-	INTEGER	\fIFH, COUNT, DATATYPE, REQUEST, IERROR\fP
-	INTEGER(KIND=MPI_OFFSET_KIND)	\fIOFFSET\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_iwrite_at_all(\fIfh\fP, \fIoffset\fP, \fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	INTEGER(KIND=MPI_OFFSET_KIND), INTENT(IN) :: offset
-	TYPE(*), DIMENSION(..) :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-offset
-File offset (integer).
-.ft R
-.TP 1i
-buf
-Initial address of buffer (choice).
-.ft R
-.TP 1i
-count
-Number of elements in buffer (integer).
-.ft R
-.TP 1i
-datatype
-Data type of each buffer element (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-request
-Request object (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_iwrite_at_all is a nonblocking version of MPI_File_write_at_all. It attempts to write into the file associated with
-.I fh
-(at the
-.I offset
-position) a total number of
-.I count
-data items having
-.I datatype
-type from the user's buffer
-.I buf.
-The offset is in
-.I etype
-units relative to the current view. That is, holes are not counted
-when locating an offset. The data is written into those parts of the
-file specified by the current view. MPI_File_iwrite_at_all stores the
-number of
-.I datatype
-elements actually written in
-.I status.
-All other fields of
-.I status
-are undefined. The request structure can be passed to MPI_Wait or MPI_Test, which will return a status with the number of bytes actually accessed.
-.sp
-It is erroneous to call this function if MPI_MODE_SEQUENTIAL mode was specified when the file was open.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fIOFFSET\fP argument only for Fortran 90.  FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_OFFSET_KIND \fIOFFSET\fP
-.fi
-.sp
-where MPI_OFFSET_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_iwrite_shared.3in b/ompi/mpi/man/man3/MPI_File_iwrite_shared.3in
deleted file mode 100644
index 33c9f2285eb..00000000000
--- a/ompi/mpi/man/man3/MPI_File_iwrite_shared.3in
+++ /dev/null
@@ -1,84 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_iwrite_shared 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_iwrite_shared\fP \- Writes a file using the shared file pointer (nonblocking, noncollective).
-
-
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_iwrite_shared(MPI_File \fIfh\fP, const void \fI*buf\fP, int \fIcount\fP, MPI_Datatype
-	\fIdatatype\fP, MPI_Request \fI*request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_IWRITE_SHARED(\fIFH\fP, \fIBUF\fP, \fICOUNT\fP, \fIDATATYPE\fP, \fIREQUEST\fP, \fIIERROR\fP)
-	<type>	\fIBUF(*)\fP
-	INTEGER	\fIFH, COUNT, DATATYPE, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_iwrite_shared(\fIfh\fP, \fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-count
-Number of elements in buffer (integer).
-.ft R
-.TP 1i
-datatype
-Data type of each buffer element (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of buffer (choice).
-.TP 1i
-request
-Request object (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_iwrite_shared is a nonblocking routine that uses the shared file pointer to write files. The order of serialization is not deterministic for this noncollective routine, so you need to use other methods of synchronization to impose a particular order.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_open.3in b/ompi/mpi/man/man3/MPI_File_open.3in
deleted file mode 100644
index 32ac6c4c229..00000000000
--- a/ompi/mpi/man/man3/MPI_File_open.3in
+++ /dev/null
@@ -1,191 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_open 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_open\fP \- Opens a file (collective).
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_open(MPI_Comm \fIcomm\fP, const char \fI*filename\fP,
-	int \fIamode\fP, MPI_Info \fIinfo\fP,
-	MPI_File \fI*fh\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_OPEN(\fICOMM\fP, \fIFILENAME\fP, \fIAMODE\fP, \fIINFO\fP, \fIFH\fP, \fIIERROR\fP)
-	CHARACTER*(*)	\fIFILENAME\fP
-	INTEGER	\fICOMM, AMODE, INFO, FH, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_open(\fIcomm\fP, \fIfilename\fP, \fIamode\fP, \fIinfo\fP, \fIfh\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	CHARACTER(LEN=*), INTENT(IN) :: \fIfilename\fP
-	INTEGER, INTENT(IN) :: \fIamode\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(MPI_File), INTENT(OUT) :: \fIfh\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-comm
-Communicator (handle).
-.TP 1i
-filename
-Name of file to open (string).
-.TP 1i
-amode
-File access mode (integer).
-.TP 1i
-info
-Info object (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-fh
-New file handle (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_open opens the file identified by the filename
-.I
-filename
-on all processes in the
-.I comm
-communicator group. MPI_File_open is a collective routine; all processes
-must provide the same value for
-.I amode,
-and all processes must provide filenames that reference the same
-file which are textually identical (note: Open MPI I/O plugins may
-have restrictions on characters that can be used in filenames. For
-example, the ROMIO plugin may disallow the colon (":") character from
-appearing in a filename). A process can open a file independently of
-other processes by using the MPI_COMM_SELF communicator. The file
-handle returned,
-.I fh,
-can be subsequently used to access the file until the file is closed
-using MPI_File_close. Before calling MPI_Finalize, the user is required to
-close (via MPI_File_close) all files that were opened with MPI_File_open. Note
-that the communicator
-.I comm
-is unaffected by MPI_File_open and continues to be usable in all MPI
-routines. Furthermore, use of
-.I comm
-will not interfere with I/O behavior.
-.sp
-Initially, all processes view the file as a linear byte stream; that is, the
-.I etype
-and
-.I filetype
-are both MPI_BYTE. The file view can be changed via the MPI_File_set_view routine.
-.sp
-The following access modes are supported (specified in amode, in a bit-vector OR in one of the following integer constants):
-.TP .5i
-  o
-MPI_MODE_APPEND
-.TP .5i
-  o
-MPI_MODE_CREATE -- Create the file if it does not exist.
-.TP .5i
-  o
-MPI_MODE_DELETE_ON_CLOSE
-.TP .5i
-  o
-MPI_MODE_EXCL -- Error creating a file that already exists.
-.TP .5i
-  o
-MPI_MODE_RDONLY -- Read only.
-.TP .5i
-  o
-MPI_MODE_RDWR -- Reading and writing.
-.TP .5i
-  o
-MPI_MODE_SEQUENTIAL
-.TP .5i
-  o
-MPI_MODE_WRONLY -- Write only.
-.TP .5i
-  o
-MPI_MODE_UNIQUE_OPEN
-.RE
-.sp
-The modes MPI_MODE_RDONLY, MPI_MODE_RDWR, MPI_MODE_WRONLY, and MPI_MODE_CREATE have
-identical semantics to their POSIX counterparts. It is erroneous to
-specify MPI_MODE_CREATE in conjunction with MPI_MODE_RDONLY. Errors related to
-the access mode are raised in the class MPI_ERR_AMODE.
-.sp
-On single-node clusters, files are opened by default using nonatomic mode file consistency
-semantics. The more stringent atomic-mode consistency semantics, required for atomicity of overlapping accesses, are the default when processors in a communicator group reside on more than one node.
-This setting can be changed using
-MPI_File_set_atomicity.
-.sp
-The MPI_File_open interface allows the user to pass information via the \fIinfo\fP argument. It can be set to MPI_INFO_NULL. See the HINTS section for a list of hints that can be set.
-
-.SH HINTS
-.ft R
-The following hints can be used as values for the \fIinfo\fP argument.
-.sp
-SETTABLE HINTS:
-.sp
-- MPI_INFO_NULL
-.sp
-- shared_file_timeout: Amount of time (in seconds) to wait for access to the
-shared file pointer before exiting with MPI_ERR_TIMEDOUT.
-.sp
-- rwlock_timeout: Amount of time (in seconds) to wait for obtaining a read or
-write lock on a contiguous chunk of a UNIX file before exiting with MPI_ERR_TIMEDOUT.
-.sp
-- noncoll_read_bufsize:  Maximum size of the buffer used by
-MPI I/O to satisfy multiple noncontiguous read requests in
-the noncollective data-access routines. (See NOTE, below.)
-.sp
-- noncoll_write_bufsize: Maximum size of the buffer used by
-MPI I/O to satisfy multiple noncontiguous write requests in
-the noncollective data-access routines. (See NOTE, below.)
-.sp
-- coll_read_bufsize:  Maximum size of the buffer used by MPI
-I/O to satisfy multiple noncontiguous read requests in the
-collective data-access routines. (See NOTE, below.)
-.sp
-- coll_write_bufsize:  Maximum size of the buffer used by MPI
-I/O to satisfy multiple noncontiguous write requests in the
-collective data-access routines. (See NOTE, below.)
-.sp
-NOTE: A buffer size smaller than the distance (in bytes) in a UNIX file between the first byte and the last byte of the access request causes MPI I/O to iterate and perform multiple UNIX read() or write() calls. If the request includes multiple noncontiguous chunks of data, and the buffer size is greater than the size of those chunks, then the UNIX read() or write() (made at the MPI I/O level) will access data not requested by this process in order to reduce the total number of write() calls made. If this is not desirable behavior, you should reduce this buffer size to equal the size of the contiguous chunks within the aggregate request.
-.sp
-- mpiio_concurrency: (boolean) controls whether nonblocking
-I/O routines can bind an extra thread to an LWP.
-.sp
-- mpiio_coll_contiguous: (boolean) controls whether subsequent collective data accesses will request collectively contiguous regions of the file.
-.sp
-NON-SETTABLE HINTS:
-.sp
-- filename: Access this hint to get the name of the file.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_preallocate.3in b/ompi/mpi/man/man3/MPI_File_preallocate.3in
deleted file mode 100644
index e6acaf3472d..00000000000
--- a/ompi/mpi/man/man3/MPI_File_preallocate.3in
+++ /dev/null
@@ -1,90 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_preallocate 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_preallocate\fP \- Preallocates a specified amount of storage space  at the beginning of a file (collective).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_preallocate(MPI_File \fIfh\fP, MPI_Offset \fIsize\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_PREALLOCATE(\fIFH\fP, \fISIZE\fP, \fIIERROR\fP)
-	INTEGER	\fIFH, IERROR\fP
-	INTEGER(KIND=MPI_OFFSET_KIND)	\fISIZE\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_preallocate(\fIfh\fP, \fIsize\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	INTEGER(KIND=MPI_OFFSET_KIND), INTENT(IN) :: \fIsize\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-size
-Size to preallocate file, in bytes (integer).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_preallocate ensures that storage space is allocated for the first \fIsize\fP bytes of the file associated with \fIfh\fP. MPI_File_preallocate can be a very time-consuming operation.
-
-MPI_File_preallocate is collective; all processes in the group must pass identical values for \fIsize\fP. Regions of the file that have previously been written are unaffected. For newly allocated regions of the file, MPI_File_preallocate has the same effect as writing undefined data. If size is larger than the current file size, the file size increases to \fIsize\fP. If \fIsize\fP is less than or equal to the current file size, the file size is unchanged.
-
-The treatment of file pointers, pending nonblocking accesses, and file consistency is the same as with MPI_File_set_size. If MPI_MODE_SEQUENTIAL mode was specified when the file was opened, it is erroneous to call this routine.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fISIZE\fP argument only for Fortran 90.  FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_OFFSET_KIND \fISIZE\fP
-.fi
-.sp
-where MPI_OFFSET_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH NOTES
-.ft R
-When using the collective routine MPI_File_set_size on a UNIX file, if the size that is set is smaller than the current file size, the file is truncated at the position defined by size. If the size is set to be larger than the current file size, the file size becomes the set size. When the file size is increased this way with MPI_File_set_size, new regions are created in the file with displacements between the old file size and the larger, newly set file size.
-.sp
-Sun MPI I/O does not necessarily allocate file space for such new regions. You may reserve file space either by using MPI_File_preallocate or by performing a read or write to certain bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_read.3in b/ompi/mpi/man/man3/MPI_File_read.3in
deleted file mode 100644
index f8b355f3721..00000000000
--- a/ompi/mpi/man/man3/MPI_File_read.3in
+++ /dev/null
@@ -1,94 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_read 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_read\fP \- Reads a file starting at the location specified by the individual file pointer (blocking, noncollective).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_read(MPI_File \fIfh\fP, void \fI*buf\fP,
-	int \fIcount\fP, MPI_Datatype \fIdatatype\fP, MPI_Status \fI*status\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_READ(\fIFH\fP, \fIBUF\fP, \fICOUNT\fP,
-	\fIDATATYPE\fP, \fISTATUS\fP, \fIIERROR\fP)
-	<type>	\fIBUF(*)\fP
-	INTEGER	\fIFH, COUNT, DATATYPE, STATUS(MPI_STATUS_SIZE),IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_read(\fIfh\fP, \fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIstatus\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	TYPE(*), DIMENSION(..) :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Status) :: \fIstatus\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-fh
-File handle (handle).
-.TP 1i
-count
-Number of elements in buffer (integer).
-.TP 1i
-datatype
-Data type of each buffer element (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of buffer (integer).
-.TP 1i
-status
-Status object (status).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_read attempts to read from the file associated with
-.I fh
-(at the current individual file pointer position maintained by the system) a total number of
-.I count
-data items having
-.I datatype
-type into the user's buffer
-.I buf.
-The data is taken out of those parts of the
-file specified by the current view. MPI_File_read stores the
-number of data-type elements actually read in
-.I status.
-All other fields of
-.I status
-are undefined.
-.sp
-It is erroneous to call this function if MPI_MODE_SEQUENTIAL mode was specified when the file was opened.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_read_all.3in b/ompi/mpi/man/man3/MPI_File_read_all.3in
deleted file mode 100644
index 038060df403..00000000000
--- a/ompi/mpi/man/man3/MPI_File_read_all.3in
+++ /dev/null
@@ -1,95 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_read_all 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_read_all\fP \- Reads a file starting at the locations specified by individual file pointers (blocking, collective).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_read_all(MPI_File \fIfh\fP, void \fI*buf\fP,
-	int \fIcount\fP, MPI_Datatype \fIdatatype\fP, MPI_Status \fI*status\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_READ_ALL(\fIFH\fP, \fIBUF\fP, \fICOUNT\fP,
-	\fIDATATYPE\fP, \fISTATUS\fP, \fIIERROR\fP)
-	<type>	\fIBUF(*)\fP
-	INTEGER	\fIFH, COUNT, DATATYPE, STATUS(MPI_STATUS_SIZE),IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_read_all(\fIfh\fP, \fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIstatus\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	TYPE(*), DIMENSION(..) :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Status) :: \fIstatus\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-fh
-File handle (handle).
-.TP 1i
-count
-Number of elements in buffer (integer).
-.TP 1i
-datatype
-Data type of each buffer element (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of buffer (choice).
-.TP 1i
-status
-Status object (status).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_read_all is a collective routine that attempts to read from the file associated with
-.I fh
-(at the current individual file pointer position maintained by the system) a total number of
-.I count
-data items having
-.I datatype
-type into the user's buffer
-.I buf.
-The data is taken out of those parts of the
-file specified by the current view. MPI_File_read_all stores the
-number of data-type elements actually read in
-.I status.
-All other fields of
-.I status
-are undefined.
-.sp
-It is erroneous to call this function if MPI_MODE_SEQUENTIAL mode was specified when the file was opened.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-
diff --git a/ompi/mpi/man/man3/MPI_File_read_all_begin.3in b/ompi/mpi/man/man3/MPI_File_read_all_begin.3in
deleted file mode 100644
index 0fdf627e35c..00000000000
--- a/ompi/mpi/man/man3/MPI_File_read_all_begin.3in
+++ /dev/null
@@ -1,90 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_read_all_begin 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_read_all_begin\fP \- Reads a file starting at the locations specified by individual file pointers; beginning part of a split collective routine (nonblocking).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_read_all_begin(MPI_File \fIfh\fP, void \fI*buf\fP,
-	int \fIcount\fP, MPI_Datatype \fIdatatype\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_READ_ALL_BEGIN(\fIFH\fP, \fIBUF\fP, \fICOUNT\fP, \fIDATATYPE\fP, \fIIERROR\fP)
-	<type>	\fIBUF\fP(*)
-	INTEGER	\fIFH, COUNT, DATATYPE, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_read_all_begin(\fIfh\fP, \fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-count
-Number of elements in buffer (integer).
-.ft R
-.TP 1i
-datatype
-Data type of each buffer element (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of buffer (choice).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_read_all_begin is the beginning part of a split collective operation that attempts to read from the file associated with
-.I fh
-(at the current individual file pointer position maintained by the system) a total number of
-.I count
-data items having
-.I datatype
-type into the user's buffer
-.I buf.
-The data is taken out of those parts of the
-file specified by the current view.
-
-.SH NOTES
-.ft R
-All the nonblocking collective routines for data access are "split" into two routines, each with _begin or _end as a suffix. These split collective routines are subject to the semantic rules described in Section 9.4.5 of the MPI-2 standard.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_read_all_end.3in b/ompi/mpi/man/man3/MPI_File_read_all_end.3in
deleted file mode 100644
index 6957ab5c135..00000000000
--- a/ompi/mpi/man/man3/MPI_File_read_all_end.3in
+++ /dev/null
@@ -1,84 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_read_all_end 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_read_all_end\fP \- Reads a file starting at the locations specified by individual file pointers; ending part of a split collective routine (blocking).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_read_all_end(MPI_File \fIfh\fP, void \fI*buf\fP,
-	MPI_Status \fI*status\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_READ_ALL_END(\fIFH\fP, \fIBUF\fP, \fISTATUS\fP, \fIIERROR\fP)
-	<type>	\fIBUF(*)\fP
-	INTEGER	\fIFH, STATUS(MPI_STATUS_SIZE), IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_read_all_end(\fIfh\fP, \fIbuf\fP, \fIstatus\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIbuf\fP
-	TYPE(MPI_Status) :: \fIstatus\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of buffer (choice).
-.ft R
-.TP 1i
-status
-Status object (status).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_read_all_end is the ending part of a split collective operation that stores the number of elements actually read from the file associated with
-.I fh
-(at the current individual file pointer position maintained by the system)
-into the user's buffer
-.I buf
-in
-.I status.
-The data is taken out of those parts of the
-file specified by the current view. All other fields of
-.I status
-are undefined.
-
-.SH NOTES
-.ft R
-All the nonblocking collective routines for data access are "split" into two routines, each with _begin or _end as a suffix. These split collective routines are subject to the semantic rules described in Section 9.4.5 of the MPI-2 standard.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_read_at.3in b/ompi/mpi/man/man3/MPI_File_read_at.3in
deleted file mode 100644
index ac1eea5583b..00000000000
--- a/ompi/mpi/man/man3/MPI_File_read_at.3in
+++ /dev/null
@@ -1,121 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_read_at 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_read_at\fP \- Reads a file at an explicitly specified offset (blocking, noncollective).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_read_at(MPI_File \fIfh\fP, MPI_Offset \fIoffset\fP,
-	void \fI*buf\fP, int \fIcount\fP, MPI_Datatype \fIdatatype\fP,
-	MPI_Status \fI*status\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_READ_AT(\fIFH\fP, \fIOFFSET\fP, \fIBUF\fP, \fICOUNT\fP,
-	\fIDATATYPE\fP, \fISTATUS\fP, \fIIERROR\fP)
-	<type>	\fIBUF\fP(*)
-	INTEGER	\fIFH, COUNT, DATATYPE, STATUS(MPI_STATUS_SIZE), IERROR\fP
-	INTEGER(KIND=MPI_OFFSET_KIND)	\fIOFFSET\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_read_at(\fIfh\fP, \fIoffset\fP, \fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIstatus\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	INTEGER(KIND=MPI_OFFSET_KIND), INTENT(IN) :: \fIoffset\fP
-	TYPE(*), DIMENSION(..) :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Status) :: \fIstatus\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-fh
-File handle (handle).
-.TP 1i
-offset
-File offset (integer).
-.TP 1i
-count
-Number of elements in buffer (integer).
-.TP 1i
-datatype
-Data type of each buffer element (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of buffer (choice).
-.TP 1i
-status
-Status object (status).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-
-MPI_File_read_at attempts to read from the file associated with
-.I fh
-(at the
-.I offset
-position) a total number of
-.I count
-data items having
-.I datatype
-type into the user's buffer
-.I buf.
-The
-.I offset
-is in
-.I etype
-units relative to the current view. That is, holes are not counted
-when locating an offset. The data is taken out of those parts of the
-file specified by the current view. MPI_File_read_at stores the
-number of
-.I datatype
-elements actually read in
-.I status.
-All other fields of
-.I status
-are undefined. It is erroneous to call this function if MPI_MODE_SEQUENTIAL mode was specified when the file was opened.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fIOFFSET\fP argument only for Fortran 90. FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_OFFSET_KIND \fIOFFSET\fP
-.fi
-.sp
-where MPI_OFFSET_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_read_at_all.3in b/ompi/mpi/man/man3/MPI_File_read_at_all.3in
deleted file mode 100644
index 715c7c18d7e..00000000000
--- a/ompi/mpi/man/man3/MPI_File_read_at_all.3in
+++ /dev/null
@@ -1,118 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_read_at_all 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_read_at_all\fP \- Reads a file at explicitly specified offsets (blocking, collective).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_read_at_all(MPI_File \fIfh\fP, MPI_Offset \fIoffset\fP,
-	void \fI*buf\fP, int \fIcount\fP, MPI_Datatype \fIdatatype\fP,
-	MPI_Status \fI*status\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_READ_AT_ALL(\fIFH\fP, \fIOFFSET\fP, \fIBUF\fP, \fICOUNT\fP,
-	\fIDATATYPE\fP, \fISTATUS\fP, \fIIERROR\fP)
-	<type>	\fIBUF\fP(*)
-	INTEGER	\fIFH, COUNT, DATATYPE, STATUS(MPI_STATUS_SIZE), IERROR\fP
-	INTEGER(KIND=MPI_OFFSET_KIND)	\fIOFFSET\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_read_at_all(\fIfh\fP, \fIoffset\fP, \fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIstatus\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	INTEGER(KIND=MPI_OFFSET_KIND), INTENT(IN) :: \fIoffset\fP
-	TYPE(*), DIMENSION(..) :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Status) :: \fIstatus\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-fh
-File handle (handle).
-.TP 1i
-offset
-File offset (integer).
-.TP 1i
-count
-Number of elements in buffer (integer).
-.TP 1i
-datatype
-Data type of each buffer element (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of buffer (choice).
-.TP 1i
-status
-Status object (status).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_read_at_all is a collective routine that attempts to read from the file associated with
-.I fh
-(at the
-.I offset
-position) a total number of
-.I count
-data items having
-.I datatype
-type into the user's buffer
-.I buf.
-The
-.I offset
-is in etype units relative to the current view. That is, holes are not counted
-when locating an offset. The data is taken out of those parts of the
-file specified by the current view. MPI_File_read_at_all stores the
-number of
-.I datatype
-elements actually read in
-.I status.
-All other fields of
-.I status
-are undefined. It is erroneous to call this function if MPI_MODE_SEQUENTIAL mode was specified when the file was opened.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fIOFFSET\fP argument only for Fortran 90. FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_OFFSET_KIND \fIOFFSET\fP
-.fi
-.sp
-where MPI_OFFSET_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_read_at_all_begin.3in b/ompi/mpi/man/man3/MPI_File_read_at_all_begin.3in
deleted file mode 100644
index 7fbd263dde9..00000000000
--- a/ompi/mpi/man/man3/MPI_File_read_at_all_begin.3in
+++ /dev/null
@@ -1,114 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_read_at_all_begin 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_read_at_all_begin\fP \- Reads a file at explicitly specified offsets; beginning part of a split collective routine (nonblocking).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_read_at_all_begin(MPI_File \fIfh\fP, MPI_Offset
-	\fIoffset\fP, void \fI*buf\fP, int \fIcount\fP, MPI_Datatype
-	\fIdatatype\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_READ_AT_ALL_BEGIN(\fIFH\fP, \fIOFFSET\fP, \fIBUF\fP,
-	\fICOUNT\fP, \fIDATATYPE\fP, \fIIERROR\fP)
-	<type>	\fIBUF\fP(*)
-	INTEGER	\fIFH, COUNT, DATATYPE, IERROR\fP
-	INTEGER(KIND=MPI_OFFSET_KIND)	\fIOFFSET\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_read_at_all_begin(\fIfh\fP, \fIoffset\fP, \fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	INTEGER(KIND=MPI_OFFSET_KIND), INTENT(IN) :: \fIoffset\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-fh
-File handle (handle).
-.ft R
-.TP 1i
-offset
-File offset (integer).
-.ft R
-.TP 1i
-count
-Number of elements in buffer (integer).
-.ft R
-.TP 1i
-datatype
-Data type of each buffer element.
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of buffer (choice).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_read_at_all_begin is the beginning part of a split collective routine that attempts to read from the file associated with
-.I fh
-(at the
-.I offset
-position) a total number of
-.I count
-data items having
-.I datatype
-type into the user's buffer
-.I buf.
-The
-.I offset
-is in etype units relative to the current view. That is, holes are not counted
-when locating an offset. The data is taken out of those parts of the
-file specified by the current view.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fIOFFSET\fP argument only for Fortran 90. FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_OFFSET_KIND \fIOFFSET\fP
-.fi
-.sp
-where MPI_OFFSET_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH NOTES
-.ft R
-All the nonblocking collective routines for data access are "split" into two routines, each with _begin or _end as a suffix. These split collective routines are subject to the semantic rules described in Section 9.4.5 of the MPI-2 standard.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_read_at_all_end.3in b/ompi/mpi/man/man3/MPI_File_read_at_all_end.3in
deleted file mode 100644
index 3a6676766bd..00000000000
--- a/ompi/mpi/man/man3/MPI_File_read_at_all_end.3in
+++ /dev/null
@@ -1,80 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_read_at_all_end 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_read_at_all_end\fP \- Reads a file at explicitly specified offsets; ending part of a split collective routine (blocking).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_read_at_all_end(MPI_File \fIfh\fP, void \fI*buf\fP,
-	MPI_Status \fI*status\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_READ_AT_ALL_END(\fIFH\fP, \fIBUF\fP, \fISTATUS\fP, \fIIERROR\fP)
-	<type>	\fIBUF(*)\fP
-	INTEGER	\fIFH, STATUS(MPI_STATUS_SIZE), IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_read_at_all_end(\fIfh\fP, \fIbuf\fP, \fIstatus\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIbuf\fP
-	TYPE(MPI_Status) :: \fIstatus\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of buffer (choice).
-.ft R
-.TP 1i
-status
-Status object (status).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_read_at_all_end is a split collective routine that stores the number of elements actually read from the file associated with
-.I fh
-in
-.I status.
-MPI_File_read_at_all_end blocks until the operation initiated by MPI_File_read_at_all_begin completes. The data is taken out of those parts of the file specified by the current view. All other fields of
-.I status
-are undefined.
-
-.SH NOTES
-.ft R
-All the nonblocking collective routines for data access are "split" into two routines, each with _begin or _end as a suffix. These split collective routines are subject to the semantic rules described in Section 9.4.5 of the MPI-2 standard.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_read_ordered.3in b/ompi/mpi/man/man3/MPI_File_read_ordered.3in
deleted file mode 100644
index 1bb16a77083..00000000000
--- a/ompi/mpi/man/man3/MPI_File_read_ordered.3in
+++ /dev/null
@@ -1,99 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015      Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_read_ordered 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_read_ordered\fP \- Reads a file at a location specified by a shared file pointer (blocking, collective).
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_read_ordered(MPI_File \fIfh\fP, void \fI*buf\fP,
-	int \fIcount\fP, MPI_Datatype \fIdatatype\fP,
-	MPI_Status \fI*status\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_READ_ORDERED(\fIFH\fP, \fIBUF\fP, \fICOUNT\fP, \fIDATATYPE\fP,
-	\fISTATUS\fP, \fIIERROR\fP)
-	<type>	\fIBUF\fP(*)
-	INTEGER	\fIFH, COUNT, DATATYPE, STATUS(MPI_STATUS_SIZE), IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_read_ordered(\fIfh\fP, \fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIstatus\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	TYPE(*), DIMENSION(..) :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Status) :: \fIstatus\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-fh
-File handle (handle).
-.TP 1i
-count
-Number of elements in buffer (integer).
-.TP 1i
-datatype
-Data type of each buffer element (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of buffer (choice).
-.TP 1i
-status
-Status object (Status).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-
-MPI_File_read_ordered is a collective routine. This routine must be
-called by all processes in the communicator group associated with the
-file handle
-.I fh.
-Each process may pass different argument values for the
-.I datatype
-and
-.I count
-arguments. Each process attempts to read, from the file associated with
-.I fh,
-a total number of
-.I count
-data items having
-.I datatype
-type into the user's buffer
-.I buf.
-For each process, the location in the file at which data is read is the position at which the shared file pointer would be after all processes whose ranks within the group are less than that of this process had read their data. MPI_File_read_ordered returns the actual number of
-.I datatype
-elements read in
-.I status.
-The shared file pointer is updated by the amounts of data requested by all processes of the group.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_read_ordered_begin.3in b/ompi/mpi/man/man3/MPI_File_read_ordered_begin.3in
deleted file mode 100644
index 276cdb9fdf5..00000000000
--- a/ompi/mpi/man/man3/MPI_File_read_ordered_begin.3in
+++ /dev/null
@@ -1,97 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_read_ordered_begin 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_read_ordered_begin\fP \- Reads a file at a location specified by a shared file pointer; beginning part of a split collective routine (nonblocking).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_read_ordered_begin(MPI_File \fIfh\fP, void \fI*buf\fP,
-	int \fIcount\fP, MPI_Datatype \fIdatatype\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_READ_ORDERED_BEGIN(\fIFH\fP, \fIBUF\fP, \fICOUNT\fP, \fIDATATYPE\fP, \fIIERROR\fP)
-	<type>	\fIBUF(*)\fP
-	INTEGER	\fIFH, COUNT, DATATYPE, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_read_ordered_begin(\fIfh\fP, \fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-count
-Number of elements in buffer (integer).
-.ft R
-.TP 1i
-datatype
-Data type of each buffer element (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of buffer (choice).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_read_ordered_begin is the beginning part of a split collective, nonblocking routine that must be
-called by all processes in the communicator group associated with the
-file handle
-.I fh.
-Each process may pass different argument values for the
-.I datatype
-and
-.I count
-arguments. Each process attempts to read, from the file associated with
-.I fh,
-a total number of
-.I count
-data items having
-.I datatype
-type into the user's buffer
-.I buf.
-For each process, the location in the file at which data is read is the position at which the shared file pointer would be after all processes whose ranks within the group are less than that of this process had read their data.
-
-.SH NOTES
-.ft R
-All the nonblocking collective routines for data access are "split" into two routines, each with _begin or _end as a suffix. These split collective routines are subject to the semantic rules described in Section 9.4.5 of the MPI-2 standard.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_read_ordered_end.3in b/ompi/mpi/man/man3/MPI_File_read_ordered_end.3in
deleted file mode 100644
index fcf80a3f562..00000000000
--- a/ompi/mpi/man/man3/MPI_File_read_ordered_end.3in
+++ /dev/null
@@ -1,81 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_read_ordered_end 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_read_ordered_end\fP \- Reads a file at a location specified by a shared file pointer; ending part of a split collective routine (blocking).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_read_ordered_end(MPI_File \fIfh\fP, void \fI*buf\fP,
-	MPI_Status \fI*status\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_READ_ORDERED_END(\fIFH\fP, \fIBUF\fP, \fISTATUS\fP, \fIIERROR\fP)
-	<type>	\fIBUF(*)\fP
-	INTEGER	\fIFH, STATUS(MPI_STATUS_SIZE), IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_read_ordered_end(\fIfh\fP, \fIbuf\fP, \fIstatus\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIbuf\fP
-	TYPE(MPI_Status) :: \fIstatus\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of buffer (choice).
-.ft R
-.TP 1i
-status
-Status object (status).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_read_ordered_end is the ending part of a split collective routine that must be called by all processes in the communicator group associated with the
-file handle
-.I fh.
-MPI_File_rad_ordered_end blocks until the operation initiated by MPI_File_read_ordered_begin completes. It attempts to read the file associated with
-.I fh
-into the user's buffer
-.I buf.
-The shared file pointer is updated by the amounts of data requested by all processes of the group. For each process, the location in the file at which data is read is the position at which the shared file pointer would be after all processes whose ranks within the group are less than that of this process had read their data.
-
-.SH NOTES
-.ft R
-All the nonblocking collective routines for data access are "split" into two routines, each with _begin or _end as a suffix. These split collective routines are subject to the semantic rules described in Section 9.4.5 of the MPI-2 standard.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_read_shared.3in b/ompi/mpi/man/man3/MPI_File_read_shared.3in
deleted file mode 100644
index e66eb158197..00000000000
--- a/ompi/mpi/man/man3/MPI_File_read_shared.3in
+++ /dev/null
@@ -1,83 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_read_shared 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_read_shared\fP \- Reads a file using the shared file pointer (blocking, noncollective).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_read_shared(MPI_File \fIfh\fP, void \fI*buf\fP, int \fIcount\fP,
-	MPI_Datatype \fIdatatype\fP, MPI_Status \fI*status\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_READ_SHARED(\fIFH\fP, \fIBUF\fP, \fICOUNT\fP, \fIDATATYPE\fP, \fISTATUS\fP,
-	\fIIERROR\fP)
-	<type>	\fIBUF(*)\fP
-	INTEGER	\fIFH, COUNT, DATATYPE,STATUS(MPI_STATUS_SIZE), IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_read_shared(\fIfh\fP, \fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIstatus\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	TYPE(*), DIMENSION(..) :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Status) :: \fIstatus\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-count
-Number of elements in buffer (integer)
-.ft R
-.TP 1i
-datatype
-Data type of each buffer element (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of buffer (choice).
-.ft R
-.TP 1i
-status
-Status object (status).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_read_shared is a blocking routine that uses the shared file pointer to read files. The order of serialization is not deterministic for this noncollective routine.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_seek.3in b/ompi/mpi/man/man3/MPI_File_seek.3in
deleted file mode 100644
index bead5615a0e..00000000000
--- a/ompi/mpi/man/man3/MPI_File_seek.3in
+++ /dev/null
@@ -1,105 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_seek 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_seek\fP \- Updates individual file pointers (noncollective).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_seek(MPI_File \fIfh\fP, MPI_Offset \fIoffset\fP,
-	int \fIwhence\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_SEEK(\fIFH\fP, \fIOFFSET\fP, \fIWHENCE\fP, \fIIERROR\fP)
-	INTEGER	\fIFH, WHENCE, IERROR\fP
-	INTEGER(KIND=MPI_OFFSET_KIND)	\fIOFFSET\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_seek(\fIfh\fP, \fIoffset\fP, \fIwhence\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	INTEGER(KIND=MPI_OFFSET_KIND), INTENT(IN) :: \fIoffset\fP
-	INTEGER, INTENT(IN) :: \fIwhence\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-fh
-File handle (handle).
-.TP 1i
-offset
-File offset (integer).
-.TP 1i
-whence
-Update mode (integer).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_seek updates the individual file pointer according to
-.I whence,
-which could have the following possible values:
-.TP
-  o
-MPI_SEEK_SET - The pointer is set to
-.I offset.
-.TP
-  o
-MPI_SEEK_CUR - The pointer is set to the current pointer position plus
-.I offset.
-.TP
-  o
-MPI_SEEK_END - The pointer is set to the end of the file plus
-.I offset.
-.sp
-.RE
-The
-.I offset
-can be negative, which allows seeking backwards. It is erroneous to
-seek to a negative position in the file. The end of the file is
-defined to be the location of the next elementary data item
-immediately after the last accessed data item, even if that location
-is a hole.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fIOFFSET\fP argument only for Fortran 90.  FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_OFFSET_KIND \fIOFFSET\fP
-.fi
-.sp
-where MPI_OFFSET_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_seek_shared.3in b/ompi/mpi/man/man3/MPI_File_seek_shared.3in
deleted file mode 100644
index 4403529b778..00000000000
--- a/ompi/mpi/man/man3/MPI_File_seek_shared.3in
+++ /dev/null
@@ -1,115 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_seek_shared 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_seek_shared\fP \- Updates the global shared file pointer (collective).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_seek_shared(MPI_File \fIfh\fP, MPI_Offset \fIoffset\fP,
-	int \fIwhence\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_SEEK_SHARED(\fIFH\fP, \fIOFFSET\fP, \fIWHENCE\fP, \fIIERROR\fP)
-	INTEGER	\fIFH, WHENCE, IERROR\fP
-	INTEGER(KIND=MPI_OFFSET_KIND)	\fIOFFSET\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_seek_shared(\fIfh\fP, \fIoffset\fP, \fIwhence\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	INTEGER(KIND=MPI_OFFSET_KIND), INTENT(IN) :: \fIoffset\fP
-	INTEGER, INTENT(IN) :: \fIwhence\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-fh
-File handle (handle).
-.TP 1i
-offset
-File offset (integer).
-.TP 1i
-whence
-Update mode (integer).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_seek_shared updates the shared file pointer according to
-.I whence,
-which could have the following possible values:
-.TP
-  o
-MPI_SEEK_SET - The pointer is set to
-.I offset.
-.TP
-  o
-MPI_SEEK_CUR - The pointer is set to the current pointer position plus
-.I offset.
-.TP
-  o
-MPI_SEEK_END - The pointer is set to the end of the file plus
-.I offset.
-.sp
-.RE
-MPI_File_seek_shared is collective; all the processes in the communicator
-group associated with the file handle
-.I fh
-must call MPI_File_seek_shared with the same
-.I offset
-and
-.I whence.
-All processes in the communicator group are synchronized before the shared file pointer is updated.
-
-.sp
-The
-.I offset
-can be negative, which allows seeking backwards. It is erroneous to
-seek to a negative position in the view. The end of the view is
-defined to be the position of the next elementary data item, relative
-to the current view, following the last whole elementary data item
-accessible.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fIOFFSET\fP argument only for Fortran 90.  FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_OFFSET_KIND \fIOFFSET\fP
-.fi
-.sp
-where MPI_OFFSET_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_set_atomicity.3in b/ompi/mpi/man/man3/MPI_File_set_atomicity.3in
deleted file mode 100644
index 93d1f4caf7d..00000000000
--- a/ompi/mpi/man/man3/MPI_File_set_atomicity.3in
+++ /dev/null
@@ -1,79 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_set_atomicity 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_set_atomicity\fP \- Sets consistency semantics for data-access operations (collective).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_set_atomicity(MPI_File \fIfh\fP, int \fIflag\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_SET_ATOMICITY(\fIFH\fP, \fIFLAG\fP, \fIIERROR\fP)
-	INTEGER	\fIFH, FLAG, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_set_atomicity(\fIfh\fP, \fIflag\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	LOGICAL, INTENT(IN) :: \fIflag\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-fh
-File handle (handle).
-.TP 1i
-flag
-\fBtrue\fP to enable atomic mode, \fBfalse\fP to enable nonatomic mode (boolean).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-The consistency semantics for data-access operations using the set of
-file handles created by one collective MPI_File_open is set by collectively
-calling MPI_File_set_atomicity. All processes in the group must pass identical values for
-.I fh
-and
-.I flag.
-If
-.I flag
-is
-.I true,
-atomic mode is set; if
-.I flag
-is
-.I false,
-nonatomic mode is set.
-.sp
-The default value on a call to MPI_File_open in Open MPI is \fItrue\fP for jobs running on more than one node, \fIfalse\fP for jobs running on a single SMP. For more information, see the MPI-2 standard.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_set_errhandler.3in b/ompi/mpi/man/man3/MPI_File_set_errhandler.3in
deleted file mode 100644
index e5a15d5f9b6..00000000000
--- a/ompi/mpi/man/man3/MPI_File_set_errhandler.3in
+++ /dev/null
@@ -1,65 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_set_errhandler 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_set_errhandler \fP \- Sets the error handler for a file.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_set_errhandler(MPI_File \fIfile\fP, MPI_Errhandler
-	\fIerrhandler\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_SET_ERRHANDLER(\fIFILE, ERRHANDLER, IERROR\fP)
-	INTEGER	\fIFILE, ERRHANDLER, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_set_errhandler(\fIfile\fP, \fIerrhandler\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfile\fP
-	TYPE(MPI_Errhandler), INTENT(IN) :: \fIerrhandler\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-file
-File (handle).
-
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-errhandler
-New error handler for file (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Attaches a new error handler to a file. The error handler must be either a predefined error handler or an error handler created by a call to MPI_File_create_errhandler.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-
diff --git a/ompi/mpi/man/man3/MPI_File_set_info.3in b/ompi/mpi/man/man3/MPI_File_set_info.3in
deleted file mode 100644
index ac63a375a18..00000000000
--- a/ompi/mpi/man/man3/MPI_File_set_info.3in
+++ /dev/null
@@ -1,105 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_set_info 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_set_info\fP \- Sets new values for hints (collective).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_set_info(MPI_File \fIfh\fP, MPI_Info \fIinfo\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_SET_INFO(\fIFH\fP, \fIINFO\fP, \fIIERROR\fP)
-	INTEGER	\fIFH, INFO, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_set_info(\fIfh\fP, \fIinfo\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-info
-Info object (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_set_info is a collective routine that sets new values for the hints of the file associated with \fIfh\fP. These hints are set for each file, using the MPI_File_open, MPI_File_delete, MPI_File_set_view, and MPI_File_set_info routines. The opaque \fIinfo\fP object, which allows you to provide hints for optimization of your code, may be different on each process, but some \fIinfo\fP entries are required to be the same on all processes: In these cases, they must appear with the same value in each process's info object. See the HINTS section for a list of hints that can be set.
-
-.SH HINTS
-.ft R
-The following hints can be used as values for the \fIinfo\fP argument.
-.sp
-SETTABLE HINTS:
-.sp
-- shared_file_timeout: Amount of time (in seconds) to wait for access to the
-shared file pointer before exiting with MPI_ERR_TIMEDOUT.
-.sp
-- rwlock_timeout: Amount of time (in seconds) to wait for obtaining a read or
-write lock on a contiguous chunk of a UNIX file before exiting with MPI_ERR_TIMEDOUT.
-.sp
-- noncoll_read_bufsize:  Maximum size of the buffer used by
-MPI I/O to satisfy read requests in
-the noncollective data-access routines. (See NOTE, below.)
-.sp
-- noncoll_write_bufsize: Maximum size of the buffer used by
-MPI I/O to satisfy write requests in
-the noncollective data-access routines. (See NOTE, below.)
-.sp
-- coll_read_bufsize:  Maximum size of the buffer used by MPI
-I/O to satisfy read requests in the
-collective data-access routines. (See NOTE, below.)
-.sp
-- coll_write_bufsize:  Maximum size of the buffer used by MPI
-I/O to satisfy write requests in the
-collective data-access routines. (See NOTE, below.)
-.sp
-NOTE: A buffer size smaller than the distance (in bytes) in a UNIX file between the first byte and the last byte of the access request causes MPI I/O to iterate and perform multiple UNIX read() or write() calls. If the request includes multiple noncontiguous chunks of data, and the buffer size is greater than the size of those chunks, then the UNIX read() or write() (made at the MPI I/O level) will access data not requested by this process in order to reduce the total number of write() calls made. If this is not desirable behavior, you should reduce this buffer size to equal the size of the contiguous chunks within the aggregate request.
-.sp
-- mpiio_concurrency: (boolean) controls whether nonblocking
-I/O routines can bind an extra thread to an LWP.
-.sp
-- mpiio_coll_contiguous: (boolean) controls whether subsequent collective data accesses will request collectively contiguous regions of the file.
-.sp
-NON-SETTABLE HINTS:
-.sp
-- filename: Access this hint to get the name of the file.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_set_size.3in b/ompi/mpi/man/man3/MPI_File_set_size.3in
deleted file mode 100644
index 6d5639b1f14..00000000000
--- a/ompi/mpi/man/man3/MPI_File_set_size.3in
+++ /dev/null
@@ -1,90 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_set_size 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_set_size\fP \- Resizes a file (collective).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_set_size(MPI_File \fIfh\fP, MPI_Offset \fIsize\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_SET_SIZE(\fIFH\fP, \fISIZE\fP, \fIIERROR\fP)
-	INTEGER	\fIFH, IERROR\fP
-	INTEGER(KIND=MPI_OFFSET_KIND)	\fISIZE\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_set_size(\fIfh\fP, \fIsize\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	INTEGER(KIND=MPI_OFFSET_KIND), INTENT(IN) :: \fIsize\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-fh
-File handle (handle).
-.TP 1i
-size
-Size to truncate or expand file (integer).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_set_size resizes the file associated with the file handle
-.I fh,
-truncating UNIX files as necessary. MPI_File_set_size is collective; all
-processes in the group must pass identical values for size.
-.sp
-When using MPI_File_set_size on a UNIX file, if \fIsize\fP is larger than the current file size, the file size becomes \fIsize\fP. If \fIsize\fP is smaller than the current file size, the file is truncated at the position defined by \fIsize\fP (from the beginning of the file and measured in bytes). Regions of the file which have been previously written are unaffected.
-.sp
-MPI_File_set_size does not affect the individual file pointers or the
-shared file pointer.
-.sp
-Note that the actual amount of storage space cannot be allocated by MPI_File_set_size. Use MPI_File_preallocate to accomplish this.
-.sp
-It is erroneous to call this function if MPI_MODE_SEQUENTIAL mode was specified when the file was opened.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fISIZE\fP argument only for Fortran 90.  FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_OFFSET_KIND \fISIZE\fP
-.fi
-.sp
-where MPI_OFFSET_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-
diff --git a/ompi/mpi/man/man3/MPI_File_set_view.3in b/ompi/mpi/man/man3/MPI_File_set_view.3in
deleted file mode 100644
index 5b6a099e39c..00000000000
--- a/ompi/mpi/man/man3/MPI_File_set_view.3in
+++ /dev/null
@@ -1,177 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_set_view 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_set_view\fP \- Changes process's view of data in file (collective).
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_set_view(MPI_File \fIfh\fP, MPI_Offset \fIdisp\fP,
-	MPI_Datatype \fIetype\fP, MPI_Datatype \fIfiletype\fP,
-	const char \fI*datarep\fP, MPI_Info \fIinfo\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_SET_VIEW(\fIFH\fP, \fIDISP\fP, \fIETYPE\fP,
-	\fIFILETYPE\fP, \fIDATAREP\fP, \fIINFO\fP, \fIIERROR\fP)
-	INTEGER	\fIFH, ETYPE, FILETYPE, INFO, IERROR\fP
-	CHARACTER*(*)	\fIDATAREP\fP
-	INTEGER(KIND=MPI_OFFSET_KIND)	\fIDISP\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_set_view(\fIfh\fP, \fIdisp\fP, \fIetype\fP, \fIfiletype\fP, \fIdatarep\fP, \fIinfo\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	INTEGER(KIND=MPI_OFFSET_KIND), INTENT(IN) :: \fIdisp\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIetype\fP, \fIfiletype\fP
-	CHARACTER(LEN=*), INTENT(IN) :: \fIdatarep\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-disp
-Displacement (integer).
-.TP 1i
-etype
-Elementary data type (handle).
-.TP 1i
-filetype
-File type (handle). See Restrictions, below.
-.TP 1i
-datarep
-Data representation (string).
-.TP 1i
-info
-Info object (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-The MPI_File_set_view routine changes the process's view of the data
-in the file -- the beginning of the data accessible in the file through
-that view is set to
-.I disp;
-the type of data is set to
-.I etype;
-and the distribution of data to processes is set to
-.I filetype.
-In addition, MPI_File_set_view resets the independent file pointers and
-the shared file pointer to zero. MPI_File_set_view is collective across the
-.IR fh ;
-all processes in the group must pass identical values for
-.IR datarep
-and provide an
-.I etype
-with an identical extent.  The values for
-.IR disp ,
-.IR filetype ,
-and
-.I info
-may vary. It is erroneous to use the shared file pointer data-access
-routines unless identical values for
-.I disp
-and
-.I filetype
-are also given. The data types passed in
-.I etype
-and
-.I filetype
-must be committed.
-.sp
-The
-.I disp
-displacement argument specifies the position (absolute offset in
-bytes from the beginning of the file) where the view begins.
-.sp
-The MPI_File_set_view interface allows the user to pass a data-representation string to MPI I/O via the \fIdatarep\fP argument. To obtain the default value pass the value "native". The user can also pass information via the \fIinfo\fP argument. See the HINTS section for a list of hints that can be set. For more information, see the MPI-2 standard.
-
-.SH HINTS
-.ft R
-The following hints can be used as values for the \fIinfo\fP argument.
-.sp
-SETTABLE HINTS:
-.sp
-- MPI_INFO_NULL
-.sp
-- shared_file_timeout: Amount of time (in seconds) to wait for access to the
-shared file pointer before exiting with MPI_ERR_TIMEDOUT.
-.sp
-- rwlock_timeout: Amount of time (in seconds) to wait for obtaining a read or
-write lock on a contiguous chunk of a UNIX file before exiting with MPI_ERR_TIMEDOUT.
-.sp
-- noncoll_read_bufsize:  Maximum size of the buffer used by
-MPI I/O to satisfy read requests in
-the noncollective data-access routines. (See NOTE, below.)
-.sp
-- noncoll_write_bufsize: Maximum size of the buffer used by
-MPI I/O to satisfy write requests in
-the noncollective data-access routines. (See NOTE, below.)
-.sp
-- coll_read_bufsize:  Maximum size of the buffer used by MPI
-I/O to satisfy read requests in the
-collective data-access routines. (See NOTE, below.)
-.sp
-- coll_write_bufsize:  Maximum size of the buffer used by MPI
-I/O to satisfy write requests in the
-collective data-access routines. (See NOTE, below.)
-.sp
-NOTE: A buffer size smaller than the distance (in bytes) in a UNIX file between the first byte and the last byte of the access request causes MPI I/O to iterate and perform multiple UNIX read() or write() calls. If the request includes multiple noncontiguous chunks of data, and the buffer size is greater than the size of those chunks, then the UNIX read() or write() (made at the MPI I/O level) will access data not requested by this process in order to reduce the total number of write() calls made. If this is not desirable behavior, you should reduce this buffer size to equal the size of the contiguous chunks within the aggregate request.
-.sp
-- mpiio_concurrency: (boolean) controls whether nonblocking
-I/O routines can bind an extra thread to an LWP.
-.sp
-- mpiio_coll_contiguous: (boolean) controls whether subsequent collective data accesses will request collectively contiguous regions of the file.
-.sp
-NON-SETTABLE HINTS:
-.sp
-- filename: Access this hint to get the name of the file.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fIDISP\fP argument only for Fortran 90.  FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_OFFSET_KIND \fIDISP\fP
-.fi
-.sp
-where MPI_OFFSET_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-
diff --git a/ompi/mpi/man/man3/MPI_File_sync.3in b/ompi/mpi/man/man3/MPI_File_sync.3in
deleted file mode 100644
index 2400083bda9..00000000000
--- a/ompi/mpi/man/man3/MPI_File_sync.3in
+++ /dev/null
@@ -1,70 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_sync 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_sync\fP \- Makes semantics consistent for data-access operations (collective).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_sync(MPI_File \fIfh\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_SYNC(\fIFH\fP, \fIIERROR\fP)
-	INTEGER	\fIFH, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_sync(\fIfh\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Calling MPI_File_sync with
-.I fh
-causes all previous writes to
-.I fh
-by the calling process to be written to permanent storage. If other processes have made updates to permanent storage, then all such updates become visible to subsequent reads of
-.I fh
-by the calling process.
-.sp
-MPI_File_sync is a collective operation. The user is responsible for ensuring that all nonblocking requests on
-.I fh
-have been completed before calling MPI_File_sync. Otherwise, the call to MPI_File_sync is erroneous.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-
diff --git a/ompi/mpi/man/man3/MPI_File_write.3in b/ompi/mpi/man/man3/MPI_File_write.3in
deleted file mode 100644
index 91c4ca0b799..00000000000
--- a/ompi/mpi/man/man3/MPI_File_write.3in
+++ /dev/null
@@ -1,101 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_write 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_write\fP \- Writes a file starting at the location specified by the individual file pointer (blocking, noncollective).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_write(MPI_File \fIfh\fP, const void \fI*buf\fP,
-	int \fIcount\fP, MPI_Datatype \fIdatatype\fP,
-	MPI_Status \fI*status\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_WRITE(\fIFH\fP, \fIBUF\fP, \fICOUNT\fP,
-	\fIDATATYPE\fP, \fISTATUS\fP, \fIIERROR\fP)
-	<type>	\fIBUF\fP(*)
-	INTEGER	\fIFH, COUNT, DATATYPE, STATUS(MPI_STATUS_SIZE), IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_write(\fIfh\fP, \fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIstatus\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Status) :: \fIstatus\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of buffer (choice).
-.TP 1i
-count
-Number of elements in buffer (integer).
-.TP 1i
-datatype
-Data type of each buffer element (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-status
-Status object (status).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_write attempts to write into the file associated with
-.I fh
-(at the current individual file pointer position maintained by the system) a total number of
-.I count
-data items having
-.I datatype
-type from the user's buffer
-.I buf.
-The data is written into those parts of the
-file specified by the current view. MPI_File_write stores the
-number of
-.I datatype
-elements actually written in
-.I status.
-All other fields of
-.I status
-are undefined.
-.sp
-It is erroneous to call this function if MPI_MODE_SEQUENTIAL mode was specified when the file was opened.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_write_all.3in b/ompi/mpi/man/man3/MPI_File_write_all.3in
deleted file mode 100644
index ec8ce2e416e..00000000000
--- a/ompi/mpi/man/man3/MPI_File_write_all.3in
+++ /dev/null
@@ -1,97 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_write_all 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_write_all\fP \- Writes a file starting at the locations specified by individual file pointers (blocking, collective).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_write_all(MPI_File \fIfh\fP, const void \fI*buf\fP,
-	int \fIcount\fP, MPI_Datatype \fIdatatype\fP, MPI_Status \fI*status\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_WRITE_ALL(\fIFH\fP, \fIBUF\fP, \fICOUNT\fP,
-	\fIDATATYPE\fP, \fISTATUS\fP, \fIIERROR\fP)
-	<type>	\fIBUF(*)\fP
-	INTEGER	\fIFH, COUNT, DATATYPE, STATUS(MPI_STATUS_SIZE), IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_write_all(\fIfh\fP, \fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIstatus\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Status) :: \fIstatus\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-fh
-File handle (handle).
-.TP 1i
-buf
-Initial address of buffer (choice).
-.TP 1i
-count
-Number of elements in buffer (integer).
-.TP 1i
-datatype
-Data type of each buffer element (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-status
-Status object (status).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_write_all is a collective routine that attempts to write into the file associated with
-.I fh
-(at the current individual file pointer position maintained by the system) a total number of
-.I count
-data items having
-.I datatype
-type from the user's buffer
-.I buf.
-The data is written into those parts of the
-file specified by the current view. MPI_File_write_all stores the
-number of
-.I datatype
-elements actually written in
-.I status.
-All other fields of
-.I status
-are undefined.
-.sp
-It is erroneous to call this function if MPI_MODE_SEQUENTIAL mode was specified when the file was opened.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_write_all_begin.3in b/ompi/mpi/man/man3/MPI_File_write_all_begin.3in
deleted file mode 100644
index 349020c5aac..00000000000
--- a/ompi/mpi/man/man3/MPI_File_write_all_begin.3in
+++ /dev/null
@@ -1,95 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_write_all_begin 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_write_all_begin\fP \- Writes a file starting at the locations specified by individual file pointers; beginning part of a split collective routine (nonblocking).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_write_all_begin(MPI_File \fIfh\fP, const void \fI*buf\fP,
-	int \fIcount\fP, MPI_Datatype \fIdatatype\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_WRITE_ALL_BEGIN(\fIFH\fP, \fIBUF\fP, \fICOUNT\fP, \fIDATATYPE\fP, \fIIERROR\fP)
-	<type>	\fIBUF(*)\fP
-	INTEGER	\fIFH, COUNT, DATATYPE, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_write_all_begin(\fIfh\fP, \fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of buffer (choice).
-.ft R
-.TP 1i
-count
-Number of elements in buffer (integer).
-.ft R
-.TP 1i
-datatype
-Data type of each buffer element (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_write_all_begin is the beginning part of a split collective, nonblocking routine that attempts to write into the file associated with
-.I fh
-(at the current individual file pointer position maintained by the system) a total number of
-.I count
-data items having
-.I datatype
-type from the user's buffer
-.I buf.
-The data is written into those parts of the
-file specified by the current view.
-
-.SH NOTES
-.ft R
-All the nonblocking collective routines for data access are "split" into two routines, each with _begin or _end as a suffix. These split collective routines are subject to the semantic rules described in Section 9.4.5 of the MPI-2 standard.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-
-
-
diff --git a/ompi/mpi/man/man3/MPI_File_write_all_end.3in b/ompi/mpi/man/man3/MPI_File_write_all_end.3in
deleted file mode 100644
index c53fb09689c..00000000000
--- a/ompi/mpi/man/man3/MPI_File_write_all_end.3in
+++ /dev/null
@@ -1,85 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_write_all_end 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_write_all_end\fP \- Writes a file starting at the locations specified by individual file pointers; ending part of a split collective routine (blocking).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_write_all_end(MPI_File \fIfh\fP, const void \fI*buf\fP, MPI_Status \fI*status\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_WRITE_ALL_END(\fIFH\fP, \fIBUF\fP, \fISTATUS\fP, \fIIERROR\fP)
-	<type>	\fIBUF(*)\fP
-	INTEGER	\fIFH, STATUS, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_write_all_end(\fIfh\fP, \fIbuf\fP, \fIstatus\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIbuf\fP
-	TYPE(MPI_Status) :: \fIstatus\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-buf
-Initial address of buffer (choice).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-status
-Status object (status).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_write_all_end is the ending part of a split collective routine that stores the
-number of elements actually written into the file associated with
-.I fh
-from the user's buffer
-.I buf
-in
-.I status.
-MPI_File_write_all_end blocks until the operation initiated by MPI_File_write_all_begin completes. The data is written into those parts of the
-file specified by the current view. All other fields of
-.I status
-are undefined.
-
-.SH NOTES
-.ft R
-All the nonblocking collective routines for data access are "split" into two routines, each with _begin or _end as a suffix. These split collective routines are subject to the semantic rules described in Section 9.4.5 of the MPI-2 standard.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
diff --git a/ompi/mpi/man/man3/MPI_File_write_at.3in b/ompi/mpi/man/man3/MPI_File_write_at.3in
deleted file mode 100644
index 12d37e98956..00000000000
--- a/ompi/mpi/man/man3/MPI_File_write_at.3in
+++ /dev/null
@@ -1,131 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_write_at 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_write_at\fP \- Writes a file at an explicitly specified offset (blocking, noncollective).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_write_at(MPI_File \fIfh\fP, MPI_Offset \fIoffset\fP, const void \fI*buf\fP,
-	int \fIcount\fP, MPI_Datatype \fIdatatype\fP, MPI_Status \fI*status\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_WRITE_AT(\fIFH\fP, \fIOFFSET\fP, \fIBUF\fP, \fICOUNT\fP,
-	\fIDATATYPE\fP, \fISTATUS\fP, \fIIERROR\fP)
-	<type>	\fIBUF\fP(*)
-	INTEGER	\fIFH, COUNT, DATATYPE, STATUS(MPI_STATUS_SIZE), IERROR\fP
-	INTEGER(KIND=MPI_OFFSET_KIND)	\fIOFFSET\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_write_at(\fIfh\fP, \fIoffset\fP, \fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIstatus\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	INTEGER(KIND=MPI_OFFSET_KIND), INTENT(IN) :: \fIoffset\fP
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Status) :: \fIstatus\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-fh
-File handle (handle).
-.TP 1i
-offset
-File offset (integer).
-.TP 1i
-buf
-Initial address of buffer (choice).
-.TP 1i
-count
-Number of elements in buffer (integer).
-.TP 1i
-datatype
-Data type of each buffer element (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-status
-Status object (status).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_write_at attempts to write into the file associated with
-.I fh
-(at the
-.I offset
-position) a total number of
-.I count
-data items having
-.I datatype
-type from the user's buffer
-.I buf.
-The offset is in
-.I etype
-units relative to the current view. That is, holes are not counted
-when locating an offset. The data is written into those parts of the
-file specified by the current view. MPI_File_write_at stores the
-number of
-.I datatype
-elements actually written in
-.I status.
-All other fields of
-.I status
-are undefined.
-.sp
-It is erroneous to call this function if MPI_MODE_SEQUENTIAL mode was specified when the file was opened.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fIOFFSET\fP argument only for Fortran 90.  FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_OFFSET_KIND \fIOFFSET\fP
-.fi
-.sp
-where MPI_OFFSET_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-MPI_File_iwrite_at
-.br
-MPI_File_write_at_all
-.br
-MPI_File_write_at_all_begin
-.br
-MPI_File_write_at_all_end
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_File_write_at_all.3in b/ompi/mpi/man/man3/MPI_File_write_at_all.3in
deleted file mode 100644
index 722b520a32c..00000000000
--- a/ompi/mpi/man/man3/MPI_File_write_at_all.3in
+++ /dev/null
@@ -1,118 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_write_at_all 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_write_at_all\fP \- Writes a file at explicitly specified offsets (blocking, collective).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_write_at_all(MPI_File \fIfh\fP, MPI_Offset \fIoffset\fP, const void \fI*buf\fP,
-	int \fIcount\fP, MPI_Datatype \fIdatatype\fP, MPI_Status \fI*status\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_WRITE_AT_ALL(\fIFH\fP, \fIOFFSET\fP, \fIBUF\fP, \fICOUNT\fP,
-	\fIDATATYPE\fP, \fISTATUS\fP, \fIIERROR\fP)
-	<type>	\fIBUF\fP(*)
-	INTEGER	\fIFH, COUNT, DATATYPE, STATUS(MPI_STATUS_SIZE), IERROR\fP
-	INTEGER(KIND=MPI_OFFSET_KIND)	\fIOFFSET\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_write_at_all(\fIfh\fP, \fIoffset\fP, \fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIstatus\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	INTEGER(KIND=MPI_OFFSET_KIND), INTENT(IN) :: \fIoffset\fP
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Status) :: \fIstatus\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-fh
-File handle (handle).
-.TP 1i
-offset
-File offset (integer).
-.TP 1i
-buf
-Initial address of buffer (choice).
-.TP 1i
-count
-Number of elements in buffer (integer).
-.TP 1i
-datatype
-Data type of each buffer element (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-status
-Status object (status).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_write_at_all is a collective routine that attempts to write into the file associated with
-.I fh
-(at the
-.I offset
-position) a total number of
-.I count
-data items having
-.I datatype
-type from the user's buffer
-.I buf.
-The offset is in etype units relative to the current view. That is, holes are not counted
-when locating an offset. The data is written into those parts of the
-file specified by the current view. MPI_File_write_at_all stores the
-number of
-.I datatype
-elements actually written in
-.I status.
-All other fields of
-.I status
-are undefined.
-.sp
-It is erroneous to call this function if MPI_MODE_SEQUENTIAL mode was specified when the file was opened.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fIOFFSET\fP argument only for Fortran 90.  FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_OFFSET_KIND \fIOFFSET\fP
-.fi
-.sp
-where MPI_OFFSET_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_write_at_all_begin.3in b/ompi/mpi/man/man3/MPI_File_write_at_all_begin.3in
deleted file mode 100644
index 885b1ad08a8..00000000000
--- a/ompi/mpi/man/man3/MPI_File_write_at_all_begin.3in
+++ /dev/null
@@ -1,114 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_write_at_all_begin 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_write_at_all_begin\fP \- Writes a file at explicitly specified offsets; beginning part of a split collective routine (nonblocking).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_write_at_all_begin(MPI_File \fIfh\fP, MPI_Offset \fIoffset\fP,
-	const void \fI*buf\fP, int \fIcount\fP, MPI_Datatype \fIdatatype\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_WRITE_AT_ALL_BEGIN(\fIFH\fP, \fIOFFSET\fP, \fIBUF\fP, \fICOUNT\fP, \fIDATATYPE\fP, \fIIERROR\fP)
-	<type>	\fIBUF\fP(*)
-	INTEGER	\fIFH, COUNT, DATATYPE, IERROR\fP
-	INTEGER(KIND=MPI_OFFSET_KIND)	\fIOFFSET\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_write_at_all_begin(\fIfh\fP, \fIoffset\fP, \fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	INTEGER(KIND=MPI_OFFSET_KIND), INTENT(IN) :: \fIoffset\fP
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-offset
-File offset (handle).
-.ft R
-.TP 1i
-buf
-Initial address of buffer (choice).
-.ft R
-.TP 1i
-count
-Number of elements in buffer (integer).
-.ft R
-.TP 1i
-datatype
-Data type of each buffer element (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_write_at_all_begin is the beginning part of a split collective, that is, a nonblocking routine that attempts to write into the file associated with
-.I fh
-(at the
-.I offset
-position) a total number of
-.I count
-data items having
-.I datatype
-type from the user's buffer
-.I buf.
-The offset is in etype units relative to the current view. That is, holes are not counted
-when locating an offset. The data is written into those parts of the
-file specified by the current view.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fIOFFSET\fP argument only for Fortran 90.  FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_OFFSET_KIND \fIOFFSET\fP
-.fi
-.sp
-where MPI_OFFSET_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH NOTES
-.ft R
-All the nonblocking collective routines for data access are "split" into two routines, each with _begin or _end as a suffix. These split collective routines are subject to the semantic rules described in Section 9.4.5 of the MPI-2 standard.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_write_at_all_end.3in b/ompi/mpi/man/man3/MPI_File_write_at_all_end.3in
deleted file mode 100644
index c94ce8475ed..00000000000
--- a/ompi/mpi/man/man3/MPI_File_write_at_all_end.3in
+++ /dev/null
@@ -1,85 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_write_at_all_end 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_write_at_all_end\fP \- Writes a file at explicitly specified offsets; ending part of a split collective routine (blocking).
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_write_at_all_end(MPI_File \fIfh\fP, const void \fI*buf\fP,
-	MPI_Status \fI*status\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_WRITE_AT_ALL_END(\fIFH\fP, \fIBUF\fP, \fISTATUS\fP, \fIIERROR\fP)
-	<type>	\fIBUF(*)\fP
-	INTEGER	\fIFH, STATUS(MPI_STATUS_SIZE), IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_write_at_all_end(\fIfh\fP, \fIbuf\fP, \fIstatus\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIbuf\fP
-	TYPE(MPI_Status) :: \fIstatus\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-fh
-File handle (handle).
-
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-buf
-Initial address of buffer (choice).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-status
-Status object (status).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_write_at_all_end is the ending part of a split collective routine that stores the
-number of elements actually written into the file associated with
-.I fh
-in
-.I status.
-The data is written into those parts of the
-file specified by the current view. All other fields of
-.I status
-are undefined.
-
-.SH NOTES
-.ft R
-All the nonblocking collective routines for data access are "split" into two routines, each with _begin or _end as a suffix. These split collective routines are subject to the semantic rules described in Section 9.4.5 of the MPI-2 standard.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_write_ordered.3in b/ompi/mpi/man/man3/MPI_File_write_ordered.3in
deleted file mode 100644
index e8cc2fde28b..00000000000
--- a/ompi/mpi/man/man3/MPI_File_write_ordered.3in
+++ /dev/null
@@ -1,105 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015      Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_File_write_ordered 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_File_write_ordered\fP \- Writes a file at a location specified by a shared file pointer (blocking, collective).
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_File_write_ordered(MPI_File \fIfh\fP, const void \fI*buf\fP,
-	int \fIcount\fP, MPI_Datatype \fIdatatype\fP,
-	MPI_Status \fI*status\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_FILE_WRITE_ORDERED(\fIFH\fP, \fIBUF\fP, \fICOUNT\fP, \fIDATATYPE\fP,
-	\fISTATUS\fP, \fIIERROR\fP)
-	<type>	\fIBUF\fP(*)
-	INTEGER	\fIFH, COUNT, DATATYPE, STATUS(MPI_STATUS_SIZE), IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_File_write_ordered(\fIfh\fP, \fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIstatus\fP, \fIierror\fP)
-	TYPE(MPI_File), INTENT(IN) :: \fIfh\fP
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Status) :: \fIstatus\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-fh
-File handle (handle).
-.TP 1i
-buf
-Initial address of buffer (choice).
-.TP 1i
-count
-Number of elements in buffer (integer).
-.TP 1i
-datatype
-Data type of each buffer element (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-status
-Status object (Status).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_File_write_ordered is a collective routine. This routine must
-be called by all processes in the communicator group associated with
-the file handle
-.I fh.
-Each process may pass different argument values
-for the
-.I datatype
-and
-.I count
-arguments. Each process attempts to
-write, into the file associated with
-.I fh,
-a total number of
-.I count
-data items having datatype type contained in the user's buffer
-.I buf.
-For
-each process, the location in the file at which data is written is the
-position at which the shared file pointer would be after all processes
-whose ranks within the group are less than that of this process had
-written their data. MPI_File_write_ordered returns the number of
-.I datatype
-elements written in
-.I status.
-The shared file pointer is
-updated by the amounts of data requested by all processes of the
-group.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_File_write_ordered.md b/ompi/mpi/man/man3/MPI_File_write_ordered.md
deleted file mode 100644
index f57cad9ec7c..00000000000
--- a/ompi/mpi/man/man3/MPI_File_write_ordered.md
+++ /dev/null
@@ -1,81 +0,0 @@
-# Name
-
-`MPI_File_write_ordered` - Writes a file at a location specified by a
-shared file pointer (blocking, collective).
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_File_write_ordered(MPI_File fh, const void *buf,
-	int count, MPI_Datatype datatype,
-	MPI_Status *status)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_FILE_WRITE_ORDERED(FH, BUF, COUNT, DATATYPE,
-    STATUS, IERROR)
-    <type>	BUF(*)
-  	INTEGER	FH, COUNT, DATATYPE, STATUS(MPI_STATUS_SIZE), IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_File_write_ordered(fh, buf, count, datatype, status, ierror)
-    TYPE(MPI_File), INTENT(IN) :: fh
-    TYPE(*), DIMENSION(..), INTENT(IN) :: buf
-    INTEGER, INTENT(IN) :: count
-    TYPE(MPI_Datatype), INTENT(IN) :: datatype
-    TYPE(MPI_Status) :: status
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameters
-
-* `fh` : File handle (handle).
-* `buf` : Initial address of buffer (choice).
-* `count` : Number of elements in buffer (integer).
-* `datatype` : Data type of each buffer element (handle).
-
-# Output Parameters
-
-* `status` : Status object (Status).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-`MPI_File_write_ordered` is a collective routine. This routine must be
-called by all processes in the communicator group associated with the
-file handle `fh`. Each process may pass different argument values for
-the `datatype` and `count` arguments. Each process attempts to write,
-into the file associated with `fh`, a total number of `count` data items
-having `datatype` type contained in the user's buffer `buf`. For each
-process, the location in the file at which data is written is the
-position at which the shared file pointer would be after all processes
-whose ranks within the group are less than that of this process had
-written their data. `MPI_File_write_ordered` returns the number of
-`datatype` elements written in `status`. The shared file pointer is
-updated by the amounts of data requested by all processes of the group.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to
-`MPI_ERRORS_RETURN`. The error handler may be changed with
-`MPI_File_set_errhandler`; the predefined error handler
-`MPI_ERRORS_ARE_FATAL` may be used to make I/O errors fatal. Note that MPI
-does not guarantee that an MPI program can continue past an error.
diff --git a/ompi/mpi/man/man3/MPI_File_write_ordered_begin.md b/ompi/mpi/man/man3/MPI_File_write_ordered_begin.md
deleted file mode 100644
index dfa7d2af061..00000000000
--- a/ompi/mpi/man/man3/MPI_File_write_ordered_begin.md
+++ /dev/null
@@ -1,88 +0,0 @@
-# Name
-
-`MPI_File_write_ordered_begin` - Writes a file at a location specified
-by a shared file pointer; beginning part of a split collective routine
-(nonblocking).
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_File_write_ordered_begin(MPI_File fh, const void *buf,
-	int count, MPI_Datatype datatype)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_FILE_WRITE_ORDERED_BEGIN(FH, BUF, COUNT, DATATYPE, IERROR)
-    <type>	BUF(*)
-    INTEGER	FH, COUNT, DATATYPE, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_File_write_ordered_begin(fh, buf, count, datatype, ierror)
-    TYPE(MPI_File), INTENT(IN) :: fh
-    TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: buf
-    INTEGER, INTENT(IN) :: count
-    TYPE(MPI_Datatype), INTENT(IN) :: datatype
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input/Output Parameter
-
-* `fh` : File handle (handle).
-
-# Input Parameters
-
-* `buf` : Initial address of buffer (choice).
-* `count` : Number of elements in buffer (integer).
-* `datatype` : Data type of each buffer element (handle).
-
-# Output Parameter
-
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-`MPI_File_write_ordered_begin` is the beginning part of a split
-collective, nonblocking routine that must be called by all processes in
-the communicator group associated with the file handle `fh`. Each
-process may pass different argument values for the `datatype` and
-`count` arguments. After all processes of the group have issued their
-respective calls, each process attempts to write, into the file
-associated with `fh`, a total number of `count` data items having
-`datatype` type contained in the user's buffer `buf`. For each process,
-the location in the file at which data is written is the position at
-which the shared file pointer would be after all processes whose ranks
-within the group are less than that of this process had written their
-data.
-
-# Notes
-
-All the nonblocking collective routines for data access are "split"
-into two routines, each with _begin or _end as a suffix. These split
-collective routines are subject to the semantic rules described in
-Section 9.4.5 of the MPI-2 standard.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to
-`MPI_ERRORS_RETURN`. The error handler may be changed with
-`MPI_File_set_errhandler`; the predefined error handler
-`MPI_ERRORS_ARE_FATAL` may be used to make I/O errors fatal. Note that MPI
-does not guarantee that an MPI program can continue past an error.
diff --git a/ompi/mpi/man/man3/MPI_File_write_ordered_end.md b/ompi/mpi/man/man3/MPI_File_write_ordered_end.md
deleted file mode 100644
index 74f3745677e..00000000000
--- a/ompi/mpi/man/man3/MPI_File_write_ordered_end.md
+++ /dev/null
@@ -1,79 +0,0 @@
-# Name
-
-`MPI_File_write_ordered_end` - Writes a file at a location specified
-by a shared file pointer; ending part of a split collective routine
-(blocking).
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_File_write_ordered_end(MPI_File fh, const void *buf,
-	MPI_Status *status)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_FILE_WRITE_ORDERED_END(FH, BUF, STATUS, IERROR)
-    <type>	BUF(*)
-  	INTEGER	FH, STATUS(MPI_STATUS_SIZE), IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_File_write_ordered_end(fh, buf, status, ierror)
-    TYPE(MPI_File), INTENT(IN) :: fh
-    TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: buf
-    TYPE(MPI_Status) :: status
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input/Output Parameter
-
-* `fh` : File handle (handle).
-
-# Input Parameter
-
-* `buf` : Initial address of buffer (choice).
-
-# Output Parameters
-
-* `status` : Status object (status).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-`MPI_File_write_ordered_end` is the ending part of a split collective
-routine that must be called by all processes in the communicator group
-associated with the file handle `fh`. `MPI_File_write_ordered_end` returns
-the number of elements written into the file associated with `fh` in
-`status`.
-
-# Notes
-
-All the nonblocking collective routines for data access are "split"
-into two routines, each with _begin or _end as a suffix. These split
-collective routines are subject to the semantic rules described in
-Section 9.4.5 of the MPI-2 standard.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to
-`MPI_ERRORS_RETURN`. The error handler may be changed with
-`MPI_File_set_errhandler`; the predefined error handler
-`MPI_ERRORS_ARE_FATAL` may be used to make I/O errors fatal. Note that MPI
-does not guarantee that an MPI program can continue past an error.
diff --git a/ompi/mpi/man/man3/MPI_File_write_shared.md b/ompi/mpi/man/man3/MPI_File_write_shared.md
deleted file mode 100644
index 2a72daef358..00000000000
--- a/ompi/mpi/man/man3/MPI_File_write_shared.md
+++ /dev/null
@@ -1,73 +0,0 @@
-# Name
-
-`MPI_File_write_shared` - Writes a file using the shared file pointer
-(blocking, noncollective).
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_File_write_shared(MPI_File fh, const void *buf, int count,
-	MPI_Datatype datatype, MPI_Status *status)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_FILE_WRITE_SHARED(FH, BUF, COUNT, DATATYPE, STATUS, IERROR)
-    <type>	BUF(*)
-	  INTEGER	FH, COUNT, DATATYPE, STATUS(MPI_STATUS_SIZE), IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_File_write_shared(fh, buf, count, datatype, status, ierror)
-    TYPE(MPI_File), INTENT(IN) :: fh
-    TYPE(*), DIMENSION(..), INTENT(IN) :: buf
-    INTEGER, INTENT(IN) :: count
-    TYPE(MPI_Datatype), INTENT(IN) :: datatype
-    TYPE(MPI_Status) :: status
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input/Output Parameter
-
-* `fh` : File handle (handle).
-
-# Input Parameters
-
-* `buf` : Initial address of buffer (choice).
-* `count` : Number of elements in buffer (integer).
-* `datatype` : Data type of each buffer element (handle).
-
-# Output Parameters
-
-* `status` : Status object (status).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-`MPI_File_write_shared` is a blocking routine that uses the shared file
-pointer to write files. The order of serialization is not deterministic
-for this noncollective routine.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to
-`MPI_ERRORS_RETURN`. The error handler may be changed with
-`MPI_File_set_errhandler`; the predefined error handler
-`MPI_ERRORS_ARE_FATAL` may be used to make I/O errors fatal. Note that MPI
-does not guarantee that an MPI program can continue past an error.
diff --git a/ompi/mpi/man/man3/MPI_Finalize.md b/ompi/mpi/man/man3/MPI_Finalize.md
deleted file mode 100644
index 350b31e9bdf..00000000000
--- a/ompi/mpi/man/man3/MPI_Finalize.md
+++ /dev/null
@@ -1,103 +0,0 @@
-# Name
-
-`MPI_Finalize`  - Terminates MPI execution environment.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Finalize()
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_FINALIZE(IERROR)
-    INTEGER	IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Finalize(ierror)
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Output Parameter
-
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-This routine cleans up all MPI states. Once this routine is called, no
-MPI routine (not even `MPI_Init`) may be called, except for
-`MPI_Get_version`, `MPI_Initialized`, and `MPI_Finalized`. Unless there has
-been a call to `MPI_Abort`, you must ensure that all pending
-communications involving a process are complete before the process calls
-`MPI_Finalize`. If the call returns, each process may either continue
-local computations or exit without participating in further
-communication with other processes. At the moment when the last process
-calls `MPI_Finalize`, all pending sends must be matched by a receive, and
-all pending receives must be matched by a send.
-
-`MPI_Finalize` is collective over all connected processes. If no processes
-were spawned, accepted, or connected, then this means it is collective
-over `MPI_COMM_WORLD`. Otherwise, it is collective over the union of all
-processes that have been and continue to be connected.
-
-# Notes
-
-All processes must call this routine before exiting. All processes will
-still exist but may not make any further MPI calls. `MPI_Finalize`
-guarantees that all local actions required by communications the user
-has completed will, in fact, occur before it returns. However,
-`MPI_Finalize` guarantees nothing about pending communications that have
-not been completed; completion is ensured only by `MPI_Wait`, `MPI_Test,`
-or `MPI_Request_free` combined with some other verification of completion.
-
-For example, a successful return from a blocking communication operation
-or from `MPI_Wait` or `MPI_Test` means that the communication is completed
-by the user and the buffer can be reused, but does not guarantee that
-the local process has no more work to do. Similarly, a successful return
-from `MPI_Request_free` with a request handle generated by an `MPI_Isend`
-nullifies the handle but does not guarantee that the operation has
-completed. The `MPI_Isend` is complete only when a matching receive has
-completed.
-
-If you would like to cause actions to happen when a process finishes,
-attach an attribute to `MPI_COMM_SELF` with a callback function. Then,
-when `MPI_Finalize` is called, it will first execute the equivalent of an
-`MPI_Comm_free` on `MPI_COMM_SELF`. This will cause the delete callback
-function to be executed on all keys associated with `MPI_COMM_SELF` in an
-arbitrary order. If no key has been attached to `MPI_COMM_SELF`, then no
-callback is invoked. This freeing of `MPI_COMM_SELF` happens before any
-other parts of MPI are affected. Calling `MPI_Finalized` will thus return
-"false" in any of these callback functions. Once you have done this
-with `MPI_COMM_SELF`, the results of `MPI_Finalize` are not specified.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-# See Also
-
-[`MPI_Init`(3)](MPI_Init.html)
-[`MPI_Init_thread`(3)](MPI_Init_thread.html)
-[`MPI_Initialized`(3)](MPI_Initialized.html)
-[`MPI_Finalized`(3)](MPI_Finalized.html)
diff --git a/ompi/mpi/man/man3/MPI_Finalized.md b/ompi/mpi/man/man3/MPI_Finalized.md
deleted file mode 100644
index 46d27f5ecc6..00000000000
--- a/ompi/mpi/man/man3/MPI_Finalized.md
+++ /dev/null
@@ -1,64 +0,0 @@
-# Name
-
-`MPI_Finalized`  - Checks whether MPI has been finalized
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Finalized(int *flag)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_FINALIZED(FLAG, IERROR)
-    LOGICAL	FLAG
-    INTEGER	IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Finalized(flag, ierror)
-    LOGICAL, INTENT(OUT) :: flag
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Output Parameter
-
-* `flag` : True if MPI was finalized, and false otherwise (logical).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-This routine may be used to determine whether MPI has been finalized. It
-is one of a small number of routines that may be called before MPI is
-initialized and after MPI has been finalized (`MPI_Initialized` is
-another).
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-# See Also
-
-[`MPI_Init`(3)](MPI_Init.html)
-[`MPI_Init_thread`(3)](MPI_Init_thread.html)
-[`MPI_Initialized`(3)](MPI_Initialized.html)
-[`MPI_Finalize`(3)](MPI_Finalize.html)
diff --git a/ompi/mpi/man/man3/MPI_Free_mem.md b/ompi/mpi/man/man3/MPI_Free_mem.md
deleted file mode 100644
index bdd0f19fe9c..00000000000
--- a/ompi/mpi/man/man3/MPI_Free_mem.md
+++ /dev/null
@@ -1,64 +0,0 @@
-# Name
-
-`MPI_Free_mem`  - Frees memory that has been allocated using
-`MPI_Alloc_mem`.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Free_mem(void *base)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_FREE_MEM(BASE, IERROR)
-    <type> BASE(*)
-    INTEGER IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Free_mem(base, ierror)
-    TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: base
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameter
-
-* `base` : Initial address of memory segment allocated by `MPI_Alloc_mem`
-(choice).
-
-# Output Parameter
-
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-`MPI_Free_mem` frees memory that has been allocated by `MPI_Alloc_mem`.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-# See Also
-
-[`MPI_Alloc_mem`(3)](MPI_Alloc_mem.html)
diff --git a/ompi/mpi/man/man3/MPI_Gather.md b/ompi/mpi/man/man3/MPI_Gather.md
deleted file mode 100644
index 5d58221317b..00000000000
--- a/ompi/mpi/man/man3/MPI_Gather.md
+++ /dev/null
@@ -1,205 +0,0 @@
-# Name
-
-`MPI_Gather`, `MPI_Igather` - Gathers values from a group of processes.
-
-# Synopsis
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Gather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-	void *recvbuf, int recvcount, MPI_Datatype recvtype, int root,
-	MPI_Comm comm)
-
-int MPI_Igather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-	void *recvbuf, int recvcount, MPI_Datatype recvtype, int root,
-	MPI_Comm comm, MPI_Request *request)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_GATHER(SENDBUF, SENDCOUNT, SENDTYPE, RECVBUF, RECVCOUNT,
-    	RECVTYPE, ROOT, COMM, IERROR)
-    <type>	SENDBUF(*), RECVBUF(*)
-    INTEGER	SENDCOUNT, SENDTYPE, RECVCOUNT, RECVTYPE, ROOT
-    INTEGER	COMM, IERROR
-
-MPI_IGATHER(SENDBUF, SENDCOUNT, SENDTYPE, RECVBUF, RECVCOUNT,
-    	RECVTYPE, ROOT, COMM, REQUEST, IERROR)
-    <type>	SENDBUF(*), RECVBUF(*)
-    INTEGER	SENDCOUNT, SENDTYPE, RECVCOUNT, RECVTYPE, ROOT
-    INTEGER	COMM, REQUEST, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Gather(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype,
-    	root, comm, ierror)
-    TYPE(*), DIMENSION(..), INTENT(IN) :: sendbuf
-    TYPE(*), DIMENSION(..) :: recvbuf
-    INTEGER, INTENT(IN) :: sendcount, recvcount, root
-    TYPE(MPI_Datatype), INTENT(IN) :: sendtype, recvtype
-    TYPE(MPI_Comm), INTENT(IN) :: comm
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-
-MPI_Igather(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype,
-    	root, comm, request, ierror)
-    TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: sendbuf
-    TYPE(*), DIMENSION(..), ASYNCHRONOUS :: recvbuf
-    INTEGER, INTENT(IN) :: sendcount, recvcount, root
-    TYPE(MPI_Datatype), INTENT(IN) :: sendtype, recvtype
-    TYPE(MPI_Comm), INTENT(IN) :: comm
-    TYPE(MPI_Request), INTENT(OUT) :: request
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-
-# Input Parameters
-
-* `sendbuf` : Starting address of send buffer (choice).
-* `sendcount` : Number of elements in send buffer (integer).
-* `sendtype` : Datatype of send buffer elements (handle).
-* `recvcount` : Number of elements for any single receive (integer, significant only
-at root).
-* `recvtype` : Datatype of recvbuffer elements (handle, significant only at root).
-* `root` : Rank of receiving process (integer).
-* `comm` : Communicator (handle).
-
-
-# Output Parameters
-
-* `recvbuf` : Address of receive buffer (choice, significant only at root).
-* `request` : Request (handle, non-blocking only).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-Each process (root process included) sends the contents of its send
-buffer to the root process. The root process receives the messages and
-stores them in rank order. The outcome is as if each of the n processes
-in the group (including the root process) had executed a call to
-
-```c
-MPI_Send(sendbuf, sendcount, sendtype, root, ...)
-```
-
-and the root had executed n calls to
-
-```c
-MPI_Recv(recfbuf + i * recvcount * extent(recvtype), recvcount, recvtype, i, ...)
-```
-
-where extent(recvtype) is the type extent obtained from a call to
-`MPI_Type_extent()`.
-
-An alternative description is that the n messages sent by the processes
-in the group are concatenated in rank order, and the resulting message
-is received by the root as if by a call to `MPI_RECV(recvbuf, recvcount*
-n, recvtype, ... )`.
-
-The receive buffer is ignored for all nonroot processes.
-
-General, derived datatypes are allowed for both sendtype and recvtype.
-The type signature of `sendcount`, `sendtype` on process i must be equal to
-the type signature of `recvcount`, `recvtype` at the root. This implies that
-the amount of data sent must be equal to the amount of data received,
-pairwise between each process and the root. Distinct type maps between
-sender and receiver are still allowed.
-
-All arguments to the function are significant on process root, while on
-other processes, only arguments `sendbuf`, `sendcount`, `sendtype`, `root`, `comm`
-are significant. The arguments `root` and `comm` must have identical values
-on all processes.
-
-The specification of counts and types should not cause any location on
-the root to be written more than once. Such a call is erroneous.
-Note that the `recvcount` argument at the root indicates the number of
-items it receives from each process, not the total number of items it
-receives.
-
-Example 1: Gather 100 ints from every process in group to root.
-
-```c
-MPI_Comm comm;
-int gsize,sendarray[100];
-int root, *rbuf;
-//...
-
-MPI_Comm_size( comm, &gsize);
-rbuf = (int *)malloc(gsize*100*sizeof(int));
-
-MPI_Gather( sendarray, 100, MPI_INT, rbuf, 100, MPI_INT, root, comm);
-```
-
-Example 2: Previous example modified -- only the root allocates
-memory for the receive buffer.
-
-```c
-MPI_Comm comm;
-int gsize,sendarray[100];
-int root, myrank, *rbuf;
-//...
-
-MPI_Comm_rank( comm, myrank);
-if ( myrank == root) {
-    MPI_Comm_size( comm, &gsize);
-    rbuf = (int *)malloc(gsize*100*sizeof(int));
-}
-MPI_Gather( sendarray, 100, MPI_INT, rbuf, 100, MPI_INT, root, comm);
-```
-
-Example 3: Do the same as the previous example, but use a derived
-datatype. Note that the type cannot be the entire set of gsize * 100
-ints since type matching is defined pairwise between the root and each
-process in the gather.
-
-```c
-MPI_Comm comm;
-int gsize,sendarray[100];
-int root, *rbuf;
-MPI_Datatype rtype;
-//...
-
-MPI_Comm_size( comm, &gsize);
-MPI_Type_contiguous( 100, MPI_INT, &rtype );
-MPI_Type_commit( &rtype );
-rbuf = (int *)malloc(gsize*100*sizeof(int));
-MPI_Gather( sendarray, 100, MPI_INT, rbuf, 1, rtype, root, comm);
-```
-
-# Use Of In-Place Option
-
-When the communicator is an intracommunicator, you can perform a gather operation in-place (the output buffer is used as the input buffer).  Use the variable `MPI_IN_PLACE` as the value of the root process `sendbuf`.  In this case, `sendcount` and `sendtype` are ignored, and the contribution of the root process to the gathered vector is assumed to already be in the correct place in the receive buffer.
-Note that `MPI_IN_PLACE` is a special kind of value; it has the same restrictions on its use as MPI_BOTTOM.
-Because the in-place option converts the receive buffer into a send-and-receive buffer, a Fortran binding that includes `INTENT` must mark these as `INOUT`, not `OUT`.
-
-# When Communicator Is An Inter-Communicator
-
-When the communicator is an inter-communicator, the root process in the first group gathers data from all the processes in the second group.  The first group defines the root process.  That process uses MPI_ROOT as the value of its `root` argument.  The remaining processes use `MPI_PROC_NULL` as the value of their `root` argument.  All processes in the second group use the rank of that root process in the first group as the value of their `root` argument.   The send buffer argument of the processes in the first group must be consistent with the receive buffer argument of the root process in the second group.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-See the MPI man page for a full list of MPI error codes.
-
-# See Also
-
-[`MPI_Gatherv`(3)](MPI_Gatherv.html)
-[`MPI_Scatter`(3)](MPI_Scatter.html)
-[`MPI_Scatterv`(3)](MPI_Scatterv.html)
diff --git a/ompi/mpi/man/man3/MPI_Gatherv.md b/ompi/mpi/man/man3/MPI_Gatherv.md
deleted file mode 100644
index 2a52322ef84..00000000000
--- a/ompi/mpi/man/man3/MPI_Gatherv.md
+++ /dev/null
@@ -1,379 +0,0 @@
-# Name
-
-`MPI_Gatherv`, `MPI_Igatherv` - Gathers varying amounts of data from all
-processes to the root process
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Gatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-    void *recvbuf, const int recvcounts[], const int displs[], MPI_Datatype recvtype,
-    int root, MPI_Comm comm)
-
-int MPI_Igatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-    void *recvbuf, const int recvcounts[], const int displs[], MPI_Datatype recvtype,
-    int root, MPI_Comm comm, MPI_Request *request)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_GATHERV(SENDBUF, SENDCOUNT, SENDTYPE, RECVBUF, RECVCOUNTS,
-    	DISPLS, RECVTYPE, ROOT, COMM, IERROR)
-    <type>	SENDBUF(*), RECVBUF(*)
-    INTEGER	SENDCOUNT, SENDTYPE, RECVCOUNTS(*), DISPLS(*)
-    INTEGER	RECVTYPE, ROOT, COMM, IERROR
-
-MPI_IGATHERV(SENDBUF, SENDCOUNT, SENDTYPE, RECVBUF, RECVCOUNTS,
-    	DISPLS, RECVTYPE, ROOT, COMM, REQUEST, IERROR)
-    <type>	SENDBUF(*), RECVBUF(*)
-    INTEGER	SENDCOUNT, SENDTYPE, RECVCOUNTS(*), DISPLS(*)
-    INTEGER	RECVTYPE, ROOT, COMM, REQUEST, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Gatherv(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs,
-    	recvtype, root, comm, ierror)
-    TYPE(*), DIMENSION(..), INTENT(IN) :: sendbuf
-    TYPE(*), DIMENSION(..) :: recvbuf
-    INTEGER, INTENT(IN) :: sendcount, recvcounts(*), displs(*), root
-    TYPE(MPI_Datatype), INTENT(IN) :: sendtype, recvtype
-    TYPE(MPI_Comm), INTENT(IN) :: comm
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-
-MPI_Igatherv(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs,
-    	recvtype, root, comm, request, ierror)
-    TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: sendbuf
-    TYPE(*), DIMENSION(..), ASYNCHRONOUS :: recvbuf
-    INTEGER, INTENT(IN) :: sendcount, root
-    INTEGER, INTENT(IN), ASYNCHRONOUS :: recvcounts(*), displs(*)
-    TYPE(MPI_Datatype), INTENT(IN) :: sendtype, recvtype
-    TYPE(MPI_Comm), INTENT(IN) :: comm
-    TYPE(MPI_Request), INTENT(OUT) :: request
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameters
-
-* `sendbuf` : Starting address of send buffer (choice).
-* `sendcount` : Number of elements in send buffer (integer).
-* `sendtype` : Datatype of send buffer elements (handle).
-* `recvcounts` : Integer array (of length group size) containing the number of
-elements that are received from each process (significant only at
-root).
-* `displs` : Integer array (of length group size). Entry i specifies the
-displacement relative to recvbuf at which to place the incoming data
-from process i (significant only at root).
-* `recvtype` : Datatype of recv buffer elements (significant only at root)
-(handle).
-* `root` : Rank of receiving process (integer).
-* `comm` : Communicator (handle).
-
-
-
-# Output Parameters
-
-* `recvbuf` : Address of receive buffer (choice, significant only at root).
-* `request` : Request (handle, non-blocking only).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-
-`MPI_Gatherv` extends the functionality of `MPI_Gather` by allowing a
-varying count of data from each process, since `recvcounts` is now an
-array. It also allows more flexibility as to where the data is placed on
-the root, by providing the new argument, `displs`.
-
-The outcome is as if each process, including the root process, sends a
-message to the root,
-
-```c
-MPI_Send(sendbuf, sendcount, sendtype, root, ...)
-```
-
-and the root executes n receives,
-
-```c
-MPI_Recv(recvbuf + disp[i] * extent(recvtype), 
-    recvcounts[i], recvtype, i, ...)
-```
-
-Messages are placed in the receive buffer of the root process in rank
-order, that is, the data sent from process j is placed in the jth
-portion of the receive buffer `recvbuf` on process root. The jth portion
-of `recvbuf` begins at offset displs[j] elements (in terms of `recvtype`)
-into `recvbuf`.
-
-The receive buffer is ignored for all nonroot processes.
-
-The type signature implied by `sendcount`, `sendtype` on process i must be
-equal to the type signature implied by `recvcounts[i]`, `recvtype` at the
-root. This implies that the amount of data sent must be equal to the
-amount of data received, pairwise between each process and the root.
-Distinct type maps between sender and receiver are still allowed, as
-illustrated in Example 2, below.
-
-All arguments to the function are significant on process `root`, while on
-other processes, only arguments `sendbuf`, `sendcount`, `sendtype`, `root`, `comm`
-are significant. The arguments `root` and `comm` must have identical values
-on all processes.
-
-The specification of counts, types, and displacements should not cause
-any location on the `root` to be written more than once. Such a call is
-erroneous.
-
-Example 1: Now have each process send 100 ints to `root`, but place
-each set (of 100) stride ints apart at receiving end. Use `MPI_Gatherv`
-and the `displs` argument to achieve this effect. Assume stride >= 100.
-
-```c
-MPI_Comm comm;
-int gsize,sendarray[100];
-int root, *rbuf, stride;
-int *displs,i,*rcounts;
-//      ...
-MPI_Comm_size(comm, &gsize);
-rbuf = (int *)malloc(gsize*stride*sizeof(int));
-displs = (int *)malloc(gsize*sizeof(int));
-rcounts = (int *)malloc(gsize*sizeof(int));
-for (i=0; i<gsize; ++i) {
-    displs[i] = i*stride;
-    rcounts[i] = 100;
-}
-MPI_Gatherv(sendarray, 100, MPI_INT, rbuf, rcounts,
-    displs, MPI_INT, root, comm);
-```
-
-Note that the program is erroneous if stride < 100.
-
-Example 2: Same as Example 1 on the receiving side, but send the 100
-ints from the 0th column of a 100  150 int array, in C.
-
-```c
-MPI_Comm comm;
-int gsize,sendarray[100][150];
-int root, *rbuf, stride;
-MPI_Datatype stype;
-int *displs,i,*rcounts;
-//      ...
-MPI_Comm_size(comm, &gsize);
-rbuf = (int *)malloc(gsize*stride*sizeof(int));
-displs = (int *)malloc(gsize*sizeof(int));
-rcounts = (int *)malloc(gsize*sizeof(int));
-for (i=0; i<gsize; ++i) {
-    displs[i] = i*stride;
-    rcounts[i] = 100;
-}
-/* Create datatype for 1 column of array
- */
-MPI_Type_vector(100, 1, 150, MPI_INT, &stype);
-MPI_Type_commit( &stype );
-MPI_Gatherv(sendarray, 1, stype, rbuf, rcounts,
-    displs, MPI_INT, root, comm);
-```
-
-Example 3: Process i sends (100-i) ints from the ith column of a 100
-x 150 int array, in C. It is received into a buffer with stride, as in
-the previous two examples.
-
-```c
-MPI_Comm comm;
-int gsize,sendarray[100][150],*sptr;
-int root, *rbuf, stride, myrank;
-MPI_Datatype stype;
-int *displs,i,*rcounts;
-//      ...
-MPI_Comm_size(comm, &gsize);
-MPI_Comm_rank( comm, &myrank );
-rbuf = (int *)malloc(gsize*stride*sizeof(int));
-displs = (int *)malloc(gsize*sizeof(int));
-rcounts = (int *)malloc(gsize*sizeof(int));
-for (i=0; i<gsize; ++i) {
-    displs[i] = i*stride;
-    rcounts[i] = 100-i;  /* note change from previous example */
-}
-/* Create datatype for the column we are sending
- */
-MPI_Type_vector(100-myrank, 1, 150, MPI_INT, &stype);
-MPI_Type_commit( &stype );
-/* sptr is the address of start of "myrank" column
- */
-sptr = &sendarray[0][myrank];
-MPI_Gatherv(sptr, 1, stype, rbuf, rcounts, displs, MPI_INT,
-   root, comm);
-```
-
-Note that a different amount of data is received from each process.
-
-Example 4: Same as Example 3, but done in a different way at the
-sending end. We create a datatype that causes the correct striding at
-the sending end so that we read a column of a C array.
-
-```c
-MPI_Comm comm;
-int gsize,sendarray[100][150],*sptr;
-int root, *rbuf, stride, myrank, disp[2], blocklen[2];
-MPI_Datatype stype,type[2];
-int *displs,i,*rcounts;
-//      ...
-MPI_Comm_size(comm, &gsize);
-MPI_Comm_rank( comm, &myrank );
-rbuf = (int *)alloc(gsize*stride*sizeof(int));
-displs = (int *)malloc(gsize*sizeof(int));
-rcounts = (int *)malloc(gsize*sizeof(int));
-for (i=0; i<gsize; ++i) {
-    displs[i] = i*stride;
-    rcounts[i] = 100-i;
-}
-/* Create datatype for one int, with extent of entire row
- */
-disp[0] = 0;       disp[1] = 150*sizeof(int);
-type[0] = MPI_INT; type[1] = MPI_UB;
-blocklen[0] = 1;   blocklen[1] = 1;
-MPI_Type_struct( 2, blocklen, disp, type, &stype );
-MPI_Type_commit( &stype );
-sptr = &sendarray[0][myrank];
-MPI_Gatherv(sptr, 100-myrank, stype, rbuf, rcounts,
-    displs, MPI_INT, root, comm);
-```
-
-Example 5: Same as Example 3 at sending side, but at receiving side
-we make the stride between received blocks vary from block to block.
-
-```c
-MPI_Comm comm;
-int gsize,sendarray[100][150],*sptr;
-int root, *rbuf, *stride, myrank, bufsize;
-MPI_Datatype stype;
-int *displs,i,*rcounts,offset;
-//      ...
-MPI_Comm_size( comm, &gsize);
-MPI_Comm_rank( comm, &myrank );
-de = (int *)malloc(gsize*sizeof(int));
-//         ...
-/* stride[i] for i = 0 to gsize-1 is set somehow
- */
-/*set up displs and rcounts vectors first
- */
-displs = (int *)malloc(gsize*sizeof(int));
-rcounts = (int *)malloc(gsize*sizeof(int));
-offset = 0;
-for (i=0; i<gsize; ++i) {
-    displs[i] = offset;
-    offset += stride[i];
-    rcounts[i] = 100-i;
-}
-/* the required buffer size for rbuf is now easily obtained
- */
-bufsize = displs[gsize-1]+rcounts[gsize-1];
-rbuf = (int *)malloc(bufsize*sizeof(int));
-/* Create datatype for the column we are sending
- */
-MPI_Type_vector(100-myrank, 1, 150, MPI_INT, &stype);
-MPI_Type_commit( &stype );
-sptr = &sendarray[0][myrank];
-MPI_Gatherv(sptr, 1, stype, rbuf, rcounts,
-    displs, MPI_INT, root, comm);
-```
-
-Example 6: Process i sends num ints from the ith column of a 100 x
-150 int array, in C. The complicating factor is that the various values
-of num are not known to `root`, so a separate gather must first be run to
-find these out. The data is placed contiguously at the receiving end.
-
-```c
-MPI_Comm comm;
-int gsize,sendarray[100][150],*sptr;
-int root, *rbuf, stride, myrank, disp[2], blocklen[2];
-MPI_Datatype stype,types[2];
-int *displs,i,*rcounts,num;
-//      ...
-MPI_Comm_size( comm, &gsize);
-MPI_Comm_rank( comm, &myrank );
-/*First, gather nums to root
- */
-rcounts = (int *)malloc(gsize*sizeof(int));
-MPI_Gather( &num, 1, MPI_INT, rcounts, 1, MPI_INT, root, comm);
-/* root now has correct rcounts, using these we set
- * displs[] so that data is placed contiguously (or
- * concatenated) at receive end
- */
-displs = (int *)malloc(gsize*sizeof(int));
-displs[0] = 0;
-for (i=1; i<gsize; ++i) {
-    displs[i] = displs[i-1]+rcounts[i-1];
-}
-/* And, create receive buffer
- */
-rbuf = (int *)malloc(gsize*(displs[gsize-1]+rcounts[gsize-1])
-        *sizeof(int));
-/* Create datatype for one int, with extent of entire row
- */
-disp[0] = 0;       disp[1] = 150*sizeof(int);
-type[0] = MPI_INT; type[1] = MPI_UB;
-blocklen[0] = 1;   blocklen[1] = 1;
-MPI_Type_struct( 2, blocklen, disp, type, &stype );
-MPI_Type_commit( &stype );
-sptr = &sendarray[0][myrank];
-MPI_Gatherv(sptr, num, stype, rbuf, rcounts,
-            displs, MPI_INT, root, comm);
-```
-
-# Use Of In-Place Option
-
-The in-place option operates in the same way as it does for `MPI_Gather.`
-When the communicator is an intracommunicator, you can perform a gather
-operation in-place (the output buffer is used as the input buffer). Use
-the variable `MPI_IN_PLACE` as the value of the root process `sendbuf`. In
-this case, `sendcount` and `sendtype` are ignored, and the contribution
-of the `root` process to the gathered vector is assumed to already be in
-the correct place in the receive buffer.
-
-Note that `MPI_IN_PLACE` is a special kind of value; it has the same
-restrictions on its use as `MPI_BOTTOM.`
-
-Because the in-place option converts the receive buffer into a
-send-and-receive buffer, a Fortran binding that includes INTENT must
-mark these as INOUT, not OUT.
-
-# When Communicator Is An Inter-Communicator
-
-When the communicator is an inter-communicator, the `root` process in the
-first group gathers data from all the processes in the second group. The
-first group defines the root process. That process uses `MPI_ROOT` as the
-value of its `root` argument. The remaining processes use `MPI_PROC_NULL`
-as the value of their `root` argument. All processes in the second group
-use the rank of that root process in the first group as the value of
-their `root` argument. The send buffer argument of the processes in the
-first group must be consistent with the receive buffer argument of the
-`root` process in the second group.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-# See Also
-
-[`MPI_Gather`(3)](MPI_Gather.html)
-[`MPI_Scatter`(3)](MPI_Scatter.html)
-[`MPI_Scatterv`(3)](MPI_Scatterv.html)
diff --git a/ompi/mpi/man/man3/MPI_Get.md b/ompi/mpi/man/man3/MPI_Get.md
deleted file mode 100644
index 3990f066145..00000000000
--- a/ompi/mpi/man/man3/MPI_Get.md
+++ /dev/null
@@ -1,131 +0,0 @@
-# Name
-
-`MPI_Get`, `MPI_Rget` - Copies data from the target memory to the
-origin.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-MPI_Get(void *origin_addr, int origin_count, MPI_Datatype
-    origin_datatype, int target_rank, MPI_Aint target_disp,
-    int target_count, MPI_Datatype target_datatype, MPI_Win win)
-
-MPI_Rget(void *origin_addr, int origin_count, MPI_Datatype
-     origin_datatype, int target_rank, MPI_Aint target_disp,
-     int target_count, MPI_Datatype target_datatype, MPI_Win win,
-         MPI_Request *request)
-```
-
-## Fortran Syntax (See Fortran 77 Notes)
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_GET(ORIGIN_ADDR, ORIGIN_COUNT, ORIGIN_DATATYPE, TARGET_RANK,
-    TARGET_DISP, TARGET_COUNT, TARGET_DATATYPE, WIN, IERROR)
-    <type> ORIGIN_ADDR(*)
-    INTEGER(KIND=MPI_ADDRESS_KIND) TARGET_DISP
-    INTEGER ORIGIN_COUNT, ORIGIN_DATATYPE, TARGET_RANK,
-    TARGET_COUNT, TARGET_DATATYPE, WIN, IERROR
-
-MPI_RGET(ORIGIN_ADDR, ORIGIN_COUNT, ORIGIN_DATATYPE, TARGET_RANK,
-     TARGET_DISP, TARGET_COUNT, TARGET_DATATYPE, WIN, REQUEST, IERROR)
-     <type> ORIGIN_ADDR(*)
-     INTEGER(KIND=MPI_ADDRESS_KIND) TARGET_DISP
-     INTEGER ORIGIN_COUNT, ORIGIN_DATATYPE, TARGET_RANK,
-     TARGET_COUNT, TARGET_DATATYPE, WIN, REQUEST, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Get(origin_addr, origin_count, origin_datatype, target_rank,
-    	target_disp, target_count, target_datatype, win, ierror)
-    TYPE(*), DIMENSION(..), ASYNCHRONOUS :: origin_addr
-    INTEGER, INTENT(IN) :: origin_count, target_rank, target_count
-    TYPE(MPI_Datatype), INTENT(IN) :: origin_datatype, target_datatype
-    INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
-    TYPE(MPI_Win), INTENT(IN) :: win
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-
-MPI_Rget(origin_addr, origin_count, origin_datatype, target_rank,
-    target_disp, target_count, target_datatype, win, request,
-    	ierror)
-    TYPE(*), DIMENSION(..), ASYNCHRONOUS :: origin_addr
-    INTEGER, INTENT(IN) :: origin_count, target_rank, target_count
-    TYPE(MPI_Datatype), INTENT(IN) :: origin_datatype, target_datatype
-    INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) :: target_disp
-    TYPE(MPI_Win), INTENT(IN) :: win
-    TYPE(MPI_Request), INTENT(OUT) :: request
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameters
-
-* `origin_addr` : Initial address of origin buffer (choice).
-* `origin_count` : Number of entries in origin buffer (nonnegative integer).
-* `origin_datatype` : Data type of each entry in origin buffer (handle).
-* `target_rank` : Rank of target (nonnegative integer).
-* `target_disp` : Displacement from window start to the beginning of the target buffer
-(nonnegative integer).
-* `target_count` : Number of entries in target buffer (nonnegative integer).
-* `target datatype` : datatype of each entry in target buffer (handle)
-* `win` : window object used for communication (handle)
-
-# Output Parameter
-
-* `request` : MPI_Rget: RMA request
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-`MPI_Get` copies data from the target memory to the origin, similar to
-`MPI_Put`, except that the direction of data transfer is reversed. The
-`origin_datatype` may not specify overlapping entries in the origin
-buffer. The target buffer must be contained within the target window,
-and the copied data must fit, without truncation, in the origin buffer.
-Only processes within the same node can access the target window.
-
-`MPI_Rget` is similar to `MPI_Get`, except that it allocates a
-communication `request` object and associates it with the `request` handle
-(the argument `request`) that can be used to wait or test for
-completion. The completion of an `MPI_Rget` operation indicates that the
-data is available in the origin buffer. If `origin_addr` points to
-memory attached to a window, then the data becomes available in the
-private copy of this window.
-
-# Fortran 77 Notes
-
-The MPI standard prescribes portable Fortran syntax for the
-`TARGET_DISP` argument only for Fortran 90. FORTRAN 77 users may use the
-non-portable syntax
-
-```fortran
-INTEGER*MPI_ADDRESS_KIND TARGET_DISP
-```
-
-where `MPI_ADDRESS_KIND` is a constant defined in mpif.h and gives the
-length of the declared integer in bytes.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-# See Also
-
-[`MPI_Put`(3)](MPI_Put.html)
diff --git a/ompi/mpi/man/man3/MPI_Get_accumulate.md b/ompi/mpi/man/man3/MPI_Get_accumulate.md
deleted file mode 100644
index d18be6b41ec..00000000000
--- a/ompi/mpi/man/man3/MPI_Get_accumulate.md
+++ /dev/null
@@ -1,193 +0,0 @@
-# Name
-
-`MPI_Get_accumulate`, `MPI_Rget_accumulate` - Combines the contents
-of the origin buffer with that of a target buffer and returns the target
-buffer value.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Get_accumulate(const void *origin_addr, int origin_count,
-    MPI_Datatype origin_datatype, void *result_addr,
-    int result_count, MPI_Datatype result_datatype,
-    int target_rank, MPI_Aint target_disp, int target_count,
-    MPI_Datatype target_datatype, MPI_Op op, MPI_Win win)
-
-int MPI_Rget_accumulate(const void *origin_addr, int origin_count,
-    MPI_Datatype origin_datatype, void *result_addr,
-    int result_count, MPI_Datatype result_datatype,
-    int target_rank, MPI_Aint target_disp, int target_count,
-    MPI_Datatype target_datatype, MPI_Op op, MPI_Win win,
-    MPI_Request *request)
-```
-
-## Fortran Syntax (See Fortran 77 Notes)
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_GET_ACCUMULATE(ORIGIN_ADDR, ORIGIN_COUNT, ORIGIN_DATATYPE, RESULT_ADDR,
-        RESULT_COUNT, RESULT_DATATYPE, TARGET_RANK, TARGET_DISP, TARGET_COUNT,
-        TARGET_DATATYPE, OP, WIN, IERROR)
-    <type> ORIGIN_ADDR, RESULT_ADDR(*)
-    INTEGER(KIND=MPI_ADDRESS_KIND) TARGET_DISP
-    INTEGER ORIGIN_COUNT, ORIGIN_DATATYPE, TARGET_COUNT, TARGET_DATATYPE,
-        TARGET_RANK, TARGET_COUNT, TARGET_DATATYPE, OP, WIN, IERROR
-
-MPI_RGET_ACCUMULATE(ORIGIN_ADDR, ORIGIN_COUNT, ORIGIN_DATATYPE, RESULT_ADDR,
-        RESULT_COUNT, RESULT_DATATYPE, TARGET_RANK, TARGET_DISP, TARGET_COUNT,
-        TARGET_DATATYPE, OP, WIN, REQUEST, IERROR)
-    <type> ORIGIN_ADDR, RESULT_ADDR(*)
-    INTEGER(KIND=MPI_ADDRESS_KIND) TARGET_DISP
-    INTEGER ORIGIN_COUNT, ORIGIN_DATATYPE, TARGET_COUNT, TARGET_DATATYPE,
-        TARGET_RANK, TARGET_COUNT, TARGET_DATATYPE, OP, WIN, REQUEST, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Get_accumulate(origin_addr, origin_count, origin_datatype, result_addr,
-    	result_count, result_datatype, target_rank, target_disp,
-    	target_count, target_datatype, op, win, ierror)
-    TYPE(*), DIMENSION(..), INTENT(IN) :: origin_addr
-    TYPE(*), DIMENSION(..) :: result_addr
-    INTEGER, INTENT(IN) :: origin_count, result_count, target_rank, target_count
-    TYPE(MPI_Datatype), INTENT(IN) :: origin_datatype, target_datatype, result_datatype
-    INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) :: target_dist
-    TYPE(MPI_Op), INTENT(IN) :: op
-    TYPE(MPI_Win), INTENT(IN) :: win
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-
-MPI_Rget_accumulate(origin_addr, origin_count, origin_datatype,
-    	result_addr, result_count, result_datatype, target_rank,
-    	target_disp, target_count, target_datatype, op, win, request,
-    	ierror)
-    TYPE(*), DIMENSION(..), INTENT(IN) :: origin_addr
-    TYPE(*), DIMENSION(..) :: result_addr
-    INTEGER, INTENT(IN) :: origin_count, result_count, target_rank, target_count
-    TYPE(MPI_Datatype), INTENT(IN) :: origin_datatype, target_datatype, result_datatype
-    INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) :: target_dist
-    TYPE(MPI_Op), INTENT(IN) :: op
-    TYPE(MPI_Win), INTENT(IN) :: win
-    TYPE(MPI_Request), INTENT(OUT) :: request
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameters
-
-* `origin_addr` : Initial address of buffer (choice).
-* `origin_count` : Number of entries in buffer (nonnegative integer).
-* `origin_datatype` : Data type of each buffer entry (handle).
-* `result_addr` : Initial address of result buffer (choice).
-* `result_count` : Number of entries in result buffer (nonnegative integer).
-* `result_datatype` : Data type of each result buffer entry (handle).
-* `target_rank` : Rank of target (nonnegative integer).
-* `target_disp` : Displacement from start of window to beginning of target buffer
-(nonnegative integer).
-* `target_count` : Number of entries in target buffer (nonnegative integer).
-* `target_datatype` : Data type of each entry in target buffer (handle).
-* `op` : Reduce operation (handle).
-* `win` : Window object (handle).
-
-# Output Parameter
-
-* `MPI_Rget_accumulate`: RMA request
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-`MPI_Get_accumulate` is a function used for one-sided MPI
-communication that adds the contents of the origin buffer (as defined by
-`origin_addr`, `origin_count`, and `origin_datatype`) to the buffer
-specified by the arguments `target_count` and `target_datatype`, at
-offset `target_disp`, in the target window specified by `target_rank`
-and `win`, using the operation `op`. `MPI_Get_accumulate` returns in
-the result buffer `result_addr` the contents of the target buffer before
-the accumulation.
-
-Any of the predefined operations for `MPI_Reduce`, as well as `MPI_NO_OP,`
-can be used. User-defined functions cannot be used. For example, if `op`
-is `MPI_SUM`, each element of the origin buffer is added to the
-corresponding element in the target, replacing the former value in the
-target.
-
-Each datatype argument must be a predefined data type or a derived data
-type, where all basic components are of the same predefined data type.
-Both datatype arguments must be constructed from the same predefined
-data type. The operation `op` applies to elements of that predefined
-type. The `target_datatype` argument must not specify overlapping
-entries, and the target buffer must fit in the target window.
-
-A new predefined operation, `MPI_REPLACE`, is defined. It corresponds to
-the associative function f(a, b) =b; that is, the current value in the
-target memory is replaced by the value supplied by the origin.
-
-A new predefined operation, `MPI_NO_OP`, is defined. It corresponds to the
-assiciative function f(a, b) = a; that is the current value in the
-target memory is returned in the result buffer at the origin and no
-operation is performed on the target buffer.
-
-`MPI_Rget_accumulate` is similar to `MPI_Get_accumulate`, except
-that it allocates a communication request object and associates it with
-the request handle (the argument request) that can be used to wait or
-test for completion. The completion of an `MPI_Rget_accumulate`
-operation indicates that the data is available in the result buffer and
-the origin buffer is free to be updated. It does not indicate that the
-operation has been completed at the target window.
-
-# Fortran 77 Notes
-
-The MPI standard prescribes portable Fortran syntax for the
-`TARGET_DISP` argument only for Fortran 90. FORTRAN 77 users may use the
-non-portable syntax
-
-```fortran
-INTEGER*MPI_ADDRESS_KIND TARGET_DISP
-```
-
-where MPI_ADDRESS_KIND is a constant defined in mpif.h and gives the
-length of the declared integer in bytes.
-
-# Notes
-
-The generic functionality of `MPI_Get_accumulate` might limit the
-performance of fetch-and-increment or fetch-and-add calls that might be
-supported by special hardware operations. `MPI_Fetch_and_op` thus allows
-for a fast implementation of a commonly used subset of the functionality
-of `MPI_Get_accumulate`.
-
-`MPI_Get` is a special case of `MPI_Get_accumulate`, with the operation
-`MPI_NO_OP`. Note, however, that `MPI_Get` and `MPI_Get_accumulate` have
-different constraints on concurrent updates.
-
-It is the user's responsibility to guarantee that, when using the
-accumulate functions, the target displacement argument is such that
-accesses to the window are properly aligned according to the data type
-arguments in the call to the `MPI_Get_accumulate` function.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-# See Also
-
-[`MPI_Put`(3)](MPI_Put.html)
-[`MPI_Get`(3)](MPI_Get.html)
-[`MPI_Accumulate`(3)](MPI_Accumulate.html)
-[`MPI_Fetch_and_op`(3)](MPI_Fetch_and_op.html)
-[`MPI_Reduce`(3)](MPI_Reduce.html)
diff --git a/ompi/mpi/man/man3/MPI_Get_address.md b/ompi/mpi/man/man3/MPI_Get_address.md
deleted file mode 100644
index d85346425fa..00000000000
--- a/ompi/mpi/man/man3/MPI_Get_address.md
+++ /dev/null
@@ -1,84 +0,0 @@
-# Name
-
-`MPI_Get_address` - Gets the address of a location in memory.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Get_address(const void *location, MPI_Aint *address)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_GET_ADDRESS(LOCATION, ADDRESS, IERROR)
-    <type> LOCATION(*)
-    INTEGER(KIND=MPI_ADDRESS_KIND) ADDRESS
-    INTEGER	IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Get_address(location, address, ierror)
-    TYPE(*), DIMENSION(..), ASYNCHRONOUS :: location
-    INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(OUT) :: address
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameters
-
-* `location` : Location in caller memory (choice).
-
-# Output Parameters
-
-* `address` : Address of location (integer).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-
-`MPI_Get_address` returns the byte `address` of a location in memory.
-
-Example: Using `MPI_Get_address` for an array.
-
-```fortran
-EAL A(100,100)
-    INTEGER I1, I2, DIFF
-    CALL MPI_GET_ADDRESS(A(1,1), I1, IERROR)
-    CALL MPI_GET_ADDRESS(A(10,10), I2, IERROR)
-    DIFF = I2 - I1
-! The value of DIFF is 909*sizeofreal; the values of I1 and I2 are
-! implementation dependent.
-```
-
-# Notes
-
-Current Fortran MPI codes will run unmodified and will port to any
-system. However, they may fail if `addresses` larger than 2^32 - 1 are
-used in the program. New codes should be written so that they use the
-new functions. This provides compatibility with C and avoids errors on
-64-bit architectures. However, such newly written codes may need to be
-(slightly) rewritten to port to old Fortran 77 environments that do not
-support KIND declarations.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
diff --git a/ompi/mpi/man/man3/MPI_Get_count.md b/ompi/mpi/man/man3/MPI_Get_count.md
deleted file mode 100644
index 6c5123848e0..00000000000
--- a/ompi/mpi/man/man3/MPI_Get_count.md
+++ /dev/null
@@ -1,87 +0,0 @@
-# Name
-
-`MPI_Get_count`  - Gets the number of top-level elements received.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Get_count(const MPI_Status *status, MPI_Datatype datatype,
-    int *count)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_GET_COUNT(STATUS, DATATYPE, COUNT, IERROR)
-    INTEGER	STATUS(MPI_STATUS_SIZE), DATATYPE, COUNT, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Get_count(status, datatype, count, ierror)
-    TYPE(MPI_Status), INTENT(IN) :: status
-    TYPE(MPI_Datatype), INTENT(IN) :: datatype
-    INTEGER, INTENT(OUT) :: count
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameters
-
-* `status` : Return status of receive operation (status).
-* `datatype` : Datatype of each receive buffer element (handle).
-
-# Output Parameters
-
-* `count` : Number of received elements (integer).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-
-Returns the number of entries received. (We count entries, each of type
-`datatype`, not bytes.) The `datatype` argument should match the argument
-provided by the receive call that set the `status` variable. (As explained
-in Section 3.12.5 in the MPI-1 Standard, "Use of General Datatypes in
-Communication," `MPI_Get_count` may, in certain situations, return the
-value `MPI_UNDEFINED`.)
-
-The `datatype` argument is passed to `MPI_Get_count` to improve performance.
-A message might be received without counting the number of elements it
-contains, and the `count` value is often not needed. Also, this allows the
-same function to be used after a call to `MPI_Probe`.
-
-# Notes
-
-If the size of the `datatype` is zero, this routine will return a `count` of
-zero. If the amount of data in `status` is not an exact multiple of the
-size of `datatype` (so that `count` would not be integral), a `count` of
-`MPI_UNDEFINED` is returned instead.
-
-# Errors
-
-If the value to be returned is larger than can fit into the `count`
-parameter, an `MPI_ERR_TRUNCATE` error is raised.
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-# See Also
-
-[`MPI_Get_elements`(3)](MPI_Get_elements.html)
diff --git a/ompi/mpi/man/man3/MPI_Get_elements.md b/ompi/mpi/man/man3/MPI_Get_elements.md
deleted file mode 100644
index 7519d11adec..00000000000
--- a/ompi/mpi/man/man3/MPI_Get_elements.md
+++ /dev/null
@@ -1,132 +0,0 @@
-# Name
-
-`MPI_Get_elements`, `MPI_Get_elements_x` - Returns the number of basic
-elements in a data type.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Get_elements(const MPI_Status *status, MPI_Datatype datatype,
-    int *count)
-
-int MPI_Get_elements_x(const MPI_Status *status, MPI_Datatype datatype,
-    MPI_Count *count)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_GET_ELEMENTS(STATUS, DATATYPE, COUNT, IERROR)
-    INTEGER	STATUS(MPI_STATUS_SIZE), DATATYPE, COUNT, IERROR
-
-MPI_GET_ELEMENTS_X(STATUS, DATATYPE, COUNT, IERROR)
-    INTEGER	STATUS(MPI_STATUS_SIZE), DATATYPE
-        INTEGER(KIND=MPI_COUNT_KIND) COUNT
-        INTEGER IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Get_elements(status, datatype, count, ierror)
-    TYPE(MPI_Status), INTENT(IN) :: status
-    TYPE(MPI_Datatype), INTENT(IN) :: datatype
-    INTEGER, INTENT(OUT) :: count
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-
-MPI_Get_elements_x(status, datatype, count, ierror)
-    TYPE(MPI_Status), INTENT(IN) :: status
-    TYPE(MPI_Datatype), INTENT(IN) :: datatype
-    INTEGER(KIND = MPI_COUNT_KIND), INTENT(OUT) :: count
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameters
-
-* `status` : Return status of receive operation (status).
-* `datatype` : Datatype used by receive operation (handle).
-
-# Output Parameters
-
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-`MPI_Get_elements` and `MPI_Get_elements_x` behave different from
-`MPI_Get_count`, which returns the number of "top-level entries"
-received, i.e., the number of "copies" of type `datatype`. `MPI_Get_count`
-may return any integer value k, where 0 =< k =< count. If
-`MPI_Get_count` returns k, then the number of basic elements received (and
-the value returned by `MPI_Get_elements` and `MPI_Get_elements_x`) is n
-k, where n is the number of basic elements in the type map of `datatype`.
-If the number of basic elements received is not a multiple of n, that
-is, if the receive operation has not received an integral number of
-`datatype` "copies," then `MPI_Get_count` returns the value `MPI_UNDEFINED.`
-For both functions, if the count parameter cannot express the value to
-be returned (e.g., if the parameter is too small to hold the output
-value), it is set to `MPI_UNDEFINED`.
-
-Example: Usage of `MPI_Get_count` and `MPI_Get_element`:
-
-```fortran
-//...
-MPI_TYPE_CONTIGUOUS(2, MPI_REAL, Type2, ierr)
-MPI_TYPE_COMMIT(Type2, ierr)
-//      ...
-MPI_COMM_RANK(comm, rank, ierr)
-IF(rank.EQ.0) THEN
-    CALL MPI_SEND(a, 2, MPI_REAL, 1, 0, comm, ierr)
-    CALL MPI_SEND(a, 3, MPI_REAL, 1, 0, comm, ierr)
-ELSE
-    CALL MPI_RECV(a, 2, Type2, 0, 0, comm, stat, ierr)
-    CALL MPI_GET_COUNT(stat, Type2, i, ierr)     ! returns i=1
-    CALL MPI_GET_ELEMENTS(stat, Type2, i, ierr)  ! returns i=2
-    CALL MPI_RECV(a, 2, Type2, 0, 0, comm, stat, ierr)
-    CALL MPI_GET_COUNT(stat, Type2, i, ierr) ! returns i=MPI_UNDEFINED
-
-    CALL MPI_GET_ELEMENTS(stat, Type2, i, ierr)  ! returns i=3
-END IF
-```
-
-The function `MPI_Get_elements` can also be used after a probe to find the
-number of elements in the probed message. Note that the two functions
-`MPI_Get_count` and `MPI_Get_elements` return the same values when they are
-used with primitive data types.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-# Fortran 77 Notes
-
-The MPI standard prescribes portable Fortran syntax for the COUNT
-argument of `MPI_Get_elements_x` only for Fortran 90. FORTRAN 77 users may
-use the non-portable syntax
-
-```Fortran
-INTEGER*MPI_COUNT_KIND COUNT
-```
-
-where `MPI_COUNT_KIND` is a constant defined in mpif.h and gives the
-length of the declared integer in bytes.
-
-# See Also
-
-[`MPI_Get_count`(3)](MPI_Get_count.html)
diff --git a/ompi/mpi/man/man3/MPI_Get_elements_x.3in b/ompi/mpi/man/man3/MPI_Get_elements_x.3in
deleted file mode 100644
index 55dfa77bded..00000000000
--- a/ompi/mpi/man/man3/MPI_Get_elements_x.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Get_elements.3
diff --git a/ompi/mpi/man/man3/MPI_Get_library_version.md b/ompi/mpi/man/man3/MPI_Get_library_version.md
deleted file mode 100644
index b119d22085e..00000000000
--- a/ompi/mpi/man/man3/MPI_Get_library_version.md
+++ /dev/null
@@ -1,78 +0,0 @@
-# Name
-
-`MPI_Get_library_version` - Returns a string of the current Open MPI
-version
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Get_library_version(char *version, int *resultlen)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_GET_LIBRARY_VERSION(VERSION, RESULTLEN, IERROR)
-    CHARACTER*(*)	NAME
-    INTEGER RESULTLEN, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Get_library_version(version, resulten, ierror)
-    CHARACTER(LEN=MPI_MAX_LIBRARY_VERSION_STRING), INTENT(OUT) :: version
-    INTEGER, INTENT(OUT) :: resultlen
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Output Parameters
-
-* `version` : A string containing the Open MPI version (string).
-* `resultlen` : Length (in characters) of result returned in `version` (integer).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-This routine returns a string representing the `version` of the MPI
-library. The `version` argument is a character string for maximum
-flexibility.
-
-The number of characters actually written is returned in the output
-argument, `resultlen`. In C, a '0' character is additionally stored
-at `version[resultlen]`. The `resultlen` cannot be larger than
-(`MPI_MAX_LIBRARY_VERSION_STRING` - 1). In Fortran, `version` is padded on
-the right with blank characters. The `resultlen` cannot be larger than `MPI_MAX_LIBRARY_VERSION_STRING`.
-
-# Note
-
-The `version` string that is passed must be at least
-`MPI_MAX_LIBRARY_VERSION_STRING` characters long.
-
-`MPI_Get_library_version` is one of the few functions that can be called
-before `MPI_Init` and after `MPI_Finalize.`
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-# See Also
-
-[`MPI_Get_version`(3)](MPI_Get_version.html)
diff --git a/ompi/mpi/man/man3/MPI_Get_processor_name.md b/ompi/mpi/man/man3/MPI_Get_processor_name.md
deleted file mode 100644
index 81cf542683f..00000000000
--- a/ompi/mpi/man/man3/MPI_Get_processor_name.md
+++ /dev/null
@@ -1,71 +0,0 @@
-# Name
-
-`MPI_Get_processor_name`  - Gets the name of the processor.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Get_processor_name(char *name, int *resultlen)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_GET_PROCESSOR_NAME(NAME, RESULTLEN, IERROR)
-    CHARACTER*(*)	NAME
-    INTEGER		RESULTLEN, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Get_processor_name(name, resultlen, ierror)
-    CHARACTER(LEN=MPI_MAX_PROCESSOR_NAME), INTENT(OUT) :: name
-    INTEGER, INTENT(OUT) :: resultlen
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Output Parameters
-
-* `name` : A unique specifier for the actual (as opposed to virtual) node.
-* `resultlen` : Length (in characters) of result returned in name.
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-This routine returns the `name` of the processor on which it was called at
-the moment of the call. The `name` is a character string for maximum
-flexibility. From this value it must be possible to identify a specific
-piece of hardware. The argument `name` must represent storage that is at
-least `MPI_MAX_PROCESSOR_NAME` characters long.
-
-The number of characters actually written is returned in the output
-argument, `resultlen`.
-
-# Notes
-
-The user must provide at least `MPI_MAX_PROCESSOR_NAME` space to write the
-processor `name`; processor `name`s can be this long. The user should
-examine the output argument, `resultlen`, to determine the actual length
-of the `name`.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
diff --git a/ompi/mpi/man/man3/MPI_Get_version.md b/ompi/mpi/man/man3/MPI_Get_version.md
deleted file mode 100644
index f6c68d350c2..00000000000
--- a/ompi/mpi/man/man3/MPI_Get_version.md
+++ /dev/null
@@ -1,62 +0,0 @@
-# Name
-
-`MPI_Get_version` - Returns the version of the standard corresponding
-to the current implementation.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Get_version(int *version, int *subversion)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_GET_VERSION(VERSION, SUBVERSION, IERROR)
-    INTEGER	VERSION, SUBVERSION, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Get_version(version, subversion, ierror)
-    INTEGER, INTENT(OUT) :: version, subversion
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Output Parameters
-
-* `version` : The major version number of the corresponding standard (integer).
-* `subversion` : The minor version number of the corresponding standard (integer).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-Since Open MPI is MPI 3.1 compliant, this function will return a `version`
-value of 3 and a subversion value of 1 for this release.
-
-# Note
-
-`MPI_Get_version` is one of the few functions that can be called before
-`MPI_Init` and after `MPI_Finalize`.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
diff --git a/ompi/mpi/man/man3/MPI_Graph_create.md b/ompi/mpi/man/man3/MPI_Graph_create.md
deleted file mode 100644
index cc6b302e236..00000000000
--- a/ompi/mpi/man/man3/MPI_Graph_create.md
+++ /dev/null
@@ -1,125 +0,0 @@
-# Name
-
-`MPI_Graph_create` - Makes a new communicator to which topology
-information has been attached.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Graph_create(MPI_Comm comm_old, int nnodes, const int index[],
-    const int edges[], int reorder, MPI_Comm *comm_graph)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_GRAPH_CREATE(COMM_OLD, NNODES, INDEX, EDGES, REORDER,
-    	COMM_GRAPH, IERROR)
-    INTEGER	COMM_OLD, NNODES, INDEX(*), EDGES(*)
-    INTEGER	COMM_GRAPH, IERROR
-    LOGICAL   REORDER
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Graph_create(comm_old, nnodes, index, edges, reorder, comm_graph,
-    	ierror)
-    TYPE(MPI_Comm), INTENT(IN) :: comm_old
-    INTEGER, INTENT(IN) :: nnodes, index(nnodes), edges(*)
-    LOGICAL, INTENT(IN) :: reorder
-    TYPE(MPI_Comm), INTENT(OUT) :: comm_graph
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-
-# Input Parameters
-
-* `comm_old` : Input communicator without topology (handle).
-* `nnodes` : Number of nodes in graph (integer).
-* `index` : Array of integers describing node degrees (see below).
-* `edges` : Array of integers describing graph edges (see below).
-* `reorder` : Ranking may be reordered (true) or not (false) (logical).
-
-# Output Parameters
-
-* `comm_graph` : Communicator with graph topology added (handle).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-`MPI_Graph_create` returns a handle to a new communicator to which the
-graph topology information is attached. If `reorder` = false then the rank
-of each process in the new group is identical to its rank in the old
-group. Otherwise, the function may `reorder` the processes. If the size,
-`nnodes`, of the graph is smaller than the size of the group of `comm_old`,
-then some processes are returned `MPI_COMM_NULL`, in analogy to
-`MPI_Cart_create` and `MPI_Comm_split`. The call is erroneous if it
-specifies a graph that is larger than the group size of the input
-communicator.
-
-The three parameters `nnodes`, `index`, and `edges` define the graph
-structure. `nnodes` is the number of nodes of the graph. The nodes are
-numbered from 0 to `nnodes`-1. The ith entry of array `index` stores the
-total number of neighbors of the first i graph nodes. The lists of
-neighbors of nodes 0, 1, ..., `nnodes`-1 are stored in consecutive
-locations in array `edges`. The array `edges` is a flattened representation
-of the edge lists. The total number of entries in `index` is `nnodes` and
-the total number of entries in `edges` is equal to the number of graph
-`edges`.
-
-The definitions of the arguments `nnodes`, `index`, and `edges` are
-illustrated with the following simple example.
-
-Example: Assume there are four processes 0, 1, 2, 3 with the
-following adjacency matrix:
-
-| Process | Neighbors |
-| ------- | --------- |
-|    0	  |   1, 3    |
-|    1	  |	0     |
-|    2	  |	3     |
-|    3	  |   0, 2    |
-
-Then, the input arguments are:
-* `nodes` = 4
-* `index`  = 2, 3, 4, 6
-* `edges`  = 1, 3, 0, 3, 0, 2
-
-Thus, in C, `index[0]` is the degree of `node` zero, and `index[i]` -
-`index[i-1]` is the degree of `node` i, i=1, . . . , nnodes-1; the list of
-neighbors of node zero is stored in `edges[j]`, for 0 <= j <=
-`index[0] - 1` and the list of neighbors of `node` i, i > 0 , is stored
-in `edges[j]`, `index[i-1]` <= j <= `index[i] - 1`.
-
-In Fortran, `index(1)` is the degree of `node` zero, and `index(i+1)` -
-`index(i)` is the degree of `node` i, i=1, . . . , nnodes-1; the list of
-neighbors of `node` zero is stored in `edges(j)`, for 1 <= j <= `index(1)`
-and the list of neighbors of `node` i, i > 0, is stored in `edges(j)`,
-`index(i) + 1` <= j <= `index(i + 1)`.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-# See Also
-
-[`MPI_Graph_get`(3)](MPI_Graph_get.html)
-[`MPI_Graphdims_get`(3)](MPI_Graphdims_get.html)
diff --git a/ompi/mpi/man/man3/MPI_Graph_get.md b/ompi/mpi/man/man3/MPI_Graph_get.md
deleted file mode 100644
index 32b438ce166..00000000000
--- a/ompi/mpi/man/man3/MPI_Graph_get.md
+++ /dev/null
@@ -1,79 +0,0 @@
-# Name
-
-`MPI_Graph_get` - Retrieves graph topology information associated
-with a communicator.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Graph_get(MPI_Comm comm, int maxindex, int maxedges,
-    int index[], int edges[])
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_GRAPH_GET(COMM, MAXINDEX, MAXEDGES, INDEX, EDGES, IERROR)
-    INTEGER	COMM, MAXINDEX, MAXEDGES, INDEX(*)
-    INTEGER	EDGES(*), IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Graph_get(comm, maxindex, maxedges, index, edges, ierror)
-    TYPE(MPI_Comm), INTENT(IN) :: comm
-    INTEGER, INTENT(IN) :: maxindex, maxedges
-    INTEGER, INTENT(OUT) :: index(maxindex), edges(maxedges)
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameters
-
-* `comm` : Communicator with graph structure (handle).
-* `maxindex` : Length of vector index in the calling program (integer).
-* `maxedges` : Length of vector edges in the calling program (integer).
-
-# Output Parameters
-
-* `index` : Array of integers containing the graph structure (for details see
-the definition of `MPI_Graph_create`).
-* `edges` : Array of integers containing the graph structure.
-* `IERROR` : Fortran only: Error status (integer).
-
-
-
-# Description
-
-Functions `MPI_Graphdims_get` and `MPI_Graph_get` retrieve the
-graph-topology information that was associated with a communicator by
-`MPI_Graph_create`.
-
-The information provided by `MPI_Graphdims_get` can be used to dimension
-the vectors `index` and `edges` correctly for a call to `MPI_Graph_get`.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-# See Also
-
-[`MPI_Graph_create`(3)](MPI_Graph_create.html)
-[`MPI_Graphdims_get`(3)](MPI_Graphdims_get.html)
diff --git a/ompi/mpi/man/man3/MPI_Graph_map.md b/ompi/mpi/man/man3/MPI_Graph_map.md
deleted file mode 100644
index 9142825f951..00000000000
--- a/ompi/mpi/man/man3/MPI_Graph_map.md
+++ /dev/null
@@ -1,75 +0,0 @@
-# Name
-
-`MPI_Graph_map`  - Maps process to graph topology information.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Graph_map(MPI_Comm comm, int nnodes, const int index[],
-    const int edges[], int *newrank)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_GRAPH_MAP(COMM, NNODES, INDEX, EDGES, NEWRANK, IERROR)
-    INTEGER	COMM, NNODES, INDEX(*), EDGES(*), NEWRANK, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Graph_map(comm, nnodes, index, edges, newrank, ierror)
-    TYPE(MPI_Comm), INTENT(IN) :: comm
-    INTEGER, INTENT(IN) :: nnodes, index(nnodes), edges(*)
-    INTEGER, INTENT(OUT) :: newrank
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameters
-
-* `comm` : Input communicator (handle).
-* `nnodes` : Number of graph nodes (integer).
-* `index` : Integer array specifying the graph structure, see
-`MPI_Graph_create`.
-* `edges` : Integer array specifying the graph structure.
-
-# Output Parameters
-
-* `newrank` : Reordered rank of the calling process; MPI_UNDEFINED if the calling
-process does not belong to graph (integer).
-* `IERROR` : Fortran only: Error status (integer).
-
-
-
-# Description
-
-`MPI_Cart_map` and `MPI_Graph_map` can be used to implement all other
-topology functions. In general they will not be called by the user
-directly, unless he or she is creating additional virtual topology
-capability other than that provided by MPI.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-# See Also
-
-[`MPI_Cart_map`(3)](MPI_Cart_map.html)
diff --git a/ompi/mpi/man/man3/MPI_Graph_neighbors.3in b/ompi/mpi/man/man3/MPI_Graph_neighbors.3in
deleted file mode 100644
index 43f6f32cd2e..00000000000
--- a/ompi/mpi/man/man3/MPI_Graph_neighbors.3in
+++ /dev/null
@@ -1,106 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Graph_neighbors 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Graph_neighbors \fP \- Returns the neighbors of a node associated with a graph topology.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Graph_neighbors(MPI_Comm \fIcomm\fP, int\fI rank\fP, int\fI maxneighbors\fP,
-	int\fI neighbors\fP[])
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_GRAPH_NEIGHBORS(\fICOMM, RANK, MAXNEIGHBORS, NEIGHBORS, IERROR\fP)
-	INTEGER	\fICOMM, RANK, MAXNEIGHBORS, NEIGHBORS(*), IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Graph_neighbors(\fIcomm\fP, \fIrank\fP, \fImaxneighbors\fP, \fIneighbors\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, INTENT(IN) :: \fIrank\fP, \fImaxneighbors\fP
-	INTEGER, INTENT(OUT) :: \fIneighbors(maxneighbors)\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-comm
-Communicator with graph topology (handle).
-.TP 1i
-rank
-Rank of process in group of comm (integer).
-.TP 1i
-maxneighbors
-Size of array neighbors (integer).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-neighbors
-Ranks of processes that are neighbors to specified process (array of integers).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-\fBExample:\fP  Suppose that comm is a communicator with a shuffle-exchange
-topology. The group has 2n members. Each process is labeled by a(1),\ ..., a(n) with a(i) E{0,1}, and has three neighbors: exchange (a(1),\ ..., a(n) = a(1),\ ..., a(n-1), a(n) (a = 1 - a), shuffle (a(1),\ ..., a(n)) = a(2),\ ..., a(n), a(1), and unshuffle (a(1),\ ..., a(n)) = a(n), a(1),\ ..., a(n-1). The graph adjacency list is illustrated below for n=3.
-.sp
-.nf
-        		exchange		shuffle		unshuffle
-    node		neighbors(1)	neighbors(2)	neighbors(3)
-    0(000)	    1		    0		    0
-    1(001)	    0		    2		    4
-    2(010)	    3		    4		    1
-    3(011)	    2		    6		    5
-    4(100)	    5		    1		    2
-    5(101)	    4		    3		    6
-    6(110)	    7		    5		    3
-    7(111)	    6		    7		    7
-.fi
-.sp
-Suppose that the communicator comm has this topology associated with it. The following code fragment cycles through the three types of neighbors and performs an appropriate permutation for each.
-.sp
-.nf
-C  assume: each process has stored a real number A.
-C  extract neighborhood information
-      CALL MPI_COMM_RANK(comm, myrank, ierr)
-      CALL MPI_GRAPH_NEIGHBORS(comm, myrank, 3, neighbors, ierr)
-C  perform exchange permutation
-      CALL MPI_SENDRECV_REPLACE(A, 1, MPI_REAL, neighbors(1), 0,
-     +     neighbors(1), 0, comm, status, ierr)
-C  perform shuffle permutation
-      CALL MPI_SENDRECV_REPLACE(A, 1, MPI_REAL, neighbors(2), 0,
-     +     neighbors(3), 0, comm, status, ierr)
-C  perform unshuffle permutation
-      CALL MPI_SENDRECV_REPLACE(A, 1, MPI_REAL, neighbors(3), 0,
-     +     neighbors(2), 0, comm, status, ierr)
-
-.fi
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Graph_neighbors_count
-
diff --git a/ompi/mpi/man/man3/MPI_Graph_neighbors_count.md b/ompi/mpi/man/man3/MPI_Graph_neighbors_count.md
deleted file mode 100644
index bf3027c75bc..00000000000
--- a/ompi/mpi/man/man3/MPI_Graph_neighbors_count.md
+++ /dev/null
@@ -1,70 +0,0 @@
-# Name
-
-`MPI_Graph_neighbors_count`  - Returns the number of neighbors of a
-node associated with a graph topology.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Graph_neighbors_count(MPI_Comm comm, int rank,
-    int *nneighbors)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_GRAPH_NEIGHBORS_COUNT(COMM, RANK, NNEIGHBORS, IERROR)
-    INTEGER	COMM, RANK, NNEIGHBORS, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Graph_neighbors_count(comm, rank, nneighbors, ierror)
-    TYPE(MPI_Comm), INTENT(IN) :: comm
-    INTEGER, INTENT(IN) :: rank
-    INTEGER, INTENT(OUT) :: nneighbors
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-
-# Input Parameters
-
-* `comm` : Communicator with graph topology (handle).
-* `rank` : Rank of process in group of comm (integer).
-
-# Output Parameters
-
-* `nneighbors` : Number of neighbors of specified process (integer).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-`MPI_Graph_neighbors_count` and `MPI_Graph_neighbors` provide adjacency
-information for a general, graph topology. `MPI_Graph_neighbors_count`
-returns the number of neighbors for the process signified by `rank`.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-# See Also
-
-[`MPI_Graph_neighbors`(3)](MPI_Graph_neighbors.html)
diff --git a/ompi/mpi/man/man3/MPI_Graphdims_get.md b/ompi/mpi/man/man3/MPI_Graphdims_get.md
deleted file mode 100644
index a830ffeff29..00000000000
--- a/ompi/mpi/man/man3/MPI_Graphdims_get.md
+++ /dev/null
@@ -1,72 +0,0 @@
-# Name
-
-`MPI_Graphdims_get` - Retrieves graph topology information associated
-with a communicator.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Graphdims_get(MPI_Comm comm, int *nnodes, int *nedges)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_GRAPHDIMS_GET(COMM, NNODES, NEDGES, IERROR)
-    INTEGER	COMM, NNODES, NEDGES, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Graphdims_get(comm, nnodes, nedges, ierror)
-    TYPE(MPI_Comm), INTENT(IN) :: comm
-    INTEGER, INTENT(OUT) :: nnodes, nedges
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-
-# Input Parameter
-
-* `comm` : Communicator for group with graph structure (handle).
-
-# Output Parameters
-
-* `nnodes` : Number of nodes in graph (integer).
-* `nedges` : Number of edges in graph (integer).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-Functions `MPI_Graphdims_get` and `MPI_Graph_get` retrieve the
-graph-topology information that was associated with a communicator by
-`MPI_Graph_create`.
-
-The information provided by `MPI_Graphdims_get` can be used to dimension
-the vectors index and edges correctly for a call to `MPI_Graph_get`.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-# See Also
-
-[`MPI_Graph_create`(3)](MPI_Graph_create.html)
-[`MPI_Graph_get`(3)](MPI_Graph_get.html)
diff --git a/ompi/mpi/man/man3/MPI_Grequest_complete.md b/ompi/mpi/man/man3/MPI_Grequest_complete.md
deleted file mode 100644
index 96160e49e34..00000000000
--- a/ompi/mpi/man/man3/MPI_Grequest_complete.md
+++ /dev/null
@@ -1,76 +0,0 @@
-# Name
-
-`MPI_Grequest_complete` - Reports that a generalized request is
-complete.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Grequest_complete(MPI_Request request)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_GREQUEST_COMPLETE(REQUEST, IERROR)
-    INTEGER	REQUEST, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Grequest_complete(request, ierror)
-    TYPE(MPI_Request), INTENT(IN) :: request
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input/Output Parameter
-
-* `request` : Generalized request (handle).
-
-# Output Parameter
-
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-`MPI_Grequest_complete` informs MPI that the operations represented by the
-generalized request `request` are complete. A call to `MPI_Wait(request, status)`
-will return, and a call to `MPI_Test(request, flag, status)` will return
-flag=true only after a call to `MPI_Grequest_complete` has
-declared that these operations are complete.
-
-MPI imposes no restrictions on the code executed by the callback
-functions. However, new nonblocking operations should be defined so that
-the general semantic rules about MPI calls such as `MPI_Test`,
-`MPI_Request_free`, or `MPI_Cancel` still hold. For example, all these calls
-are supposed to be local and nonblocking. Therefore, the callback
-functions `query_fn`, `free_fn`, or `cancel_fn` should invoke blocking
-MPI communication calls only if the context is such that these calls are
-guaranteed to return in finite time. Once `MPI_Cancel` has been invoked,
-the canceled operation should complete in finite time, regardless of the
-state of other processes (the operation has acquired "local"
-semantics). It should either succeed or fail without side-effects. The
-user should guarantee these same properties for newly defined
-operations.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
diff --git a/ompi/mpi/man/man3/MPI_Grequest_start.md b/ompi/mpi/man/man3/MPI_Grequest_start.md
deleted file mode 100644
index 2b85cd74ca7..00000000000
--- a/ompi/mpi/man/man3/MPI_Grequest_start.md
+++ /dev/null
@@ -1,201 +0,0 @@
-# Name
-
-`MPI_Grequest_start`  - Starts a generalized request and returns a
-handle to it in `request`.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Grequest_start(MPI_Grequest_query_function *query_fn,
-	MPI_Grequest_free_function *free_fn,
-	MPI_Grequest_cancel_function *cancel_fn, void *extra_state,
-	MPI_Request *request)
-```
-
-## Fortran Syntax (See Fortran 77 Notes)
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_GREQUEST_START(QUERY_FN, FREE_FN, CANCEL_FN, EXTRA_STATE,
-    REQUEST, IERROR)
-    INTEGER	REQUEST, IERROR
-    EXTERNAL QUERY_FN, FREE_FN, CANCEL_FN
-	  INTEGER(KIND=MPI_ADDRESS_KIND) EXTRA_STATE
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Grequest_start(query_fn, free_fn, cancel_fn, extra_state, request,
-    	ierror)
-    PROCEDURE(MPI_Grequest_query_function) :: query_fn
-    PROCEDURE(MPI_Grequest_free_function) :: free_fn
-    PROCEDURE(MPI_Grequest_cancel_function) :: cancel_fn
-    INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) :: extra_state
-    TYPE(MPI_Request), INTENT(OUT) :: request
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameters
-
-* `query_fn` : Callback function invoked when request status is queried (function).
-* `free_fn` : Callback function invoked when request is freed (function).
-* `cancel_fn` : Callback function invoked when request is canceled (function).
-* `extra_state` : Extra state.
-
-# Output Parameters
-
-* `request` : Generalized request (handle).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-`MPI_Grequest_start` starts a generalized `request` and returns a handle to
-it in `request`.
-
-The syntax and meaning of the callback functions are listed below. All
-callback functions are passed the `extra_state` argument that was
-associated with the `request` by the starting call `MPI_Grequest_start`.
-This can be used to maintain user-defined state for the `request`. In C,
-the query function is
-
-```c
-typedef int MPI_Grequest_query_function(void *extra_state,
-    MPI_Status *status);
-```
-
-In Fortran, it is
-
-```fortran
-SUBROUTINE GREQUEST_QUERY_FUNCTION(EXTRA_STATE, STATUS, IERROR)
-    INTEGER STATUS(MPI_STATUS_SIZE), IERROR
-    INTEGER(KIND=MPI_ADDRESS_KIND) EXTRA_STATE
-```
-
-The `query_fn` function computes the status that should be returned for
-the generalized request. The status also includes information about
-successful/unsuccessful cancellation of the request (result to be
-returned by `MPI_Test_cancelled`).
-
-The `query_fn` function is invoked by the
-`MPI_{Wait|Test}{any|some|all}` call that completed the generalized
-request associated with this callback. The callback function is also
-invoked by calls to `MPI_Request_get_status` if the request is complete
-when the call occurs. In both cases, the callback is passed a reference
-to the corresponding status variable passed by the user to the MPI call.
-If the user provided `MPI_STATUS_IGNORE` or `MPI_STATUSES_IGNORE` to the MPI
-function that causes `query_fn` to be called, then MPI will pass a valid
-status object to `query_fn`, and this status will be ignored upon return
-of the callback function. Note that `query_fn` is invoked only after
-`MPI_Grequest_complete` is called on the request; it may be invoked
-several times for the same generalized request. Note also that a call to
-`MPI_{Wait|Test}{some|all}` may cause multiple invocations of
-`query_fn` callback functions, one for each generalized request that is
-completed by the MPI call. The order of these invocations is not
-specified by MPI.
-
-In C, the free function is
-
-```c
-typedef int MPI_Grequest_free_function(void *extra_state);
-```
-
-And in Fortran, it is
-
-```fortran
-SUBROUTINE GREQUEST_FREE_FUNCTION(EXTRA_STATE, IERROR)
-    INTEGER IERROR
-    INTEGER(KIND=MPI_ADDRESS_KIND) EXTRA_STATE
-```
-
-The `free_fn` callback function is invoked to clean up user-allocated
-resources when the generalized request is freed.
-
-The `free_fn` function is invoked by the
-`MPI_{Wait|Test}{any|some|all}` call that completed the generalized
-request associated with this callback. `free_fn` is invoked after the
-call to `query_fn` for the same request. However, if the MPI call
-completed multiple generalized requests, the order in which `free_fn`
-callback functions are invoked is not specified by MPI.
-
-The `free_fn` callback is also invoked for generalized requests that are
-freed by a call to `MPI_Request_free` (no call to
-`MPI_{Wait|Test}{any|some|all}` will occur for such a request). In
-this case, the callback function will be called either in the MPI call
-`MPI_Request_free(request)` or in the MPI call
-`MPI_Grequest_complete(request)`, whichever happens last. In other words,
-in this case the actual freeing code is executed as soon as both calls
-(`MPI_Request_free` and `MPI_Grequest_complete`) have occurred. The
-`request` is not deallocated until after `free_fn` completes. Note that
-`free_fn` will be invoked only once per request by a correct program.
-
-In C, the cancel function is
-
-```c
-typedef int MPI_Grequest_cancel_function(void *extra_state, int complete);
-```
-
-In Fortran, the cancel function is
-
-```Fortran
-SUBROUTINE GREQUEST_CANCEL_FUNCTION(EXTRA_STATE, COMPLETE, IERROR)
-     INTEGER IERROR
-     INTEGER(KIND=MPI_ADDRESS_KIND) EXTRA_STATE
-     LOGICAL COMPLETE
-```
-
-The `cancel_fn` function is invoked to start the cancellation of a
-generalized request. It is called by `MPI_Request_cancel(request)`. MPI
-passes to the callback function complete=true if `MPI_Grequest_complete`
-has already been called on the request, and complete=false otherwise.
-
-# Fortran 77 Notes
-
-The MPI standard prescribes portable Fortran syntax for the
-`EXTRA_STATE` argument only for Fortran 90. FORTRAN 77 users may use the
-non-portable syntax
-
-```fortran
-INTEGER*MPI_ADDRESS_KIND EXTRA_STATE
-```
-
-where `MPI_ADDRESS_KIND` is a constant defined in mpif.h and gives the
-length of the declared integer in bytes.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-All callback functions return an error code. The code is passed back and
-dealt with as appropriate for the error code by the MPI function that
-invoked the callback function. For example, if error codes are returned,
-then the error code returned by the callback function will be returned
-by the MPI function that invoked the callback function. In the case of a
-`MPI_{Wait|Test}any` call that invokes both `query_fn` and `free_fn`,
-the MPI call will return the error code returned by the last callback,
-namely `free_fn`. If one or more of the `request`s in a call to
-`MPI_{Wait|Test}{some|all`} has failed, then the MPI call will return
-`MPI_ERR_IN_STATUS`. In such a case, if the MPI call was passed an array
-of statuses, then MPI will return in each of the statuses that
-correspond to a completed generalized `request` the error code returned by
-the corresponding invocation of its `free_fn` callback function.
-However, if the MPI function was passed `MPI_STATUSES_IGNORE`, then the
-individual error codes returned by each callback function will be lost.
-
-See the MPI man page for a full list of MPI error codes.
diff --git a/ompi/mpi/man/man3/MPI_Group_c2f.3in b/ompi/mpi/man/man3/MPI_Group_c2f.3in
deleted file mode 100644
index a13fce697dd..00000000000
--- a/ompi/mpi/man/man3/MPI_Group_c2f.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Comm_f2c.3
diff --git a/ompi/mpi/man/man3/MPI_Group_compare.md b/ompi/mpi/man/man3/MPI_Group_compare.md
deleted file mode 100644
index e8fa84e34f3..00000000000
--- a/ompi/mpi/man/man3/MPI_Group_compare.md
+++ /dev/null
@@ -1,66 +0,0 @@
-# Name
-
-`MPI_Group_compare`  - Compares two groups.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Group_compare(MPI_Group group1, MPI_Group group2,
-    int *result)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_GROUP_COMPARE(GROUP1, GROUP2, RESULT, IERROR)
-    INTEGER	GROUP1, GROUP2, RESULT, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Group_compare(group1, group2, result, ierror)
-    TYPE(MPI_Group), INTENT(IN) :: group1, group2
-    INTEGER, INTENT(OUT) :: result
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameters
-
-* `group1` : First group (handle).
-* `group2` : Second group (handle).
-
-# Output Parameters
-
-* `result` : Integer which is MPI_IDENT if the order and members of the two
-groups are the same, MPI_SIMILAR if only the members are the same,
-and MPI_UNEQUAL otherwise.
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-`MPI_IDENT` results if the group members and group order is exactly the
-same in both groups. This happens for instance if `group1` and `group2` are
-the same handle. `MPI_SIMILAR` results if the group members are the same
-but the order is different. `MPI_UNEQUAL` results otherwise.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
diff --git a/ompi/mpi/man/man3/MPI_Group_difference.md b/ompi/mpi/man/man3/MPI_Group_difference.md
deleted file mode 100644
index 592e0dea4d3..00000000000
--- a/ompi/mpi/man/man3/MPI_Group_difference.md
+++ /dev/null
@@ -1,79 +0,0 @@
-# Name
-
-`MPI_Group_difference`  - Makes a group from the difference of two
-groups.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Group_difference(MPI_Group group1, MPI_Group group2,
-	MPI_Group *newgroup)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_GROUP_DIFFERENCE(GROUP1, GROUP2, NEWGROUP, IERROR)
-    INTEGER	GROUP1, GROUP2, NEWGROUP, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Group_difference(group1, group2, newgroup, ierror)
-    TYPE(MPI_Group), INTENT(IN) :: group1, group2
-    TYPE(MPI_Group), INTENT(OUT) :: newgroup
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameters
-
-* `group1` : First group (handle).
-* `group2` : Second group (handle).
-
-# Output Parameters
-
-* `newgroup` : Difference group (handle).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-The set-like operations are defined as follows:
-* `union` -- All elements of the first group (`group1`), followed by all
-elements of second group (`group2`) that are not in the first group
-* `intersect` -- all elements of the first group that are also in the
-second group, ordered as in first group
-* `difference` -- all elements of the first group that are not in the
-second group, ordered as in the first group
-
-Note that for these operations the order of processes in the output
-group is determined primarily by order in the first group (if possible)
-and then, if necessary, by order in the second group. Neither union nor
-intersection are commutative, but both are associative.
-
-The new group can be empty, that is, equal to `MPI_GROUP_EMPTY`.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-# See Also
-
-[`MPI_Group_free`(3)](MPI_Group_free.html)
diff --git a/ompi/mpi/man/man3/MPI_Group_excl.md b/ompi/mpi/man/man3/MPI_Group_excl.md
deleted file mode 100644
index 7b489c9abdc..00000000000
--- a/ompi/mpi/man/man3/MPI_Group_excl.md
+++ /dev/null
@@ -1,81 +0,0 @@
-# Name
-
-`MPI_Group_excl` - Produces a group by reordering an existing group
-and taking only unlisted members.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Group_excl(MPI_Group group, int n, const int ranks[],
-	MPI_Group *newgroup)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_GROUP_EXCL(GROUP, N, RANKS, NEWGROUP, IERROR)
-    INTEGER	GROUP, N, RANKS(*), NEWGROUP, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Group_excl(group, n, ranks, newgroup, ierror)
-    TYPE(MPI_Group), INTENT(IN) :: group
-    INTEGER, INTENT(IN) :: n, ranks(n)
-    TYPE(MPI_Group), INTENT(OUT) :: newgroup
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameters
-
-* `group` : Group (handle).
-* `n` : Number of elements in array ranks (integer).
-* `ranks` : Array of integer ranks in group not to appear in newgroup.
-
-# Output Parameters
-
-* `newgroup` : New group derived from above, preserving the order defined by group
-(handle).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-The function `MPI_Group_excl` creates a `group` of processes `newgroup` that
-is obtained by deleting from `group` those processes with ranks
-`ranks[0]`, ... `ranks[n-1]`. The ordering of processes in `newgroup` is
-identical to the ordering in `group`. Each of the n elements of ranks must
-be a valid rank in `group` and all elements must be distinct; otherwise,
-the call is erroneous. If `n` = 0, then `newgroup` is identical to `group`.
-
-# Note
-
-Currently, each of the ranks to exclude must be a valid rank in the
-`group` and all elements must be distinct or the function is erroneous.
-This restriction is per the draft.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-# See Also
-
-[`MPI_Group_range_excl`(3)](MPI_Group_range_excl.html)
-[`MPI_Group_free`(3)](MPI_Group_free.html)
diff --git a/ompi/mpi/man/man3/MPI_Group_f2c.3in b/ompi/mpi/man/man3/MPI_Group_f2c.3in
deleted file mode 100644
index a13fce697dd..00000000000
--- a/ompi/mpi/man/man3/MPI_Group_f2c.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Comm_f2c.3
diff --git a/ompi/mpi/man/man3/MPI_Group_free.md b/ompi/mpi/man/man3/MPI_Group_free.md
deleted file mode 100644
index 387f8e24a0a..00000000000
--- a/ompi/mpi/man/man3/MPI_Group_free.md
+++ /dev/null
@@ -1,63 +0,0 @@
-# Name
-
-`MPI_Group_free`  - Frees a group.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Group_free(MPI_Group *group)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_GROUP_FREE(GROUP, IERROR)
-    INTEGER	GROUP, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Group_free(group, ierror)
-    TYPE(MPI_Group), INTENT(INOUT) :: group
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input/Output Parameter
-
-* `group` : Group (handle).
-
-# Output Parameter
-
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-This operation marks a `group` object for deallocation. The handle `group`
-is set to `MPI_GROUP_NULL` by the call. Any ongoing operation using this
-`group` will complete normally.
-
-# Note
-
-On return, `group` is set to `MPI_GROUP_NULL`.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
diff --git a/ompi/mpi/man/man3/MPI_Group_incl.md b/ompi/mpi/man/man3/MPI_Group_incl.md
deleted file mode 100644
index c755683c289..00000000000
--- a/ompi/mpi/man/man3/MPI_Group_incl.md
+++ /dev/null
@@ -1,83 +0,0 @@
-# Name
-
-`MPI_Group_incl`  - Produces a group by reordering an existing group
-and taking only listed members.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Group_incl(MPI_Group group, int n, const int ranks[],
-	MPI_Group *newgroup)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_GROUP_INCL(GROUP, N, RANKS, NEWGROUP, IERROR)
-    INTEGER	GROUP, N, RANKS(*), NEWGROUP, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Group_incl(group, n, ranks, newgroup, ierror)
-    TYPE(MPI_Group), INTENT(IN) :: group
-    INTEGER, INTENT(IN) :: n, ranks(n)
-    TYPE(MPI_Group), INTENT(OUT) :: newgroup
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameters
-
-* `group` : Group (handle).
-* `n` : Number of elements in array ranks (and size of `newgroup`)(integer).
-* `ranks` : Ranks of processes in group to appear in newgroup (array of
-integers).
-
-# Output Parameters
-
-* `newgroup` : New group derived from above, in the order defined by ranks
-(handle).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-The function `MPI_Group_incl` creates a group `group_out` that consists of
-the n processes in `group` with ranks `rank[0]`, ..., `rank[n-1]`; the
-process with rank i in `group_out` is the process with rank `ranks[i]` in
-`group`. Each of the n elements of ranks must be a valid rank in `group` and
-all elements must be distinct, or else the program is erroneous. If `n` =
-0, then `group_out` is `MPI_GROUP_EMPTY`. This function can, for instance,
-be used to reorder the elements of a `group`.
-
-# Note
-
-This implementation does not currently check to ensure that there are no
-duplicates in the list of ranks.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-# See Also
-
-[`MPI_Group_compare`(3)](MPI_Group_compare.html)
-[`MPI_Group_range_incl`(3)](MPI_Group_range_incl.html)
-[`MPI_Group_free`(3)](MPI_Group_free.html)
diff --git a/ompi/mpi/man/man3/MPI_Group_intersection.md b/ompi/mpi/man/man3/MPI_Group_intersection.md
deleted file mode 100644
index b01767f6922..00000000000
--- a/ompi/mpi/man/man3/MPI_Group_intersection.md
+++ /dev/null
@@ -1,79 +0,0 @@
-# Name
-
-`MPI_Group_intersection`  - Produces a group at the intersection of
-two existing groups.
-
-# Syntax
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Group_intersection(MPI_Group group1, MPI_Group group2,
-	MPI_Group *newgroup)
-```
-
-## Fortran Syntax
-
-```fortran
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-
-MPI_GROUP_INTERSECTION(GROUP1, GROUP2, NEWGROUP, IERROR)
-    INTEGER	GROUP1, GROUP2, NEWGROUP, IERROR
-```
-
-## Fortran 2008 Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Group_intersection(group1, group2, newgroup, ierror)
-    TYPE(MPI_Group), INTENT(IN) :: group1, group2
-    TYPE(MPI_Group), INTENT(OUT) :: newgroup
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# Input Parameters
-
-* `group1` : First group (handle).
-* `group2` : Second group (handle).
-
-# Output Parameters
-
-* `newgroup` : Intersection group (handle).
-* `IERROR` : Fortran only: Error status (integer).
-
-# Description
-
-The set-like operations are defined as follows:
-* `union` -- All elements of the first group (`group1`), followed by all
-elements of second group (`group2`) not in first.
-* `intersect` -- all elements of the first group that are also in the
-second group, ordered as in first group.
-* `difference` -- all elements of the first group that are not in the
-second group, ordered as in the first group.
-
-Note that for these operations the order of processes in the output
-group is determined primarily by order in the first group (if possible)
-and then, if necessary, by order in the second group. Neither union nor
-intersection are commutative, but both are associative.
-
-The new group can be empty, that is, equal to `MPI_GROUP_EMPTY`.
-
-# Errors
-
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-`MPI_Comm_set_errhandler`; the predefined error handler `MPI_ERRORS_RETURN`
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-# See Also
-
-[`MPI_Group_free`(3)](MPI_Group_free.html)
diff --git a/ompi/mpi/man/man3/MPI_Group_range_excl.3in b/ompi/mpi/man/man3/MPI_Group_range_excl.3in
deleted file mode 100644
index 65934e234a4..00000000000
--- a/ompi/mpi/man/man3/MPI_Group_range_excl.3in
+++ /dev/null
@@ -1,80 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Group_range_excl 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Group_range_excl\fP \- Produces a group by excluding ranges of processes from an existing group.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Group_range_excl(MPI_Group \fIgroup\fP, int\fI n\fP, int\fI ranges\fP[][3],
-	MPI_Group\fI *newgroup\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_GROUP_RANGE_EXCL(\fIGROUP, N, RANGES, NEWGROUP, IERROR\fP)
-	INTEGER	\fIGROUP, N, RANGES(3,*), NEWGROUP, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Group_range_excl(\fIgroup\fP, \fIn\fP, \fIranges\fP, \fInewgroup\fP, \fIierror\fP)
-	TYPE(MPI_Group), INTENT(IN) :: \fIgroup\fP
-	INTEGER, INTENT(IN) :: \fIn\fP, \fIranges(3,n)\fP
-	TYPE(MPI_Group), INTENT(OUT) :: \fInewgroup\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-group
-Group (handle).
-.TP 1i
-n
-Number of triplets in array ranges (integer).
-.TP 1i
-ranges
-A one-dimensional array of integer triplets of the form (first rank, last rank, stride), indicating the ranks in group of processes to be excluded from the output group newgroup.
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-newgroup
-New group derived from above, preserving the order in group (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Each computed rank must be a valid rank in group and all computed ranks must be distinct, or else the program is erroneous.
-.sp
-The functionality of this routine is specified to be equivalent to
-expanding the array of ranges to an array of the excluded ranks and passing the resulting array of ranks and other arguments to MPI_Group_excl. A call to MPI_Group_excl is equivalent to a call to MPI_Group_range_excl with each rank i in ranks replaced by the triplet (i,i,1) in the argument ranges.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Group_excl
-.br
-MPI_Group_free
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Group_range_incl.3in b/ompi/mpi/man/man3/MPI_Group_range_incl.3in
deleted file mode 100644
index c53ed850227..00000000000
--- a/ompi/mpi/man/man3/MPI_Group_range_incl.3in
+++ /dev/null
@@ -1,101 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Group_range_incl 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Group_range_incl\fP \- Creates a new group from ranges of ranks in an existing group.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Group_range_incl(MPI_Group \fIgroup\fP, int\fI n\fP, int\fI ranges\fP[][3],
-	MPI_Group\fI *newgroup\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_GROUP_RANGE_INCL(\fIGROUP, N, RANGES, NEWGROUP, IERROR\fP)
-	INTEGER	\fIGROUP, N, RANGES(3,*), NEWGROUP, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Group_range_incl(\fIgroup\fP, \fIn\fP, \fIranges\fP, \fInewgroup\fP, \fIierror\fP)
-	TYPE(MPI_Group), INTENT(IN) :: \fIgroup\fP
-	INTEGER, INTENT(IN) :: \fIn\fP, \fIranges(3,n)\fP
-	TYPE(MPI_Group), INTENT(OUT) :: \fInewgroup\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-group
-Group (handle).
-.TP 1i
-n
-Number of triplets in array ranges (integer).
-.TP 1i
-ranges
-A one-dimensional array of integer triplets, of the form (first rank, last rank, stride) indicating ranks in group or processes to be included in newgroup.
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-newgroup
-New group derived from above, in the order defined by ranges (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-If ranges consist of the triplets
-.sp
-.nf
-    (first1, last1, stride1),\ ..., (firstn, lastn, striden)
-.fi
-.sp
-then newgroup consists of the sequence of processes in group with ranks
-.sp
-.nf
-                                                 last(1)-first(1)
-  first(1), first(1) + stride(1),..., first(1) + ---------------- stride(1),...
-                                                     stride(1)
-
-                                                 last(n)-first(n)
-  first(n), first(n) + stride(n),..., first(n) + ---------------- stride(n).
-                                                     stride(n)
-.fi
-.sp
-Each computed rank must be a valid rank in group and all computed ranks must be distinct, or else the program is erroneous. Note that we may have first(i) > last(i), and stride(i) may be negative, but cannot be zero.
-.sp
-The functionality of this routine is specified to be equivalent to expanding the array of ranges to an array of the included ranks and passing the resulting array of ranks and other arguments to MPI_Group_incl. A call to MPI_Group_incl is equivalent to a call to MPI_Group_range_incl with each rank i in ranks replaced by the triplet (i,i,1) in the argument ranges.
-
-.SH NOTE
-.ft R
-This implementation does not currently check to see that the list of ranges to include are valid ranks in the group.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Group_incl
-.br
-MPI_Group_free
-
-
diff --git a/ompi/mpi/man/man3/MPI_Group_rank.3in b/ompi/mpi/man/man3/MPI_Group_rank.3in
deleted file mode 100644
index 17952e25c9b..00000000000
--- a/ompi/mpi/man/man3/MPI_Group_rank.3in
+++ /dev/null
@@ -1,61 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Group_rank 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Group_rank\fP \- Returns the rank of the calling process in the given group.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Group_rank(MPI_Group \fIgroup\fP, int *\fIrank\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_GROUP_RANK(\fIGROUP, RANK, IERROR\fP)
-	INTEGER	\fIGROUP, RANK, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Group_rank(\fIgroup\fP, \fIrank\fP, \fIierror\fP)
-	TYPE(MPI_Group), INTENT(IN) :: \fIgroup\fP
-	INTEGER, INTENT(OUT) :: \fIrank\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-group
-Group (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-rank
-Rank of the calling process in group, or MPI_UNDEFINED if the process is not a member (integer).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Group_rank returns as the output parameter \fIrank\fP the rank of the calling process in group. If the process is not a member of group then MPI_UNDEFINED is returned.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_Group_size.3in b/ompi/mpi/man/man3/MPI_Group_size.3in
deleted file mode 100644
index 04ba8c002d8..00000000000
--- a/ompi/mpi/man/man3/MPI_Group_size.3in
+++ /dev/null
@@ -1,61 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Group_size 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Group_size\fP \- Returns the size of a group.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Group_size(MPI_Group \fIgroup\fP, int \fI*size\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_GROUP_SIZE(\fIGROUP, SIZE, IERROR\fP)
-	INTEGER	\fIGROUP, SIZE, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Group_size(\fIgroup\fP, \fIsize\fP, \fIierror\fP)
-	TYPE(MPI_Group), INTENT(IN) :: \fIgroup\fP
-	INTEGER, INTENT(OUT) :: \fIsize\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-group
-Group (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-size
-Number of processes in the group (integer).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Group_size returns in \fIsize\fP the number of processes in the group. Thus, if group = MPI_GROUP_EMPTY, then the call will return size = 0. On the other hand, a call with group = MPI_GROUP_NULL is erroneous.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_Group_translate_ranks.3in b/ompi/mpi/man/man3/MPI_Group_translate_ranks.3in
deleted file mode 100644
index 5eac6382810..00000000000
--- a/ompi/mpi/man/man3/MPI_Group_translate_ranks.3in
+++ /dev/null
@@ -1,74 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Group_translate_ranks 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Group_translate_ranks\fP \- Translates the ranks of processes in one group to those in another group.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Group_translate_ranks(MPI_Group \fIgroup1\fP, int\fI n\fP,
-	const int\fI ranks1\fP[], MPI_Group\fI group2\fP, int\fI ranks2\fP[])
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_GROUP_TRANSLATE_RANKS(\fIGROUP1, N, RANKS1, GROUP2, RANKS2,
-		IERROR\fP)
-	INTEGER	\fIGROUP1, N, RANKS1(*), GROUP2, RANKS2(*), IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Group_translate_ranks(\fIgroup1\fP, \fIn\fP, \fIranks1\fP, \fIgroup2\fP, \fIranks2\fP, \fIierror\fP)
-	TYPE(MPI_Group), INTENT(IN) :: \fIgroup1\fP, \fIgroup2\fP
-	INTEGER, INTENT(IN) :: \fIn\fP, \fIranks1(n)\fP
-	INTEGER, INTENT(OUT) :: \fIranks2(n)\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-group1
-First group (handle).
-.TP 1i
-n
-Number of ranks in ranks1 and ranks2 arrays (integer).
-.TP 1i
-ranks1
-Array of zero or more valid ranks in group1.
-.TP 1i
-group2
-Second group (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-ranks2
-Array of corresponding ranks in group2, MPI_UNDEFINED when no correspondence exists.
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-This function is important for determining the relative numbering of the same processes in two different groups. For instance, if one knows the ranks of certain processes in the group of MPI_COMM_WORLD, one might want to know their ranks in a subset of that group.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_Group_union.3in b/ompi/mpi/man/man3/MPI_Group_union.3in
deleted file mode 100644
index 84a40079631..00000000000
--- a/ompi/mpi/man/man3/MPI_Group_union.3in
+++ /dev/null
@@ -1,85 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Group_union 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Group_union \fP \- Produces a group by combining two groups.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Group_union(MPI_Group \fIgroup1\fP, MPI_Group \fIgroup2\fP,
-	MPI_Group *\fInewgroup\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_GROUP_UNION(\fIGROUP1, GROUP2, NEWGROUP, IERROR\fP)
-	INTEGER	\fIGROUP1, GROUP2, NEWGROUP, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Group_union(\fIgroup1\fP, \fIgroup2\fP, \fInewgroup\fP, \fIierror\fP)
-	TYPE(MPI_Group), INTENT(IN) :: \fIgroup1\fP, \fIgroup2\fP
-	TYPE(MPI_Group), INTENT(OUT) :: \fInewgroup\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-group1
-First group (handle).
-.TP 1i
-group2
-Second group (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-newgroup
-Union group (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-The set-like operations are defined as follows:
-.TP
-  o
-union -- All elements of the first group (group1), followed by all elements
-of second group (group2) not in first.
-.TP
-  o
-intersect -- all elements of the first group that are also in the second
-group, ordered as in first group.
-.TP
-  o
-difference -- all elements of the first group that are not in the second group, ordered as in the first group.
-.sp
-.LP
-Note that for these operations the order of processes in the output group is determined primarily by order in the first group (if possible) and then, if necessary, by order in the second group. Neither union nor intersection are commutative, but both are associative.
-.sp
-The new group can be empty, that is, equal to MPI_GROUP_EMPTY.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-MPI_Group_free
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Iallgather.3in b/ompi/mpi/man/man3/MPI_Iallgather.3in
deleted file mode 100644
index f7b03f37700..00000000000
--- a/ompi/mpi/man/man3/MPI_Iallgather.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Allgather.3
diff --git a/ompi/mpi/man/man3/MPI_Iallgatherv.3in b/ompi/mpi/man/man3/MPI_Iallgatherv.3in
deleted file mode 100644
index 8fc7b812b1a..00000000000
--- a/ompi/mpi/man/man3/MPI_Iallgatherv.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Allgatherv.3
diff --git a/ompi/mpi/man/man3/MPI_Iallreduce.3in b/ompi/mpi/man/man3/MPI_Iallreduce.3in
deleted file mode 100644
index 9c97358ebe6..00000000000
--- a/ompi/mpi/man/man3/MPI_Iallreduce.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Allreduce.3
diff --git a/ompi/mpi/man/man3/MPI_Ialltoall.3in b/ompi/mpi/man/man3/MPI_Ialltoall.3in
deleted file mode 100644
index 591c20bb28d..00000000000
--- a/ompi/mpi/man/man3/MPI_Ialltoall.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Alltoall.3
diff --git a/ompi/mpi/man/man3/MPI_Ialltoallv.3in b/ompi/mpi/man/man3/MPI_Ialltoallv.3in
deleted file mode 100644
index 6cc7026e897..00000000000
--- a/ompi/mpi/man/man3/MPI_Ialltoallv.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Alltoallv.3
diff --git a/ompi/mpi/man/man3/MPI_Ialltoallw.3in b/ompi/mpi/man/man3/MPI_Ialltoallw.3in
deleted file mode 100644
index 0cca872ba48..00000000000
--- a/ompi/mpi/man/man3/MPI_Ialltoallw.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Alltoallw.3
diff --git a/ompi/mpi/man/man3/MPI_Ibarrier.3in b/ompi/mpi/man/man3/MPI_Ibarrier.3in
deleted file mode 100644
index 17e1bd261a8..00000000000
--- a/ompi/mpi/man/man3/MPI_Ibarrier.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Barrier.3
diff --git a/ompi/mpi/man/man3/MPI_Ibcast.3in b/ompi/mpi/man/man3/MPI_Ibcast.3in
deleted file mode 100644
index c0a86bebc97..00000000000
--- a/ompi/mpi/man/man3/MPI_Ibcast.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Bcast.3
diff --git a/ompi/mpi/man/man3/MPI_Ibsend.3in b/ompi/mpi/man/man3/MPI_Ibsend.3in
deleted file mode 100644
index e97d434c813..00000000000
--- a/ompi/mpi/man/man3/MPI_Ibsend.3in
+++ /dev/null
@@ -1,90 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Ibsend 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Ibsend\fP \- Starts a nonblocking buffered send.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Ibsend(const void *\fIbuf\fP, int\fI count\fP, MPI_Datatype\fI datatype\fP,
-	int\fI dest\fP, int\fI tag\fP, MPI_Comm\fI comm\fP, MPI_Request\fI *request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_IBSEND(\fIBUF, COUNT, DATATYPE, DEST, TAG, COMM, REQUEST, IERROR\fP)
-	<type>	\fIBUF\fP(*)
-	INTEGER	\fICOUNT, DATATYPE, DEST, TAG, COMM, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Ibsend(\fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIdest\fP, \fItag\fP, \fIcomm\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP, \fIdest\fP, \fItag\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of send buffer (choice).
-.TP 1i
-count
-Number of elements in send buffer (integer).
-.TP 1i
-datatype
-Data type of each send buffer element (handle).
-.TP 1i
-dest
-Rank of destination (integer).
-.TP 1i
-tag
-Message tag (integer).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-request
-Communication request (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Ibsend posts a buffered-mode, nonblocking send. Nonblocking calls allocate a communication request object and associate it with the request handle (the argument request). The request can be used later to query the status of the communication or wait for its completion.
-.sp
-A nonblocking send call indicates that the system may start copying data out of the send buffer. The sender should not modify any part of the send buffer after a nonblocking send operation is called, until the send completes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-MPI_Test
-MPI_Wait
-.br
-
-
diff --git a/ompi/mpi/man/man3/MPI_Iexscan.3in b/ompi/mpi/man/man3/MPI_Iexscan.3in
deleted file mode 100644
index c2ff4cf3254..00000000000
--- a/ompi/mpi/man/man3/MPI_Iexscan.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Exscan.3
diff --git a/ompi/mpi/man/man3/MPI_Igather.3in b/ompi/mpi/man/man3/MPI_Igather.3in
deleted file mode 100644
index d15bc2d25cf..00000000000
--- a/ompi/mpi/man/man3/MPI_Igather.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Gather.3
diff --git a/ompi/mpi/man/man3/MPI_Igatherv.3in b/ompi/mpi/man/man3/MPI_Igatherv.3in
deleted file mode 100644
index 3202cdbbd85..00000000000
--- a/ompi/mpi/man/man3/MPI_Igatherv.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Gatherv.3
diff --git a/ompi/mpi/man/man3/MPI_Improbe.3in b/ompi/mpi/man/man3/MPI_Improbe.3in
deleted file mode 100644
index ffa67e3e8cc..00000000000
--- a/ompi/mpi/man/man3/MPI_Improbe.3in
+++ /dev/null
@@ -1,114 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
-.\" Copyright 2012 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Improbe 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Improbe\fP \- Non-blocking matched probe for a message.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Improbe(int \fIsource\fP, int\fI tag\fP, MPI_Comm\fI comm\fP,
-	int\fI *flag\fP, MPI_Message\fI *message\fP, MPI_Status\fI *status\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_IMPROBE(\fISOURCE, TAG, COMM, FLAG, MESSAGE, STATUS, IERROR\fP)
-	LOGICAL	\fIFLAG\fP
-	INTEGER	\fISOURCE, TAG, COMM, MESSAGE\fP
-	INTEGER	\fISTATUS(MPI_STATUS_SIZE), IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Improbe(\fIsource\fP, \fItag\fP, \fIcomm\fP, \fIflag\fP, \fImessage\fP, \fIstatus\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIsource\fP, \fItag\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, INTENT(OUT) :: \fIflag\fP
-	TYPE(MPI_Message), INTENT(OUT) :: \fImessage\fP
-	TYPE(MPI_Status) :: \fIstatus\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-source
-Source rank or MPI_ANY_SOURCE (integer).
-.TP 1i
-tag
-Tag value or MPI_ANY_TAG (integer).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-flag
-Flag (logical).
-.ft R
-.TP 1i
-message
-Message (handle).
-.ft R
-.TP 1i
-status
-Status object (status).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Like MPI_Probe and MPI_Iprobe, the MPI_Mprobe and MPI_Improbe operations
-allow incoming messages to be queried without actually receiving
-them, except that MPI_Mprobe and MPI_Improbe provide a mechanism to
-receive the specific message that was matched regardless of other
-intervening probe or receive operations.  This gives the application
-an opportunity to decide how to receive the message, based on the
-information returned by the probe.  In particular, the application may
-allocate memory for the receive buffer according to the length of the
-probed message.
-.sp
-A matching probe with MPI_PROC_NULL as \fIsource\fP returns \fIflag\fP
-= true, \fImessage\fP = MPI_MESSAGE_NO_PROC, and the \fIstatus\fP object
-returns source = MPI_PROC_NULL, tag = MPI_ANY_TAG, and count = 0.
-.sp
-MPI_Iprobe returns a true value in \fIflag\fP if a message has been
-matched and can be received by passing the \fImessage\fP handle to the
-MPI_Mrecv or MPI_Imrecv functions, provided the \fIsource\fP was not
-MPI_PROC_NULL.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler
-MPI_ERRORS_RETURN may be used to cause error values to be
-returned. Note that MPI does not guarantee that an MPI program can
-continue past an error.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Mprobe
-MPI_Probe
-MPI_Iprobe
-MPI_Mrecv
-MPI_Imrecv
-MPI_Cancel
diff --git a/ompi/mpi/man/man3/MPI_Imrecv.3in b/ompi/mpi/man/man3/MPI_Imrecv.3in
deleted file mode 100644
index 72d0cb3a4b5..00000000000
--- a/ompi/mpi/man/man3/MPI_Imrecv.3in
+++ /dev/null
@@ -1,113 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2012 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Imrecv 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Imrecv\fP \- Non-blocking receive for a matched message
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Imrecv(void \fI*buf\fP, int\fI count\fP, MPI_Datatype\fI type\fP,
-	MPI_Message\fI *message\fP, MPI_Request\fI *request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_IMRECV(\fIBUF, COUNT, DATATYPE, MESSAGE, REQUEST, IERROR\fP)
-	<type>	\fIBUF(*)\fP
-	INTEGER	\fICOUNT, DATATYPE, MESSAGE, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Imrecv(\fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fImessage\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Message), INTENT(INOUT) :: \fImessage\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-count
-Number of elements to receive (nonnegative integer).
-.TP 1i
-datatype
-Datatype of each send buffer element (handle).
-.TP 1i
-message
-Message (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of receive buffer (choice).
-.TP 1i
-request
-Request (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-The functions MPI_Mrecv and MPI_Imrecv receive messages that have been
-previously matched by a matching probe.
-.sp
-The \fIrequest\fP returned from MPI_Imrecv can be used with any of the
-MPI_Test and MPI_Wait variants, like any non-blocking receive request.
-.sp
-If MPI_Imrecv is called with MPI_MESSAGE_NULL as the message argument,
-a call to one of the MPI_Test or MPI_Wait variants will return
-immediately with the \fIstatus\fP object set to \fIsource\fP =
-MPI_PROC_NULL, \fItag\fP = MPI_ANY_TAG, and \fIcount\fP = 0, as if a
-receive from MPI_PROC_NULL was issued.
-.sp
-If reception of a matched message is started with MPI_Imrecv, then it
-is possible to cancel the returned request with MPI_Cancel. If
-MPI_Cancel succeeds, the matched message must be found by a subsequent
-message probe (MPI_Probe, MPI_Iprobe, MPI_Mprobe, or MPI_Improbe),
-received by a subsequent receive operation or canceled by the
-sender.
-.sp
-Note, however, that is it possible for the cancellation of operations
-initiated with MPI_Imrecv to fail.  An example of a failing case is
-when canceling the matched message receive would violate MPI message
-ordering rules (e.g., if another message matching the same message
-signature has matched -- and possible received -- before this
-MPI_Imrecv is canceled).
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler
-MPI_ERRORS_RETURN may be used to cause error values to be
-returned. Note that MPI does not guarantee that an MPI program can
-continue past an error.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Mprobe
-MPI_Improbe
-MPI_Probe
-MPI_Iprobe
-MPI_Imrecv
-MPI_Cancel
diff --git a/ompi/mpi/man/man3/MPI_Ineighbor_allgather.3in b/ompi/mpi/man/man3/MPI_Ineighbor_allgather.3in
deleted file mode 100644
index f0569265c29..00000000000
--- a/ompi/mpi/man/man3/MPI_Ineighbor_allgather.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Neighbor_allgather.3
diff --git a/ompi/mpi/man/man3/MPI_Ineighbor_allgatherv.3in b/ompi/mpi/man/man3/MPI_Ineighbor_allgatherv.3in
deleted file mode 100644
index b8ce05e6462..00000000000
--- a/ompi/mpi/man/man3/MPI_Ineighbor_allgatherv.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Neighbor_allgatherv.3
diff --git a/ompi/mpi/man/man3/MPI_Ineighbor_alltoall.3in b/ompi/mpi/man/man3/MPI_Ineighbor_alltoall.3in
deleted file mode 100644
index 56f630ecd06..00000000000
--- a/ompi/mpi/man/man3/MPI_Ineighbor_alltoall.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Neighbor_alltoall.3
diff --git a/ompi/mpi/man/man3/MPI_Ineighbor_alltoallv.3in b/ompi/mpi/man/man3/MPI_Ineighbor_alltoallv.3in
deleted file mode 100644
index 13b4e89a23c..00000000000
--- a/ompi/mpi/man/man3/MPI_Ineighbor_alltoallv.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Neighbor_alltoallv.3
diff --git a/ompi/mpi/man/man3/MPI_Ineighbor_alltoallw.3in b/ompi/mpi/man/man3/MPI_Ineighbor_alltoallw.3in
deleted file mode 100644
index 299138b3a37..00000000000
--- a/ompi/mpi/man/man3/MPI_Ineighbor_alltoallw.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Neighbor_alltoallw.3
diff --git a/ompi/mpi/man/man3/MPI_Info_c2f.3in b/ompi/mpi/man/man3/MPI_Info_c2f.3in
deleted file mode 100644
index a13fce697dd..00000000000
--- a/ompi/mpi/man/man3/MPI_Info_c2f.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Comm_f2c.3
diff --git a/ompi/mpi/man/man3/MPI_Info_create.3in b/ompi/mpi/man/man3/MPI_Info_create.3in
deleted file mode 100644
index ddf5f60e4db..00000000000
--- a/ompi/mpi/man/man3/MPI_Info_create.3in
+++ /dev/null
@@ -1,67 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Info_create 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Info_create\fP \- Creates a new info object.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Info_create(MPI_Info \fI*info\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_INFO_CREATE(\fIINFO, IERROR\fP)
-	INTEGER	\fIINFO, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Info_create(\fIinfo\fP, \fIierror\fP)
-	TYPE(MPI_Info), INTENT(OUT) :: \fIinfo\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-info
-Info object created (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Info_create creates a new info object. The newly created object contains no key/value pairs.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft r
-MPI_Info_delete
-.br
-MPI_Info_dup
-.br
-MPI_Info_free
-.br
-MPI_Info_get
-.br
-MPI_Info_set
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Info_delete.3in b/ompi/mpi/man/man3/MPI_Info_delete.3in
deleted file mode 100644
index dd8ecab5d03..00000000000
--- a/ompi/mpi/man/man3/MPI_Info_delete.3in
+++ /dev/null
@@ -1,78 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Info_delete 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Info_delete\fP \- Deletes a key/value pair from \fIinfo\fP.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Info_delete(MPI_Info \fIinfo\fP, const char \fI*key\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_INFO_DELETE(\fIINFO, KEY, IERROR\fP)
-	INTEGER		\fIINFO, IERROR\fP
-	CHARACTER*(*)	\fIKEY\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Info_delete(\fIinfo\fP, \fIkey\fP, \fIierror\fP)
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	CHARACTER(LEN=*), INTENT(IN) :: \fIkey\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-info
-Info object (handle).
-
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-key
-Key (string).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Info_delete deletes a (key,value) pair from \fIinfo\fP. If \fIkey\fP is not defined in \fIinfo\fP, the call raises an error of class MPI_ERR_INFO_NOKEY.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft r
-MPI_Info_create
-.br
-MPI_Info_dup
-.br
-MPI_Info_free
-.br
-MPI_Info_get
-.br
-MPI_Info_set
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Info_dup.3in b/ompi/mpi/man/man3/MPI_Info_dup.3in
deleted file mode 100644
index 145b5831dc7..00000000000
--- a/ompi/mpi/man/man3/MPI_Info_dup.3in
+++ /dev/null
@@ -1,74 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Info_dup 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Info_dup\fP \- Duplicates an info object.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Info_dup(MPI_Info \fIinfo\fP, MPI_Info \fI*newinfo\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_INFO_DUP(\fIINFO, NEWINFO, IERROR\fP)
-	INTEGER		\fIINFO, NEWINFO, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Info_dup(\fIinfo\fP, \fInewinfo\fP, \fIierror\fP)
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(MPI_Info), INTENT(OUT) :: \fInewinfo\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-info
-Info object (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-newinfo
-Info object (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Info_dup duplicates an existing info object, creating a new object, with the same (key,value) pairs and the same ordering of keys.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft r
-MPI_Info_create
-.br
-MPI_Info_delete
-.br
-MPI_Info_free
-.br
-MPI_Info_get
-.br
-MPI_Info_set
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Info_env.3in b/ompi/mpi/man/man3/MPI_Info_env.3in
deleted file mode 100644
index c7ef970fa7d..00000000000
--- a/ompi/mpi/man/man3/MPI_Info_env.3in
+++ /dev/null
@@ -1,66 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2012 Los Alamos National Security, LLC.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" $COPYRIGHT$
-.TH MPI_INFO_ENV 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_INFO_ENV\fP \- Static MPI_Info object containing info about the application
-
-.SH DESCRIPTION
-.ft R
-The MPI-3 standard established a static MPI_Info object named \fIMPI_INFO_ENV\fP that can be used to access information about how the application was executed from the run-time.
-
-.SH SUPPORTED FIELDS
-.ft R
-.TP 1i
-command
-If available, the value will be set to argv[0]. Note that the value may not always be available - e.g., it is valid for a program to call MPI_Init with NULL parameters, in which case argv[0] will not be set if run as a singleton. This value will never be set in a Fortran program as the argv are not available.
-.TP 1i
-argv
-The argv given for the application. If no arguments are passed to the application, then this value will not be set. It will also not be set in the case of a singleton that calls MPI_Init with NULL parameters, or a Fortran program.
-.TP 1i
-maxprocs
-The number of processes in the job.
-.TP 1i
-soft
-Open MPI does not support the \fIsoft\fP option for specifying the number of processes to be executed, so this value is set to the same as \fImaxprocs\fP.
-.TP 1i
-host
-The name of the host this process is executing upon - the value returned from \fIgethostname()\fP.
-.TP 1i
-arch
-The architecture of the host this process is executing upon. This value indicates the underlying chip architecture (e.g., x86_64), if it can be determined.
-.TP 1i
-wdir
-The working directory at the time of process launch by mpiexec. Note that this value will not be set for processes launched as singletons as there is no reliable way for the MPI library to determine the location.
-.TP 1i
-file
-Although specified by the MPI-3 standard, no value is currently set for this field.
-.TP 1i
-thread_level
-The requested MPI thread level - note that this may differ from the \fIactual\fP MPI thread level of the application.
-.TP 1i
-ompi_num_apps
-The number of application contexts in an MPMD job.
-This is an Open MPI-specific field and value.
-.TP 1i
-ompi_np
-The number of processes in each application context, provided as a space-delimited list of integers.
-This is an Open MPI-specific field and value.
-.TP 1i
-ompi_first_rank
-The MPI rank of the first process in each application context, provided as a space-delimited list of integers
-This is an Open MPI-specific field and value.
-.TP 1i
-ompi_positioned_file_dir
-If Open MPI was asked to pre-position files, this field provides the top-level directory where those files were place.
-This is an Open MPI-specific field and value.
-
-.SH ERRORS
-When calling MPI_INFO_GET(3), the \fIflag\fP parameter will be set to zero (false) if a value for the field has not been set.
-.br
-
-.SH SEE ALSO
-.ft r
-MPI_Info_get
diff --git a/ompi/mpi/man/man3/MPI_Info_f2c.3in b/ompi/mpi/man/man3/MPI_Info_f2c.3in
deleted file mode 100644
index a13fce697dd..00000000000
--- a/ompi/mpi/man/man3/MPI_Info_f2c.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Comm_f2c.3
diff --git a/ompi/mpi/man/man3/MPI_Info_free.3in b/ompi/mpi/man/man3/MPI_Info_free.3in
deleted file mode 100644
index 138e9bb429c..00000000000
--- a/ompi/mpi/man/man3/MPI_Info_free.3in
+++ /dev/null
@@ -1,69 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Info_free 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Info_free\fP \- Frees an info object.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Info_free(MPI_Info \fI*info\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_INFO_FREE(\fIINFO, IERROR\fP)
-	INTEGER		\fIINFO, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Info_free(\fIinfo\fP, \fIierror\fP)
-	TYPE(MPI_Info), INTENT(INOUT) :: \fIinfo\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-info
-Info object (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Info_free frees \fIinfo\fP and sets it to MPI_INFO_NULL.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft r
-MPI_Info_create
-.br
-MPI_Info_delete
-.br
-MPI_Info_dup
-.br
-MPI_Info_get
-.br
-MPI_Info_set
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Info_get.3in b/ompi/mpi/man/man3/MPI_Info_get.3in
deleted file mode 100644
index 22df455ec9b..00000000000
--- a/ompi/mpi/man/man3/MPI_Info_get.3in
+++ /dev/null
@@ -1,100 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Info_get 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Info_get\fP \- Retrieves the value associated with a key in an info object.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Info_get(MPI_Info \fIinfo\fP, const char \fI*key\fP, int \fIvaluelen\fP, char \fI*value\fP, int *\fIflag\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_INFO_GET(\fIINFO, KEY, VALUELEN, VALUE, FLAG, IERROR\fP)
-	INTEGER	\fIINFO, VALUELEN, IERROR\fP
-	CHARACTER*(*) \fIKEY, VALUE\fP
-	LOGICAL \fIFLAG\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Info_get(\fIinfo\fP, \fIkey\fP, \fIvaluelen\fP, \fIvalue\fP, \fIflag\fP, \fIierror\fP)
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	CHARACTER(LEN=*), INTENT(IN) :: \fIkey\fP
-	INTEGER, INTENT(IN) :: \fIvaluelen\fP
-	CHARACTER(LEN=valuelen), INTENT(OUT) :: \fIvalue\fP
-	LOGICAL, INTENT(OUT) :: \fIflag\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-info
-Info object (handle).
-.ft R
-.TP 1i
-key
-Key (string).
-.ft R
-.TP 1i
-valuelen
-Length of value arg (integer).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-value
-Value (string).
-.ft R
-.TP 1i
-flag
-Returns true if key defined, false if not (boolean).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Info_get retrieves the value associated with \fIkey\fP in a previous call to MPI_Info_set. If such a key exists, it sets \fIflag\fP to true and returns the value in \fIvalue\fP; otherwise it sets \fIflag\fP to false and leaves \fIvalue\fP unchanged. \fIvaluelen\fP is the number of characters available in value. If it is less than the actual size of the value, the returned value is truncated. In C, \fIvaluelen\fP should be one less than the amount of allocated space to allow for the null terminator.
-.sp
-If \fIkey\fP is larger than MPI_MAX_INFO_KEY, the call is erroneous.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft r
-MPI_Info_create
-.br
-MPI_Info_delete
-.br
-MPI_Info_dup
-.br
-MPI_Info_free
-.br
-MPI_Info_get_valuelen
-.br
-MPI_Info_get_nkeys
-.br
-MPI_Info_get_nthkey
-.br
-MPI_Info_set
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Info_get_nkeys.3in b/ompi/mpi/man/man3/MPI_Info_get_nkeys.3in
deleted file mode 100644
index dc953933631..00000000000
--- a/ompi/mpi/man/man3/MPI_Info_get_nkeys.3in
+++ /dev/null
@@ -1,70 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Info_get_nkeys 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Info_get_nkeys\fP \- Gets the number of keys currently defined in an info object.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Info_get_nkeys(MPI_Info \fIinfo\fP, int \fI*nkeys\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_INFO_GET_NKEYS(\fIINFO, NKEYS, IERROR\fP)
-	INTEGER		\fIINFO, NKEYS, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Info_get_nkeys(\fIinfo\fP, \fInkeys\fP, \fIierror\fP)
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	INTEGER, INTENT(OUT) :: \fInkeys\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-info
-Info object (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-nkeys
-Number of defined keys (integer).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Info_get_nkeys returns the number of currently defined keys in \fIinfo\fP.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft r
-MPI_Info_get
-.br
-MPI_Info_get_nthkey
-.br
-MPI_Info_get_valuelen
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Info_get_nthkey.3in b/ompi/mpi/man/man3/MPI_Info_get_nthkey.3in
deleted file mode 100644
index 6995a3ea610..00000000000
--- a/ompi/mpi/man/man3/MPI_Info_get_nthkey.3in
+++ /dev/null
@@ -1,76 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Info_get_nthkey 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Info_get_nthkey\fP \- Returns the \fIn\fPth defined key in \fIinfo\fP.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Info_get_nthkey(MPI_Info \fIinfo\fP, int \fIn\fP, char \fI*key\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_INFO_GET_NTHKEY(\fIINFO, N, KEY, IERROR\fP)
-	INTEGER		\fIINFO, N, IERROR\fP
-	CHARACTER*(*)	\fIKEY\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Info_get_nthkey(\fIinfo\fP, \fIn\fP, \fIkey\fP, \fIierror\fP)
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	INTEGER, INTENT(IN) :: \fIn\fP
-	CHARACTER(LEN=*), INTENT(OUT) :: \fIkey\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-info
-Info object (handle).
-.ft R
-.TP 1i
-n
-Key number (integer).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-key
-Key (string).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Info_get_nthkey returns the \fIn\fPth defined key in \fIinfo\fP. Keys are numbered 0\...\fIN\fP - 1 where \fIN\fP is the value returned by MPI_Info_get_nkeys. All keys between 0 and \fIN\fP - 1 are guaranteed to be defined. The number of a given key does not change as long as \fIinfo\fP is not modified with MPI_Info_set or MPI_Info_delete.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft r
-MPI_Info_get
-.br
-MPI_Info_get_nkeys
-.br
-MPI_Info_get_valuelen
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Info_get_valuelen.3in b/ompi/mpi/man/man3/MPI_Info_get_valuelen.3in
deleted file mode 100644
index d1e91840e60..00000000000
--- a/ompi/mpi/man/man3/MPI_Info_get_valuelen.3in
+++ /dev/null
@@ -1,86 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Info_get_valuelen 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Info_get_valuelen\fP \- Retrieves the length of the key value associated with an info object.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Info_get_valuelen(MPI_Info \fIinfo\fP, const char \fI*key\fP,
-	int \fI*valuelen\fP, int \fI*flag\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_INFO_GET_VALUELEN(\fIINFO, KEY, VALUELEN, FLAG, IERROR\fP)
-	INTEGER		\fIINFO, VALUELEN, IERROR\fP
-	LOGICAL		\fIFLAG\fP
-	CHARACTER*(*)	\fIKEY\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Info_get_valuelen(\fIinfo\fP, \fIkey\fP, \fIvaluelen\fP, \fIflag\fP, \fIierror\fP)
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	CHARACTER(LEN=*), INTENT(IN) :: \fIkey\fP
-	INTEGER, INTENT(OUT) :: \fIvaluelen\fP
-	LOGICAL, INTENT(OUT) :: \fIflag\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-info
-Info object (handle).
-.ft R
-.TP 1i
-key
-Key (string).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-valuelen
-Length of value arg (integer).
-.ft R
-.TP 1i
-flag
-Returns true if key defined, false if not (boolean).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Info_get_valuelen retrieves the length of the \fIvalue\fP associated with \fIkey\fP. If \fIkey\fP is defined, \fIvaluelen\fP is set to the length of its associated value and \fIflag\fP is set to true. If \fIkey\fP is not defined, \fIvaluelen\fP is not touched and \fIflag\fP is set to false. The length returned in C does not include the end-of-string character.
-.sp
-If \fIkey\fP is larger than MPI_MAX_INFO_KEY, the call is erroneous.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft r
-MPI_Info_get
-.br
-MPI_Info_get_nkeys
-.br
-MPI_Info_get_nthkey
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Info_set.3in b/ompi/mpi/man/man3/MPI_Info_set.3in
deleted file mode 100644
index c13bdb8e0bb..00000000000
--- a/ompi/mpi/man/man3/MPI_Info_set.3in
+++ /dev/null
@@ -1,81 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Info_set 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Info_set\fP \- Adds a key/value pair to \fIinfo\fP.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Info_set(MPI_Info \fIinfo\fP, char \fI*key\fP, char \fI*value\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_INFO_SET(\fIINFO, KEY, VALUE, IERROR\fP)
-	INTEGER		\fIINFO, IERROR\fP
-	CHARACTER*(*)	\fIKEY, VALUE\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Info_set(\fIinfo\fP, \fIkey\fP, \fIvalue\fP, \fIierror\fP)
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	CHARACTER(LEN=*), INTENT(IN) :: \fIkey\fP, \fIvalue\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-info
-Info object (handle).
-
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-key
-Key (string).
-.ft R
-.TP 1i
-value
-Value (string).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Info_set adds the (key,value) pair to \fIinfo\fP and overrides the value if a value for the same key was previously set. The \fIkey\fP and \fIvalue\fP parameters are null-terminated strings in C. In Fortran, leading and trailing spaces in \fIkey\fP and \fIvalue\fP are stripped. If either \fIkey\fP or \fIvalue\fP is larger than the allowed maximums, the error MPI_ERR_INFO_KEY or MPI_ERR_INFO_VALUE is raised, respectively.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft r
-MPI_Info_create
-.br
-MPI_Info_delete
-.br
-MPI_Info_dup
-.br
-MPI_Info_free
-.br
-MPI_Info_set
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Init.3in b/ompi/mpi/man/man3/MPI_Init.3in
deleted file mode 100644
index c1790c9e38a..00000000000
--- a/ompi/mpi/man/man3/MPI_Init.3in
+++ /dev/null
@@ -1,104 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Init 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-
-.SH NAME
-\fBMPI_Init\fP \- Initializes the MPI execution environment
-
-.SH SYNTAX
-.ft R
-
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Init(int *\fIargc\fP, char ***\fIargv\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_INIT(\fIIERROR\fP)
-	INTEGER	\fIIERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Init(\fIierror\fP)
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-argc
-C only: Pointer to the number of arguments.
-.TP 1i
-argv
-C only: Argument vector.
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-This routine, or MPI_Init_thread, must be called before most other MPI
-routines are called.  There are a small number of errors, such as
-MPI_Initialized and MPI_Finalized.  MPI can be initialized at most
-once; subsequent calls to MPI_Init or MPI_Init_thread are erroneous.
-.sp
-All MPI programs must contain a call to MPI_Init or
-MPI_Init_thread. Open MPI accepts the C \fIargc\fP and \fIargv\fP
-arguments to main, but neither modifies, interprets, nor distributes
-them:
-.sp
-.nf
-	{
-		/* declare variables */
-		MPI_Init(&argc, &argv);
-		/* parse arguments */
-		/* main program */
-		MPI_Finalize();
-	}
-.fi
-
-.SH NOTES
-.ft R
-The Fortran version does not have provisions for \fIargc\fP and
-\fIargv\fP and takes only IERROR.
-.sp
-The MPI Standard does not say what a program can do before an MPI_Init
-or after an MPI_Finalize. In the Open MPI implementation, it should do
-as little as possible. In particular, avoid anything that changes the
-external state of the program, such as opening files, reading standard
-input, or writing to standard output.
-
-.SH ERRORS
-.ft R
-Almost all MPI routines return an error value; C routines as
-the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-.sp
-See the MPI man page for a full list of MPI error codes.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Init_thread
-MPI_Initialized
-MPI_Finalize
-MPI_Finalized
diff --git a/ompi/mpi/man/man3/MPI_Init_thread.3in b/ompi/mpi/man/man3/MPI_Init_thread.3in
deleted file mode 100644
index 800e1cb381c..00000000000
--- a/ompi/mpi/man/man3/MPI_Init_thread.3in
+++ /dev/null
@@ -1,172 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2010-2020 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Init_thread 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.
-.SH NAME
-\fBMPI_Init_thread\fP \- Initializes the MPI execution environment
-.
-.SH SYNTAX
-.ft R
-.
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Init_thread(int *\fIargc\fP, char ***\fIargv\fP,
-	int \fIrequired\fP, int *\fIprovided\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_INIT_THREAD(\fIREQUIRED, PROVIDED, IERROR\fP)
-	INTEGER	\fIREQUIRED, PROVIDED, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Init_thread(\fIrequired\fP, \fIprovided\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIrequired\fP
-	INTEGER, INTENT(OUT) :: \fIprovided\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-argc
-C only: Pointer to the number of arguments.
-.TP 1i
-argv
-C only: Argument vector.
-.TP 1i
-required
-Desired level of thread support (integer).
-.
-.
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-provided
-Available level of thread support (integer).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-.
-.
-.SH DESCRIPTION
-.ft R
-This routine, or MPI_Init, must be called before most other MPI
-routines are called. There are a small number of exceptions, such as
-MPI_Initialized and MPI_Finalized.  MPI can be initialized at most
-once; subsequent calls to MPI_Init or MPI_Init_thread are erroneous.
-.sp
-MPI_Init_thread, as compared to MPI_Init, has a provision to request a
-certain level of thread support in \fIrequired\fP:
-.TP 2.4i
-MPI_THREAD_SINGLE
-Only one thread will execute.
-.TP 2.4i
-MPI_THREAD_FUNNELED
-If the process is multithreaded, only the thread that called
-MPI_Init_thread will make MPI calls.
-.TP 2.4i
-MPI_THREAD_SERIALIZED
-If the process is multithreaded, only one thread will make MPI library
-calls at one time.
-.TP 2.4i
-MPI_THREAD_MULTIPLE
-If the process is multithreaded, multiple threads may call MPI at once
-with no restrictions.
-.
-.PP
-The level of thread support available to the program is set in
-\fIprovided\fP. In Open MPI, the value is dependent on how the library was
-configured and built. Note that there is no guarantee that
-\fIprovided\fP will be greater than or equal to \fIrequired\fP.
-.sp
-Also note that calling MPI_Init_thread with a
-.I required
-value of
-.I MPI_THREAD_SINGLE
-is equivalent to calling MPI_Init.
-.sp
-All MPI programs must contain a call to MPI_Init or
-MPI_Init_thread. Open MPI accepts the C \fIargc\fP and \fIargv\fP
-arguments to main, but neither modifies, interprets, nor distributes
-them:
-.sp
-.nf
-	{
-		/* declare variables */
-		MPI_Init_thread(&argc, &argv, req, &prov);
-		/* parse arguments */
-		/* main program */
-		MPI_Finalize();
-	}
-.fi
-.
-.SH NOTES
-.ft R
-The Fortran version does not have provisions for \fIargc\fP and
-\fIargv\fP and takes only IERROR.
-.sp
-It is the caller's responsibility to check the value of \fIprovided\fP,
-as it may be less than what was requested in \fIrequired\fP.
-.sp
-The MPI Standard does not say what a program can do before an
-MPI_Init_thread or after an MPI_Finalize. In the Open MPI
-implementation, it should do as little as possible. In particular,
-avoid anything that changes the external state of the program, such as
-opening files, reading standard input, or writing to standard output.
-.
-.
-.SH MPI_THREAD_MULTIPLE Support
-.
-MPI_THREAD_MULTIPLE support is included if the environment in which
-Open MPI was built supports threading.  You can check the
-output of
-.BR ompi_info (1)
-to see if Open MPI has MPI_THREAD_MULTIPLE support:
-.
-.PP
-.nf
-shell$ ompi_info | grep "Thread support"
-          Thread support: posix (MPI_THREAD_MULTIPLE: yes, OPAL support: yes, OMPI progress: no, Event lib: yes)
-shell$
-.fi
-.
-.PP
-The "MPI_THREAD_MULTIPLE: yes" portion of the above output indicates
-that Open MPI was compiled with MPI_THREAD_MULTIPLE support.
-.
-.PP
-Note that there is a small performance penalty for using
-MPI_THREAD_MULTIPLE support; latencies for short messages will be
-higher as compared to when using MPI_THREAD_SINGLE, for example.
-.
-.
-.SH ERRORS
-.ft R
-Almost all MPI routines return an error value; C routines as
-the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-.
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Init
-MPI_Initialized
-MPI_Finalize
-MPI_Finalized
diff --git a/ompi/mpi/man/man3/MPI_Initialized.3in b/ompi/mpi/man/man3/MPI_Initialized.3in
deleted file mode 100644
index ce3764ea89f..00000000000
--- a/ompi/mpi/man/man3/MPI_Initialized.3in
+++ /dev/null
@@ -1,65 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Initialized 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Initialized\fP \- Checks whether MPI has been initialized
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Initialized(int *\fIflag\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_INITIALIZED(\fIFLAG, IERROR\fP)
-	LOGICAL	\fIFLAG\fP
-	INTEGER	\fIIERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Initialized(\fIflag\fP, \fIierror\fP)
-	LOGICAL, INTENT(OUT) :: \fIflag\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-flag
-True if MPI has been initialized, and false otherwise (logical).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-This routine may be used to determine whether MPI has been
-initialized.  It is one of a small number of routines that may be
-called before MPI is initialized and after MPI has been finalized
-(MPI_Finalized is another).
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Init
-MPI_Init_thread
-MPI_Finalize
-MPI_Finalized
diff --git a/ompi/mpi/man/man3/MPI_Intercomm_create.3in b/ompi/mpi/man/man3/MPI_Intercomm_create.3in
deleted file mode 100644
index e2b56b53e85..00000000000
--- a/ompi/mpi/man/man3/MPI_Intercomm_create.3in
+++ /dev/null
@@ -1,112 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Intercomm_create 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Intercomm_create\fP \- Creates an intercommunicator from two intracommunicators.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Intercomm_create(MPI_Comm \fIlocal_comm\fP, int\fI local_leader\fP,
-	MPI_Comm\fI peer_comm\fP, int\fI remote_leader\fP, int\fI tag\fP, MPI_Comm\fI *newintercomm\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_INTERCOMM_CREATE(\fILOCAL_COMM, LOCAL_LEADER, PEER_COMM,
-		REMOTE_LEADER, TAG, NEWINTERCOMM, IERROR\fP)
-	INTEGER	\fILOCAL_COMM, LOCAL_LEADER, PEER_COMM, REMOTE_LEADER\fP
-	INTEGER	\fITAG, NEWINTERCOMM, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Intercomm_create(\fIlocal_comm\fP, \fIlocal_leader\fP, \fIpeer_comm\fP, \fIremote_leader\fP,
-		\fItag\fP, \fInewintercomm\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(IN) :: \fIlocal_comm\fP, \fIpeer_comm\fP
-	INTEGER, INTENT(IN) :: \fIlocal_leader\fP, \fIremote_leader\fP, \fItag\fP
-	TYPE(MPI_Comm), INTENT(OUT) :: \fInewintercomm\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-local_comm
-The communicator containing the process that initiates the inter-communication (handle).
-.TP 1i
-local_leader
-Rank of local group leader in local_comm (integer).
-.TP 1i
-peer_comm
-"Peer" communicator; significant only at the local_leader (handle).
-.TP 1i
-remote_leader
-Rank of remote group leader in peer_comm; significant only at the local_leader (integer).
-.TP 1i
-tag
-Message tag used to identify new intercommunicator (integer).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-newintercomm
-Created intercommunicator (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-This call creates an intercommunicator. It is collective over the union of the local and remote groups. Processes should provide identical local_comm and local_leader arguments within each group. Wildcards are not permitted for remote_leader, local_leader, and tag.
-.sp
-This call uses point-to-point communication with communicator peer_comm,
-and with tag tag between the leaders. Thus, care must be taken that there be no pending communication on peer_comm that could interfere with this communication.
-
-If multiple MPI_Intercomm_creates are being made, they should use different tags (more precisely, they should ensure that the local and remote leaders are using different tags for each MPI_intercomm_create).
-
-.SH NOTES
-We recommend using a dedicated peer communicator, such as a duplicate of MPI_COMM_WORLD, to avoid trouble with peer communicators.
-.sp
-The MPI 1.1 Standard contains two mutually exclusive comments on the
-input intracommunicators.  One says that their respective groups must be
-disjoint; the other that the leaders can be the same process.  After
-some discussion by the MPI Forum, it has been decided that the groups must
-be disjoint.  Note that the
-.B reason
-given for this in the standard is
-.B not
-the reason for this choice; rather, the
-.B other
-operations on
-intercommunicators (like
-.I MPI_Intercomm_merge
-) do not make sense if the
-groups are not disjoint.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-MPI_Intercomm_merge
-.br
-MPI_Comm_free
-.br
-MPI_Comm_remote_group
-.br
-MPI_Comm_remote_size
-
-
diff --git a/ompi/mpi/man/man3/MPI_Intercomm_merge.3in b/ompi/mpi/man/man3/MPI_Intercomm_merge.3in
deleted file mode 100644
index bb2b7ed9b34..00000000000
--- a/ompi/mpi/man/man3/MPI_Intercomm_merge.3in
+++ /dev/null
@@ -1,75 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Intercomm_merge 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Intercomm_merge\fP \- Creates an intracommunicator from an intercommunicator.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Intercomm_merge(MPI_Comm \fIintercomm\fP, int\fI high\fP,
-     MPI_Comm\fI *newintracomm\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_INTERCOMM_MERGE(\fIINTERCOMM, HIGH, NEWINTRACOMM, IERROR\fP)
-	INTEGER	\fIINTERCOMM, NEWINTRACOMM, IERROR\fP
-	LOGICAL	\fIHIGH\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Intercomm_merge(\fIintercomm\fP, \fIhigh\fP, \fInewintracomm\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(IN) :: \fIintercomm\fP
-	LOGICAL, INTENT(IN) :: \fIhigh\fP
-	TYPE(MPI_Comm), INTENT(OUT) :: \fInewintracomm\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-intercomm
-Intercommunicator (type indicator).
-.TP 1i
-high
-Used to order the groups of the two intracommunicators within comm when creating the new communicator (type indicator).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-newintracomm
-Created intracommunicator (type indicator).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-This function creates an intracommunicator from the union of the two groups that are associated with intercomm. All processes should provide the same high value within each of the two groups. If processes in one group provide the value high = false and processes in the other group provide the value high = true, then the union orders the "low" group before the "high" group. If all processes provide the same high argument, then the order of the union is arbitrary. This call is blocking and collective within the union of the two groups.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-MPI_Intercomm_create
-.br
-MPI_Comm_free
-
-
-
-
diff --git a/ompi/mpi/man/man3/MPI_Iprobe.3in b/ompi/mpi/man/man3/MPI_Iprobe.3in
deleted file mode 100644
index 856fe5a684d..00000000000
--- a/ompi/mpi/man/man3/MPI_Iprobe.3in
+++ /dev/null
@@ -1,96 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Iprobe 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Iprobe\fP \- Nonblocking test for a message.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Iprobe(int \fIsource\fP, int\fI tag\fP, MPI_Comm\fI comm\fP, int\fI *flag\fP,
-	MPI_Status\fI *status\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_IPROBE(\fISOURCE, TAG, COMM, FLAG, STATUS, IERROR\fP)
-	LOGICAL	\fIFLAG\fP
-	INTEGER	\fISOURCE, TAG, COMM, STATUS(MPI_STATUS_SIZE), IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Iprobe(\fIsource\fP, \fItag\fP, \fIcomm\fP, \fIflag\fP, \fIstatus\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIsource\fP, \fItag\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	LOGICAL, INTENT(OUT) :: \fIflag\fP
-	TYPE(MPI_Status) :: \fIstatus\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-source
-Source rank or MPI_ANY_SOURCE (integer).
-.TP 1i
-tag
-Tag value or MPI_ANY_TAG (integer).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-flag
-Message-waiting flag (logical).
-.TP 1i
-status
-Status object (status).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-The MPI_Probe and MPI_Iprobe operations allow checking of incoming messages without actual receipt of them. The user can then decide how to receive them, based on the information returned by the probe (basically, the information returned by status). In particular, the user may allocate memory for the receive buffer, according to the length of the probed message.
-.sp
-MPI_Iprobe(source, tag, comm, flag, status) returns flag = true if there is a message that can be received and that matches the pattern specified by the arguments source, tag, and comm. The call matches the same message that would have been received by a call to MPI_Recv(\&..., source, tag, comm, status) executed at the same point in the program, and returns in status the same value that would have been returned by MPI_Recv(). Otherwise, the call returns flag = false, and leaves status undefined.
-.sp
-If MPI_Iprobe returns flag = true, then the content of the status object can be subsequently accessed as described in Section 3.2.5 of the MPI-1 Standard, "Return Status," to find the source, tag, and length of the probed message.
-.sp
-A subsequent receive executed with the same context, and the source and tag returned in status by MPI_Iprobe will receive the message that was matched by the probe if no other intervening receive occurs after the probe. If the receiving process is multithreaded, it is the user's responsibility to ensure that the last condition holds.
-.sp
-The source argument of MPI_Probe can be MPI_ANY_SOURCE, and the tag argument can be MPI_ANY_TAG, so that one can probe for messages from an arbitrary source and/or with an arbitrary tag. However, a specific communication context must be provided with the comm argument.
-.sp
-If your application does not need to examine the \fIstatus\fP field, you can save resources by using the predefined constant MPI_STATUS_IGNORE as a special value for the \fIstatus\fP argument.
-.sp
-It is not necessary to receive a message immediately after it has been probed for, and the same message may be probed for several times before it is received.
-.sp
-.SH NOTE
-Users of libmpi-mt should remember that two threads may do an MPI_Iprobe that actually returns true for the same message for both threads.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Probe
-.br
-MPI_Cancel
-
diff --git a/ompi/mpi/man/man3/MPI_Irecv.3in b/ompi/mpi/man/man3/MPI_Irecv.3in
deleted file mode 100644
index a4da7c51a2e..00000000000
--- a/ompi/mpi/man/man3/MPI_Irecv.3in
+++ /dev/null
@@ -1,95 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Irecv 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Irecv\fP \- Starts a standard-mode, nonblocking receive.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Irecv(void *\fIbuf\fP, int\fI count\fP, MPI_Datatype\fI datatype\fP,
-        int\fI source\fP, int\fI tag\fP, MPI_Comm\fI comm\fP, MPI_Request\fI *request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_IRECV(\fIBUF, COUNT, DATATYPE, SOURCE, TAG, COMM, REQUEST,
-		IERROR\fP)
-	<type>	\fIBUF\fP(*)
-	INTEGER	\fICOUNT, DATATYPE, SOURCE, TAG, COMM, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Irecv(\fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIsource\fP, \fItag\fP, \fIcomm\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP, \fIsource\fP, \fItag\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of receive buffer (choice).
-.TP 1i
-count
-Number of elements in receive buffer (integer).
-.TP 1i
-datatype
-Datatype of each receive buffer element (handle).
-.TP 1i
-source
-Rank of source (integer).
-.TP 1i
-tag
-Message tag (integer).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-request
-Communication request (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Nonblocking calls allocate a communication request object and associate it with the request handle (the argument request). The request can be used later to query the status of the communication or wait for its completion.
-.sp
-A nonblocking receive call indicates that the system may start writing data into the receive buffer. The receiver should not access any part of the receive buffer after a nonblocking receive operation is called, until the receive completes.
-.sp
-A receive request can be determined being completed by calling the MPI_Wait, MPI_Waitany, MPI_Test, or MPI_Testany with request returned by this function.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-MPI_Recv
-MPI_Probe
-MPI_Test
-MPI_Testany
-MPI_Wait
-MPI_Waitany
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Ireduce.3in b/ompi/mpi/man/man3/MPI_Ireduce.3in
deleted file mode 100644
index f8c65fb7223..00000000000
--- a/ompi/mpi/man/man3/MPI_Ireduce.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Reduce.3
diff --git a/ompi/mpi/man/man3/MPI_Ireduce_scatter.3in b/ompi/mpi/man/man3/MPI_Ireduce_scatter.3in
deleted file mode 100644
index 4f03aec6068..00000000000
--- a/ompi/mpi/man/man3/MPI_Ireduce_scatter.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Reduce_scatter.3
diff --git a/ompi/mpi/man/man3/MPI_Ireduce_scatter_block.3in b/ompi/mpi/man/man3/MPI_Ireduce_scatter_block.3in
deleted file mode 100644
index f649a6c443d..00000000000
--- a/ompi/mpi/man/man3/MPI_Ireduce_scatter_block.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Reduce_scatter_block.3
diff --git a/ompi/mpi/man/man3/MPI_Irsend.3in b/ompi/mpi/man/man3/MPI_Irsend.3in
deleted file mode 100644
index a7d9e6cc21c..00000000000
--- a/ompi/mpi/man/man3/MPI_Irsend.3in
+++ /dev/null
@@ -1,86 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Irsend 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Irsend\fP \- Starts a ready-mode nonblocking send.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Irsend(const void *\fIbuf\fP, int\fI count\fP, MPI_Datatype\fI datatype\fP, int\fI dest\fP,
-	int\fI tag\fP, MPI_Comm\fI comm\fP, MPI_Request\fI *request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_IRSEND(\fIBUF, COUNT, DATATYPE, DEST, TAG, COMM, REQUEST, IERROR\fP)
-	<type>	\fIBUF\fP(*)
-	INTEGER	\fICOUNT, DATATYPE, DEST, TAG, COMM, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Irsend(\fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIdest\fP, \fItag\fP, \fIcomm\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP, \fIdest\fP, \fItag\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of send buffer (choice).
-.TP 1i
-count
-Number of elements in send buffer (integer).
-.TP 1i
-datatype
-Datatype of each send buffer element (handle).
-.TP 1i
-dest
-Rank of destination (integer).
-.TP 1i
-tag
-Message tag (integer).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-request
-Communication request (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Irsend starts a ready-mode nonblocking send. Nonblocking calls allocate a communication request object and associate it with the request handle (the argument request). The request can be used later to query the status of the communication or to wait for its completion.
-.sp
-A nonblocking send call indicates that the system may start copying data out of the send buffer. The sender should not modify any part of the send buffer after a nonblocking send operation is called, until the send completes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-MPI_Rsend
diff --git a/ompi/mpi/man/man3/MPI_Is_thread_main.3in b/ompi/mpi/man/man3/MPI_Is_thread_main.3in
deleted file mode 100644
index 5a7c6218d70..00000000000
--- a/ompi/mpi/man/man3/MPI_Is_thread_main.3in
+++ /dev/null
@@ -1,71 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Is_thread_main 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-
-.SH NAME
-\fBMPI_Is_thread_main\fP \- Determines if thread called MPI_Init
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Is_thread_main(int *\fIflag\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_IS_THREAD_MAIN(\fIFLAG, IERROR\fP)
-	LOGICAL	\fIFLAG\fP
-	INTEGER	\fIIERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Is_thread_main(\fIflag\fP, \fIierror\fP)
-	LOGICAL, INTENT(OUT) :: \fIflag\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH OUTPUT PARAMETERS
-.TP 1i
-flag
-True if calling thread is main thread (boolean).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Is_thread_main is called by a thread to find out whether the
-caller is the main thread (that is, the thread that called MPI_Init or
-MPI_Init_thread).
-
-.SH ERRORS
-.ft R
-Almost all MPI routines return an error value; C routines as
-the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-.sp
-See the MPI man page for a full list of MPI error codes.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Init
-MPI_Init_thread
-
diff --git a/ompi/mpi/man/man3/MPI_Iscan.3in b/ompi/mpi/man/man3/MPI_Iscan.3in
deleted file mode 100644
index 42cdcd65e92..00000000000
--- a/ompi/mpi/man/man3/MPI_Iscan.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Scan.3
diff --git a/ompi/mpi/man/man3/MPI_Iscatter.3in b/ompi/mpi/man/man3/MPI_Iscatter.3in
deleted file mode 100644
index 05572bc5ca1..00000000000
--- a/ompi/mpi/man/man3/MPI_Iscatter.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Scatter.3
diff --git a/ompi/mpi/man/man3/MPI_Iscatterv.3in b/ompi/mpi/man/man3/MPI_Iscatterv.3in
deleted file mode 100644
index 86a7f302ccf..00000000000
--- a/ompi/mpi/man/man3/MPI_Iscatterv.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Scatterv.3
diff --git a/ompi/mpi/man/man3/MPI_Isend.3in b/ompi/mpi/man/man3/MPI_Isend.3in
deleted file mode 100644
index c10f4292a7a..00000000000
--- a/ompi/mpi/man/man3/MPI_Isend.3in
+++ /dev/null
@@ -1,94 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2007-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Isend 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Isend\fP \- Starts a standard-mode, nonblocking send.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Isend(const void *\fIbuf\fP, int\fI count\fP, MPI_Datatype\fI datatype\fP, int\fI dest\fP,
-	int\fI tag\fP, MPI_Comm\fI comm\fP, MPI_Request\fI *request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_ISEND(\fIBUF, COUNT, DATATYPE, DEST, TAG, COMM, REQUEST, IERROR\fP)
-	<type>	\fIBUF\fP(*)
-	INTEGER	\fICOUNT, DATATYPE, DEST, TAG, COMM, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Isend(\fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIdest\fP, \fItag\fP, \fIcomm\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP, \fIdest\fP, \fItag\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of send buffer (choice).
-.TP 1i
-count
-Number of elements in send buffer (integer).
-.TP 1i
-datatype
-Datatype of each send buffer element (handle).
-.TP 1i
-dest
-Rank of destination (integer).
-.TP 1i
-tag
-Message tag (integer).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-request
-Communication request (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Isend starts a standard-mode, nonblocking send. Nonblocking calls allocate a communication request object and associate it with the request handle (the argument request). The request can be used later to query the status of the communication or wait for its completion.
-.sp
-A nonblocking send call indicates that the system may start copying data out of the send buffer. The sender should not modify any part of the send buffer after a nonblocking send operation is called, until the send completes.
-.sp
-A send request can be determined being completed by calling the MPI_Wait, MPI_Waitany, MPI_Test, or MPI_Testany with request returned by this function.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-MPI_Send
-MPI_Wait
-MPI_Waitany
-MPI_Test
-MPI_Testany
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Issend.3in b/ompi/mpi/man/man3/MPI_Issend.3in
deleted file mode 100644
index 03d907aab0a..00000000000
--- a/ompi/mpi/man/man3/MPI_Issend.3in
+++ /dev/null
@@ -1,89 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Issend 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Issend\fP \- Starts a nonblocking synchronous send.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Issend(const void *\fIbuf\fP, int\fI count\fP, MPI_Datatype\fI datatype\fP, int\fI dest\fP,
-	int\fI tag\fP, MPI_Comm\fI comm\fP, MPI_Request\fI *request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_ISSEND(\fIBUF, COUNT, DATATYPE, DEST, TAG, COMM, REQUEST, IERROR\fP)
-	<type>	\fIBUF\fP(*)
-	INTEGER	\fICOUNT, DATATYPE, DEST, TAG, COMM, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Issend(\fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIdest\fP, \fItag\fP, \fIcomm\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP, \fIdest\fP, \fItag\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of send buffer (choice).
-.TP 1i
-count
-Number of elements in send buffer (integer).
-.TP 1i
-datatype
-Datatype of each send buffer element (handle).
-.TP 1i
-dest
-Rank of destination (integer).
-.TP 1i
-tag
-Message tag (integer).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-request
-Communication request (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Starts a synchronous mode, nonblocking send.
-.sp
-Nonblocking calls allocate a communication request object and associate it with the request handle (the argument request). The request can be used later to query the status of the communication or wait for its completion.
-.sp
-A nonblocking send call indicates that the system may start copying data out of the send buffer. The sender should not modify any part of the send buffer after a nonblocking send operation is called, until the send completes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-MPI_Ssend
-.br
diff --git a/ompi/mpi/man/man3/MPI_Keyval_create.3in b/ompi/mpi/man/man3/MPI_Keyval_create.3in
deleted file mode 100644
index 4f49ef18b9b..00000000000
--- a/ompi/mpi/man/man3/MPI_Keyval_create.3in
+++ /dev/null
@@ -1,120 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Keyval_create 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Keyval_create\fP \- Generates a new attribute key -- use of this routine is deprecated.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Keyval_create(MPI_Copy_function *\fIcopy_fn\fP,
-	MPI_Delete_function *\fIdelete_fn\fP, int *\fIkeyval\fP, void *\fIextra_state\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-INCLUDE 'mpif.h'
-MPI_KEYVAL_CREATE(\fICOPY_FN, DELETE_FN, KEYVAL, EXTRA_STATE, IERROR\fP)
-	EXTERNAL	\fICOPY_FN, DELETE_FN\fP
-	INTEGER	\fIKEYVAL, EXTRA_STATE, IERROR\fP
-
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-copy_fn
-Copy callback function for keyval.
-.TP 1i
-delete_fn
-Delete callback function for keyval.
-.TP 1i
-extra_state
-Extra state for callback functions.
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-keyval
-Key value for future access (integer).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Note that use of this routine is \fIdeprecated\fP as of MPI-2. Please use MPI_Comm_create_keyval instead.
-.sp
-Generates a new attribute key. Keys are locally unique in a process and opaque to the user, though they are explicitly stored in integers. Once allocated, the key value can be used to associate attributes and access them on any locally defined communicator.
-.sp
-The copy_fn function is invoked when a communicator is duplicated by MPI_COMM_DUP. copy_fn should be of type MPI_Copy_function, which is defined as follows:
-.sp
-.nf
-  typedef int MPI_Copy_function(MPI_Comm oldcomm, int keyval,
-                                void *extra_state, void *attribute_val_in,
-                                void *attribute_val_out, int *flag)
-
-.fi
-A Fortran declaration for such a function is as follows:
-.sp
-.nf
-  SUBROUTINE COPY_FUNCTION(OLDCOMM, KEYVAL, EXTRA_STATE, ATTRIBUTE_VAL_IN,
-              ATTRIBUTE_VAL_OUT, FLAG, IERR)
-  INTEGER OLDCOMM, KEYVAL, EXTRA_STATE,
-  ATTRIBUTE_VAL_IN, ATTRIBUTE_VAL_OUT, IERR
-  LOGICAL FLAG
-.fi
-.sp
-The copy callback function is invoked for each key value in oldcomm in arbitrary order. Each call to the copy callback is made with a key value and its corresponding attribute. If it returns flag = 0, then the attribute is deleted in the duplicated communicator. Otherwise ( flag = 1), the new attribute value is set to the value returned in attribute_val_out. The function returns MPI_SUCCESS on success and an error code on failure (in which case MPI_Comm_dup will fail).
-.sp
-copy_fn may be specified as MPI_NULL_COPY_FN or MPI_DUP_FN from either C or
-Fortran; MPI_NULL_COPY_FN is a function that does nothing other than return flag = 0, and MPI_SUCCESS. MPI_DUP_FN is a simple-minded copy function that sets flag = 1, returns the value of attribute_val_in in attribute_val_out, and returns MPI_SUCCESS.
-
-.SH NOTES
-Key values are global (available for any and all communicators).
-.sp
-There are subtle differences between C and Fortran that require that the copy_fn be written in the same language that MPI_Keyval_create is called from. This should not be a problem for most users; only programmers using both Fortran and C in the same program need to be sure that they follow this rule.
-.sp
-Even though both formal arguments attribute_val_in
-and attribute_val_out are of type void*, their usage differs. The C copy function is passed by MPI in attribute_val_in the value of the attribute, and in attribute_val_out the address of the attribute, so as to allow the function to return the (new) attribute value. The use of type void* for both is to avoid messy type casts.
-.sp
-A valid copy function is one that completely duplicates the information by making a full duplicate copy of the data structures implied by an attribute; another might just make another reference to that data structure, while using a reference-count mechanism. Other types of attributes might not copy at all (they might be specific to oldcomm only).
-.sp
-Analogous to copy_fn is a callback deletion function, defined as follows. The delete_fn function is invoked when a communicator is deleted by MPI_Comm_free or when a call is made explicitly to MPI_Attr_delete. delete_fn should be of type MPI_Delete_function, which is defined as follows:
-.sp
-.nf
-  typedef int MPI_Delete_function(MPI_Comm comm, int keyval,
-      void *attribute_val, void *extra_state);
-.fi
-.sp
-A Fortran declaration for such a function is as follows:
-.sp
-.nf
-  SUBROUTINE DELETE_FUNCTION(COMM, KEYVAL,ATTRIBUTE_VAL, EXTRA_STATE, IERR)
-      INTEGER COMM, KEYVAL, ATTRIBUTE_VAL, EXTRA_STATE, IERR
-.fi
-.sp
-This function is called by MPI_Comm_free, MPI_Attr_delete, and MPI_Attr_put to do whatever is needed to remove an attribute. The function returns MPI_SUCCESS on success and an error code on failure (in which case MPI_COMM_FREE will fail).
-.sp
-delete_fn may be specified as MPI_NULL_DELETE_FN from either C or FORTRAN; MPI_NULL_DELETE_FN is a function that does nothing, other than returning MPI_SUCCESS.
-.sp
-The special key value MPI_KEYVAL_INVALID is never returned by MPI_Keyval_create. Therefore, it can be used for static initialization of key values.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-MPI_Keyval_free
-.br
-MPI_Comm_create_keyval
-.br
diff --git a/ompi/mpi/man/man3/MPI_Keyval_free.3in b/ompi/mpi/man/man3/MPI_Keyval_free.3in
deleted file mode 100644
index bd26f8b802d..00000000000
--- a/ompi/mpi/man/man3/MPI_Keyval_free.3in
+++ /dev/null
@@ -1,63 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Keyval_free 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Keyval_free\fP \- Frees attribute key for communicator cache attribute -- use of this routine is deprecated.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Keyval_free(int *\fIkeyval\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-INCLUDE 'mpif.h'
-MPI_KEYVAL_FREE(\fIKEYVAL, IERROR\fP)
-	INTEGER	\fIKEYVAL, IERROR\fP
-
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-keyval
-Frees the integer key value (integer).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Note that use of this routine is \fIdeprecated\fP as of MPI-2. Please use MPI_Comm_free_keyval instead.
-.sp
-Frees an extant attribute key. This function sets the value of keyval to  MPI_KEYVAL_INVALID. Note that it is not erroneous to free an attribute key that is in use, because the actual free does not transpire until after all references (in other communicators on the process) to the key have been freed. These references need to be explicitly freed by the program, either via calls to MPI_Attr_delete that free one attribute instance, or by calls to MPI_Comm_free that free all attribute instances associated with the freed communicator.
-
-.SH NOTE
-.ft R
-Key values are global (they can be used with any and all communicators).
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-MPI_Keyval_create
-.br
-MPI_Comm_free_keyval
-.br
-
-
-
-
diff --git a/ompi/mpi/man/man3/MPI_Lookup_name.3in b/ompi/mpi/man/man3/MPI_Lookup_name.3in
deleted file mode 100644
index 8a0367236f6..00000000000
--- a/ompi/mpi/man/man3/MPI_Lookup_name.3in
+++ /dev/null
@@ -1,133 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2007-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Lookup_name 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-
-.SH NAME
-.nf
-\fBMPI_Lookup_name\fP \- Finds port associated with a service name
-
-.fi
-.SH SYNTAX
-.ft R
-
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Lookup_name(const char *\fIservice_name\fP, MPI_Info \fIinfo\fP,
-	char *\fIport_name\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_LOOKUP_NAME(\fISERVICE_NAME, INFO, PORT_NAME, IERROR\fP)
-	CHARACTER*(*)	\fISERVICE_NAME, PORT_NAME\fP
-	INTEGER		\fIINFO, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Lookup_name(\fIservice_name\fP, \fIinfo\fP, \fIport_name\fP, \fIierror\fP)
-	CHARACTER(LEN=*), INTENT(IN) :: \fIservice_name\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	CHARACTER(LEN=MPI_MAX_PORT_NAME), INTENT(OUT) :: \fIport_name\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1.4i
-service_name
-A service name (string).
-.TP 1.4i
-info
-Options to the name service functions (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1.4i
-port_name
-a port name (string).
-.TP 1.4i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-This function retrieves a \fIport_name\fP published under
-\fIservice_name\fP by a previous invocation of MPI_Publish_name. The
-application must supply a \fIport_name\fP buffer large enough to hold
-the largest possible port name (i.e., MPI_MAX_PORT_NAME bytes).
-
-.SH INFO ARGUMENTS
-The following keys for \fIinfo\fP are recognized:
-.sp
-.sp
-.nf
-Key                   Type      Description
----                   ----      -----------
-
-ompi_lookup_order     char *    Resolution order for name lookup.
-.fi
-
-The \fIompi_lookup_order\fP info key can specify one of four valid
-string values (see the NAME SCOPE section below for more information
-on name scopes):
-
-.TP 4
-\fIlocal\fP: Only search the local scope for name resolution.
-.TP 4
-\fIglobal\fP: Only search the global scope for name resolution.
-.TP 4
-\fIlocal,global\fP: Search the local scope for name resolution.  If
-not found, try searching the global scope for name resolution.  This
-behavior is the default if the \fIompi_lookup_order\fP info key is not
-specified.
-.TP 4
-\fIglobal,local\fP: Search the global scope for name resolution.  If
-not found, try searching the local scope for name resolution.
-.PP
-If no info key is provided, the search will first check to see if a
-global server has been specified and is available. If so, then the
-search will default to global scope first, followed by local. Otherwise,
-the search will default to local.
-
-.SH NAME SCOPE
-Open MPI supports two name scopes: \fIglobal\fP and \fIlocal\fP. Local scope
-values are placed in a data store located on the mpirun of the calling
-process' job, while global scope values reside on a central server. Calls
-to MPI_Unpublish_name must correctly specify the scope to be used in
-finding the value to be removed. The function will return an error if
-the specified service name is not found on the indicated location.
-.sp
-For a more detailed description of scoping rules, please see the MPI_Publish_name
-man page.
-
-.SH ERRORS
-.ft R
-Almost all MPI routines return an error value; C routines as
-the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-.sp
-See the MPI man page for a full list of MPI error codes.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Publish_name
-MPI_Open_port
-
-
diff --git a/ompi/mpi/man/man3/MPI_Message_c2f.3in b/ompi/mpi/man/man3/MPI_Message_c2f.3in
deleted file mode 100644
index a13fce697dd..00000000000
--- a/ompi/mpi/man/man3/MPI_Message_c2f.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Comm_f2c.3
diff --git a/ompi/mpi/man/man3/MPI_Message_f2c.3in b/ompi/mpi/man/man3/MPI_Message_f2c.3in
deleted file mode 100644
index a13fce697dd..00000000000
--- a/ompi/mpi/man/man3/MPI_Message_f2c.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Comm_f2c.3
diff --git a/ompi/mpi/man/man3/MPI_Mprobe.3in b/ompi/mpi/man/man3/MPI_Mprobe.3in
deleted file mode 100644
index 32f94fb54e6..00000000000
--- a/ompi/mpi/man/man3/MPI_Mprobe.3in
+++ /dev/null
@@ -1,107 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
-.\" Copyright 2012 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Mprobe 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Mprobe\fP \- Blocking matched probe for a message.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Mprobe(int \fIsource\fP, int\fI tag\fP, MPI_Comm\fI comm\fP,
-	MPI_Message\fI *message\fP, MPI_Status\fI *status\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_MPROBE(\fISOURCE, TAG, COMM, MESSAGE, STATUS, IERROR\fP)
-	INTEGER	\fISOURCE, TAG, COMM, MESSAGE\fP
-	INTEGER	\fISTATUS(MPI_STATUS_SIZE), IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Mprobe(\fIsource\fP, \fItag\fP, \fIcomm\fP, \fImessage\fP, \fIstatus\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIsource\fP, \fItag\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Message), INTENT(OUT) :: \fImessage\fP
-	TYPE(MPI_Status) :: \fIstatus\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-source
-Source rank or MPI_ANY_SOURCE (integer).
-.TP 1i
-tag
-Tag value or MPI_ANY_TAG (integer).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-message
-Message (handle).
-.ft R
-.TP 1i
-status
-Status object (status).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Like MPI_Probe and MPI_Iprobe, the MPI_Mprobe and MPI_Improbe operations
-allow incoming messages to be queried without actually receiving
-them, except that MPI_Mprobe and MPI_Improbe provide a mechanism to
-receive the specific message that was matched regardless of other
-intervening probe or receive operations.  This gives the application
-an opportunity to decide how to receive the message, based on the
-information returned by the probe.  In particular, the application may
-allocate memory for the receive buffer according to the length of the
-probed message.
-.sp
-A matching probe with MPI_PROC_NULL as \fIsource\fP returns
-\fImessage\fP = MPI_MESSAGE_NO_PROC, and the \fIstatus\fP object returns
-source = MPI_PROC_NULL, tag = MPI_ANY_TAG, and count = 0.
-.sp
-When MPI_Mprobe returns (from a non-MPI_PROC_NULL \fIsource\fP), the
-matched message can then be received by passing the \fImessage\fP
-handle to the MPI_Mrecv or MPI_Imrecv functions.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler
-MPI_ERRORS_RETURN may be used to cause error values to be
-returned. Note that MPI does not guarantee that an MPI program can
-continue past an error.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Improbe
-MPI_Probe
-MPI_Iprobe
-MPI_Mrecv
-MPI_Imrecv
-MPI_Cancel
diff --git a/ompi/mpi/man/man3/MPI_Mrecv.3in b/ompi/mpi/man/man3/MPI_Mrecv.3in
deleted file mode 100644
index f8ef9660727..00000000000
--- a/ompi/mpi/man/man3/MPI_Mrecv.3in
+++ /dev/null
@@ -1,96 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2012 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Mrecv 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Mrecv\fP \- Blocking receive for a matched message
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Mrecv(void \fI*buf\fP, int\fI count\fP, MPI_Datatype\fI type\fP,
-	MPI_Message\fI *message\fP, MPI_Status\fI *status\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_MRECV(\fIBUF, COUNT, DATATYPE, MESSAGE, STATUS, IERROR\fP)
-	<type>	\fIBUF(*)\fP
-	INTEGER	\fICOUNT, DATATYPE, MESSAGE\fP
-	INTEGER	\fISTATUS(MPI_STATUS_SIZE), IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Mrecv(\fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fImessage\fP, \fIstatus\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..) :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Message), INTENT(INOUT) :: \fImessage\fP
-	TYPE(MPI_Status) :: \fIstatus\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-count
-Number of elements to receive (nonnegative integer).
-.TP 1i
-datatype
-Datatype of each send buffer element (handle).
-.TP 1i
-message
-Message (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of receive buffer (choice).
-.TP 1i
-status
-Status object (status).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-The functions MPI_Mrecv and MPI_Imrecv receive messages that have been
-previously matched by a matching probe.
-.sp
-If MPI_Mrecv is called with MPI_MESSAGE_NULL as the message argument,
-the call returns immediately with the \fIstatus\fP object set to
-\fIsource\fP = MPI_PROC_NULL, \fItag\fP = MPI_ANY_TAG, and \fIcount\fP
-= 0, as if a receive from MPI_PROC_NULL was issued.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler
-MPI_ERRORS_RETURN may be used to cause error values to be
-returned. Note that MPI does not guarantee that an MPI program can
-continue past an error.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Mprobe
-MPI_Improbe
-MPI_Probe
-MPI_Iprobe
-MPI_Imrecv
-MPI_Cancel
diff --git a/ompi/mpi/man/man3/MPI_Neighbor_allgather.3in b/ompi/mpi/man/man3/MPI_Neighbor_allgather.3in
deleted file mode 100644
index e152d747ce8..00000000000
--- a/ompi/mpi/man/man3/MPI_Neighbor_allgather.3in
+++ /dev/null
@@ -1,149 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" $COPYRIGHT$
-.TH MPI_Neighbor_allgather 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Neighbor_allgather, MPI_Ineighbor_allgather\fP \- Gathers and distributes data from and to all neighbors
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Neighbor_allgather(const void\fI *sendbuf\fP, int \fI sendcount\fP,
-	MPI_Datatype\fI sendtype\fP, void\fI *recvbuf\fP, int\fI recvcount\fP,
-	 MPI_Datatype\fI recvtype\fP, MPI_Comm\fI comm\fP)
-
-int MPI_Ineighbor_allgather(const void\fI *sendbuf\fP, int \fI sendcount\fP,
-	MPI_Datatype\fI sendtype\fP, void\fI *recvbuf\fP, int\fI recvcount\fP,
-	 MPI_Datatype\fI recvtype\fP, MPI_Comm\fI comm\fP, MPI_Request \fIreq\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_NEIGHBOR_ALLGATHER(\fISENDBUF\fP,\fI SENDCOUNT\fP,\fI SENDTYPE\fP,\fI RECVBUF\fP,\fI RECVCOUNT\fP,\fI
-		RECVTYPE\fP,\fI COMM\fP,\fI IERROR\fP)
-	<type>	\fISENDBUF\fP (*), \fIRECVBUF\fP (*)
-	INTEGER	\fISENDCOUNT\fP,\fI SENDTYPE\fP,\fI RECVCOUNT\fP,\fI RECVTYPE\fP,\fI COMM\fP,
-	INTEGER	\fIIERROR\fP
-
-MPI_INEIGHBOR_ALLGATHER(\fISENDBUF\fP,\fI SENDCOUNT\fP,\fI SENDTYPE\fP,\fI RECVBUF\fP,\fI RECVCOUNT\fP,\fI
-		RECVTYPE\fP,\fI COMM\fP, \fIREQUEST\fP,\fI IERROR\fP)
-	<type>	\fISENDBUF\fP (*), \fIRECVBUF\fP (*)
-	INTEGER	\fISENDCOUNT\fP,\fI SENDTYPE\fP,\fI RECVCOUNT\fP,\fI RECVTYPE\fP,\fI COMM\fP,
-	INTEGER	\fIREQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Neighbor_allgather(\fIsendbuf\fP, \fIsendcount\fP, \fIsendtype\fP, \fIrecvbuf\fP, \fIrecvcount\fP,
-		\fIrecvtype\fP, \fIcomm\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..) :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIsendcount\fP, \fIrecvcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPI_Ineighbor_allgather(\fIsendbuf\fP, \fIsendcount\fP, \fIsendtype\fP, \fIrecvbuf\fP, \fIrecvcount\fP,
-		\fIrecvtype\fP, \fIcomm\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIsendcount\fP, \fIrecvcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-sendbuf
-Starting address of send buffer (choice).
-.TP 1i
-sendcount
-Number of elements in send buffer (integer).
-.TP 1i
-sendtype
-Datatype of send buffer elements (handle).
-.TP 1i
-recvbuf
-Starting address of recv buffer (choice).
-.TP 1i
-recvcount
-Number of elements received from any process (integer).
-.TP 1i
-recvtype
-Datatype of receive buffer elements (handle).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-recvbuf
-Address of receive buffer (choice).
-.TP 1i
-request
-Request (handle, non-blocking only).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Neighbor_allgather is similar to MPI_Allgather, except that only the neighboring processes receive the result, instead of all processes. The neighbors and buffer layout is determined by the topology of \fIcomm\fP.
-.sp
-The type signature associated with sendcount, sendtype at a process must be equal to the type signature associated with recvcount, recvtype at any other process.
-.fi
-
-.sp
-.SH NEIGHBOR ORDERING
-For a distributed graph topology, created with MPI_Dist_graph_create, the sequence of neighbors
-in the send and receive buffers at each process is defined as the sequence returned by MPI_Dist_graph_neighbors
-for destinations and sources, respectively. For a general graph topology, created with MPI_Graph_create, the order of
-neighbors in the send and receive buffers is defined as the sequence of neighbors as returned by MPI_Graph_neighbors.
-Note that general graph topologies should generally be replaced by the distributed graph topologies.
-
-For a Cartesian topology, created with MPI_Cart_create, the sequence of neighbors in the send and receive
-buffers at each process is defined by order of the dimensions, first the neighbor in the negative direction
-and then in the positive direction with displacement 1. The numbers of sources and destinations in the
-communication routines are 2*ndims with ndims defined in MPI_Cart_create. If a neighbor does not exist, i.e., at
-the border of a Cartesian topology in the case of a non-periodic virtual grid dimension (i.e.,
-periods[...]==false), then this neighbor is defined to be MPI_PROC_NULL.
-
-If a neighbor in any of the functions is MPI_PROC_NULL, then the neighborhood collective communication behaves
-like a point-to-point communication with MPI_PROC_NULL in this direction. That is, the buffer is still part of
-the sequence of neighbors but it is neither communicated nor updated.
-
-.SH NOTES
-.sp
-The MPI_IN_PLACE option for \fIsendbuf\fP is not meaningful for this operation.
-
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler
-may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Neighbor_allgatherv
-MPI_Cart_create
-MPI_Garph_create
-MPI_Dist_graph_create
-.br
-MPI_Gather
-
diff --git a/ompi/mpi/man/man3/MPI_Neighbor_allgatherv.3in b/ompi/mpi/man/man3/MPI_Neighbor_allgatherv.3in
deleted file mode 100644
index fdc3e02d157..00000000000
--- a/ompi/mpi/man/man3/MPI_Neighbor_allgatherv.3in
+++ /dev/null
@@ -1,145 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2007-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" $COPYRIGHT$
-.TH MPI_Neighbor_allgatherv 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Neighbor_allgatherv, MPI_Ineighbor_allgatherv\fP \- Gathers and distributes data from and to all neighbors. Each process may contribute a different amount of data.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Neighbor_allgatherv(const void\fI *sendbuf\fP, int\fI sendcount\fP,
-	MPI_Datatype\fI sendtype\fP, void\fI *recvbuf\fP, const int\fI recvcounts[]\fP,
-	const int\fI displs[]\fP, MPI_Datatype\fI recvtype\fP, MPI_Comm\fI comm\fP)
-
-int MPI_Ineighbor_allgatherv(const void\fI *sendbuf\fP, int\fI sendcount\fP,
-	MPI_Datatype\fI sendtype\fP, void\fI *recvbuf\fP, const int\fI recvcounts[]\fP,
-	const int\fI displs[]\fP, MPI_Datatype\fI recvtype\fP, MPI_Comm\fI comm\fP,
-        MPI_Request \fI*request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_NEIGHBOR_ALLGATHERV(\fISENDBUF\fP,\fI SENDCOUNT\fP, \fISENDTYPE\fP,\fI RECVBUF\fP,
-		\fIRECVCOUNT\fP,\fI DISPLS\fP, \fIRECVTYPE\fP,\fI COMM\fP,\fI IERROR\fP)
-	<type>	\fISENDBUF\fP(*), \fIRECVBUF\fP(*)
-	INTEGER	\fISENDCOUNT\fP,\fI SENDTYPE\fP, \fIRECVCOUNT\fP(*),
-	INTEGER	\fIDISPLS\fP(*),\fI RECVTYPE\fP,\fI COMM\fP,\fI IERROR\fP
-
-MPI_INEIGHBOR_ALLGATHERV(\fISENDBUF\fP,\fI SENDCOUNT\fP, \fISENDTYPE\fP,\fI RECVBUF\fP,
-		\fIRECVCOUNT\fP,\fI DISPLS\fP, \fIRECVTYPE\fP,\fI COMM\fP,\fI REQUEST\fP,\fI IERROR\fP)
-	<type>	\fISENDBUF\fP(*), \fIRECVBUF\fP(*)
-	INTEGER	\fISENDCOUNT\fP,\fI SENDTYPE\fP, \fIRECVCOUNT\fP(*),
-	INTEGER	\fIDISPLS\fP(*),\fI RECVTYPE\fP,\fI COMM\fP,\fIREQUEST\fP,\fI IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Neighbor_allgatherv(\fIsendbuf\fP, \fIsendcount\fP, \fIsendtype\fP, \fIrecvbuf\fP, \fIrecvcounts\fP,
-		\fIdispls\fP, \fIrecvtype\fP, \fIcomm\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..) :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIsendcount\fP, \fIrecvcounts(*)\fP, \fIdispls(*)\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPI_Ineighbor_allgatherv(\fIsendbuf\fP, \fIsendcount\fP, \fIsendtype\fP, \fIrecvbuf\fP, \fIrecvcounts\fP,
-		\fIdispls\fP, \fIrecvtype\fP, \fIcomm\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIsendcount\fP
-	INTEGER, INTENT(IN), ASYNCHRONOUS :: \fIrecvcounts(*)\fP, \fIdispls(*)\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-sendbuf
-Starting address of send buffer (choice).
-.TP 1i
-sendcount
-Number of elements in send buffer (integer).
-.TP 1i
-sendtype
-Datatype of send buffer elements (handle).
-.TP 1i
-recvcount
-Integer array (of length group size) containing the number of elements that are received from each neighbor.
-.TP 1i
-displs
-Integer array (of length group size). Entry i specifies the displacement (relative to recvbuf) at which to place the incoming data from neighbor i.
-.TP 1i
-recvtype
-Datatype of receive buffer elements (handle).
-.TP 1i
-comm
-Communicator (handle).
-.sp
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-recvbuf
-Address of receive buffer (choice).
-.TP 1i
-request
-Request (handle, non-blocking only).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Neighbor_allgatherv is similar to MPI_Neighbor_allgather in that all processes gather data from all neighbors, except that each process can send a different amount of data. The block of data sent from the jth neighbor is received by every neighbor and placed in the jth block of the buffer. The neighbors and buffer layout is determined by the topology of \fIcomm\fP.
-.I recvbuf.
-.sp
-The type signature associated with sendcount, sendtype, at process j must be equal to the type signature associated with the corresponding entry in \fIrecvcounts\fP on neighboring processes.
-
-.sp
-.SH NEIGHBOR ORDERING
-For a distributed graph topology, created with MPI_Dist_graph_create, the sequence of neighbors
-in the send and receive buffers at each process is defined as the sequence returned by MPI_Dist_graph_neighbors
-for destinations and sources, respectively. For a general graph topology, created with MPI_Graph_create, the order of
-neighbors in the send and receive buffers is defined as the sequence of neighbors as returned by MPI_Graph_neighbors.
-Note that general graph topologies should generally be replaced by the distributed graph topologies.
-
-For a Cartesian topology, created with MPI_Cart_create, the sequence of neighbors in the send and receive
-buffers at each process is defined by order of the dimensions, first the neighbor in the negative direction
-and then in the positive direction with displacement 1. The numbers of sources and destinations in the
-communication routines are 2*ndims with ndims defined in MPI_Cart_create. If a neighbor does not exist, i.e., at
-the border of a Cartesian topology in the case of a non-periodic virtual grid dimension (i.e.,
-periods[...]==false), then this neighbor is defined to be MPI_PROC_NULL.
-
-If a neighbor in any of the functions is MPI_PROC_NULL, then the neighborhood collective communication behaves
-like a point-to-point communication with MPI_PROC_NULL in this direction. That is, the buffer is still part of
-the sequence of neighbors but it is neither communicated nor updated.
-
-.SH NOTES
-The MPI_IN_PLACE option for \fIsendbuf\fP is not meaningful for this operation.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler
-may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-MPI_Neighbor_allgather
-MPI_Cart_create
-MPI_Graph_create
-MPI_Dist_graph_create
diff --git a/ompi/mpi/man/man3/MPI_Neighbor_alltoall.3in b/ompi/mpi/man/man3/MPI_Neighbor_alltoall.3in
deleted file mode 100644
index 2f867a82ec3..00000000000
--- a/ompi/mpi/man/man3/MPI_Neighbor_alltoall.3in
+++ /dev/null
@@ -1,193 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" $COPYRIGHT$
-.TH MPI_Neighbor_alltoall 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-
-.SH NAME
-\fBMPI_Neighbor_alltoall, MPI_Ineighbor_alltoall\fP \- All processes send data to neighboring processes in a virtual topology communicator
-
-.SH SYNTAX
-.ft R
-
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Neighbor_alltoall(const void *\fIsendbuf\fP, int \fIsendcount\fP,
-	MPI_Datatype \fIsendtype\fP, void *\fIrecvbuf\fP, int \fIrecvcount\fP,
-	MPI_Datatype \fIrecvtype\fP, MPI_Comm \fIcomm\fP)
-
-int MPI_Ineighbor_alltoall(const void *\fIsendbuf\fP, int \fIsendcount\fP,
-	MPI_Datatype \fIsendtype\fP, void *\fIrecvbuf\fP, int \fIrecvcount\fP,
-	MPI_Datatype \fIrecvtype\fP, MPI_Comm \fIcomm\fP, MPI_Request \fI*request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_NEIGHBOR_ALLTOALL(\fISENDBUF, SENDCOUNT, SENDTYPE, RECVBUF, RECVCOUNT,
-	RECVTYPE, COMM, IERROR\fP)
-
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fISENDCOUNT, SENDTYPE, RECVCOUNT, RECVTYPE\fP
-	INTEGER	\fICOMM, IERROR\fP
-
-MPI_INEIGHBOR_ALLTOALL(\fISENDBUF, SENDCOUNT, SENDTYPE, RECVBUF, RECVCOUNT,
-	RECVTYPE, COMM, REQUEST, IERROR\fP)
-
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fISENDCOUNT, SENDTYPE, RECVCOUNT, RECVTYPE\fP
-	INTEGER	\fICOMM, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Neighbor_alltoall(\fIsendbuf\fP, \fIsendcount\fP, \fIsendtype\fP, \fIrecvbuf\fP, \fIrecvcount\fP,
-		\fIrecvtype\fP, \fIcomm\fP, \fIierror\fP)
-
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..) :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIsendcount\fP, \fIrecvcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPI_Ineighbor_alltoall(\fIsendbuf\fP, \fIsendcount\fP, \fIsendtype\fP, \fIrecvbuf\fP, \fIrecvcount\fP,
-		\fIrecvtype\fP, \fIcomm\fP, \fIrequest\fP, \fIierror\fP)
-
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIsendcount\fP, \fIrecvcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1.2i
-sendbuf
-Starting address of send buffer (choice).
-.TP 1.2i
-sendcount
-Number of elements to send to each process (integer).
-.TP 1.2i
-sendtype
-Datatype of send buffer elements (handle).
-.TP 1.2i
-recvcount
-Number of elements to receive from each process (integer).
-.TP 1.2i
-recvtype
-Datatype of receive buffer elements (handle).
-.TP 1.2i
-comm
-Communicator over which data is to be exchanged (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1.2i
-recvbuf
-Starting address of receive buffer (choice).
-.TP 1i
-request
-Request (handle, non-blocking only).
-.ft R
-.TP 1.2i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Neighbor_alltoall is a collective operation in which all processes send and receive the same amount of data to each neighbor. The operation of this routine can be represented as follows, where each process performs 2n (n being the number of neighbors in communicator \fIcomm\fP) independent point-to-point communications. The neighbors and buffer layout are determined by the topology of \fIcomm\fP.
-.sp
-Example of MPI_Neighbor_alltoall semantics for cartesian topologies:
-.sp
-.nf
-        MPI_Cart_get(\fIcomm\fP, maxdims, dims, periods, coords);
-        for (dim = 0, i = 0 ; dim < dims ; ++dim) {
-            MPI_Cart_shift(\fIcomm\fP, dim, 1, &r0, &r1);
-            MPI_Isend(\fIsendbuf\fP + i * \fIsendcount\fP * extent(\fIsendtype\fP),
-                      \fIsendcount\fP, \fIsendtype\fP, r0, ..., \fIcomm\fP, ...);
-            MPI_Irecv(\fIrecvbuf\fP + i * \fIrecvcount\fP * extent(\fIrecvtype\fP),
-                      \fIrecvcount\fP, \fIrecvtype\fP, r0, ..., \fIcomm\fP, ...);
-            ++i;
-            MPI_Isend(\fIsendbuf\fP + i * \fIsendcount\fP * extent(\fIsendtype\fP),
-                      \fIsendcount\fP, \fIsendtype\fP, r1, ..., \fIcomm\fP, &req[i]);
-            MPI_Irecv(\fIrecvbuf\fP + i * \fIrecvcount\fP * extent(\fIrecvtype\fP),
-                      \fIrecvcount\fP, \fIrecvtype\fP, r1, ..., \fIcomm\fP, ...);
-            ++i;
-        }
-
-        MPI_Waitall (...);
-.fi
-.sp
-Each process breaks up its local \fIsendbuf\fP into n blocks \- each
-containing \fIsendcount\fP elements of type \fIsendtype\fP \- and
-divides its \fIrecvbuf\fP similarly according to \fIrecvcount\fP and
-\fIrecvtype\fP. Process j sends the k-th block of its local
-\fIsendbuf\fP to neighbor k, which places the data in the j-th block of
-its local \fIrecvbuf\fP. The amount of data sent must be equal to the
-amount of data received, pairwise, between every pair of processes.
-
-.sp
-.SH NEIGHBOR ORDERING
-For a distributed graph topology, created with MPI_Dist_graph_create, the sequence of neighbors
-in the send and receive buffers at each process is defined as the sequence returned by MPI_Dist_graph_neighbors
-for destinations and sources, respectively. For a general graph topology, created with MPI_Graph_create, the order of
-neighbors in the send and receive buffers is defined as the sequence of neighbors as returned by MPI_Graph_neighbors.
-Note that general graph topologies should generally be replaced by the distributed graph topologies.
-
-For a Cartesian topology, created with MPI_Cart_create, the sequence of neighbors in the send and receive
-buffers at each process is defined by order of the dimensions, first the neighbor in the negative direction
-and then in the positive direction with displacement 1. The numbers of sources and destinations in the
-communication routines are 2*ndims with ndims defined in MPI_Cart_create. If a neighbor does not exist, i.e., at
-the border of a Cartesian topology in the case of a non-periodic virtual grid dimension (i.e.,
-periods[...]==false), then this neighbor is defined to be MPI_PROC_NULL.
-
-If a neighbor in any of the functions is MPI_PROC_NULL, then the neighborhood collective communication behaves
-like a point-to-point communication with MPI_PROC_NULL in this direction. That is, the buffer is still part of
-the sequence of neighbors but it is neither communicated nor updated.
-
-.sp
-.SH NOTES
-.ft R
-The MPI_IN_PLACE option for \fIsendbuf\fP is not meaningful for this function.
-.sp
-All arguments on all processes are significant. The \fIcomm\fP argument,
-in particular, must describe the same communicator on all processes. \fIcomm\fP
-must be either a cartesian, graph, or dist graph communicator.
-.sp
-There are two MPI library functions that are more general than
-MPI_Neighbor_alltoall. MPI_Neighbor_alltoallv allows all-to-all communication to and
-from buffers that need not be contiguous; different processes may
-send and receive different amounts of data. MPI_Neighbor_alltoallw expands
-MPI_Neighbor_alltoallv's functionality to allow the exchange of data with
-different datatypes.
-
-.SH ERRORS
-.ft R
-Almost all MPI routines return an error value; C routines as
-the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Neighbor_alltoallv
-MPI_Neighbor_alltoallw
-MPI_Cart_create
-MPI_Graph_create
-MPI_Dist_graph_create
-MPI_Dist_graph_create_adjacent
diff --git a/ompi/mpi/man/man3/MPI_Neighbor_alltoallv.3in b/ompi/mpi/man/man3/MPI_Neighbor_alltoallv.3in
deleted file mode 100644
index aaf678813ab..00000000000
--- a/ompi/mpi/man/man3/MPI_Neighbor_alltoallv.3in
+++ /dev/null
@@ -1,219 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" $COPYRIGHT$
-.TH MPI_Neighbor_alltoallv 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-
-.SH NAME
-\fBMPI_Neighbor_alltoallv, MPI_Ineighbor_alltoallv\fP \- All processes send different amounts of data to, and receive different amounts of data from, all neighbors
-.SH SYNTAX
-.ft R
-
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Neighbor_alltoallv(const void *\fIsendbuf\fP, const int \fIsendcounts\fP[],
-	const int \fIsdispls\fP[], MPI_Datatype \fIsendtype\fP,
-	void *\fIrecvbuf\fP, const int\fI recvcounts\fP[],
-	const int \fIrdispls\fP[], MPI_Datatype \fIrecvtype\fP, MPI_Comm \fIcomm\fP)
-
-int MPI_Ineighbor_alltoallv(const void *\fIsendbuf\fP, const int \fIsendcounts\fP[],
-	const int \fIsdispls\fP[], MPI_Datatype \fIsendtype\fP,
-	void *\fIrecvbuf\fP, const int\fI recvcounts\fP[],
-	const int \fIrdispls\fP[], MPI_Datatype \fIrecvtype\fP, MPI_Comm \fIcomm\fP,
-	MPI_Request \fI*request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_NEIGHBOR_ALLTOALLV(\fISENDBUF, SENDCOUNTS, SDISPLS, SENDTYPE,
-	RECVBUF, RECVCOUNTS, RDISPLS, RECVTYPE, COMM, IERROR\fP)
-
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fISENDCOUNTS(*), SDISPLS(*), SENDTYPE\fP
-	INTEGER	\fIRECVCOUNTS(*), RDISPLS(*), RECVTYPE\fP
-	INTEGER	\fICOMM, IERROR\fP
-
-MPI_INEIGHBOR_ALLTOALLV(\fISENDBUF, SENDCOUNTS, SDISPLS, SENDTYPE,
-	RECVBUF, RECVCOUNTS, RDISPLS, RECVTYPE, COMM, REQUEST, IERROR\fP)
-
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fISENDCOUNTS(*), SDISPLS(*), SENDTYPE\fP
-	INTEGER	\fIRECVCOUNTS(*), RDISPLS(*), RECVTYPE\fP
-	INTEGER	\fICOMM, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Neighbor_alltoallv(\fIsendbuf\fP, \fIsendcounts\fP, \fIsdispls\fP, \fIsendtype\fP, \fIrecvbuf\fP,
-		\fIrecvcounts\fP, \fIrdispls\fP, \fIrecvtype\fP, \fIcomm\fP, \fIierror\fP)
-
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..) :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIsendcounts(*)\fP, \fIsdispls(*)\fP, \fIrecvcounts(*),\fP
-	\fIrdispls(*)\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPI_Ineighbor_alltoallv(\fIsendbuf\fP, \fIsendcounts\fP, \fIsdispls\fP, \fIsendtype\fP, \fIrecvbuf\fP,
-		\fIrecvcounts\fP, \fIrdispls\fP, \fIrecvtype\fP, \fIcomm\fP, \fIrequest\fP, \fIierror\fP)
-
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN), ASYNCHRONOUS :: \fIsendcounts(*)\fP, \fIsdispls(*),\fP
-	\fIrecvcounts(*)\fP, \fIrdispls(*)\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1.2i
-sendbuf
-Starting address of send buffer.
-.TP 1.2i
-sendcounts
-Integer array, where entry i specifies the number of elements to send
-to neighbor i.
-.TP 1.2i
-sdispls
-Integer array, where entry i specifies the displacement (offset from
-\fIsendbuf\fP, in units of \fIsendtype\fP) from which to send data to
-neighbor i.
-.TP 1.2i
-sendtype
-Datatype of send buffer elements.
-.TP 1.2i
-recvcounts
-Integer array, where entry j specifies the number of elements to
-receive from neighbor j.
-.TP 1.2i
-rdispls
-Integer array, where entry j specifies the displacement (offset from
-\fIrecvbuf\fP, in units of \fIrecvtype\fP) to which data from neighbor j
-should be written.
-.TP 1.2i
-recvtype
-Datatype of receive buffer elements.
-.TP 1.2i
-comm
-Communicator over which data is to be exchanged.
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1.2i
-recvbuf
-Address of receive buffer.
-.TP 1i
-request
-Request (handle, non-blocking only).
-.ft R
-.TP 1.2i
-IERROR
-Fortran only: Error status.
-
-.SH DESCRIPTION
-.ft R
-MPI_Neighbor_alltoallv is a generalized collective operation in which all
-processes send data to and receive data from all neighbors. It
-adds flexibility to MPI_Neighbor_alltoall by allowing the user to specify data
-to send and receive vector-style (via a displacement and element
-count). The operation of this routine can be thought of as follows,
-where each process performs 2n (n being the number of neighbors in
-to topology of communicator \fIcomm\fP) independent point-to-point communications.
-The neighbors and buffer layout are determined by the topology of \fIcomm\fP.
-.sp
-.nf
-        MPI_Cart_get(\fIcomm\fP, maxdims, dims, periods, coords);
-        for (dim = 0, i = 0 ; dim < dims ; ++dim) {
-            MPI_Cart_shift(\fIcomm\fP, dim, 1, &r0, &r1);
-            MPI_Isend(\fIsendbuf\fP + \fIsdispls\fP[i]  * extent(\fIsendtype\fP),
-                      \fIsendcount\fP, \fIsendtype\fP, r0, ..., \fIcomm\fP, ...);
-            MPI_Irecv(\fIrecvbuf\fP + \fIrdispls\fP[i] * extent(\fIrecvtype\fP),
-                      \fIrecvcount\fP, \fIrecvtype\fP, r0, ..., \fIcomm\fP, ...);
-            ++i;
-            MPI_Isend(\fIsendbuf\fP + \fIsdispls\fP[i] * extent(\fIsendtype\fP),
-                      \fIsendcount\fP, \fIsendtype\fP, r1, ..., \fIcomm\fP, &req[i]);
-            MPI_Irecv(\fIrecvbuf\fP + \fIrdispls\fP[i] * extent(\fIrecvtype\fP),
-                      \fIrecvcount\fP, \fIrecvtype\fP, r1, ..., \fIcomm\fP, ...);
-            ++i;
-        }
-.fi
-.sp
-Process j sends the k-th block of its local \fIsendbuf\fP to neighbor
-k, which places the data in the j-th block of its local
-\fIrecvbuf\fP.
-.sp
-When a pair of processes exchanges data, each may pass different
-element count and datatype arguments so long as the sender specifies
-the same amount of data to send (in bytes) as the receiver expects
-to receive.
-.sp
-Note that process i may send a different amount of data to process j
-than it receives from process j. Also, a process may send entirely
-different amounts of data to different processes in the communicator.
-
-.sp
-.SH NEIGHBOR ORDERING
-For a distributed graph topology, created with MPI_Dist_graph_create, the sequence of neighbors
-in the send and receive buffers at each process is defined as the sequence returned by MPI_Dist_graph_neighbors
-for destinations and sources, respectively. For a general graph topology, created with MPI_Graph_create, the order of
-neighbors in the send and receive buffers is defined as the sequence of neighbors as returned by MPI_Graph_neighbors.
-Note that general graph topologies should generally be replaced by the distributed graph topologies.
-
-For a Cartesian topology, created with MPI_Cart_create, the sequence of neighbors in the send and receive
-buffers at each process is defined by order of the dimensions, first the neighbor in the negative direction
-and then in the positive direction with displacement 1. The numbers of sources and destinations in the
-communication routines are 2*ndims with ndims defined in MPI_Cart_create. If a neighbor does not exist, i.e., at
-the border of a Cartesian topology in the case of a non-periodic virtual grid dimension (i.e.,
-periods[...]==false), then this neighbor is defined to be MPI_PROC_NULL.
-
-If a neighbor in any of the functions is MPI_PROC_NULL, then the neighborhood collective communication behaves
-like a point-to-point communication with MPI_PROC_NULL in this direction. That is, the buffer is still part of
-the sequence of neighbors but it is neither communicated nor updated.
-
-.sp
-.SH NOTES
-.ft R
-The MPI_IN_PLACE option for \fIsendbuf\fP is not meaningful for this operation.
-.sp
-The specification of counts and displacements should not cause
-any location to be written more than once.
-.sp
-All arguments on all processes are significant. The \fIcomm\fP argument,
-in particular, must describe the same communicator on all processes.
-.sp
-The offsets of \fIsdispls\fP and \fIrdispls\fP are measured in units
-of \fIsendtype\fP and \fIrecvtype\fP, respectively. Compare this to
-MPI_Neighbor_alltoallw, where these offsets are measured in bytes.
-
-.SH ERRORS
-.ft R
-Almost all MPI routines return an error value; C routines as
-the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Neighbor_alltoall
-MPI_Neighbor_alltoallw
-MPI_Cart_create
-MPI_Graph_create
-MPI_Dist_graph_create
-
-
diff --git a/ompi/mpi/man/man3/MPI_Neighbor_alltoallw.3in b/ompi/mpi/man/man3/MPI_Neighbor_alltoallw.3in
deleted file mode 100644
index e71731b9951..00000000000
--- a/ompi/mpi/man/man3/MPI_Neighbor_alltoallw.3in
+++ /dev/null
@@ -1,215 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" $COPYRIGHT$
-.TH MPI_Neighbor_alltoallw 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-
-.SH NAME
-\fBMPI_Neighbor_alltoallw, MPI_Ineighbor_alltoallw\fP \- All processes send data of different types to, and receive data of different types from, all processes
-
-.SH SYNTAX
-.ft R
-
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Neighbor_alltoallw(const void *\fIsendbuf\fP, const int \fIsendcounts\fP[],
-	const MPI_Aint \fIsdispls\fP[], const MPI_Datatype \fIsendtypes\fP[],
-	void *\fIrecvbuf\fP, const int \fIrecvcounts\fP[], const MPI_Aint \fIrdispls\fP[],
-	const MPI_Datatype \fIrecvtypes\fP[], MPI_Comm \fIcomm\fP)
-
-int MPI_Ineighbor_alltoallw(const void *\fIsendbuf\fP, const int \fIsendcounts\fP[],
-	const MPI_Aint \fIsdispls\fP[], const MPI_Datatype \fIsendtypes\fP[],
-	void *\fIrecvbuf\fP, const int \fIrecvcounts\fP[], const MPI_Aint \fIrdispls\fP[],
-	const MPI_Datatype \fIrecvtypes\fP[], MPI_Comm \fIcomm\fP, MPI_Request \fI*request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_NEIGHBOR_ALLTOALLW(\fISENDBUF, SENDCOUNTS, SDISPLS, SENDTYPES,
-	RECVBUF, RECVCOUNTS, RDISPLS, RECVTYPES, COMM, IERROR\fP)
-
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fISENDCOUNTS(*), SENDTYPES(*)\fP
-	INTEGER	\fIRECVCOUNTS(*), RECVTYPES(*)\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fISDISPLS(*), RDISPLS(*)\fP
-	INTEGER	\fICOMM, IERROR\fP
-
-MPI_INEIGHBOR_ALLTOALLW(\fISENDBUF, SENDCOUNTS, SDISPLS, SENDTYPES,
-	RECVBUF, RECVCOUNTS, RDISPLS, RECVTYPES, COMM, REQUEST, IERROR\fP)
-
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fISENDCOUNTS(*), SENDTYPES(*)\fP
-	INTEGER	\fIRECVCOUNTS(*), RECVTYPES(*)\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fISDISPLS(*), RDISPLS(*)\fP
-	INTEGER	\fICOMM, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Neighbor_alltoallw(\fIsendbuf\fP, \fIsendcounts\fP, \fIsdispls\fP, \fIsendtypes\fP, \fIrecvbuf\fP,
-		\fIrecvcounts\fP, \fIrdispls\fP, \fIrecvtypes\fP, \fIcomm\fP, \fIierror\fP)
-
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..) :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIsendcounts(*)\fP, \fIrecvcounts(*)\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) :: \fIsdispls(*)\fP, \fIrdispls(*)\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtypes(*)\fP, \fIrecvtypes(*)\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPI_Ineighbor_alltoallw(\fIsendbuf\fP, \fIsendcounts\fP, \fIsdispls\fP, \fIsendtypes\fP, \fIrecvbuf\fP,
-		\fIrecvcounts\fP, \fIrdispls\fP, \fIrecvtypes\fP, \fIcomm\fP, \fIrequest\fP, \fIierror\fP)
-
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN), ASYNCHRONOUS :: \fIsendcounts(*)\fP, \fIrecvcounts(*)\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN), ASYNCHRONOUS ::
-	\fIsdispls(*)\fP, \fIrdispls(*)\fP
-	TYPE(MPI_Datatype), INTENT(IN), ASYNCHRONOUS :: \fIsendtypes(*),\fP
-	\fIrecvtypes(*)\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1.2i
-sendbuf
-Starting address of send buffer.
-.TP 1.2i
-sendcounts
-Integer array, where entry i specifies the number of elements to send
-to neighbor i.
-.TP 1.2i
-sdispls
-Integer array, where entry i specifies the displacement (in bytes,
-offset from \fIsendbuf\fP) from which to send data to neighbor i.
-.TP 1.2i
-sendtypes
-Datatype array, where entry i specifies the datatype to use when
-sending data to neighbor i.
-.TP 1.2i
-recvcounts
-Integer array, where entry j specifies the number of elements to
-receive from neighbor j.
-.TP 1.2i
-rdispls
-Integer array, where entry j specifies the displacement (in bytes,
-offset from \fIrecvbuf\fP) to which data from neighbor j should
-be written.
-.TP 1.2i
-recvtypes
-Datatype array, where entry j specifies the datatype to use when
-receiving data from neighbor j.
-.TP 1.2i
-comm
-Communicator over which data is to be exchanged.
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1.2i
-recvbuf
-Address of receive buffer.
-.TP 1i
-request
-Request (handle, non-blocking only).
-.ft R
-.TP 1.2i
-IERROR
-Fortran only: Error status.
-
-.SH DESCRIPTION
-.ft R
-MPI_Neighbor_alltoallw is a generalized collective operation in which all
-processes send data to and receive data from all neighbors. It
-adds flexibility to MPI_Neighbor_alltoallv by allowing the user to specify the
-datatype of individual data blocks (in addition to displacement and
-element count). Its operation can be thought of in the following way,
-where each process performs 2n (n being the number of neighbors in
-the topology of communicator \fIcomm\fP) independent point-to-point communications.
-The neighbors and buffer layout are determined by the topology of \fIcomm\fP.
-.sp
-.nf
-        MPI_Cart_get(\fIcomm\fP, maxdims, dims, periods, coords);
-        for (dim = 0, i = 0 ; dim < dims ; ++dim) {
-            MPI_Cart_shift(\fIcomm\fP, dim, 1, &r0, &r1);
-            MPI_Isend(\fIsendbuf\fP + \fIsdispls\fP[i]  * extent(\fIsendtype\fP),
-                      \fIsendcount\fP, \fIsendtypes\fP[i], r0, ..., \fIcomm\fP, ...);
-            MPI_Irecv(\fIrecvbuf\fP + \fIrdispls\fP[i] * extent(\fIrecvtype\fP),
-                      \fIrecvcount\fP, \fIrecvtypes\fP[i], r0, ..., \fIcomm\fP, ...);
-            ++i;
-            MPI_Isend(\fIsendbuf\fP + \fIsdispls\fP[i] * extent(\fIsendtype\fP),
-                      \fIsendcount\fP, \fIsendtypes\fP[i], r1, ..., \fIcomm\fP, &req[i]);
-            MPI_Irecv(\fIrecvbuf\fP + \fIrdispls\fP[i] * extent(\fIrecvtype\fP),
-                      \fIrecvcount\fP, \fIrecvtypes\fP[i], r1, ..., \fIcomm\fP, ...);
-            ++i;
-        }
-
-        MPI_Wait_all (...);
-
-	MPI_Comm_size(\fIcomm\fP, &n);
-	for (i = 0, i < n; i++)
-	    MPI_Send(\fIsendbuf\fP + \fIsdispls\fP[i], \fIsendcounts\fP[i],
-	        \fIsendtypes\fP[i], i, ..., \fIcomm\fP);
-	for (i = 0, i < n; i++)
-	    MPI_Recv(\fIrecvbuf\fP + \fIrdispls\fP[i], \fIrecvcounts\fP[i],
-	        \fIrecvtypes\fP[i], i, ..., \fIcomm\fP);
-.fi
-.sp
-Process j sends the k-th block of its local \fIsendbuf\fP to neighbor
-k, which places the data in the j-th block of its local
-\fIrecvbuf\fP.
-.sp
-When a pair of processes exchanges data, each may pass different
-element count and datatype arguments so long as the sender specifies
-the same amount of data to send (in bytes) as the receiver expects
-to receive.
-.sp
-Note that process i may send a different amount of data to process j
-than it receives from process j. Also, a process may send entirely
-different amounts and types of data to different processes in the
-communicator.
-
-.sp
-.SH NOTES
-.ft R
-The MPI_IN_PLACE option for \fIsendbuf\fP is not meaningful for this operation
-.sp
-The specification of counts, types, and displacements should not cause
-any location to be written more than once.
-.sp
-All arguments on all processes are significant. The \fIcomm\fP argument,
-in particular, must describe the same communicator on all processes.
-.sp
-The offsets of \fIsdispls\fP and \fIrdispls\fP are measured in bytes.
-Compare this to MPI_Neighbor_alltoallv, where these offsets are measured in units
-of \fIsendtype\fP and \fIrecvtype\fP, respectively.
-
-.SH ERRORS
-.ft R
-Almost all MPI routines return an error value; C routines as
-the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Neighbor_alltoall
-MPI_Neighbor_alltoallv
-MPI_Cart_create
-MPI_Graph_create
-MPI_Dist_graph_create
-
diff --git a/ompi/mpi/man/man3/MPI_Op_c2f.3in b/ompi/mpi/man/man3/MPI_Op_c2f.3in
deleted file mode 100644
index a13fce697dd..00000000000
--- a/ompi/mpi/man/man3/MPI_Op_c2f.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Comm_f2c.3
diff --git a/ompi/mpi/man/man3/MPI_Op_commutative.3in b/ompi/mpi/man/man3/MPI_Op_commutative.3in
deleted file mode 100644
index 84599cc49a2..00000000000
--- a/ompi/mpi/man/man3/MPI_Op_commutative.3in
+++ /dev/null
@@ -1,63 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2015 FUJITSU LIMITED.  All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Op_commutative 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Op_commutative\fP \- Query of commutativity of reduction operation.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Op_commutative(MPI_Op \fIop\fP, int *\fIcommute\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_OP_COMMUTATIVE(\fIOP, COMMUTE, IERROR\fP)
-	LOGICAL	\fICOMMUTE\fP
-	INTEGER	\fIOP, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Op_commutative(\fIop\fP, \fIcommute\fP, \fIierror\fP)
-	TYPE(MPI_Op), INTENT(IN) :: \fIop\fP
-	INTEGER, INTENT(OUT) :: \fIcommute\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.TP 1i
-op
-Operation (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-commute
-True if op is commutative, false otherwise (logical).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Reduction operations can be queried for their commutativity.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.sp
-MPI_Op_create
-
diff --git a/ompi/mpi/man/man3/MPI_Op_create.3in b/ompi/mpi/man/man3/MPI_Op_create.3in
deleted file mode 100644
index 05d3768e2e6..00000000000
--- a/ompi/mpi/man/man3/MPI_Op_create.3in
+++ /dev/null
@@ -1,190 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Op_create 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Op_create\fP \- Creates a user-defined combination function handle.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Op_create(MPI_User_function *\fIfunction\fP, int\fI commute\fP,
-	MPI_Op *\fIop\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_OP_CREATE(\fIFUNCTION, COMMUTE, OP, IERROR\fP)
-	EXTERNAL	\fIFUNCTION\fP
-	LOGICAL	\fICOMMUTE\fP
-	INTEGER	\fIOP, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Op_create(\fIuser_fn\fP, \fIcommute\fP, \fIop\fP, \fIierror\fP)
-	PROCEDURE(MPI_User_function) :: \fIuser_fn\fP
-	LOGICAL, INTENT(IN) :: \fIcommute\fP
-	TYPE(MPI_Op), INTENT(OUT) :: \fIop\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-function
-User-defined function (function).
-.TP 1i
-commute
-True if commutative; false otherwise.
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-op
-Operation (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Op_create binds a user-defined global operation to an op handle that can subsequently be used in MPI_Reduce, MPI_Allreduce, MPI_Reduce_scatter, and  MPI_Scan. The user-defined operation is assumed to be associative. If commute = true, then the operation should be both commutative and associative. If commute = false, then the order of operands is fixed and is defined to be in ascending, process rank order, beginning with process zero. The order of evaluation can be changed, taking advantage of the associativity of the operation. If commute = true then the order of evaluation can be changed, taking advantage of commutativity and associativity.
-.sp
-\fIfunction\fP is the user-defined function, which must have the following four arguments: invec, inoutvec, len, and datatype.
-
-.sp
-The ANSI-C prototype for the function is the following:
-.sp
-.nf
-  typedef void MPI_User_function(void *invec, void *inoutvec,
-                                 int *len,
-                                 MPI_Datatype *datatype);
-.fi
-.sp
-The Fortran declaration of the user-defined function appears below.
-.sp
-.nf
-  FUNCTION USER_FUNCTION( INVEC(*), INOUTVEC(*), LEN, TYPE)
-  <type> INVEC(LEN), INOUTVEC(LEN)
-   INTEGER LEN, TYPE
-.fi
-.sp
-The datatype argument is a handle to the data type that was passed into the
-call to MPI_Reduce. The user reduce function should be written such that
-the following holds: Let u[0],\ ...,\ u[len-1] be the len elements in the communication buffer described by the arguments invec, len, and datatype when the function is invoked; let v[0],\ ...,\ v[len-1] be len elements in the communication buffer described by the arguments inoutvec, len, and datatype when the function is invoked; let w[0],\ ...,\ w[len-1] be len elements in the communication buffer described by the arguments inoutvec, len, and datatype when the function returns; then w[i] = u[i] o v[i], for i=0\ ,...,\ len-1, where o is the reduce operation that the function computes.
-.sp
-Informally, we can think of invec and inoutvec as arrays of len elements
-that function is combining. The result of the reduction over-writes values
-in inoutvec, hence the name. Each invocation of the function results in the
-pointwise evaluation of the reduce operator on len elements: i.e, the
-function returns in inoutvec[i] the value invec[i] o inoutvec[i], for i =
-0\,...,\ count-1, where o is the combining operation computed by the function.
-.sp
-By internally comparing the value of the datatype argument to known, global handles, it is possible to overload the use of a single user-defined function for several different data types.
-.sp
-General datatypes may be passed to the user function. However, use of datatypes that are not contiguous is likely to lead to inefficiencies.
-.sp
-No MPI communication function may be called inside the user function.
-MPI_Abort may be called inside the function in case of an error.
-
-.SH NOTES
-Suppose one defines a library of user-defined reduce
-functions that are overloaded: The datatype argument is used to select the right execution path at each invocation, according to the types of the operands. The user-defined reduce function cannot "decode" the datatype argument that it is passed, and cannot identify, by itself, the correspondence between the datatype handles and the datatype they represent. This correspondence was established when the datatypes were created. Before the library is used, a library initialization preamble must be executed. This preamble code will define the datatypes that are used by the library and store handles to these datatypes in global, static variables that are shared by the user code and the library code.
-
-\fBExample:\fP Example of user-defined reduce:
-.sp
-Compute the product of an array of complex numbers, in C.
-.sp
-.nf
-    typedef struct {
-        double real,imag;
-    } Complex;
-
-    /* the user-defined function
-     */
-    void myProd( Complex *in, Complex *inout, int *len,
-                 MPI_Datatype *dptr )
-    {
-        int i;
-        Complex c;
-
-    for (i=0; i< *len; ++i) {
-            c.real = inout->real*in->real -
-                       inout->imag*in->imag;
-            c.imag = inout->real*in->imag +
-                       inout->imag*in->real;
-            *inout = c;
-            in++; inout++;
-        }
-    }
-
-    /* and, to call it\&...
-     */
-    \&...
-
-    /* each process has an array of 100 Complexes
-         */
-        Complex a[100], answer[100];
-        MPI_Op myOp;
-        MPI_Datatype ctype;
-
-    /* explain to MPI how type Complex is defined
-         */
-       MPI_Type_contiguous( 2, MPI_DOUBLE, &ctype );
-        MPI_Type_commit( &ctype );
-        /* create the complex-product user-op
-         */
-        MPI_Op_create( myProd, True, &myOp );
-
-        MPI_Reduce( a, answer, 100, ctype, myOp, root, comm );
-
-        /* At this point, the answer, which consists of 100 Complexes,
-         * resides on process root
-         */
-.fi
-.sp
-The Fortran version of MPI_Reduce will invoke a user-defined reduce function using the Fortran calling conventions and will pass a Fortran-type datatype argument; the C version will use C calling convention and the C representation of a datatype handle. Users who plan to mix languages should define their reduction functions accordingly.
-
-.SH NOTES ON COLLECTIVE OPERATIONS
-
-The reduction functions (
-.I MPI_Op
-) do not return an error value.  As a result,
-if the functions detect an error, all they can do is either call
-.I MPI_Abort
-or silently skip the problem.  Thus, if you change the error handler from
-.I MPI_ERRORS_ARE_FATAL
-to something else, for example,
-.I MPI_ERRORS_RETURN
-,
-then no error may be indicated.
-
-The reason for this is the performance problems in ensuring that
-all collective routines return the same error value.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-.nf
-MPI_Reduce
-MPI_Reduce_scatter
-MPI_Allreduce
-MPI_Scan
-MPI_Op_free
-
diff --git a/ompi/mpi/man/man3/MPI_Op_f2c.3in b/ompi/mpi/man/man3/MPI_Op_f2c.3in
deleted file mode 100644
index a13fce697dd..00000000000
--- a/ompi/mpi/man/man3/MPI_Op_f2c.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Comm_f2c.3
diff --git a/ompi/mpi/man/man3/MPI_Op_free.3in b/ompi/mpi/man/man3/MPI_Op_free.3in
deleted file mode 100644
index d4e436428e7..00000000000
--- a/ompi/mpi/man/man3/MPI_Op_free.3in
+++ /dev/null
@@ -1,67 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Op_free 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Op_free\fP \- Frees a user-defined combination function handle.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Op_free(MPI_Op *\fIop\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_OP_FREE(\fIOP, IERROR\fP)
-	INTEGER	\fIOP, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Op_free(\fIop\fP, \fIierror\fP)
-	TYPE(MPI_Op), INTENT(INOUT) :: \fIop\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.TP 1i
-op
-Operation (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Marks a user-defined reduction operation for deallocation and sets \fIop\fP to MPI_OP_NULL.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.sp
-MPI_Op_create
-.br
-MPI_Reduce
-.br
-MPI_Allreduce
-.br
-MPI_Reduce_scatter
-.br
-MPI_Scan
-
diff --git a/ompi/mpi/man/man3/MPI_Open_port.3in b/ompi/mpi/man/man3/MPI_Open_port.3in
deleted file mode 100644
index 8a334384bf4..00000000000
--- a/ompi/mpi/man/man3/MPI_Open_port.3in
+++ /dev/null
@@ -1,71 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2007-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Open_port 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Open_port\fP \- Establishes a network address for a server to accept connections from clients.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Open_port(MPI_Info \fIinfo\fP, char *\fIport_name\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_OPEN_PORT(\fIINFO, PORT_NAME, IERROR\fP)
-	CHARACTER*(*)	\fIPORT_NAME\fP
-	INTEGER		\fIINFO, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Open_port(\fIinfo\fP, \fIport_name\fP, \fIierror\fP)
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	CHARACTER(LEN=MPI_MAX_PORT_NAME), INTENT(OUT) :: \fIport_name\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-info
-Options on how to establish an address (handle). No options currently supported.
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-port_name
-Newly established port (string).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Open_port establishes a network address, encoded in the \fIport_name\fP string, at which the server will be able to accept connections from clients. \fIport_name\fP is supplied by the system.
-.sp
-MPI copies a system-supplied port name into \fIport_name\fP. \fIport_name\fP identifies the newly opened port and can be used by a client to contact the server. The maximum size string that may be supplied by the system is MPI_MAX_PORT_NAME.
-
-.SH SUPPORTED INFO KEYS
-None.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-MPI_Comm_accept
-MPI_Comm_connect
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Pack.3in b/ompi/mpi/man/man3/MPI_Pack.3in
deleted file mode 100644
index ac1a0fb5004..00000000000
--- a/ompi/mpi/man/man3/MPI_Pack.3in
+++ /dev/null
@@ -1,119 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Pack 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Pack\fP \- Packs data of a given datatype into contiguous memory.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Pack(const void *\fIinbuf\fP, int\fI incount\fP, MPI_Datatype\fI datatype\fP,
-	void\fI *outbuf\fP, int\fI outsize\fP, int\fI *position\fP, MPI_Comm\fI comm\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_PACK(\fIINBUF, INCOUNT, DATATYPE, OUTBUF,OUTSIZE, POSITION,
-		COMM, IERROR\fP)
-	<type>	\fIINBUF(*), OUTBUF(*)\fP
-	INTEGER	\fIINCOUNT, DATATYPE, OUTSIZE, POSITION, COMM, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Pack(\fIinbuf\fP, \fIincount\fP, \fIdatatype\fP, \fIoutbuf\fP, \fIoutsize\fP, \fIposition\fP, \fIcomm\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIinbuf\fP
-	TYPE(*), DIMENSION(..) :: \fIoutbuf\fP
-	INTEGER, INTENT(IN) :: \fIincount\fP, \fIoutsize\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	INTEGER, INTENT(INOUT) :: \fIposition\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-inbuf
-Input buffer start (choice).
-.TP 1i
-incount
-Number of input data items (integer).
-.TP 1i
-datatype
-Datatype of each input data item (handle).
-.TP 1i
-outsize
-Output buffer size, in bytes (integer).
-.TP 1i
-comm
-Communicator for packed message (handle).
-
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-position
-Current position in buffer, in bytes (integer).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-outbuf
-Output buffer start (choice).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Packs the message in the send buffer specified by \fIinbuf\fP, \fIincount\fP, \fIdatatype\fP into the buffer space specified by \fIoutbuf\fP and \fIoutsize\fP. The input buffer can be any communication buffer allowed in MPI_Send. The output buffer is a contiguous storage area containing \fIoutsize\fP bytes, starting at the address \fIoutbuf\fP (length is counted in bytes, not elements, as if it were a communication buffer for a message of type MPI_Packed).
-.sp
-The input value of \fIposition\fP is the first location in the output buffer to be used for packing. \fIposition\fP is incremented by the size of the packed message, and the output value of \fIposition\fP is the first location in the output buffer following the locations occupied by the packed message. The \fIcomm\fP argument is the communicator that will be subsequently used for sending the packed message.
-.sp
-\fBExample:\fP An example using MPI_Pack:
-.sp
-.nf
-    int position, i, j, a[2];
-    char buff[1000];
-
-    \&....
-
-    MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
-    if (myrank == 0)
-    {
-       / * SENDER CODE */
-
-    position = 0;
-      MPI_Pack(&i, 1, MPI_INT, buff, 1000, &position, MPI_COMM_WORLD);
-      MPI_Pack(&j, 1, MPI_INT, buff, 1000, &position, MPI_COMM_WORLD);
-      MPI_Send( buff, position, MPI_PACKED, 1, 0, MPI_COMM_WORLD);
-    }
-    else  /* RECEIVER CODE */
-      MPI_Recv( a, 2, MPI_INT, 0, 0, MPI_COMM_WORLD)
-
-    }
-
-.fi
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-MPI_Unpack
-.br
-MPI_Pack_size
-
diff --git a/ompi/mpi/man/man3/MPI_Pack_external.3in b/ompi/mpi/man/man3/MPI_Pack_external.3in
deleted file mode 100644
index d785c87c3ae..00000000000
--- a/ompi/mpi/man/man3/MPI_Pack_external.3in
+++ /dev/null
@@ -1,205 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Pack_external 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-
-.SH NAME
-\fBMPI_Pack_external\fP \- Writes data to a portable format
-
-.SH SYNTAX
-.ft R
-
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Pack_external(const char *\fIdatarep\fP, const void *\fIinbuf\fP,
-	int \fIincount\fP, MPI_Datatype\fI datatype\fP,
-	void *\fIoutbuf\fP, MPI_Aint \fIoutsize\fP,
-	MPI_Aint *\fIposition\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_PACK_EXTERNAL(\fIDATAREP, INBUF, INCOUNT, DATATYPE,
-	OUTBUF, OUTSIZE, POSITION, IERROR\fP)
-
-	INTEGER		\fIINCOUNT, DATATYPE, IERROR\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fIOUTSIZE, POSITION\fP
-	CHARACTER*(*)	\fIDATAREP\fP
-	<type>		\fIINBUF(*), OUTBUF(*)\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Pack_external(\fIdatarep\fP, \fIinbuf\fP, \fIincount\fP, \fIdatatype\fP, \fIoutbuf\fP, \fIoutsize\fP,
-		\fIposition\fP, \fIierror\fP)
-	CHARACTER(LEN=*), INTENT(IN) :: \fIdatarep\fP
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIinbuf\fP
-	TYPE(*), DIMENSION(..) :: \fIoutbuf\fP
-	INTEGER, INTENT(IN) :: \fIincount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) :: \fIoutsize\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(INOUT) :: \fIposition\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-datarep
-Data representation (string).
-.ft R
-.TP 1i
-inbuf
-Input buffer start (choice).
-.TP 1i
-incount
-Number of input data items (integer).
-.TP 1i
-datatype
-Datatype of each input data item (handle).
-.TP 1i
-outsize
-Output buffer size, in bytes (integer).
-
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-position
-Current position in buffer, in bytes (integer).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-outbuf
-Output buffer start (choice).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Pack_external packs data into the external32 format, a universal
-data representation defined by the MPI Forum. This format is useful
-for exchanging data between MPI implementations, or when writing data
-to a file.
-.sp
-The input buffer is specified by \fIinbuf\fP, \fIincount\fP and
-\fIdatatype\fP, and may be any communication buffer allowed in
-MPI_Send. The output buffer \fIoutbuf\fP must be a contiguous storage
-area containing \fIoutsize\fP bytes.
-.sp
-The input value of \fIposition\fP is the first position in
-\fIoutbuf\fP to be used for packing (measured in bytes, not elements,
-relative to the start of the buffer). When the function returns,
-\fIposition\fP is incremented by the size of the packed message, so
-that it points to the first location in \fIoutbuf\fP following the
-packed message. This way it may be used as input to a subsequent call
-to MPI_Pack_external.
-.sp
-
-\fBExample:\fP An example using MPI_Pack_external:
-.sp
-.nf
-	int position, i;
-	double msg[5];
-	char buf[1000];
-
-	\&...
-
-	MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
-	if (myrank == 0) {	/* SENDER CODE */
-		position = 0;
-		i = 5; /* number of doubles in msg[] */
-		MPI_Pack_external("external32", &i, 1, MPI_INT,
-		    buf, 1000, &position);
-		MPI_Pack_external("external32", &msg, i, MPI_DOUBLE,
-		    buf, 1000, &position);
-		MPI_Send(buf, position, MPI_BYTE, 1, 0,
-		    MPI_COMM_WORLD);
-	} else {		/* RECEIVER CODE */
-		MPI_Recv(buf, 1, MPI_BYTE, 0, 0, MPI_COMM_WORLD,
-		    MPI_STATUS_IGNORE);
-		MPI_Unpack_external("external32", buf, 1000,
-		    MPI_INT, &i, 1, &position);
-		MPI_Unpack_external("external32", buf, 1000,
-		    MPI_DOUBLE, &msg, i, &position);
-	}
-
-.fi
-.SH NOTES
-.ft R
-The \fIdatarep\fP argument specifies the data format. The only valid
-value in the current version of MPI is "external32". The argument is
-provided for future extensibility.
-.sp
-To understand the behavior of pack and unpack, it is convenient to
-think of the data part of a message as being the sequence obtained by
-concatenating the successive values sent in that message. The pack
-operation stores this sequence in the buffer space, as if sending the
-message to that buffer. The unpack operation retrieves this sequence
-from buffer space, as if receiving a message from that buffer. (It is
-helpful to think of internal Fortran files or sscanf in C for a
-similar function.)
-.sp
-Several messages can be successively packed into one packing
-unit. This is effected by several successive related calls to
-MPI_Pack_external, where the first call provides \fIposition\fP=0,
-and each successive call inputs the value of \fIposition\fP that was
-output by the previous call, along with the same values for
-\fIoutbuf\fP and \fIoutcount\fP. This packing unit now contains the
-equivalent information that would have been stored in a message by one
-send call with a send buffer that is the "concatenation" of the
-individual send buffers.
-.sp
-A packing unit can be sent using type MPI_BYTE. Any point-to-point
-or collective communication function can be used to move the sequence
-of bytes that forms the packing unit from one process to another. This
-packing unit can now be received using any receive operation, with any
-datatype. (The type-matching rules are relaxed for messages sent with
-type MPI_BYTE.)
-.sp
-A packing unit can be unpacked into several successive messages. This
-is effected by several successive related calls to
-MPI_Unpack_external, where the first call provides \fIposition\fP=0,
-and each successive call inputs the value of position that was output
-by the previous call, and the same values for \fIinbuf\fP and
-\fIinsize\fP.
-.sp
-The concatenation of two packing units is not necessarily a packing
-unit; nor is a substring of a packing unit necessarily a packing
-unit. Thus, one cannot concatenate two packing units and then unpack
-the result as one packing unit; nor can one unpack a substring of a
-packing unit as a separate packing unit. Each packing unit that was
-created by a related sequence of pack calls must be unpacked as a unit
-by a sequence of related unpack calls.
-
-.SH ERRORS
-.ft R
-Almost all MPI routines return an error value; C routines as
-the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-.sp
-See the MPI man page for a full list of MPI error codes.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Pack_external_size
-MPI_Send
-MPI_Unpack_external
-sscanf(3C)
-
diff --git a/ompi/mpi/man/man3/MPI_Pack_external_size.3in b/ompi/mpi/man/man3/MPI_Pack_external_size.3in
deleted file mode 100644
index 8d67ddeb66b..00000000000
--- a/ompi/mpi/man/man3/MPI_Pack_external_size.3in
+++ /dev/null
@@ -1,105 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Pack_external_size 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-
-.SH NAME
-\fBMPI_Pack_external_size\fP \- Calculates upper bound on space needed
-to write to a portable format
-
-.SH SYNTAX
-.ft R
-
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Pack_external_size(char *\fIdatarep\fP, int \fIincount\fP,
-	MPI_Datatype \fIdatatype\fP, MPI_Aint *\fIsize\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_PACK_EXTERNAL_SIZE(\fIDATAREP, INCOUNT, DATATYPE, SIZE, IERROR\fP)
-
-	INTEGER		\fIINCOUNT, DATATYPE, IERROR\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fISIZE\fP
-	CHARACTER*(*)	\fIDATAREP\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Pack_external_size(\fIdatarep\fP, \fIincount\fP, \fIdatatype\fP, \fIsize\fP, \fIierror\fP)
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	INTEGER, INTENT(IN) :: \fIincount\fP
-	CHARACTER(LEN=*), INTENT(IN) :: \fIdatarep\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(OUT) :: \fIsize\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-datarep
-Data representation (string).
-.TP 1i
-incount
-Number of input data items (integer).
-.TP 1i
-datatype
-Datatype of each input data item (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-size
-Upper bound on size of packed message, in bytes (integer).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Pack_external_size allows the application to find out how much
-space is needed to pack a message in the portable format defined by
-the MPI Forum. It returns in \fIsize\fP an upper bound on the
-increment in \fIposition\fP that would occur in a call to
-MPI_Pack_external with the same values for \fIdatarep\fP,
-\fIincount\fP, and \fIdatatype\fP.
-.sp
-The call returns an upper bound, rather than an exact bound, as the
-exact amount of space needed to pack the message may depend on context
-and alignment (e.g., the first message packed in a packing unit may
-take more space).
-
-.SH NOTES
-.ft R
-The \fIdatarep\fP argument specifies the data format. The only valid
-value in the current version of MPI is "external32". The argument is
-provided for future extensibility.
-
-.SH ERRORS
-.ft R
-Almost all MPI routines return an error value; C routines as
-the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-.sp
-See the MPI man page for a full list of MPI error codes.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Pack_external
-MPI_Unpack_external
-
diff --git a/ompi/mpi/man/man3/MPI_Pack_size.3in b/ompi/mpi/man/man3/MPI_Pack_size.3in
deleted file mode 100644
index 3abc9e435bf..00000000000
--- a/ompi/mpi/man/man3/MPI_Pack_size.3in
+++ /dev/null
@@ -1,81 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Pack_size 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Pack_size\fP \- Returns the upper bound on the amount of space needed to pack a message.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Pack_size(int \fIincount\fP, MPI_Datatype\fI datatype\fP, MPI_Comm\fI comm\fP,
-	int\fI *size\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_PACK_SIZE(\fIINCOUNT, DATATYPE, COMM, SIZE, IERROR\fP)
-	INTEGER	\fIINCOUNT, DATATYPE, COMM, SIZE, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Pack_size(\fIincount\fP, \fIdatatype\fP, \fIcomm\fP, \fIsize\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIincount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, INTENT(OUT) :: \fIsize\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-incount
-Count argument to packing call (integer).
-.TP 1i
-datatype
-Datatype argument to packing call (handle).
-.TP 1i
-comm
-Communicator argument to packing call (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-size
-Upper bound on size of packed message, in bytes (integer).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Pack_size allows the application to find out how much space is needed to pack a message. A call to MPI_Pack_size(incount, datatype, comm, size) returns in size an
-upper bound on the increment in position that would occur in a call to MPI_Pack, with the same values for \fIincount\fP, \fIdatatype\fP, and \fIcomm\fP.
-.sp
-\fBRationale:\fP  The call returns an upper bound, rather than an exact bound, since the exact amount of space needed to pack the message may depend on the context (e.g., first message packed in a packing unit may take more space).
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Pack
-.br
-MPI_Unpack
-
-
diff --git a/ompi/mpi/man/man3/MPI_Pcontrol.3in b/ompi/mpi/man/man3/MPI_Pcontrol.3in
deleted file mode 100644
index 4a15f069c1d..00000000000
--- a/ompi/mpi/man/man3/MPI_Pcontrol.3in
+++ /dev/null
@@ -1,71 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Pcontrol 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Pcontrol\fP \- Controls profiling.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Pcontrol(const int \fIlevel\fP, \&... )
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_PCONTROL(\fILEVEL\fP)
-	INTEGER	\fILEVEL\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Pcontrol(\fIlevel\fP)
-	INTEGER, INTENT(IN) :: \fIlevel\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-level
-Profiling level.
-
-.SH DESCRIPTION
-.ft R
-MPI libraries themselves make no use of this routine; they simply return immediately to the user code. However the presence of calls to this routine allows a profiling package to be explicitly called by the user.
-.sp
-Since MPI has no control of the implementation of the profiling code, we are unable to specify precisely the semantics that will be provided by calls to MPI_Pcontrol. This vagueness extends to the number of arguments to the function, and their datatypes.
-.sp
-However to provide some level of portability of user codes to different
-profiling libraries, we request the following meanings for certain values of level:
-.TP
-  o
-level==0	Profiling is disabled.
-.TP
-  o
-level==1	Profiling is enabled at a normal default level of detail.
-.TP
-  o
-level==2	Profile buffers are flushed. (This may be a no-op in some
-profilers).
-.TP
-  o
-All other values of level have profile library-defined effects and additional arguments.
-.LP
-.sp
-We also request that the default state after MPI_Init has been called is for profiling to be enabled at the normal default level (i.e., as if MPI_Pcontrol had just been called with the argument 1). This allows users to link with a profiling library and obtain profile output without having to modify their source code at all.
-.sp
-The provision of MPI_Pcontrol as a no-op in the standard MPI library allows users to modify their source code to obtain more detailed profiling information, but still be able to link exactly the same code against the standard MPI library.
-
-.SH NOTES
-.ft R
-This routine provides a common interface for profiling control. The interpretation of level and any other arguments is left to the profiling library.
-.sp
-This function does not return an error value. Consequently, the result of calling it before MPI_Init or after MPI_Finalize is undefined.
-
diff --git a/ompi/mpi/man/man3/MPI_Probe.3in b/ompi/mpi/man/man3/MPI_Probe.3in
deleted file mode 100644
index a7839a0c8af..00000000000
--- a/ompi/mpi/man/man3/MPI_Probe.3in
+++ /dev/null
@@ -1,129 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Probe 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Probe\fP \- Blocking test for a message.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Probe(int \fIsource\fP, int\fI tag\fP, MPI_Comm\fI comm\fP, MPI_Status\fI *status\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_PROBE(\fISOURCE, TAG, COMM, STATUS, IERROR\fP)
-	INTEGER	\fISOURCE, TAG, COMM, STATUS(MPI_STATUS_SIZE), IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Probe(\fIsource\fP, \fItag\fP, \fIcomm\fP, \fIstatus\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIsource\fP, \fItag\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Status) :: \fIstatus\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-source
-Source rank or MPI_ANY_SOURCE (integer).
-.TP 1i
-tag
-Tag value or MPI_ANY_TAG (integer).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-status
-Status object (status).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-The MPI_Probe and MPI_Iprobe operations allow checking of incoming messages, without actual receipt of them. The user can then decide how to receive them, based on the information returned by the probe in the status variable. For example, the user may allocate memory for the receive buffer, according to the length of the probed message.
-.sp
-MPI_Probe behaves like MPI_Iprobe except that it is a blocking call that returns only after a matching message has been found.
-.sp
-If your application does not need to examine the \fIstatus\fP field, you can save resources by using the predefined constant MPI_STATUS_IGNORE as a special value for the \fIstatus\fP argument.
-.sp
-The semantics of MPI_Probe and MPI_Iprobe guarantee progress: If a call to MPI_Probe has been issued by a process, and a send that matches the probe has been initiated by some process, then the call to MPI_Probe will return, unless the message is received by another concurrent receive operation (that is executed by another thread at the probing process). Similarly, if a process busy waits with MPI_Iprobe and a matching message has been issued, then the call to MPI_Iprobe will eventually return flag = true unless the message is received by another concurrent receive operation.
-.sp
-\fBExample 1:\fP Use blocking probe to wait for an incoming message.
-.sp
-.nf
-CALL MPI_COMM_RANK(comm, rank, ierr)
-       IF (rank.EQ.0) THEN
-            CALL MPI_SEND(i, 1, MPI_INTEGER, 2, 0, comm, ierr)
-       ELSE IF(rank.EQ.1) THEN
-            CALL MPI_SEND(x, 1, MPI_REAL, 2, 0, comm, ierr)
-       ELSE   ! rank.EQ.2
-           DO i=1, 2
-              CALL MPI_PROBE(MPI_ANY_SOURCE, 0,
-                              comm, status, ierr)
-              IF (status(MPI_SOURCE) = 0) THEN
-100                CALL MPI_RECV(i, 1, MPI_INTEGER, 0, 0, status, ierr)
-              ELSE
-200                CALL MPI_RECV(x, 1, MPI_REAL, 1, 0, status, ierr)
-              END IF
-           END DO
-       END IF
-.fi
-.sp
-Each message is received with the right type.
-.sp
-\fBExample 2:\fP A program similar to the previous example, but with a problem.
-.sp
-.nf
-CALL MPI_COMM_RANK(comm, rank, ierr)
-       IF (rank.EQ.0) THEN
-            CALL MPI_SEND(i, 1, MPI_INTEGER, 2, 0, comm, ierr)
-       ELSE IF(rank.EQ.1) THEN
-            CALL MPI_SEND(x, 1, MPI_REAL, 2, 0, comm, ierr)
-       ELSE
-           DO i=1, 2
-              CALL MPI_PROBE(MPI_ANY_SOURCE, 0,
-                              comm, status, ierr)
-              IF (status(MPI_SOURCE) = 0) THEN
-100                CALL MPI_RECV(i, 1, MPI_INTEGER, MPI_ANY_SOURCE,
-                                 0, status, ierr)
-              ELSE
-200                CALL MPI_RECV(x, 1, MPI_REAL, MPI_ANY_SOURCE,
-                                 0, status, ierr)
-              END IF
-           END DO
-       END IF
-.fi
-.sp
-We slightly modified Example 2, using MPI_ANY_SOURCE as the source argument in the two receive calls in statements labeled 100 and 200. The program is now incorrect: The receive operation may receive a message that is distinct from the message probed by the preceding call to MPI_Probe.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Iprobe
-.br
-MPI_Cancel
-
diff --git a/ompi/mpi/man/man3/MPI_Publish_name.3in b/ompi/mpi/man/man3/MPI_Publish_name.3in
deleted file mode 100644
index 36fbfc7523b..00000000000
--- a/ompi/mpi/man/man3/MPI_Publish_name.3in
+++ /dev/null
@@ -1,167 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2007-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Publish_name 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-
-.SH NAME
-.nf
-\fBMPI_Publish_name\fP \- Publishes a service name associated with a port
-
-.fi
-.SH SYNTAX
-.ft R
-
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Publish_name(const char *\fIservice_name\fP, MPI_Info \fIinfo\fP,
-	const char *\fIport_name\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_PUBLISH_NAME(\fISERVICE_NAME, INFO, PORT_NAME, IERROR\fP)
-	CHARACTER*(*)	\fISERVICE_NAME, PORT_NAME\fP
-	INTEGER		\fIINFO, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Publish_name(\fIservice_name\fP, \fIinfo\fP, \fIport_name\fP, \fIierror\fP)
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	CHARACTER(LEN=*), INTENT(IN) :: \fIservice_name\fP, \fIport_name\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1.4i
-service_name
-A service name (string).
-.TP 1.4i
-info
-Options to the name service functions (handle).
-.ft R
-.TP 1.4i
-port_name
-A port name (string).
-
-.SH OUTPUT PARAMETER
-.TP 1.4i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-This routine publishes the pair (\fIservice_name, port_name\fP) so that
-an application may retrieve \fIport_name\fP by calling MPI_Lookup_name
-with \fIservice_name\fP as an argument. It is an error to publish the same
-\fIservice_name\fP twice, or to use a \fIport_name\fP argument that was
-not previously opened by the calling process via a call to MPI_Open_port.
-
-.SH INFO ARGUMENTS
-The following keys for \fIinfo\fP are recognized:
-.sp
-.sp
-.nf
-Key                   Type      Description
----                   ----      -----------
-
-ompi_global_scope     bool      If set to true, publish the name in
-                                the global scope.  Publish in the local
-                                scope otherwise.  See the NAME SCOPE
-                                section for more details.
-
-ompi_unique           bool      If set to true, return an error if the
-                                specified service_name already exists.
-                                Default to overwriting any pre-existing
-                                value.
-.fi
-
-.sp
-\fIbool\fP info keys are actually strings but are evaluated as
-follows: if the string value is a number, it is converted to an
-integer and cast to a boolean (meaning that zero integers are false
-and non-zero values are true).  If the string value is
-(case-insensitive) "yes" or "true", the boolean is true.  If the
-string value is (case-insensitive) "no" or "false", the boolean is
-false.  All other string values are unrecognized, and therefore false.
-.PP
-If no info key is provided, the function will first check to see if a
-global server has been specified and is available. If so, then the
-publish function will default to global scope first, followed by local. Otherwise,
-the data will default to publish with local scope.
-
-.SH NAME SCOPE
-Open MPI supports two name scopes: \fIglobal\fP and \fIlocal\fP. Local scope will
-place the specified service/port pair in a data store located on the
-mpirun of the calling process' job. Thus, data published with local
-scope will only be accessible to processes in jobs spawned by that
-mpirun - e.g., processes in the calling process' job, or in jobs
-spawned via MPI_Comm_spawn.
-.sp
-Global scope places the specified service/port pair in a data store
-located on a central server that is accessible to all jobs running
-in the cluster or environment. Thus, data published with global
-scope can be accessed by multiple mpiruns and used for MPI_Comm_Connect
-and MPI_Comm_accept between jobs.
-.sp
-Note that global scope operations require both the presence of the
-central server and that the calling process be able to communicate
-to that server. MPI_Publish_name will return an error if global
-scope is specified and a global server is either not specified or
-cannot be found.
-.sp
-Open MPI provides a server called \fIompi-server\fP to support global
-scope operations. Please refer to its manual page for a more detailed
-description of data store/lookup operations.
-.sp
-As an example of the impact of these scoping rules, consider the case
-where a job has been started with
-mpirun - call this job "job1". A process in job1 creates and publishes
-a service/port pair using a local scope. Open MPI will store this
-data in the data store within mpirun.
-.sp
-A process in job1 (perhaps the same as did the publish, or perhaps
-some other process in the job) subsequently calls MPI_Comm_spawn to
-start another job (call it "job2") under this mpirun. Since the two
-jobs share a common mpirun, both jobs have access to local scope data. Hence,
-a process in job2 can perform an MPI_Lookup_name with a local scope
-to retrieve the information.
-.sp
-However, assume another user starts a job using mpirun - call
-this job "job3". Because the service/port data published by job1 specified
-local scope, processes in job3 cannot access that data. In contrast, if the
-data had been published using global scope, then any process in job3 could
-access the data, provided that mpirun was given knowledge of how to contact
-the central server and the process could establish communication
-with it.
-
-.SH ERRORS
-.ft R
-Almost all MPI routines return an error value; C routines as
-the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-.sp
-See the MPI man page for a full list of MPI error codes.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Lookup_name
-MPI_Open_port
-
-
diff --git a/ompi/mpi/man/man3/MPI_Put.3in b/ompi/mpi/man/man3/MPI_Put.3in
deleted file mode 100644
index 3f1e440e134..00000000000
--- a/ompi/mpi/man/man3/MPI_Put.3in
+++ /dev/null
@@ -1,155 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013-2014 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Put 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Put\fP, \fBMPI_Rput\fP \- Copies data from the origin memory to the target.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-MPI_Put(const void *\fIorigin_addr\fP, int \fIorigin_count\fP, MPI_Datatype
-	\fIorigin_datatype\fP, int \fItarget_rank\fP, MPI_Aint \fItarget_disp\fP,
-	int \fItarget_count\fP, MPI_Datatype \fItarget_datatype\fP, MPI_Win \fIwin\fP)
-
-MPI_Rput(const void *\fIorigin_addr\fP, int \fIorigin_count\fP, MPI_Datatype
-	 \fIorigin_datatype\fP, int \fItarget_rank\fP, MPI_Aint \fItarget_disp\fP,
-	 int \fItarget_count\fP, MPI_Datatype \fItarget_datatype\fP, MPI_Win \fIwin\fP,
-	 MPI_Request *\fIrequest\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_PUT(\fIORIGIN_ADDR, ORIGIN_COUNT, ORIGIN_DATATYPE, TARGET_RANK,
-	TARGET_DISP, TARGET_COUNT, TARGET_DATATYPE, WIN, IERROR\fP)
-	<type> \fIORIGIN_ADDR\fP(*)
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fITARGET_DISP\fP
-	INTEGER \fIORIGIN_COUNT, ORIGIN_DATATYPE, TARGET_RANK, TARGET_COUNT,
-	TARGET_DATATYPE, WIN, IERROR\fP
-
-MPI_RPUT(\fIORIGIN_ADDR, ORIGIN_COUNT, ORIGIN_DATATYPE, TARGET_RANK,
-	 TARGET_DISP, TARGET_COUNT, TARGET_DATATYPE, WIN, REQUEST, IERROR\fP)
-	 <type> \fIORIGIN_ADDR\fP(*)
-	 INTEGER(KIND=MPI_ADDRESS_KIND) \fITARGET_DISP\fP
-	 INTEGER \fIORIGIN_COUNT, ORIGIN_DATATYPE, TARGET_RANK, TARGET_COUNT,
-	 TARGET_DATATYPE, WIN, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Put(\fIorigin_addr\fP, \fIorigin_count\fP, \fIorigin_datatype\fP, \fItarget_rank\fP,
-		\fItarget_disp\fP, \fItarget_count\fP, \fItarget_datatype\fP, \fIwin\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIorigin_addr\fP
-	INTEGER, INTENT(IN) :: \fIorigin_count\fP, \fItarget_rank\fP, \fItarget_count\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIorigin_datatype\fP, \fItarget_datatype\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) :: \fItarget_disp\fP
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPI_Rput(\fIorigin_addr\fP, \fIorigin_count\fP, \fIorigin_datatype\fP, \fItarget_rank\fP,
-	\fItarget_disp\fP, \fItarget_count\fP, \fItarget_datatype\fP, \fIwin\fP, \fIrequest,\fP
-		\fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIorigin_addr\fP
-	INTEGER, INTENT(IN) :: \fIorigin_count\fP, \fItarget_rank\fP, \fItarget_count\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIorigin_datatype\fP, \fItarget_datatype\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) :: \fItarget_disp\fP
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-origin_addr
-Initial address of origin buffer (choice).
-.TP 1i
-origin_count
-Number of entries in origin buffer (nonnegative integer).
-.TP 1i
-origin_datatype
-Data type of each entry in origin buffer (handle).
-.TP 1i
-target_rank
-Rank of target (nonnegative integer).
-.TP 1i
-target_disp
-Displacement from start of window to target buffer (nonnegative integer).
-.TP 1i
-target_count
-Number of entries in target buffer (nonnegative integer).
-.TP 1i
-target_datatype
-Data type of each entry in target buffer (handle).
-.TP 1i
-win
-Window object used for communication (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-request
-MPI_Rput: RMA request
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-\fBMPI_Put\fP transfers \fIorigin_count\fP successive entries of the type specified by \fIorigin_datatype\fP, starting at address \fIorigin_addr\fP on the origin node to the target node specified by the \fIwin\fP, \fItarget_rank\fP pair. The data are written in the target buffer at address \fItarget_addr\fP = \fIwindow_base\fP + \fItarget_disp\fP x \fIdisp_unit\fP, where \fIwindow_base\fP and \fIdisp_unit\fP are the base address and window displacement unit specified at window initialization, by the target process.
-.sp
-The target buffer is specified by the arguments \fItarget_count\fP and \fItarget_datatype\fP.
-.sp
-The data transfer is the same as that which would occur if the origin process executed a send operation with arguments \fIorigin_addr\fP, \fIorigin_count\fP, \fIorigin_datatype\fP, \fItarget_rank\fP, \fItag\fP, \fIcomm\fP, and the target process executed a receive operation with arguments \fItarget_addr\fP, \fItarget_count\fP, \fItarget_datatype\fP, \fIsource\fP, \fItag\fP, \fIcomm\fP, where \fItarget_addr\fP is the target buffer address computed as explained above, and \fIcomm\fP is a communicator for the group of \fIwin\fP.
-.sp
-The communication must satisfy the same constraints as for a similar message-passing communication. The \fItarget_datatype\fP may not specify overlapping entries in the target buffer. The message sent must fit, without truncation, in the target buffer. Furthermore, the target buffer must fit in the target window. In addition, only processes within the same buffer can access the target window.
-.sp
-The \fItarget_datatype\fP argument is a handle to a datatype object defined at the origin process. However, this object is interpreted at the target process: The outcome is as if the target datatype object were defined at the target process, by the same sequence of calls used to define it at the origin process. The target data type must contain only relative displacements, not absolute addresses. The same holds for get and accumulate.
-.sp
-\fBMPI_Rput\fP is similar to \fBMPI_Put\fP, except that it allocates a communication request object and associates it with the request handle (the argument \fIrequest\fP). The completion of an MPI_Rput operation (i.e., after the corresponding test or wait) indicates that the sender is now free to update the locations in the \fIorigin_addr\fP buffer. It does not indicate that the data is available at the target window. If remote completion is required, \fBMPI_Win_flush\fP, \fBMPI_Win_flush_all\fP, \fBMPI_Win_unlock\fP, or \fBMPI_Win_unlock_all\fP can be used.
-
-.SH NOTES
-The \fItarget_datatype\fP argument is a handle to a datatype object that is defined at the origin process, even though it defines a data layout in the target process memory. This does not cause problems in a homogeneous or heterogeneous environment, as long as only portable data types are used (portable data types are defined in Section 2.4 of the MPI-2 Standard).
-.sp
-The performance of a put transfer can be significantly affected, on some systems, from the choice of window location and the shape and location of the origin and target buffer: Transfers to a target window in memory allocated by MPI_Alloc_mem may be much faster on shared memory systems; transfers from contiguous buffers will be faster on most, if not all, systems; the alignment of the communication buffers may also impact performance.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fITARGET_DISP\fP argument only for Fortran 90. FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_ADDRESS_KIND \fITARGET_DISP\fP
-.fi
-.sp
-where MPI_ADDRESS_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Get
-MPI_Rget
-.br
-MPI_Accumulate
-MPI_Win_flush
-MPI_Win_flush_all
-MPI_Win_unlock
-MPI_Win_unlock_all
-
diff --git a/ompi/mpi/man/man3/MPI_Query_thread.3in b/ompi/mpi/man/man3/MPI_Query_thread.3in
deleted file mode 100644
index c25667fc61d..00000000000
--- a/ompi/mpi/man/man3/MPI_Query_thread.3in
+++ /dev/null
@@ -1,95 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Query_thread 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-
-.SH NAME
-\fBMPI_Query_thread\fP \- Returns the current level of thread support
-
-.SH SYNTAX
-.ft R
-
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Query_thread(int *\fIprovided\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_QUERY_THREAD(\fIPROVIDED, IERROR\fP)
-	INTEGER	\fIPROVIDED, IERROR \fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Query_thread(\fIprovided\fP, \fIierror\fP)
-	INTEGER, INTENT(OUT) :: \fIprovided\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-provided
-C/Fortran only: Level of thread support (integer).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-This routine returns in \fIprovided\fP the current level of thread
-support. If MPI was initialized by a call to MPI_Init_thread,
-\fIprovided\fP will have the same value as was returned by that
-function.
-.sp
-The possible values of \fIprovided\fP are as follows:
-.TP 2.4i
-MPI_THREAD_SINGLE
-Only one thread may execute.
-.TP 2.4i
-MPI_THREAD_FUNNELED
-If the process is multithreaded, only the thread
-that called MPI_Init[_thread] may make MPI calls.
-.TP 2.4i
-MPI_THREAD_SERIALIZED
-If the process is multithreaded, only one thread
-may make MPI library calls at one time.
-.TP 2.4i
-MPI_THREAD_MULTIPLE
-If the process is multithreaded, multiple threads
-may call MPI at once with no restrictions.
-
-.SH NOTES
-.ft R
-In Open MPI, \fIprovided\fP is always MPI_THREAD_SINGLE, unless the
-program has been linked with the multithreaded library, in which case
-\fIprovided\fP is MPI_THREAD_MULTIPLE.
-
-.SH ERRORS
-.ft R
-Almost all MPI routines return an error value; C routines as
-the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-.sp
-See the MPI man page for a full list of MPI error codes.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Init
-MPI_Init_thread
-
diff --git a/ompi/mpi/man/man3/MPI_Raccumulate.3in b/ompi/mpi/man/man3/MPI_Raccumulate.3in
deleted file mode 100644
index d1e293cc85f..00000000000
--- a/ompi/mpi/man/man3/MPI_Raccumulate.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Accumulate.3
diff --git a/ompi/mpi/man/man3/MPI_Recv.3in b/ompi/mpi/man/man3/MPI_Recv.3in
deleted file mode 100644
index 8e01654457d..00000000000
--- a/ompi/mpi/man/man3/MPI_Recv.3in
+++ /dev/null
@@ -1,116 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2007-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Recv 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Recv\fP \- Performs a standard-mode blocking receive.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Recv(void *\fIbuf\fP, int\fI count\fP, MPI_Datatype\fI datatype\fP,
-	int\fI source\fP, int\fI tag\fP, MPI_Comm\fI comm\fP, MPI_Status\fI *status\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_RECV(\fIBUF, COUNT, DATATYPE, SOURCE, TAG, COMM, STATUS, IERROR\fP)
-	<type>	\fIBUF\fP(*)
-	INTEGER	\fICOUNT, DATATYPE, SOURCE, TAG, COMM\fP
-	INTEGER	\fISTATUS(MPI_STATUS_SIZE), IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Recv(\fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIsource\fP, \fItag\fP, \fIcomm\fP, \fIstatus\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..) :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP, \fIsource\fP, \fItag\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Status) :: \fIstatus\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-count
-Maximum number of elements to receive (integer).
-.TP 1i
-datatype
-Datatype of each receive buffer entry (handle).
-.TP 1i
-source
-Rank of source (integer).
-.TP 1i
-tag
-Message tag (integer).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of receive buffer (choice).
-.TP 1i
-status
-Status object (status).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-This basic receive operation, MPI_Recv, is blocking: it returns only after the receive buffer contains the newly received message. A receive can complete before the matching send has completed (of course, it can complete only after the matching send has started).
-.sp
-The blocking semantics of this call are described in Section 3.4 of the MPI-1 Standard, "Communication Modes."
-.sp
-The receive buffer contains a number (defined by the value of \fIcount\fP) of consecutive elements. The first element in the set of elements is located at \fIaddress_buf\fP. The type of each of these elements is specified by \fIdatatype\fP.
-.sp
-The length of the received message must be less than or equal to the length of the receive buffer. An  MPI_ERR_TRUNCATE is returned upon the overflow condition.
-.sp
-If a message that is shorter than the length of the receive buffer arrives, then only
-those locations corresponding to the (shorter) received message are modified.
-
-.SH NOTES
-The \fIcount\fP argument indicates the maximum number of entries of type \fIdatatype\fP that can be received in a message. Once a message is received, use the MPI_Get_count function to determine the actual number of entries within that message.
-.sp
-To receive messages of unknown length, use the MPI_Probe function. (For more information about MPI_Probe and MPI_Cancel, see their respective man pages; also, see Section 3.8 of the MPI-1 Standard, "Probe and Cancel.")
-.sp
-A message can be received by a receive operation only if it is addressed to the receiving process, and if its source, tag, and communicator (comm) values match the source, tag, and comm values specified by the receive operation. The receive operation may specify a wildcard value for source and/or tag, indicating that any source and/or tag are acceptable. The wildcard value for source is source = MPI_ANY_SOURCE. The wildcard value for tag is tag = MPI_ANY_TAG. There is no wildcard value for comm. The scope of these wildcards is limited to the proceses in the group of the specified communicator.
-.sp
-The message tag is specified by the tag argument of the receive operation.
-.sp
-The argument source, if different from MPI_ANY_SOURCE, is specified as a rank within the process group associated with that same communicator (remote process group, for intercommunicators). Thus, the range of valid values for the source argument is {0,...,n-1} {MPI_ANY_SOURCE}, where n is the number of processes in this group.
-.sp
-Note the asymmetry between send and receive operations: A receive operation may accept messages from an arbitrary sender; on the other hand, a send operation must specify a unique receiver. This matches a "push" communication mechanism, where data transfer is effected by the sender (rather than a "pull" mechanism, where data transfer is effected by the receiver).
-.sp
-Source = destination is allowed, that is, a process can send a message to itself. However, it is not recommended for a process to send messages to itself using the blocking send and receive operations described above, since this may lead to deadlock. See Section 3.5 of the MPI-1 Standard, "Semantics of Point-to-Point Communication."
-.sp
-If your application does not need to examine the \fIstatus\fP field, you can save resources by using the predefined constant MPI_STATUS_IGNORE as a special value for the \fIstatus\fP argument.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Irecv
-MPI_Probe
-
-
-
diff --git a/ompi/mpi/man/man3/MPI_Recv_init.3in b/ompi/mpi/man/man3/MPI_Recv_init.3in
deleted file mode 100644
index d24d6be4602..00000000000
--- a/ompi/mpi/man/man3/MPI_Recv_init.3in
+++ /dev/null
@@ -1,105 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Recv_init 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Recv_init\fP \- Builds a handle for a receive.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Recv_init(void *\fIbuf\fP, int\fI count\fP, MPI_Datatype\fI datatype\fP,
-	int\fI source\fP, int\fI tag\fP, MPI_Comm\fI comm\fP, MPI_Request\fI *request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_RECV_INIT(\fIBUF, COUNT, DATATYPE, SOURCE, TAG, COMM, REQUEST,
-		IERROR\fP)
-	<type>	\fIBUF\fP(*)
-	INTEGER	\fICOUNT, DATATYPE, SOURCE, TAG, COMM, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Recv_init(\fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIsource\fP, \fItag\fP, \fIcomm\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP, \fIsource\fP, \fItag\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-count
-Maximum number of elements to receive (integer).
-.TP 1i
-datatype
-Type of each entry (handle).
-.TP 1i
-source
-Rank of source (integer).
-.TP 1i
-tag
-Message tag (integer).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH INPUT/OUTPUT PARAMETER
-.TP 1i
-buf
-Initial address of receive buffer (choice).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-request
-Communication request (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Creates a persistent communication request for a receive operation. The argument \fIbuf\fP is marked as OUT because the user gives permission to write on the receive buffer by passing the argument to MPI_Recv_init.
-.sp
-A persistent communication request is inactive after it is created -- no active communication is attached to the request.
-.sp
-A communication (send or receive) that uses a persistent request is initiated by the function MPI_Start or MPI_Startall.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Bsend_init
-.br
-MPI_Rsend_init
-.br
-MPI_Send_init
-.br
-MPI_Sssend_init
-.br
-MPI_Start
-.br
-MPI_Startall
-.br
-MPI_Request_free
-
diff --git a/ompi/mpi/man/man3/MPI_Reduce.3in b/ompi/mpi/man/man3/MPI_Reduce.3in
deleted file mode 100644
index 808c92d9a6c..00000000000
--- a/ompi/mpi/man/man3/MPI_Reduce.3in
+++ /dev/null
@@ -1,472 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Reduce 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Reduce, MPI_Ireduce\fP \- Reduces values on all processes within a group.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Reduce(const void *\fIsendbuf\fP, void *\fIrecvbuf\fP, int\fI count\fP,
-               MPI_Datatype\fI datatype\fP, MPI_Op\fI op\fP, int\fI root\fP,
-               MPI_Comm\fI comm\fP)
-
-int MPI_Ireduce(const void *\fIsendbuf\fP, void *\fIrecvbuf\fP, int\fI count\fP,
-                MPI_Datatype\fI datatype\fP, MPI_Op\fI op\fP, int\fI root\fP,
-                MPI_Comm\fI comm\fP, MPI_Request \fI*request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_REDUCE(\fISENDBUF, RECVBUF, COUNT, DATATYPE, OP, ROOT, COMM,
-		IERROR\fP)
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fICOUNT, DATATYPE, OP, ROOT, COMM, IERROR\fP
-
-MPI_IREDUCE(\fISENDBUF, RECVBUF, COUNT, DATATYPE, OP, ROOT, COMM,
-            REQUEST, IERROR\fP)
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fICOUNT, DATATYPE, OP, ROOT, COMM, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Reduce(\fIsendbuf\fP, \fIrecvbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIop\fP, \fIroot\fP, \fIcomm\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..) :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP, \fIroot\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Op), INTENT(IN) :: \fIop\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPI_Ireduce(\fIsendbuf\fP, \fIrecvbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIop\fP, \fIroot\fP, \fIcomm\fP, \fIrequest\fP,
-		\fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP, \fIroot\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Op), INTENT(IN) :: \fIop\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-sendbuf
-Address of send buffer (choice).
-.TP 1i
-count
-Number of elements in send buffer (integer).
-.TP 1i
-datatype
-Data type of elements of send buffer (handle).
-.TP 1i
-op
-Reduce operation (handle).
-.TP 1i
-root
-Rank of root process (integer).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-recvbuf
-Address of receive buffer (choice, significant only at root).
-.TP 1i
-request
-Request (handle, non-blocking only).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-The global reduce functions (MPI_Reduce, MPI_Op_create, MPI_Op_free, MPI_Allreduce, MPI_Reduce_scatter, MPI_Scan) perform a global reduce operation (such as sum, max, logical AND, etc.) across all the members of a group. The reduction operation can be either one of a predefined list of operations, or a user-defined operation. The global reduction functions come in several flavors: a reduce that returns the result of the reduction at one node, an all-reduce that returns this result at all nodes, and a scan (parallel prefix) operation. In addition, a reduce-scatter operation combines the functionality of a reduce and a scatter operation.
-.sp
-MPI_Reduce combines the elements provided in the input buffer of each process in the group, using the operation op, and returns the combined value in the output buffer of the process with rank root. The input buffer is defined by the arguments sendbuf, count, and datatype; the output buffer is defined by the arguments recvbuf, count, and datatype; both have the same number of elements, with the same type. The routine is called by all group members using the same arguments for count, datatype, op, root, and comm. Thus, all processes provide input buffers and output buffers of the same length, with elements of the same type. Each process can provide one element, or a sequence of elements, in which case the combine operation is executed element-wise on each entry of the sequence. For example, if the operation is MPI_MAX and the send buffer contains two elements that are floating-point numbers (count = 2 and datatype = MPI_FLOAT), then recvbuf(1) = global max (sendbuf(1)) and recvbuf(2) = global max(sendbuf(2)).
-.sp
-.SH USE OF IN-PLACE OPTION
-When the communicator is an intracommunicator, you can perform a reduce operation in-place (the output buffer is used as the input buffer).  Use the variable MPI_IN_PLACE as the value of the root process \fIsendbuf\fR.  In this case, the input data is taken at the root from the receive buffer, where it will be replaced by the output data.
-.sp
-Note that MPI_IN_PLACE is a special kind of value; it has the same restrictions on its use as MPI_BOTTOM.
-.sp
-Because the in-place option converts the receive buffer into a send-and-receive buffer, a Fortran binding that includes INTENT must mark these as INOUT, not OUT.
-.sp
-.SH WHEN COMMUNICATOR IS AN INTER-COMMUNICATOR
-.sp
-When the communicator is an inter-communicator, the root process in the first group combines data from all the processes in the second group and then performs the \fIop\fR operation.  The first group defines the root process.  That process uses MPI_ROOT as the value of its \fIroot\fR argument.  The remaining processes use MPI_PROC_NULL as the value of their \fIroot\fR argument.  All processes in the second group use the rank of that root process in the first group as the value of their \fIroot\fR argument.  Only the send buffer arguments are significant in the second group, and only the receive buffer arguments are significant in the root process of the first group.
-.sp
-.SH PREDEFINED REDUCE OPERATIONS
-.sp
-The set of predefined operations provided by MPI is listed below (Predefined Reduce Operations). That section also enumerates the datatypes each operation can be applied to. In addition, users may define their own operations that can be overloaded to operate on several datatypes, either basic or derived. This is further explained in the description of the user-defined operations (see the man pages for MPI_Op_create and MPI_Op_free).
-.sp
-The operation op is always assumed to be associative. All predefined operations are also assumed to be commutative. Users may define operations that are assumed to be associative, but not commutative. The ``canonical'' evaluation order of a reduction is determined by the ranks of the processes in the group. However, the implementation can take advantage of associativity, or associativity and commutativity, in order to change the order of evaluation. This may change the result of the reduction for operations that are not strictly associative and commutative, such as floating point addition.
-.sp
-Predefined operators work only with the MPI types listed below (Predefined Reduce Operations, and the section MINLOC and MAXLOC, below).  User-defined operators may operate on general, derived datatypes. In this case, each argument that the reduce operation is applied to is one element described by such a datatype, which may contain several basic values. This is further explained in Section 4.9.4 of the MPI Standard, "User-Defined Operations."
-
-The following predefined operations are supplied for MPI_Reduce and related functions MPI_Allreduce, MPI_Reduce_scatter, and MPI_Scan. These operations are invoked by placing the following in op:
-.sp
-.nf
-	Name                Meaning
-     ---------           --------------------
-	MPI_MAX             maximum
-	MPI_MIN             minimum
-	MPI_SUM             sum
-	MPI_PROD            product
-	MPI_LAND            logical and
-	MPI_BAND            bit-wise and
-	MPI_LOR             logical or
-	MPI_BOR             bit-wise or
-	MPI_LXOR            logical xor
-	MPI_BXOR            bit-wise xor
-	MPI_MAXLOC          max value and location
-	MPI_MINLOC          min value and location
-.fi
-.sp
-The two operations MPI_MINLOC and MPI_MAXLOC are discussed separately below (MINLOC and MAXLOC). For the other predefined operations, we enumerate below the allowed combinations of op and datatype arguments. First, define groups of MPI basic datatypes in the following way:
-.sp
-.nf
-	C integer:            MPI_INT, MPI_LONG, MPI_SHORT,
-	                      MPI_UNSIGNED_SHORT, MPI_UNSIGNED,
-	                      MPI_UNSIGNED_LONG
-	Fortran integer:      MPI_INTEGER
-	Floating-point:       MPI_FLOAT, MPI_DOUBLE, MPI_REAL,
-	                      MPI_DOUBLE_PRECISION, MPI_LONG_DOUBLE
-	Logical:              MPI_LOGICAL
-	Complex:              MPI_COMPLEX
-	Byte:                 MPI_BYTE
-.fi
-.sp
-Now, the valid datatypes for each option is specified below.
-.sp
-.nf
-	Op                      	Allowed Types
-     ----------------         ---------------------------
-	MPI_MAX, MPI_MIN		C integer, Fortran integer,
-						floating-point
-
-	MPI_SUM, MPI_PROD 		C integer, Fortran integer,
-						floating-point, complex
-
-	MPI_LAND, MPI_LOR,		C integer, logical
-	MPI_LXOR
-
-	MPI_BAND, MPI_BOR,		C integer, Fortran integer, byte
-	MPI_BXOR
-.fi
-.sp
-\fBExample 1:\fR A routine that computes the dot product of two vectors that are distributed across a  group of processes and returns the answer at process zero.
-.sp
-.nf
-    SUBROUTINE PAR_BLAS1(m, a, b, c, comm)
-    REAL a(m), b(m)       ! local slice of array
-    REAL c                ! result (at process zero)
-    REAL sum
-    INTEGER m, comm, i, ierr
-
-    ! local sum
-    sum = 0.0
-    DO i = 1, m
-       sum = sum + a(i)*b(i)
-    END DO
-
-    ! global sum
-    CALL MPI_REDUCE(sum, c, 1, MPI_REAL, MPI_SUM, 0, comm, ierr)
-    RETURN
-.fi
-.sp
-\fBExample 2:\fR A routine that computes the product of a vector and an array that are distributed across a  group of processes and returns the answer at process zero.
-.sp
-.nf
-    SUBROUTINE PAR_BLAS2(m, n, a, b, c, comm)
-    REAL a(m), b(m,n)    ! local slice of array
-    REAL c(n)            ! result
-    REAL sum(n)
-    INTEGER n, comm, i, j, ierr
-
-    ! local sum
-    DO j= 1, n
-      sum(j) = 0.0
-      DO i = 1, m
-        sum(j) = sum(j) + a(i)*b(i,j)
-      END DO
-    END DO
-
-    ! global sum
-    CALL MPI_REDUCE(sum, c, n, MPI_REAL, MPI_SUM, 0, comm, ierr)
-
-    ! return result at process zero (and garbage at the other nodes)
-    RETURN
-
-.fi
-.SH MINLOC AND MAXLOC
-.ft R
-The operator MPI_MINLOC is used to compute a global minimum and also an index attached to the minimum value. MPI_MAXLOC similarly computes a global maximum and index. One application of these is to compute a global minimum (maximum) and the rank of the process containing this value.
-
-.sp
-The operation that defines MPI_MAXLOC is
-.sp
-.nf
-         ( u )    (  v )      ( w )
-         (   )  o (    )   =  (   )
-         ( i )    (  j )      ( k )
-
-where
-
-    w = max(u, v)
-
-and
-
-         ( i            if u > v
-         (
-   k   = ( min(i, j)    if u = v
-         (
-         (  j           if u < v)
-
-
-MPI_MINLOC is defined similarly:
-
-         ( u )    (  v )      ( w )
-         (   )  o (    )   =  (   )
-         ( i )    (  j )      ( k )
-
-where
-
-    w = min(u, v)
-
-and
-
-         ( i            if u < v
-         (
-   k   = ( min(i, j)    if u = v
-         (
-         (  j           if u > v)
-
-
-.fi
-.sp
-
-Both operations are associative and commutative. Note that if MPI_MAXLOC is
-applied to reduce a sequence of pairs (u(0), 0), (u(1), 1),\ ..., (u(n-1),
-n-1), then the value returned is (u , r), where u= max(i) u(i) and r is
-the index of the first global maximum in the sequence. Thus, if each
-process supplies a value and its rank within the group, then a reduce
-operation with op = MPI_MAXLOC will return the maximum value and the rank
-of the first process with that value. Similarly, MPI_MINLOC can be used to
-return a minimum and its index. More generally, MPI_MINLOC computes a
-lexicographic minimum, where elements are ordered according to the first
-component of each pair, and ties are resolved according to the second
-component.
-.sp
-The reduce operation is defined to operate on arguments that consist of a
-pair: value and index. For both Fortran and C, types are provided to
-describe the pair. The potentially mixed-type nature of such arguments is a
-problem in Fortran. The problem is circumvented, for Fortran, by having the
-MPI-provided type consist of a pair of the same type as value, and coercing
-the index to this type also. In C, the MPI-provided pair type has distinct
-types and the index is an int.
-.sp
-In order to use MPI_MINLOC and MPI_MAXLOC in a reduce operation, one must
-provide a datatype argument that represents a pair (value and index). MPI
-provides nine such predefined datatypes. The operations MPI_MAXLOC and
-MPI_MINLOC can be used with each of the following datatypes:
-.sp
-.nf
-    Fortran:
-    Name                     Description
-    MPI_2REAL                pair of REALs
-    MPI_2DOUBLE_PRECISION    pair of DOUBLE-PRECISION variables
-    MPI_2INTEGER             pair of INTEGERs
-
-    C:
-    Name        	    	Description
-    MPI_FLOAT_INT            float and int
-    MPI_DOUBLE_INT           double and int
-    MPI_LONG_INT             long and int
-    MPI_2INT                 pair of ints
-    MPI_SHORT_INT            short and int
-    MPI_LONG_DOUBLE_INT      long double and int
-.fi
-.sp
-The data type MPI_2REAL is equivalent to:
-.nf
-    MPI_TYPE_CONTIGUOUS(2, MPI_REAL, MPI_2REAL)
-.fi
-.sp
-Similar statements apply for MPI_2INTEGER, MPI_2DOUBLE_PRECISION, and
-MPI_2INT.
-.sp
-The datatype MPI_FLOAT_INT is as if defined by the following sequence of
-instructions.
-.sp
-.nf
-    type[0] = MPI_FLOAT
-    type[1] = MPI_INT
-    disp[0] = 0
-    disp[1] = sizeof(float)
-    block[0] = 1
-    block[1] = 1
-    MPI_TYPE_STRUCT(2, block, disp, type, MPI_FLOAT_INT)
-.fi
-.sp
-Similar statements apply for MPI_LONG_INT and MPI_DOUBLE_INT.
-.sp
-\fBExample 3:\fR Each process has an array of 30 doubles, in C. For each of
-the 30 locations, compute the value and rank of the process containing the
-largest value.
-.sp
-.nf
-        \&...
-        /* each process has an array of 30 double: ain[30]
-         */
-        double ain[30], aout[30];
-        int  ind[30];
-        struct {
-            double val;
-            int   rank;
-        } in[30], out[30];
-        int i, myrank, root;
-
-        MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
-        for (i=0; i<30; ++i) {
-            in[i].val = ain[i];
-            in[i].rank = myrank;
-        }
-        MPI_Reduce( in, out, 30, MPI_DOUBLE_INT, MPI_MAXLOC, root, comm );
-        /* At this point, the answer resides on process root
-         */
-        if (myrank == root) {
-            /* read ranks out
-             */
-            for (i=0; i<30; ++i) {
-                aout[i] = out[i].val;
-                ind[i] = out[i].rank;
-            }
-        }
-.fi
-.sp
-.fi
-\fBExample 4:\fR  Same example, in Fortran.
-.sp
-.nf
-    \&...
-    ! each process has an array of 30 double: ain(30)
-
-    DOUBLE PRECISION ain(30), aout(30)
-    INTEGER ind(30);
-    DOUBLE PRECISION in(2,30), out(2,30)
-    INTEGER i, myrank, root, ierr;
-
-    MPI_COMM_RANK(MPI_COMM_WORLD, myrank);
-        DO I=1, 30
-            in(1,i) = ain(i)
-            in(2,i) = myrank    ! myrank is coerced to a double
-        END DO
-
-    MPI_REDUCE( in, out, 30, MPI_2DOUBLE_PRECISION, MPI_MAXLOC, root,
-                                                              comm, ierr );
-    ! At this point, the answer resides on process root
-
-    IF (myrank .EQ. root) THEN
-            ! read ranks out
-            DO I= 1, 30
-                aout(i) = out(1,i)
-                ind(i) = out(2,i)  ! rank is coerced back to an integer
-            END DO
-        END IF
-.fi
-.sp
-\fBExample 5:\fR Each process has a nonempty array of values.  Find the minimum global value, the rank of the process that holds it, and its index on this process.
-.sp
-.nf
-    #define  LEN   1000
-
-    float val[LEN];        /* local array of values */
-    int count;             /* local number of values */
-    int myrank, minrank, minindex;
-    float minval;
-
-    struct {
-        float value;
-        int   index;
-    } in, out;
-
-    /* local minloc */
-    in.value = val[0];
-    in.index = 0;
-    for (i=1; i < count; i++)
-        if (in.value > val[i]) {
-            in.value = val[i];
-            in.index = i;
-        }
-
-    /* global minloc */
-    MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
-    in.index = myrank*LEN + in.index;
-    MPI_Reduce( in, out, 1, MPI_FLOAT_INT, MPI_MINLOC, root, comm );
-        /* At this point, the answer resides on process root
-         */
-    if (myrank == root) {
-        /* read answer out
-         */
-        minval = out.value;
-        minrank = out.index / LEN;
-        minindex = out.index % LEN;
-.fi
-.sp
-All MPI objects (e.g., MPI_Datatype, MPI_Comm) are of type INTEGER in Fortran.
-.SH NOTES ON COLLECTIVE OPERATIONS
-
-The reduction functions (
-.I MPI_Op
-) do not return an error value.  As a result,
-if the functions detect an error, all they can do is either call
-.I MPI_Abort
-or silently skip the problem.  Thus, if you change the error handler from
-.I MPI_ERRORS_ARE_FATAL
-to something else, for example,
-.I MPI_ERRORS_RETURN
-,
-then no error may be indicated.
-
-The reason for this is the performance problems in ensuring that
-all collective routines return the same error value.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Allreduce
-.br
-MPI_Reduce_scatter
-.br
-MPI_Scan
-.br
-MPI_Op_create
-.br
-MPI_Op_free
-
-
-
diff --git a/ompi/mpi/man/man3/MPI_Reduce_local.3in b/ompi/mpi/man/man3/MPI_Reduce_local.3in
deleted file mode 100644
index 1b5d2f1ea90..00000000000
--- a/ompi/mpi/man/man3/MPI_Reduce_local.3in
+++ /dev/null
@@ -1,287 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright 2009-2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Reduce_local 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Reduce_local\fP \- Perform a local reduction
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Reduce_local(const void *\fIinbuf\fP, void *\fIinoutbuf\fP, int\fI count\fP,
-	MPI_Datatype\fI datatype\fP, MPI_Op\fI op\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_REDUCE_LOCAL(\fIINBUF, INOUTBUF, COUNT, DATATYPE, OP, IERROR\fP)
-	<type>	\fIINBUF(*), INOUTBUF(*)\fP
-	INTEGER	\fICOUNT, DATATYPE, OP, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Reduce_local(\fIinbuf\fP, \fIinoutbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIop\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIinbuf\fP
-	TYPE(*), DIMENSION(..) :: \fIinoutbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Op), INTENT(IN) :: \fIop\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-inbuf
-Address of input buffer (choice).
-.TP 1i
-count
-Number of elements in input buffer (integer).
-.TP 1i
-datatype
-Data type of elements of input buffer (handle).
-.TP 1i
-op
-Reduce operation (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-inoutbuf
-Address of in/out buffer (choice).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-The global reduce functions (MPI_Reduce_local, MPI_Op_create, MPI_Op_free, MPI_Allreduce, MPI_Reduce_local_scatter, MPI_Scan) perform a global reduce operation (such as sum, max, logical AND, etc.) across all the members of a group. The reduction operation can be either one of a predefined list of operations, or a user-defined operation. The global reduction functions come in several flavors: a reduce that returns the result of the reduction at one node, an all-reduce that returns this result at all nodes, and a scan (parallel prefix) operation. In addition, a reduce-scatter operation combines the functionality of a reduce and a scatter operation.
-.sp
-MPI_Reduce_local combines the elements provided in the input and input/output buffers of the local process, using the operation op, and returns the combined value in the inout/output buffer. The input buffer is defined by the arguments inbuf, count, and datatype; the output buffer is defined by the arguments inoutbuf, count, and datatype; both have the same number of elements, with the same type. The routine is a local call.  The process can provide one element, or a sequence of elements, in which case the combine operation is executed element-wise on each entry of the sequence. For example, if the operation is MPI_MAX and the input buffer contains two elements that are floating-point numbers (count = 2 and datatype = MPI_FLOAT), then inoutbuf(1) = global max (inbuf(1)) and inoutbuf(2) = global max(inbuf(2)).
-.sp
-.SH USE OF IN-PLACE OPTION
-The use of MPI_IN_PLACE is disallowed with MPI_Reduce_local.
-.sp
-.SH PREDEFINED REDUCE OPERATIONS
-.sp
-The set of predefined operations provided by MPI is listed below (Predefined Reduce Operations). That section also enumerates the datatypes each operation can be applied to. In addition, users may define their own operations that can be overloaded to operate on several datatypes, either basic or derived. This is further explained in the description of the user-defined operations (see the man pages for MPI_Op_create and MPI_Op_free).
-.sp
-The operation op is always assumed to be associative. All predefined operations are also assumed to be commutative. Users may define operations that are assumed to be associative, but not commutative. The ``canonical'' evaluation order of a reduction is determined by the ranks of the processes in the group. However, the implementation can take advantage of associativity, or associativity and commutativity, in order to change the order of evaluation. This may change the result of the reduction for operations that are not strictly associative and commutative, such as floating point addition.
-.sp
-Predefined operators work only with the MPI types listed below (Predefined Reduce Operations, and the section MINLOC and MAXLOC, below).  User-defined operators may operate on general, derived datatypes. In this case, each argument that the reduce operation is applied to is one element described by such a datatype, which may contain several basic values. This is further explained in Section 4.9.4 of the MPI Standard, "User-Defined Operations."
-
-The following predefined operations are supplied for MPI_Reduce_local and related functions MPI_Allreduce, MPI_Reduce_scatter, and MPI_Scan. These operations are invoked by placing the following in op:
-.sp
-.nf
-	Name                Meaning
-     ---------           --------------------
-	MPI_MAX             maximum
-	MPI_MIN             minimum
-	MPI_SUM             sum
-	MPI_PROD            product
-	MPI_LAND            logical and
-	MPI_BAND            bit-wise and
-	MPI_LOR             logical or
-	MPI_BOR             bit-wise or
-	MPI_LXOR            logical xor
-	MPI_BXOR            bit-wise xor
-	MPI_MAXLOC          max value and location
-	MPI_MINLOC          min value and location
-.fi
-.sp
-The two operations MPI_MINLOC and MPI_MAXLOC are discussed separately below (MINLOC and MAXLOC). For the other predefined operations, we enumerate below the allowed combinations of op and datatype arguments. First, define groups of MPI basic datatypes in the following way:
-.sp
-.nf
-	C integer:            MPI_INT, MPI_LONG, MPI_SHORT,
-	                      MPI_UNSIGNED_SHORT, MPI_UNSIGNED,
-	                      MPI_UNSIGNED_LONG
-	Fortran integer:      MPI_INTEGER
-	Floating-point:       MPI_FLOAT, MPI_DOUBLE, MPI_REAL,
-	                      MPI_DOUBLE_PRECISION, MPI_LONG_DOUBLE
-	Logical:              MPI_LOGICAL
-	Complex:              MPI_COMPLEX
-	Byte:                 MPI_BYTE
-.fi
-.sp
-Now, the valid datatypes for each option is specified below.
-.sp
-.nf
-	Op                      	Allowed Types
-     ----------------         ---------------------------
-	MPI_MAX, MPI_MIN		C integer, Fortran integer,
-						floating-point
-
-	MPI_SUM, MPI_PROD 		C integer, Fortran integer,
-						floating-point, complex
-
-	MPI_LAND, MPI_LOR,		C integer, logical
-	MPI_LXOR
-
-	MPI_BAND, MPI_BOR,		C integer, Fortran integer, byte
-	MPI_BXOR
-.fi
-.sp
-.SH MINLOC AND MAXLOC
-.ft R
-The operator MPI_MINLOC is used to compute a global minimum and also an index attached to the minimum value. MPI_MAXLOC similarly computes a global maximum and index. One application of these is to compute a global minimum (maximum) and the rank of the process containing this value.
-
-.sp
-The operation that defines MPI_MAXLOC is
-.sp
-.nf
-         ( u )    (  v )      ( w )
-         (   )  o (    )   =  (   )
-         ( i )    (  j )      ( k )
-
-where
-
-    w = max(u, v)
-
-and
-
-         ( i            if u > v
-         (
-   k   = ( min(i, j)    if u = v
-         (
-         (  j           if u < v)
-
-
-MPI_MINLOC is defined similarly:
-
-         ( u )    (  v )      ( w )
-         (   )  o (    )   =  (   )
-         ( i )    (  j )      ( k )
-
-where
-
-    w = min(u, v)
-
-and
-
-         ( i            if u < v
-         (
-   k   = ( min(i, j)    if u = v
-         (
-         (  j           if u > v)
-
-
-.fi
-.sp
-
-Both operations are associative and commutative. Note that if MPI_MAXLOC is
-applied to reduce a sequence of pairs (u(0), 0), (u(1), 1),\ ..., (u(n-1),
-n-1), then the value returned is (u , r), where u= max(i) u(i) and r is
-the index of the first global maximum in the sequence. Thus, if each
-process supplies a value and its rank within the group, then a reduce
-operation with op = MPI_MAXLOC will return the maximum value and the rank
-of the first process with that value. Similarly, MPI_MINLOC can be used to
-return a minimum and its index. More generally, MPI_MINLOC computes a
-lexicographic minimum, where elements are ordered according to the first
-component of each pair, and ties are resolved according to the second
-component.
-.sp
-The reduce operation is defined to operate on arguments that consist of a
-pair: value and index. For both Fortran and C, types are provided to
-describe the pair. The potentially mixed-type nature of such arguments is a
-problem in Fortran. The problem is circumvented, for Fortran, by having the
-MPI-provided type consist of a pair of the same type as value, and coercing
-the index to this type also. In C, the MPI-provided pair type has distinct
-types and the index is an int.
-.sp
-In order to use MPI_MINLOC and MPI_MAXLOC in a reduce operation, one must
-provide a datatype argument that represents a pair (value and index). MPI
-provides nine such predefined datatypes. The operations MPI_MAXLOC and
-MPI_MINLOC can be used with each of the following datatypes:
-.sp
-.nf
-    Fortran:
-    Name                     Description
-    MPI_2REAL                pair of REALs
-    MPI_2DOUBLE_PRECISION    pair of DOUBLE-PRECISION variables
-    MPI_2INTEGER             pair of INTEGERs
-
-    C:
-    Name        	    	Description
-    MPI_FLOAT_INT            float and int
-    MPI_DOUBLE_INT           double and int
-    MPI_LONG_INT             long and int
-    MPI_2INT                 pair of ints
-    MPI_SHORT_INT            short and int
-    MPI_LONG_DOUBLE_INT      long double and int
-.fi
-.sp
-The data type MPI_2REAL is equivalent to:
-.nf
-    MPI_TYPE_CONTIGUOUS(2, MPI_REAL, MPI_2REAL)
-.fi
-.sp
-Similar statements apply for MPI_2INTEGER, MPI_2DOUBLE_PRECISION, and
-MPI_2INT.
-.sp
-The datatype MPI_FLOAT_INT is as if defined by the following sequence of
-instructions.
-.sp
-.nf
-    type[0] = MPI_FLOAT
-    type[1] = MPI_INT
-    disp[0] = 0
-    disp[1] = sizeof(float)
-    block[0] = 1
-    block[1] = 1
-    MPI_TYPE_STRUCT(2, block, disp, type, MPI_FLOAT_INT)
-.fi
-.sp
-Similar statements apply for MPI_LONG_INT and MPI_DOUBLE_INT.
-.sp
-All MPI objects (e.g., MPI_Datatype, MPI_Comm) are of type INTEGER in Fortran.
-.SH NOTES ON COLLECTIVE OPERATIONS
-
-The reduction operators (
-.I MPI_Op
-) do not return an error value.  As a result,
-if the functions detect an error, all they can do is either call
-.I MPI_Abort
-or silently skip the problem.  Thus, if you change the error handler from
-.I MPI_ERRORS_ARE_FATAL
-to something else, for example,
-.I MPI_ERRORS_RETURN
-,
-then no error may be indicated.
-
-The reason for this is the performance problems in ensuring that
-all collective routines return the same error value.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Allreduce
-.br
-MPI_Reduce
-.br
-MPI_Reduce_scatter
-.br
-MPI_Scan
-.br
-MPI_Op_create
-.br
-MPI_Op_free
-
-
-
diff --git a/ompi/mpi/man/man3/MPI_Reduce_scatter.3in b/ompi/mpi/man/man3/MPI_Reduce_scatter.3in
deleted file mode 100644
index 288c0b69e12..00000000000
--- a/ompi/mpi/man/man3/MPI_Reduce_scatter.3in
+++ /dev/null
@@ -1,136 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Reduce_scatter 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Reduce_scatter, MPI_Ireduce_scatter\fP \- Combines values and scatters the results.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Reduce_scatter(const void *\fIsendbuf\fP, void\fI *recvbuf\fP, const int\fI recvcounts\fP[],
-	MPI_Datatype\fI datatype\fP, MPI_Op\fI op\fP, MPI_Comm\fI comm\fP)
-
-int MPI_Ireduce_scatter(const void *\fIsendbuf\fP, void\fI *recvbuf\fP, const int\fI recvcounts\fP[],
-	MPI_Datatype\fI datatype\fP, MPI_Op\fI op\fP, MPI_Comm\fI comm\fP, MPI_Request \fI*request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_REDUCE_SCATTER(\fISENDBUF, RECVBUF, RECVCOUNTS, DATATYPE, OP,
-		COMM, IERROR\fP)
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fIRECVCOUNTS(*), DATATYPE, OP, COMM, IERROR \fP
-
-MPI_IREDUCE_SCATTER(\fISENDBUF, RECVBUF, RECVCOUNTS, DATATYPE, OP,
-		COMM, REQUEST, IERROR\fP)
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fIRECVCOUNTS(*), DATATYPE, OP, COMM, REQUEST, IERROR \fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Reduce_scatter(\fIsendbuf\fP, \fIrecvbuf\fP, \fIrecvcounts\fP, \fIdatatype\fP, \fIop\fP, \fIcomm\fP,
-		\fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..) :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIrecvcounts(*)\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Op), INTENT(IN) :: \fIop\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPI_Ireduce_scatter(\fIsendbuf\fP, \fIrecvbuf\fP, \fIrecvcounts\fP, \fIdatatype\fP, \fIop\fP, \fIcomm\fP,
-		\fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN), ASYNCHRONOUS :: \fIrecvcounts(*)\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Op), INTENT(IN) :: \fIop\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-sendbuf
-Starting address of send buffer (choice).
-.TP 1i
-recvcounts
-Integer array specifying the number of elements in result distributed to
-each process. Array must be identical on all calling processes.
-.TP 1i
-datatype
-Datatype of elements of input buffer (handle).
-.TP 1i
-op
-Operation (handle).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-recvbuf
-Starting address of receive buffer (choice).
-.TP 1i
-request
-Request (handle, non-blocking only).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-
-MPI_Reduce_scatter first does an element-wise reduction on vector of \fIcount\fP\
- =\ S(i)\fIrecvcounts\fP[i] elements in the send buffer defined by \fIsendbuf\fP, \fIcount\fP, and
-\fIdatatype\fP. Next, the resulting vector of results is split into n disjoint
-segments, where n is the number of processes in the group. Segment i contains
-\fIrecvcounts\fP[i] elements. The ith segment is sent to process i and stored in
-the receive buffer defined by \fIrecvbuf\fP, \fIrecvcounts\fP[i], and \fIdatatype\fP.
-
-
-.SH USE OF IN-PLACE OPTION
-When the communicator is an intracommunicator, you can perform a reduce-scatter operation in-place (the output buffer is used as the input buffer).  Use the variable MPI_IN_PLACE as the value of the \fIsendbuf\fR.  In this case, the input data is taken from the top of the receive buffer.  The area occupied by the input data may be either longer or shorter than the data filled by the output data.
-.sp
-.SH WHEN COMMUNICATOR IS AN INTER-COMMUNICATOR
-.sp
-When the communicator is an inter-communicator, the reduce-scatter operation occurs in two phases.  First, the result of the reduction performed on the data provided by the processes in the first group is scattered among the processes in the second group.  Then the reverse occurs: the reduction performed on the data provided by the processes in the second group is scattered among the processes in the first group.  For each group, all processes provide the same \fIrecvcounts\fR argument, and the sum of the \fIrecvcounts\fR values should be the same for both groups.
-.sp
-.SH NOTES ON COLLECTIVE OPERATIONS
-
-The reduction functions (
-.I MPI_Op
-) do not return an error value.  As a result,
-if the functions detect an error, all they can do is either call
-.I MPI_Abort
-or silently skip the problem.  Thus, if you change the error handler from
-.I MPI_ERRORS_ARE_FATAL
-to something else, for example,
-.I MPI_ERRORS_RETURN
-,
-then no error may be indicated.
-
-The reason for this is the performance problems in ensuring that
-all collective routines return the same error value.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_Reduce_scatter_block.3in b/ompi/mpi/man/man3/MPI_Reduce_scatter_block.3in
deleted file mode 100644
index e2d7a10facb..00000000000
--- a/ompi/mpi/man/man3/MPI_Reduce_scatter_block.3in
+++ /dev/null
@@ -1,137 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" $COPYRIGHT$
-.TH MPI_Reduce_scatter_block 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Reduce_scatter_block, MPI_Ireduce_scatter_block\fP \- Combines values and scatters the results in blocks.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Reduce_scatter_block(const void *\fIsendbuf\fP, void\fI *recvbuf\fP, int\fI recvcount\fP,
-	MPI_Datatype\fI datatype\fP, MPI_Op\fI op\fP, MPI_Comm\fI comm\fP)
-
-int MPI_Ireduce_scatter_block(const void *\fIsendbuf\fP, void\fI *recvbuf\fP, int\fI recvcount\fP,
-	MPI_Datatype\fI datatype\fP, MPI_Op\fI op\fP, MPI_Comm\fI comm\fP, MPI_Request \fI*request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_REDUCE_SCATTER_BLOCK(\fISENDBUF, RECVBUF, RECVCOUNT, DATATYPE, OP,
-		COMM, IERROR\fP)
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fIRECVCOUNT, DATATYPE, OP, COMM, IERROR \fP
-
-MPI_IREDUCE_SCATTER_BLOCK(\fISENDBUF, RECVBUF, RECVCOUNT, DATATYPE, OP,
-		COMM, REQUEST, IERROR\fP)
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fIRECVCOUNT, DATATYPE, OP, COMM, REQUEST, IERROR \fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Ireduce_scatter_block(\fIsendbuf\fP, \fIrecvbuf\fP, \fIrecvcount\fP, \fIdatatype\fP, \fIop\fP, \fIcomm\fP,
-		\fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIrecvcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Op), INTENT(IN) :: \fIop\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPI_Ireduce_scatter_block(\fIsendbuf\fP, \fIrecvbuf\fP, \fIrecvcount\fP, \fIdatatype\fP, \fIop\fP, \fIcomm\fP,
-		\fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIrecvcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Op), INTENT(IN) :: \fIop\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-sendbuf
-Starting address of send buffer (choice).
-.TP 1i
-recvcount
-lement count per block (non-negative integer).
-.TP 1i
-datatype
-Datatype of elements of input buffer (handle).
-.TP 1i
-op
-Operation (handle).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-recvbuf
-Starting address of receive buffer (choice).
-.TP 1i
-request
-Request (handle, non-blocking only).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-
-MPI_Reduce_scatter_block first does an element-wise reduction on vector of \fIcount\fP\
- =\ n * \fIrecvcount\fP elements in the send buffer defined by \fIsendbuf\fP, \fIcount\fP, and
-\fIdatatype\fP, using the operation \fIop\fP, where n is the number of
-processes in the group of \fIcomm\fP. Next, the resulting vector of results is split into n disjoint
-segments, where n is the number of processes in the group. Each segments contains \fIrecvcount\fP
-elements. The ith segment is sent to process i and stored in the receive buffer defined by
-\fIrecvbuf\fP, \fIrecvcount\fP, and \fIdatatype\fP.
-
-
-.SH USE OF IN-PLACE OPTION
-When the communicator is an intracommunicator, you can perform a reduce-scatter operation in-place (the output buffer is used as the input buffer).  Use the variable MPI_IN_PLACE as the value of the \fIsendbuf\fR.  In this case, the input data is taken from the top of the receive buffer.  The area occupied by the input data may be either longer or shorter than the data filled by the output data.
-.sp
-.SH WHEN COMMUNICATOR IS AN INTER-COMMUNICATOR
-.sp
-When the communicator is an inter-communicator, the reduce-scatter operation occurs in two phases.  First, the result of the reduction performed on the data provided by the processes in the first group is scattered among the processes in the second group.  Then the reverse occurs: the reduction performed on the data provided by the processes in the second group is scattered among the processes in the first group.  For each group, all processes provide the same \fIrecvcounts\fR argument, and the sum of the \fIrecvcounts\fR values should be the same for both groups.
-.sp
-.SH NOTES ON COLLECTIVE OPERATIONS
-
-The reduction functions (
-.I MPI_Op
-) do not return an error value.  As a result,
-if the functions detect an error, all they can do is either call
-.I MPI_Abort
-or silently skip the problem.  Thus, if you change the error handler from
-.I MPI_ERRORS_ARE_FATAL
-to something else, for example,
-.I MPI_ERRORS_RETURN
-,
-then no error may be indicated.
-
-The reason for this is the performance problems in ensuring that
-all collective routines return the same error value.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-MPI_Reduce_scatter
diff --git a/ompi/mpi/man/man3/MPI_Register_datarep.3in b/ompi/mpi/man/man3/MPI_Register_datarep.3in
deleted file mode 100644
index 60f08b269e9..00000000000
--- a/ompi/mpi/man/man3/MPI_Register_datarep.3in
+++ /dev/null
@@ -1,102 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2015-2016 Research Organization for Information Science
-.\"                     and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Register_datarep 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Register_datarep\fP \- Defines data representation.
-
-.SH SYNTAX
-.ft R
-.nf
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Register_datarep(const char \fI*datarep\fP,
-	MPI_Datarep_conversion_function \fI*read_conversion_fn\fP,
-	MPI_Datarep_conversion_function \fI*write_conversion_fn\fP,
-	MPI_Datarep_extent_function \fI*dtype_file_extent_fn\fP,
-	void \fI*extra_state\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_REGISTER_DATAREP(\fIDATAREP\fP, \fIREAD_CONVERSION_FN\fP,
-	\fIWRITE_CONVERSION_FN\fP, \fIDTYPE_FILE_EXTENT_FN\fP,
-	\fIEXTRA_STATE\fP, \fIIERROR\fP)
-	CHARACTER*(*)	\fIDATAREP\fP
-	EXTERNAL	\fIREAD_CONVERSION_FN, WRITE_CONVERSION_FN, DTYPE_FILE_EXTENT_FN\fP
-	INTEGER	\fIIERROR\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND)	\fIEXTRA_STATE\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Register_datarep(\fIdatarep\fP, \fIread_conversion_fn\fP, \fIwrite_conversion_fn\fP,
-		\fIdtype_file_extent_fn\fP, \fIextra_state\fP, \fIierror\fP)
-	CHARACTER(LEN=*), INTENT(IN) :: \fIdatarep\fP
-	PROCEDURE(MPI_Datarep_conversion_function) :: \fIread_conversion_fn\fP
-	PROCEDURE(MPI_Datarep_conversion_function) :: \fIwrite_conversion_fn\fP
-	PROCEDURE(MPI_Datarep_extent_function) :: \fIdtype_file_extent_fn\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) :: \fIextra_state\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-datarep
-Data representation identifier (string).
-.ft R
-.TP 1i
-read_conversion_fn
-Function invoked to convert from file representation to native representation (function).
-.ft R
-.TP 1i
-write_conversion_fn
-Function invoked to convert from native representation to file representation (function).
-.ft R
-.TP 1i
-dtype_file_extent_fn
-Function invoked to get the extent of a data type as represented in the file (function).
-.ft R
-.TP 1i
-extra_state
-Extra state.
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Register_datarep defines a data representation. It associates the data representation's identifier (a string) with the functions that convert from file representation to the native representation and vice versa, with the function that gets the extent of a data type as represented in the file, as well as with "extra state," which is used for passing arguments. Once a data representation has been registered using this routine, you may specify its identifier as an argument to MPI_File_set_view, causing subsequent data-access operations to call the specified conversion functions.
-
-The call associates \fIread_conversion_fn\fP, \fIwrite_conversion_fn\fP, and \fIdtype_file_extent_fn\fP with the data representation identifier \fIdatarep\fP. \fIdatarep\fP can then be used as an argument to MPI_File_set_view, causing subsequent data access operations to call the conversion functions to convert all data items accessed between file data representation and native representation. MPI_Register_datarep is a local operation and only registers the data representation for the calling MPI process. If \fIdatarep\fP is already defined, an error in the error class MPI_ERR_DUP_DATAREP is raised using the default file error handler. The length of a data representation string is limited to the value of MPI_MAX_DATAREP_STRING. MPI_MAX_DATAREP_STRING must have a value of at least 64. No routines are provided to delete data representations and free the associated resources; it is not expected that an application will generate them in significant numbers.
-
-.SH NOTES
-.ft R
-
-The Fortran version of each MPI I/O routine includes a final argument,
-IERROR, which is not defined in the PARAMETERS sections. This argument is used to return the error status of the routine in the manner typical for Fortran library routines.
-.sp
-The C version of each routine returns an error status as an integer return value.
-.sp
-Error classes are found in mpi.h (for C) and mpif.h (for Fortran).
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. For MPI I/O function errors, the default error handler is set to MPI_ERRORS_RETURN. The error handler may be changed with MPI_File_set_errhandler; the predefined error handler MPI_ERRORS_ARE_FATAL may be used to make I/O errors fatal. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_Request_c2f.3in b/ompi/mpi/man/man3/MPI_Request_c2f.3in
deleted file mode 100644
index a13fce697dd..00000000000
--- a/ompi/mpi/man/man3/MPI_Request_c2f.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Comm_f2c.3
diff --git a/ompi/mpi/man/man3/MPI_Request_f2c.3in b/ompi/mpi/man/man3/MPI_Request_f2c.3in
deleted file mode 100644
index a13fce697dd..00000000000
--- a/ompi/mpi/man/man3/MPI_Request_f2c.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Comm_f2c.3
diff --git a/ompi/mpi/man/man3/MPI_Request_free.3in b/ompi/mpi/man/man3/MPI_Request_free.3in
deleted file mode 100644
index 7000fe5066e..00000000000
--- a/ompi/mpi/man/man3/MPI_Request_free.3in
+++ /dev/null
@@ -1,140 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" Copyright (c) 2020      FUJITSU LIMITED.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Request_free 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Request_free\fP \- Frees a communication request object.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Request_free(MPI_Request *request)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_REQUEST_FREE(REQUEST, IERROR)
-	INTEGER	REQUEST, IERROR
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Request_free(\fIrequest\fP, \fIierror\fP)
-	TYPE(MPI_Request), INTENT(INOUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-request
-      Communication request (handle).
-
-.SH DESCRIPTION
-.ft R
-This operation allows a request object to be deallocated without waiting for the associated communication to complete.
-.sp
-MPI_Request_free marks the request object for deallocation and sets request
-to MPI_REQUEST_NULL. Any ongoing communication that is associated with the request will be allowed to complete. The request will be deallocated only after its completion.
-
-.SH NOTES
-Once a request is freed by a call to MPI_Request_free, it is not possible to check for the successful completion of the associated communication with calls to MPI_Wait or MPI_Test. Also, if an error occurs subsequently during the communication, an error code cannot be returned to the user -- such an error must be treated as fatal. Questions arise as to how one knows when the operations have completed when using MPI_Request_free. Depending on the program logic, there may be other ways in which the program knows that certain operations have completed and this makes usage of MPI_Request_free practical. For example, an active send request could be freed when the logic of the program is such that the receiver sends a reply to the message sent -- the arrival of the reply informs the sender that the send has completed and the send buffer can be reused. An active receive request should never be freed, as the receiver will have no way to verify that the receive has completed and the receive buffer can be reused.
-
-.sp
-\fBExample:\fR
-.sp
-.nf
-    CALL MPI_COMM_RANK(MPI_COMM_WORLD, rank)
-    IF(rank.EQ.0) THEN
-        DO i=1, n
-          CALL MPI_ISEND(outval, 1, MPI_REAL, 1, 0, req, ierr)
-          CALL MPI_REQUEST_FREE(req, ierr)
-          CALL MPI_IRECV(inval, 1, MPI_REAL, 1, 0, req, ierr)
-          CALL MPI_WAIT(req, status, ierr)
-        END DO
-    ELSE    ! rank.EQ.1
-        CALL MPI_IRECV(inval, 1, MPI_REAL, 0, 0, req, ierr)
-        CALL MPI_WAIT(req, status)
-        DO I=1, n-1
-           CALL MPI_ISEND(outval, 1, MPI_REAL, 0, 0, req, ierr)
-           CALL MPI_REQUEST_FREE(req, ierr)
-           CALL MPI_IRECV(inval, 1, MPI_REAL, 0, 0, req, ierr)
-           CALL MPI_WAIT(req, status, ierr)
-        END DO
-        CALL MPI_ISEND(outval, 1, MPI_REAL, 0, 0, req, ierr)
-        CALL MPI_WAIT(req, status)
-    END IF
-.fi
-.sp
-This routine is normally used to free persistent requests created with
-either
-.I MPI_Recv_init
-or
-.I MPI_Send_init
-and friends.  However, it can be
-used to free a request created with
-.I MPI_Irecv
-or
-.I MPI_Isend
-and friends;
-in that case the use can not use the test/wait routines on the request.
-
-It
-.B is
-permitted to free an active request.  However, once freed, you can not
-use the request in a wait or test routine (e.g.,
-.I MPI_Wait
-).
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-MPI_Isend
-.br
-MPI_Irecv
-.br
-MPI_Issend
-.br
-MPI_Ibsend
-.br
-MPI_Irsend
-.br
-MPI_Recv_init
-.br
-MPI_Send_init
-.br
-MPI_Ssend_init
-.br
-MPI_Rsend_init
-.br
-MPI_Test
-.br
-MPI_Wait
-.br
-MPI_Waitall
-.br
-MPI_Waitany
-.br
-MPI_Waitsome
-.br
-MPI_Testall
-.br
-MPI_Testany
-.br
-MPI_Testsome
-
-
-
diff --git a/ompi/mpi/man/man3/MPI_Request_get_status.3in b/ompi/mpi/man/man3/MPI_Request_get_status.3in
deleted file mode 100644
index d4e5414f38e..00000000000
--- a/ompi/mpi/man/man3/MPI_Request_get_status.3in
+++ /dev/null
@@ -1,65 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Request_get_status 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Request_get_status\fP \- Access information associated with a request without freeing the request.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Request_get_status(MPI_Request \fIrequest\fP, int \fI*flag\fP, MPI_Status \fI*status\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_REQUEST_GET_STATUS(\fIREQUEST\fP, \fIFLAG\fP, \fISTATUS\fP, \fIIERROR\fP)
-	INTEGER	REQUEST, STATUS(MPI_STATUS_SIZE), IERROR
-	LOGICAL	FLAG
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Request_get_status(\fIrequest\fP, \fIflag\fP, \fIstatus\fP, \fIierror\fP)
-	TYPE(MPI_Request), INTENT(IN) :: \fIrequest\fP
-	LOGICAL, INTENT(OUT) :: \fIflag\fP
-	TYPE(MPI_Status) :: \fIstatus\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft
-.TP 1i
-request
-      Communication request (handle).
-
-.SH OUTPUT PARAMETERS
-.ft
-.TP 1i
-flag
-Boolean flag, same as from MPI_Test (logical).
-.ft
-.TP 1i
-status
-MPI_Status object if flag is true (status).
-
-.SH DESCRIPTION
-.ft R
-MPI_Request_get_status sets \fIflag\fP=\fItrue\fP if the operation is complete or sets \fIflag\fP=\fIfalse\fP if it is not complete. If the operation is complete, it returns in \fIstatus\fP the request status. It does not deallocate or inactivate the request; a subsequent call to test, wait, or free should be executed with that request.
-.sp
-If your application does not need to examine the \fIstatus\fP field, you can save resources by using the predefined constant MPI_STATUS_IGNORE as a special value for the \fIstatus\fP argument.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_Rget.3in b/ompi/mpi/man/man3/MPI_Rget.3in
deleted file mode 100644
index 4b4410dd0da..00000000000
--- a/ompi/mpi/man/man3/MPI_Rget.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Get.3
diff --git a/ompi/mpi/man/man3/MPI_Rget_accumulate.3in b/ompi/mpi/man/man3/MPI_Rget_accumulate.3in
deleted file mode 100644
index 86db5536072..00000000000
--- a/ompi/mpi/man/man3/MPI_Rget_accumulate.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Get_accumulate.3
diff --git a/ompi/mpi/man/man3/MPI_Rput.3in b/ompi/mpi/man/man3/MPI_Rput.3in
deleted file mode 100644
index 52e806a2e8b..00000000000
--- a/ompi/mpi/man/man3/MPI_Rput.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Put.3
diff --git a/ompi/mpi/man/man3/MPI_Rsend.3in b/ompi/mpi/man/man3/MPI_Rsend.3in
deleted file mode 100644
index 14db0479e99..00000000000
--- a/ompi/mpi/man/man3/MPI_Rsend.3in
+++ /dev/null
@@ -1,79 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Rsend 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Rsend\fP \- Ready send.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Rsend(const void *\fIbuf\fP, int\fI count\fP, MPI_Datatype\fI datatype\fP, int\fI dest\fP,
-	int\fI tag\fP, MPI_Comm\fI comm\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_RSEND(\fIBUF, COUNT, DATATYPE, DEST, TAG, COMM, IERROR\fP)
-	<type>	\fIBUF\fP(*)
-	INTEGER	\fICOUNT, DATATYPE, DEST, TAG, COMM, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Rsend(\fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIdest\fP, \fItag\fP, \fIcomm\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP, \fIdest\fP, \fItag\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of send buffer (choice).
-.TP 1i
-count
-Number of elements in send buffer (nonnegative integer).
-.TP 1i
-datatype
-Datatype of each send buffer element (handle).
-.TP 1i
-dest
-Rank of destination (integer).
-.TP 1i
-tag
-Message tag (integer).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-A ready send may only be called if the user can guarantee that a receive is
-already posted. It is an error if the receive is not posted before the
-ready send is called.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_Rsend_init.3in b/ompi/mpi/man/man3/MPI_Rsend_init.3in
deleted file mode 100644
index 6746e1f903d..00000000000
--- a/ompi/mpi/man/man3/MPI_Rsend_init.3in
+++ /dev/null
@@ -1,102 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Rsend_init 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Rsend_init\fP \- Builds a handle for a ready send.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Rsend_init(const void *\fIbuf\fP, int\fI count\fP, MPI_Datatype\fI datatype\fP,
-	int\fI dest\fP, int\fI tag\fP, MPI_Comm\fI comm\fP, MPI_Request\fI *request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_RSEND_INIT(\fIBUF, COUNT, DATATYPE, DEST, TAG, COMM, REQUEST,
-		IERROR\fP)
-	<type>	\fIBUF\fP(*)
-	INTEGER	\fICOUNT, DATATYPE, DEST, TAG, COMM, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Rsend_init(\fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIdest\fP, \fItag\fP, \fIcomm\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP, \fIdest\fP, \fItag\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of send buffer (choice).
-.TP 1i
-count
-Number of elements sent (integer).
-.TP 1i
-datatype
-Type of each element (handle).
-.TP 1i
-dest
-Rank of destination (integer).
-.TP 1i
-tag
-Message tag (integer).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-request
-Communication request (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Creates a persistent communication object for a ready mode send operation, and binds to it all the arguments of a send operation.
-.sp
-A communication (send or receive) that uses a persistent request is initiated by the function MPI_Start.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Bsend_init
-.br
-MPI_Send_init
-.br
-MPI_Sssend_init
-.br
-MPI_Recv_init
-.br
-MPI_Start
-.br
-MPI_Startall
-.br
-MPI_Request_free
-
diff --git a/ompi/mpi/man/man3/MPI_Scan.3in b/ompi/mpi/man/man3/MPI_Scan.3in
deleted file mode 100644
index 5ccb8ae51a9..00000000000
--- a/ompi/mpi/man/man3/MPI_Scan.3in
+++ /dev/null
@@ -1,235 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" Copyright (c) 2020      FUJITSU LIMITED.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Scan 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-
-.SH NAME
-\fBMPI_Scan, MPI_Iscan\fP \- Computes an inclusive scan (partial reduction)
-
-.SH SYNTAX
-.ft R
-
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Scan(const void *\fIsendbuf\fP, void *\fIrecvbuf\fP, int \fIcount\fP,
-             MPI_Datatype \fIdatatype\fP, MPI_Op \fIop\fP, MPI_Comm \fIcomm\fP)
-
-int MPI_Iscan(const void *\fIsendbuf\fP, void *\fIrecvbuf\fP, int \fIcount\fP,
-              MPI_Datatype \fIdatatype\fP, MPI_Op \fIop\fP, MPI_Comm \fIcomm\fP,
-              MPI_Request \fI*request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_SCAN(\fISENDBUF, RECVBUF, COUNT, DATATYPE, OP, COMM, IERROR\fP)
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fICOUNT, DATATYPE, OP, COMM, IERROR\fP
-
-MPI_ISCAN(\fISENDBUF, RECVBUF, COUNT, DATATYPE, OP, COMM, REQUEST, IERROR\fP)
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fICOUNT, DATATYPE, OP, COMM, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Scan(\fIsendbuf\fP, \fIrecvbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIop\fP, \fIcomm\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..) :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Op), INTENT(IN) :: \fIop\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPI_Iscan(\fIsendbuf\fP, \fIrecvbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIop\fP, \fIcomm\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Op), INTENT(IN) :: \fIop\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-sendbuf
-Send buffer (choice).
-.TP 1i
-count
-Number of elements in input buffer (integer).
-.TP 1i
-datatype
-Data type of elements of input buffer (handle).
-.TP 1i
-op
-Operation (handle).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-recvbuf
-Receive buffer (choice).
-.TP 1i
-request
-Request (handle, non-blocking only).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Scan is used to perform an inclusive prefix reduction on data
-distributed across the calling processes. The operation returns, in
-the \fIrecvbuf\fP of the process with rank i, the reduction
-(calculated according to the function \fIop\fP) of the values in the
-\fIsendbuf\fPs of processes with ranks 0, ..., i (inclusive). The type
-of operations supported, their semantics, and the constraints on send
-and receive buffers are as for MPI_Reduce.
-
-.SH EXAMPLE
-.ft R
-This example uses a user-defined operation to produce a segmented
-scan. A segmented scan takes, as input, a set of values and a set of
-logicals, where the logicals delineate the various segments of the
-scan. For example,
-.sp
-.nf
-values     v1      v2      v3      v4      v5      v6      v7      v8
-logicals   0       0       1       1       1       0       0       1
-result     v1    v1+v2     v3    v3+v4  v3+v4+v5   v6    v6+v7     v8
-.fi
-.sp
-The result for rank j is thus the sum v(i) + ... + v(j), where i is
-the lowest rank such that for all ranks n, i <= n <= j, logical(n) =
-logical(j). The operator that produces this effect is
-.sp
-.nf
-      [ u ]     [ v ]     [ w ]
-      [   ]  o  [   ]  =  [   ]
-      [ i ]     [ j ]     [ j ]
-.fi
-.sp
-where
-.sp
-            ( u + v if i  = j
-      w  =  (
-            ( v     if i != j
-.fi
-.sp
-Note that this is a noncommutative operator. C code that implements it is
-given below.
-.sp
-.nf
-	typedef struct {
-		double val;
-		int log;
-	} SegScanPair;
-
-	/*
-	 * the user-defined function
-	 */
-	void segScan(SegScanPair *in, SegScanPair *inout, int *len,
-		MPI_Datatype *dptr)
-	{
-		int i;
-		SegScanPair c;
-
-		for (i = 0; i < *len; ++i) {
-			if (in->log == inout->log)
-				c.val = in->val + inout->val;
-			else
-				c.val = inout->val;
-
-			c.log = inout->log;
-			*inout = c;
-			in++;
-			inout++;
-		}
-	}
-.fi
-.sp
-Note that the inout argument to the user-defined function corresponds
-to the right-hand operand of the operator. When using this operator,
-we must be careful to specify that it is noncommutative, as in the
-following:
-.sp
-.nf
-	int			i, base;
-	SeqScanPair	a, answer;
-	MPI_Op		myOp;
-	MPI_Datatype	type[2] = {MPI_DOUBLE, MPI_INT};
-	MPI_Aint		disp[2];
-	int			blocklen[2] = {1, 1};
-	MPI_Datatype	sspair;
-
-	/*
-	 * explain to MPI how type SegScanPair is defined
-	 */
-	MPI_Get_address(a, disp);
-	MPI_Get_address(a.log, disp + 1);
-	base = disp[0];
-	for (i = 0; i < 2; ++i)
-		disp[i] -= base;
-	MPI_Type_struct(2, blocklen, disp, type, &sspair);
-	MPI_Type_commit(&sspair);
-
-	/*
-	 * create the segmented-scan user-op
-	 * noncommutative - set commute (arg 2) to 0
-	 */
-	MPI_Op_create((MPI_User_function *)segScan, 0, &myOp);
-	\&...
-	MPI_Scan(a, answer, 1, sspair, myOp, comm);
-.fi
-
-.SH USE OF IN-PLACE OPTION
-When the communicator is an intracommunicator, you can perform a scanning operation in place (the output buffer is used as the input buffer).  Use the variable MPI_IN_PLACE as the value of the \fIsendbuf\fR argument.  The input data is taken from the receive buffer and replaced by the output data.
-
-.SH NOTES ON COLLECTIVE OPERATIONS
-.ft R
-The reduction functions of type MPI_Op do not return an error value.
-As a result, if the functions detect an error, all they can do is
-either call MPI_Abort or silently skip the problem. Thus, if the
-error handler is changed from MPI_ERRORS_ARE_FATAL to something else
-(e.g., MPI_ERRORS_RETURN), then no error may be indicated.
-.sp
-The reason for this is the performance problems in ensuring that
-all collective routines return the same error value.
-
-.SH ERRORS
-.ft R
-Almost all MPI routines return an error value; C routines as
-the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-.sp
-See the MPI man page for a full list of MPI error codes.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Exscan
-MPI_Op_create
-MPI_Reduce
-
diff --git a/ompi/mpi/man/man3/MPI_Scatter.3in b/ompi/mpi/man/man3/MPI_Scatter.3in
deleted file mode 100644
index 1a97867c17c..00000000000
--- a/ompi/mpi/man/man3/MPI_Scatter.3in
+++ /dev/null
@@ -1,186 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Scatter 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Scatter, MPI_Iscatter\fP \- Sends data from one task to all tasks in a group.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Scatter(const void *\fIsendbuf\fP, int\fI sendcount\fP, MPI_Datatype\fI sendtype\fP,
-	void\fI *recvbuf\fP, int\fI recvcount\fP, MPI_Datatype\fI recvtype\fP, int\fI root\fP,
-	MPI_Comm\fI comm\fP)
-
-int MPI_Iscatter(const void *\fIsendbuf\fP, int\fI sendcount\fP, MPI_Datatype\fI sendtype\fP,
-	void\fI *recvbuf\fP, int\fI recvcount\fP, MPI_Datatype\fI recvtype\fP, int\fI root\fP,
-	MPI_Comm\fI comm\fP, MPI_Request \fI*request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_SCATTER(\fISENDBUF, SENDCOUNT, SENDTYPE, RECVBUF, RECVCOUNT,
-		RECVTYPE, ROOT, COMM, IERROR\fP)
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fISENDCOUNT, SENDTYPE, RECVCOUNT, RECVTYPE, ROOT\fP
-	INTEGER	\fICOMM, IERROR\fP
-
-MPI_ISCATTER(\fISENDBUF, SENDCOUNT, SENDTYPE, RECVBUF, RECVCOUNT,
-		RECVTYPE, ROOT, COMM, REQUEST, IERROR\fP)
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fISENDCOUNT, SENDTYPE, RECVCOUNT, RECVTYPE, ROOT\fP
-	INTEGER	\fICOMM, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Scatter(\fIsendbuf\fP, \fIsendcount\fP, \fIsendtype\fP, \fIrecvbuf\fP, \fIrecvcount\fP, \fIrecvtype\fP,
-		\fIroot\fP, \fIcomm\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..) :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIsendcount\fP, \fIrecvcount\fP, \fIroot\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPI_Iscatter(\fIsendbuf\fP, \fIsendcount\fP, \fIsendtype\fP, \fIrecvbuf\fP, \fIrecvcount\fP, \fIrecvtype\fP,
-		\fIroot\fP, \fIcomm\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIsendcount\fP, \fIrecvcount\fP, \fIroot\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-sendbuf
-Address of send buffer (choice, significant only at root).
-.TP 1i
-sendcount
-Number of elements sent to each process (integer, significant only at
-root).
-.TP 1i
-sendtype
-Datatype of send buffer elements (handle, significant only at root).
-.TP 1i
-recvcount
-Number of elements in receive buffer (integer).
-.TP 1i
-recvtype
-Datatype of receive buffer elements (handle).
-.TP 1i
-root
-Rank of sending process (integer).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-recvbuf
-Address of receive buffer (choice).
-.TP 1i
-request
-Request (handle, non-blocking only).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Scatter is the inverse operation to MPI_Gather.
-.sp
-The outcome is as if the root executed n send operations,
-.sp
-.nf
-    MPI_Send(sendbuf + i * sendcount * extent(sendtype), sendcount,
-             sendtype, i, \&...)
-.fi
-.sp
-and each process executed a receive,
-.sp
-.nf
-    MPI_Recv(recvbuf, recvcount, recvtype, i, \&...).
-.fi
-.sp
-An alternative description is that the root sends a message with
-MPI_Send(\fIsendbuf\fP, \fIsendcount\fP * \fIn\fP,\ \fIsendtype\fP, \&...). This message is split
-into \fIn\fP equal segments, the ith segment is sent to the ith process in the
-group, and each process receives this message as above.
-.sp
-The send buffer is ignored for all nonroot processes.
-.sp
-The type signature associated with \fIsendcount\fP, \fIsendtype\fP at the root must be
-equal to the type signature associated with \fIrecvcount\fP, \fIrecvtype\fP at all
-processes (however, the type maps may be different). This implies that the
-amount of data sent must be equal to the amount of data received, pairwise
-between each process and the root. Distinct type maps between sender and
-receiver are still allowed.
-.sp
-All arguments to the function are significant on process \fIroot\fP, while on
-other processes, only arguments \fIrecvbuf\fP, \fIrecvcount\fP, \fIrecvtype\fP, \fIroot\fP, \fIcomm\fP
-are significant. The arguments \fIroot\fP and \fIcomm\fP must have identical values on
-all processes.
-.sp
-The specification of counts and types should not cause any location on the
-root to be read more than once.
-.sp
-\fBRationale:\fR Though not needed, the last restriction is imposed so as
-to achieve symmetry with MPI_Gather, where the corresponding restriction (a
-multiple-write restriction) is necessary.
-.sp
-\fBExample:\fR The reverse of Example 1 in the MPI_Gather manpage. Scatter
-sets of 100 ints from the root to each process in the group.
-.sp
-.nf
-        MPI_Comm comm;
-        int gsize,*sendbuf;
-        int root, rbuf[100];
-        \&...
-        MPI_Comm_size(comm, &gsize);
-        sendbuf = (int *)malloc(gsize*100*sizeof(int));
-        \&...
-        MPI_Scatter(sendbuf, 100, MPI_INT, rbuf, 100,
-                    MPI_INT, root, comm);
-.fi
-
-.SH USE OF IN-PLACE OPTION
-When the communicator is an intracommunicator, you can perform a scatter operation in-place (the output buffer is used as the input buffer).  Use the variable MPI_IN_PLACE as the value of the root process \fIrecvbuf\fR.  In this case, \fIrecvcount\fR and \fIrecvtype\fR are ignored, and the root process sends no data to itself.
-.sp
-Note that MPI_IN_PLACE is a special kind of value; it has the same restrictions on its use as MPI_BOTTOM.
-.sp
-Because the in-place option converts the receive buffer into a send-and-receive buffer, a Fortran binding that includes INTENT must mark these as INOUT, not OUT.
-.sp
-.SH WHEN COMMUNICATOR IS AN INTER-COMMUNICATOR
-.sp
-When the communicator is an inter-communicator, the root process in the first group sends data to all processes in the second group.  The first group defines the root process.  That process uses MPI_ROOT as the value of its \fIroot\fR argument.  The remaining processes use MPI_PROC_NULL as the value of their \fIroot\fR argument.  All processes in the second group use the rank of that root process in the first group as the value of their \fIroot\fR argument.   The receive buffer argument of the root process in the first group must be consistent with the receive buffer argument of the processes in the second group.
-.sp
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-.nf
-MPI_Scatterv
-MPI_Gather
-MPI_Gatherv
-
diff --git a/ompi/mpi/man/man3/MPI_Scatterv.3in b/ompi/mpi/man/man3/MPI_Scatterv.3in
deleted file mode 100644
index fc8f46d2ec1..00000000000
--- a/ompi/mpi/man/man3/MPI_Scatterv.3in
+++ /dev/null
@@ -1,233 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" Copyright (c) 2020      FUJITSU LIMITED.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Scatterv 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Scatterv, MPI_Iscatterv\fP \- Scatters a buffer in parts to all tasks in a group.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Scatterv(const void *\fIsendbuf\fP, const int\fI sendcounts[]\fP, const int\fI displs[]\fP,
-	MPI_Datatype\fI sendtype\fP, void\fI *recvbuf\fP, int\fI recvcount\fP,
-	MPI_Datatype\fI recvtype\fP, int\fI root\fP, MPI_Comm\fI comm\fP)
-
-int MPI_Iscatterv(const void *\fIsendbuf\fP, const int\fI sendcounts[]\fP, const int\fI displs[]\fP,
-	MPI_Datatype\fI sendtype\fP, void\fI *recvbuf\fP, int\fI recvcount\fP,
-	MPI_Datatype\fI recvtype\fP, int\fI root\fP, MPI_Comm\fI comm\fP, MPI_Request \fI*request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_SCATTERV(\fISENDBUF, SENDCOUNTS, DISPLS, SENDTYPE, RECVBUF,
-		RECVCOUNT, RECVTYPE, ROOT, COMM, IERROR\fP)
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fISENDCOUNTS(*), DISPLS(*), SENDTYPE\fP
-	INTEGER	\fIRECVCOUNT, RECVTYPE, ROOT, COMM, IERROR\fP
-
-MPI_ISCATTERV(\fISENDBUF, SENDCOUNTS, DISPLS, SENDTYPE, RECVBUF,
-		RECVCOUNT, RECVTYPE, ROOT, COMM, REQUEST, IERROR\fP)
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fISENDCOUNTS(*), DISPLS(*), SENDTYPE\fP
-	INTEGER	\fIRECVCOUNT, RECVTYPE, ROOT, COMM, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Scatterv(\fIsendbuf\fP, \fIsendcounts\fP, \fIdispls\fP, \fIsendtype\fP, \fIrecvbuf\fP, \fIrecvcount\fP,
-		\fIrecvtype\fP, \fIroot\fP, \fIcomm\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..) :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIsendcounts(*)\fP, \fIdispls(*)\fP, \fIrecvcount\fP, \fIroot\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPI_Iscatterv(\fIsendbuf\fP, \fIsendcounts\fP, \fIdispls\fP, \fIsendtype\fP, \fIrecvbuf\fP, \fIrecvcount\fP,
-		\fIrecvtype\fP, \fIroot\fP, \fIcomm\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN), ASYNCHRONOUS :: \fIsendcounts(*)\fP, \fIdispls(*)\fP
-	INTEGER, INTENT(IN) :: \fIrecvcount\fP, \fIroot\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-sendbuf
-Address of send buffer (choice, significant only at root).
-.TP 1i
-sendcounts
-Integer array (of length group size) specifying the number of elements to
-send to each processor.
-.TP 1i
-displs
-Integer array (of length group size). Entry i specifies the displacement
-(relative to sendbuf) from which to take the outgoing data to process i.
-.TP 1i
-sendtype
-Datatype of send buffer elements (handle).
-.TP 1i
-recvcount
-Number of elements in receive buffer (integer).
-.TP 1i
-recvtype
-Datatype of receive buffer elements (handle).
-.TP 1i
-root
-Rank of sending process (integer).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-recvbuf
-Address of receive buffer (choice).
-.TP 1i
-request
-Request (handle, non-blocking only).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Scatterv is the inverse operation to MPI_Gatherv.
-.sp
-MPI_Scatterv extends the functionality of MPI_Scatter by allowing a varying
-count of data to be sent to each process, since \fIsendcounts\fP is now an array.
-It also allows more flexibility as to where the data is taken from on the
-root, by providing the new argument, \fIdispls\fP.
-.sp
-The outcome is as if the root executed \fIn\fP send operations,
-.sp
-.nf
-    MPI_Send(\fIsendbuf\fP + \fIdispls\fP[\fIi\fP] * \fIextent\fP(\fIsendtype\fP), \\
-             \fIsendcounts\fP[i], \fIsendtype\fP, \fIi\fP, \&...)
-
-and each process executed a receive,
-
-    MPI_Recv(\fIrecvbuf\fP, \fIrecvcount\fP, \fIrecvtype\fP, \fIroot\fP, \&...)
-
-The send buffer is ignored for all nonroot processes.
-.fi
-.sp
-The type signature implied by \fIsendcount\fP[\fIi\fP], \fIsendtype\fP at the root must be
-equal to the type signature implied by \fIrecvcount\fP, \fIrecvtype\fP at process \fIi\fP
-(however, the type maps may be different). This implies that the amount of
-data sent must be equal to the amount of data received, pairwise between
-each process and the root. Distinct type maps between sender and receiver
-are still allowed.
-.sp
-All arguments to the function are significant on process \fIroot\fP, while on
-other processes, only arguments \fIrecvbuf\fP, \fIrecvcount\fP, \fIrecvtype\fP, \fIroot\fP, \fIcomm\fP
-are significant. The arguments \fIroot\fP and \fIcomm\fP must have identical values on
-all processes.
-.sp
-The specification of counts, types, and displacements should not cause any
-location on the root to be read more than once.
-.sp
-\fBExample 1:\fR The reverse of Example 5 in the MPI_Gatherv manpage. We
-have a varying stride between blocks at sending (root) side, at the
-receiving side we receive 100 - \fIi\fP elements into the \fIi\fPth column of a 100 x 150 C array at process \fIi\fP.
-.sp
-.nf
-    MPI_Comm comm;
-        int gsize,recvarray[100][150],*rptr;
-        int root, *sendbuf, myrank, bufsize, *stride;
-        MPI_Datatype rtype;
-        int i, *displs, *scounts, offset;
-        \&...
-        MPI_Comm_size( comm, &gsize);
-        MPI_Comm_rank( comm, &myrank );
-
-        stride = (int *)malloc(gsize*sizeof(int));
-        \&...
-        /* stride[i] for i = 0 to gsize-1 is set somehow
-         * sendbuf comes from elsewhere
-         */
-        \&...
-        displs = (int *)malloc(gsize*sizeof(int));
-        scounts = (int *)malloc(gsize*sizeof(int));
-        offset = 0;
-        for (i=0; i<gsize; ++i) {
-            displs[i] = offset;
-            offset += stride[i];
-            scounts[i] = 100 - i;
-        }
-        /* Create datatype for the column we are receiving
-         */
-        MPI_Type_vector( 100-myrank, 1, 150, MPI_INT, &rtype);
-        MPI_Type_commit( &rtype );
-        rptr = &recvarray[0][myrank];
-        MPI_Scatterv(sendbuf, scounts, displs, MPI_INT,
-                     rptr, 1, rtype, root, comm);
-.fi
-.sp
-\fBExample 2:\fR The reverse of Example 1 in the MPI_Gather manpage. The
-root process scatters sets of 100 ints to the other processes, but the sets
-of 100 are stride ints apart in the sending buffer. Requires use of
-MPI_Scatterv, where \fIstride\fP >= 100.
-.sp
-.nf
-    MPI_Comm comm;
-        int gsize,*sendbuf;
-        int root, rbuf[100], i, *displs, *scounts;
-
-    \&...
-
-    MPI_Comm_size(comm, &gsize);
-        sendbuf = (int *)malloc(gsize*stride*sizeof(int));
-        \&...
-        displs = (int *)malloc(gsize*sizeof(int));
-        scounts = (int *)malloc(gsize*sizeof(int));
-        for (i=0; i<gsize; ++i) {
-            displs[i] = i*stride;
-            scounts[i] = 100;
-        }
-        MPI_Scatterv(sendbuf, scounts, displs, MPI_INT,
-                     rbuf, 100, MPI_INT, root, comm);
-.fi
-.SH USE OF IN-PLACE OPTION
-When the communicator is an intracommunicator, you can perform a scatter operation in-place (the output buffer is used as the input buffer).  Use the variable MPI_IN_PLACE as the value of the root process \fIrecvbuf\fR.  In this case, \fIrecvcount\fR and \fIrecvtype\fR are ignored, and the root process sends no data to itself.
-.sp
-Note that MPI_IN_PLACE is a special kind of value; it has the same restrictions on its use as MPI_BOTTOM.
-.sp
-Because the in-place option converts the receive buffer into a send-and-receive buffer, a Fortran binding that includes INTENT must mark these as INOUT, not OUT.
-.sp
-.SH WHEN COMMUNICATOR IS AN INTER-COMMUNICATOR
-.sp
-When the communicator is an inter-communicator, the root process in the first group sends data to all processes in the second group.  The first group defines the root process.  That process uses MPI_ROOT as the value of its \fIroot\fR argument.  The remaining processes use MPI_PROC_NULL as the value of their \fIroot\fR argument.  All processes in the second group use the rank of that root process in the first group as the value of their \fIroot\fR argument.   The receive buffer argument of the root process in the first group must be consistent with the receive buffer argument of the processes in the second group.
-.sp
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.sp
-.nf
-MPI_Gather
-MPI_Gatherv
-MPI_Scatter
-
-
-
diff --git a/ompi/mpi/man/man3/MPI_Send.3in b/ompi/mpi/man/man3/MPI_Send.3in
deleted file mode 100644
index a389ee6e883..00000000000
--- a/ompi/mpi/man/man3/MPI_Send.3in
+++ /dev/null
@@ -1,88 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2007-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Send 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Send\fP \- Performs a standard-mode blocking send.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Send(const void *\fIbuf\fP, int\fI count\fP, MPI_Datatype\fI datatype\fP, int\fI dest\fP,
-	int\fI tag\fP, MPI_Comm\fI comm\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_SEND(\fIBUF, COUNT, DATATYPE, DEST, TAG, COMM, IERROR\fP)
-	<type>	\fIBUF(*)\fP
-	INTEGER	\fICOUNT, DATATYPE, DEST, TAG, COMM, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Send(\fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIdest\fP, \fItag\fP, \fIcomm\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP, \fIdest\fP, \fItag\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of send buffer (choice).
-.TP 1i
-count
-Number of elements send (nonnegative integer).
-.TP 1i
-datatype
-Datatype of each send buffer element (handle).
-.TP 1i
-dest
-Rank of destination (integer).
-.TP 1i
-tag
-Message tag (integer).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Send performs a standard-mode, blocking send.
-
-.SH NOTE
-.ft R
-This routine will block until the message is sent to the destination. For an in-depth explanation of the semantics of the standard-mode send, refer to the MPI-1 Standard.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Isend
-MPI_Bsend
-MPI_Recv
-
diff --git a/ompi/mpi/man/man3/MPI_Send_init.3in b/ompi/mpi/man/man3/MPI_Send_init.3in
deleted file mode 100644
index 0209db645c7..00000000000
--- a/ompi/mpi/man/man3/MPI_Send_init.3in
+++ /dev/null
@@ -1,104 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Send_init 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Send_init\fP \- Builds a handle for a standard send.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Send_init(const void *\fIbuf\fP, int\fI count\fP, MPI_Datatype\fI datatype\fP,
-	int\fI dest\fP, int\fI tag\fP, MPI_Comm\fI comm\fP, MPI_Request\fI *request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_SEND_INIT(\fIBUF, COUNT, DATATYPE, DEST, TAG, COMM, REQUEST,
-		IERROR\fP)
-	<type>	\fIBUF\fP(*)
-	INTEGER	\fIREQUEST, COUNT, DATATYPE, DEST, TAG\fP
-	INTEGER	\fICOMM, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Send_init(\fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIdest\fP, \fItag\fP, \fIcomm\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP, \fIdest\fP, \fItag\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of send buffer (choice).
-.TP 1i
-count
-Number of elements to send (integer).
-.TP 1i
-datatype
-Type of each element (handle).
-.TP 1i
-dest
-Rank of destination (integer).
-.TP 1i
-tag
-Message tag (integer).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-request
-Communication request (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Creates a persistent communication request for a standard mode send operation, and binds to it all the arguments of a send operation.
-.sp
-A communication (send or receive) that uses a persistent request is initiated by the function MPI_Start or MPI_Startall.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Bsend_init
-.br
-MPI_Ssend_init
-.br
-MPI_Rsend_init
-.br
-MPI_Recv_init
-.br
-MPI_Start
-.br
-MPI_Startall
-.br
-MPI_Request_free
-
-
diff --git a/ompi/mpi/man/man3/MPI_Sendrecv.3in b/ompi/mpi/man/man3/MPI_Sendrecv.3in
deleted file mode 100644
index d7e2d757e67..00000000000
--- a/ompi/mpi/man/man3/MPI_Sendrecv.3in
+++ /dev/null
@@ -1,118 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Sendrecv 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Sendrecv\fP \- Sends and receives a message.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Sendrecv(const void *\fIsendbuf\fP, int\fI sendcount\fP, MPI_Datatype\fI sendtype\fP,
-	int\fI dest\fP, int\fI sendtag\fP, void\fI *recvbuf\fP, int\fI recvcount\fP,
-	MPI_Datatype\fI recvtype\fP, int\fI source\fP, int\fI recvtag\fP,
-	MPI_Comm\fI comm\fP, MPI_Status\fI *status\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_SENDRECV(\fISENDBUF, SENDCOUNT, SENDTYPE, DEST, SENDTAG,
-		RECVBUF, RECVCOUNT, RECVTYPE, SOURCE, RECVTAG, COMM,
-		STATUS, IERROR\fP)
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fISENDCOUNT, SENDTYPE, DEST, SENDTAG\fP
-	INTEGER	\fIRECVCOUNT, RECVTYPE, SOURCE, RECVTAG, COMM\fP
-	INTEGER	\fISTATUS(MPI_STATUS_SIZE), IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Sendrecv(\fIsendbuf\fP, \fIsendcount\fP, \fIsendtype\fP, \fIdest\fP, \fIsendtag\fP, \fIrecvbuf\fP,
-		\fIrecvcount\fP, \fIrecvtype\fP, \fIsource\fP, \fIrecvtag\fP, \fIcomm\fP, \fIstatus\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..) :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIsendcount\fP, \fIdest\fP, \fIsendtag\fP, \fIrecvcount\fP, \fIsource,\fP
-	\fIrecvtag\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Status) :: \fIstatus\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-sendbuf
-Initial address of send buffer (choice).
-.TP 1i
-sendcount
-Number of elements to send (integer).
-.TP 1i
-sendtype
-Type of elements in send buffer (handle).
-.TP 1i
-dest
-Rank of destination (integer).
-.TP 1i
-sendtag
-Send tag (integer).
-.TP 1i
-recvcount
-Maximum number of elements to receive (integer).
-.TP 1i
-recvtype
-Type of elements in receive buffer (handle).
-.TP 1i
-source
-Rank of source (integer).
-.TP 1i
-recvtag
-Receive tag (integer).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-recvbuf
-Initial address of receive buffer (choice).
-.TP 1i
-status
-Status object (status). This refers to the receive operation.
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-The send-receive operations combine in one call the sending of a message to one destination and the receiving of another message, from another process. The two (source and destination) are possibly the same. A send-receive operation is useful for executing a shift operation across a chain of processes. If blocking sends and receives are used for such a shift, then one needs to order the sends and receives correctly (for example, even processes send, then receive; odd processes receive first, then send) in order to prevent cyclic dependencies that may lead to deadlock. When a send-receive operation is used, the communication subsystem takes care of these issues. The send-receive operation can be used in conjunction with the functions described in Chapter 6 of the MPI-1 Standard, "Process Topologies," in order to perform shifts on various logical topologies. Also, a send-receive operation is useful for implementing remote procedure calls.
-.sp
-A message sent by a send-receive operation can be received by a regular receive operation or probed by a probe operation; a send-receive operation can receive a message sent by a regular send operation.
-.sp
-MPI_Sendrecv executes a blocking send and receive operation. Both send and receive use the same communicator, but possibly different tags. The send buffer and receive buffers must be disjoint, and may have different lengths and datatypes.
-.sp
-If your application does not need to examine the \fIstatus\fP field, you can save resources by using the predefined constant MPI_STATUS_IGNORE as a special value for the \fIstatus\fP argument.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Sendrecv_replace
-
-
diff --git a/ompi/mpi/man/man3/MPI_Sendrecv_replace.3in b/ompi/mpi/man/man3/MPI_Sendrecv_replace.3in
deleted file mode 100644
index 4d28f2021c2..00000000000
--- a/ompi/mpi/man/man3/MPI_Sendrecv_replace.3in
+++ /dev/null
@@ -1,107 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Sendrecv_replace 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Sendrecv_replace\fP \- Sends and receives a message using a single buffer.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Sendrecv_replace(void *\fIbuf\fP, int\fI count\fP, MPI_Datatype\fI datatype\fP,
-	int\fI dest\fP, int\fI sendtag\fP, int\fI source\fP, int\fI recvtag\fP, MPI_Comm\fI comm\fP,
-	MPI_Status\fI *status\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_SENDRECV_REPLACE(\fIBUF, COUNT, DATATYPE, DEST, SENDTAG, SOURCE,
-		RECVTAG, COMM, STATUS, IERROR\fP)
-	<type>	\fIBUF\fP(*)
-	INTEGER	\fICOUNT, DATATYPE, DEST, SENDTAG\fP
-	INTEGER	\fISOURCE, RECVTAG, COMM\fP
-	INTEGER	\fISTATUS(MPI_STATUS_SIZE), IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Sendrecv_replace(\fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIdest\fP, \fIsendtag\fP, \fIsource\fP, \fIrecvtag\fP,
-		\fIcomm\fP, \fIstatus\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..) :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP, \fIdest\fP, \fIsendtag\fP, \fIsource\fP, \fIrecvtag\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Status) :: \fIstatus\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-buf
-Initial address of send and receive buffer (choice).
-
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-count
-Number of elements in send and receive buffer (integer).
-.TP 1i
-datatype
-Type of elements to send and receive (handle).
-.TP 1i
-dest
-Rank of destination (integer).
-.TP 1i
-sendtag
-Send message tag (integer).
-.TP 1i
-source
-Rank of source (integer).
-.TP 1i
-recvtag
-Receive message tag (integer).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-status
-Status object (status).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-The send-receive operations combine in one call the sending of a message to one destination and the receiving of another message, from another process. The two (source and destination) are possibly the same. A send-receive operation is useful for executing a shift operation across a chain of processes. If blocking sends and receives are used for such a shift, then one needs to order the sends and receives correctly (for example, even processes send, then receive; odd processes receive first, then send) in order to prevent cyclic dependencies that may lead to deadlock. When a send-receive operation is used, the communication subsystem takes care of these issues. The send-receive operation can be used in conjunction with the functions described in Chapter 6 of the MPI Standard, "Process Topologies," in order to perform shifts on various logical topologies. Also, a send-receive operation is useful for implementing remote procedure calls.
-.sp
-A message sent by a send-receive operation can be received by a regular receive operation or probed by a probe operation; a send-receive operation can receive a message sent by a regular send operation.
-.sp
-MPI_Sendrecv_replace executes a blocking send and receive. The same buffer is used both for the send and for the receive, so that the message sent is replaced by the message received.
-.sp
-The semantics of a send-receive operation is what would be obtained if the caller forked two concurrent threads, one to execute the send, and one to execute the receive, followed by a join of these two threads.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Sendrecv
-
-
-
diff --git a/ompi/mpi/man/man3/MPI_Sizeof.3in b/ompi/mpi/man/man3/MPI_Sizeof.3in
deleted file mode 100644
index fb414c41a45..00000000000
--- a/ompi/mpi/man/man3/MPI_Sizeof.3in
+++ /dev/null
@@ -1,73 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Sizeof 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-
-.SH NAME
-\fBMPI_Sizeof\fP \- Returns the size, in bytes, of the given type
-
-.SH SYNTAX
-.ft R
-
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_SIZEOF(\fIX, SIZE, IERROR\fP)
-<type>	\fIX\fP
-INTEGER	\fISIZE, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Sizeof(\fIx\fP, \fIsize\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..) :: \fIx\fP
-	INTEGER, INTENT(OUT) :: \fIsize\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-X
-A Fortran variable of numeric intrinsic type (choice).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-SIZE
-Size of machine representation of that type (integer).
-.ft R
-.TP 1i
-IERROR
-Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_SIZEOF returns the size (in bytes) of the machine representation
-of the given variable. It is a generic Fortran type and has a Fortran
-binding only. This routine is similar to the sizeof builtin in
-C. However, if given an array argument, it returns the size of the
-base element, not the size of the whole array.
-
-.SH NOTES
-This function is not available in C because it is not necessary.
-
-.SH ERRORS
-.ft R
-Almost all MPI routines return an error value; C routines as
-the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-.sp
-See the MPI man page for a full list of MPI error codes.
-
diff --git a/ompi/mpi/man/man3/MPI_Ssend.3in b/ompi/mpi/man/man3/MPI_Ssend.3in
deleted file mode 100644
index 877d3628896..00000000000
--- a/ompi/mpi/man/man3/MPI_Ssend.3in
+++ /dev/null
@@ -1,77 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Ssend 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Ssend\fP \- Standard synchronous send.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Ssend(const void *\fIbuf\fP, int\fI count\fP, MPI_Datatype\fI datatype\fP, int\fI dest\fP,
-	int\fI tag\fP, MPI_Comm\fI comm\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_SSEND(\fIBUF, COUNT, DATATYPE, DEST, TAG, COMM, IERROR\fP)
-	<type>	\fIBUF\fP(*)
-	INTEGER	\fICOUNT, DATATYPE, DEST, TAG, COMM, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Ssend(\fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIdest\fP, \fItag\fP, \fIcomm\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP, \fIdest\fP, \fItag\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of send buffer (choice).
-.TP 1i
-count
-Number of elements in send buffer (nonnegative integer).
-.TP 1i
-datatype
-Datatype of each send buffer element (handle).
-.TP 1i
-dest
-Rank of destination (integer).
-.TP 1i
-tag
-Message tag (integer).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Ssend performs a synchronous-mode, blocking send. See the MPI-1 Standard for more detailed information about such sends.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_Ssend_init.3in b/ompi/mpi/man/man3/MPI_Ssend_init.3in
deleted file mode 100644
index f216cd92021..00000000000
--- a/ompi/mpi/man/man3/MPI_Ssend_init.3in
+++ /dev/null
@@ -1,103 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Ssend_init 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Ssend_init\fP \- Builds a handle for a synchronous send.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Ssend_init(const void *\fIbuf\fP, int\fI count\fP, MPI_Datatype\fI datatype\fP,
-	int\fI dest\fP, int\fI tag\fP, MPI_Comm\fI comm\fP, MPI_Request\fI *request\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_SSEND_INIT(\fIBUF, COUNT, DATATYPE, DEST, TAG, COMM, REQUEST,
-		IERROR\fP)
-	<type>	\fIBUF\fP(*)
-	INTEGER	\fICOUNT, DATATYPE, DEST, TAG, COMM, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Ssend_init(\fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIdest\fP, \fItag\fP, \fIcomm\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP, \fIdest\fP, \fItag\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-buf
-Initial address of send buffer (choice).
-.TP 1i
-count
-Number of elements to send (integer).
-.TP 1i
-datatype
-Type of each element (handle).
-.TP 1i
-dest
-Rank of destination (integer).
-.TP 1i
-tag
-Message tag (integer).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-request
-Communication request (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Creates a persistent communication object for a synchronous mode send operation, and binds to it all the arguments of a send operation.
-.sp
-A communication (send or receive) that uses a persistent request is initiated by the function MPI_Start.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Bsend_init
-.br
-MPI_Send_init
-.br
-MPI_Rsend_init
-.br
-MPI_Recv_init
-.br
-MPI_Start
-.br
-MPI_Startall
-.br
-MPI_Ssend
-
-
diff --git a/ompi/mpi/man/man3/MPI_Start.3in b/ompi/mpi/man/man3/MPI_Start.3in
deleted file mode 100644
index 32da019a2f6..00000000000
--- a/ompi/mpi/man/man3/MPI_Start.3in
+++ /dev/null
@@ -1,78 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Start 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Start\fP \- Initiates a communication using a persistent request handle.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Start(MPI_Request *\fIrequest\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_START(\fIREQUEST, IERROR\fP)
-	INTEGER	\fIREQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Start(\fIrequest\fP, \fIierror\fP)
-	TYPE(MPI_Request), INTENT(INOUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-request
-Communication request (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-A communication (send or receive) that uses a persistent request is initiated by the function MPI_Start.
-.sp
-The argument, request, is a handle returned by one of the persistent communication-request initialization functions (MPI_Send_init, MPI_Bsend_init,  MPI_Ssend_init, MPI_Rsend_init, MPI_Recv_init). The associated request should be inactive and becomes active once the call is made.
-.sp
-If the request is for a send with ready mode, then a matching receive should be posted before the call is made. From the time the call is made until after the operation completes, the communication buffer should not be accessed.
-.sp
-The call is local, with semantics similar to the nonblocking communication operations (see Section 3.7 in the MPI-1 Standard, "Nonblocking Communication.") That is, a call to MPI_Start with a request created by MPI_Send_init starts a communication in the same manner as a call to MPI_Isend; a call to MPI_Start with a request created by MPI_Bsend_init starts a communication in the same manner as a call to MPI_Ibsend; and so on.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Bsend_init
-.br
-MPI_Rsend_init
-.br
-MPI_Send_init
-.br
-MPI_Sssend_init
-.br
-MPI_Recv_init
-.br
-MPI_Startall
-
-
diff --git a/ompi/mpi/man/man3/MPI_Startall.3in b/ompi/mpi/man/man3/MPI_Startall.3in
deleted file mode 100644
index 1f2b366134e..00000000000
--- a/ompi/mpi/man/man3/MPI_Startall.3in
+++ /dev/null
@@ -1,95 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Startall 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Startall\fP \- Starts a collection of requests.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Startall(int \fIcount\fP, MPI_Request\fI array_of_requests[]\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_STARTALL(\fICOUNT, ARRAY_OF_REQUESTS, IERROR\fP)
-	INTEGER	\fICOUNT, ARRAY_OF_REQUESTS(*), IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Startall(\fIcount\fP, \fIarray_of_requests\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Request), INTENT(INOUT) :: \fIarray_of_requests(count)\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-count
-List length (integer).
-
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-array_of_requests
-Array of requests (array of handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Starts all communications associated with requests in array_of_requests. A call to  MPI_Startall(count, array_of_requests) has the same effect as calls to MPI_Start (&array_of_requests[i]), executed for i=0 ,..., count-1, in some arbitrary order.
-.sp
-A communication started with a call to MPI_Start or MPI_Startall is completed by a call to MPI_Wait, MPI_Test, or one of the derived functions MPI_Waitany, MPI_Testany, MPI_Waitall, MPI_Testall, MPI_Waitsome, MPI_Testsome (these are described in Section 3.7.5 of the MPI-1 Standard, "Multiple Completions"). The request becomes inactive after successful completion by such a call. The request is not deallocated, and it can be activated anew by another MPI_Start or MPI_Startall call.
-.sp
-A persistent request is deallocated by a call to MPI_Request_free (see Section 3.7.3  of the MPI-1 Standard, "Communication Completion").
-.sp
-The call to MPI_Request_free can occur at any point in the program after the persistent request was created. However, the request will be deallocated only after it becomes inactive. Active receive requests should not be freed. Otherwise, it will not be possible to check that the receive has completed. It is preferable, in general, to free requests when they are inactive. If this rule is followed, then the persistent communication request functions will be invoked in a sequence of the form,
-.br
-.sp
-    Create (Start Complete)* Free
-.br
-.sp
-where * indicates zero or more repetitions. If the same communication object is used in several concurrent threads, it is the user's responsibility to coordinate calls so that the correct sequence is obeyed.
-.sp
-A send operation initiated with MPI_Start can be matched with any receive operation and, likewise, a receive operation initiated with MPI_Start can receive messages generated by any send operation.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Bsend_init
-.br
-MPI_Rsend_init
-.br
-MPI_Send_init
-.br
-MPI_Ssend_init
-.br
-MPI_Recv_init
-.br
-MPI_Start
-.br
-MPI_Request_free
-
-
diff --git a/ompi/mpi/man/man3/MPI_Status_c2f.3in b/ompi/mpi/man/man3/MPI_Status_c2f.3in
deleted file mode 100644
index da04a6e9097..00000000000
--- a/ompi/mpi/man/man3/MPI_Status_c2f.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Status_f2c.3
diff --git a/ompi/mpi/man/man3/MPI_Status_c2f08.3in b/ompi/mpi/man/man3/MPI_Status_c2f08.3in
deleted file mode 100644
index b3c411bb42c..00000000000
--- a/ompi/mpi/man/man3/MPI_Status_c2f08.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Status_f082c.3
diff --git a/ompi/mpi/man/man3/MPI_Status_f082c.3.md b/ompi/mpi/man/man3/MPI_Status_f082c.3.md
deleted file mode 100644
index c9004565f7e..00000000000
--- a/ompi/mpi/man/man3/MPI_Status_f082c.3.md
+++ /dev/null
@@ -1,58 +0,0 @@
-# NAME
-
-MPI_Status_f082c, MPI_Status_c2f08 - Translates a C status into a Fortran 2008 status, or vice versa.
-
-# SYNTAX
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Status_f082c(const MPI_F08_status *f08_status, MPI_Status *c_status)
-int MPI_Status_c2f08(const MPI_Status *c_status, MPI_F08_status *f08_status)
-```
-
-# PARAMETERS
-
-* `f08_status`: `mpi_f08`-style MPI status object
-* `c_status`: C-style MPI status object
-
-# DESCRIPTION
-
-These two procedures are provided in C to convert from a Fortran 2008
-status (which is a derived type made of integers) to a C status (which
-is a structure), and vice versa. The conversion occurs on all the
-information in `status`, including that which is hidden. That is,
-no status information is lost in the conversion.
-
-When using `MPI_Status_f082c()`, if `f08_status` is a valid Fortran
-status, but not the Fortran value of `MPI_F08_STATUS_IGNORE` or
-`MPI_F08_STATUSES_IGNORE`, then `MPI_Status_f082c()` returns in
-`c_status` a valid C status with the same content. If `f08_status` is
-the Fortran value of `MPI_STATUS_IGNORE` or `MPI_STATUSES_IGNORE`, or
-if `f08_status` is not a valid Fortran status, then the call is
-erroneous.
-
-When using `MPI_Status_c2f08()`, the opposite conversion is applied. If
-`c_status` is `MPI_STATUS_IGNORE` or `MPI_STATUSES_IGNORE`, or if
-`c_status` is not a valid C status, then the call is erroneous.
-
-The input status has the same source, tag and error code values as the
-output status, and returns the same answers when queried for count,
-elements, and cancellation. The conversion function may be called with
-an input status argument that has an undefined error field, in which
-case the value of the error field in the output status argument is
-undefined.
-
-# NOTES
-
-These functions are only available in C; they are not available in any
-of the Fortran MPI interfaces.
-
-# SEE ALSO
-
-[`MPI_Status_c2f`(3)](MPI_Status_c2f.html),
-[`MPI_Status_f2c`(3)](MPI_Status_f2c.html),
-[`MPI_Status_f082f`(3)](MPI_Status_f082f.html),
-[`MPI_Status_f2f08`(3)](MPI_Status_f2f08.html)
diff --git a/ompi/mpi/man/man3/MPI_Status_f082f.3.md b/ompi/mpi/man/man3/MPI_Status_f082f.3.md
deleted file mode 100644
index a53af76bef8..00000000000
--- a/ompi/mpi/man/man3/MPI_Status_f082f.3.md
+++ /dev/null
@@ -1,92 +0,0 @@
-# NAME
-
-MPI_Status_f082f, MPI_Status_c2f08 - Translates a Fortran 2008 status into a Fortran INTEGER-style status, or vice versa.
-
-# SYNTAX
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Status_f082f(const MPI_F08_status *f08_status, MPI_Fint *f_status)
-int MPI_Status_f2f08(const MPI_Fint *f_status, MPI_F08_status *f08_status)
-```
-
-## Fortran mpi Module Syntax
-
-```fortran
-USE MPI
-
-MPI_STATUS_F082F(F08_STATUS, F_STATUS, IERROR)
-    TYPE(MPI_Status) :: F08_STATUS
-    INTEGER :: STATUS(MPI_STATUS_SIZE), IERROR
-
-MPI_STATUS_F2F08(F_STATUS, F08_STATUS, IERROR)
-    INTEGER :: F_STATUS(MPI_STATUS_SIZE), IERROR
-    TYPE(MPI_Status) :: F08_STATUS
-```
-
-## Fortran mpi_f08 Module Syntax
-
-```fortran
-USE mpi_f08
-
-MPI_Status_f082f(f08_status, f_status, ierror)
-    TYPE(MPI_Status), INTENT(IN) :: f08_status
-    INTEGER, INTENT(OUT) :: f_status(MPI_STATUS_SIZE)
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-
-MPI_Status_f2f08(f_status, f08_status, ierror)
-    INTEGER, INTENT(IN) :: f_status(MPI_STATUS_SIZE)
-    TYPE(MPI_Status), INTENT(OUT) :: f08_status
-    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-```
-
-# PARAMETERS
-
-* `f08_status`: `mpi_f08`-style MPI status object
-* `f_status`: `mpi`-style `INTEGER` MPI status object
-
-# DESCRIPTION
-
-These two procedures are provided to convert from a Fortran 2008
-status (which is a derived datatype made of integers) to a Fortran
-status (which is an array of integers), and vice versa. The conversion
-occurs on all the information in `status`, including that which is
-hidden. That is, no status information is lost in the conversion.
-
-When using `MPI_Status_f082f()`, if `f08_status` is a valid Fortran
-status, but not the Fortran value of `MPI_F08_STATUS_IGNORE` (in C),
-`MPI_STATUS_IGNORE` (in Fortran) or `MPI_F08_STATUSES_IGNORE` (in C)
-or `MPI_STATUSES_IGNORE` (in Fortran), then `MPI_Status_f082f()`
-returns in `f_status` a valid array with the same content. If
-`f08_status` is the C value of `MPI_F08_STATUS_IGNORE` or
-`MPI_F08_STATUSES_IGNORE` or the Fortran value of `MPI_STATUS_IGNORE`
-or `MPI_STATUSES_IGNORE`, or if `f08_status` is not a valid Fortran
-status, then the call is erroneous.
-
-When using `MPI_Status_f2f08()`, the opposite conversion is
-applied. If `f_status` is `MPI_STATUS_IGNORE` or
-`MPI_STATUSES_IGNORE`, or if `f_status` is not a valid Fortran status,
-then the call is erroneous.
-
-The input status has the same source, tag and error code values as the
-output status, and returns the same answers when queried for count,
-elements, and cancellation. The conversion function may be called with
-an input status argument that has an undefined error field, in which
-case the value of the error field in the output status argument is
-undefined.
-
-# NOTES
-
-The Fortran subroutines for these MPI routines are only available in
-the `mpi` and `mpi_f08` modules (including the type specification for
-`TYPE(MPI_Status); they are (intentionally) not available in `mpif.h`.
-
-# SEE ALSO
-
-[`MPI_Status_c2f`(3)](MPI_Status_c2f.html),
-[`MPI_Status_f2c`(3)](MPI_Status_c2f.html),
-[`MPI_Status_f082c`(3)](MPI_Status_f082c.html),
-[`MPI_Status_c2f08`(3)](MPI_Status_c2f08.html)
diff --git a/ompi/mpi/man/man3/MPI_Status_f2c.3.md b/ompi/mpi/man/man3/MPI_Status_f2c.3.md
deleted file mode 100644
index 4f24ffdc938..00000000000
--- a/ompi/mpi/man/man3/MPI_Status_f2c.3.md
+++ /dev/null
@@ -1,57 +0,0 @@
-# NAME
-
-MPI_Status_f2c, MPI_Status_f2c - Translates a C status into a Fortran INTEGER-style status, or vice versa.
-
-# SYNTAX
-
-## C Syntax
-
-```c
-#include <mpi.h>
-
-int MPI_Status_f2c(const MPI_Fint *f_status, MPI_Status *c_status)
-int MPI_Status_c2f(const MPI_Status *c_status, MPI_Fint *f_status)
-```
-
-# PARAMETERS
-
-* `f_status`: `mpi`-style `INTEGER` MPI status object
-* `c_status`: C-style MPI status object
-
-# DESCRIPTION
-
-These two procedures are provided in C to convert from a Fortran
-status (which is an array of integers) to a C status (which is a
-structure), and vice versa. The conversion occurs on all the
-information in `status`, including that which is hidden. That is,
-no status information is lost in the conversion.
-
-When using `MPI_Status_f2c()`, if `f_status` is a valid Fortran
-status, but not the Fortran value of `MPI_STATUS_IGNORE` or
-`MPI_STATUSES_IGNORE`, then `MPI_Status_f2c()` returns in `c_status` a
-valid C status with the same content. If `f_status` is the Fortran
-value of `MPI_STATUS_IGNORE` or `MPI_STATUSES_IGNORE`, or if
-`f_status` is not a valid Fortran status, then the call is erroneous.
-
-When using `MPI_Status_c2f()`, the opposite conversion is applied. If
-`c_status` is `MPI_STATUS_IGNORE` or `MPI_STATUSES_IGNORE`, or if
-`c_status` is not a valid C status, then the call is erroneous.
-
-The input status has the same source, tag and error code values as the
-output status, and returns the same answers when queried for count,
-elements, and cancellation. The conversion function may be called with
-an input status argument that has an undefined error field, in which
-case the value of the error field in the output status argument is
-undefined.
-
-# NOTES
-
-These functions are only available in C; they are not available in any
-of the Fortran MPI interfaces.
-
-# SEE ALSO
-
-[`MPI_Status_f082c`(3)](MPI_Status_f082c.html),
-[`MPI_Status_c2f08`(3)](MPI_Status_c2f08.html),
-[`MPI_Status_f082f`(3)](MPI_Status_f082f.html),
-[`MPI_Status_f2f08`(3)](MPI_Status_f2f08.html)
diff --git a/ompi/mpi/man/man3/MPI_Status_f2f08.3in b/ompi/mpi/man/man3/MPI_Status_f2f08.3in
deleted file mode 100644
index 67f77ae23bc..00000000000
--- a/ompi/mpi/man/man3/MPI_Status_f2f08.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Status_f082f.3
diff --git a/ompi/mpi/man/man3/MPI_Status_set_cancelled.3in b/ompi/mpi/man/man3/MPI_Status_set_cancelled.3in
deleted file mode 100644
index e7de779eb01..00000000000
--- a/ompi/mpi/man/man3/MPI_Status_set_cancelled.3in
+++ /dev/null
@@ -1,68 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Status_set_cancelled 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Status_set_cancelled\fP \- Sets \fIstatus\fP to indicate a request has been canceled.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Status_set_cancelled(MPI_Status *\fIstatus\fP, int \fIflag\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_STATUS_SET_CANCELLED(\fISTATUS, FLAG, IERROR\fP)
-	INTEGER	\fISTATUS\fP(MPI_STATUS_SIZE), \fIIERROR \fP
-	LOGICAL \fIFLAG\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Status_set_cancelled(\fIstatus\fP, \fIflag\fP, \fIierror\fP)
-	TYPE(MPI_Status), INTENT(INOUT) :: \fIstatus\fP
-	LOGICAL, INTENT(OUT) :: \fIflag\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-status
-Status with which to associate cancel flag (status).
-
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-flag
-If true, indicates request was canceled (logical).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-If \fIflag\fP is set to true, then a subsequent call to MPI_Test_cancelled(\fIstatus, flag\fP) will also return \fIflag\fP = true; otherwise it will return false.
-
-.SH NOTES
-.ft R
-Users are advised not to reuse the status fields for values other than those for which they were intended. Doing so may lead to unexpected results when using the status object. For example, calling MPI_Get_elements may cause an error if the value is out of range, or it may be impossible to detect such an error. The \fIextra_state\fP argument provided with a generalized request can be used to return information that does not logically belong in \fIstatus\fP. Furthermore, modifying the values in a status set internally by MPI, such as MPI_Recv, may lead to unpredictable results and is strongly discouraged.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_Status_set_elements.3in b/ompi/mpi/man/man3/MPI_Status_set_elements.3in
deleted file mode 100644
index ebda3c8dfc3..00000000000
--- a/ompi/mpi/man/man3/MPI_Status_set_elements.3in
+++ /dev/null
@@ -1,92 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Status_set_elements 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Status_set_elements\fP, \fBMPI_Status_set_elements_x\fP \- Modifies opaque part of \fIstatus\fP to allow MPI_Get_elements to return \fIcount\fP.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Status_set_elements(MPI_Status *\fIstatus\fP, MPI_Datatype \fIdatatype\fP, int \fIcount\fP)
-int MPI_Status_set_elements_x(MPI_Status *\fIstatus\fP, MPI_Datatype \fIdatatype\fP, MPI_Count \fIcount\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_STATUS_SET_ELEMENTS(\fISTATUS, DATATYPE, COUNT, IERROR\fP)
-	INTEGER	\fISTATUS\fP(MPI_STATUS_SIZE), DATATYPE, COUNT, IERROR\fP
-MPI_STATUS_SET_ELEMENTS_X(\fISTATUS, DATATYPE, COUNT, IERROR\fP)
-	INTEGER	\fISTATUS\fP(MPI_STATUS_SIZE), DATATYPE\fP
-        INTEGER(KIND=MPI_COUNT_KIND) \fICOUNT\fP
-        INTEGER \fIIERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Status_set_elements(\fIstatus\fP, \fIdatatype\fP, \fIcount\fP, \fIierror\fP)
-	TYPE(MPI_Status), INTENT(INOUT) :: \fIstatus\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-MPI_Status_set_elements_x(\fIstatus\fP, \fIdatatype\fP, \fIcount\fP, \fIierror\fP)
-	TYPE(MPI_Status), INTENT(INOUT) :: \fIstatus\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	INTEGER(KIND = MPI_COUNT_KIND), INTENT(IN) :: \fIcount\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-status
-Status to associate with \fIcount\fP (status).
-
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-datatype
-Data type associated with \fIcount\fP (handle).
-.TP 1i
-count
-Number of elements to associate with \fIstatus\fP (integer).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Status_set_elements modifies the opaque part of \fIstatus\fP so that a call to MPI_Get_elements or MPI_Get_elements_x will return \fIcount\fP. MPI_Get_count will return a compatible value.
-.sp
-A subsequent call to MPI_Get_count(\fIstatus, datatype, count\fP), to MPI_Get_elements(\fIstatus, datatype, count\fP), or to MPI_Get_elements_x(\fIstatus, datatype, count\fP) must use a data-type argument that has the same type signature as the data-type argument that was used in the call to MPI_Status_set_elements.
-
-.SH NOTES
-.ft R
-Users are advised not to reuse the status fields for values other than those for which they were intended. Doing so may lead to unexpected results when using the status object. For example, calling MPI_Get_elements may cause an error if the value is out of range, or it may be impossible to detect such an error. The \fIextra_state\fP argument provided with a generalized request can be used to return information that does not logically belong in \fIstatus\fP. Furthermore, modifying the values in a status set internally by MPI, such as MPI_Recv, may lead to unpredictable results and is strongly discouraged.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for the \fICOUNT\fP argument of MPI_Status_set_elements_x only for Fortran 90. FORTRAN 77 users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_COUNT_KIND \fICOUNT\fP
-.sp
-where MPI_COUNT_KIND is a constant defined in mpif.h and gives the length of the declared integer in bytes.
diff --git a/ompi/mpi/man/man3/MPI_Status_set_elements_x.3in b/ompi/mpi/man/man3/MPI_Status_set_elements_x.3in
deleted file mode 100644
index 4d643f1a3e5..00000000000
--- a/ompi/mpi/man/man3/MPI_Status_set_elements_x.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Status_set_elements.3
diff --git a/ompi/mpi/man/man3/MPI_T_category_changed.3in b/ompi/mpi/man/man3/MPI_T_category_changed.3in
deleted file mode 100644
index 095fb0d02b3..00000000000
--- a/ompi/mpi/man/man3/MPI_T_category_changed.3in
+++ /dev/null
@@ -1,38 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2010 Cisco Systems, Inc.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_T_category_changed 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.
-.SH NAME
-\fBMPI_T_category_changed\fP \- Get a timestamp for the categories
-.
-.SH SYNTAX
-.ft R
-.
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_T_category_changed(int *\fIstamp\fP)
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-stamp
-A virtual time stamp to indicate the last change to the categories.
-
-.SH DESCRIPTION
-.ft R
-If two subsequent calls to this routine return the same timestamp, it is guaranteed that
-no categories have been changed or added. If the timestamp from the second call is
-higher than some categories have been added or changed.
-
-.SH ERRORS
-.ft R
-MPI_T_category_changed() will fail if:
-.TP 1i
-[MPI_T_ERR_NOT_INITIALIZED]
-The MPI Tools interface not initialized
diff --git a/ompi/mpi/man/man3/MPI_T_category_get_categories.3in b/ompi/mpi/man/man3/MPI_T_category_get_categories.3in
deleted file mode 100644
index ff193dedc9f..00000000000
--- a/ompi/mpi/man/man3/MPI_T_category_get_categories.3in
+++ /dev/null
@@ -1,49 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2010 Cisco Systems, Inc.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_T_category_get_categories 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.
-.SH NAME
-\fBMPI_T_category_get_categories\fP \- Query which categories are in a category
-.
-.SH SYNTAX
-.ft R
-.
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_T_category_get_categories(int cat_index, int len, int indices[])
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-cat_index
-Index of the category to be queried.
-.TP 1i
-len
-The length of the indices array.
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-indices
-An integer array of size len, indicating category indices.
-
-.SH DESCRIPTION
-.ft R
-MPI_T_category_get_categories can be used to query which other categories are in
-a category.
-
-.SH ERRORS
-.ft R
-MPI_T_category_get_categories() will fail if:
-.TP 1i
-[MPI_T_ERR_NOT_INITIALIZED]
-The MPI Tools interface not initialized
-.TP 1i
-[MPI_T_ERR_INVALID_INDEX]
-The category index is invalid
diff --git a/ompi/mpi/man/man3/MPI_T_category_get_cvars.3in b/ompi/mpi/man/man3/MPI_T_category_get_cvars.3in
deleted file mode 100644
index 8041302327d..00000000000
--- a/ompi/mpi/man/man3/MPI_T_category_get_cvars.3in
+++ /dev/null
@@ -1,49 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2010 Cisco Systems, Inc.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_T_category_get_cvars 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.
-.SH NAME
-\fBMPI_T_category_get_cvars\fP \- Query which control variables are in a category
-.
-.SH SYNTAX
-.ft R
-.
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_T_category_get_cvars(int \fIcat_index\fP, int \fIlen\fP, int \fIindices\fP[])
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-cat_index
-Index of the category to be queried.
-.TP 1i
-len
-The length of the indices array.
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-indices
-An integer array of size len, indicating control variable indices.
-
-.SH DESCRIPTION
-.ft R
-MPI_T_category_get_cvars can be used to query which control variables are contained in a
-particular category.
-
-.SH ERRORS
-.ft R
-MPI_T_category_get_cvars() will fail if:
-.TP 1i
-[MPI_T_ERR_NOT_INITIALIZED]
-The MPI Tools interface not initialized
-.TP 1i
-[MPI_T_ERR_INVALID_INDEX]
-The category index is invalid
diff --git a/ompi/mpi/man/man3/MPI_T_category_get_info.3in b/ompi/mpi/man/man3/MPI_T_category_get_info.3in
deleted file mode 100644
index 811f362403a..00000000000
--- a/ompi/mpi/man/man3/MPI_T_category_get_info.3in
+++ /dev/null
@@ -1,81 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2010 Cisco Systems, Inc.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_T_category_get_info 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.
-.SH NAME
-\fBMPI_T_category_get_info\fP \- Query information from a category
-.
-.SH SYNTAX
-.ft R
-.
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_T_category_get_info(int \fIcat_index\fP, char *\fIname\fP, int *\fIname_len\fP,
-char *\fIdesc\fP, int *\fIdesc_len\fP, int *\fInum_cvars\fP, int *\fInum_pvars\fP,
-int *\fInum_categories\fP)
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-cat_index
-Index of the category to be queried.
-
-.SH INPUT/OUTPUT PARAMETERS
-.ft R
-.TP 1i
-name_len
-Length of the string and/or buffer for name.
-.TP 1i
-desc_len
-Length of the string and/or buffer for desc.
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-name
-Buffer to return the string containing the name of the
-category.
-.TP 1i
-desc
-Buffer to return the string containing the description
-of the category.
-.TP 1i
-num_cvars
-Number of control variables in the category.
-.TP 1i
-num_pvars
-Number of performance variables in the category.
-.TP 1i
-num_categories
-Number of categories contained in the category.
-
-.SH DESCRIPTION
-.ft R
-MPI_T_category_get_info can be used to query information from a category. The function returns the
-number of control variables, performance variables, and sub-categories in the queried category in
-the arguments \fInum_cvars\fP, \fInum_pvars\fP, and \fInum_categories\fP, respectively.
-
-.SH NOTES
-.ft R
-This MPI tool interface function returns two strings. This function takes two argument for each string:
-a buffer to store the string, and a length which must initially specify the size of the buffer. If the
-length passed is n then this function will copy at most n - 1 characters of the string into the
-corresponding buffer and set the length to the number of characters copied - 1. If the length argument
-is NULL or the value specified in the length is 0 the corresponding string buffer is ignored and the
-string is not returned.
-
-.SH ERRORS
-.ft R
-MPI_T_category_get_info() will fail if:
-.TP 1i
-[MPI_T_ERR_NOT_INITIALIZED]
-The MPI Tools interface not initialized
-.TP 1i
-[MPI_T_ERR_INVALID_INDEX]
-The category index is invalid
diff --git a/ompi/mpi/man/man3/MPI_T_category_get_num.3in b/ompi/mpi/man/man3/MPI_T_category_get_num.3in
deleted file mode 100644
index 3a6fa0de15f..00000000000
--- a/ompi/mpi/man/man3/MPI_T_category_get_num.3in
+++ /dev/null
@@ -1,36 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2010 Cisco Systems, Inc.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_T_category_get_num 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.
-.SH NAME
-\fBMPI_T_category_get_num\fP \- Query the number of categories
-.
-.SH SYNTAX
-.ft R
-.
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_T_category_get_num(int *\fInum_cat\fP)
-
-.fi
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-num_cat
-Current number of categories
-
-.SH DESCRIPTION
-.ft R
-MPI_T_category_get_num can be used to query the current number of categories.
-
-.SH ERRORS
-.ft R
-MPI_T_category_get_num() will fail if:
-.TP 1i
-[MPI_T_ERR_NOT_INITIALIZED]
-The MPI Tools interface not initialized
diff --git a/ompi/mpi/man/man3/MPI_T_category_get_pvars.3in b/ompi/mpi/man/man3/MPI_T_category_get_pvars.3in
deleted file mode 100644
index e8536161a2c..00000000000
--- a/ompi/mpi/man/man3/MPI_T_category_get_pvars.3in
+++ /dev/null
@@ -1,50 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2010 Cisco Systems, Inc.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_T_category_get_pvars 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.
-.SH NAME
-\fBMPI_T_category_get_pvars\fP \- Query which performance variables are in a category
-.
-.SH SYNTAX
-.ft R
-.
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_T_category_get_pvars(int cat_index, int len, int indices[])
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-cat_index
-Index of the category to be queried.
-.TP 1i
-len
-The length of the indices array.
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-indices
-An integer array of size len, indicating performance variable indices.
-
-.SH DESCRIPTION
-.ft R
-MPI_T_category_get_pvars can be used to query which performance variables are
-contained in a particular category. A category contains zero or more performance variables.
-
-.SH ERRORS
-.ft R
-MPI_T_category_get_pvars() will fail if:
-.TP 1i
-[MPI_T_ERR_NOT_INITIALIZED]
-The MPI Tools interface not initialized
-.TP 1i
-[MPI_T_ERR_INVALID_INDEX]
-The category index is invalid
-.
diff --git a/ompi/mpi/man/man3/MPI_T_cvar_get_info.3in b/ompi/mpi/man/man3/MPI_T_cvar_get_info.3in
deleted file mode 100644
index 07a150b0845..00000000000
--- a/ompi/mpi/man/man3/MPI_T_cvar_get_info.3in
+++ /dev/null
@@ -1,170 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2010 Cisco Systems, Inc.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_T_cvar_get_info 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.
-.SH NAME
-\fBMPI_T_cvar_get_info\fP \- Query information from a control variable
-.
-.SH SYNTAX
-.ft R
-.
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_T_cvar_get_info(int \fIcvar_index\fP, char *\fIname\fP, int *\fIname_len\fP,
-                        int *\fIverbosity\fP, MPI_Datatype *\fIdatatype\fP, MPI_T_enum *\fIenumtype\fP,
-                        const *\fIdesc\fP, int *\fIdesc_len\fP, int *\fIbind\fP, int *\fIscope\fP)
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-cvar_index
-Index of the control variable to be queried.
-.
-.
-.SH INPUT/OUTPUT PARAMETERS
-.ft R
-.TP 1i
-name_len
-Length of the string and/or buffer for name.
-.TP 1i
-desc_len
-Length of the string and/or buffer for desc.
-.
-.
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-name
-Buffer to return the string containing the name of the
-control variable.
-.TP 1i
-verbosity
-Verbosity level of this variable.
-.TP 1i
-datatype
-MPI datatype of the information stored in the control
-variable.
-.TP 1i
-enumtype
-Optional descriptor for enumeration information.
-.TP 1i
-desc
-Buffer to return the string containing the description
-of the control variable.
-.TP 1i
-bind
-Type of MPI object to which this variable must be
-bound.
-.TP 1i
-scope
-Scope of when changes to this variable are possible.
-.
-.
-.SH DESCRIPTION
-.ft R
-MPI_T_cvar_get_info can be used to query information about a control variable. The function returns
-the verbosity, datatype, enumeration type, binding, and scope of the queried control variable in the arguments
-\fIverbosity\fP, \fIdatatype\fP, \fIenumtype\fP, \fIbind\fP, and \fIscope\fP, respectively. Control variables
-in Open MPI are the same as MCA parameters.
-.
-.
-.SH VERBOSITY
-.ft R
-As Open MPI exposes a very large number of MCA parameters (control variables), control variables are
-categorized into nine verbosity levels corresponding to the equivalent ompi_info level. The nine levels are
-(in increasing order):
-.TP 1i
-MPI_T_VERBOSITY_USER_BASIC
-Basic information of interest to users
-.TP 1i
-MPI_T_VERBOSITY_USER_DETAIL
-Detailed information of interest to users
-.TP 1i
-MPI_T_VERBOSITY_USER_ALL
-All remaining information of interest to users
-.TP 1i
-MPI_T_VERBOSITY_TUNER_BASIC
-Basic information required for tuning
-.TP 1i
-MPI_T_VERBOSITY_TUNER_DETAIL
-Detailed information required for tuning
-.TP 1i
-MPI_T_VERBOSITY_TUNER_ALL
-All remaining information required for tuning
-.TP 1i
-MPI_T_VERBOSITY_MPIDEV_BASIC
-Basic information for MPI implementors
-.TP 1i
-MPI_T_VERBOSITY_MPIDEV_DETAIL
-Detailed information for MPI implementors
-.TP 1i
-MPI_T_VERBOSITY_MPIDEV_ALL
-All remaining information for MPI implementors
-
-For more information see MPI-3 \[char167] 14.3.1.
-
-.SH DATATYPE
-.ft R
-The datatype returned by MPI_T_cvar_get_info is restricted to one of the following datatypes: MPI_INT,
-MPI_UNSIGNED, MPI_UNSIGNED_LONG, MPI_UNSIGNED_LONG_LONG, MPI_COUNT, MPI_CHAR, and MPI_DOUBLE. For more
-information on datatypes in MPI_T see MPI-3 \[char167] 14.3.5.
-
-.SH SCOPE
-.ft R
-The scope describes when and how changes can be made to a control variable. From MPI-3 \[char167] 14.3.6, the scope may be any of the following:
-.TP 1i
-MPI_T_SCOPE_CONSTANT
-read-only, value is constant
-.TP 1i
-MPI_T_SCOPE_READONLY
-read-only, cannot be written, but can change
-.TP 1i
-MPI_T_SCOPE_LOCAL
-may be writeable, writing is a local operation
-.TP 1i
-MPI_T_SCOPE_GROUP
-may be writeable, must be done to a group of processes, all processes in a group must be set to consistent values
-.TP 1i
-MPI_T_SCOPE_GROUP_EQ
-may be writeable, must be done to a group of processes, all processes in a group must be set to the same value
-.TP 1i
-MPI_T_SCOPE_ALL
-may be writeable, must be done to all processes, all connected processes must be set to consistent values
-.TP 1i
-MPI_T_SCOPE_ALL_EQ
-may be writeable, must be done to all processes, all connected processes must be set to the same value
-
-For more information see MPI-3 \[char167] 14.3.6 Table 14.4.
-
-.SH NOTES
-.ft R
-This MPI tool interface function returns two strings. This function takes two argument for each string:
-a buffer to store the string, and a length which must initially specify the size of the buffer. If the
-length passed is n then this function will copy at most n - 1 characters of the string into the
-corresponding buffer and set the length to the number of characters copied - 1. If the length argument
-is NULL or the value specified in the length is 0 the corresponding string buffer is ignored and the
-string is not returned.
-.sp
-Open MPI does not currently support binding control variables to MPI objects.
-.
-.
-.SH ERRORS
-.ft R
-MPI_T_cvar_get_info() will fail if:
-.TP 1i
-[MPI_T_ERR_NOT_INITIALIZED]
-The MPI Tools interface not initialized
-.TP 1i
-[MPI_T_ERR_INVALID_INDEX]
-The control variable index is invalid
-.
-.SH SEE ALSO
-.ft R
-.nf
-ompi_info
diff --git a/ompi/mpi/man/man3/MPI_T_cvar_get_num.3in b/ompi/mpi/man/man3/MPI_T_cvar_get_num.3in
deleted file mode 100644
index f70c4abfecb..00000000000
--- a/ompi/mpi/man/man3/MPI_T_cvar_get_num.3in
+++ /dev/null
@@ -1,39 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2010 Cisco Systems, Inc.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_T_cvar_get_num 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.
-.SH NAME
-\fBMPI_T_cvar_get_num\fP \- Query the number of control variables
-.
-.SH SYNTAX
-.ft R
-.
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_T_cvar_get_num(int *\fInum_cvar\fP)
-
-.fi
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-num_cvar
-Current number of control variables.
-.
-.
-.SH DESCRIPTION
-.ft R
-MPI_T_cvar_get_num can be used to query the current number of control variables. The number
-of control variables may increase throughout the execution of the process but will never
-decrease.
-
-.SH ERRORS
-.ft R
-MPI_T_cvar_get_num() will fail if:
-.TP 1i
-[MPI_T_ERR_NOT_INITIALIZED]
-The MPI Tools interface not initialized
diff --git a/ompi/mpi/man/man3/MPI_T_cvar_handle_alloc.3in b/ompi/mpi/man/man3/MPI_T_cvar_handle_alloc.3in
deleted file mode 100644
index fa393b5cf66..00000000000
--- a/ompi/mpi/man/man3/MPI_T_cvar_handle_alloc.3in
+++ /dev/null
@@ -1,69 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2010 Cisco Systems, Inc.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_T_cvar_handle_alloc 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.
-.SH NAME
-\fBMPI_T_cvar_handle_alloc\fP, \fBMPI_T_cvar_handle_free\fP \- Allocate/free contol variable handles
-.
-.SH SYNTAX
-.ft R
-.
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_T_cvar_handle_alloc(int \fIcvar_index\fP, void *\fIobj_handle\fP,
-                            MPI_T_cvar_handle *\fIhandle\fP, int *\fIcount\fP)
-
-int MPI_T_cvar_handle_free(MPI_T_cvar_handle *\fIhandle\fP)
-
-.fi
-.SH DESCRIPTION
-.ft R
-MPI_T_cvar_handle_alloc binds the control variable specified in \fIcvar_index\fP to the MPI
-object specified in \fIobj_handle\fP. If MPI_T_cvar_get_info returns MPI_T_BIND_NO_OBJECT
-as the binding of the variable the \fIobj_handle\fP argument is ignored. The number of
-values represented by this control variable is returned in the \fIcount\fP parameter. If the
-control variable represents a string then \fIcount\fP will be the maximum length of the
-string.
-
-MPI_T_cvar_handle_free frees a handle allocated by MPI_T_cvar_handle_alloc and sets the
-\fIhandle\fP argument to MPI_T_CVAR_HANDLE_NULL.
-
-
-.SH NOTES
-.ft R
-Open MPI does not currently support binding MPI objects to control variables so the
-\fIobj_handle\fP argument is always ignored.
-
-
-.SH ERRORS
-.ft R
-MPI_T_cvar_handle_alloc() will fail if:
-.TP 1i
-[MPI_T_ERR_NOT_INITIALIZED]
-The MPI Tools interface not initialized
-.TP 1i
-[MPI_T_ERR_INVALID_INDEX]
-The control variable index is invalid
-.TP 1i
-[MPI_T_ERR_OUT_OF_HANDLES]
-No more handles available
-.TP 1i
-MPI_T_cvar_handle_free() will fail if:
-.TP 1i
-[MPI_T_ERR_NOT_INITIALIZED]
-The MPI Tools interface not initialized
-.TP 1i
-[MPI_T_ERR_INVALID_HANDLE]
-The handle is invalid
-
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_T_cvar_get_info
-
diff --git a/ompi/mpi/man/man3/MPI_T_cvar_handle_free.3in b/ompi/mpi/man/man3/MPI_T_cvar_handle_free.3in
deleted file mode 100644
index 3fa2264737d..00000000000
--- a/ompi/mpi/man/man3/MPI_T_cvar_handle_free.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_T_cvar_handle_alloc.3
diff --git a/ompi/mpi/man/man3/MPI_T_cvar_read.3in b/ompi/mpi/man/man3/MPI_T_cvar_read.3in
deleted file mode 100644
index 5e678d7e098..00000000000
--- a/ompi/mpi/man/man3/MPI_T_cvar_read.3in
+++ /dev/null
@@ -1,51 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2010 Cisco Systems, Inc.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_T_cvar_read 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.
-.SH NAME
-\fBMPI_T_cvar_read\fP \- Read the value of a control variable
-.
-.SH SYNTAX
-.ft R
-.
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_T_cvar_read(MPI_T_cvar_handle \fIhandle\fP, const void *\fIbuf\fP)
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-handle
-Handle of the control variable to be read.
-.TP 1i
-buf
-Initial address of storage location for variable value.
-
-.SH DESCRIPTION
-.ft R
-MPI_T_cvar_read reads the value of the control variable identified by the handle
-specified in \fIhandle\fP and stores the value in the buffer pointed to by \fIbuf\fP.
-The caller must ensure that the buffer pointed to by \fIbuf\fP is large enough to
-hold the entire value of the control variable.
-
-.SH ERRORS
-.ft R
-MPI_T_cvar_read() will fail if:
-.TP 1i
-[MPI_T_ERR_NOT_INITIALIZED]
-The MPI Tools interface not initialized
-.TP 1i
-[MPI_T_ERR_INVALID_HANDLE]
-The handle is invalid
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_T_cvar_handle_alloc
-MPI_T_cvar_get_info
diff --git a/ompi/mpi/man/man3/MPI_T_cvar_write.3in b/ompi/mpi/man/man3/MPI_T_cvar_write.3in
deleted file mode 100644
index 323cda935a7..00000000000
--- a/ompi/mpi/man/man3/MPI_T_cvar_write.3in
+++ /dev/null
@@ -1,59 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2010 Cisco Systems, Inc.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_T_cvar_write 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.
-.SH NAME
-\fBMPI_T_cvar_write\fP \- Write the value of a bound control variable
-.
-.SH SYNTAX
-.ft R
-.
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_T_cvar_write(MPI_T_cvar_handle \fIhandle\fP, const void *\fIbuf\fP)
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-handle
-Handle of the control variable to be written.
-.TP 1i
-buf
-Initial address of storage location for variable value.
-
-.SH DESCRIPTION
-.ft R
-MPI_T_cvar_write sets the value the control variable identified by the handle
-specified in \fIhandle\fP from the buffer provided in \fIbuf\fP. The caller must
-ensure that the buffer specified in \fIbuf\fP is large enough to hold the
-entire value of the control variable. If the variable has global scope, any
-write call must be issued on all connected MPI processes. For more
-information see MPI-3 \[char167] 14.3.6.
-
-.SH ERRORS
-.ft R
-MPI_T_cvar_write() will fail if:
-.TP 1i
-[MPI_T_ERR_NOT_INITIALIZED]
-The MPI Tools interface not initialized
-.TP 1i
-[MPI_T_ERR_INVALID_HANDLE]
-The handle is invalid
-.TP 1i
-[MPI_T_ERR_CVAR_SET_NOT_NOW]
-Variable cannot be set at this moment
-.TP 1i
-[MPI_T_ERR_CVAR_SET_NEVER]
-Variable cannot be set until end of execution
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_T_cvar_handle_alloc
-MPI_T_cvar_get_info
diff --git a/ompi/mpi/man/man3/MPI_T_enum_get_info.3in b/ompi/mpi/man/man3/MPI_T_enum_get_info.3in
deleted file mode 100644
index 73172561afd..00000000000
--- a/ompi/mpi/man/man3/MPI_T_enum_get_info.3in
+++ /dev/null
@@ -1,66 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2010 Cisco Systems, Inc.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_T_enum_get_info 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.
-.SH NAME
-\fBMPI_T_enum_get_info\fP \- Query information about an enumerator
-.
-.SH SYNTAX
-.ft R
-.
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_T_enum_get_info(MPI_T_enum \fIenumtype\fP, int *\fInum\fP, char *\fIname\fP, int *\fIname_len\fP)
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-enumtype
-Enumerator to be queried.
-
-.SH INPUT/OUTPUT PARAMETERS
-.ft R
-.TP 1i
-name_len
-Length of the string and/or buffer for name.
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP li
-num
-number of discrete values represented by this enumeration.
-.TP 1i
-name
-Buffer to return the string containing the name of the
-category.
-
-.SH DESCRIPTION
-.ft R
-MPI_T_enum_get_info can be used to query information about an enumerator. The function returns the
-number of discrete values represented by this enumerator in the \fInum\fP parameter.
-
-.SH NOTES
-.ft R
-This MPI tool interface function returns the name of the enumeration as a string. This function
-takes two argument for the string: \fIname\fP which specifies a buffer where the name of the
-should be stored, and \fIname_len\fP which must initially specify the size of the buffer pointed
-to by \fIname\fP. This function will copy at most \fIname_len\fP - 1 characters of the name
-and sets \fIname_len\fP to the number of characters returned + 1. If \fIname_len\fP is NULL
-or the value specified in \fIname_len\fP is 0 the \fIname\fP buffer is ignored and the name of
-the enumeration is not returned.
-
-.SH ERRORS
-.ft R
-MPI_T_enum_get_info() will fail if:
-.TP 1i
-[MPI_T_ERR_NOT_INITIALIZED]
-The MPI Tools interface not initialized
-.TP 1i
-[MPI_T_ERR_INVALID_INDEX]
-The enumeration is invalid or has been deleted
diff --git a/ompi/mpi/man/man3/MPI_T_enum_get_item.3in b/ompi/mpi/man/man3/MPI_T_enum_get_item.3in
deleted file mode 100644
index 86c22e912e1..00000000000
--- a/ompi/mpi/man/man3/MPI_T_enum_get_item.3in
+++ /dev/null
@@ -1,69 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2010 Cisco Systems, Inc.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_T_enum_get_item 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.
-.SH NAME
-\fBMPI_T_enum_get_item\fP \- Query information about an enumerator
-.
-.SH SYNTAX
-.ft R
-.
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_T_enum_get_item(MPI_T_enum \fIenumtype\fP, int \fIindex\fP, int *\fIvalue\fP, char *\fIname\fP,
-                        int *\fIname_len\fP)
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-enumtype
-Enumeration to be queried.
-.TP 1i
-index
-Number of the value to be queried in this enumeration.
-
-.SH INPUT/OUTPUT PARAMETERS
-.ft R
-.TP 1i
-name_len
-Length of the string and/or buffer for name.
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP li
-value
-Variable value.
-.TP 1i
-name
-Buffer to return the string containing the name of the
-category.
-
-.SH DESCRIPTION
-.ft R
-MPI_T_enum_get_item can be used to query information about an item in an enumerator. This function
-returns the enumeration value in the \fIvalue\fP parameter.
-
-.SH NOTES
-.ft R
-This MPI tool interface function returns the name of the item as a string. This function takes two
-arguments for the string: a buffer to store the string, and a length which must initially specify the
-size of the buffer. If the length passed is n then this function will copy at most n - 1 characters
-of the string into the buffer and sets the length to the number of characters copied - 1. If the length
-argument is NULL or the value specified in the length is 0 the string buffer is ignored and the
-string is not returned. For more information see MPI-3 \[char167] 14.3.3.
-
-.SH ERRORS
-.ft R
-MPI_T_enum_get_item() will fail if:
-.TP 1i
-[MPI_T_ERR_NOT_INITIALIZED]
-The MPI Tools interface not initialized
-.TP 1i
-[MPI_T_ERR_INVALID_INDEX]
-The enumeration is invalid or has been deleted
diff --git a/ompi/mpi/man/man3/MPI_T_finalize.3in b/ompi/mpi/man/man3/MPI_T_finalize.3in
deleted file mode 100644
index 7cb2b7dce2c..00000000000
--- a/ompi/mpi/man/man3/MPI_T_finalize.3in
+++ /dev/null
@@ -1,45 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" $COPYRIGHT$
-.TH MPI_T_finalize 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_T_finalize \fP \- Finalize the MPI tool information interface
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_T_finalize(void)
-
-.fi
-.SH DESCRIPTION
-.ft R
-MPI_T_finalize() finalizes the MPI tool information interface and must be called the same
-number of times as MPI_T_init_thread() by the end of execution. Calls to MPI tool functions
-are allowed at any point in execution as long as MPI_T_init_thread() has been called at least
-once and the number of calls to MPI_T_init_thread() is greater than the number of calls to
-MPI_T_finalize(). If at any point in execution the number of calls to MPI_T_finalize() equals
-the number of calls to MPI_T_init_thread() the MPI tool interface will no longer be available
-until another call to MPI_T_init_thread().
-
-.SH NOTES
-.ft R
-Before the end of execution the number of calls to MPI_T_init_thread() and MPI_T_finalize must
-be the same.
-
-.SH ERRORS
-.ft R
-MPI_T_finalize() will fail if:
-.TP 1i
-[MPI_T_ERR_NOT_INITIALIZED]
-The MPI Tools interface not initialized
-
-.SH SEE ALSO
-.ft T
-.nf
-MPI_T_init_thread
-
diff --git a/ompi/mpi/man/man3/MPI_T_init_thread.3.md b/ompi/mpi/man/man3/MPI_T_init_thread.3.md
deleted file mode 100644
index e0a6851f308..00000000000
--- a/ompi/mpi/man/man3/MPI_T_init_thread.3.md
+++ /dev/null
@@ -1,65 +0,0 @@
-# NAME
-
-MPI_T_init_thread - Initializes the MPI Tool information interface
-
-# SYNTAX
-
-## C Syntax
-
-```c
-#include <mpi.h>
-int MPI_T_init_thread(int required, int *provided)
-```
-
-# INPUT PARAMETERS
-
-* required: Desired level of thread support (integer).
-
-# OUTPUT PARAMETERS
-
-* provided: Available level of thread support (integer).
-
-# DESCRIPTION
-
-`MPI_T_init_thread()` initializes the MPI tool information interface.  Calls to
-MPI tool functions are allowed at any point in execution (including before
-`MPI_Init()` and after `MPI_Finalize()`) as long as `MPI_T_init_thread()` has
-been called at least once and the number of calls to `MPI_T_init_thread()` is
-greater than the number of calls to `MPI_T_finalize()`. If at any point in
-execution the number of calls to `MPI_T_finalize()` equals the number of calls
-to `MPI_T_init_thread()` the MPI tool interface will no longer be available
-until another call to `MPI_T_init_thread()`.
-
-`MPI_T_init_thread()`, like `MPI_Init_thread()`, has a provision to request a
-certain level of thread support in `required`:
-
-* `MPI_THREAD_SINGLE`: Only one thread will execute.
-* `MPI_THREAD_FUNNELED`: If the process is multithreaded, only the thread that
-  called `MPI_Init_thread()` will make MPI calls.
-* `MPI_THREAD_SERIALIZED`: If the process is multithreaded, only one thread will
-  make MPI library calls at one time.
-* `MPI_THREAD_MULTIPLE`: If the process is multithreaded, multiple threads may
-  call MPI at once with no restrictions.
-
-The level of thread support available to the program is set in
-`provided`. In Open MPI, the value is dependent on how the library was
-configured and built. Note that there is no guarantee that
-`provided` will be greater than or equal to `required`.
-
-# NOTES
-
-It is the caller's responsibility to check the value of `provided`,
-as it may be less than what was requested in `required`.
-
-# ERRORS
-
-`MPI_T_init_thread()` will fail if:
-
-* `MPI_T_ERR_MEMORY`: Out of memory
-* `MPI_T_ERR_CANNOT_INIT`: Interface not in the state to be initialized
-
-# SEE ALSO
-
-[`MPI_T`(5)](MPI_T.html),
-[`MPI_Init_thread`(3)](MPI_Init_thread.html),
-[`MPI_T_finalize`(3)](MPI_T_finalize.html)
diff --git a/ompi/mpi/man/man3/MPI_T_pvar_get_info.3in b/ompi/mpi/man/man3/MPI_T_pvar_get_info.3in
deleted file mode 100644
index 86e7dbaa7a1..00000000000
--- a/ompi/mpi/man/man3/MPI_T_pvar_get_info.3in
+++ /dev/null
@@ -1,204 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2010 Cisco Systems, Inc.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_T_pvar_get_info 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.
-.SH NAME
-\fBMPI_T_pvar_get_info\fP \- Query information from a performance variable
-.
-.SH SYNTAX
-.ft R
-.
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_T_pvar_get_info(int \fIpvar_index\fP, char *\fIname\fP, int *\fIname_len\fP,
-                        int *\fIverbosity\fP, int *\fIvar_class\fP, MPI_Datatype *\fIdatatype\fP, MPI_T_enum *\fIenumtype\fP,
-                        char *\fIdesc\fP, int *\fIdesc_len\fP, int *\fIbind\fP, int *\fIreadonly\fP, int *\fIcontinuous\fP,
-                        int *\fIatomic\fP)
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-pvar_index
-Index of the performance variable to be queried.
-
-.SH INPUT/OUTPUT PARAMETERS
-.ft R
-.TP 1i
-name_len
-Length of the string and/or buffer for name.
-.TP 1i
-desc_len
-Length of the string and/or buffer for desc.
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-name
-Buffer to return the string containing the name of the
-performance variable.
-.TP 1i
-verbosity
-Verbosity level of this variable.
-.TP 1i
-var_class
-Class of performance variable.
-.TP 1i
-datatype
-MPI datatype of the information stored in the performance variable.
-.TP 1i
-enumtype
-Optional descriptor for enumeration information.
-.TP 1i
-desc
-Buffer to return the string containing the description of the performance variable.
-.TP 1i
-bind
-Type of MPI object to which this variable must be bound.
-.TP 1i
-readonly
-Flag indicating whether the variable can be written/reset.
-.TP 1i
-continuous
-Flag indicating whether the variable can be started and stopped or is continuously active.
-.TP 1i
-atomic
-Flag indicating whether the variable can be atomically read and reset.
-.
-.
-.SH DESCRIPTION
-.ft R
-MPI_T_pvar_get_info can be used to query information from a performance variable. The function returns
-the verbosity, class, datatype, enumeration type, and binding of the queried control variable in the arguments
-\fIverbosity\fP, \fIvar_class\fP, \fIdatatype\fP, \fIenumtype\fP, and \fIbind\fP respectively. Flags indicating
-whether the variable is read-only, continuous, or atomic are returns in \fIreadonly\fP, \fIcontinuous\fP, and
-\fIatomic\fP accordingly. See MPI-3 § 14.3.7 for more information. See the man page for MPI_T_cvar_get_info
-for information on variable verbosity.
-
-.SH VARIABLE CLASS
-.ft R
-Performance variables are categorized into classes which describe their initial value, valid types, and
-behavior. The class returned in the \fIvar_class\fP parameter may be one of the following:
-.TP 2
-MPI_T_PVAR_CLASS_STATE
-Variable represents a set of discrete states that may be described by an enumerator. Variables of this class
-must be represented by an MPI_INT. The starting value is the current state of the variable.
-.TP 2
-MPI_T_PVAR_CLASS_LEVEL
-Variable represents the current utilization level of a resource. Variables of this class must be represented
-by an MPI_UNSIGNED, MPI_UNSIGNED_LONG, MPI_UNSIGNED_LONG_LONG, or MPI_DOUBLE. The starting value is the
-current utilization level of the resource.
-.TP 2
-MPI_T_PVAR_CLASS_SIZE
-Variable represents the fixed size of a resource. Variables of this class are represented by an MPI_UNSIGNED,
-MPI_UNSIGNED_LONG, MPI_UNSIGNED_LONG_LONG, or MPI_DOUBLE. The starting value is the current size of the resource.
-.TP 2
-MPI_T_PVAR_CLASS_PERCENTAGE
-Variable represents the current precentage utilization level of a resource. Variables of this class are
-represented by an MPI_DOUBLE. The starting value is the current percentage utilization of the resource.
-.TP 2
-MPI_T_PVAR_CLASS_HIGHWATERMARK
-Variable represents the high watermark of the utilization of a resource. Variables of this class are
-represented by an MPI_UNSIGNED, MPI_UNSIGNED_LONG, MPI_UNSIGNED_LONG_LONG, or MPI_DOUBLE. The starting value
-is the current utilization of the resource.
-.TP 2
-MPI_T_PVAR_CLASS_HIGHWATERMARK
-Variable represents the low watermark of the utilization of a resource. Variables of this class are
-represented by an MPI_UNSIGNED, MPI_UNSIGNED_LONG, MPI_UNSIGNED_LONG_LONG, or MPI_DOUBLE. The starting value
-is the current utilization of the resource.
-.TP 2
-MPI_T_PVAR_CLASS_COUNTER
-Variable represents a count of the number of occurrences of a specific event. Variables of this class are
-represented by an MPI_UNSIGNED, MPI_UNSIGNED_LONG, or MPI_UNSIGNED_LONG_LONG. The starting value is 0.
-.TP 2
-MPI_T_PVAR_CLASS_COUNTER
-Variable represents an aggregated value that represents a sum of arguments processed during a specific event.
-Variables of this class are represented by an MPI_UNSIGNED, MPI_UNSIGNED_LONG, MPI_UNSIGNED_LONG_LONG,
-or MPI_DOUBLE. The starting value is 0.
-.TP 2
-MPI_T_PVAR_CLASS_TIMER
-Variable represents the aggregated time spent by the MPI implementation while processing an event, type of
-event, or section of code. Variables of this class are represented by an MPI_UNSIGNED, MPI_UNSIGNED_LONG,
-MPI_UNSIGNED_LONG_LONG, or MPI_DOUBLE. If the variable is represented by an MPI_DOUBLE the units will be
-the same as those used by MPI_Wtime(). The starting value is 0.
-.TP 2
-MPI_T_PVAR_CLASS_GENERIC
-Variable does not fit into any other class. Can by represented by an type supported by the MPI tool
-information interface (see DATATYPE). Starting value is variable specific.
-
-For more information see MPI-3 \[char 167] 14.3.7.
-
-.SH DATATYPE
-.ft R
-The datatype returned by MPI_T_pvar_get_info is restricted to one of the following datatypes: MPI_INT,
-MPI_UNSIGNED, MPI_UNSIGNED_LONG, MPI_UNSIGNED_LONG_LONG, MPI_COUNT, MPI_CHAR, and MPI_DOUBLE. For more
-information on datatypes in the MPI Tool information interface see MPI-3 \[char167] 14.3.5.
-
-.SH BINDING
-.ft R
-Performance variables may be bound to an MPI object. The binding returned in the \fIbind\fP parameter may be
-one of the following:
-.TP 1i
-MPI_T_BIND_NO_OBJECT
-No object
-.TP 1i
-MPI_T_BIND_MPI_COMM
-MPI communicator
-.TP 1i
-MPI_T_BIND_MPI_DATATYPE
-MPI datatype
-.TP 1i
-MPI_T_BIND_MPI_ERRHANDLER
-MPI error handler
-.TP 1i
-MPI_T_BIND_MPI_FILE
-MPI file handle
-.TP 1i
-MPI_T_BIND_MPI_GROUP
-MPI group
-.TP 1i
-MPI_T_BIND_MPI_OP
-MPI reduction operator
-.TP 1i
-MPI_T_BIND_MPI_REQUEST
-MPI request
-.TP 1i
-MPI_T_BIND_MPI_WIN
-MPI window for one-sided communication
-.TP 1i
-MPI_T_BIND_MPI_MESSAGE
-MPI message object
-.TP 1i
-MPI_T_BIND_MPI_INFO
-MPI info object
-
-For more information see MPI-3 \[char167] 14.3.2.
-
-.SH NOTES
-.ft R
-This MPI tool interface function returns two strings. This function takes two argument for each string:
-a buffer to store the string, and a length which must initially specify the size of the buffer. If the
-length passed is n then this function will copy at most n - 1 characters of the string into the
-corresponding buffer and set the length to the number of characters copied - 1. If the length argument
-is NULL or the value specified in the length is 0 the corresponding string buffer is ignored and the
-string is not returned. For more information see MPI-3 \[char167] 14.3.3.
-
-.SH ERRORS
-.ft R
-MPI_T_pvar_get_info() will fail if:
-.TP 1i
-[MPI_T_ERR_NOT_INITIALIZED]
-The MPI Tools interface not initialized
-.TP 1i
-[MPI_T_ERR_INVALID_INDEX]
-The performance variable index is invalid
-.
-.SH SEE ALSO
-.ft R
-.nf
-MPI_T_cvar_get_info
diff --git a/ompi/mpi/man/man3/MPI_T_pvar_get_num.3in b/ompi/mpi/man/man3/MPI_T_pvar_get_num.3in
deleted file mode 100644
index 1f6fc63ac2d..00000000000
--- a/ompi/mpi/man/man3/MPI_T_pvar_get_num.3in
+++ /dev/null
@@ -1,38 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2010 Cisco Systems, Inc.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_T_pvar_get_num 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.
-.SH NAME
-\fBMPI_T_pvar_get_num\fP \- Query the number of performance variables
-.
-.SH SYNTAX
-.ft R
-.
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_T_pvar_get_num(int *\fInum_pvar\fP)
-
-.fi
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-num_pvar
-Current number of performance variables.
-
-.SH DESCRIPTION
-.ft R
-MPI_T_pvar_get_num can be used to query the current number of performance variables. The number
-of performance variables may increase throughout the exection of the process but will never
-decrease.
-
-.SH ERRORS
-.ft R
-MPI_T_pvar_get_num() will fail if:
-.TP 1i
-[MPI_T_ERR_NOT_INITIALIZED]
-The MPI Tools interface not initialized
diff --git a/ompi/mpi/man/man3/MPI_T_pvar_handle_alloc.3in b/ompi/mpi/man/man3/MPI_T_pvar_handle_alloc.3in
deleted file mode 100644
index dfcd4d19f55..00000000000
--- a/ompi/mpi/man/man3/MPI_T_pvar_handle_alloc.3in
+++ /dev/null
@@ -1,72 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2010 Cisco Systems, Inc.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_T_pvar_handle_alloc 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.
-.SH NAME
-\fBMPI_T_pvar_handle_alloc\fP, \fBMPI_T_pvar_handle_free\fP \- Allocate/free MPI performance variable handles
-.
-.SH SYNTAX
-.ft R
-.
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_T_pvar_handle_alloc(int \fIsession\fP, int \fIpvar_index\fP, void *\fIobj_handle\fP,
-                            MPI_T_pvar_handle *\fIhandle\fP, int *\fIcount\fP)
-
-int MPI_T_pvar_handle_free(int \fIsession\fP, MPI_T_pvar_handle *\fIhandle\fP)
-
-.SH DESCRIPTION
-.ft R
-MPI_T_pvar_handle_alloc binds the performance variable specified in \fIpvar_index\fP to the MPI
-object specified in \fIobj_handle\fP in the session identified by the parameter
-\fIsession\fP. The object is passed in the argument \fIobj_handle\fP as an
-address to a local variable that stores the object’s handle. If
-MPI_T_pvar_get_info returns MPI_T_BIND_NO_OBJECT as the binding
-for the variable the \fIobj_handle\fP argument is ignored. The handle
-allocated to reference the variable is returned in the argument \fIhandle\fP. Upon successful
-return, \fIcount\fP contains the number of elements (of the datatype returned by a previous
-MPI_T_PVAR_GET_INFO call) used to represent this variable.
-
-The value of \fIpvar_index\fP should be in the range 0 to \fInum_pvar - 1\fP,
-where \fInum_pvar\fP is the number of available performance variables as
-determined from a prior call to \fIMPI_T_PVAR_GET_NUM\fP. The type of the
-MPI object it references must be consistent with the type returned in the
-bind argument in a prior call to \fIMPI_T_PVAR_GET_INFO\fP.
-
-MPI_T_pvar_handle_free frees a handle allocated by MPI_T_pvar_handle_alloc and sets the
-\fIhandle\fP argument to MPI_T_PVAR_HANDLE_NULL.
-
-
-.SH ERRORS
-.ft R
-MPI_T_pvar_handle_alloc() will fail if:
-.TP 1i
-[MPI_T_ERR_NOT_INITIALIZED]
-The MPI Tools interface not initialized
-.TP 1i
-[MPI_T_ERR_INVALID_INDEX]
-The performance variable index is invalid
-.TP 1i
-[MPI_T_ERR_OUT_OF_HANDLES]
-No more handles available
-.TP 1i
-MPI_T_pvar_handle_free() will fail if:
-.TP 1i
-[MPI_T_ERR_NOT_INITIALIZED]
-The MPI Tools interface not initialized
-.TP 1i
-[MPI_T_ERR_INVALID_HANDLE]
-The handle is invalid or the handle argument passed in is not associated with the session argument
-
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_T_pvar_get_info
-MPI_T_pvar_get_num
-
diff --git a/ompi/mpi/man/man3/MPI_T_pvar_handle_free.3in b/ompi/mpi/man/man3/MPI_T_pvar_handle_free.3in
deleted file mode 100644
index 378fd2d10c8..00000000000
--- a/ompi/mpi/man/man3/MPI_T_pvar_handle_free.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_T_pvar_handle_alloc.3
diff --git a/ompi/mpi/man/man3/MPI_T_pvar_read.3in b/ompi/mpi/man/man3/MPI_T_pvar_read.3in
deleted file mode 100644
index f3d346ff2f0..00000000000
--- a/ompi/mpi/man/man3/MPI_T_pvar_read.3in
+++ /dev/null
@@ -1,59 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2010 Cisco Systems, Inc.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_T_pvar_read 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.
-.SH NAME
-\fBMPI_T_pvar_read\fP \- Read the value of a performance variable
-.
-.SH SYNTAX
-.ft R
-.
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_T_pvar_read(MPI_T_pvar_session \fIsession\fP, MPI_T_pvar_handle \fIhandle\fP, const void *\fIbuf\fP)
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-session
-Performance experiment session.
-.TP 1i
-handle
-Performance variable handle.
-.TP 1i
-buf
-Initial address of storage location for variable value.
-
-.SH DESCRIPTION
-.ft R
-MPI_T_pvar_read queries the value of a performance variable identified by the handle
-specified in \fIhandle\fP in the session specified in \fIsession\fP. The result is
-stored in the buffer pointed to by \fIbuf\fP. The caller must ensure that the
-buffer pointed to by \fIbuf\fP is large enough to hold the entire value of the
-performance variable.
-
-.SH ERRORS
-.ft R
-MPI_T_pvar_read() will fail if:
-.TP 1i
-[MPI_T_ERR_NOT_INITIALIZED]
-The MPI Tools interface not initialized
-.TP 1i
-[MPI_T_ERR_INVALID_HANDLE]
-The handle is invalid or not associated with the session
-.TP 1i
-[MPI_T_ERR_INVALID_SESSION]
-Session argument is not a valid session
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_T_pvar_handle_alloc
-MPI_T_pvar_get_info
-MPI_T_pvar_session_create
diff --git a/ompi/mpi/man/man3/MPI_T_pvar_readreset.3in b/ompi/mpi/man/man3/MPI_T_pvar_readreset.3in
deleted file mode 100644
index 007b7534830..00000000000
--- a/ompi/mpi/man/man3/MPI_T_pvar_readreset.3in
+++ /dev/null
@@ -1,69 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2010 Cisco Systems, Inc.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_T_pvar_readreset 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.
-.SH NAME
-\fBMPI_T_pvar_readreset\fP \- Atomically read and reset the value of a performance variable
-.
-.SH SYNTAX
-.ft R
-.
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_T_pvar_readreset(MPI_T_pvar_session \fIsession\fP, MPI_T_pvar_handle \fIhandle\fP, const void *\fIbuf\fP)
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-session
-Performance experiment session.
-.TP 1i
-handle
-Performance variable handle.
-.TP 1i
-buf
-Initial address of storage location for variable value.
-
-.SH DESCRIPTION
-.ft R
-MPI_T_pvar_readreset atomically queries and resets the value of a performance variable
-bound to the handle specified by \fIhandle\fP in the session specified by \fIsession\fP.
-The result is stored in the buffer pointed to by \fIbuf\fP. This function can only be
-used with performance variables that are atomic and not readonly. The caller must ensure
-that the buffer pointed to by \fIbuf\fP is large enough to hold the entire value of the
-performance variable.
-
-.SH ERRORS
-.ft R
-MPI_T_pvar_readreset() will fail if:
-.TP 1i
-[MPI_T_ERR_NOT_INITIALIZED]
-The MPI Tools interface not initialized
-.TP 1i
-[MPI_T_ERR_INVALID_HANDLE]
-The handle is invalid or not associated with the session
-.TP 1i
-[MPI_T_ERR_INVALID_SESSION]
-Session argument is not a valid session
-.TP 1i
-[MPI_T_ERR_PVAR_NO_ATOMIC]
-Variable cannot be read and written atomically
-.TP 1i
-[MPI_T_ERR_PVAR_NO_WRITE]
-Variable cannot be reset
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_T_pvar_handle_alloc
-MPI_T_pvar_get_info
-MPI_T_pvar_session_create
-MPI_T_pvar_read
-MPI_T_pvar_reset
-
diff --git a/ompi/mpi/man/man3/MPI_T_pvar_reset.3in b/ompi/mpi/man/man3/MPI_T_pvar_reset.3in
deleted file mode 100644
index df145f6a3f3..00000000000
--- a/ompi/mpi/man/man3/MPI_T_pvar_reset.3in
+++ /dev/null
@@ -1,60 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2010 Cisco Systems, Inc.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_T_pvar_reset 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.
-.SH NAME
-\fBMPI_T_pvar_reset\fP \- Reset the value of a performance variable
-.
-.SH SYNTAX
-.ft R
-.
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_T_pvar_reset(MPI_T_pvar_session \fIsession\fP, MPI_T_pvar_handle \fIhandle\fP)
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-session
-Performance experiment session.
-.TP 1i
-handle
-Performance variable handle or MPI_T_PVAR_ALL_HANDLES.
-.
-.
-.SH DESCRIPTION
-.ft R
-MPI_T_pvar_reset sets the performance variable specified by the handle in \fIhandle\fP
-to its initial value. The special value MPI_T_PVAR_ALL_HANDLES can be passed in \fIhandle\fP
-to reset all read-write handles in the session specified in \fIsession\fP.
-.
-.
-.SH ERRORS
-.ft R
-MPI_T_pvar_reset() will fail if:
-.TP 1i
-[MPI_T_ERR_NOT_INITIALIZED]
-The MPI Tools interface not initialized
-.TP 1i
-[MPI_T_ERR_INVALID_HANDLE]
-The handle is invalid
-.TP 1i
-[MPI_T_ERR_INVALID_SESSION]
-Session argument is not a valid session
-.TP 1i
-[MPI_T_ERR_PVAR_NO_WRITE]
-Variable cannot be reset
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_T_pvar_handle_alloc
-MPI_T_pvar_get_info
-MPI_T_pvar_session_create
-MPI_T_pvar_write
diff --git a/ompi/mpi/man/man3/MPI_T_pvar_session_create.3in b/ompi/mpi/man/man3/MPI_T_pvar_session_create.3in
deleted file mode 100644
index dd46817f314..00000000000
--- a/ompi/mpi/man/man3/MPI_T_pvar_session_create.3in
+++ /dev/null
@@ -1,51 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2010 Cisco Systems, Inc.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_T_pvar_session_create 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.
-.SH NAME
-\fBMPI_T_pvar_session_create\fP, \fBMPI_T_pvar_session_free\fP \- Create/free performance variable session
-.
-.SH SYNTAX
-.ft R
-.
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_T_pvar_session_create(MPI_T_pvar_session *\fIsession\fP)
-
-int MPI_T_pvar_session_free(MPI_T_pvar_session *\fIsession\fP)
-
-.fi
-.SH DESCRIPTION
-.ft R
-MPI_T_pvar_session_create creates a session for accessing performance variables. The
-new session is returned in the \fIsession\fP parameter.
-
-MPI_T_pvar_session_free releases a session allocated by MPI_T_pvar_session_create and sets
-the \fIsession\fP parameter to MPI_T_PVAR_SESSION_NULL.
-
-.SH ERRORS
-.ft R
-MPI_T_pvar_session_create() will fail if:
-.TP 1i
-[MPI_T_ERR_NOT_INITIALIZED]
-The MPI Tools interface not initialized
-.TP 1i
-[MPI_T_ERR_MEMORY]
-Out of memory
-.TP 1i
-[MPI_T_ERR_OUT_OF_SESSIONS]
-No more sessions available
-.TP 1i
-MPI_T_pvar_session_free() will fail if:
-.TP 1i
-[MPI_T_ERR_NOT_INITIALIZED]
-The MPI Tools interface not initialized
-.TP 1i
-[MPI_T_ERR_INVALID_SESSION]
-The session parameter is not a valid session
-
diff --git a/ompi/mpi/man/man3/MPI_T_pvar_session_free.3in b/ompi/mpi/man/man3/MPI_T_pvar_session_free.3in
deleted file mode 100644
index 4a42a63f6d8..00000000000
--- a/ompi/mpi/man/man3/MPI_T_pvar_session_free.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_T_pvar_session_create.3
diff --git a/ompi/mpi/man/man3/MPI_T_pvar_start.3in b/ompi/mpi/man/man3/MPI_T_pvar_start.3in
deleted file mode 100644
index 2b1c9830d9b..00000000000
--- a/ompi/mpi/man/man3/MPI_T_pvar_start.3in
+++ /dev/null
@@ -1,63 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2010 Cisco Systems, Inc.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_T_pvar_start 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.
-.SH NAME
-\fBMPI_T_pvar_start\fP, \fBMPI_T_pvar_stop\fP \- Start/stop a performance variable
-.
-.SH SYNTAX
-.ft R
-.
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_T_pvar_start(MPI_T_pvar_session \fIsession\fP, MPI_T_pvar_handle \fIhandle\fP)
-
-int MPI_T_pvar_stop(MPI_T_pvar_session \fIsession\fP, MPI_T_pvar_handle \fIhandle\fP)
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-session
-Performance experiment session.
-.TP 1i
-handle
-Performance variable handle.
-
-.SH DESCRIPTION
-.ft R
-MPI_T_pvar_start starts the performance variable with the handle specified in \fIhandle\fP.
-The special value MPI_T_PVAR_ALL_HANDLES can be passed in \fIhandle\fP to start all
-non-continuous handles in the session specified in \fIsession\fP.
-
-MPI_T_pvar_stop stops the performance variable with the handle specified in \fIhandle\fP.
-The special value MPI_T_PVAR_ALL_HANDLES can be passed in \fIhandle\fP to stop all
-non-continuous handles in the session specified in \fIsession\fP.
-
-Continuous performance variables can neither be started nor stopped.
-
-.SH ERRORS
-.ft R
-MPI_T_pvar_start() and MPI_T_pvar_stop() will fail if:
-.TP 1i
-[MPI_T_ERR_NOT_INITIALIZED]
-The MPI Tools interface not initialized
-.TP 1i
-[MPI_T_ERR_INVALID_SESSION]
-Session parameter is not a valid session
-.TP 1i
-[MPI_T_ERR_INVALID_HANDLE]
-Invalid handle or handle not associated with the session
-.TP 1i
-[MPI_T_ERR_PVAR_NO_STARTSTOP]
-The variable cannot be started or stopped
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_T_pvar_get_info
diff --git a/ompi/mpi/man/man3/MPI_T_pvar_stop.3in b/ompi/mpi/man/man3/MPI_T_pvar_stop.3in
deleted file mode 100644
index 3907001bdae..00000000000
--- a/ompi/mpi/man/man3/MPI_T_pvar_stop.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_T_pvar_start.3
diff --git a/ompi/mpi/man/man3/MPI_T_pvar_write.3in b/ompi/mpi/man/man3/MPI_T_pvar_write.3in
deleted file mode 100644
index 944a93c8e2f..00000000000
--- a/ompi/mpi/man/man3/MPI_T_pvar_write.3in
+++ /dev/null
@@ -1,62 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2010 Cisco Systems, Inc.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_T_pvar_write 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.
-.SH NAME
-\fBMPI_T_pvar_write\fP \- Write the value of a control variable
-.
-.SH SYNTAX
-.ft R
-.
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_T_pvar_write(MPI_T_pvar_session \fIsession\fP, MPI_T_pvar_handle \fIhandle\fP, const void *\fIbuf\fP)
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-session
-Performance experiment session.
-.TP 1i
-handle
-Performance variable handle.
-.TP 1i
-buf
-Initial address of storage location for variable value.
-
-.SH DESCRIPTION
-.ft R
-MPI_T_pvar_write attempts to set the value of the performance variable identified by
-the handle specified in \fIhandle\fP in the session specified in \fIsession\fP. The
-value to be written is specified in \fIbuf\fP. The caller must ensure that the buffer
-specified in \fIbuf\fP is large enough to hold the entire value of the performance
-variable.
-
-.SH ERRORS
-.ft R
-MPI_T_pvar_write() will fail if:
-.TP 1i
-[MPI_T_ERR_NOT_INITIALIZED]
-The MPI Tools interface not initialized
-.TP 1i
-[MPI_T_ERR_INVALID_HANDLE]
-The handle is invalid or not associated with the session
-.TP 1i
-[MPI_T_ERR_INVALID_SESSION]
-Session argument is not a valid session
-.TP 1i
-[MPI_T_ERR_PVAR_NO_WRITE]
-Variable cannot be written
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_T_pvar_handle_alloc
-MPI_T_pvar_get_info
-MPI_T_pvar_session_create
diff --git a/ompi/mpi/man/man3/MPI_Test.3in b/ompi/mpi/man/man3/MPI_Test.3in
deleted file mode 100644
index 36026cb9503..00000000000
--- a/ompi/mpi/man/man3/MPI_Test.3in
+++ /dev/null
@@ -1,116 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright 2007-2008 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Test 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Test\fP \- Tests for the completion of a specific send or receive.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Test(MPI_Request *\fIrequest\fP, int\fI *flag\fP, MPI_Status\fI *status\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TEST(\fIREQUEST, FLAG, STATUS, IERROR\fP)
-	LOGICAL	\fIFLAG\fP
-	INTEGER	\fIREQUEST, STATUS(MPI_STATUS_SIZE), IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Test(\fIrequest\fP, \fIflag\fP, \fIstatus\fP, \fIierror\fP)
-	TYPE(MPI_Request), INTENT(INOUT) :: \fIrequest\fP
-	LOGICAL, INTENT(OUT) :: \fIflag\fP
-	TYPE(MPI_Status) :: \fIstatus\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-request
-Communication request (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-flag
-True if operation completed (logical).
-.TP 1i
-status
-Status object (status).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-A call to MPI_Test returns flag = true if the operation identified by request is complete. In such a case, the status object is set to contain information on the completed operation; if the communication object was created by a nonblocking send or receive, then it is deallocated and the request handle is set to MPI_REQUEST_NULL. The call returns flag = false, otherwise. In this case, the value of the status object is undefined. MPI_Test is a local operation.
-.sp
-The return status object for a receive operation carries information that can be accessed as described in Section 3.2.5 of the MPI-1 Standard, "Return Status." The status object for a send operation carries information that can be accessed by a call to MPI_Test_cancelled (see Section 3.8 of the MPI-1 Standard, "Probe and Cancel").
-.sp
-If your application does not need to examine the \fIstatus\fP field, you can save resources by using the predefined constant MPI_STATUS_IGNORE as a special value for the \fIstatus\fP argument.
-.sp
-One is allowed to call MPI_Test with a null or inactive \fIrequest\fP argument. In such a case the operation returns with \fIflag\fP = true and empty \fIstatus\fP.
-.sp
-The functions MPI_Wait and MPI_Test can be used to complete both sends and
-receives.
-
-.SH NOTES
-The use of the nonblocking MPI_Test call allows the user to schedule alternative activities within a single thread of execution. An event-driven thread scheduler can be emulated with periodic calls to MPI_Test.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler, MPI_File_set_errhandler, or
-MPI_Win_set_errhandler (depending on the type of MPI handle that
-generated the request); the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does
-not guarantee that an MPI program can continue past an error.
-.sp
-Note that per MPI-1 section 3.2.5, MPI errors on requests passed
-to MPI_TEST do not set the status.MPI_ERROR field in the returned
-status.  The error code is passed to the back-end error handler
-and may be passed back to the caller through the return value of
-MPI_TEST if the back-end error handler returns it.  The
-pre-defined MPI error handler MPI_ERRORS_RETURN exhibits this
-behavior, for example.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Comm_set_errhandler
-.br
-MPI_File_set_errhandler
-.br
-MPI_Testall
-.br
-MPI_Testany
-.br
-MPI_Testsome
-.br
-MPI_Wait
-.br
-MPI_Waitall
-.br
-MPI_Waitany
-.br
-MPI_Waitsome
-.br
-MPI_Win_set_errhandler
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Test_cancelled.3in b/ompi/mpi/man/man3/MPI_Test_cancelled.3in
deleted file mode 100644
index 59db6376395..00000000000
--- a/ompi/mpi/man/man3/MPI_Test_cancelled.3in
+++ /dev/null
@@ -1,69 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Test_cancelled 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Test_cancelled\fP \- Tests whether a request was canceled.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Test_cancelled(const MPI_Status *\fIstatus\fP, int \fI*flag\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TEST_CANCELLED(\fISTATUS, FLAG, IERROR\fP)
-	LOGICAL	\fIFLAG\fP
-	INTEGER	\fISTATUS(MPI_STATUS_SIZE), IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Test_cancelled(\fIstatus\fP, \fIflag\fP, \fIierror\fP)
-	TYPE(MPI_Status), INTENT(IN) :: \fIstatus\fP
-	LOGICAL, INTENT(OUT) :: \fIflag\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-status
-Status object (status).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-flag
-True if operation was cancelled (logical).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Returns \fIflag\fP = true if the communication associated with the status object
-was canceled successfully. In such a case, all other fields of status (such as \fIcount\fP or \fItag\fP) are undefined. Otherwise, returns \fIflag\fP = false. If a receive operation might be canceled, one should call MPI_Test_cancelled first, to check whether the operation was canceled, before checking on the other fields of the return status.
-
-.SH NOTES
-Cancel can be an expensive operation that should be used only exceptionally.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-
-
diff --git a/ompi/mpi/man/man3/MPI_Testall.3in b/ompi/mpi/man/man3/MPI_Testall.3in
deleted file mode 100644
index 04b75c4b2f2..00000000000
--- a/ompi/mpi/man/man3/MPI_Testall.3in
+++ /dev/null
@@ -1,123 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2011 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Testall 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Testall\fP \- Tests for the completion of all previously initiated communications in a list.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Testall(int \fIcount\fP, MPI_Request\fI array_of_requests[]\fP,
-	int\fI *flag\fP, MPI_Status\fI array_of_statuses[]\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TESTALL(\fICOUNT, ARRAY_OF_REQUESTS, FLAG, ARRAY_OF_STATUSES,
-		IERROR\fP)
-	LOGICAL	\fIFLAG\fP
-	INTEGER	\fICOUNT, ARRAY_OF_REQUESTS(*)\fP
-	INTEGER	\fIARRAY_OF_STATUSES(MPI_STATUS_SIZE,*), IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Testall(\fIcount\fP, \fIarray_of_requests\fP, \fIflag\fP, \fIarray_of_statuses\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Request), INTENT(INOUT) :: \fIarray_of_requests(count)\fP
-	LOGICAL, INTENT(OUT) :: \fIflag\fP
-	TYPE(MPI_Status) :: \fIarray_of_statuses(*)\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-count
-Lists length (integer).
-.TP 1i
-array_of_requests
-Array of requests (array of handles).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-flag
-True if previously initiated communications are complete (logical.)
-.TP 1i
-array_of_statuses
-Array of status objects (array of status).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Returns \fIflag\fP = true if all communications associated with active handles in the array have completed (this includes the case where no handle in the list is active). In this case, each status entry that corresponds to an active handle request is set to the status of the corresponding communication; if the request was allocated by a nonblocking communication call then it is deallocated, and the handle is set to MPI_REQUEST_NULL. Each status entry that corresponds to a null or inactive handle is set to empty.
-.sp
-Otherwise, \fIflag\fP = false is returned, no request is modified and the values of the status entries are undefined. This is a local operation.
-.sp
-If your application does not need to examine the \fIarray_of_statuses\fP field, you can save resources by using the predefined constant MPI_STATUSES_IGNORE can be used as a special value for the \fIarray_of_statuses\fP argument.
-.sp
-Errors that occurred during the execution of MPI_Testall are handled in the same manner as errors in MPI_Waitall.
-
-.SH NOTE
-.ft R
-\fIflag\fP is true only if all requests have completed. Otherwise, \fIflag\fP is false,
-and neither \fIarray_of_requests\fP nor \fIarray_of_statuses\fP is modified.
-
-.SH ERRORS
-For each invocation of MPI_Testall, if one or more requests generate
-an MPI error, only the \fIfirst\fP MPI request that caused an
-error will be passed to its corresponding error handler.  No other
-error handlers will be invoked (even if multiple requests generated
-errors).  However, \fIall\fP requests that generate an error
-will have a relevant error code set in the corresponding
-status.MPI_ERROR field (unless MPI_STATUSES_IGNORE was used).
-.sp
-The default error handler aborts the MPI job, except for I/O function
-errors. The error handler may be changed with MPI_Comm_set_errhandler,
-MPI_File_set_errhandler, or MPI_Win_set_errhandler (depending on the
-type of MPI handle that generated the MPI request); the predefined
-error handler MPI_ERRORS_RETURN may be used to cause error values to
-be returned. Note that MPI does not guarantee that an MPI program can
-continue past an error.
-.sp
-If the invoked error handler allows MPI_Testall to return to the
-caller, the value MPI_ERR_IN_STATUS will be returned in the C and
-Fortran bindings.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Comm_set_errhandler
-.br
-MPI_File_set_errhandler
-.br
-MPI_Test
-.br
-MPI_Testany
-.br
-MPI_Testsome
-.br
-MPI_Wait
-.br
-MPI_Waitall
-.br
-MPI_Waitany
-.br
-MPI_Waitsome
-.br
-MPI_Win_set_errhandler
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Testany.3in b/ompi/mpi/man/man3/MPI_Testany.3in
deleted file mode 100644
index 169fd546a0d..00000000000
--- a/ompi/mpi/man/man3/MPI_Testany.3in
+++ /dev/null
@@ -1,121 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Testany 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Testany\fP \- Tests for completion of any one previously initiated communication in a list.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Testany(int \fIcount\fP, MPI_Request\fI array_of_requests[]\fP,
-	int\fI *index\fP, int\fI *flag\fP, MPI_Status\fI *status\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TESTANY(\fICOUNT, ARRAY_OF_REQUESTS, INDEX, FLAG, STATUS, IERROR\fP)
-	LOGICAL	\fIFLAG\fP
-	INTEGER	\fICOUNT, ARRAY_OF_REQUESTS(*), INDEX\fP
-	INTEGER	\fISTATUS(MPI_STATUS_SIZE), IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Testany(\fIcount\fP, \fIarray_of_requests\fP, \fIindex\fP, \fIflag\fP, \fIstatus\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Request), INTENT(INOUT) :: \fIarray_of_requests(count)\fP
-	INTEGER, INTENT(OUT) :: \fIindex\fP
-	LOGICAL, INTENT(OUT) :: \fIflag\fP
-	TYPE(MPI_Status) :: \fIstatus\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-count
-List length (integer).
-.TP 1i
-array_of_requests
-Array of requests (array of handles).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-index
-Index of operation that completed, or MPI_UNDEFINED if none completed
-(integer).
-.TP 1i
-flag
-True if one of the operations is complete (logical).
-.TP 1i
-status
-Status object (status).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Testany tests for completion of either one or none of the operations associated with active handles. In the former case, it returns \fIflag\fP = true, returns in \fIindex\fP the index of this request in the array, and returns in \fIstatus\fP the status of that operation; if the request was allocated by a nonblocking communication call then the request is deallocated and the handle is set to MPI_REQUEST_NULL. (The array is indexed from 0 in C, and from 1 in Fortran.) In the latter case (no operation completed), it returns \fIflag\fP = false, returns a value of MPI_UNDEFINED in \fIindex\fP, and \fIstatus\fP is undefined.
-.sp
-The array may contain null or inactive handles. If the array contains no active handles then the call returns immediately with \fIflag\fP = true, \fIindex\fP = MPI_UNDEFINED, and an empty \fIstatus\fP.
-.sp
-If the array of requests contains active handles then the execution of
-MPI_Testany(count, array_of_requests, index, status) has the same effect as the execution of MPI_Test(&\fIarray_of_requests[i\fP], \fIflag\fP, \fIstatus\fP), for \fIi\fP=0,1,...,count-1, in some arbitrary order, until one call returns \fIflag\fP = true, or all fail. In the former case, \fIindex\fP is set to the last value of \fIi\fP, and in the latter case, it is set to MPI_UNDEFINED. MPI_Testany with an array containing one active entry is equivalent to MPI_Test.
-.sp
-If your application does not need to examine the \fIstatus\fP field, you can save resources by using the predefined constant MPI_STATUS_IGNORE as a special value for the \fIstatus\fP argument.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler, MPI_File_set_errhandler, or
-MPI_Win_set_errhandler (depending on the type of MPI handle that
-generated the request); the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does
-not guarantee that an MPI program can continue past an error.
-.sp
-Note that per MPI-1 section 3.2.5, MPI errors on requests passed
-to MPI_TESTANY do not set the status.MPI_ERROR field in the returned
-status.  The error code is passed to the back-end error handler and
-may be passed back to the caller through the return value of
-MPI_TESTANY if the back-end error handler returns it.  The pre-defined
-MPI error handler MPI_ERRORS_RETURN exhibits this behavior, for
-example.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Comm_set_errhandler
-.br
-MPI_File_set_errhandler
-.br
-MPI_Test
-.br
-MPI_Testall
-.br
-MPI_Testsome
-.br
-MPI_Wait
-.br
-MPI_Waitall
-.br
-MPI_Waitany
-.br
-MPI_Waitsome
-.br
-MPI_Win_set_errhandler
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Testsome.3in b/ompi/mpi/man/man3/MPI_Testsome.3in
deleted file mode 100644
index 4deb07dd712..00000000000
--- a/ompi/mpi/man/man3/MPI_Testsome.3in
+++ /dev/null
@@ -1,143 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2011-2015 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Testsome 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Testsome\fP \- Tests for completion of one or more previously initiated communications in a list.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Testsome(int \fIincount\fP, MPI_Request \fIarray_of_requests[]\fP,
-	int\fI *outcount\fP, int\fI array_of_indices[]\fP,
-	MPI_Status\fI array_of_statuses[]\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TESTSOME(\fIINCOUNT, ARRAY_OF_REQUESTS, OUTCOUNT,
-		ARRAY_OF_INDICES, ARRAY_OF_STATUSES, IERROR\fP)
-	INTEGER	\fIINCOUNT, ARRAY_OF_REQUESTS(*)\fP
-	INTEGER	\fIOUTCOUNT, ARRAY_OF_INDICES(*)\fP
-	INTEGER	\fIARRAY_OF_STATUSES(MPI_STATUS_SIZE,*), IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Testsome(\fIincount\fP, \fIarray_of_requests\fP, \fIoutcount\fP, \fIarray_of_indices\fP,
-		\fIarray_of_statuses\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIincount\fP
-	TYPE(MPI_Request), INTENT(INOUT) :: \fIarray_of_requests(incount)\fP
-	INTEGER, INTENT(OUT) :: \fIoutcount\fP, \fIarray_of_indices(*)\fP
-	TYPE(MPI_Status) :: \fIarray_of_statuses(*)\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-incount
-Length of array_of_requests (integer).
-.TP 1i
-array_of_requests
-Array of requests (array of handles).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-outcount
-Number of completed requests (integer).
-.TP 1i
-array_of_indices
-Array of indices of operations that completed (array of integers).
-.TP 1i
-array_of_statuses
-Array of status objects for operations that completed (array of status).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Behaves like MPI_Waitsome, except that it returns immediately.
-.sp
-Returns in outcount the number of requests from the list
-array_of_requests that have completed. Returns in the first outcount
-locations of the array array_of_indices the indices of these
-operations (index within the array array_of_requests; the array is
-indexed from 0 in C and from 1 in Fortran). Returns in the first
-outcount locations of the array array_of_status the status for these
-completed operations. If a request that completed was allocated by a
-nonblocking communication call, then it is deallocated, and the
-associated handle is set to MPI_REQUEST_NULL.
-.sp
-If no operation has completed it returns outcount = 0. If there is no
-active handle in the list, it returns outcount = MPI_UNDEFINED.
-.sp
-MPI_Testsome is a local operation, which returns immediately, whereas MPI_Waitsome blocks until a communication completes, if it was passed a list that contains at least one active handle. Both calls fulfill a fairness requirement: If a request for a receive repeatedly appears in a list of requests passed to MPI_Waitsome or MPI_Testsome, and a matching send has been posted, then the receive will eventually succeed unless the send is satisfied by another receive; send requests also fulfill this fairness requirement.
-.sp
-Errors that occur during the execution of MPI_Testsome are handled as for
-MPI_Waitsome.
-.sp
-If your application does not need to examine the \fIarray_of_statuses\fP field, you can save resources by using the predefined constant MPI_STATUSES_IGNORE can be used as a special value for the \fIarray_of_statuses\fP argument.
-
-.SH NOTES
-The use of MPI_Testsome is likely to be more
-efficient than the use of MPI_Testany. The former returns information on all completed communications; with the latter, a new call is required for each communication that completes.
-.sp
-A server with multiple clients can use MPI_Waitsome so as not to starve any client. Clients send messages to the server with service requests. The server calls MPI_Waitsome with one receive request for each client, then handles all receives that have completed. If a call to MPI_Waitany is used instead, then one client could starve while requests from another client always sneak in first.
-
-.SH ERRORS
-For each invocation of MPI_Testsome, if one or more requests generate
-an MPI error, only the \fIfirst\fP MPI request that caused an
-error will be passed to its corresponding error handler.  No other
-error handlers will be invoked (even if multiple requests generated
-errors).  However, \fIall\fP requests that generate an error
-will have a relevant error code set in the corresponding
-status.MPI_ERROR field (unless MPI_STATUSES_IGNORE was used).
-.sp
-The default error handler aborts the MPI job, except for I/O function
-errors. The error handler may be changed with MPI_Comm_set_errhandler,
-MPI_File_set_errhandler, or MPI_Win_set_errhandler (depending on the
-type of MPI handle that generated the MPI request); the predefined
-error handler MPI_ERRORS_RETURN may be used to cause error values to
-be returned. Note that MPI does not guarantee that an MPI program can
-continue past an error.
-.sp
-If the invoked error handler allows MPI_Testsome to return to the
-caller, the value MPI_ERR_IN_STATUS will be returned in the C and
-Fortran bindings.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Comm_set_errhandler
-.br
-MPI_File_set_errhandler
-.br
-MPI_Test
-.br
-MPI_Testall
-.br
-MPI_Testany
-.br
-MPI_Wait
-.br
-MPI_Waitall
-.br
-MPI_Waitany
-.br
-MPI_Waitsome
-.br
-MPI_Win_set_errhandler
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Topo_test.3in b/ompi/mpi/man/man3/MPI_Topo_test.3in
deleted file mode 100644
index d53552a3833..00000000000
--- a/ompi/mpi/man/man3/MPI_Topo_test.3in
+++ /dev/null
@@ -1,77 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Topo_test 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Topo_test\fP \- Determines the type of topology (if any) associated with a communicator.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Topo_test(MPI_Comm \fIcomm\fP, int\fI *top_type\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TOPO_TEST(\fICOMM, TOP_TYPE, IERROR\fP)
-	INTEGER	\fICOMM, TOP_TYPE, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Topo_test(\fIcomm\fP, \fIstatus\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, INTENT(OUT) :: \fIstatus\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-		\fIDOUBLE PRECISION MPI_Wtick(\fP)
-		\fIDOUBLE PRECISION MPI_Wtime(\fP)
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-top_type
-Topology type of communicator comm (choice).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-The function MPI_Topo_test returns the type of topology that is assigned to a communicator.
-.sp
-The output value \fItop_type\fP is one of the following:
-.sp
-.nf
-    MPI_GRAPH		graph topology
-    MPI_CART		Cartesian topology
-    MPI_DIST_GRAPH	distributed graph topology
-    MPI_UNDEFINED	no topology
-
-.fi
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-MPI_Graph_create
-.br
-MPI_Cart_create
-
diff --git a/ompi/mpi/man/man3/MPI_Type_c2f.3in b/ompi/mpi/man/man3/MPI_Type_c2f.3in
deleted file mode 100644
index a13fce697dd..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_c2f.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Comm_f2c.3
diff --git a/ompi/mpi/man/man3/MPI_Type_commit.3in b/ompi/mpi/man/man3/MPI_Type_commit.3in
deleted file mode 100644
index 2e373d995bb..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_commit.3in
+++ /dev/null
@@ -1,68 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_commit 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Type_commit\fP \- Commits a data type.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_commit(MPI_Datatype *\fIdatatype\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TYPE_COMMIT(\fIDATATYPE, IERROR\fP)
-	INTEGER	\fIDATATYPE, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Type_commit(\fIdatatype\fP, \fIierror\fP)
-	TYPE(MPI_Datatype), INTENT(INOUT) :: \fIdatatype\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-datatype
-Data type (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-The commit operation commits the data type. A data type is the formal description of a communication buffer, not the content of that buffer. After a data type has been committed, it can be repeatedly reused to communicate the changing content of a buffer or, indeed, the content of different buffers, with different starting addresses.
-.sp
-\fBExample:\fP The following Fortran code fragment gives examples of using MPI_Type_commit.
-.sp
-.nf
-    INTEGER type1, type2
-    CALL MPI_TYPE_CONTIGUOUS(5, MPI_REAL, type1, ierr)
-                  ! new type object created
-    CALL MPI_TYPE_COMMIT(type1, ierr)
-                  ! now type1 can be used for communication
-.fi
-.sp
-If the data type specified in \fIdatatype\fP is already committed, it is equivalent to a no-op.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_Type_contiguous.3in b/ompi/mpi/man/man3/MPI_Type_contiguous.3in
deleted file mode 100644
index 3f0eec18efb..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_contiguous.3in
+++ /dev/null
@@ -1,94 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_contiguous 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Type_contiguous\fP \- Creates a contiguous datatype.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_contiguous(int \fIcount\fP, MPI_Datatype\fI oldtype\fP,
-	MPI_Datatype\fI *newtype\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TYPE_CONTIGUOUS(\fICOUNT, OLDTYPE, NEWTYPE, IERROR\fP)
-	INTEGER	\fICOUNT, OLDTYPE, NEWTYPE, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Type_contiguous(\fIcount\fP, \fIoldtype\fP, \fInewtype\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIoldtype\fP
-	TYPE(MPI_Datatype), INTENT(OUT) :: \fInewtype\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-count
-Replication count (nonnegative integer).
-.TP 1i
-oldtype
-Old datatype (handle).
-.sp
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-newtype
-New datatype (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-The simplest datatype constructor is MPI_Type_contiguous, which allows replication of a datatype into contiguous locations.
-.sp
-\fInewtype\fP is the datatype obtained by concatenating \fIcount\fP copies of \fIoldtype\fP. Concatenation is defined using the extent of \fIoldtype\fP as the size of the concatenated copies.
-.sp
-\fBExample:\fR Let oldtype have type map {(double, 0), (char, 8)}, with extent 16, and let count = 3. The type map of the datatype returned by newtype is
-.sp
-.nf
-    {(double, 0), (char, 8), (double, 16), (char, 24),
-    (double, 32), (char, 40)];
-.fi
-.sp
-i.e., alternating double and char elements, with displacements 0, 8, 16, 24, 32, 40.
-.sp
-In general, assume that the type map of oldtype is
-.sp
-.nf
-    {(type(0), disp(0)),...,(type(n-1), disp(n-1))},
-.fi
-.sp
-with extent ex. Then newtype has a type map with count times n entries defined by:
-.sp
-.nf
-    {(type(0), disp(0)), ...,(type(n-1), disp(n-1)),
-    (type(0), disp(0) + ex), ...,(type(n-1),
-    disp(n-1) + ex), ...,(type(0), disp(0) + ex * (count - 1)),
-    ...,(type(n-1), disp(n-1) + ex * (count - 1))}.
-.fi
-.sp
-For more information about derived datatypes, see Section 3.12 of the MPI-1 Standard.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_Type_create_darray.3in b/ompi/mpi/man/man3/MPI_Type_create_darray.3in
deleted file mode 100644
index 4cbbea129c7..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_create_darray.3in
+++ /dev/null
@@ -1,150 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_create_darray 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Type_create_darray\fP \- Creates a distributed array datatype;
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_create_darray(int \fIsize\fP, int \fIrank\fP, int \fIndims\fP,
-	const int \fIarray_of_gsizes\fP[], const int \fIarray_of_distribs\fP[],
-	const int \fIarray_of_dargs\fP[], const int \fIarray_of_psizes\fP[],
-	int \fIorder\fP, MPI_Datatype \fIoldtype\fP, MPI_Datatype \fI*newtype\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TYPE_CREATE_DARRAY(\fISIZE, RANK, NDIMS, ARRAY_OF_GSIZES,
-	ARRAY_OF_DISTRIBS, ARRAY_OF_DARGS, ARRAY_OF_PSIZES, ORDER,
-	OLDTYPE, NEWTYPE, IERROR\fP)
-
-	INTEGER	\fISIZE, RANK, NDIMS, ARRAY_OF_GSIZES(*), ARRAY_OF_DISTRIBS(*),
-		ARRAY_OF_DARGS(*), ARRAY_OF_PSIZES(*), ORDER, OLDTYPE,
-		NEWTYPE, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Type_create_darray(\fIsize\fP, \fIrank\fP, \fIndims\fP, \fIarray_of_gsizes\fP,
-	\fIarray_of_distribs\fP, \fIarray_of_dargs\fP, \fIarray_of_psizes\fP, \fIorder,\fP
-		\fIoldtype\fP, \fInewtype\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIsize\fP, \fIrank\fP, \fIndims\fP, \fIarray_of_gsizes(ndims),\fP
-	\fIarray_of_distribs(ndims)\fP, \fIarray_of_dargs(ndims),\fP
-	\fIarray_of_psizes(ndims)\fP, \fIorder\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIoldtype\fP
-	TYPE(MPI_Datatype), INTENT(OUT) :: \fInewtype\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-size
-Size of process group (positive integer).
-.TP 1i
-rank
-Rank in process group (nonnegative integer).
-.TP 1i
-ndims
-Number of array dimensions as well as process grid dimensions (positive integer).
-.sp
-.TP 1i
-array_of_gsizes
-Number of elements of type \fIoldtype\fP in each dimension of global array (array of positive integers).
-.sp
-.TP 1i
-array_of_distribs
-Distribution of array in each dimension (array of state).
-.TP 1i
-array_of_dargs
-Distribution argument in each dimension (array of positive integers).
-.sp
-.TP 1i
-array_of_psizes
-Size of process grid in each dimension (array of positive integers).
-.sp
-.TP 1i
-order
-Array storage order flag (state).
-.TP 1i
-oldtype
-Old data type (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-newtype
-New data type (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-
-MPI_Type_create_darray can be used to generate the data types corresponding to the distribution of an ndims-dimensional array of \fIoldtype\fP elements onto an \fIndims\fP-dimensional grid of logical processes. Unused dimensions of \fIarray_of_psizes\fP should be set to 1. For a call to MPI_Type_create_darray to be correct, the equation
-.sp
-.nf
-    \fIndims\fP-1
-  pi              \fIarray_of_psizes[i]\fP = \fIsize\fP
-    \fIi\fP=0
-
-.fi
-.sp
-must be satisfied. The ordering of processes in the process grid is assumed to be row-major, as in the case of virtual Cartesian process topologies in MPI-1.
-.sp
-Each dimension of the array can be distributed in one of three ways:
-.sp
-.nf
-- MPI_DISTRIBUTE_BLOCK - Block distribution
-- MPI_DISTRIBUTE_CYCLIC - Cyclic distribution
-- MPI_DISTRIBUTE_NONE - Dimension not distributed.
-.fi
-.sp
-The constant MPI_DISTRIBUTE_DFLT_DARG specifies a default distribution argument. The distribution argument for a dimension that is not distributed is ignored. For any dimension \fIi\fP in which the distribution is MPI_DISTRIBUTE_BLOCK, it erroneous to specify \fIarray_of_dargs[i]\fP \fI*\fP \fIarray_of_psizes[i]\fP < \fIarray_of_gsizes[i]\fP.
-.sp
-For example, the HPF layout ARRAY(CYCLIC(15)) corresponds to MPI_DISTRIBUTE_CYCLIC with a distribution argument of 15, and the HPF layout ARRAY(BLOCK) corresponds to MPI_DISTRIBUTE_BLOCK with a distribution argument of MPI_DISTRIBUTE_DFLT_DARG.
-.sp
-The \fIorder\fP argument is used as in MPI_TYPE_CREATE_SUBARRAY to specify the storage order. Therefore, arrays described by this type constructor may be stored in Fortran (column-major) or C (row-major) order. Valid values for order are MPI_ORDER_FORTRAN and MPI_ORDER_C.
-.sp
-This routine creates a new MPI data type with a typemap defined in terms of a function called "cyclic()" (see below).
-.sp
-Without loss of generality, it suffices to define the typemap for the MPI_DISTRIBUTE_CYCLIC case where MPI_DISTRIBUTE_DFLT_DARG is not used.
-.sp
-MPI_DISTRIBUTE_BLOCK and MPI_DISTRIBUTE_NONE can be reduced to the MPI_DISTRIBUTE_CYCLIC case for dimension \fIi\fP as follows.
-.sp
-MPI_DISTRIBUTE_BLOCK with \fIarray_of_dargs[i]\fP equal to MPI_DISTRIBUTE_DFLT_DARG is equivalent to MPI_DISTRIBUTE_CYCLIC with \fIarray_of_dargs[i]\fP set to
-.sp
-.nf
-   (\fIarray_of_gsizes[i]\fP + \fIarray_of_psizes[i]\fP - 1)/\fIarray_of_psizes[i]\fP
-.fi
-.sp
-If \fIarray_of_dargs[i]\fP is not MPI_DISTRIBUTE_DFLT_DARG, then MPI_DISTRIBUTE_BLOCK and DISTRIBUTE_CYCLIC are equivalent.
-.sp
-MPI_DISTRIBUTE_NONE is equivalent to MPI_DISTRIBUTE_CYCLIC with \fIarray_of_dargs[i]\fP set to \fIarray_of_gsizes[i]\fP.
-.sp
-Finally, MPI_DISTRIBUTE_CYCLIC with \fIarray_of_dargs[i]\fP equal to MPI_DISTRIBUTE_DFLT_DARG is equivalent to MPI_DISTRIBUTE_CYCLIC with \fIarray_of_dargs[i]\fP set to 1.
-.sp
-
-.SH NOTES
-.ft R
-For both Fortran and C arrays, the ordering of processes in the process grid is assumed to be row-major. This is consistent with the ordering used in virtual Cartesian process topologies in MPI-1. To create such virtual process topologies, or to find the coordinates of a process in the process grid, etc., users may use the corresponding functions provided in MPI-1.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-
diff --git a/ompi/mpi/man/man3/MPI_Type_create_f90_complex.3in b/ompi/mpi/man/man3/MPI_Type_create_f90_complex.3in
deleted file mode 100644
index e74b5cda97f..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_create_f90_complex.3in
+++ /dev/null
@@ -1,140 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_create_f90_complex 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-
-.SH NAME
-.nf
-\fBMPI_Type_create_f90_complex\fP \- Returns a bounded MPI complex datatype
-
-.fi
-.SH SYNTAX
-.ft R
-
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_create_f90_complex(int \fIp\fP, int \fIr\fP,
-	MPI_Datatype *\fInewtype\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TYPE_CREATE_F90_COMPLEX (\fIP, R, NEWTYPE, IERROR\fP)
-	INTEGER	\fIP, R, NEWTYPE, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Type_create_f90_complex(\fIp\fP, \fIr\fP, \fInewtype\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIp\fP, \fIr\fP
-	TYPE(MPI_Datatype), INTENT(OUT) :: \fInewtype\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-p
-Precision, in decimal digits (integer).
-.TP 1i
-r
-Decimal exponent range (integer).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-newtype
-New data type (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-This function provides a way to declare KIND-parameterized COMPLEX MPI
-datatypes. The arguments are interpreted in a similar fashion to the
-F90 function SELECTED_REAL_KIND. The parameters \fIp\fP and \fIr\fP
-must be scalar integers. The argument \fIp\fP represents the required
-level of numerical precision, in decimal digits. The \fIr\fP parameter
-indicates the range of exponents desired: the returned datatype will
-have at least one exponent between \+\fIr\fP and \-\fIr\fP (inclusive).
-.sp
-Either \fIp\fP or \fIr\fP, but not both, may be omitted from calls to
-SELECTED_REAL_KIND. Similarly, either argument to
-MPI_Type_create_f90_complex may be set to MPI_UNDEFINED.
-
-.SH NOTES
-.ft R
-It is erroneous to supply values for \fIp\fP and \fIr\fP not supported by
-the compiler.
-.sp
-The Fortran function SELECTED_REAL_KIND maps a large number of
-(\fIp,r\fP) pairs to a much smaller number of KIND parameters
-supported by the compiler. KIND parameters are not specified by the
-language and are not portable. From the point of view of the language,
-variables of the same base type and KIND parameter are equivalent,
-even if their KIND parameters were generated by different (\fIp,r\fP)
-arguments to SELECTED_REAL_KIND. However, to help facilitate
-interoperability in a heterogeneous environment, equivalency is more
-strictly defined for datatypes returned by
-MPI_Type_create_f90_complex. Two MPI datatypes, each generated by this
-function, will match if and only if they have identical values for
-both \fIp\fP and \fIr\fP.
-.sp
-The interaction between the datatypes returned by this function and
-the external32 data representation \- used by MPI_Pack_external,
-MPI_Unpack_external, and many MPI_File functions \- is subtle. The
-external32 representation of returned datatypes is as follows.
-.sp
-.nf
-	if (\fIp\fP > 33) and/or (\fIr\fP > 4931):
-		external32 size = n/a (undefined)
-	else if (\fIp\fP > 15) and/or (\fIr\fP > 307):
-		external32 size = 32
-	else if (\fIp\fP > 6) and/or (\fIr\fP > 37):
-		external32 size = 16
-	else:
-		external32 size = 8
-.fi
-.sp
-If the external32 representation of a datatype is undefined, so are
-the results of using that datatype in operations that require the
-external32 format. Care should be taken not to use incompatible
-datatypes indirectly, e.g., as part of another datatype or through a
-duplicated datatype, in these functions.
-.sp
-If a variable is declared specifying a nondefault KIND value that was
-not obtained with SELECTED_REAL_KIND (i.e., \fIp\fP and/or \fIr\fP are
-unknown), the only way to obtain a matching MPI datatype is to use the
-functions MPI_Sizeof and MPI_Type_match_size.
-
-.SH ERRORS
-.ft R
-Almost all MPI routines return an error value; C routines as
-the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-.sp
-See the MPI man page for a full list of MPI error codes.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Pack_external
-MPI_Sizeof
-MPI_Type_match_size
-MPI_Unpack_external
-SELECTED_REAL_KIND
-
diff --git a/ompi/mpi/man/man3/MPI_Type_create_f90_integer.3in b/ompi/mpi/man/man3/MPI_Type_create_f90_integer.3in
deleted file mode 100644
index b0cacf9c261..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_create_f90_integer.3in
+++ /dev/null
@@ -1,133 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_create_f90_integer 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-
-.SH NAME
-.nf
-\fBMPI_Type_create_f90_integer\fP \- Returns a bounded MPI integer datatype
-
-.fi
-.SH SYNTAX
-.ft R
-
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_create_f90_integer(int \fIr\fP, MPI_Datatype *\fInewtype\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TYPE_CREATE_F90_INTEGER (\fIR, NEWTYPE, IERROR\fP)
-	INTEGER	\fIR, NEWTYPE, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Type_create_f90_integer(\fIr\fP, \fInewtype\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIr\fP
-	TYPE(MPI_Datatype), INTENT(OUT) :: \fInewtype\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-r
-Precision, in decimal digits (integer).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-newtype
-New data type (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-This function provides a way to declare KIND-parameterized INTEGER MPI
-datatypes. The argument is interpreted in a similar fashion to the F90
-function SELECTED_INT_KIND: \fIr\fP must be a scalar integer, and
-represents the desired level of numerical precision, in decimal
-digits.
-
-.SH NOTES
-.ft R
-It is erroneous to supply a value for \fIr\fP not supported by the
-compiler.
-.sp
-The Fortran function SELECTED_INT_KIND maps a large number of \fIr\fP
-values to a much smaller number of KIND parameters supported by the
-compiler. KIND parameters are not specified by the language and are
-not portable. From the point of view of the language, variables of the
-same base type and KIND parameter are equivalent, even if their KIND
-parameters were generated by different \fIr\fP arguments to
-SELECTED_INT_KIND. However, to help facilitate interoperability in a
-heterogeneous environment, equivalency is more strictly defined for
-datatypes returned by MPI_Type_create_f90_integer. Two MPI datatypes,
-each generated by this function, will match if and only if they have
-identical values for \fIr\fP.
-.sp
-The interaction between the datatypes returned by this function and
-the external32 data representation \- used by MPI_Pack_external,
-MPI_Unpack_external and many MPI_File functions \- is subtle. The
-external32 representation of returned datatypes is as follows.
-.sp
-.nf
-	if (\fIr\fP > 38):
-		external32 size = n/a (undefined)
-	else if (\fIr\fP > 18):
-		external32 size = 16
-	else if (\fIr\fP > 9):
-		external32 size = 8
-	else if (\fIr\fP > 4):
-		external32 size = 4
-	else if (\fIr\fP > 2):
-		external32 size = 2
-	else:
-		external32 size = 1
-.fi
-.sp
-If the external32 representation of a datatype is undefined, so are
-the results of using that datatype in operations that require the
-external32 format. Care should be taken not to use incompatible
-datatypes indirectly, e.g., as part of another datatype or through a
-duplicated datatype, in these functions.
-.sp
-If a variable is declared specifying a nondefault KIND value that was
-not obtained with SELECTED_INT_KIND (i.e., \fIr\fP is unknown), the
-only way to obtain a matching MPI datatype is to use the functions
-MPI_Sizeof and MPI_Type_match_size.
-
-.SH ERRORS
-.ft R
-Almost all MPI routines return an error value; C routines as
-the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-.sp
-See the MPI man page for a full list of MPI error codes.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Pack_external
-MPI_Sizeof
-MPI_Type_match_size
-MPI_Unpack_external
-SELECTED_INT_KIND
-
diff --git a/ompi/mpi/man/man3/MPI_Type_create_f90_real.3in b/ompi/mpi/man/man3/MPI_Type_create_f90_real.3in
deleted file mode 100644
index b074399f186..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_create_f90_real.3in
+++ /dev/null
@@ -1,139 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_create_f90_real 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-
-.SH NAME
-.nf
-\fBMPI_Type_create_f90_real\fP \- Returns a bounded MPI real datatype
-
-.fi
-.SH SYNTAX
-.ft R
-
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_create_f90_real(int \fIp\fP, int \fIr\fP, MPI_Datatype *\fInewtype\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TYPE_CREATE_F90_REAL (\fIP, R, NEWTYPE, IERROR\fP)
-	INTEGER	\fIP, R, NEWTYPE, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Type_create_f90_real(\fIp\fP, \fIr\fP, \fInewtype\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIp\fP, \fIr\fP
-	TYPE(MPI_Datatype), INTENT(OUT) :: \fInewtype\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-p
-Precision, in decimal digits (integer).
-.TP 1i
-r
-Decimal exponent range (integer).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-newtype
-New data type (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-This function provides a way to declare KIND-parameterized REAL MPI
-datatypes. The arguments are interpreted in a similar fashion to the
-F90 function SELECTED_REAL_KIND. The parameters \fIp\fP and \fIr\fP
-must be scalar integers. The argument \fIp\fP represents the required
-level of numerical precision, in decimal digits. The \fIr\fP parameter
-indicates the range of exponents desired: the returned datatype will
-have at least one exponent between \+\fIr\fP and \-\fIr\fP (inclusive).
-.sp
-Either \fIp\fP or \fIr\fP, but not both, may be omitted from calls to
-SELECTED_REAL_KIND. Similarly, either argument to
-MPI_Type_create_f90_real may be set to MPI_UNDEFINED.
-
-.SH NOTES
-.ft R
-It is erroneous to supply values for \fIp\fP and \fIr\fP not supported by
-the compiler.
-.sp
-The Fortran function SELECTED_REAL_KIND maps a large number of
-(\fIp,r\fP) pairs to a much smaller number of KIND parameters
-supported by the compiler. KIND parameters are not specified by the
-language and are not portable. From the point of view of the language,
-variables of the same base type and KIND parameter are equivalent,
-even if their KIND parameters were generated by different (\fIp,r\fP)
-arguments to SELECTED_REAL_KIND. However, to help facilitate
-interoperability in a heterogeneous environment, equivalency is more
-strictly defined for datatypes returned by
-MPI_Type_create_f90_real. Two MPI datatypes, each generated by this
-function, will match if and only if they have identical values for
-both \fIp\fP and \fIr\fP.
-.sp
-The interaction between the datatypes returned by this function and
-the external32 data representation \- used by MPI_Pack_external,
-MPI_Unpack_external and many MPI_File functions \- is subtle. The
-external32 representation of returned datatypes is as follows.
-.sp
-.nf
-	if (\fIp\fP > 33) and/or (\fIr\fP > 4931):
-		external32 size = n/a (undefined)
-	else if (\fIp\fP > 15) and/or (\fIr\fP > 307):
-		external32 size = 16
-	else if (\fIp\fP > 6) and/or (\fIr\fP > 37):
-		external32 size = 8
-	else:
-		external32 size = 4
-.fi
-.sp
-If the external32 representation of a datatype is undefined, so are
-the results of using that datatype in operations that require the
-external32 format. Care should be taken not to use incompatible
-datatypes indirectly, e.g., as part of another datatype or through a
-duplicated datatype, in these functions.
-.sp
-If a variable is declared specifying a nondefault KIND value that was
-not obtained with SELECTED_REAL_KIND (i.e., \fIp\fP and/or \fIr\fP are
-unknown), the only way to obtain a matching MPI datatype is to use the
-functions MPI_Sizeof and MPI_Type_match_size.
-
-.SH ERRORS
-.ft R
-Almost all MPI routines return an error value; C routines as
-the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-.sp
-See the MPI man page for a full list of MPI error codes.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Pack_external
-MPI_Sizeof
-MPI_Type_match_size
-MPI_Unpack_external
-SELECTED_REAL_KIND
-
diff --git a/ompi/mpi/man/man3/MPI_Type_create_hindexed.3in b/ompi/mpi/man/man3/MPI_Type_create_hindexed.3in
deleted file mode 100644
index 079131d99d6..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_create_hindexed.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Type_indexed.3
diff --git a/ompi/mpi/man/man3/MPI_Type_create_hindexed_block.3in b/ompi/mpi/man/man3/MPI_Type_create_hindexed_block.3in
deleted file mode 100644
index eb17e26c190..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_create_hindexed_block.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Type_create_indexed_block.3
diff --git a/ompi/mpi/man/man3/MPI_Type_create_hvector.3in b/ompi/mpi/man/man3/MPI_Type_create_hvector.3in
deleted file mode 100644
index 25a0e3bbae5..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_create_hvector.3in
+++ /dev/null
@@ -1,96 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_create_hvector 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Type_create_hvector\fP \- Creates a vector (strided) data type with offset in bytes.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_create_hvector(int \fIcount\fP, int \fIblocklength\fP,
-	MPI_Aint \fIstride\fP, MPI_Datatype \fIoldtype\fP, MPI_Datatype *\fInewtype\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TYPE_CREATE_HVECTOR(\fICOUNT, BLOCKLENGTH, STRIDE, OLDTYPE,
-	NEWTYPE, IERROR\fP)
-
-	INTEGER	\fICOUNT, BLOCKLENGTH, OLDTYPE, NEWTYPE, IERROR\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fISTRIDE\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Type_create_hvector(\fIcount\fP, \fIblocklength\fP, \fIstride\fP, \fIoldtype\fP, \fInewtype\fP,
-		\fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIcount\fP, \fIblocklength\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) :: \fIstride\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIoldtype\fP
-	TYPE(MPI_Datatype), INTENT(OUT) :: \fInewtype\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-count
-Number of blocks (nonnegative integer).
-.TP 1i
-blocklength
-Number of elements in each block (nonnegative integer).
-.TP 1i
-stride
-Number of bytes between start of each block (integer).
-.TP 1i
-oldtype
-Old data type (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-newtype
-New data type (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-MPI_Type_create_hvector creates a vector (strided) data type with offset in bytes.
-.PP
-NOTE \- This routine replaces MPI_Type_hvector, which is deprecated. See the man page MPI_Type_hvector(3) for information about that routine.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fISTRIDE\fP argument only for Fortran 90.  FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_ADDRESS_KIND \fISTRIDE\fP
-.fi
-.sp
-where MPI_ADDRESS_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-MPI_Type_hvector
-.br
-MPI_Type_vector
-.br
diff --git a/ompi/mpi/man/man3/MPI_Type_create_indexed_block.3in b/ompi/mpi/man/man3/MPI_Type_create_indexed_block.3in
deleted file mode 100644
index ad4a92b0473..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_create_indexed_block.3in
+++ /dev/null
@@ -1,99 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" Copyright (c) 2020      FUJITSU LIMITED.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_create_indexed_block 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Type_create_indexed_block, MPI_Type_create_hindexed_block\fP \- Creates an indexed data type with the same block length for all blocks.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_create_indexed_block(int \fIcount\fP, int \fIblocklength\fP, const int \fIarray_of_displacements\fP[], MPI_Datatype \fIoldtype\fP, MPI_Datatype *\fInewtype\fP)
-
-int MPI_Type_create_hindexed_block(int \fIcount\fP, int \fIblocklength\fP, const MPI_Aint \fIarray_of_displacements\fP[], MPI_Datatype \fIoldtype\fP, MPI_Datatype *\fInewtype\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TYPE_CREATE_INDEXED_BLOCK(\fICOUNT, BLOCKLENGTH,
-		ARRAY_OF_DISPLACEMENTS, OLDTYPE, NEWTYPE, IERROR\fP)
-	INTEGER	\fICOUNT, BLOCKLENGTH, ARRAY_OF_DISPLACEMENTS(*),
-	        OLDTYPE, NEWTYPE, IERROR \fP
-
-MPI_TYPE_CREATE_HINDEXED_BLOCK(\fICOUNT, BLOCKLENGTH,
-		ARRAY_OF_DISPLACEMENTS, OLDTYPE, NEWTYPE, IERROR\fP)
-	INTEGER	\fICOUNT, BLOCKLENGTH, OLDTYPE, NEWTYPE\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fIARRAY_OF_DISPLACEMENTS(*)\fP
-	INTEGER	\fIIERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Type_create_indexed_block(\fIcount\fP, \fIblocklength\fP, \fIarray_of_displacements\fP,
-		\fIoldtype\fP, \fInewtype\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIcount\fP, \fIblocklength,\fP
-	\fIarray_of_displacements(count)\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIoldtype\fP
-	TYPE(MPI_Datatype), INTENT(OUT) :: \fInewtype\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPI_Type_create_hindexed_block(\fIcount\fP, \fIblocklength\fP, \fIarray_of_displacements\fP,
-		\fIoldtype\fP, \fInewtype\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIcount\fP, \fIblocklength\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) ::
-	\fIarray_of_displacements(count)\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIoldtype\fP
-	TYPE(MPI_Datatype), INTENT(OUT) :: \fInewtype\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-count
-Length of array of displacements (integer).
-.TP 1i
-blocklength
-Size of block (integer).
-.TP 1i
-array_of_displacements
-Array of displacements (array of integers). In units of the extent of \fIoldtype\fP for MPI_Type_create_indexed_block and bytes for MPI_Type_create_hindexed_block.
-.TP 1i
-oldtype
-Old data type (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-newtype
-New data type (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Type_create_indexed_block and MPI_Type_create_hindexed_block create an indexed data type with the same block length for all blocks. The only difference between the two functions is MPI_Type_create_indexed_block takes an array of displacements in units of the extent of \fIoldtype\fP while MPI_Type_create_hindexed_block takes displacements in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Type_indexed
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Type_create_keyval.3in b/ompi/mpi/man/man3/MPI_Type_create_keyval.3in
deleted file mode 100644
index e678ed95c62..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_create_keyval.3in
+++ /dev/null
@@ -1,127 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_create_keyval 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Type_create_keyval\fP \- Generates a new attribute key for caching on data types.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_create_keyval(MPI_Type_copy_attr_function *\fItype_copy_attr_fn\fP,
-	MPI_Type_delete_attr_function *\fItype_delete_attr_fn\fP,
-	int *\fItype_keyval\fP, void *\fIextra_state\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TYPE_CREATE_KEYVAL(\fITYPE_COPY_ATTR_FN, TYPE_DELETE_ATTR_FN,
-		TYPE_KEYVAL, EXTRA_STATE, IERROR\fP)
-	EXTERNAL \fITYPE_COPY_ATTR_FN, TYPE_DELETE_ATTR_FN\fP
-	INTEGER	\fITYPE_KEYVAL, IERROR \fP
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fIEXTRA_STATE\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Type_create_keyval(\fItype_copy_attr_fn\fP, \fItype_delete_attr_fn\fP, \fItype_keyval\fP,
-		\fIextra_state\fP, \fIierror\fP)
-	PROCEDURE(MPI_Type_copy_attr_function) :: \fItype_copy_attr_fn\fP
-	PROCEDURE(MPI_Type_delete_attr_function) :: \fItype_delete_attr_fn\fP
-	INTEGER, INTENT(OUT) :: \fItype_keyval\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) :: \fIextra_state\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-type_copy_attr_fn
-Copy callback function for \fItype_keyval\fP (function).
-.TP 1i
-type_delete_attr_fn
-Delete callback function for \fItype_keyval\fP (function).
-.TP 1i
-extra_state
-Extra state for callback functions.
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-type_keyval
-Key value for future access (integer).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Type_create_keyval generates a new attribute key for caching on data types. This routine partially replaces MPI_Keyval_create.
-.sp
-The argument \fItype_copy_attr_fn\fP may be specified as MPI_TYPE_NULL_COPY_FN or MPI_TYPE_DUP_FN from C or Fortran. MPI_TYPE_NULL_COPY_FN is a function that does nothing other than returning \fIflag\fP = 0 and MPI_SUCCESS. MPI_TYPE_DUP_FN is a simple-minded copy function that sets \fIflag\fP = 1, returns the value of \fIattribute_val_in\fP in \fIattribute_val_out\fP, and returns MPI_SUCCESS.
-.sp
-The argument \fItype_delete_attr_fn\fP may be specified as MPI_TYPE_NULL_DELETE_FN from C or Fortran. MPI_TYPE_NULL_DELETE_FN is a function that does nothing beyond returning MPI_SUCCESS.
-The C callback functions are:
-.sp
-.nf
-typedef int MPI_Type_copy_attr_function(MPI_Datatype \fIoldtype\fP,
-            int \fItype_keyval\fP, void *\fIextra_state\fP, void *\fIattribute_val_in\fP,
-            void *\fIattribute_val_out\fP, int *\fIflag\fP);
-.fi
-and
-.nf
-typedef int MPI_Type_delete_attr_function(MPI_Datatype \fItype\fP, int \fItype_keyval\fP,
-             void *\fIattribute_val\fP, void *\fIextra_state\fP);
-.fi
-.sp
-The Fortran callback functions are:
-.sp
-.nf
-SUBROUTINE TYPE_COPY_ATTR_FN(\fIOLDTYPE, TYPE_KEYVAL, EXTRA_STATE,
-             ATTRIBUTE_VAL_IN, ATTRIBUTE_VAL_OUT, FLAG, IERROR\fP)
-    INTEGER \fIOLDTYPE, TYPE KEYVAL, IERROR\fP
-    INTEGER(KIND=MPI_ADDRESS_KIND) \fIEXTRA_STATE,
-        ATTRIBUTE_VAL_IN, ATTRIBUTE_VAL_OUT\fP
-    LOGICAL \fIFLAG\fP
-.fi
-and
-.nf
-SUBROUTINE TYPE_DELETE_ATTR_FN(\fITYPE, TYPE_KEYVAL, ATTRIBUTE_VAL, EXTRA_STATE,
-             IERROR\fP)
-    INTEGER \fITYPE, TYPE_KEYVAL, IERROR\fP
-    INTEGER(KIND=MPI_ADDRESS_KIND) \fIATTRIBUTE VAL, EXTRA_STATE\fP
-.fi
-.sp
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fIEXTRA_STATE\fP argument only for Fortran 90.  FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_ADDRESS_KIND \fIEXTRA_STATE\fP
-.fi
-.sp
-where MPI_ADDRESS_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Type_free_keyval
-
diff --git a/ompi/mpi/man/man3/MPI_Type_create_resized.3in b/ompi/mpi/man/man3/MPI_Type_create_resized.3in
deleted file mode 100644
index 392b677e043..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_create_resized.3in
+++ /dev/null
@@ -1,92 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_create_resized 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Type_create_resized\fP \- Returns a new data type with new extent and upper and lower bounds.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_create_resized(MPI_Datatype \fIoldtype\fP, MPI_Aint\fI lb\fP,
-	MPI_Aint \fIextent\fP, MPI_Datatype *\fInewtype\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TYPE_CREATE_RESIZED(\fIOLDTYPE, LB, EXTENT, NEWTYPE, IERROR\fP)
-	INTEGER	\fIOLDTYPE, NEWTYPE, IERROR\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND)	\fILB, EXTENT\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Type_create_resized(\fIoldtype\fP, \fIlb\fP, \fIextent\fP, \fInewtype\fP, \fIierror\fP)
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) :: \fIlb\fP, \fIextent\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIoldtype\fP
-	TYPE(MPI_Datatype), INTENT(OUT) :: \fInewtype\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-oldtype
-Input data type (handle).
-.TP 1i
-lb
-New lower bound of data type (integer).
-.TP 1i
-extent
-New extent of data type (integer).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-newtype
-Output data type (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Type_create_resized returns in \fInewtype\fP a handle to a new data type that is identical to \fIoldtype\fP, except that the lower bound of this new data type is set to be \fIlb\fP, and its upper bound is set to be \fIlb\fP + \fIextent\fP. Any previous \fIlb\fP and \fIub\fP markers are erased, and a new pair of lower bound and upper bound markers are put in the positions indicated by the \fIlb\fP and \fIextent\fP arguments. This affects the behavior of the data type when used in communication operations, with \fIcount\fP > 1, and when used in the construction of new derived data types.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fILB\fP and \fIEXTENT\fP arguments only for Fortran 90. FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_ADDRESS_KIND \fILB\fP
-or
-     INTEGER*MPI_ADDRESS_KIND \fIEXTENT\fP
-.fi
-.sp
-where MPI_ADDRESS_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH NOTE
-.ft R
-Use of MPI_Type_create_resized is strongly recommended over the old MPI-1 functions MPI_Type_extent and MPI_Type_lb.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-
-MPI_Type_get_extent
-
diff --git a/ompi/mpi/man/man3/MPI_Type_create_struct.3in b/ompi/mpi/man/man3/MPI_Type_create_struct.3in
deleted file mode 100644
index 2e6e0d72d61..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_create_struct.3in
+++ /dev/null
@@ -1,100 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_create_struct 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Type_create_struct\fP \- Creates a structured data type.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_create_struct(int \fIcount\fP, int \fIarray_of_blocklengths\fP[],
-	const MPI_Aint \fIarray_of_displacements\fP[], const MPI_Datatype \fIarray_of_types\fP[],
-	MPI_Datatype *\fInewtype\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TYPE_CREATE_STRUCT(\fICOUNT, ARRAY_OF_BLOCKLENGTHS,
-		ARRAY_OF_DISPLACEMENTS, ARRAY_OF_TYPES, NEWTYPE, IERROR\fP)
-	INTEGER	\fICOUNT, ARRAY_OF_BLOCKLENGTHS(*), ARRAY_OF_TYPES(*),\fP
-	INTEGER \fINEWTYPE, IERROR \fP
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fIARRAY_OF_DISPLACEMENTS(*)\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Type_create_struct(\fIcount\fP, \fIarray_of_blocklengths\fP,
-		\fIarray_of_displacements\fP, \fIarray_of_types\fP, \fInewtype\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIcount\fP, \fIarray_of_blocklengths(count)\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) ::
-	\fIarray_of_displacements(count)\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIarray_of_types(count)\fP
-	TYPE(MPI_Datatype), INTENT(OUT) :: \fInewtype\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-count
-Number of blocks (integer) -- also number of entries in arrays \fIarray_of_types\fP, \fIarray_of_displacements\fP, and \fIarray_of_blocklengths\fP.
-.TP 1i
-array_of_blocklengths
-Number of elements in each block (array of integers).
-.TP 1i
-array_of_displacements
-Byte displacement of each block (array of integers).
-.TP 1i
-array_of_types
-Type of elements in each block (array of handles to data-type objects).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-newtype
-New data type (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-MPI_Type_create_struct creates a structured data type. This routine replaces MPI_Type_struct, which is now deprecated.
-.PP
-NOTE \- This routine replaces MPI_Type_struct, which is deprecated. See the man page MPI_Type_struct(3) for information about that routine.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fIARRAY_OF_DISPLACEMENTS\fP(*) argument only for Fortran 90.  FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_ADDRESS_KIND \fIARRAY_OF_DISPLACEMENTS\fP(*)
-.fi
-.sp
-where MPI_ADDRESS_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Type_struct
-.br
-MPI_Type_create_hindexed
-
diff --git a/ompi/mpi/man/man3/MPI_Type_create_subarray.3in b/ompi/mpi/man/man3/MPI_Type_create_subarray.3in
deleted file mode 100644
index a41bf5e9547..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_create_subarray.3in
+++ /dev/null
@@ -1,137 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_create_subarray 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Type_create_subarray\fP \- Creates a data type describing an \fIn\fP-dimensional subarray of an \fIn\fP-dimensional array.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_create_subarray(int \fIndims\fP, const int \fIarray_of_sizes[]\fP, const int \fIarray_of_subsizes[]\fP, const int \fIarray_of_starts[]\fP, int \fIorder\fP, MPI_Datatype \fIoldtype\fP, MPI_Datatype \fI*newtype\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TYPE_CREATE_SUBARRAY(\fINDIMS, ARRAY_OF_SIZES, ARRAY_OF_SUBSIZES,
-	ARRAY_OF_STARTS, ORDER, OLDTYPE, NEWTYPE, IERROR\fP)
-
-	INTEGER	\fINDIMS, ARRAY_OF_SIZES(*), ARRAY_OF_SUBSIZES(*),
-	ARRAY_OF_STARTS(*), ORDER, OLDTYPE, NEWTYPE, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Type_create_subarray(\fIndims\fP, \fIarray_of_sizes\fP, \fIarray_of_subsizes\fP,
-		\fIarray_of_starts\fP, \fIorder\fP, \fIoldtype\fP, \fInewtype\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIndims\fP, \fIarray_of_sizes(ndims),\fP
-	\fIarray_of_subsizes(ndims)\fP, \fIarray_of_starts(ndims)\fP, \fIorder\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIoldtype\fP
-	TYPE(MPI_Datatype), INTENT(OUT) :: \fInewtype\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-ndims
-Number of array dimensions (positive integer).
-.TP 1i
-array_of_sizes
-Number of elements of type \fIoldtype\fP in each dimension of the full array (array of positive integers).
-.TP 1i
-array_of_subsizes
-Number of elements of type \fIoldtype\fP in each dimension of the subarray (array of positive integers).
-.TP 1i
-array_of_starts
-Starting coordinates of the subarray in each dimension (array of nonnegative integers).
-.TP 1i
-order
-Array storage order flag (state).
-.TP 1i
-oldtype
-Array element data type (handle).
-
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-newtype
-New data type (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-The subarray type constructor creates an MPI data type describing an \fIn\fP-dimensional subarray of an \fIn\fP-dimensional array. The subarray may be situated anywhere within the full array, and may be of any nonzero size up to the size of the larger array as long as it is confined within this array. This type constructor facilitates creating file types to access arrays distributed in blocks among processes to a single file that contains the global array.
-.sp
-This type constructor can handle arrays with an arbitrary number of dimensions and works for both C- and Fortran-ordered matrices (that is, row-major or column-major). Note that a C program may use Fortran order and a Fortran program may use C order.
-.sp
-The \fIndims\fP parameter specifies the number of dimensions in the full data array and gives the number of elements in \fIarray_of_sizes\fP, \fIarray_of_subsizes\fP, and \fIarray_of_starts\fP.
-.sp
-The number of elements of type \fIoldtype\fP in each dimension of the \fIn\fP-dimensional array and the requested subarray are specified by \fIarray_of_sizes\fP and \fIarray_of_subsizes\fP, respectively. For any dimension \fIi\fP, it is erroneous to specify \fIarray_of_subsizes[i]\fP < 1 or \fIarray_of_subsizes[i]\fP > \fIarray of sizes[i]\fP.
-.sp
-The \fIarray_of_starts\fP contains the starting coordinates of each dimension of the subarray. Arrays are assumed to be indexed starting from zero. For any dimension \fIi\fP, it is erroneous to specify
-.sp
-.nf
-\fIarray_of_starts[i]\fP < 0
-.fi
-.sp
-or
-.sp
-.nf
-\fIarray_of_starts[i]\fP > (\fIarray_of_sizes[i]\fP - \fIarray_of_subsizes[i]\fP).
-.fi
-.sp
-The \fIorder\fP argument specifies the storage order for the subarray as well as the full array. It must be set to one of the following:
-.sp
-- MPI_ORDER_C: The ordering used by C arrays, (that is, row-major order)
-.sp
-- MPI_ORDER_FORTRAN: The ordering used by Fortran arrays, (that is, column-major order)
-.sp
-A \fIndims\fP-dimensional subarray (\fInewtype\fP) with no extra padding can be defined by the function Subarray() as follows:
-.sp
-.nf
-   newtype = Subarray(ndims, {size , size ,\..., size       },
-                                  0      1           ndims-1
-             {subsize , subsize , \..., subsize       },
-                     0         1               ndims-1
-             {start , start , \..., start       }, \fIoldtype\fP)
-                   0       1             ndims-1
-.fi
-.sp
-Let the typemap of \fIoldtype\fP have the form:
-.sp
-.nf
-   {(type , disp ), (type , disp ), \..., (type   , disp   )}
-         0      0        1      1              n-1      n-1
-.fi
-.sp
-where type\fIi\fP is a predefined MPI data type, and let \fIex\fP be the extent of \fIoldtype\fP.
-.sp
-The Subarray() function is defined recursively in three equations on page 72 of the MPI-2 standard.
-.sp
-For an example use of MPI_Type_create_subarray in the context of I/O, see Section 9.9.2 of the MPI-2 standard.
-
-
-.SH NOTES
-.ft R
-In a Fortran program with arrays indexed starting from 1, if the starting coordinate of a particular dimension of the subarray is \fIn\fP, then the entry in array of starts for that dimension is \fIn\fP-1.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-
diff --git a/ompi/mpi/man/man3/MPI_Type_delete_attr.3in b/ompi/mpi/man/man3/MPI_Type_delete_attr.3in
deleted file mode 100644
index b9f57103500..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_delete_attr.3in
+++ /dev/null
@@ -1,72 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_delete_attr 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Type_delete_attr\fP \- Deletes a datatype-caching attribute value associated with a key.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_delete_attr(MPI_Datatype \fItype\fP, int \fItype_keyval\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TYPE_DELETE_ATTR(\fITYPE, TYPE_KEYVAL, IERROR\fP)
-	INTEGER	\fITYPE, TYPE_KEYVAL, IERROR \fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Type_delete_attr(\fIdatatype\fP, \fItype_keyval\fP, \fIierror\fP)
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	INTEGER, INTENT(IN) :: \fItype_keyval\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-type
-Data type from which the attribute is deleted (handle).n
-
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-type_keyval
-Key value (integer).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Type_delete_attr deletes a datatype-caching attribute value associated with a key. This routines partially replaces MPI_Attr_delete, which is now deprecated.
-
-
-.SH NOTES
-Note that it is not defined by the MPI standard what happens if the
-delete_fn callback invokes other MPI functions.  In Open MPI, it is
-not valid for delete_fn callbacks (or any of their children) to add or
-delete attributes on the same object on which the delete_fn callback
-is being invoked.
-
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_Type_dup.3in b/ompi/mpi/man/man3/MPI_Type_dup.3in
deleted file mode 100644
index 631120cdb30..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_dup.3in
+++ /dev/null
@@ -1,75 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_dup 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Type_dup\fP \- Duplicates a data type with associated key values.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_dup(MPI_Datatype \fItype\fP, MPI_Datatype *\fInewtype\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TYPE_DUP(\fITYPE, NEWTYPE, IERROR\fP)
-	INTEGER	\fITYPE, NEWTYPE, IERROR \fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Type_dup(\fIoldtype\fP, \fInewtype\fP, \fIierror\fP)
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIoldtype\fP
-	TYPE(MPI_Datatype), INTENT(OUT) :: \fInewtype\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-type
-Data type (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-newtype
-Copy of \fItype\fP (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Type_dup is a type constructor that duplicates the existing type with associated key values. For each key value, the respective copy callback function determines the attribute value associated with this key in the new communicator. One particular action that a copy callback may take is to delete the attribute from the new data type. Returns in \fInewtype\fP a new data type with exactly the same properties as \fItype\fP, as well as any copied cached information. The new data type has identical upper bound and lower bound and yields the same net result when fully decoded with the functions described in Section 8.6 of the MPI-2 standard. \fInewtype\fP has the same committed state as the old \fItype\fP.
-
-
-.SH NOTES
-Note that it is not defined by the MPI standard what happens if the
-attribute copy callback invokes other MPI functions.  In Open MPI, it
-is not valid for attribute copy callbacks (or any of their children)
-to add or delete attributes on the same object on which the attribute
-copy callback is being invoked.
-
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Type_create_keyval
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Type_extent.3in b/ompi/mpi/man/man3/MPI_Type_extent.3in
deleted file mode 100644
index 9cb673b3e6b..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_extent.3in
+++ /dev/null
@@ -1,92 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_extent 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Type_extent\fP \- Returns the extent of a data type, the difference between the upper and lower bounds of the data type -- use of this routine is deprecated.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_extent(MPI_Datatype \fIdatatype\fP, MPI_Aint\fI *extent\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-INCLUDE 'mpif.h'
-MPI_TYPE_EXTENT(\fIDATATYPE, EXTENT, IERROR\fP)
-	INTEGER	\fIDATATYPE, EXTENT, IERROR\fP
-
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-datatype
-Datatype (handle).
-.sp
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-extent
-Datatype extent (integer).
-.sp
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Note that use of this routine is \fIdeprecated\fP as of MPI-2. Please use MPI_Type_get_extent instead.
-.sp
-MPI_Type_extent returns the extent of a data type, the difference between the upper and lower bounds of the data type.
-.sp
-In general, if
-.sp
-.nf
-    Typemap = {(type(0), disp(0)), ..., (type(n-1), disp(n-1))}
-.fi
-.sp
-then the lower bound of Typemap is defined to be
-.sp
-.nf
-              ( min(j) disp(j)                         if no entry has
-  lb(Typemap)=(                                        basic type lb
-              (min(j) {disp(j) such that type(j) = lb} otherwise
-
-.fi
-.sp
-Similarly, the upper bound of Typemap is defined to be
-.sp
-.nf
-              (max(j) disp(j) + sizeof(type(j)) + e    if no entry has
-  ub(Typemap)=(                                        basic type ub
-              (max(j) {disp(j) such that type(j) = ub} otherwise
-.fi
-.sp
-Then
-.sp
-.nf
-    extent(Typemap) = ub(Typemap) - lb(Typemap)
-.fi
-.sp
-If type(i) requires alignment to a byte address that is a multiple of k(i), then e is the least nonnegative increment needed to round extent(Typemap) to the next multiple of max(i) k(i).
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Type_get_extent
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Type_f2c.3in b/ompi/mpi/man/man3/MPI_Type_f2c.3in
deleted file mode 100644
index a13fce697dd..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_f2c.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Comm_f2c.3
diff --git a/ompi/mpi/man/man3/MPI_Type_free.3in b/ompi/mpi/man/man3/MPI_Type_free.3in
deleted file mode 100644
index dcd7781c105..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_free.3in
+++ /dev/null
@@ -1,60 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_free 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Type_free\fP \- Frees a data type.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_free(MPI_Datatype *\fIdatatype\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TYPE_FREE(\fIDATATYPE, IERROR\fP)
-	INTEGER	\fIDATATYPE, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Type_free(\fIdatatype\fP, \fIierror\fP)
-	TYPE(MPI_Datatype), INTENT(INOUT) :: \fIdatatype\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-datatype
-Datatype that is freed (handle).
-.sp
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Marks the datatype object associated with datatype for de-allocation and sets datatype to MPI_DATATYPE_NULL. Any communication that is currently using this datatype will complete normally. Derived datatypes that were defined from the freed datatype are not affected.
-.sp
-Freeing a datatype does not affect any other datatype that was built from the freed datatype. The system behaves as if input datatype arguments to derived datatype constructors are passed by value.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-
-
diff --git a/ompi/mpi/man/man3/MPI_Type_free_keyval.3in b/ompi/mpi/man/man3/MPI_Type_free_keyval.3in
deleted file mode 100644
index e459573eb40..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_free_keyval.3in
+++ /dev/null
@@ -1,61 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_free_keyval 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Type_free_keyval\fP \- Frees a previously created type key value.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_free_keyval(int *\fItype_keyval\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TYPE_FREE_KEYVAL(\fITYPE_KEYVAL, IERROR\fP)
-	INTEGER	\fITYPE_KEYVAL, IERROR \fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Type_free_keyval(\fItype_keyval\fP, \fIierror\fP)
-	INTEGER, INTENT(INOUT) :: \fItype_keyval\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-type_keyval
-Key value to free (integer).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Type_create_keyval
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Type_get_attr.3in b/ompi/mpi/man/man3/MPI_Type_get_attr.3in
deleted file mode 100644
index c27c0fa5b21..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_get_attr.3in
+++ /dev/null
@@ -1,87 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_get_attr 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Type_get_attr\fP \- Returns the attribute associated with a data type.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_get_attr(MPI_Datatype \fItype\fP, int \fItype_keyval\fP, void *\fIattribute_val\fP, int *\fIflag\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TYPE_GET_ATTR(\fITYPE, TYPE_KEYVAL, ATTRIBUTE_VAL, FLAG, IERROR\fP)
-	INTEGER	\fITYPE, TYPE_KEYVAL, IERROR \fP
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fIATTRIBUTE_VAL\fP
-	LOGICAL \fIFLAG\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Type_get_attr(\fIdatatype\fP, \fItype_keyval\fP, \fIattribute_val\fP, \fIflag\fP, \fIierror\fP)
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	INTEGER, INTENT(IN) :: \fItype_keyval\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(OUT) :: \fIattribute_val\fP
-	LOGICAL, INTENT(OUT) :: \fIflag\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-type
-Data type to which the attribute is attached (handle).
-.TP 1i
-type_keyval
-Key value (integer).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-attribute_val
-Attribute value, unless \fIflag\fP = false
-.TP 1i
-flag
-"false" if no attribute is associated with the key (logical).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-For the given data type, MPI_Type_get_attr returns an attribute value that corresponds to the specified key value.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fIATTRIBUTE_VAL\fP argument only for Fortran 90.  Sun FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_ADDRESS_KIND \fIATTRIBUTE_VAL\fP
-.fi
-.sp
-where MPI_ADDRESS_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Type_set_attr
-.br
diff --git a/ompi/mpi/man/man3/MPI_Type_get_contents.3in b/ompi/mpi/man/man3/MPI_Type_get_contents.3in
deleted file mode 100644
index 3b4cb1c33e0..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_get_contents.3in
+++ /dev/null
@@ -1,116 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_get_contents 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Type_get_contents\fP \- Returns information about arguments used in creation of a data type.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_get_contents(MPI_Datatype \fIdatatype\fP, int \fImax_integers\fP,
-	int \fImax_addresses\fP, int \fImax_datatypes\fP, int \fIarray_of_integers\fP[], MPI_Aint \fIarray_of_addresses\fP[], MPI_Datatype array_of_datatypes\fP[])
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TYPE_GET_CONTENTS(\fIDATATYPE, MAX_INTEGERS, MAX_ADDRESSES,
-		MAX_DATATYPES, ARRAY_OF_INTEGERS, ARRAY_OF_ADDRESSES,
-		ARRAY_OF_DATATYPES, IERROR\fP)
-	INTEGER	\fIDATATYPE, MAX_INTEGERS, MAX_ADDRESSES, MAX_DATATYPES\fP
-	INTEGER	\fIARRAY_OF_INTEGERS(*), ARRAY_OF_DATATYPES(*), IERROR\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fIARRAY_OF_ADDRESSES\fP(*)
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Type_get_contents(\fIdatatype\fP, \fImax_integers\fP, \fImax_addresses\fP, \fImax_datatypes\fP,
-	\fIarray_of_integers\fP, \fIarray_of_addresses\fP, \fIarray_of_datatypes,\fP
-		\fIierror\fP)
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	INTEGER, INTENT(IN) :: \fImax_integers\fP, \fImax_addresses\fP, \fImax_datatypes\fP
-	INTEGER, INTENT(OUT) :: \fIarray_of_integers(max_integers)\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(OUT) ::
-	\fIarray_of_addresses(max_addresses)\fP
-	TYPE(MPI_Datatype), INTENT(OUT) :: \fIarray_of_datatypes(max_datatypes)\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-datatype
-Data type to access (handle).
-.TP 1i
-max_integers
-Number of elements in \fIarray_of_integers\fP (nonnegative integer).
-.TP 1i
-max_addresses
-Number of elements in \fIarray_of_addresses\fP (nonnegative integer).
-.TP 1i
-max_datatypes
-Number of elements in \fIarray_of_datatypes\fP (nonnegative integer).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-array_of_integers
-Contains integer arguments used in constructing \fIdatatype\fP (array of integers).
-.TP 1i
-array_of_addresses
-Contains address arguments used in constructing \fIdatatype\fP (array of integers).
-.TP 1i
-array_of_datatypes
-Contains data-type arguments used in constructing \fIdatatype\fP (array of integers).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-For the given data type, MPI_Type_get_envelope returns information on the number and type of input arguments used in the call that created the data type. The number-of-arguments values returned can be used to provide sufficiently large arrays in the decoding routine MPI_Type_get_contents. This call and the meaning of the returned values is described below. The combiner reflects the MPI data type constructor call that was used in creating \fIdatatype\fP.
-
-The parameter \fIdatatype\fP must be a predefined unnamed or a derived data type. The call is erroneous if \fIdatatype\fP is a predefined named data type.
-.sp
-The values given for \fImax_integers\fP, \fImax_addresses\fP, and \fImax_datatypes\fP must be at least as large as the value returned in \fInum_integers\fP, \fInum_addresses\fP, and \fInum_datatypes\fP, respectively, in the call MPI_Type_get_envelope for the same \fIdatatype\fP argument.
-.sp
-The data types returned in \fIarray_of_datatypes\fP are handles to data-type objects that are equivalent to the data types used in the original construction call. If these were derived data types, then the returned data types are new data-type objects, and the user is responsible for freeing these datatypes with MPI_Type_free. If these were predefined data types, then the returned data type is equal to that (constant) predefined data type and cannot be freed.
-.sp
-The committed state of returned derived data types is undefined, that is, the data types may or may not be committed. Furthermore, the content of attributes of returned data types is undefined.
-.sp
-Note that MPI_Type_get_contents can be invoked with a data-type argument that was constructed using MPI_Type_create_f90_real, MPI_Type_create_f90_integer, or MPI_Type_create_f90_complex (an unnamed predefined data type). In such a case, an empty \fIarray_of_datatypes\fP is returned.
-.sp
-In the MPI-1 data-type constructor calls, the address arguments in Fortran are of type INTEGER. In the new MPI-2 calls, the address arguments are of type INTEGER(KIND=MPI_ADDRESS_KIND). The call MPI_Type_get_contents returns all addresses in an argument of type INTEGER(KIND=MPI_ADDRESS_KIND). This is true even if the old MPI-1 calls were used. Thus, the location of values returned can be thought of as being returned by the C bindings. It can also be determined by examining the new MPI-2 calls for data-type constructors for the deprecated MPI-1 calls that involve addresses.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fIARRAY_OF_ADDRESSES\fP argument only for Fortran 90. FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_ADDRESS_KIND \fIARRAY_OF_ADDRESSES\fP(*)
-.fi
-.sp
-where MPI_ADDRESS_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft r
-MPI_Type_get_envelope
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Type_get_envelope.3in b/ompi/mpi/man/man3/MPI_Type_get_envelope.3in
deleted file mode 100644
index 7fece14c1df..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_get_envelope.3in
+++ /dev/null
@@ -1,116 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_get_envelope 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Type_get_envelope\fP \- Returns information about input arguments associated with a data type.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_get_envelope(MPI_Datatype \fIdatatype\fP, int *\fInum_integers\fP,
-	int *\fInum_addresses\fP, int *\fInum_datatypes\fP, int *\fIcombiner\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TYPE_GET_ENVELOPE(\fIDATATYPE, NUM_INTEGERS, NUM_ADDRESSES,
-		NUM_DATATYPES, COMBINER, IERROR\fP)
-	INTEGER	\fIDATATYPE, NUM_INTEGERS, NUM_ADDRESSES\fP
-	INTEGER	\fINUM_DATATYPES, COMBINER, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Type_get_envelope(\fIdatatype\fP, \fInum_integers\fP, \fInum_addresses\fP, \fInum_datatypes\fP,
-		\fIcombiner\fP, \fIierror\fP)
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	INTEGER, INTENT(OUT) :: \fInum_integers\fP, \fInum_addresses\fP, \fInum_datatypes,\fP
-	\fIcombiner\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-datatype
-Data type to access (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-num_integers
-Number of input integers used in the call constructing \fIcombiner\fP (nonnegative integer).
-.TP 1i
-num_addresses
-Number of input addresses used in the call constructing \fIcombiner\fP (nonnegative integer).
-.TP 1i
-num_datatypes
-Number of input data types used in the call constructing \fIcombiner\fP (nonnegative integer).
-.TP 1i
-combiner
-Combiner (state).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-For the given data type, MPI_Type_get_envelope returns information on the number and type of input arguments used in the call that created the data type. The number-of-arguments values returned can be used to provide sufficiently large arrays in the decoding routine MPI_Type_get_contents. This call and the meaning of the returned values is described below. The combiner reflects the MPI data type constructor call that was used in creating \fIdatatype\fP.
-
-.SH NOTES
-.ft R
-These are the values that can be returned in \fIcombiner\fP and their associated calls:
-.sp
-.nf
-Values                          Associated Calls
-
-MPI_COMBINER_NAMED              a named predefined data type
-MPI_COMBINER_DUP                MPI_Type_dup
-MPI_COMBINER_CONTIGUOUS         MPI_Type_contiguous
-MPI_COMBINER_VECTOR             MPI_Type_vector
-MPI_COMBINER_HVECTOR_INTEGER    MPI_Type_hvector from Fortran
-MPI_COMBINER_HVECTOR            MPI_Type_hvector from C
-                                  and MPI_Type_create for
-                                  all languages
-MPI_COMBINER_INDEXED            MPI_Type_indexed
-MPI_COMBINER_HINDEXED_INTEGER   MPI_Type_hindexed from Fortran
-MPI_COMBINER_HINDEXED           MPI_Type_hindexed from C
-                                  and MPI_Type_create_hindexed
-                                  for all languages
-MPI_COMBINER_INDEXED_BLOCK      MPI_Type_create_indexed_block
-MPI_COMBINER_STRUCT_INTEGER     MPI_Type_struct from Fortran
-MPI_COMBINER_STRUCT             MPI_Type_struct from C
-                                  and MPI_Type_create_struct
-                                  for all languages
-MPI_COMBINER_SUBARRAY           MPI_Type_create_subarray
-MPI_COMBINER_DARRAY             MPI_Type_create_darray
-MPI_COMBINER_F90_REAL           MPI_Type_create_f90_real
-MPI_COMBINER_F90_COMPLEX        MPI_Type_create_f90_complex
-MPI_COMBINER_F90_INTEGER        MPI_Type_create_f90_integer
-MPI_COMBINER_RESIZED            MPI_Type_create_resized
-.fi
-.sp
-If \fIcombiner\fP is MPI_COMBINER_NAMED, then \fIdatatype\fP is a named predefined data type.
-.sp
-The actual arguments used in the creation call for a data type can be obtained from the call MPI_Type_get_contents.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft r
-MPI_Type_get_contents
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Type_get_extent.3in b/ompi/mpi/man/man3/MPI_Type_get_extent.3in
deleted file mode 100644
index e7c1368faca..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_get_extent.3in
+++ /dev/null
@@ -1,104 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_get_extent 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Type_get_extent\fP, \fBMPI_Type_get_extent_x\fP \- Returns the lower bound and extent of a data type.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_get_extent(MPI_Datatype \fIdatatype\fP, MPI_Aint\fI *lb\fP,
-	MPI_Aint *\fIextent\fP)
-int MPI_Type_get_extent_x(MPI_Datatype \fIdatatype\fP, MPI_Count\fI *lb\fP,
-	MPI_Count *\fIextent\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TYPE_GET_EXTENT(\fIDATATYPE, LB, EXTENT, IERROR\fP)
-	INTEGER	\fIDATATYPE, IERROR\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fILB, EXTENT\fP
-MPI_TYPE_GET_EXTENT_X(\fIDATATYPE, LB, EXTENT, IERROR\fP)
-	INTEGER	\fIDATATYPE, IERROR\fP
-	INTEGER(KIND=MPI_COUNT_KIND) \fILB, EXTENT\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Type_get_extent(\fIdatatype\fP, \fIlb\fP, \fIextent\fP, \fIierror\fP)
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(OUT) :: \fIlb\fP, \fIextent\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-MPI_Type_get_extent_x(\fIdatatype\fP, \fIlb\fP, \fIextent\fP, \fIierror\fP)
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	INTEGER(KIND = MPI_COUNT_KIND), INTENT(OUT) :: \fIlb\fP, \fIextent\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-datatype
-Data type (handle).
-.sp
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-lb
-Lower bound of data type (integer).
-.TP 1i
-extent
-Data type extent (integer).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Type_get_extent returns the lower bound and the extent of \fIdatatype\fP. For either function, if either the \fIlb\fP or \fIextent\fP parameter cannot express the value to be returned (e.g., if the parameter is too small to hold the output value), it is set to MPI_UNDEFINED.
-
-.SH NOTE
-.ft R
-Use of MPI_Type_get_extent is strongly recommended over the old MPI-1 functions MPI_Type_extent and MPI_Type_lb.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fILB\fP and \fIEXTENT\fP arguments only for Fortran 90. FORTRAN 77
-users may use the non-portable syntax
-.sp
-MPI_Type_get_extent:
-.sp
-.nf
-     INTEGER*MPI_ADDRESS_KIND \fILB\fP
-or
-     INTEGER*MPI_ADDRESS_KIND \fIEXTENT\fP
-.fi
-.sp
-MPI_Type_get_extent_x:
-.sp
-.nf
-     INTEGER*MPI_COUNT_KIND \fILB\fP
-or
-     INTEGER*MPI_COUNT_KIND \fIEXTENT\fP
-.fi
-.sp
-where MPI_ADDRESS_KIND and MPI_COUNT_KIND are constants defined in mpif.h
-and give the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
diff --git a/ompi/mpi/man/man3/MPI_Type_get_extent_x.3in b/ompi/mpi/man/man3/MPI_Type_get_extent_x.3in
deleted file mode 100644
index 84dc57aa67c..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_get_extent_x.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Type_get_extent.3
diff --git a/ompi/mpi/man/man3/MPI_Type_get_name.3in b/ompi/mpi/man/man3/MPI_Type_get_name.3in
deleted file mode 100644
index d636fa56469..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_get_name.3in
+++ /dev/null
@@ -1,72 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_get_name 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Type_get_name\fP \- Gets the name of a data type.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_get_name(MPI_Datatype \fItype\fP, char *\fItype_name\fP,
-	int *\fIresultlen\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TYPE_GET_NAME(\fITYPE, TYPE_NAME, RESULTLEN, IERROR\fP)
-	INTEGER	\fITYPE, RESULTLEN, IERROR \fP
-	CHARACTER*(*) \fITYPE_NAME\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Type_get_name(\fIdatatype\fP, \fItype_name\fP, \fIresultlen\fP, \fIierror\fP)
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	CHARACTER(LEN=MPI_MAX_OBJECT_NAME), INTENT(OUT) :: \fItype_name\fP
-	INTEGER, INTENT(OUT) :: \fIresultlen\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-type
-Data type whose name is to be returned (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-type_name
-The name previously stored on the data type, or an empty string if not such name exists (string).
-.TP 1i
-resultlen
-Length of returned name (integer).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Type_get_name returns the printable identifier associated with an MPI data type.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Type_set_name
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Type_get_true_extent.3in b/ompi/mpi/man/man3/MPI_Type_get_true_extent.3in
deleted file mode 100644
index a133cdac12b..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_get_true_extent.3in
+++ /dev/null
@@ -1,103 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_get_true_extent 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Type_get_true_extent\fP, \fBMPI_Type_get_true_extent_x\fP \- Returns the true lower bound and extent of a data type's corresponding typemap, ignoring MPI_UB and MPI_LB markers.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_get_true_extent(MPI_Datatype \fIdatatype\fP,
-	MPI_Aint *\fItrue_lb\fP, MPI_Aint *\fItrue_extent\fP)
-int MPI_Type_get_true_extent_x(MPI_Datatype \fIdatatype\fP,
-	MPI_Count *\fItrue_lb\fP, MPI_Count *\fItrue_extent\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TYPE_GET_TRUE_EXTENT(\fIDATATYPE, TRUE_LB, TRUE_EXTENT, IERROR\fP)
-	INTEGER	\fIDATATYPE, IERROR\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fITRUE_LB, TRUE_EXTENT\fP
-MPI_TYPE_GET_TRUE_EXTENT_X(\fIDATATYPE, TRUE_LB, TRUE_EXTENT, IERROR\fP)
-	INTEGER	\fIDATATYPE, IERROR\fP
-	INTEGER(KIND=MPI_COUNT_KIND) \fITRUE_LB, TRUE_EXTENT\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Type_get_true_extent(\fIdatatype\fP, \fItrue_lb\fP, \fItrue_extent\fP, \fIierror\fP)
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(OUT) :: \fItrue_lb\fP, \fItrue_extent\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-MPI_Type_get_true_extent_x(\fIdatatype\fP, \fItrue_lb\fP, \fItrue_extent\fP, \fIierror\fP)
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	INTEGER(KIND = MPI_COUNT_KIND), INTENT(OUT) :: \fItrue_lb\fP, \fItrue_extent\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-datatype
-Data type for which information is wanted (handle).
-.sp
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-true_lb
-True lower bound of data type (integer).
-.TP 1i
-true_extent
-True size of data type (integer).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-The \fItrue_lb\fP parameter returns the offset of the lowest unit of store that is addressed by the data type, that is, the lower bound of the corresponding typemap, ignoring MPI_LB markers. The \fItrue_extent\fP parameter returns the true size of the data type, that is, the extent of the corresponding typemap, ignoring MPI_LB and MPI_UB markers, and performing no rounding for alignment. For both functions, if either the \fItrue_lb\fP or \fItrue_extent\fP parameter cannot express the value to be returned (e.g., if the parameter is too small to hold the output value), it is set to MPI_UNDEFINED.
-.sp
-The \fItrue_extent\fP is the minimum number of bytes of memory necessary to hold a data type, uncompressed.
-.sp
-See § 4.1.8 of the MPI-3 standard for more detailed definitions of these parameters in relation to the typemap.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fITRUE_LB\fP and \fITRUE_EXTENT\fP arguments only for Fortran 90. FORTRAN 77 users may use the non-portable syntax
-.sp
-MPI_Type_get_true_extent:
-.sp
-.nf
-     INTEGER*MPI_ADDRESS_KIND \fITRUE_LB\fP
-or
-     INTEGER*MPI_ADDRESS_KIND \fITRUE_EXTENT\fP
-.fi
-.sp
-MPI_Type_get_true_extent_x:
-.sp
-.nf
-     INTEGER*MPI_COUNT_KIND \fITRUE_LB\fP
-or
-     INTEGER*MPI_COUNT_KIND \fITRUE_EXTENT\fP
-.fi
-.sp
-where MPI_ADDRESS_KIND and MPI_COUNT_KIND are constants defined in mpif.h and give the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_Type_get_true_extent_x.3in b/ompi/mpi/man/man3/MPI_Type_get_true_extent_x.3in
deleted file mode 100644
index c7e538643c9..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_get_true_extent_x.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Type_get_true_extent.3
diff --git a/ompi/mpi/man/man3/MPI_Type_hindexed.3in b/ompi/mpi/man/man3/MPI_Type_hindexed.3in
deleted file mode 100644
index 34336a02704..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_hindexed.3in
+++ /dev/null
@@ -1,99 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010-2014 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_hindexed 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Type_hindexed\fP \- Creates an indexed datatype with offsets in bytes -- use of this routine is deprecated.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_hindexed(int \fIcount\fP, int\fI *array_of_blocklengths\fP,
-	MPI_Aint\fI *array_of_displacements\fP, MPI_Datatype\fI oldtype\fP,
-	MPI_Datatype\fI *newtype\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-INCLUDE 'mpif.h'
-MPI_TYPE_HINDEXED(\fICOUNT, ARRAY_OF_BLOCKLENGTHS,
-		ARRAY_OF_DISPLACEMENTS, OLDTYPE, NEWTYPE, IERROR\fP)
-	INTEGER	\fICOUNT, ARRAY_OF_BLOCKLENGTHS(*)\fP
-	INTEGER	\fIARRAY_OF_DISPLACEMENTS(*), OLDTYPE, NEWTYPE\fP
-	INTEGER	\fIIERROR\fP
-
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-count
-Number of blocks -- also number of entries in array_of_displacements  and
-array_of_blocklengths  (integer).
-.TP 1i
-array_of_blocklengths
-Number of elements in each block (array of nonnegative integers).
-.TP 1i
-array_of_displacements
-Byte displacement of each block (C: array of
-.IR MPI_Aint ,
-Fortran: array of integer).
-.TP 1i
-oldtype
-Old datatype (handle).
-.sp
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-newtype
-New datatype (handle).
-.sp
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Note that use of this routine is \fIdeprecated\fP as of MPI-2. Use MPI_Type_create_hindexed instead.
-.sp
-The function is identical to MPI_Type_indexed, except that block displacements in array_of_displacements are specified in bytes, rather than in multiples of the oldtype extent.
-.sp
-Assume that oldtype has type map
-.sp
-.nf
-    {(type(0), disp(0)), ..., (type(n-1), disp(n-1))},
-.fi
-.sp
-with extent ex. Let B be the array_of_blocklength argument and D be the
-array_of_displacements argument. The newly created datatype has
-.nf
-n x S^count-1
-    (i=0)        B[i]  entries:
-
-  {(type(0), disp(0) + D[0]),...,(type(n-1), disp(n-1) + D[0]),...,
-  (type(0), disp(0) + (D[0] + B[0]-1)* ex),...,
-  type(n-1), disp(n-1) + (D[0]+ B[0]-1)* ex),...,
-  (type(0), disp(0) + D[count-1]),...,(type(n-1), disp(n-1) + D[count-1]),...,
-  (type(0), disp(0) +  D[count-1] + (B[count-1] -1)* ex),...,
-  (type(n-1), disp(n-1) + D[count-1] + (B[count-1] -1)* ex)}
-.fi
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-MPI_Type_create_hindexed
-.br
-MPI_Type_indexed
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Type_hvector.3in b/ompi/mpi/man/man3/MPI_Type_hvector.3in
deleted file mode 100644
index c869b5cc916..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_hvector.3in
+++ /dev/null
@@ -1,97 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_hvector 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Type_hvector\fP \- Creates a vector (strided) datatype with offset in bytes -- use of this routine is deprecated.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_hvector(int \fIcount\fP, int\fI blocklength\fP, MPI_Aint\fI stride\fP,
-	MPI_Datatype\fI oldtype\fP, MPI_Datatype\fI *newtype\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-INCLUDE 'mpif.h'
-MPI_TYPE_HVECTOR(\fICOUNT, BLOCKLENGTH, STRIDE, OLDTYPE, NEWTYPE,
-		IERROR\fP)
-	INTEGER	\fICOUNT, BLOCKLENGTH, STRIDE, OLDTYPE\fP
-	INTEGER	\fINEWTYPE, IERROR\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-count
-Number of blocks (nonnegative integer).
-.TP 1i
-blocklength
-Number of elements in each block (nonnegative integer).
-.TP 1i
-stride
-Number of bytes between start of each block (integer).
-.TP 1i
-oldtype
-Old datatype (handle).
-.sp
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-newtype
-New datatype (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Note that use of this routine is \fIdeprecated\fP as of MPI-2. Use MPI_Type_create_hvector instead.
-.sp
-The function MPI_Type_hvector is identical to MPI_Type_vector, except that
-stride is given in bytes, rather than in elements. The use for both types
-of vector constructors is illustrated in the examples in Section 3.12.7 of the MPI-1 Standard.
-.sp
-Assume that oldtype has type map
-.sp
-.nf
-    {(type(0), disp(0)), ..., (type(n-1), disp(n-1))}
-.fi
-.sp
-with extent ex. Let bl be the blocklength. The newly created datatype has a type map with   count * bl * n  entries:
-.sp
-.nf
-  {(type(0), disp(0)), ..., (type(n-1), disp(n-1)),
-  (type(0), disp(0) + ex), ..., (type(n-1), disp(n-1) + ex),
-  ..., (type(0), disp(0) + (bl -1) * ex),...,(type(n-1),
-  disp(n-1) + (bl -1) * ex), (type(0), disp(0) + stride),
-  ...,(type(n-1), disp(n-1) + stride), ..., (type(0),
-  disp(0) + stride + (bl - 1) * ex), ..., (type(n-1),
-  disp(n-1) + stride + (bl -1) * ex), ..., (type(0),
-  disp(0) + stride * (count -1)), ...,(type(n-1),
-  disp(n-1) + stride * (count -1)), ..., (type(0),
-  disp(0) + stride * (count -1) + (bl -1) * ex), ...,
-  (type(n-1), disp(n-1) + stride * (count -1) + (bl -1) * ex)}
-
-.fi
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Type_create_hvector
-.br
-MPI_Type_vector
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Type_indexed.3in b/ompi/mpi/man/man3/MPI_Type_indexed.3in
deleted file mode 100644
index 281f22ad1bc..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_indexed.3in
+++ /dev/null
@@ -1,159 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010-2014 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_indexed 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Type_indexed, MPI_Type_create_hindexed\fP \- Creates an indexed datatype.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_indexed(int \fIcount\fP, const int\fI array_of_blocklengths[]\fP,
-	const int\fI array_of_displacements[]\fP, MPI_Datatype\fI oldtype\fP,
-	MPI_Datatype\fI *newtype\fP)
-
-int MPI_Type_create_hindexed(int \fIcount\fP,
-	const int\fI array_of_blocklengths[]\fP,
-	const MPI_Aint\fI array_of_displacements[]\fP, MPI_Datatype\fI oldtype\fP,
-	MPI_Datatype\fI *newtype\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TYPE_INDEXED(\fICOUNT, ARRAY_OF_BLOCKLENGTHS,
-		ARRAY_OF_DISPLACEMENTS, OLDTYPE, NEWTYPE, IERROR\fP)
-	INTEGER	\fICOUNT, ARRAY_OF_BLOCKLENGTHS(*)\fP
-	INTEGER	\fIARRAY_OF_DISPLACEMENTS(*), OLDTYPE, NEWTYPE\fP
-	INTEGER	\fIIERROR\fP
-
-MPI_TYPE_CREATE_HINDEXED(\fICOUNT, ARRAY_OF_BLOCKLENGTHS,
-		ARRAY_OF_DISPLACEMENTS, OLDTYPE, NEWTYPE, IERROR\fP)
-	INTEGER	\fICOUNT, ARRAY_OF_BLOCKLENGTHS(*)\fP
-	INTEGER	\fIOLDTYPE, NEWTYPE\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fIARRAY_OF_DISPLACEMENTS(*)\fP
-	INTEGER	\fIIERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Type_indexed(\fIcount\fP, \fIarray_of_blocklengths\fP, \fIarray_of_displacements\fP,
-		\fIoldtype\fP, \fInewtype\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIcount\fP, \fIarray_of_blocklengths(count),\fP
-	\fIarray_of_displacements(count)\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIoldtype\fP
-	TYPE(MPI_Datatype), INTENT(OUT) :: \fInewtype\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPI_Type_create_hindexed(\fIcount\fP, \fIarray_of_blocklengths\fP,
-		\fIarray_of_displacements\fP, \fIoldtype\fP, \fInewtype\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIcount\fP, \fIarray_of_blocklengths(count)\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) ::
-	\fIarray_of_displacements(count)\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIoldtype\fP
-	TYPE(MPI_Datatype), INTENT(OUT) :: \fInewtype\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-count
-Number of blocks -- also number of entries in array_of_displacements and
-array_of_blocklengths (nonnegative integer).
-.TP 1i
-array_of_blocklengths
-Number of elements per block (array of nonnegative integers).
-.TP 1i
-array_of_displacements
-Displacement for each block, in multiples of oldtype extent for MPI_Type_indexed and bytes for MPI_Type_create_hindexed (array of
-integer for
-.BR MPI_TYPE_INDEXED ,
-array of
-.I MPI_Aint
-for
-.BR MPI_TYPE_CREATE_HINDEXED ).
-.TP 1i
-oldtype
-Old datatype (handle).
-.sp
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-newtype
-New datatype (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-The function MPI_Type_indexed allows replication of an old datatype into a sequence of blocks (each block is a concatenation of the old datatype), where each block can contain a different number of copies and have a different displacement. All block displacements are multiples of the old data type's extent.
-.sp
-
-\fBExample:\fP Let oldtype have type map {(double, 0), (char, 8)}, with extent 16. Let  B = (3, 1) and let D = (4, 0).  A call to MPI_Type_indexed(2, B, D, oldtype, newtype) returns a datatype with type map
-.sp
-.nf
-    {(double, 64), (char, 72), (double, 80), (char, 88),
-    (double, 96), (char, 104),
-    (double, 0), (char, 8)}
-.fi
-.sp
-That is, three copies of the old type starting at displacement 4 x 16 = 64, and one copy starting at displacement 0.
-.sp
-In general, assume that oldtype has type map
-.sp
-.nf
-    {(type(0), disp(0)), ..., (type(n-1), disp(n-1))},
-.fi
-.sp
-with extent ex. Let B be the array_of_blocklength argument and D be the
-array_of_displacements argument. The newly created datatype has
-.br
-.nf
-n x S ^count-1
-    i = 0           B[i]  entries:
-
-    {(type(0), disp(0) + D[0]* ex), ...,
-    (type(n-1), disp(n-1) + D[0]* ex), ...,
-    (type(0), disp(0) + (D[0] + B[0]-1)* ex), ...,
-    (type(n-1), disp(n-1) + (D[0]+ B[0]-1)* ex), ...,
-    (type(0), disp(0) + D[count-1]* ex), ...,
-    (type(n-1), disp(n-1) + D[count-1]* ex), ...,
-    (type(0), disp(0) +  (D[count-1] + B[count-1] -1)* ex), ...,
-    (type(n-1), disp(n-1) + (D[count-1] + B[count-1] -1)* ex)}
-.fi
-.sp
-A call to MPI_Type_vector(count, blocklength, stride, oldtype, newtype) is equivalent to a call to MPI_Type_indexed(count, B, D, oldtype, newtype) where
-.sp
-.nf
-    D[j] = j * stride, j = 0,..., count-1
-
-and
-
-    B[j] = blocklength, j = 0, .., count-1
-.fi
-
-The function MPI_Type_create_hindexed is identical to MPI_Type_indexed, except that block displacements in \fIarray_of_displacements\fP are specified in bytes, rather than in multiples of the \fIoldtype\fP extent.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Type_hindexed
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Type_lb.3in b/ompi/mpi/man/man3/MPI_Type_lb.3in
deleted file mode 100644
index fc21aba60fc..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_lb.3in
+++ /dev/null
@@ -1,91 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_lb 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Type_lb\fP \- Returns the lower bound of a data type -- use of this routine is deprecated.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_lb(MPI_Datatype \fIdatatype\fP, MPI_Aint\fI *displacement\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-INCLUDE 'mpif.h'
-MPI_TYPE_LB(\fIDATATYPE, DISPLACEMENT, IERROR\fP)
-	INTEGER	\fIDATATYPE, DISPLACEMENT, IERROR\fP
-
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-datatype
-Datatype (handle).
-.sp
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-displacement
-Displacement of lower bound from origin, in bytes (integer).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Note that use of this routine is \fIdeprecated\fP as of MPI-2. Please use MPI_Type_get_extent instead.
-.sp
-MPI_Type_lb returns the lower bound of a data type. This may differ from zero if the type was constructed using MPI_LB.
-.sp
-The "pseudo-datatypes," MPI_LB and MPI_UB, can be used, respectively, to mark the lower bound (or the upper bound) of a datatype. These pseudo-datatypes occupy no space (extent (MPI_LB) = extent (MPI_UB) =0. They do not affect the size or count of a datatype, and do not affect the context of a message created with this datatype. However, they do affect the definition of the extent of a datatype and, therefore, affect the outcome of a replication of this datatype by a datatype constructor.
-.sp
-In general, if
-.sp
-.nf
-    Typemap = {(type0, disp0), ..., (type(n-1), disp(n-1)}
-.fi
-.sp
-then the lower bound of Typemap is defined to be
-.nf
-
-                  (min(j) disp(j)                          if no entry has
-    lb(Typemap) = (                                        basic type lb
-                  (min(j) {disp(j) such that type(j) = lb} otherwise
-
-.fi
-Similarly, the upper bound of Typemap is defined to be
-.nf
-
-                  (max(j) disp(j) + sizeof((type(j)) + e   if no entry has
-    ub(Typemap) = (                                        basic type ub
-                  (max(j) {disp(j) such that type(j) = ub} otherwise
-
-Then
-
-    extent(Typemap) = ub(Typemap) - lb(Typemap)
-.fi
-.sp
-If type(i) requires alignment to a byte address that is a multiple of k(i),
-then e is the least nonnegative increment needed to round extent(Typemap) to the next multiple of max(i) k(i).
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Type_get_extent
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Type_match_size.3in b/ompi/mpi/man/man3/MPI_Type_match_size.3in
deleted file mode 100644
index 2834ae8b83c..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_match_size.3in
+++ /dev/null
@@ -1,96 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_match_size 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fBMPI_Type_match_size\fP \- Returns an MPI datatype of a given type and size
-
-.SH SYNTAX
-.ft R
-
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_match_size(int \fItypeclass\fP, int \fIsize\fP,
-	MPI_Datatype *\fItype\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TYPE_MATCH_SIZE(\fITYPECLASS, SIZE, TYPE, IERROR\fP)
-	INTEGER	\fITYPECLASS, SIZE, TYPE, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Type_match_size(\fItypeclass\fP, \fIsize\fP, \fIdatatype\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fItypeclass\fP, \fIsize\fP
-	TYPE(MPI_Datatype), INTENT(OUT) :: \fIdatatype\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-typeclass
-Generic type specifier (integer).
-.ft R
-.TP 1i
-size
-Size, in bytes, of representation (integer).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-type
-Datatype with correct type and size (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-The function returns an MPI datatype matching a local variable of type
-(\fItypeclass\fP, \fIsize\fP). The returned type is a reference
-(handle) to a predefined named datatype, not a duplicate. This type
-cannot be freed.
-.sp
-The value of \fItypeclass\fR may be set to one of MPI_TYPECLASS_REAL,
-MPI_TYPECLASS_INTEGER, or MPI_TYPECLASS_COMPLEX, corresponding to the
-desired datatype.
-.sp
-MPI_type_match_size can be used to obtain a size-specific type that
-matches a Fortran numeric intrinsic type: first call MPI_Sizeof to
-compute the variable size, then call MPI_Type_match_size to find a
-suitable datatype. In C use the sizeof builtin instead of MPI_Sizeof.
-.sp
-It is erroneous to specify a size not supported by the compiler.
-
-.SH ERRORS
-.ft R
-Almost all MPI routines return an error value; C routines as
-the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-.sp
-See the MPI man page for a full list of MPI error codes.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Sizeof
-MPI_Type_get_extent
-
diff --git a/ompi/mpi/man/man3/MPI_Type_set_attr.3in b/ompi/mpi/man/man3/MPI_Type_set_attr.3in
deleted file mode 100644
index bd2911ba3ea..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_set_attr.3in
+++ /dev/null
@@ -1,87 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_set_attr 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Type_set_attr\fP \- Sets a key value/attribute pair to a data type.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_set_attr(MPI_Datatype \fItype\fP, int \fItype_keyval\fP,
-	void *\fIattribute_val\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TYPE_SET_ATTR(\fITYPE, TYPE_KEYVAL, ATTRIBUTE_VAL, IERROR\fP)
-	INTEGER	\fITYPE, TYPE_KEYVAL, IERROR\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fIATTRIBUTE_VAL\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Type_set_attr(\fIdatatype\fP, \fItype_keyval\fP, \fIattribute_val\fP, \fIierror\fP)
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	INTEGER, INTENT(IN) :: \fItype_keyval\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) :: \fIattribute_val\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-type
-Data type to which attribute will be attached (handle).
-
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-type_keyval
-Key value (integer).
-.TP 1i
-attribute_val
-Attribute value.
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-For the given data type, MPI_Type_set_attr sets the key value to the value of the specified attribute.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fIATTRIBUTE_VAL\fP argument only for Fortran 90. FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_ADDRESS_KIND \fIATTRIBUTE_VAL\fP
-.fi
-.sp
-where MPI_ADDRESS_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Type_get_attr
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Type_set_name.3in b/ompi/mpi/man/man3/MPI_Type_set_name.3in
deleted file mode 100644
index 9265458ca97..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_set_name.3in
+++ /dev/null
@@ -1,72 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_set_name 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Type_set_name\fP \- Sets the name of a data type.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_set_name(MPI_Datatype \fItype\fP, const char *\fItype_name\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TYPE_SET_NAME(\fITYPE, TYPE_NAME, IERROR\fP)
-	INTEGER	\fITYPE, IERROR\fP
-	CHARACTER*(*) \fITYPE_NAME\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Type_set_name(\fIdatatype\fP, \fItype_name\fP, \fIierror\fP)
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	CHARACTER(LEN=*), INTENT(IN) :: \fItype_name\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-type
-Data type for which the identifier is to be set (handle).
-
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-type_name
-The character string remembered as the name (string).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-
-.SH DESCRIPTION
-.ft R
-MPI_Type_set_name associates a printable identifier with an MPI data type.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Type_get_name
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Type_size.3in b/ompi/mpi/man/man3/MPI_Type_size.3in
deleted file mode 100644
index cc2c78c3761..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_size.3in
+++ /dev/null
@@ -1,83 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_size 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Type_size\fP, \fBMPI_Type_size_x\fP \- Returns the number of bytes occupied by entries in a data type.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_size(MPI_Datatype \fIdatatype\fP, int\fI *size\fP)
-int MPI_Type_size_x(MPI_Datatype \fIdatatype\fP, MPI_Count\fI *size\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TYPE_SIZE(\fIDATATYPE, SIZE, IERROR\fP)
-	INTEGER	\fIDATATYPE, SIZE, IERROR\fP
-MPI_TYPE_SIZE_X(\fIDATATYPE, SIZE, IERROR\fP)
-	INTEGER	\fIDATATYPE\fP
-        INTEGER(KIND=MPI_COUNT_KIND) \fISIZE\fP
-        INTEGER \fIIERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Type_size(\fIdatatype\fP, \fIsize\fP, \fIierror\fP)
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	INTEGER, INTENT(OUT) :: \fIsize\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-MPI_Type_size_x(\fIdatatype\fP, \fIsize\fP, \fIierror\fP)
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	INTEGER(KIND=MPI_COUNT_KIND), INTENT(OUT) :: \fIsize\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-datatype
-Datatype (handle).
-.sp
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-size
-Datatype size (integer).
-.sp
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Type_size returns the total size, in bytes, of the entries in the type signature associated with datatype; i.e., the total size of the data in a message that would be created with this datatype. Entries that occur multiple times in the datatype are counted with their multiplicity. For either function, if the \fIsize\fP parameter cannot express the value to be returned (e.g., if the parameter is too small to hold the output value), it is set to MPI_UNDEFINED.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for the \fISIZE\fP argument of MPI_Type_size_x only for Fortran 90. FORTRAN 77 users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_COUNT_KIND \fISIZE\fP
-.fi
-.sp
-where MPI_COUNT_KIND is a constant defined in mpif.h and gives the length of the declared integer in bytes.
-
diff --git a/ompi/mpi/man/man3/MPI_Type_size_x.3in b/ompi/mpi/man/man3/MPI_Type_size_x.3in
deleted file mode 100644
index 4e33322d255..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_size_x.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Type_size.3
diff --git a/ompi/mpi/man/man3/MPI_Type_struct.3in b/ompi/mpi/man/man3/MPI_Type_struct.3in
deleted file mode 100644
index de6b602dd8e..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_struct.3in
+++ /dev/null
@@ -1,117 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_struct 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Type_struct\fP \- Creates a \fIstruct\fP data type -- use of this routine is deprecated.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_struct(int \fIcount\fP, int\fI *array_of_blocklengths\fP,
-	MPI_Aint\fI *array_of_displacements\fP, MPI_Datatype\fI *array_of_types\fP,
-	MPI_Datatype\fI *newtype\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-INCLUDE 'mpif.h'
-MPI_TYPE_STRUCT(\fICOUNT, ARRAY_OF_BLOCKLENGTHS,
-		ARRAY_OF_DISPLACEMENTS, ARRAY_OF_TYPES,
-		NEWTYPE, IERROR\fP)
-	INTEGER	\fICOUNT, ARRAY_OF_BLOCKLENGTHS(*)\fP
-	INTEGER	\fIARRAY_OF_DISPLACEMENTS(*)\fP
-	INTEGER	\fIARRAY_OF_TYPES(*), NEWTYPE, IERROR\fP
-
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-count
-Number of blocks (integer)  also number of entries in arrays
-array_of_types,  array_of_displacements, and array_of_blocklengths.
-.TP 1i
-array_of_blocklengths
-Number of elements in each block (array).
-.TP 1i
-array_of_displacements
-Byte displacement of each block (array).
-.TP 1i
-array_of_types
-Type of elements in each block (array of handles to datatype objects).
-.sp
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-newtype
-New datatype (handle).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Note that use of this routine is \fIdeprecated\fP as of MPI-2. Use MPI_Type_create_struct instead.
-.sp
-MPI_Type_struct is the most general type constructor. It further generalizes MPI_Type_hindexed in that it allows each block to consist of replications of different datatypes.
-.sp
-\fBExample:\fP Let type1 have type map
-.nf
-
-    {(double, 0), (char, 8)}
-
-.fi
-with extent 16. Let B = (2, 1, 3), D = (0, 16, 26), and T = (MPI_FLOAT, type1, MPI_CHAR). Then a call to MPI_Type_struct(3, B, D, T, newtype) returns a datatype with type map
-.nf
-
-    {(float, 0), (float,4), (double, 16), (char, 24),
-    (char, 26), (char, 27), (char, 28)}
-
-.fi
-That is, two copies of MPI_FLOAT starting at 0, followed by one copy of type1 starting at 16, followed by three copies of MPI_CHAR, starting at 26. (We assume that a float occupies 4 bytes.)
-.sp
-For more information, see section 3.12.1 of the MPI-1.1 Standard.
-
-.SH NOTES
-If an upperbound is set explicitly by using the MPI datatype MPI_UB, the corresponding index must be positive.
-.sp
-The MPI-1 Standard originally made vague statements about padding and alignment; this was intended to allow the simple definition of structures that could be sent with a count greater than one. For example,
-.nf
-    struct {int a; char b;} foo;
-.fi
-may have
-.nf
-    sizeof(foo) = sizeof(int) + sizeof(char);
-.fi
-defining the extent of a datatype as including an epsilon, which would have allowed an implementation to make the extent an MPI datatype for this structure equal to 2*sizeof(int). However, since different systems might define different paddings, a clarification to the standard made epsilon zero. Thus, if you define a structure datatype and wish to send or receive multiple items, you should explicitly include an MPI_UB entry as the last member of the structure.  For example, the following code can be used for the structure foo:
-.nf
-
-    blen[0] = 1; indices[0] = 0; oldtypes[0] = MPI_INT;
-    blen[1] = 1; indices[1] = &foo.b - &foo; oldtypes[1] = MPI_CHAR;
-    blen[2] = 1; indices[2] = sizeof(foo); oldtypes[2] = MPI_UB;
-    MPI_Type_struct( 3, blen, indices, oldtypes, &newtype );
-
-.fi
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Type_create_struct
-.br
-MPI_Type_create_hindexed
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Type_ub.3in b/ompi/mpi/man/man3/MPI_Type_ub.3in
deleted file mode 100644
index aad0d6b227a..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_ub.3in
+++ /dev/null
@@ -1,94 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_ub 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Type_ub\fP \- Returns the upper bound of a datatype -- use of this routine is deprecated.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_ub(MPI_Datatype \fIdatatype\fP, MPI_Aint\fI *displacement\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-INCLUDE 'mpif.h'
-MPI_TYPE_UB(\fIDATATYPE, DISPLACEMENT, IERROR\fP)
-	INTEGER	\fIDATATYPE, DISPLACEMENT, IERROR\fP
-
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-datatype
-Datatype (handle).
-.sp
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-displacement
-Displacement of upper bound from origin, in bytes (integer).
-.sp
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Note that use of this routine is \fIdeprecated\fP as of MPI-2. Please use MPI_Type_get_extent instead.
-.sp
-MPI_Type_ub returns the upper bound of a data type. This will differ from zero if the type was constructed using MPI_UB. The upper bound will take into account any alignment considerations.
-.sp
-The "pseudo-datatypes," MPI_LB and MPI_UB, can be used, respectively, to mark the upper bound (or the lower bound) of a datatype. These pseudo-datatypes occupy no space (extent (MPI_LB) = extent (MPI_UB) =0. They do not affect the size or count of a datatype, and do not affect the context of a message created with this datatype. However, they do affect the definition of the extent of a datatype and, therefore, affect the outcome of a replication of this datatype by a datatype constructor.
-.sp
-In general, if
-.nf
-
-    Typemap = {(type(0), disp(0)), ..., (type(n-1), disp(n-1))}
-
-.fi
-then the lower bound of Typemap is defined to be
-.nf
-
-                  (min(j) disp(j)                          if no entry has
-    lb(Typemap) = (                                        basic type lb
-                  (min(j) {disp(j) such that type(j) = lb} otherwise
-
-.fi
-Similarly, the upper bound of Typemap is defined to be
-.nf
-
-                  (max(j) disp(j) + sizeof(type(j) = lb}   if no entry has
-    ub(Typemap) = (                                        basic type ub
-                  (max(j) {disp(j) such that type(j) = ub} otherwise
-
-.fi
-Then
-.nf
-
-    extent(Typemap) = ub(Typemap) - lb(Typemap)
-
-.fi
-If type(i) requires alignment to a byte address that is a multiple of k(i), then e is the least nonnegative increment needed to round extent(Typemap) to the next multiple of max(i) k(i).
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Type_get_extent
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Type_vector.3in b/ompi/mpi/man/man3/MPI_Type_vector.3in
deleted file mode 100644
index 85d8533833f..00000000000
--- a/ompi/mpi/man/man3/MPI_Type_vector.3in
+++ /dev/null
@@ -1,126 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Type_vector 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Type_vector\fP \- Creates a vector (strided) datatype.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Type_vector(int \fIcount\fP, int\fI blocklength\fP, int\fI stride\fP,
-	MPI_Datatype\fI oldtype\fP, MPI_Datatype\fI *newtype\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_TYPE_VECTOR(\fICOUNT, BLOCKLENGTH, STRIDE, OLDTYPE, NEWTYPE,
-		IERROR\fP)
-	INTEGER	\fICOUNT, BLOCKLENGTH, STRIDE, OLDTYPE\fP
-	INTEGER	\fINEWTYPE, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Type_vector(\fIcount\fP, \fIblocklength\fP, \fIstride\fP, \fIoldtype\fP, \fInewtype\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIcount\fP, \fIblocklength\fP, \fIstride\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIoldtype\fP
-	TYPE(MPI_Datatype), INTENT(OUT) :: \fInewtype\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-count
-Number of blocks (nonnegative integer).
-.TP 1i
-blocklength
-Number of elements in each block (nonnegative integer).
-.TP 1i
-stride
-Number of elements between start of each block (integer).
-.TP 1i
-oldtype
-Old datatype (handle).
-.sp
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-newtype
-New datatype (handle).
-.sp
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-The function MPI_Type_vector is a general constructor that allows replication of a datatype into locations that consist of equally spaced blocks. Each block is obtained by concatenating the same number of copies of the old datatype. The spacing between blocks is a multiple of the extent of the old datatype.
-.sp
-\fBExample 1:\fP Assume, again, that oldtype has type map {(double, 0), (char, 8)}, with extent 16. A call to  MPI_Type_vector(2, 3, 4, oldtype, newtype) will create the datatype with type map
-.nf
-    {(double, 0), (char, 8), (double, 16), (char, 24),
-    (double, 32), (char, 40),
-    (double, 64), (char, 72),
-    (double, 80), (char, 88), (double, 96), (char, 104)}
-.fi
-.sp
-That is, two blocks with three copies each of the old type, with a stride of 4 elements (4 x 16 bytes) between the blocks.
-.sp
-\fBExample 2:\fP  A call to  MPI_Type_vector(3, 1, -2, oldtype, newtype) will create the datatype
-.nf
-
-    {(double, 0), (char, 8), (double, -32), (char, -24),
-    (double, -64), (char, -56)}
-
-.fi
-In general, assume that oldtype has type map
-.nf
-
-    {(type(0), disp(0)), ..., (type(n-1), disp(n-1))},
-
-.fi
-with extent ex. Let bl be the blocklength. The newly created datatype has a type map with count x bl x  n entries:
-.nf
-
-    {(type(0), disp(0)), ..., (type(n-1), disp(n-1)),
-    (type(0), disp(0) + ex), ..., (type(n-1), disp(n-1) + ex), ...,
-    (type(0), disp(0) + (bl -1) * ex),...,
-    (type(n-1), disp(n-1) + (bl -1)* ex),
-    (type(0), disp(0) + stride * ex),..., (type(n-1),
-    disp(n-1) + stride * ex), ...,
-    (type(0), disp(0) + (stride + bl - 1) * ex), ...,
-    (type(n-1), disp(n-1) + (stride + bl -1) * ex), ...,
-    (type(0), disp(0) + stride * (count -1) * ex), ...,
-    (type(n-1), disp(n-1) + stride * (count -1) * ex), ...,
-    (type(0), disp(0) + (stride * (count -1) + bl -1) * ex), ...,
-    (type(n-1), disp(n-1) + (stride * (count -1) + bl -1) * ex)}
-
-.fi
-A call to MPI_Type_contiguous(count, oldtype, newtype) is equivalent to a call to MPI_Type_vector(count, 1, 1, oldtype, newtype), or to a call to MPI_Type_vector(1, count, n, oldtype, newtype), n arbitrary.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Type_create_hvector
-.br
-MPI_Type_hvector
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Unpack.3in b/ompi/mpi/man/man3/MPI_Unpack.3in
deleted file mode 100644
index 33ca9e85fb9..00000000000
--- a/ompi/mpi/man/man3/MPI_Unpack.3in
+++ /dev/null
@@ -1,111 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Unpack 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Unpack\fP \- Unpacks a datatype into contiguous memory.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Unpack(const void *\fIinbuf\fP, int\fI insize\fP, int\fI *position\fP,
-	void\fI *outbuf\fP, int\fI outcount\fP, MPI_Datatype\fI datatype\fP,
-	MPI_Comm\fI comm\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_UNPACK(\fIINBUF, INSIZE, POSITION, OUTBUF, OUTCOUNT,
-	DATATYPE, COMM, IERROR\fP)
-	<type>	\fIINBUF(*), OUTBUF(*)\fP
-	INTEGER	\fIINSIZE, POSITION, OUTCOUNT, DATATYPE,
-		COMM, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Unpack(\fIinbuf\fP, \fIinsize\fP, \fIposition\fP, \fIoutbuf\fP, \fIoutcount\fP, \fIdatatype\fP, \fIcomm\fP,
-		\fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIinbuf\fP
-	TYPE(*), DIMENSION(..) :: \fIoutbuf\fP
-	INTEGER, INTENT(IN) :: \fIinsize\fP, \fIoutcount\fP
-	INTEGER, INTENT(INOUT) :: \fIposition\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-inbuf
-Input buffer start (choice).
-.TP 1i
-insize
-Size of input buffer, in bytes (integer).
-.TP 1i
-outcount
-Number of items to be unpacked (integer).
-.TP 1i
-datatype
-Datatype of each output data item (handle).
-.TP 1i
-comm
-Communicator for packed message (handle).
-.sp
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-position
-Current position in bytes (integer).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-outbuf
-Output buffer start (choice).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Unpacks a message into the receive buffer specified by outbuf, outcount, datatype from the buffer space specified by inbuf and insize. The output buffer can be any communication buffer allowed in MPI_Recv. The input buffer is a contiguous storage area containing insize bytes, starting at address inbuf. The input value of position is the first location in the input buffer occupied by the packed message. \fIposition\fP is incremented by the size of the packed message, so that the output value of position is the first location in the input buffer after the locations occupied by the message that was unpacked. \fIcomm\fP is the communicator used to receive the packed message.
-
-.SH NOTES
-Note the difference between MPI_Recv and MPI_Unpack: In MPI_Recv, the \fIcount\fP argument specifies the maximum number of items that can be received. The actual number of items received is determined by the length of the incoming message. In MPI_Unpack, the count argument specifies the actual number of items that are to be unpacked; the "size" of the corresponding message is the increment in position. The reason for this change is that the "incoming message size" is not predetermined since the user decides how much to unpack; nor is it easy to determine the "message size" from the number of items to be unpacked.
-.sp
-To understand the behavior of pack and unpack, it is convenient to think of the data part of a message as being the sequence obtained by concatenating the successive values sent in that message. The pack operation stores this sequence in the buffer space, as if sending the message to that buffer. The unpack operation retrieves this sequence from buffer space, as if receiving a message from that buffer. (It is helpful to think of internal Fortran files or sscanf in C for a similar function.)
-.sp
-Several messages can be successively packed into one packing unit. This is effected by several successive related calls to MPI_Pack, where the first call provides position = 0, and each successive call inputs the value of position that was output by the previous call, and the same values for outbuf, outcount, and comm. This packing unit now contains the equivalent information that would have been stored in a message by one send call with a send buffer that is the "concatenation" of the individual send buffers.
-.sp
-A packing unit can be sent using type MPI_Packed. Any point-to-point or collective communication function can be used to move the sequence of bytes that forms the packing unit from one process to another. This packing unit can now be received using any receive operation, with any datatype: The type-matching rules are relaxed for messages sent with type MPI_Packed.
-.sp
-A message sent with any type (including MPI_Packed) can be received using the type MPI_Packed. Such a message can then be unpacked by calls to MPI_Unpack.
-.sp
-A packing unit (or a message created by a regular, "typed" send) can be unpacked into several successive messages. This is effected by several successive related calls to MPI_Unpack, where the first call provides position = 0, and each successive call inputs the value of position that was output by the previous call, and the same values for inbuf, insize, and comm.
-.sp
-The concatenation of two packing units is not necessarily a packing unit; nor is a substring of a packing unit necessarily a packing unit. Thus, one cannot concatenate two packing units and then unpack the result as one packing unit; nor can one unpack a substring of a packing unit as a separate packing unit. Each packing unit that was created by a related sequence of pack calls or by a regular send must be unpacked as a unit, by a sequence of related unpack calls.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-MPI_Pack
-.br
-MPI_Pack_size
-
diff --git a/ompi/mpi/man/man3/MPI_Unpack_external.3in b/ompi/mpi/man/man3/MPI_Unpack_external.3in
deleted file mode 100644
index 2697ea3e415..00000000000
--- a/ompi/mpi/man/man3/MPI_Unpack_external.3in
+++ /dev/null
@@ -1,182 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Unpack_external 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-
-.SH NAME
-\fBMPI_Unpack_external\fP \- Reads data from a portable format
-
-.SH SYNTAX
-.ft R
-
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Unpack_external(const char \fIdatarep\fP[], const void *\fIinbuf\fP,
-	MPI_Aint \fIinsize\fP, MPI_Aint *\fIposition\fP,
-	void *\fIoutbuf\fP, int \fIoutcount\fP,
-	MPI_Datatype \fIdatatype\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_UNPACK_EXTERNAL(\fIDATAREP, INBUF, INSIZE, POSITION,
-	OUTBUF, OUTCOUNT, DATATYPE, IERROR\fP)
-
-	INTEGER		\fIOUTCOUNT, DATATYPE, IERROR\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fIINSIZE, POSITION\fP
-	CHARACTER*(*)	\fIDATAREP\fP
-	<type>		\fIINBUF(*), OUTBUF(*)\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Unpack_external(\fIdatarep\fP, \fIinbuf\fP, \fIinsize\fP, \fIposition\fP, \fIoutbuf\fP, \fIoutcount\fP,
-		\fIdatatype\fP, \fIierror\fP)
-	CHARACTER(LEN=*), INTENT(IN) :: \fIdatarep\fP
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIinbuf\fP
-	TYPE(*), DIMENSION(..) :: \fIoutbuf\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) :: \fIinsize\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(INOUT) :: \fIposition\fP
-	INTEGER, INTENT(IN) :: \fIoutcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-datarep
-Data Representation (string).
-.ft R
-.TP 1i
-inbuf
-Input buffer start (choice).
-.TP 1i
-insize
-Size of input buffer, in bytes (integer).
-.TP 1i
-outcount
-Number of items to be unpacked (integer).
-.TP 1i
-datatype
-Datatype of each output data item (handle).
-
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-position
-Current position in buffer, in bytes (integer).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-outbuf
-Output buffer start (choice).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Unpack_external unpacks data from the external32 format, a
-universal data representation defined by the MPI Forum. This format is
-useful for exchanging data between MPI implementations, or when
-writing data to a file.
-.sp
-The input buffer is a contiguous storage area pointed to by
-\fIinbuf\fP containing \fIinsize\fP bytes. The output buffer can be
-any communication buffer allowed in MPI_Recv, and is specified by
-\fIoutbuf\fP, \fIoutcount\fP, and \fIdatatype\fP.
-.sp
-The input value of \fIposition\fP is the first position in \fIinbuf\fP
-to be read for unpacking (measured in bytes, not elements, relative to
-the start of the buffer). When the function returns, \fIposition\fP is
-incremented by the size of the packed message, so that it points to
-the first location in \fIinbuf\fP following the message that was
-unpacked. This way it may be used as input to a subsequent call to
-MPI_Unpack_external.
-
-.SH NOTES
-.ft R
-Note the difference between MPI_Recv and MPI_Unpack_external: In
-MPI_Recv, the \fIcount\fP argument specifies the maximum number of
-items that can be received. In MPI_Unpack_external, the \fIoutcount\fP
-argument specifies the actual number of items that are to be
-unpacked. With a regular receive operation, the incoming message size
-determines the number of components that will be received. With
-MPI_Unpack_external, it is up to the user to specify how many
-components to unpack, since the user may wish to unpack the received
-message multiple times into various buffers.
-.sp
-To understand the behavior of pack and unpack, it is convenient to
-think of the data part of a message as being the sequence obtained by
-concatenating the successive values sent in that message. The pack
-operation stores this sequence in the buffer space, as if sending the
-message to that buffer. The unpack operation retrieves this sequence
-from buffer space, as if receiving a message from that buffer. (It is
-helpful to think of internal Fortran files or sscanf in C for a
-similar function.)
-.sp
-Several messages can be successively packed into one packing
-unit. This is effected by several successive related calls to
-MPI_Pack_external, where the first call provides \fIposition\fP=0,
-and each successive call inputs the value of \fIposition\fP that was
-output by the previous call, along with the same values for
-\fIoutbuf\fP and \fIoutcount\fP. This packing unit now contains the
-equivalent information that would have been stored in a message by one
-send call with a send buffer that is the "concatenation" of the
-individual send buffers.
-.sp
-A packing unit can be sent using type MPI_BYTE. Any point-to-point
-or collective communication function can be used to move the sequence
-of bytes that forms the packing unit from one process to another. This
-packing unit can now be received using any receive operation, with any
-datatype: The type-matching rules are relaxed for messages sent with
-type MPI_BYTE.
-.sp
-A packing unit can be unpacked into several successive messages. This
-is effected by several successive related calls to
-MPI_Unpack_external, where the first call provides \fIposition\fP=0,
-and each successive call inputs the value of position that was output
-by the previous call, and the same values for \fIinbuf\fP and
-\fIinsize\fP.
-.sp
-The concatenation of two packing units is not necessarily a packing
-unit; nor is a substring of a packing unit necessarily a packing
-unit. Thus, one cannot concatenate two packing units and then unpack
-the result as one packing unit; nor can one unpack a substring of a
-packing unit as a separate packing unit. Each packing unit that was
-created by a related sequence of pack calls must be unpacked as a unit
-by a sequence of related unpack calls.
-
-.SH ERRORS
-.ft R
-Almost all MPI routines return an error value; C routines as
-the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-.sp
-See the MPI man page for a full list of MPI error codes.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Pack_external
-MPI_Pack_external_size
-MPI_Recv
-sscanf(3C)
-
diff --git a/ompi/mpi/man/man3/MPI_Unpublish_name.3in b/ompi/mpi/man/man3/MPI_Unpublish_name.3in
deleted file mode 100644
index ca70aba504c..00000000000
--- a/ompi/mpi/man/man3/MPI_Unpublish_name.3in
+++ /dev/null
@@ -1,131 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2007-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Unpublish_name 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-
-.SH NAME
-.nf
-\fBMPI_Unpublish_name\fP \- Unpublishes a service name
-
-.fi
-.SH SYNTAX
-.ft R
-
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Unpublish_name(const char *\fIservice_name\fP, MPI_Info \fIinfo\fP,
-	const char *\fIport_name\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_UNPUBLISH_NAME(\fISERVICE_NAME, INFO, PORT_NAME, IERROR\fP)
-	CHARACTER*(*)	\fISERVICE_NAME, PORT_NAME\fP
-	INTEGER		\fIINFO, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Unpublish_name(\fIservice_name\fP, \fIinfo\fP, \fIport_name\fP, \fIierror\fP)
-	CHARACTER(LEN=*), INTENT(IN) :: \fIservice_name\fP, \fIport_name\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1.4i
-service_name
-A service name (string).
-.TP 1.4i
-info
-Options to the name service functions (handle).
-.ft R
-.TP 1.4i
-port_name
-A port name (string).
-
-.SH OUTPUT PARAMETER
-.TP 1.4i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-This routine removes the pair (\fIservice_name, port_name\fP) so that
-applications may no longer retrieve \fIport_name\fP by calling
-MPI_Lookup_name. It is an error to unpublish a \fIservice_name\fP
-that was not published via MPI_Publish_name. Both the \fIservice_name\fP
-and \fIport_name\fP arguments to MPI_Unpublish_name must be identical
-to the arguments to the previous call to MPI_Publish_name.
-
-.SH INFO ARGUMENTS
-The following keys for \fIinfo\fP are recognized:
-.sp
-.sp
-.nf
-Key                   Type      Description
----                   ----      -----------
-
-ompi_global_scope     bool      If set to true, unpublish the name from
-                                the global scope.  Unpublish from the local
-                                scope otherwise.  See the NAME SCOPE
-                                section for more details.
-
-.fi
-
-.sp
-\fIbool\fP info keys are actually strings but are evaluated as
-follows: if the string value is a number, it is converted to an
-integer and cast to a boolean (meaning that zero integers are false
-and non-zero values are true).  If the string value is
-(case-insensitive) "yes" or "true", the boolean is true.  If the
-string value is (case-insensitive) "no" or "false", the boolean is
-false.  All other string values are unrecognized, and therefore false.
-.PP
-If no info key is provided, the function will first check to see if a
-global server has been specified and is available. If so, then the
-unpublish function will default to global scope first, followed by local. Otherwise,
-the data will default to unpublish with local scope.
-
-.SH NAME SCOPE
-Open MPI supports two name scopes: \fIglobal\fP and \fIlocal\fP. Local scope
-values are placed in a data store located on the mpirun of the calling
-process' job, while global scope values reside on a central server. Calls
-to MPI_Unpublish_name must correctly specify the scope to be used in
-finding the value to be removed. The function will return an error if
-the specified service name is not found on the indicated location.
-.sp
-For a more detailed description of scoping rules, please see the MPI_Publish_name
-man page.
-
-.SH ERRORS
-.ft R
-Almost all MPI routines return an error value; C routines as
-the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does not
-guarantee that an MPI program can continue past an error.
-.sp
-See the MPI man page for a full list of MPI error codes.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Publish_name
-MPI_Lookup_name
-MPI_Open_port
-
-
diff --git a/ompi/mpi/man/man3/MPI_Wait.3in b/ompi/mpi/man/man3/MPI_Wait.3in
deleted file mode 100644
index 0d1434e00ff..00000000000
--- a/ompi/mpi/man/man3/MPI_Wait.3in
+++ /dev/null
@@ -1,122 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2007-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Wait 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Wait\fP \- Waits for an MPI send or receive to complete.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Wait(MPI_Request *\fIrequest\fP, MPI_Status\fI *status\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WAIT(\fIREQUEST, STATUS, IERROR\fP)
-	INTEGER	\fIREQUEST, STATUS(MPI_STATUS_SIZE), IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Wait(\fIrequest\fP, \fIstatus\fP, \fIierror\fP)
-	TYPE(MPI_Request), INTENT(INOUT) :: \fIrequest\fP
-	TYPE(MPI_Status) :: \fIstatus\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-request
-Request (handle).
-.sp
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-status
-Status object (status).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-A call to MPI_Wait returns when the operation identified by request is complete. If the communication object associated with this request was created by a nonblocking send or receive call, then the object is deallocated by the call to MPI_Wait and the request handle is set to MPI_REQUEST_NULL.
-.sp
-The call returns, in status, information on the completed operation. The content of the status object for a receive operation can be accessed as described in Section 3.2.5 of the MPI-1 Standard, "Return Status." The status object for a send operation may be queried by a call to MPI_Test_cancelled (see Section 3.8 of the MPI-1 Standard, "Probe and Cancel").
-.sp
-If your application does not need to examine the \fIstatus\fP field, you can save resources by using the predefined constant MPI_STATUS_IGNORE as a special value for the \fIstatus\fP argument.
-.sp
-One is allowed to call MPI_Wait with a null or inactive request argument. In this case the operation returns immediately with empty status.
-
-.SH NOTES
-Successful return of MPI_Wait after an MPI_Ibsend implies that the user send buffer can be reused  i.e., data has been sent out or copied into a buffer attached with MPI_Buffer_attach. Note that, at this point, we can no longer cancel the send (for more information, see Section 3.8 of the MPI-1 Standard, "Probe and Cancel"). If a matching receive is never posted, then the buffer cannot be freed. This runs somewhat counter to the stated goal of MPI_Cancel (always being able to free program space that was committed to the communication subsystem).
-.sp
-Example: Simple usage of nonblocking operations and  MPI_Wait.
-.sp
-.nf
-    CALL MPI_COMM_RANK(comm, rank, ierr)
-    IF(rank.EQ.0) THEN
-        CALL MPI_ISEND(a(1), 10, MPI_REAL, 1, tag, comm, request, ierr)
-        **** do some computation ****
-        CALL MPI_WAIT(request, status, ierr)
-    ELSE
-        CALL MPI_IRECV(a(1), 15, MPI_REAL, 0, tag, comm, request, ierr)
-        **** do some computation ****
-        CALL MPI_WAIT(request, status, ierr)
-    END IF
-
-.fi
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler, MPI_File_set_errhandler, or
-MPI_Win_set_errhandler (depending on the type of MPI handle that
-generated the request); the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does
-not guarantee that an MPI program can continue past an error.
-.sp
-Note that per MPI-1 section 3.2.5, MPI errors on requests passed
-to MPI_WAIT do not set the status.MPI_ERROR field in the returned
-status.  The error code is passed to the back-end error handler
-and may be passed back to the caller through the return value of
-MPI_WAIT if the back-end error handler returns it.  The
-pre-defined MPI error handler MPI_ERRORS_RETURN exhibits this
-behavior, for example.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Comm_set_errhandler
-.br
-MPI_File_set_errhandler
-.br
-MPI_Test
-.br
-MPI_Testall
-.br
-MPI_Testany
-.br
-MPI_Testsome
-.br
-MPI_Waitall
-.br
-MPI_Waitany
-.br
-MPI_Waitsome
-.br
-MPI_Win_set_errhandler
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Waitall.3in b/ompi/mpi/man/man3/MPI_Waitall.3in
deleted file mode 100644
index bb0bcd35eb5..00000000000
--- a/ompi/mpi/man/man3/MPI_Waitall.3in
+++ /dev/null
@@ -1,112 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2011 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Waitall 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Waitall\fP \- Waits for all given communications to complete.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Waitall(int \fIcount\fP, MPI_Request\fI array_of_requests[]\fP,
-	MPI_Status \fI*array_of_statuses\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WAITALL(\fICOUNT, ARRAY_OF_REQUESTS, ARRAY_OF_STATUSES, IERROR\fP)
-	INTEGER	\fICOUNT, ARRAY_OF_REQUESTS(*)\fP
-	INTEGER	\fIARRAY_OF_STATUSES(MPI_STATUS_SIZE,*), IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Waitall(\fIcount\fP, \fIarray_of_requests\fP, \fIarray_of_statuses\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Request), INTENT(INOUT) :: \fIarray_of_requests(count)\fP
-	TYPE(MPI_Status) :: \fIarray_of_statuses(*)\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-count
-Lists length (integer).
-.TP 1i
-array_of_requests
-Array of requests (array of handles).
-.sp
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-array_of_statuses
-Array of status objects (array of status).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Blocks until all communication operations associated with active handles in the list complete, and returns the status of all these operations (this includes the case where no handle in the list is active). Both arrays have the same number of valid entries. The ith entry in array_of_statuses is set to the return status of the ith operation. Requests that were created by nonblocking communication operations are deallocated, and the corresponding handles in the array are set to MPI_REQUEST_NULL. The list may contain null or inactive handles. The call sets to empty the status of each such entry.
-.sp
-The error-free execution of MPI_Waitall(count, array_of_requests, array_of_statuses) has the same effect as the execution of  MPI_Wait(&array_of_request[i], &array_of_statuses[i]), for i=0,...,count-1, in some arbitrary order. MPI_Waitall with an array of length 1 is equivalent to MPI_Wait.
-.sp
-When one or more of the communications completed by a call to MPI_Waitall fail, it is desirable to return specific information on each communication. The function MPI_Waitall will return in such case the error code MPI_ERR_IN_STATUS and will set the error field of each status to a specific error code. This code will be MPI_SUCCESS if the specific communication completed; it will be another specific error code if it failed; or it can be MPI_ERR_PENDING if it has neither failed nor completed. The function MPI_Waitall will return MPI_SUCCESS if no request had an error, or will return another error code if it failed for other reasons (such as invalid arguments). In such cases, it will not update the error fields of the statuses.
-.sp
-If your application does not need to examine the \fIarray_of_statuses\fP field, you can save resources by using the predefined constant MPI_STATUSES_IGNORE can be used as a special value for the \fIarray_of_statuses\fP argument.
-
-.SH ERRORS
-For each invocation of MPI_Waitall, if one or more requests generate
-an MPI error, only the \fIfirst\fP MPI request that caused an
-error will be passed to its corresponding error handler.  No other
-error handlers will be invoked (even if multiple requests generated
-errors).  However, \fIall\fP requests that generate an error
-will have a relevant error code set in the corresponding
-status.MPI_ERROR field (unless MPI_STATUSES_IGNORE was used).
-.sp
-The default error handler aborts the MPI job, except for I/O function
-errors. The error handler may be changed with MPI_Comm_set_errhandler,
-MPI_File_set_errhandler, or MPI_Win_set_errhandler (depending on the
-type of MPI handle that generated the MPI request); the predefined
-error handler MPI_ERRORS_RETURN may be used to cause error values to
-be returned. Note that MPI does not guarantee that an MPI program can
-continue past an error.
-.sp
-If the invoked error handler allows MPI_Waitall to return to the
-caller, the value MPI_ERR_IN_STATUS will be returned in the C and
-Fortran bindings.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Comm_set_errhandler
-.br
-MPI_File_set_errhandler
-.br
-MPI_Test
-.br
-MPI_Testall
-.br
-MPI_Testany
-.br
-MPI_Testsome
-.br
-MPI_Wait
-.br
-MPI_Waitany
-.br
-MPI_Waitsome
-.br
-MPI_Win_set_errhandler
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Waitany.3in b/ompi/mpi/man/man3/MPI_Waitany.3in
deleted file mode 100644
index f30f9988135..00000000000
--- a/ompi/mpi/man/man3/MPI_Waitany.3in
+++ /dev/null
@@ -1,142 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Waitany 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Waitany\fP \- Waits for any specified send or receive to complete.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Waitany(int \fIcount\fP, MPI_Request\fI array_of_requests[]\fP,
-	int \fI*index\fP, MPI_Status\fI *status\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WAITANY(\fICOUNT, ARRAY_OF_REQUESTS, INDEX, STATUS, IERROR\fP)
-	INTEGER	\fICOUNT, ARRAY_OF_REQUESTS(*), INDEX\fP
-	INTEGER	\fISTATUS(MPI_STATUS_SIZE), IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Waitany(\fIcount\fP, \fIarray_of_requests\fP, \fIindex\fP, \fIstatus\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Request), INTENT(INOUT) :: \fIarray_of_requests(count)\fP
-	INTEGER, INTENT(OUT) :: \fIindex\fP
-	TYPE(MPI_Status) :: \fIstatus\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-count
-List length (integer).
-.TP 1i
-array_of_requests
-Array of requests (array of handles).
-.sp
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-index
-Index of handle for operation that completed (integer). In the range 0 to
-count-1.  In Fortran, the range is 1 to count.
-.TP 1i
-status
-Status object (status).
-.sp
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-A call to MPI_Waitany can be used to wait for the completion of one out of several requests.
-.sp
-The array_of_requests list may contain null or inactive handles. If the list contains no active handles (list has length zero or all entries are null or inactive), then the call returns immediately with index = MPI_UNDEFINED, and an empty status.
-.sp
-The execution of MPI_Waitany(count, array_of_requests, index, status) has the same effect as the execution of MPI_Wait(&array_of_requests[i], status), where i is the value returned by index (unless the value of index is MPI_UNDEFINED). MPI_Waitany with an array containing one active entry is equivalent to MPI_Wait.
-.sp
-If your application does not need to examine the \fIstatus\fP field, you can save resources by using the predefined constant MPI_STATUS_IGNORE as a special value for the \fIstatus\fP argument.
-.sp
-\fBExample:\fR Client-server code (starvation can occur).
-.sp
-.nf
-    CALL MPI_COMM_SIZE(comm, size, ierr)
-    CALL MPI_COMM_RANK(comm, rank, ierr)
-    IF(rank .GT 0) THEN         ! client code
-        DO WHILE(.TRUE.)
-           CALL MPI_ISEND(a, n, MPI_REAL, 0, tag, comm, request, ierr)
-           CALL MPI_WAIT(request, status, ierr)
-        END DO
-    ELSE         ! rank=0 -- server code
-           DO i=1, size-1
-              CALL MPI_IRECV(a(1,i), n, MPI_REAL, i tag,
-                       comm, request_list(i), ierr)
-           END DO
-           DO WHILE(.TRUE.)
-              CALL MPI_WAITANY(size-1, request_list, index, status, ierr)
-              CALL DO_SERVICE(a(1,index))  ! handle one message
-              CALL MPI_IRECV(a(1, index), n, MPI_REAL, index, tag,
-                        comm, request_list(index), ierr)
-           END DO
-    END IF
-.fi
-.sp
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler, MPI_File_set_errhandler, or
-MPI_Win_set_errhandler (depending on the type of MPI handle that
-generated the request); the predefined error handler MPI_ERRORS_RETURN
-may be used to cause error values to be returned. Note that MPI does
-not guarantee that an MPI program can continue past an error.
-.sp
-Note that per MPI-1 section 3.2.5, MPI errors on requests passed
-to MPI_WAITANY do not set the status.MPI_ERROR field in the returned
-status.  The error code is passed to the back-end error handler and
-may be passed back to the caller through the return value of
-MPI_WAITANY if the back-end error handler returns it.  The pre-defined
-MPI error handler MPI_ERRORS_RETURN exhibits this behavior, for
-example.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Comm_set_errhandler
-.br
-MPI_File_set_errhandler
-.br
-MPI_Test
-.br
-MPI_Testall
-.br
-MPI_Testany
-.br
-MPI_Testsome
-.br
-MPI_Wait
-.br
-MPI_Waitall
-.br
-MPI_Waitsome
-.br
-MPI_Win_set_errhandler
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Waitsome.3in b/ompi/mpi/man/man3/MPI_Waitsome.3in
deleted file mode 100644
index ca79b2b479c..00000000000
--- a/ompi/mpi/man/man3/MPI_Waitsome.3in
+++ /dev/null
@@ -1,155 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2011 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Waitsome 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Waitsome\fP \- Waits for some given communications to complete.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Waitsome(int \fIincount\fP, MPI_Request \fIarray_of_requests[]\fP,
-	int\fI *outcount\fP, int\fI array_of_indices[]\fP,
-	MPI_Status \fIarray_of_statuses[]\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WAITSOME(\fIINCOUNT, ARRAY_OF_REQUESTS, OUTCOUNT,
-		ARRAY_OF_INDICES, ARRAY_OF_STATUSES, IERROR\fP)
-	INTEGER	\fIINCOUNT, ARRAY_OF_REQUESTS(*), OUTCOUNT\fP
-	INTEGER	\fIARRAY_OF_INDICES(*)\fP
-	INTEGER	\fIARRAY_OF_STATUSES(MPI_STATUS_SIZE*)\fP
-	INTEGER	\fIIERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Waitsome(\fIincount\fP, \fIarray_of_requests\fP, \fIoutcount\fP, \fIarray_of_indices\fP,
-		\fIarray_of_statuses\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIincount\fP
-	TYPE(MPI_Request), INTENT(INOUT) :: \fIarray_of_requests(incount)\fP
-	INTEGER, INTENT(OUT) :: \fIoutcount\fP, \fIarray_of_indices(*)\fP
-	TYPE(MPI_Status) :: \fIarray_of_statuses(*)\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-incount
-Length of array_of_requests (integer).
-.TP 1i
-array_of_requests
-Array of requests (array of handles).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-outcount
-Number of completed requests (integer).
-.TP 1i
-array_of_indices
-Array of indices of operations that completed (array of integers).
-.TP 1i
-array_of_statuses
-Array of status objects for operations that completed (array of status).
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Waits until at least one of the operations associated with active handles in the list have completed. Returns in outcount the number of requests from the list array_of_requests that have completed. Returns in the first outcount locations of the array array_of_indices the indices of these operations (index within the array array_of_requests; the array is indexed from 0 in C and from 1 in Fortran). Returns in the first outcount locations of the array array_of_status the status for these completed operations. If a request that completed was allocated by a nonblocking communication call, then it is deallocated, and the associated handle is set to MPI_REQUEST_NULL.
-.sp
-If the list contains no active handles, then the call returns immediately with outcount = MPI_UNDEFINED.
-.sp
-When one or more of the communications completed by MPI_Waitsome fails, then it is desirable to return specific information on each communication. The arguments outcount, array_of_indices, and array_of_statuses will be adjusted to indicate completion of all communications that have succeeded or failed. The call will return the error code MPI_ERR_IN_STATUS and the error field of each status returned will be set to indicate success or to indicate the specific error that occurred. The call will return MPI_SUCCESS if no request resulted in an error, and will return another error code if it failed for other reasons (such as invalid arguments). In such cases, it will not update the error fields of the statuses.
-.sp
-If your application does not need to examine the \fIarray_of_statuses\fP field, you can save resources by using the predefined constant MPI_STATUSES_IGNORE can be used as a special value for the \fIarray_of_statuses\fP argument.
-.sp
-\fBExample:\fR Same code as the example in the MPI_Waitany man page, but using MPI_Waitsome.
-.sp
-.nf
-    CALL MPI_COMM_SIZE(comm, size, ierr)
-    CALL MPI_COMM_RANK(comm, rank, ierr)
-    IF(rank .GT. 0) THEN         ! client code
-        DO WHILE(.TRUE.)
-           CALL MPI_ISEND(a, n, MPI_REAL, 0, tag, comm, request, ierr)
-           CALL MPI_WAIT(request, status, ierr)
-        END DO
-    ELSE         ! rank=0 -- server code
-        DO i=1, size-1
-           CALL MPI_IRECV(a(1,i), n, MPI_REAL, i, tag,
-                          comm, requests(i), ierr)
-        END DO
-        DO WHILE(.TRUE.)
-           CALL MPI_WAITSOME(size, request_list, numdone,
-                            indices, statuses, ierr)
-           DO i=1, numdone
-              CALL DO_SERVICE(a(1, indices(i)))
-              CALL MPI_IRECV(a(1, indices(i)), n, MPI_REAL, 0, tag,
-                           comm, requests(indices(i)), ierr)
-           END DO
-        END DO
-    END IF
-.fi
-.sp
-.SH NOTES
-.ft R
-The array of indices are in the range 0 to incount-1 for C and in the range 1 to incount for Fortran.
-
-.SH ERRORS
-For each invocation of MPI_Waitsome, if one or more requests generate
-an MPI error, only the \fIfirst\fP MPI request that caused an
-error will be passed to its corresponding error handler.  No other
-error handlers will be invoked (even if multiple requests generated
-errors).  However, \fIall\fP requests that generate an error
-will have a relevant error code set in the corresponding
-status.MPI_ERROR field (unless MPI_STATUSES_IGNORE was used).
-.sp
-The default error handler aborts the MPI job, except for I/O function
-errors. The error handler may be changed with MPI_Comm_set_errhandler,
-MPI_File_set_errhandler, or MPI_Win_set_errhandler (depending on the
-type of MPI handle that generated the MPI request); the predefined
-error handler MPI_ERRORS_RETURN may be used to cause error values to
-be returned. Note that MPI does not guarantee that an MPI program can
-continue past an error.
-.sp
-If the invoked error handler allows MPI_Waitsome to return to the
-caller, the value MPI_ERR_IN_STATUS will be returned in the C and
-Fortran bindings.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Comm_set_errhandler
-.br
-MPI_File_set_errhandler
-.br
-MPI_Test
-.br
-MPI_Testall
-.br
-MPI_Testany
-.br
-MPI_Testsome
-.br
-MPI_Wait
-.br
-MPI_Waitall
-.br
-MPI_Waitany
-.br
-MPI_Win_set_errhandler
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Win_allocate.3in b/ompi/mpi/man/man3/MPI_Win_allocate.3in
deleted file mode 100644
index 6f90f807bdc..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_allocate.3in
+++ /dev/null
@@ -1,121 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2015      Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2007-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" $COPYRIGHT$
-.TH MPI_Win_allocate 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_allocate\fP \- One-sided MPI call that allocates memory and
-returns a window object for RMA operations.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Win_allocate (MPI_Aint \fIsize\fP, int \fIdisp_unit\fP, MPI_Info \fIinfo\fP,
-                      MPI_Comm \fIcomm\fP, void *\fIbaseptr\fP, MPI_Win *\fIwin\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_ALLOCATE(\fISIZE, DISP_UNIT, INFO, COMM, BASEPTR, WIN, IERROR\fP)
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fISIZE, BASEPTR\fP
-	INTEGER \fIDISP_UNIT, INFO, COMM, WIN, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_allocate(\fIsize\fP, \fIdisp_unit\fP, \fIinfo\fP, \fIcomm\fP, \fIbaseptr\fP, \fIwin\fP, \fIierror\fP)
-	USE, INTRINSIC :: ISO_C_BINDING, ONLY : C_PTR
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) :: \fIsize\fP
-	INTEGER, INTENT(IN) :: \fIdisp_unit\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(C_PTR), INTENT(OUT) :: \fIbaseptr\fP
-	TYPE(MPI_Win), INTENT(OUT) :: \fIwin\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-size
-Size of window in bytes (nonnegative integer).
-.TP 1i
-disp_unit
-Local unit size for displacements, in bytes (positive integer).
-.TP 1i
-info
-Info argument (handle).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-baseptr
-Initial address of window.
-.TP 1i
-win
-Window object returned by the call (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-\fBMPI_Win_allocate\fP is a collective call executed by all processes
-in the group of \fIcomm\fP. On each process, it allocates memory of at
-least \fIsize\fP bytes, returns a pointer to it, and returns a window
-object that can be used by all processes in \fIcomm\fP to perform RMA
-operations. The returned memory consists of \fIsize\fP bytes local to
-each process, starting at address \fIbaseptr\fP and is associated with
-the window as if the user called \fBMPI_Win_create\fP on existing
-memory. The \fIsize\fP argument may be different at each process and
-\fIsize\fP = 0 is valid; however, a library might allocate and expose
-more memory in order to create a fast, globally symmetric
-allocation. The discussion of and rationales for \fBMPI_Alloc_mem\fP and
-\fBMPI_Free_mem\fP in MPI-3.1 \[char167] 8.2 also apply to
-\fBMPI_Win_allocate\fP; in particular, see the rationale in MPI-3.1
-\[char167] 8.2 for an explanation of the type used for \fIbaseptr\fP.
-.sp
-The displacement unit argument is provided to facilitate address
-arithmetic in RMA operations: the target displacement argument of an
-RMA operation is scaled by the factor \fIdisp_unit\fP specified by the
-target process, at window creation.
-.sp
-For supported info keys see \fBMPI_Win_create\fI.
-.sp
-
-.SH NOTES
-Common choices for \fIdisp_unit\fP are 1 (no scaling), and (in C
-syntax) \fIsizeof(type)\fP, for a window that consists of an array of
-elements of type \fItype\fP. The later choice will allow one to use
-array indices in RMA calls, and have those scaled correctly to byte
-displacements, even in a heterogeneous environment.
-.sp
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler
-MPI_ERRORS_RETURN may be used to cause error values to be
-returned. Note that MPI does not guarantee that an MPI program can
-continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Alloc_mem
-MPI_Free_mem
-MPI_Win_create
-MPI_Win_allocate_shared
diff --git a/ompi/mpi/man/man3/MPI_Win_allocate_shared.3in b/ompi/mpi/man/man3/MPI_Win_allocate_shared.3in
deleted file mode 100644
index 8c995fb186d..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_allocate_shared.3in
+++ /dev/null
@@ -1,146 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2015-2016 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2007-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" $COPYRIGHT$
-.TH MPI_Win_allocate_shared 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_allocate_shared\fP \- One-sided MPI call that allocates
-shared memory and returns a window object for RMA operations.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Win_allocate_shared (MPI_Aint \fIsize\fP, int \fIdisp_unit\fP, MPI_Info \fIinfo\fP,
-                             MPI_Comm \fIcomm\fP, void *\fIbaseptr\fP, MPI_Win *\fIwin\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_ALLOCATE_SHARED(\fISIZE, DISP_UNIT, INFO, COMM, BASEPTR, WIN, IERROR\fP)
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fISIZE, BASEPTR\fP
-	INTEGER \fIDISP_UNIT, INFO, COMM, WIN, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_allocate_shared(\fIsize\fP, \fIdisp_unit\fP, \fIinfo\fP, \fIcomm\fP, \fIbaseptr\fP, \fIwin\fP, \fIierror\fP)
-	USE, INTRINSIC :: ISO_C_BINDING, ONLY : C_PTR
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) :: \fIsize\fP
-	INTEGER, INTENT(IN) :: \fIdisp_unit\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(C_PTR), INTENT(OUT) :: \fIbaseptr\fP
-	TYPE(MPI_Win), INTENT(OUT) :: \fIwin\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-size
-Size of window in bytes (nonnegative integer).
-.TP 1i
-disp_unit
-Local unit size for displacements, in bytes (positive integer).
-.TP 1i
-info
-Info argument (handle).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-baseptr
-Initial address of window.
-.TP 1i
-win
-Window object returned by the call (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-\fBMPI_Win_allocate_shared\fP is a collective call executed by all
-processes in the group of \fIcomm\fP. On each process, it allocates
-memory of at least \fIsize\fP bytes that is shared among all processes
-in \fIcomm\fP, and returns a pointer to the locally allocated segment
-in \fIbaseptr\fP that can be used for load/store accesses on the
-calling process. The locally allocated memory can be the target of
-load/store accesses by remote processes; the base pointers for other
-processes can be queried using the function
-\fBMPI_Win_shared_query\fP. The call also returns a window object that
-can be used by all processes in \fIcomm\fP to perform RMA
-operations. The \fIsize\fP argument may be different at each process
-and \fIsize\fP = 0 is valid. It is the user's responsibility to ensure
-that the communicator \fIcomm\fP represents a group of processes that
-can create a shared memory segment that can be accessed by all
-processes in the group. The discussions of rationales for
-\fBMPI_Alloc_mem\fP and \fBMPI_Free_mem\fP in MPI-3.1 \[char167] 8.2
-also apply to \fBMPI_Win_allocate_shared\fP; in particular, see the
-rationale in MPI-3.1 \[char167] 8.2 for an explanation of the type
-used for \fIbaseptr\fP. The allocated memory is contiguous across
-process ranks unless the info key \fIalloc_shared_noncontig\fP is
-specified. Contiguous across process ranks means that the first
-address in the memory segment of process i is consecutive with the
-last address in the memory segment of process i - 1. This may enable
-the user to calculate remote address offsets with local information
-only.
-.sp
-The following info keys are supported:
-.ft R
-.TP 1i
-alloc_shared_noncontig
-If not set to \fItrue\fP, the allocation strategy is to allocate
-contiguous memory across process ranks. This may limit the performance
-on some architectures because it does not allow the implementation to
-modify the data layout (e.g., padding to reduce access latency).
-.sp
-.TP 1i
-blocking_fence
-If set to \fItrue\fP, the osc/sm component will use \fBMPI_Barrier\fP
-for \fBMPI_Win_fence\fP. If set to \fIfalse\fP a condition variable
-and counter will be used instead. The default value is
-\fIfalse\fP. This info key is Open MPI specific.
-.sp
-.TP 1i
-For additional supported info keys see \fBMPI_Win_create\fP.
-.sp
-
-.SH NOTES
-Common choices for \fIdisp_unit\fP are 1 (no scaling), and (in C
-syntax) \fIsizeof(type)\fP, for a window that consists of an array of
-elements of type \fItype\fP. The later choice will allow one to use
-array indices in RMA calls, and have those scaled correctly to byte
-displacements, even in a heterogeneous environment.
-.sp
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler
-MPI_ERRORS_RETURN may be used to cause error values to be
-returned. Note that MPI does not guarantee that an MPI program can
-continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Alloc_mem
-MPI_Free_mem
-MPI_Win_allocate
-MPI_Win_create
-MPI_Win_shared_query
diff --git a/ompi/mpi/man/man3/MPI_Win_attach.3in b/ompi/mpi/man/man3/MPI_Win_attach.3in
deleted file mode 100644
index 0f87127ec11..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_attach.3in
+++ /dev/null
@@ -1,89 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015-2019 Research Organization for Information Science
-.\"                         and Technology (RIST).  All rights reserved.
-.\" Copyright (c) 2019-2020 FUJITSU LIMITED.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Win_attach 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_attach, MPI_Win_detach\fP \- One-sided MPI call that attach / detach a window object for RMA operations.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-MPI_Win_attach(MPI_Win \fIwin\fP, void *\fIbase\fP, MPI_Aint \fIsize\fP)
-
-MPI_Win_detach(MPI_Win \fIwin\fP, void *\fIbase\fP)
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_ATTACH(\fIWIN, BASE, SIZE, IERROR\fP)
-	<type> \fIBASE\fP(*)
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fISIZE\fP
-	INTEGER \fIWIN, IERROR\fP
-
-MPI_WIN_DETACH(\fIWIN, BASE, IERROR\fP)
-	<type> \fIBASE\fP(*)
-	INTEGER \fIWIN, IERROR\fP
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_attach(\fIwin\fP, \fIbase\fP, \fIsize\fP, \fIierror\fP)
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIbase\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) :: \fIsize\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPI_Win_detach(\fIwin\fP, \fIbase\fP, \fIierror\fP)
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	TYPE(*), DIMENSION(..), INTENT(IN) :: \fIbase\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-win
-A window that was created with
-.I MPI_Win_create_dynamic
-
-.TP 1i
-base
-Initial address of window (choice).
-.TP 1i
-size
-Size of window in bytes (nonnegative integer).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-win
-Window object returned by the call (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Win_attach is a one-sided MPI communication collective call executed by all processes in the group of \fIcomm\fP. It returns a window object that can be used by these processes to perform RMA operations. Each process specifies a window of existing memory that it exposes to RMA accesses by the processes in the group of \fIcomm\fP. The window consists of \fIsize\fP bytes, starting at address \fIbase\fP. A process may elect to expose no memory by specifying \fIsize\fP = 0.
-.sp
-If the \fIbase\fP value used by MPI_Win_attach was allocated by MPI_Alloc_mem, the size of the window can be no larger than the value set by the MPI_ALLOC_MEM function.
-.sp
-
-.SH NOTES
-Use memory allocated by MPI_Alloc_mem to guarantee properly aligned window boundaries (such as word, double-word, cache line, page frame, and so on).
-.sp
-
-
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-
-
diff --git a/ompi/mpi/man/man3/MPI_Win_c2f.3in b/ompi/mpi/man/man3/MPI_Win_c2f.3in
deleted file mode 100644
index a13fce697dd..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_c2f.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Comm_f2c.3
diff --git a/ompi/mpi/man/man3/MPI_Win_call_errhandler.3in b/ompi/mpi/man/man3/MPI_Win_call_errhandler.3in
deleted file mode 100644
index bc653932ef4..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_call_errhandler.3in
+++ /dev/null
@@ -1,81 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Win_call_errhandler 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-
-.SH NAME
-\fBMPI_Win_call_errhandler\fP \- Passes the supplied error code to the
-error handler assigned to a window
-
-.SH SYNTAX
-.ft R
-
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Win_call_errhandler(MPI_Win \fIwin\fP, int \fIerrorcode\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_CALL_ERRHANDLER(\fIWIN, ERRORCODE, IERROR\fP)
-	INTEGER	\fIWIN, ERRORCODE, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_call_errhandler(\fIwin\fP, \fIerrorcode\fP, \fIierror\fP)
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	INTEGER, INTENT(IN) :: \fIerrorcode\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1.4i
-win
-Window with error handler (handle).
-.ft R
-.TP 1.4i
-errorcode
-MPI error code (integer).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1.4i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-This function invokes the error handler assigned to the window
-\fIwin\fP with the supplied error code \fIerrorcode\fP. If the error
-handler was successfully called, the process is not aborted, and the
-error handler returns, this function returns MPI_SUCCESS.
-
-.SH NOTES
-.ft R
-Users should note that the default error handler is
-MPI_ERRORS_ARE_FATAL. Thus, calling this function will abort the
-window processes if the default error handler has not been changed for
-this window.
-
-.SH ERRORS
-.ft R
-Almost all MPI routines return an error value; C routines as
-the value of the function and Fortran routines in the last argument.
-.sp
-See the MPI man page for a full list of MPI error codes.
-
-.SH SEE ALSO
-.ft R
-.nf
-MPI_Win_create_errhandler
-MPI_Win_set_errhandler
-
diff --git a/ompi/mpi/man/man3/MPI_Win_complete.3in b/ompi/mpi/man/man3/MPI_Win_complete.3in
deleted file mode 100644
index c6fbc0bc964..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_complete.3in
+++ /dev/null
@@ -1,60 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2007-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Win_complete 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_complete\fP \- Completes an RMA access epoch on \fIwin\fP started by a call to MPI_Win_start
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-MPI_Win_complete(MPI_Win \fIwin\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_COMPLETE(WIN, IERROR)
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_complete(\fIwin\fP, \fIierror\fP)
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-win
-Window object (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Win_complete is a one-sided MPI communication synchronization call, completing an RMA access epoch on \fIwin\fP started by a call to MPI_Win_start. MPI_Win_complete enforces the completion of preceding RMA calls at the origin and not at the target. A put or accumulate call may not have completed at the target when it has completed at the origin.
-
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Win_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-MPI_Win_start
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Win_create.3in b/ompi/mpi/man/man3/MPI_Win_create.3in
deleted file mode 100644
index c9104f31b50..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_create.3in
+++ /dev/null
@@ -1,153 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2015      Los Alamos National Security, LLC. All rights
-.\" reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2007-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Win_create 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_create\fP \- One-sided MPI call that returns a window object for RMA operations.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-MPI_Win_create(void *\fIbase\fP, MPI_Aint \fIsize\fP, int \fIdisp_unit\fP,
-	MPI_Info \fIinfo\fP, MPI_Comm \fIcomm\fP, MPI_Win *\fIwin\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_CREATE(\fIBASE, SIZE, DISP_UNIT, INFO, COMM, WIN, IERROR\fP)
-	<type> \fIBASE\fP(*)
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fISIZE\fP
-	INTEGER \fIDISP_UNIT, INFO, COMM, WIN, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_create(\fIbase\fP, \fIsize\fP, \fIdisp_unit\fP, \fIinfo\fP, \fIcomm\fP, \fIwin\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIbase\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) :: \fIsize\fP
-	INTEGER, INTENT(IN) :: \fIdisp_unit\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Win), INTENT(OUT) :: \fIwin\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-base
-Initial address of window (choice).
-.TP 1i
-size
-Size of window in bytes (nonnegative integer).
-.TP 1i
-disp_unit
-Local unit size for displacements, in bytes (positive integer).
-.TP 1i
-info
-Info argument (handle).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-win
-Window object returned by the call (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Win_create is a one-sided MPI communication collective call executed by all processes in the group of \fIcomm\fP. It returns a window object that can be used by these processes to perform RMA operations. Each process specifies a window of existing memory that it exposes to RMA accesses by the processes in the group of \fIcomm\fP. The window consists of \fIsize\fP bytes, starting at address \fIbase\fP. A process may elect to expose no memory by specifying \fIsize\fP = 0.
-.sp
-If the \fIbase\fP value used by MPI_Win_create was allocated by MPI_Alloc_mem, the size of the window can be no larger than the value set by the MPI_ALLOC_MEM function.
-.sp
-The displacement unit argument is provided to facilitate address arithmetic in RMA operations: the target displacement argument of an RMA operation is scaled by the factor \fIdisp_unit\fP specified by the target process, at window creation.
-.sp
-The following info keys are supported:
-.ft R
-.TP 1i
-no_locks
-If set to \fItrue\fP, then the implementation may assume that the local
-window is never locked (by a call to MPI_Win_lock or
-MPI_Win_lock_all).  Setting this value if only active synchronization
-may allow the implementation to enable certain optimizations.
-.sp
-.TP 1i
-accumulate_ordering
-By default, accumulate operations from one initiator to one target on
-the same window are strictly ordered.  If the info key
-accumulate_ordering is set to \fInone\fP, no ordering of accumulate
-operations guaranteed.  They key can also be a comma-separated list of
-required orderings consisting of \fIrar\fP, \fIwar\fP, \fIraw\fP, and \fIwaw\fP for
-read-after-read, write-after-read, read-after-write, and
-write-after-write, respectively.  Looser ordering constraints are
-likely to result in improved performance.
-.sp
-.TP 1i
-accumulate_ops
-If set to \fIsame_op\fP, the implementation will assume that all concurrent
-accumulate calls to the same target address will use the same
-operation.  If set to \fIsame_op_no_op\fP, then the implementation will
-assume that all concurrent accumulate calls to the same target address
-will use the same operation or MPI_NO_OP.  The default is \fIsame_op_no_op\fP.
-.sp
-.TP 1i
-same_size
-If set to \fItrue\fP, then the implementation may assume that the argument
-\fIsize\fP is identical on all processes, and that all processes have
-provided this info key with the same value.
-.sp
-.TP 1i
-same_disp_unit
-If set to \fItrue\fP, then the implementation may assume that the argument
-\fIdisp_unit\fP is identical on all processes, and that all processes have
-provided this info key with the same value.
-.sp
-.SH NOTES
-Common choices for \fIdisp_unit\fP are 1 (no scaling), and (in C syntax) \fIsizeof(type)\fP, for a window that consists of an array of elements of type \fItype\fP. The later choice will allow one to use array indices in RMA calls, and have those scaled correctly to byte displacements, even in a heterogeneous environment.
-.sp
-Use memory allocated by MPI_Alloc_mem to guarantee properly aligned window boundaries (such as word, double-word, cache line, page frame, and so on).
-.sp
-
-
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fISIZE\fP argument only for Fortran 90. FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_ADDRESS_KIND \fISIZE\fP
-.fi
-.sp
-where MPI_ADDRESS_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Alloc_mem
-MPI_Free_mem
-MPI_Win_allocate
-MPI_Win_allocate_shared
diff --git a/ompi/mpi/man/man3/MPI_Win_create_dynamic.3in b/ompi/mpi/man/man3/MPI_Win_create_dynamic.3in
deleted file mode 100644
index 65976023832..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_create_dynamic.3in
+++ /dev/null
@@ -1,92 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      Research Organization for Information Science
-.\"                         and Technology (RIST). All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Win_create_dynamic 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_create_dynamic\fP \- One-sided MPI call that returns a window object for RMA operations.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-MPI_Win_create_dynamic(MPI_Info \fIinfo\fP, MPI_Comm \fIcomm\fP, MPI_Win *\fIwin\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_CREATE_DYNAMIC(\fIINFO, COMM, WIN, IERROR\fP)
-	INTEGER \fIINFO, COMM, WIN, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_create_dynamic(\fIinfo\fP, \fIcomm\fP, \fIwin\fP, \fIierror\fP)
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Win), INTENT(OUT) :: \fIwin\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-info
-Info argument (handle).
-.TP 1i
-comm
-Communicator (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-win
-Window object returned by the call (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Win_create_dynamic is a one-sided MPI communication collective call executed by all processes in the group of \fIcomm\fP. It returns a window object without memory attached that can be used by these processes to perform RMA operations.
-.sp
-The following info keys are supported:
-.ft R
-.TP 1i
-no_locks
-If set to \fItrue\fP, then the implementation may assume that the local
-window is never locked (by a call to MPI_Win_lock or
-MPI_Win_lock_all).  Setting this value if only active synchronization
-may allow the implementation to enable certain optimizations.
-.sp
-.TP 1i
-accumulate_ordering
-By default, accumulate operations from one initiator to one target on
-the same window are strictly ordered.  If the info key
-accumulate_ordering is set to \fInone\fP, no ordering of accumulate
-operations guaranteed.  They key can also be a comma-separated list of
-required orderings consisting of \fIrar\fP, \fIwar\fP, \fIraw\fP, and \fIwaw\fP for
-read-after-read, write-after-read, read-after-write, and
-write-after-write, respectively.  Looser ordering constraints are
-likely to result in improved performance.
-.sp
-.TP 1i
-accumulate_ops
-If set to \fIsame_op\fP, the implementation will assume that all concurrent
-accumulate calls to the same target address will use the same
-operation.  If set to \fIsame_op_no_op\fP, then the implementation will
-assume that all concurrent accumulate calls to the same target address
-will use the same operation or MPI_NO_OP.  The default is \fIsame_op_no_op\fP.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-
-
diff --git a/ompi/mpi/man/man3/MPI_Win_create_errhandler.3in b/ompi/mpi/man/man3/MPI_Win_create_errhandler.3in
deleted file mode 100644
index f3468014b48..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_create_errhandler.3in
+++ /dev/null
@@ -1,83 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2009-2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Win_create_errhandler 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_create_errhandler\fP \- Creates an error handler for a window.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Win_create_errhandler(MPI_Win_errhandler_function *\fIfunction\fP,
-	MPI_Errhandler *\fIerrhandler\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_CREATE_ERRHANDLER(\fIFUNCTION, ERRHANDLER, IERROR\fP)
-	EXTERNAL \fIFUNCTION\fP
-	INTEGER \fIERRHANDLER, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_create_errhandler(\fIwin_errhandler_fn\fP, \fIerrhandler\fP, \fIierror\fP)
-	PROCEDURE(MPI_Win_errhandler_function) :: \fIwin_errhandler_fn\fP
-	TYPE(MPI_Errhandler), INTENT(OUT) :: \fIerrhandler\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH DEPRECATED TYPE NAME NOTE
-.ft R
-MPI-2.2 deprecated the MPI_Win_errhandler_fn and
-MPI::Win::Errhandler_fn types in favor of
-MPI_Win_errhandler_function and MPI::Win::Errhandler_function,
-respectively.  Open MPI supports both names (indeed, the _fn names are
-typedefs to the _function names).
-
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-function
-User-defined error-handling procedure (function).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-errhandler
-MPI error handler (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Win_create_errhandler should be, in C, a function of type MPI_Win_errhandler_function, which is defined as
-.sp
-.nf
-typedef void MPI_Win_errhandler_function(MPI Win *, int *, ...);
-.fi
-.sp
-The first argument is the window in use, the second is the error code to be returned.
-.sp
-In Fortran, the user routine should be of the form:
-.sp
-.nf
-SUBROUTINE WIN_ERRHANDLER_FUNCTION(WIN, ERROR_CODE, ...)
-    INTEGER WIN, ERROR_CODE
-.fi
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Win_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_Win_create_keyval.3in b/ompi/mpi/man/man3/MPI_Win_create_keyval.3in
deleted file mode 100644
index 7decaa40b7b..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_create_keyval.3in
+++ /dev/null
@@ -1,124 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Win_create_keyval 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_create_keyval\fP \- Creates a keyval for a window.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Win_create_keyval(MPI_Win_copy_attr_function *\fIwin_copy_attr_fn\fP,
-	MPI_Win_delete_attr_function *\fIwin_delete_attr_fn\fP,
-	int *\fIwin_keyval\fP, void *\fIextra_state\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_CREATE_KEYVAL(\fIWIN_COPY_ATTR_FN, WIN_DELETE_ATTR_FN,
-	WIN_KEYVAL, EXTRA_STATE, IERROR\fP)
-	EXTERNAL \fIWIN_COPY_ATTR_FN, WIN_DELETE_ATTR_FN\fP
-	INTEGER \fIWIN_KEYVAL, IERROR\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fIEXTRA_STATE\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_create_keyval(\fIwin_copy_attr_fn\fP, \fIwin_delete_attr_fn\fP, \fIwin_keyval\fP,
-		\fIextra_state\fP, \fIierror\fP)
-	PROCEDURE(MPI_Win_copy_attr_function) :: \fIwin_copy_attr_fn\fP
-	PROCEDURE(MPI_Win_delete_attr_function) :: \fIwin_delete_attr_fn\fP
-	INTEGER, INTENT(OUT) :: \fIwin_keyval\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) :: \fIextra_state\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-win_copy_attr_fn
-Copy callback function for \fIwin_keyval\fP (function).
-.TP 1i
-win_delete_attr_fn
-Delete callback function for \fIwin_keyval\fP (function).
-.TP 1i
-extra_state
-Extra state for callback functions.
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-win_keyval
-Key value for future access (integer).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-The argument \fIwin_copy_attr_fn\fP may be specified as MPI_WIN_NULL_COPY_FN or MPI_WIN_DUP_FN from either C or Fortran. MPI_WIN_NULL_COPY_FN is a function that serves only to return \fIflag\fP = 0 and MPI_SUCCESS. MPI_WIN_DUP_FN is a simple-minded copy function that sets \fIflag\fP = 1, returns the value of \fIattribute_val_in\fP in \fIattribute_val_out\fP, and returns MPI_SUCCESS.
-.sp
-The argument \fIwin_delete_attr_fn\fP may be specified as MPI_WIN_NULL_DELETE_FN from either C or Fortran. MPI_WIN_NULL_DELETE_FN is a function that serves only to return MPI_SUCCESS.
-.sp
-The C callback functions are:
-.sp
-.nf
-typedef int MPI_Win_copy_attr_function(MPI_Win \fIoldwin\fP, int \fIwin_keyval\fP,
-             void *\fIextra_state\fP, void *\fIattribute_val_in\fP,
-             void *\fIattribute_val_out\fP, int *\fIflag\fP);
-.fi
-.sp
-and
-.sp
-.nf
-typedef int MPI_Win_delete_attr_function(MPI_Win \fIwin\fP, int \fIwin_keyval\fP,
-             void *\fIattribute_val\fP, void *\fIextra_state\fP);
-.fi
-.sp
-The Fortran callback functions are:
-.sp
-.nf
-SUBROUTINE WIN_COPY_ATTR_FN(\fIOLDWIN, WIN_KEYVAL, EXTRA_STATE,
-	ATTRIBUTE_VAL_IN, ATTRIBUTE_VAL_OUT, FLAG, IERROR\fP)
-	INTEGER \fIOLDWIN, WIN_KEYVAL, IERROR\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fIEXTRA_STATE, ATTRIBUTE_VAL_IN,
-		ATTRIBUTE_VAL_OUT\fP
-	LOGICAL \fIFLAG\fP
-.fi
-.sp
-and
-.sp
-.nf
-SUBROUTINE WIN_DELETE_ATTR_FN(\fIWIN, WIN_KEYVAL, ATTRIBUTE_VAL,
-	EXTRA_STATE, IERROR\fP)
-	INTEGER \fIWIN, WIN_KEYVAL, IERROR\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fIATTRIBUTE_VAL, EXTRA_STATE\fP
-.fi
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fIEXTRA_STATE\fP argument only for Fortran 90. FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_ADDRESS_KIND \fIEXTRA_STATE\fP
-.fi
-.sp
-where MPI_ADDRESS_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_Win_delete_attr.3in b/ompi/mpi/man/man3/MPI_Win_delete_attr.3in
deleted file mode 100644
index 8ff76cec275..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_delete_attr.3in
+++ /dev/null
@@ -1,66 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Win_delete_attr 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_delete_attr\fP \- Deletes an attribute from a window.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Win_delete_attr(MPI_Win \fIwin\fP, int \fIwin_keyval\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_DELETE_ATTR(\fIWIN, WIN_KEYVAL, IERROR\fP)
-	INTEGER \fIWIN, WIN_KEYVAL, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_delete_attr(\fIwin\fP, \fIwin_keyval\fP, \fIierror\fP)
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	INTEGER, INTENT(IN) :: \fIwin_keyval\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-win
-Window from which the attribute is deleted (handle).
-
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-win_keyval
-Key value (integer).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH NOTES
-Note that it is not defined by the MPI standard what happens if the
-delete_fn callback invokes other MPI functions.  In Open MPI, it is
-not valid for delete_fn callbacks (or any of their children) to add or
-delete attributes on the same object on which the delete_fn callback
-is being invoked.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_Win_detach.3in b/ompi/mpi/man/man3/MPI_Win_detach.3in
deleted file mode 100644
index 42a7c2b2dfb..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_detach.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Win_attach.3
diff --git a/ompi/mpi/man/man3/MPI_Win_f2c.3in b/ompi/mpi/man/man3/MPI_Win_f2c.3in
deleted file mode 100644
index a13fce697dd..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_f2c.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Comm_f2c.3
diff --git a/ompi/mpi/man/man3/MPI_Win_fence.3in b/ompi/mpi/man/man3/MPI_Win_fence.3in
deleted file mode 100644
index e9cbc43e88e..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_fence.3in
+++ /dev/null
@@ -1,77 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2007-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Win_fence 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_fence\fP \- Synchronizes RMA calls on a window.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Win_fence(int \fIassert\fP, MPI_Win \fIwin\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_FENCE(\fIASSERT, WIN, IERROR\fP)
-	INTEGER \fIASSERT, WIN, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_fence(\fIassert\fP, \fIwin\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIassert\fP
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-assert
-Program assertion (integer).
-.TP 1i
-win
-Window object (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Win_fence synchronizes RMA calls on \fIwin\fP. The call is collective on the group of \fIwin\fP. All RMA operations on \fIwin\fP originating at a given process and started before the fence call will complete at that process before the fence call returns. They will be completed at their target before the fence call returns at the target. RMA operations on \fIwin\fP started by a process after the fence call returns will access their target window only after MPI_Win_fence has been called by the target process.
-.sp
-The call completes an RMA access epoch if it was preceded by another fence call and the local process issued RMA communication calls on \fIwin\fP between these two calls. The call completes an RMA exposure epoch if it was preceded by another fence call and the local window was the target of RMA accesses between these two calls. The call starts an RMA access epoch if it is followed by another fence call and by RMA communication calls issued between these two fence calls. The call starts an exposure epoch if it is followed by another fence call and the local window is the target of RMA accesses between these two fence calls. Thus, the fence call is equivalent to calls to a subset of \fIpost, start, complete, wait\fP.
-.sp
-A fence call usually entails a barrier synchronization: a process completes a call to MPI_Win_fence only after all other processes in the group have entered their matching call. However, a call to MPI_Win_fence that is known not to end any epoch (in particular, a call with \fIassert\fP = MPI_MODE_NOPRECEDE) does not necessarily act as a barrier.
-
-.SH NOTE
-Calls to MPI_Win_fence should both precede and follow calls to put, get or accumulate that are synchronized with fence calls.
-.sp
-
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-MPI_Win_create
-MPI_Win_start
-MPI_Win_post
-MPI_Win_complete
-MPI_Win_wait
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Win_flush.3in b/ompi/mpi/man/man3/MPI_Win_flush.3in
deleted file mode 100644
index 1b41798b0ba..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_flush.3in
+++ /dev/null
@@ -1,76 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2014 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2007-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" $COPYRIGHT$
-.TH MPI_Win_flush 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_flush\fP, \fBMPI_Win_flush_all\fP \- Complete all outstanding RMA operations at both the origin and the target
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Win_flush (int \fIrank\fP, MPI_Win \fIwin\fP)
-
-int MPI_Win_flush_all (MPI_Win \fIwin\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_FLUSH(\fIRANK, WIN, IERROR\fP)
-	INTEGER \fIRANK, WIN, IERROR\fP
-
-MPI_WIN_FLUSH_ALL(\fIWIN, IERROR\fP)
-	INTEGER \fIWIN, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_flush(\fIrank\fP, \fIwin\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIrank\fP
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPI_Win_flush_all(\fIwin\fP, \fIierror\fP)
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-rank
-Rank of window (nonnegative integer).
-.TP 1i
-win
-Window object (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-\fBMPI_Win_flush\fP completes all outstanding RMA operations initiated by the calling process to the target rank on the specified window. The operations are completed both at the origin and at the target. \fBMPI_Win_flush_all\fP completes all outstanding RMA operations to all targets.
-.sp
-Can only be called from within a passive target epoch.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with \fBMPI_Comm_set_errhandler\fP; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-MPI_Win_flush_local
-MPI_Win_lock
-MPI_Win_lock_all
-.br
diff --git a/ompi/mpi/man/man3/MPI_Win_flush_all.3in b/ompi/mpi/man/man3/MPI_Win_flush_all.3in
deleted file mode 100644
index b30e345a522..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_flush_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Win_flush.3
diff --git a/ompi/mpi/man/man3/MPI_Win_flush_local.3in b/ompi/mpi/man/man3/MPI_Win_flush_local.3in
deleted file mode 100644
index 440fbfe41f8..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_flush_local.3in
+++ /dev/null
@@ -1,76 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2014 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2007-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" $COPYRIGHT$
-.TH MPI_Win_flush_local 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_flush_local\fP, \fBMPI_Win_flush_local_all\fP \- Complete all outstanding RMA operations at both the origin
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Win_flush_local (int \fIrank\fP, MPI_Win \fIwin\fP)
-
-int MPI_Win_flush_local_all (MPI_Win \fIwin\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_FLUSH_LOCAL(\fIRANK, WIN, IERROR\fP)
-	INTEGER \fIRANK, WIN, IERROR\fP
-
-MPI_WIN_FLUSH_LOCAL_ALL(\fIWIN, IERROR\fP)
-	INTEGER \fIWIN, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_flush_local(\fIrank\fP, \fIwin\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIrank\fP
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPI_Win_flush_local_all(\fIwin\fP, \fIierror\fP)
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-rank
-Rank of window (nonnegative integer).
-.TP 1i
-win
-Window object (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-\fBMPI_Win_flush_local\fP locally completes at the origin all outstanding RMA operations initiated by the calling process to the target process specified by rank on the specified window. For example, after this routine completes, the user may reuse any buffers provided to put, get, or accumulate operations. \fBMPI_Win_flush_local_all\fP locally completes at the origin all outstanding RMA operations to all targets.
-.sp
-Can only be called from within a passive target epoch.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with \fBMPI_Comm_set_errhandler\fP; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-MPI_Win_flush
-MPI_Win_lock
-MPI_Win_lock_all
-.br
diff --git a/ompi/mpi/man/man3/MPI_Win_flush_local_all.3in b/ompi/mpi/man/man3/MPI_Win_flush_local_all.3in
deleted file mode 100644
index 6b740a2b3a0..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_flush_local_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPI_Win_flush_local.3
diff --git a/ompi/mpi/man/man3/MPI_Win_free.3in b/ompi/mpi/man/man3/MPI_Win_free.3in
deleted file mode 100644
index 51ca58bd455..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_free.3in
+++ /dev/null
@@ -1,62 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2007-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Win_free 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_free\fP \- Frees the window object and returns a null handle.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Win_free(MPI_Win *\fIwin\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_FREE(\fIWIN, IERROR\fP)
-	INTEGER \fIWIN, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_free(\fIwin\fP, \fIierror\fP)
-	TYPE(MPI_Win), INTENT(INOUT) :: \fIwin\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-win
-Window object (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Win_free frees the window object \fIwin\fP and returns a null handle (equal to MPI_WIN_NULL). This collective call is executed by all processes in the group associated with \fIwin\fP. It can be invoked by a process only after it has completed its involvement in RMA communications on window \fIwin\fP, that is, the process has called MPI_Win_fence, or called MPI_Win_unlock to match a previous call to MPI_Win_lock. When the call returns, the window memory can be freed.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-MPI_Win_create
-MPI_Win_fence
-MPI_Win_unlock
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Win_free_keyval.3in b/ompi/mpi/man/man3/MPI_Win_free_keyval.3in
deleted file mode 100644
index 3dbfe5eee19..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_free_keyval.3in
+++ /dev/null
@@ -1,52 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Win_free_keyval 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_free_keyval\fP \- Frees a window keyval.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Win_free_keyval(int *\fIwin_keyval\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_FREE_KEYVAL(\fIWIN_KEYVAL, IERROR\fP)
-	INTEGER \fIWIN_KEYVAL, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_free_keyval(\fIwin_keyval\fP, \fIierror\fP)
-	INTEGER, INTENT(INOUT) :: \fIwin_keyval\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-win_keyval
-Key value (integer).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_Win_get_attr.3in b/ompi/mpi/man/man3/MPI_Win_get_attr.3in
deleted file mode 100644
index f33798dad8b..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_get_attr.3in
+++ /dev/null
@@ -1,83 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Win_get_attr 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_get_attr\fP \- Obtains the value of a window attribute.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Win_get_attr(MPI_Win \fIwin\fP, int \fIwin_keyval\fP,
-	void *\fIattribute_val\fP, int *\fIflag\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_GET_ATTR(\fIWIN, WIN_KEYVAL, ATTRIBUTE_VAL, FLAG, IERROR\fP)
-	INTEGER \fIWIN, WIN_KEYVAL, IERROR\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fIATTRIBUTE_VAL\fP
-	LOGICAL \fIFLAG\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_get_attr(\fIwin\fP, \fIwin_keyval\fP, \fIattribute_val\fP, \fIflag\fP, \fIierror\fP)
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	INTEGER, INTENT(IN) :: \fIwin_keyval\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(OUT) :: \fIattribute_val\fP
-	LOGICAL, INTENT(OUT) :: \fIflag\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-win
-Window to which the attribute is attached (handle).
-.TP 1i
-win_keyval
-Key value (integer).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-attribute_val
-Attribute value, unless \fIag\fP = false
-.TP 1i
-flag
-False if no attribute is associated with the key (logical).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Obtains the value of a window attribute.
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fIATTRIBUTE_VAL\fP argument only for Fortran 90. FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_ADDRESS_KIND \fIATTRIBUTE_VAL\fP
-.fi
-.sp
-where MPI_ADDRESS_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_Win_get_errhandler.3in b/ompi/mpi/man/man3/MPI_Win_get_errhandler.3in
deleted file mode 100644
index ff9984a37f2..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_get_errhandler.3in
+++ /dev/null
@@ -1,61 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Win_get_errhandler 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_get_errhandler\fP \- Retrieves the error handler currently associated with a window.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Win_get_errhandler(MPI_Win \fIwin\fP, MPI_Errhandler *\fIerrhandler\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_GET_ERRHANDLER(\fIWIN, ERRHANDLER, IERROR\fP)
-	INTEGER \fIWIN, ERRHANDLER, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_get_errhandler(\fIwin\fP, \fIerrhandler\fP, \fIierror\fP)
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	TYPE(MPI_Errhandler), INTENT(OUT) :: \fIerrhandler\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-win
-Window (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-errhandler
-Error handler currently associated with window (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Win_get_errhandler retrieves the error handler currently associated with a window.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-
diff --git a/ompi/mpi/man/man3/MPI_Win_get_group.3in b/ompi/mpi/man/man3/MPI_Win_get_group.3in
deleted file mode 100644
index 8917ecf4ef7..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_get_group.3in
+++ /dev/null
@@ -1,60 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Win_get_group 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_get_group\fP \- Returns a duplicate of the group of the communicator used to create the window.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-MPI_Win_get_group(MPI_Win \fIwin\fP, MPI_Group *\fIgroup\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_GET_GROUP(\fIWIN, GROUP, IERROR\fP)
-	INTEGER \fIWIN, GROUP, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_get_group(\fIwin\fP, \fIgroup\fP, \fIierror\fP)
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	TYPE(MPI_Group), INTENT(OUT) :: \fIgroup\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-win
-Window object (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-group
-Group of processes that share access to the window (handle).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Win_get_group returns a duplicate of the group of the communicator used to create the window associated with \fIwin\fP. The group is returned in \fIgroup\fP.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_Win_get_info.3in b/ompi/mpi/man/man3/MPI_Win_get_info.3in
deleted file mode 100644
index 9018d782942..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_get_info.3in
+++ /dev/null
@@ -1,76 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      Research Organization for Information Science
-.\"                         and Technology (RIST). All rights reserved.
-.\" Copyright (c) 2020      FUJITSU LIMITED.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Win_get_info 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_get_info\fP \- Retrieves active window info hints
-.
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Win_get_info(MPI_Win \fIwin\fP, MPI_Info \fI*info_used\fP)
-.
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_GET_INFO(\fIWIN, INFO_USED, IERROR\fP)
-	INTEGER	\fIWIN, INFO_USED, IERROR \fP
-.
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_get_info(\fIwin\fP, \fIinfo_used\fP, \fIierror\fP)
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	TYPE(MPI_Info), INTENT(OUT) :: \fIinfo_used\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-win
-Window from which to receive active info hints
-.
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-info_used
-New info object returned with all active hints on this window.
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-.
-.SH DESCRIPTION
-.ft R
-MPI_Win_get_info returns a new info object containing the hints of
-the window associated with
-.IR win .
-The current setting of all hints actually used by the system related
-to this window is returned in
-.IR info_used .
-If no such hints exist, a handle to a newly created info object is
-returned that contains no key/value pair. The user is responsible for
-freeing info_used via MPI_Info_free.
-.
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler
-MPI_ERRORS_RETURN may be used to cause error values to be
-returned. Note that MPI does not guarantee that an MPI program can
-continue past an error.
-.
-.SH SEE ALSO
-MPI_Win_set_info,
-MPI_Win_free
diff --git a/ompi/mpi/man/man3/MPI_Win_get_name.3in b/ompi/mpi/man/man3/MPI_Win_get_name.3in
deleted file mode 100644
index 6621cb30df5..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_get_name.3in
+++ /dev/null
@@ -1,65 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Win_get_name 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_get_name\fP \- Obtains the name of a window.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Win_get_name(MPI_Win \fIwin\fP, char *\fIwin_name\fP, int *\fIresultlen\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_GET_NAME(\fIWIN, WIN_NAME, RESULTLEN, IERROR\fP)
-	INTEGER \fIWIN, RESULTLEN, IERROR\fP
-	CHARACTER*(*) \fIWIN_NAME\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_get_name(\fIwin\fP, \fIwin_name\fP, \fIresultlen\fP, \fIierror\fP)
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	CHARACTER(LEN=MPI_MAX_OBJECT_NAME), INTENT(OUT) :: \fIwin_name\fP
-	INTEGER, INTENT(OUT) :: \fIresultlen\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-win
-Window whose name is to be returned (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-win_name
-the name previously stored on the window, or an empty string if no such name exists (string).
-.TP 1i
-resultlen
-Length of returned name (integer).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-
diff --git a/ompi/mpi/man/man3/MPI_Win_lock.3in b/ompi/mpi/man/man3/MPI_Win_lock.3in
deleted file mode 100644
index 2f7d1688df1..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_lock.3in
+++ /dev/null
@@ -1,87 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2014 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2007-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Win_lock 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_lock\fP \- Starts an RMA access epoch locking access to a particular rank.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Win_lock(int \fIlock_type\fP, int \fIrank\fP, int \fIassert\fP, MPI_Win \fIwin\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_LOCK(\fILOCK_TYPE, RANK, ASSERT, WIN, IERROR\fP)
-	INTEGER \fILOCK_TYPE, RANK, ASSERT, WIN, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_lock(\fIlock_type\fP, \fIrank\fP, \fIassert\fP, \fIwin\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIlock_type\fP, \fIrank\fP, \fIassert\fP
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-lock_type
-Either MPI_LOCK_EXCLUSIVE or MPI_LOCK_SHARED (state).
-.TP 1i
-rank
-Rank of locked window (nonnegative integer).
-.TP 1i
-assert
-Program assertion (integer).
-.TP 1i
-win
-Window object (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Starts an RMA access epoch. Locks ensure that only the windows created by specific processes can be accessed by those processes (and by no other processes) during that epoch.
-.sp
-Locks are used to protect accesses to the locked target window effected by RMA calls issued between the lock and unlock call, and to protect local load/store accesses to a locked local window executed between the lock and unlock call.
-Accesses that are protected by an exclusive lock will not be concurrent at the window site with other accesses to the same window that are lock protected. Accesses that are protected by a shared lock will not be concurrent at the window site with accesses protected by an exclusive lock to the same window.
-.sp
-The \fIassert\fP argument is used to provide assertions on the context of the call that may be used for various optimizations. (See Section 6.4.4 of the MPI-2 Standard.) A value of \fIassert\fP = 0 is always valid.
-.sp
-.ft
-.SH NOTES
-.ft R
-In a client/server environment in which clients connect to
-a server and create windows that span both the client and the
-server, if a client or server that has obtained a lock
-on such a window and then terminates abnormally, the server or other clients
-may hang in a MPI_Win_lock call, failing to notice that the peer MPI job
-has terminated.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-MPI_Win_unlock
-MPI_Win_lock_all
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Win_lock_all.3in b/ompi/mpi/man/man3/MPI_Win_lock_all.3in
deleted file mode 100644
index 3c76e45534e..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_lock_all.3in
+++ /dev/null
@@ -1,79 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2014 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2007-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" $COPYRIGHT$
-.TH MPI_Win_lock_all 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_lock_all\fP \- Starts an RMA access epoch locking access to all processes in the window
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Win_lock_all(int \fIassert\fP, MPI_Win \fIwin\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_LOCK_ALL(\fIASSERT, WIN, IERROR\fP)
-	INTEGER \fIASSERT, WIN, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_lock_all(\fIassert\fP, \fIwin\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIassert\fP
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-assert
-Program assertion (integer).
-.TP 1i
-win
-Window object (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-Starts an RMA access epoch to all processes in \fIwin\fP, with a lock type of MPI_LOCK_SHARED. During the epoch, the calling process can access the window memory on all processes in \fIwin\fP by using RMA operations. A window locked with MPI_Win_lock_all must be unlocked with MPI_Win_unlock_all. This routine is not collective — the ALL refers to a lock on all members of the group of the window.
-.sp
-Locks are used to protect accesses to the locked target window effected by RMA calls issued between the lock and unlock call, and to protect local load/store accesses to a locked local window executed between the lock and unlock call.
-Accesses that are protected by an exclusive lock will not be concurrent at the window site with other accesses to the same window that are lock protected. Accesses that are protected by a shared lock will not be concurrent at the window site with accesses protected by an exclusive lock to the same window.
-.sp
-The \fIassert\fP argument is used to provide assertions on the context of the call that may be used for various optimizations. (See Section 6.4.4 of the MPI-2 Standard.) A value of \fIassert\fP = 0 is always valid.
-.sp
-.ft
-.SH NOTES
-.ft R
-In a client/server environment in which clients connect to
-a server and create windows that span both the client and the
-server, if a client or server that has obtained a lock
-on such a window and then terminates abnormally, the server or other clients
-may hang in a MPI_Win_lock_all call, failing to notice that the peer MPI job
-has terminated.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-MPI_Win_unlock_all
-MPI_Win_lock
-.br
diff --git a/ompi/mpi/man/man3/MPI_Win_post.3in b/ompi/mpi/man/man3/MPI_Win_post.3in
deleted file mode 100644
index 928393ba339..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_post.3in
+++ /dev/null
@@ -1,71 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2007-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Win_post 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_post\fP \- Starts an RMA exposure epoch for the local window associated with \fIwin\fP
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Win_post(MPI_Group \fIgroup\fP, int assert, MPI_Win \fIwin\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_POST(\fIGROUP, ASSERT, WIN, IERROR\fP)
-	INTEGER GROUP, ASSERT, WIN, IERROR
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_post(\fIgroup\fP, \fIassert\fP, \fIwin\fP, \fIierror\fP)
-	TYPE(MPI_Group), INTENT(IN) :: \fIgroup\fP
-	INTEGER, INTENT(IN) :: \fIassert\fP
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-group
-The group of origin processes (handle)
-.TP 1i
-assert
-Program assertion (integer)
-.TP 1i
-win
-Window object (handle)
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-
-Starts an RMA exposure epoch for the local window associated with \fIwin\fP. Only the processes belonging to \fIgroup\fP should access the window with RMA calls on \fIwin\fP during this epoch. Each process in \fIgroup\fP must issue a matching call to MPI_Win_start. MPI_Win_post does not block.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Win_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-MPI_Win_start
-MPI_Win_wait
-.br
-
-
-
diff --git a/ompi/mpi/man/man3/MPI_Win_set_attr.3in b/ompi/mpi/man/man3/MPI_Win_set_attr.3in
deleted file mode 100644
index 4e428f4dc28..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_set_attr.3in
+++ /dev/null
@@ -1,80 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Win_set_attr 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_set_attr\fP \- Sets the value of a window attribute.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Win_set_attr(MPI_Win \fIwin\fP, int \fIwin_keyval\fP, void *\fIattribute_val\fP)
-
-.fi
-.SH Fortran Syntax (see FORTRAN 77 NOTES)
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_SET_ATTR(\fIWIN, WIN_KEYVAL, ATTRIBUTE_VAL, IERROR\fP)
-	INTEGER \fIWIN, WIN_KEYVAL, IERROR\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fIATTRIBUTE_VAL\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_set_attr(\fIwin\fP, \fIwin_keyval\fP, \fIattribute_val\fP, \fIierror\fP)
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	INTEGER, INTENT(IN) :: \fIwin_keyval\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN) :: \fIattribute_val\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-win
-Window to which attribute will be attached (handle).
-
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-win_keyval
-Key value (integer).
-.TP 1i
-attribute_val
-Attribute value.
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-
-.SH FORTRAN 77 NOTES
-.ft R
-The MPI standard prescribes portable Fortran syntax for
-the \fIATTRIBUTE_VAL\fP argument only for Fortran 90. FORTRAN 77
-users may use the non-portable syntax
-.sp
-.nf
-     INTEGER*MPI_ADDRESS_KIND \fIATTRIBUTE_VAL\fP
-.fi
-.sp
-where MPI_ADDRESS_KIND is a constant defined in mpif.h
-and gives the length of the declared integer in bytes.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
diff --git a/ompi/mpi/man/man3/MPI_Win_set_errhandler.3in b/ompi/mpi/man/man3/MPI_Win_set_errhandler.3in
deleted file mode 100644
index 955731044f4..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_set_errhandler.3in
+++ /dev/null
@@ -1,64 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Win_set_errhandler 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_set_errhandler\fP \- Attaches a new error handler to a window.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Win_set_errhandler(MPI_Win \fIwin\fP, MPI_Errhandler \fIerrhandler\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_SET_ERRHANDLER(\fIWIN, ERRHANDLER, IERROR\fP)
-	INTEGER \fIWIN, ERRHANDLER, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_set_errhandler(\fIwin\fP, \fIerrhandler\fP, \fIierror\fP)
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	TYPE(MPI_Errhandler), INTENT(IN) :: \fIerrhandler\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-win
-Window (handle).
-
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-errhandler
-New error handler for window (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Win_set_errhandler attaches a new error handler to a window. The error handler must be either a predefined error handler or an error handler created by a call to MPI_Win_create_errhandler.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-
diff --git a/ompi/mpi/man/man3/MPI_Win_set_info.3in b/ompi/mpi/man/man3/MPI_Win_set_info.3in
deleted file mode 100644
index 51139e292cb..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_set_info.3in
+++ /dev/null
@@ -1,77 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      Research Organization for Information Science
-.\"                         and Technology (RIST). All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Win_set_info 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_set_info\fP \- Set window info hints
-.
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Win_set_info(MPI_Win \fIwin\fP, MPI_Info \fIinfo\fP)
-.
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_SET_INFO(\fIWIN, INFO, IERROR\fP)
-	INTEGER	\fIWIN, INFO, IERROR \fP
-.
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_set_info(\fIwin\fP, \fIinfo\fP, \fIierror\fP)
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-win
-Window on which to set info hints
-.TP 1i
-info
-Info object containing hints to be set on
-.I win
-.
-.SH OUTPUT PARAMETERS
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-.
-.SH DESCRIPTION
-.ft R
-MPI_WIN_SET_INFO sets new values for the hints of the window
-associated with
-.IR win.
-MPI_WIN_SET_INFO is a collective routine. The info object may be
-different on each process, but any info entries that an implementation
-requires to be the same on all processes must appear with the same
-value in each process's
-.I info
-object.
-.
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler
-MPI_ERRORS_RETURN may be used to cause error values to be
-returned. Note that MPI does not guarantee that an MPI program can
-continue past an error.
-.
-.SH SEE ALSO
-MPI_Win_get_info,
-MPI_Info_create,
-MPI_Info_set,
-MPI_Info_free
diff --git a/ompi/mpi/man/man3/MPI_Win_set_name.3in b/ompi/mpi/man/man3/MPI_Win_set_name.3in
deleted file mode 100644
index a9bb4db9e1b..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_set_name.3in
+++ /dev/null
@@ -1,65 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2013 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Win_set_name 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_set_name\fP \- Sets the name of a window.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Win_set_name(MPI_Win \fIwin\fP, const char *\fIwin_name\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_SET_NAME(\fIWIN, WIN_NAME, IERROR\fP)
-	INTEGER \fIWIN, IERROR\fP
-	CHARACTER*(*) \fIWIN_NAME\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_set_name(\fIwin\fP, \fIwin_name\fP, \fIierror\fP)
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	CHARACTER(LEN=*), INTENT(IN) :: \fIwin_name\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT/OUTPUT PARAMETER
-.ft R
-.TP 1i
-win
-Window whose identifier is to be set (handle).
-
-.SH INPUT PARAMETER
-.ft R
-.TP 1i
-win_name
-The character string used as the name (string).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-
diff --git a/ompi/mpi/man/man3/MPI_Win_shared_query.3in b/ompi/mpi/man/man3/MPI_Win_shared_query.3in
deleted file mode 100644
index 9b627ccbc25..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_shared_query.3in
+++ /dev/null
@@ -1,101 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2015      Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2007-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" $COPYRIGHT$
-.TH MPI_Win_shared_query 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_shared_query\fP \- Query a shared memory window
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Win_shared_query (MPI_Win \fIwin\fP, int \fIrank\fP, MPI_Aint *\fIsize\fP,
-                          int *\fIdisp_unit\fP, void *\fIbaseptr\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_SHARED_QUERY(WIN, RANK, SIZE, DISP_UNIT, BASEPTR, IERROR)
-        INTEGER WIN, RANK, DISP_UNIT, IERROR
-        INTEGER(KIND=MPI_ADDRESS_KIND) SIZE, BASEPTR
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_shared_query(\fIwin\fP, \fIrank\fP, \fIsize\fP, \fIdisp_unit\fP, \fIbaseptr\fP, \fIierror\fP)
-	USE, INTRINSIC :: ISO_C_BINDING, ONLY : C_PTR
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	INTEGER, INTENT(IN) :: \fIrank\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(OUT) :: \fIsize\fP
-	INTEGER, INTENT(OUT) :: \fIdisp_unit\fP
-	TYPE(C_PTR), INTENT(OUT) :: \fIbaseptr\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-win
-Shared memory window object (handle).
-.TP 1i
-rank
-Rank in the group of window \fIwin\fP (non-negative integer)
-or MPI_PROC_NULL.
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-size
-Size of the window segment (non-negative integer).
-.TP 1i
-disp_unit
-Local unit size for displacements, in bytes (positive integer).
-.TP 1i
-baseptr
-Address for load/store access to window segment
-(choice).
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-\fBMPI_Win_shared_query\fP queries the process-local address for
-remote memory segments created with MPI_Win_allocate_shared. This
-function can return different process-local addresses for the same
-physical memory on different processes. The returned memory can be
-used for load/store accesses subject to the constraints defined in
-MPI-3.1 \[char167] 11.7. This function can only be called with windows
-of flavor MPI_WIN_FLAVOR_SHARED. If the passed window is not of flavor
-MPI_WIN_FLAVOR_SHARED, the error MPI_ERR_RMA_FLAVOR is raised. When
-rank is MPI_PROC_NULL, the \fIpointer\fP, \fIdisp_unit\fP, and
-\fIsize\fP returned are the pointer, disp_unit, and size of the memory
-segment belonging the lowest rank that specified \fIsize\fP > 0. If
-all processes in the group attached to the window specified \fIsize\fP
-= 0, then the call returns \fIsize\fP = 0 and a \fIbaseptr\fP as if
-\fBMPI_Alloc_mem\fP was called with \fIsize\fP = 0.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value
-of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for
-I/O function errors. The error handler may be changed with
-MPI_Comm_set_errhandler; the predefined error handler
-MPI_ERRORS_RETURN may be used to cause error values to be
-returned. Note that MPI does not guarantee that an MPI program can
-continue past an error.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Alloc_mem
-MPI_Win_allocate_shared
diff --git a/ompi/mpi/man/man3/MPI_Win_start.3in b/ompi/mpi/man/man3/MPI_Win_start.3in
deleted file mode 100644
index 00c02c18c4a..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_start.3in
+++ /dev/null
@@ -1,80 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2007-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Win_start 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_start\fP \- Starts an RMA access epoch for \fIwin\fP
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Win_start(MPI_Group \fIgroup\fP, int assert, MPI_Win \fIwin\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_START(\fIGROUP, ASSERT, WIN, IERROR\fP)
-	INTEGER GROUP, ASSERT, WIN, IERROR
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_start(\fIgroup\fP, \fIassert\fP, \fIwin\fP, \fIierror\fP)
-	TYPE(MPI_Group), INTENT(IN) :: \fIgroup\fP
-	INTEGER, INTENT(IN) :: \fIassert\fP
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-group
-The group of target processes (handle).
-.TP 1i
-assert
-Program assertion (integer).
-.TP 1i
-win
-Window object (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Win_start is a one-sided MPI communication synchronization call that starts an RMA access epoch for \fIwin\fP. RMA calls issued on \fIwin\fP during this epoch must
-access only windows at processes in \fIgroup\fP. Each process in \fIgroup\fP must issue a matching
-call to MPI_Win_post. MPI_Win_start
-is allowed to block until the corresponding MPI_Win_post calls have been executed, but is not required to.
-.sp
-The \fIassert\fP argument is used to provide assertions on the context of the call that may be used for various optimizations. (See Section 6.4.4 of the MPI-2 Standard.) A value of \fIassert\fP = 0 is always valid. The following assertion value is supported:
-.sp
-.TP 1i
-MPI_MODE_NOCHECK
-When this value is passed in to this call, the library assumes that
-the post call on the target has been called and it is not necessary
-for the library to check to see if such a call has been made.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Win_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-MPI_Win_post
-MPI_Win_complete
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Win_sync.3in b/ompi/mpi/man/man3/MPI_Win_sync.3in
deleted file mode 100644
index 5f4c17b7de4..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_sync.3in
+++ /dev/null
@@ -1,55 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2014 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2007-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" $COPYRIGHT$
-.TH MPI_Win_sync 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_sync\fP, \- Synchronize the private and public copies of the window
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Win_sync (MPI_Win \fIwin\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_SYNC(\fIWIN, IERROR\fP)
-	INTEGER \fIWIN, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_sync(\fIwin\fP, \fIierror\fP)
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-win
-Window object (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-\fBMPI_Win_sync\fP synchronizes the private and public window copies of \fIwin\fP. For the purposes of synchronizing the private and public window, \fBMPI_Win_sync\fP has the effect of ending and reopening an access and exposure epoch on the window (note that it does not actually end an epoch or complete any pending MPI RMA operations).
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with \fBMPI_Comm_set_errhandler\fP; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
diff --git a/ompi/mpi/man/man3/MPI_Win_test.3in b/ompi/mpi/man/man3/MPI_Win_test.3in
deleted file mode 100644
index ffdeff393fe..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_test.3in
+++ /dev/null
@@ -1,73 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2007-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Win_test 3 "#OMPI_DATE#" ""#PACKAGE_VERSION#"" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_test\fP \- Attempts to complete an RMA exposure epoch; a nonblocking version of MPI_Win_wait
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Win_test(MPI_Win \fIwin\fP, int *\fIflag\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_TEST(\fI WIN, FLAG, IERROR\fP)
-	INTEGER \fI WIN, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_test(\fIwin\fP, \fIflag\fP, \fIierror\fP)
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	LOGICAL, INTENT(OUT) :: \fIflag\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-win
-Window object (handle)
-
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-.TP 1i
-flag
-The returning state of the test for epoch closure.
-
-
-.SH DESCRIPTION
-.ft R
-MPI_Win_test is a one-sided MPI communication synchronization call, a
-nonblocking version of MPI_Win_wait. It returns \fIflag = true\fP if
-MPI_Win_wait would return, \fIflag = false\fP otherwise. The effect of return of MPI_Win_test with \fIflag = true\fP is the same as the effect of a return of MPI_Win_wait. If \fIflag = false\fP is returned, then the call has no visible effect.
-.sp
-Invoke MPI_Win_test only where MPI_Win_wait can be invoked. Once
-the call has returned \fIflag = true\fP, it must not be invoked anew, until the window is posted anew.
-
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Win_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-MPI_Win_post
-MPI_Win_wait
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Win_unlock.3in b/ompi/mpi/man/man3/MPI_Win_unlock.3in
deleted file mode 100644
index 03623696c78..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_unlock.3in
+++ /dev/null
@@ -1,69 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2014 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2007-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Win_unlock 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_unlock\fP \- Completes an RMA access epoch started by a call to MPI_Win_lock.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Win_unlock(int \fIrank\fP, MPI_Win \fIwin\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_UNLOCK(\fIRANK, WIN, IERROR\fP)
-	INTEGER \fIRANK, WIN, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_unlock(\fIrank\fP, \fIwin\fP, \fIierror\fP)
-	INTEGER, INTENT(IN) :: \fIrank\fP
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-rank
-Rank of window (nonnegative integer).
-.TP 1i
-win
-Window object (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Win_unlock completes an RMA access epoch started by a call to MPI_Win_lock. RMA operations issued during this period will have completed both at the origin and at the target when the call returns.
-.sp
-Locks are used to protect accesses to the locked target window effected by RMA calls issued between the lock and unlock call, and to protect local load/store accesses to a locked local window executed between the lock and unlock call. Accesses that are protected by an exclusive lock will not be concurrent at the window site with other accesses to the same window that are lock protected. Accesses that are protected by a shared lock will not be concurrent at the window site with accesses protected by an exclusive lock to the same window.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-MPI_Win_lock
-MPI_Win_unlock_all
-.br
-
-
diff --git a/ompi/mpi/man/man3/MPI_Win_unlock_all.3in b/ompi/mpi/man/man3/MPI_Win_unlock_all.3in
deleted file mode 100644
index 480fe0dbc05..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_unlock_all.3in
+++ /dev/null
@@ -1,62 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2014 Los Alamos National Security, LLC. All rights reserved.
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2007-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" $COPYRIGHT$
-.TH MPI_Win_unlock_all 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_unlock_all\fP \- Completes an RMA access epoch started by a call to MPI_Win_lock_all.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Win_unlock_all(MPI_Win \fIwin\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_UNLOCK_ALL(\fIWIN, IERROR\fP)
-	INTEGER \fIWIN, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_unlock_all(\fIwin\fP, \fIierror\fP)
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-win
-Window object (handle).
-
-.SH OUTPUT PARAMETER
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Win_unlock_all completes an RMA access epoch started by a call to MPI_Win_lock_all. RMA operations issued during this period will have completed both at the origin and at the target when the call returns.
-.sp
-Locks are used to protect accesses to the locked target window effected by RMA calls issued between the lock and unlock call, and to protect local load/store accesses to a locked local window executed between the lock and unlock call. Accesses that are protected by an exclusive lock will not be concurrent at the window site with other accesses to the same window that are lock protected. Accesses that are protected by a shared lock will not be concurrent at the window site with accesses protected by an exclusive lock to the same window.
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-MPI_Win_lock_all
-MPI_Win_unlock
-.br
diff --git a/ompi/mpi/man/man3/MPI_Win_wait.3in b/ompi/mpi/man/man3/MPI_Win_wait.3in
deleted file mode 100644
index a33e107ace9..00000000000
--- a/ompi/mpi/man/man3/MPI_Win_wait.3in
+++ /dev/null
@@ -1,66 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright 2007-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Win_wait 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Win_wait\fP \- Completes an RMA exposure epoch started by a call to MPI_Win_post on \fIwin\fP
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-int MPI_Win_wait(MPI_Win \fIwin\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-MPI_WIN_WAIT(\fI WIN, IERROR\fP)
-	INTEGER \fI WIN, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-MPI_Win_wait(\fIwin\fP, \fIierror\fP)
-	TYPE(MPI_Win), INTENT(IN) :: \fIwin\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-win
-Window object (handle).
-
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-IERROR
-Fortran only: Error status (integer).
-
-.SH DESCRIPTION
-.ft R
-MPI_Win_wait is a one-sided MPI communication synchronization call that completes an RMA exposure epoch started by a call to MPI_Win_post on \fIwin\fP. This
-call matches calls to MPI_Win_complete(\fIwin\fP) issued by each of the processes that
-were granted access to the window during this epoch. The call to MPI_Win_wait blocks
-until all matching calls to MPI_Win_complete have occurred. This guarantees that all
-these origin processes have completed their RMA accesses to the local window. When the
-call returns, all these RMA accesses will have completed at the target window.
-
-
-.SH ERRORS
-Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument.
-.sp
-Before the error value is returned, the current MPI error handler is
-called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Win_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
-
-.SH SEE ALSO
-MPI_Win_post
-.br
-
diff --git a/ompi/mpi/man/man3/MPI_Wtick.3in b/ompi/mpi/man/man3/MPI_Wtick.3in
deleted file mode 100644
index a1eb5ba5dd5..00000000000
--- a/ompi/mpi/man/man3/MPI_Wtick.3in
+++ /dev/null
@@ -1,52 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2017 Cisco Systems, Inc.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Wtick 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Wtick\fP \- Returns the resolution of MPI_Wtime.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-double MPI_Wtick()
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-DOUBLE PRECISION MPI_WTICK()
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-DOUBLE PRECISION MPI_WTICK()
-
-.fi
-.SH RETURN VALUE
-.ft R
-Time in seconds of resolution of MPI_Wtime.
-
-.SH DESCRIPTION
-.ft R
-MPI_Wtick returns the resolution of MPI_Wtime in seconds. That is, it
-returns, as a double-precision value, the number of seconds between
-successive clock ticks. For example, if the clock is implemented by
-the hardware as a counter that is incremented every millisecond, the
-value returned by MPI_Wtick should be 10^-3.
-.PP
-
-.SH NOTE
-This function does not return an error value. Consequently, the result
-of calling it before MPI_Init or after MPI_Finalize is undefined.
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Wtime
diff --git a/ompi/mpi/man/man3/MPI_Wtime.3in b/ompi/mpi/man/man3/MPI_Wtime.3in
deleted file mode 100644
index a64c4954ff8..00000000000
--- a/ompi/mpi/man/man3/MPI_Wtime.3in
+++ /dev/null
@@ -1,88 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2006-2008 Sun Microsystems, Inc.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright (c) 2020      Google, LLC. All rights reserved.
-.\" $COPYRIGHT$
-.TH MPI_Wtime 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPI_Wtime\fP \- Returns an elapsed time on the calling processor.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-double MPI_Wtime()
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-! or the older form: INCLUDE 'mpif.h'
-DOUBLE PRECISION MPI_WTIME()
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-DOUBLE PRECISION MPI_WTIME()
-
-.fi
-.SH RETURN VALUE
-.ft R
-Time in seconds since an arbitrary time in the past.
-
-.SH DESCRIPTION
-.ft R
-MPI_Wtime returns a floating-point number of seconds, representing elapsed wall-clock time since some time in the past.
-.PP
-The "time in the past" is guaranteed not to change during the life of the process. The user is responsible for converting large numbers of seconds to other units if they are preferred.
-.PP
-This function is portable (it returns seconds, not "ticks"), it allows high resolution, and carries no unnecessary baggage. One would use it like this:
-.sp
-.nf
-    {
-       double starttime, endtime;
-       starttime = MPI_Wtime();
-        \&....  stuff to be timed  \&...
-       endtime   = MPI_Wtime();
-       printf("That took %f seconds\\n",endtime-starttime);
-    }
-.fi
-.PP
-The times returned are local to the node that called them. There is no requirement that different nodes return the "same" time.
-.SH NOTES
-The boolean variable MPI_WTIME_IS_GLOBAL, a predefined attribute key that indicates whether clocks are synchronized, does not have a valid value in Open MPI, as the clocks are not guaranteed to be synchronized.
-
-.PP
-This function is intended to be a high-resolution, elapsed (or wall) clock. See MPI_Wtick to determine the resolution of MPI_Wtime.
-.PP
-On POSIX platforms, this function may utilize a timer that is cheaper
-to invoke than the gettimeofday() system call, but will fall back to
-gettimeofday() if a cheap high-resolution timer is not available.  The
-ompi_info command can be consulted to see if Open MPI supports a
-native high-resolution timer on your platform; see the value for "MPI_WTIME
-support" (or "options:mpi-wtime" when viewing the parsable
-output).  If this value is "native", a method that is likely to be
-cheaper than gettimeofday() will be used to obtain the time when
-MPI_Wtime is invoked.
-.PP
-For example, on platforms that support it, the
-.I clock_gettime()
-function will be used to obtain a monotonic clock value with whatever
-precision is supported on that platform (e.g., nanoseconds).
-.PP
-Note, too, that the MCA parameter opal_timer_require_monotonic can
-influcence this behavior.  It defaults to true, but if set to false,
-Open MPI may use a finer-grained timing mechanism (e.g., the
-RDTSC/RDTSCP clock ticks on x86_64 platforms), but is not guaranteed
-to be monotonic in some cases (e.g., if the MPI process is not bound
-to a single processor core).
-.PP
-This function does not return an error value. Consequently, the result of calling it before MPI_Init or after MPI_Finalize is undefined.
-
-.SH SEE ALSO
-MPI_Wtick
-.br
-
diff --git a/ompi/mpi/man/man3/Makefile.am b/ompi/mpi/man/man3/Makefile.am
deleted file mode 100644
index cf62ec23a76..00000000000
--- a/ompi/mpi/man/man3/Makefile.am
+++ /dev/null
@@ -1,458 +0,0 @@
-# -*- makefile -*-
-# Copyright (c) 2006-2020 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
-# Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights reserved.
-# Copyright (c) 2020      Research Organization for Information Science
-#                         and Technology (RIST).  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-include $(top_srcdir)/Makefile.ompi-rules
-
-# For simplicity of maintenance over time, we are in the slow process
-# of converting all existing nroff pages to Markdown.  As man pages
-# are converted to Markdown, move them from TEMPLATE_FILES to
-# MD_ILES.
-#
-# Just in case someone looks for it here someday, here is a
-# conveninent reference for what Markdown pandoc supports:
-#
-# https://rmarkdown.rstudio.com/authoring_pandoc_markdown.html
-
-MD_FILES = \
-        MPI_Barrier.md \
-        MPI_Bsend.md \
-        MPI_Bsend_init.md \
-        MPI_Buffer_attach.md \
-        MPI_Buffer_detach.md \
-        MPI_Cancel.md \
-        MPI_Cart_coords.md \
-        MPI_Cart_create.md \
-        MPI_Cart_get.md \
-        MPI_Cart_map.md \
-        MPI_Cart_rank.md \
-        MPI_Cart_shift.md \
-        MPI_Close_port.md \
-        MPI_Cart_sub.md \
-        MPI_Cartdim_get.md \
-        MPI_Comm_accept.md \
-        MPI_Comm_call_errhandler.md \
-        MPI_Comm_compare.md \
-        MPI_Comm_connect.md \
-        MPI_Comm_create.md \
-        MPI_Comm_create_errhandler.md \
-        MPI_Comm_create_group.md \
-        MPI_Comm_create_keyval.md \
-        MPI_Comm_delete_attr.md \
-        MPI_Comm_disconnect.md \
-        MPI_Comm_dup.md \
-        MPI_Comm_dup_with_info.md \
-        MPI_T_init_thread.3.md \
-        MPI_Status_f2c.3.md \
-        MPI_Status_f082c.3.md \
-        MPI_Status_f082f.3.md \
-        MPI_Bcast.md \
-        MPI_File_write_ordered.md \
-        MPI_File_write_ordered_begin.md \
-        MPI_File_write_ordered_end.md \
-        MPI_File_write_shared.md \
-        MPI_Finalize.md \
-        MPI_Finalized.md \
-        MPI_Free_mem.md \
-        MPI_Gather.md \
-        MPI_Gatherv.md \
-        MPI_Get.md \
-        MPI_Get_accumulate.md \
-        MPI_Get_address.md \
-        MPI_Get_count.md \
-        MPI_Get_elements.md \
-        MPI_Get_library_version.md \
-        MPI_Get_processor_name.md \
-        MPI_Get_version.md \
-        MPI_Graph_create.md \
-        MPI_Graph_get.md \
-        MPI_Graph_map.md \
-        MPI_Graph_neighbors_count.md \
-        MPI_Graphdims_get.md \
-        MPI_Grequest_complete.md \
-        MPI_Grequest_start.md \
-        MPI_Group_compare.md \
-        MPI_Group_difference.md \
-        MPI_Group_excl.md \
-        MPI_Group_free.md \
-        MPI_Group_incl.md \
-        MPI_Group_intersection.md
-
-TEMPLATE_FILES = \
-        MPI_Abort.3in \
-        MPI_Accumulate.3in \
-        MPI_Add_error_class.3in \
-        MPI_Add_error_code.3in \
-        MPI_Add_error_string.3in \
-        MPI_Address.3in \
-	MPI_Aint_add.3in \
-	MPI_Aint_diff.3in \
-        MPI_Allgather.3in \
-        MPI_Iallgather.3in \
-        MPI_Allgatherv.3in \
-        MPI_Iallgatherv.3in \
-        MPI_Alloc_mem.3in \
-        MPI_Allreduce.3in \
-        MPI_Iallreduce.3in \
-        MPI_Alltoall.3in \
-        MPI_Ialltoall.3in \
-        MPI_Alltoallv.3in \
-        MPI_Ialltoallv.3in \
-        MPI_Alltoallw.3in \
-        MPI_Ialltoallw.3in \
-        MPI_Attr_delete.3in \
-        MPI_Attr_get.3in \
-        MPI_Attr_put.3in \
-        MPI_Ibarrier.3in \
-        MPI_Ibcast.3in \
-        MPI_Comm_c2f.3in \
-        MPI_Comm_idup.3in \
-        MPI_Comm_f2c.3in \
-        MPI_Comm_free.3in \
-        MPI_Comm_free_keyval.3in \
-        MPI_Comm_get_attr.3in \
-        MPI_Comm_get_errhandler.3in \
-        MPI_Comm_get_info.3in \
-        MPI_Comm_get_name.3in \
-        MPI_Comm_get_parent.3in \
-        MPI_Comm_group.3in \
-        MPI_Comm_join.3in \
-        MPI_Comm_rank.3in \
-        MPI_Comm_remote_group.3in \
-        MPI_Comm_remote_size.3in \
-        MPI_Comm_set_attr.3in \
-        MPI_Comm_set_errhandler.3in \
-        MPI_Comm_set_info.3in \
-        MPI_Comm_set_name.3in \
-        MPI_Comm_size.3in \
-        MPI_Comm_spawn.3in \
-        MPI_Comm_spawn_multiple.3in \
-        MPI_Comm_split.3in \
-        MPI_Comm_split_type.3in \
-        MPI_Comm_test_inter.3in \
-        MPI_Compare_and_swap.3in \
-        MPI_Dims_create.3in \
-        MPI_Dist_graph_create.3in \
-        MPI_Dist_graph_create_adjacent.3in \
-        MPI_Dist_graph_neighbors.3in \
-        MPI_Dist_graph_neighbors_count.3in \
-        MPI_Errhandler_create.3in \
-        MPI_Errhandler_free.3in \
-        MPI_Errhandler_get.3in \
-        MPI_Errhandler_set.3in \
-        MPI_Error_class.3in \
-        MPI_Error_string.3in \
-        MPI_Exscan.3in \
-        MPI_Iexscan.3in \
-        MPI_Fetch_and_op.3in \
-        MPI_File_c2f.3in \
-        MPI_File_call_errhandler.3in \
-        MPI_File_close.3in \
-        MPI_File_create_errhandler.3in \
-        MPI_File_delete.3in \
-        MPI_File_f2c.3in \
-        MPI_File_get_amode.3in \
-        MPI_File_get_atomicity.3in \
-        MPI_File_get_byte_offset.3in \
-        MPI_File_get_errhandler.3in \
-        MPI_File_get_group.3in \
-        MPI_File_get_info.3in \
-        MPI_File_get_position.3in \
-        MPI_File_get_position_shared.3in \
-        MPI_File_get_size.3in \
-        MPI_File_get_type_extent.3in \
-        MPI_File_get_view.3in \
-        MPI_File_iread.3in \
-        MPI_File_iread_at.3in \
-        MPI_File_iread_all.3in \
-        MPI_File_iread_at_all.3in \
-        MPI_File_iread_shared.3in \
-        MPI_File_iwrite.3in \
-        MPI_File_iwrite_at.3in \
-        MPI_File_iwrite_all.3in \
-        MPI_File_iwrite_at_all.3in \
-        MPI_File_iwrite_shared.3in \
-        MPI_File_open.3in \
-        MPI_File_preallocate.3in \
-        MPI_File_read.3in \
-        MPI_File_read_all.3in \
-        MPI_File_read_all_begin.3in \
-        MPI_File_read_all_end.3in \
-        MPI_File_read_at.3in \
-        MPI_File_read_at_all.3in \
-        MPI_File_read_at_all_begin.3in \
-        MPI_File_read_at_all_end.3in \
-        MPI_File_read_ordered.3in \
-        MPI_File_read_ordered_begin.3in \
-        MPI_File_read_ordered_end.3in \
-        MPI_File_read_shared.3in \
-        MPI_File_seek.3in \
-        MPI_File_seek_shared.3in \
-        MPI_File_set_atomicity.3in \
-        MPI_File_set_errhandler.3in \
-        MPI_File_set_info.3in \
-        MPI_File_set_size.3in \
-        MPI_File_set_view.3in \
-        MPI_File_sync.3in \
-        MPI_File_write.3in \
-        MPI_File_write_all.3in \
-        MPI_File_write_all_begin.3in \
-        MPI_File_write_all_end.3in \
-        MPI_File_write_at.3in \
-        MPI_File_write_at_all.3in \
-        MPI_File_write_at_all_begin.3in \
-        MPI_File_write_at_all_end.3in \
-        MPI_Igather.3in \
-        MPI_Igatherv.3in \
-        MPI_Get_elements_x.3in \
-        MPI_Graph_neighbors.3in \
-        MPI_Group_c2f.3in \
-        MPI_Group_f2c.3in \
-        MPI_Group_range_excl.3in \
-        MPI_Group_range_incl.3in \
-        MPI_Group_rank.3in \
-        MPI_Group_size.3in \
-        MPI_Group_translate_ranks.3in \
-        MPI_Group_union.3in \
-        MPI_Ibsend.3in \
-        MPI_Improbe.3in \
-        MPI_Imrecv.3in \
-        MPI_Info_c2f.3in \
-        MPI_Info_create.3in \
-        MPI_Info_delete.3in \
-        MPI_Info_dup.3in \
-        MPI_Info_env.3in \
-        MPI_Info_f2c.3in \
-        MPI_Info_free.3in \
-        MPI_Info_get.3in \
-        MPI_Info_get_nkeys.3in \
-        MPI_Info_get_nthkey.3in \
-        MPI_Info_get_valuelen.3in \
-        MPI_Info_set.3in \
-        MPI_Init.3in \
-        MPI_Initialized.3in \
-        MPI_Init_thread.3in \
-        MPI_Intercomm_create.3in \
-        MPI_Intercomm_merge.3in \
-        MPI_Iprobe.3in \
-        MPI_Irecv.3in \
-        MPI_Irsend.3in \
-        MPI_Isend.3in \
-        MPI_Issend.3in \
-        MPI_Is_thread_main.3in \
-        MPI_Keyval_create.3in \
-        MPI_Keyval_free.3in \
-        MPI_Lookup_name.3in \
-        MPI_Message_c2f.3in \
-        MPI_Message_f2c.3in \
-        MPI_Mprobe.3in \
-        MPI_Mrecv.3in \
-        MPI_Neighbor_allgather.3in \
-        MPI_Ineighbor_allgather.3in \
-        MPI_Neighbor_allgatherv.3in \
-        MPI_Ineighbor_allgatherv.3in \
-        MPI_Neighbor_alltoall.3in \
-        MPI_Ineighbor_alltoall.3in \
-        MPI_Neighbor_alltoallv.3in \
-        MPI_Ineighbor_alltoallv.3in \
-        MPI_Neighbor_alltoallw.3in \
-        MPI_Ineighbor_alltoallw.3in \
-        MPI_Op_c2f.3in \
-        MPI_Op_commutative.3in \
-        MPI_Op_create.3in \
-        MPI_Open_port.3in \
-        MPI_Op_f2c.3in \
-        MPI_Op_free.3in \
-        MPI_Pack.3in \
-        MPI_Pack_external.3in \
-        MPI_Pack_external_size.3in \
-        MPI_Pack_size.3in \
-        MPI_Pcontrol.3in \
-        MPI_Probe.3in \
-        MPI_Publish_name.3in \
-        MPI_Put.3in \
-        MPI_Query_thread.3in \
-        MPI_Raccumulate.3in \
-        MPI_Recv.3in \
-        MPI_Recv_init.3in \
-        MPI_Reduce.3in \
-        MPI_Ireduce.3in \
-        MPI_Reduce_local.3in \
-        MPI_Reduce_scatter.3in \
-        MPI_Ireduce_scatter.3in \
-        MPI_Reduce_scatter_block.3in \
-        MPI_Ireduce_scatter_block.3in \
-        MPI_Register_datarep.3in \
-        MPI_Request_c2f.3in \
-        MPI_Request_f2c.3in \
-        MPI_Request_free.3in \
-        MPI_Request_get_status.3in \
-        MPI_Rget.3in \
-        MPI_Rget_accumulate.3in \
-        MPI_Rput.3in \
-        MPI_Rsend.3in \
-        MPI_Rsend_init.3in \
-        MPI_Scan.3in \
-        MPI_Iscan.3in \
-        MPI_Scatter.3in \
-        MPI_Iscatter.3in \
-        MPI_Scatterv.3in \
-        MPI_Iscatterv.3in \
-        MPI_Send.3in \
-        MPI_Send_init.3in \
-        MPI_Sendrecv.3in \
-        MPI_Sendrecv_replace.3in \
-        MPI_Sizeof.3in \
-        MPI_Ssend.3in \
-        MPI_Ssend_init.3in \
-        MPI_Start.3in \
-        MPI_Startall.3in \
-        MPI_Status_c2f08.3in \
-        MPI_Status_c2f.3in \
-        MPI_Status_f2f08.3in \
-        MPI_Status_set_cancelled.3in \
-        MPI_Status_set_elements.3in \
-        MPI_Status_set_elements_x.3in \
-        MPI_T_category_changed.3in \
-        MPI_T_category_get_categories.3in \
-        MPI_T_category_get_cvars.3in \
-        MPI_T_category_get_info.3in \
-        MPI_T_category_get_num.3in \
-        MPI_T_category_get_pvars.3in \
-        MPI_T_cvar_get_info.3in \
-        MPI_T_cvar_get_num.3in \
-        MPI_T_cvar_handle_alloc.3in \
-        MPI_T_cvar_handle_free.3in \
-        MPI_T_cvar_read.3in \
-        MPI_T_cvar_write.3in \
-        MPI_T_enum_get_info.3in \
-        MPI_T_enum_get_item.3in \
-        MPI_T_finalize.3in \
-        MPI_T_pvar_get_info.3in \
-        MPI_T_pvar_get_num.3in \
-        MPI_T_pvar_handle_alloc.3in \
-        MPI_T_pvar_handle_free.3in \
-        MPI_T_pvar_read.3in \
-        MPI_T_pvar_readreset.3in \
-        MPI_T_pvar_reset.3in \
-        MPI_T_pvar_session_create.3in \
-        MPI_T_pvar_session_free.3in \
-        MPI_T_pvar_start.3in \
-        MPI_T_pvar_stop.3in \
-        MPI_T_pvar_write.3in \
-        MPI_Test.3in \
-        MPI_Testall.3in \
-        MPI_Testany.3in \
-        MPI_Test_cancelled.3in \
-        MPI_Testsome.3in \
-        MPI_Topo_test.3in \
-        MPI_Type_c2f.3in \
-        MPI_Type_commit.3in \
-        MPI_Type_contiguous.3in \
-        MPI_Type_create_darray.3in \
-        MPI_Type_create_f90_complex.3in \
-        MPI_Type_create_f90_integer.3in \
-        MPI_Type_create_f90_real.3in \
-        MPI_Type_create_hindexed.3in \
-        MPI_Type_create_hindexed_block.3in \
-        MPI_Type_create_hvector.3in \
-        MPI_Type_create_indexed_block.3in \
-        MPI_Type_create_keyval.3in \
-        MPI_Type_create_resized.3in \
-        MPI_Type_create_struct.3in \
-        MPI_Type_create_subarray.3in \
-        MPI_Type_delete_attr.3in \
-        MPI_Type_dup.3in \
-        MPI_Type_extent.3in \
-        MPI_Type_f2c.3in \
-        MPI_Type_free.3in \
-        MPI_Type_free_keyval.3in \
-        MPI_Type_get_attr.3in \
-        MPI_Type_get_contents.3in \
-        MPI_Type_get_envelope.3in \
-        MPI_Type_get_extent.3in \
-        MPI_Type_get_extent_x.3in \
-        MPI_Type_get_name.3in \
-        MPI_Type_get_true_extent.3in \
-        MPI_Type_get_true_extent_x.3in \
-        MPI_Type_hindexed.3in \
-        MPI_Type_hvector.3in \
-        MPI_Type_indexed.3in \
-        MPI_Type_lb.3in \
-        MPI_Type_match_size.3in \
-        MPI_Type_set_attr.3in \
-        MPI_Type_set_name.3in \
-        MPI_Type_size.3in \
-        MPI_Type_size_x.3in \
-        MPI_Type_struct.3in \
-        MPI_Type_ub.3in \
-        MPI_Type_vector.3in \
-        MPI_Unpack.3in \
-        MPI_Unpack_external.3in \
-        MPI_Unpublish_name.3in \
-        MPI_Wait.3in \
-        MPI_Waitall.3in \
-        MPI_Waitany.3in \
-        MPI_Waitsome.3in \
-        MPI_Win_allocate.3in \
-        MPI_Win_allocate_shared.3in \
-        MPI_Win_attach.3in \
-        MPI_Win_c2f.3in \
-        MPI_Win_call_errhandler.3in \
-        MPI_Win_complete.3in \
-        MPI_Win_create.3in \
-        MPI_Win_create_dynamic.3in \
-        MPI_Win_create_errhandler.3in \
-        MPI_Win_create_keyval.3in \
-        MPI_Win_delete_attr.3in \
-        MPI_Win_detach.3in \
-        MPI_Win_f2c.3in \
-        MPI_Win_fence.3in \
-        MPI_Win_flush.3in \
-        MPI_Win_flush_all.3in \
-        MPI_Win_flush_local.3in \
-        MPI_Win_flush_local_all.3in \
-        MPI_Win_free.3in \
-        MPI_Win_free_keyval.3in \
-        MPI_Win_get_attr.3in \
-        MPI_Win_get_errhandler.3in \
-        MPI_Win_get_group.3in \
-        MPI_Win_get_info.3in \
-        MPI_Win_get_name.3in \
-        MPI_Win_lock.3in \
-        MPI_Win_lock_all.3in \
-        MPI_Win_post.3in \
-        MPI_Win_set_attr.3in \
-        MPI_Win_set_errhandler.3in \
-        MPI_Win_set_info.3in \
-        MPI_Win_set_name.3in \
-        MPI_Win_shared_query.3in \
-        MPI_Win_start.3in \
-        MPI_Win_sync.3in \
-        MPI_Win_test.3in \
-        MPI_Win_unlock.3in \
-        MPI_Win_unlock_all.3in \
-        MPI_Win_wait.3in \
-        MPI_Wtick.3in \
-        MPI_Wtime.3in
-
-man_pages_from_md = $(MD_FILES:.3.md=.3)
-man_pages_from_template = $(TEMPLATE_FILES:.3in=.3)
-
-if OPAL_ENABLE_MAN_PAGES
-man_MANS = $(man_pages_from_md) $(man_pages_from_template)
-MAINTAINERCLEANFILES = $(man_pages_from_md)
-endif
-
-EXTRA_DIST = \
-        $(MD_FILES) $(man_pages_from_md) \
-        $(TEMPLATE_FILES) $(man_pages_from_template)
diff --git a/ompi/mpi/man/man5/.gitignore b/ompi/mpi/man/man5/.gitignore
deleted file mode 100644
index 5d06cd46646..00000000000
--- a/ompi/mpi/man/man5/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-*.5
diff --git a/ompi/mpi/man/man5/MPI.5 b/ompi/mpi/man/man5/MPI.5
deleted file mode 100644
index 0c58931b0a5..00000000000
--- a/ompi/mpi/man/man5/MPI.5
+++ /dev/null
@@ -1 +0,0 @@
-.so man5/Open-MPI.5
diff --git a/ompi/mpi/man/man5/MPI_T.5.md b/ompi/mpi/man/man5/MPI_T.5.md
deleted file mode 100644
index a2793cfbac4..00000000000
--- a/ompi/mpi/man/man5/MPI_T.5.md
+++ /dev/null
@@ -1,82 +0,0 @@
-# NAME
-
-Open MPI's MPI_T interface - General information
-
-# DESCRIPTION
-
-There are a few Open MPI-specific notes worth mentioning about its `MPI_T` interface implementation.
-
-## MPI_T Control Variables
-
-Open MPI's implementation of the `MPI_T` Control Variable ("cvar") APIs is an interface to Open MPI's underlying Modular Component Architecture (MCA) parameters/variables.  Simply put: using the `MPI_T` cvar interface is another mechanism to get/set Open MPI MCA parameters.
-
-In order of precedence (highest to lowest), Open MPI provides the following mechanisms to set MCA parameters:
-
-1. The `MPI_T` interface has the highest precedence.  Specifically: values set via the `MPI_T` interface will override all other settings.
-1. The `mpirun(1)` / `mpiexec(1)` command line (e.g., via the `--mca` parameter).
-1. Environment variables.
-1. Parameter files have the lowest precedence.  Specifically: values set via parameter files can be overridden by any of the other MCA-variable setting mechanisms.
-
-## MPI initialization
-
-An application may use the `MPI_T` interface before MPI is initialized to set MCA parameters.  Setting MPI-level MCA parameters before MPI is initialized may affect _how_ MPI is initialized (e.g., by influencing which frameworks and components are selected).
-
-The following example sets the `pml` and `btl` MCA params before invoking `MPI_Init(3)` in order to force a specific selection of PML and BTL components:
-
-```c
-int provided, index, count;
-MPI_T_cvar_handle pml_handle, btl_handle;
-char pml_value[64], btl_value[64];
-
-MPI_T_init_thread(MPI_THREAD_SINGLE, &provided);
-
-MPI_T_cvar_get_index("pml", &index);
-MPI_T_cvar_handle_alloc(index, NULL, &pml_handle, &count);
-MPI_T_cvar_write(pml_handle, "ob1");
-
-MPI_T_cvar_get_index("btl", &index);
-MPI_T_cvar_handle_alloc(index, NULL, &btl_handle, &count);
-MPI_T_cvar_write(btl_handle, "tcp,vader,self");
-
-MPI_T_cvar_read(pml_handle, pml_value);
-MPI_T_cvar_read(btl_handle, btl_value);
-printf("Set value of cvars: PML: %s, BTL: %s\n",
-       pml_value, btl_value);
-
-MPI_T_cvar_handle_free(&pml_handle);
-MPI_T_cvar_handle_free(&btl_handle);
-
-MPI_Init(NULL, NULL);
-
-// ...
-
-MPI_Finalize();
-
-MPI_T_finalize();
-```
-
-Note that once MPI is initialized, most Open MPI cvars become read-only.
-
-For example, after MPI is initialized, it is no longer possible to set the PML and BTL selection mechanisms.  This is because many of these MCA parameters are only used during MPI initialization; setting them after MPI has already been initialized would be meaningless, anyway.
-
-## MPI_T Categories
-
-Open MPI's MPI_T categories are organized hierarchically:
-
-1. Layer (or "project").  There are two layers in Open MPI:
-    * `ompi`: This layer contains cvars, pvars, and sub categories related to MPI characteristics.
-    * `opal`: This layer generally contains cvars, pvars, and sub categories of lower-layer constructions, such as operating system issues, networking issues, etc.
-2. Framework or section.
-    * In most cases, the next level in the hierarchy is the Open MPI MCA framework.
-        * For example, you can find the `btl` framework under the `opal` layer (because it has to do with the underlying networking).
-        * Additionally, the `pml` framework is under the `ompi` layer (because it has to do with MPI semantics of point-to-point messaging).
-    * There are a few non-MCA-framework entities under the layer, however.
-        * For example, there is an `mpi` section under both the `opal` and `ompi` layers for general/core MPI constructs.
-3. Component.
-    * If relevant, the third level in the hierarchy is the MCA component.
-    * For example, the `tcp` component can be found under the `opal` framework in the `opal` layer.
-
-# SEE ALSO
-
-[`MPI_T_init`(3)](MPI_T_init.html),
-[`MPI_T_finalize`(3)](MPI_T_finalize.html)
diff --git a/ompi/mpi/man/man5/Makefile.am b/ompi/mpi/man/man5/Makefile.am
deleted file mode 100644
index 8e7e8679c06..00000000000
--- a/ompi/mpi/man/man5/Makefile.am
+++ /dev/null
@@ -1,33 +0,0 @@
-# -*- makefile -*-
-#
-# Copyright (c) 2020 Cisco Systems, Inc.  All rights reserved.
-#
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-include $(top_srcdir)/Makefile.ompi-rules
-
-# Just in case someone looks for it here someday, here is a
-# conveninent reference for what Markdown pandoc supports:
-#
-# https://rmarkdown.rstudio.com/authoring_pandoc_markdown.html
-MD_FILES = \
-        MPI_T.5.md \
-        Open-MPI.5.md
-
-native_nroff_files = \
-        MPI.5 \
-        OpenMPI.5
-
-man_pages_from_md = $(MD_FILES:.5.md=.5)
-
-if OPAL_ENABLE_MAN_PAGES
-man_MANS = $(man_pages_from_md) $(native_nroff_files)
-MAINTAINERCLEANFILES = $(man_pages_from_md)
-endif
-
-EXTRA_DIST = $(MD_FILES) $(man_pages_from_md) $(native_nroff_files)
diff --git a/ompi/mpi/man/man5/Open-MPI.5.md b/ompi/mpi/man/man5/Open-MPI.5.md
deleted file mode 100644
index 78c97399f45..00000000000
--- a/ompi/mpi/man/man5/Open-MPI.5.md
+++ /dev/null
@@ -1,145 +0,0 @@
-# NAME
-
-Open MPI - General information
-
-# OPEN MPI
-
-Open MPI is an open source implementation of MPI (message-passing
-interface), the industry-standard specification for writing
-message-passing programs. Message passing is a programming model that
-gives the programmer explicit control over interprocess communication.
-
-The MPI specification was developed by the MPI Forum, a group of
-software developers, computer vendors, academics, and computer-science
-researchers whose goal was to develop a standard for writing
-message-passing programs that would be efficient, flexible, and
-portable.
-
-The outcome, known as the MPI Standard, was first published in 1993; its
-most recent version (MPI-3.1) was published in June 2015. Open MPI
-includes all MPI 3.1-compliant routines.
-
-For more information about Open MPI, see
-[https://www.open-mpi.org](https://www.open-mpi.org).
-
-The MPI standards are available at
-[https://www.mpi-forum.org](https://www.mpi-forum.org).
-
-# MAN PAGE SYNTAX
-
-Man pages for Open MPI and Open MPI I/O routines are named according to
-C syntax, that is, they begin with the prefix `MPI_`, all in
-uppercase, and the first letter following the `MPI_` prefix is also
-uppercase. The rest of the letters in the routine are all lowercase, for
-example, `MPI_Comm_get_attr`.
-
-# ENVIRONMENT
-
-To fine-tune your Open MPI environment, you can either use arguments
-to the `mpirun` or `mpiexec` commands, or you can use MCA parameters.
-
-For more information on arguments, see the `mpirun`(1) man page.
-
-For a complete listing of MCA parameters and their descriptions, issue
-the command `ompi_info --all`. This information also appears in the
-FAQ on the Open MPI web site at
-[https://www.open-mpi.org/faq/?category=tuning#mca-params](https://www.open-mpi.org/faq/?category=tuning#mca-params).
-
-# ERRORS
-
-All MPI routines (except `MPI_Wtime` and `MPI_Wtick`) return an error
-value; C routines as the value of the function and Fortran routines in
-the last argument. Before the value is returned, the current MPI error
-handler is called. By default, this error handler aborts the MPI job.
-The error handler may be changed with `MPI_Comm_set_errhandler`; the
-predefined error handler `MPI_ERRORS_RETURN` may be used to cause error
-values to be returned. Note that MPI does not guarantee that an MPI
-program can continue past an error.
-
-For more information on Open MPI error codes, see `mpi.h` in the
-`include` directory.
-
-Standard error return classes for Open MPI:
-
-| Error name | Error value | Description |
-|---|---|---|
-| MPI_SUCCESS | 0 | Successful return code. |
-| MPI_ERR_BUFFER | 1 | Invalid buffer pointer. |
-| MPI_ERR_COUNT | 2 | Invalid count argument. |
-| MPI_ERR_TYPE | 3 | Invalid datatype argument. |
-| MPI_ERR_TAG | 4 | Invalid tag argument. |
-| MPI_ERR_COMM | 5 | Invalid communicator. |
-| MPI_ERR_RANK | 6 | Invalid rank. |
-| MPI_ERR_REQUEST | 7 | Invalid MPI_Request handle. |
-| MPI_ERR_ROOT | 8 | Invalid root. |
-| MPI_ERR_GROUP | 9 | Null group passed to function. |
-| MPI_ERR_OP | 10 | Invalid operation. |
-| MPI_ERR_TOPOLOGY | 11 | Invalid topology. |
-| MPI_ERR_DIMS | 12 | Illegal dimension argument. |
-| MPI_ERR_ARG | 13 | Invalid argument. |
-| MPI_ERR_UNKNOWN | 14 | Unknown error. |
-| MPI_ERR_TRUNCATE | 15 | Message truncated on receive. |
-| MPI_ERR_OTHER | 16 | Other error; use Error_string. |
-| MPI_ERR_INTERN | 17 | Internal error code. |
-| MPI_ERR_IN_STATUS | 18 | Look in status for error value. |
-| MPI_ERR_PENDING | 19 | Pending request. |
-| MPI_ERR_ACCESS | 20 | Permission denied. |
-| MPI_ERR_AMODE | 21 | Unsupported amode passed to open. |
-| MPI_ERR_ASSERT | 22 | Invalid assert. |
-| MPI_ERR_BAD_FILE | 23 | Invalid file name (for example, path name too long). |
-| MPI_ERR_BASE | 24 | Invalid base. |
-| MPI_ERR_CONVERSION | 25 | An error occurred in a user-supplied data-conversion function. |
-| MPI_ERR_DISP | 26 | Invalid displacement. |
-| MPI_ERR_DUP_DATAREP | 27 | Conversion functions could not be registered because a data representation identifier that was already defined was passed to MPI_REGISTER_DATAREP. |
-| MPI_ERR_FILE_EXISTS | 28 | File exists. |
-| MPI_ERR_FILE_IN_USE | 29 | File operation could not be completed, as the file is currently open by some process. |
-| MPI_ERR_FILE | 30 | Invalid file handle. |
-| MPI_ERR_INFO_KEY | 31 | Illegal info key. |
-| MPI_ERR_INFO_NOKEY | 32 | No such key. |
-| MPI_ERR_INFO_VALUE | 33 | Illegal info value. |
-| MPI_ERR_INFO | 34 | Invalid info object. |
-| MPI_ERR_IO | 35 | I/O error. |
-| MPI_ERR_KEYVAL | 36 | Illegal key value. |
-| MPI_ERR_LOCKTYPE | 37 | Invalid locktype. |
-| MPI_ERR_NAME | 38 | Name not found. |
-| MPI_ERR_NO_MEM | 39 | Memory exhausted. |
-| MPI_ERR_NOT_SAME | 40 | Collective argument not identical on all processes, or collective routines called in a different order by different processes. |
-| MPI_ERR_NO_SPACE | 41 | Not enough space. |
-| MPI_ERR_NO_SUCH_FILE | 42 | File (or directory) does not exist. |
-| MPI_ERR_PORT | 43 | Invalid port. |
-| MPI_ERR_PROC_ABORTED | 74 | Operation failed because a remote peer has aborted. |
-| MPI_ERR_QUOTA | 44 | Quota exceeded. |
-| MPI_ERR_READ_ONLY | 45 | Read-only file system. |
-| MPI_ERR_RMA_CONFLICT | 46 | Conflicting accesses to window. |
-| MPI_ERR_RMA_SYNC | 47 | Erroneous RMA synchronization. |
-| MPI_ERR_SERVICE | 48 | Invalid publish/unpublish. |
-| MPI_ERR_SIZE | 49 | Invalid size. |
-| MPI_ERR_SPAWN | 50 | Error spawning. |
-| MPI_ERR_UNSUPPORTED_DATAREP | 51 | Unsupported datarep passed to MPI_File_set_view. |
-| MPI_ERR_UNSUPPORTED_OPERATION | 52 | Unsupported operation, such as seeking on a file that supports only sequential access. |
-| MPI_ERR_WIN | 53 | Invalid window. |
-| MPI_T_ERR_MEMORY | 54 | Out of memory. |
-| MPI_T_ERR_NOT_INITIALIZED | 55 | Interface not initialized. |
-| MPI_T_ERR_CANNOT_INIT | 56 | Interface not in the state to be initialized. |
-| MPI_T_ERR_INVALID_INDEX | 57 | The enumeration index is invalid. |
-| MPI_T_ERR_INVALID_ITEM| 58 | The item index queried is out of range. |
-| MPI_T_ERR_INVALID_HANDLE | 59 | The handle is invalid. |
-| MPI_T_ERR_OUT_OF_HANDLES | 60 | No more handles available. |
-| MPI_T_ERR_OUT_OF_SESSIONS | 61 | No more sessions available. |
-| MPI_T_ERR_INVALID_SESSION | 62 | Session argument is not a valid session. |
-| MPI_T_ERR_CVAR_SET_NOT_NOW | 63 |Variable cannot be set at this moment. |
-| MPI_T_ERR_CVAR_SET_NEVER | 64 | Variable cannot be set until end of execution. |
-| MPI_T_ERR_PVAR_NO_STARTSTOP | 65 | Variable cannot be started or stopped. |
-| MPI_T_ERR_PVAR_NO_WRITE | 66 | Variable cannot be written or reset. |
-| MPI_T_ERR_PVAR_NO_ATOMIC | 67 | Variable cannot be read and written atomically. |
-| MPI_ERR_RMA_RANGE | 68 | Target memory is not part of the window (in the case of a window created with MPI_WIN_CREATE_DYNAMIC, target memory is not attached). |
-| MPI_ERR_RMA_ATTACH | 69 | Memory cannot be attached (e.g., because of resource exhaustion). |
-| MPI_ERR_RMA_FLAVOR | 70 | Passed window has the wrong flavor for the called function. |
-| MPI_ERR_RMA_SHARED | 71 | Memory cannot be shared (e.g., some process in the group of the specified communicator cannot expose shared memory). |
-| MPI_T_ERR_INVALID | 72 | Invalid use of the interface or bad parameter values(s). |
-| MPI_T_ERR_INVALID_NAME | 73 | The variable or category name is invalid. |
-| MPI_ERR_LASTCODE | 93 | Last error code. |
-
-# SEE ALSO
-
-MPI_T(5)
diff --git a/ompi/mpi/man/man5/OpenMPI.5 b/ompi/mpi/man/man5/OpenMPI.5
deleted file mode 100644
index 0c58931b0a5..00000000000
--- a/ompi/mpi/man/man5/OpenMPI.5
+++ /dev/null
@@ -1 +0,0 @@
-.so man5/Open-MPI.5
diff --git a/ompi/mpi/tool/Makefile.am b/ompi/mpi/tool/Makefile.am
index c749cb3eb5a..72cb71b9b60 100644
--- a/ompi/mpi/tool/Makefile.am
+++ b/ompi/mpi/tool/Makefile.am
@@ -4,7 +4,11 @@
 #                         reserved.
 # Copyright (c) 2004-2006 High Performance Computing Center Stuttgart,
 #                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2012-2014 Los Alamos National Security, LLC. All rights
+# Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
+#                         reserved.
+# Copyright (c) 2018-2025 Triad National Security, LLC. All rights
+#                         reserved.
+# Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
 #                         reserved.
 # $COPYRIGHT$
 #
@@ -13,36 +17,94 @@
 # $HEADER$
 #
 
-SUBDIRS = profile
-
-AM_CPPFLAGS = -DOMPI_PROFILING_DEFINES=0
-
-noinst_LTLIBRARIES = libmpi_mpit_common.la
+# The purpose of the profiling layer is to allow intercept libraries
+# which override the MPI_ namespace symbols.  We potentially compile
+# every MPI function twice.  We always build the profiling layer,
+# because the symbols that are always implemented as functions are the
+# PMPI_ namespace symbols.  We sometimes also build the non-profiling
+# layer, if weak symbols can't be used to alias the MPI_ namespace
+# into the PMPI_ namespace.
 
+noinst_LTLIBRARIES = libmpi_mpit.la libmpi_mpit_profile.la
 if BUILD_MPI_BINDINGS_LAYER
-noinst_LTLIBRARIES += libmpi_mpit.la
+noinst_LTLIBRARIES += libmpi_mpit_noprofile.la
 endif
 
 headers = mpit-internal.h
 
-libmpi_mpit_common_la_SOURCES = mpit_common.c
-
-libmpi_mpit_la_SOURCES = init_thread.c finalize.c cvar_get_num.c \
-		     cvar_get_info.c cvar_read.c cvar_write.c \
-                     cvar_handle_alloc.c cvar_handle_free.c \
-                     category_get_num.c category_get_info.c \
-                     category_get_cvars.c category_get_pvars.c \
-                     category_get_categories.c category_changed.c \
-                     pvar_get_info.c pvar_get_num.c pvar_handle_alloc.c \
-                     pvar_handle_free.c pvar_read.c pvar_readreset.c \
-                     pvar_reset.c pvar_session_create.c pvar_session_free.c \
-                     pvar_start.c pvar_stop.c pvar_write.c \
-                     enum_get_info.c enum_get_item.c cvar_get_index.c \
-                     pvar_get_index.c category_get_index.c
+# mpit_common.c is not public functions, which does not have profiling
+# implications, so they are always built.
+libmpi_mpit_la_SOURCES = \
+	mpit_common.c
+libmpi_mpit_la_LIBADD = libmpi_mpit_profile.la
+if BUILD_MPI_BINDINGS_LAYER
+libmpi_mpit_la_LIBADD += libmpi_mpit_noprofile.la
+endif
 
 # Conditionally install the header files
-
 if WANT_INSTALL_HEADERS
 ompidir = $(ompiincludedir)/$(subdir)
 ompi_HEADERS = $(headers)
 endif
+
+#
+# List of all C files that have profile versions
+#
+interface_profile_sources = \
+	category_changed.c \
+	category_get_categories.c \
+	category_get_cvars.c \
+	category_get_events.c \
+	category_get_info.c \
+	category_get_index.c \
+	category_get_num.c \
+	category_get_num_events.c \
+	category_get_pvars.c \
+	cvar_get_info.c \
+	cvar_get_index.c \
+	cvar_get_num.c \
+	cvar_handle_alloc.c \
+	cvar_handle_free.c \
+	cvar_read.c \
+	cvar_write.c \
+	enum_get_info.c \
+	enum_get_item.c \
+	event_get_index.c \
+	event_copy.c \
+	event_callback_get_info.c \
+	event_callback_set_info.c \
+	event_handle_get_info.c \
+	event_handle_set_info.c \
+	event_get_info.c \
+	event_get_num.c \
+	event_get_source.c \
+	event_get_timestamp.c \
+	event_handle_alloc.c \
+	event_handle_free.c \
+	event_read.c \
+	event_register_callback.c \
+	event_set_dropped_handler.c \
+	finalize.c \
+	init_thread.c \
+	pvar_get_info.c \
+	pvar_get_index.c \
+	pvar_get_num.c \
+	pvar_handle_alloc.c \
+	pvar_handle_free.c \
+	pvar_read.c \
+	pvar_readreset.c \
+	pvar_reset.c \
+	pvar_session_create.c \
+	pvar_session_free.c \
+	pvar_start.c \
+	pvar_stop.c \
+	pvar_write.c \
+	source_get_num.c \
+	source_get_info.c \
+	source_get_timestamp.c
+
+libmpi_mpit_profile_la_SOURCES = $(interface_profile_sources)
+libmpi_mpit_profile_la_CPPFLAGS = -DOMPI_BUILD_MPI_PROFILING=1
+
+libmpi_mpit_noprofile_la_SOURCES = $(interface_profile_sources)
+libmpi_mpit_noprofile_la_CPPFLAGS = -DOMPI_BUILD_MPI_PROFILING=0
diff --git a/ompi/mpi/tool/category_changed.c b/ompi/mpi/tool/category_changed.c
index aed854ba669..3b3dce70036 100644
--- a/ompi/mpi/tool/category_changed.c
+++ b/ompi/mpi/tool/category_changed.c
@@ -4,6 +4,8 @@
  *                         reserved.
  * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -11,17 +13,17 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "ompi/mpi/tool/mpit-internal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_category_changed = PMPI_T_category_changed
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_category_changed PMPI_T_category_changed
 #endif
 
-
 int MPI_T_category_changed(int *stamp)
 {
     if (!mpit_is_initialized ()) {
diff --git a/ompi/mpi/tool/category_get_categories.c b/ompi/mpi/tool/category_get_categories.c
index 4f898b9c9c3..813dcc34be1 100644
--- a/ompi/mpi/tool/category_get_categories.c
+++ b/ompi/mpi/tool/category_get_categories.c
@@ -7,6 +7,8 @@
  * Copyright (c) 2020      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -14,17 +16,17 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "ompi/mpi/tool/mpit-internal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_category_get_categories = PMPI_T_category_get_categories
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_category_get_categories PMPI_T_category_get_categories
 #endif
 
-
 int MPI_T_category_get_categories(int cat_index, int len, int indices[])
 {
     const mca_base_var_group_t *group;
diff --git a/ompi/mpi/tool/category_get_cvars.c b/ompi/mpi/tool/category_get_cvars.c
index 430ba7db201..bc361f6ba9b 100644
--- a/ompi/mpi/tool/category_get_cvars.c
+++ b/ompi/mpi/tool/category_get_cvars.c
@@ -7,6 +7,8 @@
  * Copyright (c) 2020      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -14,17 +16,17 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "ompi/mpi/tool/mpit-internal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_category_get_cvars = PMPI_T_category_get_cvars
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_category_get_cvars PMPI_T_category_get_cvars
 #endif
 
-
 int MPI_T_category_get_cvars(int cat_index, int len, int indices[])
 {
     const mca_base_var_group_t *group;
diff --git a/ompi/mpi/tool/category_get_events.c b/ompi/mpi/tool/category_get_events.c
new file mode 100644
index 00000000000..3c6576b79f8
--- /dev/null
+++ b/ompi/mpi/tool/category_get_events.c
@@ -0,0 +1,40 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2020      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
+ * Copyright (c) 2025      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/tool/mpit-internal.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_T_category_get_events = PMPI_T_category_get_events
+#endif
+#define MPI_T_category_get_events PMPI_T_category_get_events
+#endif
+
+int MPI_T_category_get_events(int cat_index, int len, int indices[])
+{
+    int rc = MPI_SUCCESS;
+
+    if (!mpit_is_initialized ()) {
+        return MPI_T_ERR_NOT_INITIALIZED;
+    }
+    return rc;
+}
diff --git a/ompi/mpi/tool/category_get_index.c b/ompi/mpi/tool/category_get_index.c
index df4e1946042..996c5538f35 100644
--- a/ompi/mpi/tool/category_get_index.c
+++ b/ompi/mpi/tool/category_get_index.c
@@ -7,7 +7,8 @@
  * Copyright (c) 2020      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
-
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -15,17 +16,17 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "ompi/mpi/tool/mpit-internal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_category_get_index = PMPI_T_category_get_index
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_category_get_index PMPI_T_category_get_index
 #endif
 
-
 int MPI_T_category_get_index (const char *name, int *category_index)
 {
     int ret;
diff --git a/ompi/mpi/tool/category_get_info.c b/ompi/mpi/tool/category_get_info.c
index 1ee833509be..4906c796125 100644
--- a/ompi/mpi/tool/category_get_info.c
+++ b/ompi/mpi/tool/category_get_info.c
@@ -7,6 +7,8 @@
  * Copyright (c) 2020      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -14,17 +16,17 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "ompi/mpi/tool/mpit-internal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_category_get_info = PMPI_T_category_get_info
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_category_get_info PMPI_T_category_get_info
 #endif
 
-
 int MPI_T_category_get_info(int cat_index, char *name, int *name_len,
                             char *desc, int *desc_len, int *num_cvars,
                             int *num_pvars, int *num_categories)
diff --git a/ompi/mpi/tool/category_get_num.c b/ompi/mpi/tool/category_get_num.c
index f7b19beea20..4da64da8156 100644
--- a/ompi/mpi/tool/category_get_num.c
+++ b/ompi/mpi/tool/category_get_num.c
@@ -7,6 +7,8 @@
  * Copyright (c) 2020      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -14,17 +16,17 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "ompi/mpi/tool/mpit-internal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_category_get_num = PMPI_T_category_get_num
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_category_get_num PMPI_T_category_get_num
 #endif
 
-
 int MPI_T_category_get_num (int *num_cat)
 {
     if (!mpit_is_initialized ()) {
diff --git a/ompi/mpi/tool/category_get_num_events.c b/ompi/mpi/tool/category_get_num_events.c
new file mode 100644
index 00000000000..b7e49070323
--- /dev/null
+++ b/ompi/mpi/tool/category_get_num_events.c
@@ -0,0 +1,45 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2020      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
+ * Copyright (c) 2025      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/tool/mpit-internal.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_T_category_get_num_events = PMPI_T_category_get_num_events
+#endif
+#define MPI_T_category_get_num_events PMPI_T_category_get_num_events
+#endif
+
+int MPI_T_category_get_num_events (int cat_index, int *num_events)
+{
+    if (!mpit_is_initialized ()) {
+        return MPI_T_ERR_NOT_INITIALIZED;
+    }
+
+    if (MPI_PARAM_CHECK && NULL == num_events) {
+        return MPI_T_ERR_INVALID;
+    }
+
+    *num_events = 0;
+
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/tool/category_get_pvars.c b/ompi/mpi/tool/category_get_pvars.c
index 90eedb6c86b..c4371762fb5 100644
--- a/ompi/mpi/tool/category_get_pvars.c
+++ b/ompi/mpi/tool/category_get_pvars.c
@@ -7,6 +7,8 @@
  * Copyright (c) 2020      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -14,17 +16,17 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "ompi/mpi/tool/mpit-internal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_category_get_pvars = PMPI_T_category_get_pvars
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_category_get_pvars PMPI_T_category_get_pvars
 #endif
 
-
 int MPI_T_category_get_pvars(int cat_index, int len, int indices[])
 {
     const mca_base_var_group_t *group;
diff --git a/ompi/mpi/tool/cvar_get_index.c b/ompi/mpi/tool/cvar_get_index.c
index 2de238efa2e..ee0672f0c9b 100644
--- a/ompi/mpi/tool/cvar_get_index.c
+++ b/ompi/mpi/tool/cvar_get_index.c
@@ -7,6 +7,8 @@
  * Copyright (c) 2020      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -14,17 +16,17 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "ompi/mpi/tool/mpit-internal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_cvar_get_index = PMPI_T_cvar_get_index
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_cvar_get_index PMPI_T_cvar_get_index
 #endif
 
-
 int MPI_T_cvar_get_index (const char *name, int *cvar_index)
 {
     int ret;
diff --git a/ompi/mpi/tool/cvar_get_info.c b/ompi/mpi/tool/cvar_get_info.c
index 8b5e7fa0d14..c3f94787587 100644
--- a/ompi/mpi/tool/cvar_get_info.c
+++ b/ompi/mpi/tool/cvar_get_info.c
@@ -7,6 +7,8 @@
  * Copyright (c) 2020      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -14,17 +16,17 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "ompi/mpi/tool/mpit-internal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_cvar_get_info = PMPI_T_cvar_get_info
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_cvar_get_info PMPI_T_cvar_get_info
 #endif
 
-
 int MPI_T_cvar_get_info(int cvar_index, char *name, int *name_len, int *verbosity,
 			MPI_Datatype *datatype, MPI_T_enum *enumtype, char *desc,
 			int *desc_len, int *bind, int *scope)
diff --git a/ompi/mpi/tool/cvar_get_num.c b/ompi/mpi/tool/cvar_get_num.c
index 0b852fc32be..32ba1201cdc 100644
--- a/ompi/mpi/tool/cvar_get_num.c
+++ b/ompi/mpi/tool/cvar_get_num.c
@@ -7,6 +7,8 @@
  * Copyright (c) 2020      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -14,17 +16,17 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "ompi/mpi/tool/mpit-internal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_cvar_get_num = PMPI_T_cvar_get_num
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_cvar_get_num PMPI_T_cvar_get_num
 #endif
 
-
 int MPI_T_cvar_get_num (int *num_cvar) {
     if (!mpit_is_initialized ()) {
         return MPI_T_ERR_NOT_INITIALIZED;
diff --git a/ompi/mpi/tool/cvar_handle_alloc.c b/ompi/mpi/tool/cvar_handle_alloc.c
index 0a0ecec297a..77514825a05 100644
--- a/ompi/mpi/tool/cvar_handle_alloc.c
+++ b/ompi/mpi/tool/cvar_handle_alloc.c
@@ -7,6 +7,8 @@
  * Copyright (c) 2020      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -14,17 +16,17 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "ompi/mpi/tool/mpit-internal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_cvar_handle_alloc = PMPI_T_cvar_handle_alloc
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_cvar_handle_alloc PMPI_T_cvar_handle_alloc
 #endif
 
-
 int MPI_T_cvar_handle_alloc (int cvar_index, void *obj_handle,
                              MPI_T_cvar_handle *handle, int *count)
 {
diff --git a/ompi/mpi/tool/cvar_handle_free.c b/ompi/mpi/tool/cvar_handle_free.c
index 4d64762c8b5..7ffa6c4c30a 100644
--- a/ompi/mpi/tool/cvar_handle_free.c
+++ b/ompi/mpi/tool/cvar_handle_free.c
@@ -3,6 +3,8 @@
  * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -10,17 +12,17 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "ompi/mpi/tool/mpit-internal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_cvar_handle_free = PMPI_T_cvar_handle_free
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_cvar_handle_free PMPI_T_cvar_handle_free
 #endif
 
-
 int MPI_T_cvar_handle_free (MPI_T_cvar_handle *handle)
 {
     if (!mpit_is_initialized ()) {
diff --git a/ompi/mpi/tool/cvar_read.c b/ompi/mpi/tool/cvar_read.c
index ef53e736b93..b46db2c99c4 100644
--- a/ompi/mpi/tool/cvar_read.c
+++ b/ompi/mpi/tool/cvar_read.c
@@ -8,6 +8,8 @@
  * Copyright (c) 2020      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -15,17 +17,17 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "ompi/mpi/tool/mpit-internal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_cvar_read = PMPI_T_cvar_read
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_cvar_read PMPI_T_cvar_read
 #endif
 
-
 int MPI_T_cvar_read (MPI_T_cvar_handle handle, void *buf)
 {
     const mca_base_var_storage_t *value = NULL;
diff --git a/ompi/mpi/tool/cvar_write.c b/ompi/mpi/tool/cvar_write.c
index 23731ffb601..5dadb1e48e5 100644
--- a/ompi/mpi/tool/cvar_write.c
+++ b/ompi/mpi/tool/cvar_write.c
@@ -7,6 +7,8 @@
  * Copyright (c) 2020      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -14,17 +16,17 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "ompi/mpi/tool/mpit-internal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_cvar_write = PMPI_T_cvar_write
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_cvar_write PMPI_T_cvar_write
 #endif
 
-
 int MPI_T_cvar_write (MPI_T_cvar_handle handle, const void *buf)
 {
     int rc = MPI_SUCCESS;
diff --git a/ompi/mpi/tool/enum_get_info.c b/ompi/mpi/tool/enum_get_info.c
index 441197e389f..59de6eebe80 100644
--- a/ompi/mpi/tool/enum_get_info.c
+++ b/ompi/mpi/tool/enum_get_info.c
@@ -7,6 +7,8 @@
  * Copyright (c) 2020      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -14,17 +16,17 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "ompi/mpi/tool/mpit-internal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_enum_get_info = PMPI_T_enum_get_info
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_enum_get_info PMPI_T_enum_get_info
 #endif
 
-
 int MPI_T_enum_get_info(MPI_T_enum enumtype, int *num, char *name, int *name_len)
 {
     int rc = MPI_SUCCESS;
diff --git a/ompi/mpi/tool/enum_get_item.c b/ompi/mpi/tool/enum_get_item.c
index 0ddcd34f9b4..0a6e23c806e 100644
--- a/ompi/mpi/tool/enum_get_item.c
+++ b/ompi/mpi/tool/enum_get_item.c
@@ -7,6 +7,8 @@
  * Copyright (c) 2020      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -14,17 +16,17 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "ompi/mpi/tool/mpit-internal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_enum_get_item = PMPI_T_enum_get_item
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_enum_get_item PMPI_T_enum_get_item
 #endif
 
-
 int MPI_T_enum_get_item(MPI_T_enum enumtype, int index, int *value, char *name,
                         int *name_len)
 {
diff --git a/ompi/mpi/tool/event_callback_get_info.c b/ompi/mpi/tool/event_callback_get_info.c
new file mode 100644
index 00000000000..92519a13aac
--- /dev/null
+++ b/ompi/mpi/tool/event_callback_get_info.c
@@ -0,0 +1,38 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2019      Google, LLC. All rights reserved.
+ * Copyright (c) 2019-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+
+#include "ompi_config.h"
+#include "ompi/mpi/tool/mpit-internal.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_T_event_callback_get_info = PMPI_T_event_callback_get_info
+#endif
+#define MPI_T_event_callback_get_info PMPI_T_event_callback_get_info
+#endif
+
+
+int MPI_T_event_callback_get_info (MPI_T_event_registration event_registration,
+                                   MPI_T_cb_safety cb_safety, MPI_Info *info_used)
+{
+    if (!mpit_is_initialized ()) {
+        return MPI_T_ERR_NOT_INITIALIZED;
+    }
+
+    return MPI_T_ERR_INVALID_HANDLE;
+}
+
diff --git a/ompi/mpi/tool/event_callback_set_info.c b/ompi/mpi/tool/event_callback_set_info.c
new file mode 100644
index 00000000000..5f78d2c92de
--- /dev/null
+++ b/ompi/mpi/tool/event_callback_set_info.c
@@ -0,0 +1,36 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2019      Google, LLC. All rights reserved.
+ * Copyright (c) 2019-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/tool/mpit-internal.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_T_event_callback_set_info = PMPI_T_event_callback_set_info
+#endif
+#define MPI_T_event_callback_set_info PMPI_T_event_callback_set_info
+#endif
+
+int MPI_T_event_callback_set_info (MPI_T_event_registration event_registration,
+                                   MPI_T_cb_safety cb_safety, MPI_Info info)
+{
+    if (!mpit_is_initialized ()) {
+        return MPI_T_ERR_NOT_INITIALIZED;
+    }
+
+    return MPI_T_ERR_INVALID_HANDLE;
+}
diff --git a/ompi/mpi/tool/event_copy.c b/ompi/mpi/tool/event_copy.c
new file mode 100644
index 00000000000..f0c64bad822
--- /dev/null
+++ b/ompi/mpi/tool/event_copy.c
@@ -0,0 +1,35 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2025      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/tool/mpit-internal.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_T_event_copy = PMPI_T_event_copy
+#endif
+#define MPI_T_event_copy PMPI_T_event_copy
+#endif
+
+
+int MPI_T_event_copy (MPI_T_event_instance event, void *buffer)
+{
+    if (!mpit_is_initialized ()) {
+        return MPI_T_ERR_NOT_INITIALIZED;
+    }
+
+    return MPI_T_ERR_INVALID_HANDLE;
+}
diff --git a/ompi/mpi/tool/event_get_index.c b/ompi/mpi/tool/event_get_index.c
new file mode 100644
index 00000000000..0c0ec50cf39
--- /dev/null
+++ b/ompi/mpi/tool/event_get_index.c
@@ -0,0 +1,37 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2025      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/tool/mpit-internal.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_T_event_get_index = PMPI_T_event_get_index
+#endif
+#define MPI_T_event_get_index PMPI_T_event_get_index
+#endif
+
+int MPI_T_event_get_index (const char *name, int *event_index)
+{
+    if (!mpit_is_initialized ()) {
+        return MPI_T_ERR_NOT_INITIALIZED;
+    }
+
+    if (MPI_PARAM_CHECK && (NULL == event_index || NULL == name)) {
+        return MPI_ERR_ARG;
+    }
+
+    return MPI_T_ERR_INVALID_NAME;
+}
diff --git a/ompi/mpi/tool/event_get_info.c b/ompi/mpi/tool/event_get_info.c
new file mode 100644
index 00000000000..f7fd9134379
--- /dev/null
+++ b/ompi/mpi/tool/event_get_info.c
@@ -0,0 +1,38 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2018-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/tool/mpit-internal.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_T_event_get_info = PMPI_T_event_get_info
+#endif
+#define MPI_T_event_get_info PMPI_T_event_get_info
+#endif
+
+int MPI_T_event_get_info (int event_index, char *name, int *name_len,
+                          int *verbosity, MPI_Datatype *array_of_datatypes,
+                          MPI_Aint *array_of_displacements, int *num_elements,
+                          MPI_T_enum *enumtype, MPI_Info *info,
+                          char *desc, int *desc_len, int *bind)
+{
+    if (!mpit_is_initialized ()) {
+        return MPI_T_ERR_NOT_INITIALIZED;
+    }
+
+    return MPI_T_ERR_INVALID_INDEX;
+}
diff --git a/ompi/mpi/tool/event_get_num.c b/ompi/mpi/tool/event_get_num.c
new file mode 100644
index 00000000000..0b27caa60fc
--- /dev/null
+++ b/ompi/mpi/tool/event_get_num.c
@@ -0,0 +1,38 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2025      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/tool/mpit-internal.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_T_event_get_num = PMPI_T_event_get_num
+#endif
+#define MPI_T_event_get_num PMPI_T_event_get_num
+#endif
+
+int MPI_T_event_get_num (int *num_event)
+{
+    if (!mpit_is_initialized ()) {
+        return MPI_T_ERR_NOT_INITIALIZED;
+    }
+
+    if (MPI_PARAM_CHECK && NULL == num_event) {
+        return MPI_ERR_ARG;
+    }
+
+    *num_event = 0;
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/tool/event_get_source.c b/ompi/mpi/tool/event_get_source.c
new file mode 100644
index 00000000000..7ec12af587e
--- /dev/null
+++ b/ompi/mpi/tool/event_get_source.c
@@ -0,0 +1,34 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2025      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/tool/mpit-internal.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_T_event_get_source = PMPI_T_event_get_source
+#endif
+#define MPI_T_event_get_source PMPI_T_event_get_source
+#endif
+
+int MPI_T_event_get_source (MPI_T_event_instance event, int *source_index)
+{
+    if (!mpit_is_initialized ()) {
+        return MPI_T_ERR_NOT_INITIALIZED;
+    }
+
+    return MPI_T_ERR_INVALID_HANDLE;
+}
diff --git a/ompi/mpi/tool/event_get_timestamp.c b/ompi/mpi/tool/event_get_timestamp.c
new file mode 100644
index 00000000000..34fd49d5075
--- /dev/null
+++ b/ompi/mpi/tool/event_get_timestamp.c
@@ -0,0 +1,34 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2025      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/tool/mpit-internal.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_T_event_get_timestamp = PMPI_T_event_get_timestamp
+#endif
+#define MPI_T_event_get_timestamp PMPI_T_event_get_timestamp
+#endif
+
+int MPI_T_event_get_timestamp (MPI_T_event_instance event, MPI_Count *event_time)
+{
+    if (!mpit_is_initialized ()) {
+        return MPI_T_ERR_NOT_INITIALIZED;
+    }
+
+    return MPI_T_ERR_INVALID_HANDLE;
+}
diff --git a/ompi/mpi/tool/event_handle_alloc.c b/ompi/mpi/tool/event_handle_alloc.c
new file mode 100644
index 00000000000..0768cf2f879
--- /dev/null
+++ b/ompi/mpi/tool/event_handle_alloc.c
@@ -0,0 +1,35 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2018-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/tool/mpit-internal.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_T_event_handle_alloc = PMPI_T_event_handle_alloc
+#endif
+#define MPI_T_event_handle_alloc PMPI_T_event_handle_alloc
+#endif
+
+int MPI_T_event_handle_alloc (int event_index, void *obj_handle, MPI_Info info,
+                              MPI_T_event_registration *event_registration)
+{
+    if (!mpit_is_initialized ()) {
+        return MPI_T_ERR_NOT_INITIALIZED;
+    }
+
+    return MPI_T_ERR_INVALID_INDEX;
+}
diff --git a/ompi/mpi/tool/event_handle_free.c b/ompi/mpi/tool/event_handle_free.c
new file mode 100644
index 00000000000..7fb756b3938
--- /dev/null
+++ b/ompi/mpi/tool/event_handle_free.c
@@ -0,0 +1,36 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2018-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/tool/mpit-internal.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_T_event_handle_free = PMPI_T_event_handle_free
+#endif
+#define MPI_T_event_handle_free PMPI_T_event_handle_free
+#endif
+
+int MPI_T_event_handle_free (MPI_T_event_registration event_registration,
+                             void *user_data,
+                             MPI_T_event_free_cb_function free_cb_function)
+{
+    if (!mpit_is_initialized ()) {
+        return MPI_T_ERR_NOT_INITIALIZED;
+    }
+
+    return MPI_T_ERR_INVALID_HANDLE;
+}
diff --git a/ompi/mpi/tool/event_handle_get_info.c b/ompi/mpi/tool/event_handle_get_info.c
new file mode 100644
index 00000000000..2009d67b1ef
--- /dev/null
+++ b/ompi/mpi/tool/event_handle_get_info.c
@@ -0,0 +1,36 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2019      Google, LLC. All rights reserved.
+ * Copyright (c) 2019-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/tool/mpit-internal.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_T_event_handle_get_info = PMPI_T_event_handle_get_info
+#endif
+#define MPI_T_event_handle_get_info PMPI_T_event_handle_get_info
+#endif
+
+int MPI_T_event_handle_get_info (MPI_T_event_registration event_registration,
+                                 MPI_Info *info_used)
+{
+    if (!mpit_is_initialized ()) {
+        return MPI_T_ERR_NOT_INITIALIZED;
+    }
+
+    return MPI_T_ERR_INVALID_HANDLE;
+}
diff --git a/ompi/mpi/tool/event_handle_set_info.c b/ompi/mpi/tool/event_handle_set_info.c
new file mode 100644
index 00000000000..0c35e27d2a1
--- /dev/null
+++ b/ompi/mpi/tool/event_handle_set_info.c
@@ -0,0 +1,36 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2019      Google, LLC. All rights reserved.
+ * Copyright (c) 2019-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/tool/mpit-internal.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_T_event_handle_set_info = PMPI_T_event_handle_set_info
+#endif
+#define MPI_T_event_handle_set_info PMPI_T_event_handle_set_info
+#endif
+
+int MPI_T_event_handle_set_info (MPI_T_event_registration event_registration,
+                                 MPI_Info info)
+{
+    if (!mpit_is_initialized ()) {
+        return MPI_T_ERR_NOT_INITIALIZED;
+    }
+
+    return MPI_T_ERR_INVALID_HANDLE;
+}
diff --git a/ompi/mpi/tool/event_read.c b/ompi/mpi/tool/event_read.c
new file mode 100644
index 00000000000..e16d3e33f50
--- /dev/null
+++ b/ompi/mpi/tool/event_read.c
@@ -0,0 +1,34 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2025      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/tool/mpit-internal.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_T_event_read = PMPI_T_event_read
+#endif
+#define MPI_T_event_read PMPI_T_event_read
+#endif
+
+int MPI_T_event_read (MPI_T_event_instance event, int element_index, void *buffer)
+{
+    if (!mpit_is_initialized ()) {
+        return MPI_T_ERR_NOT_INITIALIZED;
+    }
+
+    return MPI_T_ERR_INVALID_HANDLE;
+}
diff --git a/ompi/mpi/tool/event_register_callback.c b/ompi/mpi/tool/event_register_callback.c
new file mode 100644
index 00000000000..4550fabe520
--- /dev/null
+++ b/ompi/mpi/tool/event_register_callback.c
@@ -0,0 +1,36 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2018-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/tool/mpit-internal.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_T_event_register_callback = PMPI_T_event_register_callback
+#endif
+#define MPI_T_event_register_callback PMPI_T_event_register_callback
+#endif
+
+int MPI_T_event_register_callback (MPI_T_event_registration event_registration,
+                                   MPI_T_cb_safety cb_safety, MPI_Info info, void *user_data,
+                                   MPI_T_event_cb_function event_cb_function)
+{
+    if (!mpit_is_initialized ()) {
+        return MPI_T_ERR_NOT_INITIALIZED;
+    }
+
+    return MPI_T_ERR_INVALID_HANDLE;
+}
diff --git a/ompi/mpi/tool/event_set_dropped_handler.c b/ompi/mpi/tool/event_set_dropped_handler.c
new file mode 100644
index 00000000000..fe15695abb8
--- /dev/null
+++ b/ompi/mpi/tool/event_set_dropped_handler.c
@@ -0,0 +1,35 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2025      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/tool/mpit-internal.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_T_event_set_dropped_handler = PMPI_T_event_set_dropped_handler
+#endif
+#define MPI_T_event_set_dropped_handler PMPI_T_event_set_dropped_handler
+#endif
+
+int MPI_T_event_set_dropped_handler (MPI_T_event_registration handle, MPI_T_event_dropped_cb_function dropped_cb_function)
+
+{
+    if (!mpit_is_initialized ()) {
+        return MPI_T_ERR_NOT_INITIALIZED;
+    }
+
+    return MPI_T_ERR_INVALID_HANDLE;
+}
diff --git a/ompi/mpi/tool/finalize.c b/ompi/mpi/tool/finalize.c
index dce8a40d0bf..359468b494d 100644
--- a/ompi/mpi/tool/finalize.c
+++ b/ompi/mpi/tool/finalize.c
@@ -4,6 +4,8 @@
  *                         reserved.
  * Copyright (c) 2014-2018 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -11,6 +13,7 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
 
 #include "ompi/mpi/tool/mpit-internal.h"
 
@@ -18,15 +21,13 @@
 #include "opal/include/opal/sys/atomic.h"
 #include "opal/runtime/opal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_finalize = PMPI_T_finalize
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_finalize PMPI_T_finalize
 #endif
 
-
 int MPI_T_finalize (void)
 {
     ompi_mpit_lock ();
diff --git a/ompi/mpi/tool/init_thread.c b/ompi/mpi/tool/init_thread.c
index 958985c3584..9ef39846f38 100644
--- a/ompi/mpi/tool/init_thread.c
+++ b/ompi/mpi/tool/init_thread.c
@@ -7,6 +7,8 @@
  * Copyright (c) 2020      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -14,18 +16,19 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "ompi/mpi/tool/mpit-internal.h"
 
 #include "ompi/runtime/ompi_info_support.h"
 #include "opal/include/opal/sys/atomic.h"
 #include "opal/runtime/opal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_init_thread = PMPI_T_init_thread
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_init_thread PMPI_T_init_thread
 #endif
 
 extern opal_mutex_t ompi_mpit_big_lock;
@@ -45,7 +48,7 @@ int MPI_T_init_thread (int required, int *provided)
             break;
         }
 
-        /* call opal_init_util to intialize the MCA system */
+        /* call opal_init_util to initialize the MCA system */
         rc = opal_init_util (NULL, NULL);
         if (OPAL_SUCCESS != rc) {
             rc = MPI_T_ERR_INVALID;
diff --git a/ompi/mpi/tool/mpit-internal.h b/ompi/mpi/tool/mpit-internal.h
index 9dd5d260ea8..4ee3d9e92a3 100644
--- a/ompi/mpi/tool/mpit-internal.h
+++ b/ompi/mpi/tool/mpit-internal.h
@@ -12,7 +12,7 @@
  * $HEADER$
  */
 
-#if !defined(MPIT_INTERNAL_H)
+#ifndef MPIT_INTERNAL_H
 #define MPIT_INTERNAL_H
 
 #include "opal/util/string_copy.h"
@@ -73,4 +73,4 @@ static inline void mpit_copy_string (char *dest, int *len, const char *source)
     }
 }
 
-#endif /* !defined(MPIT_INTERNAL_H) */
+#endif /* MPIT_INTERNAL_H */
diff --git a/ompi/mpi/tool/profile/Makefile.am b/ompi/mpi/tool/profile/Makefile.am
deleted file mode 100644
index d530d1c450a..00000000000
--- a/ompi/mpi/tool/profile/Makefile.am
+++ /dev/null
@@ -1,106 +0,0 @@
-# -*- makefile -*-
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2009-2014 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
-# Copyright (c) 2012      Oak Rigde National Laboratory. All rights reserved.
-# Copyright (c) 2013      Los Alamos National Security, LLC. All rights
-# Copyright (c) 2015      Research Organization for Information Science
-#                         and Technology (RIST). All rights reserved.
-#                         reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-include $(top_srcdir)/Makefile.ompi-rules
-
-#
-# OMPI_PROFILING_DEFINES flag s enabled when we want our MPI_* symbols
-# to be replaced by PMPI_*. In other words, this flag decides
-# whether "profile/defines.h" is included or not. "profile/defines.h"
-# replaces all MPI_* symbols with PMPI_* symbols. In this directory
-# we definately need it to be 1.
-#
-AM_CPPFLAGS = -DOMPI_PROFILING_DEFINES=1
-
-#
-# This build needs to go through only if profiling is required.
-# Further, this build HAS to go through if profiling is required.
-#
-
-noinst_LTLIBRARIES = libmpi_pmpit.la
-
-headers = defines.h
-
-nodist_libmpi_pmpit_la_SOURCES = \
-	pcategory_changed.c \
-	pcategory_get_categories.c \
-	pcategory_get_cvars.c \
-	pcategory_get_info.c \
-	pcategory_get_index.c \
-	pcategory_get_num.c \
-	pcategory_get_pvars.c \
-	pcvar_get_info.c \
-	pcvar_get_index.c \
-	pcvar_get_num.c \
-	pcvar_handle_alloc.c \
-	pcvar_handle_free.c \
-	pcvar_read.c \
-	pcvar_write.c \
-	penum_get_info.c \
-	penum_get_item.c \
-	pfinalize.c \
-	pinit_thread.c \
-	ppvar_get_info.c \
-	ppvar_get_index.c \
-	ppvar_get_num.c \
-	ppvar_handle_alloc.c \
-	ppvar_handle_free.c \
-	ppvar_read.c \
-	ppvar_readreset.c \
-	ppvar_reset.c \
-	ppvar_session_create.c \
-	ppvar_session_free.c \
-	ppvar_start.c \
-	ppvar_stop.c \
-	ppvar_write.c
-
-#
-# Sym link in the sources from the real MPI directory
-#
-$(nodist_libmpi_pmpit_la_SOURCES):
-	$(OMPI_V_LN_S) if test ! -r $@ ; then \
-		pname=`echo $@ | cut -b '2-'` ; \
-		$(LN_S) $(top_srcdir)/ompi/mpi/tool/$$pname $@ ; \
-	fi
-
-# Conditionally install the header files
-
-if WANT_INSTALL_HEADERS
-ompidir = $(ompiincludedir)/$(subdir)
-ompi_HEADERS = $(headers)
-endif
-
-# These files were created by targets above
-
-MAINTAINERCLEANFILES = $(nodist_libmpi_pmpit_la_SOURCES)
-
-# Don't want these targets in here
-
-tags-recursive:
-tags:
-TAGS:
-GTAGS:
-ID:
diff --git a/ompi/mpi/tool/profile/defines.h b/ompi/mpi/tool/profile/defines.h
deleted file mode 100644
index 44ef3a56eca..00000000000
--- a/ompi/mpi/tool/profile/defines.h
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
- * Copyright (c)      2012 Oak Rigde National Laboratory. All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef OMPIT_PROFILE_DEFINES_H
-#define OMPIT_PROFILE_DEFINES_H
-/*
- * This file is included in the top directory only if
- * profiling is required. Once profiling is required,
- * this file will replace all MPI_* symbols with
- * PMPI_* symbols
- */
-#define MPI_T_category_changed PMPI_T_category_changed
-#define MPI_T_category_get_categories PMPI_T_category_get_categories
-#define MPI_T_category_get_cvars PMPI_T_category_get_cvars
-#define MPI_T_category_get_info PMPI_T_category_get_info
-#define MPI_T_category_get_index PMPI_T_category_get_index
-#define MPI_T_category_get_num PMPI_T_category_get_num
-#define MPI_T_category_get_pvars PMPI_T_category_get_pvars
-#define MPI_T_cvar_get_info PMPI_T_cvar_get_info
-#define MPI_T_cvar_get_index PMPI_T_cvar_get_index
-#define MPI_T_cvar_get_num PMPI_T_cvar_get_num
-#define MPI_T_cvar_handle_alloc PMPI_T_cvar_handle_alloc
-#define MPI_T_cvar_handle_free PMPI_T_cvar_handle_free
-#define MPI_T_cvar_read PMPI_T_cvar_read
-#define MPI_T_cvar_write PMPI_T_cvar_write
-#define MPI_T_enum_get_info PMPI_T_enum_get_info
-#define MPI_T_enum_get_item PMPI_T_enum_get_item
-#define MPI_T_finalize PMPI_T_finalize
-#define MPI_T_init_thread PMPI_T_init_thread
-#define MPI_T_pvar_get_info PMPI_T_pvar_get_info
-#define MPI_T_pvar_get_index PMPI_T_pvar_get_index
-#define MPI_T_pvar_get_num PMPI_T_pvar_get_num
-#define MPI_T_pvar_handle_alloc PMPI_T_pvar_handle_alloc
-#define MPI_T_pvar_handle_free PMPI_T_pvar_handle_free
-#define MPI_T_pvar_read PMPI_T_pvar_read
-#define MPI_T_pvar_readreset PMPI_T_pvar_readreset
-#define MPI_T_pvar_reset PMPI_T_pvar_reset
-#define MPI_T_pvar_session_create PMPI_T_pvar_session_create
-#define MPI_T_pvar_session_free PMPI_T_pvar_session_free
-#define MPI_T_pvar_start PMPI_T_pvar_start
-#define MPI_T_pvar_stop PMPI_T_pvar_stop
-#define MPI_T_pvar_write PMPI_T_pvar_write
-#endif /* OMPIT_C_PROFILE_DEFINES_H */
diff --git a/ompi/mpi/tool/pvar_get_index.c b/ompi/mpi/tool/pvar_get_index.c
index 6280439b973..4f7832c4b7c 100644
--- a/ompi/mpi/tool/pvar_get_index.c
+++ b/ompi/mpi/tool/pvar_get_index.c
@@ -7,6 +7,8 @@
  * Copyright (c) 2020      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -14,17 +16,17 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "ompi/mpi/tool/mpit-internal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_pvar_get_index = PMPI_T_pvar_get_index
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_pvar_get_index PMPI_T_pvar_get_index
 #endif
 
-
 int MPI_T_pvar_get_index (const char *name, int var_class, int *pvar_index)
 {
     int ret;
diff --git a/ompi/mpi/tool/pvar_get_info.c b/ompi/mpi/tool/pvar_get_info.c
index a83984c85e3..01ad175e2d6 100644
--- a/ompi/mpi/tool/pvar_get_info.c
+++ b/ompi/mpi/tool/pvar_get_info.c
@@ -7,6 +7,8 @@
  * Copyright (c) 2020      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -14,17 +16,17 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "ompi/mpi/tool/mpit-internal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_pvar_get_info = PMPI_T_pvar_get_info
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_pvar_get_info PMPI_T_pvar_get_info
 #endif
 
-
 int MPI_T_pvar_get_info(int pvar_index, char *name, int *name_len,
                         int *verbosity, int *var_class, MPI_Datatype *datatype,
                         MPI_T_enum *enumtype, char *desc, int *desc_len, int *bind,
diff --git a/ompi/mpi/tool/pvar_get_num.c b/ompi/mpi/tool/pvar_get_num.c
index 190c8e2bc13..1e3fa0b1fb3 100644
--- a/ompi/mpi/tool/pvar_get_num.c
+++ b/ompi/mpi/tool/pvar_get_num.c
@@ -6,6 +6,8 @@
  * Copyright (c) 2020      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -13,17 +15,17 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "ompi/mpi/tool/mpit-internal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_pvar_get_num = PMPI_T_pvar_get_num
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_pvar_get_num PMPI_T_pvar_get_num
 #endif
 
-
 int MPI_T_pvar_get_num(int *num_pvar)
 {
     if (!mpit_is_initialized ()) {
diff --git a/ompi/mpi/tool/pvar_handle_alloc.c b/ompi/mpi/tool/pvar_handle_alloc.c
index 83eb650475f..0400025a11c 100644
--- a/ompi/mpi/tool/pvar_handle_alloc.c
+++ b/ompi/mpi/tool/pvar_handle_alloc.c
@@ -7,6 +7,8 @@
  * Copyright (c) 2020      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -14,17 +16,17 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "ompi/mpi/tool/mpit-internal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_pvar_handle_alloc = PMPI_T_pvar_handle_alloc
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_pvar_handle_alloc PMPI_T_pvar_handle_alloc
 #endif
 
-
 int MPI_T_pvar_handle_alloc(MPI_T_pvar_session session, int pvar_index,
                             void *obj_handle, MPI_T_pvar_handle *handle, int *count)
 {
diff --git a/ompi/mpi/tool/pvar_handle_free.c b/ompi/mpi/tool/pvar_handle_free.c
index 154d1c270f3..81ffa0707a9 100644
--- a/ompi/mpi/tool/pvar_handle_free.c
+++ b/ompi/mpi/tool/pvar_handle_free.c
@@ -7,6 +7,8 @@
  * Copyright (c) 2020      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -14,17 +16,17 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "ompi/mpi/tool/mpit-internal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_pvar_handle_free = PMPI_T_pvar_handle_free
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_pvar_handle_free PMPI_T_pvar_handle_free
 #endif
 
-
 int MPI_T_pvar_handle_free(MPI_T_pvar_session session, MPI_T_pvar_handle *handle)
 {
     int ret = MPI_SUCCESS;
diff --git a/ompi/mpi/tool/pvar_read.c b/ompi/mpi/tool/pvar_read.c
index 8314c9d4291..0c4e53653e0 100644
--- a/ompi/mpi/tool/pvar_read.c
+++ b/ompi/mpi/tool/pvar_read.c
@@ -4,6 +4,8 @@
  *                         reserved.
  * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -11,17 +13,17 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "ompi/mpi/tool/mpit-internal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_pvar_read = PMPI_T_pvar_read
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_pvar_read PMPI_T_pvar_read
 #endif
 
-
 int MPI_T_pvar_read(MPI_T_pvar_session session, MPI_T_pvar_handle handle,
                     void* buf)
 {
diff --git a/ompi/mpi/tool/pvar_readreset.c b/ompi/mpi/tool/pvar_readreset.c
index 59581bceba3..1567c4b1aef 100644
--- a/ompi/mpi/tool/pvar_readreset.c
+++ b/ompi/mpi/tool/pvar_readreset.c
@@ -3,6 +3,8 @@
  * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -10,17 +12,17 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "ompi/mpi/tool/mpit-internal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_pvar_readreset = PMPI_T_pvar_readreset
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_pvar_readreset PMPI_T_pvar_readreset
 #endif
 
-
 int MPI_T_pvar_readreset(MPI_T_pvar_session session, MPI_T_pvar_handle handle,
                          void *buf)
 {
diff --git a/ompi/mpi/tool/pvar_reset.c b/ompi/mpi/tool/pvar_reset.c
index 80e0bdeded5..a6f59d5405d 100644
--- a/ompi/mpi/tool/pvar_reset.c
+++ b/ompi/mpi/tool/pvar_reset.c
@@ -4,6 +4,8 @@
  *                         reserved.
  * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -11,17 +13,17 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "ompi/mpi/tool/mpit-internal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_pvar_reset = PMPI_T_pvar_reset
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_pvar_reset PMPI_T_pvar_reset
 #endif
 
-
 int MPI_T_pvar_reset(MPI_T_pvar_session session, MPI_T_pvar_handle handle)
 {
     int ret = MPI_SUCCESS;
diff --git a/ompi/mpi/tool/pvar_session_create.c b/ompi/mpi/tool/pvar_session_create.c
index 9e0e91cf0a4..6c153bebb8f 100644
--- a/ompi/mpi/tool/pvar_session_create.c
+++ b/ompi/mpi/tool/pvar_session_create.c
@@ -7,6 +7,8 @@
  * Copyright (c) 2020      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -14,17 +16,17 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "ompi/mpi/tool/mpit-internal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_pvar_session_create = PMPI_T_pvar_session_create
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_pvar_session_create PMPI_T_pvar_session_create
 #endif
 
-
 int MPI_T_pvar_session_create(MPI_T_pvar_session *session)
 {
     int ret = MPI_SUCCESS;
diff --git a/ompi/mpi/tool/pvar_session_free.c b/ompi/mpi/tool/pvar_session_free.c
index 98ec4a98a94..459489d6a8b 100644
--- a/ompi/mpi/tool/pvar_session_free.c
+++ b/ompi/mpi/tool/pvar_session_free.c
@@ -3,6 +3,8 @@
  * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -10,17 +12,17 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "ompi/mpi/tool/mpit-internal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_pvar_session_free = PMPI_T_pvar_session_free
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_pvar_session_free PMPI_T_pvar_session_free
 #endif
 
-
 int MPI_T_pvar_session_free(MPI_T_pvar_session *session)
 {
     int ret = MPI_SUCCESS;
diff --git a/ompi/mpi/tool/pvar_start.c b/ompi/mpi/tool/pvar_start.c
index d2fce3fa2a6..ecf99372c08 100644
--- a/ompi/mpi/tool/pvar_start.c
+++ b/ompi/mpi/tool/pvar_start.c
@@ -4,6 +4,8 @@
  *                         reserved.
  * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -11,17 +13,17 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "ompi/mpi/tool/mpit-internal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_pvar_start = PMPI_T_pvar_start
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_pvar_start PMPI_T_pvar_start
 #endif
 
-
 static int pvar_handle_start (mca_base_pvar_handle_t *handle)
 {
     if (OPAL_SUCCESS != mca_base_pvar_handle_start (handle)) {
diff --git a/ompi/mpi/tool/pvar_stop.c b/ompi/mpi/tool/pvar_stop.c
index 8923bbbf7b6..349e6f2d0ec 100644
--- a/ompi/mpi/tool/pvar_stop.c
+++ b/ompi/mpi/tool/pvar_stop.c
@@ -4,6 +4,8 @@
  *                         reserved.
  * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -11,17 +13,17 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "ompi/mpi/tool/mpit-internal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_pvar_stop = PMPI_T_pvar_stop
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_pvar_stop PMPI_T_pvar_stop
 #endif
 
-
 static int pvar_handle_stop (mca_base_pvar_handle_t *handle)
 {
     if (OPAL_SUCCESS != mca_base_pvar_handle_stop (handle)) {
diff --git a/ompi/mpi/tool/pvar_write.c b/ompi/mpi/tool/pvar_write.c
index 5bd17213600..4bd3f8d75e1 100644
--- a/ompi/mpi/tool/pvar_write.c
+++ b/ompi/mpi/tool/pvar_write.c
@@ -4,6 +4,8 @@
  *                         reserved.
  * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -11,17 +13,17 @@
  * $HEADER$
  */
 
+#include "ompi_config.h"
+
 #include "ompi/mpi/tool/mpit-internal.h"
 
-#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_T_pvar_write = PMPI_T_pvar_write
 #endif
-
-#if OMPI_PROFILING_DEFINES
-#include "ompi/mpi/tool/profile/defines.h"
+#define MPI_T_pvar_write PMPI_T_pvar_write
 #endif
 
-
 int MPI_T_pvar_write(MPI_T_pvar_session session, MPI_T_pvar_handle handle,
                      const void* buf)
 {
diff --git a/ompi/mpi/tool/source_get_info.c b/ompi/mpi/tool/source_get_info.c
new file mode 100644
index 00000000000..94017246fa2
--- /dev/null
+++ b/ompi/mpi/tool/source_get_info.c
@@ -0,0 +1,34 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2025      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/tool/mpit-internal.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_T_source_get_info = PMPI_T_source_get_info
+#endif
+#define MPI_T_source_get_info PMPI_T_source_get_info
+#endif
+
+int MPI_T_source_get_info (int source_index, char *name, int *name_len, char *desc, int *desc_len, MPI_T_source_order *ordering,
+                           MPI_Count *ticks_per_second, MPI_Count *max_timestamp, MPI_Info *info)
+{
+    if (!mpit_is_initialized ()) {
+        return MPI_T_ERR_NOT_INITIALIZED;
+    }
+
+    return MPI_T_ERR_INVALID_INDEX;
+}
diff --git a/ompi/mpi/tool/source_get_num.c b/ompi/mpi/tool/source_get_num.c
new file mode 100644
index 00000000000..6b31862ff27
--- /dev/null
+++ b/ompi/mpi/tool/source_get_num.c
@@ -0,0 +1,38 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2025      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/tool/mpit-internal.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_T_source_get_num = PMPI_T_source_get_num
+#endif
+#define MPI_T_source_get_num PMPI_T_source_get_num
+#endif
+
+int MPI_T_source_get_num (int *num_source)
+{
+    if (!mpit_is_initialized ()) {
+        return MPI_T_ERR_NOT_INITIALIZED;
+    }
+
+    if (MPI_PARAM_CHECK && NULL == num_source) {
+        return MPI_ERR_ARG;
+    }
+
+    *num_source = 0;
+    return MPI_SUCCESS;
+}
diff --git a/ompi/mpi/tool/source_get_timestamp.c b/ompi/mpi/tool/source_get_timestamp.c
new file mode 100644
index 00000000000..02a4303d286
--- /dev/null
+++ b/ompi/mpi/tool/source_get_timestamp.c
@@ -0,0 +1,37 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2025      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/mpi/tool/mpit-internal.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPI_T_source_get_timestamp = PMPI_T_source_get_timestamp
+#endif
+#define MPI_T_source_get_timestamp PMPI_T_source_get_timestamp
+#endif
+
+int MPI_T_source_get_timestamp (int source_index, MPI_Count *timestamp)
+{
+    if (!mpit_is_initialized ()) {
+        return MPI_T_ERR_NOT_INITIALIZED;
+    }
+
+    if (MPI_PARAM_CHECK && !timestamp) {
+        return MPI_ERR_ARG;
+    }
+
+    return MPI_T_ERR_INVALID_INDEX;
+}
diff --git a/ompi/mpiext/affinity/c/Makefile.am b/ompi/mpiext/affinity/c/Makefile.am
index e42dfebdafc..273e70e8219 100644
--- a/ompi/mpiext/affinity/c/Makefile.am
+++ b/ompi/mpiext/affinity/c/Makefile.am
@@ -2,7 +2,7 @@
 # Copyright (c) 2004-2009 The Trustees of Indiana University and Indiana
 #                         University Research and Technology
 #                         Corporation.  All rights reserved.
-# Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2010-2022 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2018      Research Organization for Information Science
 #                         and Technology (RIST).  All rights reserved.
 # $COPYRIGHT$
@@ -25,8 +25,6 @@ CLEANFILES = example
 # header files do the Right Thing.
 AM_CPPFLAGS = -DOMPI_PROFILE_LAYER=0 -DOMPI_COMPILING_FORTRAN_WRAPPERS=1
 
-include $(top_srcdir)/Makefile.ompi-rules
-
 # Convenience libtool library that will be slurped up into libmpi.la.
 noinst_LTLIBRARIES = libmpiext_affinity_c.la
 
@@ -45,11 +43,4 @@ libmpiext_affinity_c_la_SOURCES = \
         mpiext_affinity_str.c
 libmpiext_affinity_c_la_LDFLAGS = -module -avoid-version
 
-# Man page installation
-nodist_man_MANS = OMPI_Affinity_str.3
-
-# Man page sources
-EXTRA_DIST = $(nodist_man_MANS:.3=.3in) example.c
-
-distclean-local:
-	rm -f $(nodist_man_MANS)
+EXTRA_DIST = example.c
diff --git a/ompi/mpiext/affinity/c/OMPI_Affinity_str.3in b/ompi/mpiext/affinity/c/OMPI_Affinity_str.3in
deleted file mode 100644
index a9b0a8b1655..00000000000
--- a/ompi/mpiext/affinity/c/OMPI_Affinity_str.3in
+++ /dev/null
@@ -1,202 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright 2007-2010 Oracle and/or its affiliates.  All rights reserved.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2010 Cisco Systems, Inc.  All rights reserved.
-.\" $COPYRIGHT$
-.TH OMPI_Affinity_str 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBOMPI_Affinity_str\fP \- Obtain prettyprint strings of processor affinity information for this process
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-#include <mpi-ext.h>
-
-int OMPI_Affinity_str(ompi_affinity_fmt_type_t \fIfmt_type\fP,
-                      char \fIompi_bound\fP[OMPI_AFFINITY_STRING_MAX],
-                      char \fIcurrent_binding\fP[OMPI_AFFINITY_STRING_MAX],
-                      char \fIexists\fP[OMPI_AFFINITY_STRING_MAX])
-.fi
-.SH Fortran Syntax
-There is no Fortran binding for this function.
-.
-.SH Fortran 2008 Syntax
-There is no Fortran 2008 binding for this function.
-.
-.SH C++ Syntax
-There is no C++ binding for this function.
-.
-.SH INPUT PARAMETERS
-.ft R
-.TP 1i
-fmt_type
-An enum indicating how to format the returned ompi_bound and
-current_binding strings.  OMPI_AFFINITY_RSRC_STRING_FMT returns the
-string as human-readable resource names, such as "socket 0, core 0".
-
-OMPI_AFFINITY_LAYOUT_FMT returns ASCII art representing where this MPI
-process is bound relative to the machine resource layout.  For example
-"[. B][. .]" shows the process that called the routine is bound to
-socket 0, core 1 in a system with 2 sockets, each containing 2 cores.
-
-See below for more output examples.
-
-.
-.SH OUTPUT PARAMETERS
-.ft R
-.TP 1i
-ompi_bound
-A prettyprint string describing what processor(s) Open MPI bound this
-process to, or a string indicating that Open MPI did not bind this
-process.
-.
-.TP 1i
-current_binding
-A prettyprint string describing what processor(s) this process is
-currently bound to, or a string indicating that the process is bound
-to all available processors (and is therefore considered "unbound").
-.
-.TP 1i
-exists
-A prettyprint string describing the available sockets and sockets on
-this host.
-
-.SH DESCRIPTION
-.ft R
-Open MPI may bind a process to specific sockets and/or cores at
-process launch time.  This non-standard Open MPI function call returns
-prettyprint information about three things:
-.
-.TP
-Where Open MPI bound this process.
-The string returned in
-.B
-ompi_bound
-will either indicate that Open MPI did not bind this process to
-anything, or it will contain a prettyprint description of the
-processor(s) to which Open MPI bound this process.
-.
-.TP
-Where this process is currently bound.
-Regardless of whether Open MPI bound this process or not, another
-entity may have bound it.  The string returned in
-.B current_binding
-will indicate what the
-.I
-current
-binding is of this process, regardless of what Open MPI may have done
-earlier.  The string returned will either indicate that the process is
-unbound (meaning that it is bound to all available processors) or it
-will contain a prettyprint description of the sockets and cores to
-which the process is currently bound.
-.
-.TP
-What processors exist.
-As a convenience to the user, the
-.B
-exists
-string will contain a prettyprint description of the sockets and cores
-that this process can see (which is
-.I usually
-all processors in the system).
-
-.SH Examples
-.ft R
-\fBExample 1:\fP  Print out processes binding using resource string format.
-.sp
-.nf
-    int rank;
-    char ompi_bound[OMPI_AFFINITY_STRING_MAX];
-    char current_binding[OMPI_AFFINITY_STRING_MAX];
-    char exists[OMPI_AFFINITY_STRING_MAX];
-
-    MPI_Init(&argc, &argv);
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-
-    OMPI_Affinity_str(OMPI_AFFINITY_RSRC_STRING_FMT,
-                      ompi_bound, current_binding, exists);
-    printf("rank %d: \\n"
-           "       ompi_bound: %s\\n"
-           "  current_binding: %s\\n"
-           "           exists: %s\\n",
-           rank, ompi_bound, current_binding, exists);
-    ...
-.fi
-.PP
-Output of mpirun -np 2 -bind-to-core a.out:
-.nf
-rank 0:
-       ompi_bound: socket 0[core 0]
-  current_binding: socket 0[core 0]
-           exists: socket 0 has 4 cores
-rank 1:
-       ompi_bound: socket 0[core 1]
-  current_binding: socket 0[core 1]
-           exists: socket 0 has 4 cores
-.fi
-.PP
-Output of mpirun -np 2 -bind-to-socket a.out:
-.nf
-rank 0:
-       ompi_bound: socket 0[core 0-3]
-  current_binding: Not bound (or bound to all available processors)
-           exists: socket 0 has 4 cores
-rank 1:
-       ompi_bound: socket 0[core 0-3]
-  current_binding: Not bound (or bound to all available processors)
-           exists: socket 0 has 4 cores
-.fi
-.sp
-.br
-\fBExample 2:\fP  Print out processes binding using layout string format.
-.sp
-.nf
-    int rank;
-    char ompi_bound[OMPI_AFFINITY_STRING_MAX];
-    char current_binding[OMPI_AFFINITY_STRING_MAX];
-    char exists[OMPI_AFFINITY_STRING_MAX];
-
-    MPI_Init(&argc, &argv);
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-
-    OMPI_Affinity_str(OMPI_AFFINITY_LAYOUT_FMT,
-                      ompi_bound, current_binding, exists);
-    printf("rank %d: \\n"
-           "       ompi_bound: %s\\n"
-           "  current_binding: %s\\n"
-           "           exists: %s\\n",
-           rank, ompi_bound, current_binding, exists);
-    ...
-.fi
-.PP
-Output of mpirun -np 2 -bind-to-core a.out:
-.nf
-rank 0:
-       ompi_bound: [B . . .]
-  current_binding: [B . . .]
-           exists: [. . . .]
-rank 1:
-       ompi_bound: [. B . .]
-  current_binding: [. B . .]
-           exists: [. . . .]
-.fi
-.PP
-Output of mpirun -np 2 -bind-to-socket a.out:
-.nf
-rank 0:
-       ompi_bound: [B B B B]
-  current_binding: [B B B B]
-           exists: [. . . .]
-rank 1:
-       ompi_bound: [B B B B]
-  current_binding: [B B B B]
-           exists: [. . . .]
-.fi
-
-.SH See Also
-.ft R
-.nf
-mpirun(1)
-.fi
diff --git a/ompi/mpiext/affinity/c/mpiext_affinity_c.h b/ompi/mpiext/affinity/c/mpiext_affinity_c.h
index bf94f283c43..da6210e11b7 100644
--- a/ompi/mpiext/affinity/c/mpiext_affinity_c.h
+++ b/ompi/mpiext/affinity/c/mpiext_affinity_c.h
@@ -3,6 +3,9 @@
  *                         All rights reserved.
  * Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2021      The University of Tennessee and the University
+ *                         of Tennessee research Foundation.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
diff --git a/ompi/mpiext/affinity/c/mpiext_affinity_str.c b/ompi/mpiext/affinity/c/mpiext_affinity_str.c
index ac3fb9efc2b..03ec8c86923 100644
--- a/ompi/mpiext/affinity/c/mpiext_affinity_str.c
+++ b/ompi/mpiext/affinity/c/mpiext_affinity_str.c
@@ -94,6 +94,199 @@ int OMPI_Affinity_str(ompi_affinity_fmt_t fmt_type,
 
 /*---------------------------------------------------------------------------*/
 
+/*
+ * Make a map of socket/core/hwthread tuples
+ */
+static int build_map(int *num_sockets_arg, int *num_cores_arg, hwloc_cpuset_t cpuset, int ***map,
+                     hwloc_topology_t topo)
+{
+    int num_sockets, num_cores;
+    int socket_index, core_index, pu_index;
+    hwloc_obj_t socket, core, pu;
+    int **data;
+
+    /* Find out how many sockets we have */
+    num_sockets = hwloc_get_nbobjs_by_type(topo, HWLOC_OBJ_SOCKET);
+    /* some systems (like the iMac) only have one
+     * socket and so don't report a socket
+     */
+    if (0 == num_sockets) {
+        num_sockets = 1;
+    }
+    /* Lazy: take the total number of cores that we have in the
+       topology; that'll be more than the max number of cores
+       under any given socket */
+    num_cores = hwloc_get_nbobjs_by_type(topo, HWLOC_OBJ_CORE);
+    *num_sockets_arg = num_sockets;
+    *num_cores_arg = num_cores;
+
+    /* Alloc a 2D array: sockets x cores. */
+    data = malloc(num_sockets * sizeof(int *));
+    if (NULL == data) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+    data[0] = calloc(num_sockets * num_cores, sizeof(int));
+    if (NULL == data[0]) {
+        free(data);
+        return MPI_ERR_NO_MEM;
+    }
+    for (socket_index = 1; socket_index < num_sockets; ++socket_index) {
+        data[socket_index] = data[socket_index - 1] + num_cores;
+    }
+
+    /* Iterate the PUs in this cpuset; fill in the data[][] array with
+       the socket/core/pu triples */
+    for (pu_index = 0,
+        pu = hwloc_get_obj_inside_cpuset_by_type(topo, cpuset, HWLOC_OBJ_PU, pu_index);
+         NULL != pu;
+         pu = hwloc_get_obj_inside_cpuset_by_type(topo, cpuset, HWLOC_OBJ_PU, ++pu_index)) {
+        /* Go upward and find the core this PU belongs to */
+        core = pu;
+        while (NULL != core && core->type != HWLOC_OBJ_CORE) {
+            core = core->parent;
+        }
+        core_index = 0;
+        if (NULL != core) {
+            core_index = core->logical_index;
+        }
+
+        /* Go upward and find the socket this PU belongs to */
+        socket = pu;
+        while (NULL != socket && socket->type != HWLOC_OBJ_SOCKET) {
+            socket = socket->parent;
+        }
+        socket_index = 0;
+        if (NULL != socket) {
+            socket_index = socket->logical_index;
+        }
+
+        /* Save this socket/core/pu combo.  LAZY: Assuming that we
+           won't have more PU's per core than (sizeof(int)*8). */
+        data[socket_index][core_index] |= (1 << pu->sibling_rank);
+    }
+
+    *map = data;
+    return OMPI_SUCCESS;
+}
+
+/*
+ * Turn an int bitmap to a "a-b,c" range kind of string
+ */
+static char *bitmap2rangestr(int bitmap)
+{
+    size_t i;
+    int range_start, range_end;
+    bool first, isset;
+    char tmp[OMPI_AFFINITY_STRING_MAX - 1] = {0};
+    const int stmp = sizeof(tmp) - 1;
+    static char ret[OMPI_AFFINITY_STRING_MAX] = {0};
+
+    first = true;
+    range_start = -999;
+    for (i = 0; i < sizeof(int) * 8; ++i) {
+        isset = (bitmap & (1 << i));
+
+        /* Do we have a running range? */
+        if (range_start >= 0) {
+            if (isset) {
+                continue;
+            } else {
+                /* A range just ended; output it */
+                if (!first) {
+                    strncat(ret, ",", sizeof(ret) - strlen(ret) - 1);
+                } else {
+                    first = false;
+                }
+
+                range_end = i - 1;
+                if (range_start == range_end) {
+                    snprintf(tmp, stmp, "%d", range_start);
+                } else {
+                    snprintf(tmp, stmp, "%d-%d", range_start, range_end);
+                }
+                size_t ret_len = strlen(ret);
+                snprintf(ret + ret_len, sizeof(ret) - ret_len, "%s", tmp);
+
+                range_start = -999;
+            }
+        }
+
+        /* No running range */
+        else {
+            if (isset) {
+                range_start = i;
+            }
+        }
+    }
+
+    /* If we ended the bitmap with a range open, output it */
+    if (range_start >= 0) {
+        if (!first) {
+            strncat(ret, ",", sizeof(ret) - strlen(ret) - 1);
+            first = false;
+        }
+
+        range_end = i - 1;
+        if (range_start == range_end) {
+            snprintf(tmp, stmp, "%d", range_start);
+        } else {
+            snprintf(tmp, stmp, "%d-%d", range_start, range_end);
+        }
+        size_t ret_len = strlen(ret);
+        snprintf(ret + ret_len, sizeof(ret) - ret_len, "%s",  tmp);
+    }
+
+    return ret;
+}
+
+/*
+ * Make a prettyprint string for a hwloc_cpuset_t
+ */
+static int cset2str(char *str, int len, hwloc_topology_t topo, hwloc_cpuset_t cpuset)
+{
+    bool first;
+    int num_sockets, num_cores;
+    int ret, socket_index, core_index;
+    char tmp[OMPI_AFFINITY_STRING_MAX - 1] = {0};
+    const int stmp = sizeof(tmp) - 1;
+    int **map = NULL;
+
+    str[0] = tmp[stmp] = '\0';
+
+    /* if the cpuset is all zero, then not bound */
+    if (hwloc_bitmap_iszero(cpuset)) {
+        return OMPI_ERR_NOT_BOUND;
+    }
+
+    if (OMPI_SUCCESS != (ret = build_map(&num_sockets, &num_cores, cpuset, &map, topo))) {
+        return ret;
+    }
+    /* Iterate over the data matrix and build up the string */
+    first = true;
+    for (socket_index = 0; socket_index < num_sockets; ++socket_index) {
+        for (core_index = 0; core_index < num_cores; ++core_index) {
+            if (map[socket_index][core_index] > 0) {
+                if (!first) {
+                    strncat(str, ", ", len - strlen(str) - 1);
+                }
+                first = false;
+
+                snprintf(tmp, stmp, "socket %d[core %d[hwt %s]]", socket_index, core_index,
+                         bitmap2rangestr(map[socket_index][core_index]));
+                strncat(str, tmp, len - strlen(str) - 1);
+            }
+        }
+    }
+    if (NULL != map) {
+        if (NULL != map[0]) {
+            free(map[0]);
+        }
+        free(map);
+    }
+
+    return OMPI_SUCCESS;
+}
+
 /*
  * Where did OMPI bind this process? (prettyprint)
  */
@@ -107,10 +300,10 @@ static int get_rsrc_ompi_bound(char str[OMPI_AFFINITY_STRING_MAX])
 
     hwloc_bitmap_t cpuset = hwloc_bitmap_alloc();
     hwloc_bitmap_list_sscanf(cpuset, opal_process_info.cpuset);
-    if(OPAL_ERR_NOT_BOUND == opal_hwloc_base_cset2str(str,
-                                   OMPI_AFFINITY_STRING_MAX,
-                                   opal_hwloc_topology,
-                                   cpuset))
+    if(OMPI_ERR_NOT_BOUND == cset2str(str,
+                      OMPI_AFFINITY_STRING_MAX,
+                      opal_hwloc_topology,
+                      cpuset))
     {
        opal_string_copy(str, not_bound_str, OMPI_AFFINITY_STRING_MAX);
     }
@@ -119,6 +312,90 @@ static int get_rsrc_ompi_bound(char str[OMPI_AFFINITY_STRING_MAX])
     return OMPI_SUCCESS;
 }
 
+/* determine if there is a single cpu in a bitmap */
+static bool is_single_cpu(hwloc_cpuset_t cpuset)
+{
+    int i;
+    bool one = false;
+
+    /* count the number of bits that are set - there is
+     * one bit for each available pu. We could just
+     * subtract the first and last indices, but there
+     * may be "holes" in the bitmap corresponding to
+     * offline or unallowed cpus - so we have to
+     * search for them. Return false if we anything
+     * other than one
+     */
+    for (i = hwloc_bitmap_first(cpuset); i <= hwloc_bitmap_last(cpuset); i++) {
+        if (hwloc_bitmap_isset(cpuset, i)) {
+            if (one) {
+                return false;
+            }
+            one = true;
+        }
+    }
+
+    return one;
+}
+
+/*
+ * Make a prettyprint string for a cset in a map format.
+ * Example: [B./..]
+ * Key:  [] - signifies socket
+ *        / - divider between cores
+ *        . - signifies PU a process not bound to
+ *        B - signifies PU a process is bound to
+ */
+static int cset2mapstr(char *str, int len, hwloc_topology_t topo, hwloc_cpuset_t cpuset)
+{
+    char tmp[OMPI_AFFINITY_STRING_MAX - 1] = {0};
+    int core_index, pu_index;
+    const int stmp = sizeof(tmp) - 1;
+    hwloc_obj_t socket, core, pu;
+
+    str[0] = tmp[stmp] = '\0';
+
+    /* if the cpuset is all zero, then not bound */
+    if (hwloc_bitmap_iszero(cpuset)) {
+        return OMPI_ERR_NOT_BOUND;
+    }
+
+    /* Iterate over all existing sockets */
+    for (socket = hwloc_get_obj_by_type(topo, HWLOC_OBJ_SOCKET, 0); NULL != socket;
+         socket = socket->next_cousin) {
+        strncat(str, "[", len - strlen(str) - 1);
+
+        /* Iterate over all existing cores in this socket */
+        core_index = 0;
+        for (core = hwloc_get_obj_inside_cpuset_by_type(topo, socket->cpuset, HWLOC_OBJ_CORE,
+                                                        core_index);
+             NULL != core;
+             core = hwloc_get_obj_inside_cpuset_by_type(topo, socket->cpuset, HWLOC_OBJ_CORE,
+                                                        ++core_index)) {
+            if (core_index > 0) {
+                strncat(str, "/", len - strlen(str) - 1);
+            }
+
+            /* Iterate over all existing PUs in this core */
+            pu_index = 0;
+            for (pu = hwloc_get_obj_inside_cpuset_by_type(topo, core->cpuset, HWLOC_OBJ_PU,
+                                                          pu_index);
+                 NULL != pu; pu = hwloc_get_obj_inside_cpuset_by_type(topo, core->cpuset,
+                                                                      HWLOC_OBJ_PU, ++pu_index)) {
+
+                /* Is this PU in the cpuset? */
+                if (hwloc_bitmap_isset(cpuset, pu->os_index)) {
+                    strncat(str, "B", len - strlen(str) - 1);
+                } else {
+                    strncat(str, ".", len - strlen(str) - 1);
+                }
+            }
+        }
+        strncat(str, "]", len - strlen(str) - 1);
+    }
+
+    return OMPI_SUCCESS;
+}
 
 /*
  * Where is this process currently bound? (prettyprint)
@@ -145,24 +422,24 @@ static int get_rsrc_current_binding(char str[OMPI_AFFINITY_STRING_MAX])
         /* we are bound if the two cpusets are not equal, or if there
            is only ONE PU available to us */
         if (0 != hwloc_bitmap_compare(boundset, rootset) ||
-            opal_hwloc_base_single_cpu(rootset) ||
-            opal_hwloc_base_single_cpu(boundset)) {
+            is_single_cpu(rootset) ||
+            is_single_cpu(boundset)) {
             bound = true;
         }
     }
 
     /* If we are not bound, indicate that */
     if (!bound) {
-        strncat(str, not_bound_str, OMPI_AFFINITY_STRING_MAX - 1);
+        strncat(str, not_bound_str, OMPI_AFFINITY_STRING_MAX - strlen(str) - 1);
         ret = OMPI_SUCCESS;
     }
 
     /* If we are bound, print it out */
     else {
-        ret = opal_hwloc_base_cset2str(str, OMPI_AFFINITY_STRING_MAX,
-                                       opal_hwloc_topology,
-                                       boundset);
-        if (OPAL_ERR_NOT_BOUND == ret) {
+        ret = cset2str(str, OMPI_AFFINITY_STRING_MAX,
+                       opal_hwloc_topology,
+                       boundset);
+        if (OMPI_ERR_NOT_BOUND == ret) {
             opal_string_copy(str, not_bound_str, OMPI_AFFINITY_STRING_MAX);
             ret = OMPI_SUCCESS;
         }
@@ -182,7 +459,7 @@ static int get_rsrc_exists(char str[OMPI_AFFINITY_STRING_MAX])
 {
     bool first = true;
     int i, num_cores, num_pus;
-    char tmp[BUFSIZ];
+    char tmp[OMPI_AFFINITY_STRING_MAX - 1] = {0};
     const int stmp = sizeof(tmp) - 1;
     hwloc_obj_t socket, core, c2;
 
@@ -192,12 +469,12 @@ static int get_rsrc_exists(char str[OMPI_AFFINITY_STRING_MAX])
          NULL != socket; socket = socket->next_cousin) {
         /* If this isn't the first socket, add a delimiter */
         if (!first) {
-            strncat(str, "; ", OMPI_AFFINITY_STRING_MAX - strlen(str));
+            strncat(str, "; ", OMPI_AFFINITY_STRING_MAX - strlen(str) - 1);
         }
         first = false;
 
         snprintf(tmp, stmp, "socket %d has ", socket->os_index);
-        strncat(str, tmp, OMPI_AFFINITY_STRING_MAX - strlen(str));
+        strncat(str, tmp, OMPI_AFFINITY_STRING_MAX - strlen(str) - 1);
 
         /* Find out how many cores are inside this socket, and get an
            object pointing to the first core.  Also count how many PUs
@@ -217,13 +494,13 @@ static int get_rsrc_exists(char str[OMPI_AFFINITY_STRING_MAX])
             /* Only 1 core */
             if (1 == num_cores) {
                 strncat(str, "1 core with ",
-                        OMPI_AFFINITY_STRING_MAX - strlen(str));
+                        OMPI_AFFINITY_STRING_MAX - strlen(str) - 1);
                 if (1 == num_pus) {
                     strncat(str, "1 hwt",
-                            OMPI_AFFINITY_STRING_MAX - strlen(str));
+                            OMPI_AFFINITY_STRING_MAX - strlen(str) - 1);
                 } else {
                     snprintf(tmp, stmp, "%d hwts", num_pus);
-                    strncat(str, tmp, OMPI_AFFINITY_STRING_MAX - strlen(str));
+                    strncat(str, tmp, OMPI_AFFINITY_STRING_MAX - strlen(str) - 1);
                 }
             }
 
@@ -232,7 +509,7 @@ static int get_rsrc_exists(char str[OMPI_AFFINITY_STRING_MAX])
                 bool same = true;
 
                 snprintf(tmp, stmp, "%d cores", num_cores);
-                strncat(str, tmp, OMPI_AFFINITY_STRING_MAX - strlen(str));
+                strncat(str, tmp, OMPI_AFFINITY_STRING_MAX - strlen(str) - 1);
 
                 /* Do all the cores have the same number of PUs? */
                 for (c2 = core; NULL != c2; c2 = c2->next_cousin) {
@@ -248,32 +525,32 @@ static int get_rsrc_exists(char str[OMPI_AFFINITY_STRING_MAX])
                 /* Yes, they all have the same number of PUs */
                 if (same) {
                     snprintf(tmp, stmp, ", each with %d hwt", num_pus);
-                    strncat(str, tmp, OMPI_AFFINITY_STRING_MAX - strlen(str));
+                    strncat(str, tmp, OMPI_AFFINITY_STRING_MAX - strlen(str) - 1);
                     if (num_pus != 1) {
-                        strncat(str, "s", OMPI_AFFINITY_STRING_MAX - strlen(str));
+                        strncat(str, "s", OMPI_AFFINITY_STRING_MAX - strlen(str) - 1);
                     }
                 }
 
                 /* No, they have differing numbers of PUs */
                 else {
-                    bool first = true;
+                    bool first_iter = true;
 
-                    strncat(str, "with (", OMPI_AFFINITY_STRING_MAX - strlen(str));
+                    strncat(str, "with (", OMPI_AFFINITY_STRING_MAX - strlen(str) - 1);
                     for (c2 = core; NULL != c2; c2 = c2->next_cousin) {
-                        if (!first) {
+                        if (!first_iter) {
                             strncat(str, ", ",
-                                    OMPI_AFFINITY_STRING_MAX - strlen(str));
+                                    OMPI_AFFINITY_STRING_MAX - strlen(str) - 1);
                         }
-                        first = false;
+                        first_iter = false;
 
                         i = hwloc_get_nbobjs_inside_cpuset_by_type(opal_hwloc_topology,
                                                                    core->cpuset,
                                                                    HWLOC_OBJ_PU);
                         snprintf(tmp, stmp, "%d", i);
-                        strncat(str, tmp, OMPI_AFFINITY_STRING_MAX - strlen(str));
+                        strncat(str, tmp, OMPI_AFFINITY_STRING_MAX - strlen(str) - 1);
                     }
                     strncat(str, ") hwts",
-                            OMPI_AFFINITY_STRING_MAX - strlen(str));
+                            OMPI_AFFINITY_STRING_MAX - strlen(str) - 1);
                 }
             }
         }
@@ -297,10 +574,10 @@ static int get_layout_ompi_bound(char str[OMPI_AFFINITY_STRING_MAX])
 
     hwloc_bitmap_t cpuset = hwloc_bitmap_alloc();
     hwloc_bitmap_list_sscanf(cpuset, opal_process_info.cpuset);
-    if(OPAL_ERR_NOT_BOUND == opal_hwloc_base_cset2mapstr(str,
-                                          OMPI_AFFINITY_STRING_MAX,
-                                          opal_hwloc_topology,
-                                          cpuset))
+    if(OMPI_ERR_NOT_BOUND == cset2mapstr(str,
+                         OMPI_AFFINITY_STRING_MAX,
+                         opal_hwloc_topology,
+                         cpuset))
     {
         opal_string_copy(str, not_bound_str, OMPI_AFFINITY_STRING_MAX);
     }
@@ -333,24 +610,24 @@ static int get_layout_current_binding(char str[OMPI_AFFINITY_STRING_MAX])
         /* we are bound if the two cpusets are not equal, or if there
            is only ONE PU available to us */
         if (0 != hwloc_bitmap_compare(boundset, rootset) ||
-            opal_hwloc_base_single_cpu(rootset) ||
-            opal_hwloc_base_single_cpu(boundset)) {
+            is_single_cpu(rootset) ||
+            is_single_cpu(boundset)) {
             bound = true;
         }
     }
 
     /* If we are not bound, indicate that */
     if (!bound) {
-        strncat(str, not_bound_str, OMPI_AFFINITY_STRING_MAX - 1);
+        strncat(str, not_bound_str, OMPI_AFFINITY_STRING_MAX - strlen(str) - 1);
         ret = OMPI_SUCCESS;
     }
 
     /* If we are bound, print it out */
     else {
-        ret = opal_hwloc_base_cset2mapstr(str, OMPI_AFFINITY_STRING_MAX,
-                                          opal_hwloc_topology,
-                                          boundset);
-        if (OPAL_ERR_NOT_BOUND == ret) {
+        ret = cset2mapstr(str, OMPI_AFFINITY_STRING_MAX,
+                          opal_hwloc_topology,
+                          boundset);
+        if (OMPI_ERR_NOT_BOUND == ret) {
             opal_string_copy(str, not_bound_str, OMPI_AFFINITY_STRING_MAX);
             ret = OMPI_SUCCESS;
         }
@@ -373,7 +650,6 @@ static int get_layout_current_binding(char str[OMPI_AFFINITY_STRING_MAX])
 static int get_layout_exists(char str[OMPI_AFFINITY_STRING_MAX])
 {
     int core_index, pu_index;
-    int len = OMPI_AFFINITY_STRING_MAX;
     hwloc_obj_t socket, core, pu;
 
     str[0] = '\0';
@@ -383,7 +659,7 @@ static int get_layout_exists(char str[OMPI_AFFINITY_STRING_MAX])
                                         HWLOC_OBJ_SOCKET, 0);
          NULL != socket;
          socket = socket->next_cousin) {
-        strncat(str, "[", len - strlen(str));
+        strncat(str, "[", OMPI_AFFINITY_STRING_MAX - strlen(str) - 1);
 
         /* Iterate over all existing cores in this socket */
         core_index = 0;
@@ -395,7 +671,7 @@ static int get_layout_exists(char str[OMPI_AFFINITY_STRING_MAX])
                                                         socket->cpuset,
                                                         HWLOC_OBJ_CORE, ++core_index)) {
             if (core_index > 0) {
-                strncat(str, "/", len - strlen(str));
+                strncat(str, "/", OMPI_AFFINITY_STRING_MAX - strlen(str) - 1);
             }
 
             /* Iterate over all existing PUs in this core */
@@ -407,10 +683,10 @@ static int get_layout_exists(char str[OMPI_AFFINITY_STRING_MAX])
                  pu = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology,
                                                           core->cpuset,
                                                           HWLOC_OBJ_PU, ++pu_index)) {
-                strncat(str, ".", len - strlen(str));
+                strncat(str, ".", OMPI_AFFINITY_STRING_MAX - strlen(str) - 1);
             }
         }
-        strncat(str, "]", len - strlen(str));
+        strncat(str, "]", OMPI_AFFINITY_STRING_MAX - strlen(str) - 1);
     }
 
     return OMPI_SUCCESS;
diff --git a/ompi/mpiext/cuda/c/MPIX_Query_cuda_support.3in b/ompi/mpiext/cuda/c/MPIX_Query_cuda_support.3in
deleted file mode 100644
index 676ec570bc9..00000000000
--- a/ompi/mpiext/cuda/c/MPIX_Query_cuda_support.3in
+++ /dev/null
@@ -1,34 +0,0 @@
-.\" Copyright 2007-2010 Oracle and/or its affiliates.  All rights reserved.
-.\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2010 Cisco Systems, Inc.  All rights reserved.
-.\" Copyright (c) 2015 NVIDIA, Inc. All rights reserved.
-.TH MPIx_CUDA_SUPPORT 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPIX_Query_cuda_support\fP \- Returns 1 if there is CUDA aware support and 0 if there is not.
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-#include <mpi-ext.h>
-
-int MPIX_Query_cuda_support(void)
-.fi
-.SH Fortran Syntax
-There is no Fortran binding for this function.
-.
-.SH C++ Syntax
-There is no C++ binding for this function.
-.
-.SH DESCRIPTION
-.ft R
-
-.SH Examples
-.ft R
-
-.SH See Also
-.ft R
-.nf
-
-.fi
diff --git a/ompi/mpiext/cuda/c/Makefile.am b/ompi/mpiext/cuda/c/Makefile.am
index f303cc70824..365093abc2d 100644
--- a/ompi/mpiext/cuda/c/Makefile.am
+++ b/ompi/mpiext/cuda/c/Makefile.am
@@ -2,7 +2,7 @@
 # Copyright (c) 2004-2009 The Trustees of Indiana University and Indiana
 #                         University Research and Technology
 #                         Corporation.  All rights reserved.
-# Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2010-2022 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2015      NVIDIA, Inc. All rights reserved.
 # Copyright (c) 2018      Research Organization for Information Science
 #                         and Technology (RIST).  All rights reserved.
@@ -20,8 +20,6 @@
 # header files do the Right Thing.
 AM_CPPFLAGS = -DOMPI_PROFILE_LAYER=0 -DOMPI_COMPILING_FORTRAN_WRAPPERS=1
 
-include $(top_srcdir)/Makefile.ompi-rules
-
 # Convenience libtool library that will be slurped up into libmpi.la.
 noinst_LTLIBRARIES = libmpiext_cuda_c.la
 
@@ -39,12 +37,3 @@ libmpiext_cuda_c_la_SOURCES = \
         $(ompi_HEADERS) \
         mpiext_cuda.c
 libmpiext_cuda_c_la_LDFLAGS = -module -avoid-version
-
-# Man page installation
-nodist_man_MANS = MPIX_Query_cuda_support.3
-
-# Man page sources
-EXTRA_DIST = $(nodist_man_MANS:.3=.3in)
-
-distclean-local:
-	rm -f $(nodist_man_MANS)
diff --git a/ompi/mpiext/cuda/c/mpiext_cuda.c b/ompi/mpiext/cuda/c/mpiext_cuda.c
index 499d1441c39..84fcdaf1a8d 100644
--- a/ompi/mpiext/cuda/c/mpiext_cuda.c
+++ b/ompi/mpiext/cuda/c/mpiext_cuda.c
@@ -20,12 +20,12 @@
 #include <stdio.h>
 #include <string.h>
 
-#include "opal/constants.h"
+#include "opal/mca/accelerator/base/base.h"
 #include "ompi/mpiext/cuda/c/mpiext_cuda_c.h"
 
 /* If CUDA-aware support is configured in, return 1. Otherwise, return 0.
  * This API may be extended to return more features in the future. */
 int MPIX_Query_cuda_support(void)
 {
-    return OPAL_CUDA_SUPPORT;
+    return 0 == strcmp(opal_accelerator_base_selected_component.base_version.mca_component_name, "cuda");
 }
diff --git a/ompi/mpiext/example/README.md b/ompi/mpiext/example/README.md
index 36bb1881f7b..e0d2e46bcab 100644
--- a/ompi/mpiext/example/README.md
+++ b/ompi/mpiext/example/README.md
@@ -132,7 +132,7 @@ As such, `ompi/mpiext/EXTENSION_NAME/Makefile.am` is not traversed
 during a normal top-level `make all` target.  This `Makefile.am`
 exists for two reasons, however:
 
-1. For the conveneince of the developer, so that you can issue normal
+1. For the convenience of the developer, so that you can issue normal
    `make` commands at the top of your extension tree (e.g., `make all`
    will still build all bindings in an extension).
 1. During a top-level `make dist`, extension directories *are*
diff --git a/ompi/mpiext/example/configure.m4 b/ompi/mpiext/example/configure.m4
index 58370e0a14c..664b69e1254 100644
--- a/ompi/mpiext/example/configure.m4
+++ b/ompi/mpiext/example/configure.m4
@@ -55,3 +55,12 @@ AC_DEFUN([OMPI_MPIEXT_example_NEED_INIT], [1])
 # libraries for any of the bindings.  If finer-grained control is needed
 # someday, we may need to split this into multiple macros.
 #AC_DEFUN([OMPI_MPIEXT_example_HAVE_OBJECT], [0])
+
+# This is a hook that runs after the CONFIG macro and after the
+# configure system decides if this extension will be built (such
+# as if it was disabled by configure option).  This is a great
+# place to put any globally-visible AC_DEFINE or AM_CONDITIONAL
+# macros that are based on this component building.
+AC_DEFUN([OMPI_MPIEXT_example_POST_CONFIG], [
+  opal_show_verbose "example component build result: $1"
+])
diff --git a/ompi/mpiext/example/use-mpi-f08/Makefile.am b/ompi/mpiext/example/use-mpi-f08/Makefile.am
index f495b4414d6..ef2ddd79d13 100644
--- a/ompi/mpiext/example/use-mpi-f08/Makefile.am
+++ b/ompi/mpiext/example/use-mpi-f08/Makefile.am
@@ -2,6 +2,7 @@
 # Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2017      Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2022      IBM Corporation.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -9,6 +10,14 @@
 # $HEADER$
 #
 
+# Note that Automake's Fortran-buidling rules uses CPPFLAGS and
+# AM_CPPFLAGS.  This can cause weirdness (e.g.,
+# https://github.com/open-mpi/ompi/issues/7253 and
+# https://github.com/open-mpi/ompi/issues/9716).  Let's just zero
+# those out and rely on AM_FCFLAGS.
+CPPFLAGS =
+AM_CPPFLAGS =
+
 # This file builds the use_mpi_f08-based bindings for MPI extensions.  It
 # is optional in MPI extensions.
 
diff --git a/ompi/mpiext/ftmpi/c/Makefile.am b/ompi/mpiext/ftmpi/c/Makefile.am
index da91742691b..8a1a1a165a2 100644
--- a/ompi/mpiext/ftmpi/c/Makefile.am
+++ b/ompi/mpiext/ftmpi/c/Makefile.am
@@ -1,8 +1,9 @@
 #
 # Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
-# Copyright (c) 2016-2018 The University of Tennessee and The University
+# Copyright (c) 2016-2022 The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -27,14 +28,17 @@ libmpiext_ftmpi_c_la_SOURCES = \
     comm_revoke.c \
     comm_is_revoked.c \
     comm_shrink.c \
+    comm_ishrink.c \
     comm_failure_ack.c \
     comm_failure_get_acked.c \
+    comm_get_failed.c \
+    comm_ack_failed.c \
     comm_agree.c \
     comm_iagree.c
 libmpiext_ftmpi_c_la_LIBADD = \
     profile/libpmpiext_ftmpi_c.la
 
-dist_ompidata_DATA = help-mpi-ft.txt
+EXTRA_DIST = help-mpi-ft.txt
 
 ompidir = $(ompiincludedir)/mpiext/
 ompi_HEADERS = $(headers)
diff --git a/ompi/mpiext/ftmpi/c/comm_ack_failed.c b/ompi/mpiext/ftmpi/c/comm_ack_failed.c
new file mode 100644
index 00000000000..d4f6c625034
--- /dev/null
+++ b/ompi/mpiext/ftmpi/c/comm_ack_failed.c
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2022      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/proc/proc.h"
+#include "ompi/errhandler/errhandler.h"
+
+#include "ompi/mpiext/ftmpi/c/mpiext_ftmpi_c.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPIX_Comm_ack_failed = PMPIX_Comm_ack_failed
+#endif
+#define MPIX_Comm_ack_failed PMPIX_Comm_ack_failed
+#endif
+
+static const char FUNC_NAME[] = "MPIX_Comm_ack_failed";
+
+
+int MPIX_Comm_ack_failed(MPI_Comm comm, int num_to_ack, int* num_acked)
+{
+    int rc = MPI_SUCCESS;
+
+    /* Argument checking */
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_COMM, FUNC_NAME);
+        }
+        if (num_to_ack > ompi_comm_size(comm)) {
+            rc = MPI_ERR_ARG;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+    rc = ompi_comm_ack_failed_internal( (ompi_communicator_t*)comm, num_to_ack, num_acked );
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
+
diff --git a/ompi/mpiext/ftmpi/c/comm_get_failed.c b/ompi/mpiext/ftmpi/c/comm_get_failed.c
new file mode 100644
index 00000000000..999a627545f
--- /dev/null
+++ b/ompi/mpiext/ftmpi/c/comm_get_failed.c
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2022      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/proc/proc.h"
+
+#include "ompi/mpiext/ftmpi/c/mpiext_ftmpi_c.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPIX_Comm_get_failed = PMPIX_Comm_get_failed
+#endif
+#define MPIX_Comm_get_failed PMPIX_Comm_get_failed
+#endif
+
+static const char FUNC_NAME[] = "MPIX_Comm_get_failed";
+
+int MPIX_Comm_get_failed(MPI_Comm comm, MPI_Group *failedgrp)
+{
+    int rc = MPI_SUCCESS;
+
+    /* Argument checking */
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_COMM, FUNC_NAME);
+        }
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+    rc = ompi_comm_get_failed_internal( (ompi_communicator_t*)comm, (ompi_group_t**)failedgrp );
+    if( OMPI_SUCCESS != rc ) {
+        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+    }
+
+    return MPI_SUCCESS;
+}
+
diff --git a/ompi/mpiext/ftmpi/c/comm_ishrink.c b/ompi/mpiext/ftmpi/c/comm_ishrink.c
new file mode 100644
index 00000000000..c1c80ce9fb7
--- /dev/null
+++ b/ompi/mpiext/ftmpi/c/comm_ishrink.c
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2018-2022 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/request/request.h"
+#include "ompi/proc/proc.h"
+#include "ompi/op/op.h"
+
+#if OMPI_BUILD_MPI_PROFILING
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak MPIX_Comm_ishrink = PMPIX_Comm_ishrink
+#endif
+#define MPIX_Comm_ishrink PMPIX_Comm_ishrink
+#endif
+
+#include "ompi/mpiext/ftmpi/c/mpiext_ftmpi_c.h"
+
+static const char FUNC_NAME[] = "MPIX_Comm_ishrink";
+
+
+int MPIX_Comm_ishrink(MPI_Comm comm, MPI_Comm* newcomm, MPI_Request *request)
+{
+    int rc = MPI_SUCCESS;
+
+    /* Argument checking */
+    if (MPI_PARAM_CHECK) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+        if (ompi_comm_invalid(comm)) {
+            return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_COMM, FUNC_NAME);
+        }
+        if (NULL == newcomm) {
+            rc = MPI_ERR_ARG;
+        }
+        if (NULL == request) {
+            rc = MPI_ERR_REQUEST;
+        }
+        OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
+    }
+
+    rc = ompi_comm_ishrink_internal(comm, newcomm, request);
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
+
diff --git a/ompi/mpiext/ftmpi/c/mpiext_ftmpi_c.h b/ompi/mpiext/ftmpi/c/mpiext_ftmpi_c.h
index 8b01a4e6bb4..080729ca6a1 100644
--- a/ompi/mpiext/ftmpi/c/mpiext_ftmpi_c.h
+++ b/ompi/mpiext/ftmpi/c/mpiext_ftmpi_c.h
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2010-2020 The University of Tennessee and the University
+ * Copyright (c) 2010-2023 The University of Tennessee and the University
  *                         of Tennessee research Foundation.  All rights
  *                         reserved.
  * $COPYRIGHT$
@@ -25,19 +25,29 @@
 OMPI_DECLSPEC int MPIX_Comm_revoke(MPI_Comm comm);
 OMPI_DECLSPEC int MPIX_Comm_is_revoked(MPI_Comm comm, int *flag);
 OMPI_DECLSPEC int MPIX_Comm_shrink(MPI_Comm comm, MPI_Comm *newcomm);
+OMPI_DECLSPEC int MPIX_Comm_ishrink(MPI_Comm comm, MPI_Comm *newcomm, MPI_Request *request);
 OMPI_DECLSPEC int MPIX_Comm_failure_ack(MPI_Comm comm);
 OMPI_DECLSPEC int MPIX_Comm_failure_get_acked(MPI_Comm comm, MPI_Group *failedgrp);
+OMPI_DECLSPEC int MPIX_Comm_get_failed(MPI_Comm comm, MPI_Group *failedgroup);
+OMPI_DECLSPEC int MPIX_Comm_ack_failed(MPI_Comm comm, int num_to_ack, int *num_acked);
 OMPI_DECLSPEC int MPIX_Comm_agree(MPI_Comm comm, int *flag);
 OMPI_DECLSPEC int MPIX_Comm_iagree(MPI_Comm comm, int *flag, MPI_Request *request);
 
 OMPI_DECLSPEC int PMPIX_Comm_revoke(MPI_Comm comm);
 OMPI_DECLSPEC int PMPIX_Comm_is_revoked(MPI_Comm comm, int *flag);
 OMPI_DECLSPEC int PMPIX_Comm_shrink(MPI_Comm comm, MPI_Comm *newcomm);
+OMPI_DECLSPEC int PMPIX_Comm_ishrink(MPI_Comm comm, MPI_Comm *newcomm, MPI_Request *request);
 OMPI_DECLSPEC int PMPIX_Comm_failure_ack(MPI_Comm comm);
 OMPI_DECLSPEC int PMPIX_Comm_failure_get_acked(MPI_Comm comm, MPI_Group *failedgrp);
+OMPI_DECLSPEC int PMPIX_Comm_get_failed(MPI_Comm comm, MPI_Group *failedgroup);
+OMPI_DECLSPEC int PMPIX_Comm_ack_failed(MPI_Comm comm, int num_to_ack, int *num_acked);
 OMPI_DECLSPEC int PMPIX_Comm_agree(MPI_Comm comm, int *flag);
 OMPI_DECLSPEC int PMPIX_Comm_iagree(MPI_Comm comm, int *flag, MPI_Request *request);
 
 #include <stdbool.h>
 OMPI_DECLSPEC int OMPI_Comm_failure_inject(MPI_Comm comm, bool notify);
+/* Provide defines to facilitate the detection of the new API */
+#define OMPI_HAVE_MPIX_COMM_GET_FAILED 1
+#define OMPI_HAVE_MPIX_COMM_ACK_FAILED 1
+#define OMPI_HAVE_MPIX_COMM_ISHRINK    1
 
diff --git a/ompi/mpiext/ftmpi/c/profile/Makefile.am b/ompi/mpiext/ftmpi/c/profile/Makefile.am
index 1651dd77f44..a7090ee2bad 100644
--- a/ompi/mpiext/ftmpi/c/profile/Makefile.am
+++ b/ompi/mpiext/ftmpi/c/profile/Makefile.am
@@ -1,7 +1,8 @@
 #
-# Copyright (c) 2016-2018 The University of Tennessee and The University
+# Copyright (c) 2016-2022 The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
+# Copyright (c) 2021 Cisco Systems, Inc.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -11,7 +12,7 @@
 
 # If OMPI_BUILD_MPI_PROFILING is enabled when we want our generated MPI_* symbols
 # to be replaced by PMPI_*.
-# In this directory, we definately need it to be 1.
+# In this directory, we definitely need it to be 1.
 AM_CPPFLAGS = -DOMPI_BUILD_MPI_PROFILING=1
 
 noinst_LTLIBRARIES = libpmpiext_ftmpi_c.la
@@ -20,6 +21,9 @@ nodist_libpmpiext_ftmpi_c_la_SOURCES = \
     pcomm_revoke.c \
     pcomm_is_revoked.c \
     pcomm_shrink.c \
+    pcomm_ishrink.c \
+    pcomm_get_failed.c \
+    pcomm_ack_failed.c \
     pcomm_failure_ack.c \
     pcomm_failure_get_acked.c \
     pcomm_agree.c \
@@ -38,11 +42,3 @@ $(nodist_libpmpiext_ftmpi_c_la_SOURCES):
 # These files were created by targets above
 
 MAINTAINERCLEANFILES = $(nodist_libpmpiext_ftmpi_c_la_SOURCES)
-
-# Don't want these targets in here
-
-tags-recursive:
-tags:
-:
-GTAGS:
-ID:
diff --git a/ompi/mpiext/ftmpi/mpif-h/Makefile.am b/ompi/mpiext/ftmpi/mpif-h/Makefile.am
index e849b732881..565f5414aaa 100644
--- a/ompi/mpiext/ftmpi/mpif-h/Makefile.am
+++ b/ompi/mpiext/ftmpi/mpif-h/Makefile.am
@@ -1,6 +1,6 @@
 #
 # Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
-# Copyright (c) 2016-2020 The University of Tennessee and The University
+# Copyright (c) 2016-2022 The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
 # $COPYRIGHT$
@@ -24,6 +24,9 @@ f77_sources = \
     comm_revoke_f.c \
     comm_is_revoked_f.c \
     comm_shrink_f.c \
+    comm_ishrink_f.c \
+    comm_get_failed_f.c \
+    comm_ack_failed_f.c \
     comm_failure_ack_f.c \
     comm_failure_get_acked_f.c \
     comm_agree_f.c \
diff --git a/ompi/mpiext/ftmpi/mpif-h/comm_ack_failed_f.c b/ompi/mpiext/ftmpi/mpif-h/comm_ack_failed_f.c
new file mode 100644
index 00000000000..f1c244a50b8
--- /dev/null
+++ b/ompi/mpiext/ftmpi/mpif-h/comm_ack_failed_f.c
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2022      The University of Tennessee and the University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+#include "ompi/mpiext/ftmpi/c/mpiext_ftmpi_c.h"
+#include "ompi/mpiext/ftmpi/mpif-h/prototypes_mpi.h"
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPIX_COMM_ACK_FAILED = ompix_comm_ack_failed_f
+#pragma weak pmpix_comm_ack_failed = ompix_comm_ack_failed_f
+#pragma weak pmpix_comm_ack_failed_ = ompix_comm_ack_failed_f
+#pragma weak pmpix_comm_ack_failed__ = ompix_comm_ack_failed_f
+#pragma weak PMPIX_Comm_ack_failed_f = ompix_comm_ack_failed_f
+#pragma weak PMPIX_Comm_ack_failed_f08 = ompix_comm_ack_failed_f
+
+#pragma weak MPIX_COMM_ACK_FAILED = ompix_comm_ack_failed_f
+#pragma weak mpix_comm_ack_failed = ompix_comm_ack_failed_f
+#pragma weak mpix_comm_ack_failed_ = ompix_comm_ack_failed_f
+#pragma weak mpix_comm_ack_failed__ = ompix_comm_ack_failed_f
+#pragma weak MPIX_Comm_ack_failed_f = ompix_comm_ack_failed_f
+#pragma weak MPIX_Comm_ack_failed_f08 = ompix_comm_ack_failed_f
+
+#else /* No weak symbols */
+OMPI_GENERATE_F77_BINDINGS(PMPIX_COMM_ACK_FAILED,
+                        pmpix_comm_ack_failed,
+                        pmpix_comm_ack_failed_,
+                        pmpix_comm_ack_failed__,
+                        ompix_comm_ack_failed_f,
+                        (MPI_Fint *comm, MPI_Fint *num_to_ack, MPI_Fint *num_acked, MPI_Fint *ierr),
+                        (comm, num_to_ack, num_acked, ierr))
+
+OMPI_GENERATE_F77_BINDINGS(MPIX_COMM_ACK_FAILED,
+                        mpix_comm_ack_failed,
+                        mpix_comm_ack_failed_,
+                        mpix_comm_ack_failed__,
+                        ompix_comm_ack_failed_f,
+                        (MPI_Fint *comm, MPI_Fint *num_to_ack, MPI_Fint *num_acked, MPI_Fint *ierr),
+                        (comm, num_to_ack, num_acked, ierr))
+#endif
+
+void ompix_comm_ack_failed_f(MPI_Fint *comm, MPI_Fint *num_to_ack, MPI_Fint *num_acked, MPI_Fint *ierr)
+{
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+
+    *ierr = OMPI_INT_2_FINT(PMPIX_Comm_ack_failed(c_comm, OMPI_FINT_2_INT(*num_to_ack), OMPI_PFINT_2_PINT(num_acked)));
+}
diff --git a/ompi/mpiext/ftmpi/mpif-h/comm_get_failed_f.c b/ompi/mpiext/ftmpi/mpif-h/comm_get_failed_f.c
new file mode 100644
index 00000000000..8473a7f749c
--- /dev/null
+++ b/ompi/mpiext/ftmpi/mpif-h/comm_get_failed_f.c
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2022      The University of Tennessee and the University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+#include "ompi/mpiext/ftmpi/c/mpiext_ftmpi_c.h"
+#include "ompi/mpiext/ftmpi/mpif-h/prototypes_mpi.h"
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPIX_COMM_GET_FAILED = ompix_comm_get_failed_f
+#pragma weak pmpix_comm_get_failed = ompix_comm_get_failed_f
+#pragma weak pmpix_comm_get_failed_ = ompix_comm_get_failed_f
+#pragma weak pmpix_comm_get_failed__ = ompix_comm_get_failed_f
+#pragma weak PMPIX_Comm_get_failed_f = ompix_comm_get_failed_f
+#pragma weak PMPIX_Comm_get_failed_f08 = ompix_comm_get_failed_f
+
+#pragma weak MPIX_COMM_GET_FAILED = ompix_comm_get_failed_f
+#pragma weak mpix_comm_get_failed = ompix_comm_get_failed_f
+#pragma weak mpix_comm_get_failed_ = ompix_comm_get_failed_f
+#pragma weak mpix_comm_get_failed__ = ompix_comm_get_failed_f
+#pragma weak MPIX_Comm_get_failed_f = ompix_comm_get_failed_f
+#pragma weak MPIX_Comm_get_failed_f08 = ompix_comm_get_failed_f
+
+#else /* No weak symbols */
+OMPI_GENERATE_F77_BINDINGS(PMPIX_COMM_GET_FAILED,
+                        pmpix_comm_get_failed,
+                        pmpix_comm_get_failed_,
+                        pmpix_comm_get_failed__,
+                        ompix_comm_get_failed_f,
+                        (MPI_Fint *comm, MPI_Fint *group, MPI_Fint *ierr),
+                        (comm, group, ierr))
+
+OMPI_GENERATE_F77_BINDINGS(MPIX_COMM_GET_FAILED,
+                        mpix_comm_get_failed,
+                        mpix_comm_get_failed_,
+                        mpix_comm_get_failed__,
+                        ompix_comm_get_failed_f,
+                        (MPI_Fint *comm, MPI_Fint *group, MPI_Fint *ierr),
+                        (comm, group, ierr))
+#endif
+
+void ompix_comm_get_failed_f(MPI_Fint *comm, MPI_Fint *group, MPI_Fint *ierr)
+{
+    MPI_Group c_group;
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+
+    *ierr = OMPI_INT_2_FINT(PMPIX_Comm_get_failed(c_comm,
+                                                         &c_group));
+    if (MPI_SUCCESS == OMPI_FINT_2_INT(*ierr)) {
+        *group = PMPI_Group_c2f (c_group);
+    }
+}
diff --git a/ompi/mpiext/ftmpi/mpif-h/comm_ishrink_f.c b/ompi/mpiext/ftmpi/mpif-h/comm_ishrink_f.c
new file mode 100644
index 00000000000..30977617969
--- /dev/null
+++ b/ompi/mpiext/ftmpi/mpif-h/comm_ishrink_f.c
@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2010-2022 The University of Tennessee and the University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include "ompi/mpi/fortran/mpif-h/bindings.h"
+#include "ompi/mpi/fortran/base/constants.h"
+
+#include "ompi/mpiext/ftmpi/c/mpiext_ftmpi_c.h"
+#include "ompi/mpiext/ftmpi/mpif-h/prototypes_mpi.h"
+
+#include "ompi/communicator/communicator.h"
+#include "ompi/request/request.h"
+#include "ompi/mpi/fortran/base/fint_2_int.h"
+
+#if OPAL_HAVE_WEAK_SYMBOLS
+#pragma weak PMPIX_COMM_ISHRINK = ompix_comm_ishrink_f
+#pragma weak pmpix_comm_ishrink = ompix_comm_ishrink_f
+#pragma weak pmpix_comm_ishrink_ = ompix_comm_ishrink_f
+#pragma weak pmpix_comm_ishrink__ = ompix_comm_ishrink_f
+#pragma weak PMPIX_Comm_ishrink_f = ompix_comm_ishrink_f
+#pragma weak PMPIX_Comm_ishrink_f08 = ompix_comm_ishrink_f
+
+#pragma weak MPIX_COMM_ISHRINK = ompix_comm_ishrink_f
+#pragma weak mpix_comm_ishrink = ompix_comm_ishrink_f
+#pragma weak mpix_comm_ishrink_ = ompix_comm_ishrink_f
+#pragma weak mpix_comm_ishrink__ = ompix_comm_ishrink_f
+#pragma weak MPIX_Comm_ishrink_f = ompix_comm_ishrink_f
+#pragma weak MPIX_Comm_ishrink_f08 = ompix_comm_ishrink_f
+
+#else /* No weak symbols */
+OMPI_GENERATE_F77_BINDINGS(PMPIX_COMM_ISHRINK,
+                        pmpix_comm_ishrink,
+                        pmpix_comm_ishrink_,
+                        pmpix_comm_ishrink__,
+                        ompix_comm_ishrink_f,
+                        (MPI_Fint *comm, MPI_Fint *newcomm, MPI_Fint *request, MPI_Fint *ierr),
+                        (comm, newcomm, request, ierr))
+
+OMPI_GENERATE_F77_BINDINGS(MPIX_COMM_ISHRINK,
+                        mpix_comm_ishrink,
+                        mpix_comm_ishrink_,
+                        mpix_comm_ishrink__,
+                        ompix_comm_ishrink_f,
+                        (MPI_Fint *comm, MPI_Fint *newcomm, MPI_Fint *request, MPI_Fint *ierr),
+                        (comm, newcomm, request, ierr))
+#endif
+
+void ompix_comm_ishrink_f(MPI_Fint *comm, MPI_Fint *newcomm, MPI_Fint *request, MPI_Fint *ierr)
+{
+    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
+    MPI_Request c_req;
+    MPI_Comm c_newcomm;
+
+    *ierr = OMPI_INT_2_FINT(PMPIX_Comm_ishrink(c_comm,
+                                               &c_newcomm,
+                                               &c_req));
+
+    if (MPI_SUCCESS == OMPI_FINT_2_INT(*ierr)) {
+        *request = PMPI_Request_c2f(c_req);
+        *newcomm = PMPI_Comm_c2f(c_newcomm);
+    }
+    else {
+        *newcomm = PMPI_Comm_c2f(&ompi_mpi_comm_null.comm);
+    }
+}
diff --git a/ompi/mpiext/ftmpi/mpif-h/comm_shrink_f.c b/ompi/mpiext/ftmpi/mpif-h/comm_shrink_f.c
index 6ff0c7ae16c..f4d8c5b1c44 100644
--- a/ompi/mpiext/ftmpi/mpif-h/comm_shrink_f.c
+++ b/ompi/mpiext/ftmpi/mpif-h/comm_shrink_f.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2010-2019 The University of Tennessee and the University
+ * Copyright (c) 2010-2022 The University of Tennessee and the University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * $COPYRIGHT$
@@ -16,6 +16,8 @@
 #include "ompi/mpiext/ftmpi/c/mpiext_ftmpi_c.h"
 #include "ompi/mpiext/ftmpi/mpif-h/prototypes_mpi.h"
 
+#include "ompi/communicator/communicator.h"
+
 #if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak PMPIX_COMM_SHRINK = ompix_comm_shrink_f
 #pragma weak pmpix_comm_shrink = ompix_comm_shrink_f
@@ -59,4 +61,7 @@ void ompix_comm_shrink_f(MPI_Fint *comm, MPI_Fint *newcomm, MPI_Fint *ierr)
     if (MPI_SUCCESS == OMPI_FINT_2_INT(*ierr)) {
         *newcomm = PMPI_Comm_c2f(c_newcomm);
     }
+    else {
+        *newcomm = PMPI_Comm_c2f(&ompi_mpi_comm_null.comm);
+    }
 }
diff --git a/ompi/mpiext/ftmpi/mpif-h/prototypes_mpi.h b/ompi/mpiext/ftmpi/mpif-h/prototypes_mpi.h
index c588bca9138..661c8ce8014 100644
--- a/ompi/mpiext/ftmpi/mpif-h/prototypes_mpi.h
+++ b/ompi/mpiext/ftmpi/mpif-h/prototypes_mpi.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019      The University of Tennessee and The University
+ * Copyright (c) 2019-2022 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * $COPYRIGHT$
@@ -43,9 +43,13 @@ PN2(void, MPIX_Comm_agree, mpix_comm_agree, MPIX_COMM_AGREE, (MPI_Fint *comm, om
 PN2(void, MPIX_Comm_failure_ack, mpix_comm_failure_ack, MPIX_COMM_FAILURE_ACK, (MPI_Fint *comm, MPI_Fint *ierr));
 PN2(void, MPIX_Comm_failure_get_acked, mpix_comm_failure_get_acked, MPIX_COMM_FAILURE_GET_ACKED, (MPI_Fint *comm, MPI_Fint *group, MPI_Fint *ierr));
 PN2(void, MPIX_Comm_iagree, mpix_comm_iagree, MPIX_COMM_IAGREE, (MPI_Fint *comm, ompi_fortran_logical_t *flag, MPI_Fint *request, MPI_Fint *ierr));
+PN2(void, MPIX_Comm_ishrink, mpix_comm_ishrink, MPIX_COMM_ISHRINK, (MPI_Fint *comm, MPI_Fint *newcomm, MPI_Fint *request, MPI_Fint *ierr));
 PN2(void, MPIX_Comm_is_revoked, mpix_comm_is_revoked, MPIX_COMM_IS_REVOKED, (MPI_Fint *comm, ompi_fortran_logical_t *flag, MPI_Fint *ierr));
 PN2(void, MPIX_Comm_revoke, mpix_comm_revoke, MPIX_COMM_REVOKE, (MPI_Fint *comm, MPI_Fint *ierr));
 PN2(void, MPIX_Comm_shrink, mpix_comm_shrink, MPIX_COMM_SHRINK, (MPI_Fint *comm, MPI_Fint *newcomm, MPI_Fint *ierr));
+PN2(void, MPIX_Comm_get_failed, mpix_comm_get_failed, MPIX_COMM_GET_FAILED, (MPI_Fint *comm, MPI_Fint *group, MPI_Fint *ierr));
+PN2(void, MPIX_Comm_ack_failed, mpix_comm_ack_failed, MPI_COMM_ACK_FAILED, (MPI_Fint *comm, MPI_Fint *num_to_ack, MPI_Fint *num_acked, MPI_Fint *ierr));
+
 
 END_C_DECLS
 
diff --git a/ompi/mpiext/ftmpi/use-mpi-f08/Makefile.am b/ompi/mpiext/ftmpi/use-mpi-f08/Makefile.am
index d06c0aa6837..6be909be6c8 100644
--- a/ompi/mpiext/ftmpi/use-mpi-f08/Makefile.am
+++ b/ompi/mpiext/ftmpi/use-mpi-f08/Makefile.am
@@ -3,9 +3,10 @@
 # Copyright (c) 2017-2018 Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
 # Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-# Copyright (c) 2018      The University of Tennessee and The University
+# Copyright (c) 2018-2022 The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
+# Copyright (c) 2022      IBM Corporation.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -13,6 +14,14 @@
 # $HEADER$
 #
 
+# Note that Automake's Fortran-buidling rules uses CPPFLAGS and
+# AM_CPPFLAGS.  This can cause weirdness (e.g.,
+# https://github.com/open-mpi/ompi/issues/7253 and
+# https://github.com/open-mpi/ompi/issues/9716).  Let's just zero
+# those out and rely on AM_FCFLAGS.
+CPPFLAGS =
+AM_CPPFLAGS =
+
 # This file builds the use_mpi_f08-based bindings for MPI extensions.  It
 # is optional in MPI extensions.
 
@@ -49,18 +58,24 @@ mpi_api_files = \
     comm_is_revoked_f08.F90 \
     comm_failure_ack_f08.F90 \
     comm_failure_get_acked_f08.F90 \
+    comm_get_failed_f08.F90 \
+    comm_ack_failed_f08.F90 \
     comm_agree_f08.F90 \
     comm_iagree_f08.F90 \
-    comm_shrink_f08.F90
+    comm_shrink_f08.F90 \
+    comm_ishrink_f08.F90
 
 pmpi_api_files = \
     profile/pcomm_revoke_f08.F90 \
     profile/pcomm_is_revoked_f08.F90 \
     profile/pcomm_failure_ack_f08.F90 \
     profile/pcomm_failure_get_acked_f08.F90 \
+    profile/pcomm_get_failed_f08.F90 \
+    profile/pcomm_ack_failed_f08.F90 \
     profile/pcomm_agree_f08.F90 \
     profile/pcomm_iagree_f08.F90 \
-    profile/pcomm_shrink_f08.F90
+    profile/pcomm_shrink_f08.F90 \
+    profile/pcomm_ishrink_f08.F90
 
 mpi_api_lo_files = $(mpi_api_files:.F90=.lo)
 pmpi_api_lo_files = $(pmpi_api_files:.F90=.lo)
diff --git a/ompi/mpiext/ftmpi/use-mpi-f08/comm_ack_failed_f08.F90 b/ompi/mpiext/ftmpi/use-mpi-f08/comm_ack_failed_f08.F90
new file mode 100644
index 00000000000..77c96cb9e07
--- /dev/null
+++ b/ompi/mpiext/ftmpi/use-mpi-f08/comm_ack_failed_f08.F90
@@ -0,0 +1,33 @@
+! -*- f90 -*-
+!
+! Copyright (c) 2022      The University of Tennessee and the University
+!                         of Tennessee Research Foundation.  All rights
+!                         reserved.
+! $COPYRIGHT$
+!
+! Additional copyrights may follow
+!
+! $HEADER$
+!
+
+subroutine MPIX_Comm_ack_failed_f08(comm, num_to_ack, num_acked, ierror)
+  use :: mpi_f08_types, only : MPI_Comm
+  implicit none
+  interface
+     subroutine ompix_comm_ack_failed_f(comm, num_to_ack, num_acked, ierror) &
+          BIND(C, name="ompix_comm_ack_failed_f")
+       implicit none
+       INTEGER, INTENT(IN) :: comm, num_to_ack
+       INTEGER, INTENT(OUT) :: num_acked, ierror
+     end subroutine ompix_comm_ack_failed_f
+  end interface
+  TYPE(MPI_Comm), INTENT(IN) :: comm
+  INTEGER, INTENT(IN) :: num_to_ack
+  INTEGER, INTENT(OUT) :: num_acked
+  INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+  integer :: c_ierror
+
+  call ompix_comm_ack_failed_f(comm%MPI_VAL, num_to_ack, num_acked, c_ierror)
+  if (present(ierror)) ierror = c_ierror
+
+end subroutine MPIX_Comm_ack_failed_f08
diff --git a/ompi/mpiext/ftmpi/use-mpi-f08/comm_get_failed_f08.F90 b/ompi/mpiext/ftmpi/use-mpi-f08/comm_get_failed_f08.F90
new file mode 100644
index 00000000000..22d0cd3f616
--- /dev/null
+++ b/ompi/mpiext/ftmpi/use-mpi-f08/comm_get_failed_f08.F90
@@ -0,0 +1,32 @@
+! -*- f90 -*-
+!
+! Copyright (c) 2022      The University of Tennessee and the University
+!                         of Tennessee Research Foundation.  All rights
+!                         reserved.
+! $COPYRIGHT$
+!
+! Additional copyrights may follow
+!
+! $HEADER$
+!
+
+subroutine MPIX_Comm_get_failed_f08(comm, failedgrp, ierror)
+  use :: mpi_f08_types, only : MPI_Comm, MPI_Group
+  implicit none
+  interface
+     subroutine ompix_comm_get_failed_f(comm, failedgrp, ierror) &
+          BIND(C, name="ompix_comm_get_failed_f")
+       implicit none
+       INTEGER, INTENT(IN) :: comm
+       INTEGER, INTENT(OUT) :: failedgrp, ierror
+     end subroutine ompix_comm_get_failed_f
+  end interface
+  TYPE(MPI_Comm), INTENT(IN) :: comm
+  TYPE(MPI_Group), INTENT(OUT) :: failedgrp
+  INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+  integer :: c_ierror
+
+  call ompix_comm_get_failed_f(comm%MPI_VAL, failedgrp%MPI_VAL, c_ierror)
+  if (present(ierror)) ierror = c_ierror
+
+end subroutine MPIX_Comm_get_failed_f08
diff --git a/ompi/mpiext/ftmpi/use-mpi-f08/comm_ishrink_f08.F90 b/ompi/mpiext/ftmpi/use-mpi-f08/comm_ishrink_f08.F90
new file mode 100644
index 00000000000..610c2058bc9
--- /dev/null
+++ b/ompi/mpiext/ftmpi/use-mpi-f08/comm_ishrink_f08.F90
@@ -0,0 +1,35 @@
+! -*- f90 -*-
+!
+! Copyright (c) 2018-2022 The University of Tennessee and the University
+!                         of Tennessee Research Foundation.  All rights
+!                         reserved.
+! $COPYRIGHT$
+!
+! Additional copyrights may follow
+!
+! $HEADER$
+!
+
+subroutine MPIX_Comm_ishrink_f08(comm, newcomm, request, ierror)
+  use :: mpi_f08_types, only : MPI_Comm, MPI_Request
+  implicit none
+  interface
+     subroutine ompix_comm_ishrink_f(comm, newcomm, request, ierror) &
+          BIND(C, name="ompix_comm_shrink_f")
+       implicit none
+       INTEGER, INTENT(IN) :: comm
+       INTEGER, INTENT(OUT) :: newcomm
+       INTEGER, INTENT(OUT) :: request
+       INTEGER, INTENT(OUT) :: ierror
+     end subroutine ompix_comm_ishrink_f
+  end interface
+  TYPE(MPI_Comm), INTENT(IN) :: comm
+  TYPE(MPI_Comm), INTENT(OUT) :: newcomm
+  TYPE(MPI_Request), INTENT(OUT) :: request
+  INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+  integer :: c_ierror
+
+  call ompix_comm_ishrink_f(comm%MPI_VAL, newcomm%MPI_VAL, request%MPI_VAL, c_ierror)
+  if (present(ierror)) ierror = c_ierror
+
+end subroutine MPIX_Comm_ishrink_f08
diff --git a/ompi/mpiext/ftmpi/use-mpi-f08/mpiext_ftmpi_usempif08.h b/ompi/mpiext/ftmpi/use-mpi-f08/mpiext_ftmpi_usempif08.h
index 5a68cec8bbd..79e6531680d 100644
--- a/ompi/mpiext/ftmpi/use-mpi-f08/mpiext_ftmpi_usempif08.h
+++ b/ompi/mpiext/ftmpi/use-mpi-f08/mpiext_ftmpi_usempif08.h
@@ -1,6 +1,6 @@
 ! -*- f90 -*-
 !
-! Copyright (c) 2018-2020 The University of Tennessee and the University
+! Copyright (c) 2018-2022 The University of Tennessee and the University
 !                         of Tennessee Research Foundation.  All rights
 !                         reserved.
 ! $COPYRIGHT$
@@ -88,6 +88,25 @@ subroutine pmpix_comm_failure_get_acked_f08(comm,failedgrp,ierror)
 end subroutine pmpix_comm_failure_get_acked_f08
 end interface pmpix_comm_failure_get_acked
 
+interface mpix_comm_get_failed
+subroutine mpix_comm_get_failed_f08(comm,failedgrp,ierror)
+   use :: mpi_f08_types, only : MPI_Comm, MPI_Group
+   implicit none
+   TYPE(MPI_Comm), INTENT(IN) :: comm
+   TYPE(MPI_Group), INTENT(OUT) :: failedgrp
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+end subroutine mpix_comm_get_failed_f08
+end interface mpix_comm_get_failed
+
+interface pmpix_comm_get_failed
+subroutine pmpix_comm_get_failed_f08(comm,failedgrp,ierror)
+   use :: mpi_f08_types, only : MPI_Comm, MPI_Group
+   implicit none
+   TYPE(MPI_Comm), INTENT(IN) :: comm
+   TYPE(MPI_Group), INTENT(OUT) :: failedgrp
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+end subroutine pmpix_comm_get_failed_f08
+end interface pmpix_comm_get_failed
 interface mpix_comm_agree
 subroutine mpix_comm_agree_f08(comm,flag,ierror)
    use :: mpi_f08_types, only : MPI_Comm
@@ -98,6 +117,28 @@ subroutine mpix_comm_agree_f08(comm,flag,ierror)
 end subroutine mpix_comm_agree_f08
 end interface mpix_comm_agree
 
+interface mpix_comm_ack_failed
+subroutine mpix_comm_ack_failed_f08(comm,num_to_ack,num_acked,ierror)
+   use :: mpi_f08_types, only : MPI_Comm
+   implicit none
+   TYPE(MPI_Comm), INTENT(IN) :: comm
+   INTEGER, INTENT(IN) :: num_to_ack
+   INTEGER, INTENT(OUT) :: num_acked
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+end subroutine mpix_comm_ack_failed_f08
+end interface mpix_comm_ack_failed
+
+interface pmpix_comm_ack_failed
+subroutine pmpix_comm_ack_failed_f08(comm,num_to_ack,num_acked,ierror)
+   use :: mpi_f08_types, only : MPI_Comm, MPI_Group
+   implicit none
+   TYPE(MPI_Comm), INTENT(IN) :: comm
+   INTEGER, INTENT(IN) :: num_to_ack
+   INTEGER, INTENT(OUT) :: num_acked
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+end subroutine pmpix_comm_ack_failed_f08
+end interface pmpix_comm_ack_failed
+
 interface pmpix_comm_agree
 subroutine pmpix_comm_agree_f08(comm,flag,ierror)
    use :: mpi_f08_types, only : MPI_Comm
@@ -149,3 +190,25 @@ subroutine pmpix_comm_shrink_f08(comm,newcomm,ierror)
    INTEGER, OPTIONAL, INTENT(OUT) :: ierror
 end subroutine pmpix_comm_shrink_f08
 end interface pmpix_comm_shrink
+
+interface mpix_comm_ishrink
+subroutine mpix_comm_ishrink_f08(comm,newcomm,request,ierror)
+   use :: mpi_f08_types, only : MPI_Comm, MPI_Request
+   implicit none
+   TYPE(MPI_Comm), INTENT(IN) :: comm
+   TYPE(MPI_Comm), INTENT(OUT) :: newcomm
+   TYPE(MPI_Request), INTENT(OUT) :: request
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+end subroutine mpix_comm_ishrink_f08
+end interface mpix_comm_ishrink
+
+interface pmpix_comm_ishrink
+subroutine pmpix_comm_ishrink_f08(comm,newcomm,request,ierror)
+   use :: mpi_f08_types, only : MPI_Comm, MPI_Request
+   implicit none
+   TYPE(MPI_Comm), INTENT(IN) :: comm
+   TYPE(MPI_Comm), INTENT(OUT) :: newcomm
+   TYPE(MPI_Request), INTENT(OUT) :: request
+   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+end subroutine pmpix_comm_ishrink_f08
+end interface pmpix_comm_ishrink
diff --git a/ompi/mpiext/ftmpi/use-mpi-f08/profile/pcomm_ack_failed_f08.F90 b/ompi/mpiext/ftmpi/use-mpi-f08/profile/pcomm_ack_failed_f08.F90
new file mode 100644
index 00000000000..186a6ba5e10
--- /dev/null
+++ b/ompi/mpiext/ftmpi/use-mpi-f08/profile/pcomm_ack_failed_f08.F90
@@ -0,0 +1,33 @@
+! -*- f90 -*-
+!
+! Copyright (c) 2022      The University of Tennessee and the University
+!                         of Tennessee Research Foundation.  All rights
+!                         reserved.
+! $COPYRIGHT$
+!
+! Additional copyrights may follow
+!
+! $HEADER$
+!
+
+subroutine PMPIX_Comm_ack_failed_f08(comm, num_to_ack, num_acked, ierror)
+  use :: mpi_f08_types, only : MPI_Comm
+  implicit none
+  interface
+     subroutine ompix_comm_ack_failed_f(comm, num_to_ack, num_acked, ierror) &
+          BIND(C, name="ompix_comm_ack_failed_f")
+       implicit none
+       INTEGER, INTENT(IN) :: comm, num_to_ack
+       INTEGER, INTENT(OUT) :: num_acked, ierror
+     end subroutine ompix_comm_ack_failed_f
+  end interface
+  TYPE(MPI_Comm), INTENT(IN) :: comm
+  INTEGER, INTENT(IN) :: num_to_ack
+  INTEGER, INTENT(OUT) :: num_acked
+  INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+  integer :: c_ierror
+
+  call ompix_comm_ack_failed_f(comm%MPI_VAL, num_to_ack, num_acked, c_ierror)
+  if (present(ierror)) ierror = c_ierror
+
+end subroutine PMPIX_Comm_ack_failed_f08
diff --git a/ompi/mpiext/ftmpi/use-mpi-f08/profile/pcomm_get_failed_f08.F90 b/ompi/mpiext/ftmpi/use-mpi-f08/profile/pcomm_get_failed_f08.F90
new file mode 100644
index 00000000000..0a452103fa9
--- /dev/null
+++ b/ompi/mpiext/ftmpi/use-mpi-f08/profile/pcomm_get_failed_f08.F90
@@ -0,0 +1,32 @@
+! -*- f90 -*-
+!
+! Copyright (c) 2022      The University of Tennessee and the University
+!                         of Tennessee Research Foundation.  All rights
+!                         reserved.
+! $COPYRIGHT$
+!
+! Additional copyrights may follow
+!
+! $HEADER$
+!
+
+subroutine PMPIX_Comm_get_failed_f08(comm, failedgrp, ierror)
+  use :: mpi_f08_types, only : MPI_Comm, MPI_Group
+  implicit none
+  interface
+     subroutine ompix_comm_get_failed_f(comm, failedgrp, ierror) &
+          BIND(C, name="ompix_comm_get_failed_f")
+       implicit none
+       INTEGER, INTENT(IN) :: comm
+       INTEGER, INTENT(OUT) :: failedgrp, ierror
+     end subroutine ompix_comm_get_failed_f
+  end interface
+  TYPE(MPI_Comm), INTENT(IN) :: comm
+  TYPE(MPI_Group), INTENT(OUT) :: failedgrp
+  INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+  integer :: c_ierror
+
+  call ompix_comm_get_failed_f(comm%MPI_VAL, failedgrp%MPI_VAL, c_ierror)
+  if (present(ierror)) ierror = c_ierror
+
+end subroutine PMPIX_Comm_get_failed_f08
diff --git a/ompi/mpiext/ftmpi/use-mpi-f08/profile/pcomm_ishrink_f08.F90 b/ompi/mpiext/ftmpi/use-mpi-f08/profile/pcomm_ishrink_f08.F90
new file mode 100644
index 00000000000..8828db2fa53
--- /dev/null
+++ b/ompi/mpiext/ftmpi/use-mpi-f08/profile/pcomm_ishrink_f08.F90
@@ -0,0 +1,35 @@
+! -*- f90 -*-
+!
+! Copyright (c) 2018-2022 The University of Tennessee and the University
+!                         of Tennessee Research Foundation.  All rights
+!                         reserved.
+! $COPYRIGHT$
+!
+! Additional copyrights may follow
+!
+! $HEADER$
+!
+
+subroutine PMPIX_Comm_ishrink_f08(comm, newcomm, request, ierror)
+  use :: mpi_f08_types, only : MPI_Comm, MPI_Request
+  implicit none
+  interface
+     subroutine ompix_comm_ishrink_f(comm, newcomm, request, ierror) &
+          BIND(C, name="ompix_comm_shrink_f")
+       implicit none
+       INTEGER, INTENT(IN) :: comm
+       INTEGER, INTENT(OUT) :: newcomm
+       INTEGER, INTENT(OUT) :: request
+       INTEGER, INTENT(OUT) :: ierror
+     end subroutine ompix_comm_ishrink_f
+  end interface
+  TYPE(MPI_Comm), INTENT(IN) :: comm
+  TYPE(MPI_Comm), INTENT(OUT) :: newcomm
+  TYPE(MPI_Request), INTENT(OUT) :: request
+  INTEGER, OPTIONAL, INTENT(OUT) :: ierror
+  integer :: c_ierror
+
+  call ompix_comm_ishrink_f(comm%MPI_VAL, newcomm%MPI_VAL, request%MPI_VAL, c_ierror)
+  if (present(ierror)) ierror = c_ierror
+
+end subroutine PMPIX_Comm_ishrink_f08
diff --git a/ompi/mpiext/ftmpi/use-mpi/mpiext_ftmpi_usempi.h b/ompi/mpiext/ftmpi/use-mpi/mpiext_ftmpi_usempi.h
index 34aa8dc1f42..08d750b4981 100644
--- a/ompi/mpiext/ftmpi/use-mpi/mpiext_ftmpi_usempi.h
+++ b/ompi/mpiext/ftmpi/use-mpi/mpiext_ftmpi_usempi.h
@@ -1,6 +1,6 @@
 ! -*- fortran -*-
 ! Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
-! Copyright (c) 2010-2020 The University of Tennessee and the University
+! Copyright (c) 2010-2022 The University of Tennessee and the University
 !                         of Tennessee research foundation.  All rights
 !                         reserved.
 ! $COPYRIGHT$
@@ -61,6 +61,50 @@ interface pmpix_comm_shrink
     end subroutine pmpix_comm_shrink
 end interface pmpix_comm_shrink
 
+interface mpix_comm_ishrink
+    subroutine mpix_comm_ishrink(comm, newcomm, request, ierr)
+      integer, intent(IN) :: comm
+      integer, intent(OUT) :: newcomm, ierr
+      integer, intent(OUT) :: request
+    end subroutine mpix_comm_ishrink
+end interface mpix_comm_ishrink
+
+interface pmpix_comm_ishrink
+    subroutine pmpix_comm_ishrink(comm, newcomm, request, ierr)
+      integer, intent(IN) :: comm
+      integer, intent(OUT) :: newcomm, ierr
+      integer, intent(OUT) :: request
+    end subroutine pmpix_comm_ishrink
+end interface pmpix_comm_ishrink
+
+interface mpix_comm_get_failed
+    subroutine mpix_comm_get_failed(comm, failedgrp, ierr)
+        integer, intent(IN) :: comm
+        integer, intent(OUT) :: failedgrp, ierr
+    end subroutine mpix_comm_get_failed
+end interface mpix_comm_get_failed
+
+interface pmpix_comm_get_failed
+    subroutine pmpix_comm_get_failed(comm, failedgrp, ierr)
+        integer, intent(IN) :: comm
+        integer, intent(OUT) :: failedgrp, ierr
+    end subroutine pmpix_comm_get_failed
+end interface pmpix_comm_get_failed
+
+interface mpix_comm_ack_failed
+    subroutine mpix_comm_ack_failed(comm, num_to_ack, num_acked, ierr)
+        integer, intent(IN) :: comm, num_to_ack
+        integer, intent(OUT) :: num_acked, ierr
+    end subroutine mpix_comm_ack_failed
+end interface mpix_comm_ack_failed
+
+interface pmpix_comm_ack_failed
+    subroutine pmpix_comm_ack_failed(comm, num_to_ack, num_acked, ierr)
+        integer, intent(IN) :: comm, num_to_ack
+        integer, intent(OUT) :: num_acked, ierr
+    end subroutine pmpix_comm_ack_failed
+end interface pmpix_comm_ack_failed
+
 interface mpix_comm_failure_ack
     subroutine mpix_comm_failure_ack(comm, ierr)
       integer, intent(IN) :: comm
diff --git a/ompi/mpiext/mpiext.c b/ompi/mpiext/mpiext.c
index 3a0012f125a..1c59b520c8f 100644
--- a/ompi/mpiext/mpiext.c
+++ b/ompi/mpiext/mpiext.c
@@ -1,3 +1,14 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2018      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
 #include "ompi_config.h"
 
 #include <stdlib.h>
@@ -5,17 +16,16 @@
 #include "ompi/constants.h"
 #include "ompi/mpiext/mpiext.h"
 #include "ompi/mpiext/static-components.h"
+#include "ompi/instance/instance.h"
 
-
-int
-ompi_mpiext_init(void)
+static int ompi_mpiext_fini (void)
 {
     const ompi_mpiext_component_t **tmp = ompi_mpiext_components;
     int ret;
 
     while (NULL != (*tmp)) {
-        if (NULL != (*tmp)->init) {
-            ret = (*tmp)->init();
+        if (NULL != (*tmp)->fini) {
+            ret = (*tmp)->fini();
             if (OMPI_SUCCESS != ret) return ret;
         }
         tmp++;
@@ -24,20 +34,21 @@ ompi_mpiext_init(void)
     return OMPI_SUCCESS;
 }
 
-
 int
-ompi_mpiext_fini(void)
+ompi_mpiext_init(void)
 {
     const ompi_mpiext_component_t **tmp = ompi_mpiext_components;
     int ret;
 
     while (NULL != (*tmp)) {
-        if (NULL != (*tmp)->fini) {
-            ret = (*tmp)->fini();
+        if (NULL != (*tmp)->init) {
+            ret = (*tmp)->init();
             if (OMPI_SUCCESS != ret) return ret;
         }
         tmp++;
     }
 
+    ompi_mpi_instance_append_finalize (ompi_mpiext_fini);
+
     return OMPI_SUCCESS;
 }
diff --git a/ompi/mpiext/mpiext.h b/ompi/mpiext/mpiext.h
index 6a93563c791..e5488cdc357 100644
--- a/ompi/mpiext/mpiext.h
+++ b/ompi/mpiext/mpiext.h
@@ -1,4 +1,11 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
+ * Copyright (c) 2018      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
  * $HEADER$
  */
 #if defined(c_plusplus) || defined(__cplusplus)
@@ -8,7 +15,6 @@ extern "C" {
 #include "ompi_config.h"
 
 OMPI_DECLSPEC int ompi_mpiext_init(void);
-OMPI_DECLSPEC int ompi_mpiext_fini(void);
 
 typedef int (*ompi_mpiext_init_fn_t)(void);
 typedef int (*ompi_mpiext_fini_fn_t)(void);
diff --git a/ompi/mpiext/pcollreq/Makefile.am b/ompi/mpiext/pcollreq/Makefile.am
deleted file mode 100644
index 329a4d1a9df..00000000000
--- a/ompi/mpiext/pcollreq/Makefile.am
+++ /dev/null
@@ -1,12 +0,0 @@
-#
-# Copyright (c) 2017-2018 FUJITSU LIMITED.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-SUBDIRS = c mpif-h use-mpi use-mpi-f08
-
-EXTRA_DIST = README.md
diff --git a/ompi/mpiext/pcollreq/README.md b/ompi/mpiext/pcollreq/README.md
deleted file mode 100644
index 124019ea735..00000000000
--- a/ompi/mpiext/pcollreq/README.md
+++ /dev/null
@@ -1,14 +0,0 @@
-# Open MPI extension: pcollreq
-
-Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-
-This extension provides the feature of persistent collective
-communication operations and persistent neighborhood collective
-communication operations, which is planned to be included in the next
-MPI Standard after MPI-3.1 as of Nov. 2018.
-
-See `MPIX_Barrier_init(3)` for more details.
-
-The code will be moved to the `ompi/mpi` directory and the `MPIX_`
-prefix will be switch to the `MPI_` prefix once the MPI Standard which
-includes this feature is published.
diff --git a/ompi/mpiext/pcollreq/c/MPIX_Allgather_init.3in b/ompi/mpiext/pcollreq/c/MPIX_Allgather_init.3in
deleted file mode 100644
index a9a04748c96..00000000000
--- a/ompi/mpiext/pcollreq/c/MPIX_Allgather_init.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPIX_Barrier_init.3
diff --git a/ompi/mpiext/pcollreq/c/MPIX_Allgatherv_init.3in b/ompi/mpiext/pcollreq/c/MPIX_Allgatherv_init.3in
deleted file mode 100644
index a9a04748c96..00000000000
--- a/ompi/mpiext/pcollreq/c/MPIX_Allgatherv_init.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPIX_Barrier_init.3
diff --git a/ompi/mpiext/pcollreq/c/MPIX_Allreduce_init.3in b/ompi/mpiext/pcollreq/c/MPIX_Allreduce_init.3in
deleted file mode 100644
index a9a04748c96..00000000000
--- a/ompi/mpiext/pcollreq/c/MPIX_Allreduce_init.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPIX_Barrier_init.3
diff --git a/ompi/mpiext/pcollreq/c/MPIX_Alltoall_init.3in b/ompi/mpiext/pcollreq/c/MPIX_Alltoall_init.3in
deleted file mode 100644
index a9a04748c96..00000000000
--- a/ompi/mpiext/pcollreq/c/MPIX_Alltoall_init.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPIX_Barrier_init.3
diff --git a/ompi/mpiext/pcollreq/c/MPIX_Alltoallv_init.3in b/ompi/mpiext/pcollreq/c/MPIX_Alltoallv_init.3in
deleted file mode 100644
index a9a04748c96..00000000000
--- a/ompi/mpiext/pcollreq/c/MPIX_Alltoallv_init.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPIX_Barrier_init.3
diff --git a/ompi/mpiext/pcollreq/c/MPIX_Alltoallw_init.3in b/ompi/mpiext/pcollreq/c/MPIX_Alltoallw_init.3in
deleted file mode 100644
index a9a04748c96..00000000000
--- a/ompi/mpiext/pcollreq/c/MPIX_Alltoallw_init.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPIX_Barrier_init.3
diff --git a/ompi/mpiext/pcollreq/c/MPIX_Barrier_init.3in b/ompi/mpiext/pcollreq/c/MPIX_Barrier_init.3in
deleted file mode 100644
index db6cccc9c54..00000000000
--- a/ompi/mpiext/pcollreq/c/MPIX_Barrier_init.3in
+++ /dev/null
@@ -1,565 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-.\" $COPYRIGHT$
-.TH MPIX_Barrier_init 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-\fBMPIX_Allgather_init, MPIX_Allgatherv_init, MPIX_Allreduce_init, MPIX_Alltoall_init, MPIX_Alltoallv_init, MPIX_Alltoallw_init, MPIX_Barrier_init, MPIX_Bcast_init, MPIX_Exscan_init, MPIX_Gather_init, MPIX_Gatherv_init, MPIX_Reduce_init, MPIX_Reduce_scatter_init, MPIX_Reduce_scatter_block_init, MPIX_Scan_init, MPIX_Scatter_init, MPIX_Scatterv_init, MPIX_Neighbor_allgather_init, MPIX_Neighbor_allgatherv_init, MPIX_Neighbor_alltoall_init, MPIX_Neighbor_alltoallv_init, MPIX_Neighbor_alltoallw_init\fP \- Builds a handle for a collective communication or neighborhood collective communication
-
-.SH SYNTAX
-.ft R
-.SH C Syntax
-.nf
-#include <mpi.h>
-#include <mpi-ext.h>
-int MPIX_Allgather_init(const void *\fIsendbuf\fP, int \fIsendcount\fP,
-	MPI_Datatype \fIsendtype\fP, void *\fIrecvbuf\fP, int \fIrecvcount\fP,
-	MPI_Datatype \fIrecvtype\fP, MPI_Comm \fIcomm\fP, MPI_Info \fIinfo\fP,
-	MPI_Request *\fIrequest\fP)
-
-int MPIX_Allgatherv_init(const void *\fIsendbuf\fP, int \fIsendcount\fP,
-	MPI_Datatype \fIsendtype\fP, void *\fIrecvbuf\fP, const int \fIrecvcounts\fP[],
-	const int \fIdispls\fP[], MPI_Datatype \fIrecvtype\fP, MPI_Comm \fIcomm\fP,
-	MPI_Info \fIinfo\fP, MPI_Request *\fIrequest\fP)
-
-int MPIX_Allreduce_init(const void *\fIsendbuf\fP, void *\fIrecvbuf\fP, int \fIcount\fP,
-	MPI_Datatype \fIdatatype\fP, MPI_Op \fIop\fP, MPI_Comm \fIcomm\fP, MPI_Info \fIinfo\fP,
-	MPI_Request *\fIrequest\fP)
-
-int MPIX_Alltoall_init(const void *\fIsendbuf\fP, int \fIsendcount\fP,
-	MPI_Datatype \fIsendtype\fP, void *\fIrecvbuf\fP, int \fIrecvcount\fP,
-	MPI_Datatype \fIrecvtype\fP, MPI_Comm \fIcomm\fP, MPI_Info \fIinfo\fP,
-	MPI_Request *\fIrequest\fP)
-
-int MPIX_Alltoallv_init(const void *\fIsendbuf\fP, const int \fIsendcounts\fP[],
-	const int \fIsdispls\fP[], MPI_Datatype \fIsendtype\fP, void *\fIrecvbuf\fP,
-	const int \fIrecvcounts\fP[], const int \fIrdispls\fP[], MPI_Datatype \fIrecvtype\fP,
-	MPI_Comm \fIcomm\fP, MPI_Info \fIinfo\fP, MPI_Request *\fIrequest\fP)
-
-int MPIX_Alltoallw_init(const void *\fIsendbuf\fP, const int \fIsendcounts\fP[],
-	const int \fIsdispls\fP[], const MPI_Datatype \fIsendtypes\fP[], void *\fIrecvbuf\fP,
-	const int \fIrecvcounts\fP[], const int \fIrdispls\fP[],
-	const MPI_Datatype \fIrecvtypes\fP[], MPI_Comm \fIcomm\fP, MPI_Info \fIinfo\fP,
-	MPI_Request *\fIrequest\fP)
-
-int MPIX_Barrier_init(MPI_Comm \fIcomm\fP, MPI_Info \fIinfo\fP,
-	MPI_Request *\fIrequest\fP)
-
-int MPIX_Bcast_init(void *\fIbuffer\fP, int \fIcount\fP, MPI_Datatype \fIdatatype\fP,
-	int \fIroot\fP, MPI_Comm \fIcomm\fP, MPI_Info \fIinfo\fP, MPI_Request *\fIrequest\fP)
-
-int MPIX_Exscan_init(const void *\fIsendbuf\fP, void *\fIrecvbuf\fP, int \fIcount\fP,
-	MPI_Datatype \fIdatatype\fP, MPI_Op \fIop\fP, MPI_Comm \fIcomm\fP, MPI_Info \fIinfo\fP,
-	MPI_Request *\fIrequest\fP)
-
-int MPIX_Gather_init(const void *\fIsendbuf\fP, int \fIsendcount\fP,
-	MPI_Datatype \fIsendtype\fP, void *\fIrecvbuf\fP, int \fIrecvcount\fP,
-	MPI_Datatype \fIrecvtype\fP, int \fIroot\fP, MPI_Comm \fIcomm\fP, MPI_Info \fIinfo\fP,
-	MPI_Request *\fIrequest\fP)
-
-int MPIX_Gatherv_init(const void *\fIsendbuf\fP, int \fIsendcount\fP,
-	MPI_Datatype \fIsendtype\fP, void *\fIrecvbuf\fP, const int \fIrecvcounts\fP[],
-	const int \fIdispls\fP[], MPI_Datatype \fIrecvtype\fP, int \fIroot\fP, MPI_Comm \fIcomm\fP,
-	MPI_Info \fIinfo\fP, MPI_Request *\fIrequest\fP)
-
-int MPIX_Reduce_init(const void *\fIsendbuf\fP, void *\fIrecvbuf\fP, int \fIcount\fP,
-	MPI_Datatype \fIdatatype\fP, MPI_Op \fIop\fP, int \fIroot\fP, MPI_Comm \fIcomm\fP,
-	MPI_Info \fIinfo\fP, MPI_Request *\fIrequest\fP)
-
-int MPIX_Reduce_scatter_init(const void *\fIsendbuf\fP, void *\fIrecvbuf\fP,
-	const int \fIrecvcounts\fP[], MPI_Datatype \fIdatatype\fP, MPI_Op \fIop\fP,
-	MPI_Comm \fIcomm\fP, MPI_Info \fIinfo\fP, MPI_Request *\fIrequest\fP)
-
-int MPIX_Reduce_scatter_block_init(const void *\fIsendbuf\fP, void *\fIrecvbuf\fP,
-	int \fIrecvcount\fP, MPI_Datatype \fIdatatype\fP, MPI_Op \fIop\fP, MPI_Comm \fIcomm\fP,
-	MPI_Info \fIinfo\fP, MPI_Request *\fIrequest\fP)
-
-int MPIX_Scan_init(const void *\fIsendbuf\fP, void *\fIrecvbuf\fP, int \fIcount\fP,
-	MPI_Datatype \fIdatatype\fP, MPI_Op \fIop\fP, MPI_Comm \fIcomm\fP, MPI_Info \fIinfo\fP,
-	MPI_Request *\fIrequest\fP)
-
-int MPIX_Scatter_init(const void *\fIsendbuf\fP, int \fIsendcount\fP,
-	MPI_Datatype \fIsendtype\fP, void *\fIrecvbuf\fP, int \fIrecvcount\fP,
-	MPI_Datatype \fIrecvtype\fP, int \fIroot\fP, MPI_Comm \fIcomm\fP, MPI_Info \fIinfo\fP,
-	MPI_Request *\fIrequest\fP)
-
-int MPIX_Scatterv_init(const void *\fIsendbuf\fP, const int \fIsendcounts\fP[],
-	const int \fIdispls\fP[], MPI_Datatype \fIsendtype\fP, void *\fIrecvbuf\fP,
-	int \fIrecvcount\fP, MPI_Datatype \fIrecvtype\fP, int \fIroot\fP, MPI_Comm \fIcomm\fP,
-	MPI_Info \fIinfo\fP, MPI_Request *\fIrequest\fP)
-
-int MPIX_Neighbor_allgather_init(const void *\fIsendbuf\fP, int \fIsendcount\fP,
-	MPI_Datatype \fIsendtype\fP, void *\fIrecvbuf\fP, int \fIrecvcount\fP,
-	MPI_Datatype \fIrecvtype\fP, MPI_Comm \fIcomm\fP, MPI_Info \fIinfo\fP,
-	MPI_Request *\fIrequest\fP)
-
-int MPIX_Neighbor_allgatherv_init(const void *\fIsendbuf\fP, int \fIsendcount\fP,
-	MPI_Datatype \fIsendtype\fP, void *\fIrecvbuf\fP, const int \fIrecvcounts\fP[],
-	const int \fIdispls\fP[], MPI_Datatype \fIrecvtype\fP, MPI_Comm \fIcomm\fP,
-	MPI_Info \fIinfo\fP, MPI_Request *\fIrequest\fP)
-
-int MPIX_Neighbor_alltoall_init(const void *\fIsendbuf\fP, int \fIsendcount\fP,
-	MPI_Datatype \fIsendtype\fP, void *\fIrecvbuf\fP, int \fIrecvcount\fP,
-	MPI_Datatype \fIrecvtype\fP, MPI_Comm \fIcomm\fP, MPI_Info \fIinfo\fP,
-	MPI_Request *\fIrequest\fP)
-
-int MPIX_Neighbor_alltoallv_init(const void *\fIsendbuf\fP,
-	const int \fIsendcounts\fP[], const int \fIsdispls\fP[], MPI_Datatype \fIsendtype\fP,
-	void *\fIrecvbuf\fP, const int \fIrecvcounts\fP[], const int \fIrdispls\fP[],
-	MPI_Datatype \fIrecvtype\fP, MPI_Comm \fIcomm\fP, MPI_Info \fIinfo\fP,
-	MPI_Request *\fIrequest\fP)
-
-int MPIX_Neighbor_alltoallw_init(const void *\fIsendbuf\fP,
-	const int \fIsendcounts\fP[], const MPI_Aint \fIsdispls\fP[],
-	const MPI_Datatype \fIsendtypes\fP[], void *\fIrecvbuf\fP,
-	const int \fIrecvcounts\fP[], const MPI_Aint \fIrdispls\fP[],
-	const MPI_Datatype \fIrecvtypes\fP[], MPI_Comm \fIcomm\fP, MPI_Info \fIinfo\fP,
-	MPI_Request *\fIrequest\fP)
-
-.fi
-.SH Fortran Syntax
-.nf
-USE MPI
-USE MPI_EXT
-! or the older form: INCLUDE 'mpif.h'; INCLUDE 'mpif-ext.h'
-MPIX_ALLGATHER_INIT(\fISENDBUF, SENDCOUNT, SENDTYPE, RECVBUF, RECVCOUNT,
-		RECVTYPE, COMM, INFO, REQUEST, IERROR\fP)
-	<type>	\fISENDBUF\fP(*)\fI, RECVBUF\fP(*)
-	INTEGER	\fISENDCOUNT, SENDTYPE, RECVCOUNT, RECVTYPE, COMM, INFO\fP
-	INTEGER	\fIREQUEST, IERROR\fP
-
-MPIX_ALLGATHERV_INIT(\fISENDBUF, SENDCOUNT, SENDTYPE, RECVBUF,
-		RECVCOUNT, DISPLS, RECVTYPE, COMM, INFO, REQUEST, IERROR\fP)
-	<type>	\fISENDBUF\fP(*)\fI, RECVBUF\fP(*)
-	INTEGER	\fISENDCOUNT, SENDTYPE, RECVCOUNT\fP(*)
-	INTEGER	\fIDISPLS\fP(*)\fI, RECVTYPE, COMM, INFO, REQUEST, IERROR\fP
-
-MPIX_ALLREDUCE_INIT(\fISENDBUF, RECVBUF, COUNT, DATATYPE, OP, COMM, INFO,
-		REQUEST, IERROR\fP)
-	<type>	\fISENDBUF\fP(*)\fI, RECVBUF\fP(*)
-	INTEGER	\fICOUNT, DATATYPE, OP, COMM, INFO, REQUEST, IERROR\fP
-
-MPIX_ALLTOALL_INIT(\fISENDBUF, SENDCOUNT, SENDTYPE, RECVBUF, RECVCOUNT,
-		RECVTYPE, COMM, INFO, REQUEST, IERROR\fP)
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fISENDCOUNT, SENDTYPE, RECVCOUNT, RECVTYPE\fP
-	INTEGER	\fICOMM, INFO, REQUEST, IERROR\fP
-
-MPIX_ALLTOALLV_INIT(\fISENDBUF, SENDCOUNTS, SDISPLS, SENDTYPE,
-		RECVBUF, RECVCOUNTS, RDISPLS, RECVTYPE, COMM, INFO, REQUEST,
-		IERROR\fP)
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fISENDCOUNTS(*), SDISPLS(*), SENDTYPE\fP
-	INTEGER	\fIRECVCOUNTS(*), RDISPLS(*), RECVTYPE\fP
-	INTEGER	\fICOMM, INFO, REQUEST, IERROR\fP
-
-MPIX_ALLTOALLW_INIT(\fISENDBUF, SENDCOUNTS, SDISPLS, SENDTYPES,
-		RECVBUF, RECVCOUNTS, RDISPLS, RECVTYPES, COMM, INFO, REQUEST,
-		IERROR\fP)
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fISENDCOUNTS(*), SDISPLS(*), SENDTYPES(*)\fP
-	INTEGER	\fIRECVCOUNTS(*), RDISPLS(*), RECVTYPES(*)\fP
-	INTEGER	\fICOMM, INFO, REQUEST, IERROR\fP
-
-MPIX_BARRIER_INIT(\fICOMM\fP, \fIINFO\fP, \fIREQUEST\fP, \fIIERROR\fP)
-	INTEGER	\fICOMM\fP, \fIINFO\fP, \fIREQUEST\fP, \fIIERROR\fP
-
-MPIX_BCAST_INIT(\fIBUFFER\fP, \fICOUNT\fP, \fIDATATYPE\fP, \fIROOT\fP, \fICOMM\fP, \fIINFO\fP, \fIREQUEST\fP,
-		\fIIERROR\fP)
-	<type>	\fIBUFFER\fP(*)
-	INTEGER	\fICOUNT\fP, \fIDATATYPE\fP, \fIROOT\fP, \fICOMM\fP, \fIINFO\fP, \fIREQUEST\fP, \fIIERROR\fP
-
-MPIX_EXSCAN_INIT(\fISENDBUF, RECVBUF, COUNT, DATATYPE, OP, COMM, INFO,
-		REQUEST, IERROR\fP)
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fICOUNT, DATATYPE, OP, COMM, INFO, REQUEST, IERROR\fP
-
-MPIX_GATHER_INIT(\fISENDBUF, SENDCOUNT, SENDTYPE, RECVBUF, RECVCOUNT,
-		RECVTYPE, ROOT, COMM, INFO, REQUEST, IERROR\fP)
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fISENDCOUNT, SENDTYPE, RECVCOUNT, RECVTYPE, ROOT\fP
-	INTEGER	\fICOMM, INFO, REQUEST, IERROR\fP
-
-MPIX_GATHERV_INIT(\fISENDBUF, SENDCOUNT, SENDTYPE, RECVBUF, RECVCOUNTS,
-		DISPLS, RECVTYPE, ROOT, COMM, INFO, REQUEST, IERROR\fP)
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fISENDCOUNT, SENDTYPE, RECVCOUNTS(*), DISPLS(*)\fP
-	INTEGER	\fIRECVTYPE, ROOT, COMM, INFO, REQUEST, IERROR\fP
-
-MPIX_REDUCE_INIT(\fISENDBUF, RECVBUF, COUNT, DATATYPE, OP, ROOT, COMM,
-		INFO, REQUEST, IERROR\fP)
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fICOUNT, DATATYPE, OP, ROOT, COMM, INFO, REQUEST, IERROR\fP
-
-MPIX_REDUCE_SCATTER_INIT(\fISENDBUF, RECVBUF, RECVCOUNTS, DATATYPE, OP,
-		COMM, INFO, REQUEST, IERROR\fP)
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fIRECVCOUNTS(*), DATATYPE, OP, COMM, INFO, REQUEST, IERROR \fP
-
-MPIX_REDUCE_SCATTER_BLOCK_INIT(\fISENDBUF, RECVBUF, RECVCOUNT, DATATYPE,
-		OP, COMM, INFO, REQUEST, IERROR\fP)
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fIRECVCOUNT, DATATYPE, OP, COMM, INFO, REQUEST, IERROR \fP
-
-MPIX_SCAN_INIT(\fISENDBUF, RECVBUF, COUNT, DATATYPE, OP, COMM, INFO,
-		REQUEST, IERROR\fP)
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fICOUNT, DATATYPE, OP, COMM, INFO, REQUEST, IERROR\fP
-
-MPIX_SCATTER_INIT(\fISENDBUF, SENDCOUNT, SENDTYPE, RECVBUF, RECVCOUNT,
-		RECVTYPE, ROOT, COMM, INFO, REQUEST, IERROR\fP)
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fISENDCOUNT, SENDTYPE, RECVCOUNT, RECVTYPE, ROOT\fP
-	INTEGER	\fICOMM, INFO, REQUEST, IERROR\fP
-
-MPIX_SCATTERV_INIT(\fISENDBUF, SENDCOUNTS, DISPLS, SENDTYPE, RECVBUF,
-		RECVCOUNT, RECVTYPE, ROOT, COMM, INFO, REQUEST, IERROR\fP)
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fISENDCOUNTS(*), DISPLS(*), SENDTYPE\fP
-	INTEGER	\fIRECVCOUNT, RECVTYPE, ROOT, COMM, INFO, REQUEST, IERROR\fP
-
-MPIX_NEIGHBOR_ALLGATHER_INIT(\fISENDBUF\fP, \fISENDCOUNT\fP, \fISENDTYPE\fP, \fIRECVBUF\fP,
-		\fIRECVCOUNT\fP, \fIRECVTYPE\fP, \fICOMM\fP, \fIINFO\fP, \fIREQUEST\fP, \fIIERROR\fP)
-	<type>	\fISENDBUF\fP(*), \fIRECVBUF\fP(*)
-	INTEGER	\fISENDCOUNT\fP, \fISENDTYPE\fP, \fIRECVCOUNT\fP, \fIRECVTYPE\fP, \fICOMM\fP,
-	INTEGER	\fIINFO, REQUEST, IERROR\fP
-
-MPIX_NEIGHBOR_ALLGATHERV_INIT(\fISENDBUF\fP, \fISENDCOUNT\fP, \fISENDTYPE\fP, \fIRECVBUF\fP,
-		\fIRECVCOUNT\fP, \fIDISPLS\fP, \fIRECVTYPE\fP, \fICOMM\fP, \fIINFO\fP, \fIREQUEST\fP, \fIIERROR\fP)
-	<type>	\fISENDBUF\fP(*), \fIRECVBUF\fP(*)
-	INTEGER	\fISENDCOUNT\fP, \fISENDTYPE\fP, \fIRECVCOUNT\fP(*),
-	INTEGER	\fIDISPLS\fP(*), \fIRECVTYPE\fP, \fICOMM\fP, \fIINFO\fP, \fIREQUEST\fP, \fIIERROR\fP
-
-MPIX_NEIGHBOR_ALLTOALL_INIT(\fISENDBUF, SENDCOUNT, SENDTYPE, RECVBUF,
-		RECVCOUNT, RECVTYPE, COMM, INFO, REQUEST, IERROR\fP)
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fISENDCOUNT, SENDTYPE, RECVCOUNT, RECVTYPE\fP
-	INTEGER	\fICOMM, INFO, REQUEST, IERROR\fP
-
-MPIX_NEIGHBOR_ALLTOALLV_INIT(\fISENDBUF, SENDCOUNTS, SDISPLS, SENDTYPE,
-		RECVBUF, RECVCOUNTS, RDISPLS, RECVTYPE, COMM, INFO, REQUEST,
-		IERROR\fP)
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fISENDCOUNTS(*), SDISPLS(*), SENDTYPE\fP
-	INTEGER	\fIRECVCOUNTS(*), RDISPLS(*), RECVTYPE\fP
-	INTEGER	\fICOMM, INFO, REQUEST, IERROR\fP
-
-MPIX_NEIGHBOR_ALLTOALLW_INIT(\fISENDBUF, SENDCOUNTS, SDISPLS, SENDTYPES,
-		RECVBUF, RECVCOUNTS, RDISPLS, RECVTYPES, COMM, INFO, REQUEST,
-		IERROR\fP)
-	<type>	\fISENDBUF(*), RECVBUF(*)\fP
-	INTEGER	\fISENDCOUNTS(*), SENDTYPES(*)\fP
-	INTEGER	\fIRECVCOUNTS(*), RECVTYPES(*)\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND) \fISDISPLS(*), RDISPLS(*)\fP
-	INTEGER	\fICOMM, INFO, REQUEST, IERROR\fP
-
-.fi
-.SH Fortran 2008 Syntax
-.nf
-USE mpi_f08
-USE mpi_f08_ext
-MPIX_Allgather_init(\fIsendbuf\fP, \fIsendcount\fP, \fIsendtype\fP, \fIrecvbuf\fP, \fIrecvcount\fP,
-		\fIrecvtype\fP, \fIcomm\fP, \fIinfo\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIsendcount\fP, \fIrecvcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPIX_Allgatherv_init(\fIsendbuf\fP, \fIsendcount\fP, \fIsendtype\fP, \fIrecvbuf\fP, \fIrecvcounts\fP,
-		\fIdispls\fP, \fIrecvtype\fP, \fIcomm\fP, \fIinfo\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIsendcount\fP
-	INTEGER, INTENT(IN), ASYNCHRONOUS :: \fIrecvcounts(*)\fP, \fIdispls(*)\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPIX_Allreduce_init(\fIsendbuf\fP, \fIrecvbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIop\fP, \fIcomm\fP, \fIinfo\fP,
-		\fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Op), INTENT(IN) :: \fIop\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPIX_Alltoall_init(\fIsendbuf\fP, \fIsendcount\fP, \fIsendtype\fP, \fIrecvbuf\fP, \fIrecvcount\fP,
-		\fIrecvtype\fP, \fIcomm\fP, \fIinfo\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIsendcount\fP, \fIrecvcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPIX_Alltoallv_init(\fIsendbuf\fP, \fIsendcounts\fP, \fIsdispls\fP, \fIsendtype\fP, \fIrecvbuf\fP,
-		\fIrecvcounts\fP, \fIrdispls\fP, \fIrecvtype\fP, \fIcomm\fP, \fIinfo\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN), ASYNCHRONOUS :: \fIsendcounts(*)\fP, \fIsdispls(*)\fP,
-	\fIrecvcounts(*)\fP, \fIrdispls(*)\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPIX_Alltoallw_init(\fIsendbuf\fP, \fIsendcounts\fP, \fIsdispls\fP, \fIsendtypes\fP, \fIrecvbuf\fP,
-		\fIrecvcounts\fP, \fIrdispls\fP, \fIrecvtypes\fP, \fIcomm\fP, \fIinfo\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN), ASYNCHRONOUS :: \fIsendcounts(*)\fP, \fIsdispls(*)\fP,
-	\fIrecvcounts(*)\fP, \fIrdispls(*)\fP
-	TYPE(MPI_Datatype), INTENT(IN), ASYNCHRONOUS :: \fIsendtypes(*)\fP,
-	\fIrecvtypes(*)\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPIX_Barrier_init(\fIcomm\fP, \fIinfo\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPIX_Bcast_init(\fIbuffer\fP, \fIcount\fP, \fIdatatype\fP, \fIroot\fP, \fIcomm\fP, \fIinfo\fP, \fIrequest\fP,
-		\fIierror\fP)
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIbuffer\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP, \fIroot\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPIX_Exscan_init(\fIsendbuf\fP, \fIrecvbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIop\fP, \fIcomm\fP, \fIinfo\fP,
-		\fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Op), INTENT(IN) :: \fIop\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPIX_Gather_init(\fIsendbuf\fP, \fIsendcount\fP, \fIsendtype\fP, \fIrecvbuf\fP,
-		\fIrecvcount\fP, \fIrecvtype\fP, \fIroot\fP, \fIcomm\fP, \fIinfo\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIsendcount\fP, \fIrecvcount\fP, \fIroot\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPIX_Gatherv_init(\fIsendbuf\fP, \fIsendcount\fP, \fIsendtype\fP, \fIrecvbuf\fP, \fIrecvcounts\fP,
-		\fIdispls\fP, \fIrecvtype\fP, \fIroot\fP, \fIcomm\fP, \fIinfo\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIsendcount\fP, \fIroot\fP
-	INTEGER, INTENT(IN), ASYNCHRONOUS :: \fIrecvcounts(*)\fP, \fIdispls(*)\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPIX_Reduce_init(\fIsendbuf\fP, \fIrecvbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIop\fP, \fIroot\fP, \fIcomm\fP,
-		\fIinfo\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP, \fIroot\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Op), INTENT(IN) :: \fIop\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPIX_Reduce_scatter_init(\fIsendbuf\fP, \fIrecvbuf\fP, \fIrecvcounts\fP, \fIdatatype\fP, \fIop\fP,
-		\fIcomm\fP, \fIinfo\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN), ASYNCHRONOUS :: \fIrecvcounts(*)\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Op), INTENT(IN) :: \fIop\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPIX_Reduce_scatter_block_init(\fIsendbuf\fP, \fIrecvbuf\fP, \fIrecvcount\fP, \fIdatatype\fP,
-		\fIop\fP, \fIcomm\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIrecvcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Op), INTENT(IN) :: \fIop\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPIX_Reduce_scatter_block_init(\fIsendbuf\fP, \fIrecvbuf\fP, \fIrecvcount\fP, \fIdatatype\fP, \fIop\fP,
-		\fIcomm\fP, \fIinfo\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIrecvcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Op), INTENT(IN) :: \fIop\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPIX_Scan_init(\fIsendbuf\fP, \fIrecvbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIop\fP, \fIcomm\fP, \fIinfo\fP,
-		\fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIdatatype\fP
-	TYPE(MPI_Op), INTENT(IN) :: \fIop\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPIX_Scatter_init(\fIsendbuf\fP, \fIsendcount\fP, \fIsendtype\fP, \fIrecvbuf\fP, \fIrecvcount\fP,
-		\fIrecvtype\fP, \fIroot\fP, \fIcomm\fP, \fIinfo\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIsendcount\fP, \fIrecvcount\fP, \fIroot\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPIX_Scatterv_init(\fIsendbuf\fP, \fIsendcounts\fP, \fIdispls\fP, \fIsendtype\fP, \fIrecvbuf\fP,
-		\fIrecvcount\fP, \fIrecvtype\fP, \fIroot\fP, \fIcomm\fP, \fIinfo\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN), ASYNCHRONOUS :: \fIsendcounts(*)\fP, \fIdispls(*)\fP
-	INTEGER, INTENT(IN) :: \fIrecvcount\fP, \fIroot\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPIX_Neighbor_allgather_init(\fIsendbuf\fP, \fIsendcount\fP, \fIsendtype\fP, \fIrecvbuf\fP,
-		\fIrecvcount\fP, \fIrecvtype\fP, \fIcomm\fP, \fIinfo\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIsendcount\fP, \fIrecvcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPIX_Neighbor_allgatherv_init(\fIsendbuf\fP, \fIsendcount\fP, \fIsendtype\fP, \fIrecvbuf\fP,
-		\fIrecvcounts\fP, \fIdispls\fP, \fIrecvtype\fP, \fIcomm\fP, \fIinfo\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIsendcount\fP
-	INTEGER, INTENT(IN), ASYNCHRONOUS :: \fIrecvcounts(*)\fP, \fIdispls(*)\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPIX_Neighbor_alltoall_init(\fIsendbuf\fP, \fIsendcount\fP, \fIsendtype\fP, \fIrecvbuf\fP,
-		\fIrecvcount\fP, \fIrecvtype\fP, \fIcomm\fP, \fIinfo\fP, \fIrequest\fP, \fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN) :: \fIsendcount\fP, \fIrecvcount\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPIX_Neighbor_alltoallv_init(\fIsendbuf\fP, \fIsendcounts\fP, \fIsdispls\fP, \fIsendtype\fP,
-		\fIrecvbuf\fP, \fIrecvcounts\fP, \fIrdispls\fP, \fIrecvtype\fP, \fIcomm\fP, \fIinfo\fP, \fIrequest\fP,
-		\fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN), ASYNCHRONOUS :: \fIsendcounts(*)\fP, \fIsdispls(*)\fP,
-	\fIrecvcounts(*)\fP, \fIrdispls(*)\fP
-	TYPE(MPI_Datatype), INTENT(IN) :: \fIsendtype\fP, \fIrecvtype\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-MPIX_Neighbor_alltoallw_init(\fIsendbuf\fP, \fIsendcounts\fP, \fIsdispls\fP, \fIsendtypes\fP,
-		\fIrecvbuf\fP, \fIrecvcounts\fP, \fIrdispls\fP, \fIrecvtypes\fP, \fIcomm\fP, \fIinfo\fP, \fIrequest\fP,
-		\fIierror\fP)
-	TYPE(*), DIMENSION(..), INTENT(IN), ASYNCHRONOUS :: \fIsendbuf\fP
-	TYPE(*), DIMENSION(..), ASYNCHRONOUS :: \fIrecvbuf\fP
-	INTEGER, INTENT(IN), ASYNCHRONOUS :: \fIsendcounts(*)\fP, \fIrecvcounts(*)\fP
-	INTEGER(KIND=MPI_ADDRESS_KIND), INTENT(IN), ASYNCHRONOUS ::
-	\fIsdispls(*)\fP, \fIrdispls(*)\fP
-	TYPE(MPI_Datatype), INTENT(IN), ASYNCHRONOUS :: \fIsendtypes(*)\fP,
-	\fIrecvtypes(*)\fP
-	TYPE(MPI_Comm), INTENT(IN) :: \fIcomm\fP
-	TYPE(MPI_Info), INTENT(IN) :: \fIinfo\fP
-	TYPE(MPI_Request), INTENT(OUT) :: \fIrequest\fP
-	INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP
-
-.fi
-
-.SH DESCRIPTION
-.ft R
-Creates a persistent communication request for a collective operation or neighborhood collective operation.
-
-As of Nov. 2018, the feature of persistent collective communication operations and persistent collective neighborhood communication operations is planned to be included in the next MPI Standard after MPI-3.1.
-.nf
-
-    https://github.com/mpi-forum/mpi-issues/issues/25
-.fi
-
-Open MPI implements 2018 Draft Specification of the MPI standard shown in the following URL.
-.nf
-
-    https://www.mpi-forum.org/docs/drafts/mpi-2018-draft-report.pdf
-.fi
-
-The interface may still change in the standard. Therefore the prefix \fIMPIX_\fP is used instead of \fIMPI_\fP for these request creation routines. To start, complete, and free the created request, usual MPI routines (\fIMPI_Start\fP etc.) can be used.
-
-Future versions of Open MPI will switch to the \fIMPI_\fP prefix and will not require the C header file \fImpi-ext.h\fP, the Fortran modules \fImpi_ext\fP and \fImpi_f08_ext\fP, and the Fortran header file \fImpif-ext.h\fP once the MPI Standard which includes this feature is published.
-
-.SH EXAMPLE
-.nf
-
-    MPI_Request req;
-    MPIX_Barrier_init(MPI_COMM_WORLD, MPI_INFO_NULL, &req);
-    MPI_Start(&req);
-    MPI_Wait(&req, MPI_STATUS_IGNORE);
-    MPI_Request_free(&req);
-.fi
-
-.SH SEE ALSO
-.ft R
-.sp
-MPI_Start
-.br
-MPI_Startall
-.br
-MPI_Request_free
diff --git a/ompi/mpiext/pcollreq/c/MPIX_Bcast_init.3in b/ompi/mpiext/pcollreq/c/MPIX_Bcast_init.3in
deleted file mode 100644
index a9a04748c96..00000000000
--- a/ompi/mpiext/pcollreq/c/MPIX_Bcast_init.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPIX_Barrier_init.3
diff --git a/ompi/mpiext/pcollreq/c/MPIX_Exscan_init.3in b/ompi/mpiext/pcollreq/c/MPIX_Exscan_init.3in
deleted file mode 100644
index a9a04748c96..00000000000
--- a/ompi/mpiext/pcollreq/c/MPIX_Exscan_init.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPIX_Barrier_init.3
diff --git a/ompi/mpiext/pcollreq/c/MPIX_Gather_init.3in b/ompi/mpiext/pcollreq/c/MPIX_Gather_init.3in
deleted file mode 100644
index a9a04748c96..00000000000
--- a/ompi/mpiext/pcollreq/c/MPIX_Gather_init.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPIX_Barrier_init.3
diff --git a/ompi/mpiext/pcollreq/c/MPIX_Gatherv_init.3in b/ompi/mpiext/pcollreq/c/MPIX_Gatherv_init.3in
deleted file mode 100644
index a9a04748c96..00000000000
--- a/ompi/mpiext/pcollreq/c/MPIX_Gatherv_init.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPIX_Barrier_init.3
diff --git a/ompi/mpiext/pcollreq/c/MPIX_Neighbor_allgather_init.3in b/ompi/mpiext/pcollreq/c/MPIX_Neighbor_allgather_init.3in
deleted file mode 100644
index a9a04748c96..00000000000
--- a/ompi/mpiext/pcollreq/c/MPIX_Neighbor_allgather_init.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPIX_Barrier_init.3
diff --git a/ompi/mpiext/pcollreq/c/MPIX_Neighbor_allgatherv_init.3in b/ompi/mpiext/pcollreq/c/MPIX_Neighbor_allgatherv_init.3in
deleted file mode 100644
index a9a04748c96..00000000000
--- a/ompi/mpiext/pcollreq/c/MPIX_Neighbor_allgatherv_init.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPIX_Barrier_init.3
diff --git a/ompi/mpiext/pcollreq/c/MPIX_Neighbor_alltoall_init.3in b/ompi/mpiext/pcollreq/c/MPIX_Neighbor_alltoall_init.3in
deleted file mode 100644
index a9a04748c96..00000000000
--- a/ompi/mpiext/pcollreq/c/MPIX_Neighbor_alltoall_init.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPIX_Barrier_init.3
diff --git a/ompi/mpiext/pcollreq/c/MPIX_Neighbor_alltoallv_init.3in b/ompi/mpiext/pcollreq/c/MPIX_Neighbor_alltoallv_init.3in
deleted file mode 100644
index a9a04748c96..00000000000
--- a/ompi/mpiext/pcollreq/c/MPIX_Neighbor_alltoallv_init.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPIX_Barrier_init.3
diff --git a/ompi/mpiext/pcollreq/c/MPIX_Neighbor_alltoallw_init.3in b/ompi/mpiext/pcollreq/c/MPIX_Neighbor_alltoallw_init.3in
deleted file mode 100644
index a9a04748c96..00000000000
--- a/ompi/mpiext/pcollreq/c/MPIX_Neighbor_alltoallw_init.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPIX_Barrier_init.3
diff --git a/ompi/mpiext/pcollreq/c/MPIX_Reduce_init.3in b/ompi/mpiext/pcollreq/c/MPIX_Reduce_init.3in
deleted file mode 100644
index a9a04748c96..00000000000
--- a/ompi/mpiext/pcollreq/c/MPIX_Reduce_init.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPIX_Barrier_init.3
diff --git a/ompi/mpiext/pcollreq/c/MPIX_Reduce_scatter_block_init.3in b/ompi/mpiext/pcollreq/c/MPIX_Reduce_scatter_block_init.3in
deleted file mode 100644
index a9a04748c96..00000000000
--- a/ompi/mpiext/pcollreq/c/MPIX_Reduce_scatter_block_init.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPIX_Barrier_init.3
diff --git a/ompi/mpiext/pcollreq/c/MPIX_Reduce_scatter_init.3in b/ompi/mpiext/pcollreq/c/MPIX_Reduce_scatter_init.3in
deleted file mode 100644
index a9a04748c96..00000000000
--- a/ompi/mpiext/pcollreq/c/MPIX_Reduce_scatter_init.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPIX_Barrier_init.3
diff --git a/ompi/mpiext/pcollreq/c/MPIX_Scan_init.3in b/ompi/mpiext/pcollreq/c/MPIX_Scan_init.3in
deleted file mode 100644
index a9a04748c96..00000000000
--- a/ompi/mpiext/pcollreq/c/MPIX_Scan_init.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPIX_Barrier_init.3
diff --git a/ompi/mpiext/pcollreq/c/MPIX_Scatter_init.3in b/ompi/mpiext/pcollreq/c/MPIX_Scatter_init.3in
deleted file mode 100644
index a9a04748c96..00000000000
--- a/ompi/mpiext/pcollreq/c/MPIX_Scatter_init.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPIX_Barrier_init.3
diff --git a/ompi/mpiext/pcollreq/c/MPIX_Scatterv_init.3in b/ompi/mpiext/pcollreq/c/MPIX_Scatterv_init.3in
deleted file mode 100644
index a9a04748c96..00000000000
--- a/ompi/mpiext/pcollreq/c/MPIX_Scatterv_init.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/MPIX_Barrier_init.3
diff --git a/ompi/mpiext/pcollreq/c/Makefile.am b/ompi/mpiext/pcollreq/c/Makefile.am
deleted file mode 100644
index d4aeb8ef362..00000000000
--- a/ompi/mpiext/pcollreq/c/Makefile.am
+++ /dev/null
@@ -1,97 +0,0 @@
-#
-# Copyright (c) 2017      FUJITSU LIMITED.  All rights reserved.
-# Copyright (c) 2018      Research Organization for Information Science
-#                         and Technology (RIST). All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-SUBDIRS = profile
-
-# OMPI_BUILD_MPI_PROFILING is enabled when we want our generated MPI_* symbols
-# to be replaced by PMPI_*.
-# In this directory, we need it to be 0
-
-AM_CPPFLAGS     = -DOMPI_BUILD_MPI_PROFILING=0
-
-include $(top_srcdir)/Makefile.ompi-rules
-
-noinst_LTLIBRARIES = libmpiext_pcollreq_c.la
-
-# This is where the top-level header file (that is included in
-# <mpi-ext.h>) must be installed.
-ompidir = $(ompiincludedir)/mpiext
-
-# This is the header file that is installed.
-ompi_HEADERS = mpiext_pcollreq_c.h
-
-# Sources for the convenience libtool library.  Other than the one
-# header file, all source files in the extension have no file naming
-# conventions.
-libmpiext_pcollreq_c_la_SOURCES = \
-        $(ompi_HEADERS) \
-        mpiext_pcollreq_c.c
-
-if BUILD_MPI_BINDINGS_LAYER
-libmpiext_pcollreq_c_la_SOURCES += \
-        allgather_init.c \
-        allgatherv_init.c \
-        allreduce_init.c \
-        alltoall_init.c \
-        alltoallv_init.c \
-        alltoallw_init.c \
-        barrier_init.c \
-        bcast_init.c \
-        exscan_init.c \
-        gather_init.c \
-        gatherv_init.c \
-        reduce_init.c \
-        reduce_scatter_block_init.c \
-        reduce_scatter_init.c \
-        scan_init.c \
-        scatter_init.c \
-        scatterv_init.c \
-        \
-        neighbor_allgather_init.c \
-        neighbor_allgatherv_init.c \
-        neighbor_alltoall_init.c \
-        neighbor_alltoallv_init.c \
-        neighbor_alltoallw_init.c
-endif
-
-libmpiext_pcollreq_c_la_LIBADD = \
-        profile/libpmpiext_pcollreq_c.la
-
-# Man page installation
-nodist_man_MANS = \
-        MPIX_Allgather_init.3 \
-        MPIX_Allgatherv_init.3 \
-        MPIX_Allreduce_init.3 \
-        MPIX_Alltoall_init.3 \
-        MPIX_Alltoallv_init.3 \
-        MPIX_Alltoallw_init.3 \
-        MPIX_Barrier_init.3 \
-        MPIX_Bcast_init.3 \
-        MPIX_Exscan_init.3 \
-        MPIX_Gather_init.3 \
-        MPIX_Gatherv_init.3 \
-        MPIX_Reduce_init.3 \
-        MPIX_Reduce_scatter_block_init.3 \
-        MPIX_Reduce_scatter_init.3 \
-        MPIX_Scan_init.3 \
-        MPIX_Scatter_init.3 \
-        MPIX_Scatterv_init.3 \
-        MPIX_Neighbor_allgather_init.3 \
-        MPIX_Neighbor_allgatherv_init.3 \
-        MPIX_Neighbor_alltoall_init.3 \
-        MPIX_Neighbor_alltoallv_init.3 \
-        MPIX_Neighbor_alltoallw_init.3
-
-# Man page sources
-EXTRA_DIST = $(nodist_man_MANS:.3=.3in)
-
-distclean-local:
-	rm -f $(nodist_man_MANS)
diff --git a/ompi/mpiext/pcollreq/c/allgather_init.c b/ompi/mpiext/pcollreq/c/allgather_init.c
deleted file mode 100644
index 1207fbac685..00000000000
--- a/ompi/mpiext/pcollreq/c/allgather_init.c
+++ /dev/null
@@ -1,110 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012      Oak Ridge National Laboratory. All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_Allgather_init = PMPIX_Allgather_init
-#endif
-#define MPIX_Allgather_init PMPIX_Allgather_init
-#endif
-
-static const char FUNC_NAME[] = "MPIX_Allgather_init";
-
-
-int MPIX_Allgather_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                   void *recvbuf, int recvcount, MPI_Datatype recvtype,
-                   MPI_Comm comm, MPI_Info info, MPI_Request *request)
-{
-    int err;
-
-    SPC_RECORD(OMPI_SPC_ALLGATHER_INIT, 1);
-
-    MEMCHECKER(
-        int rank;
-        ptrdiff_t ext;
-
-        rank = ompi_comm_rank(comm);
-        ompi_datatype_type_extent(recvtype, &ext);
-
-        memchecker_datatype(recvtype);
-        memchecker_comm(comm);
-        /* check whether the actual send buffer is defined. */
-        if (MPI_IN_PLACE == sendbuf) {
-            memchecker_call(&opal_memchecker_base_isdefined,
-                            (char *)(recvbuf)+rank*ext,
-                            recvcount, recvtype);
-        } else {
-            memchecker_datatype(sendtype);
-            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-        }
-        /* check whether the receive buffer is addressable. */
-        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-
-        /* Unrooted operation -- same checks for all ranks on both
-           intracommunicators and intercommunicators */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-          OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
-        } else if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
-          err = MPI_ERR_TYPE;
-        } else if (recvcount < 0) {
-          err = MPI_ERR_COUNT;
-        } else if ((MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
-                   MPI_IN_PLACE == recvbuf) {
-          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        } else if (MPI_IN_PLACE != sendbuf) {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-        }
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_allgather_init(sendbuf, sendcount, sendtype,
-                                            recvbuf, recvcount, recvtype, comm,
-                                            info, request, comm->c_coll->coll_allgather_init_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_datatypes(*request, (MPI_IN_PLACE==sendbuf)?NULL:sendtype, recvtype);
-    }
-
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpiext/pcollreq/c/allgatherv_init.c b/ompi/mpiext/pcollreq/c/allgatherv_init.c
deleted file mode 100644
index 356a7ecdfc7..00000000000
--- a/ompi/mpiext/pcollreq/c/allgatherv_init.c
+++ /dev/null
@@ -1,135 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2010      University of Houston.  All rights reserved.
- * Copyright (c) 2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_Allgatherv_init = PMPIX_Allgatherv_init
-#endif
-#define MPIX_Allgatherv_init PMPIX_Allgatherv_init
-#endif
-
-static const char FUNC_NAME[] = "MPIX_Allgatherv_init";
-
-
-int MPIX_Allgatherv_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                         void *recvbuf, const int recvcounts[], const int displs[],
-                         MPI_Datatype recvtype, MPI_Comm comm,
-                         MPI_Info info, MPI_Request *request)
-{
-    int i, size, err;
-
-    SPC_RECORD(OMPI_SPC_ALLGATHERV_INIT, 1);
-
-    MEMCHECKER(
-        int rank;
-        ptrdiff_t ext;
-
-        rank = ompi_comm_rank(comm);
-        size = ompi_comm_size(comm);
-        ompi_datatype_type_extent(recvtype, &ext);
-
-        memchecker_datatype(recvtype);
-        memchecker_comm (comm);
-        /* check whether the receive buffer is addressable. */
-        for (i = 0; i < size; i++) {
-            memchecker_call(&opal_memchecker_base_isaddressable,
-                            (char *)(recvbuf)+displs[i]*ext,
-                            recvcounts[i], recvtype);
-        }
-
-        /* check whether the actual send buffer is defined. */
-        if (MPI_IN_PLACE == sendbuf) {
-            memchecker_call(&opal_memchecker_base_isdefined,
-                            (char *)(recvbuf)+displs[rank]*ext,
-                            recvcounts[rank], recvtype);
-        } else {
-            memchecker_datatype(sendtype);
-            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-
-        /* Unrooted operation -- same checks for all ranks on both
-           intracommunicators and intercommunicators */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if ((MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
-                   MPI_IN_PLACE == recvbuf) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        } else if (MPI_DATATYPE_NULL == recvtype) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
-        }
-
-        if (MPI_IN_PLACE != sendbuf) {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-        }
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-
-      /* We always define the remote group to be the same as the local
-         group in the case of an intracommunicator, so it's safe to
-         get the size of the remote group here for both intra- and
-         intercommunicators */
-
-        size = ompi_comm_remote_size(comm);
-        for (i = 0; i < size; ++i) {
-          if (recvcounts[i] < 0) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-          }
-        }
-
-        if (NULL == displs) {
-          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_BUFFER, FUNC_NAME);
-        }
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_allgatherv_init(sendbuf, sendcount, sendtype,
-                                             recvbuf, recvcounts, displs,
-                                             recvtype, comm, info, request,
-                                             comm->c_coll->coll_allgatherv_init_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_datatypes(*request, (MPI_IN_PLACE==sendbuf)?NULL:sendtype, recvtype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
-
diff --git a/ompi/mpiext/pcollreq/c/allreduce_init.c b/ompi/mpiext/pcollreq/c/allreduce_init.c
deleted file mode 100644
index 94971d43c2e..00000000000
--- a/ompi/mpiext/pcollreq/c/allreduce_init.c
+++ /dev/null
@@ -1,123 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2016      IBM Corporation.  All rights reserved.
- * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/op/op.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_Allreduce_init = PMPIX_Allreduce_init
-#endif
-#define MPIX_Allreduce_init PMPIX_Allreduce_init
-#endif
-
-static const char FUNC_NAME[] = "MPIX_Allreduce_init";
-
-
-int MPIX_Allreduce_init(const void *sendbuf, void *recvbuf, int count,
-                        MPI_Datatype datatype, MPI_Op op, MPI_Comm comm,
-                        MPI_Info info, MPI_Request *request)
-{
-    int err;
-
-    SPC_RECORD(OMPI_SPC_ALLREDUCE_INIT, 1);
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_comm(comm);
-
-        /* check whether receive buffer is defined. */
-        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, count, datatype);
-
-        /* check whether the actual send buffer is defined. */
-        if (MPI_IN_PLACE == sendbuf) {
-            memchecker_call(&opal_memchecker_base_isdefined, recvbuf, count, datatype);
-        } else {
-            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        char *msg;
-
-        /* Unrooted operation -- same checks for all ranks on both
-           intracommunicators and intercommunicators */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if (MPI_OP_NULL == op) {
-            err = MPI_ERR_OP;
-        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
-            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
-            free(msg);
-            return ret;
-        } else if ((MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
-                   MPI_IN_PLACE == recvbuf ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_BUFFER,
-                                          FUNC_NAME);
-        } else if( (sendbuf == recvbuf) &&
-                   (MPI_BOTTOM != sendbuf) &&
-                   (count > 1) ) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_BUFFER,
-                                          FUNC_NAME);
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-    }
-
-
-    /* MPI standard says that reductions have to have a count of at least 1,
-     * but some benchmarks (e.g., IMB) calls this function with a count of 0.
-     * So handle that case.
-     */
-    if (0 == count) {
-        err = ompi_request_persistent_noop_create(request);
-        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-
-    err = comm->c_coll->coll_allreduce_init(sendbuf, recvbuf, count, datatype,
-                                            op, comm, info, request, comm->c_coll->coll_allreduce_init_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_op(*request, op, datatype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpiext/pcollreq/c/alltoall_init.c b/ompi/mpiext/pcollreq/c/alltoall_init.c
deleted file mode 100644
index 777ba7c9efa..00000000000
--- a/ompi/mpiext/pcollreq/c/alltoall_init.c
+++ /dev/null
@@ -1,108 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012      Oak Ridge National Laboratory. All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2014-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_Alltoall_init = PMPIX_Alltoall_init
-#endif
-#define MPIX_Alltoall_init PMPIX_Alltoall_init
-#endif
-
-static const char FUNC_NAME[] = "MPIX_Alltoall_init";
-
-
-int MPIX_Alltoall_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                       void *recvbuf, int recvcount, MPI_Datatype recvtype,
-                       MPI_Comm comm, MPI_Info info, MPI_Request *request)
-{
-    size_t sendtype_size, recvtype_size;
-    int err;
-
-    SPC_RECORD(OMPI_SPC_ALLTOALL_INIT, 1);
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-        if (MPI_IN_PLACE != sendbuf) {
-            memchecker_datatype(sendtype);
-            memchecker_call(&opal_memchecker_base_isdefined, (void *)sendbuf, sendcount, sendtype);
-        }
-        memchecker_datatype(recvtype);
-        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-
-        /* Unrooted operation -- same checks for all ranks on both
-           intracommunicators and intercommunicators */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if ((MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
-                   MPI_IN_PLACE == recvbuf) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_ARG,
-                                          FUNC_NAME);
-        } else {
-            if (MPI_IN_PLACE != sendbuf) {
-                OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-                OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-            }
-            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtype, recvcount);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-
-        if (MPI_IN_PLACE != sendbuf && !OMPI_COMM_IS_INTER(comm)) {
-            ompi_datatype_type_size(sendtype, &sendtype_size);
-            ompi_datatype_type_size(recvtype, &recvtype_size);
-            if ((sendtype_size*sendcount) != (recvtype_size*recvcount)) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TRUNCATE, FUNC_NAME);
-            }
-        }
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_alltoall_init(sendbuf, sendcount, sendtype,
-                                           recvbuf, recvcount, recvtype, comm, info,
-                                           request, comm->c_coll->coll_alltoall_init_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_datatypes(*request, (MPI_IN_PLACE==sendbuf)?NULL:sendtype, recvtype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpiext/pcollreq/c/alltoallv_init.c b/ompi/mpiext/pcollreq/c/alltoallv_init.c
deleted file mode 100644
index 5cb593c0cf5..00000000000
--- a/ompi/mpiext/pcollreq/c/alltoallv_init.c
+++ /dev/null
@@ -1,138 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2014-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_Alltoallv_init = PMPIX_Alltoallv_init
-#endif
-#define MPIX_Alltoallv_init PMPIX_Alltoallv_init
-#endif
-
-static const char FUNC_NAME[] = "MPIX_Alltoallv_init";
-
-
-int MPIX_Alltoallv_init(const void *sendbuf, const int sendcounts[], const int sdispls[],
-                   MPI_Datatype sendtype, void *recvbuf, const int recvcounts[],
-                   const int rdispls[], MPI_Datatype recvtype, MPI_Comm comm,
-                   MPI_Info info, MPI_Request *request)
-{
-    int i, size, err;
-
-    SPC_RECORD(OMPI_SPC_ALLTOALLV_INIT, 1);
-
-    MEMCHECKER(
-        ptrdiff_t recv_ext;
-        ptrdiff_t send_ext;
-
-        memchecker_comm(comm);
-
-        if (MPI_IN_PLACE != sendbuf) {
-            memchecker_datatype(sendtype);
-            ompi_datatype_type_extent(sendtype, &send_ext);
-        }
-
-        memchecker_datatype(recvtype);
-        ompi_datatype_type_extent(recvtype, &recv_ext);
-
-        size = OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm);
-        for ( i = 0; i < size; i++ ) {
-            if (MPI_IN_PLACE != sendbuf) {
-                /* check if send chunks are defined. */
-                memchecker_call(&opal_memchecker_base_isdefined,
-                                (char *)(sendbuf)+sdispls[i]*send_ext,
-                                sendcounts[i], sendtype);
-            }
-            /* check if receive chunks are addressable. */
-            memchecker_call(&opal_memchecker_base_isaddressable,
-                            (char *)(recvbuf)+rdispls[i]*recv_ext,
-                            recvcounts[i], recvtype);
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-
-        /* Unrooted operation -- same checks for all ranks */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-
-        if (MPI_IN_PLACE == sendbuf) {
-            sendcounts = recvcounts;
-            sdispls = rdispls;
-            sendtype = recvtype;
-        }
-
-        if ((NULL == sendcounts) || (NULL == sdispls) ||
-            (NULL == recvcounts) || (NULL == rdispls) ||
-            (MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
-            MPI_IN_PLACE == recvbuf) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-
-        size = OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm);
-        for (i = 0; i < size; ++i) {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcounts[i]);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtype, recvcounts[i]);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-
-        if (MPI_IN_PLACE != sendbuf && !OMPI_COMM_IS_INTER(comm)) {
-            int me = ompi_comm_rank(comm);
-            size_t sendtype_size, recvtype_size;
-            ompi_datatype_type_size(sendtype, &sendtype_size);
-            ompi_datatype_type_size(recvtype, &recvtype_size);
-            if ((sendtype_size*sendcounts[me]) != (recvtype_size*recvcounts[me])) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TRUNCATE, FUNC_NAME);
-            }
-        }
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_alltoallv_init(sendbuf, sendcounts, sdispls,
-                                            sendtype, recvbuf, recvcounts, rdispls,
-                                            recvtype, comm, info, request, comm->c_coll->coll_alltoallv_init_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_datatypes(*request, (MPI_IN_PLACE==sendbuf)?NULL:sendtype, recvtype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
-
diff --git a/ompi/mpiext/pcollreq/c/alltoallw_init.c b/ompi/mpiext/pcollreq/c/alltoallw_init.c
deleted file mode 100644
index c85579ddd29..00000000000
--- a/ompi/mpiext/pcollreq/c/alltoallw_init.c
+++ /dev/null
@@ -1,135 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2014-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_Alltoallw_init = PMPIX_Alltoallw_init
-#endif
-#define MPIX_Alltoallw_init PMPIX_Alltoallw_init
-#endif
-
-static const char FUNC_NAME[] = "MPIX_Alltoallw_init";
-
-
-int MPIX_Alltoallw_init(const void *sendbuf, const int sendcounts[], const int sdispls[],
-                        const MPI_Datatype sendtypes[], void *recvbuf, const int recvcounts[],
-                        const int rdispls[], const MPI_Datatype recvtypes[], MPI_Comm comm,
-                        MPI_Info info, MPI_Request *request)
-{
-    int i, size, err;
-
-    SPC_RECORD(OMPI_SPC_ALLTOALLW_INIT, 1);
-
-    MEMCHECKER(
-        ptrdiff_t recv_ext;
-        ptrdiff_t send_ext;
-
-
-        memchecker_comm(comm);
-
-        size = OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm);
-        for ( i = 0; i < size; i++ ) {
-            if (MPI_IN_PLACE != sendbuf) {
-                memchecker_datatype(sendtypes[i]);
-                ompi_datatype_type_extent(sendtypes[i], &send_ext);
-                memchecker_call(&opal_memchecker_base_isdefined,
-                                (char *)(sendbuf)+sdispls[i]*send_ext,
-                                sendcounts[i], sendtypes[i]);
-            }
-
-            memchecker_datatype(recvtypes[i]);
-            ompi_datatype_type_extent(recvtypes[i], &recv_ext);
-            memchecker_call(&opal_memchecker_base_isaddressable,
-                            (char *)(recvbuf)+rdispls[i]*recv_ext,
-                            recvcounts[i], recvtypes[i]);
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-
-        /* Unrooted operation -- same checks for all ranks */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-
-        if (MPI_IN_PLACE == sendbuf) {
-            sendcounts = recvcounts;
-            sdispls    = rdispls;
-            sendtypes  = recvtypes;
-        }
-
-        if ((NULL == sendcounts) || (NULL == sdispls) || (NULL == sendtypes) ||
-            (NULL == recvcounts) || (NULL == rdispls) || (NULL == recvtypes) ||
-            (MPI_IN_PLACE == sendbuf && OMPI_COMM_IS_INTER(comm)) ||
-            MPI_IN_PLACE == recvbuf) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-
-        size = OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm);
-        for (i = 0; i < size; ++i) {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtypes[i], sendcounts[i]);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtypes[i], recvcounts[i]);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-
-        if (MPI_IN_PLACE != sendbuf && !OMPI_COMM_IS_INTER(comm)) {
-            int me = ompi_comm_rank(comm);
-            size_t sendtype_size, recvtype_size;
-            ompi_datatype_type_size(sendtypes[me], &sendtype_size);
-            ompi_datatype_type_size(recvtypes[me], &recvtype_size);
-            if ((sendtype_size*sendcounts[me]) != (recvtype_size*recvcounts[me])) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TRUNCATE, FUNC_NAME);
-            }
-        }
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_alltoallw_init(sendbuf, sendcounts, sdispls,
-                                            sendtypes, recvbuf, recvcounts,
-                                            rdispls, recvtypes, comm, info, request,
-                                            comm->c_coll->coll_alltoallw_init_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_datatypes_w(*request, (MPI_IN_PLACE==sendbuf)?NULL:sendtypes, recvtypes);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
-
diff --git a/ompi/mpiext/pcollreq/c/barrier_init.c b/ompi/mpiext/pcollreq/c/barrier_init.c
deleted file mode 100644
index 76084da886a..00000000000
--- a/ompi/mpiext/pcollreq/c/barrier_init.c
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2012      Oak Rigde National Laboratory. All rights reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/memchecker.h"
-#include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_Barrier_init = PMPIX_Barrier_init
-#endif
-#define MPIX_Barrier_init PMPIX_Barrier_init
-#endif
-
-static const char FUNC_NAME[] = "MPIX_Barrier_init";
-
-
-int MPIX_Barrier_init(MPI_Comm comm, MPI_Info info, MPI_Request *request)
-{
-    int err = MPI_SUCCESS;
-
-    SPC_RECORD(OMPI_SPC_BARRIER_INIT, 1);
-
-    MEMCHECKER(
-            memchecker_comm(comm);
-            );
-
-    /* Error checking */
-
-    if (MPI_PARAM_CHECK) {
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
-        }
-    }
-
-    err = comm->c_coll->coll_barrier_init(comm, info, request, comm->c_coll->coll_barrier_init_module);
-
-    /* All done */
-
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpiext/pcollreq/c/bcast_init.c b/ompi/mpiext/pcollreq/c/bcast_init.c
deleted file mode 100644
index 9a61aff8a95..00000000000
--- a/ompi/mpiext/pcollreq/c/bcast_init.c
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Copyright (c) 2012      Oak Rigde National Laboratory. All rights reserved.
- * Copyright (c) 2015-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2017-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_Bcast_init = PMPIX_Bcast_init
-#endif
-#define MPIX_Bcast_init PMPIX_Bcast_init
-#endif
-
-static const char FUNC_NAME[] = "MPIX_Bcast_init";
-
-
-int MPIX_Bcast_init(void *buffer, int count, MPI_Datatype datatype,
-                    int root, MPI_Comm comm, MPI_Info info, MPI_Request *request)
-{
-    int err;
-
-    SPC_RECORD(OMPI_SPC_BCAST_INIT, 1);
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_call(&opal_memchecker_base_isdefined, buffer, count, datatype);
-        memchecker_comm(comm);
-    );
-
-    if (MPI_PARAM_CHECK) {
-      err = MPI_SUCCESS;
-      OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-      if (ompi_comm_invalid(comm)) {
-          return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                     FUNC_NAME);
-      }
-
-      /* Errors for all ranks */
-
-      OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
-      OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-      if (MPI_IN_PLACE == buffer) {
-          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-      }
-
-      /* Errors for intracommunicators */
-
-      if (OMPI_COMM_IS_INTRA(comm)) {
-        if ((root >= ompi_comm_size(comm)) || (root < 0)) {
-          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
-        }
-      }
-
-      /* Errors for intercommunicators */
-
-      else {
-        if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
-               MPI_ROOT == root || MPI_PROC_NULL == root)) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
-        }
-      }
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-
-    err = comm->c_coll->coll_bcast_init(buffer, count, datatype, root, comm,
-                                        info, request,
-                                        comm->c_coll->coll_bcast_init_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        if (!OMPI_COMM_IS_INTRA(comm)) {
-            if (MPI_PROC_NULL == root) {
-                datatype = NULL;
-            }
-        }
-        ompi_coll_base_retain_datatypes(*request, datatype, NULL);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpiext/pcollreq/c/exscan_init.c b/ompi/mpiext/pcollreq/c/exscan_init.c
deleted file mode 100644
index 0bac072785c..00000000000
--- a/ompi/mpiext/pcollreq/c/exscan_init.c
+++ /dev/null
@@ -1,93 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/op/op.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_Exscan_init = PMPIX_Exscan_init
-#endif
-#define MPIX_Exscan_init PMPIX_Exscan_init
-#endif
-
-static const char FUNC_NAME[] = "MPIX_Exscan_init";
-
-
-int MPIX_Exscan_init(const void *sendbuf, void *recvbuf, int count,
-                     MPI_Datatype datatype, MPI_Op op, MPI_Comm comm,
-                     MPI_Info info, MPI_Request *request)
-{
-    int err;
-
-    SPC_RECORD(OMPI_SPC_EXSCAN_INIT, 1);
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
-        memchecker_comm(comm);
-    );
-
-    if (MPI_PARAM_CHECK) {
-        char *msg;
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-
-        /* Unrooted operation -- same checks for intracommunicators
-           and intercommunicators */
-        else if (MPI_OP_NULL == op) {
-            err = MPI_ERR_OP;
-        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
-            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
-            free(msg);
-            return ret;
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-
-    err = comm->c_coll->coll_exscan_init(sendbuf, recvbuf, count,
-                                         datatype, op, comm, info, request,
-                                         comm->c_coll->coll_exscan_init_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_op(*request, op, datatype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpiext/pcollreq/c/gather_init.c b/ompi/mpiext/pcollreq/c/gather_init.c
deleted file mode 100644
index 5b9b75b82b9..00000000000
--- a/ompi/mpiext/pcollreq/c/gather_init.c
+++ /dev/null
@@ -1,196 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2008      University of Houston.  All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_Gather_init = PMPIX_Gather_init
-#endif
-#define MPIX_Gather_init PMPIX_Gather_init
-#endif
-
-static const char FUNC_NAME[] = "MPIX_Gather_init";
-
-
-int MPIX_Gather_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                     void *recvbuf, int recvcount, MPI_Datatype recvtype,
-                     int root, MPI_Comm comm, MPI_Info info, MPI_Request *request)
-{
-    int err;
-
-    SPC_RECORD(OMPI_SPC_GATHER_INIT, 1);
-
-    MEMCHECKER(
-        int rank;
-        ptrdiff_t ext;
-
-        rank = ompi_comm_rank(comm);
-        ompi_datatype_type_extent(recvtype, &ext);
-
-        memchecker_comm(comm);
-        if(OMPI_COMM_IS_INTRA(comm)) {
-            if(ompi_comm_rank(comm) == root) {
-                /* check whether root's send buffer is defined. */
-                if (MPI_IN_PLACE == sendbuf) {
-                  memchecker_call(&opal_memchecker_base_isdefined,
-                                  (char *)(recvbuf)+rank*ext,
-                                  recvcount, recvtype);
-                } else {
-                    memchecker_datatype(sendtype);
-                    memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-                }
-
-                memchecker_datatype(recvtype);
-                /* check whether root's receive buffer is addressable. */
-                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-            } else {
-                memchecker_datatype(sendtype);
-                /* check whether send buffer is defined on other processes. */
-                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-            }
-        } else {
-            if (MPI_ROOT == root) {
-                memchecker_datatype(recvtype);
-                /* check whether root's receive buffer is addressable. */
-                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-            } else if (MPI_PROC_NULL != root) {
-                memchecker_datatype(sendtype);
-                /* check whether send buffer is defined. */
-                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-            }
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if ((ompi_comm_rank(comm) != root && MPI_IN_PLACE == sendbuf) ||
-                   (ompi_comm_rank(comm) == root && MPI_IN_PLACE == recvbuf)) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-
-        /* Errors for intracommunicators */
-
-        if (OMPI_COMM_IS_INTRA(comm)) {
-
-            /* Errors for all ranks */
-
-            if ((root >= ompi_comm_size(comm)) || (root < 0)) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
-            }
-
-            if (MPI_IN_PLACE != sendbuf) {
-                OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-            }
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-
-            /* Errors for the root.  Some of these could have been
-               combined into compound if statements above, but since
-               this whole section can be compiled out (or turned off at
-               run time) for efficiency, it's more clear to separate
-               them out into individual tests. */
-
-            if (ompi_comm_rank(comm) == root) {
-                if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
-                }
-
-                if (recvcount < 0) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-                }
-            }
-        }
-
-        /* Errors for intercommunicators */
-
-        else {
-            if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
-                   MPI_ROOT == root || MPI_PROC_NULL == root)) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
-            }
-
-            /* Errors for the senders */
-
-            if (MPI_ROOT != root && MPI_PROC_NULL != root) {
-                OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-                OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-            }
-
-            /* Errors for the root.  Ditto on the comment above -- these
-               error checks could have been combined above, but let's
-               make the code easier to read. */
-
-            else if (MPI_ROOT == root) {
-                if (recvcount < 0) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-                }
-
-                if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
-                }
-            }
-        }
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_gather_init(sendbuf, sendcount, sendtype, recvbuf,
-                                         recvcount, recvtype, root, comm, info, request,
-                                         comm->c_coll->coll_gather_init_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        if (OMPI_COMM_IS_INTRA(comm)) {
-            if (MPI_IN_PLACE == sendbuf) {
-                sendtype = NULL;
-            } else if (ompi_comm_rank(comm) != root) {
-                recvtype = NULL;
-            }
-        } else {
-            if (MPI_ROOT == root) {
-                sendtype = NULL;
-            } else if (MPI_PROC_NULL == root) {
-                sendtype = NULL;
-                recvtype = NULL;
-            } else {
-                recvtype = NULL;
-            }
-        }
-        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpiext/pcollreq/c/gatherv_init.c b/ompi/mpiext/pcollreq/c/gatherv_init.c
deleted file mode 100644
index 30291ee8efe..00000000000
--- a/ompi/mpiext/pcollreq/c/gatherv_init.c
+++ /dev/null
@@ -1,221 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h"
-#include "ompi/memchecker.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_Gatherv_init = PMPIX_Gatherv_init
-#endif
-#define MPIX_Gatherv_init PMPIX_Gatherv_init
-#endif
-
-static const char FUNC_NAME[] = "MPIX_Gatherv_init";
-
-
-int MPIX_Gatherv_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                      void *recvbuf, const int recvcounts[], const int displs[],
-                      MPI_Datatype recvtype, int root, MPI_Comm comm,
-                      MPI_Info info, MPI_Request *request)
-{
-    int i, size, err;
-
-    SPC_RECORD(OMPI_SPC_GATHERV_INIT, 1);
-
-    MEMCHECKER(
-        ptrdiff_t ext;
-
-        size = ompi_comm_remote_size(comm);
-        ompi_datatype_type_extent(recvtype, &ext);
-
-        memchecker_comm(comm);
-        if(OMPI_COMM_IS_INTRA(comm)) {
-            if(ompi_comm_rank(comm) == root) {
-                /* check whether root's send buffer is defined. */
-                if (MPI_IN_PLACE == sendbuf) {
-                    for (i = 0; i < size; i++) {
-                        memchecker_call(&opal_memchecker_base_isdefined,
-                                        (char *)(recvbuf)+displs[i]*ext,
-                                        recvcounts[i], recvtype);
-                    }
-                } else {
-                    memchecker_datatype(sendtype);
-                    memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-                }
-
-                memchecker_datatype(recvtype);
-                /* check whether root's receive buffer is addressable. */
-                for (i = 0; i < size; i++) {
-                    memchecker_call(&opal_memchecker_base_isaddressable,
-                                    (char *)(recvbuf)+displs[i]*ext,
-                                    recvcounts[i], recvtype);
-                }
-            } else {
-                memchecker_datatype(sendtype);
-                /* check whether send buffer is defined on other processes. */
-                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-            }
-        } else {
-            if (MPI_ROOT == root) {
-                memchecker_datatype(recvtype);
-                /* check whether root's receive buffer is addressable. */
-                for (i = 0; i < size; i++) {
-                    memchecker_call(&opal_memchecker_base_isaddressable,
-                                    (char *)(recvbuf)+displs[i]*ext,
-                                    recvcounts[i], recvtype);
-                }
-            } else if (MPI_PROC_NULL != root) {
-                memchecker_datatype(sendtype);
-                /* check whether send buffer is defined. */
-                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-            }
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if ((ompi_comm_rank(comm) != root && MPI_IN_PLACE == sendbuf) ||
-                   (ompi_comm_rank(comm) == root && MPI_IN_PLACE == recvbuf)) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-
-        /* Errors for intracommunicators */
-
-        if (OMPI_COMM_IS_INTRA(comm)) {
-
-            /* Errors for all ranks */
-
-            if ((root >= ompi_comm_size(comm)) || (root < 0)) {
-                err = MPI_ERR_ROOT;
-            } else if (MPI_IN_PLACE != sendbuf) {
-                OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-            }
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-
-            /* Errors for the root.  Some of these could have been
-               combined into compound if statements above, but since
-               this whole section can be compiled out (or turned off at
-               run time) for efficiency, it's more clear to separate
-               them out into individual tests. */
-
-            if (ompi_comm_rank(comm) == root) {
-                if (NULL == displs) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-                }
-
-                if (NULL == recvcounts) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-                }
-
-                size = ompi_comm_size(comm);
-                for (i = 0; i < size; ++i) {
-                    if (recvcounts[i] < 0) {
-                        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-                    } else if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
-                        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
-                    }
-                }
-            }
-        }
-
-        /* Errors for intercommunicators */
-
-        else {
-            if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
-                   MPI_ROOT == root || MPI_PROC_NULL == root)) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
-            }
-
-            /* Errors for the senders */
-
-            if (MPI_ROOT != root && MPI_PROC_NULL != root) {
-                OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-                OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-            }
-
-            /* Errors for the root.  Ditto on the comment above -- these
-               error checks could have been combined above, but let's
-               make the code easier to read. */
-
-            else if (MPI_ROOT == root) {
-                if (NULL == displs) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-                }
-
-                if (NULL == recvcounts) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-                }
-
-                size = ompi_comm_remote_size(comm);
-                for (i = 0; i < size; ++i) {
-                    if (recvcounts[i] < 0) {
-                        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-                    } else if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
-                        return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
-                    }
-                }
-            }
-        }
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_gatherv_init(sendbuf, sendcount, sendtype, recvbuf,
-                                          recvcounts, displs, recvtype,
-                                          root, comm, info, request,
-                                          comm->c_coll->coll_gatherv_init_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        if (OMPI_COMM_IS_INTRA(comm)) {
-            if (MPI_IN_PLACE == sendbuf) {
-                sendtype = NULL;
-            } else if (ompi_comm_rank(comm) != root) {
-                recvtype = NULL;
-            }
-        } else {
-            if (MPI_ROOT == root) {
-                sendtype = NULL;
-            } else if (MPI_PROC_NULL == root) {
-                sendtype = NULL;
-                recvtype = NULL;
-            } else {
-                recvtype = NULL;
-            }
-        }
-        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.c b/ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.c
deleted file mode 100644
index aa4d9acfab0..00000000000
--- a/ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.c
+++ /dev/null
@@ -1,15 +0,0 @@
-/*
- * Copyright (c) 2018      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- *
- */
-
-void mpiext_pcollreq_dummy(void);
-
-void mpiext_pcollreq_dummy() {
-}
diff --git a/ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h b/ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h
deleted file mode 100644
index 11bdb9937d9..00000000000
--- a/ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Copyright (c) 2017      FUJITSU LIMITED.  All rights reserved.
- * Copyright (c) 2018      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- *
- */
-
-OMPI_DECLSPEC int MPIX_Allgather_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int MPIX_Allgatherv_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int displs[], MPI_Datatype recvtype, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int MPIX_Allreduce_init(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int MPIX_Alltoall_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int MPIX_Alltoallv_init(const void *sendbuf, const int sendcounts[], const int sdispls[], MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int rdispls[], MPI_Datatype recvtype, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int MPIX_Alltoallw_init(const void *sendbuf, const int sendcounts[], const int sdispls[], const MPI_Datatype sendtypes[], void *recvbuf, const int recvcounts[], const int rdispls[], const MPI_Datatype recvtypes[], MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int MPIX_Barrier_init(MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int MPIX_Bcast_init(void *buffer, int count, MPI_Datatype datatype, int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int MPIX_Exscan_init(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int MPIX_Gather_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int MPIX_Gatherv_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int displs[], MPI_Datatype recvtype, int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int MPIX_Reduce_init(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int MPIX_Reduce_scatter_init(const void *sendbuf, void *recvbuf, const int recvcounts[], MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int MPIX_Reduce_scatter_block_init(const void *sendbuf, void *recvbuf, int recvcount, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int MPIX_Scan_init(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int MPIX_Scatter_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int MPIX_Scatterv_init(const void *sendbuf, const int sendcounts[], const int displs[], MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-
-OMPI_DECLSPEC int MPIX_Neighbor_allgather_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int MPIX_Neighbor_allgatherv_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int displs[], MPI_Datatype recvtype, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int MPIX_Neighbor_alltoall_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int MPIX_Neighbor_alltoallv_init(const void *sendbuf, const int sendcounts[], const int sdispls[], MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int rdispls[], MPI_Datatype recvtype, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int MPIX_Neighbor_alltoallw_init(const void *sendbuf, const int sendcounts[], const MPI_Aint sdispls[], const MPI_Datatype sendtypes[], void *recvbuf, const int recvcounts[], const MPI_Aint rdispls[], const MPI_Datatype recvtypes[], MPI_Comm comm, MPI_Info info, MPI_Request *request);
-
-  /*
-   * Profiling MPI API
-   */
-OMPI_DECLSPEC int PMPIX_Allgather_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int PMPIX_Allgatherv_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int displs[], MPI_Datatype recvtype, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int PMPIX_Allreduce_init(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int PMPIX_Alltoall_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int PMPIX_Alltoallv_init(const void *sendbuf, const int sendcounts[], const int sdispls[], MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int rdispls[], MPI_Datatype recvtype, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int PMPIX_Alltoallw_init(const void *sendbuf, const int sendcounts[], const int sdispls[], const MPI_Datatype sendtypes[], void *recvbuf, const int recvcounts[], const int rdispls[], const MPI_Datatype recvtypes[], MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int PMPIX_Barrier_init(MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int PMPIX_Bcast_init(void *buffer, int count, MPI_Datatype datatype, int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int PMPIX_Exscan_init(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int PMPIX_Gather_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int PMPIX_Gatherv_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int displs[], MPI_Datatype recvtype, int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int PMPIX_Reduce_init(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int PMPIX_Reduce_scatter_init(const void *sendbuf, void *recvbuf, const int recvcounts[], MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int PMPIX_Reduce_scatter_block_init(const void *sendbuf, void *recvbuf, int recvcount, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int PMPIX_Scan_init(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int PMPIX_Scatter_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int PMPIX_Scatterv_init(const void *sendbuf, const int sendcounts[], const int displs[], MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-
-OMPI_DECLSPEC int PMPIX_Neighbor_allgather_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int PMPIX_Neighbor_allgatherv_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int displs[], MPI_Datatype recvtype, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int PMPIX_Neighbor_alltoall_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int PMPIX_Neighbor_alltoallv_init(const void *sendbuf, const int sendcounts[], const int sdispls[], MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int rdispls[], MPI_Datatype recvtype, MPI_Comm comm, MPI_Info info, MPI_Request *request);
-OMPI_DECLSPEC int PMPIX_Neighbor_alltoallw_init(const void *sendbuf, const int sendcounts[], const MPI_Aint sdispls[], const MPI_Datatype sendtypes[], void *recvbuf, const int recvcounts[], const MPI_Aint rdispls[], const MPI_Datatype recvtypes[], MPI_Comm comm, MPI_Info info, MPI_Request *request);
diff --git a/ompi/mpiext/pcollreq/c/neighbor_allgather_init.c b/ompi/mpiext/pcollreq/c/neighbor_allgather_init.c
deleted file mode 100644
index b0d5a646d5c..00000000000
--- a/ompi/mpiext/pcollreq/c/neighbor_allgather_init.c
+++ /dev/null
@@ -1,130 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012      Oak Rigde National Laboratory. All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2017      IBM Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/mca/topo/topo.h"
-#include "ompi/mca/topo/base/base.h"
-#include "ompi/runtime/ompi_spc.h"
-#include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_Neighbor_allgather_init = PMPIX_Neighbor_allgather_init
-#endif
-#define MPIX_Neighbor_allgather_init PMPIX_Neighbor_allgather_init
-#endif
-
-static const char FUNC_NAME[] = "MPIX_Neighbor_allgather_init";
-
-
-int MPIX_Neighbor_allgather_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                            void *recvbuf, int recvcount, MPI_Datatype recvtype,
-                            MPI_Comm comm, MPI_Info info, MPI_Request *request)
-{
-    int err;
-
-    SPC_RECORD(OMPI_SPC_NEIGHBOR_ALLGATHER_INIT, 1);
-
-    MEMCHECKER(
-        ptrdiff_t ext;
-
-        ompi_datatype_type_extent(recvtype, &ext);
-
-        memchecker_datatype(recvtype);
-        memchecker_comm(comm);
-        /* check whether the actual send buffer is defined. */
-        if (MPI_IN_PLACE != sendbuf) {
-            memchecker_datatype(sendtype);
-            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-        }
-        /* check whether the receive buffer is addressable. */
-        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-
-        /* Unrooted operation -- same checks for all ranks on both
-           intracommunicators and intercommunicators */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
-          OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
-        } else if (! OMPI_COMM_IS_TOPO(comm)) {
-          OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TOPOLOGY, FUNC_NAME);
-        } else if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
-          err = MPI_ERR_TYPE;
-        } else if (recvcount < 0) {
-          err = MPI_ERR_COUNT;
-        } else if (MPI_IN_PLACE == sendbuf || MPI_IN_PLACE == recvbuf) {
-          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-        }
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-
-        if( OMPI_COMM_IS_CART(comm) ) {
-            const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
-            if( 0 > cart->ndims ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-        else if( OMPI_COMM_IS_GRAPH(comm) ) {
-            int degree;
-            mca_topo_base_graph_neighbors_count(comm, ompi_comm_rank(comm), &degree);
-            if( 0 > degree ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-        else if( OMPI_COMM_IS_DIST_GRAPH(comm) ) {
-            const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
-            int indegree  = dist_graph->indegree;
-            int outdegree = dist_graph->outdegree;
-            if( indegree <  0 || outdegree <  0 ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_neighbor_allgather_init(sendbuf, sendcount, sendtype, recvbuf,
-                                                     recvcount, recvtype, comm, info, request,
-                                                     comm->c_coll->coll_neighbor_allgather_init_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
-    }
-
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpiext/pcollreq/c/neighbor_allgatherv_init.c b/ompi/mpiext/pcollreq/c/neighbor_allgatherv_init.c
deleted file mode 100644
index 23da85e67a3..00000000000
--- a/ompi/mpiext/pcollreq/c/neighbor_allgatherv_init.c
+++ /dev/null
@@ -1,154 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2010      University of Houston.  All rights reserved.
- * Copyright (c) 2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2017      IBM Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/mca/topo/topo.h"
-#include "ompi/mca/topo/base/base.h"
-#include "ompi/runtime/ompi_spc.h"
-#include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_Neighbor_allgatherv_init = PMPIX_Neighbor_allgatherv_init
-#endif
-#define MPIX_Neighbor_allgatherv_init PMPIX_Neighbor_allgatherv_init
-#endif
-
-static const char FUNC_NAME[] = "MPIX_Neighbor_allgatherv_init";
-
-
-int MPIX_Neighbor_allgatherv_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                                  void *recvbuf, const int recvcounts[], const int displs[],
-                                  MPI_Datatype recvtype, MPI_Comm comm,
-                                  MPI_Info info, MPI_Request *request)
-{
-    int i, size, err;
-
-    SPC_RECORD(OMPI_SPC_NEIGHBOR_ALLGATHERV_INIT, 1);
-
-    MEMCHECKER(
-        ptrdiff_t ext;
-
-        size = ompi_comm_size(comm);
-        ompi_datatype_type_extent(recvtype, &ext);
-
-        memchecker_datatype(recvtype);
-        memchecker_comm (comm);
-        /* check whether the receive buffer is addressable. */
-        for (i = 0; i < size; i++) {
-            memchecker_call(&opal_memchecker_base_isaddressable,
-                            (char *)(recvbuf)+displs[i]*ext,
-                            recvcounts[i], recvtype);
-        }
-
-        /* check whether the actual send buffer is defined. */
-        if (MPI_IN_PLACE != sendbuf) {
-            memchecker_datatype(sendtype);
-            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-
-        /* Unrooted operation -- same checks for all ranks on both
-           intracommunicators and intercommunicators */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if (! OMPI_COMM_IS_TOPO(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TOPOLOGY,
-                                          FUNC_NAME);
-        } else if (MPI_IN_PLACE == sendbuf || MPI_IN_PLACE == recvbuf) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        } else if (MPI_DATATYPE_NULL == recvtype) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
-        }
-
-        OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-
-      /* We always define the remote group to be the same as the local
-         group in the case of an intracommunicator, so it's safe to
-         get the size of the remote group here for both intra- and
-         intercommunicators */
-
-        size = ompi_comm_remote_size(comm);
-        for (i = 0; i < size; ++i) {
-          if (recvcounts[i] < 0) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-          }
-        }
-
-        if (NULL == displs) {
-          return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_BUFFER, FUNC_NAME);
-        }
-
-        if( OMPI_COMM_IS_CART(comm) ) {
-            const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
-            if( 0 > cart->ndims ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-        else if( OMPI_COMM_IS_GRAPH(comm) ) {
-            int degree;
-            mca_topo_base_graph_neighbors_count(comm, ompi_comm_rank(comm), &degree);
-            if( 0 > degree ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-        else if( OMPI_COMM_IS_DIST_GRAPH(comm) ) {
-            const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
-            int indegree  = dist_graph->indegree;
-            int outdegree = dist_graph->outdegree;
-            if( indegree <  0 || outdegree <  0 ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_neighbor_allgatherv_init(sendbuf, sendcount, sendtype,
-                                                      recvbuf, (int *) recvcounts, (int *) displs,
-                                                      recvtype, comm, info, request,
-                                                      comm->c_coll->coll_neighbor_allgatherv_init_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
-
diff --git a/ompi/mpiext/pcollreq/c/neighbor_alltoall_init.c b/ompi/mpiext/pcollreq/c/neighbor_alltoall_init.c
deleted file mode 100644
index 68b72f4b4fe..00000000000
--- a/ompi/mpiext/pcollreq/c/neighbor_alltoall_init.c
+++ /dev/null
@@ -1,132 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012      Oak Ridge National Laboratory. All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2014-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2017      IBM Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/mca/topo/topo.h"
-#include "ompi/mca/topo/base/base.h"
-#include "ompi/runtime/ompi_spc.h"
-#include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_Neighbor_alltoall_init = PMPIX_Neighbor_alltoall_init
-#endif
-#define MPIX_Neighbor_alltoall_init PMPIX_Neighbor_alltoall_init
-#endif
-
-static const char FUNC_NAME[] = "MPIX_Neighbor_alltoall_init";
-
-
-int MPIX_Neighbor_alltoall_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                                void *recvbuf, int recvcount, MPI_Datatype recvtype,
-                                MPI_Comm comm, MPI_Info info, MPI_Request *request)
-{
-    size_t sendtype_size, recvtype_size;
-    int err;
-
-    SPC_RECORD(OMPI_SPC_NEIGHBOR_ALLTOALL_INIT, 1);
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-        if (MPI_IN_PLACE != sendbuf) {
-            memchecker_datatype(sendtype);
-            memchecker_call(&opal_memchecker_base_isdefined, (void *)sendbuf, sendcount, sendtype);
-        }
-        memchecker_datatype(recvtype);
-        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-    );
-
-    if (MPI_PARAM_CHECK) {
-
-        /* Unrooted operation -- same checks for all ranks on both
-           intracommunicators and intercommunicators */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if (! OMPI_COMM_IS_TOPO(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TOPOLOGY,
-                                          FUNC_NAME);
-        } else if (MPI_IN_PLACE == sendbuf || MPI_IN_PLACE == recvbuf) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
-                                          FUNC_NAME);
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtype, recvcount);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-
-        ompi_datatype_type_size(sendtype, &sendtype_size);
-        ompi_datatype_type_size(recvtype, &recvtype_size);
-        if ((sendtype_size*sendcount) != (recvtype_size*recvcount)) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TRUNCATE, FUNC_NAME);
-        }
-
-        if( OMPI_COMM_IS_CART(comm) ) {
-            const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
-            if( 0 > cart->ndims ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-        else if( OMPI_COMM_IS_GRAPH(comm) ) {
-            int degree;
-            mca_topo_base_graph_neighbors_count(comm, ompi_comm_rank(comm), &degree);
-            if( 0 > degree ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-        else if( OMPI_COMM_IS_DIST_GRAPH(comm) ) {
-            const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
-            int indegree  = dist_graph->indegree;
-            int outdegree = dist_graph->outdegree;
-            if( indegree <  0 || outdegree <  0 ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_neighbor_alltoall_init(sendbuf, sendcount, sendtype,
-                                                    recvbuf, recvcount, recvtype, comm,
-                                                    info, request,
-                                                    comm->c_coll->coll_neighbor_alltoall_init_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpiext/pcollreq/c/neighbor_alltoallv_init.c b/ompi/mpiext/pcollreq/c/neighbor_alltoallv_init.c
deleted file mode 100644
index 5287dac46b6..00000000000
--- a/ompi/mpiext/pcollreq/c/neighbor_alltoallv_init.c
+++ /dev/null
@@ -1,156 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2017 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2014-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2017      IBM Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/mca/topo/topo.h"
-#include "ompi/mca/topo/base/base.h"
-#include "ompi/runtime/ompi_spc.h"
-#include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_Neighbor_alltoallv_init = PMPIX_Neighbor_alltoallv_init
-#endif
-#define MPIX_Neighbor_alltoallv_init PMPIX_Neighbor_alltoallv_init
-#endif
-
-static const char FUNC_NAME[] = "MPIX_Neighbor_alltoallv_init";
-
-
-int MPIX_Neighbor_alltoallv_init(const void *sendbuf, const int sendcounts[], const int sdispls[],
-                                 MPI_Datatype sendtype, void *recvbuf, const int recvcounts[],
-                                 const int rdispls[], MPI_Datatype recvtype, MPI_Comm comm,
-                                 MPI_Info info, MPI_Request *request)
-{
-    int i, err;
-    int indegree, outdegree;
-
-    SPC_RECORD(OMPI_SPC_NEIGHBOR_ALLTOALLV_INIT, 1);
-
-    MEMCHECKER(
-        ptrdiff_t recv_ext;
-        ptrdiff_t send_ext;
-
-        memchecker_comm(comm);
-
-        if (MPI_IN_PLACE != sendbuf) {
-            memchecker_datatype(sendtype);
-            ompi_datatype_type_extent(recvtype, &recv_ext);
-        }
-
-        memchecker_datatype(recvtype);
-        ompi_datatype_type_extent(sendtype, &send_ext);
-
-        err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree);
-        if (MPI_SUCCESS == err) {
-            if (MPI_IN_PLACE != sendbuf) {
-                for ( i = 0; i < outdegree; i++ ) {
-                    /* check if send chunks are defined. */
-                    memchecker_call(&opal_memchecker_base_isdefined,
-                                    (char *)(sendbuf)+sdispls[i]*send_ext,
-                                    sendcounts[i], sendtype);
-                }
-            }
-            for ( i = 0; i < indegree; i++ ) {
-                /* check if receive chunks are addressable. */
-                memchecker_call(&opal_memchecker_base_isaddressable,
-                                (char *)(recvbuf)+rdispls[i]*recv_ext,
-                                recvcounts[i], recvtype);
-            }
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-
-        /* Unrooted operation -- same checks for all ranks */
-
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm) || OMPI_COMM_IS_INTER(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if (! OMPI_COMM_IS_TOPO(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_TOPOLOGY,
-                                          FUNC_NAME);
-        } else if ((NULL == sendcounts) || (NULL == sdispls) ||
-            (NULL == recvcounts) || (NULL == rdispls) ||
-            MPI_IN_PLACE == sendbuf || MPI_IN_PLACE == recvbuf) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-
-        err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree);
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        for (i = 0; i < outdegree; ++i) {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcounts[i]);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-        for (i = 0; i < indegree; ++i) {
-            OMPI_CHECK_DATATYPE_FOR_RECV(err, recvtype, recvcounts[i]);
-            OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-
-        if( OMPI_COMM_IS_CART(comm) ) {
-            const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart;
-            if( 0 > cart->ndims ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-        else if( OMPI_COMM_IS_GRAPH(comm) ) {
-            int degree;
-            mca_topo_base_graph_neighbors_count(comm, ompi_comm_rank(comm), &degree);
-            if( 0 > degree ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-        else if( OMPI_COMM_IS_DIST_GRAPH(comm) ) {
-            const mca_topo_base_comm_dist_graph_2_2_0_t *dist_graph = comm->c_topo->mtc.dist_graph;
-            indegree  = dist_graph->indegree;
-            outdegree = dist_graph->outdegree;
-            if( indegree <  0 || outdegree <  0 ) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-            }
-        }
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_neighbor_alltoallv_init(sendbuf, sendcounts, sdispls,
-                                                     sendtype, recvbuf, recvcounts, rdispls,
-                                                     recvtype, comm, info, request,
-                                                     comm->c_coll->coll_neighbor_alltoallv_init_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
-
diff --git a/ompi/mpiext/pcollreq/c/profile/Makefile.am b/ompi/mpiext/pcollreq/c/profile/Makefile.am
deleted file mode 100644
index 95595743229..00000000000
--- a/ompi/mpiext/pcollreq/c/profile/Makefile.am
+++ /dev/null
@@ -1,66 +0,0 @@
-#
-# Copyright (c) 2017      FUJITSU LIMITED.  All rights reserved.
-# Copyright (c) 2018      Research Organization for Information Science
-#                         and Technology (RIST).  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-# OMPI_BUILD_MPI_PROFILING is enabled when we want our generated MPI_* symbols
-# to be replaced by PMPI_*.
-# In this directory, we need it to be 1
-
-AM_CPPFLAGS     = -DOMPI_BUILD_MPI_PROFILING=1
-
-include $(top_srcdir)/Makefile.ompi-rules
-
-# Convenience libtool library that will be slurped up into libmpi.la.
-noinst_LTLIBRARIES = libpmpiext_pcollreq_c.la
-
-# This is where the top-level header file (that is included in
-# <mpi-ext.h>) must be installed.
-ompidir = $(ompiincludedir)/mpiext
-
-# This is the header file that is installed.
-ompi_HEADERS = pmpiext_pcollreq_c.h
-
-# Sources for the convenience libtool library.  Other than the one
-# header file, all source files in the extension have no file naming
-# conventions.
-nodist_libpmpiext_pcollreq_c_la_SOURCES = \
-        $(ompi_HEADERS) \
-        pallgather_init.c \
-        pallgatherv_init.c \
-        pallreduce_init.c \
-        palltoall_init.c \
-        palltoallv_init.c \
-        palltoallw_init.c \
-        pbarrier_init.c \
-        pbcast_init.c \
-        pexscan_init.c \
-        pgather_init.c \
-        pgatherv_init.c \
-        preduce_init.c \
-        preduce_scatter_block_init.c \
-        preduce_scatter_init.c \
-        pscan_init.c \
-        pscatter_init.c \
-        pscatterv_init.c \
-        \
-        pneighbor_allgather_init.c \
-        pneighbor_allgatherv_init.c \
-        pneighbor_alltoall_init.c \
-        pneighbor_alltoallv_init.c \
-        pneighbor_alltoallw_init.c
-
-#
-# Sym link in the sources from the real MPI directory
-#
-$(nodist_libpmpiext_pcollreq_c_la_SOURCES):
-	$(OMPI_V_LN_S) if test ! -r $@ ; then \
-		pname=`echo $@ | cut -b '2-'` ; \
-		$(LN_S) $(top_srcdir)/ompi/mpiext/pcollreq/c/$$pname $@ ; \
-	fi
diff --git a/ompi/mpiext/pcollreq/c/reduce_init.c b/ompi/mpiext/pcollreq/c/reduce_init.c
deleted file mode 100644
index 87378e86367..00000000000
--- a/ompi/mpiext/pcollreq/c/reduce_init.c
+++ /dev/null
@@ -1,148 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2016      IBM Corporation.  All rights reserved.
- * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/op/op.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_Reduce_init = PMPIX_Reduce_init
-#endif
-#define MPIX_Reduce_init PMPIX_Reduce_init
-#endif
-
-static const char FUNC_NAME[] = "MPIX_Reduce_init";
-
-
-int MPIX_Reduce_init(const void *sendbuf, void *recvbuf, int count,
-                     MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm,
-                     MPI_Info info, MPI_Request *request)
-{
-    int err;
-
-    SPC_RECORD(OMPI_SPC_REDUCE_INIT, 1);
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_comm(comm);
-
-        if(OMPI_COMM_IS_INTRA(comm)) {
-            if(ompi_comm_rank(comm) == root) {
-                /* check whether root's send buffer is defined. */
-                if (MPI_IN_PLACE == sendbuf) {
-                    memchecker_call(&opal_memchecker_base_isdefined, recvbuf, count, datatype);
-                } else {
-                    memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
-                }
-
-                /* check whether root's receive buffer is addressable. */
-                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, count, datatype);
-            } else {
-                /* check whether send buffer is defined on other processes. */
-                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
-            }
-        } else {
-            if (MPI_ROOT == root) {
-                /* check whether root's receive buffer is addressable. */
-                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, count, datatype);
-            } else if (MPI_PROC_NULL != root) {
-                /* check whether send buffer is defined. */
-                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
-            }
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        char *msg;
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-
-        /* Checks for all ranks */
-
-        else if (MPI_OP_NULL == op || NULL == op) {
-            err = MPI_ERR_OP;
-        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
-            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
-            free(msg);
-            return ret;
-        } else if ((ompi_comm_rank(comm) != root && MPI_IN_PLACE == sendbuf) ||
-                   (ompi_comm_rank(comm) == root && ((MPI_IN_PLACE == recvbuf) || (sendbuf == recvbuf)))) {
-            err = MPI_ERR_ARG;
-        } else {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-
-        /* Intercommunicator errors */
-
-        if (!OMPI_COMM_IS_INTRA(comm)) {
-            if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
-                   MPI_ROOT == root || MPI_PROC_NULL == root)) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
-            }
-        }
-
-        /* Intracommunicator errors */
-
-        else {
-            if (root < 0 || root >= ompi_comm_size(comm)) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
-            }
-        }
-    }
-
-    /* MPI standard says that reductions have to have a count of at least 1,
-     * but some benchmarks (e.g., IMB) calls this function with a count of 0.
-     * So handle that case.
-     */
-    if (0 == count) {
-        err = ompi_request_persistent_noop_create(request);
-        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_reduce_init(sendbuf, recvbuf, count,
-                                         datatype, op, root, comm, info, request,
-                                         comm->c_coll->coll_reduce_init_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_op(*request, op, datatype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpiext/pcollreq/c/reduce_scatter_block_init.c b/ompi/mpiext/pcollreq/c/reduce_scatter_block_init.c
deleted file mode 100644
index 8c450f449e9..00000000000
--- a/ompi/mpiext/pcollreq/c/reduce_scatter_block_init.c
+++ /dev/null
@@ -1,110 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012      Oak Ridge National Labs. All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/op/op.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_Reduce_scatter_block_init = PMPIX_Reduce_scatter_block_init
-#endif
-#define MPIX_Reduce_scatter_block_init PMPIX_Reduce_scatter_block_init
-#endif
-
-static const char FUNC_NAME[] = "MPIX_Reduce_scatter_block_init";
-
-
-int MPIX_Reduce_scatter_block_init(const void *sendbuf, void *recvbuf, int recvcount,
-                                   MPI_Datatype datatype, MPI_Op op,
-                                   MPI_Comm comm, MPI_Info info, MPI_Request *request)
-{
-    int err;
-
-    SPC_RECORD(OMPI_SPC_REDUCE_SCATTER_BLOCK_INIT, 1);
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-        memchecker_datatype(datatype);
-
-        /* check receive buffer of current proccess, whether it's addressable. */
-        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf,
-                        recvcount, datatype);
-
-        /* check whether the actual send buffer is defined. */
-        if(MPI_IN_PLACE == sendbuf) {
-            memchecker_call(&opal_memchecker_base_isdefined, recvbuf, recvcount, datatype);
-        } else {
-            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, recvcount, datatype);
-
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        char *msg;
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-
-        /* Unrooted operation; same checks for all ranks on both
-           intracommunicators and intercommunicators */
-
-        else if (MPI_OP_NULL == op || NULL == op) {
-          err = MPI_ERR_OP;
-        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
-            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
-            free(msg);
-            return ret;
-        } else if (MPI_IN_PLACE == recvbuf) {
-          err = MPI_ERR_ARG;
-        }
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-
-        OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, recvcount);
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-
-    err = comm->c_coll->coll_reduce_scatter_block_init(sendbuf, recvbuf, recvcount,
-                                                       datatype, op, comm, info, request,
-                                                       comm->c_coll->coll_reduce_scatter_block_init_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_op(*request, op, datatype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpiext/pcollreq/c/reduce_scatter_init.c b/ompi/mpiext/pcollreq/c/reduce_scatter_init.c
deleted file mode 100644
index 521833071c3..00000000000
--- a/ompi/mpiext/pcollreq/c/reduce_scatter_init.c
+++ /dev/null
@@ -1,144 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2016      IBM Corporation.  All rights reserved.
- * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/op/op.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_Reduce_scatter_init = PMPIX_Reduce_scatter_init
-#endif
-#define MPIX_Reduce_scatter_init PMPIX_Reduce_scatter_init
-#endif
-
-static const char FUNC_NAME[] = "MPIX_Reduce_scatter_init";
-
-
-int MPIX_Reduce_scatter_init(const void *sendbuf, void *recvbuf, const int recvcounts[],
-                             MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, MPI_Info info, MPI_Request *request)
-{
-    int i, err, size, count;
-
-    SPC_RECORD(OMPI_SPC_REDUCE_SCATTER_INIT, 1);
-
-    MEMCHECKER(
-        int rank;
-        int count;
-
-        size = ompi_comm_size(comm);
-        rank = ompi_comm_rank(comm);
-        for (count = i = 0; i < size; ++i) {
-            if (0 == recvcounts[i]) {
-                count += recvcounts[i];
-            }
-        }
-
-        memchecker_comm(comm);
-        memchecker_datatype(datatype);
-
-        /* check receive buffer of current proccess, whether it's addressable. */
-        memchecker_call(&opal_memchecker_base_isaddressable, recvbuf,
-                        recvcounts[rank], datatype);
-
-        /* check whether the actual send buffer is defined. */
-        if(MPI_IN_PLACE == sendbuf) {
-            memchecker_call(&opal_memchecker_base_isdefined, recvbuf, count, datatype);
-        } else {
-            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
-
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        char *msg;
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-
-        /* Unrooted operation; same checks for all ranks on both
-           intracommunicators and intercommunicators */
-
-        else if (MPI_OP_NULL == op || NULL == op) {
-          err = MPI_ERR_OP;
-        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
-            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
-            free(msg);
-            return ret;
-        } else if (NULL == recvcounts) {
-          err = MPI_ERR_COUNT;
-        } else if (MPI_IN_PLACE == recvbuf) {
-          err = MPI_ERR_ARG;
-        }
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-
-        /* Based on the standard each group has to provide the same total
-           number of elements, so the size of the recvcounts array depends
-           on the number of participants in the local group.  */
-        size = ompi_comm_size(comm);
-        for (i = 0; i < size; ++i) {
-          OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, recvcounts[i]);
-          OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-    }
-
-    /* MPI standard says that reductions have to have a count of at least 1,
-     * but some benchmarks (e.g., IMB) calls this function with a count of 0.
-     * So handle that case.
-     */
-    size = ompi_comm_size(comm);
-    for (count = i = 0; i < size; ++i) {
-        if (0 == recvcounts[i]) {
-            ++count;
-        }
-    }
-    if (size == count) {
-        err = ompi_request_persistent_noop_create(request);
-        OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-
-    err = comm->c_coll->coll_reduce_scatter_init(sendbuf, recvbuf, recvcounts,
-                                                 datatype, op, comm, info, request,
-                                                 comm->c_coll->coll_reduce_scatter_init_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_op(*request, op, datatype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpiext/pcollreq/c/scan_init.c b/ompi/mpiext/pcollreq/c/scan_init.c
deleted file mode 100644
index 6b4bc0ccad8..00000000000
--- a/ompi/mpiext/pcollreq/c/scan_init.c
+++ /dev/null
@@ -1,108 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/op/op.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_Scan_init = PMPIX_Scan_init
-#endif
-#define MPIX_Scan_init PMPIX_Scan_init
-#endif
-
-static const char FUNC_NAME[] = "MPIX_Scan_init";
-
-
-int MPIX_Scan_init(const void *sendbuf, void *recvbuf, int count,
-                   MPI_Datatype datatype, MPI_Op op, MPI_Comm comm,
-                   MPI_Info info, MPI_Request *request)
-{
-    int err;
-
-    SPC_RECORD(OMPI_SPC_SCAN_INIT, 1);
-
-    MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_comm(comm);
-        if (MPI_IN_PLACE != sendbuf) {
-            memchecker_call(&opal_memchecker_base_isdefined, sendbuf, count, datatype);
-        } else {
-            memchecker_call(&opal_memchecker_base_isdefined, recvbuf, count, datatype);
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        char *msg;
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        }
-
-        /* No intercommunicators allowed! (MPI does not define
-           MPI_SCAN on intercommunicators) */
-
-        else if (OMPI_COMM_IS_INTER(comm)) {
-          err = MPI_ERR_COMM;
-        }
-
-        /* Unrooted operation; checks for all ranks */
-
-        else if (MPI_OP_NULL == op || NULL == op) {
-          err = MPI_ERR_OP;
-        } else if (MPI_IN_PLACE == recvbuf) {
-          err = MPI_ERR_ARG;
-        } else if (!ompi_op_is_valid(op, datatype, &msg, FUNC_NAME)) {
-            int ret = OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_OP, msg);
-            free(msg);
-            return ret;
-        } else {
-          OMPI_CHECK_DATATYPE_FOR_SEND(err, datatype, count);
-        }
-        OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-    }
-
-    /* Call the coll component to actually perform the allgather */
-
-    err = comm->c_coll->coll_scan_init(sendbuf, recvbuf, count,
-                                       datatype, op, comm,
-                                       info, request,
-                                       comm->c_coll->coll_scan_init_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        ompi_coll_base_retain_op(*request, op, datatype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpiext/pcollreq/c/scatter_init.c b/ompi/mpiext/pcollreq/c/scatter_init.c
deleted file mode 100644
index aa48a0a94a1..00000000000
--- a/ompi/mpiext/pcollreq/c/scatter_init.c
+++ /dev/null
@@ -1,179 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2008      University of Houston.  All rights reserved.
- * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_Scatter_init = PMPIX_Scatter_init
-#endif
-#define MPIX_Scatter_init PMPIX_Scatter_init
-#endif
-
-static const char FUNC_NAME[] = "MPIX_Scatter_init";
-
-
-int MPIX_Scatter_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
-                      void *recvbuf, int recvcount, MPI_Datatype recvtype,
-                      int root, MPI_Comm comm, MPI_Info info, MPI_Request *request)
-{
-    int err;
-
-    SPC_RECORD(OMPI_SPC_SCATTER_INIT, 1);
-
-    MEMCHECKER(
-        memchecker_comm(comm);
-        if(OMPI_COMM_IS_INTRA(comm)) {
-            if(ompi_comm_rank(comm) == root) {
-                memchecker_datatype(sendtype);
-                /* check whether root's send buffer is defined. */
-                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-                if(MPI_IN_PLACE != recvbuf) {
-                    memchecker_datatype(recvtype);
-                    /* check whether receive buffer is addressable. */
-                    memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-                }
-            } else {
-                memchecker_datatype(recvtype);
-                /* check whether receive buffer is addressable. */
-                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-            }
-        } else {
-            if(MPI_ROOT == root) {
-                memchecker_datatype(sendtype);
-                /* check whether root's send buffer is defined. */
-                memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
-            } else if (MPI_PROC_NULL != root) {
-                memchecker_datatype(recvtype);
-                /* check whether receive buffer is addressable. */
-                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-            }
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if ((ompi_comm_rank(comm) != root && MPI_IN_PLACE == recvbuf) ||
-                   (ompi_comm_rank(comm) == root && MPI_IN_PLACE == sendbuf)) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-
-        /* Errors for intracommunicators */
-
-        if (OMPI_COMM_IS_INTRA(comm)) {
-
-          /* Errors for all ranks */
-
-          if ((root >= ompi_comm_size(comm)) || (root < 0)) {
-              err = MPI_ERR_ROOT;
-          } else if (MPI_IN_PLACE != recvbuf) {
-              if (recvcount < 0) {
-                  err = MPI_ERR_COUNT;
-              } else if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
-                  err = MPI_ERR_TYPE;
-              }
-          }
-
-          /* Errors for the root.  Some of these could have been
-             combined into compound if statements above, but since
-             this whole section can be compiled out (or turned off at
-             run time) for efficiency, it's more clear to separate
-             them out into individual tests. */
-
-          else if (ompi_comm_rank(comm) == root) {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-          }
-          OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-
-        /* Errors for intercommunicators */
-
-        else {
-          if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
-                 MPI_ROOT == root || MPI_PROC_NULL == root)) {
-            err = MPI_ERR_ROOT;
-          }
-
-          /* Errors for the receivers */
-
-          else if (MPI_ROOT != root && MPI_PROC_NULL != root) {
-            if (recvcount < 0) {
-              err = MPI_ERR_COUNT;
-            } else if (MPI_DATATYPE_NULL == recvtype) {
-              err = MPI_ERR_TYPE;
-            }
-          }
-
-          /* Errors for the root.  Ditto on the comment above -- these
-             error checks could have been combined above, but let's
-             make the code easier to read. */
-
-          else if (MPI_ROOT == root) {
-            OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcount);
-          }
-          OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-        }
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_scatter_init(sendbuf, sendcount, sendtype, recvbuf,
-                                          recvcount, recvtype, root, comm, info, request,
-                                          comm->c_coll->coll_scatter_init_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        if (OMPI_COMM_IS_INTRA(comm)) {
-            if (MPI_IN_PLACE == recvbuf) {
-                recvtype = NULL;
-            } else if (ompi_comm_rank(comm) != root) {
-                sendtype = NULL;
-            }
-        } else {
-            if (MPI_ROOT == root) {
-                recvtype = NULL;
-            } else if (MPI_PROC_NULL == root) {
-                sendtype = NULL;
-                recvtype = NULL;
-            } else {
-                sendtype = NULL;
-            }
-        }
-        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpiext/pcollreq/c/scatterv_init.c b/ompi/mpiext/pcollreq/c/scatterv_init.c
deleted file mode 100644
index 1170b79b834..00000000000
--- a/ompi/mpiext/pcollreq/c/scatterv_init.c
+++ /dev/null
@@ -1,219 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "ompi_config.h"
-#include <stdio.h>
-
-#include "ompi/mpi/c/bindings.h"
-#include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "ompi/memchecker.h"
-#include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h"
-#include "ompi/runtime/ompi_spc.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_Scatterv_init = PMPIX_Scatterv_init
-#endif
-#define MPIX_Scatterv_init PMPIX_Scatterv_init
-#endif
-
-static const char FUNC_NAME[] = "MPIX_Scatterv_init";
-
-
-int MPIX_Scatterv_init(const void *sendbuf, const int sendcounts[], const int displs[],
-                       MPI_Datatype sendtype, void *recvbuf, int recvcount,
-                       MPI_Datatype recvtype, int root, MPI_Comm comm, MPI_Info info, MPI_Request *request)
-{
-    int i, size, err;
-
-    SPC_RECORD(OMPI_SPC_SCATTERV_INIT, 1);
-
-    MEMCHECKER(
-        ptrdiff_t ext;
-
-        size = ompi_comm_remote_size(comm);
-        ompi_datatype_type_extent(recvtype, &ext);
-
-        memchecker_comm(comm);
-        if(OMPI_COMM_IS_INTRA(comm)) {
-              if(ompi_comm_rank(comm) == root) {
-                memchecker_datatype(sendtype);
-                /* check whether root's send buffer is defined. */
-                for (i = 0; i < size; i++) {
-                    memchecker_call(&opal_memchecker_base_isdefined,
-                                    (char *)(sendbuf)+displs[i]*ext,
-                                    sendcounts[i], sendtype);
-                }
-                if(MPI_IN_PLACE != recvbuf) {
-                    memchecker_datatype(recvtype);
-                    /* check whether receive buffer is addressable. */
-                    memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-                }
-              } else {
-                  memchecker_datatype(recvtype);
-                  /* check whether receive buffer is addressable. */
-                  memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-              }
-        } else {
-            if(MPI_ROOT == root) {
-                  memchecker_datatype(sendtype);
-                  /* check whether root's send buffer is defined. */
-                  for (i = 0; i < size; i++) {
-                      memchecker_call(&opal_memchecker_base_isdefined,
-                                      (char *)(sendbuf)+displs[i]*ext,
-                                      sendcounts[i], sendtype);
-                  }
-            } else if (MPI_PROC_NULL != root) {
-                /* check whether receive buffer is addressable. */
-                memchecker_call(&opal_memchecker_base_isaddressable, recvbuf, recvcount, recvtype);
-            }
-        }
-    );
-
-    if (MPI_PARAM_CHECK) {
-        err = MPI_SUCCESS;
-        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
-        if (ompi_comm_invalid(comm)) {
-            return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM,
-                                          FUNC_NAME);
-        } else if ((ompi_comm_rank(comm) != root && MPI_IN_PLACE == recvbuf) ||
-                   (ompi_comm_rank(comm) == root && MPI_IN_PLACE == sendbuf)) {
-            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-        }
-
-        /* Errors for intracommunicators */
-
-        if (OMPI_COMM_IS_INTRA(comm)) {
-
-            /* Errors for all ranks */
-
-            if ((root >= ompi_comm_size(comm)) || (root < 0)) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
-            }
-
-            if (MPI_IN_PLACE != recvbuf) {
-                if (recvcount < 0) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT,
-                                                  FUNC_NAME);
-                }
-
-                if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE,
-                                                  FUNC_NAME);
-                }
-            }
-
-            /* Errors for the root.  Some of these could have been
-               combined into compound if statements above, but since
-               this whole section can be compiled out (or turned off at
-               run time) for efficiency, it's more clear to separate
-               them out into individual tests. */
-
-            if (ompi_comm_rank(comm) == root) {
-                if (NULL == displs) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-                }
-
-                if (NULL == sendcounts) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-                }
-
-                size = ompi_comm_size(comm);
-                for (i = 0; i < size; ++i) {
-                    OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcounts[i]);
-                    OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-                }
-            }
-        }
-
-        /* Errors for intercommunicators */
-
-        else {
-            if (! ((root >= 0 && root < ompi_comm_remote_size(comm)) ||
-                   MPI_ROOT == root || MPI_PROC_NULL == root)) {
-                return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ROOT, FUNC_NAME);
-            }
-
-          /* Errors for the receivers */
-
-            if (MPI_ROOT != root && MPI_PROC_NULL != root) {
-                if (recvcount < 0) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-                }
-
-                if (MPI_DATATYPE_NULL == recvtype || NULL == recvtype) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TYPE, FUNC_NAME);
-                }
-            }
-
-            /* Errors for the root.  Ditto on the comment above -- these
-               error checks could have been combined above, but let's
-               make the code easier to read. */
-
-            else if (MPI_ROOT == root) {
-                if (NULL == displs) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME);
-                }
-
-                if (NULL == sendcounts) {
-                    return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME);
-                }
-
-                size = ompi_comm_remote_size(comm);
-                for (i = 0; i < size; ++i) {
-                    OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcounts[i]);
-                    OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME);
-                }
-            }
-        }
-    }
-
-    /* Invoke the coll component to perform the back-end operation */
-    err = comm->c_coll->coll_scatterv_init(sendbuf, sendcounts, displs,
-                                           sendtype, recvbuf, recvcount, recvtype, root, comm,
-                                           info, request, comm->c_coll->coll_scatterv_init_module);
-    if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
-        if (OMPI_COMM_IS_INTRA(comm)) {
-            if (MPI_IN_PLACE == recvbuf) {
-                recvtype = NULL;
-            } else if (ompi_comm_rank(comm) != root) {
-                sendtype = NULL;
-            }
-        } else {
-            if (MPI_ROOT == root) {
-                recvtype = NULL;
-            } else if (MPI_PROC_NULL == root) {
-                sendtype = NULL;
-                recvtype = NULL;
-            } else {
-                sendtype = NULL;
-            }
-        }
-        ompi_coll_base_retain_datatypes(*request, sendtype, recvtype);
-    }
-    OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
-}
diff --git a/ompi/mpiext/pcollreq/configure.m4 b/ompi/mpiext/pcollreq/configure.m4
deleted file mode 100644
index 26d974b9927..00000000000
--- a/ompi/mpiext/pcollreq/configure.m4
+++ /dev/null
@@ -1,30 +0,0 @@
-# -*- shell-script -*-
-#
-# Copyright (c) 2017-2018 FUJITSU LIMITED.  All rights reserved.
-# Copyright (c) 2018      Research Organization for Information Science
-#                         and Technology (RIST). All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-# OMPI_MPIEXT_pcollreq_CONFIG([action-if-found], [action-if-not-found])
-# -----------------------------------------------------------
-AC_DEFUN([OMPI_MPIEXT_pcollreq_CONFIG],[
-    AC_CONFIG_FILES([
-        ompi/mpiext/pcollreq/Makefile
-        ompi/mpiext/pcollreq/c/Makefile
-        ompi/mpiext/pcollreq/c/profile/Makefile
-        ompi/mpiext/pcollreq/mpif-h/Makefile
-        ompi/mpiext/pcollreq/mpif-h/profile/Makefile
-        ompi/mpiext/pcollreq/use-mpi/Makefile
-        ompi/mpiext/pcollreq/use-mpi-f08/Makefile
-    ])
-
-    AS_IF([test "$ENABLE_pcollreq" = "1" || \
-           test "$ENABLE_EXT_ALL" = "1"],
-          [$1],
-          [$2])
-])
diff --git a/ompi/mpiext/pcollreq/mpif-h/Makefile.am b/ompi/mpiext/pcollreq/mpif-h/Makefile.am
deleted file mode 100644
index 7282ac086d7..00000000000
--- a/ompi/mpiext/pcollreq/mpif-h/Makefile.am
+++ /dev/null
@@ -1,88 +0,0 @@
-#
-# Copyright (c) 2004-2009 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
-# Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-# Copyright (c) 2018      Research Organization for Information Science
-#                         and Technology (RIST).  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-SUBDIRS = profile
-
-# This file builds the mpif.h-based bindings for MPI extensions.  It
-# is optional in MPI extensions.
-
-# We must set these #defines so that the inner OMPI MPI prototype
-# header files do the Right Thing.
-AM_CPPFLAGS = -DOMPI_BUILD_MPI_PROFILING=0 -DOMPI_COMPILING_FORTRAN_WRAPPERS=1
-
-# Note that the mpif.h-based bindings are optional -- they can only be
-# built if OMPI is also building the Fortran-based bindings.  So we
-# initially set some Makefile macros to empty, and then conditionally
-# add to them later.
-noinst_LTLIBRARIES =
-
-# Directory where the header file to be included in mpif-ext.h must be
-# installed.
-ompidir = $(ompiincludedir)/mpiext
-
-# Just like noinst_LTLIBRARIES, set this macro to empty and
-# conditionally add to it later.
-ompi_HEADERS =
-
-# Use the Automake conditional to know if we're building the mpif.h
-# bindings.
-if OMPI_BUILD_FORTRAN_MPIFH_BINDINGS
-
-# If we are, build the convenience libtool library that will be
-# slurped up into libmpi_mpifh.la.
-noinst_LTLIBRARIES += libmpiext_pcollreq_mpifh.la
-
-# This is the header file that is installed.
-ompi_HEADERS += mpiext_pcollreq_mpifh.h
-
-noinst_HEADERS = mpiext_pcollreq_prototypes.h
-
-# Sources for the convenience libtool library.
-libmpiext_pcollreq_mpifh_la_SOURCES = \
-        $(ompi_HEADERS) \
-        mpiext_pcollreq_mpifh.c
-
-if BUILD_MPI_BINDINGS_LAYER
-libmpiext_pcollreq_mpifh_la_SOURCES += \
-        allgather_init_f.c \
-        allgatherv_init_f.c \
-        allreduce_init_f.c \
-        alltoall_init_f.c \
-        alltoallv_init_f.c \
-        alltoallw_init_f.c \
-        barrier_init_f.c \
-        bcast_init_f.c \
-        exscan_init_f.c \
-        gather_init_f.c \
-        gatherv_init_f.c \
-        reduce_init_f.c \
-        reduce_scatter_init_f.c \
-        reduce_scatter_block_init_f.c \
-        scan_init_f.c \
-        scatter_init_f.c \
-        scatterv_init_f.c \
-        \
-        neighbor_allgather_init_f.c \
-        neighbor_allgatherv_init_f.c \
-        neighbor_alltoall_init_f.c \
-        neighbor_alltoallv_init_f.c \
-        neighbor_alltoallw_init_f.c
-endif
-
-libmpiext_pcollreq_mpifh_la_LIBADD = \
-        profile/libpmpiext_pcollreq_mpifh.la
-
-endif
diff --git a/ompi/mpiext/pcollreq/mpif-h/allgather_init_f.c b/ompi/mpiext/pcollreq/mpif-h/allgather_init_f.c
deleted file mode 100644
index 84e3f4259da..00000000000
--- a/ompi/mpiext/pcollreq/mpif-h/allgather_init_f.c
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/fortran/mpif-h/bindings.h"
-#include "ompi/mpi/fortran/base/constants.h"
-#include "ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_prototypes.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak PMPIX_ALLGATHER_INIT = ompix_allgather_init_f
-#pragma weak pmpix_allgather_init = ompix_allgather_init_f
-#pragma weak pmpix_allgather_init_ = ompix_allgather_init_f
-#pragma weak pmpix_allgather_init__ = ompix_allgather_init_f
-
-#pragma weak PMPIX_Allgather_init_f = ompix_allgather_init_f
-#pragma weak PMPIX_Allgather_init_f08 = ompix_allgather_init_f
-#else
-OMPI_GENERATE_F77_BINDINGS (PMPIX_ALLGATHER_INIT,
-                            pmpix_allgather_init,
-                            pmpix_allgather_init_,
-                            pmpix_allgather_init__,
-                            pompix_allgather_init_f,
-                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, info, request, ierr) )
-#endif
-#endif
-
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_ALLGATHER_INIT = ompix_allgather_init_f
-#pragma weak mpix_allgather_init = ompix_allgather_init_f
-#pragma weak mpix_allgather_init_ = ompix_allgather_init_f
-#pragma weak mpix_allgather_init__ = ompix_allgather_init_f
-
-#pragma weak MPIX_Allgather_init_f = ompix_allgather_init_f
-#pragma weak MPIX_Allgather_init_f08 = ompix_allgather_init_f
-#else
-#if ! OMPI_BUILD_MPI_PROFILING
-OMPI_GENERATE_F77_BINDINGS (MPIX_ALLGATHER_INIT,
-                            mpix_allgather_init,
-                            mpix_allgather_init_,
-                            mpix_allgather_init__,
-                            ompix_allgather_init_f,
-                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, info, request, ierr) )
-#else
-#define ompix_allgather_init_f pompix_allgather_init_f
-#endif
-#endif
-
-
-void ompix_allgather_init_f(char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype,
-                            char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype,
-                            MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
-{
-    int ierr_c;
-    MPI_Comm c_comm;
-    MPI_Request c_req;
-    MPI_Datatype c_sendtype, c_recvtype;
-    MPI_Info c_info;
-
-    c_comm = PMPI_Comm_f2c(*comm);
-    c_sendtype = PMPI_Type_f2c(*sendtype);
-    c_recvtype = PMPI_Type_f2c(*recvtype);
-    c_info = PMPI_Info_f2c(*info);
-
-    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
-    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
-    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
-
-    ierr_c = PMPIX_Allgather_init(sendbuf,
-                                  OMPI_FINT_2_INT(*sendcount),
-                                  c_sendtype,
-                                  recvbuf,
-                                  OMPI_FINT_2_INT(*recvcount),
-                                  c_recvtype, c_comm, c_info, &c_req);
-
-    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(ierr_c);
-
-    if (MPI_SUCCESS == ierr_c) *request = PMPI_Request_c2f(c_req);
-}
diff --git a/ompi/mpiext/pcollreq/mpif-h/allgatherv_init_f.c b/ompi/mpiext/pcollreq/mpif-h/allgatherv_init_f.c
deleted file mode 100644
index 9a1576c9e00..00000000000
--- a/ompi/mpiext/pcollreq/mpif-h/allgatherv_init_f.c
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/fortran/mpif-h/bindings.h"
-#include "ompi/mpi/fortran/base/constants.h"
-#include "ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_prototypes.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak PMPIX_ALLGATHERV_INIT = ompix_allgatherv_init_f
-#pragma weak pmpix_allgatherv_init = ompix_allgatherv_init_f
-#pragma weak pmpix_allgatherv_init_ = ompix_allgatherv_init_f
-#pragma weak pmpix_allgatherv_init__ = ompix_allgatherv_init_f
-
-#pragma weak PMPIX_Allgatherv_init_f = ompix_allgatherv_init_f
-#pragma weak PMPIX_Allgatherv_init_f08 = ompix_allgatherv_init_f
-#else
-OMPI_GENERATE_F77_BINDINGS (PMPIX_ALLGATHERV_INIT,
-                            pmpix_allgatherv_init,
-                            pmpix_allgatherv_init_,
-                            pmpix_allgatherv_init__,
-                            pompix_allgatherv_init_f,
-                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *displs, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, comm, info, request, ierr) )
-#endif
-#endif
-
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_ALLGATHERV_INIT = ompix_allgatherv_init_f
-#pragma weak mpix_allgatherv_init = ompix_allgatherv_init_f
-#pragma weak mpix_allgatherv_init_ = ompix_allgatherv_init_f
-#pragma weak mpix_allgatherv_init__ = ompix_allgatherv_init_f
-
-#pragma weak MPIX_Allgatherv_init_f = ompix_allgatherv_init_f
-#pragma weak MPIX_Allgatherv_init_f08 = ompix_allgatherv_init_f
-#else
-#if ! OMPI_BUILD_MPI_PROFILING
-OMPI_GENERATE_F77_BINDINGS (MPIX_ALLGATHERV_INIT,
-                            mpix_allgatherv_init,
-                            mpix_allgatherv_init_,
-                            mpix_allgatherv_init__,
-                            ompix_allgatherv_init_f,
-                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *displs, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, comm, info, request, ierr) )
-#else
-#define ompix_allgatherv_init_f pompix_allgatherv_init_f
-#endif
-#endif
-
-
-void ompix_allgatherv_init_f(char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype,
-                             char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *displs,
-                             MPI_Fint *recvtype, MPI_Fint *comm,
-                             MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
-{
-    MPI_Comm c_comm;
-    MPI_Datatype c_sendtype, c_recvtype;
-    MPI_Request c_request;
-    MPI_Info c_info;
-    int size, ierr_c;
-    OMPI_ARRAY_NAME_DECL(recvcounts);
-    OMPI_ARRAY_NAME_DECL(displs);
-
-    c_comm = PMPI_Comm_f2c(*comm);
-    c_sendtype = PMPI_Type_f2c(*sendtype);
-    c_recvtype = PMPI_Type_f2c(*recvtype);
-    c_info = PMPI_Info_f2c(*info);
-
-    PMPI_Comm_size(c_comm, &size);
-    OMPI_ARRAY_FINT_2_INT(recvcounts, size);
-    OMPI_ARRAY_FINT_2_INT(displs, size);
-
-    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
-    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
-    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
-
-    ierr_c = PMPIX_Allgatherv_init(sendbuf,
-                                   OMPI_FINT_2_INT(*sendcount),
-                                   c_sendtype,
-                                   recvbuf,
-                                   OMPI_ARRAY_NAME_CONVERT(recvcounts),
-                                   OMPI_ARRAY_NAME_CONVERT(displs),
-                                   c_recvtype, c_comm, c_info, &c_request);
-
-    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(ierr_c);
-    if (MPI_SUCCESS == ierr_c) *request = PMPI_Request_c2f(c_request);
-
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(recvcounts);
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(displs);
-}
diff --git a/ompi/mpiext/pcollreq/mpif-h/allreduce_init_f.c b/ompi/mpiext/pcollreq/mpif-h/allreduce_init_f.c
deleted file mode 100644
index 3ed7f2748bb..00000000000
--- a/ompi/mpiext/pcollreq/mpif-h/allreduce_init_f.c
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/fortran/mpif-h/bindings.h"
-#include "ompi/mpi/fortran/base/constants.h"
-#include "ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_prototypes.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak PMPIX_ALLREDUCE_INIT = ompix_allreduce_init_f
-#pragma weak pmpix_allreduce_init = ompix_allreduce_init_f
-#pragma weak pmpix_allreduce_init_ = ompix_allreduce_init_f
-#pragma weak pmpix_allreduce_init__ = ompix_allreduce_init_f
-
-#pragma weak PMPIX_Allreduce_init_f = ompix_allreduce_init_f
-#pragma weak PMPIX_Allreduce_init_f08 = ompix_allreduce_init_f
-#else
-OMPI_GENERATE_F77_BINDINGS (PMPIX_ALLREDUCE_INIT,
-                            pmpix_allreduce_init,
-                            pmpix_allreduce_init_,
-                            pmpix_allreduce_init__,
-                            pompix_allreduce_init_f,
-                            (char *sendbuf, char *recvbuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, recvbuf, count, datatype, op, comm, info, request, ierr) )
-#endif
-#endif
-
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_ALLREDUCE_INIT = ompix_allreduce_init_f
-#pragma weak mpix_allreduce_init = ompix_allreduce_init_f
-#pragma weak mpix_allreduce_init_ = ompix_allreduce_init_f
-#pragma weak mpix_allreduce_init__ = ompix_allreduce_init_f
-
-#pragma weak MPIX_Allreduce_init_f = ompix_allreduce_init_f
-#pragma weak MPIX_Allreduce_init_f08 = ompix_allreduce_init_f
-#else
-#if ! OMPI_BUILD_MPI_PROFILING
-OMPI_GENERATE_F77_BINDINGS (MPIX_ALLREDUCE_INIT,
-                            mpix_allreduce_init,
-                            mpix_allreduce_init_,
-                            mpix_allreduce_init__,
-                            ompix_allreduce_init_f,
-                            (char *sendbuf, char *recvbuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, recvbuf, count, datatype, op, comm, info, request, ierr) )
-#else
-#define ompix_allreduce_init_f pompix_allreduce_init_f
-#endif
-#endif
-
-
-void ompix_allreduce_init_f(char *sendbuf, char *recvbuf, MPI_Fint *count,
-                            MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm,
-                            MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
-{
-    int ierr_c;
-    MPI_Comm c_comm;
-    MPI_Datatype c_type;
-    MPI_Info c_info;
-    MPI_Request c_request;
-    MPI_Op c_op;
-
-    c_comm = PMPI_Comm_f2c(*comm);
-    c_type = PMPI_Type_f2c(*datatype);
-    c_op = PMPI_Op_f2c(*op);
-    c_info = PMPI_Info_f2c(*info);
-
-    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
-    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
-    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
-
-    ierr_c = PMPIX_Allreduce_init(sendbuf, recvbuf,
-                                  OMPI_FINT_2_INT(*count),
-                                  c_type, c_op, c_comm, c_info, &c_request);
-    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(ierr_c);
-    if (MPI_SUCCESS == ierr_c) *request = PMPI_Request_c2f(c_request);
-}
diff --git a/ompi/mpiext/pcollreq/mpif-h/alltoall_init_f.c b/ompi/mpiext/pcollreq/mpif-h/alltoall_init_f.c
deleted file mode 100644
index 0dd6049d813..00000000000
--- a/ompi/mpiext/pcollreq/mpif-h/alltoall_init_f.c
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/fortran/mpif-h/bindings.h"
-#include "ompi/mpi/fortran/base/constants.h"
-#include "ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_prototypes.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak PMPIX_ALLTOALL_INIT = ompix_alltoall_init_f
-#pragma weak pmpix_alltoall_init = ompix_alltoall_init_f
-#pragma weak pmpix_alltoall_init_ = ompix_alltoall_init_f
-#pragma weak pmpix_alltoall_init__ = ompix_alltoall_init_f
-
-#pragma weak PMPIX_Alltoall_init_f = ompix_alltoall_init_f
-#pragma weak PMPIX_Alltoall_init_f08 = ompix_alltoall_init_f
-#else
-OMPI_GENERATE_F77_BINDINGS (PMPIX_ALLTOALL_INIT,
-                            pmpix_alltoall_init,
-                            pmpix_alltoall_init_,
-                            pmpix_alltoall_init__,
-                            pompix_alltoall_init_f,
-                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, info, request, ierr) )
-#endif
-#endif
-
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_ALLTOALL_INIT = ompix_alltoall_init_f
-#pragma weak mpix_alltoall_init = ompix_alltoall_init_f
-#pragma weak mpix_alltoall_init_ = ompix_alltoall_init_f
-#pragma weak mpix_alltoall_init__ = ompix_alltoall_init_f
-
-#pragma weak MPIX_Alltoall_init_f = ompix_alltoall_init_f
-#pragma weak MPIX_Alltoall_init_f08 = ompix_alltoall_init_f
-#else
-#if ! OMPI_BUILD_MPI_PROFILING
-OMPI_GENERATE_F77_BINDINGS (MPIX_ALLTOALL_INIT,
-                            mpix_alltoall_init,
-                            mpix_alltoall_init_,
-                            mpix_alltoall_init__,
-                            ompix_alltoall_init_f,
-                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, info, request, ierr) )
-#else
-#define ompix_alltoall_init_f pompix_alltoall_init_f
-#endif
-#endif
-
-
-void ompix_alltoall_init_f(char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype,
-                           char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype,
-                           MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
-{
-    int c_ierr;
-    MPI_Comm c_comm;
-    MPI_Request c_req;
-    MPI_Datatype c_sendtype, c_recvtype;
-    MPI_Info c_info;
-
-    c_comm = PMPI_Comm_f2c(*comm);
-    c_sendtype = PMPI_Type_f2c(*sendtype);
-    c_recvtype = PMPI_Type_f2c(*recvtype);
-    c_info = PMPI_Info_f2c(*info);
-
-    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
-    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
-    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
-
-    c_ierr = PMPIX_Alltoall_init(sendbuf,
-                                 OMPI_FINT_2_INT(*sendcount),
-                                 c_sendtype,
-                                 recvbuf,
-                                 OMPI_FINT_2_INT(*recvcount),
-                                 c_recvtype, c_comm, c_info, &c_req);
-    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
-
-    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_req);
-}
diff --git a/ompi/mpiext/pcollreq/mpif-h/alltoallv_init_f.c b/ompi/mpiext/pcollreq/mpif-h/alltoallv_init_f.c
deleted file mode 100644
index 562b10863ea..00000000000
--- a/ompi/mpiext/pcollreq/mpif-h/alltoallv_init_f.c
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/fortran/mpif-h/bindings.h"
-#include "ompi/mpi/fortran/base/constants.h"
-#include "ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_prototypes.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak PMPIX_ALLTOALLV_INIT = ompix_alltoallv_init_f
-#pragma weak pmpix_alltoallv_init = ompix_alltoallv_init_f
-#pragma weak pmpix_alltoallv_init_ = ompix_alltoallv_init_f
-#pragma weak pmpix_alltoallv_init__ = ompix_alltoallv_init_f
-
-#pragma weak PMPIX_Alltoallv_init_f = ompix_alltoallv_init_f
-#pragma weak PMPIX_Alltoallv_init_f08 = ompix_alltoallv_init_f
-#else
-OMPI_GENERATE_F77_BINDINGS (PMPIX_ALLTOALLV_INIT,
-                            pmpix_alltoallv_init,
-                            pmpix_alltoallv_init_,
-                            pmpix_alltoallv_init__,
-                            pompix_alltoallv_init_f,
-                            (char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *sdispls, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *rdispls, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, sendcounts, sdispls, sendtype, recvbuf, recvcounts, rdispls, recvtype, comm, info, request, ierr) )
-#endif
-#endif
-
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_ALLTOALLV_INIT = ompix_alltoallv_init_f
-#pragma weak mpix_alltoallv_init = ompix_alltoallv_init_f
-#pragma weak mpix_alltoallv_init_ = ompix_alltoallv_init_f
-#pragma weak mpix_alltoallv_init__ = ompix_alltoallv_init_f
-
-#pragma weak MPIX_Alltoallv_init_f = ompix_alltoallv_init_f
-#pragma weak MPIX_Alltoallv_init_f08 = ompix_alltoallv_init_f
-#else
-#if ! OMPI_BUILD_MPI_PROFILING
-OMPI_GENERATE_F77_BINDINGS (MPIX_ALLTOALLV_INIT,
-                            mpix_alltoallv_init,
-                            mpix_alltoallv_init_,
-                            mpix_alltoallv_init__,
-                            ompix_alltoallv_init_f,
-                            (char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *sdispls, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *rdispls, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, sendcounts, sdispls, sendtype, recvbuf, recvcounts, rdispls, recvtype, comm, info, request, ierr) )
-#else
-#define ompix_alltoallv_init_f pompix_alltoallv_init_f
-#endif
-#endif
-
-
-void ompix_alltoallv_init_f(char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *sdispls,
-                       MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts,
-                       MPI_Fint *rdispls, MPI_Fint *recvtype,
-                       MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
-{
-    MPI_Comm c_comm;
-    MPI_Datatype c_sendtype, c_recvtype;
-    MPI_Info c_info;
-    MPI_Request c_request;
-    int size, c_ierr;
-    OMPI_ARRAY_NAME_DECL(sendcounts);
-    OMPI_ARRAY_NAME_DECL(sdispls);
-    OMPI_ARRAY_NAME_DECL(recvcounts);
-    OMPI_ARRAY_NAME_DECL(rdispls);
-
-    c_comm = PMPI_Comm_f2c(*comm);
-    c_sendtype = PMPI_Type_f2c(*sendtype);
-    c_recvtype = PMPI_Type_f2c(*recvtype);
-    c_info = PMPI_Info_f2c(*info);
-
-    PMPI_Comm_size(c_comm, &size);
-    OMPI_ARRAY_FINT_2_INT(sendcounts, size);
-    OMPI_ARRAY_FINT_2_INT(sdispls, size);
-    OMPI_ARRAY_FINT_2_INT(recvcounts, size);
-    OMPI_ARRAY_FINT_2_INT(rdispls, size);
-
-    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
-    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
-    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
-
-    c_ierr = PMPIX_Alltoallv_init(sendbuf,
-                                  OMPI_ARRAY_NAME_CONVERT(sendcounts),
-                                  OMPI_ARRAY_NAME_CONVERT(sdispls),
-                                  c_sendtype,
-                                  recvbuf,
-                                  OMPI_ARRAY_NAME_CONVERT(recvcounts),
-                                  OMPI_ARRAY_NAME_CONVERT(rdispls),
-                                  c_recvtype, c_comm, c_info, &c_request);
-    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
-    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
-
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(sendcounts);
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(sdispls);
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(recvcounts);
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(rdispls);
-}
diff --git a/ompi/mpiext/pcollreq/mpif-h/alltoallw_init_f.c b/ompi/mpiext/pcollreq/mpif-h/alltoallw_init_f.c
deleted file mode 100644
index 0fae1e194db..00000000000
--- a/ompi/mpiext/pcollreq/mpif-h/alltoallw_init_f.c
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/fortran/mpif-h/bindings.h"
-#include "ompi/mpi/fortran/base/constants.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_prototypes.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak PMPIX_ALLTOALLW_INIT = ompix_alltoallw_init_f
-#pragma weak pmpix_alltoallw_init = ompix_alltoallw_init_f
-#pragma weak pmpix_alltoallw_init_ = ompix_alltoallw_init_f
-#pragma weak pmpix_alltoallw_init__ = ompix_alltoallw_init_f
-
-#pragma weak PMPIX_Alltoallw_init_f = ompix_alltoallw_init_f
-#pragma weak PMPIX_Alltoallw_init_f08 = ompix_alltoallw_init_f
-#else
-OMPI_GENERATE_F77_BINDINGS (PMPIX_ALLTOALLW_INIT,
-                            pmpix_alltoallw_init,
-                            pmpix_alltoallw_init_,
-                            pmpix_alltoallw_init__,
-                            pompix_alltoallw_init_f,
-                            (char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *sdispls, MPI_Fint *sendtypes, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *rdispls, MPI_Fint *recvtypes, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, sendcounts, sdispls, sendtypes, recvbuf, recvcounts, rdispls, recvtypes, comm, info, request, ierr) )
-#endif
-#endif
-
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_ALLTOALLW_INIT = ompix_alltoallw_init_f
-#pragma weak mpix_alltoallw_init = ompix_alltoallw_init_f
-#pragma weak mpix_alltoallw_init_ = ompix_alltoallw_init_f
-#pragma weak mpix_alltoallw_init__ = ompix_alltoallw_init_f
-
-#pragma weak MPIX_Alltoallw_init_f = ompix_alltoallw_init_f
-#pragma weak MPIX_Alltoallw_init_f08 = ompix_alltoallw_init_f
-#else
-#if ! OMPI_BUILD_MPI_PROFILING
-OMPI_GENERATE_F77_BINDINGS (MPIX_ALLTOALLW_INIT,
-                            mpix_alltoallw_init,
-                            mpix_alltoallw_init_,
-                            mpix_alltoallw_init__,
-                            ompix_alltoallw_init_f,
-                            (char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *sdispls, MPI_Fint *sendtypes, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *rdispls, MPI_Fint *recvtypes, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, sendcounts, sdispls, sendtypes, recvbuf, recvcounts, rdispls, recvtypes, comm, info, request, ierr) )
-#else
-#define ompix_alltoallw_init_f pompix_alltoallw_init_f
-#endif
-#endif
-
-
-void ompix_alltoallw_init_f(char *sendbuf, MPI_Fint *sendcounts,
-                            MPI_Fint *sdispls, MPI_Fint *sendtypes,
-                            char *recvbuf, MPI_Fint *recvcounts,
-                            MPI_Fint *rdispls, MPI_Fint *recvtypes,
-                            MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
-{
-    MPI_Comm c_comm;
-    MPI_Datatype *c_sendtypes = NULL, *c_recvtypes;
-    MPI_Info c_info;
-    MPI_Request c_request;
-    int size, c_ierr;
-    OMPI_ARRAY_NAME_DECL(sendcounts);
-    OMPI_ARRAY_NAME_DECL(sdispls);
-    OMPI_ARRAY_NAME_DECL(recvcounts);
-    OMPI_ARRAY_NAME_DECL(rdispls);
-
-    c_comm = PMPI_Comm_f2c(*comm);
-    c_info = PMPI_Info_f2c(*info);
-    size = OMPI_COMM_IS_INTER(c_comm)?ompi_comm_remote_size(c_comm):ompi_comm_size(c_comm);
-
-    if (!OMPI_IS_FORTRAN_IN_PLACE(sendbuf)) {
-        c_sendtypes = (MPI_Datatype *) malloc(size * sizeof(MPI_Datatype));
-        OMPI_ARRAY_FINT_2_INT(sendcounts, size);
-        OMPI_ARRAY_FINT_2_INT(sdispls, size);
-        for (int i=0; i<size; i++) {
-            c_sendtypes[i] = PMPI_Type_f2c(sendtypes[i]);
-        }
-    }
-
-    c_recvtypes = (MPI_Datatype *) malloc(size * sizeof(MPI_Datatype));
-    OMPI_ARRAY_FINT_2_INT(recvcounts, size);
-    OMPI_ARRAY_FINT_2_INT(rdispls, size);
-    for (int i=0; i<size; i++) {
-        c_recvtypes[i] = PMPI_Type_f2c(recvtypes[i]);
-    }
-
-    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
-    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
-    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
-
-    c_ierr = PMPIX_Alltoallw_init(sendbuf,
-                                  OMPI_ARRAY_NAME_CONVERT(sendcounts),
-                                  OMPI_ARRAY_NAME_CONVERT(sdispls),
-                                  c_sendtypes,
-                                  recvbuf,
-                                  OMPI_ARRAY_NAME_CONVERT(recvcounts),
-                                  OMPI_ARRAY_NAME_CONVERT(rdispls),
-                                  c_recvtypes, c_comm, c_info, &c_request);
-    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
-    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
-
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(sendcounts);
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(sdispls);
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(recvcounts);
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(rdispls);
-    if (NULL != c_sendtypes) {
-        free(c_sendtypes);
-    }
-    free(c_recvtypes);
-}
diff --git a/ompi/mpiext/pcollreq/mpif-h/barrier_init_f.c b/ompi/mpiext/pcollreq/mpif-h/barrier_init_f.c
deleted file mode 100644
index 3b6b6d828ce..00000000000
--- a/ompi/mpiext/pcollreq/mpif-h/barrier_init_f.c
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/fortran/mpif-h/bindings.h"
-#include "ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_prototypes.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak PMPIX_BARRIER_INIT = ompix_barrier_init_f
-#pragma weak pmpix_barrier_init = ompix_barrier_init_f
-#pragma weak pmpix_barrier_init_ = ompix_barrier_init_f
-#pragma weak pmpix_barrier_init__ = ompix_barrier_init_f
-
-#pragma weak PMPIX_Barrier_init_f = ompix_barrier_init_f
-#pragma weak PMPIX_Barrier_init_f08 = ompix_barrier_init_f
-#else
-OMPI_GENERATE_F77_BINDINGS (PMPIX_BARRIER_INIT,
-                            pmpix_barrier_init,
-                            pmpix_barrier_init_,
-                            pmpix_barrier_init__,
-                            pompix_barrier_init_f,
-                            (MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (comm, info, request, ierr) )
-#endif
-#endif
-
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_BARRIER_INIT = ompix_barrier_init_f
-#pragma weak mpix_barrier_init = ompix_barrier_init_f
-#pragma weak mpix_barrier_init_ = ompix_barrier_init_f
-#pragma weak mpix_barrier_init__ = ompix_barrier_init_f
-
-#pragma weak MPIX_Barrier_init_f = ompix_barrier_init_f
-#pragma weak MPIX_Barrier_init_f08 = ompix_barrier_init_f
-#else
-#if ! OMPI_BUILD_MPI_PROFILING
-OMPI_GENERATE_F77_BINDINGS (MPIX_BARRIER_INIT,
-                            mpix_barrier_init,
-                            mpix_barrier_init_,
-                            mpix_barrier_init__,
-                            ompix_barrier_init_f,
-                            (MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (comm, info, request, ierr) )
-#else
-#define ompix_barrier_init_f pompix_barrier_init_f
-#endif
-#endif
-
-
-void ompix_barrier_init_f(MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
-{
-    int ierr_c;
-    MPI_Comm c_comm;
-    MPI_Info c_info;
-    MPI_Request c_req;
-
-    c_comm = PMPI_Comm_f2c(*comm);
-    c_info = PMPI_Info_f2c(*info);
-
-    ierr_c = PMPIX_Barrier_init(c_comm, c_info, &c_req);
-    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(ierr_c);
-
-    if (MPI_SUCCESS == ierr_c) *request = PMPI_Request_c2f(c_req);
-}
diff --git a/ompi/mpiext/pcollreq/mpif-h/bcast_init_f.c b/ompi/mpiext/pcollreq/mpif-h/bcast_init_f.c
deleted file mode 100644
index f9385123a3a..00000000000
--- a/ompi/mpiext/pcollreq/mpif-h/bcast_init_f.c
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/fortran/mpif-h/bindings.h"
-#include "ompi/mpi/fortran/base/constants.h"
-#include "ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_prototypes.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak PMPIX_BCAST_INIT = ompix_bcast_init_f
-#pragma weak pmpix_bcast_init = ompix_bcast_init_f
-#pragma weak pmpix_bcast_init_ = ompix_bcast_init_f
-#pragma weak pmpix_bcast_init__ = ompix_bcast_init_f
-
-#pragma weak PMPIX_Bcast_init_f = ompix_bcast_init_f
-#pragma weak PMPIX_Bcast_init_f08 = ompix_bcast_init_f
-#else
-OMPI_GENERATE_F77_BINDINGS (PMPIX_BCAST_INIT,
-                            pmpix_bcast_init,
-                            pmpix_bcast_init_,
-                            pmpix_bcast_init__,
-                            pompix_bcast_init_f,
-                            (char *buffer, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (buffer, count, datatype, root, comm, info, request, ierr) )
-#endif
-#endif
-
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_BCAST_INIT = ompix_bcast_init_f
-#pragma weak mpix_bcast_init = ompix_bcast_init_f
-#pragma weak mpix_bcast_init_ = ompix_bcast_init_f
-#pragma weak mpix_bcast_init__ = ompix_bcast_init_f
-
-#pragma weak MPIX_Bcast_init_f = ompix_bcast_init_f
-#pragma weak MPIX_Bcast_init_f08 = ompix_bcast_init_f
-#else
-#if ! OMPI_BUILD_MPI_PROFILING
-OMPI_GENERATE_F77_BINDINGS (MPIX_BCAST_INIT,
-                            mpix_bcast_init,
-                            mpix_bcast_init_,
-                            mpix_bcast_init__,
-                            ompix_bcast_init_f,
-                            (char *buffer, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (buffer, count, datatype, root, comm, info, request, ierr) )
-#else
-#define ompix_bcast_init_f pompix_bcast_init_f
-#endif
-#endif
-
-
-void ompix_bcast_init_f(char *buffer, MPI_Fint *count, MPI_Fint *datatype,
-                   MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request,
-                   MPI_Fint *ierr)
-{
-    int c_ierr;
-    MPI_Comm c_comm;
-    MPI_Info c_info;
-    MPI_Request c_req;
-    MPI_Datatype c_type;
-
-    c_comm = PMPI_Comm_f2c(*comm);
-    c_type = PMPI_Type_f2c(*datatype);
-    c_info = PMPI_Info_f2c(*info);
-
-    c_ierr = PMPIX_Bcast_init(OMPI_F2C_BOTTOM(buffer),
-                              OMPI_FINT_2_INT(*count),
-                              c_type,
-                              OMPI_FINT_2_INT(*root),
-                              c_comm,
-                              c_info,
-                              &c_req);
-    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
-    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_req);
-}
diff --git a/ompi/mpiext/pcollreq/mpif-h/exscan_init_f.c b/ompi/mpiext/pcollreq/mpif-h/exscan_init_f.c
deleted file mode 100644
index d44594e7c36..00000000000
--- a/ompi/mpiext/pcollreq/mpif-h/exscan_init_f.c
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/fortran/mpif-h/bindings.h"
-#include "ompi/mpi/fortran/base/constants.h"
-#include "ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_prototypes.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak PMPIX_EXSCAN_INIT = ompix_exscan_init_f
-#pragma weak pmpix_exscan_init = ompix_exscan_init_f
-#pragma weak pmpix_exscan_init_ = ompix_exscan_init_f
-#pragma weak pmpix_exscan_init__ = ompix_exscan_init_f
-
-#pragma weak PMPIX_Exscan_init_f = ompix_exscan_init_f
-#pragma weak PMPIX_Exscan_init_f08 = ompix_exscan_init_f
-#else
-OMPI_GENERATE_F77_BINDINGS (PMPIX_EXSCAN_INIT,
-                            pmpix_exscan_init,
-                            pmpix_exscan_init_,
-                            pmpix_exscan_init__,
-                            pompix_exscan_init_f,
-                            (char *sendbuf, char *recvbuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, recvbuf, count, datatype, op, comm, info, request, ierr) )
-#endif
-#endif
-
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_EXSCAN_INIT = ompix_exscan_init_f
-#pragma weak mpix_exscan_init = ompix_exscan_init_f
-#pragma weak mpix_exscan_init_ = ompix_exscan_init_f
-#pragma weak mpix_exscan_init__ = ompix_exscan_init_f
-
-#pragma weak MPIX_Exscan_init_f = ompix_exscan_init_f
-#pragma weak MPIX_Exscan_init_f08 = ompix_exscan_init_f
-#else
-#if ! OMPI_BUILD_MPI_PROFILING
-OMPI_GENERATE_F77_BINDINGS (MPIX_EXSCAN_INIT,
-                            mpix_exscan_init,
-                            mpix_exscan_init_,
-                            mpix_exscan_init__,
-                            ompix_exscan_init_f,
-                            (char *sendbuf, char *recvbuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, recvbuf, count, datatype, op, comm, info, request, ierr) )
-#else
-#define ompix_exscan_init_f pompix_exscan_init_f
-#endif
-#endif
-
-
-void ompix_exscan_init_f(char *sendbuf, char *recvbuf, MPI_Fint *count,
-                         MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm,
-                         MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
-{
-    int c_ierr;
-    MPI_Comm c_comm;
-    MPI_Datatype c_type;
-    MPI_Info c_info;
-    MPI_Request c_request;
-    MPI_Op c_op;
-
-    c_comm = PMPI_Comm_f2c(*comm);
-    c_type = PMPI_Type_f2c(*datatype);
-    c_op = PMPI_Op_f2c(*op);
-    c_info = PMPI_Info_f2c(*info);
-
-    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
-    sendbuf = (char *) OMPI_F2C_BOTTOM (sendbuf);
-    recvbuf = (char *) OMPI_F2C_BOTTOM (recvbuf);
-
-    c_ierr = PMPIX_Exscan_init(sendbuf, recvbuf,
-                               OMPI_FINT_2_INT(*count),
-                               c_type, c_op, c_comm, c_info, &c_request);
-    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
-    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
-}
diff --git a/ompi/mpiext/pcollreq/mpif-h/gather_init_f.c b/ompi/mpiext/pcollreq/mpif-h/gather_init_f.c
deleted file mode 100644
index 87abd523926..00000000000
--- a/ompi/mpiext/pcollreq/mpif-h/gather_init_f.c
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/fortran/mpif-h/bindings.h"
-#include "ompi/mpi/fortran/base/constants.h"
-#include "ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_prototypes.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak PMPIX_GATHER_INIT = ompix_gather_init_f
-#pragma weak pmpix_gather_init = ompix_gather_init_f
-#pragma weak pmpix_gather_init_ = ompix_gather_init_f
-#pragma weak pmpix_gather_init__ = ompix_gather_init_f
-
-#pragma weak PMPIX_Gather_init_f = ompix_gather_init_f
-#pragma weak PMPIX_Gather_init_f08 = ompix_gather_init_f
-#else
-OMPI_GENERATE_F77_BINDINGS (PMPIX_GATHER_INIT,
-                            pmpix_gather_init,
-                            pmpix_gather_init_,
-                            pmpix_gather_init__,
-                            pompix_gather_init_f,
-                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, info, request, ierr) )
-#endif
-#endif
-
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_GATHER_INIT = ompix_gather_init_f
-#pragma weak mpix_gather_init = ompix_gather_init_f
-#pragma weak mpix_gather_init_ = ompix_gather_init_f
-#pragma weak mpix_gather_init__ = ompix_gather_init_f
-
-#pragma weak MPIX_Gather_init_f = ompix_gather_init_f
-#pragma weak MPIX_Gather_init_f08 = ompix_gather_init_f
-#else
-#if ! OMPI_BUILD_MPI_PROFILING
-OMPI_GENERATE_F77_BINDINGS (MPIX_GATHER_INIT,
-                            mpix_gather_init,
-                            mpix_gather_init_,
-                            mpix_gather_init__,
-                            ompix_gather_init_f,
-                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, info, request, ierr) )
-#else
-#define ompix_gather_init_f pompix_gather_init_f
-#endif
-#endif
-
-
-void ompix_gather_init_f(char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype,
-                    char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype,
-                    MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request,
-                    MPI_Fint *ierr)
-{
-    int c_ierr;
-    MPI_Comm c_comm;
-    MPI_Datatype c_sendtype, c_recvtype;
-    MPI_Info c_info;
-    MPI_Request c_request;
-
-    c_comm = PMPI_Comm_f2c(*comm);
-    c_sendtype = PMPI_Type_f2c(*sendtype);
-    c_recvtype = PMPI_Type_f2c(*recvtype);
-    c_info = PMPI_Info_f2c(*info);
-
-    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
-    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
-    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
-
-    c_ierr = PMPIX_Gather_init(sendbuf, OMPI_FINT_2_INT(*sendcount),
-                               c_sendtype, recvbuf,
-                               OMPI_FINT_2_INT(*recvcount),
-                               c_recvtype,
-                               OMPI_FINT_2_INT(*root),
-                               c_comm, c_info, &c_request);
-    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
-    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
-}
diff --git a/ompi/mpiext/pcollreq/mpif-h/gatherv_init_f.c b/ompi/mpiext/pcollreq/mpif-h/gatherv_init_f.c
deleted file mode 100644
index 96376b78ca0..00000000000
--- a/ompi/mpiext/pcollreq/mpif-h/gatherv_init_f.c
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/fortran/mpif-h/bindings.h"
-#include "ompi/mpi/fortran/base/constants.h"
-#include "ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_prototypes.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak PMPIX_GATHERV_INIT = ompix_gatherv_init_f
-#pragma weak pmpix_gatherv_init = ompix_gatherv_init_f
-#pragma weak pmpix_gatherv_init_ = ompix_gatherv_init_f
-#pragma weak pmpix_gatherv_init__ = ompix_gatherv_init_f
-
-#pragma weak PMPIX_Gatherv_init_f = ompix_gatherv_init_f
-#pragma weak PMPIX_Gatherv_init_f08 = ompix_gatherv_init_f
-#else
-OMPI_GENERATE_F77_BINDINGS (PMPIX_GATHERV_INIT,
-                            pmpix_gatherv_init,
-                            pmpix_gatherv_init_,
-                            pmpix_gatherv_init__,
-                            pompix_gatherv_init_f,
-                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *displs, MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, root, comm, info, request, ierr) )
-#endif
-#endif
-
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_GATHERV_INIT = ompix_gatherv_init_f
-#pragma weak mpix_gatherv_init = ompix_gatherv_init_f
-#pragma weak mpix_gatherv_init_ = ompix_gatherv_init_f
-#pragma weak mpix_gatherv_init__ = ompix_gatherv_init_f
-
-#pragma weak MPIX_Gatherv_init_f = ompix_gatherv_init_f
-#pragma weak MPIX_Gatherv_init_f08 = ompix_gatherv_init_f
-#else
-#if ! OMPI_BUILD_MPI_PROFILING
-OMPI_GENERATE_F77_BINDINGS (MPIX_GATHERV_INIT,
-                            mpix_gatherv_init,
-                            mpix_gatherv_init_,
-                            mpix_gatherv_init__,
-                            ompix_gatherv_init_f,
-                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *displs, MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request,MPI_Fint *ierr),
-                            (sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, root, comm, info, request, ierr) )
-#else
-#define ompix_gatherv_init_f pompix_gatherv_init_f
-#endif
-#endif
-
-
-void ompix_gatherv_init_f(char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype,
-                          char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *displs,
-                          MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm,
-                          MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
-{
-    MPI_Comm c_comm;
-    MPI_Datatype c_sendtype, c_recvtype;
-    MPI_Info c_info;
-    MPI_Request c_request;
-    int size, c_ierr;
-    OMPI_ARRAY_NAME_DECL(recvcounts);
-    OMPI_ARRAY_NAME_DECL(displs);
-
-    c_comm = PMPI_Comm_f2c(*comm);
-    c_sendtype = PMPI_Type_f2c(*sendtype);
-    c_recvtype = PMPI_Type_f2c(*recvtype);
-    c_info = PMPI_Info_f2c(*info);
-
-    PMPI_Comm_size(c_comm, &size);
-    OMPI_ARRAY_FINT_2_INT(recvcounts, size);
-    OMPI_ARRAY_FINT_2_INT(displs, size);
-
-    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
-    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
-    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
-
-    c_ierr = PMPIX_Gatherv_init(sendbuf, OMPI_FINT_2_INT(*sendcount),
-                         c_sendtype, recvbuf,
-                         OMPI_ARRAY_NAME_CONVERT(recvcounts),
-                         OMPI_ARRAY_NAME_CONVERT(displs),
-                         c_recvtype,
-                         OMPI_FINT_2_INT(*root),
-                          c_comm, c_info, &c_request);
-    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
-    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
-}
diff --git a/ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_mpifh.c b/ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_mpifh.c
deleted file mode 100644
index 36db262fad7..00000000000
--- a/ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_mpifh.c
+++ /dev/null
@@ -1,15 +0,0 @@
-/*
- * Copyright (c) 2018      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- *
- */
-
-void mpiext_pcollreq_mpifh_dummy(void);
-
-void mpiext_pcollreq_mpifh_dummy() {
-}
diff --git a/ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_mpifh.h b/ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_mpifh.h
deleted file mode 100644
index 95e9b644c6f..00000000000
--- a/ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_mpifh.h
+++ /dev/null
@@ -1,15 +0,0 @@
-! -*- fortran -*-
-!
-! Copyright (c) 2010-2011 Oak Ridge National Labs.  All rights reserved.
-! Copyright (c) 2012      Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-!
-! Additional copyrights may follow
-!
-! $HEADER$
-!
-
-! Since the OMPI mpif.h interface does not prototype subroutines, do not
-! declare any subroutines here.
diff --git a/ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_prototypes.h b/ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_prototypes.h
deleted file mode 100644
index 811a94b82e2..00000000000
--- a/ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_prototypes.h
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2013 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2015 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2011-2013 Inria.  All rights reserved.
- * Copyright (c) 2011-2013 Universite Bordeaux 1
- * Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2016-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- *
- * This file prototypes all MPI fortran functions in all four fortran
- * symbol conventions as well as all the internal real OMPI wrapper
- * functions (different from any of the four fortran symbol
- * conventions for clarity, at the cost of more typing for me...).
- * This file is included in the top-level build ONLY. The prototyping
- * is done ONLY for MPI_* bindings
- *
- * Zeroth, the OMPI wrapper functions, with a ompi_ prefix and _f
- * suffix.
- *
- * This is needed ONLY if the lower-level prototypes_pmpi.h has not
- * already been included.
- *
- * Note about function pointers: all function pointers are prototyped
- * here as (void*) rather than including the .h file that defines the
- * proper type (e.g., "op/op.h" defines ompi_op_fortran_handler_fn_t,
- * which is the function pointer type for fortran op callback
- * functions).  This is because there is no type checking coming in
- * from fortran, so why bother?  Also, including "op/op.h" (and
- * friends) makes the all the f77 bindings files dependant on these
- * files -- any change to any one of them will cause the recompilation
- * of the entire set of f77 bindings (ugh!).
- */
-
-#include "ompi_config.h"
-#include "ompi/errhandler/errhandler.h"
-#include "ompi/attribute/attribute.h"
-#include "ompi/op/op.h"
-#include "ompi/request/grequest.h"
-#include "ompi/mpi/fortran/base/datarep.h"
-
-#include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h"
-
-BEGIN_C_DECLS
-
-/* These are the prototypes for the "real" back-end fortran functions. */
-#define PN2(ret, mixed_name, lower_name, upper_name, args) \
-    /* Prototype the actual OMPI function */               \
-    OMPI_DECLSPEC ret o##lower_name##_f args;              \
-    /* Prototype the 4 versions of the MPI mpif.h name */  \
-    OMPI_DECLSPEC ret lower_name args;                     \
-    OMPI_DECLSPEC ret lower_name##_ args;                  \
-    OMPI_DECLSPEC ret lower_name##__ args;                 \
-    OMPI_DECLSPEC ret upper_name args;                     \
-    /* Prototype the use mpi/use mpi_f08 names  */         \
-    OMPI_DECLSPEC ret mixed_name##_f08 args;               \
-    OMPI_DECLSPEC ret mixed_name##_f args;                 \
-    /* Prototype the actual POMPI function */              \
-    OMPI_DECLSPEC ret po##lower_name##_f args;             \
-    /* Prototype the 4 versions of the PMPI mpif.h name */ \
-    OMPI_DECLSPEC ret p##lower_name args;                  \
-    OMPI_DECLSPEC ret p##lower_name##_ args;               \
-    OMPI_DECLSPEC ret p##lower_name##__ args;              \
-    OMPI_DECLSPEC ret P##upper_name args;                  \
-    /* Prototype the use mpi/use mpi_f08 PMPI names  */    \
-    OMPI_DECLSPEC ret P##mixed_name##_f08 args;            \
-    OMPI_DECLSPEC ret P##mixed_name##_f args
-
-PN2(void, MPIX_Allgather_init, mpix_allgather_init, MPIX_ALLGATHER_INIT, (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
-PN2(void, MPIX_Allgatherv_init, mpix_allgatherv_init, MPIX_ALLGATHERV_INIT, (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *displs, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
-END_C_DECLS
-PN2(void, MPIX_Allreduce_init, mpix_allreduce_init, MPIX_ALLREDUCE_INIT, (char *sendbuf, char *recvbuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
-PN2(void, MPIX_Alltoall_init, mpix_alltoall_init, MPIX_ALLTOALL_INIT, (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
-PN2(void, MPIX_Alltoallv_init, mpix_alltoallv_init, MPIX_ALLTOALLV_INIT, (char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *sdispls, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *rdispls, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
-PN2(void, MPIX_Alltoallw_init, mpix_alltoallw_init, MPIX_ALLTOALLW_INIT, (char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *sdispls, MPI_Fint *sendtypes, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *rdispls, MPI_Fint *recvtypes, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
-PN2(void, MPIX_Barrier_init, mpix_barrier_init, MPIX_BARRIER_INIT, (MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
-PN2(void, MPIX_Bcast_init, mpix_bcast_init, MPIX_BCAST_INIT, (char *buffer, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
-PN2(void, MPIX_Exscan_init, mpix_exscan_init, MPIX_EXSCAN_INIT, (char *sendbuf, char *recvbuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
-PN2(void, MPIX_Gather_init, mpix_gather_init, MPIX_GATHER_INIT, (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
-PN2(void, MPIX_Gatherv_init, mpix_gatherv_init, MPIX_GATHERV_INIT, (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *displs, MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
-PN2(void, MPIX_Reduce_init, mpix_reduce_init, MPIX_REDUCE_INIT, (char *sendbuf, char *recvbuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
-PN2(void, MPIX_Reduce_scatter_init, mpix_reduce_scatter_init, MPIX_REDUCE_SCATTER_INIT, (char *sendbuf, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
-PN2(void, MPIX_Reduce_scatter_block_init, mpix_reduce_scatter_block_init, MPIX_REDUCE_SCATTER_BLOCK_INIT, (char *sendbuf, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
-PN2(void, MPIX_Scan_init, mpix_scan_init, MPIX_SCAN_INIT, (char *sendbuf, char *recvbuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
-PN2(void, MPIX_Scatter_init, mpix_scatter_init, MPIX_SCATTER_INIT, (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
-PN2(void, MPIX_Scatterv_init, mpix_scatterv_init, MPIX_SCATTERV_INIT, (char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *displs, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
-PN2(void, MPIX_Neighbor_allgather_init, mpix_neighbor_allgather_init, MPIX_NEIGHBOR_ALLGATHER_INIT, (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
-PN2(void, MPIX_Neighbor_allgatherv_init, mpix_neighbor_allgatherv_init, MPIX_NEIGHBOR_ALLGATHERV_INIT, (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *displs, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
-PN2(void, MPIX_Neighbor_alltoall_init, mpix_neighbor_alltoall_init, MPIX_NEIGHBOR_ALLTOALL_INIT, (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
-PN2(void, MPIX_Neighbor_alltoallv_init, mpix_neighbor_alltoallv_init, MPIX_NEIGHBOR_ALLTOALLV_INIT, (char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *sdispls, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *rdispls, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
-PN2(void, MPIX_Neighbor_alltoallw_init, mpix_neighbor_alltoallw_init, MPIX_NEIGHBOR_ALLTOALLW_INIT, (char *sendbuf, MPI_Fint *sendcounts, MPI_Aint *sdispls, MPI_Fint *sendtypes, char *recvbuf, MPI_Fint *recvcounts, MPI_Aint *rdispls, MPI_Fint *recvtypes, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr));
diff --git a/ompi/mpiext/pcollreq/mpif-h/neighbor_allgather_init_f.c b/ompi/mpiext/pcollreq/mpif-h/neighbor_allgather_init_f.c
deleted file mode 100644
index 52ca6db627f..00000000000
--- a/ompi/mpiext/pcollreq/mpif-h/neighbor_allgather_init_f.c
+++ /dev/null
@@ -1,103 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/fortran/mpif-h/bindings.h"
-#include "ompi/mpi/fortran/base/constants.h"
-#include "ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_prototypes.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak PMPIX_NEIGHBOR_ALLGATHER_INIT = ompix_neighbor_allgather_init_f
-#pragma weak pmpix_neighbor_allgather_init = ompix_neighbor_allgather_init_f
-#pragma weak pmpix_neighbor_allgather_init_ = ompix_neighbor_allgather_init_f
-#pragma weak pmpix_neighbor_allgather_init__ = ompix_neighbor_allgather_init_f
-
-#pragma weak PMPIX_Neighbor_allgather_init_f = ompix_neighbor_allgather_init_f
-#pragma weak PMPIX_Neighbor_allgather_init_f08 = ompix_neighbor_allgather_init_f
-#else
-OMPI_GENERATE_F77_BINDINGS (PMPIX_NEIGHBOR_ALLGATHER_INIT,
-                            pmpix_neighbor_allgather_init,
-                            pmpix_neighbor_allgather_init_,
-                            pmpix_neighbor_allgather_init__,
-                            pompix_neighbor_allgather_init_f,
-                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, info, request, ierr) )
-#endif
-#endif
-
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_NEIGHBOR_ALLGATHER_INIT = ompix_neighbor_allgather_init_f
-#pragma weak mpix_neighbor_allgather_init = ompix_neighbor_allgather_init_f
-#pragma weak mpix_neighbor_allgather_init_ = ompix_neighbor_allgather_init_f
-#pragma weak mpix_neighbor_allgather_init__ = ompix_neighbor_allgather_init_f
-
-#pragma weak MPIX_Neighbor_allgather_init_f = ompix_neighbor_allgather_init_f
-#pragma weak MPIX_Neighbor_allgather_init_f08 = ompix_neighbor_allgather_init_f
-#else
-#if ! OMPI_BUILD_MPI_PROFILING
-OMPI_GENERATE_F77_BINDINGS (MPIX_NEIGHBOR_ALLGATHER_INIT,
-                            mpix_neighbor_allgather_init,
-                            mpix_neighbor_allgather_init_,
-                            mpix_neighbor_allgather_init__,
-                            ompix_neighbor_allgather_init_f,
-                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, info, request, ierr) )
-#else
-#define ompix_neighbor_allgather_init_f pompix_neighbor_allgather_init_f
-#endif
-#endif
-
-
-void ompix_neighbor_allgather_init_f(char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype,
-                                     char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype,
-                                     MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
-{
-    int ierr_c;
-    MPI_Comm c_comm;
-    MPI_Info c_info;
-    MPI_Request c_req;
-    MPI_Datatype c_sendtype, c_recvtype;
-
-    c_comm = PMPI_Comm_f2c(*comm);
-    c_sendtype = PMPI_Type_f2c(*sendtype);
-    c_recvtype = PMPI_Type_f2c(*recvtype);
-    c_info = PMPI_Info_f2c(*info);
-
-    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
-    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
-    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
-
-    ierr_c = PMPIX_Neighbor_allgather_init(sendbuf,
-                                           OMPI_FINT_2_INT(*sendcount),
-                                           c_sendtype,
-                                           recvbuf,
-                                           OMPI_FINT_2_INT(*recvcount),
-                                           c_recvtype, c_comm, c_info, &c_req);
-
-    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(ierr_c);
-
-    if (MPI_SUCCESS == ierr_c) *request = PMPI_Request_c2f(c_req);
-}
diff --git a/ompi/mpiext/pcollreq/mpif-h/neighbor_allgatherv_init_f.c b/ompi/mpiext/pcollreq/mpif-h/neighbor_allgatherv_init_f.c
deleted file mode 100644
index dae12203180..00000000000
--- a/ompi/mpiext/pcollreq/mpif-h/neighbor_allgatherv_init_f.c
+++ /dev/null
@@ -1,113 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/fortran/mpif-h/bindings.h"
-#include "ompi/mpi/fortran/base/constants.h"
-#include "ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_prototypes.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak PMPIX_NEIGHBOR_ALLGATHERV_INIT = ompix_neighbor_allgatherv_init_f
-#pragma weak pmpix_neighbor_allgatherv_init = ompix_neighbor_allgatherv_init_f
-#pragma weak pmpix_neighbor_allgatherv_init_ = ompix_neighbor_allgatherv_init_f
-#pragma weak pmpix_neighbor_allgatherv_init__ = ompix_neighbor_allgatherv_init_f
-
-#pragma weak PMPIX_Neighbor_allgatherv_init_f = ompix_neighbor_allgatherv_init_f
-#pragma weak PMPIX_Neighbor_allgatherv_init_f08 = ompix_neighbor_allgatherv_init_f
-#else
-OMPI_GENERATE_F77_BINDINGS (PMPIX_NEIGHBOR_ALLGATHERV_INIT,
-                            pmpix_neighbor_allgatherv_init,
-                            pmpix_neighbor_allgatherv_init_,
-                            pmpix_neighbor_allgatherv_init__,
-                            pompix_neighbor_allgatherv_init_f,
-                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *displs, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, comm, info, request, ierr) )
-#endif
-#endif
-
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_NEIGHBOR_ALLGATHERV_INIT = ompix_neighbor_allgatherv_init_f
-#pragma weak mpix_neighbor_allgatherv_init = ompix_neighbor_allgatherv_init_f
-#pragma weak mpix_neighbor_allgatherv_init_ = ompix_neighbor_allgatherv_init_f
-#pragma weak mpix_neighbor_allgatherv_init__ = ompix_neighbor_allgatherv_init_f
-
-#pragma weak MPIX_Neighbor_allgatherv_init_f = ompix_neighbor_allgatherv_init_f
-#pragma weak MPIX_Neighbor_allgatherv_init_f08 = ompix_neighbor_allgatherv_init_f
-#else
-#if ! OMPI_BUILD_MPI_PROFILING
-OMPI_GENERATE_F77_BINDINGS (MPIX_NEIGHBOR_ALLGATHERV_INIT,
-                            mpix_neighbor_allgatherv_init,
-                            mpix_neighbor_allgatherv_init_,
-                            mpix_neighbor_allgatherv_init__,
-                            ompix_neighbor_allgatherv_init_f,
-                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *displs, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, comm, info, request, ierr) )
-#else
-#define ompix_neighbor_allgatherv_init_f pompix_neighbor_allgatherv_init_f
-#endif
-#endif
-
-
-void ompix_neighbor_allgatherv_init_f(char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype,
-                                      char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *displs,
-                                      MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request,
-                                      MPI_Fint *ierr)
-{
-    MPI_Comm c_comm;
-    MPI_Datatype c_sendtype, c_recvtype;
-    MPI_Info c_info;
-    MPI_Request c_request;
-    int size, ierr_c;
-    OMPI_ARRAY_NAME_DECL(recvcounts);
-    OMPI_ARRAY_NAME_DECL(displs);
-
-    c_comm = PMPI_Comm_f2c(*comm);
-    c_sendtype = PMPI_Type_f2c(*sendtype);
-    c_recvtype = PMPI_Type_f2c(*recvtype);
-    c_info = PMPI_Info_f2c(*info);
-
-    PMPI_Comm_size(c_comm, &size);
-    OMPI_ARRAY_FINT_2_INT(recvcounts, size);
-    OMPI_ARRAY_FINT_2_INT(displs, size);
-
-    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
-    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
-    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
-
-    ierr_c = PMPIX_Neighbor_allgatherv_init(sendbuf,
-                                            OMPI_FINT_2_INT(*sendcount),
-                                            c_sendtype,
-                                            recvbuf,
-                                            OMPI_ARRAY_NAME_CONVERT(recvcounts),
-                                            OMPI_ARRAY_NAME_CONVERT(displs),
-                                            c_recvtype, c_comm, c_info, &c_request);
-
-    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(ierr_c);
-    if (MPI_SUCCESS == ierr_c) *request = PMPI_Request_c2f(c_request);
-
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(recvcounts);
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(displs);
-}
diff --git a/ompi/mpiext/pcollreq/mpif-h/neighbor_alltoall_init_f.c b/ompi/mpiext/pcollreq/mpif-h/neighbor_alltoall_init_f.c
deleted file mode 100644
index 561d96a55ea..00000000000
--- a/ompi/mpiext/pcollreq/mpif-h/neighbor_alltoall_init_f.c
+++ /dev/null
@@ -1,103 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/fortran/mpif-h/bindings.h"
-#include "ompi/mpi/fortran/base/constants.h"
-#include "ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_prototypes.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak PMPIX_NEIGHBOR_ALLTOALL_INIT = ompix_neighbor_alltoall_init_f
-#pragma weak pmpix_neighbor_alltoall_init = ompix_neighbor_alltoall_init_f
-#pragma weak pmpix_neighbor_alltoall_init_ = ompix_neighbor_alltoall_init_f
-#pragma weak pmpix_neighbor_alltoall_init__ = ompix_neighbor_alltoall_init_f
-
-#pragma weak PMPIX_Neighbor_alltoall_init_f = ompix_neighbor_alltoall_init_f
-#pragma weak PMPIX_Neighbor_alltoall_init_f08 = ompix_neighbor_alltoall_init_f
-#else
-OMPI_GENERATE_F77_BINDINGS (PMPIX_NEIGHBOR_ALLTOALL_INIT,
-                            pmpix_neighbor_alltoall_init,
-                            pmpix_neighbor_alltoall_init_,
-                            pmpix_neighbor_alltoall_init__,
-                            pompix_neighbor_alltoall_init_f,
-                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, info, request, ierr) )
-#endif
-#endif
-
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_NEIGHBOR_ALLTOALL_INIT = ompix_neighbor_alltoall_init_f
-#pragma weak mpix_neighbor_alltoall_init = ompix_neighbor_alltoall_init_f
-#pragma weak mpix_neighbor_alltoall_init_ = ompix_neighbor_alltoall_init_f
-#pragma weak mpix_neighbor_alltoall_init__ = ompix_neighbor_alltoall_init_f
-
-#pragma weak MPIX_Neighbor_alltoall_init_f = ompix_neighbor_alltoall_init_f
-#pragma weak MPIX_Neighbor_alltoall_init_f08 = ompix_neighbor_alltoall_init_f
-#else
-#if ! OMPI_BUILD_MPI_PROFILING
-OMPI_GENERATE_F77_BINDINGS (MPIX_NEIGHBOR_ALLTOALL_INIT,
-                            mpix_neighbor_alltoall_init,
-                            mpix_neighbor_alltoall_init_,
-                            mpix_neighbor_alltoall_init__,
-                            ompix_neighbor_alltoall_init_f,
-                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, info, request, ierr) )
-#else
-#define ompix_neighbor_alltoall_init_f pompix_neighbor_alltoall_init_f
-#endif
-#endif
-
-
-void ompix_neighbor_alltoall_init_f(char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype,
-                                    char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype,
-                                    MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
-{
-    int c_ierr;
-    MPI_Comm c_comm;
-    MPI_Info c_info;
-    MPI_Request c_req;
-    MPI_Datatype c_sendtype, c_recvtype;
-
-    c_comm = PMPI_Comm_f2c(*comm);
-    c_sendtype = PMPI_Type_f2c(*sendtype);
-    c_recvtype = PMPI_Type_f2c(*recvtype);
-    c_info = PMPI_Info_f2c(*info);
-
-    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
-    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
-    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
-
-    c_ierr = PMPIX_Neighbor_alltoall_init(sendbuf,
-                                          OMPI_FINT_2_INT(*sendcount),
-                                          c_sendtype,
-                                          recvbuf,
-                                          OMPI_FINT_2_INT(*recvcount),
-                                          c_recvtype, c_comm, c_info, &c_req);
-    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
-
-    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_req);
-}
diff --git a/ompi/mpiext/pcollreq/mpif-h/neighbor_alltoallv_init_f.c b/ompi/mpiext/pcollreq/mpif-h/neighbor_alltoallv_init_f.c
deleted file mode 100644
index b11679ac411..00000000000
--- a/ompi/mpiext/pcollreq/mpif-h/neighbor_alltoallv_init_f.c
+++ /dev/null
@@ -1,119 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/fortran/mpif-h/bindings.h"
-#include "ompi/mpi/fortran/base/constants.h"
-#include "ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_prototypes.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak PMPIX_NEIGHBOR_ALLTOALLV_INIT = ompix_neighbor_alltoallv_init_f
-#pragma weak pmpix_neighbor_alltoallv_init = ompix_neighbor_alltoallv_init_f
-#pragma weak pmpix_neighbor_alltoallv_init_ = ompix_neighbor_alltoallv_init_f
-#pragma weak pmpix_neighbor_alltoallv_init__ = ompix_neighbor_alltoallv_init_f
-
-#pragma weak PMPIX_Neighbor_alltoallv_init_f = ompix_neighbor_alltoallv_init_f
-#pragma weak PMPIX_Neighbor_alltoallv_init_f08 = ompix_neighbor_alltoallv_init_f
-#else
-OMPI_GENERATE_F77_BINDINGS (PMPIX_NEIGHBOR_ALLTOALLV_INIT,
-                            pmpix_neighbor_alltoallv_init,
-                            pmpix_neighbor_alltoallv_init_,
-                            pmpix_neighbor_alltoallv_init__,
-                            pompix_neighbor_alltoallv_init_f,
-                            (char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *sdispls, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *rdispls, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, sendcounts, sdispls, sendtype, recvbuf, recvcounts, rdispls, recvtype, comm, info, request, ierr) )
-#endif
-#endif
-
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_NEIGHBOR_ALLTOALLV_INIT = ompix_neighbor_alltoallv_init_f
-#pragma weak mpix_neighbor_alltoallv_init = ompix_neighbor_alltoallv_init_f
-#pragma weak mpix_neighbor_alltoallv_init_ = ompix_neighbor_alltoallv_init_f
-#pragma weak mpix_neighbor_alltoallv_init__ = ompix_neighbor_alltoallv_init_f
-
-#pragma weak MPIX_Neighbor_alltoallv_init_f = ompix_neighbor_alltoallv_init_f
-#pragma weak MPIX_Neighbor_alltoallv_init_f08 = ompix_neighbor_alltoallv_init_f
-#else
-#if ! OMPI_BUILD_MPI_PROFILING
-OMPI_GENERATE_F77_BINDINGS (MPIX_NEIGHBOR_ALLTOALLV_INIT,
-                            mpix_neighbor_alltoallv_init,
-                            mpix_neighbor_alltoallv_init_,
-                            mpix_neighbor_alltoallv_init__,
-                            ompix_neighbor_alltoallv_init_f,
-                            (char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *sdispls, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *rdispls, MPI_Fint *recvtype, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, sendcounts, sdispls, sendtype, recvbuf, recvcounts, rdispls, recvtype, comm, info, request, ierr) )
-#else
-#define ompix_neighbor_alltoallv_init_f pompix_neighbor_alltoallv_init_f
-#endif
-#endif
-
-
-void ompix_neighbor_alltoallv_init_f(char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *sdispls,
-                                     MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcounts,
-                                     MPI_Fint *rdispls, MPI_Fint *recvtype,
-                                     MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
-{
-    MPI_Comm c_comm;
-    MPI_Datatype c_sendtype, c_recvtype;
-    MPI_Info c_info;
-    MPI_Request c_request;
-    int size, c_ierr;
-    OMPI_ARRAY_NAME_DECL(sendcounts);
-    OMPI_ARRAY_NAME_DECL(sdispls);
-    OMPI_ARRAY_NAME_DECL(recvcounts);
-    OMPI_ARRAY_NAME_DECL(rdispls);
-
-    c_comm = PMPI_Comm_f2c(*comm);
-    c_sendtype = PMPI_Type_f2c(*sendtype);
-    c_recvtype = PMPI_Type_f2c(*recvtype);
-    c_info = PMPI_Info_f2c(*info);
-
-    PMPI_Comm_size(c_comm, &size);
-    OMPI_ARRAY_FINT_2_INT(sendcounts, size);
-    OMPI_ARRAY_FINT_2_INT(sdispls, size);
-    OMPI_ARRAY_FINT_2_INT(recvcounts, size);
-    OMPI_ARRAY_FINT_2_INT(rdispls, size);
-
-    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
-    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
-    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
-
-    c_ierr = PMPIX_Neighbor_alltoallv_init(sendbuf,
-                                           OMPI_ARRAY_NAME_CONVERT(sendcounts),
-                                           OMPI_ARRAY_NAME_CONVERT(sdispls),
-                                           c_sendtype,
-                                           recvbuf,
-                                           OMPI_ARRAY_NAME_CONVERT(recvcounts),
-                                           OMPI_ARRAY_NAME_CONVERT(rdispls),
-                                           c_recvtype, c_comm, c_info, &c_request);
-    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
-    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
-
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(sendcounts);
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(sdispls);
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(recvcounts);
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(rdispls);
-}
diff --git a/ompi/mpiext/pcollreq/mpif-h/neighbor_alltoallw_init_f.c b/ompi/mpiext/pcollreq/mpif-h/neighbor_alltoallw_init_f.c
deleted file mode 100644
index d7a0fbf6bf8..00000000000
--- a/ompi/mpiext/pcollreq/mpif-h/neighbor_alltoallw_init_f.c
+++ /dev/null
@@ -1,123 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/fortran/mpif-h/bindings.h"
-#include "ompi/mpi/fortran/base/constants.h"
-#include "ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_prototypes.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak PMPIX_NEIGHBOR_ALLTOALLW_INIT = ompix_neighbor_alltoallw_init_f
-#pragma weak pmpix_neighbor_alltoallw_init = ompix_neighbor_alltoallw_init_f
-#pragma weak pmpix_neighbor_alltoallw_init_ = ompix_neighbor_alltoallw_init_f
-#pragma weak pmpix_neighbor_alltoallw_init__ = ompix_neighbor_alltoallw_init_f
-
-#pragma weak PMPIX_Neighbor_alltoallw_init_f = ompix_neighbor_alltoallw_init_f
-#pragma weak PMPIX_Neighbor_alltoallw_init_f08 = ompix_neighbor_alltoallw_init_f
-#else
-OMPI_GENERATE_F77_BINDINGS (PMPIX_NEIGHBOR_ALLTOALLW_INIT,
-                            pmpix_neighbor_alltoallw_init,
-                            pmpix_neighbor_alltoallw_init_,
-                            pmpix_neighbor_alltoallw_init__,
-                            pompix_neighbor_alltoallw_init_f,
-                            (char *sendbuf, MPI_Fint *sendcounts, MPI_Aint *sdispls, MPI_Fint *sendtypes, char *recvbuf, MPI_Fint *recvcounts, MPI_Aint *rdispls, MPI_Fint *recvtypes, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, sendcounts, sdispls, sendtypes, recvbuf, recvcounts, rdispls, recvtypes, comm, info, request, ierr) )
-#endif
-#endif
-
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_NEIGHBOR_ALLTOALLW_INIT = ompix_neighbor_alltoallw_init_f
-#pragma weak mpix_neighbor_alltoallw_init = ompix_neighbor_alltoallw_init_f
-#pragma weak mpix_neighbor_alltoallw_init_ = ompix_neighbor_alltoallw_init_f
-#pragma weak mpix_neighbor_alltoallw_init__ = ompix_neighbor_alltoallw_init_f
-
-#pragma weak MPIX_Neighbor_alltoallw_init_f = ompix_neighbor_alltoallw_init_f
-#pragma weak MPIX_Neighbor_alltoallw_init_f08 = ompix_neighbor_alltoallw_init_f
-#else
-#if ! OMPI_BUILD_MPI_PROFILING
-OMPI_GENERATE_F77_BINDINGS (MPIX_NEIGHBOR_ALLTOALLW_INIT,
-                            mpix_neighbor_alltoallw_init,
-                            mpix_neighbor_alltoallw_init_,
-                            mpix_neighbor_alltoallw_init__,
-                            ompix_neighbor_alltoallw_init_f,
-                            (char *sendbuf, MPI_Fint *sendcounts, MPI_Aint *sdispls, MPI_Fint *sendtypes, char *recvbuf, MPI_Fint *recvcounts, MPI_Aint *rdispls, MPI_Fint *recvtypes, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, sendcounts, sdispls, sendtypes, recvbuf, recvcounts, rdispls, recvtypes, comm, info, request, ierr) )
-#else
-#define ompix_neighbor_alltoallw_init_f pompix_neighbor_alltoallw_init_f
-#endif
-#endif
-
-
-void ompix_neighbor_alltoallw_init_f(char *sendbuf, MPI_Fint *sendcounts,
-                                MPI_Aint *sdispls, MPI_Fint *sendtypes,
-                                char *recvbuf, MPI_Fint *recvcounts,
-                                MPI_Aint *rdispls, MPI_Fint *recvtypes,
-                                MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
-{
-    MPI_Comm c_comm;
-    MPI_Datatype *c_sendtypes, *c_recvtypes;
-    MPI_Info c_info;
-    MPI_Request c_request;
-    int size, c_ierr;
-    OMPI_ARRAY_NAME_DECL(sendcounts);
-    OMPI_ARRAY_NAME_DECL(recvcounts);
-
-    c_comm = PMPI_Comm_f2c(*comm);
-    c_info = PMPI_Info_f2c(*info);
-    PMPI_Comm_size(c_comm, &size);
-
-    c_sendtypes = (MPI_Datatype *) malloc(size * sizeof(MPI_Datatype));
-    c_recvtypes = (MPI_Datatype *) malloc(size * sizeof(MPI_Datatype));
-
-    OMPI_ARRAY_FINT_2_INT(sendcounts, size);
-    OMPI_ARRAY_FINT_2_INT(recvcounts, size);
-
-    while (size > 0) {
-        c_sendtypes[size - 1] = PMPI_Type_f2c(sendtypes[size - 1]);
-        c_recvtypes[size - 1] = PMPI_Type_f2c(recvtypes[size - 1]);
-        --size;
-    }
-
-    /* Neighbor_alltoallw_init does not support MPI_IN_PLACE */
-    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
-    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
-
-    c_ierr = PMPIX_Neighbor_alltoallw_init(sendbuf,
-                                           OMPI_ARRAY_NAME_CONVERT(sendcounts),
-                                           sdispls,
-                                           c_sendtypes,
-                                           recvbuf,
-                                           OMPI_ARRAY_NAME_CONVERT(recvcounts),
-                                           rdispls,
-                                           c_recvtypes, c_comm, c_info, &c_request);
-    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
-    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
-
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(sendcounts);
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(recvcounts);
-    free(c_sendtypes);
-    free(c_recvtypes);
-}
diff --git a/ompi/mpiext/pcollreq/mpif-h/profile/Makefile.am b/ompi/mpiext/pcollreq/mpif-h/profile/Makefile.am
deleted file mode 100644
index 235dd1ed5e5..00000000000
--- a/ompi/mpiext/pcollreq/mpif-h/profile/Makefile.am
+++ /dev/null
@@ -1,80 +0,0 @@
-#
-# Copyright (c) 2004-2009 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
-# Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2018      Research Organization for Information Science
-#                         and Technology (RIST).  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-# This file builds the mpif.h-based bindings for MPI extensions.  It
-# is optional in MPI extensions.
-
-# We must set these #defines so that the inner OMPI MPI prototype
-# header files do the Right Thing.
-AM_CPPFLAGS = -DOMPI_BUILD_MPI_PROFILING=1 -DOMPI_COMPILING_FORTRAN_WRAPPERS=1
-
-# Note that the mpif.h-based bindings are optional -- they can only be
-# built if OMPI is also building the Fortran-based bindings.  So we
-# initially set some Makefile macros to empty, and then conditionally
-# add to them later.
-noinst_LTLIBRARIES =
-
-# Directory where the header file to be included in mpif-ext.h must be
-# installed.
-ompidir = $(ompiincludedir)/ompi/mpiext/pcollreq/mpif-h
-
-# Just like noinst_LTLIBRARIES, set this macro to empty and
-# conditionally add to it later.
-ompi_HEADERS =
-
-# Use the Automake conditional to know if we're building the mpif.h
-# bindings.
-if OMPI_BUILD_FORTRAN_MPIFH_BINDINGS
-
-# If we are, build the convenience libtool library that will be
-# slurped up into libmpi_mpifh.la.
-noinst_LTLIBRARIES += libpmpiext_pcollreq_mpifh.la
-
-# Sources for the convenience libtool library.
-nodist_libpmpiext_pcollreq_mpifh_la_SOURCES = \
-        pallgather_init_f.c \
-        pallgatherv_init_f.c \
-        pallreduce_init_f.c \
-        palltoall_init_f.c \
-        palltoallv_init_f.c \
-        palltoallw_init_f.c \
-        pbarrier_init_f.c \
-        pbcast_init_f.c \
-        pexscan_init_f.c \
-        pgather_init_f.c \
-        pgatherv_init_f.c \
-        preduce_init_f.c \
-        preduce_scatter_init_f.c \
-        preduce_scatter_block_init_f.c \
-        pscan_init_f.c \
-        pscatter_init_f.c \
-        pscatterv_init_f.c \
-        \
-	pneighbor_allgather_init_f.c \
-        pneighbor_allgatherv_init_f.c \
-        pneighbor_alltoall_init_f.c \
-        pneighbor_alltoallv_init_f.c \
-        pneighbor_alltoallw_init_f.c
-
-#
-# Sym link in the sources from the real MPI directory
-#
-$(nodist_libpmpiext_pcollreq_mpifh_la_SOURCES):
-	$(OMPI_V_LN_S) if test ! -r $@ ; then \
-		pname=`echo $@ | cut -b '2-'` ; \
-		$(LN_S) $(top_srcdir)/ompi/mpiext/pcollreq/mpif-h/$$pname $@ ; \
-	fi
-
-endif
diff --git a/ompi/mpiext/pcollreq/mpif-h/reduce_init_f.c b/ompi/mpiext/pcollreq/mpif-h/reduce_init_f.c
deleted file mode 100644
index 2b3ed4c99c0..00000000000
--- a/ompi/mpiext/pcollreq/mpif-h/reduce_init_f.c
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/fortran/mpif-h/bindings.h"
-#include "ompi/mpi/fortran/base/constants.h"
-#include "ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_prototypes.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak PMPIX_REDUCE_INIT = ompix_reduce_init_f
-#pragma weak pmpix_reduce_init = ompix_reduce_init_f
-#pragma weak pmpix_reduce_init_ = ompix_reduce_init_f
-#pragma weak pmpix_reduce_init__ = ompix_reduce_init_f
-
-#pragma weak PMPIX_Reduce_init_f = ompix_reduce_init_f
-#pragma weak PMPIX_Reduce_init_f08 = ompix_reduce_init_f
-#else
-OMPI_GENERATE_F77_BINDINGS (PMPIX_REDUCE_INIT,
-                            pmpix_reduce_init,
-                            pmpix_reduce_init_,
-                            pmpix_reduce_init__,
-                            pompix_reduce_init_f,
-                            (char *sendbuf, char *recvbuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, recvbuf, count, datatype, op, root, comm, info, request, ierr) )
-#endif
-#endif
-
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_REDUCE_INIT = ompix_reduce_init_f
-#pragma weak mpix_reduce_init = ompix_reduce_init_f
-#pragma weak mpix_reduce_init_ = ompix_reduce_init_f
-#pragma weak mpix_reduce_init__ = ompix_reduce_init_f
-
-#pragma weak MPIX_Reduce_init_f = ompix_reduce_init_f
-#pragma weak MPIX_Reduce_init_f08 = ompix_reduce_init_f
-#else
-#if ! OMPI_BUILD_MPI_PROFILING
-OMPI_GENERATE_F77_BINDINGS (MPIX_REDUCE_INIT,
-                            mpix_reduce_init,
-                            mpix_reduce_init_,
-                            mpix_reduce_init__,
-                            ompix_reduce_init_f,
-                            (char *sendbuf, char *recvbuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, recvbuf, count, datatype, op, root, comm, info, request, ierr) )
-#else
-#define ompix_reduce_init_f pompix_reduce_init_f
-#endif
-#endif
-
-
-void ompix_reduce_init_f(char *sendbuf, char *recvbuf, MPI_Fint *count,
-                         MPI_Fint *datatype, MPI_Fint *op,
-                         MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request,
-                         MPI_Fint *ierr)
-{
-    int c_ierr;
-    MPI_Datatype c_type;
-    MPI_Info c_info;
-    MPI_Request c_request;
-    MPI_Op c_op;
-    MPI_Comm c_comm;
-
-    c_type = PMPI_Type_f2c(*datatype);
-    c_op = PMPI_Op_f2c(*op);
-    c_comm = PMPI_Comm_f2c(*comm);
-    c_info = PMPI_Info_f2c(*info);
-
-    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
-    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
-    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
-
-    c_ierr = PMPIX_Reduce_init(sendbuf, recvbuf,
-                         OMPI_FINT_2_INT(*count),
-                         c_type, c_op,
-                         OMPI_FINT_2_INT(*root),
-                         c_comm, c_info, &c_request);
-    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
-    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
-}
diff --git a/ompi/mpiext/pcollreq/mpif-h/reduce_scatter_block_init_f.c b/ompi/mpiext/pcollreq/mpif-h/reduce_scatter_block_init_f.c
deleted file mode 100644
index 953e0cb9771..00000000000
--- a/ompi/mpiext/pcollreq/mpif-h/reduce_scatter_block_init_f.c
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/fortran/mpif-h/bindings.h"
-#include "ompi/mpi/fortran/base/constants.h"
-#include "ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_prototypes.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak PMPIX_REDUCE_SCATTER_BLOCK_INIT = ompix_reduce_scatter_block_init_f
-#pragma weak pmpix_reduce_scatter_block_init = ompix_reduce_scatter_block_init_f
-#pragma weak pmpix_reduce_scatter_block_init_ = ompix_reduce_scatter_block_init_f
-#pragma weak pmpix_reduce_scatter_block_init__ = ompix_reduce_scatter_block_init_f
-
-#pragma weak PMPIX_Reduce_scatter_block_init_f = ompix_reduce_scatter_block_init_f
-#pragma weak PMPIX_Reduce_scatter_block_init_f08 = ompix_reduce_scatter_block_init_f
-#else
-OMPI_GENERATE_F77_BINDINGS (PMPIX_REDUCE_SCATTER_BLOCK_INIT,
-                            pmpix_reduce_scatter_block_init,
-                            pmpix_reduce_scatter_block_init_,
-                            pmpix_reduce_scatter_block_init__,
-                            pompix_reduce_scatter_block_init_f,
-                            (char *sendbuf, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, recvbuf, recvcounts, datatype, op, comm, info, request, ierr) )
-#endif
-#endif
-
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_REDUCE_SCATTER_BLOCK_INIT = ompix_reduce_scatter_block_init_f
-#pragma weak mpix_reduce_scatter_block_init = ompix_reduce_scatter_block_init_f
-#pragma weak mpix_reduce_scatter_block_init_ = ompix_reduce_scatter_block_init_f
-#pragma weak mpix_reduce_scatter_block_init__ = ompix_reduce_scatter_block_init_f
-
-#pragma weak MPIX_Reduce_scatter_block_init_f = ompix_reduce_scatter_block_init_f
-#pragma weak MPIX_Reduce_scatter_block_init_f08 = ompix_reduce_scatter_block_init_f
-#else
-#if ! OMPI_BUILD_MPI_PROFILING
-OMPI_GENERATE_F77_BINDINGS (MPIX_REDUCE_SCATTER_BLOCK_INIT,
-                            mpix_reduce_scatter_block_init,
-                            mpix_reduce_scatter_block_init_,
-                            mpix_reduce_scatter_block_init__,
-                            ompix_reduce_scatter_block_init_f,
-                            (char *sendbuf, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, recvbuf, recvcounts, datatype, op, comm, info, request, ierr) )
-#else
-#define ompix_reduce_scatter_block_init_f pompix_reduce_scatter_block_init_f
-#endif
-#endif
-
-
-void ompix_reduce_scatter_block_init_f(char *sendbuf, char *recvbuf,
-                                       MPI_Fint *recvcount, MPI_Fint *datatype,
-                                       MPI_Fint *op, MPI_Fint *comm,
-                                       MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
-{
-    int c_ierr;
-    MPI_Comm c_comm;
-    MPI_Datatype c_type;
-    MPI_Info c_info;
-    MPI_Request c_request;
-    MPI_Op c_op;
-    int size;
-
-    c_comm = PMPI_Comm_f2c(*comm);
-    c_type = PMPI_Type_f2c(*datatype);
-    c_op = PMPI_Op_f2c(*op);
-    c_info = PMPI_Info_f2c(*info);
-
-    PMPI_Comm_size(c_comm, &size);
-
-    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
-    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
-    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
-
-    c_ierr = PMPIX_Reduce_scatter_block_init(sendbuf, recvbuf,
-                                             OMPI_FINT_2_INT(*recvcount),
-                                             c_type, c_op, c_comm, c_info, &c_request);
-    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
-    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
-}
diff --git a/ompi/mpiext/pcollreq/mpif-h/reduce_scatter_init_f.c b/ompi/mpiext/pcollreq/mpif-h/reduce_scatter_init_f.c
deleted file mode 100644
index 1d9c619258f..00000000000
--- a/ompi/mpiext/pcollreq/mpif-h/reduce_scatter_init_f.c
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/fortran/mpif-h/bindings.h"
-#include "ompi/mpi/fortran/base/constants.h"
-#include "ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_prototypes.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak PMPIX_REDUCE_SCATTER_INIT = ompix_reduce_scatter_init_f
-#pragma weak pmpix_reduce_scatter_init = ompix_reduce_scatter_init_f
-#pragma weak pmpix_reduce_scatter_init_ = ompix_reduce_scatter_init_f
-#pragma weak pmpix_reduce_scatter_init__ = ompix_reduce_scatter_init_f
-
-#pragma weak PMPIX_Reduce_scatter_init_f = ompix_reduce_scatter_init_f
-#pragma weak PMPIX_Reduce_scatter_init_f08 = ompix_reduce_scatter_init_f
-#else
-OMPI_GENERATE_F77_BINDINGS (PMPIX_REDUCE_SCATTER_INIT,
-                            pmpix_reduce_scatter_init,
-                            pmpix_reduce_scatter_init_,
-                            pmpix_reduce_scatter_init__,
-                            pompix_reduce_scatter_init_f,
-                            (char *sendbuf, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, recvbuf, recvcounts, datatype, op, comm, info, request, ierr) )
-#endif
-#endif
-
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_REDUCE_SCATTER_INIT = ompix_reduce_scatter_init_f
-#pragma weak mpix_reduce_scatter_init = ompix_reduce_scatter_init_f
-#pragma weak mpix_reduce_scatter_init_ = ompix_reduce_scatter_init_f
-#pragma weak mpix_reduce_scatter_init__ = ompix_reduce_scatter_init_f
-
-#pragma weak MPIX_Reduce_scatter_init_f = ompix_reduce_scatter_init_f
-#pragma weak MPIX_Reduce_scatter_init_f08 = ompix_reduce_scatter_init_f
-#else
-#if ! OMPI_BUILD_MPI_PROFILING
-OMPI_GENERATE_F77_BINDINGS (MPIX_REDUCE_SCATTER_INIT,
-                            mpix_reduce_scatter_init,
-                            mpix_reduce_scatter_init_,
-                            mpix_reduce_scatter_init__,
-                            ompix_reduce_scatter_init_f,
-                            (char *sendbuf, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, recvbuf, recvcounts, datatype, op, comm, info, request, ierr) )
-#else
-#define ompix_reduce_scatter_init_f pompix_reduce_scatter_init_f
-#endif
-#endif
-
-
-void ompix_reduce_scatter_init_f(char *sendbuf, char *recvbuf,
-                                 MPI_Fint *recvcounts, MPI_Fint *datatype,
-                                 MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request,
-                                 MPI_Fint *ierr)
-{
-    int c_ierr;
-    MPI_Comm c_comm;
-    MPI_Datatype c_type;
-    MPI_Info c_info;
-    MPI_Request c_request;
-    MPI_Op c_op;
-    int size;
-    OMPI_ARRAY_NAME_DECL(recvcounts);
-
-    c_comm = PMPI_Comm_f2c(*comm);
-    c_type = PMPI_Type_f2c(*datatype);
-    c_op = PMPI_Op_f2c(*op);
-    c_info = PMPI_Info_f2c(*info);
-
-    PMPI_Comm_size(c_comm, &size);
-    OMPI_ARRAY_FINT_2_INT(recvcounts, size);
-
-    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
-    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
-    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
-
-    c_ierr = PMPIX_Reduce_scatter_init(sendbuf, recvbuf,
-                                       OMPI_ARRAY_NAME_CONVERT(recvcounts),
-                                       c_type, c_op, c_comm, c_info, &c_request);
-    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
-    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
-}
diff --git a/ompi/mpiext/pcollreq/mpif-h/scan_init_f.c b/ompi/mpiext/pcollreq/mpif-h/scan_init_f.c
deleted file mode 100644
index da0679d5d21..00000000000
--- a/ompi/mpiext/pcollreq/mpif-h/scan_init_f.c
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/fortran/mpif-h/bindings.h"
-#include "ompi/mpi/fortran/base/constants.h"
-#include "ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_prototypes.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak PMPIX_SCAN_INIT = ompix_scan_init_f
-#pragma weak pmpix_scan_init = ompix_scan_init_f
-#pragma weak pmpix_scan_init_ = ompix_scan_init_f
-#pragma weak pmpix_scan_init__ = ompix_scan_init_f
-
-#pragma weak PMPIX_Scan_init_f = ompix_scan_init_f
-#pragma weak PMPIX_Scan_init_f08 = ompix_scan_init_f
-#else
-OMPI_GENERATE_F77_BINDINGS (PMPIX_SCAN_INIT,
-                            pmpix_scan_init,
-                            pmpix_scan_init_,
-                            pmpix_scan_init__,
-                            pompix_scan_init_f,
-                            (char *sendbuf, char *recvbuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, recvbuf, count, datatype, op, comm, info, request, ierr) )
-#endif
-#endif
-
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_SCAN_INIT = ompix_scan_init_f
-#pragma weak mpix_scan_init = ompix_scan_init_f
-#pragma weak mpix_scan_init_ = ompix_scan_init_f
-#pragma weak mpix_scan_init__ = ompix_scan_init_f
-
-#pragma weak MPIX_Scan_init_f = ompix_scan_init_f
-#pragma weak MPIX_Scan_init_f08 = ompix_scan_init_f
-#else
-#if ! OMPI_BUILD_MPI_PROFILING
-OMPI_GENERATE_F77_BINDINGS (MPIX_SCAN_INIT,
-                            mpix_scan_init,
-                            mpix_scan_init_,
-                            mpix_scan_init__,
-                            ompix_scan_init_f,
-                            (char *sendbuf, char *recvbuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, recvbuf, count, datatype, op, comm, info, request, ierr) )
-#else
-#define ompix_scan_init_f pompix_scan_init_f
-#endif
-#endif
-
-
-void ompix_scan_init_f(char *sendbuf, char *recvbuf, MPI_Fint *count,
-                       MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm,
-                       MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
-{
-    int c_ierr;
-    MPI_Comm c_comm;
-    MPI_Datatype c_type;
-    MPI_Info c_info;
-    MPI_Request c_request;
-    MPI_Op c_op;
-
-    c_type = PMPI_Type_f2c(*datatype);
-    c_op = PMPI_Op_f2c(*op);
-    c_comm = PMPI_Comm_f2c(*comm);
-    c_info = PMPI_Info_f2c(*info);
-
-    sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf);
-    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
-    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
-
-    c_ierr = PMPIX_Scan_init(sendbuf, recvbuf,
-                            OMPI_FINT_2_INT(*count),
-                            c_type, c_op,
-                            c_comm, c_info, &c_request);
-    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
-    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
-}
diff --git a/ompi/mpiext/pcollreq/mpif-h/scatter_init_f.c b/ompi/mpiext/pcollreq/mpif-h/scatter_init_f.c
deleted file mode 100644
index ae48bac6929..00000000000
--- a/ompi/mpiext/pcollreq/mpif-h/scatter_init_f.c
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/fortran/mpif-h/bindings.h"
-#include "ompi/mpi/fortran/base/constants.h"
-#include "ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_prototypes.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak PMPIX_SCATTER_INIT = ompix_scatter_init_f
-#pragma weak pmpix_scatter_init = ompix_scatter_init_f
-#pragma weak pmpix_scatter_init_ = ompix_scatter_init_f
-#pragma weak pmpix_scatter_init__ = ompix_scatter_init_f
-
-#pragma weak PMPIX_Scatter_init_f = ompix_scatter_init_f
-#pragma weak PMPIX_Scatter_init_f08 = ompix_scatter_init_f
-#else
-OMPI_GENERATE_F77_BINDINGS (PMPIX_SCATTER_INIT,
-                            pmpix_scatter_init,
-                            pmpix_scatter_init_,
-                            pmpix_scatter_init__,
-                            pompix_scatter_init_f,
-                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, info, request, ierr) )
-#endif
-#endif
-
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_SCATTER_INIT = ompix_scatter_init_f
-#pragma weak mpix_scatter_init = ompix_scatter_init_f
-#pragma weak mpix_scatter_init_ = ompix_scatter_init_f
-#pragma weak mpix_scatter_init__ = ompix_scatter_init_f
-
-#pragma weak MPIX_Scatter_init_f = ompix_scatter_init_f
-#pragma weak MPIX_Scatter_init_f08 = ompix_scatter_init_f
-#else
-#if ! OMPI_BUILD_MPI_PROFILING
-OMPI_GENERATE_F77_BINDINGS (MPIX_SCATTER_INIT,
-                            mpix_scatter_init,
-                            mpix_scatter_init_,
-                            mpix_scatter_init__,
-                            ompix_scatter_init_f,
-                            (char *sendbuf, MPI_Fint *sendcount, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, info, request, ierr) )
-#else
-#define ompix_scatter_init_f pompix_scatter_init_f
-#endif
-#endif
-
-
-void ompix_scatter_init_f(char *sendbuf, MPI_Fint *sendcount,
-                          MPI_Fint *sendtype, char *recvbuf,
-                          MPI_Fint *recvcount, MPI_Fint *recvtype,
-                          MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request,
-                          MPI_Fint *ierr)
-{
-    int c_ierr;
-    MPI_Datatype c_sendtype, c_recvtype;
-    MPI_Info c_info;
-    MPI_Request c_request;
-    MPI_Comm c_comm = PMPI_Comm_f2c(*comm);
-
-    c_sendtype = PMPI_Type_f2c(*sendtype);
-    c_recvtype = PMPI_Type_f2c(*recvtype);
-    c_info = PMPI_Info_f2c(*info);
-
-    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
-    recvbuf = (char *) OMPI_F2C_IN_PLACE(recvbuf);
-    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
-
-    c_ierr = PMPIX_Scatter_init(sendbuf,OMPI_FINT_2_INT(*sendcount),
-                                c_sendtype, recvbuf,
-                                OMPI_FINT_2_INT(*recvcount),
-                                c_recvtype,
-                                OMPI_FINT_2_INT(*root), c_comm, c_info, &c_request);
-    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
-    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
-}
diff --git a/ompi/mpiext/pcollreq/mpif-h/scatterv_init_f.c b/ompi/mpiext/pcollreq/mpif-h/scatterv_init_f.c
deleted file mode 100644
index 84c535fac98..00000000000
--- a/ompi/mpiext/pcollreq/mpif-h/scatterv_init_f.c
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-
-#include "ompi/mpi/fortran/mpif-h/bindings.h"
-#include "ompi/mpi/fortran/base/constants.h"
-#include "ompi/mpiext/pcollreq/mpif-h/mpiext_pcollreq_prototypes.h"
-
-#if OMPI_BUILD_MPI_PROFILING
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak PMPIX_SCATTERV_INIT = ompix_scatterv_init_f
-#pragma weak pmpix_scatterv_init = ompix_scatterv_init_f
-#pragma weak pmpix_scatterv_init_ = ompix_scatterv_init_f
-#pragma weak pmpix_scatterv_init__ = ompix_scatterv_init_f
-
-#pragma weak PMPIX_Scatterv_init_f = ompix_scatterv_init_f
-#pragma weak PMPIX_Scatterv_init_f08 = ompix_scatterv_init_f
-#else
-OMPI_GENERATE_F77_BINDINGS (PMPIX_SCATTERV_INIT,
-                            pmpix_scatterv_init,
-                            pmpix_scatterv_init_,
-                            pmpix_scatterv_init__,
-                            pompix_scatterv_init_f,
-                            (char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *displs, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, sendcounts, displs, sendtype, recvbuf, recvcount, recvtype, root, comm, info, request, ierr) )
-#endif
-#endif
-
-#if OPAL_HAVE_WEAK_SYMBOLS
-#pragma weak MPIX_SCATTERV_INIT = ompix_scatterv_init_f
-#pragma weak mpix_scatterv_init = ompix_scatterv_init_f
-#pragma weak mpix_scatterv_init_ = ompix_scatterv_init_f
-#pragma weak mpix_scatterv_init__ = ompix_scatterv_init_f
-
-#pragma weak MPIX_Scatterv_init_f = ompix_scatterv_init_f
-#pragma weak MPIX_Scatterv_init_f08 = ompix_scatterv_init_f
-#else
-#if ! OMPI_BUILD_MPI_PROFILING
-OMPI_GENERATE_F77_BINDINGS (MPIX_SCATTERV_INIT,
-                            mpix_scatterv_init,
-                            mpix_scatterv_init_,
-                            mpix_scatterv_init__,
-                            ompix_scatterv_init_f,
-                            (char *sendbuf, MPI_Fint *sendcounts, MPI_Fint *displs, MPI_Fint *sendtype, char *recvbuf, MPI_Fint *recvcount, MPI_Fint *recvtype, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr),
-                            (sendbuf, sendcounts, displs, sendtype, recvbuf, recvcount, recvtype, root, comm, info, request, ierr) )
-#else
-#define ompix_scatterv_init_f pompix_scatterv_init_f
-#endif
-#endif
-
-
-void ompix_scatterv_init_f(char *sendbuf, MPI_Fint *sendcounts,
-                           MPI_Fint *displs, MPI_Fint *sendtype,
-                           char *recvbuf, MPI_Fint *recvcount,
-                           MPI_Fint *recvtype, MPI_Fint *root,
-                           MPI_Fint *comm, MPI_Fint *info, MPI_Fint *request, MPI_Fint *ierr)
-{
-    MPI_Comm c_comm;
-    MPI_Datatype c_sendtype, c_recvtype;
-    MPI_Info c_info;
-    MPI_Request c_request;
-    int size, c_ierr;
-    OMPI_ARRAY_NAME_DECL(sendcounts);
-    OMPI_ARRAY_NAME_DECL(displs);
-
-    c_comm = PMPI_Comm_f2c(*comm);
-    c_sendtype = PMPI_Type_f2c(*sendtype);
-    c_recvtype = PMPI_Type_f2c(*recvtype);
-    c_info = PMPI_Info_f2c(*info);
-
-    PMPI_Comm_size(c_comm, &size);
-    OMPI_ARRAY_FINT_2_INT(sendcounts, size);
-    OMPI_ARRAY_FINT_2_INT(displs, size);
-
-    sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf);
-    recvbuf = (char *) OMPI_F2C_IN_PLACE(recvbuf);
-    recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
-
-    c_ierr = PMPIX_Scatterv_init(sendbuf,
-                                OMPI_ARRAY_NAME_CONVERT(sendcounts),
-                                OMPI_ARRAY_NAME_CONVERT(displs),
-                                c_sendtype, recvbuf,
-                                OMPI_FINT_2_INT(*recvcount),
-                                c_recvtype,
-                                OMPI_FINT_2_INT(*root), c_comm, c_info, &c_request);
-    if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
-    if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
-
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(sendcounts);
-    OMPI_ARRAY_FINT_2_INT_CLEANUP(displs);
-}
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/Makefile.am b/ompi/mpiext/pcollreq/use-mpi-f08/Makefile.am
deleted file mode 100644
index b03a78e610f..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/Makefile.am
+++ /dev/null
@@ -1,115 +0,0 @@
-#
-# Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2017-2020 Research Organization for Information Science
-#                         and Technology (RIST).  All rights reserved.
-# Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-# This file builds the use_mpi_f08-based bindings for MPI extensions.  It
-# is optional in MPI extensions.
-
-# Note that Automake's Fortran-buidling rules uses CPPFLAGS and
-# AM_CPPFLAGS.  This can cause weirdness (e.g.,
-# https://github.com/open-mpi/ompi/issues/7253).  Let's just zero
-# those out and rely on AM_FCFLAGS.
-CPPFLAGS =
-AM_CPPFLAGS =
-
-# We must set these #defines and include paths so that the inner OMPI
-# MPI prototype header files do the Right Thing.
-AM_FCFLAGS = $(OMPI_FC_MODULE_FLAG)$(top_builddir)/ompi/mpi/fortran/use-mpi \
-             $(OMPI_FC_MODULE_FLAG)$(top_builddir)/ompi/mpi/fortran/use-mpi-f08/mod \
-             -I$(top_builddir) -I$(top_srcdir) $(FCFLAGS_f90)
-
-# Note that the mpi_f08-based bindings are optional -- they can only
-# be built if OMPI is also building the Fortran-based bindings.  So we
-# initially set some Makefile macros to empty, and then conditionally
-# add to them later.
-noinst_LTLIBRARIES =
-
-# Use the Automake conditional to know if we're building the mpif.h
-# bindings.
-if OMPI_BUILD_FORTRAN_USEMPIF08_BINDINGS
-
-# If we are, build the convenience libtool library that will be
-# slurped up into libmpi_usempif08.la.
-noinst_LTLIBRARIES += libmpiext_pcollreq_usempif08.la
-
-# Note that no header files are installed; instead,
-# mpiext_pcollreq_usempif08.h is automatically slurped up into the
-# mpi_f08_ext module.  It must be listed so that it is included in
-# dist tarballs.
-noinst_HEADERS = mpiext_pcollreq_usempif08.h
-
-mpi_api_files = \
-        allgather_init_f08.F90 \
-        allgatherv_init_f08.F90 \
-        allreduce_init_f08.F90 \
-        alltoall_init_f08.F90 \
-        alltoallv_init_f08.F90 \
-        alltoallw_init_f08.F90 \
-        barrier_init_f08.F90 \
-        bcast_init_f08.F90 \
-        exscan_init_f08.F90 \
-        gather_init_f08.F90 \
-        gatherv_init_f08.F90 \
-        reduce_init_f08.F90 \
-        reduce_scatter_block_init_f08.F90 \
-        reduce_scatter_init_f08.F90 \
-        scan_init_f08.F90 \
-        scatter_init_f08.F90 \
-        scatterv_init_f08.F90 \
-        \
-        neighbor_allgather_init_f08.F90 \
-        neighbor_allgatherv_init_f08.F90 \
-        neighbor_alltoall_init_f08.F90 \
-        neighbor_alltoallv_init_f08.F90 \
-        neighbor_alltoallw_init_f08.F90
-
-pmpi_api_files = \
-        profile/pallgather_init_f08.F90 \
-        profile/pallgatherv_init_f08.F90 \
-        profile/pallreduce_init_f08.F90 \
-        profile/palltoall_init_f08.F90 \
-        profile/palltoallv_init_f08.F90 \
-        profile/palltoallw_init_f08.F90 \
-        profile/pbarrier_init_f08.F90 \
-        profile/pbcast_init_f08.F90 \
-        profile/pexscan_init_f08.F90 \
-        profile/pgather_init_f08.F90 \
-        profile/pgatherv_init_f08.F90 \
-        profile/preduce_init_f08.F90 \
-        profile/preduce_scatter_block_init_f08.F90 \
-        profile/preduce_scatter_init_f08.F90 \
-        profile/pscan_init_f08.F90 \
-        profile/pscatter_init_f08.F90 \
-        profile/pscatterv_init_f08.F90 \
-        \
-        profile/pneighbor_allgather_init_f08.F90 \
-        profile/pneighbor_allgatherv_init_f08.F90 \
-        profile/pneighbor_alltoall_init_f08.F90 \
-        profile/pneighbor_alltoallv_init_f08.F90 \
-        profile/pneighbor_alltoallw_init_f08.F90
-
-mpi_api_lo_files = $(mpi_api_files:.F90=.lo)
-pmpi_api_lo_files = $(pmpi_api_files:.F90=.lo)
-
-$(mpi_api_lo_files): mpiext_pcollreq_f08.lo
-$(pmpi_api_lo_files): mpiext_pcollreq_f08.lo
-
-# Sources for the convenience libtool library.
-libmpiext_pcollreq_usempif08_la_SOURCES = \
-        mpiext_pcollreq_f08.F90 \
-        $(mpi_api_files) \
-        $(pmpi_api_files)
-
-# Remove the intermediate module file
-distclean-local:
-	rm -f mpiext_pcollreq_f08.mod
-
-endif
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/allgather_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/allgather_init_f08.F90
deleted file mode 100644
index f31cd539644..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/allgather_init_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine MPIX_Allgather_init_f08(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_allgather_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_allgather_init_f(sendbuf,sendcount,sendtype%MPI_VAL,&
-                               recvbuf,recvcount,recvtype%MPI_VAL,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPIX_Allgather_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/allgatherv_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/allgatherv_init_f08.F90
deleted file mode 100644
index db2f5eeafaf..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/allgatherv_init_f08.F90
+++ /dev/null
@@ -1,34 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine MPIX_Allgatherv_init_f08(sendbuf,sendcount,sendtype,recvbuf,recvcounts,&
-                                    displs,recvtype,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_allgatherv_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: recvcounts(*), displs(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_allgatherv_init_f(sendbuf,sendcount,sendtype%MPI_VAL,recvbuf,recvcounts,&
-                                displs,recvtype%MPI_VAL,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPIX_Allgatherv_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/allreduce_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/allreduce_init_f08.F90
deleted file mode 100644
index d31e21f88d0..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/allreduce_init_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine MPIX_Allreduce_init_f08(sendbuf,recvbuf,count,datatype,op,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_allreduce_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_allreduce_init_f(sendbuf,recvbuf,count,datatype%MPI_VAL,&
-                               op%MPI_VAL,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPIX_Allreduce_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/alltoall_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/alltoall_init_f08.F90
deleted file mode 100644
index 59728df95c7..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/alltoall_init_f08.F90
+++ /dev/null
@@ -1,33 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine MPIX_Alltoall_init_f08(sendbuf,sendcount,sendtype,recvbuf,&
-                                  recvcount,recvtype,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_alltoall_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_alltoall_init_f(sendbuf,sendcount,sendtype%MPI_VAL,recvbuf,&
-                              recvcount,recvtype%MPI_VAL,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPIX_Alltoall_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/alltoallv_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/alltoallv_init_f08.F90
deleted file mode 100644
index 133536c59ca..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/alltoallv_init_f08.F90
+++ /dev/null
@@ -1,33 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine MPIX_Alltoallv_init_f08(sendbuf,sendcounts,sdispls,sendtype,recvbuf,&
-                                   recvcounts,rdispls,recvtype,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_alltoallv_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_alltoallv_init_f(sendbuf,sendcounts,sdispls,sendtype%MPI_VAL,&
-                               recvbuf,recvcounts,rdispls,recvtype%MPI_VAL,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPIX_Alltoallv_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/alltoallw_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/alltoallw_init_f08.F90
deleted file mode 100644
index d1b91a5788c..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/alltoallw_init_f08.F90
+++ /dev/null
@@ -1,42 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2013 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine MPIX_Alltoallw_init_f08(sendbuf,sendcounts,sdispls,sendtypes,&
-                                   recvbuf,recvcounts,rdispls,recvtypes,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_alltoallw_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
-   TYPE(MPI_Datatype), INTENT(IN), ASYNCHRONOUS :: sendtypes(*), recvtypes(*)
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   ! Note that we pass a scalar here for both the sendtypes and
-   ! recvtypes arguments, even though the real Alltoallw function
-   ! expects an array of integers.  This is a hack: we know that
-   ! [send|recv]types(1)%MPI_VAL will pass the address of the first
-   ! integer in the array of Type(MPI_Datatype) derived types.  And
-   ! since Type(MPI_Datatype) are exactly memory-equivalent to a
-   ! single INTEGER, passing the address of the first one is the same
-   ! as passing the address to an array of integers.  To be clear: the
-   ! back-end ompi_alltoallw_f is expecting a pointer to an array of
-   ! integers.  So it all works out (but is a hack :-\ ).
-   call ompix_alltoallw_init_f(sendbuf,sendcounts,sdispls,sendtypes(1)%MPI_VAL,&
-                               recvbuf,recvcounts,rdispls,recvtypes(1)%MPI_VAL,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPIX_Alltoallw_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/barrier_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/barrier_init_f08.F90
deleted file mode 100644
index 538e6225125..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/barrier_init_f08.F90
+++ /dev/null
@@ -1,23 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-subroutine MPIX_Barrier_init_f08(comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_barrier_init_f
-   implicit none
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_barrier_init_f(comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPIX_Barrier_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/bcast_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/bcast_init_f08.F90
deleted file mode 100644
index 0a2b2bd9360..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/bcast_init_f08.F90
+++ /dev/null
@@ -1,29 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine MPIX_Bcast_init_f08(buffer,count,datatype,root,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_bcast_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: buffer
-   INTEGER, INTENT(IN) :: count, root
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_bcast_init_f(buffer,count,datatype%MPI_VAL,root,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPIX_Bcast_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/exscan_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/exscan_init_f08.F90
deleted file mode 100644
index ceefd482f52..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/exscan_init_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine MPIX_Exscan_init_f08(sendbuf,recvbuf,count,datatype,op,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_exscan_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_exscan_init_f(sendbuf,recvbuf,count,datatype%MPI_VAL,&
-                            op%MPI_VAL,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPIX_Exscan_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/gather_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/gather_init_f08.F90
deleted file mode 100644
index a5e93e9f1cc..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/gather_init_f08.F90
+++ /dev/null
@@ -1,33 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine MPIX_Gather_init_f08(sendbuf,sendcount,sendtype,recvbuf,recvcount,&
-                                recvtype,root,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_gather_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount, root
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_gather_init_f(sendbuf,sendcount,sendtype%MPI_VAL,recvbuf,recvcount,&
-                            recvtype%MPI_VAL,root,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPIX_Gather_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/gatherv_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/gatherv_init_f08.F90
deleted file mode 100644
index 411b557a231..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/gatherv_init_f08.F90
+++ /dev/null
@@ -1,34 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine MPIX_Gatherv_init_f08(sendbuf,sendcount,sendtype,recvbuf,recvcounts,&
-                                 displs,recvtype,root,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_gatherv_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, root
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: recvcounts(*), displs(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_gatherv_init_f(sendbuf,sendcount,sendtype%MPI_VAL,recvbuf,recvcounts,&
-                             displs,recvtype%MPI_VAL,root,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPIX_Gatherv_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/mpiext_pcollreq_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/mpiext_pcollreq_f08.F90
deleted file mode 100644
index f457c640f90..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/mpiext_pcollreq_f08.F90
+++ /dev/null
@@ -1,346 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-!                         University Research and Technology
-!                         Corporation.  All rights reserved.
-! Copyright (c) 2004-2005 The University of Tennessee and The University
-!                         of Tennessee Research Foundation.  All rights
-!                         reserved.
-! Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-!                         University of Stuttgart.  All rights reserved.
-! Copyright (c) 2004-2005 The Regents of the University of California.
-!                         All rights reserved.
-! Copyright (c) 2006-2014 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2016-2018 Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-!
-! Additional copyrights may follow
-!
-! $HEADER$
-!
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-module mpiext_pcollreq_f08
-
-  use mpi_f08_types
-  use mpi_f08_interfaces  ! this module contains the  mpi_f08 interface declarations
-  use pmpi_f08_interfaces ! this module contains the pmpi_f08 interface declarations
-  use mpi_f08_callbacks   ! this module contains the mpi_f08 attribute callback subroutines
-  use mpi_f08_interfaces_callbacks ! this module contains the mpi_f08 callback interfaces
-
-interface
-
-subroutine ompix_allgather_init_f(sendbuf,sendcount,sendtype,recvbuf, &
-                                  recvcount,recvtype,comm,info,request,ierror) &
-   BIND(C, name="ompix_allgather_init_f")
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf, recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount
-   INTEGER, INTENT(IN) :: sendtype
-   INTEGER, INTENT(IN) :: recvtype
-   INTEGER, INTENT(IN) :: comm
-   INTEGER, INTENT(IN) :: info
-   INTEGER, INTENT(OUT) :: request
-   INTEGER, INTENT(OUT) :: ierror
-end subroutine ompix_allgather_init_f
-
-subroutine ompix_allgatherv_init_f(sendbuf,sendcount,sendtype,recvbuf, &
-                                   recvcounts,displs,recvtype,comm,info,request,ierror) &
-   BIND(C, name="ompix_allgatherv_init_f")
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf, recvbuf
-   INTEGER, INTENT(IN) :: sendcount
-   INTEGER, INTENT(IN) :: recvcounts(*), displs(*)
-   INTEGER, INTENT(IN) :: sendtype
-   INTEGER, INTENT(IN) :: recvtype
-   INTEGER, INTENT(IN) :: comm
-   INTEGER, INTENT(IN) :: info
-   INTEGER, INTENT(OUT) :: request
-   INTEGER, INTENT(OUT) :: ierror
-end subroutine ompix_allgatherv_init_f
-
-subroutine ompix_allreduce_init_f(sendbuf,recvbuf,count,datatype,op,comm,info,request,ierror) &
-   BIND(C, name="ompix_allreduce_init_f")
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf, recvbuf
-   INTEGER, INTENT(IN) :: count
-   INTEGER, INTENT(IN) :: datatype
-   INTEGER, INTENT(IN) :: op
-   INTEGER, INTENT(IN) :: comm
-   INTEGER, INTENT(IN) :: info
-   INTEGER, INTENT(OUT) :: request
-   INTEGER, INTENT(OUT) :: ierror
-end subroutine ompix_allreduce_init_f
-
-subroutine ompix_alltoall_init_f(sendbuf,sendcount,sendtype,recvbuf, &
-                                 recvcount,recvtype,comm,info,request,ierror) &
-   BIND(C, name="ompix_alltoall_init_f")
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf, recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount
-   INTEGER, INTENT(IN) :: sendtype
-   INTEGER, INTENT(IN) :: recvtype
-   INTEGER, INTENT(IN) :: comm
-   INTEGER, INTENT(IN) :: info
-   INTEGER, INTENT(OUT) :: request
-   INTEGER, INTENT(OUT) :: ierror
-end subroutine ompix_alltoall_init_f
-
-subroutine ompix_alltoallv_init_f(sendbuf,sendcounts,sdispls,sendtype, &
-                                  recvbuf,recvcounts,rdispls,recvtype,comm,info,request,ierror) &
-   BIND(C, name="ompix_alltoallv_init_f")
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf, recvbuf
-   INTEGER, INTENT(IN) :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
-   INTEGER, INTENT(IN) :: sendtype
-   INTEGER, INTENT(IN) :: recvtype
-   INTEGER, INTENT(IN) :: comm
-   INTEGER, INTENT(IN) :: info
-   INTEGER, INTENT(OUT) :: request
-   INTEGER, INTENT(OUT) :: ierror
-end subroutine ompix_alltoallv_init_f
-
-subroutine ompix_alltoallw_init_f(sendbuf,sendcounts,sdispls,sendtypes, &
-                                  recvbuf,recvcounts,rdispls,recvtypes,comm,info,request,ierror) &
-   BIND(C, name="ompix_alltoallw_init_f")
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf, recvbuf
-   INTEGER, INTENT(IN) :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
-   INTEGER, INTENT(IN) :: sendtypes
-   INTEGER, INTENT(IN) :: recvtypes
-   INTEGER, INTENT(IN) :: comm
-   INTEGER, INTENT(IN) :: info
-   INTEGER, INTENT(OUT) :: request
-   INTEGER, INTENT(OUT) :: ierror
-end subroutine ompix_alltoallw_init_f
-
-subroutine ompix_barrier_init_f(comm,info,request,ierror) &
-   BIND(C, name="ompix_barrier_init_f")
-   implicit none
-   INTEGER, INTENT(IN) :: comm
-   INTEGER, INTENT(IN) :: info
-   INTEGER, INTENT(OUT) :: request
-   INTEGER, INTENT(OUT) :: ierror
-end subroutine ompix_barrier_init_f
-
-subroutine ompix_bcast_init_f(buffer,count,datatype,root,comm,info,request,ierror) &
-   BIND(C, name="ompix_bcast_init_f")
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: buffer
-   INTEGER, INTENT(IN) :: count, root
-   INTEGER, INTENT(IN) :: datatype
-   INTEGER, INTENT(IN) :: comm
-   INTEGER, INTENT(IN) :: info
-   INTEGER, INTENT(OUT) :: request
-   INTEGER, INTENT(OUT) :: ierror
-end subroutine ompix_bcast_init_f
-
-subroutine ompix_exscan_init_f(sendbuf,recvbuf,count,datatype,op,comm,info,request,ierror) &
-   BIND(C, name="ompix_exscan_init_f")
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf, recvbuf
-   INTEGER, INTENT(IN) :: count
-   INTEGER, INTENT(IN) :: datatype
-   INTEGER, INTENT(IN) :: op
-   INTEGER, INTENT(IN) :: comm
-   INTEGER, INTENT(IN) :: info
-   INTEGER, INTENT(OUT) :: request
-   INTEGER, INTENT(OUT) :: ierror
-end subroutine ompix_exscan_init_f
-
-subroutine ompix_gather_init_f(sendbuf,sendcount,sendtype,recvbuf, &
-                               recvcount,recvtype,root,comm,info,request,ierror) &
-   BIND(C, name="ompix_gather_init_f")
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf, recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount, root
-   INTEGER, INTENT(IN) :: sendtype
-   INTEGER, INTENT(IN) :: recvtype
-   INTEGER, INTENT(IN) :: comm
-   INTEGER, INTENT(IN) :: info
-   INTEGER, INTENT(OUT) :: request
-   INTEGER, INTENT(OUT) :: ierror
-end subroutine ompix_gather_init_f
-
-subroutine ompix_gatherv_init_f(sendbuf,sendcount,sendtype,recvbuf, &
-                                recvcounts,displs,recvtype,root,comm,info,request,ierror) &
-   BIND(C, name="ompix_gatherv_init_f")
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf, recvbuf
-   INTEGER, INTENT(IN) :: sendcount, root
-   INTEGER, INTENT(IN) :: recvcounts(*), displs(*)
-   INTEGER, INTENT(IN) :: sendtype
-   INTEGER, INTENT(IN) :: recvtype
-   INTEGER, INTENT(IN) :: comm
-   INTEGER, INTENT(IN) :: info
-   INTEGER, INTENT(OUT) :: request
-   INTEGER, INTENT(OUT) :: ierror
-end subroutine ompix_gatherv_init_f
-
-subroutine ompix_reduce_init_f(sendbuf,recvbuf,count,datatype,op,root,comm,info,request,ierror) &
-   BIND(C, name="ompix_reduce_init_f")
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf, recvbuf
-   INTEGER, INTENT(IN) :: count, root
-   INTEGER, INTENT(IN) :: datatype
-   INTEGER, INTENT(IN) :: op
-   INTEGER, INTENT(IN) :: comm
-   INTEGER, INTENT(IN) :: info
-   INTEGER, INTENT(OUT) :: request
-   INTEGER, INTENT(OUT) :: ierror
-end subroutine ompix_reduce_init_f
-
-subroutine ompix_reduce_scatter_init_f(sendbuf,recvbuf,recvcounts, &
-                                       datatype,op,comm,info,request,ierror) &
-   BIND(C, name="ompix_reduce_scatter_init_f")
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf, recvbuf
-   INTEGER, INTENT(IN) :: recvcounts(*)
-   INTEGER, INTENT(IN) :: datatype
-   INTEGER, INTENT(IN) :: op
-   INTEGER, INTENT(IN) :: comm
-   INTEGER, INTENT(IN) :: info
-   INTEGER, INTENT(OUT) :: request
-   INTEGER, INTENT(OUT) :: ierror
-end subroutine ompix_reduce_scatter_init_f
-
-subroutine ompix_reduce_scatter_block_init_f(sendbuf,recvbuf,recvcount, &
-                                             datatype,op,comm,info,request,ierror) &
-   BIND(C, name="ompix_reduce_scatter_block_init_f")
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf, recvbuf
-   INTEGER, INTENT(IN) :: recvcount
-   INTEGER, INTENT(IN) :: datatype
-   INTEGER, INTENT(IN) :: op
-   INTEGER, INTENT(IN) :: comm
-   INTEGER, INTENT(IN) :: info
-   INTEGER, INTENT(OUT) :: request
-   INTEGER, INTENT(OUT) :: ierror
-end subroutine ompix_reduce_scatter_block_init_f
-
-subroutine ompix_scan_init_f(sendbuf,recvbuf,count,datatype,op,comm,info,request,ierror) &
-   BIND(C, name="ompix_scan_init_f")
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf, recvbuf
-   INTEGER, INTENT(IN) :: count
-   INTEGER, INTENT(IN) :: datatype
-   INTEGER, INTENT(IN) :: op
-   INTEGER, INTENT(IN) :: comm
-   INTEGER, INTENT(IN) :: info
-   INTEGER, INTENT(OUT) :: request
-   INTEGER, INTENT(OUT) :: ierror
-end subroutine ompix_scan_init_f
-
-subroutine ompix_scatter_init_f(sendbuf,sendcount,sendtype,recvbuf, &
-                                recvcount,recvtype,root,comm,info,request,ierror) &
-   BIND(C, name="ompix_scatter_init_f")
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf, recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount, root
-   INTEGER, INTENT(IN) :: sendtype
-   INTEGER, INTENT(IN) :: recvtype
-   INTEGER, INTENT(IN) :: comm
-   INTEGER, INTENT(IN) :: info
-   INTEGER, INTENT(OUT) :: request
-   INTEGER, INTENT(OUT) :: ierror
-end subroutine ompix_scatter_init_f
-
-subroutine ompix_scatterv_init_f(sendbuf,sendcounts,displs,sendtype, &
-                                 recvbuf,recvcount,recvtype,root,comm,info,request,ierror) &
-   BIND(C, name="ompix_scatterv_init_f")
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf, recvbuf
-   INTEGER, INTENT(IN) :: recvcount, root
-   INTEGER, INTENT(IN) :: sendcounts(*), displs(*)
-   INTEGER, INTENT(IN) :: sendtype
-   INTEGER, INTENT(IN) :: recvtype
-   INTEGER, INTENT(IN) :: comm
-   INTEGER, INTENT(IN) :: info
-   INTEGER, INTENT(OUT) :: request
-   INTEGER, INTENT(OUT) :: ierror
-end subroutine ompix_scatterv_init_f
-
-subroutine ompix_neighbor_allgather_init_f(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype, &
-                                           comm,info,request,ierror) &
-   BIND(C, name="ompix_neighbor_allgather_init_f")
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount
-   INTEGER, INTENT(IN) :: sendtype, recvtype
-   INTEGER, INTENT(IN) :: comm
-   INTEGER, INTENT(IN) :: info
-   INTEGER, INTENT(OUT) :: request
-   INTEGER, INTENT(OUT) :: ierror
-end subroutine ompix_neighbor_allgather_init_f
-
-subroutine ompix_neighbor_allgatherv_init_f(sendbuf,sendcount,sendtype,recvbuf,recvcounts,displs, &
-                                            recvtype,comm,info,request,ierror) &
-   BIND(C, name="ompix_neighbor_allgatherv_init_f")
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount
-   INTEGER, INTENT(IN) :: recvcounts(*), displs(*)
-   INTEGER, INTENT(IN) :: sendtype, recvtype
-   INTEGER, INTENT(IN) :: comm
-   INTEGER, INTENT(IN) :: info
-   INTEGER, INTENT(OUT) :: request
-   INTEGER, INTENT(OUT) :: ierror
-end subroutine ompix_neighbor_allgatherv_init_f
-
-subroutine ompix_neighbor_alltoall_init_f(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype, &
-                                          comm,info,request,ierror) &
-   BIND(C, name="ompix_neighbor_alltoall_init_f")
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount
-   INTEGER, INTENT(IN) :: sendtype, recvtype
-   INTEGER, INTENT(IN) :: comm
-   INTEGER, INTENT(IN) :: info
-   INTEGER, INTENT(OUT) :: request
-   INTEGER, INTENT(OUT) :: ierror
-end subroutine ompix_neighbor_alltoall_init_f
-
-subroutine ompix_neighbor_alltoallv_init_f(sendbuf,sendcounts,sdispls,sendtype,recvbuf,recvcounts, &
-                                           rdispls,recvtype,comm,info,request,ierror) &
-   BIND(C, name="ompix_neighbor_alltoallv_init_f")
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
-   INTEGER, INTENT(IN) :: sendtype, recvtype
-   INTEGER, INTENT(IN) :: comm
-   INTEGER, INTENT(IN) :: info
-   INTEGER, INTENT(OUT) :: request
-   INTEGER, INTENT(OUT) :: ierror
-end subroutine ompix_neighbor_alltoallv_init_f
-
-subroutine ompix_neighbor_alltoallw_init_f(sendbuf,sendcounts,sdispls,sendtypes,recvbuf,recvcounts, &
-                                           rdispls,recvtypes,comm,info,request,ierror) &
-  BIND(C, name="ompix_neighbor_alltoallw_init_f")
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Request, MPI_ADDRESS_KIND
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN) :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-   INTEGER, INTENT(IN) :: sendcounts(*), recvcounts(*)
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: sdispls(*), rdispls(*)
-   INTEGER, INTENT(IN) :: sendtypes, recvtypes
-   INTEGER, INTENT(IN) :: comm
-   INTEGER, INTENT(IN) :: info
-   INTEGER, INTENT(OUT) :: request
-   INTEGER, INTENT(OUT) :: ierror
-end subroutine ompix_neighbor_alltoallw_init_f
-
-end interface
-
-end module mpiext_pcollreq_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/mpiext_pcollreq_usempif08.h b/ompi/mpiext/pcollreq/use-mpi-f08/mpiext_pcollreq_usempif08.h
deleted file mode 100644
index 33e3556cce7..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/mpiext_pcollreq_usempif08.h
+++ /dev/null
@@ -1,986 +0,0 @@
-! -*- fortran -*-
-!
-! Copyright (c) 2012      Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST). All rights reserved.
-! $COPYRIGHT$
-!
-! Additional copyrights may follow
-!
-! $HEADER$
-!
-
-! This whole file will be included in the mpi_f08_ext module interface
-! section.  Note that the extension's mpif.h file will be included
-! first, so there's no need to re-define anything that's in there (e.g.,
-! OMPI_EXAMPLE_GLOBAL).
-
-! Declare any interfaces, subroutines, and global variables/constants
-! here.  Note that the mpiext_example_mpif.h will automatically be
-! included before this, so anything declared there does not need to be
-! replicated here.
-
-interface mpix_allgather_init
-    subroutine mpix_allgather_init_f08(sendbuf, sendcount, sendtype, &
-                                       recvbuf, recvcount, recvtype, &
-                                       comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in) :: sendcount, recvcount
-        type(mpi_datatype), intent(in) :: sendtype, recvtype
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine mpix_allgather_init_f08
-end interface mpix_allgather_init
-
-interface mpix_allgatherv_init
-    subroutine mpix_allgatherv_init_f08(sendbuf, sendcount, sendtype, &
-                                        recvbuf, recvcounts, displs, recvtype, &
-                                        comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in) :: sendcount
-        integer, intent(in), asynchronous :: recvcounts(*), displs(*)
-        type(mpi_datatype), intent(in) :: sendtype, recvtype
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine mpix_allgatherv_init_f08
-end interface mpix_allgatherv_init
-
-interface mpix_allreduce_init
-    subroutine mpix_allreduce_init_f08(sendbuf, recvbuf, count, &
-                                       datatype, op, &
-                                       comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_op, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in) :: count
-        type(mpi_datatype), intent(in) :: datatype
-        type(mpi_op), intent(in) :: op
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine mpix_allreduce_init_f08
-end interface mpix_allreduce_init
-
-interface mpix_alltoall_init
-    subroutine mpix_alltoall_init_f08(sendbuf, sendcount, sendtype, &
-                                      recvbuf, recvcount, recvtype, &
-                                      comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in) :: sendcount, recvcount
-        type(mpi_datatype), intent(in) :: sendtype, recvtype
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine mpix_alltoall_init_f08
-end interface mpix_alltoall_init
-
-interface mpix_alltoallv_init
-    subroutine mpix_alltoallv_init_f08(sendbuf, sendcounts, sdispls, sendtype, &
-                                       recvbuf, recvcounts, rdispls, recvtype, &
-                                       comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in), asynchronous :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
-        type(mpi_datatype), intent(in) :: sendtype, recvtype
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine mpix_alltoallv_init_f08
-end interface mpix_alltoallv_init
-
-interface mpix_alltoallw_init
-    subroutine mpix_alltoallw_init_f08(sendbuf, sendcounts, sdispls, sendtypes, &
-                                       recvbuf, recvcounts, rdispls, recvtypes, &
-                                       comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in), asynchronous :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
-        type(mpi_datatype), intent(in), asynchronous :: sendtypes(*), recvtypes(*)
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine mpix_alltoallw_init_f08
-end interface mpix_alltoallw_init
-
-interface mpix_barrier_init
-    subroutine mpix_barrier_init_f08(comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_comm, mpi_info, mpi_request
-        implicit none
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine mpix_barrier_init_f08
-end interface mpix_barrier_init
-
-interface mpix_bcast_init
-    subroutine mpix_bcast_init_f08(buffer, count, datatype, root, &
-                                   comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: buffer
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: buffer
-        !$PRAGMA IGNORE_TKR buffer
-        !DIR$ IGNORE_TKR buffer
-        !IBM* IGNORE_TKR buffer
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: buffer
-        integer, intent(in) :: count, root
-        type(mpi_datatype), intent(in) :: datatype
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine mpix_bcast_init_f08
-end interface mpix_bcast_init
-
-interface mpix_exscan_init
-    subroutine mpix_exscan_init_f08(sendbuf, recvbuf, count, &
-                                    datatype, op, &
-                                    comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_op, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in) :: count
-        type(mpi_datatype), intent(in) :: datatype
-        type(mpi_op), intent(in) :: op
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine mpix_exscan_init_f08
-end interface mpix_exscan_init
-
-interface mpix_gather_init
-    subroutine mpix_gather_init_f08(sendbuf, sendcount, sendtype, &
-                                    recvbuf, recvcount, recvtype, root, &
-                                    comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in) :: sendcount, recvcount, root
-        type(mpi_datatype), intent(in) :: sendtype, recvtype
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine mpix_gather_init_f08
-end interface mpix_gather_init
-
-interface mpix_gatherv_init
-    subroutine mpix_gatherv_init_f08(sendbuf, sendcount, sendtype, &
-                                     recvbuf, recvcounts, displs, recvtype, root, &
-                                     comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in) :: sendcount, root
-        integer, intent(in), asynchronous :: recvcounts(*), displs(*)
-        type(mpi_datatype), intent(in) :: sendtype, recvtype
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine mpix_gatherv_init_f08
-end interface mpix_gatherv_init
-
-interface mpix_reduce_init
-    subroutine mpix_reduce_init_f08(sendbuf, recvbuf, count, &
-                                    datatype, op, root, &
-                                    comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_op, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in) :: count, root
-        type(mpi_datatype), intent(in) :: datatype
-        type(mpi_op), intent(in) :: op
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine mpix_reduce_init_f08
-end interface mpix_reduce_init
-
-interface mpix_reduce_scatter_init
-    subroutine mpix_reduce_scatter_init_f08(sendbuf, recvbuf, recvcounts, &
-                                            datatype, op, &
-                                            comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_op, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in), asynchronous :: recvcounts(*)
-        type(mpi_datatype), intent(in) :: datatype
-        type(mpi_op), intent(in) :: op
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine mpix_reduce_scatter_init_f08
-end interface mpix_reduce_scatter_init
-
-interface mpix_reduce_scatter_block_init
-    subroutine mpix_reduce_scatter_block_init_f08(sendbuf, recvbuf, recvcount, &
-                                                  datatype, op, &
-                                                  comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_op, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in) :: recvcount
-        type(mpi_datatype), intent(in) :: datatype
-        type(mpi_op), intent(in) :: op
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine mpix_reduce_scatter_block_init_f08
-end interface mpix_reduce_scatter_block_init
-
-interface mpix_scan_init
-    subroutine mpix_scan_init_f08(sendbuf, recvbuf, count, &
-                                  datatype, op, &
-                                  comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_op, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in) :: count
-        type(mpi_datatype), intent(in) :: datatype
-        type(mpi_op), intent(in) :: op
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine mpix_scan_init_f08
-end interface mpix_scan_init
-
-interface mpix_scatter_init
-    subroutine mpix_scatter_init_f08(sendbuf, sendcount, sendtype, &
-                                     recvbuf, recvcount, recvtype, root, &
-                                     comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in) :: sendcount, recvcount, root
-        type(mpi_datatype), intent(in) :: sendtype, recvtype
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine mpix_scatter_init_f08
-end interface mpix_scatter_init
-
-interface mpix_scatterv_init
-    subroutine mpix_scatterv_init_f08(sendbuf, sendcounts, displs, sendtype, &
-                                      recvbuf, recvcount, recvtype, root, &
-                                      comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in) :: recvcount, root
-        integer, intent(in), asynchronous :: sendcounts(*), displs(*)
-        type(mpi_datatype), intent(in) :: sendtype, recvtype
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine mpix_scatterv_init_f08
-end interface mpix_scatterv_init
-
-interface mpix_neighbor_allgather_init
-    subroutine mpix_neighbor_allgather_init_f08(sendbuf, sendcount, sendtype, &
-                                                recvbuf, recvcount, recvtype, &
-                                                comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in) :: sendcount, recvcount
-        type(mpi_datatype), intent(in) :: sendtype, recvtype
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine mpix_neighbor_allgather_init_f08
-end interface mpix_neighbor_allgather_init
-
-interface mpix_neighbor_allgatherv_init
-    subroutine mpix_neighbor_allgatherv_init_f08(sendbuf, sendcount, sendtype, &
-                                                 recvbuf, recvcounts, displs, recvtype, &
-                                                 comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in) :: sendcount
-        integer, intent(in), asynchronous :: recvcounts(*), displs(*)
-        type(mpi_datatype), intent(in) :: sendtype, recvtype
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine mpix_neighbor_allgatherv_init_f08
-end interface mpix_neighbor_allgatherv_init
-
-interface mpix_neighbor_alltoall_init
-    subroutine mpix_neighbor_alltoall_init_f08(sendbuf, sendcount, sendtype, &
-                                               recvbuf, recvcount, recvtype, &
-                                               comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in) :: sendcount, recvcount
-        type(mpi_datatype), intent(in) :: sendtype, recvtype
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine mpix_neighbor_alltoall_init_f08
-end interface mpix_neighbor_alltoall_init
-
-interface mpix_neighbor_alltoallv_init
-    subroutine mpix_neighbor_alltoallv_init_f08(sendbuf, sendcounts, sdispls, sendtype, &
-                                                recvbuf, recvcounts, rdispls, recvtype, &
-                                                comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in), asynchronous :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
-        type(mpi_datatype), intent(in) :: sendtype, recvtype
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine mpix_neighbor_alltoallv_init_f08
-end interface mpix_neighbor_alltoallv_init
-
-interface mpix_neighbor_alltoallw_init
-    subroutine mpix_neighbor_alltoallw_init_f08(sendbuf, sendcounts, sdispls, sendtypes, &
-                                                recvbuf, recvcounts, rdispls, recvtypes, &
-                                                comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_address_kind, mpi_datatype, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in), asynchronous :: sendcounts(*), recvcounts(*)
-        integer(mpi_address_kind), intent(in), asynchronous :: sdispls(*), rdispls(*)
-        type(mpi_datatype), intent(in), asynchronous :: sendtypes(*), recvtypes(*)
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine mpix_neighbor_alltoallw_init_f08
-end interface mpix_neighbor_alltoallw_init
-
-interface pmpix_allgather_init
-    subroutine pmpix_allgather_init_f08(sendbuf, sendcount, sendtype, &
-                                        recvbuf, recvcount, recvtype, &
-                                        comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in) :: sendcount, recvcount
-        type(mpi_datatype), intent(in) :: sendtype, recvtype
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine pmpix_allgather_init_f08
-end interface pmpix_allgather_init
-
-interface pmpix_allgatherv_init
-    subroutine pmpix_allgatherv_init_f08(sendbuf, sendcount, sendtype, &
-                                         recvbuf, recvcounts, displs, recvtype, &
-                                         comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in) :: sendcount
-        integer, intent(in), asynchronous :: recvcounts(*), displs(*)
-        type(mpi_datatype), intent(in) :: sendtype, recvtype
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine pmpix_allgatherv_init_f08
-end interface pmpix_allgatherv_init
-
-interface pmpix_allreduce_init
-    subroutine pmpix_allreduce_init_f08(sendbuf, recvbuf, count, &
-                                        datatype, op, &
-                                        comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_op, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in) :: count
-        type(mpi_datatype), intent(in) :: datatype
-        type(mpi_op), intent(in) :: op
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine pmpix_allreduce_init_f08
-end interface pmpix_allreduce_init
-
-interface pmpix_alltoall_init
-    subroutine pmpix_alltoall_init_f08(sendbuf, sendcount, sendtype, &
-                                       recvbuf, recvcount, recvtype, &
-                                       comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in) :: sendcount, recvcount
-        type(mpi_datatype), intent(in) :: sendtype, recvtype
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine pmpix_alltoall_init_f08
-end interface pmpix_alltoall_init
-
-interface pmpix_alltoallv_init
-    subroutine pmpix_alltoallv_init_f08(sendbuf, sendcounts, sdispls, sendtype, &
-                                        recvbuf, recvcounts, rdispls, recvtype, &
-                                        comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in), asynchronous :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
-        type(mpi_datatype), intent(in) :: sendtype, recvtype
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine pmpix_alltoallv_init_f08
-end interface pmpix_alltoallv_init
-
-interface pmpix_alltoallw_init
-    subroutine pmpix_alltoallw_init_f08(sendbuf, sendcounts, sdispls, sendtypes, &
-                                        recvbuf, recvcounts, rdispls, recvtypes, &
-                                        comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in), asynchronous :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
-        type(mpi_datatype), intent(in), asynchronous :: sendtypes(*), recvtypes(*)
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine pmpix_alltoallw_init_f08
-end interface pmpix_alltoallw_init
-
-interface pmpix_barrier_init
-    subroutine pmpix_barrier_init_f08(comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_comm, mpi_info, mpi_request
-        implicit none
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine pmpix_barrier_init_f08
-end interface pmpix_barrier_init
-
-interface pmpix_bcast_init
-    subroutine pmpix_bcast_init_f08(buffer, count, datatype, root, &
-                                    comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: buffer
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: buffer
-        !$PRAGMA IGNORE_TKR buffer
-        !DIR$ IGNORE_TKR buffer
-        !IBM* IGNORE_TKR buffer
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: buffer
-        integer, intent(in) :: count, root
-        type(mpi_datatype), intent(in) :: datatype
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine pmpix_bcast_init_f08
-end interface pmpix_bcast_init
-
-interface pmpix_exscan_init
-    subroutine pmpix_exscan_init_f08(sendbuf, recvbuf, count, &
-                                     datatype, op, &
-                                     comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_op, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in) :: count
-        type(mpi_datatype), intent(in) :: datatype
-        type(mpi_op), intent(in) :: op
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine pmpix_exscan_init_f08
-end interface pmpix_exscan_init
-
-interface pmpix_gather_init
-    subroutine pmpix_gather_init_f08(sendbuf, sendcount, sendtype, &
-                                     recvbuf, recvcount, recvtype, root, &
-                                     comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in) :: sendcount, recvcount, root
-        type(mpi_datatype), intent(in) :: sendtype, recvtype
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine pmpix_gather_init_f08
-end interface pmpix_gather_init
-
-interface pmpix_gatherv_init
-    subroutine pmpix_gatherv_init_f08(sendbuf, sendcount, sendtype, &
-                                      recvbuf, recvcounts, displs, recvtype, root, &
-                                      comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in) :: sendcount, root
-        integer, intent(in) :: recvcounts(*), displs(*)
-        type(mpi_datatype), intent(in) :: sendtype, recvtype
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine pmpix_gatherv_init_f08
-end interface pmpix_gatherv_init
-
-interface pmpix_reduce_init
-    subroutine pmpix_reduce_init_f08(sendbuf, recvbuf, count, &
-                                     datatype, op, root, &
-                                     comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_op, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in) :: count, root
-        type(mpi_datatype), intent(in) :: datatype
-        type(mpi_op), intent(in) :: op
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine pmpix_reduce_init_f08
-end interface pmpix_reduce_init
-
-interface pmpix_reduce_scatter_init
-    subroutine pmpix_reduce_scatter_init_f08(sendbuf, recvbuf, recvcounts, &
-                                             datatype, op, &
-                                             comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_op, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in), asynchronous :: recvcounts(*)
-        type(mpi_datatype), intent(in) :: datatype
-        type(mpi_op), intent(in) :: op
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine pmpix_reduce_scatter_init_f08
-end interface pmpix_reduce_scatter_init
-
-interface pmpix_reduce_scatter_block_init
-    subroutine pmpix_reduce_scatter_block_init_f08(sendbuf, recvbuf, recvcount, &
-                                                   datatype, op, &
-                                                   comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_op, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in) :: recvcount
-        type(mpi_datatype), intent(in) :: datatype
-        type(mpi_op), intent(in) :: op
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine pmpix_reduce_scatter_block_init_f08
-end interface pmpix_reduce_scatter_block_init
-
-interface pmpix_scan_init
-    subroutine pmpix_scan_init_f08(sendbuf, recvbuf, count, &
-                                   datatype, op, &
-                                   comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_op, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in) :: count
-        type(mpi_datatype), intent(in) :: datatype
-        type(mpi_op), intent(in) :: op
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine pmpix_scan_init_f08
-end interface pmpix_scan_init
-
-interface pmpix_scatter_init
-    subroutine pmpix_scatter_init_f08(sendbuf, sendcount, sendtype, &
-                                      recvbuf, recvcount, recvtype, root, &
-                                      comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in) :: sendcount, recvcount, root
-        type(mpi_datatype), intent(in) :: sendtype, recvtype
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine pmpix_scatter_init_f08
-end interface pmpix_scatter_init
-
-interface pmpix_scatterv_init
-    subroutine pmpix_scatterv_init_f08(sendbuf, sendcounts, displs, sendtype, &
-                                       recvbuf, recvcount, recvtype, root, &
-                                       comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in) :: recvcount, root
-        integer, intent(in), asynchronous :: sendcounts(*), displs(*)
-        type(mpi_datatype), intent(in) :: sendtype, recvtype
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine pmpix_scatterv_init_f08
-end interface pmpix_scatterv_init
-
-interface pmpix_neighbor_allgather_init
-    subroutine pmpix_neighbor_allgather_init_f08(sendbuf, sendcount, sendtype, &
-                                                 recvbuf, recvcount, recvtype, &
-                                                 comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in) :: sendcount, recvcount
-        type(mpi_datatype), intent(in) :: sendtype, recvtype
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine pmpix_neighbor_allgather_init_f08
-end interface pmpix_neighbor_allgather_init
-
-interface pmpix_neighbor_allgatherv_init
-    subroutine pmpix_neighbor_allgatherv_init_f08(sendbuf, sendcount, sendtype, &
-                                                  recvbuf, recvcounts, displs, recvtype, &
-                                                  comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in) :: sendcount
-        integer, intent(in), asynchronous :: recvcounts(*), displs(*)
-        type(mpi_datatype), intent(in) :: sendtype, recvtype
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine pmpix_neighbor_allgatherv_init_f08
-end interface pmpix_neighbor_allgatherv_init
-
-interface pmpix_neighbor_alltoall_init
-    subroutine pmpix_neighbor_alltoall_init_f08(sendbuf, sendcount, sendtype, &
-                                                recvbuf, recvcount, recvtype, &
-                                                comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in) :: sendcount, recvcount
-        type(mpi_datatype), intent(in) :: sendtype, recvtype
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine pmpix_neighbor_alltoall_init_f08
-end interface pmpix_neighbor_alltoall_init
-
-interface pmpix_neighbor_alltoallv_init
-    subroutine pmpix_neighbor_alltoallv_init_f08(sendbuf, sendcounts, sdispls, sendtype, &
-                                                 recvbuf, recvcounts, rdispls, recvtype, &
-                                                 comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_datatype, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in), asynchronous :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
-        type(mpi_datatype), intent(in) :: sendtype, recvtype
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine pmpix_neighbor_alltoallv_init_f08
-end interface pmpix_neighbor_alltoallv_init
-
-interface pmpix_neighbor_alltoallw_init
-    subroutine pmpix_neighbor_alltoallw_init_f08(sendbuf, sendcounts, sdispls, sendtypes, &
-                                                 recvbuf, recvcounts, rdispls, recvtypes, &
-                                                 comm, info, request, ierror)
-        use :: mpi_f08_types, only : mpi_address_kind, mpi_datatype, mpi_comm, mpi_info, mpi_request
-        implicit none
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in), asynchronous :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, asynchronous :: recvbuf
-        integer, intent(in), asynchronous :: sendcounts(*), recvcounts(*)
-        integer(mpi_address_kind), intent(in), asynchronous :: sdispls(*), rdispls(*)
-        type(mpi_datatype), intent(in) :: sendtypes(*), recvtypes(*)
-        type(mpi_comm), intent(in) :: comm
-        type(mpi_info), intent(in) :: info
-        type(mpi_request), intent(out) :: request
-        integer, optional, intent(out) :: ierror
-    end subroutine pmpix_neighbor_alltoallw_init_f08
-end interface pmpix_neighbor_alltoallw_init
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/neighbor_allgather_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/neighbor_allgather_init_f08.F90
deleted file mode 100644
index 3beedf850b8..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/neighbor_allgather_init_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2013 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine MPIX_Neighbor_allgather_init_f08(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_neighbor_allgather_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_neighbor_allgather_init_f(sendbuf,sendcount,sendtype%MPI_VAL,&
-                                        recvbuf,recvcount,recvtype%MPI_VAL,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPIX_Neighbor_allgather_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/neighbor_allgatherv_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/neighbor_allgatherv_init_f08.F90
deleted file mode 100644
index 524f51cf484..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/neighbor_allgatherv_init_f08.F90
+++ /dev/null
@@ -1,34 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2013 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine MPIX_Neighbor_allgatherv_init_f08(sendbuf,sendcount,sendtype,recvbuf,recvcounts,&
-                                             displs,recvtype,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_neighbor_allgatherv_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: recvcounts(*), displs(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_neighbor_allgatherv_init_f(sendbuf,sendcount,sendtype%MPI_VAL,recvbuf,recvcounts,&
-                                         displs,recvtype%MPI_VAL,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPIX_Neighbor_allgatherv_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/neighbor_alltoall_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/neighbor_alltoall_init_f08.F90
deleted file mode 100644
index 8f79b893451..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/neighbor_alltoall_init_f08.F90
+++ /dev/null
@@ -1,33 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2013 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine MPIX_Neighbor_alltoall_init_f08(sendbuf,sendcount,sendtype,recvbuf,&
-                                           recvcount,recvtype,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_neighbor_alltoall_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_neighbor_alltoall_init_f(sendbuf,sendcount,sendtype%MPI_VAL,recvbuf,&
-                                       recvcount,recvtype%MPI_VAL,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPIX_Neighbor_alltoall_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/neighbor_alltoallv_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/neighbor_alltoallv_init_f08.F90
deleted file mode 100644
index c973d652f06..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/neighbor_alltoallv_init_f08.F90
+++ /dev/null
@@ -1,34 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2013 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine MPIX_Neighbor_alltoallv_init_f08(sendbuf,sendcounts,sdispls,sendtype,recvbuf,&
-                                            recvcounts,rdispls,recvtype,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_neighbor_alltoallv_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_neighbor_alltoallv_init_f(sendbuf,sendcounts,sdispls,sendtype%MPI_VAL,&
-                                        recvbuf,recvcounts,rdispls,recvtype%MPI_VAL,&
-                                        comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPIX_Neighbor_alltoallv_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/neighbor_alltoallw_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/neighbor_alltoallw_init_f08.F90
deleted file mode 100644
index 4ad00d1f2fb..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/neighbor_alltoallw_init_f08.F90
+++ /dev/null
@@ -1,34 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2013 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2013 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine MPIX_Neighbor_alltoallw_init_f08(sendbuf,sendcounts,sdispls,sendtypes,&
-                                            recvbuf,recvcounts,rdispls,recvtypes,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Info, MPI_Request, MPI_ADDRESS_KIND
-   use :: mpiext_pcollreq_f08, only : ompix_neighbor_alltoallw_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: sendcounts(*), recvcounts(*)
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN), ASYNCHRONOUS :: sdispls(*), rdispls(*)
-   TYPE(MPI_Datatype), INTENT(IN), ASYNCHRONOUS :: sendtypes(*)
-   TYPE(MPI_Datatype), INTENT(IN), ASYNCHRONOUS :: recvtypes(*)
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_neighbor_alltoallw_init_f(sendbuf,sendcounts,sdispls,sendtypes(1)%MPI_VAL,&
-                                        recvbuf,recvcounts,rdispls,recvtypes(1)%MPI_VAL,&
-                                        comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPIX_Neighbor_alltoallw_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pallgather_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/profile/pallgather_init_f08.F90
deleted file mode 100644
index d9e1e1ffb79..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pallgather_init_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine PMPIX_Allgather_init_f08(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_allgather_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_allgather_init_f(sendbuf,sendcount,sendtype%MPI_VAL,&
-                               recvbuf,recvcount,recvtype%MPI_VAL,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine PMPIX_Allgather_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pallgatherv_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/profile/pallgatherv_init_f08.F90
deleted file mode 100644
index d719fcfeb37..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pallgatherv_init_f08.F90
+++ /dev/null
@@ -1,34 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine PMPIX_Allgatherv_init_f08(sendbuf,sendcount,sendtype,recvbuf,recvcounts,&
-                                     displs,recvtype,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_allgatherv_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: recvcounts(*), displs(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_allgatherv_init_f(sendbuf,sendcount,sendtype%MPI_VAL,recvbuf,recvcounts,&
-                                displs,recvtype%MPI_VAL,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine PMPIX_Allgatherv_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pallreduce_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/profile/pallreduce_init_f08.F90
deleted file mode 100644
index e89a652c241..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pallreduce_init_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine PMPIX_Allreduce_init_f08(sendbuf,recvbuf,count,datatype,op,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_allreduce_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_allreduce_init_f(sendbuf,recvbuf,count,datatype%MPI_VAL,&
-                               op%MPI_VAL,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine PMPIX_Allreduce_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/profile/palltoall_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/profile/palltoall_init_f08.F90
deleted file mode 100644
index 6bf635c3bd4..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/profile/palltoall_init_f08.F90
+++ /dev/null
@@ -1,33 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine PMPIX_Alltoall_init_f08(sendbuf,sendcount,sendtype,recvbuf,&
-                                   recvcount,recvtype,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_alltoall_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_alltoall_init_f(sendbuf,sendcount,sendtype%MPI_VAL,recvbuf,&
-                              recvcount,recvtype%MPI_VAL,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine PMPIX_Alltoall_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/profile/palltoallv_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/profile/palltoallv_init_f08.F90
deleted file mode 100644
index 7b16e20d404..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/profile/palltoallv_init_f08.F90
+++ /dev/null
@@ -1,33 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine PMPIX_Alltoallv_init_f08(sendbuf,sendcounts,sdispls,sendtype,recvbuf,&
-                                    recvcounts,rdispls,recvtype,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_alltoallv_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_alltoallv_init_f(sendbuf,sendcounts,sdispls,sendtype%MPI_VAL,&
-                               recvbuf,recvcounts,rdispls,recvtype%MPI_VAL,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine PMPIX_Alltoallv_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/profile/palltoallw_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/profile/palltoallw_init_f08.F90
deleted file mode 100644
index f2d108e5e0c..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/profile/palltoallw_init_f08.F90
+++ /dev/null
@@ -1,42 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2013 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine PMPIX_Alltoallw_init_f08(sendbuf,sendcounts,sdispls,sendtypes,&
-                                    recvbuf,recvcounts,rdispls,recvtypes,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_alltoallw_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
-   TYPE(MPI_Datatype), INTENT(IN), ASYNCHRONOUS :: sendtypes(*), recvtypes(*)
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   ! Note that we pass a scalar here for both the sendtypes and
-   ! recvtypes arguments, even though the real Alltoallw function
-   ! expects an array of integers.  This is a hack: we know that
-   ! [send|recv]types(1)%MPI_VAL will pass the address of the first
-   ! integer in the array of Type(MPI_Datatype) derived types.  And
-   ! since Type(MPI_Datatype) are exactly memory-equivalent to a
-   ! single INTEGER, passing the address of the first one is the same
-   ! as passing the address to an array of integers.  To be clear: the
-   ! back-end ompi_alltoallw_f is expecting a pointer to an array of
-   ! integers.  So it all works out (but is a hack :-\ ).
-   call ompix_alltoallw_init_f(sendbuf,sendcounts,sdispls,sendtypes(1)%MPI_VAL,&
-                               recvbuf,recvcounts,rdispls,recvtypes(1)%MPI_VAL,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine PMPIX_Alltoallw_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pbarrier_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/profile/pbarrier_init_f08.F90
deleted file mode 100644
index 71b12c20e8d..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pbarrier_init_f08.F90
+++ /dev/null
@@ -1,23 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-subroutine PMPIX_Barrier_init_f08(comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_barrier_init_f
-   implicit none
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_barrier_init_f(comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine PMPIX_Barrier_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pbcast_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/profile/pbcast_init_f08.F90
deleted file mode 100644
index 8b0e83cb214..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pbcast_init_f08.F90
+++ /dev/null
@@ -1,29 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine PMPIX_Bcast_init_f08(buffer,count,datatype,root,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_bcast_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: buffer
-   INTEGER, INTENT(IN) :: count, root
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_bcast_init_f(buffer,count,datatype%MPI_VAL,root,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine PMPIX_Bcast_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pexscan_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/profile/pexscan_init_f08.F90
deleted file mode 100644
index 0feb7fb45aa..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pexscan_init_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine PMPIX_Exscan_init_f08(sendbuf,recvbuf,count,datatype,op,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_exscan_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_exscan_init_f(sendbuf,recvbuf,count,datatype%MPI_VAL,&
-                            op%MPI_VAL,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine PMPIX_Exscan_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pgather_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/profile/pgather_init_f08.F90
deleted file mode 100644
index 45b8ac796e2..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pgather_init_f08.F90
+++ /dev/null
@@ -1,33 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine PMPIX_Gather_init_f08(sendbuf,sendcount,sendtype,recvbuf,recvcount,&
-                                 recvtype,root,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_gather_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount, root
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_gather_init_f(sendbuf,sendcount,sendtype%MPI_VAL,recvbuf,recvcount,&
-                            recvtype%MPI_VAL,root,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine PMPIX_Gather_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pgatherv_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/profile/pgatherv_init_f08.F90
deleted file mode 100644
index 42c7ad89722..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pgatherv_init_f08.F90
+++ /dev/null
@@ -1,34 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine PMPIX_Gatherv_init_f08(sendbuf,sendcount,sendtype,recvbuf,recvcounts,&
-                                  displs,recvtype,root,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_gatherv_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, root
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: recvcounts(*), displs(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_gatherv_init_f(sendbuf,sendcount,sendtype%MPI_VAL,recvbuf,recvcounts,&
-                             displs,recvtype%MPI_VAL,root,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine PMPIX_Gatherv_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pneighbor_allgather_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/profile/pneighbor_allgather_init_f08.F90
deleted file mode 100644
index c8a9caa8de7..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pneighbor_allgather_init_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2013 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine PMPIX_Neighbor_allgather_init_f08(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_neighbor_allgather_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_neighbor_allgather_init_f(sendbuf,sendcount,sendtype%MPI_VAL,&
-                                        recvbuf,recvcount,recvtype%MPI_VAL,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine PMPIX_Neighbor_allgather_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pneighbor_allgatherv_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/profile/pneighbor_allgatherv_init_f08.F90
deleted file mode 100644
index 66ce3aefc15..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pneighbor_allgatherv_init_f08.F90
+++ /dev/null
@@ -1,34 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2013 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine PMPIX_Neighbor_allgatherv_init_f08(sendbuf,sendcount,sendtype,recvbuf,recvcounts,&
-                                              displs,recvtype,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_neighbor_allgatherv_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: recvcounts(*), displs(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_neighbor_allgatherv_init_f(sendbuf,sendcount,sendtype%MPI_VAL,recvbuf,recvcounts,&
-                                         displs,recvtype%MPI_VAL,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine PMPIX_Neighbor_allgatherv_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pneighbor_alltoall_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/profile/pneighbor_alltoall_init_f08.F90
deleted file mode 100644
index 76c7d045b30..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pneighbor_alltoall_init_f08.F90
+++ /dev/null
@@ -1,33 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2013 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine PMPIX_Neighbor_alltoall_init_f08(sendbuf,sendcount,sendtype,recvbuf,&
-                                            recvcount,recvtype,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_neighbor_alltoall_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_neighbor_alltoall_init_f(sendbuf,sendcount,sendtype%MPI_VAL,recvbuf,&
-                                       recvcount,recvtype%MPI_VAL,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine PMPIX_Neighbor_alltoall_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pneighbor_alltoallv_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/profile/pneighbor_alltoallv_init_f08.F90
deleted file mode 100644
index 47ca428d016..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pneighbor_alltoallv_init_f08.F90
+++ /dev/null
@@ -1,34 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2013 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine PMPIX_Neighbor_alltoallv_init_f08(sendbuf,sendcounts,sdispls,sendtype,recvbuf,&
-                                             recvcounts,rdispls,recvtype,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_neighbor_alltoallv_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: sendcounts(*), sdispls(*), recvcounts(*), rdispls(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_neighbor_alltoallv_init_f(sendbuf,sendcounts,sdispls,sendtype%MPI_VAL,&
-                                        recvbuf,recvcounts,rdispls,recvtype%MPI_VAL,&
-                                        comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine PMPIX_Neighbor_alltoallv_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pneighbor_alltoallw_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/profile/pneighbor_alltoallw_init_f08.F90
deleted file mode 100644
index bd323b00055..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pneighbor_alltoallw_init_f08.F90
+++ /dev/null
@@ -1,34 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2013 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2013 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine PMPIX_Neighbor_alltoallw_init_f08(sendbuf,sendcounts,sdispls,sendtypes,&
-                                             recvbuf,recvcounts,rdispls,recvtypes,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Info, MPI_Request, MPI_ADDRESS_KIND
-   use :: mpiext_pcollreq_f08, only : ompix_neighbor_alltoallw_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: sendcounts(*), recvcounts(*)
-   INTEGER(MPI_ADDRESS_KIND), INTENT(IN), ASYNCHRONOUS :: sdispls(*), rdispls(*)
-   TYPE(MPI_Datatype), INTENT(IN), ASYNCHRONOUS :: sendtypes(*)
-   TYPE(MPI_Datatype), INTENT(IN), ASYNCHRONOUS :: recvtypes(*)
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_neighbor_alltoallw_init_f(sendbuf,sendcounts,sdispls,sendtypes(1)%MPI_VAL,&
-                                        recvbuf,recvcounts,rdispls,recvtypes(1)%MPI_VAL,&
-                                        comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine PMPIX_Neighbor_alltoallw_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/profile/preduce_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/profile/preduce_init_f08.F90
deleted file mode 100644
index ab70eed49e0..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/profile/preduce_init_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine PMPIX_Reduce_init_f08(sendbuf,recvbuf,count,datatype,op,root,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_reduce_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: count, root
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_reduce_init_f(sendbuf,recvbuf,count,datatype%MPI_VAL,&
-                            op%MPI_VAL,root,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine PMPIX_Reduce_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/profile/preduce_scatter_block_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/profile/preduce_scatter_block_init_f08.F90
deleted file mode 100644
index eeb7ccac9ca..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/profile/preduce_scatter_block_init_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine PMPIX_Reduce_scatter_block_init_f08(sendbuf,recvbuf,recvcount,datatype,op,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_reduce_scatter_block_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: recvcount
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_reduce_scatter_block_init_f(sendbuf,recvbuf,recvcount,&
-                                          datatype%MPI_VAL,op%MPI_VAL,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine PMPIX_Reduce_scatter_block_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/profile/preduce_scatter_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/profile/preduce_scatter_init_f08.F90
deleted file mode 100644
index 791ffd4ac42..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/profile/preduce_scatter_init_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine PMPIX_Reduce_scatter_init_f08(sendbuf,recvbuf,recvcounts,datatype,op,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_reduce_scatter_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: recvcounts(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_reduce_scatter_init_f(sendbuf,recvbuf,recvcounts,datatype%MPI_VAL,&
-                                    op%MPI_VAL,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine PMPIX_Reduce_scatter_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pscan_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/profile/pscan_init_f08.F90
deleted file mode 100644
index 668f0425397..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pscan_init_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine PMPIX_Scan_init_f08(sendbuf,recvbuf,count,datatype,op,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_scan_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_scan_init_f(sendbuf,recvbuf,count,datatype%MPI_VAL,&
-                          op%MPI_VAL,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine PMPIX_Scan_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pscatter_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/profile/pscatter_init_f08.F90
deleted file mode 100644
index a02b807dd05..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pscatter_init_f08.F90
+++ /dev/null
@@ -1,33 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine PMPIX_Scatter_init_f08(sendbuf,sendcount,sendtype,recvbuf,&
-                                  recvcount,recvtype,root,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_scatter_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount, root
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_scatter_init_f(sendbuf,sendcount,sendtype%MPI_VAL,recvbuf,recvcount,&
-                             recvtype%MPI_VAL,root,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine PMPIX_Scatter_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pscatterv_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/profile/pscatterv_init_f08.F90
deleted file mode 100644
index 6d138816945..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/profile/pscatterv_init_f08.F90
+++ /dev/null
@@ -1,34 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine PMPIX_Scatterv_init_f08(sendbuf,sendcounts,displs,sendtype,recvbuf,&
-                                   recvcount,recvtype,root,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_scatterv_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: recvcount, root
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: sendcounts(*), displs(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_scatterv_init_f(sendbuf,sendcounts,displs,sendtype%MPI_VAL,recvbuf,&
-                              recvcount,recvtype%MPI_VAL,root,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine PMPIX_Scatterv_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/reduce_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/reduce_init_f08.F90
deleted file mode 100644
index 24493a1b512..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/reduce_init_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine MPIX_Reduce_init_f08(sendbuf,recvbuf,count,datatype,op,root,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_reduce_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: count, root
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_reduce_init_f(sendbuf,recvbuf,count,datatype%MPI_VAL,&
-                            op%MPI_VAL,root,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPIX_Reduce_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/reduce_scatter_block_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/reduce_scatter_block_init_f08.F90
deleted file mode 100644
index b9b27823ecd..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/reduce_scatter_block_init_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine MPIX_Reduce_scatter_block_init_f08(sendbuf,recvbuf,recvcount,datatype,op,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_reduce_scatter_block_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: recvcount
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_reduce_scatter_block_init_f(sendbuf,recvbuf,recvcount,&
-                                          datatype%MPI_VAL,op%MPI_VAL,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPIX_Reduce_scatter_block_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/reduce_scatter_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/reduce_scatter_init_f08.F90
deleted file mode 100644
index 66c0717cdd7..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/reduce_scatter_init_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine MPIX_Reduce_scatter_init_f08(sendbuf,recvbuf,recvcounts,datatype,op,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_reduce_scatter_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: recvcounts(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_reduce_scatter_init_f(sendbuf,recvbuf,recvcounts,datatype%MPI_VAL,&
-                                    op%MPI_VAL,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPIX_Reduce_scatter_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/scan_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/scan_init_f08.F90
deleted file mode 100644
index 6819722929c..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/scan_init_f08.F90
+++ /dev/null
@@ -1,32 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine MPIX_Scan_init_f08(sendbuf,recvbuf,count,datatype,op,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Op, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_scan_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: count
-   TYPE(MPI_Datatype), INTENT(IN) :: datatype
-   TYPE(MPI_Op), INTENT(IN) :: op
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_scan_init_f(sendbuf,recvbuf,count,datatype%MPI_VAL,&
-                          op%MPI_VAL,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPIX_Scan_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/scatter_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/scatter_init_f08.F90
deleted file mode 100644
index 372207fbaaf..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/scatter_init_f08.F90
+++ /dev/null
@@ -1,33 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine MPIX_Scatter_init_f08(sendbuf,sendcount,sendtype,recvbuf,&
-                                 recvcount,recvtype,root,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_scatter_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: sendcount, recvcount, root
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_scatter_init_f(sendbuf,sendcount,sendtype%MPI_VAL,recvbuf,recvcount,&
-                             recvtype%MPI_VAL,root,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPIX_Scatter_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi-f08/scatterv_init_f08.F90 b/ompi/mpiext/pcollreq/use-mpi-f08/scatterv_init_f08.F90
deleted file mode 100644
index fe4a4a9d1b9..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi-f08/scatterv_init_f08.F90
+++ /dev/null
@@ -1,34 +0,0 @@
-! -*- f90 -*-
-!
-! Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
-!                         All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST).  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! $COPYRIGHT$
-
-#include "ompi/mpi/fortran/configure-fortran-output.h"
-
-subroutine MPIX_Scatterv_init_f08(sendbuf,sendcounts,displs,sendtype,recvbuf,&
-                                  recvcount,recvtype,root,comm,info,request,ierror)
-   use :: mpi_f08_types, only : MPI_Datatype, MPI_Comm, MPI_Info, MPI_Request
-   use :: mpiext_pcollreq_f08, only : ompix_scatterv_init_f
-   implicit none
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, INTENT(IN), ASYNCHRONOUS :: sendbuf
-   OMPI_FORTRAN_IGNORE_TKR_TYPE, ASYNCHRONOUS :: recvbuf
-   INTEGER, INTENT(IN) :: recvcount, root
-   INTEGER, INTENT(IN), ASYNCHRONOUS :: sendcounts(*), displs(*)
-   TYPE(MPI_Datatype), INTENT(IN) :: sendtype
-   TYPE(MPI_Datatype), INTENT(IN) :: recvtype
-   TYPE(MPI_Comm), INTENT(IN) :: comm
-   TYPE(MPI_Info), INTENT(IN) :: info
-   TYPE(MPI_Request), INTENT(OUT) :: request
-   INTEGER, OPTIONAL, INTENT(OUT) :: ierror
-   integer :: c_ierror
-
-   call ompix_scatterv_init_f(sendbuf,sendcounts,displs,sendtype%MPI_VAL,recvbuf,&
-                              recvcount,recvtype%MPI_VAL,root,comm%MPI_VAL,info%MPI_VAL,request%MPI_VAL,c_ierror)
-   if (present(ierror)) ierror = c_ierror
-
-end subroutine MPIX_Scatterv_init_f08
diff --git a/ompi/mpiext/pcollreq/use-mpi/Makefile.am b/ompi/mpiext/pcollreq/use-mpi/Makefile.am
deleted file mode 100644
index 225ffa8d589..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi/Makefile.am
+++ /dev/null
@@ -1,16 +0,0 @@
-#
-# Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2018 FUJITSU LIMITED.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-# There's nothing to build or install for the "use mpi" bindings for
-# this MPI extension.  However, we must list the header file that will
-# be slurped up to be part of the mpi_ext Fortran module so that it is
-# included in dist tarballs.
-
-noinst_HEADERS = mpiext_pcollreq_usempi.h
diff --git a/ompi/mpiext/pcollreq/use-mpi/mpiext_pcollreq_usempi.h b/ompi/mpiext/pcollreq/use-mpi/mpiext_pcollreq_usempi.h
deleted file mode 100644
index f083470d180..00000000000
--- a/ompi/mpiext/pcollreq/use-mpi/mpiext_pcollreq_usempi.h
+++ /dev/null
@@ -1,983 +0,0 @@
-! -*- fortran -*-
-!
-! Copyright (c) 2012      Cisco Systems, Inc.  All rights reserved.
-! Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
-! Copyright (c) 2018      Research Organization for Information Science
-!                         and Technology (RIST). All rights reserved.
-! $COPYRIGHT$
-!
-! Additional copyrights may follow
-!
-! $HEADER$
-!
-
-! This whole file will be included in the mpi_ext module interface
-! section.  Note that the extension's mpif.h file will be included
-! first, so there's no need to re-define anything that's in there (e.g.,
-! OMPI_EXAMPLE_GLOBAL).
-
-! Declare any interfaces, subroutines, and global variables/constants
-! here.  Note that the mpiext_example_mpif.h will automatically be
-! included before this, so anything declared there does not need to be
-! replicated here.
-
-interface mpix_allgather_init
-    subroutine mpix_allgather_init(sendbuf, sendcount, sendtype, &
-                                   recvbuf, recvcount, recvtype, &
-                                   comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, intent(in) :: sendcount
-        integer, intent(in) :: sendtype
-        integer, intent(in) :: recvcount
-        integer, intent(in) :: recvtype
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine mpix_allgather_init
-end interface mpix_allgather_init
-
-interface pmpix_allgather_init
-    subroutine pmpix_allgather_init(sendbuf, sendcount, sendtype, &
-                                    recvbuf, recvcount, recvtype, &
-                                    comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, intent(in) :: sendcount
-        integer, intent(in) :: sendtype
-        integer, intent(in) :: recvcount
-        integer, intent(in) :: recvtype
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine pmpix_allgather_init
-end interface pmpix_allgather_init
-
-interface mpix_allgatherv_init
-    subroutine mpix_allgatherv_init(sendbuf, sendcount, sendtype, &
-                                    recvbuf, recvcounts, displs, recvtype, &
-                                    comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, intent(in) :: sendcount
-        integer, intent(in) :: sendtype
-        integer, dimension(*), intent(in) :: recvcounts
-        integer, dimension(*), intent(in) :: displs
-        integer, intent(in) :: recvtype
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine mpix_allgatherv_init
-end interface mpix_allgatherv_init
-
-interface pmpix_allgatherv_init
-    subroutine pmpix_allgatherv_init(sendbuf, sendcount, sendtype, &
-                                     recvbuf, recvcounts, displs, recvtype, &
-                                     comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, intent(in) :: sendcount
-        integer, intent(in) :: sendtype
-        integer, dimension(*), intent(in) :: recvcounts
-        integer, dimension(*), intent(in) :: displs
-        integer, intent(in) :: recvtype
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine pmpix_allgatherv_init
-end interface pmpix_allgatherv_init
-
-interface mpix_allreduce_init
-    subroutine mpix_allreduce_init(sendbuf, recvbuf, count, &
-                                   datatype, op, &
-                                   comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, intent(in) :: count
-        integer, intent(in) :: datatype
-        integer, intent(in) :: op
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine mpix_allreduce_init
-end interface mpix_allreduce_init
-
-interface pmpix_allreduce_init
-    subroutine pmpix_allreduce_init(sendbuf, recvbuf, count, &
-                                    datatype, op, &
-                                    comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, intent(in) :: count
-        integer, intent(in) :: datatype
-        integer, intent(in) :: op
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine pmpix_allreduce_init
-end interface pmpix_allreduce_init
-
-interface mpix_alltoall_init
-    subroutine mpix_alltoall_init(sendbuf, sendcount, sendtype, &
-                                  recvbuf, recvcount, recvtype, &
-                                  comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, intent(in) :: sendcount
-        integer, intent(in) :: sendtype
-        integer, intent(in) :: recvcount
-        integer, intent(in) :: recvtype
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine mpix_alltoall_init
-end interface mpix_alltoall_init
-
-interface pmpix_alltoall_init
-    subroutine pmpix_alltoall_init(sendbuf, sendcount, sendtype, &
-                                   recvbuf, recvcount, recvtype, &
-                                   comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, intent(in) :: sendcount
-        integer, intent(in) :: sendtype
-        integer, intent(in) :: recvcount
-        integer, intent(in) :: recvtype
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine pmpix_alltoall_init
-end interface pmpix_alltoall_init
-
-interface mpix_alltoallv_init
-    subroutine mpix_alltoallv_init(sendbuf, sendcounts, sdispls, sendtype, &
-                                   recvbuf, recvcounts, rdispls, recvtype, &
-                                   comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, dimension(*), intent(in) :: sendcounts
-        integer, dimension(*), intent(in) :: sdispls
-        integer, intent(in) :: sendtype
-        integer, dimension(*), intent(in) :: recvcounts
-        integer, dimension(*), intent(in) :: rdispls
-        integer, intent(in) :: recvtype
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine mpix_alltoallv_init
-end interface mpix_alltoallv_init
-
-interface pmpix_alltoallv_init
-    subroutine pmpix_alltoallv_init(sendbuf, sendcounts, sdispls, sendtype, &
-                                    recvbuf, recvcounts, rdispls, recvtype, &
-                                    comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, dimension(*), intent(in) :: sendcounts
-        integer, dimension(*), intent(in) :: sdispls
-        integer, intent(in) :: sendtype
-        integer, dimension(*), intent(in) :: recvcounts
-        integer, dimension(*), intent(in) :: rdispls
-        integer, intent(in) :: recvtype
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine pmpix_alltoallv_init
-end interface pmpix_alltoallv_init
-
-interface mpix_alltoallw_init
-    subroutine mpix_alltoallw_init(sendbuf, sendcounts, sdispls, sendtypes, &
-                                   recvbuf, recvcounts, rdispls, recvtypes, &
-                                   comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, dimension(*), intent(in) :: sendcounts
-        integer, dimension(*), intent(in) :: sdispls
-        integer, dimension(*), intent(in) :: sendtypes
-        integer, dimension(*), intent(in) :: recvcounts
-        integer, dimension(*), intent(in) :: rdispls
-        integer, dimension(*), intent(in) :: recvtypes
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine mpix_alltoallw_init
-end interface mpix_alltoallw_init
-
-interface pmpix_alltoallw_init
-    subroutine pmpix_alltoallw_init(sendbuf, sendcounts, sdispls, sendtypes, &
-                                    recvbuf, recvcounts, rdispls, recvtypes, &
-                                    comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, dimension(*), intent(in) :: sendcounts
-        integer, dimension(*), intent(in) :: sdispls
-        integer, dimension(*), intent(in) :: sendtypes
-        integer, dimension(*), intent(in) :: recvcounts
-        integer, dimension(*), intent(in) :: rdispls
-        integer, dimension(*), intent(in) :: recvtypes
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine pmpix_alltoallw_init
-end interface pmpix_alltoallw_init
-
-interface mpix_barrier_init
-    subroutine mpix_barrier_init(comm, info, request, ierror)
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine mpix_barrier_init
-end interface mpix_barrier_init
-
-interface pmpix_barrier_init
-    subroutine pmpix_barrier_init(comm, info, request, ierror)
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine pmpix_barrier_init
-end interface pmpix_barrier_init
-
-interface mpix_bcast_init
-    subroutine mpix_bcast_init(buffer, count, datatype, root, &
-                               comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: buffer
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: buffer
-        !$PRAGMA IGNORE_TKR buffer
-        !DIR$ IGNORE_TKR buffer
-        !IBM* IGNORE_TKR buffer
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: buffer
-        integer, intent(in) :: count
-        integer, intent(in) :: datatype
-        integer, intent(in) :: root
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine mpix_bcast_init
-end interface mpix_bcast_init
-
-interface pmpix_bcast_init
-    subroutine pmpix_bcast_init(buffer, count, datatype, root, &
-                                comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: buffer
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: buffer
-        !$PRAGMA IGNORE_TKR buffer
-        !DIR$ IGNORE_TKR buffer
-        !IBM* IGNORE_TKR buffer
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: buffer
-        integer, intent(in) :: count
-        integer, intent(in) :: datatype
-        integer, intent(in) :: root
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine pmpix_bcast_init
-end interface pmpix_bcast_init
-
-interface mpix_exscan_init
-    subroutine mpix_exscan_init(sendbuf, recvbuf, count, &
-                                datatype, op, &
-                                comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, intent(in) :: count
-        integer, intent(in) :: datatype
-        integer, intent(in) :: op
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine mpix_exscan_init
-end interface mpix_exscan_init
-
-interface pmpix_exscan_init
-    subroutine pmpix_exscan_init(sendbuf, recvbuf, count, &
-                                 datatype, op, &
-                                 comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, intent(in) :: count
-        integer, intent(in) :: datatype
-        integer, intent(in) :: op
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine pmpix_exscan_init
-end interface pmpix_exscan_init
-
-interface mpix_gather_init
-    subroutine mpix_gather_init(sendbuf, sendcount, sendtype, &
-                                recvbuf, recvcount, recvtype, root, &
-                                comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, intent(in) :: sendcount
-        integer, intent(in) :: sendtype
-        integer, intent(in) :: recvcount
-        integer, intent(in) :: recvtype
-        integer, intent(in) :: root
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine mpix_gather_init
-end interface mpix_gather_init
-
-interface pmpix_gather_init
-    subroutine pmpix_gather_init(sendbuf, sendcount, sendtype, &
-                                 recvbuf, recvcount, recvtype, root, &
-                                 comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, intent(in) :: sendcount
-        integer, intent(in) :: sendtype
-        integer, intent(in) :: recvcount
-        integer, intent(in) :: recvtype
-        integer, intent(in) :: root
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine pmpix_gather_init
-end interface pmpix_gather_init
-
-interface mpix_gatherv_init
-    subroutine mpix_gatherv_init(sendbuf, sendcount, sendtype, &
-                                 recvbuf, recvcounts, displs, recvtype, root, &
-                                 comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, intent(in) :: sendcount
-        integer, intent(in) :: sendtype
-        integer, dimension(*), intent(in) :: recvcounts
-        integer, dimension(*), intent(in) :: displs
-        integer, intent(in) :: recvtype
-        integer, intent(in) :: root
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine mpix_gatherv_init
-end interface mpix_gatherv_init
-
-interface pmpix_gatherv_init
-    subroutine pmpix_gatherv_init(sendbuf, sendcount, sendtype, &
-                                  recvbuf, recvcounts, displs, recvtype, root, &
-                                  comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, intent(in) :: sendcount
-        integer, intent(in) :: sendtype
-        integer, dimension(*), intent(in) :: recvcounts
-        integer, dimension(*), intent(in) :: displs
-        integer, intent(in) :: recvtype
-        integer, intent(in) :: root
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine pmpix_gatherv_init
-end interface pmpix_gatherv_init
-
-interface mpix_reduce_init
-    subroutine mpix_reduce_init(sendbuf, recvbuf, count, &
-                                datatype, op, root, &
-                                comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, intent(in) :: count
-        integer, intent(in) :: datatype
-        integer, intent(in) :: op
-        integer, intent(in) :: root
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine mpix_reduce_init
-end interface mpix_reduce_init
-
-interface pmpix_reduce_init
-    subroutine pmpix_reduce_init(sendbuf, recvbuf, count, &
-                                 datatype, op, root, &
-                                 comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, intent(in) :: count
-        integer, intent(in) :: datatype
-        integer, intent(in) :: op
-        integer, intent(in) :: root
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine pmpix_reduce_init
-end interface pmpix_reduce_init
-
-interface mpix_reduce_scatter_init
-    subroutine mpix_reduce_scatter_init(sendbuf, recvbuf, recvcounts, &
-                                        datatype, op, &
-                                        comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, dimension(*), intent(in) :: recvcounts
-        integer, intent(in) :: datatype
-        integer, intent(in) :: op
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine mpix_reduce_scatter_init
-end interface mpix_reduce_scatter_init
-
-interface pmpix_reduce_scatter_init
-    subroutine pmpix_reduce_scatter_init(sendbuf, recvbuf, recvcounts, &
-                                         datatype, op, &
-                                         comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, dimension(*), intent(in) :: recvcounts
-        integer, intent(in) :: datatype
-        integer, intent(in) :: op
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine pmpix_reduce_scatter_init
-end interface pmpix_reduce_scatter_init
-
-interface mpix_reduce_scatter_block_init
-    subroutine mpix_reduce_scatter_block_init(sendbuf, recvbuf, recvcount, &
-                                              datatype, op, &
-                                              comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, intent(in) :: recvcount
-        integer, intent(in) :: datatype
-        integer, intent(in) :: op
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine mpix_reduce_scatter_block_init
-end interface mpix_reduce_scatter_block_init
-
-interface pmpix_reduce_scatter_block_init
-    subroutine pmpix_reduce_scatter_block_init(sendbuf, recvbuf, recvcount, &
-                                               datatype, op, &
-                                               comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, intent(in) :: recvcount
-        integer, intent(in) :: datatype
-        integer, intent(in) :: op
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine pmpix_reduce_scatter_block_init
-end interface pmpix_reduce_scatter_block_init
-
-interface mpix_scan_init
-    subroutine mpix_scan_init(sendbuf, recvbuf, count, &
-                              datatype, op, &
-                              comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, intent(in) :: count
-        integer, intent(in) :: datatype
-        integer, intent(in) :: op
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine mpix_scan_init
-end interface mpix_scan_init
-
-interface pmpix_scan_init
-    subroutine pmpix_scan_init(sendbuf, recvbuf, count, &
-                               datatype, op, &
-                               comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, intent(in) :: count
-        integer, intent(in) :: datatype
-        integer, intent(in) :: op
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine pmpix_scan_init
-end interface pmpix_scan_init
-
-interface mpix_scatter_init
-    subroutine mpix_scatter_init(sendbuf, sendcount, sendtype, &
-                                 recvbuf, recvcount, recvtype, root, &
-                                 comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, intent(in) :: sendcount
-        integer, intent(in) :: sendtype
-        integer, intent(in) :: recvcount
-        integer, intent(in) :: recvtype
-        integer, intent(in) :: root
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine mpix_scatter_init
-end interface mpix_scatter_init
-
-interface pmpix_scatter_init
-    subroutine pmpix_scatter_init(sendbuf, sendcount, sendtype, &
-                                  recvbuf, recvcount, recvtype, root, &
-                                  comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, intent(in) :: sendcount
-        integer, intent(in) :: sendtype
-        integer, intent(in) :: recvcount
-        integer, intent(in) :: recvtype
-        integer, intent(in) :: root
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine pmpix_scatter_init
-end interface pmpix_scatter_init
-
-interface mpix_scatterv_init
-    subroutine mpix_scatterv_init(sendbuf, sendcounts, displs, sendtype, &
-                                  recvbuf, recvcount, recvtype, root, &
-                                  comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, dimension(*), intent(in) :: sendcounts
-        integer, dimension(*), intent(in) :: displs
-        integer, intent(in) :: sendtype
-        integer, intent(in) :: recvcount
-        integer, intent(in) :: recvtype
-        integer, intent(in) :: root
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine mpix_scatterv_init
-end interface mpix_scatterv_init
-
-interface pmpix_scatterv_init
-    subroutine pmpix_scatterv_init(sendbuf, sendcounts, displs, sendtype, &
-                                   recvbuf, recvcount, recvtype, root, &
-                                   comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, dimension(*), intent(in) :: sendcounts
-        integer, dimension(*), intent(in) :: displs
-        integer, intent(in) :: sendtype
-        integer, intent(in) :: recvcount
-        integer, intent(in) :: recvtype
-        integer, intent(in) :: root
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine pmpix_scatterv_init
-end interface pmpix_scatterv_init
-
-interface mpix_neighbor_allgather_init
-    subroutine mpix_neighbor_allgather_init(sendbuf, sendcount, sendtype, &
-                                            recvbuf, recvcount, recvtype, &
-                                            comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, intent(in) :: sendcount
-        integer, intent(in) :: sendtype
-        integer, intent(in) :: recvcount
-        integer, intent(in) :: recvtype
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine mpix_neighbor_allgather_init
-end interface mpix_neighbor_allgather_init
-
-interface pmpix_neighbor_allgather_init
-    subroutine pmpix_neighbor_allgather_init(sendbuf, sendcount, sendtype, &
-                                             recvbuf, recvcount, recvtype, &
-                                             comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, intent(in) :: sendcount
-        integer, intent(in) :: sendtype
-        integer, intent(in) :: recvcount
-        integer, intent(in) :: recvtype
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine pmpix_neighbor_allgather_init
-end interface pmpix_neighbor_allgather_init
-
-interface mpix_neighbor_allgatherv_init
-    subroutine mpix_neighbor_allgatherv_init(sendbuf, sendcount, sendtype, &
-                                             recvbuf, recvcounts, displs, recvtype, &
-                                             comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, intent(in) :: sendcount
-        integer, intent(in) :: sendtype
-        integer, dimension(*), intent(in) :: recvcounts
-        integer, dimension(*), intent(in) :: displs
-        integer, intent(in) :: recvtype
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine mpix_neighbor_allgatherv_init
-end interface mpix_neighbor_allgatherv_init
-
-interface pmpix_neighbor_allgatherv_init
-    subroutine pmpix_neighbor_allgatherv_init(sendbuf, sendcount, sendtype, &
-                                              recvbuf, recvcounts, displs, recvtype, &
-                                              comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, intent(in) :: sendcount
-        integer, intent(in) :: sendtype
-        integer, dimension(*), intent(in) :: recvcounts
-        integer, dimension(*), intent(in) :: displs
-        integer, intent(in) :: recvtype
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine pmpix_neighbor_allgatherv_init
-end interface pmpix_neighbor_allgatherv_init
-
-interface mpix_neighbor_alltoall_init
-    subroutine mpix_neighbor_alltoall_init(sendbuf, sendcount, sendtype, &
-                                           recvbuf, recvcount, recvtype, &
-                                           comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, intent(in) :: sendcount
-        integer, intent(in) :: sendtype
-        integer, intent(in) :: recvcount
-        integer, intent(in) :: recvtype
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine mpix_neighbor_alltoall_init
-end interface mpix_neighbor_alltoall_init
-
-interface pmpix_neighbor_alltoall_init
-    subroutine pmpix_neighbor_alltoall_init(sendbuf, sendcount, sendtype, &
-                                            recvbuf, recvcount, recvtype, &
-                                            comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, intent(in) :: sendcount
-        integer, intent(in) :: sendtype
-        integer, intent(in) :: recvcount
-        integer, intent(in) :: recvtype
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine pmpix_neighbor_alltoall_init
-end interface pmpix_neighbor_alltoall_init
-
-interface mpix_neighbor_alltoallv_init
-    subroutine mpix_neighbor_alltoallv_init(sendbuf, sendcounts, sdispls, sendtype, &
-                                            recvbuf, recvcounts, rdispls, recvtype, &
-                                            comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, dimension(*), intent(in) :: sendcounts
-        integer, dimension(*), intent(in) :: sdispls
-        integer, intent(in) :: sendtype
-        integer, dimension(*), intent(in) :: recvcounts
-        integer, dimension(*), intent(in) :: rdispls
-        integer, intent(in) :: recvtype
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine mpix_neighbor_alltoallv_init
-end interface mpix_neighbor_alltoallv_init
-
-interface pmpix_neighbor_alltoallv_init
-    subroutine pmpix_neighbor_alltoallv_init(sendbuf, sendcounts, sdispls, sendtype, &
-                                             recvbuf, recvcounts, rdispls, recvtype, &
-                                             comm, info, request, ierror)
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, dimension(*), intent(in) :: sendcounts
-        integer, dimension(*), intent(in) :: sdispls
-        integer, intent(in) :: sendtype
-        integer, dimension(*), intent(in) :: recvcounts
-        integer, dimension(*), intent(in) :: rdispls
-        integer, intent(in) :: recvtype
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine pmpix_neighbor_alltoallv_init
-end interface pmpix_neighbor_alltoallv_init
-
-interface mpix_neighbor_alltoallw_init
-    subroutine mpix_neighbor_alltoallw_init(sendbuf, sendcounts, sdispls, sendtypes, &
-                                            recvbuf, recvcounts, rdispls, recvtypes, &
-                                            comm, info, request, ierror)
-        include 'mpif-config.h'
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, dimension(*), intent(in) :: sendcounts
-        integer(kind=mpi_address_kind), dimension(*), intent(in) :: sdispls
-        integer, dimension(*), intent(in) :: sendtypes
-        integer, dimension(*), intent(in) :: recvcounts
-        integer(kind=mpi_address_kind), dimension(*), intent(in) :: rdispls
-        integer, dimension(*), intent(in) :: recvtypes
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine mpix_neighbor_alltoallw_init
-end interface mpix_neighbor_alltoallw_init
-
-interface pmpix_neighbor_alltoallw_init
-    subroutine pmpix_neighbor_alltoallw_init(sendbuf, sendcounts, sdispls, sendtypes, &
-                                             recvbuf, recvcounts, rdispls, recvtypes, &
-                                             comm, info, request, ierror)
-        include 'mpif-config.h'
-        !DEC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !GCC$ ATTRIBUTES NO_ARG_CHECK :: sendbuf, recvbuf
-        !$PRAGMA IGNORE_TKR sendbuf, recvbuf
-        !DIR$ IGNORE_TKR sendbuf, recvbuf
-        !IBM* IGNORE_TKR sendbuf, recvbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE, intent(in) :: sendbuf
-        OMPI_FORTRAN_IGNORE_TKR_TYPE :: recvbuf
-        integer, dimension(*), intent(in) :: sendcounts
-        integer(kind=mpi_address_kind), dimension(*), intent(in) :: sdispls
-        integer, dimension(*), intent(in) :: sendtypes
-        integer, dimension(*), intent(in) :: recvcounts
-        integer(kind=mpi_address_kind), dimension(*), intent(in) :: rdispls
-        integer, dimension(*), intent(in) :: recvtypes
-        integer, intent(in) :: comm
-        integer, intent(in) :: info
-        integer, intent(out) :: request
-        integer, intent(out) :: ierror
-    end subroutine pmpix_neighbor_alltoallw_init
-end interface pmpix_neighbor_alltoallw_init
-
diff --git a/ompi/mpiext/rocm/Makefile.am b/ompi/mpiext/rocm/Makefile.am
new file mode 100644
index 00000000000..25d8b516d83
--- /dev/null
+++ b/ompi/mpiext/rocm/Makefile.am
@@ -0,0 +1,23 @@
+#
+# Copyright (c) 2004-2009 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2015      NVIDIA, Inc. All rights reserved
+# Copyright (c) 2022      Advanced Micro Devices, Inc. All rights reserved
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# This Makefile is not traversed during a normal "make all" in an OMPI
+# build.  It *is* traversed during "make dist", however.  So you can
+# put EXTRA_DIST targets in here.
+#
+# You can also use this as a convenience for building this MPI
+# extension (i.e., "make all" in this directory to invoke "make all"
+# in all the subdirectories).
+
+SUBDIRS = c
diff --git a/ompi/mpiext/rocm/c/Makefile.am b/ompi/mpiext/rocm/c/Makefile.am
new file mode 100644
index 00000000000..9315a4636dc
--- /dev/null
+++ b/ompi/mpiext/rocm/c/Makefile.am
@@ -0,0 +1,42 @@
+#
+# Copyright (c) 2004-2009 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2015      NVIDIA, Inc. All rights reserved.
+# Copyright (c) 2018      Research Organization for Information Science
+#                         and Technology (RIST).  All rights reserved.
+# Copyright (c) 2022      Advanced Micro Devices, Inc. All rights reserved.
+#
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# This file builds the C bindings for MPI extensions.  It must be
+# present in all MPI extensions.
+
+# We must set these #defines so that the inner OMPI MPI prototype
+# header files do the Right Thing.
+AM_CPPFLAGS = -DOMPI_PROFILE_LAYER=0 -DOMPI_COMPILING_FORTRAN_WRAPPERS=1
+
+# Convenience libtool library that will be slurped up into libmpi.la.
+noinst_LTLIBRARIES = libmpiext_rocm_c.la
+
+# This is where the top-level header file (that is included in
+# <mpi-ext.h>) must be installed.
+ompidir = $(ompiincludedir)/mpiext
+
+# This is the header file that is installed.
+nodist_ompi_HEADERS = mpiext_rocm_c.h
+
+# Sources for the convenience libtool library.  Other than the one
+# header file, all source files in the extension have no file naming
+# conventions.
+libmpiext_rocm_c_la_SOURCES = \
+        $(ompi_HEADERS) \
+        mpiext_rocm.c
+libmpiext_rocm_c_la_LDFLAGS = -module -avoid-version
+
diff --git a/ompi/mpiext/rocm/c/mpiext_rocm.c b/ompi/mpiext/rocm/c/mpiext_rocm.c
new file mode 100644
index 00000000000..a91d5b9b064
--- /dev/null
+++ b/ompi/mpiext/rocm/c/mpiext_rocm.c
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2004-2009 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2012      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      NVIDIA, Inc. All rights reserved.
+ * Copyright (c) 2022      Advanced Micro Devices, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include "ompi_config.h"
+
+#include <stdio.h>
+#include <string.h>
+
+#include "opal/constants.h"
+#include "opal/runtime/opal_params.h"
+#include "ompi/mpiext/rocm/c/mpiext_rocm_c.h"
+
+
+int MPIX_Query_rocm_support(void)
+{
+    return opal_built_with_rocm_support && opal_rocm_runtime_initialized;
+}
diff --git a/ompi/mpiext/rocm/c/mpiext_rocm_c.h.in b/ompi/mpiext/rocm/c/mpiext_rocm_c.h.in
new file mode 100644
index 00000000000..265282d458e
--- /dev/null
+++ b/ompi/mpiext/rocm/c/mpiext_rocm_c.h.in
@@ -0,0 +1,17 @@
+/*
+ * Copyright (c) 2004-2009 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2015      NVIDIA, Inc. All rights reserved.
+ * Copyright (c) 2022      Advanced Micro Devices, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#define MPIX_ROCM_AWARE_SUPPORT @MPIX_ROCM_AWARE_SUPPORT@
+OMPI_DECLSPEC int MPIX_Query_rocm_support(void);
diff --git a/ompi/mpiext/rocm/configure.m4 b/ompi/mpiext/rocm/configure.m4
new file mode 100644
index 00000000000..5f22505f44c
--- /dev/null
+++ b/ompi/mpiext/rocm/configure.m4
@@ -0,0 +1,35 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2010 The Trustees of Indiana University.
+#                         All rights reserved.
+# Copyright (c) 2012-2015 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2015      Intel, Inc. All rights reserved.
+# Copyright (c) 2015      NVIDIA, Inc.  All rights reserved.
+# Copyright (c) 2022      Advanced Micro Devices, Inc.  All rights reserved.
+#
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# OMPI_MPIEXT_rocm_CONFIG([action-if-found], [action-if-not-found])
+# -----------------------------------------------------------
+AC_DEFUN([OMPI_MPIEXT_rocm_CONFIG],[
+    AC_CONFIG_FILES([ompi/mpiext/rocm/Makefile])
+    AC_CONFIG_FILES([ompi/mpiext/rocm/c/Makefile])
+    AC_CONFIG_HEADERS([ompi/mpiext/rocm/c/mpiext_rocm_c.h])
+
+    AC_DEFINE_UNQUOTED([MPIX_ROCM_AWARE_SUPPORT], [$ROCM_SUPPORT],
+                       [Macro that is set to 1 when ROCM-aware support is configured in and 0 when it is not])
+
+    
+    # We compile this whether ROCm support was requested or not. It allows
+    # us to to detect if we have ROCm support.
+    AS_IF([test "$ENABLE_rocm" = "1" || \
+           test "$ENABLE_EXT_ALL" = "1"],
+          [$1],
+          [$2])
+    
+])
diff --git a/ompi/op/op.c b/ompi/op/op.c
index 39732a40187..c800dc0a1cb 100644
--- a/ompi/op/op.c
+++ b/ompi/op/op.c
@@ -17,6 +17,8 @@
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2018-2025 Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -33,6 +35,7 @@
 #include "ompi/op/op.h"
 #include "ompi/mca/op/base/base.h"
 #include "ompi/datatype/ompi_datatype_internal.h"
+#include "ompi/instance/instance.h"
 
 
 /*
@@ -47,6 +50,7 @@ opal_pointer_array_t *ompi_op_f_to_c_table = {0};
 static int add_intrinsic(ompi_op_t *op, int fort_handle, int flags,
                          const char *name);
 
+static int ompi_op_finalize (void);
 
 /*
  * Class information
@@ -228,6 +232,9 @@ int ompi_op_init(void)
     ompi_op_ddt_map[OMPI_DATATYPE_MPI_SHORT_FLOAT] = OMPI_OP_BASE_TYPE_SHORT_FLOAT;
     ompi_op_ddt_map[OMPI_DATATYPE_MPI_C_SHORT_FLOAT_COMPLEX] = OMPI_OP_BASE_TYPE_C_SHORT_FLOAT_COMPLEX;
 
+    ompi_op_ddt_map[OMPI_DATATYPE_MPI_LONG] = OMPI_OP_BASE_TYPE_LONG;
+    ompi_op_ddt_map[OMPI_DATATYPE_MPI_UNSIGNED_LONG] = OMPI_OP_BASE_TYPE_UNSIGNED_LONG;
+
     /* Create the intrinsic ops */
 
     if (OMPI_SUCCESS !=
@@ -297,15 +304,23 @@ int ompi_op_init(void)
         ompi_mpi_op_replace.op.op_type = OMPI_OP_REPLACE;
     }
 
+    ompi_mpi_instance_append_finalize (ompi_op_finalize);
+
     /* All done */
     return OMPI_SUCCESS;
 }
 
 
-/*
- * Clean up the op resources
+
+/**
+ * Finalize the op interface.
+ *
+ * @returns OMPI_SUCCESS Always
+ *
+ * Invokes on instance teardown if ompi_op_init() was called; tears down the op interface, and
+ * destroys the F2C translation table.
  */
-int ompi_op_finalize(void)
+static int ompi_op_finalize (void)
 {
     /* clean up the intrinsic ops */
     OBJ_DESTRUCT(&ompi_mpi_op_no_op);
@@ -338,6 +353,7 @@ int ompi_op_finalize(void)
  * Create a new MPI_Op
  */
 ompi_op_t *ompi_op_create_user(bool commute,
+                               bool bigcount,
                                ompi_op_fortran_handler_fn_t func)
 {
     ompi_op_t *new_op;
@@ -367,13 +383,16 @@ ompi_op_t *ompi_op_create_user(bool commute,
     if (commute) {
         new_op->o_flags |= OMPI_OP_FLAGS_COMMUTE;
     }
+    if(bigcount) {
+        new_op->o_flags |= OMPI_OP_FLAGS_BIGCOUNT;
+    }
 
     opal_string_copy(new_op->o_name, "USER OP", sizeof(new_op->o_name));
     new_op->o_name[sizeof(new_op->o_name) - 1] = '\0';
 
     /* Set the user-defined callback function.  The "fort_fn" member
        is part of a union, so it doesn't matter if this is a C or
-       Fortan callback; we'll call the right flavor (per o_flags) at
+       Fortran callback; we'll call the right flavor (per o_flags) at
        invocation time. */
     new_op->o_func.fort_fn = func;
 
diff --git a/ompi/op/op.h b/ompi/op/op.h
index 2cddcc97bb2..bf895d7c6f3 100644
--- a/ompi/op/op.h
+++ b/ompi/op/op.h
@@ -18,8 +18,9 @@
  * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * Copyright (c) 2019      Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2018      Triad National Security, LLC. All rights
+ * Copyright (c) 2018-2025 Triad National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2021      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -58,20 +59,24 @@ BEGIN_C_DECLS
  * repeated code, but it's better this way (and this typedef will
  * never change, so there's not much of a maintenance worry).
  */
-typedef void (ompi_op_c_handler_fn_t)(void *, void *, int *,
+typedef void (ompi_op_c_handler_fn_t)(const void *, void *, int *,
                                       struct ompi_datatype_t **);
+typedef void (ompi_op_c_handler_bc_fn_t)(const void *, void *, size_t *,
+                                         struct ompi_datatype_t **);
 
 /**
  * Typedef for fortran user-defined MPI_Ops.
  */
-typedef void (ompi_op_fortran_handler_fn_t)(void *, void *,
+typedef void (ompi_op_fortran_handler_fn_t)(const void *, void *,
                                             MPI_Fint *, MPI_Fint *);
+typedef void (ompi_op_fortran_handler_bc_fn_t)(const void *, void *,
+                                               size_t *, MPI_Fint *);
 
 /**
  * Typedef for Java op functions intercept (used for user-defined
  * MPI.Ops).
  */
-typedef void (ompi_op_java_handler_fn_t)(void *, void *, int *,
+typedef void (ompi_op_java_handler_fn_t)(const void *, void *, int *,
                                          struct ompi_datatype_t **,
                                          int baseType,
                                          void *jnienv, void *object);
@@ -97,8 +102,8 @@ typedef void (ompi_op_java_handler_fn_t)(void *, void *, int *,
 #define OMPI_OP_FLAGS_FLOAT_ASSOC  0x0020
 /** Set if the callback function is communative */
 #define OMPI_OP_FLAGS_COMMUTE      0x0040
-
-
+/** Set if the callback function is using bigcount */
+#define OMPI_OP_FLAGS_BIGCOUNT     0x0080
 
 
 /*
@@ -151,8 +156,12 @@ struct ompi_op_t {
         ompi_op_base_op_fns_t intrinsic;
         /** C handler function pointer */
         ompi_op_c_handler_fn_t *c_fn;
+        /** C handler function pointer  - bigcount*/
+        ompi_op_c_handler_bc_fn_t *c_fn_bc;
         /** Fortran handler function pointer */
         ompi_op_fortran_handler_fn_t *fort_fn;
+        /** Fortran handler function pointer  - bigcount*/
+        ompi_op_fortran_handler_bc_fn_t *fort_fn_bc;
         /** Java intercept function data */
         struct {
             /* The OMPI C++ callback/intercept function */
@@ -175,7 +184,7 @@ typedef struct ompi_op_t ompi_op_t;
 OMPI_DECLSPEC OBJ_CLASS_DECLARATION(ompi_op_t);
 
 /**
- * Padded struct to maintain back compatibiltiy.
+ * Padded struct to maintain back compatibility.
  * See ompi/communicator/communicator.h comments with struct ompi_communicator_t
  * for full explanation why we chose the following padding construct for predefines.
  */
@@ -326,22 +335,14 @@ extern struct opal_pointer_array_t *ompi_op_f_to_c_table;
  */
 int ompi_op_init(void);
 
-/**
- * Finalize the op interface.
- *
- * @returns OMPI_SUCCESS Always
- *
- * Invokes from ompi_mpi_finalize(); tears down the op interface, and
- * destroys the F2C translation table.
- */
-int ompi_op_finalize(void);
-
 /**
  * Create a ompi_op_t with a user-defined callback (vs. creating an
  * intrinsic ompi_op_t).
  *
  * @param commute Boolean indicating whether the operation is
  *        communative or not
+ * @param bigcount Boolean indicating whether or not the op is
+ *        using the bigcount (MPI_Count) interface
  * @param func Function pointer of the error handler
  *
  * @returns op Pointer to the ompi_op_t that will be
@@ -364,10 +365,11 @@ int ompi_op_finalize(void);
  * manually.
  */
 ompi_op_t *ompi_op_create_user(bool commute,
+                               bool bigcount,
                                ompi_op_fortran_handler_fn_t func);
 
 /**
- * Mark an MPI_Op as holding a Java calback function, and cache that
+ * Mark an MPI_Op as holding a Java callback function, and cache that
  * function in the MPI_Op.
  */
 OMPI_DECLSPEC void ompi_op_set_java_callback(ompi_op_t *op,  void *jnienv,
@@ -509,11 +511,40 @@ static inline bool ompi_op_is_valid(ompi_op_t * op, ompi_datatype_t * ddt,
  * optimization).  If you give it an intrinsic op with a datatype that
  * is not defined to have that operation, it is likely to seg fault.
  */
-static inline void ompi_op_reduce(ompi_op_t * op, void *source,
-                                  void *target, int count,
+static inline void ompi_op_reduce(ompi_op_t * op, const void *source,
+                                  void *target, size_t full_count,
                                   ompi_datatype_t * dtype)
 {
     MPI_Fint f_dtype, f_count;
+    int count = full_count;
+
+    /*
+     * If the full_count is > INT_MAX then we need to call the reduction op
+     * in iterations of counts <= INT_MAX since it has an `int *len`
+     * parameter.
+     *
+     */
+    if(OPAL_UNLIKELY((full_count > INT_MAX) &&
+        (0 == (op->o_flags & OMPI_OP_FLAGS_BIGCOUNT)))) {
+        size_t done_count = 0, shift;
+        int iter_count;
+        ptrdiff_t ext, lb;
+
+        ompi_datatype_get_extent(dtype, &lb, &ext);
+
+        while(done_count < full_count) {
+            if(done_count + INT_MAX > full_count) {
+                iter_count = full_count - done_count;
+            } else {
+                iter_count = INT_MAX;
+            }
+            shift = done_count * ext;
+            // Recurse one level in iterations of 'int'
+            ompi_op_reduce(op, (const char*)source + shift, (char*)target + shift, iter_count, dtype);
+            done_count += iter_count;
+        }
+        return;
+    }
 
     /*
      * Call the reduction function.  Two dimensions: a) if both the op
@@ -556,8 +587,12 @@ static inline void ompi_op_reduce(ompi_op_t * op, void *source,
     /* User-defined function */
     if (0 != (op->o_flags & OMPI_OP_FLAGS_FORTRAN_FUNC)) {
         f_dtype = OMPI_INT_2_FINT(dtype->d_f_to_c_index);
-        f_count = OMPI_INT_2_FINT(count);
-        op->o_func.fort_fn(source, target, &f_count, &f_dtype);
+        if (0 == (op->o_flags & OMPI_OP_FLAGS_BIGCOUNT)) {
+            f_count = OMPI_INT_2_FINT(count);
+            op->o_func.fort_fn(source, target, &f_count, &f_dtype);
+        } else {
+            op->o_func.fort_fn_bc(source, target, &full_count, &f_dtype);
+        }
         return;
     } else if (0 != (op->o_flags & OMPI_OP_FLAGS_JAVA_FUNC)) {
         op->o_func.java_data.intercept_fn(source, target, &count, &dtype,
@@ -566,15 +601,25 @@ static inline void ompi_op_reduce(ompi_op_t * op, void *source,
                                           op->o_func.java_data.object);
         return;
     }
-    op->o_func.c_fn(source, target, &count, &dtype);
+    if (0 == (op->o_flags & OMPI_OP_FLAGS_BIGCOUNT)) {
+        op->o_func.c_fn(source, target, &count, &dtype);
+    } else {
+        op->o_func.c_fn_bc(source, target, &full_count, &dtype);
+    }
     return;
 }
 
 static inline void ompi_3buff_op_user (ompi_op_t *op, void * restrict source1, void * restrict source2,
-                                       void * restrict result, int count, struct ompi_datatype_t *dtype)
+                                       void * restrict result, size_t full_count, struct ompi_datatype_t *dtype)
 {
-    ompi_datatype_copy_content_same_ddt (dtype, count, result, source1);
-    op->o_func.c_fn (source2, result, &count, &dtype);
+    ompi_datatype_copy_content_same_ddt (dtype, full_count, (char*)result, (char*)source1);
+    if (0 == (op->o_flags & OMPI_OP_FLAGS_BIGCOUNT)) {
+        assert(full_count <= INT_MAX);
+        int count = (int)full_count;  /* protected by loop in only caller of this function */
+        op->o_func.c_fn (source2, result, &count, &dtype);
+    } else {
+        op->o_func.c_fn_bc (source2, result, &full_count, &dtype);
+    }
 }
 
 /**
@@ -596,13 +641,11 @@ static inline void ompi_3buff_op_user (ompi_op_t *op, void * restrict source1, v
  * with the values in the source buffer and the result is stored in
  * the target buffer).
  *
- * This function will *only* be invoked on intrinsic MPI_Ops.
- *
  * Otherwise, this function is the same as ompi_op_reduce.
  */
 static inline void ompi_3buff_op_reduce(ompi_op_t * op, void *source1,
                                         void *source2, void *target,
-                                        int count, ompi_datatype_t * dtype)
+                                        size_t full_count, ompi_datatype_t * dtype)
 {
     void *restrict src1;
     void *restrict src2;
@@ -611,13 +654,36 @@ static inline void ompi_3buff_op_reduce(ompi_op_t * op, void *source1,
     src2 = source2;
     tgt = target;
 
+    if(OPAL_UNLIKELY((full_count > INT_MAX) &&
+        (0 == (op->o_flags & OMPI_OP_FLAGS_BIGCOUNT)))) {
+        size_t done_count = 0, shift, iter_count;
+        ptrdiff_t ext, lb;
+
+        ompi_datatype_get_extent(dtype, &lb, &ext);
+
+        while(done_count < full_count) {
+            if(done_count + INT_MAX > full_count) {
+                iter_count = full_count - done_count;
+            } else {
+                iter_count = INT_MAX;
+            }
+            shift = done_count * ext;
+            // Recurse one level in iterations of 'int'
+            ompi_3buff_op_reduce(op, (char*)source1 + shift, (char *)source2 + shift,
+                                (char*)target + shift, iter_count, dtype);
+            done_count += iter_count;
+        }
+        return;
+    }
+
     if (OPAL_LIKELY(ompi_op_is_intrinsic (op))) {
+        int count = (int)full_count;
         op->o_3buff_intrinsic.fns[ompi_op_ddt_map[dtype->id]](src1, src2,
                                                               tgt, &count,
                                                               &dtype,
                                                               op->o_3buff_intrinsic.modules[ompi_op_ddt_map[dtype->id]]);
     } else {
-        ompi_3buff_op_user (op, src1, src2, tgt, count, dtype);
+        ompi_3buff_op_user (op, src1, src2, tgt, full_count, dtype);
     }
 }
 
diff --git a/ompi/patterns/comm/Makefile.am b/ompi/patterns/comm/Makefile.am
deleted file mode 100644
index 9a733aff78a..00000000000
--- a/ompi/patterns/comm/Makefile.am
+++ /dev/null
@@ -1,17 +0,0 @@
-# Copyright (c) 2013 Oak Ridge National Laboratory.  All rights reserved.
-# Copyright (c) 2016      IBM Corporation.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-headers += \
-        patterns/comm/coll_ops.h \
-        patterns/comm/commpatterns.h
-
-lib@OMPI_LIBMPI_NAME@_la_SOURCES += \
-        patterns/comm/allreduce.c \
-        patterns/comm/allgather.c \
-        patterns/comm/bcast.c
diff --git a/ompi/patterns/comm/allgather.c b/ompi/patterns/comm/allgather.c
deleted file mode 100644
index 1dbaafae770..00000000000
--- a/ompi/patterns/comm/allgather.c
+++ /dev/null
@@ -1,294 +0,0 @@
-/*
- * Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
- * Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
- * Copyright (c) 2012      Los Alamos National Security, LLC.
- *                         All rights reserved.
- * Copyright (c) 2014-2017 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-/** @file */
-
-#include "ompi_config.h"
-
-#include "ompi/constants.h"
-#include "ompi/op/op.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/communicator/communicator.h"
-#include "opal/include/opal/sys/atomic.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/patterns/net/netpatterns.h"
-#include "coll_ops.h"
-
-/**
- * All-reduce - subgroup in communicator
- */
-OMPI_DECLSPEC int ompi_comm_allgather_pml(void *src_buf, void *dest_buf, int count,
-        ompi_datatype_t *dtype, int my_rank_in_group,
-        int n_peers, int *ranks_in_comm,ompi_communicator_t *comm)
-{
-    /* local variables */
-    int rc=OMPI_SUCCESS,msg_cnt;
-    int pair_rank,exchange,extra_rank, n_extra_nodes,n_extra;
-    int proc_block,extra_start,extra_end,iovec_len;
-    int remote_data_start_rank,remote_data_end_rank;
-    int local_data_start_rank;
-    netpatterns_pair_exchange_node_t my_exchange_node;
-    size_t message_extent,current_data_extent,current_data_count;
-    size_t dt_size;
-    ptrdiff_t dt_extent;
-    char *src_buf_current;
-    char *dest_buf_current;
-    struct iovec send_iov[2] = {{0,0},{0,0}},
-                 recv_iov[2] = {{0,0},{0,0}};
-    ompi_request_t *requests[4];
-
-    /* get size of data needed - same layout as user data, so that
-     *   we can apply the reudction routines directly on these buffers
-     */
-    rc = ompi_datatype_type_size(dtype, &dt_size);
-    if( OMPI_SUCCESS != rc ) {
-        goto Error;
-    }
-
-    rc = ompi_datatype_type_extent(dtype, &dt_extent);
-    if( OMPI_SUCCESS != rc ) {
-        goto Error;
-    }
-    message_extent = dt_extent*count;
-
-    /* place my data in the correct destination buffer */
-    rc=ompi_datatype_copy_content_same_ddt(dtype,count,
-            (char *)dest_buf+my_rank_in_group*message_extent,
-            (char *)src_buf);
-    if( OMPI_SUCCESS != rc ) {
-        goto Error;
-    }
-
-    /* 1 process special case */
-    if(1 == n_peers) {
-        return OMPI_SUCCESS;
-    }
-
-    /* get my reduction communication pattern */
-    memset(&my_exchange_node, 0, sizeof(netpatterns_pair_exchange_node_t));
-    rc = ompi_netpatterns_setup_recursive_doubling_tree_node(n_peers,
-            my_rank_in_group, &my_exchange_node);
-    if(OMPI_SUCCESS != rc){
-        return rc;
-    }
-
-    n_extra_nodes=n_peers-my_exchange_node.n_largest_pow_2;
-
-    /* get the data from the extra sources */
-    if(0 < my_exchange_node.n_extra_sources)  {
-
-        if ( EXCHANGE_NODE == my_exchange_node.node_type ) {
-
-            /*
-             ** Receive data from extra node
-             */
-
-            extra_rank=my_exchange_node.rank_extra_source;
-            /* receive the data into the correct location - will use 2
-             * messages in the recursive doubling phase */
-            dest_buf_current=(char *)dest_buf+message_extent*extra_rank;
-            rc=MCA_PML_CALL(recv(dest_buf_current,
-                    count,dtype,ranks_in_comm[extra_rank],
-                    -OMPI_COMMON_TAG_ALLREDUCE,
-                    comm, MPI_STATUSES_IGNORE));
-            if( 0 > rc ) {
-                goto  Error;
-            }
-
-        } else {
-
-            /*
-             ** Send data to "partner" node
-             */
-            extra_rank=my_exchange_node.rank_extra_source;
-            src_buf_current=(char *)src_buf;
-            rc=MCA_PML_CALL(send(src_buf_current,
-                    count,dtype,ranks_in_comm[extra_rank],
-                    -OMPI_COMMON_TAG_ALLREDUCE,
-                    MCA_PML_BASE_SEND_STANDARD,
-                    comm));
-            if( 0 > rc ) {
-                goto  Error;
-            }
-        }
-    }
-
-    current_data_extent=message_extent;
-    current_data_count=count;
-    src_buf_current=(char *)dest_buf+my_rank_in_group*message_extent;
-    proc_block=1;
-    local_data_start_rank=my_rank_in_group;
-    /* loop over data exchanges */
-    for(exchange=0 ; exchange < my_exchange_node.n_exchanges ; exchange++) {
-
-        /* is the remote data read */
-        pair_rank=my_exchange_node.rank_exchanges[exchange];
-        msg_cnt=0;
-
-        /*
-         * Power of 2 data segment
-         */
-        /* post non-blocking receive */
-        if(pair_rank > my_rank_in_group ){
-            recv_iov[0].iov_base=src_buf_current+current_data_extent;
-            recv_iov[0].iov_len=current_data_extent;
-            iovec_len=1;
-            remote_data_start_rank=local_data_start_rank+proc_block;
-            remote_data_end_rank=remote_data_start_rank+proc_block-1;
-        } else {
-            recv_iov[0].iov_base=src_buf_current-current_data_extent;
-            recv_iov[0].iov_len=current_data_extent;
-            iovec_len=1;
-            remote_data_start_rank=local_data_start_rank-proc_block;
-            remote_data_end_rank=remote_data_start_rank+proc_block-1;
-        }
-        /* the data from the non power of 2 ranks */
-        if(remote_data_start_rank<n_extra_nodes) {
-            /* figure out how much data is at the remote rank */
-            /* last rank with data */
-            extra_start=remote_data_start_rank;
-            extra_end=remote_data_end_rank;
-            if(extra_end >= n_extra_nodes ) {
-                /* if last rank exceeds the ranks with extra data,
-                 * adjust this.
-                 */
-                extra_end=n_extra_nodes-1;
-            }
-            /* get the number of ranks whos data is to be grabbed */
-            n_extra=extra_end-extra_start+1;
-
-            recv_iov[1].iov_base=(char *)dest_buf+
-                (extra_start+my_exchange_node.n_largest_pow_2)*message_extent;
-            recv_iov[1].iov_len=n_extra*count;
-            iovec_len=2;
-        }
-
-        rc=MCA_PML_CALL(irecv(recv_iov[0].iov_base,
-                    current_data_count,dtype,ranks_in_comm[pair_rank],
-                    -OMPI_COMMON_TAG_ALLREDUCE,
-                    comm,&(requests[msg_cnt])));
-        if( 0 > rc ) {
-            goto Error;
-        }
-        msg_cnt++;
-
-        if(iovec_len > 1 ) {
-            rc=MCA_PML_CALL(irecv(recv_iov[1].iov_base,
-                        recv_iov[1].iov_len,dtype,ranks_in_comm[pair_rank],
-                        -OMPI_COMMON_TAG_ALLREDUCE,
-                        comm,&(requests[msg_cnt])));
-            if( 0 > rc ) {
-                goto Error;
-            }
-            msg_cnt++;
-        }
-
-        /* post non-blocking send */
-        send_iov[0].iov_base=src_buf_current;
-        send_iov[0].iov_len=current_data_extent;
-        iovec_len=1;
-        /* the data from the non power of 2 ranks */
-        if(local_data_start_rank<n_extra_nodes) {
-            /* figure out how much data is at the remote rank */
-            /* last rank with data */
-            extra_start=local_data_start_rank;
-            extra_end=extra_start+proc_block-1;
-            if(extra_end >= n_extra_nodes ) {
-                /* if last rank exceeds the ranks with extra data,
-                 * adjust this.
-                 */
-                extra_end=n_extra_nodes-1;
-            }
-            /* get the number of ranks whos data is to be grabbed */
-            n_extra=extra_end-extra_start+1;
-
-            send_iov[1].iov_base=(char *)dest_buf+
-                (extra_start+my_exchange_node.n_largest_pow_2)*message_extent;
-            send_iov[1].iov_len=n_extra*count;
-            iovec_len=2;
-        }
-
-        rc=MCA_PML_CALL(isend(send_iov[0].iov_base,
-                    current_data_count,dtype,ranks_in_comm[pair_rank],
-                    -OMPI_COMMON_TAG_ALLREDUCE,MCA_PML_BASE_SEND_STANDARD,
-                    comm,&(requests[msg_cnt])));
-        if( 0 > rc ) {
-            goto Error;
-        }
-        msg_cnt++;
-        if( iovec_len > 1 ) {
-            rc=MCA_PML_CALL(isend(send_iov[1].iov_base,
-                        send_iov[1].iov_len,dtype,ranks_in_comm[pair_rank],
-                        -OMPI_COMMON_TAG_ALLREDUCE,MCA_PML_BASE_SEND_STANDARD,
-                        comm,&(requests[msg_cnt])));
-            if( 0 > rc ) {
-                goto Error;
-            }
-            msg_cnt++;
-        }
-
-        /* prepare the source buffer for the next iteration */
-        if(pair_rank < my_rank_in_group ){
-            src_buf_current-=current_data_extent;
-            local_data_start_rank-=proc_block;
-        }
-        proc_block*=2;
-        current_data_extent*=2;
-        current_data_count*=2;
-
-        /* wait on send and receive completion */
-        ompi_request_wait_all(msg_cnt,requests,MPI_STATUSES_IGNORE);
-    }
-
-    /* copy data in from the "extra" source, if need be */
-    if(0 < my_exchange_node.n_extra_sources)  {
-
-        if ( EXTRA_NODE == my_exchange_node.node_type ) {
-            /*
-             ** receive the data
-             ** */
-            extra_rank=my_exchange_node.rank_extra_source;
-
-            rc=MCA_PML_CALL(recv(dest_buf,
-                    count*n_peers,dtype,ranks_in_comm[extra_rank],
-                    -OMPI_COMMON_TAG_ALLREDUCE,
-                    comm,MPI_STATUSES_IGNORE));
-            if(0 > rc ) {
-                goto  Error;
-            }
-        } else {
-            /* send the data to the pair-rank outside of the power of 2 set
-             ** of ranks
-             */
-
-            extra_rank=my_exchange_node.rank_extra_source;
-            rc=MCA_PML_CALL(send(dest_buf,
-                    count*n_peers,dtype,ranks_in_comm[extra_rank],
-                    -OMPI_COMMON_TAG_ALLREDUCE,
-                    MCA_PML_BASE_SEND_STANDARD,
-                    comm));
-            if( 0 > rc ) {
-                goto  Error;
-            }
-        }
-    }
-
-    ompi_netpatterns_cleanup_recursive_doubling_tree_node(&my_exchange_node);
-
-    /* return */
-    return OMPI_SUCCESS;
-
-Error:
-    return rc;
-}
diff --git a/ompi/patterns/comm/allreduce.c b/ompi/patterns/comm/allreduce.c
deleted file mode 100644
index 7bd779a3554..00000000000
--- a/ompi/patterns/comm/allreduce.c
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
- * Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
- * Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
- * Copyright (c) 2012      Los Alamos National Security, LLC.
- *                         All rights reserved.
- * Copyright (c) 2014-2017 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-/** @file */
-
-#include "ompi_config.h"
-
-#include "ompi/constants.h"
-#include "ompi/op/op.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/communicator/communicator.h"
-#include "opal/include/opal/sys/atomic.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/patterns/net/netpatterns.h"
-#include "ompi/mca/coll/base/coll_base_util.h"
-#include "coll_ops.h"
-#include "commpatterns.h"
-
-/**
- * All-reduce for contigous primitive types
- */
-OMPI_DECLSPEC int ompi_comm_allreduce_pml(void *sbuf, void *rbuf, int count,
-        ompi_datatype_t *dtype, int my_rank_in_group,
-        struct ompi_op_t *op, int n_peers,int *ranks_in_comm,
-        ompi_communicator_t *comm)
-{
-    /* local variables */
-    int rc=OMPI_SUCCESS,n_dts_per_buffer,n_data_segments,stripe_number;
-    int pair_rank,exchange,extra_rank;
-    netpatterns_pair_exchange_node_t my_exchange_node;
-    int count_processed,count_this_stripe;
-    size_t dt_size,dt_extent;
-    char scratch_bufers[2][MAX_TMP_BUFFER];
-    int send_buffer=0,recv_buffer=1;
-    char *sbuf_current, *rbuf_current;
-
-    /* get size of data needed - same layout as user data, so that
-     *   we can apply the reudction routines directly on these buffers
-     */
-    rc = opal_datatype_type_size((opal_datatype_t *)dtype, &dt_size);
-    if( OMPI_SUCCESS != rc ) {
-        goto Error;
-    }
-    rc = ompi_datatype_type_extent(dtype, (ptrdiff_t *)&dt_extent);
-    if( OMPI_SUCCESS != rc ) {
-        goto Error;
-    }
-
-    /* 1 process special case */
-    if(1 == n_peers) {
-        /* place my data in the correct destination buffer */
-        rc=ompi_datatype_copy_content_same_ddt(dtype,count,
-                (char *)rbuf, (char *)sbuf);
-        if( OMPI_SUCCESS != rc ) {
-            goto Error;
-        }
-        return OMPI_SUCCESS;
-    }
-
-    /* number of data types copies that the scratch buffer can hold */
-    n_dts_per_buffer=((int) MAX_TMP_BUFFER)/dt_extent;
-    if ( 0 == n_dts_per_buffer ) {
-        rc=OMPI_ERROR;
-        goto Error;
-    }
-
-    /* compute number of stripes needed to process this collective */
-    n_data_segments=(count+n_dts_per_buffer -1 ) / n_dts_per_buffer ;
-
-    /* get my reduction communication pattern */
-    memset(&my_exchange_node, 0, sizeof(netpatterns_pair_exchange_node_t));
-    rc = ompi_netpatterns_setup_recursive_doubling_tree_node(n_peers,
-            my_rank_in_group, &my_exchange_node);
-    if(OMPI_SUCCESS != rc){
-        return rc;
-    }
-
-    count_processed=0;
-
-    /* get a pointer to the shared-memory working buffer */
-    /* NOTE: starting with a rather synchronous approach */
-    for( stripe_number=0 ; stripe_number < n_data_segments ; stripe_number++ ) {
-
-        /* get number of elements to process in this stripe */
-        count_this_stripe=n_dts_per_buffer;
-        if( count_processed + count_this_stripe > count )
-            count_this_stripe=count-count_processed;
-
-        /* copy data from the input buffer into the temp buffer */
-        sbuf_current=(char *)sbuf+count_processed*dt_extent;
-        rc=ompi_datatype_copy_content_same_ddt(dtype,count_this_stripe,
-                scratch_bufers[send_buffer], sbuf_current);
-        if( OMPI_SUCCESS != rc ) {
-            goto Error;
-        }
-
-        /* copy data in from the "extra" source, if need be */
-        if(0 < my_exchange_node.n_extra_sources)  {
-
-            if ( EXCHANGE_NODE == my_exchange_node.node_type ) {
-
-                /*
-                ** Receive data from extra node
-                */
-                extra_rank=my_exchange_node.rank_extra_source;
-                rc=MCA_PML_CALL(recv(scratch_bufers[recv_buffer],
-                            count_this_stripe,dtype,ranks_in_comm[extra_rank],
-                            -OMPI_COMMON_TAG_ALLREDUCE, comm,
-                            MPI_STATUSES_IGNORE));
-                if( 0 > rc ) {
-                    fprintf(stderr,"  first recv failed in ompi_comm_allreduce_pml \n");
-                    fflush(stderr);
-                    goto  Error;
-                }
-
-
-                /* apply collective operation to first half of the data */
-                if( 0 < count_this_stripe ) {
-                    ompi_op_reduce(op,
-                            (void *)scratch_bufers[send_buffer],
-                            (void *)scratch_bufers[recv_buffer],
-                            count_this_stripe,dtype);
-                }
-
-
-            } else {
-
-                /*
-                ** Send data to "partner" node
-                */
-                extra_rank=my_exchange_node.rank_extra_source;
-                rc=MCA_PML_CALL(send(scratch_bufers[send_buffer],
-                            count_this_stripe,dtype,ranks_in_comm[extra_rank],
-                            -OMPI_COMMON_TAG_ALLREDUCE, MCA_PML_BASE_SEND_STANDARD,
-                            comm));
-                if( 0 > rc ) {
-                    fprintf(stderr,"  first send failed in ompi_comm_allreduce_pml \n");
-                    fflush(stderr);
-                    goto  Error;
-                }
-            }
-
-            /* change pointer to scratch buffer - this was we can send data
-            ** that we have summed w/o a memory copy, and receive data into the
-            ** other buffer, w/o fear of over writting data that has not yet
-            ** completed being send
-            */
-            recv_buffer^=1;
-            send_buffer^=1;
-        }
-
-        /* loop over data exchanges */
-        for(exchange=0 ; exchange < my_exchange_node.n_exchanges ; exchange++) {
-
-            /* is the remote data read */
-            pair_rank=my_exchange_node.rank_exchanges[exchange];
-
-            rc=ompi_coll_base_sendrecv_actual(scratch_bufers[send_buffer],
-                                              count_this_stripe,dtype, ranks_in_comm[pair_rank],
-                                              -OMPI_COMMON_TAG_ALLREDUCE,
-                                              scratch_bufers[recv_buffer],
-                                              count_this_stripe,dtype,ranks_in_comm[pair_rank],
-                                              -OMPI_COMMON_TAG_ALLREDUCE,
-                                              comm, MPI_STATUS_IGNORE);
-            if( 0 > rc ) {
-                fprintf(stderr,"  irecv failed in  ompi_comm_allreduce_pml at iterations %d \n",
-                        exchange);
-                fflush(stderr);
-                goto Error;
-            }
-
-            /* reduce the data */
-            if( 0 < count_this_stripe ) {
-                ompi_op_reduce(op,
-                        (void *)scratch_bufers[send_buffer],
-                        (void *)scratch_bufers[recv_buffer],
-                        count_this_stripe,dtype);
-            }
-            /* get ready for next step */
-            recv_buffer^=1;
-            send_buffer^=1;
-
-        }
-
-        /* copy data in from the "extra" source, if need be */
-        if(0 < my_exchange_node.n_extra_sources)  {
-
-            if ( EXTRA_NODE == my_exchange_node.node_type ) {
-                /*
-                ** receive the data
-                ** */
-                extra_rank=my_exchange_node.rank_extra_source;
-                rc=MCA_PML_CALL(recv(scratch_bufers[recv_buffer],
-                            count_this_stripe,dtype,ranks_in_comm[extra_rank],
-                            -OMPI_COMMON_TAG_ALLREDUCE, comm,
-                            MPI_STATUSES_IGNORE));
-                if( 0 > rc ) {
-                    fprintf(stderr,"  last recv failed in ompi_comm_allreduce_pml \n");
-                    fflush(stderr);
-                    goto  Error;
-                }
-
-                recv_buffer^=1;
-                send_buffer^=1;
-            } else {
-                /* send the data to the pair-rank outside of the power of 2 set
-                ** of ranks
-                */
-
-                extra_rank=my_exchange_node.rank_extra_source;
-                rc=MCA_PML_CALL(send((char *)scratch_bufers[send_buffer],
-                            count_this_stripe,dtype,ranks_in_comm[extra_rank],
-                            -OMPI_COMMON_TAG_ALLREDUCE, MCA_PML_BASE_SEND_STANDARD,
-                            comm));
-                if( 0 > rc ) {
-                    fprintf(stderr,"  last send failed in ompi_comm_allreduce_pml \n");
-                    fflush(stderr);
-                    goto  Error;
-                }
-            }
-        }
-
-        /* copy data from the temp buffer into the output buffer */
-        rbuf_current = (char *) rbuf + count_processed * dt_size;
-        memcpy(rbuf_current,scratch_bufers[send_buffer], count_this_stripe*dt_size);
-
-        /* update the count of elements processed */
-        count_processed += count_this_stripe;
-    }
-
-    ompi_netpatterns_cleanup_recursive_doubling_tree_node(&my_exchange_node);
-
-    /* return */
-    return OMPI_SUCCESS;
-
-Error:
-    return rc;
-}
diff --git a/ompi/patterns/comm/bcast.c b/ompi/patterns/comm/bcast.c
deleted file mode 100644
index bc54613cc01..00000000000
--- a/ompi/patterns/comm/bcast.c
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
- * Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
- * Copyright (c) 2012      Los Alamos National Security, LLC.
- *                         All rights reserved.
- * Copyright (c) 2014      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-/** @file */
-
-#include "ompi_config.h"
-
-#include "ompi/constants.h"
-#include "ompi/op/op.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/communicator/communicator.h"
-#include "opal/include/opal/sys/atomic.h"
-#include "ompi/mca/pml/pml.h"
-#include "ompi/patterns/net/netpatterns.h"
-#include "coll_ops.h"
-
-/**
- * Bcast - subgroup in communicator
- *  This is a very simple algorithm - binary tree, transmitting the full
- *  message at each step.
- */
-OMPI_DECLSPEC int ompi_comm_bcast_pml(void *buffer, int root, int count,
-        ompi_datatype_t *dtype, int my_rank_in_group,
-        int n_peers, int *ranks_in_comm,ompi_communicator_t *comm)
-{
-    /* local variables */
-    int rc=OMPI_SUCCESS,msg_cnt,i;
-    ompi_request_t *requests[2];
-    int node_rank, peer_rank;
-    netpatterns_tree_node_t node_data;
-
-    /*
-     * shift rank to root==0 tree
-     */
-    node_rank=(my_rank_in_group-root+n_peers)%n_peers;
-
-    /*
-     * compute my communication pattern - binary tree
-     */
-    rc=ompi_netpatterns_setup_narray_tree(2, node_rank, n_peers,
-            &node_data);
-    if( OMPI_SUCCESS != rc ) {
-        goto Error;
-    }
-
-    /* 1 process special case */
-    if(1 == n_peers) {
-        return OMPI_SUCCESS;
-    }
-
-    /* if I have parents - wait on the data to arrive */
-    if(node_data.n_parents) {
-        /* I will have only 1 parent */
-        peer_rank=node_data.parent_rank;
-        peer_rank=(peer_rank+root)%n_peers;
-        /* translate back to actual rank */
-        rc=MCA_PML_CALL(recv(buffer, count,dtype,peer_rank,
-                    -OMPI_COMMON_TAG_BCAST, comm, MPI_STATUSES_IGNORE));
-        if( 0 > rc ) {
-            goto Error;
-        }
-    }
-
-    /* send the data to my children */
-    msg_cnt=0;
-    for(i=0 ; i < node_data.n_children ; i++ ) {
-        peer_rank=node_data.children_ranks[i];
-        peer_rank=(peer_rank+root)%n_peers;
-        rc=MCA_PML_CALL(isend(buffer,
-                    count,dtype,peer_rank,
-                    -OMPI_COMMON_TAG_BCAST,MCA_PML_BASE_SEND_STANDARD,
-                    comm,&(requests[msg_cnt])));
-        if( 0 > rc ) {
-            goto Error;
-        }
-        msg_cnt++;
-    }
-    /* wait for send completion */
-    if(msg_cnt) {
-        /* wait on send and receive completion */
-        ompi_request_wait_all(msg_cnt,requests,MPI_STATUSES_IGNORE);
-    }
-
-    if (node_data.children_ranks) {
-        free(node_data.children_ranks);
-    }
-
-    /* return */
-    return OMPI_SUCCESS;
-
-Error:
-    return rc;
-}
diff --git a/ompi/patterns/comm/coll_ops.h b/ompi/patterns/comm/coll_ops.h
deleted file mode 100644
index 5acb66c1e69..00000000000
--- a/ompi/patterns/comm/coll_ops.h
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
- * Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
- * Copyright (c) 2012      Los Alamos National Security, LLC.
- *                         All rights reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef COMM_COLL_OP_TYPES_H
-#define COMM_COLL_OP_TYPES_H
-
-#include "ompi_config.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/proc/proc.h"
-
-BEGIN_C_DECLS
-
-#define OMPI_COMMON_TAG_ALLREDUCE 99
-#define OMPI_COMMON_TAG_BCAST     98
-
-
-
-
-OMPI_DECLSPEC int ompi_comm_allgather_pml(void *src_buf, void *dest_buf, int count,
-        ompi_datatype_t *dtype, int my_rank_in_group, int n_peers,
-        int *ranks_in_comm,ompi_communicator_t *comm);
-OMPI_DECLSPEC int ompi_comm_allreduce_pml(void *sbuf, void *rbuf, int count,
-        ompi_datatype_t *dtype, int my_rank_in_group,
-        struct ompi_op_t *op, int n_peers,int *ranks_in_comm,
-        ompi_communicator_t *comm);
-OMPI_DECLSPEC int ompi_comm_bcast_pml(void *buffer, int root, int count,
-        ompi_datatype_t *dtype, int my_rank_in_group,
-        int n_peers, int *ranks_in_comm,ompi_communicator_t
-        *comm);
-
-/* reduction operations supported */
-#define OP_SUM 1
-#define OP_MAX 2
-#define OP_MIN 3
-
-#define TYPE_INT4 1
-
-
-END_C_DECLS
-
-#endif /* COMM_COLL_OP_TYPES_H */
diff --git a/ompi/patterns/comm/commpatterns.h b/ompi/patterns/comm/commpatterns.h
deleted file mode 100644
index 2e05757f111..00000000000
--- a/ompi/patterns/comm/commpatterns.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
- * Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef COMM_NETPATTERNS_H
-#define COMM_NETPATTERNS_H
-
-#include "ompi_config.h"
-
-BEGIN_C_DECLS
-
-#define MAX_TMP_BUFFER            8192
-
-END_C_DECLS
-
-#endif /* COMM_NETPATTERNS_H */
diff --git a/ompi/patterns/net/Makefile.am b/ompi/patterns/net/Makefile.am
deleted file mode 100644
index d9b07fd1422..00000000000
--- a/ompi/patterns/net/Makefile.am
+++ /dev/null
@@ -1,19 +0,0 @@
-# Copyright (c) 2013 Oak Ridge National Laboratory.  All rights reserved.
-# Copyright (c) 2016      IBM Corporation.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-headers += \
-        patterns/net/netpatterns.h \
-        patterns/net/netpatterns_knomial_tree.h \
-        patterns/net/coll_ops.h
-
-lib@OMPI_LIBMPI_NAME@_la_SOURCES += \
-		patterns/net/netpatterns_base.c \
-		patterns/net/netpatterns_multinomial_tree.c \
-		patterns/net/netpatterns_nary_tree.c \
-		patterns/net/netpatterns_knomial_tree.c
diff --git a/ompi/patterns/net/allreduce.c b/ompi/patterns/net/allreduce.c
deleted file mode 100644
index be192255507..00000000000
--- a/ompi/patterns/net/allreduce.c
+++ /dev/null
@@ -1,349 +0,0 @@
-/*
- * Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
- * Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
- * Copyright (c) 2012      Los Alamos National Security, LLC.
- *                         All rights reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
- * Copyright (c) 2019      Intel, Inc.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-/** @file */
-
-#include "ompi_config.h"
-
-#include "ompi/constants.h"
-#include "coll_sm2.h"
-#include "ompi/op/op.h"
-#include "ompi/datatype/ompi_datatype.h"
-#include "ompi/communicator/communicator.h"
-#include "ompi/runtime/ompi_rte.h"
-
-void send_completion(nt status, struct ompi_process_name_t* peer, struct iovec* msg,
-                     int count, ompi_rml_tag_t tag, void* cbdata)
-{
-    /* set send completion flag */
-    *(int *)cbdata=1;
-}
-
-
-void recv_completion(nt status, struct ompi_process_name_t* peer, struct iovec* msg,
-                     int count, ompi_rml_tag_t tag, void* cbdata)
-{
-    /* set receive completion flag */
-    MB();
-    *(int *)cbdata=1;
-}
-
-
-static void op_reduce(int op_type,(void *)src_dest_buf,(void *) src_buf, int count,
-        int data_type)
-{
-    /* local variables */
-    int ret;
-
-    /* op type */
-    switch (op_type) {
-
-        case OP_SUM:
-
-
-            switch (data_type) {
-                case TYPE_INT4:
-                    int *int_src_ptr=(int *)src_ptr;
-                    int *int_src_dst_ptr=(int *)src_dst_ptr;
-                    int cnt;
-                    for(cnt=0 ; cnt < count ; ) {
-                        (*(int_src_dst_ptr))+=(*(int_src_ptr));
-                    break;
-                default:
-                    ret=OMPI_ERROR;
-                    goto Error;
-            }
-
-            break;
-
-        default:
-        ret=OMPI_ERROR;
-        goto Error;
-    }
-Error:
-    return ret;
-}
-
-/**
- * All-reduce for contigous primitive types
- */
-static
-comm_allreduce(void *sbuf, void *rbuf, int count, opal_datatype_t *dtype,
-        int op_type, opal_list_t *peers)
-{
-    /* local variables */
-    int rc=OMPI_SUCCESS,n_dts_per_buffer,n_data_segments,stripe_number;
-    int pair_rank,exchange,extra_rank;
-    int index_read,index_write;
-    netpatterns_pair_exchange_node_t my_exchange_node;
-    int my_rank,count_processed,count_this_stripe;
-    size_t n_peers,message_extent,len_data_buffer;
-    size_t dt_size;
-    long long tag, base_tag;
-    sm_work_buffer_t *sm_buffer_desc;
-    opal_list_item_t *item;
-    char scratch_bufers[2][MAX_TMP_BUFFER];
-    int send_buffer=0;recv_buffer=1;
-    char *sbuf_current,*rbuf_current;
-    ompi_proc_t **proc_array;
-    struct iovec send_iov, recv_iov;
-    volatile int *recv_done, *send_done;
-    int recv_completion_flag, send_completion_flag;
-    int data_type;
-
-    /* get size of data needed - same layout as user data, so that
-     *   we can apply the reudction routines directly on these buffers
-     */
-    rc=opal_datatype_type_size(dtype, &dt_size);
-    if( OMPI_SUCCESS != rc ) {
-        goto Error;
-    }
-    message_extent=dt_extent*count;
-
-    /* lenght of control and data regions */
-    len_data_buffer=sm_module->data_memory_per_proc_per_segment;
-
-    /* number of data types copies that the scratch buffer can hold */
-    n_dts_per_buffer=((int) MAX_TMP_BUFFER)/dt_size;
-    if ( 0 == n_dts_per_buffer ) {
-        rc=OMPI_ERROR;
-        goto Error;
-    }
-
-    /* need a read and a write buffer for a pair-wise exchange of data */
-    n_dts_per_buffer/=2;
-    len_data_buffer=n_dts_per_buffer*dt_size;
-
-    /* compute number of stripes needed to process this collective */
-    n_data_segments=(count+n_dts_per_buffer -1 ) / n_dts_per_buffer ;
-
-    /* */
-    n_peers=opal_list_get_size(peers);
-
-    /* get my rank in the list */
-    my_rank=0;
-    for (item = opal_list_get_first(peers) ;
-            item != opal_list_get_end(peers) ;
-            item = opal_list_get_next(peers)) {
-        if(ompi_proc_local()==(ompi_proc_t *)item){
-            /* this is the pointer to my proc strucuture */
-            break;
-        }
-        my_rank++;
-    }
-    proc_array=(ompi_proc_t **)malloc(sizeof(ompi_proc_t *)*n_peers);
-    if( NULL == proc_array) {
-        goto Error;
-    }
-    cnt=0;
-    for (item = opal_list_get_first(peers) ;
-            item != opal_list_get_end(peers) ;
-            item = opal_list_get_next(peers)) {
-        proc_array[cnt]=(ompi_proc_t *)item;
-        cnt++;
-    }
-
-    /* get my reduction communication pattern */
-    ret=ompi_netpatterns_setup_recursive_doubling_tree_node(n_peers,my_rank,&my_exchange_node);
-    if(OMPI_SUCCESS != ret){
-        return ret;
-    }
-
-    /* setup flags for non-blocking communications */
-    recv_done=&recv_completion_flag;
-    send_done=&send_completion_flag;
-
-    /* set data type */
-    if(&opal_datatype_int4==dtype) {
-        data_type=TYPE_INT4;
-    }
-
-    count_processed=0;
-
-    /* get a pointer to the shared-memory working buffer */
-    /* NOTE: starting with a rather synchronous approach */
-    for( stripe_number=0 ; stripe_number < n_data_segments ; stripe_number++ ) {
-
-        /* get number of elements to process in this stripe */
-        count_this_stripe=n_dts_per_buffer;
-        if( count_processed + count_this_stripe > count )
-            count_this_stripe=count-count_processed;
-
-        /* copy data from the input buffer into the temp buffer */
-        sbuf_current=(char *)sbuf+count_processed*dt_size;
-        memcopy(scratch_bufers[send_buffer],sbuf_current,count_this_stripe*dt_size);
-
-        /* copy data in from the "extra" source, if need be */
-        if(0 < my_exchange_node->n_extra_sources)  {
-
-            if ( EXCHANGE_NODE == my_exchange_node->node_type ) {
-
-                /*
-                ** Receive data from extra node
-                */
-
-                extra_rank=my_exchange_node.rank_extra_source;
-                recv_iov.iov_base=scratch_bufers[recv_buffer];
-                recv_iov.iov_len=count_this_stripe*dt_size;
-                rc = ompi_rte_recv(&(proc_array[extra_rank]->proc_name), &recv_iov, 1,
-                        OMPI_RML_TAG_ALLREDUCE , 0);
-                if(OMPI_SUCCESS != rc ) {
-                    goto  Error;
-                }
-
-                /* apply collective operation to first half of the data */
-                if( 0 < count_this_stripe ) {
-                    op_reduce(op_type,(void *)scratch_bufers[recv_buffer],
-                            (void *)scratch_bufers[send_buffer], n_my_count,TYPE_INT4);
-                }
-
-
-            } else {
-
-                /*
-                ** Send data to "partner" node
-                */
-                extra_rank=my_exchange_node.rank_extra_source;
-                send_iov.iov_base=scratch_bufers[send_buffer];
-                send_iov.iov_len=count_this_stripe*dt_size;
-                rc = ompi_rte_send(&(proc_array[extra_rank]->proc_name), &send_iov, 1,
-                        OMPI_RML_TAG_ALLREDUCE , 0);
-                if(OMPI_SUCCESS != rc ) {
-                    goto  Error;
-                }
-            }
-
-            /* change pointer to scratch buffer - this was we can send data
-            ** that we have summed w/o a memory copy, and receive data into the
-            ** other buffer, w/o fear of over writting data that has not yet
-            ** completed being send
-            */
-            recv_buffer^=1;
-            send_buffer^=1;
-        }
-
-        MB();
-        /*
-         * Signal parent that data is ready
-         */
-        tag=base_tag+1;
-        my_ctl_pointer->flag=tag;
-
-        /* loop over data exchanges */
-        for(exchange=0 ; exchange < my_exchange_node->n_exchanges ; exchange++) {
-
-            /* debug
-            t4=opal_sys_timer_get_cycles();
-             end debug */
-
-
-            my_write_pointer=my_tmp_data_buffer[index_write];
-            my_read_pointer=my_tmp_data_buffer[index_read];
-
-            /* is the remote data read */
-            pair_rank=my_exchange_node->rank_exchanges[exchange];
-
-            *recv_done=0;
-            *send_done=0;
-            MB();
-
-            /* post non-blocking receive */
-            recv_iov.iov_base=scratch_bufers[send_buffer];
-            recv_iov.iov_len=count_this_stripe*dt_size;
-            rc = ompi_rte_recv_nb(&(proc_array[extra_rank]->proc_name), recv_iov, 1,
-                        OMPI_RML_TAG_ALLREDUCE , 0, recv_completion, recv_done);
-
-            /* post non-blocking send */
-            send_iov.iov_base=scratch_bufers[send_buffer];
-            send_iov.iov_len=count_this_stripe*dt_size;
-            rc = ompi_rte_send_nb(&(proc_array[extra_rank]->proc_name), send_iov, 1,
-                        OMPI_RML_TAG_ALLREDUCE , 0, send_completion, send_done);
-
-            /* wait on receive completion */
-            while(!(*recv_done) ) {
-                opal_progress();
-            }
-
-            /* reduce the data */
-            if( 0 < count_this_stripe ) {
-                op_reduce(op_type,(void *)scratch_bufers[recv_buffer],
-                        (void *)scratch_bufers[send_buffer], n_my_count,TYPE_INT4);
-            }
-
-
-            /* get ready for next step */
-            index_read=(exchange&1);
-            index_write=((exchange+1)&1);
-
-            /* wait on send completion */
-            while(!(*send_done) ) {
-                opal_progress();
-            }
-
-        }
-
-        /* copy data in from the "extra" source, if need be */
-        if(0 < my_exchange_node->n_extra_sources)  {
-
-            if ( EXTRA_NODE == my_exchange_node->node_type ) {
-                /*
-                ** receive the data
-                ** */
-                extra_rank=my_exchange_node->rank_extra_source;
-
-                recv_iov.iov_base=scratch_bufers[recv_buffer];
-                recv_iov.iov_len=count_this_stripe*dt_size;
-                rc = ompi_rte_recv(&(proc_array[extra_rank]->proc_name), &recv_iov, 1,
-                        OMPI_RML_TAG_ALLREDUCE , 0);
-                if(OMPI_SUCCESS != rc ) {
-                    goto  Error;
-                }
-
-            } else {
-                /* send the data to the pair-rank outside of the power of 2 set
-                ** of ranks
-                */
-
-                extra_rank=my_exchange_node->rank_extra_source;
-                send_iov.iov_base=scratch_bufers[recv_buffer];
-                send_iov.iov_len=count_this_stripe*dt_size;
-                rc = ompi_rte_recv(&(proc_array[extra_rank]->proc_name), &send_iov, 1,
-                        OMPI_RML_TAG_ALLREDUCE , 0);
-                if(OMPI_SUCCESS != rc ) {
-                    goto  Error;
-                }
-            }
-        }
-
-        /* copy data into the destination buffer */
-        rc=ompi_datatype_copy_content_same_ddt(dtype, count_this_stripe,
-                (char *)((char *)rbuf+dt_extent*count_processed),
-                (char *)my_write_pointer);
-        if( 0 != rc ) {
-            return OMPI_ERROR;
-        }
-
-        /* copy data from the temp buffer into the output buffer */
-        rbuf_current=(char *)rbuf+count_processed*dt_size;
-        memcopy(scratch_bufers[recv_buffer],rbuf_current,count_this_stripe*dt_size);
-
-        /* update the count of elements processed */
-        count_processed+=count_this_stripe;
-    }
-
-    /* return */
-    return rc;
-
-Error:
-    return rc;
-}
diff --git a/ompi/patterns/net/coll_ops.h b/ompi/patterns/net/coll_ops.h
deleted file mode 100644
index 5e52734d284..00000000000
--- a/ompi/patterns/net/coll_ops.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
- * Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef COMM_OP_TYPES_H
-#define COMM_OP_TYPES_H
-
-#include "ompi_config.h"
-
-BEGIN_C_DECLS
-
-int comm_allreduce(void *sbuf, void *rbuf, int count, opal_datatype_t *dtype,
-                int op, opal_list_t *peers);
-
-/* reduction operations supported */
-#define OP_SUM 1
-
-#define TYPE_INT4 1
-
-
-END_C_DECLS
-
-#endif /* COMM_OP_TYPES_H */
diff --git a/ompi/patterns/net/netpatterns.h b/ompi/patterns/net/netpatterns.h
deleted file mode 100644
index 9307e247e0c..00000000000
--- a/ompi/patterns/net/netpatterns.h
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
- * Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
- * Copyright (c) 2012      Los Alamos National Security, LLC.
- *                         All rights reserved.
-  * Copyright (c) 2017      IBM Corporation. All rights reserved.
-  * Copyright (c) 2019      Intel, Inc.  All rights reserved.
-  * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef COMM_PATTERNS_H
-#define COMM_PATTERNS_H
-
-#include "ompi_config.h"
-
-#include "ompi/runtime/ompi_rte.h"
-#include "netpatterns_knomial_tree.h"
-
-BEGIN_C_DECLS
-
-int ompi_netpatterns_base_err(const char* fmt, ...);
-int ompi_netpatterns_register_mca_params(void);
-
-#if OPAL_ENABLE_DEBUG
-extern int ompi_netpatterns_base_verbose; /* disabled by default */
-OMPI_DECLSPEC extern int ompi_netpatterns_base_err(const char*, ...) __opal_attribute_format__(__printf__, 1, 2);
-#define NETPATTERNS_VERBOSE(args)                                \
-    do {                                                         \
-        if(ompi_netpatterns_base_verbose > 0) {           \
-            ompi_netpatterns_base_err("[%s]%s[%s:%d:%s] ",\
-                    ompi_process_info.nodename,                  \
-                    OMPI_NAME_PRINT(OMPI_PROC_MY_NAME),          \
-                    __FILE__, __LINE__, __func__);               \
-            ompi_netpatterns_base_err args;               \
-            ompi_netpatterns_base_err("\n");              \
-        }                                                        \
-    } while(0);
-#else
-#define NETPATTERNS_VERBOSE(args)
-#endif
-
-#define FIND_BASE(base,myid,level,k)    \
-    do {                                \
-        int temp = 1;                   \
-        int jj;                         \
-        int knt2;                       \
-                                        \
-        base = 0;                       \
-        for( jj = 0; jj < level; jj++) {\
-            temp *= k;                  \
-        }                               \
-        knt2 = 1;                       \
-        while(myid >= knt2*temp){       \
-            knt2++;                     \
-        }                               \
-        base = knt2*temp - temp;        \
-    } while(0)                          \
-
-
-
-
-/* enum for node type */
-enum {
-    ROOT_NODE,
-    LEAF_NODE,
-    INTERIOR_NODE
-};
-
-
-/*
- * N-order tree node description
- */
-struct netpatterns_tree_node_t {
-    /* my rank within the group */
-    int my_rank;
-    /* my node type - root, leaf, or interior */
-    int my_node_type;
-    /* number of nodes in the tree */
-    int tree_size;
-    /* number of parents (0/1) */
-    int n_parents;
-    /* number of children */
-    int n_children;
-    /* parent rank within the group */
-    int parent_rank;
-    /* chidren ranks within the group */
-    int *children_ranks;
-};
-typedef struct netpatterns_tree_node_t netpatterns_tree_node_t;
-
-struct netpatterns_k_exchange_node_t;
-/*
- * N-order + knominal tree node description
- */
-struct netpatterns_narray_knomial_tree_node_t {
-    /* my rank within the group */
-    int my_rank;
-    /* my node type - root, leaf, or interior */
-    int my_node_type;
-    /* number of nodes in the tree */
-    int tree_size;
-    /* number of parents (0/1) */
-    int n_parents;
-    /* number of children */
-    int n_children;
-    /* parent rank within the group */
-    int parent_rank;
-    /* chidren ranks within the group */
-    int *children_ranks;
-    /* Total number of ranks on this specific level */
-    int level_size;
-    /* Rank on this node inside of level */
-    int rank_on_level;
-    /* Knomial recursive gather information */
-    struct netpatterns_k_exchange_node_t k_node;
-};
-typedef struct netpatterns_narray_knomial_tree_node_t
-netpatterns_narray_knomial_tree_node_t;
-
-
-/* Init code for common_netpatterns */
-OMPI_DECLSPEC int ompi_netpatterns_init(void);
-
-/* setup an n-array tree */
-OMPI_DECLSPEC int ompi_netpatterns_setup_narray_tree(int tree_order, int my_rank, int num_nodes,
-        netpatterns_tree_node_t *my_node);
-/* setup an n-array tree with k-nomial levels */
-OMPI_DECLSPEC int ompi_netpatterns_setup_narray_knomial_tree( int tree_order, int my_rank, int num_nodes,
-        netpatterns_narray_knomial_tree_node_t *my_node);
-/* cleanup an n-array tree setup by the above function */
-OMPI_DECLSPEC void ompi_netpatterns_cleanup_narray_knomial_tree (netpatterns_narray_knomial_tree_node_t *my_node);
-
-/* setup an multi-nomial tree - for each node in the tree
- *  this returns it's parent, and it's children
- */
-OMPI_DECLSPEC int ompi_netpatterns_setup_multinomial_tree(int tree_order, int num_nodes,
-        netpatterns_tree_node_t *tree_nodes);
-
-OMPI_DECLSPEC int ompi_netpatterns_setup_narray_tree_contigous_ranks(int tree_order,
-        int num_nodes, netpatterns_tree_node_t **tree_nodes);
-
-/* calculate the nearest power of radix that is equal to or greater
- * than size, with the specified radix.  The resulting tree is of
- * depth n_lvls.
- */
-OMPI_DECLSPEC int ompi_roundup_to_power_radix( int radix, int size, int *n_lvls );
-
-END_C_DECLS
-
-#endif /* COMM_PATTERNS_H */
diff --git a/ompi/patterns/net/netpatterns_base.c b/ompi/patterns/net/netpatterns_base.c
deleted file mode 100644
index 62669533a1e..00000000000
--- a/ompi/patterns/net/netpatterns_base.c
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- *
- * Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
- * Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include <stdarg.h>
-
-#include "ompi/include/ompi/constants.h"
-#include "netpatterns.h"
-
-int ompi_netpatterns_base_verbose = 0; /* disabled by default */
-
-int ompi_netpatterns_register_mca_params(void)
-{
-    ompi_netpatterns_base_verbose = 0;
-    mca_base_var_register("ompi", "common", "netpatterns", "base_verbose",
-                          "Verbosity level of the NETPATTERNS framework",
-                          MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                          OPAL_INFO_LVL_9,
-                          MCA_BASE_VAR_SCOPE_READONLY,
-                          &ompi_netpatterns_base_verbose);
-
-    return OMPI_SUCCESS;
-}
-
-int ompi_netpatterns_base_err(const char* fmt, ...)
-{
-    va_list list;
-    int ret;
-
-    va_start(list, fmt);
-    ret = vfprintf(stderr, fmt, list);
-    va_end(list);
-    return ret;
-}
-
-int ompi_netpatterns_init(void)
-{
-/* There is no component for common_netpatterns so every component that uses it
-   should call ompi_netpatterns_init, still we want to run it only once */
-static int was_called = 0;
-
-    if (0 == was_called) {
-        was_called = 1;
-
-        return ompi_netpatterns_register_mca_params();
-    }
-
-    return OMPI_SUCCESS;
-}
diff --git a/ompi/patterns/net/netpatterns_knomial_tree.c b/ompi/patterns/net/netpatterns_knomial_tree.c
deleted file mode 100644
index 343938a0eb9..00000000000
--- a/ompi/patterns/net/netpatterns_knomial_tree.c
+++ /dev/null
@@ -1,978 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
- * Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
- * Copyright (c) 2014      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2014      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
- * Copyright (c) 2019      Intel, Inc.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#include <sys/types.h>
-#ifdef HAVE_SYS_MMAN_H
-#include <sys/mman.h>
-#endif
-#include <fcntl.h>
-#include <stdlib.h>
-#include <assert.h>
-
-#include "ompi/constants.h"
-
-#include "ompi/runtime/ompi_rte.h"
-
-#include "netpatterns.h"
-
-/* setup recursive doubleing tree node */
-
-OMPI_DECLSPEC int ompi_netpatterns_setup_recursive_knomial_allgather_tree_node(
-        int num_nodes, int node_rank, int tree_order, int *hier_ranks,
-        netpatterns_k_exchange_node_t *exchange_node)
-{
-    /* local variables */
-    int i, j, cnt, i_temp;
-    int knt,knt2,kk, ex_node, stray;
-    int n_levels,pow_k;
-    int k_temp1;
-    int k_temp2;
-    int myid, reindex_myid = 0;
-    int base, peer_base,base_temp;
-    int peer;
-    int *prev_data = NULL;
-    int *current_data = NULL;
-    int *group_info = NULL;
-
-
-    NETPATTERNS_VERBOSE(
-            ("Enter ompi_netpatterns_setup_recursive_knomial_tree_node(num_nodes=%d, node_rank=%d, tree_order=%d)",
-                num_nodes, node_rank, tree_order));
-
-    assert(num_nodes > 1);
-    assert(tree_order > 1);
-    if (tree_order > num_nodes) {
-        tree_order = num_nodes;
-    }
-
-    /* k-nomial radix */
-    exchange_node->tree_order = tree_order;
-
-    /* Calculate the number of levels in the tree for
-     * the largest power of tree_order less than or
-     * equal to the group size
-     */
-    n_levels = 0;
-    cnt=1;
-    while ( num_nodes > cnt ) {
-        cnt *= tree_order;
-        n_levels++;
-    }
-    /* this is the actual number of recusive k-ing steps
-     * we will perform, the last step may not be a full
-     * step depending on the outcome of the next conditional
-     */
-    pow_k = n_levels;
-
-    /* figure out the largest power of tree_order that is less than or equal to
-     * num_nodes */
-    if ( cnt > num_nodes) {
-        cnt /= tree_order;
-        n_levels--;
-    }
-
-    /*exchange_node->log_tree_order = n_levels;*/
-    exchange_node->log_tree_order = pow_k;
-    exchange_node->n_largest_pow_tree_order = cnt;
-
-
-    /* find the number of complete groups of size tree_order, tree_order^2, tree_order^3,...,tree_order^pow_k */
-    /* I don't think we need to cache this info this group_info array */
-    group_info = (int *) calloc(pow_k , sizeof(int));
-    group_info[0] = num_nodes/tree_order;
-    /*fprintf(stderr,"Number of complete groups of power 1 is %d\n",group_info[0]);*/
-    for ( i = 1; i < pow_k; i ++) {
-        group_info[i] = group_info[i-1]/tree_order;
-        /*fprintf(stderr,"Number of complete groups of power %d is %d\n",i+1,group_info[i]);*/
-
-    }
-
-    /* find number of incomplete groups and number of ranks belonging to those ranks */
-    knt=0;
-    while (knt <= (pow_k - 1) && group_info[knt] > 0) {
-        knt++;
-    }
-    knt--;
-    /*fprintf(stderr,"Maximal power of k is %d and the number of incomplete groups is %d \n", knt+1 ,tree_order - group_info[knt] );*/
-
-    /* k_temp is a synonym for cnt which is the largest full power of k group */
-    /* now, start the calculation to find the first stray rank aka "extra" rank */
-    stray = 0;
-    /*fprintf(stderr,"Maximal power of k %d, first stragler rank is %d and the number of straglers is %d\n",cnt,
-                                                                           cnt*group_info[knt],
-                                                                           num_nodes - cnt*group_info[knt]);*/
-
-
-    /* cache this info, it's muy importante */
-    stray = cnt*group_info[knt];
-    exchange_node->k_nomial_stray = stray;
-
-
-
-    /* before we do this, we need to first reindex */
-    /* reindexing phase */
-     /* this is the reindex phase */
-    exchange_node->reindex_map = (int *) malloc(num_nodes*sizeof(int));
-    /* this is the inverse map */
-    exchange_node->inv_reindex_map = (int *) malloc(num_nodes*sizeof(int));
-    /*int reindex_myid;*/
-    /* reindex */
-    if( stray < num_nodes ) {
-        /* find the first proxy rank */
-        peer = stray - cnt;
-        /* fix all ranks prior to this rank */
-        for( i = 0; i < peer; i++){
-            exchange_node->reindex_map[i] = i;
-        }
-        /* now, start the swap */
-        exchange_node->reindex_map[peer] = peer;
-        for( i = (peer+1); i < (peer + (num_nodes - stray)+1); i++) {
-            exchange_node->reindex_map[i] = exchange_node->reindex_map[i-1] + 2;
-        }
-        i_temp = i;
-        for( i = i_temp; i < stray; i++) {
-            exchange_node->reindex_map[i] = exchange_node->reindex_map[i-1] + 1;
-        }
-        /* now, finish it off */
-        exchange_node->reindex_map[stray] = peer + 1;
-        for( i = (stray+1); i < num_nodes; i++) {
-            exchange_node->reindex_map[i] = exchange_node->reindex_map[i-1] + 2;
-        }
-        /* debug print */
-        /*
-        for( i = 0; i < np; i++){
-            fprintf(stderr,"%d ",reindex_map[i]);
-        }
-        fprintf(stderr,"\n");
-        */
-    } else {
-        /* we have no extras, trivial reindexing */
-        for( i = 0; i < num_nodes; i++){
-            exchange_node->reindex_map[i] = i;
-        }
-    }
-    /* finished reindexing */
-
-    /* Now, I need to get my rank in the new indexing */
-    for( i = 0; i < num_nodes; i++ ){
-        if( node_rank == exchange_node->reindex_map[i] ){
-            exchange_node->reindex_myid = i;
-            break;
-        }
-    }
-    /* Now, let's compute the inverse mapping here */
-    for( i = 0; i < num_nodes; i++){
-        j = 0;
-        while(exchange_node->reindex_map[j] != i ){
-            j++;
-        }
-        exchange_node->inv_reindex_map[i] = j;
-    }
-
-
-    /* Now we get the data sizes we should expect at each level */
-    /* now get the size of the data I am to receive from each peer */
-    /*int **payload_info;*/
-    prev_data = (int *) malloc( num_nodes*sizeof(int) );
-    if( NULL == prev_data ) {
-        goto Error;
-    }
-
-    current_data = (int *) malloc( num_nodes*sizeof(int) );
-    if( NULL == current_data ) {
-        goto Error;
-    }
-
-
-    exchange_node->payload_info = (netpatterns_payload_t **) malloc(sizeof(netpatterns_payload_t *)*pow_k);
-    if( NULL == exchange_node->payload_info) {
-        goto Error;
-    }
-
-    for(i = 0; i < pow_k; i++){
-        exchange_node->payload_info[i] = (netpatterns_payload_t *) malloc(sizeof(netpatterns_payload_t)*(tree_order-1));
-        if( NULL == exchange_node->payload_info[i]) {
-            goto Error;
-        }
-
-    }
-    /* intialize the payload array
-       This is the money struct, just need to initialize this with
-       the subgroup information */
-    /*
-    for(i = 0; i < num_nodes; i++){
-        prev_data[i] = 1;
-        current_data[i] = 1;
-    }
-    */
-
-    for(i = 0; i < num_nodes; i++){
-        prev_data[i] = hier_ranks[i];
-        current_data[i] = hier_ranks[i];
-    }
-
-    /* everyone will need to do this loop over all ranks
-     * Phase I calculate the contribution from the extra ranks
-     */
-    for( myid = 0; myid < num_nodes; myid++) {
-        /* get my new rank */
-        for( j = 0; j < num_nodes; j++ ){
-            /* this will be satisfied for one of the indices */
-            if( myid == exchange_node->reindex_map[j] ){
-                reindex_myid = j;
-                break;
-            }
-        }
-
-        for( j = stray; j < num_nodes; j++) {
-            if(reindex_myid == ( j - cnt )) {
-                /* then this is a proxy rank */
-                prev_data[myid] += prev_data[exchange_node->reindex_map[j]];
-                break;
-            }
-
-        }
-    }
-
-    /* Phase II calculate the contribution from each recursive k - ing level
-     *
-     */
-    k_temp1 = tree_order; /* k^1 */
-    k_temp2 = 1;   /* k^0 */
-    peer_base = 0;
-    base_temp = 0;
-    for( i = 0; i < pow_k; i++) {
-        /* get my new rank */
-        for( myid = 0; myid < num_nodes; myid++){
-            current_data[myid] = prev_data[myid];
-            /*fprintf(stderr,"my current data at level %d is %d\n",i+1,current_data[myid]);*/
-            for( j = 0; j < num_nodes; j++ ){
-                if( myid == exchange_node->reindex_map[j] ){
-                    reindex_myid = j;
-                    break;
-                }
-            }
-            if( reindex_myid < stray ) {
-                /* now start the actual algorithm */
-                FIND_BASE(base,reindex_myid,i+1,tree_order);
-                for( j = 0; j < ( tree_order - 1 ); j ++ ) {
-                    peer = base + (reindex_myid + k_temp2*(j+1))%k_temp1;
-                    if( peer < stray ) {
-                        /*fprintf(stderr,"getting %d bytes \n",prev_data[reindex_map[peer]]);*/
-                        /* then get the data */
-                        if( node_rank == myid ){
-                            exchange_node->payload_info[i][j].r_len = prev_data[exchange_node->reindex_map[peer]];
-                            /*fprintf(stderr,"exchange_node->payload_info[%d][%d].r_len %d\n",i,j,prev_data[exchange_node->reindex_map[peer]]);*/
-                            if( i > 0 ) {
-
-                                /* find my len and offset */
-                                FIND_BASE(peer_base,peer,i,tree_order);
-                                /* I do not want to mess with this, but it seems that I have no choice */
-                               ex_node = exchange_node->reindex_map[peer_base];
-                               /* now, find out how far down the line this guy really is */
-                               knt2 =0;
-                               for(kk = 0; kk < ex_node; kk++){
-                                   knt2 += hier_ranks[kk];
-                               }
-                                exchange_node->payload_info[i][j].r_offset = knt2;
-                                /*fprintf(stderr,"exchange_node->payload_info[%d][%d].r_offset %d\n",i,j,exchange_node->payload_info[i][j].r_offset);*/
-
-                                FIND_BASE(base_temp,reindex_myid,i,tree_order);
-                                ex_node = exchange_node->reindex_map[base_temp];
-                                knt2 = 0;
-                                for( kk = 0; kk < ex_node; kk++){
-                                    knt2 += hier_ranks[kk];
-                                }
-                                exchange_node->payload_info[i][j].s_offset =
-                                                                  knt2; /* exchange_node->reindex_map[base_temp]; */
-                                /*fprintf(stderr,"exchange_node->payload_info[%d][%d].s_offset %d\n",i,j,exchange_node->payload_info[i][j].s_offset);*/
-                            } else {
-                                ex_node = exchange_node->reindex_map[peer];
-                                knt2 =0;
-                                for(kk = 0; kk < ex_node; kk++){
-                                    knt2 += hier_ranks[kk];
-                                }
-                                exchange_node->payload_info[i][j].r_offset =
-                                    knt2; /*exchange_node->reindex_map[peer]; */
-                                /*fprintf(stderr,"exchange_node->payload_info[%d][%d].r_offset %d\n",i,j,exchange_node->payload_info[i][j].r_offset);*/
-                                knt2 = 0;
-                                for(kk = 0; kk < myid; kk++){
-                                    knt2 += hier_ranks[kk];
-                                }
-                                exchange_node->payload_info[i][j].s_offset = knt2;
-                                /*fprintf(stderr,"exchange_node->payload_info[%d][%d].s_offset %d\n",i,j, exchange_node->payload_info[i][j].s_offset);*/
-                            }
-                            /* how much I am to receive from this peer on this level */
-                            /* how much I am to send to this peer on this level */
-                            exchange_node->payload_info[i][j].s_len = prev_data[node_rank];
-                            /*fprintf(stderr,"exchange_node->payload_info[%d][%d].s_len %d\n",i,j,prev_data[node_rank]);*/
-                            /*fprintf(stderr,"I am rank %d receiveing %d bytes from rank %d at level %d\n",node_rank,
-                                                                        prev_data[exchange_node->reindex_map[peer]],
-                                                                        exchange_node->reindex_map[peer], i+1);*/
-                            /*fprintf(stderr,"I am rank %d sending %d bytes to rank %d at level %d\n",node_rank,prev_data[myid],
-                                      exchange_node->reindex_map[peer],i+1);*/
-                        }
-
-                        current_data[myid] += prev_data[exchange_node->reindex_map[peer]];
-                    }
-                }
-            }
-
-
-        }
-        k_temp1 *= tree_order;
-        k_temp2 *= tree_order;
-        /* debug print */
-       /* fprintf(stderr,"Level %d current data ",i+1);*/
-        for( j = 0; j < num_nodes; j++){
-           /* fprintf(stderr,"%d ",current_data[j]); */
-            prev_data[j] = current_data[j];
-        }
-       /* fprintf(stderr,"\n");*/
-
-    }
-
-
-    /* this is the natural way to do recursive k-ing */
-    /* should never have more than one extra rank per proxy */
-    if( exchange_node->reindex_myid >= stray ){
-        /*fprintf(stderr,"Rank %d is mapped onto proxy rank %d \n",exchange_node->reindex_myid,exchange_node->reindex_myid - cnt);*/
-        exchange_node->node_type = EXTRA_NODE;
-    } else {
-        exchange_node->node_type = EXCHANGE_NODE;
-    }
-
-    /* set node characteristics - node that is not within the largest
-     * power of tree_order will just send its data to node that will participate
-     * in the recursive k-ing, and get the result back at the end.
-     * set the initial and final data exchanges - those that are not
-     * part of the recursive k-ing.
-     */
-    if (EXCHANGE_NODE == exchange_node->node_type)  {
-        exchange_node->n_extra_sources = 0;
-        for( i = stray; i < num_nodes; i++) {
-            if(exchange_node->reindex_myid == ( i - cnt )) {
-                /* then I am a proxy rank and there is only a
-                 * single extra source
-                 */
-                exchange_node->n_extra_sources = 1;
-                break;
-            }
-        }
-
-        if (exchange_node->n_extra_sources > 0) {
-            exchange_node->rank_extra_sources_array = (int *) malloc
-                (exchange_node->n_extra_sources * sizeof(int));
-            if( NULL == exchange_node->rank_extra_sources_array ) {
-                goto Error;
-            }
-            /* you broke above */
-            exchange_node->rank_extra_sources_array[0] = exchange_node->reindex_map[i];
-        } else {
-            exchange_node->rank_extra_sources_array = NULL;
-        }
-    } else {
-        /* I am an extra rank, find my proxy rank */
-        exchange_node->n_extra_sources = 1;
-
-        exchange_node->rank_extra_sources_array = (int *) malloc
-            (exchange_node->n_extra_sources * sizeof(int));
-        if( NULL == exchange_node->rank_extra_sources_array ) {
-            goto Error;
-        }
-        exchange_node->rank_extra_sources_array[0] = exchange_node->reindex_map[exchange_node->reindex_myid - cnt];
-    }
-
-
-    /* set the exchange pattern */
-    if (EXCHANGE_NODE == exchange_node->node_type) {
-        /* yep, that's right PLUS 1 */
-        exchange_node->n_exchanges = n_levels + 1;
-        /* initialize this */
-        exchange_node->n_actual_exchanges = 0;
-        /* Allocate 2 dimension array thak keeps
-         rank exchange information for each step*/
-        exchange_node->rank_exchanges = (int **) malloc
-            (exchange_node->n_exchanges * sizeof(int *));
-        if(NULL == exchange_node->rank_exchanges) {
-            goto Error;
-        }
-        for (i = 0; i < exchange_node->n_exchanges; i++) {
-            exchange_node->rank_exchanges[i] = (int *) malloc
-                ((tree_order - 1) * sizeof(int));
-            if( NULL == exchange_node->rank_exchanges ) {
-                goto Error;
-            }
-        }
-        k_temp1 = tree_order;
-        k_temp2 = 1;
-        /* fill in exchange partners */
-        /* Ok, now we start with the actual algorithm */
-        for( i = 0; i < exchange_node->n_exchanges; i ++) {
-            /*fprintf(stderr,"Starting Level %d\n",i+1);*/
-
-            FIND_BASE(base,exchange_node->reindex_myid,i+1,tree_order);
-            /*fprintf(stderr,"Myid %d base %d\n",node_rank,base);*/
-            for( j = 0; j < (tree_order-1); j ++ ) {
-                peer = base + (exchange_node->reindex_myid + k_temp2*(j+1))%k_temp1;
-                if ( peer < stray ) {
-                    exchange_node->rank_exchanges[i][j] = exchange_node->reindex_map[peer];
-                    /* an actual exchange occurs, bump the counter */
-
-                } else {
-                    /* out of range, skip it - do not bump the n_actual_exchanges counter */
-                    exchange_node->rank_exchanges[i][j] = -1;
-                }
-
-            }
-            k_temp1 *= tree_order;
-            k_temp2 *= tree_order;
-        }
-        for(i = 0; i < pow_k; i++){
-            for(j = 0; j < (tree_order-1); j++){
-                if(-1 != exchange_node->rank_exchanges[i][j]){
-                    /* then bump the counter */
-                    exchange_node->n_actual_exchanges++;
-                }
-            }
-        }
-
-    } else {
-        /* we are extra ranks and we don't participate in the exchange :( */
-        exchange_node->n_exchanges=0;
-        exchange_node->rank_exchanges=NULL;
-    }
-
-
-    /* set the number of tags needed per stripe - this must be the
-     *   same across all procs in the communicator.
-     */
-    /* do we need this one */
-    exchange_node->n_tags = tree_order * n_levels + 1;
-
-    free(prev_data);
-    free(current_data);
-    free(group_info);
-
-    /* successful return */
-    return OMPI_SUCCESS;
-
-Error:
-
-    if (NULL != exchange_node->rank_extra_sources_array) {
-        free(exchange_node->rank_extra_sources_array);
-    }
-
-    if (NULL != exchange_node->rank_exchanges) {
-        for (i = 0; i < exchange_node->n_exchanges; i++) {
-            if (NULL != exchange_node->rank_exchanges[i]) {
-                free(exchange_node->rank_exchanges[i]);
-            }
-        }
-        free(exchange_node->rank_exchanges);
-    }
-
-    if (NULL != prev_data ){
-        free(prev_data);
-    }
-
-    if(NULL != current_data) {
-        free(current_data);
-    }
-
-    if(NULL != group_info) {
-        free(group_info);
-    }
-
-    /* error return */
-    return OMPI_ERROR;
-}
-
-OMPI_DECLSPEC void ompi_netpatterns_cleanup_recursive_knomial_allgather_tree_node(
-        netpatterns_k_exchange_node_t *exchange_node)
-{
-    int i;
-
-    free(exchange_node->reindex_map);
-    free(exchange_node->inv_reindex_map);
-    if (exchange_node->n_extra_sources > 0) {
-        free(exchange_node->rank_extra_sources_array) ;
-        exchange_node->n_extra_sources = 0;
-        exchange_node->rank_extra_sources_array = NULL;
-    }
-    if (exchange_node->n_exchanges > 0) {
-        for (i=0; i < exchange_node->n_exchanges; i++) {
-            free(exchange_node->rank_exchanges[i]);
-            exchange_node->rank_exchanges[i] = NULL;
-        }
-        free(exchange_node->rank_exchanges);
-        exchange_node->rank_exchanges = NULL;
-        exchange_node->n_exchanges = 0;
-    }
-    for(i = 0; i < exchange_node->log_tree_order; i++){
-        free(exchange_node->payload_info[i]);
-    }
-    free(exchange_node->payload_info);
-}
-
-OMPI_DECLSPEC int ompi_netpatterns_setup_recursive_knomial_tree_node(
-        int num_nodes, int node_rank, int tree_order,
-        netpatterns_k_exchange_node_t *exchange_node)
-{
-    /* local variables */
-    int i, j, tmp, cnt;
-    int n_levels;
-    int k_base, kpow_num, peer;
-
-    NETPATTERNS_VERBOSE(
-            ("Enter ompi_netpatterns_setup_recursive_knomial_tree_node(num_nodes=%d, node_rank=%d, tree_order=%d)",
-                num_nodes, node_rank, tree_order));
-
-    assert(num_nodes > 1);
-    assert(tree_order > 1);
-    if (tree_order > num_nodes) {
-        tree_order = num_nodes;
-    }
-
-    exchange_node->tree_order = tree_order;
-
-    /* figure out number of levels in the tree */
-    n_levels = 0;
-    /* cnt - number of ranks in given level */
-    cnt=1;
-    while ( num_nodes > cnt ) {
-        cnt *= tree_order;
-        n_levels++;
-    };
-
-    /* figure out the largest power of tree_order that is less than or equal to
-     * num_nodes */
-    if ( cnt > num_nodes) {
-        cnt /= tree_order;
-        n_levels--;
-    }
-
-    exchange_node->log_tree_order = n_levels;
-    exchange_node->n_largest_pow_tree_order = cnt;
-
-    /* set node characteristics - node that is not within the largest
-     *  power of tree_order will just send it's data to node that will participate
-     *  in the recursive doubling, and get the result back at the end.
-     */
-    if (node_rank + 1 > cnt) {
-        exchange_node->node_type = EXTRA_NODE;
-    } else {
-        exchange_node->node_type = EXCHANGE_NODE;
-    }
-
-
-    /* set the initial and final data exchanges - those that are not
-     *   part of the recursive doubling.
-     */
-    if (EXCHANGE_NODE == exchange_node->node_type)  {
-        exchange_node->n_extra_sources = 0;
-        for (i = 0, tmp = node_rank * (tree_order - 1) + cnt + i;
-                tmp < num_nodes && i < tree_order - 1;
-                ++i, ++tmp) {
-            ++exchange_node->n_extra_sources;
-        }
-
-        assert(exchange_node->n_extra_sources < tree_order);
-
-        if (exchange_node->n_extra_sources > 0) {
-            exchange_node->rank_extra_sources_array = (int *) malloc
-                (exchange_node->n_extra_sources * sizeof(int));
-            if( NULL == exchange_node->rank_extra_sources_array ) {
-                goto Error;
-            }
-            for (i = 0, tmp = node_rank * (tree_order - 1) + cnt;
-                    i < tree_order - 1 && tmp < num_nodes; ++i, ++tmp) {
-                NETPATTERNS_VERBOSE(("extra_source#%d = %d", i, tmp));
-                exchange_node->rank_extra_sources_array[i] = tmp;
-            }
-        } else {
-            exchange_node->rank_extra_sources_array = NULL;
-        }
-    } else {
-        exchange_node->n_extra_sources = 1;
-        exchange_node->rank_extra_sources_array = (int *) malloc (sizeof(int));
-        if( NULL == exchange_node->rank_extra_sources_array ) {
-            goto Error;
-        }
-        exchange_node->rank_extra_sources_array[0] = (node_rank - cnt) / (tree_order - 1);
-        NETPATTERNS_VERBOSE(("extra_source#%d = %d", 0,
-                    exchange_node->rank_extra_sources_array[0] ));
-    }
-
-    /* set the exchange pattern */
-    if (EXCHANGE_NODE == exchange_node->node_type) {
-        exchange_node->n_exchanges = n_levels;
-        /* Allocate 2 dimension array thak keeps
-         rank exchange information for each step*/
-        exchange_node->rank_exchanges = (int **) malloc
-            (exchange_node->n_exchanges * sizeof(int *));
-        if(NULL == exchange_node->rank_exchanges) {
-            goto Error;
-        }
-        for (i = 0; i < exchange_node->n_exchanges; i++) {
-            exchange_node->rank_exchanges[i] = (int *) malloc
-                ((tree_order - 1) * sizeof(int));
-            if( NULL == exchange_node->rank_exchanges ) {
-                goto Error;
-            }
-        }
-        /* fill in exchange partners */
-        for(i = 0, kpow_num = 1; i < exchange_node->n_exchanges;
-                                      i++, kpow_num *= tree_order) {
-            k_base = node_rank / (kpow_num * tree_order);
-            for(j = 1; j < tree_order; j++) {
-                peer = node_rank + kpow_num * j;
-                if (k_base != peer/(kpow_num * tree_order)) {
-                    /* Wraparound the number */
-                    peer = k_base * (kpow_num * tree_order)  +
-                        peer % (kpow_num * tree_order);
-                }
-                exchange_node->rank_exchanges[i][j - 1] = peer;
-                NETPATTERNS_VERBOSE(("rank_exchanges#(%d,%d)/%d = %d",
-                            i, j, tree_order, peer));
-            }
-        }
-    } else {
-        exchange_node->n_exchanges=0;
-        exchange_node->rank_exchanges=NULL;
-    }
-
-    /* set the number of tags needed per stripe - this must be the
-     *   same across all procs in the communicator.
-     */
-    /* do we need this one */
-    exchange_node->n_tags = tree_order * n_levels + 1;
-
-    /* successful return */
-    return OMPI_SUCCESS;
-
-Error:
-
-    ompi_netpatterns_cleanup_recursive_knomial_tree_node (exchange_node);
-
-    /* error return */
-    return OMPI_ERROR;
-}
-
-OMPI_DECLSPEC void ompi_netpatterns_cleanup_recursive_knomial_tree_node(
-        netpatterns_k_exchange_node_t *exchange_node)
-{
-    int i;
-
-    if (exchange_node->n_extra_sources > 0) {
-        free(exchange_node->rank_extra_sources_array);
-        exchange_node->rank_extra_sources_array = NULL;
-        exchange_node->n_extra_sources = 0;
-    }
-    if (exchange_node->n_exchanges > 0) {
-        for (i=0 ; i<exchange_node->n_exchanges; i++) {
-            free(exchange_node->rank_exchanges[i]);
-            exchange_node->rank_exchanges[i] = NULL;
-        }
-        free(exchange_node->rank_exchanges);
-        exchange_node->rank_exchanges = NULL;
-        exchange_node->n_exchanges = 0;
-    }
-}
-
-#if 1
-OMPI_DECLSPEC int ompi_netpatterns_setup_recursive_doubling_n_tree_node(int num_nodes, int node_rank, int tree_order,
-        netpatterns_pair_exchange_node_t *exchange_node)
-{
-    /* local variables */
-    int i, tmp, cnt;
-    int n_levels;
-    int shift, mask;
-
-    NETPATTERNS_VERBOSE(("Enter ompi_netpatterns_setup_recursive_doubling_n_tree_node(num_nodes=%d, node_rank=%d, tree_order=%d)", num_nodes, node_rank, tree_order));
-
-    assert(num_nodes > 1);
-    while (tree_order > num_nodes) {
-        tree_order /= 2;
-    }
-
-    exchange_node->tree_order = tree_order;
-    /* We support only tree_order that are power of two */
-    assert(0 == (tree_order & (tree_order - 1)));
-
-    /* figure out number of levels in the tree */
-    n_levels = 0;
-    /* cnt - number of ranks in given level */
-    cnt=1;
-    while ( num_nodes > cnt ) {
-        cnt *= tree_order;
-        n_levels++;
-    };
-
-    /* figure out the largest power of tree_order that is less than or equal to
-     * num_nodes */
-    if ( cnt > num_nodes) {
-        cnt /= tree_order;
-        n_levels--;
-    }
-    exchange_node->log_tree_order = n_levels;
-    if (2 == tree_order) {
-        exchange_node->log_2 = exchange_node->log_tree_order;
-    }
-
-    tmp=1;
-    for (i=0 ; i < n_levels ; i++ ) {
-        tmp *= tree_order;
-    }
-    /* Ishai: I see no reason for calculating tmp. Add an assert before deleting it */
-    assert(tmp == cnt);
-
-    exchange_node->n_largest_pow_tree_order = tmp;
-    if (2 == tree_order) {
-        exchange_node->n_largest_pow_2 = exchange_node->n_largest_pow_tree_order;
-    }
-
-    /* set node characteristics - node that is not within the largest
-     *  power of tree_order will just send it's data to node that will participate
-     *  in the recursive doubling, and get the result back at the end.
-     */
-    if ( node_rank + 1 > cnt ) {
-        exchange_node->node_type = EXTRA_NODE;
-    } else {
-        exchange_node->node_type = EXCHANGE_NODE;
-    }
-
-    /* set the initial and final data exchanges - those that are not
-     *   part of the recursive doubling.
-     */
-    if ( EXCHANGE_NODE == exchange_node->node_type ) {
-        exchange_node->n_extra_sources = 0;
-        for (tmp = node_rank + cnt; tmp < num_nodes; tmp += cnt) {
-            ++exchange_node->n_extra_sources;
-        }
-        if (exchange_node->n_extra_sources > 0) {
-            exchange_node->rank_extra_sources_array = (int *) malloc
-                (exchange_node->n_extra_sources * sizeof(int));
-            if( NULL == exchange_node->rank_extra_sources_array ) {
-                goto Error;
-            }
-            for (i = 0, tmp = node_rank + cnt; tmp < num_nodes; ++i, tmp += cnt) {
-                NETPATTERNS_VERBOSE(("extra_source#%d = %d", i, tmp));
-                exchange_node->rank_extra_sources_array[i] = tmp;
-            }
-        } else {
-            exchange_node->rank_extra_sources_array = NULL;
-        }
-    } else {
-        exchange_node->n_extra_sources = 1;
-        exchange_node->rank_extra_sources_array = (int *) malloc (sizeof(int));
-        if( NULL == exchange_node->rank_extra_sources_array ) {
-            goto Error;
-        }
-        exchange_node->rank_extra_sources_array[0] = node_rank & (cnt - 1);
-        NETPATTERNS_VERBOSE(("extra_source#%d = %d", 0, node_rank & (cnt - 1)));
-    }
-
-    /* Ishai: To be compatable with the old structure - should be remoived later */
-    if (1 == exchange_node->n_extra_sources) {
-        exchange_node->rank_extra_source = exchange_node->rank_extra_sources_array[0];
-    } else {
-        exchange_node->rank_extra_source = -1;
-    }
-
-    /* set the exchange pattern */
-    if ( EXCHANGE_NODE == exchange_node->node_type ) {
-        exchange_node->n_exchanges = n_levels * (tree_order - 1);
-        exchange_node->rank_exchanges = (int *) malloc
-            (exchange_node->n_exchanges * sizeof(int));
-        if( NULL == exchange_node->rank_exchanges ) {
-            goto Error;
-        }
-
-        /* fill in exchange partners */
-        for ( i = 0, shift = 1 ; i < exchange_node->n_exchanges ; shift *= tree_order ) {
-            for ( mask = 1 ; mask < tree_order ; ++mask, ++i ) {
-                exchange_node->rank_exchanges[i] = node_rank ^ (mask * shift);
-                NETPATTERNS_VERBOSE(("rank_exchanges#%d/%d = %d", i, tree_order, node_rank ^ (mask * shift)));
-            }
-        }
-
-    } else {
-
-        exchange_node->n_exchanges=0;
-        exchange_node->rank_exchanges=NULL;
-
-    }
-
-    /* set the number of tags needed per stripe - this must be the
-     *   same across all procs in the communicator.
-     */
-    /* Ishai: Need to find out what is n_tags */
-    exchange_node->n_tags = tree_order * n_levels + 1;
-
-    /* successful return */
-    return OMPI_SUCCESS;
-
-Error:
-    if (exchange_node->rank_extra_sources_array != NULL) {
-        free(exchange_node->rank_extra_sources_array);
-    }
-
-    /* error return */
-    return OMPI_ERROR;
-}
-
-OMPI_DECLSPEC void ompi_netpatterns_cleanup_recursive_doubling_tree_node(
-    netpatterns_pair_exchange_node_t *exchange_node)
-{
-    NETPATTERNS_VERBOSE(("About to release rank_extra_sources_array and rank_exchanges"));
-    if (exchange_node->rank_extra_sources_array != NULL) {
-        free(exchange_node->rank_extra_sources_array);
-    }
-
-    if (exchange_node->rank_exchanges != NULL) {
-        free(exchange_node->rank_exchanges);
-    }
-}
-#endif
-
-OMPI_DECLSPEC int ompi_netpatterns_setup_recursive_doubling_tree_node(int num_nodes, int node_rank,
-        netpatterns_pair_exchange_node_t *exchange_node)
-{
-    return ompi_netpatterns_setup_recursive_doubling_n_tree_node(num_nodes, node_rank, 2, exchange_node);
-}
-
-#if 0
-/*OMPI_DECLSPEC int old_netpatterns_setup_recursive_doubling_tree_node(int num_nodes, int node_rank,*/
-OMPI_DECLSPEC int ompi_netpatterns_setup_recursive_doubling_n_tree_node(int num_nodes, int node_rank,int tree_order,
-        netpatterns_pair_exchange_node_t *exchange_node)
-{
-    /* local variables */
-    /*int tree_order;*/
-    int i,tmp,cnt,result,n_extra_nodes;
-    int n_exchanges;
-
-    /* figure out number of levels in the tree */
-
-    n_exchanges=0;
-    result=num_nodes;
-/*    tree_order=2;*/
-    /* cnt - number of ranks in given level */
-    cnt=1;
-    while( num_nodes > cnt ) {
-        cnt*=tree_order;
-        n_exchanges++;
-    };
-
-    /* figure out the largest power of 2 that is less than or equal to
-     * num_nodes */
-    if( cnt > num_nodes) {
-        cnt/=tree_order;
-        n_exchanges--;
-    }
-    exchange_node->log_2=n_exchanges;
-
-    tmp=1;
-    for(i=0 ; i < n_exchanges ; i++ ) {
-        tmp*=2;
-    }
-    exchange_node->n_largest_pow_2=tmp;
-
-    /* set node characteristics - node that is not within the largest
-     *  power of 2 will just send it's data to node that will participate
-     *  in the recursive doubling, and get the result back at the end.
-     */
-    if( node_rank+1 > cnt ) {
-        exchange_node->node_type=EXTRA_NODE;
-    } else {
-        exchange_node->node_type=EXCHANGE_NODE;
-    }
-
-    /* set the initial and final data exchanges - those that are not
-     *   part of the recursive doubling.
-     */
-    n_extra_nodes=num_nodes-cnt;
-
-    if ( EXCHANGE_NODE == exchange_node->node_type ) {
-
-        if( node_rank < n_extra_nodes ) {
-            exchange_node->n_extra_sources=1;
-            exchange_node->rank_extra_source=cnt+node_rank;
-        } else {
-            exchange_node->n_extra_sources=0;
-            exchange_node->rank_extra_source=-1;
-        }
-
-    } else {
-            exchange_node->n_extra_sources=1;
-            exchange_node->rank_extra_source=node_rank-cnt;
-    }
-
-    /* set the exchange pattern */
-    if( EXCHANGE_NODE == exchange_node->node_type ) {
-
-        exchange_node->n_exchanges=n_exchanges;
-        exchange_node->rank_exchanges=(int *) malloc
-            (n_exchanges*sizeof(int));
-        if( NULL == exchange_node->rank_exchanges ) {
-            goto Error;
-        }
-
-        /* fill in exchange partners */
-        result=1;
-        tmp=node_rank;
-        for( i=0 ; i < n_exchanges ; i++ ) {
-            if(tmp & 1 ) {
-                exchange_node->rank_exchanges[i]=
-                    node_rank-result;
-            } else {
-                exchange_node->rank_exchanges[i]=
-                    node_rank+result;
-            }
-            result*=2;
-            tmp/=2;
-        }
-
-    } else {
-
-        exchange_node->n_exchanges=0;
-        exchange_node->rank_exchanges=NULL;
-
-    }
-
-    /* set the number of tags needed per stripe - this must be the
-     *   same across all procs in the communicator.
-     */
-    exchange_node->n_tags=2*n_exchanges+1;
-
-    /* Ishai: to make sure free will work also for people that call this function */
-    exchange_node->rank_extra_sources_array = NULL;
-
-    /* successful return */
-    return OMPI_SUCCESS;
-
-Error:
-
-    /* error return */
-    return OMPI_ERROR;
-}
-#endif
-
diff --git a/ompi/patterns/net/netpatterns_knomial_tree.h b/ompi/patterns/net/netpatterns_knomial_tree.h
deleted file mode 100644
index 16dd6d81868..00000000000
--- a/ompi/patterns/net/netpatterns_knomial_tree.h
+++ /dev/null
@@ -1,262 +0,0 @@
-/*
- * Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
- * Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
- * Copyright (c) 2012      Los Alamos National Security, LLC.
- *                         All rights reserved.
- * Copyright (c) 2014      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef COMM_PATTERNS_KNOMIAL_TREE_H
-#define COMM_PATTERNS_KNOMIAL_TREE_H
-
-#include "ompi_config.h"
-
-BEGIN_C_DECLS
-
-
-/*
- * Pair-wise data exchange
- */
-
-/* enum for node type */
-enum {
-    EXCHANGE_NODE,
-    EXTRA_NODE
-};
-
-struct netpatterns_pair_exchange_node_t {
-
-    /* Order of a node in the tree - usually 2 */
-    int tree_order;
-
-    /* number of nodes this node will exchange data with */
-    int n_exchanges;
-
-    /* ranks of nodes involved in data exchnge */
-    int *rank_exchanges;
-
-    /* number of extra sources of data - outside largest power of 2 in
-     *  this group */
-    int n_extra_sources;
-
-    /* rank of the extra source */
-    /* deprecated */ int rank_extra_source;
-    int *rank_extra_sources_array;
-
-    /* number of tags needed per stripe */
-    int n_tags;
-
-    /* log 2 of largest full power of 2 for this node set */
-    /* deprecated */ int log_2;
-    int log_tree_order;
-
-    /* largest power of 2 that fits in this group */
-    /* deprecated */ int n_largest_pow_2;
-    int n_largest_pow_tree_order;
-
-    /* node type */
-    int node_type;
-
-};
-typedef struct netpatterns_pair_exchange_node_t netpatterns_pair_exchange_node_t;
-
-struct netpatterns_payload_t {
-    int s_len;
-    int r_len;
-    int s_offset;
-    int r_offset;
-};
-typedef struct netpatterns_payload_t netpatterns_payload_t;
-
-struct netpatterns_k_exchange_node_t {
-    /* Order of a node in the tree - usually 2 */
-    int tree_order;
-    /* number of nodes this node will exchange data with */
-    int n_exchanges;
-    /* total number of exchanges that I actually participate in */
-    int n_actual_exchanges;
-    /* ranks of nodes involved in data exchnge */
-    int **rank_exchanges;
-    /* number of extra sources of data - outside largest power of 2 in
-     *  this group */
-    int n_extra_sources;
-    /* rank/s of the extra source */
-    int *rank_extra_sources_array;
-    /* number of tags needed per stripe */
-    int n_tags;
-    /* log k of largest full power of k for this node set */
-    int log_tree_order;
-    /* largest power of k that fits in this group */
-    int n_largest_pow_tree_order;
-    /* node type */
-    int node_type;
-    /* start of extra ranks k_nomial */
-    int k_nomial_stray;
-    /* reindex map */
-    int *reindex_map;
-    /* inverse of reindex map, i.e. given a reindexed id find out its actual rank */
-    int *inv_reindex_map;
-    /* reindexed node_rank */
-    int reindex_myid;
-    /* 2-d array that hold payload info for each level of recursive k-ing */
-    netpatterns_payload_t **payload_info;
-};
-typedef struct netpatterns_k_exchange_node_t
-               netpatterns_k_exchange_node_t;
-
-OMPI_DECLSPEC int ompi_netpatterns_setup_recursive_doubling_n_tree_node(int num_nodes, int node_rank, int tree_order,
-    netpatterns_pair_exchange_node_t *exchange_node);
-
-OMPI_DECLSPEC void ompi_netpatterns_cleanup_recursive_doubling_tree_node(
-    netpatterns_pair_exchange_node_t *exchange_node);
-
-OMPI_DECLSPEC int ompi_netpatterns_setup_recursive_doubling_tree_node(int num_nodes, int node_rank,
-    netpatterns_pair_exchange_node_t *exchange_node);
-
-OMPI_DECLSPEC int ompi_netpatterns_setup_recursive_knomial_tree_node(
-   int num_nodes, int node_rank, int tree_order,
-   netpatterns_k_exchange_node_t *exchange_node);
-
-OMPI_DECLSPEC void ompi_netpatterns_cleanup_recursive_knomial_tree_node(
-   netpatterns_k_exchange_node_t *exchange_node);
-
-OMPI_DECLSPEC int ompi_netpatterns_setup_recursive_knomial_allgather_tree_node(
-        int num_nodes, int node_rank, int tree_order, int *hier_ranks,
-        netpatterns_k_exchange_node_t *exchange_node);
-
-OMPI_DECLSPEC void ompi_netpatterns_cleanup_recursive_knomial_allgather_tree_node(
-        netpatterns_k_exchange_node_t *exchange_node);
-
-/* Input: k_exchange_node structure
-      Output: index in rank_exchanges array that points
-      to the "start_point" for outgoing send.
-
-      Please see below example of usage:
-      for (i = start_point ; i > 0; i--)
-          for (k = 0; k < tree_radix; k++)
-              send messages to exchange_node->rank_exchanges[i][k];
-*/
-
-static inline __opal_attribute_always_inline__
-int netpatterns_get_knomial_level(
-    int my_rank, int src_rank,
-    int radix,   int size,
-    int *k_level)
-{
-    int distance,
-        pow_k;
-    int logk_level = 0;
-
-    /* Calculate disctance from source of data */
-    distance = src_rank - my_rank;
-
-    /* Wrap around */
-    if (0 > distance) {
-        distance += size;
-    }
-
-    pow_k = 1;
-    while(distance / (pow_k * radix)) {
-        pow_k *= radix;
-        ++logk_level;
-    }
-    --logk_level;
-
-    *k_level = pow_k;
-    return logk_level;
-}
-
-/* Input: my_rank, root, radix, size
- * Output: source of the data, offset in power of K
- */
-static inline __opal_attribute_always_inline__
-int netpatterns_get_knomial_data_source(
-    int my_rank, int root, int radix, int size,
-    int *k_level, int *logk_level)
-{
-    int level = radix;
-    int step = 0;
-
-    /* Calculate source of the data */
-    while((0 == (root - my_rank) % level)
-            && (level <= size)) {
-        level *= radix;
-        ++step;
-    }
-
-    *k_level = level/radix;
-    *logk_level = step;
-    return my_rank - (my_rank % level - root % level);
-}
-
-/* Input: my_rank, radix,
- *        k_level - that you get from netpatterns_get_knomial_data_source
- *        k_step - some integer
- * Output: peer - next children in the tree
- * Usage:
- *         src = netpatterns_get_knomial_data_source(
- *                  my_rank, root, radix, size,
- *                  &k_level, &logk_level)
- *         recv_from(src......);
- *
- *         MCA_COMMON_NETPATTERNS_GET_NEXT_KNOMIAL_INIT(step_info, k_level, my_rank);
- *         while(MCA_COMMON_NETPATTERNS_GET_NEXT_KNOMIAL_PEER_CHECK_LEVEL(step_info)) {
- *              MCA_COMMON_NETPATTERNS_GET_NEXT_KNOMIAL_PEER(my_rank, radix, step_info, peer);
- *              send_to(peer....);
- *         }
- * for more example please grep in ptpcoll bcol bcast files
- */
-
-typedef struct netpatterns_knomial_step_info_t {
-    int k_step;
-    int k_level;
-    int k_tmp_peer;
-} netpatterns_knomial_step_info_t;
-
-#define MCA_COMMON_NETPATTERNS_GET_NEXT_KNOMIAL_UPDATE_LEVEL_FOR_BCAST(step_info, radix)\
-do {                                                                                    \
-    if (1 != step_info.k_step) {                                                        \
-        step_info.k_level /= radix;                                                     \
-    }                                                                                   \
-} while (0)                                                                             \
-
-#define MCA_COMMON_NETPATTERNS_GET_NEXT_KNOMIAL_INIT(step_info, in_k_level, in_peer)\
-do {                                                                                \
-    step_info.k_step  = 1;                                                          \
-    step_info.k_level = in_k_level;                                                 \
-    step_info.k_tmp_peer = in_peer;                                                 \
-} while (0)
-
-#define MCA_COMMON_NETPATTERNS_GET_NEXT_KNOMIAL_PEER_CHECK_LEVEL(step_info) \
-                                                    (step_info.k_level > 1)
-
-#define MCA_COMMON_NETPATTERNS_GET_NEXT_KNOMIAL_PEER(my_rank, radix, step_info, peer)           \
-do {                                                                                            \
-    int rank_radix_base = my_rank/step_info.k_level;                                            \
-                                                                                                \
-    peer = step_info.k_tmp_peer + step_info.k_level/radix;                                      \
-    if (rank_radix_base != peer/step_info.k_level) {                                            \
-        /* Wraparound the number */                                                             \
-        peer -= step_info.k_level;                                                              \
-        assert(peer >=0);                                                                       \
-    }                                                                                           \
-    ++step_info.k_step;                                                                         \
-    if (radix == step_info.k_step) {                                                            \
-        step_info.k_level /= radix;                                                             \
-        step_info.k_step = 1;                                                                   \
-        step_info.k_tmp_peer = my_rank;                                                         \
-    } else {                                                                                    \
-        step_info.k_tmp_peer = peer;                                                            \
-    }                                                                                           \
-                                                                                                \
-} while (0)
-
-END_C_DECLS
-#endif
diff --git a/ompi/patterns/net/netpatterns_multinomial_tree.c b/ompi/patterns/net/netpatterns_multinomial_tree.c
deleted file mode 100644
index bb397c91238..00000000000
--- a/ompi/patterns/net/netpatterns_multinomial_tree.c
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
- * Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#include <sys/types.h>
-#ifdef HAVE_SYS_MMAN_H
-#include <sys/mman.h>
-#endif
-#include <fcntl.h>
-#include <stdlib.h>
-
-#include "ompi/constants.h"
-#include "netpatterns.h"
-
-
-/* setup an multi-nomial tree - for each node in the tree
- *  this returns it's parent, and it's children */
-
-OMPI_DECLSPEC int ompi_netpatterns_setup_multinomial_tree(int tree_order, int num_nodes,
-        netpatterns_tree_node_t *tree_nodes)
-{
-    /* local variables */
-    int i,result;
-    int cnt, n_nodes_in_this_level,node_index;
-    int n_cum_nodes,current_level,node,n_nodes_prev_level,rank,parent_rank;
-    int n_nodes_in_last_level,n_full_stripes,n_in_partial_stipe,n_children;
-    int n_lvls_in_tree;
-
-    /* sanity check */
-    if( 1 >= tree_order ) {
-        goto Error;
-    }
-
-
-    /* figure out number of levels in the tree */
-
-    n_lvls_in_tree=0;
-    result=num_nodes;
-    /* cnt - number of ranks in given level */
-    cnt=1;
-    /*  cummulative count of ranks */
-    while( 0 < result ) {
-        result-=cnt;
-        cnt*=tree_order;
-        n_lvls_in_tree++;
-    };
-
-    /* loop over tree levels */
-    n_nodes_in_this_level=1;
-    node_index=-1;
-    n_cum_nodes=0;
-    for( current_level = 0 ; current_level < n_lvls_in_tree ; current_level++) {
-
-        /* loop over nodes in current level */
-        for ( node=0 ; node < n_nodes_in_this_level ; node++ ) {
-            /* get node index */
-            node_index++;
-
-            /* break if reach group size */
-            if( node_index == num_nodes) {
-                break;
-            }
-
-            tree_nodes[node_index].my_rank=node_index;
-            tree_nodes[node_index].children_ranks=NULL;
-
-            /*
-             *  Parents
-             */
-            if( 0 == current_level ) {
-                tree_nodes[node_index].n_parents=0;
-                /* get parent index */
-                tree_nodes[node_index].parent_rank=-1;
-            } else {
-                tree_nodes[node_index].n_parents=1;
-                /* get parent index */
-                n_nodes_prev_level=n_nodes_in_this_level/tree_order;
-                if( current_level == n_lvls_in_tree -1 ) {
-                    /* load balance the lowest level */
-                    parent_rank=node-
-                        (node/n_nodes_prev_level)*n_nodes_prev_level;
-                    parent_rank=n_cum_nodes-n_nodes_prev_level+
-                        parent_rank;
-                    tree_nodes[node_index].parent_rank=parent_rank;
-                } else {
-                    tree_nodes[node_index].parent_rank=
-                        (n_cum_nodes-n_nodes_prev_level)+node/tree_order;
-                }
-            }
-
-            /*
-             * Children
-             */
-
-            /* get number of children */
-            if( (n_lvls_in_tree-1) == current_level ) {
-                /* leaves have no nodes */
-                tree_nodes[node_index].n_children=0;
-                tree_nodes[node_index].children_ranks=NULL;
-            } else {
-                /* take into account last level being incomplete */
-                if( (n_lvls_in_tree-2) == current_level ) {
-                    /* last level is load balanced */
-                    n_nodes_in_last_level=num_nodes-
-                        (n_cum_nodes+n_nodes_in_this_level);
-                    n_full_stripes=n_nodes_in_last_level/n_nodes_in_this_level;
-                    n_in_partial_stipe=n_nodes_in_last_level-
-                        n_full_stripes*n_nodes_in_this_level;
-                    n_children=n_full_stripes;
-                    if( n_full_stripes < tree_order ) {
-                        if( node <= n_in_partial_stipe-1 ) {
-                            n_children++;
-                        }
-                    }
-                    tree_nodes[node_index].n_children=n_children;
-                    if( 0 < n_children ) {
-                        tree_nodes[node_index].children_ranks=(int *)
-                            malloc(sizeof(int)*n_children);
-                        if( NULL == tree_nodes[node_index].children_ranks) {
-                            goto Error;
-                        }
-                    } else {
-                        tree_nodes[node_index].children_ranks=NULL;
-                    }
-                    /* fill in list */
-                    for( rank=0 ; rank < n_children ; rank++ ) {
-                        tree_nodes[node_index].children_ranks[rank]=
-                            node+rank*n_nodes_in_this_level;
-                        tree_nodes[node_index].children_ranks[rank]+=
-                            (n_cum_nodes+n_nodes_in_this_level);
-                    }
-                } else {
-                    n_children=tree_order;
-                    tree_nodes[node_index].n_children=tree_order;
-                    tree_nodes[node_index].children_ranks=(int *)
-                        malloc(sizeof(int)*n_children);
-                    if( NULL == tree_nodes[node_index].children_ranks) {
-                        goto Error;
-                    }
-                    for( rank=0 ; rank < n_children ; rank++ ) {
-                        tree_nodes[node_index].children_ranks[rank]=
-                            rank+tree_order*node;
-                        tree_nodes[node_index].children_ranks[rank]+=
-                            (n_cum_nodes+n_nodes_in_this_level);
-                    }
-                }
-            }
-
-        } /* end node loop */
-
-        /* update helper counters */
-        n_cum_nodes+=n_nodes_in_this_level;
-        n_nodes_in_this_level*=tree_order;
-    }
-
-    /* set node type */
-    for(i=0 ; i < num_nodes ; i++ ) {
-        if( 0 == tree_nodes[i].n_parents ) {
-            tree_nodes[i].my_node_type=ROOT_NODE;
-        } else if ( 0 == tree_nodes[i].n_children ) {
-            tree_nodes[i].my_node_type=LEAF_NODE;
-        } else {
-            tree_nodes[i].my_node_type=INTERIOR_NODE;
-        }
-    }
-
-    /* successful return */
-    return OMPI_SUCCESS;
-
-Error:
-    /* free allocated memory */
-    for( i=0 ; i < num_nodes ; i++ ) {
-        if( NULL != tree_nodes[i].children_ranks ) {
-            free(tree_nodes[i].children_ranks);
-        }
-    }
-
-    /* error return */
-    return OMPI_ERROR;
-}
diff --git a/ompi/patterns/net/netpatterns_nary_tree.c b/ompi/patterns/net/netpatterns_nary_tree.c
deleted file mode 100644
index 08f1543173d..00000000000
--- a/ompi/patterns/net/netpatterns_nary_tree.c
+++ /dev/null
@@ -1,458 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
- * Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
- * Copyright (c) 2014      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "ompi_config.h"
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#include <sys/types.h>
-#ifdef HAVE_SYS_MMAN_H
-#include <sys/mman.h>
-#endif
-#include <fcntl.h>
-#include <errno.h>
-#include <stdlib.h>
-
-#include "ompi/constants.h"
-#include "netpatterns.h"
-
-/*
- * Create mmaped shared file
- */
-
-/* setup an n-array tree */
-
-int ompi_netpatterns_setup_narray_tree(int tree_order, int my_rank, int num_nodes,
-        netpatterns_tree_node_t *my_node)
-{
-    /* local variables */
-    int n_levels, result;
-    int my_level_in_tree, cnt;
-    int lvl,cum_cnt, my_rank_in_my_level,n_lvls_in_tree;
-    int start_index,end_index;
-
-    /* sanity check */
-    if( 1 >= tree_order ) {
-        goto Error;
-    }
-
-    my_node->my_rank=my_rank;
-    my_node->tree_size=num_nodes;
-
-    /* figure out number of levels in tree */
-    n_levels=0;
-    result=num_nodes-1;
-    while (0 < result ) {
-        result/=tree_order;
-        n_levels++;
-    };
-
-    /* figure out who my children and parents are */
-    my_level_in_tree=-1;
-    result=my_rank;
-    /* cnt - number of ranks in given level */
-    cnt=1;
-    /*  cummulative count of ranks */
-    while( 0 <= result ) {
-        result-=cnt;
-        cnt*=tree_order;
-        my_level_in_tree++;
-    };
-    /* int my_level_in_tree, n_children, n_parents; */
-
-    if( 0 == my_rank ) {
-        my_node->n_parents=0;
-        my_node->parent_rank=-1;
-        my_rank_in_my_level=0;
-    } else {
-        my_node->n_parents=1;
-        cnt=1;
-        cum_cnt=0;
-        for (lvl = 0 ; lvl < my_level_in_tree ; lvl ++ ) {
-            /* cummulative count up to this level */
-            cum_cnt+=cnt;
-            /* number of ranks in this level */
-            cnt*=tree_order;
-        }
-        my_rank_in_my_level=my_rank-cum_cnt;
-        /* tree_order consecutive ranks have the same parent */
-        my_node->parent_rank=cum_cnt-cnt/tree_order+my_rank_in_my_level/tree_order;
-    }
-
-    /* figure out number of levels in the tree */
-    n_lvls_in_tree=0;
-    result=num_nodes;
-    /* cnt - number of ranks in given level */
-    cnt=1;
-    /*  cummulative count of ranks */
-    while( 0 < result ) {
-        result-=cnt;
-        cnt*=tree_order;
-        n_lvls_in_tree++;
-    };
-
-    my_node->children_ranks=(int *)NULL;
-
-    /* get list of children */
-    if( my_level_in_tree == (n_lvls_in_tree -1 ) ) {
-        /* last level has no children */
-        my_node->n_children=0;
-    } else {
-        cum_cnt=0;
-        cnt=1;
-        for( lvl=0 ; lvl <= my_level_in_tree ; lvl++ ) {
-            cum_cnt+=cnt;
-            cnt*=tree_order;
-        }
-        start_index=cum_cnt+my_rank_in_my_level*tree_order;
-        end_index=start_index+tree_order-1;
-
-        /* don't go out of bounds at the end of the list */
-        if( end_index >= num_nodes ) {
-            end_index = num_nodes-1;
-        }
-
-        if( start_index <= (num_nodes-1) ) {
-            my_node->n_children=end_index-start_index+1;
-        } else {
-            my_node->n_children=0;
-        }
-
-        my_node->children_ranks=NULL;
-        if( 0 < my_node->n_children ) {
-            my_node->children_ranks=
-                (int *)malloc( sizeof(int)*my_node->n_children);
-            if( NULL == my_node->children_ranks) {
-                goto Error;
-            }
-            for (lvl= start_index ; lvl <= end_index ; lvl++ ) {
-                my_node->children_ranks[lvl-start_index]=lvl;
-            }
-        }
-    }
-    /* set node type */
-    if( 0 == my_node->n_parents ) {
-        my_node->my_node_type=ROOT_NODE;
-    } else if ( 0 == my_node->n_children ) {
-        my_node->my_node_type=LEAF_NODE;
-    } else {
-        my_node->my_node_type=INTERIOR_NODE;
-    }
-
-
-    /* successful return */
-    return OMPI_SUCCESS;
-
-Error:
-
-    /* error return */
-    return OMPI_ERROR;
-}
-
-void ompi_netpatterns_cleanup_narray_knomial_tree (netpatterns_narray_knomial_tree_node_t *my_node)
-{
-    if (my_node->children_ranks) {
-	free (my_node->children_ranks);
-	my_node->children_ranks = NULL;
-    }
-
-    if (0 != my_node->my_rank) {
-	ompi_netpatterns_cleanup_recursive_knomial_tree_node (&my_node->k_node);
-    }
-}
-
-int ompi_netpatterns_setup_narray_knomial_tree(
-        int tree_order, int my_rank, int num_nodes,
-        netpatterns_narray_knomial_tree_node_t *my_node)
-{
-    /* local variables */
-    int n_levels, result;
-    int my_level_in_tree, cnt ;
-    int lvl,cum_cnt, my_rank_in_my_level,n_lvls_in_tree;
-    int start_index,end_index;
-    int rc;
-
-    /* sanity check */
-    if( 1 >= tree_order ) {
-        goto Error;
-    }
-
-    my_node->my_rank=my_rank;
-    my_node->tree_size=num_nodes;
-
-    /* figure out number of levels in tree */
-    n_levels=0;
-    result=num_nodes-1;
-    while (0 < result ) {
-        result/=tree_order;
-        n_levels++;
-    };
-
-    /* figure out who my children and parents are */
-    my_level_in_tree=-1;
-    result=my_rank;
-    /* cnt - number of ranks in given level */
-    cnt=1;
-    /*  cummulative count of ranks */
-    while( 0 <= result ) {
-        result-=cnt;
-        cnt*=tree_order;
-        my_level_in_tree++;
-    };
-    /* int my_level_in_tree, n_children, n_parents; */
-
-    if( 0 == my_rank ) {
-        my_node->n_parents=0;
-        my_node->parent_rank=-1;
-        my_rank_in_my_level=0;
-    } else {
-        my_node->n_parents=1;
-        cnt=1;
-        cum_cnt=0;
-        for (lvl = 0 ; lvl < my_level_in_tree ; lvl ++ ) {
-            /* cummulative count up to this level */
-            cum_cnt+=cnt;
-            /* number of ranks in this level */
-            cnt*=tree_order;
-        }
-
-        my_node->rank_on_level =
-            my_rank_in_my_level =
-            my_rank-cum_cnt;
-        my_node->level_size = cnt;
-
-        rc = ompi_netpatterns_setup_recursive_knomial_tree_node(
-                my_node->level_size, my_node->rank_on_level,
-                tree_order, &my_node->k_node);
-        if (OMPI_SUCCESS != rc) {
-            goto Error;
-        }
-
-        /* tree_order consecutive ranks have the same parent */
-        my_node->parent_rank=cum_cnt-cnt/tree_order+my_rank_in_my_level/tree_order;
-    }
-
-    /* figure out number of levels in the tree */
-    n_lvls_in_tree=0;
-    result=num_nodes;
-    /* cnt - number of ranks in given level */
-    cnt=1;
-    /*  cummulative count of ranks */
-    while( 0 < result ) {
-        result-=cnt;
-        cnt*=tree_order;
-        n_lvls_in_tree++;
-    };
-
-    if(result < 0) {
-        /* reset the size on group */
-        num_nodes = cnt / tree_order;
-    }
-
-    my_node->children_ranks=(int *)NULL;
-
-    /* get list of children */
-    if( my_level_in_tree == (n_lvls_in_tree -1 ) ) {
-        /* last level has no children */
-        my_node->n_children=0;
-    } else {
-        cum_cnt=0;
-        cnt=1;
-        for( lvl=0 ; lvl <= my_level_in_tree ; lvl++ ) {
-            cum_cnt+=cnt;
-            cnt*=tree_order;
-        }
-        start_index=cum_cnt+my_rank_in_my_level*tree_order;
-        end_index=start_index+tree_order-1;
-
-        /* don't go out of bounds at the end of the list */
-        if( end_index >= num_nodes ) {
-            end_index = num_nodes-1;
-        }
-
-        if( start_index <= (num_nodes-1) ) {
-            my_node->n_children=end_index-start_index+1;
-        } else {
-            my_node->n_children=0;
-        }
-
-        my_node->children_ranks=NULL;
-        if( 0 < my_node->n_children ) {
-            my_node->children_ranks=
-                (int *)malloc( sizeof(int)*my_node->n_children);
-            if( NULL == my_node->children_ranks) {
-                goto Error;
-            }
-            for (lvl= start_index ; lvl <= end_index ; lvl++ ) {
-                my_node->children_ranks[lvl-start_index]=lvl;
-            }
-        }
-    }
-    /* set node type */
-    if( 0 == my_node->n_parents ) {
-        my_node->my_node_type=ROOT_NODE;
-    } else if ( 0 == my_node->n_children ) {
-        my_node->my_node_type=LEAF_NODE;
-    } else {
-        my_node->my_node_type=INTERIOR_NODE;
-    }
-
-
-    /* successful return */
-    return OMPI_SUCCESS;
-
-Error:
-
-    /* error return */
-    return OMPI_ERROR;
-}
-
-/* calculate the nearest power of radix that is equal to or greater
- * than size, with the specified radix.  The resulting tree is of
- * depth n_lvls.
- */
-int ompi_roundup_to_power_radix ( int radix, int size, int *n_lvls )
-{
-    int n_levels=0, return_value=1;
-    int result;
-    if( 1 > size ) {
-        return 0;
-    }
-
-    result=size-1;
-    while (0 < result ) {
-        result/=radix;
-        n_levels++;
-        return_value*=radix;
-    };
-    *n_lvls=n_levels;
-    return return_value;
-}
-
-static int fill_in_node_data(int tree_order, int num_nodes, int my_node,
-        netpatterns_tree_node_t *nodes_data)
-{
-    /* local variables */
-    int rc, num_ranks_per_child, num_children, n_extra;
-    int child, rank, n_to_offset, n_ranks_to_child;
-
-    /* figure out who are my children */
-    num_ranks_per_child=num_nodes/tree_order;
-    if( num_ranks_per_child ) {
-        num_children=tree_order;
-        n_extra=num_nodes-num_ranks_per_child*tree_order;
-    } else {
-        num_children=num_nodes;
-        /* each child has the same number of descendents - 1 */
-        n_extra=0;
-        /* when there is a child, there is at least one
-         * descendent */
-        num_ranks_per_child=1;
-    }
-
-    nodes_data[my_node].n_children=num_children;
-    if( num_children ) {
-        nodes_data[my_node].children_ranks=(int *)
-            malloc(sizeof(int)*num_children);
-        if(!nodes_data[my_node].children_ranks) {
-
-            if ( NULL == nodes_data[my_node].children_ranks )
-            {
-                fprintf(stderr, "Cannot allocate memory for children_ranks.\n");
-                rc = OMPI_ERR_OUT_OF_RESOURCE;
-                goto error;
-            }
-        }
-    }
-
-    rank = my_node;
-    for( child=0 ; child < num_children ; child ++ ) {
-
-    /* set parent information */
-        nodes_data[rank].n_parents=1;
-        nodes_data[rank].parent_rank=my_node;
-        if( n_extra ) {
-            n_to_offset=child;
-            if( n_to_offset > n_extra){
-                n_to_offset=n_extra;
-            }
-        } else {
-            n_to_offset=0;
-        }
-
-        rank=my_node+1+child*num_ranks_per_child;
-        rank+=n_to_offset;
-
-        /* set parent information */
-        nodes_data[rank].n_parents=1;
-        nodes_data[rank].parent_rank=my_node;
-
-        n_ranks_to_child=num_ranks_per_child;
-        if(n_extra && (child < n_extra) ) {
-            n_ranks_to_child++;
-        }
-
-        /* set child information */
-        nodes_data[my_node].children_ranks[child]=rank;
-
-        /* remove the child from the list of ranks */
-        n_ranks_to_child--;
-        rc=fill_in_node_data(tree_order, n_ranks_to_child, rank, nodes_data);
-        if( OMPI_SUCCESS != rc ) {
-            goto error;
-        }
-
-    }
-
-    /* return */
-    return OMPI_SUCCESS;
-
-    /* Error */
-error:
-    return rc;
-
-}
-
-/*
- * This routine sets up the array describing the communication tree for
- * a k-ary tree where the children form a contiguous range of ranks at
- * each level.  The assumption here is that rank 0 is always the root -
- * ranks may be rotated based on who the actual root is, to obtain the
- * appropriate communication pattern for such roots.
- */
-OMPI_DECLSPEC int ompi_netpatterns_setup_narray_tree_contigous_ranks(
-        int tree_order, int num_nodes,
-        netpatterns_tree_node_t **tree_nodes)
-{
-    /* local variables */
-    int num_descendent_ranks=num_nodes-1;
-    int rc=OMPI_SUCCESS;
-
-    *tree_nodes=(netpatterns_tree_node_t *)malloc(
-            sizeof(netpatterns_tree_node_t)*
-            num_nodes);
-    if(!(*tree_nodes) ) {
-        fprintf(stderr, "Cannot allocate memory for tree_nodes.\n");
-        rc = OMPI_ERR_OUT_OF_RESOURCE;
-        return rc;
-    }
-
-    (*tree_nodes)[0].n_parents=0;
-    rc=fill_in_node_data(tree_order,
-            num_descendent_ranks, 0, *tree_nodes);
-
-    /* successful return */
-    return rc;
-
-}
diff --git a/ompi/peruse/peruse.h b/ompi/peruse/peruse.h
index f1f0d817219..ea37cab3ceb 100644
--- a/ompi/peruse/peruse.h
+++ b/ompi/peruse/peruse.h
@@ -36,7 +36,7 @@ typedef int (peruse_comm_callback_f)(peruse_event_h event_h,
 
 /* PERUSE constants */
 enum {
-    PERUSE_SUCCESS = 0,       /* Success *//* XXX Devation from 1.11 */
+    PERUSE_SUCCESS = 0,       /* Success *//* XXX Deviation from 1.11 */
     PERUSE_ERR_INIT,          /* PERUSE initialization failure */
     PERUSE_ERR_GENERIC,       /* Generic unspecified error */
     PERUSE_ERR_MALLOC,        /* Memory-related error */
@@ -49,7 +49,7 @@ enum {
 };
 
 enum {
-    PERUSE_EVENT_INVALID = -1, /* Must differ in value from PERUSE_SUCCESS. Devation from 1.11 */
+    PERUSE_EVENT_INVALID = -1, /* Must differ in value from PERUSE_SUCCESS. Deviation from 1.11 */
 
     /* Point-to-point request events */
     PERUSE_COMM_REQ_ACTIVATE,
@@ -80,13 +80,13 @@ enum {
 
 /* Scope of message queues */
 enum {
-    PERUSE_PER_COMM=0,                  /* XXX Devation from 1.11 */
+    PERUSE_PER_COMM=0,                  /* XXX Deviation from 1.11 */
     PERUSE_GLOBAL
 };
 
 /* Operation values */
 enum {
-    PERUSE_SEND=0,                      /* XXX Devation from 1.11 */
+    PERUSE_SEND=0,                      /* XXX Deviation from 1.11 */
     PERUSE_RECV
 };
 
diff --git a/ompi/proc/proc.c b/ompi/proc/proc.c
index e10068514a3..080506925c6 100644
--- a/ompi/proc/proc.c
+++ b/ompi/proc/proc.c
@@ -352,7 +352,7 @@ int ompi_proc_complete_init(void)
      * are already in the hash table
      */
     if (ompi_process_info.num_procs < ompi_add_procs_cutoff) {
-        /* sinse ompi_proc_for_name is locking internally -
+        /* since ompi_proc_for_name is locking internally -
          * we need to release lock here
          */
         opal_mutex_unlock (&ompi_proc_lock);
diff --git a/ompi/proc/proc.h b/ompi/proc/proc.h
index 5c5d5576cf8..028d1348d79 100644
--- a/ompi/proc/proc.h
+++ b/ompi/proc/proc.h
@@ -286,8 +286,8 @@ OMPI_DECLSPEC ompi_proc_t * ompi_proc_find_and_add(const ompi_process_name_t * n
  * Pack proc list into portable buffer
  *
  * This function takes a list of ompi_proc_t pointers (e.g. as given
- * in groups) and returns a orte buffer containing all information
- * needed to add the proc to a remote list.  This includes the ORTE
+ * in groups) and returns a pmix buffer containing all information
+ * needed to add the proc to a remote list.  This includes the
  * process name, the architecture, and the hostname.  Ordering is
  * maintained.  The buffer is packed to be sent to a remote node with
  * different architecture (endian or word size).
@@ -355,7 +355,7 @@ OMPI_DECLSPEC int ompi_proc_unpack(pmix_data_buffer_t *buf,
  *
  * Refresh the Open MPI process subsystem. This function will update
  * the list of proc instances in the current MPI_COMM_WORLD with
- * data from the run-time environemnt.
+ * data from the run-time environment.
  *
  * @note This is primarily used when restarting a process and thus
  * need to update the jobid and node name.
@@ -471,6 +471,7 @@ static inline void ompi_proc_mark_as_failed(ompi_proc_t *proc) {
         abort();
     }
     proc->proc_active = false;
+    opal_atomic_wmb(); /* non-locked update needs a memory barrier to propagate */
 }
 #endif /* OPAL_ENABLE_FT_MPI */
 
diff --git a/ompi/request/grequest.c b/ompi/request/grequest.c
index 02affd642aa..6125d134a9c 100644
--- a/ompi/request/grequest.c
+++ b/ompi/request/grequest.c
@@ -9,8 +9,9 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2006-2021 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2009      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2021      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -23,6 +24,32 @@
 #include "ompi/request/grequest.h"
 #include "ompi/mpi/fortran/base/fint_2_int.h"
 
+/**
+ * Internal function to specialize the call to the user provided free_fn
+ * for generalized requests.
+ * @return The return value of the user specified callback or MPI_SUCCESS.
+ */
+static inline int ompi_grequest_internal_free(ompi_grequest_t* greq)
+{
+    int rc = MPI_SUCCESS;
+    if (NULL != greq->greq_free.c_free) {
+        /* We were already putting query_fn()'s return value into
+         * status.MPI_ERROR but for MPI_{Wait,Test}*.  If there's a
+         * free callback to invoke, the standard says to use the
+         * return value from free_fn() callback, too.
+         */
+        if (greq->greq_funcs_are_c) {
+            greq->greq_base.req_status.MPI_ERROR =
+                greq->greq_free.c_free(greq->greq_state);
+        } else {
+            MPI_Fint ierr;
+            greq->greq_free.f_free((MPI_Aint*)greq->greq_state, &ierr);
+            greq->greq_base.req_status.MPI_ERROR = OMPI_FINT_2_INT(ierr);
+        }
+        rc = greq->greq_base.req_status.MPI_ERROR;
+    }
+    return rc;
+ }
 
 /*
  * See the comment in the grequest destructor for the weird semantics
@@ -36,9 +63,21 @@
  */
 static int ompi_grequest_free(ompi_request_t** req)
 {
-    OBJ_RELEASE(*req);
-    *req = MPI_REQUEST_NULL;
-    return OMPI_SUCCESS;
+    ompi_grequest_t* greq = (ompi_grequest_t*)*req;
+    int rc = OMPI_SUCCESS;
+
+    if( greq->greq_user_freed ) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+    greq->greq_user_freed = true;
+    if( REQUEST_COMPLETE(*req) ) {
+        rc = ompi_grequest_internal_free(greq);
+    }
+    if (OMPI_SUCCESS == rc ) {
+        OBJ_RELEASE(*req);
+        *req = MPI_REQUEST_NULL;
+    }
+    return rc;
 }
 
 static int ompi_grequest_cancel(ompi_request_t* req, int flag)
@@ -71,6 +110,7 @@ static void ompi_grequest_construct(ompi_grequest_t* greq)
        override this value if the gen request was created from
        Fortran */
     greq->greq_funcs_are_c = true;
+    greq->greq_user_freed = false;
 }
 
 /*
@@ -85,7 +125,7 @@ static void ompi_grequest_construct(ompi_grequest_t* greq)
  *
  * 1. Call to MPI_GREQUEST_COMPLETE and then a corresponding call to
  * some flavor of MPI_TEST* or MPI_WAIT*.  This will both complete the
- * requests and destroy the coresponding MPI generalized request
+ * requests and destroy the corresponding MPI generalized request
  * object.
  *
  * 2. Call MPI_REQUEST_FREE and then (!) -- with some other
@@ -110,7 +150,7 @@ static void ompi_grequest_construct(ompi_grequest_t* greq)
  * TEST* / WAIT* notified the user as such, and this function is also
  * invoked by REQUEST_FREE).  Hence, these two functions will *always*
  * be invoked, but the order in which they are invoked is up to the
- * user.  So this is a perfect opprotunity for the OBJ_* reference
+ * user.  So this is a perfect opportunity for the OBJ_* reference
  * count system.  When we create an ompi_grequest_t in
  * ompi_grequest_start(), we both OBJ_NEW and OBJ_RETAIN it so that
  * its reference count goes to 0.  Then in ompi_grequest_complete()
@@ -121,16 +161,6 @@ static void ompi_grequest_construct(ompi_grequest_t* greq)
  */
 static void ompi_grequest_destruct(ompi_grequest_t* greq)
 {
-    MPI_Fint ierr;
-
-    if (greq->greq_free.c_free != NULL) {
-        if (greq->greq_funcs_are_c) {
-            greq->greq_free.c_free(greq->greq_state);
-        } else {
-            greq->greq_free.f_free((MPI_Aint*)greq->greq_state, &ierr);
-        }
-    }
-
     OMPI_REQUEST_FINI(&greq->greq_base);
 }
 
@@ -180,9 +210,13 @@ int ompi_grequest_start(
  */
 int ompi_grequest_complete(ompi_request_t *req)
 {
+    ompi_grequest_t* greq = (ompi_grequest_t*)req;
     int rc;
 
     rc = ompi_request_complete(req, true);
+    if( greq->greq_user_freed ) {
+        rc = ompi_grequest_internal_free(greq);
+    }
     OBJ_RELEASE(req);
     return rc;
 }
@@ -214,8 +248,21 @@ int ompi_grequest_invoke_query(ompi_request_t *request,
         if (g->greq_funcs_are_c) {
             rc = g->greq_query.c_query(g->greq_state, status);
         } else {
+            /* request->req_status.MPI_ERROR was initialized to success
+             * and it's meant to be unmodified in the case of callback
+             * success, and set when callbacks return a failure.  But
+             * if we leave fstatus uninitialized this sets
+             * req_status.MPI_ERROR to whatever happened to be on the
+             * stack at fstatus (f_query isn't supposed to directly set
+             * its status.MPI_ERROR, according to the standard)
+             *
+             * So the Status_c2f below only really cares about transferring
+             * the MPI_ERROR setting into fstatus so that when it's transferred
+             * back in the f2c call, it has the starting value.
+             */
             MPI_Fint ierr;
             MPI_Fint fstatus[sizeof(MPI_Status) / sizeof(int)];
+            MPI_Status_c2f(status, fstatus);
             g->greq_query.f_query((MPI_Aint*)g->greq_state, fstatus, &ierr);
             MPI_Status_f2c(fstatus, status);
             rc = OMPI_FINT_2_INT(ierr);
diff --git a/ompi/request/grequest.h b/ompi/request/grequest.h
index 09dc5cdc0e4..93545e10b35 100644
--- a/ompi/request/grequest.h
+++ b/ompi/request/grequest.h
@@ -104,6 +104,7 @@ struct ompi_grequest_t {
 #endif
     void *greq_state;
     bool greq_funcs_are_c;
+    bool greq_user_freed;
 };
 /**
  * Convenience typedef
diff --git a/ompi/request/req_ft.c b/ompi/request/req_ft.c
index c6dac235c2b..2c53ce076b0 100644
--- a/ompi/request/req_ft.c
+++ b/ompi/request/req_ft.c
@@ -13,6 +13,9 @@
  * Copyright (c) 2006-2008 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved.
  * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2021      Triad National Security, LLC. All rights
+ *                         reserved.
+ *
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -91,9 +94,9 @@ bool ompi_request_is_failed_fn(ompi_request_t *req)
         req->req_status.MPI_ERROR  = MPI_ERR_REVOKED;
 
         opal_output_verbose(10, ompi_ftmpi_output_handle,
-                            "%s ompi_request_is_failed: %p (peer %d, tag %d) is on communicator %s(%d) that has been revoked!",
+                            "%s ompi_request_is_failed: %p (peer %d, tag %d) is on communicator %s(%s) that has been revoked!",
                             OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), (void*)req, pml_req->req_peer, pml_req->req_tag,
-                            req->req_mpi_object.comm->c_name, req->req_mpi_object.comm->c_contextid);
+                            req->req_mpi_object.comm->c_name, ompi_comm_print_cid(req->req_mpi_object.comm));
         goto return_with_error;
     }
 
@@ -129,9 +132,9 @@ bool ompi_request_is_failed_fn(ompi_request_t *req)
                 req->req_status.MPI_ERROR  = MPI_ERR_PROC_FAILED;
             }
             opal_output_verbose(10, ompi_ftmpi_output_handle,
-                                "%s ompi_request_is_failed: Request %p (peer %d, tag %d) in comm %s(%d) peer ANY_SOURCE %s!",
+                                "%s ompi_request_is_failed: Request %p (peer %d, tag %d) in comm %s(%s) peer ANY_SOURCE %s!",
                                 OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), (void*)req, pml_req->req_peer, pml_req->req_tag,
-                                req->req_mpi_object.comm->c_name, req->req_mpi_object.comm->c_contextid,
+                                req->req_mpi_object.comm->c_name, ompi_comm_print_cid(req->req_mpi_object.comm),
                                 ompi_mpi_errnum_get_string(req->req_status.MPI_ERROR));
             goto return_with_error;
         }
@@ -144,9 +147,9 @@ bool ompi_request_is_failed_fn(ompi_request_t *req)
         req->req_status.MPI_ERROR  = MPI_ERR_PROC_FAILED;
         assert(MPI_ANY_SOURCE != pml_req->req_peer); /* this case is handled above, so... */
         opal_output_verbose(10, ompi_ftmpi_output_handle,
-                            "%s ompi_request_is_failed: Request %p (peer %d, tag %d) in comm %s(%d) mpi_source %3d failed - Ret %s",
+                            "%s ompi_request_is_failed: Request %p (peer %d, tag %d) in comm %s(%s) mpi_source %3d failed - Ret %s",
                             OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), (void*)req, pml_req->req_peer, pml_req->req_tag,
-                            req->req_mpi_object.comm->c_name, req->req_mpi_object.comm->c_contextid,
+                            req->req_mpi_object.comm->c_name, ompi_comm_print_cid(req->req_mpi_object.comm),
                             req->req_status.MPI_SOURCE,
                             ompi_mpi_errnum_get_string(req->req_status.MPI_ERROR));
         goto return_with_error;
@@ -170,14 +173,14 @@ bool ompi_request_is_failed_fn(ompi_request_t *req)
             MCA_PML_CALL(dump(req->req_mpi_object.comm, ompi_ftmpi_output_handle));
         }
 #endif /* OPAL_ENABLE_DEBUG */
-        /* Cancel and force completion immmediately
+        /* Cancel and force completion immediately
          * However, for Revoked and Collective error we can't complete
          * with an error before the buffer is unpinned (i.e. the request gets
          * wire cancelled).
          */
         ompi_request_cancel(req);
         req->req_status._cancelled = cancelled; /* This request is not user cancelled here, it is completed in error */
-        return REQUEST_COMPLETE(req); /* If this request is not complete yet, it is stil ok and needs more spinning */
+        return REQUEST_COMPLETE(req); /* If this request is not complete yet, it is still ok and needs more spinning */
     }
     return (MPI_SUCCESS != req->req_status.MPI_ERROR);
 }
diff --git a/ompi/request/req_test.c b/ompi/request/req_test.c
index 8fc088ac9c1..b28ade8a67a 100644
--- a/ompi/request/req_test.c
+++ b/ompi/request/req_test.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -35,13 +35,12 @@ int ompi_request_default_test(ompi_request_t ** rptr,
 #if OPAL_ENABLE_PROGRESS_THREADS == 0
     int do_it_once = 0;
 
- recheck_request_status:
+recheck_request_status:
 #endif
-    opal_atomic_mb();
     if( request->req_state == OMPI_REQUEST_INACTIVE ) {
         *completed = true;
         if (MPI_STATUS_IGNORE != status) {
-            *status = ompi_status_empty;
+            OMPI_COPY_STATUS(status, ompi_status_empty, false);
         }
         return OMPI_SUCCESS;
     }
@@ -53,17 +52,11 @@ int ompi_request_default_test(ompi_request_t ** rptr,
            STATUS_IGNORE.  See MPI-2:8.2. */
         if (OMPI_REQUEST_GEN == request->req_type) {
             ompi_grequest_invoke_query(request, &request->req_status);
-            if (MPI_STATUS_IGNORE != status) {
-                int old_error = status->MPI_ERROR;
-                *status = request->req_status;
-                status->MPI_ERROR = old_error;
-            }
-        } else if (MPI_STATUS_IGNORE != status) {
-            /* Do *NOT* set a new value for status->MPI_ERROR here!
-               See MPI-1.1 doc, sec 3.2.5, p.22 */
-            int old_error = status->MPI_ERROR;
-            *status = request->req_status;
-            status->MPI_ERROR = old_error;
+        }
+        if (MPI_STATUS_IGNORE != status) {
+            /* make sure we get the correct status */
+            opal_atomic_rmb();
+            OMPI_COPY_STATUS(status, request->req_status, false);
         }
         if( request->req_persistent ) {
             request->req_state = OMPI_REQUEST_INACTIVE;
@@ -94,9 +87,10 @@ int ompi_request_default_test(ompi_request_t ** rptr,
          * If we run the opal_progress then check the status of the request before
          * leaving. We will call the opal_progress only once per call.
          */
-        opal_progress();
-        do_it_once++;
-        goto recheck_request_status;
+        ++do_it_once;
+        if (0 != opal_progress()) {
+            goto recheck_request_status;
+        }
     }
 #endif
     *completed = false;
@@ -115,7 +109,6 @@ int ompi_request_default_test_any(
     ompi_request_t **rptr;
     ompi_request_t *request;
 
-    opal_atomic_mb();
     rptr = requests;
     for (i = 0; i < count; i++, rptr++) {
         request = *rptr;
@@ -134,19 +127,11 @@ int ompi_request_default_test_any(
                STATUS_IGNORE */
             if (OMPI_REQUEST_GEN == request->req_type) {
                 ompi_grequest_invoke_query(request, &request->req_status);
-                if (MPI_STATUS_IGNORE != status) {
-                    /* Do *NOT* set a new value for status->MPI_ERROR
-                       here!  See MPI-1.1 doc, sec 3.2.5, p.22 */
-                    int old_error = status->MPI_ERROR;
-                    *status = request->req_status;
-                    status->MPI_ERROR = old_error;
-                }
-            } else if (MPI_STATUS_IGNORE != status) {
-                /* Do *NOT* set a new value for status->MPI_ERROR
-                   here!  See MPI-1.1 doc, sec 3.2.5, p.22 */
-                int old_error = status->MPI_ERROR;
-                *status = request->req_status;
-                status->MPI_ERROR = old_error;
+            }
+            if (MPI_STATUS_IGNORE != status) {
+                /* make sure we get the correct status */
+                opal_atomic_rmb();
+                OMPI_COPY_STATUS(status, request->req_status, false);
             }
 
             if( request->req_persistent ) {
@@ -184,7 +169,7 @@ int ompi_request_default_test_any(
     } else {
         *completed = true;
         if (MPI_STATUS_IGNORE != status) {
-            *status = ompi_status_empty;
+            OMPI_COPY_STATUS(status, ompi_status_empty, false);
         }
     }
     return OMPI_SUCCESS;
@@ -201,15 +186,14 @@ int ompi_request_default_test_all(
     ompi_request_t **rptr;
     size_t num_completed = 0;
     ompi_request_t *request;
+    int do_it_once = 0;
 
-    opal_atomic_mb();
-    rptr = requests;
-    for (i = 0; i < count; i++, rptr++) {
-        request = *rptr;
+    for (i = 0; i < count; i++) {
+        request = requests[i];
 
-        if( request->req_state == OMPI_REQUEST_INACTIVE ||
-            REQUEST_COMPLETE(request) ) {
+        if( request->req_state == OMPI_REQUEST_INACTIVE || REQUEST_COMPLETE(request) ) {
             num_completed++;
+            continue;
         }
 #if OPAL_ENABLE_FT_MPI
         /* Check for dead requests due to process failure */
@@ -217,20 +201,29 @@ int ompi_request_default_test_all(
         if(OPAL_UNLIKELY( ompi_request_is_failed(request) &&
                           MPI_ERR_PROC_FAILED_PENDING == request->req_status.MPI_ERROR )) {
             if (MPI_STATUSES_IGNORE != statuses) {
-                statuses[i] = request->req_status;
+                OMPI_COPY_STATUS(&statuses[i], request->req_status, true);
                 statuses[i].MPI_ERROR = MPI_ERR_PROC_FAILED_PENDING;
             }
             *completed = false;
             return MPI_ERR_PROC_FAILED_PENDING;
         }
 #endif /* OPAL_ENABLE_FT_MPI */
+#if OPAL_ENABLE_PROGRESS_THREADS == 0
+        if (0 == do_it_once) {
+            ++do_it_once;
+            if (0 != opal_progress()) {
+                /* continue walking the list, retest the current request */
+                --i;
+                continue;
+            }
+        }
+#endif /* OPAL_ENABLE_PROGRESS_THREADS */
+        /* short-circuit */
+        break;
     }
 
     if (num_completed != count) {
         *completed = false;
-#if OPAL_ENABLE_PROGRESS_THREADS == 0
-        opal_progress();
-#endif
         return OMPI_SUCCESS;
     }
 
@@ -239,6 +232,8 @@ int ompi_request_default_test_all(
 
     rc = MPI_SUCCESS;
     if (MPI_STATUSES_IGNORE != statuses) {
+        /* make sure we get the correct statuses */
+        opal_atomic_rmb();
         /* fill out completion status and free request if required */
         for( i = 0; i < count; i++, rptr++ ) {
             request  = *rptr;
@@ -246,13 +241,13 @@ int ompi_request_default_test_all(
              * to ompi_status_empty.
              */
             if( request->req_state == OMPI_REQUEST_INACTIVE ) {
-                statuses[i] = ompi_status_empty;
+                OMPI_COPY_STATUS(&statuses[i], ompi_status_empty, true);
                 continue;
             }
             if (OMPI_REQUEST_GEN == request->req_type) {
                 ompi_grequest_invoke_query(request, &request->req_status);
             }
-            statuses[i] = request->req_status;
+            OMPI_COPY_STATUS(&statuses[i], request->req_status, true);
             if( request->req_persistent ) {
                 request->req_state = OMPI_REQUEST_INACTIVE;
                 continue;
@@ -326,7 +321,6 @@ int ompi_request_default_test_some(
     ompi_request_t **rptr;
     ompi_request_t *request;
 
-    opal_atomic_mb();
     rptr = requests;
     for (i = 0; i < count; i++, rptr++) {
         request = *rptr;
@@ -336,6 +330,7 @@ int ompi_request_default_test_some(
         }
         if( REQUEST_COMPLETE(request) ) {
             indices[num_requests_done++] = i;
+            continue;
         }
 #if OPAL_ENABLE_FT_MPI
         /* Check for dead requests due to process failure */
@@ -364,6 +359,9 @@ int ompi_request_default_test_some(
         return OMPI_SUCCESS;
     }
 
+    /* make sure we get the correct statuses */
+    opal_atomic_rmb();
+
     /* fill out completion status and free request if required */
     for( i = 0; i < num_requests_done; i++) {
         request = requests[indices[i]];
@@ -372,7 +370,7 @@ int ompi_request_default_test_some(
         /* Special case for MPI_ANY_SOURCE */
         if(OPAL_UNLIKELY( MPI_ERR_PROC_FAILED_PENDING == request->req_status.MPI_ERROR )) {
             if (MPI_STATUSES_IGNORE != statuses) {
-                statuses[i] = request->req_status;
+                OMPI_COPY_STATUS(&statuses[i], request->req_status, true);
                 statuses[i].MPI_ERROR = MPI_ERR_PROC_FAILED_PENDING;
             }
             rc = MPI_ERR_PROC_FAILED_PENDING;
@@ -387,7 +385,7 @@ int ompi_request_default_test_some(
             ompi_grequest_invoke_query(request, &request->req_status);
         }
         if (MPI_STATUSES_IGNORE != statuses) {
-            statuses[i] = request->req_status;
+            OMPI_COPY_STATUS(&statuses[i], request->req_status, true);
         }
 
         if (MPI_SUCCESS != request->req_status.MPI_ERROR) {
diff --git a/ompi/request/req_wait.c b/ompi/request/req_wait.c
index 1b46888b905..d66eccf9f79 100644
--- a/ompi/request/req_wait.c
+++ b/ompi/request/req_wait.c
@@ -39,37 +39,32 @@ int ompi_request_default_wait(
 
     ompi_request_wait_completion(req);
 
+    /* make sure we get the correct status */
+    opal_atomic_rmb();
+
 #if OPAL_ENABLE_FT_MPI
     /* Special case for MPI_ANY_SOURCE */
     if( MPI_ERR_PROC_FAILED_PENDING == req->req_status.MPI_ERROR ) {
         if( MPI_STATUS_IGNORE != status ) {
-            status->MPI_TAG    = req->req_status.MPI_TAG;
-            status->MPI_SOURCE = req->req_status.MPI_SOURCE;
-            status->_ucount = req->req_status._ucount;
-            status->_cancelled = req->req_status._cancelled;
+            OMPI_COPY_STATUS(status, req->req_status, false);
         }
         return MPI_ERR_PROC_FAILED_PENDING;
     }
 #endif /* OPAL_ENABLE_FT_MPI */
 
     /* return status.  If it's a generalized request, we *have* to
-       invoke the query_fn, even if the user procided STATUS_IGNORE.
+       invoke the query_fn, even if the user provided STATUS_IGNORE.
        MPI-2:8.2. */
     if (OMPI_REQUEST_GEN == req->req_type) {
         ompi_grequest_invoke_query(req, &req->req_status);
     }
     if( MPI_STATUS_IGNORE != status ) {
-        /* Do *NOT* set status->MPI_ERROR here!  See MPI-1.1 doc, sec
-           3.2.5, p.22 */
-        status->MPI_TAG    = req->req_status.MPI_TAG;
-        status->MPI_SOURCE = req->req_status.MPI_SOURCE;
-        status->_ucount    = req->req_status._ucount;
-        status->_cancelled = req->req_status._cancelled;
+        OMPI_COPY_STATUS(status, req->req_status, false);
     }
     if( req->req_persistent ) {
         if( req->req_state == OMPI_REQUEST_INACTIVE ) {
             if (MPI_STATUS_IGNORE != status) {
-                *status = ompi_status_empty;
+                OMPI_COPY_STATUS(status, ompi_status_empty, false);
             }
             return OMPI_SUCCESS;
         }
@@ -142,7 +137,7 @@ int ompi_request_default_wait_any(size_t count,
     if(num_requests_null_inactive == count) {
         *index = MPI_UNDEFINED;
         if (MPI_STATUS_IGNORE != status) {
-            *status = ompi_status_empty;
+            OMPI_COPY_STATUS(status, ompi_status_empty, false);
         }
         /* No signal-in-flight can be in this case */
         WAIT_SYNC_RELEASE_NOWAIT(&sync);
@@ -206,11 +201,9 @@ int ompi_request_default_wait_any(size_t count,
         rc = ompi_grequest_invoke_query(request, &request->req_status);
     }
     if (MPI_STATUS_IGNORE != status) {
-        /* Do *NOT* set status->MPI_ERROR here!  See MPI-1.1 doc,
-           sec 3.2.5, p.22 */
-        int old_error = status->MPI_ERROR;
-        *status = request->req_status;
-        status->MPI_ERROR = old_error;
+        /* make sure we get the correct status */
+        opal_atomic_rmb();
+        OMPI_COPY_STATUS(status, request->req_status, false);
     }
     rc = request->req_status.MPI_ERROR;
     if( request->req_persistent ) {
@@ -255,7 +248,7 @@ int ompi_request_default_wait_all( size_t count,
             continue;
         }
 
-        if (!OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&request->req_complete, &_tmp_ptr, &sync)) {
+        if (REQUEST_COMPLETE(request) || !OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&request->req_complete, &_tmp_ptr, &sync)) {
             if( OPAL_LIKELY( REQUEST_COMPLETE(request) ) ) {
                 if( OPAL_UNLIKELY( MPI_SUCCESS != request->req_status.MPI_ERROR ) ) {
                     failed++;
@@ -314,6 +307,9 @@ int ompi_request_default_wait_all( size_t count,
  finish:
     rptr = requests;
     if (MPI_STATUSES_IGNORE != statuses) {
+        /* make sure we get the correct status */
+        opal_atomic_rmb();
+
         /* fill out status and free request if required */
         for( i = 0; i < count; i++, rptr++ ) {
             void *_tmp_ptr = &sync;
@@ -321,14 +317,14 @@ int ompi_request_default_wait_all( size_t count,
             request = *rptr;
 
             if( request->req_state == OMPI_REQUEST_INACTIVE ) {
-                statuses[i] = ompi_status_empty;
+                OMPI_COPY_STATUS(&statuses[i], ompi_status_empty, true);
                 continue;
             }
 
             if( OPAL_UNLIKELY(0 < failed) ) {
                 /* if we have failed requests we skipped the waiting on the sync. Thus,
                  * some of the requests might not be properly completed, in which case
-                 * we must detach all requests from the sync. However, if we can succesfully
+                 * we must detach all requests from the sync. However, if we can successfully
                  * mark the request as pending then it is neither failed nor complete, and
                  * we must stop altering it.
                  */
@@ -356,7 +352,7 @@ int ompi_request_default_wait_all( size_t count,
                 ompi_grequest_invoke_query(request, &request->req_status);
             }
 
-            statuses[i] = request->req_status;
+            OMPI_COPY_STATUS(&statuses[i], request->req_status, true);
 
             if( request->req_persistent ) {
                 request->req_state = OMPI_REQUEST_INACTIVE;
@@ -513,7 +509,7 @@ int ompi_request_default_wait_some(size_t count,
 
     if(num_requests_null_inactive == count) {
         *outcount = MPI_UNDEFINED;
-        /* nobody will signall us */
+        /* nobody will signal us */
         WAIT_SYNC_RELEASE_NOWAIT(&sync);
         return rc;
     }
@@ -586,6 +582,9 @@ int ompi_request_default_wait_some(size_t count,
 
     *outcount = num_requests_done;
 
+    /* make sure we get the correct status */
+    opal_atomic_rmb();
+
     for (size_t i = 0; i < num_requests_done; i++) {
         request = requests[indices[i]];
 #if OPAL_ENABLE_FT_MPI
@@ -593,7 +592,7 @@ int ompi_request_default_wait_some(size_t count,
         if( MPI_ERR_PROC_FAILED_PENDING == request->req_status.MPI_ERROR ) {
             rc = MPI_ERR_IN_STATUS;
             if (MPI_STATUSES_IGNORE != statuses) {
-                statuses[i] = request->req_status;
+                OMPI_COPY_STATUS(&statuses[i], request->req_status, true);
                 statuses[i].MPI_ERROR = MPI_ERR_PROC_FAILED_PENDING;
             } else {
                 if( (MPI_ERR_PROC_FAILED == request->req_status.MPI_ERROR) ||
@@ -612,7 +611,7 @@ int ompi_request_default_wait_some(size_t count,
             ompi_grequest_invoke_query(request, &request->req_status);
         }
         if (MPI_STATUSES_IGNORE != statuses) {
-            statuses[i] = request->req_status;
+            OMPI_COPY_STATUS(&statuses[i], request->req_status, true);
         }
 
         if (MPI_SUCCESS != request->req_status.MPI_ERROR) {
diff --git a/ompi/request/request.c b/ompi/request/request.c
index abf33449d89..3f72a23fde0 100644
--- a/ompi/request/request.c
+++ b/ompi/request/request.c
@@ -15,9 +15,13 @@
  * Copyright (c) 2012      Oak Ridge National Labs.  All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
  *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015-2024 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2018      Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -32,11 +36,17 @@
 #include "ompi/request/request.h"
 #include "ompi/request/request_default.h"
 #include "ompi/constants.h"
+#if MPI_VERSION >= 4
+#include "ompi/mca/pml/base/pml_base_sendreq.h"
+#endif
 
 opal_pointer_array_t             ompi_request_f_to_c_table = {{0}};
 ompi_predefined_request_t        ompi_request_null = {{{{{0}}}}};
 ompi_predefined_request_t        *ompi_request_null_addr = &ompi_request_null;
 ompi_request_t                   ompi_request_empty = {{{{0}}}};
+#if MPI_VERSION >= 4
+ompi_request_t                   ompi_request_empty_send = {{{{0}}}};
+#endif
 ompi_status_public_t             ompi_status_empty = {0};
 ompi_request_fns_t               ompi_request_functions = {
     ompi_request_default_test,
@@ -104,6 +114,16 @@ OBJ_CLASS_INSTANCE(
     ompi_request_destruct);
 
 
+static int ompi_request_finalize (void)
+{
+    OMPI_REQUEST_FINI( &ompi_request_null.request );
+    OBJ_DESTRUCT( &ompi_request_null.request );
+    OMPI_REQUEST_FINI( &ompi_request_empty );
+    OBJ_DESTRUCT( &ompi_request_empty );
+    OBJ_DESTRUCT( &ompi_request_f_to_c_table );
+    return OMPI_SUCCESS;
+}
+
 int ompi_request_init(void)
 {
 
@@ -167,27 +187,47 @@ int ompi_request_init(void)
         return OMPI_ERR_REQUEST;
     }
 
+#if MPI_VERSION >= 4
+    /*
+     * This is a copy of the ompi_request_empty object where the only difference
+     * is that the req_cancel callback is set to a callback which issues a
+     * message that calling MPI_Cancel on a non-blocking send request is
+     * deprecated.
+     */
+    OBJ_CONSTRUCT(&ompi_request_empty_send, ompi_request_t);
+    ompi_request_empty_send.req_type = OMPI_REQUEST_NULL;
+    ompi_request_empty_send.req_status.MPI_SOURCE = MPI_PROC_NULL;
+    ompi_request_empty_send.req_status.MPI_TAG = MPI_ANY_TAG;
+    ompi_request_empty_send.req_status.MPI_ERROR = MPI_SUCCESS;
+    ompi_request_empty_send.req_status._ucount = 0;
+    ompi_request_empty_send.req_status._cancelled = 0;
+
+    ompi_request_empty_send.req_complete = REQUEST_COMPLETED;
+    ompi_request_empty_send.req_state = OMPI_REQUEST_ACTIVE;
+    ompi_request_empty_send.req_persistent = false;
+    ompi_request_empty_send.req_f_to_c_index =
+        opal_pointer_array_add(&ompi_request_f_to_c_table, &ompi_request_empty_send);
+    ompi_request_empty_send.req_start = NULL; /* should not be called */
+    ompi_request_empty_send.req_free = ompi_request_empty_free;
+    ompi_request_empty_send.req_cancel = mca_pml_cancel_send_callback;
+    ompi_request_empty_send.req_mpi_object.comm = &ompi_mpi_comm_world.comm;
+
+    if (2 != ompi_request_empty_send.req_f_to_c_index) {
+        return OMPI_ERR_REQUEST;
+    }
+#endif
+
     ompi_status_empty.MPI_SOURCE = MPI_ANY_SOURCE;
     ompi_status_empty.MPI_TAG = MPI_ANY_TAG;
     ompi_status_empty.MPI_ERROR = MPI_SUCCESS;
     ompi_status_empty._ucount = 0;
     ompi_status_empty._cancelled = 0;
 
-    return OMPI_SUCCESS;
-}
+    ompi_mpi_instance_append_finalize (ompi_request_finalize);
 
-
-int ompi_request_finalize(void)
-{
-    OMPI_REQUEST_FINI( &ompi_request_null.request );
-    OBJ_DESTRUCT( &ompi_request_null.request );
-    OMPI_REQUEST_FINI( &ompi_request_empty );
-    OBJ_DESTRUCT( &ompi_request_empty );
-    OBJ_DESTRUCT( &ompi_request_f_to_c_table );
     return OMPI_SUCCESS;
 }
 
-
 int ompi_request_persistent_noop_create(ompi_request_t** request)
 {
     ompi_request_t *req;
@@ -209,3 +249,28 @@ int ompi_request_persistent_noop_create(ompi_request_t** request)
     *request = req;
     return OMPI_SUCCESS;
 }
+
+bool ompi_request_check_same_instance(ompi_request_t** requests,
+                                      int count)
+{
+    ompi_instance_t* base_instance = NULL;
+    ompi_request_t *req;
+
+    for(int idx = 0; idx < count; idx++ ) {
+        req = requests[idx];
+        if(OMPI_REQUEST_NULL == req->req_type)  /* predefined requests are generic */
+            continue;
+        /* Only PML requests have support for MPI sessions */
+        if(OMPI_REQUEST_PML != req->req_type)
+            continue;
+        if(NULL == base_instance) {
+            base_instance = req->req_mpi_object.comm->instance;
+            continue;
+        }
+        if(&ompi_mpi_comm_null.comm == req->req_mpi_object.comm)
+            continue;
+        if(base_instance != req->req_mpi_object.comm->instance)
+            return false;
+    }
+    return true;
+}
diff --git a/ompi/request/request.h b/ompi/request/request.h
index eb3f829b45b..548c053a7bc 100644
--- a/ompi/request/request.h
+++ b/ompi/request/request.h
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -16,6 +16,10 @@
  * Copyright (c) 2015-2017 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2018      Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -127,7 +131,7 @@ struct ompi_request_t {
     opal_free_list_item_t super;                /**< Base type */
     ompi_request_type_t req_type;               /**< Enum indicating the type of the request */
     ompi_status_public_t req_status;            /**< Completion status */
-    volatile void *req_complete;                /**< Flag indicating wether request has completed */
+    volatile void *req_complete;                /**< Flag indicating whether request has completed */
     volatile ompi_request_state_t req_state;    /**< enum indicate state of the request */
     bool req_persistent;                        /**< flag indicating if the this is a persistent request */
     int req_f_to_c_index;                       /**< Index in Fortran <-> C translation array */
@@ -146,7 +150,7 @@ typedef struct ompi_request_t ompi_request_t;
 
 
 /**
- * Padded struct to maintain back compatibiltiy.
+ * Padded struct to maintain back compatibility.
  * See ompi/communicator/communicator.h comments with struct ompi_communicator_t
  * for full explanation why we chose the following padding construct for predefines.
  */
@@ -193,7 +197,7 @@ typedef struct ompi_predefined_request_t ompi_predefined_request_t;
  * This function should be called only from the MPI layer. It should
  * never be called from the PML. It take care of the upper level clean-up.
  * When the user call MPI_Request_free we should release all MPI level
- * ressources, so we have to call this function too.
+ * resources, so we have to call this function too.
  */
 #define OMPI_REQUEST_FINI(request)                                      \
 do {                                                                    \
@@ -205,6 +209,28 @@ do {                                                                    \
     }                                                                   \
 } while (0);
 
+/*
+ * Except in procedures that return MPI_ERR_IN_STATUS, the MPI_ERROR
+ * field of a status object shall never be modified
+ * See MPI-1.1 doc, sec 3.2.5, p.22
+ *
+ * Add a small macro that helps setting the status appropriately
+ * depending on the use case
+ */
+#define OMPI_COPY_STATUS(pdst, src, is_err_in_status)                   \
+do {                                                                    \
+    if (is_err_in_status) {                                             \
+        *(pdst) = (src);                                                \
+    }                                                                   \
+    else {                                                              \
+        (pdst)->MPI_TAG = (src).MPI_TAG;                                \
+        (pdst)->MPI_SOURCE = (src).MPI_SOURCE;                          \
+        (pdst)->_ucount = (src)._ucount;                                \
+        (pdst)->_cancelled = (src)._cancelled;                          \
+    }                                                                   \
+} while(0);
+
+
 /**
  * Non-blocking test for request completion.
  *
@@ -242,7 +268,7 @@ typedef int (*ompi_request_test_any_fn_t)(size_t count,
  *
  * @param count (IN)      Number of requests
  * @param requests (IN)   Array of requests
- * @param completed (OUT) Flag indicating wether all requests completed.
+ * @param completed (OUT) Flag indicating whether all requests completed.
  * @param statuses (OUT)  Array of completion statuses.
  * @return                OMPI_SUCCESS or failure status.
  *
@@ -359,17 +385,15 @@ OMPI_DECLSPEC extern ompi_predefined_request_t        *ompi_request_null_addr;
 OMPI_DECLSPEC extern ompi_request_t         ompi_request_empty;
 OMPI_DECLSPEC extern ompi_status_public_t   ompi_status_empty;
 OMPI_DECLSPEC extern ompi_request_fns_t     ompi_request_functions;
+#if MPI_VERSION >= 4
+OMPI_DECLSPEC extern ompi_request_t         ompi_request_empty_send;
+#endif
 
 /**
  * Initialize the MPI_Request subsystem; invoked during MPI_INIT.
  */
 int ompi_request_init(void);
 
-/**
- * Shut down the MPI_Request subsystem; invoked during MPI_FINALIZE.
- */
-int ompi_request_finalize(void);
-
 /**
  * Create a persistent request that does nothing (e.g., to MPI_PROC_NULL).
  */
@@ -426,39 +450,44 @@ static inline bool ompi_request_tag_is_collective(int tag) {
 
 static inline void ompi_request_wait_completion(ompi_request_t *req)
 {
-    if (opal_using_threads () && !REQUEST_COMPLETE(req)) {
-        void *_tmp_ptr;
-        ompi_wait_sync_t sync;
+    if (opal_using_threads ()) {
+        if(!REQUEST_COMPLETE(req)) {
+            void *_tmp_ptr;
+            ompi_wait_sync_t sync;
+
+
 #if OPAL_ENABLE_FT_MPI
-redo:
-        if(OPAL_UNLIKELY( ompi_request_is_failed(req) )) {
-            return;
-        }
+    redo:
+            if(OPAL_UNLIKELY( ompi_request_is_failed(req) )) {
+                return;
+            }
 #endif /* OPAL_ENABLE_FT_MPI */
-        _tmp_ptr = REQUEST_PENDING;
+            _tmp_ptr = REQUEST_PENDING;
 
-        WAIT_SYNC_INIT(&sync, 1);
+            WAIT_SYNC_INIT(&sync, 1);
 
-        if (OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&req->req_complete, &_tmp_ptr, &sync)) {
-            SYNC_WAIT(&sync);
-        } else {
-            /* completed before we had a chance to swap in the sync object */
-            WAIT_SYNC_SIGNALLED(&sync);
-        }
+            if (OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&req->req_complete, &_tmp_ptr, &sync)) {
+                SYNC_WAIT(&sync);
+            } else {
+                /* completed before we had a chance to swap in the sync object */
+                WAIT_SYNC_SIGNALLED(&sync);
+            }
 
 #if OPAL_ENABLE_FT_MPI
-        if (OPAL_UNLIKELY(OMPI_SUCCESS != sync.status)) {
-            OPAL_OUTPUT_VERBOSE((50, ompi_ftmpi_output_handle, "Status %d reported for sync %p rearming req %p", sync.status, (void*)&sync, (void*)req));
-            _tmp_ptr = &sync;
-            if (OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&req->req_complete, &_tmp_ptr, REQUEST_PENDING)) {
-                opal_output_verbose(10, ompi_ftmpi_output_handle, "Status %d reported for sync %p rearmed req %p", sync.status, (void*)&sync, (void*)req);
-                WAIT_SYNC_RELEASE(&sync);
-                goto redo;
+            if (OPAL_UNLIKELY(OMPI_SUCCESS != sync.status)) {
+                OPAL_OUTPUT_VERBOSE((50, ompi_ftmpi_output_handle, "Status %d reported for sync %p rearming req %p", sync.status, (void*)&sync, (void*)req));
+                _tmp_ptr = &sync;
+                if (OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&req->req_complete, &_tmp_ptr, REQUEST_PENDING)) {
+                    opal_output_verbose(10, ompi_ftmpi_output_handle, "Status %d reported for sync %p rearmed req %p", sync.status, (void*)&sync, (void*)req);
+                    WAIT_SYNC_RELEASE(&sync);
+                    goto redo;
+                }
             }
-        }
 #endif /* OPAL_ENABLE_FT_MPI */
-        assert(REQUEST_COMPLETE(req));
-        WAIT_SYNC_RELEASE(&sync);
+            assert(REQUEST_COMPLETE(req));
+            WAIT_SYNC_RELEASE(&sync);
+     }
+     opal_atomic_rmb();
     } else {
         while(!REQUEST_COMPLETE(req)) {
             opal_progress();
@@ -498,18 +527,18 @@ static inline int ompi_request_complete(ompi_request_t* request, bool with_signa
     }
 
     if (0 == rc) {
-        if( OPAL_LIKELY(with_signal) ) {
-            void *_tmp_ptr = REQUEST_PENDING;
-
-            if(!OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&request->req_complete, &_tmp_ptr, REQUEST_COMPLETED)) {
-                ompi_wait_sync_t *tmp_sync = (ompi_wait_sync_t *) OPAL_ATOMIC_SWAP_PTR(&request->req_complete,
-                                                                                       REQUEST_COMPLETED);
-                /* In the case where another thread concurrently changed the request to REQUEST_PENDING */
-                if( REQUEST_PENDING != tmp_sync )
-                    wait_sync_update(tmp_sync, 1, request->req_status.MPI_ERROR);
+        if (OPAL_LIKELY(with_signal)) {
+            /* make sure everything in the request is visible before we mark it complete */
+            opal_atomic_wmb();
+
+            ompi_wait_sync_t *tmp_sync = (ompi_wait_sync_t *) OPAL_ATOMIC_SWAP_PTR(&request->req_complete,
+                                                                                    REQUEST_COMPLETED);
+            if( REQUEST_PENDING != tmp_sync ) {
+                wait_sync_update(tmp_sync, 1, request->req_status.MPI_ERROR);
             }
-        } else
+        } else {
             request->req_complete = REQUEST_COMPLETED;
+        }
     }
 
     return OMPI_SUCCESS;
@@ -530,6 +559,22 @@ static inline int ompi_request_set_callback(ompi_request_t* request,
     return OMPI_SUCCESS;
 }
 
+/**
+ * Check if an array of requests contains only requests that belong to the same
+ * instance or MPI Session. Exclude predefined requests, that are generic and part
+ * of all instances.
+ * Right now, only tests for PML requests as they are the only requests that have
+ * support for MPI Sessions.
+ *
+ * @param requests (IN)   Array of requests
+ * @param count (IN)      Number of requests
+ * @return                true if all requests are part of the same instance
+ *                        or MPI Session, false otherwise.
+ *
+ */
+bool ompi_request_check_same_instance(ompi_request_t** requests,
+                                      int count);
+
 END_C_DECLS
 
 #endif
diff --git a/ompi/request/request_dbg.h b/ompi/request/request_dbg.h
index 7251b96dc5b..5929374ade4 100644
--- a/ompi/request/request_dbg.h
+++ b/ompi/request/request_dbg.h
@@ -1,6 +1,7 @@
 /* -*- Mode: C; c-basic-offset:4 ; -*- */
 /*
  * Copyright (c) 2009      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -17,7 +18,7 @@
  */
 
 /**
- * Enum inidicating the type of the request
+ * Enum indicating the type of the request
  */
 typedef enum {
     OMPI_REQUEST_PML,      /**< MPI point-to-point request */
@@ -28,6 +29,7 @@ typedef enum {
     OMPI_REQUEST_NULL,     /**< NULL request */
     OMPI_REQUEST_NOOP,     /**< A request that does nothing (e.g., to PROC_NULL) */
     OMPI_REQUEST_COMM,     /**< MPI-3 non-blocking communicator duplication */
+    OMPI_REQUEST_PART,     /**< MPI-4 partitioned communication request */
     OMPI_REQUEST_MAX       /**< Maximum request type */
 } ompi_request_type_t;
 
diff --git a/ompi/runtime/Makefile.am b/ompi/runtime/Makefile.am
index 2a31779b9fd..957045ed116 100644
--- a/ompi/runtime/Makefile.am
+++ b/ompi/runtime/Makefile.am
@@ -12,6 +12,7 @@
 # Copyright (c) 2006-2015 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2014-2019 Intel, Inc.  All rights reserved.
 # Copyright (c) 2016      IBM Corporation.  All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -21,7 +22,7 @@
 
 # This makefile.am does not stand on its own - it is included from ompi/Makefile.am
 
-dist_ompidata_DATA += runtime/help-mpi-runtime.txt
+EXTRA_DIST += runtime/help-mpi-runtime.txt
 
 headers += \
         runtime/mpiruntime.h \
diff --git a/ompi/runtime/help-mpi-runtime.txt b/ompi/runtime/help-mpi-runtime.txt
index 1fcb93a35e9..c20b723b3ea 100644
--- a/ompi/runtime/help-mpi-runtime.txt
+++ b/ompi/runtime/help-mpi-runtime.txt
@@ -10,7 +10,7 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2007-2018 Cisco Systems, Inc.  All rights reserved
+# Copyright (c) 2007-2022 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2013      NVIDIA Corporation.  All rights reserved.
 # Copyright (c) 2017-2020 Intel, Inc.  All rights reserved.
 # $COPYRIGHT$
@@ -95,16 +95,6 @@ Open MPI with --enable-heterogeneous.
 The user requested CUDA support with the --mca mpi_cuda_support 1 flag
 but the library was not compiled with any support.
 #
-[noconxcpt]
-The user has called an operation involving MPI_Connect and/or MPI_Accept,
-but this environment lacks the necessary infrastructure support for
-that operation. Open MPI relies on the PMIx_Publish/Lookup (or one of
-its predecessors) APIs for this operation.
-
-This typically happens when launching outside of mpirun where the underlying
-resource manager does not provide publish/lookup support. One way of solving
-the problem is to simply use mpirun to start the application.
-#
 [lib-call-fail]
 A library call unexpectedly failed.  This is a terminal error; please
 show this message to an Open MPI wizard:
@@ -128,3 +118,7 @@ PMIx_Init failed for the following reason:
 Open MPI requires access to a local PMIx server to execute. Please ensure
 that either you are operating in a PMIx-enabled environment, or use "mpirun"
 to execute the job.
+#
+[no-pmix-but]
+No PMIx server was reachable, but a PMI1/2 was detected.
+If srun is being used to launch application,  %d singletons will be started.
diff --git a/ompi/runtime/mpiruntime.h b/ompi/runtime/mpiruntime.h
index 81c9741c2e2..5ce1ce53539 100644
--- a/ompi/runtime/mpiruntime.h
+++ b/ompi/runtime/mpiruntime.h
@@ -90,7 +90,7 @@ typedef enum {
 
 /*
  * These variables are for the MPI F03 bindings (F03 must bind Fortran
- * varaiables to symbols; it cannot bind Fortran variables to the
+ * variables to symbols; it cannot bind Fortran variables to the
  * address of a C variable).
  */
 
@@ -151,7 +151,7 @@ OMPI_DECLSPEC extern struct ompi_status_public_t *ompi_mpi_statuses_ignore_addr;
     MPI_FINALIZE. */
 OMPI_DECLSPEC extern opal_list_t ompi_registered_datareps;
 
-/** In ompi_mpi_init: the lists of Fortran 90 mathing datatypes.
+/** In ompi_mpi_init: the lists of Fortran 90 matching datatypes.
  * We need these lists and hashtables in order to satisfy the new
  * requirements introduced in MPI 2-1 Sect. 10.2.5,
  * MPI_TYPE_CREATE_F90_xxxx, page 295, line 47.
@@ -184,12 +184,12 @@ void ompi_mpi_thread_level(int requested, int *provided);
  * @returns MPI_SUCCESS if successful
  * @returns Error code if unsuccessful
  *
- * Intialize all support code needed for MPI applications.  This
+ * Initialize all support code needed for MPI applications.  This
  * function should only be called by MPI applications (including
  * singletons).  If this function is called, ompi_init() and
  * ompi_rte_init() should *not* be called.
  *
- * It is permissable to pass in (0, NULL) for (argc, argv).
+ * It is permissible to pass in (0, NULL) for (argc, argv).
  */
 int ompi_mpi_init(int argc, char **argv, int requested, int *provided,
                   bool reinit_ok);
@@ -242,12 +242,6 @@ void ompi_mpi_dynamics_disable(const char *msg);
  */
 bool ompi_mpi_dynamics_is_enabled(const char *function);
 
-/**
- * Clean up memory / resources by the MPI dynamics process
- * functionality checker
- */
-void ompi_mpi_dynamics_finalize(void);
-
 END_C_DECLS
 
 #endif /* OMPI_MPI_MPIRUNTIME_H */
diff --git a/ompi/runtime/ompi_info_support.h b/ompi/runtime/ompi_info_support.h
index 17283182f6c..ed3392bbc36 100644
--- a/ompi/runtime/ompi_info_support.h
+++ b/ompi/runtime/ompi_info_support.h
@@ -20,7 +20,7 @@
  * $HEADER$
  */
 
-#if !defined(OMPI_INFO_SUPPORT_H)
+#ifndef OMPI_INFO_SUPPORT_H
 #define OMPI_INFO_SUPPORT_H
 
 #include "opal/class/opal_pointer_array.h"
@@ -33,4 +33,4 @@ OMPI_DECLSPEC void  ompi_info_close_components(void);
 
 OMPI_DECLSPEC void ompi_info_show_ompi_version(const char *scope);
 
-#endif /* !defined(OMPI_INFO_SUPPORT_H) */
+#endif /* OMPI_INFO_SUPPORT_H */
diff --git a/ompi/runtime/ompi_mpi_abort.c b/ompi/runtime/ompi_mpi_abort.c
index 1c9961215c0..bfb78114590 100644
--- a/ompi/runtime/ompi_mpi_abort.c
+++ b/ompi/runtime/ompi_mpi_abort.c
@@ -19,7 +19,7 @@
  * Copyright (c) 2015      Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2017      FUJITSU LIMITED.  All rights reserved.
- * Copyright (c) 2019      Triad National Security, LLC. All rights
+ * Copyright (c) 2019-2021 Triad National Security, LLC. All rights
  *                         reserved.
  * $COPYRIGHT$
  *
@@ -189,8 +189,7 @@ ompi_mpi_abort(struct ompi_communicator_t* comm,
     /* We can fall through to here in a few cases:
 
        1. The attempt to kill just a subset of peers via
-          try_kill_peers() failed (e.g., as of July 2014, ORTE does
-          returns NOT_IMPLENTED from orte_rte_abort_peers()).
+          try_kill_peers() failed.
        2. MPI wasn't initialized, was already finalized, or we got a
           NULL communicator.
 
@@ -198,5 +197,6 @@ ompi_mpi_abort(struct ompi_communicator_t* comm,
        kill the entire job.  Wah wah. */
     ompi_rte_abort(errcode, NULL);
 
-    /* Does not return */
+    /* Does not return - but we add a return to keep compiler warnings at bay*/
+    return 0;
 }
diff --git a/ompi/runtime/ompi_mpi_dynamics.c b/ompi/runtime/ompi_mpi_dynamics.c
index 9e9f92f84f8..25403c91892 100644
--- a/ompi/runtime/ompi_mpi_dynamics.c
+++ b/ompi/runtime/ompi_mpi_dynamics.c
@@ -28,16 +28,30 @@
 
 #include "ompi/runtime/params.h"
 #include "ompi/runtime/mpiruntime.h"
+#include "ompi/instance/instance.h"
 
 static char *ompi_mpi_dynamics_disabled_msg = "Enabled";
 
 
+static int ompi_mpi_dynamics_finalize (void)
+{
+    // If dynamics were disabled, then we have a message to free
+    if (!ompi_mpi_dynamics_enabled) {
+        free(ompi_mpi_dynamics_disabled_msg);
+        ompi_mpi_dynamics_disabled_msg = NULL;
+    }
+
+    return OMPI_SUCCESS;
+}
+
 void ompi_mpi_dynamics_disable(const char *msg)
 {
     assert(msg);
 
     ompi_mpi_dynamics_enabled = false;
     ompi_mpi_dynamics_disabled_msg = strdup(msg);
+
+    ompi_mpi_instance_append_finalize (ompi_mpi_dynamics_finalize);
 }
 
 bool ompi_mpi_dynamics_is_enabled(const char *function)
@@ -53,12 +67,3 @@ bool ompi_mpi_dynamics_is_enabled(const char *function)
                    ompi_mpi_dynamics_disabled_msg);
     return false;
 }
-
-void ompi_mpi_dynamics_finalize(void)
-{
-    // If dynamics were disabled, then we have a message to free
-    if (!ompi_mpi_dynamics_enabled) {
-        free(ompi_mpi_dynamics_disabled_msg);
-        ompi_mpi_dynamics_disabled_msg = NULL;
-    }
-}
diff --git a/ompi/runtime/ompi_mpi_finalize.c b/ompi/runtime/ompi_mpi_finalize.c
index 6274db06e86..ad8a328dc55 100644
--- a/ompi/runtime/ompi_mpi_finalize.c
+++ b/ompi/runtime/ompi_mpi_finalize.c
@@ -15,13 +15,12 @@
  *                         reserved.
  * Copyright (c) 2006      University of Houston. All rights reserved.
  * Copyright (c) 2009      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2011-2020 Sandia National Laboratories. All rights reserved.
  * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- *
  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
- * Copyright (c) 2019      Triad National Security, LLC. All rights
+ * Copyright (c) 2019-2022 Triad National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2020      Amazon.com, Inc. or its affiliates.
  *                         All Rights reserved.
@@ -61,7 +60,6 @@
 #include "opal/mca/allocator/base/base.h"
 #include "opal/mca/pmix/pmix-internal.h"
 #include "opal/util/timings.h"
-
 #include "mpi.h"
 #include "ompi/constants.h"
 #include "ompi/errhandler/errcode.h"
@@ -78,6 +76,7 @@
 #include "ompi/mca/pml/base/base.h"
 #include "ompi/mca/bml/base/base.h"
 #include "ompi/mca/osc/base/base.h"
+#include "ompi/mca/part/base/base.h"
 #include "ompi/mca/coll/base/coll_base_functions.h"
 #include "ompi/mca/coll/base/base.h"
 #include "ompi/runtime/ompi_rte.h"
@@ -86,9 +85,15 @@
 #include "ompi/mca/io/base/base.h"
 #include "ompi/mca/pml/base/pml_base_bsend.h"
 #include "ompi/runtime/params.h"
-#include "ompi/dpm/dpm.h"
-#include "ompi/mpiext/mpiext.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/mca/hook/hook.h"
 #include "ompi/mca/hook/base/base.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/attribute/attribute.h"
+#include "ompi/instance/instance.h"
+
+#include "mpi.h"
+#include "ompi/constants.h"
 
 extern bool ompi_enable_timing;
 
@@ -103,13 +108,8 @@ static void fence_cbfunc(pmix_status_t status, void *cbdata)
 int ompi_mpi_finalize(void)
 {
     int ret = MPI_SUCCESS;
-    opal_list_item_t *item;
-    ompi_proc_t** procs;
-    size_t nprocs;
-    volatile bool active;
-    uint32_t key;
-    ompi_datatype_t * datatype;
     pmix_status_t rc;
+    volatile bool active;
 
     ompi_hook_base_mpi_finalize_top();
 
@@ -137,8 +137,6 @@ int ompi_mpi_finalize(void)
     opal_atomic_wmb();
     opal_atomic_swap_32(&ompi_mpi_state, OMPI_MPI_STATE_FINALIZE_STARTED);
 
-    ompi_mpiext_fini();
-
     /* Per MPI-2:4.8, we have to free MPI_COMM_SELF before doing
        anything else in MPI_FINALIZE (to include setting up such that
        MPI_FINALIZED will return true). */
@@ -195,19 +193,10 @@ int ompi_mpi_finalize(void)
     opal_atomic_swap_32(&ompi_mpi_state,
                         OMPI_MPI_STATE_FINALIZE_PAST_COMM_SELF_DESTRUCT);
 
-    /* As finalize is the last legal MPI call, we are allowed to force the release
-     * of the user buffer used for bsend, before going anywhere further.
-     */
-    (void)mca_pml_base_bsend_detach(NULL, NULL);
-
 #if OPAL_ENABLE_PROGRESS_THREADS == 0
     opal_progress_set_event_flag(OPAL_EVLOOP_ONCE | OPAL_EVLOOP_NONBLOCK);
 #endif
 
-    /* Redo ORTE calling opal_progress_event_users_increment() during
-       MPI lifetime, to get better latency when not using TCP */
-    opal_progress_event_users_increment();
-
     /* NOTE: MPI-2.1 requires that MPI_FINALIZE is "collective" across
        *all* connected processes.  This only means that all processes
        have to call it.  It does *not* mean that all connected
@@ -283,7 +272,7 @@ int ompi_mpi_finalize(void)
        del_procs behavior around May of 2014 (see
        https://svn.open-mpi.org/trac/ompi/ticket/4669#comment:4 for
        more details). */
-    if (!ompi_async_mpi_finalize && !ompi_singleton) {
+    if (!ompi_async_mpi_finalize && !opal_process_info.is_singleton) {
         active = true;
         OPAL_POST_OBJECT(&active);
         /* Note that use of the non-blocking PMIx fence will
@@ -302,205 +291,7 @@ int ompi_mpi_finalize(void)
         OMPI_LAZY_WAIT_FOR_COMPLETION(active);
     }
 
-    /* Shut down any bindings-specific issues: C++, F77, F90 */
-
-    /* Remove all memory associated by MPI_REGISTER_DATAREP (per
-       MPI-2:9.5.3, there is no way for an MPI application to
-       *un*register datareps, but we don't want the OMPI layer causing
-       memory leaks). */
-    while (NULL != (item = opal_list_remove_first(&ompi_registered_datareps))) {
-        OBJ_RELEASE(item);
-    }
-    OBJ_DESTRUCT(&ompi_registered_datareps);
-
-    /* Remove all F90 types from the hash tables */
-    OPAL_HASH_TABLE_FOREACH(key, uint32, datatype, &ompi_mpi_f90_integer_hashtable)
-        OBJ_RELEASE(datatype);
-    OBJ_DESTRUCT(&ompi_mpi_f90_integer_hashtable);
-    OPAL_HASH_TABLE_FOREACH(key, uint32, datatype, &ompi_mpi_f90_real_hashtable)
-        OBJ_RELEASE(datatype);
-    OBJ_DESTRUCT(&ompi_mpi_f90_real_hashtable);
-    OPAL_HASH_TABLE_FOREACH(key, uint32, datatype, &ompi_mpi_f90_complex_hashtable)
-        OBJ_RELEASE(datatype);
-    OBJ_DESTRUCT(&ompi_mpi_f90_complex_hashtable);
-
-    /* Free communication objects */
-
-    /* free file resources */
-    if (OMPI_SUCCESS != (ret = ompi_file_finalize())) {
-        goto done;
-    }
-
-    /* free window resources */
-    if (OMPI_SUCCESS != (ret = ompi_win_finalize())) {
-        goto done;
-    }
-    if (OMPI_SUCCESS != (ret = ompi_osc_base_finalize())) {
-        goto done;
-    }
-
-    /* free communicator resources. this MUST come before finalizing the PML
-     * as this will call into the pml */
-    if (OMPI_SUCCESS != (ret = ompi_comm_finalize())) {
-        goto done;
-    }
-
-    /* call del_procs on all allocated procs even though some may not be known
-     * to the pml layer. the pml layer is expected to be resilient and ignore
-     * any unknown procs. */
-    nprocs = 0;
-    procs = ompi_proc_get_allocated (&nprocs);
-    MCA_PML_CALL(del_procs(procs, nprocs));
-    free(procs);
-
-    /* free pml resource */
-    if(OMPI_SUCCESS != (ret = mca_pml_base_finalize())) {
-        goto done;
-    }
-
-    /* free requests */
-    if (OMPI_SUCCESS != (ret = ompi_request_finalize())) {
-        goto done;
-    }
-
-    if (OMPI_SUCCESS != (ret = ompi_message_finalize())) {
-        goto done;
-    }
-
-    /* If requested, print out a list of memory allocated by ALLOC_MEM
-       but not freed by FREE_MEM */
-    if (0 != ompi_debug_show_mpi_alloc_mem_leaks) {
-        mca_mpool_base_tree_print(ompi_debug_show_mpi_alloc_mem_leaks);
-    }
-
-    /* Now that all MPI objects dealing with communications are gone,
-       shut down MCA types having to do with communications */
-    if (OMPI_SUCCESS != (ret = mca_base_framework_close(&ompi_pml_base_framework) ) ) {
-        OMPI_ERROR_LOG(ret);
-        goto done;
-    }
-
-    /* shut down buffered send code */
-    mca_pml_base_bsend_fini();
-
-    /* Free secondary resources */
-
-    /* free attr resources */
-    if (OMPI_SUCCESS != (ret = ompi_attr_finalize())) {
-        goto done;
-    }
-
-    /* free group resources */
-    if (OMPI_SUCCESS != (ret = ompi_group_finalize())) {
-        goto done;
-    }
-
-    /* finalize the DPM subsystem */
-    if ( OMPI_SUCCESS != (ret = ompi_dpm_finalize())) {
-        goto done;
-    }
-
-    /* free internal error resources */
-    if (OMPI_SUCCESS != (ret = ompi_errcode_intern_finalize())) {
-        goto done;
-    }
-
-    /* free error code resources */
-    if (OMPI_SUCCESS != (ret = ompi_mpi_errcode_finalize())) {
-        goto done;
-    }
-
-    /* free errhandler resources */
-    if (OMPI_SUCCESS != (ret = ompi_errhandler_finalize())) {
-        goto done;
-    }
-
-    /* Free all other resources */
-
-    /* free op resources */
-    if (OMPI_SUCCESS != (ret = ompi_op_finalize())) {
-        goto done;
-    }
-
-    /* free ddt resources */
-    if (OMPI_SUCCESS != (ret = ompi_datatype_finalize())) {
-        goto done;
-    }
-
-    /* free info resources */
-    if (OMPI_SUCCESS != (ret = ompi_mpiinfo_finalize())) {
-        goto done;
-    }
-
-    /* Close down MCA modules */
-
-    /* io is opened lazily, so it's only necessary to close it if it
-       was actually opened */
-    if (0 < ompi_io_base_framework.framework_refcnt) {
-        /* May have been "opened" multiple times. We want it closed now */
-        ompi_io_base_framework.framework_refcnt = 1;
-
-        if (OMPI_SUCCESS != mca_base_framework_close(&ompi_io_base_framework)) {
-            goto done;
-        }
-    }
-    (void) mca_base_framework_close(&ompi_topo_base_framework);
-    if (OMPI_SUCCESS != (ret = mca_base_framework_close(&ompi_osc_base_framework))) {
-        goto done;
-    }
-    if (OMPI_SUCCESS != (ret = mca_base_framework_close(&ompi_coll_base_framework))) {
-        goto done;
-    }
-    if (OMPI_SUCCESS != (ret = mca_base_framework_close(&ompi_bml_base_framework))) {
-        goto done;
-    }
-    if (OMPI_SUCCESS != (ret = mca_base_framework_close(&opal_mpool_base_framework))) {
-        goto done;
-    }
-    if (OMPI_SUCCESS != (ret = mca_base_framework_close(&opal_rcache_base_framework))) {
-        goto done;
-    }
-    if (OMPI_SUCCESS != (ret = mca_base_framework_close(&opal_allocator_base_framework))) {
-        goto done;
-    }
-
-    /* free proc resources */
-    if ( OMPI_SUCCESS != (ret = ompi_proc_finalize())) {
-        goto done;
-    }
-
-    if (NULL != ompi_mpi_main_thread) {
-        OBJ_RELEASE(ompi_mpi_main_thread);
-        ompi_mpi_main_thread = NULL;
-    }
-
-    /* Clean up memory/resources from the MPI dynamic process
-       functionality checker */
-    ompi_mpi_dynamics_finalize();
-
-    /* Leave the RTE */
-
-    if (OMPI_SUCCESS != (ret = ompi_rte_finalize())) {
-        goto done;
-    }
-    ompi_rte_initialized = false;
-
-    /* Now close the hook framework */
-    if (OMPI_SUCCESS != (ret = mca_base_framework_close(&ompi_hook_base_framework) ) ) {
-        OMPI_ERROR_LOG(ret);
-        goto done;
-    }
-
-    if (OPAL_SUCCESS != (ret = opal_finalize_util())) {
-        goto done;
-    }
-
-    if (0 == opal_initialized) {
-        /* if there is no MPI_T_init_thread that has been MPI_T_finalize'd,
-         * then be gentle to the app and release all the memory now (instead
-         * of the opal library destructor */
-        opal_class_finalize();
-    }
+    ompi_mpi_instance_finalize (&ompi_mpi_instance_default);
 
     /* cleanup environment */
     opal_unsetenv("OMPI_COMMAND", &environ);
@@ -508,7 +299,7 @@ int ompi_mpi_finalize(void)
 
     /* All done */
 
-  done:
+   done:
     opal_atomic_wmb();
     opal_atomic_swap_32(&ompi_mpi_state, OMPI_MPI_STATE_FINALIZE_COMPLETED);
 
diff --git a/ompi/runtime/ompi_mpi_init.c b/ompi/runtime/ompi_mpi_init.c
index 45673c10e6f..19c0999d163 100644
--- a/ompi/runtime/ompi_mpi_init.c
+++ b/ompi/runtime/ompi_mpi_init.c
@@ -10,22 +10,25 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2006-2018 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2006-2022 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2006-2015 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2006-2009 University of Houston. All rights reserved.
  * Copyright (c) 2008-2009 Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2011-2020 Sandia National Laboratories. All rights reserved.
  * Copyright (c) 2012-2013 Inria.  All rights reserved.
  * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
- * Copyright (c) 2014-2016 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2014-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2016-2018 Mellanox Technologies Ltd. All rights reserved.
  *
  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
  * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
  * Copyright (c) 2020      Amazon.com, Inc. or its affiliates.
  *                         All Rights reserved.
+ * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2021-2022 Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -63,7 +66,6 @@
 #include "opal/mca/mpool/base/base.h"
 #include "opal/mca/btl/base/base.h"
 #include "opal/mca/pmix/base/base.h"
-#include "opal/util/timings.h"
 #include "opal/util/opal_environ.h"
 
 #include "ompi/constants.h"
@@ -87,6 +89,7 @@
 #include "ompi/mca/pml/base/base.h"
 #include "ompi/mca/bml/base/base.h"
 #include "ompi/mca/osc/base/base.h"
+#include "ompi/mca/part/base/base.h"
 #include "ompi/mca/coll/base/base.h"
 #include "ompi/mca/io/io.h"
 #include "ompi/mca/io/base/base.h"
@@ -103,7 +106,7 @@
 #ifdef HAVE___MALLOC_INITIALIZE_HOOK
 #include "opal/mca/memory/base/base.h"
 /* So this sucks, but with OPAL in its own library that is brought in
-   implicity from libmpi, there are times when the malloc initialize
+   implicitly from libmpi, there are times when the malloc initialize
    hook in the memory component doesn't work.  So we have to do it
    from here, since any MPI code is going to call MPI_Init... */
 OPAL_DECLSPEC void (*__malloc_initialize_hook) (void) =
@@ -115,11 +118,6 @@ OPAL_DECLSPEC void (*__malloc_initialize_hook) (void) =
  */
 #include <float.h>
 
-#if OPAL_CC_USE_PRAGMA_IDENT
-#pragma ident OMPI_IDENT_STRING
-#elif OPAL_CC_USE_IDENT
-#ident OMPI_IDENT_STRING
-#endif
 const char ompi_version_string[] = OMPI_IDENT_STRING;
 
 /*
@@ -137,7 +135,7 @@ opal_thread_t *ompi_mpi_main_thread = NULL;
 
 /*
  * These variables are for the MPI F08 bindings (F08 must bind Fortran
- * varaiables to symbols; it cannot bind Fortran variables to the
+ * variables to symbols; it cannot bind Fortran variables to the
  * address of a C variable).
  */
 
@@ -215,21 +213,31 @@ struct ompi_status_public_t *ompi_mpi_statuses_ignore_addr =
 #  if OMPI_FORTRAN_CAPS
 MPI_Fint *MPI_F_STATUS_IGNORE = (MPI_Fint*) &MPI_FORTRAN_STATUS_IGNORE;
 MPI_Fint *MPI_F_STATUSES_IGNORE = (MPI_Fint*) &MPI_FORTRAN_STATUSES_IGNORE;
+MPI_Fint *MPI_F08_STATUS_IGNORE = (MPI_Fint*) &MPI_FORTRAN_STATUS_IGNORE;
+MPI_Fint *MPI_F08_STATUSES_IGNORE = (MPI_Fint*) &MPI_FORTRAN_STATUSES_IGNORE;
 #  elif OMPI_FORTRAN_PLAIN
 MPI_Fint *MPI_F_STATUS_IGNORE = (MPI_Fint*) &mpi_fortran_status_ignore;
 MPI_Fint *MPI_F_STATUSES_IGNORE = (MPI_Fint*) &mpi_fortran_statuses_ignore;
+MPI_Fint *MPI_F08_STATUS_IGNORE = (MPI_Fint*) &mpi_fortran_status_ignore;
+MPI_Fint *MPI_F08_STATUSES_IGNORE = (MPI_Fint*) &mpi_fortran_statuses_ignore;
 #  elif OMPI_FORTRAN_SINGLE_UNDERSCORE
 MPI_Fint *MPI_F_STATUS_IGNORE = (MPI_Fint*) &mpi_fortran_status_ignore_;
 MPI_Fint *MPI_F_STATUSES_IGNORE = (MPI_Fint*) &mpi_fortran_statuses_ignore_;
+MPI_Fint *MPI_F08_STATUS_IGNORE = (MPI_Fint*) &mpi_fortran_status_ignore_;
+MPI_Fint *MPI_F08_STATUSES_IGNORE = (MPI_Fint*) &mpi_fortran_statuses_ignore_;
 #  elif OMPI_FORTRAN_DOUBLE_UNDERSCORE
 MPI_Fint *MPI_F_STATUS_IGNORE = (MPI_Fint*) &mpi_fortran_status_ignore__;
 MPI_Fint *MPI_F_STATUSES_IGNORE = (MPI_Fint*) &mpi_fortran_statuses_ignore__;
+MPI_Fint *MPI_F08_STATUS_IGNORE = (MPI_Fint*) &mpi_fortran_status_ignore__;
+MPI_Fint *MPI_F08_STATUSES_IGNORE = (MPI_Fint*) &mpi_fortran_statuses_ignore__;
 #  else
 #    error Unrecognized Fortran name mangling scheme
 #  endif
 #else
 MPI_Fint *MPI_F_STATUS_IGNORE = NULL;
 MPI_Fint *MPI_F_STATUSES_IGNORE = NULL;
+MPI_Fint *MPI_F08_STATUS_IGNORE = NULL;
+MPI_Fint *MPI_F08_STATUSES_IGNORE = NULL;
 #endif  /* OMPI_BUILD_FORTRAN_BINDINGS */
 
 
@@ -249,7 +257,7 @@ MPI_Fint *MPI_F_STATUSES_IGNORE = NULL;
    in a linker warning).  FWIW, if you initialize these variables in
    functions (i.e., not at the instantiation in the global scope), the
    linker somehow "figures it all out" (w.r.t. different alignments
-   between fortan common blocks and the corresponding C variables) and
+   between fortran common blocks and the corresponding C variables) and
    no linker warnings occur.
 
    Note that the rationale for the types of each of these variables is
@@ -260,57 +268,6 @@ MPI_Fint *MPI_F_STATUSES_IGNORE = NULL;
 
 #include "mpif-c-constants.h"
 
-/*
- * Hash tables for MPI_Type_create_f90* functions
- */
-opal_hash_table_t ompi_mpi_f90_integer_hashtable = {{0}};
-opal_hash_table_t ompi_mpi_f90_real_hashtable = {{0}};
-opal_hash_table_t ompi_mpi_f90_complex_hashtable = {{0}};
-
-/*
- * Per MPI-2:9.5.3, MPI_REGISTER_DATAREP is a memory leak.  There is
- * no way to *de*register datareps once they've been registered.  So
- * we have to track all registrations here so that they can be
- * de-registered during MPI_FINALIZE so that memory-tracking debuggers
- * don't show Open MPI as leaking memory.
- */
-opal_list_t ompi_registered_datareps = {{0}};
-
-bool ompi_enable_timing = false;
-extern bool ompi_mpi_yield_when_idle;
-extern int ompi_mpi_event_tick_rate;
-
-/**
- * Static functions used to configure the interactions between the OPAL and
- * the runtime.
- */
-static char*
-_process_name_print_for_opal(const opal_process_name_t procname)
-{
-    ompi_process_name_t* rte_name = (ompi_process_name_t*)&procname;
-    return OMPI_NAME_PRINT(rte_name);
-}
-
-static int
-_process_name_compare(const opal_process_name_t p1, const opal_process_name_t p2)
-{
-    ompi_process_name_t* o1 = (ompi_process_name_t*)&p1;
-    ompi_process_name_t* o2 = (ompi_process_name_t*)&p2;
-    return ompi_rte_compare_name_fields(OMPI_RTE_CMP_ALL, o1, o2);
-}
-
-static int _convert_string_to_process_name(opal_process_name_t *name,
-                                           const char* name_string)
-{
-    return ompi_rte_convert_string_to_process_name(name, name_string);
-}
-
-static int _convert_process_name_to_string(char** name_string,
-                                          const opal_process_name_t *name)
-{
-    return ompi_rte_convert_process_name_to_string(name_string, name);
-}
-
 void ompi_mpi_thread_level(int requested, int *provided)
 {
     /**
@@ -334,41 +291,6 @@ void ompi_mpi_thread_level(int requested, int *provided)
                                 MPI_THREAD_MULTIPLE);
 }
 
-static int ompi_register_mca_variables(void)
-{
-    int ret;
-
-    /* Register MPI variables */
-    if (OMPI_SUCCESS != (ret = ompi_mpi_register_params())) {
-        return ret;
-    }
-
-    /* check to see if we want timing information */
-    /* TODO: enable OMPI init and OMPI finalize timings if
-     * this variable was set to 1!
-     */
-    ompi_enable_timing = false;
-    (void) mca_base_var_register("ompi", "ompi", NULL, "timing",
-                                 "Request that critical timing loops be measured",
-                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
-                                 OPAL_INFO_LVL_9,
-                                 MCA_BASE_VAR_SCOPE_READONLY,
-                                 &ompi_enable_timing);
-
-#if OPAL_ENABLE_FT_MPI
-    /* Before loading any other part of the MPI library, we need to load
-     * the ft-mpi tune file to override default component selection when
-     * FT is desired ON; this does override openmpi-params.conf, but not
-     * command line or env.
-     */
-    if( ompi_ftmpi_enabled ) {
-        mca_base_var_load_extra_files("ft-mpi", false);
-    }
-#endif /* OPAL_ENABLE_FT_MPI */
-
-    return OMPI_SUCCESS;
-}
-
 static void fence_release(pmix_status_t status, void *cbdata)
 {
     volatile bool *active = (volatile bool*)cbdata;
@@ -377,31 +299,19 @@ static void fence_release(pmix_status_t status, void *cbdata)
     OPAL_POST_OBJECT(active);
 }
 
-static void evhandler_reg_callbk(pmix_status_t status,
-                                 size_t evhandler_ref,
-                                 void *cbdata)
-{
-    opal_pmix_lock_t *lock = (opal_pmix_lock_t*)cbdata;
-
-    lock->status = status;
-    OPAL_PMIX_WAKEUP_THREAD(lock);
-}
-
-
 int ompi_mpi_init(int argc, char **argv, int requested, int *provided,
                   bool reinit_ok)
 {
     int ret;
-    ompi_proc_t** procs;
-    size_t nprocs;
     char *error = NULL;
+#if OPAL_USING_INTERNAL_PMIX
+    char *evar;
+#endif
     volatile bool active;
     bool background_fence = false;
     pmix_info_t info[2];
-    pmix_status_t codes[1] = { PMIX_ERR_PROC_ABORTED };
     pmix_status_t rc;
     OMPI_TIMING_INIT(64);
-    opal_pmix_lock_t mylock;
 
     ompi_hook_base_mpi_init_top(argc, argv, requested, provided);
 
@@ -436,227 +346,63 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided,
         }
     }
 
-    /* Figure out the final MPI thread levels.  If we were not
-       compiled for support for MPI threads, then don't allow
-       MPI_THREAD_MULTIPLE.  Set this stuff up here early in the
-       process so that other components can make decisions based on
-       this value. */
-
-    ompi_mpi_thread_level(requested, provided);
-
-    /* Setup enough to check get/set MCA params */
-    if (OPAL_SUCCESS != (ret = opal_init_util(&argc, &argv))) {
-        error = "ompi_mpi_init: opal_init_util failed";
-        goto error;
-    }
-    OMPI_TIMING_IMPORT_OPAL("opal_init_util");
-
-    /* If thread support was enabled, then setup OPAL to allow for them. This must be done
-     * early to prevent a race condition that can occur with orte_init(). */
-    if (*provided != MPI_THREAD_SINGLE) {
-        opal_set_using_threads(true);
-    }
-
-    /* Convince OPAL to use our naming scheme */
-    opal_process_name_print = _process_name_print_for_opal;
-    opal_compare_proc = _process_name_compare;
-    opal_convert_string_to_process_name = _convert_string_to_process_name;
-    opal_convert_process_name_to_string = _convert_process_name_to_string;
-    opal_proc_for_name = ompi_proc_for_name;
-
-    /* Register MCA variables */
-    if (OPAL_SUCCESS != (ret = ompi_register_mca_variables())) {
-        error = "ompi_mpi_init: ompi_register_mca_variables failed";
-        goto error;
+    /* deal with OPAL_PREFIX to ensure that an internal PMIx installation
+     * is also relocated if necessary */
+#if OPAL_USING_INTERNAL_PMIX
+    if (NULL != (evar = getenv("OPAL_PREFIX"))) {
+        opal_setenv("PMIX_PREFIX", evar, true, &environ);
     }
+#endif
 
-    if (OPAL_SUCCESS != (ret = opal_arch_set_fortran_logical_size(sizeof(ompi_fortran_logical_t)))) {
-        error = "ompi_mpi_init: opal_arch_set_fortran_logical_size failed";
-        goto error;
-    }
-
-    /* _After_ opal_init_util() but _before_ orte_init(), we need to
-       set an MCA param that tells libevent that it's ok to use any
-       mechanism in libevent that is available on this platform (e.g.,
-       epoll and friends).  Per opal/event/event.s, we default to
-       select/poll -- but we know that MPI processes won't be using
-       pty's with the event engine, so it's ok to relax this
-       constraint and let any fd-monitoring mechanism be used. */
-
-    ret = mca_base_var_find("opal", "event", "*", "event_include");
-    if (ret >= 0) {
-        char *allvalue = "all";
-        /* We have to explicitly "set" the MCA param value here
-           because libevent initialization will re-register the MCA
-           param and therefore override the default. Setting the value
-           here puts the desired value ("all") in different storage
-           that is not overwritten if/when the MCA param is
-           re-registered. This is unless the user has specified a different
-           value for this MCA parameter. Make sure we check to see if the
-           default is specified before forcing "all" in case that is not what
-           the user desires. Note that we do *NOT* set this value as an
-           environment variable, just so that it won't be inherited by
-           any spawned processes and potentially cause unintented
-           side-effects with launching RTE tools... */
-        mca_base_var_set_value(ret, allvalue, 4, MCA_BASE_VAR_SOURCE_DEFAULT, NULL);
-    }
+    ompi_mpi_thread_level(requested, provided);
 
-    /* open the ompi hook framework */
-    if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_hook_base_framework, 0))) {
-        error = "ompi_hook_base_open() failed";
+    ret = ompi_mpi_instance_init (*provided, &ompi_mpi_info_null.info.super, MPI_ERRORS_ARE_FATAL, &ompi_mpi_instance_default, argc, argv);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
+        error = "ompi_mpi_init: ompi_mpi_instance_init failed";
         goto error;
     }
 
     ompi_hook_base_mpi_init_top_post_opal(argc, argv, requested, provided);
 
-
-    OMPI_TIMING_NEXT("initialization");
-
-    /* Setup RTE */
-    if (OMPI_SUCCESS != (ret = ompi_rte_init(&argc, &argv))) {
-        error = "ompi_mpi_init: ompi_rte_init failed";
+    /* initialize communicator subsystem */
+    if (OMPI_SUCCESS != (ret = ompi_comm_init_mpi3 ())) {
+        error = "ompi_mpi_init: ompi_comm_init_mpi3 failed";
         goto error;
     }
-    OMPI_TIMING_NEXT("rte_init");
-    OMPI_TIMING_IMPORT_OPAL("orte_ess_base_app_setup");
-    OMPI_TIMING_IMPORT_OPAL("rte_init");
-
-    ompi_rte_initialized = true;
 
-    /* Register the default errhandler callback  */
-    /* we want to go first */
-    PMIX_INFO_LOAD(&info[0], PMIX_EVENT_HDLR_PREPEND, NULL, PMIX_BOOL);
-    /* give it a name so we can distinguish it */
-    PMIX_INFO_LOAD(&info[1], PMIX_EVENT_HDLR_NAME, "MPI-Default", PMIX_STRING);
-    OPAL_PMIX_CONSTRUCT_LOCK(&mylock);
-    PMIx_Register_event_handler(codes, 1, info, 2, ompi_errhandler_callback, evhandler_reg_callbk, (void*)&mylock);
-    OPAL_PMIX_WAIT_THREAD(&mylock);
-    rc = mylock.status;
-    OPAL_PMIX_DESTRUCT_LOCK(&mylock);
-    PMIX_INFO_DESTRUCT(&info[0]);
-    PMIX_INFO_DESTRUCT(&info[1]);
-    if (PMIX_SUCCESS != rc) {
-        error = "Error handler registration";
-        ret = opal_pmix_convert_status(rc);
+    /* Bozo argument check */
+    if (NULL == argv && argc > 1) {
+        ret = OMPI_ERR_BAD_PARAM;
+        error = "argc > 1, but argv == NULL";
         goto error;
     }
 
-    /* declare our presence for interlib coordination, and
-     * register for callbacks when other libs declare */
-    if (OMPI_SUCCESS != (ret = ompi_interlib_declare(*provided, OMPI_IDENT_STRING))) {
-        error = "ompi_interlib_declare";
-        goto error;
-    }
-
-    /* initialize datatypes. This step should be done early as it will
-     * create the local convertor and local arch used in the proc
-     * init.
+    /* if we were not externally started, then we need to setup
+     * some envars so the MPI_INFO_ENV can get the cmd name
+     * and argv (but only if the user supplied a non-NULL argv!), and
+     * the requested thread level
      */
-    if (OMPI_SUCCESS != (ret = ompi_datatype_init())) {
-        error = "ompi_datatype_init() failed";
-        goto error;
-    }
-
-    /* Initialize OMPI procs */
-    if (OMPI_SUCCESS != (ret = ompi_proc_init())) {
-        error = "mca_proc_init() failed";
-        goto error;
-    }
-
-    /* Initialize the op framework. This has to be done *after*
-       ddt_init, but befor mca_coll_base_open, since some collective
-       modules (e.g., the hierarchical coll component) may need ops in
-       their query function. */
-    if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_op_base_framework, 0))) {
-        error = "ompi_op_base_open() failed";
-        goto error;
-    }
-    if (OMPI_SUCCESS !=
-        (ret = ompi_op_base_find_available(OPAL_ENABLE_PROGRESS_THREADS,
-                                           ompi_mpi_thread_multiple))) {
-        error = "ompi_op_base_find_available() failed";
-        goto error;
+    if (NULL == getenv("OMPI_COMMAND") && NULL != argv && NULL != argv[0]) {
+        opal_setenv("OMPI_COMMAND", argv[0], true, &environ);
     }
-    if (OMPI_SUCCESS != (ret = ompi_op_init())) {
-        error = "ompi_op_init() failed";
-        goto error;
+    if (NULL == getenv("OMPI_ARGV") && 1 < argc) {
+        char *tmp;
+        tmp = opal_argv_join(&argv[1], ' ');
+        opal_setenv("OMPI_ARGV", tmp, true, &environ);
+        free(tmp);
     }
 
-    /* Open up MPI-related MCA components */
-
-    if (OMPI_SUCCESS != (ret = mca_base_framework_open(&opal_allocator_base_framework, 0))) {
-        error = "mca_allocator_base_open() failed";
-        goto error;
-    }
-    if (OMPI_SUCCESS != (ret = mca_base_framework_open(&opal_rcache_base_framework, 0))) {
-        error = "mca_rcache_base_open() failed";
-        goto error;
-    }
-    if (OMPI_SUCCESS != (ret = mca_base_framework_open(&opal_mpool_base_framework, 0))) {
-        error = "mca_mpool_base_open() failed";
-        goto error;
-    }
-    if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_bml_base_framework, 0))) {
-        error = "mca_bml_base_open() failed";
-        goto error;
-    }
-    if (OMPI_SUCCESS != (ret = mca_bml_base_init (1, ompi_mpi_thread_multiple))) {
-        error = "mca_bml_base_init() failed";
-        goto error;
-    }
-    if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_pml_base_framework, 0))) {
-        error = "mca_pml_base_open() failed";
-        goto error;
-    }
-    if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_coll_base_framework, 0))) {
-        error = "mca_coll_base_open() failed";
-        goto error;
-    }
-
-    if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_osc_base_framework, 0))) {
-        error = "ompi_osc_base_open() failed";
-        goto error;
-    }
-
-    /* In order to reduce the common case for MPI apps (where they
-       don't use MPI-2 IO or MPI-1 topology functions), the io and
-       topo frameworks are initialized lazily, at the first use of
-       relevant functions (e.g., MPI_FILE_*, MPI_CART_*, MPI_GRAPH_*),
-       so they are not opened here. */
-
-    /* Select which MPI components to use */
-
-    if (OMPI_SUCCESS !=
-        (ret = mca_pml_base_select(OPAL_ENABLE_PROGRESS_THREADS,
-                                   ompi_mpi_thread_multiple))) {
-        error = "mca_pml_base_select() failed";
-        goto error;
-    }
-
-    OMPI_TIMING_IMPORT_OPAL("orte_init");
-    OMPI_TIMING_NEXT("rte_init-commit");
-
-    /* exchange connection info - this function may also act as a barrier
-     * if data exchange is required. The modex occurs solely across procs
-     * in our job. If a barrier is required, the "modex" function will
-     * perform it internally */
-    rc = PMIx_Commit();
-    if (PMIX_SUCCESS != rc) {
-        ret = opal_pmix_convert_status(rc);
-        error = "PMIx_Commit()";
-        goto error;
-    }
-    OMPI_TIMING_NEXT("commit");
 #if (OPAL_ENABLE_TIMING)
     if (OMPI_TIMING_ENABLED && !opal_pmix_base_async_modex &&
-            opal_pmix_collect_all_data && !ompi_singleton) {
+            opal_pmix_collect_all_data && !opal_process_info.is_singleton) {
         if (PMIX_SUCCESS != (rc = PMIx_Fence(NULL, 0, NULL, 0))) {
-            ret - opal_pmix_convert_status(rc);
+            ret = opal_pmix_convert_status(rc);
             error = "timing: pmix-barrier-1 failed";
             goto error;
         }
         OMPI_TIMING_NEXT("pmix-barrier-1");
         if (PMIX_SUCCESS != (rc = PMIx_Fence(NULL, 0, NULL, 0))) {
+            ret = opal_pmix_convert_status(rc);
             error = "timing: pmix-barrier-2 failed";
             goto error;
         }
@@ -664,7 +410,7 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided,
     }
 #endif
 
-    if (!ompi_singleton) {
+    if (!opal_process_info.is_singleton) {
         if (opal_pmix_base_async_modex) {
             /* if we are doing an async modex, but we are collecting all
              * data, then execute the non-blocking modex in the background.
@@ -709,142 +455,6 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided,
 
     OMPI_TIMING_NEXT("modex");
 
-    /* select buffered send allocator component to be used */
-    if( OMPI_SUCCESS !=
-        (ret = mca_pml_base_bsend_init(ompi_mpi_thread_multiple))) {
-        error = "mca_pml_base_bsend_init() failed";
-        goto error;
-    }
-
-    if (OMPI_SUCCESS !=
-        (ret = mca_coll_base_find_available(OPAL_ENABLE_PROGRESS_THREADS,
-                                            ompi_mpi_thread_multiple))) {
-        error = "mca_coll_base_find_available() failed";
-        goto error;
-    }
-
-    if (OMPI_SUCCESS !=
-        (ret = ompi_osc_base_find_available(OPAL_ENABLE_PROGRESS_THREADS,
-                                            ompi_mpi_thread_multiple))) {
-        error = "ompi_osc_base_find_available() failed";
-        goto error;
-    }
-
-    /* io and topo components are not selected here -- see comment
-       above about the io and topo frameworks being loaded lazily */
-
-    /* Initialize each MPI handle subsystem */
-    /* initialize requests */
-    if (OMPI_SUCCESS != (ret = ompi_request_init())) {
-        error = "ompi_request_init() failed";
-        goto error;
-    }
-
-    if (OMPI_SUCCESS != (ret = ompi_message_init())) {
-        error = "ompi_message_init() failed";
-        goto error;
-    }
-
-    /* initialize error handlers */
-    if (OMPI_SUCCESS != (ret = ompi_errhandler_init())) {
-        error = "ompi_errhandler_init() failed";
-        goto error;
-    }
-
-    /* initialize error codes */
-    if (OMPI_SUCCESS != (ret = ompi_mpi_errcode_init())) {
-        error = "ompi_mpi_errcode_init() failed";
-        goto error;
-    }
-
-    /* initialize internal error codes */
-    if (OMPI_SUCCESS != (ret = ompi_errcode_intern_init())) {
-        error = "ompi_errcode_intern_init() failed";
-        goto error;
-    }
-
-    /* initialize info */
-    if (OMPI_SUCCESS != (ret = ompi_mpiinfo_init())) {
-        error = "ompi_info_init() failed";
-        goto error;
-    }
-
-    /* initialize groups  */
-    if (OMPI_SUCCESS != (ret = ompi_group_init())) {
-        error = "ompi_group_init() failed";
-        goto error;
-    }
-
-    /* initialize communicators */
-    if (OMPI_SUCCESS != (ret = ompi_comm_init())) {
-        error = "ompi_comm_init() failed";
-        goto error;
-    }
-
-    /* initialize file handles */
-    if (OMPI_SUCCESS != (ret = ompi_file_init())) {
-        error = "ompi_file_init() failed";
-        goto error;
-    }
-
-    /* initialize windows */
-    if (OMPI_SUCCESS != (ret = ompi_win_init())) {
-        error = "ompi_win_init() failed";
-        goto error;
-    }
-
-    /* initialize attribute meta-data structure for comm/win/dtype */
-    if (OMPI_SUCCESS != (ret = ompi_attr_init())) {
-        error = "ompi_attr_init() failed";
-        goto error;
-    }
-
-    /* identify the architectures of remote procs and setup
-     * their datatype convertors, if required
-     */
-    if (OMPI_SUCCESS != (ret = ompi_proc_complete_init())) {
-        error = "ompi_proc_complete_init failed";
-        goto error;
-    }
-
-    /* start PML/BTL's */
-    ret = MCA_PML_CALL(enable(true));
-    if( OMPI_SUCCESS != ret ) {
-        error = "PML control failed";
-        goto error;
-    }
-
-    /* some btls/mtls require we call add_procs with all procs in the job.
-     * since the btls/mtls have no visibility here it is up to the pml to
-     * convey this requirement */
-    if (mca_pml_base_requires_world ()) {
-        if (NULL == (procs = ompi_proc_world (&nprocs))) {
-            error = "ompi_proc_world () failed";
-            goto error;
-        }
-    } else {
-        /* add all allocated ompi_proc_t's to PML (below the add_procs limit this
-         * behaves identically to ompi_proc_world ()) */
-        if (NULL == (procs = ompi_proc_get_allocated (&nprocs))) {
-            error = "ompi_proc_get_allocated () failed";
-            goto error;
-        }
-    }
-    ret = MCA_PML_CALL(add_procs(procs, nprocs));
-    free(procs);
-    /* If we got "unreachable", then print a specific error message.
-       Otherwise, if we got some other failure, fall through to print
-       a generic message. */
-    if (OMPI_ERR_UNREACH == ret) {
-        opal_show_help("help-mpi-runtime.txt",
-                       "mpi_init:startup:pml-add-procs-fail", true);
-        error = NULL;
-        goto error;
-    } else if (OMPI_SUCCESS != ret) {
-        error = "PML add procs failed";
-        goto error;
-    }
-
     MCA_PML_CALL(add_comm(&ompi_mpi_comm_world.comm));
     MCA_PML_CALL(add_comm(&ompi_mpi_comm_self.comm));
 
@@ -852,7 +462,6 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided,
     /* initialize the fault tolerant infrastructure (revoke, detector,
      * propagator) */
     if( ompi_ftmpi_enabled ) {
-        int rc;
         const char *evmethod;
         rc = ompi_comm_rbcast_init();
         if( OMPI_SUCCESS != rc ) return rc;
@@ -875,7 +484,7 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided,
      */
     if (ompi_mpi_show_mca_params) {
         ompi_show_all_mca_params(ompi_mpi_comm_world.comm.c_my_rank,
-                                 nprocs,
+                                 ompi_process_info.num_procs,
                                  ompi_process_info.nodename);
     }
 
@@ -885,7 +494,7 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided,
     /* Next timing measurement */
     OMPI_TIMING_NEXT("modex-barrier");
 
-    if (!ompi_singleton) {
+    if (!opal_process_info.is_singleton) {
         /* if we executed the above fence in the background, then
          * we have to wait here for it to complete. However, there
          * is no reason to do two barriers! */
@@ -920,7 +529,8 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided,
        CPU utilization for the remainder of MPI_INIT when we are
        blocking on RTE-level events, but may greatly reduce non-TCP
        latency. */
-    opal_progress_set_event_flag(OPAL_EVLOOP_NONBLOCK);
+    int old_event_flags = opal_progress_set_event_flag(0);
+    opal_progress_set_event_flag(old_event_flags | OPAL_EVLOOP_NONBLOCK);
 #endif
 
     /* wire up the mpi interface, if requested.  Do this after the
@@ -932,23 +542,6 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided,
         goto error;
     }
 
-    /* Setup the dynamic process management (DPM) subsystem */
-    if (OMPI_SUCCESS != (ret = ompi_dpm_init())) {
-        error = "ompi_dpm_init() failed";
-        goto error;
-    }
-
-    /* Determine the overall threadlevel support of all processes
-       in MPI_COMM_WORLD. This has to be done before calling
-       coll_base_comm_select, since some of the collective components
-       e.g. hierarch, might create subcommunicators. The threadlevel
-       requested by all processes is required in order to know
-       which cid allocation algorithm can be used. */
-    if (OMPI_SUCCESS != ( ret = ompi_comm_cid_init ())) {
-        error = "ompi_mpi_init: ompi_comm_cid_init failed";
-        goto error;
-    }
-
     /* Init coll for the comms. This has to be after dpm_base_select,
        (since dpm.mark_dyncomm is not set in the communicator creation
        function else), but before dpm.dyncom_init, since this function
@@ -965,48 +558,21 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided,
         goto error;
     }
 
-    /* Check whether we have been spawned or not.  We introduce that
-       at the very end, since we need collectives, datatypes, ptls
-       etc. up and running here.... */
-    if (OMPI_SUCCESS != (ret = ompi_dpm_dyn_init())) {
-        error = "ompi_dpm_dyn_init() failed";
-        goto error;
-    }
-
-    /* Undo OPAL calling opal_progress_event_users_increment() during
-       opal_init, to get better latency when not using TCP.  Do
-       this *after* dyn_init, as dyn init uses lots of RTE
-       communication and we don't want to hinder the performance of
-       that code. */
-    opal_progress_event_users_decrement();
-
-    /* see if yield_when_idle was specified - if so, use it */
-    opal_progress_set_yield_when_idle(ompi_mpi_yield_when_idle);
-
-    /* negative value means use default - just don't do anything */
-    if (ompi_mpi_event_tick_rate >= 0) {
-        opal_progress_set_event_poll_rate(ompi_mpi_event_tick_rate);
-    }
-
-    /* At this point, we are fully configured and in MPI mode.  Any
-       communication calls here will work exactly like they would in
-       the user's code.  Setup the connections between procs and warm
-       them up with simple sends, if requested */
-
-    if (OMPI_SUCCESS != (ret = ompi_mpiext_init())) {
-        error = "ompi_mpiext_init";
-        goto error;
-    }
-
 #if OPAL_ENABLE_FT_MPI
     /* start the failure detector */
     if( ompi_ftmpi_enabled ) {
-        int rc;
         rc = ompi_comm_failure_detector_start();
         if( OMPI_SUCCESS != rc ) return rc;
     }
 #endif
 
+    /* Check whether we have been spawned or not.  We introduce that
+       at the very end, since we need collectives, datatypes, ptls
+       etc. up and running here.... */
+    if (OMPI_SUCCESS != (ret = ompi_dpm_dyn_init())) {
+        return ret;
+    }
+
     /* Fall through */
  error:
     if (ret != OMPI_SUCCESS) {
@@ -1022,21 +588,6 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided,
         return ret;
     }
 
-    /* Initialize the registered datarep list to be empty */
-    OBJ_CONSTRUCT(&ompi_registered_datareps, opal_list_t);
-
-    /* Initialize the arrays used to store the F90 types returned by the
-     *  MPI_Type_create_f90_XXX functions.
-     */
-    OBJ_CONSTRUCT( &ompi_mpi_f90_integer_hashtable, opal_hash_table_t);
-    opal_hash_table_init(&ompi_mpi_f90_integer_hashtable, 16 /* why not? */);
-
-    OBJ_CONSTRUCT( &ompi_mpi_f90_real_hashtable, opal_hash_table_t);
-    opal_hash_table_init(&ompi_mpi_f90_real_hashtable, FLT_MAX_10_EXP);
-
-    OBJ_CONSTRUCT( &ompi_mpi_f90_complex_hashtable, opal_hash_table_t);
-    opal_hash_table_init(&ompi_mpi_f90_complex_hashtable, FLT_MAX_10_EXP);
-
     /* All done.  Wasn't that simple? */
     opal_atomic_wmb();
     opal_atomic_swap_32(&ompi_mpi_state, OMPI_MPI_STATE_INIT_COMPLETED);
diff --git a/ompi/runtime/ompi_mpi_params.c b/ompi/runtime/ompi_mpi_params.c
index 0371228ed97..c747d55ee7d 100644
--- a/ompi/runtime/ompi_mpi_params.c
+++ b/ompi/runtime/ompi_mpi_params.c
@@ -18,8 +18,12 @@
  * Copyright (c) 2013-2019 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Mellanox Technologies, Inc.
  *                         All rights reserved.
- * Copyright (c) 2016-2019 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2018-2025 Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -51,7 +55,7 @@
  *
  * As a deviation from the norm, ompi_mpi_param_check is also
  * extern'ed in src/mpi/interface/c/bindings.h because it is already
- * included in all MPI function imlementation files
+ * included in all MPI function implementation files
  *
  * The values below are the default values.
  */
@@ -79,16 +83,19 @@ bool ompi_async_mpi_finalize = false;
 uint32_t ompi_add_procs_cutoff = OMPI_ADD_PROCS_CUTOFF_DEFAULT;
 bool ompi_mpi_dynamics_enabled = true;
 
-bool ompi_mpi_compat_mpi3 = false;
+bool ompi_mpi_compat_mpi3 = true;
 
 char *ompi_mpi_spc_attach_string = NULL;
 bool ompi_mpi_spc_dump_enabled = false;
+uint32_t ompi_pmix_connect_timeout = 0;
+
+bool ompi_enable_timing = false;
 
 static bool show_default_mca_params = false;
 static bool show_file_mca_params = false;
 static bool show_enviro_mca_params = false;
 static bool show_override_mca_params = false;
-static bool ompi_mpi_oversubscribe = false;
+bool ompi_mpi_oversubscribed = false;
 
 #if OPAL_ENABLE_FT_MPI
 int ompi_ftmpi_output_handle = 0;
@@ -96,6 +103,9 @@ bool ompi_ftmpi_enabled = false;
 #include "ompi/communicator/communicator.h"
 #endif /* OPAL_ENABLE_FT_MPI */
 
+static int ompi_stream_buffering_mode = -1;
+int ompi_comm_verbose_level = 0;
+
 int ompi_mpi_register_params(void)
 {
     int value;
@@ -144,20 +154,7 @@ int ompi_mpi_register_params(void)
         ompi_mpi_param_check = false;
     }
 
-    /*
-     * opal_progress: decide whether to yield and the event library
-     * tick rate
-     */
-    ompi_mpi_oversubscribe = false;
-    (void) mca_base_var_register("ompi", "mpi", NULL, "oversubscribe",
-                                 "Internal MCA parameter set by the runtime environment when oversubscribing nodes",
-                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
-                                 OPAL_INFO_LVL_9,
-                                 MCA_BASE_VAR_SCOPE_READONLY,
-                                 &ompi_mpi_oversubscribe);
-
     /* yield if the node is oversubscribed and allow users to override */
-    ompi_mpi_yield_when_idle |= ompi_mpi_oversubscribe;
     (void) mca_base_var_register("ompi", "mpi", NULL, "yield_when_idle",
                                  "Yield the processor when waiting for MPI communication (for MPI processes, will default to 1 when oversubscribing nodes)",
                                  MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
@@ -196,7 +193,7 @@ int ompi_mpi_register_params(void)
         ompi_mpi_param_check = true;
         if (!MPI_PARAM_CHECK) {
             opal_output(0, "WARNING: MCA parameter mpi_no_free_handles set to true, but MPI");
-            opal_output(0, "WARNING: parameter checking has been compiled out of Open MPI.");
+            opal_output(0, "WARNING: parameter checking has been compiled out of " OMPI_IDENT_STRING ".");
             opal_output(0, "WARNING: mpi_no_free_handles is therefore only partially effective!");
         }
     }
@@ -213,7 +210,7 @@ int ompi_mpi_register_params(void)
     /* Whether or not to print all MCA parameters in MPI_INIT */
     ompi_mpi_show_mca_params_string = NULL;
     (void) mca_base_var_register("ompi", "mpi", NULL, "show_mca_params",
-                                 "Whether to show all MCA parameter values during MPI_INIT or not (good for reproducability of MPI jobs "
+                                 "Whether to show all MCA parameter values during MPI_INIT or not (good for reproducibility of MPI jobs "
                                  "for debug purposes). Accepted values are all, default, file, api, and enviro - or a comma "
                                  "delimited combination of them",
                                  MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
@@ -255,7 +252,8 @@ int ompi_mpi_register_params(void)
 
     /* File to use when dumping the parameters */
     (void) mca_base_var_register("ompi", "mpi", NULL, "show_mca_params_file",
-                                 "If mpi_show_mca_params is true, setting this string to a valid filename tells Open MPI to dump all the MCA parameter values into a file suitable for reading via the mca_param_files parameter (good for reproducability of MPI jobs)",
+                                 "If mpi_show_mca_params is true, setting this string to a valid filename tells "
+                                 OMPI_IDENT_STRING " to dump all the MCA parameter values into a file suitable for reading via the mca_param_files parameter (good for reproducibility of MPI jobs)",
                                  MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
                                  OPAL_INFO_LVL_9,
                                  MCA_BASE_VAR_SCOPE_READONLY,
@@ -281,7 +279,7 @@ int ompi_mpi_register_params(void)
 
     /* Sparse group storage support */
     (void) mca_base_var_register("ompi", "mpi", NULL, "have_sparse_group_storage",
-                                 "Whether this Open MPI installation supports storing of data in MPI groups in \"sparse\" formats (good for extremely large process count MPI jobs that create many communicators/groups)",
+                                 "Whether this " OMPI_IDENT_STRING " installation supports storing of data in MPI groups in \"sparse\" formats (good for extremely large process count MPI jobs that create many communicators/groups)",
                                  MCA_BASE_VAR_TYPE_BOOL, NULL, 0,
                                  MCA_BASE_VAR_FLAG_DEFAULT_ONLY,
                                  OPAL_INFO_LVL_9,
@@ -365,7 +363,7 @@ int ompi_mpi_register_params(void)
                                       MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
     }
 
-    ompi_mpi_compat_mpi3 = false;
+    ompi_mpi_compat_mpi3 = true;
     (void) mca_base_var_register("ompi", "mpi", NULL, "compat_mpi3",
                                  "A boolean value for whether Open MPI operates in MPI-3 compatibility mode; this changes the following behavior: in operations without a handle, errors are raised on (true) MPI_COMM_WORLD (MPI-3 behavior) or (false) MPI_COMM_SELF (MPI-4 behavior).",
                                  MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
@@ -391,6 +389,68 @@ int ompi_mpi_register_params(void)
                                  &ompi_mpi_spc_dump_enabled);
 #endif // SPC_ENABLE
 
+    ompi_pmix_connect_timeout = 0; /* infinite timeout - see PMIx standard */
+    (void) mca_base_var_register ("ompi", "mpi", NULL, "pmix_connect_timeout",
+                                  "Timeout(secs) for calls to PMIx_Connect and PMIx_Group_construct/destruct. Default is no timeout.",
+                                  MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL,
+                                  0, 0, OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_LOCAL,
+                                  &ompi_pmix_connect_timeout);
+
+    /* check to see if we want timing information */
+    /* TODO: enable OMPI init and OMPI finalize timings if
+     * this variable was set to 1!
+     */
+    ompi_enable_timing = false;
+    (void) mca_base_var_register("ompi", "ompi", NULL, "timing",
+                                 "Request that critical timing loops be measured",
+                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
+                                 OPAL_INFO_LVL_9,
+                                 MCA_BASE_VAR_SCOPE_READONLY,
+                                 &ompi_enable_timing);
+
+    /*
+     * stdout/stderr buffering
+     * If the user requested to override the default setting then do
+     * as they wish.
+     */
+    (void) mca_base_var_register("ompi", "ompi", NULL, "stream_buffering",
+                                 "Adjust buffering for stdout/stderr. "
+                                 "(0) unbuffered, (1) line buffered, (2) fully buffered.",
+                                 MCA_BASE_VAR_TYPE_INT,
+                                 NULL, 0, 0,
+                                 OPAL_INFO_LVL_3,
+                                 MCA_BASE_VAR_SCOPE_READONLY,
+                                 &ompi_stream_buffering_mode);
+    if(0 == ompi_stream_buffering_mode) {
+        setvbuf(stdout, NULL, _IONBF, 0);
+        setvbuf(stderr, NULL, _IONBF, 0);
+    }
+    else if(1 == ompi_stream_buffering_mode) {
+        setvbuf(stdout, NULL, _IOLBF, 0);
+        setvbuf(stderr, NULL, _IOLBF, 0);
+    }
+    else if(2 == ompi_stream_buffering_mode) {
+        setvbuf(stdout, NULL, _IOFBF, 0);
+        setvbuf(stderr, NULL, _IOFBF, 0);
+    }
+
+
+#if OPAL_ENABLE_FT_MPI
+    /* Before loading any other part of the MPI library, we need to load
+     * the ft-mpi tune file to override default component selection when
+     * FT is desired ON; this does override openmpi-params.conf, but not
+     * command line or env.
+     */
+    if( ompi_ftmpi_enabled ) {
+        mca_base_var_load_extra_files("ft-mpi", false);
+    }
+#endif /* OPAL_ENABLE_FT_MPI */
+
+    (void) mca_base_var_register ("ompi", "mpi", "comm", "verbose",
+                                  "Verbosity level for communicator management subsystem",
+                                  MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
+                                  OPAL_INFO_LVL_8, MCA_BASE_VAR_SCOPE_LOCAL, &ompi_comm_verbose_level);
+
     return OMPI_SUCCESS;
 }
 
diff --git a/ompi/runtime/ompi_rte.c b/ompi/runtime/ompi_rte.c
index 4e74888e571..4e7719c73af 100644
--- a/ompi/runtime/ompi_rte.c
+++ b/ompi/runtime/ompi_rte.c
@@ -8,14 +8,14 @@
  *                         reserved.
  * Copyright (c) 2014-2018 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
- * Copyright (c) 2018-2020 Triad National Security, LLC. All rights
+ * Copyright (c) 2018-2023 Triad National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2019      Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2020      Amazon.com, Inc. or its affiliates.  All Rights
  *                         reserved.
- * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
- * Copyright (c) 2021      IBM Corporation. All rights reserved.
+ * Copyright (c) 2021-2025 Nanook Consulting  All rights reserved.
+ * Copyright (c) 2021-2022 IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  */
 #include "ompi_config.h"
@@ -61,7 +61,15 @@
 /* storage to support OMPI */
 opal_process_name_t pmix_name_wildcard = {UINT32_MAX-1, UINT32_MAX-1};
 opal_process_name_t pmix_name_invalid = {UINT32_MAX, UINT32_MAX};
-bool ompi_singleton = false;
+
+/**
+ * Flag used to indicate whether we setup (and should destroy) our job session
+ * directory. We keep track of this information because we may be using run-time
+ * infrastructure that manages its structure (e.g., OpenPMIx). If we setup this
+ * session directory structure, then we shall cleanup after ourselves.
+ */
+static bool destroy_job_session_dir = false;
+static bool destroy_proc_session_dir = false;
 
 static int _setup_top_session_dir(char **sdir);
 static int _setup_job_session_dir(char **sdir);
@@ -97,6 +105,7 @@ buffer_cleanup(void *value)
         }
         free (ptr);
     }
+    fns_init = false;
 }
 
 static opal_print_args_buffers_t*
@@ -540,10 +549,13 @@ int ompi_rte_init(int *pargc, char ***pargv)
     pmix_value_t pval;
     pmix_status_t rc;
     char **tmp;
+    bool singleton = false;
+    const static char *pmi_sentinels[] = {"PMI_FD", /* SLURM PMI1,2 */
+                                          "PMI_CONTROL_PORT", /* Cray Shasta */
+                                          NULL};
 
     u32ptr = &u32;
     u16ptr = &u16;
-    memset(&opal_process_info, 0, sizeof(opal_process_info));
 
     /* Convince OPAL to use our naming scheme */
     opal_process_name_print = _process_name_print_for_opal;
@@ -565,11 +577,39 @@ int ompi_rte_init(int *pargc, char ***pargv)
     opal_pmix_setup_nspace_tracker();
 
     /* initialize the selected module */
-    if (PMIX_SUCCESS != (ret = PMIx_Init(&opal_process_info.myprocid, NULL, 0))) {
+    if (!PMIx_Initialized() && (PMIX_SUCCESS != (ret = PMIx_Init(&opal_process_info.myprocid, NULL, 0)))) {
         /* if we get PMIX_ERR_UNREACH indicating that we cannot reach the
          * server, then we assume we are operating as a singleton */
         if (PMIX_ERR_UNREACH == ret) {
-            ompi_singleton = true;
+            bool found_a_pmi = false;
+            int n = 0;
+            /* if we are in a PMI environment with two tasks or more,
+             * we probably do not want to start singletons */
+            while (pmi_sentinels[n] != NULL) {
+                if (NULL != getenv(pmi_sentinels[n])) {
+                    found_a_pmi = true;
+                    break;
+                }
+                n++;
+            }
+            if (found_a_pmi) {
+                char *size_str = getenv("PMI_SIZE");
+                if (NULL == size_str) {
+                    size_str = getenv("SLURM_NPROCS");
+                }
+                int size = (NULL != size_str)?atoi(size_str):1;
+                if (1 < size) {
+                    char *rank_str = getenv("PMI_RANK");
+                    if (NULL == rank_str) {
+                        rank_str = getenv("SLURM_PROCID");
+                    }
+                    int rank = (NULL != rank_str)?atoi(rank_str):0;
+                    if (0 == rank) {
+                        opal_show_help("help-mpi-runtime.txt", "no-pmix-but", false, size);
+                    }
+                }
+            }
+            singleton = true;
         } else {
             /* we cannot run - this could be due to being direct launched
              * without the required PMI support being built, so print
@@ -588,6 +628,12 @@ int ompi_rte_init(int *pargc, char ***pargv)
     OPAL_PROC_MY_NAME.vpid = pname.vpid;
     opal_process_info.my_name.jobid = OPAL_PROC_MY_NAME.jobid;
     opal_process_info.my_name.vpid = OPAL_PROC_MY_NAME.vpid;
+    if (singleton) {
+        opal_process_info.is_singleton = true;
+    } else {
+        opal_process_info.is_singleton = false;
+    }
+
 
     /* set our hostname */
     ev1 = NULL;
@@ -605,7 +651,7 @@ int ompi_rte_init(int *pargc, char ***pargv)
     OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_LOCAL_RANK,
                                    &opal_process_info.my_name, &u16ptr, PMIX_UINT16);
     if (PMIX_SUCCESS != rc) {
-        if (ompi_singleton) {
+        if (opal_process_info.is_singleton) {
             /* just assume 0 */
             u16 = 0;
         } else {
@@ -620,7 +666,7 @@ int ompi_rte_init(int *pargc, char ***pargv)
     OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_NODE_RANK,
                                    &opal_process_info.my_name, &u16ptr, PMIX_UINT16);
     if (PMIX_SUCCESS != rc) {
-        if (ompi_singleton) {
+        if (opal_process_info.is_singleton) {
             /* just assume 0 */
             u16 = 0;
         } else {
@@ -639,7 +685,7 @@ int ompi_rte_init(int *pargc, char ***pargv)
     OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_JOB_SIZE,
                                    &pname, &u32ptr, PMIX_UINT32);
     if (PMIX_SUCCESS != rc) {
-        if (ompi_singleton) {
+        if ( opal_process_info.is_singleton) {
             /* just assume 1 */
             u32 = 1;
         } else {
@@ -654,7 +700,7 @@ int ompi_rte_init(int *pargc, char ***pargv)
     OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_UNIV_SIZE,
                                    &pname, &u32ptr, PMIX_UINT32);
     if (PMIX_SUCCESS != rc) {
-        if (ompi_singleton) {
+        if (opal_process_info.is_singleton) {
             /* just assume 1 */
             u32 = 1;
         } else {
@@ -800,6 +846,11 @@ int ompi_rte_init(int *pargc, char ***pargv)
         opal_process_info.initial_wdir = val;
         val = NULL;  // protect the string
     }
+    else {
+        // Probably singleton case. Just assume cwd.
+        opal_process_info.initial_wdir = calloc(1, OPAL_PATH_MAX + 1);
+        opal_getcwd(opal_process_info.initial_wdir, OPAL_PATH_MAX);
+    }
 
     /* identify our location */
     val = NULL;
@@ -825,19 +876,21 @@ int ompi_rte_init(int *pargc, char ***pargv)
 
     /* retrieve the local peers - defaults to local node */
     val = NULL;
-    OPAL_MODEX_RECV_VALUE(rc, PMIX_LOCAL_PEERS,
-                          &pname, &val, PMIX_STRING);
+    OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_LOCAL_PEERS,
+                                   &pname, &val, PMIX_STRING);
     if (PMIX_SUCCESS == rc && NULL != val) {
         peers = opal_argv_split(val, ',');
         free(val);
     } else {
-        ret = opal_pmix_convert_status(rc);
-        error = "local peers";
-        goto error;
+        peers = NULL;
     }
     /* if we were unable to retrieve the #local peers, set it here */
     if (0 == opal_process_info.num_local_peers) {
-        opal_process_info.num_local_peers = opal_argv_count(peers) - 1;
+        if (NULL != peers) {
+            opal_process_info.num_local_peers = opal_argv_count(peers) - 1;
+        } else {
+            opal_process_info.num_local_peers = 1;
+        }
     }
     /* if my local rank if too high, then that's an error */
     if (opal_process_info.num_local_peers < opal_process_info.my_local_rank) {
@@ -880,25 +933,15 @@ int ompi_rte_init(int *pargc, char ***pargv)
         opal_argv_free(peers);
     }
 
-    /*
-     * stdout/stderr buffering
-     * If the user requested to override the default setting then do
-     * as they wish.
-     */
-    OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, "OMPI_STREAM_BUFFERING",
-                                   &opal_process_info.my_name, &u16ptr, PMIX_UINT16);
-    if (PMIX_SUCCESS == rc) {
-        if (0 == u16) {
-            setvbuf(stdout, NULL, _IONBF, 0);
-            setvbuf(stderr, NULL, _IONBF, 0);
-        } else if (1 == u16) {
-            setvbuf(stdout, NULL, _IOLBF, 0);
-            setvbuf(stderr, NULL, _IOLBF, 0);
-        } else if (2 == u16 ) {
-            setvbuf(stdout, NULL, _IOFBF, 0);
-            setvbuf(stderr, NULL, _IOFBF, 0);
-        }
+#ifdef PMIX_NODE_OVERSUBSCRIBED
+    pname.jobid = opal_process_info.my_name.jobid;
+    pname.vpid = OPAL_VPID_WILDCARD;
+    OPAL_MODEX_RECV_VALUE_OPTIONAL(ret, PMIX_NODE_OVERSUBSCRIBED, &pname,
+                                   NULL, PMIX_BOOL);
+    if (PMIX_SUCCESS == ret) {
+        ompi_mpi_oversubscribed = true;
     }
+#endif
 
     return OPAL_SUCCESS;
 
@@ -938,15 +981,14 @@ static bool check_file(const char *root, const char *path)
 
 int ompi_rte_finalize(void)
 {
-    /* shutdown pmix */
-    PMIx_Finalize(NULL, 0);
 
     /* cleanup the session directory we created */
-    if (NULL != opal_process_info.job_session_dir) {
+    if (NULL != opal_process_info.job_session_dir && destroy_job_session_dir) {
         opal_os_dirpath_destroy(opal_process_info.job_session_dir,
                                 false, check_file);
         free(opal_process_info.job_session_dir);
         opal_process_info.job_session_dir = NULL;
+        destroy_job_session_dir = false;
     }
 
     if (NULL != opal_process_info.top_session_dir) {
@@ -954,9 +996,12 @@ int ompi_rte_finalize(void)
         opal_process_info.top_session_dir = NULL;
     }
 
-    if (NULL != opal_process_info.proc_session_dir) {
+    if (NULL != opal_process_info.proc_session_dir && destroy_proc_session_dir) {
+        opal_os_dirpath_destroy(opal_process_info.proc_session_dir,
+                                false, check_file);
         free(opal_process_info.proc_session_dir);
         opal_process_info.proc_session_dir = NULL;
+        destroy_proc_session_dir = false;
     }
 
     if (NULL != opal_process_info.app_sizes) {
@@ -997,6 +1042,11 @@ int ompi_rte_finalize(void)
     opal_pmix_finalize_nspace_tracker();
 
 
+    opal_finalize ();
+
+    /* shutdown pmix */
+    PMIx_Finalize(NULL, 0);
+
     return OMPI_SUCCESS;
 }
 
@@ -1051,32 +1101,28 @@ static void _release_fn(size_t refid, pmix_status_t status,
     debugger_event_active = false;
 }
 
-/*
- * Wait for a debugger if asked.  We support two ways of waiting for
- * attaching debuggers
- */
-void ompi_rte_wait_for_debugger(void)
+void ompi_rte_breakpoint(char *name)
 {
     pmix_info_t directive;
     char *evar;
-    int time, code = PMIX_ERR_DEBUGGER_RELEASE;
-    pmix_info_t info;
+    int rc, code = PMIX_DEBUGGER_RELEASE;
+    pmix_info_t info[2];
+    opal_process_name_t pname;
 
-    /* check PMIx to see if we are under a debugger */
-    if (NULL == getenv("PMIX_DEBUG_WAIT_FOR_NOTIFY") &&
-        NULL == getenv("PMIX_TEST_DEBUGGER_ATTACH")) {
-        /* if not, just return */
+    if (NULL != name
+        && NULL != (evar = getenv("OMPI_BREAKPOINT"))
+        && 0 != strcasecmp(evar, name)) {
+        /* they don't want to stop here */
         return;
     }
 
-    /* if we are being debugged, then we need to find
-     * the correct plug-ins
-     */
-    ompi_debugger_setup_dlls();
-
-    if (NULL != (evar = getenv("PMIX_TEST_DEBUGGER_SLEEP"))) {
-        time = strtol(evar, NULL, 10);
-        sleep(time);
+    /* check PMIx to see if we are under a debugger */
+    pname.jobid = opal_process_info.my_name.jobid;
+    pname.vpid = OPAL_VPID_WILDCARD;
+    OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_DEBUG_STOP_IN_APP,
+                                   &pname, NULL, PMIX_BOOL);
+    if (PMIX_SUCCESS != rc) {
+        /* if not, just return */
         return;
     }
 
@@ -1085,12 +1131,14 @@ void ompi_rte_wait_for_debugger(void)
     PMIx_Register_event_handler(&code, 1, &directive, 1, _release_fn, NULL, NULL);
     PMIX_INFO_DESTRUCT(&directive);
 
-    /* notify the host that we are waiting */
-    PMIX_INFO_LOAD(&info, PMIX_EVENT_NON_DEFAULT, NULL, PMIX_BOOL);
-    PMIx_Notify_event(PMIX_DEBUG_WAITING_FOR_NOTIFY,
+    /* notify the host that we are waiting in MPI_Init */
+    PMIX_INFO_LOAD(&info[0], PMIX_EVENT_NON_DEFAULT, NULL, PMIX_BOOL);
+    PMIX_INFO_LOAD(&info[1], PMIX_BREAKPOINT, "mpi-init", PMIX_STRING);
+    PMIx_Notify_event(PMIX_READY_FOR_DEBUG,
                       &opal_process_info.myprocid,
-                      PMIX_RANGE_RM, &info, 1, NULL, NULL);
-    PMIX_INFO_DESTRUCT(&info);
+                      PMIX_RANGE_RM, info, 2, NULL, NULL);
+    PMIX_INFO_DESTRUCT(&info[0]);
+    PMIX_INFO_DESTRUCT(&info[1]);
 
     /* let the MPI progress engine run while we wait for debugger release */
     OMPI_WAIT_FOR_COMPLETION(debugger_event_active);
@@ -1099,6 +1147,22 @@ void ompi_rte_wait_for_debugger(void)
     PMIx_Deregister_event_handler(handler, NULL, NULL);
 }
 
+/*
+ * Wait for a debugger if asked.  We support two ways of waiting for
+ * attaching debuggers
+ */
+void ompi_rte_wait_for_debugger(void)
+{
+    if (NULL != getenv("PMIX_TEST_DEBUGGER_ATTACH")
+        || NULL == getenv("OMPI_BREAKPOINT")) {
+        ompi_rte_breakpoint(NULL);
+        return;
+    }
+
+    /* check for the "mpi-init" breakpoint */
+    ompi_rte_breakpoint("mpi-init");
+}
+
 static int _setup_top_session_dir(char **sdir)
 {
     char *tmpdir;
@@ -1114,6 +1178,7 @@ static int _setup_top_session_dir(char **sdir)
 
 static int _setup_job_session_dir(char **sdir)
 {
+    int rc;
     /* get the effective uid */
     uid_t uid = geteuid();
 
@@ -1125,18 +1190,34 @@ static int _setup_job_session_dir(char **sdir)
         opal_process_info.job_session_dir = NULL;
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
-
+    rc = opal_os_dirpath_create(opal_process_info.job_session_dir, 0755);
+    if (OPAL_SUCCESS != rc) {
+        // could not create session dir
+        free(opal_process_info.job_session_dir);
+        opal_process_info.job_session_dir = NULL;
+        return rc;
+    }
+    destroy_job_session_dir = true;
     return OPAL_SUCCESS;
 }
 
 static int _setup_proc_session_dir(char **sdir)
 {
+    int rc;
+
     if (0 > opal_asprintf(sdir,  "%s/%d",
                           opal_process_info.job_session_dir,
                           opal_process_info.my_name.vpid)) {
         opal_process_info.proc_session_dir = NULL;
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
-
+    rc = opal_os_dirpath_create(opal_process_info.proc_session_dir, 0755);
+    if (OPAL_SUCCESS != rc) {
+        // could not create session dir
+        free(opal_process_info.proc_session_dir);
+        opal_process_info.proc_session_dir = NULL;
+        return rc;
+    }
+    destroy_proc_session_dir = true;
     return OPAL_SUCCESS;
 }
diff --git a/ompi/runtime/ompi_rte.h b/ompi/runtime/ompi_rte.h
index c6cb0610cc9..996fae9700b 100644
--- a/ompi/runtime/ompi_rte.h
+++ b/ompi/runtime/ompi_rte.h
@@ -6,7 +6,7 @@
  * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2019      Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2020      Triad National Security, LLC. All rights
+ * Copyright (c) 2020-2022  Triad National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2020      Amazon.com, Inc. or its affiliates.  All Rights
  *                         reserved.
@@ -23,8 +23,8 @@
  * to define when it should be built.
  *
  * Each component must provide a number of types and functions that mimic
- * those provided by ORTE. These include (where flexibility exists, the
- * ORTE data type is shown, but any compatible type is allowed. For example,
+ * those provided by PRTE. These include (where flexibility exists, the
+ * pmix data type is shown, but any compatible type is allowed. For example,
  * the jobid field in ompi_process_name_t could be any type of integer, but
  * cannot be a string):
  *
@@ -69,7 +69,7 @@
  *     1. ompi_rte_collective_t - an OPAL object used during RTE collective operations
  *        such as modex and barrier. It must be an opal_list_item_t and contain the
  *        following fields:
- *           a. id (ORTE type: int32_t)
+ *           a. id (pmix type: int32_t)
  *           b. bool active
  *              flag that user can poll on to know when collective
  *              has completed - set to false just prior to
@@ -216,6 +216,8 @@ OMPI_DECLSPEC int ompi_rte_convert_string_to_process_name(opal_process_name_t *n
 OMPI_DECLSPEC int ompi_rte_convert_process_name_to_string(char** name_string,
                                                           const opal_process_name_t *name);
 
+OMPI_DECLSPEC void ompi_rte_breakpoint(char *name);
+
 #define OMPI_LOCAL_JOBID(n) \
     ( (n) & 0x0000ffff)
 #define OMPI_JOB_FAMILY(n)  \
@@ -257,7 +259,6 @@ OMPI_DECLSPEC void ompi_rte_abort_peers(opal_process_name_t *procs,
 OMPI_DECLSPEC int ompi_rte_init(int *argc, char ***argv);
 OMPI_DECLSPEC int ompi_rte_finalize(void);
 OMPI_DECLSPEC void ompi_rte_wait_for_debugger(void);
-OMPI_DECLSPEC extern bool ompi_singleton;
 
 /* In a few places, we need to barrier until something happens
  * that changes a flag to indicate we can release - e.g., waiting
diff --git a/ompi/runtime/ompi_spc.c b/ompi/runtime/ompi_spc.c
index 4c0ed5a1b5e..6f1d8aa7d6a 100644
--- a/ompi/runtime/ompi_spc.c
+++ b/ompi/runtime/ompi_spc.c
@@ -9,6 +9,7 @@
  * Copyright (c) 2019      Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2020      IBM Corporation. All rights reserved.
+ * Copyright (c) 2022      Triad National Security, LLC. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -165,7 +166,11 @@ static const ompi_spc_event_t ompi_spc_events_desc[OMPI_SPC_NUM_COUNTERS] = {
     SET_COUNTER_ARRAY(OMPI_SPC_MAX_UNEXPECTED_IN_QUEUE, "The maximum number of messages that the unexpected message queue(s) within an MPI process "
                                                     "contained at once since the last reset of this counter. Note: This counter is reset each time it is read.", true, false),
     SET_COUNTER_ARRAY(OMPI_SPC_MAX_OOS_IN_QUEUE, "The maximum number of messages that the out of sequence message queue(s) within an MPI process "
-                                             "contained at once since the last reset of this counter. Note: This counter is reset each time it is read.", true, false)
+                                             "contained at once since the last reset of this counter. Note: This counter is reset each time it is read.", true, false),
+    SET_COUNTER_ARRAY(OMPI_SPC_ISENDRECV, "The number of times MPI_Isendrecv was called.", false, false),
+    SET_COUNTER_ARRAY(OMPI_SPC_ISENDRECV_REPLACE, "The number of times MPI_Isendrecv_replace was called.", false, false),
+    SET_COUNTER_ARRAY(OMPI_SPC_PARRIVED, "The number of times MPI_Parrived was called.", false, false),
+    SET_COUNTER_ARRAY(OMPI_SPC_PREADY, "The number of times MPI_Pready (or similar functions) was called.", false, false),
 };
 
 /* An array of event structures to store the event data (value, attachments, flags) */
@@ -392,7 +397,7 @@ static void ompi_spc_dump(void)
     ompi_spc_comm->c_coll->coll_barrier(ompi_spc_comm, ompi_spc_comm->c_coll->coll_barrier_module);
 }
 
-/* Frees any dynamically alocated OMPI SPC data structures */
+/* Frees any dynamically allocated OMPI SPC data structures */
 void ompi_spc_fini(void)
 {
     if (ompi_mpi_spc_dump_enabled) {
diff --git a/ompi/runtime/ompi_spc.h b/ompi/runtime/ompi_spc.h
index 374c50a5334..76ec7f25f16 100644
--- a/ompi/runtime/ompi_spc.h
+++ b/ompi/runtime/ompi_spc.h
@@ -152,6 +152,10 @@ typedef enum ompi_spc_counters {
     OMPI_SPC_OOS_IN_QUEUE,
     OMPI_SPC_MAX_UNEXPECTED_IN_QUEUE,
     OMPI_SPC_MAX_OOS_IN_QUEUE,
+    OMPI_SPC_ISENDRECV,
+    OMPI_SPC_ISENDRECV_REPLACE,
+    OMPI_SPC_PARRIVED,
+    OMPI_SPC_PREADY,
     OMPI_SPC_NUM_COUNTERS /* This serves as the number of counters.  It must be last. */
 } ompi_spc_counters_t;
 
diff --git a/ompi/runtime/params.h b/ompi/runtime/params.h
index 91a9120638a..db4e9043d7b 100644
--- a/ompi/runtime/params.h
+++ b/ompi/runtime/params.h
@@ -16,6 +16,9 @@
  * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
  * Copyright (c) 2013      NVIDIA Corporation.  All rights reserved.
  * Copyright (c) 2013      Intel, Inc. All rights reserved
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -28,6 +31,8 @@
 
 #include "ompi_config.h"
 
+#include "ompi/runtime/mpiruntime.h"
+
 BEGIN_C_DECLS
 
 /*
@@ -58,7 +63,7 @@ OMPI_DECLSPEC extern bool ompi_mpi_param_check;
  * handles that are still allocated during MPI_FINALIZE.
  *
  * This is good debugging for user applications to find out if they
- * are inadvertantly orphaning MPI handles.
+ * are inadvertently orphaning MPI handles.
  */
 OMPI_DECLSPEC extern bool ompi_debug_show_handle_leaks;
 
@@ -68,7 +73,7 @@ OMPI_DECLSPEC extern bool ompi_debug_show_handle_leaks;
  * freed via MPI_FREE_MEM will be displayed during MPI_FINALIZE.
  *
  * This is good debugging for user applications to find out if they
- * are inadvertantly orphaning MPI "special" memory.
+ * are inadvertently orphaning MPI "special" memory.
  */
 OMPI_DECLSPEC extern int ompi_debug_show_mpi_alloc_mem_leaks;
 
@@ -79,7 +84,7 @@ OMPI_DECLSPEC extern int ompi_debug_show_mpi_alloc_mem_leaks;
  * attempt to use them will result in an MPI error.
  *
  * This is good debugging for user applications to find out if they
- * are inadvertantly using MPI handles after they have been freed.
+ * are inadvertently using MPI handles after they have been freed.
  */
 OMPI_DECLSPEC extern bool ompi_debug_no_free_handles;
 
@@ -118,8 +123,8 @@ OMPI_DECLSPEC extern int ompi_mpi_abort_delay;
 /**
  * Whether we operate in MPI3 compatibility, or MPI4 mode (default).
  *
- * true: use MPI3 compatibility
- * false: use MPI4 compatibility (default)
+ * true: use MPI3 compatibility (default)
+ * false: use MPI4 compatibility
  *
  * Behavioral changes:
  *   - errors in operations without a handle are raised on MPI_COMM_WORLD (MPI-3 behavior) or MPI_COMM_SELF (MPI-4 behavior)
@@ -172,6 +177,25 @@ OMPI_DECLSPEC extern char * ompi_mpi_spc_attach_string;
  */
 OMPI_DECLSPEC extern bool ompi_mpi_spc_dump_enabled;
 
+/**
+ * Timeout for calls to PMIx_Connect(default 0, no timeout)
+ */
+OMPI_DECLSPEC extern uint32_t ompi_pmix_connect_timeout;
+
+ /**
+ * A boolean value that determines whether or not to enable runtime timing of
+ * init and finalize.
+ */
+OMPI_DECLSPEC extern bool ompi_enable_timing;
+
+OMPI_DECLSPEC extern int ompi_mpi_event_tick_rate;
+OMPI_DECLSPEC extern bool ompi_mpi_yield_when_idle;
+
+ /**
+ * An integer value specifying verbosity level for communicator management
+ * subsystem.
+ */
+OMPI_DECLSPEC extern int ompi_comm_verbose_level;
 
 /**
  * Register MCA parameters used by the MPI layer.
@@ -183,6 +207,7 @@ OMPI_DECLSPEC extern bool ompi_mpi_spc_dump_enabled;
  */
 OMPI_DECLSPEC int ompi_mpi_register_params(void);
 
+
 /**
  * Display all MCA parameters used
  *
@@ -192,6 +217,12 @@ OMPI_DECLSPEC int ompi_mpi_register_params(void);
  */
 int ompi_show_all_mca_params(int32_t, int, char *);
 
+/**
+ * Set by checking PMIx to see if we are running in an oversubscribed
+ * environment or not.
+ */
+OMPI_DECLSPEC extern bool ompi_mpi_oversubscribed;
+
 END_C_DECLS
 
 #endif /* OMPI_RUNTIME_PARAMS_H */
diff --git a/ompi/tools/mpirun/Makefile.am b/ompi/tools/mpirun/Makefile.am
index bcbc90fe95a..7167ccdf6dd 100644
--- a/ompi/tools/mpirun/Makefile.am
+++ b/ompi/tools/mpirun/Makefile.am
@@ -1,8 +1,10 @@
 #
 # Copyright (c) 2019-2020 Intel, Inc.  All rights reserved.
 # Copyright (c) 2020      IBM Corporation.  All rights reserved.
-# Copyright (c) 2021      Amazon.com, Inc. or its affiliates.
-#                         All Rights reserved.
+# Copyright (c) 2021-2022 Amazon.com, Inc. or its affiliates.  All Rights reserved.
+# Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+# Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -11,10 +13,25 @@
 #
 
 if OMPI_WANT_PRRTE
+
+bin_PROGRAMS = mpirun
+
+EXTRA_DIST = help-mpirun.txt
+
+mpirun_SOURCES = \
+    main.c
+
+mpirun_LDADD = \
+	$(top_builddir)/opal/libopen-pal_core.la
+
+mpirun_CPPFLAGS = \
+    -DMCA_oshmem_FRAMEWORKS="\"$(MCA_oshmem_FRAMEWORKS)\"" \
+    -DMCA_ompi_FRAMEWORKS="\"$(MCA_ompi_FRAMEWORKS)\"" \
+    -DMCA_opal_FRAMEWORKS="\"$(MCA_opal_FRAMEWORKS)\""
+
 install-exec-hook:
-	(cd $(DESTDIR)$(bindir); rm -f mpirun$(EXEEXT); $(LN_S) $(PRTE_PATH)$(EXEEXT) mpirun$(EXEEXT))
-	(cd $(DESTDIR)$(bindir); rm -f mpiexec$(EXEEXT); $(LN_S) $(PRTE_PATH)$(EXEEXT) mpiexec$(EXEEXT))
-	(cd $(DESTDIR)$(bindir); rm -f oshrun$(EXEEXT); $(LN_S) $(PRTE_PATH)$(EXEEXT) oshrun$(EXEEXT))
+	(cd $(DESTDIR)$(bindir); rm -f mpiexec$(EXEEXT); $(LN_S) mpirun$(EXEEXT) mpiexec$(EXEEXT))
+	(cd $(DESTDIR)$(bindir); rm -f oshrun$(EXEEXT); $(LN_S) mpirun$(EXEEXT) oshrun$(EXEEXT))
 
 uninstall-local:
 	rm -f $(DESTDIR)$(bindir)/mpirun$(EXEEXT) \
diff --git a/ompi/tools/mpirun/help-mpirun.txt b/ompi/tools/mpirun/help-mpirun.txt
new file mode 100644
index 00000000000..7df77c785cf
--- /dev/null
+++ b/ompi/tools/mpirun/help-mpirun.txt
@@ -0,0 +1,23 @@
+# -*- text -*-
+#
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+# This is the US/English help file for Open MPI wrapper compiler error
+# messages.
+#
+[no-prterun-found]
+Open MPI's mpirun command was unable to find an underlying prterun
+command to execute.  Consider setting the OMPI_PRTERUN environment
+variable to help mpirun find the correct underlying prterun.
+[prterun-exec-failed]
+Open MPI's mpirun command could not execute the underlying prterun
+command.  The prterun command we tried to execute and the error
+message from exec() are below:
+
+  Command: %s
+  Error Message: %s
diff --git a/ompi/tools/mpirun/main.c b/ompi/tools/mpirun/main.c
new file mode 100644
index 00000000000..3a4e18dced9
--- /dev/null
+++ b/ompi/tools/mpirun/main.c
@@ -0,0 +1,217 @@
+/***************************************************************************
+ *                                                                         *
+ *          Open MPI: Open Source High Performance Computing               *
+ *                                                                         *
+ *                   https://www.open-mpi.org/                             *
+ *                                                                         *
+ ***************************************************************************/
+#include "ompi_config.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <errno.h>
+#if HAVE_SYS_STAT_H
+#    include <sys/stat.h>
+#endif /* HAVE_SYS_STAT_H */
+#include <string.h>
+
+#include "opal/mca/base/base.h"
+#include "opal/mca/installdirs/base/base.h"
+#include "opal/runtime/opal.h"
+#include "opal/util/argv.h"
+#include "opal/util/basename.h"
+#include "opal/util/opal_environ.h"
+#include "opal/util/os_dirpath.h"
+#include "opal/util/os_path.h"
+#include "opal/util/path.h"
+#include "opal/util/printf.h"
+#include "opal/util/show_help.h"
+#include "ompi/constants.h"
+
+static char *find_prterun(void)
+{
+    char *filename = NULL;
+#if !OMPI_USING_INTERNAL_PRRTE
+    char *prrte_prefix = NULL;
+#endif
+
+    /* 1) Did the user tell us exactly where to find prterun? */
+    filename = getenv("OMPI_PRTERUN");
+    if (NULL != filename) {
+        return filename;
+    }
+
+#if OMPI_USING_INTERNAL_PRRTE
+    /* 2) If using internal PRRTE, use our bindir.  Note that this
+     * will obey OPAL_PREFIX and OPAL_DESTDIR */
+    opal_asprintf(&filename, "%s%sprterun", opal_install_dirs.bindir, OPAL_PATH_SEP);
+    return filename;
+#else
+
+    /* 3) Look in ${PRTE_PREFIX}/bin */
+    prrte_prefix = getenv("PRTE_PREFIX");
+    if (NULL != prrte_prefix) {
+        opal_asprintf(&filename, "%s%sbin%sprterun", prrte_prefix, OPAL_PATH_SEP, OPAL_PATH_SEP);
+        return filename;
+    }
+
+    /* 4) See if configure told us where to look, if set */
+#if defined(OMPI_PRTERUN_PATH)
+    return strdup(OMPI_PRTERUN_PATH);
+#else
+
+    /* 5) Use path search */
+    filename = opal_find_absolute_path("prterun");
+
+    return filename;
+#endif
+#endif
+}
+
+static void append_prefixes(char ***out, const char *in)
+{
+    if (NULL == in) {
+        return;
+    }
+
+    char **tokenized;
+    tokenized = opal_argv_split(in, ' ');
+    if (NULL == tokenized) {
+        return;
+    }
+
+    int count = opal_argv_count(*out);
+    for (int i = 0; tokenized[i] != NULL; ++i) {
+        // Skip adding the names "common" and "pmix" to the list
+        if (strcmp(tokenized[i], "common") == 0 ||
+            strcmp(tokenized[i], "pmix") == 0) {
+            continue;
+        }
+        opal_argv_append(&count, out, tokenized[i]);
+    }
+
+    opal_argv_free(tokenized);
+}
+
+static void setup_mca_prefixes(void)
+{
+    int count = 0;
+    char **tmp = NULL;
+
+    opal_argv_append(&count, &tmp, "mca");
+    opal_argv_append(&count, &tmp, "opal");
+    opal_argv_append(&count, &tmp, "ompi");
+
+    append_prefixes(&tmp, MCA_oshmem_FRAMEWORKS);
+    append_prefixes(&tmp, MCA_ompi_FRAMEWORKS);
+    append_prefixes(&tmp, MCA_opal_FRAMEWORKS);
+
+    char *env_str = opal_argv_join(tmp, ',');
+    opal_setenv("OMPI_MCA_PREFIXES", env_str, true,
+                &environ);
+    free(env_str);
+
+    opal_argv_free(tmp);
+}
+
+
+int main(int argc, char *argv[])
+{
+    char *opal_prefix = getenv("OPAL_PREFIX");
+    char *full_prterun_path = NULL;
+    char **prterun_args = NULL;
+    int ret;
+    size_t i;
+
+    ret = opal_init_util(&argc, &argv);
+    if (OMPI_SUCCESS != ret) {
+        fprintf(stderr, "Failed initializing opal: %d\n", ret);
+        exit(1);
+    }
+
+    /* note that we just modify our environment rather than create a
+     * child environment because it is easier and we're not going to
+     * be around long enough for it to matter (since we exec prterun
+     * asap */
+    setenv("PRTE_MCA_schizo_proxy", "ompi", 1);
+    setenv("OMPI_VERSION", OMPI_VERSION, 1);
+    char *base_tool_name = opal_basename(argv[0]);
+    setenv("OMPI_TOOL_NAME", base_tool_name, 1);
+    free(base_tool_name);
+
+    /* TODO: look for --prefix and compare with OPAL_PREFIX and pick
+     * one */
+
+    /* as a special case, if OPAL_PREFIX was set and either PRRTE or
+     * PMIx are internal builds, set their prefix variables as well */
+    if (NULL != opal_prefix) {
+#if OMPI_USING_INTERNAL_PRRTE
+        setenv("PRTE_PREFIX", opal_prefix, 1);
+#endif
+#if OPAL_USING_INTERNAL_PMIX
+        setenv("PMIX_PREFIX", opal_prefix, 1);
+#endif
+    }
+
+    full_prterun_path = find_prterun();
+    if (NULL == full_prterun_path) {
+        opal_show_help("help-mpirun.txt", "no-prterun-found", 1);
+        exit(1);
+    }
+
+    /*
+     * set environment variable for our install location
+     * used within the OMPI prrte schizo component
+     */
+
+    setenv("OMPI_LIBDIR_LOC", opal_install_dirs.libdir, 1);
+
+    // Set environment variable to tell PRTE what MCA prefixes belong
+    // to Open MPI.
+    setup_mca_prefixes();
+
+    /* calling mpirun (and now prterun) with a full path has a special
+     * meaning in terms of -prefix behavior, so copy that behavior
+     * into prterun */
+    if (opal_path_is_absolute(argv[0])) {
+        opal_argv_append_nosize(&prterun_args, full_prterun_path);
+    } else {
+        opal_argv_append_nosize(&prterun_args, "prterun");
+    }
+
+    /* Copy all the mpirun arguments to prterun.
+     * TODO: Need to handle --prefix rationally here. */
+    for (i = 1; NULL != argv[i]; i++) {
+        opal_argv_append_nosize(&prterun_args, argv[i]);
+    }
+    ret = execv(full_prterun_path, prterun_args);
+    opal_show_help("help-mpirun.txt", "prterun-exec-failed",
+                   1, full_prterun_path, strerror(errno));
+    exit(1);
+}
+
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2017-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2020-2022 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2022      Triad National Security, LLC. All rights
+ *                         reserved.
+
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
diff --git a/ompi/tools/mpisync/Makefile.am b/ompi/tools/mpisync/Makefile.am
index c02046c4af7..e0d499a8b74 100644
--- a/ompi/tools/mpisync/Makefile.am
+++ b/ompi/tools/mpisync/Makefile.am
@@ -15,9 +15,8 @@
 #                         All rights reserved.
 # Copyright (c) 2014      Artem Polyakov <artpol84@gmail.com>
 # Copyright (c) 2016      IBM Corporation.  All rights reserved.
-# Copyright (c) 2017      Research Organization for Information Science
-#                         and Technology (RIST). All rights reserved.
-#
+# Copyright (c) 2017-2022 Research Organization for Information Science
+#                         and Technology (RIST).  All rights reserved.
 # Copyright (c) 2019      Intel, Inc.  All rights reserved.
 # $COPYRIGHT$
 #
@@ -26,31 +25,9 @@
 # $HEADER$
 #
 
-
-
-AM_CFLAGS = \
-            -DOPAL_CONFIGURE_USER="\"@OPAL_CONFIGURE_USER@\"" \
-            -DOPAL_CONFIGURE_HOST="\"@OPAL_CONFIGURE_HOST@\"" \
-            -DOPAL_CONFIGURE_DATE="\"@OPAL_CONFIGURE_DATE@\"" \
-            -DOMPI_BUILD_USER="\"$$USER\"" \
-            -DOMPI_BUILD_HOST="\"$${HOSTNAME:-`(hostname || uname -n) | sed 1q`}\"" \
-            -DOMPI_BUILD_DATE="\"`$(top_srcdir)/config/getdate.sh`\"" \
-            -DOMPI_BUILD_CFLAGS="\"@CFLAGS@\"" \
-            -DOMPI_BUILD_CPPFLAGS="\"@CPPFLAGS@\"" \
-            -DOMPI_BUILD_CXXFLAGS="\"@CXXFLAGS@\"" \
-            -DOMPI_BUILD_CXXCPPFLAGS="\"@CXXCPPFLAGS@\"" \
-            -DOMPI_BUILD_FFLAGS="\"@FFLAGS@\"" \
-            -DOMPI_BUILD_FCFLAGS="\"@FCFLAGS@\"" \
-            -DOMPI_BUILD_LDFLAGS="\"@LDFLAGS@\"" \
-            -DOMPI_BUILD_LIBS="\"@LIBS@\"" \
-            -DOPAL_CC_ABSOLUTE="\"@OPAL_CC_ABSOLUTE@\"" \
-            -DOMPI_CXX_ABSOLUTE="\"@OMPI_CXX_ABSOLUTE@\""
-
 include $(top_srcdir)/Makefile.ompi-rules
 
-man_pages = mpisync.1
-EXTRA_DIST = $(man_pages:.1=.1in)\
-	     mpirun_prof \
+EXTRA_DIST = mpirun_prof \
 	     ompi_timing_post
 
 if OPAL_INSTALL_TIMING_BINARIES
@@ -58,14 +35,6 @@ if OPAL_INSTALL_TIMING_BINARIES
 bin_PROGRAMS = mpisync
 bin_SCRIPTS = mpirun_prof ompi_timing_post
 
-nodist_man_MANS = $(man_pages)
-
-$(nodist_man_MANS): $(top_builddir)/opal/include/opal_config.h
-
-install-data-hook:
-	(cd $(DESTDIR)$(mandir)/man1; rm -f mpirun_prof.1; $(LN_S) mpisync.1 mpirun_prof.1)
-	(cd $(DESTDIR)$(mandir)/man1; rm -f ompi_timing_post.1; $(LN_S) mpisync.1 ompi_timing_post.1)
-
 endif
 
 mpisync_SOURCES = \
@@ -76,10 +45,4 @@ mpisync_SOURCES = \
         sync.c
 
 mpisync_LDADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la
-mpisync_LDADD += $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
-
-clean-local:
-	test -z "$(OMPI_CXX_TEMPLATE_REPOSITORY)" || rm -rf $(OMPI_CXX_TEMPLATE_REPOSITORY)
-
-distclean-local:
-	rm -f $(man_pages)
+mpisync_LDADD += $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
diff --git a/ompi/tools/mpisync/mpisync.1in b/ompi/tools/mpisync/mpisync.1in
deleted file mode 100644
index 219d89f034f..00000000000
--- a/ompi/tools/mpisync/mpisync.1in
+++ /dev/null
@@ -1,48 +0,0 @@
-.\" Copyright (c) 2014      Artem Polyakov <artpol84@gmail.com>.  All rights reserved.
-.TH MPISYNC 1 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-Open MPI timing tools
-.
-.SH SYNTAX
-.B mpisync
-[\fIoptions\fR]
-.br
-.B mpirun_prof
-[\fIoptions\fR]
-.br
-.B ompi_timing_post
-[\fI<timing-file>\fR] [\fI<processed-file>\fR]
-.
-.SH DESCRIPTION
-.PP
-.BR mpisync
-determines clock offsets relative to Head Node Process (HNP). It accepts the following options:
-.TP
-\fB\-o\fR, \fB\-\-output\fR
-The name of output file where offsets related to HNP will be written
-.TP
-\fB\-h\fR, \fB\-\-help\fR
-Print help information
-.PP
-.BR ompi_timing_post
-takes the timing output file as input parameter. The events are sorted by the timestamps. Next, the timestamps are replaced with time offsets relative to the
-.BR first
-:
-.BR previous
-event.
-.PP
-.BR mpirun_prof
-is a wrapper around
-.BR mpirun
-that performs clock synchronisation and post-processing of the timing output file.
-
-.SH NOTES
-.PP
-The mpisync code was derived from MPIPerf project:
-http://mpiperf.cpct.sibsutis.ru/index.php/Main/Documentation
-.
-.SH FILES
-.PP
-The output file has following format:
-.PP
-<hostname> <round-trip-time> <offset-from-hnp>
diff --git a/ompi/tools/ompi_info/Makefile.am b/ompi/tools/ompi_info/Makefile.am
index fe739548c8c..4ec193f2390 100644
--- a/ompi/tools/ompi_info/Makefile.am
+++ b/ompi/tools/ompi_info/Makefile.am
@@ -9,7 +9,7 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2008-2020 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2008-2022 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
 # Copyright (c) 2012      Los Alamos National Security, LLC.
 #                         All rights reserved.
@@ -42,11 +42,6 @@ AM_CFLAGS = \
             -DOPAL_CC_ABSOLUTE="\"@OPAL_CC_ABSOLUTE@\"" \
             -DOMPI_CXX_ABSOLUTE="\"@OMPI_CXX_ABSOLUTE@\""
 
-include $(top_srcdir)/Makefile.ompi-rules
-
-man_pages = ompi_info.1
-EXTRA_DIST = $(man_pages:.1=.1in)
-
 if OPAL_INSTALL_BINARIES
 
 bin_PROGRAMS = ompi_info
@@ -66,10 +61,7 @@ ompi_info_SOURCES = \
         param.c
 
 ompi_info_LDADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la
-ompi_info_LDADD += $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
-
-clean-local:
-	test -z "$(OMPI_CXX_TEMPLATE_REPOSITORY)" || rm -rf $(OMPI_CXX_TEMPLATE_REPOSITORY)
+ompi_info_LDADD += $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 
 distclean-local:
 	rm -f $(man_pages)
diff --git a/ompi/tools/ompi_info/ompi_info.1in b/ompi/tools/ompi_info/ompi_info.1in
deleted file mode 100644
index 89cc2551ec1..00000000000
--- a/ompi/tools/ompi_info/ompi_info.1in
+++ /dev/null
@@ -1,287 +0,0 @@
-.\" Man page contributed by Dirk Eddelbuettel <edd@debian.org>
-.\" and released under the BSD license
-.\" Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
-.\" Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
-.TH OMPI_INFO 1 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-ompi_info - Display information about the Open MPI installation
-.
-.\" **************************
-.\"    Synopsis Section
-.\" **************************
-.SH SYNOPSIS
-.B ompi_info [options]
-.
-.\" **************************
-.\"    Description Section
-.\" **************************
-.SH DESCRIPTION
-.PP
-.B ompi_info
-provides detailed information about the Open MPI installation.  It can
-be useful for at least three common scenarios:
-.PP
-1. Checking local configuration and seeing how Open MPI was installed.
-.PP
-2. Submitting bug reports / help requests to the Open MPI community
-(see
-.IR https://www.open-mpi.org/community/help/ )
-.PP
-3. Seeing a list of installed Open MPI plugins and querying what
-MCA parameters they support.
-.PP
-.B NOTE:
-.B ompi_info
-defaults to only showing a few MCA parameters by default (i.e., level
-1 parameters).  Use the
-.B --level
-option to enable showing more options (see the LEVELS section for more
-information).
-.
-.\" **************************
-.\"    Options Section
-.\" **************************
-.SH OPTIONS
-.B ompi_info
-accepts the following options:
-.TP 8
-.I \-a|\-\-all
-Show all configuration options and MCA parameters.  Also changes the
-default MCA parameter level to 9, unless
-.I --level
-is also specified.
-.TP 8
-.I \-\-arch
-Show architecture on which Open MPI was compiled.
-.TP 8
-.I \-c|\-\-config
-Show configuration options
-.TP 8
-.I \-gmca|\-\-gmca <param> <value>
-Pass global MCA parameters that are applicable to all contexts.
-.TP 8
-.I \-h|\-\-help
-Shows help / usage message.
-.TP 8
-.I \-\-hostname
-Show the hostname on which Open MPI was configured and built.
-.TP 8
-.I \-\-internal
-Show internal MCA parameters (not meant to be modified by users).
-.TP 8
-.I \-\-level <level>
-Show only variables with at most this level (1-9). The default is 1 unless
-\-\-all is specified without \-\-level, in which case the default is 9. See
-the LEVELS section for more information.
-.TP 8
-.I \-mca|\-\-mca <param> <value>
-Pass context-specific MCA parameters; they are considered global if --gmca is
-not used and only one context is specified.
-.TP 8
-.I \-\-param <type> <component>
-Show MCA parameters.  The first parameter is the type of the component
-to display; the second parameter is the specific component to display
-(or the keyword "all", meaning "display all components of this type").
-.TP 8
-.I \-t|\-\-type
-Show MCA parameters of the type specified in the parameter. Accepts the
-following parameters: unsigned_int, unsigned_long, unsigned_long_long,
-size_t, string, version_string, bool, double. By default level
-is 1 unless it is specified with \-\-level.
-.TP 8
-.I \-\-parsable
-When used in conjunction with other parameters, the output is
-displayed in a machine-parsable format
-.I \-\-parseable
-Synonym for --parsable
-.TP 8
-.I \-\-path <type>
-Show paths that Open MPI was configured with.  Accepts the following
-parameters: prefix, bindir, libdir, incdir, pkglibdir, sysconfdir.
-.TP 8
-.I \-\-pretty
-When used in conjunction with other parameters, the output is
-displayed in 'prettyprint' format (default)
-.TP 8
-.I \-\-selected-only
-Show only variables from selected components.
-.TP 8
-.I \-V|\-\-version
-Show version of Open MPI.
-.
-.\" **************************
-.\"    Levels Section
-.\" **************************
-.SH LEVELS
-Open MPI has many, many run-time tunable parameters (called "MCA
-parameters"), and usually only a handfull of them are useful to a
-given user.
-.
-.
-.PP
-As such, Open MPI has divided these parameters up into nine distinct
-levels, broken down into three categories, each with three
-sub-categories.
-.
-.
-.PP
-Note that since each MCA parameter is accessible through the MPI_T
-control variable API (introduced in MPI-3.0), these levels exactly
-correspond to the nine MPI_T cvar levels.
-.
-.
-.PP
-The three categories are:
-.TP 4
-.B End user
-Generally, these are parameters that are required for correctness,
-meaning that a user may need to set these just to get their MPI
-application to run correctly. For example, BTL "if_include" and
-"if_exclude" parameters fit into this category.
-.
-.TP
-.B Application tuner
-Generally, these are parameters that can be used to tweak MPI
-application performance. This even includes parameters that control
-resource exhaustion levels (e.g., number of free list entries, size of
-buffers, etc.), and could be considered "correctness" parameters if
-they're set too low. But, really -- they're tuning parameters.
-.
-.TP
-.B Open MPI developer
-Parameters in this category either don't fit in the other two, or are
-specifically intended for debugging / development of Open MPI itself.
-.
-.
-.PP
-And within each category, there are three sub-categories:
-.TP 4
-.B Basic
-This sub-category is for parameters that everyone in this category
-will want to see -- even less-advanced end users, application tuners,
-and new OMPI developers.
-.
-.TP
-.B Detailed
-This sub-category is for parameters that are generally useful, but
-users probably won't need to change them often.
-.
-.TP
-.B All
-This sub-category is for all other parameters. Such parameters are
-likely fairly esoteric.
-.
-.
-.PP
-Combining the categories and sub-categories, here's how Open MPI
-defines all nine levels:
-.TP 4
-1
-Basic information of interest to end users.
-.TP
-2
-Detailed information of interest to end users.
-.TP
-3
-All remaining information of interest to end users.
-.TP
-4
-Basic information required for application tuners.
-.TP
-5
-Detailed information required for application tuners.
-.TP
-6
-All remaining information required for application tuners.
-.TP
-7
-Basic information for Open MPI implementors.
-.TP
-8
-Detailed information for Open MPI implementors.
-.TP
-9
-All remaining information for Open MPI implementors.
-.
-.
-.PP
-By default,
-.B ompi_info
-only shows level 1 MCA parameters.  To see more MCA parameters, use
-the
-.B --level
-command line option.
-.
-.
-.PP
-.\" **************************
-.\"    Examples Section
-.\" **************************
-.SH EXAMPLES
-.TP 4
-ompi_info
-Show the default output of options and listing of installed
-components in a human-readable / prettyprint format.
-.
-.
-.TP
-ompi_info --parsable
-Show the default output of options and listing of installed
-components in a machine-parsable format.
-.
-.
-.TP
-ompi_info --param btl tcp
-Show the level 1 MCA parameters of the "tcp" BTL component in a
-human-readable / prettyprint format.
-.
-.
-.TP
-ompi_info --param btl tcp --level 6
-Show the level 1 through level 6 MCA parameters of the "tcp" BTL
-component in a human-readable / prettyprint format.
-.
-.
-.TP
-ompi_info --param btl tcp --parsable
-Show the level 1 MCA parameters of the "tcp" BTL component in a
-machine-parsable format.
-.
-.
-.TP
-ompi_info --type string --pretty-print --level 3
-Show the level 3 MCA parameters of string type in a human-readable /
-prettyprint format.
-.
-.
-.TP
-ompi_info --path bindir
-Show the "bindir" that Open MPI was configured with.
-.
-.
-.TP
-ompi_info --version
-Show the version of Open MPI version numbers in a prettyprint format.
-.
-.
-.TP
-ompi_info --all
-Show
-.I all
-information about the Open MPI installation, including all components
-that can be found, all the MCA parameters that they support (i.e.,
-levels 1 through 9), versions of Open MPI and the components, etc.
-.
-.
-.\" **************************
-.\"    Authors Section
-.\" **************************
-.SH AUTHORS
-The Open MPI maintainers -- see
-.I https://www.open-mpi.org/
-or the file
-.IR AUTHORS .
-.PP
-This manual page was originally contributed by Dirk Eddelbuettel
-<edd@debian.org>, one of the Debian GNU/Linux maintainers for Open
-MPI, and may be used by others.
diff --git a/ompi/tools/ompi_info/ompi_info.c b/ompi/tools/ompi_info/ompi_info.c
index 01b33c933e1..cec93c4b431 100644
--- a/ompi/tools/ompi_info/ompi_info.c
+++ b/ompi/tools/ompi_info/ompi_info.c
@@ -96,8 +96,8 @@ int main(int argc, char *argv[])
     }
 
     if (opal_cmd_line_is_taken(ompi_info_cmd_line, "version")) {
-        fprintf(stdout, "Open MPI v%s\n\n%s\n",
-                OPAL_VERSION, PACKAGE_BUGREPORT);
+        fprintf(stdout, "%s v%s\n\n%s\n",
+                PACKAGE_NAME, OPAL_VERSION, PACKAGE_BUGREPORT);
         exit(0);
     }
 
diff --git a/ompi/tools/ompi_info/param.c b/ompi/tools/ompi_info/param.c
index 785bc65b6ac..f112bc5ed0c 100644
--- a/ompi/tools/ompi_info/param.c
+++ b/ompi/tools/ompi_info/param.c
@@ -14,7 +14,7 @@
  * Copyright (c) 2014-2019 Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2015-2019 Intel, Inc.  All rights reserved.
- * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2018-2022 Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
  * $COPYRIGHT$
  *
@@ -40,6 +40,7 @@
 
 #include MCA_timer_IMPLEMENTATION_HEADER
 #include "opal/include/opal/version.h"
+#include "opal/opal_portable_platform.h"
 #include "opal/class/opal_value_array.h"
 #include "opal/class/opal_pointer_array.h"
 #include "opal/util/printf.h"
@@ -47,7 +48,6 @@
 #include "opal/runtime/opal_info_support.h"
 
 #include "ompi/tools/ompi_info/ompi_info.h"
-#include "ompi/include/mpi_portable_platform.h"
 
 
 const char *ompi_info_deprecated_value = "deprecated-ompi-info-value";
@@ -62,10 +62,10 @@ static void append(char *dest, size_t max, int *first, char *src)
 
     len = max - strlen(dest);
     if (!(*first)) {
-        strncat(dest, ", ", len);
+        strncat(dest, ", ", len - 1);
         len = max - strlen(dest);
     }
-    strncat(dest, src, len);
+    strncat(dest, src, len - 1);
     *first = 0;
 }
 
@@ -287,7 +287,7 @@ void ompi_info_do_config(bool want_all)
 
     if (OMPI_FORTRAN_HAVE_IGNORE_TKR) {
         /* OMPI_FORTRAN_IGNORE_TKR_PREDECL is already in quotes; it
-           didn't work consistently to put it in _STRINGIFY because
+           didn't work consistently to put it in PLATFORM_STRINGIFY because
            sometimes the compiler would actually interpret the pragma
            in there before stringify-ing it. */
         (void)opal_asprintf(&fortran_have_ignore_tkr, "yes (%s)",
@@ -336,9 +336,9 @@ void ompi_info_do_config(bool want_all)
     opal_info_out("C compiler absolute", "compiler:c:absolute",
                   OPAL_CC_ABSOLUTE);
     opal_info_out("C compiler family name", "compiler:c:familyname",
-                  _STRINGIFY(OPAL_BUILD_PLATFORM_COMPILER_FAMILYNAME));
+                  PLATFORM_STRINGIFY(PLATFORM_COMPILER_FAMILYNAME));
     opal_info_out("C compiler version", "compiler:c:version",
-                  _STRINGIFY(OPAL_BUILD_PLATFORM_COMPILER_VERSION_STR));
+                  PLATFORM_COMPILER_VERSION_STR);
 
     if (want_all) {
         opal_info_out_int("C char size", "compiler:c:sizeof:char", sizeof(char));
@@ -608,16 +608,16 @@ void ompi_info_do_config(bool want_all)
         opal_info_out("Build LDFLAGS", "option:build:ldflags", OMPI_BUILD_LDFLAGS);
         opal_info_out("Build LIBS", "option:build:libs", OMPI_BUILD_LIBS);
 
-        opal_info_out("Wrapper extra CFLAGS", "option:wrapper:extra_cflags",
-                      WRAPPER_EXTRA_CFLAGS);
-        opal_info_out("Wrapper extra CXXFLAGS", "option:wrapper:extra_cxxflags",
-                      WRAPPER_EXTRA_CXXFLAGS);
-        opal_info_out("Wrapper extra FCFLAGS", "option:wrapper:extra_fcflags",
-                      WRAPPER_EXTRA_FCFLAGS);
-        opal_info_out("Wrapper extra LDFLAGS", "option:wrapper:extra_ldflags",
-                      WRAPPER_EXTRA_LDFLAGS);
-        opal_info_out("Wrapper extra LIBS", "option:wrapper:extra_libs",
-                      WRAPPER_EXTRA_LIBS);
+        opal_info_out("Wrapper CFLAGS", "option:wrapper:cflags",
+                      OMPI_WRAPPER_CFLAGS);
+        opal_info_out("Wrapper CXXFLAGS", "option:wrapper:cxxflags",
+                      OMPI_WRAPPER_CXXFLAGS);
+        opal_info_out("Wrapper FCFLAGS", "option:wrapper:fcflags",
+                      OMPI_WRAPPER_FCFLAGS);
+        opal_info_out("Wrapper LDFLAGS", "option:wrapper:ldflags",
+                      OMPI_WRAPPER_LDFLAGS);
+        opal_info_out("Wrapper LIBS", "option:wrapper:libs",
+                      OMPI_WRAPPER_LIBS);
     }
 
     opal_info_out("Internal debug support", "option:debug", debug);
diff --git a/ompi/tools/wrappers/Makefile.am b/ompi/tools/wrappers/Makefile.am
index 933eb3d7620..1d5b24a9372 100644
--- a/ompi/tools/wrappers/Makefile.am
+++ b/ompi/tools/wrappers/Makefile.am
@@ -9,7 +9,7 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2006-2014 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2006-2022 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2012      Los Alamos National Security, LLC.  All rights reserved.
 # Copyright (c) 2013      Intel, Inc. All rights reserved.
 # Copyright (c) 2014      Research Organization for Information Science
@@ -22,17 +22,20 @@
 # $HEADER$
 #
 
-include $(top_srcdir)/Makefile.ompi-rules
-
-generated_man_pages = mpicc.1 mpic++.1 mpicxx.1 mpifort.1 mpif77.1 mpif90.1
+if OPAL_INSTALL_BINARIES
+pkgconfigdir = $(libdir)/pkgconfig
+pkgconfig_DATA = ompi.pc ompi-c.pc
 
-if OMPI_WANT_JAVA_BINDINGS
-generated_man_pages += mpijavac.1
+if OMPI_HAVE_CXX_COMPILER
+pkgconfig_DATA += ompi-cxx.pc
 endif
 
-man_pages = $(generated_man_pages)
+if OMPI_HAVE_FORTRAN_COMPILER
+pkgconfig_DATA += ompi-fort.pc
+endif
+endif # OPAL_INSTALL_BINARIES
 
-EXTRA_DIST = mpif77.1in mpijavac.1in mpijavac.pl.in
+#-----------------
 
 if OPAL_WANT_SCRIPT_WRAPPER_COMPILERS
 
@@ -45,11 +48,15 @@ CLEANFILES += $(bin_SCRIPTS)
 install-exec-hook-always:
 	test -z "$(bindir)" || $(MKDIR_P) "$(DESTDIR)$(bindir)"
 	(cd $(DESTDIR)$(bindir); rm -f mpicc; $(LN_S) ompi_wrapper_script mpicc)
+if OMPI_HAVE_CXX_COMPILER
 	(cd $(DESTDIR)$(bindir); rm -f mpic++; $(LN_S) ompi_wrapper_script mpic++)
 	(cd $(DESTDIR)$(bindir); rm -f mpicxx; $(LN_S) ompi_wrapper_script mpicxx)
+endif
+if OMPI_HAVE_FORTRAN_COMPILER
 	(cd $(DESTDIR)$(bindir); rm -f mpifort; $(LN_S) ompi_wrapper_script mpifort)
 	(cd $(DESTDIR)$(bindir); rm -f mpif77; $(LN_S) ompi_wrapper_script mpif77)
 	(cd $(DESTDIR)$(bindir); rm -f mpif90; $(LN_S) ompi_wrapper_script mpif90)
+endif
 if OMPI_WANT_JAVA_BINDINGS
 	(cp mpijavac.pl $(DESTDIR)$(bindir))
 	(cd $(DESTDIR)$(bindir); chmod +x mpijavac.pl; rm -f mpijavac; $(LN_S) mpijavac.pl mpijavac)
@@ -64,7 +71,7 @@ uninstall-local-always:
 	$(DESTDIR)$(bindir)/mpif90 \
 	$(DESTDIR)$(bindir)/mpijavac
 
-if CASE_SENSITIVE_FS
+if CASE_SENSITIVE_FS_AND_HAVE_CXX_COMPILER
 install-exec-hook: install-exec-hook-always
 	(cd $(DESTDIR)$(bindir); rm -f mpiCC; $(LN_S) ompi_wrapper_script mpiCC)
 uninstall-local: uninstall-local-always
@@ -78,39 +85,47 @@ else # OPAL_WANT_SCRIPT_WRAPPER_COMPILERS
 
 if OPAL_INSTALL_BINARIES
 
-pkgconfigdir = $(libdir)/pkgconfig
-pkgconfig_DATA = ompi.pc ompi-c.pc ompi-cxx.pc ompi-fort.pc
-
 if OMPI_WANT_JAVA_BINDINGS
 bin_SCRIPTS = mpijavac.pl
 endif
 
-man_MANS = $(man_pages)
+nodist_ompidata_DATA = mpicc-wrapper-data.txt
 
-nodist_ompidata_DATA = \
-        mpicc-wrapper-data.txt \
-	mpic++-wrapper-data.txt \
-	mpifort-wrapper-data.txt
+if OMPI_HAVE_CXX_COMPILER
+nodist_ompidata_DATA += mpic++-wrapper-data.txt
+endif
+
+if OMPI_HAVE_FORTRAN_COMPILER
+nodist_ompidata_DATA += mpifort-wrapper-data.txt
+endif
 
 install-exec-hook-always:
 	test -z "$(bindir)" || $(MKDIR_P) "$(DESTDIR)$(bindir)"
 	(cd $(DESTDIR)$(bindir); rm -f mpicc$(EXEEXT); $(LN_S) opal_wrapper$(EXEEXT) mpicc$(EXEEXT))
+if OMPI_HAVE_CXX_COMPILER
 	(cd $(DESTDIR)$(bindir); rm -f mpic++$(EXEEXT); $(LN_S) opal_wrapper$(EXEEXT) mpic++$(EXEEXT))
 	(cd $(DESTDIR)$(bindir); rm -f mpicxx$(EXEEXT); $(LN_S) opal_wrapper$(EXEEXT) mpicxx$(EXEEXT))
+endif
+if OMPI_HAVE_FORTRAN_COMPILER
 	(cd $(DESTDIR)$(bindir); rm -f mpifort$(EXEEXT); $(LN_S) opal_wrapper$(EXEEXT) mpifort$(EXEEXT))
 	(cd $(DESTDIR)$(bindir); rm -f mpif77$(EXEEXT); $(LN_S) opal_wrapper$(EXEEXT) mpif77$(EXEEXT))
 	(cd $(DESTDIR)$(bindir); rm -f mpif90$(EXEEXT); $(LN_S) opal_wrapper$(EXEEXT) mpif90$(EXEEXT))
+endif
 if OMPI_WANT_JAVA_BINDINGS
 	(cp mpijavac.pl $(DESTDIR)$(bindir))
 	(cd $(DESTDIR)$(bindir); chmod +x mpijavac.pl; rm -f mpijavac; $(LN_S) mpijavac.pl mpijavac)
 endif
 
 install-data-hook-always:
+if OMPI_HAVE_CXX_COMPILER
 	(cd $(DESTDIR)$(pkgdatadir); rm -f mpicxx-wrapper-data.txt; $(LN_S) mpic++-wrapper-data.txt mpicxx-wrapper-data.txt)
+endif
+if OMPI_HAVE_FORTRAN_COMPILER
 	(cd $(DESTDIR)$(pkgdatadir); rm -f mpif77-wrapper-data.txt; $(LN_S) mpifort-wrapper-data.txt mpif77-wrapper-data.txt)
 	(cd $(DESTDIR)$(pkgdatadir); rm -f mpif90-wrapper-data.txt; $(LN_S) mpifort-wrapper-data.txt mpif90-wrapper-data.txt)
 	(cd $(DESTDIR)$(pkgconfigdir); rm -f ompi-f77.pc; $(LN_S) ompi-fort.pc ompi-f77.pc)
 	(cd $(DESTDIR)$(pkgconfigdir); rm -f ompi-f90.pc; $(LN_S) ompi-fort.pc ompi-f90.pc)
+endif
 
 uninstall-local-always:
 	rm -f $(DESTDIR)$(bindir)/mpicc$(EXEEXT) \
@@ -128,17 +143,15 @@ uninstall-local-always:
 	$(DESTDIR)$(bindir)/mpijavac \
 	$(DESTDIR)$(bindir)/mpijavac.pl
 
-if CASE_SENSITIVE_FS
+if CASE_SENSITIVE_FS_AND_HAVE_CXX_COMPILER
 install-exec-hook: install-exec-hook-always
 	(cd $(DESTDIR)$(bindir); rm -f mpiCC$(EXEEXT); $(LN_S) opal_wrapper$(EXEEXT) mpiCC$(EXEEXT))
 
 install-data-hook: install-data-hook-always
 	(cd $(DESTDIR)$(pkgdatadir); rm -f mpiCC-wrapper-data.txt; $(LN_S) mpic++-wrapper-data.txt mpiCC-wrapper-data.txt)
-	(cd $(DESTDIR)$(mandir)/man1; rm -f mpiCC.1; $(LN_S) mpic++.1 mpiCC.1)
 
 uninstall-local: uninstall-local-always
 	rm -f $(DESTDIR)$(bindir)/mpiCC$(EXEEXT) \
-	$(DESTDIR)$(mandir)/man1/mpiCC.1 \
 	$(DESTDIR)$(pkgdatadir)/mpiCC-wrapper-data.txt
 
 else # CASE_SENSITIVE_FS
@@ -151,47 +164,3 @@ endif # CASE_SENSITIVE_FS
 endif # OPAL_INSTALL_BINARIES
 
 endif # OPAL_WANT_SCRIPT_WRAPPER_COMPILERS
-
-
-########################################################
-#
-# Man page generation / handling
-#
-########################################################
-distclean-local:
-	rm -f $(generated_man_pages)
-
-$(top_builddir)/opal/tools/wrappers/generic_wrapper.1:
-	(cd $(top_builddir)/opal/tools/wrappers && $(MAKE) $(AM_MAKEFLAGS) generic_wrapper.1)
-
-mpicc.1: $(top_builddir)/opal/tools/wrappers/generic_wrapper.1
-	rm -f mpicc.1
-	sed -e 's/#COMMAND#/mpicc/g' -e 's/#PROJECT#/Open MPI/g' -e 's/#PROJECT_SHORT#/OMPI/g' -e 's/#LANGUAGE#/C/g' < $(top_builddir)/opal/tools/wrappers/generic_wrapper.1 > mpicc.1
-
-mpic++.1: $(top_builddir)/opal/tools/wrappers/generic_wrapper.1
-	rm -f mpic++.1
-	sed -e 's/#COMMAND#/mpic++/g' -e 's/#PROJECT#/Open MPI/g' -e 's/#PROJECT_SHORT#/OMPI/g' -e 's/#LANGUAGE#/C++/g' < $(top_builddir)/opal/tools/wrappers/generic_wrapper.1 > mpic++.1
-
-mpicxx.1: $(top_builddir)/opal/tools/wrappers/generic_wrapper.1
-	rm -f mpicxx.1
-	sed -e 's/#COMMAND#/mpicxx/g' -e 's/#PROJECT#/Open MPI/g' -e 's/#PROJECT_SHORT#/OMPI/g' -e 's/#LANGUAGE#/C++/g' < $(top_builddir)/opal/tools/wrappers/generic_wrapper.1 > mpicxx.1
-
-mpifort.1: $(top_builddir)/opal/tools/wrappers/generic_wrapper.1
-	rm -f mpifort.1
-	sed -e 's/#COMMAND#/mpifort/g' -e 's/#PROJECT#/Open MPI/g' -e 's/#PROJECT_SHORT#/OMPI/g' -e 's/#LANGUAGE#/Fortran/g' < $(top_builddir)/opal/tools/wrappers/generic_wrapper.1 > mpifort.1
-
-# Deprecated
-mpif77.1: mpif77.1in
-	rm -f mpif77.1
-	sed -e 's/#PROJECT#/Open MPI/g' \
-	    -e 's/#PROJECT_SHORT#/OMPI/g' \
-	    -e 's/#LANGUAGE#/Fortran/g' \
-	    -e 's/#PACKAGE_NAME#/@PACKAGE_NAME@/g' \
-	    -e 's/#PACKAGE_VERSION#/@PACKAGE_VERSION@/g' \
-	    -e 's/#OMPI_DATE#/@OMPI_RELEASE_DATE@/g' \
-	    < $(top_srcdir)/ompi/tools/wrappers/mpif77.1in > mpif77.1
-
-# Deprecated
-mpif90.1: mpif77.1
-	@ rm -f mpif90.1
-	$(OMPI_V_LN_S) $(LN_S) mpif77.1 mpif90.1
diff --git a/ompi/tools/wrappers/mpic++-wrapper-data.txt.in b/ompi/tools/wrappers/mpic++-wrapper-data.txt.in
index 0b27e6ae34e..79d6ddb6a5a 100644
--- a/ompi/tools/wrappers/mpic++-wrapper-data.txt.in
+++ b/ompi/tools/wrappers/mpic++-wrapper-data.txt.in
@@ -12,16 +12,13 @@ language=C++
 compiler_env=CXX
 compiler_flags_env=CXXFLAGS
 compiler=@CXX@
-preprocessor_flags=@OMPI_WRAPPER_EXTRA_CPPFLAGS@
-compiler_flags_prefix=@OMPI_WRAPPER_EXTRA_CXXFLAGS_PREFIX@
-compiler_flags=@OMPI_WRAPPER_EXTRA_CXXFLAGS@
-linker_flags=@OMPI_WRAPPER_EXTRA_LDFLAGS@
-# Note that per https://svn.open-mpi.org/trac/ompi/ticket/3422, we
-# intentionally only link in the MPI libraries (OPAL, etc. are
-# pulled in implicitly) because we intend MPI applications to only use
-# the MPI API.
-libs=@OMPI_WRAPPER_CXX_LIB@ -l@OMPI_LIBMPI_NAME@
-libs_static=@OMPI_WRAPPER_CXX_LIB@ -l@OMPI_LIBMPI_NAME@ -l@OPAL_LIB_PREFIX@open-pal @OMPI_WRAPPER_EXTRA_LIBS@
+preprocessor_flags=@OMPI_WRAPPER_CPPFLAGS@
+compiler_flags_prefix=@OMPI_WRAPPER_CXXFLAGS_PREFIX@
+compiler_flags=@OMPI_WRAPPER_CXXFLAGS@
+linker_flags=@OMPI_WRAPPER_LDFLAGS@
+linker_flags_static=@OMPI_WRAPPER_LDFLAGS_STATIC@
+libs=@OMPI_WRAPPER_LIBS@
+libs_static=@OMPI_WRAPPER_LIBS_STATIC@
 dyn_lib_file=lib@OMPI_LIBMPI_NAME@.@OPAL_DYN_LIB_SUFFIX@
 static_lib_file=lib@OMPI_LIBMPI_NAME@.a
 required_file=@OMPI_WRAPPER_CXX_REQUIRED_FILE@
diff --git a/ompi/tools/wrappers/mpicc-wrapper-data.txt.in b/ompi/tools/wrappers/mpicc-wrapper-data.txt.in
index dcbf10caad2..75cc53f101a 100644
--- a/ompi/tools/wrappers/mpicc-wrapper-data.txt.in
+++ b/ompi/tools/wrappers/mpicc-wrapper-data.txt.in
@@ -12,16 +12,13 @@ language=C
 compiler_env=CC
 compiler_flags_env=CFLAGS
 compiler=@WRAPPER_CC@
-preprocessor_flags=@OMPI_WRAPPER_EXTRA_CPPFLAGS@
-compiler_flags_prefix=@OMPI_WRAPPER_EXTRA_CFLAGS_PREFIX@
-compiler_flags=@OMPI_WRAPPER_EXTRA_CFLAGS@
-linker_flags=@OMPI_WRAPPER_EXTRA_LDFLAGS@
-# Note that per https://svn.open-mpi.org/trac/ompi/ticket/3422, we
-# intentionally only link in the MPI libraries (OPAL, etc. are
-# pulled in implicitly) because we intend MPI applications to only use
-# the MPI API.
-libs=-l@OMPI_LIBMPI_NAME@
-libs_static=-l@OMPI_LIBMPI_NAME@ -l@OPAL_LIB_PREFIX@open-pal @OMPI_WRAPPER_EXTRA_LIBS@
+preprocessor_flags=@OMPI_WRAPPER_CPPFLAGS@
+compiler_flags_prefix=@OMPI_WRAPPER_CFLAGS_PREFIX@
+compiler_flags=@OMPI_WRAPPER_CFLAGS@
+linker_flags=@OMPI_WRAPPER_LDFLAGS@
+linker_flags_static=@OMPI_WRAPPER_LDFLAGS_STATIC@
+libs=@OMPI_WRAPPER_LIBS@
+libs_static=@OMPI_WRAPPER_LIBS_STATIC@
 dyn_lib_file=lib@OMPI_LIBMPI_NAME@.@OPAL_DYN_LIB_SUFFIX@
 static_lib_file=lib@OMPI_LIBMPI_NAME@.a
 required_file=
diff --git a/ompi/tools/wrappers/mpif77.1in b/ompi/tools/wrappers/mpif77.1in
deleted file mode 100644
index 2653305f10f..00000000000
--- a/ompi/tools/wrappers/mpif77.1in
+++ /dev/null
@@ -1,48 +0,0 @@
-.\" Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
-.\" Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
-.TH "mpif77,mpif90" 1 "#OMPI_DATE#" "#PACKAGE_VERSION#" "Open MPI"
-.
-.SH NAME
-mpif77, mpif90 -- Deprecated Open MPI Fortran wrapper compilers
-.
-.SH SYNTAX
-mpif90 ...
-.
-.\" **************************
-.\"    Description Section
-.\" **************************
-.
-.SH DESCRIPTION
-.PP
-The
-.I mpif77
-and
-.I mpif90
-wrapper compiler names are deprecated, and will disappear in a future
-version of Open MPI.  You should use the
-.I mpifort
-wrapper compiler, instead.  While they are deprecated,
-.I mpif77
-and
-.I mpif90
-accept all the same parameters as
-.IR mpifort ,
-and behaves the same as
-.IR mpifort .
-.
-.PP
-With
-.IR mpifort ,
-you can compile any Fortran program that uses the "mpif.h", "use mpi",
-and/or "use mpi_f08" MPI Fortran interfaces.
-.
-.PP
-See mpifort(1) for more details.
-.
-.\" **************************
-.\"    See Also Section
-.\" **************************
-.
-.SH SEE ALSO
-mpifort(1)
-.
diff --git a/ompi/tools/wrappers/mpifort-wrapper-data.txt.in b/ompi/tools/wrappers/mpifort-wrapper-data.txt.in
index d189536ba05..e2dc2581f65 100644
--- a/ompi/tools/wrappers/mpifort-wrapper-data.txt.in
+++ b/ompi/tools/wrappers/mpifort-wrapper-data.txt.in
@@ -13,14 +13,12 @@ compiler_env=FC
 compiler_flags_env=FCFLAGS
 compiler=@FC@
 preprocessor_flags=
-compiler_flags=@OMPI_WRAPPER_EXTRA_FCFLAGS@
-linker_flags=@OMPI_WRAPPER_EXTRA_FC_LDFLAGS@
-# Note that per https://svn.open-mpi.org/trac/ompi/ticket/3422, we
-# intentionally only link in the MPI libraries (OPAL, etc. are
-# pulled in implicitly) because we intend MPI applications to only use
-# the MPI API.
-libs=@OMPI_FORTRAN_USEMPIF08_LIB@ @OMPI_FORTRAN_USEMPI_LIB@ -l@OMPI_LIBMPI_NAME@_mpifh -l@OMPI_LIBMPI_NAME@
-libs_static=@OMPI_FORTRAN_USEMPIF08_LIB@ @OMPI_FORTRAN_USEMPI_LIB@ -l@OMPI_LIBMPI_NAME@_mpifh -l@OMPI_LIBMPI_NAME@ -l@OPAL_LIB_PREFIX@open-pal  @OMPI_WRAPPER_EXTRA_LIBS@
+compiler_flags_prefix=@OMPI_WRAPPER_FCFLAGS_PREFIX@
+compiler_flags=@OMPI_WRAPPER_FCFLAGS@
+linker_flags=@OMPI_WRAPPER_FC_LDFLAGS@
+linker_flags_static=@OMPI_WRAPPER_FC_LDFLAGS_STATIC@
+libs=@OMPI_WRAPPER_FC_LIBS@
+libs_static=@OMPI_WRAPPER_FC_LIBS_STATIC@
 dyn_lib_file=lib@OMPI_LIBMPI_NAME@.@OPAL_DYN_LIB_SUFFIX@
 static_lib_file=lib@OMPI_LIBMPI_NAME@.a
 required_file=@OMPI_WRAPPER_FORTRAN_REQUIRED_FILE@
diff --git a/ompi/tools/wrappers/mpijavac.1in b/ompi/tools/wrappers/mpijavac.1in
deleted file mode 100644
index e95016e6aa5..00000000000
--- a/ompi/tools/wrappers/mpijavac.1in
+++ /dev/null
@@ -1,147 +0,0 @@
-.\" Copyright (c) 2012      Los Alamos National Security, LLC.  All rights reserved.
-.\" Copyright (c) 2017      FUJITSU LIMITED.  All rights reserved.
-.TH mpijava 1 "#OPAL_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.
-.SH NAME
-mpijava -- Open MPI Java wrapper compiler
-.
-.SH SYNTAX
-mpijava [-showme|-showme:compile|-showme:link] ...
-.
-.SH OPTIONS
-.TP
---showme
-This option comes in several different variants (see below).  None of
-the variants invokes the underlying compiler; they all provide
-information on how the underlying compiler would have been invoked had
-.I --showme
-not been used.
-The basic
-.I --showme
-option outputs the command line that would be executed to compile the
-program.  \fBNOTE:\fR If a non-filename argument is passed on the
-command line, the \fI-showme\fR option will \fInot\fR display any
-additional flags.  For example, both "mpijava --showme" and
-"mpijava --showme my_source.java" will show all the wrapper-supplied
-flags.  But "mpijava --showme -v" will only show the underlying
-compiler name and "-v".
-.TP
---showme:compile
-Output the compiler flags that would have been supplied to the
-java compiler.
-.TP
---showme:link
-Output the linker flags that would have been supplied to the
-java compiler.
-.TP
---showme:command
-Outputs the underlying java compiler command (which may be one
-or more tokens).
-.TP
---showme:incdirs
-Outputs a space-delimited (but otherwise undecorated) list of
-directories that the wrapper compiler would have provided to the
-underlying java compiler to indicate where relevant header files
-are located.
-.TP
---showme:libdirs
-Outputs a space-delimited (but otherwise undecorated) list of
-directories that the wrapper compiler would have provided to the
-underlying linker to indicate where relevant libraries are located.
-.TP
---showme:libs
-Outputs a space-delimited (but otherwise undecorated) list of library
-names that the wrapper compiler would have used to link an
-application.  For example: "mpi open-rte open-pal util".
-.TP
---showme:version
-Outputs the version number of Open MPI.
-.PP
-See the man page for your underlying java compiler for other
-options that can be passed through mpijava.
-.
-.
-.SH DESCRIPTION
-.PP
-Conceptually, the role of these commands is quite simple:
-transparently add relevant compiler and linker flags to the user's
-command line that are necessary to compile / link Open MPI
-programs, and then invoke the underlying compiler to actually perform
-the command.
-.
-.PP
-As such, these commands are frequently referred to as "wrapper"
-compilers because they do not actually compile or link applications
-themselves; they only add in command line flags and invoke the
-back-end compiler.
-.
-.
-.SS Overview
-\fImpijava\fR is a convenience wrapper for the underlying
-java compiler.  Translation of an Open MPI program requires the
-linkage of the Open MPI-specific libraries which may not reside in
-one of the standard search directories of ld(1).  It also often
-requires the inclusion of header files what may also not be found in a
-standard location.
-.
-.PP
-\fImpijava\fR passes its arguments to the underlying java
-compiler along with the -I, -L and -l options required by Open MPI
-programs.
-.
-.PP
-The Open MPI Team \fIstrongly\fR encourages using the wrapper
-compilers instead of attempting to link to the Open MPI libraries
-manually.  This allows the specific implementation of Open MPI to
-change without forcing changes to linker directives in users'
-Makefiles.  Indeed, the specific set of flags and libraries used by
-the wrapper compilers depends on how Open MPI was configured and
-built; the values can change between different installations of the
-same version of Open MPI.
-.
-.PP
-Indeed, since the wrappers are simply thin shells on top of an
-underlying compiler, there are very, very few compelling reasons
-\fInot\fR to use \fImpijava\fR.  When it is not possible to use the
-wrappers directly, the \fI-showme:compile\fR and \fI-showme:link\fR
-options should be used to determine what flags the wrappers would have
-used.
-.
-.
-.SH NOTES
-.PP
-It is possible to make the wrapper compilers multi-lib aware.  That
-is, the libraries and includes specified may differ based on the
-compiler flags specified (for example, with the GNU compilers on
-Linux, a different library path may be used if -m32 is seen versus
--m64 being seen).  This is not the default behavior in a standard
-build, but can be activated (for example, in a binary package
-providing both 32 and 64 bit support).  More information can be found
-at:
-.PP
-  https://svn.open-mpi.org/trac/ompi/wiki/compilerwrapper3264
-.
-.
-.SH FILES
-.PP
-The string that the wrapper compilers insert into the command line
-before invoking the underlying compiler are stored in a text file
-created by Open MPI and installed to
-\fI$pkgdata/mpijava-wrapper-data.txt\fR, where \fI$pkgdata\fR
-is typically \fI$prefix/share/openmpi\fR, and \fI$prefix\fR is the top
-installation directory of Open MPI.
-.
-.PP
-It is rarely necessary to edit this file, but it can be examined to
-gain insight into what flags the wrappers are placing on the command
-line.
-.
-.
-.SH ENVIRONMENT VARIABLES
-.PP
-By default, the wrappers use the compilers that were selected when
-Open MPI was configured.  These compilers were either found
-automatically by Open MPI's "configure" script, or were selected by
-the user in the CC, CXX, F77, JAVAC, and/or FC environment variables
-before "configure" was invoked.  Additionally, other arguments
-specific to the compiler may have been selected by configure.
diff --git a/ompi/tools/wrappers/ompi-c.pc.in b/ompi/tools/wrappers/ompi-c.pc.in
index 3f7c50efee4..4ab7eb97e2c 100644
--- a/ompi/tools/wrappers/ompi-c.pc.in
+++ b/ompi/tools/wrappers/ompi-c.pc.in
@@ -11,12 +11,9 @@ exec_prefix=@exec_prefix@
 includedir=@includedir@
 libdir=@libdir@
 pkgincludedir=@opalincludedir@
-#
-# Note that -lopen-pal and all the EXTRA_LIBS are only necessary when
-# static linking (they're pulled in by libopen-rte.so's implicit
-# dependencies), so only list these in Libs.private.
-#
-Libs: -L${libdir} @OMPI_PKG_CONFIG_LDFLAGS@ -l@OMPI_LIBMPI_NAME@
-Libs.private: -lopen-rte -lopen-pal @OMPI_WRAPPER_EXTRA_LIBS@
-#
-Cflags: -I${includedir} @OMPI_WRAPPER_EXTRA_CPPFLAGS@ @OMPI_WRAPPER_EXTRA_CFLAGS@
+
+Libs: @OMPI_PC_LIBS@
+Libs.private: @OMPI_PC_LIBS_PRIVATE@
+Cflags: @OMPI_PC_CFLAGS@
+Requires: @OMPI_PC_MODULES@
+Requires.private: @OMPI_PC_MODULES_PRIVATE@
diff --git a/ompi/tools/wrappers/ompi-cxx.pc.in b/ompi/tools/wrappers/ompi-cxx.pc.in
index b1f80a656b7..4ab7eb97e2c 100644
--- a/ompi/tools/wrappers/ompi-cxx.pc.in
+++ b/ompi/tools/wrappers/ompi-cxx.pc.in
@@ -11,12 +11,9 @@ exec_prefix=@exec_prefix@
 includedir=@includedir@
 libdir=@libdir@
 pkgincludedir=@opalincludedir@
-#
-# Note that -lopen-pal and all the EXTRA_LIBS are only necessary when
-# static linking (they're pulled in by libopen-rte.so's implicit
-# dependencies), so only list these in Libs.private.
-#
-Libs: -L${libdir} @OMPI_PKG_CONFIG_LDFLAGS@ @OMPI_WRAPPER_CXX_LIB@ -l@OMPI_LIBMPI_NAME@
-Libs.private: -lopen-rte -lopen-pal @OMPI_WRAPPER_EXTRA_LIBS@
-#
-Cflags: -I${includedir} @OMPI_WRAPPER_EXTRA_CPPFLAGS@ @OMPI_WRAPPER_EXTRA_CXXFLAGS@
+
+Libs: @OMPI_PC_LIBS@
+Libs.private: @OMPI_PC_LIBS_PRIVATE@
+Cflags: @OMPI_PC_CFLAGS@
+Requires: @OMPI_PC_MODULES@
+Requires.private: @OMPI_PC_MODULES_PRIVATE@
diff --git a/ompi/tools/wrappers/ompi-fort.pc.in b/ompi/tools/wrappers/ompi-fort.pc.in
index a40ce57668b..233641c8a7b 100644
--- a/ompi/tools/wrappers/ompi-fort.pc.in
+++ b/ompi/tools/wrappers/ompi-fort.pc.in
@@ -11,11 +11,9 @@ exec_prefix=@exec_prefix@
 includedir=@includedir@
 libdir=@libdir@
 pkgincludedir=@opalincludedir@
-#
-# Note that -lopen-pal and all the EXTRA_LIBS are only necessary when
-# static linking (they're pulled in by libopen-rte.so's implicit
-# dependencies), so only list these in Libs.private.
-#
-Libs: -L${libdir} @OMPI_PKG_CONFIG_LDFLAGS@ @OMPI_FORTRAN_USEMPIF08_LIB@ @OMPI_FORTRAN_USEMPI_LIB@ -l@OMPI_LIBMPI_NAME@_mpifh -l@OMPI_LIBMPI_NAME@
-Libs.private: -lopen-rte -lopen-pal @OMPI_WRAPPER_EXTRA_LIBS@
-Cflags: -I${includedir} @OMPI_WRAPPER_EXTRA_CPPFLAGS@ @OMPI_WRAPPER_EXTRA_FCFLAGS@
+
+Libs: @OMPI_PC_FC_LIBS@
+Libs.private: @OMPI_PC_FC_LIBS_PRIVATE@
+Cflags: @OMPI_PC_FC_CFLAGS@
+Requires: @OMPI_PC_MODULES@
+Requires.private: @OMPI_PC_MODULES_PRIVATE@
diff --git a/ompi/tools/wrappers/ompi.pc.in b/ompi/tools/wrappers/ompi.pc.in
index 3b9649c50d2..4ab7eb97e2c 100644
--- a/ompi/tools/wrappers/ompi.pc.in
+++ b/ompi/tools/wrappers/ompi.pc.in
@@ -11,12 +11,9 @@ exec_prefix=@exec_prefix@
 includedir=@includedir@
 libdir=@libdir@
 pkgincludedir=@opalincludedir@
-#
-# Note that -lopen-pal and all the EXTRA_LIBS are only necessary when
-# static linking (they're pulled in by libopen-rte.so's implicit
-# dependencies), so only list these in Libs.private.
-#
-Libs: -L${libdir} @OMPI_PKG_CONFIG_LDFLAGS@ -l@OMPI_LIBMPI_NAME@
-Libs.private: @OMPI_WRAPPER_EXTRA_LIBS@
-#
-Cflags: -I${includedir} @OMPI_WRAPPER_EXTRA_CPPFLAGS@ @OMPI_WRAPPER_EXTRA_CFLAGS@
+
+Libs: @OMPI_PC_LIBS@
+Libs.private: @OMPI_PC_LIBS_PRIVATE@
+Cflags: @OMPI_PC_CFLAGS@
+Requires: @OMPI_PC_MODULES@
+Requires.private: @OMPI_PC_MODULES_PRIVATE@
diff --git a/ompi/tools/wrappers/ompi_wrapper_script.in b/ompi/tools/wrappers/ompi_wrapper_script.in
index 0822a5843c0..803e4b9092f 100644
--- a/ompi/tools/wrappers/ompi_wrapper_script.in
+++ b/ompi/tools/wrappers/ompi_wrapper_script.in
@@ -10,6 +10,7 @@
 # Copyright (c) 2016      IBM Corporation.  All rights reserved.
 # Copyright (c) 2016      Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -36,57 +37,48 @@ if (exists($ENV{'OPAL_DESTDIR'})
 my $CC = "@WRAPPER_CC@";
 my $CXX = "@CXX@";
 my $FC = "@FC@";
-my $extra_cppflags = "@OMPI_WRAPPER_EXTRA_CPPFLAGS@";
-my $extra_cflags = "@OMPI_WRAPPER_EXTRA_CFLAGS@";
-my $extra_cflags_prefix = "@OMPI_WRAPPER_EXTRA_CFLAGS_PREFIX@";
-my $extra_cxxflags = "@OMPI_WRAPPER_EXTRA_CXXFLAGS@";
-my $extra_cxxflags_prefix = "@OMPI_WRAPPER_EXTRA_CXXFLAGS_PREFIX@";
-my $extra_fcflags = "@OMPI_WRAPPER_EXTRA_FCFLAGS@";
-my $extra_fcflags_prefix = "@OMPI_WRAPPER_EXTRA_FCFLAGS_PREFIX@";
-my $extra_ldflags = "@OMPI_PKG_CONFIG_LDFLAGS@";
-my $extra_libs = "@OMPI_WRAPPER_EXTRA_LIBS@";
-my $cxx_lib = "@OMPI_WRAPPER_CXX_LIB@";
-my $fc_module_flag = "@OMPI_FC_MODULE_FLAG@";
-my $dynamic_lib_suffix = "@OPAL_DYN_LIB_SUFFIX@";
-my $fortran_libs = "@OMPI_FORTRAN_USEMPIF08_LIB@ @OMPI_FORTRAN_USEMPI_LIB@";
-my $ompi_libmpi_name = "@OMPI_LIBMPI_NAME@";
-
-# Someone might want to fix for windows
-my $include_flag = "-I";
-my $libdir_flag = "-L";
+my $wrapper_cppflags = "@OMPI_WRAPPER_CPPFLAGS@";
+my $wrapper_cflags = "@OMPI_WRAPPER_CFLAGS@";
+my $wrapper_cflags_prefix = "@OMPI_WRAPPER_CFLAGS_PREFIX@";
+my $wrapper_cxxflags = "@OMPI_WRAPPER_CXXFLAGS@";
+my $wrapper_cxxflags_prefix = "@OMPI_WRAPPER_CXXFLAGS_PREFIX@";
+my $wrapper_fcflags = "@OMPI_WRAPPER_FCFLAGS@";
+my $wrapper_fcflags_prefix = "@OMPI_WRAPPER_FCFLAGS_PREFIX@";
+my $wrapper_ldflags = "@OMPI_WRAPPER_LDFLAGS@";
+my $wrapper_ldflags_static = "@OMPI_WRAPPER_LDFLAGS_STATIC@";
+my $wrapper_libs = "@OMPI_WRAPPER_LIBS@";
+my $wrapper_libs_static = "@OMPI_WRAPPER_LIBS_STATIC@";
+my $wrapper_fc_ldflags = "@OMPI_WRAPPER_FC_LDFLAGS@";
+my $wrapper_fc_ldflags_static = "@OMPI_WRAPPER_FC_LDFLAGS_STATIC@";
+my $wrapper_fc_libs = "@OMPI_WRAPPER_FC_LIBS@";
+my $wrapper_fc_libs_static = "@OMPI_WRAPPER_FC_LIBS_STATIC@";
 
 my $lang = "none";
 my $comp = ""; # this is a sentinal from configure
-my $preproc_flags = $include_flag . $includedir;
+my $preproc_flags = "";
 my $comp_flags = "";
 my $comp_flags_prefix = "";
-my $linker_flags = $libdir_flag . $libdir . " " . $extra_ldflags;
-# Note that per https://svn.open-mpi.org/trac/ompi/ticket/3422, we
-# intentionally only link in the MPI libraries (ORTE, OPAL, etc. are
-# pulled in implicitly) because we intend MPI applications to only use
-# the MPI API.
-my $libs = "-l".$ompi_libmpi_name." " . $extra_libs;
-my $libs_static = "-l".$ompi_libmpi_name." -lopen-rte -lopen-pal " . $extra_libs;
-
-my $have_dynamic = 0;
-if (-e $libdir . "/lib".$ompi_libmpi_name."." . $dynamic_lib_suffix) {
-    $have_dynamic = 1;
-}
-my $have_static = 0;
-if (-e $libdir . "/lib".$ompi_libmpi_name.".a") {
-    $have_static = 1;
-}
+my $linker_flags = "";
+my $linker_flags_static = "";
+my $libs = "";
+my $libs_static = "";
 
 # run flags through regex to fix directories...
-$extra_cppflags =~ s/\$\{includedir\}/$includedir/g;
-$extra_cflags =~ s/\$\{includedir\}/$includedir/g;
-$extra_cflags_prefix =~ s/\$\{includedir\}/$includedir/g;
-$extra_cxxflags =~ s/\$\{includedir\}/$includedir/g;
-$extra_cxxflags_prefix =~ s/\$\{includedir\}/$includedir/g;
-$extra_fcflags =~ s/\$\{includedir\}/$includedir/g;
-$extra_fcflags_prefix =~ s/\$\{includedir\}/$includedir/g;
-$extra_ldflags =~ s/\$\{libdir\}/$libdir/g;
-$extra_libs =~ s/\$\{libdir\}/$libdir/g;
+$wrapper_cppflags =~ s/\$\{includedir\}/$includedir/g;
+$wrapper_cflags =~ s/\$\{includedir\}/$includedir/g;
+$wrapper_cflags_prefix =~ s/\$\{includedir\}/$includedir/g;
+$wrapper_cxxflags =~ s/\$\{includedir\}/$includedir/g;
+$wrapper_cxxflags_prefix =~ s/\$\{includedir\}/$includedir/g;
+$wrapper_fcflags =~ s/\$\{includedir\}/$includedir/g;
+$wrapper_fcflags_prefix =~ s/\$\{includedir\}/$includedir/g;
+$wrapper_ldflags =~ s/\$\{libdir\}/$libdir/g;
+$wrapper_ldflags_static =~ s/\$\{libdir\}/$libdir/g;
+$wrapper_libs =~ s/\$\{libdir\}/$libdir/g;
+$wrapper_libs_static =~ s/\$\{libdir\}/$libdir/g;
+$wrapper_fc_ldflags =~ s/\$\{libdir\}/$libdir/g;
+$wrapper_fc_ldflags_static =~ s/\$\{libdir\}/$libdir/g;
+$wrapper_fc_libs =~ s/\$\{libdir\}/$libdir/g;
+$wrapper_fc_libs_static =~ s/\$\{libdir\}/$libdir/g;
 
 sub check_env {
     my $envvar = shift;
@@ -107,27 +99,36 @@ sub check_env {
 if (basename($0) eq "mpicc") {
     $lang = "C";
     $comp = check_env("CC", $CC);
-    $preproc_flags .= " " . $extra_cppflags;
-    $comp_flags = $extra_cflags;
-    $comp_flags_prefix = $extra_cflags_prefix;
-    # no special libs for C
+    $preproc_flags = $wrapper_cppflags;
+    $comp_flags = $wrapper_cflags;
+    $comp_flags_prefix = $wrapper_cflags_prefix;
+    $linker_flags = $wrapper_ldflags;
+    $linker_flags_static = $wrapper_ldflags_static;
+    $libs = $wrapper_libs;
+    $libs_static = $wrapper_libs_static;
 } elsif (basename($0) eq "mpic++" || basename($0) eq "mpiCC" || basename($0) eq "mpicxx") {
     $lang = "C++";
     $comp = check_env("CXX", $CXX);
-    $preproc_flags .= " " . $extra_cppflags;
-    $comp_flags = $extra_cxxflags;
-    $comp_flags_prefix = $extra_cxxflags_prefix;
-    $libs = $cxx_lib . " " . $libs;
+    $preproc_flags = $wrapper_cppflags;
+    $comp_flags = $wrapper_cxxflags;
+    $comp_flags_prefix = $wrapper_cxxflags_prefix;
+    $linker_flags = $wrapper_ldflags;
+    $linker_flags_static = $wrapper_ldflags_static;
+    $libs = $wrapper_libs;
+    $libs_static = $wrapper_libs_static;
 }
 # mpifort is now preferred; mpif77/mpif90 are legacy names
 elsif (basename($0) eq "mpifort" ||
          basename($0) eq "mpif77" || basename($0) eq "mpif90") {
     $lang = "Fortran";
     $comp = check_env("FC", $FC);
-    # no extra includes for Fortran.
-    $comp_flags = $extra_fcflags;
-    $comp_flags_prefix = $extra_fcflags_prefix;
-    $libs = $fortran_libs . " -l".$ompi_libmpi_name."_mpifh " . $libs;
+    $preproc_flags = $wrapper_cppflags;
+    $comp_flags = $wrapper_fcflags;
+    $comp_flags_prefix = $wrapper_fcflags_prefix;
+    $linker_flags = $wrapper_fc_ldflags;
+    $linker_flags_static = $wrapper_fc_ldflags_static;
+    $libs = $wrapper_fc_libs;
+    $libs_static = $wrapper_fc_libs_static;
 }
 
 if ($lang eq "none") {
@@ -202,8 +203,9 @@ if ($disable_flags == 1 && !($dry_run == 1 && $real_flag == 0)) {
     $want_preproc = $want_compile = $want_link = 0;
 }
 
-if ($want_static == 1 || $have_dynamic == 0) {
+if ($want_static == 1) {
     $libs = $libs_static;
+    $linker_flags = $linker_flags_static;
 }
 
 my @exec_argv = ();
diff --git a/ompi/util/Makefile.am b/ompi/util/Makefile.am
index 45f01c77069..53a89a074eb 100644
--- a/ompi/util/Makefile.am
+++ b/ompi/util/Makefile.am
@@ -10,4 +10,5 @@
 
 # Source code files
 headers += \
-	util/timings.h
+	util/timings.h \
+	util/count_disp_array.h
diff --git a/ompi/util/count_disp_array.h b/ompi/util/count_disp_array.h
new file mode 100644
index 00000000000..f95d65dc858
--- /dev/null
+++ b/ompi/util/count_disp_array.h
@@ -0,0 +1,133 @@
+/*
+ * Copyright (c) 2024   Triad National Security, LLC. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#ifndef OMPI_UTIL_COUNT_DISP_ARRAY_H
+#define OMPI_UTIL_COUNT_DISP_ARRAY_H
+
+#include <stdlib.h>
+#include <stddef.h>
+#include <stdint.h>
+
+/*
+ * NOTE: This code chooses between 64-bit and 32-bit pointers by using the
+ *       least significant bit as a flag (which is possible since these
+ *       pointers will always be multiples of 4 or 8).
+ */
+typedef intptr_t ompi_count_array_t;
+
+/* Initialize an int variant of the count array */
+static inline void ompi_count_array_init(ompi_count_array_t *array, const int *data)
+{
+    *array = (intptr_t)data | 0x1L;
+}
+
+/* Initialize a bigcount variant of the count array */
+static inline void ompi_count_array_init_c(ompi_count_array_t *array, const size_t *data)
+{
+    *array = (intptr_t)data;
+}
+
+#if OPAL_C_HAVE__GENERIC
+#define OMPI_COUNT_ARRAY_INIT(array, data) _Generic((data), \
+                                                    int *: ompi_count_array_init, \
+                                                    const int *: ompi_count_array_init, \
+                                                    size_t *: ompi_count_array_init_c, \
+                                                    const size_t *: ompi_count_array_init_c, \
+                                                    const MPI_Count *: ompi_count_array_init_c)(array, (const void *) data)
+#else
+#define OMPI_COUNT_ARRAY_INIT(array, data) \
+    do { \
+        if (sizeof(*(data)) == sizeof(int)) { \
+            ompi_count_array_init(array, (const int *) (data)); \
+        } else if (sizeof(*(data)) == sizeof(size_t)) { \
+            ompi_count_array_init_c(array, (const size_t *) (data)); \
+        } \
+    } while (0)
+#endif
+
+/* Return if the internal type is 64-bit or not */
+static inline bool ompi_count_array_is_64bit(ompi_count_array_t array)
+{
+    return !(array & 0x1L) && sizeof(size_t) == 8;
+}
+
+static inline const void *ompi_count_array_ptr(ompi_count_array_t array)
+{
+    if (OPAL_LIKELY(array & 0x1L)){
+        return (const void *)(array & ~0x1L);
+    }
+    return (const void *) array;
+}
+
+/* Get a count in the array at index i */
+static inline size_t ompi_count_array_get(ompi_count_array_t array, size_t i)
+{
+    if (OPAL_LIKELY(array & 0x1L)){
+        const int *iptr = (const int *)(array & ~0x1L);
+        return iptr[i];
+    }
+    return ((const size_t *)array)[i];
+}
+
+typedef intptr_t ompi_disp_array_t;
+
+/* Initialize an int variant of the disp array */
+static inline void ompi_disp_array_init(ompi_disp_array_t *array, const int *data)
+{
+    *array = (intptr_t)data | 0x1L;
+}
+
+/* Initialize a bigcount variant of the disp array */
+static inline void ompi_disp_array_init_c(ompi_disp_array_t *array, const ptrdiff_t *data)
+{
+    *array = (intptr_t)data;
+}
+
+#if OPAL_C_HAVE__GENERIC
+#define OMPI_DISP_ARRAY_INIT(array, data) _Generic((data), \
+                                                   int *: ompi_disp_array_init, \
+                                                   const int *: ompi_disp_array_init, \
+                                                   ptrdiff_t *: ompi_disp_array_init_c, \
+                                                   const ptrdiff_t *: ompi_disp_array_init_c)(array, data)
+#else
+#define OMPI_DISP_ARRAY_INIT(array, data) \
+    do { \
+        if (sizeof(*(data)) == sizeof(int)) { \
+            ompi_disp_array_init(array, (const int *) (data)); \
+        } else if (sizeof(*(data)) == sizeof(ptrdiff_t)) { \
+            ompi_disp_array_init_c(array, (const ptrdiff_t *) (data)); \
+        } \
+    } while(0)
+#endif
+
+/* Return if the internal type is 64-bit or not */
+static inline bool ompi_disp_array_is_64bit(ompi_disp_array_t array)
+{
+    return !(array & 0x1L) && sizeof(ptrdiff_t) == 8;
+}
+
+/* Get a displacement in the array at index i */
+static inline ptrdiff_t ompi_disp_array_get(ompi_disp_array_t array, size_t i)
+{
+    if (OPAL_LIKELY(array & 0x1L)){
+        const int *iptr = (const int *)(array & ~0x1L);
+        return iptr[i];
+    }
+    return ((const ptrdiff_t *)array)[i];
+}
+
+/* Get a direct pointer to the data */
+static inline const void *ompi_disp_array_ptr(ompi_disp_array_t array)
+{
+    if (OPAL_LIKELY(array & 0x1L)){
+        return (const void *)(array & ~0x1L);
+    }
+    return (const void *)array;
+}
+
+#endif
diff --git a/ompi/util/timings.h b/ompi/util/timings.h
index b582466661e..55188f6563e 100644
--- a/ompi/util/timings.h
+++ b/ompi/util/timings.h
@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2017-2018 Mellanox Technologies Ltd. All rights reserved.
  * Copyright (c) 2017      Intel, Inc. All rights reserved.
+ * Copyright (c) 2021-2022 Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -12,7 +13,6 @@
 #define OMPI_UTIL_TIMING_H
 
 #include "opal/util/timings.h"
-/* TODO: we need access to MPI_* functions */
 
 #if (OPAL_ENABLE_TIMING)
 
@@ -166,16 +166,17 @@ typedef struct ompi_timing_t {
 #define OMPI_TIMING_OUT                                                           \
     do {                                                                          \
         if (OMPI_TIMING.enabled) {                                                \
-            int i, size, rank;                                                    \
-            MPI_Comm_size(MPI_COMM_WORLD, &size);                                 \
-            MPI_Comm_rank(MPI_COMM_WORLD, &rank);                                 \
-            int error = 0;                                                        \
+            int i;                                                                \
+            int size = ompi_comm_size(MPI_COMM_WORLD);                            \
+            int rank = ompi_comm_rank(MPI_COMM_WORLD);                            \
+            int timing_error = 0;                                                 \
             int imported = 0;                                                     \
                                                                                   \
-            MPI_Reduce(&OMPI_TIMING.error, &error, 1,                             \
-                MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);                             \
+            MPI_COMM_WORLD->c_coll->coll_reduce(&OMPI_TIMING.error, &timing_error, 1, \
+                                                MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD, \
+                                                MPI_COMM_WORLD->c_coll->coll_reduce_module); \
                                                                                   \
-            if (error) {                                                          \
+            if (timing_error) {                                                   \
                 if (0 == rank) {                                                  \
                     printf("==OMPI_TIMING== error: something went wrong, timings doesn't work\n"); \
                 }                                                                 \
@@ -196,12 +197,15 @@ typedef struct ompi_timing_t {
                     do {                                                          \
                         int use;                                                  \
                         for (use = 0; use < timing->use; use++) {                 \
-                            MPI_Reduce(&timing->val[use].ts, avg + i, 1,          \
-                                MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);          \
-                            MPI_Reduce(&timing->val[use].ts, min + i, 1,          \
-                                MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);          \
-                            MPI_Reduce(&timing->val[use].ts, max + i, 1,          \
-                                MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);          \
+                            MPI_COMM_WORLD->c_coll->coll_reduce(&timing->val[use].ts, avg + i, 1,         \
+                                                                MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD, \
+                                                                MPI_COMM_WORLD->c_coll->coll_reduce_module); \
+                            MPI_COMM_WORLD->c_coll->coll_reduce(&timing->val[use].ts, min + i, 1, \
+                                                                MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD, \
+                                                                MPI_COMM_WORLD->c_coll->coll_reduce_module); \
+                            MPI_COMM_WORLD->c_coll->coll_reduce(&timing->val[use].ts, max + i, 1, \
+                                                                MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD, \
+                                                                MPI_COMM_WORLD->c_coll->coll_reduce_module); \
                             desc[i] = timing->val[use].desc;                      \
                             prefix[i] = timing->val[use].prefix;                  \
                             file[i] = timing->val[use].file;                      \
diff --git a/ompi/win/win.c b/ompi/win/win.c
index b4bc150d893..87556a44c24 100644
--- a/ompi/win/win.c
+++ b/ompi/win/win.c
@@ -17,6 +17,9 @@
  * Copyright (c) 2015-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
+ * Copyright (c) 2018-2019 Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2024      Advanced Micro Devices, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -36,6 +39,7 @@
 #include "ompi/attribute/attribute.h"
 #include "ompi/group/group.h"
 #include "ompi/info/info.h"
+#include "ompi/info/info_memkind.h"
 #include "ompi/mca/osc/base/base.h"
 #include "ompi/mca/osc/osc.h"
 
@@ -52,13 +56,13 @@ ompi_predefined_win_t *ompi_mpi_win_null_addr = &ompi_mpi_win_null;
 mca_base_var_enum_t *ompi_win_accumulate_ops = NULL;
 mca_base_var_enum_flag_t *ompi_win_accumulate_order = NULL;
 
-static mca_base_var_enum_value_t accumulate_ops_values[] = {
+static const mca_base_var_enum_value_t accumulate_ops_values[] = {
     {.value = OMPI_WIN_ACCUMULATE_OPS_SAME_OP_NO_OP, .string = "same_op_no_op",},
     {.value = OMPI_WIN_ACCUMULATE_OPS_SAME_OP, .string = "same_op",},
     {.value = -1, .string = NULL},
 };
 
-static mca_base_var_enum_value_flag_t accumulate_order_flags[] = {
+static const mca_base_var_enum_value_flag_t accumulate_order_flags[] = {
     {.flag = OMPI_WIN_ACC_ORDER_NONE, .string = "none", .conflicting_flag = OMPI_WIN_ACC_ORDER_RAR |
      OMPI_WIN_ACC_ORDER_WAR | OMPI_WIN_ACC_ORDER_RAW | OMPI_WIN_ACC_ORDER_WAW},
     {.flag = OMPI_WIN_ACC_ORDER_RAR, .string = "rar", .conflicting_flag = OMPI_WIN_ACC_ORDER_NONE},
@@ -74,8 +78,40 @@ static void ompi_win_destruct(ompi_win_t *win);
 OBJ_CLASS_INSTANCE(ompi_win_t, opal_infosubscriber_t,
                    ompi_win_construct, ompi_win_destruct);
 
-int
-ompi_win_init(void)
+
+static void ompi_win_dump (ompi_win_t *win)
+{
+    opal_output(0, "Dumping information for window: %s\n", win->w_name);
+    opal_output(0,"  Fortran window handle: %d, window size: %d\n",
+                win->w_f_to_c_index, ompi_group_size (win->w_group));
+}
+
+static int ompi_win_finalize(void)
+{
+    size_t size = opal_pointer_array_get_size (&ompi_mpi_windows);
+    /* start at 1 to skip win null */
+    for (size_t i = 1 ; i < size ; ++i) {
+        ompi_win_t *win =
+            (ompi_win_t *) opal_pointer_array_get_item (&ompi_mpi_windows, i);
+        if (NULL != win) {
+            if (ompi_debug_show_handle_leaks && !ompi_win_invalid(win)){
+                opal_output(0,"WARNING: MPI_Win still allocated in MPI_Finalize\n");
+                ompi_win_dump (win);
+            }
+            ompi_win_free (win);
+        }
+    }
+
+    OBJ_DESTRUCT(&ompi_mpi_win_null.win);
+    OBJ_DESTRUCT(&ompi_mpi_windows);
+    OBJ_RELEASE(ompi_win_accumulate_ops);
+    OBJ_RELEASE(ompi_win_accumulate_order);
+
+    /* release a reference to the attributes subsys */
+    return ompi_attr_put_ref();
+}
+
+int ompi_win_init (void)
 {
     int ret;
 
@@ -106,36 +142,12 @@ ompi_win_init(void)
         return ret;
     }
 
-    return OMPI_SUCCESS;
-}
-
-static void ompi_win_dump (ompi_win_t *win)
-{
-    opal_output(0, "Dumping information for window: %s\n", win->w_name);
-    opal_output(0,"  Fortran window handle: %d, window size: %d\n",
-                win->w_f_to_c_index, ompi_group_size (win->w_group));
-}
-
-int ompi_win_finalize(void)
-{
-    size_t size = opal_pointer_array_get_size (&ompi_mpi_windows);
-    /* start at 1 to skip win null */
-    for (size_t i = 1 ; i < size ; ++i) {
-        ompi_win_t *win =
-            (ompi_win_t *) opal_pointer_array_get_item (&ompi_mpi_windows, i);
-        if (NULL != win) {
-            if (ompi_debug_show_handle_leaks && !ompi_win_invalid(win)){
-                opal_output(0,"WARNING: MPI_Win still allocated in MPI_Finalize\n");
-                ompi_win_dump (win);
-            }
-            ompi_win_free (win);
-        }
+    ret = ompi_attr_get_ref();
+    if (OMPI_SUCCESS != ret) {
+        return ret;
     }
 
-    OBJ_DESTRUCT(&ompi_mpi_win_null.win);
-    OBJ_DESTRUCT(&ompi_mpi_windows);
-    OBJ_RELEASE(ompi_win_accumulate_ops);
-    OBJ_RELEASE(ompi_win_accumulate_order);
+    ompi_mpi_instance_append_finalize (ompi_win_finalize);
 
     return OMPI_SUCCESS;
 }
@@ -152,7 +164,15 @@ static int alloc_window(struct ompi_communicator_t *comm, opal_info_t *info, int
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
 
-    ret = opal_info_get_value_enum (info, "accumulate_ops", &acc_ops,
+    /* Copy the info for the info layer */
+    win->super.s_info = OBJ_NEW(opal_info_t);
+    if (info) {
+        opal_info_dup(info, &(win->super.s_info));
+    }
+    ompi_info_memkind_assert_type type;
+    ompi_info_memkind_copy_or_set (&comm->instance->super, &win->super, info, &type);
+
+    ret = opal_info_get_value_enum (win->super.s_info, "accumulate_ops", &acc_ops,
                                     OMPI_WIN_ACCUMULATE_OPS_SAME_OP_NO_OP,
                                     ompi_win_accumulate_ops, &flag);
     if (OMPI_SUCCESS != ret) {
@@ -162,7 +182,7 @@ static int alloc_window(struct ompi_communicator_t *comm, opal_info_t *info, int
 
     win->w_acc_ops = (ompi_win_accumulate_ops_t)acc_ops;
 
-    ret = opal_info_get_value_enum (info, "accumulate_order", &acc_order,
+    ret = opal_info_get_value_enum (win->super.s_info, "accumulate_order", &acc_order,
                                     OMPI_WIN_ACC_ORDER_RAR | OMPI_WIN_ACC_ORDER_WAR |
                                     OMPI_WIN_ACC_ORDER_RAW | OMPI_WIN_ACC_ORDER_WAW,
                                     &(ompi_win_accumulate_order->super), &flag);
@@ -180,12 +200,6 @@ static int alloc_window(struct ompi_communicator_t *comm, opal_info_t *info, int
     OBJ_RETAIN(group);
     win->w_group = group;
 
-    /* Copy the info for the info layer */
-    win->super.s_info = OBJ_NEW(opal_info_t);
-    if (info) {
-        opal_info_dup(info, &(win->super.s_info));
-    }
-
     *win_out = win;
 
     return OMPI_SUCCESS;
@@ -243,7 +257,7 @@ ompi_win_create(void *base, size_t size,
         return ret;
     }
 
-    ret = ompi_osc_base_select(win, &base, size, disp_unit, comm, info, MPI_WIN_FLAVOR_CREATE, &model);
+    ret = ompi_osc_base_select(win, &base, size, disp_unit, comm, MPI_WIN_FLAVOR_CREATE, &model);
     if (OMPI_SUCCESS != ret) {
         OBJ_RELEASE(win);
         return ret;
@@ -260,7 +274,6 @@ ompi_win_create(void *base, size_t size,
     return OMPI_SUCCESS;
 }
 
-
 int
 ompi_win_allocate(size_t size, int disp_unit, opal_info_t *info,
                   ompi_communicator_t *comm, void *baseptr, ompi_win_t **newwin)
@@ -275,7 +288,7 @@ ompi_win_allocate(size_t size, int disp_unit, opal_info_t *info,
         return ret;
     }
 
-    ret = ompi_osc_base_select(win, &base, size, disp_unit, comm, info, MPI_WIN_FLAVOR_ALLOCATE, &model);
+    ret = ompi_osc_base_select(win, &base, size, disp_unit, comm, MPI_WIN_FLAVOR_ALLOCATE, &model);
     if (OMPI_SUCCESS != ret) {
         OBJ_RELEASE(win);
         return ret;
@@ -293,7 +306,6 @@ ompi_win_allocate(size_t size, int disp_unit, opal_info_t *info,
     return OMPI_SUCCESS;
 }
 
-
 int
 ompi_win_allocate_shared(size_t size, int disp_unit, opal_info_t *info,
                          ompi_communicator_t *comm, void *baseptr, ompi_win_t **newwin)
@@ -308,7 +320,7 @@ ompi_win_allocate_shared(size_t size, int disp_unit, opal_info_t *info,
         return ret;
     }
 
-    ret = ompi_osc_base_select(win, &base, size, disp_unit, comm, info, MPI_WIN_FLAVOR_SHARED, &model);
+    ret = ompi_osc_base_select(win, &base, size, disp_unit, comm, MPI_WIN_FLAVOR_SHARED, &model);
     if (OMPI_SUCCESS != ret) {
         OBJ_RELEASE(win);
         return ret;
@@ -326,7 +338,6 @@ ompi_win_allocate_shared(size_t size, int disp_unit, opal_info_t *info,
     return OMPI_SUCCESS;
 }
 
-
 int
 ompi_win_create_dynamic(opal_info_t *info, ompi_communicator_t *comm, ompi_win_t **newwin)
 {
@@ -339,7 +350,7 @@ ompi_win_create_dynamic(opal_info_t *info, ompi_communicator_t *comm, ompi_win_t
         return ret;
     }
 
-    ret = ompi_osc_base_select(win, MPI_BOTTOM, 0, 1, comm, info, MPI_WIN_FLAVOR_DYNAMIC, &model);
+    ret = ompi_osc_base_select(win, MPI_BOTTOM, 0, 1, comm, MPI_WIN_FLAVOR_DYNAMIC, &model);
     if (OMPI_SUCCESS != ret) {
         OBJ_RELEASE(win);
         return ret;
@@ -356,7 +367,6 @@ ompi_win_create_dynamic(opal_info_t *info, ompi_communicator_t *comm, ompi_win_t
     return OMPI_SUCCESS;
 }
 
-
 int
 ompi_win_free(ompi_win_t *win)
 {
diff --git a/ompi/win/win.h b/ompi/win/win.h
index 63aec9de14a..a4f48246cc4 100644
--- a/ompi/win/win.h
+++ b/ompi/win/win.h
@@ -15,6 +15,8 @@
  * Copyright (c) 2013-2015 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
+ * Copyright (c) 2018      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -115,7 +117,7 @@ typedef struct ompi_win_t ompi_win_t;
 OMPI_DECLSPEC OBJ_CLASS_DECLARATION(ompi_win_t);
 
 /**
- * Padded struct to maintain back compatibiltiy.
+ * Padded struct to maintain back compatibility.
  * See ompi/communicator/communicator.h comments with struct ompi_communicator_t
  * for full explanation why we chose the following padding construct for predefines.
  */
@@ -131,7 +133,6 @@ OMPI_DECLSPEC extern ompi_predefined_win_t ompi_mpi_win_null;
 OMPI_DECLSPEC extern ompi_predefined_win_t *ompi_mpi_win_null_addr;
 
 int ompi_win_init(void);
-int ompi_win_finalize(void);
 
 int ompi_win_create(void *base, size_t size, int disp_unit,
                     ompi_communicator_t *comm, opal_info_t *info,
@@ -141,7 +142,6 @@ int ompi_win_allocate(size_t size, int disp_unit, opal_info_t *info,
 int ompi_win_allocate_shared(size_t size, int disp_unit, opal_info_t *info,
                       ompi_communicator_t *comm, void *baseptr, ompi_win_t **newwin);
 int ompi_win_create_dynamic(opal_info_t *info, ompi_communicator_t *comm, ompi_win_t **newwin);
-
 int ompi_win_free(ompi_win_t *win);
 
 OMPI_DECLSPEC int ompi_win_set_name(ompi_win_t *win, const char *win_name);
@@ -149,8 +149,8 @@ OMPI_DECLSPEC int ompi_win_get_name(ompi_win_t *win, char *win_name, int *length
 
 OMPI_DECLSPEC int ompi_win_group(ompi_win_t *win, ompi_group_t **group);
 
-/* Note that the defintion of an "invalid" window is closely related
-   to the defintion of an "invalid" communicator.  See a big comment
+/* Note that the definition of an "invalid" window is closely related
+   to the definition of an "invalid" communicator.  See a big comment
    in ompi/communicator/communicator.h about this. */
 static inline int ompi_win_invalid(ompi_win_t *win) {
     if (NULL == win ||
diff --git a/opal/Makefile.am b/opal/Makefile.am
index 8c208632dd2..100e91fccc1 100644
--- a/opal/Makefile.am
+++ b/opal/Makefile.am
@@ -2,7 +2,7 @@
 # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
 #                         University Research and Technology
 #                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2009 The University of Tennessee and The University
+# Copyright (c) 2004-2022 The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
 # Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
@@ -13,8 +13,9 @@
 # Copyright (c) 2015-2016 Intel, Inc. All rights reserved.
 # Copyright (c) 2016      Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
-# Copyright (c) 2020      Amazon.com, Inc. or its affiliates.
-#                         All Rights reserved.
+# Copyright (c) 2020-2022 Amazon.com, Inc. or its affiliates.  All Rights reserved.
+# Copyright (c) 2022      Advanced Micro Devices, Inc. All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -22,10 +23,8 @@
 # $HEADER$
 #
 
-if OPAL_cuda_support
 SUBDIRS = \
 	include \
-        mca/common/cuda \
         datatype \
         etc \
         util \
@@ -34,74 +33,45 @@ SUBDIRS = \
 	$(MCA_opal_FRAMEWORK_COMPONENT_STATIC_SUBDIRS) \
         . \
 	$(MCA_opal_FRAMEWORK_COMPONENT_DSO_SUBDIRS)
-# libltdl is included by variable because if --disable-dlopen was
-# used, there will be no generated Makefile in that directory (and
-# therefore make distclean will fail).
 DIST_SUBDIRS = \
 	include \
-        mca/common/cuda \
         datatype \
         etc \
 	util \
 	mca/base \
 	$(MCA_opal_FRAMEWORKS_SUBDIRS) \
 	$(MCA_opal_FRAMEWORK_COMPONENT_ALL_SUBDIRS)
-else
-SUBDIRS = \
-	include \
-        datatype \
-        etc \
-        util \
-	mca/base \
-	$(MCA_opal_FRAMEWORKS_SUBDIRS) \
-	$(MCA_opal_FRAMEWORK_COMPONENT_STATIC_SUBDIRS) \
-        . \
-	$(MCA_opal_FRAMEWORK_COMPONENT_DSO_SUBDIRS)
-# libltdl is included by variable because if --disable-dlopen was
-# used, there will be no generated Makefile in that directory (and
-# therefore make distclean will fail).
-DIST_SUBDIRS = \
-	include \
-        datatype \
-        etc \
-	util \
-	mca/base \
-	$(MCA_opal_FRAMEWORKS_SUBDIRS) \
-	$(MCA_opal_FRAMEWORK_COMPONENT_ALL_SUBDIRS)
-endif
 
-# Build the main OPAL library
-lib_LTLIBRARIES = lib@OPAL_LIB_PREFIX@open-pal.la
-lib@OPAL_LIB_PREFIX@open_pal_la_SOURCES =
-lib@OPAL_LIB_PREFIX@open_pal_la_LIBADD = \
-        datatype/libdatatype.la \
+noinst_LTLIBRARIES = libopen-pal_core.la
+lib_LTLIBRARIES = lib@OPAL_LIB_NAME@.la
+
+libopen_pal_core_la_SOURCES =
+libopen_pal_core_la_LIBADD = \
+        mca/base/libmca_base.la \
+        util/libopalutil_core.la \
+	$(MCA_opal_FRAMEWORK_CORE_LIBS)
+libopen_pal_core_la_DEPENDENCIES = \
         mca/base/libmca_base.la \
+        util/libopalutil_core.la \
+	$(MCA_opal_FRAMEWORK_CORE_LIBS)
+
+lib@OPAL_LIB_NAME@_la_SOURCES =
+lib@OPAL_LIB_NAME@_la_LIBADD = \
+	libopen-pal_core.la \
+        datatype/libdatatype.la \
         util/libopalutil.la \
-	$(MCA_opal_FRAMEWORK_LIBS) \
-	$(opal_libevent_LIBS) \
-	$(opal_hwloc_LIBS) \
-	$(opal_pmix_LIBS)
-lib@OPAL_LIB_PREFIX@open_pal_la_DEPENDENCIES = \
+	$(MCA_opal_FRAMEWORK_LIBS)
+lib@OPAL_LIB_NAME@_la_DEPENDENCIES = \
+	libopen-pal_core.la \
         datatype/libdatatype.la \
-        mca/base/libmca_base.la \
         util/libopalutil.la \
         $(MCA_opal_FRAMEWORK_LIBS)
-if OPAL_cuda_support
-lib@OPAL_LIB_PREFIX@open_pal_la_LIBADD += \
-        mca/common/cuda/libmca_common_cuda.la
-lib@OPAL_LIB_PREFIX@open_pal_la_DEPENDENCIES += \
-        mca/common/cuda/libmca_common_cuda.la
-endif
-lib@OPAL_LIB_PREFIX@open_pal_la_LDFLAGS = -version-info $(libopen_pal_so_version) \
-	$(opal_libevent_LDFLAGS) \
-	$(opal_hwloc_LDFLAGS) \
-	$(opal_pmix_LDFLAGS)
+lib@OPAL_LIB_NAME@_la_LDFLAGS = -version-info @libopen_pal_so_version@
 
 # included subdirectory Makefile.am's and appended-to variables
 headers =
-noinst_LTLIBRARIES =
-dist_opaldata_DATA =
-lib@OPAL_LIB_PREFIX@open_pal_la_SOURCES += $(headers)
+EXTRA_DIST =
+lib@OPAL_LIB_NAME@_la_SOURCES += $(headers)
 
 # Conditionally install the header files
 
diff --git a/opal/class/Makefile.am b/opal/class/Makefile.am
index 1d3cdeab6e5..1d3beb50a78 100644
--- a/opal/class/Makefile.am
+++ b/opal/class/Makefile.am
@@ -3,7 +3,7 @@
 # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
 #                         University Research and Technology
 #                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2007 The University of Tennessee and The University
+# Copyright (c) 2004-2022 The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
 # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -14,6 +14,7 @@
 # Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights
 #                         reserved.
 # Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -27,6 +28,7 @@
 # Source code files
 headers += \
         class/opal_bitmap.h \
+        class/opal_cstring.h \
         class/opal_free_list.h \
         class/opal_hash_table.h \
         class/opal_hotel.h \
@@ -41,8 +43,9 @@ headers += \
         class/opal_rb_tree.h \
         class/opal_interval_tree.h
 
-lib@OPAL_LIB_PREFIX@open_pal_la_SOURCES += \
+libopen_pal_core_la_SOURCES += \
         class/opal_bitmap.c \
+        class/opal_cstring.c \
         class/opal_free_list.c \
         class/opal_hash_table.c \
         class/opal_hotel.c \
diff --git a/opal/class/opal_bitmap.c b/opal/class/opal_bitmap.c
index d4e1e9f6b50..d113ed53537 100644
--- a/opal/class/opal_bitmap.c
+++ b/opal/class/opal_bitmap.c
@@ -23,35 +23,30 @@
 
 #include "opal_config.h"
 
-#include <stdio.h>
 #include <limits.h>
+#include <stdio.h>
 
-#include "opal/constants.h"
 #include "opal/class/opal_bitmap.h"
+#include "opal/constants.h"
 
 /* The number of bits in the underlying type of the bitmap field
  * in the opal_bitmap_t struct
  */
-#define SIZE_OF_BASE_TYPE  64
+#define SIZE_OF_BASE_TYPE 64
 
 static void opal_bitmap_construct(opal_bitmap_t *bm);
 static void opal_bitmap_destruct(opal_bitmap_t *bm);
 
-OBJ_CLASS_INSTANCE(opal_bitmap_t, opal_object_t,
-                   opal_bitmap_construct, opal_bitmap_destruct);
-
+OBJ_CLASS_INSTANCE(opal_bitmap_t, opal_object_t, opal_bitmap_construct, opal_bitmap_destruct);
 
-static void
-opal_bitmap_construct(opal_bitmap_t *bm)
+static void opal_bitmap_construct(opal_bitmap_t *bm)
 {
     bm->bitmap = NULL;
     bm->array_size = 0;
     bm->max_size = INT_MAX;
 }
 
-
-static void
-opal_bitmap_destruct(opal_bitmap_t *bm)
+static void opal_bitmap_destruct(opal_bitmap_t *bm)
 {
     if (NULL != bm->bitmap) {
         free(bm->bitmap);
@@ -59,8 +54,7 @@ opal_bitmap_destruct(opal_bitmap_t *bm)
     }
 }
 
-
-int opal_bitmap_set_max_size (opal_bitmap_t *bm, int max_size)
+int opal_bitmap_set_max_size(opal_bitmap_t *bm, int max_size)
 {
     if (NULL == bm) {
         return OPAL_ERR_BAD_PARAM;
@@ -71,14 +65,12 @@ int opal_bitmap_set_max_size (opal_bitmap_t *bm, int max_size)
      * we set it (in numbers of bits!), otherwise it is
      * set to INT_MAX in the constructor.
      */
-    bm->max_size = (int)(((size_t)max_size + SIZE_OF_BASE_TYPE - 1) / SIZE_OF_BASE_TYPE);
+    bm->max_size = (int) (((size_t) max_size + SIZE_OF_BASE_TYPE - 1) / SIZE_OF_BASE_TYPE);
 
     return OPAL_SUCCESS;
 }
 
-
-int
-opal_bitmap_init(opal_bitmap_t *bm, int size)
+int opal_bitmap_init(opal_bitmap_t *bm, int size)
 {
     /*
      * Only if the caller set the maximum size before initializing,
@@ -89,13 +81,14 @@ opal_bitmap_init(opal_bitmap_t *bm, int size)
         return OPAL_ERR_BAD_PARAM;
     }
 
-    bm->array_size = (int)(((size_t)size + SIZE_OF_BASE_TYPE - 1) / SIZE_OF_BASE_TYPE);
-    if( NULL != bm->bitmap ) {
+    bm->array_size = (int) (((size_t) size + SIZE_OF_BASE_TYPE - 1) / SIZE_OF_BASE_TYPE);
+    if (NULL != bm->bitmap) {
         free(bm->bitmap);
-        if(bm->max_size < bm->array_size)
+        if (bm->max_size < bm->array_size) {
             bm->max_size = bm->array_size;
+        }
     }
-    bm->bitmap = (uint64_t*) malloc(bm->array_size * sizeof(uint64_t));
+    bm->bitmap = (uint64_t *) malloc(bm->array_size * sizeof(uint64_t));
     if (NULL == bm->bitmap) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
@@ -104,9 +97,7 @@ opal_bitmap_init(opal_bitmap_t *bm, int size)
     return OPAL_SUCCESS;
 }
 
-
-int
-opal_bitmap_set_bit(opal_bitmap_t *bm, int bit)
+int opal_bitmap_set_bit(opal_bitmap_t *bm, int bit)
 {
     int index, offset, new_size;
 
@@ -124,12 +115,13 @@ opal_bitmap_set_bit(opal_bitmap_t *bm, int bit)
          valid and we simply expand the bitmap */
 
         new_size = index + 1;
-        if( new_size > bm->max_size )
+        if (new_size > bm->max_size) {
             new_size = bm->max_size;
+        }
 
         /* New size is just a multiple of the original size to fit in
          the index. */
-        bm->bitmap = (uint64_t*)realloc(bm->bitmap, new_size*sizeof(uint64_t));
+        bm->bitmap = (uint64_t *) realloc(bm->bitmap, new_size * sizeof(uint64_t));
         if (NULL == bm->bitmap) {
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
@@ -147,9 +139,7 @@ opal_bitmap_set_bit(opal_bitmap_t *bm, int bit)
     return OPAL_SUCCESS;
 }
 
-
-int
-opal_bitmap_clear_bit(opal_bitmap_t *bm, int bit)
+int opal_bitmap_clear_bit(opal_bitmap_t *bm, int bit)
 {
     int index, offset;
 
@@ -164,9 +154,7 @@ opal_bitmap_clear_bit(opal_bitmap_t *bm, int bit)
     return OPAL_SUCCESS;
 }
 
-
-bool
-opal_bitmap_is_set_bit(opal_bitmap_t *bm, int bit)
+bool opal_bitmap_is_set_bit(opal_bitmap_t *bm, int bit)
 {
     int index, offset;
 
@@ -184,9 +172,7 @@ opal_bitmap_is_set_bit(opal_bitmap_t *bm, int bit)
     return false;
 }
 
-
-int
-opal_bitmap_clear_all_bits(opal_bitmap_t *bm)
+int opal_bitmap_clear_all_bits(opal_bitmap_t *bm)
 {
     if (NULL == bm) {
         return OPAL_ERR_BAD_PARAM;
@@ -196,9 +182,7 @@ opal_bitmap_clear_all_bits(opal_bitmap_t *bm)
     return OPAL_SUCCESS;
 }
 
-
-int
-opal_bitmap_set_all_bits(opal_bitmap_t *bm)
+int opal_bitmap_set_all_bits(opal_bitmap_t *bm)
 {
     if (NULL == bm) {
         return OPAL_ERR_BAD_PARAM;
@@ -209,9 +193,7 @@ opal_bitmap_set_all_bits(opal_bitmap_t *bm)
     return OPAL_SUCCESS;
 }
 
-
-int
-opal_bitmap_find_and_set_first_unset_bit(opal_bitmap_t *bm, int *position)
+int opal_bitmap_find_and_set_first_unset_bit(opal_bitmap_t *bm, int *position)
 {
     int i = 0;
     uint64_t temp, all_ones = 0xffffffffffffffffUL;
@@ -222,7 +204,7 @@ opal_bitmap_find_and_set_first_unset_bit(opal_bitmap_t *bm, int *position)
 
     /* Neglect all which don't have an unset bit */
     *position = 0;
-    while((i < bm->array_size) && (bm->bitmap[i] == all_ones)) {
+    while ((i < bm->array_size) && (bm->bitmap[i] == all_ones)) {
         ++i;
     }
 
@@ -236,8 +218,8 @@ opal_bitmap_find_and_set_first_unset_bit(opal_bitmap_t *bm, int *position)
 
     temp = bm->bitmap[i];
     bm->bitmap[i] |= (bm->bitmap[i] + 1); /* Set the first zero bit */
-    temp ^= bm->bitmap[i];  /* Compute the change: the first unset bit in the original number */
-    while( !(temp & 0x1) ) {
+    temp ^= bm->bitmap[i]; /* Compute the change: the first unset bit in the original number */
+    while (!(temp & 0x1)) {
         ++(*position);
         temp >>= 1;
     }
@@ -253,17 +235,17 @@ int opal_bitmap_bitwise_and_inplace(opal_bitmap_t *dest, opal_bitmap_t *right)
     /*
      * Sanity check
      */
-    if( NULL == dest || NULL == right ) {
+    if (NULL == dest || NULL == right) {
         return OPAL_ERR_BAD_PARAM;
     }
-    if( dest->array_size != right->array_size ) {
+    if (dest->array_size != right->array_size) {
         return OPAL_ERR_BAD_PARAM;
     }
 
     /*
      * Bitwise AND
      */
-    for(i = 0; i < dest->array_size; ++i) {
+    for (i = 0; i < dest->array_size; ++i) {
         dest->bitmap[i] &= right->bitmap[i];
     }
 
@@ -277,17 +259,17 @@ int opal_bitmap_bitwise_or_inplace(opal_bitmap_t *dest, opal_bitmap_t *right)
     /*
      * Sanity check
      */
-    if( NULL == dest || NULL == right ) {
+    if (NULL == dest || NULL == right) {
         return OPAL_ERR_BAD_PARAM;
     }
-    if( dest->array_size != right->array_size ) {
+    if (dest->array_size != right->array_size) {
         return OPAL_ERR_BAD_PARAM;
     }
 
     /*
      * Bitwise OR
      */
-    for(i = 0; i < dest->array_size; ++i) {
+    for (i = 0; i < dest->array_size; ++i) {
         dest->bitmap[i] |= right->bitmap[i];
     }
 
@@ -301,17 +283,17 @@ int opal_bitmap_bitwise_xor_inplace(opal_bitmap_t *dest, opal_bitmap_t *right)
     /*
      * Sanity check
      */
-    if( NULL == dest || NULL == right ) {
+    if (NULL == dest || NULL == right) {
         return OPAL_ERR_BAD_PARAM;
     }
-    if( dest->array_size != right->array_size ) {
+    if (dest->array_size != right->array_size) {
         return OPAL_ERR_BAD_PARAM;
     }
 
     /*
      * Bitwise XOR
      */
-    for(i = 0; i < dest->array_size; ++i) {
+    for (i = 0; i < dest->array_size; ++i) {
         dest->bitmap[i] ^= right->bitmap[i];
     }
 
@@ -325,19 +307,19 @@ bool opal_bitmap_are_different(opal_bitmap_t *left, opal_bitmap_t *right)
     /*
      * Sanity check
      */
-    if( NULL == left || NULL == right ) {
+    if (NULL == left || NULL == right) {
         return OPAL_ERR_BAD_PARAM;
     }
 
-    if( opal_bitmap_size(left) != opal_bitmap_size(right) ) {
+    if (opal_bitmap_size(left) != opal_bitmap_size(right)) {
         return true;
     }
 
     /*
      * Direct comparison
      */
-    for(i = 0; i < left->array_size; ++i) {
-        if( left->bitmap[i] != right->bitmap[i] ) {
+    for (i = 0; i < left->array_size; ++i) {
+        if (left->bitmap[i] != right->bitmap[i]) {
             return true;
         }
     }
@@ -345,12 +327,12 @@ bool opal_bitmap_are_different(opal_bitmap_t *left, opal_bitmap_t *right)
     return false;
 }
 
-char * opal_bitmap_get_string(opal_bitmap_t *bitmap)
+char *opal_bitmap_get_string(opal_bitmap_t *bitmap)
 {
     int i;
     char *bitmap_str = NULL;
 
-    if( NULL == bitmap) {
+    if (NULL == bitmap) {
         return NULL;
     }
 
@@ -360,8 +342,8 @@ char * opal_bitmap_get_string(opal_bitmap_t *bitmap)
     }
     bitmap_str[bitmap->array_size * SIZE_OF_BASE_TYPE] = '\0';
 
-    for( i = 0; i < (bitmap->array_size * SIZE_OF_BASE_TYPE); ++i) {
-        if( opal_bitmap_is_set_bit(bitmap, i) ) {
+    for (i = 0; i < (bitmap->array_size * SIZE_OF_BASE_TYPE); ++i) {
+        if (opal_bitmap_is_set_bit(bitmap, i)) {
             bitmap_str[i] = 'X';
         } else {
             bitmap_str[i] = '_';
@@ -387,12 +369,14 @@ int opal_bitmap_num_set_bits(opal_bitmap_t *bm, int len)
     }
 #endif
 
-    for(i = 0; i < len; ++i) {
-        if( 0 == (val = bm->bitmap[i]) ) continue;
+    for (i = 0; i < len; ++i) {
+        if (0 == (val = bm->bitmap[i])) {
+            continue;
+        }
         /*  Peter Wegner in CACM 3 (1960), 322. This method goes through as many
          *  iterations as there are set bits. */
-        for( ; val; cnt++ ) {
-            val &= val - 1;  /* clear the least significant bit set */
+        for (; val; cnt++) {
+            val &= val - 1; /* clear the least significant bit set */
         }
     }
 
diff --git a/opal/class/opal_bitmap.h b/opal/class/opal_bitmap.h
index cda93888451..9de445a55f3 100644
--- a/opal/class/opal_bitmap.h
+++ b/opal/class/opal_bitmap.h
@@ -49,10 +49,10 @@
 BEGIN_C_DECLS
 
 struct opal_bitmap_t {
-    opal_object_t  super;       /**< Subclass of opal_object_t */
-    uint64_t      *bitmap;      /**< The actual bitmap array of characters */
-    int            array_size;  /**< The actual array size that maintains the bitmap */
-    int            max_size;    /**< The maximum size that this bitmap may grow (optional) */
+    opal_object_t super; /**< Subclass of opal_object_t */
+    uint64_t *bitmap;    /**< The actual bitmap array of characters */
+    int array_size;      /**< The actual array size that maintains the bitmap */
+    int max_size;        /**< The maximum size that this bitmap may grow (optional) */
 };
 
 typedef struct opal_bitmap_t opal_bitmap_t;
@@ -68,8 +68,7 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_bitmap_t);
  * @return OPAL error code or success
  *
  */
-OPAL_DECLSPEC int opal_bitmap_set_max_size (opal_bitmap_t *bm, int max_size);
-
+OPAL_DECLSPEC int opal_bitmap_set_max_size(opal_bitmap_t *bm, int max_size);
 
 /**
  * Initializes the bitmap and sets its size. This must be called
@@ -80,12 +79,11 @@ OPAL_DECLSPEC int opal_bitmap_set_max_size (opal_bitmap_t *bm, int max_size);
  * @return OPAL error code or success
  *
  */
-OPAL_DECLSPEC int opal_bitmap_init (opal_bitmap_t *bm, int size);
-
+OPAL_DECLSPEC int opal_bitmap_init(opal_bitmap_t *bm, int size);
 
 /**
  * Set a bit of the bitmap. If the bit asked for is beyond the current
- * size of the bitmap, then the bitmap is extended to accomodate the
+ * size of the bitmap, then the bitmap is extended to accommodate the
  * bit
  *
  * @param  bitmap The input bitmap (IN)
@@ -95,7 +93,6 @@ OPAL_DECLSPEC int opal_bitmap_init (opal_bitmap_t *bm, int size);
  */
 OPAL_DECLSPEC int opal_bitmap_set_bit(opal_bitmap_t *bm, int bit);
 
-
 /**
  * Clear/unset a bit of the bitmap. If the bit is beyond the current
  * size of the bitmap, an error is returned
@@ -107,21 +104,19 @@ OPAL_DECLSPEC int opal_bitmap_set_bit(opal_bitmap_t *bm, int bit);
  */
 OPAL_DECLSPEC int opal_bitmap_clear_bit(opal_bitmap_t *bm, int bit);
 
-
 /**
-  * Find out if a bit is set in the bitmap
-  *
-  * @param  bitmap  The input bitmap (IN)
-  * @param  bit     The bit which is to be checked (IN)
-  * @return true    if the bit is set
-  *         false   if the bit is not set OR the index
-  *                 is outside the bounds of the provided
-  *                 bitmap
-  *
-  */
+ * Find out if a bit is set in the bitmap
+ *
+ * @param  bitmap  The input bitmap (IN)
+ * @param  bit     The bit which is to be checked (IN)
+ * @return true    if the bit is set
+ *         false   if the bit is not set OR the index
+ *                 is outside the bounds of the provided
+ *                 bitmap
+ *
+ */
 OPAL_DECLSPEC bool opal_bitmap_is_set_bit(opal_bitmap_t *bm, int bit);
 
-
 /**
  * Find the first clear bit in the bitmap and set it
  *
@@ -130,9 +125,7 @@ OPAL_DECLSPEC bool opal_bitmap_is_set_bit(opal_bitmap_t *bm, int bit);
 
  * @return err        OPAL_SUCCESS on success
  */
-OPAL_DECLSPEC int opal_bitmap_find_and_set_first_unset_bit(opal_bitmap_t *bm,
-                                                           int *position);
-
+OPAL_DECLSPEC int opal_bitmap_find_and_set_first_unset_bit(opal_bitmap_t *bm, int *position);
 
 /**
  * Clear all bits in the bitmap
@@ -143,7 +136,6 @@ OPAL_DECLSPEC int opal_bitmap_find_and_set_first_unset_bit(opal_bitmap_t *bm,
  */
 OPAL_DECLSPEC int opal_bitmap_clear_all_bits(opal_bitmap_t *bm);
 
-
 /**
  * Set all bits in the bitmap
  * @param bitmap The input bitmap (IN)
@@ -152,7 +144,6 @@ OPAL_DECLSPEC int opal_bitmap_clear_all_bits(opal_bitmap_t *bm);
  */
 OPAL_DECLSPEC int opal_bitmap_set_all_bits(opal_bitmap_t *bm);
 
-
 /**
  * Gives the current size (number of bits) in the bitmap. This is the
  * legal (accessible) number of bits
@@ -166,7 +157,6 @@ static inline int opal_bitmap_size(opal_bitmap_t *bm)
     return (NULL == bm) ? 0 : (bm->array_size * ((int) (sizeof(*bm->bitmap) * 8)));
 }
 
-
 /**
  * Copy a bitmap
  *
@@ -176,10 +166,11 @@ static inline int opal_bitmap_size(opal_bitmap_t *bm)
  */
 static inline void opal_bitmap_copy(opal_bitmap_t *dest, opal_bitmap_t *src)
 {
-    if( dest->array_size < src->array_size ) {
-        if( NULL != dest->bitmap) free(dest->bitmap);
+    if (dest->array_size < src->array_size) {
+        if (NULL != dest->bitmap)
+            free(dest->bitmap);
         dest->max_size = src->max_size;
-        dest->bitmap = (uint64_t*)malloc(src->array_size*sizeof(uint64_t));
+        dest->bitmap = (uint64_t *) malloc(src->array_size * sizeof(uint64_t));
     }
     memcpy(dest->bitmap, src->bitmap, src->array_size * sizeof(uint64_t));
     dest->array_size = src->array_size;
@@ -228,7 +219,7 @@ OPAL_DECLSPEC bool opal_bitmap_are_different(opal_bitmap_t *left, opal_bitmap_t
  * @param bitmap Point to the bitmap to represent
  * @return Pointer to the string (caller must free if not NULL)
  */
-OPAL_DECLSPEC char * opal_bitmap_get_string(opal_bitmap_t *bitmap);
+OPAL_DECLSPEC char *opal_bitmap_get_string(opal_bitmap_t *bitmap);
 
 /**
  * Return the number of 'unset' bits, upto the specified length
diff --git a/opal/class/opal_cstring.c b/opal/class/opal_cstring.c
new file mode 100644
index 00000000000..674c3a6741b
--- /dev/null
+++ b/opal/class/opal_cstring.c
@@ -0,0 +1,149 @@
+/*
+ * Copyright (c) 2020      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2020      High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "opal_cstring.h"
+
+#include "opal/constants.h"
+#include "opal/util/string_copy.h"
+#include <ctype.h>
+#include <errno.h>
+#include <stddef.h>
+
+static void opal_cstring_ctor(opal_cstring_t *obj);
+
+OBJ_CLASS_INSTANCE(opal_cstring_t, opal_object_t, &opal_cstring_ctor, NULL);
+
+/* make sure we have sufficient padding to always null-terminate the string */
+#if (__STDC_VERSION__ >= 201112L)
+_Static_assert(sizeof(opal_cstring_t) > offsetof(opal_cstring_t, string),
+               "Insufficient padding available in opal_cstring_t");
+#endif // (__STDC_VERSION__ >= 201112L)
+
+static void opal_cstring_ctor(opal_cstring_t *obj)
+{
+    *(size_t *) &(obj->length) = 0;
+    /* make sure the string is null-terminated */
+    ((char *) obj->string)[0] = '\0';
+}
+
+static inline size_t opal_cstring_alloc_size(size_t len)
+{
+    /* the size required for the object and the string, incl. the null-terminator */
+    size_t res = sizeof(opal_cstring_t) + len + 1;
+    /* adjust for the additional padding that is used for the string anyway */
+    res -= (sizeof(opal_cstring_t) - offsetof(opal_cstring_t, string));
+    /* make sure we allocate at least sizeof(opal_cstring_t) */
+    return (res > sizeof(opal_cstring_t)) ? res : sizeof(opal_cstring_t);
+}
+
+opal_cstring_t *opal_cstring_create_l(const char *string, size_t len)
+{
+    if (NULL == string || 0 == len) {
+        return OBJ_NEW(opal_cstring_t);
+    }
+
+    /* Allocate space for the object, the characters in \c string and the terminating null */
+    opal_cstring_t *res = (opal_cstring_t *) malloc(opal_cstring_alloc_size(len));
+    if (NULL == res) {
+        return NULL;
+    }
+    OBJ_CONSTRUCT(res, opal_cstring_t);
+
+    /* cast away const for setting the member values */
+    *(size_t *) &(res->length) = len;
+    opal_string_copy((char *) res->string, string, len + 1);
+
+    return res;
+}
+
+opal_cstring_t *opal_cstring_create(const char *string)
+{
+    if (NULL == string) {
+        return OBJ_NEW(opal_cstring_t);
+    }
+    size_t len = strlen(string);
+    return opal_cstring_create_l(string, len);
+}
+
+int opal_cstring_to_int(opal_cstring_t *string, int *interp)
+{
+    long tmp;
+    char *endp;
+
+    if (NULL == string || '\0' == string->string[0]) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    errno = 0;
+    tmp = strtol(string->string, &endp, 10);
+    /* we found something not a number */
+    if (*endp != '\0') {
+        return OPAL_ERR_BAD_PARAM;
+    }
+    /* base errors */
+    if (tmp == 0 && errno == EINVAL) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+    /* overflow/underflow of long int (return of strtol) */
+    if (errno == ERANGE && (tmp == LONG_MIN || tmp == LONG_MAX)) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+    /* overflow/underflow of int (for cases where long int is larger) */
+    if (tmp < INT_MIN || tmp > INT_MAX) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    *interp = (int) tmp;
+
+    return OPAL_SUCCESS;
+}
+
+static int opal_str_to_bool_impl(const char *string, bool *interp)
+{
+    const char *ptr = string;
+
+    if (NULL != string) {
+        /* Trim leading whitespace */
+        while (isspace(*ptr)) {
+            ++ptr;
+        }
+
+        if ('\0' != *ptr) {
+            if (isdigit(*ptr)) {
+                *interp = (bool) atoi(ptr);
+                return OPAL_SUCCESS;
+            } else if (0 == strncasecmp(ptr, "yes", 3) || 0 == strncasecmp(ptr, "true", 4)) {
+                *interp = true;
+                return OPAL_SUCCESS;
+            } else if (0 == strncasecmp(ptr, "no", 2) || 0 == strncasecmp(ptr, "false", 5)) {
+                *interp = false;
+                return OPAL_SUCCESS;
+            }
+        }
+    }
+
+    *interp = false;
+    return OPAL_ERR_BAD_PARAM;
+}
+
+int opal_cstring_to_bool(opal_cstring_t *string, bool *interp)
+{
+    return opal_str_to_bool_impl(string->string, interp);
+}
+
+bool opal_str_to_bool(const char *string)
+{
+    bool res;
+    opal_str_to_bool_impl(string, &res);
+    return res;
+}
diff --git a/opal/class/opal_cstring.h b/opal/class/opal_cstring.h
new file mode 100644
index 00000000000..87c8f43f262
--- /dev/null
+++ b/opal/class/opal_cstring.h
@@ -0,0 +1,160 @@
+/*
+ * Copyright (c) 2020      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2020      High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/**
+ * @file
+ *
+ * Implementation of a reference-counted immutable string object.
+ * The string object is created using either \c opal_cstring_create(string) or
+ * \c opal_cstring_create_l(string, len) with the latter accepting the number
+ * of characters to take from the input string.
+ *
+ * The reference counting is done using opal's \c OBJ_RETAIN / \c OBJ_RELEASE mechanism
+ * and it is the user's responsibility to ensure that the string is eventually
+ * free'd by decrementing the reference counter using OBJ_RETAIN.
+ *
+ * The structure contains two relevant members:
+ *
+ * - \c length: the length of the string, i.e., the number of characters in \c string *not*
+ *              including the null-terminator.
+ * - \c string: the array of characters that make up the string.
+ *
+ * Both fields are \c const and should not be altered by the user. If the string
+ * contained in an \c opal_cstring_t object should be modified it has to be copied
+ * to a different buffer (e.g., using strdup).
+ *
+ * The \c string is always guaranteed to be null-terminated, even if the
+ * \c opal_cstring_t object was created using \c OBJ_NEW (which results in an
+ * empty string with the \c length field set to zero and the \c string field
+ * pointing to the null-terminator).
+ *
+ */
+
+#ifndef OPAL_STRING_H
+#define OPAL_STRING_H
+
+#include "opal/class/opal_object.h"
+#include "opal/mca/base/mca_base_var_enum.h"
+
+#include <string.h>
+
+/**
+ * Reference-counted immutable string object.
+ *
+ * The two relevant members are:
+ *
+ * - \c length: the length of the string, i.e., the number of characters in \c string *not*
+ *              including the null-terminator.
+ * - \c string: the array of characters that make up the string.
+ *
+ * The string is eventually free'd by calling \c OBJ_RELEASE on it.
+ *
+ * If allocated using \c OBJ_NEW the object will contain an empty string.
+ * The member field \c _ignored is used to force the existence of padding bytes
+ * that can be used to write the null-terminator even if no additional memory
+ * was allocated succeeding the object itself and is ignored.
+ */
+struct opal_cstring_t {
+    opal_object_t super;
+    const size_t length; //< the number of characters not including the null-terminator
+    char _ignored; //< single char forcing additional padding to always ensure null-termination
+    const char string[]; //< FMA containing the string, making use of padding bytes
+};
+
+typedef struct opal_cstring_t opal_cstring_t;
+
+BEGIN_C_DECLS
+
+/**
+ * \internal
+ *
+ * The class for string objects.
+ */
+OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_cstring_t);
+
+/**
+ * Create a new instance of a reference-counted immutable string object
+ * (\ref opal_cstring_t) containing the characters of \c string.
+ *
+ * @param string Value of the new string
+ * @return An object representing the null-terminated string with value \c string
+ *
+ * If \c string is \c NULL then the resulting string will be empty.
+ *
+ */
+OPAL_DECLSPEC
+opal_cstring_t *opal_cstring_create(const char *string) __opal_attribute_malloc__;
+
+/**
+ * Create a new instance of a reference-counted immutable string object
+ * (\ref opal_cstring_t) containing the first \c length characters of \c string.
+ *
+ * @param string Value of the new string
+ * @return An object representing null-terminated string with the first
+ *         \c length characters of \c string
+ *
+ * If \c string is \c NULL or \c length is zero the resulting string will be empty.
+ */
+OPAL_DECLSPEC
+opal_cstring_t *opal_cstring_create_l(const char *string, size_t length) __opal_attribute_malloc__;
+
+/**
+ * Convert string to integer
+ *
+ * Convert \c string into an integer, adhering to the
+ * interpretation rules specified in MPI-4 Chapter 10.
+ * All others will return \c OPAL_ERR_BAD_PARAM
+ *
+ * @param string Value string to interpret
+ * @param interp returned interpretation of the value key
+ *
+ * @retval OPAL_SUCCESS string was successfully interpreted
+ * @retval OPAL_ERR_BAD_PARAM string could not be interpreted
+ *
+ */
+OPAL_DECLSPEC
+int opal_cstring_to_int(opal_cstring_t *string, int *interp);
+
+/**
+ * Convert string to boolean
+ *
+ * Convert \c string into a boolean, adhering to the
+ * interpretation rules specified in MPI-4 Chapter 10.
+ *
+ * @param value Value string to interpret
+ * @param interp returned interpretation of the value key
+ *
+ * @retval OPAL_SUCCESS string was successfully interpreted
+ * @retval OPAL_ERR_BAD_PARAM string was not able to be interpreted
+ *
+ *   The string value will be cast to the boolean output in
+ *   the following manner:
+ *
+ *   - If the string value is digits, the return value is "(bool)
+ *     atoi(value)"
+ *   - If the string value is (case-insensitive) "yes" or "true", the
+ *     result is true
+ *   - If the string value is (case-insensitive) "no" or "false", the
+ *     result is false
+ *   - All other values will lead to a return value of OPAL_ERR_BAD_PARAM and
+ *     \c interp will be set to false.
+ */
+OPAL_DECLSPEC
+int opal_cstring_to_bool(opal_cstring_t *string, bool *interp);
+
+OPAL_DECLSPEC
+bool opal_str_to_bool(const char *string);
+
+END_C_DECLS
+
+#endif // OPAL_STRING_H
diff --git a/opal/class/opal_fifo.c b/opal/class/opal_fifo.c
index 238801d4922..c3cd93d1c31 100644
--- a/opal/class/opal_fifo.c
+++ b/opal/class/opal_fifo.c
@@ -22,7 +22,7 @@
 #include "opal_config.h"
 #include "opal/class/opal_fifo.h"
 
-static void opal_fifo_construct (opal_fifo_t *fifo)
+static void opal_fifo_construct(opal_fifo_t *fifo)
 {
     OBJ_CONSTRUCT(&fifo->opal_fifo_ghost, opal_list_item_t);
 
diff --git a/opal/class/opal_fifo.h b/opal/class/opal_fifo.h
index 3cc88393cd1..816b2fb0ecd 100644
--- a/opal/class/opal_fifo.h
+++ b/opal/class/opal_fifo.h
@@ -13,7 +13,7 @@
  * Copyright (c) 2007      Voltaire All rights reserved.
  * Copyright (c) 2010      IBM Corporation.  All rights reserved.
  * Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights
- *                         reseved.
+ *                         reserved.
  * Copyright (c) 2021      Triad National Security, LLC. All rights reserved.
  * Copyright (c) 2021      Google, LLC. All rights reserved.
  * $COPYRIGHT$
@@ -29,8 +29,8 @@
 #include "opal_config.h"
 #include "opal/class/opal_lifo.h"
 
-#include "opal/sys/atomic.h"
 #include "opal/mca/threads/mutex.h"
+#include "opal/sys/atomic.h"
 
 BEGIN_C_DECLS
 
@@ -59,12 +59,12 @@ typedef struct opal_fifo_t opal_fifo_t;
 
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_fifo_t);
 
-static inline opal_list_item_t *opal_fifo_head (opal_fifo_t* fifo)
+static inline opal_list_item_t *opal_fifo_head(opal_fifo_t *fifo)
 {
     return (opal_list_item_t *) fifo->opal_fifo_head.data.item;
 }
 
-static inline opal_list_item_t *opal_fifo_tail (opal_fifo_t* fifo)
+static inline opal_list_item_t *opal_fifo_tail(opal_fifo_t *fifo)
 {
     return (opal_list_item_t *) fifo->opal_fifo_tail.data.item;
 }
@@ -73,9 +73,9 @@ static inline opal_list_item_t *opal_fifo_tail (opal_fifo_t* fifo)
  * compare-and-swap. On most architectures the reading of a pointer is an
  * atomic operation so we don't have to protect it.
  */
-static inline bool opal_fifo_is_empty( opal_fifo_t* fifo )
+static inline bool opal_fifo_is_empty(opal_fifo_t *fifo)
 {
-    return opal_fifo_head (fifo) == &fifo->opal_fifo_ghost;
+    return opal_fifo_head(fifo) == &fifo->opal_fifo_ghost;
 }
 
 #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 && !OPAL_HAVE_ATOMIC_LLSC_PTR
@@ -84,28 +84,27 @@ static inline bool opal_fifo_is_empty( opal_fifo_t* fifo )
  * to allow the upper level to detect if this element is the first one in the
  * list (if the list was empty before this operation).
  */
-static inline opal_list_item_t *opal_fifo_push_atomic (opal_fifo_t *fifo,
-                                                       opal_list_item_t *item)
+static inline opal_list_item_t *opal_fifo_push_atomic(opal_fifo_t *fifo, opal_list_item_t *item)
 {
     opal_counted_pointer_t tail = {.value = fifo->opal_fifo_tail.value};
-    const opal_list_item_t * const ghost = &fifo->opal_fifo_ghost;
+    const opal_list_item_t *const ghost = &fifo->opal_fifo_ghost;
 
     item->opal_list_next = (opal_list_item_t *) ghost;
 
-    opal_atomic_wmb ();
+    opal_atomic_wmb();
 
     do {
-        if (opal_update_counted_pointer (&fifo->opal_fifo_tail, &tail, item)) {
+        if (opal_update_counted_pointer(&fifo->opal_fifo_tail, &tail, item)) {
             break;
         }
     } while (1);
 
-    opal_atomic_wmb ();
+    opal_atomic_wmb();
 
-  if ((intptr_t) ghost == tail.data.item) {
+    if ((intptr_t) ghost == tail.data.item) {
         /* update the head */
         opal_counted_pointer_t head = {.value = fifo->opal_fifo_head.value};
-        opal_update_counted_pointer (&fifo->opal_fifo_head, &head, item);
+        opal_update_counted_pointer(&fifo->opal_fifo_head, &head, item);
     } else {
         /* update previous item */
         ((opal_list_item_t *) tail.data.item)->opal_list_next = item;
@@ -117,16 +116,16 @@ static inline opal_list_item_t *opal_fifo_push_atomic (opal_fifo_t *fifo,
 /* Retrieve one element from the FIFO. If we reach the ghost element then the FIFO
  * is empty so we return NULL.
  */
-static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo)
+static inline opal_list_item_t *opal_fifo_pop_atomic(opal_fifo_t *fifo)
 {
     opal_list_item_t *item, *next, *ghost = &fifo->opal_fifo_ghost;
     opal_counted_pointer_t head, tail;
 
-    opal_read_counted_pointer (&fifo->opal_fifo_head, &head);
+    opal_read_counted_pointer(&fifo->opal_fifo_head, &head);
 
     do {
         tail.value = fifo->opal_fifo_tail.value;
-        opal_atomic_rmb ();
+        opal_atomic_rmb();
 
         item = (opal_list_item_t *) head.data.item;
         next = (opal_list_item_t *) item->opal_list_next;
@@ -136,29 +135,30 @@ static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo)
         }
 
         /* the head or next pointer are in an inconsistent state. keep looping. */
-        if (tail.data.item != (intptr_t) item && (intptr_t) ghost != tail.data.item && ghost == next) {
-            opal_read_counted_pointer (&fifo->opal_fifo_head, &head);
+        if (tail.data.item != (intptr_t) item && (intptr_t) ghost != tail.data.item
+            && ghost == next) {
+            opal_read_counted_pointer(&fifo->opal_fifo_head, &head);
             continue;
         }
 
         /* try popping the head */
-        if (opal_update_counted_pointer (&fifo->opal_fifo_head, &head, next)) {
+        if (opal_update_counted_pointer(&fifo->opal_fifo_head, &head, next)) {
             break;
         }
     } while (1);
 
-    opal_atomic_wmb ();
+    opal_atomic_wmb();
 
     /* check for tail and head consistency */
     if (ghost == next) {
         /* the head was just set to &fifo->opal_fifo_ghost. try to update the tail as well */
-        if (!opal_update_counted_pointer (&fifo->opal_fifo_tail, &tail, ghost)) {
+        if (!opal_update_counted_pointer(&fifo->opal_fifo_tail, &tail, ghost)) {
             /* tail was changed by a push operation. wait for the item's next pointer to be se then
              * update the head */
 
             /* wait for next pointer to be updated by push */
             do {
-                opal_atomic_rmb ();
+                opal_atomic_rmb();
             } while (ghost == item->opal_list_next);
 
             /* update the head with the real next value. note that no other thread
@@ -166,7 +166,7 @@ static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo)
              * with the next pointer. push will not see an empty list and other pop
              * operations will loop until the head is consistent. */
             fifo->opal_fifo_head.data.item = (intptr_t) item->opal_list_next;
-            opal_atomic_wmb ();
+            opal_atomic_wmb();
         }
     }
 
@@ -180,20 +180,20 @@ static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo)
 /* When compare-and-set 128 is not available we avoid the ABA problem by
  * using a spin-lock on the head (using the head counter). Otherwise
  * the algorithm is identical to the compare-and-set 128 version. */
-static inline opal_list_item_t *opal_fifo_push_atomic (opal_fifo_t *fifo,
-                                                       opal_list_item_t *item)
+static inline opal_list_item_t *opal_fifo_push_atomic(opal_fifo_t *fifo, opal_list_item_t *item)
 {
-    const opal_list_item_t * const ghost = &fifo->opal_fifo_ghost;
+    const opal_list_item_t *const ghost = &fifo->opal_fifo_ghost;
     opal_list_item_t *tail_item;
 
     item->opal_list_next = (opal_list_item_t *) ghost;
 
-    opal_atomic_wmb ();
+    opal_atomic_wmb();
 
     /* try to get the tail */
-    tail_item = (opal_list_item_t *) opal_atomic_swap_ptr (&fifo->opal_fifo_tail.data.item, (intptr_t) item);
+    tail_item = (opal_list_item_t *) opal_atomic_swap_ptr(&fifo->opal_fifo_tail.data.item,
+                                                          (intptr_t) item);
 
-    opal_atomic_wmb ();
+    opal_atomic_wmb();
 
     if (ghost == tail_item) {
         /* update the head */
@@ -203,7 +203,7 @@ static inline opal_list_item_t *opal_fifo_push_atomic (opal_fifo_t *fifo,
         tail_item->opal_list_next = item;
     }
 
-    opal_atomic_wmb ();
+    opal_atomic_wmb();
 
     return (opal_list_item_t *) tail_item;
 }
@@ -211,20 +211,20 @@ static inline opal_list_item_t *opal_fifo_push_atomic (opal_fifo_t *fifo,
 /* Retrieve one element from the FIFO. If we reach the ghost element then the FIFO
  * is empty so we return NULL.
  */
-static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo)
+static inline opal_list_item_t *opal_fifo_pop_atomic(opal_fifo_t *fifo)
 {
-    const opal_list_item_t * const ghost = &fifo->opal_fifo_ghost;
+    const opal_list_item_t *const ghost = &fifo->opal_fifo_ghost;
 
-#if OPAL_HAVE_ATOMIC_LLSC_PTR
+#    if OPAL_HAVE_ATOMIC_LLSC_PTR
     register opal_list_item_t *item, *next;
     int attempt = 0, ret = 0;
 
     /* use load-linked store-conditional to avoid ABA issues */
     do {
         if (++attempt == 5) {
-            /* deliberatly suspend this thread to allow other threads to run. this should
+            /* deliberately suspend this thread to allow other threads to run. this should
              * only occur during periods of contention on the lifo. */
-            _opal_lifo_release_cpu ();
+            opal_lifo_release_cpu();
             attempt = 0;
         }
 
@@ -243,21 +243,21 @@ static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo)
         opal_atomic_sc_ptr(&fifo->opal_fifo_head.data.item, next, ret);
     } while (!ret);
 
-#else
+#    else
     opal_list_item_t *item, *next;
 
     /* protect against ABA issues by "locking" the head */
     do {
-        if (!opal_atomic_swap_32 ((opal_atomic_int32_t *) &fifo->opal_fifo_head.data.counter, 1)) {
+        if (!opal_atomic_swap_32((opal_atomic_int32_t *) &fifo->opal_fifo_head.data.counter, 1)) {
             break;
         }
 
-        opal_atomic_wmb ();
+        opal_atomic_wmb();
     } while (1);
 
     opal_atomic_wmb();
 
-    item = opal_fifo_head (fifo);
+    item = opal_fifo_head(fifo);
     if (ghost == item) {
         fifo->opal_fifo_head.data.counter = 0;
         return NULL;
@@ -265,21 +265,22 @@ static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo)
 
     next = (opal_list_item_t *) item->opal_list_next;
     fifo->opal_fifo_head.data.item = (uintptr_t) next;
-#endif
+#    endif
 
     if (ghost == next) {
         void *tmp = item;
 
-        if (!opal_atomic_compare_exchange_strong_ptr (&fifo->opal_fifo_tail.data.item, (intptr_t *) &tmp, (intptr_t) ghost)) {
+        if (!opal_atomic_compare_exchange_strong_ptr(&fifo->opal_fifo_tail.data.item,
+                                                     (intptr_t *) &tmp, (intptr_t) ghost)) {
             do {
-                opal_atomic_rmb ();
+                opal_atomic_rmb();
             } while (ghost == item->opal_list_next);
 
             fifo->opal_fifo_head.data.item = (intptr_t) item->opal_list_next;
         }
     }
 
-    opal_atomic_wmb ();
+    opal_atomic_wmb();
 
     /* unlock the head */
     fifo->opal_fifo_head.data.counter = 0;
@@ -292,15 +293,14 @@ static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo)
 #endif
 
 /* single threaded versions of push/pop */
-static inline opal_list_item_t *opal_fifo_push_st (opal_fifo_t *fifo,
-                                                   opal_list_item_t *item)
+static inline opal_list_item_t *opal_fifo_push_st(opal_fifo_t *fifo, opal_list_item_t *item)
 {
-    opal_list_item_t *prev = opal_fifo_tail (fifo);
+    opal_list_item_t *prev = opal_fifo_tail(fifo);
 
     item->opal_list_next = &fifo->opal_fifo_ghost;
 
     fifo->opal_fifo_tail.data.item = (intptr_t) item;
-    if (&fifo->opal_fifo_ghost == opal_fifo_head (fifo)) {
+    if (&fifo->opal_fifo_ghost == opal_fifo_head(fifo)) {
         fifo->opal_fifo_head.data.item = (intptr_t) item;
     } else {
         prev->opal_list_next = item;
@@ -309,16 +309,16 @@ static inline opal_list_item_t *opal_fifo_push_st (opal_fifo_t *fifo,
     return (opal_list_item_t *) item->opal_list_next;
 }
 
-static inline opal_list_item_t *opal_fifo_pop_st (opal_fifo_t *fifo)
+static inline opal_list_item_t *opal_fifo_pop_st(opal_fifo_t *fifo)
 {
-    opal_list_item_t *item = opal_fifo_head (fifo);
+    opal_list_item_t *item = opal_fifo_head(fifo);
 
     if (item == &fifo->opal_fifo_ghost) {
         return NULL;
     }
 
     fifo->opal_fifo_head.data.item = (intptr_t) item->opal_list_next;
-    if (&fifo->opal_fifo_ghost == opal_fifo_head (fifo)) {
+    if (&fifo->opal_fifo_ghost == opal_fifo_head(fifo)) {
         fifo->opal_fifo_tail.data.item = (intptr_t) &fifo->opal_fifo_ghost;
     }
 
@@ -327,25 +327,24 @@ static inline opal_list_item_t *opal_fifo_pop_st (opal_fifo_t *fifo)
 }
 
 /* push/pop versions conditioned off opal_using_threads() */
-static inline opal_list_item_t *opal_fifo_push (opal_fifo_t *fifo,
-                                                opal_list_item_t *item)
+static inline opal_list_item_t *opal_fifo_push(opal_fifo_t *fifo, opal_list_item_t *item)
 {
-    if (opal_using_threads ()) {
-        return opal_fifo_push_atomic (fifo, item);
+    if (opal_using_threads()) {
+        return opal_fifo_push_atomic(fifo, item);
     }
 
-    return opal_fifo_push_st (fifo, item);
+    return opal_fifo_push_st(fifo, item);
 }
 
-static inline opal_list_item_t *opal_fifo_pop (opal_fifo_t *fifo)
+static inline opal_list_item_t *opal_fifo_pop(opal_fifo_t *fifo)
 {
-    if (opal_using_threads ()) {
-        return opal_fifo_pop_atomic (fifo);
+    if (opal_using_threads()) {
+        return opal_fifo_pop_atomic(fifo);
     }
 
-    return opal_fifo_pop_st (fifo);
+    return opal_fifo_pop_st(fifo);
 }
 
 END_C_DECLS
 
-#endif  /* OPAL_FIFO_H_HAS_BEEN_INCLUDED */
+#endif /* OPAL_FIFO_H_HAS_BEEN_INCLUDED */
diff --git a/opal/class/opal_free_list.c b/opal/class/opal_free_list.c
index 7fb010effc0..259642918f2 100644
--- a/opal/class/opal_free_list.c
+++ b/opal/class/opal_free_list.c
@@ -24,21 +24,19 @@
 
 #include "opal_config.h"
 
-#include "opal/class/opal_free_list.h"
 #include "opal/align.h"
-#include "opal/util/output.h"
-#include "opal/mca/mpool/mpool.h"
+#include "opal/class/opal_free_list.h"
 #include "opal/mca/mpool/base/base.h"
+#include "opal/mca/mpool/mpool.h"
 #include "opal/mca/rcache/rcache.h"
+#include "opal/util/output.h"
 #include "opal/util/sys_limits.h"
 
 typedef struct opal_free_list_item_t opal_free_list_memory_t;
 
-OBJ_CLASS_INSTANCE(opal_free_list_item_t,
-                   opal_list_item_t,
-                   NULL, NULL);
+OBJ_CLASS_INSTANCE(opal_free_list_item_t, opal_list_item_t, NULL, NULL);
 
-static void opal_free_list_construct(opal_free_list_t* fl)
+static void opal_free_list_construct(opal_free_list_t *fl)
 {
     OBJ_CONSTRUCT(&fl->fl_lock, opal_mutex_t);
     OBJ_CONSTRUCT(&fl->fl_condition, opal_condition_t);
@@ -54,22 +52,21 @@ static void opal_free_list_construct(opal_free_list_t* fl)
     fl->fl_mpool = NULL;
     fl->fl_rcache = NULL;
     /* default flags */
-    fl->fl_rcache_reg_flags = MCA_RCACHE_FLAGS_CACHE_BYPASS |
-        MCA_RCACHE_FLAGS_CUDA_REGISTER_MEM;
+    fl->fl_rcache_reg_flags = MCA_RCACHE_FLAGS_CACHE_BYPASS | MCA_RCACHE_FLAGS_ACCELERATOR_REGISTER_MEM;
     fl->ctx = NULL;
     OBJ_CONSTRUCT(&(fl->fl_allocations), opal_list_t);
 }
 
-static void opal_free_list_allocation_release (opal_free_list_t *fl, opal_free_list_memory_t *fl_mem)
+static void opal_free_list_allocation_release(opal_free_list_t *fl, opal_free_list_memory_t *fl_mem)
 {
     if (NULL != fl->fl_rcache) {
-        fl->fl_rcache->rcache_deregister (fl->fl_rcache, fl_mem->registration);
+        fl->fl_rcache->rcache_deregister(fl->fl_rcache, fl_mem->registration);
     }
 
     if (NULL != fl->fl_mpool) {
-        fl->fl_mpool->mpool_free (fl->fl_mpool, fl_mem->ptr);
+        fl->fl_mpool->mpool_free(fl->fl_mpool, fl_mem->ptr);
     } else if (fl_mem->ptr) {
-        free (fl_mem->ptr);
+        free(fl_mem->ptr);
     }
 
     /* destruct the item (we constructed it), then free the memory chunk */
@@ -90,8 +87,8 @@ static void opal_free_list_destruct(opal_free_list_t *fl)
     }
 #endif
 
-    while(NULL != (item = opal_lifo_pop(&(fl->super)))) {
-        fl_item = (opal_free_list_item_t*)item;
+    while (NULL != (item = opal_lifo_pop(&(fl->super)))) {
+        fl_item = (opal_free_list_item_t *) item;
 
         /* destruct the item (we constructed it), the underlying memory will be
          * reclaimed when we free the slab (opal_free_list_memory_t ptr)
@@ -99,8 +96,8 @@ static void opal_free_list_destruct(opal_free_list_t *fl)
         OBJ_DESTRUCT(fl_item);
     }
 
-    while(NULL != (item = opal_list_remove_first(&fl->fl_allocations))) {
-        opal_free_list_allocation_release (fl, (opal_free_list_memory_t *) item);
+    while (NULL != (item = opal_list_remove_first(&fl->fl_allocations))) {
+        opal_free_list_allocation_release(fl, (opal_free_list_memory_t *) item);
     }
 
     OBJ_DESTRUCT(&fl->fl_allocations);
@@ -111,14 +108,13 @@ static void opal_free_list_destruct(opal_free_list_t *fl)
 OBJ_CLASS_INSTANCE(opal_free_list_t, opal_lifo_t, opal_free_list_construct,
                    opal_free_list_destruct);
 
-
-int opal_free_list_init (opal_free_list_t *flist, size_t frag_size, size_t frag_alignment,
-                         opal_class_t *frag_class, size_t payload_buffer_size,
-                         size_t payload_buffer_alignment, int num_elements_to_alloc,
-                         int max_elements_to_alloc, int num_elements_per_alloc,
-                         mca_mpool_base_module_t *mpool, int rcache_reg_flags,
-                         mca_rcache_base_module_t *rcache, opal_free_list_item_init_fn_t item_init,
-                         void *ctx)
+int opal_free_list_init(opal_free_list_t *flist, size_t frag_size, size_t frag_alignment,
+                        opal_class_t *frag_class, size_t payload_buffer_size,
+                        size_t payload_buffer_alignment, int num_elements_to_alloc,
+                        int max_elements_to_alloc, int num_elements_per_alloc,
+                        mca_mpool_base_module_t *mpool, int rcache_reg_flags,
+                        mca_rcache_base_module_t *rcache, opal_free_list_item_init_fn_t item_init,
+                        void *ctx)
 {
     /* alignment must be more than zero and power of two */
     if (frag_alignment <= 1 || (frag_alignment & (frag_alignment - 1))) {
@@ -126,8 +122,10 @@ int opal_free_list_init (opal_free_list_t *flist, size_t frag_size, size_t frag_
     }
 
     if (0 < payload_buffer_size) {
-        if (payload_buffer_alignment <= 1 || (payload_buffer_alignment & (payload_buffer_alignment - 1)))
+        if (payload_buffer_alignment <= 1
+            || (payload_buffer_alignment & (payload_buffer_alignment - 1))) {
             return OPAL_ERROR;
+        }
     }
 
     if (frag_class && frag_size < frag_class->cls_sizeof) {
@@ -155,13 +153,14 @@ int opal_free_list_init (opal_free_list_t *flist, size_t frag_size, size_t frag_
     flist->ctx = ctx;
 
     if (num_elements_to_alloc) {
-        return opal_free_list_grow_st (flist, num_elements_to_alloc, NULL);
+        return opal_free_list_grow_st(flist, num_elements_to_alloc, NULL);
     }
 
     return OPAL_SUCCESS;
 }
 
-int opal_free_list_grow_st (opal_free_list_t* flist, size_t num_elements, opal_free_list_item_t **item_out)
+int opal_free_list_grow_st(opal_free_list_t *flist, size_t num_elements,
+                           opal_free_list_item_t **item_out)
 {
     unsigned char *ptr, *payload_ptr = NULL;
     opal_free_list_memory_t *alloc_ptr;
@@ -169,8 +168,8 @@ int opal_free_list_grow_st (opal_free_list_t* flist, size_t num_elements, opal_f
     mca_rcache_base_registration_t *reg = NULL;
     int rc = OPAL_SUCCESS;
 
-    if (flist->fl_max_to_alloc && (flist->fl_num_allocated + num_elements) >
-        flist->fl_max_to_alloc) {
+    if (flist->fl_max_to_alloc
+        && (flist->fl_num_allocated + num_elements) > flist->fl_max_to_alloc) {
         num_elements = flist->fl_max_to_alloc - flist->fl_num_allocated;
     }
 
@@ -182,17 +181,17 @@ int opal_free_list_grow_st (opal_free_list_t* flist, size_t num_elements, opal_f
 
     /* NTH: calculate allocation alignment first as it might change the number of elements */
     if (0 != flist->fl_payload_buffer_size) {
-        elem_size = OPAL_ALIGN(flist->fl_payload_buffer_size,
-                               flist->fl_payload_buffer_alignment, size_t);
+        elem_size = OPAL_ALIGN(flist->fl_payload_buffer_size, flist->fl_payload_buffer_alignment,
+                               size_t);
 
         /* elem_size should not be 0 here */
-        assert (elem_size > 0);
+        assert(elem_size > 0);
 
         buffer_size = num_elements * elem_size;
         align = flist->fl_payload_buffer_alignment;
 
-        if (MCA_RCACHE_FLAGS_CUDA_REGISTER_MEM & flist->fl_rcache_reg_flags) {
-            size_t pagesize = opal_getpagesize ();
+        if (MCA_RCACHE_FLAGS_ACCELERATOR_REGISTER_MEM & flist->fl_rcache_reg_flags) {
+            size_t pagesize = opal_getpagesize();
             /* CUDA cannot handle registering overlapping regions, so make
              * sure each region is page sized and page aligned. */
             align = OPAL_ALIGN(align, pagesize, size_t);
@@ -204,8 +203,8 @@ int opal_free_list_grow_st (opal_free_list_t* flist, size_t num_elements, opal_f
     }
 
     /* calculate head allocation size */
-    alloc_size = num_elements * head_size + sizeof(opal_free_list_memory_t) +
-        flist->fl_frag_alignment;
+    alloc_size = num_elements * head_size + sizeof(opal_free_list_memory_t)
+                 + flist->fl_frag_alignment;
 
     alloc_ptr = (opal_free_list_memory_t *) malloc(alloc_size);
     if (OPAL_UNLIKELY(NULL == alloc_ptr)) {
@@ -214,18 +213,21 @@ int opal_free_list_grow_st (opal_free_list_t* flist, size_t num_elements, opal_f
 
     if (0 != flist->fl_payload_buffer_size) {
         /* allocate the rest from the mpool (or use memalign/malloc) */
-        payload_ptr = (unsigned char *) flist->fl_mpool->mpool_alloc(flist->fl_mpool, buffer_size, align, 0);
+        payload_ptr = (unsigned char *) flist->fl_mpool->mpool_alloc(flist->fl_mpool, buffer_size,
+                                                                     align, 0);
         if (NULL == payload_ptr) {
             free(alloc_ptr);
             return OPAL_ERR_TEMP_OUT_OF_RESOURCE;
         }
 
         if (flist->fl_rcache) {
-            rc = flist->fl_rcache->rcache_register (flist->fl_rcache, payload_ptr, num_elements * elem_size,
-                                                    flist->fl_rcache_reg_flags, MCA_RCACHE_ACCESS_ANY, &reg);
+            rc = flist->fl_rcache->rcache_register(flist->fl_rcache, payload_ptr,
+                                                   num_elements * elem_size,
+                                                   flist->fl_rcache_reg_flags,
+                                                   MCA_RCACHE_ACCESS_ANY, &reg);
             if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-                free (alloc_ptr);
-                flist->fl_mpool->mpool_free (flist->fl_mpool, payload_ptr);
+                free(alloc_ptr);
+                flist->fl_mpool->mpool_free(flist->fl_mpool, payload_ptr);
 
                 return rc;
             }
@@ -235,16 +237,16 @@ int opal_free_list_grow_st (opal_free_list_t* flist, size_t num_elements, opal_f
     /* make the alloc_ptr a list item, save the chunk in the allocations list,
      * and have ptr point to memory right after the list item structure */
     OBJ_CONSTRUCT(alloc_ptr, opal_free_list_item_t);
-    opal_list_append(&(flist->fl_allocations), (opal_list_item_t*)alloc_ptr);
+    opal_list_append(&(flist->fl_allocations), (opal_list_item_t *) alloc_ptr);
 
     alloc_ptr->registration = reg;
     alloc_ptr->ptr = payload_ptr;
 
-    ptr = (unsigned char*)alloc_ptr + sizeof(opal_free_list_memory_t);
-    ptr = OPAL_ALIGN_PTR(ptr, flist->fl_frag_alignment, unsigned char*);
+    ptr = (unsigned char *) alloc_ptr + sizeof(opal_free_list_memory_t);
+    ptr = OPAL_ALIGN_PTR(ptr, flist->fl_frag_alignment, unsigned char *);
 
-    for(size_t i = 0; i < num_elements ; ++i) {
-        opal_free_list_item_t* item = (opal_free_list_item_t*)ptr;
+    for (size_t i = 0; i < num_elements; ++i) {
+        opal_free_list_item_t *item = (opal_free_list_item_t *) ptr;
         item->registration = reg;
         item->ptr = payload_ptr;
 
@@ -255,7 +257,7 @@ int opal_free_list_grow_st (opal_free_list_t* flist, size_t num_elements, opal_f
         if (flist->item_init) {
             if (OPAL_SUCCESS != (rc = flist->item_init(item, flist->ctx))) {
                 num_elements = i;
-                OBJ_DESTRUCT (item);
+                OBJ_DESTRUCT(item);
                 break;
             }
         }
@@ -268,7 +270,7 @@ int opal_free_list_grow_st (opal_free_list_t* flist, size_t num_elements, opal_f
              * if one is available */
             *item_out = item;
         } else {
-            opal_lifo_push_atomic (&flist->super, &item->super);
+            opal_lifo_push_atomic(&flist->super, &item->super);
         }
 
         ptr += head_size;
@@ -279,8 +281,8 @@ int opal_free_list_grow_st (opal_free_list_t* flist, size_t num_elements, opal_f
 
     if (OPAL_SUCCESS != rc && 0 == num_elements) {
         /* couldn't initialize any items */
-        opal_list_remove_item (&flist->fl_allocations, (opal_list_item_t *) alloc_ptr);
-        opal_free_list_allocation_release (flist, alloc_ptr);
+        opal_list_remove_item(&flist->fl_allocations, (opal_list_item_t *) alloc_ptr);
+        opal_free_list_allocation_release(flist, alloc_ptr);
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
@@ -304,16 +306,16 @@ int opal_free_list_resize_mt(opal_free_list_t *flist, size_t size)
         return OPAL_SUCCESS;
     }
 
-    opal_mutex_lock (&flist->fl_lock);
+    opal_mutex_lock(&flist->fl_lock);
     do {
-        ret = opal_free_list_grow_st (flist, flist->fl_num_per_alloc, NULL);
+        ret = opal_free_list_grow_st(flist, flist->fl_num_per_alloc, NULL);
         if (OPAL_SUCCESS != ret) {
             break;
         }
 
-        inc_num = (ssize_t)size - (ssize_t)flist->fl_num_allocated;
+        inc_num = (ssize_t) size - (ssize_t) flist->fl_num_allocated;
     } while (inc_num > 0);
-    opal_mutex_unlock (&flist->fl_lock);
+    opal_mutex_unlock(&flist->fl_lock);
 
     return ret;
 }
diff --git a/opal/class/opal_free_list.h b/opal/class/opal_free_list.h
index ddc383fb912..6e956998ed9 100644
--- a/opal/class/opal_free_list.h
+++ b/opal/class/opal_free_list.h
@@ -26,9 +26,9 @@
 
 #include "opal_config.h"
 #include "opal/class/opal_lifo.h"
-#include "opal/prefetch.h"
-#include "opal/mca/threads/condition.h"
 #include "opal/constants.h"
+#include "opal/mca/threads/condition.h"
+#include "opal/prefetch.h"
 #include "opal/runtime/opal.h"
 
 BEGIN_C_DECLS
@@ -50,8 +50,7 @@ struct opal_free_list_item_t;
  * function should return OPAL_SUCCESS. On any error
  * opal_free_list_grow will stop initializing new items.
  */
-typedef int (*opal_free_list_item_init_fn_t) (
-        struct opal_free_list_item_t *item, void *ctx);
+typedef int (*opal_free_list_item_init_fn_t)(struct opal_free_list_item_t *item, void *ctx);
 
 struct opal_free_list_t {
     /** Items in a free list are stored last-in first-out */
@@ -97,8 +96,7 @@ typedef struct opal_free_list_t opal_free_list_t;
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_free_list_t);
 
 struct mca_mpool_base_registration_t;
-struct opal_free_list_item_t
-{
+struct opal_free_list_item_t {
     opal_list_item_t super;
     struct mca_rcache_base_registration_t *registration;
     void *ptr;
@@ -106,14 +104,14 @@ struct opal_free_list_item_t
 typedef struct opal_free_list_item_t opal_free_list_item_t;
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_free_list_item_t);
 
-
 /**
  * Initialize a free list.
  *
  * @param free_list                (IN)  Free list.
  * @param frag_size                (IN)  Size of each element - allocated by malloc.
  * @param frag_alignment           (IN)  Fragment alignment.
- * @param frag_class               (IN)  opal_class_t of element - used to initialize allocated elements.
+ * @param frag_class               (IN)  opal_class_t of element - used to initialize allocated
+ * elements.
  * @param payload_buffer_size      (IN)  Size of payload buffer - allocated from mpool.
  * @param payload_buffer_alignment (IN)  Payload buffer alignment.
  * @param num_elements_to_alloc    (IN)  Initial number of elements to allocate.
@@ -126,20 +124,14 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_free_list_item_t);
  * @param ctx                      (IN)  Initialization function context.
  */
 
-OPAL_DECLSPEC int opal_free_list_init (opal_free_list_t *free_list,
-                                       size_t frag_size,
-                                       size_t frag_alignment,
-                                       opal_class_t* frag_class,
-                                       size_t payload_buffer_size,
-                                       size_t payload_buffer_alignment,
-                                       int num_elements_to_alloc,
-                                       int max_elements_to_alloc,
-                                       int num_elements_per_alloc,
-                                       struct mca_mpool_base_module_t *mpool,
-                                       int rcache_reg_flags,
-                                       struct mca_rcache_base_module_t *rcache,
-                                       opal_free_list_item_init_fn_t item_init,
-                                       void *ctx);
+OPAL_DECLSPEC int opal_free_list_init(opal_free_list_t *free_list, size_t frag_size,
+                                      size_t frag_alignment, opal_class_t *frag_class,
+                                      size_t payload_buffer_size, size_t payload_buffer_alignment,
+                                      int num_elements_to_alloc, int max_elements_to_alloc,
+                                      int num_elements_per_alloc,
+                                      struct mca_mpool_base_module_t *mpool, int rcache_reg_flags,
+                                      struct mca_rcache_base_module_t *rcache,
+                                      opal_free_list_item_init_fn_t item_init, void *ctx);
 
 /**
  * Grow the free list by at most num_elements elements.
@@ -163,7 +155,8 @@ OPAL_DECLSPEC int opal_free_list_init (opal_free_list_t *free_list,
  * and assumes NULL is an out of memory condition (which it wasn't necessarily
  * before this parameter was added).
  */
-OPAL_DECLSPEC int opal_free_list_grow_st (opal_free_list_t *flist, size_t num_elements, opal_free_list_item_t **item_out);
+OPAL_DECLSPEC int opal_free_list_grow_st(opal_free_list_t *flist, size_t num_elements,
+                                         opal_free_list_item_t **item_out);
 
 /**
  * Grow the free list to be at least size elements.
@@ -179,60 +172,56 @@ OPAL_DECLSPEC int opal_free_list_grow_st (opal_free_list_t *flist, size_t num_el
  * chunks). This function is thread-safe and will obtain the free list lock before
  * growing the free list.
  */
-OPAL_DECLSPEC int opal_free_list_resize_mt (opal_free_list_t *flist, size_t size);
-
+OPAL_DECLSPEC int opal_free_list_resize_mt(opal_free_list_t *flist, size_t size);
 
 /**
- * Attemp to obtain an item from a free list.
+ * Attempt to obtain an item from a free list.
  *
  * @param fl (IN)        Free list.
  * @param item (OUT)     Allocated item.
  *
  * If the requested item is not available the free list is grown to
- * accomodate the request - unless the max number of allocations has
+ * accommodate the request - unless the max number of allocations has
  * been reached.  If this is the case - a NULL pointer is returned
  * to the caller. This function comes in three flavor: thread safe
  * (opal_free_list_get_mt), single threaded (opal_free_list_get_st),
  * and opal_using_threads conditioned (opal_free_list_get).
  */
-static inline opal_free_list_item_t *opal_free_list_get_mt (opal_free_list_t *flist)
+static inline opal_free_list_item_t *opal_free_list_get_mt(opal_free_list_t *flist)
 {
-    opal_free_list_item_t *item =
-        (opal_free_list_item_t*) opal_lifo_pop_atomic (&flist->super);
+    opal_free_list_item_t *item = (opal_free_list_item_t *) opal_lifo_pop_atomic(&flist->super);
 
     if (OPAL_UNLIKELY(NULL == item)) {
-        opal_mutex_lock (&flist->fl_lock);
-        opal_free_list_grow_st (flist, flist->fl_num_per_alloc, &item);
-        opal_mutex_unlock (&flist->fl_lock);
+        opal_mutex_lock(&flist->fl_lock);
+        opal_free_list_grow_st(flist, flist->fl_num_per_alloc, &item);
+        opal_mutex_unlock(&flist->fl_lock);
     }
 
     return item;
 }
 
-static inline opal_free_list_item_t *opal_free_list_get_st (opal_free_list_t *flist)
+static inline opal_free_list_item_t *opal_free_list_get_st(opal_free_list_t *flist)
 {
-    opal_free_list_item_t *item =
-        (opal_free_list_item_t*) opal_lifo_pop_st (&flist->super);
+    opal_free_list_item_t *item = (opal_free_list_item_t *) opal_lifo_pop_st(&flist->super);
 
     if (OPAL_UNLIKELY(NULL == item)) {
-        opal_free_list_grow_st (flist, flist->fl_num_per_alloc, &item);
+        opal_free_list_grow_st(flist, flist->fl_num_per_alloc, &item);
     }
 
     return item;
 }
 
-static inline opal_free_list_item_t *opal_free_list_get (opal_free_list_t *flist)
+static inline opal_free_list_item_t *opal_free_list_get(opal_free_list_t *flist)
 {
-    if (opal_using_threads ()) {
-        return opal_free_list_get_mt (flist);
+    if (opal_using_threads()) {
+        return opal_free_list_get_mt(flist);
     }
 
-    return opal_free_list_get_st (flist);
+    return opal_free_list_get_st(flist);
 }
 
 /** compatibility macro */
-#define OPAL_FREE_LIST_GET(fl, item)            \
-    (item) = opal_free_list_get (fl)
+#define OPAL_FREE_LIST_GET(fl, item) (item) = opal_free_list_get(fl)
 
 /**
  * Blocking call to obtain an item from a free list.
@@ -241,77 +230,74 @@ static inline opal_free_list_item_t *opal_free_list_get (opal_free_list_t *flist
  * @param item (OUT)     Allocated item.
  *
  * If the requested item is not available the free list is grown to
- * accomodate the request - unless the max number of allocations has
+ * accommodate the request - unless the max number of allocations has
  * been reached. In this case the caller is blocked until an item
  * is returned to the list.
  */
 
 /** compatibility macro */
-#define OPAL_FREE_LIST_WAIT(fl, item)           \
-    (item) = opal_free_list_wait (fl)
+#define OPAL_FREE_LIST_WAIT(fl, item) (item) = opal_free_list_wait(fl)
 
-static inline opal_free_list_item_t *opal_free_list_wait_mt (opal_free_list_t *fl)
+static inline opal_free_list_item_t *opal_free_list_wait_mt(opal_free_list_t *fl)
 {
-    opal_free_list_item_t *item =
-        (opal_free_list_item_t *) opal_lifo_pop_atomic (&fl->super);
+    opal_free_list_item_t *item = (opal_free_list_item_t *) opal_lifo_pop_atomic(&fl->super);
 
     while (NULL == item) {
-        if (!opal_mutex_trylock (&fl->fl_lock)) {
-            if (fl->fl_max_to_alloc <= fl->fl_num_allocated ||
-                OPAL_SUCCESS != opal_free_list_grow_st (fl, fl->fl_num_per_alloc, &item)) {
+        if (!opal_mutex_trylock(&fl->fl_lock)) {
+            if (fl->fl_max_to_alloc <= fl->fl_num_allocated
+                || OPAL_SUCCESS != opal_free_list_grow_st(fl, fl->fl_num_per_alloc, &item)) {
                 fl->fl_num_waiting++;
-                opal_condition_wait (&fl->fl_condition, &fl->fl_lock);
+                opal_condition_wait(&fl->fl_condition, &fl->fl_lock);
                 fl->fl_num_waiting--;
             } else {
                 if (0 < fl->fl_num_waiting) {
                     if (1 == fl->fl_num_waiting) {
-                        opal_condition_signal (&fl->fl_condition);
+                        opal_condition_signal(&fl->fl_condition);
                     } else {
-                        opal_condition_broadcast (&fl->fl_condition);
+                        opal_condition_broadcast(&fl->fl_condition);
                     }
                 }
             }
         } else {
-            /* If I wasn't able to get the lock in the begining when I finaly grab it
-             * the one holding the lock in the begining already grow the list. I will
+            /* If I wasn't able to get the lock in the beginning when I finally grab it
+             * the one holding the lock in the beginning already grow the list. I will
              * release the lock and try to get a new element until I succeed.
              */
-            opal_mutex_lock (&fl->fl_lock);
+            opal_mutex_lock(&fl->fl_lock);
         }
-        opal_mutex_unlock (&fl->fl_lock);
+        opal_mutex_unlock(&fl->fl_lock);
         if (NULL == item) {
-            item = (opal_free_list_item_t *) opal_lifo_pop_atomic (&fl->super);
+            item = (opal_free_list_item_t *) opal_lifo_pop_atomic(&fl->super);
         }
     }
 
     return item;
 }
 
-static inline opal_free_list_item_t *opal_free_list_wait_st (opal_free_list_t *fl)
+static inline opal_free_list_item_t *opal_free_list_wait_st(opal_free_list_t *fl)
 {
-    opal_free_list_item_t *item =
-        (opal_free_list_item_t *) opal_lifo_pop (&fl->super);
+    opal_free_list_item_t *item = (opal_free_list_item_t *) opal_lifo_pop(&fl->super);
 
     while (NULL == item) {
-        if (fl->fl_max_to_alloc <= fl->fl_num_allocated ||
-            OPAL_SUCCESS != opal_free_list_grow_st (fl, fl->fl_num_per_alloc, &item)) {
+        if (fl->fl_max_to_alloc <= fl->fl_num_allocated
+            || OPAL_SUCCESS != opal_free_list_grow_st(fl, fl->fl_num_per_alloc, &item)) {
             /* try to make progress */
-            opal_progress ();
+            opal_progress();
         }
         if (NULL == item) {
-            item = (opal_free_list_item_t *) opal_lifo_pop (&fl->super);
+            item = (opal_free_list_item_t *) opal_lifo_pop(&fl->super);
         }
     }
 
     return item;
 }
 
-static inline opal_free_list_item_t *opal_free_list_wait (opal_free_list_t *fl)
+static inline opal_free_list_item_t *opal_free_list_wait(opal_free_list_t *fl)
 {
-    if (opal_using_threads ()) {
-        return opal_free_list_wait_mt (fl);
+    if (opal_using_threads()) {
+        return opal_free_list_wait_mt(fl);
     } else {
-        return opal_free_list_wait_st (fl);
+        return opal_free_list_wait_st(fl);
     }
 }
 
@@ -322,54 +308,49 @@ static inline opal_free_list_item_t *opal_free_list_wait (opal_free_list_t *fl)
  * @param item (OUT)     Allocated item.
  *
  */
-static inline void opal_free_list_return_mt (opal_free_list_t *flist,
-                                             opal_free_list_item_t *item)
+static inline void opal_free_list_return_mt(opal_free_list_t *flist, opal_free_list_item_t *item)
 {
-    opal_list_item_t* original;
+    opal_list_item_t *original;
 
-    original = opal_lifo_push_atomic (&flist->super, &item->super);
+    original = opal_lifo_push_atomic(&flist->super, &item->super);
     if (&flist->super.opal_lifo_ghost == original) {
         if (flist->fl_num_waiting > 0) {
             /* one one item is being returned so it doesn't make sense to wake
              * more than a single waiting thread. additionally, posix thread
              * semantics do not require that the lock be held to signal the
              * condition variable. */
-            opal_condition_signal (&flist->fl_condition);
+            opal_condition_signal(&flist->fl_condition);
         }
     }
 }
 
-static inline void opal_free_list_return_st (opal_free_list_t *flist,
-                                             opal_free_list_item_t *item)
+static inline void opal_free_list_return_st(opal_free_list_t *flist, opal_free_list_item_t *item)
 {
-    opal_list_item_t* original;
+    opal_list_item_t *original;
 
-    original = opal_lifo_push_st (&flist->super, &item->super);
+    original = opal_lifo_push_st(&flist->super, &item->super);
     if (&flist->super.opal_lifo_ghost == original) {
         if (flist->fl_num_waiting > 0) {
             /* one one item is being returned so it doesn't make sense to wake
              * more than a single waiting thread. additionally, posix thread
              * semantics do not require that the lock be held to signal the
              * condition variable. */
-            opal_condition_signal (&flist->fl_condition);
+            opal_condition_signal(&flist->fl_condition);
         }
     }
 }
 
-static inline void opal_free_list_return (opal_free_list_t *flist,
-                                          opal_free_list_item_t *item)
+static inline void opal_free_list_return(opal_free_list_t *flist, opal_free_list_item_t *item)
 {
-    if (opal_using_threads ()) {
-        opal_free_list_return_mt (flist, item);
+    if (opal_using_threads()) {
+        opal_free_list_return_mt(flist, item);
     } else {
-        opal_free_list_return_st (flist, item);
+        opal_free_list_return_st(flist, item);
     }
 }
 
 /** compatibility macro */
-#define OPAL_FREE_LIST_RETURN(fl, item)         \
-    opal_free_list_return (fl, item)
+#define OPAL_FREE_LIST_RETURN(fl, item) opal_free_list_return(fl, item)
 
 END_C_DECLS
 #endif
-
diff --git a/opal/class/opal_graph.c b/opal/class/opal_graph.c
index 8ee0e88702f..aa4f787fd74 100644
--- a/opal/class/opal_graph.c
+++ b/opal/class/opal_graph.c
@@ -23,9 +23,9 @@
 
 #include "opal_config.h"
 
+#include "opal/class/opal_graph.h"
 #include "opal/class/opal_list.h"
 #include "opal/constants.h"
-#include "opal/class/opal_graph.h"
 #include "opal/util/output.h"
 
 static int compare_vertex_distance(const void *item1, const void *item2);
@@ -34,49 +34,28 @@ static int compare_vertex_distance(const void *item1, const void *item2);
  *  Graph classes
  */
 
-
 static void opal_graph_vertex_construct(opal_graph_vertex_t *vertex);
 static void opal_graph_vertex_destruct(opal_graph_vertex_t *vertex);
 
-OBJ_CLASS_INSTANCE(
-    opal_graph_vertex_t,
-    opal_list_item_t,
-    opal_graph_vertex_construct,
-    opal_graph_vertex_destruct
-);
-
+OBJ_CLASS_INSTANCE(opal_graph_vertex_t, opal_list_item_t, opal_graph_vertex_construct,
+                   opal_graph_vertex_destruct);
 
 static void opal_graph_edge_construct(opal_graph_edge_t *edge);
 static void opal_graph_edge_destruct(opal_graph_edge_t *edge);
 
-OBJ_CLASS_INSTANCE(
-    opal_graph_edge_t,
-    opal_list_item_t,
-    opal_graph_edge_construct,
-    opal_graph_edge_destruct
-);
+OBJ_CLASS_INSTANCE(opal_graph_edge_t, opal_list_item_t, opal_graph_edge_construct,
+                   opal_graph_edge_destruct);
 
 static void opal_graph_construct(opal_graph_t *graph);
 static void opal_graph_destruct(opal_graph_t *graph);
 
-OBJ_CLASS_INSTANCE(
-    opal_graph_t,
-    opal_object_t,
-    opal_graph_construct,
-    opal_graph_destruct
-);
+OBJ_CLASS_INSTANCE(opal_graph_t, opal_object_t, opal_graph_construct, opal_graph_destruct);
 
 static void opal_adjacency_list_construct(opal_adjacency_list_t *aj_list);
 static void opal_adjacency_list_destruct(opal_adjacency_list_t *aj_list);
 
-OBJ_CLASS_INSTANCE(
-    opal_adjacency_list_t,
-    opal_list_item_t,
-    opal_adjacency_list_construct,
-    opal_adjacency_list_destruct
-);
-
-
+OBJ_CLASS_INSTANCE(opal_adjacency_list_t, opal_list_item_t, opal_adjacency_list_construct,
+                   opal_adjacency_list_destruct);
 
 /*
  *
@@ -112,7 +91,6 @@ static void opal_graph_vertex_destruct(opal_graph_vertex_t *vertex)
     vertex->print_vertex = NULL;
 }
 
-
 /*
  *
  *      opal_graph_edge_t interface
@@ -127,7 +105,6 @@ static void opal_graph_edge_construct(opal_graph_edge_t *edge)
     edge->in_adj_list = NULL;
 }
 
-
 static void opal_graph_edge_destruct(opal_graph_edge_t *edge)
 {
     edge->end = NULL;
@@ -136,14 +113,12 @@ static void opal_graph_edge_destruct(opal_graph_edge_t *edge)
     edge->in_adj_list = NULL;
 }
 
-
 /*
  *
  *     opal_graph_t  interface
  *
  */
 
-
 static void opal_graph_construct(opal_graph_t *graph)
 {
     graph->adjacency_list = OBJ_NEW(opal_list_t);
@@ -172,8 +147,8 @@ static void opal_adjacency_list_construct(opal_adjacency_list_t *aj_list)
 
 static void opal_adjacency_list_destruct(opal_adjacency_list_t *aj_list)
 {
-   aj_list->vertex = NULL;
-   OPAL_LIST_RELEASE(aj_list->edges);
+    aj_list->vertex = NULL;
+    OPAL_LIST_RELEASE(aj_list->edges);
 }
 
 /**
@@ -191,17 +166,17 @@ static void delete_all_edges_conceded_to_vertex(opal_graph_t *graph, opal_graph_
     /**
      * for all the adjacency list in the graph
      */
-    OPAL_LIST_FOREACH(aj_list, graph->adjacency_list, opal_adjacency_list_t) {
+    OPAL_LIST_FOREACH (aj_list, graph->adjacency_list, opal_adjacency_list_t) {
         /**
          * for all the edges in the adjacency list
          */
-        OPAL_LIST_FOREACH_SAFE(edge, next, aj_list->edges, opal_graph_edge_t) {
+        OPAL_LIST_FOREACH_SAFE (edge, next, aj_list->edges, opal_graph_edge_t) {
             /**
              * if the edge is ended in the vertex
              */
             if (edge->end == vertex) {
                 /* Delete this edge  */
-                opal_list_remove_item(edge->in_adj_list->edges, (opal_list_item_t*)edge);
+                opal_list_remove_item(edge->in_adj_list->edges, (opal_list_item_t *) edge);
                 /* distract this edge */
                 OBJ_RELEASE(edge);
             }
@@ -218,30 +193,29 @@ static void delete_all_edges_conceded_to_vertex(opal_graph_t *graph, opal_graph_
  */
 void opal_graph_add_vertex(opal_graph_t *graph, opal_graph_vertex_t *vertex)
 {
-   opal_adjacency_list_t *aj_list;
-
-   /**
-    * Find if this vertex already exists in the graph.
-    */
-   OPAL_LIST_FOREACH(aj_list, graph->adjacency_list, opal_adjacency_list_t) {
-       if (aj_list->vertex == vertex) {
-           /* If this vertex exists, dont do anything. */
-           return;
-       }
-   }
-   /* Construct a new adjacency list */
-   aj_list = OBJ_NEW(opal_adjacency_list_t);
-   aj_list->vertex = vertex;
-   /* point the vertex to the adjacency list of the vertex (for easy searching) */
-   vertex->in_adj_list = aj_list;
-   /* Append the new creates adjacency list to the graph */
-   opal_list_append(graph->adjacency_list, (opal_list_item_t*)aj_list);
-   /* point the vertex to the graph it belongs to (mostly for debug uses)*/
-   vertex->in_graph = graph;
-   /* increase the number of vertices in the graph */
-   graph->number_of_vertices++;
-}
+    opal_adjacency_list_t *aj_list;
 
+    /**
+     * Find if this vertex already exists in the graph.
+     */
+    OPAL_LIST_FOREACH (aj_list, graph->adjacency_list, opal_adjacency_list_t) {
+        if (aj_list->vertex == vertex) {
+            /* If this vertex exists, dont do anything. */
+            return;
+        }
+    }
+    /* Construct a new adjacency list */
+    aj_list = OBJ_NEW(opal_adjacency_list_t);
+    aj_list->vertex = vertex;
+    /* point the vertex to the adjacency list of the vertex (for easy searching) */
+    vertex->in_adj_list = aj_list;
+    /* Append the new creates adjacency list to the graph */
+    opal_list_append(graph->adjacency_list, (opal_list_item_t *) aj_list);
+    /* point the vertex to the graph it belongs to (mostly for debug uses)*/
+    vertex->in_graph = graph;
+    /* increase the number of vertices in the graph */
+    graph->number_of_vertices++;
+}
 
 /**
  * This graph API adds an edge (connection between two
@@ -256,14 +230,13 @@ void opal_graph_add_vertex(opal_graph_t *graph, opal_graph_vertex_t *vertex)
  */
 int opal_graph_add_edge(opal_graph_t *graph, opal_graph_edge_t *edge)
 {
-    opal_adjacency_list_t *aj_list, *start_aj_list= NULL;
+    opal_adjacency_list_t *aj_list, *start_aj_list = NULL;
     bool end_found = false;
 
-
     /**
      * find the vertices that this edge should connect.
      */
-    OPAL_LIST_FOREACH(aj_list, graph->adjacency_list, opal_adjacency_list_t) {
+    OPAL_LIST_FOREACH (aj_list, graph->adjacency_list, opal_adjacency_list_t) {
         if (aj_list->vertex == edge->start) {
             start_aj_list = aj_list;
         }
@@ -279,15 +252,14 @@ int opal_graph_add_edge(opal_graph_t *graph, opal_graph_edge_t *edge)
         return OPAL_ERROR;
     }
     /* point the edge to the adjacency list of the start vertex (for easy search) */
-    edge->in_adj_list=start_aj_list;
+    edge->in_adj_list = start_aj_list;
     /* append the edge to the adjacency list of the start vertex */
-    opal_list_append(start_aj_list->edges, (opal_list_item_t*)edge);
+    opal_list_append(start_aj_list->edges, (opal_list_item_t *) edge);
     /* increase the graph size */
     graph->number_of_edges++;
     return OPAL_SUCCESS;
 }
 
-
 /**
  * This graph API removes an edge (a connection between two
  * vertices) from the graph. The most common use of this API is
@@ -299,10 +271,10 @@ int opal_graph_add_edge(opal_graph_t *graph, opal_graph_edge_t *edge)
  * @param graph The graph that this edge will be remove from.
  * @param edge the edge that we want to remove.
  */
-void opal_graph_remove_edge (opal_graph_t *graph, opal_graph_edge_t *edge)
+void opal_graph_remove_edge(opal_graph_t *graph, opal_graph_edge_t *edge)
 {
     /* remove the edge from the list it belongs to */
-    opal_list_remove_item(edge->in_adj_list->edges, (opal_list_item_t*)edge);
+    opal_list_remove_item(edge->in_adj_list->edges, (opal_list_item_t *) edge);
     /* decrees the number of edges in the graph */
     graph->number_of_edges--;
     /* Note that the edge is not destructed - the caller should destruct this edge. */
@@ -328,7 +300,7 @@ void opal_graph_remove_vertex(opal_graph_t *graph, opal_graph_vertex_t *vertex)
      * remove the adjscency list of this vertex from the graph and
      * destruct it.
      */
-    opal_list_remove_item(graph->adjacency_list, (opal_list_item_t*)adj_list);
+    opal_list_remove_item(graph->adjacency_list, (opal_list_item_t *) adj_list);
     OBJ_RELEASE(adj_list);
     /**
      * delete all the edges that connected *to* the vertex.
@@ -351,7 +323,8 @@ void opal_graph_remove_vertex(opal_graph_t *graph, opal_graph_vertex_t *vertex)
  *         vertices or infinity if the vertices are not
  *         connected.
  */
-uint32_t opal_graph_adjacent(opal_graph_t *graph, opal_graph_vertex_t *vertex1, opal_graph_vertex_t *vertex2)
+uint32_t opal_graph_adjacent(opal_graph_t *graph, opal_graph_vertex_t *vertex1,
+                             opal_graph_vertex_t *vertex2)
 {
     opal_adjacency_list_t *adj_list;
     opal_graph_edge_t *edge;
@@ -360,14 +333,16 @@ uint32_t opal_graph_adjacent(opal_graph_t *graph, opal_graph_vertex_t *vertex1,
      * Verify that the first vertex belongs to the graph.
      */
     if (graph != vertex1->in_graph) {
-        OPAL_OUTPUT((0,"opal_graph_adjacent 1 Vertex1 %p not in the graph %p\n",(void *)vertex1,(void *)graph));
+        OPAL_OUTPUT((0, "opal_graph_adjacent 1 Vertex1 %p not in the graph %p\n", (void *) vertex1,
+                     (void *) graph));
         return DISTANCE_INFINITY;
     }
     /**
      * Verify that the second vertex belongs to the graph.
      */
     if (graph != vertex2->in_graph) {
-        OPAL_OUTPUT((0,"opal_graph_adjacent 2 Vertex2 %p not in the graph %p\n",(void *)vertex2,(void *)graph));
+        OPAL_OUTPUT((0, "opal_graph_adjacent 2 Vertex2 %p not in the graph %p\n", (void *) vertex2,
+                     (void *) graph));
         return DISTANCE_INFINITY;
     }
     /**
@@ -382,9 +357,10 @@ uint32_t opal_graph_adjacent(opal_graph_t *graph, opal_graph_vertex_t *vertex1,
      * vertex.
      */
     adj_list = (opal_adjacency_list_t *) vertex1->in_adj_list;
-    OPAL_LIST_FOREACH(edge, adj_list->edges, opal_graph_edge_t) {
+    OPAL_LIST_FOREACH (edge, adj_list->edges, opal_graph_edge_t) {
         if (edge->end == vertex2) {
-            /* if the second vertex was found in the adjacency list of the first one, return the weight */
+            /* if the second vertex was found in the adjacency list of the first one, return the
+             * weight */
             return edge->weight;
         }
     }
@@ -434,7 +410,7 @@ opal_graph_vertex_t *opal_graph_find_vertex(opal_graph_t *graph, void *vertex_da
     /**
      * Run on all the vertices of the graph
      */
-    OPAL_LIST_FOREACH(aj_list, graph->adjacency_list, opal_adjacency_list_t) {
+    OPAL_LIST_FOREACH (aj_list, graph->adjacency_list, opal_adjacency_list_t) {
         if (NULL != aj_list->vertex->compare_vertex) {
             /* if the vertex data of a vertex is equal to the vertex data */
             if (0 == aj_list->vertex->compare_vertex(aj_list->vertex->vertex_data, vertex_data)) {
@@ -447,7 +423,6 @@ opal_graph_vertex_t *opal_graph_find_vertex(opal_graph_t *graph, void *vertex_da
     return NULL;
 }
 
-
 /**
  * This graph API returns an array of pointers of all the
  * vertices in the graph.
@@ -471,9 +446,9 @@ int opal_graph_get_graph_vertices(opal_graph_t *graph, opal_pointer_array_t *ver
         return 0;
     }
     /* Run on all the vertices of the graph */
-    OPAL_LIST_FOREACH(aj_list, graph->adjacency_list, opal_adjacency_list_t) {
+    OPAL_LIST_FOREACH (aj_list, graph->adjacency_list, opal_adjacency_list_t) {
         /* Add the vertex to the vertices array */
-        opal_pointer_array_add(vertices_list,(void *)aj_list->vertex);
+        opal_pointer_array_add(vertices_list, (void *) aj_list->vertex);
     }
     /* return the vertices list */
     return graph->number_of_vertices;
@@ -492,7 +467,8 @@ int opal_graph_get_graph_vertices(opal_graph_t *graph, opal_pointer_array_t *ver
  *
  * @return int the number of adjacents in the list.
  */
-int opal_graph_get_adjacent_vertices(opal_graph_t *graph, opal_graph_vertex_t *vertex, opal_value_array_t *adjacents)
+int opal_graph_get_adjacent_vertices(opal_graph_t *graph, opal_graph_vertex_t *vertex,
+                                     opal_value_array_t *adjacents)
 {
     opal_adjacency_list_t *adj_list;
     opal_graph_edge_t *edge;
@@ -503,7 +479,7 @@ int opal_graph_get_adjacent_vertices(opal_graph_t *graph, opal_graph_vertex_t *v
      * Verify that the vertex belongs to the graph.
      */
     if (graph != vertex->in_graph) {
-        OPAL_OUTPUT((0,"Vertex %p not in the graph %p\n", (void *)vertex, (void *)graph));
+        OPAL_OUTPUT((0, "Vertex %p not in the graph %p\n", (void *) vertex, (void *) graph));
         return 0;
     }
     /**
@@ -513,7 +489,7 @@ int opal_graph_get_adjacent_vertices(opal_graph_t *graph, opal_graph_vertex_t *v
     /* find the number of adjcents of this vertex */
     adjacents_number = opal_list_get_size(adj_list->edges);
     /* Run on all the edges from this vertex */
-    OPAL_LIST_FOREACH(edge, adj_list->edges, opal_graph_edge_t) {
+    OPAL_LIST_FOREACH (edge, adj_list->edges, opal_graph_edge_t) {
         /* assign vertices and their weight in the adjcents list */
         distance_from.vertex = edge->end;
         distance_from.weight = edge->weight;
@@ -533,7 +509,8 @@ int opal_graph_get_adjacent_vertices(opal_graph_t *graph, opal_graph_vertex_t *v
  * @return uint32_t the distance between the two vertices.
  */
 
-uint32_t opal_graph_spf(opal_graph_t *graph, opal_graph_vertex_t *vertex1, opal_graph_vertex_t *vertex2)
+uint32_t opal_graph_spf(opal_graph_t *graph, opal_graph_vertex_t *vertex1,
+                        opal_graph_vertex_t *vertex2)
 {
     opal_value_array_t *distance_array;
     uint32_t items_in_distance_array, spf = DISTANCE_INFINITY;
@@ -544,14 +521,16 @@ uint32_t opal_graph_spf(opal_graph_t *graph, opal_graph_vertex_t *vertex1, opal_
      * Verify that the first vertex belongs to the graph.
      */
     if (graph != vertex1->in_graph) {
-        OPAL_OUTPUT((0,"opal_graph_spf 1 Vertex1 %p not in the graph %p\n",(void *)vertex1,(void *)graph));
+        OPAL_OUTPUT((0, "opal_graph_spf 1 Vertex1 %p not in the graph %p\n", (void *) vertex1,
+                     (void *) graph));
         return DISTANCE_INFINITY;
     }
     /**
      * Verify that the second vertex belongs to the graph.
      */
     if (graph != vertex2->in_graph) {
-        OPAL_OUTPUT((0,"opal_graph_spf 2 Vertex2 %p not in the graph %p\n",(void *)vertex2,(void *)graph));
+        OPAL_OUTPUT((0, "opal_graph_spf 2 Vertex2 %p not in the graph %p\n", (void *) vertex2,
+                     (void *) graph));
         return DISTANCE_INFINITY;
     }
     /**
@@ -559,7 +538,7 @@ uint32_t opal_graph_spf(opal_graph_t *graph, opal_graph_vertex_t *vertex1, opal_
      */
     distance_array = OBJ_NEW(opal_value_array_t);
     opal_value_array_init(distance_array, sizeof(vertex_distance_from_t));
-    opal_value_array_reserve(distance_array,50);
+    opal_value_array_reserve(distance_array, 50);
     items_in_distance_array = opal_graph_dijkstra(graph, vertex1, distance_array);
     /**
      * find the end vertex in the distance array that Dijkstra
@@ -595,8 +574,8 @@ static int compare_vertex_distance(const void *item1, const void *item2)
     vertex_distance_from_t *vertex_dist1, *vertex_dist2;
 
     /* convert the void pointers to vertex distance pointers. */
-    vertex_dist1 = (vertex_distance_from_t *)item1;
-    vertex_dist2 = (vertex_distance_from_t *)item2;
+    vertex_dist1 = (vertex_distance_from_t *) item1;
+    vertex_dist2 = (vertex_distance_from_t *) item2;
 
     /* If the first item weight is higher then the second item weight return 1*/
     if (vertex_dist1->weight > vertex_dist2->weight) {
@@ -610,7 +589,6 @@ static int compare_vertex_distance(const void *item1, const void *item2)
     return -1;
 }
 
-
 /**
  * This graph API returns the distance (weight) from a reference
  * vertex to all other vertices in the graph using the Dijkstra
@@ -623,7 +601,8 @@ static int compare_vertex_distance(const void *item1, const void *item2)
  *
  * @return uint32_t the size of the distance array
  */
-uint32_t opal_graph_dijkstra(opal_graph_t *graph, opal_graph_vertex_t *vertex, opal_value_array_t *distance_array)
+uint32_t opal_graph_dijkstra(opal_graph_t *graph, opal_graph_vertex_t *vertex,
+                             opal_value_array_t *distance_array)
 {
     int graph_order;
     vertex_distance_from_t *Q, *q_start, *current_vertex;
@@ -632,23 +611,23 @@ uint32_t opal_graph_dijkstra(opal_graph_t *graph, opal_graph_vertex_t *vertex, o
     int i;
     uint32_t weight;
 
-
     /**
      * Verify that the reference vertex belongs to the graph.
      */
     if (graph != vertex->in_graph) {
-        OPAL_OUTPUT((0,"opal:graph:dijkstra: vertex %p not in the graph %p\n",(void *)vertex,(void *)graph));
+        OPAL_OUTPUT((0, "opal:graph:dijkstra: vertex %p not in the graph %p\n", (void *) vertex,
+                     (void *) graph));
         return 0;
     }
     /* get the order of the graph and allocate a working queue accordingly */
     graph_order = opal_graph_get_order(graph);
-    Q = (vertex_distance_from_t *)malloc(graph_order * sizeof(vertex_distance_from_t));
+    Q = (vertex_distance_from_t *) malloc(graph_order * sizeof(vertex_distance_from_t));
     /* assign a pointer to the start of the queue */
     q_start = Q;
     /* run on all the vertices of the graph */
     i = 0;
-    OPAL_LIST_FOREACH(adj_list, graph->adjacency_list, opal_adjacency_list_t) {
-        /* insert the vertices pointes to the working queue */
+    OPAL_LIST_FOREACH (adj_list, graph->adjacency_list, opal_adjacency_list_t) {
+        /* insert the vertices points to the working queue */
         Q[i].vertex = adj_list->vertex;
         /**
          * assign an infinity distance to all the vertices in the queue
@@ -667,7 +646,8 @@ uint32_t opal_graph_dijkstra(opal_graph_t *graph, opal_graph_vertex_t *vertex, o
         q_start++;
         /* decrees the number of vertices in the queue */
         number_of_items_in_q--;
-        /* find the distance of all other vertices in the queue from the first vertex in the queue */
+        /* find the distance of all other vertices in the queue from the first vertex in the queue
+         */
         for (i = 0; i < number_of_items_in_q; i++) {
             weight = opal_graph_adjacent(graph, current_vertex->vertex, q_start[i].vertex);
             /**
@@ -682,11 +662,12 @@ uint32_t opal_graph_dijkstra(opal_graph_t *graph, opal_graph_vertex_t *vertex, o
             }
         }
         /* sort again the working queue */
-        qsort(q_start, number_of_items_in_q, sizeof(vertex_distance_from_t), compare_vertex_distance);
+        qsort(q_start, number_of_items_in_q, sizeof(vertex_distance_from_t),
+              compare_vertex_distance);
     }
     /* copy the working queue the the returned distance array */
-    for (i = 0; i < graph_order-1; i++) {
-        opal_value_array_append_item(distance_array, (void *)&(Q[i+1]));
+    for (i = 0; i < graph_order - 1; i++) {
+        opal_value_array_append_item(distance_array, (void *) &(Q[i + 1]));
     }
     /* free the working queue */
     free(Q);
@@ -694,10 +675,9 @@ uint32_t opal_graph_dijkstra(opal_graph_t *graph, opal_graph_vertex_t *vertex, o
     return graph_order - 1;
 }
 
-
 /**
  * This graph API duplicates a graph. Note that this API does
- * not copy the graph but builds a new graph while coping just
+ * not copy the graph but builds a new graph while copying just
  * the vertex data.
  *
  * @param dest The new created graph.
@@ -712,7 +692,7 @@ void opal_graph_duplicate(opal_graph_t **dest, opal_graph_t *src)
     /* construct a new graph */
     *dest = OBJ_NEW(opal_graph_t);
     /* Run on all the vertices of the src graph */
-    OPAL_LIST_FOREACH(aj_list, src->adjacency_list, opal_adjacency_list_t) {
+    OPAL_LIST_FOREACH (aj_list, src->adjacency_list, opal_adjacency_list_t) {
         /* for each vertex in the src graph, construct a new vertex */
         vertex = OBJ_NEW(opal_graph_vertex_t);
         /* associate the new vertex to a vertex from the original graph */
@@ -741,14 +721,15 @@ void opal_graph_duplicate(opal_graph_t **dest, opal_graph_t *src)
      * Now, copy all the edges from the source graph
      */
     /* Run on all the adjscency lists in the graph */
-    OPAL_LIST_FOREACH(aj_list, src->adjacency_list, opal_adjacency_list_t) {
+    OPAL_LIST_FOREACH (aj_list, src->adjacency_list, opal_adjacency_list_t) {
         /* for all the edges in the adjscency list */
-        OPAL_LIST_FOREACH(edge, aj_list->edges, opal_graph_edge_t) {
+        OPAL_LIST_FOREACH (edge, aj_list->edges, opal_graph_edge_t) {
             /* construct new edge for the new graph */
             new_edge = OBJ_NEW(opal_graph_edge_t);
             /* copy the edge weight from the original edge */
             new_edge->weight = edge->weight;
-            /* connect the new edge according to start and end associations to the vertices of the src graph */
+            /* connect the new edge according to start and end associations to the vertices of the
+             * src graph */
             new_edge->start = edge->start->sibling;
             new_edge->end = edge->end->sibling;
             /* add the new edge to the new graph */
@@ -772,31 +753,29 @@ void opal_graph_print(opal_graph_t *graph)
     opal_output(0, "      Graph         ");
     opal_output(0, "====================");
     /* run on all the vertices of the graph */
-    OPAL_LIST_FOREACH(aj_list, graph->adjacency_list, opal_adjacency_list_t) {
+    OPAL_LIST_FOREACH (aj_list, graph->adjacency_list, opal_adjacency_list_t) {
         /* print vertex data to temporary string*/
         if (NULL != aj_list->vertex->print_vertex) {
             need_free1 = true;
             tmp_str1 = aj_list->vertex->print_vertex(aj_list->vertex->vertex_data);
-        }
-        else {
+        } else {
             need_free1 = false;
             tmp_str1 = "";
         }
         /* print vertex */
-        opal_output(0, "V(%s) Connections:",tmp_str1);
+        opal_output(0, "V(%s) Connections:", tmp_str1);
         /* run on all the edges of the vertex */
-        OPAL_LIST_FOREACH(edge, aj_list->edges, opal_graph_edge_t) {
+        OPAL_LIST_FOREACH (edge, aj_list->edges, opal_graph_edge_t) {
             /* print the vertex data of the vertex in the end of the edge to a temporary string */
             if (NULL != edge->end->print_vertex) {
                 need_free2 = true;
                 tmp_str2 = edge->end->print_vertex(edge->end->vertex_data);
-            }
-            else {
+            } else {
                 need_free2 = false;
                 tmp_str2 = "";
             }
             /* print the edge */
-            opal_output(0, "    E(%s -> %d -> %s)",tmp_str1, edge->weight, tmp_str2);
+            opal_output(0, "    E(%s -> %d -> %s)", tmp_str1, edge->weight, tmp_str2);
             if (need_free2) {
                 free(tmp_str2);
             }
@@ -806,4 +785,3 @@ void opal_graph_print(opal_graph_t *graph)
         }
     }
 }
-
diff --git a/opal/class/opal_graph.h b/opal/class/opal_graph.h
index 47b5f54bdb9..1685d0830b7 100644
--- a/opal/class/opal_graph.h
+++ b/opal/class/opal_graph.h
@@ -19,7 +19,7 @@
 /**
  * @file
  * The opal_graph interface is used to provide a generic graph infrastructure
- * to Open-MPI. The graph is represented as an adjacentcy list.
+ * to Open-MPI. The graph is represented as an adjacency list.
  * The graph is a list of vertices. The graph is a weighted directional graph.
  * Each vertex contains a pointer to a vertex data.
  * This pointer can point to the structure that this vertex belongs to.
@@ -28,39 +28,39 @@
 #define OPAL_GRAPH_H
 
 #include "opal_config.h"
-#include <stdio.h>
-#include <stdlib.h>
-#include "opal/class/opal_object.h"
 #include "opal/class/opal_list.h"
+#include "opal/class/opal_object.h"
 #include "opal/class/opal_pointer_array.h"
 #include "opal/class/opal_value_array.h"
+#include <stdio.h>
+#include <stdlib.h>
 
 BEGIN_C_DECLS
 
-    /* When two vertices are not connected, the distance between them is infinite. */
+/* When two vertices are not connected, the distance between them is infinite. */
 #define DISTANCE_INFINITY 0x7fffffff
 
-    /* A class for vertex */
+/* A class for vertex */
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_graph_vertex_t);
 
-    /* A class for an edge (a connection between two verices) */
+/* A class for an edge (a connection between two verices) */
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_graph_edge_t);
 
-    /* A class for an adjacency list  */
+/* A class for an adjacency list  */
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_adjacency_list_t);
 
-    /* A class for graph */
+/* A class for graph */
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_graph_t);
 
-  /**
-   * Function pointer for coping a vertex data from one vertex to
-   * another
-   *
-   * @param dst The destination pointer of vertex_data
-   * @param src The source pointer of the vertex_data
-   *
-   *
-   */
+/**
+ * Function pointer for copying a vertex data from one vertex to
+ * another
+ *
+ * @param dst The destination pointer of vertex_data
+ * @param src The source pointer of the vertex_data
+ *
+ *
+ */
 typedef void (*opal_graph_copy_vertex_data)(void **dst, void *src);
 
 /**
@@ -83,12 +83,12 @@ typedef void *(*opal_graph_alloc_vertex_data)(void);
  *@param vertex_data1
  *@param vertex_data2
  *
- *@return int The comparition results. 1- vertex_data1 is bigger
+ *@return int The comparison results. 1- vertex_data1 is bigger
  *        then vertex_data2, 0- vertex_data1 is equal to
  *        vertex_data2 and -1- vertex_data1 is smaller the
  *        vertex_data2.
  */
-typedef int  (*opal_graph_compare_vertex_data)(void *vertex_data1, void *vertex_data2);
+typedef int (*opal_graph_compare_vertex_data)(void *vertex_data1, void *vertex_data2);
 
 /**
  * print a vertex data.
@@ -97,24 +97,24 @@ typedef int  (*opal_graph_compare_vertex_data)(void *vertex_data1, void *vertex_
  */
 typedef char *(*opal_graph_print_vertex)(void *vertex_data);
 
-
 /**
  * A vertex class.
  */
 struct opal_graph_vertex_t {
-    opal_list_item_t             super; /* A pointer to a vertex parent */
-    void                         *in_graph; /* A pointer to the graph that this vertex belongs to */
-    void                         *in_adj_list; /* A pointer to the adjacency that this vertex belongs to */
-    void                         *vertex_data;/* A pointer to some data. this pointer can point to the struct the this*/
-                                              /* vertex belongs to*/
-    struct opal_graph_vertex_t   *sibling;/* A pointer to a sibling vertex. */
-                                          /* if this vertex was copied this pointer will point to the source vertex */
-                                          /* This pointer is for internal uses. */
-    opal_graph_copy_vertex_data  copy_vertex_data; /* A function to copy vertex data */
-    opal_graph_free_vertex_data  free_vertex_data; /* A function to print vertex data */
-    opal_graph_alloc_vertex_data alloc_vertex_data;/* A function to allocate vertex data */
-    opal_graph_compare_vertex_data compare_vertex; /* A function to compare between two vertices data */
-    opal_graph_print_vertex        print_vertex;   /* A function to print vertex data */
+    opal_list_item_t super; /* A pointer to a vertex parent */
+    void *in_graph;         /* A pointer to the graph that this vertex belongs to */
+    void *in_adj_list;      /* A pointer to the adjacency that this vertex belongs to */
+    void *vertex_data; /* A pointer to some data. this pointer can point to the struct the this*/
+                       /* vertex belongs to*/
+    struct opal_graph_vertex_t *sibling; /* A pointer to a sibling vertex. */
+    /* if this vertex was copied this pointer will point to the source vertex */
+    /* This pointer is for internal uses. */
+    opal_graph_copy_vertex_data copy_vertex_data;   /* A function to copy vertex data */
+    opal_graph_free_vertex_data free_vertex_data;   /* A function to print vertex data */
+    opal_graph_alloc_vertex_data alloc_vertex_data; /* A function to allocate vertex data */
+    opal_graph_compare_vertex_data
+        compare_vertex;                   /* A function to compare between two vertices data */
+    opal_graph_print_vertex print_vertex; /* A function to print vertex data */
 };
 
 /**
@@ -126,9 +126,9 @@ typedef struct opal_graph_vertex_t opal_graph_vertex_t;
  * An opal_adjacency_list_t class
  */
 struct opal_adjacency_list_t {
-    opal_list_item_t     super;   /* A pointer to vertex parent */
-    opal_graph_vertex_t *vertex;  /* The adjacency_list is for adjacent of this vertex */
-    opal_list_t         *edges;   /* An edge list for all the adjacent and their weights */
+    opal_list_item_t super;      /* A pointer to vertex parent */
+    opal_graph_vertex_t *vertex; /* The adjacency_list is for adjacent of this vertex */
+    opal_list_t *edges;          /* An edge list for all the adjacent and their weights */
 };
 
 /**
@@ -144,13 +144,13 @@ typedef struct opal_adjacency_list_t opal_adjacency_list_t;
  * contains a weight field.
  */
 struct opal_graph_edge_t {
-    opal_list_item_t         super;  /* A pointer to the edge parent */
-    opal_graph_vertex_t      *start; /* The start vertex. */
-    opal_graph_vertex_t      *end;   /* The end vertex */
-    uint32_t                 weight; /* The weight of this edge */
-    opal_adjacency_list_t    *in_adj_list; /* The adjacency list in witch this edge in.*/
-                                           /* This adjacency list contains the start vertex of this edge*/
-                                           /* and its for internal uses */
+    opal_list_item_t super;             /* A pointer to the edge parent */
+    opal_graph_vertex_t *start;         /* The start vertex. */
+    opal_graph_vertex_t *end;           /* The end vertex */
+    uint32_t weight;                    /* The weight of this edge */
+    opal_adjacency_list_t *in_adj_list; /* The adjacency list in witch this edge in.*/
+    /* This adjacency list contains the start vertex of this edge*/
+    /* and its for internal uses */
 };
 
 /**
@@ -158,15 +158,14 @@ struct opal_graph_edge_t {
  */
 typedef struct opal_graph_edge_t opal_graph_edge_t;
 
-
 /**
  * A graph class.
  */
 struct opal_graph_t {
-    opal_object_t       super;
-    opal_list_t         *adjacency_list;
-    int                 number_of_edges;
-    int                 number_of_vertices;
+    opal_object_t super;
+    opal_list_t *adjacency_list;
+    int number_of_edges;
+    int number_of_vertices;
 };
 
 /**
@@ -180,7 +179,7 @@ typedef struct opal_graph_t opal_graph_t;
  */
 struct vertex_distance_from_t {
     opal_graph_vertex_t *vertex;
-    uint32_t            weight;
+    uint32_t weight;
 };
 
 /**
@@ -231,7 +230,7 @@ OPAL_DECLSPEC int opal_graph_add_edge(opal_graph_t *graph, opal_graph_edge_t *ed
  * @param graph The graph that this edge will be remove from.
  * @param edge the edge that we want to remove.
  */
-OPAL_DECLSPEC void opal_graph_remove_edge (opal_graph_t *graph, opal_graph_edge_t *edge);
+OPAL_DECLSPEC void opal_graph_remove_edge(opal_graph_t *graph, opal_graph_edge_t *edge);
 
 /**
  * This graph API tell us if two vertices are adjacent
@@ -244,7 +243,8 @@ OPAL_DECLSPEC void opal_graph_remove_edge (opal_graph_t *graph, opal_graph_edge_
  *         vertices or infinity if the vertices are not
  *         connected.
  */
-OPAL_DECLSPEC uint32_t opal_graph_adjacent(opal_graph_t *graph, opal_graph_vertex_t *vertex1, opal_graph_vertex_t *vertex2);
+OPAL_DECLSPEC uint32_t opal_graph_adjacent(opal_graph_t *graph, opal_graph_vertex_t *vertex1,
+                                           opal_graph_vertex_t *vertex2);
 
 /**
  * This Graph API returns the order of the graph (number of
@@ -277,7 +277,6 @@ OPAL_DECLSPEC int opal_graph_get_size(opal_graph_t *graph);
  */
 OPAL_DECLSPEC opal_graph_vertex_t *opal_graph_find_vertex(opal_graph_t *graph, void *vertex_data);
 
-
 /**
  * This graph API returns an array of pointers of all the
  * vertices in the graph.
@@ -290,7 +289,8 @@ OPAL_DECLSPEC opal_graph_vertex_t *opal_graph_find_vertex(opal_graph_t *graph, v
  * @return int returning the graph order (the
  *                    number of vertices in the returned array)
  */
-OPAL_DECLSPEC int opal_graph_get_graph_vertices(opal_graph_t *graph, opal_pointer_array_t *vertices_list);
+OPAL_DECLSPEC int opal_graph_get_graph_vertices(opal_graph_t *graph,
+                                                opal_pointer_array_t *vertices_list);
 
 /**
  * This graph API returns all the adjacent of a vertex and the
@@ -305,11 +305,12 @@ OPAL_DECLSPEC int opal_graph_get_graph_vertices(opal_graph_t *graph, opal_pointe
  *
  * @return int the number of adjacent in the list.
  */
-OPAL_DECLSPEC int opal_graph_get_adjacent_vertices(opal_graph_t *graph, opal_graph_vertex_t *vertex, opal_value_array_t *adjacent);
+OPAL_DECLSPEC int opal_graph_get_adjacent_vertices(opal_graph_t *graph, opal_graph_vertex_t *vertex,
+                                                   opal_value_array_t *adjacent);
 
 /**
  * This graph API duplicates a graph. Note that this API does
- * not copy the graph but builds a new graph while coping just
+ * not copy the graph but builds a new graph while copying just
  * the vertices data.
  *
  * @param dest The new created graph.
@@ -326,7 +327,8 @@ OPAL_DECLSPEC void opal_graph_duplicate(opal_graph_t **dest, opal_graph_t *src);
  *
  * @return uint32_t the distance between the two vertices.
  */
-OPAL_DECLSPEC uint32_t opal_graph_spf(opal_graph_t *graph, opal_graph_vertex_t *vertex1, opal_graph_vertex_t *vertex2);
+OPAL_DECLSPEC uint32_t opal_graph_spf(opal_graph_t *graph, opal_graph_vertex_t *vertex1,
+                                      opal_graph_vertex_t *vertex2);
 
 /**
  * This graph API returns the distance (weight) from a reference
@@ -340,7 +342,8 @@ OPAL_DECLSPEC uint32_t opal_graph_spf(opal_graph_t *graph, opal_graph_vertex_t *
  *
  * @return uint32_t the size of the distance array
  */
-OPAL_DECLSPEC uint32_t opal_graph_dijkstra(opal_graph_t *graph, opal_graph_vertex_t *vertex, opal_value_array_t *distance_array);
+OPAL_DECLSPEC uint32_t opal_graph_dijkstra(opal_graph_t *graph, opal_graph_vertex_t *vertex,
+                                           opal_value_array_t *distance_array);
 
 /**
  * This graph API prints a graph - mostly for debug uses.
@@ -350,6 +353,4 @@ OPAL_DECLSPEC void opal_graph_print(opal_graph_t *graph);
 
 END_C_DECLS
 
-
 #endif /* OPAL_GRAPH_H */
-
diff --git a/opal/class/opal_hash_table.c b/opal/class/opal_hash_table.c
index 628440dc1fa..94f81615ab7 100644
--- a/opal/class/opal_hash_table.c
+++ b/opal/class/opal_hash_table.c
@@ -24,12 +24,12 @@
 
 #include "opal_config.h"
 
-#include <string.h>
 #include <stdlib.h>
+#include <string.h>
 
-#include "opal/util/output.h"
 #include "opal/class/opal_hash_table.h"
 #include "opal/constants.h"
+#include "opal/util/output.h"
 
 /*
  * opal_hash_table_t
@@ -37,7 +37,7 @@
  * Sketch: [Contributed by David Linden of Hewlett-Packard]
  *
  * This has been found to be good for search and insert and
- * (seldom-)remove, all with probablistic O(1) time.  Having a good
+ * (seldom-)remove, all with probabilistic O(1) time.  Having a good
  * distribution of the hash indices is important, so even if you know
  * the keys distribute well under a mask, that micro-optimization
  * isn't worth doing.
@@ -92,16 +92,16 @@
  */
 
 struct opal_hash_element_t {
-    int         valid;          /* whether this element is valid */
-    union {                     /* the key, in its various forms */
-        uint32_t        u32;
-        uint64_t        u64;
+    int valid; /* whether this element is valid */
+    union {    /* the key, in its various forms */
+        uint32_t u32;
+        uint64_t u64;
         struct {
-            const void * key;
-            size_t      key_size;
-        }       ptr;
-    }           key;
-    void *      value;          /* the value */
+            const void *key;
+            size_t key_size;
+        } ptr;
+    } key;
+    void *value; /* the value */
 };
 typedef struct opal_hash_element_t opal_hash_element_t;
 
@@ -110,35 +110,29 @@ struct opal_hash_type_methods_t {
      * The value is not owned by the hash table
      * The key,key_size of pointer keys is
      */
-    void        (*elt_destructor)(opal_hash_element_t * elt);
+    void (*elt_destructor)(opal_hash_element_t *elt);
     /* Hash the key of the element -- for growing and adjusting-after-removal */
-    uint64_t    (*hash_elt)(opal_hash_element_t * elt);
+    uint64_t (*hash_elt)(opal_hash_element_t *elt);
 };
 
 /* interact with the class-like mechanism */
 
-static void opal_hash_table_construct(opal_hash_table_t* ht);
-static void opal_hash_table_destruct(opal_hash_table_t* ht);
+static void opal_hash_table_construct(opal_hash_table_t *ht);
+static void opal_hash_table_destruct(opal_hash_table_t *ht);
 
-OBJ_CLASS_INSTANCE(
-    opal_hash_table_t,
-    opal_object_t,
-    opal_hash_table_construct,
-    opal_hash_table_destruct
-);
+OBJ_CLASS_INSTANCE(opal_hash_table_t, opal_object_t, opal_hash_table_construct,
+                   opal_hash_table_destruct);
 
-static void
-opal_hash_table_construct(opal_hash_table_t* ht)
+static void opal_hash_table_construct(opal_hash_table_t *ht)
 {
-  ht->ht_table = NULL;
-  ht->ht_capacity = ht->ht_size = ht->ht_growth_trigger = 0;
-  ht->ht_density_numer = ht->ht_density_denom = 0;
-  ht->ht_growth_numer = ht->ht_growth_denom = 0;
-  ht->ht_type_methods = NULL;
+    ht->ht_table = NULL;
+    ht->ht_capacity = ht->ht_size = ht->ht_growth_trigger = 0;
+    ht->ht_density_numer = ht->ht_density_denom = 0;
+    ht->ht_growth_numer = ht->ht_growth_denom = 0;
+    ht->ht_type_methods = NULL;
 }
 
-static void
-opal_hash_table_destruct(opal_hash_table_t* ht)
+static void opal_hash_table_destruct(opal_hash_table_t *ht)
 {
     opal_hash_table_remove_all(ht);
     free(ht->ht_table);
@@ -148,49 +142,47 @@ opal_hash_table_destruct(opal_hash_table_t* ht)
  * Init, etc
  */
 
-static size_t
-opal_hash_round_capacity_up(size_t capacity)
+static size_t opal_hash_round_capacity_up(size_t capacity)
 {
     /* round up to (1 mod 30) */
-    return ((capacity+29)/30*30 + 1);
+    return ((capacity + 29) / 30 * 30 + 1);
 }
 
 /* this could be the new init if people wanted a more general API */
 /* (that's why it isn't static) */
-int                             /* OPAL_ return code */
-opal_hash_table_init2(opal_hash_table_t* ht, size_t estimated_max_size,
-                      int density_numer, int density_denom,
-                      int growth_numer, int growth_denom)
+int /* OPAL_ return code */
+opal_hash_table_init2(opal_hash_table_t *ht, size_t estimated_max_size, int density_numer,
+                      int density_denom, int growth_numer, int growth_denom)
 {
     size_t est_capacity = estimated_max_size * density_denom / density_numer;
     size_t capacity = opal_hash_round_capacity_up(est_capacity);
-    ht->ht_table = (opal_hash_element_t*) calloc(capacity, sizeof(opal_hash_element_t));
+    ht->ht_table = (opal_hash_element_t *) calloc(capacity, sizeof(opal_hash_element_t));
     if (NULL == ht->ht_table) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
-    ht->ht_capacity       = capacity;
-    ht->ht_density_numer  = density_numer;
-    ht->ht_density_denom  = density_denom;
-    ht->ht_growth_numer   = growth_numer;
-    ht->ht_growth_denom   = growth_denom;
+    ht->ht_capacity = capacity;
+    ht->ht_density_numer = density_numer;
+    ht->ht_density_denom = density_denom;
+    ht->ht_growth_numer = growth_numer;
+    ht->ht_growth_denom = growth_denom;
     ht->ht_growth_trigger = capacity * density_numer / density_denom;
-    ht->ht_type_methods   = NULL;
+    ht->ht_type_methods = NULL;
     return OPAL_SUCCESS;
 }
 
-int                             /* OPAL_ return code */
-opal_hash_table_init(opal_hash_table_t* ht, size_t table_size)
+int /* OPAL_ return code */
+opal_hash_table_init(opal_hash_table_t *ht, size_t table_size)
 {
     /* default to density of 1/2 and growth of 2/1 */
     return opal_hash_table_init2(ht, table_size, 1, 2, 2, 1);
 }
 
-int                             /* OPAL_ return code */
-opal_hash_table_remove_all(opal_hash_table_t* ht)
+int /* OPAL_ return code */
+opal_hash_table_remove_all(opal_hash_table_t *ht)
 {
     size_t ii;
     for (ii = 0; ii < ht->ht_capacity; ii += 1) {
-        opal_hash_element_t * elt = &ht->ht_table[ii];
+        opal_hash_element_t *elt = &ht->ht_table[ii];
         if (elt->valid && ht->ht_type_methods && ht->ht_type_methods->elt_destructor) {
             ht->ht_type_methods->elt_destructor(elt);
         }
@@ -204,22 +196,22 @@ opal_hash_table_remove_all(opal_hash_table_t* ht)
     return OPAL_SUCCESS;
 }
 
-static int                      /* OPAL_ return code */
-opal_hash_grow(opal_hash_table_t * ht)
+static int /* OPAL_ return code */
+opal_hash_grow(opal_hash_table_t *ht)
 {
     size_t jj, ii;
-    opal_hash_element_t* old_table;
-    opal_hash_element_t* new_table;
+    opal_hash_element_t *old_table;
+    opal_hash_element_t *new_table;
     size_t old_capacity;
     size_t new_capacity;
 
-    old_table    = ht->ht_table;
+    old_table = ht->ht_table;
     old_capacity = ht->ht_capacity;
 
     new_capacity = old_capacity * ht->ht_growth_numer / ht->ht_growth_denom;
     new_capacity = opal_hash_round_capacity_up(new_capacity);
 
-    new_table    = (opal_hash_element_t*) calloc(new_capacity, sizeof(new_table[0]));
+    new_table = (opal_hash_element_t *) calloc(new_capacity, sizeof(new_table[0]));
     if (NULL == new_table) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
@@ -233,14 +225,16 @@ opal_hash_grow(opal_hash_table_t * ht)
        deleted, so we still own the ptr key storage, just in the new
        table now */
     for (jj = 0; jj < old_capacity; jj += 1) {
-        opal_hash_element_t * old_elt;
-        opal_hash_element_t * new_elt;
-        old_elt =  &old_table[jj];
+        opal_hash_element_t *old_elt;
+        opal_hash_element_t *new_elt;
+        old_elt = &old_table[jj];
         if (old_elt->valid) {
-            for (ii = (ht->ht_type_methods->hash_elt(old_elt)%new_capacity); ; ii += 1) {
-                if (ii == new_capacity) { ii = 0; }
+            for (ii = (ht->ht_type_methods->hash_elt(old_elt) % new_capacity);; ii += 1) {
+                if (ii == new_capacity) {
+                    ii = 0;
+                }
                 new_elt = &new_table[ii];
-                if (! new_elt->valid) {
+                if (!new_elt->valid) {
                     *new_elt = *old_elt;
                     break;
                 }
@@ -259,16 +253,16 @@ opal_hash_grow(opal_hash_table_t * ht)
    removed.  With the help of the type methods this can be generic.
    The important thing is to rehash any valid elements immediately
    following the element-being-removed */
-static int                      /* OPAL_ return code */
-opal_hash_table_remove_elt_at(opal_hash_table_t * ht, size_t ii)
+static int /* OPAL_ return code */
+opal_hash_table_remove_elt_at(opal_hash_table_t *ht, size_t ii)
 {
     size_t jj, capacity = ht->ht_capacity;
-    opal_hash_element_t* elts = ht->ht_table;
-    opal_hash_element_t * elt;
+    opal_hash_element_t *elts = ht->ht_table;
+    opal_hash_element_t *elt;
 
     elt = &elts[ii];
 
-    if (! elt->valid) {
+    if (!elt->valid) {
         /* huh?  removing a not-valid element? */
         return OPAL_ERROR;
     }
@@ -288,20 +282,24 @@ opal_hash_table_remove_elt_at(opal_hash_table_t * ht, size_t ii)
      * then  z moves down a little:                  XYyabCz..
      * then  . means we're done
      */
-    for (ii = ii+1; ; ii += 1) { /* scan immediately following elements */
-        if (ii == capacity) { ii = 0; }
+    for (ii = ii + 1;; ii += 1) { /* scan immediately following elements */
+        if (ii == capacity) {
+            ii = 0;
+        }
         elt = &elts[ii];
-        if (! elt->valid) {
-            break;              /* done */
+        if (!elt->valid) {
+            break; /* done */
         }
         /* rehash it and move it if necessary */
-        for (jj = ht->ht_type_methods->hash_elt(elt)%capacity; ; jj += 1) {
-            if (jj == capacity) { jj = 0; }
+        for (jj = ht->ht_type_methods->hash_elt(elt) % capacity;; jj += 1) {
+            if (jj == capacity) {
+                jj = 0;
+            }
             if (jj == ii) {
                 /* already in place, either ideal or best-for-now */
                 break;
-            } else if (! elts[jj].valid) {
-                /* move it down, and invaildate where it came from */
+            } else if (!elts[jj].valid) {
+                /* move it down, and invalidate where it came from */
                 elts[jj] = elts[ii];
                 elts[ii].valid = 0;
                 break;
@@ -314,46 +312,42 @@ opal_hash_table_remove_elt_at(opal_hash_table_t * ht, size_t ii)
     return OPAL_SUCCESS;
 }
 
-
 /***************************************************************************/
 
-static uint64_t
-opal_hash_hash_elt_uint32(opal_hash_element_t * elt)
+static uint64_t opal_hash_hash_elt_uint32(opal_hash_element_t *elt)
 {
-  return elt->key.u32;
+    return elt->key.u32;
 }
 
-static const struct opal_hash_type_methods_t
-opal_hash_type_methods_uint32 = {
-    NULL,
-    opal_hash_hash_elt_uint32
-};
+static const struct opal_hash_type_methods_t opal_hash_type_methods_uint32
+    = {NULL, opal_hash_hash_elt_uint32};
 
-int                             /* OPAL_ return code */
-opal_hash_table_get_value_uint32(opal_hash_table_t* ht, uint32_t key, void * *value)
+int /* OPAL_ return code */
+opal_hash_table_get_value_uint32(opal_hash_table_t *ht, uint32_t key, void **value)
 {
     size_t ii, capacity = ht->ht_capacity;
-    opal_hash_element_t * elt;
+    opal_hash_element_t *elt;
 
 #if OPAL_ENABLE_DEBUG
-    if(capacity == 0) {
+    if (capacity == 0) {
         opal_output(0, "opal_hash_table_get_value_uint32:"
-                    "opal_hash_table_init() has not been called");
+                       "opal_hash_table_init() has not been called");
         return OPAL_ERROR;
     }
-    if (NULL != ht->ht_type_methods &&
-        &opal_hash_type_methods_uint32 != ht->ht_type_methods) {
+    if (NULL != ht->ht_type_methods && &opal_hash_type_methods_uint32 != ht->ht_type_methods) {
         opal_output(0, "opal_hash_table_get_value_uint32:"
-                    "hash table is for a different key type");
-            return OPAL_ERROR;
+                       "hash table is for a different key type");
+        return OPAL_ERROR;
     }
 #endif
 
     ht->ht_type_methods = &opal_hash_type_methods_uint32;
-    for (ii = key%capacity; ; ii += 1) {
-        if (ii == capacity) { ii = 0; }
+    for (ii = key % capacity;; ii += 1) {
+        if (ii == capacity) {
+            ii = 0;
+        }
         elt = &ht->ht_table[ii];
-        if (! elt->valid) {
+        if (!elt->valid) {
             return OPAL_ERR_NOT_FOUND;
         } else if (elt->key.u32 == key) {
             *value = elt->value;
@@ -362,35 +356,35 @@ opal_hash_table_get_value_uint32(opal_hash_table_t* ht, uint32_t key, void * *va
             /* keey looking */
         }
     }
-
 }
 
-int                             /* OPAL_ return code */
-opal_hash_table_set_value_uint32(opal_hash_table_t * ht, uint32_t key, void * value)
+int /* OPAL_ return code */
+opal_hash_table_set_value_uint32(opal_hash_table_t *ht, uint32_t key, void *value)
 {
     int rc;
     size_t ii, capacity = ht->ht_capacity;
-    opal_hash_element_t * elt;
+    opal_hash_element_t *elt;
 
 #if OPAL_ENABLE_DEBUG
-    if(capacity == 0) {
+    if (capacity == 0) {
         opal_output(0, "opal_hash_table_set_value_uint32:"
-                   "opal_hash_table_init() has not been called");
+                       "opal_hash_table_init() has not been called");
         return OPAL_ERR_BAD_PARAM;
     }
-    if (NULL != ht->ht_type_methods &&
-        &opal_hash_type_methods_uint32 != ht->ht_type_methods) {
+    if (NULL != ht->ht_type_methods && &opal_hash_type_methods_uint32 != ht->ht_type_methods) {
         opal_output(0, "opal_hash_table_set_value_uint32:"
-                    "hash table is for a different key type");
-            return OPAL_ERROR;
+                       "hash table is for a different key type");
+        return OPAL_ERROR;
     }
 #endif
 
     ht->ht_type_methods = &opal_hash_type_methods_uint32;
-    for (ii = key%capacity; ; ii += 1) {
-        if (ii == capacity) { ii = 0; }
+    for (ii = key % capacity;; ii += 1) {
+        if (ii == capacity) {
+            ii = 0;
+        }
         elt = &ht->ht_table[ii];
-        if (! elt->valid) {
+        if (!elt->valid) {
             /* new entry */
             elt->key.u32 = key;
             elt->value = value;
@@ -412,31 +406,31 @@ opal_hash_table_set_value_uint32(opal_hash_table_t * ht, uint32_t key, void * va
     }
 }
 
-int
-opal_hash_table_remove_value_uint32(opal_hash_table_t * ht, uint32_t key)
+int opal_hash_table_remove_value_uint32(opal_hash_table_t *ht, uint32_t key)
 {
     size_t ii, capacity = ht->ht_capacity;
 
 #if OPAL_ENABLE_DEBUG
-    if(capacity == 0) {
+    if (capacity == 0) {
         opal_output(0, "opal_hash_table_get_value_uint32:"
-                    "opal_hash_table_init() has not been called");
+                       "opal_hash_table_init() has not been called");
         return OPAL_ERROR;
     }
-    if (NULL != ht->ht_type_methods &&
-        &opal_hash_type_methods_uint32 != ht->ht_type_methods) {
+    if (NULL != ht->ht_type_methods && &opal_hash_type_methods_uint32 != ht->ht_type_methods) {
         opal_output(0, "opal_hash_table_remove_value_uint32:"
-                    "hash table is for a different key type");
-            return OPAL_ERROR;
+                       "hash table is for a different key type");
+        return OPAL_ERROR;
     }
 #endif
 
     ht->ht_type_methods = &opal_hash_type_methods_uint32;
-    for (ii = key%capacity; ; ii += 1) {
-        opal_hash_element_t * elt;
-        if (ii == capacity) ii = 0;
+    for (ii = key % capacity;; ii += 1) {
+        opal_hash_element_t *elt;
+        if (ii == capacity) {
+            ii = 0;
+        }
         elt = &ht->ht_table[ii];
-        if (! elt->valid) {
+        if (!elt->valid) {
             return OPAL_ERR_NOT_FOUND;
         } else if (elt->key.u32 == key) {
             return opal_hash_table_remove_elt_at(ht, ii);
@@ -446,48 +440,43 @@ opal_hash_table_remove_value_uint32(opal_hash_table_t * ht, uint32_t key)
     }
 }
 
-
 /***************************************************************************/
 
-
-static uint64_t
-opal_hash_hash_elt_uint64(opal_hash_element_t * elt)
+static uint64_t opal_hash_hash_elt_uint64(opal_hash_element_t *elt)
 {
-  return elt->key.u64;
+    return elt->key.u64;
 }
 
-static const struct opal_hash_type_methods_t
-opal_hash_type_methods_uint64 = {
-    NULL,
-    opal_hash_hash_elt_uint64
-};
+static const struct opal_hash_type_methods_t opal_hash_type_methods_uint64
+    = {NULL, opal_hash_hash_elt_uint64};
 
-int                             /* OPAL_ return code */
-opal_hash_table_get_value_uint64(opal_hash_table_t * ht, uint64_t key, void * *value)
+int /* OPAL_ return code */
+opal_hash_table_get_value_uint64(opal_hash_table_t *ht, uint64_t key, void **value)
 {
     size_t ii;
     size_t capacity = ht->ht_capacity;
-    opal_hash_element_t * elt;
+    opal_hash_element_t *elt;
 
 #if OPAL_ENABLE_DEBUG
-    if(capacity == 0) {
+    if (capacity == 0) {
         opal_output(0, "opal_hash_table_get_value_uint64:"
-                   "opal_hash_table_init() has not been called");
+                       "opal_hash_table_init() has not been called");
         return OPAL_ERROR;
     }
-    if (NULL != ht->ht_type_methods &&
-        &opal_hash_type_methods_uint64 != ht->ht_type_methods) {
+    if (NULL != ht->ht_type_methods && &opal_hash_type_methods_uint64 != ht->ht_type_methods) {
         opal_output(0, "opal_hash_table_get_value_uint64:"
-                    "hash table is for a different key type");
-            return OPAL_ERROR;
+                       "hash table is for a different key type");
+        return OPAL_ERROR;
     }
 #endif
 
     ht->ht_type_methods = &opal_hash_type_methods_uint64;
-    for (ii = key%capacity; ; ii += 1) {
-        if (ii == capacity) { ii = 0; }
+    for (ii = key % capacity;; ii += 1) {
+        if (ii == capacity) {
+            ii = 0;
+        }
         elt = &ht->ht_table[ii];
-        if (! elt->valid) {
+        if (!elt->valid) {
             return OPAL_ERR_NOT_FOUND;
         } else if (elt->key.u64 == key) {
             *value = elt->value;
@@ -496,35 +485,35 @@ opal_hash_table_get_value_uint64(opal_hash_table_t * ht, uint64_t key, void * *v
             /* keep looking */
         }
     }
-
 }
 
-int                             /* OPAL_ return code */
-opal_hash_table_set_value_uint64(opal_hash_table_t * ht, uint64_t key, void * value)
+int /* OPAL_ return code */
+opal_hash_table_set_value_uint64(opal_hash_table_t *ht, uint64_t key, void *value)
 {
     int rc;
     size_t ii, capacity = ht->ht_capacity;
-    opal_hash_element_t * elt;
+    opal_hash_element_t *elt;
 
 #if OPAL_ENABLE_DEBUG
-    if(capacity == 0) {
+    if (capacity == 0) {
         opal_output(0, "opal_hash_table_set_value_uint64:"
-                   "opal_hash_table_init() has not been called");
+                       "opal_hash_table_init() has not been called");
         return OPAL_ERR_BAD_PARAM;
     }
-    if (NULL != ht->ht_type_methods &&
-        &opal_hash_type_methods_uint64 != ht->ht_type_methods) {
+    if (NULL != ht->ht_type_methods && &opal_hash_type_methods_uint64 != ht->ht_type_methods) {
         opal_output(0, "opal_hash_table_set_value_uint64:"
-                    "hash table is for a different key type");
-            return OPAL_ERROR;
+                       "hash table is for a different key type");
+        return OPAL_ERROR;
     }
 #endif
 
     ht->ht_type_methods = &opal_hash_type_methods_uint64;
-    for (ii = key%capacity; ; ii += 1) {
-        if (ii == capacity) { ii = 0; }
+    for (ii = key % capacity;; ii += 1) {
+        if (ii == capacity) {
+            ii = 0;
+        }
         elt = &ht->ht_table[ii];
-        if (! elt->valid) {
+        if (!elt->valid) {
             /* new entry */
             elt->key.u64 = key;
             elt->value = value;
@@ -545,32 +534,32 @@ opal_hash_table_set_value_uint64(opal_hash_table_t * ht, uint64_t key, void * va
     }
 }
 
-
-int                             /* OPAL_ return code */
-opal_hash_table_remove_value_uint64(opal_hash_table_t * ht, uint64_t key)
+int /* OPAL_ return code */
+opal_hash_table_remove_value_uint64(opal_hash_table_t *ht, uint64_t key)
 {
     size_t ii, capacity = ht->ht_capacity;
 
 #if OPAL_ENABLE_DEBUG
-    if(capacity == 0) {
+    if (capacity == 0) {
         opal_output(0, "opal_hash_table_get_value_uint64:"
-                    "opal_hash_table_init() has not been called");
+                       "opal_hash_table_init() has not been called");
         return OPAL_ERROR;
     }
-    if (NULL != ht->ht_type_methods &&
-        &opal_hash_type_methods_uint64 != ht->ht_type_methods) {
+    if (NULL != ht->ht_type_methods && &opal_hash_type_methods_uint64 != ht->ht_type_methods) {
         opal_output(0, "opal_hash_table_remove_value_uint64:"
-                    "hash table is for a different key type");
-            return OPAL_ERROR;
+                       "hash table is for a different key type");
+        return OPAL_ERROR;
     }
 #endif
 
     ht->ht_type_methods = &opal_hash_type_methods_uint64;
-    for (ii = key%capacity; ; ii += 1) {
-        opal_hash_element_t * elt;
-        if (ii == capacity) { ii = 0; }
+    for (ii = key % capacity;; ii += 1) {
+        opal_hash_element_t *elt;
+        if (ii == capacity) {
+            ii = 0;
+        }
         elt = &ht->ht_table[ii];
-        if (! elt->valid) {
+        if (!elt->valid) {
             return OPAL_ERR_NOT_FOUND;
         } else if (elt->key.u64 == key) {
             return opal_hash_table_remove_elt_at(ht, ii);
@@ -580,80 +569,72 @@ opal_hash_table_remove_value_uint64(opal_hash_table_t * ht, uint64_t key)
     }
 }
 
-
 /***************************************************************************/
 
 /* helper function used in several places */
-static uint64_t
-opal_hash_hash_key_ptr(const void * key, size_t key_size)
+static uint64_t opal_hash_hash_key_ptr(const void *key, size_t key_size)
 {
     uint64_t hash;
     const unsigned char *scanner;
     size_t ii;
 
     hash = 0;
-    scanner = (const unsigned char *)key;
+    scanner = (const unsigned char *) key;
     for (ii = 0; ii < key_size; ii += 1) {
-        hash = HASH_MULTIPLIER*hash + *scanner++;
+        hash = HASH_MULTIPLIER * hash + *scanner++;
     }
     return hash;
 }
 
 /* ptr methods */
 
-static void
-opal_hash_destruct_elt_ptr(opal_hash_element_t * elt)
+static void opal_hash_destruct_elt_ptr(opal_hash_element_t *elt)
 {
     elt->key.ptr.key_size = 0;
-    void * key = (void *) elt->key.ptr.key; /* cast away const so we can free it */
+    void *key = (void *) elt->key.ptr.key; /* cast away const so we can free it */
     if (NULL != key) {
         elt->key.ptr.key = NULL;
         free(key);
     }
 }
 
-static uint64_t
-opal_hash_hash_elt_ptr(opal_hash_element_t * elt)
+static uint64_t opal_hash_hash_elt_ptr(opal_hash_element_t *elt)
 {
     return opal_hash_hash_key_ptr(elt->key.ptr.key, elt->key.ptr.key_size);
 }
 
-static const struct opal_hash_type_methods_t
-opal_hash_type_methods_ptr = {
-    opal_hash_destruct_elt_ptr,
-    opal_hash_hash_elt_ptr
-};
+static const struct opal_hash_type_methods_t opal_hash_type_methods_ptr
+    = {opal_hash_destruct_elt_ptr, opal_hash_hash_elt_ptr};
 
-int                             /* OPAL_ return code */
-opal_hash_table_get_value_ptr(opal_hash_table_t * ht,
-                              const void * key, size_t key_size,
-                              void * *value)
+int /* OPAL_ return code */
+opal_hash_table_get_value_ptr(opal_hash_table_t *ht, const void *key, size_t key_size, void **value)
 {
     size_t ii, capacity = ht->ht_capacity;
-    opal_hash_element_t * elt;
+    opal_hash_element_t *elt;
 
 #if OPAL_ENABLE_DEBUG
-    if(capacity == 0) {
+    if (capacity == 0) {
         opal_output(0, "opal_hash_table_get_value_ptr:"
-                   "opal_hash_table_init() has not been called");
+                       "opal_hash_table_init() has not been called");
         return OPAL_ERROR;
     }
-    if (NULL != ht->ht_type_methods &&
-        &opal_hash_type_methods_ptr != ht->ht_type_methods) {
+    if (NULL != ht->ht_type_methods && &opal_hash_type_methods_ptr != ht->ht_type_methods) {
         opal_output(0, "opal_hash_table_get_value_ptr:"
-                    "hash table is for a different key type");
-            return OPAL_ERROR;
+                       "hash table is for a different key type");
+        return OPAL_ERROR;
     }
 #endif
 
     ht->ht_type_methods = &opal_hash_type_methods_ptr;
-    for (ii = opal_hash_hash_key_ptr(key, key_size)%capacity; ; ii += 1) {
-        if (ii == capacity) { ii = 0; }
+    for (ii = opal_hash_hash_key_ptr(key, key_size) % capacity;; ii += 1) {
+        if (ii == capacity) {
+            ii = 0;
+        }
         elt = &ht->ht_table[ii];
-        if (! elt->valid) {
+        if (!elt->valid) {
             return OPAL_ERR_NOT_FOUND;
-        } else if (elt->key.ptr.key_size == key_size &&
-                   0 == memcmp(elt->key.ptr.key, key, key_size)) {
+        } else if (elt->key.ptr.key_size == key_size
+                   && 0 == memcmp(elt->key.ptr.key, key, key_size)) {
             *value = elt->value;
             return OPAL_SUCCESS;
         } else {
@@ -662,38 +643,37 @@ opal_hash_table_get_value_ptr(opal_hash_table_t * ht,
     }
 }
 
-int                             /* OPAL_ return code */
-opal_hash_table_set_value_ptr(opal_hash_table_t * ht,
-                              const void * key, size_t key_size,
-                              void * value)
+int /* OPAL_ return code */
+opal_hash_table_set_value_ptr(opal_hash_table_t *ht, const void *key, size_t key_size, void *value)
 {
     int rc;
     size_t ii, capacity = ht->ht_capacity;
-    opal_hash_element_t * elt;
+    opal_hash_element_t *elt;
 
 #if OPAL_ENABLE_DEBUG
-    if(capacity == 0) {
+    if (capacity == 0) {
         opal_output(0, "opal_hash_table_set_value_ptr:"
-                   "opal_hash_table_init() has not been called");
+                       "opal_hash_table_init() has not been called");
         return OPAL_ERR_BAD_PARAM;
     }
-    if (NULL != ht->ht_type_methods &&
-        &opal_hash_type_methods_ptr != ht->ht_type_methods) {
+    if (NULL != ht->ht_type_methods && &opal_hash_type_methods_ptr != ht->ht_type_methods) {
         opal_output(0, "opal_hash_table_set_value_ptr:"
-                    "hash table is for a different key type");
-            return OPAL_ERROR;
+                       "hash table is for a different key type");
+        return OPAL_ERROR;
     }
 #endif
 
     ht->ht_type_methods = &opal_hash_type_methods_ptr;
-    for (ii = opal_hash_hash_key_ptr(key, key_size)%capacity; ; ii += 1) {
-        if (ii == capacity) { ii = 0; }
+    for (ii = opal_hash_hash_key_ptr(key, key_size) % capacity;; ii += 1) {
+        if (ii == capacity) {
+            ii = 0;
+        }
         elt = &ht->ht_table[ii];
-        if (! elt->valid) {
+        if (!elt->valid) {
             /* new entry */
-            void * key_local = malloc(key_size);
+            void *key_local = malloc(key_size);
             memcpy(key_local, key, key_size);
-            elt->key.ptr.key      = key_local;
+            elt->key.ptr.key = key_local;
             elt->key.ptr.key_size = key_size;
             elt->value = value;
             elt->valid = 1;
@@ -704,8 +684,8 @@ opal_hash_table_set_value_ptr(opal_hash_table_t * ht,
                 }
             }
             return OPAL_SUCCESS;
-        } else if (elt->key.ptr.key_size == key_size &&
-                   0 == memcmp(elt->key.ptr.key, key, key_size)) {
+        } else if (elt->key.ptr.key_size == key_size
+                   && 0 == memcmp(elt->key.ptr.key, key, key_size)) {
             /* replace existing value */
             elt->value = value;
             return OPAL_SUCCESS;
@@ -715,35 +695,35 @@ opal_hash_table_set_value_ptr(opal_hash_table_t * ht,
     }
 }
 
-int                             /* OPAL_ return code */
-opal_hash_table_remove_value_ptr(opal_hash_table_t * ht,
-                                 const void * key, size_t key_size)
+int /* OPAL_ return code */
+opal_hash_table_remove_value_ptr(opal_hash_table_t *ht, const void *key, size_t key_size)
 {
     size_t ii, capacity = ht->ht_capacity;
 
 #if OPAL_ENABLE_DEBUG
-    if(capacity == 0) {
+    if (capacity == 0) {
         opal_output(0, "opal_hash_table_get_value_ptr:"
-                    "opal_hash_table_init() has not been called");
+                       "opal_hash_table_init() has not been called");
         return OPAL_ERROR;
     }
-    if (NULL != ht->ht_type_methods &&
-        &opal_hash_type_methods_ptr != ht->ht_type_methods) {
+    if (NULL != ht->ht_type_methods && &opal_hash_type_methods_ptr != ht->ht_type_methods) {
         opal_output(0, "opal_hash_table_remove_value_ptr:"
-                    "hash table is for a different key type");
-            return OPAL_ERROR;
+                       "hash table is for a different key type");
+        return OPAL_ERROR;
     }
 #endif
 
     ht->ht_type_methods = &opal_hash_type_methods_ptr;
-    for (ii = opal_hash_hash_key_ptr(key, key_size)%capacity; ; ii += 1) {
-        opal_hash_element_t * elt;
-        if (ii == capacity) { ii = 0; }
+    for (ii = opal_hash_hash_key_ptr(key, key_size) % capacity;; ii += 1) {
+        opal_hash_element_t *elt;
+        if (ii == capacity) {
+            ii = 0;
+        }
         elt = &ht->ht_table[ii];
-        if (! elt->valid) {
+        if (!elt->valid) {
             return OPAL_ERR_NOT_FOUND;
-        } else if (elt->key.ptr.key_size == key_size &&
-                   0 == memcmp(elt->key.ptr.key, key, key_size)) {
+        } else if (elt->key.ptr.key_size == key_size
+                   && 0 == memcmp(elt->key.ptr.key, key, key_size)) {
             return opal_hash_table_remove_elt_at(ht, ii);
         } else {
             /* keep looking */
@@ -754,115 +734,103 @@ opal_hash_table_remove_value_ptr(opal_hash_table_t * ht,
 /***************************************************************************/
 /* Traversals */
 
-static int                      /* OPAL_ return code */
+static int /* OPAL_ return code */
 opal_hash_table_get_next_elt(opal_hash_table_t *ht,
-                             opal_hash_element_t * prev_elt, /* NULL means find first */
-                             opal_hash_element_t * *next_elt)
+                             opal_hash_element_t *prev_elt, /* NULL means find first */
+                             opal_hash_element_t **next_elt)
 {
-  opal_hash_element_t* elts = ht->ht_table;
-  size_t ii, capacity = ht->ht_capacity;
+    opal_hash_element_t *elts = ht->ht_table;
+    size_t ii, capacity = ht->ht_capacity;
 
-  for (ii = (NULL == prev_elt ? 0 : (prev_elt-elts)+1); ii < capacity; ii += 1) {
-    opal_hash_element_t * elt = &elts[ii];
-    if (elt->valid) {
-      *next_elt = elt;
-      return OPAL_SUCCESS;
+    for (ii = (NULL == prev_elt ? 0 : (prev_elt - elts) + 1); ii < capacity; ii += 1) {
+        opal_hash_element_t *elt = &elts[ii];
+        if (elt->valid) {
+            *next_elt = elt;
+            return OPAL_SUCCESS;
+        }
     }
-  }
-  return OPAL_ERROR;
+    return OPAL_ERROR;
 }
 
-int                             /* OPAL_ return code */
-opal_hash_table_get_first_key_uint32(opal_hash_table_t * ht,
-                                     uint32_t *key, void * *value,
-                                     void * *node)
+int /* OPAL_ return code */
+opal_hash_table_get_first_key_uint32(opal_hash_table_t *ht, uint32_t *key, void **value,
+                                     void **node)
 {
-  return opal_hash_table_get_next_key_uint32(ht, key, value, NULL, node);
+    return opal_hash_table_get_next_key_uint32(ht, key, value, NULL, node);
 }
 
-int                             /* OPAL_ return code */
-opal_hash_table_get_next_key_uint32(opal_hash_table_t * ht,
-                                    uint32_t *key, void * *value,
-                                    void * in_node, void * *out_node)
+int /* OPAL_ return code */
+opal_hash_table_get_next_key_uint32(opal_hash_table_t *ht, uint32_t *key, void **value,
+                                    void *in_node, void **out_node)
 {
-  opal_hash_element_t * elt;
-  if (OPAL_SUCCESS == opal_hash_table_get_next_elt(ht, (opal_hash_element_t *) in_node, &elt)) {
-    *key       = elt->key.u32;
-    *value     = elt->value;
-    *out_node  = elt;
-    return OPAL_SUCCESS;
-  }
-  return OPAL_ERROR;
+    opal_hash_element_t *elt;
+    if (OPAL_SUCCESS == opal_hash_table_get_next_elt(ht, (opal_hash_element_t *) in_node, &elt)) {
+        *key = elt->key.u32;
+        *value = elt->value;
+        *out_node = elt;
+        return OPAL_SUCCESS;
+    }
+    return OPAL_ERROR;
 }
 
-int                             /* OPAL_ return code */
-opal_hash_table_get_first_key_ptr(opal_hash_table_t * ht,
-                                  void * *key, size_t *key_size, void * *value,
-                                  void * *node)
+int /* OPAL_ return code */
+opal_hash_table_get_first_key_ptr(opal_hash_table_t *ht, void **key, size_t *key_size, void **value,
+                                  void **node)
 {
-  return opal_hash_table_get_next_key_ptr(ht, key, key_size, value, NULL, node);
+    return opal_hash_table_get_next_key_ptr(ht, key, key_size, value, NULL, node);
 }
 
-int                             /* OPAL_ return code */
-opal_hash_table_get_next_key_ptr(opal_hash_table_t * ht,
-                                 void * *key, size_t *key_size, void * *value,
-                                 void * in_node, void * *out_node)
+int /* OPAL_ return code */
+opal_hash_table_get_next_key_ptr(opal_hash_table_t *ht, void **key, size_t *key_size, void **value,
+                                 void *in_node, void **out_node)
 {
-  opal_hash_element_t * elt;
-  if (OPAL_SUCCESS == opal_hash_table_get_next_elt(ht, (opal_hash_element_t *) in_node, &elt)) {
-    *key       = (void *)elt->key.ptr.key;
-    *key_size  = elt->key.ptr.key_size;
-    *value     = elt->value;
-    *out_node  = elt;
-    return OPAL_SUCCESS;
-  }
-  return OPAL_ERROR;
+    opal_hash_element_t *elt;
+    if (OPAL_SUCCESS == opal_hash_table_get_next_elt(ht, (opal_hash_element_t *) in_node, &elt)) {
+        *key = (void *) elt->key.ptr.key;
+        *key_size = elt->key.ptr.key_size;
+        *value = elt->value;
+        *out_node = elt;
+        return OPAL_SUCCESS;
+    }
+    return OPAL_ERROR;
 }
 
-int                             /* OPAL_ return code */
-opal_hash_table_get_first_key_uint64(opal_hash_table_t * ht,
-                                     uint64_t *key, void * *value,
-                                     void * *node)
+int /* OPAL_ return code */
+opal_hash_table_get_first_key_uint64(opal_hash_table_t *ht, uint64_t *key, void **value,
+                                     void **node)
 {
-  return opal_hash_table_get_next_key_uint64(ht, key, value, NULL, node);
+    return opal_hash_table_get_next_key_uint64(ht, key, value, NULL, node);
 }
 
-int                             /* OPAL_ return code */
-opal_hash_table_get_next_key_uint64(opal_hash_table_t * ht,
-                                    uint64_t *key, void * *value,
-                                    void * in_node, void * *out_node)
+int /* OPAL_ return code */
+opal_hash_table_get_next_key_uint64(opal_hash_table_t *ht, uint64_t *key, void **value,
+                                    void *in_node, void **out_node)
 {
-  opal_hash_element_t * elt;
-  if (OPAL_SUCCESS == opal_hash_table_get_next_elt(ht, (opal_hash_element_t *) in_node, &elt)) {
-    *key       = elt->key.u64;
-    *value     = elt->value;
-    *out_node  = elt;
-    return OPAL_SUCCESS;
-  }
-  return OPAL_ERROR;
+    opal_hash_element_t *elt;
+    if (OPAL_SUCCESS == opal_hash_table_get_next_elt(ht, (opal_hash_element_t *) in_node, &elt)) {
+        *key = elt->key.u64;
+        *value = elt->value;
+        *out_node = elt;
+        return OPAL_SUCCESS;
+    }
+    return OPAL_ERROR;
 }
 
 /* there was/is no traversal for the ptr case; it would go here */
 /* interact with the class-like mechanism */
 
-static void opal_proc_table_construct(opal_proc_table_t* pt);
-static void opal_proc_table_destruct(opal_proc_table_t* pt);
+static void opal_proc_table_construct(opal_proc_table_t *pt);
+static void opal_proc_table_destruct(opal_proc_table_t *pt);
 
-OBJ_CLASS_INSTANCE(
-    opal_proc_table_t,
-    opal_hash_table_t,
-    opal_proc_table_construct,
-    opal_proc_table_destruct
-);
+OBJ_CLASS_INSTANCE(opal_proc_table_t, opal_hash_table_t, opal_proc_table_construct,
+                   opal_proc_table_destruct);
 
-static void
-opal_proc_table_construct(opal_proc_table_t* pt)
+static void opal_proc_table_construct(opal_proc_table_t *pt)
 {
-  pt->pt_size = 0;
+    pt->pt_size = 0;
 }
 
-static void
-opal_proc_table_destruct(opal_proc_table_t* pt)
+static void opal_proc_table_destruct(opal_proc_table_t *pt)
 {
 }
 
@@ -870,22 +838,24 @@ opal_proc_table_destruct(opal_proc_table_t* pt)
  * Init, etc
  */
 
-int opal_proc_table_init(opal_proc_table_t* pt, size_t jobids, size_t vpids) {
+int opal_proc_table_init(opal_proc_table_t *pt, size_t jobids, size_t vpids)
+{
     int rc;
-    if (OPAL_SUCCESS != (rc=opal_hash_table_init(&pt->super, jobids))) {
+    if (OPAL_SUCCESS != (rc = opal_hash_table_init(&pt->super, jobids))) {
         return rc;
     }
     pt->vpids_size = vpids;
     return OPAL_SUCCESS;
 }
 
-int opal_proc_table_remove_all(opal_proc_table_t *pt) {
+int opal_proc_table_remove_all(opal_proc_table_t *pt)
+{
     int rc;
-    opal_hash_table_t * vpids;
+    opal_hash_table_t *vpids;
     uint32_t jobid;
-    void * node;
+    void *node;
 
-    rc = opal_hash_table_get_first_key_uint32(&pt->super, &jobid, (void **)&vpids, &node);
+    rc = opal_hash_table_get_first_key_uint32(&pt->super, &jobid, (void **) &vpids, &node);
 
     if (OPAL_SUCCESS == rc) {
         do {
@@ -893,19 +863,19 @@ int opal_proc_table_remove_all(opal_proc_table_t *pt) {
                 opal_hash_table_remove_all(vpids);
                 OBJ_RELEASE(vpids);
             }
-            rc = opal_hash_table_get_next_key_uint32 (&pt->super, &jobid,
-                                               (void **) &vpids, node, &node);
+            rc = opal_hash_table_get_next_key_uint32(&pt->super, &jobid, (void **) &vpids, node,
+                                                     &node);
         } while (OPAL_SUCCESS == rc);
     }
 
     return rc;
 }
 
-int opal_proc_table_get_value(opal_proc_table_t* pt, opal_process_name_t key,
-                              void** ptr) {
+int opal_proc_table_get_value(opal_proc_table_t *pt, opal_process_name_t key, void **ptr)
+{
     int rc;
-    opal_hash_table_t * vpids;
-    rc = opal_hash_table_get_value_uint32(&pt->super, key.jobid, (void **)&vpids);
+    opal_hash_table_t *vpids;
+    rc = opal_hash_table_get_value_uint32(&pt->super, key.jobid, (void **) &vpids);
     if (rc != OPAL_SUCCESS) {
         return rc;
     }
@@ -913,20 +883,21 @@ int opal_proc_table_get_value(opal_proc_table_t* pt, opal_process_name_t key,
     return rc;
 }
 
-int opal_proc_table_set_value(opal_proc_table_t* pt, opal_process_name_t key, void* value) {
+int opal_proc_table_set_value(opal_proc_table_t *pt, opal_process_name_t key, void *value)
+{
     int rc;
-    opal_hash_table_t * vpids;
-    rc = opal_hash_table_get_value_uint32(&pt->super, key.jobid, (void **)&vpids);
+    opal_hash_table_t *vpids;
+    rc = opal_hash_table_get_value_uint32(&pt->super, key.jobid, (void **) &vpids);
     if (rc != OPAL_SUCCESS) {
         vpids = OBJ_NEW(opal_hash_table_t);
         if (NULL == vpids) {
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
-        if (OPAL_SUCCESS != (rc=opal_hash_table_init(vpids, pt->vpids_size))) {
+        if (OPAL_SUCCESS != (rc = opal_hash_table_init(vpids, pt->vpids_size))) {
             OBJ_RELEASE(vpids);
             return rc;
         }
-        if (OPAL_SUCCESS != (rc=opal_hash_table_set_value_uint32(&pt->super, key.jobid, vpids))) {
+        if (OPAL_SUCCESS != (rc = opal_hash_table_set_value_uint32(&pt->super, key.jobid, vpids))) {
             OBJ_RELEASE(vpids);
             return rc;
         }
@@ -935,13 +906,15 @@ int opal_proc_table_set_value(opal_proc_table_t* pt, opal_process_name_t key, vo
     return rc;
 }
 
-int opal_proc_table_remove_value(opal_proc_table_t* pt, opal_process_name_t key) {
+int opal_proc_table_remove_value(opal_proc_table_t *pt, opal_process_name_t key)
+{
     int rc;
-    opal_hash_table_t * vpids;
-    if (OPAL_SUCCESS != (rc=opal_hash_table_get_value_uint32(&pt->super, key.jobid, (void **)&vpids))) {
+    opal_hash_table_t *vpids;
+    if (OPAL_SUCCESS
+        != (rc = opal_hash_table_get_value_uint32(&pt->super, key.jobid, (void **) &vpids))) {
         return rc;
     }
-    if (OPAL_SUCCESS == (rc=opal_hash_table_remove_value_uint32(vpids, key.vpid))) {
+    if (OPAL_SUCCESS == (rc = opal_hash_table_remove_value_uint32(vpids, key.vpid))) {
         if (0 == vpids->ht_size) {
             opal_hash_table_remove_value_uint32(&pt->super, key.jobid);
             OBJ_RELEASE(vpids);
@@ -950,12 +923,15 @@ int opal_proc_table_remove_value(opal_proc_table_t* pt, opal_process_name_t key)
     return rc;
 }
 
-int opal_proc_table_get_first_key(opal_proc_table_t *pt, opal_process_name_t *key,
-                                  void **value, void **node1, void **node2) {
+int opal_proc_table_get_first_key(opal_proc_table_t *pt, opal_process_name_t *key, void **value,
+                                  void **node1, void **node2)
+{
     int rc;
     uint32_t jobid, vpid;
-    opal_hash_table_t * vpids;
-    if (OPAL_SUCCESS != (rc=opal_hash_table_get_first_key_uint32(&pt->super, &jobid, (void **)&vpids, node1))) {
+    opal_hash_table_t *vpids;
+    if (OPAL_SUCCESS
+        != (rc = opal_hash_table_get_first_key_uint32(&pt->super, &jobid, (void **) &vpids,
+                                                      node1))) {
         return rc;
     }
     rc = opal_hash_table_get_first_key_uint32(vpids, &vpid, value, node2);
@@ -966,13 +942,13 @@ int opal_proc_table_get_first_key(opal_proc_table_t *pt, opal_process_name_t *ke
     return rc;
 }
 
-int opal_proc_table_get_next_key(opal_proc_table_t *pt, opal_process_name_t *key,
-                                 void **value, void *in_node1, void **out_node1,
-                                 void *in_node2, void **out_node2) {
+int opal_proc_table_get_next_key(opal_proc_table_t *pt, opal_process_name_t *key, void **value,
+                                 void *in_node1, void **out_node1, void *in_node2, void **out_node2)
+{
     int rc;
-    uint32_t jobid = ((opal_hash_element_t *)in_node1)->key.u32;
+    uint32_t jobid = ((opal_hash_element_t *) in_node1)->key.u32;
     uint32_t vpid;
-    opal_hash_table_t * vpids = ((opal_hash_element_t *)in_node1)->value;
+    opal_hash_table_t *vpids = ((opal_hash_element_t *) in_node1)->value;
 
     rc = opal_hash_table_get_next_key_uint32(vpids, &vpid, value, in_node2, out_node2);
     if (OPAL_SUCCESS == rc) {
@@ -981,7 +957,9 @@ int opal_proc_table_get_next_key(opal_proc_table_t *pt, opal_process_name_t *key
         *out_node1 = in_node1;
         return rc;
     }
-    if (OPAL_SUCCESS != (rc=opal_hash_table_get_next_key_uint32(&pt->super, &jobid, (void **)&vpids, in_node1, out_node1))) {
+    if (OPAL_SUCCESS
+        != (rc = opal_hash_table_get_next_key_uint32(&pt->super, &jobid, (void **) &vpids, in_node1,
+                                                     out_node1))) {
         return rc;
     }
     rc = opal_hash_table_get_first_key_uint32(vpids, &vpid, value, out_node2);
diff --git a/opal/class/opal_hash_table.h b/opal/class/opal_hash_table.h
index 4622ed4a81f..37ef25ad3b0 100644
--- a/opal/class/opal_hash_table.h
+++ b/opal/class/opal_hash_table.h
@@ -37,29 +37,26 @@
 
 #include "opal_config.h"
 
-#include <stdint.h>
 #include "opal/class/opal_list.h"
 #include "opal/util/proc.h"
+#include <stdint.h>
 
 BEGIN_C_DECLS
 
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_hash_table_t);
 
-struct opal_hash_table_t
-{
-    opal_object_t        super;          /**< subclass of opal_object_t */
-    struct opal_hash_element_t * ht_table;       /**< table of elements (opaque to users) */
-    size_t               ht_capacity;    /**< allocated size (capacity) of table */
-    size_t               ht_size;        /**< number of extant entries */
-    size_t               ht_growth_trigger; /**< size hits this and table is grown  */
-    int                  ht_density_numer, ht_density_denom; /**< max allowed density of table */
-    int                  ht_growth_numer, ht_growth_denom;   /**< growth factor when grown  */
-    const struct opal_hash_type_methods_t * ht_type_methods;
+struct opal_hash_table_t {
+    opal_object_t super;                    /**< subclass of opal_object_t */
+    struct opal_hash_element_t *ht_table;   /**< table of elements (opaque to users) */
+    size_t ht_capacity;                     /**< allocated size (capacity) of table */
+    size_t ht_size;                         /**< number of extant entries */
+    size_t ht_growth_trigger;               /**< size hits this and table is grown  */
+    int ht_density_numer, ht_density_denom; /**< max allowed density of table */
+    int ht_growth_numer, ht_growth_denom;   /**< growth factor when grown  */
+    const struct opal_hash_type_methods_t *ht_type_methods;
 };
 typedef struct opal_hash_table_t opal_hash_table_t;
 
-
-
 /**
  *  Initializes the table size, must be called before using
  *  the table.
@@ -71,12 +68,12 @@ typedef struct opal_hash_table_t opal_hash_table_t;
  *
  */
 
-OPAL_DECLSPEC int opal_hash_table_init(opal_hash_table_t* ht, size_t table_size);
+OPAL_DECLSPEC int opal_hash_table_init(opal_hash_table_t *ht, size_t table_size);
 
 /* this could be the new init if people wanted a more general API */
-OPAL_DECLSPEC int opal_hash_table_init2(opal_hash_table_t* ht, size_t estimated_max_size,
-                                        int density_numer, int density_denom,
-                                        int growth_numer, int growth_denom);
+OPAL_DECLSPEC int opal_hash_table_init2(opal_hash_table_t *ht, size_t estimated_max_size,
+                                        int density_numer, int density_denom, int growth_numer,
+                                        int growth_denom);
 
 /**
  *  Returns the number of elements currently stored in the table.
@@ -114,8 +111,8 @@ OPAL_DECLSPEC int opal_hash_table_remove_all(opal_hash_table_t *ht);
  *
  */
 
-OPAL_DECLSPEC int opal_hash_table_get_value_uint32(opal_hash_table_t* table, uint32_t key,
-                                                   void** ptr);
+OPAL_DECLSPEC int opal_hash_table_get_value_uint32(opal_hash_table_t *table, uint32_t key,
+                                                   void **ptr);
 
 /**
  *  Set value based on uint32_t key.
@@ -127,7 +124,8 @@ OPAL_DECLSPEC int opal_hash_table_get_value_uint32(opal_hash_table_t* table, uin
  *
  */
 
-OPAL_DECLSPEC int opal_hash_table_set_value_uint32(opal_hash_table_t* table, uint32_t key, void* value);
+OPAL_DECLSPEC int opal_hash_table_set_value_uint32(opal_hash_table_t *table, uint32_t key,
+                                                   void *value);
 
 /**
  *  Remove value based on uint32_t key.
@@ -138,7 +136,7 @@ OPAL_DECLSPEC int opal_hash_table_set_value_uint32(opal_hash_table_t* table, uin
  *
  */
 
-OPAL_DECLSPEC int opal_hash_table_remove_value_uint32(opal_hash_table_t* table, uint32_t key);
+OPAL_DECLSPEC int opal_hash_table_remove_value_uint32(opal_hash_table_t *table, uint32_t key);
 
 /**
  *  Retrieve value via uint64_t key.
@@ -166,7 +164,8 @@ OPAL_DECLSPEC int opal_hash_table_get_value_uint64(opal_hash_table_t *table, uin
  *
  */
 
-OPAL_DECLSPEC int opal_hash_table_set_value_uint64(opal_hash_table_t *table, uint64_t key, void* value);
+OPAL_DECLSPEC int opal_hash_table_set_value_uint64(opal_hash_table_t *table, uint64_t key,
+                                                   void *value);
 
 /**
  *  Remove value based on uint64_t key.
@@ -192,7 +191,7 @@ OPAL_DECLSPEC int opal_hash_table_remove_value_uint64(opal_hash_table_t *table,
  *
  */
 
-OPAL_DECLSPEC int opal_hash_table_get_value_ptr(opal_hash_table_t *table, const void* key,
+OPAL_DECLSPEC int opal_hash_table_get_value_ptr(opal_hash_table_t *table, const void *key,
                                                 size_t keylen, void **ptr);
 
 /**
@@ -205,7 +204,8 @@ OPAL_DECLSPEC int opal_hash_table_get_value_ptr(opal_hash_table_t *table, const
  *
  */
 
-OPAL_DECLSPEC int opal_hash_table_set_value_ptr(opal_hash_table_t *table, const void* key, size_t keylen, void* value);
+OPAL_DECLSPEC int opal_hash_table_set_value_ptr(opal_hash_table_t *table, const void *key,
+                                                size_t keylen, void *value);
 
 /**
  *  Remove value based on arbitrary length binary key.
@@ -216,8 +216,8 @@ OPAL_DECLSPEC int opal_hash_table_set_value_ptr(opal_hash_table_t *table, const
  *
  */
 
-OPAL_DECLSPEC int opal_hash_table_remove_value_ptr(opal_hash_table_t *table, const void* key, size_t keylen);
-
+OPAL_DECLSPEC int opal_hash_table_remove_value_ptr(opal_hash_table_t *table, const void *key,
+                                                   size_t keylen);
 
 /** The following functions are only for allowing iterating through
     the hash table. The calls return along with a key, a pointer to
@@ -242,8 +242,7 @@ OPAL_DECLSPEC int opal_hash_table_remove_value_ptr(opal_hash_table_t *table, con
  */
 
 OPAL_DECLSPEC int opal_hash_table_get_first_key_uint32(opal_hash_table_t *table, uint32_t *key,
-                                        void **value, void **node);
-
+                                                       void **value, void **node);
 
 /**
  *  Get the next 32 bit key from the hash table, knowing the current key
@@ -260,9 +259,7 @@ OPAL_DECLSPEC int opal_hash_table_get_first_key_uint32(opal_hash_table_t *table,
  */
 
 OPAL_DECLSPEC int opal_hash_table_get_next_key_uint32(opal_hash_table_t *table, uint32_t *key,
-                                       void **value, void *in_node,
-                                       void **out_node);
-
+                                                      void **value, void *in_node, void **out_node);
 
 /**
  *  Get the first 64 key from the hash table, which can be used later to
@@ -278,8 +275,7 @@ OPAL_DECLSPEC int opal_hash_table_get_next_key_uint32(opal_hash_table_t *table,
  */
 
 OPAL_DECLSPEC int opal_hash_table_get_first_key_uint64(opal_hash_table_t *table, uint64_t *key,
-                                       void **value, void **node);
-
+                                                       void **value, void **node);
 
 /**
  *  Get the next 64 bit key from the hash table, knowing the current key
@@ -296,9 +292,7 @@ OPAL_DECLSPEC int opal_hash_table_get_first_key_uint64(opal_hash_table_t *table,
  */
 
 OPAL_DECLSPEC int opal_hash_table_get_next_key_uint64(opal_hash_table_t *table, uint64_t *key,
-                                       void **value, void *in_node,
-                                       void **out_node);
-
+                                                      void **value, void *in_node, void **out_node);
 
 /**
  *  Get the first ptr bit key from the hash table, which can be used later to
@@ -314,9 +308,8 @@ OPAL_DECLSPEC int opal_hash_table_get_next_key_uint64(opal_hash_table_t *table,
  *
  */
 
-OPAL_DECLSPEC int opal_hash_table_get_first_key_ptr(opal_hash_table_t *table, void* *key,
-                                        size_t *key_size, void **value, void **node);
-
+OPAL_DECLSPEC int opal_hash_table_get_first_key_ptr(opal_hash_table_t *table, void **key,
+                                                    size_t *key_size, void **value, void **node);
 
 /**
  *  Get the next ptr bit key from the hash table, knowing the current key
@@ -333,28 +326,23 @@ OPAL_DECLSPEC int opal_hash_table_get_first_key_ptr(opal_hash_table_t *table, vo
  *
  */
 
-OPAL_DECLSPEC int opal_hash_table_get_next_key_ptr(opal_hash_table_t *table, void* *key,
-                                       size_t *key_size, void **value,
-                                       void *in_node, void **out_node);
-
-
+OPAL_DECLSPEC int opal_hash_table_get_next_key_ptr(opal_hash_table_t *table, void **key,
+                                                   size_t *key_size, void **value, void *in_node,
+                                                   void **out_node);
 
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_proc_table_t);
 
-struct opal_proc_table_t
-{
-    opal_hash_table_t    super;          /**< subclass of opal_object_t */
-    size_t               pt_size;        /**< number of extant entries */
-    size_t               vpids_size;
+struct opal_proc_table_t {
+    opal_hash_table_t super; /**< subclass of opal_object_t */
+    size_t pt_size;          /**< number of extant entries */
+    size_t vpids_size;
     // FIXME
     // Begin KLUDGE!!  So ompi/debuggers/ompi_common_dll.c doesn't complain
-    size_t              pt_table_size;  /**< size of table */
+    size_t pt_table_size; /**< size of table */
     // End KLUDGE
 };
 typedef struct opal_proc_table_t opal_proc_table_t;
 
-
-
 /**
  *  Initializes the table size, must be called before using
  *  the table.
@@ -368,7 +356,7 @@ typedef struct opal_proc_table_t opal_proc_table_t;
  *
  */
 
-OPAL_DECLSPEC int opal_proc_table_init(opal_proc_table_t* pt, size_t jobids, size_t vpids);
+OPAL_DECLSPEC int opal_proc_table_init(opal_proc_table_t *pt, size_t jobids, size_t vpids);
 
 /**
  *  Remove all elements from the table.
@@ -393,8 +381,8 @@ OPAL_DECLSPEC int opal_proc_table_remove_all(opal_proc_table_t *pt);
  *
  */
 
-OPAL_DECLSPEC int opal_proc_table_get_value(opal_proc_table_t* pt, opal_process_name_t key,
-                                                   void** ptr);
+OPAL_DECLSPEC int opal_proc_table_get_value(opal_proc_table_t *pt, opal_process_name_t key,
+                                            void **ptr);
 
 /**
  *  Set value based on opal_process_name_t key.
@@ -406,7 +394,8 @@ OPAL_DECLSPEC int opal_proc_table_get_value(opal_proc_table_t* pt, opal_process_
  *
  */
 
-OPAL_DECLSPEC int opal_proc_table_set_value(opal_proc_table_t* pt, opal_process_name_t key, void* value);
+OPAL_DECLSPEC int opal_proc_table_set_value(opal_proc_table_t *pt, opal_process_name_t key,
+                                            void *value);
 
 /**
  *  Remove value based on opal_process_name_t key.
@@ -417,8 +406,7 @@ OPAL_DECLSPEC int opal_proc_table_set_value(opal_proc_table_t* pt, opal_process_
  *
  */
 
-OPAL_DECLSPEC int opal_proc_table_remove_value(opal_proc_table_t* pt, opal_process_name_t key);
-
+OPAL_DECLSPEC int opal_proc_table_remove_value(opal_proc_table_t *pt, opal_process_name_t key);
 
 /**
  *  Get the first opal_process_name_t key from the hash table, which can be used later to
@@ -439,7 +427,6 @@ OPAL_DECLSPEC int opal_proc_table_remove_value(opal_proc_table_t* pt, opal_proce
 OPAL_DECLSPEC int opal_proc_table_get_first_key(opal_proc_table_t *pt, opal_process_name_t *key,
                                                 void **value, void **node1, void **node2);
 
-
 /**
  *  Get the next opal_process_name_t key from the hash table, knowing the current key
  *  @param  pt       The hash table pointer (IN)
@@ -483,16 +470,19 @@ OPAL_DECLSPEC int opal_proc_table_get_next_key(opal_proc_table_t *pt, opal_proce
  * }
  */
 #define OPAL_HASH_TABLE_FOREACH(key, type, value, ht) \
-  for (void *_nptr=NULL;                                   \
-       OPAL_SUCCESS == opal_hash_table_get_next_key_##type(ht, &key, (void **)&value, _nptr, &_nptr);)
-
-#define OPAL_HASH_TABLE_FOREACH_PTR(key, value, ht, body)		\
-  {									\
-    size_t key_size_;							\
-    for (void *_nptr=NULL;						\
-	 OPAL_SUCCESS == opal_hash_table_get_next_key_ptr (ht, &key, &key_size_, (void **)&value, _nptr, &_nptr);) \
-      body								\
-  }
+    for (void *_nptr = NULL;                          \
+         OPAL_SUCCESS                                 \
+         == opal_hash_table_get_next_key_##type(ht, &key, (void **) &value, _nptr, &_nptr);)
+
+#define OPAL_HASH_TABLE_FOREACH_PTR(key, value, ht, body)                                       \
+    {                                                                                           \
+        size_t key_size_;                                                                       \
+        for (void *_nptr = NULL;                                                                \
+             OPAL_SUCCESS                                                                       \
+             == opal_hash_table_get_next_key_ptr(ht, &key, &key_size_, (void **) &value, _nptr, \
+                                                 &_nptr);)                                      \
+            body                                                                                \
+    }
 END_C_DECLS
 
-#endif  /* OPAL_HASH_TABLE_H */
+#endif /* OPAL_HASH_TABLE_H */
diff --git a/opal/class/opal_hotel.c b/opal/class/opal_hotel.c
index 50b46e10a58..4777a54aba9 100644
--- a/opal/class/opal_hotel.c
+++ b/opal/class/opal_hotel.c
@@ -13,17 +13,16 @@
 
 #include "opal_config.h"
 
-#include <stdio.h>
 #include <stddef.h>
+#include <stdio.h>
 
-#include "opal/util/event.h"
 #include "opal/class/opal_hotel.h"
-
+#include "opal/util/event.h"
 
 static void local_eviction_callback(int fd, short flags, void *arg)
 {
-    opal_hotel_room_eviction_callback_arg_t *eargs =
-        (opal_hotel_room_eviction_callback_arg_t*) arg;
+    opal_hotel_room_eviction_callback_arg_t *eargs = (opal_hotel_room_eviction_callback_arg_t *)
+        arg;
     void *occupant = eargs->hotel->rooms[eargs->room_num].occupant;
 
     /* Remove the occurpant from the room.
@@ -39,23 +38,17 @@ static void local_eviction_callback(int fd, short flags, void *arg)
     hotel->unoccupied_rooms[hotel->last_unoccupied_room] = eargs->room_num;
 
     /* Invoke the user callback to tell them that they were evicted */
-    hotel->evict_callback_fn(hotel,
-                             eargs->room_num,
-                             occupant);
+    hotel->evict_callback_fn(hotel, eargs->room_num, occupant);
 }
 
-
-int opal_hotel_init(opal_hotel_t *h, int num_rooms,
-                    opal_event_base_t *evbase,
-                    uint32_t eviction_timeout,
-                    int eviction_event_priority,
+int opal_hotel_init(opal_hotel_t *h, int num_rooms, opal_event_base_t *evbase,
+                    uint32_t eviction_timeout, int eviction_event_priority,
                     opal_hotel_eviction_callback_fn_t evict_callback_fn)
 {
     int i;
 
     /* Bozo check */
-    if (num_rooms <= 0 ||
-        NULL == evict_callback_fn) {
+    if (num_rooms <= 0 || NULL == evict_callback_fn) {
         return OPAL_ERR_BAD_PARAM;
     }
 
@@ -64,12 +57,12 @@ int opal_hotel_init(opal_hotel_t *h, int num_rooms,
     h->eviction_timeout.tv_usec = eviction_timeout % 1000000;
     h->eviction_timeout.tv_sec = eviction_timeout / 1000000;
     h->evict_callback_fn = evict_callback_fn;
-    h->rooms = (opal_hotel_room_t*)malloc(num_rooms * sizeof(opal_hotel_room_t));
+    h->rooms = (opal_hotel_room_t *) malloc(num_rooms * sizeof(opal_hotel_room_t));
     if (NULL != evict_callback_fn) {
-        h->eviction_args =
-            (opal_hotel_room_eviction_callback_arg_t*)malloc(num_rooms * sizeof(opal_hotel_room_eviction_callback_arg_t));
+        h->eviction_args = (opal_hotel_room_eviction_callback_arg_t *) malloc(
+            num_rooms * sizeof(opal_hotel_room_eviction_callback_arg_t));
     }
-    h->unoccupied_rooms = (int*) malloc(num_rooms * sizeof(int));
+    h->unoccupied_rooms = (int *) malloc(num_rooms * sizeof(int));
     h->last_unoccupied_room = num_rooms - 1;
 
     for (i = 0; i < num_rooms; ++i) {
@@ -85,14 +78,11 @@ int opal_hotel_init(opal_hotel_t *h, int num_rooms,
 
         /* Create this room's event (but don't add it) */
         if (NULL != h->evbase) {
-            opal_event_set(h->evbase,
-                           &(h->rooms[i].eviction_timer_event),
-                           -1, 0, local_eviction_callback,
-                           &(h->eviction_args[i]));
+            opal_event_set(h->evbase, &(h->rooms[i].eviction_timer_event), -1, 0,
+                           local_eviction_callback, &(h->eviction_args[i]));
 
             /* Set the priority so it gets serviced properly */
-            opal_event_set_priority(&(h->rooms[i].eviction_timer_event),
-                                    eviction_event_priority);
+            opal_event_set_priority(&(h->rooms[i].eviction_timer_event), eviction_event_priority);
         }
     }
 
@@ -136,7 +126,4 @@ static void destructor(opal_hotel_t *h)
     }
 }
 
-OBJ_CLASS_INSTANCE(opal_hotel_t,
-                   opal_object_t,
-                   constructor,
-                   destructor);
+OBJ_CLASS_INSTANCE(opal_hotel_t, opal_object_t, constructor, destructor);
diff --git a/opal/class/opal_hotel.h b/opal/class/opal_hotel.h
index 52cf5a4b1b4..b8406efe776 100644
--- a/opal/class/opal_hotel.h
+++ b/opal/class/opal_hotel.h
@@ -55,19 +55,18 @@
 
 #include "opal_config.h"
 
+#include "opal/class/opal_object.h"
 #include "opal/constants.h"
-#include "opal/util/output.h"
 #include "opal/prefetch.h"
-#include "opal/class/opal_object.h"
 #include "opal/util/event.h"
+#include "opal/util/output.h"
 
 BEGIN_C_DECLS
 
 struct opal_hotel_t;
 
 /* User-supplied function to be invoked when an occupant is evicted. */
-typedef void (*opal_hotel_eviction_callback_fn_t)(struct opal_hotel_t *hotel,
-                                                  int room_num,
+typedef void (*opal_hotel_eviction_callback_fn_t)(struct opal_hotel_t *hotel, int room_num,
                                                   void *occupant);
 
 /* Note that this is an internal data structure; it is not part of the
@@ -155,13 +154,11 @@ OBJ_CLASS_DECLARATION(opal_hotel_t);
  * already been ("forcibly") checked out *before* the
  * eviction_callback_fn is invoked.
  *
- * @return OPAL_SUCCESS if all initializations were succesful. Otherwise,
+ * @return OPAL_SUCCESS if all initializations were successful. Otherwise,
  *  the error indicate what went wrong in the function.
  */
-OPAL_DECLSPEC int opal_hotel_init(opal_hotel_t *hotel, int num_rooms,
-                                  opal_event_base_t *evbase,
-                                  uint32_t eviction_timeout,
-                                  int eviction_event_priority,
+OPAL_DECLSPEC int opal_hotel_init(opal_hotel_t *hotel, int num_rooms, opal_event_base_t *evbase,
+                                  uint32_t eviction_timeout, int eviction_event_priority,
                                   opal_hotel_eviction_callback_fn_t evict_callback_fn);
 
 /**
@@ -184,9 +181,7 @@ OPAL_DECLSPEC int opal_hotel_init(opal_hotel_t *hotel, int num_rooms,
  * @return OPAL_ERR_TEMP_OUT_OF_RESOURCE is the hotel is full.  Try
  * again later.
  */
-static inline int opal_hotel_checkin(opal_hotel_t *hotel,
-                                     void *occupant,
-                                     int *room_num)
+static inline int opal_hotel_checkin(opal_hotel_t *hotel, void *occupant, int *room_num)
 {
     opal_hotel_room_t *room;
 
@@ -202,8 +197,7 @@ static inline int opal_hotel_checkin(opal_hotel_t *hotel,
 
     /* Assign the event and make it pending */
     if (NULL != hotel->evbase) {
-        opal_event_add(&(room->eviction_timer_event),
-                       &(hotel->eviction_timeout));
+        opal_event_add(&(room->eviction_timer_event), &(hotel->eviction_timeout));
     }
 
     return OPAL_SUCCESS;
@@ -213,9 +207,7 @@ static inline int opal_hotel_checkin(opal_hotel_t *hotel,
  * Same as opal_hotel_checkin(), but slightly optimized for when the
  * caller *knows* that there is a room available.
  */
-static inline void opal_hotel_checkin_with_res(opal_hotel_t *hotel,
-                                     void *occupant,
-                                     int *room_num)
+static inline void opal_hotel_checkin_with_res(opal_hotel_t *hotel, void *occupant, int *room_num)
 {
     opal_hotel_room_t *room;
 
@@ -227,8 +219,7 @@ static inline void opal_hotel_checkin_with_res(opal_hotel_t *hotel,
 
     /* Assign the event and make it pending */
     if (NULL != hotel->evbase) {
-        opal_event_add(&(room->eviction_timer_event),
-                       &(hotel->eviction_timeout));
+        opal_event_add(&(room->eviction_timer_event), &(hotel->eviction_timeout));
     }
 }
 
@@ -281,7 +272,8 @@ static inline void opal_hotel_checkout(opal_hotel_t *hotel, int room_num)
  *
  * Use this checkout and when caller needs the occupant
  */
-static inline void opal_hotel_checkout_and_return_occupant(opal_hotel_t *hotel, int room_num, void **occupant)
+static inline void opal_hotel_checkout_and_return_occupant(opal_hotel_t *hotel, int room_num,
+                                                           void **occupant)
 {
     opal_hotel_room_t *room;
 
@@ -291,7 +283,7 @@ static inline void opal_hotel_checkout_and_return_occupant(opal_hotel_t *hotel,
     /* If there's an occupant in the room, check them out */
     room = &(hotel->rooms[room_num]);
     if (OPAL_LIKELY(NULL != room->occupant)) {
-        opal_output (10, "checking out occupant %p from room num %d", room->occupant, room_num);
+        opal_output(10, "checking out occupant %p from room num %d", room->occupant, room_num);
         /* Do not change this logic without also changing the same
            logic in opal_hotel_checkout() and
            opal_hotel.c:local_eviction_callback(). */
@@ -303,8 +295,7 @@ static inline void opal_hotel_checkout_and_return_occupant(opal_hotel_t *hotel,
         hotel->last_unoccupied_room++;
         assert(hotel->last_unoccupied_room < hotel->num_rooms);
         hotel->unoccupied_rooms[hotel->last_unoccupied_room] = room_num;
-    }
-    else {
+    } else {
         *occupant = NULL;
     }
 }
@@ -315,7 +306,7 @@ static inline void opal_hotel_checkout_and_return_occupant(opal_hotel_t *hotel,
  * @return bool true if empty false if there is a occupant(s)
  *
  */
-static inline bool opal_hotel_is_empty (opal_hotel_t *hotel)
+static inline bool opal_hotel_is_empty(opal_hotel_t *hotel)
 {
     if (hotel->last_unoccupied_room == hotel->num_rooms - 1)
         return true;
@@ -345,7 +336,7 @@ static inline void opal_hotel_knock(opal_hotel_t *hotel, int room_num, void **oc
     /* If there's an occupant in the room, have them come to the door */
     room = &(hotel->rooms[room_num]);
     if (OPAL_LIKELY(NULL != room->occupant)) {
-        opal_output (10, "occupant %p in room num %d responded to knock", room->occupant, room_num);
+        opal_output(10, "occupant %p in room num %d responded to knock", room->occupant, room_num);
         *occupant = room->occupant;
     }
 }
diff --git a/opal/class/opal_interval_tree.c b/opal/class/opal_interval_tree.c
index 8b6fb2dd61f..7b3577082c6 100644
--- a/opal/class/opal_interval_tree.c
+++ b/opal/class/opal_interval_tree.c
@@ -13,6 +13,7 @@
  * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2020      Google, LLC. All rights reserved.
+ * Copyright (c) 2025      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -29,25 +30,29 @@
 #include <limits.h>
 
 /* Private functions */
-static void opal_interval_tree_insert_node (opal_interval_tree_t *tree, opal_interval_tree_node_t *node);
+static void opal_interval_tree_insert_node(opal_interval_tree_t *tree,
+                                           opal_interval_tree_node_t *node);
 
 /* tree rebalancing functions */
-static void opal_interval_tree_delete_fixup (opal_interval_tree_t *tree, opal_interval_tree_node_t *node,
-                                             opal_interval_tree_node_t *parent);
-static void opal_interval_tree_insert_fixup (opal_interval_tree_t *tree, opal_interval_tree_node_t *x);
+static void opal_interval_tree_delete_fixup(opal_interval_tree_t *tree,
+                                            opal_interval_tree_node_t *node,
+                                            opal_interval_tree_node_t *parent);
+static void opal_interval_tree_insert_fixup(opal_interval_tree_t *tree,
+                                            opal_interval_tree_node_t *x);
 
-static opal_interval_tree_node_t *opal_interval_tree_next (opal_interval_tree_t *tree,
-                                                           opal_interval_tree_node_t *node);
-static opal_interval_tree_node_t * opal_interval_tree_find_node(opal_interval_tree_t *tree,
-                                                                uint64_t low, uint64_t high,
-                                                                void *data);
+static opal_interval_tree_node_t *opal_interval_tree_next(opal_interval_tree_t *tree,
+                                                          opal_interval_tree_node_t *node);
+static opal_interval_tree_node_t *
+opal_interval_tree_find_node(opal_interval_tree_t *tree, uint64_t low, uint64_t high, void *data);
 
-static opal_interval_tree_node_t *left_rotate (opal_interval_tree_t *tree, opal_interval_tree_node_t *x);
-static opal_interval_tree_node_t *right_rotate (opal_interval_tree_t *tree, opal_interval_tree_node_t *x);
+static opal_interval_tree_node_t *left_rotate(opal_interval_tree_t *tree,
+                                              opal_interval_tree_node_t *x);
+static opal_interval_tree_node_t *right_rotate(opal_interval_tree_t *tree,
+                                               opal_interval_tree_node_t *x);
 
-static void inorder_destroy(opal_interval_tree_t *tree, opal_interval_tree_node_t * node);
+static void inorder_destroy(opal_interval_tree_t *tree, opal_interval_tree_node_t *node);
 
-#define max(x,y) (((x) > (y)) ? (x) : (y))
+#define max(x, y) (((x) > (y)) ? (x) : (y))
 
 /**
  * the constructor function. creates the free list to get the nodes from
@@ -56,7 +61,7 @@ static void inorder_destroy(opal_interval_tree_t *tree, opal_interval_tree_node_
  *
  * @retval NONE
  */
-static void opal_interval_tree_construct (opal_interval_tree_t *tree)
+static void opal_interval_tree_construct(opal_interval_tree_t *tree)
 {
     OBJ_CONSTRUCT(&tree->root, opal_interval_tree_node_t);
     OBJ_CONSTRUCT(&tree->nill, opal_interval_tree_node_t);
@@ -81,11 +86,12 @@ static void opal_interval_tree_construct (opal_interval_tree_t *tree)
     tree->tree_size = 0;
     tree->lock = 0;
     tree->reader_count = 0;
+    tree->reader_id = 0;
     tree->epoch = 0;
 
-    /* set all reader epochs to UINT_MAX. this value is used to simplfy
+    /* set all reader epochs to UINT_MAX. this value is used to simplify
      * checks against the current epoch. */
-    for (int i = 0 ; i < OPAL_INTERVAL_TREE_MAX_READERS ; ++i) {
+    for (int i = 0; i < OPAL_INTERVAL_TREE_MAX_READERS; ++i) {
         tree->reader_epochs[i] = UINT_MAX;
     }
 }
@@ -95,9 +101,9 @@ static void opal_interval_tree_construct (opal_interval_tree_t *tree)
  *
  * @param object the tree object
  */
-static void opal_interval_tree_destruct (opal_interval_tree_t *tree)
+static void opal_interval_tree_destruct(opal_interval_tree_t *tree)
 {
-    opal_interval_tree_destroy (tree);
+    opal_interval_tree_destroy(tree);
 
     OBJ_DESTRUCT(&tree->free_list);
     OBJ_DESTRUCT(&tree->root);
@@ -114,7 +120,7 @@ typedef int32_t opal_interval_tree_token_t;
 /**
  * @brief pick and return a reader slot
  */
-static opal_interval_tree_token_t opal_interval_tree_reader_get_token (opal_interval_tree_t *tree)
+static opal_interval_tree_token_t opal_interval_tree_reader_get_token(opal_interval_tree_t *tree)
 {
     opal_interval_tree_token_t token = -1;
 
@@ -125,49 +131,56 @@ static opal_interval_tree_token_t opal_interval_tree_reader_get_token (opal_inte
          * using atomics. */
         token = tree->reader_id++ % OPAL_INTERVAL_TREE_MAX_READERS;
         while (OPAL_UNLIKELY(reader_count <= token)) {
-            if (opal_atomic_compare_exchange_strong_32 (&tree->reader_count, &reader_count, token + 1)) {
+            if (opal_atomic_compare_exchange_strong_32(&tree->reader_count, &reader_count,
+                                                       token + 1)) {
                 break;
             }
         }
     }
 
-    while (!OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_32((opal_atomic_int32_t *) &tree->reader_epochs[token],
-                                                   &(int32_t) {UINT_MAX}, tree->epoch));
+    while (
+        !OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_32((opal_atomic_int32_t *) &tree->reader_epochs[token],
+                                                &(int32_t){UINT_MAX}, tree->epoch)) {
+    }
 
     return token;
 }
 
-static void opal_interval_tree_reader_return_token (opal_interval_tree_t *tree, opal_interval_tree_token_t token)
+static void opal_interval_tree_reader_return_token(opal_interval_tree_t *tree,
+                                                   opal_interval_tree_token_t token)
 {
     tree->reader_epochs[token] = UINT_MAX;
 }
 
 /* Create the tree */
-int opal_interval_tree_init (opal_interval_tree_t *tree)
+int opal_interval_tree_init(opal_interval_tree_t *tree)
 {
-    return opal_free_list_init (&tree->free_list, sizeof(opal_interval_tree_node_t),
-                                opal_cache_line_size, OBJ_CLASS(opal_interval_tree_node_t),
-                                0, opal_cache_line_size, 0, -1 , 128, NULL, 0, NULL, NULL, NULL);
+    return opal_free_list_init(&tree->free_list, sizeof(opal_interval_tree_node_t),
+                               opal_cache_line_size, OBJ_CLASS(opal_interval_tree_node_t), 0,
+                               opal_cache_line_size, 0, -1, 128, NULL, 0, NULL, NULL, NULL);
 }
 
-static bool opal_interval_tree_write_trylock (opal_interval_tree_t *tree)
+static bool opal_interval_tree_write_trylock(opal_interval_tree_t *tree)
 {
-    opal_atomic_rmb ();
-    return !(tree->lock || opal_atomic_swap_32 (&tree->lock, 1));
+    opal_atomic_rmb();
+    return !(tree->lock || opal_atomic_swap_32(&tree->lock, 1));
 }
 
-static void opal_interval_tree_write_lock (opal_interval_tree_t *tree)
+static void opal_interval_tree_write_lock(opal_interval_tree_t *tree)
 {
-    while (!opal_interval_tree_write_trylock (tree));
+    while (!opal_interval_tree_write_trylock(tree)) {
+    }
 }
 
-static void opal_interval_tree_write_unlock (opal_interval_tree_t *tree)
+static void opal_interval_tree_write_unlock(opal_interval_tree_t *tree)
 {
-    opal_atomic_wmb ();
+    opal_atomic_wmb();
     tree->lock = 0;
 }
 
-static void opal_interval_tree_insert_fixup_helper (opal_interval_tree_t *tree, opal_interval_tree_node_t *node) {
+static void opal_interval_tree_insert_fixup_helper(opal_interval_tree_t *tree,
+                                                   opal_interval_tree_node_t *node)
+{
     opal_interval_tree_node_t *y, *parent = node->parent;
     bool rotate_right = false;
 
@@ -186,13 +199,13 @@ static void opal_interval_tree_insert_fixup_helper (opal_interval_tree_t *tree,
         parent->color = OPAL_INTERVAL_TREE_COLOR_BLACK;
         y->color = OPAL_INTERVAL_TREE_COLOR_BLACK;
         parent->parent->color = OPAL_INTERVAL_TREE_COLOR_RED;
-        opal_interval_tree_insert_fixup_helper (tree, parent->parent);
+        opal_interval_tree_insert_fixup_helper(tree, parent->parent);
         return;
     }
 
     if (rotate_right) {
         if (node == parent->right) {
-            node = left_rotate (tree, parent);
+            node = left_rotate(tree, parent);
             parent = node->parent;
         }
 
@@ -209,14 +222,16 @@ static void opal_interval_tree_insert_fixup_helper (opal_interval_tree_t *tree,
         (void) left_rotate(tree, parent->parent);
     }
 
-    opal_interval_tree_insert_fixup_helper (tree, node);
+    opal_interval_tree_insert_fixup_helper(tree, node);
 }
 
-static void opal_interval_tree_insert_fixup (opal_interval_tree_t *tree, opal_interval_tree_node_t *node) {
+static void opal_interval_tree_insert_fixup(opal_interval_tree_t *tree,
+                                            opal_interval_tree_node_t *node)
+{
     /* do the rotations */
     /* usually one would have to check for NULL, but because of the sentinal,
      * we don't have to   */
-    opal_interval_tree_insert_fixup_helper (tree, node);
+    opal_interval_tree_insert_fixup_helper(tree, node);
 
     /* after the rotations the root is black */
     tree->root.left->color = OPAL_INTERVAL_TREE_COLOR_BLACK;
@@ -232,8 +247,8 @@ static void opal_interval_tree_insert_fixup (opal_interval_tree_t *tree, opal_in
  * @returns the next node to fixup or root if done
  */
 static inline opal_interval_tree_node_t *
-opal_interval_tree_delete_fixup_helper (opal_interval_tree_t *tree, opal_interval_tree_node_t *node,
-                                        opal_interval_tree_node_t *parent, const bool left)
+opal_interval_tree_delete_fixup_helper(opal_interval_tree_t *tree, opal_interval_tree_node_t *node,
+                                       opal_interval_tree_node_t *parent, const bool left)
 {
     opal_interval_tree_node_t *w;
 
@@ -251,7 +266,8 @@ opal_interval_tree_delete_fixup_helper (opal_interval_tree_t *tree, opal_interva
         }
     }
 
-    if ((w->left->color == OPAL_INTERVAL_TREE_COLOR_BLACK) && (w->right->color == OPAL_INTERVAL_TREE_COLOR_BLACK)) {
+    if ((w->left->color == OPAL_INTERVAL_TREE_COLOR_BLACK)
+        && (w->right->color == OPAL_INTERVAL_TREE_COLOR_BLACK)) {
         w->color = OPAL_INTERVAL_TREE_COLOR_RED;
         return parent;
     }
@@ -282,14 +298,15 @@ opal_interval_tree_delete_fixup_helper (opal_interval_tree_t *tree, opal_interva
 
     /* return the root */
     return tree->root.left;
- }
+}
 
 /* Fixup the balance of the btree after deletion    */
-static void opal_interval_tree_delete_fixup (opal_interval_tree_t *tree, opal_interval_tree_node_t *node,
-                                             opal_interval_tree_node_t *parent)
+static void opal_interval_tree_delete_fixup(opal_interval_tree_t *tree,
+                                            opal_interval_tree_node_t *node,
+                                            opal_interval_tree_node_t *parent)
 {
     while ((node != tree->root.left) && (node->color == OPAL_INTERVAL_TREE_COLOR_BLACK)) {
-        node = opal_interval_tree_delete_fixup_helper (tree, node, parent, node == parent->left);
+        node = opal_interval_tree_delete_fixup_helper(tree, node, parent, node == parent->left);
         parent = node->parent;
     }
 
@@ -299,44 +316,45 @@ static void opal_interval_tree_delete_fixup (opal_interval_tree_t *tree, opal_in
 
 /* traverse the garbage-collection list and return any nodes that can not have any
  * references. this function MUST be called with the writer lock held. */
-static void opal_interval_tree_gc_clean (opal_interval_tree_t *tree)
+static void opal_interval_tree_gc_clean(opal_interval_tree_t *tree)
 {
     opal_interval_tree_node_t *node, *next;
     uint32_t oldest_epoch = UINT_MAX;
 
-    if (0 == opal_list_get_size (&tree->gc_list)) {
+    if (0 == opal_list_get_size(&tree->gc_list)) {
         return;
     }
 
-    for (int i = 0 ; i < tree->reader_count ; ++i) {
-        oldest_epoch = (oldest_epoch < tree->reader_epochs[i]) ? oldest_epoch : tree->reader_epochs[i];
+    for (int i = 0; i < tree->reader_count; ++i) {
+        oldest_epoch = (oldest_epoch < tree->reader_epochs[i]) ? oldest_epoch
+                                                               : tree->reader_epochs[i];
     }
 
-    OPAL_LIST_FOREACH_SAFE(node, next, &tree->gc_list, opal_interval_tree_node_t) {
+    OPAL_LIST_FOREACH_SAFE (node, next, &tree->gc_list, opal_interval_tree_node_t) {
         if (node->epoch < oldest_epoch) {
-            opal_list_remove_item (&tree->gc_list, &node->super.super);
-            opal_free_list_return_st (&tree->free_list, &node->super);
+            opal_list_remove_item(&tree->gc_list, &node->super.super);
+            opal_free_list_return_st(&tree->free_list, &node->super);
         }
     }
 }
 
 /* This inserts a node into the tree based on the passed values. */
-int opal_interval_tree_insert (opal_interval_tree_t *tree, void *value, uint64_t low, uint64_t high)
+int opal_interval_tree_insert(opal_interval_tree_t *tree, void *value, uint64_t low, uint64_t high)
 {
-    opal_interval_tree_node_t * node;
+    opal_interval_tree_node_t *node;
 
     if (low > high) {
         return OPAL_ERR_BAD_PARAM;
     }
 
-    opal_interval_tree_write_lock (tree);
+    opal_interval_tree_write_lock(tree);
 
-    opal_interval_tree_gc_clean (tree);
+    opal_interval_tree_gc_clean(tree);
 
     /* get the memory for a node */
-    node = (opal_interval_tree_node_t *) opal_free_list_get (&tree->free_list);
+    node = (opal_interval_tree_node_t *) opal_free_list_get(&tree->free_list);
     if (OPAL_UNLIKELY(NULL == node)) {
-        opal_interval_tree_write_unlock (tree);
+        opal_interval_tree_write_unlock(tree);
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
@@ -348,17 +366,19 @@ int opal_interval_tree_insert (opal_interval_tree_t *tree, void *value, uint64_t
     node->epoch = tree->epoch;
 
     /* insert the node into the tree */
-    opal_interval_tree_insert_node (tree, node);
+    opal_interval_tree_insert_node(tree, node);
 
-    opal_interval_tree_insert_fixup (tree, node);
-    opal_interval_tree_write_unlock (tree);
+    opal_interval_tree_insert_fixup(tree, node);
+    opal_interval_tree_write_unlock(tree);
 
     return OPAL_SUCCESS;
 }
 
-static int opal_interval_tree_compare_node(opal_interval_tree_node_t *node, uint64_t low, uint64_t high, void *data) {
-    if ((data && node->low == low && node->high == high && node->data == data) ||
-        (!data && node->low <= low && node->high >= high)) {
+static int opal_interval_tree_compare_node(opal_interval_tree_node_t *node, uint64_t low,
+                                           uint64_t high, void *data)
+{
+    if ((data && node->low == low && node->high == high && node->data == data)
+        || (!data && node->low <= low && node->high >= high)) {
         return 0;
     }
     if (node->low > low) {
@@ -379,8 +399,10 @@ static int opal_interval_tree_compare_node(opal_interval_tree_node_t *node, uint
     return 1;
 }
 
-static opal_interval_tree_node_t *opal_interval_tree_find_interval(opal_interval_tree_t *tree, opal_interval_tree_node_t *node, uint64_t low,
-                                                                   uint64_t high, void *data)
+static opal_interval_tree_node_t *opal_interval_tree_find_interval(opal_interval_tree_t *tree,
+                                                                   opal_interval_tree_node_t *node,
+                                                                   uint64_t low, uint64_t high,
+                                                                   void *data)
 {
     if (node == &tree->nill) {
         return NULL;
@@ -392,136 +414,144 @@ static opal_interval_tree_node_t *opal_interval_tree_find_interval(opal_interval
     }
 
     if (-1 == check) {
-        return opal_interval_tree_find_interval (tree, node->left, low, high, data);
+        return opal_interval_tree_find_interval(tree, node->left, low, high, data);
     }
 
-    return opal_interval_tree_find_interval (tree, node->right, low, high, data);
+    return opal_interval_tree_find_interval(tree, node->right, low, high, data);
 }
 
 /* Finds the node in the tree based on the key and returns a pointer
  * to the node. This is a bit a code duplication, but this has to be fast
  * so we go ahead with the duplication */
-static opal_interval_tree_node_t *opal_interval_tree_find_node(opal_interval_tree_t *tree, uint64_t low, uint64_t high, void *data)
+static opal_interval_tree_node_t *
+opal_interval_tree_find_node(opal_interval_tree_t *tree, uint64_t low, uint64_t high, void *data)
 {
-    return opal_interval_tree_find_interval (tree, tree->root.left, low, high, data);
+    return opal_interval_tree_find_interval(tree, tree->root.left, low, high, data);
 }
 
-void *opal_interval_tree_find_overlapping (opal_interval_tree_t *tree, uint64_t low, uint64_t high)
+void *opal_interval_tree_find_overlapping(opal_interval_tree_t *tree, uint64_t low, uint64_t high)
 {
     opal_interval_tree_token_t token;
     opal_interval_tree_node_t *node;
 
-    token = opal_interval_tree_reader_get_token (tree);
-    node = opal_interval_tree_find_node (tree, low, high, NULL);
-    opal_interval_tree_reader_return_token (tree, token);
+    token = opal_interval_tree_reader_get_token(tree);
+    node = opal_interval_tree_find_node(tree, low, high, NULL);
+    opal_interval_tree_reader_return_token(tree, token);
 
     return node ? node->data : NULL;
 }
 
-static size_t opal_interval_tree_depth_node (opal_interval_tree_t *tree, opal_interval_tree_node_t *node)
+static size_t opal_interval_tree_depth_node(opal_interval_tree_t *tree,
+                                            opal_interval_tree_node_t *node)
 {
     if (&tree->nill == node) {
         return 0;
     }
 
-    return 1 + max (opal_interval_tree_depth_node (tree, node->right), opal_interval_tree_depth_node (tree, node->left));
+    return 1
+           + max(opal_interval_tree_depth_node(tree, node->right),
+                 opal_interval_tree_depth_node(tree, node->left));
 }
 
-size_t opal_interval_tree_depth (opal_interval_tree_t *tree)
+size_t opal_interval_tree_depth(opal_interval_tree_t *tree)
 {
     opal_interval_tree_token_t token;
     size_t depth;
 
-    token = opal_interval_tree_reader_get_token (tree);
-    depth = opal_interval_tree_depth_node (tree, &tree->root);
-    opal_interval_tree_reader_return_token (tree, token);
+    token = opal_interval_tree_reader_get_token(tree);
+    depth = opal_interval_tree_depth_node(tree, &tree->root);
+    opal_interval_tree_reader_return_token(tree, token);
 
     return depth;
 }
 
 /* update the value of a tree pointer */
-static inline void rp_publish (opal_interval_tree_node_t **ptr, opal_interval_tree_node_t *node)
+static inline void rp_publish(opal_interval_tree_node_t **ptr, opal_interval_tree_node_t *node)
 {
     /* ensure all writes complete before continuing */
-    opal_atomic_wmb ();
+    opal_atomic_wmb();
     /* just set the value */
     *ptr = node;
 }
 
-
-static inline void rp_wait_for_readers (opal_interval_tree_t *tree)
+static inline void rp_wait_for_readers(opal_interval_tree_t *tree)
 {
     uint32_t epoch_id = ++tree->epoch;
 
     /* wait for all readers to see the new tree version */
-    for (int i = 0 ; i < tree->reader_count ; ++i) {
-        while (tree->reader_epochs[i] < epoch_id);
+    for (int i = 0; i < tree->reader_count; ++i) {
+        while (tree->reader_epochs[i] < epoch_id) {
+        }
     }
 }
 
 /* waits for all writers to finish with the node then releases the last reference */
-static inline void rp_free_wait (opal_interval_tree_t *tree, opal_interval_tree_node_t *node)
+static inline void rp_free_wait(opal_interval_tree_t *tree, opal_interval_tree_node_t *node)
 {
-    rp_wait_for_readers (tree);
+    rp_wait_for_readers(tree);
     /* no other threads are working on this node so go ahead and return it */
-    opal_free_list_return_st (&tree->free_list, &node->super);
+    opal_free_list_return_st(&tree->free_list, &node->super);
 }
 
 /* schedules the node for releasing */
-static inline void rp_free (opal_interval_tree_t *tree, opal_interval_tree_node_t *node)
+static inline void rp_free(opal_interval_tree_t *tree, opal_interval_tree_node_t *node)
 {
-    opal_list_append (&tree->gc_list, &node->super.super);
+    opal_list_append(&tree->gc_list, &node->super.super);
 }
 
-static opal_interval_tree_node_t *opal_interval_tree_node_copy (opal_interval_tree_t *tree, opal_interval_tree_node_t *node)
+static opal_interval_tree_node_t *opal_interval_tree_node_copy(opal_interval_tree_t *tree,
+                                                               opal_interval_tree_node_t *node)
 {
-    opal_interval_tree_node_t *copy = (opal_interval_tree_node_t *) opal_free_list_wait_st (&tree->free_list);
+    opal_interval_tree_node_t *copy = (opal_interval_tree_node_t *) opal_free_list_wait_st(
+        &tree->free_list);
     size_t color_offset = offsetof(opal_interval_tree_node_t, color);
-    assert (NULL != copy);
-    memcpy ((unsigned char *) copy + color_offset, (unsigned char *) node + color_offset,
-            sizeof (*node) - color_offset);
+    assert(NULL != copy);
+    memcpy((unsigned char *) copy + color_offset, (unsigned char *) node + color_offset,
+           sizeof(*node) - color_offset);
     return copy;
 }
 
 /* this function deletes a node that is either a left or right leaf (or both) */
-static void opal_interval_tree_delete_leaf (opal_interval_tree_t *tree, opal_interval_tree_node_t *node)
+static void opal_interval_tree_delete_leaf(opal_interval_tree_t *tree,
+                                           opal_interval_tree_node_t *node)
 {
     const opal_interval_tree_node_t *nill = &tree->nill;
     opal_interval_tree_node_t **parent_ptr, *next, *parent = node->parent;
     opal_interval_tree_nodecolor_t color = node->color;
 
-    assert (node->left == nill || node->right == nill);
+    assert(node->left == nill || node->right == nill);
 
     parent_ptr = (parent->right == node) ? &parent->right : &parent->left;
 
     next = (node->right == nill) ? node->left : node->right;
 
     next->parent = node->parent;
-    rp_publish (parent_ptr, next);
+    rp_publish(parent_ptr, next);
 
-    rp_free (tree, node);
+    rp_free(tree, node);
 
     if (OPAL_INTERVAL_TREE_COLOR_BLACK == color) {
         if (OPAL_INTERVAL_TREE_COLOR_RED == next->color) {
             next->color = OPAL_INTERVAL_TREE_COLOR_BLACK;
         } else {
-            opal_interval_tree_delete_fixup (tree, next, parent);
+            opal_interval_tree_delete_fixup(tree, next, parent);
         }
     }
 }
 
-static void opal_interval_tree_delete_interior (opal_interval_tree_t *tree, opal_interval_tree_node_t *node)
+static void opal_interval_tree_delete_interior(opal_interval_tree_t *tree,
+                                               opal_interval_tree_node_t *node)
 {
     opal_interval_tree_node_t **parent_ptr, *next, *next_copy, *parent = node->parent;
     opal_interval_tree_nodecolor_t color = node->color, next_color;
 
     parent_ptr = (parent->right == node) ? &parent->right : &parent->left;
-    next = opal_interval_tree_next (tree, node);
+    next = opal_interval_tree_next(tree, node);
     next_color = next->color;
 
     if (next != node->right) {
         /* case 3 */
-        next_copy = opal_interval_tree_node_copy (tree, next);
+        next_copy = opal_interval_tree_node_copy(tree, next);
         next_copy->color = node->color;
         next_copy->left = node->left;
         next_copy->left->parent = next_copy;
@@ -529,40 +559,40 @@ static void opal_interval_tree_delete_interior (opal_interval_tree_t *tree, opal
         next_copy->right->parent = next_copy;
         next_copy->parent = node->parent;
 
-        rp_publish (parent_ptr, next_copy);
-        rp_free_wait (tree, node);
+        rp_publish(parent_ptr, next_copy);
+        rp_free_wait(tree, node);
 
-        opal_interval_tree_delete_leaf (tree, next);
+        opal_interval_tree_delete_leaf(tree, next);
     } else {
         /* case 2. no copies are needed */
         next->color = color;
         next->left = node->left;
         next->left->parent = next;
         next->parent = node->parent;
-        rp_publish (parent_ptr, next);
-        rp_free (tree, node);
+        rp_publish(parent_ptr, next);
+        rp_free(tree, node);
 
-	/* since we are actually "deleting" the next node the fixup needs to happen on the
-	 * right child of next (by definition next was a left child) */
+        /* since we are actually "deleting" the next node the fixup needs to happen on the
+         * right child of next (by definition next was a left child) */
         if (OPAL_INTERVAL_TREE_COLOR_BLACK == next_color) {
-	    if (OPAL_INTERVAL_TREE_COLOR_RED == next->right->color) {
-		next->right->color = OPAL_INTERVAL_TREE_COLOR_BLACK;
+            if (OPAL_INTERVAL_TREE_COLOR_RED == next->right->color) {
+                next->right->color = OPAL_INTERVAL_TREE_COLOR_BLACK;
             } else {
-		opal_interval_tree_delete_fixup (tree, next->right, next);
-	    }
-	}
+                opal_interval_tree_delete_fixup(tree, next->right, next);
+            }
+        }
     }
 }
 
 /* Delete a node from the tree based on the key */
-int opal_interval_tree_delete (opal_interval_tree_t *tree, uint64_t low, uint64_t high, void *data)
+int opal_interval_tree_delete(opal_interval_tree_t *tree, uint64_t low, uint64_t high, void *data)
 {
     opal_interval_tree_node_t *node;
 
-    opal_interval_tree_write_lock (tree);
-    node = opal_interval_tree_find_node (tree, low, high, data);
+    opal_interval_tree_write_lock(tree);
+    node = opal_interval_tree_find_node(tree, low, high, data);
     if (NULL == node) {
-        opal_interval_tree_write_unlock (tree);
+        opal_interval_tree_write_unlock(tree);
         return OPAL_ERR_NOT_FOUND;
     }
 
@@ -575,23 +605,23 @@ int opal_interval_tree_delete (opal_interval_tree_t *tree, uint64_t low, uint64_
      * 3) p is a interior node
      *    we replace node with next(node)
      */
-    
+
     if ((node->left == &tree->nill) || (node->right == &tree->nill)) {
         /* handle case 1 */
-        opal_interval_tree_delete_leaf (tree, node);
+        opal_interval_tree_delete_leaf(tree, node);
     } else {
         /* handle case 2 and 3 */
-        opal_interval_tree_delete_interior (tree, node);
+        opal_interval_tree_delete_interior(tree, node);
     }
 
     --tree->tree_size;
 
-    opal_interval_tree_write_unlock (tree);
+    opal_interval_tree_write_unlock(tree);
 
     return OPAL_SUCCESS;
 }
 
-int opal_interval_tree_destroy (opal_interval_tree_t *tree)
+int opal_interval_tree_destroy(opal_interval_tree_t *tree)
 {
     /* Recursive inorder traversal for delete */
     inorder_destroy(tree, &tree->root);
@@ -599,9 +629,9 @@ int opal_interval_tree_destroy (opal_interval_tree_t *tree)
     return OPAL_SUCCESS;
 }
 
-
 /* Find the next inorder successor of a node    */
-static opal_interval_tree_node_t *opal_interval_tree_next (opal_interval_tree_t *tree, opal_interval_tree_node_t *node)
+static opal_interval_tree_node_t *opal_interval_tree_next(opal_interval_tree_t *tree,
+                                                          opal_interval_tree_node_t *node)
 {
     opal_interval_tree_node_t *p = node->right;
 
@@ -629,7 +659,8 @@ static opal_interval_tree_node_t *opal_interval_tree_next (opal_interval_tree_t
 /* Insert an element in the normal binary search tree fashion    */
 /* this function goes through the tree and finds the leaf where
  * the node will be inserted   */
-static void opal_interval_tree_insert_node (opal_interval_tree_t *tree, opal_interval_tree_node_t *node)
+static void opal_interval_tree_insert_node(opal_interval_tree_t *tree,
+                                           opal_interval_tree_node_t *node)
 {
     opal_interval_tree_node_t *parent = &tree->root;
     opal_interval_tree_node_t *n = parent->left; /* the real root of the tree */
@@ -646,7 +677,7 @@ static void opal_interval_tree_insert_node (opal_interval_tree_t *tree, opal_int
     while (n != nill) {
         check = opal_interval_tree_compare_node(n, node->low, node->high, node->data);
         /* node already exists */
-        assert (0 != check);
+        assert(0 != check);
 
         if (n->max < node->high) {
             n->max = node->high;
@@ -654,7 +685,7 @@ static void opal_interval_tree_insert_node (opal_interval_tree_t *tree, opal_int
 
         parent = n;
         n = (-1 == check) ? n->left : n->right;
-        assert (nill == n || n->parent == parent);
+        assert(nill == n || n->parent == parent);
     }
 
     /* place it on either the left or the right */
@@ -670,9 +701,9 @@ static void opal_interval_tree_insert_node (opal_interval_tree_t *tree, opal_int
     ++tree->tree_size;
 }
 
-static int inorder_traversal (opal_interval_tree_t *tree, uint64_t low, uint64_t high,
-			      bool partial_ok, opal_interval_tree_action_fn_t action,
-			      opal_interval_tree_node_t * node, void *ctx)
+static int inorder_traversal(opal_interval_tree_t *tree, uint64_t low, uint64_t high,
+                             bool partial_ok, opal_interval_tree_action_fn_t action,
+                             opal_interval_tree_node_t *node, void *ctx)
 {
     int rc;
 
@@ -685,12 +716,12 @@ static int inorder_traversal (opal_interval_tree_t *tree, uint64_t low, uint64_t
         return rc;
     }
 
-    if ((!partial_ok && (node->low <= low && node->high >= high)) ||
-        (partial_ok && ((low >= node->low && low <= node->high) ||
-                        (high >= node->low && high <= node->high) ||
-                        (node->low >= low && node->low <= high) ||
-                        (node->high >= high && node->high <= high)))) {
-        rc = action (node->low, node->high, node->data, ctx);
+    if ((!partial_ok && (node->low <= low && node->high >= high))
+        || (partial_ok
+            && ((low >= node->low && low <= node->high) || (high >= node->low && high <= node->high)
+                || (node->low >= low && node->low <= high)
+                || (node->high >= high && node->high <= high)))) {
+        rc = action(node->low, node->high, node->data, ctx);
         if (OPAL_SUCCESS != rc) {
             return rc;
         }
@@ -701,7 +732,7 @@ static int inorder_traversal (opal_interval_tree_t *tree, uint64_t low, uint64_t
 
 /* Free the nodes in inorder fashion    */
 
-static void inorder_destroy (opal_interval_tree_t *tree, opal_interval_tree_node_t *node)
+static void inorder_destroy(opal_interval_tree_t *tree, opal_interval_tree_node_t *node)
 {
     if (node == &tree->nill) {
         return;
@@ -711,18 +742,18 @@ static void inorder_destroy (opal_interval_tree_t *tree, opal_interval_tree_node
     inorder_destroy(tree, node->right);
 
     if (node->left != &tree->nill) {
-        opal_free_list_return_st (&tree->free_list, &node->left->super);
+        opal_free_list_return_st(&tree->free_list, &node->left->super);
     }
 
     if (node->right != &tree->nill) {
-        opal_free_list_return_st (&tree->free_list, &node->right->super);
+        opal_free_list_return_st(&tree->free_list, &node->right->super);
     }
 }
 
 /* Try to access all the elements of the hashmap conditionally */
 
-int opal_interval_tree_traverse (opal_interval_tree_t *tree, uint64_t low, uint64_t high,
-                                 bool partial_ok, opal_interval_tree_action_fn_t action, void *ctx)
+int opal_interval_tree_traverse(opal_interval_tree_t *tree, uint64_t low, uint64_t high,
+                                bool partial_ok, opal_interval_tree_action_fn_t action, void *ctx)
 {
     opal_interval_tree_token_t token;
     int rc;
@@ -731,16 +762,17 @@ int opal_interval_tree_traverse (opal_interval_tree_t *tree, uint64_t low, uint6
         return OPAL_ERR_BAD_PARAM;
     }
 
-    token = opal_interval_tree_reader_get_token (tree);
-    rc = inorder_traversal (tree, low, high, partial_ok, action, tree->root.left, ctx);
-    opal_interval_tree_reader_return_token (tree, token);
+    token = opal_interval_tree_reader_get_token(tree);
+    rc = inorder_traversal(tree, low, high, partial_ok, action, tree->root.left, ctx);
+    opal_interval_tree_reader_return_token(tree, token);
     return rc;
 }
 
 /* Left rotate the tree    */
 /* basically what we want to do is to make x be the left child
  * of its right child    */
-static opal_interval_tree_node_t *left_rotate (opal_interval_tree_t *tree, opal_interval_tree_node_t *x)
+static opal_interval_tree_node_t *left_rotate(opal_interval_tree_t *tree,
+                                              opal_interval_tree_node_t *x)
 {
     opal_interval_tree_node_t *x_copy = x;
     opal_interval_tree_node_t *y = x->right;
@@ -754,16 +786,16 @@ static opal_interval_tree_node_t *left_rotate (opal_interval_tree_t *tree, opal_
     /* x's parent is now y */
     x_copy->parent = y;
     x_copy->right = y->left;
-    x_copy->max = max (x_copy->high, max (x_copy->left->max, x_copy->left->max));
+    x_copy->max = max(x_copy->high, max(x_copy->left->max, x_copy->left->max));
 
-    rp_publish (&y->left, x_copy);
+    rp_publish(&y->left, x_copy);
 
-    /* normlly we would have to check to see if we are at the root.
+    /* normally we would have to check to see if we are at the root.
      * however, the root sentinal takes care of it for us */
     if (x == parent->left) {
-        rp_publish (&parent->left, y);
+        rp_publish(&parent->left, y);
     } else {
-        rp_publish (&parent->right, y);
+        rp_publish(&parent->right, y);
     }
 
     /* the old parent of x is now y's parent */
@@ -772,11 +804,11 @@ static opal_interval_tree_node_t *left_rotate (opal_interval_tree_t *tree, opal_
     return x_copy;
 }
 
-
 /* Right rotate the tree    */
 /* basically what we want to do is to make x be the right child
  * of its left child */
-static opal_interval_tree_node_t *right_rotate (opal_interval_tree_t *tree, opal_interval_tree_node_t *x)
+static opal_interval_tree_node_t *right_rotate(opal_interval_tree_t *tree,
+                                               opal_interval_tree_node_t *x)
 {
     opal_interval_tree_node_t *x_copy = x;
     opal_interval_tree_node_t *y = x->left;
@@ -790,7 +822,7 @@ static opal_interval_tree_node_t *right_rotate (opal_interval_tree_t *tree, opal
     x_copy->left = y->right;
     x_copy->parent = y;
 
-    rp_publish (&y->right, x_copy);
+    rp_publish(&y->right, x_copy);
 
     /* the maximum value in the subtree rooted at y is now the value it
      * was at x */
@@ -798,9 +830,9 @@ static opal_interval_tree_node_t *right_rotate (opal_interval_tree_t *tree, opal
     y->parent = parent;
 
     if (parent->left == x) {
-        rp_publish (&parent->left, y);
+        rp_publish(&parent->left, y);
     } else {
-        rp_publish (&parent->right, y);
+        rp_publish(&parent->right, y);
     }
 
     return x_copy;
@@ -812,17 +844,18 @@ size_t opal_interval_tree_size(opal_interval_tree_t *tree)
     return tree->tree_size;
 }
 
-static bool opal_interval_tree_verify_node (opal_interval_tree_t *tree, opal_interval_tree_node_t *node, int black_depth,
-                                            int current_black_depth)
+static bool opal_interval_tree_verify_node(opal_interval_tree_t *tree,
+                                           opal_interval_tree_node_t *node, int black_depth,
+                                           int current_black_depth)
 {
     if (node == &tree->nill) {
         return true;
     }
 
-    if (OPAL_INTERVAL_TREE_COLOR_RED == node->color &&
-        (OPAL_INTERVAL_TREE_COLOR_BLACK != node->left->color ||
-         OPAL_INTERVAL_TREE_COLOR_BLACK != node->right->color)) {
-        fprintf (stderr, "Red node has a red child!\n");
+    if (OPAL_INTERVAL_TREE_COLOR_RED == node->color
+        && (OPAL_INTERVAL_TREE_COLOR_BLACK != node->left->color
+            || OPAL_INTERVAL_TREE_COLOR_BLACK != node->right->color)) {
+        fprintf(stderr, "Red node has a red child!\n");
         return false;
     }
 
@@ -832,18 +865,20 @@ static bool opal_interval_tree_verify_node (opal_interval_tree_t *tree, opal_int
 
     if (node->left == &tree->nill && node->right == &tree->nill) {
         if (black_depth != current_black_depth) {
-            fprintf (stderr, "Found leaf with unexpected black depth: %d, expected: %d\n", current_black_depth, black_depth);
+            fprintf(stderr, "Found leaf with unexpected black depth: %d, expected: %d\n",
+                    current_black_depth, black_depth);
             return false;
         }
 
         return true;
     }
 
-    return opal_interval_tree_verify_node (tree, node->left, black_depth, current_black_depth) ||
-        opal_interval_tree_verify_node (tree, node->right, black_depth, current_black_depth);
+    return opal_interval_tree_verify_node(tree, node->left, black_depth, current_black_depth)
+           || opal_interval_tree_verify_node(tree, node->right, black_depth, current_black_depth);
 }
 
-static int opal_interval_tree_black_depth (opal_interval_tree_t *tree, opal_interval_tree_node_t *node, int depth)
+static int opal_interval_tree_black_depth(opal_interval_tree_t *tree,
+                                          opal_interval_tree_node_t *node, int depth)
 {
     if (node == &tree->nill) {
         return depth;
@@ -854,36 +889,37 @@ static int opal_interval_tree_black_depth (opal_interval_tree_t *tree, opal_inte
         depth++;
     }
 
-    return opal_interval_tree_black_depth (tree, node->left, depth);
+    return opal_interval_tree_black_depth(tree, node->left, depth);
 }
 
-bool opal_interval_tree_verify (opal_interval_tree_t *tree)
+bool opal_interval_tree_verify(opal_interval_tree_t *tree)
 {
     int black_depth;
 
     if (OPAL_INTERVAL_TREE_COLOR_BLACK != tree->root.left->color) {
-        fprintf (stderr, "Root node of tree is NOT black!\n");
+        fprintf(stderr, "Root node of tree is NOT black!\n");
         return false;
     }
 
     if (OPAL_INTERVAL_TREE_COLOR_BLACK != tree->nill.color) {
-        fprintf (stderr, "Leaf node color is NOT black!\n");
+        fprintf(stderr, "Leaf node color is NOT black!\n");
         return false;
     }
 
-    black_depth = opal_interval_tree_black_depth (tree, tree->root.left, 0);
+    black_depth = opal_interval_tree_black_depth(tree, tree->root.left, 0);
 
-    return opal_interval_tree_verify_node (tree, tree->root.left, black_depth, 0);
+    return opal_interval_tree_verify_node(tree, tree->root.left, black_depth, 0);
 }
 
-static void opal_interval_tree_dump_node (opal_interval_tree_t *tree, opal_interval_tree_node_t *node, int black_rank, FILE *fh)
+static void opal_interval_tree_dump_node(opal_interval_tree_t *tree,
+                                         opal_interval_tree_node_t *node, int black_rank, FILE *fh)
 {
     const char *color = (node->color == OPAL_INTERVAL_TREE_COLOR_BLACK) ? "black" : "red";
     uintptr_t left = (uintptr_t) node->left, right = (uintptr_t) node->right;
     opal_interval_tree_node_t *nill = &tree->nill;
 
     if (node->color == OPAL_INTERVAL_TREE_COLOR_BLACK) {
-	++black_rank;
+        ++black_rank;
     }
 
     if (nill == node) {
@@ -893,46 +929,48 @@ static void opal_interval_tree_dump_node (opal_interval_tree_t *tree, opal_inter
     /* print out nill nodes if any */
     if ((uintptr_t) nill == left) {
         left = (uintptr_t) node | 0x1;
-        fprintf (fh, "  Node%lx [color=black,label=nill];\n\n", left);
+        fprintf(fh, "  Node%lx [color=black,label=nill];\n\n", left);
     } else {
         left = (uintptr_t) node->left;
     }
 
     if ((uintptr_t) nill == right) {
         right = (uintptr_t) node | 0x2;
-        fprintf (fh, "  Node%lx [color=black,label=nill];\n\n", right);
+        fprintf(fh, "  Node%lx [color=black,label=nill];\n\n", right);
     } else {
         right = (uintptr_t) node->right;
     }
 
     /* print out this node and its edges */
-    fprintf (fh, "  Node%lx [color=%s,shape=box,label=\"[0x%" PRIx64 ",0x%" PRIx64 "]\\nmax=0x%" PRIx64
-             "\\ndata=0x%lx\\nblack rank=%d\"];\n", (uintptr_t) node, color, node->low, node->high, node->max,
-             (uintptr_t) node->data, black_rank);
-    fprintf (fh, "  Node%lx -> Node%lx;\n", (uintptr_t) node, left);
-    fprintf (fh, "  Node%lx -> Node%lx;\n\n", (uintptr_t) node, right);
+    fprintf(fh,
+            "  Node%lx [color=%s,shape=box,label=\"[0x%" PRIx64 ",0x%" PRIx64 "]\\nmax=0x%" PRIx64
+            "\\ndata=0x%lx\\nblack rank=%d\"];\n",
+            (uintptr_t) node, color, node->low, node->high, node->max, (uintptr_t) node->data,
+            black_rank);
+    fprintf(fh, "  Node%lx -> Node%lx;\n", (uintptr_t) node, left);
+    fprintf(fh, "  Node%lx -> Node%lx;\n\n", (uintptr_t) node, right);
     if (node != tree->root.left) {
-        fprintf (fh, "  Node%lx -> Node%lx;\n\n", (uintptr_t) node, (uintptr_t) node->parent);
+        fprintf(fh, "  Node%lx -> Node%lx;\n\n", (uintptr_t) node, (uintptr_t) node->parent);
     }
-    opal_interval_tree_dump_node (tree, node->left, black_rank, fh);
-    opal_interval_tree_dump_node (tree, node->right, black_rank, fh);
+    opal_interval_tree_dump_node(tree, node->left, black_rank, fh);
+    opal_interval_tree_dump_node(tree, node->right, black_rank, fh);
 }
 
-int opal_interval_tree_dump (opal_interval_tree_t *tree, const char *path)
+int opal_interval_tree_dump(opal_interval_tree_t *tree, const char *path)
 {
     FILE *fh;
 
-    fh = fopen (path, "w");
+    fh = fopen(path, "w");
     if (NULL == fh) {
         return OPAL_ERR_BAD_PARAM;
     }
 
-    fprintf (fh, "digraph {\n");
-    fprintf (fh, "  graph [ordering=\"out\"];");
-    opal_interval_tree_dump_node (tree, tree->root.left, 0, fh);
-    fprintf (fh, "}\n");
+    fprintf(fh, "digraph {\n");
+    fprintf(fh, "  graph [ordering=\"out\"];");
+    opal_interval_tree_dump_node(tree, tree->root.left, 0, fh);
+    fprintf(fh, "}\n");
 
-    fclose (fh);
+    fclose(fh);
 
     return OPAL_SUCCESS;
 }
diff --git a/opal/class/opal_interval_tree.h b/opal/class/opal_interval_tree.h
index 68f6fe4cef5..2490141ce94 100644
--- a/opal/class/opal_interval_tree.h
+++ b/opal/class/opal_interval_tree.h
@@ -29,10 +29,10 @@
 #define OPAL_INTERVAL_TREE_H
 
 #include "opal_config.h"
-#include <stdlib.h>
-#include "opal/constants.h"
-#include "opal/class/opal_object.h"
 #include "opal/class/opal_free_list.h"
+#include "opal/class/opal_object.h"
+#include "opal/constants.h"
+#include <stdlib.h>
 
 BEGIN_C_DECLS
 /*
@@ -40,20 +40,22 @@ BEGIN_C_DECLS
  */
 
 /**
-  * red and black enum
-  */
-typedef enum {OPAL_INTERVAL_TREE_COLOR_RED, OPAL_INTERVAL_TREE_COLOR_BLACK} opal_interval_tree_nodecolor_t;
+ * red and black enum
+ */
+typedef enum {
+    OPAL_INTERVAL_TREE_COLOR_RED,
+    OPAL_INTERVAL_TREE_COLOR_BLACK
+} opal_interval_tree_nodecolor_t;
 
 /**
-  * node data structure
-  */
-struct opal_interval_tree_node_t
-{
-    opal_free_list_item_t super;        /**< the parent class */
+ * node data structure
+ */
+struct opal_interval_tree_node_t {
+    opal_free_list_item_t super;              /**< the parent class */
     opal_interval_tree_nodecolor_t color;     /**< the node color */
-    struct opal_interval_tree_node_t *parent;/**< the parent node, can be NULL */
-    struct opal_interval_tree_node_t *left;  /**< the left child - can be nill */
-    struct opal_interval_tree_node_t *right; /**< the right child - can be nill */
+    struct opal_interval_tree_node_t *parent; /**< the parent node, can be NULL */
+    struct opal_interval_tree_node_t *left;   /**< the left child - can be nill */
+    struct opal_interval_tree_node_t *right;  /**< the right child - can be nill */
     /** edit epoch associated with this node */
     uint32_t epoch;
     /** data for this interval */
@@ -71,22 +73,22 @@ typedef struct opal_interval_tree_node_t opal_interval_tree_node_t;
 #define OPAL_INTERVAL_TREE_MAX_READERS 128
 
 /**
-  * the data structure that holds all the needed information about the tree.
-  */
+ * the data structure that holds all the needed information about the tree.
+ */
 struct opal_interval_tree_t {
-    opal_object_t super;           /**< the parent class */
+    opal_object_t super; /**< the parent class */
     /* this root pointer doesn't actually point to the root of the tree.
      * rather, it points to a sentinal node who's left branch is the real
      * root of the tree. This is done to eliminate special cases */
-    opal_interval_tree_node_t root; /**< a pointer to the root of the tree */
-    opal_interval_tree_node_t nill;     /**< the nill sentinal node */
-    opal_free_list_t free_list;   /**< the free list to get the memory from */
-    opal_list_t gc_list;          /**< list of nodes that need to be released */
-    uint32_t epoch;               /**< current update epoch */
-    opal_atomic_size_t tree_size;    /**< the current size of the tree */
-    opal_atomic_int32_t lock;        /**< update lock */
-    opal_atomic_int32_t reader_count;    /**< current highest reader slot to check */
-    volatile uint32_t reader_id;  /**< next reader slot to check */
+    opal_interval_tree_node_t root;   /**< a pointer to the root of the tree */
+    opal_interval_tree_node_t nill;   /**< the nill sentinal node */
+    opal_free_list_t free_list;       /**< the free list to get the memory from */
+    opal_list_t gc_list;              /**< list of nodes that need to be released */
+    uint32_t epoch;                   /**< current update epoch */
+    opal_atomic_size_t tree_size;     /**< the current size of the tree */
+    opal_atomic_int32_t lock;         /**< update lock */
+    opal_atomic_int32_t reader_count; /**< current highest reader slot to check */
+    volatile uint32_t reader_id;      /**< next reader slot to check */
     opal_atomic_uint32_t reader_epochs[OPAL_INTERVAL_TREE_MAX_READERS];
 };
 typedef struct opal_interval_tree_t opal_interval_tree_t;
@@ -98,10 +100,10 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_interval_tree_t);
 
 /* Function pointers for map traversal function */
 /**
-  * this function is used for the opal_interval_tree_traverse function.
-  * it is passed a pointer to the value for each node and, if it returns
-  * a one, the action function is called on that node. Otherwise, the node is ignored.
-  */
+ * this function is used for the opal_interval_tree_traverse function.
+ * it is passed a pointer to the value for each node and, if it returns
+ * a one, the action function is called on that node. Otherwise, the node is ignored.
+ */
 typedef int (*opal_interval_tree_condition_fn_t)(void *);
 /**
  * this function is used for the user to perform any action on the passed
@@ -112,97 +114,100 @@ typedef int (*opal_interval_tree_condition_fn_t)(void *);
 typedef int (*opal_interval_tree_action_fn_t)(uint64_t low, uint64_t high, void *data, void *ctx);
 
 /*
- * Public function protoypes
+ * Public function prototypes
  */
 
 /**
-  * the function creates a new tree
-  *
-  * @param tree a pointer to an allocated area of memory for the main
-  *  tree data structure.
-  * @param comp a pointer to the function to use for comaparing 2 nodes
-  *
-  * @retval OPAL_SUCCESS if it is successful
-  * @retval OPAL_ERR_TEMP_OUT_OF_RESOURCE if unsuccessful
-  */
-OPAL_DECLSPEC int opal_interval_tree_init(opal_interval_tree_t * tree);
-
+ * the function creates a new tree
+ *
+ * @param tree a pointer to an allocated area of memory for the main
+ *  tree data structure.
+ * @param comp a pointer to the function to use for comaparing 2 nodes
+ *
+ * @retval OPAL_SUCCESS if it is successful
+ * @retval OPAL_ERR_TEMP_OUT_OF_RESOURCE if unsuccessful
+ */
+OPAL_DECLSPEC int opal_interval_tree_init(opal_interval_tree_t *tree);
 
 /**
-  * inserts a node into the tree
-  *
-  * @param tree a pointer to the tree data structure
-  * @param key the key for the node
-  * @param value the value for the node
-  *
-  * @retval OPAL_SUCCESS
-  * @retval OPAL_ERR_TEMP_OUT_OF_RESOURCE if unsuccessful
-  */
-OPAL_DECLSPEC int opal_interval_tree_insert(opal_interval_tree_t *tree, void *value, uint64_t low, uint64_t high);
+ * inserts a node into the tree
+ *
+ * @param tree a pointer to the tree data structure
+ * @param key the key for the node
+ * @param value the value for the node
+ *
+ * @retval OPAL_SUCCESS
+ * @retval OPAL_ERR_TEMP_OUT_OF_RESOURCE if unsuccessful
+ */
+OPAL_DECLSPEC int opal_interval_tree_insert(opal_interval_tree_t *tree, void *value, uint64_t low,
+                                            uint64_t high);
 
 /**
-  * finds a value in the tree based on the passed key using passed
-  * compare function
-  *
-  * @param tree a pointer to the tree data structure
-  * @param key a pointer to the key
-  * @param compare function
-  *
-  * @retval pointer to the value if found
-  * @retval NULL if not found
-  */
-OPAL_DECLSPEC void *opal_interval_tree_find_overlapping (opal_interval_tree_t *tree, uint64_t low, uint64_t high);
+ * finds a value in the tree based on the passed key using passed
+ * compare function
+ *
+ * @param tree a pointer to the tree data structure
+ * @param key a pointer to the key
+ * @param compare function
+ *
+ * @retval pointer to the value if found
+ * @retval NULL if not found
+ */
+OPAL_DECLSPEC void *opal_interval_tree_find_overlapping(opal_interval_tree_t *tree, uint64_t low,
+                                                        uint64_t high);
 
 /**
-  * deletes a node based on its interval
-  *
-  * @param tree a pointer to the tree data structure
-  * @param low low value of interval
-  * @param high high value of interval
-  * @param data data to match (NULL for any)
-  *
-  * @retval OPAL_SUCCESS if the node is found and deleted
-  * @retval OPAL_ERR_NOT_FOUND if the node is not found
-  *
-  * This function finds and deletes an interval from the tree that exactly matches
-  * the given range.
-  */
-OPAL_DECLSPEC int opal_interval_tree_delete(opal_interval_tree_t *tree, uint64_t low, uint64_t high, void *data);
+ * deletes a node based on its interval
+ *
+ * @param tree a pointer to the tree data structure
+ * @param low low value of interval
+ * @param high high value of interval
+ * @param data data to match (NULL for any)
+ *
+ * @retval OPAL_SUCCESS if the node is found and deleted
+ * @retval OPAL_ERR_NOT_FOUND if the node is not found
+ *
+ * This function finds and deletes an interval from the tree that exactly matches
+ * the given range.
+ */
+OPAL_DECLSPEC int opal_interval_tree_delete(opal_interval_tree_t *tree, uint64_t low, uint64_t high,
+                                            void *data);
 
 /**
-  * frees all the nodes on the tree
-  *
-  * @param tree a pointer to the tree data structure
-  *
-  * @retval OPAL_SUCCESS
-  */
+ * frees all the nodes on the tree
+ *
+ * @param tree a pointer to the tree data structure
+ *
+ * @retval OPAL_SUCCESS
+ */
 OPAL_DECLSPEC int opal_interval_tree_destroy(opal_interval_tree_t *tree);
 
 /**
-  * traverses the entire tree, performing the cond function on each of the
-  * values and if it returns one it performs the action function on the values
-  *
-  * @param tree a pointer to the tree
-  * @param low low value of interval
-  * @param high high value of interval
-  * @param partial_ok traverse nodes that parially overlap the given range
-  * @param action a pointer to the action function
-  * @param ctx context to pass to action function
-  *
-  * @retval OPAL_SUCCESS
-  * @retval OPAL_ERROR if there is an error
-  */
-OPAL_DECLSPEC int opal_interval_tree_traverse (opal_interval_tree_t *tree, uint64_t low, uint64_t high,
-                                               bool complete, opal_interval_tree_action_fn_t action, void *ctx);
+ * traverses the entire tree, performing the cond function on each of the
+ * values and if it returns one it performs the action function on the values
+ *
+ * @param tree a pointer to the tree
+ * @param low low value of interval
+ * @param high high value of interval
+ * @param partial_ok traverse nodes that partially overlap the given range
+ * @param action a pointer to the action function
+ * @param ctx context to pass to action function
+ *
+ * @retval OPAL_SUCCESS
+ * @retval OPAL_ERROR if there is an error
+ */
+OPAL_DECLSPEC int opal_interval_tree_traverse(opal_interval_tree_t *tree, uint64_t low,
+                                              uint64_t high, bool complete,
+                                              opal_interval_tree_action_fn_t action, void *ctx);
 
 /**
-  * returns the size of the tree
-  *
-  * @param tree a pointer to the tree data structure
-  *
-  * @retval int the nuber of items on the tree
-  */
-OPAL_DECLSPEC size_t opal_interval_tree_size (opal_interval_tree_t *tree);
+ * returns the size of the tree
+ *
+ * @param tree a pointer to the tree data structure
+ *
+ * @retval int the number of items on the tree
+ */
+OPAL_DECLSPEC size_t opal_interval_tree_size(opal_interval_tree_t *tree);
 
 /**
  * Diagnostic function to get the max depth of an interval tree.
@@ -213,7 +218,7 @@ OPAL_DECLSPEC size_t opal_interval_tree_size (opal_interval_tree_t *tree);
  * maximum depth. For a valid interval tree this depth will always be
  * O(log(n)) where n is the number of intervals in the tree.
  */
-OPAL_DECLSPEC size_t opal_interval_tree_depth (opal_interval_tree_t *tree);
+OPAL_DECLSPEC size_t opal_interval_tree_depth(opal_interval_tree_t *tree);
 
 /**
  * Diagnostic function that can be used to verify that an interval tree
@@ -224,7 +229,7 @@ OPAL_DECLSPEC size_t opal_interval_tree_depth (opal_interval_tree_t *tree);
  * @returns true if the tree is a valid interval tree
  * @returns false otherwise
  */
-OPAL_DECLSPEC bool opal_interval_tree_verify (opal_interval_tree_t *tree);
+OPAL_DECLSPEC bool opal_interval_tree_verify(opal_interval_tree_t *tree);
 
 /**
  * Dump a DOT representation of the interval tree
@@ -235,7 +240,7 @@ OPAL_DECLSPEC bool opal_interval_tree_verify (opal_interval_tree_t *tree);
  * This function dumps the tree and includes: color, data value, interval, and sub-tree
  * min and max.
  */
-OPAL_DECLSPEC int opal_interval_tree_dump (opal_interval_tree_t *tree, const char *path);
+OPAL_DECLSPEC int opal_interval_tree_dump(opal_interval_tree_t *tree, const char *path);
 
 END_C_DECLS
 #endif /* OPAL_INTERVAL_TREE_H */
diff --git a/opal/class/opal_lifo.c b/opal/class/opal_lifo.c
index c2fc30c51c6..0af332e3e3a 100644
--- a/opal/class/opal_lifo.c
+++ b/opal/class/opal_lifo.c
@@ -19,10 +19,14 @@
  * $HEADER$
  */
 
+/* needed for nanosleep() */
+#define _POSIX_C_SOURCE 200809L
+
 #include "opal_config.h"
+#include <time.h>
 #include "opal/class/opal_lifo.h"
 
-static void opal_lifo_construct (opal_lifo_t *lifo)
+static void opal_lifo_construct(opal_lifo_t *lifo)
 {
     OBJ_CONSTRUCT(&lifo->opal_lifo_ghost, opal_list_item_t);
     lifo->opal_lifo_ghost.opal_list_next = &lifo->opal_lifo_ghost;
@@ -31,3 +35,16 @@ static void opal_lifo_construct (opal_lifo_t *lifo)
 }
 
 OBJ_CLASS_INSTANCE(opal_lifo_t, opal_object_t, opal_lifo_construct, NULL);
+
+
+void opal_lifo_release_cpu(void)
+{
+    /* NTH: there are many ways to cause the current thread to be suspended. This one
+     * should work well in most cases. Another approach would be to use poll (NULL, 0, ) but
+     * the interval will be forced to be in ms (instead of ns or us). Note that there
+     * is a performance improvement for the lifo test when this call is made on detection
+     * of contention but it may not translate into actually MPI or application performance
+     * improvements. */
+    static struct timespec interval = {.tv_sec = 0, .tv_nsec = 100};
+    nanosleep(&interval, NULL);
+}
diff --git a/opal/class/opal_lifo.h b/opal/class/opal_lifo.h
index 4c34fbe5f5c..72cc531e3b8 100644
--- a/opal/class/opal_lifo.h
+++ b/opal/class/opal_lifo.h
@@ -13,7 +13,7 @@
  * Copyright (c) 2007      Voltaire All rights reserved.
  * Copyright (c) 2010      IBM Corporation.  All rights reserved.
  * Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights
- *                         reseved.
+ *                         reserved.
  * Copyright (c) 2016-2018 Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2021      Triad National Security, LLC. All rights reserved.
@@ -29,17 +29,16 @@
 #define OPAL_LIFO_H_HAS_BEEN_INCLUDED
 
 #include "opal_config.h"
-#include <time.h>
 #include "opal/class/opal_list.h"
 
-#include "opal/sys/atomic.h"
 #include "opal/mca/threads/mutex.h"
+#include "opal/sys/atomic.h"
 
 BEGIN_C_DECLS
 
 /* NTH: temporarily suppress warnings about this not being defined */
 #if !defined(OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128)
-#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 0
+#    define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 0
 #endif
 
 /**
@@ -61,28 +60,29 @@ union opal_counted_pointer_t {
 };
 typedef union opal_counted_pointer_t opal_counted_pointer_t;
 
-
 #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 && !OPAL_HAVE_ATOMIC_LLSC_PTR
 
 /* Add one element to the FIFO. We will return the last head of the list
  * to allow the upper level to detect if this element is the first one in the
  * list (if the list was empty before this operation).
  */
-static inline bool opal_update_counted_pointer (volatile opal_counted_pointer_t * volatile addr, opal_counted_pointer_t *old,
-                                                opal_list_item_t *item)
+static inline bool opal_update_counted_pointer(volatile opal_counted_pointer_t *volatile addr,
+                                               opal_counted_pointer_t *old, opal_list_item_t *item)
 {
     opal_counted_pointer_t new_p;
     new_p.data.item = (intptr_t) item;
     new_p.data.counter = old->data.counter + 1;
-    return opal_atomic_compare_exchange_strong_128 (&addr->atomic_value, &old->value, new_p.value);
+    return opal_atomic_compare_exchange_strong_128(&addr->atomic_value, &old->value, new_p.value);
 }
 
-__opal_attribute_always_inline__
-static inline void opal_read_counted_pointer (volatile opal_counted_pointer_t * volatile addr, opal_counted_pointer_t *value)
+__opal_attribute_always_inline__ static inline void
+opal_read_counted_pointer(volatile opal_counted_pointer_t *volatile addr,
+                          opal_counted_pointer_t *value)
 {
-    /* most platforms do not read the value atomically so make sure we read the counted pointer in a specific order */
+    /* most platforms do not read the value atomically so make sure we read the counted pointer in a
+     * specific order */
     value->data.counter = addr->data.counter;
-    opal_atomic_rmb ();
+    opal_atomic_rmb();
     value->data.item = addr->data.item;
 }
 
@@ -91,18 +91,7 @@ static inline void opal_read_counted_pointer (volatile opal_counted_pointer_t *
 /**
  * @brief Helper function for lifo/fifo to sleep this thread if excessive contention is detected
  */
-static inline void _opal_lifo_release_cpu (void)
-{
-    /* NTH: there are many ways to cause the current thread to be suspended. This one
-     * should work well in most cases. Another approach would be to use poll (NULL, 0, ) but
-     * the interval will be forced to be in ms (instead of ns or us). Note that there
-     * is a performance improvement for the lifo test when this call is made on detection
-     * of contention but it may not translate into actually MPI or application performance
-     * improvements. */
-    static struct timespec interval = { .tv_sec = 0, .tv_nsec = 100 };
-    nanosleep (&interval, NULL);
-}
-
+void opal_lifo_release_cpu(void);
 
 /* Atomic Last In First Out lists. If we are in a multi-threaded environment then the
  * atomicity is insured via the compare-and-swap operation, if not we simply do a read
@@ -127,34 +116,32 @@ typedef struct opal_lifo_t opal_lifo_t;
 
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_lifo_t);
 
-
 /* The ghost pointer will never change. The head will change via an atomic
  * compare-and-swap. On most architectures the reading of a pointer is an
  * atomic operation so we don't have to protect it.
  */
-static inline bool opal_lifo_is_empty( opal_lifo_t* lifo )
+static inline bool opal_lifo_is_empty(opal_lifo_t *lifo)
 {
     return (opal_list_item_t *) lifo->opal_lifo_head.data.item == &lifo->opal_lifo_ghost;
 }
 
-
 #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 && !OPAL_HAVE_ATOMIC_LLSC_PTR
 
 /* Add one element to the LIFO. We will return the last head of the list
  * to allow the upper level to detect if this element is the first one in the
  * list (if the list was empty before this operation).
  */
-static inline opal_list_item_t *opal_lifo_push_atomic (opal_lifo_t *lifo,
-                                                       opal_list_item_t *item)
+static inline opal_list_item_t *opal_lifo_push_atomic(opal_lifo_t *lifo, opal_list_item_t *item)
 {
     opal_list_item_t *next = (opal_list_item_t *) lifo->opal_lifo_head.data.item;
 
     do {
         item->opal_list_next = next;
-        opal_atomic_wmb ();
+        opal_atomic_wmb();
 
         /* to protect against ABA issues it is sufficient to only update the counter in pop */
-        if (opal_atomic_compare_exchange_strong_ptr (&lifo->opal_lifo_head.data.item, (intptr_t *) &next, (intptr_t) item)) {
+        if (opal_atomic_compare_exchange_strong_ptr(&lifo->opal_lifo_head.data.item,
+                                                    (intptr_t *) &next, (intptr_t) item)) {
             return next;
         }
         /* DO some kind of pause to release the bus */
@@ -164,12 +151,12 @@ static inline opal_list_item_t *opal_lifo_push_atomic (opal_lifo_t *lifo,
 /* Retrieve one element from the LIFO. If we reach the ghost element then the LIFO
  * is empty so we return NULL.
  */
-static inline opal_list_item_t *opal_lifo_pop_atomic (opal_lifo_t* lifo)
+static inline opal_list_item_t *opal_lifo_pop_atomic(opal_lifo_t *lifo)
 {
     opal_counted_pointer_t old_head;
     opal_list_item_t *item;
 
-    opal_read_counted_pointer (&lifo->opal_lifo_head, &old_head);
+    opal_read_counted_pointer(&lifo->opal_lifo_head, &old_head);
 
     do {
         item = (opal_list_item_t *) old_head.data.item;
@@ -177,9 +164,9 @@ static inline opal_list_item_t *opal_lifo_pop_atomic (opal_lifo_t* lifo)
             return NULL;
         }
 
-        if (opal_update_counted_pointer (&lifo->opal_lifo_head, &old_head,
-                                         (opal_list_item_t *) item->opal_list_next)) {
-            opal_atomic_wmb ();
+        if (opal_update_counted_pointer(&lifo->opal_lifo_head, &old_head,
+                                        (opal_list_item_t *) item->opal_list_next)) {
+            opal_atomic_wmb();
             item->opal_list_next = NULL;
             return item;
         }
@@ -192,8 +179,7 @@ static inline opal_list_item_t *opal_lifo_pop_atomic (opal_lifo_t* lifo)
  * to allow the upper level to detect if this element is the first one in the
  * list (if the list was empty before this operation).
  */
-static inline opal_list_item_t *opal_lifo_push_atomic (opal_lifo_t *lifo,
-                                                       opal_list_item_t *item)
+static inline opal_list_item_t *opal_lifo_push_atomic(opal_lifo_t *lifo, opal_list_item_t *item)
 {
     opal_list_item_t *next = (opal_list_item_t *) lifo->opal_lifo_head.data.item;
 
@@ -203,8 +189,9 @@ static inline opal_list_item_t *opal_lifo_push_atomic (opal_lifo_t *lifo,
     do {
         item->opal_list_next = next;
         opal_atomic_wmb();
-        if (opal_atomic_compare_exchange_strong_ptr (&lifo->opal_lifo_head.data.item, (intptr_t *) &next, (intptr_t) item)) {
-            opal_atomic_wmb ();
+        if (opal_atomic_compare_exchange_strong_ptr(&lifo->opal_lifo_head.data.item,
+                                                    (intptr_t *) &next, (intptr_t) item)) {
+            opal_atomic_wmb();
             /* now safe to pop this item */
             item->item_free = 0;
             return next;
@@ -213,21 +200,21 @@ static inline opal_list_item_t *opal_lifo_push_atomic (opal_lifo_t *lifo,
     } while (1);
 }
 
-#if OPAL_HAVE_ATOMIC_LLSC_PTR
+#    if OPAL_HAVE_ATOMIC_LLSC_PTR
 
 /* Retrieve one element from the LIFO. If we reach the ghost element then the LIFO
  * is empty so we return NULL.
  */
-static inline opal_list_item_t *opal_lifo_pop_atomic (opal_lifo_t* lifo)
+static inline opal_list_item_t *opal_lifo_pop_atomic(opal_lifo_t *lifo)
 {
     register opal_list_item_t *item, *next;
     int attempt = 0, ret;
 
     do {
         if (++attempt == 5) {
-            /* deliberatly suspend this thread to allow other threads to run. this should
+            /* deliberately suspend this thread to allow other threads to run. this should
              * only occur during periods of contention on the lifo. */
-            _opal_lifo_release_cpu ();
+            opal_lifo_release_cpu();
             attempt = 0;
         }
 
@@ -240,33 +227,34 @@ static inline opal_list_item_t *opal_lifo_pop_atomic (opal_lifo_t* lifo)
         opal_atomic_sc_ptr(&lifo->opal_lifo_head.data.item, next, ret);
     } while (!ret);
 
-    opal_atomic_wmb ();
+    opal_atomic_wmb();
 
     item->opal_list_next = NULL;
     return item;
 }
 
-#else
+#    else
 
 /* Retrieve one element from the LIFO. If we reach the ghost element then the LIFO
  * is empty so we return NULL.
  */
-static inline opal_list_item_t *opal_lifo_pop_atomic (opal_lifo_t* lifo)
+static inline opal_list_item_t *opal_lifo_pop_atomic(opal_lifo_t *lifo)
 {
     opal_list_item_t *item, *head, *ghost = &lifo->opal_lifo_ghost;
 
-    while ((item=(opal_list_item_t *)lifo->opal_lifo_head.data.item) != ghost) {
+    while ((item = (opal_list_item_t *) lifo->opal_lifo_head.data.item) != ghost) {
         /* ensure it is safe to pop the head */
         if (opal_atomic_swap_32((opal_atomic_int32_t *) &item->item_free, 1)) {
             continue;
         }
 
-        opal_atomic_wmb ();
+        opal_atomic_wmb();
 
         head = item;
         /* try to swap out the head pointer */
-        if (opal_atomic_compare_exchange_strong_ptr (&lifo->opal_lifo_head.data.item, (intptr_t *) &head,
-                                                     (intptr_t) item->opal_list_next)) {
+        if (opal_atomic_compare_exchange_strong_ptr(&lifo->opal_lifo_head.data.item,
+                                                    (intptr_t *) &head,
+                                                    (intptr_t) item->opal_list_next)) {
             break;
         }
 
@@ -281,19 +269,18 @@ static inline opal_list_item_t *opal_lifo_pop_atomic (opal_lifo_t* lifo)
         return NULL;
     }
 
-    opal_atomic_wmb ();
+    opal_atomic_wmb();
 
     item->opal_list_next = NULL;
     return item;
 }
 
-#endif /* OPAL_HAVE_ATOMIC_LLSC_PTR */
+#    endif /* OPAL_HAVE_ATOMIC_LLSC_PTR */
 
 #endif
 
 /* single-threaded versions of the lifo functions */
-static inline opal_list_item_t *opal_lifo_push_st (opal_lifo_t *lifo,
-                                                   opal_list_item_t *item)
+static inline opal_list_item_t *opal_lifo_push_st(opal_lifo_t *lifo, opal_list_item_t *item)
 {
     item->opal_list_next = (opal_list_item_t *) lifo->opal_lifo_head.data.item;
     item->item_free = 0;
@@ -301,7 +288,7 @@ static inline opal_list_item_t *opal_lifo_push_st (opal_lifo_t *lifo,
     return (opal_list_item_t *) item->opal_list_next;
 }
 
-static inline opal_list_item_t *opal_lifo_pop_st (opal_lifo_t *lifo)
+static inline opal_list_item_t *opal_lifo_pop_st(opal_lifo_t *lifo)
 {
     opal_list_item_t *item;
     item = (opal_list_item_t *) lifo->opal_lifo_head.data.item;
@@ -316,25 +303,24 @@ static inline opal_list_item_t *opal_lifo_pop_st (opal_lifo_t *lifo)
 }
 
 /* conditional versions of lifo functions. use atomics if opal_using_threads is set */
-static inline opal_list_item_t *opal_lifo_push (opal_lifo_t *lifo,
-                                                opal_list_item_t *item)
+static inline opal_list_item_t *opal_lifo_push(opal_lifo_t *lifo, opal_list_item_t *item)
 {
-    if (opal_using_threads ()) {
-        return opal_lifo_push_atomic (lifo, item);
+    if (opal_using_threads()) {
+        return opal_lifo_push_atomic(lifo, item);
     }
 
-    return opal_lifo_push_st (lifo, item);
+    return opal_lifo_push_st(lifo, item);
 }
 
-static inline opal_list_item_t *opal_lifo_pop (opal_lifo_t *lifo)
+static inline opal_list_item_t *opal_lifo_pop(opal_lifo_t *lifo)
 {
-    if (opal_using_threads ()) {
-        return opal_lifo_pop_atomic (lifo);
+    if (opal_using_threads()) {
+        return opal_lifo_pop_atomic(lifo);
     }
 
-    return opal_lifo_pop_st (lifo);
+    return opal_lifo_pop_st(lifo);
 }
 
 END_C_DECLS
 
-#endif  /* OPAL_LIFO_H_HAS_BEEN_INCLUDED */
+#endif /* OPAL_LIFO_H_HAS_BEEN_INCLUDED */
diff --git a/opal/class/opal_list.c b/opal/class/opal_list.c
index f61580eeabd..e28b445b953 100644
--- a/opal/class/opal_list.c
+++ b/opal/class/opal_list.c
@@ -26,26 +26,16 @@
  *  List classes
  */
 
-static void opal_list_item_construct(opal_list_item_t*);
-static void opal_list_item_destruct(opal_list_item_t*);
+static void opal_list_item_construct(opal_list_item_t *);
+static void opal_list_item_destruct(opal_list_item_t *);
 
-OBJ_CLASS_INSTANCE(
-    opal_list_item_t,
-    opal_object_t,
-    opal_list_item_construct,
-    opal_list_item_destruct
-);
+OBJ_CLASS_INSTANCE(opal_list_item_t, opal_object_t, opal_list_item_construct,
+                   opal_list_item_destruct);
 
-static void opal_list_construct(opal_list_t*);
-static void opal_list_destruct(opal_list_t*);
-
-OBJ_CLASS_INSTANCE(
-    opal_list_t,
-    opal_object_t,
-    opal_list_construct,
-    opal_list_destruct
-);
+static void opal_list_construct(opal_list_t *);
+static void opal_list_destruct(opal_list_t *);
 
+OBJ_CLASS_INSTANCE(opal_list_t, opal_object_t, opal_list_construct, opal_list_destruct);
 
 /*
  *
@@ -66,12 +56,11 @@ static void opal_list_item_construct(opal_list_item_t *item)
 static void opal_list_item_destruct(opal_list_item_t *item)
 {
 #if OPAL_ENABLE_DEBUG
-    assert( 0 == item->opal_list_item_refcount );
-    assert( NULL == item->opal_list_item_belong_to );
-#endif  /* OPAL_ENABLE_DEBUG */
+    assert(0 == item->opal_list_item_refcount);
+    assert(NULL == item->opal_list_item_belong_to);
+#endif /* OPAL_ENABLE_DEBUG */
 }
 
-
 /*
  *
  *      opal_list_list_t interface
@@ -85,8 +74,8 @@ static void opal_list_construct(opal_list_t *list)
        should never be removed from this list, added to another list,
        etc.  So set them to sentinel values. */
 
-    OBJ_CONSTRUCT( &(list->opal_list_sentinel), opal_list_item_t );
-    list->opal_list_sentinel.opal_list_item_refcount  = 1;
+    OBJ_CONSTRUCT(&(list->opal_list_sentinel), opal_list_item_t);
+    list->opal_list_sentinel.opal_list_item_refcount = 1;
     list->opal_list_sentinel.opal_list_item_belong_to = list;
 #endif
 
@@ -95,7 +84,6 @@ static void opal_list_construct(opal_list_t *list)
     list->opal_list_length = 0;
 }
 
-
 /*
  * Reset all the pointers to be NULL -- do not actually destroy
  * anything.
@@ -105,60 +93,8 @@ static void opal_list_destruct(opal_list_t *list)
     opal_list_construct(list);
 }
 
-
-/*
- * Insert an item at a specific place in a list
- */
-bool opal_list_insert(opal_list_t *list, opal_list_item_t *item, long long idx)
-{
-    /* Adds item to list at index and retains item. */
-    int     i;
-    volatile opal_list_item_t *ptr, *next;
-
-    if ( idx >= (long long)list->opal_list_length ) {
-        return false;
-    }
-
-    if ( 0 == idx )
-    {
-        opal_list_prepend(list, item);
-    } else {
-#if OPAL_ENABLE_DEBUG
-        /* Spot check: ensure that this item is previously on no
-           lists */
-
-        assert(0 == item->opal_list_item_refcount);
-#endif
-        /* pointer to element 0 */
-        ptr = list->opal_list_sentinel.opal_list_next;
-        for ( i = 0; i < idx-1; i++ )
-            ptr = ptr->opal_list_next;
-
-        next = ptr->opal_list_next;
-        item->opal_list_next = next;
-        item->opal_list_prev = ptr;
-        next->opal_list_prev = item;
-        ptr->opal_list_next = item;
-
-#if OPAL_ENABLE_DEBUG
-        /* Spot check: ensure this item is only on the list that we
-           just inserted it into */
-
-        opal_atomic_add ( &(item->opal_list_item_refcount), 1 );
-        assert(1 == item->opal_list_item_refcount);
-        item->opal_list_item_belong_to = list;
-#endif
-    }
-
-    list->opal_list_length++;
-    return true;
-}
-
-
-static
-void
-opal_list_transfer(opal_list_item_t *pos, opal_list_item_t *begin,
-                   opal_list_item_t *end)
+static void opal_list_transfer(opal_list_item_t *pos, opal_list_item_t *begin,
+                               opal_list_item_t *end)
 {
     volatile opal_list_item_t *tmp;
 
@@ -175,25 +111,21 @@ opal_list_transfer(opal_list_item_t *pos, opal_list_item_t *begin,
         begin->opal_list_prev = tmp;
 #if OPAL_ENABLE_DEBUG
         {
-            volatile opal_list_item_t* item = begin;
-            while( pos != item ) {
+            volatile opal_list_item_t *item = begin;
+            while (pos != item) {
                 item->opal_list_item_belong_to = pos->opal_list_item_belong_to;
                 item = item->opal_list_next;
                 assert(NULL != item);
             }
         }
-#endif  /* OPAL_ENABLE_DEBUG */
+#endif /* OPAL_ENABLE_DEBUG */
     }
 }
 
-
-void
-opal_list_join(opal_list_t *thislist, opal_list_item_t *pos,
-               opal_list_t *xlist)
+void opal_list_join(opal_list_t *thislist, opal_list_item_t *pos, opal_list_t *xlist)
 {
     if (0 != opal_list_get_size(xlist)) {
-        opal_list_transfer(pos, opal_list_get_first(xlist),
-                           opal_list_get_end(xlist));
+        opal_list_transfer(pos, opal_list_get_first(xlist), opal_list_get_end(xlist));
 
         /* fix the sizes */
         thislist->opal_list_length += xlist->opal_list_length;
@@ -201,11 +133,8 @@ opal_list_join(opal_list_t *thislist, opal_list_item_t *pos,
     }
 }
 
-
-void
-opal_list_splice(opal_list_t *thislist, opal_list_item_t *pos,
-                 opal_list_t *xlist, opal_list_item_t *first,
-                 opal_list_item_t *last)
+void opal_list_splice(opal_list_t *thislist, opal_list_item_t *pos, opal_list_t *xlist,
+                      opal_list_item_t *first, opal_list_item_t *last)
 {
     size_t change = 0;
     opal_list_item_t *tmp;
@@ -215,7 +144,7 @@ opal_list_splice(opal_list_t *thislist, opal_list_item_t *pos,
          * first, since last might be end and then we wouldn't be able
          * to run the loop)
          */
-        for (tmp = first ; tmp != last ; tmp = opal_list_get_next(tmp)) {
+        for (tmp = first; tmp != last; tmp = opal_list_get_next(tmp)) {
             change++;
         }
 
@@ -227,31 +156,28 @@ opal_list_splice(opal_list_t *thislist, opal_list_item_t *pos,
     }
 }
 
-
-int opal_list_sort(opal_list_t* list, opal_list_item_compare_fn_t compare)
+int opal_list_sort(opal_list_t *list, opal_list_item_compare_fn_t compare)
 {
-    opal_list_item_t* item;
-    opal_list_item_t** items;
-    size_t i, index=0;
+    opal_list_item_t *item;
+    opal_list_item_t **items;
+    size_t i, index = 0;
 
     if (0 == list->opal_list_length) {
         return OPAL_SUCCESS;
     }
-    items = (opal_list_item_t**)malloc(sizeof(opal_list_item_t*) *
-                                       list->opal_list_length);
+    items = (opal_list_item_t **) malloc(sizeof(opal_list_item_t *) * list->opal_list_length);
 
     if (NULL == items) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
-    while(NULL != (item = opal_list_remove_first(list))) {
+    while (NULL != (item = opal_list_remove_first(list))) {
         items[index++] = item;
     }
 
-    qsort(items, index, sizeof(opal_list_item_t*),
-          (int(*)(const void*,const void*))compare);
-    for (i=0; i<index; i++) {
-        opal_list_append(list,items[i]);
+    qsort(items, index, sizeof(opal_list_item_t *), (int (*)(const void *, const void *)) compare);
+    for (i = 0; i < index; i++) {
+        opal_list_append(list, items[i]);
     }
     free(items);
     return OPAL_SUCCESS;
diff --git a/opal/class/opal_list.h b/opal/class/opal_list.h
index 28390d06a84..a829b6e328a 100644
--- a/opal/class/opal_list.h
+++ b/opal/class/opal_list.h
@@ -68,14 +68,14 @@
 #define OPAL_LIST_H
 
 #include "opal_config.h"
+#include "opal/class/opal_object.h"
 #include <stdio.h>
 #include <stdlib.h>
-#include "opal/class/opal_object.h"
 
 #if OPAL_ENABLE_DEBUG
 /* Need atomics for debugging (reference counting) */
-#include "opal/sys/atomic.h"
-#include "opal/mca/threads/mutex.h"
+#    include "opal/mca/threads/mutex.h"
+#    include "opal/sys/atomic.h"
 #endif
 
 BEGIN_C_DECLS
@@ -93,19 +93,17 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_list_t);
  */
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_list_item_t);
 
-
 /**
  * \internal
  *
  * Struct of an opal_list_item_t
  */
-struct opal_list_item_t
-{
+struct opal_list_item_t {
     opal_object_t super;
     /**< Generic parent class for all Open MPI objects */
-    volatile struct opal_list_item_t * volatile opal_list_next;
+    volatile struct opal_list_item_t *volatile opal_list_next;
     /**< Pointer to next list item */
-    volatile struct opal_list_item_t * volatile opal_list_prev;
+    volatile struct opal_list_item_t *volatile opal_list_prev;
     /**< Pointer to previous list item */
     int32_t item_free;
 
@@ -113,7 +111,7 @@ struct opal_list_item_t
     /** Atomic reference count for debugging */
     opal_atomic_int32_t opal_list_item_refcount;
     /** The list this item belong to */
-    volatile struct opal_list_t* opal_list_item_belong_to;
+    volatile struct opal_list_t *opal_list_item_belong_to;
 #endif
 };
 /**
@@ -121,7 +119,6 @@ struct opal_list_item_t
  */
 typedef struct opal_list_item_t opal_list_item_t;
 
-
 /**
  * Get the next item in a list.
  *
@@ -130,7 +127,7 @@ typedef struct opal_list_item_t opal_list_item_t;
  * @returns The next item in the list
  */
 #define opal_list_get_next(item) \
-    ((item) ? ((opal_list_item_t*) ((opal_list_item_t*)(item))->opal_list_next) : NULL)
+    ((item) ? ((opal_list_item_t *) ((opal_list_item_t *) (item))->opal_list_next) : NULL)
 
 /**
  * Get the next item in a list.
@@ -140,21 +137,19 @@ typedef struct opal_list_item_t opal_list_item_t;
  * @returns The next item in the list
  */
 #define opal_list_get_prev(item) \
-    ((item) ? ((opal_list_item_t*) ((opal_list_item_t*)(item))->opal_list_prev) : NULL)
-
+    ((item) ? ((opal_list_item_t *) ((opal_list_item_t *) (item))->opal_list_prev) : NULL)
 
 /**
  * \internal
  *
  * Struct of an opal_list_t
  */
-struct opal_list_t
-{
-    opal_object_t       super;
+struct opal_list_t {
+    opal_object_t super;
     /**< Generic parent class for all Open MPI objects */
-    opal_list_item_t    opal_list_sentinel;
+    opal_list_item_t opal_list_sentinel;
     /**< Head and tail item of the list */
-    volatile size_t     opal_list_length;
+    volatile size_t opal_list_length;
     /**< Quick reference to the number of items in the list */
 };
 /**
@@ -170,28 +165,27 @@ typedef struct opal_list_t opal_list_t;
  *
  * @param[in] list List to destruct or release
  */
-#define OPAL_LIST_DESTRUCT(list)                                  \
-    do {                                                          \
-        opal_list_item_t *it;                                     \
-        if (1 == ((opal_object_t*)(list))->obj_reference_count) { \
-          while (NULL != (it = opal_list_remove_first(list))) {   \
-              OBJ_RELEASE(it);                                    \
-          }                                                       \
-        }                                                         \
-        OBJ_DESTRUCT(list);                                       \
-    } while(0);
-
-#define OPAL_LIST_RELEASE(list)                                   \
-    do {                                                          \
-        opal_list_item_t *it;                                     \
-        if (1 == ((opal_object_t*)(list))->obj_reference_count) { \
-          while (NULL != (it = opal_list_remove_first(list))) {   \
-              OBJ_RELEASE(it);                                    \
-          }                                                       \
-        }                                                         \
-        OBJ_RELEASE(list);                                        \
-    } while(0);
-
+#define OPAL_LIST_DESTRUCT(list)                                    \
+    do {                                                            \
+        opal_list_item_t *it;                                       \
+        if (1 == ((opal_object_t *) (list))->obj_reference_count) { \
+            while (NULL != (it = opal_list_remove_first(list))) {   \
+                OBJ_RELEASE(it);                                    \
+            }                                                       \
+        }                                                           \
+        OBJ_DESTRUCT(list);                                         \
+    } while (0);
+
+#define OPAL_LIST_RELEASE(list)                                     \
+    do {                                                            \
+        opal_list_item_t *it;                                       \
+        if (1 == ((opal_object_t *) (list))->obj_reference_count) { \
+            while (NULL != (it = opal_list_remove_first(list))) {   \
+                OBJ_RELEASE(it);                                    \
+            }                                                       \
+        }                                                           \
+        OBJ_RELEASE(list);                                          \
+    } while (0);
 
 /**
  * Loop over a list.
@@ -210,15 +204,15 @@ typedef struct opal_list_t opal_list_t;
  *    do something(foo);
  * }
  */
-#define OPAL_LIST_FOREACH(item, list, type)                             \
-  for (item = (type *) (list)->opal_list_sentinel.opal_list_next ;      \
-       item != (type *) &(list)->opal_list_sentinel ;                   \
-       item = (type *) ((opal_list_item_t *) (item))->opal_list_next)
+#define OPAL_LIST_FOREACH(item, list, type)                         \
+    for (item = (type *) (list)->opal_list_sentinel.opal_list_next; \
+         item != (type *) &(list)->opal_list_sentinel;              \
+         item = (type *) ((opal_list_item_t *) (item))->opal_list_next)
 
-#define OPAL_LIST_FOREACH_DECL(item, list, type)			\
-  for (type *item = (type *) (list)->opal_list_sentinel.opal_list_next ; \
-       item != (type *) &(list)->opal_list_sentinel ;                   \
-       item = (type *) ((opal_list_item_t *) (item))->opal_list_next)
+#define OPAL_LIST_FOREACH_DECL(item, list, type)                          \
+    for (type *item = (type *) (list)->opal_list_sentinel.opal_list_next; \
+         item != (type *) &(list)->opal_list_sentinel;                    \
+         item = (type *) ((opal_list_item_t *) (item))->opal_list_next)
 
 /**
  * Loop over a list in reverse.
@@ -237,10 +231,10 @@ typedef struct opal_list_t opal_list_t;
  *    do something;
  * }
  */
-#define OPAL_LIST_FOREACH_REV(item, list, type)                         \
-  for (item = (type *) (list)->opal_list_sentinel.opal_list_prev ;      \
-       item != (type *) &(list)->opal_list_sentinel ;                   \
-       item = (type *) ((opal_list_item_t *) (item))->opal_list_prev)
+#define OPAL_LIST_FOREACH_REV(item, list, type)                     \
+    for (item = (type *) (list)->opal_list_sentinel.opal_list_prev; \
+         item != (type *) &(list)->opal_list_sentinel;              \
+         item = (type *) ((opal_list_item_t *) (item))->opal_list_prev)
 
 /**
  * Loop over a list in a *safe* way
@@ -261,11 +255,11 @@ typedef struct opal_list_t opal_list_t;
  *    opal_list_remove_item (list, (opal_list_item_t *) foo);
  * }
  */
-#define OPAL_LIST_FOREACH_SAFE(item, next, list, type)                  \
-  for (item = (type *) (list)->opal_list_sentinel.opal_list_next,       \
-         next = (type *) ((opal_list_item_t *) (item))->opal_list_next ;\
-       item != (type *) &(list)->opal_list_sentinel ;                   \
-       item = next, next = (type *) ((opal_list_item_t *) (item))->opal_list_next)
+#define OPAL_LIST_FOREACH_SAFE(item, next, list, type)                 \
+    for (item = (type *) (list)->opal_list_sentinel.opal_list_next,    \
+        next = (type *) ((opal_list_item_t *) (item))->opal_list_next; \
+         item != (type *) &(list)->opal_list_sentinel;                 \
+         item = next, next = (type *) ((opal_list_item_t *) (item))->opal_list_next)
 
 /**
  * Loop over a list in a *safe* way
@@ -286,12 +280,11 @@ typedef struct opal_list_t opal_list_t;
  *    opal_list_remove_item (list, (opal_list_item_t *) foo);
  * }
  */
-#define OPAL_LIST_FOREACH_SAFE_REV(item, prev, list, type)              \
-  for (item = (type *) (list)->opal_list_sentinel.opal_list_prev,       \
-         prev = (type *) ((opal_list_item_t *) (item))->opal_list_prev ;\
-       item != (type *) &(list)->opal_list_sentinel ;                   \
-       item = prev, prev = (type *) ((opal_list_item_t *) (item))->opal_list_prev)
-
+#define OPAL_LIST_FOREACH_SAFE_REV(item, prev, list, type)             \
+    for (item = (type *) (list)->opal_list_sentinel.opal_list_prev,    \
+        prev = (type *) ((opal_list_item_t *) (item))->opal_list_prev; \
+         item != (type *) &(list)->opal_list_sentinel;                 \
+         item = prev, prev = (type *) ((opal_list_item_t *) (item))->opal_list_prev)
 
 /**
  * Check for empty list
@@ -305,13 +298,11 @@ typedef struct opal_list_t opal_list_t;
  * This is an inlined function in compilers that support inlining,
  * so it's usually a cheap operation.
  */
-static inline bool opal_list_is_empty(opal_list_t* list)
+static inline bool opal_list_is_empty(opal_list_t *list)
 {
-    return (list->opal_list_sentinel.opal_list_next ==
-        &(list->opal_list_sentinel) ? true : false);
+    return (list->opal_list_sentinel.opal_list_next == &(list->opal_list_sentinel) ? true : false);
 }
 
-
 /**
  * Return the first item on the list (does not remove it).
  *
@@ -326,14 +317,14 @@ static inline bool opal_list_is_empty(opal_list_t* list)
  * This is an inlined function in compilers that support inlining, so
  * it's usually a cheap operation.
  */
-static inline opal_list_item_t* opal_list_get_first(opal_list_t* list)
+static inline opal_list_item_t *opal_list_get_first(opal_list_t *list)
 {
-    opal_list_item_t* item = (opal_list_item_t*)list->opal_list_sentinel.opal_list_next;
+    opal_list_item_t *item = (opal_list_item_t *) list->opal_list_sentinel.opal_list_next;
 #if OPAL_ENABLE_DEBUG
     /* Spot check: ensure that the first item is only on one list */
 
     assert(1 == item->opal_list_item_refcount);
-    assert( list == item->opal_list_item_belong_to );
+    assert(list == item->opal_list_item_belong_to);
 #endif
 
     return item;
@@ -353,14 +344,14 @@ static inline opal_list_item_t* opal_list_get_first(opal_list_t* list)
  * This is an inlined function in compilers that support inlining, so
  * it's usually a cheap operation.
  */
-static inline opal_list_item_t* opal_list_get_last(opal_list_t* list)
+static inline opal_list_item_t *opal_list_get_last(opal_list_t *list)
 {
-    opal_list_item_t* item = (opal_list_item_t *)list->opal_list_sentinel.opal_list_prev;
+    opal_list_item_t *item = (opal_list_item_t *) list->opal_list_sentinel.opal_list_prev;
 #if OPAL_ENABLE_DEBUG
     /* Spot check: ensure that the last item is only on one list */
 
-    assert( 1 == item->opal_list_item_refcount );
-    assert( list == item->opal_list_item_belong_to );
+    assert(1 == item->opal_list_item_refcount);
+    assert(list == item->opal_list_item_belong_to);
 #endif
 
     return item;
@@ -378,12 +369,12 @@ static inline opal_list_item_t* opal_list_get_last(opal_list_t* list)
  * Similar to the STL, this is a special invalid list item -- it
  * should \em not be used for storage.  It is only suitable for
  * comparison to other items in the list to see if they are valid or
- * not; it's ususally used when iterating through the items in a list.
+ * not; it's usually used when iterating through the items in a list.
  *
  * This is an inlined function in compilers that support inlining, so
  * it's usually a cheap operation.
  */
-static inline opal_list_item_t* opal_list_get_begin(opal_list_t* list)
+static inline opal_list_item_t *opal_list_get_begin(opal_list_t *list)
 {
     return &(list->opal_list_sentinel);
 }
@@ -400,17 +391,16 @@ static inline opal_list_item_t* opal_list_get_begin(opal_list_t* list)
  * Similar to the STL, this is a special invalid list item -- it
  * should \em not be used for storage.  It is only suitable for
  * comparison to other items in the list to see if they are valid or
- * not; it's ususally used when iterating through the items in a list.
+ * not; it's usually used when iterating through the items in a list.
  *
  * This is an inlined function in compilers that support inlining, so
  * it's usually a cheap operation.
  */
-static inline opal_list_item_t* opal_list_get_end(opal_list_t* list)
+static inline opal_list_item_t *opal_list_get_end(opal_list_t *list)
 {
     return &(list->opal_list_sentinel);
 }
 
-
 /**
  * Return the number of items in a list
  *
@@ -430,7 +420,7 @@ static inline opal_list_item_t* opal_list_get_end(opal_list_t* list)
  * never called on the specified list, this function will always be
  * O(1).
  */
-static inline size_t opal_list_get_size(opal_list_t* list)
+static inline size_t opal_list_get_size(opal_list_t *list)
 {
 #if OPAL_ENABLE_DEBUG && 0
     /* not sure if we really want this running in devel, as it does
@@ -440,14 +430,15 @@ static inline size_t opal_list_get_size(opal_list_t* list)
     size_t check_len = 0;
     opal_list_item_t *item;
 
-    for (item = opal_list_get_first(list) ;
-         item != opal_list_get_end(list) ;
+    for (item = opal_list_get_first(list); item != opal_list_get_end(list);
          item = opal_list_get_next(item)) {
         check_len++;
     }
 
     if (check_len != list->opal_list_length) {
-        fprintf(stderr," Error :: opal_list_get_size - opal_list_length does not match actual list length\n");
+        fprintf(
+            stderr,
+            " Error :: opal_list_get_size - opal_list_length does not match actual list length\n");
         fflush(stderr);
         abort();
     }
@@ -456,7 +447,6 @@ static inline size_t opal_list_get_size(opal_list_t* list)
     return list->opal_list_length;
 }
 
-
 /**
  * Remove an item from a list.
  *
@@ -478,51 +468,49 @@ static inline size_t opal_list_get_size(opal_list_t* list)
  * This is an inlined function in compilers that support inlining, so
  * it's usually a cheap operation.
  */
-static inline opal_list_item_t *opal_list_remove_item
-  (opal_list_t *list, opal_list_item_t *item)
+static inline opal_list_item_t *opal_list_remove_item(opal_list_t *list, opal_list_item_t *item)
 {
 #if OPAL_ENABLE_DEBUG
     opal_list_item_t *item_ptr;
     bool found = false;
 
     /* check to see that the item is in the list */
-    for (item_ptr = opal_list_get_first(list);
-            item_ptr != opal_list_get_end(list);
-            item_ptr = (opal_list_item_t *)(item_ptr->opal_list_next)) {
+    for (item_ptr = opal_list_get_first(list); item_ptr != opal_list_get_end(list);
+         item_ptr = (opal_list_item_t *) (item_ptr->opal_list_next)) {
         if (item_ptr == (opal_list_item_t *) item) {
             found = true;
             break;
         }
     }
     if (!found) {
-        fprintf(stderr," Warning :: opal_list_remove_item - the item %p is not on the list %p \n",(void*) item, (void*) list);
+        fprintf(stderr, " Warning :: opal_list_remove_item - the item %p is not on the list %p \n",
+                (void *) item, (void *) list);
         fflush(stderr);
-        return (opal_list_item_t *)NULL;
+        return (opal_list_item_t *) NULL;
     }
 
-    assert( list == item->opal_list_item_belong_to );
+    assert(list == item->opal_list_item_belong_to);
 #endif
 
     /* reset next pointer of previous element */
-    item->opal_list_prev->opal_list_next=item->opal_list_next;
+    item->opal_list_prev->opal_list_next = item->opal_list_next;
 
     /* reset previous pointer of next element */
-    item->opal_list_next->opal_list_prev=item->opal_list_prev;
+    item->opal_list_next->opal_list_prev = item->opal_list_prev;
 
     list->opal_list_length--;
 
 #if OPAL_ENABLE_DEBUG
     /* Spot check: ensure that this item is still only on one list */
 
-    OPAL_THREAD_ADD_FETCH32( &(item->opal_list_item_refcount), -1 );
+    OPAL_THREAD_ADD_FETCH32(&(item->opal_list_item_refcount), -1);
     assert(0 == item->opal_list_item_refcount);
     item->opal_list_item_belong_to = NULL;
 #endif
 
-    return (opal_list_item_t *)item->opal_list_prev;
+    return (opal_list_item_t *) item->opal_list_prev;
 }
 
-
 /**
  * Append an item to the end of the list.
  *
@@ -538,55 +526,53 @@ static inline opal_list_item_t *opal_list_remove_item
  */
 
 #if OPAL_ENABLE_DEBUG
-#define opal_list_append(l,i) \
-_opal_list_append(l,i,__FILE__,__LINE__)
+#    define opal_list_append(l, i) _opal_list_append(l, i, __FILE__, __LINE__)
 #else
-#define opal_list_append(l,i) \
-_opal_list_append(l,i)
-#endif  /* OPAL_ENABLE_DEBUG */
+#    define opal_list_append(l, i) _opal_list_append(l, i)
+#endif /* OPAL_ENABLE_DEBUG */
 
 static inline void _opal_list_append(opal_list_t *list, opal_list_item_t *item
 #if OPAL_ENABLE_DEBUG
-                                     , const char* FILE_NAME, int LINENO
-#endif  /* OPAL_ENABLE_DEBUG */
-                                     )
+                                     ,
+                                     const char *FILE_NAME, int LINENO
+#endif /* OPAL_ENABLE_DEBUG */
+)
 {
-    opal_list_item_t* sentinel = &(list->opal_list_sentinel);
+    opal_list_item_t *sentinel = &(list->opal_list_sentinel);
 #if OPAL_ENABLE_DEBUG
-  /* Spot check: ensure that this item is previously on no lists */
+    /* Spot check: ensure that this item is previously on no lists */
 
-  assert(0 == item->opal_list_item_refcount);
-  assert( NULL == item->opal_list_item_belong_to );
-  item->super.cls_init_file_name = FILE_NAME;
-  item->super.cls_init_lineno    = LINENO;
+    assert(0 == item->opal_list_item_refcount);
+    assert(NULL == item->opal_list_item_belong_to);
+    item->super.cls_init_file_name = FILE_NAME;
+    item->super.cls_init_lineno = LINENO;
 #endif
 
-  /* set new element's previous pointer */
-  item->opal_list_prev = sentinel->opal_list_prev;
+    /* set new element's previous pointer */
+    item->opal_list_prev = sentinel->opal_list_prev;
 
-  /* reset previous pointer on current last element */
-  sentinel->opal_list_prev->opal_list_next = item;
+    /* reset previous pointer on current last element */
+    sentinel->opal_list_prev->opal_list_next = item;
 
-  /* reset new element's next pointer */
-  item->opal_list_next = sentinel;
+    /* reset new element's next pointer */
+    item->opal_list_next = sentinel;
 
-  /* reset the list's tail element previous pointer */
-  sentinel->opal_list_prev = item;
+    /* reset the list's tail element previous pointer */
+    sentinel->opal_list_prev = item;
 
-  /* increment list element counter */
-  list->opal_list_length++;
+    /* increment list element counter */
+    list->opal_list_length++;
 
 #if OPAL_ENABLE_DEBUG
-  /* Spot check: ensure this item is only on the list that we just
-     appended it to */
+    /* Spot check: ensure this item is only on the list that we just
+       appended it to */
 
-  OPAL_THREAD_ADD_FETCH32( &(item->opal_list_item_refcount), 1 );
-  assert(1 == item->opal_list_item_refcount);
-  item->opal_list_item_belong_to = list;
+    OPAL_THREAD_ADD_FETCH32(&(item->opal_list_item_refcount), 1);
+    assert(1 == item->opal_list_item_refcount);
+    item->opal_list_item_belong_to = list;
 #endif
 }
 
-
 /**
  * Prepend an item to the beginning of the list.
  *
@@ -600,43 +586,41 @@ static inline void _opal_list_append(opal_list_t *list, opal_list_item_t *item
  * This is an inlined function in compilers that support inlining, so
  * it's usually a cheap operation.
  */
-static inline void opal_list_prepend(opal_list_t *list,
-                                     opal_list_item_t *item)
+static inline void opal_list_prepend(opal_list_t *list, opal_list_item_t *item)
 {
-    opal_list_item_t* sentinel = &(list->opal_list_sentinel);
+    opal_list_item_t *sentinel = &(list->opal_list_sentinel);
 #if OPAL_ENABLE_DEBUG
-  /* Spot check: ensure that this item is previously on no lists */
+    /* Spot check: ensure that this item is previously on no lists */
 
-  assert(0 == item->opal_list_item_refcount);
-  assert( NULL == item->opal_list_item_belong_to );
+    assert(0 == item->opal_list_item_refcount);
+    assert(NULL == item->opal_list_item_belong_to);
 #endif
 
-  /* reset item's next pointer */
-  item->opal_list_next = sentinel->opal_list_next;
+    /* reset item's next pointer */
+    item->opal_list_next = sentinel->opal_list_next;
 
-  /* reset item's previous pointer */
-  item->opal_list_prev = sentinel;
+    /* reset item's previous pointer */
+    item->opal_list_prev = sentinel;
 
-  /* reset previous first element's previous poiner */
-  sentinel->opal_list_next->opal_list_prev = item;
+    /* reset previous first element's previous pointer */
+    sentinel->opal_list_next->opal_list_prev = item;
 
-  /* reset head's next pointer */
-  sentinel->opal_list_next = item;
+    /* reset head's next pointer */
+    sentinel->opal_list_next = item;
 
-  /* increment list element counter */
-  list->opal_list_length++;
+    /* increment list element counter */
+    list->opal_list_length++;
 
 #if OPAL_ENABLE_DEBUG
-  /* Spot check: ensure this item is only on the list that we just
-     prepended it to */
+    /* Spot check: ensure this item is only on the list that we just
+       prepended it to */
 
-  OPAL_THREAD_ADD_FETCH32( &(item->opal_list_item_refcount), 1 );
-  assert(1 == item->opal_list_item_refcount);
-  item->opal_list_item_belong_to = list;
+    OPAL_THREAD_ADD_FETCH32(&(item->opal_list_item_refcount), 1);
+    assert(1 == item->opal_list_item_refcount);
+    item->opal_list_item_belong_to = list;
 #endif
 }
 
-
 /**
  * Remove the first item from the list and return it.
  *
@@ -655,50 +639,49 @@ static inline void opal_list_prepend(opal_list_t *list,
  */
 static inline opal_list_item_t *opal_list_remove_first(opal_list_t *list)
 {
-  /*  Removes and returns first item on list.
-      Caller now owns the item and should release the item
-      when caller is done with it.
-  */
-  opal_list_item_t *item;
-  if ( 0 == list->opal_list_length ) {
-    return (opal_list_item_t *)NULL;
-  }
+    /*  Removes and returns first item on list.
+        Caller now owns the item and should release the item
+        when caller is done with it.
+    */
+    opal_list_item_t *item;
+    if (0 == list->opal_list_length) {
+        return (opal_list_item_t *) NULL;
+    }
 
 #if OPAL_ENABLE_DEBUG
-  /* Spot check: ensure that the first item is only on this list */
+    /* Spot check: ensure that the first item is only on this list */
 
-  assert(1 == list->opal_list_sentinel.opal_list_next->opal_list_item_refcount);
+    assert(1 == list->opal_list_sentinel.opal_list_next->opal_list_item_refcount);
 #endif
 
-  /* reset list length counter */
-  list->opal_list_length--;
+    /* reset list length counter */
+    list->opal_list_length--;
 
-  /* get pointer to first element on the list */
-  item = (opal_list_item_t *) list->opal_list_sentinel.opal_list_next;
+    /* get pointer to first element on the list */
+    item = (opal_list_item_t *) list->opal_list_sentinel.opal_list_next;
 
-  /* reset previous pointer of next item on the list */
-  item->opal_list_next->opal_list_prev = item->opal_list_prev;
+    /* reset previous pointer of next item on the list */
+    item->opal_list_next->opal_list_prev = item->opal_list_prev;
 
-  /* reset the head next pointer */
-  list->opal_list_sentinel.opal_list_next = item->opal_list_next;
+    /* reset the head next pointer */
+    list->opal_list_sentinel.opal_list_next = item->opal_list_next;
 
 #if OPAL_ENABLE_DEBUG
-  assert( list == item->opal_list_item_belong_to );
-  item->opal_list_item_belong_to = NULL;
-  item->opal_list_prev=(opal_list_item_t *)NULL;
-  item->opal_list_next=(opal_list_item_t *)NULL;
+    assert(list == item->opal_list_item_belong_to);
+    item->opal_list_item_belong_to = NULL;
+    item->opal_list_prev = (opal_list_item_t *) NULL;
+    item->opal_list_next = (opal_list_item_t *) NULL;
 
-  /* Spot check: ensure that the item we're returning is now on no
-     lists */
+    /* Spot check: ensure that the item we're returning is now on no
+       lists */
 
-  OPAL_THREAD_ADD_FETCH32( &item->opal_list_item_refcount, -1 );
-  assert(0 == item->opal_list_item_refcount);
+    OPAL_THREAD_ADD_FETCH32(&item->opal_list_item_refcount, -1);
+    assert(0 == item->opal_list_item_refcount);
 #endif
 
-  return item;
+    return item;
 }
 
-
 /**
  * Remove the last item from the list and return it.
  *
@@ -717,57 +700,57 @@ static inline opal_list_item_t *opal_list_remove_first(opal_list_t *list)
  */
 static inline opal_list_item_t *opal_list_remove_last(opal_list_t *list)
 {
-  /*  Removes, releases and returns last item on list.
-      Caller now owns the item and should release the item
-      when caller is done with it.
-  */
-  opal_list_item_t  *item;
-  if ( 0 == list->opal_list_length ) {
-      return (opal_list_item_t *)NULL;
-  }
+    /*  Removes, releases and returns last item on list.
+        Caller now owns the item and should release the item
+        when caller is done with it.
+    */
+    opal_list_item_t *item;
+    if (0 == list->opal_list_length) {
+        return (opal_list_item_t *) NULL;
+    }
 
 #if OPAL_ENABLE_DEBUG
-  /* Spot check: ensure that the first item is only on this list */
+    /* Spot check: ensure that the first item is only on this list */
 
-  assert(1 == list->opal_list_sentinel.opal_list_prev->opal_list_item_refcount);
+    assert(1 == list->opal_list_sentinel.opal_list_prev->opal_list_item_refcount);
 #endif
 
-  /* reset list length counter */
-  list->opal_list_length--;
+    /* reset list length counter */
+    list->opal_list_length--;
 
-  /* get item */
-  item = (opal_list_item_t *) list->opal_list_sentinel.opal_list_prev;
+    /* get item */
+    item = (opal_list_item_t *) list->opal_list_sentinel.opal_list_prev;
 
-  /* reset previous pointer on next to last pointer */
-  item->opal_list_prev->opal_list_next = item->opal_list_next;
+    /* reset previous pointer on next to last pointer */
+    item->opal_list_prev->opal_list_next = item->opal_list_next;
 
-  /* reset tail's previous pointer */
-  list->opal_list_sentinel.opal_list_prev = item->opal_list_prev;
+    /* reset tail's previous pointer */
+    list->opal_list_sentinel.opal_list_prev = item->opal_list_prev;
 
 #if OPAL_ENABLE_DEBUG
-  assert( list == item->opal_list_item_belong_to );
-  item->opal_list_next = item->opal_list_prev = (opal_list_item_t *)NULL;
+    assert(list == item->opal_list_item_belong_to);
+    item->opal_list_next = item->opal_list_prev = (opal_list_item_t *) NULL;
 
-  /* Spot check: ensure that the item we're returning is now on no
-     lists */
+    /* Spot check: ensure that the item we're returning is now on no
+       lists */
 
-  OPAL_THREAD_ADD_FETCH32(&item->opal_list_item_refcount, -1 );
-  assert(0 == item->opal_list_item_refcount);
-  item->opal_list_item_belong_to = NULL;
+    OPAL_THREAD_ADD_FETCH32(&item->opal_list_item_refcount, -1);
+    assert(0 == item->opal_list_item_refcount);
+    item->opal_list_item_belong_to = NULL;
 #endif
 
-  return item;
+    return item;
 }
 
-  /**
-   * Add an item to the list before a given element
-   *
-   * @param list The list container
-   * @param pos List element to insert \c item before
-   * @param item The item to insert
-   *
-   * Inserts \c item before \c pos.  This is an O(1) operation.
-   */
+/**
+ * Add an item to the list before a given element
+ *
+ * @param list The list container
+ * @param pos List element to insert \c item before
+ * @param item The item to insert
+ *
+ * Inserts \c item before \c pos.  This is an O(1) operation.
+ */
 static inline void opal_list_insert_pos(opal_list_t *list, opal_list_item_t *pos,
                                         opal_list_item_t *item)
 {
@@ -776,7 +759,7 @@ static inline void opal_list_insert_pos(opal_list_t *list, opal_list_item_t *pos
        not on any list */
 
     assert(0 == item->opal_list_item_refcount);
-    assert( NULL == item->opal_list_item_belong_to );
+    assert(NULL == item->opal_list_item_belong_to);
 #endif
 
     /* point item at the existing elements */
@@ -794,126 +777,101 @@ static inline void opal_list_insert_pos(opal_list_t *list, opal_list_item_t *pos
     /* Spot check: double check that this item is only on the list
        that we just added it to */
 
-    OPAL_THREAD_ADD_FETCH32( &(item->opal_list_item_refcount), 1 );
+    OPAL_THREAD_ADD_FETCH32(&(item->opal_list_item_refcount), 1);
     assert(1 == item->opal_list_item_refcount);
     item->opal_list_item_belong_to = list;
 #endif
 }
 
-  /**
-   * Add an item to the list at a specific index location in the list.
-   *
-   * @param list The list container
-   * @param item The item to insert
-   * @param index Location to add the item
-   *
-   * @returns true if insertion succeeded; otherwise false
-   *
-   * This is potentially an O(N) operation to traverse down to the
-   * correct location in the list and add an item.
-   *
-   * Example: if idx = 2 and list = item1->item2->item3->item4, then
-   * after insert, list = item1->item2->item->item3->item4.
-   *
-   * If index is greater than the length of the list, no action is
-   * performed and false is returned.
-   */
-  OPAL_DECLSPEC bool opal_list_insert(opal_list_t *list, opal_list_item_t *item,
-                                      long long idx);
-
-
-    /**
-     * Join a list into another list
-     *
-     * @param thislist List container for list being operated on
-     * @param pos List item in \c thislist marking the position before
-     *              which items are inserted
-     * @param xlist List container for list being spliced from
-     *
-     * Join a list into another list.  All of the elements of \c xlist
-     * are inserted before \c pos and removed from \c xlist.
-     *
-     * This operation is an O(1) operation.  Both \c thislist and \c
-     * xlist must be valid list containsers.  \c xlist will be empty
-     * but valid after the call.  All pointers to \c opal_list_item_t
-     * containers remain valid, including those that point to elements
-     * in \c xlist.
-     */
-    OPAL_DECLSPEC void opal_list_join(opal_list_t *thislist, opal_list_item_t *pos,
-                                      opal_list_t *xlist);
-
-
-    /**
-     * Splice a list into another list
-     *
-     * @param thislist List container for list being operated on
-     * @param pos List item in \c thislist marking the position before
-     *             which items are inserted
-     * @param xlist List container for list being spliced from
-     * @param first List item in \c xlist marking the start of elements
-     *             to be copied into \c thislist
-     * @param last List item in \c xlist marking the end of elements
-     * to be copied into \c thislist
-     *
-     * Splice a subset of a list into another list.  The \c [first,
-     * last) elements of \c xlist are moved into \c thislist,
-     * inserting them before \c pos.  \c pos must be a valid iterator
-     * in \c thislist and \c [first, last) must be a valid range in \c
-     * xlist.  \c postition must not be in the range \c [first, last).
-     * It is, however, valid for \c xlist and \c thislist to be the
-     * same list.
-     *
-     * This is an O(N) operation because the length of both lists must
-     * be recomputed.
-     */
-    OPAL_DECLSPEC void opal_list_splice(opal_list_t *thislist, opal_list_item_t *pos,
-                                        opal_list_t *xlist, opal_list_item_t *first,
-                                        opal_list_item_t *last);
-
-    /**
-     * Comparison function for opal_list_sort(), below.
-     *
-     * @param a Pointer to a pointer to an opal_list_item_t.
-     * Explanation below.
-     * @param b Pointer to a pointer to an opal_list_item_t.
-     * Explanation below.
-     * @retval 1 if \em a is greater than \em b
-     * @retval 0 if \em a is equal to \em b
-     * @retval -1 if \em a is less than \em b
-     *
-     * This function is invoked by qsort(3) from within
-     * opal_list_sort().  It is important to understand what
-     * opal_list_sort() does before invoking qsort, so go read that
-     * documentation first.
-     *
-     * The important thing to realize here is that a and b will be \em
-     * double pointers to the items that you need to compare.  Here's
-     * a sample compare function to illustrate this point:
-     */
-    typedef int (*opal_list_item_compare_fn_t)(opal_list_item_t **a,
-                                               opal_list_item_t **b);
-
-    /**
-     * Sort a list with a provided compare function.
-     *
-     * @param list The list to sort
-     * @param compare Compare function
-     *
-     * Put crassly, this function's complexity is O(N) + O(log(N)).
-     * Its algorithm is:
-     *
-     * - remove every item from the list and put the corresponding
-     *    (opal_list_item_t*)'s in an array
-     * - call qsort(3) with that array and your compare function
-     * - re-add every element of the now-sorted array to the list
-     *
-     * The resulting list is now ordered.  Note, however, that since
-     * an array of pointers is sorted, the comparison function must do
-     * a double de-reference to get to the actual opal_list_item_t (or
-     * whatever the underlying type is).  See the documentation of
-     * opal_list_item_compare_fn_t for an example).
-     */
-    OPAL_DECLSPEC int opal_list_sort(opal_list_t* list, opal_list_item_compare_fn_t compare);
+/**
+ * Join a list into another list
+ *
+ * @param thislist List container for list being operated on
+ * @param pos List item in \c thislist marking the position before
+ *              which items are inserted
+ * @param xlist List container for list being spliced from
+ *
+ * Join a list into another list.  All of the elements of \c xlist
+ * are inserted before \c pos and removed from \c xlist.
+ *
+ * This operation is an O(1) operation.  Both \c thislist and \c
+ * xlist must be valid list containsers.  \c xlist will be empty
+ * but valid after the call.  All pointers to \c opal_list_item_t
+ * containers remain valid, including those that point to elements
+ * in \c xlist.
+ */
+OPAL_DECLSPEC void opal_list_join(opal_list_t *thislist, opal_list_item_t *pos, opal_list_t *xlist);
+
+/**
+ * Splice a list into another list
+ *
+ * @param thislist List container for list being operated on
+ * @param pos List item in \c thislist marking the position before
+ *             which items are inserted
+ * @param xlist List container for list being spliced from
+ * @param first List item in \c xlist marking the start of elements
+ *             to be copied into \c thislist
+ * @param last List item in \c xlist marking the end of elements
+ * to be copied into \c thislist
+ *
+ * Splice a subset of a list into another list.  The \c [first,
+ * last) elements of \c xlist are moved into \c thislist,
+ * inserting them before \c pos.  \c pos must be a valid iterator
+ * in \c thislist and \c [first, last) must be a valid range in \c
+ * xlist.  \c position must not be in the range \c [first, last).
+ * It is, however, valid for \c xlist and \c thislist to be the
+ * same list.
+ *
+ * This is an O(N) operation because the length of both lists must
+ * be recomputed.
+ */
+OPAL_DECLSPEC void opal_list_splice(opal_list_t *thislist, opal_list_item_t *pos,
+                                    opal_list_t *xlist, opal_list_item_t *first,
+                                    opal_list_item_t *last);
+
+/**
+ * Comparison function for opal_list_sort(), below.
+ *
+ * @param a Pointer to a pointer to an opal_list_item_t.
+ * Explanation below.
+ * @param b Pointer to a pointer to an opal_list_item_t.
+ * Explanation below.
+ * @retval 1 if \em a is greater than \em b
+ * @retval 0 if \em a is equal to \em b
+ * @retval -1 if \em a is less than \em b
+ *
+ * This function is invoked by qsort(3) from within
+ * opal_list_sort().  It is important to understand what
+ * opal_list_sort() does before invoking qsort, so go read that
+ * documentation first.
+ *
+ * The important thing to realize here is that a and b will be \em
+ * double pointers to the items that you need to compare.  Here's
+ * a sample compare function to illustrate this point:
+ */
+typedef int (*opal_list_item_compare_fn_t)(opal_list_item_t **a, opal_list_item_t **b);
+
+/**
+ * Sort a list with a provided compare function.
+ *
+ * @param list The list to sort
+ * @param compare Compare function
+ *
+ * Put crassly, this function's complexity is O(N) + O(log(N)).
+ * Its algorithm is:
+ *
+ * - remove every item from the list and put the corresponding
+ *    (opal_list_item_t*)'s in an array
+ * - call qsort(3) with that array and your compare function
+ * - re-add every element of the now-sorted array to the list
+ *
+ * The resulting list is now ordered.  Note, however, that since
+ * an array of pointers is sorted, the comparison function must do
+ * a double de-reference to get to the actual opal_list_item_t (or
+ * whatever the underlying type is).  See the documentation of
+ * opal_list_item_compare_fn_t for an example).
+ */
+OPAL_DECLSPEC int opal_list_sort(opal_list_t *list, opal_list_item_compare_fn_t compare);
 
 END_C_DECLS
 
diff --git a/opal/class/opal_object.c b/opal/class/opal_object.c
index 64fef5712fc..c7127d1f587 100644
--- a/opal/class/opal_object.c
+++ b/opal/class/opal_object.c
@@ -29,9 +29,9 @@
 
 #include <stdio.h>
 
-#include "opal/sys/atomic.h"
 #include "opal/class/opal_object.h"
 #include "opal/constants.h"
+#include "opal/sys/atomic.h"
 
 /*
  * Instantiation of class descriptor for the base class.  This is
@@ -56,27 +56,25 @@ int opal_class_init_epoch = 1;
  * Local variables
  */
 static opal_atomic_lock_t class_lock = OPAL_ATOMIC_LOCK_INIT;
-static void** classes = NULL;
+static void **classes = NULL;
 static int num_classes = 0;
 static int max_classes = 0;
 static const int increment = 10;
 
-
 /*
  * Local functions
  */
 static void save_class(opal_class_t *cls);
 static void expand_array(void);
 
-
 /*
  * Lazy initialization of class descriptor.
  */
 void opal_class_initialize(opal_class_t *cls)
 {
     opal_class_t *c;
-    opal_construct_t* cls_construct_array;
-    opal_destruct_t* cls_destruct_array;
+    opal_construct_t *cls_construct_array;
+    opal_destruct_t *cls_destruct_array;
     int cls_construct_array_count;
     int cls_destruct_array_count;
     int i;
@@ -107,12 +105,12 @@ void opal_class_initialize(opal_class_t *cls)
 
     cls->cls_depth = 0;
     cls_construct_array_count = 0;
-    cls_destruct_array_count  = 0;
+    cls_destruct_array_count = 0;
     for (c = cls; c; c = c->cls_parent) {
-        if( NULL != c->cls_construct ) {
+        if (NULL != c->cls_construct) {
             cls_construct_array_count++;
         }
-        if( NULL != c->cls_destruct ) {
+        if (NULL != c->cls_destruct) {
             cls_destruct_array_count++;
         }
         cls->cls_depth++;
@@ -123,38 +121,35 @@ void opal_class_initialize(opal_class_t *cls)
      * plus for each a NULL-sentinel
      */
 
-    cls->cls_construct_array =
-        (void (**)(opal_object_t*))malloc((cls_construct_array_count +
-                                           cls_destruct_array_count + 2) *
-                                          sizeof(opal_construct_t) );
+    cls->cls_construct_array = (void (**)(opal_object_t *)) malloc(
+        (cls_construct_array_count + cls_destruct_array_count + 2) * sizeof(opal_construct_t));
     if (NULL == cls->cls_construct_array) {
         perror("Out of memory");
         exit(-1);
     }
-    cls->cls_destruct_array =
-        cls->cls_construct_array + cls_construct_array_count + 1;
+    cls->cls_destruct_array = cls->cls_construct_array + cls_construct_array_count + 1;
 
     /*
      * The constructor array is reversed, so start at the end
      */
 
     cls_construct_array = cls->cls_construct_array + cls_construct_array_count;
-    cls_destruct_array  = cls->cls_destruct_array;
+    cls_destruct_array = cls->cls_destruct_array;
 
     c = cls;
-    *cls_construct_array = NULL;  /* end marker for the constructors */
+    *cls_construct_array = NULL; /* end marker for the constructors */
     for (i = 0; i < cls->cls_depth; i++) {
-        if( NULL != c->cls_construct ) {
+        if (NULL != c->cls_construct) {
             --cls_construct_array;
             *cls_construct_array = c->cls_construct;
         }
-        if( NULL != c->cls_destruct ) {
+        if (NULL != c->cls_destruct) {
             *cls_destruct_array = c->cls_destruct;
             cls_destruct_array++;
         }
         c = c->cls_parent;
     }
-    *cls_destruct_array = NULL;  /* end marker for the destructors */
+    *cls_destruct_array = NULL; /* end marker for the destructors */
 
     cls->cls_initialized = opal_class_init_epoch;
     save_class(cls);
@@ -164,7 +159,6 @@ void opal_class_initialize(opal_class_t *cls)
     opal_atomic_unlock(&class_lock);
 }
 
-
 /*
  * Note that this is finalize for *all* classes.
  */
@@ -193,7 +187,6 @@ int opal_class_finalize(void)
     return OPAL_SUCCESS;
 }
 
-
 static void save_class(opal_class_t *cls)
 {
     if (num_classes >= max_classes) {
@@ -204,13 +197,12 @@ static void save_class(opal_class_t *cls)
     ++num_classes;
 }
 
-
 static void expand_array(void)
 {
     int i;
 
     max_classes += increment;
-    classes = (void**)realloc(classes, sizeof(opal_class_t*) * max_classes);
+    classes = (void **) realloc(classes, sizeof(opal_class_t *) * max_classes);
     if (NULL == classes) {
         perror("class malloc failed");
         exit(-1);
@@ -219,4 +211,3 @@ static void expand_array(void)
         classes[i] = NULL;
     }
 }
-
diff --git a/opal/class/opal_object.h b/opal/class/opal_object.h
index 2a2f1edbbe3..40cd9e5adbe 100644
--- a/opal/class/opal_object.h
+++ b/opal/class/opal_object.h
@@ -129,16 +129,15 @@ BEGIN_C_DECLS
 
 #if OPAL_ENABLE_DEBUG
 /* Any kind of unique ID should do the job */
-#define OPAL_OBJ_MAGIC_ID ((0xdeafbeedULL << 32) + 0xdeafbeedULL)
+#    define OPAL_OBJ_MAGIC_ID ((0xdeafbeedULL << 32) + 0xdeafbeedULL)
 #endif
 
 /* typedefs ***********************************************************/
 
 typedef struct opal_object_t opal_object_t;
 typedef struct opal_class_t opal_class_t;
-typedef void (*opal_construct_t) (opal_object_t *);
-typedef void (*opal_destruct_t) (opal_object_t *);
-
+typedef void (*opal_construct_t)(opal_object_t *);
+typedef void (*opal_destruct_t)(opal_object_t *);
 
 /* types **************************************************************/
 
@@ -156,10 +155,10 @@ struct opal_class_t {
     int cls_initialized;            /**< is class initialized */
     int cls_depth;                  /**< depth of class hierarchy tree */
     opal_construct_t *cls_construct_array;
-                                    /**< array of parent class constructors */
+    /**< array of parent class constructors */
     opal_destruct_t *cls_destruct_array;
-                                    /**< array of parent class destructors */
-    size_t cls_sizeof;              /**< size of an object instance */
+    /**< array of parent class destructors */
+    size_t cls_sizeof; /**< size of an object instance */
 };
 
 extern int opal_class_init_epoch;
@@ -170,20 +169,16 @@ extern int opal_class_init_epoch;
  * @param NAME   Name of the class to initialize
  */
 #if OPAL_ENABLE_DEBUG
-#define OPAL_OBJ_STATIC_INIT(BASE_CLASS)        \
-    {                                           \
-        .obj_magic_id = OPAL_OBJ_MAGIC_ID,      \
-        .obj_class = OBJ_CLASS(BASE_CLASS),     \
-        .obj_reference_count = 1,               \
-        .cls_init_file_name = __FILE__,         \
-        .cls_init_lineno = __LINE__,            \
-    }
+#    define OPAL_OBJ_STATIC_INIT(BASE_CLASS)                                                       \
+        {                                                                                          \
+            .obj_magic_id = OPAL_OBJ_MAGIC_ID, .obj_class = OBJ_CLASS(BASE_CLASS),                 \
+            .obj_reference_count = 1, .cls_init_file_name = __FILE__, .cls_init_lineno = __LINE__, \
+        }
 #else
-#define OPAL_OBJ_STATIC_INIT(BASE_CLASS)        \
-    {                                           \
-        .obj_class = OBJ_CLASS(BASE_CLASS),     \
-        .obj_reference_count = 1,               \
-    }
+#    define OPAL_OBJ_STATIC_INIT(BASE_CLASS)                              \
+        {                                                                 \
+            .obj_class = OBJ_CLASS(BASE_CLASS), .obj_reference_count = 1, \
+        }
 #endif
 
 /**
@@ -197,12 +192,13 @@ struct opal_object_t {
         struct's memory */
     uint64_t obj_magic_id;
 #endif
-    opal_class_t *obj_class;            /**< class descriptor */
-    opal_atomic_int32_t obj_reference_count;   /**< reference count */
+    opal_class_t *obj_class;                 /**< class descriptor */
+    opal_atomic_int32_t obj_reference_count; /**< reference count */
 #if OPAL_ENABLE_DEBUG
-   const char* cls_init_file_name;        /**< In debug mode store the file where the object get contructed */
-   int   cls_init_lineno;           /**< In debug mode store the line number where the object get contructed */
-#endif  /* OPAL_ENABLE_DEBUG */
+    const char
+        *cls_init_file_name; /**< In debug mode store the file where the object get constructed */
+    int cls_init_lineno; /**< In debug mode store the line number where the object get constructed */
+#endif /* OPAL_ENABLE_DEBUG */
 };
 
 /* macros ************************************************************/
@@ -214,8 +210,7 @@ struct opal_object_t {
  * @param NAME          Name of class
  * @return              Pointer to class descriptor
  */
-#define OBJ_CLASS(NAME)     (&(NAME ## _class))
-
+#define OBJ_CLASS(NAME) (&(NAME##_class))
 
 /**
  * Static initializer for a class descriptor
@@ -227,16 +222,16 @@ struct opal_object_t {
  *
  * Put this in NAME.c
  */
-#define OBJ_CLASS_INSTANCE(NAME, PARENT, CONSTRUCTOR, DESTRUCTOR)       \
-    opal_class_t NAME ## _class = {                                     \
-        # NAME,                                                         \
-        OBJ_CLASS(PARENT),                                              \
-        (opal_construct_t) CONSTRUCTOR,                                 \
-        (opal_destruct_t) DESTRUCTOR,                                   \
-        0, 0, NULL, NULL,                                               \
-        sizeof(NAME)                                                    \
-    }
-
+#define OBJ_CLASS_INSTANCE(NAME, PARENT, CONSTRUCTOR, DESTRUCTOR) \
+    opal_class_t NAME##_class = {#NAME,                           \
+                                 OBJ_CLASS(PARENT),               \
+                                 (opal_construct_t) CONSTRUCTOR,  \
+                                 (opal_destruct_t) DESTRUCTOR,    \
+                                 0,                               \
+                                 0,                               \
+                                 NULL,                            \
+                                 NULL,                            \
+                                 sizeof(NAME)}
 
 /**
  * Declaration for class descriptor
@@ -245,9 +240,7 @@ struct opal_object_t {
  *
  * Put this in NAME.h
  */
-#define OBJ_CLASS_DECLARATION(NAME)             \
-    extern opal_class_t NAME ## _class
-
+#define OBJ_CLASS_DECLARATION(NAME) extern opal_class_t NAME##_class
 
 /**
  * Create an object: dynamically allocate storage and run the class
@@ -256,22 +249,20 @@ struct opal_object_t {
  * @param type          Type (class) of the object
  * @return              Pointer to the object
  */
-static inline opal_object_t *opal_obj_new(opal_class_t * cls);
+static inline opal_object_t *opal_obj_new(opal_class_t *cls);
 #if OPAL_ENABLE_DEBUG
-static inline opal_object_t *opal_obj_new_debug(opal_class_t* type, const char* file, int line)
+static inline opal_object_t *opal_obj_new_debug(opal_class_t *type, const char *file, int line)
 {
-    opal_object_t* object = opal_obj_new(type);
+    opal_object_t *object = opal_obj_new(type);
     object->obj_magic_id = OPAL_OBJ_MAGIC_ID;
     object->cls_init_file_name = file;
     object->cls_init_lineno = line;
     return object;
 }
-#define OBJ_NEW(type)                                   \
-    ((type *)opal_obj_new_debug(OBJ_CLASS(type), __FILE__, __LINE__))
+#    define OBJ_NEW(type) ((type *) opal_obj_new_debug(OBJ_CLASS(type), __FILE__, __LINE__))
 #else
-#define OBJ_NEW(type)                                   \
-    ((type *) opal_obj_new(OBJ_CLASS(type)))
-#endif  /* OPAL_ENABLE_DEBUG */
+#    define OBJ_NEW(type) ((type *) opal_obj_new(OBJ_CLASS(type)))
+#endif /* OPAL_ENABLE_DEBUG */
 
 /**
  * Retain an object (by incrementing its reference count)
@@ -279,15 +270,15 @@ static inline opal_object_t *opal_obj_new_debug(opal_class_t* type, const char*
  * @param object        Pointer to the object
  */
 #if OPAL_ENABLE_DEBUG
-#define OBJ_RETAIN(object)                                              \
-    do {                                                                \
-        assert(NULL != ((opal_object_t *) (object))->obj_class);        \
-        assert(OPAL_OBJ_MAGIC_ID == ((opal_object_t *) (object))->obj_magic_id); \
-        opal_obj_update((opal_object_t *) (object), 1);                 \
-        assert(((opal_object_t *) (object))->obj_reference_count >= 0); \
-    } while (0)
+#    define OBJ_RETAIN(object)                                                       \
+        do {                                                                         \
+            assert(NULL != ((opal_object_t *) (object))->obj_class);                 \
+            assert(OPAL_OBJ_MAGIC_ID == ((opal_object_t *) (object))->obj_magic_id); \
+            opal_obj_update((opal_object_t *) (object), 1);                          \
+            assert(((opal_object_t *) (object))->obj_reference_count >= 0);          \
+        } while (0)
 #else
-#define OBJ_RETAIN(object)  opal_obj_update((opal_object_t *) (object), 1);
+#    define OBJ_RETAIN(object) opal_obj_update((opal_object_t *) (object), 1);
 #endif
 
 /**
@@ -295,19 +286,19 @@ static inline opal_object_t *opal_obj_new_debug(opal_class_t* type, const char*
  * an object change.
  */
 #if OPAL_ENABLE_DEBUG
-#define OBJ_REMEMBER_FILE_AND_LINENO( OBJECT, FILE, LINENO )    \
-    do {                                                        \
-        ((opal_object_t*)(OBJECT))->cls_init_file_name = FILE;  \
-        ((opal_object_t*)(OBJECT))->cls_init_lineno = LINENO;   \
-    } while(0)
-#define OBJ_SET_MAGIC_ID( OBJECT, VALUE )                       \
-    do {                                                        \
-        ((opal_object_t*)(OBJECT))->obj_magic_id = (VALUE);     \
-    } while(0)
+#    define OBJ_REMEMBER_FILE_AND_LINENO(OBJECT, FILE, LINENO)       \
+        do {                                                         \
+            ((opal_object_t *) (OBJECT))->cls_init_file_name = FILE; \
+            ((opal_object_t *) (OBJECT))->cls_init_lineno = LINENO;  \
+        } while (0)
+#    define OBJ_SET_MAGIC_ID(OBJECT, VALUE)                       \
+        do {                                                      \
+            ((opal_object_t *) (OBJECT))->obj_magic_id = (VALUE); \
+        } while (0)
 #else
-#define OBJ_REMEMBER_FILE_AND_LINENO( OBJECT, FILE, LINENO )
-#define OBJ_SET_MAGIC_ID( OBJECT, VALUE )
-#endif  /* OPAL_ENABLE_DEBUG */
+#    define OBJ_REMEMBER_FILE_AND_LINENO(OBJECT, FILE, LINENO)
+#    define OBJ_SET_MAGIC_ID(OBJECT, VALUE)
+#endif /* OPAL_ENABLE_DEBUG */
 
 /**
  * Release an object (by decrementing its reference count).  If the
@@ -322,49 +313,49 @@ static inline opal_object_t *opal_obj_new_debug(opal_class_t* type, const char*
  *
  */
 #if OPAL_ENABLE_DEBUG
-#define OBJ_RELEASE(object)                                             \
-    do {                                                                \
-        assert(OPAL_OBJ_MAGIC_ID == ((opal_object_t *) (object))->obj_magic_id); \
-        assert(NULL != ((opal_object_t *) (object))->obj_class);        \
-        if (0 == opal_obj_update((opal_object_t *) (object), -1)) {     \
-            OBJ_SET_MAGIC_ID((object), 0);                              \
-            opal_obj_run_destructors((opal_object_t *) (object));       \
-            OBJ_REMEMBER_FILE_AND_LINENO( object, __FILE__, __LINE__ ); \
-            free((void *) object);                                      \
-            object = NULL;                                              \
-        }                                                               \
-    } while (0)
+#    define OBJ_RELEASE(object)                                                      \
+        do {                                                                         \
+            assert(OPAL_OBJ_MAGIC_ID == ((opal_object_t *) (object))->obj_magic_id); \
+            assert(NULL != ((opal_object_t *) (object))->obj_class);                 \
+            if (0 == opal_obj_update((opal_object_t *) (object), -1)) {              \
+                OBJ_SET_MAGIC_ID((object), 0);                                       \
+                opal_obj_run_destructors((opal_object_t *) (object));                \
+                OBJ_REMEMBER_FILE_AND_LINENO(object, __FILE__, __LINE__);            \
+                free((void *) object);                                               \
+                object = NULL;                                                       \
+            }                                                                        \
+        } while (0)
 #else
-#define OBJ_RELEASE(object)                                             \
-    do {                                                                \
-        if (0 == opal_obj_update((opal_object_t *) (object), -1)) {     \
-            opal_obj_run_destructors((opal_object_t *) (object));       \
-            free((void *) object);                                      \
-            object = NULL;                                              \
-        }                                                               \
-    } while (0)
+#    define OBJ_RELEASE(object)                                         \
+        do {                                                            \
+            if (0 == opal_obj_update((opal_object_t *) (object), -1)) { \
+                opal_obj_run_destructors((opal_object_t *) (object));   \
+                free((void *) object);                                  \
+                object = NULL;                                          \
+            }                                                           \
+        } while (0)
 #endif
 
 #if OPAL_ENABLE_DEBUG
-#define OBJ_RELEASE_NO_NULLIFY(object)                                  \
-    do {                                                                \
-        assert(OPAL_OBJ_MAGIC_ID == ((opal_object_t *) (object))->obj_magic_id); \
-        assert(NULL != ((opal_object_t *) (object))->obj_class);        \
-        if (0 == opal_obj_update((opal_object_t *) (object), -1)) {     \
-            OBJ_SET_MAGIC_ID((object), 0);                              \
-            opal_obj_run_destructors((opal_object_t *) (object));       \
-            OBJ_REMEMBER_FILE_AND_LINENO( object, __FILE__, __LINE__ ); \
-            free((void *) object);                                      \
-        }                                                               \
-    } while (0)
+#    define OBJ_RELEASE_NO_NULLIFY(object)                                           \
+        do {                                                                         \
+            assert(OPAL_OBJ_MAGIC_ID == ((opal_object_t *) (object))->obj_magic_id); \
+            assert(NULL != ((opal_object_t *) (object))->obj_class);                 \
+            if (0 == opal_obj_update((opal_object_t *) (object), -1)) {              \
+                OBJ_SET_MAGIC_ID((object), 0);                                       \
+                opal_obj_run_destructors((opal_object_t *) (object));                \
+                OBJ_REMEMBER_FILE_AND_LINENO(object, __FILE__, __LINE__);            \
+                free((void *) object);                                               \
+            }                                                                        \
+        } while (0)
 #else
-#define OBJ_RELEASE_NO_NULLIFY(object)                                   \
-    do {                                                                \
-        if (0 == opal_obj_update((opal_object_t *) (object), -1)) {     \
-            opal_obj_run_destructors((opal_object_t *) (object));       \
-            free((void *) object);                                      \
-        }                                                               \
-    } while (0)
+#    define OBJ_RELEASE_NO_NULLIFY(object)                              \
+        do {                                                            \
+            if (0 == opal_obj_update((opal_object_t *) (object), -1)) { \
+                opal_obj_run_destructors((opal_object_t *) (object));   \
+                free((void *) object);                                  \
+            }                                                           \
+        } while (0)
 #endif
 
 /**
@@ -374,23 +365,22 @@ static inline opal_object_t *opal_obj_new_debug(opal_class_t* type, const char*
  * @param type          The object type
  */
 
-#define OBJ_CONSTRUCT(object, type)                             \
-do {                                                            \
-    OBJ_CONSTRUCT_INTERNAL((object), OBJ_CLASS(type));          \
-} while (0)
-
-#define OBJ_CONSTRUCT_INTERNAL(object, type)                        \
-do {                                                                \
-    OBJ_SET_MAGIC_ID((object), OPAL_OBJ_MAGIC_ID);                  \
-    if (opal_class_init_epoch != (type)->cls_initialized) {         \
-        opal_class_initialize((type));                              \
-    }                                                               \
-    ((opal_object_t *) (object))->obj_class = (type);               \
-    ((opal_object_t *) (object))->obj_reference_count = 1;          \
-    opal_obj_run_constructors((opal_object_t *) (object));          \
-    OBJ_REMEMBER_FILE_AND_LINENO( object, __FILE__, __LINE__ ); \
-} while (0)
+#define OBJ_CONSTRUCT(object, type)                        \
+    do {                                                   \
+        OBJ_CONSTRUCT_INTERNAL((object), OBJ_CLASS(type)); \
+    } while (0)
 
+#define OBJ_CONSTRUCT_INTERNAL(object, type)                      \
+    do {                                                          \
+        OBJ_SET_MAGIC_ID((object), OPAL_OBJ_MAGIC_ID);            \
+        if (opal_class_init_epoch != (type)->cls_initialized) {   \
+            opal_class_initialize((type));                        \
+        }                                                         \
+        ((opal_object_t *) (object))->obj_class = (type);         \
+        ((opal_object_t *) (object))->obj_reference_count = 1;    \
+        opal_obj_run_constructors((opal_object_t *) (object));    \
+        OBJ_REMEMBER_FILE_AND_LINENO(object, __FILE__, __LINE__); \
+    } while (0)
 
 /**
  * Destruct (finalize) an object that is not dynamically allocated.
@@ -398,19 +388,19 @@ do {                                                                \
  * @param object        Pointer to the object
  */
 #if OPAL_ENABLE_DEBUG
-#define OBJ_DESTRUCT(object)                                    \
-do {                                                            \
-    assert(OPAL_OBJ_MAGIC_ID == ((opal_object_t *) (object))->obj_magic_id); \
-    OBJ_SET_MAGIC_ID((object), 0);                              \
-    opal_obj_run_destructors((opal_object_t *) (object));       \
-    OBJ_REMEMBER_FILE_AND_LINENO( object, __FILE__, __LINE__ ); \
-} while (0)
+#    define OBJ_DESTRUCT(object)                                                     \
+        do {                                                                         \
+            assert(OPAL_OBJ_MAGIC_ID == ((opal_object_t *) (object))->obj_magic_id); \
+            OBJ_SET_MAGIC_ID((object), 0);                                           \
+            opal_obj_run_destructors((opal_object_t *) (object));                    \
+            OBJ_REMEMBER_FILE_AND_LINENO(object, __FILE__, __LINE__);                \
+        } while (0)
 #else
-#define OBJ_DESTRUCT(object)                                    \
-do {                                                            \
-    opal_obj_run_destructors((opal_object_t *) (object));       \
-    OBJ_REMEMBER_FILE_AND_LINENO( object, __FILE__, __LINE__ ); \
-} while (0)
+#    define OBJ_DESTRUCT(object)                                      \
+        do {                                                          \
+            opal_obj_run_destructors((opal_object_t *) (object));     \
+            OBJ_REMEMBER_FILE_AND_LINENO(object, __FILE__, __LINE__); \
+        } while (0)
 #endif
 
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_object_t);
@@ -450,20 +440,19 @@ OPAL_DECLSPEC int opal_class_finalize(void);
  * Hardwired for fairly shallow inheritance trees
  * @param size          Pointer to the object.
  */
-static inline void opal_obj_run_constructors(opal_object_t * object)
+static inline void opal_obj_run_constructors(opal_object_t *object)
 {
-    opal_construct_t* cls_construct;
+    opal_construct_t *cls_construct;
 
     assert(NULL != object->obj_class);
 
     cls_construct = object->obj_class->cls_construct_array;
-    while( NULL != *cls_construct ) {
+    while (NULL != *cls_construct) {
         (*cls_construct)(object);
         cls_construct++;
     }
 }
 
-
 /**
  * Run the hierarchy of class destructors for this object, in a
  * parent-last order.
@@ -472,20 +461,19 @@ static inline void opal_obj_run_constructors(opal_object_t * object)
  *
  * @param size          Pointer to the object.
  */
-static inline void opal_obj_run_destructors(opal_object_t * object)
+static inline void opal_obj_run_destructors(opal_object_t *object)
 {
-    opal_destruct_t* cls_destruct;
+    opal_destruct_t *cls_destruct;
 
     assert(NULL != object->obj_class);
 
     cls_destruct = object->obj_class->cls_destruct_array;
-    while( NULL != *cls_destruct ) {
+    while (NULL != *cls_destruct) {
         (*cls_destruct)(object);
         cls_destruct++;
     }
 }
 
-
 /**
  * Create new object: dynamically allocate storage and run the class
  * constructor.
@@ -496,7 +484,7 @@ static inline void opal_obj_run_destructors(opal_object_t * object)
  * @param cls           Pointer to the class descriptor of this object
  * @return              Pointer to the object
  */
-static inline opal_object_t *opal_obj_new(opal_class_t * cls)
+static inline opal_object_t *opal_obj_new(opal_class_t *cls)
 {
     opal_object_t *object;
     assert(cls->cls_sizeof >= sizeof(opal_object_t));
@@ -517,7 +505,6 @@ static inline opal_object_t *opal_obj_new(opal_class_t * cls)
     return object;
 }
 
-
 /**
  * Atomically update the object's reference count by some increment.
  *
diff --git a/opal/class/opal_pointer_array.c b/opal/class/opal_pointer_array.c
index b28337a616c..f00d5fa8bb0 100644
--- a/opal/class/opal_pointer_array.c
+++ b/opal/class/opal_pointer_array.c
@@ -19,20 +19,19 @@
 
 #include "opal_config.h"
 
-#include <stdlib.h>
-#include <stdio.h>
 #include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
 
-#include "opal/constants.h"
 #include "opal/class/opal_pointer_array.h"
+#include "opal/constants.h"
 #include "opal/util/output.h"
 
 static void opal_pointer_array_construct(opal_pointer_array_t *);
 static void opal_pointer_array_destruct(opal_pointer_array_t *);
 static bool grow_table(opal_pointer_array_t *table, int at_least);
 
-OBJ_CLASS_INSTANCE(opal_pointer_array_t, opal_object_t,
-                   opal_pointer_array_construct,
+OBJ_CLASS_INSTANCE(opal_pointer_array_t, opal_object_t, opal_pointer_array_construct,
                    opal_pointer_array_destruct);
 
 /*
@@ -56,11 +55,11 @@ static void opal_pointer_array_construct(opal_pointer_array_t *array)
 static void opal_pointer_array_destruct(opal_pointer_array_t *array)
 {
     /* free table */
-    if( NULL != array->free_bits) {
+    if (NULL != array->free_bits) {
         free(array->free_bits);
         array->free_bits = NULL;
     }
-    if( NULL != array->addr ) {
+    if (NULL != array->addr) {
         free(array->addr);
         array->addr = NULL;
     }
@@ -76,12 +75,12 @@ static void opal_pointer_array_destruct(opal_pointer_array_t *array)
  * Translate an index position into the free bits array into 2 values, the
  * index of the element and the index of the bit position.
  */
-#define GET_BIT_POS(IDX, BIDX, PIDX)                                    \
-    do {                                                                \
-        uint32_t __idx = (uint32_t)(IDX);                               \
-        (BIDX) = (__idx / (8 * sizeof(uint64_t)));                      \
-        (PIDX) = (__idx % (8 * sizeof(uint64_t)));                      \
-    } while(0)
+#define GET_BIT_POS(IDX, BIDX, PIDX)               \
+    do {                                           \
+        uint32_t __idx = (uint32_t)(IDX);          \
+        (BIDX) = (__idx / (8 * sizeof(uint64_t))); \
+        (PIDX) = (__idx % (8 * sizeof(uint64_t))); \
+    } while (0)
 
 /**
  * A classical find first zero bit (ffs) on a large array. It checks starting
@@ -92,61 +91,67 @@ static void opal_pointer_array_destruct(opal_pointer_array_t *array)
  * indicator to constants (the type is inferred by the compiler according to
  * the number of bits necessary to represent it).
  */
-#define FIND_FIRST_ZERO(START_IDX, STORE)                               \
-    do {                                                                \
-        uint32_t __b_idx, __b_pos;                                      \
-        if( 0 == table->number_free ) {                                 \
-            (STORE) = table->size;                                      \
-            break;                                                      \
-        }                                                               \
-        GET_BIT_POS((START_IDX), __b_idx, __b_pos);                     \
-        for (; table->free_bits[__b_idx] == 0xFFFFFFFFFFFFFFFFu; __b_idx++); \
-        assert(__b_idx < (uint32_t)table->size);                        \
-        uint64_t __check_value = table->free_bits[__b_idx];             \
-        __b_pos = 0;                                                    \
-                                                                        \
-        if( 0x00000000FFFFFFFFu == (__check_value & 0x00000000FFFFFFFFu) ) { \
-            __check_value >>= 32; __b_pos += 32;                        \
-        }                                                               \
-        if( 0x000000000000FFFFu == (__check_value & 0x000000000000FFFFu) ) { \
-            __check_value >>= 16; __b_pos += 16;                        \
-        }                                                               \
-        if( 0x00000000000000FFu == (__check_value & 0x00000000000000FFu) ) { \
-            __check_value >>= 8; __b_pos += 8;                          \
-        }                                                               \
-        if( 0x000000000000000Fu == (__check_value & 0x000000000000000Fu) ) { \
-            __check_value >>= 4; __b_pos += 4;                          \
-        }                                                               \
-        if( 0x0000000000000003u == (__check_value & 0x0000000000000003u) ) { \
-            __check_value >>= 2; __b_pos += 2;                          \
-        }                                                               \
-        if( 0x0000000000000001u == (__check_value & 0x0000000000000001u) ) { \
-            __b_pos += 1;                                               \
-        }                                                               \
-        (STORE) = (__b_idx * 8 * sizeof(uint64_t)) + __b_pos;           \
-    } while(0)
+#define FIND_FIRST_ZERO(START_IDX, STORE)                                   \
+    do {                                                                    \
+        uint32_t __b_idx, __b_pos;                                          \
+        if (0 == table->number_free) {                                      \
+            (STORE) = table->size;                                          \
+            break;                                                          \
+        }                                                                   \
+        GET_BIT_POS((START_IDX), __b_idx, __b_pos);                         \
+        for (; table->free_bits[__b_idx] == 0xFFFFFFFFFFFFFFFFu; __b_idx++) \
+            ;                                                               \
+        assert(__b_idx < (uint32_t) table->size);                           \
+        uint64_t __check_value = table->free_bits[__b_idx];                 \
+        __b_pos = 0;                                                        \
+                                                                            \
+        if (0x00000000FFFFFFFFu == (__check_value & 0x00000000FFFFFFFFu)) { \
+            __check_value >>= 32;                                           \
+            __b_pos += 32;                                                  \
+        }                                                                   \
+        if (0x000000000000FFFFu == (__check_value & 0x000000000000FFFFu)) { \
+            __check_value >>= 16;                                           \
+            __b_pos += 16;                                                  \
+        }                                                                   \
+        if (0x00000000000000FFu == (__check_value & 0x00000000000000FFu)) { \
+            __check_value >>= 8;                                            \
+            __b_pos += 8;                                                   \
+        }                                                                   \
+        if (0x000000000000000Fu == (__check_value & 0x000000000000000Fu)) { \
+            __check_value >>= 4;                                            \
+            __b_pos += 4;                                                   \
+        }                                                                   \
+        if (0x0000000000000003u == (__check_value & 0x0000000000000003u)) { \
+            __check_value >>= 2;                                            \
+            __b_pos += 2;                                                   \
+        }                                                                   \
+        if (0x0000000000000001u == (__check_value & 0x0000000000000001u)) { \
+            __b_pos += 1;                                                   \
+        }                                                                   \
+        (STORE) = (__b_idx * 8 * sizeof(uint64_t)) + __b_pos;               \
+    } while (0)
 
 /**
  * Set the IDX bit in the free_bits array. The bit should be previously unset.
  */
-#define SET_BIT(IDX)                                                    \
-    do {                                                                \
-        uint32_t __b_idx, __b_pos;                                      \
-        GET_BIT_POS((IDX), __b_idx, __b_pos);                           \
-        assert( 0 == (table->free_bits[__b_idx] & (((uint64_t)1) << __b_pos))); \
-        table->free_bits[__b_idx] |= (((uint64_t)1) << __b_pos);        \
-    } while(0)
+#define SET_BIT(IDX)                                                            \
+    do {                                                                        \
+        uint32_t __b_idx, __b_pos;                                              \
+        GET_BIT_POS((IDX), __b_idx, __b_pos);                                   \
+        assert(0 == (table->free_bits[__b_idx] & (((uint64_t) 1) << __b_pos))); \
+        table->free_bits[__b_idx] |= (((uint64_t) 1) << __b_pos);               \
+    } while (0)
 
 /**
  * Unset the IDX bit in the free_bits array. The bit should be previously set.
  */
-#define UNSET_BIT(IDX)                                                  \
-    do {                                                                \
-        uint32_t __b_idx, __b_pos;                                      \
-        GET_BIT_POS((IDX), __b_idx, __b_pos);                           \
-        assert( (table->free_bits[__b_idx] & (((uint64_t)1) << __b_pos))); \
-        table->free_bits[__b_idx] ^= (((uint64_t)1) << __b_pos);        \
-    } while(0)
+#define UNSET_BIT(IDX)                                                     \
+    do {                                                                   \
+        uint32_t __b_idx, __b_pos;                                         \
+        GET_BIT_POS((IDX), __b_idx, __b_pos);                              \
+        assert((table->free_bits[__b_idx] & (((uint64_t) 1) << __b_pos))); \
+        table->free_bits[__b_idx] ^= (((uint64_t) 1) << __b_pos);          \
+    } while (0)
 
 #if 0
 /**
@@ -175,9 +180,8 @@ static void opal_pointer_array_validate(opal_pointer_array_t *array)
 /**
  * initialize an array object
  */
-int opal_pointer_array_init(opal_pointer_array_t* array,
-                            int initial_allocation,
-                            int max_size, int block_size)
+int opal_pointer_array_init(opal_pointer_array_t *array, int initial_allocation, int max_size,
+                            int block_size)
 {
     size_t num_bytes;
 
@@ -193,12 +197,12 @@ int opal_pointer_array_init(opal_pointer_array_t* array,
     num_bytes = (0 < initial_allocation ? initial_allocation : block_size);
 
     /* Allocate and set the array to NULL */
-    array->addr = (void **)calloc(num_bytes, sizeof(void*));
+    array->addr = (void **) calloc(num_bytes, sizeof(void *));
     if (NULL == array->addr) { /* out of memory */
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
-    array->free_bits = (uint64_t*)calloc(TYPE_ELEM_COUNT(uint64_t, num_bytes), sizeof(uint64_t));
-    if (NULL == array->free_bits) {  /* out of memory */
+    array->free_bits = (uint64_t *) calloc(TYPE_ELEM_COUNT(uint64_t, num_bytes), sizeof(uint64_t));
+    if (NULL == array->free_bits) { /* out of memory */
         free(array->addr);
         array->addr = NULL;
         return OPAL_ERR_OUT_OF_RESOURCE;
@@ -225,15 +229,15 @@ int opal_pointer_array_add(opal_pointer_array_t *table, void *ptr)
 
     if (table->number_free == 0) {
         /* need to grow table */
-        if (!grow_table(table, index) ) {
+        if (!grow_table(table, index)) {
             OPAL_THREAD_UNLOCK(&(table->lock));
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
     }
 
-    assert( (table->addr != NULL) && (table->size > 0) );
-    assert( (table->lowest_free >= 0) && (table->lowest_free < table->size) );
-    assert( (table->number_free > 0) && (table->number_free <= table->size) );
+    assert((table->addr != NULL) && (table->size > 0));
+    assert((table->lowest_free >= 0) && (table->lowest_free < table->size));
+    assert((table->number_free > 0) && (table->number_free <= table->size));
 
     /*
      * add pointer to table, and return the index
@@ -268,8 +272,7 @@ int opal_pointer_array_add(opal_pointer_array_t *table, void *ptr)
  *
  * Assumption: NULL element is free element.
  */
-int opal_pointer_array_set_item(opal_pointer_array_t *table, int index,
-                                void * value)
+int opal_pointer_array_set_item(opal_pointer_array_t *table, int index, void *value)
 {
     assert(table != NULL);
 
@@ -288,8 +291,8 @@ int opal_pointer_array_set_item(opal_pointer_array_t *table, int index,
     }
     assert(table->size > index);
     /* mark element as free, if NULL element */
-    if( NULL == value ) {
-        if( NULL != table->addr[index] ) {
+    if (NULL == value) {
+        if (NULL != table->addr[index]) {
             if (index < table->lowest_free) {
                 table->lowest_free = index;
             }
@@ -301,11 +304,11 @@ int opal_pointer_array_set_item(opal_pointer_array_t *table, int index,
             table->number_free--;
             SET_BIT(index);
             /* Reset lowest_free if required */
-            if ( index == table->lowest_free ) {
+            if (index == table->lowest_free) {
                 FIND_FIRST_ZERO(index, table->lowest_free);
             }
         } else {
-            assert( index != table->lowest_free );
+            assert(index != table->lowest_free);
         }
     }
     table->addr[index] = value;
@@ -337,8 +340,7 @@ int opal_pointer_array_set_item(opal_pointer_array_t *table, int index,
  * In contrary to array_set, this function does not allow to overwrite
  * a value, unless the previous value is NULL ( equiv. to free ).
  */
-bool opal_pointer_array_test_and_set_item (opal_pointer_array_t *table,
-                                           int index, void *value)
+bool opal_pointer_array_test_and_set_item(opal_pointer_array_t *table, int index, void *value)
 {
     assert(table != NULL);
     assert(index >= 0);
@@ -353,7 +355,7 @@ bool opal_pointer_array_test_and_set_item (opal_pointer_array_t *table,
 
     /* expand table if required to set a specific index */
     OPAL_THREAD_LOCK(&(table->lock));
-    if ( index < table->size && table->addr[index] != NULL ) {
+    if (index < table->size && table->addr[index] != NULL) {
         /* This element is already in use */
         OPAL_THREAD_UNLOCK(&(table->lock));
         return false;
@@ -376,8 +378,8 @@ bool opal_pointer_array_test_and_set_item (opal_pointer_array_t *table,
     table->number_free--;
     SET_BIT(index);
     /* Reset lowest_free if required */
-    if( table->number_free > 0 ) {
-        if ( index == table->lowest_free ) {
+    if (table->number_free > 0) {
+        if (index == table->lowest_free) {
             FIND_FIRST_ZERO(index, table->lowest_free);
         }
     } else {
@@ -400,7 +402,7 @@ bool opal_pointer_array_test_and_set_item (opal_pointer_array_t *table,
 int opal_pointer_array_set_size(opal_pointer_array_t *array, int new_size)
 {
     OPAL_THREAD_LOCK(&(array->lock));
-    if(new_size > array->size) {
+    if (new_size > array->size) {
         if (!grow_table(array, new_size)) {
             OPAL_THREAD_UNLOCK(&(array->lock));
             return OPAL_ERROR;
@@ -416,9 +418,9 @@ static bool grow_table(opal_pointer_array_t *table, int at_least)
     void *p;
 
     new_size = table->block_size * ((at_least + 1 + table->block_size - 1) / table->block_size);
-    if( new_size >= table->max_size ) {
+    if (new_size >= table->max_size) {
         new_size = table->max_size;
-        if( at_least >= table->max_size ) {
+        if (at_least >= table->max_size) {
             return false;
         }
     }
@@ -429,19 +431,18 @@ static bool grow_table(opal_pointer_array_t *table, int at_least)
     }
 
     table->number_free += (new_size - table->size);
-    table->addr = (void**)p;
+    table->addr = (void **) p;
     for (i = table->size; i < new_size; ++i) {
         table->addr[i] = NULL;
     }
     new_size_int = TYPE_ELEM_COUNT(uint64_t, new_size);
-    if( (int)(TYPE_ELEM_COUNT(uint64_t, table->size)) != new_size_int ) {
-        p = (uint64_t*)realloc(table->free_bits, new_size_int * sizeof(uint64_t));
+    if ((int) (TYPE_ELEM_COUNT(uint64_t, table->size)) != new_size_int) {
+        p = (uint64_t *) realloc(table->free_bits, new_size_int * sizeof(uint64_t));
         if (NULL == p) {
             return false;
         }
-        table->free_bits = (uint64_t*)p;
-        for (i = TYPE_ELEM_COUNT(uint64_t, table->size);
-             i < new_size_int; i++ ) {
+        table->free_bits = (uint64_t *) p;
+        for (i = TYPE_ELEM_COUNT(uint64_t, table->size); i < new_size_int; i++) {
             table->free_bits[i] = 0;
         }
     }
diff --git a/opal/class/opal_pointer_array.h b/opal/class/opal_pointer_array.h
index ec6a883f5e4..b05e832d5fd 100644
--- a/opal/class/opal_pointer_array.h
+++ b/opal/class/opal_pointer_array.h
@@ -31,8 +31,8 @@
 
 #include "opal_config.h"
 
-#include "opal/mca/threads/mutex.h"
 #include "opal/class/opal_object.h"
+#include "opal/mca/threads/mutex.h"
 #include "opal/prefetch.h"
 
 BEGIN_C_DECLS
@@ -59,7 +59,7 @@ struct opal_pointer_array_t {
     /** block size for each allocation */
     int block_size;
     /** pointer to an array of bits to speed up the research for an empty position. */
-    uint64_t* free_bits;
+    uint64_t *free_bits;
     /** pointer to array of pointers */
     void **addr;
 };
@@ -83,12 +83,11 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_pointer_array_t);
  * @param max_size The maximum size of the array (IN)
  * @param block_size The size for all subsequent grows of the array (IN).
  *
- * @return OPAL_SUCCESS if all initializations were succesfull. Otherwise,
+ * @return OPAL_SUCCESS if all initializations were successful. Otherwise,
  *  the error indicate what went wrong in the function.
  */
-OPAL_DECLSPEC int opal_pointer_array_init( opal_pointer_array_t* array,
-                                           int initial_allocation,
-                                           int max_size, int block_size );
+OPAL_DECLSPEC int opal_pointer_array_init(opal_pointer_array_t *array, int initial_allocation,
+                                          int max_size, int block_size);
 
 /**
  * Add a pointer to the array (Grow the array, if need be)
@@ -110,8 +109,7 @@ OPAL_DECLSPEC int opal_pointer_array_add(opal_pointer_array_t *array, void *ptr)
  *
  * @return Error code.  (-1) indicates an error.
  */
-OPAL_DECLSPEC int opal_pointer_array_set_item(opal_pointer_array_t *array,
-                                int index, void *value);
+OPAL_DECLSPEC int opal_pointer_array_set_item(opal_pointer_array_t *array, int index, void *value);
 
 /**
  * Get the value of an element in array
@@ -122,12 +120,11 @@ OPAL_DECLSPEC int opal_pointer_array_set_item(opal_pointer_array_t *array,
  * @return Error code.  NULL indicates an error.
  */
 
-static inline void *opal_pointer_array_get_item(opal_pointer_array_t *table,
-                                                int element_index)
+static inline void *opal_pointer_array_get_item(opal_pointer_array_t *table, int element_index)
 {
     void *p;
 
-    if( OPAL_UNLIKELY(0 > element_index || table->size <= element_index) ) {
+    if (OPAL_UNLIKELY(0 > element_index || table->size <= element_index)) {
         return NULL;
     }
     OPAL_THREAD_LOCK(&(table->lock));
@@ -136,7 +133,6 @@ static inline void *opal_pointer_array_get_item(opal_pointer_array_t *table,
     return p;
 }
 
-
 /**
  * Get the size of the pointer array
  *
@@ -149,7 +145,7 @@ static inline void *opal_pointer_array_get_item(opal_pointer_array_t *table,
  */
 static inline int opal_pointer_array_get_size(opal_pointer_array_t *array)
 {
-  return array->size;
+    return array->size;
 }
 
 /**
@@ -178,9 +174,8 @@ OPAL_DECLSPEC int opal_pointer_array_set_size(opal_pointer_array_t *array, int s
  * In contrary to array_set, this function does not allow to overwrite
  * a value, unless the previous value is NULL ( equiv. to free ).
  */
-OPAL_DECLSPEC bool opal_pointer_array_test_and_set_item (opal_pointer_array_t *table,
-                                          int index,
-                                          void *value);
+OPAL_DECLSPEC bool opal_pointer_array_test_and_set_item(opal_pointer_array_t *table, int index,
+                                                        void *value);
 
 /**
  * Empty the array.
@@ -191,16 +186,17 @@ OPAL_DECLSPEC bool opal_pointer_array_test_and_set_item (opal_pointer_array_t *t
 static inline void opal_pointer_array_remove_all(opal_pointer_array_t *array)
 {
     int i;
-    if( array->number_free == array->size )
-        return;  /* nothing to do here this time (the array is already empty) */
+    if (array->number_free == array->size)
+        return; /* nothing to do here this time (the array is already empty) */
 
     OPAL_THREAD_LOCK(&array->lock);
     array->lowest_free = 0;
     array->number_free = array->size;
-    for(i = 0; i < array->size; i++) {
+    for (i = 0; i < array->size; i++) {
         array->addr[i] = NULL;
     }
-    for(i = 0; i < (int)((array->size + 8*sizeof(uint64_t) - 1) / (8*sizeof(uint64_t))); i++) {
+    for (i = 0; i < (int) ((array->size + 8 * sizeof(uint64_t) - 1) / (8 * sizeof(uint64_t)));
+         i++) {
         array->free_bits[i] = 0;
     }
     OPAL_THREAD_UNLOCK(&array->lock);
diff --git a/opal/class/opal_rb_tree.c b/opal/class/opal_rb_tree.c
index cdbaa187642..298b1035229 100644
--- a/opal/class/opal_rb_tree.c
+++ b/opal/class/opal_rb_tree.c
@@ -27,19 +27,15 @@
 #include "opal/class/opal_rb_tree.h"
 
 /* Private functions */
-static void btree_insert(opal_rb_tree_t *tree, opal_rb_tree_node_t * node);
-static void btree_delete_fixup(opal_rb_tree_t *tree, opal_rb_tree_node_t * x);
-static opal_rb_tree_node_t * btree_successor(opal_rb_tree_t * tree,
-                                             opal_rb_tree_node_t * node);
-static opal_rb_tree_node_t * opal_rb_tree_find_node(opal_rb_tree_t *tree, void *key);
-static void left_rotate(opal_rb_tree_t *tree, opal_rb_tree_node_t * x);
-static void right_rotate(opal_rb_tree_t *tree, opal_rb_tree_node_t * x);
-static void inorder_destroy(opal_rb_tree_t *tree, opal_rb_tree_node_t * node);
-static void inorder_traversal(opal_rb_tree_t *tree,
-                              opal_rb_tree_condition_fn_t cond,
-                              opal_rb_tree_action_fn_t action,
-                              opal_rb_tree_node_t * node);
-
+static void btree_insert(opal_rb_tree_t *tree, opal_rb_tree_node_t *node);
+static void btree_delete_fixup(opal_rb_tree_t *tree, opal_rb_tree_node_t *x);
+static opal_rb_tree_node_t *btree_successor(opal_rb_tree_t *tree, opal_rb_tree_node_t *node);
+static opal_rb_tree_node_t *opal_rb_tree_find_node(opal_rb_tree_t *tree, void *key);
+static void left_rotate(opal_rb_tree_t *tree, opal_rb_tree_node_t *x);
+static void right_rotate(opal_rb_tree_t *tree, opal_rb_tree_node_t *x);
+static void inorder_destroy(opal_rb_tree_t *tree, opal_rb_tree_node_t *node);
+static void inorder_traversal(opal_rb_tree_t *tree, opal_rb_tree_condition_fn_t cond,
+                              opal_rb_tree_action_fn_t action, opal_rb_tree_node_t *node);
 
 /**
  * the constructor function. creates the free list to get the nodes from
@@ -48,14 +44,14 @@ static void inorder_traversal(opal_rb_tree_t *tree,
  *
  * @retval NONE
  */
-static void opal_rb_tree_construct(opal_object_t * object)
+static void opal_rb_tree_construct(opal_object_t *object)
 {
-    opal_rb_tree_t * tree = (opal_rb_tree_t *) object;
+    opal_rb_tree_t *tree = (opal_rb_tree_t *) object;
     tree->root_ptr = NULL;
     OBJ_CONSTRUCT(&(tree->free_list), opal_free_list_t);
-    opal_free_list_init (&(tree->free_list), sizeof(opal_rb_tree_node_t),
-            opal_cache_line_size, OBJ_CLASS(opal_rb_tree_node_t),
-            0,opal_cache_line_size, 0, -1 , 128, NULL, 0, NULL, NULL, NULL);
+    opal_free_list_init(&(tree->free_list), sizeof(opal_rb_tree_node_t), opal_cache_line_size,
+                        OBJ_CLASS(opal_rb_tree_node_t), 0, opal_cache_line_size, 0, -1, 128, NULL,
+                        0, NULL, NULL, NULL);
 }
 
 /**
@@ -63,35 +59,33 @@ static void opal_rb_tree_construct(opal_object_t * object)
  *
  * @param object the tree object
  */
-static void opal_rb_tree_destruct(opal_object_t * object)
+static void opal_rb_tree_destruct(opal_object_t *object)
 {
-    if(NULL != ((opal_rb_tree_t *)object)->root_ptr) {
+    if (NULL != ((opal_rb_tree_t *) object)->root_ptr) {
         opal_rb_tree_destroy((opal_rb_tree_t *) object);
     }
-    OBJ_DESTRUCT(&(((opal_rb_tree_t *)object)->free_list));
+    OBJ_DESTRUCT(&(((opal_rb_tree_t *) object)->free_list));
     return;
 }
 
 /* declare the instance of the classes  */
 OBJ_CLASS_INSTANCE(opal_rb_tree_node_t, opal_free_list_item_t, NULL, NULL);
-OBJ_CLASS_INSTANCE(opal_rb_tree_t, opal_object_t, opal_rb_tree_construct,
-                   opal_rb_tree_destruct);
+OBJ_CLASS_INSTANCE(opal_rb_tree_t, opal_object_t, opal_rb_tree_construct, opal_rb_tree_destruct);
 
 /* Create the tree */
-int opal_rb_tree_init(opal_rb_tree_t * tree,
-                      opal_rb_tree_comp_fn_t comp)
+int opal_rb_tree_init(opal_rb_tree_t *tree, opal_rb_tree_comp_fn_t comp)
 {
-    opal_free_list_item_t * node;
+    opal_free_list_item_t *node;
     /* we need to get memory for the root pointer from the free list */
-    node = opal_free_list_get (&(tree->free_list));
+    node = opal_free_list_get(&(tree->free_list));
     tree->root_ptr = (opal_rb_tree_node_t *) node;
     if (NULL == node) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
-    node = opal_free_list_get (&(tree->free_list));
+    node = opal_free_list_get(&(tree->free_list));
     if (NULL == node) {
-        opal_free_list_return (&tree->free_list, (opal_free_list_item_t*)tree->root_ptr);
+        opal_free_list_return(&tree->free_list, (opal_free_list_item_t *) tree->root_ptr);
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
     tree->nill = (opal_rb_tree_node_t *) node;
@@ -115,16 +109,15 @@ int opal_rb_tree_init(opal_rb_tree_t * tree,
     return OPAL_SUCCESS;
 }
 
-
 /* This inserts a node into the tree based on the passed values. */
-int opal_rb_tree_insert(opal_rb_tree_t *tree, void * key, void * value)
+int opal_rb_tree_insert(opal_rb_tree_t *tree, void *key, void *value)
 {
-    opal_rb_tree_node_t * y;
-    opal_rb_tree_node_t * node;
-    opal_free_list_item_t * item;
+    opal_rb_tree_node_t *y;
+    opal_rb_tree_node_t *node;
+    opal_free_list_item_t *item;
 
     /* get the memory for a node */
-    item = opal_free_list_get (&tree->free_list);
+    item = opal_free_list_get(&tree->free_list);
     if (NULL == item) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
@@ -180,10 +173,9 @@ int opal_rb_tree_insert(opal_rb_tree_t *tree, void * key, void * value)
 }
 
 /* Finds the node in the tree based on the key */
-void * opal_rb_tree_find_with(opal_rb_tree_t *tree, void *key,
-        opal_rb_tree_comp_fn_t compfn)
+void *opal_rb_tree_find_with(opal_rb_tree_t *tree, void *key, opal_rb_tree_comp_fn_t compfn)
 {
-    opal_rb_tree_node_t * node;
+    opal_rb_tree_node_t *node;
     int compvalue;
 
     node = tree->root_ptr->left;
@@ -203,9 +195,9 @@ void * opal_rb_tree_find_with(opal_rb_tree_t *tree, void *key,
 /* Finds the node in the tree based on the key and returns a pointer
  * to the node. This is a bit a code duplication, but this has to be fast
  * so we go ahead with the duplication */
-static opal_rb_tree_node_t * opal_rb_tree_find_node(opal_rb_tree_t *tree, void *key)
+static opal_rb_tree_node_t *opal_rb_tree_find_node(opal_rb_tree_t *tree, void *key)
 {
-    opal_rb_tree_node_t * node;
+    opal_rb_tree_node_t *node;
     int compvalue;
 
     node = tree->root_ptr->left;
@@ -225,10 +217,10 @@ static opal_rb_tree_node_t * opal_rb_tree_find_node(opal_rb_tree_t *tree, void *
 /* Delete a node from the tree based on the key */
 int opal_rb_tree_delete(opal_rb_tree_t *tree, void *key)
 {
-    opal_rb_tree_node_t * p;
-    opal_rb_tree_node_t * todelete;
-    opal_rb_tree_node_t * y;
-    opal_free_list_item_t * item;
+    opal_rb_tree_node_t *p;
+    opal_rb_tree_node_t *todelete;
+    opal_rb_tree_node_t *y;
+    opal_free_list_item_t *item;
 
     p = opal_rb_tree_find_node(tree, key);
     if (NULL == p) {
@@ -252,7 +244,7 @@ int opal_rb_tree_delete(opal_rb_tree_t *tree, void *key)
         tree->root_ptr->left = y;
     } else {
         if (todelete == todelete->parent->left) {
-         todelete->parent->left = y;
+            todelete->parent->left = y;
         } else {
             todelete->parent->right = y;
         }
@@ -267,16 +259,15 @@ int opal_rb_tree_delete(opal_rb_tree_t *tree, void *key)
         btree_delete_fixup(tree, y);
     }
     item = (opal_free_list_item_t *) todelete;
-    opal_free_list_return (&(tree->free_list), item);
+    opal_free_list_return(&(tree->free_list), item);
     --tree->tree_size;
     return OPAL_SUCCESS;
 }
 
-
 /* Destroy the hashmap    */
 int opal_rb_tree_destroy(opal_rb_tree_t *tree)
 {
-    opal_free_list_item_t * item;
+    opal_free_list_item_t *item;
     /* Recursive inorder traversal for delete    */
 
     inorder_destroy(tree, tree->root_ptr);
@@ -287,16 +278,15 @@ int opal_rb_tree_destroy(opal_rb_tree_t *tree)
 
     /* free the tree->nill node */
     item = (opal_free_list_item_t *) tree->nill;
-    opal_free_list_return (&(tree->free_list), item);
+    opal_free_list_return(&(tree->free_list), item);
     return OPAL_SUCCESS;
 }
 
-
 /* Find the next inorder successor of a node    */
 
-static opal_rb_tree_node_t * btree_successor(opal_rb_tree_t * tree, opal_rb_tree_node_t * node)
+static opal_rb_tree_node_t *btree_successor(opal_rb_tree_t *tree, opal_rb_tree_node_t *node)
 {
-    opal_rb_tree_node_t * p;
+    opal_rb_tree_node_t *p;
 
     if (node->right == tree->nill) {
         p = node->parent;
@@ -304,27 +294,26 @@ static opal_rb_tree_node_t * btree_successor(opal_rb_tree_t * tree, opal_rb_tree
             node = p;
             p = p->parent;
         }
-        if(p == tree->root_ptr) {
+        if (p == tree->root_ptr) {
             return tree->nill;
         }
         return p;
     }
 
     p = node->right;
-    while(p->left != tree->nill) {
+    while (p->left != tree->nill) {
         p = p->left;
     }
     return p;
 }
 
-
 /* Insert an element in the normal binary search tree fashion    */
 /* this function goes through the tree and finds the leaf where
  * the node will be inserted   */
-static void btree_insert(opal_rb_tree_t *tree, opal_rb_tree_node_t * node)
+static void btree_insert(opal_rb_tree_t *tree, opal_rb_tree_node_t *node)
 {
-    opal_rb_tree_node_t * parent = tree->root_ptr;
-    opal_rb_tree_node_t * n = parent->left; /* the real root of the tree */
+    opal_rb_tree_node_t *parent = tree->root_ptr;
+    opal_rb_tree_node_t *n = parent->left; /* the real root of the tree */
 
     /* set up initial values for the node */
     node->color = RED;
@@ -339,7 +328,7 @@ static void btree_insert(opal_rb_tree_t *tree, opal_rb_tree_node_t * node)
     }
 
     /* place it on either the left or the right */
-    if((parent == tree->root_ptr) || (tree->comp(node->key, parent->key) <= 0)) {
+    if ((parent == tree->root_ptr) || (tree->comp(node->key, parent->key) <= 0)) {
         parent->left = node;
     } else {
         parent->right = node;
@@ -354,10 +343,10 @@ static void btree_insert(opal_rb_tree_t *tree, opal_rb_tree_node_t * node)
 }
 
 /* Fixup the balance of the btree after deletion    */
-static void btree_delete_fixup(opal_rb_tree_t *tree, opal_rb_tree_node_t * x)
+static void btree_delete_fixup(opal_rb_tree_t *tree, opal_rb_tree_node_t *x)
 {
-    opal_rb_tree_node_t * w;
-    opal_rb_tree_node_t * root = tree->root_ptr->left;
+    opal_rb_tree_node_t *w;
+    opal_rb_tree_node_t *root = tree->root_ptr->left;
     while ((x != root) && (x->color == BLACK)) {
         if (x == x->parent->left) {
             w = x->parent->right;
@@ -415,13 +404,11 @@ static void btree_delete_fixup(opal_rb_tree_t *tree, opal_rb_tree_node_t * x)
     return;
 }
 
-
 /* Free the nodes in inorder fashion    */
 
-static void
-inorder_destroy(opal_rb_tree_t *tree, opal_rb_tree_node_t * node)
+static void inorder_destroy(opal_rb_tree_t *tree, opal_rb_tree_node_t *node)
 {
-    opal_free_list_item_t * item;
+    opal_free_list_item_t *item;
 
     if (node == tree->nill) {
         return;
@@ -432,21 +419,20 @@ inorder_destroy(opal_rb_tree_t *tree, opal_rb_tree_node_t * node)
     if (node->left != tree->nill) {
         item = (opal_free_list_item_t *) node->left;
         --tree->tree_size;
-        opal_free_list_return (&tree->free_list, item);
+        opal_free_list_return(&tree->free_list, item);
     }
 
     inorder_destroy(tree, node->right);
     if (node->right != tree->nill) {
         item = (opal_free_list_item_t *) node->right;
         --tree->tree_size;
-        opal_free_list_return (&tree->free_list, item);
+        opal_free_list_return(&tree->free_list, item);
     }
 }
 
 /* Try to access all the elements of the hashmap conditionally */
 
-int opal_rb_tree_traverse(opal_rb_tree_t *tree,
-                          opal_rb_tree_condition_fn_t cond,
+int opal_rb_tree_traverse(opal_rb_tree_t *tree, opal_rb_tree_condition_fn_t cond,
                           opal_rb_tree_action_fn_t action)
 {
     if ((cond == NULL) || (action == NULL)) {
@@ -458,11 +444,8 @@ int opal_rb_tree_traverse(opal_rb_tree_t *tree,
     return OPAL_SUCCESS;
 }
 
-
-static void inorder_traversal(opal_rb_tree_t *tree,
-                              opal_rb_tree_condition_fn_t cond,
-                              opal_rb_tree_action_fn_t action,
-                              opal_rb_tree_node_t * node)
+static void inorder_traversal(opal_rb_tree_t *tree, opal_rb_tree_condition_fn_t cond,
+                              opal_rb_tree_action_fn_t action, opal_rb_tree_node_t *node)
 {
     if (node == tree->nill) {
         return;
@@ -480,17 +463,17 @@ static void inorder_traversal(opal_rb_tree_t *tree,
 /* Left rotate the tree    */
 /* basically what we want to do is to make x be the left child
  * of its right child    */
-static void left_rotate(opal_rb_tree_t *tree, opal_rb_tree_node_t * x)
+static void left_rotate(opal_rb_tree_t *tree, opal_rb_tree_node_t *x)
 {
-    opal_rb_tree_node_t * y;
+    opal_rb_tree_node_t *y;
 
     y = x->right;
     /* make the left child of y's parent be x if it is not the sentinal node*/
     if (y->left != tree->nill) {
-        y->left->parent=x;
+        y->left->parent = x;
     }
 
-    /* normlly we would have to check to see if we are at the root.
+    /* normally we would have to check to see if we are at the root.
      * however, the root sentinal takes care of it for us */
     if (x == x->parent->left) {
         x->parent->left = y;
@@ -507,17 +490,16 @@ static void left_rotate(opal_rb_tree_t *tree, opal_rb_tree_node_t * x)
     return;
 }
 
-
 /* Right rotate the tree    */
 /* basically what we want to do is to make x be the right child
  * of its left child */
-static void right_rotate(opal_rb_tree_t *tree, opal_rb_tree_node_t * x)
+static void right_rotate(opal_rb_tree_t *tree, opal_rb_tree_node_t *x)
 {
-    opal_rb_tree_node_t * y;
+    opal_rb_tree_node_t *y;
 
     y = x->left;
 
-    if(y->right != tree->nill) {
+    if (y->right != tree->nill) {
         y->right->parent = x;
     }
 
@@ -535,16 +517,15 @@ static void right_rotate(opal_rb_tree_t *tree, opal_rb_tree_node_t * x)
     return;
 }
 
-
 /* returns the size of the tree */
 int opal_rb_tree_size(opal_rb_tree_t *tree)
 {
-        return tree->tree_size;
+    return tree->tree_size;
 }
 
 /* below are a couple of debugging functions */
 #if 0
-#include <stdio.h>
+#    include <stdio.h>
 static void inorder(opal_rb_tree_t * tree, opal_rb_tree_node_t * node);
 static void print_inorder(opal_rb_tree_t * tree);
 
diff --git a/opal/class/opal_rb_tree.h b/opal/class/opal_rb_tree.h
index c405904b699..9e042818d5c 100644
--- a/opal/class/opal_rb_tree.h
+++ b/opal/class/opal_rb_tree.h
@@ -29,10 +29,10 @@
 #define OPAL_RB_TREE_H
 
 #include "opal_config.h"
-#include <stdlib.h>
-#include "opal/constants.h"
-#include "opal/class/opal_object.h"
 #include "opal/class/opal_free_list.h"
+#include "opal/class/opal_object.h"
+#include "opal/constants.h"
+#include <stdlib.h>
 
 BEGIN_C_DECLS
 /*
@@ -40,43 +40,42 @@ BEGIN_C_DECLS
  */
 
 /**
-  * red and black enum
-  */
-typedef enum {RED, BLACK} opal_rb_tree_nodecolor_t;
+ * red and black enum
+ */
+typedef enum { RED, BLACK } opal_rb_tree_nodecolor_t;
 
 /**
-  * node data structure
-  */
-struct opal_rb_tree_node_t
-{
+ * node data structure
+ */
+struct opal_rb_tree_node_t {
     opal_free_list_item_t super;        /**< the parent class */
     opal_rb_tree_nodecolor_t color;     /**< the node color */
-    struct opal_rb_tree_node_t * parent;/**< the parent node, can be NULL */
-    struct opal_rb_tree_node_t * left;  /**< the left child - can be nill */
-    struct opal_rb_tree_node_t * right; /**< the right child - can be nill */
+    struct opal_rb_tree_node_t *parent; /**< the parent node, can be NULL */
+    struct opal_rb_tree_node_t *left;   /**< the left child - can be nill */
+    struct opal_rb_tree_node_t *right;  /**< the right child - can be nill */
     void *key;                          /**< a pointer to the key */
     void *value;                        /**< a pointer to the value */
 };
 typedef struct opal_rb_tree_node_t opal_rb_tree_node_t;
 
 /**
-  * the compare function typedef. This function is used to compare 2 nodes.
-  */
+ * the compare function typedef. This function is used to compare 2 nodes.
+ */
 typedef int (*opal_rb_tree_comp_fn_t)(void *key1, void *key2);
 
 /**
-  * the data structure that holds all the needed information about the tree.
-  */
+ * the data structure that holds all the needed information about the tree.
+ */
 struct opal_rb_tree_t {
-    opal_object_t parent;           /**< the parent class */
+    opal_object_t parent; /**< the parent class */
     /* this root pointer doesn't actually point to the root of the tree.
      * rather, it points to a sentinal node who's left branch is the real
      * root of the tree. This is done to eliminate special cases */
-    opal_rb_tree_node_t * root_ptr; /**< a pointer to the root of the tree */
-    opal_rb_tree_node_t * nill;     /**< the nill sentinal node */
-    opal_rb_tree_comp_fn_t comp;    /**< the compare function */
-    opal_free_list_t free_list;   /**< the free list to get the memory from */
-    size_t tree_size;                  /**< the size of the tree */
+    opal_rb_tree_node_t *root_ptr; /**< a pointer to the root of the tree */
+    opal_rb_tree_node_t *nill;     /**< the nill sentinal node */
+    opal_rb_tree_comp_fn_t comp;   /**< the compare function */
+    opal_free_list_t free_list;    /**< the free list to get the memory from */
+    size_t tree_size;              /**< the size of the tree */
 };
 typedef struct opal_rb_tree_t opal_rb_tree_t;
 
@@ -87,10 +86,10 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_rb_tree_t);
 
 /* Function pointers for map traversal function */
 /**
-  * this function is used for the opal_rb_tree_traverse function.
-  * it is passed a pointer to the value for each node and, if it returns
-  * a one, the action function is called on that node. Otherwise, the node is ignored.
-  */
+ * this function is used for the opal_rb_tree_traverse function.
+ * it is passed a pointer to the value for each node and, if it returns
+ * a one, the action function is called on that node. Otherwise, the node is ignored.
+ */
 typedef int (*opal_rb_tree_condition_fn_t)(void *);
 /**
  * this function is used for the user to perform any action on the passed
@@ -101,105 +100,103 @@ typedef int (*opal_rb_tree_condition_fn_t)(void *);
 typedef void (*opal_rb_tree_action_fn_t)(void *, void *);
 
 /*
- * Public function protoypes
+ * Public function prototypes
  */
 
 /**
-  * the function creates a new tree
-  *
-  * @param tree a pointer to an allocated area of memory for the main
-  *  tree data structure.
-  * @param comp a pointer to the function to use for comaparing 2 nodes
-  *
-  * @retval OPAL_SUCCESS if it is successful
-  * @retval OPAL_ERR_TEMP_OUT_OF_RESOURCE if unsuccessful
-  */
-OPAL_DECLSPEC int opal_rb_tree_init(opal_rb_tree_t * tree, opal_rb_tree_comp_fn_t comp);
-
+ * the function creates a new tree
+ *
+ * @param tree a pointer to an allocated area of memory for the main
+ *  tree data structure.
+ * @param comp a pointer to the function to use for comaparing 2 nodes
+ *
+ * @retval OPAL_SUCCESS if it is successful
+ * @retval OPAL_ERR_TEMP_OUT_OF_RESOURCE if unsuccessful
+ */
+OPAL_DECLSPEC int opal_rb_tree_init(opal_rb_tree_t *tree, opal_rb_tree_comp_fn_t comp);
 
 /**
-  * inserts a node into the tree
-  *
-  * @param tree a pointer to the tree data structure
-  * @param key the key for the node
-  * @param value the value for the node
-  *
-  * @retval OPAL_SUCCESS
-  * @retval OPAL_ERR_TEMP_OUT_OF_RESOURCE if unsuccessful
-  */
-OPAL_DECLSPEC int opal_rb_tree_insert(opal_rb_tree_t *tree, void * key, void * value);
+ * inserts a node into the tree
+ *
+ * @param tree a pointer to the tree data structure
+ * @param key the key for the node
+ * @param value the value for the node
+ *
+ * @retval OPAL_SUCCESS
+ * @retval OPAL_ERR_TEMP_OUT_OF_RESOURCE if unsuccessful
+ */
+OPAL_DECLSPEC int opal_rb_tree_insert(opal_rb_tree_t *tree, void *key, void *value);
 
 /**
-  * finds a value in the tree based on the passed key using passed
-  * compare function
-  *
-  * @param tree a pointer to the tree data structure
-  * @param key a pointer to the key
-  * @param compare function
-  *
-  * @retval pointer to the value if found
-  * @retval NULL if not found
-  */
-OPAL_DECLSPEC void * opal_rb_tree_find_with(opal_rb_tree_t *tree, void *key, opal_rb_tree_comp_fn_t compfn);
+ * finds a value in the tree based on the passed key using passed
+ * compare function
+ *
+ * @param tree a pointer to the tree data structure
+ * @param key a pointer to the key
+ * @param compare function
+ *
+ * @retval pointer to the value if found
+ * @retval NULL if not found
+ */
+OPAL_DECLSPEC void *opal_rb_tree_find_with(opal_rb_tree_t *tree, void *key,
+                                           opal_rb_tree_comp_fn_t compfn);
 
 /**
-  * finds a value in the tree based on the passed key
-  *
-  * @param tree a pointer to the tree data structure
-  * @param key a pointer to the key
-  *
-  * @retval pointer to the value if found
-  * @retval NULL if not found
-  */
-static inline void * opal_rb_tree_find(opal_rb_tree_t *tree, void *key)
+ * finds a value in the tree based on the passed key
+ *
+ * @param tree a pointer to the tree data structure
+ * @param key a pointer to the key
+ *
+ * @retval pointer to the value if found
+ * @retval NULL if not found
+ */
+static inline void *opal_rb_tree_find(opal_rb_tree_t *tree, void *key)
 {
     return opal_rb_tree_find_with(tree, key, tree->comp);
 }
 
 /**
-  * deletes a node based on its key
-  *
-  * @param tree a pointer to the tree data structure
-  * @param key a pointer to the key
-  *
-  * @retval OPAL_SUCCESS if the node is found and deleted
-  * @retval OPAL_ERR_NOT_FOUND if the node is not found
-  */
+ * deletes a node based on its key
+ *
+ * @param tree a pointer to the tree data structure
+ * @param key a pointer to the key
+ *
+ * @retval OPAL_SUCCESS if the node is found and deleted
+ * @retval OPAL_ERR_NOT_FOUND if the node is not found
+ */
 OPAL_DECLSPEC int opal_rb_tree_delete(opal_rb_tree_t *tree, void *key);
 
 /**
-  * frees all the nodes on the tree
-  *
-  * @param tree a pointer to the tree data structure
-  *
-  * @retval OPAL_SUCCESS
-  */
+ * frees all the nodes on the tree
+ *
+ * @param tree a pointer to the tree data structure
+ *
+ * @retval OPAL_SUCCESS
+ */
 OPAL_DECLSPEC int opal_rb_tree_destroy(opal_rb_tree_t *tree);
 
 /**
-  * traverses the entire tree, performing the cond function on each of the
-  * values and if it returns one it performs the action function on the values
-  *
-  * @param tree a pointer to the tree
-  * @param cond a pointer to the condition function
-  * @param action a pointer to the action function
-  *
-  * @retval OPAL_SUCCESS
-  * @retval OPAL_ERROR if there is an error
-  */
-OPAL_DECLSPEC int opal_rb_tree_traverse(opal_rb_tree_t *tree,
-                          opal_rb_tree_condition_fn_t cond,
-                          opal_rb_tree_action_fn_t action);
+ * traverses the entire tree, performing the cond function on each of the
+ * values and if it returns one it performs the action function on the values
+ *
+ * @param tree a pointer to the tree
+ * @param cond a pointer to the condition function
+ * @param action a pointer to the action function
+ *
+ * @retval OPAL_SUCCESS
+ * @retval OPAL_ERROR if there is an error
+ */
+OPAL_DECLSPEC int opal_rb_tree_traverse(opal_rb_tree_t *tree, opal_rb_tree_condition_fn_t cond,
+                                        opal_rb_tree_action_fn_t action);
 
 /**
-  * returns the size of the tree
-  *
-  * @param tree a pointer to the tree data structure
-  *
-  * @retval int the nuber of items on the tree
-  */
+ * returns the size of the tree
+ *
+ * @param tree a pointer to the tree data structure
+ *
+ * @retval int the number of items on the tree
+ */
 OPAL_DECLSPEC int opal_rb_tree_size(opal_rb_tree_t *tree);
 
 END_C_DECLS
 #endif /* OPAL_RB_TREE_H */
-
diff --git a/opal/class/opal_ring_buffer.c b/opal/class/opal_ring_buffer.c
index e06b00eb889..60eed931562 100644
--- a/opal/class/opal_ring_buffer.c
+++ b/opal/class/opal_ring_buffer.c
@@ -20,19 +20,18 @@
 
 #include "opal_config.h"
 
-#include <stdlib.h>
-#include <stdio.h>
 #include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
 
-#include "opal/constants.h"
 #include "opal/class/opal_ring_buffer.h"
+#include "opal/constants.h"
 #include "opal/util/output.h"
 
 static void opal_ring_buffer_construct(opal_ring_buffer_t *);
 static void opal_ring_buffer_destruct(opal_ring_buffer_t *);
 
-OBJ_CLASS_INSTANCE(opal_ring_buffer_t, opal_object_t,
-                   opal_ring_buffer_construct,
+OBJ_CLASS_INSTANCE(opal_ring_buffer_t, opal_object_t, opal_ring_buffer_construct,
                    opal_ring_buffer_destruct);
 
 /*
@@ -54,7 +53,7 @@ static void opal_ring_buffer_construct(opal_ring_buffer_t *ring)
  */
 static void opal_ring_buffer_destruct(opal_ring_buffer_t *ring)
 {
-    if( NULL != ring->addr) {
+    if (NULL != ring->addr) {
         free(ring->addr);
         ring->addr = NULL;
     }
@@ -68,7 +67,7 @@ static void opal_ring_buffer_destruct(opal_ring_buffer_t *ring)
 /**
  * initialize a ring object
  */
-int opal_ring_buffer_init(opal_ring_buffer_t* ring, int size)
+int opal_ring_buffer_init(opal_ring_buffer_t *ring, int size)
 {
     /* check for errors */
     if (NULL == ring) {
@@ -76,7 +75,7 @@ int opal_ring_buffer_init(opal_ring_buffer_t* ring, int size)
     }
 
     /* Allocate and set the ring to NULL */
-    ring->addr = (char **)calloc(size * sizeof(char*), 1);
+    ring->addr = (char **) calloc(size * sizeof(char *), 1);
     if (NULL == ring->addr) { /* out of memory */
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
@@ -85,20 +84,20 @@ int opal_ring_buffer_init(opal_ring_buffer_t* ring, int size)
     return OPAL_SUCCESS;
 }
 
-void* opal_ring_buffer_push(opal_ring_buffer_t *ring, void *ptr)
+void *opal_ring_buffer_push(opal_ring_buffer_t *ring, void *ptr)
 {
-    char *p=NULL;
+    char *p = NULL;
 
     OPAL_ACQUIRE_THREAD(&(ring->lock), &(ring->cond), &(ring->in_use));
     if (NULL != ring->addr[ring->head]) {
-        p = (char*)ring->addr[ring->head];
+        p = (char *) ring->addr[ring->head];
         if (ring->tail == ring->size - 1) {
             ring->tail = 0;
         } else {
             ring->tail = ring->head + 1;
         }
     }
-    ring->addr[ring->head] = (char*)ptr;
+    ring->addr[ring->head] = (char *) ptr;
     if (ring->tail < 0) {
         ring->tail = ring->head;
     }
@@ -108,21 +107,21 @@ void* opal_ring_buffer_push(opal_ring_buffer_t *ring, void *ptr)
         ring->head++;
     }
     OPAL_RELEASE_THREAD(&(ring->lock), &(ring->cond), &(ring->in_use));
-    return (void*)p;
+    return (void *) p;
 }
 
-void* opal_ring_buffer_pop(opal_ring_buffer_t *ring)
+void *opal_ring_buffer_pop(opal_ring_buffer_t *ring)
 {
-    char *p=NULL;
+    char *p = NULL;
 
     OPAL_ACQUIRE_THREAD(&(ring->lock), &(ring->cond), &(ring->in_use));
     if (-1 == ring->tail) {
         /* nothing has been put on the ring yet */
         p = NULL;
     } else {
-        p = (char*)ring->addr[ring->tail];
+        p = (char *) ring->addr[ring->tail];
         ring->addr[ring->tail] = NULL;
-        if (ring->tail == ring->size-1) {
+        if (ring->tail == ring->size - 1) {
             ring->tail = 0;
         } else {
             ring->tail++;
@@ -133,12 +132,12 @@ void* opal_ring_buffer_pop(opal_ring_buffer_t *ring)
         }
     }
     OPAL_RELEASE_THREAD(&(ring->lock), &(ring->cond), &(ring->in_use));
-    return (void*)p;
+    return (void *) p;
 }
 
- void* opal_ring_buffer_poke(opal_ring_buffer_t *ring, int i)
- {
-    char *p=NULL;
+void *opal_ring_buffer_poke(opal_ring_buffer_t *ring, int i)
+{
+    char *p = NULL;
     int offset;
 
     OPAL_ACQUIRE_THREAD(&(ring->lock), &(ring->cond), &(ring->in_use));
@@ -161,5 +160,5 @@ void* opal_ring_buffer_pop(opal_ring_buffer_t *ring)
         p = ring->addr[offset];
     }
     OPAL_RELEASE_THREAD(&(ring->lock), &(ring->cond), &(ring->in_use));
-    return (void*)p;
+    return (void *) p;
 }
diff --git a/opal/class/opal_ring_buffer.h b/opal/class/opal_ring_buffer.h
index ed607709dfd..8f2ff58b739 100644
--- a/opal/class/opal_ring_buffer.h
+++ b/opal/class/opal_ring_buffer.h
@@ -26,8 +26,8 @@
 
 #include "opal_config.h"
 
-#include "opal/mca/threads/threads.h"
 #include "opal/class/opal_object.h"
+#include "opal/mca/threads/threads.h"
 #include "opal/util/output.h"
 
 BEGIN_C_DECLS
@@ -65,10 +65,10 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_ring_buffer_t);
  * @param ring Pointer to a ring buffer (IN/OUT)
  * @param size The number of elements in the ring (IN)
  *
- * @return OPAL_SUCCESS if all initializations were succesful. Otherwise,
+ * @return OPAL_SUCCESS if all initializations were successful. Otherwise,
  *  the error indicate what went wrong in the function.
  */
-OPAL_DECLSPEC int opal_ring_buffer_init(opal_ring_buffer_t* ring, int size);
+OPAL_DECLSPEC int opal_ring_buffer_init(opal_ring_buffer_t *ring, int size);
 
 /**
  * Push an item onto the ring buffer
@@ -79,8 +79,7 @@ OPAL_DECLSPEC int opal_ring_buffer_init(opal_ring_buffer_t* ring, int size);
  * @return OPAL_SUCCESS. Returns error if ring cannot take
  *  another entry
  */
-OPAL_DECLSPEC void* opal_ring_buffer_push(opal_ring_buffer_t *ring, void *ptr);
-
+OPAL_DECLSPEC void *opal_ring_buffer_push(opal_ring_buffer_t *ring, void *ptr);
 
 /**
  * Pop an item off of the ring. The oldest entry on the ring will be
@@ -91,14 +90,14 @@ OPAL_DECLSPEC void* opal_ring_buffer_push(opal_ring_buffer_t *ring, void *ptr);
  * @return Error code.  NULL indicates an error.
  */
 
-OPAL_DECLSPEC void* opal_ring_buffer_pop(opal_ring_buffer_t *ring);
+OPAL_DECLSPEC void *opal_ring_buffer_pop(opal_ring_buffer_t *ring);
 
 /*
  * Access an element of the ring, without removing it, indexed
  * starting at the tail - a value of -1 will return the element
  * at the head of the ring
  */
-OPAL_DECLSPEC void* opal_ring_buffer_poke(opal_ring_buffer_t *ring, int i);
+OPAL_DECLSPEC void *opal_ring_buffer_poke(opal_ring_buffer_t *ring, int i);
 
 END_C_DECLS
 
diff --git a/opal/class/opal_value_array.c b/opal/class/opal_value_array.c
index a9615dc1cba..18936b1fdf7 100644
--- a/opal/class/opal_value_array.c
+++ b/opal/class/opal_value_array.c
@@ -20,8 +20,7 @@
 
 #include "opal/class/opal_value_array.h"
 
-
-static void opal_value_array_construct(opal_value_array_t* array)
+static void opal_value_array_construct(opal_value_array_t *array)
 {
     array->array_items = NULL;
     array->array_size = 0;
@@ -29,38 +28,36 @@ static void opal_value_array_construct(opal_value_array_t* array)
     array->array_alloc_size = 0;
 }
 
-static void opal_value_array_destruct(opal_value_array_t* array)
+static void opal_value_array_destruct(opal_value_array_t *array)
 {
-    if (NULL != array->array_items)
+    if (NULL != array->array_items) {
         free(array->array_items);
+    }
 }
 
-OBJ_CLASS_INSTANCE(
-    opal_value_array_t,
-    opal_object_t,
-    opal_value_array_construct,
-    opal_value_array_destruct
-);
+OBJ_CLASS_INSTANCE(opal_value_array_t, opal_object_t, opal_value_array_construct,
+                   opal_value_array_destruct);
 
-
-int opal_value_array_set_size(opal_value_array_t* array, size_t size)
+int opal_value_array_set_size(opal_value_array_t *array, size_t size)
 {
 #if OPAL_ENABLE_DEBUG
-    if(array->array_item_sizeof == 0) {
+    if (array->array_item_sizeof == 0) {
         opal_output(0, "opal_value_array_set_size: item size must be initialized");
         return OPAL_ERR_BAD_PARAM;
     }
 #endif
 
-    if(size > array->array_alloc_size) {
-        while(array->array_alloc_size < size)
+    if (size > array->array_alloc_size) {
+        while (array->array_alloc_size < size) {
             array->array_alloc_size <<= 1;
-        array->array_items = (unsigned char *)realloc(array->array_items,
-            array->array_alloc_size * array->array_item_sizeof);
-        if (NULL == array->array_items)
+        }
+        array->array_items = (unsigned char *) realloc(array->array_items,
+                                                       array->array_alloc_size
+                                                           * array->array_item_sizeof);
+        if (NULL == array->array_items) {
             return OPAL_ERR_OUT_OF_RESOURCE;
+        }
     }
     array->array_size = size;
     return OPAL_SUCCESS;
 }
-
diff --git a/opal/class/opal_value_array.h b/opal/class/opal_value_array.h
index de59eb358af..aa7f149132d 100644
--- a/opal/class/opal_value_array.h
+++ b/opal/class/opal_value_array.h
@@ -23,12 +23,12 @@
 
 #include <string.h>
 #ifdef HAVE_STRINGS_H
-#include <strings.h>
+#    include <strings.h>
 #endif /* HAVE_STRINGS_H */
 
 #include "opal/class/opal_object.h"
 #if OPAL_ENABLE_DEBUG
-#include "opal/util/output.h"
+#    include "opal/util/output.h"
 #endif
 #include "opal/constants.h"
 
@@ -38,13 +38,12 @@ BEGIN_C_DECLS
  *  @file  Array of elements maintained by value.
  */
 
-struct opal_value_array_t
-{
-    opal_object_t    super;
-    unsigned char*  array_items;
-    size_t          array_item_sizeof;
-    size_t          array_size;
-    size_t          array_alloc_size;
+struct opal_value_array_t {
+    opal_object_t super;
+    unsigned char *array_items;
+    size_t array_item_sizeof;
+    size_t array_size;
+    size_t array_alloc_size;
 };
 typedef struct opal_value_array_t opal_value_array_t;
 
@@ -68,11 +67,11 @@ static inline int opal_value_array_init(opal_value_array_t *array, size_t item_s
     array->array_item_sizeof = item_sizeof;
     array->array_alloc_size = 1;
     array->array_size = 0;
-    array->array_items = (unsigned char*)realloc(array->array_items, item_sizeof * array->array_alloc_size);
+    array->array_items = (unsigned char *) realloc(array->array_items,
+                                                   item_sizeof * array->array_alloc_size);
     return (NULL != array->array_items) ? OPAL_SUCCESS : OPAL_ERR_OUT_OF_RESOURCE;
 }
 
-
 /**
  *  Reserve space in the array for new elements, but do not change the size.
  *
@@ -81,35 +80,33 @@ static inline int opal_value_array_init(opal_value_array_t *array, size_t item_s
  *  @return  OPAL error code.
  */
 
-static inline int opal_value_array_reserve(opal_value_array_t* array, size_t size)
+static inline int opal_value_array_reserve(opal_value_array_t *array, size_t size)
 {
-     if(size > array->array_alloc_size) {
-         array->array_items = (unsigned char*)realloc(array->array_items, array->array_item_sizeof * size);
-         if(NULL == array->array_items) {
-             array->array_size = 0;
-             array->array_alloc_size = 0;
-             return OPAL_ERR_OUT_OF_RESOURCE;
-         }
-         array->array_alloc_size = size;
-     }
-     return OPAL_SUCCESS;
+    if (size > array->array_alloc_size) {
+        array->array_items = (unsigned char *) realloc(array->array_items,
+                                                       array->array_item_sizeof * size);
+        if (NULL == array->array_items) {
+            array->array_size = 0;
+            array->array_alloc_size = 0;
+            return OPAL_ERR_OUT_OF_RESOURCE;
+        }
+        array->array_alloc_size = size;
+    }
+    return OPAL_SUCCESS;
 }
 
-
-
 /**
- *  Retreives the number of elements in the array.
+ *  Retrieves the number of elements in the array.
  *
  *  @param   array   The input array (IN).
  *  @return  The number of elements currently in use.
  */
 
-static inline size_t opal_value_array_get_size(opal_value_array_t* array)
+static inline size_t opal_value_array_get_size(opal_value_array_t *array)
 {
     return array->array_size;
 }
 
-
 /**
  *  Set the number of elements in the array.
  *
@@ -125,8 +122,7 @@ static inline size_t opal_value_array_get_size(opal_value_array_t* array)
  *  return the new size.
  */
 
-OPAL_DECLSPEC int opal_value_array_set_size(opal_value_array_t* array, size_t size);
-
+OPAL_DECLSPEC int opal_value_array_set_size(opal_value_array_t *array, size_t size);
 
 /**
  *  Macro to retrieve an item from the array by value.
@@ -143,7 +139,7 @@ OPAL_DECLSPEC int opal_value_array_set_size(opal_value_array_t* array, size_t si
  */
 
 #define OPAL_VALUE_ARRAY_GET_ITEM(array, item_type, item_index) \
-    ((item_type*)((array)->array_items))[item_index]
+    ((item_type *) ((array)->array_items))[item_index]
 
 /**
  *  Retrieve an item from the array by reference.
@@ -157,9 +153,10 @@ OPAL_DECLSPEC int opal_value_array_set_size(opal_value_array_t* array, size_t si
  *  array size, the array is grown to satisfy the request.
  */
 
-static inline void* opal_value_array_get_item(opal_value_array_t *array, size_t item_index)
+static inline void *opal_value_array_get_item(opal_value_array_t *array, size_t item_index)
 {
-    if(item_index >= array->array_size && opal_value_array_set_size(array, item_index+1) != OPAL_SUCCESS)
+    if (item_index >= array->array_size
+        && opal_value_array_set_size(array, item_index + 1) != OPAL_SUCCESS)
         return NULL;
     return array->array_items + (item_index * array->array_item_sizeof);
 }
@@ -181,7 +178,7 @@ static inline void* opal_value_array_get_item(opal_value_array_t *array, size_t
  */
 
 #define OPAL_VALUE_ARRAY_SET_ITEM(array, item_type, item_index, item_value) \
-    (((item_type*)((array)->array_items))[item_index] = item_value)
+    (((item_type *) ((array)->array_items))[item_index] = item_value)
 
 /**
  *  Set an array element by value.
@@ -197,17 +194,18 @@ static inline void* opal_value_array_get_item(opal_value_array_t *array, size_t
  * copied into the array by value.
  */
 
-static inline int opal_value_array_set_item(opal_value_array_t *array, size_t item_index, const void* item)
+static inline int opal_value_array_set_item(opal_value_array_t *array, size_t item_index,
+                                            const void *item)
 {
     int rc;
-    if(item_index >= array->array_size &&
-       (rc = opal_value_array_set_size(array, item_index+1)) != OPAL_SUCCESS)
+    if (item_index >= array->array_size
+        && (rc = opal_value_array_set_size(array, item_index + 1)) != OPAL_SUCCESS)
         return rc;
-    memcpy(array->array_items + (item_index * array->array_item_sizeof), item, array->array_item_sizeof);
+    memcpy(array->array_items + (item_index * array->array_item_sizeof), item,
+           array->array_item_sizeof);
     return OPAL_SUCCESS;
 }
 
-
 /**
  *  Appends an item to the end of the array.
  *
@@ -227,7 +225,6 @@ static inline int opal_value_array_append_item(opal_value_array_t *array, const
     return opal_value_array_set_item(array, array->array_size, item);
 }
 
-
 /**
  *  Remove a specific item from the array.
  *
@@ -244,12 +241,13 @@ static inline int opal_value_array_remove_item(opal_value_array_t *array, size_t
 {
 #if OPAL_ENABLE_DEBUG
     if (item_index >= array->array_size) {
-        opal_output(0, "opal_value_array_remove_item: invalid index %lu\n", (unsigned long)item_index);
+        opal_output(0, "opal_value_array_remove_item: invalid index %lu\n",
+                    (unsigned long) item_index);
         return OPAL_ERR_BAD_PARAM;
     }
 #endif
-    memmove(array->array_items+(array->array_item_sizeof * item_index),
-            array->array_items+(array->array_item_sizeof * (item_index+1)),
+    memmove(array->array_items + (array->array_item_sizeof * item_index),
+            array->array_items + (array->array_item_sizeof * (item_index + 1)),
             array->array_item_sizeof * (array->array_size - item_index - 1));
     array->array_size--;
     return OPAL_SUCCESS;
@@ -271,10 +269,8 @@ static inline int opal_value_array_remove_item(opal_value_array_t *array, size_t
  * number of pointer dereferences.
  */
 
-#define OPAL_VALUE_ARRAY_GET_BASE(array, item_type) \
-  ((item_type*) ((array)->array_items))
+#define OPAL_VALUE_ARRAY_GET_BASE(array, item_type) ((item_type *) ((array)->array_items))
 
 END_C_DECLS
 
 #endif
-
diff --git a/opal/datatype/opal_convertor.c b/opal/datatype/opal_convertor.c
index 36736a3007f..8550683a60d 100644
--- a/opal/datatype/opal_convertor.c
+++ b/opal/datatype/opal_convertor.c
@@ -15,6 +15,8 @@
  * Copyright (c) 2013-2018 Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2017      Intel, Inc. All rights reserved
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2022      Advanced Micro Devices, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -25,63 +27,78 @@
 #include "opal_config.h"
 
 #include <stddef.h>
-#include <stdio.h>
 #include <stdint.h>
+#include <stdio.h>
 
 #include "opal/prefetch.h"
 #include "opal/util/arch.h"
 #include "opal/util/output.h"
 
-#include "opal/datatype/opal_datatype_internal.h"
-#include "opal/datatype/opal_datatype.h"
 #include "opal/datatype/opal_convertor.h"
+#include "opal/datatype/opal_convertor_internal.h"
+#include "opal/datatype/opal_datatype.h"
 #include "opal/datatype/opal_datatype_checksum.h"
+#include "opal/datatype/opal_datatype_internal.h"
 #include "opal/datatype/opal_datatype_prototypes.h"
-#include "opal/datatype/opal_convertor_internal.h"
-#if OPAL_CUDA_SUPPORT
-#include "opal/mca/common/cuda/common_cuda.h"
-#define MEMCPY_CUDA( DST, SRC, BLENGTH, CONVERTOR ) \
-    CONVERTOR->cbmemcpy( (DST), (SRC), (BLENGTH), (CONVERTOR) )
-#endif
+#include "opal/mca/accelerator/accelerator.h"
 
-static void opal_convertor_construct( opal_convertor_t* convertor )
+#define MEMCPY_ACCELERATOR(DST, SRC, BLENGTH, CONVERTOR) \
+    CONVERTOR->cbmemcpy((DST), (SRC), (BLENGTH), (CONVERTOR))
+
+static void *opal_convertor_accelerator_memcpy(void *dest, const void *src, size_t size, opal_convertor_t *convertor)
 {
-    convertor->pStack         = convertor->static_stack;
-    convertor->stack_size     = DT_STATIC_STACK_SIZE;
-    convertor->partial_length = 0;
-    convertor->remoteArch     = opal_local_arch;
-    convertor->flags          = OPAL_DATATYPE_FLAG_NO_GAPS | CONVERTOR_COMPLETED;
-#if OPAL_CUDA_SUPPORT
-    convertor->cbmemcpy       = &opal_cuda_memcpy;
-#endif
+    int res;
+    if (!(convertor->flags & CONVERTOR_ACCELERATOR)) {
+        return MEMCPY(dest, src, size);
+    }
+
+    res = opal_accelerator.mem_copy(MCA_ACCELERATOR_NO_DEVICE_ID, MCA_ACCELERATOR_NO_DEVICE_ID,
+                                  dest, src, size, MCA_ACCELERATOR_TRANSFER_UNSPEC);
+    if (OPAL_SUCCESS != res) {
+        opal_output(0, "Error in accelerator memcpy");
+        abort();
+    } else {
+        return dest;
+    }
 }
 
+static void opal_convertor_construct(opal_convertor_t *convertor)
+{
+    convertor->pStack = convertor->static_stack;
+    convertor->stack_size = DT_STATIC_STACK_SIZE;
+    convertor->partial_length = 0;
+    convertor->remoteArch = opal_local_arch;
+    convertor->flags = OPAL_DATATYPE_FLAG_NO_GAPS | CONVERTOR_COMPLETED;
+    convertor->cbmemcpy = &opal_convertor_accelerator_memcpy;
+}
 
-static void opal_convertor_destruct( opal_convertor_t* convertor )
+static void opal_convertor_destruct(opal_convertor_t *convertor)
 {
-    opal_convertor_cleanup( convertor );
+    opal_convertor_cleanup(convertor);
 }
 
-OBJ_CLASS_INSTANCE(opal_convertor_t, opal_object_t, opal_convertor_construct, opal_convertor_destruct );
+OBJ_CLASS_INSTANCE(opal_convertor_t, opal_object_t, opal_convertor_construct,
+                   opal_convertor_destruct);
 
-static opal_convertor_master_t* opal_convertor_master_list = NULL;
+static opal_convertor_master_t *opal_convertor_master_list = NULL;
 
 extern conversion_fct_t opal_datatype_heterogeneous_copy_functions[OPAL_DATATYPE_MAX_PREDEFINED];
 extern conversion_fct_t opal_datatype_copy_functions[OPAL_DATATYPE_MAX_PREDEFINED];
 
-void opal_convertor_destroy_masters( void )
+void opal_convertor_destroy_masters(void)
 {
-    opal_convertor_master_t* master = opal_convertor_master_list;
+    opal_convertor_master_t *master = opal_convertor_master_list;
 
-    while( NULL != master ) {
+    while (NULL != master) {
         opal_convertor_master_list = master->next;
         master->next = NULL;
         /* Cleanup the conversion function if not one of the defaults */
-        if( (master->pFunctions != opal_datatype_heterogeneous_copy_functions) &&
-            (master->pFunctions != opal_datatype_copy_functions) )
-            free( master->pFunctions );
+        if ((master->pFunctions != opal_datatype_heterogeneous_copy_functions)
+            && (master->pFunctions != opal_datatype_copy_functions)) {
+            free(master->pFunctions);
+        }
 
-        free( master );
+        free(master);
         master = opal_convertor_master_list;
     }
 }
@@ -91,161 +108,167 @@ void opal_convertor_destroy_masters( void )
  * is already a master convertor for this architecture then return it.
  * Otherwise, create and initialize a full featured master convertor.
  */
-opal_convertor_master_t* opal_convertor_find_or_create_master( uint32_t remote_arch )
+opal_convertor_master_t *opal_convertor_find_or_create_master(uint32_t remote_arch)
 {
-    opal_convertor_master_t* master = opal_convertor_master_list;
+    opal_convertor_master_t *master = opal_convertor_master_list;
     int i;
-    size_t* remote_sizes;
+    size_t *remote_sizes;
 
-    while( NULL != master ) {
-        if( master->remote_arch == remote_arch )
+    while (NULL != master) {
+        if (master->remote_arch == remote_arch) {
             return master;
+        }
         master = master->next;
     }
     /**
      * Create a new convertor matching the specified architecture and add it to the
      * master convertor list.
      */
-    master = (opal_convertor_master_t*)malloc( sizeof(opal_convertor_master_t) );
+    master = (opal_convertor_master_t *) malloc(sizeof(opal_convertor_master_t));
     master->next = opal_convertor_master_list;
     opal_convertor_master_list = master;
     master->remote_arch = remote_arch;
-    master->flags       = 0;
+    master->flags = 0;
     master->hetero_mask = 0;
     /**
      * Most of the sizes will be identical, so for now just make a copy of
      * the local ones. As master->remote_sizes is defined as being an array of
      * consts we have to manually cast it before using it for writing purposes.
      */
-    remote_sizes = (size_t*)master->remote_sizes;
+    remote_sizes = (size_t *) master->remote_sizes;
     memcpy(remote_sizes, opal_datatype_local_sizes, sizeof(size_t) * OPAL_DATATYPE_MAX_PREDEFINED);
     /**
      * If the local and remote architecture are the same there is no need
      * to check for the remote data sizes. They will always be the same as
      * the local ones.
      */
-    if( master->remote_arch == opal_local_arch ) {
+    if (master->remote_arch == opal_local_arch) {
         master->pFunctions = opal_datatype_copy_functions;
         master->flags |= CONVERTOR_HOMOGENEOUS;
         return master;
     }
 
     /* Find out the remote bool size */
-    if( opal_arch_checkmask( &master->remote_arch, OPAL_ARCH_BOOLIS8 ) ) {
+    if (opal_arch_checkmask(&master->remote_arch, OPAL_ARCH_BOOLIS8)) {
         remote_sizes[OPAL_DATATYPE_BOOL] = 1;
-    } else if( opal_arch_checkmask( &master->remote_arch, OPAL_ARCH_BOOLIS16 ) ) {
+    } else if (opal_arch_checkmask(&master->remote_arch, OPAL_ARCH_BOOLIS16)) {
         remote_sizes[OPAL_DATATYPE_BOOL] = 2;
-    } else if( opal_arch_checkmask( &master->remote_arch, OPAL_ARCH_BOOLIS32 ) ) {
+    } else if (opal_arch_checkmask(&master->remote_arch, OPAL_ARCH_BOOLIS32)) {
         remote_sizes[OPAL_DATATYPE_BOOL] = 4;
     } else {
-        opal_output( 0, "Unknown sizeof(bool) for the remote architecture\n" );
+        opal_output(0, "Unknown sizeof(bool) for the remote architecture\n");
+    }
+    if (opal_arch_checkmask(&master->remote_arch, OPAL_ARCH_LONGIS64)) {
+        remote_sizes[OPAL_DATATYPE_LONG] = 8;
+        remote_sizes[OPAL_DATATYPE_UNSIGNED_LONG] = 8;
+    } else {
+        remote_sizes[OPAL_DATATYPE_LONG] = 4;
+        remote_sizes[OPAL_DATATYPE_UNSIGNED_LONG] = 4;
     }
-
     /**
      * Now we can compute the conversion mask. For all sizes where the remote
      * and local architecture differ a conversion is needed. Moreover, if the
-     * 2 architectures don't have the same endianess all data with a length
+     * 2 architectures don't have the same endianness all data with a length
      * over 2 bytes (with the exception of logicals) have to be byte-swapped.
      */
-    for( i = OPAL_DATATYPE_FIRST_TYPE; i < OPAL_DATATYPE_MAX_PREDEFINED; i++ ) {
-        if( remote_sizes[i] != opal_datatype_local_sizes[i] )
-            master->hetero_mask |= (((uint32_t)1) << i);
+    for (i = OPAL_DATATYPE_FIRST_TYPE; i < OPAL_DATATYPE_MAX_PREDEFINED; i++) {
+        if (remote_sizes[i] != opal_datatype_local_sizes[i]) {
+            master->hetero_mask |= (((uint32_t) 1) << i);
+        }
     }
-    if( opal_arch_checkmask( &master->remote_arch, OPAL_ARCH_ISBIGENDIAN ) !=
-        opal_arch_checkmask( &opal_local_arch, OPAL_ARCH_ISBIGENDIAN ) ) {
+    if (opal_arch_checkmask(&master->remote_arch, OPAL_ARCH_ISBIGENDIAN)
+        != opal_arch_checkmask(&opal_local_arch, OPAL_ARCH_ISBIGENDIAN)) {
         uint32_t hetero_mask = 0;
 
-        for( i = OPAL_DATATYPE_FIRST_TYPE; i < OPAL_DATATYPE_MAX_PREDEFINED; i++ ) {
-            if( remote_sizes[i] > 1 )
-                hetero_mask |= (((uint32_t)1) << i);
+        for (i = OPAL_DATATYPE_FIRST_TYPE; i < OPAL_DATATYPE_MAX_PREDEFINED; i++) {
+            if (remote_sizes[i] > 1) {
+                hetero_mask |= (((uint32_t) 1) << i);
+            }
         }
-        hetero_mask &= ~(((uint32_t)1) << OPAL_DATATYPE_BOOL);
+        hetero_mask &= ~(((uint32_t) 1) << OPAL_DATATYPE_BOOL);
         master->hetero_mask |= hetero_mask;
     }
-    master->pFunctions = (conversion_fct_t*)malloc( sizeof(opal_datatype_heterogeneous_copy_functions) );
+    master->pFunctions = (conversion_fct_t *) malloc(
+        sizeof(opal_datatype_heterogeneous_copy_functions));
     /**
      * Usually the heterogeneous functions are slower than the copy ones. Let's
      * try to minimize the usage of the heterogeneous versions.
      */
-    for( i = OPAL_DATATYPE_FIRST_TYPE; i < OPAL_DATATYPE_MAX_PREDEFINED; i++ ) {
-        if( master->hetero_mask & (((uint32_t)1) << i) )
+    for (i = OPAL_DATATYPE_FIRST_TYPE; i < OPAL_DATATYPE_MAX_PREDEFINED; i++) {
+        if (master->hetero_mask & (((uint32_t) 1) << i)) {
             master->pFunctions[i] = opal_datatype_heterogeneous_copy_functions[i];
-        else
+        } else {
             master->pFunctions[i] = opal_datatype_copy_functions[i];
+        }
     }
 
-    /* We're done so far, return the mater convertor */
+    /* We're done so far, return the master convertor */
     return master;
 }
 
-
-opal_convertor_t* opal_convertor_create( int32_t remote_arch, int32_t mode )
+opal_convertor_t *opal_convertor_create(int32_t remote_arch, int32_t mode)
 {
-    opal_convertor_t* convertor = OBJ_NEW(opal_convertor_t);
-    opal_convertor_master_t* master;
+    opal_convertor_t *convertor = OBJ_NEW(opal_convertor_t);
+    opal_convertor_master_t *master;
 
-    master = opal_convertor_find_or_create_master( remote_arch );
+    master = opal_convertor_find_or_create_master(remote_arch);
 
     convertor->remoteArch = remote_arch;
-    convertor->stack_pos  = 0;
-    convertor->flags      = master->flags;
-    convertor->master     = master;
+    convertor->stack_pos = 0;
+    convertor->flags = master->flags;
+    convertor->master = master;
 
     return convertor;
 }
 
-#define OPAL_CONVERTOR_SET_STATUS_BEFORE_PACK_UNPACK( CONVERTOR, IOV, OUT, MAX_DATA ) \
-    do {                                                                \
-        /* protect against over packing data */                         \
-        if( OPAL_UNLIKELY((CONVERTOR)->flags & CONVERTOR_COMPLETED) ) { \
-            (IOV)[0].iov_len = 0;                                       \
-            *(OUT) = 0;                                                 \
-            *(MAX_DATA) = 0;                                            \
-            return 1;  /* nothing to do */                              \
-        }                                                               \
-        (CONVERTOR)->checksum = OPAL_CSUM_ZERO;                         \
-        (CONVERTOR)->csum_ui1 = 0;                                      \
-        (CONVERTOR)->csum_ui2 = 0;                                      \
-        assert( (CONVERTOR)->bConverted < (CONVERTOR)->local_size );    \
-    } while(0)
+#define OPAL_CONVERTOR_SET_STATUS_BEFORE_PACK_UNPACK(CONVERTOR, IOV, OUT, MAX_DATA) \
+    do {                                                                            \
+        /* protect against over packing data */                                     \
+        if (OPAL_UNLIKELY((CONVERTOR)->flags & CONVERTOR_COMPLETED)) {              \
+            (IOV)[0].iov_len = 0;                                                   \
+            *(OUT) = 0;                                                             \
+            *(MAX_DATA) = 0;                                                        \
+            return 1; /* nothing to do */                                           \
+        }                                                                           \
+        (CONVERTOR)->checksum = OPAL_CSUM_ZERO;                                     \
+        (CONVERTOR)->csum_ui1 = 0;                                                  \
+        (CONVERTOR)->csum_ui2 = 0;                                                  \
+        assert((CONVERTOR)->bConverted < (CONVERTOR)->local_size);                  \
+    } while (0)
 
 /**
  * Return 0 if everything went OK and if there is still room before the complete
  *          conversion of the data (need additional call with others input buffers )
- *        1 if everything went fine and the data was completly converted
+ *        1 if everything went fine and the data was completely converted
  *       -1 something wrong occurs.
  */
-int32_t opal_convertor_pack( opal_convertor_t* pConv,
-                             struct iovec* iov, uint32_t* out_size,
-                             size_t* max_data )
+int32_t opal_convertor_pack(opal_convertor_t *pConv, struct iovec *iov, uint32_t *out_size,
+                            size_t *max_data)
 {
-    OPAL_CONVERTOR_SET_STATUS_BEFORE_PACK_UNPACK( pConv, iov, out_size, max_data );
+    OPAL_CONVERTOR_SET_STATUS_BEFORE_PACK_UNPACK(pConv, iov, out_size, max_data);
 
-    if( OPAL_LIKELY(pConv->flags & CONVERTOR_NO_OP) ) {
+    if (OPAL_LIKELY(pConv->flags & CONVERTOR_NO_OP)) {
         /**
          * We are doing conversion on a contiguous datatype on a homogeneous
          * environment. The convertor contain minimal information, we only
          * use the bConverted to manage the conversion.
          */
         uint32_t i;
-        unsigned char* base_pointer;
+        unsigned char *base_pointer;
         size_t pending_length = pConv->local_size - pConv->bConverted;
 
         *max_data = pending_length;
-        opal_convertor_get_current_pointer( pConv, (void**)&base_pointer );
+        opal_convertor_get_current_pointer(pConv, (void **) &base_pointer);
 
-        for( i = 0; i < *out_size; i++ ) {
-            if( iov[i].iov_len >= pending_length ) {
+        for (i = 0; i < *out_size; i++) {
+            if (iov[i].iov_len >= pending_length) {
                 goto complete_contiguous_data_pack;
             }
-            if( OPAL_LIKELY(NULL == iov[i].iov_base) )
+            if (OPAL_LIKELY(NULL == iov[i].iov_base)) {
                 iov[i].iov_base = (IOVBASE_TYPE *) base_pointer;
-            else
-#if OPAL_CUDA_SUPPORT
-                MEMCPY_CUDA( iov[i].iov_base, base_pointer, iov[i].iov_len, pConv );
-#else
-                MEMCPY( iov[i].iov_base, base_pointer, iov[i].iov_len );
-#endif
+            } else {
+                MEMCPY_ACCELERATOR(iov[i].iov_base, base_pointer, iov[i].iov_len, pConv);
+            }
             pending_length -= iov[i].iov_len;
             base_pointer += iov[i].iov_len;
         }
@@ -253,54 +276,46 @@ int32_t opal_convertor_pack( opal_convertor_t* pConv,
         pConv->bConverted += (*max_data);
         return 0;
 
-complete_contiguous_data_pack:
+    complete_contiguous_data_pack:
         iov[i].iov_len = pending_length;
-        if( OPAL_LIKELY(NULL == iov[i].iov_base) )
+        if (OPAL_LIKELY(NULL == iov[i].iov_base)) {
             iov[i].iov_base = (IOVBASE_TYPE *) base_pointer;
-        else
-#if OPAL_CUDA_SUPPORT
-            MEMCPY_CUDA( iov[i].iov_base, base_pointer, iov[i].iov_len, pConv );
-#else
-            MEMCPY( iov[i].iov_base, base_pointer, iov[i].iov_len );
-#endif
+        } else {
+            MEMCPY_ACCELERATOR(iov[i].iov_base, base_pointer, iov[i].iov_len, pConv);
+        }
         pConv->bConverted = pConv->local_size;
         *out_size = i + 1;
         pConv->flags |= CONVERTOR_COMPLETED;
         return 1;
     }
 
-    return pConv->fAdvance( pConv, iov, out_size, max_data );
+    return pConv->fAdvance(pConv, iov, out_size, max_data);
 }
 
-
-int32_t opal_convertor_unpack( opal_convertor_t* pConv,
-                               struct iovec* iov, uint32_t* out_size,
-                               size_t* max_data )
+int32_t opal_convertor_unpack(opal_convertor_t *pConv, struct iovec *iov, uint32_t *out_size,
+                              size_t *max_data)
 {
-    OPAL_CONVERTOR_SET_STATUS_BEFORE_PACK_UNPACK( pConv, iov, out_size, max_data );
+    OPAL_CONVERTOR_SET_STATUS_BEFORE_PACK_UNPACK(pConv, iov, out_size, max_data);
 
-    if( OPAL_LIKELY(pConv->flags & CONVERTOR_NO_OP) ) {
+    if (OPAL_LIKELY(pConv->flags & CONVERTOR_NO_OP)) {
         /**
          * We are doing conversion on a contiguous datatype on a homogeneous
-         * environment. The convertor contain minimal informations, we only
+         * environment. The convertor contain minimal information, we only
          * use the bConverted to manage the conversion.
          */
         uint32_t i;
-        unsigned char* base_pointer;
+        unsigned char *base_pointer;
         size_t pending_length = pConv->local_size - pConv->bConverted;
 
         *max_data = pending_length;
-        opal_convertor_get_current_pointer( pConv, (void**)&base_pointer );
+        opal_convertor_get_current_pointer(pConv, (void **) &base_pointer);
 
-        for( i = 0; i < *out_size; i++ ) {
-            if( iov[i].iov_len >= pending_length ) {
+        for (i = 0; i < *out_size; i++) {
+            if (iov[i].iov_len >= pending_length) {
                 goto complete_contiguous_data_unpack;
             }
-#if OPAL_CUDA_SUPPORT
-            MEMCPY_CUDA( base_pointer, iov[i].iov_base, iov[i].iov_len, pConv );
-#else
-            MEMCPY( base_pointer, iov[i].iov_base, iov[i].iov_len );
-#endif
+            MEMCPY_ACCELERATOR(base_pointer, iov[i].iov_base, iov[i].iov_len, pConv);
+
             pending_length -= iov[i].iov_len;
             base_pointer += iov[i].iov_len;
         }
@@ -308,29 +323,25 @@ int32_t opal_convertor_unpack( opal_convertor_t* pConv,
         pConv->bConverted += (*max_data);
         return 0;
 
-complete_contiguous_data_unpack:
+    complete_contiguous_data_unpack:
         iov[i].iov_len = pending_length;
-#if OPAL_CUDA_SUPPORT
-        MEMCPY_CUDA( base_pointer, iov[i].iov_base, iov[i].iov_len, pConv );
-#else
-        MEMCPY( base_pointer, iov[i].iov_base, iov[i].iov_len );
-#endif
+        MEMCPY_ACCELERATOR(base_pointer, iov[i].iov_base, iov[i].iov_len, pConv);
         pConv->bConverted = pConv->local_size;
         *out_size = i + 1;
         pConv->flags |= CONVERTOR_COMPLETED;
         return 1;
     }
 
-    return pConv->fAdvance( pConv, iov, out_size, max_data );
+    return pConv->fAdvance(pConv, iov, out_size, max_data);
 }
 
-static inline int
-opal_convertor_create_stack_with_pos_contig( opal_convertor_t* pConvertor,
-                                             size_t starting_point, const size_t* sizes )
+static inline int opal_convertor_create_stack_with_pos_contig(opal_convertor_t *pConvertor,
+                                                              size_t starting_point,
+                                                              const size_t *sizes)
 {
-    dt_stack_t* pStack;   /* pointer to the position on the stack */
-    const opal_datatype_t* pData = pConvertor->pDesc;
-    dt_elem_desc_t* pElems;
+    dt_stack_t *pStack; /* pointer to the position on the stack */
+    const opal_datatype_t *pData = pConvertor->pDesc;
+    dt_elem_desc_t *pElems;
     size_t count;
     ptrdiff_t extent;
 
@@ -344,39 +355,38 @@ opal_convertor_create_stack_with_pos_contig( opal_convertor_t* pConvertor,
     count = starting_point / pData->size;
     extent = pData->ub - pData->lb;
 
-    pStack[0].type     = OPAL_DATATYPE_LOOP;  /* the first one is always the loop */
-    pStack[0].count    = pConvertor->count - count;
-    pStack[0].index    = -1;
-    pStack[0].disp     = count * extent;
+    pStack[0].type = OPAL_DATATYPE_LOOP; /* the first one is always the loop */
+    pStack[0].count = pConvertor->count - count;
+    pStack[0].index = -1;
+    pStack[0].disp = count * extent;
 
     /* now compute the number of pending bytes */
     count = starting_point % pData->size;
     /**
-     * We save the current displacement starting from the begining
+     * We save the current displacement starting from the beginning
      * of this data.
      */
-    if( OPAL_LIKELY(0 == count) ) {
-        pStack[1].type     = pElems->elem.common.type;
-        pStack[1].count    = pElems->elem.blocklen;
+    if (OPAL_LIKELY(0 == count)) {
+        pStack[1].type = pElems->elem.common.type;
+        pStack[1].count = pElems->elem.blocklen;
     } else {
-        pStack[1].type  = OPAL_DATATYPE_UINT1;
+        pStack[1].type = OPAL_DATATYPE_UINT1;
         pStack[1].count = pData->size - count;
     }
-    pStack[1].disp  = count;
-    pStack[1].index = 0;  /* useless */
+    pStack[1].disp = count;
+    pStack[1].index = 0; /* useless */
 
     pConvertor->bConverted = starting_point;
     pConvertor->stack_pos = 1;
-    assert( 0 == pConvertor->partial_length );
+    assert(0 == pConvertor->partial_length);
     return OPAL_SUCCESS;
 }
 
-static inline int
-opal_convertor_create_stack_at_begining( opal_convertor_t* convertor,
-                                         const size_t* sizes )
+static inline int opal_convertor_create_stack_at_begining(opal_convertor_t *convertor,
+                                                          const size_t *sizes)
 {
-    dt_stack_t* pStack = convertor->pStack;
-    dt_elem_desc_t* pElems;
+    dt_stack_t *pStack = convertor->pStack;
+    dt_elem_desc_t *pElems;
 
     /**
      * The prepare function already make the selection on which data representation
@@ -384,9 +394,9 @@ opal_convertor_create_stack_at_begining( opal_convertor_t* convertor,
      */
     pElems = convertor->use_desc->desc;
 
-    convertor->stack_pos      = 1;
+    convertor->stack_pos = 1;
     convertor->partial_length = 0;
-    convertor->bConverted     = 0;
+    convertor->bConverted = 0;
     /**
      * Fill the first position on the stack. This one correspond to the
      * last fake OPAL_DATATYPE_END_LOOP that we add to the data representation and
@@ -394,24 +404,22 @@ opal_convertor_create_stack_at_begining( opal_convertor_t* convertor,
      */
     pStack[0].index = -1;
     pStack[0].count = convertor->count;
-    pStack[0].disp  = 0;
-    pStack[0].type  = OPAL_DATATYPE_LOOP;
+    pStack[0].disp = 0;
+    pStack[0].type = OPAL_DATATYPE_LOOP;
 
     pStack[1].index = 0;
     pStack[1].disp = 0;
-    if( pElems[0].elem.common.type == OPAL_DATATYPE_LOOP ) {
+    if (pElems[0].elem.common.type == OPAL_DATATYPE_LOOP) {
         pStack[1].count = pElems[0].loop.loops;
-        pStack[1].type  = OPAL_DATATYPE_LOOP;
+        pStack[1].type = OPAL_DATATYPE_LOOP;
     } else {
-        pStack[1].count = (size_t)pElems[0].elem.count * pElems[0].elem.blocklen;
-        pStack[1].type  = pElems[0].elem.common.type;
+        pStack[1].count = (size_t) pElems[0].elem.count * pElems[0].elem.blocklen;
+        pStack[1].type = pElems[0].elem.common.type;
     }
     return OPAL_SUCCESS;
 }
 
-
-int32_t opal_convertor_set_position_nocheck( opal_convertor_t* convertor,
-                                             size_t* position )
+int32_t opal_convertor_set_position_nocheck(opal_convertor_t *convertor, size_t *position)
 {
     int32_t rc;
 
@@ -421,23 +429,25 @@ int32_t opal_convertor_set_position_nocheck( opal_convertor_t* convertor,
      * if we plan to rollback the convertor then first we have to reset it at
      * the beginning.
      */
-    if( OPAL_LIKELY(convertor->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS) ) {
-        rc = opal_convertor_create_stack_with_pos_contig( convertor, (*position),
-                                                          opal_datatype_local_sizes );
+    if (OPAL_LIKELY(convertor->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS)) {
+        rc = opal_convertor_create_stack_with_pos_contig(convertor, (*position),
+                                                         opal_datatype_local_sizes);
     } else {
-        if( (0 == (*position)) || ((*position) < convertor->bConverted) ) {
-            rc = opal_convertor_create_stack_at_begining( convertor, opal_datatype_local_sizes );
-            if( 0 == (*position) ) return rc;
+        if ((0 == (*position)) || ((*position) < convertor->bConverted)) {
+            rc = opal_convertor_create_stack_at_begining(convertor, opal_datatype_local_sizes);
+            if (0 == (*position)) {
+                return rc;
+            }
         }
-        rc = opal_convertor_generic_simple_position( convertor, position );
+        rc = opal_convertor_generic_simple_position(convertor, position);
         /**
-         * If we have a non-contigous send convertor don't allow it move in the middle
+         * If we have a non-contiguous send convertor don't allow it move in the middle
          * of a predefined datatype, it won't be able to copy out the left-overs
          * anyway. Instead force the position to stay on predefined datatypes
          * boundaries. As we allow partial predefined datatypes on the contiguous
          * case, we should be accepted by any receiver convertor.
          */
-        if( CONVERTOR_SEND & convertor->flags ) {
+        if (CONVERTOR_SEND & convertor->flags) {
             convertor->bConverted -= convertor->partial_length;
             convertor->partial_length = 0;
         }
@@ -446,50 +456,27 @@ int32_t opal_convertor_set_position_nocheck( opal_convertor_t* convertor,
     return rc;
 }
 
-static size_t
-opal_datatype_compute_remote_size( const opal_datatype_t* pData,
-                                   const size_t* sizes )
-{
-    uint32_t typeMask = pData->bdt_used;
-    size_t length = 0;
-
-    if (opal_datatype_is_predefined(pData)) {
-        return sizes[pData->desc.desc->elem.common.type];
-    }
-
-    if( OPAL_UNLIKELY(NULL == pData->ptypes) ) {
-        /* Allocate and fill the array of types used in the datatype description */
-        opal_datatype_compute_ptypes( (opal_datatype_t*)pData );
-    }
-
-    for( int i = OPAL_DATATYPE_FIRST_TYPE; typeMask && (i < OPAL_DATATYPE_MAX_PREDEFINED); i++ ) {
-        if( typeMask & ((uint32_t)1 << i) ) {
-            length += (pData->ptypes[i] * sizes[i]);
-            typeMask ^= ((uint32_t)1 << i);
-        }
-    }
-    return length;
-}
-
 /**
  * Compute the remote size. If necessary remove the homogeneous flag
  * and redirect the convertor description toward the non-optimized
  * datatype representation.
  */
-size_t opal_convertor_compute_remote_size( opal_convertor_t* pConvertor )
+size_t opal_convertor_compute_remote_size(opal_convertor_t *pConvertor)
 {
-    opal_datatype_t* datatype = (opal_datatype_t*)pConvertor->pDesc;
-    
+    opal_datatype_t *datatype = (opal_datatype_t *) pConvertor->pDesc;
+
     pConvertor->remote_size = pConvertor->local_size;
-    if( OPAL_UNLIKELY(datatype->bdt_used & pConvertor->master->hetero_mask) ) {
+    if (OPAL_UNLIKELY(datatype->bdt_used & pConvertor->master->hetero_mask)) {
         pConvertor->flags &= (~CONVERTOR_HOMOGENEOUS);
-        if (!(pConvertor->flags & CONVERTOR_SEND && pConvertor->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS)) {
+        /* Can we use the optimized description? */
+        if (pConvertor->flags & OPAL_DATATYPE_OPTIMIZED_RESTRICTED) {
             pConvertor->use_desc = &(datatype->desc);
         }
-        if( 0 == (pConvertor->flags & CONVERTOR_HAS_REMOTE_SIZE) ) {
+        if (0 == (pConvertor->flags & CONVERTOR_HAS_REMOTE_SIZE)) {
             /* This is for a single datatype, we must update it with the count */
-            pConvertor->remote_size = opal_datatype_compute_remote_size(datatype,
-                                                                        pConvertor->master->remote_sizes);
+            pConvertor->remote_size =
+                opal_datatype_compute_remote_size(datatype,
+                                                  pConvertor->master->remote_sizes);
             pConvertor->remote_size *= pConvertor->count;
         }
     }
@@ -504,151 +491,171 @@ size_t opal_convertor_compute_remote_size( opal_convertor_t* pConvertor )
  * here that the convertor is clean, either never initialized or already
  * cleaned.
  */
-#define OPAL_CONVERTOR_PREPARE( convertor, datatype, count, pUserBuf )  \
-    {                                                                   \
-        convertor->local_size = count * datatype->size;                 \
-        convertor->pBaseBuf   = (unsigned char*)pUserBuf;               \
-        convertor->count      = count;                                  \
-        convertor->pDesc      = (opal_datatype_t*)datatype;             \
-        convertor->bConverted = 0;                                      \
-        convertor->use_desc   = &(datatype->opt_desc);                  \
-        /* If the data is empty we just mark the convertor as           \
-         * completed. With this flag set the pack and unpack functions  \
-         * will not do anything.                                        \
-         */                                                             \
-        if( OPAL_UNLIKELY((0 == count) || (0 == datatype->size)) ) {    \
-            convertor->flags |= (OPAL_DATATYPE_FLAG_NO_GAPS | CONVERTOR_COMPLETED | CONVERTOR_HAS_REMOTE_SIZE); \
-            convertor->local_size = convertor->remote_size = 0;         \
-            return OPAL_SUCCESS;                                        \
-        }                                                               \
-                                                                        \
-        /* Grab the datatype part of the flags */                       \
-        convertor->flags     &= CONVERTOR_TYPE_MASK;                    \
-        convertor->flags     |= (CONVERTOR_DATATYPE_MASK & datatype->flags); \
-        convertor->flags     |= (CONVERTOR_NO_OP | CONVERTOR_HOMOGENEOUS); \
-                                                                        \
-        convertor->remote_size = convertor->local_size;                 \
-        if( OPAL_LIKELY(convertor->remoteArch == opal_local_arch) ) {   \
-            if( !(convertor->flags & CONVERTOR_WITH_CHECKSUM) &&        \
-                ((convertor->flags & OPAL_DATATYPE_FLAG_NO_GAPS) || \
-                 ((convertor->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS) && (1 == count))) ) { \
-                return OPAL_SUCCESS;                                    \
-            }                                                           \
-        }                                                               \
-                                                                        \
-        assert( (convertor)->pDesc == (datatype) );                     \
-        opal_convertor_compute_remote_size( convertor );                \
-        assert( NULL != convertor->use_desc->desc );                    \
-        /* For predefined datatypes (contiguous) do nothing more */     \
-        /* if checksum is enabled then always continue */               \
-        if( ((convertor->flags & (CONVERTOR_WITH_CHECKSUM | OPAL_DATATYPE_FLAG_NO_GAPS)) \
-             == OPAL_DATATYPE_FLAG_NO_GAPS) &&                          \
-            ((convertor->flags & (CONVERTOR_SEND | CONVERTOR_HOMOGENEOUS)) == \
-             (CONVERTOR_SEND | CONVERTOR_HOMOGENEOUS)) ) {              \
-            return OPAL_SUCCESS;                                        \
-        }                                                               \
-        convertor->flags &= ~CONVERTOR_NO_OP;                           \
-        {                                                               \
-            uint32_t required_stack_length = datatype->loops + 1;       \
-                                                                        \
-            if( required_stack_length > convertor->stack_size ) {       \
-                assert(convertor->pStack == convertor->static_stack);   \
-                convertor->stack_size = required_stack_length;          \
-                convertor->pStack     = (dt_stack_t*)malloc(sizeof(dt_stack_t) * \
-                                                            convertor->stack_size ); \
-            }                                                           \
-        }                                                               \
-        opal_convertor_create_stack_at_begining( convertor, opal_datatype_local_sizes ); \
-    }
-
-
-int32_t opal_convertor_prepare_for_recv( opal_convertor_t* convertor,
-                                         const struct opal_datatype_t* datatype,
-                                         size_t count,
-                                         const void* pUserBuf )
+#define OPAL_CONVERTOR_PREPARE(convertor, datatype, count, pUserBuf)                            \
+    {                                                                                           \
+        convertor->local_size = count * datatype->size;                                         \
+        convertor->pBaseBuf = (unsigned char *) pUserBuf;                                       \
+        convertor->count = count;                                                               \
+        convertor->pDesc = (opal_datatype_t *) datatype;                                        \
+        convertor->bConverted = 0;                                                              \
+        convertor->use_desc = &(datatype->opt_desc);                                            \
+        /* If the data is empty we just mark the convertor as                                   \
+         * completed. With this flag set the pack and unpack functions                          \
+         * will not do anything.                                                                \
+         */                                                                                     \
+        if (OPAL_UNLIKELY((0 == count) || (0 == datatype->size))) {                             \
+            convertor->flags |= (OPAL_DATATYPE_FLAG_NO_GAPS | CONVERTOR_COMPLETED               \
+                                 | CONVERTOR_HAS_REMOTE_SIZE);                                  \
+            convertor->local_size = convertor->remote_size = 0;                                 \
+            return OPAL_SUCCESS;                                                                \
+        }                                                                                       \
+                                                                                                \
+        /* Grab the datatype part of the flags */                                               \
+        convertor->flags &= CONVERTOR_TYPE_MASK;                                                \
+        convertor->flags |= (CONVERTOR_DATATYPE_MASK & datatype->flags);                        \
+        convertor->flags |= (CONVERTOR_NO_OP | CONVERTOR_HOMOGENEOUS);                          \
+                                                                                                \
+        convertor->remote_size = convertor->local_size;                                         \
+        if (OPAL_LIKELY(convertor->remoteArch == opal_local_arch)) {                            \
+            if (!(convertor->flags & CONVERTOR_WITH_CHECKSUM)                                   \
+                && ((convertor->flags & OPAL_DATATYPE_FLAG_NO_GAPS)                             \
+                    || ((convertor->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS) && (1 == count)))) { \
+                return OPAL_SUCCESS;                                                            \
+            }                                                                                   \
+        }                                                                                       \
+                                                                                                \
+        assert((convertor)->pDesc == (datatype));                                               \
+        opal_convertor_compute_remote_size(convertor);                                          \
+        assert(NULL != convertor->use_desc->desc);                                              \
+        /* For predefined datatypes (contiguous) do nothing more */                             \
+        /* if checksum is enabled then always continue */                                       \
+        if (((convertor->flags & (CONVERTOR_WITH_CHECKSUM | OPAL_DATATYPE_FLAG_NO_GAPS))        \
+             == OPAL_DATATYPE_FLAG_NO_GAPS)                                                     \
+            && ((convertor->flags & (CONVERTOR_SEND | CONVERTOR_HOMOGENEOUS))                   \
+                == (CONVERTOR_SEND | CONVERTOR_HOMOGENEOUS))) {                                 \
+            return OPAL_SUCCESS;                                                                \
+        }                                                                                       \
+        convertor->flags &= ~CONVERTOR_NO_OP;                                                   \
+        {                                                                                       \
+            uint32_t required_stack_length = datatype->loops + 1;                               \
+                                                                                                \
+            if (required_stack_length > convertor->stack_size) {                                \
+                assert(convertor->pStack == convertor->static_stack);                           \
+                convertor->stack_size = required_stack_length;                                  \
+                convertor->pStack = (dt_stack_t *) malloc(sizeof(dt_stack_t)                    \
+                                                          * convertor->stack_size);             \
+            }                                                                                   \
+        }                                                                                       \
+        opal_convertor_create_stack_at_begining(convertor, opal_datatype_local_sizes);          \
+    }
+
+static void opal_convertor_accelerator_init(opal_convertor_t *convertor, const void *addr)
+{
+    uint64_t flags = 0;
+    int dev_id;
+    /* This is needed to handle case where convertor is not fully initialized
+     * like when trying to do a sendi with convertor on the stack */
+    convertor->cbmemcpy = &opal_convertor_accelerator_memcpy;
+
+    if (opal_accelerator.check_addr(addr, &dev_id, &flags) > 0) {
+        convertor->flags |= CONVERTOR_ACCELERATOR;
+    }
+    if (flags & MCA_ACCELERATOR_FLAGS_UNIFIED_MEMORY) {
+        convertor->flags |= CONVERTOR_ACCELERATOR_UNIFIED;
+    }
+    return;
+}
+
+int32_t opal_convertor_prepare_for_recv(opal_convertor_t *convertor,
+                                        const struct opal_datatype_t *datatype, size_t count,
+                                        const void *pUserBuf)
 {
     /* Here I should check that the data is not overlapping */
 
     convertor->flags |= CONVERTOR_RECV;
-#if OPAL_CUDA_SUPPORT
-    if (!( convertor->flags & CONVERTOR_SKIP_CUDA_INIT )) {
-        mca_cuda_convertor_init(convertor, pUserBuf);
+    if (!(convertor->flags & CONVERTOR_SKIP_ACCELERATOR_INIT)) {
+        opal_convertor_accelerator_init(convertor, pUserBuf);
     }
-#endif
 
-    assert(! (convertor->flags & CONVERTOR_SEND));
-    OPAL_CONVERTOR_PREPARE( convertor, datatype, count, pUserBuf );
+    assert(!(convertor->flags & CONVERTOR_SEND));
+    OPAL_CONVERTOR_PREPARE(convertor, datatype, count, pUserBuf);
 
 #if defined(CHECKSUM)
-    if( OPAL_UNLIKELY(convertor->flags & CONVERTOR_WITH_CHECKSUM) ) {
-        if( OPAL_UNLIKELY(!(convertor->flags & CONVERTOR_HOMOGENEOUS)) ) {
+    if (OPAL_UNLIKELY(convertor->flags & CONVERTOR_WITH_CHECKSUM)) {
+        if (OPAL_UNLIKELY(!(convertor->flags & CONVERTOR_HOMOGENEOUS))) {
             convertor->fAdvance = opal_unpack_general_checksum;
         } else {
-            if( convertor->pDesc->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS ) {
+            if (convertor->pDesc->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS) {
                 convertor->fAdvance = opal_unpack_homogeneous_contig_checksum;
             } else {
                 convertor->fAdvance = opal_generic_simple_unpack_checksum;
             }
         }
-    } else
-#endif  /* defined(CHECKSUM) */
-        if( OPAL_UNLIKELY(!(convertor->flags & CONVERTOR_HOMOGENEOUS)) ) {
+    } else {
+#endif /* defined(CHECKSUM) */
+        if (OPAL_UNLIKELY(!(convertor->flags & CONVERTOR_HOMOGENEOUS))) {
             convertor->fAdvance = opal_unpack_general;
         } else {
-            if( convertor->pDesc->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS ) {
+            if (convertor->pDesc->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS) {
                 convertor->fAdvance = opal_unpack_homogeneous_contig;
             } else {
                 convertor->fAdvance = opal_generic_simple_unpack;
             }
         }
+#if defined(CHECKSUM)
+    }
+#endif
     return OPAL_SUCCESS;
 }
 
-
-int32_t opal_convertor_prepare_for_send( opal_convertor_t* convertor,
-                                         const struct opal_datatype_t* datatype,
-                                         size_t count,
-                                         const void* pUserBuf )
+int32_t opal_convertor_prepare_for_send(opal_convertor_t *convertor,
+                                        const struct opal_datatype_t *datatype, size_t count,
+                                        const void *pUserBuf)
 {
     convertor->flags |= CONVERTOR_SEND;
-#if OPAL_CUDA_SUPPORT
-    if (!( convertor->flags & CONVERTOR_SKIP_CUDA_INIT )) {
-        mca_cuda_convertor_init(convertor, pUserBuf);
+    if (!(convertor->flags & CONVERTOR_SKIP_ACCELERATOR_INIT)) {
+        opal_convertor_accelerator_init(convertor, pUserBuf);
     }
-#endif
 
-    OPAL_CONVERTOR_PREPARE( convertor, datatype, count, pUserBuf );
+    OPAL_CONVERTOR_PREPARE(convertor, datatype, count, pUserBuf);
 
 #if defined(CHECKSUM)
-    if( convertor->flags & CONVERTOR_WITH_CHECKSUM ) {
-        if( CONVERTOR_SEND_CONVERSION == (convertor->flags & (CONVERTOR_SEND_CONVERSION|CONVERTOR_HOMOGENEOUS)) ) {
+    if (convertor->flags & CONVERTOR_WITH_CHECKSUM) {
+        if (CONVERTOR_SEND_CONVERSION
+            == (convertor->flags & (CONVERTOR_SEND_CONVERSION | CONVERTOR_HOMOGENEOUS))) {
             convertor->fAdvance = opal_pack_general_checksum;
         } else {
-            if( datatype->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS ) {
-                if( ((datatype->ub - datatype->lb) == (ptrdiff_t)datatype->size)
-                    || (1 >= convertor->count) )
+            if (datatype->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS) {
+                if (((datatype->ub - datatype->lb) == (ptrdiff_t) datatype->size)
+                    || (1 >= convertor->count)) {
                     convertor->fAdvance = opal_pack_homogeneous_contig_checksum;
-                else
+                } else {
                     convertor->fAdvance = opal_pack_homogeneous_contig_with_gaps_checksum;
+                }
             } else {
                 convertor->fAdvance = opal_generic_simple_pack_checksum;
             }
         }
-    } else
-#endif  /* defined(CHECKSUM) */
-        if( CONVERTOR_SEND_CONVERSION == (convertor->flags & (CONVERTOR_SEND_CONVERSION|CONVERTOR_HOMOGENEOUS)) ) {
+    } else {
+#endif /* defined(CHECKSUM) */
+        if (CONVERTOR_SEND_CONVERSION
+            == (convertor->flags & (CONVERTOR_SEND_CONVERSION | CONVERTOR_HOMOGENEOUS))) {
             convertor->fAdvance = opal_pack_general;
         } else {
-            if( datatype->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS ) {
-                if( ((datatype->ub - datatype->lb) == (ptrdiff_t)datatype->size)
-                    || (1 >= convertor->count) )
+            if (datatype->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS) {
+                if (((datatype->ub - datatype->lb) == (ptrdiff_t) datatype->size)
+                    || (1 >= convertor->count)) {
                     convertor->fAdvance = opal_pack_homogeneous_contig;
-                else
+                } else {
                     convertor->fAdvance = opal_pack_homogeneous_contig_with_gaps;
+                }
             } else {
                 convertor->fAdvance = opal_generic_simple_pack;
             }
         }
+#if defined(CHECKSUM)
+    }
+#endif
     return OPAL_SUCCESS;
 }
 
@@ -662,90 +669,109 @@ int32_t opal_convertor_prepare_for_send( opal_convertor_t* convertor,
  * ready to use starting from the old position. If copy_stack is false then the convertor
  * is created with a empty stack (you have to use opal_convertor_set_position before using it).
  */
-int opal_convertor_clone( const opal_convertor_t* source,
-                          opal_convertor_t* destination,
-                          int32_t copy_stack )
+int opal_convertor_clone(const opal_convertor_t *source, opal_convertor_t *destination,
+                         int32_t copy_stack)
 {
-    destination->remoteArch        = source->remoteArch;
-    destination->flags             = source->flags;
-    destination->pDesc             = source->pDesc;
-    destination->use_desc          = source->use_desc;
-    destination->count             = source->count;
-    destination->pBaseBuf          = source->pBaseBuf;
-    destination->fAdvance          = source->fAdvance;
-    destination->master            = source->master;
-    destination->local_size        = source->local_size;
-    destination->remote_size       = source->remote_size;
+    destination->remoteArch = source->remoteArch;
+    destination->flags = source->flags;
+    destination->pDesc = source->pDesc;
+    destination->use_desc = source->use_desc;
+    destination->count = source->count;
+    destination->pBaseBuf = source->pBaseBuf;
+    destination->fAdvance = source->fAdvance;
+    destination->master = source->master;
+    destination->local_size = source->local_size;
+    destination->remote_size = source->remote_size;
     /* create the stack */
-    if( OPAL_UNLIKELY(source->stack_size > DT_STATIC_STACK_SIZE) ) {
-        destination->pStack = (dt_stack_t*)malloc(sizeof(dt_stack_t) * source->stack_size );
+    if (OPAL_UNLIKELY(source->stack_size > DT_STATIC_STACK_SIZE)) {
+        destination->pStack = (dt_stack_t *) malloc(sizeof(dt_stack_t) * source->stack_size);
     } else {
         destination->pStack = destination->static_stack;
     }
     destination->stack_size = source->stack_size;
 
     /* initialize the stack */
-    if( OPAL_LIKELY(0 == copy_stack) ) {
+    if (OPAL_LIKELY(0 == copy_stack)) {
         destination->bConverted = -1;
-        destination->stack_pos  = -1;
+        destination->stack_pos = -1;
     } else {
-        memcpy( destination->pStack, source->pStack, sizeof(dt_stack_t) * (source->stack_pos+1) );
+        memcpy(destination->pStack, source->pStack, sizeof(dt_stack_t) * (source->stack_pos + 1));
         destination->bConverted = source->bConverted;
-        destination->stack_pos  = source->stack_pos;
+        destination->stack_pos = source->stack_pos;
     }
-#if OPAL_CUDA_SUPPORT
-    destination->cbmemcpy   = source->cbmemcpy;
-#endif
+
+    destination->cbmemcpy = source->cbmemcpy;
+
     return OPAL_SUCCESS;
 }
 
-
-void opal_convertor_dump( opal_convertor_t* convertor )
+void opal_convertor_dump(opal_convertor_t *convertor)
 {
-    opal_output( 0, "Convertor %p count %" PRIsize_t " stack position %u bConverted %" PRIsize_t "\n"
-                 "\tlocal_size %" PRIsize_t " remote_size %" PRIsize_t " flags %X stack_size %u pending_length %" PRIsize_t "\n"
-                 "\tremote_arch %u local_arch %u\n",
-                 (void*)convertor,
-                 convertor->count, convertor->stack_pos, convertor->bConverted,
-                 convertor->local_size, convertor->remote_size,
-                 convertor->flags, convertor->stack_size, convertor->partial_length,
-                 convertor->remoteArch, opal_local_arch );
-    if( convertor->flags & CONVERTOR_RECV ) opal_output( 0, "unpack ");
-    if( convertor->flags & CONVERTOR_SEND ) opal_output( 0, "pack ");
-    if( convertor->flags & CONVERTOR_SEND_CONVERSION ) opal_output( 0, "conversion ");
-    if( convertor->flags & CONVERTOR_HOMOGENEOUS ) opal_output( 0, "homogeneous " );
-    else opal_output( 0, "heterogeneous ");
-    if( convertor->flags & CONVERTOR_NO_OP ) opal_output( 0, "no_op ");
-    if( convertor->flags & CONVERTOR_WITH_CHECKSUM ) opal_output( 0, "checksum ");
-    if( convertor->flags & CONVERTOR_CUDA ) opal_output( 0, "CUDA ");
-    if( convertor->flags & CONVERTOR_CUDA_ASYNC ) opal_output( 0, "CUDA Async ");
-    if( convertor->flags & CONVERTOR_COMPLETED ) opal_output( 0, "COMPLETED ");
-
-    opal_datatype_dump( convertor->pDesc );
-    if( !((0 == convertor->stack_pos) &&
-          ((size_t)convertor->pStack[convertor->stack_pos].index > convertor->pDesc->desc.length)) ) {
+    opal_output(0,
+                "Convertor %p count %" PRIsize_t " stack position %u bConverted %" PRIsize_t "\n"
+                "\tlocal_size %" PRIsize_t " remote_size %" PRIsize_t
+                " flags %X stack_size %u pending_length %" PRIsize_t "\n"
+                "\tremote_arch %u local_arch %u\n",
+                (void *) convertor, convertor->count, convertor->stack_pos, convertor->bConverted,
+                convertor->local_size, convertor->remote_size, convertor->flags,
+                convertor->stack_size, convertor->partial_length, convertor->remoteArch,
+                opal_local_arch);
+    if (convertor->flags & CONVERTOR_RECV) {
+        opal_output(0, "unpack ");
+    }
+    if (convertor->flags & CONVERTOR_SEND) {
+        opal_output(0, "pack ");
+    }
+    if (convertor->flags & CONVERTOR_SEND_CONVERSION) {
+        opal_output(0, "conversion ");
+    }
+    if (convertor->flags & CONVERTOR_HOMOGENEOUS) {
+        opal_output(0, "homogeneous ");
+    } else {
+        opal_output(0, "heterogeneous ");
+    }
+    if (convertor->flags & CONVERTOR_NO_OP) {
+        opal_output(0, "no_op ");
+    }
+    if (convertor->flags & CONVERTOR_WITH_CHECKSUM) {
+        opal_output(0, "checksum ");
+    }
+    if (convertor->flags & CONVERTOR_ACCELERATOR) {
+        opal_output(0, "ACCELERATOR ");
+    }
+    if (convertor->flags & CONVERTOR_ACCELERATOR_ASYNC) {
+        opal_output(0, "ACCELERATOR Async ");
+    }
+    if (convertor->flags & CONVERTOR_COMPLETED) {
+        opal_output(0, "COMPLETED ");
+    }
+
+    opal_datatype_dump(convertor->pDesc);
+    if (!((0 == convertor->stack_pos)
+          && ((size_t) convertor->pStack[convertor->stack_pos].index
+              > convertor->pDesc->desc.length))) {
         /* only if the convertor is completely initialized */
-        opal_output( 0, "Actual stack representation\n" );
-        opal_datatype_dump_stack( convertor->pStack, convertor->stack_pos,
-                                  convertor->pDesc->desc.desc, convertor->pDesc->name );
+        opal_output(0, "Actual stack representation\n");
+        opal_datatype_dump_stack(convertor->pStack, convertor->stack_pos,
+                                 convertor->pDesc->desc.desc, convertor->pDesc->name);
     }
 }
 
-
-void opal_datatype_dump_stack( const dt_stack_t* pStack, int stack_pos,
-                               const union dt_elem_desc* pDesc, const char* name )
+void opal_datatype_dump_stack(const dt_stack_t *pStack, int stack_pos,
+                              const union dt_elem_desc *pDesc, const char *name)
 {
-    opal_output( 0, "\nStack %p stack_pos %d name %s\n", (void*)pStack, stack_pos, name );
-    for( ; stack_pos >= 0; stack_pos-- ) {
-        opal_output( 0, "%d: pos %d count %" PRIsize_t " disp %ld ", stack_pos, pStack[stack_pos].index,
-                     pStack[stack_pos].count, pStack[stack_pos].disp );
-        if( pStack->index != -1 )
-            opal_output( 0, "\t[desc count %lu disp %ld extent %ld]\n",
-                         (unsigned long)pDesc[pStack[stack_pos].index].elem.count,
-                         (long)pDesc[pStack[stack_pos].index].elem.disp,
-                         (long)pDesc[pStack[stack_pos].index].elem.extent );
-        else
-            opal_output( 0, "\n" );
-    }
-    opal_output( 0, "\n" );
+    opal_output(0, "\nStack %p stack_pos %d name %s\n", (void *) pStack, stack_pos, name);
+    for (; stack_pos >= 0; stack_pos--) {
+        opal_output(0, "%d: pos %d count %" PRIsize_t " disp %ld ", stack_pos,
+                    pStack[stack_pos].index, pStack[stack_pos].count, pStack[stack_pos].disp);
+        if (pStack->index != -1) {
+            opal_output(0, "\t[desc count %lu disp %ld extent %ld]\n",
+                        (unsigned long) pDesc[pStack[stack_pos].index].elem.count,
+                        (long) pDesc[pStack[stack_pos].index].elem.disp,
+                        (long) pDesc[pStack[stack_pos].index].elem.extent);
+        } else {
+            opal_output(0, "\n");
+        }
+    }
+    opal_output(0, "\n");
 }
diff --git a/opal/datatype/opal_convertor.h b/opal/datatype/opal_convertor.h
index b24d94c37b0..d2e73a65b80 100644
--- a/opal/datatype/opal_convertor.h
+++ b/opal/datatype/opal_convertor.h
@@ -15,6 +15,7 @@
  * Copyright (c) 2017-2018 Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2017      Intel, Inc. All rights reserved
+ * Copyright (c) 2022      Advanced Micro Devices, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -28,129 +29,123 @@
 #include "opal_config.h"
 
 #ifdef HAVE_SYS_UIO_H
-#include <sys/uio.h>
+#    include <sys/uio.h>
 #endif
 
 #include "opal/constants.h"
 #include "opal/datatype/opal_datatype.h"
 #include "opal/prefetch.h"
+#include "opal/mca/accelerator/accelerator.h"
 
 BEGIN_C_DECLS
 /*
  * CONVERTOR SECTION
  */
 /* keep the last 16 bits free for data flags */
-#define CONVERTOR_DATATYPE_MASK    0x0000FFFF
-#define CONVERTOR_SEND_CONVERSION  0x00010000
-#define CONVERTOR_RECV             0x00020000
-#define CONVERTOR_SEND             0x00040000
-#define CONVERTOR_HOMOGENEOUS      0x00080000
-#define CONVERTOR_NO_OP            0x00100000
-#define CONVERTOR_WITH_CHECKSUM    0x00200000
-#define CONVERTOR_CUDA             0x00400000
-#define CONVERTOR_CUDA_ASYNC       0x00800000
-#define CONVERTOR_TYPE_MASK        0x10FF0000
-#define CONVERTOR_STATE_START      0x01000000
-#define CONVERTOR_STATE_COMPLETE   0x02000000
-#define CONVERTOR_STATE_ALLOC      0x04000000
-#define CONVERTOR_COMPLETED        0x08000000
-#define CONVERTOR_CUDA_UNIFIED     0x10000000
-#define CONVERTOR_HAS_REMOTE_SIZE  0x20000000
-#define CONVERTOR_SKIP_CUDA_INIT   0x40000000
+#define CONVERTOR_DATATYPE_MASK          0x0000FFFF
+#define CONVERTOR_SEND_CONVERSION        0x00010000
+#define CONVERTOR_RECV                   0x00020000
+#define CONVERTOR_SEND                   0x00040000
+#define CONVERTOR_HOMOGENEOUS            0x00080000
+#define CONVERTOR_NO_OP                  0x00100000
+#define CONVERTOR_WITH_CHECKSUM          0x00200000
+#define CONVERTOR_ACCELERATOR            0x00400000
+#define CONVERTOR_ACCELERATOR_ASYNC      0x00800000
+#define CONVERTOR_TYPE_MASK              0x10FF0000
+#define CONVERTOR_STATE_START            0x01000000
+#define CONVERTOR_STATE_COMPLETE         0x02000000
+#define CONVERTOR_STATE_ALLOC            0x04000000
+#define CONVERTOR_COMPLETED              0x08000000
+#define CONVERTOR_ACCELERATOR_UNIFIED    0x10000000
+#define CONVERTOR_HAS_REMOTE_SIZE        0x20000000
+#define CONVERTOR_SKIP_ACCELERATOR_INIT  0x40000000
 
 union dt_elem_desc;
 typedef struct opal_convertor_t opal_convertor_t;
 
-typedef int32_t (*convertor_advance_fct_t)( opal_convertor_t* pConvertor,
-                                            struct iovec* iov,
-                                            uint32_t* out_size,
-                                            size_t* max_data );
-typedef void*(*memalloc_fct_t)( size_t* pLength, void* userdata );
-typedef void*(*memcpy_fct_t)( void* dest, const void* src, size_t n, opal_convertor_t* pConvertor );
+typedef int32_t (*convertor_advance_fct_t)(opal_convertor_t *pConvertor, struct iovec *iov,
+                                           uint32_t *out_size, size_t *max_data);
+typedef void *(*memalloc_fct_t)(size_t *pLength, void *userdata);
+typedef void *(*memcpy_fct_t)(void *dest, const void *src, size_t n, opal_convertor_t *pConvertor);
 
 /* The master convertor struct (defined in convertor_internal.h) */
 struct opal_convertor_master_t;
 
 struct dt_stack_t {
-    int32_t           index;    /**< index in the element description */
-    int16_t           type;     /**< the type used for the last pack/unpack (original or OPAL_DATATYPE_UINT1) */
-    int16_t           padding;
-    size_t            count;    /**< number of times we still have to do it */
-    ptrdiff_t         disp;     /**< actual displacement depending on the count field */
+    int32_t index; /**< index in the element description */
+    int16_t type;  /**< the type used for the last pack/unpack (original or OPAL_DATATYPE_UINT1) */
+    int16_t padding;
+    size_t count;   /**< number of times we still have to do it */
+    ptrdiff_t disp; /**< actual displacement depending on the count field */
 };
 typedef struct dt_stack_t dt_stack_t;
 
 /**
  *
  */
-#define DT_STATIC_STACK_SIZE   5                /**< This should be sufficient for most applications */
+#define DT_STATIC_STACK_SIZE 5 /**< This should be sufficient for most applications */
 
 struct opal_convertor_t {
-    opal_object_t                 super;          /**< basic superclass */
-    uint32_t                      remoteArch;     /**< the remote architecture */
-    uint32_t                      flags;          /**< the properties of this convertor */
-    size_t                        local_size;     /**< overall length data on local machine, compared to bConverted */
-    size_t                        remote_size;    /**< overall length data on remote machine, compared to bConverted */
-    const opal_datatype_t*        pDesc;          /**< the datatype description associated with the convertor */
-    const dt_type_desc_t*         use_desc;       /**< the version used by the convertor (normal or optimized) */
-    opal_datatype_count_t         count;          /**< the total number of full datatype elements */
+    opal_object_t super; /**< basic superclass */
+    uint32_t remoteArch; /**< the remote architecture */
+    uint32_t flags;      /**< the properties of this convertor */
+    size_t local_size;   /**< overall length data on local machine, compared to bConverted */
+    size_t remote_size;  /**< overall length data on remote machine, compared to bConverted */
+    const opal_datatype_t *pDesc;   /**< the datatype description associated with the convertor */
+    const dt_type_desc_t *use_desc; /**< the version used by the convertor (normal or optimized) */
+    opal_datatype_count_t count;    /**< the total number of full datatype elements */
 
     /* --- cacheline boundary (64 bytes - if 64bits arch and !OPAL_ENABLE_DEBUG) --- */
-    uint32_t                      stack_size;     /**< size of the allocated stack */
-    unsigned char*                pBaseBuf;       /**< initial buffer as supplied by the user */
-    dt_stack_t*                   pStack;         /**< the local stack for the actual conversion */
-    convertor_advance_fct_t       fAdvance;       /**< pointer to the pack/unpack functions */
+    uint32_t stack_size;              /**< size of the allocated stack */
+    unsigned char *pBaseBuf;          /**< initial buffer as supplied by the user */
+    dt_stack_t *pStack;               /**< the local stack for the actual conversion */
+    convertor_advance_fct_t fAdvance; /**< pointer to the pack/unpack functions */
 
     /* --- cacheline boundary (96 bytes - if 64bits arch and !OPAL_ENABLE_DEBUG) --- */
-    struct opal_convertor_master_t* master;       /**< the master convertor */
+    struct opal_convertor_master_t *master; /**< the master convertor */
 
     /* All others fields get modified for every call to pack/unpack functions */
-    uint32_t                      stack_pos;      /**< the actual position on the stack */
-    size_t                        partial_length; /**< amount of data left over from the last unpack */
-    size_t                        bConverted;     /**< # of bytes already converted */
+    uint32_t stack_pos;    /**< the actual position on the stack */
+    size_t partial_length; /**< amount of data left over from the last unpack */
+    size_t bConverted;     /**< # of bytes already converted */
 
     /* --- cacheline boundary (128 bytes - if 64bits arch and !OPAL_ENABLE_DEBUG) --- */
-    uint32_t                      checksum;       /**< checksum computed by pack/unpack operation */
-    uint32_t                      csum_ui1;       /**< partial checksum computed by pack/unpack operation */
-    size_t                        csum_ui2;       /**< partial checksum computed by pack/unpack operation */
+    uint32_t checksum; /**< checksum computed by pack/unpack operation */
+    uint32_t csum_ui1; /**< partial checksum computed by pack/unpack operation */
+    size_t csum_ui2;   /**< partial checksum computed by pack/unpack operation */
 
     /* --- fields are no more aligned on cacheline --- */
-    dt_stack_t                    static_stack[DT_STATIC_STACK_SIZE];  /**< local stack for small datatypes */
+    dt_stack_t static_stack[DT_STATIC_STACK_SIZE]; /**< local stack for small datatypes */
 
-#if OPAL_CUDA_SUPPORT
-    memcpy_fct_t                  cbmemcpy;       /**< memcpy or cuMemcpy */
-    void *                        stream;         /**< CUstream for async copy */
-#endif
+    memcpy_fct_t cbmemcpy; /**< memcpy or accelerator memcpy */
+    opal_accelerator_stream_t *stream;  /**<Accelerator stream for async copy */
 };
-OPAL_DECLSPEC OBJ_CLASS_DECLARATION( opal_convertor_t );
-
+OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_convertor_t);
 
 /*
  *
  */
-static inline uint32_t opal_convertor_get_checksum( opal_convertor_t* convertor )
+static inline uint32_t opal_convertor_get_checksum(opal_convertor_t *convertor)
 {
     return convertor->checksum;
 }
 
-
 /*
  *
  */
-OPAL_DECLSPEC int32_t opal_convertor_pack( opal_convertor_t* pConv, struct iovec* iov,
-                                           uint32_t* out_size, size_t* max_data );
+OPAL_DECLSPEC int32_t opal_convertor_pack(opal_convertor_t *pConv, struct iovec *iov,
+                                          uint32_t *out_size, size_t *max_data);
 
 /*
  *
  */
-OPAL_DECLSPEC int32_t opal_convertor_unpack( opal_convertor_t* pConv, struct iovec* iov,
-                                             uint32_t* out_size, size_t* max_data );
+OPAL_DECLSPEC int32_t opal_convertor_unpack(opal_convertor_t *pConv, struct iovec *iov,
+                                            uint32_t *out_size, size_t *max_data);
 
 /*
  *
  */
-OPAL_DECLSPEC opal_convertor_t* opal_convertor_create( int32_t remote_arch, int32_t mode );
-
+OPAL_DECLSPEC opal_convertor_t *opal_convertor_create(int32_t remote_arch, int32_t mode);
 
 /**
  * The cleanup function will put the convertor in exactly the same state as after a call
@@ -159,21 +154,20 @@ OPAL_DECLSPEC opal_convertor_t* opal_convertor_create( int32_t remote_arch, int3
  * cache. The OBJ_CONSTRUCT on the convertor should be called only on the first creation
  * of a request (not when extracted from the cache).
  */
-static inline int opal_convertor_cleanup( opal_convertor_t* convertor )
+static inline int opal_convertor_cleanup(opal_convertor_t *convertor)
 {
-    if( OPAL_UNLIKELY(convertor->stack_size > DT_STATIC_STACK_SIZE) ) {
-        free( convertor->pStack );
-        convertor->pStack     = convertor->static_stack;
+    if (OPAL_UNLIKELY(convertor->stack_size > DT_STATIC_STACK_SIZE)) {
+        free(convertor->pStack);
+        convertor->pStack = convertor->static_stack;
         convertor->stack_size = DT_STATIC_STACK_SIZE;
     }
-    convertor->pDesc     = NULL;
+    convertor->pDesc = NULL;
     convertor->stack_pos = 0;
-    convertor->flags     = OPAL_DATATYPE_FLAG_NO_GAPS | CONVERTOR_COMPLETED;
+    convertor->flags = OPAL_DATATYPE_FLAG_NO_GAPS | CONVERTOR_COMPLETED;
 
     return OPAL_SUCCESS;
 }
 
-
 /**
  * Return:   0 if no packing is required for sending (the upper layer
  *             can use directly the pointer to the contiguous user
@@ -182,50 +176,60 @@ static inline int opal_convertor_cleanup( opal_convertor_t* convertor )
  *             (source arch != dest arch) or non contiguous memory
  *             layout.
  */
-static inline int32_t opal_convertor_need_buffers( const opal_convertor_t* pConvertor )
+static inline int32_t opal_convertor_need_buffers(const opal_convertor_t *pConvertor)
 {
-    if (OPAL_UNLIKELY(0 == (pConvertor->flags & CONVERTOR_HOMOGENEOUS))) return 1;
-#if OPAL_CUDA_SUPPORT
-    if( pConvertor->flags & (CONVERTOR_CUDA | CONVERTOR_CUDA_UNIFIED)) return 1;
-#endif
-    if( pConvertor->flags & OPAL_DATATYPE_FLAG_NO_GAPS ) return 0;
-    if( (pConvertor->count == 1) && (pConvertor->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS) ) return 0;
+    if (OPAL_UNLIKELY(0 == (pConvertor->flags & CONVERTOR_HOMOGENEOUS)))
+        return 1;
+    if (pConvertor->flags & OPAL_DATATYPE_FLAG_NO_GAPS)
+        return 0;
+    if ((pConvertor->count == 1) && (pConvertor->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS))
+        return 0;
     return 1;
 }
 
+static inline int32_t opal_convertor_on_device(const opal_convertor_t *pConvertor)
+{
+    return !!(pConvertor->flags & CONVERTOR_ACCELERATOR);
+}
+
+static inline int32_t opal_convertor_on_discrete_device(const opal_convertor_t *pConvertor)
+{
+    return (CONVERTOR_ACCELERATOR == ((pConvertor->flags & CONVERTOR_ACCELERATOR) |
+                                      (pConvertor->flags & CONVERTOR_ACCELERATOR_UNIFIED)));
+}
+
+static inline int32_t opal_convertor_on_unified_device(const opal_convertor_t *pConvertor)
+{
+    return (!!(pConvertor->flags & CONVERTOR_ACCELERATOR) &&
+            !!(pConvertor->flags & CONVERTOR_ACCELERATOR_UNIFIED));
+}
+
 /**
  * Update the size of the remote datatype representation. The size will
  * depend on the configuration of the master convertor. In homogeneous
  * environments, the local and remote sizes are identical.
  */
-size_t
-opal_convertor_compute_remote_size( opal_convertor_t* pConv );
+size_t opal_convertor_compute_remote_size(opal_convertor_t *pConv);
 
 /**
- * Return the local size of the convertor (count times the size of the datatype).
+ * Return the packed size of the memory layout represented by this
+ * convertor. This is the size of the buffer that would be needed
+ * for the conversion (takes in account the type of the operation,
+ * aka pack or unpack, as well as which side is supposed to do the
+ * type conversion).
  */
-static inline void opal_convertor_get_packed_size( const opal_convertor_t* pConv,
-                                                   size_t* pSize )
+static inline void
+opal_convertor_get_packed_size(const opal_convertor_t *pConv, size_t *pSize)
 {
     *pSize = pConv->local_size;
-}
-
-
-/**
- * Return the remote size of the convertor (count times the remote size of the
- * datatype). On homogeneous environments the local and remote sizes are
- * identical.
- */
-static inline void opal_convertor_get_unpacked_size( const opal_convertor_t* pConv,
-                                                     size_t* pSize )
-{
-    if( pConv->flags & CONVERTOR_HOMOGENEOUS ) {
-        *pSize = pConv->local_size;
+    if ((pConv->flags & CONVERTOR_HOMOGENEOUS) ||
+        ((pConv->flags & CONVERTOR_SEND) && !(pConv->flags & CONVERTOR_SEND_CONVERSION)) ||
+        ((pConv->flags & CONVERTOR_RECV) && (pConv->flags & CONVERTOR_SEND_CONVERSION))) {
         return;
     }
-    if( 0 == (CONVERTOR_HAS_REMOTE_SIZE & pConv->flags) ) {
-        assert(! (pConv->flags & CONVERTOR_SEND));
-        opal_convertor_compute_remote_size( (opal_convertor_t*)pConv);
+    if (0 == (CONVERTOR_HAS_REMOTE_SIZE & pConv->flags)) {
+        assert(!(pConv->flags & CONVERTOR_SEND));
+        opal_convertor_compute_remote_size((opal_convertor_t *) pConv);
     }
     *pSize = pConv->remote_size;
 }
@@ -235,89 +239,75 @@ static inline void opal_convertor_get_unpacked_size( const opal_convertor_t* pCo
  * mostly useful for contiguous datatypes, when we need to get the pointer to the
  * contiguous piece of memory.
  */
-static inline void opal_convertor_get_current_pointer( const opal_convertor_t* pConv,
-                                                       void** position )
+static inline void opal_convertor_get_current_pointer(const opal_convertor_t *pConv,
+                                                      void **position)
 {
-    unsigned char* base = pConv->pBaseBuf + pConv->bConverted + pConv->pDesc->true_lb;
-    *position = (void*)base;
+    unsigned char *base = pConv->pBaseBuf + pConv->bConverted + pConv->pDesc->true_lb;
+    *position = (void *) base;
 }
 
-static inline void opal_convertor_get_offset_pointer( const opal_convertor_t* pConv,
-                                                      size_t offset, void** position )
+static inline void opal_convertor_get_offset_pointer(const opal_convertor_t *pConv, size_t offset,
+                                                     void **position)
 {
-    unsigned char* base = pConv->pBaseBuf + offset + pConv->pDesc->true_lb;
-    *position = (void*)base;
+    unsigned char *base = pConv->pBaseBuf + offset + pConv->pDesc->true_lb;
+    *position = (void *) base;
 }
 
-
 /*
  *
  */
-OPAL_DECLSPEC int32_t opal_convertor_prepare_for_send( opal_convertor_t* convertor,
-                                                       const struct opal_datatype_t* datatype,
-                                                       size_t count,
-                                                       const void* pUserBuf);
-
-static inline int32_t opal_convertor_copy_and_prepare_for_send( const opal_convertor_t* pSrcConv,
-                                                                const struct opal_datatype_t* datatype,
-                                                                size_t count,
-                                                                const void* pUserBuf,
-                                                                int32_t flags,
-                                                                opal_convertor_t* convertor )
+OPAL_DECLSPEC int32_t opal_convertor_prepare_for_send(opal_convertor_t *convertor,
+                                                      const struct opal_datatype_t *datatype,
+                                                      size_t count, const void *pUserBuf);
+
+static inline int32_t opal_convertor_copy_and_prepare_for_send(
+    const opal_convertor_t *pSrcConv, const struct opal_datatype_t *datatype, size_t count,
+    const void *pUserBuf, int32_t flags, opal_convertor_t *convertor)
 {
     convertor->remoteArch = pSrcConv->remoteArch;
-    convertor->flags      = pSrcConv->flags | flags;
-    convertor->master     = pSrcConv->master;
+    convertor->flags = pSrcConv->flags | flags;
+    convertor->master = pSrcConv->master;
 
-    return opal_convertor_prepare_for_send( convertor, datatype, count, pUserBuf );
+    return opal_convertor_prepare_for_send(convertor, datatype, count, pUserBuf);
 }
 
 /*
  *
  */
-OPAL_DECLSPEC int32_t opal_convertor_prepare_for_recv( opal_convertor_t* convertor,
-                                                       const struct opal_datatype_t* datatype,
-                                                       size_t count,
-                                                       const void* pUserBuf );
-static inline int32_t opal_convertor_copy_and_prepare_for_recv( const opal_convertor_t* pSrcConv,
-                                                                const struct opal_datatype_t* datatype,
-                                                                size_t count,
-                                                                const void* pUserBuf,
-                                                                int32_t flags,
-                                                                opal_convertor_t* convertor )
+OPAL_DECLSPEC int32_t opal_convertor_prepare_for_recv(opal_convertor_t *convertor,
+                                                      const struct opal_datatype_t *datatype,
+                                                      size_t count, const void *pUserBuf);
+static inline int32_t opal_convertor_copy_and_prepare_for_recv(
+    const opal_convertor_t *pSrcConv, const struct opal_datatype_t *datatype, size_t count,
+    const void *pUserBuf, int32_t flags, opal_convertor_t *convertor)
 {
     convertor->remoteArch = pSrcConv->remoteArch;
-    convertor->flags      = (pSrcConv->flags | flags);
-    convertor->master     = pSrcConv->master;
+    convertor->flags = (pSrcConv->flags | flags);
+    convertor->master = pSrcConv->master;
 
-    return opal_convertor_prepare_for_recv( convertor, datatype, count, pUserBuf );
+    return opal_convertor_prepare_for_recv(convertor, datatype, count, pUserBuf);
 }
 
 /*
  * Give access to the raw memory layout based on the datatype.
  */
-OPAL_DECLSPEC int32_t
-opal_convertor_raw( opal_convertor_t* convertor,  /* [IN/OUT] */
-                    struct iovec* iov,            /* [IN/OUT] */
-                    uint32_t* iov_count,          /* [IN/OUT] */
-                    size_t* length );             /* [OUT]    */
-
+OPAL_DECLSPEC int32_t opal_convertor_raw(opal_convertor_t *convertor, /* [IN/OUT] */
+                                         struct iovec *iov,           /* [IN/OUT] */
+                                         uint32_t *iov_count,         /* [IN/OUT] */
+                                         size_t *length);             /* [OUT]    */
 
 /*
  * Upper level does not need to call the _nocheck function directly.
  */
-OPAL_DECLSPEC int32_t
-opal_convertor_set_position_nocheck( opal_convertor_t* convertor,
-                                     size_t* position );
-static inline int32_t
-opal_convertor_set_position( opal_convertor_t* convertor,
-                             size_t* position )
+OPAL_DECLSPEC int32_t opal_convertor_set_position_nocheck(opal_convertor_t *convertor,
+                                                          size_t *position);
+static inline int32_t opal_convertor_set_position(opal_convertor_t *convertor, size_t *position)
 {
     /*
      * Do not allow the convertor to go outside the data boundaries. This test include
      * the check for datatype with size zero as well as for convertors with a count of zero.
      */
-    if( OPAL_UNLIKELY(convertor->local_size <= *position) ) {
+    if (OPAL_UNLIKELY(convertor->local_size <= *position)) {
         convertor->flags |= CONVERTOR_COMPLETED;
         convertor->bConverted = convertor->local_size;
         *position = convertor->bConverted;
@@ -327,76 +317,66 @@ opal_convertor_set_position( opal_convertor_t* convertor,
     /*
      * If the convertor is already at the correct position we are happy.
      */
-    if( OPAL_LIKELY((*position) == convertor->bConverted) ) return OPAL_SUCCESS;
+    if (OPAL_LIKELY((*position) == convertor->bConverted))
+        return OPAL_SUCCESS;
 
     /* Remove the completed flag if it's already set */
     convertor->flags &= ~CONVERTOR_COMPLETED;
 
-    if( (convertor->flags & OPAL_DATATYPE_FLAG_NO_GAPS) &&
+    if ((convertor->flags & OPAL_DATATYPE_FLAG_NO_GAPS) &&
 #if defined(CHECKSUM)
         !(convertor->flags & CONVERTOR_WITH_CHECKSUM) &&
-#endif  /* defined(CHECKSUM) */
-        (convertor->flags & (CONVERTOR_SEND | CONVERTOR_HOMOGENEOUS)) ) {
+#endif /* defined(CHECKSUM) */
+        (convertor->flags & (CONVERTOR_SEND | CONVERTOR_HOMOGENEOUS))) {
         /* Contiguous and no checkpoint and no homogeneous unpack */
         convertor->bConverted = *position;
         return OPAL_SUCCESS;
     }
 
-    return opal_convertor_set_position_nocheck( convertor, position );
+    return opal_convertor_set_position_nocheck(convertor, position);
 }
 
 /*
  *
  */
-static inline int32_t
-opal_convertor_personalize( opal_convertor_t* convertor,
-                            uint32_t flags,
-                            size_t* position )
+static inline int32_t opal_convertor_personalize(opal_convertor_t *convertor, uint32_t flags,
+                                                 size_t *position)
 {
     convertor->flags |= flags;
 
-    if( OPAL_UNLIKELY(NULL == position) )
+    if (OPAL_UNLIKELY(NULL == position))
         return OPAL_SUCCESS;
-    return opal_convertor_set_position( convertor, position );
+    return opal_convertor_set_position(convertor, position);
 }
 
 /*
  *
  */
-OPAL_DECLSPEC int
-opal_convertor_clone( const opal_convertor_t* source,
-                      opal_convertor_t* destination,
-                      int32_t copy_stack );
-
-static inline int
-opal_convertor_clone_with_position( const opal_convertor_t* source,
-                                    opal_convertor_t* destination,
-                                    int32_t copy_stack,
-                                    size_t* position )
+OPAL_DECLSPEC int opal_convertor_clone(const opal_convertor_t *source,
+                                       opal_convertor_t *destination, int32_t copy_stack);
+
+static inline int opal_convertor_clone_with_position(const opal_convertor_t *source,
+                                                     opal_convertor_t *destination,
+                                                     int32_t copy_stack, size_t *position)
 {
-    (void)opal_convertor_clone( source, destination, copy_stack );
-    return opal_convertor_set_position( destination, position );
+    (void) opal_convertor_clone(source, destination, copy_stack);
+    return opal_convertor_set_position(destination, position);
 }
 
 /*
  *
  */
-OPAL_DECLSPEC void
-opal_convertor_dump( opal_convertor_t* convertor );
+OPAL_DECLSPEC void opal_convertor_dump(opal_convertor_t *convertor);
 
-OPAL_DECLSPEC void
-opal_datatype_dump_stack( const dt_stack_t* pStack,
-                          int stack_pos,
-                          const union dt_elem_desc* pDesc,
-                          const char* name );
+OPAL_DECLSPEC void opal_datatype_dump_stack(const dt_stack_t *pStack, int stack_pos,
+                                            const union dt_elem_desc *pDesc, const char *name);
 
 /*
  *
  */
-OPAL_DECLSPEC int
-opal_convertor_generic_simple_position( opal_convertor_t* pConvertor,
-                                        size_t* position );
+OPAL_DECLSPEC int opal_convertor_generic_simple_position(opal_convertor_t *pConvertor,
+                                                         size_t *position);
 
 END_C_DECLS
 
-#endif  /* OPAL_CONVERTOR_H_HAS_BEEN_INCLUDED */
+#endif /* OPAL_CONVERTOR_H_HAS_BEEN_INCLUDED */
diff --git a/opal/datatype/opal_convertor_internal.h b/opal/datatype/opal_convertor_internal.h
index 39690f5bd19..d44db12c673 100644
--- a/opal/datatype/opal_convertor_internal.h
+++ b/opal/datatype/opal_convertor_internal.h
@@ -22,34 +22,32 @@
 
 BEGIN_C_DECLS
 
-typedef int32_t (*conversion_fct_t)( opal_convertor_t* pConvertor, uint32_t count,
-                                     const void* from, size_t from_len, ptrdiff_t from_extent,
-                                     void* to, size_t to_length, ptrdiff_t to_extent,
-                                     ptrdiff_t *advance );
+typedef int32_t (*conversion_fct_t)(opal_convertor_t *pConvertor, uint32_t count, const void *from,
+                                    size_t from_len, ptrdiff_t from_extent, void *to,
+                                    size_t to_length, ptrdiff_t to_extent, ptrdiff_t *advance);
 
 typedef struct opal_convertor_master_t {
-    struct opal_convertor_master_t* next;
-    uint32_t                        remote_arch;
-    uint32_t                        flags;
-    uint32_t                        hetero_mask;
-    const size_t                    remote_sizes[OPAL_DATATYPE_MAX_PREDEFINED];
-    conversion_fct_t*               pFunctions;   /**< the convertor functions pointer */
+    struct opal_convertor_master_t *next;
+    uint32_t remote_arch;
+    uint32_t flags;
+    uint32_t hetero_mask;
+    const size_t remote_sizes[OPAL_DATATYPE_MAX_PREDEFINED];
+    conversion_fct_t *pFunctions; /**< the convertor functions pointer */
 } opal_convertor_master_t;
 
 /*
  * Find or create a new master convertor based on a specific architecture. The master
- * convertor hold all informations related to a defined architecture, such as the sizes
+ * convertor hold all information related to a defined architecture, such as the sizes
  * of the predefined data-types, the conversion functions, ...
  */
-opal_convertor_master_t* opal_convertor_find_or_create_master( uint32_t remote_arch );
+opal_convertor_master_t *opal_convertor_find_or_create_master(uint32_t remote_arch);
 
 /*
  * Destroy all pending master convertors. This function is usually called when we
  * shutdown the data-type engine, once all convertors have been destroyed.
  */
-void opal_convertor_destroy_masters( void );
-
+void opal_convertor_destroy_masters(void);
 
 END_C_DECLS
 
-#endif  /* OPAL_CONVERTOR_INTERNAL_HAS_BEEN_INCLUDED */
+#endif /* OPAL_CONVERTOR_INTERNAL_HAS_BEEN_INCLUDED */
diff --git a/opal/datatype/opal_convertor_raw.c b/opal/datatype/opal_convertor_raw.c
index 7e938a53c7c..068bd53ad00 100644
--- a/opal/datatype/opal_convertor_raw.c
+++ b/opal/datatype/opal_convertor_raw.c
@@ -23,11 +23,14 @@
 #include "opal_stdint.h"
 
 #if OPAL_ENABLE_DEBUG
-#include "opal/util/output.h"
+#    include "opal/util/output.h"
 
-#define DO_DEBUG(INST)  if( opal_ddt_raw_debug ) { INST }
+#    define DO_DEBUG(INST)        \
+        if (opal_ddt_raw_debug) { \
+            INST                  \
+        }
 #else
-#define DO_DEBUG(INST)
+#    define DO_DEBUG(INST)
 #endif /* OPAL_ENABLE_DEBUG */
 
 /* Take a new iovec (base + len) and try to merge it with what we already
@@ -35,18 +38,18 @@
  * iovec location. If we need to advance position and we reach the end
  * of the iovec array, return 1 to signal we did not saved the last iovec.
  */
-static inline int
-opal_convertor_merge_iov( struct iovec* iov, uint32_t* iov_count,
-                          IOVBASE_TYPE* base, size_t len,
-                          uint32_t* idx )
+static inline int opal_convertor_merge_iov(struct iovec *iov, uint32_t *iov_count,
+                                           IOVBASE_TYPE *base, size_t len, uint32_t *idx)
 {
-    if( 0 != iov[*idx].iov_len ) {
-        if( base == ((char*)iov[*idx].iov_base + iov[*idx].iov_len) ) {
-            iov[*idx].iov_len += len;  /* merge with previous iovec */
+    if (0 != iov[*idx].iov_len) {
+        if (base == ((char *) iov[*idx].iov_base + iov[*idx].iov_len)) {
+            iov[*idx].iov_len += len; /* merge with previous iovec */
             return 0;
-        }  /* cannot merge, move to the next position */
+        } /* cannot merge, move to the next position */
         *idx = *idx + 1;
-        if( *idx == *iov_count ) return 1;  /* do not overwrite outside the iovec array boundaries */
+        if (*idx == *iov_count) {
+            return 1; /* do not overwrite outside the iovec array boundaries */
+        }
     }
     iov[*idx].iov_base = base;
     iov[*idx].iov_len = len;
@@ -58,45 +61,43 @@ opal_convertor_merge_iov( struct iovec* iov, uint32_t* iov_count,
  * conversion (i.e. no heterogeneity) is taken into account, and that all
  * length we're working on are local.
  */
-int32_t
-opal_convertor_raw( opal_convertor_t* pConvertor,
-                    struct iovec* iov, uint32_t* iov_count,
-                    size_t* length )
+int32_t opal_convertor_raw(opal_convertor_t *pConvertor, struct iovec *iov, uint32_t *iov_count,
+                           size_t *length)
 {
     const opal_datatype_t *pData = pConvertor->pDesc;
-    dt_stack_t* pStack;       /* pointer to the position on the stack */
-    uint32_t pos_desc;        /* actual position in the description of the derived datatype */
-    size_t count_desc;        /* the number of items already done in the actual pos_desc */
+    dt_stack_t *pStack; /* pointer to the position on the stack */
+    uint32_t pos_desc;  /* actual position in the description of the derived datatype */
+    size_t count_desc;  /* the number of items already done in the actual pos_desc */
     size_t do_now, blength;
-    dt_elem_desc_t* description, *pElem;
-    unsigned char *source_base;  /* origin of the data */
-    size_t sum_iov_len = 0;      /* sum of raw data lengths in the iov_len fields */
-    uint32_t index = 0;          /* the iov index and a simple counter */
+    dt_elem_desc_t *description, *pElem;
+    unsigned char *source_base; /* origin of the data */
+    size_t sum_iov_len = 0;     /* sum of raw data lengths in the iov_len fields */
+    uint32_t index = 0;         /* the iov index and a simple counter */
 
-    assert( (*iov_count) > 0 );
-    if( OPAL_LIKELY(pConvertor->flags & CONVERTOR_COMPLETED) ) {
+    assert((*iov_count) > 0);
+    if (OPAL_LIKELY(pConvertor->flags & CONVERTOR_COMPLETED)) {
         iov[0].iov_base = NULL;
-        iov[0].iov_len  = 0;
-        *iov_count      = 0;
-        *length         = iov[0].iov_len;
-        return 1;  /* We're still done */
+        iov[0].iov_len = 0;
+        *iov_count = 0;
+        *length = iov[0].iov_len;
+        return 1; /* We're still done */
     }
-    if( OPAL_LIKELY(pConvertor->flags & CONVERTOR_NO_OP) ) {
-        /* The convertor contain minimal informations, we only use the bConverted
+    if (OPAL_LIKELY(pConvertor->flags & CONVERTOR_NO_OP)) {
+        /* The convertor contain minimal information, we only use the bConverted
          * to manage the conversion. This function work even after the convertor
          * was moved to a specific position.
          */
-        opal_convertor_get_current_pointer( pConvertor, (void**)&iov[0].iov_base );
+        opal_convertor_get_current_pointer(pConvertor, (void **) &iov[0].iov_base);
         iov[0].iov_len = pConvertor->local_size - pConvertor->bConverted;
         *length = iov[0].iov_len;
         pConvertor->bConverted = pConvertor->local_size;
         pConvertor->flags |= CONVERTOR_COMPLETED;
         *iov_count = 1;
-        return 1;  /* we're done */
+        return 1; /* we're done */
     }
 
-    DO_DEBUG( opal_output( 0, "opal_convertor_raw( %p, {%p, %" PRIu32 "}, %"PRIsize_t " )\n", (void*)pConvertor,
-                           (void*)iov, *iov_count, *length ); );
+    DO_DEBUG(opal_output(0, "opal_convertor_raw( %p, {%p, %" PRIu32 "}, %" PRIsize_t " )\n",
+                         (void *) pConvertor, (void *) iov, *iov_count, *length););
 
     description = pConvertor->use_desc->desc;
 
@@ -105,91 +106,99 @@ opal_convertor_raw( opal_convertor_t* pConvertor,
      * due to the fact that the convertor can stop in the middle of a data with a count
      */
     pStack = pConvertor->pStack + pConvertor->stack_pos;
-    pos_desc     = pStack->index;
-    source_base  = pConvertor->pBaseBuf + pStack->disp;
-    count_desc   = pStack->count;
+    pos_desc = pStack->index;
+    source_base = pConvertor->pBaseBuf + pStack->disp;
+    count_desc = pStack->count;
     pStack--;
     pConvertor->stack_pos--;
     pElem = &(description[pos_desc]);
 
-    DO_DEBUG( opal_output( 0, "raw start pos_desc %d count_desc %" PRIsize_t " disp %ld\n"
-                           "stack_pos %d pos_desc %d count_desc %" PRIsize_t " disp %ld\n",
-                           pos_desc, count_desc, (long)(source_base - pConvertor->pBaseBuf),
-                           pConvertor->stack_pos, pStack->index, pStack->count, (long)pStack->disp ); );
+    DO_DEBUG(opal_output(0,
+                         "raw start pos_desc %d count_desc %" PRIsize_t " disp %ld\n"
+                         "stack_pos %d pos_desc %d count_desc %" PRIsize_t " disp %ld\n",
+                         pos_desc, count_desc, (long) (source_base - pConvertor->pBaseBuf),
+                         pConvertor->stack_pos, pStack->index, pStack->count,
+                         (long) pStack->disp););
 
     iov[index].iov_len = 0;
     /* Special case if we start from a position that is in the middle of a data element blocklen.
      * We can treat this outside the loop as it is an exception that can only happen once,
      * and will simplify the loop handling.
      */
-    if( pElem->elem.common.flags & OPAL_DATATYPE_FLAG_DATA ) {
-        const ddt_elem_desc_t* current = &(pElem->elem);
+    if (pElem->elem.common.flags & OPAL_DATATYPE_FLAG_DATA) {
+        const ddt_elem_desc_t *current = &(pElem->elem);
 
-        if( count_desc != ((size_t)current->count * current->blocklen) ) {  /* Not the full element description */
-            if( (do_now = count_desc % current->blocklen) ) {
-                do_now = current->blocklen - do_now;  /* how much left in the block */
+        if (count_desc
+            != ((size_t) current->count
+                * current->blocklen)) { /* Not the full element description */
+            if ((do_now = count_desc % current->blocklen)) {
+                do_now = current->blocklen - do_now; /* how much left in the block */
                 source_base += current->disp;
                 blength = do_now * opal_datatype_basicDatatypes[current->common.type]->size;
-                OPAL_DATATYPE_SAFEGUARD_POINTER( source_base, blength, pConvertor->pBaseBuf,
-                                                 pConvertor->pDesc, pConvertor->count );
-                DO_DEBUG( opal_output( 0, "raw 1. iov[%d] = {base %p, length %" PRIsize_t "}\n",
-                                       index, (void*)source_base, blength ); );
-                opal_convertor_merge_iov( iov, iov_count,
-                                          (IOVBASE_TYPE *) source_base, blength, &index );
+                OPAL_DATATYPE_SAFEGUARD_POINTER(source_base, blength, pConvertor->pBaseBuf,
+                                                pConvertor->pDesc, pConvertor->count);
+                DO_DEBUG(opal_output(0, "raw 1. iov[%d] = {base %p, length %" PRIsize_t "}\n",
+                                     index, (void *) source_base, blength););
+                opal_convertor_merge_iov(iov, iov_count, (IOVBASE_TYPE *) source_base, blength,
+                                         &index);
                 /* ignore the return value, we know there was at least one element in the iovec */
                 sum_iov_len += blength;
                 count_desc -= do_now;
 
-                source_base += (blength - current->blocklen * opal_datatype_basicDatatypes[current->common.type]->size +
-                                current->extent - current->disp);
+                source_base += (blength
+                                - current->blocklen
+                                      * opal_datatype_basicDatatypes[current->common.type]->size
+                                + current->extent - current->disp);
             }
         }
     }
 
-    while( 1 ) {
-        while( pElem->elem.common.flags & OPAL_DATATYPE_FLAG_DATA ) {
-            const ddt_elem_desc_t* current = &(pElem->elem);
+    while (1) {
+        while (pElem->elem.common.flags & OPAL_DATATYPE_FLAG_DATA) {
+            const ddt_elem_desc_t *current = &(pElem->elem);
             source_base += current->disp;
 
             do_now = current->count;
-            if( count_desc != ((size_t)current->count * current->blocklen) ) {
+            if (count_desc != ((size_t) current->count * current->blocklen)) {
                 do_now = count_desc / current->blocklen;
-                assert( 0 == (count_desc % current->blocklen) );
+                assert(0 == (count_desc % current->blocklen));
             }
 
             blength = current->blocklen * opal_datatype_basicDatatypes[current->common.type]->size;
-            for(size_t _i = 0; _i < do_now; _i++ ) {
-                OPAL_DATATYPE_SAFEGUARD_POINTER( source_base, blength, pConvertor->pBaseBuf,
-                                                 pConvertor->pDesc, pConvertor->count );
-                DO_DEBUG( opal_output( 0, "raw 2. iov[%d] = {base %p, length %" PRIsize_t "}\n",
-                                       index, (void*)source_base, blength ); );
-                if( opal_convertor_merge_iov( iov, iov_count,
-                                              (IOVBASE_TYPE *) source_base, blength, &index ) )
-                    break;  /* no more iovec available, bail out */
+            for (size_t _i = 0; _i < do_now; _i++) {
+                OPAL_DATATYPE_SAFEGUARD_POINTER(source_base, blength, pConvertor->pBaseBuf,
+                                                pConvertor->pDesc, pConvertor->count);
+                DO_DEBUG(opal_output(0, "raw 2. iov[%d] = {base %p, length %" PRIsize_t "}\n",
+                                     index, (void *) source_base, blength););
+                if (opal_convertor_merge_iov(iov, iov_count, (IOVBASE_TYPE *) source_base, blength,
+                                             &index)) {
+                    break; /* no more iovec available, bail out */
+                }
 
                 source_base += current->extent;
                 sum_iov_len += blength;
                 count_desc -= current->blocklen;
             }
 
-            if( 0 == count_desc ) {  /* completed */
+            if (0 == count_desc) { /* completed */
                 source_base = pConvertor->pBaseBuf + pStack->disp;
-                pos_desc++;  /* advance to the next data */
-                UPDATE_INTERNAL_COUNTERS( description, pos_desc, pElem, count_desc );
+                pos_desc++; /* advance to the next data */
+                UPDATE_INTERNAL_COUNTERS(description, pos_desc, pElem, count_desc);
                 continue;
             }
             source_base -= current->disp;
             goto complete_loop;
         }
-        if( OPAL_DATATYPE_END_LOOP == pElem->elem.common.type ) { /* end of the current loop */
-            DO_DEBUG( opal_output( 0, "raw end_loop count %" PRIsize_t " stack_pos %d"
-                                   " pos_desc %d disp %ld space %" PRIsize_t "\n",
-                                   pStack->count, pConvertor->stack_pos,
-                                   pos_desc, (long)pStack->disp, sum_iov_len ); );
-            if( --(pStack->count) == 0 ) { /* end of loop */
-                if( 0 == pConvertor->stack_pos ) {
+        if (OPAL_DATATYPE_END_LOOP == pElem->elem.common.type) { /* end of the current loop */
+            DO_DEBUG(opal_output(0,
+                                 "raw end_loop count %" PRIsize_t " stack_pos %d"
+                                 " pos_desc %d disp %ld space %" PRIsize_t "\n",
+                                 pStack->count, pConvertor->stack_pos, pos_desc,
+                                 (long) pStack->disp, sum_iov_len););
+            if (--(pStack->count) == 0) { /* end of loop */
+                if (0 == pConvertor->stack_pos) {
                     /* we're done. Force the exit of the main for loop (around iovec) */
-                    index++;  /* account for the currently updating iovec */
+                    index++; /* account for the currently updating iovec */
                     goto complete_loop;
                 }
                 pConvertor->stack_pos--;
@@ -197,32 +206,35 @@ opal_convertor_raw( opal_convertor_t* pConvertor,
                 pos_desc++;
             } else {
                 pos_desc = pStack->index + 1;
-                if( pStack->index == -1 ) {
+                if (pStack->index == -1) {
                     pStack->disp += (pData->ub - pData->lb);
                 } else {
-                    assert( OPAL_DATATYPE_LOOP == description[pStack->index].loop.common.type );
-                    pStack->disp += description[pStack->index].loop.extent;  /* jump by the loop extent */
+                    assert(OPAL_DATATYPE_LOOP == description[pStack->index].loop.common.type);
+                    pStack->disp += description[pStack->index]
+                                        .loop.extent; /* jump by the loop extent */
                 }
             }
             source_base = pConvertor->pBaseBuf + pStack->disp;
-            UPDATE_INTERNAL_COUNTERS( description, pos_desc, pElem, count_desc );
-            DO_DEBUG( opal_output( 0, "raw new_loop count %" PRIsize_t " stack_pos %d "
-                                   "pos_desc %d disp %ld space %" PRIsize_t "\n",
-                                   pStack->count, pConvertor->stack_pos,
-                                   pos_desc, (long)pStack->disp, sum_iov_len ); );
+            UPDATE_INTERNAL_COUNTERS(description, pos_desc, pElem, count_desc);
+            DO_DEBUG(opal_output(0,
+                                 "raw new_loop count %" PRIsize_t " stack_pos %d "
+                                 "pos_desc %d disp %ld space %" PRIsize_t "\n",
+                                 pStack->count, pConvertor->stack_pos, pos_desc,
+                                 (long) pStack->disp, sum_iov_len););
         }
-        if( OPAL_DATATYPE_LOOP == pElem->elem.common.type ) {
-            ptrdiff_t local_disp = (ptrdiff_t)source_base;
-            ddt_endloop_desc_t* end_loop = (ddt_endloop_desc_t*)(pElem + pElem->loop.items);
+        if (OPAL_DATATYPE_LOOP == pElem->elem.common.type) {
+            ptrdiff_t local_disp = (ptrdiff_t) source_base;
+            ddt_endloop_desc_t *end_loop = (ddt_endloop_desc_t *) (pElem + pElem->loop.items);
 
-            if( pElem->loop.common.flags & OPAL_DATATYPE_FLAG_CONTIGUOUS ) {
+            if (pElem->loop.common.flags & OPAL_DATATYPE_FLAG_CONTIGUOUS) {
                 ptrdiff_t offset = end_loop->first_elem_disp;
                 source_base += offset;
-                for(; count_desc > 0; ) {
-                    OPAL_DATATYPE_SAFEGUARD_POINTER( source_base, end_loop->size, pConvertor->pBaseBuf,
-                                                     pConvertor->pDesc, pConvertor->count );
-                    if( opal_convertor_merge_iov( iov, iov_count,
-                                                  (IOVBASE_TYPE *) source_base, end_loop->size, &index ) ) {
+                for (; count_desc > 0;) {
+                    OPAL_DATATYPE_SAFEGUARD_POINTER(source_base, end_loop->size,
+                                                    pConvertor->pBaseBuf, pConvertor->pDesc,
+                                                    pConvertor->count);
+                    if (opal_convertor_merge_iov(iov, iov_count, (IOVBASE_TYPE *) source_base,
+                                                 end_loop->size, &index)) {
                         source_base -= offset;
                         goto complete_loop;
                     }
@@ -230,36 +242,41 @@ opal_convertor_raw( opal_convertor_t* pConvertor,
                     source_base += pElem->loop.extent;
                     sum_iov_len += end_loop->size;
                     count_desc--;
-                    DO_DEBUG( opal_output( 0, "raw contig loop generate iov[%d] = {base %p, length %" PRIsize_t "}"
-                                           "space %" PRIsize_t " [pos_desc %d]\n",
-                                           index, iov[index].iov_base, iov[index].iov_len,
-                                           sum_iov_len, pos_desc ); );
+                    DO_DEBUG(opal_output(
+                                 0,
+                                 "raw contig loop generate iov[%d] = {base %p, length %" PRIsize_t
+                                 "}"
+                                 "space %" PRIsize_t " [pos_desc %d]\n",
+                                 index, iov[index].iov_base, iov[index].iov_len, sum_iov_len,
+                                 pos_desc););
                 }
                 source_base -= offset;
                 pos_desc += pElem->loop.items + 1;
             } else {
-                local_disp = (ptrdiff_t)source_base - local_disp;
-                PUSH_STACK( pStack, pConvertor->stack_pos, pos_desc, OPAL_DATATYPE_LOOP, count_desc,
-                            pStack->disp + local_disp);
+                local_disp = (ptrdiff_t) source_base - local_disp;
+                PUSH_STACK(pStack, pConvertor->stack_pos, pos_desc, OPAL_DATATYPE_LOOP, count_desc,
+                           pStack->disp + local_disp);
                 pos_desc++;
             }
             source_base = pConvertor->pBaseBuf + pStack->disp;
-            UPDATE_INTERNAL_COUNTERS( description, pos_desc, pElem, count_desc );
-            DDT_DUMP_STACK( pConvertor->pStack, pConvertor->stack_pos, pElem, "advance loop" );
+            UPDATE_INTERNAL_COUNTERS(description, pos_desc, pElem, count_desc);
+            DDT_DUMP_STACK(pConvertor->pStack, pConvertor->stack_pos, pElem, "advance loop");
         }
     }
- complete_loop:
-    pConvertor->bConverted += sum_iov_len;  /* update the already converted bytes */
+complete_loop:
+    pConvertor->bConverted += sum_iov_len; /* update the already converted bytes */
     *length = sum_iov_len;
     *iov_count = index;
-    if( pConvertor->bConverted == pConvertor->local_size ) {
+    if (pConvertor->bConverted == pConvertor->local_size) {
         pConvertor->flags |= CONVERTOR_COMPLETED;
         return 1;
     }
     /* I complete an element, next step I should go to the next one */
-    PUSH_STACK( pStack, pConvertor->stack_pos, pos_desc, OPAL_DATATYPE_UINT1, count_desc,
-                source_base - pConvertor->pBaseBuf );
-    DO_DEBUG( opal_output( 0, "raw save stack stack_pos %d pos_desc %d count_desc %" PRIsize_t " disp %ld\n",
-                           pConvertor->stack_pos, pStack->index, pStack->count, (long)pStack->disp ); );
+    PUSH_STACK(pStack, pConvertor->stack_pos, pos_desc, OPAL_DATATYPE_UINT1, count_desc,
+               source_base - pConvertor->pBaseBuf);
+    DO_DEBUG(
+        opal_output(0,
+                    "raw save stack stack_pos %d pos_desc %d count_desc %" PRIsize_t " disp %ld\n",
+                    pConvertor->stack_pos, pStack->index, pStack->count, (long) pStack->disp););
     return 0;
 }
diff --git a/opal/datatype/opal_copy_functions.c b/opal/datatype/opal_copy_functions.c
index ad315f787e4..fec19a2a70e 100644
--- a/opal/datatype/opal_copy_functions.c
+++ b/opal/datatype/opal_copy_functions.c
@@ -19,11 +19,11 @@
 
 #include <stddef.h>
 
-#include "opal/datatype/opal_datatype.h"
 #include "opal/datatype/opal_convertor.h"
-#include "opal/datatype/opal_datatype_internal.h"
-#include "opal/datatype/opal_datatype_checksum.h"
 #include "opal/datatype/opal_convertor_internal.h"
+#include "opal/datatype/opal_datatype.h"
+#include "opal/datatype/opal_datatype_checksum.h"
+#include "opal/datatype/opal_datatype_internal.h"
 
 /*
  * This function is used to copy data from one buffer to another.  The assumption
@@ -39,43 +39,42 @@
  *
  * Return value: Number of elements of type TYPE copied
  */
-#define COPY_TYPE( TYPENAME, TYPE, COUNT )                                              \
-static int copy_##TYPENAME( opal_convertor_t *pConvertor, size_t count,                 \
-                            char* from, size_t from_len, ptrdiff_t from_extent,         \
-                            char* to, size_t to_len, ptrdiff_t to_extent,               \
-                            ptrdiff_t *advance)                                         \
-{                                                                                       \
-    size_t remote_TYPE_size = sizeof(TYPE) * (COUNT); /* TODO */                        \
-    size_t local_TYPE_size = (COUNT) * sizeof(TYPE);                                    \
-                                                                                        \
-    /* make sure the remote buffer is large enough to hold the data */                  \
-    if( (remote_TYPE_size * count) > from_len ) {                                       \
-        count = from_len / remote_TYPE_size;                                            \
-        if( (count * remote_TYPE_size) != from_len ) {                                  \
-            DUMP( "oops should I keep this data somewhere (excedent %d bytes)?\n",      \
-                  from_len - (count * remote_TYPE_size) );                              \
-        }                                                                               \
-        DUMP( "correct: copy %s count %d from buffer %p with length %d to %p space %d\n", \
-              #TYPE, count, from, from_len, to, to_len );                               \
-    } else                                                                              \
-        DUMP( "         copy %s count %d from buffer %p with length %d to %p space %d\n", \
-              #TYPE, count, from, from_len, to, to_len );                               \
-                                                                                        \
-    if( (from_extent == (ptrdiff_t)local_TYPE_size) &&                          \
-        (to_extent == (ptrdiff_t)remote_TYPE_size) ) {                          \
-        /* copy of contigous data at both source and destination */                     \
-        MEMCPY( to, from, count * local_TYPE_size );                                    \
-    } else {                                                                            \
-        /* source or destination are non-contigous */                                   \
-        for(size_t i = 0; i < count; i++ ) {                                            \
-            MEMCPY( to, from, local_TYPE_size );                                        \
-            to += to_extent;                                                            \
-            from += from_extent;                                                        \
-        }                                                                               \
-    }                                                                                   \
-    *advance = count * from_extent;                                                     \
-    return count;                                                                       \
-}
+#define COPY_TYPE(TYPENAME, TYPE, COUNT)                                                        \
+    static int copy_##TYPENAME(opal_convertor_t *pConvertor, size_t count, char *from,          \
+                               size_t from_len, ptrdiff_t from_extent, char *to, size_t to_len, \
+                               ptrdiff_t to_extent, ptrdiff_t *advance)                         \
+    {                                                                                           \
+        size_t remote_TYPE_size = sizeof(TYPE) * (COUNT); /* TODO */                            \
+        size_t local_TYPE_size = (COUNT) * sizeof(TYPE);                                        \
+                                                                                                \
+        /* make sure the remote buffer is large enough to hold the data */                      \
+        if ((remote_TYPE_size * count) > from_len) {                                            \
+            count = from_len / remote_TYPE_size;                                                \
+            if ((count * remote_TYPE_size) != from_len) {                                       \
+                DUMP("oops should I keep this data somewhere (excedent %d bytes)?\n",           \
+                     from_len - (count * remote_TYPE_size));                                    \
+            }                                                                                   \
+            DUMP("correct: copy %s count %d from buffer %p with length %d to %p space %d\n",    \
+                 #TYPE, count, from, from_len, to, to_len);                                     \
+        } else                                                                                  \
+            DUMP("         copy %s count %d from buffer %p with length %d to %p space %d\n",    \
+                 #TYPE, count, from, from_len, to, to_len);                                     \
+                                                                                                \
+        if ((from_extent == (ptrdiff_t) local_TYPE_size)                                        \
+            && (to_extent == (ptrdiff_t) remote_TYPE_size)) {                                   \
+            /* copy of contiguous data at both source and destination */                         \
+            MEMCPY(to, from, count *local_TYPE_size);                                           \
+        } else {                                                                                \
+            /* source or destination are non-contiguous */                                       \
+            for (size_t i = 0; i < count; i++) {                                                \
+                MEMCPY(to, from, local_TYPE_size);                                              \
+                to += to_extent;                                                                \
+                from += from_extent;                                                            \
+            }                                                                                   \
+        }                                                                                       \
+        *advance = count * from_extent;                                                         \
+        return count;                                                                           \
+    }
 
 /*
  * This function is used to copy data from one buffer to another.  The assumption
@@ -91,195 +90,201 @@ static int copy_##TYPENAME( opal_convertor_t *pConvertor, size_t count,
  *
  * Return value: Number of elements of type TYPE copied
  */
-#define COPY_CONTIGUOUS_BYTES( TYPENAME, COUNT )                                          \
-static size_t copy_##TYPENAME##_##COUNT( opal_convertor_t *pConvertor, size_t count,         \
-                                         char* from, size_t from_len, ptrdiff_t from_extent, \
-                                         char* to, size_t to_len, ptrdiff_t to_extent,       \
-                                         ptrdiff_t *advance )              \
-{                                                                               \
-    size_t remote_TYPE_size = (size_t)(COUNT); /* TODO */                       \
-    size_t local_TYPE_size = (size_t)(COUNT);                                   \
-                                                                                \
-    if( (remote_TYPE_size * count) > from_len ) {                               \
-        count = from_len / remote_TYPE_size;                                    \
-        if( (count * remote_TYPE_size) != from_len ) {                          \
-            DUMP( "oops should I keep this data somewhere (excedent %d bytes)?\n", \
-                  from_len - (count * remote_TYPE_size) );                      \
-        }                                                                       \
-        DUMP( "correct: copy %s count %d from buffer %p with length %d to %p space %d\n", \
-              #TYPENAME, count, from, from_len, to, to_len );                   \
-    } else                                                                      \
-        DUMP( "         copy %s count %d from buffer %p with length %d to %p space %d\n", \
-              #TYPENAME, count, from, from_len, to, to_len );                   \
-                                                                                \
-    if( (from_extent == (ptrdiff_t)local_TYPE_size) &&                  \
-        (to_extent == (ptrdiff_t)remote_TYPE_size) ) {                  \
-        MEMCPY( to, from, count * local_TYPE_size );                            \
-    } else {                                                                    \
-        for(size_t i = 0; i < count; i++ ) {                                    \
-            MEMCPY( to, from, local_TYPE_size );                                \
-            to += to_extent;                                                    \
-            from += from_extent;                                                \
-        }                                                                       \
-    }                                                                           \
-    *advance = count * from_extent;                                             \
-    return count;                                                               \
-}
+#define COPY_CONTIGUOUS_BYTES(TYPENAME, COUNT)                                                  \
+    static size_t copy_##TYPENAME##_##COUNT(opal_convertor_t *pConvertor, size_t count,         \
+                                            char *from, size_t from_len, ptrdiff_t from_extent, \
+                                            char *to, size_t to_len, ptrdiff_t to_extent,       \
+                                            ptrdiff_t *advance)                                 \
+    {                                                                                           \
+        size_t remote_TYPE_size = (size_t)(COUNT); /* TODO */                                   \
+        size_t local_TYPE_size = (size_t)(COUNT);                                               \
+                                                                                                \
+        if ((remote_TYPE_size * count) > from_len) {                                            \
+            count = from_len / remote_TYPE_size;                                                \
+            if ((count * remote_TYPE_size) != from_len) {                                       \
+                DUMP("oops should I keep this data somewhere (excedent %d bytes)?\n",           \
+                     from_len - (count * remote_TYPE_size));                                    \
+            }                                                                                   \
+            DUMP("correct: copy %s count %d from buffer %p with length %d to %p space %d\n",    \
+                 #TYPENAME, count, from, from_len, to, to_len);                                 \
+        } else                                                                                  \
+            DUMP("         copy %s count %d from buffer %p with length %d to %p space %d\n",    \
+                 #TYPENAME, count, from, from_len, to, to_len);                                 \
+                                                                                                \
+        if ((from_extent == (ptrdiff_t) local_TYPE_size)                                        \
+            && (to_extent == (ptrdiff_t) remote_TYPE_size)) {                                   \
+            MEMCPY(to, from, count *local_TYPE_size);                                           \
+        } else {                                                                                \
+            for (size_t i = 0; i < count; i++) {                                                \
+                MEMCPY(to, from, local_TYPE_size);                                              \
+                to += to_extent;                                                                \
+                from += from_extent;                                                            \
+            }                                                                                   \
+        }                                                                                       \
+        *advance = count * from_extent;                                                         \
+        return count;                                                                           \
+    }
 
 /* set up copy functions for the basic C MPI data types */
 /* per default, select all of them */
-#define REQUIRE_COPY_BYTES_1 1
-#define REQUIRE_COPY_BYTES_2 1
-#define REQUIRE_COPY_BYTES_4 1
-#define REQUIRE_COPY_BYTES_8 1
+#define REQUIRE_COPY_BYTES_1  1
+#define REQUIRE_COPY_BYTES_2  1
+#define REQUIRE_COPY_BYTES_4  1
+#define REQUIRE_COPY_BYTES_8  1
 #define REQUIRE_COPY_BYTES_16 1
 
 #if REQUIRE_COPY_BYTES_1
-COPY_CONTIGUOUS_BYTES( bytes, 1 )
+COPY_CONTIGUOUS_BYTES(bytes, 1)
 #endif
 #if REQUIRE_COPY_BYTES_2
-COPY_CONTIGUOUS_BYTES( bytes, 2 )
+COPY_CONTIGUOUS_BYTES(bytes, 2)
 #endif
 #if REQUIRE_COPY_BYTES_4
-COPY_CONTIGUOUS_BYTES( bytes, 4 )
+COPY_CONTIGUOUS_BYTES(bytes, 4)
 #endif
 #if REQUIRE_COPY_BYTES_8
-COPY_CONTIGUOUS_BYTES( bytes, 8 )
+COPY_CONTIGUOUS_BYTES(bytes, 8)
 #endif
 #if REQUIRE_COPY_BYTES_16
-COPY_CONTIGUOUS_BYTES( bytes, 16 )
+COPY_CONTIGUOUS_BYTES(bytes, 16)
 #endif
 
 #if defined(HAVE_SHORT_FLOAT) && SIZEOF_SHORT_FLOAT == 2
-COPY_TYPE( float_2, short float, 1 )
+COPY_TYPE(float_2, short float, 1)
 #elif SIZEOF_FLOAT == 2
-COPY_TYPE( float_2, float, 1 )
+COPY_TYPE(float_2, float, 1)
 #elif SIZEOF_DOUBLE == 2
-COPY_TYPE( float_2, double, 1 )
+COPY_TYPE(float_2, double, 1)
 #elif SIZEOF_LONG_DOUBLE == 2
-COPY_TYPE( float_2, long double, 1 )
+COPY_TYPE(float_2, long double, 1)
 #elif defined(HAVE_OPAL_SHORT_FLOAT_T) && SIZEOF_OPAL_SHORT_FLOAT_T == 2
-COPY_TYPE( float_2, opal_short_float_t, 1 )
+COPY_TYPE(float_2, opal_short_float_t, 1)
 #else
 /* #error No basic type for copy function for opal_datatype_float2 found */
-#define copy_float_2 NULL
+#    define copy_float_2 NULL
 #endif
 
 #if defined(HAVE_SHORT_FLOAT) && SIZEOF_SHORT_FLOAT == 4
-COPY_TYPE( float_4, short float, 1 )
+COPY_TYPE(float_4, short float, 1)
 #elif SIZEOF_FLOAT == 4
-COPY_TYPE( float_4, float, 1 )
+COPY_TYPE(float_4, float, 1)
 #elif SIZEOF_DOUBLE == 4
-COPY_TYPE( float_4, double, 1 )
+COPY_TYPE(float_4, double, 1)
 #elif SIZEOF_LONG_DOUBLE == 4
-COPY_TYPE( float_4, long double, 1 )
+COPY_TYPE(float_4, long double, 1)
 #elif defined(HAVE_OPAL_SHORT_FLOAT_T) && SIZEOF_OPAL_SHORT_FLOAT_T == 4
-COPY_TYPE( float_4, opal_short_float_t, 1 )
+COPY_TYPE(float_4, opal_short_float_t, 1)
 #else
-#error No basic type for copy function for opal_datatype_float4 found
+#    error No basic type for copy function for opal_datatype_float4 found
 #endif
 
 #if defined(HAVE_SHORT_FLOAT) && SIZEOF_SHORT_FLOAT == 8
-COPY_TYPE( float_8, short float, 1 )
+COPY_TYPE(float_8, short float, 1)
 #elif SIZEOF_FLOAT == 8
-COPY_TYPE( float_8, float, 1 )
+COPY_TYPE(float_8, float, 1)
 #elif SIZEOF_DOUBLE == 8
-COPY_TYPE( float_8, double, 1 )
+COPY_TYPE(float_8, double, 1)
 #elif SIZEOF_LONG_DOUBLE == 8
-COPY_TYPE( float_8, long double, 1 )
+COPY_TYPE(float_8, long double, 1)
 #elif defined(HAVE_OPAL_SHORT_FLOAT_T) && SIZEOF_OPAL_SHORT_FLOAT_T == 8
-COPY_TYPE( float_8, opal_short_float_t, 1 )
+COPY_TYPE(float_8, opal_short_float_t, 1)
 #else
-#error No basic type for copy function for opal_datatype_float8 found
+#    error No basic type for copy function for opal_datatype_float8 found
 #endif
 
 #if defined(HAVE_SHORT_FLOAT) && SIZEOF_SHORT_FLOAT == 12
-COPY_TYPE( float_12, short float, 1 )
+COPY_TYPE(float_12, short float, 1)
 #elif SIZEOF_FLOAT == 12
-COPY_TYPE( float_12, float, 1 )
+COPY_TYPE(float_12, float, 1)
 #elif SIZEOF_DOUBLE == 12
-COPY_TYPE( float_12, double, 1 )
+COPY_TYPE(float_12, double, 1)
 #elif SIZEOF_LONG_DOUBLE == 12
-COPY_TYPE( float_12, long double, 1 )
+COPY_TYPE(float_12, long double, 1)
 #elif defined(HAVE_OPAL_SHORT_FLOAT_T) && SIZEOF_OPAL_SHORT_FLOAT_T == 12
-COPY_TYPE( float_12, opal_short_float_t, 1 )
+COPY_TYPE(float_12, opal_short_float_t, 1)
 #else
 /* #error No basic type for copy function for opal_datatype_float12 found */
-#define copy_float_12 NULL
+#    define copy_float_12 NULL
 #endif
 
 #if defined(HAVE_SHORT_FLOAT) && SIZEOF_SHORT_FLOAT == 16
-COPY_TYPE( float_16, short float, 1 )
+COPY_TYPE(float_16, short float, 1)
 #elif SIZEOF_FLOAT == 16
-COPY_TYPE( float_16, float, 1 )
+COPY_TYPE(float_16, float, 1)
 #elif SIZEOF_DOUBLE == 16
-COPY_TYPE( float_16, double, 1 )
+COPY_TYPE(float_16, double, 1)
 #elif SIZEOF_LONG_DOUBLE == 16
-COPY_TYPE( float_16, long double, 1 )
+COPY_TYPE(float_16, long double, 1)
 #elif defined(HAVE_OPAL_SHORT_FLOAT_T) && SIZEOF_OPAL_SHORT_FLOAT_T == 16
-COPY_TYPE( float_16, opal_short_float_t, 1 )
+COPY_TYPE(float_16, opal_short_float_t, 1)
 #else
 /* #error No basic type for copy function for opal_datatype_float16 found */
-#define copy_float_16 NULL
+#    define copy_float_16 NULL
 #endif
 
 #if defined(HAVE_SHORT_FLOAT__COMPLEX)
-COPY_TYPE ( short_float_complex, short float _Complex, 1)
+COPY_TYPE(short_float_complex, short float _Complex, 1)
 #elif defined(HAVE_OPAL_SHORT_FLOAT_COMPLEX_T)
-COPY_TYPE ( short_float_complex, opal_short_float_complex_t, 1)
+COPY_TYPE(short_float_complex, opal_short_float_complex_t, 1)
 #else
 /* #error No basic type for copy function for opal_datatype_short_float_complex found */
-#define copy_short_float_complex NULL
+#    define copy_short_float_complex NULL
 #endif
 
-COPY_TYPE ( float_complex, float _Complex, 1)
+COPY_TYPE(float_complex, float _Complex, 1)
 
-COPY_TYPE ( double_complex, double _Complex, 1)
+COPY_TYPE(double_complex, double _Complex, 1)
 
-COPY_TYPE ( long_double_complex, long double _Complex, 1)
+COPY_TYPE(long_double_complex, long double _Complex, 1)
 
 #if SIZEOF__BOOL == SIZEOF_CHAR
-COPY_TYPE (bool, char, 1)
+COPY_TYPE(bool, char, 1)
 #elif SIZEOF__BOOL == SIZEOF_SHORT
-COPY_TYPE (bool, short, 1)
+COPY_TYPE(bool, short, 1)
 #elif SIZEOF__BOOL == SIZEOF_INT
-COPY_TYPE (bool, int, 1)
+COPY_TYPE(bool, int, 1)
 #elif SIZEOF__BOOL == SIZEOF_LONG
-COPY_TYPE (bool, long, 1)
+COPY_TYPE(bool, long, 1)
 #else
-#error No basic type for copy function for opal_datatype_bool found
+#    error No basic type for copy function for opal_datatype_bool found
 #endif
 
-COPY_TYPE (wchar, wchar_t, 1)
-
+COPY_TYPE(wchar, wchar_t, 1)
 
 /* Table of predefined copy functions - one for each OPAL type */
 /* NOTE: The order of this array *MUST* match the order in opal_datatype_basicDatatypes */
 conversion_fct_t opal_datatype_copy_functions[OPAL_DATATYPE_MAX_PREDEFINED] = {
-    (conversion_fct_t)NULL,                      /* OPAL_DATATYPE_LOOP         */
-    (conversion_fct_t)NULL,                      /* OPAL_DATATYPE_END_LOOP     */
-    (conversion_fct_t)NULL,                      /* OPAL_DATATYPE_LB           */
-    (conversion_fct_t)NULL,                      /* OPAL_DATATYPE_UB           */
-    (conversion_fct_t)copy_bytes_1,              /* OPAL_DATATYPE_INT1         */
-    (conversion_fct_t)copy_bytes_2,              /* OPAL_DATATYPE_INT2         */
-    (conversion_fct_t)copy_bytes_4,              /* OPAL_DATATYPE_INT4         */
-    (conversion_fct_t)copy_bytes_8,              /* OPAL_DATATYPE_INT8         */
-    (conversion_fct_t)copy_bytes_16,             /* OPAL_DATATYPE_INT16        */
-    (conversion_fct_t)copy_bytes_1,              /* OPAL_DATATYPE_UINT1        */
-    (conversion_fct_t)copy_bytes_2,              /* OPAL_DATATYPE_UINT2        */
-    (conversion_fct_t)copy_bytes_4,              /* OPAL_DATATYPE_UINT4        */
-    (conversion_fct_t)copy_bytes_8,              /* OPAL_DATATYPE_UINT8        */
-    (conversion_fct_t)copy_bytes_16,             /* OPAL_DATATYPE_UINT16       */
-    (conversion_fct_t)copy_float_2,              /* OPAL_DATATYPE_FLOAT2       */
-    (conversion_fct_t)copy_float_4,              /* OPAL_DATATYPE_FLOAT4       */
-    (conversion_fct_t)copy_float_8,              /* OPAL_DATATYPE_FLOAT8       */
-    (conversion_fct_t)copy_float_12,             /* OPAL_DATATYPE_FLOAT12       */
-    (conversion_fct_t)copy_float_16,             /* OPAL_DATATYPE_FLOAT16      */
-    (conversion_fct_t)copy_short_float_complex,  /* OPAL_DATATYPE_SHORT_FLOAT_COMPLEX */
-    (conversion_fct_t)copy_float_complex,        /* OPAL_DATATYPE_FLOAT_COMPLEX */
-    (conversion_fct_t)copy_double_complex,       /* OPAL_DATATYPE_DOUBLE_COMPLEX */
-    (conversion_fct_t)copy_long_double_complex,  /* OPAL_DATATYPE_LONG_DOUBLE_COMPLEX */
-    (conversion_fct_t)copy_bool,                 /* OPAL_DATATYPE_BOOL         */
-    (conversion_fct_t)copy_wchar,                /* OPAL_DATATYPE_WCHAR        */
-    (conversion_fct_t)NULL                       /* OPAL_DATATYPE_UNAVAILABLE  */
+    [OPAL_DATATYPE_LOOP]                =     (conversion_fct_t) NULL,
+    [OPAL_DATATYPE_END_LOOP]            =     (conversion_fct_t) NULL,
+    [OPAL_DATATYPE_LB]                  =     (conversion_fct_t) NULL,
+    [OPAL_DATATYPE_UB]                  =     (conversion_fct_t) NULL,
+    [OPAL_DATATYPE_INT1]                =     (conversion_fct_t) copy_bytes_1,
+    [OPAL_DATATYPE_INT2]                =     (conversion_fct_t) copy_bytes_2,
+    [OPAL_DATATYPE_INT4]                =     (conversion_fct_t) copy_bytes_4,
+    [OPAL_DATATYPE_INT8]                =     (conversion_fct_t) copy_bytes_8,
+    [OPAL_DATATYPE_INT16]               =     (conversion_fct_t) copy_bytes_16,
+    [OPAL_DATATYPE_UINT1]               =     (conversion_fct_t) copy_bytes_1,
+    [OPAL_DATATYPE_UINT2]               =     (conversion_fct_t) copy_bytes_2,
+    [OPAL_DATATYPE_UINT4]               =     (conversion_fct_t) copy_bytes_4,
+    [OPAL_DATATYPE_UINT8]               =     (conversion_fct_t) copy_bytes_8,
+    [OPAL_DATATYPE_UINT16]              =     (conversion_fct_t) copy_bytes_16,
+    [OPAL_DATATYPE_FLOAT2]              =     (conversion_fct_t) copy_float_2,
+    [OPAL_DATATYPE_FLOAT4]              =     (conversion_fct_t) copy_float_4,
+    [OPAL_DATATYPE_FLOAT8]              =     (conversion_fct_t) copy_float_8,
+    [OPAL_DATATYPE_FLOAT12]             =     (conversion_fct_t) copy_float_12,
+    [OPAL_DATATYPE_FLOAT16]             =     (conversion_fct_t) copy_float_16,
+    [OPAL_DATATYPE_SHORT_FLOAT_COMPLEX] =     (conversion_fct_t) copy_short_float_complex,
+    [OPAL_DATATYPE_FLOAT_COMPLEX]       =     (conversion_fct_t) copy_float_complex,
+    [OPAL_DATATYPE_DOUBLE_COMPLEX]      =     (conversion_fct_t) copy_double_complex,
+    [OPAL_DATATYPE_LONG_DOUBLE_COMPLEX] =     (conversion_fct_t) copy_long_double_complex,
+    [OPAL_DATATYPE_BOOL]                =     (conversion_fct_t) copy_bool,
+    [OPAL_DATATYPE_WCHAR]               =     (conversion_fct_t) copy_wchar,
+#if SIZEOF_LONG == 4
+    [OPAL_DATATYPE_LONG]                =     (conversion_fct_t)copy_bytes_4,
+    [OPAL_DATATYPE_UNSIGNED_LONG]       =     (conversion_fct_t)copy_bytes_4,
+#elif SIZEOF_LONG == 8
+    [OPAL_DATATYPE_LONG]                =     (conversion_fct_t)copy_bytes_8,
+    [OPAL_DATATYPE_UNSIGNED_LONG]       =     (conversion_fct_t)copy_bytes_8,
+#endif
+    [OPAL_DATATYPE_UNAVAILABLE]         =     NULL,
 };
diff --git a/opal/datatype/opal_copy_functions_heterogeneous.c b/opal/datatype/opal_copy_functions_heterogeneous.c
index 7a3e7d57f92..fa654b20138 100644
--- a/opal/datatype/opal_copy_functions_heterogeneous.c
+++ b/opal/datatype/opal_copy_functions_heterogeneous.c
@@ -7,6 +7,8 @@
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2021      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -17,7 +19,7 @@
 #include "opal_config.h"
 
 #ifdef HAVE_IEEE754_H
-#include <ieee754.h>
+#    include <ieee754.h>
 #endif
 
 #include <stddef.h>
@@ -25,13 +27,12 @@
 
 #include "opal/util/arch.h"
 
-#include "opal/types.h"
-#include "opal/datatype/opal_datatype.h"
 #include "opal/datatype/opal_convertor.h"
-#include "opal/datatype/opal_datatype_internal.h"
-#include "opal/datatype/opal_datatype_checksum.h"
 #include "opal/datatype/opal_convertor_internal.h"
-
+#include "opal/datatype/opal_datatype.h"
+#include "opal/datatype/opal_datatype_checksum.h"
+#include "opal/datatype/opal_datatype_internal.h"
+#include "opal/types.h"
 
 /*
  * Long-Term TODO:
@@ -44,276 +45,863 @@
  * A better way would be to have a conversion registration functionality.
  */
 
-static inline void
-opal_dt_swap_bytes(void *to_p, const void *from_p, const size_t size, size_t count)
+static inline void opal_dt_swap_bytes(void *to_p, const void *from_p, const size_t size,
+                                      size_t count)
 {
     size_t i;
     size_t back_i = size - 1;
-    uint8_t *to = (uint8_t*) to_p;
-    uint8_t *from = (uint8_t*) from_p;
+    uint8_t *to = (uint8_t *) to_p;
+    uint8_t *from = (uint8_t *) from_p;
 
     /* Do the first element */
-    for (i = 0 ; i < size ; i++, back_i--) {
+    for (i = 0; i < size; i++, back_i--) {
         to[back_i] = from[i];
     }
     /* Do all the others if any */
-    while(count > 1) {
+    while (count > 1) {
         to += size;
         from += size;
         count--;
-        for (i = 0, back_i = size - 1 ; i < size ; i++, back_i--) {
+        for (i = 0, back_i = size - 1; i < size; i++, back_i--) {
             to[back_i] = from[i];
         }
     }
 }
 
+static inline void opal_dt_swap_bytes_inplace(void *buf_p, const size_t size,
+                                      size_t count)
+{
+    size_t i;
+    size_t back_i = size - 1;
+    uint8_t *buf = (uint8_t *) buf_p;
+    uint8_t copy[32];
+
+    assert(size <= 32);
+
+    /* Do the first element */
+    for (i = 0; i < size; i++) {
+        copy[i] = buf[i];
+    }
+    for (i = 0; i < size; i++, back_i--) {
+        buf[back_i] = copy[i];
+    }
+    /* Do all the others if any */
+    while (count > 1) {
+        buf += size;
+        count--;
+        for (i = 0; i < size; i++) {
+            copy[i] = buf[i];
+        }
+        for (i = 0, back_i = size - 1; i < size; i++, back_i--) {
+            buf[back_i] = copy[i];
+        }
+    }
+}
+
 #ifdef HAVE_IEEE754_H
 struct bit128 {
-    unsigned int mantissa3:32;
-    unsigned int mantissa2:32;
-    unsigned int mantissa1:32;
-    unsigned int mantissa0:16;
-    unsigned int exponent:15;
-    unsigned int negative:1;
+    unsigned int mantissa3 : 32;
+    unsigned int mantissa2 : 32;
+    unsigned int mantissa1 : 32;
+    unsigned int mantissa0 : 16;
+    unsigned int exponent : 15;
+    unsigned int negative : 1;
 };
 
 struct bit80 {
-    unsigned int pad:32;
-    unsigned int empty:16;
-    unsigned int negative:1;
-    unsigned int exponent:15;
-    unsigned int mantissa0:32;
-    unsigned int mantissa1:32;
+    unsigned int pad : 32;
+    unsigned int empty : 16;
+    unsigned int negative : 1;
+    unsigned int exponent : 15;
+    unsigned int mantissa0 : 32;
+    unsigned int mantissa1 : 32;
 };
 
-static inline void
-opal_dt_swap_long_double(void *to_p, const void *from_p, const size_t size, size_t count, uint32_t remoteArch)
+static inline void opal_dt_swap_long_double(void *to_p, const void *from_p, const size_t size,
+                                            size_t count, uint32_t remoteArch)
 {
 
-#ifdef HAVE_IEEE754_H
+#    ifdef HAVE_IEEE754_H
 
-    if ((opal_local_arch&OPAL_ARCH_LDISINTEL) && !(remoteArch&OPAL_ARCH_LDISINTEL)) {
-#ifdef __x86_64
-        long double*to = (long double *) to_p;
+    if ((opal_local_arch & OPAL_ARCH_LDISINTEL) && !(remoteArch & OPAL_ARCH_LDISINTEL)) {
+#        ifdef __x86_64
+        long double *to = (long double *) to_p;
 
-        for (size_t i=0; i<count; i++, to++) {
+        for (size_t i = 0; i < count; i++, to++) {
             union ieee854_long_double ld;
-            struct bit128 * b = (struct bit128 *)to;
+            struct bit128 *b = (struct bit128 *) to;
             ld.ieee.empty = 0;
-            ld.ieee.mantissa0 = 0x80000000 | (((unsigned int)b->mantissa0 << 15) & 0x7FFF8000) | ((b->mantissa1 >> 17) & 0x00007FFF);
-            ld.ieee.mantissa1 = ((b->mantissa1 << 15) & 0xFFFF8000) | ((b->mantissa2 << 17) & 0x000007FFF);
+            ld.ieee.mantissa0 = 0x80000000 | (((unsigned int) b->mantissa0 << 15) & 0x7FFF8000)
+                                | ((b->mantissa1 >> 17) & 0x00007FFF);
+            ld.ieee.mantissa1 = ((b->mantissa1 << 15) & 0xFFFF8000)
+                                | ((b->mantissa2 << 17) & 0x000007FFF);
             ld.ieee.exponent = b->exponent;
             ld.ieee.negative = b->negative;
-            MEMCPY( to, &ld, sizeof(long double));
+            MEMCPY(to, &ld, sizeof(long double));
         }
-#endif
-    } else if (!(opal_local_arch&OPAL_ARCH_LDISINTEL) && (remoteArch&OPAL_ARCH_LDISINTEL)) {
-#ifdef __sparcv9
-        long double*to = (long double *) to_p;
+#        endif
+    } else if (!(opal_local_arch & OPAL_ARCH_LDISINTEL) && (remoteArch & OPAL_ARCH_LDISINTEL)) {
+#        ifdef __sparcv9
+        long double *to = (long double *) to_p;
 
-        for (size_t i=0; i<count; i++, to++) {
+        for (size_t i = 0; i < count; i++, to++) {
             union ieee854_long_double ld;
-            struct bit80 * b = (struct bit80 *)to;
+            struct bit80 *b = (struct bit80 *) to;
             ld.ieee.mantissa3 = 0;
             ld.ieee.mantissa2 = 0;
             ld.ieee.mantissa0 = (b->mantissa0 << 1) | (b->mantissa1 & 0x80000000);
             ld.ieee.mantissa1 = (b->mantissa1 << 1) & 0xFFFFFFFE;
             ld.ieee.exponent = b->exponent;
             ld.ieee.negative = b->negative;
-            MEMCPY( to, &ld, sizeof(long double));
+            MEMCPY(to, &ld, sizeof(long double));
         }
+#        endif
+    }
+#    else
+    assert(0);
+#    endif
+}
+#else
+#    define opal_dt_swap_long_double(to_p, from_p, size, count, remoteArch)
+#endif
+
+union fp_float64
+{
+  double value;
+  struct {
+#if defined(WORDS_BIGENDIAN)
+    unsigned sign  :  1;
+    unsigned exp   : 11;
+    unsigned frac1 : 20;
+    unsigned frac0 : 32;
+#else
+    unsigned frac0 : 32;
+    unsigned frac1 : 20;
+    unsigned exp   : 11;
+    unsigned sign  :  1;
+#endif
+  } bits;
+  char bytes[sizeof(double)];
+};
+
+union fp_float80
+{
+  long double value;
+  struct {
+#if defined(WORDS_BIGENDIAN)
+    unsigned sign  :  1;
+    unsigned exp   : 15;
+    unsigned pad   : 16;
+    unsigned frac1 : 32;
+    unsigned frac0 : 32;
+#else
+    unsigned frac0 : 32;
+    unsigned frac1 : 32;
+    unsigned exp   : 15;
+    unsigned sign  :  1;
+    unsigned pad   : 16;
+#endif
+  } bits;
+  char bytes[sizeof(long double)];
+};
+
+union fp_float128
+{
+  /*__float128 value;*/
+  struct {
+#if defined(WORDS_BIGENDIAN)
+    unsigned sign  :  1;
+    unsigned exp   : 15;
+    unsigned frac3 : 16;
+    unsigned frac2 : 32;
+    unsigned frac1 : 32;
+    unsigned frac0 : 32;
+#else
+    unsigned frac0 : 32;
+    unsigned frac1 : 32;
+    unsigned frac2 : 32;
+    unsigned frac3 : 16;
+    unsigned exp   : 15;
+    unsigned sign  :  1;
 #endif
+  } bits;
+  char bytes[16];
+};
+
+// f64_to_f128 (copies a float64(local_endian) to a float128(local_endian))
+static inline
+void
+f64_to_f128(unsigned char *f128_buf_to, const unsigned char *f64_buf_from, ssize_t count, ptrdiff_t from_extent)
+{
+    unsigned s,e,f[4],f0,f1;
+    union fp_float64 ud;
+    union fp_float128 uq;
+    int f64_is_aligned;
+
+    f64_is_aligned = 1;
+    if ((uintptr_t)f64_buf_from & 0x7) {
+        f64_is_aligned = 0;
+    }
+    if ((uintptr_t)from_extent & 0x7) {
+        f64_is_aligned = 0;
     }
+
+    do {
+        /* input */
+        if (f64_is_aligned) {
+            ud.value = *(double*)f64_buf_from;
+        } else {
+            memcpy(&ud.value, f64_buf_from, sizeof(ud));
+        }
+
+        /* unpack */
+        s = ud.bits.sign;
+        e = ud.bits.exp;
+        f0 = ud.bits.frac0;
+        f1 = ud.bits.frac1;
+
+        /* bias */
+        if (e) e += 16383 - 1023;
+
+        /* extend */
+        f[3] = (f1 >> 4);
+        f[2] = (f1 << 28) | (f0 >> 4);
+        f[1] = (f0 << 28);
+        f[0] = 0;
+
+        /* pack */
+        uq.bits.sign  = s;
+        uq.bits.exp   = e;
+        uq.bits.frac0 = f[0];
+        uq.bits.frac1 = f[1];
+        uq.bits.frac2 = f[2];
+        uq.bits.frac3 = f[3];
+
+        /* output */
+        memcpy(f128_buf_to,uq.bytes,sizeof(uq));
+
+        f64_buf_from += from_extent;
+        f128_buf_to += sizeof(uq);
+        count--;
+   } while (count > 0);
+}
+
+// f80_to_f128 (copies an intel80(local_endian) to a float128(local_endian))
+static inline
+void
+f80_to_f128(unsigned char *f128_buf_to, const unsigned char *f80_buf_from, ssize_t count, ptrdiff_t from_extent)
+{
+    unsigned s,e,f[4],f0,f1;
+    union fp_float80 ul;
+    union fp_float128 uq;
+    int f80_is_aligned;
+
+    f80_is_aligned = 1;
+    if ((uintptr_t)f80_buf_from & 0xF) {
+        f80_is_aligned = 0;
+    }
+    if ((uintptr_t)from_extent & 0xF) {
+        f80_is_aligned = 0;
+    }
+
+    do {
+        /* input */
+        if (f80_is_aligned) {
+            ul.value = *(long double*)f80_buf_from;
+        } else {
+            memcpy(&ul.value, f80_buf_from, sizeof(ul));
+        }
+
+        /* unpack */
+        s = ul.bits.sign;
+        e = ul.bits.exp;
+        f0 = ul.bits.frac0;
+        f1 = ul.bits.frac1;
+
+        /* implicit bit */
+        f1 &= ~(1 << 31);
+
+        /* extend */
+        f[3] = (f1 >> 15);
+        f[2] = (f1 << 17) | (f0 >> 15);
+        f[1] = (f0 << 17);
+        f[0] = 0;
+
+        /* pack */
+        uq.bits.sign  = s;
+        uq.bits.exp   = e;
+        uq.bits.frac0 = f[0];
+        uq.bits.frac1 = f[1];
+        uq.bits.frac2 = f[2];
+        uq.bits.frac3 = f[3];
+
+        /* output */
+        memcpy(f128_buf_to,uq.bytes,sizeof(uq));
+
+        f80_buf_from += from_extent;
+        f128_buf_to += sizeof(uq);
+        count--;
+   } while (count > 0);
+}
+
+// f128_to_f64 (copies a float128(local_endian) to a float64(local_endian))
+static inline
+void
+f128_to_f64(unsigned char *f64_buf_to, const unsigned char *f128_buf_from, ssize_t count, ptrdiff_t to_extent)
+{
+    unsigned s,e,f[4],f0,f1;
+    union fp_float64 ud;
+    union fp_float128 uq;
+    int f64_is_aligned;
+
+    f64_is_aligned = 1;
+    if ((uintptr_t)f64_buf_to & 0x7) {
+        f64_is_aligned = 0;
+    }
+    if ((uintptr_t)to_extent & 0x7) {
+        f64_is_aligned = 0;
+    }
+
+    do {
+        /* input */
+        memcpy(uq.bytes,f128_buf_from,sizeof(uq));
+
+        /* unpack */
+        s = uq.bits.sign;
+        e = uq.bits.exp;
+        f[0] = uq.bits.frac0;
+        f[1] = uq.bits.frac1;
+        f[2] = uq.bits.frac2;
+        f[3] = uq.bits.frac3;
+
+        /* bias */
+        if (e) e -= 16383 - 1023;
+
+        /* truncate */
+        f1 = (f[3] << 4) | (f[2] >> 28);
+        f0 = (f[2] << 4) | (f[1] >> 28);
+
+        /* pack */
+        ud.bits.sign  = s;
+        ud.bits.exp   = e;
+        ud.bits.frac0 = f0;
+        ud.bits.frac1 = f1;
+
+        /* output */
+        if (f64_is_aligned) {
+            *(double*)f64_buf_to = ud.value;
+        } else {
+            memcpy(f64_buf_to, &ud.value, sizeof(ud));
+        }
+
+        f64_buf_to += to_extent;
+        f128_buf_from += sizeof(uq);
+        count--;
+    } while (count > 0);
+}
+
+// f128_to_f80 (copies a float128(local_endian) to an intel80(local_endian))
+static inline
+void
+f128_to_f80(unsigned char *f80_buf_to, const unsigned char *f128_buf_from, ssize_t count, ptrdiff_t to_extent)
+{
+    unsigned s,e,f[4],f0,f1;
+    union fp_float80 ul;
+    union fp_float128 uq;
+    int f80_is_aligned;
+
+    f80_is_aligned = 1;
+    if ((uintptr_t)f80_buf_to & 0xF) {
+        f80_is_aligned = 0;
+    }
+    if ((uintptr_t)to_extent & 0xF) {
+        f80_is_aligned = 0;
+    }
+
+    do {
+        /* input */
+        memcpy(uq.bytes,f128_buf_from,sizeof(uq));
+
+        /* unpack */
+        s = uq.bits.sign;
+        e = uq.bits.exp;
+        f[0] = uq.bits.frac0;
+        f[1] = uq.bits.frac1;
+        f[2] = uq.bits.frac2;
+        f[3] = uq.bits.frac3;
+
+        /* truncate */
+        f1 = (f[3] << 15) | (f[2] >> 17);
+        f0 = (f[2] << 15) | (f[1] >> 17);
+
+        /* implicit bit */
+        if (e)
+            f1 |= (1 << 31);
+        else
+            f1 &= ~(1 << 31);
+
+        /* pack */
+        ul.bits.sign  = s;
+        ul.bits.exp   = e;
+        ul.bits.frac0 = f0;
+        ul.bits.frac1 = f1;
+
+        /* output */
+        /* this started as *f80_buf_to = ul.value;
+           but I'm reluctant to assume alignment */
+        if (f80_is_aligned) {
+            *(long double*)f80_buf_to = ul.value;
+        } else {
+            memcpy(f80_buf_to, &ul.value, sizeof(ul));
+        }
+
+        f80_buf_to += to_extent;
+        f128_buf_from += sizeof(uq);
+        count--;
+    } while (count > 0);
+}
+
+#define LDBL_IS_F64(arch) \
+  (                                                                        \
+    (((arch) & OPAL_ARCH_LDMANTDIGISxx) == OPAL_ARCH_LDMANTDIGIS53)        \
+    &&                                                                     \
+    (((arch) & OPAL_ARCH_LDEXPSIZEISxx) == OPAL_ARCH_LDEXPSIZEIS11)        \
+  )
+#define LDBL_IS_F80(arch) \
+  (                                                                        \
+    (((arch) & OPAL_ARCH_LDMANTDIGISxx) == OPAL_ARCH_LDMANTDIGIS64)        \
+    &&                                                                     \
+    (((arch) & OPAL_ARCH_LDEXPSIZEISxx) == OPAL_ARCH_LDEXPSIZEIS15)        \
+  )
+#define LDBL_IS_F128(arch) \
+  (                                                                        \
+    (((arch) & OPAL_ARCH_LDMANTDIGISxx) == OPAL_ARCH_LDMANTDIGIS113)       \
+    &&                                                                     \
+    (((arch) & OPAL_ARCH_LDEXPSIZEISxx) == OPAL_ARCH_LDEXPSIZEIS15)        \
+  )
+#define LDBL_INFO_MASK (OPAL_ARCH_LDMANTDIGISxx | OPAL_ARCH_LDEXPSIZEISxx)
+
+#ifdef HAVE___FLOAT128
+/*
+ *  I'm not sure about the portability of alignof() so I'm handling things
+ *  like the possibility of sizeof(long double) == 12 in a slower way.  The
+ *  alignment requirement in that case would be 4 (largest power of 2 that
+ *  divides into the sizeof).
+ *
+ *  And saving it static to just compute it once without running a loop
+ *  every call.
+ */
+static inline
+size_t
+alignment_of_long_double(void) {
+    static size_t val = 0;
+
+    if (val == 0) {
+        val = 1;
+        while (sizeof(long double) % (val*2) == 0) {
+            val *= 2;
+        }
+    }
+    return val;
+}
+#endif
+
+// ldbl_to_f128 (copies a long double(from_arch format) to a float128(local_endian))
+static inline
+void
+ldbl_to_f128(unsigned char *f128_buf_to, const unsigned char *ldbl_buf_from, ssize_t count, int from_arch, ptrdiff_t from_extent)
+{
+#ifdef HAVE___FLOAT128
+    int ldbl_is_aligned;
+
+    ldbl_is_aligned = 1;
+    int alignment_mask = alignment_of_long_double() - 1;
+    if ((uintptr_t)ldbl_buf_from & alignment_mask) {
+        ldbl_is_aligned = 0;
+    }
+    if ((uintptr_t)from_extent & alignment_mask) {
+        ldbl_is_aligned = 0;
+    }
+
+    int f128_is_aligned;
+    f128_is_aligned = 1;
+    if ((uintptr_t)f128_buf_to & 0xF) {
+        f128_is_aligned = 0;
+    }
+
+    do {
+        if (ldbl_is_aligned && f128_is_aligned) {
+            *(__float128*)f128_buf_to = *(long double*)ldbl_buf_from;
+        } else {
+            __float128 f128;
+            long double ldbl;
+            memcpy(&ldbl, ldbl_buf_from, sizeof(ldbl));
+            f128 = ldbl;
+            memcpy(f128_buf_to, &f128, sizeof(f128));
+        }
+
+        ldbl_buf_from += from_extent;
+        f128_buf_to += sizeof(__float128);
+        count--;
+    } while (count > 0);
 #else
-    assert(0);
+    if (LDBL_IS_F64(from_arch)) {
+        f64_to_f128(f128_buf_to, ldbl_buf_from, count, from_extent);
+    } else if (LDBL_IS_F80(from_arch)) {
+        f80_to_f128(f128_buf_to, ldbl_buf_from, count, from_extent);
+    } else {
+/*
+ *  This could be an error condition, eg we're trying to process a
+ *  long double from a format that isn't f128 (or doesn't appear to be)
+ *  into f128.  But I think the reason not to error out is confidence
+ *  in the detection.  I wouldn't want to produce a false failure.
+ */
+        do {
+            memcpy(f128_buf_to, ldbl_buf_from, from_extent);
+
+            ldbl_buf_from += from_extent;
+            f128_buf_to += 16;
+            count--;
+        } while (count > 0);
+    }
 #endif
 }
+
+// f128_to_ldbl (copies a float128(local_endian) to a long double(to_arch format))
+static inline
+void
+f128_to_ldbl(unsigned char *ldbl_buf_to, const unsigned char *f128_buf_from, ssize_t count, int to_arch, ptrdiff_t to_extent)
+{
+#ifdef HAVE___FLOAT128
+    int ldbl_is_aligned;
+
+    ldbl_is_aligned = 1;
+    int alignment_mask = alignment_of_long_double() - 1;
+    if ((uintptr_t)ldbl_buf_to & alignment_mask) {
+        ldbl_is_aligned = 0;
+    }
+    if ((uintptr_t)to_extent & alignment_mask) {
+        ldbl_is_aligned = 0;
+    }
+
+    int f128_is_aligned;
+    f128_is_aligned = 1;
+    if ((uintptr_t)f128_buf_from & 0xF) {
+        f128_is_aligned = 0;
+    }
+
+    do {
+        if (ldbl_is_aligned && f128_is_aligned) {
+            *(long double*)ldbl_buf_to = *(__float128*)f128_buf_from;
+        } else {
+            __float128 f128;
+            long double ldbl;
+            memcpy(&f128, f128_buf_from, sizeof(f128));
+            ldbl = f128;
+            memcpy(ldbl_buf_to, &ldbl, sizeof(ldbl));
+        }
+
+        ldbl_buf_to += to_extent;
+        f128_buf_from += sizeof(__float128);
+        count--;
+    } while (count > 0);
 #else
-#define opal_dt_swap_long_double(to_p, from_p, size, count, remoteArch)
+    if (LDBL_IS_F64(to_arch)) {
+        f128_to_f64(ldbl_buf_to, f128_buf_from, count, to_extent);
+    } else if (LDBL_IS_F80(to_arch)) {
+        f128_to_f80(ldbl_buf_to, f128_buf_from, count, to_extent);
+    } else {
+/*
+ *  This could be an error condition, eg we're trying to process an
+ *  f128 into a long double of a format that isn't f128 (or doesn't
+ *  appear to be).  But I think the reason not to error out is confidence
+ *  in the detection.  I wouldn't want to produce a false failure.
+ */
+        do {
+            memcpy(ldbl_buf_to, f128_buf_from, to_extent);
+
+            ldbl_buf_to += to_extent;
+            f128_buf_from += 16;
+            count--;
+        } while (count > 0);
+    }
 #endif
+}
 
 /**
  * BEWARE: Do not use the following macro with composed types such as
  * complex. As the swap is done using the entire type sizeof, the
- * wrong endianess translation will be done.  Instead, use the
+ * wrong endianness translation will be done.  Instead, use the
  * COPY_2SAMETYPE_HETEROGENEOUS.
  */
-#define COPY_TYPE_HETEROGENEOUS( TYPENAME, TYPE )                                         \
-            COPY_TYPE_HETEROGENEOUS_INTERNAL( TYPENAME, TYPE, 0 )
-
-#define COPY_TYPE_HETEROGENEOUS_INTERNAL( TYPENAME, TYPE, LONG_DOUBLE )                   \
-static int32_t                                                                            \
-copy_##TYPENAME##_heterogeneous(opal_convertor_t *pConvertor, size_t count,               \
-                                const char* from, size_t from_len, ptrdiff_t from_extent, \
-                                char* to, size_t to_length, ptrdiff_t to_extent,          \
-                                ptrdiff_t *advance)             \
-{                                                                       \
-    size_t i;                                                           \
-                                                                        \
-    datatype_check( #TYPE, sizeof(TYPE), sizeof(TYPE), &count,          \
-                   from, from_len, from_extent,                         \
-                   to, to_length, to_extent);                           \
-                                                                        \
-    if ((pConvertor->remoteArch & OPAL_ARCH_ISBIGENDIAN) !=             \
-        (opal_local_arch & OPAL_ARCH_ISBIGENDIAN)) {                    \
-        if( (to_extent == from_extent) && (to_extent == sizeof(TYPE)) ) { \
-            opal_dt_swap_bytes(to, from, sizeof(TYPE), count);          \
-            if (LONG_DOUBLE) {                                          \
-                opal_dt_swap_long_double(to, from, sizeof(TYPE), count, pConvertor->remoteArch);\
-            }                                                           \
-        } else {                                                        \
-            for( i = 0; i < count; i++ ) {                              \
-                opal_dt_swap_bytes(to, from, sizeof(TYPE), 1);          \
-                if (LONG_DOUBLE) {                                      \
-                    opal_dt_swap_long_double(to, from, sizeof(TYPE), 1, pConvertor->remoteArch);\
-                }                                                       \
-                to += to_extent;                                        \
-                from += from_extent;                                    \
-            }                                                           \
-        }                                                               \
-    } else if ((ptrdiff_t)sizeof(TYPE) == to_extent &&          \
-               (ptrdiff_t)sizeof(TYPE) == from_extent) {        \
-         MEMCPY( to, from, count * sizeof(TYPE) );                      \
-    } else {                                                            \
-         /* source or destination are non-contigous */                  \
-         for( i = 0; i < count; i++ ) {                                 \
-             MEMCPY( to, from, sizeof(TYPE) );                          \
-             to += to_extent;                                           \
-             from += from_extent;                                       \
-         }                                                              \
-    }                                                                   \
-    *advance = count * from_extent;                                     \
-    return count;                                                       \
-}
+#define COPY_TYPE_HETEROGENEOUS(TYPENAME, TYPE) COPY_TYPE_HETEROGENEOUS_INTERNAL(TYPENAME, TYPE, 0)
 
-#define COPY_2SAMETYPE_HETEROGENEOUS( TYPENAME, TYPE )                                         \
-            COPY_2SAMETYPE_HETEROGENEOUS_INTERNAL( TYPENAME, TYPE, 0)
-
-#define COPY_2SAMETYPE_HETEROGENEOUS_INTERNAL( TYPENAME, TYPE, LONG_DOUBLE)                 \
-static int32_t                                                                            \
-copy_##TYPENAME##_heterogeneous(opal_convertor_t *pConvertor, size_t count,               \
-                                const char* from, size_t from_len, ptrdiff_t from_extent, \
-                                char* to, size_t to_length, ptrdiff_t to_extent,          \
-                                ptrdiff_t *advance)             \
-{                                                                       \
-    size_t i;                                                           \
-                                                                        \
-    datatype_check( #TYPE, sizeof(TYPE), sizeof(TYPE), &count,          \
-                   from, from_len, from_extent,                         \
-                   to, to_length, to_extent);                           \
-                                                                        \
-    if ((pConvertor->remoteArch & OPAL_ARCH_ISBIGENDIAN) !=             \
-        (opal_local_arch & OPAL_ARCH_ISBIGENDIAN)) {                    \
-        if( (to_extent == from_extent) && (to_extent == (2 * sizeof(TYPE))) ) { \
-            opal_dt_swap_bytes(to, from, sizeof(TYPE), 2 * count);      \
-            if (LONG_DOUBLE) {                                          \
-                opal_dt_swap_long_double(to, from, sizeof(TYPE), 2*count, pConvertor->remoteArch);\
-            }                                                           \
-        } else {                                                        \
-            for( i = 0; i < count; i++ ) {                              \
-                opal_dt_swap_bytes(to, from, sizeof(TYPE), 2);          \
-                if (LONG_DOUBLE) {                                      \
-                    opal_dt_swap_long_double(to, from, sizeof(TYPE), 2, pConvertor->remoteArch);\
-                }                                                       \
-                to += to_extent;                                        \
-                from += from_extent;                                    \
-            }                                                           \
-        }                                                               \
-    } else if ((ptrdiff_t)sizeof(TYPE) == to_extent &&          \
-               (ptrdiff_t)sizeof(TYPE) == from_extent) {        \
-         MEMCPY( to, from, count * sizeof(TYPE) );                      \
-    } else {                                                            \
-         /* source or destination are non-contigous */                  \
-         for( i = 0; i < count; i++ ) {                                 \
-             MEMCPY( to, from, sizeof(TYPE) );                          \
-             to += to_extent;                                           \
-             from += from_extent;                                       \
-         }                                                              \
-    }                                                                   \
-    *advance = count * from_extent;                                     \
-    return count;                                                       \
-}
+/*
+ *  Summaryizing the logic of the pFunc copy functions
+ *  with regard to long doubles:
+ *
+ *  For terminology I'll use
+ *  f64 : float64 which some architectures use as their long double
+ *  f80 : x86 double extended format that uses 80 bytes, commonly used for long double
+ *  f128 : ieee quad precision, sometimes available as __float128
+ *
+ *    if !LONG_DOUBLE or both architecture have the same long double format:
+ *      byte swap based on local/remote endianness differing
+ *    else:
+ *      if from_arch is not local endianness: byte swap to local endianness
+ *      if from_arch isn't f128 : ldbl_to_f128
+ *        if we have __float128         : convert to __float128
+ *        else if from_arch LDBL is f80 : f80_to_f128
+ *        else if from_arch LDBL is f64 : f64_to_f128
+ *      if to_arch isn't f128 : f128_to_ldbl
+ *        if we have __float128         : convert from __float128 to
+ *        if to_arch LDBL is f80   : f128_to_f80
+ *        if to_arch LDBL is f64   : f128_to_f64
+ *      if to_arch is not local endianness : byte swap
+ *
+ *  And for all the above conversions the logic for handling size difference
+ *  between the from/to type is the same:
+ *    if (to_extent == from_extent == sizeof(TYPE))
+ *      opal_dt_swap_bytes(to, from, sizeof(TYPE), count);
+ *    else
+ *      loop i=0..count-1
+ *        opal_dt_swap_bytes(to, from, sizeof(TYPE), 1);
+ *        to += to_extent;
+ *        from += from_extent;
+ *  so that's handled by a do while as an outer loop.
+ */
 
-#define COPY_2TYPE_HETEROGENEOUS( TYPENAME, TYPE1, TYPE2 )              \
-static int32_t                                                          \
-copy_##TYPENAME##_heterogeneous(opal_convertor_t *pConvertor, size_t count, \
-                                const char* from, size_t from_len, ptrdiff_t from_extent, \
-                                char* to, size_t to_length, ptrdiff_t to_extent, \
-                                ptrdiff_t *advance)             \
-{                                                                       \
-    size_t i;                                                           \
-                                                                        \
-    datatype_check( #TYPENAME, sizeof(TYPE1) + sizeof(TYPE2),           \
-                   sizeof(TYPE1) + sizeof(TYPE2), &count,               \
-                   from, from_len, from_extent,                         \
-                   to, to_length, to_extent);                           \
-                                                                        \
-    if ((pConvertor->remoteArch & OPAL_ARCH_ISBIGENDIAN) !=             \
-        (opal_local_arch & OPAL_ARCH_ISBIGENDIAN)) {                    \
-        /* source and destination are different endianness */           \
-        for( i = 0; i < count; i++ ) {                                  \
-            TYPE1* to_1, *from_1;                                       \
-            TYPE2* to_2, *from_2;                                       \
-            to_1 = (TYPE1*) to; from_1 = (TYPE1*) from;                 \
-            opal_dt_swap_bytes(to_1, from_1, sizeof(TYPE1), 1);         \
-            to_2 = (TYPE2*) (to_1 + 1); from_2 = (TYPE2*) (from_1 + 1); \
-            opal_dt_swap_bytes(to_2, from_2, sizeof(TYPE2), 1);         \
-            to += to_extent;                                            \
-            from += from_extent;                                        \
+#define COPY_TYPE_HETEROGENEOUS_INTERNAL(TYPENAME, TYPE, LONG_DOUBLE)                              \
+    static int32_t copy_##TYPENAME##_heterogeneous(opal_convertor_t *pConvertor, size_t count,     \
+                                                   const char *from, size_t from_len,              \
+                                                   ptrdiff_t from_extent, char *to,                \
+                                                   size_t to_length, ptrdiff_t to_extent,          \
+                                                   ptrdiff_t *advance)                            \
+    {                                                                                              \
+        size_t countperblock, nblocksleft;                                                         \
+        int from_arch, to_arch ;                                        \
+        if (pConvertor->flags & CONVERTOR_SEND_CONVERSION) { /* pack */ \
+            from_arch = opal_local_arch;                                \
+            to_arch = pConvertor->remoteArch;                           \
+        } else { /* unpack */                                           \
+            from_arch = pConvertor->remoteArch;                         \
+            to_arch = opal_local_arch;                                  \
         }                                                               \
-    } else if ((ptrdiff_t)(sizeof(TYPE1) + sizeof(TYPE2)) == to_extent &&   \
-               (ptrdiff_t)(sizeof(TYPE1) + sizeof(TYPE2)) == from_extent) { \
-        /* source and destination are contigous */                      \
-        MEMCPY( to, from, count * (sizeof(TYPE1) + sizeof(TYPE2)) );    \
-    } else {                                                            \
-        /* source or destination are non-contigous */                   \
-        for( i = 0; i < count; i++ ) {                                  \
-            MEMCPY( to, from, sizeof(TYPE1) + sizeof(TYPE2) );          \
-            to += to_extent;                                            \
-            from += from_extent;                                        \
+        datatype_check(#TYPE, sizeof(TYPE), sizeof(TYPE), &count, from, from_len, from_extent, to, \
+                       to_length, to_extent);                                                      \
+        if ((to_extent == from_extent) && (to_extent == sizeof(TYPE))) {                           \
+            countperblock = count;                                                                 \
+            nblocksleft = 1;                                                                       \
+        } else {                                                                                   \
+            countperblock = 1;                                                                     \
+            nblocksleft = count;                                                                   \
+        }                                                                                          \
+        do {                                                                                       \
+            if (!(LONG_DOUBLE) || ((from_arch & LDBL_INFO_MASK) == (to_arch & LDBL_INFO_MASK))) {  \
+                if ((from_arch & OPAL_ARCH_ISBIGENDIAN)                                            \
+                    != (to_arch & OPAL_ARCH_ISBIGENDIAN))                                          \
+                {                                                                                  \
+                    opal_dt_swap_bytes(to, from, sizeof(TYPE), countperblock);                     \
+                } else {                                                                           \
+                    MEMCPY(to, from, countperblock * sizeof(TYPE));                                \
+                }                                                                                  \
+            } else {                                                                               \
+                const char *tmp_from = from;                                                       \
+                if ((from_arch & OPAL_ARCH_ISBIGENDIAN)                                            \
+                    != (opal_local_arch & OPAL_ARCH_ISBIGENDIAN))                                  \
+                {                                                                                  \
+                    opal_dt_swap_bytes(to, tmp_from, sizeof(TYPE), countperblock);                 \
+                    tmp_from = to;                                                                 \
+                }                                                                                  \
+                if (!LDBL_IS_F128(from_arch)) {                                                    \
+                    ldbl_to_f128((unsigned char*)to, (const unsigned char*)tmp_from,               \
+                        countperblock, from_arch, from_extent);                                    \
+                    tmp_from = to;                                                                 \
+                }                                                                                  \
+                if (!LDBL_IS_F128(to_arch)) {                                                      \
+                    f128_to_ldbl((unsigned char*)to, (const unsigned char*)tmp_from,               \
+                        countperblock, to_arch, to_extent);                                        \
+                    tmp_from = to;                                                                 \
+                }                                                                                  \
+                if ((to_arch & OPAL_ARCH_ISBIGENDIAN)                                              \
+                    != (opal_local_arch & OPAL_ARCH_ISBIGENDIAN))                                  \
+                {                                                                                  \
+                    if (tmp_from == from) {                                                        \
+                        opal_dt_swap_bytes(to, from, sizeof(TYPE), countperblock);                 \
+                    } else {                                                                       \
+                        opal_dt_swap_bytes_inplace(to, sizeof(TYPE), countperblock);               \
+                    }                                                                              \
+                }                                                                                  \
+            }                                                                                      \
+                                                                                                   \
+            to += to_extent;                                                                       \
+            from += from_extent;                                                                   \
+            nblocksleft--;                                                                         \
+        } while (nblocksleft > 0);                                                                 \
+                                                                                                   \
+        *advance = count * from_extent;                                                            \
+        return count;                                                                              \
+    }
+
+#define COPY_2SAMETYPE_HETEROGENEOUS(TYPENAME, TYPE) \
+    COPY_2SAMETYPE_HETEROGENEOUS_INTERNAL(TYPENAME, TYPE, 0)
+
+#define COPY_2SAMETYPE_HETEROGENEOUS_INTERNAL(TYPENAME, TYPE, LONG_DOUBLE)                         \
+    static int32_t copy_##TYPENAME##_heterogeneous(opal_convertor_t *pConvertor, size_t count,     \
+                                                   const char *from, size_t from_len,              \
+                                                   ptrdiff_t from_extent, char *to,                \
+                                                   size_t to_length, ptrdiff_t to_extent,          \
+                                                   ptrdiff_t *advance)                             \
+    {                                                                                              \
+        size_t countperblock, nblocksleft;                                                         \
+        int from_arch, to_arch ;                                        \
+        if (pConvertor->flags & CONVERTOR_SEND_CONVERSION) { /* pack */ \
+            from_arch = opal_local_arch;                                \
+            to_arch = pConvertor->remoteArch;                           \
+        } else { /* unpack */                                           \
+            from_arch = pConvertor->remoteArch;                         \
+            to_arch = opal_local_arch;                                  \
         }                                                               \
-    }                                                                   \
-    *advance = count * from_extent;                                     \
-    return count;                                                       \
-}
+        datatype_check(#TYPE, sizeof(TYPE), sizeof(TYPE), &count, from, from_len, from_extent, to, \
+                       to_length, to_extent);                                                      \
+        if ((to_extent == from_extent) && (to_extent == 2 * sizeof(TYPE))) {                       \
+            countperblock = count * 2;                                                             \
+            nblocksleft = 1;                                                                       \
+        } else {                                                                                   \
+            countperblock = 2;                                                                     \
+            nblocksleft = count;                                                                   \
+        }                                                                                          \
+        do {                                                                                       \
+                                                                                                   \
+            if (!(LONG_DOUBLE) || ((from_arch & LDBL_INFO_MASK) == (to_arch & LDBL_INFO_MASK))) {  \
+                if ((from_arch & OPAL_ARCH_ISBIGENDIAN)                                            \
+                    != (to_arch & OPAL_ARCH_ISBIGENDIAN))                                          \
+                {                                                                                  \
+                    opal_dt_swap_bytes(to, from, sizeof(TYPE), countperblock);                     \
+                } else {                                                                           \
+                    MEMCPY(to, from, countperblock * sizeof(TYPE));                                \
+                }                                                                                  \
+            } else {                                                                               \
+                const char *tmp_from = from;                                                       \
+                if ((from_arch & OPAL_ARCH_ISBIGENDIAN)                                            \
+                    != (opal_local_arch & OPAL_ARCH_ISBIGENDIAN))                                  \
+                {                                                                                  \
+                    opal_dt_swap_bytes(to, tmp_from, sizeof(TYPE), countperblock);                 \
+                    tmp_from = to;                                                                 \
+                }                                                                                  \
+                if (!LDBL_IS_F128(from_arch)) {                                                    \
+                    ldbl_to_f128((unsigned char*)to, (const unsigned char*)tmp_from,               \
+                        countperblock, from_arch, from_extent/2);                                  \
+                    tmp_from = to;                                                                 \
+                }                                                                                  \
+                if (!LDBL_IS_F128(to_arch)) {                                                      \
+                    f128_to_ldbl((unsigned char*)to, (const unsigned char*)tmp_from,               \
+                        countperblock, to_arch, to_extent/2);                                      \
+                    tmp_from = to;                                                                 \
+                }                                                                                  \
+                if ((to_arch & OPAL_ARCH_ISBIGENDIAN)                                              \
+                    != (opal_local_arch & OPAL_ARCH_ISBIGENDIAN))                                  \
+                {                                                                                  \
+                    if (tmp_from == from) {                                                        \
+                        opal_dt_swap_bytes(to, from, sizeof(TYPE), countperblock);                 \
+                    } else {                                                                       \
+                        opal_dt_swap_bytes_inplace(to, sizeof(TYPE), countperblock);               \
+                    }                                                                              \
+                }                                                                                  \
+            }                                                                                      \
+                                                                                                   \
+            to += to_extent;                                                                       \
+            from += from_extent;                                                                   \
+            nblocksleft--;                                                                         \
+        } while (nblocksleft > 0);                                                                 \
+                                                                                                   \
+        *advance = count * from_extent;                                                            \
+        return count;                                                                              \
+    }
 
+#define COPY_2TYPE_HETEROGENEOUS(TYPENAME, TYPE1, TYPE2)                                        \
+    static int32_t copy_##TYPENAME##_heterogeneous(opal_convertor_t *pConvertor, size_t count,  \
+                                                   const char *from, size_t from_len,           \
+                                                   ptrdiff_t from_extent, char *to,             \
+                                                   size_t to_length, ptrdiff_t to_extent,       \
+                                                   ptrdiff_t *advance)                          \
+    {                                                                                           \
+        size_t i;                                                                               \
+        int from_arch, to_arch ;                                        \
+        if (pConvertor->flags & CONVERTOR_SEND_CONVERSION) { /* pack */ \
+            from_arch = opal_local_arch;                                \
+            to_arch = pConvertor->remoteArch;                           \
+        } else { /* unpack */                                           \
+            from_arch = pConvertor->remoteArch;                         \
+            to_arch = opal_local_arch;                                  \
+        }                                                               \
+        datatype_check(#TYPENAME, sizeof(TYPE1) + sizeof(TYPE2), sizeof(TYPE1) + sizeof(TYPE2), \
+                       &count, from, from_len, from_extent, to, to_length, to_extent);          \
+                                                                                                \
+        if ((pConvertor->remoteArch & OPAL_ARCH_ISBIGENDIAN)                                    \
+            != (opal_local_arch & OPAL_ARCH_ISBIGENDIAN)) {                                     \
+            /* source and destination are different endianness */                               \
+            for (i = 0; i < count; i++) {                                                       \
+                TYPE1 *to_1, *from_1;                                                           \
+                TYPE2 *to_2, *from_2;                                                           \
+                to_1 = (TYPE1 *) to;                                                            \
+                from_1 = (TYPE1 *) from;                                                        \
+                opal_dt_swap_bytes(to_1, from_1, sizeof(TYPE1), 1);                             \
+                to_2 = (TYPE2 *) (to_1 + 1);                                                    \
+                from_2 = (TYPE2 *) (from_1 + 1);                                                \
+                opal_dt_swap_bytes(to_2, from_2, sizeof(TYPE2), 1);                             \
+                to += to_extent;                                                                \
+                from += from_extent;                                                            \
+            }                                                                                   \
+        } else if ((ptrdiff_t)(sizeof(TYPE1) + sizeof(TYPE2)) == to_extent                      \
+                   && (ptrdiff_t)(sizeof(TYPE1) + sizeof(TYPE2)) == from_extent) {              \
+            /* source and destination are contiguous */                                          \
+            MEMCPY(to, from, count *(sizeof(TYPE1) + sizeof(TYPE2)));                           \
+        } else {                                                                                \
+            /* source or destination are non-contiguous */                                       \
+            for (i = 0; i < count; i++) {                                                       \
+                MEMCPY(to, from, sizeof(TYPE1) + sizeof(TYPE2));                                \
+                to += to_extent;                                                                \
+                from += from_extent;                                                            \
+            }                                                                                   \
+        }                                                                                       \
+        *advance = count * from_extent;                                                         \
+        return count;                                                                           \
+    }
 
-static inline void
-datatype_check(char *type, size_t local_size, size_t remote_size, size_t *count,
-               const char* from, size_t from_len, ptrdiff_t from_extent,
-               char* to, size_t to_len, ptrdiff_t to_extent)
+static inline void datatype_check(char *type, size_t local_size, size_t remote_size, size_t *count,
+                                  const char *from, size_t from_len, ptrdiff_t from_extent,
+                                  char *to, size_t to_len, ptrdiff_t to_extent)
 {
     /* make sure the remote buffer is large enough to hold the data */
-    if( (remote_size * *count) > from_len ) {
+    if ((remote_size * *count) > from_len) {
         *count = from_len / remote_size;
-        if( (*count * remote_size) != from_len ) {
-            DUMP( "oops should I keep this data somewhere (excedent %d bytes)?\n",
-                  from_len - (*count * remote_size) );
+        if ((*count * remote_size) != from_len) {
+            DUMP("oops should I keep this data somewhere (excedent %d bytes)?\n",
+                 from_len - (*count * remote_size));
         }
-        DUMP( "correct: copy %s count %d from buffer %p with length %d to %p space %d\n",
-              "char", *count, from, from_len, to, to_len );
+        DUMP("correct: copy %s count %d from buffer %p with length %d to %p space %d\n", "char",
+             *count, from, from_len, to, to_len);
     } else {
-        DUMP( "         copy %s count %d from buffer %p with length %d to %p space %d\n",
-              "char", *count, from, from_len, to, to_len );
+        DUMP("         copy %s count %d from buffer %p with length %d to %p space %d\n", "char",
+             *count, from, from_len, to, to_len);
     }
 }
 
-#define CXX_BOOL_COPY_LOOP(TYPE)                        \
-    for(size_t i = 0; i < count; i++ ) {                \
-        bool *to_real = (bool*) to;                     \
-        *to_real = *((TYPE*) from) == 0 ? false : true; \
-        to += to_extent;                                \
-        from += from_extent;                            \
+#define CXX_BOOL_COPY_LOOP(TYPE)                         \
+    for (size_t i = 0; i < count; i++) {                 \
+        bool *to_real = (bool *) to;                     \
+        *to_real = *((TYPE *) from) == 0 ? false : true; \
+        to += to_extent;                                 \
+        from += from_extent;                             \
     }
-static int32_t
-copy_cxx_bool_heterogeneous(opal_convertor_t *pConvertor, size_t count,
-                            const char* from, size_t from_len, ptrdiff_t from_extent,
-                            char* to, size_t to_length, ptrdiff_t to_extent,
-                            ptrdiff_t *advance)
+static int32_t copy_cxx_bool_heterogeneous(opal_convertor_t *pConvertor, size_t count,
+                                           const char *from, size_t from_len, ptrdiff_t from_extent,
+                                           char *to, size_t to_length, ptrdiff_t to_extent,
+                                           ptrdiff_t *advance)
 {
     /* fix up the from extent */
-    if ((pConvertor->remoteArch & OPAL_ARCH_BOOLISxx) !=
-        (opal_local_arch & OPAL_ARCH_BOOLISxx)) {
+    if ((pConvertor->remoteArch & OPAL_ARCH_BOOLISxx) != (opal_local_arch & OPAL_ARCH_BOOLISxx)) {
         switch (pConvertor->remoteArch & OPAL_ARCH_BOOLISxx) {
         case OPAL_ARCH_BOOLIS8:
             from_extent = 1;
@@ -327,13 +915,12 @@ copy_cxx_bool_heterogeneous(opal_convertor_t *pConvertor, size_t count,
         }
     }
 
-    datatype_check( "bool", sizeof(bool), sizeof(bool), &count,
-                   from, from_len, from_extent,
-                   to, to_length, to_extent);
+    datatype_check("bool", sizeof(bool), sizeof(bool), &count, from, from_len, from_extent, to,
+                   to_length, to_extent);
 
-    if ((to_extent != sizeof(bool) || from_extent != sizeof(bool)) ||
-        ((pConvertor->remoteArch & OPAL_ARCH_BOOLISxx) !=
-         (opal_local_arch & OPAL_ARCH_BOOLISxx))) {
+    if ((to_extent != sizeof(bool) || from_extent != sizeof(bool))
+        || ((pConvertor->remoteArch & OPAL_ARCH_BOOLISxx)
+            != (opal_local_arch & OPAL_ARCH_BOOLISxx))) {
         switch (pConvertor->remoteArch & OPAL_ARCH_BOOLISxx) {
         case OPAL_ARCH_BOOLIS8:
             CXX_BOOL_COPY_LOOP(int8_t);
@@ -346,148 +933,369 @@ copy_cxx_bool_heterogeneous(opal_convertor_t *pConvertor, size_t count,
             break;
         }
     } else {
-        MEMCPY( to, from, count * sizeof(bool) );
+        MEMCPY(to, from, count * sizeof(bool));
     }
 
     *advance = count * from_extent;
     return count;
 }
 
-
 COPY_TYPE_HETEROGENEOUS(int1, int8_t)
 COPY_TYPE_HETEROGENEOUS(int2, int16_t)
 COPY_TYPE_HETEROGENEOUS(int4, int32_t)
 #ifdef HAVE_INT64_T
 COPY_TYPE_HETEROGENEOUS(int8, int64_t)
 #else
-#define copy_int8_heterogeneous NULL
+#    define copy_int8_heterogeneous NULL
 #endif
 
 #ifdef HAVE_INT128_T
 COPY_TYPE_HETEROGENEOUS(int16, int128_t)
 #else
-#define copy_int16_heterogeneous NULL
+#    define copy_int16_heterogeneous NULL
 #endif
 
-
 #if defined(HAVE_SHORT_FLOAT) && SIZEOF_SHORT_FLOAT == 2
-COPY_TYPE_HETEROGENEOUS( float2, short float )
+COPY_TYPE_HETEROGENEOUS(float2, short float)
 #elif SIZEOF_FLOAT == 2
-COPY_TYPE_HETEROGENEOUS( float2, float )
+COPY_TYPE_HETEROGENEOUS(float2, float)
 #elif SIZEOF_DOUBLE == 2
-COPY_TYPE_HETEROGENEOUS( float2, double )
+COPY_TYPE_HETEROGENEOUS(float2, double)
 #elif SIZEOF_LONG_DOUBLE == 2
-COPY_TYPE_HETEROGENEOUS( float2, long double )
+COPY_TYPE_HETEROGENEOUS(float2, long double)
 #elif defined(HAVE_OPAL_SHORT_FLOAT_T) && SIZEOF_OPAL_SHORT_FLOAT_T == 2
-COPY_TYPE_HETEROGENEOUS( float2, opal_short_float_t )
+COPY_TYPE_HETEROGENEOUS(float2, opal_short_float_t)
 #else
 /* #error No basic type for copy function for opal_datatype_float2 found */
-#define copy_float2_heterogeneous NULL
+#    define copy_float2_heterogeneous NULL
 #endif
 
 #if defined(HAVE_SHORT_FLOAT) && SIZEOF_SHORT_FLOAT == 4
-COPY_TYPE_HETEROGENEOUS( float4, short float )
+COPY_TYPE_HETEROGENEOUS(float4, short float)
 #elif SIZEOF_FLOAT == 4
-COPY_TYPE_HETEROGENEOUS( float4, float )
+COPY_TYPE_HETEROGENEOUS(float4, float)
 #elif SIZEOF_DOUBLE == 4
-COPY_TYPE_HETEROGENEOUS( float4, double )
+COPY_TYPE_HETEROGENEOUS(float4, double)
 #elif SIZEOF_LONG_DOUBLE == 4
-COPY_TYPE_HETEROGENEOUS( float4, long double )
+COPY_TYPE_HETEROGENEOUS(float4, long double)
 #elif defined(HAVE_OPAL_SHORT_FLOAT_T) && SIZEOF_OPAL_SHORT_FLOAT_T == 4
-COPY_TYPE_HETEROGENEOUS( float4, opal_short_float_t )
+COPY_TYPE_HETEROGENEOUS(float4, opal_short_float_t)
 #else
 /* #error No basic type for copy function for opal_datatype_float4 found */
-#define copy_float4_heterogeneous NULL
+#    define copy_float4_heterogeneous NULL
 #endif
 
 #if defined(HAVE_SHORT_FLOAT) && SIZEOF_SHORT_FLOAT == 8
-COPY_TYPE_HETEROGENEOUS( float8, short float )
+COPY_TYPE_HETEROGENEOUS(float8, short float)
 #elif SIZEOF_FLOAT == 8
-COPY_TYPE_HETEROGENEOUS( float8, float )
+COPY_TYPE_HETEROGENEOUS(float8, float)
 #elif SIZEOF_DOUBLE == 8
-COPY_TYPE_HETEROGENEOUS( float8, double )
+COPY_TYPE_HETEROGENEOUS(float8, double)
 #elif SIZEOF_LONG_DOUBLE == 8
-COPY_TYPE_HETEROGENEOUS( float8, long double )
+COPY_TYPE_HETEROGENEOUS(float8, long double)
 #elif defined(HAVE_OPAL_SHORT_FLOAT_T) && SIZEOF_OPAL_SHORT_FLOAT_T == 8
-COPY_TYPE_HETEROGENEOUS( float8, opal_short_float_t )
+COPY_TYPE_HETEROGENEOUS(float8, opal_short_float_t)
 #else
 /* #error No basic type for copy function for opal_datatype_float8 found */
-#define copy_float8_heterogeneous NULL
+#    define copy_float8_heterogeneous NULL
 #endif
 
 #if defined(HAVE_SHORT_FLOAT) && SIZEOF_SHORT_FLOAT == 12
-COPY_TYPE_HETEROGENEOUS( float12, short float )
+COPY_TYPE_HETEROGENEOUS(float12, short float)
 #elif SIZEOF_FLOAT == 12
-COPY_TYPE_HETEROGENEOUS( float12, float )
+COPY_TYPE_HETEROGENEOUS(float12, float)
 #elif SIZEOF_DOUBLE == 12
-COPY_TYPE_HETEROGENEOUS( float12, double )
+COPY_TYPE_HETEROGENEOUS(float12, double)
 #elif SIZEOF_LONG_DOUBLE == 12
-COPY_TYPE_HETEROGENEOUS( float12, long double )
+COPY_TYPE_HETEROGENEOUS(float12, long double)
 #elif defined(HAVE_OPAL_SHORT_FLOAT_T) && SIZEOF_OPAL_SHORT_FLOAT_T == 12
-COPY_TYPE_HETEROGENEOUS( float12, opal_short_float_t )
+COPY_TYPE_HETEROGENEOUS(float12, opal_short_float_t)
 #else
 /* #error No basic type for copy function for opal_datatype_float12 found */
-#define copy_float12_heterogeneous NULL
+#    define copy_float12_heterogeneous NULL
 #endif
 
 #if defined(HAVE_SHORT_FLOAT) && SIZEOF_SHORT_FLOAT == 16
-COPY_TYPE_HETEROGENEOUS( float16, short float )
+COPY_TYPE_HETEROGENEOUS(float16, short float)
 #elif SIZEOF_FLOAT == 16
-COPY_TYPE_HETEROGENEOUS( float16, float )
+COPY_TYPE_HETEROGENEOUS(float16, float)
 #elif SIZEOF_DOUBLE == 16
-COPY_TYPE_HETEROGENEOUS( float16, double )
+COPY_TYPE_HETEROGENEOUS(float16, double)
 #elif SIZEOF_LONG_DOUBLE == 16
-COPY_TYPE_HETEROGENEOUS_INTERNAL( float16, long double, 1)
+COPY_TYPE_HETEROGENEOUS_INTERNAL(float16, long double, 1)
 #elif defined(HAVE_OPAL_SHORT_FLOAT_T) && SIZEOF_OPAL_SHORT_FLOAT_T == 16
-COPY_TYPE_HETEROGENEOUS( float16, opal_short_float_t )
+COPY_TYPE_HETEROGENEOUS(float16, opal_short_float_t)
 #else
 /* #error No basic type for copy function for opal_datatype_float16 found */
-#define copy_float16_heterogeneous NULL
+#    define copy_float16_heterogeneous NULL
 #endif
 
 #if defined(HAVE_SHORT_FLOAT__COMPLEX)
-COPY_2SAMETYPE_HETEROGENEOUS( short_float_complex, short float _Complex )
+COPY_2SAMETYPE_HETEROGENEOUS(short_float_complex, short float _Complex)
 #elif defined(HAVE_OPAL_SHORT_FLOAT_COMPLEX_T)
-COPY_2SAMETYPE_HETEROGENEOUS( short_float_complex, opal_short_float_complex_t )
+COPY_2SAMETYPE_HETEROGENEOUS(short_float_complex, opal_short_float_complex_t)
 #else
 /* #error No basic type for copy function for opal_datatype_short_float_complex found */
-#define copy_short_float_complex_heterogeneous NULL
+#    define copy_short_float_complex_heterogeneous NULL
 #endif
 
-COPY_2SAMETYPE_HETEROGENEOUS( float_complex, float )
+COPY_2SAMETYPE_HETEROGENEOUS(float_complex, float)
+
+COPY_2SAMETYPE_HETEROGENEOUS(double_complex, double)
+
+COPY_2SAMETYPE_HETEROGENEOUS_INTERNAL(long_double_complex, long double, 1)
 
-COPY_2SAMETYPE_HETEROGENEOUS( double_complex, double )
+COPY_TYPE_HETEROGENEOUS(wchar, wchar_t)
 
-COPY_2SAMETYPE_HETEROGENEOUS_INTERNAL( long_double_complex, long double, 1)
+#if SIZEOF_LONG == 8
+static int32_t
+copy_long_heterogeneous(opal_convertor_t *pConvertor, size_t count,
+                        const char* from, size_t from_len, ptrdiff_t from_extent,
+                        char* to, size_t to_length, ptrdiff_t to_extent,
+                        ptrdiff_t *advance)
+{
+    size_t i;
+
+    datatype_check("long", sizeof(long), pConvertor->master->remote_sizes[OPAL_DATATYPE_LONG], &count, from, from_len, from_extent, to,
+                   to_length, to_extent);
+    if (!((pConvertor->remoteArch ^ opal_local_arch) & OPAL_ARCH_LONGIS64)) {  /* same sizeof(long) */
+        if ((pConvertor->remoteArch ^ opal_local_arch) & OPAL_ARCH_ISBIGENDIAN) {  /* different endianness */
+            for (i = 0; i < count; i++) {
+                opal_dt_swap_bytes(to, from, sizeof(long), 1);
+                to += to_extent;
+                from += from_extent;
+            }
+        } else {
+            for (i = 0; i < count; i++) {
+                *(long*)to = *(long*)from;
+                to += to_extent;
+                from += from_extent;
+            }
+        }
+    } else {
+        /* the two sides have different lengths for sizeof(long) */
+        if( CONVERTOR_SEND & pConvertor->flags ) { /* we're doing a pack */
+            assert(CONVERTOR_SEND_CONVERSION & pConvertor->flags);
+            if ((pConvertor->remoteArch ^ opal_local_arch) & OPAL_ARCH_ISBIGENDIAN) {
+                /* different sizeof, we need to convert */
+                if (opal_local_arch & OPAL_ARCH_LONGIS64) {
+                    for (i = 0; i < count; i++) { /* from 8 to 4 bytes */
+                        int64_t val = *(int64_t*)from;
+                        int32_t i32 = (int32_t)val;
+                        opal_dt_swap_bytes(to, &i32, sizeof(int32_t), 1);
+                        to += to_extent;
+                        from += from_extent;
+                    }
+                } else {
+                    for (i = 0; i < count; i++) { /* from 4 to 8 bytes */
+                        int32_t val = *(int32_t*)from;
+                        int64_t i64 = (int64_t)val;
+                        opal_dt_swap_bytes(to, &i64, sizeof(int64_t), 1);
+                        to += to_extent;
+                        from += from_extent;
+                    }
+                }
+            } else {  /* both have the same endianness */
+                if (opal_local_arch & OPAL_ARCH_LONGIS64) {
+                    for (i = 0; i < count; i++) { /* from 8 to 4 bytes */
+                        long val = *(long*)from;
+                        *(int32_t*)to = (int32_t)val;
+                        to += to_extent;
+                        from += from_extent;
+                    }
+                } else {
+                    for (i = 0; i < count; i++) { /* from 4 to 8 bytes */
+                        long val = *(long*)from;
+                        *(int64_t*)to = (int64_t)val;
+                        to += to_extent;
+                        from += from_extent;
+                    }
+                }
+            }
+        } else {  /* unpack */
+            if ((pConvertor->remoteArch ^ opal_local_arch) & OPAL_ARCH_ISBIGENDIAN) {
+                /* different endianness */
+                if (opal_local_arch & OPAL_ARCH_LONGIS64) {
+                    for (i = 0; i < count; i++) { /* from 4 to 8 bytes */
+                        int32_t val;
+                        opal_dt_swap_bytes(&val, from, sizeof(int32_t), 1);
+                        *(long*)to = (long)val;
+                        to += to_extent;
+                        from += from_extent;
+                    }
+                } else {
+                    for (i = 0; i < count; i++) { /* from 8 to 4 bytes */
+                        int64_t val;
+                        opal_dt_swap_bytes(&val, from, sizeof(int64_t), 1);
+                        *(long*)to = (long)val;
+                        to += to_extent;
+                        from += from_extent;
+                    }
+                }
+            } else {  /* both have the same endianness */
+                if (opal_local_arch & OPAL_ARCH_LONGIS64) {
+                    for (i = 0; i < count; i++) { /* from 8 to 4 bytes */
+                        int32_t val = *(int32_t*)from;
+                        *(long*)to = (long)val;
+                        to += to_extent;
+                        from += from_extent;
+                    }
+                } else {
+                    for (i = 0; i < count; i++) { /* from 4 to 8 bytes */
+                        int64_t val = *(int64_t*)from;
+                        *(long*)to = (long)val;
+                        to += to_extent;
+                        from += from_extent;
+                    }
+                }
+            }
+        }
+    }
+    *advance = count * from_extent;
+    return count;
+}
+
+static int32_t
+copy_unsigned_long_heterogeneous(opal_convertor_t *pConvertor, size_t count,
+                                 const char* from, size_t from_len, ptrdiff_t from_extent,
+                                 char* to, size_t to_length, ptrdiff_t to_extent,
+                                 ptrdiff_t *advance)
+{
+    size_t i;
 
-COPY_TYPE_HETEROGENEOUS (wchar, wchar_t)
+    datatype_check("unsigned long", sizeof(unsigned long), pConvertor->master->remote_sizes[OPAL_DATATYPE_UNSIGNED_LONG],
+                   &count, from, from_len, from_extent, to, to_length, to_extent);
+    if (!((pConvertor->remoteArch ^ opal_local_arch) & OPAL_ARCH_LONGIS64)) {  /* same sizeof(long) */
+        if ((pConvertor->remoteArch ^ opal_local_arch) & OPAL_ARCH_ISBIGENDIAN) {  /* different endianness */
+            for (i = 0; i < count; i++) {
+                opal_dt_swap_bytes(to, from, sizeof(unsigned long), 1);
+                to += to_extent;
+                from += from_extent;
+            }
+        } else {
+            for (i = 0; i < count; i++) {
+                *(unsigned long*)to = *(unsigned long*)from;
+                to += to_extent;
+                from += from_extent;
+            }
+        }
+    } else {
+        /* the two sides have different lengths for sizeof(long) */
+        if( CONVERTOR_SEND & pConvertor->flags ) { /* we're doing a pack */
+            assert(CONVERTOR_SEND_CONVERSION & pConvertor->flags);
+            if ((pConvertor->remoteArch ^ opal_local_arch) & OPAL_ARCH_ISBIGENDIAN) {
+                /* different sizeof, we need to convert */
+                if (opal_local_arch & OPAL_ARCH_LONGIS64) {
+                    for (i = 0; i < count; i++) { /* from 8 to 4 bytes */
+                        uint64_t val = *(uint64_t*)from;
+                        uint32_t i32 = (uint32_t)val;
+                        opal_dt_swap_bytes(to, &i32, sizeof(uint32_t), 1);
+                        to += to_extent;
+                        from += from_extent;
+                    }
+                } else {
+                    for (i = 0; i < count; i++) { /* from 4 to 8 bytes */
+                        uint32_t val = *(uint32_t*)from;
+                        uint64_t i64 = (uint64_t)val;
+                        opal_dt_swap_bytes(to, &i64, sizeof(uint64_t), 1);
+                        to += to_extent;
+                        from += from_extent;
+                    }
+                }
+            } else {  /* both have the same endianness */
+                if (opal_local_arch & OPAL_ARCH_LONGIS64) {
+                    for (i = 0; i < count; i++) { /* from 8 to 4 bytes */
+                        unsigned long val = *(unsigned long*)from;
+                        *(uint32_t*)to = (uint32_t)val;
+                        to += to_extent;
+                        from += from_extent;
+                    }
+                } else {
+                    for (i = 0; i < count; i++) { /* from 4 to 8 bytes */
+                        unsigned long val = *(unsigned long*)from;
+                        *(uint64_t*)to = (uint64_t)val;
+                        to += to_extent;
+                        from += from_extent;
+                    }
+                }
+            }
+        } else {  /* unpack */
+            if ((pConvertor->remoteArch ^ opal_local_arch) & OPAL_ARCH_ISBIGENDIAN) {
+                /* different endianness */
+                if (opal_local_arch & OPAL_ARCH_LONGIS64) {
+                    for (i = 0; i < count; i++) { /* from 4 to 8 bytes */
+                        uint32_t val;
+                        opal_dt_swap_bytes(&val, from, sizeof(uint32_t), 1);
+                        *(unsigned long*)to = (unsigned long)val;
+                        to += to_extent;
+                        from += from_extent;
+                    }
+                } else {
+                    for (i = 0; i < count; i++) { /* from 8 to 4 bytes */
+                        uint64_t val;
+                        opal_dt_swap_bytes(&val, from, sizeof(uint64_t), 1);
+                        *(unsigned long*)to = (unsigned long)val;
+                        to += to_extent;
+                        from += from_extent;
+                    }
+                }
+            } else {  /* both have the same endianness */
+                if (opal_local_arch & OPAL_ARCH_LONGIS64) {
+                    for (i = 0; i < count; i++) { /* from 8 to 4 bytes */
+                        uint32_t val = *(uint32_t*)from;
+                        *(unsigned long*)to = (unsigned long)val;
+                        to += to_extent;
+                        from += from_extent;
+                    }
+                } else {
+                    for (i = 0; i < count; i++) { /* from 4 to 8 bytes */
+                        uint64_t val = *(uint64_t*)from;
+                        *(unsigned long*)to = (unsigned long)val;
+                        to += to_extent;
+                        from += from_extent;
+                    }
+                }
+            }
+        }
+    }
+    *advance = count * from_extent;
+    return count;
+}
+#endif  /* SIZEOF_LONG == 8 */
 
 /* table of predefined copy functions - one for each MPI type */
 conversion_fct_t opal_datatype_heterogeneous_copy_functions[OPAL_DATATYPE_MAX_PREDEFINED] = {
-   [OPAL_DATATYPE_LOOP]                = NULL,
-   [OPAL_DATATYPE_END_LOOP]            = NULL,
-   [OPAL_DATATYPE_LB]                  = NULL,
-   [OPAL_DATATYPE_UB]                  = NULL,
-   [OPAL_DATATYPE_INT1]                = (conversion_fct_t) copy_int1_heterogeneous,
-   [OPAL_DATATYPE_INT2]                = (conversion_fct_t) copy_int2_heterogeneous,
-   [OPAL_DATATYPE_INT4]                = (conversion_fct_t) copy_int4_heterogeneous,
-   [OPAL_DATATYPE_INT8]                = (conversion_fct_t) copy_int8_heterogeneous,
-   [OPAL_DATATYPE_INT16]               = (conversion_fct_t) copy_int16_heterogeneous,
-   [OPAL_DATATYPE_UINT1]               = (conversion_fct_t) copy_int1_heterogeneous,
-   [OPAL_DATATYPE_UINT2]               = (conversion_fct_t) copy_int2_heterogeneous,
-   [OPAL_DATATYPE_UINT4]               = (conversion_fct_t) copy_int4_heterogeneous,
-   [OPAL_DATATYPE_UINT8]               = (conversion_fct_t) copy_int8_heterogeneous,
-   [OPAL_DATATYPE_UINT16]              = (conversion_fct_t) copy_int16_heterogeneous,
-   [OPAL_DATATYPE_FLOAT2]              = (conversion_fct_t) copy_float2_heterogeneous,
-   [OPAL_DATATYPE_FLOAT4]              = (conversion_fct_t) copy_float4_heterogeneous,
-   [OPAL_DATATYPE_FLOAT8]              = (conversion_fct_t) copy_float8_heterogeneous,
-   [OPAL_DATATYPE_FLOAT12]             = (conversion_fct_t) copy_float12_heterogeneous,
-   [OPAL_DATATYPE_FLOAT16]             = (conversion_fct_t) copy_float16_heterogeneous,
-   [OPAL_DATATYPE_SHORT_FLOAT_COMPLEX] = (conversion_fct_t) copy_short_float_complex_heterogeneous,
-   [OPAL_DATATYPE_FLOAT_COMPLEX]       = (conversion_fct_t) copy_float_complex_heterogeneous,
-   [OPAL_DATATYPE_DOUBLE_COMPLEX]      = (conversion_fct_t) copy_double_complex_heterogeneous,
-   [OPAL_DATATYPE_LONG_DOUBLE_COMPLEX] = (conversion_fct_t) copy_long_double_complex_heterogeneous,
-   [OPAL_DATATYPE_BOOL]                = (conversion_fct_t) copy_cxx_bool_heterogeneous,
-   [OPAL_DATATYPE_WCHAR]               = (conversion_fct_t) copy_wchar_heterogeneous,
-   [OPAL_DATATYPE_UNAVAILABLE]         = NULL,
+    [OPAL_DATATYPE_LOOP]                = NULL,
+    [OPAL_DATATYPE_END_LOOP]            = NULL,
+    [OPAL_DATATYPE_LB]                  = NULL,
+    [OPAL_DATATYPE_UB]                  = NULL,
+    [OPAL_DATATYPE_INT1]                = (conversion_fct_t) copy_int1_heterogeneous,
+    [OPAL_DATATYPE_INT2]                = (conversion_fct_t) copy_int2_heterogeneous,
+    [OPAL_DATATYPE_INT4]                = (conversion_fct_t) copy_int4_heterogeneous,
+    [OPAL_DATATYPE_INT8]                = (conversion_fct_t) copy_int8_heterogeneous,
+    [OPAL_DATATYPE_INT16]               = (conversion_fct_t) copy_int16_heterogeneous,
+    [OPAL_DATATYPE_UINT1]               = (conversion_fct_t) copy_int1_heterogeneous,
+    [OPAL_DATATYPE_UINT2]               = (conversion_fct_t) copy_int2_heterogeneous,
+    [OPAL_DATATYPE_UINT4]               = (conversion_fct_t) copy_int4_heterogeneous,
+    [OPAL_DATATYPE_UINT8]               = (conversion_fct_t) copy_int8_heterogeneous,
+    [OPAL_DATATYPE_UINT16]              = (conversion_fct_t) copy_int16_heterogeneous,
+    [OPAL_DATATYPE_FLOAT2]              = (conversion_fct_t) copy_float2_heterogeneous,
+    [OPAL_DATATYPE_FLOAT4]              = (conversion_fct_t) copy_float4_heterogeneous,
+    [OPAL_DATATYPE_FLOAT8]              = (conversion_fct_t) copy_float8_heterogeneous,
+    [OPAL_DATATYPE_FLOAT12]             = (conversion_fct_t) copy_float12_heterogeneous,
+    [OPAL_DATATYPE_FLOAT16]             = (conversion_fct_t) copy_float16_heterogeneous,
+    [OPAL_DATATYPE_SHORT_FLOAT_COMPLEX] = (conversion_fct_t) copy_short_float_complex_heterogeneous,
+    [OPAL_DATATYPE_FLOAT_COMPLEX]       = (conversion_fct_t) copy_float_complex_heterogeneous,
+    [OPAL_DATATYPE_DOUBLE_COMPLEX]      = (conversion_fct_t) copy_double_complex_heterogeneous,
+    [OPAL_DATATYPE_LONG_DOUBLE_COMPLEX] = (conversion_fct_t) copy_long_double_complex_heterogeneous,
+    [OPAL_DATATYPE_BOOL]                = (conversion_fct_t) copy_cxx_bool_heterogeneous,
+    [OPAL_DATATYPE_WCHAR]               = (conversion_fct_t) copy_wchar_heterogeneous,
+#if SIZEOF_LONG == 4
+    [OPAL_DATATYPE_LONG]                = (conversion_fct_t) copy_int4_heterogeneous,
+    [OPAL_DATATYPE_UNSIGNED_LONG]       = (conversion_fct_t) copy_int4_heterogeneous,
+#else
+    [OPAL_DATATYPE_LONG]                = (conversion_fct_t) copy_long_heterogeneous,
+    [OPAL_DATATYPE_UNSIGNED_LONG]       = (conversion_fct_t) copy_unsigned_long_heterogeneous,
+#endif
+    [OPAL_DATATYPE_UNAVAILABLE]         = NULL,
 };
diff --git a/opal/datatype/opal_datatype.h b/opal/datatype/opal_datatype.h
index 8f054f7c7b0..5f7fc53fa7d 100644
--- a/opal/datatype/opal_datatype.h
+++ b/opal/datatype/opal_datatype.h
@@ -52,37 +52,46 @@ BEGIN_C_DECLS
  * This must match the same definition as in opal_datatype_internal.h
  */
 #if !defined(OPAL_DATATYPE_MAX_PREDEFINED)
-#define OPAL_DATATYPE_MAX_PREDEFINED 26
+#    define OPAL_DATATYPE_MAX_PREDEFINED 28
 #endif
 /*
- * No more than this number of _Basic_ datatypes in C/CPP or Fortran
- * are supported (in order to not change setup and usage of the predefined
- * datatypes).
+ * Upper limit of the number of _Basic_ datatypes supported (in order to
+ * not change setup and usage of the predefined datatypes).
  *
  * BEWARE: This constant should reflect whatever the OMPI-layer needs.
  */
-#define OPAL_DATATYPE_MAX_SUPPORTED  50
-
+#define OPAL_DATATYPE_MAX_SUPPORTED 64
 
 /* flags for the datatypes. */
-#define OPAL_DATATYPE_FLAG_UNAVAILABLE   0x0001  /**< datatypes unavailable on the build (OS or compiler dependant) */
-#define OPAL_DATATYPE_FLAG_PREDEFINED    0x0002  /**< cannot be removed: initial and predefined datatypes */
-#define OPAL_DATATYPE_FLAG_COMMITTED     0x0004  /**< ready to be used for a send/recv operation */
-#define OPAL_DATATYPE_FLAG_OVERLAP       0x0008  /**< datatype is unpropper for a recv operation */
-#define OPAL_DATATYPE_FLAG_CONTIGUOUS    0x0010  /**< contiguous datatype */
-#define OPAL_DATATYPE_FLAG_NO_GAPS       0x0020  /**< no gaps around the datatype, aka OPAL_DATATYPE_FLAG_CONTIGUOUS and extent == size */
-#define OPAL_DATATYPE_FLAG_USER_LB       0x0040  /**< has a user defined LB */
-#define OPAL_DATATYPE_FLAG_USER_UB       0x0080  /**< has a user defined UB */
-#define OPAL_DATATYPE_FLAG_DATA          0x0100  /**< data or control structure */
+#define OPAL_DATATYPE_FLAG_UNAVAILABLE \
+    0x0001 /**< datatypes unavailable on the build (OS or compiler dependant) */
+#define OPAL_DATATYPE_FLAG_PREDEFINED \
+    0x0002 /**< cannot be removed: initial and predefined datatypes */
+#define OPAL_DATATYPE_FLAG_COMMITTED  0x0004 /**< ready to be used for a send/recv operation */
+#define OPAL_DATATYPE_FLAG_OVERLAP    0x0008 /**< datatype is unpropper for a recv operation */
+#define OPAL_DATATYPE_FLAG_CONTIGUOUS 0x0010 /**< contiguous datatype */
+#define OPAL_DATATYPE_FLAG_NO_GAPS                                                                \
+    0x0020 /**< no gaps around the datatype, aka OPAL_DATATYPE_FLAG_CONTIGUOUS and extent == size \
+            */
+#define OPAL_DATATYPE_FLAG_USER_LB 0x0040 /**< has a user defined LB */
+#define OPAL_DATATYPE_FLAG_USER_UB 0x0080 /**< has a user defined UB */
+#define OPAL_DATATYPE_FLAG_DATA    0x0100 /**< data or control structure */
 /*
  * We should make the difference here between the predefined contiguous and non contiguous
  * datatypes. The OPAL_DATATYPE_FLAG_BASIC is held by all predefined contiguous datatypes.
  */
-#define OPAL_DATATYPE_FLAG_BASIC         (OPAL_DATATYPE_FLAG_PREDEFINED | \
-                                          OPAL_DATATYPE_FLAG_CONTIGUOUS | \
-                                          OPAL_DATATYPE_FLAG_NO_GAPS |    \
-                                          OPAL_DATATYPE_FLAG_DATA |       \
-                                          OPAL_DATATYPE_FLAG_COMMITTED)
+#define OPAL_DATATYPE_FLAG_BASIC                                                                \
+    (OPAL_DATATYPE_FLAG_PREDEFINED | OPAL_DATATYPE_FLAG_CONTIGUOUS | OPAL_DATATYPE_FLAG_NO_GAPS \
+     | OPAL_DATATYPE_FLAG_DATA | OPAL_DATATYPE_FLAG_COMMITTED)
+/*
+ * If during the datatype optimization process we collapse contiguous elements with
+ * different types, we cannot use this optimized description for any communication
+ * in a heterogeneous setting, especially not for the exteranl32 support.
+ *
+ * A datatype with this flag cannot use the optimized description in heterogeneous
+ * setups.
+ */
+#define OPAL_DATATYPE_OPTIMIZED_RESTRICTED  0x1000
 
 /**
  * The number of supported entries in the data-type definition and the
@@ -94,44 +103,43 @@ typedef size_t opal_datatype_count_t;
 typedef union dt_elem_desc dt_elem_desc_t;
 
 struct dt_type_desc_t {
-    opal_datatype_count_t  length;  /**< the maximum number of elements in the description array */
-    opal_datatype_count_t  used;    /**< the number of used elements in the description array */
-    dt_elem_desc_t*        desc;
+    opal_datatype_count_t length; /**< the maximum number of elements in the description array */
+    opal_datatype_count_t used;   /**< the number of used elements in the description array */
+    dt_elem_desc_t *desc;
 };
 typedef struct dt_type_desc_t dt_type_desc_t;
 
-
 /*
  * The datatype description.
  */
 struct opal_datatype_t {
-    opal_object_t      super;    /**< basic superclass */
-    uint16_t           flags;    /**< the flags */
-    uint16_t           id;       /**< data id, normally the index in the data array. */
-    uint32_t           bdt_used; /**< bitset of which basic datatypes are used in the data description */
-    size_t             size;     /**< total size in bytes of the memory used by the data if
-                                      the data is put on a contiguous buffer */
-    ptrdiff_t          true_lb;  /**< the true lb of the data without user defined lb and ub */
-    ptrdiff_t          true_ub;  /**< the true ub of the data without user defined lb and ub */
-    ptrdiff_t          lb;       /**< lower bound in memory */
-    ptrdiff_t          ub;       /**< upper bound in memory */
+    opal_object_t super; /**< basic superclass */
+    uint16_t flags;      /**< the flags */
+    uint16_t id;         /**< data id, normally the index in the data array. */
+    uint32_t bdt_used;   /**< bitset of which basic datatypes are used in the data description */
+    size_t size;         /**< total size in bytes of the memory used by the data if
+                              the data is put on a contiguous buffer */
+    ptrdiff_t true_lb;   /**< the true lb of the data without user defined lb and ub */
+    ptrdiff_t true_ub;   /**< the true ub of the data without user defined lb and ub */
+    ptrdiff_t lb;        /**< lower bound in memory */
+    ptrdiff_t ub;        /**< upper bound in memory */
     /* --- cacheline 1 boundary (64 bytes) --- */
-    size_t             nbElems;  /**< total number of elements inside the datatype */
-    uint32_t           align;    /**< data should be aligned to */
-    uint32_t           loops;    /**< number of loops on the iternal type stack */
+    size_t nbElems; /**< total number of elements inside the datatype */
+    uint32_t align; /**< data should be aligned to */
+    uint32_t loops; /**< number of loops on the iternal type stack */
 
     /* Attribute fields */
-    char               name[OPAL_MAX_OBJECT_NAME];  /**< name of the datatype */
-    dt_type_desc_t     desc;     /**< the data description */
-    dt_type_desc_t     opt_desc; /**< short description of the data used when conversion is useless
-                                      or in the send case (without conversion) */
-
-    size_t             *ptypes;  /**< array of basic predefined types that facilitate the computing
-                                      of the remote size in heterogeneous environments. The length of the
-                                      array is dependent on the maximum number of predefined datatypes of
-                                      all language interfaces (because Fortran is not known at the OPAL
-                                      layer). This field should never be initialized in homogeneous
-                                      environments */
+    char name[OPAL_MAX_OBJECT_NAME]; /**< name of the datatype */
+    dt_type_desc_t desc;             /**< the data description */
+    dt_type_desc_t opt_desc; /**< short description of the data used when conversion is useless
+                                  or in the send case (without conversion) */
+
+    size_t *ptypes; /**< array of basic predefined types that facilitate the computing
+                         of the remote size in heterogeneous environments. The length of the
+                         array is dependent on the maximum number of predefined datatypes of
+                         all language interfaces (because Fortran is not known at the OPAL
+                         layer). This field should never be initialized in homogeneous
+                         environments */
     /* --- cacheline 5 boundary (320 bytes) was 32-36 bytes ago --- */
 
     /* size: 352, cachelines: 6, members: 15 */
@@ -140,9 +148,10 @@ struct opal_datatype_t {
 
 typedef struct opal_datatype_t opal_datatype_t;
 
-OPAL_DECLSPEC OBJ_CLASS_DECLARATION( opal_datatype_t );
+OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_datatype_t);
 
-OPAL_DECLSPEC extern const opal_datatype_t* opal_datatype_basicDatatypes[OPAL_DATATYPE_MAX_PREDEFINED];
+OPAL_DECLSPEC extern const opal_datatype_t
+    *opal_datatype_basicDatatypes[OPAL_DATATYPE_MAX_PREDEFINED];
 OPAL_DECLSPEC extern const size_t opal_datatype_local_sizes[OPAL_DATATYPE_MAX_PREDEFINED];
 
 /* Local Architecture as provided by opal_arch_compute_local_id() */
@@ -156,60 +165,57 @@ OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_loop;
 OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_end_loop;
 OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_lb;
 OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_ub;
-OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_int1;       /* in bytes */
-OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_int2;       /* in bytes */
-OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_int4;       /* in bytes */
-OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_int8;       /* in bytes */
-OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_int16;      /* in bytes */
-OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_uint1;      /* in bytes */
-OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_uint2;      /* in bytes */
-OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_uint4;      /* in bytes */
-OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_uint8;      /* in bytes */
-OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_uint16;     /* in bytes */
-OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_float2;     /* in bytes */
-OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_float4;     /* in bytes */
-OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_float8;     /* in bytes */
-OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_float12;    /* in bytes */
-OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_float16;    /* in bytes */
+OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_int1;    /* in bytes */
+OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_int2;    /* in bytes */
+OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_int4;    /* in bytes */
+OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_int8;    /* in bytes */
+OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_int16;   /* in bytes */
+OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_uint1;   /* in bytes */
+OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_uint2;   /* in bytes */
+OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_uint4;   /* in bytes */
+OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_uint8;   /* in bytes */
+OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_uint16;  /* in bytes */
+OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_float2;  /* in bytes */
+OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_float4;  /* in bytes */
+OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_float8;  /* in bytes */
+OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_float12; /* in bytes */
+OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_float16; /* in bytes */
 OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_short_float_complex;
 OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_float_complex;
 OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_double_complex;
 OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_long_double_complex;
 OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_bool;
 OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_wchar;
-
+OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_long;
+OPAL_DECLSPEC extern const opal_datatype_t opal_datatype_unsigned_long;
 
 /*
  * Functions exported externally
  */
 int opal_datatype_register_params(void);
-OPAL_DECLSPEC int32_t opal_datatype_init( void );
-OPAL_DECLSPEC opal_datatype_t* opal_datatype_create( int32_t expectedSize );
-OPAL_DECLSPEC int32_t opal_datatype_create_desc( opal_datatype_t * datatype, int32_t expectedSize );
-OPAL_DECLSPEC int32_t opal_datatype_commit( opal_datatype_t * pData );
-OPAL_DECLSPEC int32_t opal_datatype_destroy( opal_datatype_t** );
-OPAL_DECLSPEC int32_t opal_datatype_is_monotonic( opal_datatype_t* type);
-
-static inline int32_t
-opal_datatype_is_committed( const opal_datatype_t* type )
+OPAL_DECLSPEC int32_t opal_datatype_init(void);
+OPAL_DECLSPEC opal_datatype_t *opal_datatype_create(int32_t expectedSize);
+OPAL_DECLSPEC int32_t opal_datatype_create_desc(opal_datatype_t *datatype, int32_t expectedSize);
+OPAL_DECLSPEC int32_t opal_datatype_commit(opal_datatype_t *pData);
+OPAL_DECLSPEC int32_t opal_datatype_destroy(opal_datatype_t **);
+OPAL_DECLSPEC int32_t opal_datatype_is_monotonic(opal_datatype_t *type);
+
+static inline int32_t opal_datatype_is_committed(const opal_datatype_t *type)
 {
     return ((type->flags & OPAL_DATATYPE_FLAG_COMMITTED) == OPAL_DATATYPE_FLAG_COMMITTED);
 }
 
-static inline int32_t
-opal_datatype_is_overlapped( const opal_datatype_t* type )
+static inline int32_t opal_datatype_is_overlapped(const opal_datatype_t *type)
 {
     return ((type->flags & OPAL_DATATYPE_FLAG_OVERLAP) == OPAL_DATATYPE_FLAG_OVERLAP);
 }
 
-static inline int32_t
-opal_datatype_is_valid( const opal_datatype_t* type )
+static inline int32_t opal_datatype_is_valid(const opal_datatype_t *type)
 {
     return !((type->flags & OPAL_DATATYPE_FLAG_UNAVAILABLE) == OPAL_DATATYPE_FLAG_UNAVAILABLE);
 }
 
-static inline int32_t
-opal_datatype_is_predefined( const opal_datatype_t* type )
+static inline int32_t opal_datatype_is_predefined(const opal_datatype_t *type)
 {
     return (type->flags & OPAL_DATATYPE_FLAG_PREDEFINED);
 }
@@ -218,103 +224,101 @@ opal_datatype_is_predefined( const opal_datatype_t* type )
  * This function return true (1) if the datatype representation depending on the count
  * is contiguous in the memory. And false (0) otherwise.
  */
-static inline int32_t
-opal_datatype_is_contiguous_memory_layout( const opal_datatype_t* datatype, int32_t count )
+static inline int32_t opal_datatype_is_contiguous_memory_layout(const opal_datatype_t *datatype,
+                                                                int32_t count)
 {
-    if( !(datatype->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS) ) return 0;
-    if( (count == 1) || (datatype->flags & OPAL_DATATYPE_FLAG_NO_GAPS) ) return 1;
+    if (!(datatype->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS))
+        return 0;
+    if ((count == 1) || (datatype->flags & OPAL_DATATYPE_FLAG_NO_GAPS))
+        return 1;
     return 0;
 }
 
-
-OPAL_DECLSPEC void
-opal_datatype_dump( const opal_datatype_t* pData );
+OPAL_DECLSPEC void opal_datatype_dump(const opal_datatype_t *pData);
 
 /* data creation functions */
 
 /**
  * Create a duplicate of the source datatype.
  */
-OPAL_DECLSPEC int32_t
-opal_datatype_clone( const opal_datatype_t* src_type,
-                     opal_datatype_t* dest_type );
+OPAL_DECLSPEC int32_t opal_datatype_clone(const opal_datatype_t *src_type,
+                                          opal_datatype_t *dest_type);
 /**
  * A contiguous array of identical datatypes.
  */
-OPAL_DECLSPEC int32_t
-opal_datatype_create_contiguous( int count, const opal_datatype_t* oldType,
-                                 opal_datatype_t** newType );
+OPAL_DECLSPEC int32_t opal_datatype_create_contiguous(int count, const opal_datatype_t *oldType,
+                                                      opal_datatype_t **newType);
 /**
  * Add a new datatype to the base type description. The count is the number
  * repetitions of the same element to be added, and the extent is the extent
  * of each element. The displacement is the initial displacement of the
  * first element.
  */
-OPAL_DECLSPEC int32_t
-opal_datatype_add( opal_datatype_t* pdtBase,
-                   const opal_datatype_t* pdtAdd, size_t count,
-                   ptrdiff_t disp, ptrdiff_t extent );
+OPAL_DECLSPEC int32_t opal_datatype_add(opal_datatype_t *pdtBase, const opal_datatype_t *pdtAdd,
+                                        size_t count, ptrdiff_t disp, ptrdiff_t extent);
 
 /**
  * Alter the lb and extent of an existing datatype in place.
  */
-OPAL_DECLSPEC int32_t
-opal_datatype_resize( opal_datatype_t* type,
-                      ptrdiff_t lb,
-                      ptrdiff_t extent );
+OPAL_DECLSPEC int32_t opal_datatype_resize(opal_datatype_t *type, ptrdiff_t lb, ptrdiff_t extent);
 
-static inline int32_t
-opal_datatype_type_lb( const opal_datatype_t* pData, ptrdiff_t* disp )
+static inline int32_t opal_datatype_type_lb(const opal_datatype_t *pData, ptrdiff_t *disp)
 {
     *disp = pData->lb;
     return 0;
 }
 
-static inline int32_t
-opal_datatype_type_ub( const opal_datatype_t* pData, ptrdiff_t* disp )
+static inline int32_t opal_datatype_type_ub(const opal_datatype_t *pData, ptrdiff_t *disp)
 {
     *disp = pData->ub;
     return 0;
 }
 
-static inline int32_t
-opal_datatype_type_size( const opal_datatype_t* pData, size_t *size )
+static inline int32_t opal_datatype_type_size(const opal_datatype_t *pData, size_t *size)
 {
     *size = pData->size;
     return 0;
 }
 
-static inline int32_t
-opal_datatype_type_extent( const opal_datatype_t* pData, ptrdiff_t* extent )
+static inline int32_t opal_datatype_type_extent(const opal_datatype_t *pData, ptrdiff_t *extent)
 {
     *extent = pData->ub - pData->lb;
     return 0;
 }
 
-static inline int32_t
-opal_datatype_get_extent( const opal_datatype_t* pData, ptrdiff_t* lb, ptrdiff_t* extent)
+static inline int32_t opal_datatype_get_extent(const opal_datatype_t *pData, ptrdiff_t *lb,
+                                               ptrdiff_t *extent)
 {
-    *lb = pData->lb; *extent = pData->ub - pData->lb;
+    *lb = pData->lb;
+    *extent = pData->ub - pData->lb;
     return 0;
 }
 
-static inline int32_t
-opal_datatype_get_true_extent( const opal_datatype_t* pData, ptrdiff_t* true_lb, ptrdiff_t* true_extent)
+static inline int32_t opal_datatype_get_true_extent(const opal_datatype_t *pData,
+                                                    ptrdiff_t *true_lb, ptrdiff_t *true_extent)
 {
     *true_lb = pData->true_lb;
     *true_extent = (pData->true_ub - pData->true_lb);
     return 0;
 }
 
-OPAL_DECLSPEC ssize_t
-opal_datatype_get_element_count( const opal_datatype_t* pData, size_t iSize );
-OPAL_DECLSPEC int32_t
-opal_datatype_set_element_count( const opal_datatype_t* pData, size_t count, size_t* length );
-OPAL_DECLSPEC int32_t
-opal_datatype_copy_content_same_ddt( const opal_datatype_t* pData, int32_t count,
-                                     char* pDestBuf, char* pSrcBuf );
+OPAL_DECLSPEC ssize_t opal_datatype_get_element_count(const opal_datatype_t *pData, size_t iSize);
+OPAL_DECLSPEC int32_t opal_datatype_set_element_count(const opal_datatype_t *pData, size_t count,
+                                                      size_t *length);
+OPAL_DECLSPEC int32_t opal_datatype_copy_content_same_ddt(const opal_datatype_t *pData,
+                                                          int32_t count, char *pDestBuf,
+                                                          char *pSrcBuf);
 
-OPAL_DECLSPEC int opal_datatype_compute_ptypes( opal_datatype_t* datatype );
+OPAL_DECLSPEC int opal_datatype_compute_ptypes(opal_datatype_t *datatype);
+
+/*
+ * Compute the size of the datatype using a specific set of predefined type sizes.
+ * This function allows to compute the size of a packed buffer without creating
+ * a fully fledged specialized convertor for the remote peer.
+ */
+OPAL_DECLSPEC size_t
+opal_datatype_compute_remote_size(const opal_datatype_t *pData,
+                                  const size_t *sizes);
 
 /* Compute the span in memory of count datatypes. This function help with temporary
  * memory allocations for receiving already typed data (such as those used for reduce
@@ -325,9 +329,8 @@ OPAL_DECLSPEC int opal_datatype_compute_ptypes( opal_datatype_t* datatype );
  * Returns: the memory span of count repetition of the datatype, and in the gap
  *          argument, the number of bytes of the gap at the beginning.
  */
-static inline ptrdiff_t
-opal_datatype_span( const opal_datatype_t* pData, size_t count,
-                    ptrdiff_t* gap)
+static inline ptrdiff_t opal_datatype_span(const opal_datatype_t *pData, size_t count,
+                                           ptrdiff_t *gap)
 {
     if (OPAL_UNLIKELY(0 == pData->size) || (0 == count)) {
         *gap = 0;
@@ -344,12 +347,12 @@ opal_datatype_span( const opal_datatype_t* pData, size_t count,
  * Set a breakpoint to this function in your favorite debugger
  * to make it stop on all pack and unpack errors.
  */
-OPAL_DECLSPEC int
-opal_datatype_safeguard_pointer_debug_breakpoint( const void* actual_ptr, int length,
-                                                  const void* initial_ptr,
-                                                  const opal_datatype_t* pData,
-                                                  int count );
-#endif  /* OPAL_ENABLE_DEBUG */
+OPAL_DECLSPEC int opal_datatype_safeguard_pointer_debug_breakpoint(const void *actual_ptr,
+                                                                   int length,
+                                                                   const void *initial_ptr,
+                                                                   const opal_datatype_t *pData,
+                                                                   int count);
+#endif /* OPAL_ENABLE_DEBUG */
 
 END_C_DECLS
-#endif  /* OPAL_DATATYPE_H_HAS_BEEN_INCLUDED */
+#endif /* OPAL_DATATYPE_H_HAS_BEEN_INCLUDED */
diff --git a/opal/datatype/opal_datatype_add.c b/opal/datatype/opal_datatype_add.c
index 7038d37a4c4..2618ad3ba5b 100644
--- a/opal/datatype/opal_datatype_add.c
+++ b/opal/datatype/opal_datatype_add.c
@@ -26,66 +26,88 @@
 #include <stddef.h>
 
 #include "opal/constants.h"
-#include "opal/util/output.h"
 #include "opal/datatype/opal_datatype.h"
 #include "opal/datatype/opal_datatype_internal.h"
+#include "opal/util/output.h"
 
 /* macros to play with the flags */
-#define SET_CONTIGUOUS_FLAG( INT_VALUE )     (INT_VALUE) = (INT_VALUE) | (OPAL_DATATYPE_FLAG_CONTIGUOUS)
-#define SET_NO_GAP_FLAG( INT_VALUE )         (INT_VALUE) = (INT_VALUE) | (OPAL_DATATYPE_FLAG_NO_GAPS)
-#define UNSET_CONTIGUOUS_FLAG( INT_VALUE )   (INT_VALUE) = (INT_VALUE) & (~(OPAL_DATATYPE_FLAG_CONTIGUOUS | OPAL_DATATYPE_FLAG_NO_GAPS))
+#define SET_CONTIGUOUS_FLAG(INT_VALUE) (INT_VALUE) = (INT_VALUE) | (OPAL_DATATYPE_FLAG_CONTIGUOUS)
+#define SET_NO_GAP_FLAG(INT_VALUE)     (INT_VALUE) = (INT_VALUE) | (OPAL_DATATYPE_FLAG_NO_GAPS)
+#define UNSET_CONTIGUOUS_FLAG(INT_VALUE) \
+    (INT_VALUE) = (INT_VALUE) & (~(OPAL_DATATYPE_FLAG_CONTIGUOUS | OPAL_DATATYPE_FLAG_NO_GAPS))
 
 #if defined(__GNUC__) && !defined(__STDC__)
-#define LMAX(A,B)  ({ ptrdiff_t _a = (A), _b = (B); (_a < _b ? _b : _a) })
-#define LMIN(A,B)  ({ ptrdiff_t _a = (A), _b = (B); (_a < _b ? _a : _b); })
-#define IMAX(A,B)  ({ int _a = (A), _b = (B); (_a < _b ? _b : _a); })
+#    define LMAX(A, B)                    \
+        ({                                \
+            ptrdiff_t _a = (A), _b = (B); \
+            (_a < _b ? _b : _a)           \
+        })
+#    define LMIN(A, B)                    \
+        ({                                \
+            ptrdiff_t _a = (A), _b = (B); \
+            (_a < _b ? _a : _b);          \
+        })
+#    define IMAX(A, B)              \
+        ({                          \
+            int _a = (A), _b = (B); \
+            (_a < _b ? _b : _a);    \
+        })
 #else
-static inline ptrdiff_t LMAX( ptrdiff_t a, ptrdiff_t b ) { return ( a < b ? b : a ); }
-static inline ptrdiff_t LMIN( ptrdiff_t a, ptrdiff_t b ) { return ( a < b ? a : b ); }
-static inline int  IMAX( int a, int b ) { return ( a < b ? b : a ); }
-#endif  /* __GNU__ */
-
-#define OPAL_DATATYPE_COMPUTE_REQUIRED_ENTRIES( _pdtAdd, _count, _extent, _place_needed) \
-{ \
-    if( (_pdtAdd)->flags & OPAL_DATATYPE_FLAG_PREDEFINED ) { /* add a basic datatype */ \
-        (_place_needed) = ((_extent) == (ptrdiff_t)(_pdtAdd)->size ? 1 : 3); \
-    } else { \
-        (_place_needed) = (_pdtAdd)->desc.used; \
-        if( (_count) != 1 ) { \
-            if( (_place_needed) < (MAX_DT_COMPONENT_COUNT - 2) ) { \
-                (_place_needed) += 2;  /* for the loop markers */ \
-            } else { \
-                /* The data-type contain too many elements. We will be unable \
-                 * to handle it, so let's just complain by now. \
-                 */ \
-                opal_output( 0, "Too many elements in the datatype. The limit is %ud\n", \
-                             MAX_DT_COMPONENT_COUNT ); \
-                return OPAL_ERROR; \
-            } \
-        } \
-    } \
+static inline ptrdiff_t LMAX(ptrdiff_t a, ptrdiff_t b)
+{
+    return (a < b ? b : a);
 }
-
-#define OPAL_DATATYPE_LB_UB_CONT( _count, _disp, _old_lb, _old_ub, _old_extent, _new_lb, _new_ub ) \
-{ \
-    if( 0 == _count ) { \
-        _new_lb = (_old_lb) + (_disp); \
-        _new_ub = (_old_ub) + (_disp); \
-    } else { \
-        ptrdiff_t lower, upper; \
-        upper = (_disp) + (_old_extent) * ((_count) - 1); \
-        lower = (_disp); \
-        if( lower < upper ) { \
-            _new_lb = lower; \
-            _new_ub = upper; \
-         } else { \
-            _new_lb = upper; \
-            _new_ub = lower; \
-         } \
-         _new_lb += (_old_lb); \
-         _new_ub += (_old_ub); \
-    }\
+static inline ptrdiff_t LMIN(ptrdiff_t a, ptrdiff_t b)
+{
+    return (a < b ? a : b);
 }
+static inline int IMAX(int a, int b)
+{
+    return (a < b ? b : a);
+}
+#endif /* __GNU__ */
+
+#define OPAL_DATATYPE_COMPUTE_REQUIRED_ENTRIES(_pdtAdd, _count, _extent, _place_needed)     \
+    {                                                                                       \
+        if ((_pdtAdd)->flags & OPAL_DATATYPE_FLAG_PREDEFINED) { /* add a basic datatype */  \
+            (_place_needed) = ((_extent) == (ptrdiff_t)(_pdtAdd)->size ? 1 : 3);            \
+        } else {                                                                            \
+            (_place_needed) = (_pdtAdd)->desc.used;                                         \
+            if ((_count) != 1) {                                                            \
+                if ((_place_needed) < (MAX_DT_COMPONENT_COUNT - 2)) {                       \
+                    (_place_needed) += 2; /* for the loop markers */                        \
+                } else {                                                                    \
+                    /* The data-type contain too many elements. We will be unable           \
+                     * to handle it, so let's just complain by now.                         \
+                     */                                                                     \
+                    opal_output(0, "Too many elements in the datatype. The limit is %ud\n", \
+                                MAX_DT_COMPONENT_COUNT);                                    \
+                    return OPAL_ERROR;                                                      \
+                }                                                                           \
+            }                                                                               \
+        }                                                                                   \
+    }
+
+#define OPAL_DATATYPE_LB_UB_CONT(_count, _disp, _old_lb, _old_ub, _old_extent, _new_lb, _new_ub) \
+    {                                                                                            \
+        if (0 == _count) {                                                                       \
+            _new_lb = (_old_lb) + (_disp);                                                       \
+            _new_ub = (_old_ub) + (_disp);                                                       \
+        } else {                                                                                 \
+            ptrdiff_t lower, upper;                                                              \
+            upper = (_disp) + (_old_extent) * ((_count) -1);                                     \
+            lower = (_disp);                                                                     \
+            if (lower < upper) {                                                                 \
+                _new_lb = lower;                                                                 \
+                _new_ub = upper;                                                                 \
+            } else {                                                                             \
+                _new_lb = upper;                                                                 \
+                _new_ub = lower;                                                                 \
+            }                                                                                    \
+            _new_lb += (_old_lb);                                                                \
+            _new_ub += (_old_ub);                                                                \
+        }                                                                                        \
+    }
 
 /* When we add a datatype we should update it's definition depending on the
  * initial displacement for the whole data, so the displacement of all elements
@@ -93,7 +115,7 @@ static inline int  IMAX( int a, int b ) { return ( a < b ? b : a ); }
  * displacement.
  */
 
-/* we have 3 differents structures to update:
+/* we have 3 different structures to update:
  * - the first is the real representation of the datatype
  * - the second is the internal representation using extents
  * - the last is the representation used for send operations
@@ -102,11 +124,11 @@ static inline int  IMAX( int a, int b ) { return ( a < b ? b : a ); }
  * be sure that the pdtBase datatype is correctly initialized with all fields
  * set to ZERO if it's a empty datatype.
  */
-int32_t opal_datatype_add( opal_datatype_t* pdtBase, const opal_datatype_t* pdtAdd,
-                           size_t count, ptrdiff_t disp, ptrdiff_t extent )
+int32_t opal_datatype_add(opal_datatype_t *pdtBase, const opal_datatype_t *pdtAdd, size_t count,
+                          ptrdiff_t disp, ptrdiff_t extent)
 {
     uint32_t newLength, place_needed = 0, i;
-    short localFlags = 0;  /* no specific options yet */
+    short localFlags = 0; /* no specific options yet */
     dt_elem_desc_t *pLast, *pLoop = NULL;
     ptrdiff_t lb, ub, true_lb, true_ub, epsilon, old_true_ub;
 
@@ -116,42 +138,46 @@ int32_t opal_datatype_add( opal_datatype_t* pdtBase, const opal_datatype_t* pdtA
      * non-negative integers. If the value is zero, no elements are generated in
      * the type map and there is no effect on datatype bounds or extent.
      */
-    if( 0 == count ) return OPAL_SUCCESS;
+    if (0 == count) {
+        return OPAL_SUCCESS;
+    }
 
     /* the extent should always be positive. So a negative value here have a
      * special meaning ie. default extent as computed by ub - lb
      */
-    if( extent == -1 ) extent = (pdtAdd->ub - pdtAdd->lb);
+    if (extent == -1) {
+        extent = (pdtAdd->ub - pdtAdd->lb);
+    }
 
     /* Deal with the special markers (OPAL_DATATYPE_LB and OPAL_DATATYPE_UB) */
-    if( OPAL_DATATYPE_LB == pdtAdd->id ) {
-        pdtBase->bdt_used |= (((uint32_t)1) << OPAL_DATATYPE_LB);
-        if( pdtBase->flags & OPAL_DATATYPE_FLAG_USER_LB ) {
-            pdtBase->lb = LMIN( pdtBase->lb, disp );
+    if (OPAL_DATATYPE_LB == pdtAdd->id) {
+        pdtBase->bdt_used |= (((uint32_t) 1) << OPAL_DATATYPE_LB);
+        if (pdtBase->flags & OPAL_DATATYPE_FLAG_USER_LB) {
+            pdtBase->lb = LMIN(pdtBase->lb, disp);
         } else {
             pdtBase->lb = disp;
             pdtBase->flags |= OPAL_DATATYPE_FLAG_USER_LB;
         }
-        if( (pdtBase->ub - pdtBase->lb) != (ptrdiff_t)pdtBase->size ) {
+        if ((pdtBase->ub - pdtBase->lb) != (ptrdiff_t) pdtBase->size) {
             pdtBase->flags &= ~OPAL_DATATYPE_FLAG_NO_GAPS;
         }
         return OPAL_SUCCESS; /* Just ignore the OPAL_DATATYPE_LOOP and OPAL_DATATYPE_END_LOOP */
-    } else if( OPAL_DATATYPE_UB == pdtAdd->id ) {
-        pdtBase->bdt_used |= (((uint32_t)1) << OPAL_DATATYPE_UB);
-        if( pdtBase->flags & OPAL_DATATYPE_FLAG_USER_UB ) {
-            pdtBase->ub = LMAX( pdtBase->ub, disp );
+    } else if (OPAL_DATATYPE_UB == pdtAdd->id) {
+        pdtBase->bdt_used |= (((uint32_t) 1) << OPAL_DATATYPE_UB);
+        if (pdtBase->flags & OPAL_DATATYPE_FLAG_USER_UB) {
+            pdtBase->ub = LMAX(pdtBase->ub, disp);
         } else {
             pdtBase->ub = disp;
             pdtBase->flags |= OPAL_DATATYPE_FLAG_USER_UB;
         }
-        if( (pdtBase->ub - pdtBase->lb) != (ptrdiff_t)pdtBase->size ) {
+        if ((pdtBase->ub - pdtBase->lb) != (ptrdiff_t) pdtBase->size) {
             pdtBase->flags &= ~OPAL_DATATYPE_FLAG_NO_GAPS;
         }
         return OPAL_SUCCESS; /* Just ignore the OPAL_DATATYPE_LOOP and OPAL_DATATYPE_END_LOOP */
     }
 
     /* Compute the number of entries we need in the datatype description */
-    OPAL_DATATYPE_COMPUTE_REQUIRED_ENTRIES( pdtAdd, count, extent, place_needed );
+    OPAL_DATATYPE_COMPUTE_REQUIRED_ENTRIES(pdtAdd, count, extent, place_needed);
 
     /*
      * Compute the lower and upper bound of the datatype. We do it in 2 steps.
@@ -159,15 +185,15 @@ int32_t opal_datatype_add( opal_datatype_t* pdtBase, const opal_datatype_t* pdtA
      * count. Then update the lb value depending on the user markers and
      * update the global lb and ub.
      */
-    OPAL_DATATYPE_LB_UB_CONT( count, disp, pdtAdd->lb, pdtAdd->ub, extent, lb, ub );
+    OPAL_DATATYPE_LB_UB_CONT(count, disp, pdtAdd->lb, pdtAdd->ub, extent, lb, ub);
 
     /* Compute the true_lb and true_ub for the datatype to be added, taking
-     * in account the number of repetions. These values do not include the
-     * potential gaps at the begining and at the end of the datatype.
+     * in account the number of repetitions. These values do not include the
+     * potential gaps at the beginning and at the end of the datatype.
      */
     true_lb = lb - (pdtAdd->lb - pdtAdd->true_lb);
     true_ub = ub - (pdtAdd->ub - pdtAdd->true_ub);
-    if( true_lb > true_ub ) {
+    if (true_lb > true_ub) {
         old_true_ub = true_lb;
         true_lb = true_ub;
         true_ub = old_true_ub;
@@ -188,29 +214,29 @@ int32_t opal_datatype_add( opal_datatype_t* pdtBase, const opal_datatype_t* pdtA
      * if the USER has explicitly set it. The result lb is the MIN between
      * the all lb + disp if and only if all or nobody flags's contain the LB.
      */
-    if( (pdtAdd->flags ^ pdtBase->flags) & OPAL_DATATYPE_FLAG_USER_LB ) {
-        if( pdtBase->flags & OPAL_DATATYPE_FLAG_USER_LB ) {
-            lb = pdtBase->lb;  /* base type has a user provided lb */
+    if ((pdtAdd->flags ^ pdtBase->flags) & OPAL_DATATYPE_FLAG_USER_LB) {
+        if (pdtBase->flags & OPAL_DATATYPE_FLAG_USER_LB) {
+            lb = pdtBase->lb; /* base type has a user provided lb */
         }
         pdtBase->flags |= OPAL_DATATYPE_FLAG_USER_LB;
     } else {
         /* both of them have the LB flag or both of them dont have it */
-        lb = LMIN( pdtBase->lb, lb );
+        lb = LMIN(pdtBase->lb, lb);
     }
 
     /* the same apply for the upper bound except for the case where
      * either of them has the flag UB, in which case we should
-     * compute the UB including the natural alignement of the data.
+     * compute the UB including the natural alignment of the data.
      */
-    if( (pdtBase->flags ^ pdtAdd->flags) & OPAL_DATATYPE_FLAG_USER_UB ) {
-        if( pdtBase->flags & OPAL_DATATYPE_FLAG_USER_UB ) {
+    if ((pdtBase->flags ^ pdtAdd->flags) & OPAL_DATATYPE_FLAG_USER_UB) {
+        if (pdtBase->flags & OPAL_DATATYPE_FLAG_USER_UB) {
             ub = pdtBase->ub;
         }
         pdtBase->flags |= OPAL_DATATYPE_FLAG_USER_UB;
     } else {
         /* both of them have the UB flag or both of them dont have it */
-        /* we should compute the extent depending on the alignement */
-        ub = LMAX( pdtBase->ub, ub );
+        /* we should compute the extent depending on the alignment */
+        ub = LMAX(pdtBase->ub, ub);
     }
     /* While the true_lb and true_ub have to be ordered to have the true_lb lower
      * than the true_ub, the ub and lb do not have to be ordered. They should be
@@ -219,18 +245,18 @@ int32_t opal_datatype_add( opal_datatype_t* pdtBase, const opal_datatype_t* pdtA
     pdtBase->lb = lb;
     pdtBase->ub = ub;
 
-    /* compute the new memory alignement */
-    pdtBase->align = IMAX( pdtBase->align, pdtAdd->align );
+    /* compute the new memory alignment */
+    pdtBase->align = IMAX(pdtBase->align, pdtAdd->align);
 
     /* Now that we have the new ub and the alignment we should update the ub to match
-     * the new alignement. We have to add an epsilon that is the least nonnegative
-     * increment needed to roung the extent to the next multiple of the alignment.
+     * the new alignment. We have to add an epsilon that is the least nonnegative
+     * increment needed to round the extent to the next multiple of the alignment.
      * This rule apply only if there is user specified upper bound as stated in the
      * MPI standard MPI 1.2 page 71.
      */
-    if( !(pdtBase->flags & OPAL_DATATYPE_FLAG_USER_UB) ) {
+    if (!(pdtBase->flags & OPAL_DATATYPE_FLAG_USER_UB)) {
         epsilon = (pdtBase->ub - pdtBase->lb) % pdtBase->align;
-        if( 0 != epsilon ) {
+        if (0 != epsilon) {
             pdtBase->ub += (pdtBase->align - epsilon);
         }
     }
@@ -247,7 +273,7 @@ int32_t opal_datatype_add( opal_datatype_t* pdtBase, const opal_datatype_t* pdtA
      * extension, the count set to zero can be used to reset the alignment of
      * the data, but not for changing the true_lb and true_ub.
      */
-    if( (0 == count) || (0 == pdtAdd->size) ) {
+    if ((0 == count) || (0 == pdtAdd->size)) {
         return OPAL_SUCCESS;
     }
 
@@ -255,11 +281,14 @@ int32_t opal_datatype_add( opal_datatype_t* pdtBase, const opal_datatype_t* pdtA
      * the data-type we can update the size, true_lb and true_ub.
      */
     pdtBase->size += count * pdtAdd->size;
-    if( 0 == pdtBase->nbElems ) old_true_ub = disp;
-    else                        old_true_ub = pdtBase->true_ub;
-    if( 0 != pdtBase->size ) {
-        pdtBase->true_lb = LMIN( true_lb, pdtBase->true_lb );
-        pdtBase->true_ub = LMAX( true_ub, pdtBase->true_ub );
+    if (0 == pdtBase->nbElems) {
+        old_true_ub = disp;
+    } else {
+        old_true_ub = pdtBase->true_ub;
+    }
+    if (0 != pdtBase->size) {
+        pdtBase->true_lb = LMIN(true_lb, pdtBase->true_lb);
+        pdtBase->true_ub = LMAX(true_ub, pdtBase->true_ub);
     } else {
         pdtBase->true_lb = true_lb;
         pdtBase->true_ub = true_ub;
@@ -267,10 +296,10 @@ int32_t opal_datatype_add( opal_datatype_t* pdtBase, const opal_datatype_t* pdtA
 
     pdtBase->bdt_used |= pdtAdd->bdt_used;
     newLength = pdtBase->desc.used + place_needed;
-    if( newLength > pdtBase->desc.length ) {
-        newLength = ((newLength / DT_INCREASE_STACK) + 1 ) * DT_INCREASE_STACK;
-        pdtBase->desc.desc   = (dt_elem_desc_t*)realloc( pdtBase->desc.desc,
-                                                         sizeof(dt_elem_desc_t) * newLength );
+    if (newLength > pdtBase->desc.length) {
+        newLength = ((newLength / DT_INCREASE_STACK) + 1) * DT_INCREASE_STACK;
+        pdtBase->desc.desc = (dt_elem_desc_t *) realloc(pdtBase->desc.desc,
+                                                        sizeof(dt_elem_desc_t) * newLength);
         pdtBase->desc.length = newLength;
     }
     pLast = &(pdtBase->desc.desc[pdtBase->desc.used]);
@@ -278,23 +307,26 @@ int32_t opal_datatype_add( opal_datatype_t* pdtBase, const opal_datatype_t* pdtA
      * of an predefined contiguous datatype. This part is unable to handle any
      * predefined non contiguous datatypes (like MPI_SHORT_INT).
      */
-    if( (pdtAdd->flags & (OPAL_DATATYPE_FLAG_PREDEFINED | OPAL_DATATYPE_FLAG_DATA)) == (OPAL_DATATYPE_FLAG_PREDEFINED | OPAL_DATATYPE_FLAG_DATA) ) {
-        if( NULL != pdtBase->ptypes )
+    if ((pdtAdd->flags & (OPAL_DATATYPE_FLAG_PREDEFINED | OPAL_DATATYPE_FLAG_DATA))
+        == (OPAL_DATATYPE_FLAG_PREDEFINED | OPAL_DATATYPE_FLAG_DATA)) {
+        if (NULL != pdtBase->ptypes) {
             pdtBase->ptypes[pdtAdd->id] += count;
+        }
 
-        pLast->elem.common.flags     = pdtAdd->flags & ~(OPAL_DATATYPE_FLAG_COMMITTED);
-        pLast->elem.common.type      = pdtAdd->id;
-        pLast->elem.disp             = disp;
-        pLast->elem.extent           = (ptrdiff_t)count * extent;
+        pLast->elem.common.flags = pdtAdd->flags & ~(OPAL_DATATYPE_FLAG_COMMITTED);
+        pLast->elem.common.type = pdtAdd->id;
+        pLast->elem.disp = disp;
+        pLast->elem.extent = (ptrdiff_t) count * extent;
         /* assume predefined datatypes without extent, aka. contiguous */
-        pLast->elem.count            = 1;
-        pLast->elem.blocklen         = count;
-        if( extent != (ptrdiff_t)pdtAdd->size ) {  /* not contiguous: let's fix */
-            pLast->elem.count            = count;
-            pLast->elem.blocklen         = 1;
-            pLast->elem.extent           = extent;
-            if( count > 1 ) {  /* gaps around the predefined datatype */
-                pLast->elem.common.flags &= ~(OPAL_DATATYPE_FLAG_CONTIGUOUS | OPAL_DATATYPE_FLAG_NO_GAPS);
+        pLast->elem.count = 1;
+        pLast->elem.blocklen = count;
+        if (extent != (ptrdiff_t) pdtAdd->size) { /* not contiguous: let's fix */
+            pLast->elem.count = count;
+            pLast->elem.blocklen = 1;
+            pLast->elem.extent = extent;
+            if (count > 1) { /* gaps around the predefined datatype */
+                pLast->elem.common.flags &= ~(OPAL_DATATYPE_FLAG_CONTIGUOUS
+                                              | OPAL_DATATYPE_FLAG_NO_GAPS);
             }
         }
         pdtBase->desc.used++;
@@ -303,43 +335,46 @@ int32_t opal_datatype_add( opal_datatype_t* pdtBase, const opal_datatype_t* pdtA
         pdtBase->loops += pdtAdd->loops;
         pdtBase->flags |= (pdtAdd->flags & OPAL_DATATYPE_FLAG_USER_LB);
         pdtBase->flags |= (pdtAdd->flags & OPAL_DATATYPE_FLAG_USER_UB);
-        if( (NULL != pdtBase->ptypes) && (NULL != pdtAdd->ptypes) ) {
-            for( i = OPAL_DATATYPE_FIRST_TYPE; i < OPAL_DATATYPE_MAX_PREDEFINED; i++ )
-                if( pdtAdd->ptypes[i] != 0 ) pdtBase->ptypes[i] += (count * pdtAdd->ptypes[i]);
+        if ((NULL != pdtBase->ptypes) && (NULL != pdtAdd->ptypes)) {
+            for (i = OPAL_DATATYPE_FIRST_TYPE; i < OPAL_DATATYPE_MAX_PREDEFINED; i++) {
+                if (pdtAdd->ptypes[i] != 0) {
+                    pdtBase->ptypes[i] += (count * pdtAdd->ptypes[i]);
+                }
+            }
         }
-        if( 1 == pdtAdd->desc.used ) {
-            pLast->elem        = pdtAdd->desc.desc[0].elem;
-            pLast->elem.disp  += disp;
-            if( 1 == count ) {
+        if (1 == pdtAdd->desc.used) {
+            pLast->elem = pdtAdd->desc.desc[0].elem;
+            pLast->elem.disp += disp;
+            if (1 == count) {
                 /* Extent only has a meaning when there are multiple elements. Bail out */
-            } else if( 1 == pLast->elem.count ) {
-                /* The size and true_extent of the added datatype are identical, signaling a datatype
-                 * that is mostly contiguous with the exception of the initial and final gaps. These
-                 * gaps do not matter here as they will amended (the initial gaps being shifted by the
-                 * new displacement and the final gap being replaced with the new gap
+            } else if (1 == pLast->elem.count) {
+                /* The size and true_extent of the added datatype are identical, signaling a
+                 * datatype that is mostly contiguous with the exception of the initial and final
+                 * gaps. These gaps do not matter here as they will amended (the initial gaps being
+                 * shifted by the new displacement and the final gap being replaced with the new gap
                  */
-                if( pdtAdd->desc.desc[0].elem.extent == extent ) {
+                if (pdtAdd->desc.desc[0].elem.extent == extent) {
                     /* pure bliss everything is fully contiguous and we can collapse
                      * everything by updating the blocklen and extent
                      */
                     pLast->elem.blocklen *= count;
-                    pLast->elem.extent   *= count;
+                    pLast->elem.extent *= count;
                 } else {
                     pLast->elem.count = count;
                     pLast->elem.extent = extent;
                 }
-            } else if( extent == ((ptrdiff_t)pLast->elem.count * pLast->elem.extent) ) {
+            } else if (extent == ((ptrdiff_t) pLast->elem.count * pLast->elem.extent)) {
                 /* It's just a repetition of the same element, increase the count */
                 /* We need to protect against the case where the multiplication below results in a
-                 * number larger than the max uint32_t. In the unlikely situation where that's the case
-                 * we should not try to optimize the item further but instead fall back and build a loop
-                 * around it.
+                 * number larger than the max uint32_t. In the unlikely situation where that's the
+                 * case we should not try to optimize the item further but instead fall back and
+                 * build a loop around it.
                  */
                 uint32_t cnt = pLast->elem.count * count;
-                if( cnt < pLast->elem.count ) {
+                if (cnt < pLast->elem.count) {
                     goto build_loop;
                 }
-                pLast->elem.count = cnt;  /* we're good, merge the elements */
+                pLast->elem.count = cnt; /* we're good, merge the elements */
             } else {
                 /* No luck here, no optimization can be applied. Fall back to the
                  * normal case where we add a loop around the datatype.
@@ -349,34 +384,34 @@ int32_t opal_datatype_add( opal_datatype_t* pdtBase, const opal_datatype_t* pdtA
             pdtBase->desc.used++;
         } else {
 
-build_loop:
+        build_loop:
             /* if the extent of the datatype is the same as the extent of the loop
              * description of the datatype then we simply have to update the main loop.
              */
-            if( count != 1 ) {
+            if (count != 1) {
                 pLoop = pLast;
-                CREATE_LOOP_START( pLast, count, pdtAdd->desc.used + 1, extent,
-                                   (pdtAdd->flags & ~(OPAL_DATATYPE_FLAG_COMMITTED)) );
-                pdtBase->loops     += 2;
+                CREATE_LOOP_START(pLast, count, pdtAdd->desc.used + 1, extent,
+                                  (pdtAdd->flags & ~(OPAL_DATATYPE_FLAG_COMMITTED)));
+                pdtBase->loops += 2;
                 pdtBase->desc.used += 2;
                 pLast++;
             }
 
-            for( i = 0; i < pdtAdd->desc.used; i++ ) {
-                pLast->elem               = pdtAdd->desc.desc[i].elem;
-                if( OPAL_DATATYPE_FLAG_DATA & pLast->elem.common.flags )
+            for (i = 0; i < pdtAdd->desc.used; i++) {
+                pLast->elem = pdtAdd->desc.desc[i].elem;
+                if (OPAL_DATATYPE_FLAG_DATA & pLast->elem.common.flags) {
                     pLast->elem.disp += disp;
-                else if( OPAL_DATATYPE_END_LOOP == pLast->elem.common.type ) {
+                } else if (OPAL_DATATYPE_END_LOOP == pLast->elem.common.type) {
                     pLast->end_loop.first_elem_disp += disp;
                 }
                 pLast++;
             }
             pdtBase->desc.used += pdtAdd->desc.used;
-            if( pLoop != NULL ) {
-                int index = GET_FIRST_NON_LOOP( pLoop );
-                assert( pLoop[index].elem.common.flags & OPAL_DATATYPE_FLAG_DATA );
-                CREATE_LOOP_END( pLast, pdtAdd->desc.used + 1, pLoop[index].elem.disp,
-                                 pdtAdd->size, pLoop->loop.common.flags );
+            if (pLoop != NULL) {
+                int index = GET_FIRST_NON_LOOP(pLoop);
+                assert(pLoop[index].elem.common.flags & OPAL_DATATYPE_FLAG_DATA);
+                CREATE_LOOP_END(pLast, pdtAdd->desc.used + 1, pLoop[index].elem.disp, pdtAdd->size,
+                                pLoop->loop.common.flags);
             }
         }
         /* should I add some space until the extent of this datatype ? */
@@ -389,19 +424,20 @@ int32_t opal_datatype_add( opal_datatype_t* pdtBase, const opal_datatype_t* pdtA
      */
     localFlags = pdtBase->flags & pdtAdd->flags;
     UNSET_CONTIGUOUS_FLAG(pdtBase->flags);
-    if( (localFlags & OPAL_DATATYPE_FLAG_CONTIGUOUS)             /* both type were contiguous */
-        && ((disp + pdtAdd->true_lb) == old_true_ub)  /* and there is no gap between them */
-        && ( ((ptrdiff_t)pdtAdd->size == extent)      /* the size and the extent of the
-                                                       * added type have to match */
-             || (count < 2)) ) {                      /* if the count is bigger than 2 */
-            SET_CONTIGUOUS_FLAG(pdtBase->flags);
-            if( (ptrdiff_t)pdtBase->size == (pdtBase->ub - pdtBase->lb) )
-                SET_NO_GAP_FLAG(pdtBase->flags);
+    if ((localFlags & OPAL_DATATYPE_FLAG_CONTIGUOUS) /* both type were contiguous */
+        && ((disp + pdtAdd->true_lb) == old_true_ub) /* and there is no gap between them */
+        && (((ptrdiff_t) pdtAdd->size == extent)     /* the size and the extent of the
+                                                      * added type have to match */
+            || (count < 2))) {                       /* if the count is bigger than 2 */
+        SET_CONTIGUOUS_FLAG(pdtBase->flags);
+        if ((ptrdiff_t) pdtBase->size == (pdtBase->ub - pdtBase->lb)) {
+            SET_NO_GAP_FLAG(pdtBase->flags);
+        }
     }
 
     /* If the NO_GAP flag is set the contiguous have to be set too */
-    if( pdtBase->flags & OPAL_DATATYPE_FLAG_NO_GAPS ) {
-        assert( pdtBase->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS );
+    if (pdtBase->flags & OPAL_DATATYPE_FLAG_NO_GAPS) {
+        assert(pdtBase->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS);
     }
     pdtBase->nbElems += (count * pdtAdd->nbElems);
 
diff --git a/opal/datatype/opal_datatype_checksum.h b/opal/datatype/opal_datatype_checksum.h
index 64fecd179b9..7b59e4f7930 100644
--- a/opal/datatype/opal_datatype_checksum.h
+++ b/opal/datatype/opal_datatype_checksum.h
@@ -19,37 +19,42 @@
 #ifndef DATATYPE_CHECKSUM_H_HAS_BEEN_INCLUDED
 #define DATATYPE_CHECKSUM_H_HAS_BEEN_INCLUDED
 
-
 #include "opal/datatype/opal_datatype_memcpy.h"
 #include "opal/util/crc.h"
 
 #if defined(CHECKSUM)
 
-#if defined (OPAL_CSUM_DST)
-#define MEMCPY_CSUM( DST, SRC, BLENGTH, CONVERTOR ) \
-do { \
-    (CONVERTOR)->checksum += OPAL_CSUM_BCOPY_PARTIAL( (SRC), (DST), (BLENGTH), (BLENGTH), &(CONVERTOR)->csum_ui1, &(CONVERTOR)->csum_ui2 ); \
-} while (0)
+#    if defined(OPAL_CSUM_DST)
+#        define MEMCPY_CSUM(DST, SRC, BLENGTH, CONVERTOR)                                 \
+            do {                                                                          \
+                (CONVERTOR)->checksum += OPAL_CSUM_BCOPY_PARTIAL((SRC), (DST), (BLENGTH), \
+                                                                 (BLENGTH),               \
+                                                                 &(CONVERTOR)->csum_ui1,  \
+                                                                 &(CONVERTOR)->csum_ui2); \
+            } while (0)
 
-#else  /* if OPAL_CSUM_DST */
+#    else /* if OPAL_CSUM_DST */
 
-#define MEMCPY_CSUM( DST, SRC, BLENGTH, CONVERTOR ) \
-do { \
-    (CONVERTOR)->checksum += OPAL_CSUM_BCOPY_PARTIAL( (SRC), (DST), (BLENGTH), (BLENGTH), &(CONVERTOR)->csum_ui1, &(CONVERTOR)->csum_ui2 ); \
-} while (0)
-#endif  /* if OPAL_CSUM_DST */
+#        define MEMCPY_CSUM(DST, SRC, BLENGTH, CONVERTOR)                                 \
+            do {                                                                          \
+                (CONVERTOR)->checksum += OPAL_CSUM_BCOPY_PARTIAL((SRC), (DST), (BLENGTH), \
+                                                                 (BLENGTH),               \
+                                                                 &(CONVERTOR)->csum_ui1,  \
+                                                                 &(CONVERTOR)->csum_ui2); \
+            } while (0)
+#    endif /* if OPAL_CSUM_DST */
 
-#define COMPUTE_CSUM( SRC, BLENGTH, CONVERTOR ) \
-do { \
-    (CONVERTOR)->checksum += OPAL_CSUM_PARTIAL( (SRC), (BLENGTH), &(CONVERTOR)->csum_ui1, &(CONVERTOR)->csum_ui2 ); \
-} while (0)
+#    define COMPUTE_CSUM(SRC, BLENGTH, CONVERTOR)                                                \
+        do {                                                                                     \
+            (CONVERTOR)->checksum += OPAL_CSUM_PARTIAL((SRC), (BLENGTH), &(CONVERTOR)->csum_ui1, \
+                                                       &(CONVERTOR)->csum_ui2);                  \
+        } while (0)
 
-#else  /* if CHECKSUM */
+#else /* if CHECKSUM */
 
-#define MEMCPY_CSUM( DST, SRC, BLENGTH, CONVERTOR ) \
-    MEMCPY( (DST), (SRC), (BLENGTH) )
+#    define MEMCPY_CSUM(DST, SRC, BLENGTH, CONVERTOR) MEMCPY((DST), (SRC), (BLENGTH))
 
-#define COMPUTE_CSUM( SRC, BLENGTH, CONVERTOR )
+#    define COMPUTE_CSUM(SRC, BLENGTH, CONVERTOR)
 
-#endif  /* if CHECKSUM */
-#endif  /* DATATYPE_CHECKSUM_H_HAS_BEEN_INCLUDED */
+#endif /* if CHECKSUM */
+#endif /* DATATYPE_CHECKSUM_H_HAS_BEEN_INCLUDED */
diff --git a/opal/datatype/opal_datatype_clone.c b/opal/datatype/opal_datatype_clone.c
index 59e82bb40cf..2ef94d7c036 100644
--- a/opal/datatype/opal_datatype_clone.c
+++ b/opal/datatype/opal_datatype_clone.c
@@ -32,15 +32,15 @@
  *
  * Clone all the values from oldType into newType without allocating a new datatype.
  */
-int32_t opal_datatype_clone( const opal_datatype_t * src_type, opal_datatype_t * dest_type )
+int32_t opal_datatype_clone(const opal_datatype_t *src_type, opal_datatype_t *dest_type)
 {
-    int32_t desc_length = src_type->desc.used + 1;  /* +1 because of the fake OPAL_DATATYPE_END_LOOP entry */
-    dt_elem_desc_t* temp = dest_type->desc.desc;    /* temporary copy of the desc pointer */
+    int32_t desc_length = src_type->desc.used
+                          + 1; /* +1 because of the fake OPAL_DATATYPE_END_LOOP entry */
+    dt_elem_desc_t *temp = dest_type->desc.desc; /* temporary copy of the desc pointer */
 
     /* copy _excluding_ the super object, we want to keep the cls_destruct_array */
-    memcpy( (char*)dest_type + sizeof(opal_object_t),
-            (char*)src_type + sizeof(opal_object_t),
-            sizeof(opal_datatype_t)-sizeof(opal_object_t) );
+    memcpy((char *) dest_type + sizeof(opal_object_t), (char *) src_type + sizeof(opal_object_t),
+           sizeof(opal_datatype_t) - sizeof(opal_object_t));
 
     dest_type->flags &= (~OPAL_DATATYPE_FLAG_PREDEFINED);
     dest_type->ptypes = NULL;
@@ -49,27 +49,29 @@ int32_t opal_datatype_clone( const opal_datatype_t * src_type, opal_datatype_t *
     /**
      * Allow duplication of MPI_UB and MPI_LB.
      */
-    if( 0 != src_type->desc.used ) {
-        memcpy( dest_type->desc.desc, src_type->desc.desc, sizeof(dt_elem_desc_t) * desc_length );
-        if( 0 != src_type->opt_desc.used ) {
-            if( src_type->opt_desc.desc == src_type->desc.desc) {
+    if (0 != src_type->desc.used) {
+        memcpy(dest_type->desc.desc, src_type->desc.desc, sizeof(dt_elem_desc_t) * desc_length);
+        if (0 != src_type->opt_desc.used) {
+            if (src_type->opt_desc.desc == src_type->desc.desc) {
                 dest_type->opt_desc = dest_type->desc;
             } else {
                 desc_length = dest_type->opt_desc.used + 1;
-                dest_type->opt_desc.desc = (dt_elem_desc_t*)malloc( desc_length * sizeof(dt_elem_desc_t) );
+                dest_type->opt_desc.desc = (dt_elem_desc_t *) malloc(desc_length
+                                                                     * sizeof(dt_elem_desc_t));
                 /*
                  * Yes, the dest_type->opt_desc.length is just the opt_desc.used of the old Type.
                  */
                 dest_type->opt_desc.length = src_type->opt_desc.used;
                 dest_type->opt_desc.used = src_type->opt_desc.used;
-                memcpy( dest_type->opt_desc.desc, src_type->opt_desc.desc, desc_length * sizeof(dt_elem_desc_t) );
+                memcpy(dest_type->opt_desc.desc, src_type->opt_desc.desc,
+                       desc_length * sizeof(dt_elem_desc_t));
             }
         } else {
-            assert( NULL == dest_type->opt_desc.desc );
-            assert( 0 == dest_type->opt_desc.length );
+            assert(NULL == dest_type->opt_desc.desc);
+            assert(0 == dest_type->opt_desc.length);
         }
     }
-    dest_type->id  = src_type->id;  /* preserve the default id. This allow us to
-                                     * copy predefined types. */
+    dest_type->id = src_type->id; /* preserve the default id. This allow us to
+                                   * copy predefined types. */
     return OPAL_SUCCESS;
 }
diff --git a/opal/datatype/opal_datatype_copy.c b/opal/datatype/opal_datatype_copy.c
index e13d17fa59c..81b621a9d6a 100644
--- a/opal/datatype/opal_datatype_copy.c
+++ b/opal/datatype/opal_datatype_copy.c
@@ -15,6 +15,9 @@
  * Copyright (c) 2013      Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2015-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2022      Advanced Micro Devices, Inc. All rights reserved.
+ * Copyright (c) 2024      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -27,92 +30,130 @@
 #include <stddef.h>
 #include <stdlib.h>
 
-#include "opal/prefetch.h"
-#include "opal/util/output.h"
-#include "opal/datatype/opal_datatype.h"
 #include "opal/datatype/opal_convertor.h"
-#include "opal/datatype/opal_datatype_internal.h"
+#include "opal/datatype/opal_datatype.h"
 #include "opal/datatype/opal_datatype_checksum.h"
-
+#include "opal/datatype/opal_datatype_internal.h"
+#include "opal/prefetch.h"
+#include "opal/util/output.h"
+#include "opal/mca/accelerator/accelerator.h"
 
 #if OPAL_ENABLE_DEBUG
-#define DO_DEBUG(INST)  if( opal_ddt_copy_debug ) { INST }
+#    define DO_DEBUG(INST)         \
+        if (opal_ddt_copy_debug) { \
+            INST                   \
+        }
 #else
-#define DO_DEBUG(INST)
-#endif  /* OPAL_ENABLE_DEBUG */
-
-static size_t opal_datatype_memop_block_size = 128 * 1024;
-
-/**
- * Non overlapping memory regions
- */
-#undef MEM_OP_NAME
-#define MEM_OP_NAME  non_overlap
-#undef MEM_OP
-#define MEM_OP       MEMCPY
-#include "opal_datatype_copy.h"
-
-#define MEMMOVE(d, s, l)                                  \
-    do {                                                  \
-        if( (((d) < (s)) && (((d) + (l)) > (s))) ||       \
-            (((s) < (d)) && (((s) + (l)) > (d))) ) {      \
-            memmove( (d), (s), (l) );                     \
-        } else {                                          \
-            MEMCPY( (d), (s), (l) );                      \
-        }                                                 \
+#    define DO_DEBUG(INST)
+#endif /* OPAL_ENABLE_DEBUG */
+
+#define MEMMOVE(d, s, l)                                                                    \
+    do {                                                                                    \
+        if ((((d) < (s)) && (((d) + (l)) > (s))) || (((s) < (d)) && (((s) + (l)) > (d)))) { \
+            memmove((d), (s), (l));                                                         \
+        } else {                                                                            \
+            MEMCPY((d), (s), (l));                                                          \
+        }                                                                                   \
     } while (0)
 
-/**
- * Overlapping memory regions
- */
-#undef MEM_OP_NAME
-#define MEM_OP_NAME  overlap
-#undef MEM_OP
-#define MEM_OP       MEMMOVE
-#include "opal_datatype_copy.h"
+static void *opal_datatype_accelerator_memcpy(void *dest, const void *src, size_t size)
+{
+    int res;
+    int dst_type, dst_dev, src_type, src_dev;
+    int copy_type = MCA_ACCELERATOR_TRANSFER_DTOD;
+    uint64_t flags;
+    /* If accelerator check addr returns an error, we can only
+     * assume it is a host buffer. If device buffer checking fails,
+     * it's also highly likely that a device copy will fail. The best
+     * we can do is fail as this is not a recoverable/ignorable failure
+     * and retries are also unlikely to succeed. We identify these
+     * buffers as host buffers as attempting a memcpy would provide
+     * a chance to succeed. */
+    dst_type = opal_accelerator.check_addr(dest, &dst_dev, &flags);
+    src_type = opal_accelerator.check_addr(src, &src_dev, &flags);
+    if (0 >= dst_type && 0 >= src_type) {
+        return memcpy(dest, src, size);
+    }
+    else if (0 >= dst_type && 0 < src_type) {
+	copy_type = MCA_ACCELERATOR_TRANSFER_DTOH;
+    }
+    else if (0 < dst_type && 0 >= src_type) {
+	copy_type = MCA_ACCELERATOR_TRANSFER_HTOD;
+    }
+    res = opal_accelerator.mem_copy(dst_dev, src_dev,
+				    dest, src, size, copy_type);
+    if (OPAL_SUCCESS != res) {
+        opal_output(0, "Error in accelerator memcpy");
+        abort();
+    }
+    return dest;
+}
 
-#if OPAL_CUDA_SUPPORT
-#include "opal/mca/common/cuda/common_cuda.h"
+static void *opal_datatype_accelerator_memmove(void *dest, const void *src, size_t size)
+{
+    int res;
+    int dst_type, dst_dev, src_type, src_dev;
+    int copy_type = MCA_ACCELERATOR_TRANSFER_DTOD;
+    uint64_t flags;
+    /* If accelerator check addr returns an error, we can only
+     * assume it is a host buffer. If device buffer checking fails,
+     * it's also highly likely that a device copy will fail. The best
+     * we can do is fail as this is not a recoverable/ignorable failure
+     * and retries are also unlikely to succeed. We identify these
+     * buffers as host buffers as attempting a memmove would provide
+     * a chance to succeed. */
+    dst_type = opal_accelerator.check_addr(dest, &dst_dev, &flags);
+    src_type = opal_accelerator.check_addr(src, &src_dev, &flags);
+    if (0 >= dst_type && 0 >= src_type) {
+        return memmove(dest, src, size);
+    }
+    else if (0 >= dst_type && 0 < src_type) {
+	copy_type = MCA_ACCELERATOR_TRANSFER_DTOH;
+    }
+    else if (0 < dst_type && 0 >= src_type) {
+	copy_type = MCA_ACCELERATOR_TRANSFER_HTOD;
+    }
+    res = opal_accelerator.mem_move(dst_dev, src_dev,
+                                    dest, src, size, copy_type);
+    if (OPAL_SUCCESS != res) {
+        opal_output(0, "Error in accelerator memmove");
+        abort();
+    }
+    return dest;
+}
 
+#undef MEM_OP_BLOCK_SIZE
+#define MEM_OP_BLOCK_SIZE total_length
 #undef MEM_OP_NAME
-#define MEM_OP_NAME non_overlap_cuda
+#define MEM_OP_NAME non_overlap_accelerator
 #undef MEM_OP
-#define MEM_OP opal_cuda_memcpy_sync
+#define MEM_OP opal_datatype_accelerator_memcpy
 #include "opal_datatype_copy.h"
 
+#undef MEM_OP_BLOCK_SIZE
+#define MEM_OP_BLOCK_SIZE total_length
 #undef MEM_OP_NAME
-#define MEM_OP_NAME overlap_cuda
+#define MEM_OP_NAME overlap_accelerator
 #undef MEM_OP
-#define MEM_OP opal_cuda_memmove
+#define MEM_OP opal_datatype_accelerator_memmove
 #include "opal_datatype_copy.h"
 
-#define SET_CUDA_COPY_FCT(cuda_device_bufs, fct, copy_function)     \
-    do {                                                            \
-        if (true == cuda_device_bufs) {                             \
-            fct = copy_function;                                    \
-        }                                                           \
-    } while(0)
-#else
-#define SET_CUDA_COPY_FCT(cuda_device_bufs, fct, copy_function)
-#endif
-
-int32_t opal_datatype_copy_content_same_ddt( const opal_datatype_t* datatype, int32_t count,
-                                             char* destination_base, char* source_base )
+int32_t opal_datatype_copy_content_same_ddt(const opal_datatype_t *datatype, int32_t count,
+                                            char *destination_base, char *source_base)
 {
     ptrdiff_t extent;
-    int32_t (*fct)( const opal_datatype_t*, int32_t, char*, char*);
-
-#if OPAL_CUDA_SUPPORT
-    bool cuda_device_bufs = opal_cuda_check_bufs(destination_base, source_base);
-#endif
+    int32_t (*fct)(const opal_datatype_t *, int32_t, char *, char *);
 
-    DO_DEBUG( opal_output( 0, "opal_datatype_copy_content_same_ddt( %p, %d, dst %p, src %p )\n",
-                           (void*)datatype, count, (void*)destination_base, (void*)source_base ); );
+    DO_DEBUG(opal_output(0, "opal_datatype_copy_content_same_ddt( %p, %d, dst %p, src %p )\n",
+                         (void *) datatype, count, (void *) destination_base,
+                         (void *) source_base););
 
     /* empty data ? then do nothing. This should normally be trapped
      * at a higher level.
      */
-    if( 0 == count ) return 1;
+    if (0 == count) {
+        return 1;
+    }
 
     /**
      * see discussion in coll_basic_reduce.c for the computation of extent when
@@ -121,21 +162,17 @@ int32_t opal_datatype_copy_content_same_ddt( const opal_datatype_t* datatype, in
      */
     extent = (datatype->true_ub - datatype->true_lb) + (count - 1) * (datatype->ub - datatype->lb);
 
-    fct = non_overlap_copy_content_same_ddt;
-    SET_CUDA_COPY_FCT(cuda_device_bufs, fct, non_overlap_cuda_copy_content_same_ddt);
-    if( destination_base < source_base ) {
-        if( (destination_base + extent) > source_base ) {
+    fct = non_overlap_accelerator_copy_content_same_ddt;
+    if (destination_base < source_base) {
+        if ((destination_base + extent) > source_base) {
             /* memmove */
-            fct = overlap_copy_content_same_ddt;
-            SET_CUDA_COPY_FCT(cuda_device_bufs, fct, overlap_cuda_copy_content_same_ddt);
+            fct = overlap_accelerator_copy_content_same_ddt;
         }
     } else {
-        if( (source_base + extent) > destination_base ) {
+        if ((source_base + extent) > destination_base) {
             /* memmove */
-            fct = overlap_copy_content_same_ddt;
-            SET_CUDA_COPY_FCT(cuda_device_bufs, fct, overlap_cuda_copy_content_same_ddt);
+            fct = overlap_accelerator_copy_content_same_ddt;
         }
     }
-    return fct( datatype, count, destination_base, source_base );
+    return fct(datatype, count, destination_base, source_base);
 }
-
diff --git a/opal/datatype/opal_datatype_copy.h b/opal/datatype/opal_datatype_copy.h
index 1a5c021e760..1e10b03ed27 100644
--- a/opal/datatype/opal_datatype_copy.h
+++ b/opal/datatype/opal_datatype_copy.h
@@ -14,46 +14,44 @@
  */
 
 #ifdef HAVE_ALLOCA_H
-#include <alloca.h>
+#    include <alloca.h>
 #endif
 
 #if !defined(MEM_OP_NAME)
-#error
-#endif  /* !defined((MEM_OP_NAME) */
+#    error
+#endif /* !defined((MEM_OP_NAME) */
 #if !defined(MEM_OP)
-#error
-#endif  /* !defined(MEM_OP) */
+#    error
+#endif /* !defined(MEM_OP) */
+#if !defined(MEM_OP_BLOCK_SIZE)
+#    error
+#endif  /* !defined(MEM_OP_BLOCK_SIZE) */
 
 #ifndef STRINGIFY
-#  define STRINGIFY_(arg)          #arg
-#  define STRINGIFY(arg)           STRINGIFY_(arg)
+#    define STRINGIFY_(arg) #    arg
+#    define STRINGIFY(arg)  STRINGIFY_(arg)
 #endif
 
 #ifndef DT_CONCAT
-#  define DT__CONCAT(a, b)        a##b
-#  define DT_CONCAT(a, b)         DT__CONCAT(a, b)
+#    define DT__CONCAT(a, b) a##b
+#    define DT_CONCAT(a, b)  DT__CONCAT(a, b)
 #endif
 
+#define _predefined_data       DT_CONCAT(MEM_OP_NAME, _predefined_data)
+#define _contiguous_loop       DT_CONCAT(MEM_OP_NAME, _contiguous_loop)
+#define _copy_content_same_ddt DT_CONCAT(MEM_OP_NAME, _copy_content_same_ddt)
 
-#define _predefined_data        DT_CONCAT(MEM_OP_NAME,_predefined_data)
-#define _contiguous_loop        DT_CONCAT(MEM_OP_NAME,_contiguous_loop)
-#define _copy_content_same_ddt  DT_CONCAT(MEM_OP_NAME,_copy_content_same_ddt)
-
-static inline void _predefined_data( const dt_elem_desc_t* ELEM,
-                                     const opal_datatype_t* DATATYPE,
-                                     unsigned char* SOURCE_BASE,
-                                     size_t TOTAL_COUNT,
-                                     size_t COUNT,
-                                     unsigned char* SOURCE,
-                                     unsigned char* DESTINATION,
-                                     size_t* SPACE )
+static inline void _predefined_data(const dt_elem_desc_t *ELEM, const opal_datatype_t *DATATYPE,
+                                    unsigned char *SOURCE_BASE, size_t TOTAL_COUNT, size_t COUNT,
+                                    unsigned char *SOURCE, unsigned char *DESTINATION,
+                                    size_t *SPACE)
 {
-    const ddt_elem_desc_t* _elem = &((ELEM)->elem);
-    unsigned char* _source = (SOURCE) + _elem->disp;
-    unsigned char* _destination = (DESTINATION) + _elem->disp;
+    const ddt_elem_desc_t *_elem = &((ELEM)->elem);
+    unsigned char *_source = (SOURCE) + _elem->disp;
+    unsigned char *_destination = (DESTINATION) + _elem->disp;
     size_t do_now = _elem->count, do_now_bytes;
 
-    assert( (COUNT) == (do_now * _elem->blocklen));
+    assert((COUNT) == (do_now * _elem->blocklen));
 
     /* We don't a prologue and epilogue here as we are __always__ working
      * with full copies of the data description.
@@ -63,179 +61,187 @@ static inline void _predefined_data( const dt_elem_desc_t* ELEM,
      * Compute how many full blocklen we need to do and do them.
      */
     do_now_bytes = _elem->blocklen * opal_datatype_basicDatatypes[_elem->common.type]->size;
-    assert( (do_now * do_now_bytes) <= (*SPACE) );
-
-    for(size_t _i = 0; _i < do_now; _i++ ) {
-        OPAL_DATATYPE_SAFEGUARD_POINTER( _source, do_now_bytes, (SOURCE_BASE),
-                                         (DATATYPE), (TOTAL_COUNT) );
-        DO_DEBUG( opal_output( 0, "copy %s( %p, %p, %" PRIsize_t " ) => space %" PRIsize_t "\n",
-                               STRINGIFY(MEM_OP_NAME), (void*)_destination, (void*)_source, do_now_bytes, *(SPACE) - _i * do_now_bytes ); );
-        MEM_OP( _destination, _source, do_now_bytes );
+    assert((do_now * do_now_bytes) <= (*SPACE));
+
+    for (size_t _i = 0; _i < do_now; _i++) {
+        OPAL_DATATYPE_SAFEGUARD_POINTER(_source, do_now_bytes, (SOURCE_BASE), (DATATYPE),
+                                        (TOTAL_COUNT));
+        DO_DEBUG(opal_output(0, "copy %s( %p, %p, %" PRIsize_t " ) => space %" PRIsize_t "\n",
+                             STRINGIFY(MEM_OP_NAME), (void *) _destination, (void *) _source,
+                             do_now_bytes, *(SPACE) -_i * do_now_bytes););
+        MEM_OP(_destination, _source, do_now_bytes);
         _destination += _elem->extent;
-        _source      += _elem->extent;
+        _source += _elem->extent;
     }
     *(SPACE) -= (do_now_bytes * do_now);
 }
 
-static inline void _contiguous_loop( const dt_elem_desc_t* ELEM,
-                                     const opal_datatype_t* DATATYPE,
-                                     unsigned char* SOURCE_BASE,
-                                     size_t TOTAL_COUNT,
-                                     size_t COUNT,
-                                     unsigned char* SOURCE,
-                                     unsigned char* DESTINATION,
-                                     size_t* SPACE )
+static inline void _contiguous_loop(const dt_elem_desc_t *ELEM, const opal_datatype_t *DATATYPE,
+                                    unsigned char *SOURCE_BASE, size_t TOTAL_COUNT, size_t COUNT,
+                                    unsigned char *SOURCE, unsigned char *DESTINATION,
+                                    size_t *SPACE)
 {
-    ddt_loop_desc_t *_loop = (ddt_loop_desc_t*)(ELEM);
-    ddt_endloop_desc_t* _end_loop = (ddt_endloop_desc_t*)((ELEM) + _loop->items);
-    unsigned char* _source = (SOURCE) + _end_loop->first_elem_disp;
-    unsigned char* _destination = (DESTINATION) + _end_loop->first_elem_disp;
+    ddt_loop_desc_t *_loop = (ddt_loop_desc_t *) (ELEM);
+    ddt_endloop_desc_t *_end_loop = (ddt_endloop_desc_t *) ((ELEM) + _loop->items);
+    unsigned char *_source = (SOURCE) + _end_loop->first_elem_disp;
+    unsigned char *_destination = (DESTINATION) + _end_loop->first_elem_disp;
     size_t _copy_loops = (COUNT);
 
-    if( _loop->extent == (ptrdiff_t)_end_loop->size ) {  /* the loop is contiguous */
+    if (_loop->extent == (ptrdiff_t) _end_loop->size) { /* the loop is contiguous */
         _copy_loops *= _end_loop->size;
-        OPAL_DATATYPE_SAFEGUARD_POINTER( _source, _copy_loops, (SOURCE_BASE),
-                                    (DATATYPE), (TOTAL_COUNT) );
-        MEM_OP( _destination, _source, _copy_loops );
+        OPAL_DATATYPE_SAFEGUARD_POINTER(_source, _copy_loops, (SOURCE_BASE), (DATATYPE),
+                                        (TOTAL_COUNT));
+        MEM_OP(_destination, _source, _copy_loops);
     } else {
-        for(size_t _i = 0; _i < _copy_loops; _i++ ) {
-            OPAL_DATATYPE_SAFEGUARD_POINTER( _source, _end_loop->size, (SOURCE_BASE),
-                                        (DATATYPE), (TOTAL_COUNT) );
-            DO_DEBUG( opal_output( 0, "copy 3. %s( %p, %p, %" PRIsize_t " ) => space %" PRIsize_t "\n",
-                                   STRINGIFY(MEM_OP_NAME), (void*)_destination, (void*)_source, _end_loop->size, *(SPACE) - _i * _end_loop->size ); );
-            MEM_OP( _destination, _source, _end_loop->size );
-            _source      += _loop->extent;
+        for (size_t _i = 0; _i < _copy_loops; _i++) {
+            OPAL_DATATYPE_SAFEGUARD_POINTER(_source, _end_loop->size, (SOURCE_BASE), (DATATYPE),
+                                            (TOTAL_COUNT));
+            DO_DEBUG(opal_output(0,
+                                 "copy 3. %s( %p, %p, %" PRIsize_t " ) => space %" PRIsize_t "\n",
+                                 STRINGIFY(MEM_OP_NAME), (void *) _destination, (void *) _source,
+                                 _end_loop->size, *(SPACE) -_i * _end_loop->size););
+            MEM_OP(_destination, _source, _end_loop->size);
+            _source += _loop->extent;
             _destination += _loop->extent;
         }
         _copy_loops *= _end_loop->size;
     }
-    *(SPACE)      -= _copy_loops;
+    *(SPACE) -= _copy_loops;
 }
 
-static inline int32_t _copy_content_same_ddt( const opal_datatype_t* datatype, int32_t count,
-                                              char* destination_base, char* source_base )
+static inline int32_t _copy_content_same_ddt(const opal_datatype_t *datatype, int32_t count,
+                                             char *destination_base, char *source_base)
 {
-    dt_stack_t* pStack;       /* pointer to the position on the stack */
-    int32_t stack_pos;        /* index of the stack level */
-    uint32_t pos_desc;        /* actual position in the description of the derived datatype */
-    uint32_t count_desc;      /* the number of items already done in the actual pos_desc */
-    dt_elem_desc_t* description;
-    dt_elem_desc_t* pElem;
+    dt_stack_t *pStack;  /* pointer to the position on the stack */
+    int32_t stack_pos;   /* index of the stack level */
+    uint32_t pos_desc;   /* actual position in the description of the derived datatype */
+    uint32_t count_desc; /* the number of items already done in the actual pos_desc */
+    dt_elem_desc_t *description;
+    dt_elem_desc_t *pElem;
     size_t iov_len_local;
-    unsigned char *source = (unsigned char*)source_base,
-                  *destination = (unsigned char*)destination_base;
+    unsigned char *source = (unsigned char *) source_base,
+                  *destination = (unsigned char *) destination_base;
 
-    DO_DEBUG( opal_output( 0, "_copy_content_same_ddt( %p, %d, dst %p, src %p )\n",
-                           (void*)datatype, count, (void*)destination_base, (void*)source_base ); );
+    DO_DEBUG(opal_output(0, "_copy_content_same_ddt( %p, %d, dst %p, src %p )\n", (void *) datatype,
+                         count, (void *) destination_base, (void *) source_base););
 
-    iov_len_local = (size_t)count * datatype->size;
+    iov_len_local = (size_t) count * datatype->size;
 
     /* If we have to copy a contiguous datatype then simply
      * do a MEM_OP.
      */
-    if( datatype->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS ) {
+    if (datatype->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS) {
         ptrdiff_t extent = (datatype->ub - datatype->lb);
         /* Now that we know the datatype is contiguous, we should move the 2 pointers
          * source and destination to the correct displacement.
          */
         destination += datatype->true_lb;
-        source      += datatype->true_lb;
-        if( (ptrdiff_t)datatype->size == extent ) {  /* all contiguous == no gaps around */
+        source += datatype->true_lb;
+        if ((ptrdiff_t) datatype->size == extent) { /* all contiguous == no gaps around */
             size_t total_length = iov_len_local;
-            size_t memop_chunk = opal_datatype_memop_block_size;
-            OPAL_DATATYPE_SAFEGUARD_POINTER( source, iov_len_local,
-                                             (unsigned char*)source_base, datatype, count );
-            while( total_length > 0 ) {
-                if( memop_chunk > total_length ) memop_chunk = total_length;
-                DO_DEBUG( opal_output( 0, "copy c1. %s( %p, %p, %lu ) => space %lu\n",
-                                       STRINGIFY(MEM_OP_NAME), (void*)destination, (void*)source, (unsigned long)memop_chunk, (unsigned long)total_length ); );
-                MEM_OP( destination, source, memop_chunk );
-                destination   += memop_chunk;
-                source        += memop_chunk;
-                total_length  -= memop_chunk;
+            size_t memop_chunk = MEM_OP_BLOCK_SIZE;
+            OPAL_DATATYPE_SAFEGUARD_POINTER(source, iov_len_local, (unsigned char *) source_base,
+                                            datatype, count);
+            while (total_length > 0) {
+                if (memop_chunk > total_length)
+                    memop_chunk = total_length;
+                DO_DEBUG(opal_output(0, "copy c1. %s( %p, %p, %lu ) => space %lu\n",
+                                     STRINGIFY(MEM_OP_NAME), (void *) destination, (void *) source,
+                                     (unsigned long) memop_chunk, (unsigned long) total_length););
+                MEM_OP(destination, source, memop_chunk);
+                destination += memop_chunk;
+                source += memop_chunk;
+                total_length -= memop_chunk;
             }
-            return 0;  /* completed */
+            return 0; /* completed */
         }
-        for( pos_desc = 0; (int32_t)pos_desc < count; pos_desc++ ) {
-            OPAL_DATATYPE_SAFEGUARD_POINTER( destination, datatype->size,
-                                        (unsigned char*)destination_base, datatype, count );
-            OPAL_DATATYPE_SAFEGUARD_POINTER( source, datatype->size,
-                                        (unsigned char*)source_base, datatype, count );
-            DO_DEBUG( opal_output( 0, "copy c2. %s( %p, %p, %lu ) => space %lu\n",
-                                   STRINGIFY(MEM_OP_NAME), (void*)destination, (void*)source, (unsigned long)datatype->size,
-                                   (unsigned long)(iov_len_local - (pos_desc * datatype->size)) ); );
-            MEM_OP( destination, source, datatype->size );
+        for (pos_desc = 0; (int32_t) pos_desc < count; pos_desc++) {
+            OPAL_DATATYPE_SAFEGUARD_POINTER(destination, datatype->size,
+                                            (unsigned char *) destination_base, datatype, count);
+            OPAL_DATATYPE_SAFEGUARD_POINTER(source, datatype->size, (unsigned char *) source_base,
+                                            datatype, count);
+            DO_DEBUG(opal_output(0, "copy c2. %s( %p, %p, %lu ) => space %lu\n",
+                                 STRINGIFY(MEM_OP_NAME), (void *) destination, (void *) source,
+                                 (unsigned long) datatype->size,
+                                 (unsigned long) (iov_len_local - (pos_desc * datatype->size))););
+            MEM_OP(destination, source, datatype->size);
             destination += extent;
             source += extent;
         }
-        return 0;  /* completed */
+        return 0; /* completed */
     }
 
-    pStack = (dt_stack_t*)alloca( sizeof(dt_stack_t) * (datatype->loops + 1) );
+    pStack = (dt_stack_t *) alloca(sizeof(dt_stack_t) * (datatype->loops + 1));
     pStack->count = count;
-    pStack->index   = -1;
-    pStack->disp    = 0;
+    pStack->index = -1;
+    pStack->disp = 0;
     pos_desc = 0;
     stack_pos = 0;
 
     description = datatype->opt_desc.desc;
-    if( NULL == description ) {
+    if (NULL == description) {
         description = datatype->desc.desc;
     }
 
-    UPDATE_INTERNAL_COUNTERS( description, 0, pElem, count_desc );
+    UPDATE_INTERNAL_COUNTERS(description, 0, pElem, count_desc);
 
-    while( 1 ) {
-        while( OPAL_LIKELY(pElem->elem.common.flags & OPAL_DATATYPE_FLAG_DATA) ) {
+    while (1) {
+        while (OPAL_LIKELY(pElem->elem.common.flags & OPAL_DATATYPE_FLAG_DATA)) {
             /* now here we have a basic datatype */
-            _predefined_data( pElem, datatype, (unsigned char*)source_base, count, count_desc,
-                              source, destination, &iov_len_local );
-            pos_desc++;  /* advance to the next data */
-            UPDATE_INTERNAL_COUNTERS( description, pos_desc, pElem, count_desc );
+            _predefined_data(pElem, datatype, (unsigned char *) source_base, count, count_desc,
+                             source, destination, &iov_len_local);
+            pos_desc++; /* advance to the next data */
+            UPDATE_INTERNAL_COUNTERS(description, pos_desc, pElem, count_desc);
         }
-        if( OPAL_DATATYPE_END_LOOP == pElem->elem.common.type ) { /* end of the current loop */
-            DO_DEBUG( opal_output( 0, "copy end_loop count %" PRIsize_t " stack_pos %d pos_desc %d disp %ld space %lu\n",
-                                   pStack->count, stack_pos, pos_desc, pStack->disp, (unsigned long)iov_len_local ); );
-            if( --(pStack->count) == 0 ) { /* end of loop */
-                if( stack_pos == 0 ) {
-                    assert( iov_len_local == 0 );
-                    return 0;  /* completed */
+        if (OPAL_DATATYPE_END_LOOP == pElem->elem.common.type) { /* end of the current loop */
+            DO_DEBUG(opal_output(0,
+                                 "copy end_loop count %" PRIsize_t
+                                 " stack_pos %d pos_desc %d disp %ld space %lu\n",
+                                 pStack->count, stack_pos, pos_desc, pStack->disp,
+                                 (unsigned long) iov_len_local););
+            if (--(pStack->count) == 0) { /* end of loop */
+                if (stack_pos == 0) {
+                    assert(iov_len_local == 0);
+                    return 0; /* completed */
                 }
                 stack_pos--;
                 pStack--;
                 pos_desc++;
             } else {
                 pos_desc = pStack->index + 1;
-                if( pStack->index == -1 ) {
+                if (pStack->index == -1) {
                     pStack->disp += (datatype->ub - datatype->lb);
                 } else {
-                    assert( OPAL_DATATYPE_LOOP == description[pStack->index].loop.common.type );
+                    assert(OPAL_DATATYPE_LOOP == description[pStack->index].loop.common.type);
                     pStack->disp += description[pStack->index].loop.extent;
                 }
             }
-            source      = (unsigned char*)source_base + pStack->disp;
-            destination = (unsigned char*)destination_base + pStack->disp;
-            UPDATE_INTERNAL_COUNTERS( description, pos_desc, pElem, count_desc );
-            DO_DEBUG( opal_output( 0, "copy new_loop count %" PRIsize_t " stack_pos %d pos_desc %d disp %ld space %lu\n",
-                                   pStack->count, stack_pos, pos_desc, pStack->disp, (unsigned long)iov_len_local ); );
+            source = (unsigned char *) source_base + pStack->disp;
+            destination = (unsigned char *) destination_base + pStack->disp;
+            UPDATE_INTERNAL_COUNTERS(description, pos_desc, pElem, count_desc);
+            DO_DEBUG(opal_output(0,
+                                 "copy new_loop count %" PRIsize_t
+                                 " stack_pos %d pos_desc %d disp %ld space %lu\n",
+                                 pStack->count, stack_pos, pos_desc, pStack->disp,
+                                 (unsigned long) iov_len_local););
         }
-        if( OPAL_DATATYPE_LOOP == pElem->elem.common.type ) {
-            ptrdiff_t local_disp = (ptrdiff_t)source;
-            if( pElem->loop.common.flags & OPAL_DATATYPE_FLAG_CONTIGUOUS ) {
-                _contiguous_loop( pElem, datatype, (unsigned char*)source_base, count, count_desc,
-                                  source, destination, &iov_len_local );
+        if (OPAL_DATATYPE_LOOP == pElem->elem.common.type) {
+            ptrdiff_t local_disp = (ptrdiff_t) source;
+            if (pElem->loop.common.flags & OPAL_DATATYPE_FLAG_CONTIGUOUS) {
+                _contiguous_loop(pElem, datatype, (unsigned char *) source_base, count, count_desc,
+                                 source, destination, &iov_len_local);
                 pos_desc += pElem->loop.items + 1;
                 goto update_loop_description;
             }
-            local_disp = (ptrdiff_t)source - local_disp;
-            PUSH_STACK( pStack, stack_pos, pos_desc, OPAL_DATATYPE_LOOP, count_desc,
-                        pStack->disp + local_disp);
+            local_disp = (ptrdiff_t) source - local_disp;
+            PUSH_STACK(pStack, stack_pos, pos_desc, OPAL_DATATYPE_LOOP, count_desc,
+                       pStack->disp + local_disp);
             pos_desc++;
-        update_loop_description:  /* update the current state */
-            source      = (unsigned char*)source_base + pStack->disp;
-            destination = (unsigned char*)destination_base + pStack->disp;
-            UPDATE_INTERNAL_COUNTERS( description, pos_desc, pElem, count_desc );
-            DDT_DUMP_STACK( pStack, stack_pos, pElem, "advance loop" );
+        update_loop_description: /* update the current state */
+            source = (unsigned char *) source_base + pStack->disp;
+            destination = (unsigned char *) destination_base + pStack->disp;
+            UPDATE_INTERNAL_COUNTERS(description, pos_desc, pElem, count_desc);
+            DDT_DUMP_STACK(pStack, stack_pos, pElem, "advance loop");
             continue;
         }
     }
diff --git a/opal/datatype/opal_datatype_create.c b/opal/datatype/opal_datatype_create.c
index 122521989b8..536bdb6bd87 100644
--- a/opal/datatype/opal_datatype_create.c
+++ b/opal/datatype/opal_datatype_create.c
@@ -24,62 +24,63 @@
 
 #include <stddef.h>
 
+#include "limits.h"
 #include "opal/constants.h"
 #include "opal/datatype/opal_datatype.h"
 #include "opal/datatype/opal_datatype_internal.h"
-#include "limits.h"
 #include "opal/prefetch.h"
 
-static void opal_datatype_construct( opal_datatype_t* pData )
+static void opal_datatype_construct(opal_datatype_t *pData)
 {
-    pData->size               = 0;
-    pData->flags              = OPAL_DATATYPE_FLAG_CONTIGUOUS;
-    pData->id                 = 0;
-    pData->bdt_used           = 0;
-    pData->size               = 0;
-    pData->true_lb            = LONG_MAX;
-    pData->true_ub            = LONG_MIN;
-    pData->lb                 = LONG_MAX;
-    pData->ub                 = LONG_MIN;
-    pData->align              = 1;
-    pData->nbElems            = 0;
+    pData->size = 0;
+    pData->flags = OPAL_DATATYPE_FLAG_CONTIGUOUS;
+    pData->id = 0;
+    pData->bdt_used = 0;
+    pData->size = 0;
+    pData->true_lb = LONG_MAX;
+    pData->true_ub = LONG_MIN;
+    pData->lb = LONG_MAX;
+    pData->ub = LONG_MIN;
+    pData->align = 1;
+    pData->nbElems = 0;
     memset(pData->name, 0, OPAL_MAX_OBJECT_NAME);
 
-    pData->desc.desc          = NULL;
-    pData->desc.length        = 0;
-    pData->desc.used          = 0;
+    pData->desc.desc = NULL;
+    pData->desc.length = 0;
+    pData->desc.used = 0;
 
-    pData->opt_desc.desc      = NULL;
-    pData->opt_desc.length    = 0;
-    pData->opt_desc.used      = 0;
+    pData->opt_desc.desc = NULL;
+    pData->opt_desc.length = 0;
+    pData->opt_desc.used = 0;
 
-    pData->ptypes             = NULL;
-    pData->loops              = 0;
+    pData->ptypes = NULL;
+    pData->loops = 0;
 }
 
-static void opal_datatype_destruct( opal_datatype_t* datatype )
+static void opal_datatype_destruct(opal_datatype_t *datatype)
 {
     /**
      * As the default description and the optimized description might point to the
      * same data description we should start by cleaning the optimized description.
      */
-    if( NULL != datatype->opt_desc.desc ) {
-        if( datatype->opt_desc.desc != datatype->desc.desc )
-            free( datatype->opt_desc.desc );
+    if (NULL != datatype->opt_desc.desc) {
+        if (datatype->opt_desc.desc != datatype->desc.desc) {
+            free(datatype->opt_desc.desc);
+        }
         datatype->opt_desc.length = 0;
-        datatype->opt_desc.used   = 0;
-        datatype->opt_desc.desc   = NULL;
+        datatype->opt_desc.used = 0;
+        datatype->opt_desc.desc = NULL;
     }
     if (!opal_datatype_is_predefined(datatype)) {
-        if( NULL != datatype->desc.desc ) {
-            free( datatype->desc.desc );
+        if (NULL != datatype->desc.desc) {
+            free(datatype->desc.desc);
             datatype->desc.length = 0;
-            datatype->desc.used   = 0;
-            datatype->desc.desc   = NULL;
+            datatype->desc.used = 0;
+            datatype->desc.desc = NULL;
         }
     }
     /* dont free the ptypes of predefined types (it was not dynamically allocated) */
-    if( (NULL != datatype->ptypes) && (!opal_datatype_is_predefined(datatype)) ) {
+    if ((NULL != datatype->ptypes) && (!opal_datatype_is_predefined(datatype))) {
         free(datatype->ptypes);
         datatype->ptypes = NULL;
     }
@@ -90,27 +91,32 @@ static void opal_datatype_destruct( opal_datatype_t* datatype )
 
 OBJ_CLASS_INSTANCE(opal_datatype_t, opal_object_t, opal_datatype_construct, opal_datatype_destruct);
 
-opal_datatype_t* opal_datatype_create( int32_t expectedSize )
+opal_datatype_t *opal_datatype_create(int32_t expectedSize)
 {
-    opal_datatype_t* datatype = (opal_datatype_t*)OBJ_NEW(opal_datatype_t);
+    opal_datatype_t *datatype = (opal_datatype_t *) OBJ_NEW(opal_datatype_t);
 
-    if( expectedSize == -1 ) expectedSize = DT_INCREASE_STACK;
-    datatype->desc.length = expectedSize + 1;  /* one for the fake elem at the end */
-    datatype->desc.used   = 0;
-    datatype->desc.desc   = (dt_elem_desc_t*)calloc(datatype->desc.length, sizeof(dt_elem_desc_t));
-    /* BEWARE: an upper-layer configured with OPAL_MAX_OBJECT_NAME different than the OPAL-layer will not work! */
-    memset( datatype->name, 0, OPAL_MAX_OBJECT_NAME );
+    if (expectedSize == -1) {
+        expectedSize = DT_INCREASE_STACK;
+    }
+    datatype->desc.length = expectedSize + 1; /* one for the fake elem at the end */
+    datatype->desc.used = 0;
+    datatype->desc.desc = (dt_elem_desc_t *) calloc(datatype->desc.length, sizeof(dt_elem_desc_t));
+    /* BEWARE: an upper-layer configured with OPAL_MAX_OBJECT_NAME different than the OPAL-layer
+     * will not work! */
+    memset(datatype->name, 0, OPAL_MAX_OBJECT_NAME);
     return datatype;
 }
 
-int32_t opal_datatype_create_desc( opal_datatype_t * datatype, int32_t expectedSize )
+int32_t opal_datatype_create_desc(opal_datatype_t *datatype, int32_t expectedSize)
 {
-    if( expectedSize == -1 )
+    if (expectedSize == -1) {
         expectedSize = DT_INCREASE_STACK;
-    datatype->desc.length = expectedSize + 1;  /* one for the fake elem at the end */
-    datatype->desc.used   = 0;
-    datatype->desc.desc   = (dt_elem_desc_t*)calloc(datatype->desc.length, sizeof(dt_elem_desc_t));
-    if (NULL == datatype->desc.desc)
+    }
+    datatype->desc.length = expectedSize + 1; /* one for the fake elem at the end */
+    datatype->desc.used = 0;
+    datatype->desc.desc = (dt_elem_desc_t *) calloc(datatype->desc.length, sizeof(dt_elem_desc_t));
+    if (NULL == datatype->desc.desc) {
         return OPAL_ERR_OUT_OF_RESOURCE;
+    }
     return OPAL_SUCCESS;
 }
diff --git a/opal/datatype/opal_datatype_create_contiguous.c b/opal/datatype/opal_datatype_create_contiguous.c
index 3db53000d16..28664d64c8c 100644
--- a/opal/datatype/opal_datatype_create_contiguous.c
+++ b/opal/datatype/opal_datatype_create_contiguous.c
@@ -24,17 +24,17 @@
 #include "opal/datatype/opal_datatype.h"
 #include "opal/datatype/opal_datatype_internal.h"
 
-int32_t opal_datatype_create_contiguous( int count, const opal_datatype_t* oldType,
-                                         opal_datatype_t** newType )
+int32_t opal_datatype_create_contiguous(int count, const opal_datatype_t *oldType,
+                                        opal_datatype_t **newType)
 {
-    opal_datatype_t* pdt;
+    opal_datatype_t *pdt;
 
-    if( 0 == count ) {
-        pdt = opal_datatype_create( 0 );
-        opal_datatype_add( pdt, &opal_datatype_empty, 0, 0, 0 );
+    if (0 == count) {
+        pdt = opal_datatype_create(0);
+        opal_datatype_add(pdt, &opal_datatype_empty, 0, 0, 0);
     } else {
-        pdt = opal_datatype_create( oldType->desc.used + 2 );
-        opal_datatype_add( pdt, oldType, count, 0, (oldType->ub - oldType->lb) );
+        pdt = opal_datatype_create(oldType->desc.used + 2);
+        opal_datatype_add(pdt, oldType, count, 0, (oldType->ub - oldType->lb));
     }
     *newType = pdt;
     return OPAL_SUCCESS;
diff --git a/opal/datatype/opal_datatype_destroy.c b/opal/datatype/opal_datatype_destroy.c
index d468cd07e8c..3381df622bf 100644
--- a/opal/datatype/opal_datatype_destroy.c
+++ b/opal/datatype/opal_datatype_destroy.c
@@ -23,15 +23,15 @@
 #include "opal/datatype/opal_datatype.h"
 #include "opal/datatype/opal_datatype_internal.h"
 
-int32_t opal_datatype_destroy( opal_datatype_t** dt )
+int32_t opal_datatype_destroy(opal_datatype_t **dt)
 {
-    opal_datatype_t* pData = *dt;
+    opal_datatype_t *pData = *dt;
 
-    if( (pData->flags & OPAL_DATATYPE_FLAG_PREDEFINED) &&
-        (pData->super.obj_reference_count <= 1) )
+    if ((pData->flags & OPAL_DATATYPE_FLAG_PREDEFINED) && (pData->super.obj_reference_count <= 1)) {
         return OPAL_ERROR;
+    }
 
-    OBJ_RELEASE( pData );
+    OBJ_RELEASE(pData);
     *dt = NULL;
     return OPAL_SUCCESS;
 }
diff --git a/opal/datatype/opal_datatype_dump.c b/opal/datatype/opal_datatype_dump.c
index 27903db657e..259dcc3152f 100644
--- a/opal/datatype/opal_datatype_dump.c
+++ b/opal/datatype/opal_datatype_dump.c
@@ -24,134 +24,183 @@
 
 #include "opal_config.h"
 
-#include <stdlib.h>
 #include <stdio.h>
+#include <stdlib.h>
 
 #include "opal/constants.h"
-#include "opal/util/output.h"
 #include "opal/datatype/opal_datatype.h"
 #include "opal/datatype/opal_datatype_internal.h"
+#include "opal/util/output.h"
 
 /********************************************************
  * Data dumping functions
  ********************************************************/
 
-int opal_datatype_contain_basic_datatypes( const opal_datatype_t* pData, char* ptr, size_t length )
+int opal_datatype_contain_basic_datatypes(const opal_datatype_t *pData, char *ptr, size_t length)
 {
     int i;
     int32_t index = 0;
     uint64_t mask = 1;
 
-    if( pData->flags & OPAL_DATATYPE_FLAG_USER_LB ) index += snprintf( ptr, length - index, "lb " );
-    if( pData->flags & OPAL_DATATYPE_FLAG_USER_UB ) index += snprintf( ptr + index, length - index, "ub " );
-    for( i = 0; i < OPAL_DATATYPE_MAX_PREDEFINED; i++ ) {
-        if( pData->bdt_used & mask ) {
-            if( NULL == pData->ptypes ) {
-                index += snprintf( ptr + index, length - index, "%s:* ", opal_datatype_basicDatatypes[i]->name );
+    if (pData->flags & OPAL_DATATYPE_FLAG_USER_LB) {
+        index += snprintf(ptr, length - index, "lb ");
+    }
+    if (pData->flags & OPAL_DATATYPE_FLAG_USER_UB) {
+        index += snprintf(ptr + index, length - index, "ub ");
+    }
+    for (i = 0; i < OPAL_DATATYPE_MAX_PREDEFINED; i++) {
+        if (pData->bdt_used & mask) {
+            if (NULL == pData->ptypes) {
+                index += snprintf(ptr + index, length - index, "%s:* ",
+                                  opal_datatype_basicDatatypes[i]->name);
             } else {
-                index += snprintf( ptr + index, length - index, "%s:%" PRIsize_t " ", opal_datatype_basicDatatypes[i]->name,
-                                   pData->ptypes[i]);
+                index += snprintf(ptr + index, length - index, "%s:%" PRIsize_t " ",
+                                  opal_datatype_basicDatatypes[i]->name, pData->ptypes[i]);
             }
         }
         mask <<= 1;
-        if( length <= (size_t)index ) break;
+        if (length <= (size_t) index) {
+            break;
+        }
     }
     return index;
 }
 
-int opal_datatype_dump_data_flags( unsigned short usflags, char* ptr, size_t length )
+int opal_datatype_dump_data_flags(unsigned short usflags, char *ptr, size_t length)
 {
     int index = 0;
-    if( length < 22 ) return 0;
-    index = snprintf( ptr, 22, "-----------[---][---]" );  /* set everything to - */
-    if( usflags & OPAL_DATATYPE_FLAG_COMMITTED )  ptr[1]  = 'c';
-    if( usflags & OPAL_DATATYPE_FLAG_CONTIGUOUS ) ptr[2]  = 'C';
-    if( usflags & OPAL_DATATYPE_FLAG_OVERLAP )    ptr[3]  = 'o';
-    if( usflags & OPAL_DATATYPE_FLAG_USER_LB )    ptr[4]  = 'l';
-    if( usflags & OPAL_DATATYPE_FLAG_USER_UB )    ptr[5]  = 'u';
-    if( usflags & OPAL_DATATYPE_FLAG_PREDEFINED ) ptr[6]  = 'P';
-    if( !(usflags & OPAL_DATATYPE_FLAG_NO_GAPS) ) ptr[7]  = 'G';
-    if( usflags & OPAL_DATATYPE_FLAG_DATA )       ptr[8]  = 'D';
-    if( (usflags & OPAL_DATATYPE_FLAG_BASIC) == OPAL_DATATYPE_FLAG_BASIC ) ptr[9]  = 'B';
+    if (length < 22) {
+        return 0;
+    }
+    index = snprintf(ptr, 22, "-----------[---][---]"); /* set everything to - */
+    if (usflags & OPAL_DATATYPE_FLAG_COMMITTED) {
+        ptr[1] = 'c';
+    }
+    if (usflags & OPAL_DATATYPE_FLAG_CONTIGUOUS) {
+        ptr[2] = 'C';
+    }
+    if (usflags & OPAL_DATATYPE_FLAG_OVERLAP) {
+        ptr[3] = 'o';
+    }
+    if (usflags & OPAL_DATATYPE_FLAG_USER_LB) {
+        ptr[4] = 'l';
+    }
+    if (usflags & OPAL_DATATYPE_FLAG_USER_UB) {
+        ptr[5] = 'u';
+    }
+    if (usflags & OPAL_DATATYPE_FLAG_PREDEFINED) {
+        ptr[6] = 'P';
+    }
+    if (!(usflags & OPAL_DATATYPE_FLAG_NO_GAPS)) {
+        ptr[7] = 'G';
+    }
+    if (usflags & OPAL_DATATYPE_FLAG_DATA) {
+        ptr[8] = 'D';
+    }
+    if ((usflags & OPAL_DATATYPE_FLAG_BASIC) == OPAL_DATATYPE_FLAG_BASIC) {
+        ptr[9] = 'B';
+    } else if (usflags & OPAL_DATATYPE_OPTIMIZED_RESTRICTED) {
+        ptr[9] = 'H';  /* optimized description restricted to homogeneous cases */
+    }
     /* We know nothing about the upper level language or flags! */
     /* ... */
     return index;
 }
 
-
-int opal_datatype_dump_data_desc( dt_elem_desc_t* pDesc, int nbElems, char* ptr, size_t length )
+int opal_datatype_dump_data_desc(dt_elem_desc_t *pDesc, int nbElems, char *ptr, size_t length)
 {
     int i;
     int32_t index = 0;
 
-    for( i = 0; i < nbElems; i++ ) {
-        index += opal_datatype_dump_data_flags( pDesc->elem.common.flags, ptr + index, length );
-        if( length <= (size_t)index ) break;
-        index += snprintf( ptr + index, length - index, "%15s ", opal_datatype_basicDatatypes[pDesc->elem.common.type]->name );
-        if( length <= (size_t)index ) break;
-        if( OPAL_DATATYPE_LOOP == pDesc->elem.common.type )
-            index += snprintf( ptr + index, length - index, "%u times the next %u elements extent %td\n",
-                               pDesc->loop.loops, pDesc->loop.items,
-                               pDesc->loop.extent );
-        else if( OPAL_DATATYPE_END_LOOP == pDesc->elem.common.type )
-            index += snprintf( ptr + index, length - index, "prev %u elements first elem displacement %td size of data %" PRIsize_t "\n",
-                               pDesc->end_loop.items, pDesc->end_loop.first_elem_disp,
-                               pDesc->end_loop.size );
-        else
-            index += snprintf( ptr + index, length - index, "count %u disp 0x%tx (%td) blen %" PRIsize_t " extent %td (size %zd)\n",
-                               pDesc->elem.count, pDesc->elem.disp, pDesc->elem.disp, pDesc->elem.blocklen,
-                               pDesc->elem.extent, (pDesc->elem.count * pDesc->elem.blocklen * opal_datatype_basicDatatypes[pDesc->elem.common.type]->size) );
+    for (i = 0; i < nbElems; i++) {
+        index += opal_datatype_dump_data_flags(pDesc->elem.common.flags, ptr + index, length);
+        if (length <= (size_t) index) {
+            break;
+        }
+        index += snprintf(ptr + index, length - index, "%15s ",
+                          opal_datatype_basicDatatypes[pDesc->elem.common.type]->name);
+        if (length <= (size_t) index) {
+            break;
+        }
+        if (OPAL_DATATYPE_LOOP == pDesc->elem.common.type) {
+            index += snprintf(ptr + index, length - index,
+                              "%u times the next %u elements extent %td\n", pDesc->loop.loops,
+                              pDesc->loop.items, pDesc->loop.extent);
+        } else if (OPAL_DATATYPE_END_LOOP == pDesc->elem.common.type) {
+            index += snprintf(
+                ptr + index, length - index,
+                "prev %u elements first elem displacement %td size of data %" PRIsize_t "\n",
+                pDesc->end_loop.items, pDesc->end_loop.first_elem_disp, pDesc->end_loop.size);
+        } else {
+            index += snprintf(ptr + index, length - index,
+                              "count %u disp 0x%tx (%td) blen %" PRIsize_t
+                              " extent %td (size %zd)\n",
+                              pDesc->elem.count, pDesc->elem.disp, pDesc->elem.disp,
+                              pDesc->elem.blocklen, pDesc->elem.extent,
+                              (pDesc->elem.count * pDesc->elem.blocklen
+                               * opal_datatype_basicDatatypes[pDesc->elem.common.type]->size));
+        }
         pDesc++;
 
-        if( length <= (size_t)index ) break;
+        if (length <= (size_t) index) {
+            break;
+        }
     }
     return index;
 }
 
-
-void opal_datatype_dump( const opal_datatype_t* pData )
+void opal_datatype_dump(const opal_datatype_t *pData)
 {
     size_t length;
     int index = 0;
-    char* buffer;
+    char *buffer;
 
     length = pData->opt_desc.used + pData->desc.used;
     length = length * 100 + 500;
-    buffer = (char*)malloc( length );
-    index += snprintf( buffer, length - index, "Datatype %p[%s] size %" PRIsize_t " align %u id %u length %" PRIsize_t " used %" PRIsize_t "\n"
-                                               "true_lb %td true_ub %td (true_extent %td) lb %td ub %td (extent %td)\n"
-                                               "nbElems %" PRIsize_t " loops %u flags %X (",
-                       (void*)pData, pData->name, pData->size, pData->align, (uint32_t)pData->id, pData->desc.length, pData->desc.used,
-                       pData->true_lb, pData->true_ub, pData->true_ub - pData->true_lb,
-                       pData->lb, pData->ub, pData->ub - pData->lb,
-                       pData->nbElems, pData->loops, (int)pData->flags );
+    buffer = (char *) malloc(length);
+    index += snprintf(buffer, length - index,
+                      "Datatype %p[%s] size %" PRIsize_t " align %u id %u length %" PRIsize_t
+                      " used %" PRIsize_t "\n"
+                      "true_lb %td true_ub %td (true_extent %td) lb %td ub %td (extent %td)\n"
+                      "nbElems %" PRIsize_t " loops %u flags %X (",
+                      (void *) pData, pData->name, pData->size, pData->align, (uint32_t) pData->id,
+                      pData->desc.length, pData->desc.used, pData->true_lb, pData->true_ub,
+                      pData->true_ub - pData->true_lb, pData->lb, pData->ub, pData->ub - pData->lb,
+                      pData->nbElems, pData->loops, (int) pData->flags);
     /* dump the flags */
-    if( pData->flags == OPAL_DATATYPE_FLAG_PREDEFINED )
-        index += snprintf( buffer + index, length - index, "predefined " );
-    else {
-        if( pData->flags & OPAL_DATATYPE_FLAG_COMMITTED ) index += snprintf( buffer + index, length - index, "committed " );
-        if( pData->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS) index += snprintf( buffer + index, length - index, "contiguous " );
-    }
-    index += snprintf( buffer + index, length - index, ")" );
-    index += opal_datatype_dump_data_flags( pData->flags, buffer + index, length - index );
+    if (pData->flags == OPAL_DATATYPE_FLAG_PREDEFINED) {
+        index += snprintf(buffer + index, length - index, "predefined ");
+    } else {
+        if (pData->flags & OPAL_DATATYPE_FLAG_COMMITTED) {
+            index += snprintf(buffer + index, length - index, "committed ");
+        }
+        if (pData->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS) {
+            index += snprintf(buffer + index, length - index, "contiguous ");
+        }
+    }
+    index += snprintf(buffer + index, length - index, ")");
+    index += opal_datatype_dump_data_flags(pData->flags, buffer + index, length - index);
     {
-        index += snprintf( buffer + index, length - index, "\n   contain " );
-        index += opal_datatype_contain_basic_datatypes( pData, buffer + index, length - index );
-        index += snprintf( buffer + index, length - index, "\n" );
+        index += snprintf(buffer + index, length - index, "\n   contain ");
+        index += opal_datatype_contain_basic_datatypes(pData, buffer + index, length - index);
+        index += snprintf(buffer + index, length - index, "\n");
     }
-    if( (pData->opt_desc.desc != pData->desc.desc) && (NULL != pData->opt_desc.desc) ) {
+    if ((pData->opt_desc.desc != pData->desc.desc) && (NULL != pData->opt_desc.desc)) {
         /* If the data is already committed print everything including the last
          * fake OPAL_DATATYPE_END_LOOP entry.
          */
-        index += opal_datatype_dump_data_desc( pData->desc.desc, pData->desc.used + 1, buffer + index, length - index );
-        index += snprintf( buffer + index, length - index, "Optimized description \n" );
-        index += opal_datatype_dump_data_desc( pData->opt_desc.desc, pData->opt_desc.used + 1, buffer + index, length - index );
+        index += opal_datatype_dump_data_desc(pData->desc.desc, pData->desc.used + 1,
+                                              buffer + index, length - index);
+        index += snprintf(buffer + index, length - index, "Optimized description \n");
+        index += opal_datatype_dump_data_desc(pData->opt_desc.desc, pData->opt_desc.used + 1,
+                                              buffer + index, length - index);
     } else {
-        index += opal_datatype_dump_data_desc( pData->desc.desc, pData->desc.used, buffer + index, length - index );
-        index += snprintf( buffer + index, length - index, "No optimized description\n" );
+        index += opal_datatype_dump_data_desc(pData->desc.desc, pData->desc.used, buffer + index,
+                                              length - index);
+        index += snprintf(buffer + index, length - index, "No optimized description\n");
     }
-    buffer[index] = '\0';  /* make sure we end the string with 0 */
-    opal_output( 0, "%s\n", buffer );
+    buffer[index] = '\0'; /* make sure we end the string with 0 */
+    opal_output(0, "%s\n", buffer);
 
     free(buffer);
 }
diff --git a/opal/datatype/opal_datatype_fake_stack.c b/opal/datatype/opal_datatype_fake_stack.c
index aee2ccf9058..dffd20be8a8 100644
--- a/opal/datatype/opal_datatype_fake_stack.c
+++ b/opal/datatype/opal_datatype_fake_stack.c
@@ -26,32 +26,31 @@
 #include <stdlib.h>
 
 #ifdef HAVE_ALLOCA_H
-#include <alloca.h>
+#    include <alloca.h>
 #endif
 
-#include "opal/datatype/opal_datatype.h"
 #include "opal/datatype/opal_convertor.h"
+#include "opal/datatype/opal_datatype.h"
 #include "opal/datatype/opal_datatype_internal.h"
 
+extern int opal_convertor_create_stack_with_pos_general(opal_convertor_t *convertor,
+                                                        size_t starting_point, const size_t *sizes);
 
-extern int opal_convertor_create_stack_with_pos_general( opal_convertor_t* convertor,
-                                                         size_t starting_point, const size_t* sizes );
-
-int opal_convertor_create_stack_with_pos_general( opal_convertor_t* pConvertor,
-                                                  size_t starting_point, const size_t* sizes )
+int opal_convertor_create_stack_with_pos_general(opal_convertor_t *pConvertor,
+                                                 size_t starting_point, const size_t *sizes)
 {
-    dt_stack_t* pStack;   /* pointer to the position on the stack */
-    int pos_desc;         /* actual position in the description of the derived datatype */
+    dt_stack_t *pStack; /* pointer to the position on the stack */
+    int pos_desc;       /* actual position in the description of the derived datatype */
     size_t lastLength = 0;
-    const opal_datatype_t* pData = pConvertor->pDesc;
+    const opal_datatype_t *pData = pConvertor->pDesc;
     size_t loop_length, *remoteLength, remote_size;
     size_t resting_place = starting_point;
-    dt_elem_desc_t* pElems;
+    dt_elem_desc_t *pElems;
     size_t count;
 
-    assert( 0 != starting_point );
-    assert( pConvertor->bConverted != starting_point );
-    assert( starting_point <=(pConvertor->count * pData->size) );
+    assert(0 != starting_point);
+    assert(pConvertor->bConverted != starting_point);
+    assert(starting_point <= (pConvertor->count * pData->size));
 
     /*opal_output( 0, "Data extent %d size %d count %d total_size %d starting_point %d\n",
                  pData->ub - pData->lb, pData->size, pConvertor->count,
@@ -64,24 +63,25 @@ int opal_convertor_create_stack_with_pos_general( opal_convertor_t* pConvertor,
      */
     pElems = pConvertor->use_desc->desc;
 
-    if( (pConvertor->flags & CONVERTOR_HOMOGENEOUS) && (pData->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS) ) {
+    if ((pConvertor->flags & CONVERTOR_HOMOGENEOUS)
+        && (pData->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS)) {
         /* Special case for contiguous datatypes */
         int32_t cnt = (int32_t)(starting_point / pData->size);
         ptrdiff_t extent = pData->ub - pData->lb;
 
-        loop_length = GET_FIRST_NON_LOOP( pElems );
-        pStack[0].disp  = pElems[loop_length].elem.disp;
-        pStack[0].type  = OPAL_DATATYPE_LOOP;
+        loop_length = GET_FIRST_NON_LOOP(pElems);
+        pStack[0].disp = pElems[loop_length].elem.disp;
+        pStack[0].type = OPAL_DATATYPE_LOOP;
         pStack[0].count = pConvertor->count - cnt;
-        cnt = (int32_t)(starting_point - cnt * pData->size);  /* number of bytes after the loop */
-        pStack[1].index    = 0;
-        pStack[1].type     = OPAL_DATATYPE_UINT1;
-        pStack[1].disp     = pStack[0].disp;
-        pStack[1].count    = pData->size - cnt;
+        cnt = (int32_t)(starting_point - cnt * pData->size); /* number of bytes after the loop */
+        pStack[1].index = 0;
+        pStack[1].type = OPAL_DATATYPE_UINT1;
+        pStack[1].disp = pStack[0].disp;
+        pStack[1].count = pData->size - cnt;
 
-        if( (ptrdiff_t)pData->size == extent ) { /* all elements are contiguous */
+        if ((ptrdiff_t) pData->size == extent) { /* all elements are contiguous */
             pStack[1].disp += starting_point;
-        } else {  /* each is contiguous but there are gaps inbetween */
+        } else { /* each is contiguous but there are gaps inbetween */
             pStack[1].disp += (pConvertor->count - pStack[0].count) * extent + cnt;
         }
 
@@ -91,47 +91,47 @@ int opal_convertor_create_stack_with_pos_general( opal_convertor_t* pConvertor,
     }
 
     /* remove from the main loop all the complete datatypes */
-    assert (! (pConvertor->flags & CONVERTOR_SEND));
-    remote_size    = opal_convertor_compute_remote_size( pConvertor );
-    count          = starting_point / remote_size;
+    assert(!(pConvertor->flags & CONVERTOR_SEND));
+    remote_size = opal_convertor_compute_remote_size(pConvertor);
+    count = starting_point / remote_size;
     resting_place -= (remote_size * count);
-    pStack->count  = pConvertor->count - count;
-    pStack->index  = -1;
+    pStack->count = pConvertor->count - count;
+    pStack->index = -1;
 
-    loop_length = GET_FIRST_NON_LOOP( pElems );
+    loop_length = GET_FIRST_NON_LOOP(pElems);
     pStack->disp = count * (pData->ub - pData->lb) + pElems[loop_length].elem.disp;
 
-    pos_desc  = 0;
-    remoteLength = (size_t*)alloca( sizeof(size_t) * (pConvertor->pDesc->loops + 1));
-    remoteLength[0] = 0;  /* initial value set to ZERO */
+    pos_desc = 0;
+    remoteLength = (size_t *) alloca(sizeof(size_t) * (pConvertor->pDesc->loops + 1));
+    remoteLength[0] = 0; /* initial value set to ZERO */
     loop_length = 0;
 
     /* The only way to get out of this loop is when we reach the desired position or
      * when we finish the whole datatype.
      */
-    while( pos_desc < (int32_t)pConvertor->use_desc->used ) {
-        if( OPAL_DATATYPE_END_LOOP == pElems->elem.common.type ) { /* end of the current loop */
-            ddt_endloop_desc_t* end_loop = (ddt_endloop_desc_t*)pElems;
+    while (pos_desc < (int32_t) pConvertor->use_desc->used) {
+        if (OPAL_DATATYPE_END_LOOP == pElems->elem.common.type) { /* end of the current loop */
+            ddt_endloop_desc_t *end_loop = (ddt_endloop_desc_t *) pElems;
             ptrdiff_t extent;
 
-            if( (loop_length * pStack->count) > resting_place ) {
+            if ((loop_length * pStack->count) > resting_place) {
                 /* We will stop somewhere on this loop. To avoid moving inside the loop
                  * multiple times, we can compute the index of the loop where we will
                  * stop. Once this index is computed we can then reparse the loop once
                  * until we find the correct position.
                  */
                 int32_t cnt = (int32_t)(resting_place / loop_length);
-                if( pStack->index == -1 ) {
+                if (pStack->index == -1) {
                     extent = pData->ub - pData->lb;
                 } else {
-                    assert( OPAL_DATATYPE_LOOP == (pElems - end_loop->items)->loop.common.type );
-                    extent = ((ddt_loop_desc_t*)(pElems - end_loop->items))->extent;
+                    assert(OPAL_DATATYPE_LOOP == (pElems - end_loop->items)->loop.common.type);
+                    extent = ((ddt_loop_desc_t *) (pElems - end_loop->items))->extent;
                 }
                 pStack->count -= (cnt + 1);
                 resting_place -= cnt * loop_length;
                 pStack->disp += (cnt + 1) * extent;
                 /* reset the remoteLength as we act as restarting the last loop */
-                pos_desc -= (end_loop->items - 1);  /* go back to the first element in the loop */
+                pos_desc -= (end_loop->items - 1); /* go back to the first element in the loop */
                 pElems -= (end_loop->items - 1);
                 remoteLength[pConvertor->stack_pos] = 0;
                 loop_length = 0;
@@ -140,7 +140,7 @@ int opal_convertor_create_stack_with_pos_general( opal_convertor_t* pConvertor,
             /* Not in this loop. Cleanup the stack and advance to the
              * next data description.
              */
-            resting_place -= (loop_length * (pStack->count - 1));  /* update the resting place */
+            resting_place -= (loop_length * (pStack->count - 1)); /* update the resting place */
             pStack--;
             pConvertor->stack_pos--;
             pos_desc++;
@@ -149,34 +149,33 @@ int opal_convertor_create_stack_with_pos_general( opal_convertor_t* pConvertor,
             loop_length = remoteLength[pConvertor->stack_pos];
             continue;
         }
-        if( OPAL_DATATYPE_LOOP == pElems->elem.common.type ) {
+        if (OPAL_DATATYPE_LOOP == pElems->elem.common.type) {
             remoteLength[pConvertor->stack_pos] += loop_length;
-            PUSH_STACK( pStack, pConvertor->stack_pos, pos_desc, OPAL_DATATYPE_LOOP,
-                        pElems->loop.loops, pStack->disp );
+            PUSH_STACK(pStack, pConvertor->stack_pos, pos_desc, OPAL_DATATYPE_LOOP,
+                       pElems->loop.loops, pStack->disp);
             pos_desc++;
             pElems++;
             remoteLength[pConvertor->stack_pos] = 0;
-            loop_length = 0;  /* starting a new loop */
+            loop_length = 0; /* starting a new loop */
         }
-        while( pElems->elem.common.flags & OPAL_DATATYPE_FLAG_DATA ) {
+        while (pElems->elem.common.flags & OPAL_DATATYPE_FLAG_DATA) {
             /* now here we have a basic datatype */
-            const opal_datatype_t* basic_type = BASIC_DDT_FROM_ELEM( (*pElems) );
-            lastLength = (size_t)pElems->elem.count * basic_type->size;
-            if( resting_place < lastLength ) {
+            const opal_datatype_t *basic_type = BASIC_DDT_FROM_ELEM((*pElems));
+            lastLength = (size_t) pElems->elem.count * basic_type->size;
+            if (resting_place < lastLength) {
                 int32_t cnt = (int32_t)(resting_place / basic_type->size);
                 loop_length += (cnt * basic_type->size);
                 resting_place -= (cnt * basic_type->size);
-                PUSH_STACK( pStack, pConvertor->stack_pos, pos_desc, pElems->elem.common.type,
-                            pElems->elem.count - cnt,
-                            pElems->elem.disp + cnt * pElems->elem.extent );
+                PUSH_STACK(pStack, pConvertor->stack_pos, pos_desc, pElems->elem.common.type,
+                           pElems->elem.count - cnt, pElems->elem.disp + cnt * pElems->elem.extent);
                 pConvertor->bConverted = starting_point - resting_place;
-                DDT_DUMP_STACK( pConvertor->pStack, pConvertor->stack_pos,
-                                pConvertor->pDesc->desc.desc, pConvertor->pDesc->name );
+                DDT_DUMP_STACK(pConvertor->pStack, pConvertor->stack_pos,
+                               pConvertor->pDesc->desc.desc, pConvertor->pDesc->name);
                 return OPAL_SUCCESS;
             }
             loop_length += lastLength;
             resting_place -= lastLength;
-            pos_desc++;  /* advance to the next data */
+            pos_desc++; /* advance to the next data */
             pElems++;
         }
     }
diff --git a/opal/datatype/opal_datatype_get_count.c b/opal/datatype/opal_datatype_get_count.c
index 12dba4d0d48..e4b3ccc2431 100644
--- a/opal/datatype/opal_datatype_get_count.c
+++ b/opal/datatype/opal_datatype_get_count.c
@@ -4,6 +4,7 @@
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2009      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -12,12 +13,12 @@
  */
 
 #include "opal_config.h"
-#include "opal/datatype/opal_datatype.h"
 #include "opal/datatype/opal_convertor.h"
+#include "opal/datatype/opal_datatype.h"
 #include "opal/datatype/opal_datatype_internal.h"
 
 #ifdef HAVE_ALLOCA_H
-#include <alloca.h>
+#    include <alloca.h>
 #endif
 
 /* Get the number of elements from the data-type that can be
@@ -28,68 +29,76 @@
  *   positive = number of basic elements inside
  *   negative = some error occurs
  */
-ssize_t opal_datatype_get_element_count( const opal_datatype_t* datatype, size_t iSize )
+ssize_t opal_datatype_get_element_count(const opal_datatype_t *datatype, size_t iSize)
 {
-    dt_stack_t* pStack;   /* pointer to the position on the stack */
-    uint32_t pos_desc;    /* actual position in the description of the derived datatype */
+    dt_stack_t *pStack; /* pointer to the position on the stack */
+    uint32_t pos_desc;  /* actual position in the description of the derived datatype */
     ssize_t nbElems = 0, stack_pos = 0;
     size_t local_size;
-    dt_elem_desc_t* pElems;
+    dt_elem_desc_t *pElems;
 
     /* Normally the size should be less or equal to the size of the datatype.
      * This function does not support a iSize bigger than the size of the datatype.
      */
-    assert( iSize <= datatype->size );
-    DUMP( "dt_count_elements( %p, %ul )\n", (void*)datatype, (unsigned long)iSize );
-    pStack = (dt_stack_t*)alloca( sizeof(dt_stack_t) * (datatype->loops + 2) );
-    pStack->count    = 1;
-    pStack->index    = -1;
-    pStack->disp     = 0;
-    pElems           = datatype->desc.desc;
-    pos_desc         = 0;
-
-    while( 1 ) {  /* loop forever the exit condition is on the last OPAL_DATATYPE_END_LOOP */
-        if( OPAL_DATATYPE_END_LOOP == pElems[pos_desc].elem.common.type ) { /* end of the current loop */
-            if( --(pStack->count) == 0 ) { /* end of loop */
-                stack_pos--; pStack--;
-                if( stack_pos == -1 ) return nbElems;  /* completed */
-                pos_desc++;  /* advance to the next element after the end loop */
+    assert(iSize <= datatype->size);
+    DUMP("dt_count_elements( %p, %ul )\n", (void *) datatype, (unsigned long) iSize);
+    pStack = (dt_stack_t *) alloca(sizeof(dt_stack_t) * (datatype->loops + 2));
+    pStack->count = 1;
+    pStack->index = -1;
+    pStack->disp = 0;
+    pElems = datatype->desc.desc;
+    pos_desc = 0;
+
+    while (1) { /* loop forever the exit condition is on the last OPAL_DATATYPE_END_LOOP */
+        if (OPAL_DATATYPE_END_LOOP
+            == pElems[pos_desc].elem.common.type) { /* end of the current loop */
+            if (--(pStack->count) == 0) {           /* end of loop */
+                stack_pos--;
+                pStack--;
+                if (stack_pos == -1) {
+                    return nbElems; /* completed */
+                }
+                pos_desc++; /* advance to the next element after the end loop */
             } else {
-                pos_desc = pStack->index + 1;  /* go back to the begining of the loop */
+                pos_desc = pStack->index + 1; /* go back to the beginning of the loop */
             }
             continue;
         }
-        if( OPAL_DATATYPE_LOOP == pElems[pos_desc].elem.common.type ) {
+        if (OPAL_DATATYPE_LOOP == pElems[pos_desc].elem.common.type) {
             do {
-                PUSH_STACK( pStack, stack_pos, pos_desc, OPAL_DATATYPE_LOOP, pElems[pos_desc].loop.loops, 0 );
+                PUSH_STACK(pStack, stack_pos, pos_desc, OPAL_DATATYPE_LOOP,
+                           pElems[pos_desc].loop.loops, 0);
                 pos_desc++;
-            } while( OPAL_DATATYPE_LOOP == pElems[pos_desc].elem.common.type ); /* let's start another loop */
-            DDT_DUMP_STACK( pStack, stack_pos, pElems, "advance loops" );
+            } while (OPAL_DATATYPE_LOOP
+                     == pElems[pos_desc].elem.common.type); /* let's start another loop */
+            DDT_DUMP_STACK(pStack, stack_pos, pElems, "advance loops");
         }
-        while( pElems[pos_desc].elem.common.flags & OPAL_DATATYPE_FLAG_DATA ) {
+        while (pElems[pos_desc].elem.common.flags & OPAL_DATATYPE_FLAG_DATA) {
             /* now here we have a basic datatype */
-            const opal_datatype_t* basic_type = BASIC_DDT_FROM_ELEM(pElems[pos_desc]);
-            local_size = ((size_t)pElems[pos_desc].elem.count * pElems[pos_desc].elem.blocklen) * basic_type->size;
-            if( local_size >= iSize ) {
+            const opal_datatype_t *basic_type = BASIC_DDT_FROM_ELEM(pElems[pos_desc]);
+            local_size = ((size_t) pElems[pos_desc].elem.count * pElems[pos_desc].elem.blocklen)
+                         * basic_type->size;
+            if (local_size >= iSize) {
                 local_size = iSize / basic_type->size;
-                nbElems += (int32_t)local_size;
+                nbElems += (int32_t) local_size;
                 iSize -= local_size * basic_type->size;
                 return (iSize == 0 ? nbElems : -1);
             }
-            nbElems += ((size_t)pElems[pos_desc].elem.count * pElems[pos_desc].elem.blocklen);
+            nbElems += ((size_t) pElems[pos_desc].elem.count * pElems[pos_desc].elem.blocklen);
             iSize -= local_size;
-            pos_desc++;  /* advance to the next data */
+            pos_desc++; /* advance to the next data */
         }
     }
 }
 
-int32_t opal_datatype_set_element_count( const opal_datatype_t* datatype, size_t count, size_t* length )
+int32_t opal_datatype_set_element_count(const opal_datatype_t *datatype, size_t count,
+                                        size_t *length)
 {
-    dt_stack_t* pStack;   /* pointer to the position on the stack */
+    dt_stack_t *pStack; /* pointer to the position on the stack */
     size_t pos_desc;    /* actual position in the description of the derived datatype */
     int32_t stack_pos = 0;
     size_t local_length = 0;
-    dt_elem_desc_t* pElems;
+    dt_elem_desc_t *pElems;
 
     /**
      * Handle all complete multiple of the datatype.
@@ -98,47 +107,53 @@ int32_t opal_datatype_set_element_count( const opal_datatype_t* datatype, size_t
     pos_desc = count / local_length;
     count = count % local_length;
     *length = datatype->size * pos_desc;
-    if( 0 == count ) {
+    if (0 == count) {
         return 0;
     }
 
-    DUMP( "dt_set_element_count( %p, %d )\n", (void*)datatype, count );
-    pStack = (dt_stack_t*)alloca( sizeof(dt_stack_t) * (datatype->loops + 2) );
-    pStack->count    = 1;
-    pStack->index    = -1;
-    pStack->disp     = 0;
-    pElems           = datatype->desc.desc;
-    pos_desc         = 0;
-
-    while( 1 ) {  /* loop forever the exit condition is on the last OPAL_DATATYPE_END_LOOP */
-        if( OPAL_DATATYPE_END_LOOP == pElems[pos_desc].elem.common.type ) { /* end of the current loop */
-            if( --(pStack->count) == 0 ) { /* end of loop */
-                stack_pos--; pStack--;
-                if( stack_pos == -1 ) return 0;
-                pos_desc++;  /* advance to the next element after the end loop */
+    DUMP("dt_set_element_count( %p, %d )\n", (void *) datatype, count);
+    pStack = (dt_stack_t *) alloca(sizeof(dt_stack_t) * (datatype->loops + 2));
+    pStack->count = 1;
+    pStack->index = -1;
+    pStack->disp = 0;
+    pElems = datatype->desc.desc;
+    pos_desc = 0;
+
+    while (1) { /* loop forever the exit condition is on the last OPAL_DATATYPE_END_LOOP */
+        if (OPAL_DATATYPE_END_LOOP
+            == pElems[pos_desc].elem.common.type) { /* end of the current loop */
+            if (--(pStack->count) == 0) {           /* end of loop */
+                stack_pos--;
+                pStack--;
+                if (stack_pos == -1) {
+                    return 0;
+                }
+                pos_desc++; /* advance to the next element after the end loop */
             } else {
-                pos_desc = pStack->index + 1;  /* go back to the begining of the loop */
+                pos_desc = pStack->index + 1; /* go back to the beginning of the loop */
             }
             continue;
         }
-        if( OPAL_DATATYPE_LOOP == pElems[pos_desc].elem.common.type ) {
+        if (OPAL_DATATYPE_LOOP == pElems[pos_desc].elem.common.type) {
             do {
-                PUSH_STACK( pStack, stack_pos, pos_desc, OPAL_DATATYPE_LOOP, pElems[pos_desc].loop.loops, 0 );
+                PUSH_STACK(pStack, stack_pos, pos_desc, OPAL_DATATYPE_LOOP,
+                           pElems[pos_desc].loop.loops, 0);
                 pos_desc++;
-            } while( OPAL_DATATYPE_LOOP == pElems[pos_desc].elem.common.type ); /* let's start another loop */
-            DDT_DUMP_STACK( pStack, stack_pos, pElems, "advance loops" );
+            } while (OPAL_DATATYPE_LOOP
+                     == pElems[pos_desc].elem.common.type); /* let's start another loop */
+            DDT_DUMP_STACK(pStack, stack_pos, pElems, "advance loops");
         }
-        while( pElems[pos_desc].elem.common.flags & OPAL_DATATYPE_FLAG_DATA ) {
+        while (pElems[pos_desc].elem.common.flags & OPAL_DATATYPE_FLAG_DATA) {
             /* now here we have a basic datatype */
-            const opal_datatype_t* basic_type = BASIC_DDT_FROM_ELEM(pElems[pos_desc]);
-            local_length = ((size_t)pElems[pos_desc].elem.count * pElems[pos_desc].elem.blocklen);
-            if( local_length >= count ) {
+            const opal_datatype_t *basic_type = BASIC_DDT_FROM_ELEM(pElems[pos_desc]);
+            local_length = ((size_t) pElems[pos_desc].elem.count * pElems[pos_desc].elem.blocklen);
+            if (local_length >= count) {
                 *length += count * basic_type->size;
                 return 0;
             }
             *length += local_length * basic_type->size;
             count -= local_length;
-            pos_desc++;  /* advance to the next data */
+            pos_desc++; /* advance to the next data */
         }
     }
 }
@@ -150,51 +165,91 @@ int32_t opal_datatype_set_element_count( const opal_datatype_t* datatype, size_t
  * when we use get_element_count). Thus, we will pay the cost once per
  * datatype, but we will only update this array if/when needed.
  */
-int opal_datatype_compute_ptypes( opal_datatype_t* datatype )
+int opal_datatype_compute_ptypes(opal_datatype_t *datatype)
 {
-    dt_stack_t* pStack;   /* pointer to the position on the stack */
-    uint32_t pos_desc;    /* actual position in the description of the derived datatype */
-    ssize_t nbElems = 0, stack_pos = 0;
-    dt_elem_desc_t* pElems;
-
-    if( NULL != datatype->ptypes ) return 0;
-    datatype->ptypes = (size_t*)calloc(OPAL_DATATYPE_MAX_SUPPORTED, sizeof(size_t));
-
-    DUMP( "opal_datatype_compute_ptypes( %p )\n", (void*)datatype );
-    pStack = (dt_stack_t*)alloca( sizeof(dt_stack_t) * (datatype->loops + 2) );
-    pStack->count    = 1;
-    pStack->index    = -1;
-    pStack->disp     = 0;
-    pElems           = datatype->desc.desc;
-    pos_desc         = 0;
-
-    while( 1 ) {  /* loop forever the exit condition is on the last OPAL_DATATYPE_END_LOOP */
-        if( OPAL_DATATYPE_END_LOOP == pElems[pos_desc].elem.common.type ) { /* end of the current loop */
-            if( --(pStack->count) == 0 ) { /* end of loop */
-                stack_pos--; pStack--;
-                if( stack_pos == -1 ) return 0;  /* completed */
-                pos_desc++;  /* advance to the next element after the end loop */
+    dt_stack_t *pStack; /* pointer to the position on the stack */
+    uint32_t pos_desc;  /* actual position in the description of the derived datatype */
+    ssize_t stack_pos = 0;
+#if defined(VERBOSE)
+    ssize_t nbElems = 0;
+#endif
+    dt_elem_desc_t *pElems;
+
+    if (NULL != datatype->ptypes) {
+        return 0;
+    }
+    datatype->ptypes = (size_t *) calloc(OPAL_DATATYPE_MAX_SUPPORTED, sizeof(size_t));
+
+    DUMP("opal_datatype_compute_ptypes( %p )\n", (void *) datatype);
+    pStack = (dt_stack_t *) alloca(sizeof(dt_stack_t) * (datatype->loops + 2));
+    pStack->count = 1;
+    pStack->index = -1;
+    pStack->disp = 0;
+    pElems = datatype->desc.desc;
+    pos_desc = 0;
+
+    while (1) { /* loop forever the exit condition is on the last OPAL_DATATYPE_END_LOOP */
+        if (OPAL_DATATYPE_END_LOOP
+            == pElems[pos_desc].elem.common.type) { /* end of the current loop */
+            if (--(pStack->count) == 0) {           /* end of loop */
+                stack_pos--;
+                pStack--;
+                if (stack_pos == -1) {
+                    return 0; /* completed */
+                }
+                pos_desc++; /* advance to the next element after the end loop */
             } else {
-                pos_desc = pStack->index + 1;  /* go back to the begining of the loop */
+                pos_desc = pStack->index + 1; /* go back to the beginning of the loop */
             }
             continue;
         }
-        if( OPAL_DATATYPE_LOOP == pElems[pos_desc].elem.common.type ) {
+        if (OPAL_DATATYPE_LOOP == pElems[pos_desc].elem.common.type) {
             do {
-                PUSH_STACK( pStack, stack_pos, pos_desc, OPAL_DATATYPE_LOOP, pElems[pos_desc].loop.loops, 0 );
+                PUSH_STACK(pStack, stack_pos, pos_desc, OPAL_DATATYPE_LOOP,
+                           pElems[pos_desc].loop.loops, 0);
                 pos_desc++;
-            } while( OPAL_DATATYPE_LOOP == pElems[pos_desc].elem.common.type ); /* let's start another loop */
-            DDT_DUMP_STACK( pStack, stack_pos, pElems, "advance loops" );
+            } while (OPAL_DATATYPE_LOOP
+                     == pElems[pos_desc].elem.common.type); /* let's start another loop */
+            DDT_DUMP_STACK(pStack, stack_pos, pElems, "advance loops");
         }
-        while( pElems[pos_desc].elem.common.flags & OPAL_DATATYPE_FLAG_DATA ) {
+        while (pElems[pos_desc].elem.common.flags & OPAL_DATATYPE_FLAG_DATA) {
             /* now here we have a basic datatype */
-            datatype->ptypes[pElems[pos_desc].elem.common.type] += (size_t)pElems[pos_desc].elem.count * pElems[pos_desc].elem.blocklen;
-            nbElems += (size_t)pElems[pos_desc].elem.count * pElems[pos_desc].elem.blocklen;
+            datatype->ptypes[pElems[pos_desc].elem.common.type] += (size_t) pElems[pos_desc]
+                                                                       .elem.count
+                                                                   * pElems[pos_desc].elem.blocklen;
+#if defined(VERBOSE)
+            nbElems += (size_t) pElems[pos_desc].elem.count * pElems[pos_desc].elem.blocklen;
 
-            DUMP( "  compute_ptypes-add: type %d count %"PRIsize_t" (total type %u total %lld)\n",
-                  pElems[pos_desc].elem.common.type, datatype->ptypes[pElems[pos_desc].elem.common.type],
-                  pElems[pos_desc].elem.count, nbElems );
-            pos_desc++;  /* advance to the next data */
+            DUMP("  compute_ptypes-add: type %d count %" PRIsize_t " (total type %u total %lld)\n",
+                 pElems[pos_desc].elem.common.type,
+                 datatype->ptypes[pElems[pos_desc].elem.common.type], pElems[pos_desc].elem.count,
+                 nbElems);
+#endif
+            pos_desc++; /* advance to the next data */
         }
     }
 }
+
+size_t opal_datatype_compute_remote_size(const opal_datatype_t *pData, const size_t *sizes)
+{
+    uint32_t typeMask = pData->bdt_used;
+    size_t length = 0;
+
+    if (opal_datatype_is_predefined(pData)) {
+        return sizes[pData->desc.desc->elem.common.type];
+    }
+
+    if (OPAL_UNLIKELY(NULL == pData->ptypes)) {
+        /* Allocate and fill the array of types used in the datatype description */
+        opal_datatype_compute_ptypes((opal_datatype_t *) pData);
+    }
+
+    for (int i = OPAL_DATATYPE_FIRST_TYPE; typeMask && (i < OPAL_DATATYPE_MAX_PREDEFINED); i++) {
+        if (typeMask & ((uint32_t) 1 << i)) {
+            length += (pData->ptypes[i] * sizes[i]);
+            typeMask ^= ((uint32_t) 1 << i);
+        }
+    }
+    return length;
+}
+
diff --git a/opal/datatype/opal_datatype_internal.h b/opal/datatype/opal_datatype_internal.h
index b14acdf6168..47f6e0107cf 100644
--- a/opal/datatype/opal_datatype_internal.h
+++ b/opal/datatype/opal_datatype_internal.h
@@ -34,57 +34,21 @@
 #include <string.h>
 
 #if defined(VERBOSE)
-#include "opal/util/output.h"
+#    include "opal/util/output.h"
 
 extern int opal_datatype_dfd;
 
-#  define DDT_DUMP_STACK( PSTACK, STACK_POS, PDESC, NAME ) \
-     opal_datatype_dump_stack( (PSTACK), (STACK_POS), (PDESC), (NAME) )
-#  if defined(ACCEPT_C99)
-#    define DUMP( ARGS... )          opal_output(opal_datatype_dfd, __VA_ARGS__)
-#  else
-#    if defined(__GNUC__) && !defined(__STDC__)
-#      define DUMP(ARGS...)          opal_output( opal_datatype_dfd, ARGS)
-#  else
-static inline void DUMP( char* fmt, ... )
-{
-   va_list list;
+#    define DDT_DUMP_STACK(PSTACK, STACK_POS, PDESC, NAME) \
+        opal_datatype_dump_stack((PSTACK), (STACK_POS), (PDESC), (NAME))
+
+#    define DUMP(...) opal_output(opal_datatype_dfd, __VA_ARGS__)
 
-   va_start( list, fmt );
-   opal_output_vverbose( 0, opal_datatype_dfd, fmt, list );
-   va_end( list );
-}
-#    endif  /* __GNUC__ && !__STDC__ */
-#  endif  /* ACCEPT_C99 */
 #else
-#  define DDT_DUMP_STACK( PSTACK, STACK_POS, PDESC, NAME )
-#  if defined(ACCEPT_C99)
-#    define DUMP(ARGS...)
-#  else
-#    if defined(__GNUC__) && !defined(__STDC__)
-#      define DUMP(ARGS...)
-#    else
-       /* If we do not compile with PGI, mark the parameter as unused */
-#      if !defined(__PGI)
-#        define __opal_attribute_unused_tmp__  __opal_attribute_unused__
-#      else
-#        define __opal_attribute_unused_tmp__
-#      endif
-static inline void DUMP( char* fmt __opal_attribute_unused_tmp__, ... )
-{
-#if defined(__PGI)
-           /* Some compilers complain if we have "..." arguments and no
-              corresponding va_start() */
-           va_list arglist;
-           va_start(arglist, fmt);
-           va_end(arglist);
-#endif
-}
-#         undef __opal_attribute_unused_tmp__
-#    endif  /* __GNUC__ && !__STDC__ */
-#  endif  /* ACCEPT_C99 */
-#endif  /* VERBOSE */
 
+#    define DDT_DUMP_STACK(PSTACK, STACK_POS, PDESC, NAME)
+#    define DUMP(...)
+
+#endif /* VERBOSE */
 
 /*
  * There 3 types of predefined data types.
@@ -97,56 +61,60 @@ static inline void DUMP( char* fmt __opal_attribute_unused_tmp__, ... )
  *
  * At the OPAL-level we do not care from which language the datatype came from
  * (C, C++ or FORTRAN), we only focus on their internal representation in
- * the host memory.
+ * the host memory. There is one notable exception, the long predefined type
+ * which need to be handled at the lowest level due to it's variable size but
+ * fixed XDR representation.
  *
  * NOTE: This predefined datatype order should be matched by any upper-level
  * users of the OPAL datatype.
  */
-#define OPAL_DATATYPE_LOOP           0
-#define OPAL_DATATYPE_END_LOOP       1
-#define OPAL_DATATYPE_LB             2
-#define OPAL_DATATYPE_UB             3
-#define OPAL_DATATYPE_FIRST_TYPE     4 /* Number of first real type */
-#define OPAL_DATATYPE_INT1           4
-#define OPAL_DATATYPE_INT2           5
-#define OPAL_DATATYPE_INT4           6
-#define OPAL_DATATYPE_INT8           7
-#define OPAL_DATATYPE_INT16          8
-#define OPAL_DATATYPE_UINT1          9
-#define OPAL_DATATYPE_UINT2          10
-#define OPAL_DATATYPE_UINT4          11
-#define OPAL_DATATYPE_UINT8          12
-#define OPAL_DATATYPE_UINT16         13
-#define OPAL_DATATYPE_FLOAT2         14
-#define OPAL_DATATYPE_FLOAT4         15
-#define OPAL_DATATYPE_FLOAT8         16
-#define OPAL_DATATYPE_FLOAT12        17
-#define OPAL_DATATYPE_FLOAT16        18
+#define OPAL_DATATYPE_LOOP                0
+#define OPAL_DATATYPE_END_LOOP            1
+#define OPAL_DATATYPE_LB                  2
+#define OPAL_DATATYPE_UB                  3
+#define OPAL_DATATYPE_FIRST_TYPE          4 /* Number of first real type */
+#define OPAL_DATATYPE_INT1                4
+#define OPAL_DATATYPE_INT2                5
+#define OPAL_DATATYPE_INT4                6
+#define OPAL_DATATYPE_INT8                7
+#define OPAL_DATATYPE_INT16               8
+#define OPAL_DATATYPE_UINT1               9
+#define OPAL_DATATYPE_UINT2               10
+#define OPAL_DATATYPE_UINT4               11
+#define OPAL_DATATYPE_UINT8               12
+#define OPAL_DATATYPE_UINT16              13
+#define OPAL_DATATYPE_FLOAT2              14
+#define OPAL_DATATYPE_FLOAT4              15
+#define OPAL_DATATYPE_FLOAT8              16
+#define OPAL_DATATYPE_FLOAT12             17
+#define OPAL_DATATYPE_FLOAT16             18
 #define OPAL_DATATYPE_SHORT_FLOAT_COMPLEX 19
-#define OPAL_DATATYPE_FLOAT_COMPLEX  20
-#define OPAL_DATATYPE_DOUBLE_COMPLEX 21
+#define OPAL_DATATYPE_FLOAT_COMPLEX       20
+#define OPAL_DATATYPE_DOUBLE_COMPLEX      21
 #define OPAL_DATATYPE_LONG_DOUBLE_COMPLEX 22
-#define OPAL_DATATYPE_BOOL           23
-#define OPAL_DATATYPE_WCHAR          24
-#define OPAL_DATATYPE_UNAVAILABLE    25
+#define OPAL_DATATYPE_BOOL                23
+#define OPAL_DATATYPE_WCHAR               24
+#define OPAL_DATATYPE_LONG                25
+#define OPAL_DATATYPE_UNSIGNED_LONG       26
+#define OPAL_DATATYPE_UNAVAILABLE         27
 
 #ifndef OPAL_DATATYPE_MAX_PREDEFINED
-#define OPAL_DATATYPE_MAX_PREDEFINED (OPAL_DATATYPE_UNAVAILABLE+1)
+#    define OPAL_DATATYPE_MAX_PREDEFINED (OPAL_DATATYPE_UNAVAILABLE + 1)
 #elif OPAL_DATATYPE_MAX_PREDEFINED <= OPAL_DATATYPE_UNAVAILABLE
 /*
  * If the number of basic datatype should change update
  * OPAL_DATATYPE_MAX_PREDEFINED in opal_datatype.h
  */
-#error OPAL_DATATYPE_MAX_PREDEFINED should be updated to the next value after the OPAL_DATATYPE_UNAVAILABLE define
+#    error OPAL_DATATYPE_MAX_PREDEFINED should be updated to the next value after the OPAL_DATATYPE_UNAVAILABLE define
 #endif
 
-#define DT_INCREASE_STACK     8
+#define DT_INCREASE_STACK 8
 
 BEGIN_C_DECLS
 
 struct ddt_elem_id_description {
-    uint16_t   flags;  /**< flags for the record */
-    uint16_t   type;   /**< the basic data type id */
+    uint16_t flags; /**< flags for the record */
+    uint16_t type;  /**< the basic data type id */
 };
 typedef struct ddt_elem_id_description ddt_elem_id_description;
 
@@ -156,16 +124,16 @@ typedef struct ddt_elem_id_description ddt_elem_id_description;
  * and then an extent for all the extra count.
  */
 struct ddt_elem_desc {
-    ddt_elem_id_description common;           /**< basic data description and flags */
-    uint32_t                count;            /**< number of blocks */
-    size_t                  blocklen;         /**< number of elements on each block */
-    ptrdiff_t               extent;           /**< extent of each block (in bytes) */
-    ptrdiff_t               disp;             /**< displacement of the first block */
+    ddt_elem_id_description common; /**< basic data description and flags */
+    uint32_t count;                 /**< number of blocks */
+    size_t blocklen;                /**< number of elements on each block */
+    ptrdiff_t extent;               /**< extent of each block (in bytes) */
+    ptrdiff_t disp;                 /**< displacement of the first block */
 };
 typedef struct ddt_elem_desc ddt_elem_desc_t;
 
 /**
- * The loop description, with it's two markers: one for the begining and one for
+ * The loop description, with it's two markers: one for the beginning and one for
  * the end. The initial marker contains the number of repetitions, the number of
  * elements in the loop, and the extent of each loop. The end marker contains in
  * addition to the number of elements (so that we can easily pair together the
@@ -173,73 +141,73 @@ typedef struct ddt_elem_desc ddt_elem_desc_t;
  * the first element.
  */
 struct ddt_loop_desc {
-    ddt_elem_id_description common;           /**< basic data description and flags */
-    uint32_t                items;            /**< number of items in the loop */
-    uint32_t                loops;            /**< number of elements */
-    size_t                  unused;           /**< not used right now */
-    ptrdiff_t               extent;           /**< extent of the whole loop */
+    ddt_elem_id_description common; /**< basic data description and flags */
+    uint32_t items;                 /**< number of items in the loop */
+    uint32_t loops;                 /**< number of elements */
+    size_t unused;                  /**< not used right now */
+    ptrdiff_t extent;               /**< extent of the whole loop */
 };
 typedef struct ddt_loop_desc ddt_loop_desc_t;
 
 struct ddt_endloop_desc {
-    ddt_elem_id_description common;           /**< basic data description and flags */
-    uint32_t                items;            /**< number of elements */
-    uint32_t                unused;           /**< not used right now */
-    size_t                  size;             /**< real size of the data in the loop */
-    ptrdiff_t               first_elem_disp;  /**< the displacement of the first block in the loop */
+    ddt_elem_id_description common; /**< basic data description and flags */
+    uint32_t items;                 /**< number of elements */
+    uint32_t unused;                /**< not used right now */
+    size_t size;                    /**< real size of the data in the loop */
+    ptrdiff_t first_elem_disp;      /**< the displacement of the first block in the loop */
 };
 typedef struct ddt_endloop_desc ddt_endloop_desc_t;
 
 union dt_elem_desc {
-    ddt_elem_desc_t    elem;
-    ddt_loop_desc_t    loop;
+    ddt_elem_desc_t elem;
+    ddt_loop_desc_t loop;
     ddt_endloop_desc_t end_loop;
 };
 
-#define CREATE_LOOP_START( _place, _count, _items, _extent, _flags )           \
-    do {                                                                       \
-        (_place)->loop.common.type   = OPAL_DATATYPE_LOOP;                     \
-        (_place)->loop.common.flags  = (_flags) & ~OPAL_DATATYPE_FLAG_DATA;    \
-        (_place)->loop.loops         = (_count);                               \
-        (_place)->loop.items         = (_items);                               \
-        (_place)->loop.extent        = (_extent);                              \
-        (_place)->loop.unused        = -1;                                     \
-    } while(0)
-
-#define CREATE_LOOP_END( _place, _items, _first_item_disp, _size, _flags )     \
+#define CREATE_LOOP_START(_place, _count, _items, _extent, _flags)         \
+    do {                                                                   \
+        (_place)->loop.common.type = OPAL_DATATYPE_LOOP;                   \
+        (_place)->loop.common.flags = (_flags) & ~OPAL_DATATYPE_FLAG_DATA; \
+        (_place)->loop.loops = (_count);                                   \
+        (_place)->loop.items = (_items);                                   \
+        (_place)->loop.extent = (_extent);                                 \
+        (_place)->loop.unused = -1;                                        \
+    } while (0)
+
+#define CREATE_LOOP_END(_place, _items, _first_item_disp, _size, _flags)       \
     do {                                                                       \
         (_place)->end_loop.common.type = OPAL_DATATYPE_END_LOOP;               \
         (_place)->end_loop.common.flags = (_flags) & ~OPAL_DATATYPE_FLAG_DATA; \
         (_place)->end_loop.items = (_items);                                   \
         (_place)->end_loop.first_elem_disp = (_first_item_disp);               \
-        (_place)->end_loop.size = (_size);  /* the size inside the loop */     \
+        (_place)->end_loop.size = (_size); /* the size inside the loop */      \
         (_place)->end_loop.unused = -1;                                        \
-    } while(0)
-
+    } while (0)
 
 /**
  * Create an element entry in the description. If the element is contiguous
  * collapse everything into the blocklen.
  */
-#define CREATE_ELEM(_place, _type, _flags, _blocklen, _count, _disp, _extent)  \
-    do {                                                                       \
-        (_place)->elem.common.flags = (_flags) | OPAL_DATATYPE_FLAG_DATA;      \
-        (_place)->elem.common.type  = (_type);                                 \
-        (_place)->elem.blocklen     = (_blocklen);                             \
-        (_place)->elem.count        = (_count);                                \
-        (_place)->elem.extent       = (_extent);                               \
-        (_place)->elem.disp         = (_disp);                                 \
-        if( _extent == (ptrdiff_t)(_blocklen * opal_datatype_basicDatatypes[_type]->size) ) { \
-            /* collapse it into a single large blocklen */              \
-            (_place)->elem.blocklen *= _count;                          \
-            (_place)->elem.extent   *= _count;                          \
-            (_place)->elem.count     = 1;                               \
-        }                                                               \
-    } while(0)
+#define CREATE_ELEM(_place, _type, _flags, _blocklen, _count, _disp, _extent)                 \
+    do {                                                                                      \
+        (_place)->elem.common.flags = (_flags) | OPAL_DATATYPE_FLAG_DATA;                     \
+        (_place)->elem.common.type = (_type);                                                 \
+        (_place)->elem.blocklen = (_blocklen);                                                \
+        (_place)->elem.count = (_count);                                                      \
+        (_place)->elem.extent = (_extent);                                                    \
+        (_place)->elem.disp = (_disp);                                                        \
+        if (_extent == (ptrdiff_t) (_blocklen * opal_datatype_basicDatatypes[_type]->size)) { \
+            /* collapse it into a single large blocklen */                                    \
+            (_place)->elem.blocklen *= _count;                                                \
+            (_place)->elem.extent *= _count;                                                  \
+            (_place)->elem.count = 1;                                                         \
+        }                                                                                     \
+    } while (0)
 /*
  * This array holds the descriptions desc.desc[2] of the predefined basic datatypes.
  */
-OPAL_DECLSPEC extern union dt_elem_desc opal_datatype_predefined_elem_desc[2 * OPAL_DATATYPE_MAX_PREDEFINED];
+OPAL_DECLSPEC extern union dt_elem_desc
+    opal_datatype_predefined_elem_desc[2 * OPAL_DATATYPE_MAX_PREDEFINED];
 struct opal_datatype_t;
 
 /* Other fields starting after bdt_used (index of OPAL_DATATYPE_LOOP should be ONE) */
@@ -252,7 +220,11 @@ struct opal_datatype_t;
  */
 
 #define OPAL_DATATYPE_INIT_PTYPES_ARRAY_UNAVAILABLE NULL
-#define OPAL_DATATYPE_INIT_PTYPES_ARRAY(NAME) (size_t[OPAL_DATATYPE_MAX_PREDEFINED]){ [OPAL_DATATYPE_ ## NAME] = 1, [OPAL_DATATYPE_MAX_PREDEFINED-1] = 0 }
+#define OPAL_DATATYPE_INIT_PTYPES_ARRAY(NAME)                              \
+    (size_t[OPAL_DATATYPE_MAX_PREDEFINED])                                 \
+    {                                                                      \
+        [OPAL_DATATYPE_##NAME] = 1, [OPAL_DATATYPE_MAX_PREDEFINED - 1] = 0 \
+    }
 
 #define OPAL_DATATYPE_INIT_NAME(NAME) "OPAL_" #NAME
 
@@ -261,345 +233,361 @@ struct opal_datatype_t;
  * into the array opal_datatype_predefined_type_desc, which is initialized at
  * runtime in opal_datatype_init(). Each basic type has two desc-elements....
  */
-#define OPAL_DATATYPE_INIT_DESC_PREDEFINED(NAME)                                     \
-    {                                                                                \
-        .length = 1, .used = 1,                                                      \
-        .desc = &(opal_datatype_predefined_elem_desc[2 * OPAL_DATATYPE_ ## NAME])    \
+#define OPAL_DATATYPE_INIT_DESC_PREDEFINED(NAME)                                \
+    {                                                                           \
+        .length = 1, .used = 1,                                                 \
+        .desc = &(opal_datatype_predefined_elem_desc[2 * OPAL_DATATYPE_##NAME]) \
+    }
+#define OPAL_DATATYPE_INIT_DESC_NULL         \
+    {                                        \
+        .length = 0, .used = 0, .desc = NULL \
     }
-#define OPAL_DATATYPE_INIT_DESC_NULL  {.length = 0, .used = 0, .desc = NULL}
-
-#define OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED( NAME, FLAGS )                   \
-    {                                                                                \
-        .super = OPAL_OBJ_STATIC_INIT(opal_datatype_t),                              \
-        .flags = OPAL_DATATYPE_FLAG_UNAVAILABLE | OPAL_DATATYPE_FLAG_PREDEFINED | (FLAGS), \
-        .id = OPAL_DATATYPE_ ## NAME,                                                \
-        .bdt_used = 0,                                                               \
-        .size = 0,                                                                   \
-        .true_lb = 0, .true_ub = 0, .lb = 0, .ub = 0,                                \
-        .align = 0,                                                                  \
-        .nbElems = 1,                                                                \
-        .name = OPAL_DATATYPE_INIT_NAME(NAME),                                       \
-        .desc = OPAL_DATATYPE_INIT_DESC_PREDEFINED(UNAVAILABLE),                     \
-        .opt_desc = OPAL_DATATYPE_INIT_DESC_PREDEFINED(UNAVAILABLE),                 \
-        .ptypes = OPAL_DATATYPE_INIT_PTYPES_ARRAY_UNAVAILABLE                        \
+
+#define OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED(NAME, FLAGS)                                   \
+    {                                                                                              \
+        .super = OPAL_OBJ_STATIC_INIT(opal_datatype_t),                                            \
+        .flags = OPAL_DATATYPE_FLAG_UNAVAILABLE | OPAL_DATATYPE_FLAG_PREDEFINED | (FLAGS),         \
+        .id = OPAL_DATATYPE_##NAME, .bdt_used = 0, .size = 0, .true_lb = 0, .true_ub = 0, .lb = 0, \
+        .ub = 0, .align = 0, .nbElems = 1, .name = OPAL_DATATYPE_INIT_NAME(NAME),                  \
+        .desc = OPAL_DATATYPE_INIT_DESC_PREDEFINED(UNAVAILABLE),                                   \
+        .opt_desc = OPAL_DATATYPE_INIT_DESC_PREDEFINED(UNAVAILABLE),                               \
+        .ptypes = OPAL_DATATYPE_INIT_PTYPES_ARRAY_UNAVAILABLE                                      \
     }
 
-#define OPAL_DATATYPE_INITIALIZER_UNAVAILABLE( FLAGS )                               \
-    OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED( UNAVAILABLE, (FLAGS) )
-
-#define OPAL_DATATYPE_INITIALIZER_EMPTY( FLAGS )                        \
-    {                                                                   \
-        .super = OPAL_OBJ_STATIC_INIT(opal_datatype_t),                 \
-        .flags = OPAL_DATATYPE_FLAG_PREDEFINED | (FLAGS),               \
-        .id = 0,                                                        \
-        .bdt_used = 0,                                                  \
-        .size = 0,                                                      \
-        .true_lb = 0, .true_ub = 0, .lb = 0, .ub = 0,                   \
-        .align = 0,                                                     \
-        .nbElems = 1,                                                   \
-        .name = OPAL_DATATYPE_INIT_NAME(EMPTY),                         \
-        .desc = OPAL_DATATYPE_INIT_DESC_NULL,                           \
-        .opt_desc = OPAL_DATATYPE_INIT_DESC_NULL,                       \
-        .ptypes = OPAL_DATATYPE_INIT_PTYPES_ARRAY_UNAVAILABLE           \
+#define OPAL_DATATYPE_INITIALIZER_UNAVAILABLE(FLAGS) \
+    OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED(UNAVAILABLE, (FLAGS))
+
+#define OPAL_DATATYPE_INITIALIZER_EMPTY(FLAGS)                                               \
+    {                                                                                        \
+        .super = OPAL_OBJ_STATIC_INIT(opal_datatype_t),                                      \
+        .flags = OPAL_DATATYPE_FLAG_PREDEFINED | (FLAGS), .id = 0, .bdt_used = 0, .size = 0, \
+        .true_lb = 0, .true_ub = 0, .lb = 0, .ub = 0, .align = 0, .nbElems = 1,              \
+        .name = OPAL_DATATYPE_INIT_NAME(EMPTY), .desc = OPAL_DATATYPE_INIT_DESC_NULL,        \
+        .opt_desc = OPAL_DATATYPE_INIT_DESC_NULL,                                            \
+        .ptypes = OPAL_DATATYPE_INIT_PTYPES_ARRAY_UNAVAILABLE                                \
     }
 
-#define OPAL_DATATYPE_INIT_BASIC_TYPE( TYPE, NAME, FLAGS )              \
-    {                                                                   \
-        .super = OPAL_OBJ_STATIC_INIT(opal_datatype_t),                 \
-        .flags = OPAL_DATATYPE_FLAG_PREDEFINED | (FLAGS),               \
-        .id = TYPE,                                                     \
-        .bdt_used = (((uint32_t)1)<<(TYPE)),                            \
-        .size = 0,                                                      \
-        .true_lb = 0, .true_ub = 0, .lb = 0, .ub = 0,                   \
-        .align = 0,                                                     \
-        .nbElems = 1,                                                   \
-        .name = OPAL_DATATYPE_INIT_NAME(NAME),                          \
-        .desc = OPAL_DATATYPE_INIT_DESC_NULL,                           \
-        .opt_desc = OPAL_DATATYPE_INIT_DESC_NULL,                       \
-        .ptypes = OPAL_DATATYPE_INIT_PTYPES_ARRAY_UNAVAILABLE           \
+#define OPAL_DATATYPE_INIT_BASIC_TYPE(TYPE, NAME, FLAGS)                                        \
+    {                                                                                           \
+        .super = OPAL_OBJ_STATIC_INIT(opal_datatype_t),                                         \
+        .flags = OPAL_DATATYPE_FLAG_PREDEFINED | (FLAGS), .id = TYPE,                           \
+        .bdt_used = (((uint32_t) 1) << (TYPE)), .size = 0, .true_lb = 0, .true_ub = 0, .lb = 0, \
+        .ub = 0, .align = 0, .nbElems = 1, .name = OPAL_DATATYPE_INIT_NAME(NAME),               \
+        .desc = OPAL_DATATYPE_INIT_DESC_NULL, .opt_desc = OPAL_DATATYPE_INIT_DESC_NULL,         \
+        .ptypes = OPAL_DATATYPE_INIT_PTYPES_ARRAY_UNAVAILABLE                                   \
     }
 
-#define OPAL_DATATYPE_INIT_BASIC_DATATYPE( TYPE, ALIGN, NAME, FLAGS )                \
-    {                                                                                \
-        .super = OPAL_OBJ_STATIC_INIT(opal_datatype_t),                              \
-        .flags = OPAL_DATATYPE_FLAG_BASIC | (FLAGS),                                 \
-        .id = OPAL_DATATYPE_ ## NAME,                                                \
-        .bdt_used = (((uint32_t)1)<<(OPAL_DATATYPE_ ## NAME)),                       \
-        .size = sizeof(TYPE),                                                        \
-        .true_lb = 0, .true_ub = sizeof(TYPE), .lb = 0, .ub = sizeof(TYPE),          \
-        .align = (ALIGN),                                                            \
-        .nbElems = 1,                                                                \
-        .name = OPAL_DATATYPE_INIT_NAME(NAME),                                       \
-        .desc = OPAL_DATATYPE_INIT_DESC_PREDEFINED(NAME),                            \
-        .opt_desc = OPAL_DATATYPE_INIT_DESC_PREDEFINED(NAME),                        \
-        .ptypes = OPAL_DATATYPE_INIT_PTYPES_ARRAY_UNAVAILABLE                        \
+#define OPAL_DATATYPE_INIT_BASIC_DATATYPE(TYPE, ALIGN, NAME, FLAGS)                           \
+    {                                                                                         \
+        .super = OPAL_OBJ_STATIC_INIT(opal_datatype_t),                                       \
+        .flags = OPAL_DATATYPE_FLAG_BASIC | (FLAGS), .id = OPAL_DATATYPE_##NAME,              \
+        .bdt_used = (((uint32_t) 1) << (OPAL_DATATYPE_##NAME)), .size = sizeof(TYPE),         \
+        .true_lb = 0, .true_ub = sizeof(TYPE), .lb = 0, .ub = sizeof(TYPE), .align = (ALIGN), \
+        .nbElems = 1, .name = OPAL_DATATYPE_INIT_NAME(NAME),                                  \
+        .desc = OPAL_DATATYPE_INIT_DESC_PREDEFINED(NAME),                                     \
+        .opt_desc = OPAL_DATATYPE_INIT_DESC_PREDEFINED(NAME),                                 \
+        .ptypes = OPAL_DATATYPE_INIT_PTYPES_ARRAY_UNAVAILABLE                                 \
     }
 
-#define OPAL_DATATYPE_INITIALIZER_INT1(FLAGS)                  \
-             OPAL_DATATYPE_HANDLE_INT1(                        \
-             OPAL_DATATYPE_INIT_BASIC_DATATYPE,                \
-             OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
-#define OPAL_DATATYPE_INITIALIZER_INT2(FLAGS)                  \
-             OPAL_DATATYPE_HANDLE_INT2(                        \
-             OPAL_DATATYPE_INIT_BASIC_DATATYPE,                \
-             OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
-#define OPAL_DATATYPE_INITIALIZER_INT4(FLAGS)                  \
-             OPAL_DATATYPE_HANDLE_INT4(                        \
-             OPAL_DATATYPE_INIT_BASIC_DATATYPE,                \
-             OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
-#define OPAL_DATATYPE_INITIALIZER_INT8(FLAGS)                  \
-             OPAL_DATATYPE_HANDLE_INT8(                        \
-             OPAL_DATATYPE_INIT_BASIC_DATATYPE,                \
-             OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
-#define OPAL_DATATYPE_INITIALIZER_INT16(FLAGS)                 \
-             OPAL_DATATYPE_HANDLE_INT16(                       \
-             OPAL_DATATYPE_INIT_BASIC_DATATYPE,                \
-             OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
-#define OPAL_DATATYPE_INITIALIZER_UINT1(FLAGS)                 \
-             OPAL_DATATYPE_HANDLE_UINT1(                       \
-             OPAL_DATATYPE_INIT_BASIC_DATATYPE,                \
-             OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
-#define OPAL_DATATYPE_INITIALIZER_UINT2(FLAGS)                 \
-             OPAL_DATATYPE_HANDLE_UINT2(                       \
-             OPAL_DATATYPE_INIT_BASIC_DATATYPE,                \
-             OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
-#define OPAL_DATATYPE_INITIALIZER_UINT4(FLAGS)                 \
-             OPAL_DATATYPE_HANDLE_UINT4(                       \
-             OPAL_DATATYPE_INIT_BASIC_DATATYPE,                \
-             OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
-#define OPAL_DATATYPE_INITIALIZER_UINT8(FLAGS)                 \
-             OPAL_DATATYPE_HANDLE_UINT8(                       \
-             OPAL_DATATYPE_INIT_BASIC_DATATYPE,                \
-             OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
-#define OPAL_DATATYPE_INITIALIZER_UINT16(FLAGS)                \
-             OPAL_DATATYPE_HANDLE_UINT16(                      \
-             OPAL_DATATYPE_INIT_BASIC_DATATYPE,                \
-             OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
-#define OPAL_DATATYPE_INITIALIZER_FLOAT2(FLAGS)                \
-             OPAL_DATATYPE_HANDLE_FLOAT2(                      \
-             OPAL_DATATYPE_INIT_BASIC_DATATYPE,                \
-             OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
-#define OPAL_DATATYPE_INITIALIZER_FLOAT4(FLAGS)                \
-             OPAL_DATATYPE_HANDLE_FLOAT4(                      \
-             OPAL_DATATYPE_INIT_BASIC_DATATYPE,                \
-             OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
-#define OPAL_DATATYPE_INITIALIZER_FLOAT8(FLAGS)                \
-             OPAL_DATATYPE_HANDLE_FLOAT8(                      \
-             OPAL_DATATYPE_INIT_BASIC_DATATYPE,                \
-             OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
-#define OPAL_DATATYPE_INITIALIZER_FLOAT12(FLAGS)               \
-             OPAL_DATATYPE_HANDLE_FLOAT12(                     \
-             OPAL_DATATYPE_INIT_BASIC_DATATYPE,                \
-             OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
-#define OPAL_DATATYPE_INITIALIZER_FLOAT16(FLAGS)               \
-             OPAL_DATATYPE_HANDLE_FLOAT16(                     \
-             OPAL_DATATYPE_INIT_BASIC_DATATYPE,                \
-             OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
-#define OPAL_DATATYPE_INITIALIZER_SHORT_FLOAT_COMPLEX(FLAGS)   \
-             OPAL_DATATYPE_HANDLE_SHORT_FLOAT_COMPLEX(         \
-             OPAL_DATATYPE_INIT_BASIC_DATATYPE,                \
-             OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
-#define OPAL_DATATYPE_INITIALIZER_FLOAT_COMPLEX(FLAGS)         \
-             OPAL_DATATYPE_HANDLE_FLOAT_COMPLEX(               \
-             OPAL_DATATYPE_INIT_BASIC_DATATYPE,                \
-             OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
-#define OPAL_DATATYPE_INITIALIZER_DOUBLE_COMPLEX(FLAGS)        \
-             OPAL_DATATYPE_HANDLE_DOUBLE_COMPLEX(              \
-             OPAL_DATATYPE_INIT_BASIC_DATATYPE,                \
-             OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
-#define OPAL_DATATYPE_INITIALIZER_LONG_DOUBLE_COMPLEX(FLAGS)   \
-             OPAL_DATATYPE_HANDLE_LONG_DOUBLE_COMPLEX(         \
-             OPAL_DATATYPE_INIT_BASIC_DATATYPE,                \
-             OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
-#define OPAL_DATATYPE_INITIALIZER_BOOL(FLAGS)                  \
-             OPAL_DATATYPE_HANDLE_BOOL(                        \
-             OPAL_DATATYPE_INIT_BASIC_DATATYPE,                \
-             OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
-#define OPAL_DATATYPE_INITIALIZER_WCHAR(FLAGS)                 \
-             OPAL_DATATYPE_HANDLE_WCHAR(                       \
-             OPAL_DATATYPE_INIT_BASIC_DATATYPE,                \
-             OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
-#define OPAL_DATATYPE_INITIALIZER_LOOP(FLAGS)       OPAL_DATATYPE_INIT_BASIC_TYPE( OPAL_DATATYPE_LOOP, LOOP_S, FLAGS )
-#define OPAL_DATATYPE_INITIALIZER_END_LOOP(FLAGS)   OPAL_DATATYPE_INIT_BASIC_TYPE( OPAL_DATATYPE_END_LOOP, LOOP_E, FLAGS )
-#define OPAL_DATATYPE_INITIALIZER_LB(FLAGS)         OPAL_DATATYPE_INIT_BASIC_TYPE( OPAL_DATATYPE_LB, LB, FLAGS )
-#define OPAL_DATATYPE_INITIALIZER_UB(FLAGS)         OPAL_DATATYPE_INIT_BASIC_TYPE( OPAL_DATATYPE_UB, UB, FLAGS )
-
-
-#define OPAL_DATATYPE_HANDLE_INT1(AV, NOTAV, FLAGS)       AV( int8_t, OPAL_ALIGNMENT_INT8, INT1, FLAGS )
-#define OPAL_DATATYPE_HANDLE_INT2(AV, NOTAV, FLAGS)       AV( int16_t, OPAL_ALIGNMENT_INT16, INT2, FLAGS )
-#define OPAL_DATATYPE_HANDLE_INT4(AV, NOTAV, FLAGS)       AV( int32_t, OPAL_ALIGNMENT_INT32, INT4, FLAGS )
-#define OPAL_DATATYPE_HANDLE_INT8(AV, NOTAV, FLAGS)       AV( int64_t, OPAL_ALIGNMENT_INT64, INT8, FLAGS )
+#define OPAL_DATATYPE_INITIALIZER_INT1(FLAGS)                    \
+    OPAL_DATATYPE_HANDLE_INT1(OPAL_DATATYPE_INIT_BASIC_DATATYPE, \
+                              OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
+#define OPAL_DATATYPE_INITIALIZER_INT2(FLAGS)                    \
+    OPAL_DATATYPE_HANDLE_INT2(OPAL_DATATYPE_INIT_BASIC_DATATYPE, \
+                              OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
+#define OPAL_DATATYPE_INITIALIZER_INT4(FLAGS)                    \
+    OPAL_DATATYPE_HANDLE_INT4(OPAL_DATATYPE_INIT_BASIC_DATATYPE, \
+                              OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
+#define OPAL_DATATYPE_INITIALIZER_INT8(FLAGS)                    \
+    OPAL_DATATYPE_HANDLE_INT8(OPAL_DATATYPE_INIT_BASIC_DATATYPE, \
+                              OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
+#define OPAL_DATATYPE_INITIALIZER_INT16(FLAGS)                    \
+    OPAL_DATATYPE_HANDLE_INT16(OPAL_DATATYPE_INIT_BASIC_DATATYPE, \
+                               OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
+#define OPAL_DATATYPE_INITIALIZER_UINT1(FLAGS)                    \
+    OPAL_DATATYPE_HANDLE_UINT1(OPAL_DATATYPE_INIT_BASIC_DATATYPE, \
+                               OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
+#define OPAL_DATATYPE_INITIALIZER_UINT2(FLAGS)                    \
+    OPAL_DATATYPE_HANDLE_UINT2(OPAL_DATATYPE_INIT_BASIC_DATATYPE, \
+                               OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
+#define OPAL_DATATYPE_INITIALIZER_UINT4(FLAGS)                    \
+    OPAL_DATATYPE_HANDLE_UINT4(OPAL_DATATYPE_INIT_BASIC_DATATYPE, \
+                               OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
+#define OPAL_DATATYPE_INITIALIZER_UINT8(FLAGS)                    \
+    OPAL_DATATYPE_HANDLE_UINT8(OPAL_DATATYPE_INIT_BASIC_DATATYPE, \
+                               OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
+#define OPAL_DATATYPE_INITIALIZER_UINT16(FLAGS)                    \
+    OPAL_DATATYPE_HANDLE_UINT16(OPAL_DATATYPE_INIT_BASIC_DATATYPE, \
+                                OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
+#define OPAL_DATATYPE_INITIALIZER_FLOAT2(FLAGS)                    \
+    OPAL_DATATYPE_HANDLE_FLOAT2(OPAL_DATATYPE_INIT_BASIC_DATATYPE, \
+                                OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
+#define OPAL_DATATYPE_INITIALIZER_FLOAT4(FLAGS)                    \
+    OPAL_DATATYPE_HANDLE_FLOAT4(OPAL_DATATYPE_INIT_BASIC_DATATYPE, \
+                                OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
+#define OPAL_DATATYPE_INITIALIZER_FLOAT8(FLAGS)                    \
+    OPAL_DATATYPE_HANDLE_FLOAT8(OPAL_DATATYPE_INIT_BASIC_DATATYPE, \
+                                OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
+#define OPAL_DATATYPE_INITIALIZER_FLOAT12(FLAGS)                    \
+    OPAL_DATATYPE_HANDLE_FLOAT12(OPAL_DATATYPE_INIT_BASIC_DATATYPE, \
+                                 OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
+#define OPAL_DATATYPE_INITIALIZER_FLOAT16(FLAGS)                    \
+    OPAL_DATATYPE_HANDLE_FLOAT16(OPAL_DATATYPE_INIT_BASIC_DATATYPE, \
+                                 OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
+#define OPAL_DATATYPE_INITIALIZER_SHORT_FLOAT_COMPLEX(FLAGS)                    \
+    OPAL_DATATYPE_HANDLE_SHORT_FLOAT_COMPLEX(OPAL_DATATYPE_INIT_BASIC_DATATYPE, \
+                                             OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
+#define OPAL_DATATYPE_INITIALIZER_FLOAT_COMPLEX(FLAGS)                    \
+    OPAL_DATATYPE_HANDLE_FLOAT_COMPLEX(OPAL_DATATYPE_INIT_BASIC_DATATYPE, \
+                                       OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
+#define OPAL_DATATYPE_INITIALIZER_DOUBLE_COMPLEX(FLAGS)                    \
+    OPAL_DATATYPE_HANDLE_DOUBLE_COMPLEX(OPAL_DATATYPE_INIT_BASIC_DATATYPE, \
+                                        OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
+#define OPAL_DATATYPE_INITIALIZER_LONG_DOUBLE_COMPLEX(FLAGS)                    \
+    OPAL_DATATYPE_HANDLE_LONG_DOUBLE_COMPLEX(OPAL_DATATYPE_INIT_BASIC_DATATYPE, \
+                                             OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
+#define OPAL_DATATYPE_INITIALIZER_BOOL(FLAGS)                    \
+    OPAL_DATATYPE_HANDLE_BOOL(OPAL_DATATYPE_INIT_BASIC_DATATYPE, \
+                              OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
+#define OPAL_DATATYPE_INITIALIZER_WCHAR(FLAGS)                    \
+    OPAL_DATATYPE_HANDLE_WCHAR(OPAL_DATATYPE_INIT_BASIC_DATATYPE, \
+                               OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED, FLAGS)
+#define OPAL_DATATYPE_INITIALIZER_LOOP(FLAGS) \
+    OPAL_DATATYPE_INIT_BASIC_TYPE(OPAL_DATATYPE_LOOP, LOOP_S, FLAGS)
+#define OPAL_DATATYPE_INITIALIZER_END_LOOP(FLAGS) \
+    OPAL_DATATYPE_INIT_BASIC_TYPE(OPAL_DATATYPE_END_LOOP, LOOP_E, FLAGS)
+#define OPAL_DATATYPE_INITIALIZER_LB(FLAGS) \
+    OPAL_DATATYPE_INIT_BASIC_TYPE(OPAL_DATATYPE_LB, LB, FLAGS)
+#define OPAL_DATATYPE_INITIALIZER_UB(FLAGS) \
+    OPAL_DATATYPE_INIT_BASIC_TYPE(OPAL_DATATYPE_UB, UB, FLAGS)
+
+#define OPAL_DATATYPE_HANDLE_INT1(AV, NOTAV, FLAGS) AV(int8_t, OPAL_ALIGNMENT_INT8, INT1, FLAGS)
+#define OPAL_DATATYPE_HANDLE_INT2(AV, NOTAV, FLAGS) AV(int16_t, OPAL_ALIGNMENT_INT16, INT2, FLAGS)
+#define OPAL_DATATYPE_HANDLE_INT4(AV, NOTAV, FLAGS) AV(int32_t, OPAL_ALIGNMENT_INT32, INT4, FLAGS)
+#define OPAL_DATATYPE_HANDLE_INT8(AV, NOTAV, FLAGS) AV(int64_t, OPAL_ALIGNMENT_INT64, INT8, FLAGS)
 #ifdef HAVE_INT128_T
-#define OPAL_DATATYPE_HANDLE_INT16(AV, NOTAV, FLAGS)      AV( int128_t, OPAL_ALIGNMENT_INT128, INT16, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_INT16(AV, NOTAV, FLAGS) \
+        AV(int128_t, OPAL_ALIGNMENT_INT128, INT16, FLAGS)
 #else
-#define OPAL_DATATYPE_HANDLE_INT16(AV, NOTAV, FLAGS)      NOTAV( INT16, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_INT16(AV, NOTAV, FLAGS) NOTAV(INT16, FLAGS)
 #endif
-#define OPAL_DATATYPE_HANDLE_UINT1(AV, NOTAV, FLAGS)      AV( uint8_t, OPAL_ALIGNMENT_INT8, UINT1, FLAGS )
-#define OPAL_DATATYPE_HANDLE_UINT2(AV, NOTAV, FLAGS)      AV( uint16_t, OPAL_ALIGNMENT_INT16, UINT2, FLAGS )
-#define OPAL_DATATYPE_HANDLE_UINT4(AV, NOTAV, FLAGS)      AV( uint32_t, OPAL_ALIGNMENT_INT32, UINT4, FLAGS )
-#define OPAL_DATATYPE_HANDLE_UINT8(AV, NOTAV, FLAGS)      AV( uint64_t, OPAL_ALIGNMENT_INT64, UINT8, FLAGS )
+#define OPAL_DATATYPE_HANDLE_UINT1(AV, NOTAV, FLAGS) AV(uint8_t, OPAL_ALIGNMENT_INT8, UINT1, FLAGS)
+#define OPAL_DATATYPE_HANDLE_UINT2(AV, NOTAV, FLAGS) \
+    AV(uint16_t, OPAL_ALIGNMENT_INT16, UINT2, FLAGS)
+#define OPAL_DATATYPE_HANDLE_UINT4(AV, NOTAV, FLAGS) \
+    AV(uint32_t, OPAL_ALIGNMENT_INT32, UINT4, FLAGS)
+#define OPAL_DATATYPE_HANDLE_UINT8(AV, NOTAV, FLAGS) \
+    AV(uint64_t, OPAL_ALIGNMENT_INT64, UINT8, FLAGS)
 #ifdef HAVE_UINT128_T
-#define OPAL_DATATYPE_HANDLE_UINT16(AV, NOTAV, FLAGS)     AV( uint128_t, OPAL_ALIGNMENT_INT128, UINT16, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_UINT16(AV, NOTAV, FLAGS) \
+        AV(uint128_t, OPAL_ALIGNMENT_INT128, UINT16, FLAGS)
 #else
-#define OPAL_DATATYPE_HANDLE_UINT16(AV, NOTAV, FLAGS)     NOTAV( INT16, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_UINT16(AV, NOTAV, FLAGS) NOTAV(INT16, FLAGS)
 #endif
 
+
+#define OPAL_DATATYPE_INITIALIZER_LONG(FLAGS)  \
+     OPAL_DATATYPE_INIT_BASIC_DATATYPE(long, OPAL_ALIGNMENT_LONG, LONG, FLAGS)
+#define OPAL_DATATYPE_INITIALIZER_UNSIGNED_LONG(FLAGS)  \
+     OPAL_DATATYPE_INIT_BASIC_DATATYPE(unsigned long, OPAL_ALIGNMENT_LONG, UNSIGNED_LONG, FLAGS)
+
 #if defined(HAVE_SHORT_FLOAT) && SIZEOF_SHORT_FLOAT == 2
-#define OPAL_DATATYPE_HANDLE_FLOAT2(AV, NOTAV, FLAGS)     AV( short float, OPAL_ALIGNMENT_SHORT_FLOAT, FLOAT2, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_FLOAT2(AV, NOTAV, FLAGS) \
+        AV(short float, OPAL_ALIGNMENT_SHORT_FLOAT, FLOAT2, FLAGS)
 #elif SIZEOF_FLOAT == 2
-#define OPAL_DATATYPE_HANDLE_FLOAT2(AV, NOTAV, FLAGS)     AV( float, OPAL_ALIGNMENT_FLOAT, FLOAT2, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_FLOAT2(AV, NOTAV, FLAGS) \
+        AV(float, OPAL_ALIGNMENT_FLOAT, FLOAT2, FLAGS)
 #elif SIZEOF_DOUBLE == 2
-#define OPAL_DATATYPE_HANDLE_FLOAT2(AV, NOTAV, FLAGS)     AV( double, OPAL_ALIGNMENT_DOUBLE, FLOAT2, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_FLOAT2(AV, NOTAV, FLAGS) \
+        AV(double, OPAL_ALIGNMENT_DOUBLE, FLOAT2, FLAGS)
 #elif SIZEOF_LONG_DOUBLE == 2
-#define OPAL_DATATYPE_HANDLE_FLOAT2(AV, NOTAV, FLAGS)     AV( long double, OPAL_ALIGNMENT_LONG_DOUBLE, FLOAT2, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_FLOAT2(AV, NOTAV, FLAGS) \
+        AV(long double, OPAL_ALIGNMENT_LONG_DOUBLE, FLOAT2, FLAGS)
 #elif defined(HAVE_OPAL_SHORT_FLOAT_T) && SIZEOF_OPAL_SHORT_FLOAT_T == 2
-#define OPAL_DATATYPE_HANDLE_FLOAT2(AV, NOTAV, FLAGS)     AV( opal_short_float_t, OPAL_ALIGNMENT_OPAL_SHORT_FLOAT_T, FLOAT2, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_FLOAT2(AV, NOTAV, FLAGS) \
+        AV(opal_short_float_t, OPAL_ALIGNMENT_OPAL_SHORT_FLOAT_T, FLOAT2, FLAGS)
 #else
-#define OPAL_DATATYPE_HANDLE_FLOAT2(AV, NOTAV, FLAGS)     NOTAV( FLOAT2, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_FLOAT2(AV, NOTAV, FLAGS) NOTAV(FLOAT2, FLAGS)
 #endif
 
 #if defined(HAVE_SHORT_FLOAT) && SIZEOF_SHORT_FLOAT == 4
-#define OPAL_DATATYPE_HANDLE_FLOAT4(AV, NOTAV, FLAGS)     AV( short float, OPAL_ALIGNMENT_SHORT_FLOAT, FLOAT4, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_FLOAT4(AV, NOTAV, FLAGS) \
+        AV(short float, OPAL_ALIGNMENT_SHORT_FLOAT, FLOAT4, FLAGS)
 #elif SIZEOF_FLOAT == 4
-#define OPAL_DATATYPE_HANDLE_FLOAT4(AV, NOTAV, FLAGS)     AV( float, OPAL_ALIGNMENT_FLOAT, FLOAT4, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_FLOAT4(AV, NOTAV, FLAGS) \
+        AV(float, OPAL_ALIGNMENT_FLOAT, FLOAT4, FLAGS)
 #elif SIZEOF_DOUBLE == 4
-#define OPAL_DATATYPE_HANDLE_FLOAT4(AV, NOTAV, FLAGS)     AV( double, OPAL_ALIGNMENT_DOUBLE, FLOAT4, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_FLOAT4(AV, NOTAV, FLAGS) \
+        AV(double, OPAL_ALIGNMENT_DOUBLE, FLOAT4, FLAGS)
 #elif SIZEOF_LONG_DOUBLE == 4
-#define OPAL_DATATYPE_HANDLE_FLOAT4(AV, NOTAV, FLAGS)     AV( long double, OPAL_ALIGNMENT_LONG_DOUBLE, FLOAT4, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_FLOAT4(AV, NOTAV, FLAGS) \
+        AV(long double, OPAL_ALIGNMENT_LONG_DOUBLE, FLOAT4, FLAGS)
 #elif defined(HAVE_OPAL_SHORT_FLOAT_T) && SIZEOF_OPAL_SHORT_FLOAT_T == 4
-#define OPAL_DATATYPE_HANDLE_FLOAT4(AV, NOTAV, FLAGS)     AV( opal_short_float_t, OPAL_ALIGNMENT_OPAL_SHORT_FLOAT_T, FLOAT4, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_FLOAT4(AV, NOTAV, FLAGS) \
+        AV(opal_short_float_t, OPAL_ALIGNMENT_OPAL_SHORT_FLOAT_T, FLOAT4, FLAGS)
 #else
-#define OPAL_DATATYPE_HANDLE_FLOAT4(AV, NOTAV, FLAGS)     NOTAV( FLOAT4, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_FLOAT4(AV, NOTAV, FLAGS) NOTAV(FLOAT4, FLAGS)
 #endif
 
 #if defined(HAVE_SHORT_FLOAT) && SIZEOF_SHORT_FLOAT == 8
-#define OPAL_DATATYPE_HANDLE_FLOAT8(AV, NOTAV, FLAGS)     AV( short float, OPAL_ALIGNMENT_SHORT_FLOAT, FLOAT8, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_FLOAT8(AV, NOTAV, FLAGS) \
+        AV(short float, OPAL_ALIGNMENT_SHORT_FLOAT, FLOAT8, FLAGS)
 #elif SIZEOF_FLOAT == 8
-#define OPAL_DATATYPE_HANDLE_FLOAT8(AV, NOTAV, FLAGS)     AV( float, OPAL_ALIGNMENT_FLOAT, FLOAT8, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_FLOAT8(AV, NOTAV, FLAGS) \
+        AV(float, OPAL_ALIGNMENT_FLOAT, FLOAT8, FLAGS)
 #elif SIZEOF_DOUBLE == 8
-#define OPAL_DATATYPE_HANDLE_FLOAT8(AV, NOTAV, FLAGS)     AV( double, OPAL_ALIGNMENT_DOUBLE, FLOAT8, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_FLOAT8(AV, NOTAV, FLAGS) \
+        AV(double, OPAL_ALIGNMENT_DOUBLE, FLOAT8, FLAGS)
 #elif SIZEOF_LONG_DOUBLE == 8
-#define OPAL_DATATYPE_HANDLE_FLOAT8(AV, NOTAV, FLAGS)     AV( long double, OPAL_ALIGNMENT_LONG_DOUBLE, FLOAT8, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_FLOAT8(AV, NOTAV, FLAGS) \
+        AV(long double, OPAL_ALIGNMENT_LONG_DOUBLE, FLOAT8, FLAGS)
 #elif defined(HAVE_OPAL_SHORT_FLOAT_T) && SIZEOF_OPAL_SHORT_FLOAT_T == 8
-#define OPAL_DATATYPE_HANDLE_FLOAT8(AV, NOTAV, FLAGS)     AV( opal_short_float_t, OPAL_ALIGNMENT_OPAL_SHORT_FLOAT_T, FLOAT8, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_FLOAT8(AV, NOTAV, FLAGS) \
+        AV(opal_short_float_t, OPAL_ALIGNMENT_OPAL_SHORT_FLOAT_T, FLOAT8, FLAGS)
 #else
-#define OPAL_DATATYPE_HANDLE_FLOAT8(AV, NOTAV, FLAGS)     NOTAV( FLOAT8, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_FLOAT8(AV, NOTAV, FLAGS) NOTAV(FLOAT8, FLAGS)
 #endif
 
 #if defined(HAVE_SHORT_FLOAT) && SIZEOF_SHORT_FLOAT == 12
-#define OPAL_DATATYPE_HANDLE_FLOAT12(AV, NOTAV, FLAGS)    AV( short float, OPAL_ALIGNMENT_SHORT_FLOAT, FLOAT12, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_FLOAT12(AV, NOTAV, FLAGS) \
+        AV(short float, OPAL_ALIGNMENT_SHORT_FLOAT, FLOAT12, FLAGS)
 #elif SIZEOF_FLOAT == 12
-#define OPAL_DATATYPE_HANDLE_FLOAT12(AV, NOTAV, FLAGS)    AV( float, OPAL_ALIGNMENT_FLOAT, FLOAT12, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_FLOAT12(AV, NOTAV, FLAGS) \
+        AV(float, OPAL_ALIGNMENT_FLOAT, FLOAT12, FLAGS)
 #elif SIZEOF_DOUBLE == 12
-#define OPAL_DATATYPE_HANDLE_FLOAT12(AV, NOTAV, FLAGS)    AV( double, OPAL_ALIGNMENT_DOUBLE, FLOAT12, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_FLOAT12(AV, NOTAV, FLAGS) \
+        AV(double, OPAL_ALIGNMENT_DOUBLE, FLOAT12, FLAGS)
 #elif SIZEOF_LONG_DOUBLE == 12
-#define OPAL_DATATYPE_HANDLE_FLOAT12(AV, NOTAV, FLAGS)    AV( long double, OPAL_ALIGNMENT_LONG_DOUBLE, FLOAT12, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_FLOAT12(AV, NOTAV, FLAGS) \
+        AV(long double, OPAL_ALIGNMENT_LONG_DOUBLE, FLOAT12, FLAGS)
 #elif defined(HAVE_OPAL_SHORT_FLOAT_T) && SIZEOF_OPAL_SHORT_FLOAT_T == 12
-#define OPAL_DATATYPE_HANDLE_FLOAT12(AV, NOTAV, FLAGS)    AV( opal_short_float_t, OPAL_ALIGNMENT_OPAL_SHORT_FLOAT_T, FLOAT12, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_FLOAT12(AV, NOTAV, FLAGS) \
+        AV(opal_short_float_t, OPAL_ALIGNMENT_OPAL_SHORT_FLOAT_T, FLOAT12, FLAGS)
 #else
-#define OPAL_DATATYPE_HANDLE_FLOAT12(AV, NOTAV, FLAGS)    NOTAV( FLOAT12, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_FLOAT12(AV, NOTAV, FLAGS) NOTAV(FLOAT12, FLAGS)
 #endif
 
 #if defined(HAVE_SHORT_FLOAT) && SIZEOF_SHORT_FLOAT == 16
-#define OPAL_DATATYPE_HANDLE_FLOAT16(AV, NOTAV, FLAGS)    AV( short float, OPAL_ALIGNMENT_SHORT_FLOAT, FLOAT16, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_FLOAT16(AV, NOTAV, FLAGS) \
+        AV(short float, OPAL_ALIGNMENT_SHORT_FLOAT, FLOAT16, FLAGS)
 #elif SIZEOF_FLOAT == 16
-#define OPAL_DATATYPE_HANDLE_FLOAT16(AV, NOTAV, FLAGS)    AV( float, OPAL_ALIGNMENT_FLOAT, FLOAT16, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_FLOAT16(AV, NOTAV, FLAGS) \
+        AV(float, OPAL_ALIGNMENT_FLOAT, FLOAT16, FLAGS)
 #elif SIZEOF_DOUBLE == 16
-#define OPAL_DATATYPE_HANDLE_FLOAT16(AV, NOTAV, FLAGS)    AV( double, OPAL_ALIGNMENT_DOUBLE, FLOAT16, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_FLOAT16(AV, NOTAV, FLAGS) \
+        AV(double, OPAL_ALIGNMENT_DOUBLE, FLOAT16, FLAGS)
 #elif SIZEOF_LONG_DOUBLE == 16
-#define OPAL_DATATYPE_HANDLE_FLOAT16(AV, NOTAV, FLAGS)    AV( long double, OPAL_ALIGNMENT_LONG_DOUBLE, FLOAT16, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_FLOAT16(AV, NOTAV, FLAGS) \
+        AV(long double, OPAL_ALIGNMENT_LONG_DOUBLE, FLOAT16, FLAGS)
 #elif defined(HAVE_OPAL_SHORT_FLOAT_T) && SIZEOF_OPAL_SHORT_FLOAT_T == 16
-#define OPAL_DATATYPE_HANDLE_FLOAT16(AV, NOTAV, FLAGS)    AV( opal_short_float_t, OPAL_ALIGNMENT_OPAL_SHORT_FLOAT_T, FLOAT16, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_FLOAT16(AV, NOTAV, FLAGS) \
+        AV(opal_short_float_t, OPAL_ALIGNMENT_OPAL_SHORT_FLOAT_T, FLOAT16, FLAGS)
 #else
-#define OPAL_DATATYPE_HANDLE_FLOAT16(AV, NOTAV, FLAGS)    NOTAV( FLOAT16, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_FLOAT16(AV, NOTAV, FLAGS) NOTAV(FLOAT16, FLAGS)
 #endif
 
 #if defined(HAVE_SHORT_FLOAT__COMPLEX)
-#define OPAL_DATATYPE_HANDLE_SHORT_FLOAT_COMPLEX(AV, NOTAV, FLAGS) AV( short float _Complex, OPAL_ALIGNMENT_SHORT_FLOAT_COMPLEX, SHORT_FLOAT_COMPLEX, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_SHORT_FLOAT_COMPLEX(AV, NOTAV, FLAGS) \
+        AV(short float _Complex, OPAL_ALIGNMENT_SHORT_FLOAT_COMPLEX, SHORT_FLOAT_COMPLEX, FLAGS)
 #elif defined(HAVE_OPAL_SHORT_FLOAT_COMPLEX_T)
-#define OPAL_DATATYPE_HANDLE_SHORT_FLOAT_COMPLEX(AV, NOTAV, FLAGS) AV( opal_short_float_complex_t, OPAL_ALIGNMENT_OPAL_SHORT_FLOAT_T, SHORT_FLOAT_COMPLEX, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_SHORT_FLOAT_COMPLEX(AV, NOTAV, FLAGS)                         \
+        AV(opal_short_float_complex_t, OPAL_ALIGNMENT_OPAL_SHORT_FLOAT_T, SHORT_FLOAT_COMPLEX, \
+           FLAGS)
 #else
-#define OPAL_DATATYPE_HANDLE_SHORT_FLOAT_COMPLEX(AV, NOTAV, FLAGS) NOTAV( SHORT_FLOAT_COMPLEX, FLAGS)
+#    define OPAL_DATATYPE_HANDLE_SHORT_FLOAT_COMPLEX(AV, NOTAV, FLAGS) \
+        NOTAV(SHORT_FLOAT_COMPLEX, FLAGS)
 #endif
 
-#define OPAL_DATATYPE_HANDLE_FLOAT_COMPLEX(AV, NOTAV, FLAGS) AV( float _Complex, OPAL_ALIGNMENT_FLOAT_COMPLEX, FLOAT_COMPLEX, FLAGS )
+#define OPAL_DATATYPE_HANDLE_FLOAT_COMPLEX(AV, NOTAV, FLAGS) \
+    AV(float _Complex, OPAL_ALIGNMENT_FLOAT_COMPLEX, FLOAT_COMPLEX, FLAGS)
 
-#define OPAL_DATATYPE_HANDLE_DOUBLE_COMPLEX(AV, NOTAV, FLAGS) AV( double _Complex, OPAL_ALIGNMENT_DOUBLE_COMPLEX, DOUBLE_COMPLEX, FLAGS )
+#define OPAL_DATATYPE_HANDLE_DOUBLE_COMPLEX(AV, NOTAV, FLAGS) \
+    AV(double _Complex, OPAL_ALIGNMENT_DOUBLE_COMPLEX, DOUBLE_COMPLEX, FLAGS)
 
-#define OPAL_DATATYPE_HANDLE_LONG_DOUBLE_COMPLEX(AV, NOTAV, FLAGS) AV( long double _Complex, OPAL_ALIGNMENT_LONG_DOUBLE_COMPLEX, LONG_DOUBLE_COMPLEX, FLAGS )
+#define OPAL_DATATYPE_HANDLE_LONG_DOUBLE_COMPLEX(AV, NOTAV, FLAGS) \
+    AV(long double _Complex, OPAL_ALIGNMENT_LONG_DOUBLE_COMPLEX, LONG_DOUBLE_COMPLEX, FLAGS)
 
-#define OPAL_DATATYPE_HANDLE_BOOL(AV, NOTAV, FLAGS)       AV( _Bool, OPAL_ALIGNMENT_BOOL, BOOL, FLAGS )
+#define OPAL_DATATYPE_HANDLE_BOOL(AV, NOTAV, FLAGS) \
+    AV(_Bool, OPAL_ALIGNMENT_BOOL, BOOL, FLAGS)
 
 #if OPAL_ALIGNMENT_WCHAR != 0
-#define OPAL_DATATYPE_HANDLE_WCHAR(AV, NOTAV, FLAGS)      AV( wchar_t, OPAL_ALIGNMENT_WCHAR, WCHAR, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_WCHAR(AV, NOTAV, FLAGS) \
+        AV(wchar_t, OPAL_ALIGNMENT_WCHAR, WCHAR, FLAGS)
 #else
-#define OPAL_DATATYPE_HANDLE_WCHAR(AV, NOTAV, FLAGS)      NOTAV( WCHAR, FLAGS )
+#    define OPAL_DATATYPE_HANDLE_WCHAR(AV, NOTAV, FLAGS) NOTAV(WCHAR, FLAGS)
 #endif
 
-#define BASIC_DDT_FROM_ELEM( ELEM ) (opal_datatype_basicDatatypes[(ELEM).elem.common.type])
+#define BASIC_DDT_FROM_ELEM(ELEM) (opal_datatype_basicDatatypes[(ELEM).elem.common.type])
 
-#define SAVE_STACK( PSTACK, INDEX, TYPE, COUNT, DISP) \
-do { \
-   (PSTACK)->index    = (INDEX); \
-   (PSTACK)->type     = (TYPE); \
-   (PSTACK)->count    = (COUNT); \
-   (PSTACK)->disp     = (DISP); \
-} while(0)
+#define SAVE_STACK(PSTACK, INDEX, TYPE, COUNT, DISP) \
+    do {                                             \
+        (PSTACK)->index = (INDEX);                   \
+        (PSTACK)->type = (TYPE);                     \
+        (PSTACK)->count = (COUNT);                   \
+        (PSTACK)->disp = (DISP);                     \
+    } while (0)
 
-#define PUSH_STACK( PSTACK, STACK_POS, INDEX, TYPE, COUNT, DISP) \
-do { \
-   dt_stack_t* pTempStack = (PSTACK) + 1; \
-   SAVE_STACK( pTempStack, (INDEX), (TYPE), (COUNT), (DISP) );  \
-   (STACK_POS)++; \
-   (PSTACK) = pTempStack; \
-} while(0)
+#define PUSH_STACK(PSTACK, STACK_POS, INDEX, TYPE, COUNT, DISP)   \
+    do {                                                          \
+        dt_stack_t *pTempStack = (PSTACK) + 1;                    \
+        SAVE_STACK(pTempStack, (INDEX), (TYPE), (COUNT), (DISP)); \
+        (STACK_POS)++;                                            \
+        (PSTACK) = pTempStack;                                    \
+    } while (0)
 
 #if OPAL_ENABLE_DEBUG
-#define OPAL_DATATYPE_SAFEGUARD_POINTER( ACTPTR, LENGTH, INITPTR, PDATA, COUNT ) \
-    {                                                                   \
-        unsigned char *__lower_bound = (INITPTR), *__upper_bound;       \
-        assert( ((LENGTH) != 0) && ((COUNT) != 0) );                    \
-        __lower_bound += (PDATA)->true_lb;                              \
-        __upper_bound = (INITPTR) + (PDATA)->true_ub +                  \
-            ((PDATA)->ub - (PDATA)->lb) * ((COUNT) - 1);                \
-        if( ((ACTPTR) < __lower_bound) || ((ACTPTR) >= __upper_bound) ) { \
-            opal_datatype_safeguard_pointer_debug_breakpoint( (ACTPTR), (LENGTH), (INITPTR), (PDATA), (COUNT) ); \
-            opal_output( 0, "%s:%d\n\tPointer %p size %lu is outside [%p,%p] for\n\tbase ptr %p count %lu and data \n", \
-                         __FILE__, __LINE__, (void*)(ACTPTR), (unsigned long)(LENGTH), (void*)__lower_bound, (void*)__upper_bound, \
-                         (void*)(INITPTR), (unsigned long)(COUNT) );    \
-            opal_datatype_dump( (PDATA) );                              \
-        }                                                               \
-    }
+#    define OPAL_DATATYPE_SAFEGUARD_POINTER(ACTPTR, LENGTH, INITPTR, PDATA, COUNT)                \
+        {                                                                                         \
+            unsigned char *__lower_bound = (INITPTR), *__upper_bound;                             \
+            assert( (COUNT) != 0 );                                                               \
+            __lower_bound += (PDATA)->true_lb;                                                    \
+            __upper_bound = (INITPTR) + (PDATA)->true_ub +                                        \
+                            ((PDATA)->ub - (PDATA)->lb) * ((COUNT) -1);                           \
+            if (((ACTPTR) < __lower_bound) || ((ACTPTR) >= __upper_bound)) {                      \
+                opal_datatype_safeguard_pointer_debug_breakpoint((ACTPTR), (LENGTH), (INITPTR),   \
+                                                                 (PDATA), (COUNT));               \
+                opal_output(0,                                                                    \
+                            "%s:%d\n\tPointer %p size %lu is outside [%p,%p] for\n\tbase ptr %p " \
+                            "count %lu and data \n",                                              \
+                            __FILE__, __LINE__, (void *) (ACTPTR), (unsigned long) (LENGTH),      \
+                            (void *) __lower_bound, (void *) __upper_bound, (void *) (INITPTR),   \
+                            (unsigned long) (COUNT));                                             \
+                opal_datatype_dump((PDATA));                                                      \
+            }                                                                                     \
+        }
 
 #else
-#define OPAL_DATATYPE_SAFEGUARD_POINTER( ACTPTR, LENGTH, INITPTR, PDATA, COUNT )
-#endif  /* OPAL_ENABLE_DEBUG */
+#    define OPAL_DATATYPE_SAFEGUARD_POINTER(ACTPTR, LENGTH, INITPTR, PDATA, COUNT)
+#endif /* OPAL_ENABLE_DEBUG */
 
-static inline int GET_FIRST_NON_LOOP( const union dt_elem_desc* _pElem )
+static inline int GET_FIRST_NON_LOOP(const union dt_elem_desc *_pElem)
 {
     int element_index = 0;
 
     /* We dont have to check for the end as we always put an END_LOOP
      * at the end of all datatype descriptions.
      */
-    while( _pElem->elem.common.type == OPAL_DATATYPE_LOOP ) {
-        ++_pElem; element_index++;
+    while (_pElem->elem.common.type == OPAL_DATATYPE_LOOP) {
+        ++_pElem;
+        element_index++;
     }
     return element_index;
 }
 
-#define UPDATE_INTERNAL_COUNTERS( DESCRIPTION, POSITION, ELEMENT, COUNTER ) \
-    do {                                                                    \
-        (ELEMENT) = &((DESCRIPTION)[(POSITION)]);                           \
-        if( OPAL_DATATYPE_LOOP == (ELEMENT)->elem.common.type )             \
-            (COUNTER) = (ELEMENT)->loop.loops;                              \
-        else                                                                \
-            (COUNTER) = (ELEMENT)->elem.count * (ELEMENT)->elem.blocklen;   \
+#define UPDATE_INTERNAL_COUNTERS(DESCRIPTION, POSITION, ELEMENT, COUNTER) \
+    do {                                                                  \
+        (ELEMENT) = &((DESCRIPTION)[(POSITION)]);                         \
+        if (OPAL_DATATYPE_LOOP == (ELEMENT)->elem.common.type)            \
+            (COUNTER) = (ELEMENT)->loop.loops;                            \
+        else                                                              \
+            (COUNTER) = (ELEMENT)->elem.count * (ELEMENT)->elem.blocklen; \
     } while (0)
 
-OPAL_DECLSPEC int opal_datatype_contain_basic_datatypes( const struct opal_datatype_t* pData, char* ptr, size_t length );
-OPAL_DECLSPEC int opal_datatype_dump_data_flags( unsigned short usflags, char* ptr, size_t length );
-OPAL_DECLSPEC int opal_datatype_dump_data_desc( union dt_elem_desc* pDesc, int nbElems, char* ptr, size_t length );
+OPAL_DECLSPEC int opal_datatype_contain_basic_datatypes(const struct opal_datatype_t *pData,
+                                                        char *ptr, size_t length);
+OPAL_DECLSPEC int opal_datatype_dump_data_flags(unsigned short usflags, char *ptr, size_t length);
+OPAL_DECLSPEC int opal_datatype_dump_data_desc(union dt_elem_desc *pDesc, int nbElems, char *ptr,
+                                               size_t length);
 
 extern bool opal_ddt_position_debug;
 extern bool opal_ddt_copy_debug;
@@ -608,4 +596,4 @@ extern bool opal_ddt_pack_debug;
 extern bool opal_ddt_raw_debug;
 
 END_C_DECLS
-#endif  /* OPAL_DATATYPE_INTERNAL_H_HAS_BEEN_INCLUDED */
+#endif /* OPAL_DATATYPE_INTERNAL_H_HAS_BEEN_INCLUDED */
diff --git a/opal/datatype/opal_datatype_memcpy.h b/opal/datatype/opal_datatype_memcpy.h
index 972009ac96a..daa6b474bba 100644
--- a/opal/datatype/opal_datatype_memcpy.h
+++ b/opal/datatype/opal_datatype_memcpy.h
@@ -14,7 +14,6 @@
 #ifndef OPAL_DATATYPE_MEMCPY_H_HAS_BEEN_INCLUDED
 #define OPAL_DATATYPE_MEMCPY_H_HAS_BEEN_INCLUDED
 
-#define MEMCPY( DST, SRC, BLENGTH ) \
-    memcpy( (DST), (SRC), (BLENGTH) )
+#define MEMCPY(DST, SRC, BLENGTH) memcpy((DST), (SRC), (BLENGTH))
 
-#endif  /* OPAL_DATATYPE_MEMCPY_H_HAS_BEEN_INCLUDED */
+#endif /* OPAL_DATATYPE_MEMCPY_H_HAS_BEEN_INCLUDED */
diff --git a/opal/datatype/opal_datatype_module.c b/opal/datatype/opal_datatype_module.c
index 1ce57e67b50..527bb310bb1 100644
--- a/opal/datatype/opal_datatype_module.c
+++ b/opal/datatype/opal_datatype_module.c
@@ -31,83 +31,89 @@
 
 #include <stddef.h>
 
+#include "opal/datatype/opal_convertor_internal.h"
+#include "opal/datatype/opal_datatype.h"
+#include "opal/datatype/opal_datatype_internal.h"
+#include "opal/mca/base/mca_base_var.h"
 #include "opal/runtime/opal.h"
 #include "opal/util/arch.h"
 #include "opal/util/output.h"
-#include "opal/datatype/opal_datatype_internal.h"
-#include "opal/datatype/opal_datatype.h"
-#include "opal/datatype/opal_convertor_internal.h"
-#include "opal/mca/base/mca_base_var.h"
 
-/* by default the debuging is turned off */
+/* by default the debugging is turned off */
 int opal_datatype_dfd = -1;
 bool opal_ddt_unpack_debug = false;
 bool opal_ddt_pack_debug = false;
 bool opal_ddt_position_debug = false;
 bool opal_ddt_copy_debug = false;
 bool opal_ddt_raw_debug = false;
-int opal_ddt_verbose = -1;  /* Has the datatype verbose it's own output stream */
+int opal_ddt_verbose = -1; /* Has the datatype verbose it's own output stream */
 
-extern int opal_cuda_verbose;
-
-/* Using this macro implies that at this point _all_ informations needed
+/* Using this macro implies that at this point _all_ information needed
  * to fill up the datatype are known.
  * We fill all the static information, the pointer to desc.desc is setup
  * into an array, which is initialized at runtime.
  * Everything is constant.
  */
-OPAL_DECLSPEC const opal_datatype_t opal_datatype_empty =       OPAL_DATATYPE_INITIALIZER_EMPTY(OPAL_DATATYPE_FLAG_CONTIGUOUS);
+OPAL_DECLSPEC const opal_datatype_t opal_datatype_empty = OPAL_DATATYPE_INITIALIZER_EMPTY(
+    OPAL_DATATYPE_FLAG_CONTIGUOUS);
 
-OPAL_DECLSPEC const opal_datatype_t opal_datatype_loop =        OPAL_DATATYPE_INITIALIZER_LOOP(0);
-OPAL_DECLSPEC const opal_datatype_t opal_datatype_end_loop =    OPAL_DATATYPE_INITIALIZER_END_LOOP(0);
-OPAL_DECLSPEC const opal_datatype_t opal_datatype_lb =          OPAL_DATATYPE_INITIALIZER_LB(0);
-OPAL_DECLSPEC const opal_datatype_t opal_datatype_ub =          OPAL_DATATYPE_INITIALIZER_UB(0);
-OPAL_DECLSPEC const opal_datatype_t opal_datatype_int1 =        OPAL_DATATYPE_INITIALIZER_INT1(0);
-OPAL_DECLSPEC const opal_datatype_t opal_datatype_int2 =        OPAL_DATATYPE_INITIALIZER_INT2(0);
-OPAL_DECLSPEC const opal_datatype_t opal_datatype_int4 =        OPAL_DATATYPE_INITIALIZER_INT4(0);
-OPAL_DECLSPEC const opal_datatype_t opal_datatype_int8 =        OPAL_DATATYPE_INITIALIZER_INT8(0);
-OPAL_DECLSPEC const opal_datatype_t opal_datatype_int16 =       OPAL_DATATYPE_INITIALIZER_INT16(0);
-OPAL_DECLSPEC const opal_datatype_t opal_datatype_uint1 =       OPAL_DATATYPE_INITIALIZER_UINT1(0);
-OPAL_DECLSPEC const opal_datatype_t opal_datatype_uint2 =       OPAL_DATATYPE_INITIALIZER_UINT2(0);
-OPAL_DECLSPEC const opal_datatype_t opal_datatype_uint4 =       OPAL_DATATYPE_INITIALIZER_UINT4(0);
-OPAL_DECLSPEC const opal_datatype_t opal_datatype_uint8 =       OPAL_DATATYPE_INITIALIZER_UINT8(0);
-OPAL_DECLSPEC const opal_datatype_t opal_datatype_uint16 =      OPAL_DATATYPE_INITIALIZER_UINT16(0);
-OPAL_DECLSPEC const opal_datatype_t opal_datatype_float2 =      OPAL_DATATYPE_INITIALIZER_FLOAT2(0);
-OPAL_DECLSPEC const opal_datatype_t opal_datatype_float4 =      OPAL_DATATYPE_INITIALIZER_FLOAT4(0);
-OPAL_DECLSPEC const opal_datatype_t opal_datatype_float8 =      OPAL_DATATYPE_INITIALIZER_FLOAT8(0);
-OPAL_DECLSPEC const opal_datatype_t opal_datatype_float12 =     OPAL_DATATYPE_INITIALIZER_FLOAT12(0);
-OPAL_DECLSPEC const opal_datatype_t opal_datatype_float16 =     OPAL_DATATYPE_INITIALIZER_FLOAT16(0);
-OPAL_DECLSPEC const opal_datatype_t opal_datatype_short_float_complex = OPAL_DATATYPE_INITIALIZER_SHORT_FLOAT_COMPLEX(0);
-OPAL_DECLSPEC const opal_datatype_t opal_datatype_float_complex = OPAL_DATATYPE_INITIALIZER_FLOAT_COMPLEX(0);
-OPAL_DECLSPEC const opal_datatype_t opal_datatype_double_complex = OPAL_DATATYPE_INITIALIZER_DOUBLE_COMPLEX(0);
-OPAL_DECLSPEC const opal_datatype_t opal_datatype_long_double_complex = OPAL_DATATYPE_INITIALIZER_LONG_DOUBLE_COMPLEX(0);
-OPAL_DECLSPEC const opal_datatype_t opal_datatype_bool =        OPAL_DATATYPE_INITIALIZER_BOOL(0);
-OPAL_DECLSPEC const opal_datatype_t opal_datatype_wchar =       OPAL_DATATYPE_INITIALIZER_WCHAR(0);
-OPAL_DECLSPEC const opal_datatype_t opal_datatype_unavailable = OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED(UNAVAILABLE, 0);
+OPAL_DECLSPEC const opal_datatype_t opal_datatype_loop = OPAL_DATATYPE_INITIALIZER_LOOP(0);
+OPAL_DECLSPEC const opal_datatype_t opal_datatype_end_loop = OPAL_DATATYPE_INITIALIZER_END_LOOP(0);
+OPAL_DECLSPEC const opal_datatype_t opal_datatype_lb = OPAL_DATATYPE_INITIALIZER_LB(0);
+OPAL_DECLSPEC const opal_datatype_t opal_datatype_ub = OPAL_DATATYPE_INITIALIZER_UB(0);
+OPAL_DECLSPEC const opal_datatype_t opal_datatype_int1 = OPAL_DATATYPE_INITIALIZER_INT1(0);
+OPAL_DECLSPEC const opal_datatype_t opal_datatype_int2 = OPAL_DATATYPE_INITIALIZER_INT2(0);
+OPAL_DECLSPEC const opal_datatype_t opal_datatype_int4 = OPAL_DATATYPE_INITIALIZER_INT4(0);
+OPAL_DECLSPEC const opal_datatype_t opal_datatype_int8 = OPAL_DATATYPE_INITIALIZER_INT8(0);
+OPAL_DECLSPEC const opal_datatype_t opal_datatype_int16 = OPAL_DATATYPE_INITIALIZER_INT16(0);
+OPAL_DECLSPEC const opal_datatype_t opal_datatype_uint1 = OPAL_DATATYPE_INITIALIZER_UINT1(0);
+OPAL_DECLSPEC const opal_datatype_t opal_datatype_uint2 = OPAL_DATATYPE_INITIALIZER_UINT2(0);
+OPAL_DECLSPEC const opal_datatype_t opal_datatype_uint4 = OPAL_DATATYPE_INITIALIZER_UINT4(0);
+OPAL_DECLSPEC const opal_datatype_t opal_datatype_uint8 = OPAL_DATATYPE_INITIALIZER_UINT8(0);
+OPAL_DECLSPEC const opal_datatype_t opal_datatype_uint16 = OPAL_DATATYPE_INITIALIZER_UINT16(0);
+OPAL_DECLSPEC const opal_datatype_t opal_datatype_float2 = OPAL_DATATYPE_INITIALIZER_FLOAT2(0);
+OPAL_DECLSPEC const opal_datatype_t opal_datatype_float4 = OPAL_DATATYPE_INITIALIZER_FLOAT4(0);
+OPAL_DECLSPEC const opal_datatype_t opal_datatype_float8 = OPAL_DATATYPE_INITIALIZER_FLOAT8(0);
+OPAL_DECLSPEC const opal_datatype_t opal_datatype_float12 = OPAL_DATATYPE_INITIALIZER_FLOAT12(0);
+OPAL_DECLSPEC const opal_datatype_t opal_datatype_float16 = OPAL_DATATYPE_INITIALIZER_FLOAT16(0);
+OPAL_DECLSPEC const opal_datatype_t opal_datatype_short_float_complex
+    = OPAL_DATATYPE_INITIALIZER_SHORT_FLOAT_COMPLEX(0);
+OPAL_DECLSPEC const opal_datatype_t opal_datatype_float_complex
+    = OPAL_DATATYPE_INITIALIZER_FLOAT_COMPLEX(0);
+OPAL_DECLSPEC const opal_datatype_t opal_datatype_double_complex
+    = OPAL_DATATYPE_INITIALIZER_DOUBLE_COMPLEX(0);
+OPAL_DECLSPEC const opal_datatype_t opal_datatype_long_double_complex
+    = OPAL_DATATYPE_INITIALIZER_LONG_DOUBLE_COMPLEX(0);
+OPAL_DECLSPEC const opal_datatype_t opal_datatype_bool = OPAL_DATATYPE_INITIALIZER_BOOL(0);
+OPAL_DECLSPEC const opal_datatype_t opal_datatype_wchar = OPAL_DATATYPE_INITIALIZER_WCHAR(0);
+OPAL_DECLSPEC const opal_datatype_t opal_datatype_long =  OPAL_DATATYPE_INITIALIZER_LONG(0);
+OPAL_DECLSPEC const opal_datatype_t opal_datatype_unsigned_long =  OPAL_DATATYPE_INITIALIZER_UNSIGNED_LONG(0);
+OPAL_DECLSPEC const opal_datatype_t opal_datatype_unavailable
+    = OPAL_DATATYPE_INITIALIZER_UNAVAILABLE_NAMED(UNAVAILABLE, 0);
 
-OPAL_DECLSPEC dt_elem_desc_t opal_datatype_predefined_elem_desc[2 * OPAL_DATATYPE_MAX_PREDEFINED] = {{{{0}}}};
+OPAL_DECLSPEC dt_elem_desc_t opal_datatype_predefined_elem_desc[2 * OPAL_DATATYPE_MAX_PREDEFINED]
+    = {{{{0}}}};
 
 /*
  * NOTE: The order of this array *MUST* match the order in opal_datatype_basicDatatypes
  * (use of designated initializers should relax this restrictions some)
  */
-OPAL_DECLSPEC const size_t opal_datatype_local_sizes[OPAL_DATATYPE_MAX_PREDEFINED] =
-{
+OPAL_DECLSPEC const size_t opal_datatype_local_sizes[OPAL_DATATYPE_MAX_PREDEFINED] = {
     [OPAL_DATATYPE_INT1] = sizeof(int8_t),
     [OPAL_DATATYPE_INT2] = sizeof(int16_t),
     [OPAL_DATATYPE_INT4] = sizeof(int32_t),
     [OPAL_DATATYPE_INT8] = sizeof(int64_t),
-    [OPAL_DATATYPE_INT16] = 16,    /* sizeof (int128_t) */
+    [OPAL_DATATYPE_INT16] = 16, /* sizeof (int128_t) */
     [OPAL_DATATYPE_UINT1] = sizeof(uint8_t),
     [OPAL_DATATYPE_UINT2] = sizeof(uint16_t),
     [OPAL_DATATYPE_UINT4] = sizeof(uint32_t),
     [OPAL_DATATYPE_UINT8] = sizeof(uint64_t),
-    [OPAL_DATATYPE_UINT16] = 16,    /* sizeof (uint128_t) */
-    [OPAL_DATATYPE_FLOAT2] = 2,     /* sizeof (float2) */
-    [OPAL_DATATYPE_FLOAT4] = 4,     /* sizeof (float4) */
-    [OPAL_DATATYPE_FLOAT8] = 8,     /* sizeof (float8) */
-    [OPAL_DATATYPE_FLOAT12] = 12,   /* sizeof (float12) */
-    [OPAL_DATATYPE_FLOAT16] = 16,   /* sizeof (float16) */
+    [OPAL_DATATYPE_UINT16] = 16,  /* sizeof (uint128_t) */
+    [OPAL_DATATYPE_FLOAT2] = 2,   /* sizeof (float2) */
+    [OPAL_DATATYPE_FLOAT4] = 4,   /* sizeof (float4) */
+    [OPAL_DATATYPE_FLOAT8] = 8,   /* sizeof (float8) */
+    [OPAL_DATATYPE_FLOAT12] = 12, /* sizeof (float12) */
+    [OPAL_DATATYPE_FLOAT16] = 16, /* sizeof (float16) */
 #if defined(HAVE_SHORT_FLOAT__COMPLEX)
     [OPAL_DATATYPE_SHORT_FLOAT_COMPLEX] = sizeof(short float _Complex),
 #elif defined(HAVE_OPAL_SHORT_FLOAT_COMPLEX_T)
@@ -118,15 +124,17 @@ OPAL_DECLSPEC const size_t opal_datatype_local_sizes[OPAL_DATATYPE_MAX_PREDEFINE
     [OPAL_DATATYPE_FLOAT_COMPLEX] = sizeof(float _Complex),
     [OPAL_DATATYPE_DOUBLE_COMPLEX] = sizeof(double _Complex),
     [OPAL_DATATYPE_LONG_DOUBLE_COMPLEX] = sizeof(long double _Complex),
-    [OPAL_DATATYPE_BOOL] = sizeof (_Bool),
-    [OPAL_DATATYPE_WCHAR] = sizeof (wchar_t),
+    [OPAL_DATATYPE_BOOL] = sizeof(_Bool),
+    [OPAL_DATATYPE_WCHAR] = sizeof(wchar_t),
+    [OPAL_DATATYPE_LONG] = sizeof(long),
+    [OPAL_DATATYPE_UNSIGNED_LONG] = sizeof(unsigned long),
 };
 
 /*
  * NOTE: The order of this array *MUST* match what is listed in datatype.h
  * (use of designated initializers should relax this restrictions some)
  */
-OPAL_DECLSPEC const opal_datatype_t* opal_datatype_basicDatatypes[OPAL_DATATYPE_MAX_PREDEFINED] = {
+OPAL_DECLSPEC const opal_datatype_t *opal_datatype_basicDatatypes[OPAL_DATATYPE_MAX_PREDEFINED] = {
     [OPAL_DATATYPE_LOOP] = &opal_datatype_loop,
     [OPAL_DATATYPE_END_LOOP] = &opal_datatype_end_loop,
     [OPAL_DATATYPE_LB] = &opal_datatype_lb,
@@ -135,12 +143,14 @@ OPAL_DECLSPEC const opal_datatype_t* opal_datatype_basicDatatypes[OPAL_DATATYPE_
     [OPAL_DATATYPE_INT2] = &opal_datatype_int2,
     [OPAL_DATATYPE_INT4] = &opal_datatype_int4,
     [OPAL_DATATYPE_INT8] = &opal_datatype_int8,
-    [OPAL_DATATYPE_INT16] = &opal_datatype_int16,       /* Yes, double-machine word integers are available */
+    [OPAL_DATATYPE_INT16] = &opal_datatype_int16, /* Yes, double-machine word integers are available
+                                                   */
     [OPAL_DATATYPE_UINT1] = &opal_datatype_uint1,
     [OPAL_DATATYPE_UINT2] = &opal_datatype_uint2,
     [OPAL_DATATYPE_UINT4] = &opal_datatype_uint4,
     [OPAL_DATATYPE_UINT8] = &opal_datatype_uint8,
-    [OPAL_DATATYPE_UINT16] = &opal_datatype_uint16,      /* Yes, double-machine word integers are available */
+    [OPAL_DATATYPE_UINT16] = &opal_datatype_uint16, /* Yes, double-machine word integers are
+                                                       available */
     [OPAL_DATATYPE_FLOAT2] = &opal_datatype_float2,
     [OPAL_DATATYPE_FLOAT4] = &opal_datatype_float4,
     [OPAL_DATATYPE_FLOAT8] = &opal_datatype_float8,
@@ -152,134 +162,128 @@ OPAL_DECLSPEC const opal_datatype_t* opal_datatype_basicDatatypes[OPAL_DATATYPE_
     [OPAL_DATATYPE_LONG_DOUBLE_COMPLEX] = &opal_datatype_long_double_complex,
     [OPAL_DATATYPE_BOOL] = &opal_datatype_bool,
     [OPAL_DATATYPE_WCHAR] = &opal_datatype_wchar,
+    [OPAL_DATATYPE_LONG] = &opal_datatype_long,
+    [OPAL_DATATYPE_UNSIGNED_LONG] = &opal_datatype_unsigned_long,
     [OPAL_DATATYPE_UNAVAILABLE] = &opal_datatype_unavailable,
 };
 
-
 int opal_datatype_register_params(void)
 {
 #if OPAL_ENABLE_DEBUG
     int ret;
 
-    ret = mca_base_var_register ("opal", "mpi", NULL, "ddt_unpack_debug",
-                                 "Whether to output debugging information in the ddt unpack functions (nonzero = enabled)",
-                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
-                                 MCA_BASE_VAR_SCOPE_LOCAL, &opal_ddt_unpack_debug);
+    ret = mca_base_var_register(
+        "opal", "mpi", NULL, "ddt_unpack_debug",
+        "Whether to output debugging information in the ddt unpack functions (nonzero = enabled)",
+        MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
+        MCA_BASE_VAR_SCOPE_LOCAL, &opal_ddt_unpack_debug);
     if (0 > ret) {
         return ret;
     }
 
-    ret = mca_base_var_register ("opal", "mpi", NULL, "ddt_pack_debug",
-                                 "Whether to output debugging information in the ddt pack functions (nonzero = enabled)",
-                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
-                                 MCA_BASE_VAR_SCOPE_LOCAL, &opal_ddt_pack_debug);
+    ret = mca_base_var_register(
+        "opal", "mpi", NULL, "ddt_pack_debug",
+        "Whether to output debugging information in the ddt pack functions (nonzero = enabled)",
+        MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
+        MCA_BASE_VAR_SCOPE_LOCAL, &opal_ddt_pack_debug);
     if (0 > ret) {
         return ret;
     }
 
-    ret = mca_base_var_register ("opal", "mpi", NULL, "ddt_raw_debug",
-                                 "Whether to output debugging information in the ddt raw functions (nonzero = enabled)",
-                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
-                                 MCA_BASE_VAR_SCOPE_LOCAL, &opal_ddt_raw_debug);
+    ret = mca_base_var_register(
+        "opal", "mpi", NULL, "ddt_raw_debug",
+        "Whether to output debugging information in the ddt raw functions (nonzero = enabled)",
+        MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
+        MCA_BASE_VAR_SCOPE_LOCAL, &opal_ddt_raw_debug);
     if (0 > ret) {
         return ret;
     }
 
-    ret = mca_base_var_register ("opal", "mpi", NULL, "ddt_position_debug",
-                                 "Non zero lead to output generated by the datatype position functions",
-                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
-                                 MCA_BASE_VAR_SCOPE_LOCAL, &opal_ddt_position_debug);
+    ret = mca_base_var_register(
+        "opal", "mpi", NULL, "ddt_position_debug",
+        "Non zero lead to output generated by the datatype position functions",
+        MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
+        MCA_BASE_VAR_SCOPE_LOCAL, &opal_ddt_position_debug);
     if (0 > ret) {
         return ret;
     }
 
-    ret = mca_base_var_register ("opal", "mpi", NULL, "ddt_copy_debug",
-                                 "Whether to output debugging information in the ddt copy functions (nonzero = enabled)",
-                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
-                                 MCA_BASE_VAR_SCOPE_LOCAL, &opal_ddt_copy_debug);
+    ret = mca_base_var_register(
+        "opal", "mpi", NULL, "ddt_copy_debug",
+        "Whether to output debugging information in the ddt copy functions (nonzero = enabled)",
+        MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
+        MCA_BASE_VAR_SCOPE_LOCAL, &opal_ddt_copy_debug);
     if (0 > ret) {
         return ret;
     }
 
-    ret = mca_base_var_register ("opal", "opal", NULL, "ddt_verbose",
-                                 "Set level of opal datatype verbosity",
-                                 MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-                                 OPAL_INFO_LVL_8, MCA_BASE_VAR_SCOPE_LOCAL,
-                                 &opal_ddt_verbose);
-    if (0 > ret) {
-        return ret;
-    }
-#if OPAL_CUDA_SUPPORT
-    /* Set different levels of verbosity in the cuda related code. */
-    ret = mca_base_var_register ("opal", "opal", NULL, "cuda_verbose",
-                                 "Set level of opal cuda verbosity",
-                                 MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-                                 OPAL_INFO_LVL_8, MCA_BASE_VAR_SCOPE_LOCAL,
-                                 &opal_cuda_verbose);
+    ret = mca_base_var_register("opal", "opal", NULL, "ddt_verbose",
+                                "Set level of opal datatype verbosity", MCA_BASE_VAR_TYPE_INT, NULL,
+                                0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_8,
+                                MCA_BASE_VAR_SCOPE_LOCAL, &opal_ddt_verbose);
     if (0 > ret) {
         return ret;
     }
-#endif
 
 #endif /* OPAL_ENABLE_DEBUG */
 
     return OPAL_SUCCESS;
 }
 
-static void opal_datatype_finalize (void)
+static void opal_datatype_finalize(void)
 {
     /* As the synonyms are just copies of the internal data we should not free them.
      * Anyway they are over the limit of OPAL_DATATYPE_MAX_PREDEFINED so they will never get freed.
      */
 
     /* As they are statically allocated they cannot be released. But we
-     * can call OBJ_DESTRUCT, just to free all internally allocated ressources.
+     * can call OBJ_DESTRUCT, just to free all internally allocated resources.
      */
     /* clear all master convertors */
     opal_convertor_destroy_masters();
 
-    opal_output_close (opal_datatype_dfd);
+    opal_output_close(opal_datatype_dfd);
     opal_datatype_dfd = -1;
 }
 
-int32_t opal_datatype_init( void )
+int32_t opal_datatype_init(void)
 {
-    const opal_datatype_t* datatype;
+    const opal_datatype_t *datatype;
     int32_t i;
 
     /**
-     * Force he initialization of the opal_datatype_t class. This will allow us to
+     * Force the initialization of the opal_datatype_t class. This will allow us to
      * call OBJ_DESTRUCT without going too deep in the initialization process.
      */
     opal_class_initialize(OBJ_CLASS(opal_datatype_t));
-    for( i = OPAL_DATATYPE_FIRST_TYPE; i < OPAL_DATATYPE_MAX_PREDEFINED; i++ ) {
+    for (i = OPAL_DATATYPE_FIRST_TYPE; i < OPAL_DATATYPE_MAX_PREDEFINED; i++) {
         datatype = opal_datatype_basicDatatypes[i];
 
         /* All of the predefined OPAL types don't have any GAPS! */
-        datatype->desc.desc[0].elem.common.flags = OPAL_DATATYPE_FLAG_PREDEFINED |
-                                                   OPAL_DATATYPE_FLAG_DATA |
-                                                   OPAL_DATATYPE_FLAG_CONTIGUOUS |
-                                                   OPAL_DATATYPE_FLAG_NO_GAPS;
-        datatype->desc.desc[0].elem.common.type  = i;
-        datatype->desc.desc[0].elem.count        = 1;
-        datatype->desc.desc[0].elem.blocklen     = 1;
-        datatype->desc.desc[0].elem.disp         = 0;
-        datatype->desc.desc[0].elem.extent       = datatype->size;
+        datatype->desc.desc[0].elem.common.flags = OPAL_DATATYPE_FLAG_PREDEFINED
+                                                   | OPAL_DATATYPE_FLAG_DATA
+                                                   | OPAL_DATATYPE_FLAG_CONTIGUOUS
+                                                   | OPAL_DATATYPE_FLAG_NO_GAPS;
+        datatype->desc.desc[0].elem.common.type = i;
+        datatype->desc.desc[0].elem.count = 1;
+        datatype->desc.desc[0].elem.blocklen = 1;
+        datatype->desc.desc[0].elem.disp = 0;
+        datatype->desc.desc[0].elem.extent = datatype->size;
 
-        datatype->desc.desc[1].end_loop.common.flags    = 0;
-        datatype->desc.desc[1].end_loop.common.type     = OPAL_DATATYPE_END_LOOP;
-        datatype->desc.desc[1].end_loop.items           = 1;
+        datatype->desc.desc[1].end_loop.common.flags = 0;
+        datatype->desc.desc[1].end_loop.common.type = OPAL_DATATYPE_END_LOOP;
+        datatype->desc.desc[1].end_loop.items = 1;
         datatype->desc.desc[1].end_loop.first_elem_disp = datatype->desc.desc[0].elem.disp;
-        datatype->desc.desc[1].end_loop.size            = datatype->size;
+        datatype->desc.desc[1].end_loop.size = datatype->size;
     }
 
     /* Enable a private output stream for datatype */
-    if( opal_ddt_verbose > 0 ) {
+    if (opal_ddt_verbose > 0) {
         opal_datatype_dfd = opal_output_open(NULL);
         opal_output_set_verbosity(opal_datatype_dfd, opal_ddt_verbose);
     }
 
-    opal_finalize_register_cleanup (opal_datatype_finalize);
+    opal_finalize_register_cleanup(opal_datatype_finalize);
 
     return OPAL_SUCCESS;
 }
@@ -289,11 +293,10 @@ int32_t opal_datatype_init( void )
  * Set a breakpoint to this function in your favorite debugger
  * to make it stop on all pack and unpack errors.
  */
-int opal_datatype_safeguard_pointer_debug_breakpoint( const void* actual_ptr, int length,
-                                                      const void* initial_ptr,
-                                                      const opal_datatype_t* pData,
-                                                      int count )
+int opal_datatype_safeguard_pointer_debug_breakpoint(const void *actual_ptr, int length,
+                                                     const void *initial_ptr,
+                                                     const opal_datatype_t *pData, int count)
 {
     return 0;
 }
-#endif  /* OPAL_ENABLE_DEBUG */
+#endif /* OPAL_ENABLE_DEBUG */
diff --git a/opal/datatype/opal_datatype_monotonic.c b/opal/datatype/opal_datatype_monotonic.c
index 247fd66142d..67498332ca2 100644
--- a/opal/datatype/opal_datatype_monotonic.c
+++ b/opal/datatype/opal_datatype_monotonic.c
@@ -17,9 +17,9 @@
 #include <stddef.h>
 
 #include "opal/constants.h"
+#include "opal/datatype/opal_convertor.h"
 #include "opal/datatype/opal_datatype.h"
 #include "opal/datatype/opal_datatype_internal.h"
-#include "opal/datatype/opal_convertor.h"
 
 #define OPAL_DATATYPE_MAX_MONOTONIC_IOVEC 32
 
@@ -28,41 +28,42 @@
  * the contiguous pieces are always advancing in the same direction, i.e.
  * there is no potential for overlap.
  */
-int32_t opal_datatype_is_monotonic(opal_datatype_t* type )
+int32_t opal_datatype_is_monotonic(opal_datatype_t *type)
 {
     struct iovec iov[OPAL_DATATYPE_MAX_MONOTONIC_IOVEC];
-    ptrdiff_t upper_limit = (ptrdiff_t)type->true_lb;  /* as conversion base will be NULL the first address is true_lb */
+    ptrdiff_t upper_limit = (ptrdiff_t) type->true_lb; /* as conversion base will be NULL the first
+                                                          address is true_lb */
     size_t max_data = 0x7FFFFFFF;
     opal_convertor_t *pConv;
     bool monotonic = true;
     uint32_t iov_count;
     int rc;
 
-    pConv  = opal_convertor_create( opal_local_arch, 0 );
+    pConv = opal_convertor_create(opal_local_arch, 0);
     if (OPAL_UNLIKELY(NULL == pConv)) {
         return -1;
     }
-    rc = opal_convertor_prepare_for_send( pConv, type, 1, NULL );
-    if( OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
+    rc = opal_convertor_prepare_for_send(pConv, type, 1, NULL);
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
         OBJ_RELEASE(pConv);
         return -1;
     }
 
     do {
         iov_count = OPAL_DATATYPE_MAX_MONOTONIC_IOVEC;
-        rc = opal_convertor_raw( pConv, iov, &iov_count, &max_data);
+        rc = opal_convertor_raw(pConv, iov, &iov_count, &max_data);
         for (uint32_t i = 0; i < iov_count; i++) {
-            if ((ptrdiff_t)iov[i].iov_base < upper_limit) {
+            if ((ptrdiff_t) iov[i].iov_base < upper_limit) {
                 monotonic = false;
                 goto cleanup;
             }
             /* The new upper bound is at the end of the iovec */
-            upper_limit = (ptrdiff_t)iov[i].iov_base + iov[i].iov_len;
+            upper_limit = (ptrdiff_t) iov[i].iov_base + iov[i].iov_len;
         }
     } while (rc != 1);
 
-  cleanup:
-    OBJ_RELEASE( pConv );
+cleanup:
+    OBJ_RELEASE(pConv);
 
     return monotonic;
 }
diff --git a/opal/datatype/opal_datatype_optimize.c b/opal/datatype/opal_datatype_optimize.c
index 2e661b95daa..dbf7aeb86a3 100644
--- a/opal/datatype/opal_datatype_optimize.c
+++ b/opal/datatype/opal_datatype_optimize.c
@@ -26,71 +26,77 @@
 #include <stddef.h>
 #include <stdlib.h>
 
-#include "opal/datatype/opal_datatype.h"
 #include "opal/datatype/opal_convertor.h"
+#include "opal/datatype/opal_datatype.h"
 #include "opal/datatype/opal_datatype_internal.h"
 
-static int32_t
-opal_datatype_optimize_short( opal_datatype_t* pData,
-                              size_t count,
-                              dt_type_desc_t* pTypeDesc )
+static int32_t opal_datatype_optimize_short(opal_datatype_t *pData, size_t count,
+                                            dt_type_desc_t *pTypeDesc)
 {
-    dt_elem_desc_t* pElemDesc;
+    dt_elem_desc_t *pElemDesc;
     dt_stack_t *pOrigStack, *pStack; /* pointer to the position on the stack */
-    int32_t pos_desc = 0;            /* actual position in the description of the derived datatype */
+    int32_t pos_desc = 0; /* actual position in the description of the derived datatype */
     int32_t stack_pos = 0;
     int32_t nbElems = 0;
     ptrdiff_t total_disp = 0;
     ddt_elem_desc_t last = {.common.flags = 0xFFFF /* all on */, .count = 0, .disp = 0}, compress;
-    ddt_elem_desc_t* current;
+    ddt_elem_desc_t *current;
 
-    pOrigStack = pStack = (dt_stack_t*)malloc( sizeof(dt_stack_t) * (pData->loops+2) );
-    SAVE_STACK( pStack, -1, 0, count, 0 );
+    pOrigStack = pStack = (dt_stack_t *) malloc(sizeof(dt_stack_t) * (pData->loops + 2));
+    SAVE_STACK(pStack, -1, 0, count, 0);
 
-    pTypeDesc->length = 2 * pData->desc.used + 1 /* for the fake OPAL_DATATYPE_END_LOOP at the end */;
-    pTypeDesc->desc = pElemDesc = (dt_elem_desc_t*)malloc( sizeof(dt_elem_desc_t) * pTypeDesc->length );
+    pTypeDesc->length = 2 * pData->desc.used
+                        + 1 /* for the fake OPAL_DATATYPE_END_LOOP at the end */;
+    pTypeDesc->desc = pElemDesc = (dt_elem_desc_t *) malloc(sizeof(dt_elem_desc_t)
+                                                            * pTypeDesc->length);
     pTypeDesc->used = 0;
 
-    assert( OPAL_DATATYPE_END_LOOP == pData->desc.desc[pData->desc.used].elem.common.type );
+    assert(OPAL_DATATYPE_END_LOOP == pData->desc.desc[pData->desc.used].elem.common.type);
 
-    while( stack_pos >= 0 ) {
-        if( OPAL_DATATYPE_END_LOOP == pData->desc.desc[pos_desc].elem.common.type ) { /* end of the current loop */
-            ddt_endloop_desc_t* end_loop = &(pData->desc.desc[pos_desc].end_loop);
-            if( 0 != last.count ) {
-                CREATE_ELEM( pElemDesc, last.common.type, OPAL_DATATYPE_FLAG_BASIC,
-                             last.blocklen, last.count, last.disp, last.extent );
-                pElemDesc++; nbElems++;
-                last.count= 0;
+    while (stack_pos >= 0) {
+        if (OPAL_DATATYPE_END_LOOP
+            == pData->desc.desc[pos_desc].elem.common.type) { /* end of the current loop */
+            ddt_endloop_desc_t *end_loop = &(pData->desc.desc[pos_desc].end_loop);
+            if (0 != last.count) {
+                CREATE_ELEM(pElemDesc, last.common.type, OPAL_DATATYPE_FLAG_BASIC, last.blocklen,
+                            last.count, last.disp, last.extent);
+                pElemDesc++;
+                nbElems++;
+                last.count = 0;
             }
-            CREATE_LOOP_END( pElemDesc, nbElems - pStack->index + 1,  /* # of elems in this loop */
-                             end_loop->first_elem_disp, end_loop->size, end_loop->common.flags );
-            if( --stack_pos >= 0 ) {  /* still something to do ? */
-                ddt_loop_desc_t* pStartLoop = &(pTypeDesc->desc[pStack->index - 1].loop);
+            CREATE_LOOP_END(pElemDesc, nbElems - pStack->index + 1, /* # of elems in this loop */
+                            end_loop->first_elem_disp, end_loop->size, end_loop->common.flags);
+            if (--stack_pos >= 0) { /* still something to do ? */
+                ddt_loop_desc_t *pStartLoop = &(pTypeDesc->desc[pStack->index - 1].loop);
                 pStartLoop->items = pElemDesc->end_loop.items;
-                total_disp = pStack->disp;  /* update the displacement position */
+                total_disp = pStack->disp; /* update the displacement position */
             }
-            pElemDesc++; nbElems++;
-            pStack--;  /* go down one position on the stack */
+            pElemDesc++;
+            nbElems++;
+            pStack--; /* go down one position on the stack */
             pos_desc++;
             continue;
         }
-        if( OPAL_DATATYPE_LOOP == pData->desc.desc[pos_desc].elem.common.type ) {
-            ddt_loop_desc_t* loop = (ddt_loop_desc_t*)&(pData->desc.desc[pos_desc]);
-            int index = GET_FIRST_NON_LOOP( &(pData->desc.desc[pos_desc]) );
+        if (OPAL_DATATYPE_LOOP == pData->desc.desc[pos_desc].elem.common.type) {
+            ddt_loop_desc_t *loop = (ddt_loop_desc_t *) &(pData->desc.desc[pos_desc]);
+            int index = GET_FIRST_NON_LOOP(&(pData->desc.desc[pos_desc]));
 
-            if( loop->common.flags & OPAL_DATATYPE_FLAG_CONTIGUOUS ) {
-                ddt_endloop_desc_t* end_loop = (ddt_endloop_desc_t*)&(pData->desc.desc[pos_desc + loop->items]);
+            if (loop->common.flags & OPAL_DATATYPE_FLAG_CONTIGUOUS) {
+                ddt_endloop_desc_t *end_loop = (ddt_endloop_desc_t *) &(
+                    pData->desc.desc[pos_desc + loop->items]);
 
                 assert(pData->desc.desc[pos_desc + index].elem.disp == end_loop->first_elem_disp);
                 compress.common.flags = loop->common.flags;
-                compress.common.type =  pData->desc.desc[pos_desc + index].elem.common.type;
+                compress.common.type = pData->desc.desc[pos_desc + index].elem.common.type;
                 compress.blocklen = pData->desc.desc[pos_desc + index].elem.blocklen;
-                for( uint32_t i = index+1; i < loop->items; i++ ) {
+                for (uint32_t i = index + 1; i < loop->items; i++) {
                     current = &pData->desc.desc[pos_desc + i].elem;
-                    assert(1 ==  current->count);
-                    if( (current->common.type == OPAL_DATATYPE_LOOP) ||
-                        compress.common.type != current->common.type ) {
-                        compress.common.type = OPAL_DATATYPE_UINT1;
+                    assert(1 == current->count);
+                    if ((current->common.type == OPAL_DATATYPE_LOOP)
+                        || compress.common.type != current->common.type) {
+                        compress.common.type   = OPAL_DATATYPE_UINT1;
+                        compress.common.flags |= OPAL_DATATYPE_OPTIMIZED_RESTRICTED;
+                        pData->flags          |= OPAL_DATATYPE_OPTIMIZED_RESTRICTED;
                         compress.blocklen = end_loop->size;
                         break;
                     }
@@ -99,11 +105,14 @@ opal_datatype_optimize_short( opal_datatype_t* pData,
                 compress.count = loop->loops;
                 compress.extent = loop->extent;
                 compress.disp = end_loop->first_elem_disp;
-                if( compress.extent == (ptrdiff_t)(compress.blocklen * opal_datatype_basicDatatypes[compress.common.type]->size) ) {
-                    /* The compressed element is contiguous: collapse it into a single large blocklen */
+                if (compress.extent
+                    == (ptrdiff_t)(compress.blocklen
+                                   * opal_datatype_basicDatatypes[compress.common.type]->size)) {
+                    /* The compressed element is contiguous: collapse it into a single large
+                     * blocklen */
                     compress.blocklen *= compress.count;
-                    compress.extent   *= compress.count;
-                    compress.count     = 1;
+                    compress.extent *= compress.count;
+                    compress.count = 1;
                 }
                 /**
                  * The current loop has been compressed and can now be treated as if it
@@ -121,23 +130,26 @@ opal_datatype_optimize_short( opal_datatype_t* pData,
              * in reducing the number of memcpy during pack/unpack.
              */
 
-            if( 0 != last.count ) {  /* Generate the pending element */
-                CREATE_ELEM( pElemDesc, last.common.type, OPAL_DATATYPE_FLAG_BASIC,
-                             last.blocklen, last.count, last.disp, last.extent );
-                pElemDesc++; nbElems++;
-                last.count       = 0;
+            if (0 != last.count) { /* Generate the pending element */
+                CREATE_ELEM(pElemDesc, last.common.type, OPAL_DATATYPE_FLAG_BASIC, last.blocklen,
+                            last.count, last.disp, last.extent);
+                pElemDesc++;
+                nbElems++;
+                last.count = 0;
                 last.common.type = OPAL_DATATYPE_LOOP;
             }
 
             /* Can we unroll the loop? */
-            if( (loop->items <= 3) && (loop->loops <= 2) ) {
+            if ((loop->items <= 3) && (loop->loops <= 2)) {
                 ptrdiff_t elem_displ = 0;
-                for( uint32_t i = 0; i < loop->loops; i++ ) {
-                    for( uint32_t j = 0; j < (loop->items - 1); j++ ) {
+                for (uint32_t i = 0; i < loop->loops; i++) {
+                    for (uint32_t j = 0; j < (loop->items - 1); j++) {
                         current = &pData->desc.desc[pos_desc + index + j].elem;
-                        CREATE_ELEM( pElemDesc, current->common.type, current->common.flags,
-                                     current->blocklen, current->count, current->disp + elem_displ, current->extent );
-                        pElemDesc++; nbElems++;
+                        CREATE_ELEM(pElemDesc, current->common.type, current->common.flags,
+                                    current->blocklen, current->count, current->disp + elem_displ,
+                                    current->extent);
+                        pElemDesc++;
+                        nbElems++;
                     }
                     elem_displ += loop->extent;
                 }
@@ -145,26 +157,31 @@ opal_datatype_optimize_short( opal_datatype_t* pData,
                 goto complete_loop;
             }
 
-            CREATE_LOOP_START( pElemDesc, loop->loops, loop->items, loop->extent, loop->common.flags );
-            pElemDesc++; nbElems++;
-            PUSH_STACK( pStack, stack_pos, nbElems, OPAL_DATATYPE_LOOP, loop->loops, total_disp );
+            CREATE_LOOP_START(pElemDesc, loop->loops, loop->items, loop->extent,
+                              loop->common.flags);
+            pElemDesc++;
+            nbElems++;
+            PUSH_STACK(pStack, stack_pos, nbElems, OPAL_DATATYPE_LOOP, loop->loops, total_disp);
             pos_desc++;
-            DDT_DUMP_STACK( pStack, stack_pos, pData->desc.desc, "advance loops" );
+            DDT_DUMP_STACK(pStack, stack_pos, pData->desc.desc, "advance loops");
 
         complete_loop:
-            total_disp = pStack->disp;  /* update the displacement */
+            total_disp = pStack->disp; /* update the displacement */
             continue;
         }
-        while( pData->desc.desc[pos_desc].elem.common.flags & OPAL_DATATYPE_FLAG_DATA ) {  /* go over all basic datatype elements */
+        while (pData->desc.desc[pos_desc].elem.common.flags
+               & OPAL_DATATYPE_FLAG_DATA) { /* go over all basic datatype elements */
             current = &pData->desc.desc[pos_desc].elem;
-            pos_desc++;  /* point to the next element as current points to the current one */
+            pos_desc++; /* point to the next element as current points to the current one */
 
-          fuse_loops:
-            if( 0 == last.count ) {  /* first data of the datatype */
+        fuse_loops:
+            if (0 == last.count) { /* first data of the datatype */
                 last = *current;
-                continue;  /* next data */
-            } else {  /* can we merge it in order to decrease count */
-                if( (ptrdiff_t)last.blocklen * (ptrdiff_t)opal_datatype_basicDatatypes[last.common.type]->size == last.extent ) {
+                continue; /* next data */
+            } else {      /* can we merge it in order to decrease count */
+                if ((ptrdiff_t) last.blocklen
+                        * (ptrdiff_t) opal_datatype_basicDatatypes[last.common.type]->size
+                    == last.extent) {
                     last.extent *= last.count;
                     last.blocklen *= last.count;
                     last.count = 1;
@@ -174,33 +191,36 @@ opal_datatype_optimize_short( opal_datatype_t* pData,
             /* are the two elements compatible: aka they have very similar values and they
              * can be merged together by increasing the count, and/or changing the extent.
              */
-            if( (last.blocklen * opal_datatype_basicDatatypes[last.common.type]->size) ==
-                (current->blocklen * opal_datatype_basicDatatypes[current->common.type]->size) ) {
-                ddt_elem_desc_t save = last;  /* safekeep the type and blocklen */
-                if( last.common.type != current->common.type ) {
-                    last.blocklen    *= opal_datatype_basicDatatypes[last.common.type]->size;
-                    last.common.type  = OPAL_DATATYPE_UINT1;
+            if ((last.blocklen * opal_datatype_basicDatatypes[last.common.type]->size)
+                == (current->blocklen * opal_datatype_basicDatatypes[current->common.type]->size)) {
+                ddt_elem_desc_t save = last; /* safekeep the type and blocklen */
+                if (last.common.type != current->common.type) {
+                    last.blocklen *= opal_datatype_basicDatatypes[last.common.type]->size;
+                    last.common.type   = OPAL_DATATYPE_UINT1;
+                    last.common.flags |= OPAL_DATATYPE_OPTIMIZED_RESTRICTED;
+                    pData->flags      |= OPAL_DATATYPE_OPTIMIZED_RESTRICTED;
                 }
 
-                if( (last.extent * (ptrdiff_t)last.count + last.disp) == current->disp ) {
-                    if( 1 == current->count ) {
+                if ((last.extent * (ptrdiff_t) last.count + last.disp) == current->disp) {
+                    if (1 == current->count) {
                         last.count++;
                         continue;
                     }
-                    if( last.extent == current->extent ) {
+                    if (last.extent == current->extent) {
                         last.count += current->count;
                         continue;
                     }
                 }
-                if( 1 == last.count ) {
-                    /* we can ignore the extent of the element with count == 1 and merge them together if their displacements match */
-                    if( 1 == current->count ) {
+                if (1 == last.count) {
+                    /* we can ignore the extent of the element with count == 1 and merge them
+                     * together if their displacements match */
+                    if (1 == current->count) {
                         last.extent = current->disp - last.disp;
                         last.count++;
                         continue;
                     }
                     /* can we compute a matching displacement ? */
-                    if( (last.disp + current->extent) == current->disp ) {
+                    if ((last.disp + current->extent) == current->disp) {
                         last.extent = current->extent;
                         last.count = current->count + last.count;
                         continue;
@@ -210,105 +230,117 @@ opal_datatype_optimize_short( opal_datatype_t* pData,
                 last.common.type = save.common.type;
                 /* try other optimizations */
             }
-            /* are the elements fusionable such that we can fusion the last blocklen of one with the first
-             * blocklen of the other.
+            /* are the elements fusionable such that we can fusion the last blocklen of one with the
+             * first blocklen of the other.
              */
-            if( (ptrdiff_t)(last.disp + (last.count - 1) * last.extent + last.blocklen * opal_datatype_basicDatatypes[last.common.type]->size) ==
-                current->disp ) {
-                if( last.count != 1 ) {
-                    CREATE_ELEM( pElemDesc, last.common.type, OPAL_DATATYPE_FLAG_BASIC,
-                                 last.blocklen, last.count - 1, last.disp, last.extent );
-                    pElemDesc++; nbElems++;
+            if ((ptrdiff_t)(last.disp + (last.count - 1) * last.extent
+                            + last.blocklen * opal_datatype_basicDatatypes[last.common.type]->size)
+                == current->disp) {
+                if (last.count != 1) {
+                    CREATE_ELEM(pElemDesc, last.common.type, OPAL_DATATYPE_FLAG_BASIC,
+                                last.blocklen, last.count - 1, last.disp, last.extent);
+                    pElemDesc++;
+                    nbElems++;
                     last.disp += (last.count - 1) * last.extent;
                     last.count = 1;
                 }
-                if( last.common.type == current->common.type ) {
+                if (last.common.type == current->common.type) {
                     last.blocklen += current->blocklen;
                 } else {
-                    last.blocklen = ((last.blocklen * opal_datatype_basicDatatypes[last.common.type]->size) +
-                                     (current->blocklen * opal_datatype_basicDatatypes[current->common.type]->size));
-                    last.common.type = OPAL_DATATYPE_UINT1;
+                    last.blocklen = ((last.blocklen
+                                      * opal_datatype_basicDatatypes[last.common.type]->size)
+                                     + (current->blocklen
+                                        * opal_datatype_basicDatatypes[current->common.type]
+                                              ->size));
+                    last.common.type   = OPAL_DATATYPE_UINT1;
+                    last.common.flags |= OPAL_DATATYPE_OPTIMIZED_RESTRICTED;
+                    pData->flags      |= OPAL_DATATYPE_OPTIMIZED_RESTRICTED;
                 }
                 last.extent += current->extent;
-                if( current->count != 1 ) {
-                    CREATE_ELEM( pElemDesc, last.common.type, OPAL_DATATYPE_FLAG_BASIC,
-                                 last.blocklen, last.count, last.disp, last.extent );
-                    pElemDesc++; nbElems++;
+                if (current->count != 1) {
+                    CREATE_ELEM(pElemDesc, last.common.type, OPAL_DATATYPE_FLAG_BASIC,
+                                last.blocklen, last.count, last.disp, last.extent);
+                    pElemDesc++;
+                    nbElems++;
                     last = *current;
                     last.count -= 1;
                     last.disp += last.extent;
                 }
                 continue;
             }
-            CREATE_ELEM( pElemDesc, last.common.type, OPAL_DATATYPE_FLAG_BASIC,
-                         last.blocklen, last.count, last.disp, last.extent );
-            pElemDesc++; nbElems++;
+            CREATE_ELEM(pElemDesc, last.common.type, OPAL_DATATYPE_FLAG_BASIC, last.blocklen,
+                        last.count, last.disp, last.extent);
+            pElemDesc++;
+            nbElems++;
             last = *current;
         }
     }
 
-    if( 0 != last.count ) {
-        CREATE_ELEM( pElemDesc, last.common.type, OPAL_DATATYPE_FLAG_BASIC,
-                     last.blocklen, last.count, last.disp, last.extent );
-        pElemDesc++; nbElems++;
+    if (0 != last.count) {
+        CREATE_ELEM(pElemDesc, last.common.type, OPAL_DATATYPE_FLAG_BASIC, last.blocklen,
+                    last.count, last.disp, last.extent);
+        pElemDesc++;
+        nbElems++;
     }
     /* cleanup the stack */
-    pTypeDesc->used = nbElems - 1;  /* except the last fake END_LOOP */
+    pTypeDesc->used = nbElems - 1; /* except the last fake END_LOOP */
     free(pOrigStack);
     return OPAL_SUCCESS;
 }
 
-int32_t opal_datatype_commit( opal_datatype_t * pData )
+int32_t opal_datatype_commit(opal_datatype_t *pData)
 {
-    ddt_endloop_desc_t* pLast = &(pData->desc.desc[pData->desc.used].end_loop);
+    ddt_endloop_desc_t *pLast = &(pData->desc.desc[pData->desc.used].end_loop);
     ptrdiff_t first_elem_disp = 0;
 
-    if( pData->flags & OPAL_DATATYPE_FLAG_COMMITTED ) return OPAL_SUCCESS;
+    if (pData->flags & OPAL_DATATYPE_FLAG_COMMITTED) {
+        return OPAL_SUCCESS;
+    }
     pData->flags |= OPAL_DATATYPE_FLAG_COMMITTED;
 
     /* We have to compute the displacement of the first non loop item in the
      * description.
      */
-    if( 0 != pData->size ) {
+    if (0 != pData->size) {
         int index;
-        dt_elem_desc_t* pElem = pData->desc.desc;
+        dt_elem_desc_t *pElem = pData->desc.desc;
 
-        index = GET_FIRST_NON_LOOP( pElem );
-        assert( pElem[index].elem.common.flags & OPAL_DATATYPE_FLAG_DATA );
+        index = GET_FIRST_NON_LOOP(pElem);
+        assert(pElem[index].elem.common.flags & OPAL_DATATYPE_FLAG_DATA);
         first_elem_disp = pElem[index].elem.disp;
     }
 
     /* let's add a fake element at the end just to avoid useless comparaisons
      * in pack/unpack functions.
      */
-    pLast->common.type     = OPAL_DATATYPE_END_LOOP;
-    pLast->common.flags    = 0;
-    pLast->items           = pData->desc.used;
+    pLast->common.type = OPAL_DATATYPE_END_LOOP;
+    pLast->common.flags = 0;
+    pLast->items = pData->desc.used;
     pLast->first_elem_disp = first_elem_disp;
-    pLast->size            = pData->size;
+    pLast->size = pData->size;
 
     /* If there is no datatype description how can we have an optimized description ? */
-    if( 0 == pData->desc.used ) {
+    if (0 == pData->desc.used) {
         pData->opt_desc.length = 0;
-        pData->opt_desc.desc   = NULL;
-        pData->opt_desc.used   = 0;
+        pData->opt_desc.desc = NULL;
+        pData->opt_desc.used = 0;
         return OPAL_SUCCESS;
     }
 
     /* If the data is contiguous is useless to generate an optimized version. */
     /*if( pData->size == (pData->true_ub - pData->true_lb) ) return OPAL_SUCCESS; */
 
-    (void)opal_datatype_optimize_short( pData, 1, &(pData->opt_desc) );
-    if( 0 != pData->opt_desc.used ) {
+    (void) opal_datatype_optimize_short(pData, 1, &(pData->opt_desc));
+    if (0 != pData->opt_desc.used) {
         /* let's add a fake element at the end just to avoid useless comparaisons
          * in pack/unpack functions.
          */
         pLast = &(pData->opt_desc.desc[pData->opt_desc.used].end_loop);
-        pLast->common.type     = OPAL_DATATYPE_END_LOOP;
-        pLast->common.flags    = 0;
-        pLast->items           = pData->opt_desc.used;
+        pLast->common.type = OPAL_DATATYPE_END_LOOP;
+        pLast->common.flags = 0;
+        pLast->items = pData->opt_desc.used;
         pLast->first_elem_disp = first_elem_disp;
-        pLast->size            = pData->size;
+        pLast->size = pData->size;
     }
     return OPAL_SUCCESS;
 }
diff --git a/opal/datatype/opal_datatype_pack.c b/opal/datatype/opal_datatype_pack.c
index b5225017a59..3ada1860595 100644
--- a/opal/datatype/opal_datatype_pack.c
+++ b/opal/datatype/opal_datatype_pack.c
@@ -29,40 +29,40 @@
 #include "opal/datatype/opal_datatype_internal.h"
 
 #if OPAL_ENABLE_DEBUG
-#include "opal/util/output.h"
+#    include "opal/util/output.h"
 
-#define DO_DEBUG(INST)  if( opal_ddt_pack_debug ) { INST }
+#    define DO_DEBUG(INST)         \
+        if (opal_ddt_pack_debug) { \
+            INST                   \
+        }
 #else
-#define DO_DEBUG(INST)
-#endif  /* OPAL_ENABLE_DEBUG */
+#    define DO_DEBUG(INST)
+#endif /* OPAL_ENABLE_DEBUG */
 
 #include "opal/datatype/opal_datatype_checksum.h"
 #include "opal/datatype/opal_datatype_pack.h"
 #include "opal/datatype/opal_datatype_prototypes.h"
 
 #if defined(CHECKSUM)
-#define opal_pack_homogeneous_contig_function           opal_pack_homogeneous_contig_checksum
-#define opal_pack_homogeneous_contig_with_gaps_function opal_pack_homogeneous_contig_with_gaps_checksum
-#define opal_generic_simple_pack_function               opal_generic_simple_pack_checksum
-#define opal_pack_general_function                      opal_pack_general_checksum
+#    define opal_pack_homogeneous_contig_function opal_pack_homogeneous_contig_checksum
+#    define opal_pack_homogeneous_contig_with_gaps_function \
+        opal_pack_homogeneous_contig_with_gaps_checksum
+#    define opal_generic_simple_pack_function opal_generic_simple_pack_checksum
+#    define opal_pack_general_function        opal_pack_general_checksum
 #else
-#define opal_pack_homogeneous_contig_function           opal_pack_homogeneous_contig
-#define opal_pack_homogeneous_contig_with_gaps_function opal_pack_homogeneous_contig_with_gaps
-#define opal_generic_simple_pack_function               opal_generic_simple_pack
-#define opal_pack_general_function                      opal_pack_general
-#endif  /* defined(CHECKSUM) */
-
+#    define opal_pack_homogeneous_contig_function           opal_pack_homogeneous_contig
+#    define opal_pack_homogeneous_contig_with_gaps_function opal_pack_homogeneous_contig_with_gaps
+#    define opal_generic_simple_pack_function               opal_generic_simple_pack
+#    define opal_pack_general_function                      opal_pack_general
+#endif /* defined(CHECKSUM) */
 
 /* the contig versions does not use the stack. They can easily retrieve
- * the status with just the informations from pConvertor->bConverted.
+ * the status with just the information from pConvertor->bConverted.
  */
-int32_t
-opal_pack_homogeneous_contig_function( opal_convertor_t* pConv,
-                                       struct iovec* iov,
-                                       uint32_t* out_size,
-                                       size_t* max_data )
+int32_t opal_pack_homogeneous_contig_function(opal_convertor_t *pConv, struct iovec *iov,
+                                              uint32_t *out_size, size_t *max_data)
 {
-    dt_stack_t* pStack = pConv->pStack;
+    dt_stack_t *pStack = pConv->pStack;
     unsigned char *source_base = NULL;
     uint32_t iov_count;
     size_t length = pConv->local_size - pConv->bConverted, initial_amount = pConv->bConverted;
@@ -72,18 +72,21 @@ opal_pack_homogeneous_contig_function( opal_convertor_t* pConv,
     /* There are some optimizations that can be done if the upper level
      * does not provide a buffer.
      */
-    for( iov_count = 0; iov_count < (*out_size); iov_count++ ) {
-        if( 0 == length ) break;
-        if( (size_t)iov[iov_count].iov_len > length )
+    for (iov_count = 0; iov_count < (*out_size); iov_count++) {
+        if (0 == length) {
+            break;
+        }
+        if ((size_t) iov[iov_count].iov_len > length) {
             iov[iov_count].iov_len = length;
-        if( iov[iov_count].iov_base == NULL ) {
+        }
+        if (iov[iov_count].iov_base == NULL) {
             iov[iov_count].iov_base = (IOVBASE_TYPE *) source_base;
-            COMPUTE_CSUM( iov[iov_count].iov_base, iov[iov_count].iov_len, pConv );
+            COMPUTE_CSUM(iov[iov_count].iov_base, iov[iov_count].iov_len, pConv);
         } else {
             /* contiguous data just memcpy the smallest data in the user buffer */
-            OPAL_DATATYPE_SAFEGUARD_POINTER( source_base, iov[iov_count].iov_len,
-                                        pConv->pBaseBuf, pConv->pDesc, pConv->count );
-            MEMCPY_CSUM( iov[iov_count].iov_base, source_base, iov[iov_count].iov_len, pConv );
+            OPAL_DATATYPE_SAFEGUARD_POINTER(source_base, iov[iov_count].iov_len, pConv->pBaseBuf,
+                                            pConv->pDesc, pConv->count);
+            MEMCPY_CSUM(iov[iov_count].iov_base, source_base, iov[iov_count].iov_len, pConv);
         }
         length -= iov[iov_count].iov_len;
         pConv->bConverted += iov[iov_count].iov_len;
@@ -94,131 +97,140 @@ opal_pack_homogeneous_contig_function( opal_convertor_t* pConv,
     /* update the return value */
     *max_data = pConv->bConverted - initial_amount;
     *out_size = iov_count;
-    if( pConv->bConverted == pConv->local_size ) {
+    if (pConv->bConverted == pConv->local_size) {
         pConv->flags |= CONVERTOR_COMPLETED;
         return 1;
     }
     return 0;
 }
 
-
-int32_t
-opal_pack_homogeneous_contig_with_gaps_function( opal_convertor_t* pConv,
-                                                 struct iovec* iov,
-                                                 uint32_t* out_size,
-                                                 size_t* max_data )
+int32_t opal_pack_homogeneous_contig_with_gaps_function(opal_convertor_t *pConv, struct iovec *iov,
+                                                        uint32_t *out_size, size_t *max_data)
 {
     size_t remaining, length, initial_bytes_converted = pConv->bConverted;
-    const opal_datatype_t* pData = pConv->pDesc;
-    dt_stack_t* stack = pConv->pStack;
+    const opal_datatype_t *pData = pConv->pDesc;
+    dt_stack_t *stack = pConv->pStack;
     ptrdiff_t extent = pData->ub - pData->lb;
     unsigned char *user_memory, *packed_buffer;
     uint32_t idx;
     size_t i;
 
-    /* The memory layout is contiguous with gaps in the begining and at the end. The datatype true_lb
-     * is the initial displacement, the size the length of the contiguous area and the extent represent
-     * how much we should jump between elements.
+    /* The memory layout is contiguous with gaps in the beginning and at the end. The datatype
+     * true_lb is the initial displacement, the size the length of the contiguous area and the
+     * extent represent how much we should jump between elements.
      */
-    assert( (pData->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS) && ((ptrdiff_t)pData->size != extent) );
-    assert( pData->opt_desc.used <= 1 );
-    DO_DEBUG( opal_output( 0, "pack_homogeneous_contig( pBaseBuf %p, iov_count %d )\n",
-                           (void*)pConv->pBaseBuf, *out_size ); );
-    if( stack[1].type != opal_datatype_uint1.id ) {
+    assert((pData->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS) && ((ptrdiff_t) pData->size != extent));
+    assert(pData->opt_desc.used <= 1);
+    DO_DEBUG(opal_output(0, "pack_homogeneous_contig( pBaseBuf %p, iov_count %d )\n",
+                         (void *) pConv->pBaseBuf, *out_size););
+    if (stack[1].type != opal_datatype_uint1.id) {
         stack[1].count *= opal_datatype_basicDatatypes[stack[1].type]->size;
-        stack[1].type   = opal_datatype_uint1.id;
+        stack[1].type = opal_datatype_uint1.id;
     }
     /* We can provide directly the pointers in the user buffers (like the convertor_raw) */
-    if( NULL == iov[0].iov_base ) {
+    if (NULL == iov[0].iov_base) {
         user_memory = pConv->pBaseBuf + pData->true_lb;
 
-        for( idx = 0; (idx < (*out_size)) && stack[0].count; idx++ ) {
+        for (idx = 0; (idx < (*out_size)) && stack[0].count; idx++) {
             iov[idx].iov_base = user_memory + stack[0].disp + stack[1].disp;
-            iov[idx].iov_len  = stack[1].count;
-            COMPUTE_CSUM( iov[idx].iov_base, iov[idx].iov_len, pConv );
+            iov[idx].iov_len = stack[1].count;
+            COMPUTE_CSUM(iov[idx].iov_base, iov[idx].iov_len, pConv);
 
             pConv->bConverted += stack[1].count;
 
             stack[0].disp += extent;
             stack[0].count--;
-            stack[1].disp  = 0;
-            stack[1].count = pData->size;  /* we might need this to update the partial
-                                            * length for the first iteration */
+            stack[1].disp = 0;
+            stack[1].count = pData->size; /* we might need this to update the partial
+                                           * length for the first iteration */
         }
         goto update_status_and_return;
     }
 
-    for( idx = 0; idx < (*out_size); idx++ ) {
+    for (idx = 0; idx < (*out_size); idx++) {
         /* Limit the amount of packed data to the data left over on this convertor */
         remaining = pConv->local_size - pConv->bConverted;
-        if( 0 == remaining ) break;  /* we're done this time */
-        if( remaining > iov[idx].iov_len )
+        if (0 == remaining) {
+            break; /* we're done this time */
+        }
+        if (remaining > iov[idx].iov_len) {
             remaining = iov[idx].iov_len;
-        packed_buffer = (unsigned char *)iov[idx].iov_base;
+        }
+        packed_buffer = (unsigned char *) iov[idx].iov_base;
         pConv->bConverted += remaining;
         user_memory = pConv->pBaseBuf + pData->true_lb + stack[0].disp + stack[1].disp;
 
-        DO_DEBUG( opal_output( 0, "pack_homogeneous_contig( user_memory %p, packed_buffer %p length %" PRIsize_t "\n",
-                               (void*)user_memory, (void*)packed_buffer, remaining ); );
+        DO_DEBUG(opal_output(
+                     0,
+                     "pack_homogeneous_contig( user_memory %p, packed_buffer %p length %" PRIsize_t
+                     "\n",
+                     (void *) user_memory, (void *) packed_buffer, remaining););
 
-        length = (0 == pConv->stack_pos ? 0 : stack[1].count);  /* left over from the last pack */
+        length = (0 == pConv->stack_pos ? 0 : stack[1].count); /* left over from the last pack */
         /* data left from last round and enough space in the buffer */
-        if( (pData->size != length) && (length <= remaining)) {
+        if ((pData->size != length) && (length <= remaining)) {
             /* copy the partial left-over from the previous round */
-            OPAL_DATATYPE_SAFEGUARD_POINTER( user_memory, length, pConv->pBaseBuf,
-                                             pData, pConv->count );
-            DO_DEBUG( opal_output( 0, "pack dest %p src %p length %" PRIsize_t " [prologue]\n",
-                                   (void*)user_memory, (void*)packed_buffer, length ); );
-            MEMCPY_CSUM( packed_buffer, user_memory, length, pConv );
-            packed_buffer  += length;
-            remaining      -= length;
+            OPAL_DATATYPE_SAFEGUARD_POINTER(user_memory, length, pConv->pBaseBuf, pData,
+                                            pConv->count);
+            DO_DEBUG(opal_output(0, "pack dest %p src %p length %" PRIsize_t " [prologue]\n",
+                                 (void *) user_memory, (void *) packed_buffer, length););
+            MEMCPY_CSUM(packed_buffer, user_memory, length, pConv);
+            packed_buffer += length;
+            remaining -= length;
             stack[1].count -= length;
-            stack[1].disp  += length;  /* just in case, we overwrite this below */
-            if( 0 == stack[1].count) { /* one completed element */
+            stack[1].disp += length;   /* just in case, we overwrite this below */
+            if (0 == stack[1].count) { /* one completed element */
                 stack[0].count--;
                 stack[0].disp += extent;
-                if( 0 == stack[0].count )  /* not yet done */
+                if (0 == stack[0].count) { /* not yet done */
                     break;
+                }
                 stack[1].count = pData->size;
                 stack[1].disp = 0;
             }
             user_memory = pConv->pBaseBuf + pData->true_lb + stack[0].disp + stack[1].disp;
         }
 
-        for( i = 0; pData->size <= remaining; i++ ) {
-            OPAL_DATATYPE_SAFEGUARD_POINTER( user_memory, pData->size, pConv->pBaseBuf,
-                                             pData, pConv->count );
-            DO_DEBUG( opal_output( 0, "pack dest %p src %p length %" PRIsize_t " [%" PRIsize_t "/%" PRIsize_t "\n",
-                                   (void*)user_memory, (void*)packed_buffer, pData->size, remaining, iov[idx].iov_len ); );
-            MEMCPY_CSUM( packed_buffer, user_memory, pData->size, pConv );
+        for (i = 0; pData->size <= remaining; i++) {
+            OPAL_DATATYPE_SAFEGUARD_POINTER(user_memory, pData->size, pConv->pBaseBuf, pData,
+                                            pConv->count);
+            DO_DEBUG(opal_output(0,
+                                 "pack dest %p src %p length %" PRIsize_t " [%" PRIsize_t
+                                 "/%" PRIsize_t "\n",
+                                 (void *) user_memory, (void *) packed_buffer, pData->size,
+                                 remaining, iov[idx].iov_len););
+            MEMCPY_CSUM(packed_buffer, user_memory, pData->size, pConv);
             packed_buffer += pData->size;
-            user_memory   += extent;
-            remaining     -= pData->size;
+            user_memory += extent;
+            remaining -= pData->size;
         }
-        stack[0].count -= i;  /* the entire datatype copied above */
-        stack[0].disp  += (i * extent);
+        stack[0].count -= i; /* the entire datatype copied above */
+        stack[0].disp += (i * extent);
 
         /* Copy the last bits */
-        if( 0 != remaining ) {
-            OPAL_DATATYPE_SAFEGUARD_POINTER( user_memory, remaining, pConv->pBaseBuf,
-                                             pData, pConv->count );
-            DO_DEBUG( opal_output( 0, "4. pack dest %p src %p length %" PRIsize_t "\n",
-                                   (void*)user_memory, (void*)packed_buffer, remaining ); );
-            MEMCPY_CSUM( packed_buffer, user_memory, remaining, pConv );
+        if (0 != remaining) {
+            OPAL_DATATYPE_SAFEGUARD_POINTER(user_memory, remaining, pConv->pBaseBuf, pData,
+                                            pConv->count);
+            DO_DEBUG(opal_output(0, "4. pack dest %p src %p length %" PRIsize_t "\n",
+                                 (void *) user_memory, (void *) packed_buffer, remaining););
+            MEMCPY_CSUM(packed_buffer, user_memory, remaining, pConv);
             stack[1].count -= remaining;
-            stack[1].disp  += remaining;  /* keep the += in case we are copying less that the datatype size */
-            if( 0 == stack[1].count ) {  /* prepare for the next element */
+            stack[1].disp += remaining; /* keep the += in case we are copying less that the datatype
+                                           size */
+            if (0 == stack[1].count) {  /* prepare for the next element */
                 stack[1].count = pData->size;
-                stack[1].disp  = 0;
+                stack[1].disp = 0;
             }
         }
     }
 
- update_status_and_return:
+update_status_and_return:
     *out_size = idx;
     *max_data = pConv->bConverted - initial_bytes_converted;
-    if( pConv->bConverted == pConv->local_size ) pConv->flags |= CONVERTOR_COMPLETED;
-    return !!(pConv->flags & CONVERTOR_COMPLETED);  /* done or not */
+    if (pConv->bConverted == pConv->local_size) {
+        pConv->flags |= CONVERTOR_COMPLETED;
+    }
+    return !!(pConv->flags & CONVERTOR_COMPLETED); /* done or not */
 }
 
 /* The pack/unpack functions need a cleanup. I have to create a proper interface to access
@@ -227,29 +239,27 @@ opal_pack_homogeneous_contig_with_gaps_function( opal_convertor_t* pConv,
  * But first let's make some global assumptions:
  * - a datatype (with the flag DT_DATA set) will have the contiguous flags set if and only if
  *   the data is really contiguous (extent equal with size)
- * - for the OPAL_DATATYPE_LOOP type the DT_CONTIGUOUS flag set means that the content of the loop is
- *   contiguous but with a gap in the begining or at the end.
+ * - for the OPAL_DATATYPE_LOOP type the DT_CONTIGUOUS flag set means that the content of the loop
+ * is contiguous but with a gap in the beginning or at the end.
  * - the DT_CONTIGUOUS flag for the type OPAL_DATATYPE_END_LOOP is meaningless.
  */
-int32_t
-opal_generic_simple_pack_function( opal_convertor_t* pConvertor,
-                                   struct iovec* iov, uint32_t* out_size,
-                                   size_t* max_data )
+int32_t opal_generic_simple_pack_function(opal_convertor_t *pConvertor, struct iovec *iov,
+                                          uint32_t *out_size, size_t *max_data)
 {
-    dt_stack_t* pStack;       /* pointer to the position on the stack */
-    uint32_t pos_desc;        /* actual position in the description of the derived datatype */
-    size_t count_desc;        /* the number of items already done in the actual pos_desc */
-    size_t total_packed = 0;  /* total amount packed this time */
-    dt_elem_desc_t* description;
-    dt_elem_desc_t* pElem;
+    dt_stack_t *pStack;      /* pointer to the position on the stack */
+    uint32_t pos_desc;       /* actual position in the description of the derived datatype */
+    size_t count_desc;       /* the number of items already done in the actual pos_desc */
+    size_t total_packed = 0; /* total amount packed this time */
+    dt_elem_desc_t *description;
+    dt_elem_desc_t *pElem;
     const opal_datatype_t *pData = pConvertor->pDesc;
     unsigned char *conv_ptr, *iov_ptr;
     size_t iov_len_local;
     uint32_t iov_count;
 
-    DO_DEBUG( opal_output( 0, "opal_convertor_generic_simple_pack( %p:%p, {%p, %lu}, %d )\n",
-                           (void*)pConvertor, (void*)pConvertor->pBaseBuf,
-                           (void*)iov[0].iov_base, (unsigned long)iov[0].iov_len, *out_size ); );
+    DO_DEBUG(opal_output(0, "opal_convertor_generic_simple_pack( %p:%p, {%p, %lu}, %d )\n",
+                         (void *) pConvertor, (void *) pConvertor->pBaseBuf,
+                         (void *) iov[0].iov_base, (unsigned long) iov[0].iov_len, *out_size););
 
     description = pConvertor->use_desc->desc;
 
@@ -258,114 +268,124 @@ opal_generic_simple_pack_function( opal_convertor_t* pConvertor,
      * due to the fact that the convertor can stop in the middle of a data with a count
      */
     pStack = pConvertor->pStack + pConvertor->stack_pos;
-    pos_desc   = pStack->index;
-    conv_ptr   = pConvertor->pBaseBuf + pStack->disp;
+    pos_desc = pStack->index;
+    conv_ptr = pConvertor->pBaseBuf + pStack->disp;
     count_desc = pStack->count;
     pStack--;
     pConvertor->stack_pos--;
     pElem = &(description[pos_desc]);
 
-    DO_DEBUG( opal_output( 0, "pack start pos_desc %d count_desc %" PRIsize_t " disp %ld\n"
-                           "stack_pos %d pos_desc %d count_desc %" PRIsize_t " disp %ld\n",
-                           pos_desc, count_desc, (long)(conv_ptr - pConvertor->pBaseBuf),
-                           pConvertor->stack_pos, pStack->index, pStack->count, pStack->disp ); );
+    DO_DEBUG(opal_output(0,
+                         "pack start pos_desc %d count_desc %" PRIsize_t " disp %ld\n"
+                         "stack_pos %d pos_desc %d count_desc %" PRIsize_t " disp %ld\n",
+                         pos_desc, count_desc, (long) (conv_ptr - pConvertor->pBaseBuf),
+                         pConvertor->stack_pos, pStack->index, pStack->count, pStack->disp););
 
-    for( iov_count = 0; iov_count < (*out_size); iov_count++ ) {
+    for (iov_count = 0; iov_count < (*out_size); iov_count++) {
         iov_ptr = (unsigned char *) iov[iov_count].iov_base;
         iov_len_local = iov[iov_count].iov_len;
 
-        if( pElem->elem.common.flags & OPAL_DATATYPE_FLAG_DATA ) {
-            if( ((size_t)pElem->elem.count * pElem->elem.blocklen) != count_desc ) {
+        if (pElem->elem.common.flags & OPAL_DATATYPE_FLAG_DATA) {
+            if (((size_t) pElem->elem.count * pElem->elem.blocklen) != count_desc) {
                 /* we have a partial (less than blocklen) basic datatype */
-                int rc = PACK_PARTIAL_BLOCKLEN( pConvertor, pElem, count_desc,
-                                                conv_ptr, iov_ptr, iov_len_local );
-                if( 0 == rc )  /* not done */
+                int rc = PACK_PARTIAL_BLOCKLEN(pConvertor, pElem, count_desc, conv_ptr, iov_ptr,
+                                               iov_len_local);
+                if (0 == rc) { /* not done */
                     goto complete_loop;
-                if( 0 == count_desc ) {
+                }
+                if (0 == count_desc) {
                     conv_ptr = pConvertor->pBaseBuf + pStack->disp;
-                    pos_desc++;  /* advance to the next data */
-                    UPDATE_INTERNAL_COUNTERS( description, pos_desc, pElem, count_desc );
+                    pos_desc++; /* advance to the next data */
+                    UPDATE_INTERNAL_COUNTERS(description, pos_desc, pElem, count_desc);
                 }
             }
         }
 
-        while( 1 ) {
-            while( pElem->elem.common.flags & OPAL_DATATYPE_FLAG_DATA ) {
+        while (1) {
+            while (pElem->elem.common.flags & OPAL_DATATYPE_FLAG_DATA) {
                 /* we have a basic datatype (working on full blocks) */
-                PACK_PREDEFINED_DATATYPE( pConvertor, pElem, count_desc,
-                                          conv_ptr, iov_ptr, iov_len_local );
-                if( 0 != count_desc )  /* completed? */
+                PACK_PREDEFINED_DATATYPE(pConvertor, pElem, count_desc, conv_ptr, iov_ptr,
+                                         iov_len_local);
+                if (0 != count_desc) { /* completed? */
                     goto complete_loop;
+                }
                 conv_ptr = pConvertor->pBaseBuf + pStack->disp;
-                pos_desc++;  /* advance to the next data */
-                UPDATE_INTERNAL_COUNTERS( description, pos_desc, pElem, count_desc );
+                pos_desc++; /* advance to the next data */
+                UPDATE_INTERNAL_COUNTERS(description, pos_desc, pElem, count_desc);
             }
-            if( OPAL_DATATYPE_END_LOOP == pElem->elem.common.type ) { /* end of the current loop */
-                DO_DEBUG( opal_output( 0, "pack end_loop count %" PRIsize_t " stack_pos %d"
-                                       " pos_desc %d disp %ld space %lu\n",
-                                       pStack->count, pConvertor->stack_pos,
-                                       pos_desc, pStack->disp, (unsigned long)iov_len_local ); );
-                if( --(pStack->count) == 0 ) { /* end of loop */
-                    if( 0 == pConvertor->stack_pos ) {
+            if (OPAL_DATATYPE_END_LOOP == pElem->elem.common.type) { /* end of the current loop */
+                DO_DEBUG(opal_output(0,
+                                     "pack end_loop count %" PRIsize_t " stack_pos %d"
+                                     " pos_desc %d disp %ld space %lu\n",
+                                     pStack->count, pConvertor->stack_pos, pos_desc, pStack->disp,
+                                     (unsigned long) iov_len_local););
+                if (--(pStack->count) == 0) { /* end of loop */
+                    if (0 == pConvertor->stack_pos) {
                         /* we're done. Force the exit of the main for loop (around iovec) */
                         *out_size = iov_count;
                         goto complete_loop;
                     }
-                    pConvertor->stack_pos--;  /* go one position up on the stack */
+                    pConvertor->stack_pos--; /* go one position up on the stack */
                     pStack--;
-                    pos_desc++;  /* and move to the next element */
+                    pos_desc++; /* and move to the next element */
                 } else {
-                    pos_desc = pStack->index + 1;  /* jump back to the begining of the loop */
-                    if( pStack->index == -1 ) {  /* If it's the datatype count loop */
-                        pStack->disp += (pData->ub - pData->lb);  /* jump by the datatype extent */
+                    pos_desc = pStack->index + 1; /* jump back to the beginning of the loop */
+                    if (pStack->index == -1) {    /* If it's the datatype count loop */
+                        pStack->disp += (pData->ub - pData->lb); /* jump by the datatype extent */
                     } else {
-                        assert( OPAL_DATATYPE_LOOP == description[pStack->index].loop.common.type );
-                        pStack->disp += description[pStack->index].loop.extent;  /* jump by the loop extent */
+                        assert(OPAL_DATATYPE_LOOP == description[pStack->index].loop.common.type);
+                        pStack->disp += description[pStack->index]
+                                            .loop.extent; /* jump by the loop extent */
                     }
                 }
                 conv_ptr = pConvertor->pBaseBuf + pStack->disp;
-                UPDATE_INTERNAL_COUNTERS( description, pos_desc, pElem, count_desc );
-                DO_DEBUG( opal_output( 0, "pack new_loop count %" PRIsize_t " stack_pos %d pos_desc %d count_desc %" PRIsize_t " disp %ld space %lu\n",
-                                       pStack->count, pConvertor->stack_pos, pos_desc,
-                                       count_desc, pStack->disp, (unsigned long)iov_len_local ); );
+                UPDATE_INTERNAL_COUNTERS(description, pos_desc, pElem, count_desc);
+                DO_DEBUG(opal_output(0,
+                                     "pack new_loop count %" PRIsize_t
+                                     " stack_pos %d pos_desc %d count_desc %" PRIsize_t
+                                     " disp %ld space %lu\n",
+                                     pStack->count, pConvertor->stack_pos, pos_desc, count_desc,
+                                     pStack->disp, (unsigned long) iov_len_local););
             }
-            if( OPAL_DATATYPE_LOOP == pElem->elem.common.type ) {
-                ptrdiff_t local_disp = (ptrdiff_t)conv_ptr;
-                if( pElem->loop.common.flags & OPAL_DATATYPE_FLAG_CONTIGUOUS ) {
-                    PACK_CONTIGUOUS_LOOP( pConvertor, pElem, count_desc,
-                                          conv_ptr, iov_ptr, iov_len_local );
-                    if( 0 == count_desc ) {  /* completed */
+            if (OPAL_DATATYPE_LOOP == pElem->elem.common.type) {
+                ptrdiff_t local_disp = (ptrdiff_t) conv_ptr;
+                if (pElem->loop.common.flags & OPAL_DATATYPE_FLAG_CONTIGUOUS) {
+                    PACK_CONTIGUOUS_LOOP(pConvertor, pElem, count_desc, conv_ptr, iov_ptr,
+                                         iov_len_local);
+                    if (0 == count_desc) { /* completed */
                         pos_desc += pElem->loop.items + 1;
                         goto update_loop_description;
                     }
                     /* Save the stack with the correct last_count value. */
                 }
-                local_disp = (ptrdiff_t)conv_ptr - local_disp;
-                PUSH_STACK( pStack, pConvertor->stack_pos, pos_desc, OPAL_DATATYPE_LOOP, count_desc,
-                            pStack->disp + local_disp);
+                local_disp = (ptrdiff_t) conv_ptr - local_disp;
+                PUSH_STACK(pStack, pConvertor->stack_pos, pos_desc, OPAL_DATATYPE_LOOP, count_desc,
+                           pStack->disp + local_disp);
                 pos_desc++;
-            update_loop_description:  /* update the current state */
+            update_loop_description: /* update the current state */
                 conv_ptr = pConvertor->pBaseBuf + pStack->disp;
-                UPDATE_INTERNAL_COUNTERS( description, pos_desc, pElem, count_desc );
-                DDT_DUMP_STACK( pConvertor->pStack, pConvertor->stack_pos, pElem, "advance loop" );
+                UPDATE_INTERNAL_COUNTERS(description, pos_desc, pElem, count_desc);
+                DDT_DUMP_STACK(pConvertor->pStack, pConvertor->stack_pos, pElem, "advance loop");
             }
         }
     complete_loop:
-        iov[iov_count].iov_len -= iov_len_local;  /* update the amount of valid data */
+        iov[iov_count].iov_len -= iov_len_local; /* update the amount of valid data */
         total_packed += iov[iov_count].iov_len;
     }
     *max_data = total_packed;
-    pConvertor->bConverted += total_packed;  /* update the already converted bytes */
+    pConvertor->bConverted += total_packed; /* update the already converted bytes */
     *out_size = iov_count;
-    if( pConvertor->bConverted == pConvertor->local_size ) {
+    if (pConvertor->bConverted == pConvertor->remote_size) {
         pConvertor->flags |= CONVERTOR_COMPLETED;
         return 1;
     }
     /* Save the global position for the next round */
-    PUSH_STACK( pStack, pConvertor->stack_pos, pos_desc, pElem->elem.common.type, count_desc,
-                conv_ptr - pConvertor->pBaseBuf );
-    DO_DEBUG( opal_output( 0, "pack save stack stack_pos %d pos_desc %d count_desc %" PRIsize_t " disp %ld\n",
-                           pConvertor->stack_pos, pStack->index, pStack->count, pStack->disp ); );
+    PUSH_STACK(pStack, pConvertor->stack_pos, pos_desc, pElem->elem.common.type, count_desc,
+               conv_ptr - pConvertor->pBaseBuf);
+    DO_DEBUG(opal_output(0,
+                         "pack save stack stack_pos %d pos_desc %d count_desc %" PRIsize_t
+                         " disp %ld\n",
+                         pConvertor->stack_pos, pStack->index, pStack->count, pStack->disp););
     return 0;
 }
 
@@ -378,68 +398,107 @@ opal_generic_simple_pack_function( opal_convertor_t* pConvertor,
  * to a contiguous output buffer with a predefined size.
  * return OPAL_SUCCESS if everything went OK and if there is still room before the complete
  *          conversion of the data (need additional call with others input buffers )
- *        1 if everything went fine and the data was completly converted
+ *        1 if everything went fine and the data was completely converted
  *       -1 something wrong occurs.
  */
 
 static inline void
-pack_predefined_heterogeneous( opal_convertor_t* CONVERTOR,
-                               const dt_elem_desc_t* ELEM,
-                               size_t* COUNT,
-                               unsigned char** SOURCE,
-                               unsigned char** DESTINATION,
-                               size_t* SPACE )
+pack_predefined_heterogeneous(opal_convertor_t *CONVERTOR,
+                              const dt_elem_desc_t *ELEM, size_t *COUNT,
+                              unsigned char **memory,
+                              unsigned char **packed, size_t *SPACE)
 {
-    const opal_convertor_master_t* master = (CONVERTOR)->master;
-    const ddt_elem_desc_t* _elem = &((ELEM)->elem);
-    unsigned char* _source = (*SOURCE) + _elem->disp;
-    ptrdiff_t advance;
-    size_t _count = *(COUNT);
-    size_t _r_blength;
-
-    _r_blength = master->remote_sizes[_elem->common.type];
-    if( (_count * _r_blength) > *(SPACE) ) {
-        _count = (*(SPACE) / _r_blength);
-        if( 0 == _count ) return;  /* nothing to do */
+    const opal_convertor_master_t *master = (CONVERTOR)->master;
+    const ddt_elem_desc_t *_elem = &((ELEM)->elem);
+    size_t cando_count = *(COUNT), do_now_bytes;
+    size_t local_elem_size = opal_datatype_basicDatatypes[_elem->common.type]->size;
+    size_t remote_elem_size = master->remote_sizes[_elem->common.type];
+    size_t blocklen_bytes = remote_elem_size;
+    unsigned char *_memory = (*memory) + _elem->disp;
+    unsigned char *_packed = *packed;
+    ptrdiff_t advance = 0;
+
+    assert(0 == (cando_count % _elem->blocklen)); /* no partials here */
+    assert(*(COUNT) <= ((size_t) _elem->count * _elem->blocklen));
+
+    if ((remote_elem_size * cando_count) > *(SPACE))
+        cando_count = (*SPACE) / blocklen_bytes;
+
+    /* preemptively update the number of COUNT we will return. */
+    *(COUNT) -= cando_count;
+
+    if (_elem->blocklen == 1) {
+        master->pFunctions[_elem->common.type](CONVERTOR, cando_count,
+                                               _memory, *SPACE, _elem->extent,
+                                               _packed, *SPACE, remote_elem_size,
+                                               &advance);
+        _memory += cando_count * _elem->extent;
+        _packed += cando_count * remote_elem_size;
+        goto update_and_return;
+    }
+
+    if ((1 < _elem->count) && (_elem->blocklen <= cando_count)) {
+        blocklen_bytes = remote_elem_size * _elem->blocklen;
+
+        do { /* Do as many full blocklen as possible */
+            OPAL_DATATYPE_SAFEGUARD_POINTER(_memory, blocklen_bytes, (CONVERTOR)->pBaseBuf,
+                                            (CONVERTOR)->pDesc, (CONVERTOR)->count);
+            DO_DEBUG(opal_output(0, "pack 2. memcpy( %p, %p, %lu ) => space %lu\n",
+                                 (void *) _packed, (void *) _memory, (unsigned long) blocklen_bytes,
+                                 (unsigned long) (*(SPACE) - (_packed - *(packed)))););
+            master->pFunctions[_elem->common.type](CONVERTOR, _elem->blocklen,
+                                                   _memory, *SPACE, local_elem_size,
+                                                   _packed, *SPACE, remote_elem_size,
+                                                   &advance);
+            _packed += blocklen_bytes;
+            _memory += _elem->extent;
+            cando_count -= _elem->blocklen;
+        } while (_elem->blocklen <= cando_count);
+    }
+
+    /**
+     * As an epilog do anything left from the last blocklen.
+     */
+    if (0 != cando_count) {
+        assert((cando_count < _elem->blocklen)
+               || ((1 == _elem->count) && (cando_count <= _elem->blocklen)));
+        do_now_bytes = cando_count * remote_elem_size;
+        OPAL_DATATYPE_SAFEGUARD_POINTER(_memory, do_now_bytes, (CONVERTOR)->pBaseBuf,
+                                        (CONVERTOR)->pDesc, (CONVERTOR)->count);
+        DO_DEBUG(opal_output(0, "pack 3. memcpy( %p, %p, %lu ) => space %lu [epilog]\n",
+                             (void *) _packed, (void *) _memory, (unsigned long) do_now_bytes,
+                             (unsigned long) (*(SPACE) - (_packed - *(packed)))););
+        master->pFunctions[_elem->common.type](CONVERTOR, cando_count,
+                                               _memory, *SPACE, local_elem_size,
+                                               _packed, *SPACE, remote_elem_size,
+                                               &advance);
+        _memory += cando_count * local_elem_size;
+        _packed += do_now_bytes;
     }
 
-    OPAL_DATATYPE_SAFEGUARD_POINTER( _source, (_count * _elem->extent), (CONVERTOR)->pBaseBuf,
-                                     (CONVERTOR)->pDesc, (CONVERTOR)->count );
-    DO_DEBUG( opal_output( 0, "pack [l %s r %s] memcpy( %p, %p, %lu ) => space %lu\n",
-                           ((ptrdiff_t)(opal_datatype_basicDatatypes[_elem->common.type]->size) == _elem->extent) ? "cont" : "----",
-                           ((ptrdiff_t)_r_blength == _elem->extent) ? "cont" : "----",
-                           (void*)*(DESTINATION), (void*)_source, (unsigned long)_r_blength,
-                           (unsigned long)(*(SPACE)) ); );
-    master->pFunctions[_elem->common.type]( CONVERTOR, _count,
-                                            _source, *SPACE, _elem->extent,
-                                            *DESTINATION, *SPACE, _r_blength,
-                                            &advance );
-    _r_blength     *= _count;  /* update the remote length to encompass all the elements */
-    *(SOURCE)      += _count * _elem->extent;
-    *(DESTINATION) += _r_blength;
-    *(SPACE)       -= _r_blength;
-    *(COUNT)       -= _count;
+update_and_return:
+    *(memory) = _memory - _elem->disp;
+    *(SPACE) -= (_packed - *packed);
+    *(packed) = _packed;
 }
 
-int32_t
-opal_pack_general_function( opal_convertor_t* pConvertor,
-                            struct iovec* iov, uint32_t* out_size,
-                            size_t* max_data )
+int32_t opal_pack_general_function(opal_convertor_t *pConvertor, struct iovec *iov,
+                                   uint32_t *out_size, size_t *max_data)
 {
-    dt_stack_t* pStack;       /* pointer to the position on the stack */
-    uint32_t pos_desc;        /* actual position in the description of the derived datatype */
-    size_t count_desc;      /* the number of items already done in the actual pos_desc */
-    size_t total_packed = 0;  /* total amount packed this time */
-    dt_elem_desc_t* description;
-    dt_elem_desc_t* pElem;
+    dt_stack_t *pStack;      /* pointer to the position on the stack */
+    uint32_t pos_desc;       /* actual position in the description of the derived datatype */
+    size_t count_desc;       /* the number of items already done in the actual pos_desc */
+    size_t total_packed = 0; /* total amount packed this time */
+    dt_elem_desc_t *description;
+    dt_elem_desc_t *pElem;
     const opal_datatype_t *pData = pConvertor->pDesc;
     unsigned char *conv_ptr, *iov_ptr;
     size_t iov_len_local;
     uint32_t iov_count;
 
-    DO_DEBUG( opal_output( 0, "opal_convertor_general_pack( %p:%p, {%p, %lu}, %d )\n",
-                           (void*)pConvertor, (void*)pConvertor->pBaseBuf,
-                           (void*)iov[0].iov_base, (unsigned long)iov[0].iov_len, *out_size ); );
+    DO_DEBUG(opal_output(0, "opal_convertor_general_pack( %p:%p, {%p, %lu}, %d )\n",
+                         (void *) pConvertor, (void *) pConvertor->pBaseBuf,
+                         (void *) iov[0].iov_base, (unsigned long) iov[0].iov_len, *out_size););
 
     description = pConvertor->use_desc->desc;
 
@@ -448,77 +507,83 @@ opal_pack_general_function( opal_convertor_t* pConvertor,
      * due to the fact that the convertor can stop in the middle of a data with a count
      */
     pStack = pConvertor->pStack + pConvertor->stack_pos;
-    pos_desc   = pStack->index;
-    conv_ptr   = pConvertor->pBaseBuf + pStack->disp;
+    pos_desc = pStack->index;
+    conv_ptr = pConvertor->pBaseBuf + pStack->disp;
     count_desc = pStack->count;
     pStack--;
     pConvertor->stack_pos--;
     pElem = &(description[pos_desc]);
 
-    DO_DEBUG( opal_output( 0, "pack start pos_desc %d count_desc %" PRIsize_t " disp %ld\n"
-                           "stack_pos %d pos_desc %d count_desc %" PRIsize_t " disp %ld\n",
-                           pos_desc, count_desc, (long)(conv_ptr - pConvertor->pBaseBuf),
-                           pConvertor->stack_pos, pStack->index, pStack->count, pStack->disp ); );
+    DO_DEBUG(opal_output(0,
+                         "pack start pos_desc %d count_desc %" PRIsize_t " disp %ld\n"
+                         "stack_pos %d pos_desc %d count_desc %" PRIsize_t " disp %ld\n",
+                         pos_desc, count_desc, (long) (conv_ptr - pConvertor->pBaseBuf),
+                         pConvertor->stack_pos, pStack->index, pStack->count, pStack->disp););
 
-    for( iov_count = 0; iov_count < (*out_size); iov_count++ ) {
+    for (iov_count = 0; iov_count < (*out_size); iov_count++) {
         iov_ptr = (unsigned char *) iov[iov_count].iov_base;
         iov_len_local = iov[iov_count].iov_len;
-        while( 1 ) {
-            while( pElem->elem.common.flags & OPAL_DATATYPE_FLAG_DATA ) {
+        while (1) {
+            while (pElem->elem.common.flags & OPAL_DATATYPE_FLAG_DATA) {
                 /* now here we have a basic datatype */
-                DO_DEBUG( opal_output( 0, "pack (%p:%ld, %" PRIsize_t ", %ld) -> (%p, %ld) type %s\n",
-                                       (void*)pConvertor->pBaseBuf, conv_ptr + pElem->elem.disp - pConvertor->pBaseBuf,
-                                       count_desc, description[pos_desc].elem.extent,
-                                       (void*)iov_ptr, iov_len_local,
-                                       opal_datatype_basicDatatypes[pElem->elem.common.type]->name ); );
-
-                pack_predefined_heterogeneous( pConvertor, pElem, &count_desc,
-                                               &conv_ptr, &iov_ptr, &iov_len_local);
+                DO_DEBUG(opal_output(0, "pack (%p:%ld, %" PRIsize_t ", %ld) -> (%p, %ld) type %s\n",
+                                     (void *) pConvertor->pBaseBuf,
+                                     conv_ptr + pElem->elem.disp - pConvertor->pBaseBuf, count_desc,
+                                     description[pos_desc].elem.extent, (void *) iov_ptr,
+                                     iov_len_local,
+                                     opal_datatype_basicDatatypes[pElem->elem.common.type]->name););
+
+                pack_predefined_heterogeneous(pConvertor, pElem, &count_desc, &conv_ptr, &iov_ptr,
+                                              &iov_len_local);
 #if 0
                 PACK_PREDEFINED_DATATYPE( pConvertor, pElem, count_desc,
                                           conv_ptr, iov_ptr, iov_len_local );
 #endif
-                if( 0 == count_desc ) {  /* completed */
+                if (0 == count_desc) { /* completed */
                     conv_ptr = pConvertor->pBaseBuf + pStack->disp;
-                    pos_desc++;  /* advance to the next data */
-                    UPDATE_INTERNAL_COUNTERS( description, pos_desc, pElem, count_desc );
+                    pos_desc++; /* advance to the next data */
+                    UPDATE_INTERNAL_COUNTERS(description, pos_desc, pElem, count_desc);
                     continue;
                 }
                 goto complete_loop;
             }
-            if( OPAL_DATATYPE_END_LOOP == pElem->elem.common.type ) { /* end of the current loop */
-                DO_DEBUG( opal_output( 0, "pack end_loop count %" PRIsize_t " stack_pos %d"
-                                       " pos_desc %d disp %ld space %lu\n",
-                                       pStack->count, pConvertor->stack_pos,
-                                       pos_desc, pStack->disp, (unsigned long)iov_len_local ); );
-                if( --(pStack->count) == 0 ) { /* end of loop */
-                    if( 0 == pConvertor->stack_pos ) {
+            if (OPAL_DATATYPE_END_LOOP == pElem->elem.common.type) { /* end of the current loop */
+                DO_DEBUG(opal_output(0,
+                                     "pack end_loop count %" PRIsize_t " stack_pos %d"
+                                     " pos_desc %d disp %ld space %lu\n",
+                                     pStack->count, pConvertor->stack_pos, pos_desc, pStack->disp,
+                                     (unsigned long) iov_len_local););
+                if (--(pStack->count) == 0) { /* end of loop */
+                    if (0 == pConvertor->stack_pos) {
                         /* we lie about the size of the next element in order to
                          * make sure we exit the main loop.
                          */
                         *out_size = iov_count;
-                        goto complete_loop;  /* completed */
+                        goto complete_loop; /* completed */
                     }
                     pConvertor->stack_pos--;
                     pStack--;
                     pos_desc++;
                 } else {
                     pos_desc = pStack->index + 1;
-                    if( pStack->index == -1 ) {
+                    if (pStack->index == -1) {
                         pStack->disp += (pData->ub - pData->lb);
                     } else {
-                        assert( OPAL_DATATYPE_LOOP == description[pStack->index].loop.common.type );
+                        assert(OPAL_DATATYPE_LOOP == description[pStack->index].loop.common.type);
                         pStack->disp += description[pStack->index].loop.extent;
                     }
                 }
                 conv_ptr = pConvertor->pBaseBuf + pStack->disp;
-                UPDATE_INTERNAL_COUNTERS( description, pos_desc, pElem, count_desc );
-                DO_DEBUG( opal_output( 0, "pack new_loop count %" PRIsize_t " stack_pos %d pos_desc %d count_desc %" PRIsize_t " disp %ld space %lu\n",
-                                       pStack->count, pConvertor->stack_pos, pos_desc,
-                                       count_desc, pStack->disp, (unsigned long)iov_len_local ); );
+                UPDATE_INTERNAL_COUNTERS(description, pos_desc, pElem, count_desc);
+                DO_DEBUG(opal_output(0,
+                                     "pack new_loop count %" PRIsize_t
+                                     " stack_pos %d pos_desc %d count_desc %" PRIsize_t
+                                     " disp %ld space %lu\n",
+                                     pStack->count, pConvertor->stack_pos, pos_desc, count_desc,
+                                     pStack->disp, (unsigned long) iov_len_local););
             }
-            if( OPAL_DATATYPE_LOOP == pElem->elem.common.type ) {
-                ptrdiff_t local_disp = (ptrdiff_t)conv_ptr;
+            if (OPAL_DATATYPE_LOOP == pElem->elem.common.type) {
+                ptrdiff_t local_disp = (ptrdiff_t) conv_ptr;
 #if 0
                 if( pElem->loop.common.flags & OPAL_DATATYPE_FLAG_CONTIGUOUS ) {
                     PACK_CONTIGUOUS_LOOP( pConvertor, pElem, count_desc,
@@ -529,35 +594,39 @@ opal_pack_general_function( opal_convertor_t* pConvertor,
                     }
                     /* Save the stack with the correct last_count value. */
                 }
-#endif  /* in a heterogeneous environment we can't handle the contiguous loops */
-                local_disp = (ptrdiff_t)conv_ptr - local_disp;
-                PUSH_STACK( pStack, pConvertor->stack_pos, pos_desc, OPAL_DATATYPE_LOOP, count_desc,
-                            pStack->disp + local_disp);
+#endif /* in a heterogeneous environment we can't handle the contiguous loops */
+                local_disp = (ptrdiff_t) conv_ptr - local_disp;
+                PUSH_STACK(pStack, pConvertor->stack_pos, pos_desc, OPAL_DATATYPE_LOOP, count_desc,
+                           pStack->disp + local_disp);
                 pos_desc++;
 #if 0
             update_loop_description:  /* update the current state */
-#endif  /* in a heterogeneous environment we can't handle the contiguous loops */
+#endif /* in a heterogeneous environment we can't handle the contiguous loops */
                 conv_ptr = pConvertor->pBaseBuf + pStack->disp;
-                UPDATE_INTERNAL_COUNTERS( description, pos_desc, pElem, count_desc );
-                DDT_DUMP_STACK( pConvertor->pStack, pConvertor->stack_pos, pElem, "advance loop" );
+                UPDATE_INTERNAL_COUNTERS(description, pos_desc, pElem, count_desc);
+                DDT_DUMP_STACK(pConvertor->pStack, pConvertor->stack_pos, pElem, "advance loop");
                 continue;
             }
         }
     complete_loop:
-        iov[iov_count].iov_len -= iov_len_local;  /* update the amount of valid data */
+        iov[iov_count].iov_len -= iov_len_local; /* update the amount of valid data */
         total_packed += iov[iov_count].iov_len;
     }
     *max_data = total_packed;
-    pConvertor->bConverted += total_packed;  /* update the already converted bytes */
+    pConvertor->bConverted += total_packed; /* update the already converted bytes */
     *out_size = iov_count;
-    if( pConvertor->bConverted == pConvertor->local_size ) {
+    size_t expected_packed_size;
+    opal_convertor_get_packed_size(pConvertor, &expected_packed_size);
+    if (pConvertor->bConverted == expected_packed_size) {
         pConvertor->flags |= CONVERTOR_COMPLETED;
         return 1;
     }
     /* Save the global position for the next round */
-    PUSH_STACK( pStack, pConvertor->stack_pos, pos_desc, pElem->elem.common.type, count_desc,
-                conv_ptr - pConvertor->pBaseBuf );
-    DO_DEBUG( opal_output( 0, "pack save stack stack_pos %d pos_desc %d count_desc %" PRIsize_t" disp %ld\n",
-                           pConvertor->stack_pos, pStack->index, pStack->count, pStack->disp ); );
+    PUSH_STACK(pStack, pConvertor->stack_pos, pos_desc, pElem->elem.common.type, count_desc,
+               conv_ptr - pConvertor->pBaseBuf);
+    DO_DEBUG(opal_output(0,
+                         "pack save stack stack_pos %d pos_desc %d count_desc %" PRIsize_t
+                         " disp %ld\n",
+                         pConvertor->stack_pos, pStack->index, pStack->count, pStack->disp););
     return 0;
 }
diff --git a/opal/datatype/opal_datatype_pack.h b/opal/datatype/opal_datatype_pack.h
index 2031a005e70..58856e398b6 100644
--- a/opal/datatype/opal_datatype_pack.h
+++ b/opal/datatype/opal_datatype_pack.h
@@ -9,6 +9,7 @@
  *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2020-2021 IBM Corporation.  All rights reserved.
+ * Copyright (c) 2022      Advanced Micro Devices, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -22,35 +23,31 @@
 #include "opal_config.h"
 #include "opal/datatype/opal_datatype_pack_unpack_predefined.h"
 
-#if !defined(CHECKSUM) && OPAL_CUDA_SUPPORT
-/* Make use of existing macro to do CUDA style memcpy */
-#undef MEMCPY_CSUM
-#define MEMCPY_CSUM( DST, SRC, BLENGTH, CONVERTOR ) \
-    CONVERTOR->cbmemcpy( (DST), (SRC), (BLENGTH), (CONVERTOR) )
+#if !defined(CHECKSUM)
+/* Make use of existing macro to do device style memcpy */
+#    undef MEMCPY_CSUM
+#    define MEMCPY_CSUM(DST, SRC, BLENGTH, CONVERTOR) \
+        CONVERTOR->cbmemcpy((DST), (SRC), (BLENGTH), (CONVERTOR))
 #endif
 
 /**
- * This function deals only with partial elements. The COUNT points however to the whole leftover count,
- * but this function is only expected to operate on an amount less than blength, that would allow the rest
- * of the pack process to handle only entire blength blocks (plus the left over).
+ * This function deals only with partial elements. The COUNT points however to the whole leftover
+ * count, but this function is only expected to operate on an amount less than blength, that would
+ * allow the rest of the pack process to handle only entire blength blocks (plus the left over).
  *
  * Return 1 if we are now aligned on a block, 0 otherwise.
  */
-static inline int
-pack_partial_blocklen( opal_convertor_t* CONVERTOR,
-                       const dt_elem_desc_t* ELEM,
-                       size_t* COUNT,
-                       unsigned char** memory,
-                       unsigned char** packed,
-                       size_t* SPACE )
+static inline int pack_partial_blocklen(opal_convertor_t *CONVERTOR, const dt_elem_desc_t *ELEM,
+                                        size_t *COUNT, unsigned char **memory,
+                                        unsigned char **packed, size_t *SPACE)
 {
-    const ddt_elem_desc_t* _elem = &((ELEM)->elem);
+    const ddt_elem_desc_t *_elem = &((ELEM)->elem);
     size_t do_now_bytes = opal_datatype_basicDatatypes[_elem->common.type]->size;
     size_t do_now = *(COUNT);
-    unsigned char* _memory = (*memory) + _elem->disp;
-    unsigned char* _packed = *packed;
+    unsigned char *_memory = (*memory) + _elem->disp;
+    unsigned char *_packed = *packed;
 
-    assert( *(COUNT) <= ((size_t)_elem->count * _elem->blocklen) );
+    assert(*(COUNT) <= ((size_t) _elem->count * _elem->blocklen));
 
     /**
      * First check if we already did something on this element ? The COUNT is the number
@@ -58,88 +55,92 @@ pack_partial_blocklen( opal_convertor_t* CONVERTOR,
      * should be manipulated in the current call (this number is instead reflected on the
      * SPACE).
      */
-    if( 0 == (do_now = (*COUNT) % _elem->blocklen) )
+    if (0 == (do_now = (*COUNT) % _elem->blocklen))
         return 1;
 
-    size_t left_in_block = do_now;  /* left in the current blocklen */
+    size_t left_in_block = do_now; /* left in the current blocklen */
 
-    if( (do_now_bytes * do_now) > *(SPACE) )
+    if ((do_now_bytes * do_now) > *(SPACE))
         do_now = (*SPACE) / do_now_bytes;
 
     do_now_bytes *= do_now;
 
-    OPAL_DATATYPE_SAFEGUARD_POINTER( _memory, do_now_bytes, (CONVERTOR)->pBaseBuf,
-                                     (CONVERTOR)->pDesc, (CONVERTOR)->count );
-    DO_DEBUG( opal_output( 0, "pack memcpy( %p, %p, %lu ) => space %lu [partial]\n",
-                           (void*) _packed, (void*)_memory, (unsigned long)do_now_bytes, (unsigned long)(*(SPACE)) ); );
-    MEMCPY_CSUM( _packed, _memory, do_now_bytes, (CONVERTOR) );
-    *(memory)     += (ptrdiff_t)do_now_bytes;
-    if( do_now == left_in_block )  /* compensate if completed a blocklen */
-        *(memory) += _elem->extent - (_elem->blocklen * opal_datatype_basicDatatypes[_elem->common.type]->size);
-
-    *(COUNT)  -= do_now;
-    *(SPACE)  -= do_now_bytes;
+    OPAL_DATATYPE_SAFEGUARD_POINTER(_memory, do_now_bytes, (CONVERTOR)->pBaseBuf,
+                                    (CONVERTOR)->pDesc, (CONVERTOR)->count);
+    DO_DEBUG(opal_output(0, "pack memcpy( %p, %p, %lu ) => space %lu [partial]\n", (void *) _packed,
+                         (void *) _memory, (unsigned long) do_now_bytes,
+                         (unsigned long) (*(SPACE))););
+    MEMCPY_CSUM(_packed, _memory, do_now_bytes, (CONVERTOR));
+    *(memory) += (ptrdiff_t) do_now_bytes;
+    if (do_now == left_in_block) /* compensate if completed a blocklen */
+        *(memory) += _elem->extent
+                     - (_elem->blocklen * opal_datatype_basicDatatypes[_elem->common.type]->size);
+
+    *(COUNT) -= do_now;
+    *(SPACE) -= do_now_bytes;
     *(packed) += do_now_bytes;
     return (do_now == left_in_block);
 }
 
 /**
- * Pack entire blocks, plus a possible remainder if SPACE is constrained to less than COUNT elements.
+ * Pack entire blocks, plus a possible remainder if SPACE is constrained to less than COUNT
+ * elements.
  */
-static inline void
-pack_predefined_data( opal_convertor_t* CONVERTOR,
-                      const dt_elem_desc_t* ELEM,
-                      size_t* COUNT,
-                      unsigned char** memory,
-                      unsigned char** packed,
-                      size_t* SPACE )
+static inline void pack_predefined_data(opal_convertor_t *CONVERTOR, const dt_elem_desc_t *ELEM,
+                                        size_t *COUNT, unsigned char **memory,
+                                        unsigned char **packed, size_t *SPACE)
 {
-    const ddt_elem_desc_t* _elem = &((ELEM)->elem);
+    const ddt_elem_desc_t *_elem = &((ELEM)->elem);
     size_t blocklen_bytes = opal_datatype_basicDatatypes[_elem->common.type]->size;
     size_t cando_count = *(COUNT), do_now_bytes;
-    unsigned char* _memory = (*memory) + _elem->disp;
-    unsigned char* _packed = *packed;
+    unsigned char *_memory = (*memory) + _elem->disp;
+    unsigned char *_packed = *packed;
 
-    assert( 0 == (cando_count % _elem->blocklen) );  /* no partials here */
-    assert( *(COUNT) <= ((size_t)_elem->count * _elem->blocklen) );
+    assert(0 == (cando_count % _elem->blocklen)); /* no partials here */
+    assert(*(COUNT) <= ((size_t) _elem->count * _elem->blocklen));
 
-    if( (blocklen_bytes * cando_count) > *(SPACE) )
+    if ((blocklen_bytes * cando_count) > *(SPACE))
         cando_count = (*SPACE) / blocklen_bytes;
 
-    /* premptively update the number of COUNT we will return. */
+    /* preemptively update the number of COUNT we will return. */
     *(COUNT) -= cando_count;
 
-    if(_elem->blocklen < 9) {
-        if(OPAL_LIKELY(OPAL_SUCCESS == opal_datatype_pack_predefined_element(&_memory, &_packed, cando_count, _elem)))   {
+    if (_elem->blocklen < 9) {
+        if (!(CONVERTOR->flags & CONVERTOR_ACCELERATOR)
+            && OPAL_LIKELY(
+                OPAL_SUCCESS
+                == opal_datatype_pack_predefined_element(&_memory, &_packed, cando_count, _elem))) {
             goto update_and_return;
         }
         /* else unrecognized _elem->common.type, use the memcpy path */
     }
 
-    if(_elem->blocklen == 1) {
-        for(; cando_count > 0; cando_count--) {
-            OPAL_DATATYPE_SAFEGUARD_POINTER( _memory, blocklen_bytes, (CONVERTOR)->pBaseBuf,
-                                             (CONVERTOR)->pDesc, (CONVERTOR)->count );
-            DO_DEBUG( opal_output( 0, "pack memcpy( %p, %p, %lu ) => space %lu [blen = 1]\n",
-                                   (void*)_packed, (void*)_memory, (unsigned long)blocklen_bytes, (unsigned long)(*(SPACE) - (_packed - *(packed))) ); );
-            MEMCPY_CSUM( _packed, _memory, blocklen_bytes, (CONVERTOR) );
-            _packed     += blocklen_bytes;
-            _memory     += _elem->extent;
+    if (_elem->blocklen == 1) {
+        for (; cando_count > 0; cando_count--) {
+            OPAL_DATATYPE_SAFEGUARD_POINTER(_memory, blocklen_bytes, (CONVERTOR)->pBaseBuf,
+                                            (CONVERTOR)->pDesc, (CONVERTOR)->count);
+            DO_DEBUG(opal_output(0, "pack memcpy( %p, %p, %lu ) => space %lu [blen = 1]\n",
+                                 (void *) _packed, (void *) _memory, (unsigned long) blocklen_bytes,
+                                 (unsigned long) (*(SPACE) - (_packed - *(packed)))););
+            MEMCPY_CSUM(_packed, _memory, blocklen_bytes, (CONVERTOR));
+            _packed += blocklen_bytes;
+            _memory += _elem->extent;
         }
         goto update_and_return;
     }
 
-    if( (1 < _elem->count) && (_elem->blocklen <= cando_count) ) {
+    if ((1 < _elem->count) && (_elem->blocklen <= cando_count)) {
         blocklen_bytes *= _elem->blocklen;
 
         do { /* Do as many full blocklen as possible */
-            OPAL_DATATYPE_SAFEGUARD_POINTER( _memory, blocklen_bytes, (CONVERTOR)->pBaseBuf,
-                                             (CONVERTOR)->pDesc, (CONVERTOR)->count );
-            DO_DEBUG( opal_output( 0, "pack 2. memcpy( %p, %p, %lu ) => space %lu\n",
-                                   (void*)_packed, (void*)_memory, (unsigned long)blocklen_bytes, (unsigned long)(*(SPACE) - (_packed - *(packed))) ); );
-            MEMCPY_CSUM( _packed, _memory, blocklen_bytes, (CONVERTOR) );
-            _packed     += blocklen_bytes;
-            _memory     += _elem->extent;
+            OPAL_DATATYPE_SAFEGUARD_POINTER(_memory, blocklen_bytes, (CONVERTOR)->pBaseBuf,
+                                            (CONVERTOR)->pDesc, (CONVERTOR)->count);
+            DO_DEBUG(opal_output(0, "pack 2. memcpy( %p, %p, %lu ) => space %lu\n",
+                                 (void *) _packed, (void *) _memory, (unsigned long) blocklen_bytes,
+                                 (unsigned long) (*(SPACE) - (_packed - *(packed)))););
+            MEMCPY_CSUM(_packed, _memory, blocklen_bytes, (CONVERTOR));
+            _packed += blocklen_bytes;
+            _memory += _elem->extent;
             cando_count -= _elem->blocklen;
         } while (_elem->blocklen <= cando_count);
     }
@@ -147,70 +148,69 @@ pack_predefined_data( opal_convertor_t* CONVERTOR,
     /**
      * As an epilog do anything left from the last blocklen.
      */
-    if( 0 != cando_count ) {
-        assert( (cando_count < _elem->blocklen) ||
-                ((1 == _elem->count) && (cando_count <= _elem->blocklen)) );
+    if (0 != cando_count) {
+        assert((cando_count < _elem->blocklen)
+               || ((1 == _elem->count) && (cando_count <= _elem->blocklen)));
         do_now_bytes = cando_count * opal_datatype_basicDatatypes[_elem->common.type]->size;
-        OPAL_DATATYPE_SAFEGUARD_POINTER( _memory, do_now_bytes, (CONVERTOR)->pBaseBuf,
-                                         (CONVERTOR)->pDesc, (CONVERTOR)->count );
-        DO_DEBUG( opal_output( 0, "pack 3. memcpy( %p, %p, %lu ) => space %lu [epilog]\n",
-                               (void*)_packed, (void*)_memory, (unsigned long)do_now_bytes, (unsigned long)(*(SPACE) - (_packed - *(packed))) ); );
-        MEMCPY_CSUM( _packed, _memory, do_now_bytes, (CONVERTOR) );
-        _memory   += do_now_bytes;
-        _packed   += do_now_bytes;
+        OPAL_DATATYPE_SAFEGUARD_POINTER(_memory, do_now_bytes, (CONVERTOR)->pBaseBuf,
+                                        (CONVERTOR)->pDesc, (CONVERTOR)->count);
+        DO_DEBUG(opal_output(0, "pack 3. memcpy( %p, %p, %lu ) => space %lu [epilog]\n",
+                             (void *) _packed, (void *) _memory, (unsigned long) do_now_bytes,
+                             (unsigned long) (*(SPACE) - (_packed - *(packed)))););
+        MEMCPY_CSUM(_packed, _memory, do_now_bytes, (CONVERTOR));
+        _memory += do_now_bytes;
+        _packed += do_now_bytes;
     }
 
- update_and_return:
-    *(memory)  = _memory - _elem->disp;
-    *(SPACE)  -= (_packed - *packed);
-    *(packed)  = _packed;
+update_and_return:
+    *(memory) = _memory - _elem->disp;
+    *(SPACE) -= (_packed - *packed);
+    *(packed) = _packed;
 }
 
-static inline void pack_contiguous_loop( opal_convertor_t* CONVERTOR,
-                                         const dt_elem_desc_t* ELEM,
-                                         size_t* COUNT,
-                                         unsigned char** memory,
-                                         unsigned char** packed,
-                                         size_t* SPACE )
+static inline void pack_contiguous_loop(opal_convertor_t *CONVERTOR, const dt_elem_desc_t *ELEM,
+                                        size_t *COUNT, unsigned char **memory,
+                                        unsigned char **packed, size_t *SPACE)
 {
-    const ddt_loop_desc_t *_loop = (ddt_loop_desc_t*)(ELEM);
-    const ddt_endloop_desc_t* _end_loop = (ddt_endloop_desc_t*)((ELEM) + _loop->items);
-    unsigned char* _memory = (*memory) + _end_loop->first_elem_disp;
+    const ddt_loop_desc_t *_loop = (ddt_loop_desc_t *) (ELEM);
+    const ddt_endloop_desc_t *_end_loop = (ddt_endloop_desc_t *) ((ELEM) + _loop->items);
+    unsigned char *_memory = (*memory) + _end_loop->first_elem_disp;
     size_t _copy_loops = *(COUNT);
 
-    if( (_copy_loops * _end_loop->size) > *(SPACE) )
+    if ((_copy_loops * _end_loop->size) > *(SPACE))
         _copy_loops = (*(SPACE) / _end_loop->size);
-    for(size_t _i = 0; _i < _copy_loops; _i++ ) {
-        OPAL_DATATYPE_SAFEGUARD_POINTER( _memory, _end_loop->size, (CONVERTOR)->pBaseBuf,
-                                    (CONVERTOR)->pDesc, (CONVERTOR)->count );
-        DO_DEBUG( opal_output( 0, "pack 3. memcpy( %p, %p, %lu ) => space %lu\n",
-                               (void*)*(packed), (void*)_memory, (unsigned long)_end_loop->size, (unsigned long)(*(SPACE) - _i * _end_loop->size) ); );
-        MEMCPY_CSUM( *(packed), _memory, _end_loop->size, (CONVERTOR) );
+    for (size_t _i = 0; _i < _copy_loops; _i++) {
+        OPAL_DATATYPE_SAFEGUARD_POINTER(_memory, _end_loop->size, (CONVERTOR)->pBaseBuf,
+                                        (CONVERTOR)->pDesc, (CONVERTOR)->count);
+        DO_DEBUG(opal_output(0, "pack 3. memcpy( %p, %p, %lu ) => space %lu\n", (void *) *(packed),
+                             (void *) _memory, (unsigned long) _end_loop->size,
+                             (unsigned long) (*(SPACE) -_i * _end_loop->size)););
+        MEMCPY_CSUM(*(packed), _memory, _end_loop->size, (CONVERTOR));
         *(packed) += _end_loop->size;
-        _memory   += _loop->extent;
+        _memory += _loop->extent;
     }
     *(memory) = _memory - _end_loop->first_elem_disp;
     *(SPACE) -= _copy_loops * _end_loop->size;
     *(COUNT) -= _copy_loops;
 }
 
-#define PACK_PARTIAL_BLOCKLEN( CONVERTOR,    /* the convertor */                       \
-                               ELEM,         /* the basic element to be packed */ \
-                               COUNT,        /* the number of elements */ \
-                               MEMORY,       /* the source pointer (char*) */ \
-                               PACKED,       /* the destination pointer (char*) */ \
-                               SPACE )       /* the space in the destination buffer */ \
-pack_partial_blocklen( (CONVERTOR), (ELEM), &(COUNT), &(MEMORY), &(PACKED), &(SPACE) )
-
-#define PACK_PREDEFINED_DATATYPE( CONVERTOR,    /* the convertor */     \
-                                  ELEM,         /* the basic element to be packed */      \
-                                  COUNT,        /* the number of elements */              \
-                                  MEMORY,       /* the source pointer (char*) */          \
-                                  PACKED,       /* the destination pointer (char*) */     \
-                                  SPACE )       /* the space in the destination buffer */ \
-pack_predefined_data( (CONVERTOR), (ELEM), &(COUNT), &(MEMORY), &(PACKED), &(SPACE) )
-
-#define PACK_CONTIGUOUS_LOOP( CONVERTOR, ELEM, COUNT, MEMORY, PACKED, SPACE ) \
-    pack_contiguous_loop( (CONVERTOR), (ELEM), &(COUNT), &(MEMORY), &(PACKED), &(SPACE) )
-
-#endif  /* OPAL_DATATYPE_PACK_H_HAS_BEEN_INCLUDED */
+#define PACK_PARTIAL_BLOCKLEN(CONVERTOR, /* the convertor */                       \
+                              ELEM,      /* the basic element to be packed */      \
+                              COUNT,     /* the number of elements */              \
+                              MEMORY,    /* the source pointer (char*) */          \
+                              PACKED,    /* the destination pointer (char*) */     \
+                              SPACE)     /* the space in the destination buffer */ \
+    pack_partial_blocklen((CONVERTOR), (ELEM), &(COUNT), &(MEMORY), &(PACKED), &(SPACE))
+
+#define PACK_PREDEFINED_DATATYPE(CONVERTOR, /* the convertor */                       \
+                                 ELEM,      /* the basic element to be packed */      \
+                                 COUNT,     /* the number of elements */              \
+                                 MEMORY,    /* the source pointer (char*) */          \
+                                 PACKED,    /* the destination pointer (char*) */     \
+                                 SPACE)     /* the space in the destination buffer */ \
+    pack_predefined_data((CONVERTOR), (ELEM), &(COUNT), &(MEMORY), &(PACKED), &(SPACE))
+
+#define PACK_CONTIGUOUS_LOOP(CONVERTOR, ELEM, COUNT, MEMORY, PACKED, SPACE) \
+    pack_contiguous_loop((CONVERTOR), (ELEM), &(COUNT), &(MEMORY), &(PACKED), &(SPACE))
+
+#endif /* OPAL_DATATYPE_PACK_H_HAS_BEEN_INCLUDED */
diff --git a/opal/datatype/opal_datatype_pack_unpack_predefined.h b/opal/datatype/opal_datatype_pack_unpack_predefined.h
index be98cf2716c..b16fce0f111 100644
--- a/opal/datatype/opal_datatype_pack_unpack_predefined.h
+++ b/opal/datatype/opal_datatype_pack_unpack_predefined.h
@@ -2,6 +2,7 @@
  * Copyright (c) 2020-2021 IBM Corporation. All rights reserved.
  * Copyright (c) 2002      University of Chicago
  * Copyright (c) 2001      Argonne National Laboratory
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -54,7 +55,6 @@
 #define OPAL_DATATYPE_PACK_UNPACK_PREDEFINED_H_HAS_BEEN_INCLUDED
 
 #include "opal_config.h"
-#include "opal/mca/common/cuda/common_cuda.h"
 #include <stdint.h>
 
 /*  Improve predefined pack/unpack performance using mpich methods.
@@ -68,221 +68,200 @@
  *   do a manual copy for larger blocklengths, but more data will have
  *   to be gathered to see if an implementation would be
  *   better over the current implementation.
-*/
+ */
 
-#define OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_ONE(stride, blocklen) { \
-  for (; i; i--) {            \
-    *_dest   = *_src;         \
-     _src    += stride;       \
-     _dest   += blocklen;     \
-  }                           \
-}
+#define OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_ONE(stride, blocklen) \
+    {                                                                \
+        for (; i; i--) {                                             \
+            *_dest = *_src;                                          \
+            _src += stride;                                          \
+            _dest += blocklen;                                       \
+        }                                                            \
+    }
 
-#define OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_TWO(stride, blocklen) { \
-  for (; i > 1; i -= 2) {     \
-    _dest[0]  = _src[0];      \
-    _dest[1]  = _src[1];      \
-    _src     += stride;       \
-    _dest    += blocklen;     \
-  }                           \
-}
+#define OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_TWO(stride, blocklen) \
+    {                                                                \
+        for (; i > 1; i -= 2) {                                      \
+            _dest[0] = _src[0];                                      \
+            _dest[1] = _src[1];                                      \
+            _src += stride;                                          \
+            _dest += blocklen;                                       \
+        }                                                            \
+    }
 
-#define OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_THREE(stride, blocklen) { \
-  for (; i > 2; i -= 3) {     \
-    _dest[0]  = _src[0];      \
-    _dest[1]  = _src[1];      \
-    _dest[2]  = _src[2];      \
-    _src     += stride;       \
-    _dest    += blocklen;     \
-  }                           \
-}
+#define OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_THREE(stride, blocklen) \
+    {                                                                  \
+        for (; i > 2; i -= 3) {                                        \
+            _dest[0] = _src[0];                                        \
+            _dest[1] = _src[1];                                        \
+            _dest[2] = _src[2];                                        \
+            _src += stride;                                            \
+            _dest += blocklen;                                         \
+        }                                                              \
+    }
 
-#define OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_FOUR(stride, blocklen) { \
-  for (; i > 3; i -= 4) {     \
-    _dest[0]  = _src[0];      \
-    _dest[1]  = _src[1];      \
-    _dest[2]  = _src[2];      \
-    _dest[3]  = _src[3];      \
-    _src     += stride;       \
-    _dest    += blocklen;     \
-  }                           \
-}
+#define OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_FOUR(stride, blocklen) \
+    {                                                                 \
+        for (; i > 3; i -= 4) {                                       \
+            _dest[0] = _src[0];                                       \
+            _dest[1] = _src[1];                                       \
+            _dest[2] = _src[2];                                       \
+            _dest[3] = _src[3];                                       \
+            _src += stride;                                           \
+            _dest += blocklen;                                        \
+        }                                                             \
+    }
 
-#define OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_FIVE(stride, blocklen) { \
-  for (; i > 4; i -= 5) {     \
-    _dest[0]  = _src[0];      \
-    _dest[1]  = _src[1];      \
-    _dest[2]  = _src[2];      \
-    _dest[3]  = _src[3];      \
-    _dest[4]  = _src[4];      \
-    _src     += stride;       \
-    _dest    += blocklen;     \
-  }                           \
-}
+#define OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_FIVE(stride, blocklen) \
+    {                                                                 \
+        for (; i > 4; i -= 5) {                                       \
+            _dest[0] = _src[0];                                       \
+            _dest[1] = _src[1];                                       \
+            _dest[2] = _src[2];                                       \
+            _dest[3] = _src[3];                                       \
+            _dest[4] = _src[4];                                       \
+            _src += stride;                                           \
+            _dest += blocklen;                                        \
+        }                                                             \
+    }
 
-#define OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_SIX(stride, blocklen) { \
-  for (; i > 5; i -= 6) {     \
-    _dest[0]  = _src[0];      \
-    _dest[1]  = _src[1];      \
-    _dest[2]  = _src[2];      \
-    _dest[3]  = _src[3];      \
-    _dest[4]  = _src[4];      \
-    _dest[5]  = _src[5];      \
-    _src     += stride;       \
-    _dest    += blocklen;     \
-  }                           \
-}
+#define OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_SIX(stride, blocklen) \
+    {                                                                \
+        for (; i > 5; i -= 6) {                                      \
+            _dest[0] = _src[0];                                      \
+            _dest[1] = _src[1];                                      \
+            _dest[2] = _src[2];                                      \
+            _dest[3] = _src[3];                                      \
+            _dest[4] = _src[4];                                      \
+            _dest[5] = _src[5];                                      \
+            _src += stride;                                          \
+            _dest += blocklen;                                       \
+        }                                                            \
+    }
 
-#define OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_SEVEN(stride, blocklen) { \
-  for (; i > 6; i -= 7) {     \
-    _dest[0]  = _src[0];      \
-    _dest[1]  = _src[1];      \
-    _dest[2]  = _src[2];      \
-    _dest[3]  = _src[3];      \
-    _dest[4]  = _src[4];      \
-    _dest[5]  = _src[5];      \
-    _dest[6]  = _src[6];      \
-    _src     += stride;       \
-    _dest    += blocklen;     \
-  }                           \
-}
+#define OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_SEVEN(stride, blocklen) \
+    {                                                                  \
+        for (; i > 6; i -= 7) {                                        \
+            _dest[0] = _src[0];                                        \
+            _dest[1] = _src[1];                                        \
+            _dest[2] = _src[2];                                        \
+            _dest[3] = _src[3];                                        \
+            _dest[4] = _src[4];                                        \
+            _dest[5] = _src[5];                                        \
+            _dest[6] = _src[6];                                        \
+            _src += stride;                                            \
+            _dest += blocklen;                                         \
+        }                                                              \
+    }
 
-#define OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_EIGHT(stride, blocklen) { \
-  for (; i > 7; i -= 8) {     \
-    _dest[0]  = _src[0];      \
-    _dest[1]  = _src[1];      \
-    _dest[2]  = _src[2];      \
-    _dest[3]  = _src[3];      \
-    _dest[4]  = _src[4];      \
-    _dest[5]  = _src[5];      \
-    _dest[6]  = _src[6];      \
-    _dest[7]  = _src[7];      \
-    _src     += stride;       \
-    _dest    += blocklen;     \
-  }                           \
-}
+#define OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_EIGHT(stride, blocklen) \
+    {                                                                  \
+        for (; i > 7; i -= 8) {                                        \
+            _dest[0] = _src[0];                                        \
+            _dest[1] = _src[1];                                        \
+            _dest[2] = _src[2];                                        \
+            _dest[3] = _src[3];                                        \
+            _dest[4] = _src[4];                                        \
+            _dest[5] = _src[5];                                        \
+            _dest[6] = _src[6];                                        \
+            _dest[7] = _src[7];                                        \
+            _src += stride;                                            \
+            _dest += blocklen;                                         \
+        }                                                              \
+    }
 
-#define OPAL_DATATYPE_PACK_PREDEFINED_RESIDUAL_DATA() { \
-  if(i != 0) {                          \
-   for (; i > 0; i--) {                 \
-     *_dest++ = *_src++;                \
-   }                                    \
-  }                                     \
-}
+#define OPAL_DATATYPE_PACK_PREDEFINED_RESIDUAL_DATA() \
+    {                                                 \
+        if (i != 0) {                                 \
+            for (; i > 0; i--) {                      \
+                *_dest++ = *_src++;                   \
+            }                                         \
+        }                                             \
+    }
 
-#define OPAL_DATATYPE_PACK_PREDEFINED_ELEMENT(src_base, dest_base, count, stride, blocklen, type) { \
-  type* _src  = (type *) src_base;                                        \
-  type* _dest = (type *) dest_base;                                       \
-  register unsigned long i = count;                                       \
-  if(blocklen == 1) {                                                     \
-    OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_ONE(stride, blocklen);         \
-  }                                                                       \
-  else if (blocklen == 2) {                                               \
-    OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_TWO(stride, blocklen);         \
-  }                                                                       \
-  else if (blocklen == 3) {                                               \
-    OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_THREE(stride, blocklen);       \
-  }                                                                       \
-  else if (blocklen == 4) {                                               \
-    OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_FOUR(stride, blocklen);        \
-  }                                                                       \
-  else if (blocklen == 5) {                                               \
-    OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_FIVE(stride, blocklen);        \
-  }                                                                       \
-  else if (blocklen == 6) {                                               \
-    OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_SIX(stride, blocklen);         \
-  }                                                                       \
-  else if (blocklen == 7) {                                               \
-    OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_SEVEN(stride, blocklen);       \
-  }                                                                       \
-  else if (blocklen == 8) {                                               \
-    OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_EIGHT(stride, blocklen);       \
-  }                                                                       \
-  OPAL_DATATYPE_PACK_PREDEFINED_RESIDUAL_DATA()                           \
-  src_base  = (unsigned char *) _src;                                     \
-  dest_base = (unsigned char *) _dest;                                    \
-}                                                                         \
+#define OPAL_DATATYPE_PACK_PREDEFINED_ELEMENT(src_base, dest_base, count, blocklen, type, align)  \
+    {                                                                                             \
+        register unsigned long i = count;                                                         \
+        if (((uintptr_t) src_base)  % (align) ||                                                  \
+            ((uintptr_t) dest_base) % (align) ||                                                  \
+            (elem->extent % (align) && cando_count > blocklen)) {                                 \
+            return OPAL_ERROR;                                                                    \
+        }                                                                                         \
+        type *_src = (type *) src_base;                                                           \
+        type *_dest = (type *) dest_base;                                                         \
+        size_t stride = elem->extent / sizeof(type);                                              \
+        if (blocklen == 1) {                                                                      \
+            OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_ONE(stride, blocklen);                         \
+        } else if (blocklen == 2) {                                                               \
+            OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_TWO(stride, blocklen);                         \
+        } else if (blocklen == 3) {                                                               \
+            OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_THREE(stride, blocklen);                       \
+        } else if (blocklen == 4) {                                                               \
+            OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_FOUR(stride, blocklen);                        \
+        } else if (blocklen == 5) {                                                               \
+            OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_FIVE(stride, blocklen);                        \
+        } else if (blocklen == 6) {                                                               \
+            OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_SIX(stride, blocklen);                         \
+        } else if (blocklen == 7) {                                                               \
+            OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_SEVEN(stride, blocklen);                       \
+        } else if (blocklen == 8) {                                                               \
+            OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_EIGHT(stride, blocklen);                       \
+        }                                                                                         \
+        OPAL_DATATYPE_PACK_PREDEFINED_RESIDUAL_DATA()                                             \
+        src_base = (unsigned char *) _src;                                                        \
+        dest_base = (unsigned char *) _dest;                                                      \
+    }
 
-#define OPAL_DATATYPE_UNPACK_PREDEFINED_ELEMENT(src_base, dest_base, count, stride, blocklen, type) { \
-  type* _src  = (type *) src_base;                                        \
-  type* _dest = (type *) dest_base;                                       \
-  register unsigned long i = count;                                       \
-  /* (reversing the meanings of blocklen and stride and using the "PACK" macro) */ \
-  if(blocklen == 1) {                                                     \
-    OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_ONE(blocklen, stride);         \
-  }                                                                       \
-  else if (blocklen == 2) {                                               \
-    OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_TWO(blocklen, stride);         \
-  }                                                                       \
-  else if (blocklen == 3) {                                               \
-    OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_THREE(blocklen, stride);       \
-  }                                                                       \
-  else if (blocklen == 4) {                                               \
-    OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_FOUR(blocklen, stride);        \
-  }                                                                       \
-  else if (blocklen == 5) {                                               \
-    OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_FIVE(blocklen, stride);        \
-  }                                                                       \
-  else if (blocklen == 6) {                                               \
-    OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_SIX(blocklen, stride);         \
-  }                                                                       \
-  else if (blocklen == 7) {                                               \
-    OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_SEVEN(blocklen, stride);       \
-  }                                                                       \
-  else if (blocklen == 8) {                                               \
-    OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_EIGHT(blocklen, stride);       \
-  }                                                                       \
-  OPAL_DATATYPE_PACK_PREDEFINED_RESIDUAL_DATA()                           \
-  src_base  = (unsigned char *) _src;                                     \
-  dest_base = (unsigned char *) _dest;                                    \
-}                                                                         \
+#define OPAL_DATATYPE_UNPACK_PREDEFINED_ELEMENT(src_base, dest_base, count, blocklen,         \
+                                                type, align)                                  \
+    {                                                                                         \
+        if (((uintptr_t) src_base)  % (align)  ||                                             \
+            ((uintptr_t) dest_base) % (align) ||                                              \
+            (elem->extent % (align) && cando_count > blocklen)) {                             \
+            return OPAL_ERROR;                                                                \
+        }                                                                                     \
+        type *_src = (type *) src_base;                                                       \
+        type *_dest = (type *) dest_base;                                                     \
+        /* elem's extent but in terms of count rather than bytes */                           \
+        size_t stride = elem->extent / sizeof(type);                                          \
+        register unsigned long i = count;                                                     \
+        /* (reversing the meanings of blocklen and stride and using the "PACK" macro) */      \
+        if (blocklen == 1) {                                                                  \
+            OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_ONE(blocklen, stride);                     \
+        } else if (blocklen == 2) {                                                           \
+            OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_TWO(blocklen, stride);                     \
+        } else if (blocklen == 3) {                                                           \
+            OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_THREE(blocklen, stride);                   \
+        } else if (blocklen == 4) {                                                           \
+            OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_FOUR(blocklen, stride);                    \
+        } else if (blocklen == 5) {                                                           \
+            OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_FIVE(blocklen, stride);                    \
+        } else if (blocklen == 6) {                                                           \
+            OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_SIX(blocklen, stride);                     \
+        } else if (blocklen == 7) {                                                           \
+            OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_SEVEN(blocklen, stride);                   \
+        } else if (blocklen == 8) {                                                           \
+            OPAL_DATATYPE_PACK_PREDEFINED_BLOCKLEN_EIGHT(blocklen, stride);                   \
+        }                                                                                     \
+        OPAL_DATATYPE_PACK_PREDEFINED_RESIDUAL_DATA()                                         \
+        src_base = (unsigned char *) _src;                                                    \
+        dest_base = (unsigned char *) _dest;                                                  \
+    }
 
-static inline int
-opal_datatype_unpack_predefined_element( unsigned char** rtn_src,
-                                unsigned char** rtn_dest,
-                                size_t cando_count,
-                                const ddt_elem_desc_t* elem)
+static inline int opal_datatype_unpack_predefined_element(unsigned char **rtn_src,
+                                                          unsigned char **rtn_dest,
+                                                          size_t cando_count,
+                                                          const ddt_elem_desc_t *elem)
 {
-    size_t stride; // elem's extent but in terms of count rather than bytes
     size_t blocklen;
     int id;
-    int align;
 
     id = elem->common.type;
     blocklen = elem->blocklen;
-    stride = elem->extent / opal_datatype_basicDatatypes[id]->size;
-    align = opal_datatype_basicDatatypes[id]->align;
 
     unsigned char *src = *rtn_src;
     unsigned char *dest = *rtn_dest;
 
-#if OPAL_CUDA_SUPPORT
-    if (opal_cuda_check_bufs(dest, src)) {
-        return OPAL_ERROR;
-    }
-/*
- *  For checking if elem contains cuda memory, I think it's mostly okay
- *  to only check the first element as done above.  Although a complete
- *  MPI datatype could easily be made to span both gpu and system memory,
- *  I don't think that's true for the individual vector elements that make
- *  up a datatype's description.  The only way I can even conceive of that
- *  being untrue is if the element has only two entries with a crazy
- *  extent sized to hit both locations.  I don't really think that's
- *  possible, but I'm checking it anyway below.
- */
-    if (elem->count == 2 && cando_count >= blocklen &&
-       (opal_cuda_check_bufs(dest + elem->extent, src)))
-    {
-        return OPAL_ERROR;
-    }
-#endif
-  if ((uintptr_t)src % align  ||
-      (uintptr_t)dest % align ||
-      (elem->extent % align && cando_count > blocklen))
-  {
-      return OPAL_ERROR;
-  }
-
 /*
  *  Here as an example of how we want to call our macro, if the incoming id
  *  were OPAL_DATATYPE_INT4, we want
@@ -298,234 +277,210 @@ opal_datatype_unpack_predefined_element( unsigned char** rtn_src,
  *  Otherwise we'd have to copy and maintain essentially the same blob of
  *  macros that already exist in opal_datatype_internal.h.
  */
-#define OPAL_DATATYPE_MYUNPACK(NAME)               \
-    do {                                           \
-        OPAL_DATATYPE_HANDLE_ ## NAME(             \
-            OPAL_DATATYPE_MYUNPACK_AVAILABLE,      \
-            OPAL_DATATYPE_MYUNPACK_NOTAVAIL, 0);   \
+#define OPAL_DATATYPE_MYUNPACK(NAME)                                     \
+    do {                                                                 \
+        OPAL_DATATYPE_HANDLE_##NAME(OPAL_DATATYPE_MYUNPACK_AVAILABLE,    \
+                                    OPAL_DATATYPE_MYUNPACK_NOTAVAIL, 0); \
     } while (0)
 
-#define OPAL_DATATYPE_MYUNPACK_AVAILABLE(TYPE, unused_ALIGN, NAME, unused) \
-    do { \
-        OPAL_DATATYPE_UNPACK_PREDEFINED_ELEMENT(src, dest, cando_count, stride, blocklen, TYPE); \
-        success = true; \
+#define OPAL_DATATYPE_MYUNPACK_AVAILABLE(TYPE, ALIGN, NAME, unused)                              \
+    do {                                                                                         \
+        OPAL_DATATYPE_UNPACK_PREDEFINED_ELEMENT(src, dest, cando_count, blocklen, TYPE, ALIGN);  \
+        success = true;                                                                          \
     } while (0)
 
 #define OPAL_DATATYPE_MYUNPACK_NOTAVAIL(NAME, unused) \
-    do { \
-        success = false; \
+    do {                                              \
+        success = false;                              \
     } while (0)
 
     bool success = false;
-    switch(id) {
-        case OPAL_DATATYPE_INT1:
-            // The below macro should expand to
-            // OPAL_DATATYPE_UNPACK_PREDEFINED_ELEMENT(src, dest, cando_count, stride, blocklen, int8_t);
-            // by using OPAL_DATATYPE_HANDLE_* where it finds that INT1 means int8_t etc
-            OPAL_DATATYPE_MYUNPACK(INT1);
-            break;
-        case OPAL_DATATYPE_INT2:
-            OPAL_DATATYPE_MYUNPACK(INT2);
-            break;
-        case OPAL_DATATYPE_INT4:
-            OPAL_DATATYPE_MYUNPACK(INT4);
-            break;
-        case OPAL_DATATYPE_INT8:
-            OPAL_DATATYPE_MYUNPACK(INT8);
-            break;
-        case OPAL_DATATYPE_INT16:
-            OPAL_DATATYPE_MYUNPACK(INT16);
-            break;
-        case OPAL_DATATYPE_UINT1:
-            OPAL_DATATYPE_MYUNPACK(UINT1);
-            break;
-        case OPAL_DATATYPE_UINT2:
-            OPAL_DATATYPE_MYUNPACK(UINT2);
-            break;
-        case OPAL_DATATYPE_UINT4:
-            OPAL_DATATYPE_MYUNPACK(UINT4);
-            break;
-        case OPAL_DATATYPE_UINT8:
-            OPAL_DATATYPE_MYUNPACK(UINT8);
-            break;
-        case OPAL_DATATYPE_UINT16:
-            OPAL_DATATYPE_MYUNPACK(UINT16);
-            break;
-        case OPAL_DATATYPE_FLOAT2:
-            OPAL_DATATYPE_MYUNPACK(FLOAT2);
-            break;
-        case OPAL_DATATYPE_FLOAT4:
-            OPAL_DATATYPE_MYUNPACK(FLOAT4);
-            break;
-        case OPAL_DATATYPE_FLOAT8:
-            OPAL_DATATYPE_MYUNPACK(FLOAT8);
-            break;
-        case OPAL_DATATYPE_FLOAT12:
-            OPAL_DATATYPE_MYUNPACK(FLOAT12);
-            break;
-        case OPAL_DATATYPE_FLOAT16:
-            OPAL_DATATYPE_MYUNPACK(FLOAT16);
-            break;
-        case OPAL_DATATYPE_SHORT_FLOAT_COMPLEX:
-            OPAL_DATATYPE_MYUNPACK(SHORT_FLOAT_COMPLEX);
-            break;
-        case OPAL_DATATYPE_FLOAT_COMPLEX:
-            OPAL_DATATYPE_MYUNPACK(FLOAT_COMPLEX);
-            break;
-        case OPAL_DATATYPE_DOUBLE_COMPLEX:
-            OPAL_DATATYPE_MYUNPACK(DOUBLE_COMPLEX);
-            break;
-        case OPAL_DATATYPE_LONG_DOUBLE_COMPLEX:
-            OPAL_DATATYPE_MYUNPACK(LONG_DOUBLE_COMPLEX);
-            break;
-        case OPAL_DATATYPE_BOOL:
-            OPAL_DATATYPE_MYUNPACK(BOOL);
-            break;
-        case OPAL_DATATYPE_WCHAR:
-            OPAL_DATATYPE_MYUNPACK(WCHAR);
-            break;
-        default:
-            return OPAL_ERROR;
+    switch (id) {
+    case OPAL_DATATYPE_INT1:
+        // The below macro should expand to
+        // OPAL_DATATYPE_UNPACK_PREDEFINED_ELEMENT(src, dest, cando_count, stride, blocklen,
+        // int8_t); by using OPAL_DATATYPE_HANDLE_* where it finds that INT1 means int8_t etc
+        OPAL_DATATYPE_MYUNPACK(INT1);
+        break;
+    case OPAL_DATATYPE_INT2:
+        OPAL_DATATYPE_MYUNPACK(INT2);
+        break;
+    case OPAL_DATATYPE_INT4:
+        OPAL_DATATYPE_MYUNPACK(INT4);
+        break;
+    case OPAL_DATATYPE_INT8:
+        OPAL_DATATYPE_MYUNPACK(INT8);
+        break;
+    case OPAL_DATATYPE_INT16:
+        OPAL_DATATYPE_MYUNPACK(INT16);
+        break;
+    case OPAL_DATATYPE_UINT1:
+        OPAL_DATATYPE_MYUNPACK(UINT1);
+        break;
+    case OPAL_DATATYPE_UINT2:
+        OPAL_DATATYPE_MYUNPACK(UINT2);
+        break;
+    case OPAL_DATATYPE_UINT4:
+        OPAL_DATATYPE_MYUNPACK(UINT4);
+        break;
+    case OPAL_DATATYPE_UINT8:
+        OPAL_DATATYPE_MYUNPACK(UINT8);
+        break;
+    case OPAL_DATATYPE_UINT16:
+        OPAL_DATATYPE_MYUNPACK(UINT16);
+        break;
+    case OPAL_DATATYPE_FLOAT2:
+        OPAL_DATATYPE_MYUNPACK(FLOAT2);
+        break;
+    case OPAL_DATATYPE_FLOAT4:
+        OPAL_DATATYPE_MYUNPACK(FLOAT4);
+        break;
+    case OPAL_DATATYPE_FLOAT8:
+        OPAL_DATATYPE_MYUNPACK(FLOAT8);
+        break;
+    case OPAL_DATATYPE_FLOAT12:
+        OPAL_DATATYPE_MYUNPACK(FLOAT12);
+        break;
+    case OPAL_DATATYPE_FLOAT16:
+        OPAL_DATATYPE_MYUNPACK(FLOAT16);
+        break;
+    case OPAL_DATATYPE_SHORT_FLOAT_COMPLEX:
+        OPAL_DATATYPE_MYUNPACK(SHORT_FLOAT_COMPLEX);
+        break;
+    case OPAL_DATATYPE_FLOAT_COMPLEX:
+        OPAL_DATATYPE_MYUNPACK(FLOAT_COMPLEX);
+        break;
+    case OPAL_DATATYPE_DOUBLE_COMPLEX:
+        OPAL_DATATYPE_MYUNPACK(DOUBLE_COMPLEX);
+        break;
+    case OPAL_DATATYPE_LONG_DOUBLE_COMPLEX:
+        OPAL_DATATYPE_MYUNPACK(LONG_DOUBLE_COMPLEX);
+        break;
+    case OPAL_DATATYPE_BOOL:
+        OPAL_DATATYPE_MYUNPACK(BOOL);
+        break;
+    case OPAL_DATATYPE_WCHAR:
+        OPAL_DATATYPE_MYUNPACK(WCHAR);
+        break;
+    default:
+        return OPAL_ERROR;
     }
     if (!success) {
         return OPAL_ERROR;
     }
 
-    *rtn_src  = src;
+    *rtn_src = src;
     *rtn_dest = dest;
     return OPAL_SUCCESS;
 }
 
-static inline int
-opal_datatype_pack_predefined_element( unsigned char** rtn_src,
-                                unsigned char** rtn_dest,
-                                size_t cando_count,
-                                const ddt_elem_desc_t* elem)
+static inline int opal_datatype_pack_predefined_element(unsigned char **rtn_src,
+                                                        unsigned char **rtn_dest,
+                                                        size_t cando_count,
+                                                        const ddt_elem_desc_t *elem)
 {
-    size_t stride; // elem's extent but in terms of count rather than bytes
     size_t blocklen;
     int id;
-    int align;
 
     id = elem->common.type;
     blocklen = elem->blocklen;
-    stride = elem->extent / opal_datatype_basicDatatypes[id]->size;
-    align = opal_datatype_basicDatatypes[id]->align;
 
     unsigned char *src = *rtn_src;
     unsigned char *dest = *rtn_dest;
 
-#if OPAL_CUDA_SUPPORT
-    if (opal_cuda_check_bufs(dest, src)) {
-        return OPAL_ERROR;
-    }
-    if (elem->count == 2 && cando_count >= blocklen &&
-       (opal_cuda_check_bufs(dest, src + elem->extent)))
-    {
-        return OPAL_ERROR;
-    }
-#endif
-  if ((uintptr_t)src % align  ||
-      (uintptr_t)dest % align ||
-      (elem->extent % align && cando_count > blocklen))
-  {
-      return OPAL_ERROR;
-  }
-
-#define OPAL_DATATYPE_MYPACK(NAME)                 \
-    do {                                           \
-        OPAL_DATATYPE_HANDLE_ ## NAME(             \
-            OPAL_DATATYPE_MYPACK_AVAILABLE,        \
-            OPAL_DATATYPE_MYPACK_NOTAVAIL, 0);     \
+#define OPAL_DATATYPE_MYPACK(NAME)                                                                 \
+    do {                                                                                           \
+        OPAL_DATATYPE_HANDLE_##NAME(OPAL_DATATYPE_MYPACK_AVAILABLE, OPAL_DATATYPE_MYPACK_NOTAVAIL, \
+                                    0);                                                            \
     } while (0)
 
-#define OPAL_DATATYPE_MYPACK_AVAILABLE(TYPE, unused_ALIGN, NAME, unused) \
-    do { \
-        OPAL_DATATYPE_PACK_PREDEFINED_ELEMENT(src, dest, cando_count, stride, blocklen, TYPE); \
-        success = true; \
+#define OPAL_DATATYPE_MYPACK_AVAILABLE(TYPE, ALIGN, NAME, unused)                             \
+    do {                                                                                      \
+        OPAL_DATATYPE_PACK_PREDEFINED_ELEMENT(src, dest, cando_count, blocklen, TYPE, ALIGN); \
+        success = true;                                                                       \
     } while (0)
 
 #define OPAL_DATATYPE_MYPACK_NOTAVAIL(NAME, unused) \
-    do { \
-        success = false; \
+    do {                                            \
+        success = false;                            \
     } while (0)
 
     bool success = false;
-    switch(id) {
-        case OPAL_DATATYPE_INT1:
-            // The below macro should expand to
-            // OPAL_DATATYPE_PACK_PREDEFINED_ELEMENT(src, dest, cando_count, stride, blocklen, int8_t);
-            // by using OPAL_DATATYPE_HANDLE_* where it finds that INT1 means int8_t etc
-            OPAL_DATATYPE_MYPACK(INT1);
-            break;
-        case OPAL_DATATYPE_INT2:
-            OPAL_DATATYPE_MYPACK(INT2);
-            break;
-        case OPAL_DATATYPE_INT4:
-            OPAL_DATATYPE_MYPACK(INT4);
-            break;
-        case OPAL_DATATYPE_INT8:
-            OPAL_DATATYPE_MYPACK(INT8);
-            break;
-        case OPAL_DATATYPE_INT16:
-            OPAL_DATATYPE_MYPACK(INT16);
-            break;
-        case OPAL_DATATYPE_UINT1:
-            OPAL_DATATYPE_MYPACK(UINT1);
-            break;
-        case OPAL_DATATYPE_UINT2:
-            OPAL_DATATYPE_MYPACK(UINT2);
-            break;
-        case OPAL_DATATYPE_UINT4:
-            OPAL_DATATYPE_MYPACK(UINT4);
-            break;
-        case OPAL_DATATYPE_UINT8:
-            OPAL_DATATYPE_MYPACK(UINT8);
-            break;
-        case OPAL_DATATYPE_UINT16:
-            OPAL_DATATYPE_MYPACK(UINT16);
-            break;
-        case OPAL_DATATYPE_FLOAT2:
-            OPAL_DATATYPE_MYPACK(FLOAT2);
-            break;
-        case OPAL_DATATYPE_FLOAT4:
-            OPAL_DATATYPE_MYPACK(FLOAT4);
-            break;
-        case OPAL_DATATYPE_FLOAT8:
-            OPAL_DATATYPE_MYPACK(FLOAT8);
-            break;
-        case OPAL_DATATYPE_FLOAT12:
-            OPAL_DATATYPE_MYPACK(FLOAT12);
-            break;
-        case OPAL_DATATYPE_FLOAT16:
-            OPAL_DATATYPE_MYPACK(FLOAT16);
-            break;
-        case OPAL_DATATYPE_SHORT_FLOAT_COMPLEX:
-            OPAL_DATATYPE_MYPACK(SHORT_FLOAT_COMPLEX);
-            break;
-        case OPAL_DATATYPE_FLOAT_COMPLEX:
-            OPAL_DATATYPE_MYPACK(FLOAT_COMPLEX);
-            break;
-        case OPAL_DATATYPE_DOUBLE_COMPLEX:
-            OPAL_DATATYPE_MYPACK(DOUBLE_COMPLEX);
-            break;
-        case OPAL_DATATYPE_LONG_DOUBLE_COMPLEX:
-            OPAL_DATATYPE_MYPACK(LONG_DOUBLE_COMPLEX);
-            break;
-        case OPAL_DATATYPE_BOOL:
-            OPAL_DATATYPE_MYPACK(BOOL);
-            break;
-        case OPAL_DATATYPE_WCHAR:
-            OPAL_DATATYPE_MYPACK(WCHAR);
-            break;
-        default:
-            return OPAL_ERROR;
+    switch (id) {
+    case OPAL_DATATYPE_INT1:
+        // The below macro should expand to
+        // OPAL_DATATYPE_PACK_PREDEFINED_ELEMENT(src, dest, cando_count, stride, blocklen, int8_t);
+        // by using OPAL_DATATYPE_HANDLE_* where it finds that INT1 means int8_t etc
+        OPAL_DATATYPE_MYPACK(INT1);
+        break;
+    case OPAL_DATATYPE_INT2:
+        OPAL_DATATYPE_MYPACK(INT2);
+        break;
+    case OPAL_DATATYPE_INT4:
+        OPAL_DATATYPE_MYPACK(INT4);
+        break;
+    case OPAL_DATATYPE_INT8:
+        OPAL_DATATYPE_MYPACK(INT8);
+        break;
+    case OPAL_DATATYPE_INT16:
+        OPAL_DATATYPE_MYPACK(INT16);
+        break;
+    case OPAL_DATATYPE_UINT1:
+        OPAL_DATATYPE_MYPACK(UINT1);
+        break;
+    case OPAL_DATATYPE_UINT2:
+        OPAL_DATATYPE_MYPACK(UINT2);
+        break;
+    case OPAL_DATATYPE_UINT4:
+        OPAL_DATATYPE_MYPACK(UINT4);
+        break;
+    case OPAL_DATATYPE_UINT8:
+        OPAL_DATATYPE_MYPACK(UINT8);
+        break;
+    case OPAL_DATATYPE_UINT16:
+        OPAL_DATATYPE_MYPACK(UINT16);
+        break;
+    case OPAL_DATATYPE_FLOAT2:
+        OPAL_DATATYPE_MYPACK(FLOAT2);
+        break;
+    case OPAL_DATATYPE_FLOAT4:
+        OPAL_DATATYPE_MYPACK(FLOAT4);
+        break;
+    case OPAL_DATATYPE_FLOAT8:
+        OPAL_DATATYPE_MYPACK(FLOAT8);
+        break;
+    case OPAL_DATATYPE_FLOAT12:
+        OPAL_DATATYPE_MYPACK(FLOAT12);
+        break;
+    case OPAL_DATATYPE_FLOAT16:
+        OPAL_DATATYPE_MYPACK(FLOAT16);
+        break;
+    case OPAL_DATATYPE_SHORT_FLOAT_COMPLEX:
+        OPAL_DATATYPE_MYPACK(SHORT_FLOAT_COMPLEX);
+        break;
+    case OPAL_DATATYPE_FLOAT_COMPLEX:
+        OPAL_DATATYPE_MYPACK(FLOAT_COMPLEX);
+        break;
+    case OPAL_DATATYPE_DOUBLE_COMPLEX:
+        OPAL_DATATYPE_MYPACK(DOUBLE_COMPLEX);
+        break;
+    case OPAL_DATATYPE_LONG_DOUBLE_COMPLEX:
+        OPAL_DATATYPE_MYPACK(LONG_DOUBLE_COMPLEX);
+        break;
+    case OPAL_DATATYPE_BOOL:
+        OPAL_DATATYPE_MYPACK(BOOL);
+        break;
+    case OPAL_DATATYPE_WCHAR:
+        OPAL_DATATYPE_MYPACK(WCHAR);
+        break;
+    default:
+        return OPAL_ERROR;
     }
     if (!success) {
         return OPAL_ERROR;
     }
 
-    *rtn_src  = src;
+    *rtn_src = src;
     *rtn_dest = dest;
     return OPAL_SUCCESS;
 }
diff --git a/opal/datatype/opal_datatype_position.c b/opal/datatype/opal_datatype_position.c
index 6dc38e4ff4a..3d82d73c0c6 100644
--- a/opal/datatype/opal_datatype_position.c
+++ b/opal/datatype/opal_datatype_position.c
@@ -26,17 +26,20 @@
 #include <stddef.h>
 #include <stdlib.h>
 
-#include "opal/datatype/opal_datatype.h"
 #include "opal/datatype/opal_convertor.h"
+#include "opal/datatype/opal_datatype.h"
 #include "opal/datatype/opal_datatype_internal.h"
 
 #if OPAL_ENABLE_DEBUG
-#include "opal/util/output.h"
+#    include "opal/util/output.h"
 
-#define DO_DEBUG(INST)  if( opal_ddt_position_debug ) { INST }
+#    define DO_DEBUG(INST)             \
+        if (opal_ddt_position_debug) { \
+            INST                       \
+        }
 #else
-#define DO_DEBUG(INST)
-#endif  /* OPAL_ENABLE_DEBUG */
+#    define DO_DEBUG(INST)
+#endif /* OPAL_ENABLE_DEBUG */
 
 /* The pack/unpack functions need a cleanup. I have to create a proper interface to access
  * all basic functionalities, hence using them as basic blocks for all conversion functions.
@@ -44,75 +47,77 @@
  * But first let's make some global assumptions:
  * - a datatype (with the flag DT_DATA set) will have the contiguous flags set if and only if
  *   the data is really contiguous (extent equal with size)
- * - for the OPAL_DATATYPE_LOOP type the DT_CONTIGUOUS flag set means that the content of the loop is
- *   contiguous but with a gap in the begining or at the end.
+ * - for the OPAL_DATATYPE_LOOP type the DT_CONTIGUOUS flag set means that the content of the loop
+ * is contiguous but with a gap in the beginning or at the end.
  * - the DT_CONTIGUOUS flag for the type OPAL_DATATYPE_END_LOOP is meaningless.
  */
 
-static inline void
-position_single_block(opal_convertor_t* CONVERTOR,
-                      unsigned char** mem, ptrdiff_t mem_update,
-                      size_t* space, size_t space_update,
-                      size_t* cnt, size_t cnt_update)
+static inline void position_single_block(opal_convertor_t *CONVERTOR, unsigned char **mem,
+                                         ptrdiff_t mem_update, size_t *space, size_t space_update,
+                                         size_t *cnt, size_t cnt_update)
 {
-    OPAL_DATATYPE_SAFEGUARD_POINTER( *mem, mem_update, (CONVERTOR)->pBaseBuf,
-                                     (CONVERTOR)->pDesc, (CONVERTOR)->count );
-    DO_DEBUG( opal_output( 0, "position( %p, %lu ) => space %lu [prolog]\n",
-                           (void*)*mem, (unsigned long)space_update, (unsigned long)(*space) ); );
-    *mem   += mem_update;
+    OPAL_DATATYPE_SAFEGUARD_POINTER(*mem, mem_update, (CONVERTOR)->pBaseBuf, (CONVERTOR)->pDesc,
+                                    (CONVERTOR)->count);
+    DO_DEBUG(opal_output(0, "position( %p, %lu ) => space %lu [prolog]\n", (void *) *mem,
+                         (unsigned long) space_update, (unsigned long) (*space)););
+    *mem += mem_update;
     *space -= space_update;
-    *cnt   -= cnt_update;
+    *cnt -= cnt_update;
 }
 
 /**
- * Advance the convertors' position according. Update the pointer and the remaining space
- * accordingly.
+ * Advance the convertors' position according to account for *COUNT elements. Update
+ * the pointer and the remaining space accordingly.
  */
-static inline void
-position_predefined_data( opal_convertor_t* CONVERTOR,
-                          dt_elem_desc_t* ELEM,
-                          size_t* COUNT,
-                          unsigned char** POINTER,
-                          size_t* SPACE )
+static inline void position_predefined_data(opal_convertor_t *CONVERTOR, dt_elem_desc_t *ELEM,
+                                            size_t *COUNT, unsigned char **POINTER, size_t *SPACE)
 {
-    const ddt_elem_desc_t* _elem = &((ELEM)->elem);
-    size_t total_count = (size_t)_elem->count * _elem->blocklen;
+    const ddt_elem_desc_t *_elem = &((ELEM)->elem);
+    size_t total_count = (size_t) _elem->count * _elem->blocklen;
     size_t cando_count = (*SPACE) / opal_datatype_basicDatatypes[_elem->common.type]->size;
     size_t do_now, do_now_bytes = opal_datatype_basicDatatypes[_elem->common.type]->size;
-    unsigned char* _memory = (*POINTER) + _elem->disp;
+    unsigned char *_memory = (*POINTER) + _elem->disp;
 
-    assert( *(COUNT) <= ((size_t)_elem->count * _elem->blocklen) );
+    assert(*(COUNT) <= ((size_t) _elem->count * _elem->blocklen));
 
-    if( cando_count > *(COUNT) )
+    if (cando_count > *(COUNT)) {
         cando_count = *(COUNT);
+    } else if( 0 == cando_count )
+        return;
 
-    if( 1 == _elem->blocklen ) {
-        DO_DEBUG( opal_output( 0, "position( %p, %" PRIsize_t " ) x (count %" PRIsize_t ", extent %ld) => space %lu [prolog]\n",
-                               (void*)_memory, (unsigned long)do_now_bytes, cando_count, _elem->extent, (unsigned long)(*SPACE) ); );
-        _memory     += cando_count * _elem->extent;
-        *SPACE      -= cando_count * do_now_bytes;
-        *COUNT      -= cando_count;
+    if (1 == _elem->blocklen) {
+        DO_DEBUG(opal_output(0,
+                             "position( %p, %" PRIsize_t " ) x (count %" PRIsize_t
+                             ", extent %ld) => space %lu [prolog]\n",
+                             (void *) _memory, (unsigned long) do_now_bytes, cando_count,
+                             _elem->extent, (unsigned long) (*SPACE)););
+        _memory += cando_count * _elem->extent;
+        *SPACE -= cando_count * do_now_bytes;
+        *COUNT -= cando_count;
         goto update_and_return;
     }
 
     /**
      * First check if we already did something on this element ?
      */
-    do_now = (total_count - *(COUNT));  /* done elements */
-    if( 0 != do_now ) {
-        do_now = do_now % _elem->blocklen;  /* partial blocklen? */
+    do_now = (total_count - *(COUNT)); /* done elements */
+    if (0 != do_now) {
+        do_now = do_now % _elem->blocklen; /* partial blocklen? */
 
-        if( 0 != do_now ) {
-            size_t left_in_block = _elem->blocklen - do_now;  /* left in the current blocklen */
-            do_now = (left_in_block > cando_count ) ? cando_count : left_in_block;
+        if (0 != do_now) {
+            size_t left_in_block = _elem->blocklen - do_now; /* left in the current blocklen */
+            do_now = (left_in_block > cando_count) ? cando_count : left_in_block;
             do_now_bytes = do_now * opal_datatype_basicDatatypes[_elem->common.type]->size;
 
-            position_single_block( CONVERTOR, &_memory, do_now_bytes,
-                                   SPACE, do_now_bytes, COUNT, do_now );
+            position_single_block(CONVERTOR, &_memory, do_now_bytes, SPACE, do_now_bytes, COUNT,
+                                  do_now);
 
             /* compensate if we just completed a blocklen */
-            if( do_now == left_in_block )
-                _memory += _elem->extent - (_elem->blocklen * opal_datatype_basicDatatypes[_elem->common.type]->size);
+            if (do_now == left_in_block) {
+                _memory += _elem->extent
+                           - (_elem->blocklen
+                              * opal_datatype_basicDatatypes[_elem->common.type]->size);
+            }
             cando_count -= do_now;
         }
     }
@@ -121,49 +126,49 @@ position_predefined_data( opal_convertor_t* CONVERTOR,
      * Compute how many full blocklen we need to do and do them.
      */
     do_now = cando_count / _elem->blocklen;
-    if( 0 != do_now ) {
+    if (0 != do_now) {
         do_now_bytes = _elem->blocklen * opal_datatype_basicDatatypes[_elem->common.type]->size;
 #if OPAL_ENABLE_DEBUG
-        for(size_t _i = 0; _i < do_now; _i++ ) {
-            position_single_block( CONVERTOR, &_memory, _elem->extent,
-                                   SPACE, do_now_bytes, COUNT, _elem->blocklen );
+        for (size_t _i = 0; _i < do_now; _i++) {
+            position_single_block(CONVERTOR, &_memory, _elem->extent, SPACE, do_now_bytes, COUNT,
+                                  _elem->blocklen);
             cando_count -= _elem->blocklen;
         }
 #else
-        _memory     += do_now * _elem->extent;
-        *SPACE      -= do_now * do_now_bytes;
-        *COUNT      -= do_now * _elem->blocklen;
+        _memory += do_now * _elem->extent;
+        *SPACE -= do_now * do_now_bytes;
+        *COUNT -= do_now * _elem->blocklen;
         cando_count -= do_now * _elem->blocklen;
-#endif  /* OPAL_ENABLE_DEBUG */
+#endif /* OPAL_ENABLE_DEBUG */
     }
 
     /**
      * As an epilog do anything left from the last blocklen.
      */
     do_now = cando_count;
-    if( 0 != do_now ) {
+    if (0 != do_now) {
         do_now_bytes = do_now * opal_datatype_basicDatatypes[_elem->common.type]->size;
-        position_single_block( CONVERTOR, &_memory, do_now_bytes,
-                               SPACE, do_now_bytes, COUNT, do_now );
+        position_single_block(CONVERTOR, &_memory, do_now_bytes, SPACE, do_now_bytes, COUNT,
+                              do_now);
     }
 
- update_and_return:
-    *(POINTER)  = _memory - _elem->disp;
+update_and_return:
+    *(POINTER) = _memory - _elem->disp;
 }
 
-int opal_convertor_generic_simple_position( opal_convertor_t* pConvertor,
-                                            size_t* position )
+int opal_convertor_generic_simple_position(opal_convertor_t *pConvertor, size_t *position)
 {
-    dt_stack_t* pStack;       /* pointer to the position on the stack */
-    uint32_t pos_desc;        /* actual position in the description of the derived datatype */
-    size_t count_desc;       /* the number of items already done in the actual pos_desc */
+    dt_stack_t *pStack; /* pointer to the position on the stack */
+    uint32_t pos_desc;  /* actual position in the description of the derived datatype */
+    size_t count_desc;  /* the number of items already done in the actual pos_desc */
     size_t iov_len_local;
-    dt_elem_desc_t* description = pConvertor->use_desc->desc;
-    dt_elem_desc_t* pElem;    /* current position */
+    dt_elem_desc_t *description = pConvertor->use_desc->desc;
+    dt_elem_desc_t *pElem; /* current position */
     unsigned char *base_pointer = pConvertor->pBaseBuf;
     ptrdiff_t extent = pConvertor->pDesc->ub - pConvertor->pDesc->lb;
 
-    DUMP( "opal_convertor_generic_simple_position( %p, &%ld )\n", (void*)pConvertor, (long)*position );
+    DUMP("opal_convertor_generic_simple_position( %p, &%ld )\n", (void *) pConvertor,
+         (long) *position);
     assert(*position > pConvertor->bConverted);
 
     /* We dont want to have to parse the datatype multiple times. What we are interested in
@@ -172,42 +177,50 @@ int opal_convertor_generic_simple_position( opal_convertor_t* pConvertor,
      * The only problem is that we have to modify all the elements on the stack.
      */
     iov_len_local = *position - pConvertor->bConverted;
-    if( iov_len_local > pConvertor->pDesc->size ) {
-        pStack = pConvertor->pStack;  /* we're working with the full stack */
+    if (iov_len_local > pConvertor->pDesc->size) {
+        pStack = pConvertor->pStack; /* we're working with the full stack */
         count_desc = iov_len_local / pConvertor->pDesc->size;
-        DO_DEBUG( opal_output( 0, "position before %lu asked %lu data size %lu"
-                               " iov_len_local %lu count_desc %" PRIsize_t "\n",
-                               (unsigned long)pConvertor->bConverted, (unsigned long)*position, (unsigned long)pConvertor->pDesc->size,
-                               (unsigned long)iov_len_local, count_desc ); );
+        DO_DEBUG(opal_output(0,
+                             "position before %lu asked %lu data size %lu"
+                             " iov_len_local %lu count_desc %" PRIsize_t "\n",
+                             (unsigned long) pConvertor->bConverted, (unsigned long) *position,
+                             (unsigned long) pConvertor->pDesc->size, (unsigned long) iov_len_local,
+                             count_desc););
         /* Update all the stack including the last one */
-        for( pos_desc = 0; pos_desc <= pConvertor->stack_pos; pos_desc++ )
+        for (pos_desc = 0; pos_desc <= pConvertor->stack_pos; pos_desc++) {
             pStack[pos_desc].disp += count_desc * extent;
+        }
         pConvertor->bConverted += count_desc * pConvertor->pDesc->size;
         iov_len_local = *position - pConvertor->bConverted;
         pStack[0].count -= count_desc;
-        DO_DEBUG( opal_output( 0, "after bConverted %lu remaining count %lu iov_len_local %lu\n",
-                               (unsigned long)pConvertor->bConverted, (unsigned long)pStack[0].count, (unsigned long)iov_len_local ); );
+        DO_DEBUG(opal_output(0, "after bConverted %lu remaining count %lu iov_len_local %lu\n",
+                             (unsigned long) pConvertor->bConverted,
+                             (unsigned long) pStack[0].count, (unsigned long) iov_len_local););
     }
 
     pStack = pConvertor->pStack + pConvertor->stack_pos;
-    pos_desc      = pStack->index;
+    pos_desc = pStack->index;
     base_pointer += pStack->disp;
-    count_desc    = pStack->count;
+    count_desc = pStack->count;
     pStack--;
     pConvertor->stack_pos--;
     pElem = &(description[pos_desc]);
 
-    DO_DEBUG( opal_output( 0, "position start pos_desc %d count_desc %" PRIsize_t " disp %llx\n"
-                           "stack_pos %d pos_desc %d count_desc %" PRIsize_t " disp %llx\n",
-                           pos_desc, count_desc, (unsigned long long)(base_pointer - pConvertor->pBaseBuf),
-                           pConvertor->stack_pos, pStack->index, pStack->count, (unsigned long long)pStack->disp ); );
+    DO_DEBUG(opal_output(0,
+                         "position start pos_desc %d count_desc %" PRIsize_t " disp %llx\n"
+                         "stack_pos %d pos_desc %d count_desc %" PRIsize_t " disp %llx\n",
+                         pos_desc, count_desc,
+                         (unsigned long long) (base_pointer - pConvertor->pBaseBuf),
+                         pConvertor->stack_pos, pStack->index, pStack->count,
+                         (unsigned long long) pStack->disp););
     /* Last data has been only partially converted. Compute the relative position */
-    if( 0 != pConvertor->partial_length ) {
+    if (0 != pConvertor->partial_length) {
         size_t element_length = opal_datatype_basicDatatypes[pElem->elem.common.type]->size;
         size_t missing_length = element_length - pConvertor->partial_length;
-        if( missing_length >= iov_len_local ) {
-            pConvertor->partial_length = (pConvertor->partial_length + iov_len_local) % element_length;
-            pConvertor->bConverted    += iov_len_local;
+        if (missing_length >= iov_len_local) {
+            pConvertor->partial_length = (pConvertor->partial_length + iov_len_local)
+                                         % element_length;
+            pConvertor->bConverted += iov_len_local;
             assert(pConvertor->partial_length < element_length);
             return 0;
         }
@@ -216,89 +229,101 @@ int opal_convertor_generic_simple_position( opal_convertor_t* pConvertor,
         iov_len_local -= missing_length;
         count_desc--;
     }
-    while( 1 ) {
-        if( OPAL_DATATYPE_END_LOOP == pElem->elem.common.type ) { /* end of the the entire datatype */
-            DO_DEBUG( opal_output( 0, "position end_loop count %" PRIsize_t " stack_pos %d pos_desc %d disp %lx space %lu\n",
-                                   pStack->count, pConvertor->stack_pos, pos_desc,
-                                   pStack->disp, (unsigned long)iov_len_local ); );
-            if( --(pStack->count) == 0 ) { /* end of loop */
-                if( pConvertor->stack_pos == 0 ) {
+    while (1) {
+        if (OPAL_DATATYPE_END_LOOP
+            == pElem->elem.common.type) { /* end of the the entire datatype */
+            DO_DEBUG(opal_output(0,
+                                 "position end_loop count %" PRIsize_t
+                                 " stack_pos %d pos_desc %d disp %lx space %lu\n",
+                                 pStack->count, pConvertor->stack_pos, pos_desc, pStack->disp,
+                                 (unsigned long) iov_len_local););
+            if (--(pStack->count) == 0) { /* end of loop */
+                if (pConvertor->stack_pos == 0) {
                     pConvertor->flags |= CONVERTOR_COMPLETED;
-                    goto complete_loop;  /* completed */
+                    goto complete_loop; /* completed */
                 }
                 pConvertor->stack_pos--;
                 pStack--;
                 pos_desc++;
             } else {
-                if( pStack->index == -1 ) {
+                if (pStack->index == -1) {
                     pStack->disp += extent;
-                    pos_desc = 0;  /* back to the first element */
+                    pos_desc = 0; /* back to the first element */
                 } else {
-                    assert( OPAL_DATATYPE_LOOP == description[pStack->index].loop.common.type );
+                    assert(OPAL_DATATYPE_LOOP == description[pStack->index].loop.common.type);
                     pStack->disp += description[pStack->index].loop.extent;
-                    pos_desc = pStack->index;  /* go back to the loop start itself to give a chance 
-                                                * to move forward by entire loops */
+                    pos_desc = pStack->index; /* go back to the loop start itself to give a chance
+                                               * to move forward by entire loops */
                 }
             }
             base_pointer = pConvertor->pBaseBuf + pStack->disp;
-            UPDATE_INTERNAL_COUNTERS( description, pos_desc, pElem, count_desc );
-            DO_DEBUG( opal_output( 0, "position new_loop count %" PRIsize_t " stack_pos %d pos_desc %d disp %lx space %lu\n",
-                                   pStack->count, pConvertor->stack_pos, pos_desc,
-                                   pStack->disp, (unsigned long)iov_len_local ); );
+            UPDATE_INTERNAL_COUNTERS(description, pos_desc, pElem, count_desc);
+            DO_DEBUG(opal_output(0,
+                                 "position new_loop count %" PRIsize_t
+                                 " stack_pos %d pos_desc %d disp %lx space %lu\n",
+                                 pStack->count, pConvertor->stack_pos, pos_desc, pStack->disp,
+                                 (unsigned long) iov_len_local););
         }
-        if( OPAL_DATATYPE_LOOP == pElem->elem.common.type ) {
-            ptrdiff_t local_disp = (ptrdiff_t)base_pointer;
-            ddt_endloop_desc_t* end_loop = (ddt_endloop_desc_t*)(pElem + pElem->loop.items);
+        if (OPAL_DATATYPE_LOOP == pElem->elem.common.type) {
+            ptrdiff_t local_disp = (ptrdiff_t) base_pointer;
+            ddt_endloop_desc_t *end_loop = (ddt_endloop_desc_t *) (pElem + pElem->loop.items);
             size_t full_loops = iov_len_local / end_loop->size;
             full_loops = count_desc <= full_loops ? count_desc : full_loops;
-            if( full_loops ) {
-                base_pointer  += full_loops * pElem->loop.extent;
+            if (full_loops) {
+                base_pointer += full_loops * pElem->loop.extent;
                 iov_len_local -= full_loops * end_loop->size;
-                count_desc    -= full_loops;
+                count_desc -= full_loops;
 
-                if( 0 == count_desc ) {  /* completed */
+                if (0 == count_desc) { /* completed */
                     pos_desc += pElem->loop.items + 1;
                     goto update_loop_description;
                 }
                 /* Save the stack with the correct last_count value. */
             }
-            local_disp = (ptrdiff_t)base_pointer - local_disp;
-            PUSH_STACK( pStack, pConvertor->stack_pos, pos_desc, OPAL_DATATYPE_LOOP, count_desc,
-                        pStack->disp + local_disp );
+            local_disp = (ptrdiff_t) base_pointer - local_disp;
+            PUSH_STACK(pStack, pConvertor->stack_pos, pos_desc, OPAL_DATATYPE_LOOP, count_desc,
+                       pStack->disp + local_disp);
             pos_desc++;
-        update_loop_description:  /* update the current state */
+        update_loop_description: /* update the current state */
             base_pointer = pConvertor->pBaseBuf + pStack->disp;
-            UPDATE_INTERNAL_COUNTERS( description, pos_desc, pElem, count_desc );
-            DDT_DUMP_STACK( pConvertor->pStack, pConvertor->stack_pos, pElem, "advance loop" );
-            DO_DEBUG( opal_output( 0, "position set loop count %" PRIsize_t " stack_pos %d pos_desc %d disp %lx space %lu\n",
-                                   pStack->count, pConvertor->stack_pos, pos_desc,
-                                   pStack->disp, (unsigned long)iov_len_local ); );
+            UPDATE_INTERNAL_COUNTERS(description, pos_desc, pElem, count_desc);
+            DDT_DUMP_STACK(pConvertor->pStack, pConvertor->stack_pos, pElem, "advance loop");
+            DO_DEBUG(opal_output(0,
+                                 "position set loop count %" PRIsize_t
+                                 " stack_pos %d pos_desc %d disp %lx space %lu\n",
+                                 pStack->count, pConvertor->stack_pos, pos_desc, pStack->disp,
+                                 (unsigned long) iov_len_local););
             continue;
         }
-        while( pElem->elem.common.flags & OPAL_DATATYPE_FLAG_DATA ) {
+        while (pElem->elem.common.flags & OPAL_DATATYPE_FLAG_DATA) {
             /* now here we have a basic datatype */
-            position_predefined_data( pConvertor, pElem, &count_desc, &base_pointer, &iov_len_local );
-            if( 0 != count_desc ) {  /* completed */
+            position_predefined_data(pConvertor, pElem, &count_desc, &base_pointer, &iov_len_local);
+            if (0 != count_desc) { /* completed */
                 pConvertor->partial_length = iov_len_local;
                 goto complete_loop;
             }
             base_pointer = pConvertor->pBaseBuf + pStack->disp;
-            pos_desc++;  /* advance to the next data */
-            UPDATE_INTERNAL_COUNTERS( description, pos_desc, pElem, count_desc );
-            DO_DEBUG( opal_output( 0, "position set loop count %" PRIsize_t " stack_pos %d pos_desc %d disp %lx space %lu\n",
-                                   pStack->count, pConvertor->stack_pos, pos_desc,
-                                   pStack->disp, (unsigned long)iov_len_local ); );
+            pos_desc++; /* advance to the next data */
+            UPDATE_INTERNAL_COUNTERS(description, pos_desc, pElem, count_desc);
+            DO_DEBUG(opal_output(0,
+                                 "position set loop count %" PRIsize_t
+                                 " stack_pos %d pos_desc %d disp %lx space %lu\n",
+                                 pStack->count, pConvertor->stack_pos, pos_desc, pStack->disp,
+                                 (unsigned long) iov_len_local););
         }
     }
- complete_loop:
-    pConvertor->bConverted = *position;  /* update the already converted bytes */
+complete_loop:
+    pConvertor->bConverted = *position; /* update the already converted bytes */
 
-    if( !(pConvertor->flags & CONVERTOR_COMPLETED) ) {
+    if (!(pConvertor->flags & CONVERTOR_COMPLETED)) {
         /* I complete an element, next step I should go to the next one */
-        PUSH_STACK( pStack, pConvertor->stack_pos, pos_desc, pElem->elem.common.type, count_desc,
-                    base_pointer - pConvertor->pBaseBuf );
-        DO_DEBUG( opal_output( 0, "position save stack stack_pos %d pos_desc %d count_desc %" PRIsize_t " disp %llx\n",
-                               pConvertor->stack_pos, pStack->index, pStack->count, (unsigned long long)pStack->disp ); );
+        PUSH_STACK(pStack, pConvertor->stack_pos, pos_desc, pElem->elem.common.type, count_desc,
+                   base_pointer - pConvertor->pBaseBuf);
+        DO_DEBUG(opal_output(0,
+                             "position save stack stack_pos %d pos_desc %d count_desc %" PRIsize_t
+                             " disp %llx\n",
+                             pConvertor->stack_pos, pStack->index, pStack->count,
+                             (unsigned long long) pStack->disp););
         return 0;
     }
     return 1;
diff --git a/opal/datatype/opal_datatype_prototypes.h b/opal/datatype/opal_datatype_prototypes.h
index 668397112b8..4a0e1188bd2 100644
--- a/opal/datatype/opal_datatype_prototypes.h
+++ b/opal/datatype/opal_datatype_prototypes.h
@@ -16,74 +16,45 @@
 
 #include "opal_config.h"
 
-
 BEGIN_C_DECLS
 
 /*
  * First the public ones
  */
 
-OPAL_DECLSPEC int32_t
-opal_pack_general( opal_convertor_t* pConvertor,
-                   struct iovec* iov, uint32_t* out_size,
-                   size_t* max_data );
-OPAL_DECLSPEC int32_t
-opal_pack_general_checksum( opal_convertor_t* pConvertor,
-                            struct iovec* iov, uint32_t* out_size,
-                            size_t* max_data );
-OPAL_DECLSPEC int32_t
-opal_unpack_general( opal_convertor_t* pConvertor,
-                     struct iovec* iov, uint32_t* out_size,
-                     size_t* max_data );
-OPAL_DECLSPEC int32_t
-opal_unpack_general_checksum( opal_convertor_t* pConvertor,
-                              struct iovec* iov, uint32_t* out_size,
-                              size_t* max_data );
+OPAL_DECLSPEC int32_t opal_pack_general(opal_convertor_t *pConvertor, struct iovec *iov,
+                                        uint32_t *out_size, size_t *max_data);
+OPAL_DECLSPEC int32_t opal_pack_general_checksum(opal_convertor_t *pConvertor, struct iovec *iov,
+                                                 uint32_t *out_size, size_t *max_data);
+OPAL_DECLSPEC int32_t opal_unpack_general(opal_convertor_t *pConvertor, struct iovec *iov,
+                                          uint32_t *out_size, size_t *max_data);
+OPAL_DECLSPEC int32_t opal_unpack_general_checksum(opal_convertor_t *pConvertor, struct iovec *iov,
+                                                   uint32_t *out_size, size_t *max_data);
 
 /*
  * Now the internal functions
  */
-int32_t
-opal_pack_homogeneous_contig( opal_convertor_t* pConv,
-                          struct iovec* iov, uint32_t* out_size,
-                          size_t* max_data );
-int32_t
-opal_pack_homogeneous_contig_checksum( opal_convertor_t* pConv,
-                                   struct iovec* iov, uint32_t* out_size,
-                                   size_t* max_data );
-int32_t
-opal_pack_homogeneous_contig_with_gaps( opal_convertor_t* pConv,
-                                    struct iovec* iov, uint32_t* out_size,
-                                    size_t* max_data );
-int32_t
-opal_pack_homogeneous_contig_with_gaps_checksum( opal_convertor_t* pConv,
-                                             struct iovec* iov, uint32_t* out_size,
-                                             size_t* max_data );
-int32_t
-opal_generic_simple_pack( opal_convertor_t* pConvertor,
-                          struct iovec* iov, uint32_t* out_size,
-                          size_t* max_data );
-int32_t
-opal_generic_simple_pack_checksum( opal_convertor_t* pConvertor,
-                                   struct iovec* iov, uint32_t* out_size,
-                                   size_t* max_data );
-int32_t
-opal_unpack_homogeneous_contig( opal_convertor_t* pConv,
-                                struct iovec* iov, uint32_t* out_size,
-                                size_t* max_data );
-int32_t
-opal_unpack_homogeneous_contig_checksum( opal_convertor_t* pConv,
-                                         struct iovec* iov, uint32_t* out_size,
-                                         size_t* max_data );
-int32_t
-opal_generic_simple_unpack( opal_convertor_t* pConvertor,
-                            struct iovec* iov, uint32_t* out_size,
-                            size_t* max_data );
-int32_t
-opal_generic_simple_unpack_checksum( opal_convertor_t* pConvertor,
-                                     struct iovec* iov, uint32_t* out_size,
-                                     size_t* max_data );
+int32_t opal_pack_homogeneous_contig(opal_convertor_t *pConv, struct iovec *iov, uint32_t *out_size,
+                                     size_t *max_data);
+int32_t opal_pack_homogeneous_contig_checksum(opal_convertor_t *pConv, struct iovec *iov,
+                                              uint32_t *out_size, size_t *max_data);
+int32_t opal_pack_homogeneous_contig_with_gaps(opal_convertor_t *pConv, struct iovec *iov,
+                                               uint32_t *out_size, size_t *max_data);
+int32_t opal_pack_homogeneous_contig_with_gaps_checksum(opal_convertor_t *pConv, struct iovec *iov,
+                                                        uint32_t *out_size, size_t *max_data);
+int32_t opal_generic_simple_pack(opal_convertor_t *pConvertor, struct iovec *iov,
+                                 uint32_t *out_size, size_t *max_data);
+int32_t opal_generic_simple_pack_checksum(opal_convertor_t *pConvertor, struct iovec *iov,
+                                          uint32_t *out_size, size_t *max_data);
+int32_t opal_unpack_homogeneous_contig(opal_convertor_t *pConv, struct iovec *iov,
+                                       uint32_t *out_size, size_t *max_data);
+int32_t opal_unpack_homogeneous_contig_checksum(opal_convertor_t *pConv, struct iovec *iov,
+                                                uint32_t *out_size, size_t *max_data);
+int32_t opal_generic_simple_unpack(opal_convertor_t *pConvertor, struct iovec *iov,
+                                   uint32_t *out_size, size_t *max_data);
+int32_t opal_generic_simple_unpack_checksum(opal_convertor_t *pConvertor, struct iovec *iov,
+                                            uint32_t *out_size, size_t *max_data);
 
 END_C_DECLS
 
-#endif  /* OPAL_DATATYPE_PROTOTYPES_H_HAS_BEEN_INCLUDED */
+#endif /* OPAL_DATATYPE_PROTOTYPES_H_HAS_BEEN_INCLUDED */
diff --git a/opal/datatype/opal_datatype_resize.c b/opal/datatype/opal_datatype_resize.c
index 81b3c414243..fa9a2d794bc 100644
--- a/opal/datatype/opal_datatype_resize.c
+++ b/opal/datatype/opal_datatype_resize.c
@@ -19,7 +19,7 @@
 #include "opal/datatype/opal_datatype.h"
 #include "opal/datatype/opal_datatype_internal.h"
 
-int32_t opal_datatype_resize( opal_datatype_t* type, ptrdiff_t lb, ptrdiff_t extent )
+int32_t opal_datatype_resize(opal_datatype_t *type, ptrdiff_t lb, ptrdiff_t extent)
 {
     type->lb = lb;
     type->ub = lb + extent;
@@ -27,8 +27,7 @@ int32_t opal_datatype_resize( opal_datatype_t* type, ptrdiff_t lb, ptrdiff_t ext
     type->flags &= ~OPAL_DATATYPE_FLAG_NO_GAPS;
     type->flags |= OPAL_DATATYPE_FLAG_USER_LB;
     type->flags |= OPAL_DATATYPE_FLAG_USER_UB;
-    if( (extent == (ptrdiff_t)type->size) &&
-        (type->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS) ) {
+    if ((extent == (ptrdiff_t) type->size) && (type->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS)) {
         type->flags |= OPAL_DATATYPE_FLAG_NO_GAPS;
     }
     return OPAL_SUCCESS;
diff --git a/opal/datatype/opal_datatype_unpack.c b/opal/datatype/opal_datatype_unpack.c
index 6f9fdce2774..e1f2d93acbf 100644
--- a/opal/datatype/opal_datatype_unpack.c
+++ b/opal/datatype/opal_datatype_unpack.c
@@ -15,6 +15,8 @@
  * Copyright (c) 2013      Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2017-2018 Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -29,29 +31,33 @@
 
 #include "opal/datatype/opal_convertor_internal.h"
 #include "opal/datatype/opal_datatype_internal.h"
+#include "opal/mca/accelerator/accelerator.h"
+#include "opal/mca/accelerator/base/base.h"
 
 #if OPAL_ENABLE_DEBUG
-#include "opal/util/output.h"
+#    include "opal/util/output.h"
 
-#define DO_DEBUG(INST)  if( opal_ddt_unpack_debug ) { INST }
+#    define DO_DEBUG(INST)           \
+        if (opal_ddt_unpack_debug) { \
+            INST                     \
+        }
 #else
-#define DO_DEBUG(INST)
-#endif  /* OPAL_ENABLE_DEBUG */
+#    define DO_DEBUG(INST)
+#endif /* OPAL_ENABLE_DEBUG */
 
 #include "opal/datatype/opal_datatype_checksum.h"
-#include "opal/datatype/opal_datatype_unpack.h"
 #include "opal/datatype/opal_datatype_prototypes.h"
+#include "opal/datatype/opal_datatype_unpack.h"
 
 #if defined(CHECKSUM)
-#define opal_unpack_general_function            opal_unpack_general_checksum
-#define opal_unpack_homogeneous_contig_function opal_unpack_homogeneous_contig_checksum
-#define opal_generic_simple_unpack_function     opal_generic_simple_unpack_checksum
+#    define opal_unpack_general_function            opal_unpack_general_checksum
+#    define opal_unpack_homogeneous_contig_function opal_unpack_homogeneous_contig_checksum
+#    define opal_generic_simple_unpack_function     opal_generic_simple_unpack_checksum
 #else
-#define opal_unpack_general_function            opal_unpack_general
-#define opal_unpack_homogeneous_contig_function opal_unpack_homogeneous_contig
-#define opal_generic_simple_unpack_function     opal_generic_simple_unpack
-#endif  /* defined(CHECKSUM) */
-
+#    define opal_unpack_general_function            opal_unpack_general
+#    define opal_unpack_homogeneous_contig_function opal_unpack_homogeneous_contig
+#    define opal_generic_simple_unpack_function     opal_generic_simple_unpack
+#endif /* defined(CHECKSUM) */
 
 /**
  * This function will be used to unpack all datatypes that have the contiguous flag set.
@@ -62,90 +68,102 @@
  * taken in account directly in the opal_convertor_unpack function (in convertor.c) for
  * the homogeneous case.
  */
-int32_t
-opal_unpack_homogeneous_contig_function( opal_convertor_t* pConv,
-                                         struct iovec* iov,
-                                         uint32_t* out_size,
-                                         size_t* max_data )
+int32_t opal_unpack_homogeneous_contig_function(opal_convertor_t *pConv, struct iovec *iov,
+                                                uint32_t *out_size, size_t *max_data)
 {
     const opal_datatype_t *pData = pConv->pDesc;
     unsigned char *user_memory, *packed_buffer;
-    uint32_t iov_idx, i;
+    uint32_t iov_idx;
+    uint32_t __opal_attribute_unused__ i;
     size_t remaining, initial_bytes_converted = pConv->bConverted;
-    dt_stack_t* stack = pConv->pStack;
+    dt_stack_t *stack = pConv->pStack;
     ptrdiff_t extent = pData->ub - pData->lb;
 
-    DO_DEBUG( opal_output( 0, "unpack_homogeneous_contig( pBaseBuf %p, iov count %d )\n",
-                           (void*)pConv->pBaseBuf, *out_size ); );
-    if( stack[1].type != opal_datatype_uint1.id ) {
+    DO_DEBUG(opal_output(0, "unpack_homogeneous_contig( pBaseBuf %p, iov count %d )\n",
+                         (void *) pConv->pBaseBuf, *out_size););
+    if (stack[1].type != opal_datatype_uint1.id) {
         stack[1].count *= opal_datatype_basicDatatypes[stack[1].type]->size;
         stack[1].type = opal_datatype_uint1.id;
     }
 
-    if( (ptrdiff_t)pData->size == extent ) {
-        for( iov_idx = 0; iov_idx < (*out_size); iov_idx++ ) {
+    if ((ptrdiff_t) pData->size == extent) {
+        for (iov_idx = 0; iov_idx < (*out_size); iov_idx++) {
             remaining = pConv->local_size - pConv->bConverted;
-            if( 0 == remaining ) break;  /* we're done this time */
-            if( remaining > iov[iov_idx].iov_len )
+            if (0 == remaining) {
+                break; /* we're done this time */
+            }
+            if (remaining > iov[iov_idx].iov_len) {
                 remaining = iov[iov_idx].iov_len;
+            }
 
-            packed_buffer = (unsigned char*)iov[iov_idx].iov_base;
+            packed_buffer = (unsigned char *) iov[iov_idx].iov_base;
             user_memory = pConv->pBaseBuf + pData->true_lb + pConv->bConverted;
 
             /* contiguous data or basic datatype with count */
-            OPAL_DATATYPE_SAFEGUARD_POINTER( user_memory, remaining,
-                                             pConv->pBaseBuf, pData, pConv->count );
-            DO_DEBUG( opal_output( 0, "unpack contig [%d] dest %p src %p length %" PRIsize_t "\n",
-                                   iov_idx, (void*)user_memory, (void*)packed_buffer, remaining ); );
-            MEMCPY_CSUM( user_memory, packed_buffer, remaining, pConv );
+            OPAL_DATATYPE_SAFEGUARD_POINTER(user_memory, remaining, pConv->pBaseBuf, pData,
+                                            pConv->count);
+            DO_DEBUG(opal_output(0, "unpack contig [%d] dest %p src %p length %" PRIsize_t "\n",
+                                 iov_idx, (void *) user_memory, (void *) packed_buffer,
+                                 remaining););
+            MEMCPY_CSUM(user_memory, packed_buffer, remaining, pConv);
             pConv->bConverted += remaining; /* how much will get unpacked this time */
         }
     } else {
-        for( iov_idx = 0; iov_idx < (*out_size); iov_idx++ ) {
+        for (iov_idx = 0; iov_idx < (*out_size); iov_idx++) {
             remaining = pConv->local_size - pConv->bConverted;
-            if( 0 == remaining ) break;  /* we're done this time */
-            if( remaining > iov[iov_idx].iov_len )
+            if (0 == remaining) {
+                break; /* we're done this time */
+            }
+            if (remaining > iov[iov_idx].iov_len) {
                 remaining = iov[iov_idx].iov_len;
+            }
 
-            packed_buffer = (unsigned char*)iov[iov_idx].iov_base;
+            packed_buffer = (unsigned char *) iov[iov_idx].iov_base;
             user_memory = pConv->pBaseBuf + pData->true_lb + stack[0].disp + stack[1].disp;
             pConv->bConverted += remaining; /* how much will get unpacked this time */
 
-            for( i = 0; stack[1].count <= remaining; i++ ) {  /* partial or full data */
-                OPAL_DATATYPE_SAFEGUARD_POINTER( user_memory, stack[1].count, pConv->pBaseBuf,
-                                                 pData, pConv->count );
-                DO_DEBUG( opal_output( 0, "unpack gaps [%d] dest %p src %p length %" PRIsize_t " [%d]\n",
-                                       iov_idx, (void*)user_memory, (void*)packed_buffer, stack[1].count, i ); );
-                MEMCPY_CSUM( user_memory, packed_buffer, stack[1].count, pConv );
+            for (i = 0; stack[1].count <= remaining; i++) { /* partial or full data */
+                OPAL_DATATYPE_SAFEGUARD_POINTER(user_memory, stack[1].count, pConv->pBaseBuf, pData,
+                                                pConv->count);
+                DO_DEBUG(opal_output(0,
+                                     "unpack gaps [%d] dest %p src %p length %" PRIsize_t " [%d]\n",
+                                     iov_idx, (void *) user_memory, (void *) packed_buffer,
+                                     stack[1].count, i););
+                MEMCPY_CSUM(user_memory, packed_buffer, stack[1].count, pConv);
 
                 packed_buffer += stack[1].count;
-                remaining     -= stack[1].count;
+                remaining -= stack[1].count;
 
                 stack[0].count--;
                 stack[0].disp += extent;
                 stack[1].count = pData->size;
-                stack[1].disp  = 0;
+                stack[1].disp = 0;
 
                 user_memory = pConv->pBaseBuf + pData->true_lb + stack[0].disp;
             }
 
             /* Copy the last bits */
-            if( 0 != remaining ) {
-                OPAL_DATATYPE_SAFEGUARD_POINTER( user_memory, remaining, pConv->pBaseBuf,
-                                                 pData, pConv->count );
-                DO_DEBUG( opal_output( 0, "unpack gaps [%d] dest %p src %p length %" PRIsize_t " [epilog]\n",
-                                       iov_idx, (void*)user_memory, (void*)packed_buffer, remaining ); );
-                MEMCPY_CSUM( user_memory, packed_buffer, remaining, pConv );
+            if (0 != remaining) {
+                OPAL_DATATYPE_SAFEGUARD_POINTER(user_memory, remaining, pConv->pBaseBuf, pData,
+                                                pConv->count);
+                DO_DEBUG(
+                    opal_output(0,
+                                "unpack gaps [%d] dest %p src %p length %" PRIsize_t " [epilog]\n",
+                                iov_idx, (void *) user_memory, (void *) packed_buffer, remaining););
+                MEMCPY_CSUM(user_memory, packed_buffer, remaining, pConv);
                 stack[1].count -= remaining;
-                stack[1].disp  += remaining;  /* keep the += in case we are copying less that the datatype size */
-                assert( stack[1].count );
+                stack[1].disp += remaining; /* keep the += in case we are copying less that the
+                                               datatype size */
+                assert(stack[1].count);
             }
         }
     }
-    *out_size = iov_idx; /* we only reach this line after the for loop succesfully complete */
+    *out_size = iov_idx; /* we only reach this line after the for loop successfully complete */
     *max_data = pConv->bConverted - initial_bytes_converted;
-    if( pConv->bConverted == pConv->local_size ) pConv->flags |= CONVERTOR_COMPLETED;
-    return !!(pConv->flags & CONVERTOR_COMPLETED);  /* done or not */
+    if (pConv->bConverted == pConv->local_size) {
+        pConv->flags |= CONVERTOR_COMPLETED;
+    }
+    return !!(pConv->flags & CONVERTOR_COMPLETED); /* done or not */
 }
 
 /**
@@ -161,74 +179,90 @@ opal_unpack_homogeneous_contig_function( opal_convertor_t* pConv,
  * of the exponent or mantissa).
  */
 static inline void
-opal_unpack_partial_datatype( opal_convertor_t* pConvertor, dt_elem_desc_t* pElem,
-                              unsigned char* partial_data,
-                              ptrdiff_t start_position, size_t length,
-                              unsigned char** user_buffer )
+opal_unpack_partial_predefined(opal_convertor_t *pConvertor, const dt_elem_desc_t *pElem,
+                               size_t *COUNT, unsigned char **packed,
+                               unsigned char **memory, size_t *SPACE)
 {
     char unused_byte = 0x7F, saved_data[16];
     unsigned char temporary[16], *temporary_buffer = temporary;
-    unsigned char* user_data = *user_buffer + pElem->elem.disp;
-    size_t count_desc = 1;
+    unsigned char *user_data = *memory + pElem->elem.disp;
     size_t data_length = opal_datatype_basicDatatypes[pElem->elem.common.type]->size;
+    unsigned char *partial_data = *packed;
+    ptrdiff_t start_position = pConvertor->partial_length;
+    size_t length = data_length - start_position;
+    size_t count_desc = 1;
+    dt_elem_desc_t single_elem = { .elem = { .common = pElem->elem.common, .count = 1, .blocklen = 1,
+                                   .extent = data_length,  /* advance by a full data element */
+                                   .disp = 0  /* right where the pointer is */ } };
+    if( *SPACE < length ) {
+        length = *SPACE;
+    }
 
     DO_DEBUG( opal_output( 0, "unpack partial data start %lu end %lu data_length %lu user %p\n"
                            "\tbConverted %lu total_length %lu count %ld\n",
-                           (unsigned long)start_position, (unsigned long)start_position + length, (unsigned long)data_length, (void*)*user_buffer,
-                           (unsigned long)pConvertor->bConverted, (unsigned long)pConvertor->local_size, pConvertor->count ); );
-
-    /* Find a byte that is not used in the partial buffer */
+                           (unsigned long)start_position, (unsigned long)start_position + length,
+                           (unsigned long)data_length, (void*)*memory,
+                           (unsigned long)pConvertor->bConverted,
+                           (unsigned long)pConvertor->local_size, pConvertor->count ); );
+    COMPUTE_CSUM( partial_data, length, pConvertor );
+
+    /* Find a byte value that is not used in the partial buffer. We use it as a marker
+     * to identify what has not been modified by the unpack call. */
  find_unused_byte:
-    for(size_t i = 0; i < length; i++ ) {
+    for (size_t i = 0; i < length; i++ ) {
         if( unused_byte == partial_data[i] ) {
             unused_byte--;
             goto find_unused_byte;
         }
     }
 
-    /* Copy and fill the rest of the buffer with the unused byte */
+    /* Prepare an full element of the predefined type, by populating an entire type
+     * with the unused byte and then put the partial data at the right position. */
     memset( temporary, unused_byte, data_length );
     MEMCPY( temporary + start_position, partial_data, length );
 
-#if OPAL_CUDA_SUPPORT
+    /* Save the original content of the user memory */
     /* In the case where the data is being unpacked from device memory, need to
-     * use the special host to device memory copy.  Note this code path was only
-     * seen on large receives of noncontiguous data via buffered sends. */
+     * use the special host to device memory copy. */
     pConvertor->cbmemcpy(saved_data, user_data, data_length, pConvertor );
-#else
-    /* Save the content of the user memory */
-    MEMCPY( saved_data, user_data, data_length );
-#endif
 
     /* Then unpack the data into the user memory */
-    UNPACK_PREDEFINED_DATATYPE( pConvertor, pElem, count_desc,
-                                temporary_buffer, *user_buffer, data_length );
+    UNPACK_PREDEFINED_DATATYPE(pConvertor, &single_elem, count_desc, temporary_buffer, user_data,
+                               data_length);
 
-    /* reload the length as it is reset by the macro */
+    /* reload the length and user buffer as they have been updated by the macro */
     data_length = opal_datatype_basicDatatypes[pElem->elem.common.type]->size;
+    user_data = *memory + pElem->elem.disp;
 
-    /* For every occurence of the unused byte move data from the saved
-     * buffer back into the user memory.
-     */
-#if OPAL_CUDA_SUPPORT
+    /* Rebuild the data by pulling back the unmodified bytes from the original
+     * content in the user memory. */
     /* Need to copy the modified user_data again so we can see which
-     * bytes need to be converted back to their original values.  Note
-     * this code path was only seen on large receives of noncontiguous
-     * data via buffered sends. */
-    {
+     * bytes need to be converted back to their original values. */
+    if (0 != strcmp(opal_accelerator_base_selected_component.base_version.mca_component_name, "null")) {
         char resaved_data[16];
-        pConvertor->cbmemcpy(resaved_data, user_data, data_length, pConvertor );
-        for(size_t i = 0; i < data_length; i++ ) {
-            if( unused_byte == resaved_data[i] )
+        pConvertor->cbmemcpy(resaved_data, user_data, data_length, pConvertor);
+        for (size_t i = 0; i < data_length; i++) {
+            if (unused_byte == resaved_data[i])
                 pConvertor->cbmemcpy(&user_data[i], &saved_data[i], 1, pConvertor);
         }
+    } else {
+        for (size_t i = 0; i < data_length; i++) {
+            if (unused_byte == user_data[i]) {
+                user_data[i] = saved_data[i];
+            }
+        }
     }
-#else
-    for(size_t i = 0; i < data_length; i++ ) {
-        if( unused_byte == user_data[i] )
-            user_data[i] = saved_data[i];
+
+    pConvertor->partial_length = (pConvertor->partial_length + length) % data_length;
+    *SPACE  -= length;
+    *packed += length;
+    if (0 == pConvertor->partial_length) {
+        (*COUNT)--;  /* we have enough to complete one full predefined type */
+        *memory += data_length;
+        if (0 == (*COUNT % pElem->elem.blocklen)) {
+            *memory += pElem->elem.extent - (pElem->elem.blocklen * data_length);
+        }
     }
-#endif
 }
 
 /* The pack/unpack functions need a cleanup. I have to create a proper interface to access
@@ -237,28 +271,26 @@ opal_unpack_partial_datatype( opal_convertor_t* pConvertor, dt_elem_desc_t* pEle
  * But first let's make some global assumptions:
  * - a datatype (with the flag DT_DATA set) will have the contiguous flags set if and only if
  *   the data is really contiguous (extent equal with size)
- * - for the OPAL_DATATYPE_LOOP type the DT_CONTIGUOUS flag set means that the content of the loop is
- *   contiguous but with a gap in the begining or at the end.
+ * - for the OPAL_DATATYPE_LOOP type the DT_CONTIGUOUS flag set means that the content of the loop
+ * is contiguous but with a gap in the beginning or at the end.
  * - the DT_CONTIGUOUS flag for the type OPAL_DATATYPE_END_LOOP is meaningless.
  */
-int32_t
-opal_generic_simple_unpack_function( opal_convertor_t* pConvertor,
-                                     struct iovec* iov, uint32_t* out_size,
-                                     size_t* max_data )
+int32_t opal_generic_simple_unpack_function(opal_convertor_t *pConvertor, struct iovec *iov,
+                                            uint32_t *out_size, size_t *max_data)
 {
-    dt_stack_t* pStack;                /* pointer to the position on the stack */
-    uint32_t pos_desc;                 /* actual position in the description of the derived datatype */
-    size_t count_desc;                 /* the number of items already done in the actual pos_desc */
-    size_t total_unpacked = 0;         /* total size unpacked this time */
-    dt_elem_desc_t* description;
-    dt_elem_desc_t* pElem;
+    dt_stack_t *pStack;        /* pointer to the position on the stack */
+    uint32_t pos_desc;         /* actual position in the description of the derived datatype */
+    size_t count_desc;         /* the number of items already done in the actual pos_desc */
+    size_t total_unpacked = 0; /* total size unpacked this time */
+    dt_elem_desc_t *description;
+    dt_elem_desc_t *pElem;
     const opal_datatype_t *pData = pConvertor->pDesc;
     unsigned char *conv_ptr, *iov_ptr;
     size_t iov_len_local;
     uint32_t iov_count;
 
-    DO_DEBUG( opal_output( 0, "opal_convertor_generic_simple_unpack( %p, {%p, %lu}, %u )\n",
-                           (void*)pConvertor, (void*)iov[0].iov_base, (unsigned long)iov[0].iov_len, *out_size ); );
+    DO_DEBUG( opal_output( 0, "opal_convertor_generic_simple_unpack( %p, iov[%u] = {%p, %lu} )\n",
+                           (void*)pConvertor, *out_size, (void*)iov[0].iov_base, (unsigned long)iov[0].iov_len ); );
 
     description = pConvertor->use_desc->desc;
 
@@ -266,75 +298,80 @@ opal_generic_simple_unpack_function( opal_convertor_t* pConvertor,
      * main while loop we will set back the source_base to the correct value. This is
      * due to the fact that the convertor can stop in the middle of a data with a count
      */
-    pStack     = pConvertor->pStack + pConvertor->stack_pos;
-    pos_desc   = pStack->index;
-    conv_ptr   = pConvertor->pBaseBuf + pStack->disp;
+    pStack = pConvertor->pStack + pConvertor->stack_pos;
+    pos_desc = pStack->index;
+    conv_ptr = pConvertor->pBaseBuf + pStack->disp;
     count_desc = pStack->count;
     pStack--;
     pConvertor->stack_pos--;
     pElem = &(description[pos_desc]);
 
-    DO_DEBUG( opal_output( 0, "unpack start pos_desc %d count_desc %" PRIsize_t " disp %ld\n"
-                           "stack_pos %d pos_desc %d count_desc %" PRIsize_t " disp %ld\n",
-                           pos_desc, count_desc, (long)(conv_ptr - pConvertor->pBaseBuf),
-                           pConvertor->stack_pos, pStack->index, pStack->count, (long)(pStack->disp) ); );
+    DO_DEBUG(opal_output(0,
+                         "unpack start pos_desc %d count_desc %" PRIsize_t " disp %ld\n"
+                         "stack_pos %d pos_desc %d count_desc %" PRIsize_t " disp %ld\n",
+                         pos_desc, count_desc, (long) (conv_ptr - pConvertor->pBaseBuf),
+                         pConvertor->stack_pos, pStack->index, pStack->count,
+                         (long) (pStack->disp)););
 
-    for( iov_count = 0; iov_count < (*out_size); iov_count++ ) {
+    for (iov_count = 0; iov_count < (*out_size); iov_count++) {
         iov_ptr = (unsigned char *) iov[iov_count].iov_base;
         iov_len_local = iov[iov_count].iov_len;
 
-        if( 0 != pConvertor->partial_length ) {
-            size_t element_length = opal_datatype_basicDatatypes[pElem->elem.common.type]->size;
-            size_t missing_length = element_length - pConvertor->partial_length;
-
-            assert( pElem->elem.common.flags & OPAL_DATATYPE_FLAG_DATA );
-            COMPUTE_CSUM( iov_ptr, missing_length, pConvertor );
-            opal_unpack_partial_datatype( pConvertor, pElem,
-                                          iov_ptr,
-                                          pConvertor->partial_length, (size_t)(element_length - pConvertor->partial_length),
-                                          &conv_ptr );
-            --count_desc;
-            if( 0 == count_desc ) {
-                conv_ptr = pConvertor->pBaseBuf + pStack->disp;
-                pos_desc++;  /* advance to the next data */
-                UPDATE_INTERNAL_COUNTERS( description, pos_desc, pElem, count_desc );
+        /* Deal with all types of partial predefined datatype unpacking, including when
+         * unpacking a partial predefined element and when unpacking a part smaller than
+         * the blocklen.
+         */
+        if (pElem->elem.common.flags & OPAL_DATATYPE_FLAG_DATA) {
+            if (0 != pConvertor->partial_length) {  /* partial predefined element */
+                assert( pElem->elem.common.flags & OPAL_DATATYPE_FLAG_DATA );
+                opal_unpack_partial_predefined( pConvertor, pElem, &count_desc,
+                                                &iov_ptr, &conv_ptr, &iov_len_local );
+                if (0 == count_desc) {  /* the end of the vector ? */
+                    assert( 0 == pConvertor->partial_length );
+                    conv_ptr = pConvertor->pBaseBuf + pStack->disp;
+                    pos_desc++; /* advance to the next data */
+                    UPDATE_INTERNAL_COUNTERS(description, pos_desc, pElem, count_desc);
+                    goto next_vector;
+                }
+                if( 0 == iov_len_local )
+                    goto complete_loop;
             }
-            iov_ptr       += missing_length;
-            iov_len_local -= missing_length;
-            pConvertor->partial_length = 0;  /* nothing more inside */
-        }
-        if( pElem->elem.common.flags & OPAL_DATATYPE_FLAG_DATA ) {
-            if( ((size_t)pElem->elem.count * pElem->elem.blocklen) != count_desc ) {
+            if (((size_t) pElem->elem.count * pElem->elem.blocklen) != count_desc) {
                 /* we have a partial (less than blocklen) basic datatype */
-                int rc = UNPACK_PARTIAL_BLOCKLEN( pConvertor, pElem, count_desc,
-                                                  iov_ptr, conv_ptr, iov_len_local );
-                if( 0 == rc )  /* not done */
+                int rc = UNPACK_PARTIAL_BLOCKLEN(pConvertor, pElem, count_desc, iov_ptr, conv_ptr,
+                                                 iov_len_local);
+                if (0 == rc) { /* not done */
                     goto complete_loop;
-                if( 0 == count_desc ) {
+                }
+                if (0 == count_desc) {
                     conv_ptr = pConvertor->pBaseBuf + pStack->disp;
-                    pos_desc++;  /* advance to the next data */
-                    UPDATE_INTERNAL_COUNTERS( description, pos_desc, pElem, count_desc );
+                    pos_desc++; /* advance to the next data */
+                    UPDATE_INTERNAL_COUNTERS(description, pos_desc, pElem, count_desc);
                 }
             }
         }
 
-        while( 1 ) {
-            while( pElem->elem.common.flags & OPAL_DATATYPE_FLAG_DATA ) {
+        while (1) {
+          next_vector:
+            while (pElem->elem.common.flags & OPAL_DATATYPE_FLAG_DATA) {
                 /* we have a basic datatype (working on full blocks) */
-                UNPACK_PREDEFINED_DATATYPE( pConvertor, pElem, count_desc,
-                                            iov_ptr, conv_ptr, iov_len_local );
-                if( 0 != count_desc )  /* completed? */
+                UNPACK_PREDEFINED_DATATYPE(pConvertor, pElem, count_desc, iov_ptr, conv_ptr,
+                                           iov_len_local);
+                if (0 != count_desc) { /* completed? */
                     goto complete_loop;
+                }
                 conv_ptr = pConvertor->pBaseBuf + pStack->disp;
-                pos_desc++;  /* advance to the next data */
-                UPDATE_INTERNAL_COUNTERS( description, pos_desc, pElem, count_desc );
+                pos_desc++; /* advance to the next data */
+                UPDATE_INTERNAL_COUNTERS(description, pos_desc, pElem, count_desc);
             }
-            if( OPAL_DATATYPE_END_LOOP == pElem->elem.common.type ) { /* end of the current loop */
-                DO_DEBUG( opal_output( 0, "unpack end_loop count %" PRIsize_t " stack_pos %d pos_desc %d disp %ld space %lu\n",
-                                       pStack->count, pConvertor->stack_pos, pos_desc,
-                                       pStack->disp, (unsigned long)iov_len_local ); );
-                if( --(pStack->count) == 0 ) { /* end of loop */
-                    if( 0 == pConvertor->stack_pos ) {
+            if (OPAL_DATATYPE_END_LOOP == pElem->elem.common.type) { /* end of the current loop */
+                DO_DEBUG(opal_output(0,
+                                     "unpack end_loop count %" PRIsize_t
+                                     " stack_pos %d pos_desc %d disp %ld space %lu\n",
+                                     pStack->count, pConvertor->stack_pos, pos_desc, pStack->disp,
+                                     (unsigned long) iov_len_local););
+                if (--(pStack->count) == 0) { /* end of loop */
+                    if (0 == pConvertor->stack_pos) {
                         /* we're done. Force the exit of the main for loop (around iovec) */
                         *out_size = iov_count;
                         goto complete_loop;
@@ -344,38 +381,40 @@ opal_generic_simple_unpack_function( opal_convertor_t* pConvertor,
                     pos_desc++;
                 } else {
                     pos_desc = pStack->index + 1;
-                    if( pStack->index == -1 ) {
+                    if (pStack->index == -1) {
                         pStack->disp += (pData->ub - pData->lb);
                     } else {
-                        assert( OPAL_DATATYPE_LOOP == description[pStack->index].loop.common.type );
+                        assert(OPAL_DATATYPE_LOOP == description[pStack->index].loop.common.type);
                         pStack->disp += description[pStack->index].loop.extent;
                     }
                 }
                 conv_ptr = pConvertor->pBaseBuf + pStack->disp;
-                UPDATE_INTERNAL_COUNTERS( description, pos_desc, pElem, count_desc );
-                DO_DEBUG( opal_output( 0, "unpack new_loop count %" PRIsize_t " stack_pos %d pos_desc %d disp %ld space %lu\n",
-                                       pStack->count, pConvertor->stack_pos, pos_desc,
-                                       pStack->disp, (unsigned long)iov_len_local ); );
+                UPDATE_INTERNAL_COUNTERS(description, pos_desc, pElem, count_desc);
+                DO_DEBUG(opal_output(0,
+                                     "unpack new_loop count %" PRIsize_t
+                                     " stack_pos %d pos_desc %d disp %ld space %lu\n",
+                                     pStack->count, pConvertor->stack_pos, pos_desc, pStack->disp,
+                                     (unsigned long) iov_len_local););
             }
-            if( OPAL_DATATYPE_LOOP == pElem->elem.common.type ) {
-                ptrdiff_t local_disp = (ptrdiff_t)conv_ptr;
-                if( pElem->loop.common.flags & OPAL_DATATYPE_FLAG_CONTIGUOUS ) {
-                    UNPACK_CONTIGUOUS_LOOP( pConvertor, pElem, count_desc,
-                                            iov_ptr, conv_ptr, iov_len_local );
-                    if( 0 == count_desc ) {  /* completed */
+            if (OPAL_DATATYPE_LOOP == pElem->elem.common.type) {
+                ptrdiff_t local_disp = (ptrdiff_t) conv_ptr;
+                if (pElem->loop.common.flags & OPAL_DATATYPE_FLAG_CONTIGUOUS) {
+                    UNPACK_CONTIGUOUS_LOOP(pConvertor, pElem, count_desc, iov_ptr, conv_ptr,
+                                           iov_len_local);
+                    if (0 == count_desc) { /* completed */
                         pos_desc += pElem->loop.items + 1;
                         goto update_loop_description;
                     }
                     /* Save the stack with the correct last_count value. */
                 }
-                local_disp = (ptrdiff_t)conv_ptr - local_disp;
-                PUSH_STACK( pStack, pConvertor->stack_pos, pos_desc, OPAL_DATATYPE_LOOP, count_desc,
-                            pStack->disp + local_disp);
+                local_disp = (ptrdiff_t) conv_ptr - local_disp;
+                PUSH_STACK(pStack, pConvertor->stack_pos, pos_desc, OPAL_DATATYPE_LOOP, count_desc,
+                           pStack->disp + local_disp);
                 pos_desc++;
-            update_loop_description:  /* update the current state */
+            update_loop_description: /* update the current state */
                 conv_ptr = pConvertor->pBaseBuf + pStack->disp;
-                UPDATE_INTERNAL_COUNTERS( description, pos_desc, pElem, count_desc );
-                DDT_DUMP_STACK( pConvertor->pStack, pConvertor->stack_pos, pElem, "advance loop" );
+                UPDATE_INTERNAL_COUNTERS(description, pos_desc, pElem, count_desc);
+                DDT_DUMP_STACK(pConvertor->pStack, pConvertor->stack_pos, pElem, "advance loop");
             }
         }
     complete_loop:
@@ -386,31 +425,27 @@ opal_generic_simple_unpack_function( opal_convertor_t* pConvertor,
              * and keep it hot until the next round.
              */
             assert( iov_len_local < opal_datatype_basicDatatypes[pElem->elem.common.type]->size );
-            COMPUTE_CSUM( iov_ptr, iov_len_local, pConvertor );
-
-            opal_unpack_partial_datatype( pConvertor, pElem,
-                                          iov_ptr, 0, iov_len_local,
-                                          &temp );
-
-            pConvertor->partial_length = iov_len_local;
-            iov_len_local = 0;
+            opal_unpack_partial_predefined(pConvertor, pElem, &count_desc, &iov_ptr, &temp, &iov_len_local);
         }
 
-        iov[iov_count].iov_len -= iov_len_local;  /* update the amount of valid data */
+        iov[iov_count].iov_len -= iov_len_local; /* update the amount of valid data */
         total_unpacked += iov[iov_count].iov_len;
     }
     *max_data = total_unpacked;
-    pConvertor->bConverted += total_unpacked;  /* update the already converted bytes */
+    pConvertor->bConverted += total_unpacked; /* update the already converted bytes */
     *out_size = iov_count;
-    if( pConvertor->bConverted == pConvertor->remote_size ) {
+    if (pConvertor->bConverted == pConvertor->local_size) {
         pConvertor->flags |= CONVERTOR_COMPLETED;
         return 1;
     }
     /* Save the global position for the next round */
-    PUSH_STACK( pStack, pConvertor->stack_pos, pos_desc, pElem->elem.common.type, count_desc,
-                conv_ptr - pConvertor->pBaseBuf );
-    DO_DEBUG( opal_output( 0, "unpack save stack stack_pos %d pos_desc %d count_desc %" PRIsize_t " disp %ld\n",
-                           pConvertor->stack_pos, pStack->index, pStack->count, (long)pStack->disp ); );
+    PUSH_STACK(pStack, pConvertor->stack_pos, pos_desc, pElem->elem.common.type, count_desc,
+               conv_ptr - pConvertor->pBaseBuf);
+    DO_DEBUG(opal_output(0,
+                         "unpack save stack stack_pos %d pos_desc %d count_desc %" PRIsize_t
+                         " disp %ld\n",
+                         pConvertor->stack_pos, pStack->index, pStack->count,
+                         (long) pStack->disp););
     return 0;
 }
 
@@ -423,32 +458,106 @@ opal_generic_simple_unpack_function( opal_convertor_t* pConvertor,
  * to a contiguous output buffer with a predefined size.
  * return OPAL_SUCCESS if everything went OK and if there is still room before the complete
  *          conversion of the data (need additional call with others input buffers )
- *        1 if everything went fine and the data was completly converted
+ *        1 if everything went fine and the data was completely converted
  *       -1 something wrong occurs.
  */
-int32_t
-opal_unpack_general_function( opal_convertor_t* pConvertor,
-                              struct iovec* iov, uint32_t* out_size,
-                              size_t* max_data )
+static inline void
+unpack_predefined_heterogeneous(opal_convertor_t *CONVERTOR,
+                                const dt_elem_desc_t *ELEM, size_t *COUNT,
+                                unsigned char **memory,
+                                unsigned char **packed, size_t *SPACE)
+{
+    const opal_convertor_master_t *master = (CONVERTOR)->master;
+    const ddt_elem_desc_t *_elem = &((ELEM)->elem);
+    size_t cando_count = *(COUNT), do_now_bytes;
+    size_t local_elem_size = opal_datatype_basicDatatypes[_elem->common.type]->size;
+    size_t remote_elem_size = master->remote_sizes[_elem->common.type];
+    size_t blocklen_bytes = remote_elem_size;
+    unsigned char *_memory = (*memory) + _elem->disp;
+    unsigned char *_packed = *packed;
+    ptrdiff_t advance = 0;
+
+    assert(0 == (cando_count % _elem->blocklen)); /* no partials here */
+    assert(*(COUNT) <= ((size_t) _elem->count * _elem->blocklen));
+
+    if ((remote_elem_size * cando_count) > *(SPACE))
+        cando_count = (*SPACE) / blocklen_bytes;
+
+    /* preemptively update the number of COUNT we will return. */
+    *(COUNT) -= cando_count;
+
+    if (_elem->blocklen == 1) {
+        master->pFunctions[_elem->common.type](CONVERTOR, cando_count,
+                                               _packed, *SPACE, remote_elem_size,
+                                               _memory, *SPACE, _elem->extent,
+                                               &advance);
+        _memory += cando_count * _elem->extent;
+        _packed += cando_count * remote_elem_size;
+        goto update_and_return;
+    }
+
+    if ((1 < _elem->count) && (_elem->blocklen <= cando_count)) {
+        blocklen_bytes = remote_elem_size * _elem->blocklen;
+
+        do { /* Do as many full blocklen as possible */
+            OPAL_DATATYPE_SAFEGUARD_POINTER(_memory, blocklen_bytes, (CONVERTOR)->pBaseBuf,
+                                            (CONVERTOR)->pDesc, (CONVERTOR)->count);
+            DO_DEBUG(opal_output(0, "pack 2. memcpy( %p, %p, %lu ) => space %lu\n",
+                                 (void *) _packed, (void *) _memory, (unsigned long) blocklen_bytes,
+                                 (unsigned long) (*(SPACE) - (_packed - *(packed)))););
+            master->pFunctions[_elem->common.type](CONVERTOR, _elem->blocklen,
+                                                   _packed, *SPACE, remote_elem_size,
+                                                   _memory, *SPACE, local_elem_size,
+                                                   &advance);
+            _packed += blocklen_bytes;
+            _memory += _elem->extent;
+            cando_count -= _elem->blocklen;
+        } while (_elem->blocklen <= cando_count);
+    }
+
+    /**
+     * As an epilog do anything left from the last blocklen.
+     */
+    if (0 != cando_count) {
+        assert((cando_count < _elem->blocklen)
+               || ((1 == _elem->count) && (cando_count <= _elem->blocklen)));
+        do_now_bytes = cando_count * remote_elem_size;
+        OPAL_DATATYPE_SAFEGUARD_POINTER(_memory, do_now_bytes, (CONVERTOR)->pBaseBuf,
+                                        (CONVERTOR)->pDesc, (CONVERTOR)->count);
+        DO_DEBUG(opal_output(0, "pack 3. memcpy( %p, %p, %lu ) => space %lu [epilog]\n",
+                             (void *) _packed, (void *) _memory, (unsigned long) do_now_bytes,
+                             (unsigned long) (*(SPACE) - (_packed - *(packed)))););
+        master->pFunctions[_elem->common.type](CONVERTOR, cando_count,
+                                               _packed, *SPACE, remote_elem_size,
+                                               _memory, *SPACE, local_elem_size,
+                                               &advance);
+        _memory += cando_count * local_elem_size;
+        _packed += do_now_bytes;
+    }
+
+update_and_return:
+    *(memory) = _memory - _elem->disp;
+    *(SPACE) -= (_packed - *packed);
+    *(packed) = _packed;
+}
+
+int32_t opal_unpack_general_function(opal_convertor_t *pConvertor, struct iovec *iov,
+                                     uint32_t *out_size, size_t *max_data)
 {
-    dt_stack_t* pStack;                /* pointer to the position on the stack */
-    uint32_t pos_desc;                 /* actual position in the description of the derived datatype */
-    size_t count_desc;                 /* the number of items already done in the actual pos_desc */
-    uint16_t type = OPAL_DATATYPE_MAX_PREDEFINED; /* type at current position */
-    size_t total_unpacked = 0;         /* total size unpacked this time */
-    dt_elem_desc_t* description;
-    dt_elem_desc_t* pElem;
+    dt_stack_t *pStack; /* pointer to the position on the stack */
+    uint32_t pos_desc;  /* actual position in the description of the derived datatype */
+    size_t count_desc;  /* the number of items already done in the actual pos_desc */
+    size_t total_unpacked = 0;                    /* total size unpacked this time */
+    dt_elem_desc_t *description;
+    dt_elem_desc_t *pElem;
     const opal_datatype_t *pData = pConvertor->pDesc;
     unsigned char *conv_ptr, *iov_ptr;
     uint32_t iov_count;
     size_t iov_len_local;
 
-    const opal_convertor_master_t* master = pConvertor->master;
-    ptrdiff_t advance;       /* number of bytes that we should advance the buffer */
-    size_t rc;
-
-    DO_DEBUG( opal_output( 0, "opal_convertor_general_unpack( %p, {%p, %lu}, %d )\n",
-                           (void*)pConvertor, (void*)iov[0].iov_base, (unsigned long)iov[0].iov_len, *out_size ); );
+    DO_DEBUG(opal_output(0, "opal_convertor_general_unpack( %p, {%p, %lu}, %d )\n",
+                         (void *) pConvertor, (void *) iov[0].iov_base,
+                         (unsigned long) iov[0].iov_len, *out_size););
 
     description = pConvertor->use_desc->desc;
 
@@ -456,75 +565,69 @@ opal_unpack_general_function( opal_convertor_t* pConvertor,
      * main while loop we will set back the source_base to the correct value. This is
      * due to the fact that the convertor can stop in the middle of a data with a count
      */
-    pStack     = pConvertor->pStack + pConvertor->stack_pos;
-    pos_desc   = pStack->index;
-    conv_ptr   = pConvertor->pBaseBuf + pStack->disp;
+    pStack = pConvertor->pStack + pConvertor->stack_pos;
+    pos_desc = pStack->index;
+    conv_ptr = pConvertor->pBaseBuf + pStack->disp;
     count_desc = pStack->count;
     pStack--;
     pConvertor->stack_pos--;
     pElem = &(description[pos_desc]);
 
-    DO_DEBUG( opal_output( 0, "unpack start pos_desc %d count_desc %" PRIsize_t " disp %ld\n"
-                           "stack_pos %d pos_desc %d count_desc %" PRIsize_t " disp %ld\n",
-                           pos_desc, count_desc, (long)(conv_ptr - pConvertor->pBaseBuf),
-                           pConvertor->stack_pos, pStack->index, pStack->count, (long)(pStack->disp) ); );
+    DO_DEBUG(opal_output(0,
+                         "unpack start pos_desc %d count_desc %" PRIsize_t " disp %ld\n"
+                         "stack_pos %d pos_desc %d count_desc %" PRIsize_t " disp %ld\n",
+                         pos_desc, count_desc, (long) (conv_ptr - pConvertor->pBaseBuf),
+                         pConvertor->stack_pos, pStack->index, pStack->count,
+                         (long) (pStack->disp)););
 
-    for( iov_count = 0; iov_count < (*out_size); iov_count++ ) {
+    for (iov_count = 0; iov_count < (*out_size); iov_count++) {
         iov_ptr = (unsigned char *) iov[iov_count].iov_base;
         iov_len_local = iov[iov_count].iov_len;
-        assert( 0 == pConvertor->partial_length );
-        while( 1 ) {
-            while( pElem->elem.common.flags & OPAL_DATATYPE_FLAG_DATA ) {
+        assert(0 == pConvertor->partial_length);
+        while (1) {
+            while (pElem->elem.common.flags & OPAL_DATATYPE_FLAG_DATA) {
                 /* now here we have a basic datatype */
-                type = description[pos_desc].elem.common.type;
-                OPAL_DATATYPE_SAFEGUARD_POINTER( conv_ptr + pElem->elem.disp, pData->size, pConvertor->pBaseBuf,
-                                                 pData, pConvertor->count );
-                DO_DEBUG( opal_output( 0, "unpack (%p, %ld) -> (%p:%ld, %" PRIsize_t ", %ld) type %s\n",
-                                       (void*)iov_ptr, iov_len_local,
-                                       (void*)pConvertor->pBaseBuf, conv_ptr + pElem->elem.disp - pConvertor->pBaseBuf,
-                                       count_desc, description[pos_desc].elem.extent,
-                                       opal_datatype_basicDatatypes[type]->name ); );
-                rc = master->pFunctions[type]( pConvertor, count_desc,
-                                               iov_ptr, iov_len_local, opal_datatype_basicDatatypes[type]->size,
-                                               conv_ptr + pElem->elem.disp,
-                                               (pConvertor->pDesc->ub - pConvertor->pDesc->lb) * pConvertor->count,
-                                               description[pos_desc].elem.extent, &advance );
-                iov_len_local -= advance;  /* decrease the available space in the buffer */
-                iov_ptr += advance;        /* increase the pointer to the buffer */
-                count_desc -= rc;          /* compute leftovers */
-                if( 0 == count_desc ) {  /* completed */
+                OPAL_DATATYPE_SAFEGUARD_POINTER(conv_ptr + pElem->elem.disp, pData->size,
+                                                pConvertor->pBaseBuf, pData, pConvertor->count);
+                DO_DEBUG(opal_output(0,
+                                     "unpack (%p, %ld) -> (%p:%ld, %" PRIsize_t ", %ld) type %s\n",
+                                     (void *) iov_ptr, iov_len_local, (void *) pConvertor->pBaseBuf,
+                                     conv_ptr + pElem->elem.disp - pConvertor->pBaseBuf, count_desc,
+                                     description[pos_desc].elem.extent,
+                                     opal_datatype_basicDatatypes[description[pos_desc].elem.common.type]->name););
+                unpack_predefined_heterogeneous(pConvertor, pElem, &count_desc, &conv_ptr, &iov_ptr,
+                                                &iov_len_local);
+                if (0 == count_desc) {    /* completed */
                     conv_ptr = pConvertor->pBaseBuf + pStack->disp;
-                    pos_desc++;  /* advance to the next data */
-                    UPDATE_INTERNAL_COUNTERS( description, pos_desc, pElem, count_desc );
-                    if( 0 == iov_len_local ) goto complete_loop;  /* escape if we're done */
+                    pos_desc++; /* advance to the next data */
+                    UPDATE_INTERNAL_COUNTERS(description, pos_desc, pElem, count_desc);
+                    if (0 == iov_len_local) {
+                        goto complete_loop; /* escape if we're done */
+                    }
                     continue;
                 }
-                conv_ptr += rc * description[pos_desc].elem.extent;
-                assert( pElem->elem.common.type < OPAL_DATATYPE_MAX_PREDEFINED );
-                assert( 0 == iov_len_local );
-                if( 0 != iov_len_local ) {
-                    unsigned char* temp = conv_ptr;
+                assert(pElem->elem.common.type < OPAL_DATATYPE_MAX_PREDEFINED);
+                assert(0 == iov_len_local);
+                if (0 != iov_len_local) {
+                    unsigned char *temp = conv_ptr;
                     /* We have some partial data here. Let's copy it into the convertor
                      * and keep it hot until the next round.
                      */
-                    assert( iov_len_local < opal_datatype_basicDatatypes[pElem->elem.common.type]->size );
-                    COMPUTE_CSUM( iov_ptr, iov_len_local, pConvertor );
-
-                    opal_unpack_partial_datatype( pConvertor, pElem,
-                                                  iov_ptr, 0, iov_len_local,
-                                                  &temp );
-
-                    pConvertor->partial_length = iov_len_local;
-                    iov_len_local = 0;
+                    assert(iov_len_local < opal_datatype_basicDatatypes[pElem->elem.common.type]->size);
+                    opal_unpack_partial_predefined(pConvertor, pElem, &count_desc, &iov_ptr,
+                                                   &temp, &iov_len_local);
+                    assert( 0 == iov_len_local );
                 }
                 goto complete_loop;
             }
-            if( OPAL_DATATYPE_END_LOOP == pElem->elem.common.type ) { /* end of the current loop */
-                DO_DEBUG( opal_output( 0, "unpack end_loop count %" PRIsize_t " stack_pos %d pos_desc %d disp %ld space %lu\n",
-                                       pStack->count, pConvertor->stack_pos, pos_desc,
-                                       pStack->disp, (unsigned long)iov_len_local ); );
-                if( --(pStack->count) == 0 ) { /* end of loop */
-                    if( 0 == pConvertor->stack_pos ) {
+            if (OPAL_DATATYPE_END_LOOP == pElem->elem.common.type) { /* end of the current loop */
+                DO_DEBUG(opal_output(0,
+                                     "unpack end_loop count %" PRIsize_t
+                                     " stack_pos %d pos_desc %d disp %ld space %lu\n",
+                                     pStack->count, pConvertor->stack_pos, pos_desc, pStack->disp,
+                                     (unsigned long) iov_len_local););
+                if (--(pStack->count) == 0) { /* end of loop */
+                    if (0 == pConvertor->stack_pos) {
                         /* we're done. Force the exit of the main for loop (around iovec) */
                         *out_size = iov_count;
                         goto complete_loop;
@@ -534,44 +637,51 @@ opal_unpack_general_function( opal_convertor_t* pConvertor,
                     pos_desc++;
                 } else {
                     pos_desc = pStack->index + 1;
-                    if( pStack->index == -1 ) {
+                    if (pStack->index == -1) {
                         pStack->disp += (pData->ub - pData->lb);
                     } else {
-                        assert( OPAL_DATATYPE_LOOP == description[pStack->index].loop.common.type );
+                        assert(OPAL_DATATYPE_LOOP == description[pStack->index].loop.common.type);
                         pStack->disp += description[pStack->index].loop.extent;
                     }
                 }
                 conv_ptr = pConvertor->pBaseBuf + pStack->disp;
-                UPDATE_INTERNAL_COUNTERS( description, pos_desc, pElem, count_desc );
-                DO_DEBUG( opal_output( 0, "unpack new_loop count %" PRIsize_t " stack_pos %d pos_desc %d disp %ld space %lu\n",
-                                       pStack->count, pConvertor->stack_pos, pos_desc,
-                                       pStack->disp, (unsigned long)iov_len_local ); );
+                UPDATE_INTERNAL_COUNTERS(description, pos_desc, pElem, count_desc);
+                DO_DEBUG(opal_output(0,
+                                     "unpack new_loop count %" PRIsize_t
+                                     " stack_pos %d pos_desc %d disp %ld space %lu\n",
+                                     pStack->count, pConvertor->stack_pos, pos_desc, pStack->disp,
+                                     (unsigned long) iov_len_local););
             }
-            if( OPAL_DATATYPE_LOOP == pElem->elem.common.type ) {
-                PUSH_STACK( pStack, pConvertor->stack_pos, pos_desc, OPAL_DATATYPE_LOOP, count_desc,
-                            pStack->disp );
+            if (OPAL_DATATYPE_LOOP == pElem->elem.common.type) {
+                PUSH_STACK(pStack, pConvertor->stack_pos, pos_desc, OPAL_DATATYPE_LOOP, count_desc,
+                           pStack->disp);
                 pos_desc++;
                 conv_ptr = pConvertor->pBaseBuf + pStack->disp;
-                UPDATE_INTERNAL_COUNTERS( description, pos_desc, pElem, count_desc );
-                DDT_DUMP_STACK( pConvertor->pStack, pConvertor->stack_pos, pElem, "advance loop" );
+                UPDATE_INTERNAL_COUNTERS(description, pos_desc, pElem, count_desc);
+                DDT_DUMP_STACK(pConvertor->pStack, pConvertor->stack_pos, pElem, "advance loop");
                 continue;
             }
         }
     complete_loop:
-        iov[iov_count].iov_len -= iov_len_local;  /* update the amount of valid data */
+        iov[iov_count].iov_len -= iov_len_local; /* update the amount of valid data */
         total_unpacked += iov[iov_count].iov_len;
     }
     *max_data = total_unpacked;
-    pConvertor->bConverted += total_unpacked;  /* update the already converted bytes */
+    pConvertor->bConverted += total_unpacked; /* update the already converted bytes */
     *out_size = iov_count;
-    if( pConvertor->bConverted == pConvertor->remote_size ) {
+    size_t expected_packed_size;
+    opal_convertor_get_packed_size(pConvertor, &expected_packed_size);
+    if (pConvertor->bConverted == expected_packed_size) {
         pConvertor->flags |= CONVERTOR_COMPLETED;
         return 1;
     }
     /* Save the global position for the next round */
-    PUSH_STACK( pStack, pConvertor->stack_pos, pos_desc, pElem->elem.common.type, count_desc,
-                conv_ptr - pConvertor->pBaseBuf );
-    DO_DEBUG( opal_output( 0, "unpack save stack stack_pos %d pos_desc %d count_desc %" PRIsize_t" disp %ld\n",
-                           pConvertor->stack_pos, pStack->index, pStack->count, (long)pStack->disp ); );
+    PUSH_STACK(pStack, pConvertor->stack_pos, pos_desc, pElem->elem.common.type, count_desc,
+               conv_ptr - pConvertor->pBaseBuf);
+    DO_DEBUG(opal_output(0,
+                         "unpack save stack stack_pos %d pos_desc %d count_desc %" PRIsize_t
+                         " disp %ld\n",
+                         pConvertor->stack_pos, pStack->index, pStack->count,
+                         (long) pStack->disp););
     return 0;
 }
diff --git a/opal/datatype/opal_datatype_unpack.h b/opal/datatype/opal_datatype_unpack.h
index a786a2fc7e9..6180bf99062 100644
--- a/opal/datatype/opal_datatype_unpack.h
+++ b/opal/datatype/opal_datatype_unpack.h
@@ -8,6 +8,7 @@
  * Copyright (c) 2017-2018 Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2020-2021 IBM Corporation. All rights reserved.
+ * Copyright (c) 2022      Advanced Micro Devices, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -21,56 +22,56 @@
 #include "opal_config.h"
 #include "opal/datatype/opal_datatype_pack_unpack_predefined.h"
 
-#if !defined(CHECKSUM) && OPAL_CUDA_SUPPORT
-/* Make use of existing macro to do CUDA style memcpy */
-#undef MEMCPY_CSUM
-#define MEMCPY_CSUM( DST, SRC, BLENGTH, CONVERTOR ) \
-    CONVERTOR->cbmemcpy( (DST), (SRC), (BLENGTH), (CONVERTOR) )
+#if !defined(CHECKSUM)
+/* Make use of existing macro to do device style memcpy */
+#    undef MEMCPY_CSUM
+#    define MEMCPY_CSUM(DST, SRC, BLENGTH, CONVERTOR) \
+        CONVERTOR->cbmemcpy((DST), (SRC), (BLENGTH), (CONVERTOR))
 #endif
 
 /**
- * This function deals only with partial elements. The COUNT points however to the whole leftover count,
- * but this function is only expected to operate on an amount less than blength, that would allow the rest
- * of the pack process to handle only entire blength blocks (plus the left over).
+ * This function deals only with partial elements. The COUNT points however to
+ * the whole leftover count, but this function is only expected to operate on
+ * an amount less than blength, that would allow the rest of the pack process
+ * to handle only entire blength blocks (plus the left over).
  *
  * Return 1 if we are now aligned on a block, 0 otherwise.
  */
-static inline int
-unpack_partial_blocklen( opal_convertor_t* CONVERTOR,
-                        const dt_elem_desc_t* ELEM,
-                        size_t* COUNT,
-                        unsigned char** packed,
-                        unsigned char** memory,
-                        size_t* SPACE )
+static inline int unpack_partial_blocklen(opal_convertor_t *CONVERTOR, const dt_elem_desc_t *ELEM,
+                                          size_t *COUNT, unsigned char **packed,
+                                          unsigned char **memory, size_t *SPACE)
 {
-    const ddt_elem_desc_t* _elem = &((ELEM)->elem);
+    const ddt_elem_desc_t *_elem = &((ELEM)->elem);
     size_t do_now_bytes = opal_datatype_basicDatatypes[_elem->common.type]->size;
     size_t do_now = (*COUNT);
-    unsigned char* _memory = (*memory) + _elem->disp;
-    unsigned char* _packed = *packed;
+    unsigned char *_memory = (*memory) + _elem->disp;
+    unsigned char *_packed = *packed;
 
-    assert( *(COUNT) <= ((size_t)(_elem->count * _elem->blocklen)) );
+    assert(*(COUNT) <= ((size_t)(_elem->count * _elem->blocklen)));
 
+    if( (*SPACE) < do_now_bytes ) /* Can we do anything ? */
+        return 0;
     /**
      * First check if we already did something on this element ? The COUNT is the number
      * of remaining predefined types in the current elem, not how many predefined types
      * should be manipulated in the current call (this number is instead reflected on the
      * SPACE).
      */
-    if( 0 == (do_now = (*COUNT) % _elem->blocklen) )
+    if (0 == (do_now = (*COUNT) % _elem->blocklen))
         return 1;
 
-    size_t left_in_block = do_now;  /* left in the current blocklen */
+    size_t left_in_block = do_now; /* left in the current blocklen */
 
-    if( (do_now_bytes * do_now) > *(SPACE) )
+    if ((do_now_bytes * do_now) > *(SPACE))
         do_now = (*SPACE) / do_now_bytes;
 
     do_now_bytes *= do_now;
 
     OPAL_DATATYPE_SAFEGUARD_POINTER( _memory, do_now_bytes, (CONVERTOR)->pBaseBuf,
                                      (CONVERTOR)->pDesc, (CONVERTOR)->count );
-    DO_DEBUG( opal_output( 0, "unpack memcpy( %p, %p, %lu ) => space %lu [prolog]\n",
-                           (void*)_memory, (void*)_packed, (unsigned long)do_now_bytes, (unsigned long)(*(SPACE)) ); );
+    DO_DEBUG( opal_output( 0, "unpack memcpy( %p [%ld], %p, %lu ) => space %lu [prolog]\n",
+                           (void*)_memory, _memory - CONVERTOR->pBaseBuf,
+                           (void*)_packed, (unsigned long)do_now_bytes, (unsigned long)(*(SPACE)) ); );
     MEMCPY_CSUM( _memory, _packed, do_now_bytes, (CONVERTOR) );
     *(memory)     += (ptrdiff_t)do_now_bytes;
     if( do_now == left_in_block )  /* compensate if completed a blocklen */
@@ -82,42 +83,43 @@ unpack_partial_blocklen( opal_convertor_t* CONVERTOR,
     return (do_now == left_in_block);
 }
 
-static inline void
-unpack_predefined_data( opal_convertor_t* CONVERTOR,
-                        const dt_elem_desc_t* ELEM,
-                        size_t* COUNT,
-                        unsigned char** packed,
-                        unsigned char** memory,
-                        size_t* SPACE )
+static inline void unpack_predefined_data(opal_convertor_t *CONVERTOR, const dt_elem_desc_t *ELEM,
+                                          size_t *COUNT, unsigned char **packed,
+                                          unsigned char **memory, size_t *SPACE)
 {
-    const ddt_elem_desc_t* _elem = &((ELEM)->elem);
+    const ddt_elem_desc_t *_elem = &((ELEM)->elem);
     size_t blocklen_bytes = opal_datatype_basicDatatypes[_elem->common.type]->size;
     size_t cando_count = (*COUNT), do_now_bytes;
-    unsigned char* _memory = (*memory) + _elem->disp;
-    unsigned char* _packed = *packed;
+    unsigned char *_memory = (*memory) + _elem->disp;
+    unsigned char *_packed = *packed;
 
-    assert( 0 == (cando_count % _elem->blocklen) );  /* no partials here */
-    assert( *(COUNT) <= ((size_t)_elem->count * _elem->blocklen) );
+    assert(0 == (cando_count % _elem->blocklen)); /* no partials here */
+    assert(*(COUNT) <= ((size_t) _elem->count * _elem->blocklen));
 
-    if( (blocklen_bytes * cando_count) > *(SPACE) )
+    if ((blocklen_bytes * cando_count) > *(SPACE))
         cando_count = (*SPACE) / blocklen_bytes;
 
-    /* premptively update the number of COUNT we will return. */
+    /* preemptively update the number of COUNT we will return. */
     *(COUNT) -= cando_count;
 
-    if( _elem->blocklen < 9 ) {
-        if(OPAL_LIKELY(OPAL_SUCCESS == opal_datatype_unpack_predefined_element(&_packed, &_memory, cando_count, _elem))) {
+    if (_elem->blocklen < 9) {
+        if (!(CONVERTOR->flags & CONVERTOR_ACCELERATOR)
+            && OPAL_LIKELY(OPAL_SUCCESS
+                           == opal_datatype_unpack_predefined_element(&_packed, &_memory,
+                                                                      cando_count, _elem))) {
             goto update_and_return;
         }
         /* else unrecognized _elem->common.type, use the memcpy path */
     }
 
-    if( 1 == _elem->blocklen ) {  /* Do as many full blocklen as possible */
-        for(; cando_count > 0; cando_count--) {
+    if (1 == _elem->blocklen) {  /* Do as many full blocklen as possible */
+        for (; cando_count > 0; cando_count--) {
             OPAL_DATATYPE_SAFEGUARD_POINTER( _memory, blocklen_bytes, (CONVERTOR)->pBaseBuf,
                                              (CONVERTOR)->pDesc, (CONVERTOR)->count );
-            DO_DEBUG( opal_output( 0, "unpack memcpy( %p, %p, %lu ) => space %lu [blen = 1]\n",
-                                   (void*)_memory, (void*)_packed, (unsigned long)blocklen_bytes, (unsigned long)(*(SPACE) - (_packed - *(packed))) ); );
+            DO_DEBUG( opal_output( 0, "unpack memcpy( %p [%ld], %p [%ld], %lu ) => space %lu [blen = 1]\n",
+                                   (void*)_memory, _memory - CONVERTOR->pBaseBuf,
+                                   (void*)_packed, _packed - *packed,
+                                   (unsigned long)blocklen_bytes, (unsigned long)(*(SPACE) - (_packed - *(packed))) ); );
             MEMCPY_CSUM( _memory, _packed, blocklen_bytes, (CONVERTOR) );
             _packed     += blocklen_bytes;
             _memory     += _elem->extent;
@@ -125,14 +127,16 @@ unpack_predefined_data( opal_convertor_t* CONVERTOR,
         goto update_and_return;
     }
 
-    if( (1 < _elem->count) && (_elem->blocklen <= cando_count) ) {
+    if ((1 < _elem->count) && (_elem->blocklen <= cando_count)) {
         blocklen_bytes *= _elem->blocklen;
 
         do { /* Do as many full blocklen as possible */
             OPAL_DATATYPE_SAFEGUARD_POINTER( _memory, blocklen_bytes, (CONVERTOR)->pBaseBuf,
                                              (CONVERTOR)->pDesc, (CONVERTOR)->count );
-            DO_DEBUG( opal_output( 0, "unpack 2. memcpy( %p, %p, %lu ) => space %lu\n",
-                                   (void*)_memory, (void*)_packed, (unsigned long)blocklen_bytes, (unsigned long)(*(SPACE) - (_packed - *(packed))) ); );
+            DO_DEBUG( opal_output( 0, "unpack 2. memcpy( %p [%ld], %p [%ld], %lu ) => space %lu\n",
+                                   (void*)_memory, _memory - CONVERTOR->pBaseBuf,
+                                   (void*)_packed, _packed - *packed,
+                                   (unsigned long)blocklen_bytes, (unsigned long)(*(SPACE) - (_packed - *(packed))) ); );
             MEMCPY_CSUM( _memory, _packed, blocklen_bytes, (CONVERTOR) );
             _packed     += blocklen_bytes;
             _memory     += _elem->extent;
@@ -143,70 +147,70 @@ unpack_predefined_data( opal_convertor_t* CONVERTOR,
     /**
      * As an epilog do anything left from the last blocklen.
      */
-    if( 0 != cando_count ) {
-        assert( (cando_count < _elem->blocklen) ||
-                ((1 == _elem->count) && (cando_count <= _elem->blocklen)) );
+    if (0 != cando_count) {
+        assert((cando_count < _elem->blocklen)
+               || ((1 == _elem->count) && (cando_count <= _elem->blocklen)));
         do_now_bytes = cando_count * opal_datatype_basicDatatypes[_elem->common.type]->size;
         OPAL_DATATYPE_SAFEGUARD_POINTER( _memory, do_now_bytes, (CONVERTOR)->pBaseBuf,
                                          (CONVERTOR)->pDesc, (CONVERTOR)->count );
-        DO_DEBUG( opal_output( 0, "unpack 3. memcpy( %p, %p, %lu ) => space %lu [epilog]\n",
-                               (void*)_memory, (void*)_packed, (unsigned long)do_now_bytes, (unsigned long)(*(SPACE) - (_packed - *(packed))) ); );
+        DO_DEBUG( opal_output( 0, "unpack 3. memcpy( %p [%ld], %p [%ld], %lu ) => space %lu [epilog]\n",
+                               (void*)_memory, _memory - CONVERTOR->pBaseBuf,
+                               (void*)_packed, _packed - *packed,
+                               (unsigned long)do_now_bytes, (unsigned long)(*(SPACE) - (_packed - *(packed))) ); );
         MEMCPY_CSUM( _memory, _packed, do_now_bytes, (CONVERTOR) );
         _memory   += do_now_bytes;
         _packed   += do_now_bytes;
     }
 
- update_and_return:
-    *(memory)  = _memory - _elem->disp;
-    *(SPACE)  -= (_packed - *packed);
-    *(packed)  = _packed;
+update_and_return:
+    *(memory) = _memory - _elem->disp;
+    *(SPACE) -= (_packed - *packed);
+    *(packed) = _packed;
 }
 
-static inline void unpack_contiguous_loop( opal_convertor_t* CONVERTOR,
-                                           const dt_elem_desc_t* ELEM,
-                                           size_t* COUNT,
-                                           unsigned char** packed,
-                                           unsigned char** memory,
-                                           size_t* SPACE )
+static inline void unpack_contiguous_loop(opal_convertor_t *CONVERTOR, const dt_elem_desc_t *ELEM,
+                                          size_t *COUNT, unsigned char **packed,
+                                          unsigned char **memory, size_t *SPACE)
 {
-    const ddt_loop_desc_t *_loop = (ddt_loop_desc_t*)(ELEM);
-    const ddt_endloop_desc_t* _end_loop = (ddt_endloop_desc_t*)((ELEM) + _loop->items);
-    unsigned char* _memory = (*memory) + _end_loop->first_elem_disp;
+    const ddt_loop_desc_t *_loop = (ddt_loop_desc_t *) (ELEM);
+    const ddt_endloop_desc_t *_end_loop = (ddt_endloop_desc_t *) ((ELEM) + _loop->items);
+    unsigned char *_memory = (*memory) + _end_loop->first_elem_disp;
     size_t _copy_loops = *(COUNT);
 
-    if( (_copy_loops * _end_loop->size) > *(SPACE) )
+    if ((_copy_loops * _end_loop->size) > *(SPACE))
         _copy_loops = (*(SPACE) / _end_loop->size);
-    for(size_t _i = 0; _i < _copy_loops; _i++ ) {
-        OPAL_DATATYPE_SAFEGUARD_POINTER( _memory, _end_loop->size, (CONVERTOR)->pBaseBuf,
-                                    (CONVERTOR)->pDesc, (CONVERTOR)->count );
-        DO_DEBUG( opal_output( 0, "unpack 3. memcpy( %p, %p, %lu ) => space %lu\n",
-                               (void*)_memory, (void*)*(packed), (unsigned long)_end_loop->size, (unsigned long)(*(SPACE) - _i * _end_loop->size) ); );
-        MEMCPY_CSUM( _memory, *(packed), _end_loop->size, (CONVERTOR) );
+    for (size_t _i = 0; _i < _copy_loops; _i++) {
+        OPAL_DATATYPE_SAFEGUARD_POINTER(_memory, _end_loop->size, (CONVERTOR)->pBaseBuf,
+                                        (CONVERTOR)->pDesc, (CONVERTOR)->count);
+        DO_DEBUG(opal_output(0, "unpack 3. memcpy( %p, %p, %lu ) => space %lu\n", (void *) _memory,
+                             (void *) *(packed), (unsigned long) _end_loop->size,
+                             (unsigned long) (*(SPACE) -_i * _end_loop->size)););
+        MEMCPY_CSUM(_memory, *(packed), _end_loop->size, (CONVERTOR));
         *(packed) += _end_loop->size;
-        _memory   += _loop->extent;
+        _memory += _loop->extent;
     }
-    *(memory)  = _memory - _end_loop->first_elem_disp;
-    *(SPACE)  -= _copy_loops * _end_loop->size;
-    *(COUNT)  -= _copy_loops;
+    *(memory) = _memory - _end_loop->first_elem_disp;
+    *(SPACE) -= _copy_loops * _end_loop->size;
+    *(COUNT) -= _copy_loops;
 }
 
-#define UNPACK_PARTIAL_BLOCKLEN( CONVERTOR,    /* the convertor */       \
-                                 ELEM,         /* the basic element to be packed */ \
-                                 COUNT,        /* the number of elements */ \
-                                 PACKED,       /* the destination pointer (char*) */ \
-                                 MEMORY,       /* the source pointer (char*) */ \
-                                 SPACE )       /* the space in the destination buffer */ \
-unpack_partial_blocklen( (CONVERTOR), (ELEM), &(COUNT), &(PACKED), &(MEMORY), &(SPACE) )
-
-#define UNPACK_PREDEFINED_DATATYPE( CONVERTOR,    /* the convertor */    \
-                                    ELEM,         /* the basic element to be packed */ \
-                                    COUNT,        /* the number of elements */ \
-                                    PACKED,       /* the destination pointer (char*) */ \
-                                    MEMORY,       /* the source pointer (char*) */ \
-                                    SPACE )       /* the space in the destination buffer */ \
-unpack_predefined_data( (CONVERTOR), (ELEM), &(COUNT), &(PACKED), &(MEMORY), &(SPACE) )
-
-#define UNPACK_CONTIGUOUS_LOOP( CONVERTOR, ELEM, COUNT, PACKED, MEMORY, SPACE ) \
-    unpack_contiguous_loop( (CONVERTOR), (ELEM), &(COUNT), &(PACKED), &(MEMORY), &(SPACE) )
-
-#endif  /* OPAL_DATATYPE_UNPACK_H_HAS_BEEN_INCLUDED */
+#define UNPACK_PARTIAL_BLOCKLEN(CONVERTOR, /* the convertor */                       \
+                                ELEM,      /* the basic element to be packed */      \
+                                COUNT,     /* the number of elements */              \
+                                PACKED,    /* the destination pointer (char*) */     \
+                                MEMORY,    /* the source pointer (char*) */          \
+                                SPACE)     /* the space in the destination buffer */ \
+    unpack_partial_blocklen((CONVERTOR), (ELEM), &(COUNT), &(PACKED), &(MEMORY), &(SPACE))
+
+#define UNPACK_PREDEFINED_DATATYPE(CONVERTOR, /* the convertor */                       \
+                                   ELEM,      /* the basic element to be packed */      \
+                                   COUNT,     /* the number of elements */              \
+                                   PACKED,    /* the destination pointer (char*) */     \
+                                   MEMORY,    /* the source pointer (char*) */          \
+                                   SPACE)     /* the space in the destination buffer */ \
+    unpack_predefined_data((CONVERTOR), (ELEM), &(COUNT), &(PACKED), &(MEMORY), &(SPACE))
+
+#define UNPACK_CONTIGUOUS_LOOP(CONVERTOR, ELEM, COUNT, PACKED, MEMORY, SPACE) \
+    unpack_contiguous_loop((CONVERTOR), (ELEM), &(COUNT), &(PACKED), &(MEMORY), &(SPACE))
+
+#endif /* OPAL_DATATYPE_UNPACK_H_HAS_BEEN_INCLUDED */
diff --git a/opal/etc/Makefile.am b/opal/etc/Makefile.am
index 6325cefe752..b9c6394cd4c 100644
--- a/opal/etc/Makefile.am
+++ b/opal/etc/Makefile.am
@@ -54,7 +54,7 @@ install-data-local:
 
 # Only remove if exactly the same as what in our tree
 # NOTE TO READER: Bourne shell if ... fi evaluates the body if
-#    the return of the evaluted command is 0 (as opposed to non-zero
+#    the return of the evaluated command is 0 (as opposed to non-zero
 #    as used by everyone else)
 uninstall-local:
 	@ p="$(opal_config_files)"; \
diff --git a/opal/include/opal/Makefile.am b/opal/include/opal/Makefile.am
index f0389a25815..138e463df42 100644
--- a/opal/include/opal/Makefile.am
+++ b/opal/include/opal/Makefile.am
@@ -10,6 +10,7 @@
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
 # Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2021      Nanook Consulting.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -27,7 +28,9 @@ headers += \
 	opal/prefetch.h \
         opal/hash_string.h \
 	opal/frameworks.h \
-	opal/opal_portable_platform.h
+	opal/opal_portable_platform.h \
+	opal/opal_portable_platform_real.h \
+	opal/opal_gpu.h
 
 nodist_headers += \
 	opal/version.h
diff --git a/opal/include/opal/align.h b/opal/include/opal/align.h
index ba87490f8b2..5df9d9485ea 100644
--- a/opal/include/opal/align.h
+++ b/opal/include/opal/align.h
@@ -5,7 +5,7 @@
  * Copyright (c) 2004-2005 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ * Copyright (c) 2004-2020 High Performance Computing Center Stuttgart,
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
@@ -22,25 +22,25 @@
 #ifndef OPAL_ALIGN_H
 #define OPAL_ALIGN_H
 
-#define OPAL_DOWN_ALIGN(x,a,t) ((x) & ~(((t)(a)-1)))
-#define OPAL_DOWN_ALIGN_PTR(x,a,t) ((t)OPAL_DOWN_ALIGN((uintptr_t)x, a, uintptr_t))
-#define OPAL_ALIGN(x,a,t) (((x)+((t)(a)-1)) & ~(((t)(a)-1)))
-#define OPAL_ALIGN_PTR(x,a,t) ((t)OPAL_ALIGN((uintptr_t)x, a, uintptr_t))
-#define OPAL_ALIGN_PAD_AMOUNT(x,s) ((~((uintptr_t)(x))+1) & ((uintptr_t)(s)-1))
+#define OPAL_DOWN_ALIGN(x,a,t)       ((x) & ~(((t)(a)-1)))
+#define OPAL_DOWN_ALIGN_PTR(x,a,t)   ((t)OPAL_DOWN_ALIGN((uintptr_t)x, a, uintptr_t))
+#define OPAL_ALIGN(x,a,t)            (((x)+((t)(a)-1)) & ~(((t)(a)-1)))
+#define OPAL_ALIGN_PTR(x,a,t)        ((t)OPAL_ALIGN((uintptr_t)x, a, uintptr_t))
+#define OPAL_ALIGN_PAD_AMOUNT(x,s)   ((~((uintptr_t)(x))+1) & ((uintptr_t)(s)+(!(uintptr_t)(s))-1))
 
 #if __STDC_VERSION__ >= 201101L
-#include <stddef.h>
-#define OPAL_ALIGN_MIN (_Alignof(max_align_t))
+#    include <stddef.h>
+#    define OPAL_ALIGN_MIN (_Alignof(max_align_t))
 #else
-#if defined(OPAL_ALIGNMENT___FLOAT128)
-#define OPAL_ALIGN_MIN (OPAL_ALIGNMENT___FLOAT128)
-#elif defined(OPAL_ALIGNMENT_LONG_DOUBLE_COMPLEX)
-#define OPAL_ALIGN_MIN (OPAL_ALIGNMENT_LONG_DOUBLE_COMPLEX)
-#elif defined (OPAL_ALIGNMENT_LONG_DOUBLE)
-#define OPAL_ALIGN_MIN (OPAL_ALIGNMENT_LONG_DOUBLE)
-#else
-#define OPAL_ALIGN_MIN (OPAL_ALIGNMENT_DOUBLE)
-#endif
+#    if defined(OPAL_ALIGNMENT___FLOAT128)
+#        define OPAL_ALIGN_MIN (OPAL_ALIGNMENT___FLOAT128)
+#    elif defined(OPAL_ALIGNMENT_LONG_DOUBLE_COMPLEX)
+#        define OPAL_ALIGN_MIN (OPAL_ALIGNMENT_LONG_DOUBLE_COMPLEX)
+#    elif defined(OPAL_ALIGNMENT_LONG_DOUBLE)
+#        define OPAL_ALIGN_MIN (OPAL_ALIGNMENT_LONG_DOUBLE)
+#    else
+#        define OPAL_ALIGN_MIN (OPAL_ALIGNMENT_DOUBLE)
+#    endif
 #endif /* __STDC_VERSION__ >= 201101L */
 
 #endif /* OPAL_ALIGN_H */
diff --git a/opal/include/opal/constants.h b/opal/include/opal/constants.h
index 9ea9d0362da..72dc75d5934 100644
--- a/opal/include/opal/constants.h
+++ b/opal/include/opal/constants.h
@@ -23,85 +23,85 @@
 
 /* error codes - don't forget to update opal/rutime/opal_init.c when
    adding to this list */
-#define OPAL_ERR_BASE             0 /* internal use only */
+#define OPAL_ERR_BASE 0 /* internal use only */
 
 enum {
-    OPAL_SUCCESS                            = (OPAL_ERR_BASE),
+    OPAL_SUCCESS = (OPAL_ERR_BASE),
 
-    OPAL_ERROR                              = (OPAL_ERR_BASE -  1),
-    OPAL_ERR_OUT_OF_RESOURCE                = (OPAL_ERR_BASE -  2), /* fatal error */
-    OPAL_ERR_TEMP_OUT_OF_RESOURCE           = (OPAL_ERR_BASE -  3), /* try again later */
-    OPAL_ERR_RESOURCE_BUSY                  = (OPAL_ERR_BASE -  4),
-    OPAL_ERR_BAD_PARAM                      = (OPAL_ERR_BASE -  5),  /* equivalent to MPI_ERR_ARG error code */
-    OPAL_ERR_FATAL                          = (OPAL_ERR_BASE -  6),
-    OPAL_ERR_NOT_IMPLEMENTED                = (OPAL_ERR_BASE -  7),
-    OPAL_ERR_NOT_SUPPORTED                  = (OPAL_ERR_BASE -  8),
-    OPAL_ERR_INTERRUPTED                    = (OPAL_ERR_BASE -  9),
-    OPAL_ERR_WOULD_BLOCK                    = (OPAL_ERR_BASE - 10),
-    OPAL_ERR_IN_ERRNO                       = (OPAL_ERR_BASE - 11),
-    OPAL_ERR_UNREACH                        = (OPAL_ERR_BASE - 12),
-    OPAL_ERR_NOT_FOUND                      = (OPAL_ERR_BASE - 13),
-    OPAL_EXISTS                             = (OPAL_ERR_BASE - 14), /* indicates that the specified object already exists */
-    OPAL_ERR_TIMEOUT                        = (OPAL_ERR_BASE - 15),
-    OPAL_ERR_NOT_AVAILABLE                  = (OPAL_ERR_BASE - 16),
-    OPAL_ERR_PERM                           = (OPAL_ERR_BASE - 17), /* no permission */
-    OPAL_ERR_VALUE_OUT_OF_BOUNDS            = (OPAL_ERR_BASE - 18),
-    OPAL_ERR_FILE_READ_FAILURE              = (OPAL_ERR_BASE - 19),
-    OPAL_ERR_FILE_WRITE_FAILURE             = (OPAL_ERR_BASE - 20),
-    OPAL_ERR_FILE_OPEN_FAILURE              = (OPAL_ERR_BASE - 21),
-    OPAL_ERR_PACK_MISMATCH                  = (OPAL_ERR_BASE - 22),
-    OPAL_ERR_PACK_FAILURE                   = (OPAL_ERR_BASE - 23),
-    OPAL_ERR_UNPACK_FAILURE                 = (OPAL_ERR_BASE - 24),
-    OPAL_ERR_UNPACK_INADEQUATE_SPACE        = (OPAL_ERR_BASE - 25),
+    OPAL_ERROR = (OPAL_ERR_BASE - 1),
+    OPAL_ERR_OUT_OF_RESOURCE = (OPAL_ERR_BASE - 2),      /* fatal error */
+    OPAL_ERR_TEMP_OUT_OF_RESOURCE = (OPAL_ERR_BASE - 3), /* try again later */
+    OPAL_ERR_RESOURCE_BUSY = (OPAL_ERR_BASE - 4),
+    OPAL_ERR_BAD_PARAM = (OPAL_ERR_BASE - 5), /* equivalent to MPI_ERR_ARG error code */
+    OPAL_ERR_FATAL = (OPAL_ERR_BASE - 6),
+    OPAL_ERR_NOT_IMPLEMENTED = (OPAL_ERR_BASE - 7),
+    OPAL_ERR_NOT_SUPPORTED = (OPAL_ERR_BASE - 8),
+    OPAL_ERR_INTERRUPTED = (OPAL_ERR_BASE - 9),
+    OPAL_ERR_WOULD_BLOCK = (OPAL_ERR_BASE - 10),
+    OPAL_ERR_IN_ERRNO = (OPAL_ERR_BASE - 11),
+    OPAL_ERR_UNREACH = (OPAL_ERR_BASE - 12),
+    OPAL_ERR_NOT_FOUND = (OPAL_ERR_BASE - 13),
+    OPAL_EXISTS = (OPAL_ERR_BASE - 14), /* indicates that the specified object already exists */
+    OPAL_ERR_TIMEOUT = (OPAL_ERR_BASE - 15),
+    OPAL_ERR_NOT_AVAILABLE = (OPAL_ERR_BASE - 16),
+    OPAL_ERR_PERM = (OPAL_ERR_BASE - 17), /* no permission */
+    OPAL_ERR_VALUE_OUT_OF_BOUNDS = (OPAL_ERR_BASE - 18),
+    OPAL_ERR_FILE_READ_FAILURE = (OPAL_ERR_BASE - 19),
+    OPAL_ERR_FILE_WRITE_FAILURE = (OPAL_ERR_BASE - 20),
+    OPAL_ERR_FILE_OPEN_FAILURE = (OPAL_ERR_BASE - 21),
+    OPAL_ERR_PACK_MISMATCH = (OPAL_ERR_BASE - 22),
+    OPAL_ERR_PACK_FAILURE = (OPAL_ERR_BASE - 23),
+    OPAL_ERR_UNPACK_FAILURE = (OPAL_ERR_BASE - 24),
+    OPAL_ERR_UNPACK_INADEQUATE_SPACE = (OPAL_ERR_BASE - 25),
     OPAL_ERR_UNPACK_READ_PAST_END_OF_BUFFER = (OPAL_ERR_BASE - 26),
-    OPAL_ERR_TYPE_MISMATCH                  = (OPAL_ERR_BASE - 27),
-    OPAL_ERR_OPERATION_UNSUPPORTED          = (OPAL_ERR_BASE - 28),
-    OPAL_ERR_UNKNOWN_DATA_TYPE              = (OPAL_ERR_BASE - 29),
-    OPAL_ERR_BUFFER                         = (OPAL_ERR_BASE - 30),
-    OPAL_ERR_DATA_TYPE_REDEF                = (OPAL_ERR_BASE - 31),
-    OPAL_ERR_DATA_OVERWRITE_ATTEMPT         = (OPAL_ERR_BASE - 32),
-    OPAL_ERR_MODULE_NOT_FOUND               = (OPAL_ERR_BASE - 33),
-    OPAL_ERR_TOPO_SLOT_LIST_NOT_SUPPORTED   = (OPAL_ERR_BASE - 34),
-    OPAL_ERR_TOPO_SOCKET_NOT_SUPPORTED      = (OPAL_ERR_BASE - 35),
-    OPAL_ERR_TOPO_CORE_NOT_SUPPORTED        = (OPAL_ERR_BASE - 36),
-    OPAL_ERR_NOT_ENOUGH_SOCKETS             = (OPAL_ERR_BASE - 37),
-    OPAL_ERR_NOT_ENOUGH_CORES               = (OPAL_ERR_BASE - 38),
-    OPAL_ERR_INVALID_PHYS_CPU               = (OPAL_ERR_BASE - 39),
-    OPAL_ERR_MULTIPLE_AFFINITIES            = (OPAL_ERR_BASE - 40),
-    OPAL_ERR_SLOT_LIST_RANGE                = (OPAL_ERR_BASE - 41),
-    OPAL_ERR_NETWORK_NOT_PARSEABLE          = (OPAL_ERR_BASE - 42),
-    OPAL_ERR_SILENT                         = (OPAL_ERR_BASE - 43),
-    OPAL_ERR_NOT_INITIALIZED                = (OPAL_ERR_BASE - 44),
-    OPAL_ERR_NOT_BOUND                      = (OPAL_ERR_BASE - 45),
-    OPAL_ERR_TAKE_NEXT_OPTION               = (OPAL_ERR_BASE - 46),
-    OPAL_ERR_PROC_ENTRY_NOT_FOUND           = (OPAL_ERR_BASE - 47),
-    OPAL_ERR_DATA_VALUE_NOT_FOUND           = (OPAL_ERR_BASE - 48),
-    OPAL_ERR_CONNECTION_FAILED              = (OPAL_ERR_BASE - 49),
-    OPAL_ERR_AUTHENTICATION_FAILED          = (OPAL_ERR_BASE - 50),
-    OPAL_ERR_COMM_FAILURE                   = (OPAL_ERR_BASE - 51),
-    OPAL_ERR_SERVER_NOT_AVAIL               = (OPAL_ERR_BASE - 52),
-    OPAL_ERR_IN_PROCESS                     = (OPAL_ERR_BASE - 53),
+    OPAL_ERR_TYPE_MISMATCH = (OPAL_ERR_BASE - 27),
+    OPAL_ERR_OPERATION_UNSUPPORTED = (OPAL_ERR_BASE - 28),
+    OPAL_ERR_UNKNOWN_DATA_TYPE = (OPAL_ERR_BASE - 29),
+    OPAL_ERR_BUFFER = (OPAL_ERR_BASE - 30),
+    OPAL_ERR_DATA_TYPE_REDEF = (OPAL_ERR_BASE - 31),
+    OPAL_ERR_DATA_OVERWRITE_ATTEMPT = (OPAL_ERR_BASE - 32),
+    OPAL_ERR_MODULE_NOT_FOUND = (OPAL_ERR_BASE - 33),
+    OPAL_ERR_TOPO_SLOT_LIST_NOT_SUPPORTED = (OPAL_ERR_BASE - 34),
+    OPAL_ERR_TOPO_SOCKET_NOT_SUPPORTED = (OPAL_ERR_BASE - 35),
+    OPAL_ERR_TOPO_CORE_NOT_SUPPORTED = (OPAL_ERR_BASE - 36),
+    OPAL_ERR_NOT_ENOUGH_SOCKETS = (OPAL_ERR_BASE - 37),
+    OPAL_ERR_NOT_ENOUGH_CORES = (OPAL_ERR_BASE - 38),
+    OPAL_ERR_INVALID_PHYS_CPU = (OPAL_ERR_BASE - 39),
+    OPAL_ERR_MULTIPLE_AFFINITIES = (OPAL_ERR_BASE - 40),
+    OPAL_ERR_SLOT_LIST_RANGE = (OPAL_ERR_BASE - 41),
+    OPAL_ERR_NETWORK_NOT_PARSEABLE = (OPAL_ERR_BASE - 42),
+    OPAL_ERR_SILENT = (OPAL_ERR_BASE - 43),
+    OPAL_ERR_NOT_INITIALIZED = (OPAL_ERR_BASE - 44),
+    OPAL_ERR_NOT_BOUND = (OPAL_ERR_BASE - 45),
+    OPAL_ERR_TAKE_NEXT_OPTION = (OPAL_ERR_BASE - 46),
+    OPAL_ERR_PROC_ENTRY_NOT_FOUND = (OPAL_ERR_BASE - 47),
+    OPAL_ERR_DATA_VALUE_NOT_FOUND = (OPAL_ERR_BASE - 48),
+    OPAL_ERR_CONNECTION_FAILED = (OPAL_ERR_BASE - 49),
+    OPAL_ERR_AUTHENTICATION_FAILED = (OPAL_ERR_BASE - 50),
+    OPAL_ERR_COMM_FAILURE = (OPAL_ERR_BASE - 51),
+    OPAL_ERR_SERVER_NOT_AVAIL = (OPAL_ERR_BASE - 52),
+    OPAL_ERR_IN_PROCESS = (OPAL_ERR_BASE - 53),
     /* PMIx equivalents for notification support */
-    OPAL_ERR_DEBUGGER_RELEASE               = (OPAL_ERR_BASE - 54),
-    OPAL_ERR_HANDLERS_COMPLETE              = (OPAL_ERR_BASE - 55),
-    OPAL_ERR_PARTIAL_SUCCESS                = (OPAL_ERR_BASE - 56),
-    OPAL_ERR_PROC_ABORTED                   = (OPAL_ERR_BASE - 57),
-    OPAL_ERR_PROC_REQUESTED_ABORT           = (OPAL_ERR_BASE - 58),
-    OPAL_ERR_PROC_ABORTING                  = (OPAL_ERR_BASE - 59),
-    OPAL_ERR_NODE_DOWN                      = (OPAL_ERR_BASE - 60),
-    OPAL_ERR_NODE_OFFLINE                   = (OPAL_ERR_BASE - 61),
-    OPAL_ERR_JOB_TERMINATED                 = (OPAL_ERR_BASE - 62),
-    OPAL_ERR_PROC_RESTART                   = (OPAL_ERR_BASE - 63),
-    OPAL_ERR_PROC_CHECKPOINT                = (OPAL_ERR_BASE - 64),
-    OPAL_ERR_PROC_MIGRATE                   = (OPAL_ERR_BASE - 65),
-    OPAL_ERR_EVENT_REGISTRATION             = (OPAL_ERR_BASE - 66),
-    OPAL_ERR_HEARTBEAT_ALERT                = (OPAL_ERR_BASE - 67),
-    OPAL_ERR_FILE_ALERT                     = (OPAL_ERR_BASE - 68),
-    OPAL_ERR_MODEL_DECLARED                 = (OPAL_ERR_BASE - 69),
-    OPAL_PMIX_LAUNCH_DIRECTIVE              = (OPAL_ERR_BASE - 70),
-    OPAL_OPERATION_SUCCEEDED                = (OPAL_ERR_BASE - 71)
+    OPAL_ERR_DEBUGGER_RELEASE = (OPAL_ERR_BASE - 54),
+    OPAL_ERR_HANDLERS_COMPLETE = (OPAL_ERR_BASE - 55),
+    OPAL_ERR_PARTIAL_SUCCESS = (OPAL_ERR_BASE - 56),
+    OPAL_ERR_PROC_ABORTED = (OPAL_ERR_BASE - 57),
+    OPAL_ERR_PROC_REQUESTED_ABORT = (OPAL_ERR_BASE - 58),
+    OPAL_ERR_PROC_ABORTING = (OPAL_ERR_BASE - 59),
+    OPAL_ERR_NODE_DOWN = (OPAL_ERR_BASE - 60),
+    OPAL_ERR_NODE_OFFLINE = (OPAL_ERR_BASE - 61),
+    OPAL_ERR_JOB_TERMINATED = (OPAL_ERR_BASE - 62),
+    OPAL_ERR_PROC_RESTART = (OPAL_ERR_BASE - 63),
+    OPAL_ERR_PROC_CHECKPOINT = (OPAL_ERR_BASE - 64),
+    OPAL_ERR_PROC_MIGRATE = (OPAL_ERR_BASE - 65),
+    OPAL_ERR_EVENT_REGISTRATION = (OPAL_ERR_BASE - 66),
+    OPAL_ERR_HEARTBEAT_ALERT = (OPAL_ERR_BASE - 67),
+    OPAL_ERR_FILE_ALERT = (OPAL_ERR_BASE - 68),
+    OPAL_ERR_MODEL_DECLARED = (OPAL_ERR_BASE - 69),
+    OPAL_PMIX_LAUNCH_DIRECTIVE = (OPAL_ERR_BASE - 70),
+    OPAL_OPERATION_SUCCEEDED = (OPAL_ERR_BASE - 71)
 };
 
-#define OPAL_ERR_MAX                (OPAL_ERR_BASE - 100)
+#define OPAL_ERR_MAX (OPAL_ERR_BASE - 100)
 
 #endif /* OPAL_CONSTANTS_H */
diff --git a/opal/include/opal/hash_string.h b/opal/include/opal/hash_string.h
index c4af2153806..58b71a3d02e 100644
--- a/opal/include/opal/hash_string.h
+++ b/opal/include/opal/hash_string.h
@@ -25,24 +25,24 @@
  *  @param hash (OUT)   Where the hash value will be stored (uint32_t)
  *  @param length (OUT) The computed length of the string (uint32_t)
  */
-#define OPAL_HASH_STRLEN( str, hash, length ) \
-    do {                                      \
-        register const char *_str = (str);    \
-        register uint32_t    _hash = 0;       \
-        register uint32_t    _len = 0;        \
-                                              \
-        while( *_str ) {                      \
-            _len++;                           \
-            _hash += *_str++;                 \
-            _hash += (_hash << 10);           \
-            _hash ^= (_hash >> 6);            \
-        }                                     \
-                                              \
-        _hash += (_hash << 3);                \
-        _hash ^= (_hash >> 11);               \
-        (hash) = (_hash + (_hash << 15));     \
-        (length)  = _len;                     \
-    } while(0)
+#define OPAL_HASH_STRLEN(str, hash, length) \
+    do {                                    \
+        register const char *_str = (str);  \
+        register uint32_t _hash = 0;        \
+        register uint32_t _len = 0;         \
+                                            \
+        while (*_str) {                     \
+            _len++;                         \
+            _hash += *_str++;               \
+            _hash += (_hash << 10);         \
+            _hash ^= (_hash >> 6);          \
+        }                                   \
+                                            \
+        _hash += (_hash << 3);              \
+        _hash ^= (_hash >> 11);             \
+        (hash) = (_hash + (_hash << 15));   \
+        (length) = _len;                    \
+    } while (0)
 
 /**
  *  Compute the hash value
@@ -50,20 +50,20 @@
  *  @param str (IN)     The string which will be parsed   (char*)
  *  @param hash (OUT)   Where the hash value will be stored (uint32_t)
  */
-#define OPAL_HASH_STR( str, hash )            \
-    do {                                      \
-        register const char *_str = (str);    \
-        register uint32_t    _hash = 0;       \
-                                              \
-        while( *_str ) {                      \
-            _hash += *_str++;                 \
-            _hash += (_hash << 10);           \
-            _hash ^= (_hash >> 6);            \
-        }                                     \
-                                              \
-        _hash += (_hash << 3);                \
-        _hash ^= (_hash >> 11);               \
-        (hash) = (_hash + (_hash << 15));     \
-    } while(0)
+#define OPAL_HASH_STR(str, hash)           \
+    do {                                   \
+        register const char *_str = (str); \
+        register uint32_t _hash = 0;       \
+                                           \
+        while (*_str) {                    \
+            _hash += *_str++;              \
+            _hash += (_hash << 10);        \
+            _hash ^= (_hash >> 6);         \
+        }                                  \
+                                           \
+        _hash += (_hash << 3);             \
+        _hash ^= (_hash >> 11);            \
+        (hash) = (_hash + (_hash << 15));  \
+    } while (0)
 
-#endif  /* OPAL_HASH_STRING_H */
+#endif /* OPAL_HASH_STRING_H */
diff --git a/opal/include/opal/opal_gpu.h b/opal/include/opal/opal_gpu.h
new file mode 100644
index 00000000000..36e9844dbb7
--- /dev/null
+++ b/opal/include/opal/opal_gpu.h
@@ -0,0 +1,47 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2024-2006 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2013 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2006 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2015 NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * Copyright (c) 2023      Advanced Micro Devices, Inc. 
+ *                         All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#ifndef OPAL_GPU_H
+#define OPAL_GPU_H
+#include "opal/mca/rcache/rcache.h"
+#include "opal/mca/accelerator/accelerator.h"
+
+struct mca_opal_gpu_reg_data_t {
+    opal_accelerator_ipc_handle_t       ipcHandle;
+    opal_accelerator_ipc_event_handle_t ipcEventHandle;
+    opal_accelerator_event_t            *event;
+    opal_ptr_t                          memh_seg_addr;
+    size_t                              memh_seg_len;
+};
+typedef struct mca_opal_gpu_reg_data_t mca_opal_gpu_reg_data_t;
+
+struct mca_opal_gpu_reg_t {
+    mca_rcache_base_registration_t base;
+    mca_opal_gpu_reg_data_t data;
+};
+typedef struct mca_opal_gpu_reg_t mca_opal_gpu_reg_t;
+#endif /* OPAL_GPU_H */
diff --git a/opal/include/opal/opal_portable_platform.h b/opal/include/opal/opal_portable_platform.h
index 5eaa28998b3..7924c530baf 100644
--- a/opal/include/opal/opal_portable_platform.h
+++ b/opal/include/opal/opal_portable_platform.h
@@ -1,401 +1,23 @@
 /*
- * Header file with preprocessor magic to figure out, which compiler the user has been calling!
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
+ * Copyright (c) 2024      Jeffrey M. Squyres.  All rights reserved.
+ * $COPYRIGHT$
  *
- * This code is adapted from the file other/portable_platform.h of GASnet-1.14.0:
- *  - Ripping out the required parts.
- *  - Get rid of brackets as it messes up autoconf
- *  - Delete version tests for older PGI versions (#include "omp.h" not acceptabe)
- *  - Indent ('#' should be in column 0)
+ * Additional copyrights may follow
  *
- * External packages (i.e., romio) depend on top_build_dir/ompi/include, therefore
- * although this is not changed in the configure process, this has to be set as
- * a .in file...
- * ---------------------------------------------------------------------------
- */
-#ifndef OPAL_PORTABLE_PLATFORM_H
-#define OPAL_PORTABLE_PLATFORM_H
-
-/* All files in this directory and all sub-directories (except where otherwise noted)
- * are subject to the following licensing terms:
- *
- * ---------------------------------------------------------------------------
- * "Copyright (c) 2000-2003 The Regents of the University of California.
- * All rights reserved.
- *
- * Permission to use, copy, modify, and distribute this software and its
- * documentation for any purpose, without fee, and without written agreement is
- * hereby granted, provided that the above copyright notice and the following
- * two paragraphs appear in all copies of this software.
- *
- * IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR
- * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT
- * OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF THE UNIVERSITY OF
- * CALIFORNIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES,
- * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
- * AND FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
- * ON AN "AS IS" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATION TO
- * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS."
- * ---------------------------------------------------------------------------
- *
- * Please see the license.txt files within the gm-conduit, lapi-conduit and
- * vapi-conduit directories for the licensing terms governing those
- * contributed components.
- *
- * The authors/contributors of GASNet include:
- *
- *  Dan Bonachea <bonachea@cs.berkeley.edu>:
- *    General infrastructure & documentation
- *    mpi-conduit
- *    elan-conduit
- *    smp-conduit
- *    udp-conduit
- *    extended-ref
- *    template-conduit
- *  Christian Bell <csbell@cs.berkeley.edu>: gm-conduit, shmem-conduit
- *  Mike Welcome <mlwelcome@lbl.gov>: lapi-conduit, portals-conduit
- *  Paul H. Hargrove <phhargrove@lbl.gov>: vapi-conduit, ibv-conduit
- *  Rajesh Nishtala <rajeshn@cs.berkeley.edu>: collectives, dcmf-conduit
- *  Parry Husbands (PJRHusbands@lbl.gov): lapi-conduit
- *
- * For more information about GASNet, visit our home page at:
- *  http://gasnet.cs.berkeley.edu/
- * Or send email to:
- *  <upc@lbl.gov>
- *
- * Source code contributions (fixes, patches, extensions etc.) should be
- * sent to <upc@lbl.gov> to be reviewed for acceptance into the primary
- * distribution. Contributions are most likely to be accepted if they
- * are provided as public domain, or under a BSD-style license such as
- * the one above.
+ * $HEADER$
  *
+ * Wrapper around GASNet's gasnet_portable_platform.h to avoid
+ * compiler warnings.  This wrapper used to be necessary to add some
+ * additional #defines, but those issues have now been resolved
+ * upstream at gasnet.  However, we left this wrapper file just to be
+ * able to handle any future workarounds, if necessary.
  */
-#ifndef _STRINGIFY
-#define _STRINGIFY_HELPER(x) #x
-#define _STRINGIFY(x) _STRINGIFY_HELPER(x)
-#endif
-
-#if defined(__INTEL_COMPILER)
-#    define PLATFORM_COMPILER_FAMILYNAME INTEL
-#    define PLATFORM_COMPILER_FAMILYID 2
-#    ifdef __cplusplus
-#        define PLATFORM_COMPILER_INTEL_CXX  1
-#    else
-#        define PLATFORM_COMPILER_INTEL_C  1
-#    endif
-#    define _PLATFORM_COMPILER_INTEL_MIN_BUILDDATE 19700000 /* year 1970: predates most intel products :) */
-#    ifdef __INTEL_COMPILER_BUILD_DATE
-#        define _PLATFORM_INTEL_COMPILER_BUILD_DATE __INTEL_COMPILER_BUILD_DATE
-#    else
-#        define _PLATFORM_INTEL_COMPILER_BUILD_DATE _PLATFORM_COMPILER_INTEL_MIN_BUILDDATE
-#    endif
-     /* patch number is a decimal build date: YYYYMMDD */
-#    define PLATFORM_COMPILER_VERSION_INT(maj,min,pat)         \
-         (((((maj) * 10) | (min)) << 20) |                      \
-            ((pat) < _PLATFORM_COMPILER_INTEL_MIN_BUILDDATE ?   \
-             _PLATFORM_COMPILER_INTEL_MIN_BUILDDATE : ((pat)-_PLATFORM_COMPILER_INTEL_MIN_BUILDDATE)))
-#    define PLATFORM_COMPILER_VERSION \
-         PLATFORM_COMPILER_VERSION_INT(__INTEL_COMPILER/10, __INTEL_COMPILER/100, _PLATFORM_INTEL_COMPILER_BUILD_DATE)
-#    define PLATFORM_COMPILER_VERSION_STR \
-         _STRINGIFY(__INTEL_COMPILER) "." _STRINGIFY(_PLATFORM_INTEL_COMPILER_BUILD_DATE)
-
-#elif defined(__PATHSCALE__)
-#    define PLATFORM_COMPILER_PATHSCALE  1
-#    define PLATFORM_COMPILER_FAMILYNAME PATHSCALE
-#    define PLATFORM_COMPILER_FAMILYID 3
-#    ifdef __cplusplus
-#        define PLATFORM_COMPILER_PATHSCALE_CXX  1
-#    else
-#        define PLATFORM_COMPILER_PATHSCALE_C  1
-#    endif
-#    define PLATFORM_COMPILER_VERSION \
-         PLATFORM_COMPILER_VERSION_INT(__PATHCC__,__PATHCC_MINOR__,__PATHCC_PATCHLEVEL__)
-#    define PLATFORM_COMPILER_VERSION_STR __PATHSCALE__
-
-#elif defined(__PGI)
-#    define PLATFORM_COMPILER_PGI  1
-#    define PLATFORM_COMPILER_FAMILYNAME PGI
-#    define PLATFORM_COMPILER_FAMILYID 4
-#    ifdef __cplusplus
-#        define PLATFORM_COMPILER_PGI_CXX  1
-#    else
-#        define PLATFORM_COMPILER_PGI_C  1
-#    endif
-#    if __PGIC__ == 99
-      /* bug 2230: PGI versioning was broken for some platforms in 7.0
-         no way to know exact version, but provide something slightly more accurate */
-#      define PLATFORM_COMPILER_VERSION 0x070000
-#      define PLATFORM_COMPILER_VERSION_STR "7.?-?"
-#    elif defined(__PGIC__) && defined(__PGIC_MINOR__) && defined(__PGIC_PATCHLEVEL__)
-#      define PLATFORM_COMPILER_VERSION \
-            PLATFORM_COMPILER_VERSION_INT(__PGIC__,__PGIC_MINOR__,__PGIC_PATCHLEVEL__)
-#      define PLATFORM_COMPILER_VERSION_STR \
-            _STRINGIFY(__PGIC__) "." _STRINGIFY(__PGIC_MINOR__) "-" _STRINGIFY(__PGIC_PATCHLEVEL__)
-#    else
-       /* PGI before 6.1-4 lacks any version ID preprocessor macros - so use this filthy hack */
-       /* XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-        * We cannot do these within mpi.h.in, as we should not include ompi.h
-        * Hopefully, compilers with integrated preprocessors will not analyse code within the #if 0-block
-        * XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-        */
-#if 0
-#        ifdef PLATFORM_PGI_IS_ANCIENT
-         /* Include below might fail for ancient versions lacking this header, but testing shows it
-            works back to at least 5.1-3 (Nov 2003), and based on docs probably back to 3.2 (Sep 2000) */
-#            define PLATFORM_COMPILER_VERSION 0
-#        elif defined(__x86_64__) /* bug 1753 - 64-bit omp.h upgrade happenned in <6.0-8,6.1-1) */
-#            include "omp.h"
-#            if defined(_PGOMP_H)
-                 /* 6.1.1 or newer */
-#                define PLATFORM_COMPILER_VERSION 0x060101
-#                define PLATFORM_COMPILER_VERSION_STR ">=6.1-1"
-#            else
-                 /* 6.0.8 or older */
-#                define PLATFORM_COMPILER_VERSION 0
-#                define PLATFORM_COMPILER_VERSION_STR "<=6.0-8"
-#            endif
-#        else /* 32-bit omp.h upgrade happenned in <5.2-4,6.0-8 */
-#            include "omp.h"
-#            if defined(_PGOMP_H)
-                 /* 6.0-8 or newer */
-#                define PLATFORM_COMPILER_VERSION 0x060008
-#                define PLATFORM_COMPILER_VERSION_STR ">=6.0-8"
-#            else
-                 /* 5.2-4 or older */
-#                define PLATFORM_COMPILER_VERSION 0
-#                define PLATFORM_COMPILER_VERSION_STR "<=5.2-4"
-#            endif
-#        endif
-#endif /* 0 */
-       /* XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX */
-#    endif
-
-#elif defined(__xlC__) || defined(__ibmxl__) || defined(__IBMC__) || defined(__IBMCPP__)
-#    define PLATFORM_COMPILER_XLC  1
-#    define PLATFORM_COMPILER_FAMILYNAME XLC
-#    define PLATFORM_COMPILER_FAMILYID 5
-#    ifdef __cplusplus
-#        define PLATFORM_COMPILER_XLC_CXX  1
-#    else
-#        define PLATFORM_COMPILER_XLC_C  1
-#    endif
-#    define PLATFORM_COMPILER_VERSION __xlC__
-#    define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \
-         ( ((maj) << 8) | ((min) << 4) | (pat) )
-
-#elif defined(__DECC) || defined(__DECCXX)
-#    define PLATFORM_COMPILER_COMPAQ  1
-#    define PLATFORM_COMPILER_FAMILYNAME COMPAQ
-#    define PLATFORM_COMPILER_FAMILYID 6
-#    ifdef __cplusplus
-#        define PLATFORM_COMPILER_COMPAQ_CXX  1
-#    else
-#        define PLATFORM_COMPILER_COMPAQ_C  1
-#    endif
-#    if defined(__DECC_VER)
-#        define PLATFORM_COMPILER_VERSION __DECC_VER
-#    elif defined(__DECCXX_VER)
-#        define PLATFORM_COMPILER_VERSION __DECCXX_VER
-#    endif
-
-#    define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \
-         ( ((maj) * 10000000) + ((min) * 100000) + (90000) + (pat) )
-         /* 90000 = official ver, 80000 = customer special ver, 60000 = field test ver */
 
-#elif defined(__SUNPRO_C) || defined(__SUNPRO_CC)
-#    define PLATFORM_COMPILER_SUN  1
-#    define PLATFORM_COMPILER_FAMILYNAME SUN
-#    define PLATFORM_COMPILER_FAMILYID 7
-#    ifdef __cplusplus
-#        define PLATFORM_COMPILER_SUN_CXX  1
-#    else
-#        define PLATFORM_COMPILER_SUN_C  1
-#    endif
-#    if defined(__SUNPRO_C) && __SUNPRO_C > 0
-#        define PLATFORM_COMPILER_VERSION __SUNPRO_C
-#    elif defined(__SUNPRO_CC) && __SUNPRO_CC > 0
-#        define PLATFORM_COMPILER_VERSION __SUNPRO_CC
-#    endif
-#    define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \
-         ( ((maj) << 8) | ((min) << 4) | (pat) )
-
-#elif defined(__HP_cc) || defined(__HP_aCC)
-#    define PLATFORM_COMPILER_HP  1
-#    define PLATFORM_COMPILER_FAMILYNAME HP
-#    define PLATFORM_COMPILER_FAMILYID 8
-#    ifdef __cplusplus
-#        define PLATFORM_COMPILER_HP_CXX  1
-#    else
-#        define PLATFORM_COMPILER_HP_C  1
-#    endif
-#    if defined(__HP_cc) && __HP_cc > 0
-#        define PLATFORM_COMPILER_VERSION __HP_cc
-#    elif defined(__HP_aCC) && __HP_aCC > 0
-#        define PLATFORM_COMPILER_VERSION __HP_aCC
-#    endif
-#    define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \
-         ( ((maj) << 16) | ((min) << 8) | (pat) )
-
-#elif defined(_SGI_COMPILER_VERSION) || \
-   (defined(_COMPILER_VERSION) && defined(__sgi) && !defined(__GNUC__)) /* 7.3.0 and earlier lack _SGI_COMPILER_VERSION */
-#    define PLATFORM_COMPILER_SGI  1
-#    define PLATFORM_COMPILER_FAMILYNAME SGI
-#    define PLATFORM_COMPILER_FAMILYID 9
-#    ifdef __cplusplus
-#        define PLATFORM_COMPILER_SGI_CXX  1
-#    else
-#        define PLATFORM_COMPILER_SGI_C  1
-#    endif
-#    if defined(_SGI_COMPILER_VERSION) && _SGI_COMPILER_VERSION > 0
-#        define PLATFORM_COMPILER_VERSION _SGI_COMPILER_VERSION
-#    elif defined(_COMPILER_VERSION) && _COMPILER_VERSION > 0
-#        define PLATFORM_COMPILER_VERSION _COMPILER_VERSION
-#    endif
-#    define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \
-         ( ((maj) << 8) | ((min) << 4) | (pat) )
-
-#elif defined(_CRAYC)
-#    define PLATFORM_COMPILER_CRAY  1
-#    define PLATFORM_COMPILER_FAMILYNAME CRAY
-#    define PLATFORM_COMPILER_FAMILYID 10
-#    ifdef __cplusplus
-#        define PLATFORM_COMPILER_CRAY_CXX  1
-#    else
-#        define PLATFORM_COMPILER_CRAY_C  1
-#    endif
-#    if defined(_RELEASE) && defined(_RELEASE_MINOR) /* X1 and XT */
-#        define PLATFORM_COMPILER_VERSION \
-             PLATFORM_COMPILER_VERSION_INT(_RELEASE,_RELEASE_MINOR,0)
-#     elif defined(_RELEASE) /* T3E */
-#        define PLATFORM_COMPILER_VERSION \
-             PLATFORM_COMPILER_VERSION_INT(_RELEASE,0,0)
-#     endif
-#     ifdef _RELEASE_STRING /* X1 and XT */
-#        define PLATFORM_COMPILER_VERSION_STR _RELEASE_STRING
-#     endif
-
-#elif defined(__KCC)
-#    define PLATFORM_COMPILER_KAI  1
-#    define PLATFORM_COMPILER_FAMILYNAME KAI
-#    define PLATFORM_COMPILER_FAMILYID 11
-#    ifdef __cplusplus
-#        define PLATFORM_COMPILER_KAI_CXX  1
-#    else
-#        define PLATFORM_COMPILER_KAI_C  1
-#    endif
-
-#elif defined(__MTA__)
-#    define PLATFORM_COMPILER_MTA  1
-#    define PLATFORM_COMPILER_FAMILYNAME MTA
-#    define PLATFORM_COMPILER_FAMILYID 12
-#    ifdef __cplusplus
-#        define PLATFORM_COMPILER_MTA_CXX  1
-#    else
-#        define PLATFORM_COMPILER_MTA_C  1
-#    endif
-
-#elif defined(_SX)
-#    define PLATFORM_COMPILER_NECSX  1
-#    define PLATFORM_COMPILER_FAMILYNAME NECSX
-#    define PLATFORM_COMPILER_FAMILYID 13
-#    ifdef __cplusplus
-#        define PLATFORM_COMPILER_NECSX_CXX  1
-#    else
-#        define PLATFORM_COMPILER_NECSX_C  1
-#    endif
-
-#elif defined(_MSC_VER)
-#    define PLATFORM_COMPILER_MICROSOFT  1
-#    define PLATFORM_COMPILER_FAMILYNAME MICROSOFT
-#    define PLATFORM_COMPILER_FAMILYID 14
-#    ifdef __cplusplus
-#        define PLATFORM_COMPILER_MICROSOFT_CXX  1
-#    else
-#        define PLATFORM_COMPILER_MICROSOFT_C  1
-#    endif
-#    define PLATFORM_COMPILER_VERSION _MSC_VER
-
-#elif defined(__TINYC__)
-#    define PLATFORM_COMPILER_TINY  1
-#    define PLATFORM_COMPILER_FAMILYNAME TINY
-#    define PLATFORM_COMPILER_FAMILYID 15
-#    ifdef __cplusplus
-#        define PLATFORM_COMPILER_TINY_CXX  1
-#    else
-#        define PLATFORM_COMPILER_TINY_C  1
-#    endif
-
-#elif defined(__LCC__)
-#    define PLATFORM_COMPILER_LCC 1
-#    define PLATFORM_COMPILER_FAMILYNAME LCC
-#    define PLATFORM_COMPILER_FAMILYID 16
-#    ifdef __cplusplus
-#        define PLATFORM_COMPILER_LCC_CXX  1
-#    else
-#        define PLATFORM_COMPILER_LCC_C  1
-#    endif
-
-#else /* unknown compiler */
-#     define PLATFORM_COMPILER_UNKNOWN  1
-#endif
-
-/* this stanza comes last, because many vendor compilers lie and claim
-   to be GNU C for compatibility reasons and/or because they share a frontend */
-#if defined(__GNUC__)
-#    undef PLATFORM_COMPILER_UNKNOWN
-#    ifndef PLATFORM_COMPILER_FAMILYID
-#        define PLATFORM_COMPILER_GNU  1
-#        define PLATFORM_COMPILER_FAMILYNAME GNU
-#        define PLATFORM_COMPILER_FAMILYID 1
-#        ifdef __cplusplus
-#            define PLATFORM_COMPILER_GNU_CXX  1
-#        else
-#            define PLATFORM_COMPILER_GNU_C  1
-#        endif
-#        if defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__)
-#            define PLATFORM_COMPILER_VERSION \
-                 PLATFORM_COMPILER_VERSION_INT(__GNUC__,__GNUC_MINOR__,__GNUC_PATCHLEVEL__)
-#        elif defined(__GNUC_MINOR__) /* older versions of egcs lack __GNUC_PATCHLEVEL__ */
-#            define PLATFORM_COMPILER_VERSION \
-                 PLATFORM_COMPILER_VERSION_INT(__GNUC__,__GNUC_MINOR__,0)
-#        else
-#            define PLATFORM_COMPILER_VERSION \
-                 PLATFORM_COMPILER_VERSION_INT(__GNUC__,0,0)
-#        endif
-#        define PLATFORM_COMPILER_VERSION_STR __PLATFORM_COMPILER_GNU_VERSION_STR
-#    else
-#        define _PLATFORM_COMPILER_GNU_VERSION_STR __PLATFORM_COMPILER_GNU_VERSION_STR
-#    endif
-     /* gather any advertised GNU version number info, even for non-gcc compilers */
-#    if defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__)
-#        define __PLATFORM_COMPILER_GNU_VERSION_STR \
-             _STRINGIFY(__GNUC__) "." _STRINGIFY(__GNUC_MINOR__) "." _STRINGIFY(__GNUC_PATCHLEVEL__)
-#    elif defined(__GNUC_MINOR__)
-#        define __PLATFORM_COMPILER_GNU_VERSION_STR \
-             _STRINGIFY(__GNUC__) "." _STRINGIFY(__GNUC_MINOR__) ".?"
-#    else
-#        define __PLATFORM_COMPILER_GNU_VERSION_STR \
-             _STRINGIFY(__GNUC__) ".?.?"
-#    endif
-#elif defined(PLATFORM_COMPILER_UNKNOWN) /* unknown compiler */
-#    define PLATFORM_COMPILER_FAMILYNAME UNKNOWN
-#    define PLATFORM_COMPILER_FAMILYID 0
-#endif
-
-/* Default Values */
-#ifndef PLATFORM_COMPILER_VERSION
-#    define PLATFORM_COMPILER_VERSION 0 /* don't know */
-#endif
+#ifndef OPAL_PORTABLE_PLATFORM_H
+#define OPAL_PORTABLE_PLATFORM_H 1
 
-#ifndef PLATFORM_COMPILER_VERSION_STR
-#    define PLATFORM_COMPILER_VERSION_STR _STRINGIFY(PLATFORM_COMPILER_VERSION)
-#endif
+#include "opal/opal_portable_platform_real.h"
 
-#ifndef PLATFORM_COMPILER_VERSION_INT
-#    define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \
-        (((maj) << 16) | ((min) << 8) | (pat))
 #endif
-
-
-#endif /* OPAL_PORTABLE_PLATFORM_H */
diff --git a/opal/include/opal/opal_portable_platform_real.h b/opal/include/opal/opal_portable_platform_real.h
new file mode 100644
index 00000000000..6f21dbfa8d4
--- /dev/null
+++ b/opal/include/opal/opal_portable_platform_real.h
@@ -0,0 +1,1185 @@
+/*   $Source: bitbucket.org:berkeleylab/gasnet.git/other/portable_platform.h $
+ * Description: Portable platform detection header
+ * Copyright 2006, Dan Bonachea 
+ * Copyright 2018, The Regents of the University of California
+ * Terms of Use: In ADDITION to the license information in license.txt, 
+ *  anyone redistributing this header agrees not to change any part of this notice, or
+ *  the version handshake in the header versioning section below. 
+ *  Furthermore, redistributed copies of any portion of this header must
+ *  not appear within files named "portable_platform.h" or "gasnet_portable_platform.h",
+ *  unless it is embedded within a complete copy of the GASNet distribution.
+ *  These restrictions are designed to prevent conflicts for end users 
+ *  who compose multiple projects using the PLATFORM_ namespace.
+ *
+ * The canonical version of this header is hosted in the GASNet project at:
+ *   https://bitbucket.org/berkeleylab/gasnet   
+ *
+ * Developers who clone this header into their own project are HIGHLY encouraged to  
+ * contribute any improvements (especially addition of new platforms) back to the 
+ * canonical version, for the benefit of the community. 
+ * Contributions and bug reports should be directed to:
+ *   https://gasnet-bugs.lbl.gov or gasnet-staff@lbl.gov
+ */
+/* ------------------------------------------------------------------------------------ */
+/* Header versioning: DO NOT CHANGE ANYTHING IN THIS SECTION 
+ * The license terms for this header prohibit modifying this section in ANY way.
+   Clones should continue to advertise a PLATFORM_HEADER_VERSION equal to the canonical version they cloned,
+   and should not modify the handshake logic which ensures the highest canonical header version is used.
+ */
+/* Publish and enforce version number for the public interface to this header */
+/* YOU ARE NOT PERMITTED TO CHANGE THIS SECTION WITHOUT DIRECT APPROVAL FROM DAN BONACHEA */
+#if !defined(_PORTABLE_PLATFORM_H) || !defined(PLATFORM_HEADER_VERSION) \
+     || _PORTABLE_PLATFORM_H != PLATFORM_HEADER_VERSION \
+     || PLATFORM_HEADER_VERSION < 22
+#undef  PLATFORM_HEADER_VERSION 
+#define PLATFORM_HEADER_VERSION 22
+#undef  _PORTABLE_PLATFORM_H
+#define _PORTABLE_PLATFORM_H PLATFORM_HEADER_VERSION
+/* End Header versioning handshake */
+/* ------------------------------------------------------------------------------------ */
+
+/* make sure that previously-included older/broken clones of this header do not pollute our namespace */
+#undef PLATFORM_COMPILER_FAMILYNAME
+#undef PLATFORM_COMPILER_FAMILYID
+#undef PLATFORM_COMPILER_ID
+#undef PLATFORM_COMPILER_VERSION
+#undef PLATFORM_COMPILER_VERSION_STR
+#undef PLATFORM_COMPILER_VERSION_INT
+#undef PLATFORM_COMPILER_IDSTR
+#undef PLATFORM_COMPILER_VERSION_GT
+#undef PLATFORM_COMPILER_VERSION_GE
+#undef PLATFORM_COMPILER_VERSION_EQ
+#undef PLATFORM_COMPILER_VERSION_LE
+#undef PLATFORM_COMPILER_VERSION_LT
+#undef PLATFORM_COMPILER_C_LANGLVL
+#undef PLATFORM_COMPILER_CXX_LANGLVL
+#undef PLATFORM_COMPILER_INTEL
+#undef PLATFORM_COMPILER_INTEL_C
+#undef PLATFORM_COMPILER_INTEL_CXX
+#undef PLATFORM_COMPILER_PATHSCALE
+#undef PLATFORM_COMPILER_PATHSCALE_C
+#undef PLATFORM_COMPILER_PATHSCALE_CXX
+#undef PLATFORM_COMPILER_PGI
+#undef PLATFORM_COMPILER_PGI_C
+#undef PLATFORM_COMPILER_PGI_CXX
+#undef PLATFORM_COMPILER_XLC
+#undef PLATFORM_COMPILER_XLC_C
+#undef PLATFORM_COMPILER_XLC_CXX
+#undef PLATFORM_COMPILER_COMPAQ
+#undef PLATFORM_COMPILER_COMPAQ_C
+#undef PLATFORM_COMPILER_COMPAQ_CXX
+#undef PLATFORM_COMPILER_SUN
+#undef PLATFORM_COMPILER_SUN_C
+#undef PLATFORM_COMPILER_SUN_CXX
+#undef PLATFORM_COMPILER_HP
+#undef PLATFORM_COMPILER_HP_C
+#undef PLATFORM_COMPILER_HP_CXX
+#undef PLATFORM_COMPILER_SGI
+#undef PLATFORM_COMPILER_SGI_C
+#undef PLATFORM_COMPILER_SGI_CXX
+#undef PLATFORM_COMPILER_CRAY
+#undef PLATFORM_COMPILER_CRAY_C
+#undef PLATFORM_COMPILER_CRAY_CXX
+#undef PLATFORM_COMPILER_KAI
+#undef PLATFORM_COMPILER_KAI_C
+#undef PLATFORM_COMPILER_KAI_CXX
+#undef PLATFORM_COMPILER_MTA
+#undef PLATFORM_COMPILER_MTA_C
+#undef PLATFORM_COMPILER_MTA_CXX
+#undef PLATFORM_COMPILER_NECSX
+#undef PLATFORM_COMPILER_NECSX_C
+#undef PLATFORM_COMPILER_NECSX_CXX
+#undef PLATFORM_COMPILER_MICROSOFT
+#undef PLATFORM_COMPILER_MICROSOFT_C
+#undef PLATFORM_COMPILER_MICROSOFT_CXX
+#undef PLATFORM_COMPILER_TINY
+#undef PLATFORM_COMPILER_TINY_C
+#undef PLATFORM_COMPILER_TINY_CXX
+#undef PLATFORM_COMPILER_LCC
+#undef PLATFORM_COMPILER_LCC_C
+#undef PLATFORM_COMPILER_LCC_CXX
+#undef PLATFORM_COMPILER_OPEN64
+#undef PLATFORM_COMPILER_OPEN64_C
+#undef PLATFORM_COMPILER_OPEN64_CXX
+#undef PLATFORM_COMPILER_PCC
+#undef PLATFORM_COMPILER_PCC_C
+#undef PLATFORM_COMPILER_PCC_CXX
+#undef PLATFORM_COMPILER_CLANG
+#undef PLATFORM_COMPILER_CLANG_C
+#undef PLATFORM_COMPILER_CLANG_CXX
+#undef PLATFORM_COMPILER_NVHPC
+#undef PLATFORM_COMPILER_NVHPC_C
+#undef PLATFORM_COMPILER_NVHPC_CXX
+#undef PLATFORM_COMPILER_GNU
+#undef PLATFORM_COMPILER_GNU_C
+#undef PLATFORM_COMPILER_GNU_CXX
+#undef PLATFORM_COMPILER_UNKNOWN
+
+#undef PLATFORM_OS_FAMILYNAME
+#undef PLATFORM_OS_SUBFAMILYNAME
+#undef PLATFORM_OS_CATAMOUNT
+#undef PLATFORM_OS_BGP
+#undef PLATFORM_OS_BGQ
+#undef PLATFORM_OS_K42
+#undef PLATFORM_OS_UCLINUX
+#undef PLATFORM_OS_LINUX
+#undef PLATFORM_OS_CNL
+#undef PLATFORM_OS_SUBFAMILY_CNL
+#undef PLATFORM_OS_WSL
+#undef PLATFORM_OS_SUBFAMILY_WSL
+#undef PLATFORM_OS_BLRTS
+#undef PLATFORM_OS_CYGWIN
+#undef PLATFORM_OS_MSWINDOWS
+#undef PLATFORM_OS_AIX
+#undef PLATFORM_OS_TRU64
+#undef PLATFORM_OS_FREEBSD
+#undef PLATFORM_OS_NETBSD
+#undef PLATFORM_OS_OPENBSD
+#undef PLATFORM_OS_SOLARIS
+#undef PLATFORM_OS_DARWIN
+#undef PLATFORM_OS_IRIX
+#undef PLATFORM_OS_HPUX
+#undef PLATFORM_OS_UNICOS
+#undef PLATFORM_OS_MTA
+#undef PLATFORM_OS_SUPERUX
+#undef PLATFORM_OS_UNKNOWN
+
+#undef PLATFORM_ARCH_FAMILYNAME
+#undef PLATFORM_ARCH_32
+#undef _PLATFORM_ARCH_32
+#undef PLATFORM_ARCH_64
+#undef _PLATFORM_ARCH_64
+#undef PLATFORM_ARCH_BIG_ENDIAN
+#undef _PLATFORM_ARCH_BIG_ENDIAN
+#undef PLATFORM_ARCH_LITTLE_ENDIAN
+#undef _PLATFORM_ARCH_LITTLE_ENDIAN
+#undef PLATFORM_ARCH_POWERPC
+#undef PLATFORM_ARCH_MIC
+#undef PLATFORM_ARCH_X86_64
+#undef PLATFORM_ARCH_IA64
+#undef PLATFORM_ARCH_X86
+#undef PLATFORM_ARCH_ALPHA
+#undef PLATFORM_ARCH_MIPS
+#undef PLATFORM_ARCH_SPARC
+#undef PLATFORM_ARCH_PARISC
+#undef PLATFORM_ARCH_CRAYX1
+#undef PLATFORM_ARCH_CRAYT3E
+#undef PLATFORM_ARCH_MTA
+#undef PLATFORM_ARCH_NECSX
+#undef PLATFORM_ARCH_MICROBLAZE
+#undef PLATFORM_ARCH_ARM
+#undef PLATFORM_ARCH_AARCH64
+#undef PLATFORM_ARCH_TILE
+#undef PLATFORM_ARCH_S390
+#undef PLATFORM_ARCH_RISCV
+#undef PLATFORM_ARCH_UNKNOWN
+
+/* prevent known old/broken versions of this header from loading */
+#undef  OMPI_PORTABLE_PLATFORM_H
+#define OMPI_PORTABLE_PLATFORM_H
+#undef  OPAL_PORTABLE_PLATFORM_H
+#define OPAL_PORTABLE_PLATFORM_H
+
+/* ------------------------------------------------------------------------------------ */
+/* most of this file was written based on information in vendor documents, system headers,
+   and inspecting verbose compiler output. 
+   Another useful source of information: http://predef.sourceforge.net/
+*/
+
+/* ------------------------------------------------------------------------------------ */
+/* helpers */
+
+#undef  _PLATFORM_STRINGIFY_HELPER
+#define _PLATFORM_STRINGIFY_HELPER(x) #x
+#undef  PLATFORM_STRINGIFY
+#define PLATFORM_STRINGIFY(x) _PLATFORM_STRINGIFY_HELPER(x)
+
+/* ------------------------------------------------------------------------------------ */
+/* Compiler detection */
+/* 
+  PLATFORM_COMPILER_<family>:
+     defined to 1 if compiler is a given family, undef otherwise
+  PLATFORM_COMPILER_<family>_C
+  PLATFORM_COMPILER_<family>_CXX
+     defined to 1 if compiler is a given family, and is the C or C++ compiler, respectively
+  PLATFORM_COMPILER_FAMILYNAME:
+     unquoted token which provides the compiler family name
+  PLATFORM_COMPILER_FAMILYID:
+     defined to a positive integral value which is unique to a given compiler family
+     or zero if the compiler is unrecognized
+  PLATFORM_COMPILER_ID:
+     same as PLATFORM_COMPILER_FAMILYID, except C and C++ compilers are differentiated
+  PLATFORM_COMPILER_VERSION:
+     defined to an integral expression which is guaranteed to be monotonically non-decreasing 
+     with increasing compiler versions. Will be zero for unrecognized compilers.
+     The exact encoding of compiler version tuples into this constant may occasionally
+     change when this header is upgraded, so code should use the (in)equality macros below
+     to check against particular compiler versions, instead of embedding an encoded constant.
+  PLATFORM_COMPILER_VERSION_STR:
+     A string representation of the compiler version, which may contain additional info
+  PLATFORM_COMPILER_VERSION_[GT,GE,EQ,LE,LT](maj,min,pat):
+     evaluate to non-zero iff the compiler version in use is respectively 
+     greater-than, greater-or-equal, equal, less-or-equal, less-than 
+     the provided version components
+  PLATFORM_COMPILER_IDSTR:
+     a string which uniquely identifies recognized compilers
+  PLATFORM_COMPILER_C_LANGLVL and PLATFORM_COMPILER_CXX_LANGLVL: (in PLATFORM_HEADER_VERSION >= 5)
+     defined to a positive integral value corresponding to the C or C++ (respectively) 
+     language standard to which the current compiler advertises conformance.
+     Otherwise undef (in particular at most one of these is defined in a given compilation).
+*/
+
+#if defined(__INTEL_COMPILER)
+  #define PLATFORM_COMPILER_INTEL  1
+  #define PLATFORM_COMPILER_FAMILYNAME INTEL
+  #define PLATFORM_COMPILER_FAMILYID 2
+  #ifdef __cplusplus
+    #define PLATFORM_COMPILER_INTEL_CXX  1
+  #else
+    #define PLATFORM_COMPILER_INTEL_C  1
+  #endif
+  /* Intel compiler version "patch number"
+   * -------------------------------------
+   * Intel compiler versioning is unfortunately complicated by behavioral changes.
+   * Versions prior to Intel 14.0.0 (Sept 2013) lacked a preprocessor symbol to supply the "update" number.
+   * Version 14.0.0 and later supply a __INTEL_COMPILER_UPDATE symbol, but sadly several releases of Version 19 
+   * report the wrong value in this field (bug 3876).
+   * For now, the "patch" field of the PLATFORM_COMPILER_VERSION for Intel is the release package BUILD DATE,
+   * in the same decimal YYYYMMDD format as __INTEL_COMPILER_BUILD_DATE, as this is the only indicator that has
+   * remained reliably stable/correct across versions. 
+   * So for example to check for icc --version "19.0.1.144 20181018" or later, pass:
+   *   PLATFORM_COMPILER_VERSION_GE(19, 0, 20181018)
+   * NOTE 1: this build-date is unfortunately OS-DEPENDENT, sometimes differing by several days or weeks 
+   * between the Linux and OSX releases. For a complete mapping, see:
+   * https://software.intel.com/en-us/articles/intel-compiler-and-composer-update-version-numbers-to-compiler-version-number-mapping
+   * NOTE 2: some of the build-date entries in the table linked above have been observed to be incorrect,
+   * so when possible it's safest to verify the build-date from `icc --version` on both Linux and macOS.
+   */
+  #undef  _PLATFORM_INTEL_COMPILER_BUILD_DATE
+  #undef  _PLATFORM_COMPILER_INTEL_MIN_BUILDDATE
+  #define _PLATFORM_COMPILER_INTEL_MIN_BUILDDATE 19900000 /* year 1990: corresponds roughly to Intel v4.5 (1992) */
+  /* MIN_BUILDDATE is used to normalize build dates to a bit-saving range for the encoding
+   * Intel officially supports the current release and two prior (currently back to 2016)
+   * Our 1990 floor corresponds to Intel v4.x that only worked on MS-DOS and predates both Linux and BSD-based macOS
+   */
+  #ifdef __INTEL_COMPILER_BUILD_DATE
+    #define _PLATFORM_INTEL_COMPILER_BUILD_DATE __INTEL_COMPILER_BUILD_DATE
+  #else
+    #define _PLATFORM_INTEL_COMPILER_BUILD_DATE _PLATFORM_COMPILER_INTEL_MIN_BUILDDATE
+  #endif
+  /* Intel patch number is a decimal build date: YYYYMMDD - do NOT pass the "update number" */
+  #define PLATFORM_COMPILER_VERSION_INT(maj,min,pat)         \
+        (((((maj) * 100) + (min)) << 19) |                    \
+           ((pat) < _PLATFORM_COMPILER_INTEL_MIN_BUILDDATE ? \
+             0 : ((pat)-_PLATFORM_COMPILER_INTEL_MIN_BUILDDATE)))
+  #undef _PLATFORM__INTEL_COMPILER
+  #if __INTEL_COMPILER == 9999  /* Seen in 20110811 release of 12.1.0 - overflows VERSION_INT() */
+    #define _PLATFORM__INTEL_COMPILER 1201
+  #else
+    #define _PLATFORM__INTEL_COMPILER __INTEL_COMPILER
+  #endif
+  #define PLATFORM_COMPILER_VERSION \
+          PLATFORM_COMPILER_VERSION_INT(_PLATFORM__INTEL_COMPILER/100, _PLATFORM__INTEL_COMPILER%100, _PLATFORM_INTEL_COMPILER_BUILD_DATE)
+  #define PLATFORM_COMPILER_VERSION_STR \
+          PLATFORM_STRINGIFY(_PLATFORM__INTEL_COMPILER) "." PLATFORM_STRINGIFY(_PLATFORM_INTEL_COMPILER_BUILD_DATE)
+
+#elif defined(__PATHSCALE__)
+  #define PLATFORM_COMPILER_PATHSCALE  1
+  #define PLATFORM_COMPILER_FAMILYNAME PATHSCALE
+  #define PLATFORM_COMPILER_FAMILYID 3
+  #ifdef __cplusplus
+    #define PLATFORM_COMPILER_PATHSCALE_CXX  1
+  #else
+    #define PLATFORM_COMPILER_PATHSCALE_C  1
+  #endif
+  #define PLATFORM_COMPILER_VERSION \
+          PLATFORM_COMPILER_VERSION_INT(__PATHCC__,__PATHCC_MINOR__,__PATHCC_PATCHLEVEL__+0)
+  #define PLATFORM_COMPILER_VERSION_STR __PATHSCALE__
+
+#elif defined(__NVCOMPILER) /* Must occur prior to PGI and CLANG */
+  #define PLATFORM_COMPILER_NVHPC  1
+  #define PLATFORM_COMPILER_FAMILYNAME NVHPC
+  #define PLATFORM_COMPILER_FAMILYID 20
+  #ifdef __cplusplus
+    #define PLATFORM_COMPILER_NVHPC_CXX  1
+  #else
+    #define PLATFORM_COMPILER_NVHPC_C  1
+  #endif
+  #define PLATFORM_COMPILER_VERSION \
+          PLATFORM_COMPILER_VERSION_INT(__NVCOMPILER_MAJOR__,__NVCOMPILER_MINOR__,__NVCOMPILER_PATCHLEVEL__)
+  #define PLATFORM_COMPILER_VERSION_STR \
+      PLATFORM_STRINGIFY(__NVCOMPILER_MAJOR__) "." PLATFORM_STRINGIFY(__NVCOMPILER_MINOR__) "-" PLATFORM_STRINGIFY(__NVCOMPILER_PATCHLEVEL__)
+
+#elif defined(__PGI)
+  #define PLATFORM_COMPILER_PGI  1
+  #define PLATFORM_COMPILER_FAMILYNAME PGI
+  #define PLATFORM_COMPILER_FAMILYID 4
+  #ifdef __cplusplus
+    #define PLATFORM_COMPILER_PGI_CXX  1
+  #else
+    #define PLATFORM_COMPILER_PGI_C  1
+  #endif
+  #if __PGIC__ == 99 
+    /* bug 2230: PGI versioning was broken for some platforms in 7.0
+                 no way to know exact version, but provide something slightly more accurate */
+    #define PLATFORM_COMPILER_VERSION 0x070000
+    #define PLATFORM_COMPILER_VERSION_STR "7.?-?"
+  #elif defined(__PGIC__) && defined(__PGIC_MINOR__) && defined(__PGIC_PATCHLEVEL__)
+    #define PLATFORM_COMPILER_VERSION \
+            PLATFORM_COMPILER_VERSION_INT(__PGIC__,__PGIC_MINOR__,__PGIC_PATCHLEVEL__)
+    #define PLATFORM_COMPILER_VERSION_STR \
+            PLATFORM_STRINGIFY(__PGIC__) "." PLATFORM_STRINGIFY(__PGIC_MINOR__) "-" PLATFORM_STRINGIFY(__PGIC_PATCHLEVEL__)
+  #else
+    /* PGI before 6.1-4 lacks any version ID preprocessor macros - so use this filthy hack */
+    #ifdef PLATFORM_PGI_IS_ANCIENT
+      /* Include below might fail for ancient versions lacking this header, but testing shows it
+         works back to at least 5.1-3 (Nov 2003), and based on docs probably back to 3.2 (Sep 2000) */
+        #define PLATFORM_COMPILER_VERSION 0       
+    #elif defined(__x86_64__) /* bug 1753 - 64-bit omp.h upgrade happenned in <6.0-8,6.1-1] */
+      #include "omp.h"
+      #if defined(_PGOMP_H)
+        /* 6.1.1 or newer */
+        #define PLATFORM_COMPILER_VERSION 0x060101
+        #define PLATFORM_COMPILER_VERSION_STR ">=6.1-1"
+      #else
+        /* 6.0.8 or older */
+        #define PLATFORM_COMPILER_VERSION 0
+        #define PLATFORM_COMPILER_VERSION_STR "<=6.0-8"
+      #endif
+    #else /* 32-bit omp.h upgrade happenned in <5.2-4,6.0-8] */
+      #include "omp.h"
+      #if defined(_PGOMP_H)
+        /* 6.0-8 or newer */
+        #define PLATFORM_COMPILER_VERSION 0x060008
+        #define PLATFORM_COMPILER_VERSION_STR ">=6.0-8"
+      #else
+        /* 5.2-4 or older */
+        #define PLATFORM_COMPILER_VERSION 0
+        #define PLATFORM_COMPILER_VERSION_STR "<=5.2-4"
+      #endif
+    #endif
+  #endif
+
+#elif defined(__xlC__) || defined(__ibmxl__)
+  #define PLATFORM_COMPILER_XLC  1
+  #define PLATFORM_COMPILER_FAMILYNAME XLC
+  #define PLATFORM_COMPILER_FAMILYID 5
+  #ifdef __cplusplus
+    #define PLATFORM_COMPILER_XLC_CXX  1
+  #else
+    #define PLATFORM_COMPILER_XLC_C  1
+  #endif
+  #ifdef __ibmxl_version__
+    #define PLATFORM_COMPILER_VERSION \
+      (__ibmxl_version__ << 24 | __ibmxl_release__ << 16 | \
+       __ibmxl_modification__ << 8 | __ibmxl_ptf_fix_level__)
+    #define PLATFORM_COMPILER_VERSION_STR \
+      PLATFORM_STRINGIFY(__ibmxl_version__) "." PLATFORM_STRINGIFY(__ibmxl_release__) "." PLATFORM_STRINGIFY(__ibmxl_modification__) "." PLATFORM_STRINGIFY(__ibmxl_ptf_fix_level__)
+  #else
+    #ifdef __xlC_ver__
+      #define PLATFORM_COMPILER_VERSION (__xlC__ << 16 | __xlC_ver__)
+    #else
+      #define PLATFORM_COMPILER_VERSION (__xlC__ << 16)
+    #endif
+    #ifdef __xlc__
+      #define PLATFORM_COMPILER_VERSION_STR __xlc__
+    #else
+      #define PLATFORM_COMPILER_VERSION_STR PLATFORM_STRINGIFY(__xlC__)
+    #endif
+  #endif
+  #define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \
+        ( ((maj) << 24) | ((min) << 16) | ((pat) << 8) )
+
+#elif defined(__DECC) || defined(__DECCXX)
+  #define PLATFORM_COMPILER_COMPAQ  1
+  #define PLATFORM_COMPILER_FAMILYNAME COMPAQ
+  #define PLATFORM_COMPILER_FAMILYID 6
+  #ifdef __cplusplus
+    #define PLATFORM_COMPILER_COMPAQ_CXX  1
+  #else
+    #define PLATFORM_COMPILER_COMPAQ_C  1
+  #endif
+  #if defined(__DECC_VER)
+    #define PLATFORM_COMPILER_VERSION __DECC_VER
+  #elif defined(__DECCXX_VER)
+    #define PLATFORM_COMPILER_VERSION __DECCXX_VER
+  #endif
+
+  #define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \
+        ( ((maj) * 10000000) + ((min) * 100000) + (90000) + (pat) )
+  /* 90000 = official ver, 80000 = customer special ver, 60000 = field test ver */
+
+#elif defined(__SUNPRO_C) || defined(__SUNPRO_CC)
+  #define PLATFORM_COMPILER_SUN  1
+  #define PLATFORM_COMPILER_FAMILYNAME SUN
+  #define PLATFORM_COMPILER_FAMILYID 7
+  #ifdef __cplusplus
+    #define PLATFORM_COMPILER_SUN_CXX  1
+  #else
+    #define PLATFORM_COMPILER_SUN_C  1
+  #endif
+  #if defined(__SUNPRO_C) && __SUNPRO_C > 0
+    #define PLATFORM_COMPILER_VERSION __SUNPRO_C
+  #elif defined(__SUNPRO_CC) && __SUNPRO_CC > 0
+    #define PLATFORM_COMPILER_VERSION __SUNPRO_CC
+  #endif
+  /* Sun version numbers look like hex but are actually a sloppy concatenation of decimal version numbers
+   * leading to weird discontinuities in the version space, luckily it remains monotonic (so far)
+   */
+  #define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) ( \
+        (min) < 10 ?                                   \
+        ( ((maj) << 8) | ((min) << 4) | (pat) ) :      \
+        ( ((maj) << 12) | (((min)/10) << 8) | (((min)%10) << 4) | (pat) )  )
+
+#elif defined(__HP_cc) || defined(__HP_aCC)
+  #define PLATFORM_COMPILER_HP  1
+  #define PLATFORM_COMPILER_FAMILYNAME HP
+  #define PLATFORM_COMPILER_FAMILYID 8
+  #ifdef __cplusplus
+    #define PLATFORM_COMPILER_HP_CXX  1
+  #else
+    #define PLATFORM_COMPILER_HP_C  1
+  #endif
+  #if defined(__HP_cc) && __HP_cc > 0
+    #define PLATFORM_COMPILER_VERSION __HP_cc
+  #elif defined(__HP_aCC) && __HP_aCC > 0
+    #define PLATFORM_COMPILER_VERSION __HP_aCC
+  #endif
+  #define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \
+        ( ((maj) << 16) | ((min) << 8) | (pat) )
+
+#elif defined(_SGI_COMPILER_VERSION) || \
+   (defined(_COMPILER_VERSION) && defined(__sgi) && !defined(__GNUC__)) /* 7.3.0 and earlier lack _SGI_COMPILER_VERSION */
+  #define PLATFORM_COMPILER_SGI  1
+  #define PLATFORM_COMPILER_FAMILYNAME SGI
+  #define PLATFORM_COMPILER_FAMILYID 9
+  #ifdef __cplusplus
+    #define PLATFORM_COMPILER_SGI_CXX  1
+  #else
+    #define PLATFORM_COMPILER_SGI_C  1
+  #endif
+  #if defined(_SGI_COMPILER_VERSION) && _SGI_COMPILER_VERSION > 0
+    #define PLATFORM_COMPILER_VERSION _SGI_COMPILER_VERSION
+  #elif defined(_COMPILER_VERSION) && _COMPILER_VERSION > 0
+    #define PLATFORM_COMPILER_VERSION _COMPILER_VERSION
+  #endif
+  #define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \
+        ( ((maj) << 8) | ((min) << 4) | (pat) )
+
+#elif defined(_CRAYC) 
+  #define PLATFORM_COMPILER_CRAY  1
+  #define PLATFORM_COMPILER_FAMILYNAME CRAY
+  #define PLATFORM_COMPILER_FAMILYID 10
+  #ifdef __cplusplus
+    #define PLATFORM_COMPILER_CRAY_CXX  1
+  #else
+    #define PLATFORM_COMPILER_CRAY_C  1
+  #endif
+  #if defined(_RELEASE_MAJOR) && defined(_RELEASE_MINOR) /* XE, XK, XC */
+    #define PLATFORM_COMPILER_VERSION \
+            PLATFORM_COMPILER_VERSION_INT(_RELEASE_MAJOR,_RELEASE_MINOR,0)
+  #elif defined(_RELEASE) && defined(_RELEASE_MINOR) /* X1 and XT */
+    #define PLATFORM_COMPILER_VERSION \
+            PLATFORM_COMPILER_VERSION_INT(_RELEASE,_RELEASE_MINOR,0)
+  #elif defined(_RELEASE) /* T3E */
+    #define PLATFORM_COMPILER_VERSION \
+            PLATFORM_COMPILER_VERSION_INT(_RELEASE,0,0)
+  #endif
+  #ifdef _RELEASE_STRING /* X1 and XT, XK, XC */
+    #define PLATFORM_COMPILER_VERSION_STR _RELEASE_STRING
+  #endif
+
+#elif defined(__KCC)
+  #define PLATFORM_COMPILER_KAI  1
+  #define PLATFORM_COMPILER_FAMILYNAME KAI
+  #define PLATFORM_COMPILER_FAMILYID 11
+  #ifdef __cplusplus
+    #define PLATFORM_COMPILER_KAI_CXX  1
+  #else
+    #define PLATFORM_COMPILER_KAI_C  1
+  #endif
+
+#elif defined(__MTA__)
+  #define PLATFORM_COMPILER_MTA  1
+  #define PLATFORM_COMPILER_FAMILYNAME MTA
+  #define PLATFORM_COMPILER_FAMILYID 12
+  #ifdef __cplusplus
+    #define PLATFORM_COMPILER_MTA_CXX  1
+  #else
+    #define PLATFORM_COMPILER_MTA_C  1
+  #endif
+
+#elif defined(_SX)
+  #define PLATFORM_COMPILER_NECSX  1
+  #define PLATFORM_COMPILER_FAMILYNAME NECSX
+  #define PLATFORM_COMPILER_FAMILYID 13
+  #ifdef __cplusplus
+    #define PLATFORM_COMPILER_NECSX_CXX  1
+  #else
+    #define PLATFORM_COMPILER_NECSX_C  1
+  #endif
+
+#elif defined(_MSC_VER)
+  #define PLATFORM_COMPILER_MICROSOFT  1
+  #define PLATFORM_COMPILER_FAMILYNAME MICROSOFT
+  #define PLATFORM_COMPILER_FAMILYID 14
+  #ifdef __cplusplus
+    #define PLATFORM_COMPILER_MICROSOFT_CXX  1
+  #else
+    #define PLATFORM_COMPILER_MICROSOFT_C  1
+  #endif
+  #define PLATFORM_COMPILER_VERSION _MSC_VER
+
+#elif defined(__TINYC__)
+  #define PLATFORM_COMPILER_TINY  1
+  #define PLATFORM_COMPILER_FAMILYNAME TINY
+  #define PLATFORM_COMPILER_FAMILYID 15
+  #ifdef __cplusplus
+    #define PLATFORM_COMPILER_TINY_CXX  1
+  #else
+    #define PLATFORM_COMPILER_TINY_C  1
+  #endif
+
+#elif defined(__LCC__)
+  #define PLATFORM_COMPILER_LCC 1
+  #define PLATFORM_COMPILER_FAMILYNAME LCC
+  #define PLATFORM_COMPILER_FAMILYID 16
+  #ifdef __cplusplus
+    #define PLATFORM_COMPILER_LCC_CXX  1
+  #else
+    #define PLATFORM_COMPILER_LCC_C  1
+  #endif
+
+#elif defined(__OPENCC__)
+  #define PLATFORM_COMPILER_OPEN64  1
+  #define PLATFORM_COMPILER_FAMILYNAME OPEN64
+  #define PLATFORM_COMPILER_FAMILYID 17
+  #ifdef __cplusplus
+    #define PLATFORM_COMPILER_OPEN64_CXX  1
+  #else
+    #define PLATFORM_COMPILER_OPEN64_C  1
+  #endif
+  /* Note: can't use __OPENCC_PATCHLEVEL__ because it is sometimes non-integer (eg 3.2).
+     Adding a cast would not result in a preprocessor constant expression. */
+  #define PLATFORM_COMPILER_VERSION \
+          PLATFORM_COMPILER_VERSION_INT(__OPENCC__,__OPENCC_MINOR__,0)
+  #define PLATFORM_COMPILER_VERSION_STR __OPEN64__
+
+#elif defined(__PCC__)
+  #define PLATFORM_COMPILER_PCC  1
+  #define PLATFORM_COMPILER_FAMILYNAME PCC
+  #define PLATFORM_COMPILER_FAMILYID 18
+  #ifdef __cplusplus
+    #define PLATFORM_COMPILER_PCC_CXX  1
+  #else
+    #define PLATFORM_COMPILER_PCC_C  1
+  #endif
+  #define PLATFORM_COMPILER_VERSION \
+          PLATFORM_COMPILER_VERSION_INT(__PCC__,__PCC_MINOR__,__PCC_MINORMINOR__)
+  #define PLATFORM_COMPILER_VERSION_STR \
+      PLATFORM_STRINGIFY(__PCC__) "." PLATFORM_STRINGIFY(__PCC_MINOR__) "." PLATFORM_STRINGIFY(__PCC_MINORMINOR__)
+
+#elif defined(__clang__)
+  #define PLATFORM_COMPILER_CLANG  1
+  #define PLATFORM_COMPILER_FAMILYNAME CLANG
+  #define PLATFORM_COMPILER_FAMILYID 19
+  #ifdef __cplusplus
+    #define PLATFORM_COMPILER_CLANG_CXX  1
+  #else
+    #define PLATFORM_COMPILER_CLANG_C  1
+  #endif
+  #ifdef __clang_version__
+    /* clang 2.7 (gcc 4.2.1 compliant) and earlier lacked specific version identification */
+    #define PLATFORM_COMPILER_VERSION \
+            PLATFORM_COMPILER_VERSION_INT(__clang_major__,__clang_minor__,__clang_patchlevel__)
+    #define PLATFORM_COMPILER_VERSION_STR __clang_version__
+  #endif
+
+/* NOTE: PLATFORM_COMPILER_FAMILYID "20" is allocted to NVHPC, appearing earlier */
+
+#else /* unknown compiler */
+  #define PLATFORM_COMPILER_UNKNOWN  1
+#endif
+
+/* this stanza comes last, because many vendor compilers lie and claim 
+   to be GNU C for compatibility reasons and/or because they share a frontend */ 
+#undef _PLATFORM_COMPILER_GNU_VERSION_STR
+#undef __PLATFORM_COMPILER_GNU_VERSION_STR
+#if defined(__GNUC__)
+  #undef PLATFORM_COMPILER_UNKNOWN
+  #ifndef PLATFORM_COMPILER_FAMILYID
+    #define PLATFORM_COMPILER_GNU  1
+    #define PLATFORM_COMPILER_FAMILYNAME GNU
+    #define PLATFORM_COMPILER_FAMILYID 1
+    #ifdef __cplusplus
+      #define PLATFORM_COMPILER_GNU_CXX  1
+    #else
+      #define PLATFORM_COMPILER_GNU_C  1
+    #endif
+   #if defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__)
+    #define PLATFORM_COMPILER_VERSION \
+            PLATFORM_COMPILER_VERSION_INT(__GNUC__,__GNUC_MINOR__,__GNUC_PATCHLEVEL__)
+   #elif defined(__GNUC_MINOR__) /* older versions of egcs lack __GNUC_PATCHLEVEL__ */
+    #define PLATFORM_COMPILER_VERSION \
+            PLATFORM_COMPILER_VERSION_INT(__GNUC__,__GNUC_MINOR__,0)
+   #else
+    #define PLATFORM_COMPILER_VERSION \
+            PLATFORM_COMPILER_VERSION_INT(__GNUC__,0,0)
+   #endif
+    #define PLATFORM_COMPILER_VERSION_STR __PLATFORM_COMPILER_GNU_VERSION_STR
+  #else
+    #define _PLATFORM_COMPILER_GNU_VERSION_STR __PLATFORM_COMPILER_GNU_VERSION_STR
+  #endif
+  /* gather any advertised GNU version number info, even for non-gcc compilers */
+  #if defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__)
+    #define __PLATFORM_COMPILER_GNU_VERSION_STR \
+      PLATFORM_STRINGIFY(__GNUC__) "." PLATFORM_STRINGIFY(__GNUC_MINOR__) "." PLATFORM_STRINGIFY(__GNUC_PATCHLEVEL__)
+  #elif defined(__GNUC_MINOR__)
+    #define __PLATFORM_COMPILER_GNU_VERSION_STR \
+      PLATFORM_STRINGIFY(__GNUC__) "." PLATFORM_STRINGIFY(__GNUC_MINOR__) ".?"
+  #else
+    #define __PLATFORM_COMPILER_GNU_VERSION_STR \
+      PLATFORM_STRINGIFY(__GNUC__) ".?.?"
+  #endif
+#elif defined(PLATFORM_COMPILER_UNKNOWN) /* unknown compiler */
+  #define PLATFORM_COMPILER_FAMILYNAME UNKNOWN
+  #define PLATFORM_COMPILER_FAMILYID 0 
+#endif
+
+/* defaulting */
+
+#ifndef PLATFORM_COMPILER_VERSION
+#define PLATFORM_COMPILER_VERSION 0 /* don't know */
+#endif
+
+#ifndef PLATFORM_COMPILER_VERSION_STR
+#define PLATFORM_COMPILER_VERSION_STR PLATFORM_STRINGIFY(PLATFORM_COMPILER_VERSION)
+#endif
+
+#ifndef PLATFORM_COMPILER_VERSION_INT
+#define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \
+        (((maj) << 16) | ((min) << 8) | (pat))
+#endif
+
+/* version check macros */
+
+#define PLATFORM_COMPILER_VERSION_GT(maj,min,pat) \
+        PLATFORM_COMPILER_VERSION >  PLATFORM_COMPILER_VERSION_INT(maj,min,pat)
+#define PLATFORM_COMPILER_VERSION_GE(maj,min,pat) \
+        PLATFORM_COMPILER_VERSION >= PLATFORM_COMPILER_VERSION_INT(maj,min,pat)
+#define PLATFORM_COMPILER_VERSION_EQ(maj,min,pat) \
+        PLATFORM_COMPILER_VERSION == PLATFORM_COMPILER_VERSION_INT(maj,min,pat)
+#define PLATFORM_COMPILER_VERSION_LE(maj,min,pat) \
+        PLATFORM_COMPILER_VERSION <= PLATFORM_COMPILER_VERSION_INT(maj,min,pat)
+#define PLATFORM_COMPILER_VERSION_LT(maj,min,pat) \
+        PLATFORM_COMPILER_VERSION <  PLATFORM_COMPILER_VERSION_INT(maj,min,pat)
+
+/* misc feature detection */
+
+#ifdef __cplusplus
+  #define PLATFORM_COMPILER_ID (10000+PLATFORM_COMPILER_FAMILYID)
+#else
+  #define PLATFORM_COMPILER_ID PLATFORM_COMPILER_FAMILYID
+#endif
+
+/* default language spec conformance detection */
+#if !defined(PLATFORM_COMPILER_C_LANGLVL) && !defined(PLATFORM_COMPILER_CXX_LANGLVL)
+  #if defined(__cplusplus) && (__cplusplus > 0)  /* C++98 or newer */
+    #define PLATFORM_COMPILER_CXX_LANGLVL  __cplusplus
+  #elif defined(__STDC_VERSION__) && (__STDC_VERSION__ > 0)  /* C95 or newer */
+    #define PLATFORM_COMPILER_C_LANGLVL  __STDC_VERSION__
+  #elif defined(__STDC__) && !defined(__cplusplus) && !defined(__STDC_VERSION__) /* C89/C90 */
+    #define PLATFORM_COMPILER_C_LANGLVL  199000L
+  #else 
+    /* unknown - leave both undef */
+  #endif
+#endif
+
+#undef _PLATFORM_COMPILER_STD_STDC
+#ifdef __STDC__
+  #define _PLATFORM_COMPILER_STD_STDC "__STDC__"
+#else
+  #define _PLATFORM_COMPILER_STD_STDC "-"
+#endif
+#undef _PLATFORM_COMPILER_STD_STDC_VERSION
+#ifdef __STDC_VERSION__
+  #define _PLATFORM_COMPILER_STD_STDC_VERSION ",__STDC_VERSION__=" PLATFORM_STRINGIFY(__STDC_VERSION__)
+#else
+  #define _PLATFORM_COMPILER_STD_STDC_VERSION
+#endif
+#undef _PLATFORM_COMPILER_STD_STDC_EXT
+#ifdef __STDC_EXT__
+  #define _PLATFORM_COMPILER_STD_STDC_EXT ",__STDC_EXT__=" PLATFORM_STRINGIFY(__STDC_EXT__)
+#else
+  #define _PLATFORM_COMPILER_STD_STDC_EXT 
+#endif
+#undef _PLATFORM_COMPILER_STD_CPLUSPLUS
+#ifdef __cplusplus
+  #define _PLATFORM_COMPILER_STD_CPLUSPLUS ",__cplusplus=" PLATFORM_STRINGIFY(__cplusplus)
+#else
+  #define _PLATFORM_COMPILER_STD_CPLUSPLUS 
+#endif
+
+#undef _PLATFORM_COMPILER_MISC_VERSION_STR
+#ifndef _PLATFORM_COMPILER_MISC_VERSION_STR
+  #ifdef __VERSION__
+    #define _PLATFORM_COMPILER_MISC_VERSION_STR "|misc:" __VERSION__
+  #else
+    #define _PLATFORM_COMPILER_MISC_VERSION_STR
+  #endif
+#endif
+#undef _PLATFORM_COMPILER_GNU_VERSION_STR_HELP
+#ifdef _PLATFORM_COMPILER_GNU_VERSION_STR
+    #define _PLATFORM_COMPILER_GNU_VERSION_STR_HELP "|GNU:" _PLATFORM_COMPILER_GNU_VERSION_STR
+#else
+    #define _PLATFORM_COMPILER_GNU_VERSION_STR_HELP
+#endif
+
+#define PLATFORM_COMPILER_IDSTR                                      \
+        "|COMPILER_FAMILY:"                                          \
+          PLATFORM_STRINGIFY(PLATFORM_COMPILER_FAMILYNAME)           \
+        "|COMPILER_VERSION:" PLATFORM_COMPILER_VERSION_STR           \
+        "|COMPILER_FAMILYID:"                                        \
+          PLATFORM_STRINGIFY(PLATFORM_COMPILER_FAMILYID)             \
+         _PLATFORM_COMPILER_GNU_VERSION_STR_HELP                     \
+        "|STD:" _PLATFORM_COMPILER_STD_STDC                          \
+                _PLATFORM_COMPILER_STD_STDC_VERSION                  \
+                _PLATFORM_COMPILER_STD_STDC_EXT                      \
+                _PLATFORM_COMPILER_STD_CPLUSPLUS                     \
+         _PLATFORM_COMPILER_MISC_VERSION_STR                         \
+        "|"
+
+/* ------------------------------------------------------------------------------------ */
+/* OS detection */
+/* 
+   PLATFORM_OS_<family>:
+     defined to a positive value if OS belongs to a given family, undef otherwise
+   PLATFORM_OS_FAMILYNAME:
+     unquoted token which provides the OS family name
+
+   Some systems also define a subfamily:
+    PLATFORM_OS_SUBFAMILY_<subfamily>: positive value or undef
+    PLATFORM_OS_SUBFAMILYNAME: unquoted token for subfamily name or undef
+*/
+
+#if defined(__LIBCATAMOUNT__) || defined(__QK_USER__)
+  #define PLATFORM_OS_CATAMOUNT 1
+  #define PLATFORM_OS_FAMILYNAME CATAMOUNT
+
+#elif defined(GASNETI_ARCH_BGP) || defined(__bgp__)
+  #define PLATFORM_OS_BGP 1
+  #define PLATFORM_OS_FAMILYNAME BGP
+
+#elif defined(GASNETI_ARCH_BGQ) || defined(__bgq__)
+  #define PLATFORM_OS_BGQ 1
+  #define PLATFORM_OS_FAMILYNAME BGQ
+
+#elif defined(__K42)
+  #define PLATFORM_OS_K42 1
+  #define PLATFORM_OS_FAMILYNAME K42
+
+#elif defined(__uClinux__)
+  #define PLATFORM_OS_UCLINUX 1
+  #define PLATFORM_OS_FAMILYNAME UCLINUX
+
+#elif defined(__linux) || defined(__linux__) || defined(__gnu_linux__)
+  #define PLATFORM_OS_LINUX 1
+  #define PLATFORM_OS_FAMILYNAME LINUX
+  #if defined(GASNETI_ARCH_WSL)
+    #define PLATFORM_OS_SUBFAMILY_WSL 1
+    #define PLATFORM_OS_SUBFAMILYNAME WSL
+  #elif defined(__CRAYXT_COMPUTE_LINUX_TARGET)
+    /* NOTE: As of 2022-07 this is ONLY defined for the Cray cc/CC wrappers, and not the raw PrgEnv compilers */
+    #define PLATFORM_OS_SUBFAMILY_CNL 1
+    #define PLATFORM_OS_SUBFAMILYNAME CNL
+  #endif
+
+#elif defined(__blrts) || defined(__blrts__) || defined(__gnu_blrts__)
+  #define PLATFORM_OS_BLRTS 1
+  #define PLATFORM_OS_FAMILYNAME BLRTS
+
+#elif defined(__CYGWIN__)
+  #define PLATFORM_OS_CYGWIN 1
+  #define PLATFORM_OS_FAMILYNAME CYGWIN
+
+#elif defined(_WIN32)
+  #define PLATFORM_OS_MSWINDOWS 1
+  #define PLATFORM_OS_FAMILYNAME MSWINDOWS
+
+#elif defined(_AIX)
+  #define PLATFORM_OS_AIX 1
+  #define PLATFORM_OS_FAMILYNAME AIX
+
+#elif defined(__osf__) || defined(__digital__)
+  #define PLATFORM_OS_TRU64 1
+  #define PLATFORM_OS_FAMILYNAME TRU64
+
+#elif defined(__FreeBSD) || defined(__FreeBSD__)
+  #define PLATFORM_OS_FREEBSD 1
+  #define PLATFORM_OS_FAMILYNAME FREEBSD
+
+#elif defined(__NetBSD) || defined(__NetBSD__)
+  #define PLATFORM_OS_NETBSD 1
+  #define PLATFORM_OS_FAMILYNAME NETBSD
+
+#elif defined(__OpenBSD__)
+  #define PLATFORM_OS_OPENBSD 1
+  #define PLATFORM_OS_FAMILYNAME OPENBSD
+
+#elif defined(__sun) || defined(__sun__)
+  #define PLATFORM_OS_SOLARIS 1
+  #define PLATFORM_OS_FAMILYNAME SOLARIS
+
+#elif (defined(__APPLE__) && defined(__MACH__)) || \
+      defined(__osx86__) /* PGI on OSX */
+  #define PLATFORM_OS_DARWIN 1
+  #define PLATFORM_OS_FAMILYNAME DARWIN
+
+#elif defined(__sgi) || defined(__sgi__)
+  #define PLATFORM_OS_IRIX 1
+  #define PLATFORM_OS_FAMILYNAME IRIX
+
+#elif defined(__hpux) || defined(__hpux__)
+  #define PLATFORM_OS_HPUX 1
+  #define PLATFORM_OS_FAMILYNAME HPUX
+
+#elif defined(_CRAY) || defined(_UNICOSMP)
+  #define PLATFORM_OS_UNICOS 1
+  #define PLATFORM_OS_FAMILYNAME UNICOS
+
+#elif defined(__MTA__)
+  #define PLATFORM_OS_MTA 1
+  #define PLATFORM_OS_FAMILYNAME MTA
+
+#elif defined(_SX)
+  #define PLATFORM_OS_SUPERUX 1
+  #define PLATFORM_OS_FAMILYNAME SUPERUX
+
+#else
+  #define PLATFORM_OS_UNKNOWN 1
+  #define PLATFORM_OS_FAMILYNAME UNKNOWN
+#endif
+
+/* ------------------------------------------------------------------------------------ */
+/* Architecture detection */
+/* 
+   PLATFORM_ARCH_<family>:
+     defined to positive value if CPU belongs to a given family, undef otherwise
+   PLATFORM_ARCH_FAMILYNAME:
+     unquoted token which provides the CPU family name
+
+   PLATFORM_ARCH_32              - 32-bit pointers
+   PLATFORM_ARCH_64              - 64-bit pointers
+   PLATFORM_ARCH_BIG_ENDIAN      - big-endian word order
+   PLATFORM_ARCH_LITTLE_ENDIAN   - little-endian word order
+     defined to positive value if CPU is known to have the indicated property, undef otherwise
+ */
+
+#if defined(__ppc64) || defined(__ppc64__) || \
+    defined(__PPC64) || defined(__PPC64__) || \
+    defined(__powerpc64) || defined(__powerpc64__) 
+  #define PLATFORM_ARCH_POWERPC 1
+  #define PLATFORM_ARCH_FAMILYNAME POWERPC
+  #define _PLATFORM_ARCH_64 1
+  #define _PLATFORM_ARCH_BIG_ENDIAN 1
+
+#elif defined(_POWER) || \
+      defined(__PPC)  || defined(__PPC__) || \
+      defined(__powerpc) || defined(__powerpc__) || \
+      defined(__ppc) || defined(__ppc__) || \
+      defined(__POWERPC__)
+  #define PLATFORM_ARCH_POWERPC 1
+  #define PLATFORM_ARCH_FAMILYNAME POWERPC
+  #define _PLATFORM_ARCH_32 1
+  #define _PLATFORM_ARCH_BIG_ENDIAN 1
+
+#elif defined(_ARCH_PPC) || defined(_ARCH_PPC64)
+  #define PLATFORM_ARCH_POWERPC 1
+  #define PLATFORM_ARCH_FAMILYNAME POWERPC
+  #define _PLATFORM_ARCH_BIG_ENDIAN 1
+
+#elif defined(__KNC__) || defined(__MIC__)
+  #define PLATFORM_ARCH_MIC 1
+  #define PLATFORM_ARCH_FAMILYNAME MIC
+  #define _PLATFORM_ARCH_64 1
+  #define _PLATFORM_ARCH_LITTLE_ENDIAN 1
+
+#elif defined(__x86_64) || defined(__x86_64__) || \
+    defined(__athlon) || defined(__athlon__) || \
+    defined(__amd64)  || defined(__amd64__)
+  #define PLATFORM_ARCH_X86_64 1
+  #define PLATFORM_ARCH_FAMILYNAME X86_64
+  #define _PLATFORM_ARCH_64 1
+  #define _PLATFORM_ARCH_LITTLE_ENDIAN 1
+
+#elif defined(__ia64__) || defined(__ia64)
+  #define PLATFORM_ARCH_IA64 1
+  #define PLATFORM_ARCH_FAMILYNAME IA64
+  #define _PLATFORM_ARCH_64 1
+  #if defined(PLATFORM_OS_LINUX) || defined(PLATFORM_OS_FREEBSD)
+    #define _PLATFORM_ARCH_LITTLE_ENDIAN 1
+  #elif defined(PLATFORM_OS_HPUX)
+    #define _PLATFORM_ARCH_BIG_ENDIAN 1
+  #else
+    /* Unknown.  Hope one of the other mechanisms can sort it out. */
+  #endif
+
+#elif defined(__i386__) || defined(__i386) || \
+      defined(__i486__) || defined(__i486) || \
+      defined(__i586__) || defined(__i586) || \
+      defined(__i686__) || defined(__i686) || \
+      defined(__pentiumpro) || defined(__pentiumpro__) || \
+      defined(_M_IX86)
+  #define PLATFORM_ARCH_X86 1
+  #define PLATFORM_ARCH_FAMILYNAME X86
+  #define _PLATFORM_ARCH_32 1
+  #define _PLATFORM_ARCH_LITTLE_ENDIAN 1
+
+#elif defined(__alpha) || defined(__alpha__)
+  #define PLATFORM_ARCH_ALPHA 1
+  #define PLATFORM_ARCH_FAMILYNAME ALPHA
+  #define _PLATFORM_ARCH_64 1
+  #define _PLATFORM_ARCH_LITTLE_ENDIAN 1
+
+#elif defined(_mips) || defined(__mips) || defined(__mips__) || \
+    defined(__host_mips) || defined(__host_mips__) || \
+    defined(_MIPS_ARCH) || defined(__R4000)
+  #define PLATFORM_ARCH_MIPS 1
+  #define PLATFORM_ARCH_FAMILYNAME MIPS
+  #ifdef _MIPSEL /* MIPS cores support both little and big endian modes */
+    /* SiCortex */
+    #define _PLATFORM_ARCH_LITTLE_ENDIAN 1
+  #else
+    /* IRIX */
+    #define _PLATFORM_ARCH_BIG_ENDIAN 1
+  #endif
+  #ifdef _MIPS_SZPTR
+    #if _MIPS_SZPTR == 32
+      #define _PLATFORM_ARCH_32 1
+    #elif _MIPS_SZPTR == 64
+      #define _PLATFORM_ARCH_64 1
+    #endif
+  #endif
+
+#elif defined(__sparc) || defined(__sparc__) || \
+    defined(__sparclet__) || defined(__sparclite__) || \
+    defined(__sparcv8) || defined(__sparcv9)
+  #define PLATFORM_ARCH_SPARC 1
+  #define PLATFORM_ARCH_FAMILYNAME SPARC
+  #define _PLATFORM_ARCH_BIG_ENDIAN 1
+
+#elif defined(__hppa) || defined(__hppa__) || \
+    defined(__parisc) || defined(__parisc__) || \
+    defined(_PA_RISC1_1) || defined(_PA_RISC2_0)
+  #define PLATFORM_ARCH_PARISC 1
+  #define PLATFORM_ARCH_FAMILYNAME PARISC
+  #define _PLATFORM_ARCH_BIG_ENDIAN 1
+
+#elif defined(__crayx1)
+  #define PLATFORM_ARCH_CRAYX1 1
+  #define PLATFORM_ARCH_FAMILYNAME CRAYX1
+  #define _PLATFORM_ARCH_BIG_ENDIAN 1
+  #define _PLATFORM_ARCH_64 1
+
+#elif defined(_CRAYT3E)
+  #define PLATFORM_ARCH_CRAYT3E 1
+  #define PLATFORM_ARCH_FAMILYNAME CRAYT3E
+  #define _PLATFORM_ARCH_BIG_ENDIAN 1
+  #define _PLATFORM_ARCH_64 1
+
+#elif defined(__MTA__)
+  #define PLATFORM_ARCH_MTA 1
+  #define PLATFORM_ARCH_FAMILYNAME MTA
+
+#elif defined(_SX)
+  #define PLATFORM_ARCH_NECSX 1
+  #define PLATFORM_ARCH_FAMILYNAME NECSX
+
+#elif defined(__MICROBLAZE__)
+  #define PLATFORM_ARCH_MICROBLAZE 1
+  #define PLATFORM_ARCH_FAMILYNAME MICROBLAZE
+  #define _PLATFORM_ARCH_BIG_ENDIAN 1
+  #define _PLATFORM_ARCH_32 1
+
+#elif defined(__arm__)
+  #define PLATFORM_ARCH_ARM 1
+  #define PLATFORM_ARCH_FAMILYNAME ARM
+  #define _PLATFORM_ARCH_32 1
+  #if defined(__ARMEB__)
+    #define _PLATFORM_ARCH_BIG_ENDIAN 1
+  #elif defined(__ARMEL__)
+    #define _PLATFORM_ARCH_LITTLE_ENDIAN 1
+  #endif
+
+#elif defined(__aarch64__)
+  #define PLATFORM_ARCH_AARCH64 1
+  #define PLATFORM_ARCH_FAMILYNAME AARCH64
+  #if defined(__AARCH64EB__)
+    #define _PLATFORM_ARCH_BIG_ENDIAN 1
+  #elif defined(__AARCH64EL__)
+    #define _PLATFORM_ARCH_LITTLE_ENDIAN 1
+  #endif
+
+#elif defined(__tile__)
+  #define PLATFORM_ARCH_TILE 1
+  #define PLATFORM_ARCH_FAMILYNAME TILE
+  #define _PLATFORM_ARCH_LITTLE_ENDIAN 1
+  #if defined(__tilegx__)
+    #define _PLATFORM_ARCH_64 1
+  #else
+    #define _PLATFORM_ARCH_32 1
+  #endif
+
+#elif defined(__s390__)
+  #define PLATFORM_ARCH_S390 1
+  #define PLATFORM_ARCH_FAMILYNAME S390
+  #define _PLATFORM_ARCH_BIG_ENDIAN 1
+  #if defined(__s390x__)
+    #define _PLATFORM_ARCH_64 1
+  #else
+    #define _PLATFORM_ARCH_32 1
+  #endif
+
+#elif defined(__riscv)
+  #define PLATFORM_ARCH_RISCV 1
+  #define PLATFORM_ARCH_FAMILYNAME RISCV
+  #define _PLATFORM_ARCH_LITTLE_ENDIAN 1
+  #if __riscv_xlen == 32
+    #define _PLATFORM_ARCH_32 1
+  #else  /* (__riscv_xlen == 64) || (__riscv_xlen == 128) */
+    #define _PLATFORM_ARCH_64 1
+  #endif
+
+#else /* unknown CPU */
+  #define PLATFORM_ARCH_UNKNOWN 1
+  #define PLATFORM_ARCH_FAMILYNAME UNKNOWN
+#endif
+
+/* generic chip properties */
+
+#if defined(PLATFORM_ARCH_BIG_ENDIAN) || defined(PLATFORM_ARCH_LITTLE_ENDIAN)
+  #error internal error in endianness configuration
+#endif
+
+/* PLATFORM_ARCH_{BIG,LITTLE}_ENDIAN:
+    first detect common preprocessor defines
+    then default to any arch-specific value provided
+ */
+
+#if defined(__BIG_ENDIAN__) || defined(WORDS_BIGENDIAN) || \
+    ( __BYTE_ORDER__ > 0 && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ )
+  #define PLATFORM_ARCH_BIG_ENDIAN 1
+#elif defined(__LITTLE_ENDIAN__) || defined(WORDS_LITTLEENDIAN) || \
+    ( __BYTE_ORDER__ > 0 && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ )
+  #define PLATFORM_ARCH_LITTLE_ENDIAN 1
+#elif defined(_PLATFORM_ARCH_BIG_ENDIAN)
+  #define PLATFORM_ARCH_BIG_ENDIAN 1
+#elif defined(_PLATFORM_ARCH_LITTLE_ENDIAN)
+  #define PLATFORM_ARCH_LITTLE_ENDIAN 1
+#endif
+#undef _PLATFORM_ARCH_BIG_ENDIAN
+#undef _PLATFORM_ARCH_LITTLE_ENDIAN
+
+#if defined(PLATFORM_ARCH_BIG_ENDIAN) && defined(PLATFORM_ARCH_LITTLE_ENDIAN)
+  #error conflicting endianness information
+#endif
+
+/* PLATFORM_ARCH_{32,64}: 
+    first trust SIZEOF_VOID_P, which is most likely to be accurate
+    next, detect common 32/64 preprocessor defines
+    finally default to any arch-specific value provided
+ */
+#if defined(PLATFORM_ARCH_64) || defined(PLATFORM_ARCH_32)
+  #error internal error in bit width configuration
+#endif
+
+#if SIZEOF_VOID_P == 8
+  #define PLATFORM_ARCH_64 1
+#elif SIZEOF_VOID_P == 4
+  #define PLATFORM_ARCH_32 1
+#elif defined(_LP64) || defined(__LP64__) || \
+      defined(__arch64__) || defined(__64BIT__) || \
+      __INTPTR_MAX__ > 2147483647
+  #define PLATFORM_ARCH_64 1
+#elif defined(_ILP32) || defined(__ILP32__) || \
+      defined(__arch32__) || defined(__32BIT__) || \
+      __INTPTR_MAX__ == 2147483647
+  #define PLATFORM_ARCH_32 1
+#elif defined(_PLATFORM_ARCH_64)
+  #define PLATFORM_ARCH_64 1
+#elif defined(_PLATFORM_ARCH_32)
+  #define PLATFORM_ARCH_32 1
+#endif
+#undef _PLATFORM_ARCH_64
+#undef _PLATFORM_ARCH_32
+
+#if defined(PLATFORM_ARCH_64) && defined(PLATFORM_ARCH_32)
+  #error conflicting bit width information
+#elif !defined(PLATFORM_ARCH_64) && !defined(PLATFORM_ARCH_32)
+  #error missing bit width information
+#endif
+
+/* ------------------------------------------------------------------------------------ */
+/* handy test code that can be parsed after preprocess or executed to show platform results */
+#ifdef PLATFORM_SHOW
+#include <stdio.h>
+#include <assert.h>
+const char *
+COMPILER_FAMILYNAME = PLATFORM_STRINGIFY(PLATFORM_COMPILER_FAMILYNAME)
+, *
+COMPILER_FAMILYID = PLATFORM_STRINGIFY(PLATFORM_COMPILER_FAMILYID)
+, *
+COMPILER_VERSION_STR = PLATFORM_COMPILER_VERSION_STR
+, *
+COMPILER_IDSTR = PLATFORM_COMPILER_IDSTR
+, *
+OS_FAMILYNAME = PLATFORM_STRINGIFY(PLATFORM_OS_FAMILYNAME)
+, *
+ARCH_FAMILYNAME = PLATFORM_STRINGIFY(PLATFORM_ARCH_FAMILYNAME)
+;
+int main(void) {
+  #define PLATFORM_DISP(x) printf("PLATFORM_"#x"=%s\n",x)
+  #define PLATFORM_DISPI(x) printf("PLATFORM_"#x"=%li\n",(long int)PLATFORM_##x)
+  #define PLATFORM_DISPX(x) printf("PLATFORM_"#x"=0x%lx\n",(long int)PLATFORM_##x)
+  PLATFORM_DISP(COMPILER_FAMILYNAME);
+  PLATFORM_DISP(COMPILER_FAMILYID);
+  PLATFORM_DISPI(COMPILER_ID);
+  PLATFORM_DISPX(COMPILER_VERSION);
+  PLATFORM_DISP(COMPILER_VERSION_STR);
+  PLATFORM_DISP(COMPILER_IDSTR);
+  #ifdef PLATFORM_COMPILER_C_LANGLVL
+    PLATFORM_DISPI(COMPILER_C_LANGLVL);
+  #elif defined(PLATFORM_COMPILER_CXX_LANGLVL)
+    PLATFORM_DISPI(COMPILER_CXX_LANGLVL);
+  #else
+    printf("WARNING: Missing PLATFORM_COMPILER_C(XX)_LANGLVL!");
+  #endif
+  PLATFORM_DISP(OS_FAMILYNAME);
+  #ifdef PLATFORM_OS_SUBFAMILYNAME
+  { const char * OS_SUBFAMILYNAME = PLATFORM_STRINGIFY(PLATFORM_OS_SUBFAMILYNAME);
+    PLATFORM_DISP(OS_SUBFAMILYNAME);
+  }
+  #endif
+  PLATFORM_DISP(ARCH_FAMILYNAME);
+  #ifdef PLATFORM_ARCH_32
+    PLATFORM_DISPI(ARCH_32);
+    assert(sizeof(void *) == 4);
+  #else
+    PLATFORM_DISPI(ARCH_64);
+    assert(sizeof(void *) == 8);
+  #endif
+  { int x = 0x00FF;
+    unsigned char *p = (unsigned char *)&x;
+  #ifdef PLATFORM_ARCH_BIG_ENDIAN
+    PLATFORM_DISPI(ARCH_BIG_ENDIAN);
+    assert(*p == 0);
+  #else
+    PLATFORM_DISPI(ARCH_LITTLE_ENDIAN);
+    assert(*p == 0xFF);
+  #endif
+  }
+  return 0;
+}
+#endif
+/* ------------------------------------------------------------------------------------ */
+#endif
diff --git a/opal/include/opal/opal_socket_errno.h b/opal/include/opal/opal_socket_errno.h
index 1e3f346a0cb..385bf8767a4 100644
--- a/opal/include/opal/opal_socket_errno.h
+++ b/opal/include/opal/opal_socket_errno.h
@@ -18,8 +18,8 @@
 #ifndef OPAL_GET_SOCKET_ERROR_H
 #define OPAL_GET_SOCKET_ERROR_H
 
-#include <errno.h>
 #include "opal/constants.h"
+#include <errno.h>
 
 #define opal_socket_errno errno
 
diff --git a/opal/include/opal/prefetch.h b/opal/include/opal/prefetch.h
index 059b3a023f4..a94a26ca2f7 100644
--- a/opal/include/opal/prefetch.h
+++ b/opal/include/opal/prefetch.h
@@ -22,36 +22,36 @@
 #if defined(c_plusplus) || defined(__cplusplus)
 /* C++ code */
 
-#if OMPI_CXX_HAVE_BUILTIN_EXPECT
-#define OPAL_LIKELY(expression) __builtin_expect(!!(expression), 1)
-#define OPAL_UNLIKELY(expression) __builtin_expect(!!(expression), 0)
-#else
-#define OPAL_LIKELY(expression) (expression)
-#define OPAL_UNLIKELY(expression) (expression)
-#endif
-
-#if OMPI_CXX_HAVE_BUILTIN_PREFETCH
-#define OPAL_PREFETCH(address,rw,locality) __builtin_prefetch(address,rw,locality)
-#else
-#define OPAL_PREFETCH(address,rw,locality)
-#endif
+#    if OMPI_CXX_HAVE_BUILTIN_EXPECT
+#        define OPAL_LIKELY(expression)   __builtin_expect(!!(expression), 1)
+#        define OPAL_UNLIKELY(expression) __builtin_expect(!!(expression), 0)
+#    else
+#        define OPAL_LIKELY(expression)   (expression)
+#        define OPAL_UNLIKELY(expression) (expression)
+#    endif
+
+#    if OMPI_CXX_HAVE_BUILTIN_PREFETCH
+#        define OPAL_PREFETCH(address, rw, locality) __builtin_prefetch(address, rw, locality)
+#    else
+#        define OPAL_PREFETCH(address, rw, locality)
+#    endif
 
 #else
 /* C code */
 
-#if OPAL_C_HAVE_BUILTIN_EXPECT
-#define OPAL_LIKELY(expression) __builtin_expect(!!(expression), 1)
-#define OPAL_UNLIKELY(expression) __builtin_expect(!!(expression), 0)
-#else
-#define OPAL_LIKELY(expression) (expression)
-#define OPAL_UNLIKELY(expression) (expression)
-#endif
-
-#if OPAL_C_HAVE_BUILTIN_PREFETCH
-#define OPAL_PREFETCH(address,rw,locality) __builtin_prefetch(address,rw,locality)
-#else
-#define OPAL_PREFETCH(address,rw,locality)
-#endif
+#    if OPAL_C_HAVE_BUILTIN_EXPECT
+#        define OPAL_LIKELY(expression)   __builtin_expect(!!(expression), 1)
+#        define OPAL_UNLIKELY(expression) __builtin_expect(!!(expression), 0)
+#    else
+#        define OPAL_LIKELY(expression)   (expression)
+#        define OPAL_UNLIKELY(expression) (expression)
+#    endif
+
+#    if OPAL_C_HAVE_BUILTIN_PREFETCH
+#        define OPAL_PREFETCH(address, rw, locality) __builtin_prefetch(address, rw, locality)
+#    else
+#        define OPAL_PREFETCH(address, rw, locality)
+#    endif
 
 #endif
 
diff --git a/opal/include/opal/sys/Makefile.am b/opal/include/opal/sys/Makefile.am
index bf869e8fbbe..b56e909b376 100644
--- a/opal/include/opal/sys/Makefile.am
+++ b/opal/include/opal/sys/Makefile.am
@@ -15,7 +15,9 @@
 #                         reserved.
 # Copyright (c) 2017      Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
-# Copyright (c) 2020      Google, LLC. All rights reserved.
+# Copyright (c) 2020-2021 Google, LLC. All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+#                         All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -26,16 +28,18 @@
 # This makefile.am does not stand on its own - it is included from opal/Makefile.am
 
 headers += \
-	opal/sys/architecture.h \
 	opal/sys/atomic.h \
 	opal/sys/atomic_stdc.h \
-	opal/sys/atomic_impl.h \
+	opal/sys/atomic_impl_minmax_math.h \
+	opal/sys/atomic_impl_ptr_cswap.h \
+	opal/sys/atomic_impl_ptr_llsc.h \
+	opal/sys/atomic_impl_ptr_swap.h \
+	opal/sys/atomic_impl_size_t_math.h \
+	opal/sys/atomic_impl_spinlock.h \
 	opal/sys/timer.h \
         opal/sys/cma.h
 
 include opal/sys/x86_64/Makefile.am
-include opal/sys/arm/Makefile.am
 include opal/sys/arm64/Makefile.am
-include opal/sys/ia32/Makefile.am
 include opal/sys/powerpc/Makefile.am
 include opal/sys/gcc_builtin/Makefile.am
diff --git a/opal/include/opal/sys/architecture.h b/opal/include/opal/sys/architecture.h
deleted file mode 100644
index 38338f1ecb2..00000000000
--- a/opal/include/opal/sys/architecture.h
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
- * Copyright (c) 2014      Intel, Inc. All rights reserved
- * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2017      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2020      Google, LLC. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-/*
- * List of supported architectures
- */
-
-#ifndef OPAL_SYS_ARCHITECTURE_H
-#define OPAL_SYS_ARCHITECTURE_H
-
-/* Architectures */
-#define OPAL_UNSUPPORTED    0000
-#define OPAL_IA32           0010
-#define OPAL_X86_64         0030
-#define OPAL_POWERPC32      0050
-#define OPAL_POWERPC64      0051
-#define OPAL_ARM            0100
-#define OPAL_ARM64          0101
-#define OPAL_BUILTIN_GCC    0202
-#define OPAL_BUILTIN_NO     0203
-#define OPAL_BUILTIN_C11    0204
-
-/* Formats */
-#define OPAL_DEFAULT        1000  /* standard for given architecture */
-
-#endif /* #ifndef OPAL_SYS_ARCHITECTURE_H */
diff --git a/opal/include/opal/sys/arm/Makefile.am b/opal/include/opal/sys/arm/Makefile.am
deleted file mode 100644
index 602cce94bed..00000000000
--- a/opal/include/opal/sys/arm/Makefile.am
+++ /dev/null
@@ -1,24 +0,0 @@
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2008 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-# This makefile.am does not stand on its own - it is included from opal/include/Makefile.am
-
-headers += \
-       opal/sys/arm/atomic.h \
-       opal/sys/arm/timer.h
-
diff --git a/opal/include/opal/sys/arm/atomic.h b/opal/include/opal/sys/arm/atomic.h
deleted file mode 100644
index ff2c5bab351..00000000000
--- a/opal/include/opal/sys/arm/atomic.h
+++ /dev/null
@@ -1,254 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2010      IBM Corporation.  All rights reserved.
- * Copyright (c) 2010      ARM ltd.  All rights reserved.
- * Copyright (c) 2017-2018 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-/*
- * ARMv5 and earlier lack robust atomic operations and therefore this file uses
- * Linux kernel support where needed.  The kernel also provides memory barriers
- * and this file uses them for ARMv5 and earlier processors, which lack the
- * memory barrier instruction.  These kernel functions are available on kernel
- * versions 2.6.15 and greater; using them will result in undefined behavior on
- * older kernels.
- * See Documentation/arm/kernel_user_helpers.txt in the kernel tree for details
- */
-
-#ifndef OPAL_SYS_ARCH_ATOMIC_H
-#define OPAL_SYS_ARCH_ATOMIC_H 1
-
-#if (OPAL_ASM_ARM_VERSION >= 7)
-
-#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
-/* use the DMB instruction if available... */
-
-#define MB()  __asm__ __volatile__ ("dmb" : : : "memory")
-#define RMB() __asm__ __volatile__ ("dmb" : : : "memory")
-#define WMB() __asm__ __volatile__ ("dmb" : : : "memory")
-
-#elif (OPAL_ASM_ARM_VERSION == 6)
-
-#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
-/* ...or the v6-specific equivalent... */
-
-#define MB()  __asm__ __volatile__ ("mcr p15, 0, r0, c7, c10, 5" : : : "memory")
-#define RMB() MB()
-#define WMB() MB()
-
-#else
-
-#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
-/* ...otherwise use the Linux kernel-provided barrier */
-
-#define MB() (*((void (*)(void))(0xffff0fa0)))()
-#define RMB() MB()
-#define WMB() MB()
-
-#endif
-
-/**********************************************************************
- *
- * Memory Barriers
- *
- *********************************************************************/
-
-#if (OPAL_HAVE_ATOMIC_MEM_BARRIER == 1)
-
-static inline
-void opal_atomic_mb(void)
-{
-    MB();
-}
-
-
-static inline
-void opal_atomic_rmb(void)
-{
-    RMB();
-}
-
-
-static inline
-void opal_atomic_wmb(void)
-{
-    WMB();
-}
-
-static inline
-void opal_atomic_isync(void)
-{
-}
-
-#endif
-
-
-/**********************************************************************
- *
- * Atomic math operations
- *
- *********************************************************************/
-
-#if (OPAL_GCC_INLINE_ASSEMBLY && (OPAL_ASM_ARM_VERSION >= 6))
-
-#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
-#define OPAL_HAVE_ATOMIC_MATH_32 1
-static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
-{
-  int32_t prev, tmp;
-  bool ret;
-
-   __asm__ __volatile__ (
-                         "1:  ldrex   %0, [%2]        \n"
-                         "    cmp     %0, %3          \n"
-                         "    bne     2f              \n"
-                         "    strex   %1, %4, [%2]    \n"
-                         "    cmp     %1, #0          \n"
-                         "    bne     1b              \n"
-                         "2:                          \n"
-
-                         : "=&r" (prev), "=&r" (tmp)
-                         : "r" (addr), "r" (*oldval), "r" (newval)
-                         : "cc", "memory");
-
-   ret = (prev == *oldval);
-   *oldval = prev;
-   return ret;
-}
-
-/* these two functions aren't inlined in the non-gcc case because then
-   there would be two function calls (since neither cmpset_32 nor
-   atomic_?mb can be inlined).  Instead, we "inline" them by hand in
-   the assembly, meaning there is one function call overhead instead
-   of two */
-static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
-{
-    bool rc;
-
-    rc = opal_atomic_compare_exchange_strong_32 (addr, oldval, newval);
-    opal_atomic_rmb();
-
-    return rc;
-}
-
-
-static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
-{
-    opal_atomic_wmb();
-    return opal_atomic_compare_exchange_strong_32 (addr, oldval, newval);
-}
-
-#if (OPAL_ASM_SUPPORT_64BIT == 1)
-
-#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
-static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
-{
-    int64_t prev;
-    int tmp;
-    bool ret;
-
-    __asm__ __volatile__ (
-                          "1:  ldrexd  %0, %H0, [%2]           \n"
-                          "    cmp     %0, %3                  \n"
-                          "    it      eq                      \n"
-                          "    cmpeq   %H0, %H3                \n"
-                          "    bne     2f                      \n"
-                          "    strexd  %1, %4, %H4, [%2]       \n"
-                          "    cmp     %1, #0                  \n"
-                          "    bne     1b                      \n"
-                          "2:                                    \n"
-
-                          : "=&r" (prev), "=&r" (tmp)
-                          : "r" (addr), "r" (*oldval), "r" (newval)
-                          : "cc", "memory");
-
-    ret = (prev == *oldval);
-    *oldval = prev;
-    return ret;
-}
-
-/* these two functions aren't inlined in the non-gcc case because then
-   there would be two function calls (since neither cmpset_64 nor
-   atomic_?mb can be inlined).  Instead, we "inline" them by hand in
-   the assembly, meaning there is one function call overhead instead
-   of two */
-static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
-{
-    bool rc;
-
-    rc = opal_atomic_compare_exchange_strong_64 (addr, oldval, newval);
-    opal_atomic_rmb();
-
-    return rc;
-}
-
-
-static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
-{
-    opal_atomic_wmb();
-    return opal_atomic_compare_exchange_strong_64 (addr, oldval, newval);
-}
-
-#endif
-
-
-#define OPAL_HAVE_ATOMIC_ADD_32 1
-static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t* v, int inc)
-{
-    int32_t t, old;
-    int tmp;
-
-    __asm__ __volatile__(
-                         "1:  ldrex   %1, [%3]        \n"
-                         "    add     %0, %1, %4      \n"
-                         "    strex   %2, %0, [%3]    \n"
-                         "    cmp     %2, #0          \n"
-                         "    bne     1b              \n"
-
-                         : "=&r" (t), "=&r" (old), "=&r" (tmp)
-                         : "r" (v), "r" (inc)
-                         : "cc", "memory");
-
-
-    return old;
-}
-
-#define OPAL_HAVE_ATOMIC_SUB_32 1
-static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t* v, int dec)
-{
-    int32_t t, old;
-    int tmp;
-
-    __asm__ __volatile__(
-                         "1:  ldrex   %1, [%3]        \n"
-                         "    sub     %0, %1, %4      \n"
-                         "    strex   %2, %0, [%3]    \n"
-                         "    cmp     %2, #0          \n"
-                         "    bne     1b              \n"
-
-                         : "=&r" (t), "=&r" (old), "=&r" (tmp)
-                         : "r" (v), "r" (dec)
-                         : "cc", "memory");
-
-    return t;
-}
-
-#endif
-
-#endif /* ! OPAL_SYS_ARCH_ATOMIC_H */
diff --git a/opal/include/opal/sys/arm/timer.h b/opal/include/opal/sys/arm/timer.h
deleted file mode 100644
index b93689c908d..00000000000
--- a/opal/include/opal/sys/arm/timer.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright (c) 2008      The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef OPAL_SYS_ARCH_TIMER_H
-#define OPAL_SYS_ARCH_TIMER_H 1
-
-#include <sys/times.h>
-
-typedef uint64_t opal_timer_t;
-
-static inline opal_timer_t
-opal_sys_timer_get_cycles(void)
-{
-    opal_timer_t ret;
-    struct tms accurate_clock;
-
-    times(&accurate_clock);
-    ret = accurate_clock.tms_utime + accurate_clock.tms_stime;
-
-    return ret;
-}
-
-#define OPAL_HAVE_SYS_TIMER_GET_CYCLES 1
-
-#endif /* ! OPAL_SYS_ARCH_TIMER_H */
diff --git a/opal/include/opal/sys/arm/update.sh b/opal/include/opal/sys/arm/update.sh
deleted file mode 100644
index 94d8ed2714b..00000000000
--- a/opal/include/opal/sys/arm/update.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/sh
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-CFILE=/tmp/opal_atomic_$$.c
-
-trap "/bin/rm -f $CFILE; exit 0" 0 1 2 15
-
-echo Updating atomic.s from atomic.h using gcc
-
-cat > $CFILE<<EOF
-#include <stdlib.h>
-#include <inttypes.h>
-#define static
-#define inline
-#define OPAL_GCC_INLINE_ASSEMBLY 1
-#include "../architecture.h"
-#include "atomic.h"
-EOF
-
-gcc -O1 -I. -S $CFILE -o atomic.s
diff --git a/opal/include/opal/sys/arm64/atomic.h b/opal/include/opal/sys/arm64/atomic.h
index 8495901ace1..6fb7b6db268 100644
--- a/opal/include/opal/sys/arm64/atomic.h
+++ b/opal/include/opal/sys/arm64/atomic.h
@@ -16,6 +16,8 @@
  *                         reserved.
  * Copyright (c) 2021      Triad National Security, LLC. All rights reserved.
  * Copyright (c) 2021      Google, LLC. All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -25,32 +27,9 @@
 
 #include "atomic_llsc.h"
 
-#if !defined(OPAL_SYS_ARCH_ATOMIC_H)
-
+#ifndef OPAL_SYS_ARCH_ATOMIC_H
 #define OPAL_SYS_ARCH_ATOMIC_H 1
 
-#if OPAL_GCC_INLINE_ASSEMBLY
-
-#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
-#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
-#define OPAL_HAVE_ATOMIC_SWAP_32 1
-#define OPAL_HAVE_ATOMIC_MATH_32 1
-#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
-#define OPAL_HAVE_ATOMIC_SWAP_64 1
-#define OPAL_HAVE_ATOMIC_ADD_32 1
-#define OPAL_HAVE_ATOMIC_AND_32 1
-#define OPAL_HAVE_ATOMIC_OR_32 1
-#define OPAL_HAVE_ATOMIC_XOR_32 1
-#define OPAL_HAVE_ATOMIC_SUB_32 1
-#define OPAL_HAVE_ATOMIC_ADD_64 1
-#define OPAL_HAVE_ATOMIC_AND_64 1
-#define OPAL_HAVE_ATOMIC_OR_64 1
-#define OPAL_HAVE_ATOMIC_XOR_64 1
-#define OPAL_HAVE_ATOMIC_SUB_64 1
-
-#define MB()  __asm__ __volatile__ ("dmb sy" : : : "memory")
-#define RMB() __asm__ __volatile__ ("dmb ld" : : : "memory")
-#define WMB() __asm__ __volatile__ ("dmb st" : : : "memory")
 
 /**********************************************************************
  *
@@ -58,211 +37,260 @@
  *
  *********************************************************************/
 
-static inline void opal_atomic_mb (void)
+static inline void opal_atomic_mb(void)
 {
-    MB();
+    __asm__ __volatile__("dmb sy" : : : "memory");
 }
 
-static inline void opal_atomic_rmb (void)
+static inline void opal_atomic_rmb(void)
 {
-    RMB();
+    __asm__ __volatile__("dmb ld" : : : "memory");
 }
 
-static inline void opal_atomic_wmb (void)
+static inline void opal_atomic_wmb(void)
 {
-    WMB();
+    __asm__ __volatile__("dmb st" : : : "memory");
 }
 
-static inline void opal_atomic_isync (void)
+static inline void opal_atomic_isync(void)
 {
-    __asm__ __volatile__ ("isb");
+    __asm__ __volatile__("isb");
 }
 
+
 /**********************************************************************
  *
- * Atomic math operations
+ * Compare and Swap
  *
  *********************************************************************/
 
-static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
+static inline bool opal_atomic_compare_exchange_strong_32(opal_atomic_int32_t *addr,
+                                                          int32_t *oldval, int32_t newval)
 {
     int32_t prev, tmp;
     bool ret;
 
-    __asm__ __volatile__ ("1:  ldaxr    %w0, [%2]      \n"
-                          "    cmp     %w0, %w3        \n"
-                          "    bne     2f              \n"
-                          "    stxr    %w1, %w4, [%2]  \n"
-                          "    cbnz    %w1, 1b         \n"
-                          "2:                          \n"
-                          : "=&r" (prev), "=&r" (tmp)
-                          : "r" (addr), "r" (*oldval), "r" (newval)
-                          : "cc", "memory");
+    __asm__ __volatile__("1:  ldaxr    %w0, [%2]      \n"
+                         "    cmp     %w0, %w3        \n"
+                         "    bne     2f              \n"
+                         "    stxr    %w1, %w4, [%2]  \n"
+                         "    cbnz    %w1, 1b         \n"
+                         "2:                          \n"
+                         : "=&r"(prev), "=&r"(tmp)
+                         : "r"(addr), "r"(*oldval), "r"(newval)
+                         : "cc", "memory");
 
     ret = (prev == *oldval);
     *oldval = prev;
     return ret;
 }
 
-static inline int32_t opal_atomic_swap_32(opal_atomic_int32_t *addr, int32_t newval)
-{
-    int32_t ret, tmp;
-
-    __asm__ __volatile__ ("1:  ldaxr   %w0, [%2]       \n"
-                          "    stlxr   %w1, %w3, [%2]  \n"
-                          "    cbnz    %w1, 1b         \n"
-                          : "=&r" (ret), "=&r" (tmp)
-                          : "r" (addr), "r" (newval)
-                          : "cc", "memory");
-
-    return ret;
-}
-
 /* these two functions aren't inlined in the non-gcc case because then
    there would be two function calls (since neither cmpset_32 nor
    atomic_?mb can be inlined).  Instead, we "inline" them by hand in
    the assembly, meaning there is one function call overhead instead
    of two */
-static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
+static inline bool opal_atomic_compare_exchange_strong_acq_32(opal_atomic_int32_t *addr,
+                                                              int32_t *oldval, int32_t newval)
 {
     int32_t prev, tmp;
     bool ret;
 
-    __asm__ __volatile__ ("1:  ldaxr   %w0, [%2]       \n"
-                          "    cmp     %w0, %w3        \n"
-                          "    bne     2f              \n"
-                          "    stxr    %w1, %w4, [%2]  \n"
-                          "    cbnz    %w1, 1b         \n"
-                          "2:                          \n"
-                          : "=&r" (prev), "=&r" (tmp)
-                          : "r" (addr), "r" (*oldval), "r" (newval)
-                          : "cc", "memory");
+    __asm__ __volatile__("1:  ldaxr   %w0, [%2]       \n"
+                         "    cmp     %w0, %w3        \n"
+                         "    bne     2f              \n"
+                         "    stxr    %w1, %w4, [%2]  \n"
+                         "    cbnz    %w1, 1b         \n"
+                         "2:                          \n"
+                         : "=&r"(prev), "=&r"(tmp)
+                         : "r"(addr), "r"(*oldval), "r"(newval)
+                         : "cc", "memory");
 
     ret = (prev == *oldval);
     *oldval = prev;
     return ret;
 }
 
-
-static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
+static inline bool opal_atomic_compare_exchange_strong_rel_32(opal_atomic_int32_t *addr,
+                                                              int32_t *oldval, int32_t newval)
 {
     int32_t prev, tmp;
     bool ret;
 
-    __asm__ __volatile__ ("1:  ldxr    %w0, [%2]       \n"
-                          "    cmp     %w0, %w3        \n"
-                          "    bne     2f              \n"
-                          "    stlxr   %w1, %w4, [%2]  \n"
-                          "    cbnz    %w1, 1b         \n"
-                          "2:                          \n"
-                          : "=&r" (prev), "=&r" (tmp)
-                          : "r" (addr), "r" (*oldval), "r" (newval)
-                          : "cc", "memory");
+    __asm__ __volatile__("1:  ldxr    %w0, [%2]       \n"
+                         "    cmp     %w0, %w3        \n"
+                         "    bne     2f              \n"
+                         "    stlxr   %w1, %w4, [%2]  \n"
+                         "    cbnz    %w1, 1b         \n"
+                         "2:                          \n"
+                         : "=&r"(prev), "=&r"(tmp)
+                         : "r"(addr), "r"(*oldval), "r"(newval)
+                         : "cc", "memory");
 
     ret = (prev == *oldval);
     *oldval = prev;
     return ret;
 }
 
-static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
+static inline bool opal_atomic_compare_exchange_strong_64(opal_atomic_int64_t *addr,
+                                                          int64_t *oldval, int64_t newval)
 {
     int64_t prev;
     int tmp;
     bool ret;
 
-    __asm__ __volatile__ ("1:  ldaxr    %0, [%2]       \n"
-                          "    cmp     %0, %3          \n"
-                          "    bne     2f              \n"
-                          "    stxr    %w1, %4, [%2]   \n"
-                          "    cbnz    %w1, 1b         \n"
-                          "2:                          \n"
-                          : "=&r" (prev), "=&r" (tmp)
-                          : "r" (addr), "r" (*oldval), "r" (newval)
-                          : "cc", "memory");
+    __asm__ __volatile__("1:  ldaxr    %0, [%2]       \n"
+                         "    cmp     %0, %3          \n"
+                         "    bne     2f              \n"
+                         "    stxr    %w1, %4, [%2]   \n"
+                         "    cbnz    %w1, 1b         \n"
+                         "2:                          \n"
+                         : "=&r"(prev), "=&r"(tmp)
+                         : "r"(addr), "r"(*oldval), "r"(newval)
+                         : "cc", "memory");
 
     ret = (prev == *oldval);
     *oldval = prev;
     return ret;
 }
 
-static inline int64_t opal_atomic_swap_64 (opal_atomic_int64_t *addr, int64_t newval)
-{
-    int64_t ret;
-    int tmp;
-
-    __asm__ __volatile__ ("1:  ldaxr   %0, [%2]        \n"
-                          "    stlxr   %w1, %3, [%2]   \n"
-                          "    cbnz    %w1, 1b         \n"
-                          : "=&r" (ret), "=&r" (tmp)
-                          : "r" (addr), "r" (newval)
-                          : "cc", "memory");
-
-    return ret;
-}
-
 /* these two functions aren't inlined in the non-gcc case because then
    there would be two function calls (since neither cmpset_64 nor
    atomic_?mb can be inlined).  Instead, we "inline" them by hand in
    the assembly, meaning there is one function call overhead instead
    of two */
-static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
+static inline bool opal_atomic_compare_exchange_strong_acq_64(opal_atomic_int64_t *addr,
+                                                              int64_t *oldval, int64_t newval)
 {
     int64_t prev;
     int tmp;
     bool ret;
 
-    __asm__ __volatile__ ("1:  ldaxr   %0, [%2]        \n"
-                          "    cmp     %0, %3          \n"
-                          "    bne     2f              \n"
-                          "    stxr    %w1, %4, [%2]   \n"
-                          "    cbnz    %w1, 1b         \n"
-                          "2:                          \n"
-                          : "=&r" (prev), "=&r" (tmp)
-                          : "r" (addr), "r" (*oldval), "r" (newval)
-                          : "cc", "memory");
+    __asm__ __volatile__("1:  ldaxr   %0, [%2]        \n"
+                         "    cmp     %0, %3          \n"
+                         "    bne     2f              \n"
+                         "    stxr    %w1, %4, [%2]   \n"
+                         "    cbnz    %w1, 1b         \n"
+                         "2:                          \n"
+                         : "=&r"(prev), "=&r"(tmp)
+                         : "r"(addr), "r"(*oldval), "r"(newval)
+                         : "cc", "memory");
 
     ret = (prev == *oldval);
     *oldval = prev;
     return ret;
 }
 
-
-static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
+static inline bool opal_atomic_compare_exchange_strong_rel_64(opal_atomic_int64_t *addr,
+                                                              int64_t *oldval, int64_t newval)
 {
     int64_t prev;
     int tmp;
     bool ret;
 
-    __asm__ __volatile__ ("1:  ldxr    %0, [%2]        \n"
-                          "    cmp     %0, %3          \n"
-                          "    bne     2f              \n"
-                          "    stlxr   %w1, %4, [%2]   \n"
-                          "    cbnz    %w1, 1b         \n"
-                          "2:                          \n"
-                          : "=&r" (prev), "=&r" (tmp)
-                          : "r" (addr), "r" (*oldval), "r" (newval)
-                          : "cc", "memory");
+    __asm__ __volatile__("1:  ldxr    %0, [%2]        \n"
+                         "    cmp     %0, %3          \n"
+                         "    bne     2f              \n"
+                         "    stlxr   %w1, %4, [%2]   \n"
+                         "    cbnz    %w1, 1b         \n"
+                         "2:                          \n"
+                         : "=&r"(prev), "=&r"(tmp)
+                         : "r"(addr), "r"(*oldval), "r"(newval)
+                         : "cc", "memory");
 
     ret = (prev == *oldval);
     *oldval = prev;
     return ret;
 }
 
-#define OPAL_ASM_MAKE_ATOMIC(type, bits, name, inst, reg)                   \
-    static inline type opal_atomic_fetch_ ## name ## _ ## bits (opal_atomic_ ## type *addr, type value) \
-    {                                                                   \
-        type newval, old;                                               \
-        int32_t tmp;                                                    \
-                                                                        \
-        __asm__ __volatile__("1:  ldxr   %" reg "1, [%3]        \n"     \
-                             "    " inst "   %" reg "0, %" reg "1, %" reg "4 \n" \
-                             "    stxr   %w2, %" reg "0, [%3]   \n"     \
-                             "    cbnz   %w2, 1b         \n"            \
-                             : "=&r" (newval), "=&r" (old), "=&r" (tmp) \
-                             : "r" (addr), "r" (value)                  \
-                             : "cc", "memory");                         \
-                                                                        \
-        return old;                                                     \
+#include "opal/sys/atomic_impl_ptr_cswap.h"
+
+
+/**********************************************************************
+ *
+ * Swap
+ *
+ *********************************************************************/
+
+static inline int32_t opal_atomic_swap_32(opal_atomic_int32_t *addr, int32_t newval)
+{
+    int32_t ret, tmp;
+
+    __asm__ __volatile__("1:  ldaxr   %w0, [%2]       \n"
+                         "    stlxr   %w1, %w3, [%2]  \n"
+                         "    cbnz    %w1, 1b         \n"
+                         : "=&r"(ret), "=&r"(tmp)
+                         : "r"(addr), "r"(newval)
+                         : "cc", "memory");
+
+    return ret;
+}
+
+static inline int64_t opal_atomic_swap_64(opal_atomic_int64_t *addr, int64_t newval)
+{
+    int64_t ret;
+    int tmp;
+
+    __asm__ __volatile__("1:  ldaxr   %0, [%2]        \n"
+                         "    stlxr   %w1, %3, [%2]   \n"
+                         "    cbnz    %w1, 1b         \n"
+                         : "=&r"(ret), "=&r"(tmp)
+                         : "r"(addr), "r"(newval)
+                         : "cc", "memory");
+
+    return ret;
+}
+
+#include "opal/sys/atomic_impl_ptr_swap.h"
+
+
+/**********************************************************************
+ *
+ * Atomic spinlocks
+ *
+ *********************************************************************/
+
+#include "opal/sys/atomic_impl_spinlock.h"
+
+
+/**********************************************************************
+ *
+ * Atomic math operations
+ *
+ *********************************************************************/
+
+#define OPAL_ASM_MAKE_ATOMIC(type, bits, name, inst, reg)                          \
+    static inline type opal_atomic_fetch_##name##_##bits(opal_atomic_##type *addr, \
+                                                         type value)               \
+    {                                                                              \
+        type newval, old;                                                          \
+        int32_t tmp;                                                               \
+                                                                                   \
+        __asm__ __volatile__("1:  ldxr   %" reg "1, [%3]        \n"                \
+                             "    " inst "   %" reg "0, %" reg "1, %" reg "4 \n"   \
+                             "    stxr   %w2, %" reg "0, [%3]   \n"                \
+                             "    cbnz   %w2, 1b         \n"                       \
+                             : "=&r"(newval), "=&r"(old), "=&r"(tmp)               \
+                             : "r"(addr), "r"(value)                               \
+                             : "cc", "memory");                                    \
+                                                                                   \
+        return old;                                                                \
+    }                                                                              \
+    static inline type opal_atomic_##name##_fetch_##bits(opal_atomic_##type *addr, \
+                                                         type value)               \
+    {                                                                              \
+        type newval, old;                                                          \
+        int32_t tmp;                                                               \
+                                                                                   \
+        __asm__ __volatile__("1:  ldxr   %" reg "1, [%3]        \n"                \
+                             "    " inst "   %" reg "0, %" reg "1, %" reg "4 \n"   \
+                             "    stxr   %w2, %" reg "0, [%3]   \n"                \
+                             "    cbnz   %w2, 1b         \n"                       \
+                             : "=&r"(newval), "=&r"(old), "=&r"(tmp)               \
+                             : "r"(addr), "r"(value)                               \
+                             : "cc", "memory");                                    \
+                                                                                   \
+        return newval;                                                             \
     }
 
 OPAL_ASM_MAKE_ATOMIC(int32_t, 32, add, "add", "w")
@@ -270,12 +298,15 @@ OPAL_ASM_MAKE_ATOMIC(int32_t, 32, and, "and", "w")
 OPAL_ASM_MAKE_ATOMIC(int32_t, 32, or, "orr", "w")
 OPAL_ASM_MAKE_ATOMIC(int32_t, 32, xor, "eor", "w")
 OPAL_ASM_MAKE_ATOMIC(int32_t, 32, sub, "sub", "w")
+
 OPAL_ASM_MAKE_ATOMIC(int64_t, 64, add, "add", "")
 OPAL_ASM_MAKE_ATOMIC(int64_t, 64, and, "and", "")
 OPAL_ASM_MAKE_ATOMIC(int64_t, 64, or, "orr", "")
 OPAL_ASM_MAKE_ATOMIC(int64_t, 64, xor, "eor", "")
 OPAL_ASM_MAKE_ATOMIC(int64_t, 64, sub, "sub", "")
 
-#endif /* OPAL_GCC_INLINE_ASSEMBLY */
+#include "opal/sys/atomic_impl_minmax_math.h"
+#include "opal/sys/atomic_impl_size_t_math.h"
+
 
 #endif /* ! OPAL_SYS_ARCH_ATOMIC_H */
diff --git a/opal/include/opal/sys/arm64/atomic_llsc.h b/opal/include/opal/sys/arm64/atomic_llsc.h
index 56dff4934c3..f51ab4a3481 100644
--- a/opal/include/opal/sys/arm64/atomic_llsc.h
+++ b/opal/include/opal/sys/arm64/atomic_llsc.h
@@ -16,6 +16,8 @@
  *                         reserved.
  * Copyright (c) 2021      Triad National Security, LLC. All rights reserved.
  * Copyright (c) 2021      Google, LLC. All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -23,70 +25,63 @@
  * $HEADER$
  */
 
-#if !defined(OPAL_SYS_ARCH_ATOMIC_LLSC_H)
+#ifndef OPAL_SYS_ARCH_ATOMIC_LLSC_H
+#define OPAL_SYS_ARCH_ATOMIC_LLSC_H 1
 
-#define OPAL_SYS_ARCH_ATOMIC_LLSC_H
+/*
+ * this file is included even when C11 or GCC built-in atomics are
+ * used, which is why we must check for gcc inline assembly support.
+ */
 
-#if OPAL_C_GCC_INLINE_ASSEMBLY
+#    if OPAL_C_GCC_INLINE_ASSEMBLY
 
-#undef OPAL_HAVE_ATOMIC_LLSC_32
-#undef OPAL_HAVE_ATOMIC_LLSC_64
+#        define OPAL_HAVE_ATOMIC_LLSC_32 1
+#        define OPAL_HAVE_ATOMIC_LLSC_64 1
 
-#define OPAL_HAVE_ATOMIC_LLSC_32 1
-#define OPAL_HAVE_ATOMIC_LLSC_64 1
+#        define opal_atomic_ll_32(addr, ret)                                                       \
+            do {                                                                                   \
+                opal_atomic_int32_t *_addr = (addr);                                               \
+                                                                                                   \
+                __asm__ __volatile__("ldaxr    %w0, [%1]          \n" : "=&r"(ret) : "r"(_addr));  \
+            } while (0)
 
-#define opal_atomic_ll_32(addr, ret)                                    \
-    do {                                                                \
-        opal_atomic_int32_t *_addr = (addr);                               \
-        int32_t _ret;                                                   \
-                                                                        \
-        __asm__ __volatile__ ("ldaxr    %w0, [%1]          \n"          \
-                              : "=&r" (_ret)                            \
-                              : "r" (_addr));                           \
-                                                                        \
-        ret = (typeof(ret)) _ret;                                       \
-    } while (0)
+#        define opal_atomic_sc_32(addr, newval, ret)                  \
+            do {                                                      \
+                opal_atomic_int32_t *_addr = (addr);                  \
+                int32_t _newval = (int32_t) newval;                   \
+                int _ret;                                             \
+                                                                      \
+                __asm__ __volatile__("stlxr    %w0, %w2, [%1]     \n" \
+                                     : "=&r"(_ret)                    \
+                                     : "r"(_addr), "r"(_newval)       \
+                                     : "cc", "memory");               \
+                                                                      \
+                ret = (_ret == 0);                                    \
+            } while (0)
 
-#define opal_atomic_sc_32(addr, newval, ret)                            \
-    do {                                                                \
-        opal_atomic_int32_t *_addr = (addr);                               \
-        int32_t _newval = (int32_t) newval;                             \
-        int _ret;                                                       \
-                                                                        \
-        __asm__ __volatile__ ("stlxr    %w0, %w2, [%1]     \n"          \
-                              : "=&r" (_ret)                            \
-                              : "r" (_addr), "r" (_newval)              \
-                              : "cc", "memory");                        \
-                                                                        \
-        ret = (_ret == 0);                                              \
-    } while (0)
+#        define opal_atomic_ll_64(addr, ret)                                                      \
+            do {                                                                                  \
+                opal_atomic_int64_t *_addr = (addr);                                              \
+                                                                                                  \
+                __asm__ __volatile__("ldaxr    %0, [%1]          \n" : "=&r"(ret) : "r"(_addr)); \
+            } while (0)
 
-#define opal_atomic_ll_64(addr, ret)                            \
-    do {                                                        \
-        opal_atomic_int64_t *_addr = (addr);                       \
-        int64_t _ret;                                           \
-                                                                \
-        __asm__ __volatile__ ("ldaxr    %0, [%1]          \n"   \
-                              : "=&r" (_ret)                    \
-                              : "r" (_addr));                   \
-                                                                \
-        ret = (typeof(ret)) _ret;                               \
-    } while (0)
+#        define opal_atomic_sc_64(addr, newval, ret)                 \
+            do {                                                     \
+                opal_atomic_int64_t *_addr = (addr);                 \
+                int64_t _newval = (int64_t) newval;                  \
+                int _ret;                                            \
+                                                                     \
+                __asm__ __volatile__("stlxr    %w0, %2, [%1]     \n" \
+                                     : "=&r"(_ret)                   \
+                                     : "r"(_addr), "r"(_newval)      \
+                                     : "cc", "memory");              \
+                                                                     \
+                ret = (_ret == 0);                                   \
+            } while (0)
 
-#define opal_atomic_sc_64(addr, newval, ret)                            \
-    do {                                                                \
-        opal_atomic_int64_t *_addr = (addr);                               \
-        int64_t _newval = (int64_t) newval;                             \
-        int _ret;                                                       \
-                                                                        \
-        __asm__ __volatile__ ("stlxr    %w0, %2, [%1]     \n"           \
-                              : "=&r" (_ret)                            \
-                              : "r" (_addr), "r" (_newval)              \
-                              : "cc", "memory");                        \
-                                                                        \
-        ret = (_ret == 0);                                              \
-    } while (0)
+#include "opal/sys/atomic_impl_ptr_llsc.h"
 
-#endif /* OPAL_GCC_INLINE_ASSEMBLY */
+#    endif /* OPAL_C_GCC_INLINE_ASSEMBLY */
 
 #endif /* ! OPAL_SYS_ARCH_ATOMIC_LLSC_H */
diff --git a/opal/include/opal/sys/arm64/timer.h b/opal/include/opal/sys/arm64/timer.h
index adf2dc57b99..0f237e81506 100644
--- a/opal/include/opal/sys/arm64/timer.h
+++ b/opal/include/opal/sys/arm64/timer.h
@@ -6,6 +6,9 @@
  * Copyright (c) 2016      Broadcom Limited. All rights reserved.
  * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2021      Google, LLC. All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -16,29 +19,42 @@
 #ifndef OPAL_SYS_ARCH_TIMER_H
 #define OPAL_SYS_ARCH_TIMER_H 1
 
+#if defined(PLATFORM_ARCH_AARCH64)
 typedef uint64_t opal_timer_t;
+#else
+typedef uint32_t opal_timer_t;
+#endif
 
-static inline opal_timer_t
-opal_sys_timer_get_cycles(void)
+#if OPAL_C_GCC_INLINE_ASSEMBLY
+
+static inline opal_timer_t opal_sys_timer_get_cycles(void)
 {
     opal_timer_t ret;
 
-    __asm__ __volatile__ ("isb" ::: "memory");
-    __asm__ __volatile__ ("mrs %0,  CNTVCT_EL0" : "=r" (ret));
+    __asm__ __volatile__("isb" ::: "memory");
+#if defined(PLATFORM_ARCH_AARCH64)
+    __asm__ __volatile__("mrs %0,  CNTVCT_EL0" : "=r"(ret));
+#else
+    __asm__ __volatile__("mrs %0,  CNTVCT" : "=r"(ret));
+#endif
 
     return ret;
 }
 
-
-static inline opal_timer_t
-opal_sys_timer_get_freq(void)
+static inline opal_timer_t opal_sys_timer_get_freq(void)
 {
     opal_timer_t freq;
-    __asm__ __volatile__ ("mrs %0,  CNTFRQ_EL0" : "=r" (freq));
+#if defined(PLATFORM_ARCH_AARCH64)
+    __asm__ __volatile__("mrs %0,  CNTFRQ_EL0" : "=r"(freq));
+#else
+    __asm__ __volatile__("mrs %0,  CNTFRQ" : "=r"(freq));
+#endif
     return (opal_timer_t)(freq);
 }
 
 #define OPAL_HAVE_SYS_TIMER_GET_CYCLES 1
-#define OPAL_HAVE_SYS_TIMER_GET_FREQ 1
+#define OPAL_HAVE_SYS_TIMER_GET_FREQ   1
+
+#endif /* OPAL_C_GCC_INLINE_ASSEMBLY */
 
 #endif /* ! OPAL_SYS_ARCH_TIMER_H */
diff --git a/opal/include/opal/sys/arm64/update.sh b/opal/include/opal/sys/arm64/update.sh
deleted file mode 100644
index 94d8ed2714b..00000000000
--- a/opal/include/opal/sys/arm64/update.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/sh
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-CFILE=/tmp/opal_atomic_$$.c
-
-trap "/bin/rm -f $CFILE; exit 0" 0 1 2 15
-
-echo Updating atomic.s from atomic.h using gcc
-
-cat > $CFILE<<EOF
-#include <stdlib.h>
-#include <inttypes.h>
-#define static
-#define inline
-#define OPAL_GCC_INLINE_ASSEMBLY 1
-#include "../architecture.h"
-#include "atomic.h"
-EOF
-
-gcc -O1 -I. -S $CFILE -o atomic.s
diff --git a/opal/include/opal/sys/atomic.h b/opal/include/opal/sys/atomic.h
index e17212e9fee..cc047b59d3d 100644
--- a/opal/include/opal/sys/atomic.h
+++ b/opal/include/opal/sys/atomic.h
@@ -16,7 +16,9 @@
  *                         reserved.
  * Copyright (c) 2017      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2020      Google, LLC. All rights reserved.
+ * Copyright (c) 2020-2021 Google, LLC. All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -39,11 +41,6 @@
  * The following #defines will be true / false based on
  * assembly support:
  *
- *  - \c OPAL_HAVE_ATOMIC_MEM_BARRIER atomic memory barriers
- *  - \c OPAL_HAVE_ATOMIC_SPINLOCKS atomic spinlocks
- *  - \c OPAL_HAVE_ATOMIC_MATH_32 if 32 bit add/sub/compare-exchange can be done "atomicly"
- *  - \c OPAL_HAVE_ATOMIC_MATH_64 if 64 bit add/sub/compare-exchange can be done "atomicly"
- *
  * Note that for the Atomic math, atomic add/sub may be implemented as
  * C code using opal_atomic_compare_exchange.  The appearance of atomic
  * operation will be upheld in these cases.
@@ -52,167 +49,18 @@
 #ifndef OPAL_SYS_ATOMIC_H
 #define OPAL_SYS_ATOMIC_H 1
 
-#include "opal_config.h"
-
 #include <stdbool.h>
 
-#include "opal/sys/architecture.h"
+#include "opal/opal_portable_platform.h"
 #include "opal_stdatomic.h"
 
-/* do some quick #define cleanup in cases where we are doing
-   testing... */
-#ifdef OPAL_DISABLE_INLINE_ASM
-#undef OPAL_C_GCC_INLINE_ASSEMBLY
-#define OPAL_C_GCC_INLINE_ASSEMBLY 0
-#endif
-
-#if OPAL_ASSEMBLY_BUILTIN == OPAL_BUILTIN_C11 && !defined(__INTEL_COMPILER)
-
-#include "atomic_stdc.h"
-
-#else /* !OPAL_C_HAVE__ATOMIC */
-
-/* define OPAL_{GCC,DEC,XLC}_INLINE_ASSEMBLY based on the
-   OPAL_C_{GCC,DEC,XLC}_INLINE_ASSEMBLY defines and whether we
-   are in C or C++ */
-#if defined(c_plusplus) || defined(__cplusplus)
-/* We no longer support inline assembly for C++ as OPAL is a C-only interface */
-#define OPAL_GCC_INLINE_ASSEMBLY 0
-#else
-#define OPAL_GCC_INLINE_ASSEMBLY OPAL_C_GCC_INLINE_ASSEMBLY
-#endif
-
-
 BEGIN_C_DECLS
-/**********************************************************************
- *
- * Data structures for atomic ops
- *
- *********************************************************************/
-/**
- * Volatile lock object (with optional padding).
- *
- * \note The internals of the lock are included here, but should be
- * considered private.  The implementation currently in use may choose
- * to use an int or unsigned char as the lock value - the user is not
- * informed either way.
- */
-struct opal_atomic_lock_t {
-    union {
-        opal_atomic_int32_t lock;     /**< The lock address (an integer) */
-        volatile unsigned char sparc_lock; /**< The lock address on sparc */
-        char padding[sizeof(int)]; /**< Array for optional padding */
-    } u;
-};
-typedef struct opal_atomic_lock_t opal_atomic_lock_t;
 
 /**********************************************************************
  *
- * Set or unset these macros in the architecture-specific atomic.h
- * files if we need to specify them as inline or non-inline
+ * Memory Barriers
  *
  *********************************************************************/
-#if !OPAL_GCC_INLINE_ASSEMBLY
-#define OPAL_HAVE_INLINE_ATOMIC_MEM_BARRIER 0
-#define OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32 0
-#define OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64 0
-#define OPAL_HAVE_INLINE_ATOMIC_ADD_32 0
-#define OPAL_HAVE_INLINE_ATOMIC_AND_32 0
-#define OPAL_HAVE_INLINE_ATOMIC_OR_32 0
-#define OPAL_HAVE_INLINE_ATOMIC_XOR_32 0
-#define OPAL_HAVE_INLINE_ATOMIC_SUB_32 0
-#define OPAL_HAVE_INLINE_ATOMIC_ADD_64 0
-#define OPAL_HAVE_INLINE_ATOMIC_AND_64 0
-#define OPAL_HAVE_INLINE_ATOMIC_OR_64 0
-#define OPAL_HAVE_INLINE_ATOMIC_XOR_64 0
-#define OPAL_HAVE_INLINE_ATOMIC_SUB_64 0
-#define OPAL_HAVE_INLINE_ATOMIC_SWAP_32 0
-#define OPAL_HAVE_INLINE_ATOMIC_SWAP_64 0
-#else
-#define OPAL_HAVE_INLINE_ATOMIC_MEM_BARRIER 1
-#define OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32 1
-#define OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64 1
-#define OPAL_HAVE_INLINE_ATOMIC_ADD_32 1
-#define OPAL_HAVE_INLINE_ATOMIC_AND_32 1
-#define OPAL_HAVE_INLINE_ATOMIC_OR_32 1
-#define OPAL_HAVE_INLINE_ATOMIC_XOR_32 1
-#define OPAL_HAVE_INLINE_ATOMIC_SUB_32 1
-#define OPAL_HAVE_INLINE_ATOMIC_ADD_64 1
-#define OPAL_HAVE_INLINE_ATOMIC_AND_64 1
-#define OPAL_HAVE_INLINE_ATOMIC_OR_64 1
-#define OPAL_HAVE_INLINE_ATOMIC_XOR_64 1
-#define OPAL_HAVE_INLINE_ATOMIC_SUB_64 1
-#define OPAL_HAVE_INLINE_ATOMIC_SWAP_32 1
-#define OPAL_HAVE_INLINE_ATOMIC_SWAP_64 1
-#endif
-
-/**
- * Enumeration of lock states
- */
-enum {
-    OPAL_ATOMIC_LOCK_UNLOCKED = 0,
-    OPAL_ATOMIC_LOCK_LOCKED = 1
-};
-
-#define OPAL_ATOMIC_LOCK_INIT {.u = {.lock = OPAL_ATOMIC_LOCK_UNLOCKED}}
-
-/**********************************************************************
- *
- * Load the appropriate architecture files and set some reasonable
- * default values for our support
- *
- *********************************************************************/
-#if defined(DOXYGEN)
-/* don't include system-level gorp when generating doxygen files */
-#elif OPAL_ASSEMBLY_BUILTIN == OPAL_BUILTIN_GCC
-#include "opal/sys/gcc_builtin/atomic.h"
-#elif OPAL_ASSEMBLY_ARCH == OPAL_X86_64
-#include "opal/sys/x86_64/atomic.h"
-#elif OPAL_ASSEMBLY_ARCH == OPAL_ARM
-#include "opal/sys/arm/atomic.h"
-#elif OPAL_ASSEMBLY_ARCH == OPAL_ARM64
-#include "opal/sys/arm64/atomic.h"
-#elif OPAL_ASSEMBLY_ARCH == OPAL_IA32
-#include "opal/sys/ia32/atomic.h"
-#elif OPAL_ASSEMBLY_ARCH == OPAL_POWERPC32
-#include "opal/sys/powerpc/atomic.h"
-#elif OPAL_ASSEMBLY_ARCH == OPAL_POWERPC64
-#include "opal/sys/powerpc/atomic.h"
-#endif
-
-#ifndef DOXYGEN
-/* compare and set operations can't really be emulated from software,
-   so if these defines aren't already set, they should be set to 0
-   now */
-#ifndef OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
-#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 0
-#endif
-#ifndef OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
-#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 0
-#endif
-#ifndef OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128
-#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 0
-#endif
-#ifndef OPAL_HAVE_ATOMIC_LLSC_32
-#define OPAL_HAVE_ATOMIC_LLSC_32 0
-#endif
-#ifndef OPAL_HAVE_ATOMIC_LLSC_64
-#define OPAL_HAVE_ATOMIC_LLSC_64 0
-#endif
-#endif /* DOXYGEN */
-
-/**********************************************************************
- *
- * Memory Barriers - defined here if running doxygen or have barriers
- *                   but can't inline
- *
- *********************************************************************/
-#if !defined(OPAL_HAVE_ATOMIC_MEM_BARRIER) && !defined(DOXYGEN)
-/* no way to emulate in C code */
-#define OPAL_HAVE_ATOMIC_MEM_BARRIER 0
-#endif
-
-#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MEM_BARRIER
 /**
  * Memory barrier
  *
@@ -226,11 +74,7 @@ enum {
  * generally grinding the memory controller's performance.  Use only
  * if you need *both* read and write barriers.
  */
-
-#if OPAL_HAVE_INLINE_ATOMIC_MEM_BARRIER
-static inline
-#endif
-void opal_atomic_mb(void);
+static inline void opal_atomic_mb(void);
 
 /**
  * Read memory barrier
@@ -241,11 +85,7 @@ void opal_atomic_mb(void);
  * next read.  Nothing is said about the ordering of writes when using
  * \c opal_atomic_rmb().
  */
-
-#if OPAL_HAVE_INLINE_ATOMIC_MEM_BARRIER
-static inline
-#endif
-void opal_atomic_rmb(void);
+static inline void opal_atomic_rmb(void);
 
 /**
  * Write memory barrier.
@@ -256,145 +96,237 @@ void opal_atomic_rmb(void);
  * next write.  Nothing is said about the ordering of reads when using
  * \c opal_atomic_wmb().
  */
-
-#if OPAL_HAVE_INLINE_ATOMIC_MEM_BARRIER
-static inline
-#endif
-void opal_atomic_wmb(void);
-
-#endif /* defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MEM_BARRIER */
+static inline void opal_atomic_wmb(void);
 
 
 /**********************************************************************
  *
- * Atomic spinlocks - always inlined, if have atomic compare-and-swap
+ * Compare and Swap
+ *
+ * Implementations must provide 32 and 64 bit compare-and-swap
+ * operations, but may provide the ptr implementation by including
+ * atomic_cmpx_ptr_impl.h (which implements the ptr implementation
+ * over the 32 and 64 bit implementations).
  *
  *********************************************************************/
 
-#if !defined(OPAL_HAVE_ATOMIC_SPINLOCKS) && !defined(DOXYGEN)
-/* 0 is more like "pending" - we'll fix up at the end after all
-   the static inline functions are declared */
-#define OPAL_HAVE_ATOMIC_SPINLOCKS 0
-#endif
-
-#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_SPINLOCKS || (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
+/*
+ * The stdc implementation is implemetned as macros around the C11
+ * atomic interface (which is a type-independent interface).  While it
+ * would be better to have type checking so developers using the C11
+ * interface didn't accidentally munge something that broke on other
+ * implementations, there are a ton of warnings due to volatile casing
+ * in the opal_lifo code. Don't enforce the types of the function
+ * calls on C11 until we can sort that out.
+ */
+#if OPAL_USE_C11_ATOMICS == 0
 
 /**
- * Initialize a lock to value
+ * Atomic compare and set of 32 bit integers with acquire and release semantics.
  *
- * @param lock         Address of the lock
- * @param value        Initial value to set lock to
+ * @param addr          Address of value to be swapped
+ * @param oldval        Comparison value
+ * @param newval        New value to set if comparison is true
+ *
+ * @returns If newval was written into addr, the function returns
+ * true.  Otherwise, the function returns false and the value of addr
+ * at the time of the comparison is returned in oldval.
  */
-#if OPAL_HAVE_ATOMIC_SPINLOCKS == 0
-static inline
-#endif
-void opal_atomic_lock_init(opal_atomic_lock_t* lock, int32_t value);
-
+static inline bool opal_atomic_compare_exchange_strong_32(opal_atomic_int32_t *addr, int32_t *oldval,
+                                                          int32_t newval);
 
 /**
- * Try to acquire a lock.
+ * Atomic compare and set of 32 bit integers with acquire semantics.
  *
- * @param lock          Address of the lock.
- * @return              0 if the lock was acquired, 1 otherwise.
+ * @param addr          Address of value to be swapped
+ * @param oldval        Comparison value
+ * @param newval        New value to set if comparison is true
+ *
+ * @returns If newval was written into addr, the function returns
+ * true.  Otherwise, the function returns false and the value of addr
+ * at the time of the comparison is returned in oldval.
  */
-#if OPAL_HAVE_ATOMIC_SPINLOCKS == 0
-static inline
-#endif
-int opal_atomic_trylock(opal_atomic_lock_t *lock);
-
+static inline bool opal_atomic_compare_exchange_strong_acq_32(opal_atomic_int32_t *addr, int32_t *oldval,
+                                                              int32_t newval);
 
 /**
- * Acquire a lock by spinning.
+ * Atomic compare and set of 32 bit integers with release semantics.
  *
- * @param lock          Address of the lock.
+ * @param addr          Address of value to be swapped
+ * @param oldval        Comparison value
+ * @param newval        New value to set if comparison is true
+ *
+ * @returns If newval was written into addr, the function returns
+ * true.  Otherwise, the function returns false and the value of addr
+ * at the time of the comparison is returned in oldval.
  */
-#if OPAL_HAVE_ATOMIC_SPINLOCKS == 0
-static inline
-#endif
-void opal_atomic_lock(opal_atomic_lock_t *lock);
+static inline bool opal_atomic_compare_exchange_strong_rel_32(opal_atomic_int32_t *addr, int32_t *oldval,
+                                                              int32_t newval);
 
+/**
+ * Atomic compare and set of 64 bit integers with acquire and release semantics.
+ *
+ * @param addr          Address of value to be swapped
+ * @param oldval        Comparison value
+ * @param newval        New value to set if comparison is true
+ *
+ * @returns If newval was written into addr, the function returns
+ * true.  Otherwise, the function returns false and the value of addr
+ * at the time of the comparison is returned in oldval.
+ */
+static inline bool opal_atomic_compare_exchange_strong_64(opal_atomic_int64_t *addr, int64_t *oldval,
+                                                          int64_t newval);
 
 /**
- * Release a lock.
+ * Atomic compare and set of 64 bit integers with acquire semantics.
  *
- * @param lock          Address of the lock.
+ * @param addr          Address of value to be swapped
+ * @param oldval        Comparison value
+ * @param newval        New value to set if comparison is true
+ *
+ * @returns If newval was written into addr, the function returns
+ * true.  Otherwise, the function returns false and the value of addr
+ * at the time of the comparison is returned in oldval.
  */
-#if OPAL_HAVE_ATOMIC_SPINLOCKS == 0
-static inline
-#endif
-void opal_atomic_unlock(opal_atomic_lock_t *lock);
+static inline bool opal_atomic_compare_exchange_strong_acq_64(opal_atomic_int64_t *addr, int64_t *oldval,
+                                                              int64_t newval);
 
+/**
+ * Atomic compare and set of 64 bit integers with release semantics.
+ *
+ * @param addr          Address of value to be swapped
+ * @param oldval        Comparison value
+ * @param newval        New value to set if comparison is true
+ *
+ * @returns If newval was written into addr, the function returns
+ * true.  Otherwise, the function returns false and the value of addr
+ * at the time of the comparison is returned in oldval.
+ */
+static inline bool opal_atomic_compare_exchange_strong_rel_64(opal_atomic_int64_t *addr, int64_t *oldval,
+                                                              int64_t newval);
 
-#if OPAL_HAVE_ATOMIC_SPINLOCKS == 0
-#undef OPAL_HAVE_ATOMIC_SPINLOCKS
-#define OPAL_HAVE_ATOMIC_SPINLOCKS (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
-#define OPAL_NEED_INLINE_ATOMIC_SPINLOCKS 1
-#endif
+/**
+ * Atomic compare and set of pointer-sized integers with acquire and release semantics.
+ *
+ * @param addr          Address of value to be swapped
+ * @param oldval        Comparison value
+ * @param newval        New value to set if comparison is true
+ *
+ * @returns If newval was written into addr, the function returns
+ * true.  Otherwise, the function returns false and the value of addr
+ * at the time of the comparison is returned in oldval.
+ */
+static inline bool opal_atomic_compare_exchange_strong_ptr(opal_atomic_intptr_t *addr,
+                                                           intptr_t *oldval, intptr_t newval);
 
-#endif /* OPAL_HAVE_ATOMIC_SPINLOCKS */
+/**
+ * Atomic compare and set of pointer-sized integers with acquire semantics.
+ *
+ * @param addr          Address of value to be swapped
+ * @param oldval        Comparison value
+ * @param newval        New value to set if comparison is true
+ *
+ * @returns If newval was written into addr, the function returns
+ * true.  Otherwise, the function returns false and the value of addr
+ * at the time of the comparison is returned in oldval.
+ */
+static inline bool opal_atomic_compare_exchange_strong_acq_ptr(opal_atomic_intptr_t *addr,
+                                                               intptr_t *oldval, intptr_t newval);
 
+/**
+ * Atomic compare and set of pointer-sized integers with release semantics.
+ *
+ * @param addr          Address of value to be swapped
+ * @param oldval        Comparison value
+ * @param newval        New value to set if comparison is true
+ *
+ * @returns If newval was written into addr, the function returns
+ * true.  Otherwise, the function returns false and the value of addr
+ * at the time of the comparison is returned in oldval.
+ */
+static inline bool opal_atomic_compare_exchange_strong_rel_ptr(opal_atomic_intptr_t *addr,
+                                                               intptr_t *oldval, intptr_t newval);
 
 /**********************************************************************
  *
- * Atomic math operations
+ * Swap
+ *
+ * Implementations may provide a native implementation of these
+ * operations or include atomic_swap_impl.h, which provides
+ * implementations over compare-and-swap.
  *
  *********************************************************************/
-#if !defined(OPAL_HAVE_ATOMIC_CMPSET_32) && !defined(DOXYGEN)
-#define OPAL_HAVE_ATOMIC_CMPSET_32 0
-#endif
-#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_CMPSET_32
-
-#if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32
-static inline
-#endif
-bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval,
-                                             int32_t newval);
+/**
+ * Atomic swap of 32 bit value
+ * @param addr           Address of value to be swapped
+ * @param newval         New value to set in addr
+ *
+ * @returns Value in addr before swap
+ */
+static inline int32_t opal_atomic_swap_32(opal_atomic_int32_t *addr, int32_t newval);
 
-#if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32
-static inline
-#endif
-bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval,
-                                                 int32_t newval);
+/**
+ * Atomic swap of 32 bit value
+ * @param addr           Address of value to be swapped
+ * @param newval         New value to set in addr
+ *
+ * @returns Value in addr before swap
+ */
+static inline int64_t opal_atomic_swap_64(opal_atomic_int64_t *addr, int64_t newval);
 
-#if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32
-static inline
-#endif
-bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval,
-                                                 int32_t newval);
-#endif
+/**
+ * Atomic swap of 32 bit value
+ * @param addr           Address of value to be swapped
+ * @param newval         New value to set in addr
+ *
+ * @returns Value in addr before swap
+ */
+static inline intptr_t opal_atomic_swap_ptr(opal_atomic_intptr_t *addr, intptr_t newval);
 
+#endif /* #if !(OPAL_ASSEMBLY_BUILTIN == OPAL_BUILTIN_C11 && !defined(__INTEL_COMPILER)) */
 
-#if !defined(OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64) && !defined(DOXYGEN)
-#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 0
-#endif
-#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
 
-#if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64
-static inline
-#endif
-bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval,
-                                             int64_t newval);
+/**********************************************************************
+ *
+ * Atomic spinlocks
+ *
+ *********************************************************************/
+/**
+ * Initialize a lock to value
+ *
+ * @param lock         Address of the lock
+ * @param value        Initial value to set lock to
+ */
+static inline void opal_atomic_lock_init(opal_atomic_lock_t *lock, int32_t value);
 
-#if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64
-static inline
-#endif
-bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval,
-                                                 int64_t newval);
+/**
+ * Try to acquire a lock.
+ *
+ * @param lock          Address of the lock.
+ * @return              0 if the lock was acquired, 1 otherwise.
+ */
+static inline int opal_atomic_trylock(opal_atomic_lock_t *lock);
 
-#if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64
-static inline
-#endif
-bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval,
-                                                 int64_t newval);
+/**
+ * Acquire a lock by spinning.
+ *
+ * @param lock          Address of the lock.
+ */
+static inline void opal_atomic_lock(opal_atomic_lock_t *lock);
 
-#endif
+/**
+ * Release a lock.
+ *
+ * @param lock          Address of the lock.
+ */
+static inline void opal_atomic_unlock(opal_atomic_lock_t *lock);
 
-#if !defined(OPAL_HAVE_ATOMIC_MATH_32) && !defined(DOXYGEN)
-  /* define to 0 for these tests.  WIll fix up later. */
-  #define OPAL_HAVE_ATOMIC_MATH_32 0
-#endif
 
-#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
+/**********************************************************************
+ *
+ * Atomic math operations
+ *
+ *********************************************************************/
 
 static inline int32_t opal_atomic_add_fetch_32(opal_atomic_int32_t *addr, int delta);
 static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t *addr, int delta);
@@ -406,25 +338,10 @@ static inline int32_t opal_atomic_xor_fetch_32(opal_atomic_int32_t *addr, int32_
 static inline int32_t opal_atomic_fetch_xor_32(opal_atomic_int32_t *addr, int32_t value);
 static inline int32_t opal_atomic_sub_fetch_32(opal_atomic_int32_t *addr, int delta);
 static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t *addr, int delta);
-static inline int32_t opal_atomic_min_fetch_32 (opal_atomic_int32_t *addr, int32_t value);
-static inline int32_t opal_atomic_fetch_min_32 (opal_atomic_int32_t *addr, int32_t value);
-static inline int32_t opal_atomic_max_fetch_32 (opal_atomic_int32_t *addr, int32_t value);
-static inline int32_t opal_atomic_fetch_max_32 (opal_atomic_int32_t *addr, int32_t value);
-
-#endif /* OPAL_HAVE_ATOMIC_MATH_32 */
-
-#if ! OPAL_HAVE_ATOMIC_MATH_32
-/* fix up the value of opal_have_atomic_math_32 to allow for C versions */
-#undef OPAL_HAVE_ATOMIC_MATH_32
-#define OPAL_HAVE_ATOMIC_MATH_32 OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
-#endif
-
-#ifndef OPAL_HAVE_ATOMIC_MATH_64
-/* define to 0 for these tests.  WIll fix up later. */
-#define OPAL_HAVE_ATOMIC_MATH_64 0
-#endif
-
-#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MATH_64 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
+static inline int32_t opal_atomic_min_fetch_32(opal_atomic_int32_t *addr, int32_t value);
+static inline int32_t opal_atomic_fetch_min_32(opal_atomic_int32_t *addr, int32_t value);
+static inline int32_t opal_atomic_max_fetch_32(opal_atomic_int32_t *addr, int32_t value);
+static inline int32_t opal_atomic_fetch_max_32(opal_atomic_int32_t *addr, int32_t value);
 
 static inline int64_t opal_atomic_add_fetch_64(opal_atomic_int64_t *addr, int64_t delta);
 static inline int64_t opal_atomic_fetch_add_64(opal_atomic_int64_t *addr, int64_t delta);
@@ -435,254 +352,116 @@ static inline int64_t opal_atomic_fetch_or_64(opal_atomic_int64_t *addr, int64_t
 static inline int64_t opal_atomic_fetch_xor_64(opal_atomic_int64_t *addr, int64_t value);
 static inline int64_t opal_atomic_sub_fetch_64(opal_atomic_int64_t *addr, int64_t delta);
 static inline int64_t opal_atomic_fetch_sub_64(opal_atomic_int64_t *addr, int64_t delta);
-static inline int64_t opal_atomic_min_fetch_64 (opal_atomic_int64_t *addr, int64_t value);
-static inline int64_t opal_atomic_fetch_min_64 (opal_atomic_int64_t *addr, int64_t value);
-static inline int64_t opal_atomic_max_fetch_64 (opal_atomic_int64_t *addr, int64_t value);
-static inline int64_t opal_atomic_fetch_max_64 (opal_atomic_int64_t *addr, int64_t value);
-
-#endif /* OPAL_HAVE_ATOMIC_MATH_64 */
-
-#if ! OPAL_HAVE_ATOMIC_MATH_64
-/* fix up the value of opal_have_atomic_math_64 to allow for C versions */
-#undef OPAL_HAVE_ATOMIC_MATH_64
-#define OPAL_HAVE_ATOMIC_MATH_64 OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
-#endif
-
-/* provide a size_t add/subtract.  When in debug mode, make it an
- * inline function so that we don't have any casts in the
- *  interface and can catch type errors.  When not in debug mode,
- * just make it a macro, so that there's no performance penalty
- */
-#if defined(DOXYGEN) || OPAL_ENABLE_DEBUG
-static inline size_t
-opal_atomic_add_fetch_size_t(opal_atomic_size_t *addr, size_t delta)
-{
-#if SIZEOF_SIZE_T == 4
-    return (size_t) opal_atomic_add_fetch_32((int32_t*) addr, delta);
-#elif SIZEOF_SIZE_T == 8
-    return (size_t) opal_atomic_add_fetch_64((int64_t*) addr, delta);
-#else
-#error "Unknown size_t size"
-#endif
-}
-
-static inline size_t
-opal_atomic_fetch_add_size_t(opal_atomic_size_t *addr, size_t delta)
-{
-#if SIZEOF_SIZE_T == 4
-    return (size_t) opal_atomic_fetch_add_32((int32_t*) addr, delta);
-#elif SIZEOF_SIZE_T == 8
-    return (size_t) opal_atomic_fetch_add_64((int64_t*) addr, delta);
-#else
-#error "Unknown size_t size"
-#endif
-}
-
-static inline size_t
-opal_atomic_sub_fetch_size_t(opal_atomic_size_t *addr, size_t delta)
-{
-#if SIZEOF_SIZE_T == 4
-    return (size_t) opal_atomic_sub_fetch_32((int32_t*) addr, delta);
-#elif SIZEOF_SIZE_T == 8
-    return (size_t) opal_atomic_sub_fetch_64((int64_t*) addr, delta);
-#else
-#error "Unknown size_t size"
-#endif
-}
-
-static inline size_t
-opal_atomic_fetch_sub_size_t(opal_atomic_size_t *addr, size_t delta)
-{
-#if SIZEOF_SIZE_T == 4
-    return (size_t) opal_atomic_fetch_sub_32((int32_t*) addr, delta);
-#elif SIZEOF_SIZE_T == 8
-    return (size_t) opal_atomic_fetch_sub_64((int64_t*) addr, delta);
-#else
-#error "Unknown size_t size"
-#endif
-}
-
-#else
-#if SIZEOF_SIZE_T == 4
-#define opal_atomic_add_fetch_size_t(addr, delta) ((size_t) opal_atomic_add_fetch_32((opal_atomic_int32_t *) addr, delta))
-#define opal_atomic_fetch_add_size_t(addr, delta) ((size_t) opal_atomic_fetch_add_32((opal_atomic_int32_t *) addr, delta))
-#define opal_atomic_sub_fetch_size_t(addr, delta) ((size_t) opal_atomic_sub_fetch_32((opal_atomic_int32_t *) addr, delta))
-#define opal_atomic_fetch_sub_size_t(addr, delta) ((size_t) opal_atomic_fetch_sub_32((opal_atomic_int32_t *) addr, delta))
-#elif SIZEOF_SIZE_T == 8
-#define opal_atomic_add_fetch_size_t(addr, delta) ((size_t) opal_atomic_add_fetch_64((opal_atomic_int64_t *) addr, delta))
-#define opal_atomic_fetch_add_size_t(addr, delta) ((size_t) opal_atomic_fetch_add_64((opal_atomic_int64_t *) addr, delta))
-#define opal_atomic_sub_fetch_size_t(addr, delta) ((size_t) opal_atomic_sub_fetch_64((opal_atomic_int64_t *) addr, delta))
-#define opal_atomic_fetch_sub_size_t(addr, delta) ((size_t) opal_atomic_fetch_sub_64((opal_atomic_int64_t *) addr, delta))
-#else
-#error "Unknown size_t size"
-#endif
-#endif
-
-#if defined(DOXYGEN) || (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
-/* these are always done with inline functions, so always mark as
-   static inline */
-
-static inline bool opal_atomic_compare_exchange_strong_xx (opal_atomic_intptr_t *addr, intptr_t *oldval,
-                                                           int64_t newval, size_t length);
-static inline bool opal_atomic_compare_exchange_strong_acq_xx (opal_atomic_intptr_t *addr, intptr_t *oldval,
-                                                               int64_t newval, size_t length);
-static inline bool opal_atomic_compare_exchange_strong_rel_xx (opal_atomic_intptr_t *addr, intptr_t *oldval,
-                                                               int64_t newval, size_t length);
-
-
-static inline bool opal_atomic_compare_exchange_strong_ptr (opal_atomic_intptr_t* addr, intptr_t *oldval,
-                                                            intptr_t newval);
-static inline bool opal_atomic_compare_exchange_strong_acq_ptr (opal_atomic_intptr_t* addr, intptr_t *oldval,
-                                                                intptr_t newval);
-static inline bool opal_atomic_compare_exchange_strong_rel_ptr (opal_atomic_intptr_t* addr, intptr_t *oldval,
-                                                                intptr_t newval);
+static inline int64_t opal_atomic_min_fetch_64(opal_atomic_int64_t *addr, int64_t value);
+static inline int64_t opal_atomic_fetch_min_64(opal_atomic_int64_t *addr, int64_t value);
+static inline int64_t opal_atomic_max_fetch_64(opal_atomic_int64_t *addr, int64_t value);
+static inline int64_t opal_atomic_fetch_max_64(opal_atomic_int64_t *addr, int64_t value);
 
-/**
- * Atomic compare and set of generic type with relaxed semantics. This
- * macro detect at compile time the type of the first argument and
- * choose the correct function to be called.
- *
- * \note This macro should only be used for integer types.
- *
- * @param addr          Address of <TYPE>.
- * @param oldval        Comparison value address of <TYPE>.
- * @param newval        New value to set if comparision is true <TYPE>.
- *
- * See opal_atomic_compare_exchange_* for pseudo-code.
- */
-#define opal_atomic_compare_exchange_strong( ADDR, OLDVAL, NEWVAL )                  \
-    opal_atomic_compare_exchange_strong_xx( (opal_atomic_intptr_t*)(ADDR), (intptr_t *)(OLDVAL), \
-                                            (intptr_t)(NEWVAL), sizeof(*(ADDR)) )
-
-/**
- * Atomic compare and set of generic type with acquire semantics. This
- * macro detect at compile time the type of the first argument and
- * choose the correct function to be called.
- *
- * \note This macro should only be used for integer types.
- *
- * @param addr          Address of <TYPE>.
- * @param oldval        Comparison value address of <TYPE>.
- * @param newval        New value to set if comparision is true <TYPE>.
- *
- * See opal_atomic_compare_exchange_acq_* for pseudo-code.
- */
-#define opal_atomic_compare_exchange_strong_acq( ADDR, OLDVAL, NEWVAL )                  \
-    opal_atomic_compare_exchange_strong_acq_xx( (opal_atomic_intptr_t*)(ADDR), (intptr_t *)(OLDVAL), \
-                                                (intptr_t)(NEWVAL), sizeof(*(ADDR)) )
+static inline size_t opal_atomic_add_fetch_size_t(opal_atomic_size_t *addr, size_t delta);
+static inline size_t opal_atomic_fetch_add_size_t(opal_atomic_size_t *addr, size_t delta);
 
+#ifdef DOXYGEN /* because this isn't a proper C prototype */
 /**
- * Atomic compare and set of generic type with release semantics. This
- * macro detect at compile time the type of the first argument and
- * choose the correct function to be called.
+ * Atomically add delta to addr, type independent
  *
- * \note This macro should only be used for integer types.
+ * @param addr   Address of value to update
+ * @param delta  Value by which to change the value in addr
  *
- * @param addr          Address of <TYPE>.
- * @param oldval        Comparison value address of <TYPE>.
- * @param newval        New value to set if comparision is true <TYPE>.
- *
- * See opal_atomic_compare_exchange_rel_* for pseudo-code.
+ * Generally implemented as a macro (except for when implemented as a
+ * compiler built-in), this function provides a type-independent math
+ * operator.
  */
-#define opal_atomic_compare_exchange_strong_rel( ADDR, OLDVAL, NEWVAL ) \
-    opal_atomic_compare_exchange_strong_rel_xx( (opal_atomic_intptr_t*)(ADDR), (intptr_t *)(OLDVAL), \
-                                                (intptr_t)(NEWVAL), sizeof(*(ADDR)) )
-
-
-#endif /* (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64) */
-
-#if defined(DOXYGEN) || (OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64)
-
-static inline void opal_atomic_add_xx(opal_atomic_intptr_t* addr,
-                                      int32_t value, size_t length);
-static inline void opal_atomic_sub_xx(opal_atomic_intptr_t* addr,
-                                      int32_t value, size_t length);
+static inline void opal_atomic_add(type *addr, type delta);
+#endif
 
-static inline intptr_t opal_atomic_add_fetch_ptr( opal_atomic_intptr_t* addr, void* delta );
-static inline intptr_t opal_atomic_fetch_add_ptr( opal_atomic_intptr_t* addr, void* delta );
-static inline intptr_t opal_atomic_sub_fetch_ptr( opal_atomic_intptr_t* addr, void* delta );
-static inline intptr_t opal_atomic_fetch_sub_ptr( opal_atomic_intptr_t* addr, void* delta );
 
-/**
- * Atomically increment the content depending on the type. This
- * macro detect at compile time the type of the first argument
- * and choose the correct function to be called.
+/**********************************************************************
  *
- * \note This macro should only be used for integer types.
+ * Load-linked, Store Conditional
  *
- * @param addr          Address of <TYPE>
- * @param delta         Value to add (converted to <TYPE>).
- */
-#define opal_atomic_add( ADDR, VALUE )                                  \
-   opal_atomic_add_xx( (opal_atomic_intptr_t*)(ADDR), (int32_t)(VALUE), \
-                       sizeof(*(ADDR)) )
-
-/**
- * Atomically decrement the content depending on the type. This
- * macro detect at compile time the type of the first argument
- * and choose the correct function to be called.
+ * Optional.  Check OPAL_HAVE_ATOMIC_LLSC_32,
+ * OPAL_HAVE_ATOMIC_LLSC_64, or OPAL_HAVE_ATOMIC_LLSC_PTR before
+ * using.  Implemented as macros due to function call behaviors;
+ * prototyped here as C++-style functions for readability.
  *
- * \note This macro should only be used for integer types.
+ * C11 and GCC built-in atomics don't provide native LL/SC support, so
+ * if there is an architectural implementation, we use it even if
+ * we are using the C11 or GCC built-in atomics.
  *
- * @param addr          Address of <TYPE>
- * @param delta         Value to substract (converted to <TYPE>).
- */
-#define opal_atomic_sub( ADDR, VALUE )                                  \
-   opal_atomic_sub_xx( (opal_atomic_intptr_t*)(ADDR), (int32_t)(VALUE),        \
-                      sizeof(*(ADDR)) )
+ *********************************************************************/
 
-#endif /* OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64 */
+#ifdef DOXYGEN
 
+static inline void opal_atomic_ll_32(opal_atomic_int32_t *addr, int32_t &ret);
 
-/*
- * Include inline implementations of everything not defined directly
- * in assembly
- */
-#include "opal/sys/atomic_impl.h"
+static inline void opal_atomic_sc_32(opal_atomic_int32_t *addr, int32_t newval, int &ret);
 
-#endif /* !OPAL_C_HAVE__ATOMIC */
+static inline void opal_atomic_ll_64(opal_atomic_int64_t *addr, int64_t &ret);
 
-/****** load-linked, store-conditional atomic implementations ******/
+static inline void opal_atomic_sc_64(opal_atomic_int64_t *addr, int64_t newval, int &ret);
 
-/* C11 atomics do not expose the low-level load-linked, store-conditional
- * instructions. Open MPI can use these instructions to implement a more
- * efficient version of the lock-free lifo and fifo. On Apple Silicon the
- * LL/SC fifo and lifo are ~ 2-20x faster than the CAS128 implementation. */
-#if OPAL_ASSEMBLY_ARCH == OPAL_ARM64
-#include "opal/sys/arm64/atomic_llsc.h"
-#endif
+static inline void opal_atomic_ll_ptr(opal_atomic_intptr_t *addr, intptr_t &ret);
 
-#if !defined(OPAL_HAVE_ATOMIC_LLSC_32)
-#define OPAL_HAVE_ATOMIC_LLSC_32 0
-#endif
+static inline void opal_atomic_sc_ptr(opal_atomic_intptr_t *addr, intptr_t newval, int &ret);
 
-#if !defined(OPAL_HAVE_ATOMIC_LLSC_64)
-#define OPAL_HAVE_ATOMIC_LLSC_64 0
 #endif
 
-#if (OPAL_HAVE_ATOMIC_LLSC_32 || OPAL_HAVE_ATOMIC_LLSC_64)
 
-#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_LLSC_32
-
-#define opal_atomic_ll_ptr(addr, ret) opal_atomic_ll_32((opal_atomic_int32_t *) (addr), ret)
-#define opal_atomic_sc_ptr(addr, value, ret) opal_atomic_sc_32((opal_atomic_int32_t *) (addr), (intptr_t) (value), ret)
+/**********************************************************************
+ *
+ * Load the appropriate architecture files and set some reasonable
+ * default values for our support
+ *
+ *********************************************************************/
 
-#define OPAL_HAVE_ATOMIC_LLSC_PTR 1
+#if defined(DOXYGEN)
+/* don't include system-level gorp when generating doxygen files */
+#elif OPAL_USE_C11_ATOMICS == 1
+#    include "opal/sys/atomic_stdc.h"
+#elif OPAL_USE_GCC_BUILTIN_ATOMICS == 1
+#    include "opal/sys/gcc_builtin/atomic.h"
+#elif OPAL_USE_ASM_ATOMICS == 1
+#    if defined(PLATFORM_ARCH_X86_64)
+#        include "opal/sys/x86_64/atomic.h"
+#    elif defined(PLATFORM_ARCH_AARCH64)
+#        include "opal/sys/arm64/atomic.h"
+#    elif defined(PLATFORM_ARCH_POWERPC) && defined(PLATFORM_ARCH_64)
+#        include "opal/sys/powerpc/atomic.h"
+#    else
+#        error "No asm support found."
+#    endif
+#else
+#error "No atomics support found."
+#endif
 
-#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_LLSC_64
+#if defined(PLATFORM_ARCH_AARCH64)
+#    include "opal/sys/arm64/atomic_llsc.h"
+#elif defined(PLATFORM_ARCH_POWERPC) && defined(PLATFORM_ARCH_64)
+#    include "opal/sys/powerpc/atomic_llsc.h"
+#endif
 
-#define opal_atomic_ll_ptr(addr, ret) opal_atomic_ll_64((opal_atomic_int64_t *) (addr), ret)
-#define opal_atomic_sc_ptr(addr, value, ret) opal_atomic_sc_64((opal_atomic_int64_t *) (addr), (intptr_t) (value), ret)
 
-#define OPAL_HAVE_ATOMIC_LLSC_PTR 1
+/**********************************************************************
+ *
+ * Ensure defines for the few optional features are always defined
+ *
+ *********************************************************************/
 
+#ifndef OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128
+#    define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 0
 #endif
 
-#else
+#ifndef OPAL_HAVE_ATOMIC_LLSC_32
+#    define OPAL_HAVE_ATOMIC_LLSC_32 0
+#endif
 
-#define OPAL_HAVE_ATOMIC_LLSC_PTR 0
+#ifndef OPAL_HAVE_ATOMIC_LLSC_64
+#    define OPAL_HAVE_ATOMIC_LLSC_64 0
+#endif
 
-#endif /* (OPAL_HAVE_ATOMIC_LLSC_32 || OPAL_HAVE_ATOMIC_LLSC_64)*/
+#ifndef OPAL_HAVE_ATOMIC_LLSC_PTR
+#    define OPAL_HAVE_ATOMIC_LLSC_PTR 0
+#endif
 
 END_C_DECLS
 
diff --git a/opal/include/opal/sys/atomic_impl.h b/opal/include/opal/sys/atomic_impl.h
deleted file mode 100644
index 9df747eede7..00000000000
--- a/opal/include/opal/sys/atomic_impl.h
+++ /dev/null
@@ -1,497 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2014 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-/* Inline C implementation of the functions defined in atomic.h */
-
-#include <stdlib.h>
-
-/**********************************************************************
- *
- * Atomic math operations
- *
- * All the architectures provide a compare_and_set atomic operations. If
- * they dont provide atomic additions and/or substractions then we can
- * define these operations using the atomic compare_and_set.
- *
- * Some architectures do not provide support for the 64 bits
- * atomic operations. Until we find a better solution let's just
- * undefine all those functions if there is no 64 bit compare-exchange
- *
- *********************************************************************/
-#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
-
-#if !defined(OPAL_HAVE_ATOMIC_MIN_32)
-static inline int32_t opal_atomic_fetch_min_32 (opal_atomic_int32_t *addr, int32_t value)
-{
-    int32_t old = *addr;
-    do {
-        if (old <= value) {
-            break;
-        }
-    } while (!opal_atomic_compare_exchange_strong_32 (addr, &old, value));
-
-    return old;
-}
-
-#define OPAL_HAVE_ATOMIC_MIN_32 1
-
-#endif /* OPAL_HAVE_ATOMIC_MIN_32 */
-
-#if !defined(OPAL_HAVE_ATOMIC_MAX_32)
-static inline int32_t opal_atomic_fetch_max_32 (opal_atomic_int32_t *addr, int32_t value)
-{
-    int32_t old = *addr;
-    do {
-        if (old >= value) {
-            break;
-        }
-    } while (!opal_atomic_compare_exchange_strong_32 (addr, &old, value));
-
-    return old;
-}
-
-#define OPAL_HAVE_ATOMIC_MAX_32 1
-#endif /* OPAL_HAVE_ATOMIC_MAX_32 */
-
-#define OPAL_ATOMIC_DEFINE_CMPXCG_OP(type, bits, operation, name)  \
-    static inline type opal_atomic_fetch_ ## name ## _ ## bits (opal_atomic_ ## type *addr, type value) \
-    {                                                                   \
-        type oldval;                                                    \
-        do {                                                            \
-            oldval = *addr;                                             \
-        } while (!opal_atomic_compare_exchange_strong_ ## bits (addr, &oldval, oldval operation value)); \
-                                                                        \
-        return oldval;                                                  \
-    }
-
-#if !defined(OPAL_HAVE_ATOMIC_SWAP_32)
-#define OPAL_HAVE_ATOMIC_SWAP_32 1
-static inline int32_t opal_atomic_swap_32(opal_atomic_int32_t *addr,
-                                          int32_t newval)
-{
-    int32_t old = *addr;
-    do {
-    } while (!opal_atomic_compare_exchange_strong_32 (addr, &old, newval));
-
-    return old;
-}
-#endif /* OPAL_HAVE_ATOMIC_SWAP_32 */
-
-#if !defined(OPAL_HAVE_ATOMIC_ADD_32)
-#define OPAL_HAVE_ATOMIC_ADD_32 1
-
-OPAL_ATOMIC_DEFINE_CMPXCG_OP(int32_t, 32, +, add)
-
-#endif  /* OPAL_HAVE_ATOMIC_ADD_32 */
-
-#if !defined(OPAL_HAVE_ATOMIC_AND_32)
-#define OPAL_HAVE_ATOMIC_AND_32 1
-
-OPAL_ATOMIC_DEFINE_CMPXCG_OP(int32_t, 32, &, and)
-
-#endif  /* OPAL_HAVE_ATOMIC_AND_32 */
-
-#if !defined(OPAL_HAVE_ATOMIC_OR_32)
-#define OPAL_HAVE_ATOMIC_OR_32 1
-
-OPAL_ATOMIC_DEFINE_CMPXCG_OP(int32_t, 32, |, or)
-
-#endif  /* OPAL_HAVE_ATOMIC_OR_32 */
-
-#if !defined(OPAL_HAVE_ATOMIC_XOR_32)
-#define OPAL_HAVE_ATOMIC_XOR_32 1
-
-OPAL_ATOMIC_DEFINE_CMPXCG_OP(int32_t, 32, ^, xor)
-
-#endif  /* OPAL_HAVE_ATOMIC_XOR_32 */
-
-
-#if !defined(OPAL_HAVE_ATOMIC_SUB_32)
-#define OPAL_HAVE_ATOMIC_SUB_32 1
-
-OPAL_ATOMIC_DEFINE_CMPXCG_OP(int32_t, 32, -, sub)
-
-#endif  /* OPAL_HAVE_ATOMIC_SUB_32 */
-
-#endif /* OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 */
-
-
-#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
-
-#if !defined(OPAL_HAVE_ATOMIC_MIN_64)
-static inline int64_t opal_atomic_fetch_min_64 (opal_atomic_int64_t *addr, int64_t value)
-{
-    int64_t old = *addr;
-    do {
-        if (old <= value) {
-            break;
-        }
-    } while (!opal_atomic_compare_exchange_strong_64 (addr, &old, value));
-
-    return old;
-}
-
-#define OPAL_HAVE_ATOMIC_MIN_64 1
-
-#endif /* OPAL_HAVE_ATOMIC_MIN_64 */
-
-#if !defined(OPAL_HAVE_ATOMIC_MAX_64)
-static inline int64_t opal_atomic_fetch_max_64 (opal_atomic_int64_t *addr, int64_t value)
-{
-    int64_t old = *addr;
-    do {
-        if (old >= value) {
-            break;
-        }
-    } while (!opal_atomic_compare_exchange_strong_64 (addr, &old, value));
-
-    return old;
-}
-
-#define OPAL_HAVE_ATOMIC_MAX_64 1
-#endif /* OPAL_HAVE_ATOMIC_MAX_64 */
-
-#if !defined(OPAL_HAVE_ATOMIC_SWAP_64)
-#define OPAL_HAVE_ATOMIC_SWAP_64 1
-static inline int64_t opal_atomic_swap_64(opal_atomic_int64_t *addr,
-                                          int64_t newval)
-{
-    int64_t old = *addr;
-    do {
-    } while (!opal_atomic_compare_exchange_strong_64 (addr, &old, newval));
-
-    return old;
-}
-#endif /* OPAL_HAVE_ATOMIC_SWAP_64 */
-
-#if !defined(OPAL_HAVE_ATOMIC_ADD_64)
-#define OPAL_HAVE_ATOMIC_ADD_64 1
-
-OPAL_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, +, add)
-
-#endif  /* OPAL_HAVE_ATOMIC_ADD_64 */
-
-#if !defined(OPAL_HAVE_ATOMIC_AND_64)
-#define OPAL_HAVE_ATOMIC_AND_64 1
-
-OPAL_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, &, and)
-
-#endif  /* OPAL_HAVE_ATOMIC_AND_64 */
-
-#if !defined(OPAL_HAVE_ATOMIC_OR_64)
-#define OPAL_HAVE_ATOMIC_OR_64 1
-
-OPAL_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, |, or)
-
-#endif  /* OPAL_HAVE_ATOMIC_OR_64 */
-
-#if !defined(OPAL_HAVE_ATOMIC_XOR_64)
-#define OPAL_HAVE_ATOMIC_XOR_64 1
-
-OPAL_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, ^, xor)
-
-#endif  /* OPAL_HAVE_ATOMIC_XOR_64 */
-
-#if !defined(OPAL_HAVE_ATOMIC_SUB_64)
-#define OPAL_HAVE_ATOMIC_SUB_64 1
-
-OPAL_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, -, sub)
-
-#endif  /* OPAL_HAVE_ATOMIC_SUB_64 */
-
-#else
-
-#if !defined(OPAL_HAVE_ATOMIC_ADD_64)
-#define OPAL_HAVE_ATOMIC_ADD_64 0
-#endif
-
-#if !defined(OPAL_HAVE_ATOMIC_SUB_64)
-#define OPAL_HAVE_ATOMIC_SUB_64 0
-#endif
-
-#endif  /* OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 */
-
-#if (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
-
-#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
-#define OPAL_ATOMIC_DEFINE_CMPXCG_XX(semantics)                         \
-    static inline bool                                                  \
-    opal_atomic_compare_exchange_strong ## semantics ## xx (opal_atomic_intptr_t* addr, intptr_t *oldval, \
-                                                            int64_t newval, const size_t length) \
-    {                                                                   \
-        switch (length) {                                               \
-        case 4:                                                         \
-            return opal_atomic_compare_exchange_strong_32 ((opal_atomic_int32_t *) addr, \
-                                                           (int32_t *) oldval, (int32_t) newval); \
-        case 8:                                                         \
-            return opal_atomic_compare_exchange_strong_64 ((opal_atomic_int64_t *) addr, \
-                                                           (int64_t *) oldval, (int64_t) newval); \
-        }                                                               \
-        abort();                                                        \
-    }
-#elif OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
-#define OPAL_ATOMIC_DEFINE_CMPXCG_XX(semantics)                         \
-    static inline bool                                                  \
-    opal_atomic_compare_exchange_strong ## semantics ## xx (opal_atomic_intptr_t* addr, intptr_t *oldval, \
-                                                            int64_t newval, const size_t length) \
-    {                                                                   \
-        switch (length) {                                               \
-        case 4:                                                         \
-            return opal_atomic_compare_exchange_strong_32 ((opal_atomic_int32_t *) addr, \
-                                                           (int32_t *) oldval, (int32_t) newval); \
-        }                                                               \
-        abort();                                                        \
-    }
-#else
-#error "Platform does not have required atomic compare-and-swap functionality"
-#endif
-
-OPAL_ATOMIC_DEFINE_CMPXCG_XX(_)
-OPAL_ATOMIC_DEFINE_CMPXCG_XX(_acq_)
-OPAL_ATOMIC_DEFINE_CMPXCG_XX(_rel_)
-
-#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
-#define OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(semantics)                     \
-    static inline bool                                                  \
-        opal_atomic_compare_exchange_strong ## semantics ## ptr (opal_atomic_intptr_t* addr, intptr_t *oldval, intptr_t newval) \
-    {                                                                   \
-        return opal_atomic_compare_exchange_strong_32 ((opal_atomic_int32_t *) addr, (int32_t *) oldval, (int32_t) newval); \
-    }
-#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
-#define OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(semantics)                     \
-    static inline bool                                                  \
-        opal_atomic_compare_exchange_strong ## semantics ## ptr (opal_atomic_intptr_t* addr, intptr_t *oldval, intptr_t newval) \
-    {                                                                   \
-        return opal_atomic_compare_exchange_strong_64 ((opal_atomic_int64_t *) addr, (int64_t *) oldval, (int64_t) newval); \
-    }
-#else
-#error "Can not define opal_atomic_compare_exchange_strong_ptr with existing atomics"
-#endif
-
-OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(_)
-OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(_acq_)
-OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(_rel_)
-
-#endif /* (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64) */
-
-
-#if (OPAL_HAVE_ATOMIC_SWAP_32 || OPAL_HAVE_ATOMIC_SWAP_64)
-
-#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_SWAP_32
-#define opal_atomic_swap_ptr(addr, value) (intptr_t) opal_atomic_swap_32((opal_atomic_int32_t *) addr, (int32_t) value)
-#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_SWAP_64
-#define opal_atomic_swap_ptr(addr, value) (intptr_t) opal_atomic_swap_64((opal_atomic_int64_t *) addr, (int64_t) value)
-#endif
-
-#endif /* (OPAL_HAVE_ATOMIC_SWAP_32 || OPAL_HAVE_ATOMIC_SWAP_64) */
-
-#if OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64
-
-static inline void
-    opal_atomic_add_xx(opal_atomic_intptr_t* addr, int32_t value, size_t length)
-{
-   switch( length ) {
-#if OPAL_HAVE_ATOMIC_ADD_32
-   case 4:
-       (void) opal_atomic_fetch_add_32( (opal_atomic_int32_t*)addr, (int32_t)value );
-      break;
-#endif  /* OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 */
-
-#if OPAL_HAVE_ATOMIC_ADD_64
-   case 8:
-       (void) opal_atomic_fetch_add_64( (opal_atomic_int64_t*)addr, (int64_t)value );
-      break;
-#endif  /* OPAL_HAVE_ATOMIC_ADD_64 */
-   default:
-       /* This should never happen, so deliberately abort (hopefully
-          leaving a corefile for analysis) */
-       abort();
-   }
-}
-
-
-static inline void
-opal_atomic_sub_xx(opal_atomic_intptr_t* addr, int32_t value, size_t length)
-{
-   switch( length ) {
-#if OPAL_HAVE_ATOMIC_SUB_32
-   case 4:
-       (void) opal_atomic_fetch_sub_32( (opal_atomic_int32_t*)addr, (int32_t)value );
-      break;
-#endif  /* OPAL_HAVE_ATOMIC_SUB_32 */
-
-#if OPAL_HAVE_ATOMIC_SUB_64
-   case 8:
-       (void) opal_atomic_fetch_sub_64( (opal_atomic_int64_t*)addr, (int64_t)value );
-      break;
-#endif  /* OPAL_HAVE_ATOMIC_SUB_64 */
-   default:
-       /* This should never happen, so deliberately abort (hopefully
-          leaving a corefile for analysis) */
-       abort();
-   }
-}
-
-#define OPAL_ATOMIC_DEFINE_OP_FETCH(op, operation, type, ptr_type, suffix) \
-    static inline type opal_atomic_ ## op ## _fetch_ ## suffix (opal_atomic_ ## ptr_type *addr, type value) \
-    {                                                                   \
-        return opal_atomic_fetch_ ## op ## _ ## suffix (addr, value) operation value; \
-    }
-
-OPAL_ATOMIC_DEFINE_OP_FETCH(add, +, int32_t, int32_t, 32)
-OPAL_ATOMIC_DEFINE_OP_FETCH(and, &, int32_t, int32_t, 32)
-OPAL_ATOMIC_DEFINE_OP_FETCH(or, |, int32_t, int32_t, 32)
-OPAL_ATOMIC_DEFINE_OP_FETCH(xor, ^, int32_t, int32_t, 32)
-OPAL_ATOMIC_DEFINE_OP_FETCH(sub, -, int32_t, int32_t, 32)
-
-static inline int32_t opal_atomic_min_fetch_32 (opal_atomic_int32_t *addr, int32_t value)
-{
-    int32_t old = opal_atomic_fetch_min_32 (addr, value);
-    return old <= value ? old : value;
-}
-
-static inline int32_t opal_atomic_max_fetch_32 (opal_atomic_int32_t *addr, int32_t value)
-{
-    int32_t old = opal_atomic_fetch_max_32 (addr, value);
-    return old >= value ? old : value;
-}
-
-#if OPAL_HAVE_ATOMIC_MATH_64
-OPAL_ATOMIC_DEFINE_OP_FETCH(add, +, int64_t, int64_t, 64)
-OPAL_ATOMIC_DEFINE_OP_FETCH(and, &, int64_t, int64_t, 64)
-OPAL_ATOMIC_DEFINE_OP_FETCH(or, |, int64_t, int64_t, 64)
-OPAL_ATOMIC_DEFINE_OP_FETCH(xor, ^, int64_t, int64_t, 64)
-OPAL_ATOMIC_DEFINE_OP_FETCH(sub, -, int64_t, int64_t, 64)
-
-static inline int64_t opal_atomic_min_fetch_64 (opal_atomic_int64_t *addr, int64_t value)
-{
-    int64_t old = opal_atomic_fetch_min_64 (addr, value);
-    return old <= value ? old : value;
-}
-
-static inline int64_t opal_atomic_max_fetch_64 (opal_atomic_int64_t *addr, int64_t value)
-{
-    int64_t old = opal_atomic_fetch_max_64 (addr, value);
-    return old >= value ? old : value;
-}
-
-#endif
-
-static inline intptr_t opal_atomic_fetch_add_ptr( opal_atomic_intptr_t* addr,
-                                           void* delta )
-{
-#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_ADD_32
-    return opal_atomic_fetch_add_32((opal_atomic_int32_t*) addr, (unsigned long) delta);
-#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_ADD_64
-    return opal_atomic_fetch_add_64((opal_atomic_int64_t*) addr, (unsigned long) delta);
-#else
-    abort ();
-    return 0;
-#endif
-}
-
-static inline intptr_t opal_atomic_add_fetch_ptr( opal_atomic_intptr_t* addr,
-                                           void* delta )
-{
-#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_ADD_32
-    return opal_atomic_add_fetch_32((opal_atomic_int32_t*) addr, (unsigned long) delta);
-#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_ADD_64
-    return opal_atomic_add_fetch_64((opal_atomic_int64_t*) addr, (unsigned long) delta);
-#else
-    abort ();
-    return 0;
-#endif
-}
-
-static inline intptr_t opal_atomic_fetch_sub_ptr( opal_atomic_intptr_t* addr,
-                                           void* delta )
-{
-#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_SUB_32
-    return opal_atomic_fetch_sub_32((opal_atomic_int32_t*) addr, (unsigned long) delta);
-#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_SUB_32
-    return opal_atomic_fetch_sub_64((opal_atomic_int64_t*) addr, (unsigned long) delta);
-#else
-    abort();
-    return 0;
-#endif
-}
-
-static inline intptr_t opal_atomic_sub_fetch_ptr( opal_atomic_intptr_t* addr,
-                                           void* delta )
-{
-#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_SUB_32
-    return opal_atomic_sub_fetch_32((opal_atomic_int32_t*) addr, (unsigned long) delta);
-#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_SUB_32
-    return opal_atomic_sub_fetch_64((opal_atomic_int64_t*) addr, (unsigned long) delta);
-#else
-    abort();
-    return 0;
-#endif
-}
-
-#endif /* OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64 */
-
-/**********************************************************************
- *
- * Atomic spinlocks
- *
- *********************************************************************/
-#ifdef OPAL_NEED_INLINE_ATOMIC_SPINLOCKS
-
-/*
- * Lock initialization function. It set the lock to UNLOCKED.
- */
-static inline void
-opal_atomic_lock_init( opal_atomic_lock_t* lock, int32_t value )
-{
-   lock->u.lock = value;
-}
-
-
-static inline int
-opal_atomic_trylock(opal_atomic_lock_t *lock)
-{
-    int32_t unlocked = OPAL_ATOMIC_LOCK_UNLOCKED;
-    bool ret = opal_atomic_compare_exchange_strong_acq_32 (&lock->u.lock, &unlocked, OPAL_ATOMIC_LOCK_LOCKED);
-    return (ret == false) ? 1 : 0;
-}
-
-
-static inline void
-opal_atomic_lock(opal_atomic_lock_t *lock)
-{
-    while (opal_atomic_trylock (lock)) {
-        while (lock->u.lock == OPAL_ATOMIC_LOCK_LOCKED) {
-            /* spin */ ;
-        }
-    }
-}
-
-
-static inline void
-opal_atomic_unlock(opal_atomic_lock_t *lock)
-{
-   opal_atomic_wmb();
-   lock->u.lock=OPAL_ATOMIC_LOCK_UNLOCKED;
-}
-
-#endif /* OPAL_HAVE_ATOMIC_SPINLOCKS */
diff --git a/opal/include/opal/sys/atomic_impl_math.h b/opal/include/opal/sys/atomic_impl_math.h
new file mode 100644
index 00000000000..7f48e50a23b
--- /dev/null
+++ b/opal/include/opal/sys/atomic_impl_math.h
@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2014 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ * Implementation of the required atomic math functions in terms of
+ * compare and swap operators.
+ */
+
+#ifndef ATOMIC_IMPL_MATH_H
+#define ATOMIC_IMPL_MATH_H 1
+
+#define OPAL_ATOMIC_DEFINE_OP(type, bits, operation, name)                                         \
+        static inline type opal_atomic_fetch_##name##_##bits(opal_atomic_##type *addr, type value) \
+        {                                                                                          \
+            type oldval;                                                                           \
+            do {                                                                                   \
+                oldval = *addr;                                                                    \
+            } while (!opal_atomic_compare_exchange_strong_##bits(addr, &oldval,                    \
+                                                                 oldval operation value));         \
+                                                                                                   \
+            return oldval;                                                                         \
+        }                                                                                          \
+                                                                                                   \
+        static inline type opal_atomic_##name##_fetch_##bits(opal_atomic_##type *addr, type value) \
+        {                                                                                          \
+            type oldval, newval;                                                                   \
+            do {                                                                                   \
+                oldval = *addr;                                                                    \
+                newval = oldval operation value;                                                   \
+            } while (!opal_atomic_compare_exchange_strong_##bits(addr, &oldval, newval);           \
+                                                                                                   \
+            return newval;                                                                         \
+        }
+
+OPAL_ATOMIC_DEFINE_OP(int32_t, 32, +, add)
+OPAL_ATOMIC_DEFINE_OP(int32_t, 32, &, and)
+OPAL_ATOMIC_DEFINE_OP(int32_t, 32, |, or)
+OPAL_ATOMIC_DEFINE_OP(int32_t, 32, ^, xor)
+OPAL_ATOMIC_DEFINE_OP(int32_t, 32, -, sub)
+
+OPAL_ATOMIC_DEFINE_OP(int64_t, 64, +, add)
+OPAL_ATOMIC_DEFINE_OP(int64_t, 64, &, and)
+OPAL_ATOMIC_DEFINE_OP(int64_t, 64, |, or)
+OPAL_ATOMIC_DEFINE_OP(int64_t, 64, ^, xor)
+OPAL_ATOMIC_DEFINE_OP(int64_t, 64, -, sub)
+
+#include "opal/sys/atomic_impl_minmax_math.h"
+
+#endif /* #ifndef ATOMIC_MATH_IMPL_H */
diff --git a/opal/include/opal/sys/atomic_impl_minmax_math.h b/opal/include/opal/sys/atomic_impl_minmax_math.h
new file mode 100644
index 00000000000..7ec5920ad46
--- /dev/null
+++ b/opal/include/opal/sys/atomic_impl_minmax_math.h
@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2014 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ * Implementation of the min/max atomic functions in terms of compare
+ * and swap.  These are broken out from the basic atomic_impl_math.h
+ * functions because most atomic implementations do not provide native
+ * min/max interfaces.
+ */
+
+#ifndef ATOMIC_IMPL_MINMAX_MATH_H
+#define ATOMIC_IMPL_MINMAX_MATH_H 1
+
+static inline int32_t opal_atomic_fetch_min_32(opal_atomic_int32_t *addr, int32_t value)
+{
+    int32_t old = *addr;
+    do {
+        if (old <= value) {
+            break;
+        }
+    } while (!opal_atomic_compare_exchange_strong_32(addr, &old, value));
+
+    return old;
+}
+
+static inline int32_t opal_atomic_min_fetch_32(opal_atomic_int32_t *addr, int32_t value)
+{
+    int32_t old = opal_atomic_fetch_min_32(addr, value);
+    return old <= value ? old : value;
+}
+
+static inline int32_t opal_atomic_fetch_max_32(opal_atomic_int32_t *addr, int32_t value)
+{
+    int32_t old = *addr;
+    do {
+        if (old >= value) {
+            break;
+        }
+    } while (!opal_atomic_compare_exchange_strong_32(addr, &old, value));
+
+    return old;
+}
+
+static inline int32_t opal_atomic_max_fetch_32(opal_atomic_int32_t *addr, int32_t value)
+{
+    int32_t old = opal_atomic_fetch_max_32(addr, value);
+    return old >= value ? old : value;
+}
+
+static inline int64_t opal_atomic_fetch_min_64(opal_atomic_int64_t *addr, int64_t value)
+{
+    int64_t old = *addr;
+    do {
+        if (old <= value) {
+            break;
+        }
+    } while (!opal_atomic_compare_exchange_strong_64(addr, &old, value));
+
+    return old;
+}
+
+static inline int64_t opal_atomic_fetch_max_64(opal_atomic_int64_t *addr, int64_t value)
+{
+    int64_t old = *addr;
+    do {
+        if (old >= value) {
+            break;
+        }
+    } while (!opal_atomic_compare_exchange_strong_64(addr, &old, value));
+
+    return old;
+}
+
+static inline int64_t opal_atomic_min_fetch_64(opal_atomic_int64_t *addr, int64_t value)
+{
+    int64_t old = opal_atomic_fetch_min_64(addr, value);
+    return old <= value ? old : value;
+}
+
+static inline int64_t opal_atomic_max_fetch_64(opal_atomic_int64_t *addr, int64_t value)
+{
+    int64_t old = opal_atomic_fetch_max_64(addr, value);
+    return old >= value ? old : value;
+}
+
+#endif /* #ifndef ATOMIC_MATH_MINMAX_IMPL_H */
diff --git a/opal/include/opal/sys/atomic_impl_ptr_cswap.h b/opal/include/opal/sys/atomic_impl_ptr_cswap.h
new file mode 100644
index 00000000000..a7246c04ee3
--- /dev/null
+++ b/opal/include/opal/sys/atomic_impl_ptr_cswap.h
@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2014 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef ATOMIC_IMPL_PTR_CSWAP_H
+#define ATOMIC_IMPL_PTR_CSWAP_H 1
+
+static inline bool opal_atomic_compare_exchange_strong_ptr(opal_atomic_intptr_t *addr,
+                                                           intptr_t *oldval, intptr_t newval)
+{
+#if SIZEOF_VOID_P == 4
+    return opal_atomic_compare_exchange_strong_32((opal_atomic_int32_t*)addr,
+                                                  (int32_t*)oldval,
+                                                  (int32_t)newval);
+#elif SIZEOF_VOID_P == 8
+    return opal_atomic_compare_exchange_strong_64((opal_atomic_int64_t*)addr,
+                                                  (int64_t*)oldval,
+                                                  (int64_t)newval);
+#else
+#error "No implementation of opal_atomic_compare_exchange_strong_ptr"
+#endif
+}
+
+static inline bool opal_atomic_compare_exchange_strong_acq_ptr(opal_atomic_intptr_t *addr,
+                                                               intptr_t *oldval, intptr_t newval)
+{
+#if SIZEOF_VOID_P == 4
+    return opal_atomic_compare_exchange_strong_acq_32((opal_atomic_int32_t*)addr,
+                                                      (int32_t*)oldval,
+                                                      (int32_t)newval);
+#elif SIZEOF_VOID_P == 8
+    return opal_atomic_compare_exchange_strong_acq_64((opal_atomic_int64_t*)addr,
+                                                      (int64_t*)oldval,
+                                                      (int64_t)newval);
+#else
+#error "No implementation of opal_atomic_compare_exchange_strong_acq_ptr"
+#endif
+}
+
+static inline bool opal_atomic_compare_exchange_strong_rel_ptr(opal_atomic_intptr_t *addr,
+                                                               intptr_t *oldval, intptr_t newval)
+{
+#if SIZEOF_VOID_P == 4
+    return opal_atomic_compare_exchange_strong_rel_32((opal_atomic_int32_t*)addr,
+                                                      (int32_t*)oldval,
+                                                      (int32_t)newval);
+#elif SIZEOF_VOID_P == 8
+    return opal_atomic_compare_exchange_strong_rel_64((opal_atomic_int64_t*)addr,
+                                                      (int64_t*)oldval,
+                                                      (int64_t)newval);
+#else
+#error "No implementation of opal_atomic_compare_exchange_strong_rel_ptr"
+#endif
+}
+
+#endif /* #ifndef ATOMIC_IMPL_PTR_CSWAP_H */
diff --git a/opal/include/opal/sys/atomic_impl_ptr_llsc.h b/opal/include/opal/sys/atomic_impl_ptr_llsc.h
new file mode 100644
index 00000000000..076e768cba5
--- /dev/null
+++ b/opal/include/opal/sys/atomic_impl_ptr_llsc.h
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2011-2017 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2017      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2020-2021 Google, LLC. All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ * Pointer-sized wrapper for LL/SC calls, wrappers around size-defined
+ * calls.  Note that these must be macros, as LL/SC may not work
+ * across function calls.
+ */
+
+
+#ifndef ATOMIC_IMPL_PTR_LLSC_H
+#define ATOMIC_IMPL_PTR_LLSC_H 1
+
+#if SIZEOF_VOID_P == 4 && defined(OPAL_HAVE_ATOMIC_LLSC_32) && OPAL_HAVE_ATOMIC_LLSC_32
+
+#    define opal_atomic_ll_ptr(addr, ret) opal_atomic_ll_32((opal_atomic_int32_t *) (addr), ret)
+#    define opal_atomic_sc_ptr(addr, value, ret) \
+        opal_atomic_sc_32((opal_atomic_int32_t *) (addr), (intptr_t)(value), ret)
+
+#    define OPAL_HAVE_ATOMIC_LLSC_PTR 1
+
+#elif SIZEOF_VOID_P == 8 && defined(OPAL_HAVE_ATOMIC_LLSC_64) && OPAL_HAVE_ATOMIC_LLSC_64
+
+#    define opal_atomic_ll_ptr(addr, ret) opal_atomic_ll_64((opal_atomic_int64_t *) (addr), ret)
+#    define opal_atomic_sc_ptr(addr, value, ret) \
+        opal_atomic_sc_64((opal_atomic_int64_t *) (addr), (intptr_t)(value), ret)
+
+#    define OPAL_HAVE_ATOMIC_LLSC_PTR 1
+
+#endif /* SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_LLSC_64 */
+
+#endif /* ATOMIC_IMPL_PTR_LLSC_H */
diff --git a/opal/include/opal/sys/atomic_impl_ptr_swap.h b/opal/include/opal/sys/atomic_impl_ptr_swap.h
new file mode 100644
index 00000000000..97615961fc7
--- /dev/null
+++ b/opal/include/opal/sys/atomic_impl_ptr_swap.h
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2014 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef ATOMIC_SWAP_PTR_IMPL_H
+#define ATOMIC_SWAP_PTR_IMPL_H
+
+static inline intptr_t opal_atomic_swap_ptr(opal_atomic_intptr_t *addr, intptr_t newval)
+{
+#if SIZEOF_VOID_P == 4
+    return (intptr_t)opal_atomic_swap_32((opal_atomic_int32_t *) addr, (int32_t) newval);
+#elif SIZEOF_VOID_P == 8
+    return (intptr_t)opal_atomic_swap_64((opal_atomic_int64_t *) addr, (int64_t) newval);
+#else
+#error "No implementation of opal_atomic_swap_ptr"
+#endif
+}
+
+#endif /* #ifndef ATOMIC_SWAP_PTR_IMPL_H */
diff --git a/opal/include/opal/sys/atomic_impl_size_t_math.h b/opal/include/opal/sys/atomic_impl_size_t_math.h
new file mode 100644
index 00000000000..4316bd9a52c
--- /dev/null
+++ b/opal/include/opal/sys/atomic_impl_size_t_math.h
@@ -0,0 +1,103 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2014 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ * Implementation of size_t atomic add functions as wrappers around
+ * sized implementations.
+ */
+
+#ifndef ATOMIC_IMPL_SIZE_T_MATH_H
+#define ATOMIC_IMPL_SIZE_T_MATH_H 1
+
+#include <stdlib.h>
+
+static inline size_t opal_atomic_add_fetch_size_t(opal_atomic_size_t *addr, size_t delta)
+{
+#if SIZEOF_SIZE_T == 4
+    return (size_t)opal_atomic_add_fetch_32((opal_atomic_int32_t *) addr, delta);
+#elif SIZEOF_SIZE_T == 8
+    return (size_t)opal_atomic_add_fetch_64((opal_atomic_int64_t *) addr, delta);
+#else
+#error "No implementation of opal_atomic_add_fetch_size_t"
+#endif
+}
+
+static inline size_t opal_atomic_fetch_add_size_t(opal_atomic_size_t *addr, size_t delta)
+{
+#if SIZEOF_SIZE_T == 4
+    return (size_t)opal_atomic_fetch_add_32((opal_atomic_int32_t *) addr, delta);
+#elif SIZEOF_SIZE_T == 8
+    return (size_t)opal_atomic_fetch_add_64((opal_atomic_int64_t *) addr, delta);
+#else
+#error "No implementation of opal_atomic_fetch_add_size_t"
+#endif
+}
+
+static inline size_t opal_atomic_sub_fetch_size_t(opal_atomic_size_t *addr, size_t delta)
+{
+#if SIZEOF_SIZE_T == 4
+    return (size_t)opal_atomic_sub_fetch_32((opal_atomic_int32_t *) addr, delta);
+#elif SIZEOF_SIZE_T == 8
+    return (size_t)opal_atomic_sub_fetch_64((opal_atomic_int64_t *) addr, delta);
+#else
+#error "No implementation of opal_atomic_sub_fetch_size_t"
+#endif
+}
+
+static inline size_t opal_atomic_fetch_sub_size_t(opal_atomic_size_t *addr, size_t delta)
+{
+#if SIZEOF_SIZE_T == 4
+    return (size_t)opal_atomic_fetch_sub_32((opal_atomic_int32_t *) addr, delta);
+#elif SIZEOF_SIZE_T == 8
+    return (size_t)opal_atomic_fetch_sub_64((opal_atomic_int64_t *) addr, delta);
+#else
+#error "No implementation of opal_atomic_fetch_sub_size_t"
+#endif
+}
+
+/**
+ * Atomically increment the content depending on the type. This
+ * macro detect at compile time the type of the first argument
+ * and choose the correct function to be called.
+ *
+ * \note This macro should only be used for integer types.
+ *
+ * @param addr          Address of <TYPE>
+ * @param delta         Value to add (converted to <TYPE>).
+ */
+#define opal_atomic_add(ADDR, VALUE) \
+    opal_atomic_add_xx((opal_atomic_intptr_t *) (ADDR), (int32_t)(VALUE), sizeof(*(ADDR)))
+
+static inline void opal_atomic_add_xx(opal_atomic_intptr_t *addr, int32_t value, size_t length)
+{
+    switch (length) {
+        case 4:
+            (void)opal_atomic_fetch_add_32((opal_atomic_int32_t*)addr, value);
+            break;
+        case 8:
+            (void)opal_atomic_fetch_add_64((opal_atomic_int64_t*)addr, value);
+            break;
+        default:
+            abort();
+    }
+}
+
+#endif
diff --git a/opal/include/opal/sys/atomic_impl_spinlock.h b/opal/include/opal/sys/atomic_impl_spinlock.h
new file mode 100644
index 00000000000..1f92f5ae72b
--- /dev/null
+++ b/opal/include/opal/sys/atomic_impl_spinlock.h
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2014 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ * Compare-and-swap based implementation of the atomic interface
+ */
+
+#ifndef ATOMIC_SPINLOCK_IMPL_H
+#define ATOMIC_SPINLOCK_IMPL_H
+
+static inline void opal_atomic_lock_init(opal_atomic_lock_t *lock, int32_t value)
+{
+    *lock = value;
+    opal_atomic_wmb();
+}
+
+static inline int opal_atomic_trylock(opal_atomic_lock_t *lock)
+{
+    int32_t unlocked = OPAL_ATOMIC_LOCK_UNLOCKED;
+    bool ret = opal_atomic_compare_exchange_strong_acq_32(lock, &unlocked,
+                                                          OPAL_ATOMIC_LOCK_LOCKED);
+    return (ret == false) ? 1 : 0;
+}
+
+static inline void opal_atomic_lock(opal_atomic_lock_t *lock)
+{
+    while (opal_atomic_trylock(lock)) {
+        while (*lock == OPAL_ATOMIC_LOCK_LOCKED) {
+            /* spin */;
+        }
+    }
+}
+
+static inline void opal_atomic_unlock(opal_atomic_lock_t *lock)
+{
+    opal_atomic_wmb();
+    *lock = OPAL_ATOMIC_LOCK_UNLOCKED;
+}
+
+#endif /* #ifndef ATOMIC_SPINLOCK_IMPL_H */
diff --git a/opal/include/opal/sys/atomic_stdc.h b/opal/include/opal/sys/atomic_stdc.h
index 1c31df8b7b2..5eca86c9edb 100644
--- a/opal/include/opal/sys/atomic_stdc.h
+++ b/opal/include/opal/sys/atomic_stdc.h
@@ -4,9 +4,11 @@
  *                         reserved.
  * Copyright (c) 2018      Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2019      Google, LLC. All rights reserved.
+ * Copyright (c) 2019-2021 Google, LLC. All rights reserved.
  * Copyright (c) 2019      Triad National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -24,246 +26,190 @@
  * definitions to be worthwhile. */
 
 #if !defined(OPAL_ATOMIC_STDC_H)
-#define OPAL_ATOMIC_STDC_H
+#    define OPAL_ATOMIC_STDC_H
 
-#include <stdatomic.h>
-#include <stdint.h>
-#include "opal_stdint.h"
+#    include "opal_stdint.h"
+#    include "opal/opal_portable_platform.h"
+#    include <stdatomic.h>
+#    include <stdint.h>
 
-#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
-
-#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
-#define OPAL_HAVE_ATOMIC_SWAP_32 1
-
-#define OPAL_HAVE_ATOMIC_MATH_32 1
-#define OPAL_HAVE_ATOMIC_ADD_32 1
-#define OPAL_HAVE_ATOMIC_AND_32 1
-#define OPAL_HAVE_ATOMIC_OR_32 1
-#define OPAL_HAVE_ATOMIC_XOR_32 1
-#define OPAL_HAVE_ATOMIC_SUB_32 1
-
-#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
-#define OPAL_HAVE_ATOMIC_SWAP_64 1
-
-#define OPAL_HAVE_ATOMIC_MATH_64 1
-#define OPAL_HAVE_ATOMIC_ADD_64 1
-#define OPAL_HAVE_ATOMIC_AND_64 1
-#define OPAL_HAVE_ATOMIC_OR_64 1
-#define OPAL_HAVE_ATOMIC_XOR_64 1
-#define OPAL_HAVE_ATOMIC_SUB_64 1
-
-#define OPAL_HAVE_ATOMIC_MIN_32 1
-#define OPAL_HAVE_ATOMIC_MAX_32 1
-
-#define OPAL_HAVE_ATOMIC_MIN_64 1
-#define OPAL_HAVE_ATOMIC_MAX_64 1
-
-#define OPAL_HAVE_ATOMIC_SPINLOCKS 1
+/**********************************************************************
+ *
+ * Memory Barriers
+ *
+ *********************************************************************/
 
-static inline void opal_atomic_mb (void)
+static inline void opal_atomic_mb(void)
 {
-    atomic_thread_fence (memory_order_seq_cst);
+    atomic_thread_fence(memory_order_seq_cst);
 }
 
-static inline void opal_atomic_wmb (void)
+static inline void opal_atomic_wmb(void)
 {
-    atomic_thread_fence (memory_order_release);
+    atomic_thread_fence(memory_order_release);
 }
 
-static inline void opal_atomic_rmb (void)
+static inline void opal_atomic_rmb(void)
 {
-#if OPAL_ASSEMBLY_ARCH == OPAL_X86_64
+#    if defined(PLATFORM_ARCH_X86_64) && defined(PLATFORM_COMPILER_GNU) && __GNUC__ < 8
     /* work around a bug in older gcc versions (observed in gcc 6.x)
      * where acquire seems to get treated as a no-op instead of being
-     * equivalent to __asm__ __volatile__("": : :"memory") on x86_64 */
-    opal_atomic_mb ();
-#else
-    atomic_thread_fence (memory_order_acquire);
-#endif
+     * equivalent to __asm__ __volatile__("": : :"memory") on x86_64.
+     * The issue has been fixed in the GCC 8 release series:
+     * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80640
+     */
+    opal_atomic_mb();
+#    else
+    atomic_thread_fence(memory_order_acquire);
+#    endif
 }
 
-#define opal_atomic_compare_exchange_strong_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_relaxed, memory_order_relaxed)
-#define opal_atomic_compare_exchange_strong_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_relaxed, memory_order_relaxed)
-#define opal_atomic_compare_exchange_strong_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_relaxed, memory_order_relaxed)
-#define opal_atomic_compare_exchange_strong_acq_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed)
-#define opal_atomic_compare_exchange_strong_acq_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed)
-#define opal_atomic_compare_exchange_strong_acq_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed)
-
-#define opal_atomic_compare_exchange_strong_rel_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed)
-#define opal_atomic_compare_exchange_strong_rel_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed)
-#define opal_atomic_compare_exchange_strong_rel_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed)
-
-#define opal_atomic_compare_exchange_strong(addr, oldval, newval) atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_relaxed, memory_order_relaxed)
-#define opal_atomic_compare_exchange_strong_acq(addr, oldval, newval)  atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_acquire, memory_order_relaxed)
-#define opal_atomic_compare_exchange_strong_rel(addr, oldval, newval)  atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_release, memory_order_relaxed)
-
-#define opal_atomic_swap_32(addr, value) atomic_exchange_explicit ((_Atomic unsigned int *)addr, value, memory_order_relaxed)
-#define opal_atomic_swap_64(addr, value) atomic_exchange_explicit ((_Atomic unsigned long *)addr, value, memory_order_relaxed)
-#define opal_atomic_swap_ptr(addr, value) atomic_exchange_explicit ((_Atomic unsigned long *)addr, value, memory_order_relaxed)
-
-#define OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(op, bits, type, operator)      \
-    static inline type opal_atomic_fetch_ ## op ##_## bits (opal_atomic_ ## type *addr, type value) \
-    {                                                                   \
-        return atomic_fetch_ ## op ## _explicit (addr, value, memory_order_relaxed); \
-    }                                                                   \
-                                                                        \
-    static inline type opal_atomic_## op ## _fetch_ ## bits (opal_atomic_ ## type *addr, type value) \
-    {                                                                   \
-        return atomic_fetch_ ## op ## _explicit (addr, value, memory_order_relaxed) operator value; \
-    }
-
-OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, 32, int32_t, +)
-OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, 64, int64_t, +)
-OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, size_t, size_t, +)
 
-OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, 32, int32_t, -)
-OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, 64, int64_t, -)
-OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, size_t, size_t, -)
-
-OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(or, 32, int32_t, |)
-OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(or, 64, int64_t, |)
+/**********************************************************************
+ *
+ * Compare and Swap
+ *
+ *********************************************************************/
+
+#    define opal_atomic_compare_exchange_strong_32(addr, compare, value)                    \
+        atomic_compare_exchange_strong_explicit(addr, compare, value, memory_order_relaxed, \
+                                                memory_order_relaxed)
+#    define opal_atomic_compare_exchange_strong_acq_32(addr, compare, value)                \
+        atomic_compare_exchange_strong_explicit(addr, compare, value, memory_order_acquire, \
+                                                memory_order_relaxed)
+#    define opal_atomic_compare_exchange_strong_rel_32(addr, compare, value)                \
+        atomic_compare_exchange_strong_explicit(addr, compare, value, memory_order_release, \
+                                                memory_order_relaxed)
+
+#    define opal_atomic_compare_exchange_strong_64(addr, compare, value)                    \
+        atomic_compare_exchange_strong_explicit(addr, compare, value, memory_order_relaxed, \
+                                                memory_order_relaxed)
+#    define opal_atomic_compare_exchange_strong_acq_64(addr, compare, value)                \
+        atomic_compare_exchange_strong_explicit(addr, compare, value, memory_order_acquire, \
+                                                memory_order_relaxed)
+#    define opal_atomic_compare_exchange_strong_rel_64(addr, compare, value)                \
+        atomic_compare_exchange_strong_explicit(addr, compare, value, memory_order_release, \
+                                                memory_order_relaxed)
+
+#    define opal_atomic_compare_exchange_strong_ptr(addr, compare, value)                   \
+        atomic_compare_exchange_strong_explicit(addr, compare, value, memory_order_relaxed, \
+                                                memory_order_relaxed)
+#    define opal_atomic_compare_exchange_strong_acq_ptr(addr, compare, value)               \
+        atomic_compare_exchange_strong_explicit(addr, compare, value, memory_order_acquire, \
+                                                memory_order_relaxed)
+#    define opal_atomic_compare_exchange_strong_rel_ptr(addr, compare, value)               \
+        atomic_compare_exchange_strong_explicit(addr, compare, value, memory_order_release, \
+                                                memory_order_relaxed)
+
+#    if OPAL_HAVE_C11_CSWAP_INT128
 
-OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(xor, 32, int32_t, ^)
-OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(xor, 64, int64_t, ^)
+/* the C11 atomic compare-exchange is lock free so use it */
+#        define opal_atomic_compare_exchange_strong_128 atomic_compare_exchange_strong
 
-OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(and, 32, int32_t, &)
-OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(and, 64, int64_t, &)
+#        define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
 
-#define opal_atomic_add(addr, value) (void) atomic_fetch_add_explicit (addr, value, memory_order_relaxed)
+#    elif OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128
 
-static inline int32_t opal_atomic_fetch_min_32 (opal_atomic_int32_t *addr, int32_t value)
+/* fall back on the __sync builtin if available since it will emit the expected instruction on
+ * x86_64 (cmpxchng16b) */
+__opal_attribute_always_inline__ static inline bool
+opal_atomic_compare_exchange_strong_128(opal_atomic_int128_t *addr, opal_int128_t *oldval,
+                                        opal_int128_t newval)
 {
-    int32_t old = *addr;
-    do {
-        if (old <= value) {
-            break;
-        }
-    } while (!opal_atomic_compare_exchange_strong_32 (addr, &old, value));
-
-    return old;
+    opal_int128_t prev = __sync_val_compare_and_swap(addr, *oldval, newval);
+    bool ret = prev == *oldval;
+    *oldval = prev;
+    return ret;
 }
 
-static inline int32_t opal_atomic_fetch_max_32 (opal_atomic_int32_t *addr, int32_t value)
-{
-    int32_t old = *addr;
-    do {
-        if (old >= value) {
-            break;
-        }
-    } while (!opal_atomic_compare_exchange_strong_32 (addr, &old, value));
-
-    return old;
-}
+#        define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
 
-static inline int64_t opal_atomic_fetch_min_64 (opal_atomic_int64_t *addr, int64_t value)
-{
-    int64_t old = *addr;
-    do {
-        if (old <= value) {
-            break;
-        }
-    } while (!opal_atomic_compare_exchange_strong_64 (addr, &old, value));
+#    else
 
-    return old;
-}
+#        define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 0
 
-static inline int64_t opal_atomic_fetch_max_64 (opal_atomic_int64_t *addr, int64_t value)
-{
-    int64_t old = *addr;
-    do {
-        if (old >= value) {
-            break;
-        }
-    } while (!opal_atomic_compare_exchange_strong_64 (addr, &old, value));
+#    endif
 
-    return old;
-}
 
-static inline int32_t opal_atomic_min_fetch_32 (opal_atomic_int32_t *addr, int32_t value)
-{
-    int32_t old = opal_atomic_fetch_min_32 (addr, value);
-    return old <= value ? old : value;
-}
-
-static inline int32_t opal_atomic_max_fetch_32 (opal_atomic_int32_t *addr, int32_t value)
-{
-    int32_t old = opal_atomic_fetch_max_32 (addr, value);
-    return old >= value ? old : value;
-}
-
-static inline int64_t opal_atomic_min_fetch_64 (opal_atomic_int64_t *addr, int64_t value)
-{
-    int64_t old = opal_atomic_fetch_min_64 (addr, value);
-    return old <= value ? old : value;
-}
-
-static inline int64_t opal_atomic_max_fetch_64 (opal_atomic_int64_t *addr, int64_t value)
-{
-    int64_t old = opal_atomic_fetch_max_64 (addr, value);
-    return old >= value ? old : value;
-}
-
-#define OPAL_ATOMIC_LOCK_UNLOCKED false
-#define OPAL_ATOMIC_LOCK_LOCKED true
+/**********************************************************************
+ *
+ * Swap
+ *
+ *********************************************************************/
 
-#define OPAL_ATOMIC_LOCK_INIT ATOMIC_FLAG_INIT
+#    define opal_atomic_swap_32(addr, value) \
+        atomic_exchange_explicit((_Atomic unsigned int *) addr, value, memory_order_relaxed)
+#    define opal_atomic_swap_64(addr, value) \
+        atomic_exchange_explicit((_Atomic unsigned long *) addr, value, memory_order_relaxed)
+#    define opal_atomic_swap_ptr(addr, value) \
+        atomic_exchange_explicit((_Atomic unsigned long *) addr, value, memory_order_relaxed)
 
-typedef atomic_flag opal_atomic_lock_t;
 
+/**********************************************************************
+ *
+ * Atomic spinlocks
+ *
+ *********************************************************************/
 /*
  * Lock initialization function. It set the lock to UNLOCKED.
  */
-static inline void opal_atomic_lock_init (opal_atomic_lock_t *lock, bool value)
+static inline void opal_atomic_lock_init(opal_atomic_lock_t *lock, int32_t value)
 {
-    atomic_flag_clear (lock);
+    atomic_flag_clear_explicit(lock, memory_order_relaxed);
 }
 
-
-static inline int opal_atomic_trylock (opal_atomic_lock_t *lock)
+static inline int opal_atomic_trylock(opal_atomic_lock_t *lock)
 {
-    return (int) atomic_flag_test_and_set (lock);
+    return (int) atomic_flag_test_and_set(lock);
 }
 
-
 static inline void opal_atomic_lock(opal_atomic_lock_t *lock)
 {
-    while (opal_atomic_trylock (lock)) {
+    while (opal_atomic_trylock(lock)) {
     }
 }
 
-
-static inline void opal_atomic_unlock (opal_atomic_lock_t *lock)
+static inline void opal_atomic_unlock(opal_atomic_lock_t *lock)
 {
-    atomic_flag_clear (lock);
+    atomic_flag_clear(lock);
 }
 
 
-#if OPAL_HAVE_C11_CSWAP_INT128
-
-/* the C11 atomic compare-exchange is lock free so use it */
-#define opal_atomic_compare_exchange_strong_128 atomic_compare_exchange_strong
-
-#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
-
-#elif OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128
+/**********************************************************************
+ *
+ * Atomic math operations
+ *
+ *********************************************************************/
+
+#    define OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(op, bits, type, operator)                             \
+        static inline type opal_atomic_fetch_##op##_##bits(opal_atomic_##type *addr, type value)   \
+        {                                                                                          \
+            return atomic_fetch_##op##_explicit(addr, value, memory_order_relaxed);                \
+        }                                                                                          \
+                                                                                                   \
+        static inline type opal_atomic_##op##_fetch_##bits(opal_atomic_##type *addr, type value)   \
+        {                                                                                          \
+            return atomic_fetch_##op##_explicit(addr, value, memory_order_relaxed) operator value; \
+        }
 
-/* fall back on the __sync builtin if available since it will emit the expected instruction on x86_64 (cmpxchng16b) */
-__opal_attribute_always_inline__
-static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_int128_t *addr,
-                                                            opal_int128_t *oldval, opal_int128_t newval)
-{
-    opal_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval);
-    bool ret = prev == *oldval;
-    *oldval = prev;
-    return ret;
-}
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, 32, int32_t, +)
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(and, 32, int32_t, &)
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(or, 32, int32_t, |)
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(xor, 32, int32_t, ^)
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, 32, int32_t, -)
 
-#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, 64, int64_t, +)
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(and, 64, int64_t, &)
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(or, 64, int64_t, |)
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(xor, 64, int64_t, ^)
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, 64, int64_t, -)
 
-#else
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, size_t, size_t, +)
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, size_t, size_t, -)
 
-#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 0
+#    define opal_atomic_add(addr, value) \
+        (void) atomic_fetch_add_explicit(addr, value, memory_order_relaxed)
 
-#endif
+#include "opal/sys/atomic_impl_minmax_math.h"
 
 #endif /* !defined(OPAL_ATOMIC_STDC_H) */
diff --git a/opal/include/opal/sys/cma.h b/opal/include/opal/sys/cma.h
index 273b482e4c3..b1db5f7700c 100644
--- a/opal/include/opal/sys/cma.h
+++ b/opal/include/opal/sys/cma.h
@@ -5,6 +5,8 @@
  * Copyright (c) 2017      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2020      Google, LLC. All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  */
 
@@ -19,74 +21,60 @@
 #ifndef OPAL_SYS_CMA_H
 #define OPAL_SYS_CMA_H 1
 
-#if !defined(OPAL_ASSEMBLY_ARCH)
-/* need opal_config.h for the assembly architecture */
-#include "opal_config.h"
-#endif
-
-#include "opal/sys/architecture.h"
+#include "opal/opal_portable_platform.h"
 
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 
 #ifdef HAVE_UNISTD_H
-#include <sys/unistd.h>
+#    include <sys/unistd.h>
 #endif
 
-#ifdef __linux__
+#ifdef PLATFORM_OS_LINUX
 
 /* Cross Memory Attach is so far only supported under linux */
 
-#if OPAL_ASSEMBLY_ARCH == OPAL_X86_64
-#define __NR_process_vm_readv 310
-#define __NR_process_vm_writev 311
-#elif OPAL_ASSEMBLY_ARCH == OPAL_IA32
-#define __NR_process_vm_readv 347
-#define __NR_process_vm_writev 348
-#elif OPAL_ASSEMBLY_ARCH == OPAL_POWERPC32
-#define __NR_process_vm_readv 351
-#define __NR_process_vm_writev 352
-#elif OPAL_ASSEMBLY_ARCH == OPAL_POWERPC64
-#define __NR_process_vm_readv 351
-#define __NR_process_vm_writev 352
-#elif OPAL_ASSEMBLY_ARCH == OPAL_ARM
-
-#define __NR_process_vm_readv 376
-#define __NR_process_vm_writev 377
-
-#elif OPAL_ASSEMBLY_ARCH == OPAL_ARM64
+#    if defined(PLATFORM_ARCH_x86_64)
+#        define __NR_process_vm_readv  310
+#        define __NR_process_vm_writev 311
+#    elif defined(PLATFORM_ARCH_X86)
+#        define __NR_process_vm_readv  347
+#        define __NR_process_vm_writev 348
+#    elif defined(PLATFORM_ARCH_POWERPC) && defined(PLATFORM_ARCH_32)
+#        define __NR_process_vm_readv  351
+#        define __NR_process_vm_writev 352
+#    elif defined(PLATFORM_ARCH_POWERPC) && defined(PLATFORM_ARCH_64)
+#        define __NR_process_vm_readv  351
+#        define __NR_process_vm_writev 352
+#    elif defined(PLATFORM_ARCH_ARM)
+
+#        define __NR_process_vm_readv  376
+#        define __NR_process_vm_writev 377
+
+#    elif defined(PLATFORM_ARCH_AARCH64)
 
 /* ARM64 uses the asm-generic syscall numbers */
 
-#define __NR_process_vm_readv 270
-#define __NR_process_vm_writev 271
-
-#else
-#error "Unsupported architecture for process_vm_readv and process_vm_writev syscalls"
-#endif
+#        define __NR_process_vm_readv  270
+#        define __NR_process_vm_writev 271
 
+#    else
+#        error "Unsupported architecture for process_vm_readv and process_vm_writev syscalls"
+#    endif
 
-static inline ssize_t
-process_vm_readv(pid_t pid,
-                 const struct iovec  *lvec,
-                 unsigned long liovcnt,
-                 const struct iovec *rvec,
-                 unsigned long riovcnt,
-                 unsigned long flags)
+static inline ssize_t process_vm_readv(pid_t pid, const struct iovec *lvec, unsigned long liovcnt,
+                                       const struct iovec *rvec, unsigned long riovcnt,
+                                       unsigned long flags)
 {
-  return syscall(__NR_process_vm_readv, pid, lvec, liovcnt, rvec, riovcnt, flags);
+    return syscall(__NR_process_vm_readv, pid, lvec, liovcnt, rvec, riovcnt, flags);
 }
 
-static inline ssize_t
-process_vm_writev(pid_t pid,
-                  const struct iovec  *lvec,
-                  unsigned long liovcnt,
-                  const struct iovec *rvec,
-                  unsigned long riovcnt,
-                  unsigned long flags)
+static inline ssize_t process_vm_writev(pid_t pid, const struct iovec *lvec, unsigned long liovcnt,
+                                        const struct iovec *rvec, unsigned long riovcnt,
+                                        unsigned long flags)
 {
-  return syscall(__NR_process_vm_writev, pid, lvec, liovcnt, rvec, riovcnt, flags);
+    return syscall(__NR_process_vm_writev, pid, lvec, liovcnt, rvec, riovcnt, flags);
 }
 
 #endif /* __linux__ */
diff --git a/opal/include/opal/sys/gcc_builtin/atomic.h b/opal/include/opal/sys/gcc_builtin/atomic.h
index f42559bc781..66a3f899456 100644
--- a/opal/include/opal/sys/gcc_builtin/atomic.h
+++ b/opal/include/opal/sys/gcc_builtin/atomic.h
@@ -13,10 +13,13 @@
  * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
  * Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights
  *                         reserved.
- * Copyright (c) 2016-2017 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2018      Triad National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2021      Google, LLC. All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -32,231 +35,242 @@
  * Memory Barriers
  *
  *********************************************************************/
-#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
-
-#define OPAL_HAVE_ATOMIC_MATH_32 1
-#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
-#define OPAL_HAVE_ATOMIC_ADD_32 1
-#define OPAL_HAVE_ATOMIC_AND_32 1
-#define OPAL_HAVE_ATOMIC_OR_32 1
-#define OPAL_HAVE_ATOMIC_XOR_32 1
-#define OPAL_HAVE_ATOMIC_SUB_32 1
-#define OPAL_HAVE_ATOMIC_SWAP_32 1
-#define OPAL_HAVE_ATOMIC_MATH_64 1
-#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
-#define OPAL_HAVE_ATOMIC_ADD_64 1
-#define OPAL_HAVE_ATOMIC_AND_64 1
-#define OPAL_HAVE_ATOMIC_OR_64 1
-#define OPAL_HAVE_ATOMIC_XOR_64 1
-#define OPAL_HAVE_ATOMIC_SUB_64 1
-#define OPAL_HAVE_ATOMIC_SWAP_64 1
 
+#if defined(PLATFORM_ARCH_X86_64) && defined(PLATFORM_COMPILER_GNU) && __GNUC__ < 8
+    /* work around a bug in older gcc versions where ACQUIRE seems to get
+     * treated as a no-op instead */
+#define OPAL_BUSTED_ATOMIC_MB 1
+#else
+#define OPAL_BUSTED_ATOMIC_MB 0
+#endif
 
 static inline void opal_atomic_mb(void)
 {
-    __atomic_thread_fence (__ATOMIC_SEQ_CST);
+    __atomic_thread_fence(__ATOMIC_SEQ_CST);
 }
 
 static inline void opal_atomic_rmb(void)
 {
-#if OPAL_ASSEMBLY_ARCH == OPAL_X86_64
-    /* work around a bug in older gcc versions where ACQUIRE seems to get
-     * treated as a no-op instead of being equivalent to
-     * __asm__ __volatile__("": : :"memory") */
-    __atomic_thread_fence (__ATOMIC_SEQ_CST);
+#if OPAL_BUSTED_ATOMIC_MB
+    __asm__ __volatile__("" : : : "memory");
 #else
-    __atomic_thread_fence (__ATOMIC_ACQUIRE);
+    __atomic_thread_fence(__ATOMIC_ACQUIRE);
 #endif
 }
 
 static inline void opal_atomic_wmb(void)
 {
-    __atomic_thread_fence (__ATOMIC_RELEASE);
+    __atomic_thread_fence(__ATOMIC_RELEASE);
 }
 
-#define MB() opal_atomic_mb()
 
 /**********************************************************************
  *
- * Atomic math operations
+ * Compare and Swap
  *
  *********************************************************************/
 
 /*
  * Suppress numerous (spurious ?) warnings from Oracle Studio compilers
  * see https://community.oracle.com/thread/3968347
- */ 
+ */
 #if defined(__SUNPRO_C) || defined(__SUNPRO_CC)
-#pragma error_messages(off, E_ARG_INCOMPATIBLE_WITH_ARG_L)
+#    pragma error_messages(off, E_ARG_INCOMPATIBLE_WITH_ARG_L)
 #endif
 
-static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
+static inline bool opal_atomic_compare_exchange_strong_32(opal_atomic_int32_t *addr,
+                                                          int32_t *oldval, int32_t newval)
 {
-    return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
+    return __atomic_compare_exchange_n(addr, oldval, newval, false, __ATOMIC_ACQUIRE,
+                                       __ATOMIC_RELAXED);
 }
 
-
-static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
+static inline bool opal_atomic_compare_exchange_strong_acq_32(opal_atomic_int32_t *addr,
+                                                              int32_t *oldval, int32_t newval)
 {
-    return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+    return __atomic_compare_exchange_n(addr, oldval, newval, false, __ATOMIC_ACQUIRE,
+                                       __ATOMIC_RELAXED);
 }
 
-static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
+static inline bool opal_atomic_compare_exchange_strong_rel_32(opal_atomic_int32_t *addr,
+                                                              int32_t *oldval, int32_t newval)
 {
-    return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
+    return __atomic_compare_exchange_n(addr, oldval, newval, false, __ATOMIC_RELEASE,
+                                       __ATOMIC_RELAXED);
 }
 
-static inline int32_t opal_atomic_swap_32 (opal_atomic_int32_t *addr, int32_t newval)
+static inline bool opal_atomic_compare_exchange_strong_64(opal_atomic_int64_t *addr,
+                                                          int64_t *oldval, int64_t newval)
 {
-    int32_t oldval;
-    __atomic_exchange (addr, &newval, &oldval, __ATOMIC_RELAXED);
-    return oldval;
+    return __atomic_compare_exchange_n(addr, oldval, newval, false, __ATOMIC_ACQUIRE,
+                                       __ATOMIC_RELAXED);
 }
 
-static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t *addr, int32_t delta)
+static inline bool opal_atomic_compare_exchange_strong_acq_64(opal_atomic_int64_t *addr,
+                                                              int64_t *oldval, int64_t newval)
 {
-    return __atomic_fetch_add (addr, delta, __ATOMIC_RELAXED);
+    return __atomic_compare_exchange_n(addr, oldval, newval, false, __ATOMIC_ACQUIRE,
+                                       __ATOMIC_RELAXED);
 }
 
-static inline int32_t opal_atomic_fetch_and_32(opal_atomic_int32_t *addr, int32_t value)
+static inline bool opal_atomic_compare_exchange_strong_rel_64(opal_atomic_int64_t *addr,
+                                                              int64_t *oldval, int64_t newval)
 {
-    return __atomic_fetch_and (addr, value, __ATOMIC_RELAXED);
+    return __atomic_compare_exchange_n(addr, oldval, newval, false, __ATOMIC_RELEASE,
+                                       __ATOMIC_RELAXED);
 }
 
-static inline int32_t opal_atomic_fetch_or_32(opal_atomic_int32_t *addr, int32_t value)
-{
-    return __atomic_fetch_or (addr, value, __ATOMIC_RELAXED);
-}
+#include "opal/sys/atomic_impl_ptr_cswap.h"
 
-static inline int32_t opal_atomic_fetch_xor_32(opal_atomic_int32_t *addr, int32_t value)
-{
-    return __atomic_fetch_xor (addr, value, __ATOMIC_RELAXED);
-}
+#if OPAL_HAVE_GCC_BUILTIN_CSWAP_INT128
 
-static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t *addr, int32_t delta)
-{
-    return __atomic_fetch_sub (addr, delta, __ATOMIC_RELAXED);
-}
+#    define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
 
-static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
+static inline bool opal_atomic_compare_exchange_strong_128(opal_atomic_int128_t *addr,
+                                                           opal_int128_t *oldval,
+                                                           opal_int128_t newval)
 {
-    return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
+    return __atomic_compare_exchange_n(addr, oldval, newval, false, __ATOMIC_ACQUIRE,
+                                       __ATOMIC_RELAXED);
 }
 
-static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
-{
-    return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
-}
+#elif defined(OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128) && OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128
 
+#    define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
 
-static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
-{
-    return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
-}
+/* __atomic version is not lock-free so use legacy __sync version */
 
-static inline int64_t opal_atomic_swap_64 (opal_atomic_int64_t *addr, int64_t newval)
+static inline bool opal_atomic_compare_exchange_strong_128(opal_atomic_int128_t *addr,
+                                                           opal_int128_t *oldval,
+                                                           opal_int128_t newval)
 {
-    int64_t oldval;
-    __atomic_exchange (addr, &newval, &oldval, __ATOMIC_RELAXED);
-    return oldval;
+    opal_int128_t prev = __sync_val_compare_and_swap(addr, *oldval, newval);
+    bool ret = prev == *oldval;
+    *oldval = prev;
+    return ret;
 }
 
-static inline int64_t opal_atomic_fetch_add_64(opal_atomic_int64_t *addr, int64_t delta)
-{
-    return __atomic_fetch_add (addr, delta, __ATOMIC_RELAXED);
-}
+#endif
 
-static inline int64_t opal_atomic_fetch_and_64(opal_atomic_int64_t *addr, int64_t value)
-{
-    return __atomic_fetch_and (addr, value, __ATOMIC_RELAXED);
-}
 
-static inline int64_t opal_atomic_fetch_or_64(opal_atomic_int64_t *addr, int64_t value)
-{
-    return __atomic_fetch_or (addr, value, __ATOMIC_RELAXED);
-}
+/**********************************************************************
+ *
+ * Swap
+ *
+ *********************************************************************/
 
-static inline int64_t opal_atomic_fetch_xor_64(opal_atomic_int64_t *addr, int64_t value)
+static inline int32_t opal_atomic_swap_32(opal_atomic_int32_t *addr, int32_t newval)
 {
-    return __atomic_fetch_xor (addr, value, __ATOMIC_RELAXED);
+    int32_t oldval;
+    __atomic_exchange(addr, &newval, &oldval, __ATOMIC_RELAXED);
+    return oldval;
 }
 
-static inline int64_t opal_atomic_fetch_sub_64(opal_atomic_int64_t *addr, int64_t delta)
+static inline int64_t opal_atomic_swap_64(opal_atomic_int64_t *addr, int64_t newval)
 {
-    return __atomic_fetch_sub (addr, delta, __ATOMIC_RELAXED);
+    int64_t oldval;
+    __atomic_exchange(addr, &newval, &oldval, __ATOMIC_RELAXED);
+    return oldval;
 }
 
-#if OPAL_HAVE_GCC_BUILTIN_CSWAP_INT128
-
-#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
-
-static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_int128_t *addr,
-                                                            opal_int128_t *oldval, opal_int128_t newval)
+static inline intptr_t opal_atomic_swap_ptr(opal_atomic_intptr_t *addr, intptr_t newval)
 {
-    return __atomic_compare_exchange_n (addr, oldval, newval, false,
-                                        __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
+    intptr_t oldval;
+    __atomic_exchange(addr, &newval, &oldval, __ATOMIC_RELAXED);
+    return oldval;
 }
 
-#elif defined(OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128) && OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128
-
-#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
-
-/* __atomic version is not lock-free so use legacy __sync version */
-
-static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_opal_int128_t *addr,
-                                                            opal_int128_t *oldval, opal_int128_t newval)
-{
-    opal_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval);
-    bool ret = prev == *oldval;
-    *oldval = prev;
-    return ret;
-}
 
-#endif
+/**********************************************************************
+ *
+ * Atomic spinlocks
+ *
+ *********************************************************************/
 
 #if defined(__HLE__)
 
-#include <immintrin.h>
+#    include <immintrin.h>
 
-#define OPAL_HAVE_ATOMIC_SPINLOCKS 1
-
-static inline void opal_atomic_lock_init (opal_atomic_lock_t* lock, int32_t value)
+static inline void opal_atomic_lock_init(opal_atomic_lock_t *lock, int32_t value)
 {
-   lock->u.lock = value;
+    *lock = value;
 }
 
 static inline int opal_atomic_trylock(opal_atomic_lock_t *lock)
 {
-    int ret = __atomic_exchange_n (&lock->u.lock, OPAL_ATOMIC_LOCK_LOCKED,
-                                   __ATOMIC_ACQUIRE | __ATOMIC_HLE_ACQUIRE);
+    int ret = __atomic_exchange_n(&lock, OPAL_ATOMIC_LOCK_LOCKED,
+                                  __ATOMIC_ACQUIRE | __ATOMIC_HLE_ACQUIRE);
     if (OPAL_ATOMIC_LOCK_LOCKED == ret) {
         /* abort the transaction */
-        _mm_pause ();
+        _mm_pause();
         return 1;
     }
 
     return 0;
 }
 
-static inline void opal_atomic_lock (opal_atomic_lock_t *lock)
+static inline void opal_atomic_lock(opal_atomic_lock_t *lock)
 {
-    while (OPAL_ATOMIC_LOCK_LOCKED == __atomic_exchange_n (&lock->u.lock, OPAL_ATOMIC_LOCK_LOCKED,
-                                                      __ATOMIC_ACQUIRE | __ATOMIC_HLE_ACQUIRE)) {
+    while (OPAL_ATOMIC_LOCK_LOCKED
+           == __atomic_exchange_n(&lock, OPAL_ATOMIC_LOCK_LOCKED,
+                                  __ATOMIC_ACQUIRE | __ATOMIC_HLE_ACQUIRE)) {
         /* abort the transaction */
-        _mm_pause ();
+        _mm_pause();
     }
 }
 
-static inline void opal_atomic_unlock (opal_atomic_lock_t *lock)
+static inline void opal_atomic_unlock(opal_atomic_lock_t *lock)
 {
-    __atomic_store_n (&lock->u.lock, OPAL_ATOMIC_LOCK_UNLOCKED,
-                       __ATOMIC_RELEASE | __ATOMIC_HLE_RELEASE);
+    __atomic_store_n(&lock, OPAL_ATOMIC_LOCK_UNLOCKED,
+                     __ATOMIC_RELEASE | __ATOMIC_HLE_RELEASE);
 }
 
+#else  /* #if defined(__HLE__) */
+
+#include "opal/sys/atomic_impl_spinlock.h"
+
 #endif
 
+
+/**********************************************************************
+ *
+ * Atomic math operations
+ *
+ *********************************************************************/
+
+#define OPAL_ATOMIC_DEFINE_OP(type, bits, operator, name)                                          \
+        static inline type opal_atomic_fetch_##name##_##bits(opal_atomic_##type *addr, type value) \
+        {                                                                                          \
+            return __atomic_fetch_##name(addr, value, __ATOMIC_RELAXED);                           \
+        }                                                                                          \
+                                                                                                   \
+        static inline type opal_atomic_##name##_fetch_##bits(opal_atomic_##type *addr, type value) \
+        {                                                                                          \
+            return __atomic_##name##_fetch(addr, value, __ATOMIC_RELAXED);                         \
+        }
+
+OPAL_ATOMIC_DEFINE_OP(int32_t, 32, +, add)
+OPAL_ATOMIC_DEFINE_OP(int32_t, 32, &, and)
+OPAL_ATOMIC_DEFINE_OP(int32_t, 32, |, or)
+OPAL_ATOMIC_DEFINE_OP(int32_t, 32, ^, xor)
+OPAL_ATOMIC_DEFINE_OP(int32_t, 32, -, sub)
+
+OPAL_ATOMIC_DEFINE_OP(int64_t, 64, +, add)
+OPAL_ATOMIC_DEFINE_OP(int64_t, 64, &, and)
+OPAL_ATOMIC_DEFINE_OP(int64_t, 64, |, or)
+OPAL_ATOMIC_DEFINE_OP(int64_t, 64, ^, xor)
+OPAL_ATOMIC_DEFINE_OP(int64_t, 64, -, sub)
+
+OPAL_ATOMIC_DEFINE_OP(size_t, size_t, +, add)
+OPAL_ATOMIC_DEFINE_OP(size_t, size_t, -, sub)
+
+#define opal_atomic_add(ADDR, VALUE) \
+     (void) __atomic_fetch_add(ADDR, VALUE, __ATOMIC_RELAXED)
+
+#include "opal/sys/atomic_impl_minmax_math.h"
+
 #if defined(__SUNPRO_C) || defined(__SUNPRO_CC)
-#pragma error_messages(default, E_ARG_INCOMPATIBLE_WITH_ARG_L)
+#    pragma error_messages(default, E_ARG_INCOMPATIBLE_WITH_ARG_L)
 #endif
 
 #endif /* ! OPAL_SYS_ARCH_ATOMIC_H */
diff --git a/opal/include/opal/sys/ia32/Makefile.am b/opal/include/opal/sys/ia32/Makefile.am
deleted file mode 100644
index b3a0efcd554..00000000000
--- a/opal/include/opal/sys/ia32/Makefile.am
+++ /dev/null
@@ -1,23 +0,0 @@
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-# This makefile.am does not stand on its own - it is included from opal/include/Makefile.am
-
-headers += \
-	opal/sys/ia32/atomic.h \
-	opal/sys/ia32/timer.h
diff --git a/opal/include/opal/sys/ia32/atomic.h b/opal/include/opal/sys/ia32/atomic.h
deleted file mode 100644
index eac18b4b007..00000000000
--- a/opal/include/opal/sys/ia32/atomic.h
+++ /dev/null
@@ -1,167 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2010 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007-2010 Oracle and/or its affiliates.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef OPAL_SYS_ARCH_ATOMIC_H
-#define OPAL_SYS_ARCH_ATOMIC_H 1
-
-/*
- * On ia32, we use cmpxchg.
- */
-
-#define SMPLOCK "lock; "
-#define MB() __asm__ __volatile__("": : :"memory")
-
-
-/**********************************************************************
- *
- * Define constants for IA32
- *
- *********************************************************************/
-#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
-
-#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
-
-#define OPAL_HAVE_ATOMIC_MATH_32 1
-#define OPAL_HAVE_ATOMIC_ADD_32 1
-#define OPAL_HAVE_ATOMIC_SUB_32 1
-
-/**********************************************************************
- *
- * Memory Barriers
- *
- *********************************************************************/
-#if OPAL_GCC_INLINE_ASSEMBLY
-
-static inline void opal_atomic_mb(void)
-{
-    MB();
-}
-
-
-static inline void opal_atomic_rmb(void)
-{
-    MB();
-}
-
-
-static inline void opal_atomic_wmb(void)
-{
-    MB();
-}
-
-static inline void opal_atomic_isync(void)
-{
-}
-
-#endif /* OPAL_GCC_INLINE_ASSEMBLY */
-
-
-/**********************************************************************
- *
- * Atomic math operations
- *
- *********************************************************************/
-#if OPAL_GCC_INLINE_ASSEMBLY
-
-static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
-{
-   unsigned char ret;
-   __asm__ __volatile__ (
-                       SMPLOCK "cmpxchgl %3,%2   \n\t"
-                               "sete     %0      \n\t"
-                       : "=qm" (ret), "+a" (*oldval), "+m" (*addr)
-                       : "q"(newval)
-                       : "memory", "cc");
-
-   return (bool) ret;
-}
-
-#endif /* OPAL_GCC_INLINE_ASSEMBLY */
-
-#define opal_atomic_compare_exchange_strong_acq_32 opal_atomic_compare_exchange_strong_32
-#define opal_atomic_compare_exchange_strong_rel_32 opal_atomic_compare_exchange_strong_32
-
-#if OPAL_GCC_INLINE_ASSEMBLY
-
-#define OPAL_HAVE_ATOMIC_SWAP_32 1
-
-static inline int32_t opal_atomic_swap_32( opal_atomic_int32_t *addr,
-					   int32_t newval)
-{
-    int32_t oldval;
-
-    __asm__ __volatile__("xchg %1, %0" :
-			 "=r" (oldval), "=m" (*addr) :
-			 "0" (newval), "m" (*addr) :
-			 "memory");
-    return oldval;
-}
-
-#endif /* OPAL_GCC_INLINE_ASSEMBLY */
-
-
-#if OPAL_GCC_INLINE_ASSEMBLY
-
-/**
- * atomic_add - add integer to atomic variable
- * @i: integer value to add
- * @v: pointer of type int
- *
- * Atomically adds @i to @v.
- */
-static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t* v, int i)
-{
-    int ret = i;
-   __asm__ __volatile__(
-                        SMPLOCK "xaddl %1,%0"
-                        :"+m" (*v), "+r" (ret)
-                        :
-                        :"memory", "cc"
-                        );
-   return ret;
-}
-
-
-/**
- * atomic_sub - subtract the atomic variable
- * @i: integer value to subtract
- * @v: pointer of type int
- *
- * Atomically subtracts @i from @v.
- */
-static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t* v, int i)
-{
-    int ret = -i;
-   __asm__ __volatile__(
-                        SMPLOCK "xaddl %1,%0"
-                        :"+m" (*v), "+r" (ret)
-                        :
-                        :"memory", "cc"
-                        );
-   return ret;
-}
-
-#endif /* OPAL_GCC_INLINE_ASSEMBLY */
-
-#endif /* ! OPAL_SYS_ARCH_ATOMIC_H */
diff --git a/opal/include/opal/sys/ia32/timer.h b/opal/include/opal/sys/ia32/timer.h
deleted file mode 100644
index 2fb2beef639..00000000000
--- a/opal/include/opal/sys/ia32/timer.h
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2014 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef OPAL_SYS_ARCH_TIMER_H
-#define OPAL_SYS_ARCH_TIMER_H 1
-
-
-typedef uint64_t opal_timer_t;
-
-/* Using RDTSC(P) results in non-monotonic timers across cores */
-#undef OPAL_TIMER_MONOTONIC
-#define OPAL_TIMER_MONOTONIC 0
-
-#if OPAL_GCC_INLINE_ASSEMBLY
-
-static inline opal_timer_t
-opal_sys_timer_get_cycles(void)
-{
-    opal_timer_t ret;
-    int tmp;
-
-    __asm__ __volatile__(
-                         "xchgl %%ebx, %1\n"
-                         "cpuid\n"
-                         "xchgl %%ebx, %1\n"
-                         "rdtsc\n"
-                         : "=A"(ret), "=r"(tmp)
-                         :: "ecx");
-
-    return ret;
-}
-
-#define OPAL_HAVE_SYS_TIMER_GET_CYCLES 1
-
-#else
-
-#define OPAL_HAVE_SYS_TIMER_GET_CYCLES 0
-
-#endif /* OPAL_GCC_INLINE_ASSEMBLY */
-
-#endif /* ! OPAL_SYS_ARCH_TIMER_H */
diff --git a/opal/include/opal/sys/ia32/update.sh b/opal/include/opal/sys/ia32/update.sh
deleted file mode 100644
index 8d7107f362b..00000000000
--- a/opal/include/opal/sys/ia32/update.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/sh
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-CFILE=/tmp/opal_asm_$$.c
-
-trap "/bin/rm -f $CFILE; exit 0" 0 1 2 15
-
-echo Updating asm.s from atomic.h and timer.h using gcc
-
-cat > $CFILE<<EOF
-#include <stdlib.h>
-#include <inttypes.h>
-#define static
-#define inline
-#define OPAL_GCC_INLINE_ASSEMBLY 1
-#include "atomic.h"
-#include "timer.h"
-EOF
-
-gcc -O3 -I. -S $CFILE -o asm.s
diff --git a/opal/include/opal/sys/powerpc/Makefile.am b/opal/include/opal/sys/powerpc/Makefile.am
index 612dd2e4d7f..d2dbeeba0a5 100644
--- a/opal/include/opal/sys/powerpc/Makefile.am
+++ b/opal/include/opal/sys/powerpc/Makefile.am
@@ -20,4 +20,6 @@
 
 headers += \
 	opal/sys/powerpc/atomic.h \
+	opal/sys/powerpc/atomic_helper.h \
+	opal/sys/powerpc/atomic_llsc.h \
 	opal/sys/powerpc/timer.h
diff --git a/opal/include/opal/sys/powerpc/atomic.h b/opal/include/opal/sys/powerpc/atomic.h
index 393b92acba5..a92457c6da1 100644
--- a/opal/include/opal/sys/powerpc/atomic.h
+++ b/opal/include/opal/sys/powerpc/atomic.h
@@ -13,6 +13,9 @@
  * Copyright (c) 2010-2021 IBM Corporation.  All rights reserved.
  * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2021      Google, LLC. All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -27,375 +30,205 @@
  * On powerpc ...
  */
 
-#define MB()  __asm__ __volatile__ ("sync" : : : "memory")
-#define RMB() __asm__ __volatile__ ("lwsync" : : : "memory")
-#define WMB() __asm__ __volatile__ ("lwsync" : : : "memory")
-#define ISYNC() __asm__ __volatile__ ("isync" : : : "memory")
-
-
-/**********************************************************************
- *
- * Define constants for PowerPC 32
- *
- *********************************************************************/
-#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
-
-#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
-#define OPAL_HAVE_ATOMIC_SWAP_32 1
-#define OPAL_HAVE_ATOMIC_LLSC_32 1
-
-#define OPAL_HAVE_ATOMIC_MATH_32 1
-#define OPAL_HAVE_ATOMIC_ADD_32 1
-#define OPAL_HAVE_ATOMIC_AND_32 1
-#define OPAL_HAVE_ATOMIC_OR_32 1
-#define OPAL_HAVE_ATOMIC_XOR_32 1
-#define OPAL_HAVE_ATOMIC_SUB_32 1
-
-
-#if (OPAL_ASSEMBLY_ARCH == OPAL_POWERPC64) || OPAL_ASM_SUPPORT_64BIT
-#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
-#define OPAL_HAVE_ATOMIC_SWAP_64 1
-#define OPAL_HAVE_ATOMIC_LLSC_64 1
-#define OPAL_HAVE_ATOMIC_MATH_64 1
-#define OPAL_HAVE_ATOMIC_ADD_64 1
-#define OPAL_HAVE_ATOMIC_AND_64 1
-#define OPAL_HAVE_ATOMIC_OR_64 1
-#define OPAL_HAVE_ATOMIC_XOR_64 1
-#define OPAL_HAVE_ATOMIC_SUB_64 1
-#endif
-
+#include "opal/sys/powerpc/atomic_helper.h"
 
 /**********************************************************************
  *
  * Memory Barriers
  *
  *********************************************************************/
-#if OPAL_GCC_INLINE_ASSEMBLY
 
-static inline
-void opal_atomic_mb(void)
+static inline void opal_atomic_mb(void)
 {
-    MB();
+    __asm__ __volatile__("sync" : : : "memory");
 }
 
-
-static inline
-void opal_atomic_rmb(void)
+static inline void opal_atomic_rmb(void)
 {
-    RMB();
+    __asm__ __volatile__ ("isync" : : : "memory");
 }
 
-
-static inline
-void opal_atomic_wmb(void)
+static inline void opal_atomic_wmb(void)
 {
-    WMB();
+    __asm__ __volatile__("lwsync" : : : "memory");
 }
 
-static inline
-void opal_atomic_isync(void)
+static inline void opal_atomic_isync(void)
 {
-    ISYNC();
+    __asm__ __volatile__("isync" : : : "memory");
 }
 
-#endif /* end OPAL_GCC_INLINE_ASSEMBLY */
 
 /**********************************************************************
  *
- * Atomic math operations
+ * Compare and Swap
  *
  *********************************************************************/
-#if OPAL_GCC_INLINE_ASSEMBLY
-
-#if defined(__xlC__) || defined(__IBMC__) || defined(__IBMCPP__) || defined(__ibmxl__)
-/* work-around bizzare xlc bug in which it sign-extends
-   a pointer to a 32-bit signed integer */
-#define OPAL_ASM_ADDR(a) ((uintptr_t)a)
-#else
-#define OPAL_ASM_ADDR(a) (a)
-#endif
-
-#if defined(__PGI)
-/* work-around for bug in PGI 16.5-16.7 where the compiler fails to
- * correctly emit load instructions for 64-bit operands. without this
- * it will emit lwz instead of ld to load the 64-bit operand. */
-#define OPAL_ASM_VALUE64(x) (void *)(intptr_t) (x)
-#else
-#define OPAL_ASM_VALUE64(x) x
-#endif
-
-static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
+
+static inline bool opal_atomic_compare_exchange_strong_32(opal_atomic_int32_t *addr,
+                                                          int32_t *oldval, int32_t newval)
 {
     int32_t prev;
     bool ret;
 
-    __asm__ __volatile__ (
-                          "1: lwarx   %0, 0, %2  \n\t"
-                          "   cmpw    0, %0, %3  \n\t"
-                          "   bne-    2f         \n\t"
-                          "   stwcx.  %4, 0, %2  \n\t"
-                          "   bne-    1b         \n\t"
-                          "2:"
-                          : "=&r" (prev), "=m" (*addr)
-                          : "r" OPAL_ASM_ADDR(addr), "r" (*oldval), "r" (newval), "m" (*addr)
-                          : "cc", "memory");
+    __asm__ __volatile__("1: lwarx   %0, 0, %2  \n\t"
+                         "   cmpw    0, %0, %3  \n\t"
+                         "   bne-    2f         \n\t"
+                         "   stwcx.  %4, 0, %2  \n\t"
+                         "   bne-    1b         \n\t"
+                         "2:"
+                         : "=&r"(prev), "=m"(*addr)
+                         : "r" OPAL_ASM_ADDR(addr), "r"(*oldval), "r"(newval), "m"(*addr)
+                         : "cc", "memory");
 
     ret = (prev == *oldval);
     *oldval = prev;
     return ret;
 }
 
-/* NTH: the LL/SC support is done through macros due to issues with non-optimized builds. The reason
- * is that even with an always_inline attribute the compiler may still emit instructions to store then
- * load the arguments to/from the stack. This sequence may cause the ll reservation to be cancelled. */
-#define opal_atomic_ll_32(addr, ret)                                    \
-    do {                                                                \
-        opal_atomic_int32_t *_addr = (addr);                               \
-        int32_t _ret;                                                   \
-        __asm__ __volatile__ ("lwarx   %0, 0, %1  \n\t"                 \
-                              : "=&r" (_ret)                            \
-                              : "r" (_addr)                             \
-                              );                                        \
-        ret = (typeof(ret)) _ret;                                       \
-    } while (0)
-
-#define opal_atomic_sc_32(addr, value, ret)                             \
-    do {                                                                \
-        opal_atomic_int32_t *_addr = (addr);                               \
-        int32_t _ret, _foo, _newval = (int32_t) value;                  \
-                                                                        \
-        __asm__ __volatile__ ("   stwcx.  %4, 0, %3  \n\t"              \
-                              "   li      %0,0       \n\t"              \
-                              "   bne-    1f         \n\t"              \
-                              "   ori     %0,%0,1    \n\t"              \
-                              "1:"                                      \
-                              : "=r" (_ret), "=m" (*_addr), "=r" (_foo) \
-                              : "r" (_addr), "r" (_newval)              \
-                              : "cc", "memory");                        \
-        ret = _ret;                                                     \
-    } while (0)
-
-/* these two functions aren't inlined in the non-gcc case because then
-   there would be two function calls (since neither cmpset_32 nor
-   atomic_?mb can be inlined).  Instead, we "inline" them by hand in
-   the assembly, meaning there is one function call overhead instead
-   of two */
-static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
+static inline bool opal_atomic_compare_exchange_strong_acq_32(opal_atomic_int32_t *addr,
+                                                              int32_t *oldval, int32_t newval)
 {
     bool rc;
 
-    rc = opal_atomic_compare_exchange_strong_32 (addr, oldval, newval);
+    rc = opal_atomic_compare_exchange_strong_32(addr, oldval, newval);
     opal_atomic_rmb();
 
     return rc;
 }
 
-
-static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
+static inline bool opal_atomic_compare_exchange_strong_rel_32(opal_atomic_int32_t *addr,
+                                                              int32_t *oldval, int32_t newval)
 {
     opal_atomic_wmb();
-    return opal_atomic_compare_exchange_strong_32 (addr, oldval, newval);
-}
-
-static inline int32_t opal_atomic_swap_32(opal_atomic_int32_t *addr, int32_t newval)
-{
-    int32_t ret;
-
-    __asm__ __volatile__ ("1: lwarx   %0, 0, %2  \n\t"
-                          "   stwcx.  %3, 0, %2  \n\t"
-                          "   bne-    1b         \n\t"
-                          : "=&r" (ret), "=m" (*addr)
-                          : "r" (addr), "r" (newval)
-                          : "cc", "memory");
-
-   return ret;
-}
-
-#endif /* OPAL_GCC_INLINE_ASSEMBLY */
-
-
-#if (OPAL_ASSEMBLY_ARCH == OPAL_POWERPC64)
-
-#if  OPAL_GCC_INLINE_ASSEMBLY
-
-#define OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(type, instr)               \
-static inline int64_t opal_atomic_fetch_ ## type ## _64(opal_atomic_int64_t* v, int64_t val) \
-{                                                                       \
-    int64_t t, old;                                                     \
-                                                                        \
-    __asm__ __volatile__(                                               \
-                         "1:   ldarx   %1, 0, %4    \n\t"               \
-                         "     " #instr "     %0, %3, %1   \n\t"        \
-                         "     stdcx.  %0, 0, %4    \n\t"               \
-                         "     bne-    1b           \n\t"               \
-                         : "=&r" (t), "=&r" (old), "=m" (*v)            \
-                         : "r" (OPAL_ASM_VALUE64(val)), "r" OPAL_ASM_ADDR(v), "m" (*v) \
-                         : "cc");                                       \
-                                                                        \
-    return old;                                                         \
+    return opal_atomic_compare_exchange_strong_32(addr, oldval, newval);
 }
 
-OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(add, add)
-OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(and, and)
-OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(or, or)
-OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(xor, xor)
-OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(sub, subf)
 
-static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
+static inline bool opal_atomic_compare_exchange_strong_64(opal_atomic_int64_t *addr,
+                                                          int64_t *oldval, int64_t newval)
 {
     int64_t prev;
     bool ret;
 
-    __asm__ __volatile__ (
-                          "1: ldarx   %0, 0, %2  \n\t"
-                          "   cmpd    0, %0, %3  \n\t"
-                          "   bne-    2f         \n\t"
-                          "   stdcx.  %4, 0, %2  \n\t"
-                          "   bne-    1b         \n\t"
-                          "2:"
-                          : "=&r" (prev), "=m" (*addr)
-                          : "r" (addr), "r" (OPAL_ASM_VALUE64(*oldval)), "r" (OPAL_ASM_VALUE64(newval)), "m" (*addr)
-                          : "cc", "memory");
+    __asm__ __volatile__("1: ldarx   %0, 0, %2  \n\t"
+                         "   cmpd    0, %0, %3  \n\t"
+                         "   bne-    2f         \n\t"
+                         "   stdcx.  %4, 0, %2  \n\t"
+                         "   bne-    1b         \n\t"
+                         "2:"
+                         : "=&r"(prev), "=m"(*addr)
+                         : "r"(addr), "r"(OPAL_ASM_VALUE64(*oldval)), "r"(OPAL_ASM_VALUE64(newval)),
+                           "m"(*addr)
+                         : "cc", "memory");
 
     ret = (prev == *oldval);
     *oldval = prev;
     return ret;
 }
 
-#define opal_atomic_ll_64(addr, ret)                                    \
-    do {                                                                \
-        opal_atomic_int64_t *_addr = (addr);                               \
-        int64_t _ret;                                                   \
-        __asm__ __volatile__ ("ldarx   %0, 0, %1  \n\t"                 \
-                              : "=&r" (_ret)                            \
-                              : "r" (_addr)                             \
-                              );                                        \
-        ret = (typeof(ret)) _ret;                                       \
-    } while (0)
-
-#define opal_atomic_sc_64(addr, value, ret)                             \
-    do {                                                                \
-        opal_atomic_int64_t *_addr = (addr);                               \
-        int64_t _newval = (int64_t) value;                              \
-        int32_t _ret;                                                   \
-                                                                        \
-        __asm__ __volatile__ ("   stdcx.  %2, 0, %1  \n\t"              \
-                              "   li      %0,0       \n\t"              \
-                              "   bne-    1f         \n\t"              \
-                              "   ori     %0,%0,1    \n\t"              \
-                              "1:"                                      \
-                              : "=r" (_ret)                             \
-                              : "r" (_addr), "r" (OPAL_ASM_VALUE64(_newval)) \
-                              : "cc", "memory");                        \
-        ret = _ret;                                                     \
-    } while (0)
-
-static inline int64_t opal_atomic_swap_64(opal_atomic_int64_t *addr, int64_t newval)
+static inline bool opal_atomic_compare_exchange_strong_acq_64(opal_atomic_int64_t *addr,
+                                                              int64_t *oldval, int64_t newval)
 {
-   int64_t ret;
+    bool rc;
 
-   __asm__ __volatile__ ("1: ldarx   %0, 0, %2  \n\t"
-                         "   stdcx.  %3, 0, %2  \n\t"
-                         "   bne-    1b         \n\t"
-                         : "=&r" (ret), "=m" (*addr)
-                         : "r" (addr), "r" (OPAL_ASM_VALUE64(newval))
-                         : "cc", "memory");
+    rc = opal_atomic_compare_exchange_strong_64(addr, oldval, newval);
+    opal_atomic_rmb();
 
-   return ret;
+    return rc;
 }
 
-#endif /* OPAL_GCC_INLINE_ASSEMBLY */
+static inline bool opal_atomic_compare_exchange_strong_rel_64(opal_atomic_int64_t *addr,
+                                                              int64_t *oldval, int64_t newval)
+{
+    opal_atomic_wmb();
+    return opal_atomic_compare_exchange_strong_64(addr, oldval, newval);
+}
 
-#elif (OPAL_ASSEMBLY_ARCH == OPAL_POWERPC32) && OPAL_ASM_SUPPORT_64BIT
+#include "opal/sys/atomic_impl_ptr_cswap.h"
 
-#ifndef ll_low /* GLIBC provides these somewhere, so protect */
-#define ll_low(x)       *(((unsigned int*)&(x))+0)
-#define ll_high(x)      *(((unsigned int*)&(x))+1)
-#endif
 
-#if  OPAL_GCC_INLINE_ASSEMBLY
+/**********************************************************************
+ *
+ * Swap
+ *
+ *********************************************************************/
 
-static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
+static inline int32_t opal_atomic_swap_32(opal_atomic_int32_t *addr, int32_t newval)
 {
-    int64_t prev;
-    int ret;
-
-    /*
-     * We force oldval and newval into memory because PPC doesn't
-     * appear to have a way to do a move register with offset.  Since
-     * this is 32-bit code, a 64 bit integer will be loaded into two
-     * registers (assuming no inlining, addr will be in r3, oldval
-     * will be in r4 and r5, and newval will be r6 and r7.  We need
-     * to load the whole thing into one register.  So we have the
-     * compiler push the values into memory and load the double word
-     * into registers.  We use r4,r5 so that the main block of code
-     * is very similar to the pure 64 bit version.
-     */
-   __asm__ __volatile__ (
-                         "ld r4,%3         \n\t"
-                         "ld r5,%4        \n\t"
-                         "1: ldarx   %1, 0, %2  \n\t"
-                         "   cmpd    0, %1, r4  \n\t"
-                         "   bne-    2f         \n\t"
-                         "   stdcx.  r5, 0, %2  \n\t"
+    int32_t ret;
+
+    __asm__ __volatile__("1: lwarx   %0, 0, %2  \n\t"
+                         "   stwcx.  %3, 0, %2  \n\t"
                          "   bne-    1b         \n\t"
-                         "2:                    \n\t"
-                         "xor r5,r4,%1          \n\t"
-                         "subfic r9,r5,0        \n\t"
-                         "adde %0,r9,r5         \n\t"
-                         : "=&r" (ret), "+r" (prev)
-                         : "r"OPAL_ASM_ADDR(addr),
-                           "m"(*oldval), "m"(newval)
-                         : "r4", "r5", "r9", "cc", "memory");
-   *oldval = prev;
-   return (bool) ret;
+                         : "=&r"(ret), "=m"(*addr)
+                         : "r"(addr), "r"(newval)
+                         : "cc", "memory");
+
+    return ret;
 }
 
-#endif /* OPAL_GCC_INLINE_ASSEMBLY */
+static inline int64_t opal_atomic_swap_64(opal_atomic_int64_t *addr, int64_t newval)
+{
+    int64_t ret;
 
-#endif /* OPAL_ASM_SUPPORT_64BIT */
+    __asm__ __volatile__("1: ldarx   %0, 0, %2  \n\t"
+                         "   stdcx.  %3, 0, %2  \n\t"
+                         "   bne-    1b         \n\t"
+                         : "=&r"(ret), "=m"(*addr)
+                         : "r"(addr), "r"(OPAL_ASM_VALUE64(newval))
+                         : "cc", "memory");
 
-#if OPAL_GCC_INLINE_ASSEMBLY
+    return ret;
+}
 
-/* these two functions aren't inlined in the non-gcc case because then
-   there would be two function calls (since neither cmpset_64 nor
-   atomic_?mb can be inlined).  Instead, we "inline" them by hand in
-   the assembly, meaning there is one function call overhead instead
-   of two */
-static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
-{
-    bool rc;
+#include "opal/sys/atomic_impl_ptr_swap.h"
 
-    rc = opal_atomic_compare_exchange_strong_64 (addr, oldval, newval);
-    opal_atomic_rmb();
 
-    return rc;
-}
+/**********************************************************************
+ *
+ * Atomic spinlocks
+ *
+ *********************************************************************/
 
+#include "opal/sys/atomic_impl_spinlock.h"
 
-static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
-{
-    opal_atomic_wmb();
-    return opal_atomic_compare_exchange_strong_64 (addr, oldval, newval);
-}
 
+/**********************************************************************
+ *
+ * Atomic math operations
+ *
+ *********************************************************************/
 
-#define OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_32(type, instr)               \
-static inline int32_t opal_atomic_fetch_ ## type ## _32(opal_atomic_int32_t* v, int val) \
-{                                                                       \
-    int32_t t, old;                                                     \
-                                                                        \
-    __asm__ __volatile__(                                               \
-                         "1:   lwarx   %1, 0, %4    \n\t"               \
-                         "     " #instr "     %0, %3, %1   \n\t"        \
-                         "     stwcx.  %0, 0, %4    \n\t"               \
-                         "     bne-    1b           \n\t"               \
-                         : "=&r" (t), "=&r" (old), "=m" (*v)            \
-                         : "r" (val), "r" OPAL_ASM_ADDR(v), "m" (*v)    \
-                         : "cc");                                       \
-                                                                        \
-    return old;                                                         \
-}
+#define OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_32(type, instr)                                \
+    static inline int32_t opal_atomic_fetch_##type##_32(opal_atomic_int32_t *v, int val) \
+    {                                                                                    \
+        int32_t newval, old;                                                             \
+                                                                                         \
+        __asm__ __volatile__("1:   lwarx   %1, 0, %4    \n\t"                            \
+                             "     " #instr "     %0, %3, %1   \n\t"                     \
+                             "     stwcx.  %0, 0, %4    \n\t"                            \
+                             "     bne-    1b           \n\t"                            \
+                             : "=&r"(newval), "=&r"(old), "=m"(*v)                       \
+                             : "r"(val), "r" OPAL_ASM_ADDR(v), "m"(*v)                   \
+                             : "cc");                                                    \
+                                                                                         \
+        return old;                                                                      \
+    }                                                                                    \
+    static inline int32_t opal_atomic_##type##_fetch_32(opal_atomic_int32_t *v, int val) \
+    {                                                                                    \
+        int32_t newval, old;                                                             \
+                                                                                         \
+        __asm__ __volatile__("1:   lwarx   %1, 0, %4    \n\t"                            \
+                             "     " #instr "     %0, %3, %1   \n\t"                     \
+                             "     stwcx.  %0, 0, %4    \n\t"                            \
+                             "     bne-    1b           \n\t"                            \
+                             : "=&r"(newval), "=&r"(old), "=m"(*v)                       \
+                             : "r"(val), "r" OPAL_ASM_ADDR(v), "m"(*v)                   \
+                             : "cc");                                                    \
+                                                                                         \
+        return newval;                                                                   \
+    }
 
 OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_32(add, add)
 OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_32(and, and)
@@ -403,6 +236,43 @@ OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_32(or, or)
 OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_32(xor, xor)
 OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_32(sub, subf)
 
-#endif /* OPAL_GCC_INLINE_ASSEMBLY */
+#define OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(type, instr)                                    \
+    static inline int64_t opal_atomic_fetch_##type##_64(opal_atomic_int64_t *v, int64_t val) \
+    {                                                                                        \
+        int64_t newval, old;                                                                 \
+                                                                                             \
+        __asm__ __volatile__("1:   ldarx   %1, 0, %4    \n\t"                                \
+                             "     " #instr "     %0, %3, %1   \n\t"                         \
+                             "     stdcx.  %0, 0, %4    \n\t"                                \
+                             "     bne-    1b           \n\t"                                \
+                             : "=&r"(newval), "=&r"(old), "=m"(*v)                           \
+                             : "r"(OPAL_ASM_VALUE64(val)), "r" OPAL_ASM_ADDR(v), "m"(*v)     \
+                             : "cc");                                                        \
+                                                                                             \
+        return old;                                                                          \
+    }                                                                                        \
+    static inline int64_t opal_atomic_##type##_fetch_64(opal_atomic_int64_t *v, int64_t val) \
+    {                                                                                        \
+        int64_t newval, old;                                                                 \
+                                                                                             \
+        __asm__ __volatile__("1:   ldarx   %1, 0, %4    \n\t"                                \
+                             "     " #instr "     %0, %3, %1   \n\t"                         \
+                             "     stdcx.  %0, 0, %4    \n\t"                                \
+                             "     bne-    1b           \n\t"                                \
+                             : "=&r"(newval), "=&r"(old), "=m"(*v)                           \
+                             : "r"(OPAL_ASM_VALUE64(val)), "r" OPAL_ASM_ADDR(v), "m"(*v)     \
+                             : "cc");                                                        \
+                                                                                             \
+        return newval;                                                                       \
+    }
+
+OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(add, add)
+OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(and, and)
+OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(or, or)
+OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(xor, xor)
+OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(sub, subf)
+
+#include "opal/sys/atomic_impl_minmax_math.h"
+#include "opal/sys/atomic_impl_size_t_math.h"
 
 #endif /* ! OPAL_SYS_ARCH_ATOMIC_H */
diff --git a/opal/include/opal/sys/powerpc/atomic_helper.h b/opal/include/opal/sys/powerpc/atomic_helper.h
new file mode 100644
index 00000000000..bcbb3210dd8
--- /dev/null
+++ b/opal/include/opal/sys/powerpc/atomic_helper.h
@@ -0,0 +1,44 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ > * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2021 IBM Corporation.  All rights reserved.
+ * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2021      Google, LLC. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef OPAL_SYS_ARCH_ATOMIC_HELPER_H
+#define OPAL_SYS_ARCH_ATOMIC_HELPER_H 1
+
+#if defined(__xlC__) || defined(__IBMC__) || defined(__IBMCPP__) || defined(__ibmxl__)
+/* work-around bizarre xlc bug in which it sign-extends
+   a pointer to a 32-bit signed integer */
+#    define OPAL_ASM_ADDR(a) ((uintptr_t) a)
+#else
+#    define OPAL_ASM_ADDR(a) (a)
+#endif
+
+#if defined(__PGI)
+/* work-around for bug in PGI 16.5-16.7 where the compiler fails to
+ * correctly emit load instructions for 64-bit operands. without this
+ * it will emit lwz instead of ld to load the 64-bit operand. */
+#    define OPAL_ASM_VALUE64(x) (void *) (intptr_t)(x)
+#else
+#    define OPAL_ASM_VALUE64(x) x
+#endif
+
+#endif /* OPAL_SYS_ARCH_ATOMIC_HELPER_H */
diff --git a/opal/include/opal/sys/powerpc/atomic_llsc.h b/opal/include/opal/sys/powerpc/atomic_llsc.h
new file mode 100644
index 00000000000..9e6c7aa5863
--- /dev/null
+++ b/opal/include/opal/sys/powerpc/atomic_llsc.h
@@ -0,0 +1,94 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2021 IBM Corporation.  All rights reserved.
+ * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2021      Google, LLC. All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef OPAL_SYS_ARCH_ATOMIC_LLSC_H
+#define OPAL_SYS_ARCH_ATOMIC_LLSC_H 1
+
+/*
+ * this file is included even when C11 or GCC built-in atomics are
+ * used, which is why we must check for gcc inline assembly support.
+ */
+
+#if OPAL_C_GCC_INLINE_ASSEMBLY
+
+#include "opal/sys/powerpc/atomic_helper.h"
+
+#define OPAL_HAVE_ATOMIC_LLSC_32             1
+#define OPAL_HAVE_ATOMIC_LLSC_64             1
+
+/* NTH: the LL/SC support is done through macros due to issues with non-optimized builds. The reason
+ * is that even with an always_inline attribute the compiler may still emit instructions to store
+ * then load the arguments to/from the stack. This sequence may cause the ll reservation to be
+ * cancelled. */
+#define opal_atomic_ll_32(addr, ret)                                                \
+    do {                                                                            \
+        opal_atomic_int32_t *_addr = (addr);                                        \
+        __asm__ __volatile__("lwarx   %0, 0, %1  \n\t" : "=&r"(ret) : "r"(_addr));  \
+    } while (0)
+
+#define opal_atomic_sc_32(addr, value, ret)                         \
+    do {                                                            \
+        opal_atomic_int32_t *_addr = (addr);                        \
+        int32_t _ret, _foo, _newval = (int32_t) value;              \
+                                                                    \
+        __asm__ __volatile__("   stwcx.  %4, 0, %3  \n\t"           \
+                             "   li      %0,0       \n\t"           \
+                             "   bne-    1f         \n\t"           \
+                             "   ori     %0,%0,1    \n\t"           \
+                             "1:"                                   \
+                             : "=r"(_ret), "=m"(*_addr), "=r"(_foo) \
+                             : "r"(_addr), "r"(_newval)             \
+                             : "cc", "memory");                     \
+        ret = _ret;                                                 \
+   } while (0)
+
+#define opal_atomic_ll_64(addr, ret)                                                \
+    do {                                                                            \
+        opal_atomic_int64_t *_addr = (addr);                                        \
+        __asm__ __volatile__("ldarx   %0, 0, %1  \n\t" : "=&r"(ret) : "r"(_addr));  \
+    } while (0)
+
+#define opal_atomic_sc_64(addr, value, ret)                               \
+    do {                                                                  \
+        opal_atomic_int64_t *_addr = (addr);                              \
+        int64_t _newval = (int64_t) value;                                \
+        int32_t _ret;                                                     \
+                                                                          \
+        __asm__ __volatile__("   stdcx.  %2, 0, %1  \n\t"                 \
+                             "   li      %0,0       \n\t"                 \
+                             "   bne-    1f         \n\t"                 \
+                             "   ori     %0,%0,1    \n\t"                 \
+                             "1:"                                         \
+                             : "=r"(_ret)                                 \
+                             : "r"(_addr), "r"(OPAL_ASM_VALUE64(_newval)) \
+                             : "cc", "memory");                           \
+        ret = _ret;                                                       \
+    } while (0)
+
+#include "opal/sys/atomic_impl_ptr_llsc.h"
+
+#endif /* OPAL_C_GCC_INLINE_ASSEMBLY */
+
+#endif /* OPAL_SYS_ARCH_ATOMIC_LLSC_H */
diff --git a/opal/include/opal/sys/powerpc/timer.h b/opal/include/opal/sys/powerpc/timer.h
index 09d9f2cf89a..216b7da01e7 100644
--- a/opal/include/opal/sys/powerpc/timer.h
+++ b/opal/include/opal/sys/powerpc/timer.h
@@ -9,6 +9,8 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -19,32 +21,29 @@
 #ifndef OPAL_SYS_ARCH_TIMER_H
 #define OPAL_SYS_ARCH_TIMER_H 1
 
-
 typedef uint64_t opal_timer_t;
 
+#if OPAL_C_GCC_INLINE_ASSEMBLY
 
-#if OPAL_GCC_INLINE_ASSEMBLY
-
-static inline opal_timer_t
-opal_sys_timer_get_cycles(void)
+static inline opal_timer_t opal_sys_timer_get_cycles(void)
 {
     unsigned int tbl, tbu0, tbu1;
 
     do {
-        __asm__ __volatile__ ("mftbu %0" : "=r"(tbu0));
-        __asm__ __volatile__ ("mftb %0" : "=r"(tbl));
-        __asm__ __volatile__ ("mftbu %0" : "=r"(tbu1));
+        __asm__ __volatile__("mftbu %0" : "=r"(tbu0));
+        __asm__ __volatile__("mftb %0" : "=r"(tbl));
+        __asm__ __volatile__("mftbu %0" : "=r"(tbu1));
     } while (tbu0 != tbu1);
 
-    return (((unsigned long long)tbu0) << 32) | tbl;
+    return (((unsigned long long) tbu0) << 32) | tbl;
 }
 
-#define OPAL_HAVE_SYS_TIMER_GET_CYCLES 1
+#    define OPAL_HAVE_SYS_TIMER_GET_CYCLES 1
 
 #else
 
-#define OPAL_HAVE_SYS_TIMER_GET_CYCLES 0
+#    define OPAL_HAVE_SYS_TIMER_GET_CYCLES 0
 
-#endif /* OPAL_GCC_INLINE_ASSEMBLY */
+#endif /* OPAL_C_GCC_INLINE_ASSEMBLY */
 
 #endif /* ! OPAL_SYS_ARCH_TIMER_H */
diff --git a/opal/include/opal/sys/powerpc/update.sh b/opal/include/opal/sys/powerpc/update.sh
deleted file mode 100644
index 095868d4fb5..00000000000
--- a/opal/include/opal/sys/powerpc/update.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/bin/sh
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-CFILE=/tmp/opal_asm_$$.c
-
-trap "/bin/rm -f $CFILE; exit 0" 0 1 2 15
-
-echo Updating asm.s from atomic.h and timer.h using gcc
-
-cat > $CFILE<<EOF
-#include <stdlib.h>
-#include <inttypes.h>
-#define static
-#define inline
-#define OPAL_GCC_INLINE_ASSEMBLY 1
-#include "../architecture.h"
-#include "atomic.h"
-#include "timer.h"
-EOF
-
-gcc -O1 -mpowerpc64 -mcpu=970 -DOPAL_ASSEMBLY_ARCH=POWERPC32 -DOPAL_ASM_SUPPORT_64BIT=1 -I. -S $CFILE -o asm-32-64.s
-gcc -O1 -DOPAL_ASSEMBLY_ARCH=OPAL_POWERPC32 -DOPAL_ASM_SUPPORT_64BIT=0 -I. -S $CFILE -o asm-32.s
-gcc -m64 -O1 -finline-functions -DOPAL_ASSEMBLY_ARCH=OPAL_POWERPC64 -DOPAL_ASM_SUPPORT64BIT=1 -I. -S $CFILE -o asm-64.s
diff --git a/opal/include/opal/sys/timer.h b/opal/include/opal/sys/timer.h
index 9cf844192e0..3f63839a48c 100644
--- a/opal/include/opal/sys/timer.h
+++ b/opal/include/opal/sys/timer.h
@@ -15,6 +15,8 @@
  *                         reserved.
  * Copyright (c) 2020      Intel, Inc.  All rights reserved.
  * Copyright (c) 2020      Google, LLC. All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -31,29 +33,12 @@
 #ifndef OPAL_SYS_TIMER_H
 #define OPAL_SYS_TIMER_H 1
 
-#include "opal_config.h"
-
-#include "opal/sys/architecture.h"
+#include "opal/opal_portable_platform.h"
 
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-
-/* do some quick #define cleanup in cases where we are doing
-   testing... */
-#ifdef OPAL_DISABLE_INLINE_ASM
-#undef OPAL_C_GCC_INLINE_ASSEMBLY
-#define OPAL_C_GCC_INLINE_ASSEMBLY 0
+#    include <sys/types.h>
 #endif
 
-/* define OPAL_{GCC,DEC,XLC}_INLINE_ASSEMBLY based on the
-   OPAL_{C,CXX}_{GCC,DEC,XLC}_INLINE_ASSEMBLY defines and whether we
-   are in C or C++ */
-#if defined(c_plusplus) || defined(__cplusplus)
-#define OPAL_GCC_INLINE_ASSEMBLY OPAL_CXX_GCC_INLINE_ASSEMBLY
-#else
-#define OPAL_GCC_INLINE_ASSEMBLY OPAL_C_GCC_INLINE_ASSEMBLY
-#endif
 
 /**********************************************************************
  *
@@ -72,37 +57,31 @@ BEGIN_C_DECLS
 
 #if defined(DOXYGEN)
 /* don't include system-level gorp when generating doxygen files */
-#elif OPAL_ASSEMBLY_ARCH == OPAL_X86_64
-#include "opal/sys/x86_64/timer.h"
-#elif OPAL_ASSEMBLY_ARCH == OPAL_ARM
-#include "opal/sys/arm/timer.h"
-#elif OPAL_ASSEMBLY_ARCH == OPAL_ARM64
-#include "opal/sys/arm64/timer.h"
-#elif OPAL_ASSEMBLY_ARCH == OPAL_IA32
-#include "opal/sys/ia32/timer.h"
-#elif OPAL_ASSEMBLY_ARCH == OPAL_POWERPC32
-#include "opal/sys/powerpc/timer.h"
-#elif OPAL_ASSEMBLY_ARCH == OPAL_POWERPC64
-#include "opal/sys/powerpc/timer.h"
+#elif defined(PLATFORM_ARCH_X86_64) || defined(PLATFORM_ARCH_X86)
+#    include "opal/sys/x86_64/timer.h"
+#elif defined(PLATFORM_ARCH_ARM) || defined(PLATFORM_ARCH_AARCH64)
+#    include "opal/sys/arm64/timer.h"
+#elif defined(PLATFORM_ARCH_POWERPC)
+#    include "opal/sys/powerpc/timer.h"
 #endif
 
 #ifndef DOXYGEN
-#ifndef OPAL_HAVE_SYS_TIMER_GET_CYCLES
-#define OPAL_HAVE_SYS_TIMER_GET_CYCLES 0
+#    ifndef OPAL_HAVE_SYS_TIMER_GET_CYCLES
+#        define OPAL_HAVE_SYS_TIMER_GET_CYCLES 0
 
 typedef long opal_timer_t;
-#endif
+#    endif
 
-#ifndef OPAL_HAVE_SYS_TIMER_GET_FREQ
-#define OPAL_HAVE_SYS_TIMER_GET_FREQ 0
-#endif
+#    ifndef OPAL_HAVE_SYS_TIMER_GET_FREQ
+#        define OPAL_HAVE_SYS_TIMER_GET_FREQ 0
+#    endif
 #endif
 
 #ifndef OPAL_HAVE_SYS_TIMER_IS_MONOTONIC
 
-#define OPAL_HAVE_SYS_TIMER_IS_MONOTONIC 1
+#    define OPAL_HAVE_SYS_TIMER_IS_MONOTONIC 1
 
-static inline bool opal_sys_timer_is_monotonic (void)
+static inline bool opal_sys_timer_is_monotonic(void)
 {
     return OPAL_TIMER_MONOTONIC;
 }
diff --git a/opal/include/opal/sys/x86_64/atomic.h b/opal/include/opal/sys/x86_64/atomic.h
index 8853eb5c2b9..f16133bda25 100644
--- a/opal/include/opal/sys/x86_64/atomic.h
+++ b/opal/include/opal/sys/x86_64/atomic.h
@@ -10,11 +10,14 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2007      Sun Microsystems, Inc.  All rights reserverd.
+ * Copyright (c) 2007      Sun Microsystems, Inc.  All rights reserved.
  * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2016-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2021      Google, LLC. All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -28,249 +31,220 @@
  * On x86_64, we use cmpxchg.
  */
 
-
 #define SMPLOCK "lock; "
-#define MB() __asm__ __volatile__("": : :"memory")
-
-
-/**********************************************************************
- *
- * Define constants for AMD64 / x86_64 / EM64T / ...
- *
- *********************************************************************/
-#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
 
-#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
-
-#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
 
 /**********************************************************************
  *
  * Memory Barriers
  *
  *********************************************************************/
-#if OPAL_GCC_INLINE_ASSEMBLY
 
 static inline void opal_atomic_mb(void)
 {
-    MB();
+    __asm__ __volatile__("mfence": : :"memory");
 }
 
-
 static inline void opal_atomic_rmb(void)
 {
-    MB();
+    __asm__ __volatile__("" : : : "memory");
 }
 
-
 static inline void opal_atomic_wmb(void)
 {
-    MB();
+    __asm__ __volatile__("" : : : "memory");
 }
 
 static inline void opal_atomic_isync(void)
 {
 }
 
-#endif /* OPAL_GCC_INLINE_ASSEMBLY */
-
 
 /**********************************************************************
  *
- * Atomic math operations
+ * Compare and Swap
  *
  *********************************************************************/
-#if OPAL_GCC_INLINE_ASSEMBLY
 
-static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
+static inline bool opal_atomic_compare_exchange_strong_32(opal_atomic_int32_t *addr,
+                                                          int32_t *oldval, int32_t newval)
 {
-   unsigned char ret;
-   __asm__ __volatile__ (
-                       SMPLOCK "cmpxchgl %3,%2   \n\t"
-                               "sete     %0      \n\t"
-                       : "=qm" (ret), "+a" (*oldval), "+m" (*addr)
-                       : "q"(newval)
-                       : "memory", "cc");
-
-   return (bool) ret;
-}
+    unsigned char ret;
+    __asm__ __volatile__(SMPLOCK "cmpxchgl %3,%2   \n\t"
+                                 "sete     %0      \n\t"
+                         : "=qm"(ret), "+a"(*oldval), "+m"(*addr)
+                         : "q"(newval)
+                         : "memory", "cc");
 
-#endif /* OPAL_GCC_INLINE_ASSEMBLY */
+    return (bool) ret;
+}
 
 #define opal_atomic_compare_exchange_strong_acq_32 opal_atomic_compare_exchange_strong_32
 #define opal_atomic_compare_exchange_strong_rel_32 opal_atomic_compare_exchange_strong_32
 
-#if OPAL_GCC_INLINE_ASSEMBLY
-
-static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
+static inline bool opal_atomic_compare_exchange_strong_64(opal_atomic_int64_t *addr,
+                                                          int64_t *oldval, int64_t newval)
 {
-   unsigned char ret;
-   __asm__ __volatile__ (
-                       SMPLOCK "cmpxchgq %3,%2   \n\t"
-                               "sete     %0      \n\t"
-                       : "=qm" (ret), "+a" (*oldval), "+m" (*((opal_atomic_long_t *)addr))
-                       : "q"(newval)
-                       : "memory", "cc"
-                       );
-
-   return (bool) ret;
-}
+    unsigned char ret;
+    __asm__ __volatile__(SMPLOCK "cmpxchgq %3,%2   \n\t"
+                                 "sete     %0      \n\t"
+                         : "=qm"(ret), "+a"(*oldval), "+m"(*((opal_atomic_long_t *) addr))
+                         : "q"(newval)
+                         : "memory", "cc");
 
-#endif /* OPAL_GCC_INLINE_ASSEMBLY */
+    return (bool) ret;
+}
 
 #define opal_atomic_compare_exchange_strong_acq_64 opal_atomic_compare_exchange_strong_64
 #define opal_atomic_compare_exchange_strong_rel_64 opal_atomic_compare_exchange_strong_64
 
-#if OPAL_GCC_INLINE_ASSEMBLY && OPAL_HAVE_CMPXCHG16B && HAVE_OPAL_INT128_T
+#include "opal/sys/atomic_impl_ptr_cswap.h"
 
-static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_int128_t *addr, opal_int128_t *oldval, opal_int128_t newval)
+#if OPAL_HAVE_CMPXCHG16B && HAVE_OPAL_INT128_T
+
+static inline bool opal_atomic_compare_exchange_strong_128(opal_atomic_int128_t *addr,
+                                                           opal_int128_t *oldval,
+                                                           opal_int128_t newval)
 {
     unsigned char ret;
 
     /* cmpxchg16b compares the value at the address with eax:edx (low:high). if the values are
      * the same the contents of ebx:ecx are stores at the address. in all cases the value stored
      * at the address is returned in eax:edx. */
-    __asm__ __volatile__ (SMPLOCK "cmpxchg16b (%%rsi)   \n\t"
-                                  "sete     %0      \n\t"
-                          : "=qm" (ret), "+a" (((int64_t *)oldval)[0]), "+d" (((int64_t *)oldval)[1])
-                          : "S" (addr), "b" (((int64_t *)&newval)[0]), "c" (((int64_t *)&newval)[1])
-                          : "memory", "cc");
+    __asm__ __volatile__(SMPLOCK "cmpxchg16b (%%rsi)   \n\t"
+                                 "sete     %0      \n\t"
+                         : "=qm"(ret), "+a"(((int64_t *) oldval)[0]), "+d"(((int64_t *) oldval)[1])
+                         : "S"(addr), "b"(((int64_t *) &newval)[0]), "c"(((int64_t *) &newval)[1])
+                         : "memory", "cc");
 
     return (bool) ret;
 }
 
-#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
-
-#endif /* OPAL_GCC_INLINE_ASSEMBLY */
+#    define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
 
+#endif
 
-#if OPAL_GCC_INLINE_ASSEMBLY
 
-#define OPAL_HAVE_ATOMIC_SWAP_32 1
-
-#define OPAL_HAVE_ATOMIC_SWAP_64 1
+/**********************************************************************
+ *
+ * Swap
+ *
+ *********************************************************************/
 
-static inline int32_t opal_atomic_swap_32( opal_atomic_int32_t *addr,
-					   int32_t newval)
+static inline int32_t opal_atomic_swap_32(opal_atomic_int32_t *addr, int32_t newval)
 {
     int32_t oldval;
 
-    __asm__ __volatile__("xchg %1, %0" :
-			 "=r" (oldval), "+m" (*addr) :
-			 "0" (newval) :
-			 "memory");
+    __asm__ __volatile__("xchg %1, %0" : "=r"(oldval), "+m"(*addr) : "0"(newval) : "memory");
     return oldval;
 }
 
-#endif /* OPAL_GCC_INLINE_ASSEMBLY */
-
-#if OPAL_GCC_INLINE_ASSEMBLY
-
-static inline int64_t opal_atomic_swap_64( opal_atomic_int64_t *addr,
-                                           int64_t newval)
+static inline int64_t opal_atomic_swap_64(opal_atomic_int64_t *addr, int64_t newval)
 {
     int64_t oldval;
 
-    __asm__ __volatile__("xchgq %1, %0" :
-			 "=r" (oldval), "+m" (*addr) :
-			 "0" (newval) :
-			 "memory");
+    __asm__ __volatile__("xchgq %1, %0" : "=r"(oldval), "+m"(*addr) : "0"(newval) : "memory");
     return oldval;
 }
 
-#endif /* OPAL_GCC_INLINE_ASSEMBLY */
+#include "opal/sys/atomic_impl_ptr_swap.h"
 
 
+/**********************************************************************
+ *
+ * Atomic spinlocks
+ *
+ *********************************************************************/
 
-#if OPAL_GCC_INLINE_ASSEMBLY
-
-#define OPAL_HAVE_ATOMIC_MATH_32 1
-#define OPAL_HAVE_ATOMIC_MATH_64 1
+#include "opal/sys/atomic_impl_spinlock.h"
 
-#define OPAL_HAVE_ATOMIC_ADD_32 1
 
-/**
- * atomic_add - add integer to atomic variable
- * @i: integer value to add
- * @v: pointer of type int
+/**********************************************************************
  *
- * Atomically adds @i to @v.
- */
-static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t* v, int i)
+ * Atomic math operations
+ *
+ *********************************************************************/
+
+static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t *v, int i)
 {
     int ret = i;
-   __asm__ __volatile__(
-                        SMPLOCK "xaddl %1,%0"
-                        :"+m" (*v), "+r" (ret)
-                        :
-                        :"memory", "cc"
-                        );
-   return ret;
+    __asm__ __volatile__(SMPLOCK "xaddl %1,%0" : "+m"(*v), "+r"(ret) : : "memory", "cc");
+    return ret;
 }
 
-#define OPAL_HAVE_ATOMIC_ADD_64 1
+static inline int32_t opal_atomic_add_fetch_32(opal_atomic_int32_t *v, int i)
+{
+    return opal_atomic_fetch_add_32(v, i) + i;
+}
 
-/**
- * atomic_add - add integer to atomic variable
- * @i: integer value to add
- * @v: pointer of type int
- *
- * Atomically adds @i to @v.
- */
-static inline int64_t opal_atomic_fetch_add_64(opal_atomic_int64_t* v, int64_t i)
+static inline int64_t opal_atomic_fetch_add_64(opal_atomic_int64_t *v, int64_t i)
 {
     int64_t ret = i;
-   __asm__ __volatile__(
-                        SMPLOCK "xaddq %1,%0"
-                        :"+m" (*v), "+r" (ret)
-                        :
-                        :"memory", "cc"
-                        );
-   return ret;
+    __asm__ __volatile__(SMPLOCK "xaddq %1,%0" : "+m"(*v), "+r"(ret) : : "memory", "cc");
+    return ret;
 }
 
-#define OPAL_HAVE_ATOMIC_SUB_32 1
+static inline int64_t opal_atomic_add_fetch_64(opal_atomic_int64_t *v, int64_t i)
+{
+    return opal_atomic_fetch_add_64(v, i) + i;
+}
 
-/**
- * atomic_sub - subtract the atomic variable
- * @i: integer value to subtract
- * @v: pointer of type int
- *
- * Atomically subtracts @i from @v.
- */
-static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t* v, int i)
+static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t *v, int i)
 {
     int ret = -i;
-   __asm__ __volatile__(
-                        SMPLOCK "xaddl %1,%0"
-                        :"+m" (*v), "+r" (ret)
-                        :
-                        :"memory", "cc"
-                        );
-   return ret;
+    __asm__ __volatile__(SMPLOCK "xaddl %1,%0" : "+m"(*v), "+r"(ret) : : "memory", "cc");
+    return ret;
 }
 
-#define OPAL_HAVE_ATOMIC_SUB_64 1
+static inline int32_t opal_atomic_sub_fetch_32(opal_atomic_int32_t *v, int i)
+{
+    return opal_atomic_fetch_sub_32(v, i) - i;
+}
 
-/**
- * atomic_sub - subtract the atomic variable
- * @i: integer value to subtract
- * @v: pointer of type int
- *
- * Atomically subtracts @i from @v.
- */
-static inline int64_t opal_atomic_fetch_sub_64(opal_atomic_int64_t* v, int64_t i)
+static inline int64_t opal_atomic_fetch_sub_64(opal_atomic_int64_t *v, int64_t i)
 {
     int64_t ret = -i;
-   __asm__ __volatile__(
-                        SMPLOCK "xaddq %1,%0"
-                        :"+m" (*v), "+r" (ret)
-                        :
-                        :"memory", "cc"
-                        );
-   return ret;
+    __asm__ __volatile__(SMPLOCK "xaddq %1,%0" : "+m"(*v), "+r"(ret) : : "memory", "cc");
+    return ret;
+}
+
+static inline int64_t opal_atomic_sub_fetch_64(opal_atomic_int64_t *v, int64_t i)
+{
+    return opal_atomic_sub_fetch_64(v, i) - i;
 }
 
-#endif /* OPAL_GCC_INLINE_ASSEMBLY */
+#define OPAL_ATOMIC_DEFINE_OP(type, bits, operation, name)                                         \
+        static inline type opal_atomic_fetch_##name##_##bits(opal_atomic_##type *addr, type value) \
+        {                                                                                          \
+            type oldval;                                                                           \
+            do {                                                                                   \
+                oldval = *addr;                                                                    \
+            } while (!opal_atomic_compare_exchange_strong_##bits(addr, &oldval,                    \
+                                                                 oldval operation value));         \
+                                                                                                   \
+            return oldval;                                                                         \
+        }                                                                                          \
+                                                                                                   \
+        static inline type opal_atomic_##name##_fetch_##bits(opal_atomic_##type *addr, type value) \
+        {                                                                                          \
+            type oldval, newval;                                                                   \
+            do {                                                                                   \
+                oldval = *addr;                                                                    \
+                newval = oldval operation value;                                                   \
+            } while (!opal_atomic_compare_exchange_strong_##bits(addr, &oldval, newval));          \
+                                                                                                   \
+            return newval;                                                                         \
+        }
+
+OPAL_ATOMIC_DEFINE_OP(int32_t, 32, &, and)
+OPAL_ATOMIC_DEFINE_OP(int32_t, 32, |, or)
+OPAL_ATOMIC_DEFINE_OP(int32_t, 32, ^, xor)
+
+OPAL_ATOMIC_DEFINE_OP(int64_t, 64, &, and)
+OPAL_ATOMIC_DEFINE_OP(int64_t, 64, |, or)
+OPAL_ATOMIC_DEFINE_OP(int64_t, 64, ^, xor)
+
+#include "opal/sys/atomic_impl_minmax_math.h"
+#include "opal/sys/atomic_impl_size_t_math.h"
 
 #endif /* ! OPAL_SYS_ARCH_ATOMIC_H */
diff --git a/opal/include/opal/sys/x86_64/timer.h b/opal/include/opal/sys/x86_64/timer.h
index 9eb7e084208..cbef5e82e95 100644
--- a/opal/include/opal/sys/x86_64/timer.h
+++ b/opal/include/opal/sys/x86_64/timer.h
@@ -12,6 +12,8 @@
  *                         All rights reserved.
  * Copyright (c) 2016      Los Alamos National Security, LLC. ALl rights
  *                         reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -22,27 +24,27 @@
 #ifndef OPAL_SYS_ARCH_TIMER_H
 #define OPAL_SYS_ARCH_TIMER_H 1
 
-
 typedef uint64_t opal_timer_t;
 
 /* Using RDTSC(P) results in non-monotonic timers across cores */
 #undef OPAL_TIMER_MONOTONIC
 #define OPAL_TIMER_MONOTONIC 0
 
-#if OPAL_GCC_INLINE_ASSEMBLY
+#if OPAL_C_GCC_INLINE_ASSEMBLY
+
+#    if defined(PLATFORM_ARCH_X86_64)
 
 /* TODO: add AMD mfence version and dispatch at init */
-static inline opal_timer_t
-opal_sys_timer_get_cycles(void)
+static inline opal_timer_t opal_sys_timer_get_cycles(void)
 {
-     uint32_t l, h;
-     __asm__ __volatile__ ("lfence\n\t"
-                           "rdtsc\n\t"
-                           : "=a" (l), "=d" (h));
-     return ((opal_timer_t)l) | (((opal_timer_t)h) << 32);
+    uint32_t l, h;
+    __asm__ __volatile__("lfence\n\t"
+                         "rdtsc\n\t"
+                         : "=a"(l), "=d"(h));
+    return ((opal_timer_t) l) | (((opal_timer_t) h) << 32);
 }
 
-static inline bool opal_sys_timer_is_monotonic (void)
+static inline bool opal_sys_timer_is_monotonic(void)
 {
     int64_t tmp;
     int32_t cpuid1, cpuid2;
@@ -50,23 +52,41 @@ static inline bool opal_sys_timer_is_monotonic (void)
 
     /* cpuid clobbers ebx but it must be restored for -fPIC so save
      * then restore ebx */
-    __asm__ volatile ("xchg %%rbx, %2\n"
-                      "cpuid\n"
-                      "xchg %%rbx, %2\n":
-                      "=a" (cpuid1), "=d" (cpuid2), "=r" (tmp) :
-                      "a" (level) :
-                      "ecx", "ebx");
+    __asm__ volatile("xchg %%rbx, %2\n"
+                     "cpuid\n"
+                     "xchg %%rbx, %2\n"
+                     : "=a"(cpuid1), "=d"(cpuid2), "=r"(tmp)
+                     : "a"(level)
+                     : "ecx", "ebx");
     /* bit 8 of edx contains the invariant tsc flag */
     return !!(cpuid2 & (1 << 8));
 }
 
-#define OPAL_HAVE_SYS_TIMER_GET_CYCLES 1
-#define OPAL_HAVE_SYS_TIMER_IS_MONOTONIC 1
+#    define OPAL_HAVE_SYS_TIMER_IS_MONOTONIC 1
+#    else
+
+static inline opal_timer_t opal_sys_timer_get_cycles(void)
+{
+    opal_timer_t ret;
+    int tmp;
+
+    __asm__ __volatile__("xchgl %%ebx, %1\n"
+                         "cpuid\n"
+                         "xchgl %%ebx, %1\n"
+                         "rdtsc\n"
+                         : "=A"(ret), "=r"(tmp)::"ecx");
+
+    return ret;
+}
+
+#    endif
+
+#    define OPAL_HAVE_SYS_TIMER_GET_CYCLES 1
 
 #else
 
-#define OPAL_HAVE_SYS_TIMER_GET_CYCLES 0
+#    define OPAL_HAVE_SYS_TIMER_GET_CYCLES 0
 
-#endif /* OPAL_GCC_INLINE_ASSEMBLY */
+#endif /* OPAL_C_GCC_INLINE_ASSEMBLY */
 
 #endif /* ! OPAL_SYS_ARCH_TIMER_H */
diff --git a/opal/include/opal/sys/x86_64/update.sh b/opal/include/opal/sys/x86_64/update.sh
deleted file mode 100644
index dbef4d61cd0..00000000000
--- a/opal/include/opal/sys/x86_64/update.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/sh
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-CFILE=/tmp/opal_atomic_$$.c
-
-trap "/bin/rm -f $CFILE; exit 0" 0 1 2 15
-
-echo Updating asm.s from atomic.h and timer.h using gcc
-
-cat > $CFILE<<EOF
-#include <stdlib.h>
-#include <inttypes.h>
-#define static
-#define inline
-#define OPAL_GCC_INLINE_ASSEMBLY 1
-#include "atomic.h"
-#include "timer.h"
-EOF
-
-gcc -O3 -I. -S $CFILE -o asm.s
diff --git a/opal/include/opal/types.h b/opal/include/opal/types.h
index 2089057365a..bca24de238f 100644
--- a/opal/include/opal/types.h
+++ b/opal/include/opal/types.h
@@ -24,23 +24,23 @@
 
 #include <stdint.h>
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 #ifdef HAVE_SYS_SOCKET_H
-#include <sys/socket.h>
+#    include <sys/socket.h>
 #endif
 #ifdef HAVE_SYS_SELECT_H
-#include <sys/select.h>
+#    include <sys/select.h>
 #endif
 #ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
+#    include <netinet/in.h>
 #endif
 #ifdef HAVE_ARPA_INET_H
-#include <arpa/inet.h>
+#    include <arpa/inet.h>
 #endif
 
 #if OPAL_ENABLE_DEBUG
-#include "opal/util/output.h"
+#    include "opal/util/output.h"
 #endif
 
 /*
@@ -48,13 +48,13 @@
  */
 
 typedef union {
-   uint64_t lval;
-   uint32_t ival;
-   void*    pval;
-   struct {
-       uint32_t uval;
-       uint32_t lval;
-   } sval;
+    uint64_t lval;
+    uint32_t ival;
+    void *pval;
+    struct {
+        uint32_t uval;
+        uint32_t lval;
+    } sval;
 } opal_ptr_t;
 
 /*
@@ -62,11 +62,11 @@ typedef union {
  */
 
 #if defined(__APPLE__) || defined(__WINDOWS__)
-typedef char* opal_iov_base_ptr_t;
-#define OPAL_IOVBASE char
+typedef char *opal_iov_base_ptr_t;
+#    define OPAL_IOVBASE char
 #else
-#define OPAL_IOVBASE void
-typedef void* opal_iov_base_ptr_t;
+#    define OPAL_IOVBASE void
+typedef void *opal_iov_base_ptr_t;
 #endif
 
 /*
@@ -79,7 +79,6 @@ typedef socklen_t opal_socklen_t;
 typedef int opal_socklen_t;
 #endif
 
-
 /*
  * Convert a 64 bit value to network byte order.
  */
@@ -87,12 +86,13 @@ static inline uint64_t hton64(uint64_t val) __opal_attribute_const__;
 static inline uint64_t hton64(uint64_t val)
 {
 #ifdef HAVE_UNIX_BYTESWAP
-    union { uint64_t ll;
-            uint32_t l[2];
+    union {
+        uint64_t ll;
+        uint32_t l[2];
     } w, r;
 
     /* platform already in network byte order? */
-    if(htonl(1) == 1L)
+    if (htonl(1) == 1L)
         return val;
     w.ll = val;
     r.l[0] = htonl(w.l[1]);
@@ -111,12 +111,13 @@ static inline uint64_t ntoh64(uint64_t val) __opal_attribute_const__;
 static inline uint64_t ntoh64(uint64_t val)
 {
 #ifdef HAVE_UNIX_BYTESWAP
-    union { uint64_t ll;
-            uint32_t l[2];
+    union {
+        uint64_t ll;
+        uint32_t l[2];
     } w, r;
 
     /* platform already in network byte order? */
-    if(htonl(1) == 1L)
+    if (htonl(1) == 1L)
         return val;
     w.ll = val;
     r.l[0] = ntohl(w.l[1]);
@@ -127,33 +128,33 @@ static inline uint64_t ntoh64(uint64_t val)
 #endif
 }
 
-
 /**
  * Convert between a local representation of pointer and a 64 bits value.
  */
-static inline uint64_t opal_ptr_ptol( void* ptr ) __opal_attribute_const__;
-static inline uint64_t opal_ptr_ptol( void* ptr )
+static inline uint64_t opal_ptr_ptol(void *ptr) __opal_attribute_const__;
+static inline uint64_t opal_ptr_ptol(void *ptr)
 {
     return (uint64_t)(uintptr_t) ptr;
 }
 
-static inline void* opal_ptr_ltop( uint64_t value ) __opal_attribute_const__;
-static inline void* opal_ptr_ltop( uint64_t value )
+static inline void *opal_ptr_ltop(uint64_t value) __opal_attribute_const__;
+static inline void *opal_ptr_ltop(uint64_t value)
 {
 #if SIZEOF_VOID_P == 4 && OPAL_ENABLE_DEBUG
     if (value > ((1ULL << 32) - 1ULL)) {
         opal_output(0, "Warning: truncating value in opal_ptr_ltop");
     }
 #endif
-    return (void*)(uintptr_t) value;
+    return (void *) (uintptr_t) value;
 }
 
 #if defined(WORDS_BIGENDIAN) || !defined(HAVE_UNIX_BYTESWAP)
 static inline uint16_t opal_swap_bytes2(uint16_t val) __opal_attribute_const__;
 static inline uint16_t opal_swap_bytes2(uint16_t val)
 {
-    union { uint16_t bigval;
-            uint8_t  arrayval[2];
+    union {
+        uint16_t bigval;
+        uint8_t arrayval[2];
     } w, r;
 
     w.bigval = val;
@@ -166,8 +167,9 @@ static inline uint16_t opal_swap_bytes2(uint16_t val)
 static inline uint32_t opal_swap_bytes4(uint32_t val) __opal_attribute_const__;
 static inline uint32_t opal_swap_bytes4(uint32_t val)
 {
-    union { uint32_t bigval;
-            uint8_t  arrayval[4];
+    union {
+        uint32_t bigval;
+        uint8_t arrayval[4];
     } w, r;
 
     w.bigval = val;
@@ -182,8 +184,9 @@ static inline uint32_t opal_swap_bytes4(uint32_t val)
 static inline uint64_t opal_swap_bytes8(uint64_t val) __opal_attribute_const__;
 static inline uint64_t opal_swap_bytes8(uint64_t val)
 {
-    union { uint64_t bigval;
-            uint8_t  arrayval[8];
+    union {
+        uint64_t bigval;
+        uint8_t arrayval[8];
     } w, r;
 
     w.bigval = val;
@@ -200,9 +203,9 @@ static inline uint64_t opal_swap_bytes8(uint64_t val)
 }
 
 #else
-#define opal_swap_bytes2 htons
-#define opal_swap_bytes4 htonl
-#define opal_swap_bytes8 hton64
+#    define opal_swap_bytes2 htons
+#    define opal_swap_bytes4 htonl
+#    define opal_swap_bytes8 hton64
 #endif /* WORDS_BIGENDIAN || !HAVE_UNIX_BYTESWAP */
 
 #endif /* OPAL_TYPES_H */
diff --git a/opal/include/opal_config_bottom.h b/opal/include/opal_config_bottom.h
index d6f47f102f3..39a8839a0d0 100644
--- a/opal/include/opal_config_bottom.h
+++ b/opal/include/opal_config_bottom.h
@@ -10,12 +10,14 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2009      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2009-2013 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2009-2022 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2013      Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2015-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2015-2017 Intel, Inc. All rights reserved.
+ * Copyright (c) 2021      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2023      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -32,7 +34,7 @@
  */
 
 #ifndef OPAL_CONFIG_H
-#error "opal_config_bottom.h should only be included from opal_config.h"
+#    error "opal_config_bottom.h should only be included from opal_config.h"
 #endif
 
 /*
@@ -51,7 +53,7 @@
  * default to 0.
  */
 #ifndef OMPI_BUILDING
-#define OMPI_BUILDING 1
+#    define OMPI_BUILDING 1
 #endif
 
 /*
@@ -61,7 +63,7 @@
  * know this file is not available, we can prevent flex from including it.
  */
 #ifndef HAVE_UNISTD_H
-#define YY_NO_UNISTD_H
+#    define YY_NO_UNISTD_H
 #endif
 
 /***********************************************************************
@@ -79,11 +81,11 @@
 #undef BEGIN_C_DECLS
 #undef END_C_DECLS
 #if defined(c_plusplus) || defined(__cplusplus)
-# define BEGIN_C_DECLS extern "C" {
-# define END_C_DECLS }
+#    define BEGIN_C_DECLS extern "C" {
+#    define END_C_DECLS   }
 #else
-#define BEGIN_C_DECLS          /* empty */
-#define END_C_DECLS            /* empty */
+#    define BEGIN_C_DECLS /* empty */
+#    define END_C_DECLS   /* empty */
 #endif
 
 /**
@@ -91,8 +93,8 @@
  * usage.
  */
 #if OPAL_HAVE_ATTRIBUTE_ALIGNED
-#    define __opal_attribute_aligned__(a)    __attribute__((__aligned__(a)))
-#    define __opal_attribute_aligned_max__   __attribute__((__aligned__))
+#    define __opal_attribute_aligned__(a)  __attribute__((__aligned__(a)))
+#    define __opal_attribute_aligned_max__ __attribute__((__aligned__))
 #else
 #    define __opal_attribute_aligned__(a)
 #    define __opal_attribute_aligned_max__
@@ -105,107 +107,107 @@
 #endif
 
 #if OPAL_HAVE_ATTRIBUTE_COLD
-#    define __opal_attribute_cold__          __attribute__((__cold__))
+#    define __opal_attribute_cold__ __attribute__((__cold__))
 #else
 #    define __opal_attribute_cold__
 #endif
 
 #if OPAL_HAVE_ATTRIBUTE_CONST
-#    define __opal_attribute_const__         __attribute__((__const__))
+#    define __opal_attribute_const__ __attribute__((__const__))
 #else
 #    define __opal_attribute_const__
 #endif
 
 #if OPAL_HAVE_ATTRIBUTE_DEPRECATED
-#    define __opal_attribute_deprecated__    __attribute__((__deprecated__))
+#    define __opal_attribute_deprecated__ __attribute__((__deprecated__))
 #else
 #    define __opal_attribute_deprecated__
 #endif
 
 #if OPAL_HAVE_ATTRIBUTE_FORMAT
-#    define __opal_attribute_format__(a,b,c) __attribute__((__format__(a, b, c)))
+#    define __opal_attribute_format__(a, b, c) __attribute__((__format__(a, b, c)))
 #else
-#    define __opal_attribute_format__(a,b,c)
+#    define __opal_attribute_format__(a, b, c)
 #endif
 
-/* Use this __atribute__ on function-ptr declarations, only */
+/* Use this __attribute__ on function-ptr declarations, only */
 #if OPAL_HAVE_ATTRIBUTE_FORMAT_FUNCPTR
-#    define __opal_attribute_format_funcptr__(a,b,c) __attribute__((__format__(a, b, c)))
+#    define __opal_attribute_format_funcptr__(a, b, c) __attribute__((__format__(a, b, c)))
 #else
-#    define __opal_attribute_format_funcptr__(a,b,c)
+#    define __opal_attribute_format_funcptr__(a, b, c)
 #endif
 
 #if OPAL_HAVE_ATTRIBUTE_HOT
-#    define __opal_attribute_hot__           __attribute__((__hot__))
+#    define __opal_attribute_hot__ __attribute__((__hot__))
 #else
 #    define __opal_attribute_hot__
 #endif
 
 #if OPAL_HAVE_ATTRIBUTE_MALLOC
-#    define __opal_attribute_malloc__        __attribute__((__malloc__))
+#    define __opal_attribute_malloc__ __attribute__((__malloc__))
 #else
 #    define __opal_attribute_malloc__
 #endif
 
 #if OPAL_HAVE_ATTRIBUTE_MAY_ALIAS
-#    define __opal_attribute_may_alias__     __attribute__((__may_alias__))
+#    define __opal_attribute_may_alias__ __attribute__((__may_alias__))
 #else
 #    define __opal_attribute_may_alias__
 #endif
 
 #if OPAL_HAVE_ATTRIBUTE_NO_INSTRUMENT_FUNCTION
-#    define __opal_attribute_no_instrument_function__  __attribute__((__no_instrument_function__))
+#    define __opal_attribute_no_instrument_function__ __attribute__((__no_instrument_function__))
 #else
 #    define __opal_attribute_no_instrument_function__
 #endif
 
 #if OPAL_HAVE_ATTRIBUTE_NOINLINE
-#    define __opal_attribute_noinline__  __attribute__((__noinline__))
+#    define __opal_attribute_noinline__ __attribute__((__noinline__))
 #else
 #    define __opal_attribute_noinline__
 #endif
 
 #if OPAL_HAVE_ATTRIBUTE_NONNULL
-#    define __opal_attribute_nonnull__(a)    __attribute__((__nonnull__(a)))
-#    define __opal_attribute_nonnull_all__   __attribute__((__nonnull__))
+#    define __opal_attribute_nonnull__(a)  __attribute__((__nonnull__(a)))
+#    define __opal_attribute_nonnull_all__ __attribute__((__nonnull__))
 #else
 #    define __opal_attribute_nonnull__(a)
 #    define __opal_attribute_nonnull_all__
 #endif
 
 #if OPAL_HAVE_ATTRIBUTE_NORETURN
-#    define __opal_attribute_noreturn__      __attribute__((__noreturn__))
+#    define __opal_attribute_noreturn__ __attribute__((__noreturn__))
 #else
 #    define __opal_attribute_noreturn__
 #endif
 
-/* Use this __atribute__ on function-ptr declarations, only */
+/* Use this __attribute__ on function-ptr declarations, only */
 #if OPAL_HAVE_ATTRIBUTE_NORETURN_FUNCPTR
-#    define __opal_attribute_noreturn_funcptr__  __attribute__((__noreturn__))
+#    define __opal_attribute_noreturn_funcptr__ __attribute__((__noreturn__))
 #else
 #    define __opal_attribute_noreturn_funcptr__
 #endif
 
 #if OPAL_HAVE_ATTRIBUTE_PACKED
-#    define __opal_attribute_packed__        __attribute__((__packed__))
+#    define __opal_attribute_packed__ __attribute__((__packed__))
 #else
 #    define __opal_attribute_packed__
 #endif
 
 #if OPAL_HAVE_ATTRIBUTE_PURE
-#    define __opal_attribute_pure__          __attribute__((__pure__))
+#    define __opal_attribute_pure__ __attribute__((__pure__))
 #else
 #    define __opal_attribute_pure__
 #endif
 
 #if OPAL_HAVE_ATTRIBUTE_SENTINEL
-#    define __opal_attribute_sentinel__      __attribute__((__sentinel__))
+#    define __opal_attribute_sentinel__ __attribute__((__sentinel__))
 #else
 #    define __opal_attribute_sentinel__
 #endif
 
 #if OPAL_HAVE_ATTRIBUTE_UNUSED
-#    define __opal_attribute_unused__        __attribute__((__unused__))
+#    define __opal_attribute_unused__ __attribute__((__unused__))
 #else
 #    define __opal_attribute_unused__
 #endif
@@ -228,37 +230,41 @@
 #    define __opal_attribute_weak_alias__(a)
 #endif
 
+#if OPAL_HAVE_ATTRIBUTE_CONSTRUCTOR
+#    define __opal_attribute_constructor__ __attribute__((__constructor__))
+#else
+#    define __opal_attribute_constructor__
+#endif
+
 #if OPAL_HAVE_ATTRIBUTE_DESTRUCTOR
-#    define __opal_attribute_destructor__    __attribute__((__destructor__))
+#    define __opal_attribute_destructor__ __attribute__((__destructor__))
 #else
 #    define __opal_attribute_destructor__
 #endif
 
 #if OPAL_HAVE_ATTRIBUTE_OPTNONE
-#    define __opal_attribute_optnone__    __attribute__((__optnone__))
+#    define __opal_attribute_optnone__ __attribute__((__optnone__))
 #else
 #    define __opal_attribute_optnone__
 #endif
 
 #if OPAL_HAVE_ATTRIBUTE_EXTENSION
-#    define __opal_attribute_extension__    __extension__
+#    define __opal_attribute_extension__ __extension__
 #else
 #    define __opal_attribute_extension__
 #endif
 
-#  if OPAL_C_HAVE_VISIBILITY
-#    define OPAL_DECLSPEC           __opal_attribute_visibility__("default")
-#    define OPAL_MODULE_DECLSPEC    __opal_attribute_visibility__("default")
-#  else
+#if OPAL_C_HAVE_VISIBILITY
+#    define OPAL_DECLSPEC        __opal_attribute_visibility__("default")
+#else
 #    define OPAL_DECLSPEC
-#    define OPAL_MODULE_DECLSPEC
-#  endif
+#endif
 
-#if !defined(__STDC_LIMIT_MACROS) && (defined(c_plusplus) || defined (__cplusplus))
+#if !defined(__STDC_LIMIT_MACROS) && (defined(c_plusplus) || defined(__cplusplus))
 /* When using a C++ compiler, the max / min value #defines for std
    types are only included if __STDC_LIMIT_MACROS is set before
    including stdint.h */
-#define __STDC_LIMIT_MACROS
+#    define __STDC_LIMIT_MACROS
 #endif
 #include "opal_stdint.h"
 
@@ -274,38 +280,38 @@
 /*
  * Maximum size of a filename path.
  */
-#include <limits.h>
-#ifdef HAVE_SYS_PARAM_H
-#include <sys/param.h>
-#endif
-#if defined(PATH_MAX)
-#define OPAL_PATH_MAX   (PATH_MAX + 1)
-#elif defined(_POSIX_PATH_MAX)
-#define OPAL_PATH_MAX   (_POSIX_PATH_MAX + 1)
-#else
-#define OPAL_PATH_MAX   256
-#endif
+#    include <limits.h>
+#    ifdef HAVE_SYS_PARAM_H
+#        include <sys/param.h>
+#    endif
+#    if defined(PATH_MAX)
+#        define OPAL_PATH_MAX (PATH_MAX + 1)
+#    elif defined(_POSIX_PATH_MAX)
+#        define OPAL_PATH_MAX (_POSIX_PATH_MAX + 1)
+#    else
+#        define OPAL_PATH_MAX 256
+#    endif
 
 /*
  * Set the compile-time path-separator on this system and variable separator
  */
-#define OPAL_PATH_SEP "/"
-#define OPAL_ENV_SEP  ':'
-
-#if defined(MAXHOSTNAMELEN)
-#define OPAL_MAXHOSTNAMELEN (MAXHOSTNAMELEN + 1)
-#elif defined(HOST_NAME_MAX)
-#define OPAL_MAXHOSTNAMELEN (HOST_NAME_MAX + 1)
-#else
+#    define OPAL_PATH_SEP "/"
+#    define OPAL_ENV_SEP  ':'
+
+#    if defined(MAXHOSTNAMELEN)
+#        define OPAL_MAXHOSTNAMELEN (MAXHOSTNAMELEN + 1)
+#    elif defined(HOST_NAME_MAX)
+#        define OPAL_MAXHOSTNAMELEN (HOST_NAME_MAX + 1)
+#    else
 /* SUSv2 guarantees that "Host names are limited to 255 bytes". */
-#define OPAL_MAXHOSTNAMELEN (255 + 1)
-#endif
+#        define OPAL_MAXHOSTNAMELEN (255 + 1)
+#    endif
 
-#define OPAL_LOCAL_MAXHOSTNAMELEN 4096
+#    define OPAL_LOCAL_MAXHOSTNAMELEN 4096
 
-#if (OPAL_MAXHOSTNAMELEN > OPAL_LOCAL_MAXHOSTNAMELEN)
-#define OPAL_LOCAL_MAXHOSTNAMELEN OPAL_MAXHOSTNAMELEN
-#endif
+#    if (OPAL_MAXHOSTNAMELEN > OPAL_LOCAL_MAXHOSTNAMELEN)
+#        define OPAL_LOCAL_MAXHOSTNAMELEN OPAL_MAXHOSTNAMELEN
+#    endif
 
 /*
  * Do we want memory debugging?
@@ -326,94 +332,107 @@
  * this stuff enabled (like the memory manager code) a way to turn us
  * off
  */
-#if OPAL_ENABLE_MEM_DEBUG && !defined(OPAL_DISABLE_ENABLE_MEM_DEBUG)
+#    if OPAL_ENABLE_MEM_DEBUG && !defined(OPAL_DISABLE_ENABLE_MEM_DEBUG)
 
 /* It is safe to include opal/util/malloc.h here because a) it will only
    happen when we are building OMPI and therefore have a full OMPI
    source tree [including headers] available, and b) we guaranteed to
    *not* to include anything else via opal/util/malloc.h, so we won't
    have Cascading Includes Of Death. */
-#    include "opal/util/malloc.h"
-#    if defined(malloc)
-#        undef malloc
-#    endif
-#    define malloc(size) opal_malloc((size), __FILE__, __LINE__)
-#    if defined(calloc)
-#        undef calloc
-#    endif
-#    define calloc(nmembers, size) opal_calloc((nmembers), (size), __FILE__, __LINE__)
-#    if defined(realloc)
-#        undef realloc
-#    endif
-#    define realloc(ptr, size) opal_realloc((ptr), (size), __FILE__, __LINE__)
-#    if defined(free)
-#        undef free
-#    endif
-#    define free(ptr) opal_free((ptr), __FILE__, __LINE__)
+#        include "opal/util/malloc.h"
+#        if defined(malloc)
+#            undef malloc
+#        endif
+#        define malloc(size) opal_malloc((size), __FILE__, __LINE__)
+#        if defined(calloc)
+#            undef calloc
+#        endif
+#        define calloc(nmembers, size) opal_calloc((nmembers), (size), __FILE__, __LINE__)
+#        if defined(realloc)
+#            undef realloc
+#        endif
+#        define realloc(ptr, size) opal_realloc((ptr), (size), __FILE__, __LINE__)
+#        if defined(free)
+#            undef free
+#        endif
+#        define free(ptr) opal_free((ptr), __FILE__, __LINE__)
 
 /*
  * If we're mem debugging, make the OPAL_DEBUG_ZERO resolve to memset
  */
-#    include <string.h>
-#    define OPAL_DEBUG_ZERO(obj) memset(&(obj), 0, sizeof(obj))
-#else
-#    define OPAL_DEBUG_ZERO(obj)
-#endif
+#        include <string.h>
+#        define OPAL_DEBUG_ZERO(obj) memset(&(obj), 0, sizeof(obj))
+#    else
+#        define OPAL_DEBUG_ZERO(obj)
+#    endif
 
 /*
  * printf functions for portability (only when building Open MPI)
  */
-#if !defined(HAVE_VASPRINTF) || !defined(HAVE_VSNPRINTF)
-#include <stdarg.h>
-#include <stdlib.h>
-#endif
+#    if !defined(HAVE_VASPRINTF) || !defined(HAVE_VSNPRINTF)
+#        include <stdarg.h>
+#        include <stdlib.h>
+#    endif
 
-#if !defined(HAVE_ASPRINTF) || !defined(HAVE_SNPRINTF) || !defined(HAVE_VASPRINTF) || !defined(HAVE_VSNPRINTF)
-#include "opal/util/printf.h"
-#endif
+#    if !defined(HAVE_ASPRINTF) || !defined(HAVE_SNPRINTF) || !defined(HAVE_VASPRINTF) \
+        || !defined(HAVE_VSNPRINTF)
+#        include "opal/util/printf.h"
+#    endif
 
-#ifndef HAVE_ASPRINTF
-# define asprintf opal_asprintf
-#endif
+#    ifndef HAVE_ASPRINTF
+#        define asprintf opal_asprintf
+#    endif
 
-#ifndef HAVE_SNPRINTF
-# define snprintf opal_snprintf
-#endif
+#    ifndef HAVE_SNPRINTF
+#        define snprintf opal_snprintf
+#    endif
 
-#ifndef HAVE_VASPRINTF
-# define vasprintf opal_vasprintf
-#endif
+#    ifndef HAVE_VASPRINTF
+#        define vasprintf opal_vasprintf
+#    endif
 
-#ifndef HAVE_VSNPRINTF
-# define vsnprintf opal_vsnprintf
-#endif
+#    ifndef HAVE_VSNPRINTF
+#        define vsnprintf opal_vsnprintf
+#    endif
 
 /*
  * Some platforms (Solaris) have a broken qsort implementation.  Work
  * around by using our own.
  */
-#if OPAL_HAVE_BROKEN_QSORT
-#ifdef qsort
-#undef qsort
-#endif
+#    if OPAL_HAVE_BROKEN_QSORT
+#        ifdef qsort
+#            undef qsort
+#        endif
 
-#include "opal/util/qsort.h"
-#define qsort opal_qsort
-#endif
+#        include "opal/util/qsort.h"
+#        define qsort opal_qsort
+#    endif
 
 /*
- * On some homogenous big-iron machines (Sandia's Red Storm), there
+ * On some homogeneous big-iron machines (Sandia's Red Storm), there
  * are no htonl and friends.  If that's the case, provide stubs.  I
  * would hope we never find a platform that doesn't have these macros
  * and would want to talk to the outside world... On other platforms
  * we fail to detect them correctly.
  */
-#if !defined(HAVE_UNIX_BYTESWAP)
-static inline uint32_t htonl(uint32_t hostvar) { return hostvar; }
-static inline uint32_t ntohl(uint32_t netvar) { return netvar; }
-static inline uint16_t htons(uint16_t hostvar) { return hostvar; }
-static inline uint16_t ntohs(uint16_t netvar) { return netvar; }
-#endif
+#    if !defined(HAVE_UNIX_BYTESWAP)
+static inline uint32_t htonl(uint32_t hostvar)
+{
+    return hostvar;
+}
+static inline uint32_t ntohl(uint32_t netvar)
+{
+    return netvar;
+}
+static inline uint16_t htons(uint16_t hostvar)
+{
+    return hostvar;
+}
+static inline uint16_t ntohs(uint16_t netvar)
+{
+    return netvar;
+}
+#    endif
 
 /*
  * Define __func__-preprocessor directive if the compiler does not
@@ -422,112 +441,143 @@ static inline uint16_t ntohs(uint16_t netvar) { return netvar; }
  * coming from (assuming that __func__ is typically used for
  * printf-style debugging).
  */
-#if defined(HAVE_DECL___FUNC__) && !HAVE_DECL___FUNC__
-#define __func__ __FILE__
-#endif
+#    if defined(HAVE_DECL___FUNC__) && !HAVE_DECL___FUNC__
+#        define __func__ __FILE__
+#    endif
 
-#define IOVBASE_TYPE  void
+#    define IOVBASE_TYPE void
 
 /* ensure the bool type is defined as it is used everywhere */
-#include <stdbool.h>
+#    include <stdbool.h>
 
 /**
  * If we generate our own bool type, we need a special way to cast the result
  * in such a way to keep the compilers silent.
  */
-#  define OPAL_INT_TO_BOOL(VALUE)  (bool)(VALUE)
+#    define OPAL_INT_TO_BOOL(VALUE) (bool) (VALUE)
 
 /**
  * Top level define to check 2 things: a) if we want ipv6 support, and
  * b) the underlying system supports ipv6.  Having one #define for
  * this makes it simpler to check throughout the code base.
  */
-#if OPAL_ENABLE_IPV6 && defined(HAVE_STRUCT_SOCKADDR_IN6)
-#define OPAL_ENABLE_IPV6 1
-#else
-#define OPAL_ENABLE_IPV6 0
-#endif
+#    if OPAL_ENABLE_IPV6 && defined(HAVE_STRUCT_SOCKADDR_IN6)
+#        define OPAL_ENABLE_IPV6 1
+#    else
+#        define OPAL_ENABLE_IPV6 0
+#    endif
 
-#if !defined(HAVE_STRUCT_SOCKADDR_STORAGE) && defined(HAVE_STRUCT_SOCKADDR_IN)
-#define sockaddr_storage sockaddr
-#define ss_family sa_family
-#endif
+#    if !defined(HAVE_STRUCT_SOCKADDR_STORAGE) && defined(HAVE_STRUCT_SOCKADDR_IN)
+#        define sockaddr_storage sockaddr
+#        define ss_family        sa_family
+#    endif
 
 /* Compatibility structure so that we don't have to have as many
    #if checks in the code base */
-#if !defined(HAVE_STRUCT_SOCKADDR_IN6) && defined(HAVE_STRUCT_SOCKADDR_IN)
-#define sockaddr_in6 sockaddr_in
-#define sin6_len sin_len
-#define sin6_family sin_family
-#define sin6_port sin_port
-#define sin6_addr sin_addr
-#endif
+#    if !defined(HAVE_STRUCT_SOCKADDR_IN6) && defined(HAVE_STRUCT_SOCKADDR_IN)
+#        define sockaddr_in6 sockaddr_in
+#        define sin6_len     sin_len
+#        define sin6_family  sin_family
+#        define sin6_port    sin_port
+#        define sin6_addr    sin_addr
+#    endif
 
-#if !HAVE_DECL_AF_UNSPEC
-#define AF_UNSPEC 0
-#endif
-#if !HAVE_DECL_PF_UNSPEC
-#define PF_UNSPEC 0
-#endif
-#if !HAVE_DECL_AF_INET6
-#define AF_INET6 AF_UNSPEC
-#endif
-#if !HAVE_DECL_PF_INET6
-#define PF_INET6 PF_UNSPEC
-#endif
+#    if !HAVE_DECL_AF_UNSPEC
+#        define AF_UNSPEC 0
+#    endif
+#    if !HAVE_DECL_PF_UNSPEC
+#        define PF_UNSPEC 0
+#    endif
+#    if !HAVE_DECL_AF_INET6
+#        define AF_INET6 AF_UNSPEC
+#    endif
+#    if !HAVE_DECL_PF_INET6
+#        define PF_INET6 PF_UNSPEC
+#    endif
 
-#if defined(__APPLE__) && defined(HAVE_INTTYPES_H)
+#    if defined(__APPLE__) && defined(HAVE_INTTYPES_H)
 /* Prior to Mac OS X 10.3, the length modifier "ll" wasn't
    supported, but "q" was for long long.  This isn't ANSI
    C and causes a warning when using PRI?64 macros.  We
-   don't support versions prior to OS X 10.3, so we dont'
+   don't support versions prior to OS X 10.3, so we don't
    need such backward compatibility.  Instead, redefine
    the macros to be "ll", which is ANSI C and doesn't
    cause a compiler warning. */
-#include <inttypes.h>
-#if defined(__PRI_64_LENGTH_MODIFIER__)
-#undef __PRI_64_LENGTH_MODIFIER__
-#define __PRI_64_LENGTH_MODIFIER__ "ll"
-#endif
-#if defined(__SCN_64_LENGTH_MODIFIER__)
-#undef __SCN_64_LENGTH_MODIFIER__
-#define __SCN_64_LENGTH_MODIFIER__ "ll"
-#endif
-#endif
+#        include <inttypes.h>
+#        if defined(__PRI_64_LENGTH_MODIFIER__)
+#            undef __PRI_64_LENGTH_MODIFIER__
+#            define __PRI_64_LENGTH_MODIFIER__ "ll"
+#        endif
+#        if defined(__SCN_64_LENGTH_MODIFIER__)
+#            undef __SCN_64_LENGTH_MODIFIER__
+#            define __SCN_64_LENGTH_MODIFIER__ "ll"
+#        endif
+#    endif
 
-#ifdef MCS_VXWORKS
+#    ifdef MCS_VXWORKS
 /* VXWorks puts some common functions in oddly named headers.  Rather
    than update all the places the functions are used, which would be a
    maintenance disatster, just update here... */
-#ifdef HAVE_IOLIB_H
+#        ifdef HAVE_IOLIB_H
 /* pipe(), ioctl() */
-#include <ioLib.h>
-#endif
-#ifdef HAVE_SOCKLIB_H
+#            include <ioLib.h>
+#        endif
+#        ifdef HAVE_SOCKLIB_H
 /* socket() */
-#include <sockLib.h>
-#endif
-#ifdef HAVE_HOSTLIB_H
+#            include <sockLib.h>
+#        endif
+#        ifdef HAVE_HOSTLIB_H
 /* gethostname() */
-#include <hostLib.h>
-#endif
-#endif
+#            include <hostLib.h>
+#        endif
+#    endif
 
 /* If we're in C++, then just undefine restrict and then define it to
    nothing.  "restrict" is not part of the C++ language, and we don't
    have a corresponding AC_CXX_RESTRICT to figure out what the C++
    compiler supports. */
-#if defined(c_plusplus) || defined(__cplusplus)
-#undef restrict
-#define restrict
-#endif
+#    if defined(c_plusplus) || defined(__cplusplus)
+#        undef restrict
+#        define restrict
+#    endif
+
+/* We need to define the opal_short_float_t macro in the configure script
+   because we use it in some other places of the configure script. However
+   we cannot define it as a macro like "__extension__ [TYPENAME]" because
+   the __extension__ keyword can only be attached to a C expression. As a
+   workaround, we once define opal_short_float_t as a macro without
+   __extension__ in configure and redefine it using typedef here. */
+#    ifdef HAVE_OPAL_SHORT_FLOAT_T
+#        undef opal_short_float_t
+#        if OPAL_SHORT_FLOAT_NEEDS_EXTENSION
+__extension__ typedef OPAL_SHORT_FLOAT_TYPE opal_short_float_t;
+#        else
+typedef OPAL_SHORT_FLOAT_TYPE opal_short_float_t;
+#        endif
+#    endif
 
-#ifdef HAVE_OPAL_SHORT_FLOAT_COMPLEX_T
-#undef opal_short_float_complex_t
+/* We need to define the opal_short_float_complex_t macro in the configure
+   script because we use it in some other places of the configure script.
+   However we cannot define it as a macro like "struct { ... }" in configure.
+   As a workaround, we once define opal_short_float_complex_t as an array
+   in configure and redefine it as a structure using typedef here. */
+#    ifdef HAVE_OPAL_SHORT_FLOAT_COMPLEX_T
+#        undef opal_short_float_complex_t
 typedef struct {
     opal_short_float_t real;
     opal_short_float_t imag;
 } opal_short_float_complex_t;
+#    endif
+
+/* gcc 13 does not define SSIZE_MAX as required by the POSIX standard.
+ * As a workaround we define ours.
+ */
+#ifndef SSIZE_MAX
+#  if SIZEOF_SSIZE_T == SIZEOF_LONG
+#    define SSIZE_MAX LONG_MAX
+#  elif SIZEOF_SSIZE_T == SIZEOF_LONG_LONG
+#    define SSIZE_MAX LONG_LONG_MAX
+#  endif
 #endif
 
 #else
@@ -538,15 +588,15 @@ typedef struct {
    directly in opal_config.h because they'll be turned into #defines'
    via autoconf.
 
-   So put them here in case any only else includes OMPI/ORTE/OPAL's
+   So put them here in case any only else includes OMPI/OPAL's
    config.h files. */
 
-#undef PACKAGE_BUGREPORT
-#undef PACKAGE_NAME
-#undef PACKAGE_STRING
-#undef PACKAGE_TARNAME
-#undef PACKAGE_VERSION
-#undef PACKAGE_URL
-#undef HAVE_CONFIG_H
+#    undef PACKAGE_BUGREPORT
+#    undef PACKAGE_NAME
+#    undef PACKAGE_STRING
+#    undef PACKAGE_TARNAME
+#    undef PACKAGE_VERSION
+#    undef PACKAGE_URL
+#    undef HAVE_CONFIG_H
 
 #endif /* OMPI_BUILDING */
diff --git a/opal/include/opal_config_top.h b/opal/include/opal_config_top.h
index 1ce5267c389..da75b279cd3 100644
--- a/opal/include/opal_config_top.h
+++ b/opal/include/opal_config_top.h
@@ -16,13 +16,13 @@
  */
 
 #ifndef OPAL_CONFIG_H
-#error "opal_config_top.h should only be included from opal_config.h"
+#    error "opal_config_top.h should only be included from opal_config.h"
 #endif
 
 /* The only purpose of this file is to undef the PACKAGE_<foo> macros
    that are put in by autoconf/automake projects.  Specifically, if
    you include a .h file from another project that defines these
-   macros (e.g., gmp.h) and then include OMPI/ORTE/OPAL's config.h,
+   macros (e.g., gmp.h) and then include OMPI/OPAL's config.h,
    you'll get a preprocessor conflict.  So put these undef's here to
    protect us from other package's PACKAGE_<foo> macros.
 
diff --git a/opal/include/opal_stdatomic.h b/opal/include/opal_stdatomic.h
index 4c76f59c26a..35437551208 100644
--- a/opal/include/opal_stdatomic.h
+++ b/opal/include/opal_stdatomic.h
@@ -2,6 +2,8 @@
 /*
  * Copyright (c) 2018      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -10,11 +12,11 @@
  */
 
 #if !defined(OPAL_STDATOMIC_H)
-#define OPAL_STDATOMIC_H
+#    define OPAL_STDATOMIC_H
 
-#include "opal_stdint.h"
+#    include "opal_stdint.h"
 
-#if (OPAL_ASSEMBLY_BUILTIN != OPAL_BUILTIN_C11) || defined(__INTEL_COMPILER)
+#if OPAL_USE_C11_ATOMICS == 0
 
 typedef volatile int opal_atomic_int_t;
 typedef volatile long opal_atomic_long_t;
@@ -29,15 +31,16 @@ typedef volatile ssize_t opal_atomic_ssize_t;
 typedef volatile intptr_t opal_atomic_intptr_t;
 typedef volatile uintptr_t opal_atomic_uintptr_t;
 
-#else /* OPAL_HAVE_C__ATOMIC */
+typedef opal_atomic_int32_t opal_atomic_lock_t;
 
-#include <stdatomic.h>
+enum { OPAL_ATOMIC_LOCK_UNLOCKED = 0,
+       OPAL_ATOMIC_LOCK_LOCKED = 1 };
 
-#ifdef __INTEL_COMPILER
-#if __INTEL_COMPILER_BUILD_DATE <= 20200310
-#warning C11 _Atomic type not fully supported. The C11 atomic support should have been disabled.
-#endif
-#endif
+#    define OPAL_ATOMIC_LOCK_INIT OPAL_ATOMIC_LOCK_UNLOCKED
+
+#else /* OPAL_USE_C11_ATOMICS == 0 */
+
+#    include <stdatomic.h>
 
 typedef atomic_int opal_atomic_int_t;
 typedef atomic_long opal_atomic_long_t;
@@ -52,21 +55,27 @@ typedef _Atomic ssize_t opal_atomic_ssize_t;
 typedef _Atomic intptr_t opal_atomic_intptr_t;
 typedef _Atomic uintptr_t opal_atomic_uintptr_t;
 
-#endif /* OPAL_HAVE_C__ATOMIC */
+typedef atomic_flag opal_atomic_lock_t;
+
+#    define OPAL_ATOMIC_LOCK_UNLOCKED false
+#    define OPAL_ATOMIC_LOCK_LOCKED   true
+
+#    define OPAL_ATOMIC_LOCK_INIT ATOMIC_FLAG_INIT
+
+#    endif /* OPAL_USE_C11_ATOMICS == 0 */
 
-#if HAVE_OPAL_INT128_T
+#    if HAVE_OPAL_INT128_T
 
-/* do not use C11 atomics for __int128 if they are not lock free */
-#if OPAL_HAVE_C11_CSWAP_INT128 && ! defined(__INTEL_COMPILER)
+#        if OPAL_USE_C11_ATOMICS && OPAL_HAVE_C11_CSWAP_INT128
 
 typedef _Atomic opal_int128_t opal_atomic_int128_t;
 
-#else
+#        else
 
 typedef volatile opal_int128_t opal_atomic_int128_t;
 
-#endif
+#        endif
 
-#endif
+#    endif
 
 #endif /* !defined(OPAL_STDATOMIC_H) */
diff --git a/opal/include/opal_stdint.h b/opal/include/opal_stdint.h
index e514b3854db..c5601f421cb 100644
--- a/opal/include/opal_stdint.h
+++ b/opal/include/opal_stdint.h
@@ -37,7 +37,7 @@
 #include <stdint.h>
 
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 
 /* 128-bit */
@@ -47,29 +47,29 @@
 typedef int128_t opal_int128_t;
 typedef uint128_t opal_uint128_t;
 
-#define HAVE_OPAL_INT128_T 1
+#    define HAVE_OPAL_INT128_T 1
 
 #elif defined(HAVE___INT128)
 
 /* suppress warning about __int128 type */
-#pragma GCC diagnostic push
+#    pragma GCC diagnostic push
 /* Clang won't quietly accept "-pedantic", but GCC versions older than ~4.8
  * won't quietly accept "-Wpedanic".  The whole "#pragma GCC diagnostic ..."
  * facility only was added to GCC as of version 4.6. */
-#if defined(__clang__) || (defined(__GNUC__) && __GNUC__ >= 6)
-#pragma GCC diagnostic ignored "-Wpedantic"
-#else
-#pragma GCC diagnostic ignored "-pedantic"
-#endif
+#    if defined(__clang__) || (defined(__GNUC__) && __GNUC__ >= 6)
+#        pragma GCC diagnostic ignored "-Wpedantic"
+#    else
+#        pragma GCC diagnostic ignored "-pedantic"
+#    endif
 typedef __int128 opal_int128_t;
 typedef unsigned __int128 opal_uint128_t;
-#pragma GCC diagnostic pop
+#    pragma GCC diagnostic pop
 
-#define HAVE_OPAL_INT128_T 1
+#    define HAVE_OPAL_INT128_T 1
 
 #else
 
-#define HAVE_OPAL_INT128_T 0
+#    define HAVE_OPAL_INT128_T 0
 
 #endif
 
@@ -77,53 +77,52 @@ typedef unsigned __int128 opal_uint128_t;
 
 #if SIZEOF_VOID_P == SIZEOF_INT
 
-#ifndef HAVE_INTPTR_T
+#    ifndef HAVE_INTPTR_T
 typedef signed int intptr_t;
-#endif
+#    endif
 
-#ifndef HAVE_UINTPTR_T
+#    ifndef HAVE_UINTPTR_T
 typedef unsigned int uintptr_t;
-#endif
+#    endif
 
 #elif SIZEOF_VOID_P == SIZEOF_LONG
 
-#ifndef HAVE_INTPTR_T
+#    ifndef HAVE_INTPTR_T
 typedef signed long intptr_t;
-#endif
+#    endif
 
-#ifndef HAVE_UINTPTR_T
+#    ifndef HAVE_UINTPTR_T
 typedef unsigned long uintptr_t;
-#endif
+#    endif
 
 #elif SIZEOF_VOID_P == SIZEOF_LONG_LONG
 
-#ifndef HAVE_INTPTR_T
+#    ifndef HAVE_INTPTR_T
 typedef signed long long intptr_t;
-#endif
-#ifndef HAVE_UINTPTR_T
+#    endif
+#    ifndef HAVE_UINTPTR_T
 typedef unsigned long long uintptr_t;
-#endif
+#    endif
 
 #else
 
-#error Failed to define pointer-sized integer types
+#    error Failed to define pointer-sized integer types
 
 #endif
 
 /* inttypes.h printf specifiers */
-# include <inttypes.h>
+#include <inttypes.h>
 
 #ifndef PRIsize_t
-# if defined(ACCEPT_C99)
-#   define PRIsize_t "zu"
-# elif SIZEOF_SIZE_T == SIZEOF_LONG
-#   define PRIsize_t "lu"
-# elif SIZEOF_SIZE_T == SIZEOF_LONG_LONG
-#   define PRIsize_t "llu"
-# else
-#   define PRIsize_t "u"
-# endif
+#    if defined(ACCEPT_C99)
+#        define PRIsize_t "zu"
+#    elif SIZEOF_SIZE_T == SIZEOF_LONG
+#        define PRIsize_t "lu"
+#    elif SIZEOF_SIZE_T == SIZEOF_LONG_LONG
+#        define PRIsize_t "llu"
+#    else
+#        define PRIsize_t "u"
+#    endif
 #endif
 
 #endif /* OPAL_STDINT_H */
-
diff --git a/opal/mca/accelerator/Makefile.am b/opal/mca/accelerator/Makefile.am
new file mode 100644
index 00000000000..e296808aad2
--- /dev/null
+++ b/opal/mca/accelerator/Makefile.am
@@ -0,0 +1,29 @@
+#
+# Copyright (c) 2014      Intel, Inc.  All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+#                         All Rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# main library setup
+noinst_LTLIBRARIES = libmca_accelerator.la
+libmca_accelerator_la_SOURCES =
+
+# local files
+headers = accelerator.h
+libmca_accelerator_la_SOURCES += $(headers)
+
+# Conditionally install the header files
+if WANT_INSTALL_HEADERS
+opaldir = $(opalincludedir)/$(subdir)
+nobase_opal_HEADERS = $(headers)
+endif
+
+include base/Makefile.am
+
+distclean-local:
+	rm -f base/static-components.h
diff --git a/opal/mca/accelerator/accelerator.h b/opal/mca/accelerator/accelerator.h
new file mode 100644
index 00000000000..12f025f53c2
--- /dev/null
+++ b/opal/mca/accelerator/accelerator.h
@@ -0,0 +1,762 @@
+/*
+ * Copyright (c) 2014-2021 Intel, Inc. All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c)           Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All Rights reserved.
+ * Copyright (c) 2024      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+/**
+ * @file
+ * Accelerator Framework
+ *
+ * Accelerator initialization and selection:
+ *
+ * During opal initialization, all available accelerator components
+ * will be attempted to be initialized. Only the null component and
+ * at most, one other component are expected to succeed initialization
+ * or we will abort the accelerator initialization process.
+ *
+ * The framework selection function will set the global opal_accelerator
+ * module to allow usage of the accelerator API.
+ *
+ *
+ * Transfer types:
+ *
+ * There are five transfer types that can be passed to memory copy functions.
+ * They are defined in the enum, opal_accelerator_transfer_type_t:
+ *
+ * UNSPEC - Not specified
+ * HTOH - Host to Host
+ * HTOD - Host to Device
+ * DTOH - Device to Host
+ * DTOD - Device to Device
+ *
+ * The UNSPEC transfer type should be used when the detection of buffer types
+ * will be deferred to the implementation. For certain optimizations, the caller
+ * may pass the transfer type to the memory copying functions.
+ *
+ *
+ * Streams and Events:
+ *
+ * A stream is a sequence of operations that execute on the device in the order
+ * in which they are issued by the host code. A single stream guarantees execution
+ * in order, but multiple streams may have interleaved or concurrent execution.
+ *
+ * An event is a synchronization marker that can be used to monitor the device's
+ * progress. They can be appended to a stream through the record_event api.
+ * The event can then be queried for completion through the query_event api.
+ *
+ * For a more brute force method of synchronization, the synchronize_stream api
+ * was added which will wait until all operations on the stream are completed
+ * before progressing.
+ *
+ *
+ * Asynchronous copies:
+ *
+ * Asynchronous copies are appended to a stream using the memcpy_async api.
+ * A subsequent event can be appended to the stream for querying.
+ * There is currently no callback functionality for asynchronous copy completion,
+ * and thus asynchronous copy completions must be progressed manually.
+ *
+ *
+ * Host register/unregister:
+ *
+ * Host register page-locks host register and informs the device. This allows
+ * the device to perform fast operations using DMA (direct memory access) which
+ * can increase the performance of certain memory copies.
+ *
+ */
+#ifndef OPAL_ACCELERATOR_H
+#define OPAL_ACCELERATOR_H
+
+#include "opal/class/opal_object.h"
+#include "opal/mca/mca.h"
+#include "ompi/info/info_memkind.h"
+
+BEGIN_C_DECLS
+
+#define MCA_ACCELERATOR_NO_DEVICE_ID -1
+/**
+ * Accelerator flags
+ */
+/* Unified memory buffers */
+#define MCA_ACCELERATOR_FLAGS_UNIFIED_MEMORY 0x00000001
+
+/**
+ * Transfer types.
+ * UNSPEC - Not specified
+ * HTOH - Host to Host
+ * HTOD - Host to Device
+ * DTOH - Device to Host
+ * DTOD - Device to Device
+ */
+typedef enum {
+    MCA_ACCELERATOR_TRANSFER_UNSPEC = 0,
+    MCA_ACCELERATOR_TRANSFER_HTOH,
+    MCA_ACCELERATOR_TRANSFER_HTOD,
+    MCA_ACCELERATOR_TRANSFER_DTOH,
+    MCA_ACCELERATOR_TRANSFER_DTOD,
+} opal_accelerator_transfer_type_t;
+
+typedef uint64_t opal_accelerator_buffer_id_t;
+
+struct opal_accelerator_stream_t {
+    opal_object_t super;
+    /* Stream object */
+    void *stream;
+};
+typedef struct opal_accelerator_stream_t opal_accelerator_stream_t;
+OBJ_CLASS_DECLARATION(opal_accelerator_stream_t);
+
+/* Constant indicating the default/zero stream */
+#define MCA_ACCELERATOR_STREAM_DEFAULT (opal_accelerator_stream_t *)0x00000002
+
+#define IPC_MAX_HANDLE_SIZE 64
+struct opal_accelerator_ipc_handle_t {
+    opal_object_t super;
+    size_t size;
+    uint8_t handle[IPC_MAX_HANDLE_SIZE];
+    void* dev_ptr;
+};
+typedef struct opal_accelerator_ipc_handle_t opal_accelerator_ipc_handle_t;
+OBJ_CLASS_DECLARATION(opal_accelerator_ipc_handle_t);
+
+struct opal_accelerator_ipc_event_handle_t {
+    opal_object_t super;
+    size_t size;
+    uint8_t handle[IPC_MAX_HANDLE_SIZE];
+};
+typedef struct opal_accelerator_ipc_event_handle_t opal_accelerator_ipc_event_handle_t;
+OBJ_CLASS_DECLARATION(opal_accelerator_ipc_event_handle_t);
+
+struct opal_accelerator_pci_attr_t {
+    uint16_t domain_id;
+    uint8_t bus_id;
+    uint8_t device_id;
+    uint8_t function_id;
+};
+typedef struct opal_accelerator_pci_attr_t opal_accelerator_pci_attr_t;
+
+
+struct opal_accelerator_event_t {
+    opal_object_t super;
+    /* Event object */
+    void *event;
+};
+typedef struct opal_accelerator_event_t opal_accelerator_event_t;
+OBJ_CLASS_DECLARATION(opal_accelerator_event_t);
+
+/**
+ * Check whether a pointer belongs to an accelerator or not.
+ * interfaces
+ *
+ * @param[IN] addr           Pointer to check
+ * @param[OUT] dev_id        Returns the device id against which the memory was allocated
+ *                           or MCA_ACCELERATOR_NO_DEVICE_ID
+ * @param[OUT] flags         Set to 0 or an Accelerator flag to indicate additional
+ *                           information about the corresponding pointer
+ *
+ * @retval <0                An error has occurred.
+ * @retval 0                 The buffer does not belong to a managed buffer
+ *                           in device memory.
+ * @retval >0                The buffer belongs to a managed buffer in
+ *                           device memory.
+ */
+typedef int (*opal_accelerator_base_module_check_addr_fn_t)(
+    const void *addr, int *dev_id, uint64_t *flags);
+
+/**
+ * Creates a stream for asynchonous operations. This function will allocate
+ * memory for the object. To release the memory, call OBJ_RELEASE(*stream);
+ *
+ * @param[IN] dev_id         Associated device for the stream or
+ *                           MCA_ACCELERATOR_NO_DEVICE_ID
+ * @param[IN] stream         Stream to create
+ *
+ * @return                   OPAL_SUCCESS or error status on failure
+ */
+typedef int (*opal_accelerator_base_module_create_stream_fn_t)(
+    int dev_id, opal_accelerator_stream_t **stream);
+
+/**
+ * Wait for the completion of all operations inserted into the stream.
+ *
+ * @param[IN] stram          The stream to wait for.
+ *
+ * @return                   OPAL_SUCCESS or error status on failure
+ */
+typedef int (*opal_accelerator_base_module_sync_stream_fn_t)(
+    opal_accelerator_stream_t *stream);
+
+/**
+ * Creates an event. An event is a synchronization marker that can be
+ * appended to a stream to monitor device progress or synchronize the
+ * corresponding stream. This function will allocate memory for the object.
+ * To release the memory, call OBJ_RELEASE(*event);
+ *
+ * @param[IN]  dev_id        Associated device for the event or
+ *                           MCA_ACCELERATOR_NO_DEVICE_ID
+ * @param[OUT] event         Event to create
+ * @param[IN]  enable_ipc    support inter-process tracking of the event
+ *
+ * @return                   OPAL_SUCCESS or error status on failure.
+ */
+typedef int (*opal_accelerator_base_module_create_event_fn_t)(
+      int dev_id, opal_accelerator_event_t **event, bool enable_ipc);
+
+/**
+ * Records an event on a stream. An event recorded on the stream is
+ * a synchronization marker that can be used to monitor the device's
+ * progress or to synchronize the corresponding stream. This API appends
+ * the given event onto the given stream.
+ *
+ * @param[IN] dev_id         Associated device for the event or
+ *                           MCA_ACCELERATOR_NO_DEVICE_ID
+ * @param[IN] event          Event to record
+ * @param[IN] stream         Stream to record event for
+ *
+ * @return                   OPAL_SUCCESS or error status on failure.
+ */
+typedef int (*opal_accelerator_base_module_record_event_fn_t)(
+    int dev_id, opal_accelerator_event_t *event, opal_accelerator_stream_t *stream);
+
+/**
+ * Queries an event's status. This can be used to monitor the device
+ * progress, as events placed on the stream execute in order.
+ *
+ * @param[IN] dev_id         Associated device for the event or
+ *                           MCA_ACCELERATOR_NO_DEVICE_ID
+ * @param[IN] event          Event to query
+ *
+ * @return                   OPAL_SUCCESS on event completion, OPAL_ERROR on error,
+ *                           or OPAL_ERR_RESOURCE_BUSY if any work is incomplete.
+ */
+typedef int (*opal_accelerator_base_module_query_event_fn_t)(
+    int dev_id, opal_accelerator_event_t *event);
+
+/**
+ * Make a stream wait on an event
+ *
+ * @param[IN] dev_id         Associated device for the event or
+ *                           MCA_ACCELERATOR_NO_DEVICE_ID
+ * @param[IN] event          Event to wait on
+ * @param[IN] stream         Stream to wait
+ *
+ * @return                   OPAL_SUCCESS or error status on failure
+ */
+typedef int (*opal_accelerator_base_module_wait_event_fn_t)(
+   int dev_id, opal_accelerator_event_t *event,  opal_accelerator_stream_t *stream);
+
+/**
+ * Copies memory asynchronously from src to dest. Memory of dest and src
+ * may not overlap. Optionally can specify the transfer type to
+ * avoid pointer detection for performance.
+ *
+ * @param[IN] dest_dev_id    Associated device to copy to or
+ *                           MCA_ACCELERATOR_NO_DEVICE_ID
+ * @param[IN] src_dev_id     Associated device to copy from or
+ *                           MCA_ACCELERATOR_NO_DEVICE_ID
+ * @param[IN] dest           Destination to copy memory to
+ * @param[IN] src            Source to copy memory from
+ * @param[IN] size           Size of memory to copy
+ * @param[IN] stream         Stream to perform asynchronous copy on
+ * @param[IN] type           Transfer type field for performance
+ *                           Can be set to MCA_ACCELERATOR_TRANSFER_UNSPEC
+ *                           if caller is unsure of the transfer direction.
+ *
+ * @return                   OPAL_SUCCESS or error status on failure
+ */
+typedef int (*opal_accelerator_base_module_memcpy_async_fn_t)(
+    int dest_dev_id, int src_dev_id, void *dest, const void *src, size_t size,
+    opal_accelerator_stream_t *stream, opal_accelerator_transfer_type_t type);
+
+/**
+ * Copies memory synchronously from src to dest. Memory of dest and src
+ * may not overlap. Optionally can specify the transfer type to
+ * avoid pointer detection for performance.
+ *
+ * @param[IN] dest_dev_id    Associated device to copy to or
+ *                           MCA_ACCELERATOR_NO_DEVICE_ID
+ * @param[IN] src_dev_id     Associated device to copy from or
+ *                           MCA_ACCELERATOR_NO_DEVICE_ID
+ * @param[IN] dest           Destination to copy memory to
+ * @param[IN] src            Source to copy memory from
+ * @param[IN] size           Size of memory to copy
+ * @param[IN] type           Transfer type field for performance
+ *                           Can be set to MCA_ACCELERATOR_TRANSFER_UNSPEC
+ *                           if caller is unsure of the transfer direction.
+ *
+ * @return                   OPAL_SUCCESS or error status on failure
+ */
+typedef int (*opal_accelerator_base_module_memcpy_fn_t)(
+    int dest_dev_id, int src_dev_id, void *dest, const void *src, size_t size,
+    opal_accelerator_transfer_type_t type);
+
+/**
+ * Copies memory synchronously from src to dest. Memory of dest and src
+ * may overlap. Optionally can specify the transfer type to
+ * avoid pointer detection for performance.
+ *
+ * @param[IN] dest_dev_id    Associated device to copy to or
+ *                           MCA_ACCELERATOR_NO_DEVICE_ID
+ * @param[IN] src_dev_id     Associated device to copy from or
+ *                           MCA_ACCELERATOR_NO_DEVICE_ID
+ * @param[IN] dest           Destination to copy memory to
+ * @param[IN] src            Source to copy memory from
+ * @param[IN] size           Size of memory to copy
+ * @param[IN] type           Transfer type field for performance
+ *                           Can be set to MCA_ACCELERATOR_TRANSFER_UNSPEC
+ *                           if caller is unsure of the transfer direction.
+ *
+ * @return                   OPAL_SUCCESS or error status on failure
+ */
+typedef int (*opal_accelerator_base_module_memmove_fn_t)(
+    int dest_dev_id, int src_dev_id, void *dest, const void *src, size_t size,
+    opal_accelerator_transfer_type_t type);
+
+
+/**
+ * Copies memory asynchronously from src to dest. Memory of dest and src
+ * may overlap. Optionally can specify the transfer type to
+ * avoid pointer detection for performance. The operations will be enqueued
+ * into the provided stream but are not guaranteed to be complete upon return.
+ *
+ * @param[IN] dest_dev_id    Associated device to copy to or
+ *                           MCA_ACCELERATOR_NO_DEVICE_ID
+ * @param[IN] src_dev_id     Associated device to copy from or
+ *                           MCA_ACCELERATOR_NO_DEVICE_ID
+ * @param[IN] dest           Destination to copy memory to
+ * @param[IN] src            Source to copy memory from
+ * @param[IN] size           Size of memory to copy
+ * @param[IN] stream         Stream to perform asynchronous move on
+ * @param[IN] type           Transfer type field for performance
+ *                           Can be set to MCA_ACCELERATOR_TRANSFER_UNSPEC
+ *                           if caller is unsure of the transfer direction.
+ *
+ * @return                   OPAL_SUCCESS or error status on failure
+ */
+typedef int (*opal_accelerator_base_module_memmove_async_fn_t)(
+    int dest_dev_id, int src_dev_id, void *dest, const void *src, size_t size,
+    opal_accelerator_stream_t *stream, opal_accelerator_transfer_type_t type);
+
+/**
+ * Allocates size bytes memory from the device and sets ptr to the
+ * pointer of the allocated memory. The memory is not initialized.
+ *
+ * @param[IN] dev_id         Associated device for the allocation or
+ *                           MCA_ACCELERATOR_NO_DEVICE_ID
+ * @param[OUT] ptr           Returns pointer to allocated memory
+ * @param[IN] size           Size of memory to allocate
+ *
+ * @return                   OPAL_SUCCESS or error status on failure
+ */
+typedef int (*opal_accelerator_base_module_mem_alloc_fn_t)(
+    int dev_id, void **ptr, size_t size);
+
+/**
+ * Frees the memory space pointed to by ptr which has been returned by
+ * a previous call to an opal_accelerator_base_module_mem_alloc_fn_t().
+ * If the function is called on a ptr that has already been freed,
+ * undefined behavior occurs. If ptr is NULL, no operation is performed,
+ * and the function returns OPAL_SUCCESS.
+ *
+ * @param[IN] dev_id         Associated device for the allocation or
+ *                           MCA_ACCELERATOR_NO_DEVICE_ID
+ * @param[IN] ptr            Pointer to free
+ *
+ * @return                   OPAL_SUCCESS or error status on failure
+ */
+typedef int (*opal_accelerator_base_module_mem_release_fn_t)(
+    int dev_id, void *ptr);
+
+
+/**
+ * Allocates size bytes memory from the device and sets ptr to the
+ * pointer of the allocated memory. The memory is not initialized.
+ * The allocation request is placed into the stream object.
+ * Any use of the memory must succeed the completion of this
+ * operation on the stream.
+ *
+ * @param[IN] dev_id         Associated device for the allocation or
+ *                           MCA_ACCELERATOR_NO_DEVICE_ID
+ * @param[OUT] ptr           Returns pointer to allocated memory
+ * @param[IN] size           Size of memory to allocate
+ * @param[IN] stream         Stream into which to insert the allocation request
+ *
+ * @return                   OPAL_SUCCESS or error status on failure
+ */
+typedef int (*opal_accelerator_base_module_mem_alloc_stream_fn_t)(
+    int dev_id, void **ptr, size_t size, opal_accelerator_stream_t *stream);
+
+/**
+ * Frees the memory space pointed to by ptr which has been returned by
+ * a previous call to an opal_accelerator_base_module_mem_alloc_stream_fn_t().
+ * If the function is called on a ptr that has already been freed,
+ * undefined behavior occurs. If ptr is NULL, no operation is performed,
+ * and the function returns OPAL_SUCCESS.
+ * The release of the memory will be inserted into the stream and occurs after
+ * all previous operations have completed.
+ *
+ * @param[IN] dev_id         Associated device for the allocation or
+ *                           MCA_ACCELERATOR_NO_DEVICE_ID
+ * @param[IN] ptr            Pointer to free
+ * @param[IN] stream         Stream into which to insert the free operation
+ *
+ * @return                   OPAL_SUCCESS or error status on failure
+ */
+typedef int (*opal_accelerator_base_module_mem_release_stream_fn_t)(
+    int dev_id, void *ptr, opal_accelerator_stream_t *stream);
+
+
+
+/**
+ * Retrieves the base address and/or size of a memory allocation of the
+ * device.
+ *
+ * @param[IN] dev_id         Associated device for the allocation or
+ *                           MCA_ACCELERATOR_NO_DEVICE_ID
+ * @param[IN] ptr            Pointer to device memory to get base/size from
+ * @param[OUT] base          Base address of the memory allocation
+ * @param[OUT] size          Size of the memory allocation
+ *
+ * @return                   OPAL_SUCCESS or error status on failure
+ */
+typedef int (*opal_accelerator_base_module_get_address_range_fn_t)(
+    int dev_id, const void *ptr, void **base, size_t *size);
+
+/*********************************************************/
+/****   Inter Process Communication (IPC) Functions   ****/
+/*********************************************************/
+
+/**
+ * Queries whether the device supports IPC or not.
+ *
+ * If true, the functions:
+ *
+ * opal_accelerator_base_module_get_ipc_handle_fn_t()
+ * opal_accelerator_base_module_open_ipc_handle_fn_t()
+ * opal_accelerator_base_module_import_ipc_handle_fn_t()
+ * opal_accelerator_base_module_compare_ipc_handles_fn_t()
+ * opal_accelerator_base_module_get_ipc_event_handle_fn_t()
+ * opal_accelerator_base_module_open_ipc_event_handle_fn_t()
+ * opal_accelerator_base_module_import_ipc_event_handle_fn_t()
+ *
+ * must be implemented.
+ *
+ * @return true              IPC supported
+ * @return false             IPC not supported
+ */
+typedef bool (*opal_accelerator_base_module_is_ipc_enabled_fn_t)(void);
+
+/**
+ * Gets an IPC memory handle for an existing device memory allocation.
+ * This interface assumes that the object has been declared statically,
+ * hence one has to call OBJ_DESTRUCT(handle) on it.
+ *
+ * @param[IN]  dev_id        Associated device for the IPC memory handle or
+ *                           MCA_ACCELERATOR_NO_DEVICE_ID
+ * @param[IN]  dev_ptr       Device memory address
+ * @param[OUT] handle        Pointer to IPC handle object
+ *
+ * @return                   OPAL_SUCCESS or error status on failure
+ *
+ */
+typedef int (*opal_accelerator_base_module_get_ipc_handle_fn_t)(
+    int dev_id, void *dev_ptr, opal_accelerator_ipc_handle_t *handle);
+
+/**
+ * Creates an opal_accelerator_ipc_handle object given the 64byte IPC handle,
+ * which was created using module_get_ipc_handle_fn on another process.
+ * This interface assumes that the object has been declared statically,
+ * hence one has to call OBJ_DESTRUCT(handle) on it.
+ *
+ * @param[IN]  dev_id        Associated device for the IPC memory handle or
+ *                           MCA_ACCELERATOR_NO_DEVICE_ID
+ * @param[IN]  ipc_handle    64 byte IPC handle transfered from another process
+ * @param[OUT] handle        Pointer to IPC handle object
+ *
+ * @return                   OPAL_SUCCESS or error status on failure
+ *
+ */
+typedef int (*opal_accelerator_base_module_import_ipc_handle_fn_t)(
+    int dev_id, uint8_t ipc_handle[IPC_MAX_HANDLE_SIZE], opal_accelerator_ipc_handle_t *handle);
+
+/**
+ * Opens an IPC memory handle from another process and returns
+ * a device pointer usable in the local process.
+ *
+ * @param[IN]  dev_id        Associated device for the IPC memory handle or
+ *                           MCA_ACCELERATOR_NO_DEVICE_ID
+ * @param[IN]  handle        IPC handle created using the module_create_ipc_handle_fn
+ * @param[OUT] dev_ptr       Returned device pointer
+ *
+ * @return                   OPAL_SUCCESS on success,
+ *                           OPAL_ERR_WOULD_BLOCK if the memory region is already mapped
+ *                           or error status on other failures
+ */
+typedef int (*opal_accelerator_base_module_open_ipc_handle_fn_t)(
+    int dev_id, opal_accelerator_ipc_handle_t *handle, void **dev_ptr);
+
+/**
+ * Compare two IPC handles
+ *
+ * @param[IN] handle_1       First IPC handle
+ * @param[IN] handle_2       Second IPC handle
+ *
+ * @return                   zero if IPC handles are identical
+ *                           non-zero value otherwise
+ */
+
+typedef int (*opal_accelerator_base_module_compare_ipc_handles_fn_t)(
+    uint8_t handle_1[IPC_MAX_HANDLE_SIZE], uint8_t handle_2[IPC_MAX_HANDLE_SIZE]);
+
+/**
+ * Gets an IPC event handle for an event created by opal_accelerator_base_module_create_event_fn_t.
+ * This interface assumes that the object has been declared statically,
+ * hence one has to call OBJ_DESTRUCT(handle) on it.
+ *
+ * @param[IN] event          Event created previously
+ * @param[IN] handle         Pointer to IPC event handle object
+ *
+ * @return                   OPAL_SUCCESS or error status on failure
+ */
+typedef int (*opal_accelerator_base_module_get_ipc_event_handle_fn_t)(
+    opal_accelerator_event_t *event, opal_accelerator_ipc_event_handle_t *handle);
+
+/**
+ * Creates an opal_accelerator_ipc_event_handle object using the 64 byte IPC event handle
+ * which was created using module_get_ipc_event_handle_fn on another process.
+ * This interface assumes that the object has been declared statically,
+ * hence one has to call OBJ_DESTRUCT(handle) on it.
+ *
+ * @param[IN] event          Event created previously
+ * @param[IN] ipc_handle     64 byte IPC handle object
+ * @param[OUT] handle        Pointer to IPC event handle object
+ *
+ * @return                   OPAL_SUCCESS or error status on failure
+ */
+typedef int (*opal_accelerator_base_module_import_ipc_event_handle_fn_t)(
+     uint8_t ipc_handle[IPC_MAX_HANDLE_SIZE], opal_accelerator_ipc_event_handle_t *handle);
+
+/**
+ * Opens an IPC event handle from another process opened by
+ * opal_accelerator_base_module_get_ipc_event_handle_fn_t.
+ *
+ * @param[IN]  handle        IPC event handle from another process
+ * @param[OUT] event         Pointer to store the opened event
+ *
+ * @return                   OPAL_SUCCESS or error status on failure
+ */
+typedef int (*opal_accelerator_base_module_open_ipc_event_handle_fn_t)(
+    opal_accelerator_ipc_event_handle_t *handle, opal_accelerator_event_t *event);
+
+/**
+ * Page-locks the memory range specified by ptr and size
+ *
+ * @param[IN] dev_id         Associated device to register host memory with
+ *                           MCA_ACCELERATOR_NO_DEVICE_ID
+ * @param[IN] ptr            Host pointer to memory to page-lock
+ * @param[IN] size           Size in bytes of the address range to page-lock in bytes
+ *
+ * @return                   OPAL_SUCCESS or error status on failure
+ */
+typedef int (*opal_accelerator_base_module_host_register_fn_t)(
+    int dev_id, void *ptr, size_t size);
+
+/**
+ * Unregisters a memory range that was registered with
+ * opal_accelerator_base_module_host_register_fn_t.
+ *
+ * @param[IN] dev_id         Associated device to deregister host memory with
+ *                           MCA_ACCELERATOR_NO_DEVICE_ID
+ * @param[IN] ptr            Host pointer to memory to unregister
+ *
+ * @return                   OPAL_SUCCESS or error status on failure
+ */
+typedef int (*opal_accelerator_base_module_host_unregister_fn_t)(
+    int dev_id, void *ptr);
+
+/**
+ * Retrieves current device id for a device associated with the local process.
+ * If MCA_ACCELERATOR_NO_DEVICE_ID is provided, there is no device/process pairing.
+ *
+ * @param[OUT] dev_id        ID of the device or MCA_ACCELERATOR_NO_DEVICE_ID
+ *
+ * @return                   OPAL_SUCCESS or error status on failure
+ */
+typedef int (*opal_accelerator_base_module_get_device_fn_t)(
+    int *dev_id);
+
+/**
+ * Retrieves PCI attributes of an accelerator device.
+ *
+ * @param[int] dev_id        Accelerator device id
+ * @param[out] pci_attr      PCI attributes of the requested device
+ *
+ * @return                   OPAL_SUCCESS or error status on failure
+ */
+typedef int (*opal_accelerator_base_module_get_device_pci_attr_fn_t)(
+    int dev_id, opal_accelerator_pci_attr_t *pci_attr);
+
+/**
+ * Queries if a device may directly access a peer device's memory.
+ *
+ * @param[OUT] access        Returns 1 if dev1 can directly access memory on dev2
+ *                           Returns 0 if dev1 can not directly access memory on dev2
+ * @param[IN] dev1           ID of device checking if peer device memory can be accessed
+ * @param[IN] dev2           ID of peer device on which the memory allocations
+ *                           reside.
+ *
+ * @return                   OPAL_SUCCESS or error status on failure
+ */
+typedef int (*opal_accelerator_base_module_device_can_access_peer_fn_t)(
+    int *access, int dev1, int dev2);
+
+/**
+ * Retrieves current device id for a device associated with the local process.
+ * If MCA_ACCELERATOR_NO_DEVICE_ID is provided, there is no device/process pairing.
+ *
+ * @param[IN] dev_id         ID of the device or MCA_ACCELERATOR_NO_DEVICE_ID
+ * @param[IN] addr           Buffer pointer to check
+ * @param[OUT] buf_id        ID of the given buffer
+ *
+ *
+ * @return                   OPAL_SUCCESS or error status on failure
+ */
+typedef int (*opal_accelerator_base_module_get_buffer_id_fn_t)(
+    int dev_id, const void *addr, opal_accelerator_buffer_id_t *buf_id);
+
+/**
+ * Get the number of devices available.
+ *
+ * @param[OUT] stram         Number of devices.
+ *
+ * @return                   OPAL_SUCCESS or error status on failure
+ */
+typedef int (*opal_accelerator_base_module_get_num_devices_fn_t)(int *num_devices);
+
+/**
+ * Get the memory bandwidth of the device.
+ *
+ * @param[IN] device         The device to query.
+ * @param[OUT] bw            The returned bandwidth for the device.
+ *
+ * @return                   OPAL_SUCCESS or error status on failure
+ */
+typedef int (*opal_accelerator_base_module_get_mem_bw_fn_t)(int device, float *bw);
+
+/**
+ * Get the memkind information of the accelerator component.
+ * @param[OUT] supported     Memory alloc kinds supported by component
+ *
+ */
+typedef void (*opal_accelerator_base_module_get_memkind_fn_t)(ompi_memkind_t *memkind);
+
+/*
+ * the standard public API data structure
+ */
+typedef struct {
+    /* accelerator function table */
+    opal_accelerator_base_module_check_addr_fn_t check_addr;
+
+    opal_accelerator_base_module_create_stream_fn_t create_stream;
+    opal_accelerator_base_module_sync_stream_fn_t sync_stream;
+    opal_accelerator_base_module_create_event_fn_t create_event;
+    opal_accelerator_base_module_record_event_fn_t record_event;
+    opal_accelerator_base_module_query_event_fn_t query_event;
+    opal_accelerator_base_module_wait_event_fn_t wait_event;
+
+    opal_accelerator_base_module_memcpy_async_fn_t mem_copy_async;
+    opal_accelerator_base_module_memcpy_fn_t mem_copy;
+    opal_accelerator_base_module_memmove_async_fn_t mem_move_async;
+    opal_accelerator_base_module_memmove_fn_t mem_move;
+
+    opal_accelerator_base_module_mem_alloc_fn_t mem_alloc;
+    opal_accelerator_base_module_mem_release_fn_t mem_release;
+    opal_accelerator_base_module_mem_alloc_stream_fn_t mem_alloc_stream;
+    opal_accelerator_base_module_mem_release_stream_fn_t mem_release_stream;
+    opal_accelerator_base_module_get_address_range_fn_t get_address_range;
+
+    opal_accelerator_base_module_is_ipc_enabled_fn_t is_ipc_enabled;
+    opal_accelerator_base_module_get_ipc_handle_fn_t get_ipc_handle;
+    opal_accelerator_base_module_import_ipc_handle_fn_t import_ipc_handle;
+    opal_accelerator_base_module_open_ipc_handle_fn_t open_ipc_handle;
+    opal_accelerator_base_module_compare_ipc_handles_fn_t compare_ipc_handles;
+    opal_accelerator_base_module_get_ipc_event_handle_fn_t get_ipc_event_handle;
+    opal_accelerator_base_module_import_ipc_event_handle_fn_t import_ipc_event_handle;
+    opal_accelerator_base_module_open_ipc_event_handle_fn_t open_ipc_event_handle;
+
+    opal_accelerator_base_module_host_register_fn_t host_register;
+    opal_accelerator_base_module_host_unregister_fn_t host_unregister;
+
+    opal_accelerator_base_module_get_device_fn_t get_device;
+    opal_accelerator_base_module_get_device_pci_attr_fn_t get_device_pci_attr;
+    opal_accelerator_base_module_device_can_access_peer_fn_t device_can_access_peer;
+
+    opal_accelerator_base_module_get_buffer_id_fn_t get_buffer_id;
+
+    opal_accelerator_base_module_get_num_devices_fn_t num_devices;
+    opal_accelerator_base_module_get_mem_bw_fn_t get_mem_bw;
+    opal_accelerator_base_module_get_memkind_fn_t get_memkind;
+} opal_accelerator_base_module_t;
+
+/**
+ * Accelerator component initialization.
+ * Called by MCA framework to initialize the component.
+ *
+ * This should initialize any component level data.
+ *
+ * This should discover accelerators that are available.
+ * We assume that only one accelerator will be present
+ * on any given node.
+ *
+ * @return                   Initialized module or NULL if init failed.
+ */
+typedef opal_accelerator_base_module_t * (*mca_accelerator_base_component_init_fn_t)(void);
+
+/**
+ * Accelerator component finalization
+ * Called by MCA framework to finalize the component.
+ *
+ * This should finalize the given accelerator component.
+ * Any component level data should be cleaned up, including
+ * any allocated during component_init() and data created
+ * during the lifetime of the component, including outstanding
+ * modules.
+ *
+ * @param[IN] module    If the component performed allocation within
+ *                      the module, allow the component the to perform
+ *                      the required cleanup
+ *
+ * No return since error will likely be ignored anyway.
+ */
+typedef void (*mca_accelerator_base_component_fini_fn_t)(opal_accelerator_base_module_t* module);
+
+typedef struct {
+    mca_base_component_t base_version;
+    mca_base_component_data_t base_data;
+    mca_accelerator_base_component_init_fn_t accelerator_init;
+    mca_accelerator_base_component_fini_fn_t accelerator_finalize;
+} opal_accelerator_base_component_t;
+
+/*
+ * Macro for use in components that are of type accelerator
+ */
+#define OPAL_ACCELERATOR_BASE_VERSION_1_0_0 OPAL_MCA_BASE_VERSION_2_1_0("accelerator", 1, 0, 0)
+
+/* Global structure for accessing accelerator functions */
+OPAL_DECLSPEC extern opal_accelerator_base_module_t opal_accelerator;
+
+END_C_DECLS
+
+#endif
diff --git a/opal/mca/accelerator/base/Makefile.am b/opal/mca/accelerator/base/Makefile.am
new file mode 100644
index 00000000000..91357b0ee8a
--- /dev/null
+++ b/opal/mca/accelerator/base/Makefile.am
@@ -0,0 +1,22 @@
+#
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2012-2013 Los Alamos National Security, Inc.  All rights reserved.
+# Copyright (c) 2014      Intel, Inc. All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+#                         All Rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+EXTRA_DIST = base/help-accelerator-base.txt
+
+headers += \
+	base/base.h
+
+libmca_accelerator_la_SOURCES += \
+	base/accelerator_base_frame.c \
+	base/accelerator_base_select.c
diff --git a/opal/mca/accelerator/base/accelerator_base_frame.c b/opal/mca/accelerator/base/accelerator_base_frame.c
new file mode 100644
index 00000000000..55e13f3773b
--- /dev/null
+++ b/opal/mca/accelerator/base/accelerator_base_frame.c
@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2014      Intel, Inc. All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "opal_config.h"
+
+#include "opal/constants.h"
+#include "opal/mca/accelerator/base/base.h"
+#include "opal/mca/base/base.h"
+#include "opal/mca/mca.h"
+
+/*
+ * The following file was created by configure.  It contains extern
+ * components and the definition of an array of pointers to each
+ * module's public mca_base_module_t struct.
+ */
+#include "opal/mca/accelerator/base/static-components.h"
+
+
+opal_accelerator_base_module_t opal_accelerator = {0};
+opal_accelerator_base_component_t opal_accelerator_base_selected_component = {{0}};
+
+static int opal_accelerator_base_frame_register(mca_base_register_flag_t flags)
+{
+    return OPAL_SUCCESS;
+}
+
+static int opal_accelerator_base_frame_close(void)
+{
+    return mca_base_framework_components_close(&opal_accelerator_base_framework, NULL);
+}
+
+static int opal_accelerator_base_frame_open(mca_base_open_flag_t flags)
+{
+    return mca_base_framework_components_open(&opal_accelerator_base_framework, flags);
+}
+
+OBJ_CLASS_INSTANCE(
+    opal_accelerator_stream_t,
+    opal_object_t,
+    NULL,
+    NULL);
+
+OBJ_CLASS_INSTANCE(
+    opal_accelerator_event_t,
+    opal_object_t,
+    NULL,
+    NULL);
+
+OBJ_CLASS_INSTANCE(
+    opal_accelerator_ipc_handle_t,
+    opal_object_t,
+    NULL,
+    NULL);
+
+OBJ_CLASS_INSTANCE(
+    opal_accelerator_ipc_event_handle_t,
+    opal_object_t,
+    NULL,
+    NULL);
+
+
+MCA_BASE_FRAMEWORK_DECLARE(opal, accelerator, "OPAL Accelerator Framework",
+                           opal_accelerator_base_frame_register, opal_accelerator_base_frame_open,
+                           opal_accelerator_base_frame_close, mca_accelerator_base_static_components,
+                           0);
diff --git a/opal/mca/accelerator/base/accelerator_base_select.c b/opal/mca/accelerator/base/accelerator_base_select.c
new file mode 100644
index 00000000000..86be62ace5c
--- /dev/null
+++ b/opal/mca/accelerator/base/accelerator_base_select.c
@@ -0,0 +1,182 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2012-2015 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015-2020 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2020-2022 Amazon.com, Inc. or its affiliates.  All Rights
+ * Copyright (c) 2018-2020 Triad National Security, LLC. All rights
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "opal_config.h"
+
+#include <string.h>
+
+#include "opal/mca/accelerator/accelerator.h"
+#include "opal/mca/accelerator/base/base.h"
+#include "opal/class/opal_list.h"
+#include "opal/constants.h"
+#include "opal/mca/base/base.h"
+#include "opal/mca/mca.h"
+#include "opal/util/output.h"
+#include "opal/util/show_help.h"
+#include "opal/util/proc.h"
+
+typedef struct accelerator_list_item_t {
+  opal_list_item_t super;
+  opal_accelerator_base_component_t *accelerator_component;
+  opal_accelerator_base_module_t *accelerator_module;
+} accelerator_list_item_t;
+
+static void multiple_accelerators_found_help_message(opal_list_t *components);
+
+int opal_accelerator_base_select(void)
+{
+    mca_base_component_list_item_t *cli = NULL;
+    mca_base_component_t *skip = NULL;
+    opal_accelerator_base_component_t *component = NULL;
+    opal_accelerator_base_module_t *module = NULL;
+    accelerator_list_item_t *ali = NULL, *ali2 = NULL;
+    opal_list_t ordered_list, initialized_list;
+    opal_list_item_t *item = NULL;
+
+    OBJ_CONSTRUCT(&ordered_list, opal_list_t);
+    OBJ_CONSTRUCT(&initialized_list, opal_list_t);
+
+    /* Traverse the list of available components and create a new ordered list with
+       the NULL Component on the back of the new list */
+    OPAL_LIST_FOREACH(cli, &opal_accelerator_base_framework.framework_components, mca_base_component_list_item_t) {
+        ali = (accelerator_list_item_t*) malloc(sizeof(accelerator_list_item_t));
+        if (NULL == ali) {
+            return OPAL_ERR_OUT_OF_RESOURCE;
+        }
+
+        OBJ_CONSTRUCT(ali, opal_list_item_t);
+        ali->accelerator_component = (opal_accelerator_base_component_t *) cli->cli_component;
+
+        if (0 == strcmp(cli->cli_component->mca_component_name, "null")) {
+            opal_list_append(&ordered_list, (opal_list_item_t*) ali);
+        }
+        else {
+            opal_list_prepend(&ordered_list, (opal_list_item_t*) ali);
+        }
+    }
+
+    /* Traverse the ordered list of available components and try and initialize every component.
+     * Save all initialized components in initialized component list. */
+    OPAL_LIST_FOREACH(ali, &ordered_list, accelerator_list_item_t) {
+        component = ali->accelerator_component;
+        if (NULL == component->accelerator_init) {
+            opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                                 "select: no init function; ignoring component %s",
+                                 component->base_version.mca_component_name);
+            continue;
+        }
+
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "select: initializing %s component %s",
+                            component->base_version.mca_type_name,
+                            component->base_version.mca_component_name);
+
+        module = component->accelerator_init();
+
+        if (NULL != module) {
+            ali2 = (accelerator_list_item_t*) malloc(sizeof(accelerator_list_item_t));
+            if (NULL == ali2) {
+                return OPAL_ERR_OUT_OF_RESOURCE;
+            }
+
+            OBJ_CONSTRUCT(ali2, opal_list_item_t);
+
+            ali2->accelerator_component = ali->accelerator_component;
+            ali2->accelerator_module = module;
+
+            opal_list_append(&initialized_list, (opal_list_item_t*) ali2);
+        }
+    }
+
+    /* There are four possible cases to handle:
+     * 1. List empty, someone selected a component with --mca accelerator <component> and it fails to initialize
+     * 2. There is 1 component in the list (NULL will always initialize, or someone selected a component)
+     * 3. There are 2 components in the list (User Defined, and NULL)
+     * 4. There are 3+ components in the list (User Defined1, User Defined 2, ..., NULL)
+     *
+     * Case 1: Abort and show simple help message
+     * Case 2 & 3 can be handled the same way by selecting the first component in the list
+     * Case 4: Show help (with a string containing all initialized components) and abort
+    */
+    if (0 == initialized_list.opal_list_length) {
+        opal_show_help("help-accelerator-base.txt", "No Accelerators Found", true);
+        /* leaks memory, but doesn't matter b/c ompi is aborting */
+        return OPAL_ERR_FATAL;
+    } else if (2 >= initialized_list.opal_list_length) {
+        ali = (accelerator_list_item_t *) opal_list_get_first(&initialized_list);
+        opal_accelerator_base_selected_component = *ali->accelerator_component;
+        skip = (mca_base_component_t *) ali->accelerator_component;
+        opal_accelerator = *ali->accelerator_module;
+    } else {
+        multiple_accelerators_found_help_message(&initialized_list);
+        /* leaks memory, but doesn't matter b/c ompi is aborting */
+        return OPAL_ERR_FATAL;
+    }
+
+    opal_output_verbose(10, opal_accelerator_base_framework.framework_output, "selected %s\n",
+                        opal_accelerator_base_selected_component.base_version.mca_component_name);
+
+    /* This base function closes, unloads, and removes from the available list all
+     * unselected components. The available list will contain only the selected component. */
+    mca_base_framework_components_close(&opal_accelerator_base_framework, skip);
+
+    /* Cleanup Lists */
+    for (item = opal_list_remove_first(&ordered_list); NULL != item; item = opal_list_remove_first(&ordered_list)) {
+        OBJ_DESTRUCT(item);
+        free(item);
+    }
+
+    for (item = opal_list_remove_first(&initialized_list); NULL != item; item = opal_list_remove_first(&initialized_list)) {
+        OBJ_DESTRUCT(item);
+        free(item);
+    }
+
+    OBJ_DESTRUCT(&ordered_list);
+    OBJ_DESTRUCT(&initialized_list);
+
+    return OPAL_SUCCESS;
+}
+
+static void multiple_accelerators_found_help_message(opal_list_t *components)
+{
+    accelerator_list_item_t *ali = NULL;
+    int i = 0;
+    int list_length = components->opal_list_length;
+    /* NULL Terminator + Do not include NULL component in list */
+    char* component_names = (char*) malloc(1 + ((list_length - 1) * MCA_BASE_MAX_COMPONENT_NAME_LEN));
+    component_names[0] = '\0';
+
+    OPAL_LIST_FOREACH(ali, components, accelerator_list_item_t) {
+        strcat(component_names, ali->accelerator_component->base_version.mca_component_name);
+        /* Do not include NULL component in list */
+        if (++i == list_length - 1) {
+            break;
+        }
+        strcat(component_names, ", ");
+    }
+
+    opal_show_help("help-accelerator-base.txt", "Multiple Accelerators Found", true, component_names);
+    free(component_names);
+}
diff --git a/opal/mca/accelerator/base/base.h b/opal/mca/accelerator/base/base.h
new file mode 100644
index 00000000000..e5922032ea8
--- /dev/null
+++ b/opal/mca/accelerator/base/base.h
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2014      Intel, Inc. All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+/** @file:
+ */
+
+#ifndef MCA_ACCELERATOR_BASE_H
+#define MCA_ACCELERATOR_BASE_H
+
+#include "opal_config.h"
+
+#include "opal/mca/accelerator/accelerator.h"
+#include "opal/mca/base/mca_base_framework.h"
+#include "opal/mca/mca.h"
+
+
+BEGIN_C_DECLS
+
+OPAL_DECLSPEC extern mca_base_framework_t opal_accelerator_base_framework;
+
+/**
+ * Select an accelerator module
+ */
+OPAL_DECLSPEC int opal_accelerator_base_select(void);
+
+OPAL_DECLSPEC extern opal_accelerator_base_component_t opal_accelerator_base_selected_component;
+
+END_C_DECLS
+
+#endif
diff --git a/opal/mca/accelerator/base/help-accelerator-base.txt b/opal/mca/accelerator/base/help-accelerator-base.txt
new file mode 100644
index 00000000000..6aaf53907c7
--- /dev/null
+++ b/opal/mca/accelerator/base/help-accelerator-base.txt
@@ -0,0 +1,20 @@
+# -*- text -*-
+#
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+#                         All Rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+[Multiple Accelerators Found]
+ERROR: Open MPI only supports one accelerator framework per node. Please pick one of the
+initialized accelerator components via command line argument "--mca accelerator <component>".
+The following accelerators initialized successfully: %s.
+The program will abort to allow you to resolve the accelerator conflict.
+
+
+[No Accelerators Found]
+ERROR: User specified an accelerator component via "--mca accelerator <component>", and that
+component failed to initialize successfully.  The program will abort to allow you to fix the issue.
diff --git a/opal/mca/accelerator/base/owner.txt b/opal/mca/accelerator/base/owner.txt
new file mode 100644
index 00000000000..c124ea811b5
--- /dev/null
+++ b/opal/mca/accelerator/base/owner.txt
@@ -0,0 +1,7 @@
+#
+# owner/status file
+# owner: institution that is responsible for this package
+# status: e.g. active, maintenance, unmaintained
+#
+owner: Amazon
+status: active
diff --git a/opal/mca/accelerator/cuda/Makefile.am b/opal/mca/accelerator/cuda/Makefile.am
new file mode 100644
index 00000000000..2c5260d8718
--- /dev/null
+++ b/opal/mca/accelerator/cuda/Makefile.am
@@ -0,0 +1,45 @@
+#
+# Copyright (c) 2014      Intel, Inc.  All rights reserved.
+# Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+#                         All Rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+
+AM_CPPFLAGS = $(common_cuda_CPPFLAGS)
+EXTRA_DIST = help-accelerator-cuda.txt
+sources = \
+        accelerator_cuda.h \
+        accelerator_cuda_component.c \
+        accelerator_cuda.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_opal_accelerator_cuda_DSO
+component_noinst =
+component_install = mca_accelerator_cuda.la
+else
+component_noinst = libmca_accelerator_cuda.la
+component_install =
+endif
+
+mcacomponentdir = $(opallibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+
+mca_accelerator_cuda_la_SOURCES = $(sources)
+mca_accelerator_cuda_la_LDFLAGS = -module -avoid-version $(accelerator_cuda_LDFLAGS)
+mca_accelerator_cuda_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
+        $(accelerator_cuda_LIBS)
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_accelerator_cuda_la_SOURCES =$(sources)
+libmca_accelerator_cuda_la_LDFLAGS = -module -avoid-version $(accelerator_cuda_LDFLAGS)
+libmca_accelerator_cuda_la_LIBADD = $(accelerator_cuda_LIBS)
diff --git a/opal/mca/accelerator/cuda/accelerator_cuda.c b/opal/mca/accelerator/cuda/accelerator_cuda.c
new file mode 100644
index 00000000000..6a88d98df48
--- /dev/null
+++ b/opal/mca/accelerator/cuda/accelerator_cuda.c
@@ -0,0 +1,1235 @@
+/*
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2014-2015 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2014      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c)           Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * Copyright (c) 2024      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "opal_config.h"
+
+#include <cuda.h>
+
+#include "accelerator_cuda.h"
+#include "opal/mca/accelerator/base/base.h"
+#include "opal/mca/rcache/rcache.h"
+#include "opal/util/show_help.h"
+#include "opal/util/proc.h"
+#include "ompi/info/info_memkind.h"
+/* Accelerator API's */
+static int accelerator_cuda_check_addr(const void *addr, int *dev_id, uint64_t *flags);
+static int accelerator_cuda_create_stream(int dev_id, opal_accelerator_stream_t **stream);
+
+static int accelerator_cuda_create_event(int dev_id, opal_accelerator_event_t **event, bool enable_ipc);
+static int accelerator_cuda_record_event(int dev_id, opal_accelerator_event_t *event, opal_accelerator_stream_t *stream);
+static int accelerator_cuda_query_event(int dev_id, opal_accelerator_event_t *event);
+static int accelerator_cuda_wait_event(int dev_id, opal_accelerator_event_t *event, opal_accelerator_stream_t *stream);
+
+static int accelerator_cuda_memcpy_async(int dest_dev_id, int src_dev_id, void *dest, const void *src, size_t size,
+                                  opal_accelerator_stream_t *stream, opal_accelerator_transfer_type_t type);
+static int accelerator_cuda_memcpy(int dest_dev_id, int src_dev_id, void *dest, const void *src,
+                            size_t size, opal_accelerator_transfer_type_t type);
+static int accelerator_cuda_memmove_async(int dest_dev_id, int src_dev_id, void *dest, const void *src,
+                                          size_t size, opal_accelerator_stream_t *stream,
+                                          opal_accelerator_transfer_type_t type);
+static int accelerator_cuda_memmove(int dest_dev_id, int src_dev_id, void *dest, const void *src, size_t size,
+                             opal_accelerator_transfer_type_t type);
+static int accelerator_cuda_mem_alloc(int dev_id, void **ptr, size_t size);
+static int accelerator_cuda_mem_release(int dev_id, void *ptr);
+static int accelerator_cuda_mem_alloc_stream(int dev_id, void **ptr, size_t size,
+                                             opal_accelerator_stream_t *stream);
+static int accelerator_cuda_mem_release_stream(int dev_id, void *ptr, opal_accelerator_stream_t *stream);
+static int accelerator_cuda_get_address_range(int dev_id, const void *ptr, void **base,
+                                              size_t *size);
+
+static bool accelerator_cuda_is_ipc_enabled(void);
+static int accelerator_cuda_get_ipc_handle(int dev_id, void *dev_ptr,
+                                           opal_accelerator_ipc_handle_t *handle);
+static int accelerator_cuda_import_ipc_handle(int dev_id, uint8_t ipc_handle[IPC_MAX_HANDLE_SIZE],
+                                              opal_accelerator_ipc_handle_t *handle);
+static int accelerator_cuda_open_ipc_handle(int dev_id, opal_accelerator_ipc_handle_t *handle,
+                                            void **dev_ptr);
+static int accelerator_cuda_compare_ipc_handles(uint8_t handle_1[IPC_MAX_HANDLE_SIZE],
+                                                uint8_t handle_2[IPC_MAX_HANDLE_SIZE]);
+static int accelerator_cuda_get_ipc_event_handle(opal_accelerator_event_t *event,
+                                                 opal_accelerator_ipc_event_handle_t *handle);
+static int accelerator_cuda_import_ipc_event_handle(uint8_t ipc_handle[IPC_MAX_HANDLE_SIZE],
+                                                    opal_accelerator_ipc_event_handle_t *handle);
+static int accelerator_cuda_open_ipc_event_handle(opal_accelerator_ipc_event_handle_t *handle,
+                                                  opal_accelerator_event_t *event);
+
+static int accelerator_cuda_host_register(int dev_id, void *ptr, size_t size);
+static int accelerator_cuda_host_unregister(int dev_id, void *ptr);
+
+static int accelerator_cuda_get_device(int *dev_id);
+static int accelerator_cuda_get_device_pci_attr(int dev_id, opal_accelerator_pci_attr_t *pci_attr);
+static int accelerator_cuda_device_can_access_peer( int *access, int dev1, int dev2);
+
+static int accelerator_cuda_get_buffer_id(int dev_id, const void *addr, opal_accelerator_buffer_id_t *buf_id);
+
+static int accelerator_cuda_sync_stream(opal_accelerator_stream_t *stream);
+static int accelerator_cuda_get_num_devices(int *num_devices);
+static int accelerator_cuda_get_mem_bw(int device, float *bw);
+static void accelerator_cuda_get_memkind(ompi_memkind_t *memkind);
+
+#define GET_STREAM(_stream) \
+    ((_stream) == MCA_ACCELERATOR_STREAM_DEFAULT ? 0 : *((CUstream *) (_stream)->stream))
+
+opal_accelerator_base_module_t opal_accelerator_cuda_module =
+{
+    accelerator_cuda_check_addr,
+
+    accelerator_cuda_create_stream,
+    accelerator_cuda_sync_stream,
+
+    accelerator_cuda_create_event,
+    accelerator_cuda_record_event,
+    accelerator_cuda_query_event,
+    accelerator_cuda_wait_event,
+
+    accelerator_cuda_memcpy_async,
+    accelerator_cuda_memcpy,
+    accelerator_cuda_memmove_async,
+    accelerator_cuda_memmove,
+    accelerator_cuda_mem_alloc,
+    accelerator_cuda_mem_release,
+    accelerator_cuda_mem_alloc_stream,
+    accelerator_cuda_mem_release_stream,
+    accelerator_cuda_get_address_range,
+
+    accelerator_cuda_is_ipc_enabled,
+    accelerator_cuda_get_ipc_handle,
+    accelerator_cuda_import_ipc_handle,
+    accelerator_cuda_open_ipc_handle,
+    accelerator_cuda_compare_ipc_handles,
+    accelerator_cuda_get_ipc_event_handle,
+    accelerator_cuda_import_ipc_event_handle,
+    accelerator_cuda_open_ipc_event_handle,
+
+    accelerator_cuda_host_register,
+    accelerator_cuda_host_unregister,
+
+    accelerator_cuda_get_device,
+    accelerator_cuda_get_device_pci_attr,
+    accelerator_cuda_device_can_access_peer,
+
+    accelerator_cuda_get_buffer_id,
+
+    accelerator_cuda_get_num_devices,
+    accelerator_cuda_get_mem_bw,
+    accelerator_cuda_get_memkind
+};
+
+static inline int opal_accelerator_cuda_delayed_init_check(void)
+{
+    if (OPAL_UNLIKELY(true != mca_accelerator_cuda_init_complete)) {
+        return opal_accelerator_cuda_delayed_init();
+    }
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_cuda_get_device_id(CUcontext mem_ctx) {
+    /* query the device from the context */
+    int dev_id = -1;
+    CUdevice ptr_dev;
+    cuCtxPushCurrent(mem_ctx);
+    cuCtxGetDevice(&ptr_dev);
+    for (int i = 0; i < opal_accelerator_cuda_num_devices; ++i) {
+        CUdevice dev;
+        cuDeviceGet(&dev, i);
+        if (dev == ptr_dev) {
+            dev_id = i;
+            break;
+        }
+    }
+    cuCtxPopCurrent(&mem_ctx);
+    return dev_id;
+}
+
+static int accelerator_cuda_check_vmm(CUdeviceptr dbuf, CUmemorytype *mem_type,
+                                      int *dev_id)
+{
+#if OPAL_CUDA_VMM_SUPPORT
+    static int device_count = -1;
+    CUmemAllocationProp prop;
+    CUmemLocation location;
+    CUresult result;
+    unsigned long long flags;
+    CUmemGenericAllocationHandle alloc_handle;
+
+    if (device_count == -1) {
+        result = cuDeviceGetCount(&device_count);
+        if (result != CUDA_SUCCESS) {
+            return 0;
+        }
+    }
+
+    result = cuMemRetainAllocationHandle(&alloc_handle, (void*)dbuf);
+    if (result != CUDA_SUCCESS) {
+        return 0;
+    }
+
+    result = cuMemGetAllocationPropertiesFromHandle(&prop, alloc_handle);
+    if (result != CUDA_SUCCESS) {
+        cuMemRelease(alloc_handle);
+        return 0;
+    }
+
+    if (prop.location.type == CU_MEM_LOCATION_TYPE_DEVICE) {
+        *mem_type = CU_MEMORYTYPE_DEVICE;
+        *dev_id  = prop.location.id;
+        cuMemRelease(alloc_handle);
+        return 1;
+    }
+
+    if (prop.location.type == CU_MEM_LOCATION_TYPE_HOST_NUMA) {
+        /* check if device has access */
+        for (int i = 0; i < device_count; i++) {
+            location.type = CU_MEM_LOCATION_TYPE_DEVICE;
+            location.id   = i;
+            result = cuMemGetAccess(&flags, &location, dbuf);
+            if ((CUDA_SUCCESS == result) &&
+                (CU_MEM_ACCESS_FLAGS_PROT_READWRITE == flags)) {
+                *mem_type = CU_MEMORYTYPE_DEVICE;
+                *dev_id  = i;
+                cuMemRelease(alloc_handle);
+                return 1;
+            }
+        }
+    }
+
+    /* host must have access as device access possibility is exhausted */
+    *mem_type = CU_MEMORYTYPE_HOST;
+    *dev_id = MCA_ACCELERATOR_NO_DEVICE_ID;
+    cuMemRelease(alloc_handle);
+    return 1;
+
+#endif
+
+    return 0;
+}
+
+static int accelerator_cuda_check_mpool(CUdeviceptr dbuf, CUmemorytype *mem_type,
+                                        int *dev_id)
+{
+#if OPAL_CUDA_VMM_SUPPORT
+    static int device_count = -1;
+    static int mpool_supported = -1;
+    CUresult result;
+    CUmemoryPool mpool;
+    CUmemAccess_flags flags;
+    CUmemLocation location;
+
+    if (mpool_supported <= 0) {
+        if (mpool_supported == -1) {
+            if (device_count == -1) {
+                result = cuDeviceGetCount(&device_count);
+                if (result != CUDA_SUCCESS || (0 == device_count)) {
+                    mpool_supported = 0;  /* never check again */
+                    device_count = 0;
+                    return 0;
+                }
+            }
+
+            /* assume uniformity of devices */
+            result = cuDeviceGetAttribute(&mpool_supported,
+                    CU_DEVICE_ATTRIBUTE_MEMORY_POOLS_SUPPORTED, 0);
+            if (result != CUDA_SUCCESS) {
+                mpool_supported = 0;
+            }
+        }
+        if (0 == mpool_supported) {
+            return 0;
+        }
+    }
+
+    result = cuPointerGetAttribute(&mpool, CU_POINTER_ATTRIBUTE_MEMPOOL_HANDLE,
+                                   dbuf);
+    if (CUDA_SUCCESS != result) {
+        return 0;
+    }
+
+    /* check if device has access */
+    for (int i = 0; i < device_count; i++) {
+        location.type = CU_MEM_LOCATION_TYPE_DEVICE;
+        location.id   = i;
+        result = cuMemPoolGetAccess(&flags, mpool, &location);
+        if ((CUDA_SUCCESS == result) &&
+            (CU_MEM_ACCESS_FLAGS_PROT_READWRITE == flags)) {
+            *mem_type = CU_MEMORYTYPE_DEVICE;
+            *dev_id  = i;
+            return 1;
+        }
+    }
+
+    /* host must have access as device access possibility is exhausted */
+    *mem_type = CU_MEMORYTYPE_HOST;
+    *dev_id = MCA_ACCELERATOR_NO_DEVICE_ID;
+    return 0;
+#endif
+
+    return 0;
+}
+
+static int accelerator_cuda_get_primary_context(CUdevice dev_id, CUcontext *pctx)
+{
+    CUresult result;
+    unsigned int flags;
+    int active;
+
+    result =  cuDevicePrimaryCtxGetState(dev_id, &flags, &active);
+    if (CUDA_SUCCESS != result) {
+        return OPAL_ERROR;
+    }
+
+    if (active) {
+        result = cuDevicePrimaryCtxRetain(pctx, dev_id);
+        return OPAL_SUCCESS;
+    }
+
+    return OPAL_ERROR;
+}
+
+static int accelerator_cuda_check_addr(const void *addr, int *dev_id, uint64_t *flags)
+{
+    CUresult result;
+    int is_vmm = 0;
+    int is_mpool_ptr = 0;
+    int vmm_dev_id = MCA_ACCELERATOR_NO_DEVICE_ID;
+    int mpool_dev_id = MCA_ACCELERATOR_NO_DEVICE_ID;
+    CUmemorytype vmm_mem_type = 0;
+    CUmemorytype mpool_mem_type = 0;
+    CUmemorytype mem_type = 0;
+    CUdeviceptr dbuf = (CUdeviceptr) addr;
+    CUcontext ctx = NULL, mem_ctx = NULL;
+    *dev_id = MCA_ACCELERATOR_NO_DEVICE_ID;
+
+    if (NULL == addr || NULL == flags) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    *flags = 0;
+
+    is_vmm = accelerator_cuda_check_vmm(dbuf, &vmm_mem_type, &vmm_dev_id);
+    is_mpool_ptr = accelerator_cuda_check_mpool(dbuf, &mpool_mem_type, &mpool_dev_id);
+
+#if OPAL_CUDA_GET_ATTRIBUTES
+    uint32_t is_managed = 0;
+    /* With CUDA 7.0, we can get multiple attributes with a single call */
+    CUpointer_attribute attributes[3] = {CU_POINTER_ATTRIBUTE_MEMORY_TYPE,
+                                         CU_POINTER_ATTRIBUTE_CONTEXT,
+                                         CU_POINTER_ATTRIBUTE_IS_MANAGED};
+    void *attrdata[] = {(void *) &mem_type, (void *) &mem_ctx, (void *) &is_managed};
+
+    result = cuPointerGetAttributes(3, attributes, attrdata, dbuf);
+    OPAL_OUTPUT_VERBOSE((101, opal_accelerator_base_framework.framework_output,
+                         "dbuf=%p, mem_type=%d, mem_ctx=%p, is_managed=%d, result=%d", (void *) dbuf,
+                         (int) mem_type, (void *) mem_ctx, is_managed, result));
+
+    /* Mark unified memory buffers with a flag.  This will allow all unified
+     * memory to be forced through host buffers.  Note that this memory can
+     * be either host or device so we need to set this flag prior to that check. */
+    if (1 == is_managed) {
+        *flags |= MCA_ACCELERATOR_FLAGS_UNIFIED_MEMORY;
+    }
+    if (CUDA_SUCCESS != result) {
+        /* If cuda is not initialized, assume it is a host buffer. It can also
+         * return invalid value if the ptr was not allocated by, mapped by,
+         * or registered with a CUcontext */
+        if (CUDA_ERROR_NOT_INITIALIZED == result || CUDA_ERROR_INVALID_VALUE == result) {
+            return 0;
+        } else {
+            return OPAL_ERROR;
+        }
+    } else if (CU_MEMORYTYPE_HOST == mem_type) {
+        if (is_vmm && (vmm_mem_type == CU_MEMORYTYPE_DEVICE)) {
+            mem_type = CU_MEMORYTYPE_DEVICE;
+            *dev_id = vmm_dev_id;
+        } else if (is_mpool_ptr && (mpool_mem_type == CU_MEMORYTYPE_DEVICE)) {
+            mem_type = CU_MEMORYTYPE_DEVICE;
+            *dev_id = mpool_dev_id;
+        } else {
+            /* Host memory, nothing to do here */
+            return 0;
+        }
+    } else if (0 == mem_type) {
+        /* This can happen when CUDA is initialized but dbuf is not valid CUDA pointer */
+        return 0;
+    } else {
+        if (is_vmm) {
+            *dev_id = vmm_dev_id;
+        } else if (is_mpool_ptr) {
+            *dev_id = mpool_dev_id;
+        } else {
+            /* query the device from the context */
+            *dev_id = accelerator_cuda_get_device_id(mem_ctx);
+        }
+    }
+#else /* OPAL_CUDA_GET_ATTRIBUTES */
+    result = cuPointerGetAttribute(&mem_type, CU_POINTER_ATTRIBUTE_MEMORY_TYPE, dbuf);
+    if (CUDA_SUCCESS != result) {
+        /* If cuda is not initialized, assume it is a host buffer. */
+        if (CUDA_ERROR_NOT_INITIALIZED == result) {
+            return 0;
+        } else {
+            return OPAL_ERROR;
+        }
+    } else if (CU_MEMORYTYPE_HOST == mem_type) {
+        if (is_vmm && (vmm_mem_type == CU_MEMORYTYPE_DEVICE)) {
+            mem_type = CU_MEMORYTYPE_DEVICE;
+            *dev_id = vmm_dev_id;
+        } else if (is_mpool_ptr && (mpool_mem_type == CU_MEMORYTYPE_DEVICE)) {
+            mem_type = CU_MEMORYTYPE_DEVICE;
+            *dev_id = mpool_dev_id;
+        } else {
+            /* Host memory, nothing to do here */
+            return 0;
+        }
+    } else {
+        if (is_vmm) {
+            *dev_id = vmm_dev_id;
+        } else if (is_mpool_ptr) {
+            *dev_id = mpool_dev_id;
+        } else {
+            result = cuPointerGetAttribute(&mem_ctx,
+                                           CU_POINTER_ATTRIBUTE_CONTEXT, dbuf);
+            /* query the device from the context */
+            *dev_id = accelerator_cuda_get_device_id(mem_ctx);
+        }
+    }
+#endif /* OPAL_CUDA_GET_ATTRIBUTES */
+
+    /* Must be a device pointer */
+    assert(CU_MEMORYTYPE_DEVICE == mem_type);
+
+    /* This piece of code was added in to handle in a case involving
+     * OMP threads.  The user had initialized CUDA and then spawned
+     * two threads.  The first thread had the CUDA context, but the
+     * second thread did not.  We therefore had no context to act upon
+     * and future CUDA driver calls would fail.  Therefore, if we have
+     * GPU memory, but no context, get the context from the GPU memory
+     * and set the current context to that.  It is rare that we will not
+     * have a context. */
+    result = cuCtxGetCurrent(&ctx);
+    if (OPAL_UNLIKELY(NULL == ctx)) {
+        if (CUDA_SUCCESS == result) {
+#if !OPAL_CUDA_GET_ATTRIBUTES
+            result = cuPointerGetAttribute(&mem_ctx, CU_POINTER_ATTRIBUTE_CONTEXT, dbuf);
+            if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
+                opal_output(0,
+                            "CUDA: error calling cuPointerGetAttribute: "
+                            "result=%d, ptr=%p aborting...",
+                            result, addr);
+                return OPAL_ERROR;
+            }
+#endif /* OPAL_CUDA_GET_ATTRIBUTES */
+            if (is_vmm || is_mpool_ptr) {
+                if (OPAL_SUCCESS ==
+                        accelerator_cuda_get_primary_context(
+                            is_vmm ? vmm_dev_id : mpool_dev_id, &mem_ctx)) {
+                    /* As VMM/mempool allocations have no context associated
+                     * with them, check if device primary context can be set */
+                } else {
+                    opal_output(0,
+                                "CUDA: unable to set ctx with the given pointer"
+                                "ptr=%p aborting...", addr);
+                    return OPAL_ERROR;
+                }
+            }
+
+            result = cuCtxSetCurrent(mem_ctx);
+            if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
+                opal_output(0,
+                            "CUDA: error calling cuCtxSetCurrent: "
+                            "result=%d, ptr=%p aborting...",
+                            result, addr);
+                return OPAL_ERROR;
+            } else {
+                OPAL_OUTPUT_VERBOSE(
+                    (10, opal_accelerator_base_framework.framework_output, "CUDA: cuCtxSetCurrent passed: ptr=%p", addr));
+            }
+        } else {
+            /* Print error and proceed */
+            opal_output(0,
+                        "CUDA: error calling cuCtxGetCurrent: "
+                        "result=%d, ptr=%p aborting...",
+                        result, addr);
+            return OPAL_ERROR;
+        }
+    }
+
+    /* WORKAROUND - There are times when the above code determines a pice of memory
+     * is GPU memory, but it actually is not.  That has been seen on multi-GPU systems
+     * with 6 or 8 GPUs on them. Therefore, we will do this extra check.  Note if we
+     * made it this far, then the assumption at this point is we have GPU memory.
+     * Unfotunately, this extra call is costing us another 100 ns almost doubling
+     * the cost of this entire function. */
+    if (OPAL_LIKELY(((CUDA_VERSION > 7000) ? 0 : 1))) {
+        CUdeviceptr pbase;
+        size_t psize;
+        result = cuMemGetAddressRange(&pbase, &psize, dbuf);
+        if (CUDA_SUCCESS != result) {
+            opal_output_verbose(5, opal_accelerator_base_framework.framework_output,
+                                "CUDA: cuMemGetAddressRange failed on this pointer: result=%d, buf=%p "
+                                "Overriding check and setting to host pointer. ",
+                                result, (void *) dbuf);
+            /* This cannot be GPU memory if the previous call failed */
+            return 0;
+        }
+    }
+    /* First access on a device pointer finalizes CUDA support initialization. */
+    (void)opal_accelerator_cuda_delayed_init_check();
+    return 1;
+}
+
+static int accelerator_cuda_create_stream(int dev_id, opal_accelerator_stream_t **stream)
+{
+    CUresult result;
+    int delayed_init = opal_accelerator_cuda_delayed_init_check();
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != delayed_init)) {
+        return delayed_init;
+    }
+    *stream = (opal_accelerator_stream_t*)OBJ_NEW(opal_accelerator_cuda_stream_t);
+    if (NULL == *stream) {
+        return OPAL_ERR_OUT_OF_RESOURCE;
+    }
+
+    (*stream)->stream = malloc(sizeof(CUstream));
+    if (NULL == (*stream)->stream) {
+        OBJ_RELEASE(*stream);
+        return OPAL_ERR_OUT_OF_RESOURCE;
+    }
+
+    result = cuStreamCreate((*stream)->stream, 0);
+    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
+        opal_show_help("help-accelerator-cuda.txt", "cuStreamCreate failed", true,
+                       OPAL_PROC_MY_HOSTNAME, result);
+        free((*stream)->stream);
+        OBJ_RELEASE(*stream);
+        return OPAL_ERROR;
+    }
+    return OPAL_SUCCESS;
+}
+
+static void opal_accelerator_cuda_stream_destruct(opal_accelerator_cuda_stream_t *stream)
+{
+    CUresult result;
+
+    if (MCA_ACCELERATOR_STREAM_DEFAULT != (opal_accelerator_stream_t *)stream &&
+        NULL != stream->base.stream) {
+        result = cuStreamDestroy(*(CUstream *)stream->base.stream);
+        if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
+            opal_show_help("help-accelerator-cuda.txt", "cuStreamDestroy failed", true,
+                           result);
+        }
+        free(stream->base.stream);
+    }
+}
+
+OBJ_CLASS_INSTANCE(
+    opal_accelerator_cuda_stream_t,
+    opal_accelerator_stream_t,
+    NULL,
+    opal_accelerator_cuda_stream_destruct);
+
+static int accelerator_cuda_create_event(int dev_id, opal_accelerator_event_t **event, bool enable_ipc)
+{
+    CUresult result;
+    int delayed_init = opal_accelerator_cuda_delayed_init_check();
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != delayed_init)) {
+        return delayed_init;
+    }
+
+    *event = (opal_accelerator_event_t*)OBJ_NEW(opal_accelerator_cuda_event_t);
+    if (NULL == *event) {
+        return OPAL_ERR_OUT_OF_RESOURCE;
+    }
+
+    (*event)->event = malloc(sizeof(CUevent));
+    if (NULL == (*event)->event) {
+        OBJ_RELEASE(*event);
+        return OPAL_ERR_OUT_OF_RESOURCE;
+    }
+    result = cuEventCreate((*event)->event, enable_ipc ? CU_EVENT_DISABLE_TIMING|CU_EVENT_INTERPROCESS :
+			   CU_EVENT_DISABLE_TIMING);
+    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
+        opal_show_help("help-accelerator-cuda.txt", "cuEventCreate failed", true,
+                       OPAL_PROC_MY_HOSTNAME, result);
+        free((*event)->event);
+        OBJ_RELEASE(*event);
+        return OPAL_ERROR;
+    }
+    return OPAL_SUCCESS;
+}
+
+static void opal_accelerator_cuda_event_destruct(opal_accelerator_cuda_event_t *event)
+{
+    CUresult result;
+    if (NULL != event->base.event) {
+        result = cuEventDestroy(*(CUevent *)event->base.event);
+        if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
+            opal_show_help("help-accelerator-cuda.txt", "cuEventDestroy failed", true,
+                           result);
+        }
+        free(event->base.event);
+    }
+}
+
+OBJ_CLASS_INSTANCE(
+    opal_accelerator_cuda_event_t,
+    opal_accelerator_event_t,
+    NULL,
+    opal_accelerator_cuda_event_destruct);
+
+static int accelerator_cuda_record_event(int dev_id, opal_accelerator_event_t *event, opal_accelerator_stream_t *stream)
+{
+    CUresult result;
+
+    if ((MCA_ACCELERATOR_STREAM_DEFAULT != stream &&
+        (NULL == stream || NULL == stream->stream)) ||
+        NULL == event) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    result = cuEventRecord(*(CUevent *)event->event, GET_STREAM(stream));
+    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
+        opal_show_help("help-accelerator-cuda.txt", "cuEventRecord failed", true,
+                       OPAL_PROC_MY_HOSTNAME, result);
+        return OPAL_ERROR;
+    }
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_cuda_query_event(int dev_id, opal_accelerator_event_t *event)
+{
+    CUresult result;
+
+    if (NULL == event) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    result = cuEventQuery(*(CUevent *)event->event);
+    switch (result) {
+        case CUDA_SUCCESS:
+            {
+                return OPAL_SUCCESS;
+                break;
+            }
+        case CUDA_ERROR_NOT_READY:
+            {
+                return OPAL_ERR_RESOURCE_BUSY;
+                break;
+            }
+        default:
+            {
+                opal_show_help("help-accelerator-cuda.txt", "cuEventQuery failed", true,
+                               result);
+                return OPAL_ERROR;
+            }
+    }
+}
+static int accelerator_cuda_wait_event(int dev_id, opal_accelerator_event_t *event, opal_accelerator_stream_t *stream)
+{
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
+
+static int accelerator_cuda_memcpy_async(int dest_dev_id, int src_dev_id, void *dest, const void *src, size_t size,
+                                  opal_accelerator_stream_t *stream, opal_accelerator_transfer_type_t type)
+{
+    CUresult result;
+
+    int delayed_init = opal_accelerator_cuda_delayed_init_check();
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != delayed_init)) {
+        return delayed_init;
+    }
+
+    if ((MCA_ACCELERATOR_STREAM_DEFAULT != stream && NULL == stream) ||
+        NULL == dest || NULL == src || size < 0) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+    if (0 == size) {
+        return OPAL_SUCCESS;
+    }
+
+    result = cuMemcpyAsync((CUdeviceptr) dest, (CUdeviceptr) src, size, GET_STREAM(stream));
+    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
+        opal_show_help("help-accelerator-cuda.txt", "cuMemcpyAsync failed", true, dest, src,
+                       size, result);
+        return OPAL_ERROR;
+    }
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_cuda_memcpy(int dest_dev_id, int src_dev_id, void *dest, const void *src,
+                            size_t size, opal_accelerator_transfer_type_t type)
+{
+    CUresult result;
+
+    int delayed_init = opal_accelerator_cuda_delayed_init_check();
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != delayed_init)) {
+        return delayed_init;
+    }
+
+    if (NULL == dest || NULL == src || size < 0) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+    if (0 == size) {
+        return OPAL_SUCCESS;
+    }
+
+    /* Async copy then synchronize is the default behavior as some applications
+     * cannot utilize synchronous copies. In addition, host memory does not need
+     * to be page-locked if an Async memory copy is done (It just makes it synchronous
+     * which is what we want anyway):
+     * https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#concurrent-execution-host-device
+     * Additionally, cuMemcpy is not necessarily always synchronous. See:
+     * https://docs.nvidia.com/cuda/cuda-driver-api/api-sync-behavior.html
+     * TODO: Add optimizations for type field */
+    result = cuMemcpyAsync((CUdeviceptr) dest, (CUdeviceptr) src, size,
+                           *(CUstream *) opal_accelerator_cuda_memcpy_stream.base.stream);
+    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
+        opal_show_help("help-accelerator-cuda.txt", "cuMemcpyAsync failed", true, dest, src,
+                       size, result);
+        return OPAL_ERROR;
+    }
+    result = cuStreamSynchronize(*(CUstream *) opal_accelerator_cuda_memcpy_stream.base.stream);
+    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
+        opal_show_help("help-accelerator-cuda.txt", "cuStreamSynchronize failed", true,
+                       OPAL_PROC_MY_HOSTNAME, result);
+        return OPAL_ERROR;
+    }
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_cuda_memmove_async(int dest_dev_id, int src_dev_id, void *dest,
+                                          const void *src, size_t size,
+                                          opal_accelerator_stream_t *stream,
+                                          opal_accelerator_transfer_type_t type)
+{
+    CUdeviceptr tmp;
+    CUresult result;
+    void *ptr;
+
+    int delayed_init = opal_accelerator_cuda_delayed_init_check();
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != delayed_init)) {
+        return delayed_init;
+    }
+
+    if (NULL == dest || NULL == src || size <= 0) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    result = accelerator_cuda_mem_alloc_stream(src_dev_id, &ptr, size, stream);
+    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
+        return OPAL_ERROR;
+    }
+    tmp = (CUdeviceptr)ptr;
+    result = cuMemcpyAsync(tmp, (CUdeviceptr) src, size, *(CUstream*)stream->stream);
+    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
+        opal_show_help("help-accelerator-cuda.txt", "cuMemcpyAsync failed", true, tmp, src, size,
+                       result);
+        return OPAL_ERROR;
+    }
+    result = cuMemcpyAsync((CUdeviceptr) dest, tmp, size, *(CUstream*)stream->stream);
+    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
+        opal_show_help("help-accelerator-cuda.txt", "cuMemcpyAsync failed", true, dest, tmp,
+                       size, result);
+        return OPAL_ERROR;
+    }
+    return accelerator_cuda_mem_release_stream(src_dev_id, ptr, stream);
+}
+
+static int accelerator_cuda_memmove(int dest_dev_id, int src_dev_id, void *dest, const void *src, size_t size,
+                                    opal_accelerator_transfer_type_t type)
+{
+    int ret;
+
+    ret = accelerator_cuda_memmove_async(dest_dev_id, src_dev_id, dest, src, size, &opal_accelerator_cuda_memcpy_stream.base, type);
+    if (OPAL_SUCCESS != ret) {
+        return OPAL_ERROR;
+    }
+    ret = accelerator_cuda_sync_stream(&opal_accelerator_cuda_memcpy_stream.base);
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != ret)) {
+        opal_show_help("help-accelerator-cuda.txt", "cuStreamSynchronize failed", true,
+                       OPAL_PROC_MY_HOSTNAME, ret);
+        return OPAL_ERROR;
+    }
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_cuda_mem_alloc(int dev_id, void **ptr, size_t size)
+{
+    CUresult result;
+
+    int delayed_init = opal_accelerator_cuda_delayed_init_check();
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != delayed_init)) {
+        return delayed_init;
+    }
+
+    if (NULL == ptr || 0 == size) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    result = cuMemAlloc((CUdeviceptr *) ptr, size);
+    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
+        opal_show_help("help-accelerator-cuda.txt", "cuMemAlloc failed", true,
+                        OPAL_PROC_MY_HOSTNAME, result);
+        return OPAL_ERROR;
+    }
+    return OPAL_SUCCESS;
+}
+
+
+
+static int accelerator_cuda_mem_alloc_stream(int dev_id, void **addr, size_t size,
+                                             opal_accelerator_stream_t *stream)
+{
+
+    int delayed_init = opal_accelerator_cuda_delayed_init();
+    if (OPAL_UNLIKELY(0 != delayed_init)) {
+        return delayed_init;
+    }
+
+    /* fall-back to regular stream allocation */
+
+    CUresult result = cuMemAllocAsync((CUdeviceptr*)addr, size, *(CUstream*)stream->stream);
+    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
+        opal_show_help("help-accelerator-cuda.txt", "cuMemAlloc failed", true,
+                        OPAL_PROC_MY_HOSTNAME, result);
+        return OPAL_ERROR;
+    }
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_cuda_mem_release(int dev_id, void *ptr)
+{
+    CUresult result;
+    if (NULL != ptr) {
+        result = cuMemFree((CUdeviceptr) ptr);
+        if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
+            opal_show_help("help-accelerator-cuda.txt", "cuMemFree failed", true,
+                           OPAL_PROC_MY_HOSTNAME, result);
+            return OPAL_ERROR;
+        }
+    }
+    return 0;
+}
+
+static int accelerator_cuda_mem_release_stream(int dev_id, void *addr,
+                                               opal_accelerator_stream_t *stream)
+{
+    CUresult result;
+
+    if (NULL == stream || NULL == addr) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    result = cuMemFreeAsync((CUdeviceptr)addr, *(CUstream*)stream->stream);
+    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
+        opal_show_help("help-accelerator-cuda.txt", "cuMemFree failed", true,
+                        OPAL_PROC_MY_HOSTNAME, result);
+        return OPAL_ERROR;
+    }
+    return OPAL_SUCCESS;
+}
+
+
+static int accelerator_cuda_sync_stream(opal_accelerator_stream_t *stream)
+{
+    CUresult result;
+    result = cuStreamSynchronize(*(CUstream*)stream->stream);
+    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
+        opal_show_help("help-accelerator-cuda.txt", "cuStreamSynchronize failed", true,
+                       OPAL_PROC_MY_HOSTNAME, result);
+        return OPAL_ERROR;
+    }
+    return OPAL_SUCCESS;
+}
+
+
+static int accelerator_cuda_get_address_range(int dev_id, const void *ptr, void **base,
+                                       size_t *size)
+{
+    CUresult result;
+
+    int delayed_init = opal_accelerator_cuda_delayed_init_check();
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != delayed_init)) {
+        return delayed_init;
+    }
+
+    if (NULL == ptr || NULL == base || NULL == size) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    result = cuMemGetAddressRange((CUdeviceptr *) base, size, (CUdeviceptr) ptr);
+    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
+        opal_show_help("help-accelerator-cuda.txt", "cuMemGetAddressRange failed 2", true,
+                       OPAL_PROC_MY_HOSTNAME, result, ptr);
+        return OPAL_ERROR;
+    } else {
+        opal_output_verbose(50, opal_accelerator_base_framework.framework_output,
+                            "CUDA: cuMemGetAddressRange passed: addr=%p, pbase=%p, psize=%lu ",
+                            ptr, *(char **) base, *size);
+    }
+    return 0;
+}
+
+static bool accelerator_cuda_is_ipc_enabled(void)
+{
+    return true;
+}
+
+static void mca_accelerator_cuda_ipc_handle_destruct(opal_accelerator_cuda_ipc_handle_t *handle)
+{
+    if (NULL != handle && NULL != handle->base.dev_ptr) {
+        cuIpcCloseMemHandle((CUdeviceptr) handle->base.dev_ptr);
+        handle->base.dev_ptr = NULL;
+    }
+}
+
+OBJ_CLASS_INSTANCE(
+    opal_accelerator_cuda_ipc_handle_t,
+    opal_accelerator_ipc_handle_t,
+    NULL,
+    mca_accelerator_cuda_ipc_handle_destruct);
+
+static int accelerator_cuda_get_ipc_handle(int dev_id, void *dev_ptr,
+                                           opal_accelerator_ipc_handle_t *handle)
+{
+    if (NULL == dev_ptr || NULL == handle) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    CUipcMemHandle cuda_ipc_handle;
+    opal_accelerator_cuda_ipc_handle_t *cuda_handle = (opal_accelerator_cuda_ipc_handle_t *) handle;
+
+    OBJ_CONSTRUCT(cuda_handle, opal_accelerator_cuda_ipc_handle_t);
+    cuda_handle->base.dev_ptr = NULL;
+
+    CUresult err = cuIpcGetMemHandle(&cuda_ipc_handle,
+				     (CUdeviceptr)dev_ptr);
+    if (OPAL_UNLIKELY(CUDA_SUCCESS != err)) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "Error in cuIpcGetMemHandle dev_ptr %p", dev_ptr);
+        OBJ_DESTRUCT(cuda_handle);
+        return OPAL_ERROR;
+    }
+    memcpy(cuda_handle->base.handle, &cuda_ipc_handle, IPC_MAX_HANDLE_SIZE);
+
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_cuda_import_ipc_handle(int dev_id, uint8_t ipc_handle[IPC_MAX_HANDLE_SIZE],
+                                              opal_accelerator_ipc_handle_t *handle)
+{
+    opal_accelerator_cuda_ipc_handle_t *cuda_handle = (opal_accelerator_cuda_ipc_handle_t *) handle;
+    OBJ_CONSTRUCT(cuda_handle, opal_accelerator_cuda_ipc_handle_t);
+    memcpy(cuda_handle->base.handle, ipc_handle, IPC_MAX_HANDLE_SIZE);
+
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_cuda_open_ipc_handle(int dev_id, opal_accelerator_ipc_handle_t *handle,
+                                            void **dev_ptr)
+{
+    if (NULL == dev_ptr || NULL == handle) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    CUresult err = cuIpcOpenMemHandle((CUdeviceptr *) &handle->dev_ptr,
+				      *(CUipcMemHandle*)handle->handle,
+				      CU_IPC_MEM_LAZY_ENABLE_PEER_ACCESS);
+    if (CUDA_ERROR_ALREADY_MAPPED == err) {
+        return OPAL_ERR_WOULD_BLOCK;
+    }
+    else if (CUDA_SUCCESS != err) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "error in cuIpcOpenMemHandle");
+        return OPAL_ERROR;
+    }
+    *dev_ptr = handle->dev_ptr;
+
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_cuda_compare_ipc_handles(uint8_t handle_1[IPC_MAX_HANDLE_SIZE],
+                                                uint8_t handle_2[IPC_MAX_HANDLE_SIZE])
+{
+    return memcmp(handle_1, handle_2, IPC_MAX_HANDLE_SIZE);
+}
+
+static void mca_accelerator_cuda_ipc_event_handle_destruct(opal_accelerator_cuda_ipc_handle_t *handle)
+{
+    // Just a place holder, there is no cuIpcCloseEventHandle.
+}
+
+OBJ_CLASS_INSTANCE(
+    opal_accelerator_cuda_ipc_event_handle_t,
+    opal_accelerator_ipc_event_handle_t,
+    NULL,
+    mca_accelerator_cuda_ipc_event_handle_destruct);
+
+static int accelerator_cuda_get_ipc_event_handle(opal_accelerator_event_t *event,
+                                                 opal_accelerator_ipc_event_handle_t *handle)
+{
+    if (NULL == event || NULL == handle) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    CUipcEventHandle cuda_ipc_handle;
+    opal_accelerator_cuda_ipc_event_handle_t *cuda_handle = (opal_accelerator_cuda_ipc_event_handle_t *) handle;
+    OBJ_CONSTRUCT(cuda_handle, opal_accelerator_cuda_ipc_event_handle_t);
+
+    memset(cuda_ipc_handle.reserved, 0, CU_IPC_HANDLE_SIZE);
+    CUresult err = cuIpcGetEventHandle(&cuda_ipc_handle,
+                                          *((CUevent *)event->event));
+    if (OPAL_UNLIKELY(CUDA_SUCCESS != err)) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "error in cuIpcGetEventHandle");
+        OBJ_DESTRUCT(cuda_handle);
+        return OPAL_ERROR;
+    }
+    memcpy(cuda_handle->base.handle, &cuda_ipc_handle, IPC_MAX_HANDLE_SIZE);
+
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_cuda_import_ipc_event_handle(uint8_t ipc_handle[IPC_MAX_HANDLE_SIZE],
+                                                    opal_accelerator_ipc_event_handle_t *handle)
+{
+    opal_accelerator_cuda_ipc_handle_t *cuda_handle = (opal_accelerator_cuda_ipc_handle_t *) handle;
+
+    OBJ_CONSTRUCT(cuda_handle, opal_accelerator_cuda_ipc_handle_t);
+    memcpy(cuda_handle->base.handle, ipc_handle, IPC_MAX_HANDLE_SIZE);
+
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_cuda_open_ipc_event_handle(opal_accelerator_ipc_event_handle_t *handle,
+                                                  opal_accelerator_event_t *event)
+{
+    if (NULL == event || NULL == handle) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    opal_accelerator_cuda_ipc_event_handle_t *cuda_handle = (opal_accelerator_cuda_ipc_event_handle_t *) handle;
+    opal_accelerator_cuda_event_t *cuda_event = (opal_accelerator_cuda_event_t *) event;
+    OBJ_CONSTRUCT(cuda_event, opal_accelerator_cuda_event_t);
+    cuda_event->base.event = malloc(sizeof(CUevent));
+    if (NULL == cuda_event->base.event) {
+        return OPAL_ERR_OUT_OF_RESOURCE;
+    }
+
+    CUresult err = cuIpcOpenEventHandle( (CUevent *)cuda_event->base.event,
+					 *((CUipcEventHandle*)cuda_handle->base.handle));
+    if (OPAL_UNLIKELY(CUDA_SUCCESS != err)) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "error in cuIpcOpenEventHandle");
+        return OPAL_ERROR;
+    }
+
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_cuda_host_register(int dev_id, void *ptr, size_t size)
+{
+    CUresult result;
+    int delayed_init = opal_accelerator_cuda_delayed_init_check();
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != delayed_init)) {
+        return delayed_init;
+    }
+
+    if (NULL == ptr && size > 0) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    result = cuMemHostRegister(ptr, size, 0);
+    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
+        opal_show_help("help-accelerator-cuda.txt", "cuMemHostRegister failed", true,
+                       ptr, size, OPAL_PROC_MY_HOSTNAME, result);
+        return OPAL_ERROR;
+    }
+
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_cuda_host_unregister(int dev_id, void *ptr)
+{
+    CUresult result;
+    if (NULL != ptr) {
+        result = cuMemHostUnregister(ptr);
+        if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
+            opal_show_help("help-accelerator-cuda.txt", "cuMemHostUnregister failed", true,
+                           ptr, OPAL_PROC_MY_HOSTNAME, result);
+            return OPAL_ERROR;
+        }
+    }
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_cuda_get_device(int *dev_id)
+{
+    CUdevice cuDev;
+    CUresult result;
+
+    int delayed_init = opal_accelerator_cuda_delayed_init_check();
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != delayed_init)) {
+        return delayed_init;
+    }
+
+    if (NULL == dev_id) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    result = cuCtxGetDevice(&cuDev);
+    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
+        opal_show_help("help-accelerator-cuda.txt", "cuCtxGetDevice failed", true,
+                       result);
+        return OPAL_ERROR;
+    }
+    *dev_id = cuDev;
+    return 0;
+}
+
+static int accelerator_cuda_get_device_pci_attr(int dev_id, opal_accelerator_pci_attr_t *pci_attr)
+{
+    CUresult result;
+    int ret;
+    static const int PCI_BUS_ID_LENGTH = 13;
+    char pci_bus_id[PCI_BUS_ID_LENGTH];
+    char domain_id[5] = {0}, bus_id[3] = {0}, device_id[3] = {0}, function_id[2] = {0};
+
+    if (NULL == pci_attr) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    result = cuDeviceGetPCIBusId(pci_bus_id, PCI_BUS_ID_LENGTH, dev_id);
+
+    if (CUDA_SUCCESS != result) {
+        opal_output_verbose(5, opal_accelerator_base_framework.framework_output,
+                            "CUDA: Failed to get device PCI bus id");
+        return OPAL_ERROR;
+    }
+
+    ret = sscanf(pci_bus_id, "%4s:%2s:%2s.%1s", domain_id, bus_id, device_id, function_id);
+    if (4 > ret) {
+        opal_output_verbose(5, opal_accelerator_base_framework.framework_output,
+                            "CUDA: Failed to parse device PCI bus id");
+        return OPAL_ERROR;
+    }
+
+    errno = 0;
+    pci_attr->domain_id = strtol(domain_id, NULL, 16);
+    pci_attr->bus_id = strtol(bus_id, NULL, 16);
+    pci_attr->device_id = strtol(device_id, NULL, 16);
+    pci_attr->function_id = strtol(function_id, NULL, 16);
+    if (0 != errno) {
+        return OPAL_ERROR;
+    }
+
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_cuda_device_can_access_peer(int *access, int dev1, int dev2)
+{
+    CUresult result;
+
+    int delayed_init = opal_accelerator_cuda_delayed_init_check();
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != delayed_init)) {
+        return delayed_init;
+    }
+
+    if (NULL == access) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    result = cuDeviceCanAccessPeer(access, (CUdevice) dev1, (CUdevice) dev2);
+    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
+        opal_show_help("help-accelerator-cuda.txt", "cuDeviceCanAccessPeer failed", true,
+                       OPAL_PROC_MY_HOSTNAME, result);
+        return OPAL_ERROR;
+    }
+    return 0;
+}
+
+/*
+ * Get the buffer ID from the memory.
+ * This is needed to ensure the cached registration is not stale.  If
+ * we fail to get buffer ID, print an error and set buffer ID to 0.
+ * Also set SYNC_MEMOPS on any GPU registration to ensure that
+ * synchronous copies complete before the buffer is accessed.
+ */
+static int accelerator_cuda_get_buffer_id(int dev_id, const void *addr, opal_accelerator_buffer_id_t *buf_id)
+{
+    CUresult result;
+    int enable = 1;
+
+    int delayed_init = opal_accelerator_cuda_delayed_init_check();
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != delayed_init)) {
+        return delayed_init;
+    }
+
+    result = cuPointerGetAttribute((unsigned long long *)buf_id, CU_POINTER_ATTRIBUTE_BUFFER_ID, (CUdeviceptr) addr);
+    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
+        opal_show_help("help-accelerator-cuda.txt", "bufferID failed", true, OPAL_PROC_MY_HOSTNAME,
+                       result);
+        return OPAL_ERROR;
+    }
+    result = cuPointerSetAttribute(&enable, CU_POINTER_ATTRIBUTE_SYNC_MEMOPS,
+                                       (CUdeviceptr) addr);
+    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
+        opal_show_help("help-accelerator-cuda.txt", "cuPointerSetAttribute failed", true,
+                       OPAL_PROC_MY_HOSTNAME, result, addr);
+        return OPAL_ERROR;
+    }
+    return OPAL_SUCCESS;
+}
+
+
+
+static int accelerator_cuda_get_num_devices(int *num_devices)
+{
+
+    int delayed_init = opal_accelerator_cuda_delayed_init();
+    if (OPAL_UNLIKELY(0 != delayed_init)) {
+        return delayed_init;
+    }
+
+    *num_devices = opal_accelerator_cuda_num_devices;
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_cuda_get_mem_bw(int device, float *bw)
+{
+    int delayed_init = opal_accelerator_cuda_delayed_init();
+    if (OPAL_UNLIKELY(0 != delayed_init)) {
+        return delayed_init;
+    }
+    assert(opal_accelerator_cuda_mem_bw != NULL);
+
+    *bw = opal_accelerator_cuda_mem_bw[device];
+    return OPAL_SUCCESS;
+}
+
+static void accelerator_cuda_get_memkind (ompi_memkind_t *memkind)
+{
+  memkind->im_name = strdup("cuda");
+  memkind->im_no_restrictors = false;
+  memkind->im_num_restrictors = 3;
+  memkind->im_restrictors[0] = strdup("host");
+  memkind->im_restrictors[1] = strdup("device");
+  memkind->im_restrictors[2] = strdup("managed");
+
+  return;
+}
diff --git a/opal/mca/accelerator/cuda/accelerator_cuda.h b/opal/mca/accelerator/cuda/accelerator_cuda.h
new file mode 100644
index 00000000000..70091729918
--- /dev/null
+++ b/opal/mca/accelerator/cuda/accelerator_cuda.h
@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2014      Intel, Inc.  All rights reserved.
+ * Copyright (c) 2017-2022 Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2024      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef MCA_ACCELERATOR_CUDA_H
+#define MCA_ACCELERATOR_CUDA_H
+
+#include "opal_config.h"
+
+#include <cuda.h>
+
+#include "opal/mca/accelerator/accelerator.h"
+#include "opal/mca/threads/mutex.h"
+
+BEGIN_C_DECLS
+
+typedef struct {
+    opal_accelerator_base_component_t super;
+} opal_accelerator_cuda_component_t;
+
+struct opal_accelerator_cuda_stream_t {
+    opal_accelerator_stream_t base;
+};
+typedef struct opal_accelerator_cuda_stream_t opal_accelerator_cuda_stream_t;
+OBJ_CLASS_DECLARATION(opal_accelerator_cuda_stream_t);
+
+struct opal_accelerator_cuda_event_t {
+    opal_accelerator_event_t base;
+};
+typedef struct opal_accelerator_cuda_event_t opal_accelerator_cuda_event_t;
+OBJ_CLASS_DECLARATION(opal_accelerator_cuda_event_t);
+
+struct opal_accelerator_cuda_ipc_handle_t {
+    opal_accelerator_ipc_handle_t base;
+};
+typedef struct opal_accelerator_cuda_ipc_handle_t opal_accelerator_cuda_ipc_handle_t;
+OBJ_CLASS_DECLARATION(opal_accelerator_cuda_ipc_handle_t);
+
+struct opal_accelerator_cuda_ipc_event_handle_t {
+    opal_accelerator_ipc_event_handle_t base;
+};
+typedef struct opal_accelerator_cuda_ipc_event_handle_t opal_accelerator_cuda_ipc_event_handle_t;
+OBJ_CLASS_DECLARATION(opal_accelerator_cuda_ipc_event_handle_t);
+
+/* Declare extern variables, defined in accelerator_cuda_component.c */
+extern opal_accelerator_cuda_stream_t opal_accelerator_cuda_memcpy_stream;
+extern opal_mutex_t opal_accelerator_cuda_stream_lock;
+extern bool mca_accelerator_cuda_init_complete;
+
+OPAL_DECLSPEC extern opal_accelerator_cuda_component_t mca_accelerator_cuda_component;
+
+extern opal_accelerator_base_module_t opal_accelerator_cuda_module;
+
+extern int opal_accelerator_cuda_delayed_init(void);
+
+OPAL_DECLSPEC extern int opal_accelerator_cuda_num_devices;
+
+OPAL_DECLSPEC extern float *opal_accelerator_cuda_mem_bw;
+
+OPAL_DECLSPEC extern int opal_accelerator_cuda_delayed_init(void);
+
+END_C_DECLS
+
+#endif /* MCA_ACCELERATOR_CUDA_H */
diff --git a/opal/mca/accelerator/cuda/accelerator_cuda_component.c b/opal/mca/accelerator/cuda/accelerator_cuda_component.c
new file mode 100644
index 00000000000..e93e61a4a0d
--- /dev/null
+++ b/opal/mca/accelerator/cuda/accelerator_cuda_component.c
@@ -0,0 +1,284 @@
+/*
+ * Copyright (c) 2014      Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2017-2022 Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2024      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ * These symbols are in a file by themselves to provide nice linker
+ * semantics.  Since linkers generally pull in symbols by object
+ * files, keeping these symbols as the only symbols in this file
+ * prevents utility programs such as "ompi_info" from having to import
+ * entire components just to query their version and parameters.
+ */
+
+#include "opal_config.h"
+
+#include <cuda.h>
+
+#include "accelerator_cuda.h"
+#include "opal/mca/accelerator/base/base.h"
+#include "opal/mca/dl/base/base.h"
+#include "opal/runtime/opal_params.h"
+#include "opal/util/argv.h"
+#include "opal/util/printf.h"
+#include "opal/util/proc.h"
+#include "opal/util/show_help.h"
+#include "opal/sys/atomic.h"
+
+/* Define global variables, used in accelerator_cuda.c */
+opal_accelerator_cuda_stream_t opal_accelerator_cuda_memcpy_stream = {0};
+int opal_accelerator_cuda_num_devices = 0;
+
+/* Initialization lock for delayed cuda initialization */
+static opal_mutex_t accelerator_cuda_init_lock;
+bool mca_accelerator_cuda_init_complete = false;
+
+float *opal_accelerator_cuda_mem_bw = NULL;
+
+#define STRINGIFY2(x) #x
+#define STRINGIFY(x)  STRINGIFY2(x)
+
+/* Unused variable that we register at init time and unregister at fini time.
+ * This is used to detect if user has done a device reset prior to MPI_Finalize.
+ * This is a workaround to avoid SEGVs.
+ */
+static int checkmem;
+static int ctx_ok = 1;
+
+/*
+ * Public string showing the accelerator cuda component version number
+ */
+const char *opal_accelerator_cuda_component_version_string
+    = "OPAL cuda accelerator MCA component version " OPAL_VERSION;
+
+/*
+ * Local function
+ */
+static int accelerator_cuda_open(void);
+static int accelerator_cuda_close(void);
+static int accelerator_cuda_component_register(void);
+static opal_accelerator_base_module_t* accelerator_cuda_init(void);
+static void accelerator_cuda_finalize(opal_accelerator_base_module_t* module);
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+
+opal_accelerator_cuda_component_t mca_accelerator_cuda_component = {{
+
+    /* First, the mca_component_t struct containing meta information
+     * about the component itself */
+
+    .base_version =
+        {
+            /* Indicate that we are a accelerator v1.1.0 component (which also
+             * implies a specific MCA version) */
+
+            OPAL_ACCELERATOR_BASE_VERSION_1_0_0,
+
+            /* Component name and version */
+
+            .mca_component_name = "cuda",
+            MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
+                                  OPAL_RELEASE_VERSION),
+
+            /* Component open and close functions */
+
+            .mca_open_component = accelerator_cuda_open,
+            .mca_close_component = accelerator_cuda_close,
+            .mca_register_component_params = accelerator_cuda_component_register,
+
+        },
+    /* Next the MCA v1.0.0 component meta data */
+    .base_data =
+        { /* The component is checkpoint ready */
+         MCA_BASE_METADATA_PARAM_CHECKPOINT},
+    .accelerator_init = accelerator_cuda_init,
+    .accelerator_finalize = accelerator_cuda_finalize,
+}};
+
+static int accelerator_cuda_open(void)
+{
+    /* construct the component fields */
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_cuda_close(void)
+{
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_cuda_component_register(void)
+{
+    return OPAL_SUCCESS;
+}
+
+int opal_accelerator_cuda_delayed_init()
+{
+    int result = OPAL_SUCCESS;
+    CUcontext cuContext;
+
+    /* Double checked locking to avoid having to
+     * grab locks post lazy-initialization.  */
+    opal_atomic_rmb();
+    if (true == mca_accelerator_cuda_init_complete) {
+        return OPAL_SUCCESS;
+    }
+    OPAL_THREAD_LOCK(&accelerator_cuda_init_lock);
+
+    /* If already initialized, just exit */
+    if (true == mca_accelerator_cuda_init_complete) {
+        goto out;
+    }
+
+    cuDeviceGetCount(&opal_accelerator_cuda_num_devices);
+
+    /* Check to see if this process is running in a CUDA context.  If
+     * so, all is good.  If not, then disable registration of memory. */
+    result = cuCtxGetCurrent(&cuContext);
+    if (CUDA_SUCCESS != result) {
+        result = OPAL_ERR_NOT_INITIALIZED;
+        opal_output_verbose(20, opal_accelerator_base_framework.framework_output, "CUDA: cuCtxGetCurrent failed");
+        goto out;
+    } else if ((CUDA_SUCCESS == result) && (NULL == cuContext)) {
+        opal_output_verbose(20, opal_accelerator_base_framework.framework_output, "CUDA: cuCtxGetCurrent returned NULL context");
+
+        /* create a context for each device */
+        for (int i = 0; i < opal_accelerator_cuda_num_devices; ++i) {
+            CUdevice dev;
+            result = cuDeviceGet(&dev, i);
+            if (CUDA_SUCCESS != result) {
+                opal_output_verbose(20, opal_accelerator_base_framework.framework_output,
+                                    "CUDA: cuDeviceGet failed");
+                result = OPAL_ERROR;
+                goto out;
+            }
+            result = cuDevicePrimaryCtxRetain(&cuContext, dev);
+            if (CUDA_SUCCESS != result) {
+                opal_output_verbose(20, opal_accelerator_base_framework.framework_output,
+                                    "CUDA: cuDevicePrimaryCtxRetain failed");
+                result = OPAL_ERROR;
+                goto out;
+            }
+            if (0 == i) {
+                result = cuCtxPushCurrent(cuContext);
+                if (CUDA_SUCCESS != result) {
+                    opal_output_verbose(20, opal_accelerator_base_framework.framework_output,
+                                        "CUDA: cuCtxPushCurrent failed");
+                    result = OPAL_ERROR;
+                    goto out;
+                }
+            }
+        }
+
+
+    } else {
+        opal_output_verbose(20, opal_accelerator_base_framework.framework_output, "CUDA: cuCtxGetCurrent succeeded");
+    }
+
+    /* Create stream for use in cuMemcpyAsync synchronous copies */
+    CUstream memcpy_stream;
+    result = cuStreamCreate(&memcpy_stream, 0);
+    if (OPAL_UNLIKELY(result != CUDA_SUCCESS)) {
+        opal_show_help("help-accelerator-cuda.txt", "cuStreamCreate failed", true,
+                       OPAL_PROC_MY_HOSTNAME, result);
+        goto out;
+    }
+    opal_accelerator_cuda_memcpy_stream.base.stream = malloc(sizeof(CUstream));
+    *(CUstream*)opal_accelerator_cuda_memcpy_stream.base.stream = memcpy_stream;
+
+    result = cuMemHostRegister(&checkmem, sizeof(int), 0);
+    if (result != CUDA_SUCCESS) {
+        /* If registering the memory fails, print a message and continue.
+         * This is not a fatal error. */
+        opal_show_help("help-accelerator-cuda.txt", "cuMemHostRegister during init failed", true,
+                       &checkmem, sizeof(int), OPAL_PROC_MY_HOSTNAME, result, "checkmem");
+    } else {
+        opal_output_verbose(20, opal_accelerator_base_framework.framework_output,
+                            "CUDA: cuMemHostRegister OK on test region");
+    }
+
+    /* determine the memory bandwidth */
+    opal_accelerator_cuda_mem_bw = malloc(sizeof(float)*opal_accelerator_cuda_num_devices);
+    for (int i = 0; i < opal_accelerator_cuda_num_devices; ++i) {
+        CUdevice dev;
+        result = cuDeviceGet(&dev, i);
+        if (CUDA_SUCCESS != result) {
+            opal_output_verbose(20, opal_accelerator_base_framework.framework_output,
+                                "CUDA: cuDeviceGet failed");
+            goto out;
+        }
+        int mem_clock_rate; // kHz
+        result = cuDeviceGetAttribute(&mem_clock_rate,
+                                CU_DEVICE_ATTRIBUTE_MEMORY_CLOCK_RATE,
+                                dev);
+        int bus_width; // bit
+        result = cuDeviceGetAttribute(&bus_width,
+                                CU_DEVICE_ATTRIBUTE_GLOBAL_MEMORY_BUS_WIDTH,
+                                dev);
+        /* bw = clock_rate * bus width * 2bit multiplier
+         * See https://forums.developer.nvidia.com/t/memory-clock-rate/107940
+         */
+        float bw = ((float)mem_clock_rate*(float)bus_width*2.0) / 1024 / 1024 / 8;
+        opal_accelerator_cuda_mem_bw[i] = bw;
+    }
+
+    result = OPAL_SUCCESS;
+    opal_atomic_wmb();
+    mca_accelerator_cuda_init_complete = true;
+out:
+    OPAL_THREAD_UNLOCK(&accelerator_cuda_init_lock);
+    return result;
+}
+
+static opal_accelerator_base_module_t* accelerator_cuda_init(void)
+{
+    OBJ_CONSTRUCT(&accelerator_cuda_init_lock, opal_mutex_t);
+    OBJ_CONSTRUCT(&opal_accelerator_cuda_memcpy_stream, opal_accelerator_cuda_stream_t);
+
+    /* First check if the support is enabled.  In the case that the user has
+     * turned it off, we do not need to continue with any CUDA specific
+     * initialization.  Do this after MCA parameter registration. */
+    if (!opal_cuda_support) {
+        return NULL;
+    }
+
+    (void)opal_accelerator_cuda_delayed_init();
+    return &opal_accelerator_cuda_module;
+}
+
+static void accelerator_cuda_finalize(opal_accelerator_base_module_t* module)
+{
+    CUresult result;
+    /* This call is in here to make sure the context is still valid.
+     * This was the one way of checking which did not cause problems
+     * while calling into the CUDA library.  This check will detect if
+     * a user has called cudaDeviceReset prior to MPI_Finalize. If so,
+     * then this call will fail and we skip cleaning up CUDA resources. */
+    result = cuMemHostUnregister(&checkmem);
+    if (CUDA_SUCCESS != result) {
+        ctx_ok = 0;
+    }
+
+    if ((NULL != opal_accelerator_cuda_memcpy_stream.base.stream) && ctx_ok) {
+        OBJ_DESTRUCT(&opal_accelerator_cuda_memcpy_stream);
+    }
+
+    free(opal_accelerator_cuda_mem_bw);
+    opal_accelerator_cuda_mem_bw = NULL;
+
+    OBJ_DESTRUCT(&accelerator_cuda_init_lock);
+    return;
+}
diff --git a/opal/mca/accelerator/cuda/configure.m4 b/opal/mca/accelerator/cuda/configure.m4
new file mode 100644
index 00000000000..aa67623c8b2
--- /dev/null
+++ b/opal/mca/accelerator/cuda/configure.m4
@@ -0,0 +1,36 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2011-2013 NVIDIA Corporation.  All rights reserved.
+# Copyright (c) 2013      The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+#                         All Rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+#
+# If CUDA support was requested, then build the CUDA support library.
+# This code checks makes sure the check was done earlier by the
+# opal_check_cuda.m4 code. It also copies the flags and libs under
+# opal_cuda_CPPFLAGS, opal_cuda_LDFLAGS, and opal_cuda_LIBS
+
+AC_DEFUN([MCA_opal_accelerator_cuda_CONFIG],[
+
+    AC_CONFIG_FILES([opal/mca/accelerator/cuda/Makefile])
+
+    OPAL_CHECK_CUDA([accelerator_cuda])
+
+    AS_IF([test "x$CUDA_SUPPORT" = "x1"],
+          [$1],
+          [$2])
+
+    AC_SUBST([accelerator_cuda_CPPFLAGS])
+    AC_SUBST([accelerator_cuda_LDFLAGS])
+    AC_SUBST([accelerator_cuda_LIBS])
+
+])dnl
diff --git a/opal/mca/accelerator/cuda/help-accelerator-cuda.txt b/opal/mca/accelerator/cuda/help-accelerator-cuda.txt
new file mode 100644
index 00000000000..2cf7a14bf5d
--- /dev/null
+++ b/opal/mca/accelerator/cuda/help-accelerator-cuda.txt
@@ -0,0 +1,264 @@
+# -*- text -*-
+#
+# Copyright (c) 2011-2015 NVIDIA.  All rights reserved.
+# Copyright (c) 2015      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+#                         All Rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+[cuCtxGetCurrent failed not initialized]
+WARNING: The call to cuCtxGetCurrent() failed while attempting to register
+internal memory with the CUDA environment.  The program will continue to run,
+but the performance of GPU memory transfers may be reduced.  This failure
+indicates that the CUDA environment is not yet initialized.  To eliminate
+this warning, ensure that CUDA is initialized prior to calling MPI_Init.
+
+NOTE: You can turn off this warning by setting the MCA parameter
+      mpi_common_cuda_warning to 0.
+#
+[cuCtxGetCurrent failed]
+WARNING: The call to cuCtxGetCurrent() failed while attempting to register
+internal memory with the CUDA environment.  The program will continue to run,
+but the performance of GPU memory transfers may be reduced.
+  cuCtxGetCurrent return value:   %d
+
+NOTE: You can turn off this warning by setting the MCA parameter
+      mpi_common_cuda_warning to 0.
+#
+[cuCtxGetCurrent returned NULL]
+WARNING: The call to cuCtxGetCurrent() failed while attempting to register
+internal memory with the CUDA environment.  The program will continue to run,
+but the performance of GPU memory transfers may be reduced.  This failure
+indicates that there is no CUDA context yet.  To eliminate this warning,
+ensure that there is a CUDA context prior to calling MPI_Init.
+
+NOTE: You can turn off this warning by setting the MCA parameter
+      mpi_common_cuda_warning to 0.
+#
+[cuCtxGetDevice failed]
+WARNING: The call to cuCtxGetDevice() failed.
+  cuCtxGetDevice return value:   %d
+
+NOTE: You can turn off this warning by setting the MCA parameter
+      mpi_common_cuda_warning to 0.
+#
+[cuMemHostRegister during init failed]
+The call to cuMemHostRegister(%p, %d, 0) failed.
+  Host:  %s
+  cuMemHostRegister return value:  %d
+  Registration cache:  %s
+#
+[cuMemHostRegister failed]
+The call to cuMemHostRegister(%p, %d, 0) failed.
+  Host:  %s
+  cuMemHostRegister return value:  %d
+  Registration cache:  %s
+#
+[cuMemHostUnregister failed]
+The call to cuMemHostRegister(%p) failed.
+  Host:  %s
+  cuMemHostUnregister return value:  %d
+#
+[cuIpcGetMemHandle failed]
+The call to cuIpcGetMemHandle failed. This means the GPU RDMA protocol
+cannot be used.
+  cuIpcGetMemHandle return value:   %d
+  address: %p
+Check the cuda.h file for what the return value means. Perhaps a reboot
+of the node will clear the problem.
+#
+[cuMemGetAddressRange failed]
+The call to cuMemGetAddressRange failed. This means the GPU RDMA protocol
+cannot be used.
+  cuMemGetAddressRange return value:   %d
+  address: %p
+Check the cuda.h file for what the return value means. Perhaps a reboot
+of the node will clear the problem.
+#
+[cuMemGetAddressRange failed 2]
+The call to cuMemGetAddressRange failed during the GPU RDMA protocol.
+  Host:  %s
+  cuMemGetAddressRange return value:  %d
+  address:  %p
+Check the cuda.h file for what the return value means. This is highly
+unusual and should not happen. The program will probably abort.
+#
+[Out of cuEvent handles]
+The library has exceeded its number of outstanding event handles.
+For better performance, this number should be increased.
+  Current maximum handles:   %4d
+  Suggested new maximum:     %4d
+Rerun with --mca mpi_common_cuda_event_max %d
+#
+[cuIpcOpenMemHandle failed]
+The call to cuIpcOpenMemHandle failed. This is an unrecoverable error
+and will cause the program to abort.
+  Hostname:                         %s
+  cuIpcOpenMemHandle return value:  %d
+  address:                          %p
+Check the cuda.h file for what the return value means. A possible cause
+for this is not enough free device memory.  Try to reduce the device
+memory footprint of your application.
+#
+[cuIpcCloseMemHandle failed]
+The call to cuIpcCloseMemHandle failed. This is a warning and the program
+will continue to run.
+  cuIpcCloseMemHandle return value:   %d
+  address: %p
+Check the cuda.h file for what the return value means. Perhaps a reboot
+of the node will clear the problem.
+#
+[cuMemcpyAsync failed]
+The call to cuMemcpyAsync failed. This is a unrecoverable error and will
+cause the program to abort.
+  cuMemcpyAsync(%p, %p, %d) returned value %d
+Check the cuda.h file for what the return value means.
+#
+[cuEventCreate failed]
+The call to cuEventCreate failed. This is a unrecoverable error and will
+cause the program to abort.
+  Hostname:                     %s
+  cuEventCreate return value:   %d
+Check the cuda.h file for what the return value means.
+#
+[cuEventRecord failed]
+The call to cuEventRecord failed. This is a unrecoverable error and will
+cause the program to abort.
+  Hostname:                     %s
+  cuEventRecord return value:   %d
+Check the cuda.h file for what the return value means.
+#
+[cuEventQuery failed]
+The call to cuEventQuery failed. This is a unrecoverable error and will
+cause the program to abort.
+  cuEventQuery return value:   %d
+Check the cuda.h file for what the return value means.
+#
+[cuEventSynchronize failed]
+The call to cuEventSynchronize failed. This is highly unusual and should
+not happen.  Please report this error to the Open MPI developers.
+  cuEventSynchronize return value:     %d
+#
+[cuIpcGetEventHandle failed]
+The call to cuIpcGetEventHandle failed. This is a unrecoverable error and will
+cause the program to abort.
+  cuIpcGetEventHandle return value:   %d
+Check the cuda.h file for what the return value means.
+#
+[cuIpcOpenEventHandle failed]
+The call to cuIpcOpenEventHandle failed. This is a unrecoverable error and will
+cause the program to abort.
+  cuIpcOpenEventHandle return value:   %d
+Check the cuda.h file for what the return value means.
+#
+[cuStreamWaitEvent failed]
+The call to cuStreamWaitEvent failed. This is a unrecoverable error and will
+cause the program to abort.
+  cuStreamWaitEvent return value:   %d
+Check the cuda.h file for what the return value means.
+#
+[cuEventDestroy failed]
+The call to cuEventDestroy failed. This is a unrecoverable error and will
+cause the program to abort.
+  cuEventDestroy return value:   %d
+Check the cuda.h file for what the return value means.
+#
+[cuStreamCreate failed]
+The call to cuStreamCreate failed.  This is a unrecoverable error and will
+cause the program to abort.
+  Hostname:                      %s
+  cuStreamCreate return value:   %d
+Check the cuda.h file for what the return vale means.
+#
+[cuStreamDestroy failed]
+The call to cuStreamDestroy failed. This is highly unusual and should
+not happen.  Please report this error to the Open MPI developers.
+  cuStreamDestroy return value:         %d
+Check the cuda.h file for what the return value means.
+#
+[dlopen disabled]
+Open MPI was compiled without dynamic library support (e.g., with the
+ --disable-dlopen flag), and therefore cannot utilize CUDA support.
+
+If you need CUDA support, reconfigure Open MPI with dynamic library support enabled.
+#
+[dlopen failed]
+The library attempted to open the following supporting CUDA libraries,
+but each of them failed.  CUDA-aware support is disabled.
+%s
+If you do not require CUDA-aware support, then run with
+--mca opal_warn_on_missing_libcuda 0 to suppress this message.  If you do
+require CUDA-aware support, then try setting LD_LIBRARY_PATH to the location
+of libcuda.so.1 to resolve this issue.
+#
+[dlsym failed]
+An error occurred while trying to map in the address of a function.
+  Function Name: %s
+  Error string:  %s
+CUDA-aware support is disabled.
+#
+[bufferID failed]
+An error occurred while trying to get the BUFFER_ID of a GPU memory
+region.  This could cause incorrect results.  Turn of GPU Direct RDMA
+support by running with --mca btl_openib_cuda_want_gdr_support 0.
+  Hostname:                             %s
+  cuPointerGetAttribute return value:   %d
+Check the cuda.h file for what the return value means.
+[cuPointerSetAttribute failed]
+The call to cuPointerSetAttribute with CU_POINTER_ATTRIBUTE_SYNC_MEMOPS
+failed. This is highly unusual and should not happen.  The program will
+continue, but report this error to the Open MPI developers.
+  Hostname:                             %s
+  cuPointerSetAttribute return value:   %d
+  Address:                              %p
+Check the cuda.h file for what the return value means.
+#
+[cuStreamSynchronize failed]
+The call to cuStreamSynchronize failed. This is highly unusual and should
+not happen.  Please report this error to the Open MPI developers.
+  Hostname:                             %s
+  cuStreamSynchronize return value:     %d
+Check the cuda.h file for what the return value means.
+#
+[cuMemcpy failed]
+The call to cuMemcpy failed. This is highly unusual and should
+not happen.  Please report this error to the Open MPI developers.
+  Hostname:                  %s
+  cuMemcpy return value:     %d
+Check the cuda.h file for what the return value means.
+#
+[cuMemcpy2D failed]
+The call to cuMemcpy2D failed. This is highly unusual and should
+not happen.  Please report this error to the Open MPI developers.
+  Hostname:                  %s
+  cuMemcpy2D return value:   %d
+Check the cuda.h file for what the return value means.
+#
+[cuMemAlloc failed]
+The call to cuMemAlloc failed. This is highly unusual and should
+not happen.  Please report this error to the Open MPI developers.
+  Hostname:                  %s
+  cuMemAlloc return value:   %d
+Check the cuda.h file for what the return value means.
+#
+[cuMemFree failed]
+The call to cuMemFree failed. This is highly unusual and should
+not happen.  Please report this error to the Open MPI developers.
+  Hostname:                  %s
+  cuMemFree return value:    %d
+Check the cuda.h file for what the return value means.
+#
+[cuDeviceCanAccessPeer failed]
+The call to cuDeviceCanAccessPeer failed.
+  Hostname:                              %s
+  cuDeviceCanAccessPeer return value:    %d
+Check the cuda.h file for what the return value means.
+#
+[No memory]
+A call to allocate memory within the CUDA support failed.  This is
+an unrecoverable error and will cause the program to abort.
+  Hostname:  %s
diff --git a/opal/mca/accelerator/cuda/owner.txt b/opal/mca/accelerator/cuda/owner.txt
new file mode 100644
index 00000000000..0177f4391ab
--- /dev/null
+++ b/opal/mca/accelerator/cuda/owner.txt
@@ -0,0 +1,7 @@
+#
+# owner/status file
+# owner: institution that is responsible for this package
+# status: e.g. active, maintenance, unmaintained
+#
+owner: NVIDIA
+status: active
diff --git a/opal/mca/accelerator/null/Makefile.am b/opal/mca/accelerator/null/Makefile.am
new file mode 100644
index 00000000000..c9bef40b15a
--- /dev/null
+++ b/opal/mca/accelerator/null/Makefile.am
@@ -0,0 +1,37 @@
+#
+# Copyright (c) 2014      Intel, Inc.  All rights reserved.
+# Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+#                         All Rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+sources = \
+        accelerator_null_component.h \
+        accelerator_null_component.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_opal_accelerator_null_DSO
+component_noinst =
+component_install = mca_accelerator_null.la
+else
+component_noinst = libmca_accelerator_null.la
+component_install =
+endif
+
+mcacomponentdir = $(opallibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_accelerator_null_la_SOURCES = $(sources)
+mca_accelerator_null_la_LDFLAGS = -module -avoid-version
+mca_accelerator_null_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_accelerator_null_la_SOURCES =$(sources)
+libmca_accelerator_null_la_LDFLAGS = -module -avoid-version
diff --git a/opal/mca/accelerator/null/accelerator_null_component.c b/opal/mca/accelerator/null/accelerator_null_component.c
new file mode 100644
index 00000000000..996e5d59bce
--- /dev/null
+++ b/opal/mca/accelerator/null/accelerator_null_component.c
@@ -0,0 +1,407 @@
+/*
+ * Copyright (c) 2014-2015 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2014      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c)           Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * Copyright (c) 2023      Advanced Micro Devices, Inc. All rights reserved.
+ * Copyright (c) 2024      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "opal_config.h"
+
+#include "accelerator_null_component.h"
+#include "opal/constants.h"
+#include "ompi/info/info_memkind.h"
+#include <string.h>
+
+/*
+ * Public string showing the accelerator null component version number
+ */
+const char *opal_accelerator_null_component_version_string
+    = "OPAL null accelerator MCA component version " OPAL_VERSION;
+
+/*
+ * Component API functions
+ */
+static int accelerator_null_open(void);
+static int accelerator_null_close(void);
+static int accelerator_null_component_register(void);
+static opal_accelerator_base_module_t* accelerator_null_init(void);
+static void accelerator_null_finalize(opal_accelerator_base_module_t* module);
+
+/* Accelerator API's */
+static int accelerator_null_check_addr(const void *addr, int *dev_id, uint64_t *flags);
+
+static int accelerator_null_create_stream(int dev_id, opal_accelerator_stream_t **stream);
+static int accelerator_null_create_event(int dev_id, opal_accelerator_event_t **event, bool enable_ipc);
+static int accelerator_null_record_event(int dev_id, opal_accelerator_event_t *event, opal_accelerator_stream_t *stream);
+static int accelerator_null_query_event(int dev_id, opal_accelerator_event_t *event);
+static int accelerator_null_wait_event(int dev_id, opal_accelerator_event_t *event, opal_accelerator_stream_t *stream);
+
+static int accelerator_null_memcpy_async(int dest_dev_id, int src_dev_id, void *dest, const void *src, size_t size,
+                                         opal_accelerator_stream_t *stream, opal_accelerator_transfer_type_t type);
+static int accelerator_null_memcpy(int dest_dev_id, int src_dev_id, void *dest, const void *src,
+                                   size_t size, opal_accelerator_transfer_type_t type);
+static int accelerator_null_memmove_async(int dest_dev_id, int src_dev_id, void *dest, const void *src, size_t size,
+                                          opal_accelerator_stream_t *stream, opal_accelerator_transfer_type_t type);
+static int accelerator_null_memmove(int dest_dev_id, int src_dev_id, void *dest, const void *src, size_t size,
+                                    opal_accelerator_transfer_type_t type);
+
+static int accelerator_null_mem_alloc(int dev_id, void **ptr, size_t size);
+static int accelerator_null_mem_release(int dev_id, void *ptr);
+static int accelerator_null_mem_alloc_stream(int dev_id, void **ptr, size_t size, opal_accelerator_stream_t* stream);
+static int accelerator_null_mem_release_stream(int dev_id, void *ptr, opal_accelerator_stream_t *stream);
+static int accelerator_null_get_address_range(int dev_id, const void *ptr, void **base, size_t *size);
+
+static bool accelerator_null_is_ipc_enabled(void);
+static int accelerator_null_get_ipc_handle(int dev_id, void *dev_ptr,
+                                           opal_accelerator_ipc_handle_t *handle);
+static int accelerator_null_import_ipc_handle(int dev_id, uint8_t ipc_handle[IPC_MAX_HANDLE_SIZE],
+                                              opal_accelerator_ipc_handle_t *handle);
+static int accelerator_null_open_ipc_handle(int dev_id, opal_accelerator_ipc_handle_t *handle,
+                                            void **dev_ptr);
+static int accelerator_null_compare_ipc_handles(uint8_t handle_1[IPC_MAX_HANDLE_SIZE],
+                                                uint8_t handle_2[IPC_MAX_HANDLE_SIZE]);
+static int accelerator_null_get_ipc_event_handle(opal_accelerator_event_t *event,
+                                                 opal_accelerator_ipc_event_handle_t *handle);
+static int accelerator_null_import_ipc_event_handle(uint8_t ipc_handle[IPC_MAX_HANDLE_SIZE],
+                                                    opal_accelerator_ipc_event_handle_t *handle);
+static int accelerator_null_open_ipc_event_handle(opal_accelerator_ipc_event_handle_t *handle,
+                                                  opal_accelerator_event_t *event);
+
+static int accelerator_null_host_register(int dev_id, void *ptr, size_t size);
+static int accelerator_null_host_unregister(int dev_id, void *ptr);
+
+static int accelerator_null_get_device(int *dev_id);
+static int accelerator_null_get_device_pci_attr(int dev_id, opal_accelerator_pci_attr_t *pci_attr);
+static int accelerator_null_device_can_access_peer(int *access, int dev1, int dev2);
+
+static int accelerator_null_get_buffer_id(int dev_id, const void *addr, opal_accelerator_buffer_id_t *buf_id);
+
+static int accelerator_null_sync_stream(opal_accelerator_stream_t *stream);
+
+static int accelerator_null_get_num_devices(int *num_devices);
+
+static int accelerator_null_get_mem_bw(int device, float *bw);
+static void accelerator_null_get_memkind(ompi_memkind_t *memkind);
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+
+opal_accelerator_null_component_t mca_accelerator_null_component = {{
+
+    /* First, the mca_component_t struct containing meta information
+     * about the component itself */
+
+    .base_version =
+        {
+            /* Indicate that we are a accelerator v1.1.0 component (which also
+             * implies a specific MCA version) */
+
+            OPAL_ACCELERATOR_BASE_VERSION_1_0_0,
+
+            /* Component name and version */
+
+            .mca_component_name = "null",
+            MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
+                                  OPAL_RELEASE_VERSION),
+
+            /* Component open and close functions */
+
+            .mca_open_component = accelerator_null_open,
+            .mca_close_component = accelerator_null_close,
+            .mca_register_component_params = accelerator_null_component_register,
+
+        },
+    /* Next the MCA v1.0.0 component meta data */
+    .base_data =
+        { /* The component is checkpoint ready */
+         MCA_BASE_METADATA_PARAM_CHECKPOINT},
+    .accelerator_init = accelerator_null_init,
+    .accelerator_finalize = accelerator_null_finalize,
+}};
+
+opal_accelerator_base_module_t opal_accelerator_null_module =
+{
+    accelerator_null_check_addr,
+
+    accelerator_null_create_stream,
+    accelerator_null_sync_stream,
+
+    accelerator_null_create_event,
+    accelerator_null_record_event,
+    accelerator_null_query_event,
+    accelerator_null_wait_event,
+
+    accelerator_null_memcpy_async,
+    accelerator_null_memcpy,
+    accelerator_null_memmove_async,
+    accelerator_null_memmove,
+    accelerator_null_mem_alloc,
+    accelerator_null_mem_release,
+    accelerator_null_mem_alloc_stream,
+    accelerator_null_mem_release_stream,
+    accelerator_null_get_address_range,
+
+    accelerator_null_is_ipc_enabled,
+    accelerator_null_get_ipc_handle,
+    accelerator_null_import_ipc_handle,
+    accelerator_null_open_ipc_handle,
+    accelerator_null_compare_ipc_handles,
+    accelerator_null_get_ipc_event_handle,
+    accelerator_null_import_ipc_event_handle,
+    accelerator_null_open_ipc_event_handle,
+
+    accelerator_null_host_register,
+    accelerator_null_host_unregister,
+
+    accelerator_null_get_device,
+    accelerator_null_get_device_pci_attr,
+    accelerator_null_device_can_access_peer,
+
+    accelerator_null_get_buffer_id,
+
+    accelerator_null_get_num_devices,
+    accelerator_null_get_mem_bw,
+    accelerator_null_get_memkind
+};
+
+static int accelerator_null_open(void)
+{
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_null_close(void)
+{
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_null_component_register(void)
+{
+    return OPAL_SUCCESS;
+}
+
+static opal_accelerator_base_module_t* accelerator_null_init(void)
+{
+    return &opal_accelerator_null_module;
+}
+
+static void accelerator_null_finalize(opal_accelerator_base_module_t* module)
+{
+    return;
+}
+
+/* Accelerator API's Implementation */
+static int accelerator_null_check_addr(const void *addr, int *dev_id, uint64_t *flags)
+{
+    /* Always return that the pointer belongs to the host */
+    return 0;
+}
+
+static int accelerator_null_create_stream(int dev_id, opal_accelerator_stream_t **stream)
+{
+    *stream = OBJ_NEW(opal_accelerator_stream_t);
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_null_create_event(int dev_id, opal_accelerator_event_t **event, bool enable_ipc)
+{
+    *event = OBJ_NEW(opal_accelerator_event_t);
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_null_record_event(int dev_id, opal_accelerator_event_t *event, opal_accelerator_stream_t *stream)
+{
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_null_query_event(int dev_id, opal_accelerator_event_t *event)
+{
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_null_wait_event(int dev_id, opal_accelerator_event_t *event, opal_accelerator_stream_t *stream)
+{
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_null_memcpy_async(int dest_dev_id, int src_dev_id, void *dest, const void *src, size_t size,
+                                  opal_accelerator_stream_t *stream, opal_accelerator_transfer_type_t type)
+{
+    memcpy(dest, src, size);
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_null_memcpy(int dest_dev_id, int src_dev_id, void *dest, const void *src,
+                            size_t size, opal_accelerator_transfer_type_t type)
+{
+    memcpy(dest, src, size);
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_null_memmove(int dest_dev_id, int src_dev_id, void *dest, const void *src, size_t size,
+                             opal_accelerator_transfer_type_t type)
+{
+    memmove(dest, src, size);
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_null_memmove_async(int dest_dev_id, int src_dev_id, void *dest, const void *src, size_t size,
+                                          opal_accelerator_stream_t *stream, opal_accelerator_transfer_type_t type)
+{
+    memmove(dest, src, size);
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_null_mem_alloc(int dev_id, void **ptr, size_t size)
+{
+    *ptr = malloc(size);
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_null_mem_release(int dev_id, void *ptr)
+{
+    free(ptr);
+    return OPAL_SUCCESS;
+}
+
+
+static int accelerator_null_mem_alloc_stream(int dev_id, void **ptr, size_t size,
+                                             opal_accelerator_stream_t *stream)
+{
+    (void)stream;
+    *ptr = malloc(size);
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_null_mem_release_stream(int dev_id, void *ptr,
+                                               opal_accelerator_stream_t *stream)
+{
+    (void)stream;
+    free(ptr);
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_null_get_address_range(int dev_id, const void *ptr, void **base,
+                                              size_t *size)
+{
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
+
+static bool accelerator_null_is_ipc_enabled(void)
+{
+    return false;
+}
+
+static int accelerator_null_get_ipc_handle(int dev_id, void *dev_ptr,
+                                           opal_accelerator_ipc_handle_t *handle)
+{
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
+
+static int accelerator_null_import_ipc_handle(int dev_id, uint8_t ipc_handle[IPC_MAX_HANDLE_SIZE],
+                                              opal_accelerator_ipc_handle_t *handle)
+{
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
+
+static int accelerator_null_open_ipc_handle(int dev_id, opal_accelerator_ipc_handle_t *handle,
+                                            void **dev_ptr)
+{
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
+
+static int accelerator_null_compare_ipc_handles(uint8_t handle_1[IPC_MAX_HANDLE_SIZE],
+						uint8_t handle_2[IPC_MAX_HANDLE_SIZE])
+{
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
+
+static int accelerator_null_get_ipc_event_handle(opal_accelerator_event_t *event,
+                                                 opal_accelerator_ipc_event_handle_t *handle)
+{
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
+
+static int accelerator_null_import_ipc_event_handle(uint8_t ipc_handle[IPC_MAX_HANDLE_SIZE],
+                                                    opal_accelerator_ipc_event_handle_t *handle)
+{
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
+
+static int accelerator_null_open_ipc_event_handle(opal_accelerator_ipc_event_handle_t *handle,
+                                                  opal_accelerator_event_t *event)
+{
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
+
+static int accelerator_null_host_register(int dev_id, void *ptr, size_t size)
+{
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
+
+static int accelerator_null_host_unregister(int dev_id, void *ptr)
+{
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
+
+static int accelerator_null_get_device(int *dev_id)
+{
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
+
+static int accelerator_null_get_device_pci_attr(int dev_id, opal_accelerator_pci_attr_t *pci_attr)
+{
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
+
+static int accelerator_null_device_can_access_peer( int *access, int dev1, int dev2)
+{
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
+
+static int accelerator_null_get_buffer_id(int dev_id, const void *addr, opal_accelerator_buffer_id_t *buf_id)
+{
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
+
+static int accelerator_null_sync_stream(opal_accelerator_stream_t *stream)
+{
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_null_get_num_devices(int *num_devices)
+{
+    *num_devices = 0;
+    return OPAL_SUCCESS;
+}
+
+
+static int accelerator_null_get_mem_bw(int device, float *bw)
+{
+    *bw = 1.0; // return something that is not 0
+    return OPAL_SUCCESS;
+}
+
+static void accelerator_null_get_memkind (ompi_memkind_t *memkind)
+{
+  memkind->im_name = NULL;
+  memkind->im_no_restrictors = false;
+  memkind->im_num_restrictors = 0;
+
+  return;
+}
diff --git a/opal/mca/accelerator/null/accelerator_null_component.h b/opal/mca/accelerator/null/accelerator_null_component.h
new file mode 100644
index 00000000000..0f23a41a218
--- /dev/null
+++ b/opal/mca/accelerator/null/accelerator_null_component.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2014      Intel, Inc.  All rights reserved.
+ * Copyright (c) 2017-2022 Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef MCA_ACCELERATOR_BASE_NULL_COMPONENT_H
+#define MCA_ACCELERATOR_BASE_NULL_COMPONENT_H
+
+#include "opal/mca/accelerator/accelerator.h"
+
+BEGIN_C_DECLS
+
+typedef struct {
+    opal_accelerator_base_component_t super;
+} opal_accelerator_null_component_t;
+
+OPAL_DECLSPEC extern opal_accelerator_null_component_t mca_accelerator_null_component;
+
+OPAL_DECLSPEC extern opal_accelerator_base_module_t opal_accelerator_null_module;
+
+END_C_DECLS
+
+#endif /* MCA_ACCELERATOR_BASE_NULL_COMPONENT_H */
diff --git a/opal/mca/accelerator/null/configure.m4 b/opal/mca/accelerator/null/configure.m4
new file mode 100644
index 00000000000..61456c66aaa
--- /dev/null
+++ b/opal/mca/accelerator/null/configure.m4
@@ -0,0 +1,28 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2024      NVIDIA Corporation.
+#
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+AC_DEFUN([MCA_opal_accelerator_null_PRIORITY], [0])
+
+#
+# Force this component to compile in static-only mode
+#
+AC_DEFUN([MCA_opal_accelerator_null_COMPILE_MODE], [
+    AC_MSG_CHECKING([for MCA component $2:$3 compile mode])
+    $4="static"
+    AC_MSG_RESULT([$$4])
+])
+
+# MCA_accelerator_null_CONFIG([action-if-can-compile],
+#                             [action-if-cant-compile])
+# ------------------------------------------------
+AC_DEFUN([MCA_opal_accelerator_null_CONFIG],[
+    AC_CONFIG_FILES([opal/mca/accelerator/null/Makefile])
+])
diff --git a/opal/mca/accelerator/null/owner.txt b/opal/mca/accelerator/null/owner.txt
new file mode 100644
index 00000000000..c124ea811b5
--- /dev/null
+++ b/opal/mca/accelerator/null/owner.txt
@@ -0,0 +1,7 @@
+#
+# owner/status file
+# owner: institution that is responsible for this package
+# status: e.g. active, maintenance, unmaintained
+#
+owner: Amazon
+status: active
diff --git a/opal/mca/accelerator/rocm/Makefile.am b/opal/mca/accelerator/rocm/Makefile.am
new file mode 100644
index 00000000000..cb89cae2697
--- /dev/null
+++ b/opal/mca/accelerator/rocm/Makefile.am
@@ -0,0 +1,45 @@
+#
+# Copyright (c) 2014      Intel, Inc.  All rights reserved.
+# Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+#                         All Rights reserved.
+# Copyright (c) 2022      Advanced Micro Devices, Inc.
+#                         All Rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+
+AM_CPPFLAGS = $(opal_rocm_CPPFLAGS)
+
+sources = \
+        accelerator_rocm.h \
+        accelerator_rocm_component.c \
+        accelerator_rocm_module.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_opal_accelerator_rocm_DSO
+component_noinst =
+component_install = mca_accelerator_rocm.la
+else
+component_noinst = libmca_accelerator_rocm.la
+component_install =
+endif
+
+mcacomponentdir = $(opallibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_accelerator_rocm_la_SOURCES = $(sources)
+mca_accelerator_rocm_la_LDFLAGS = -module -avoid-version $(opal_rocm_LDFLAGS)
+mca_accelerator_rocm_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
+	$(opal_rocm_LIBS)
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_accelerator_rocm_la_SOURCES =$(sources)
+libmca_accelerator_rocm_la_LDFLAGS = -module -avoid-version $(opal_rocm_LDFLAGS)
+libmca_accelerator_rocm_la_LIBADD = $(opal_rocm_LIBS)
diff --git a/opal/mca/accelerator/rocm/accelerator_rocm.h b/opal/mca/accelerator/rocm/accelerator_rocm.h
new file mode 100644
index 00000000000..4d4e1f0e65f
--- /dev/null
+++ b/opal/mca/accelerator/rocm/accelerator_rocm.h
@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2022-2023  Advanced Micro Devices, Inc. All rights reserved.
+ * Copyright (c) 2024      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef OPAL_ACCELERATOR_ROCM_H
+#define OPAL_ACCELERATOR_ROCM_H
+
+#include "opal_config.h"
+
+#include <stdio.h>
+
+/* Not interested in warnings generated in hip_runtime_api.h */
+#pragma GCC diagnostic push
+/* Clang won't quietly accept "-pedantic", but GCC versions older than ~4.8
+ * won't quietly accept "-Wpedanic".  The whole "#pragma GCC diagnostic ..."
+ * facility only was added to GCC as of version 4.6. */
+#if defined(__clang__) || (defined(__GNUC__) && __GNUC__ >= 6)
+#    pragma GCC diagnostic ignored "-Wpedantic"
+#    pragma GCC diagnostic ignored "-Wundef"
+#    pragma GCC diagnostic ignored "-Wstrict-prototypes"
+#else
+#    pragma GCC diagnostic ignored "-pedantic"
+#endif
+#include <hip/hip_runtime_api.h>
+#include <hip/hip_version.h>
+/* Restore warnings to original state */
+#pragma GCC diagnostic pop
+
+
+#include "opal/mca/accelerator/accelerator.h"
+#include "opal/mca/threads/mutex.h"
+
+typedef struct {
+    opal_accelerator_base_component_t super;
+} opal_accelerator_rocm_component_t;
+
+OPAL_DECLSPEC extern opal_accelerator_rocm_component_t mca_accelerator_rocm_component;
+extern opal_accelerator_base_module_t opal_accelerator_rocm_module;
+
+struct opal_accelerator_rocm_stream_t {
+    opal_accelerator_stream_t base;
+};
+typedef struct opal_accelerator_rocm_stream_t opal_accelerator_rocm_stream_t;
+OBJ_CLASS_DECLARATION(opal_accelerator_rocm_stream_t);
+
+struct opal_accelerator_rocm_event_t {
+    opal_accelerator_event_t base;
+};
+typedef struct opal_accelerator_rocm_event_t opal_accelerator_rocm_event_t;
+OBJ_CLASS_DECLARATION(opal_accelerator_rocm_event_t);
+
+struct opal_accelerator_rocm_ipc_handle_t {
+    opal_accelerator_ipc_handle_t base;
+};
+typedef struct opal_accelerator_rocm_ipc_handle_t opal_accelerator_rocm_ipc_handle_t;
+OBJ_CLASS_DECLARATION(opal_accelerator_rocm_ipc_handle_t);
+
+struct opal_accelerator_rocm_ipc_event_handle_t {
+    opal_accelerator_ipc_event_handle_t base;
+};
+typedef struct opal_accelerator_rocm_ipc_event_handle_t opal_accelerator_rocm_ipc_event_handle_t;
+OBJ_CLASS_DECLARATION(opal_accelerator_rocm_ipc_event_handle_t);
+
+extern hipStream_t *opal_accelerator_rocm_MemcpyStream;
+extern int opal_accelerator_rocm_memcpy_async;
+extern int opal_accelerator_rocm_verbose;
+extern size_t opal_accelerator_rocm_memcpyH2D_limit;
+extern size_t opal_accelerator_rocm_memcpyD2H_limit;
+extern int opal_accelerator_rocm_num_devices;
+extern float *opal_accelerator_rocm_mem_bw;
+
+extern int opal_accelerator_rocm_lazy_init(void);
+
+#endif
diff --git a/opal/mca/accelerator/rocm/accelerator_rocm_component.c b/opal/mca/accelerator/rocm/accelerator_rocm_component.c
new file mode 100644
index 00000000000..8e11f8ea7ee
--- /dev/null
+++ b/opal/mca/accelerator/rocm/accelerator_rocm_component.c
@@ -0,0 +1,335 @@
+/*
+ * Copyright (c) 2014      Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2017-2022 Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All Rights reserved.
+ * Copyright (c) 2024      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "opal_config.h"
+
+#include <stdio.h>
+
+#include "opal/mca/dl/base/base.h"
+#include "opal/mca/accelerator/base/base.h"
+#include "opal/runtime/opal_params.h"
+#include "opal/util/proc.h"
+#include "accelerator_rocm.h"
+
+int opal_accelerator_rocm_memcpy_async = 0;
+int opal_accelerator_rocm_verbose = 0;
+size_t opal_accelerator_rocm_memcpyD2H_limit=1024;
+size_t opal_accelerator_rocm_memcpyH2D_limit=1048576;
+
+/* Initialization lock for lazy rocm initialization */
+static opal_mutex_t accelerator_rocm_init_lock;
+static bool accelerator_rocm_init_complete = false;
+
+/* Define global variables, used in accelerator_rocm.c */
+int opal_accelerator_rocm_num_devices = 0;
+float *opal_accelerator_rocm_mem_bw = NULL;
+hipStream_t *opal_accelerator_rocm_MemcpyStream = NULL;
+
+/*
+ * Public string showing the accelerator rocm component version number
+ */
+const char *opal_accelerator_rocm_component_version_string
+    = "OPAL rocm accelerator MCA component version " OPAL_VERSION;
+
+
+#define HIP_CHECK(condition)                                                 \
+{                                                                            \
+    hipError_t error = condition;                                            \
+    if (hipSuccess != error){                                                \
+        const char* msg = hipGetErrorString(error);                \
+        opal_output(0, "HIP error: %d %s file: %s line: %d\n", error, msg, __FILE__, __LINE__); \
+        return error;                                                        \
+    }                                                                        \
+}
+
+#define HIP_CHECK_RETNULL(condition)                                         \
+{                                                                            \
+    hipError_t error = condition;                                            \
+    if (hipSuccess != error){                                                \
+        const char* msg = hipGetErrorString(error);                \
+        opal_output(0, "HIP error: %d %s file: %s line: %d\n", error, msg, __FILE__, __LINE__); \
+        return NULL;                                                         \
+    }                                                                        \
+}
+
+
+/*
+ * Local function
+ */
+static int accelerator_rocm_open(void);
+static int accelerator_rocm_close(void);
+static int accelerator_rocm_component_register(void);
+static opal_accelerator_base_module_t* accelerator_rocm_init(void);
+static void accelerator_rocm_finalize(opal_accelerator_base_module_t* module);
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+
+opal_accelerator_rocm_component_t mca_accelerator_rocm_component = {{
+
+    /* First, the mca_component_t struct containing meta information
+     * about the component itself */
+
+    .base_version =
+        {
+            /* Indicate that we are a accelerator v1.1.0 component (which also
+             * implies a specific MCA version) */
+
+            OPAL_ACCELERATOR_BASE_VERSION_1_0_0,
+
+            /* Component name and version */
+
+            .mca_component_name = "rocm",
+            MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
+                                  OPAL_RELEASE_VERSION),
+
+            /* Component open and close functions */
+
+            .mca_open_component = accelerator_rocm_open,
+            .mca_close_component = accelerator_rocm_close,
+            .mca_register_component_params = accelerator_rocm_component_register,
+
+        },
+    /* Next the MCA v1.0.0 component meta data */
+    .base_data =
+        { /* The component is checkpoint ready */
+         MCA_BASE_METADATA_PARAM_CHECKPOINT},
+    .accelerator_init = accelerator_rocm_init,
+    .accelerator_finalize = accelerator_rocm_finalize,
+}};
+
+static int accelerator_rocm_open(void)
+{
+    /* construct the component fields */
+
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_rocm_close(void)
+{
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_rocm_component_register(void)
+{
+    int var_id;
+
+    /* Set verbosity in the rocm related code. */
+    opal_accelerator_rocm_verbose = 0;
+    var_id = mca_base_component_var_register (&mca_accelerator_rocm_component.super.base_version,
+                                              "verbose", "Set level of verbosity of rocm component",
+                                              MCA_BASE_VAR_TYPE_INT, NULL, 0,
+                                              0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+                                              &opal_accelerator_rocm_verbose);
+    (void) mca_base_var_register_synonym (var_id, "ompi", "mpi", "accelerator_rocm", "verbose",
+                                          MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
+
+    /* Switching point between using memcpy and hipMemcpy* functions. */
+    opal_accelerator_rocm_memcpyD2H_limit = 1024;
+    var_id = mca_base_component_var_register (&mca_accelerator_rocm_component.super.base_version,
+                                              "memcpyD2H_limit",
+                                              "Max. msg. length to use memcpy instead of hip functions "
+                                              "for device-to-host copy operations",
+                                              MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                              OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+                                              &opal_accelerator_rocm_memcpyD2H_limit);
+    (void) mca_base_var_register_synonym (var_id, "ompi", "mpi", "accelerator_rocm", "memcpyD2H_limit",
+                                          MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
+    
+    /* Switching point between using memcpy and hipMemcpy* functions. */
+    opal_accelerator_rocm_memcpyH2D_limit = 1048576;
+    var_id = mca_base_component_var_register (&mca_accelerator_rocm_component.super.base_version,
+                                              "memcpyH2D_limit",
+                                              "Max. msg. length to use memcpy instead of hip functions "
+                                              "for host-to-device copy operations",
+                                              MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                              OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+                                              &opal_accelerator_rocm_memcpyH2D_limit);
+    (void) mca_base_var_register_synonym (var_id, "ompi", "mpi", "accelerator_rocm", "memcpyH2D_limit",
+                                          MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
+
+    /* Use this flag to test async vs sync copies */
+    opal_accelerator_rocm_memcpy_async = 0;
+    var_id = mca_base_component_var_register (&mca_accelerator_rocm_component.super.base_version,
+                                              "memcpy_async",
+                                              "Set to 1 to force using hipMemcpyAsync instead of hipMemcpy",
+                                              MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9,
+                                              MCA_BASE_VAR_SCOPE_READONLY,
+                                              &opal_accelerator_rocm_memcpy_async);
+    (void) mca_base_var_register_synonym (var_id, "ompi", "mpi", "accelerator_rocm", "memcpy_async",
+                                          MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
+    return OPAL_SUCCESS;
+}
+
+int opal_accelerator_rocm_lazy_init()
+{
+    hipError_t hip_err;
+    int err = OPAL_SUCCESS;
+
+    /* Double checked locking to avoid having to
+     * grab locks post lazy-initialization.  */
+    opal_atomic_rmb();
+    if (true == accelerator_rocm_init_complete) {
+        return OPAL_SUCCESS;
+    }
+    OPAL_THREAD_LOCK(&accelerator_rocm_init_lock);
+
+    /* If already initialized, just exit */
+    if (true == accelerator_rocm_init_complete) {
+        goto out;
+    }
+
+    hip_err = hipGetDeviceCount(&opal_accelerator_rocm_num_devices);
+    if (hipSuccess != hip_err) {
+        opal_output(0, "Failed to query device count, err=%d %s\n",
+                hip_err, hipGetErrorString(hip_err));
+        err = OPAL_ERROR;
+        goto out;
+    }
+
+    hipStream_t memcpy_stream;
+    hip_err = hipStreamCreate(&memcpy_stream);
+    if (hipSuccess != hip_err) {
+        opal_output(0, "Could not create hipStream, err=%d %s\n",
+                hip_err, hipGetErrorString(hip_err));
+        err = OPAL_ERROR;  // we got hipErrorInvalidValue, pretty bad
+        goto out;
+    }
+
+    opal_accelerator_rocm_MemcpyStream = malloc(sizeof(hipStream_t));
+    if (NULL == opal_accelerator_rocm_MemcpyStream) {
+        opal_output(0, "Could not allocate hipStream\n");
+        err = OPAL_ERR_OUT_OF_RESOURCE;
+        goto out;
+    }
+    *opal_accelerator_rocm_MemcpyStream = memcpy_stream;
+
+    opal_accelerator_rocm_mem_bw = malloc(sizeof(float)*opal_accelerator_rocm_num_devices);
+    if (NULL == opal_accelerator_rocm_mem_bw) {
+        opal_output(0, "Could not allocate memory_bw array\n");
+        err = OPAL_ERR_OUT_OF_RESOURCE;
+        goto out;
+    }
+
+    for (int i = 0; i < opal_accelerator_rocm_num_devices; ++i) {
+        int mem_clock_rate; // kHz
+        hip_err = hipDeviceGetAttribute(&mem_clock_rate,
+                                        hipDeviceAttributeMemoryClockRate,
+                                        i);
+        if (hipSuccess != hip_err) {
+            opal_output(0, "Failed to query device MemoryClockRate, err=%d %s\n",
+                        hip_err, hipGetErrorString(hip_err));
+            err = OPAL_ERROR;
+            goto out;
+        }
+
+        int bus_width; // bit
+        hip_err = hipDeviceGetAttribute(&bus_width,
+                                        hipDeviceAttributeMemoryBusWidth,
+                                        i);
+        if (hipSuccess != hip_err) {
+            opal_output(0, "Failed to query device MemoryBusWidth, err=%d %s\n",
+                        hip_err, hipGetErrorString(hip_err));
+            err = OPAL_ERROR;
+            goto out;
+        }
+
+        /* bw = clock_rate * bus width * 2bit multiplier */
+        float bw = ((float)mem_clock_rate*(float)bus_width*2.0) / 1024 / 1024 / 8;
+        opal_accelerator_rocm_mem_bw[i] = bw;
+    }
+
+#if HIP_VERSION >= 60000000
+    int dev_id;
+    hip_err = hipGetDevice(&dev_id);
+    if (hipSuccess != hip_err) {
+        opal_output(0, "error retrieving current device");
+        err = OPAL_ERROR;
+        goto out;
+    }
+
+    int has_large_bar = 0;
+    hip_err = hipDeviceGetAttribute (&has_large_bar, hipDeviceAttributeIsLargeBar,
+                                     dev_id);
+    if (hipSuccess != hip_err) {
+        opal_output(0, "error retrieving current device");
+        err = OPAL_ERROR;
+        goto out;
+    }
+
+    if (0 == has_large_bar) {
+        // Without large BAR we have to use hipMemcpy(Async) for all data transfers
+        opal_output(0, "Large BAR support is not enabled on current device. "
+                    "Enable large BAR support in BIOS (Above 4G Encoding) for "
+                    "better performance\n.");
+        opal_accelerator_rocm_memcpyH2D_limit = 0;
+        opal_accelerator_rocm_memcpyD2H_limit = 0;
+    }
+#endif
+
+    err = OPAL_SUCCESS;
+    opal_atomic_wmb();
+    accelerator_rocm_init_complete = true;
+out:
+    OPAL_THREAD_UNLOCK(&accelerator_rocm_init_lock);
+    return err;
+}
+
+static opal_accelerator_base_module_t* accelerator_rocm_init(void)
+{
+    OBJ_CONSTRUCT(&accelerator_rocm_init_lock, opal_mutex_t);
+
+    hipError_t err;
+
+    if (opal_rocm_runtime_initialized) {
+        return NULL;
+    }
+
+    int count=0;
+    err = hipGetDeviceCount(&count);
+    if (hipSuccess != err || 0 == count) {
+        opal_output(0, "No HIP capabale device found. Disabling component.\n");
+        return NULL;
+    }
+
+    opal_atomic_mb();
+    opal_rocm_runtime_initialized = true;
+
+    return &opal_accelerator_rocm_module;
+}
+
+static void accelerator_rocm_finalize(opal_accelerator_base_module_t* module)
+{
+    if (NULL != opal_accelerator_rocm_MemcpyStream) {
+        hipError_t err = hipStreamDestroy(*opal_accelerator_rocm_MemcpyStream);
+        if (hipSuccess != err) {
+            opal_output_verbose(10, 0, "hip_dl_finalize: error while destroying the hipStream\n");
+        }
+        free(opal_accelerator_rocm_MemcpyStream);
+        opal_accelerator_rocm_MemcpyStream = NULL;
+
+        free(opal_accelerator_rocm_mem_bw);
+        opal_accelerator_rocm_mem_bw = NULL;
+    }
+
+    OBJ_DESTRUCT(&accelerator_rocm_init_lock);
+    return;
+}
diff --git a/opal/mca/accelerator/rocm/accelerator_rocm_module.c b/opal/mca/accelerator/rocm/accelerator_rocm_module.c
new file mode 100644
index 00000000000..c66467cadec
--- /dev/null
+++ b/opal/mca/accelerator/rocm/accelerator_rocm_module.c
@@ -0,0 +1,962 @@
+/*
+ * Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All Rights reserved.
+ * Copyright (c) 2024      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "opal_config.h"
+
+#include "accelerator_rocm.h"
+#include "opal/mca/accelerator/base/base.h"
+#include "opal/constants.h"
+#include "opal/util/output.h"
+#include "ompi/info/info_memkind.h"
+
+/* Accelerator API's */
+static int mca_accelerator_rocm_check_addr(const void *addr, int *dev_id, uint64_t *flags);
+static int mca_accelerator_rocm_create_stream(int dev_id, opal_accelerator_stream_t **stream);
+
+static int mca_accelerator_rocm_create_event(int dev_id, opal_accelerator_event_t **event, bool enable_ipc);
+static int mca_accelerator_rocm_record_event(int dev_id, opal_accelerator_event_t *event, opal_accelerator_stream_t *stream);
+static int mca_accelerator_rocm_query_event(int dev_id, opal_accelerator_event_t *event);
+static int mca_accelerator_rocm_wait_event(int dev_id, opal_accelerator_event_t *event, opal_accelerator_stream_t *stream);
+
+static int mca_accelerator_rocm_memcpy_async(int dest_dev_id, int src_dev_id, void *dest, const void *src, size_t size,
+                                  opal_accelerator_stream_t *stream, opal_accelerator_transfer_type_t type);
+static int mca_accelerator_rocm_memcpy(int dest_dev_id, int src_dev_id, void *dest, const void *src,
+                            size_t size, opal_accelerator_transfer_type_t type);
+static int mca_accelerator_rocm_memmove_async(int dest_dev_id, int src_dev_id, void *dest,
+                                              const void *src, size_t size,
+                                              opal_accelerator_stream_t *stream,
+                                              opal_accelerator_transfer_type_t type);
+static int mca_accelerator_rocm_memmove(int dest_dev_id, int src_dev_id, void *dest, const void *src, size_t size,
+                                        opal_accelerator_transfer_type_t type);
+static int mca_accelerator_rocm_mem_alloc(int dev_id, void **ptr, size_t size);
+static int mca_accelerator_rocm_mem_release(int dev_id, void *ptr);
+static int mca_accelerator_rocm_mem_alloc_stream(int dev_id, void **ptr, size_t size,
+                                                 opal_accelerator_stream_t *stream);
+static int mca_accelerator_rocm_mem_release_stream(int dev_id, void *ptr, opal_accelerator_stream_t *stream);
+static int mca_accelerator_rocm_get_address_range(int dev_id, const void *ptr, void **base,
+                                                  size_t *size);
+
+static bool mca_accelerator_rocm_is_ipc_enabled(void);
+static int mca_accelerator_rocm_get_ipc_handle(int dev_id, void *dev_ptr,
+                                               opal_accelerator_ipc_handle_t *handle);
+static int mca_accelerator_rocm_import_ipc_handle(int dev_id, uint8_t ipc_handle[IPC_MAX_HANDLE_SIZE],
+                                                  opal_accelerator_ipc_handle_t *handle);
+static int mca_accelerator_rocm_open_ipc_handle(int dev_id, opal_accelerator_ipc_handle_t *handle,
+                                                void **dev_ptr);
+static int mca_accelerator_rocm_compare_ipc_handles(uint8_t handle_1[IPC_MAX_HANDLE_SIZE],
+                                                    uint8_t handle_2[IPC_MAX_HANDLE_SIZE]);
+static int mca_accelerator_rocm_get_ipc_event_handle(opal_accelerator_event_t *event,
+                                                     opal_accelerator_ipc_event_handle_t *handle);
+static int mca_accelerator_rocm_import_ipc_event_handle(uint8_t ipc_handle[IPC_MAX_HANDLE_SIZE],
+                                                        opal_accelerator_ipc_event_handle_t *handle);
+static int mca_accelerator_rocm_open_ipc_event_handle(opal_accelerator_ipc_event_handle_t *handle,
+                                                      opal_accelerator_event_t *event);
+
+static int mca_accelerator_rocm_host_register(int dev_id, void *ptr, size_t size);
+static int mca_accelerator_rocm_host_unregister(int dev_id, void *ptr);
+
+static int mca_accelerator_rocm_get_device(int *dev_id);
+static int mca_accelerator_rocm_get_device_pci_attr(int dev_id, opal_accelerator_pci_attr_t *pci_attr);
+static int mca_accelerator_rocm_device_can_access_peer( int *access, int dev1, int dev2);
+
+static int mca_accelerator_rocm_get_buffer_id(int dev_id, const void *addr, opal_accelerator_buffer_id_t *buf_id);
+
+static int mca_accelerator_rocm_sync_stream(opal_accelerator_stream_t *stream);
+
+static int mca_accelerator_rocm_get_num_devices(int *num_devices);
+
+static int mca_accelerator_rocm_get_mem_bw(int device, float *bw);
+static void mca_accelerator_rocm_get_memkind(ompi_memkind_t *memkind);
+
+#define GET_STREAM(_stream) (_stream == MCA_ACCELERATOR_STREAM_DEFAULT ? 0 : *((hipStream_t *)_stream->stream))
+
+opal_accelerator_base_module_t opal_accelerator_rocm_module =
+{
+    .check_addr = mca_accelerator_rocm_check_addr,
+
+    .create_stream = mca_accelerator_rocm_create_stream,
+    .sync_stream = mca_accelerator_rocm_sync_stream,
+
+    .create_event = mca_accelerator_rocm_create_event,
+    .record_event = mca_accelerator_rocm_record_event,
+    .query_event = mca_accelerator_rocm_query_event,
+    .wait_event = mca_accelerator_rocm_wait_event,
+
+    .mem_copy_async = mca_accelerator_rocm_memcpy_async,
+    .mem_copy = mca_accelerator_rocm_memcpy,
+    .mem_move_async = mca_accelerator_rocm_memmove_async,
+    .mem_move = mca_accelerator_rocm_memmove,
+    .mem_alloc = mca_accelerator_rocm_mem_alloc,
+    .mem_release = mca_accelerator_rocm_mem_release,
+    .mem_alloc_stream = mca_accelerator_rocm_mem_alloc_stream,
+    .mem_release_stream = mca_accelerator_rocm_mem_release_stream,
+    .get_address_range = mca_accelerator_rocm_get_address_range,
+
+    .is_ipc_enabled = mca_accelerator_rocm_is_ipc_enabled,
+    .get_ipc_handle = mca_accelerator_rocm_get_ipc_handle,
+    .import_ipc_handle = mca_accelerator_rocm_import_ipc_handle,
+    .open_ipc_handle = mca_accelerator_rocm_open_ipc_handle,
+    .compare_ipc_handles = mca_accelerator_rocm_compare_ipc_handles,
+    .get_ipc_event_handle = mca_accelerator_rocm_get_ipc_event_handle,
+    .import_ipc_event_handle = mca_accelerator_rocm_import_ipc_event_handle,
+    .open_ipc_event_handle = mca_accelerator_rocm_open_ipc_event_handle,
+
+    .host_register = mca_accelerator_rocm_host_register,
+    .host_unregister = mca_accelerator_rocm_host_unregister,
+
+    .get_device = mca_accelerator_rocm_get_device,
+    .get_device_pci_attr = mca_accelerator_rocm_get_device_pci_attr,
+    .device_can_access_peer = mca_accelerator_rocm_device_can_access_peer,
+
+    .get_buffer_id = mca_accelerator_rocm_get_buffer_id,
+
+    .num_devices = mca_accelerator_rocm_get_num_devices,
+    .get_mem_bw = mca_accelerator_rocm_get_mem_bw,
+    .get_memkind = mca_accelerator_rocm_get_memkind
+};
+
+
+static int mca_accelerator_rocm_check_addr (const void *addr, int *dev_id, uint64_t *flags)
+{
+    int ret = 0;
+    hipPointerAttribute_t srcAttr;
+    hipError_t err;
+
+    *dev_id = MCA_ACCELERATOR_NO_DEVICE_ID;
+
+    if (NULL == addr || NULL == flags) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    *flags = 0;
+    err = hipPointerGetAttributes(&srcAttr, addr);
+    if (hipSuccess == err) {
+#if HIP_VERSION >= 50731921
+        if (hipMemoryTypeDevice == srcAttr.type) {
+#else
+        if (hipMemoryTypeDevice == srcAttr.memoryType) {
+#endif
+            opal_accelerator_rocm_lazy_init();
+            *dev_id = srcAttr.device;
+            ret = 1;
+#if HIP_VERSION >= 50731921
+        } else if (hipMemoryTypeUnified == srcAttr.type) {
+#else
+        } else if (hipMemoryTypeUnified == srcAttr.memoryType) {
+#endif
+            *flags |= MCA_ACCELERATOR_FLAGS_UNIFIED_MEMORY;
+            opal_accelerator_rocm_lazy_init();
+            *dev_id = srcAttr.device;
+            ret = 1;
+        }
+    }
+
+    return ret;
+}
+
+static int mca_accelerator_rocm_create_stream(int dev_id, opal_accelerator_stream_t **stream)
+{
+    if (NULL == stream) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+    *stream = (opal_accelerator_stream_t*)OBJ_NEW(opal_accelerator_rocm_stream_t);
+    if (NULL == *stream) {
+        return OPAL_ERR_OUT_OF_RESOURCE;
+    }
+
+    (*stream)->stream = (hipStream_t *)malloc(sizeof(hipStream_t));
+    if (NULL == (*stream)->stream) {
+        OBJ_RELEASE(*stream);
+        return OPAL_ERR_OUT_OF_RESOURCE;
+    }
+
+    hipError_t err = hipStreamCreate((hipStream_t *)(*stream)->stream);
+    if (hipSuccess != err) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "Could not create hipStream, err=%d %s\n",
+                            err, hipGetErrorString(err));
+        free((*stream)->stream);
+        OBJ_RELEASE(*stream);
+        return OPAL_ERROR;
+    }
+
+    return OPAL_SUCCESS;
+}
+
+static void mca_accelerator_rocm_stream_destruct(opal_accelerator_rocm_stream_t *stream)
+{
+    if (MCA_ACCELERATOR_STREAM_DEFAULT != (opal_accelerator_stream_t *)stream &&
+	NULL != stream->base.stream) {
+        hipError_t err = hipStreamDestroy(*(hipStream_t *)stream->base.stream);
+        if (hipSuccess != err) {
+            opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                                "error while destroying the hipStream\n");
+        }
+        free(stream->base.stream);
+    }
+}
+
+OBJ_CLASS_INSTANCE(
+    opal_accelerator_rocm_stream_t,
+    opal_accelerator_stream_t,
+    NULL,
+    mca_accelerator_rocm_stream_destruct);
+
+ static int mca_accelerator_rocm_create_event(int dev_id, opal_accelerator_event_t **event, bool enable_ipc)
+{
+    if (NULL == event) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    *event = (opal_accelerator_event_t*)OBJ_NEW(opal_accelerator_rocm_event_t);
+    if (NULL == *event) {
+        return OPAL_ERR_OUT_OF_RESOURCE;
+    }
+
+    (*event)->event = malloc(sizeof(hipEvent_t));
+    if (NULL == (*event)->event) {
+        OBJ_RELEASE(*event);
+        return OPAL_ERR_OUT_OF_RESOURCE;
+    }
+    hipError_t err = hipEventCreateWithFlags((hipEvent_t*)(*event)->event,
+                                             enable_ipc ? hipEventDisableTiming|hipEventInterprocess :
+                                             hipEventDisableTiming);
+    if (hipSuccess != err) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "error creating event\n");
+        free((*event)->event);
+        OBJ_RELEASE(*event);
+        return OPAL_ERROR;
+    }
+
+    return OPAL_SUCCESS;
+}
+
+static void mca_accelerator_rocm_event_destruct(opal_accelerator_rocm_event_t *event)
+{
+    if (NULL != event->base.event) {
+        hipError_t err = hipEventDestroy(*(hipEvent_t*)event->base.event);
+        if (hipSuccess != err) {
+            opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                                "error destroying event\n");
+        }
+        free(event->base.event);
+    }
+}
+
+OBJ_CLASS_INSTANCE(
+    opal_accelerator_rocm_event_t,
+    opal_accelerator_event_t,
+    NULL,
+    mca_accelerator_rocm_event_destruct);
+
+static int mca_accelerator_rocm_record_event(int dev_id, opal_accelerator_event_t *event,
+                                             opal_accelerator_stream_t *stream)
+{
+    if (NULL == event || NULL == event->event){
+        return OPAL_ERR_BAD_PARAM;
+    }
+    if (MCA_ACCELERATOR_STREAM_DEFAULT != stream  &&
+	(NULL == stream || NULL == stream->stream)){
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    hipError_t err = hipEventRecord(*((hipEvent_t *)event->event), GET_STREAM(stream));
+    if (hipSuccess != err) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "error recording event\n");
+        return OPAL_ERROR;
+    }
+
+    return OPAL_SUCCESS;
+}
+
+static int mca_accelerator_rocm_query_event(int dev_id, opal_accelerator_event_t *event)
+{
+    if (NULL == event || NULL == event->event) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    hipError_t err = hipEventQuery(*((hipEvent_t *)event->event));
+    switch (err) {
+        case hipSuccess:
+            return OPAL_SUCCESS;
+            break;
+        case hipErrorNotReady:
+            return OPAL_ERR_RESOURCE_BUSY;
+            break;
+        default:
+            opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                                "error while querying event\n");
+            return OPAL_ERROR;
+    }
+
+    return OPAL_SUCCESS;
+}
+
+static int mca_accelerator_rocm_wait_event(int dev_id, opal_accelerator_event_t *event, opal_accelerator_stream_t *stream)
+{
+    if (NULL == event || NULL == event->event) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    if (MCA_ACCELERATOR_STREAM_DEFAULT != stream &&
+	(NULL == stream || NULL == stream->stream)){
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    hipError_t err = hipStreamWaitEvent(GET_STREAM(stream), *((hipEvent_t *)event->event), 0);
+    if (hipSuccess != err) {
+        return OPAL_ERROR;
+    }
+
+    return OPAL_SUCCESS;
+}
+
+static int mca_accelerator_rocm_memcpy_async(int dest_dev_id, int src_dev_id, void *dest, const void *src,
+                                             size_t size, opal_accelerator_stream_t *stream,
+                                             opal_accelerator_transfer_type_t type)
+{
+    if ((MCA_ACCELERATOR_STREAM_DEFAULT != stream &&
+        (NULL == stream || NULL == stream->stream)) ||
+        NULL == src || NULL == dest || size < 0) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+    if (0 == size) {
+        return OPAL_SUCCESS;
+    }
+
+    hipError_t err = hipMemcpyAsync(dest, src, size, hipMemcpyDefault,
+				    GET_STREAM(stream));
+    if (hipSuccess != err ) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "error while starting asynchronous copy\n");
+        return OPAL_ERROR;
+    }
+
+    return OPAL_SUCCESS;
+}
+
+static int mca_accelerator_rocm_memcpy(int dest_dev_id, int src_dev_id, void *dest,
+                                       const void *src, size_t size,
+                                       opal_accelerator_transfer_type_t type)
+{
+    hipError_t err;
+
+    if (NULL == src || NULL == dest || size < 0) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+    if (0 == size) {
+        return OPAL_SUCCESS;
+    }
+
+    if ((type == MCA_ACCELERATOR_TRANSFER_DTOH ||
+	 type == MCA_ACCELERATOR_TRANSFER_UNSPEC) &&
+	size <= opal_accelerator_rocm_memcpyD2H_limit) {
+        memcpy(dest, src, size);
+        return OPAL_SUCCESS;
+    }
+
+    if (type == MCA_ACCELERATOR_TRANSFER_HTOD && size <= opal_accelerator_rocm_memcpyH2D_limit) {
+        memcpy(dest, src, size);
+        return OPAL_SUCCESS;
+    }
+
+    if (opal_accelerator_rocm_memcpy_async) {
+        err = hipMemcpyAsync(dest, src, size, hipMemcpyDefault,
+                                       *opal_accelerator_rocm_MemcpyStream);
+        if (hipSuccess != err ) {
+            opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                                "error starting async copy\n");
+            return OPAL_ERROR;
+        }
+
+        err = hipStreamSynchronize(*opal_accelerator_rocm_MemcpyStream);
+        if (hipSuccess != err ) {
+            opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                                "error synchronizing stream after async copy\n");
+            return OPAL_ERROR;
+        }
+    } else {
+        err = hipMemcpy(dest, src, size, hipMemcpyDefault);
+        if (hipSuccess != err ) {
+            opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                                "error during synchronous copy\n");
+            return OPAL_ERROR;
+        }
+        err = hipStreamSynchronize(0);
+        if (hipSuccess != err ) {
+            opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                                "error synchronizing default stream after hipMemcpy\n");
+            return OPAL_ERROR;
+        }
+    }
+
+    return OPAL_SUCCESS;
+}
+
+static int mca_accelerator_rocm_memmove_async(int dest_dev_id, int src_dev_id, void *dest, const void *src,
+                                              size_t size, opal_accelerator_stream_t *stream,
+                                              opal_accelerator_transfer_type_t type)
+{
+    hipDeviceptr_t tmp;
+    hipError_t result;
+    int ret;
+    void *ptr;
+
+    int delayed_init = opal_accelerator_rocm_lazy_init();
+    if (OPAL_UNLIKELY(0 != delayed_init)) {
+        return delayed_init;
+    }
+
+    if (NULL == dest || NULL == src || size <= 0) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    ret = mca_accelerator_rocm_mem_alloc_stream(src_dev_id, &ptr, size, stream);
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != ret)) {
+        return OPAL_ERROR;
+    }
+    tmp = (hipDeviceptr_t)ptr;
+    result = hipMemcpyAsync(tmp, (hipDeviceptr_t) src, size, hipMemcpyDefault, *(hipStream_t*)stream->stream);
+    if (OPAL_UNLIKELY(hipSuccess != result)) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "error during synchronous copy\n");
+        return OPAL_ERROR;
+    }
+    result = hipMemcpyAsync((hipDeviceptr_t) dest, tmp, size, hipMemcpyDefault, *(hipStream_t*)stream->stream);
+    if (OPAL_UNLIKELY(hipSuccess != result)) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "error during synchronous copy\n");
+        return OPAL_ERROR;
+    }
+    return mca_accelerator_rocm_mem_release_stream(src_dev_id, ptr, stream);
+}
+
+static int mca_accelerator_rocm_memmove(int dest_dev_id, int src_dev_id, void *dest,
+                                        const void *src, size_t size,
+                                        opal_accelerator_transfer_type_t type)
+{
+    char *tmp = NULL;
+    hipError_t err;
+
+    if (NULL == src || NULL == dest || size <= 0) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    err = hipMalloc((void **)&tmp, size);
+    if (hipSuccess != err ) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "error allocating memory for memmove\n");
+        return OPAL_ERROR;
+    }
+
+    if (opal_accelerator_rocm_memcpy_async) {
+        err = hipMemcpyAsync(tmp, src, size, hipMemcpyDefault,
+                                       *opal_accelerator_rocm_MemcpyStream);
+        if (hipSuccess != err ) {
+            opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                                "error in async memcpy for memmove\n");
+            return OPAL_ERROR;
+        }
+
+        err = hipMemcpyAsync(dest, tmp, size, hipMemcpyDefault,
+                                       *opal_accelerator_rocm_MemcpyStream);
+        if (hipSuccess != err ) {
+            opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                                "error in async memcpy for memmove\n");
+            return OPAL_ERROR;
+        }
+
+        err = hipStreamSynchronize(*opal_accelerator_rocm_MemcpyStream);
+        if (hipSuccess != err ) {
+            opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                                "error synchronizing stream for memmove\n");
+            return OPAL_ERROR;
+        }
+    } else {
+        err = hipMemcpy(tmp, src, size, hipMemcpyDefault);
+        if (hipSuccess != err ) {
+            opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                                "error in memcpy for memmove\n");
+            return OPAL_ERROR;
+        }
+        err = hipMemcpy(dest, tmp, size, hipMemcpyDefault);
+        if (hipSuccess != err ) {
+            opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                                "error in memcpy for memmove\n");
+            return OPAL_ERROR;
+        }
+    }
+
+    err = hipFree(tmp);
+    if (hipSuccess != err ) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "error in hipFree for memmove\n");
+        return OPAL_ERROR;
+    }
+
+    return OPAL_SUCCESS;
+}
+
+static int mca_accelerator_rocm_mem_alloc(int dev_id, void **ptr, size_t size)
+{
+    if (NULL == ptr || size <= 0) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    hipError_t err = hipMalloc(ptr, size);
+    if (hipSuccess != err ) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "error allocating memory\n");
+        return OPAL_ERROR;
+    }
+
+    return OPAL_SUCCESS;
+}
+
+static int mca_accelerator_rocm_mem_release(int dev_id, void *ptr)
+{
+    if (NULL == ptr) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    hipError_t err = hipFree(ptr);
+    if (hipSuccess != err ) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "error freeing memory\n");
+        return OPAL_ERROR;
+    }
+
+    return OPAL_SUCCESS;
+}
+
+static int mca_accelerator_rocm_get_address_range(int dev_id, const void *ptr, void **base,
+                                                  size_t *size)
+{
+    hipError_t err;
+    hipDeviceptr_t tBase;
+    size_t tSize;
+
+    if (NULL == ptr || NULL == base || NULL == size) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    err = hipMemGetAddressRange(&tBase, &tSize, (hipDeviceptr_t) ptr);
+    if (hipSuccess != err) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "couldn't get address range for pointer %p/%lu", ptr, *size);
+        return OPAL_ERROR;
+    }
+
+    *size = tSize;
+    *base = (char *) tBase;
+
+    return OPAL_SUCCESS;
+}
+
+static bool mca_accelerator_rocm_is_ipc_enabled(void)
+{
+    return true;
+}
+
+static void mca_accelerator_rocm_ipc_handle_destruct(opal_accelerator_rocm_ipc_handle_t *handle)
+{
+    if (NULL != handle && NULL != handle->base.dev_ptr) {
+        hipIpcCloseMemHandle((hipDeviceptr_t) handle->base.dev_ptr);
+	handle->base.dev_ptr = NULL;
+    }
+}
+
+OBJ_CLASS_INSTANCE(
+    opal_accelerator_rocm_ipc_handle_t,
+    opal_accelerator_ipc_handle_t,
+    NULL,
+    mca_accelerator_rocm_ipc_handle_destruct);
+
+static int mca_accelerator_rocm_get_ipc_handle(int dev_id, void *dev_ptr,
+                                               opal_accelerator_ipc_handle_t *handle)
+{
+    if (NULL == dev_ptr || NULL == handle) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    hipIpcMemHandle_t rocm_ipc_handle;
+    opal_accelerator_rocm_ipc_handle_t *rocm_handle = (opal_accelerator_rocm_ipc_handle_t *) handle;
+
+    OBJ_CONSTRUCT(rocm_handle, opal_accelerator_rocm_ipc_handle_t);
+    rocm_handle->base.dev_ptr = NULL;
+
+    memset(rocm_ipc_handle.reserved, 0, HIP_IPC_HANDLE_SIZE);
+    hipError_t err = hipIpcGetMemHandle(&rocm_ipc_handle,
+                                        (hipDeviceptr_t)dev_ptr);
+    if (hipSuccess != err) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "Error in hipIpcGetMemHandle dev_ptr %p", dev_ptr);
+        OBJ_DESTRUCT(rocm_handle);
+        return OPAL_ERROR;
+    }
+    memcpy(rocm_handle->base.handle, &rocm_ipc_handle, IPC_MAX_HANDLE_SIZE);
+
+    return OPAL_SUCCESS;
+}
+
+static int mca_accelerator_rocm_import_ipc_handle(int dev_id, uint8_t ipc_handle[IPC_MAX_HANDLE_SIZE],
+                                                  opal_accelerator_ipc_handle_t *handle)
+{
+    opal_accelerator_rocm_ipc_handle_t *rocm_handle = (opal_accelerator_rocm_ipc_handle_t *) handle;
+    OBJ_CONSTRUCT(rocm_handle, opal_accelerator_rocm_ipc_handle_t);
+    memcpy(rocm_handle->base.handle, ipc_handle, IPC_MAX_HANDLE_SIZE);
+
+    return OPAL_SUCCESS;
+}
+
+static int mca_accelerator_rocm_open_ipc_handle(int dev_id, opal_accelerator_ipc_handle_t *handle,
+                                                void **dev_ptr)
+{
+    if (NULL == dev_ptr || NULL == handle) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    hipError_t err = hipIpcOpenMemHandle((hipDeviceptr_t *) &handle->dev_ptr,
+                                         *(hipIpcMemHandle_t*)handle->handle,
+                                         hipIpcMemLazyEnablePeerAccess);
+    if (hipErrorMapFailed == err) {
+        return OPAL_ERR_WOULD_BLOCK;
+    }
+    else if (hipSuccess != err) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "error in hipIpcOpenMemHandle");
+        return OPAL_ERROR;
+    }
+    *dev_ptr = handle->dev_ptr;
+
+    return OPAL_SUCCESS;
+}
+
+static int mca_accelerator_rocm_compare_ipc_handles(uint8_t handle_1[IPC_MAX_HANDLE_SIZE],
+                                                    uint8_t handle_2[IPC_MAX_HANDLE_SIZE])
+{
+    /*
+     * The HIP IPC handles consists of multiple elements.
+     * We will only use the ROCr IPC handle (32 bytes, starting at pos 0)
+     * and the process ID for comparison.
+     * We definitily need to exclude the offset component in the comparison.
+     */
+    static const int rocr_ipc_handle_size = 32;
+    static const int pos = rocr_ipc_handle_size + 2*sizeof(size_t);
+    int *pid_1 = (int *)&handle_1[pos];
+    int *pid_2 = (int *)&handle_2[pos];
+
+    if (*pid_1 != *pid_2) {
+        return 1;
+    }
+
+    return memcmp(handle_1, handle_2, rocr_ipc_handle_size);
+}
+
+static void mca_accelerator_rocm_ipc_event_handle_destruct(opal_accelerator_rocm_ipc_handle_t *handle)
+{
+    // Just a place holder, there is no hipIpcCloseEventHandle.
+}
+
+OBJ_CLASS_INSTANCE(
+    opal_accelerator_rocm_ipc_event_handle_t,
+    opal_accelerator_ipc_event_handle_t,
+    NULL,
+    mca_accelerator_rocm_ipc_event_handle_destruct);
+
+
+static int mca_accelerator_rocm_get_ipc_event_handle(opal_accelerator_event_t *event,
+                                                     opal_accelerator_ipc_event_handle_t *handle)
+{
+    if (NULL == event || NULL == handle) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    hipIpcEventHandle_t rocm_ipc_handle;
+    opal_accelerator_rocm_ipc_event_handle_t *rocm_handle = (opal_accelerator_rocm_ipc_event_handle_t *) handle;
+    OBJ_CONSTRUCT(rocm_handle, opal_accelerator_rocm_ipc_event_handle_t);
+
+    memset(rocm_ipc_handle.reserved, 0, HIP_IPC_HANDLE_SIZE);
+    hipError_t err = hipIpcGetEventHandle(&rocm_ipc_handle,
+                                          *((hipEvent_t *)event->event));
+    if (hipSuccess != err) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "error in hipIpcGetEventHandle");
+        OBJ_DESTRUCT(rocm_handle);
+        return OPAL_ERROR;
+    }
+    memcpy(rocm_handle->base.handle, &rocm_ipc_handle, IPC_MAX_HANDLE_SIZE);
+
+    return OPAL_SUCCESS;
+}
+
+static int mca_accelerator_rocm_import_ipc_event_handle(uint8_t ipc_handle[IPC_MAX_HANDLE_SIZE],
+                                                        opal_accelerator_ipc_event_handle_t *handle)
+{
+    opal_accelerator_rocm_ipc_handle_t *rocm_handle = (opal_accelerator_rocm_ipc_handle_t *) handle;
+
+    OBJ_CONSTRUCT(rocm_handle, opal_accelerator_rocm_ipc_handle_t);
+    memcpy(rocm_handle->base.handle, ipc_handle, IPC_MAX_HANDLE_SIZE);
+
+    return OPAL_SUCCESS;
+}
+
+static int mca_accelerator_rocm_open_ipc_event_handle(opal_accelerator_ipc_event_handle_t *handle,
+                                                      opal_accelerator_event_t *event)
+{
+    if (NULL == event || NULL == handle) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    opal_accelerator_rocm_ipc_event_handle_t *rocm_handle = (opal_accelerator_rocm_ipc_event_handle_t *) handle;
+    opal_accelerator_rocm_event_t *rocm_event = (opal_accelerator_rocm_event_t *) event;
+    OBJ_CONSTRUCT(rocm_event, opal_accelerator_rocm_event_t);
+    rocm_event->base.event = malloc(sizeof(hipEvent_t));
+    if (NULL == rocm_event->base.event) {
+        return OPAL_ERR_OUT_OF_RESOURCE;
+    }
+
+    hipError_t err = hipIpcOpenEventHandle( (hipEvent_t *)rocm_event->base.event,
+                                            *((hipIpcEventHandle_t*)rocm_handle->base.handle));
+    if (hipSuccess != err) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "error in hipIpcOpenEventHandle");
+        return OPAL_ERROR;
+    }
+
+    return OPAL_SUCCESS;
+}
+
+static int mca_accelerator_rocm_host_register(int dev_id, void *ptr, size_t size)
+{
+    if (NULL == ptr && size > 0) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    hipError_t err = hipHostRegister(ptr, size, 0);
+    if (hipSuccess != err) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "error registering address %p", ptr);
+        return OPAL_ERROR;
+    }
+
+    return OPAL_SUCCESS;
+}
+
+static int mca_accelerator_rocm_host_unregister(int dev_id, void *ptr)
+{
+    if (NULL == ptr) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    hipError_t err = hipHostUnregister(ptr);
+    if (hipSuccess != err) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "error unregistering address %p", ptr);
+        return OPAL_ERROR;
+    }
+
+    return OPAL_SUCCESS;
+}
+
+static int mca_accelerator_rocm_get_device(int *dev_id)
+{
+    if (NULL == dev_id) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    hipError_t err = hipGetDevice(dev_id);
+    if (hipSuccess != err) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "error retrieviung current device");
+        return OPAL_ERROR;
+    }
+
+    return OPAL_SUCCESS;
+}
+
+static int mca_accelerator_rocm_get_device_pci_attr(int dev_id, opal_accelerator_pci_attr_t *pci_attr)
+{
+    hipError_t err;
+    int ret;
+    static const int PCI_BUS_ID_LENGTH = 13;
+    char pci_bus_id[PCI_BUS_ID_LENGTH];
+    char domain_id[5] = {0}, bus_id[3] = {0}, device_id[3] = {0}, function_id[2] = {0};
+
+    if (NULL == pci_attr) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    err = hipDeviceGetPCIBusId(pci_bus_id, PCI_BUS_ID_LENGTH, dev_id);
+    if(hipSuccess != err) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "error retrieving device PCI attributes");
+        return OPAL_ERROR;
+    }
+
+    ret = sscanf(pci_bus_id, "%4s:%2s:%2s.%1s", domain_id, bus_id, device_id, function_id);
+    if (4 > ret) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "error parsing device PCI attributes");
+        return OPAL_ERROR;
+    }
+
+    errno = 0;
+    pci_attr->domain_id = strtol(domain_id, NULL, 16);
+    pci_attr->bus_id = strtol(bus_id, NULL, 16);
+    pci_attr->device_id = strtol(device_id, NULL, 16);
+    pci_attr->function_id = strtol(function_id, NULL, 16);
+    if (0 != errno) {
+        return OPAL_ERROR;
+    }
+
+    return OPAL_SUCCESS;
+}
+
+static int mca_accelerator_rocm_device_can_access_peer(int *access, int dev1, int dev2)
+{
+    if (NULL == access || dev1 < 0 || dev2 < 0){
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    if (dev1 == dev2) {
+        *access = 1;
+        return OPAL_SUCCESS;
+    }
+
+    hipError_t err = hipDeviceCanAccessPeer(access, dev1, dev2);
+    if (hipSuccess != err) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "error in hipDeviceCanAccessPerr dev1 %d dev2 %d", dev1, dev2);
+        return OPAL_ERROR;
+    }
+
+    return OPAL_SUCCESS;
+}
+
+static int mca_accelerator_rocm_get_buffer_id(int dev_id, const void *addr, opal_accelerator_buffer_id_t *buf_id)
+{
+    *buf_id = 0;
+
+#if HIP_VERSION >= 50120531
+    hipError_t result = hipPointerGetAttribute((unsigned long long *)buf_id, HIP_POINTER_ATTRIBUTE_BUFFER_ID,
+                                               (hipDeviceptr_t)addr);
+    if (hipSuccess != result) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "error in hipPointerGetAttribute, could not retrieve buffer_id");
+        return OPAL_ERROR;
+    }
+#endif
+
+#if HIP_VERSION >= 50530201
+    int enable = 1;
+    hipError_t err = hipPointerSetAttribute(&enable, HIP_POINTER_ATTRIBUTE_SYNC_MEMOPS,
+                                            (hipDeviceptr_t)addr);
+    if (hipSuccess != err) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "error in hipPointerSetAttribute, could not set SYNC_MEMOPS");
+        return OPAL_ERROR;
+    }
+#endif
+    return OPAL_SUCCESS;
+}
+
+
+static int mca_accelerator_rocm_mem_alloc_stream(
+    int dev_id,
+    void **addr,
+    size_t size,
+    opal_accelerator_stream_t *stream)
+{
+    hipError_t result;
+
+    int delayed_init = opal_accelerator_rocm_lazy_init();
+    if (OPAL_UNLIKELY(0 != delayed_init)) {
+        return delayed_init;
+    }
+
+    if (NULL == stream || NULL == addr || 0 == size) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    result = hipMallocAsync(addr, size, *(hipStream_t*)stream->stream);
+    if (OPAL_UNLIKELY(hipSuccess != result)) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "error allocating memory\n");
+        return OPAL_ERROR;
+    }
+    return OPAL_SUCCESS;
+}
+
+static int mca_accelerator_rocm_mem_release_stream(
+    int dev_id,
+    void *addr,
+    opal_accelerator_stream_t *stream)
+{
+    hipError_t result;
+
+    if (NULL == stream || NULL == addr) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    result = hipFreeAsync(addr, *(hipStream_t*)stream->stream);
+    if (OPAL_UNLIKELY(hipSuccess != result)) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "error freeing memory\n");
+        return OPAL_ERROR;
+    }
+    return OPAL_SUCCESS;
+}
+
+static int mca_accelerator_rocm_sync_stream(opal_accelerator_stream_t *stream)
+{
+    hipError_t result;
+    result = hipStreamSynchronize(*(hipStream_t*)stream->stream);
+    if (OPAL_UNLIKELY(hipSuccess != result)) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "error synchronizing stream\n");
+        return OPAL_ERROR;
+    }
+    return OPAL_SUCCESS;
+}
+
+
+static int mca_accelerator_rocm_get_num_devices(int *num_devices)
+{
+    *num_devices = opal_accelerator_rocm_num_devices;
+    return OPAL_SUCCESS;
+}
+
+static int mca_accelerator_rocm_get_mem_bw(int device, float *bw)
+{
+    int delayed_init = opal_accelerator_rocm_lazy_init();
+    if (OPAL_UNLIKELY(0 != delayed_init)) {
+        return delayed_init;
+    }
+    assert(opal_accelerator_rocm_mem_bw != NULL);
+
+    *bw = opal_accelerator_rocm_mem_bw[device];
+    return OPAL_SUCCESS;
+}
+
+static void mca_accelerator_rocm_get_memkind (ompi_memkind_t *memkind)
+{
+  memkind->im_name = strdup("rocm");
+  memkind->im_no_restrictors = false;
+  memkind->im_num_restrictors = 3;
+  memkind->im_restrictors[0] = strdup("host");
+  memkind->im_restrictors[1] = strdup("device");
+  memkind->im_restrictors[2] = strdup("managed");
+
+  return;
+}
diff --git a/opal/mca/accelerator/rocm/configure.m4 b/opal/mca/accelerator/rocm/configure.m4
new file mode 100644
index 00000000000..780e0d8c793
--- /dev/null
+++ b/opal/mca/accelerator/rocm/configure.m4
@@ -0,0 +1,28 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2022      Advanced Micro Devices, Inc. All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# MCA_opal_accelerator_rocm_CONFIG([action-if-can-compile],
+#                                  [action-if-cant-compile])
+# ------------------------------------------------
+AC_DEFUN([MCA_opal_accelerator_rocm_CONFIG],[
+    AC_CONFIG_FILES([opal/mca/accelerator/rocm/Makefile])
+
+    OPAL_CHECK_ROCM([opal_rocm],
+	        [opal_rocm_happy="yes"],
+	        [opal_rocm_happy="no"])
+    OPAL_SUMMARY_ADD([Accelerators], [ROCm support], [], [$opal_rocm_happy])
+
+    AS_IF([test "$opal_rocm_happy" = "yes"],
+	  [$1],
+	  [$2])
+    AC_SUBST([opal_rocm_LDFLAGS])
+    AC_SUBST([opal_rocm_LIBS])
+    AC_SUBST([opal_rocm_CPPFLAGS])
+])dnl
diff --git a/opal/mca/accelerator/rocm/owner.txt b/opal/mca/accelerator/rocm/owner.txt
new file mode 100644
index 00000000000..d44b1deaf99
--- /dev/null
+++ b/opal/mca/accelerator/rocm/owner.txt
@@ -0,0 +1,7 @@
+#
+# owner/status file
+# owner: institution that is responsible for this package
+# status: e.g. active, maintenance, unmaintained
+#
+owner: AMD
+status: active
diff --git a/opal/mca/accelerator/ze/Makefile.am b/opal/mca/accelerator/ze/Makefile.am
new file mode 100644
index 00000000000..0d3c40c4b12
--- /dev/null
+++ b/opal/mca/accelerator/ze/Makefile.am
@@ -0,0 +1,45 @@
+#
+# Copyright (c) 2014      Intel, Inc.  All rights reserved.
+# Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+#                         All Rights reserved.
+# Copyright (c) 2022      Advanced Micro Devices, Inc.
+#                         All Rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+
+AM_CPPFLAGS = $(common_ze_CPPFLAGS)
+
+sources = \
+        accelerator_ze.h \
+        accelerator_ze_component.c \
+        accelerator_ze_module.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_opal_accelerator_ze_DSO
+component_noinst =
+component_install = mca_accelerator_ze.la
+else
+component_noinst = libmca_accelerator_ze.la
+component_install =
+endif
+
+mcacomponentdir = $(opallibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_accelerator_ze_la_SOURCES = $(sources)
+mca_accelerator_ze_la_LDFLAGS = -module -avoid-version $(opal_ze_LDFLAGS)
+mca_accelerator_ze_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
+	$(opal_ze_LIBS)
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_accelerator_ze_la_SOURCES =$(sources)
+libmca_accelerator_ze_la_LDFLAGS = -module -avoid-version $(opal_ze_LDFLAGS)
+libmca_accelerator_ze_la_LIBADD = $(opal_ze_LIBS)
diff --git a/opal/mca/accelerator/ze/accelerator_ze.h b/opal/mca/accelerator/ze/accelerator_ze.h
new file mode 100644
index 00000000000..9bf7a0b4b07
--- /dev/null
+++ b/opal/mca/accelerator/ze/accelerator_ze.h
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2022-2023  Advanced Micro Devices, Inc. All rights reserved.
+ * Copyright (c) 2023       Triad National Security, LLC. All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef OPAL_ACCELERATOR_ZE_H
+#define OPAL_ACCELERATOR_ZE_H
+
+#include "opal_config.h"
+
+#include "level_zero/ze_api.h"
+
+#include "opal/mca/accelerator/accelerator.h"
+#include "opal/mca/threads/mutex.h"
+
+typedef struct {
+    opal_accelerator_base_component_t super;
+} opal_accelerator_ze_component_t;
+
+OPAL_DECLSPEC extern opal_accelerator_ze_component_t mca_accelerator_ze_component;
+OPAL_DECLSPEC extern opal_accelerator_base_module_t opal_accelerator_ze_module;
+
+struct opal_accelerator_ze_stream_t {
+    opal_accelerator_stream_t base;
+    ze_command_queue_handle_t hCommandQueue;
+    ze_command_list_handle_t hCommandList;
+    int dev_id;
+};
+typedef struct opal_accelerator_ze_stream_t opal_accelerator_ze_stream_t;
+OBJ_CLASS_DECLARATION(opal_accelerator_ze_stream_t);
+
+struct opal_accelerator_ze_event_t {
+    opal_accelerator_event_t base;
+};
+typedef struct opal_accelerator_ze_event_t opal_accelerator_ze_event_t;
+OBJ_CLASS_DECLARATION(opal_accelerator_ze_event_t);
+
+OPAL_DECLSPEC extern uint32_t opal_accelerator_ze_device_count;
+OPAL_DECLSPEC extern ze_device_handle_t *opal_accelerator_ze_devices_handle;
+OPAL_DECLSPEC extern ze_driver_handle_t opal_accelerator_ze_driver_handle;
+OPAL_DECLSPEC extern ze_context_handle_t opal_accelerator_ze_context;
+OPAL_DECLSPEC extern ze_event_pool_handle_t opal_accelerator_ze_event_pool;
+OPAL_DECLSPEC extern opal_accelerator_stream_t **opal_accelerator_ze_MemcpyStream;
+
+OPAL_DECLSPEC extern int opal_accelerator_ze_memcpy_async;
+OPAL_DECLSPEC extern int opal_accelerator_ze_verbose;
+
+OPAL_DECLSPEC extern int opal_accelerator_ze_lazy_init(void);
+
+#endif
diff --git a/opal/mca/accelerator/ze/accelerator_ze_component.c b/opal/mca/accelerator/ze/accelerator_ze_component.c
new file mode 100644
index 00000000000..2a7aa985afe
--- /dev/null
+++ b/opal/mca/accelerator/ze/accelerator_ze_component.c
@@ -0,0 +1,410 @@
+/*
+ * Copyright (c) 2014      Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2017-2022 Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * Copyright (c) 2022      Advanced Micro Devices, Inc. All Rights reserved.
+ * Copyright (c) 2023      Triad National Security, LLC. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "opal_config.h"
+
+#include <stdio.h>
+#include <dlfcn.h>
+
+#include "opal/mca/dl/base/base.h"
+#include "opal/runtime/opal_params.h"
+#include "accelerator_ze.h"
+#include "opal/mca/accelerator/base/base.h"
+#include "opal_config.h"
+#include "opal/util/argv.h"
+#include "opal/util/printf.h"
+#include "opal/util/output.h"                
+
+
+int opal_accelerator_ze_memcpy_async = 1;
+int opal_accelerator_ze_verbose = 0;
+uint32_t opal_accelerator_ze_device_count = 0;
+ze_device_handle_t *opal_accelerator_ze_devices_handle = NULL;
+ze_driver_handle_t opal_accelerator_ze_driver_handle = NULL;
+ze_context_handle_t opal_accelerator_ze_context = NULL;
+ze_event_pool_handle_t opal_accelerator_ze_event_pool = NULL;
+opal_accelerator_stream_t **opal_accelerator_ze_MemcpyStream = NULL;
+
+/* Initialization lock for lazy ze initialization */
+static opal_mutex_t accelerator_ze_init_lock;
+static bool accelerator_ze_init_complete = false;
+
+/*
+ * Public string showing the accelerator ze component version number
+ */
+const char *opal_accelerator_ze_component_version_string
+    = "OPAL ze accelerator MCA component version " OPAL_VERSION;
+
+/*
+ * Local function
+ */
+static int accelerator_ze_open(void);
+static int accelerator_ze_close(void);
+static int accelerator_ze_component_register(void);
+static opal_accelerator_base_module_t* accelerator_ze_init(void);
+static void accelerator_ze_finalize(opal_accelerator_base_module_t* module);
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+
+opal_accelerator_ze_component_t mca_accelerator_ze_component = {{
+
+    /* First, the mca_component_t struct containing meta information
+     * about the component itself */
+
+    .base_version =
+        {
+            /* Indicate that we are a accelerator v1.1.0 component (which also
+             * implies a specific MCA version) */
+
+            OPAL_ACCELERATOR_BASE_VERSION_1_0_0,
+
+            /* Component name and version */
+
+            .mca_component_name = "ze",
+            MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
+                                  OPAL_RELEASE_VERSION),
+
+            /* Component open and close functions */
+
+            .mca_open_component = accelerator_ze_open,
+            .mca_close_component = accelerator_ze_close,
+            .mca_register_component_params = accelerator_ze_component_register,
+
+        },
+    /* Next the MCA v1.0.0 component meta data */
+    .base_data =
+        { /* The component is checkpoint ready */
+         MCA_BASE_METADATA_PARAM_CHECKPOINT},
+    .accelerator_init = accelerator_ze_init,
+    .accelerator_finalize = accelerator_ze_finalize,
+}};
+
+static int accelerator_ze_open(void)
+{
+    /* construct the component fields */
+
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_ze_close(void)
+{
+    return OPAL_SUCCESS;
+}
+
+static int accelerator_ze_component_register(void)
+{
+    int var_id;
+
+    /* Set verbosity in the ze related code. */
+    opal_accelerator_ze_verbose = 0;
+    var_id = mca_base_component_var_register(&mca_accelerator_ze_component.super.base_version,
+                                             "verbose", "Set level of ze verbosity",
+                                             MCA_BASE_VAR_TYPE_INT, NULL,
+                                             0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+                                             &opal_accelerator_ze_verbose);
+    (void) mca_base_var_register_synonym (var_id, "ompi", "mpi", "accelerator_ze", "verbose",
+                                          MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
+
+    return OPAL_SUCCESS;
+}
+
+/*
+ * If this method is invoked it means we already
+ * initialized ZE in the accelerator_ze_init method below
+ */
+
+int opal_accelerator_ze_lazy_init(void)
+{
+    uint32_t i,d;
+    int err = OPAL_SUCCESS;
+    ze_result_t zret;
+    uint32_t driver_count = 0;
+    ze_driver_handle_t *all_drivers = NULL;
+
+    /* Double checked locking to avoid having to
+     * grab locks post lazy-initialization.  */
+
+    opal_atomic_rmb();
+    if (true == accelerator_ze_init_complete) {
+        return OPAL_SUCCESS;
+    }
+    OPAL_THREAD_LOCK(&accelerator_ze_init_lock);
+
+    /* If already initialized, just exit */
+    if (true == accelerator_ze_init_complete) {
+        goto fn_fail;
+    }
+
+    opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                       "ZE: starting lazy init");
+
+    zret = zeDriverGet(&driver_count, NULL);
+    if (ZE_RESULT_SUCCESS != zret) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "ZE: zeDriverGet returned %d\n", zret);
+        err =  OPAL_ERR_NOT_INITIALIZED;
+        goto fn_fail;
+    }
+
+    /*
+     * driver count should not be zero as to get here ZE component
+     * was successfully init'd.
+     */
+    if (0 == driver_count) {
+        err = OPAL_ERR_NOT_FOUND;
+    }
+
+    all_drivers = (ze_driver_handle_t *)malloc(driver_count * sizeof(ze_driver_handle_t));
+    if (all_drivers == NULL) {
+        err = OPAL_ERR_OUT_OF_RESOURCE;
+        goto fn_fail;
+    }
+
+    zret = zeDriverGet(&driver_count, all_drivers);
+    if (ZE_RESULT_SUCCESS != zret) {
+        err = OPAL_ERR_NOT_FOUND;
+        goto fn_fail;
+    }
+
+    /*
+     * Current design of ZE component assumes we find the first driver with a GPU device.
+     * we'll create a single ZE context if we do find such a device.  This may need to
+     * be revisited at some point but would impact areas of code outside of the 
+     * accelerator framework.
+     */
+
+    for (i = 0; i < driver_count; ++i) {
+        opal_accelerator_ze_device_count = 0;
+        zret = zeDeviceGet(all_drivers[i], &opal_accelerator_ze_device_count, NULL);
+        if (ZE_RESULT_SUCCESS != zret) {
+            opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                               "ZE: zeDeviceGet returned %d\n", zret);
+            err = OPAL_ERROR;
+            goto fn_fail;
+        }
+        opal_accelerator_ze_devices_handle =
+            malloc(opal_accelerator_ze_device_count * sizeof(ze_device_handle_t));
+        if (NULL == opal_accelerator_ze_devices_handle) {
+            err = OPAL_ERR_OUT_OF_RESOURCE;
+            goto fn_fail;
+        }
+        zret = zeDeviceGet(all_drivers[i], &opal_accelerator_ze_device_count, opal_accelerator_ze_devices_handle);
+        if (ZE_RESULT_SUCCESS != zret) {
+            opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                               "ZE: zeDeviceGet returned %d\n", zret);
+            err = OPAL_ERROR;
+            goto fn_fail;
+        }
+        /* Check if the driver supports a gpu */
+        for (d = 0; d < opal_accelerator_ze_device_count; ++d) {
+            ze_device_properties_t device_properties;
+            zret = zeDeviceGetProperties(opal_accelerator_ze_devices_handle[d], &device_properties);
+            if (ZE_RESULT_SUCCESS != zret) {
+                opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                                   "ZE: zeDeviceGetProperties returned %d\n", zret);
+                err = OPAL_ERROR;
+                goto fn_fail;
+            }
+
+            if (ZE_DEVICE_TYPE_GPU == device_properties.type) {
+                opal_accelerator_ze_driver_handle = all_drivers[i];
+                break;
+            }
+        }
+
+        if (NULL != opal_accelerator_ze_driver_handle) {
+            break;
+        } else {
+            free(opal_accelerator_ze_devices_handle);
+            opal_accelerator_ze_devices_handle = NULL;
+        }
+    }
+
+    ze_context_desc_t contextDesc = {
+        .stype = ZE_STRUCTURE_TYPE_CONTEXT_DESC,
+        .pNext = NULL,
+        .flags = 0,
+    };
+    zret = zeContextCreate(opal_accelerator_ze_driver_handle, 
+                          &contextDesc, 
+                          &opal_accelerator_ze_context);
+    if (ZE_RESULT_SUCCESS != zret) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                           "ZE: zeContextCreate returned %d\n", zret);
+        err = OPAL_ERROR;
+        goto fn_fail;
+    }
+
+    /*
+     * allocate synchronous memcpy stream handles, but delay creating streams till needed
+     */
+
+    opal_accelerator_ze_MemcpyStream = (opal_accelerator_stream_t **)calloc((size_t)opal_accelerator_ze_device_count,
+                                                                            sizeof(opal_accelerator_stream_t *));
+    if (NULL == opal_accelerator_ze_MemcpyStream) {
+        err = OPAL_ERR_OUT_OF_RESOURCE;
+        goto fn_fail;
+    }
+
+    /*
+     * set up an event pool
+     */
+
+    ze_event_pool_desc_t eventPoolDesc = {
+            .stype = ZE_STRUCTURE_TYPE_EVENT_POOL_DESC,
+            .pNext = NULL,
+            .flags = 0,
+            .count = 1000,  /* TODO: fix this! */
+    };
+
+    /*
+     * create an event pool that can be used by all devices associated with this ze context
+     */
+    zret = zeEventPoolCreate(opal_accelerator_ze_context,
+                             &eventPoolDesc,
+                             0,
+                             NULL,
+                             &opal_accelerator_ze_event_pool);
+    if (ZE_RESULT_SUCCESS != zret) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                           "ZE: zeEventPoolCreate returned %d\n", zret);
+        err = OPAL_ERROR;
+        goto fn_fail;
+    }
+
+    opal_atomic_wmb();
+    opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                       "ZE: found %d devices", opal_accelerator_ze_device_count);
+    accelerator_ze_init_complete = true;
+
+    return OPAL_SUCCESS;
+
+fn_fail:
+    if (NULL != opal_accelerator_ze_MemcpyStream) {
+        free(opal_accelerator_ze_MemcpyStream);
+        opal_accelerator_ze_MemcpyStream = NULL;
+    }
+
+    if (NULL != all_drivers) {
+        free(all_drivers);
+    }
+
+    if (OPAL_SUCCESS != err)  {
+        free(opal_accelerator_ze_devices_handle);
+        opal_accelerator_ze_devices_handle = NULL;
+    }
+      
+    OPAL_THREAD_UNLOCK(&accelerator_ze_init_lock);
+    return err;
+}
+
+static opal_accelerator_base_module_t* accelerator_ze_init(void)
+{
+    uint32_t driver_count=0;
+    ze_result_t zret;
+    ze_init_flag_t flags = ZE_INIT_FLAG_GPU_ONLY;
+
+    OBJ_CONSTRUCT(&accelerator_ze_init_lock, opal_mutex_t);
+
+    if (opal_ze_runtime_initialized) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output, "ZE: runtime not initialized");
+        return NULL;
+    }
+
+    /* 
+     * Initialize ze, this function can be called multiple times
+     */
+
+    zret = zeInit(flags);
+    if (ZE_RESULT_SUCCESS != zret) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output, "ZE: zeInit returned %d flags = %d\n", zret, flags);
+        return NULL;
+    }
+
+    /*
+     * zeDriverGet can return:
+     * ZE_RESULT_SUCCESS
+     * ZE_RESULT_ERROR_UNINITIALIZED
+     * ZE_RESULT_ERROR_DEVICE_LOST
+     * ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY
+     * ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY
+     */
+    zret = zeDriverGet(&driver_count, NULL);
+    if (ZE_RESULT_SUCCESS != zret || 0 == driver_count) {
+        if (ZE_RESULT_SUCCESS != zret) {
+            opal_output_verbose(10, opal_accelerator_base_framework.framework_output, 
+                               "ZE: zeDriverGet returned %d\n", zret);
+        } else {
+            opal_output_verbose(10, opal_accelerator_base_framework.framework_output, 
+                               "ZE: no device drivers found\n");
+        }
+        return NULL;
+    } else {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output, 
+                                   "ZE: %d device drivers found\n", driver_count);
+    }
+
+    opal_atomic_mb();
+    opal_ze_runtime_initialized = true;
+
+    return &opal_accelerator_ze_module;
+}
+
+static void accelerator_ze_finalize(opal_accelerator_base_module_t* module)
+{
+    ze_result_t zret;
+
+    opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "ZE: finalizing component\n");
+
+    if (NULL != opal_accelerator_ze_event_pool) {
+        zret = zeEventPoolDestroy(opal_accelerator_ze_event_pool);
+        if (ZE_RESULT_SUCCESS != zret) {
+            opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                                "zeEventPoolDestroy returned %d", zret);
+        }
+        opal_accelerator_ze_event_pool = NULL;
+    }
+
+
+    if (NULL != opal_accelerator_ze_MemcpyStream) {
+        for (uint32_t i = 0; i < opal_accelerator_ze_device_count; i++) {
+            if (NULL != opal_accelerator_ze_MemcpyStream[i]) {
+                OBJ_RELEASE(opal_accelerator_ze_MemcpyStream[i]);
+            }
+        }
+        free(opal_accelerator_ze_MemcpyStream);
+        opal_accelerator_ze_MemcpyStream = NULL;
+    }
+
+    if (NULL != (void *)opal_accelerator_ze_context) {
+        zret = zeContextDestroy(opal_accelerator_ze_context);
+        if (ZE_RESULT_SUCCESS != zret) {
+            opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                                "zeContextDestroy returned %d", zret);
+        }
+        opal_accelerator_ze_context = NULL;
+    }
+
+    opal_accelerator_ze_device_count = 0;
+
+    OBJ_DESTRUCT(&accelerator_ze_init_lock);
+    return;
+}
diff --git a/opal/mca/accelerator/ze/accelerator_ze_module.c b/opal/mca/accelerator/ze/accelerator_ze_module.c
new file mode 100644
index 00000000000..cb6cff21fef
--- /dev/null
+++ b/opal/mca/accelerator/ze/accelerator_ze_module.c
@@ -0,0 +1,890 @@
+/*
+ * Copyright (c) 2022      Advanced Micro Devices, Inc. All Rights reserved.
+ * Copyright (c) 2023-2024 Triad National Security, LLC. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include <string.h>
+
+#include "opal_config.h"
+
+#include "accelerator_ze.h"
+#include "opal/mca/accelerator/base/base.h"
+#include "opal/util/argv.h"
+#include "opal/util/printf.h"
+#include "opal/constants.h"
+#include "opal/util/output.h"
+#include "ompi/info/info_memkind.h"
+
+/* Accelerator API's */
+static int mca_accelerator_ze_check_addr(const void *addr, int *dev_id, uint64_t *flags);
+static int mca_accelerator_ze_create_stream(int dev_id, opal_accelerator_stream_t **stream);
+
+static int mca_accelerator_ze_create_event(int dev_id, opal_accelerator_event_t **event, bool enable_ipc);
+static int mca_accelerator_ze_record_event(int dev_id, opal_accelerator_event_t *event, opal_accelerator_stream_t *stream);
+static int mca_accelerator_ze_query_event(int dev_id, opal_accelerator_event_t *event);
+static int mca_accelerator_ze_wait_event(int dev_id, opal_accelerator_event_t *event, opal_accelerator_stream_t *stream);
+
+static int mca_accelerator_ze_memcpy_async(int dest_dev_id, int src_dev_id, void *dest, const void *src, size_t size,
+                                  opal_accelerator_stream_t *stream, opal_accelerator_transfer_type_t type);
+static int mca_accelerator_ze_memcpy(int dest_dev_id, int src_dev_id, void *dest, const void *src,
+                            size_t size, opal_accelerator_transfer_type_t type);
+static int mca_accelerator_ze_memmove_async(int dest_dev_id, int src_dev_id, void *dest,
+                                            const void *src, size_t size,
+                                            opal_accelerator_stream_t *stream,
+                                            opal_accelerator_transfer_type_t type);
+static int mca_accelerator_ze_memmove(int dest_dev_id, int src_dev_id, void *dest, const void *src, size_t size,
+                                        opal_accelerator_transfer_type_t type);
+static int mca_accelerator_ze_mem_alloc(int dev_id, void **ptr, size_t size);
+static int mca_accelerator_ze_mem_release(int dev_id, void *ptr);
+static int mca_accelerator_ze_mem_alloc_stream(int dev_id, void **ptr, size_t size,
+                                               opal_accelerator_stream_t *stream);
+static int mca_accelerator_ze_mem_release_stream(int dev_id, void *ptr, opal_accelerator_stream_t *stream);
+static int mca_accelerator_ze_get_address_range(int dev_id, const void *ptr, void **base,
+                                                  size_t *size);
+
+static bool mca_accelerator_ze_is_ipc_enabled(void);
+static int mca_accelerator_ze_get_ipc_handle(int dev_id, void *dev_ptr,
+                                             opal_accelerator_ipc_handle_t *handle);
+static int mca_accelerator_ze_import_ipc_handle(int dev_id, uint8_t ipc_handle[IPC_MAX_HANDLE_SIZE],
+                                                opal_accelerator_ipc_handle_t *handle);
+static int mca_accelerator_ze_open_ipc_handle(int dev_id, opal_accelerator_ipc_handle_t *handle,
+                                              void **dev_ptr);
+static int mca_accelerator_ze_compare_ipc_handles(uint8_t handle_1[IPC_MAX_HANDLE_SIZE],
+                                             uint8_t handle_2[IPC_MAX_HANDLE_SIZE]);
+static int mca_accelerator_ze_get_ipc_event_handle(opal_accelerator_event_t *event,
+                                                   opal_accelerator_ipc_event_handle_t *handle);
+static int mca_accelerator_ze_import_ipc_event_handle(uint8_t ipc_handle[IPC_MAX_HANDLE_SIZE],
+                                                      opal_accelerator_ipc_event_handle_t *handle);
+static int mca_accelerator_ze_open_ipc_event_handle(opal_accelerator_ipc_event_handle_t *handle,
+                                                    opal_accelerator_event_t *event);
+
+static int mca_accelerator_ze_host_register(int dev_id, void *ptr, size_t size);
+static int mca_accelerator_ze_host_unregister(int dev_id, void *ptr);
+
+static int mca_accelerator_ze_get_device(int *dev_id);
+static int mca_accelerator_ze_device_can_access_peer( int *access, int dev1, int dev2);
+
+static int mca_accelerator_ze_get_device_pci_attr(int dev_id, opal_accelerator_pci_attr_t *pci_attr);
+
+static int mca_accelerator_ze_get_buffer_id(int dev_id, const void *addr, opal_accelerator_buffer_id_t *buf_id);
+
+static int mca_accelerator_ze_sync_stream(opal_accelerator_stream_t *stream);
+
+static int mca_accelerator_ze_get_num_devices(int *num_devices);
+
+static int mca_accelerator_ze_get_mem_bw(int device, float *bw);
+static void mca_accelerator_ze_get_memkind(ompi_memkind_t *memkind);
+
+opal_accelerator_base_module_t opal_accelerator_ze_module =
+{
+    .check_addr = mca_accelerator_ze_check_addr,
+
+    .create_stream = mca_accelerator_ze_create_stream,
+    .sync_stream = mca_accelerator_ze_sync_stream,
+    .create_event = mca_accelerator_ze_create_event,
+    .record_event = mca_accelerator_ze_record_event,
+    .query_event = mca_accelerator_ze_query_event,
+    .wait_event = mca_accelerator_ze_wait_event,
+
+    .mem_copy_async = mca_accelerator_ze_memcpy_async,
+    .mem_copy = mca_accelerator_ze_memcpy,
+    .mem_move_async = mca_accelerator_ze_memmove_async,
+    .mem_move = mca_accelerator_ze_memmove,
+
+    .mem_alloc = mca_accelerator_ze_mem_alloc,
+    .mem_release = mca_accelerator_ze_mem_release,
+    .mem_alloc_stream = mca_accelerator_ze_mem_alloc_stream,
+    .mem_release_stream = mca_accelerator_ze_mem_release_stream,
+    .get_address_range = mca_accelerator_ze_get_address_range,
+
+    .is_ipc_enabled = mca_accelerator_ze_is_ipc_enabled,
+    .get_ipc_handle = mca_accelerator_ze_get_ipc_handle,
+    .import_ipc_handle = mca_accelerator_ze_import_ipc_handle,
+    .open_ipc_handle = mca_accelerator_ze_open_ipc_handle,
+    .compare_ipc_handles = mca_accelerator_ze_compare_ipc_handles,
+    .get_ipc_event_handle = mca_accelerator_ze_get_ipc_event_handle,
+    .import_ipc_event_handle = mca_accelerator_ze_import_ipc_event_handle,
+    .open_ipc_event_handle = mca_accelerator_ze_open_ipc_event_handle,
+
+    .host_register = mca_accelerator_ze_host_register,
+    .host_unregister = mca_accelerator_ze_host_unregister,
+
+    .get_device= mca_accelerator_ze_get_device,
+    .get_device_pci_attr = mca_accelerator_ze_get_device_pci_attr,
+    .device_can_access_peer = mca_accelerator_ze_device_can_access_peer,
+
+    .get_buffer_id = mca_accelerator_ze_get_buffer_id,
+    .num_devices = mca_accelerator_ze_get_num_devices,
+    .get_mem_bw = mca_accelerator_ze_get_mem_bw,
+    .get_memkind = mca_accelerator_ze_get_memkind
+};
+
+static int accelerator_ze_dev_handle_to_dev_id(ze_device_handle_t hDevice)
+{
+    int i, ret = MCA_ACCELERATOR_NO_DEVICE_ID;
+
+    for (i = 0; i < (int)opal_accelerator_ze_device_count; i++) {
+        if (opal_accelerator_ze_devices_handle[i] == hDevice) {
+            ret = i;
+            break;
+        }
+    }
+
+    return ret;
+}
+
+static int mca_accelerator_ze_check_addr (const void *addr, int *dev_id, uint64_t *flags)
+{
+    ze_result_t zret;
+    int ret = 0;
+    ze_memory_allocation_properties_t attr;
+    ze_device_handle_t hDevice;
+
+    *dev_id = MCA_ACCELERATOR_NO_DEVICE_ID;
+    *flags = 0;
+
+    if (NULL == addr || NULL == flags) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    ret = opal_accelerator_ze_lazy_init();
+    if (OPAL_SUCCESS != ret) {
+        return ret;
+    }
+
+    memset(&attr, 0, sizeof(ze_memory_allocation_properties_t));
+
+    zret = zeMemGetAllocProperties(opal_accelerator_ze_context,
+                                  addr,
+                                  &attr,
+                                  &hDevice);
+    if (ZE_RESULT_SUCCESS != zret) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                               "zeMemGetAllocProperties returned %d", zret);
+        goto fn_fail;
+    }
+    switch (attr.type) {
+        case ZE_MEMORY_TYPE_UNKNOWN:
+        case ZE_MEMORY_TYPE_HOST:
+            break;
+        case ZE_MEMORY_TYPE_DEVICE:
+        case ZE_MEMORY_TYPE_SHARED:
+            ret = 1;
+            *dev_id = accelerator_ze_dev_handle_to_dev_id(hDevice);
+            break;
+        default:
+            goto fn_fail;
+    }
+
+fn_fail:
+
+    return ret;
+}
+
+static int mca_accelerator_ze_create_stream(int dev_id, opal_accelerator_stream_t **stream)
+{
+    int ret;
+    ze_result_t zret;
+    ze_device_handle_t hDevice;
+    opal_accelerator_ze_stream_t *ze_stream;
+
+    ze_command_queue_desc_t cmdQueueDesc = {
+            .stype = ZE_STRUCTURE_TYPE_COMMAND_QUEUE_DESC,
+            .pNext = NULL,
+            .index = 0,
+            .flags = 0,
+            .ordinal = 0,
+            .mode = ZE_COMMAND_QUEUE_MODE_ASYNCHRONOUS,
+            .priority = ZE_COMMAND_QUEUE_PRIORITY_NORMAL,
+    };
+
+    if (NULL == stream) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    ret = opal_accelerator_ze_lazy_init();
+    if (OPAL_SUCCESS != ret) {
+        return ret;
+    }
+
+    *stream = (opal_accelerator_stream_t*)OBJ_NEW(opal_accelerator_ze_stream_t);
+    if (NULL == *stream) {
+        return OPAL_ERR_OUT_OF_RESOURCE;
+    }
+
+    ze_stream = (opal_accelerator_ze_stream_t *)malloc(sizeof(opal_accelerator_ze_stream_t));
+    if (NULL == ze_stream) {
+        OBJ_RELEASE(*stream);
+        return OPAL_ERR_OUT_OF_RESOURCE;
+    }
+
+    if (MCA_ACCELERATOR_NO_DEVICE_ID == dev_id) {
+        hDevice = opal_accelerator_ze_devices_handle[0];
+    } else {
+        hDevice = opal_accelerator_ze_devices_handle[dev_id];
+    }
+    ze_stream->dev_id = dev_id;
+
+    zret = zeCommandQueueCreate(opal_accelerator_ze_context,
+                                hDevice,
+                                &cmdQueueDesc,
+                                &ze_stream->hCommandQueue);
+    if (ZE_RESULT_SUCCESS != zret) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "zeCommandQueueCreate returned %d", zret);
+        return OPAL_ERROR;
+    }
+
+    /*
+     * create a command list
+     */
+
+    ze_command_list_desc_t commandListDesc = {
+        .stype =  ZE_STRUCTURE_TYPE_COMMAND_LIST_DESC,
+        .pNext = NULL,
+        .commandQueueGroupOrdinal = 0,
+        .flags = 0,
+    };
+
+    zret = zeCommandListCreate(opal_accelerator_ze_context,
+                               opal_accelerator_ze_devices_handle[0],
+                               &commandListDesc,
+                               &ze_stream->hCommandList);
+    if (ZE_RESULT_SUCCESS != zret) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "zeCommandListCreate returned %d", zret);
+        return OPAL_ERROR;
+    }
+    (*stream)->stream = (void *)ze_stream;
+
+    return OPAL_SUCCESS;
+}
+
+static void mca_accelerator_ze_stream_destruct(opal_accelerator_ze_stream_t *stream)
+{
+    ze_result_t zret;
+    opal_accelerator_ze_stream_t *ze_stream;
+
+    if (NULL != stream->base.stream) {
+        ze_stream = (opal_accelerator_ze_stream_t  *)stream->base.stream;
+        zret = zeCommandQueueDestroy(ze_stream->hCommandQueue);
+        if (ZE_RESULT_SUCCESS != zret) {
+            opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                                "error while destroying the zeCommandQueue");
+        }
+
+        free(stream->base.stream);
+    }
+}
+
+OBJ_CLASS_INSTANCE(
+    opal_accelerator_ze_stream_t,
+    opal_accelerator_stream_t,
+    NULL,
+    mca_accelerator_ze_stream_destruct);
+
+static int mca_accelerator_ze_create_event(int dev_id, opal_accelerator_event_t **event, bool enable_ipc)
+{
+    ze_result_t zret;
+
+    ze_event_desc_t eventDesc = {
+       .stype = ZE_STRUCTURE_TYPE_EVENT_DESC,
+       .pNext = NULL,
+       .index = 0,
+       .signal = ZE_EVENT_SCOPE_FLAG_HOST,
+       .wait = ZE_EVENT_SCOPE_FLAG_HOST,
+    };
+
+    if (NULL == event) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    *event = (opal_accelerator_event_t*)OBJ_NEW(opal_accelerator_ze_event_t);
+    if (NULL == *event) {
+        return OPAL_ERR_OUT_OF_RESOURCE;
+    }
+
+    (*event)->event = malloc(sizeof(ze_event_handle_t));
+    if (NULL == (*event)->event) {
+        OBJ_RELEASE(*event);
+        return OPAL_ERR_OUT_OF_RESOURCE;
+    }
+
+    zret = zeEventCreate(opal_accelerator_ze_event_pool, &eventDesc, (ze_event_handle_t *)(*event)->event);
+    if (ZE_RESULT_SUCCESS != zret) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "error creating event %d", zret);
+        free((*event)->event);
+        OBJ_RELEASE(*event);
+        return OPAL_ERROR;
+    }
+
+    return OPAL_SUCCESS;
+
+}
+
+static void mca_accelerator_ze_event_destruct(opal_accelerator_ze_event_t *event)
+{
+    ze_result_t zret;
+
+    if (NULL != event->base.event) {
+        zret = zeEventDestroy(*(ze_event_handle_t *)event->base.event);
+        if (ZE_RESULT_SUCCESS != zret) {
+            opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                                "error destroying event %d", zret);
+        }
+        free(event->base.event);
+    }
+}
+
+OBJ_CLASS_INSTANCE(
+    opal_accelerator_ze_event_t,
+    opal_accelerator_event_t,
+    NULL,
+    mca_accelerator_ze_event_destruct);
+
+static int mca_accelerator_ze_record_event(int dev_id, opal_accelerator_event_t *event,
+                                             opal_accelerator_stream_t *stream)
+{
+    ze_result_t zret = ZE_RESULT_SUCCESS;
+    opal_accelerator_ze_stream_t *ze_stream;
+
+    if (NULL == event || NULL == event->event){
+        return OPAL_ERR_BAD_PARAM;
+    }
+    if (NULL == stream || NULL == stream->stream){
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    ze_stream = (opal_accelerator_ze_stream_t  *)stream->stream;
+
+    zret = zeCommandListAppendSignalEvent(ze_stream->hCommandList,
+                                          *(ze_event_handle_t *)event->event);
+    if (ZE_RESULT_SUCCESS != zret) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                           "zeCommandListAppendSignalEvent returned %d", zret);
+        return OPAL_ERROR;
+    }
+
+    /*
+     * okay now close the command list and submit
+     */
+
+    zret = zeCommandListClose(ze_stream->hCommandList);
+    if (ZE_RESULT_SUCCESS != zret) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                           "zeCommandListClose returned %d", zret);
+        return OPAL_ERROR;
+    }
+
+    zret = zeCommandQueueExecuteCommandLists(ze_stream->hCommandQueue,
+                                             1,
+                                             &ze_stream->hCommandList,
+                                             NULL);
+    if (ZE_RESULT_SUCCESS != zret) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                           "zeCommandQueueExecuteCommandList returned %d", zret);
+        return OPAL_ERROR;
+    }
+
+    return OPAL_SUCCESS;
+}
+
+static int mca_accelerator_ze_wait_event(int __opal_attribute_unused__ dev_id,
+                                         opal_accelerator_event_t *event,
+                                         opal_accelerator_stream_t * __opal_attribute_unused__ stream)
+{
+    ze_result_t zret;
+
+    zret = zeEventHostSynchronize(*((ze_event_handle_t *)event->event),
+                                  UINT64_MAX);
+    switch (zret) {
+        case ZE_RESULT_SUCCESS:
+            return OPAL_SUCCESS;
+            break;
+        case ZE_RESULT_NOT_READY:
+            return OPAL_ERR_RESOURCE_BUSY;
+            break;
+        default:
+            opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                                "zeEventHostSynchronize returned %d", zret);
+            return OPAL_ERROR;
+    }
+
+    return OPAL_SUCCESS;
+}
+
+
+
+static int mca_accelerator_ze_query_event(int dev_id, opal_accelerator_event_t *event)
+{
+    ze_result_t zret;
+
+    if (NULL == event || NULL == event->event) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    zret = zeEventQueryStatus(*((ze_event_handle_t *)event->event));
+    switch (zret) {
+        case ZE_RESULT_SUCCESS:
+            return OPAL_SUCCESS;
+            break;
+        case ZE_RESULT_NOT_READY:
+            return OPAL_ERR_RESOURCE_BUSY;
+            break;
+        default:
+            opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                                "zeEventQueryStatus returned %d", zret);
+            return OPAL_ERROR;
+    }
+
+    return OPAL_SUCCESS;
+}
+
+static int mca_accelerator_ze_memcpy_async(int dest_dev_id, int src_dev_id, void *dest, const void *src,
+                                             size_t size, opal_accelerator_stream_t *stream,
+                                             opal_accelerator_transfer_type_t type)
+{
+   ze_result_t zret;
+   opal_accelerator_ze_stream_t *ze_stream = NULL;
+
+   if (NULL == stream || NULL == src ||
+        NULL == dest   || size < 0) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+   if (0 == size) {
+       return OPAL_SUCCESS;
+   }
+
+   ze_stream = (opal_accelerator_ze_stream_t  *)stream->stream;
+   assert(NULL != ze_stream);
+
+    zret = zeCommandListAppendMemoryCopy(ze_stream->hCommandList,
+                                         dest,
+                                         src,
+                                         size,
+                                         NULL,
+                                         0,
+                                         NULL);
+    if (ZE_RESULT_SUCCESS != zret) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "zeCommandListAppendMemoryCopy returned %d", zret);
+        return OPAL_ERROR;
+    }
+
+    return OPAL_SUCCESS;
+}
+
+static int mca_accelerator_ze_memcpy(int dest_dev_id, int src_dev_id, void *dest,
+				       const void *src, size_t size,
+                                       opal_accelerator_transfer_type_t type)
+{
+    int ret, dev_id;
+    ze_result_t zret;
+
+    opal_accelerator_ze_stream_t *ze_stream = NULL;
+
+    if (NULL == src || NULL == dest || size <0) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+    if (0 == size) {
+        return OPAL_SUCCESS;
+    }
+
+    if (MCA_ACCELERATOR_NO_DEVICE_ID == src_dev_id) {
+        dev_id = 0;
+    } else {
+        dev_id = src_dev_id;
+    }
+
+    if (NULL == opal_accelerator_ze_MemcpyStream[dev_id]) {
+        ret = mca_accelerator_ze_create_stream(dev_id,
+                                               (opal_accelerator_stream_t **)&opal_accelerator_ze_MemcpyStream[dev_id]);
+        if (OPAL_SUCCESS != ret) {
+            return ret;
+        }
+    }
+
+    ze_stream = (opal_accelerator_ze_stream_t *)opal_accelerator_ze_MemcpyStream[dev_id]->stream;
+    zret = zeCommandListAppendMemoryCopy(ze_stream->hCommandList,
+                                         dest,
+                                         src,
+                                         size,
+                                         NULL,
+                                         0,
+                                         NULL);
+    if (ZE_RESULT_SUCCESS != zret) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "zeCommandListAppendMemoryCopy returned %d", zret);
+        return OPAL_ERROR;
+    }
+
+    zret = zeCommandListClose(ze_stream->hCommandList);
+    if (ZE_RESULT_SUCCESS != zret) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "zeCommandListClose returned %d", zret);
+        return OPAL_ERROR;
+    }
+
+    zret = zeCommandQueueExecuteCommandLists(ze_stream->hCommandQueue,
+                                             1,
+                                             &ze_stream->hCommandList,
+                                             NULL);
+    if (ZE_RESULT_SUCCESS != zret) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "zeCommandQueueExecuteCommandList returned %d", zret);
+        return OPAL_ERROR;
+    }
+
+    zret = zeCommandQueueSynchronize(ze_stream->hCommandQueue,
+                                     UINT32_MAX);
+    if (ZE_RESULT_SUCCESS != zret) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "zeCommandQueueSynchronize returned %d", zret);
+        return OPAL_ERROR;
+    }
+
+    zret = zeCommandListReset(ze_stream->hCommandList);
+    if (ZE_RESULT_SUCCESS != zret) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "zeCommandListReset returned %d", zret);
+        return OPAL_ERROR;
+    }
+
+
+    return OPAL_SUCCESS;
+}
+
+static int mca_accelerator_ze_memmove(int dest_dev_id, int src_dev_id, void *dest,
+					const void *src, size_t size,
+                                        opal_accelerator_transfer_type_t type)
+{
+    /*
+     * TODO
+     */
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
+
+static int mca_accelerator_ze_memmove_async(int dest_dev_id, int src_dev_id, void *dest,
+                                            const void *src, size_t size,
+                                            opal_accelerator_stream_t *stream,
+                                            opal_accelerator_transfer_type_t type)
+{
+    /*
+     * TODO
+     */
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
+
+static int mca_accelerator_ze_mem_alloc(int dev_id, void **ptr, size_t size)
+{
+   ze_result_t zret;
+   size_t mem_alignment;
+   ze_device_handle_t hDevice;
+
+   ze_device_mem_alloc_desc_t device_desc = {
+        .stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC,
+        .pNext = NULL,
+        .flags = 0,
+        .ordinal = 0,   /* We currently support a single memory type */
+    };
+
+    if (MCA_ACCELERATOR_NO_DEVICE_ID == dev_id) {
+        hDevice = opal_accelerator_ze_devices_handle[0];
+    } else {
+        hDevice = opal_accelerator_ze_devices_handle[dev_id];
+    }
+
+    /* Currently ZE ignores this argument and uses an internal alignment
+     * value. However, this behavior can change in the future. */
+    mem_alignment = 1;
+    zret = zeMemAllocDevice(opal_accelerator_ze_context,
+                           &device_desc,
+                           size,
+                           mem_alignment,
+                           hDevice,
+                           ptr);
+    if (ZE_RESULT_SUCCESS != zret) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "zeMemAllocDevice returned %d", zret);
+        goto fn_fail;
+    }
+
+    return OPAL_SUCCESS;
+  fn_fail:
+    return OPAL_ERROR;
+}
+
+static int mca_accelerator_ze_mem_release(int dev_id, void *ptr)
+{
+    ze_result_t zret;
+
+    zret = zeMemFree(opal_accelerator_ze_context, ptr);
+    if (ZE_RESULT_SUCCESS != zret) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "zeMemFree returned %d", zret);
+        goto fn_fail;
+    }
+
+    return OPAL_SUCCESS;
+  fn_fail:
+    return OPAL_ERROR;
+}
+
+static int mca_accelerator_ze_mem_alloc_stream(int dev_id, void **ptr, size_t size,
+                                               opal_accelerator_stream_t *stream)
+{
+    /*
+     * TODO
+     */
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
+
+static int mca_accelerator_ze_mem_release_stream(int dev_id, void *ptr, opal_accelerator_stream_t *stream)
+{
+    /*
+     * TODO
+     */
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
+
+static int mca_accelerator_ze_get_address_range(int dev_id, const void *ptr, void **base,
+						  size_t *size)
+{
+    ze_result_t zret;
+    void *pBase;
+    size_t pSize;
+
+    if (NULL == ptr || NULL == base || NULL == size) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    zret = zeMemGetAddressRange(opal_accelerator_ze_context,
+                                ptr,
+                                &pBase,
+                                &pSize);
+    if (ZE_RESULT_SUCCESS != zret) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "couldn't get address range for pointer %p/%lu %d", ptr, *size, zret);
+        return OPAL_ERROR;
+    }
+
+    *size = pSize;
+    *base = (char *) pBase;
+
+    return OPAL_SUCCESS;
+}
+
+static bool mca_accelerator_ze_is_ipc_enabled(void)
+{
+    return false;
+}
+
+static int mca_accelerator_ze_get_ipc_handle(int dev_id, void *dev_ptr,
+                                             opal_accelerator_ipc_handle_t *handle)
+{
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
+
+static int mca_accelerator_ze_import_ipc_handle(int dev_id, uint8_t ipc_handle[IPC_MAX_HANDLE_SIZE],
+                                                opal_accelerator_ipc_handle_t *handle)
+{
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
+
+static int mca_accelerator_ze_open_ipc_handle(int dev_id, opal_accelerator_ipc_handle_t *handle,
+                                              void **dev_ptr)
+{
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
+
+static int mca_accelerator_ze_compare_ipc_handles(uint8_t handle_1[IPC_MAX_HANDLE_SIZE],
+                                              uint8_t handle_2[IPC_MAX_HANDLE_SIZE])
+{
+    return memcmp(handle_1, handle_2, IPC_MAX_HANDLE_SIZE);
+}
+
+static int mca_accelerator_ze_get_ipc_event_handle(opal_accelerator_event_t *event,
+                                                   opal_accelerator_ipc_event_handle_t *handle)
+{
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
+
+static int mca_accelerator_ze_import_ipc_event_handle(uint8_t ipc_handle[IPC_MAX_HANDLE_SIZE],
+                                                      opal_accelerator_ipc_event_handle_t *handle)
+{
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
+
+static int mca_accelerator_ze_open_ipc_event_handle(opal_accelerator_ipc_event_handle_t *handle,
+                                                    opal_accelerator_event_t *event)
+{
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
+
+/*
+ * ZE doesn't have explicit host memory registration functions
+ */
+
+static int mca_accelerator_ze_host_register(int dev_id, void *ptr, size_t size)
+{
+    return OPAL_SUCCESS;
+}
+
+static int mca_accelerator_ze_host_unregister(int dev_id, void *ptr)
+{
+    return OPAL_SUCCESS;
+}
+
+static int mca_accelerator_ze_get_device(int *dev_id)
+{
+    /*
+     * this method does not map to the Zero Level API, just return 0.
+     * This may just work if the runtime is use the ZE_AFFINITY_MASK
+     * environment variable to control the visible PV(s) for a given process.
+     */
+
+    if (NULL == dev_id) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    *dev_id = 0;
+
+    return OPAL_SUCCESS;
+}
+
+static int mca_accelerator_ze_get_device_pci_attr(int dev_id, opal_accelerator_pci_attr_t *pci_attr)
+{
+    ze_result_t zret;
+    ze_device_handle_t hDevice;
+    ze_pci_ext_properties_t pPciProperties;
+
+    if (NULL == pci_attr) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    if (MCA_ACCELERATOR_NO_DEVICE_ID == dev_id) {
+        hDevice = opal_accelerator_ze_devices_handle[0];
+    } else {
+        hDevice = opal_accelerator_ze_devices_handle[dev_id];
+    }
+
+    zret = zeDevicePciGetPropertiesExt(hDevice, &pPciProperties);
+    if(ZE_RESULT_SUCCESS != zret) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "zeDevicePciGetPropertiesExt returned %d", zret);
+        return OPAL_ERROR;
+    }
+
+    pci_attr->domain_id = (uint16_t)pPciProperties.address.domain;
+    pci_attr->bus_id = (uint8_t) pPciProperties.address.bus;
+    pci_attr->device_id = (uint8_t)pPciProperties.address.device;
+    pci_attr->function_id = (uint8_t)pPciProperties.address.function;
+
+    return OPAL_SUCCESS;
+}
+
+
+/*
+ * could zeDeviceGetP2PProperties be used instead here?
+ */
+static int mca_accelerator_ze_device_can_access_peer(int *access, int dev1, int dev2)
+{
+    ze_result_t zret;
+    ze_bool_t value;
+    ze_device_handle_t hDevice;
+    ze_device_handle_t hPeerDevice;
+
+    if (NULL == access || dev1 < 0 || dev2 < 0){
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    hDevice = opal_accelerator_ze_devices_handle[dev1];
+    hPeerDevice = opal_accelerator_ze_devices_handle[dev2];
+
+    zret = zeDeviceCanAccessPeer(hDevice,
+                                 hPeerDevice,
+                                 &value);
+    if (ZE_RESULT_SUCCESS != zret) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "zeDeviceCanAccessPeer returned %d", zret);
+        return OPAL_ERROR;
+    }
+
+    *access = (value == 1) ? 1 : 0;
+
+    return OPAL_SUCCESS;
+}
+
+static int mca_accelerator_ze_get_buffer_id(int dev_id, const void *addr, opal_accelerator_buffer_id_t *buf_id)
+{
+    ze_result_t zret;
+    ze_memory_allocation_properties_t pMemAllocProperties;
+    ze_device_handle_t hDevice;
+
+    if (NULL == buf_id) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    if (MCA_ACCELERATOR_NO_DEVICE_ID == dev_id) {
+        hDevice = opal_accelerator_ze_devices_handle[0];
+    } else {
+        hDevice = opal_accelerator_ze_devices_handle[dev_id];
+    }
+
+    zret = zeMemGetAllocProperties(opal_accelerator_ze_context,
+                                   addr,
+                                   &pMemAllocProperties,
+                                   &hDevice);
+    if (ZE_RESULT_SUCCESS != zret) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "zeMemGetAllocProperties returned %d", zret);
+        return OPAL_ERROR;
+    }
+
+    *buf_id = pMemAllocProperties.id;
+
+    return OPAL_SUCCESS;
+}
+
+
+static int mca_accelerator_ze_wait_stream(opal_accelerator_stream_t *stream)
+{
+    /*
+     * TODO
+     */
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
+
+static int mca_accelerator_ze_get_num_devices(int *num_devices)
+{
+    /*
+     * TODO
+     */
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
+
+static int mca_accelerator_ze_get_mem_bw(int device, float *bw)
+{
+    /*
+     * TODO
+     */
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
+
+static void mca_accelerator_ze_get_memkind (ompi_memkind_t *memkind)
+{
+  memkind->im_name = strdup("level_zero");
+  memkind->im_no_restrictors = false;
+  memkind->im_num_restrictors = 3;
+  memkind->im_restrictors[0] = strdup("host");
+  memkind->im_restrictors[1] = strdup("device");
+  memkind->im_restrictors[2] = strdup("shared");
+
+  return;
+}
diff --git a/opal/mca/accelerator/ze/configure.m4 b/opal/mca/accelerator/ze/configure.m4
new file mode 100644
index 00000000000..48b5256eda8
--- /dev/null
+++ b/opal/mca/accelerator/ze/configure.m4
@@ -0,0 +1,28 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2022      Advanced Micro Devices, Inc. All rights reserved.
+# Copyright (c) 2023      Triad National Security, LLC. All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# MCA_opal_accelerator_ze_CONFIG([action-if-can-compile],
+#                                  [action-if-cant-compile])
+# ------------------------------------------------
+AC_DEFUN([MCA_opal_accelerator_ze_CONFIG],[
+    AC_CONFIG_FILES([opal/mca/accelerator/ze/Makefile])
+
+    OPAL_CHECK_ZE([opal_ze],
+	        [opal_ze_happy="yes"],
+	        [opal_ze_happy="no"])
+    OPAL_SUMMARY_ADD([Accelerators], [Intel ZE support], [], [$opal_ze_happy])
+
+    AS_IF([test "$opal_ze_happy" = "yes"],
+	  [$1],
+	  [$2])
+    AC_SUBST([opal_ze_LDFLAGS])
+    AC_SUBST([opal_ze_LIBS])
+])dnl
diff --git a/opal/mca/allocator/allocator.h b/opal/mca/allocator/allocator.h
index 77180462c3d..28b87ddad36 100644
--- a/opal/mca/allocator/allocator.h
+++ b/opal/mca/allocator/allocator.h
@@ -19,9 +19,9 @@
  * $HEADER$
  */
 /**
-  * @file
-  * The public definition of the MCA Allocator framework.
-  */
+ * @file
+ * The public definition of the MCA Allocator framework.
+ */
 #ifndef MCA_ALLOCATOR_H
 #define MCA_ALLOCATOR_H
 
@@ -34,43 +34,35 @@ BEGIN_C_DECLS
 struct mca_allocator_base_module_t;
 
 /**
-  * The allocate function typedef for the function to be provided by the component.
-  */
-typedef void* (*mca_allocator_base_module_alloc_fn_t)(
-    struct mca_allocator_base_module_t*,
-    size_t size,
-    size_t align);
+ * The allocate function typedef for the function to be provided by the component.
+ */
+typedef void *(*mca_allocator_base_module_alloc_fn_t)(struct mca_allocator_base_module_t *,
+                                                      size_t size, size_t align);
 
 /**
-  * The realloc function typedef
-  */
-typedef void* (*mca_allocator_base_module_realloc_fn_t)(
-    struct mca_allocator_base_module_t*,
-    void*, size_t);
+ * The realloc function typedef
+ */
+typedef void *(*mca_allocator_base_module_realloc_fn_t)(struct mca_allocator_base_module_t *,
+                                                        void *, size_t);
 
 /**
-  * Free function typedef
-  */
-typedef void(*mca_allocator_base_module_free_fn_t)(
-    struct mca_allocator_base_module_t*, void *);
-
+ * Free function typedef
+ */
+typedef void (*mca_allocator_base_module_free_fn_t)(struct mca_allocator_base_module_t *, void *);
 
 /**
  * compact/return memory to higher level allocator
  */
 
 typedef int (*mca_allocator_base_module_compact_fn_t)(
-    struct mca_allocator_base_module_t* allocator
-);
-
+    struct mca_allocator_base_module_t *allocator);
 
 /**
  * cleanup (free) any resources held by allocator
  */
 
 typedef int (*mca_allocator_base_module_finalize_fn_t)(
-    struct mca_allocator_base_module_t* allocator
-);
+    struct mca_allocator_base_module_t *allocator);
 
 /**
  * The data structure for each component.
@@ -94,34 +86,26 @@ struct mca_allocator_base_module_t {
  */
 typedef struct mca_allocator_base_module_t mca_allocator_base_module_t;
 
-
 /**
-  * A function to get more memory from the system. This function is to be
-  * provided by the module to the allocator framework.
-  */
+ * A function to get more memory from the system. This function is to be
+ * provided by the module to the allocator framework.
+ */
 
-typedef void* (*mca_allocator_base_component_segment_alloc_fn_t)(void *ctx,
-                                                                 size_t *size);
+typedef void *(*mca_allocator_base_component_segment_alloc_fn_t)(void *ctx, size_t *size);
 
 /**
-  * A function to free memory from the control of the allocator framework
-  * back to the system. This function is to be provided by the module to the
-  * allocator framework.
-  */
-typedef void (*mca_allocator_base_component_segment_free_fn_t)(void *ctx,
-                                                               void *segment);
-
+ * A function to free memory from the control of the allocator framework
+ * back to the system. This function is to be provided by the module to the
+ * allocator framework.
+ */
+typedef void (*mca_allocator_base_component_segment_free_fn_t)(void *ctx, void *segment);
 
 /**
-  * The function used to initialize the component.
-  */
-typedef struct mca_allocator_base_module_t*
-    (*mca_allocator_base_component_init_fn_t)(
-    bool enable_mpi_threads,
-    mca_allocator_base_component_segment_alloc_fn_t segment_alloc,
-    mca_allocator_base_component_segment_free_fn_t segment_free,
-    void *context
-);
+ * The function used to initialize the component.
+ */
+typedef struct mca_allocator_base_module_t *(*mca_allocator_base_component_init_fn_t)(
+    bool enable_mpi_threads, mca_allocator_base_component_segment_alloc_fn_t segment_alloc,
+    mca_allocator_base_component_segment_free_fn_t segment_free, void *context);
 
 /**
  * The data structure provided by each component to the framework which
@@ -144,10 +128,8 @@ typedef struct mca_allocator_base_component_2_0_0_t mca_allocator_base_component
 /**
  * Macro for use in components that are of type allocator
  */
-#define MCA_ALLOCATOR_BASE_VERSION_2_0_0 \
-    OPAL_MCA_BASE_VERSION_2_1_0("allocator", 2, 0, 0)
+#define MCA_ALLOCATOR_BASE_VERSION_2_0_0 OPAL_MCA_BASE_VERSION_2_1_0("allocator", 2, 0, 0)
 
 END_C_DECLS
 
 #endif /* MCA_ALLOCATOR_H */
-
diff --git a/opal/mca/allocator/base/allocator_base_frame.c b/opal/mca/allocator/base/allocator_base_frame.c
index cf8559af2af..890d315a826 100644
--- a/opal/mca/allocator/base/allocator_base_frame.c
+++ b/opal/mca/allocator/base/allocator_base_frame.c
@@ -16,15 +16,14 @@
  * $HEADER$
  */
 
-
 #include "opal_config.h"
 #include <stdio.h>
 #include <string.h>
 
-#include "opal/mca/mca.h"
-#include "opal/mca/base/base.h"
 #include "opal/mca/allocator/allocator.h"
 #include "opal/mca/allocator/base/base.h"
+#include "opal/mca/base/base.h"
+#include "opal/mca/mca.h"
 
 /*
  * The following file was created by configure.  It contains extern
@@ -49,18 +48,17 @@ MCA_BASE_FRAMEWORK_DECLARE(opal, allocator, NULL, NULL, NULL, NULL,
  * @retval mca_allocator_base_component_t* pointer to the requested component
  * @retval NULL if the requested component is not found
  */
-mca_allocator_base_component_t* mca_allocator_component_lookup(const char* name)
+mca_allocator_base_component_t *mca_allocator_component_lookup(const char *name)
 {
     /* Traverse the list of available components; call their init functions. */
     mca_base_component_list_item_t *cli;
-    OPAL_LIST_FOREACH(cli, &opal_allocator_base_framework.framework_components, mca_base_component_list_item_t) {
-         mca_allocator_base_component_t* component = (mca_allocator_base_component_t *) cli->cli_component;
-         if(strcmp(component->allocator_version.mca_component_name,
-                   name) == 0) {
-             return component;
-         }
+    OPAL_LIST_FOREACH (cli, &opal_allocator_base_framework.framework_components,
+                       mca_base_component_list_item_t) {
+        mca_allocator_base_component_t *component = (mca_allocator_base_component_t *)
+                                                        cli->cli_component;
+        if (strcmp(component->allocator_version.mca_component_name, name) == 0) {
+            return component;
+        }
     }
     return NULL;
 }
-
-
diff --git a/opal/mca/allocator/base/base.h b/opal/mca/allocator/base/base.h
index 4a75ffd824f..f77f14ae15a 100644
--- a/opal/mca/allocator/base/base.h
+++ b/opal/mca/allocator/base/base.h
@@ -26,20 +26,20 @@
 #include "opal/mca/base/mca_base_framework.h"
 
 #include "opal/class/opal_list.h"
-#include "opal/mca/mca.h"
 #include "opal/mca/allocator/allocator.h"
+#include "opal/mca/mca.h"
 
 BEGIN_C_DECLS
 /**
  * Structure which describes a selected module.
  */
 struct mca_allocator_base_selected_module_t {
-  opal_list_item_t super;
-  /**< Makes this an object of type opal_list_item */
-  mca_allocator_base_component_t *allocator_component;
-  /**< Info about the module */
-  mca_allocator_base_module_t *allocator_module;
-  /**< The function pointers for all the module's functions. */
+    opal_list_item_t super;
+    /**< Makes this an object of type opal_list_item */
+    mca_allocator_base_component_t *allocator_component;
+    /**< Info about the module */
+    mca_allocator_base_module_t *allocator_module;
+    /**< The function pointers for all the module's functions. */
 };
 /**
  * Convenience typedef.
@@ -51,9 +51,7 @@ typedef struct mca_allocator_base_selected_module_t mca_allocator_base_selected_
  */
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(mca_allocator_base_selected_module_t);
 
-
-OPAL_DECLSPEC mca_allocator_base_component_t* mca_allocator_component_lookup(const char* name);
-
+OPAL_DECLSPEC mca_allocator_base_component_t *mca_allocator_component_lookup(const char *name);
 
 /*
  * Globals
diff --git a/opal/mca/allocator/basic/Makefile.am b/opal/mca/allocator/basic/Makefile.am
index b385131f194..053d834d6b1 100644
--- a/opal/mca/allocator/basic/Makefile.am
+++ b/opal/mca/allocator/basic/Makefile.am
@@ -38,7 +38,7 @@ mcacomponentdir = $(opallibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_allocator_basic_la_SOURCES = $(sources)
 mca_allocator_basic_la_LDFLAGS = -module -avoid-version
-mca_allocator_basic_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+mca_allocator_basic_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 
 noinst_LTLIBRARIES = $(component_noinst)
 libmca_allocator_basic_la_SOURCES = $(sources)
diff --git a/opal/mca/allocator/basic/allocator_basic.c b/opal/mca/allocator/basic/allocator_basic.c
index 53c40ad85f1..ddb4d8193d7 100644
--- a/opal/mca/allocator/basic/allocator_basic.c
+++ b/opal/mca/allocator/basic/allocator_basic.c
@@ -26,62 +26,45 @@
 #include "allocator_basic.h"
 #include "opal/constants.h"
 
-
-
 mca_allocator_base_component_t mca_allocator_basic_component = {
 
-  /* First, the mca_base_module_t struct containing meta information
-     about the module itself */
-
-  {
-    MCA_ALLOCATOR_BASE_VERSION_2_0_0,
+    /* First, the mca_base_module_t struct containing meta information
+       about the module itself */
 
-    "basic", /* MCA module name */
-    OPAL_MAJOR_VERSION,
-    OPAL_MINOR_VERSION,
-    OPAL_RELEASE_VERSION,
-    mca_allocator_basic_component_open,  /* module open */
-    mca_allocator_basic_component_close  /* module close */
-  },
-  {
-      /* The component is checkpoint ready */
-      MCA_BASE_METADATA_PARAM_CHECKPOINT
-  },
-  mca_allocator_basic_component_init
-};
+    {
+        MCA_ALLOCATOR_BASE_VERSION_2_0_0,
 
+        "basic", /* MCA module name */
+        OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION, OPAL_RELEASE_VERSION,
+        mca_allocator_basic_component_open, /* module open */
+        mca_allocator_basic_component_close /* module close */
+    },
+    {/* The component is checkpoint ready */
+     MCA_BASE_METADATA_PARAM_CHECKPOINT},
+    mca_allocator_basic_component_init};
 
-OBJ_CLASS_INSTANCE(
-    mca_allocator_basic_segment_t,
-    opal_free_list_item_t,
-    NULL,
-    NULL);
-
+OBJ_CLASS_INSTANCE(mca_allocator_basic_segment_t, opal_free_list_item_t, NULL, NULL);
 
 int mca_allocator_basic_component_open(void)
 {
     return OPAL_SUCCESS;
 }
 
-
 int mca_allocator_basic_component_close(void)
 {
     return OPAL_SUCCESS;
 }
 
-
 /**
-  *
-  */
-
-mca_allocator_base_module_t* mca_allocator_basic_component_init(
-    bool enable_mpi_threads,
-    mca_allocator_base_component_segment_alloc_fn_t segment_alloc,
-    mca_allocator_base_component_segment_free_fn_t segment_free,
-    void *context)
+ *
+ */
+
+mca_allocator_base_module_t *mca_allocator_basic_component_init(
+    bool enable_mpi_threads, mca_allocator_base_component_segment_alloc_fn_t segment_alloc,
+    mca_allocator_base_component_segment_free_fn_t segment_free, void *context)
 {
-    mca_allocator_basic_module_t *module = (mca_allocator_basic_module_t *)
-                                            malloc(sizeof(mca_allocator_basic_module_t));
+    mca_allocator_basic_module_t *module = (mca_allocator_basic_module_t *) malloc(
+        sizeof(mca_allocator_basic_module_t));
     if (NULL == module) {
         return NULL;
     }
@@ -98,76 +81,68 @@ mca_allocator_base_module_t* mca_allocator_basic_component_init(
     OBJ_CONSTRUCT(&module->seg_lock, opal_mutex_t);
     OBJ_CONSTRUCT(&module->seg_descriptors, opal_free_list_t);
 
-    opal_free_list_init (&module->seg_descriptors,
-        sizeof(mca_allocator_basic_segment_t),
-        opal_cache_line_size,
-        OBJ_CLASS(mca_allocator_basic_segment_t),
-        0,opal_cache_line_size,
-        0,  /* initial size */
-        -1, /* maximum size */
-        16, /* increment to grow by */
-        NULL, 0, NULL, NULL, NULL);
+    opal_free_list_init(&module->seg_descriptors, sizeof(mca_allocator_basic_segment_t),
+                        opal_cache_line_size, OBJ_CLASS(mca_allocator_basic_segment_t), 0,
+                        opal_cache_line_size, 0, /* initial size */
+                        -1,                      /* maximum size */
+                        16,                      /* increment to grow by */
+                        NULL, 0, NULL, NULL, NULL);
 
     return &module->super;
 }
 
 /**
-  * Combine adjacent segments together.
+ * Combine adjacent segments together.
  */
 
-static void mca_allocator_basic_combine_prev(
-    mca_allocator_basic_module_t* module,
-    mca_allocator_basic_segment_t* seg)
+static void mca_allocator_basic_combine_prev(mca_allocator_basic_module_t *module,
+                                             mca_allocator_basic_segment_t *seg)
 {
-    opal_list_item_t* item = opal_list_get_prev(seg);
-    if(item != opal_list_get_begin(&module->seg_list)) {
-        mca_allocator_basic_segment_t *prev = (mca_allocator_basic_segment_t*)item;
-        if(prev->seg_addr + prev->seg_size == seg->seg_addr) {
+    opal_list_item_t *item = opal_list_get_prev(seg);
+    if (item != opal_list_get_begin(&module->seg_list)) {
+        mca_allocator_basic_segment_t *prev = (mca_allocator_basic_segment_t *) item;
+        if (prev->seg_addr + prev->seg_size == seg->seg_addr) {
             prev->seg_size += seg->seg_size;
             opal_list_remove_item(&module->seg_list, &seg->seg_item.super);
-            opal_free_list_return (&module->seg_descriptors, &seg->seg_item);
+            opal_free_list_return(&module->seg_descriptors, &seg->seg_item);
             return;
         }
     }
 }
 
-static void mca_allocator_basic_combine_next(
-    mca_allocator_basic_module_t* module,
-    mca_allocator_basic_segment_t* seg)
+static void mca_allocator_basic_combine_next(mca_allocator_basic_module_t *module,
+                                             mca_allocator_basic_segment_t *seg)
 {
     opal_list_item_t *item = opal_list_get_next(seg);
-    if(item != opal_list_get_end(&module->seg_list)) {
-        mca_allocator_basic_segment_t *next = (mca_allocator_basic_segment_t*)item;
-        if(seg->seg_addr + seg->seg_size == next->seg_addr) {
+    if (item != opal_list_get_end(&module->seg_list)) {
+        mca_allocator_basic_segment_t *next = (mca_allocator_basic_segment_t *) item;
+        if (seg->seg_addr + seg->seg_size == next->seg_addr) {
             next->seg_addr = seg->seg_addr;
             next->seg_size += seg->seg_size;
             opal_list_remove_item(&module->seg_list, &seg->seg_item.super);
-            opal_free_list_return (&module->seg_descriptors, &seg->seg_item);
+            opal_free_list_return(&module->seg_descriptors, &seg->seg_item);
             return;
         }
     }
 }
 
 /**
-  * Accepts a request for memory in a specific region defined by the
-  * mca_allocator_basic_options_t struct and returns a pointer to memory in that
-  * region or NULL if there was an error
-  *
-  * @param mem A pointer to the appropriate struct for the area of memory.
-  * @param size The size of the requested area of memory
-  *
-  * @retval Pointer to the area of memory if the allocation was successful
-  * @retval NULL if the allocation was unsuccessful
-  */
-
-void *mca_allocator_basic_alloc(
-    mca_allocator_base_module_t * base,
-    size_t size,
-    size_t align)
+ * Accepts a request for memory in a specific region defined by the
+ * mca_allocator_basic_options_t struct and returns a pointer to memory in that
+ * region or NULL if there was an error
+ *
+ * @param mem A pointer to the appropriate struct for the area of memory.
+ * @param size The size of the requested area of memory
+ *
+ * @retval Pointer to the area of memory if the allocation was successful
+ * @retval NULL if the allocation was unsuccessful
+ */
+
+void *mca_allocator_basic_alloc(mca_allocator_base_module_t *base, size_t size, size_t align)
 {
-    mca_allocator_basic_module_t* module = (mca_allocator_basic_module_t*)base;
-    mca_allocator_basic_segment_t* seg;
-    unsigned char* addr;
+    mca_allocator_basic_module_t *module = (mca_allocator_basic_module_t *) base;
+    mca_allocator_basic_segment_t *seg;
+    unsigned char *addr;
     size_t allocated_size;
     OPAL_THREAD_LOCK(&module->seg_lock);
 
@@ -176,109 +151,106 @@ void *mca_allocator_basic_alloc(
     /* normalize size so we don't end up with seg_addr on an odd boundary */
     size += sizeof(size_t) - (size & (sizeof(size_t) - 1));
     /* search the list for a segment of the required size */
-    OPAL_LIST_FOREACH(seg, &module->seg_list, mca_allocator_basic_segment_t) {
+    OPAL_LIST_FOREACH (seg, &module->seg_list, mca_allocator_basic_segment_t) {
         /* split the segment */
-        if(seg->seg_size > size) {
+        if (seg->seg_size > size) {
             addr = seg->seg_addr;
             seg->seg_addr += size;
             seg->seg_size -= size;
             OPAL_THREAD_UNLOCK(&module->seg_lock);
-            *(size_t*)addr = size;
-            return addr+sizeof(size_t);
+            *(size_t *) addr = size;
+            return addr + sizeof(size_t);
         } else if (seg->seg_size == size) {
             addr = seg->seg_addr;
             opal_list_remove_item(&module->seg_list, (opal_list_item_t *) seg);
-            opal_free_list_return (&module->seg_descriptors, (opal_free_list_item_t *) seg);
+            opal_free_list_return(&module->seg_descriptors, (opal_free_list_item_t *) seg);
             OPAL_THREAD_UNLOCK(&module->seg_lock);
-            *(size_t*)addr = size;
-            return addr+sizeof(size_t);
+            *(size_t *) addr = size;
+            return addr + sizeof(size_t);
         }
     }
 
     /* request additional block */
     allocated_size = size;
-    if(NULL == (addr = (unsigned char *)module->seg_alloc(module->super.alc_context, &allocated_size))) {
+    if (NULL
+        == (addr = (unsigned char *) module->seg_alloc(module->super.alc_context,
+                                                       &allocated_size))) {
         OPAL_THREAD_UNLOCK(&module->seg_lock);
         return NULL;
     }
 
     /* create a segment for any extra allocation */
-    if(allocated_size > size) {
-        seg = (mca_allocator_basic_segment_t *) opal_free_list_get (&module->seg_descriptors);
+    if (allocated_size > size) {
+        seg = (mca_allocator_basic_segment_t *) opal_free_list_get(&module->seg_descriptors);
         if (NULL == seg) {
             OPAL_THREAD_UNLOCK(&module->seg_lock);
             return NULL;
         }
         seg->seg_addr = addr + size;
         seg->seg_size = allocated_size - size;
-        opal_list_append (&module->seg_list, (opal_list_item_t *) seg);
+        opal_list_append(&module->seg_list, (opal_list_item_t *) seg);
     }
 
-    *(size_t*)addr = size;
+    *(size_t *) addr = size;
     OPAL_THREAD_UNLOCK(&module->seg_lock);
-    return addr+sizeof(size_t);
+    return addr + sizeof(size_t);
 }
 
-
 /**
-  * Attempts to resize the passed region of memory into a larger or a smaller
-  * region. If it is unsuccessful, it will return NULL and the passed area of
-  * memory will be untouched.
-  *
-  * @param mem A pointer to the appropriate struct for the area of
-  * memory.
-  * @param size The size of the requested area of memory
-  * @param ptr A pointer to the region of memory to be resized
-  *
-  * @retval Pointer to the area of memory if the reallocation was successful
-  * @retval NULL if the allocation was unsuccessful
-  *
-  */
-
-void * mca_allocator_basic_realloc(
-    mca_allocator_base_module_t * base,
-    void * ptr,
-    size_t size)
+ * Attempts to resize the passed region of memory into a larger or a smaller
+ * region. If it is unsuccessful, it will return NULL and the passed area of
+ * memory will be untouched.
+ *
+ * @param mem A pointer to the appropriate struct for the area of
+ * memory.
+ * @param size The size of the requested area of memory
+ * @param ptr A pointer to the region of memory to be resized
+ *
+ * @retval Pointer to the area of memory if the reallocation was successful
+ * @retval NULL if the allocation was unsuccessful
+ *
+ */
+
+void *mca_allocator_basic_realloc(mca_allocator_base_module_t *base, void *ptr, size_t size)
 {
-    unsigned char* addr = ((unsigned char*)ptr) - sizeof(size_t);
-    size_t alloc_size = *(size_t*)addr;
-    if(size <= alloc_size)
+    unsigned char *addr = ((unsigned char *) ptr) - sizeof(size_t);
+    size_t alloc_size = *(size_t *) addr;
+    if (size <= alloc_size) {
         return ptr;
-    addr = (unsigned char *)mca_allocator_basic_alloc(base, size, 0);
-    if(addr == NULL)
+    }
+    addr = (unsigned char *) mca_allocator_basic_alloc(base, size, 0);
+    if (addr == NULL) {
         return addr;
-    memcpy(addr,ptr,alloc_size);
-    mca_allocator_basic_free(base,ptr);
+    }
+    memcpy(addr, ptr, alloc_size);
+    mca_allocator_basic_free(base, ptr);
     return addr;
 }
 
-
 /**
-  * Frees the passed region of memory
-  *
-  * @param mem A pointer to the appropriate struct for the area of
-  * memory.
-  * @param ptr A pointer to the region of memory to be freed
-  *
-  * @retval None
-  *
-  */
-void mca_allocator_basic_free(
-    mca_allocator_base_module_t * base,
-    void * ptr)
+ * Frees the passed region of memory
+ *
+ * @param mem A pointer to the appropriate struct for the area of
+ * memory.
+ * @param ptr A pointer to the region of memory to be freed
+ *
+ * @retval None
+ *
+ */
+void mca_allocator_basic_free(mca_allocator_base_module_t *base, void *ptr)
 {
-    mca_allocator_basic_module_t* module = (mca_allocator_basic_module_t*)base;
-    mca_allocator_basic_segment_t* seg;
-    unsigned char* addr = (unsigned char*)ptr - sizeof(size_t);
-    size_t size = *(size_t*)addr;
+    mca_allocator_basic_module_t *module = (mca_allocator_basic_module_t *) base;
+    mca_allocator_basic_segment_t *seg;
+    unsigned char *addr = (unsigned char *) ptr - sizeof(size_t);
+    size_t size = *(size_t *) addr;
     OPAL_THREAD_LOCK(&module->seg_lock);
 
     /* maintain the free list in sorted order by address */
-    OPAL_LIST_FOREACH(seg, &module->seg_list, mca_allocator_basic_segment_t) {
+    OPAL_LIST_FOREACH (seg, &module->seg_list, mca_allocator_basic_segment_t) {
         if (seg->seg_addr < addr) {
 
             /* can we grow the current entry */
-            if(seg->seg_addr + seg->seg_size == addr) {
+            if (seg->seg_addr + seg->seg_size == addr) {
                 seg->seg_size += size;
                 mca_allocator_basic_combine_next(module, seg);
                 OPAL_THREAD_UNLOCK(&module->seg_lock);
@@ -286,28 +258,29 @@ void mca_allocator_basic_free(
             }
             /* otherwise continue to check next larger entry */
 
-        } else  {
+        } else {
 
             /* can this be combined with current entry */
-            if(addr + size == seg->seg_addr) {
+            if (addr + size == seg->seg_addr) {
                 seg->seg_addr = addr;
                 seg->seg_size += size;
                 mca_allocator_basic_combine_prev(module, seg);
                 OPAL_THREAD_UNLOCK(&module->seg_lock);
                 return;
 
-            /* insert before larger entry */
+                /* insert before larger entry */
             } else {
-                mca_allocator_basic_segment_t* new_seg;
-                new_seg = (mca_allocator_basic_segment_t *)
-                    opal_free_list_get (&module->seg_descriptors);
-                if(NULL == new_seg) {
+                mca_allocator_basic_segment_t *new_seg;
+                new_seg = (mca_allocator_basic_segment_t *) opal_free_list_get(
+                    &module->seg_descriptors);
+                if (NULL == new_seg) {
                     OPAL_THREAD_UNLOCK(&module->seg_lock);
                     return;
                 }
                 new_seg->seg_addr = addr;
                 new_seg->seg_size = size;
-                opal_list_insert_pos(&module->seg_list, &seg->seg_item.super, (opal_list_item_t *) new_seg);
+                opal_list_insert_pos(&module->seg_list, &seg->seg_item.super,
+                                     (opal_list_item_t *) new_seg);
                 OPAL_THREAD_UNLOCK(&module->seg_lock);
                 return;
             }
@@ -315,8 +288,8 @@ void mca_allocator_basic_free(
     }
 
     /* append to the end of the list */
-    seg = (mca_allocator_basic_segment_t *) opal_free_list_get (&module->seg_descriptors);
-    if(NULL == seg) {
+    seg = (mca_allocator_basic_segment_t *) opal_free_list_get(&module->seg_descriptors);
+    if (NULL == seg) {
         OPAL_THREAD_UNLOCK(&module->seg_lock);
         return;
     }
@@ -326,40 +299,39 @@ void mca_allocator_basic_free(
     OPAL_THREAD_UNLOCK(&module->seg_lock);
 }
 
-
 /**
-  * Frees all the memory from all the basics back to the system. Note that
-  * this function only frees memory that was previously freed with
-  * mca_allocator_basic_free().
-  *
-  * @param mem A pointer to the appropriate struct for the area of
-  * memory.
-  *
-  * @retval None
-  *
-  */
-
-int mca_allocator_basic_compact(mca_allocator_base_module_t * mem)
+ * Frees all the memory from all the basics back to the system. Note that
+ * this function only frees memory that was previously freed with
+ * mca_allocator_basic_free().
+ *
+ * @param mem A pointer to the appropriate struct for the area of
+ * memory.
+ *
+ * @retval None
+ *
+ */
+
+int mca_allocator_basic_compact(mca_allocator_base_module_t *mem)
 {
     return OPAL_SUCCESS;
 }
 
 /**
-  * Cleanup all resources held by this allocator.
-  *
-  * @param mem A pointer to the appropriate struct for the area of
-  * memory.
-  *
-  * @retval None
-  *
-  */
-
-int mca_allocator_basic_finalize(mca_allocator_base_module_t * base)
+ * Cleanup all resources held by this allocator.
+ *
+ * @param mem A pointer to the appropriate struct for the area of
+ * memory.
+ *
+ * @retval None
+ *
+ */
+
+int mca_allocator_basic_finalize(mca_allocator_base_module_t *base)
 {
-    mca_allocator_basic_module_t* module = (mca_allocator_basic_module_t*)base;
+    mca_allocator_basic_module_t *module = (mca_allocator_basic_module_t *) base;
     // clear the list as we don't own these bits
     while (NULL != opal_list_remove_first(&module->seg_list)) {
-      continue;
+        continue;
     }
     OBJ_DESTRUCT(&module->seg_list);
     OBJ_DESTRUCT(&module->seg_lock);
@@ -367,5 +339,3 @@ int mca_allocator_basic_finalize(mca_allocator_base_module_t * base)
     free(module);
     return OPAL_SUCCESS;
 }
-
-
diff --git a/opal/mca/allocator/basic/allocator_basic.h b/opal/mca/allocator/basic/allocator_basic.h
index 4766a994c8d..feb041b0803 100644
--- a/opal/mca/allocator/basic/allocator_basic.h
+++ b/opal/mca/allocator/basic/allocator_basic.h
@@ -27,12 +27,11 @@
 #define ALLOCATOR_BASIC_H
 
 #include "opal_config.h"
-#include <stdlib.h>
-#include <string.h>
-#include "opal/mca/threads/mutex.h"
 #include "opal/class/opal_free_list.h"
 #include "opal/mca/allocator/allocator.h"
-
+#include "opal/mca/threads/mutex.h"
+#include <stdlib.h>
+#include <string.h>
 
 /*
  * Free list of allocated segments
@@ -40,12 +39,11 @@
 
 struct mca_allocator_basic_segment_t {
     opal_free_list_item_t seg_item;
-    unsigned char* seg_addr;
+    unsigned char *seg_addr;
     size_t seg_size;
 };
 typedef struct mca_allocator_basic_segment_t mca_allocator_basic_segment_t;
 
-
 /*
  * Basic allocator module
  */
@@ -60,7 +58,6 @@ struct mca_allocator_basic_module_t {
 };
 typedef struct mca_allocator_basic_module_t mca_allocator_basic_module_t;
 
-
 BEGIN_C_DECLS
 
 /*
@@ -71,99 +68,79 @@ int mca_allocator_basic_component_open(void);
 int mca_allocator_basic_component_close(void);
 
 /**
-  * The function used to initialize the component.
-  */
-mca_allocator_base_module_t* mca_allocator_basic_component_init(
-    bool enable_mpi_threads,
-    mca_allocator_base_component_segment_alloc_fn_t segment_alloc,
-    mca_allocator_base_component_segment_free_fn_t segment_free,
-    void *ctx
-);
+ * The function used to initialize the component.
+ */
+mca_allocator_base_module_t *mca_allocator_basic_component_init(
+    bool enable_mpi_threads, mca_allocator_base_component_segment_alloc_fn_t segment_alloc,
+    mca_allocator_base_component_segment_free_fn_t segment_free, void *ctx);
 
 /**
-   * Accepts a request for memory in a specific region defined by the
-   * mca_allocator_basic_options_t struct and returns a pointer to memory in that
-   * region or NULL if there was an error
-   *
-   * @param mem A pointer to the appropriate struct for the area of memory.
-   * @param size The size of the requested area of memory
-   *
-   * @retval Pointer to the area of memory if the allocation was successful
-   * @retval NULL if the allocation was unsuccessful
-   */
-    void * mca_allocator_basic_alloc(
-        mca_allocator_base_module_t * mem,
-        size_t size,
-        size_t align);
+ * Accepts a request for memory in a specific region defined by the
+ * mca_allocator_basic_options_t struct and returns a pointer to memory in that
+ * region or NULL if there was an error
+ *
+ * @param mem A pointer to the appropriate struct for the area of memory.
+ * @param size The size of the requested area of memory
+ *
+ * @retval Pointer to the area of memory if the allocation was successful
+ * @retval NULL if the allocation was unsuccessful
+ */
+void *mca_allocator_basic_alloc(mca_allocator_base_module_t *mem, size_t size, size_t align);
 
 /**
-   * Attempts to resize the passed region of memory into a larger or a smaller
-   * region. If it is unsuccessful, it will return NULL and the passed area of
-   * memory will be untouched.
-   *
-   * @param mem A pointer to the appropriate struct for the area of
-   * memory.
-   * @param size The size of the requested area of memory
-   * @param ptr A pointer to the region of memory to be resized
-   *
-   * @retval Pointer to the area of memory if the reallocation was successful
-   * @retval NULL if the allocation was unsuccessful
-   *
-   */
-    void * mca_allocator_basic_realloc(
-        mca_allocator_base_module_t * mem,
-        void * ptr,
-        size_t size);
+ * Attempts to resize the passed region of memory into a larger or a smaller
+ * region. If it is unsuccessful, it will return NULL and the passed area of
+ * memory will be untouched.
+ *
+ * @param mem A pointer to the appropriate struct for the area of
+ * memory.
+ * @param size The size of the requested area of memory
+ * @param ptr A pointer to the region of memory to be resized
+ *
+ * @retval Pointer to the area of memory if the reallocation was successful
+ * @retval NULL if the allocation was unsuccessful
+ *
+ */
+void *mca_allocator_basic_realloc(mca_allocator_base_module_t *mem, void *ptr, size_t size);
 
 /**
-   * Frees the passed region of memory
-   *
-   * @param mem A pointer to the appropriate struct for the area of
-   * memory.
-   * @param ptr A pointer to the region of memory to be freed
-   *
-   * @retval None
-   *
-   */
-    void mca_allocator_basic_free(
-        mca_allocator_base_module_t * mem,
-        void * ptr);
+ * Frees the passed region of memory
+ *
+ * @param mem A pointer to the appropriate struct for the area of
+ * memory.
+ * @param ptr A pointer to the region of memory to be freed
+ *
+ * @retval None
+ *
+ */
+void mca_allocator_basic_free(mca_allocator_base_module_t *mem, void *ptr);
 
 /**
-   * Frees all the memory from all the basics back to the system. Note that
-   * this function only frees memory that was previously freed with
-   * mca_allocator_basic_free().
-   *
-   * @param mem A pointer to the appropriate struct for the area of
-   * memory.
-   *
-   * @retval None
-   *
-   */
-    int mca_allocator_basic_compact(
-        mca_allocator_base_module_t * mem);
+ * Frees all the memory from all the basics back to the system. Note that
+ * this function only frees memory that was previously freed with
+ * mca_allocator_basic_free().
+ *
+ * @param mem A pointer to the appropriate struct for the area of
+ * memory.
+ *
+ * @retval None
+ *
+ */
+int mca_allocator_basic_compact(mca_allocator_base_module_t *mem);
 
 /**
-   * Cleanup all resources held by this allocator.
-   *
-   * @param mem A pointer to the appropriate struct for the area of
-   * memory.
-   *
-   * @retval None
-   *
-   */
-    int mca_allocator_basic_finalize(
-        mca_allocator_base_module_t * mem);
+ * Cleanup all resources held by this allocator.
+ *
+ * @param mem A pointer to the appropriate struct for the area of
+ * memory.
+ *
+ * @retval None
+ *
+ */
+int mca_allocator_basic_finalize(mca_allocator_base_module_t *mem);
 
 OPAL_DECLSPEC extern mca_allocator_base_component_t mca_allocator_basic_component;
 
 END_C_DECLS
 
 #endif /* ALLOCATOR_BUCKET_ALLOC_H */
-
-
-
-
-
-
-
diff --git a/opal/mca/allocator/bucket/Makefile.am b/opal/mca/allocator/bucket/Makefile.am
index ba50d9398de..f868f7ed88f 100644
--- a/opal/mca/allocator/bucket/Makefile.am
+++ b/opal/mca/allocator/bucket/Makefile.am
@@ -11,6 +11,7 @@
 #                         All rights reserved.
 # Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -18,6 +19,8 @@
 # $HEADER$
 #
 
+EXTRA_DIST = help-mca-allocator-bucket.txt
+
 sources = \
         allocator_bucket.c \
         allocator_bucket_alloc.c \
@@ -39,9 +42,8 @@ mcacomponentdir = $(opallibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_allocator_bucket_la_SOURCES = $(sources)
 mca_allocator_bucket_la_LDFLAGS = -module -avoid-version
-mca_allocator_bucket_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+mca_allocator_bucket_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 
 noinst_LTLIBRARIES = $(component_noinst)
 libmca_allocator_bucket_la_SOURCES = $(sources)
 libmca_allocator_bucket_la_LDFLAGS = -module -avoid-version
-
diff --git a/opal/mca/allocator/bucket/allocator_bucket.c b/opal/mca/allocator/bucket/allocator_bucket.c
index e57b1648cb7..3d9a2617af6 100644
--- a/opal/mca/allocator/bucket/allocator_bucket.c
+++ b/opal/mca/allocator/bucket/allocator_bucket.c
@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2014      Los Alamos National Security, LLC. All rights
- *                         reseved.
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -21,75 +21,67 @@
  */
 
 #include "opal_config.h"
-#include "opal/mca/allocator/allocator.h"
 #include "opal/constants.h"
+#include "opal/mca/allocator/allocator.h"
 #include "opal/mca/allocator/bucket/allocator_bucket_alloc.h"
 #include "opal/mca/base/mca_base_var.h"
 
-struct mca_allocator_base_module_t* mca_allocator_bucket_module_init(
-    bool enable_mpi_threads,
-    mca_allocator_base_component_segment_alloc_fn_t segment_alloc,
-    mca_allocator_base_component_segment_free_fn_t segment_free,
-    void *context);
+struct mca_allocator_base_module_t *mca_allocator_bucket_module_init(
+    bool enable_mpi_threads, mca_allocator_base_component_segment_alloc_fn_t segment_alloc,
+    mca_allocator_base_component_segment_free_fn_t segment_free, void *context);
 
 int mca_allocator_bucket_module_open(void);
 
 int mca_allocator_bucket_module_close(void);
 
-void * mca_allocator_bucket_alloc_wrapper(
-    struct mca_allocator_base_module_t* allocator,
-    size_t size, size_t align);
+void *mca_allocator_bucket_alloc_wrapper(struct mca_allocator_base_module_t *allocator, size_t size,
+                                         size_t align);
 
 static int mca_allocator_num_buckets;
 
-
-
-int mca_allocator_bucket_finalize(struct mca_allocator_base_module_t* allocator)
+int mca_allocator_bucket_finalize(struct mca_allocator_base_module_t *allocator)
 {
     mca_allocator_bucket_t *bucket = (mca_allocator_bucket_t *) allocator;
 
     mca_allocator_bucket_cleanup(allocator);
 
-    for (int i = 0 ; i < bucket->num_buckets ; ++i) {
+    for (int i = 0; i < bucket->num_buckets; ++i) {
         OBJ_DESTRUCT(&bucket->buckets[i].lock);
     }
 
-    free (bucket->buckets);
+    free(bucket->buckets);
     free(allocator);
 
-    return(OPAL_SUCCESS);
+    return (OPAL_SUCCESS);
 }
 
-struct mca_allocator_base_module_t* mca_allocator_bucket_module_init(
-    bool enable_mpi_threads,
-    mca_allocator_base_component_segment_alloc_fn_t segment_alloc,
-    mca_allocator_base_component_segment_free_fn_t segment_free,
-    void *context)
+struct mca_allocator_base_module_t *mca_allocator_bucket_module_init(
+    bool enable_mpi_threads, mca_allocator_base_component_segment_alloc_fn_t segment_alloc,
+    mca_allocator_base_component_segment_free_fn_t segment_free, void *context)
 {
     size_t alloc_size = sizeof(mca_allocator_bucket_t);
-    mca_allocator_bucket_t * retval;
-    mca_allocator_bucket_t * allocator = (mca_allocator_bucket_t *) malloc(alloc_size);
-    if(NULL == allocator) {
+    mca_allocator_bucket_t *retval;
+    mca_allocator_bucket_t *allocator = (mca_allocator_bucket_t *) malloc(alloc_size);
+    if (NULL == allocator) {
         return NULL;
     }
     retval = mca_allocator_bucket_init((mca_allocator_base_module_t *) allocator,
-        mca_allocator_num_buckets,
-        segment_alloc,
-        segment_free);
-    if(NULL == retval) {
+                                       mca_allocator_num_buckets, segment_alloc, segment_free);
+    if (NULL == retval) {
         free(allocator);
         return NULL;
     }
-    allocator->super.alc_alloc =  mca_allocator_bucket_alloc_wrapper;
+    allocator->super.alc_alloc = mca_allocator_bucket_alloc_wrapper;
     allocator->super.alc_realloc = mca_allocator_bucket_realloc;
-    allocator->super.alc_free =  mca_allocator_bucket_free;
+    allocator->super.alc_free = mca_allocator_bucket_free;
     allocator->super.alc_compact = mca_allocator_bucket_cleanup;
     allocator->super.alc_finalize = mca_allocator_bucket_finalize;
     allocator->super.alc_context = context;
     return (mca_allocator_base_module_t *) allocator;
 }
 
-static int mca_allocator_bucket_module_register(void) {
+static int mca_allocator_bucket_module_register(void)
+{
     mca_allocator_num_buckets = 30;
     (void) mca_base_component_var_register(&mca_allocator_bucket_component.allocator_version,
                                            "num_buckets", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0,
@@ -98,47 +90,37 @@ static int mca_allocator_bucket_module_register(void) {
     return OPAL_SUCCESS;
 }
 
-int mca_allocator_bucket_module_open(void) {
+int mca_allocator_bucket_module_open(void)
+{
     return OPAL_SUCCESS;
 }
 
-int mca_allocator_bucket_module_close(void) {
+int mca_allocator_bucket_module_close(void)
+{
     return OPAL_SUCCESS;
 }
 
-void * mca_allocator_bucket_alloc_wrapper(
-    struct mca_allocator_base_module_t* allocator,
-    size_t size,
-    size_t align)
+void *mca_allocator_bucket_alloc_wrapper(struct mca_allocator_base_module_t *allocator, size_t size,
+                                         size_t align)
 {
-    if(0 == align){
+    if (0 == align) {
         return mca_allocator_bucket_alloc(allocator, size);
     }
     return mca_allocator_bucket_alloc_align(allocator, size, align);
 }
 
-
 mca_allocator_base_component_t mca_allocator_bucket_component = {
 
-  /* First, the mca_base_module_t struct containing meta information
-     about the module itself */
-
-  {
-    MCA_ALLOCATOR_BASE_VERSION_2_0_0,
-
-    "bucket", /* MCA module name */
-    OPAL_MAJOR_VERSION,
-    OPAL_MINOR_VERSION,
-    OPAL_RELEASE_VERSION,
-    mca_allocator_bucket_module_open,  /* module open */
-    mca_allocator_bucket_module_close, /* module close */
-    NULL,
-    mca_allocator_bucket_module_register
-  },
-  {
-      /* The component is checkpoint ready */
-      MCA_BASE_METADATA_PARAM_CHECKPOINT
-  },
-  mca_allocator_bucket_module_init
-};
+    /* First, the mca_base_module_t struct containing meta information
+       about the module itself */
+
+    {MCA_ALLOCATOR_BASE_VERSION_2_0_0,
 
+     "bucket", /* MCA module name */
+     OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION, OPAL_RELEASE_VERSION,
+     mca_allocator_bucket_module_open,  /* module open */
+     mca_allocator_bucket_module_close, /* module close */
+     NULL, mca_allocator_bucket_module_register},
+    {/* The component is checkpoint ready */
+     MCA_BASE_METADATA_PARAM_CHECKPOINT},
+    mca_allocator_bucket_module_init};
diff --git a/opal/mca/allocator/bucket/allocator_bucket_alloc.c b/opal/mca/allocator/bucket/allocator_bucket_alloc.c
index be3db944d06..5edf998c686 100644
--- a/opal/mca/allocator/bucket/allocator_bucket_alloc.c
+++ b/opal/mca/allocator/bucket/allocator_bucket_alloc.c
@@ -18,43 +18,49 @@
  */
 
 #include "opal_config.h"
-#include "opal/constants.h"
 #include "opal/mca/allocator/bucket/allocator_bucket_alloc.h"
+#include "opal/constants.h"
+#include "opal/util/show_help.h"
+
 /**
-  * The define controls the size in bytes of the 1st bucket and hence every one
-  * afterwards.
-  */
+ * The define controls the size in bytes of the 1st bucket and hence every one
+ * afterwards.
+ */
 #define MCA_ALLOCATOR_BUCKET_1_SIZE 8
 /**
-  * This is the number of left bit shifts from 1 needed to get to the number of
-  * bytes in the initial memory buckets
-  */
+ * This is the number of left bit shifts from 1 needed to get to the number of
+ * bytes in the initial memory buckets
+ */
 #define MCA_ALLOCATOR_BUCKET_1_BITSHIFTS 3
 
- /*
-   * Initializes the mca_allocator_bucket_options_t data structure for the passed
-   * parameters.
-   */
-mca_allocator_bucket_t * mca_allocator_bucket_init(
-    mca_allocator_base_module_t * mem,
-    int num_buckets,
-    mca_allocator_base_component_segment_alloc_fn_t get_mem_funct,
-    mca_allocator_base_component_segment_free_fn_t free_mem_funct)
+static int max_bucket_idx;
+
+/*
+ * Initializes the mca_allocator_bucket_options_t data structure for the passed
+ * parameters.
+ */
+mca_allocator_bucket_t *
+mca_allocator_bucket_init(mca_allocator_base_module_t *mem, int num_buckets,
+                          mca_allocator_base_component_segment_alloc_fn_t get_mem_funct,
+                          mca_allocator_base_component_segment_free_fn_t free_mem_funct)
 {
-    mca_allocator_bucket_t * mem_options = (mca_allocator_bucket_t *) mem;
+    mca_allocator_bucket_t *mem_options = (mca_allocator_bucket_t *) mem;
     int i;
     size_t size;
     /* if a bad value is used for the number of buckets, default to 30 */
-    if(num_buckets <= 0) {
+    if (num_buckets <= 0) {
         num_buckets = 30;
     }
+
+    max_bucket_idx = num_buckets - 1;
+
     /* initialize the array of buckets */
     size = sizeof(mca_allocator_bucket_bucket_t) * num_buckets;
-    mem_options->buckets = (mca_allocator_bucket_bucket_t*) malloc(size);
-    if(NULL == mem_options->buckets) {
-        return(NULL);
+    mem_options->buckets = (mca_allocator_bucket_bucket_t *) malloc(size);
+    if (NULL == mem_options->buckets) {
+        return (NULL);
     }
-    for(i = 0; i < num_buckets; i++) {
+    for (i = 0; i < num_buckets; i++) {
         mem_options->buckets[i].free_chunk = NULL;
         mem_options->buckets[i].segment_head = NULL;
         OBJ_CONSTRUCT(&(mem_options->buckets[i].lock), opal_mutex_t);
@@ -62,39 +68,45 @@ mca_allocator_bucket_t * mca_allocator_bucket_init(
     mem_options->num_buckets = num_buckets;
     mem_options->get_mem_fn = get_mem_funct;
     mem_options->free_mem_fn = free_mem_funct;
-    return(mem_options);
+    return (mem_options);
 }
 
 /*
-   * Accepts a request for memory in a specific region defined by the
-   * mca_allocator_bucket_options_t struct and returns a pointer to memory in that
-   * region or NULL if there was an error
-   *
-   */
-void * mca_allocator_bucket_alloc(mca_allocator_base_module_t * mem,
-                                  size_t size)
+ * Accepts a request for memory in a specific region defined by the
+ * mca_allocator_bucket_options_t struct and returns a pointer to memory in that
+ * region or NULL if there was an error
+ *
+ */
+void *mca_allocator_bucket_alloc(mca_allocator_base_module_t *mem, size_t size)
 {
-    mca_allocator_bucket_t * mem_options = (mca_allocator_bucket_t *) mem;
+    mca_allocator_bucket_t *mem_options = (mca_allocator_bucket_t *) mem;
     /* initialize for the later bit shifts */
     int bucket_num = 0;
     size_t bucket_size = MCA_ALLOCATOR_BUCKET_1_SIZE;
     size_t allocated_size;
-    mca_allocator_bucket_chunk_header_t * chunk;
-    mca_allocator_bucket_chunk_header_t * first_chunk;
-    mca_allocator_bucket_segment_head_t * segment_header;
+    mca_allocator_bucket_chunk_header_t *chunk;
+    mca_allocator_bucket_chunk_header_t *first_chunk;
+    mca_allocator_bucket_segment_head_t *segment_header;
     /* add the size of the header into the amount we need to request */
     size += sizeof(mca_allocator_bucket_chunk_header_t);
 
     /* figure out which bucket it will come from. */
-    while(size > bucket_size) {
+    while (size > bucket_size) {
         bucket_num++;
         bucket_size <<= 1;
     }
 
+    if( bucket_num > max_bucket_idx ) {
+        size_t sz_bucket = MCA_ALLOCATOR_BUCKET_1_SIZE;
+        opal_show_help ("help-mca-allocator-bucket.txt", "buffer too large", 1, size, sz_bucket << max_bucket_idx,
+                        "allocator_bucket_num_buckets", bucket_num + 1);
+        return (NULL);
+    }
+
     /* now that we know what bucket it will come from, we must get the lock */
     OPAL_THREAD_LOCK(&(mem_options->buckets[bucket_num].lock));
     /* see if there is already a free chunk */
-    if(NULL != mem_options->buckets[bucket_num].free_chunk) {
+    if (NULL != mem_options->buckets[bucket_num].free_chunk) {
         chunk = mem_options->buckets[bucket_num].free_chunk;
         mem_options->buckets[bucket_num].free_chunk = chunk->u.next_free;
         chunk->u.bucket = bucket_num;
@@ -102,7 +114,7 @@ void * mca_allocator_bucket_alloc(mca_allocator_base_module_t * mem,
         chunk += 1;
         /*release the lock */
         OPAL_THREAD_UNLOCK(&(mem_options->buckets[bucket_num].lock));
-        return((void *) chunk);
+        return ((void *) chunk);
     }
     /* figure out the size of bucket we need */
     allocated_size = bucket_size;
@@ -111,29 +123,29 @@ void * mca_allocator_bucket_alloc(mca_allocator_base_module_t * mem,
     allocated_size += sizeof(mca_allocator_bucket_segment_head_t);
     /* attempt to get the memory */
     segment_header = (mca_allocator_bucket_segment_head_t *)
-                   mem_options->get_mem_fn(mem_options->super.alc_context, &allocated_size);
-    if(NULL == segment_header) {
+                         mem_options->get_mem_fn(mem_options->super.alc_context, &allocated_size);
+    if (NULL == segment_header) {
         /* release the lock */
         OPAL_THREAD_UNLOCK(&(mem_options->buckets[bucket_num].lock));
-        return(NULL);
+        return (NULL);
     }
     /* if were allocated more memory then we actually need, then we will try to
      * break it up into multiple chunks in the current bucket */
     allocated_size -= (sizeof(mca_allocator_bucket_segment_head_t) + bucket_size);
-    chunk = first_chunk = segment_header->first_chunk =
-                  (mca_allocator_bucket_chunk_header_t *) (segment_header + 1);
+    chunk = first_chunk = segment_header->first_chunk = (mca_allocator_bucket_chunk_header_t
+                                                             *) (segment_header + 1);
     /* add the segment into the segment list */
     segment_header->next_segment = mem_options->buckets[bucket_num].segment_head;
     mem_options->buckets[bucket_num].segment_head = segment_header;
-    if(allocated_size >= bucket_size) {
-        mem_options->buckets[bucket_num].free_chunk =
-                        (mca_allocator_bucket_chunk_header_t *) ((char *) chunk + bucket_size);
-        chunk->next_in_segment = (mca_allocator_bucket_chunk_header_t *)
-                                   ((char *)chunk + bucket_size);
-        while(allocated_size >= bucket_size) {
+    if (allocated_size >= bucket_size) {
+        mem_options->buckets[bucket_num].free_chunk = (mca_allocator_bucket_chunk_header_t
+                                                           *) ((char *) chunk + bucket_size);
+        chunk->next_in_segment = (mca_allocator_bucket_chunk_header_t *) ((char *) chunk
+                                                                          + bucket_size);
+        while (allocated_size >= bucket_size) {
             chunk = (mca_allocator_bucket_chunk_header_t *) ((char *) chunk + bucket_size);
-            chunk->u.next_free = (mca_allocator_bucket_chunk_header_t *)
-                                 ((char *) chunk + bucket_size);
+            chunk->u.next_free = (mca_allocator_bucket_chunk_header_t *) ((char *) chunk
+                                                                          + bucket_size);
             chunk->next_in_segment = chunk->u.next_free;
             allocated_size -= bucket_size;
         }
@@ -145,24 +157,24 @@ void * mca_allocator_bucket_alloc(mca_allocator_base_module_t * mem,
     first_chunk->u.bucket = bucket_num;
     OPAL_THREAD_UNLOCK(&(mem_options->buckets[bucket_num].lock));
     /* return the memory moved past the header */
-    return((void *) (first_chunk + 1));
+    return ((void *) (first_chunk + 1));
 }
 
 /*
-  * allocates an aligned region of memory
-  */
-void * mca_allocator_bucket_alloc_align(mca_allocator_base_module_t * mem,
-                                        size_t size, size_t alignment)
+ * allocates an aligned region of memory
+ */
+void *mca_allocator_bucket_alloc_align(mca_allocator_base_module_t *mem, size_t size,
+                                       size_t alignment)
 {
-    mca_allocator_bucket_t * mem_options = (mca_allocator_bucket_t *) mem;
+    mca_allocator_bucket_t *mem_options = (mca_allocator_bucket_t *) mem;
     int bucket_num = 1;
-    void * ptr;
+    void *ptr;
     size_t aligned_max_size, bucket_size;
     size_t alignment_off, allocated_size;
-    mca_allocator_bucket_chunk_header_t * chunk;
-    mca_allocator_bucket_chunk_header_t * first_chunk;
-    mca_allocator_bucket_segment_head_t * segment_header;
-    char * aligned_memory;
+    mca_allocator_bucket_chunk_header_t *chunk;
+    mca_allocator_bucket_chunk_header_t *first_chunk;
+    mca_allocator_bucket_segment_head_t *segment_header;
+    char *aligned_memory;
 
     /* since we do not have a way to get pre aligned memory, we need to request
      * a chunk then return an aligned spot in it. In the worst case we need
@@ -173,8 +185,8 @@ void * mca_allocator_bucket_alloc_align(mca_allocator_base_module_t * mem,
     allocated_size = aligned_max_size;
     /* get some memory */
     ptr = mem_options->get_mem_fn(mem_options->super.alc_context, &allocated_size);
-    if(NULL == ptr) {
-        return(NULL);
+    if (NULL == ptr) {
+        return (NULL);
     }
     /* the first part of the memory is the segment header */
     segment_header = (mca_allocator_bucket_segment_head_t *) ptr;
@@ -184,15 +196,23 @@ void * mca_allocator_bucket_alloc_align(mca_allocator_base_module_t * mem,
     /* we want to align the memory right after the header, so we go past the header */
     aligned_memory = (char *) (first_chunk + 1);
     /* figure out how much the alignment is off by */
-    alignment_off = ((size_t)  aligned_memory) % alignment;
+    alignment_off = ((size_t) aligned_memory) % alignment;
     aligned_memory += (alignment - alignment_off);
     /* we now have an aligned piece of memory. Now we have to put the chunk
      * header right before the aligned memory                           */
     first_chunk = (mca_allocator_bucket_chunk_header_t *) aligned_memory - 1;
-    while(bucket_size > MCA_ALLOCATOR_BUCKET_1_SIZE) {
+    while (bucket_size > MCA_ALLOCATOR_BUCKET_1_SIZE) {
         bucket_size >>= 1;
         bucket_num++;
     }
+
+    if( bucket_num > max_bucket_idx ) {
+        size_t sz_bucket = MCA_ALLOCATOR_BUCKET_1_SIZE;
+        opal_show_help ("help-mca-allocator-bucket.txt", "aligned buffer too large", 1, allocated_size, sz_bucket << max_bucket_idx,
+                        "allocator_bucket_num_buckets", bucket_num + 1);
+        return (NULL);
+    }
+
     bucket_size = 1;
     bucket_size <<= MCA_ALLOCATOR_BUCKET_1_BITSHIFTS + bucket_num;
 
@@ -205,15 +225,15 @@ void * mca_allocator_bucket_alloc_align(mca_allocator_base_module_t * mem,
     /* add the segment into the segment list */
     segment_header->next_segment = mem_options->buckets[bucket_num].segment_head;
     mem_options->buckets[bucket_num].segment_head = segment_header;
-    if(allocated_size >= bucket_size) {
-        mem_options->buckets[bucket_num].free_chunk =
-                        (mca_allocator_bucket_chunk_header_t *) ((char *) chunk + bucket_size);
-        chunk->next_in_segment = (mca_allocator_bucket_chunk_header_t *)
-                                   ((char *)chunk + bucket_size);
-        while(allocated_size >= bucket_size) {
+    if (allocated_size >= bucket_size) {
+        mem_options->buckets[bucket_num].free_chunk = (mca_allocator_bucket_chunk_header_t
+                                                           *) ((char *) chunk + bucket_size);
+        chunk->next_in_segment = (mca_allocator_bucket_chunk_header_t *) ((char *) chunk
+                                                                          + bucket_size);
+        while (allocated_size >= bucket_size) {
             chunk = (mca_allocator_bucket_chunk_header_t *) ((char *) chunk + bucket_size);
-            chunk->u.next_free = (mca_allocator_bucket_chunk_header_t *)
-                                 ((char *) chunk + bucket_size);
+            chunk->u.next_free = (mca_allocator_bucket_chunk_header_t *) ((char *) chunk
+                                                                          + bucket_size);
             chunk->next_in_segment = chunk->u.next_free;
             allocated_size -= bucket_size;
         }
@@ -225,22 +245,21 @@ void * mca_allocator_bucket_alloc_align(mca_allocator_base_module_t * mem,
     first_chunk->u.bucket = bucket_num;
     OPAL_THREAD_UNLOCK(&(mem_options->buckets[bucket_num].lock));
     /* return the aligned memory */
-    return((void *) (aligned_memory));
+    return ((void *) (aligned_memory));
 }
 
 /*
-  * function to reallocate the segment of memory
-  */
-void * mca_allocator_bucket_realloc(mca_allocator_base_module_t * mem,
-                                    void * ptr, size_t size)
+ * function to reallocate the segment of memory
+ */
+void *mca_allocator_bucket_realloc(mca_allocator_base_module_t *mem, void *ptr, size_t size)
 {
-    mca_allocator_bucket_t * mem_options = (mca_allocator_bucket_t *) mem;
+    mca_allocator_bucket_t *mem_options = (mca_allocator_bucket_t *) mem;
     /* initialize for later bit shifts */
     size_t bucket_size = 1;
     int bucket_num;
-    void * ret_ptr;
+    void *ret_ptr;
     /* get the header of the chunk */
-    mca_allocator_bucket_chunk_header_t * chunk = (mca_allocator_bucket_chunk_header_t *) ptr - 1;
+    mca_allocator_bucket_chunk_header_t *chunk = (mca_allocator_bucket_chunk_header_t *) ptr - 1;
     bucket_num = chunk->u.bucket;
 
     bucket_size <<= (bucket_num + MCA_ALLOCATOR_BUCKET_1_BITSHIFTS);
@@ -249,31 +268,30 @@ void * mca_allocator_bucket_realloc(mca_allocator_base_module_t * mem,
     bucket_size -= sizeof(mca_allocator_bucket_chunk_header_t);
     /* if the requested size is less than or equal to what they ask for,
      * just give them back what they passed in      */
-    if(size <= bucket_size) {
-        return(ptr);
+    if (size <= bucket_size) {
+        return (ptr);
     }
     /* we need a new space in memory, so let's get it */
     ret_ptr = mca_allocator_bucket_alloc((mca_allocator_base_module_t *) mem_options, size);
-    if(NULL == ret_ptr) {
+    if (NULL == ret_ptr) {
         /* we were unable to get a larger area of memory */
-        return(NULL);
+        return (NULL);
     }
     /* copy what they have in memory to the new spot */
     memcpy(ret_ptr, ptr, bucket_size);
     /* free the old area in memory */
     mca_allocator_bucket_free((mca_allocator_base_module_t *) mem_options, ptr);
-    return(ret_ptr);
+    return (ret_ptr);
 }
 
-
 /*
-   * Frees the passed region of memory
-   *
-   */
-void mca_allocator_bucket_free(mca_allocator_base_module_t * mem, void * ptr)
+ * Frees the passed region of memory
+ *
+ */
+void mca_allocator_bucket_free(mca_allocator_base_module_t *mem, void *ptr)
 {
-    mca_allocator_bucket_t * mem_options = (mca_allocator_bucket_t *) mem;
-    mca_allocator_bucket_chunk_header_t * chunk  = (mca_allocator_bucket_chunk_header_t *) ptr - 1;
+    mca_allocator_bucket_t *mem_options = (mca_allocator_bucket_t *) mem;
+    mca_allocator_bucket_chunk_header_t *chunk = (mca_allocator_bucket_chunk_header_t *) ptr - 1;
     int bucket_num = chunk->u.bucket;
     OPAL_THREAD_LOCK(&(mem_options->buckets[bucket_num].lock));
     chunk->u.next_free = mem_options->buckets[bucket_num].free_chunk;
@@ -282,26 +300,26 @@ void mca_allocator_bucket_free(mca_allocator_base_module_t * mem, void * ptr)
 }
 
 /*
-   * Frees all the memory from all the buckets back to the system. Note that
-   * this function only frees memory that was previously freed with
-   * mca_allocator_bucket_free().
-   *
-   */
-int mca_allocator_bucket_cleanup(mca_allocator_base_module_t * mem)
+ * Frees all the memory from all the buckets back to the system. Note that
+ * this function only frees memory that was previously freed with
+ * mca_allocator_bucket_free().
+ *
+ */
+int mca_allocator_bucket_cleanup(mca_allocator_base_module_t *mem)
 {
-    mca_allocator_bucket_t * mem_options = (mca_allocator_bucket_t *) mem;
+    mca_allocator_bucket_t *mem_options = (mca_allocator_bucket_t *) mem;
     int i;
-    mca_allocator_bucket_chunk_header_t * next_chunk;
-    mca_allocator_bucket_chunk_header_t * chunk;
-    mca_allocator_bucket_chunk_header_t * first_chunk;
-    mca_allocator_bucket_segment_head_t ** segment_header;
-    mca_allocator_bucket_segment_head_t * segment;
+    mca_allocator_bucket_chunk_header_t *next_chunk;
+    mca_allocator_bucket_chunk_header_t *chunk;
+    mca_allocator_bucket_chunk_header_t *first_chunk;
+    mca_allocator_bucket_segment_head_t **segment_header;
+    mca_allocator_bucket_segment_head_t *segment;
     bool empty = true;
 
-    for(i = 0; i < mem_options->num_buckets; i++) {
+    for (i = 0; i < mem_options->num_buckets; i++) {
         OPAL_THREAD_LOCK(&(mem_options->buckets[i].lock));
         segment_header = &(mem_options->buckets[i].segment_head);
-        if( NULL == (*segment_header) ) {
+        if (NULL == (*segment_header)) {
             OPAL_THREAD_UNLOCK(&(mem_options->buckets[i].lock));
             continue;
         }
@@ -312,73 +330,75 @@ int mca_allocator_bucket_cleanup(mca_allocator_base_module_t * mem)
          */
         empty = true;
         segment = mem_options->buckets[i].segment_head;
-        while( (true == empty) && (NULL != segment) ) {
+        while ((true == empty) && (NULL != segment)) {
             first_chunk = segment->first_chunk;
             chunk = first_chunk;
             /* determine if the segment is free */
             do {
-                if(chunk->u.bucket == i) {
+                if (chunk->u.bucket == i) {
                     empty = false;
                     break;
                 }
                 chunk = chunk->next_in_segment;
-            } while(chunk != first_chunk);
+            } while (chunk != first_chunk);
             /* go to next segment */
             segment = segment->next_segment;
         }
-        if( true == empty ) {  /* all segments ready for release */
-            mca_allocator_bucket_segment_head_t* next_segment;
+        if (true == empty) { /* all segments ready for release */
+            mca_allocator_bucket_segment_head_t *next_segment;
             segment = mem_options->buckets[i].segment_head;
-            while( NULL != segment ) {
+            while (NULL != segment) {
                 next_segment = segment->next_segment;
                 /* free the memory */
-                if(mem_options->free_mem_fn)
+                if (mem_options->free_mem_fn) {
                     mem_options->free_mem_fn(mem->alc_context, segment);
+                }
                 segment = next_segment;
             }
             mem_options->buckets[i].free_chunk = NULL;
             mem_options->buckets[i].segment_head = NULL;
         } else {
             /* traverse the list of segment headers until we hit NULL */
-            while(NULL != *segment_header) {
+            while (NULL != *segment_header) {
                 first_chunk = (*segment_header)->first_chunk;
                 chunk = first_chunk;
                 empty = true;
                 /* determine if the segment is free */
                 do {
-                    if(chunk->u.bucket == i) {
+                    if (chunk->u.bucket == i) {
                         empty = false;
                     }
                     chunk = chunk->next_in_segment;
-                } while(empty && (chunk != first_chunk));
-                if(empty) {
+                } while (empty && (chunk != first_chunk));
+                if (empty) {
                     chunk = first_chunk;
                     /* remove the chunks from the free list */
                     do {
-                        if(mem_options->buckets[i].free_chunk == chunk) {
+                        if (mem_options->buckets[i].free_chunk == chunk) {
                             mem_options->buckets[i].free_chunk = chunk->u.next_free;
                         } else {
                             next_chunk = mem_options->buckets[i].free_chunk;
-                            while(next_chunk->u.next_free != chunk) {
+                            while (next_chunk->u.next_free != chunk) {
                                 next_chunk = next_chunk->u.next_free;
                             }
                             next_chunk->u.next_free = chunk->u.next_free;
                         }
-                    } while((chunk = chunk->next_in_segment) != first_chunk);
+                    } while ((chunk = chunk->next_in_segment) != first_chunk);
                     /* set the segment list to point to the next segment */
                     segment = *segment_header;
                     *segment_header = segment->next_segment;
                     /* free the memory */
-                    if(mem_options->free_mem_fn)
+                    if (mem_options->free_mem_fn) {
                         mem_options->free_mem_fn(mem->alc_context, segment);
+                    }
                 } else {
                     /* go to next segment */
                     segment_header = &((*segment_header)->next_segment);
                 }
             }
         }
-        /* relese the lock on the bucket */
+        /* release the lock on the bucket */
         OPAL_THREAD_UNLOCK(&(mem_options->buckets[i].lock));
     }
-    return(OPAL_SUCCESS);
+    return (OPAL_SUCCESS);
 }
diff --git a/opal/mca/allocator/bucket/allocator_bucket_alloc.h b/opal/mca/allocator/bucket/allocator_bucket_alloc.h
index 103b611d7b4..80b6b8738b3 100644
--- a/opal/mca/allocator/bucket/allocator_bucket_alloc.h
+++ b/opal/mca/allocator/bucket/allocator_bucket_alloc.h
@@ -11,7 +11,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
- *                         reseved.
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -27,190 +27,182 @@
 #define ALLOCATOR_BUCKET_ALLOC_H
 
 #include "opal_config.h"
+#include "opal/mca/allocator/allocator.h"
+#include "opal/mca/threads/mutex.h"
 #include <stdlib.h>
 #include <string.h>
-#include "opal/mca/threads/mutex.h"
-#include "opal/mca/allocator/allocator.h"
 
 BEGIN_C_DECLS
 
 /**
-  * Structure for the header of each memory chunk
-  */
+ * Structure for the header of each memory chunk
+ */
 struct mca_allocator_bucket_chunk_header_t {
-    struct mca_allocator_bucket_chunk_header_t * next_in_segment;
-               /**< The next chunk in the memory segment */
+    struct mca_allocator_bucket_chunk_header_t *next_in_segment;
+    /**< The next chunk in the memory segment */
     /**
-      * Union which holds either a pointer to the next free chunk
-      * or the bucket number. Based on the current location of the chunk
-      * we use one or the other of these fields. If the chunk is owned
-      * by the user, then the bucket field is set, which allow us to know
-      * in which specific bucket we have to put it back on free (as the
-      * chunk don't have the size attached). When the allocator own the
-      * chunk, the next_free fild is used, which allow us to put these
-      * chunks a list of free elements.
-      */
+     * Union which holds either a pointer to the next free chunk
+     * or the bucket number. Based on the current location of the chunk
+     * we use one or the other of these fields. If the chunk is owned
+     * by the user, then the bucket field is set, which allow us to know
+     * in which specific bucket we have to put it back on free (as the
+     * chunk don't have the size attached). When the allocator own the
+     * chunk, the next_free fild is used, which allow us to put these
+     * chunks a list of free elements.
+     */
     union u {
-        struct mca_allocator_bucket_chunk_header_t * next_free;
+        struct mca_allocator_bucket_chunk_header_t *next_free;
         /**< if the chunk is free this will point to the next free chunk in the bucket */
-        int bucket;                  /**< the bucket number it belongs to */
-    } u; /**< the union */
+        int bucket; /**< the bucket number it belongs to */
+    } u;            /**< the union */
 };
- /**
-   * Typedef so we don't have to use struct
-   */
+/**
+ * Typedef so we don't have to use struct
+ */
 typedef struct mca_allocator_bucket_chunk_header_t mca_allocator_bucket_chunk_header_t;
 
 /**
-  * Structure that heads each segment
-  */
+ * Structure that heads each segment
+ */
 struct mca_allocator_bucket_segment_head_t {
-    struct mca_allocator_bucket_chunk_header_t * first_chunk; /**< the first chunk of the header */
-    struct mca_allocator_bucket_segment_head_t * next_segment; /**< the next segment in the
-                                                 bucket */
+    struct mca_allocator_bucket_chunk_header_t *first_chunk;  /**< the first chunk of the header */
+    struct mca_allocator_bucket_segment_head_t *next_segment; /**< the next segment in the
+                                                bucket */
 };
 /**
-  * Typedef so we don't have to use struct
-  */
+ * Typedef so we don't have to use struct
+ */
 typedef struct mca_allocator_bucket_segment_head_t mca_allocator_bucket_segment_head_t;
 
 /**
-  * Structure for each bucket
-  */
+ * Structure for each bucket
+ */
 struct mca_allocator_bucket_bucket_t {
-    mca_allocator_bucket_chunk_header_t * free_chunk; /**< the first free chunk of memory */
-    opal_mutex_t lock;                /**< the lock on the bucket */
-    mca_allocator_bucket_segment_head_t * segment_head; /**< the list of segment headers */
+    mca_allocator_bucket_chunk_header_t *free_chunk;   /**< the first free chunk of memory */
+    opal_mutex_t lock;                                 /**< the lock on the bucket */
+    mca_allocator_bucket_segment_head_t *segment_head; /**< the list of segment headers */
 };
 /**
-  * Typedef so we don't have to use struct
-  */
+ * Typedef so we don't have to use struct
+ */
 typedef struct mca_allocator_bucket_bucket_t mca_allocator_bucket_bucket_t;
 
 /**
-  * Structure that holds the necessary information for each area of memory
-  */
+ * Structure that holds the necessary information for each area of memory
+ */
 struct mca_allocator_bucket_t {
-    mca_allocator_base_module_t super;          /**< makes this a child of class mca_allocator_t */
-    mca_allocator_bucket_bucket_t * buckets; /**< the array of buckets */
-    int num_buckets;                         /**< the number of buckets */
+    mca_allocator_base_module_t super;      /**< makes this a child of class mca_allocator_t */
+    mca_allocator_bucket_bucket_t *buckets; /**< the array of buckets */
+    int num_buckets;                        /**< the number of buckets */
     mca_allocator_base_component_segment_alloc_fn_t get_mem_fn;
     /**< pointer to the function to get more memory */
     mca_allocator_base_component_segment_free_fn_t free_mem_fn;
     /**< pointer to the function to free memory */
 };
 /**
-  * Typedef so we don't have to use struct
-  */
+ * Typedef so we don't have to use struct
+ */
 typedef struct mca_allocator_bucket_t mca_allocator_bucket_t;
 
-  /**
-   * Initializes the mca_allocator_bucket_options_t data structure for the passed
-   * parameters.
-   * @param mem a pointer to the mca_allocator_t struct to be filled in
-   * @param num_buckets The number of buckets the allocator will use
-   * @param get_mem_funct A pointer to the function that the allocator
-   * will use to get more memory
-   * @param free_mem_funct A pointer to the function that the allocator
-   * will use to free memory
-   *
-   * @retval Pointer to the initialized mca_allocator_bucket_options_t structure
-   * @retval NULL if there was an error
-   */
-    mca_allocator_bucket_t *mca_allocator_bucket_init(mca_allocator_base_module_t * mem,
-                                       int num_buckets,
-                                       mca_allocator_base_component_segment_alloc_fn_t get_mem_funct,
-                                       mca_allocator_base_component_segment_free_fn_t free_mem_funct);
 /**
-   * Accepts a request for memory in a specific region defined by the
-   * mca_allocator_bucket_options_t struct and returns a pointer to memory in that
-   * region or NULL if there was an error
-   *
-   * @param mem A pointer to the appropriate struct for the area of memory.
-   * @param size The size of the requested area of memory
-   *
-   * @retval Pointer to the area of memory if the allocation was successful
-   * @retval NULL if the allocation was unsuccessful
-   */
-    void * mca_allocator_bucket_alloc(
-        mca_allocator_base_module_t * mem,
-        size_t size);
+ * Initializes the mca_allocator_bucket_options_t data structure for the passed
+ * parameters.
+ * @param mem a pointer to the mca_allocator_t struct to be filled in
+ * @param num_buckets The number of buckets the allocator will use
+ * @param get_mem_funct A pointer to the function that the allocator
+ * will use to get more memory
+ * @param free_mem_funct A pointer to the function that the allocator
+ * will use to free memory
+ *
+ * @retval Pointer to the initialized mca_allocator_bucket_options_t structure
+ * @retval NULL if there was an error
+ */
+mca_allocator_bucket_t *
+mca_allocator_bucket_init(mca_allocator_base_module_t *mem, int num_buckets,
+                          mca_allocator_base_component_segment_alloc_fn_t get_mem_funct,
+                          mca_allocator_base_component_segment_free_fn_t free_mem_funct);
+/**
+ * Accepts a request for memory in a specific region defined by the
+ * mca_allocator_bucket_options_t struct and returns a pointer to memory in that
+ * region or NULL if there was an error
+ *
+ * @param mem A pointer to the appropriate struct for the area of memory.
+ * @param size The size of the requested area of memory
+ *
+ * @retval Pointer to the area of memory if the allocation was successful
+ * @retval NULL if the allocation was unsuccessful
+ */
+void *mca_allocator_bucket_alloc(mca_allocator_base_module_t *mem, size_t size);
 
 /**
-   * Accepts a request for memory in a specific region defined by the
-   * mca_allocator_bucket_options_t struct and aligned by the specified amount and
-   * returns a pointer to memory in that region or NULL if there was an error
-   *
-   * @param mem A pointer to the appropriate struct for the area of
-   * memory.
-   * @param size The size of the requested area of memory
-   * @param alignment The requested alignment of the new area of memory. This
-   * MUST be a power of 2.
-   *
-   * @retval Pointer to the area of memory if the allocation was successful
-   * @retval NULL if the allocation was unsuccessful
-   *
-   */
-    void * mca_allocator_bucket_alloc_align(
-        mca_allocator_base_module_t * mem,
-        size_t size,
-        size_t alignment);
+ * Accepts a request for memory in a specific region defined by the
+ * mca_allocator_bucket_options_t struct and aligned by the specified amount and
+ * returns a pointer to memory in that region or NULL if there was an error
+ *
+ * @param mem A pointer to the appropriate struct for the area of
+ * memory.
+ * @param size The size of the requested area of memory
+ * @param alignment The requested alignment of the new area of memory. This
+ * MUST be a power of 2.
+ *
+ * @retval Pointer to the area of memory if the allocation was successful
+ * @retval NULL if the allocation was unsuccessful
+ *
+ */
+void *mca_allocator_bucket_alloc_align(mca_allocator_base_module_t *mem, size_t size,
+                                       size_t alignment);
 
 /**
-   * Attempts to resize the passed region of memory into a larger or a smaller
-   * region. If it is unsuccessful, it will return NULL and the passed area of
-   * memory will be untouched.
-   *
-   * @param mem A pointer to the appropriate struct for the area of
-   * memory.
-   * @param size The size of the requested area of memory
-   * @param ptr A pointer to the region of memory to be resized
-   *
-   * @retval Pointer to the area of memory if the reallocation was successful
-   * @retval NULL if the allocation was unsuccessful
-   *
-   */
-    void * mca_allocator_bucket_realloc(
-        mca_allocator_base_module_t * mem,
-        void * ptr,
-        size_t size);
+ * Attempts to resize the passed region of memory into a larger or a smaller
+ * region. If it is unsuccessful, it will return NULL and the passed area of
+ * memory will be untouched.
+ *
+ * @param mem A pointer to the appropriate struct for the area of
+ * memory.
+ * @param size The size of the requested area of memory
+ * @param ptr A pointer to the region of memory to be resized
+ *
+ * @retval Pointer to the area of memory if the reallocation was successful
+ * @retval NULL if the allocation was unsuccessful
+ *
+ */
+void *mca_allocator_bucket_realloc(mca_allocator_base_module_t *mem, void *ptr, size_t size);
 
 /**
-   * Frees the passed region of memory
-   *
-   * @param mem A pointer to the appropriate struct for the area of
-   * memory.
-   * @param ptr A pointer to the region of memory to be freed
-   *
-   * @retval None
-   *
-   */
-    void mca_allocator_bucket_free(mca_allocator_base_module_t * mem,
-                                   void * ptr);
+ * Frees the passed region of memory
+ *
+ * @param mem A pointer to the appropriate struct for the area of
+ * memory.
+ * @param ptr A pointer to the region of memory to be freed
+ *
+ * @retval None
+ *
+ */
+void mca_allocator_bucket_free(mca_allocator_base_module_t *mem, void *ptr);
 
 /**
-   * Frees all the memory from all the buckets back to the system. Note that
-   * this function only frees memory that was previously freed with
-   * mca_allocator_bucket_free().
-   *
-   * @param mem A pointer to the appropriate struct for the area of
-   * memory.
-   *
-   * @retval None
-   *
-   */
-    int mca_allocator_bucket_cleanup(mca_allocator_base_module_t * mem);
+ * Frees all the memory from all the buckets back to the system. Note that
+ * this function only frees memory that was previously freed with
+ * mca_allocator_bucket_free().
+ *
+ * @param mem A pointer to the appropriate struct for the area of
+ * memory.
+ *
+ * @retval None
+ *
+ */
+int mca_allocator_bucket_cleanup(mca_allocator_base_module_t *mem);
 
 /**
-   * Cleanup all resources held by this allocator.
-   *
-   * @param mem A pointer to the appropriate struct for the area of
-   * memory.
-   *
-   * @retval None
-   *
-   */
-    int mca_allocator_bucket_finalize(mca_allocator_base_module_t * mem);
+ * Cleanup all resources held by this allocator.
+ *
+ * @param mem A pointer to the appropriate struct for the area of
+ * memory.
+ *
+ * @retval None
+ *
+ */
+int mca_allocator_bucket_finalize(mca_allocator_base_module_t *mem);
 
 OPAL_DECLSPEC extern mca_allocator_base_component_t mca_allocator_bucket_component;
 
diff --git a/opal/mca/allocator/bucket/help-mca-allocator-bucket.txt b/opal/mca/allocator/bucket/help-mca-allocator-bucket.txt
new file mode 100644
index 00000000000..27edbb92fa4
--- /dev/null
+++ b/opal/mca/allocator/bucket/help-mca-allocator-bucket.txt
@@ -0,0 +1,19 @@
+# -*- text -*-
+#
+# Copyright (c) 2021      IBM Corporation.  All rights reserved
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+# This is the US/English help file for Open MPI's allocator bucket support
+#
+[buffer too large]
+ERROR: Requested buffer size %zu exceeds limit of %zu
+Consider setting "%s" to %d
+#
+[aligned buffer too large]
+ERROR: Requested aligned buffer size %zu exceeds limit of %zu
+Consider setting "%s" to %d
+#
diff --git a/opal/mca/backtrace/backtrace.h b/opal/mca/backtrace/backtrace.h
index 9ca5658cdea..7ab82c495fc 100644
--- a/opal/mca/backtrace/backtrace.h
+++ b/opal/mca/backtrace/backtrace.h
@@ -25,8 +25,8 @@
 
 #include "opal_config.h"
 
-#include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
+#include "opal/mca/mca.h"
 #include "opal/util/stacktrace.h"
 
 BEGIN_C_DECLS
@@ -37,7 +37,6 @@ BEGIN_C_DECLS
  * components.
  */
 
-
 /*
  * Print back trace to FILE file with a prefix for each line.
  * First strip lines are not printed.
@@ -55,8 +54,7 @@ OPAL_DECLSPEC int opal_backtrace_print(FILE *file, char *prefix, int strip);
  * \note Probably bad to call this from a signal handler.
  *
  */
-OPAL_DECLSPEC int opal_backtrace_buffer(char*** messages, int *len);
-
+OPAL_DECLSPEC int opal_backtrace_buffer(char ***messages, int *len);
 
 /**
  * Structure for backtrace components.
@@ -75,8 +73,7 @@ typedef struct opal_backtrace_base_component_2_0_0_t opal_backtrace_base_compone
 /*
  * Macro for use in components that are of type backtrace
  */
-#define OPAL_BACKTRACE_BASE_VERSION_2_0_0 \
-    OPAL_MCA_BASE_VERSION_2_1_0("backtrace", 2, 0, 0)
+#define OPAL_BACKTRACE_BASE_VERSION_2_0_0 OPAL_MCA_BASE_VERSION_2_1_0("backtrace", 2, 0, 0)
 
 END_C_DECLS
 
diff --git a/opal/mca/backtrace/base/backtrace_component.c b/opal/mca/backtrace/base/backtrace_component.c
index 995fa2328c6..18811e27743 100644
--- a/opal/mca/backtrace/base/backtrace_component.c
+++ b/opal/mca/backtrace/base/backtrace_component.c
@@ -18,7 +18,6 @@
  * $HEADER$
  */
 
-
 #include "opal_config.h"
 
 #include "opal/mca/backtrace/base/base.h"
@@ -30,11 +29,10 @@
  */
 #include "opal/mca/backtrace/base/static-components.h"
 
-
 /*
  * Globals
  */
 
 /* Uses default register/open/close functions */
 MCA_BASE_FRAMEWORK_DECLARE(opal, backtrace, NULL, NULL, NULL, NULL,
-			   mca_backtrace_base_static_components, 0);
+                           mca_backtrace_base_static_components, 0);
diff --git a/opal/mca/backtrace/configure.m4 b/opal/mca/backtrace/configure.m4
index 0ee4261f3e8..4f6f8478d3a 100644
--- a/opal/mca/backtrace/configure.m4
+++ b/opal/mca/backtrace/configure.m4
@@ -10,6 +10,7 @@ dnl Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
 dnl                         University of Stuttgart.  All rights reserved.
 dnl Copyright (c) 2004-2006 The Regents of the University of California.
 dnl                         All rights reserved.
+dnl Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
@@ -17,5 +18,8 @@ dnl
 dnl $HEADER$
 dnl
 
+dnl need in the core library
+AC_DEFUN([MCA_opal_backtrace_CORE_LIB], [1])
+
 dnl we only want one :)
 m4_define(MCA_opal_backtrace_CONFIGURE_MODE, STOP_AT_FIRST)
diff --git a/opal/mca/backtrace/execinfo/backtrace_execinfo.c b/opal/mca/backtrace/execinfo/backtrace_execinfo.c
index 0f17c514c1d..f0cbf572819 100644
--- a/opal/mca/backtrace/execinfo/backtrace_execinfo.c
+++ b/opal/mca/backtrace/execinfo/backtrace_execinfo.c
@@ -23,25 +23,25 @@
 #include <stdio.h>
 #include <string.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 #ifdef HAVE_EXECINFO_H
-#include <execinfo.h>
+#    include <execinfo.h>
 #endif
 
 #include "opal/constants.h"
+#include "opal/util/output.h"
 #include "opal/mca/backtrace/backtrace.h"
 
-int
-opal_backtrace_print(FILE *file, char *prefix, int strip)
+int opal_backtrace_print(FILE *file, char *prefix, int strip)
 {
     int i, len;
     int trace_size;
-    void * trace[32];
+    void *trace[32];
     char buf[6];
     int fd = opal_stacktrace_output_fileno;
 
-    if( NULL != file ) {
+    if (NULL != file) {
         fd = fileno(file);
     }
 
@@ -49,30 +49,28 @@ opal_backtrace_print(FILE *file, char *prefix, int strip)
         return OPAL_ERR_BAD_PARAM;
     }
 
-    trace_size = backtrace (trace, 32);
+    trace_size = backtrace(trace, 32);
 
     for (i = strip; i < trace_size; i++) {
         if (NULL != prefix) {
-            write (fd, prefix, strlen (prefix));
+            opal_best_effort_write(fd, prefix, strlen(prefix));
         }
-        len = snprintf (buf, sizeof(buf), "[%2d] ", i - strip);
-        write (fd, buf, len);
-        backtrace_symbols_fd (&trace[i], 1, fd);
+        len = snprintf(buf, sizeof(buf), "[%2d] ", i - strip);
+        opal_best_effort_write(fd, buf, len);
+        backtrace_symbols_fd(&trace[i], 1, fd);
     }
 
     return OPAL_SUCCESS;
 }
 
-
-int
-opal_backtrace_buffer(char ***message_out, int *len_out)
+int opal_backtrace_buffer(char ***message_out, int *len_out)
 {
     int trace_size;
-    void * trace[32];
-    char ** funcs = (char **)NULL;
+    void *trace[32];
+    char **funcs = (char **) NULL;
 
-    trace_size = backtrace (trace, 32);
-    funcs = backtrace_symbols (trace, trace_size);
+    trace_size = backtrace(trace, 32);
+    funcs = backtrace_symbols(trace, trace_size);
 
     *message_out = funcs;
     *len_out = trace_size;
diff --git a/opal/mca/backtrace/execinfo/backtrace_execinfo_component.c b/opal/mca/backtrace/execinfo/backtrace_execinfo_component.c
index cb44d2ce692..fd36bbffe0c 100644
--- a/opal/mca/backtrace/execinfo/backtrace_execinfo_component.c
+++ b/opal/mca/backtrace/execinfo/backtrace_execinfo_component.c
@@ -26,16 +26,16 @@
 const opal_backtrace_base_component_2_0_0_t mca_backtrace_execinfo_component = {
     /* First, the mca_component_t struct containing meta information
        about the component itself */
-    .backtracec_version = {
-        OPAL_BACKTRACE_BASE_VERSION_2_0_0,
+    .backtracec_version =
+        {
+            OPAL_BACKTRACE_BASE_VERSION_2_0_0,
 
-        /* Component name and version */
-        .mca_component_name = "execinfo",
-        MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
-                              OPAL_RELEASE_VERSION),
-    },
-    .backtracec_data = {
-        /* The component is checkpoint ready */
-        MCA_BASE_METADATA_PARAM_CHECKPOINT
-    },
+            /* Component name and version */
+            .mca_component_name = "execinfo",
+            MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
+                                  OPAL_RELEASE_VERSION),
+        },
+    .backtracec_data =
+        {/* The component is checkpoint ready */
+         MCA_BASE_METADATA_PARAM_CHECKPOINT},
 };
diff --git a/opal/mca/backtrace/none/backtrace_none.c b/opal/mca/backtrace/none/backtrace_none.c
index ba343f22dbf..57b8baba456 100644
--- a/opal/mca/backtrace/none/backtrace_none.c
+++ b/opal/mca/backtrace/none/backtrace_none.c
@@ -23,15 +23,12 @@
 #include "opal/constants.h"
 #include "opal/mca/backtrace/backtrace.h"
 
-int
-opal_backtrace_print(FILE *file, char *prefix, int strip)
+int opal_backtrace_print(FILE *file, char *prefix, int strip)
 {
     return OPAL_ERR_NOT_IMPLEMENTED;
 }
 
-
-int
-opal_backtrace_buffer(char ***message_out, int *len_out)
+int opal_backtrace_buffer(char ***message_out, int *len_out)
 {
     *message_out = NULL;
     *len_out = 0;
diff --git a/opal/mca/backtrace/none/backtrace_none_component.c b/opal/mca/backtrace/none/backtrace_none_component.c
index 9f820ea695a..a5021183b86 100644
--- a/opal/mca/backtrace/none/backtrace_none_component.c
+++ b/opal/mca/backtrace/none/backtrace_none_component.c
@@ -24,22 +24,22 @@
 #include "opal/mca/backtrace/backtrace.h"
 
 BEGIN_C_DECLS
-    OPAL_DECLSPEC extern const opal_backtrace_base_component_2_0_0_t mca_backtrace_none_component;
+OPAL_DECLSPEC extern const opal_backtrace_base_component_2_0_0_t mca_backtrace_none_component;
 END_C_DECLS
 
 const opal_backtrace_base_component_2_0_0_t mca_backtrace_none_component = {
     /* First, the mca_component_t struct containing meta information
        about the component itself */
-    .backtracec_version = {
-        OPAL_BACKTRACE_BASE_VERSION_2_0_0,
+    .backtracec_version =
+        {
+            OPAL_BACKTRACE_BASE_VERSION_2_0_0,
 
-        /* Component name and version */
-        .mca_component_name = "none",
-        MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
-                              OPAL_RELEASE_VERSION),
-    },
-    .backtracec_data = {
-        /* The component is checkpoint ready */
-        MCA_BASE_METADATA_PARAM_CHECKPOINT
-    },
+            /* Component name and version */
+            .mca_component_name = "none",
+            MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
+                                  OPAL_RELEASE_VERSION),
+        },
+    .backtracec_data =
+        {/* The component is checkpoint ready */
+         MCA_BASE_METADATA_PARAM_CHECKPOINT},
 };
diff --git a/opal/mca/backtrace/printstack/backtrace_printstack.c b/opal/mca/backtrace/printstack/backtrace_printstack.c
index 214cacfb14b..e95335b6cf4 100644
--- a/opal/mca/backtrace/printstack/backtrace_printstack.c
+++ b/opal/mca/backtrace/printstack/backtrace_printstack.c
@@ -25,12 +25,11 @@
 #include "opal/constants.h"
 #include "opal/mca/backtrace/backtrace.h"
 
-int
-opal_backtrace_print(FILE *file, char *prefix, int strip)
+int opal_backtrace_print(FILE *file, char *prefix, int strip)
 {
     int fd = opal_stacktrace_output_fileno;
 
-    if( NULL != file ) {
+    if (NULL != file) {
         fd = fileno(file);
     }
 
@@ -39,9 +38,7 @@ opal_backtrace_print(FILE *file, char *prefix, int strip)
     return OPAL_SUCCESS;
 }
 
-
-int
-opal_backtrace_buffer(char ***message_out, int *len_out)
+int opal_backtrace_buffer(char ***message_out, int *len_out)
 {
     *message_out = NULL;
     *len_out = 0;
diff --git a/opal/mca/backtrace/printstack/backtrace_printstack_component.c b/opal/mca/backtrace/printstack/backtrace_printstack_component.c
index 3a37269969d..7174383bb72 100644
--- a/opal/mca/backtrace/printstack/backtrace_printstack_component.c
+++ b/opal/mca/backtrace/printstack/backtrace_printstack_component.c
@@ -26,16 +26,16 @@
 const opal_backtrace_base_component_2_0_0_t mca_backtrace_printstack_component = {
     /* First, the mca_component_t struct containing meta information
        about the component itself */
-    .backtracec_version = {
-        OPAL_BACKTRACE_BASE_VERSION_2_0_0,
+    .backtracec_version =
+        {
+            OPAL_BACKTRACE_BASE_VERSION_2_0_0,
 
-        /* Component name and version */
-        .mca_component_name = "printstack",
-        MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
-                              OPAL_RELEASE_VERSION),
-    },
-    .backtracec_data = {
-        /* The component is checkpoint ready */
-        MCA_BASE_METADATA_PARAM_CHECKPOINT
-    },
+            /* Component name and version */
+            .mca_component_name = "printstack",
+            MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
+                                  OPAL_RELEASE_VERSION),
+        },
+    .backtracec_data =
+        {/* The component is checkpoint ready */
+         MCA_BASE_METADATA_PARAM_CHECKPOINT},
 };
diff --git a/opal/mca/base/Makefile.am b/opal/mca/base/Makefile.am
index 2e4cd3f2f3f..c6f57e0283a 100644
--- a/opal/mca/base/Makefile.am
+++ b/opal/mca/base/Makefile.am
@@ -11,6 +11,7 @@
 #                         All rights reserved.
 # Copyright (c) 2010 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2020      Google LLC. All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -26,7 +27,7 @@ AM_CPPFLAGS = \
 
 noinst_LTLIBRARIES = libmca_base.la
 
-dist_opaldata_DATA = help-mca-base.txt help-mca-var.txt
+EXTRA_DIST = help-mca-base.txt help-mca-var.txt
 
 # Source code files
 
diff --git a/opal/mca/base/base.h b/opal/mca/base/base.h
index d180d17961d..789cbc52030 100644
--- a/opal/mca/base/base.h
+++ b/opal/mca/base/base.h
@@ -10,7 +10,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2009-2022 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2015      Research Organization for Information Science
@@ -30,15 +30,15 @@
 
 #include "opal_config.h"
 
-#include "opal/class/opal_object.h"
 #include "opal/class/opal_list.h"
+#include "opal/class/opal_object.h"
 
 /*
  * These units are large enough to warrant their own .h files
  */
-#include "opal/mca/mca.h"
-#include "opal/mca/base/mca_base_var.h"
 #include "opal/mca/base/mca_base_framework.h"
+#include "opal/mca/base/mca_base_var.h"
+#include "opal/mca/mca.h"
 #include "opal/util/cmd_line.h"
 #include "opal/util/output.h"
 
@@ -61,8 +61,7 @@ struct mca_base_component_priority_list_item_t {
     mca_base_component_list_item_t super;
     int cpli_priority;
 };
-typedef struct mca_base_component_priority_list_item_t
-    mca_base_component_priority_list_item_t;
+typedef struct mca_base_component_priority_list_item_t mca_base_component_priority_list_item_t;
 
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(mca_base_component_priority_list_item_t);
 
@@ -70,7 +69,7 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(mca_base_component_priority_list_item_t);
  * Public variables
  */
 OPAL_DECLSPEC extern char *mca_base_component_path;
-OPAL_DECLSPEC extern bool mca_base_component_show_load_errors;
+OPAL_DECLSPEC extern char *mca_base_component_show_load_errors;
 OPAL_DECLSPEC extern bool mca_base_component_track_load_errors;
 OPAL_DECLSPEC extern bool mca_base_component_disable_dlopen;
 OPAL_DECLSPEC extern char *mca_base_system_default_path;
@@ -81,23 +80,23 @@ OPAL_DECLSPEC extern char *mca_base_user_default_path;
  */
 enum {
     /** total silence */
-    MCA_BASE_VERBOSE_NONE  = -1,
+    MCA_BASE_VERBOSE_NONE = -1,
     /** only errors are printed */
     MCA_BASE_VERBOSE_ERROR = 0,
     /** emit messages about component selection, open, and unloading */
     MCA_BASE_VERBOSE_COMPONENT = 10,
     /** also emit warnings */
-    MCA_BASE_VERBOSE_WARN  = 20,
+    MCA_BASE_VERBOSE_WARN = 20,
     /** also emit general, user-relevant information, such as rationale as to why certain choices
      * or code paths were taken, information gleaned from probing the local system, etc. */
-    MCA_BASE_VERBOSE_INFO  = 40,
+    MCA_BASE_VERBOSE_INFO = 40,
     /** also emit relevant tracing information (e.g., which functions were invoked /
      * call stack entry/exit info) */
     MCA_BASE_VERBOSE_TRACE = 60,
     /** also emit Open MPI-developer-level (i.e,. highly detailed) information */
     MCA_BASE_VERBOSE_DEBUG = 80,
     /** also output anything else that might be useful */
-    MCA_BASE_VERBOSE_MAX   = 100,
+    MCA_BASE_VERBOSE_MAX = 100,
 };
 
 /*
@@ -127,7 +126,7 @@ OPAL_DECLSPEC int mca_base_open(void);
  * @return OPAL_ERROR Upon failure
  *
  * This function closes down the entire MCA.  It clears all MCA
- * parameters and closes down the MCA component respository.
+ * parameters and closes down the MCA component repository.
  *
  * It must be the last MCA function invoked.  It is normally invoked
  * during the finalize stage.
@@ -141,8 +140,7 @@ OPAL_DECLSPEC void mca_base_close(void);
 OPAL_DECLSPEC int mca_base_select(const char *type_name, int output_id,
                                   opal_list_t *components_available,
                                   mca_base_module_t **best_module,
-                                  mca_base_component_t **best_component,
-                                  int *priority_out);
+                                  mca_base_component_t **best_component, int *priority_out);
 
 /**
  * A function for component query functions to discover if they have
@@ -153,15 +151,13 @@ OPAL_DECLSPEC int mca_base_select(const char *type_name, int output_id,
  *
  */
 OPAL_DECLSPEC int mca_base_is_component_required(opal_list_t *components_available,
-                                                 mca_base_component_t *component,
-                                                 bool exclusive,
+                                                 mca_base_component_t *component, bool exclusive,
                                                  bool *is_required);
 
 /* mca_base_cmd_line.c */
 
 OPAL_DECLSPEC int mca_base_cmd_line_setup(opal_cmd_line_t *cmd);
-OPAL_DECLSPEC int mca_base_cmd_line_process_args(opal_cmd_line_t *cmd,
-                                                 char ***app_env,
+OPAL_DECLSPEC int mca_base_cmd_line_process_args(opal_cmd_line_t *cmd, char ***app_env,
                                                  char ***global_env);
 OPAL_DECLSPEC void mca_base_cmd_line_wrap_args(char **args);
 
@@ -173,19 +169,19 @@ OPAL_DECLSPEC int mca_base_component_compare(const mca_base_component_t *a,
                                              const mca_base_component_t *b);
 OPAL_DECLSPEC int mca_base_component_compatible(const mca_base_component_t *a,
                                                 const mca_base_component_t *b);
-OPAL_DECLSPEC char * mca_base_component_to_string(const mca_base_component_t *a);
+OPAL_DECLSPEC char *mca_base_component_to_string(const mca_base_component_t *a);
 
 /* mca_base_component_find.c */
 
-OPAL_DECLSPEC int mca_base_component_find (const char *directory, mca_base_framework_t *framework,
-                                           bool ignore_requested, bool open_dso_components);
+OPAL_DECLSPEC int mca_base_component_find(const char *directory, mca_base_framework_t *framework,
+                                          bool ignore_requested, bool open_dso_components);
 
 /**
  * Parse the requested component string and return an opal_argv of the requested
  * (or not requested) components.
  */
-int mca_base_component_parse_requested (const char *requested, bool *include_mode,
-                                        char ***requested_component_names);
+int mca_base_component_parse_requested(const char *requested, bool *include_mode,
+                                       char ***requested_component_names);
 
 /**
  * Filter a list of components based on a comma-delimted list of names and/or
@@ -205,9 +201,8 @@ int mca_base_component_parse_requested (const char *requested, bool *include_mod
  * This function closes and releases any components that do not match the filter_name and
  * filter flags.
  */
-OPAL_DECLSPEC int mca_base_components_filter (mca_base_framework_t *framework, uint32_t filter_flags);
-
-
+OPAL_DECLSPEC int mca_base_components_filter(mca_base_framework_t *framework,
+                                             uint32_t filter_flags);
 
 /* Safely release some memory allocated by mca_base_component_find()
    (i.e., is safe to call even if you never called
@@ -215,12 +210,16 @@ OPAL_DECLSPEC int mca_base_components_filter (mca_base_framework_t *framework, u
 OPAL_DECLSPEC int mca_base_component_find_finalize(void);
 
 /* mca_base_components_register.c */
-OPAL_DECLSPEC int mca_base_framework_components_register (struct mca_base_framework_t *framework,
-                                                          mca_base_register_flag_t flags);
+OPAL_DECLSPEC int mca_base_framework_components_register(struct mca_base_framework_t *framework,
+                                                         mca_base_register_flag_t flags);
 
 /* mca_base_components_open.c */
-OPAL_DECLSPEC int mca_base_framework_components_open (struct mca_base_framework_t *framework,
-                                                      mca_base_open_flag_t flags);
+OPAL_DECLSPEC int mca_base_show_load_errors_init(void);
+OPAL_DECLSPEC int mca_base_show_load_errors_finalize(void);
+OPAL_DECLSPEC bool mca_base_show_load_errors(const char *framework_name,
+                                             const char *component_name);
+OPAL_DECLSPEC int mca_base_framework_components_open(struct mca_base_framework_t *framework,
+                                                     mca_base_open_flag_t flags);
 
 OPAL_DECLSPEC int mca_base_components_open(const char *type_name, int output_id,
                                            const mca_base_component_t **static_components,
@@ -236,7 +235,7 @@ OPAL_DECLSPEC int mca_base_components_open(const char *type_name, int output_id,
  *
  * After calling this function the component may no longer be used.
  */
-OPAL_DECLSPEC void mca_base_component_close (const mca_base_component_t *component, int output_id);
+OPAL_DECLSPEC void mca_base_component_close(const mca_base_component_t *component, int output_id);
 
 /**
  * Release a component without closing it.
@@ -245,13 +244,13 @@ OPAL_DECLSPEC void mca_base_component_close (const mca_base_component_t *compone
  *
  * After calling this function the component may no longer be used.
  */
-void mca_base_component_unload (const mca_base_component_t *component, int output_id);
+void mca_base_component_unload(const mca_base_component_t *component, int output_id);
 
 OPAL_DECLSPEC int mca_base_components_close(int output_id, opal_list_t *components_available,
                                             const mca_base_component_t *skip);
 
-OPAL_DECLSPEC int mca_base_framework_components_close (struct mca_base_framework_t *framework,
-						       const mca_base_component_t *skip);
+OPAL_DECLSPEC int mca_base_framework_components_close(struct mca_base_framework_t *framework,
+                                                      const mca_base_component_t *skip);
 
 END_C_DECLS
 
diff --git a/opal/mca/base/help-mca-base.txt b/opal/mca/base/help-mca-base.txt
index 543b5c51f25..2637fcce1fa 100644
--- a/opal/mca/base/help-mca-base.txt
+++ b/opal/mca/base/help-mca-base.txt
@@ -10,7 +10,7 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2008-2014 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2008-2022 Cisco Systems, Inc.  All rights reserved
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -59,3 +59,23 @@ all components *except* a and b", while "c,d" specifies the inclusive
 behavior and means "use *only* components c and d."
 
 You cannot mix inclusive and exclusive behavior.
+#
+[internal error during init]
+An internal error has occurred during the startup of Open MPI.  This
+is highly unusual and shouldn't happen.  Open MPI will now abort your
+job.
+
+The following message may provide additional insight into the error:
+
+  Failure at:      %s (%s:%d)
+  Error:           %d (%s)
+#
+[show_load_errors: too many /]
+The opal_mca_base_component_show_load_errors MCA variable cannot
+contain a token that has more than one "/" character in it.
+
+The opal_mca_base_component_show_load_errors MCA variable can only
+contain the values: all, none, or a comma-delimited list of tokens in
+the form of "framework" or "framework/component".
+
+  Erroneous value: %s
diff --git a/opal/mca/base/help-mca-var.txt b/opal/mca/base/help-mca-var.txt
index 316342bb005..56010c1dcb5 100644
--- a/opal/mca/base/help-mca-var.txt
+++ b/opal/mca/base/help-mca-var.txt
@@ -74,7 +74,7 @@ variables are supposed to affect.
 [re-register-with-different-type]
 An MCA variable was re-registered with a different type (i.e., it was
 either originally registered as an INT and re-registered as a STRING,
-or it was originially registered as a STRING and re-registered as an
+or it was originally registered as a STRING and re-registered as an
 INT).  This a developer error; your job may abort.
 
   MCA variable name: %s
diff --git a/opal/mca/base/mca_base_alias.c b/opal/mca/base/mca_base_alias.c
index ad5e557d18d..e9a8b991796 100644
--- a/opal/mca/base/mca_base_alias.c
+++ b/opal/mca/base/mca_base_alias.c
@@ -12,25 +12,26 @@
 #include "opal/class/opal_hash_table.h"
 #include "opal/runtime/opal.h"
 
-static void mca_base_alias_init (mca_base_alias_t *alias)
+static void mca_base_alias_init(mca_base_alias_t *alias)
 {
     OBJ_CONSTRUCT(&alias->component_aliases, opal_list_t);
 }
 
-static void mca_base_alias_fini (mca_base_alias_t *alias)
+static void mca_base_alias_fini(mca_base_alias_t *alias)
 {
     OPAL_LIST_DESTRUCT(&alias->component_aliases);
 }
 
-OBJ_CLASS_INSTANCE(mca_base_alias_t, opal_object_t, mca_base_alias_init,
-                   mca_base_alias_fini);
+OBJ_CLASS_INSTANCE(mca_base_alias_t, opal_object_t, mca_base_alias_init, mca_base_alias_fini);
 
-static void mca_base_alias_item_init (mca_base_alias_item_t *alias_item) {
+static void mca_base_alias_item_init(mca_base_alias_item_t *alias_item)
+{
     alias_item->component_alias = NULL;
 }
 
-static void mca_base_alias_item_fini (mca_base_alias_item_t *alias_item) {
-    free (alias_item->component_alias);
+static void mca_base_alias_item_fini(mca_base_alias_item_t *alias_item)
+{
+    free(alias_item->component_alias);
 }
 
 OBJ_CLASS_INSTANCE(mca_base_alias_item_t, opal_list_item_t, mca_base_alias_item_init,
@@ -41,7 +42,7 @@ OBJ_CLASS_INSTANCE(mca_base_alias_item_t, opal_list_item_t, mca_base_alias_item_
  */
 static opal_hash_table_t *alias_hash_table;
 
-static void mca_base_alias_cleanup (void)
+static void mca_base_alias_cleanup(void)
 {
     if (!alias_hash_table) {
         return;
@@ -49,28 +50,27 @@ static void mca_base_alias_cleanup (void)
 
     void *key;
     opal_object_t *value;
-    OPAL_HASH_TABLE_FOREACH_PTR(key, value, alias_hash_table, {
-        OBJ_RELEASE(value);
-    });
+    OPAL_HASH_TABLE_FOREACH_PTR (key, value, alias_hash_table, { OBJ_RELEASE(value); })
+        ;
 
     OBJ_RELEASE(alias_hash_table);
     alias_hash_table = NULL;
 }
 
-static int mca_base_alias_setup (void)
+static int mca_base_alias_setup(void)
 {
     if (NULL != alias_hash_table) {
         return OPAL_SUCCESS;
     }
 
-    opal_finalize_register_cleanup (mca_base_alias_cleanup);
+    opal_finalize_register_cleanup(mca_base_alias_cleanup);
 
     alias_hash_table = OBJ_NEW(opal_hash_table_t);
     if (NULL == alias_hash_table) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
-    int ret = opal_hash_table_init (alias_hash_table, 32);
+    int ret = opal_hash_table_init(alias_hash_table, 32);
     if (OPAL_SUCCESS != ret) {
         OBJ_RELEASE(alias_hash_table);
         alias_hash_table = NULL;
@@ -80,19 +80,20 @@ static int mca_base_alias_setup (void)
     return OPAL_SUCCESS;
 }
 
-static char *mca_base_alias_generate_name (const char *project, const char *framework, const char *component_name)
+static char *mca_base_alias_generate_name(const char *project, const char *framework,
+                                          const char *component_name)
 {
-    size_t project_length = project ? strlen (project) : 0;
-    size_t framework_length = framework ? strlen (framework) : 0;
-    size_t component_name_length = strlen (component_name);
+    size_t project_length = project ? strlen(project) : 0;
+    size_t framework_length = framework ? strlen(framework) : 0;
+    size_t component_name_length = strlen(component_name);
     size_t length = project_length + framework_length + component_name_length + 2;
-    char *tmp = calloc (1, length + 1);
+    char *tmp = calloc(1, length + 1);
     if (NULL == tmp) {
         return tmp;
     }
 
     if (project_length) {
-        strncat (tmp, project, length);
+        strncat(tmp, project, length);
         // NOTE: We use strcat() here (and not strncat()) because
         // we're appending a constant string with a fixed/hard-coded
         // length.  It's the exact equivalent of strncat(tmp, "_", 1),
@@ -101,85 +102,88 @@ static char *mca_base_alias_generate_name (const char *project, const char *fram
         // constant string as the source and a hard-coded length that
         // is equivalent to the length of that compile-time constant
         // string.  So avoid the warning and use strcat().
-        strcat (tmp, "_");
+        strcat(tmp, "_");
         length -= project_length + 1;
     }
 
     if (framework_length) {
-        strncat (tmp, framework, length);
+        strncat(tmp, framework, length);
         // Use strcat() here instead of strncat(); see the comment
         // above for an explanation.
-        strcat (tmp, "_");
+        strcat(tmp, "_");
         length -= framework_length + 1;
     }
 
-    strncat (tmp, component_name, length);
+    strncat(tmp, component_name, length);
 
     return tmp;
 }
 
-static mca_base_alias_t *mca_base_alias_lookup_internal (const char *name)
+static mca_base_alias_t *mca_base_alias_lookup_internal(const char *name)
 {
     mca_base_alias_t *alias = NULL;
     if (NULL == alias_hash_table) {
         return NULL;
     }
 
-    (void) opal_hash_table_get_value_ptr (alias_hash_table, name, strlen (name), (void **) &alias);
+    (void) opal_hash_table_get_value_ptr(alias_hash_table, name, strlen(name), (void **) &alias);
     return alias;
 }
 
-int mca_base_alias_register (const char *project, const char *framework, const char *component_name,
-                             const char *component_alias, uint32_t alias_flags)
+int mca_base_alias_register(const char *project, const char *framework, const char *component_name,
+                            const char *component_alias, uint32_t alias_flags)
 {
     if (NULL == component_name) {
         return OPAL_ERR_BAD_PARAM;
     }
 
-    int ret = mca_base_alias_setup ();
+    int ret = mca_base_alias_setup();
     if (OPAL_SUCCESS != ret) {
         return ret;
     }
 
-    char *name = mca_base_alias_generate_name (project, framework, component_name);
-    assert (NULL != name);
+    char *name = mca_base_alias_generate_name(project, framework, component_name);
+    assert(NULL != name);
 
-    mca_base_alias_t *alias = mca_base_alias_lookup_internal (name);
+    mca_base_alias_t *alias = mca_base_alias_lookup_internal(name);
     if (NULL == alias) {
         alias = OBJ_NEW(mca_base_alias_t);
         if (NULL == alias) {
+            free(name);
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
 
-        opal_hash_table_set_value_ptr (alias_hash_table, name, strlen(name), alias);
-        free (name);
+        opal_hash_table_set_value_ptr(alias_hash_table, name, strlen(name), alias);
+        free(name);
         name = NULL;
     }
 
     mca_base_alias_item_t *alias_item = OBJ_NEW(mca_base_alias_item_t);
     if (NULL == alias_item) {
-        if (NULL != name) free(name);
+        free(name);
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
-    
-    alias_item->component_alias = strdup (component_alias);
+
+    alias_item->component_alias = strdup(component_alias);
     alias_item->alias_flags = alias_flags;
 
-    opal_list_append (&alias->component_aliases, &alias_item->super);
+    opal_list_append(&alias->component_aliases, &alias_item->super);
 
+    free(name);
     return OPAL_SUCCESS;
 }
 
-const mca_base_alias_t *mca_base_alias_lookup(const char *project, const char *framework, const char *component_name)
+const mca_base_alias_t *mca_base_alias_lookup(const char *project, const char *framework,
+                                              const char *component_name)
 {
     if (NULL == component_name) {
         return NULL;
     }
 
-    char *name = mca_base_alias_generate_name (project, framework, component_name);
-    assert (NULL != name);
-    const mca_base_alias_t *alias = mca_base_alias_lookup_internal (name);
-    free (name);
+    char *name = mca_base_alias_generate_name(project, framework, component_name);
+    assert(NULL != name);
+    const mca_base_alias_t *alias = mca_base_alias_lookup_internal(name);
+    free(name);
 
     return alias;
 }
diff --git a/opal/mca/base/mca_base_alias.h b/opal/mca/base/mca_base_alias.h
index 552464ce6da..83bcbafbd95 100644
--- a/opal/mca/base/mca_base_alias.h
+++ b/opal/mca/base/mca_base_alias.h
@@ -17,9 +17,9 @@
 BEGIN_C_DECLS
 
 enum mca_base_alias_flags_t {
-      MCA_BASE_ALIAS_FLAG_NONE       = 0,
-      /** The aliased name has been deprecated. */
-      MCA_BASE_ALIAS_FLAG_DEPRECATED = 1,
+    MCA_BASE_ALIAS_FLAG_NONE = 0,
+    /** The aliased name has been deprecated. */
+    MCA_BASE_ALIAS_FLAG_DEPRECATED = 1,
 };
 
 typedef enum mca_base_alias_flags_t mca_base_alias_flags_t;
@@ -67,10 +67,9 @@ OBJ_CLASS_DECLARATION(mca_base_alias_t);
  * component if the synonym is registered in the btl framework
  * registration function.
  */
-OPAL_DECLSPEC int mca_base_alias_register (const char *project, const char *framework,
-                                           const char *component_name,
-                                           const char *component_alias,
-                                           uint32_t alias_flags);
+OPAL_DECLSPEC int mca_base_alias_register(const char *project, const char *framework,
+                                          const char *component_name, const char *component_alias,
+                                          uint32_t alias_flags);
 
 /**
  * @brief Check for aliases for a component.
@@ -79,8 +78,7 @@ OPAL_DECLSPEC int mca_base_alias_register (const char *project, const char *fram
  * @param[in] frameworek     Framework name (may be NULL)
  * @param[in] component_name Component name (may not be NULL)
  */
-OPAL_DECLSPEC const mca_base_alias_t *mca_base_alias_lookup(const char *project,
-                                                            const char *framework,
-                                                            const char *component_name);
+OPAL_DECLSPEC const mca_base_alias_t *
+mca_base_alias_lookup(const char *project, const char *framework, const char *component_name);
 
 #endif /* OPAL_MCA_BASE_ALIAS_H */
diff --git a/opal/mca/base/mca_base_close.c b/opal/mca/base/mca_base_close.c
index ef4929e9a5d..0f2f0fa9ff2 100644
--- a/opal/mca/base/mca_base_close.c
+++ b/opal/mca/base/mca_base_close.c
@@ -10,7 +10,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2009-2022 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
  * $COPYRIGHT$
@@ -22,11 +22,11 @@
 
 #include "opal_config.h"
 
-#include "opal/util/output.h"
-#include "opal/mca/mca.h"
+#include "opal/constants.h"
 #include "opal/mca/base/base.h"
 #include "opal/mca/base/mca_base_component_repository.h"
-#include "opal/constants.h"
+#include "opal/mca/mca.h"
+#include "opal/util/output.h"
 
 extern int mca_base_opened;
 
@@ -34,25 +34,25 @@ extern int mca_base_opened;
  * Main MCA shutdown.
  */
 
-void mca_base_close (void)
+void mca_base_close(void)
 {
-    assert (mca_base_opened);
+    assert(mca_base_opened);
     if (--mca_base_opened) {
         return;
     }
 
     /* deregister all MCA base parameters */
-    int group_id = mca_base_var_group_find ("opal", "mca", "base");
+    int group_id = mca_base_var_group_find("opal", "mca", "base");
 
     if (-1 < group_id) {
-        mca_base_var_group_deregister (group_id);
+        mca_base_var_group_deregister(group_id);
     }
 
     /* release the default paths */
-    free (mca_base_system_default_path);
+    free(mca_base_system_default_path);
     mca_base_system_default_path = NULL;
 
-    free (mca_base_user_default_path);
+    free(mca_base_user_default_path);
     mca_base_user_default_path = NULL;
 
     /* Close down the component repository */
@@ -61,6 +61,9 @@ void mca_base_close (void)
     /* Shut down the dynamic component finder */
     mca_base_component_find_finalize();
 
+    /* Shut down the show_load_errors processing */
+    mca_base_show_load_errors_finalize();
+
     /* Close opal output stream 0 */
     opal_output_close(0);
 }
diff --git a/opal/mca/base/mca_base_cmd_line.c b/opal/mca/base/mca_base_cmd_line.c
index dae1dac3e5d..08fe29e62f1 100644
--- a/opal/mca/base/mca_base_cmd_line.c
+++ b/opal/mca/base/mca_base_cmd_line.c
@@ -24,14 +24,13 @@
 #include <stdio.h>
 #include <string.h>
 
-#include "opal/util/cmd_line.h"
+#include "opal/constants.h"
+#include "opal/mca/base/base.h"
 #include "opal/util/argv.h"
+#include "opal/util/cmd_line.h"
 #include "opal/util/opal_environ.h"
-#include "opal/util/show_help.h"
 #include "opal/util/printf.h"
-#include "opal/mca/base/base.h"
-#include "opal/constants.h"
-
+#include "opal/util/show_help.h"
 
 /*
  * Private variables
@@ -40,11 +39,9 @@
 /*
  * Private functions
  */
-static int process_arg(const char *param, const char *value,
-                       char ***params, char ***values);
+static int process_arg(const char *param, const char *value, char ***params, char ***values);
 static void add_to_env(char **params, char **values, char ***env);
 
-
 /*
  * Add -mca to the possible command line options list
  */
@@ -53,25 +50,32 @@ int mca_base_cmd_line_setup(opal_cmd_line_t *cmd)
     int ret = OPAL_SUCCESS;
 
     ret = opal_cmd_line_make_opt3(cmd, '\0', OPAL_MCA_CMD_LINE_ID, OPAL_MCA_CMD_LINE_ID, 2,
-                                  "Pass context-specific MCA parameters; they are considered global if --g"OPAL_MCA_CMD_LINE_ID" is not used and only one context is specified (arg0 is the parameter name; arg1 is the parameter value)");
+                                  "Pass context-specific MCA parameters; they are considered "
+                                  "global if --g" OPAL_MCA_CMD_LINE_ID
+                                  " is not used and only one context is specified (arg0 is the "
+                                  "parameter name; arg1 is the parameter value)");
     if (OPAL_SUCCESS != ret) {
         return ret;
     }
 
-    ret = opal_cmd_line_make_opt3(cmd, '\0', "g"OPAL_MCA_CMD_LINE_ID, "g"OPAL_MCA_CMD_LINE_ID, 2,
-                                  "Pass global MCA parameters that are applicable to all contexts (arg0 is the parameter name; arg1 is the parameter value)");
+    ret = opal_cmd_line_make_opt3(cmd, '\0', "g" OPAL_MCA_CMD_LINE_ID, "g" OPAL_MCA_CMD_LINE_ID, 2,
+                                  "Pass global MCA parameters that are applicable to all contexts "
+                                  "(arg0 is the parameter name; arg1 is the parameter value)");
 
     if (OPAL_SUCCESS != ret) {
         return ret;
     }
 
     {
-        opal_cmd_line_init_t entry =
-            {"mca_base_param_file_prefix", '\0', "am", NULL, 1,
-             NULL, OPAL_CMD_LINE_TYPE_STRING,
-             "Aggregate MCA parameter set file list",
-             OPAL_CMD_LINE_OTYPE_LAUNCH
-            };
+        opal_cmd_line_init_t entry = {"mca_base_param_file_prefix",
+                                      '\0',
+                                      "am",
+                                      NULL,
+                                      1,
+                                      NULL,
+                                      OPAL_CMD_LINE_TYPE_STRING,
+                                      "Aggregate MCA parameter set file list",
+                                      OPAL_CMD_LINE_OTYPE_LAUNCH};
         ret = opal_cmd_line_make_opt_mca(cmd, entry);
         if (OPAL_SUCCESS != ret) {
             return ret;
@@ -79,12 +83,15 @@ int mca_base_cmd_line_setup(opal_cmd_line_t *cmd)
     }
 
     {
-        opal_cmd_line_init_t entry =
-            {"mca_base_envar_file_prefix", '\0', "tune", NULL, 1,
-             NULL, OPAL_CMD_LINE_TYPE_STRING,
-             "Application profile options file list",
-             OPAL_CMD_LINE_OTYPE_DEBUG
-            };
+        opal_cmd_line_init_t entry = {"mca_base_envar_file_prefix",
+                                      '\0',
+                                      "tune",
+                                      NULL,
+                                      1,
+                                      NULL,
+                                      OPAL_CMD_LINE_TYPE_STRING,
+                                      "Application profile options file list",
+                                      OPAL_CMD_LINE_OTYPE_DEBUG};
         ret = opal_cmd_line_make_opt_mca(cmd, entry);
         if (OPAL_SUCCESS != ret) {
             return ret;
@@ -94,12 +101,10 @@ int mca_base_cmd_line_setup(opal_cmd_line_t *cmd)
     return ret;
 }
 
-
 /*
  * Look for and handle any -mca options on the command line
  */
-int mca_base_cmd_line_process_args(opal_cmd_line_t *cmd,
-                                   char ***context_env, char ***global_env)
+int mca_base_cmd_line_process_args(opal_cmd_line_t *cmd, char ***context_env, char ***global_env)
 {
     int i, num_insts, rc;
     char **params;
@@ -107,8 +112,8 @@ int mca_base_cmd_line_process_args(opal_cmd_line_t *cmd,
 
     /* If no relevant parameters were given, just return */
 
-    if (!opal_cmd_line_is_taken(cmd, OPAL_MCA_CMD_LINE_ID) &&
-        !opal_cmd_line_is_taken(cmd, "g"OPAL_MCA_CMD_LINE_ID)) {
+    if (!opal_cmd_line_is_taken(cmd, OPAL_MCA_CMD_LINE_ID)
+        && !opal_cmd_line_is_taken(cmd, "g" OPAL_MCA_CMD_LINE_ID)) {
         return OPAL_SUCCESS;
     }
 
@@ -117,9 +122,10 @@ int mca_base_cmd_line_process_args(opal_cmd_line_t *cmd,
     num_insts = opal_cmd_line_get_ninsts(cmd, OPAL_MCA_CMD_LINE_ID);
     params = values = NULL;
     for (i = 0; i < num_insts; ++i) {
-        if (OPAL_SUCCESS != (rc = process_arg(opal_cmd_line_get_param(cmd, OPAL_MCA_CMD_LINE_ID, i, 0),
-                                              opal_cmd_line_get_param(cmd, OPAL_MCA_CMD_LINE_ID, i, 1),
-                                              &params, &values))) {
+        if (OPAL_SUCCESS
+            != (rc = process_arg(opal_cmd_line_get_param(cmd, OPAL_MCA_CMD_LINE_ID, i, 0),
+                                 opal_cmd_line_get_param(cmd, OPAL_MCA_CMD_LINE_ID, i, 1), &params,
+                                 &values))) {
             return rc;
         }
     }
@@ -131,12 +137,13 @@ int mca_base_cmd_line_process_args(opal_cmd_line_t *cmd,
 
     /* Handle global parameters */
 
-    num_insts = opal_cmd_line_get_ninsts(cmd, "g"OPAL_MCA_CMD_LINE_ID);
+    num_insts = opal_cmd_line_get_ninsts(cmd, "g" OPAL_MCA_CMD_LINE_ID);
     params = values = NULL;
     for (i = 0; i < num_insts; ++i) {
-        if (OPAL_SUCCESS != (rc = process_arg(opal_cmd_line_get_param(cmd, "g"OPAL_MCA_CMD_LINE_ID, i, 0),
-                                              opal_cmd_line_get_param(cmd, "g"OPAL_MCA_CMD_LINE_ID, i, 1),
-                                              &params, &values))) {
+        if (OPAL_SUCCESS
+            != (rc = process_arg(opal_cmd_line_get_param(cmd, "g" OPAL_MCA_CMD_LINE_ID, i, 0),
+                                 opal_cmd_line_get_param(cmd, "g" OPAL_MCA_CMD_LINE_ID, i, 1),
+                                 &params, &values))) {
             return rc;
         }
     }
@@ -151,21 +158,18 @@ int mca_base_cmd_line_process_args(opal_cmd_line_t *cmd,
     return OPAL_SUCCESS;
 }
 
-
-
 /*
  * Process a single MCA argument.
  */
-static int process_arg(const char *param, const char *value,
-                       char ***params, char ***values)
+static int process_arg(const char *param, const char *value, char ***params, char ***values)
 {
     int i;
     char *p1;
 
     /* check for quoted value */
-    if ('\"' == value[0] && '\"' == value[strlen(value)-1]) {
+    if ('\"' == value[0] && '\"' == value[strlen(value) - 1]) {
         p1 = strdup(&value[1]);
-        p1[strlen(p1)-1] = '\0';
+        p1[strlen(p1) - 1] = '\0';
     } else {
         p1 = strdup(value);
     }
@@ -202,7 +206,6 @@ static int process_arg(const char *param, const char *value,
     return OPAL_SUCCESS;
 }
 
-
 static void add_to_env(char **params, char **values, char ***env)
 {
     int i;
@@ -212,7 +215,7 @@ static void add_to_env(char **params, char **values, char ***env)
        vars of the form OPAL_MCA_PREFIX*=value. */
 
     for (i = 0; NULL != params && NULL != params[i]; ++i) {
-        (void) mca_base_var_env_name (params[i], &name);
+        (void) mca_base_var_env_name(params[i], &name);
         opal_setenv(name, values[i], true, env);
         free(name);
     }
@@ -223,10 +226,10 @@ void mca_base_cmd_line_wrap_args(char **args)
     int i;
     char *tstr;
 
-    for (i=0; NULL != args && NULL != args[i]; i++) {
-        if (0 == strcmp(args[i], "-"OPAL_MCA_CMD_LINE_ID) ||
-            0 == strcmp(args[i], "--"OPAL_MCA_CMD_LINE_ID)) {
-            if (NULL == args[i+1] || NULL == args[i+2]) {
+    for (i = 0; NULL != args && NULL != args[i]; i++) {
+        if (0 == strcmp(args[i], "-" OPAL_MCA_CMD_LINE_ID)
+            || 0 == strcmp(args[i], "--" OPAL_MCA_CMD_LINE_ID)) {
+            if (NULL == args[i + 1] || NULL == args[i + 2]) {
                 /* this should be impossible as the error would
                  * have been detected well before here, but just
                  * be safe */
diff --git a/opal/mca/base/mca_base_component_compare.c b/opal/mca/base/mca_base_component_compare.c
index ee385ad57a7..ad0950f1fe4 100644
--- a/opal/mca/base/mca_base_component_compare.c
+++ b/opal/mca/base/mca_base_component_compare.c
@@ -21,8 +21,8 @@
 
 #include <string.h>
 
-#include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
+#include "opal/mca/mca.h"
 #include "opal/util/printf.h"
 
 /*
@@ -31,108 +31,87 @@
  * the types of the modules are the same.  Sort first by priority,
  * second by module name, third by module version.
  *
- * Note that we acutally want a *reverse* ordering here -- the al_*
+ * Note that we actually want a *reverse* ordering here -- the al_*
  * functions will put "smaller" items at the head, and "larger" items
  * at the tail.  Since we want the highest priority at the head, it
  * may help the gentle reader to consider this an inverse comparison.
  * :-)
  */
-int
-mca_base_component_compare_priority(mca_base_component_priority_list_item_t *a,
-                                    mca_base_component_priority_list_item_t *b)
+int mca_base_component_compare_priority(mca_base_component_priority_list_item_t *a,
+                                        mca_base_component_priority_list_item_t *b)
 {
-  /* First, compare the priorties */
-
-  if (a->cpli_priority > b->cpli_priority) {
-    return -1;
-  } else if (a->cpli_priority < b->cpli_priority) {
-    return 1;
-  } else {
-    return mca_base_component_compare(a->super.cli_component,
-                                      b->super.cli_component);
-  }
+    /* First, compare the priorties */
+
+    if (a->cpli_priority > b->cpli_priority) {
+        return -1;
+    } else if (a->cpli_priority < b->cpli_priority) {
+        return 1;
+    } else {
+        return mca_base_component_compare(a->super.cli_component, b->super.cli_component);
+    }
 }
 
-
-int mca_base_component_compare(const mca_base_component_t* aa,
-                               const mca_base_component_t* bb)
+int mca_base_component_compare(const mca_base_component_t *aa, const mca_base_component_t *bb)
 {
     int val;
 
-    val = strncmp(aa->mca_type_name, bb->mca_type_name,
-                  MCA_BASE_MAX_TYPE_NAME_LEN);
+    val = strncmp(aa->mca_type_name, bb->mca_type_name, MCA_BASE_MAX_TYPE_NAME_LEN);
     if (val != 0) {
-      return -val;
+        return -val;
     }
 
-    val = strncmp(aa->mca_component_name, bb->mca_component_name,
-                      MCA_BASE_MAX_COMPONENT_NAME_LEN);
+    val = strncmp(aa->mca_component_name, bb->mca_component_name, MCA_BASE_MAX_COMPONENT_NAME_LEN);
     if (val != 0) {
-      return -val;
+        return -val;
     }
 
     /* The names were equal, so compare the versions */
 
-    if (aa->mca_component_major_version >
-        bb->mca_component_major_version) {
-      return -1;
-    } else if (aa->mca_component_major_version <
-               bb->mca_component_major_version) {
-      return 1;
-    } else if (aa->mca_component_minor_version >
-               bb->mca_component_minor_version) {
-      return -1;
-    } else if (aa->mca_component_minor_version <
-               bb->mca_component_minor_version) {
-      return 1;
-    } else if (aa->mca_component_release_version >
-               bb->mca_component_release_version) {
-      return -1;
-    } else if (aa->mca_component_release_version <
-               bb->mca_component_release_version) {
-      return 1;
+    if (aa->mca_component_major_version > bb->mca_component_major_version) {
+        return -1;
+    } else if (aa->mca_component_major_version < bb->mca_component_major_version) {
+        return 1;
+    } else if (aa->mca_component_minor_version > bb->mca_component_minor_version) {
+        return -1;
+    } else if (aa->mca_component_minor_version < bb->mca_component_minor_version) {
+        return 1;
+    } else if (aa->mca_component_release_version > bb->mca_component_release_version) {
+        return -1;
+    } else if (aa->mca_component_release_version < bb->mca_component_release_version) {
+        return 1;
     }
 
     return 0;
 }
 
-
 /**
  * compare but exclude the release version - declare compatible
  * if the major/minor version are the same.
  */
-int mca_base_component_compatible(
-    const mca_base_component_t* aa,
-    const mca_base_component_t* bb)
+int mca_base_component_compatible(const mca_base_component_t *aa, const mca_base_component_t *bb)
 {
     int val;
 
-    val = strncmp(aa->mca_type_name, bb->mca_type_name,
-                  MCA_BASE_MAX_TYPE_NAME_LEN);
+    val = strncmp(aa->mca_type_name, bb->mca_type_name, MCA_BASE_MAX_TYPE_NAME_LEN);
     if (val != 0) {
-      return -val;
+        return -val;
     }
 
-    val = strncmp(aa->mca_component_name, bb->mca_component_name,
-                  MCA_BASE_MAX_COMPONENT_NAME_LEN);
+    val = strncmp(aa->mca_component_name, bb->mca_component_name, MCA_BASE_MAX_COMPONENT_NAME_LEN);
     if (val != 0) {
-      return -val;
+        return -val;
     }
 
     /* The names were equal, so compare the versions */
 
-    if (aa->mca_component_major_version >
-        bb->mca_component_major_version) {
-      return -1;
-    } else if (aa->mca_component_major_version <
-               bb->mca_component_major_version) {
-      return 1;
-    } else if (aa->mca_component_minor_version >
-               bb->mca_component_minor_version) {
-      return -1;
-    } else if (aa->mca_component_minor_version <
-               bb->mca_component_minor_version) {
-      return 1;
+    if (aa->mca_component_major_version > bb->mca_component_major_version) {
+        return -1;
+    } else if (aa->mca_component_major_version < bb->mca_component_major_version) {
+        return 1;
+    } else if (aa->mca_component_minor_version > bb->mca_component_minor_version) {
+        return -1;
+    } else if (aa->mca_component_minor_version < bb->mca_component_minor_version) {
+        return 1;
     }
     return 0;
 }
@@ -141,13 +120,12 @@ int mca_base_component_compatible(
  * Returns a string which represents the component name and version.
  * Has the form: comp_type.comp_name.major_version.minor_version
  */
-char * mca_base_component_to_string(const mca_base_component_t *a) {
-    char * str = NULL;
-    if(0 > opal_asprintf(&str, "%s.%s.%d.%d", a->mca_type_name,
-                    a->mca_component_name, a->mca_component_major_version,
-                    a->mca_component_minor_version)) {
+char *mca_base_component_to_string(const mca_base_component_t *a)
+{
+    char *str = NULL;
+    if (0 > opal_asprintf(&str, "%s.%s.%d.%d", a->mca_type_name, a->mca_component_name,
+                          a->mca_component_major_version, a->mca_component_minor_version)) {
         return NULL;
     }
     return str;
 }
-
diff --git a/opal/mca/base/mca_base_component_find.c b/opal/mca/base/mca_base_component_find.c
index 477fd3f16c4..3f22f57a4d8 100644
--- a/opal/mca/base/mca_base_component_find.c
+++ b/opal/mca/base/mca_base_component_find.c
@@ -28,39 +28,39 @@
 
 #include "opal_config.h"
 
-#include <stdio.h>
-#include <string.h>
 #include <ctype.h>
+#include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 #ifdef HAVE_SYS_STAT_H
-#include <sys/stat.h>
+#    include <sys/stat.h>
 #endif
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 #ifdef HAVE_SYS_PARAM_H
-#include <sys/param.h>
+#    include <sys/param.h>
 #endif
 #ifdef HAVE_NETDB_H
-#include <netdb.h>
+#    include <netdb.h>
 #endif
 
-#include "opal/runtime/opal.h"
-#include "opal/mca/installdirs/installdirs.h"
-#include "opal/util/opal_environ.h"
-#include "opal/util/output.h"
-#include "opal/util/argv.h"
-#include "opal/util/show_help.h"
 #include "opal/class/opal_list.h"
-#include "opal/mca/mca.h"
+#include "opal/constants.h"
 #include "opal/mca/base/base.h"
 #include "opal/mca/base/mca_base_alias.h"
 #include "opal/mca/base/mca_base_component_repository.h"
-#include "opal/constants.h"
 #include "opal/mca/dl/base/base.h"
+#include "opal/mca/installdirs/installdirs.h"
+#include "opal/mca/mca.h"
+#include "opal/runtime/opal.h"
+#include "opal/util/argv.h"
+#include "opal/util/opal_environ.h"
+#include "opal/util/output.h"
+#include "opal/util/show_help.h"
 
 #if OPAL_HAVE_DL_SUPPORT
 /*
@@ -71,7 +71,7 @@ static void find_dyn_components(const char *path, mca_base_framework_t *framewor
 
 #endif /* OPAL_HAVE_DL_SUPPORT */
 
-static int component_find_check (mca_base_framework_t *framework, char **requested_component_names);
+static int component_find_check(mca_base_framework_t *framework, char **requested_component_names);
 
 /*
  * Dummy structure for casting for open_only logic
@@ -86,11 +86,8 @@ typedef struct mca_base_open_only_dummy_component_t mca_base_open_only_dummy_com
 
 static char negate[] = "^";
 
-static bool use_component(const mca_base_framework_t *framework,
-                          const bool include_mode,
-                          const char **requested_component_names,
-                          const char *component_name);
-
+static bool use_component(const mca_base_framework_t *framework, const bool include_mode,
+                          const char **requested_component_names, const char *component_name);
 
 /*
  * Function to find as many components of a given type as possible.  This
@@ -101,8 +98,8 @@ static bool use_component(const mca_base_framework_t *framework,
  * Return one consolidated array of (mca_base_component_t*) pointing to all
  * available components.
  */
-int mca_base_component_find (const char *directory, mca_base_framework_t *framework,
-                             bool ignore_requested, bool open_dso_components)
+int mca_base_component_find(const char *directory, mca_base_framework_t *framework,
+                            bool ignore_requested, bool open_dso_components)
 {
     const mca_base_component_t **static_components = framework->framework_static_components;
     char **requested_component_names = NULL;
@@ -111,8 +108,8 @@ int mca_base_component_find (const char *directory, mca_base_framework_t *framew
     int ret;
 
     if (!ignore_requested) {
-        ret = mca_base_component_parse_requested (framework->framework_selection, &include_mode,
-                                                  &requested_component_names);
+        ret = mca_base_component_parse_requested(framework->framework_selection, &include_mode,
+                                                 &requested_component_names);
         if (OPAL_SUCCESS != ret) {
             return ret;
         }
@@ -120,10 +117,9 @@ int mca_base_component_find (const char *directory, mca_base_framework_t *framew
 
     /* Find all the components that were statically linked in */
     if (static_components) {
-        for (int i = 0 ; NULL != static_components[i]; ++i) {
-            if ( use_component(framework, include_mode,
-                               (const char**)requested_component_names,
-                               static_components[i]->mca_component_name) ) {
+        for (int i = 0; NULL != static_components[i]; ++i) {
+            if (use_component(framework, include_mode, (const char **) requested_component_names,
+                              static_components[i]->mca_component_name)) {
                 cli = OBJ_NEW(mca_base_component_list_item_t);
                 if (NULL == cli) {
                     ret = OPAL_ERR_OUT_OF_RESOURCE;
@@ -138,17 +134,17 @@ int mca_base_component_find (const char *directory, mca_base_framework_t *framew
 #if OPAL_HAVE_DL_SUPPORT
     /* Find any available dynamic components in the specified directory */
     if (open_dso_components && !mca_base_component_disable_dlopen) {
-        find_dyn_components(directory, framework, (const char**)requested_component_names,
+        find_dyn_components(directory, framework, (const char **) requested_component_names,
                             include_mode);
     } else {
-        opal_output_verbose (MCA_BASE_VERBOSE_INFO, 0,
+        opal_output_verbose(MCA_BASE_VERBOSE_INFO, 0,
                             "mca: base: component_find: dso loading for %s MCA components disabled",
                             framework->framework_name);
     }
 #endif
 
     if (include_mode) {
-        ret = component_find_check (framework, requested_component_names);
+        ret = component_find_check(framework, requested_component_names);
     } else {
         ret = OPAL_SUCCESS;
     }
@@ -169,7 +165,7 @@ int mca_base_component_find_finalize(void)
     return OPAL_SUCCESS;
 }
 
-int mca_base_components_filter (mca_base_framework_t *framework, uint32_t filter_flags)
+int mca_base_components_filter(mca_base_framework_t *framework, uint32_t filter_flags)
 {
     opal_list_t *components = &framework->framework_components;
     int output_id = framework->framework_output;
@@ -178,58 +174,56 @@ int mca_base_components_filter (mca_base_framework_t *framework, uint32_t filter
     bool include_mode, can_use;
     int ret;
 
-    assert (NULL != components);
+    assert(NULL != components);
 
     if (0 == filter_flags && NULL == framework->framework_selection) {
         return OPAL_SUCCESS;
     }
 
-    ret = mca_base_component_parse_requested (framework->framework_selection, &include_mode,
-                                              &requested_component_names);
+    ret = mca_base_component_parse_requested(framework->framework_selection, &include_mode,
+                                             &requested_component_names);
     if (OPAL_SUCCESS != ret) {
         return ret;
     }
 
-    OPAL_LIST_FOREACH_SAFE(cli, next, components, mca_base_component_list_item_t) {
+    OPAL_LIST_FOREACH_SAFE (cli, next, components, mca_base_component_list_item_t) {
         const mca_base_component_t *component = cli->cli_component;
-        mca_base_open_only_dummy_component_t *dummy =
-            (mca_base_open_only_dummy_component_t *) cli->cli_component;
+        mca_base_open_only_dummy_component_t *dummy = (mca_base_open_only_dummy_component_t *)
+                                                          cli->cli_component;
 
-        can_use = use_component (framework, include_mode, (const char **) requested_component_names,
-                                 cli->cli_component->mca_component_name);
+        can_use = use_component(framework, include_mode, (const char **) requested_component_names,
+                                cli->cli_component->mca_component_name);
 
         if (!can_use || (filter_flags & dummy->data.param_field) != filter_flags) {
-            if (can_use && (filter_flags & MCA_BASE_METADATA_PARAM_CHECKPOINT) &&
-                !(MCA_BASE_METADATA_PARAM_CHECKPOINT & dummy->data.param_field)) {
-                opal_output_verbose (MCA_BASE_VERBOSE_COMPONENT, output_id,
-                                     "mca: base: components_filter: "
-                                     "(%s) Component %s is *NOT* Checkpointable - Disabled",
-                                     component->reserved,
-                                     component->mca_component_name);
+            if (can_use && (filter_flags & MCA_BASE_METADATA_PARAM_CHECKPOINT)
+                && !(MCA_BASE_METADATA_PARAM_CHECKPOINT & dummy->data.param_field)) {
+                opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT, output_id,
+                                    "mca: base: components_filter: "
+                                    "(%s) Component %s is *NOT* Checkpointable - Disabled",
+                                    component->reserved, component->mca_component_name);
             }
 
-            opal_list_remove_item (components, &cli->super);
+            opal_list_remove_item(components, &cli->super);
 
-            mca_base_component_unload (component, output_id);
+            mca_base_component_unload(component, output_id);
 
             OBJ_RELEASE(cli);
         } else if (filter_flags & MCA_BASE_METADATA_PARAM_CHECKPOINT) {
-            opal_output_verbose (MCA_BASE_VERBOSE_COMPONENT, output_id,
-                                 "mca: base: components_filter: "
-                                 "(%s) Component %s is Checkpointable",
-                                 component->reserved,
-                                 component->mca_component_name);
+            opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT, output_id,
+                                "mca: base: components_filter: "
+                                "(%s) Component %s is Checkpointable",
+                                component->reserved, component->mca_component_name);
         }
     }
 
     if (include_mode) {
-        ret = component_find_check (framework, requested_component_names);
+        ret = component_find_check(framework, requested_component_names);
     } else {
         ret = OPAL_SUCCESS;
     }
 
     if (NULL != requested_component_names) {
-        opal_argv_free (requested_component_names);
+        opal_argv_free(requested_component_names);
     }
 
     return ret;
@@ -253,31 +247,30 @@ static void find_dyn_components(const char *path, mca_base_framework_t *framewor
     int ret;
 
     if (NULL != path) {
-        ret = mca_base_component_repository_add (path);
+        ret = mca_base_component_repository_add(path);
         if (OPAL_SUCCESS != ret) {
             return;
         }
     }
 
-    ret = mca_base_component_repository_get_components (framework, &dy_components);
+    ret = mca_base_component_repository_get_components(framework, &dy_components);
     if (OPAL_SUCCESS != ret) {
         return;
     }
 
     /* Iterate through the repository and find components that can be included */
-    OPAL_LIST_FOREACH(ri, dy_components, mca_base_component_repository_item_t) {
+    OPAL_LIST_FOREACH (ri, dy_components, mca_base_component_repository_item_t) {
         if (use_component(framework, include_mode, names, ri->ri_name)) {
-            mca_base_component_repository_open (framework, ri);
+            mca_base_component_repository_open(framework, ri);
         }
     }
 }
 
 #endif /* OPAL_HAVE_DL_SUPPORT */
 
-static bool component_in_list (const char **requested_component_names,
-                               const char *component_name)
+static bool component_in_list(const char **requested_component_names, const char *component_name)
 {
-    for (int i = 0 ; requested_component_names[i] ; ++i) {
+    for (int i = 0; requested_component_names[i]; ++i) {
         if (strcmp(component_name, requested_component_names[i]) == 0) {
             return true;
         }
@@ -286,10 +279,8 @@ static bool component_in_list (const char **requested_component_names,
     return false;
 }
 
-static bool use_component(const mca_base_framework_t *framework,
-                          const bool include_mode,
-                          const char **requested_component_names,
-                          const char *component_name)
+static bool use_component(const mca_base_framework_t *framework, const bool include_mode,
+                          const char **requested_component_names, const char *component_name)
 {
     /*
      * If no selection is specified then we use all components
@@ -299,14 +290,15 @@ static bool use_component(const mca_base_framework_t *framework,
         return true;
     }
 
-    bool found = component_in_list (requested_component_names, component_name);
+    bool found = component_in_list(requested_component_names, component_name);
 
     if (!found) {
-        const mca_base_alias_t *alias = mca_base_alias_lookup (framework->framework_project,
-                                                               framework->framework_name, component_name);
+        const mca_base_alias_t *alias = mca_base_alias_lookup(framework->framework_project,
+                                                              framework->framework_name,
+                                                              component_name);
         if (alias) {
-            OPAL_LIST_FOREACH_DECL(alias_item, &alias->component_aliases, mca_base_alias_item_t) {
-                found = component_in_list (requested_component_names, alias_item->component_alias);
+            OPAL_LIST_FOREACH_DECL (alias_item, &alias->component_aliases, mca_base_alias_item_t) {
+                found = component_in_list(requested_component_names, alias_item->component_alias);
                 if (found) {
                     break;
                 }
@@ -332,7 +324,7 @@ static bool use_component(const mca_base_framework_t *framework,
 
 /* Ensure that *all* requested components exist.  Print a warning
    and abort if they do not. */
-static int component_find_check (mca_base_framework_t *framework, char **requested_component_names)
+static int component_find_check(mca_base_framework_t *framework, char **requested_component_names)
 {
     opal_list_t *components = &framework->framework_components;
 
@@ -340,21 +332,23 @@ static int component_find_check (mca_base_framework_t *framework, char **request
         return OPAL_SUCCESS;
     }
 
-    for (int i = 0 ; requested_component_names[i] ; ++i) {
+    for (int i = 0; requested_component_names[i]; ++i) {
         bool found = false;
 
-        OPAL_LIST_FOREACH_DECL(cli, components, mca_base_component_list_item_t) {
-            if (0 == strcmp (requested_component_names[i], cli->cli_component->mca_component_name)) {
+        OPAL_LIST_FOREACH_DECL (cli, components, mca_base_component_list_item_t) {
+            if (0 == strcmp(requested_component_names[i], cli->cli_component->mca_component_name)) {
                 found = true;
                 break;
             }
 
-            const mca_base_alias_t *alias = mca_base_alias_lookup (framework->framework_project,
-                                                                   framework->framework_name,
-                                                                   cli->cli_component->mca_component_name);
+            const mca_base_alias_t *alias = mca_base_alias_lookup(framework->framework_project,
+                                                                  framework->framework_name,
+                                                                  cli->cli_component
+                                                                      ->mca_component_name);
             if (alias) {
-                OPAL_LIST_FOREACH_DECL(alias_item, &alias->component_aliases, mca_base_alias_item_t) {
-                    if (0 == strcmp (requested_component_names[i], alias_item->component_alias)) {
+                OPAL_LIST_FOREACH_DECL (alias_item, &alias->component_aliases,
+                                        mca_base_alias_item_t) {
+                    if (0 == strcmp(requested_component_names[i], alias_item->component_alias)) {
                         found = true;
                         break;
                     }
@@ -368,9 +362,8 @@ static int component_find_check (mca_base_framework_t *framework, char **request
         if (!found) {
             const char *h;
             h = opal_gethostname();
-            opal_show_help("help-mca-base.txt",
-                           "find-available:not-valid", true,
-                           h, framework->framework_name, requested_component_names[i]);
+            opal_show_help("help-mca-base.txt", "find-available:not-valid", true, h,
+                           framework->framework_name, requested_component_names[i]);
             return OPAL_ERR_NOT_FOUND;
         }
     }
@@ -378,8 +371,8 @@ static int component_find_check (mca_base_framework_t *framework, char **request
     return OPAL_SUCCESS;
 }
 
-int mca_base_component_parse_requested (const char *requested, bool *include_mode,
-                                        char ***requested_component_names)
+int mca_base_component_parse_requested(const char *requested, bool *include_mode,
+                                       char ***requested_component_names)
 {
     const char *requested_orig = requested;
 
@@ -387,7 +380,7 @@ int mca_base_component_parse_requested (const char *requested, bool *include_mod
     *include_mode = true;
 
     /* See if the user requested anything */
-    if (NULL == requested || 0 == strlen (requested)) {
+    if (NULL == requested || 0 == strlen(requested)) {
         return OPAL_SUCCESS;
     }
 
@@ -397,14 +390,13 @@ int mca_base_component_parse_requested (const char *requested, bool *include_mod
     *include_mode = requested[0] != negate[0];
 
     /* skip over all negate symbols at the beginning */
-    requested += strspn (requested, negate);
+    requested += strspn(requested, negate);
 
     /* Double check to ensure that the user did not specify the negate
        character anywhere else in the value. */
-    if (NULL != strstr (requested, negate)) {
-        opal_show_help("help-mca-base.txt",
-                       "framework-param:too-many-negates",
-                       true, requested_orig);
+    if (NULL != strstr(requested, negate)) {
+        opal_show_help("help-mca-base.txt", "framework-param:too-many-negates", true,
+                       requested_orig);
         return OPAL_ERROR;
     }
 
diff --git a/opal/mca/base/mca_base_component_repository.c b/opal/mca/base/mca_base_component_repository.c
index 7c677ea186f..f359cfa9019 100644
--- a/opal/mca/base/mca_base_component_repository.c
+++ b/opal/mca/base/mca_base_component_repository.c
@@ -10,7 +10,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2008-2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2008-2022 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2015      Research Organization for Information Science
@@ -24,28 +24,27 @@
  * $HEADER$
  */
 
-
 #include "opal_config.h"
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
-#include <string.h>
-#include <stdlib.h>
 #include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 
+#include "opal/class/opal_hash_table.h"
 #include "opal/class/opal_list.h"
-#include "opal/mca/mca.h"
+#include "opal/constants.h"
 #include "opal/mca/base/base.h"
 #include "opal/mca/base/mca_base_component_repository.h"
 #include "opal/mca/dl/base/base.h"
-#include "opal/constants.h"
-#include "opal/class/opal_hash_table.h"
+#include "opal/mca/mca.h"
 #include "opal/util/basename.h"
-#include "opal/util/string_copy.h"
 #include "opal/util/printf.h"
+#include "opal/util/string_copy.h"
 
 #if OPAL_HAVE_DL_SUPPORT
 
@@ -54,17 +53,15 @@
  */
 static void ri_constructor(mca_base_component_repository_item_t *ri);
 static void ri_destructor(mca_base_component_repository_item_t *ri);
-OBJ_CLASS_INSTANCE(mca_base_component_repository_item_t, opal_list_item_t,
-                   ri_constructor, ri_destructor);
+OBJ_CLASS_INSTANCE(mca_base_component_repository_item_t, opal_list_item_t, ri_constructor,
+                   ri_destructor);
 
 #endif /* OPAL_HAVE_DL_SUPPORT */
 
 static void clf_constructor(opal_object_t *obj);
 static void clf_destructor(opal_object_t *obj);
 
-OBJ_CLASS_INSTANCE(mca_base_failed_component_t, opal_list_item_t,
-                   clf_constructor, clf_destructor);
-
+OBJ_CLASS_INSTANCE(mca_base_failed_component_t, opal_list_item_t, clf_constructor, clf_destructor);
 
 static void clf_constructor(opal_object_t *obj)
 {
@@ -77,7 +74,7 @@ static void clf_destructor(opal_object_t *obj)
 {
     mca_base_failed_component_t *cli = (mca_base_failed_component_t *) obj;
     cli->comp = NULL;
-    if( NULL != cli->error_msg ) {
+    if (NULL != cli->error_msg) {
         free(cli->error_msg);
         cli->error_msg = NULL;
     }
@@ -88,16 +85,15 @@ static void clf_destructor(opal_object_t *obj)
  */
 static bool initialized = false;
 
-
 #if OPAL_HAVE_DL_SUPPORT
 
 static opal_hash_table_t mca_base_component_repository;
 
 /* two-level macro for stringifying a number */
-#define STRINGIFYX(x) #x
-#define STRINGIFY(x) STRINGIFYX(x)
+#    define STRINGIFYX(x) #    x
+#    define STRINGIFY(x)  STRINGIFYX(x)
 
-static int process_repository_item (const char *filename, void *data)
+static int process_repository_item(const char *filename, void *data)
 {
     char name[MCA_BASE_MAX_COMPONENT_NAME_LEN + 1];
     char type[MCA_BASE_MAX_TYPE_NAME_LEN + 1];
@@ -106,21 +102,23 @@ static int process_repository_item (const char *filename, void *data)
     char *base;
     int ret;
 
-    base = opal_basename (filename);
+    base = opal_basename(filename);
     if (NULL == base) {
         return OPAL_ERROR;
     }
 
     /* check if the plugin has the appropriate prefix */
-    if (0 != strncmp (base, "mca_", 4)) {
-        free (base);
+    if (0 != strncmp(base, "mca_", 4)) {
+        free(base);
         return OPAL_SUCCESS;
     }
 
     /* read framework and component names. framework names may not include an _
      * but component names may */
-    ret = sscanf (base, "mca_%" STRINGIFY(MCA_BASE_MAX_TYPE_NAME_LEN) "[^_]_%"
-                  STRINGIFY(MCA_BASE_MAX_COMPONENT_NAME_LEN) "s", type, name);
+    ret = sscanf(base,
+                 "mca_%" STRINGIFY(MCA_BASE_MAX_TYPE_NAME_LEN) "[^_]_%" STRINGIFY(
+                     MCA_BASE_MAX_COMPONENT_NAME_LEN) "s",
+                 type, name);
     if (0 > ret) {
         /* does not patch the expected template. skip */
         free(base);
@@ -128,52 +126,52 @@ static int process_repository_item (const char *filename, void *data)
     }
 
     /* lookup the associated framework list and create if it doesn't already exist */
-    ret = opal_hash_table_get_value_ptr (&mca_base_component_repository, type,
-                                         strlen (type), (void **) &component_list);
+    ret = opal_hash_table_get_value_ptr(&mca_base_component_repository, type, strlen(type),
+                                        (void **) &component_list);
     if (OPAL_SUCCESS != ret) {
         component_list = OBJ_NEW(opal_list_t);
         if (NULL == component_list) {
-            free (base);
+            free(base);
             /* OOM. nothing to do but fail */
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
 
-        ret = opal_hash_table_set_value_ptr (&mca_base_component_repository, type,
-                                             strlen (type), (void *) component_list);
+        ret = opal_hash_table_set_value_ptr(&mca_base_component_repository, type, strlen(type),
+                                            (void *) component_list);
         if (OPAL_SUCCESS != ret) {
-            free (base);
+            free(base);
             OBJ_RELEASE(component_list);
             return ret;
         }
     }
 
     /* check for duplicate components */
-    OPAL_LIST_FOREACH(ri, component_list, mca_base_component_repository_item_t) {
-        if (0 == strcmp (ri->ri_name, name)) {
+    OPAL_LIST_FOREACH (ri, component_list, mca_base_component_repository_item_t) {
+        if (0 == strcmp(ri->ri_name, name)) {
             /* already scanned this component */
-            free (base);
+            free(base);
             return OPAL_SUCCESS;
         }
     }
 
     ri = OBJ_NEW(mca_base_component_repository_item_t);
     if (NULL == ri) {
-        free (base);
+        free(base);
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
     ri->ri_base = base;
 
-    ri->ri_path = strdup (filename);
+    ri->ri_path = strdup(filename);
     if (NULL == ri->ri_path) {
         OBJ_RELEASE(ri);
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
-    opal_string_copy (ri->ri_type, type, MCA_BASE_MAX_TYPE_NAME_LEN);
-    opal_string_copy (ri->ri_name, name, MCA_BASE_MAX_COMPONENT_NAME_LEN);
+    opal_string_copy(ri->ri_type, type, MCA_BASE_MAX_TYPE_NAME_LEN);
+    opal_string_copy(ri->ri_name, name, MCA_BASE_MAX_COMPONENT_NAME_LEN);
 
-    opal_list_append (component_list, &ri->super);
+    opal_list_append(component_list, &ri->super);
 
     return OPAL_SUCCESS;
 }
@@ -184,7 +182,7 @@ static int file_exists(const char *filename, const char *ext)
     int ret;
 
     if (NULL == ext) {
-        return access (filename, F_OK) == 0;
+        return access(filename, F_OK) == 0;
     }
 
     ret = opal_asprintf(&final, "%s.%s", filename, ext);
@@ -192,14 +190,14 @@ static int file_exists(const char *filename, const char *ext)
         return 0;
     }
 
-    ret = access (final, F_OK);
+    ret = access(final, F_OK);
     free(final);
     return (0 == ret);
 }
 
 #endif /* OPAL_HAVE_DL_SUPPORT */
 
-int mca_base_component_repository_add (const char *path)
+int mca_base_component_repository_add(const char *path)
 {
 #if OPAL_HAVE_DL_SUPPORT
     char *path_to_use = NULL, *dir, *ctx;
@@ -210,92 +208,98 @@ int mca_base_component_repository_add (const char *path)
         return OPAL_SUCCESS;
     }
 
-    path_to_use = strdup (path);
+    path_to_use = strdup(path);
 
-    dir = strtok_r (path_to_use, sep, &ctx);
+    dir = strtok_r(path_to_use, sep, &ctx);
+    if (NULL == dir) {
+        goto done;
+    }
     do {
         if ((0 == strcmp(dir, "USER_DEFAULT") || 0 == strcmp(dir, "USR_DEFAULT"))
             && NULL != mca_base_user_default_path) {
             dir = mca_base_user_default_path;
-        } else if (0 == strcmp(dir, "SYS_DEFAULT") ||
-                   0 == strcmp(dir, "SYSTEM_DEFAULT")) {
+        } else if (0 == strcmp(dir, "SYS_DEFAULT") || 0 == strcmp(dir, "SYSTEM_DEFAULT")) {
             dir = mca_base_system_default_path;
         }
 
         if (0 != opal_dl_foreachfile(dir, process_repository_item, NULL)) {
             break;
         }
-    } while (NULL != (dir = strtok_r (NULL, sep, &ctx)));
+    } while (NULL != (dir = strtok_r(NULL, sep, &ctx)));
 
-    free (path_to_use);
+ done:
+    free(path_to_use);
 
 #endif /* OPAL_HAVE_DL_SUPPORT */
 
     return OPAL_SUCCESS;
 }
 
-
 /*
  * Initialize the repository
  */
 int mca_base_component_repository_init(void)
 {
-  /* Setup internal structures */
+    /* Setup internal structures */
 
-  if (!initialized) {
+    if (!initialized) {
 #if OPAL_HAVE_DL_SUPPORT
 
-    /* Initialize the dl framework */
-    int ret = mca_base_framework_open(&opal_dl_base_framework, 0);
-    if (OPAL_SUCCESS != ret) {
-        opal_output(0, "%s %d:%s failed -- process will likely abort (open the dl framework returned %d instead of OPAL_SUCCESS)\n",
-                    __FILE__, __LINE__, __func__, ret);
-        return ret;
-    }
-    opal_dl_base_select();
+        /* Initialize the dl framework */
+        int ret = mca_base_framework_open(&opal_dl_base_framework, 0);
+        if (OPAL_SUCCESS != ret) {
+            opal_output(0,
+                        "%s %d:%s failed -- process will likely abort (open the dl framework "
+                        "returned %d instead of OPAL_SUCCESS)\n",
+                        __FILE__, __LINE__, __func__, ret);
+            return ret;
+        }
+        opal_dl_base_select();
 
-    OBJ_CONSTRUCT(&mca_base_component_repository, opal_hash_table_t);
-    ret = opal_hash_table_init (&mca_base_component_repository, 128);
-    if (OPAL_SUCCESS != ret) {
-        (void) mca_base_framework_close (&opal_dl_base_framework);
-        return ret;
-    }
+        OBJ_CONSTRUCT(&mca_base_component_repository, opal_hash_table_t);
+        ret = opal_hash_table_init(&mca_base_component_repository, 128);
+        if (OPAL_SUCCESS != ret) {
+            (void) mca_base_framework_close(&opal_dl_base_framework);
+            return ret;
+        }
 
-    ret = mca_base_component_repository_add (mca_base_component_path);
-    if (OPAL_SUCCESS != ret) {
-        OBJ_DESTRUCT(&mca_base_component_repository);
-        (void) mca_base_framework_close (&opal_dl_base_framework);
-        return ret;
-    }
+        ret = mca_base_component_repository_add(mca_base_component_path);
+        if (OPAL_SUCCESS != ret) {
+            OBJ_DESTRUCT(&mca_base_component_repository);
+            (void) mca_base_framework_close(&opal_dl_base_framework);
+            return ret;
+        }
 #endif
 
-    initialized = true;
-  }
+        initialized = true;
+    }
 
-  /* All done */
+    /* All done */
 
-  return OPAL_SUCCESS;
+    return OPAL_SUCCESS;
 }
 
-int mca_base_component_repository_get_components (mca_base_framework_t *framework,
-                                                  opal_list_t **framework_components)
+int mca_base_component_repository_get_components(mca_base_framework_t *framework,
+                                                 opal_list_t **framework_components)
 {
     *framework_components = NULL;
 #if OPAL_HAVE_DL_SUPPORT
-    return opal_hash_table_get_value_ptr (&mca_base_component_repository, framework->framework_name,
-                                          strlen (framework->framework_name), (void **) framework_components);
+    return opal_hash_table_get_value_ptr(&mca_base_component_repository, framework->framework_name,
+                                         strlen(framework->framework_name),
+                                         (void **) framework_components);
 #endif
     return OPAL_ERR_NOT_FOUND;
 }
 
 #if OPAL_HAVE_DL_SUPPORT
-static void mca_base_component_repository_release_internal (mca_base_component_repository_item_t *ri) {
+static void mca_base_component_repository_release_internal(mca_base_component_repository_item_t *ri)
+{
     int group_id;
 
-    group_id = mca_base_var_group_find (NULL, ri->ri_type, ri->ri_name);
+    group_id = mca_base_var_group_find(NULL, ri->ri_type, ri->ri_name);
     if (0 <= group_id) {
         /* ensure all variables are deregistered before we dlclose the component */
-        mca_base_var_group_deregister (group_id);
+        mca_base_var_group_deregister(group_id);
     }
 
     /* Close the component (and potentially unload it from memory */
@@ -307,21 +311,21 @@ static void mca_base_component_repository_release_internal (mca_base_component_r
 #endif
 
 #if OPAL_HAVE_DL_SUPPORT
-static mca_base_component_repository_item_t *find_component (const char *type, const char *name)
+static mca_base_component_repository_item_t *find_component(const char *type, const char *name)
 {
     mca_base_component_repository_item_t *ri;
     opal_list_t *component_list;
     int ret;
 
-    ret = opal_hash_table_get_value_ptr (&mca_base_component_repository, type,
-                                         strlen (type), (void **) &component_list);
+    ret = opal_hash_table_get_value_ptr(&mca_base_component_repository, type, strlen(type),
+                                        (void **) &component_list);
     if (OPAL_SUCCESS != ret) {
         /* component does not exist in the repository */
         return NULL;
     }
 
-    OPAL_LIST_FOREACH(ri, component_list, mca_base_component_repository_item_t) {
-        if (0 == strcmp (ri->ri_name, name)) {
+    OPAL_LIST_FOREACH (ri, component_list, mca_base_component_repository_item_t) {
+        if (0 == strcmp(ri->ri_name, name)) {
             return ri;
         }
     }
@@ -335,14 +339,14 @@ void mca_base_component_repository_release(const mca_base_component_t *component
 #if OPAL_HAVE_DL_SUPPORT
     mca_base_component_repository_item_t *ri;
 
-    ri = find_component (component->mca_type_name, component->mca_component_name);
+    ri = find_component(component->mca_type_name, component->mca_component_name);
     if (NULL != ri && !(--ri->ri_refcnt)) {
-        mca_base_component_repository_release_internal (ri);
+        mca_base_component_repository_release_internal(ri);
     }
 #endif
 }
 
-int mca_base_component_repository_retain_component (const char *type, const char *name)
+int mca_base_component_repository_retain_component(const char *type, const char *name)
 {
 #if OPAL_HAVE_DL_SUPPORT
     mca_base_component_repository_item_t *ri = find_component(type, name);
@@ -358,8 +362,8 @@ int mca_base_component_repository_retain_component (const char *type, const char
 #endif
 }
 
-int mca_base_component_repository_open (mca_base_framework_t *framework,
-                                        mca_base_component_repository_item_t *ri)
+int mca_base_component_repository_open(mca_base_framework_t *framework,
+                                       mca_base_component_repository_item_t *ri)
 {
 #if OPAL_HAVE_DL_SUPPORT
     mca_base_component_t *component_struct;
@@ -367,10 +371,13 @@ int mca_base_component_repository_open (mca_base_framework_t *framework,
     char *struct_name = NULL;
     int vl, ret;
 
-    opal_output_verbose(MCA_BASE_VERBOSE_INFO, 0, "mca_base_component_repository_open: examining dynamic "
-                        "%s MCA component \"%s\" at path %s", ri->ri_type, ri->ri_name, ri->ri_path);
+    opal_output_verbose(MCA_BASE_VERBOSE_INFO, 0,
+                        "mca_base_component_repository_open: examining dynamic "
+                        "%s MCA component \"%s\" at path %s",
+                        ri->ri_type, ri->ri_name, ri->ri_path);
 
-    vl = mca_base_component_show_load_errors ? MCA_BASE_VERBOSE_ERROR : MCA_BASE_VERBOSE_INFO;
+    vl = mca_base_show_load_errors(ri->ri_type,
+                                   ri->ri_name) ? MCA_BASE_VERBOSE_ERROR : MCA_BASE_VERBOSE_INFO;
 
     /* Ensure that this component is not already loaded (should only happen
        if it was statically loaded).  It's an error if it's already
@@ -378,9 +385,10 @@ int mca_base_component_repository_open (mca_base_framework_t *framework,
        Hence, returning OPAL_ERR_PARAM indicates that the *file* failed
        to load, not the component. */
 
-    OPAL_LIST_FOREACH(mitem, &framework->framework_components, mca_base_component_list_item_t) {
+    OPAL_LIST_FOREACH (mitem, &framework->framework_components, mca_base_component_list_item_t) {
         if (0 == strcmp(mitem->cli_component->mca_component_name, ri->ri_name)) {
-            opal_output_verbose (MCA_BASE_VERBOSE_INFO, 0, "mca_base_component_repository_open: already loaded (ignored)");
+            opal_output_verbose(MCA_BASE_VERBOSE_INFO, 0,
+                                "mca_base_component_repository_open: already loaded (ignored)");
             return OPAL_ERR_BAD_PARAM;
         }
     }
@@ -389,23 +397,25 @@ int mca_base_component_repository_open (mca_base_framework_t *framework,
     mitem = NULL;
 
     if (NULL != ri->ri_dlhandle) {
-        opal_output_verbose (MCA_BASE_VERBOSE_INFO, 0, "mca_base_component_repository_open: already loaded. returning cached component");
+        opal_output_verbose(
+            MCA_BASE_VERBOSE_INFO, 0,
+            "mca_base_component_repository_open: already loaded. returning cached component");
         mitem = OBJ_NEW(mca_base_component_list_item_t);
         if (NULL == mitem) {
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
 
         mitem->cli_component = ri->ri_component_struct;
-        opal_list_append (&framework->framework_components, &mitem->super);
+        opal_list_append(&framework->framework_components, &mitem->super);
 
         return OPAL_SUCCESS;
     }
 
-    if (0 != strcmp (ri->ri_type, framework->framework_name)) {
+    if (0 != strcmp(ri->ri_type, framework->framework_name)) {
         /* shouldn't happen. attempting to open a component belonging to
          * another framework. if this happens it is likely a MCA base
          * bug so assert */
-        assert (0);
+        assert(0);
         return OPAL_ERR_NOT_SUPPORTED;
     }
 
@@ -422,17 +432,16 @@ int mca_base_component_repository_open (mca_base_framework_t *framework,
            (e.g., missing symbol) -- do some simple huersitics and if
            the file [probably] does exist, print a slightly better error
            message. */
-        if (0 == strcasecmp("file not found", err_msg) &&
-            (file_exists(ri->ri_path, "lo") ||
-             file_exists(ri->ri_path, "so") ||
-             file_exists(ri->ri_path, "dylib") ||
-             file_exists(ri->ri_path, "dll"))) {
+        if (0 == strcasecmp("file not found", err_msg)
+            && (file_exists(ri->ri_path, "lo") || file_exists(ri->ri_path, "so")
+                || file_exists(ri->ri_path, "dylib") || file_exists(ri->ri_path, "dll"))) {
             err_msg = "perhaps a missing symbol, or compiled for a different version of Open MPI?";
         }
-        opal_output_verbose(vl, 0, "mca_base_component_repository_open: unable to open %s: %s (ignored)",
+        opal_output_verbose(vl, 0,
+                            "mca_base_component_repository_open: unable to open %s: %s (ignored)",
                             ri->ri_base, err_msg);
 
-        if( mca_base_component_track_load_errors ) {
+        if (mca_base_component_track_load_errors) {
             mca_base_failed_component_t *f_comp = OBJ_NEW(mca_base_failed_component_t);
             f_comp->comp = ri;
             opal_asprintf(&(f_comp->error_msg), "%s", err_msg);
@@ -446,7 +455,7 @@ int mca_base_component_repository_open (mca_base_framework_t *framework,
        Malloc out enough space for it. */
 
     do {
-        ret = opal_asprintf (&struct_name, "mca_%s_%s_component", ri->ri_type, ri->ri_name);
+        ret = opal_asprintf(&struct_name, "mca_%s_%s_component", ri->ri_type, ri->ri_name);
         if (0 > ret) {
             ret = OPAL_ERR_OUT_OF_RESOURCE;
             break;
@@ -459,43 +468,49 @@ int mca_base_component_repository_open (mca_base_framework_t *framework,
         }
 
         err_msg = NULL;
-        ret = opal_dl_lookup(ri->ri_dlhandle, struct_name, (void**) &component_struct, &err_msg);
+        ret = opal_dl_lookup(ri->ri_dlhandle, struct_name, (void **) &component_struct, &err_msg);
         if (OPAL_SUCCESS != ret || NULL == component_struct) {
             if (NULL == err_msg) {
                 err_msg = "opal_dl_loookup() error message was NULL!";
             }
-            opal_output_verbose(vl, 0, "mca_base_component_repository_open: \"%s\" does not appear to be a valid "
-                                "%s MCA dynamic component (ignored): %s. ret %d", ri->ri_base, ri->ri_type, err_msg, ret);
+            opal_output_verbose(
+                vl, 0,
+                "mca_base_component_repository_open: \"%s\" does not appear to be a valid "
+                "%s MCA dynamic component (ignored): %s. ret %d",
+                ri->ri_base, ri->ri_type, err_msg, ret);
 
             ret = OPAL_ERR_BAD_PARAM;
             break;
         }
 
         /* done with the structure name */
-        free (struct_name);
+        free(struct_name);
         struct_name = NULL;
 
         /* We found the public struct.  Make sure its MCA major.minor
            version is the same as ours. TODO -- add checks for project version (from framework) */
-        if (!(MCA_BASE_VERSION_MAJOR == component_struct->mca_major_version &&
-              MCA_BASE_VERSION_MINOR == component_struct->mca_minor_version)) {
-            opal_output_verbose(vl, 0, "mca_base_component_repository_open: %s \"%s\" uses an MCA interface that is "
-                                "not recognized (component MCA v%d.%d.%d != supported MCA v%d.%d.%d) -- ignored",
-                                ri->ri_type, ri->ri_path, component_struct->mca_major_version,
-                                component_struct->mca_minor_version, component_struct->mca_release_version,
-                                MCA_BASE_VERSION_MAJOR, MCA_BASE_VERSION_MINOR, MCA_BASE_VERSION_RELEASE);
+        if (!(MCA_BASE_VERSION_MAJOR == component_struct->mca_major_version
+              && MCA_BASE_VERSION_MINOR == component_struct->mca_minor_version)) {
+            opal_output_verbose(
+                vl, 0,
+                "mca_base_component_repository_open: %s \"%s\" uses an MCA interface that is "
+                "not recognized (component MCA v%d.%d.%d != supported MCA v%d.%d.%d) -- ignored",
+                ri->ri_type, ri->ri_path, component_struct->mca_major_version,
+                component_struct->mca_minor_version, component_struct->mca_release_version,
+                MCA_BASE_VERSION_MAJOR, MCA_BASE_VERSION_MINOR, MCA_BASE_VERSION_RELEASE);
             ret = OPAL_ERR_BAD_PARAM;
             break;
         }
 
         /* Also check that the component struct framework and component
            names match the expected names from the filename */
-        if (0 != strcmp(component_struct->mca_type_name, ri->ri_type) ||
-            0 != strcmp(component_struct->mca_component_name, ri->ri_name)) {
-            opal_output_verbose(vl, 0, "Component file data does not match filename: %s (%s / %s) != %s %s -- ignored",
-                                ri->ri_path, ri->ri_type, ri->ri_name,
-                                component_struct->mca_type_name,
-                                component_struct->mca_component_name);
+        if (0 != strcmp(component_struct->mca_type_name, ri->ri_type)
+            || 0 != strcmp(component_struct->mca_component_name, ri->ri_name)) {
+            opal_output_verbose(
+                vl, 0,
+                "Component file data does not match filename: %s (%s / %s) != %s %s -- ignored",
+                ri->ri_path, ri->ri_type, ri->ri_name, component_struct->mca_type_name,
+                component_struct->mca_component_name);
             ret = OPAL_ERR_BAD_PARAM;
             break;
         }
@@ -507,8 +522,10 @@ int mca_base_component_repository_open (mca_base_framework_t *framework,
         ri->ri_refcnt = 1;
         opal_list_append(&framework->framework_components, &mitem->super);
 
-        opal_output_verbose (MCA_BASE_VERBOSE_INFO, 0, "mca_base_component_repository_open: opened dynamic %s MCA "
-                             "component \"%s\"", ri->ri_type, ri->ri_name);
+        opal_output_verbose(MCA_BASE_VERBOSE_INFO, 0,
+                            "mca_base_component_repository_open: opened dynamic %s MCA "
+                            "component \"%s\"",
+                            ri->ri_type, ri->ri_name);
 
         return OPAL_SUCCESS;
     } while (0);
@@ -518,10 +535,10 @@ int mca_base_component_repository_open (mca_base_framework_t *framework,
     }
 
     if (struct_name) {
-        free (struct_name);
+        free(struct_name);
     }
 
-    opal_dl_close (ri->ri_dlhandle);
+    opal_dl_close(ri->ri_dlhandle);
     ri->ri_dlhandle = NULL;
 
     return ret;
@@ -549,13 +566,12 @@ void mca_base_component_repository_finalize(void)
     size_t key_size;
     int ret;
 
-    ret = opal_hash_table_get_first_key_ptr (&mca_base_component_repository, &key, &key_size,
-                                             (void **) &component_list, &node);
+    ret = opal_hash_table_get_first_key_ptr(&mca_base_component_repository, &key, &key_size,
+                                            (void **) &component_list, &node);
     while (OPAL_SUCCESS == ret) {
         OPAL_LIST_RELEASE(component_list);
-        ret = opal_hash_table_get_next_key_ptr (&mca_base_component_repository, &key,
-                                                &key_size, (void **) &component_list,
-                                                node, &node);
+        ret = opal_hash_table_get_next_key_ptr(&mca_base_component_repository, &key, &key_size,
+                                               (void **) &component_list, node, &node);
     }
 
     (void) mca_base_framework_close(&opal_dl_base_framework);
@@ -568,7 +584,7 @@ void mca_base_component_repository_finalize(void)
 /*
  * Basic sentinel values, and construct the inner list
  */
-static void ri_constructor (mca_base_component_repository_item_t *ri)
+static void ri_constructor(mca_base_component_repository_item_t *ri)
 {
     memset(ri->ri_type, 0, sizeof(ri->ri_type));
     ri->ri_dlhandle = NULL;
@@ -576,14 +592,13 @@ static void ri_constructor (mca_base_component_repository_item_t *ri)
     ri->ri_path = NULL;
 }
 
-
 /*
  * Close a component
  */
-static void ri_destructor (mca_base_component_repository_item_t *ri)
+static void ri_destructor(mca_base_component_repository_item_t *ri)
 {
     /* dlclose the component if it is still open */
-    mca_base_component_repository_release_internal (ri);
+    mca_base_component_repository_release_internal(ri);
 
     /* It should be obvious, but I'll state it anyway because it bit me
        during debugging: after the dlclose(), the mca_base_component_t
@@ -591,11 +606,11 @@ static void ri_destructor (mca_base_component_repository_item_t *ri)
        unloaded from memory.  So don't try to use it.  :-) */
 
     if (ri->ri_path) {
-        free (ri->ri_path);
+        free(ri->ri_path);
     }
 
     if (ri->ri_base) {
-        free (ri->ri_base);
+        free(ri->ri_base);
     }
 }
 
diff --git a/opal/mca/base/mca_base_component_repository.h b/opal/mca/base/mca_base_component_repository.h
index 08babe70511..66822edcb43 100644
--- a/opal/mca/base/mca_base_component_repository.h
+++ b/opal/mca/base/mca_base_component_repository.h
@@ -39,8 +39,8 @@
 
 #include "opal_config.h"
 
-#include "opal/mca/dl/dl.h"
 #include "opal/mca/dl/base/base.h"
+#include "opal/mca/dl/dl.h"
 
 BEGIN_C_DECLS
 struct mca_base_component_repository_item_t {
@@ -88,8 +88,7 @@ OPAL_DECLSPEC int mca_base_component_repository_init(void);
  *
  * @param[in] path        delimited list of search paths to add
  */
-OPAL_DECLSPEC int mca_base_component_repository_add (const char *path);
-
+OPAL_DECLSPEC int mca_base_component_repository_add(const char *path);
 
 /**
  * @brief return the list of components that match a given framework
@@ -100,8 +99,8 @@ OPAL_DECLSPEC int mca_base_component_repository_add (const char *path);
  * The list returned in {framework_components} is owned by the component
  * repository and CAN NOT be modified by the caller.
  */
-OPAL_DECLSPEC int mca_base_component_repository_get_components (mca_base_framework_t *framework,
-                                                                opal_list_t **framework_components);
+OPAL_DECLSPEC int mca_base_component_repository_get_components(mca_base_framework_t *framework,
+                                                               opal_list_t **framework_components);
 
 /**
  * @brief finalize the mca component repository
@@ -115,14 +114,13 @@ OPAL_DECLSPEC void mca_base_component_repository_finalize(void);
  * @param[in] framework   framework that matches the component
  * @param[in] ri          dynamic component to open
  */
-int mca_base_component_repository_open (mca_base_framework_t *framework,
-                                        mca_base_component_repository_item_t *ri);
-
+int mca_base_component_repository_open(mca_base_framework_t *framework,
+                                       mca_base_component_repository_item_t *ri);
 
 /**
  * @brief Reduce the reference count of a component and dlclose it if necessary
  */
-void mca_base_component_repository_release (const mca_base_component_t *component);
+void mca_base_component_repository_release(const mca_base_component_t *component);
 
 /**
  * @brief Increase the reference count of a component
@@ -137,7 +135,7 @@ void mca_base_component_repository_release (const mca_base_component_t *componen
  * @note all components are automatically unloaded by the
  * mca_base_component_repository_finalize() call.
  */
-int mca_base_component_repository_retain_component (const char *type, const char *name);
+int mca_base_component_repository_retain_component(const char *type, const char *name);
 
 END_C_DECLS
 
diff --git a/opal/mca/base/mca_base_components_close.c b/opal/mca/base/mca_base_components_close.c
index 0239df3b61e..9dbb24cf296 100644
--- a/opal/mca/base/mca_base_components_close.c
+++ b/opal/mca/base/mca_base_components_close.c
@@ -22,55 +22,53 @@
 #include "opal_config.h"
 
 #include "opal/class/opal_list.h"
-#include "opal/util/output.h"
-#include "opal/mca/mca.h"
+#include "opal/constants.h"
 #include "opal/mca/base/base.h"
 #include "opal/mca/base/mca_base_component_repository.h"
-#include "opal/constants.h"
+#include "opal/mca/mca.h"
+#include "opal/util/output.h"
 
-void mca_base_component_unload (const mca_base_component_t *component, int output_id)
+void mca_base_component_unload(const mca_base_component_t *component, int output_id)
 {
     int ret;
 
     /* Unload */
-    opal_output_verbose (MCA_BASE_VERBOSE_COMPONENT, output_id,
-                         "mca: base: close: unloading component %s",
-                         component->mca_component_name);
+    opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT, output_id,
+                        "mca: base: close: unloading component %s", component->mca_component_name);
 
-    ret = mca_base_var_group_find (component->mca_project_name, component->mca_type_name,
-                                   component->mca_component_name);
+    ret = mca_base_var_group_find(component->mca_project_name, component->mca_type_name,
+                                  component->mca_component_name);
     if (0 <= ret) {
-        mca_base_var_group_deregister (ret);
+        mca_base_var_group_deregister(ret);
     }
 
-    mca_base_component_repository_release (component);
+    mca_base_component_repository_release(component);
 }
 
-void mca_base_component_close (const mca_base_component_t *component, int output_id)
+void mca_base_component_close(const mca_base_component_t *component, int output_id)
 {
     /* Close */
     if (NULL != component->mca_close_component) {
-        if( OPAL_SUCCESS == component->mca_close_component() ) {
-            opal_output_verbose (MCA_BASE_VERBOSE_COMPONENT, output_id,
-                                 "mca: base: close: component %s closed",
-                                 component->mca_component_name);
+        if (OPAL_SUCCESS == component->mca_close_component()) {
+            opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT, output_id,
+                                "mca: base: close: component %s closed",
+                                component->mca_component_name);
         } else {
-            opal_output_verbose (MCA_BASE_VERBOSE_COMPONENT, output_id,
-                                 "mca: base: close: component %s refused to close [drop it]",
-                                 component->mca_component_name);
+            opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT, output_id,
+                                "mca: base: close: component %s refused to close [drop it]",
+                                component->mca_component_name);
             return;
         }
     }
 
-    mca_base_component_unload (component, output_id);
+    mca_base_component_unload(component, output_id);
 }
 
-int mca_base_framework_components_close (mca_base_framework_t *framework,
-                                         const mca_base_component_t *skip)
+int mca_base_framework_components_close(mca_base_framework_t *framework,
+                                        const mca_base_component_t *skip)
 {
-    return mca_base_components_close (framework->framework_output,
-                                      &framework->framework_components,
-                                      skip);
+    return mca_base_components_close(framework->framework_output, &framework->framework_components,
+                                     skip);
 }
 
 int mca_base_components_close(int output_id, opal_list_t *components,
@@ -83,13 +81,13 @@ int mca_base_components_close(int output_id, opal_list_t *components,
        components.  It's easier to simply remove the entire list and
        then simply re-add the skip entry when done. */
 
-    OPAL_LIST_FOREACH_SAFE(cli, next, components, mca_base_component_list_item_t) {
+    OPAL_LIST_FOREACH_SAFE (cli, next, components, mca_base_component_list_item_t) {
         if (skip == cli->cli_component) {
             continue;
         }
 
-        mca_base_component_close (cli->cli_component, output_id);
-        opal_list_remove_item (components, &cli->super);
+        mca_base_component_close(cli->cli_component, output_id);
+        opal_list_remove_item(components, &cli->super);
 
         OBJ_RELEASE(cli);
     }
diff --git a/opal/mca/base/mca_base_components_open.c b/opal/mca/base/mca_base_components_open.c
index b18540d2068..dc451ae2c71 100644
--- a/opal/mca/base/mca_base_components_open.c
+++ b/opal/mca/base/mca_base_components_open.c
@@ -10,10 +10,11 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2008-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2008-2022 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2011-2015 Los Alamos National Security, LLC.
  *                         All rights reserved.
  * Copyright (c) 2014      Hochschule Esslingen.  All rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -24,15 +25,17 @@
 #include "opal_config.h"
 
 #include <stdio.h>
-#include <string.h>
 #include <stdlib.h>
+#include <string.h>
 
 #include "opal/class/opal_list.h"
+#include "opal/class/opal_cstring.h"
+#include "opal/constants.h"
+#include "opal/mca/base/base.h"
+#include "opal/mca/mca.h"
 #include "opal/util/argv.h"
 #include "opal/util/output.h"
-#include "opal/mca/mca.h"
-#include "opal/mca/base/base.h"
-#include "opal/constants.h"
+#include "opal/util/show_help.h"
 
 /*
  * Local functions
@@ -44,12 +47,240 @@ struct mca_base_dummy_framework_list_item_t {
     mca_base_framework_t framework;
 };
 
+typedef struct fc_pair {
+    opal_list_item_t li;
+    char *framework_name;
+    char *component_name;
+} opal_base_fc_pair_t;
+
+OBJ_CLASS_DECLARATION(opal_base_fc_pair_t);
+
+typedef enum {
+    SHOW_LOAD_ERRORS_ALL,
+    SHOW_LOAD_ERRORS_INCLUDE,
+    SHOW_LOAD_ERRORS_EXCLUDE,
+    SHOW_LOAD_ERRORS_NONE,
+} show_load_type_t;
+
+static show_load_type_t show_load_errors = SHOW_LOAD_ERRORS_ALL;
+static opal_list_t show_load_errors_include = {0};
+static opal_list_t show_load_errors_exclude = {0};
+
+
+static void fc_pair_constructor(struct fc_pair *obj)
+{
+    obj->framework_name = NULL;
+    obj->component_name = NULL;
+}
+
+static void fc_pair_destructor(struct fc_pair *obj)
+{
+    free(obj->framework_name);
+    obj->framework_name = NULL;
+    free(obj->component_name);
+    obj->component_name = NULL;
+}
+
+OBJ_CLASS_INSTANCE(opal_base_fc_pair_t, opal_list_item_t,
+                   fc_pair_constructor,
+                   fc_pair_destructor);
+
+/*
+ * Parse the content of the show_load_errors value
+ *
+ * Valid values:
+ * - "all"
+ * - "none"
+ * - comma-delimited list of items, each of which is
+ *   "framework[/component]"
+ *
+ * The comma-delimited list may be prefixed with a "^".
+ */
+int mca_base_show_load_errors_init(void)
+{
+    OBJ_CONSTRUCT(&show_load_errors_include, opal_list_t);
+    OBJ_CONSTRUCT(&show_load_errors_exclude, opal_list_t);
+
+    // Check to see if mca_base_component_show_load_errors is a
+    // boolean value
+    opal_cstring_t *cstr = opal_cstring_create(mca_base_component_show_load_errors);
+    if (NULL == cstr) {
+        int ret = OPAL_ERROR;
+        opal_show_help("help-mca-base.txt",
+                       "internal error during init", true,
+                       __func__, __FILE__, __LINE__,
+                       ret,
+                       "Failed to create opal_cstring");
+        return ret;
+    }
+    bool value;
+    int ret = opal_cstring_to_bool(cstr, &value);
+    OBJ_RELEASE(cstr);
+
+    if (OPAL_SUCCESS == ret) {
+        // True true values as a synonym for "all", and false values
+        // as a synonym for "none".  This is mainly for backwards
+        // compatibility with Open MPI <= v4.x, where
+        // mca_base_component_show_load_errors was a boolean value.
+        if (value) {
+            show_load_errors = SHOW_LOAD_ERRORS_ALL;
+        } else {
+            show_load_errors = SHOW_LOAD_ERRORS_NONE;
+        }
+    } else if (strcasecmp(mca_base_component_show_load_errors, "all") == 0) {
+        show_load_errors = SHOW_LOAD_ERRORS_ALL;
+    }
+    else if (strcasecmp(mca_base_component_show_load_errors, "none") == 0) {
+        show_load_errors = SHOW_LOAD_ERRORS_NONE;
+    } else {
+        // We have a comma-delimited list of values.  Is it
+        // "include"-style, or "exclude" style?
+        size_t pos = 0;
+        opal_list_t *list = &show_load_errors_include;
+        show_load_errors = SHOW_LOAD_ERRORS_INCLUDE;
+        if (mca_base_component_show_load_errors[0] == '^') {
+            pos = 1;
+            list = &show_load_errors_exclude;
+            show_load_errors = SHOW_LOAD_ERRORS_EXCLUDE;
+        }
+
+        // Examine each of the values in the comma-delimited list.
+        // Each value can be of the form "framework" or
+        // "framework/component".
+        char **values = opal_argv_split(mca_base_component_show_load_errors + pos,
+                                        ',');
+        if (values == NULL) {
+            ret = OPAL_ERROR;
+            opal_show_help("help-mca-base.txt",
+                           "internal error during init", true,
+                           __func__, __FILE__, __LINE__,
+                           ret,
+                           "Failed to argv split opal_mca_base_component_show_load_errors");
+            return ret;
+        }
+
+        char **split;
+        int argc;
+        opal_base_fc_pair_t *fcp;
+        for (int i = 0; values[i] != NULL; ++i) {
+            split = opal_argv_split(values[i], '/');
+            if (NULL == split) {
+                ret = OPAL_ERROR;
+                opal_show_help("help-mca-base.txt",
+                               "internal error during init", true,
+                               __func__, __FILE__, __LINE__,
+                               ret,
+                               "Failed to argv split opal_mca_base_component_show_load_errors value");
+                return ret;
+            }
+
+            argc = opal_argv_count(split);
+            if (0 == argc) {
+                // This should never happen
+                ret = OPAL_ERROR;
+                opal_show_help("help-mca-base.txt",
+                               "internal error during init", true,
+                               __func__, __FILE__, __LINE__,
+                               ret,
+                               "Argv split resulted in 0 tokens");
+                return ret;
+            }
+
+            if (strlen(split[0]) == 0) {
+                // Empty entry (e.g., consecutive commas); silently
+                // skip it
+                continue;
+            }
+
+            if (argc > 2) {
+                ret = OPAL_ERR_BAD_PARAM;
+                opal_show_help("help-mca-base.txt",
+                               "show_load_errors: too many /", true,
+                               values[i]);
+                return ret;
+            }
+
+            fcp = OBJ_NEW(opal_base_fc_pair_t);
+            if (NULL == fcp) {
+                ret = OPAL_ERR_OUT_OF_RESOURCE;
+                opal_show_help("help-mca-base.txt",
+                               "internal error during init", true,
+                               __func__, __FILE__, __LINE__,
+                               ret,
+                               "Failed to alloc new fc_pair_t");
+                return ret;
+            }
+
+            fcp->framework_name = split[0];
+            if (2 == argc) {
+                fcp->component_name = split[1];
+            }
+
+            opal_list_append(list, &fcp->li);
+        }
+        opal_argv_free(values);
+    }
+
+    return OPAL_SUCCESS;
+}
+
+
+bool mca_base_show_load_errors(const char *framework_name,
+                               const char *component_name)
+{
+    if (SHOW_LOAD_ERRORS_ALL == show_load_errors) {
+        return true;
+    } else if (SHOW_LOAD_ERRORS_NONE == show_load_errors) {
+        return false;
+    }
+
+    // If we get here, it means we have an include or exclude list.
+    // Setup for what to do based on whether it's an include or
+    // exclude list.
+    opal_list_t *list;
+    bool value_if_match_found;
+
+    if (SHOW_LOAD_ERRORS_INCLUDE == show_load_errors) {
+        list = &show_load_errors_include;
+        value_if_match_found = true;
+    } else {
+        list = &show_load_errors_exclude;
+        value_if_match_found = false;
+    }
+
+    // See if the framework_name/component_name pair is found in the
+    // active list.
+    opal_base_fc_pair_t *item;
+    OPAL_LIST_FOREACH(item, list, opal_base_fc_pair_t) {
+        if (strcmp(framework_name, item->framework_name) == 0) {
+            if (NULL == item->component_name) {
+                // If there's no component name, then we're matching
+                // all components in this framework.
+                return value_if_match_found;
+            } else if (strcmp(component_name, item->component_name) == 0) {
+                // We matched both the framework *and* component name.
+                return value_if_match_found;
+            }
+        }
+    }
+
+    // We didn't find a match.
+    return !value_if_match_found;
+}
+
+int mca_base_show_load_errors_finalize(void)
+{
+    OBJ_DESTRUCT(&show_load_errors_include);
+    OBJ_DESTRUCT(&show_load_errors_exclude);
+
+    return OPAL_SUCCESS;
+}
+
 /**
  * Function for finding and opening either all MCA components, or the
  * one that was specifically requested via a MCA parameter.
  */
-int mca_base_framework_components_open (mca_base_framework_t *framework,
-                                        mca_base_open_flag_t flags)
+int mca_base_framework_components_open(mca_base_framework_t *framework, mca_base_open_flag_t flags)
 {
     /* Open flags are not used at this time. Suppress compiler warning. */
     if (flags & MCA_BASE_OPEN_FIND_COMPONENTS) {
@@ -62,7 +293,7 @@ int mca_base_framework_components_open (mca_base_framework_t *framework,
     }
 
     /* Open all registered components */
-    return open_components (framework);
+    return open_components(framework);
 }
 
 /*
@@ -87,7 +318,7 @@ static int open_components(mca_base_framework_t *framework)
      *
      * JJH Note: Currently checkpoint/restart is the only user of this
      *           functionality. If other component constraint options are
-     *           added, then this logic can be used for all contraint
+     *           added, then this logic can be used for all constraint
      *           options.
      *
      * NTH: Logic moved to mca_base_components_filter.
@@ -95,64 +326,66 @@ static int open_components(mca_base_framework_t *framework)
 
     /* If mca_base_framework_register_components was called with the MCA_BASE_COMPONENTS_ALL flag
        we need to trim down and close any extra components we do not want open */
-    ret = mca_base_components_filter (framework, open_only_flags);
+    ret = mca_base_components_filter(framework, open_only_flags);
     if (OPAL_SUCCESS != ret) {
         return ret;
     }
 
     /* Announce */
-    opal_output_verbose (MCA_BASE_VERBOSE_COMPONENT, output_id, "mca: base: components_open: opening %s components",
-                         framework->framework_name);
+    opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT, output_id,
+                        "mca: base: components_open: opening %s components",
+                        framework->framework_name);
 
     /* Traverse the list of components */
-    OPAL_LIST_FOREACH_SAFE(cli, next, components, mca_base_component_list_item_t) {
+    OPAL_LIST_FOREACH_SAFE (cli, next, components, mca_base_component_list_item_t) {
         const mca_base_component_t *component = cli->cli_component;
 
-        opal_output_verbose (MCA_BASE_VERBOSE_COMPONENT, output_id,
-                             "mca: base: components_open: found loaded component %s",
-                             component->mca_component_name);
+        opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT, output_id,
+                            "mca: base: components_open: found loaded component %s",
+                            component->mca_component_name);
 
-	if (NULL != component->mca_open_component) {
-	    /* Call open if register didn't call it already */
+        if (NULL != component->mca_open_component) {
+            /* Call open if register didn't call it already */
             ret = component->mca_open_component();
 
             if (OPAL_SUCCESS == ret) {
-                opal_output_verbose (MCA_BASE_VERBOSE_COMPONENT, output_id,
-                                     "mca: base: components_open: "
-                                     "component %s open function successful",
-                                     component->mca_component_name);
+                opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT, output_id,
+                                    "mca: base: components_open: "
+                                    "component %s open function successful",
+                                    component->mca_component_name);
             } else {
-		if (OPAL_ERR_NOT_AVAILABLE != ret) {
-		    /* If the component returns OPAL_ERR_NOT_AVAILABLE,
-		       it's a cue to "silently ignore me" -- it's not a
-		       failure, it's just a way for the component to say
-		       "nope!".
-
-		       Otherwise, however, display an error.  We may end
-		       up displaying this twice, but it may go to separate
-		       streams.  So better to be redundant than to not
-		       display the error in the stream where it was
-		       expected. */
-
-		    if (mca_base_component_show_load_errors) {
-			opal_output_verbose (MCA_BASE_VERBOSE_ERROR, output_id,
-                                             "mca: base: components_open: component %s "
-                                             "/ %s open function failed",
-                                             component->mca_type_name,
-                                             component->mca_component_name);
-		    }
-		    opal_output_verbose (MCA_BASE_VERBOSE_COMPONENT, output_id,
-                                         "mca: base: components_open: "
-                                         "component %s open function failed",
-                                         component->mca_component_name);
-		}
-
-                mca_base_component_close (component, output_id);
-
-		opal_list_remove_item (components, &cli->super);
-		OBJ_RELEASE(cli);
-	    }
-	}
+                if (OPAL_ERR_NOT_AVAILABLE != ret) {
+                    /* If the component returns OPAL_ERR_NOT_AVAILABLE,
+                       it's a cue to "silently ignore me" -- it's not a
+                       failure, it's just a way for the component to say
+                       "nope!".
+
+                       Otherwise, however, display an error.  We may end
+                       up displaying this twice, but it may go to separate
+                       streams.  So better to be redundant than to not
+                       display the error in the stream where it was
+                       expected. */
+
+                    if (mca_base_show_load_errors(component->mca_type_name,
+                                                  component->mca_component_name)) {
+                        opal_output_verbose(MCA_BASE_VERBOSE_ERROR, output_id,
+                                            "mca: base: components_open: component %s "
+                                            "/ %s open function failed",
+                                            component->mca_type_name,
+                                            component->mca_component_name);
+                    }
+                    opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT, output_id,
+                                        "mca: base: components_open: "
+                                        "component %s open function failed",
+                                        component->mca_component_name);
+                }
+
+                mca_base_component_close(component, output_id);
+
+                opal_list_remove_item(components, &cli->super);
+                OBJ_RELEASE(cli);
+            }
+        }
     }
 
     /* All done */
diff --git a/opal/mca/base/mca_base_components_register.c b/opal/mca/base/mca_base_components_register.c
index 1598a533030..18bf74ccec5 100644
--- a/opal/mca/base/mca_base_components_register.c
+++ b/opal/mca/base/mca_base_components_register.c
@@ -10,7 +10,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2008-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2008-2022 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2011-2015 Los Alamos National Security, LLC.
  *                         All rights reserved.
  * $COPYRIGHT$
@@ -23,18 +23,18 @@
 #include "opal_config.h"
 
 #include <stdio.h>
-#include <string.h>
 #include <stdlib.h>
+#include <string.h>
 
 #include "opal/class/opal_list.h"
+#include "opal/constants.h"
+#include "opal/mca/base/base.h"
+#include "opal/mca/base/mca_base_component_repository.h"
+#include "opal/mca/base/mca_base_framework.h"
+#include "opal/mca/mca.h"
 #include "opal/util/argv.h"
 #include "opal/util/output.h"
 #include "opal/util/show_help.h"
-#include "opal/mca/mca.h"
-#include "opal/mca/base/base.h"
-#include "opal/mca/base/mca_base_framework.h"
-#include "opal/mca/base/mca_base_component_repository.h"
-#include "opal/constants.h"
 
 /*
  * Local functions
@@ -44,8 +44,8 @@ static int register_components(mca_base_framework_t *framework);
  * Function for finding and opening either all MCA components, or the
  * one that was specifically requested via a MCA parameter.
  */
-int mca_base_framework_components_register (mca_base_framework_t *framework,
-                                            mca_base_register_flag_t flags)
+int mca_base_framework_components_register(mca_base_framework_t *framework,
+                                           mca_base_register_flag_t flags)
 {
     bool open_dso_components = !(flags & MCA_BASE_REGISTER_STATIC_ONLY);
     bool ignore_requested = !!(flags & MCA_BASE_REGISTER_ALL);
@@ -76,25 +76,27 @@ static int register_components(mca_base_framework_t *framework)
     int output_id = framework->framework_output;
 
     /* Announce */
-    opal_output_verbose (MCA_BASE_VERBOSE_COMPONENT, output_id,
-                         "mca: base: components_register: registering framework %s components",
-                         framework->framework_name);
+    opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT, output_id,
+                        "mca: base: components_register: registering framework %s components",
+                        framework->framework_name);
 
     /* Traverse the list of found components */
 
-    OPAL_LIST_FOREACH_SAFE(cli, next, &framework->framework_components, mca_base_component_list_item_t) {
-        component = (mca_base_component_t *)cli->cli_component;
+    OPAL_LIST_FOREACH_SAFE (cli, next, &framework->framework_components,
+                            mca_base_component_list_item_t) {
+        component = (mca_base_component_t *) cli->cli_component;
 
         opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT, output_id,
                             "mca: base: components_register: found loaded component %s",
                             component->mca_component_name);
 
-        /* Call the component's MCA parameter registration function (or open if register doesn't exist) */
+        /* Call the component's MCA parameter registration function (or open if register doesn't
+         * exist) */
         if (NULL == component->mca_register_component_params) {
-            opal_output_verbose (MCA_BASE_VERBOSE_COMPONENT, output_id,
-                                 "mca: base: components_register: "
-                                 "component %s has no register or open function",
-                                 component->mca_component_name);
+            opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT, output_id,
+                                "mca: base: components_register: "
+                                "component %s has no register or open function",
+                                component->mca_component_name);
             ret = OPAL_SUCCESS;
         } else {
             ret = component->mca_register_component_params();
@@ -113,21 +115,21 @@ static int register_components(mca_base_framework_t *framework)
                    display the error in the stream where it was
                    expected. */
 
-                if (mca_base_component_show_load_errors) {
-                    opal_output_verbose (MCA_BASE_VERBOSE_ERROR, output_id,
-                                         "mca: base: components_register: component %s "
-                                         "/ %s register function failed",
-                                         component->mca_type_name,
-                                         component->mca_component_name);
+                if (mca_base_show_load_errors(component->mca_type_name,
+                                              component->mca_component_name)) {
+                    opal_output_verbose(MCA_BASE_VERBOSE_ERROR, output_id,
+                                        "mca: base: components_register: component %s "
+                                        "/ %s register function failed",
+                                        component->mca_type_name, component->mca_component_name);
                 }
 
-                opal_output_verbose (MCA_BASE_VERBOSE_COMPONENT, output_id,
-                                     "mca: base: components_register: "
-                                     "component %s register function failed",
-                                     component->mca_component_name);
+                opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT, output_id,
+                                    "mca: base: components_register: "
+                                    "component %s register function failed",
+                                    component->mca_component_name);
             }
 
-            opal_list_remove_item (&framework->framework_components, &cli->super);
+            opal_list_remove_item(&framework->framework_components, &cli->super);
 
             /* Release this list item */
             OBJ_RELEASE(cli);
@@ -135,24 +137,28 @@ static int register_components(mca_base_framework_t *framework)
         }
 
         if (NULL != component->mca_register_component_params) {
-            opal_output_verbose (MCA_BASE_VERBOSE_COMPONENT, output_id, "mca: base: components_register: "
-                                 "component %s register function successful",
-                                 component->mca_component_name);
+            opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT, output_id,
+                                "mca: base: components_register: "
+                                "component %s register function successful",
+                                component->mca_component_name);
         }
 
         /* Register this component's version */
-        mca_base_component_var_register (component, "major_version", NULL, MCA_BASE_VAR_TYPE_INT, NULL,
-                                         0, MCA_BASE_VAR_FLAG_DEFAULT_ONLY | MCA_BASE_VAR_FLAG_INTERNAL,
-                                         OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_CONSTANT,
-                                         &component->mca_component_major_version);
-        mca_base_component_var_register (component, "minor_version", NULL, MCA_BASE_VAR_TYPE_INT, NULL,
-                                         0, MCA_BASE_VAR_FLAG_DEFAULT_ONLY | MCA_BASE_VAR_FLAG_INTERNAL,
-                                         OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_CONSTANT,
-                                         &component->mca_component_minor_version);
-        mca_base_component_var_register (component, "release_version", NULL, MCA_BASE_VAR_TYPE_INT, NULL,
-                                         0, MCA_BASE_VAR_FLAG_DEFAULT_ONLY | MCA_BASE_VAR_FLAG_INTERNAL,
-                                         OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_CONSTANT,
-                                         &component->mca_component_release_version);
+        mca_base_component_var_register(component, "major_version", NULL, MCA_BASE_VAR_TYPE_INT,
+                                        NULL, 0,
+                                        MCA_BASE_VAR_FLAG_DEFAULT_ONLY | MCA_BASE_VAR_FLAG_INTERNAL,
+                                        OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_CONSTANT,
+                                        &component->mca_component_major_version);
+        mca_base_component_var_register(component, "minor_version", NULL, MCA_BASE_VAR_TYPE_INT,
+                                        NULL, 0,
+                                        MCA_BASE_VAR_FLAG_DEFAULT_ONLY | MCA_BASE_VAR_FLAG_INTERNAL,
+                                        OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_CONSTANT,
+                                        &component->mca_component_minor_version);
+        mca_base_component_var_register(component, "release_version", NULL, MCA_BASE_VAR_TYPE_INT,
+                                        NULL, 0,
+                                        MCA_BASE_VAR_FLAG_DEFAULT_ONLY | MCA_BASE_VAR_FLAG_INTERNAL,
+                                        OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_CONSTANT,
+                                        &component->mca_component_release_version);
     }
 
     /* All done */
diff --git a/opal/mca/base/mca_base_components_select.c b/opal/mca/base/mca_base_components_select.c
index cf45bf903ec..d2bc2d22af5 100644
--- a/opal/mca/base/mca_base_components_select.c
+++ b/opal/mca/base/mca_base_components_select.c
@@ -16,25 +16,22 @@
 #include "opal_config.h"
 
 #include <stdio.h>
-#include <string.h>
 #include <stdlib.h>
+#include <string.h>
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 
-#include "opal/runtime/opal.h"
 #include "opal/class/opal_list.h"
-#include "opal/util/output.h"
-#include "opal/mca/mca.h"
+#include "opal/constants.h"
 #include "opal/mca/base/base.h"
 #include "opal/mca/base/mca_base_component_repository.h"
-#include "opal/constants.h"
-
+#include "opal/mca/mca.h"
+#include "opal/runtime/opal.h"
+#include "opal/util/output.h"
 
-int mca_base_select(const char *type_name, int output_id,
-                    opal_list_t *components_available,
-                    mca_base_module_t **best_module,
-                    mca_base_component_t **best_component,
+int mca_base_select(const char *type_name, int output_id, opal_list_t *components_available,
+                    mca_base_module_t **best_module, mca_base_component_t **best_component,
                     int *priority_out)
 {
     mca_base_component_list_item_t *cli = NULL;
@@ -46,33 +43,33 @@ int mca_base_select(const char *type_name, int output_id,
     *best_module = NULL;
     *best_component = NULL;
 
-    opal_output_verbose (MCA_BASE_VERBOSE_COMPONENT, output_id,
-                         "mca:base:select: Auto-selecting %s components",
-                         type_name);
+    opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT, output_id,
+                        "mca:base:select: Auto-selecting %s components", type_name);
 
     /*
      * Traverse the list of available components.
      * For each call their 'query' functions to determine relative priority.
      */
-    OPAL_LIST_FOREACH(cli, components_available, mca_base_component_list_item_t) {
+    OPAL_LIST_FOREACH (cli, components_available, mca_base_component_list_item_t) {
         component = (mca_base_component_t *) cli->cli_component;
 
         /*
          * If there is a query function then use it.
          */
         if (NULL == component->mca_query_component) {
-            opal_output_verbose (MCA_BASE_VERBOSE_COMPONENT, output_id,
-                                 "mca:base:select:(%5s) Skipping component [%s]. It does not implement a query function",
-                                 type_name, component->mca_component_name );
+            opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT, output_id,
+                                "mca:base:select:(%5s) Skipping component [%s]. It does not "
+                                "implement a query function",
+                                type_name, component->mca_component_name);
             continue;
         }
 
         /*
          * Query this component for the module and priority
          */
-        opal_output_verbose (MCA_BASE_VERBOSE_COMPONENT, output_id,
-                             "mca:base:select:(%5s) Querying component [%s]",
-                             type_name, component->mca_component_name);
+        opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT, output_id,
+                            "mca:base:select:(%5s) Querying component [%s]", type_name,
+                            component->mca_component_name);
 
         rc = component->mca_query_component(&module, &priority);
         if (OPAL_ERR_FATAL == rc) {
@@ -81,7 +78,7 @@ int mca_base_select(const char *type_name, int output_id,
              * not find it. In this case, we must not continue as we might
              * find some other component that could run, causing us to do
              * something the user didn't want */
-             return rc;
+            return rc;
         } else if (OPAL_SUCCESS != rc) {
             /* silently skip this component */
             continue;
@@ -91,22 +88,23 @@ int mca_base_select(const char *type_name, int output_id,
          * If no module was returned, then skip component
          */
         if (NULL == module) {
-            opal_output_verbose (MCA_BASE_VERBOSE_COMPONENT, output_id,
-                                 "mca:base:select:(%5s) Skipping component [%s]. Query failed to return a module",
-                                 type_name, component->mca_component_name );
+            opal_output_verbose(
+                MCA_BASE_VERBOSE_COMPONENT, output_id,
+                "mca:base:select:(%5s) Skipping component [%s]. Query failed to return a module",
+                type_name, component->mca_component_name);
             continue;
         }
 
         /*
          * Determine if this is the best module we have seen by looking the priority
          */
-        opal_output_verbose (MCA_BASE_VERBOSE_COMPONENT, output_id,
-                             "mca:base:select:(%5s) Query of component [%s] set priority to %d",
-                             type_name, component->mca_component_name, priority);
+        opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT, output_id,
+                            "mca:base:select:(%5s) Query of component [%s] set priority to %d",
+                            type_name, component->mca_component_name, priority);
         if (priority > best_priority) {
-            best_priority  = priority;
+            best_priority = priority;
             *best_component = component;
-            *best_module    = module;
+            *best_module = module;
         }
     }
 
@@ -119,29 +117,25 @@ int mca_base_select(const char *type_name, int output_id,
      * Make sure we found something in the process.
      */
     if (NULL == *best_component) {
-        opal_output_verbose (MCA_BASE_VERBOSE_COMPONENT, output_id,
-                            "mca:base:select:(%5s) No component selected!",
-                            type_name);
+        opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT, output_id,
+                            "mca:base:select:(%5s) No component selected!", type_name);
         /*
          * Still close the non-selected components
          */
         mca_base_components_close(0, /* Pass 0 to keep this from closing the output handle */
-                                  components_available,
-                                  NULL);
+                                  components_available, NULL);
         return OPAL_ERR_NOT_FOUND;
     }
 
-    opal_output_verbose (MCA_BASE_VERBOSE_COMPONENT, output_id,
-                         "mca:base:select:(%5s) Selected component [%s]",
-                         type_name, (*best_component)->mca_component_name);
+    opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT, output_id,
+                        "mca:base:select:(%5s) Selected component [%s]", type_name,
+                        (*best_component)->mca_component_name);
 
     /*
      * Close the non-selected components
      */
-    mca_base_components_close(output_id,
-                              components_available,
+    mca_base_components_close(output_id, components_available,
                               (mca_base_component_t *) (*best_component));
 
-
     return OPAL_SUCCESS;
 }
diff --git a/opal/mca/base/mca_base_framework.c b/opal/mca/base/mca_base_framework.c
index 54edc9b552c..81a76731650 100644
--- a/opal/mca/base/mca_base_framework.c
+++ b/opal/mca/base/mca_base_framework.c
@@ -7,6 +7,7 @@
  * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * Copyright (c) 2018      Triad National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -24,49 +25,48 @@
 #include "mca_base_var.h"
 #include "opal/mca/base/base.h"
 
-bool mca_base_framework_is_registered (struct mca_base_framework_t *framework)
+bool mca_base_framework_is_registered(struct mca_base_framework_t *framework)
 {
     return !!(framework->framework_flags & MCA_BASE_FRAMEWORK_FLAG_REGISTERED);
 }
 
-bool mca_base_framework_is_open (struct mca_base_framework_t *framework)
+bool mca_base_framework_is_open(struct mca_base_framework_t *framework)
 {
     return !!(framework->framework_flags & MCA_BASE_FRAMEWORK_FLAG_OPEN);
 }
 
-static void framework_open_output (struct mca_base_framework_t *framework)
+static void framework_open_output(struct mca_base_framework_t *framework)
 {
     if (0 < framework->framework_verbose) {
         if (-1 == framework->framework_output) {
-            framework->framework_output = opal_output_open (NULL);
+            framework->framework_output = opal_output_open(NULL);
         }
-        opal_output_set_verbosity(framework->framework_output,
-                                  framework->framework_verbose);
+        opal_output_set_verbosity(framework->framework_output, framework->framework_verbose);
     } else if (-1 != framework->framework_output) {
-        opal_output_close (framework->framework_output);
+        opal_output_close(framework->framework_output);
         framework->framework_output = -1;
     }
 }
 
-static void framework_close_output (struct mca_base_framework_t *framework)
+static void framework_close_output(struct mca_base_framework_t *framework)
 {
     if (-1 != framework->framework_output) {
-        opal_output_close (framework->framework_output);
+        opal_output_close(framework->framework_output);
         framework->framework_output = -1;
     }
 }
 
-int mca_base_framework_register (struct mca_base_framework_t *framework,
-                                 mca_base_register_flag_t flags)
+int mca_base_framework_register(struct mca_base_framework_t *framework,
+                                mca_base_register_flag_t flags)
 {
     char *desc;
     int ret;
 
-    assert (NULL != framework);
+    assert(NULL != framework);
 
     framework->framework_refcnt++;
 
-    if (mca_base_framework_is_registered (framework)) {
+    if (mca_base_framework_is_registered(framework)) {
         return OPAL_SUCCESS;
     }
 
@@ -79,39 +79,38 @@ int mca_base_framework_register (struct mca_base_framework_t *framework,
 
     if (!(MCA_BASE_FRAMEWORK_FLAG_NOREGISTER & framework->framework_flags)) {
         /* register this framework with the MCA variable system */
-        ret = mca_base_var_group_register (framework->framework_project,
-                                           framework->framework_name,
-                                           NULL, framework->framework_description);
+        ret = mca_base_var_group_register(framework->framework_project, framework->framework_name,
+                                          NULL, framework->framework_description);
         if (0 > ret) {
             return ret;
         }
 
-        opal_asprintf (&desc, "Default selection set of components for the %s framework (<none>"
-                  " means use all components that can be found)", framework->framework_name);
-        ret = mca_base_var_register (framework->framework_project, framework->framework_name,
-                                     NULL, NULL, desc, MCA_BASE_VAR_TYPE_STRING, NULL, 0,
-                                     MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_2,
-                                     MCA_BASE_VAR_SCOPE_ALL_EQ, &framework->framework_selection);
-        free (desc);
+        opal_asprintf(&desc,
+                      "Default selection set of components for the %s framework (<none>"
+                      " means use all components that can be found)",
+                      framework->framework_name);
+        ret = mca_base_var_register(framework->framework_project, framework->framework_name, NULL,
+                                    NULL, desc, MCA_BASE_VAR_TYPE_STRING, NULL, 0,
+                                    MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_2,
+                                    MCA_BASE_VAR_SCOPE_ALL_EQ, &framework->framework_selection);
+        free(desc);
         if (0 > ret) {
             return ret;
         }
 
         /* register a verbosity variable for this framework */
-        ret = opal_asprintf (&desc, "Verbosity level for the %s framework (default: 0)",
-                        framework->framework_name);
+        ret = opal_asprintf(&desc, "Verbosity level for the %s framework (default: 0)",
+                            framework->framework_name);
         if (0 > ret) {
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
 
         framework->framework_verbose = MCA_BASE_VERBOSE_ERROR;
-        ret = mca_base_framework_var_register (framework, "verbose", desc,
-                                               MCA_BASE_VAR_TYPE_INT,
-                                               &mca_base_var_enum_verbose, 0,
-                                               MCA_BASE_VAR_FLAG_SETTABLE,
-                                               OPAL_INFO_LVL_8,
-                                               MCA_BASE_VAR_SCOPE_LOCAL,
-                                               &framework->framework_verbose);
+        ret = mca_base_framework_var_register(framework, "verbose", desc, MCA_BASE_VAR_TYPE_INT,
+                                              &mca_base_var_enum_verbose, 0,
+                                              MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_8,
+                                              MCA_BASE_VAR_SCOPE_LOCAL,
+                                              &framework->framework_verbose);
         free(desc);
         if (0 > ret) {
             return ret;
@@ -119,18 +118,18 @@ int mca_base_framework_register (struct mca_base_framework_t *framework,
 
         /* check the initial verbosity and open the output if necessary. we
            will recheck this on open */
-        framework_open_output (framework);
+        framework_open_output(framework);
 
         /* register framework variables */
         if (NULL != framework->framework_register) {
-            ret = framework->framework_register (flags);
+            ret = framework->framework_register(flags);
             if (OPAL_SUCCESS != ret) {
                 return ret;
             }
         }
 
         /* register components variables */
-        ret = mca_base_framework_components_register (framework, flags);
+        ret = mca_base_framework_components_register(framework, flags);
         if (OPAL_SUCCESS != ret) {
             return ret;
         }
@@ -142,14 +141,15 @@ int mca_base_framework_register (struct mca_base_framework_t *framework,
     return OPAL_SUCCESS;
 }
 
-int mca_base_framework_register_list (mca_base_framework_t **frameworks, mca_base_register_flag_t flags)
+int mca_base_framework_register_list(mca_base_framework_t **frameworks,
+                                     mca_base_register_flag_t flags)
 {
     if (NULL == frameworks) {
         return OPAL_ERR_BAD_PARAM;
     }
 
-    for (int i = 0 ; frameworks[i] ; ++i) {
-        int ret = mca_base_framework_register (frameworks[i], flags);
+    for (int i = 0; frameworks[i]; ++i) {
+        int ret = mca_base_framework_register(frameworks[i], flags);
         if (OPAL_UNLIKELY(OPAL_SUCCESS != ret && OPAL_ERR_NOT_AVAILABLE != ret)) {
             return ret;
         }
@@ -158,20 +158,20 @@ int mca_base_framework_register_list (mca_base_framework_t **frameworks, mca_bas
     return OPAL_SUCCESS;
 }
 
-int mca_base_framework_open (struct mca_base_framework_t *framework,
-                             mca_base_open_flag_t flags) {
+int mca_base_framework_open(struct mca_base_framework_t *framework, mca_base_open_flag_t flags)
+{
     int ret;
 
-    assert (NULL != framework);
+    assert(NULL != framework);
 
     /* register this framework before opening it */
-    ret = mca_base_framework_register (framework, MCA_BASE_REGISTER_DEFAULT);
+    ret = mca_base_framework_register(framework, MCA_BASE_REGISTER_DEFAULT);
     if (OPAL_SUCCESS != ret) {
         return ret;
     }
 
     /* check if this framework is already open */
-    if (mca_base_framework_is_open (framework)) {
+    if (mca_base_framework_is_open(framework)) {
         return OPAL_SUCCESS;
     }
 
@@ -184,18 +184,16 @@ int mca_base_framework_open (struct mca_base_framework_t *framework,
     }
 
     /* lock all of this frameworks's variables */
-    ret = mca_base_var_group_find (framework->framework_project,
-                                   framework->framework_name,
-                                   NULL);
-    mca_base_var_group_set_var_flag (ret, MCA_BASE_VAR_FLAG_SETTABLE, false);
+    ret = mca_base_var_group_find(framework->framework_project, framework->framework_name, NULL);
+    mca_base_var_group_set_var_flag(ret, MCA_BASE_VAR_FLAG_SETTABLE, false);
 
     /* check the verbosity level and open (or close) the output */
-    framework_open_output (framework);
+    framework_open_output(framework);
 
     if (NULL != framework->framework_open) {
-        ret = framework->framework_open (flags);
+        ret = framework->framework_open(flags);
     } else {
-        ret = mca_base_framework_components_open (framework, flags);
+        ret = mca_base_framework_components_open(framework, flags);
     }
 
     if (OPAL_SUCCESS != ret) {
@@ -207,14 +205,14 @@ int mca_base_framework_open (struct mca_base_framework_t *framework,
     return ret;
 }
 
-int mca_base_framework_open_list (mca_base_framework_t **frameworks, mca_base_open_flag_t flags)
+int mca_base_framework_open_list(mca_base_framework_t **frameworks, mca_base_open_flag_t flags)
 {
     if (NULL == frameworks) {
         return OPAL_ERR_BAD_PARAM;
     }
 
-    for (int i = 0 ; frameworks[i] ; ++i) {
-        int ret = mca_base_framework_open (frameworks[i], flags);
+    for (int i = 0; frameworks[i]; ++i) {
+        int ret = mca_base_framework_open(frameworks[i], flags);
         if (OPAL_UNLIKELY(OPAL_SUCCESS != ret && OPAL_ERR_NOT_AVAILABLE != ret)) {
             return ret;
         }
@@ -223,35 +221,38 @@ int mca_base_framework_open_list (mca_base_framework_t **frameworks, mca_base_op
     return OPAL_SUCCESS;
 }
 
-int mca_base_framework_close (struct mca_base_framework_t *framework) {
-    bool is_open = mca_base_framework_is_open (framework);
-    bool is_registered = mca_base_framework_is_registered (framework);
+int mca_base_framework_close(struct mca_base_framework_t *framework)
+{
+    bool is_open = mca_base_framework_is_open(framework);
+    bool is_registered = mca_base_framework_is_registered(framework);
     int ret, group_id;
 
-    assert (NULL != framework);
+    assert(NULL != framework);
 
     if (!(is_open || is_registered)) {
         return OPAL_SUCCESS;
     }
 
-    assert (framework->framework_refcnt);
+    assert(framework->framework_refcnt);
     if (--framework->framework_refcnt) {
         return OPAL_SUCCESS;
     }
 
     /* find and deregister all component groups and variables */
-    group_id = mca_base_var_group_find (framework->framework_project,
-                                        framework->framework_name, NULL);
+    group_id = mca_base_var_group_find(framework->framework_project, framework->framework_name,
+                                       NULL);
     if (0 <= group_id) {
-        (void) mca_base_var_group_deregister (group_id);
+        (void) mca_base_var_group_deregister(group_id);
     }
 
     /* close the framework and all of its components */
     if (is_open) {
+        ret = OPAL_SUCCESS;
         if (NULL != framework->framework_close) {
-            ret = framework->framework_close ();
-        } else {
-            ret = mca_base_framework_components_close (framework, NULL);
+            ret = framework->framework_close();
+        }
+        if (OPAL_SUCCESS == ret) {
+            ret = mca_base_framework_components_close(framework, NULL);
         }
 
         if (OPAL_SUCCESS != ret) {
@@ -259,37 +260,37 @@ int mca_base_framework_close (struct mca_base_framework_t *framework) {
         }
     } else {
         opal_list_item_t *item;
-        while (NULL != (item = opal_list_remove_first (&framework->framework_components))) {
+        while (NULL != (item = opal_list_remove_first(&framework->framework_components))) {
             mca_base_component_list_item_t *cli;
-            cli = (mca_base_component_list_item_t*) item;
-            mca_base_component_unload(cli->cli_component,
-                                      framework->framework_output);
+            cli = (mca_base_component_list_item_t *) item;
+            mca_base_component_unload(cli->cli_component, framework->framework_output);
             OBJ_RELEASE(item);
         }
-        while (NULL != (item = opal_list_remove_first (&framework->framework_failed_components))) {
+        while (NULL != (item = opal_list_remove_first(&framework->framework_failed_components))) {
             OBJ_RELEASE(item);
         }
         ret = OPAL_SUCCESS;
     }
 
-    framework->framework_flags &= ~(MCA_BASE_FRAMEWORK_FLAG_REGISTERED | MCA_BASE_FRAMEWORK_FLAG_OPEN);
+    framework->framework_flags &= ~(MCA_BASE_FRAMEWORK_FLAG_REGISTERED
+                                    | MCA_BASE_FRAMEWORK_FLAG_OPEN);
 
     OBJ_DESTRUCT(&framework->framework_components);
     OBJ_DESTRUCT(&framework->framework_failed_components);
 
-    framework_close_output (framework);
+    framework_close_output(framework);
 
     return ret;
 }
 
-int mca_base_framework_close_list (mca_base_framework_t **frameworks)
+int mca_base_framework_close_list(mca_base_framework_t **frameworks)
 {
     if (NULL == frameworks) {
         return OPAL_ERR_BAD_PARAM;
     }
 
-    for (int i = 0 ; frameworks[i] ; ++i) {
-        int ret = mca_base_framework_close (frameworks[i]);
+    for (int i = 0; frameworks[i]; ++i) {
+        int ret = mca_base_framework_close(frameworks[i]);
         if (OPAL_UNLIKELY(OPAL_SUCCESS != ret)) {
             return ret;
         }
diff --git a/opal/mca/base/mca_base_framework.h b/opal/mca/base/mca_base_framework.h
index a5187d7dd3c..d79428cb43d 100644
--- a/opal/mca/base/mca_base_framework.h
+++ b/opal/mca/base/mca_base_framework.h
@@ -13,18 +13,18 @@
  */
 
 #if !defined(OPAL_MCA_BASE_FRAMEWORK_H)
-#define OPAL_MCA_BASE_FRAMEWORK_H
+#    define OPAL_MCA_BASE_FRAMEWORK_H
 
-#include "opal/mca/mca.h"
-#include "opal/class/opal_list.h"
+#    include "opal/class/opal_list.h"
+#    include "opal/mca/mca.h"
 
 /*
  * Register and open flags
  */
 enum mca_base_register_flag_t {
-    MCA_BASE_REGISTER_DEFAULT     = 0,
+    MCA_BASE_REGISTER_DEFAULT = 0,
     /** Register all components (ignore selection MCA variables) */
-    MCA_BASE_REGISTER_ALL         = 1,
+    MCA_BASE_REGISTER_ALL = 1,
     /** Do not register DSO components */
     MCA_BASE_REGISTER_STATIC_ONLY = 2
 };
@@ -32,18 +32,17 @@ enum mca_base_register_flag_t {
 typedef enum mca_base_register_flag_t mca_base_register_flag_t;
 
 enum mca_base_open_flag_t {
-    MCA_BASE_OPEN_DEFAULT         = 0,
+    MCA_BASE_OPEN_DEFAULT = 0,
     /** Find components in mca_base_components_find. Used by
      mca_base_framework_open() when NOREGISTER is specified
      by the framework */
     MCA_BASE_OPEN_FIND_COMPONENTS = 1,
     /** Do not open DSO components */
-    MCA_BASE_OPEN_STATIC_ONLY     = 2,
+    MCA_BASE_OPEN_STATIC_ONLY = 2,
 };
 
 typedef enum mca_base_open_flag_t mca_base_open_flag_t;
 
-
 /**
  * Register the MCA framework parameters
  *
@@ -58,7 +57,7 @@ typedef enum mca_base_open_flag_t mca_base_open_flag_t;
  * Frameworks are NOT required to provide this function. It
  * may be NULL.
  */
-typedef int (*mca_base_framework_register_params_fn_t) (mca_base_register_flag_t flags);
+typedef int (*mca_base_framework_register_params_fn_t)(mca_base_register_flag_t flags);
 
 /**
  * Initialize the MCA framework
@@ -89,7 +88,7 @@ typedef int (*mca_base_framework_register_params_fn_t) (mca_base_register_flag_t
  * an open function it will need to call mca_base_framework_components_open()
  * if it needs to open any components.
  */
-typedef int (*mca_base_framework_open_fn_t) (mca_base_open_flag_t flags);
+typedef int (*mca_base_framework_open_fn_t)(mca_base_open_flag_t flags);
 
 /**
  * Shut down the MCA framework.
@@ -109,18 +108,18 @@ typedef int (*mca_base_framework_open_fn_t) (mca_base_open_flag_t flags);
  * a close function it will need to call mca_base_framework_components_close()
  * if any components were opened.
  */
-typedef int (*mca_base_framework_close_fn_t) (void);
+typedef int (*mca_base_framework_close_fn_t)(void);
 
 typedef enum {
-    MCA_BASE_FRAMEWORK_FLAG_DEFAULT    = 0,
+    MCA_BASE_FRAMEWORK_FLAG_DEFAULT = 0,
     /** Don't register any variables for this framework */
     MCA_BASE_FRAMEWORK_FLAG_NOREGISTER = 1,
     /** Internal. Don't set outside mca_base_framework.h */
     MCA_BASE_FRAMEWORK_FLAG_REGISTERED = 2,
     /** Framework does not have any DSO components */
-    MCA_BASE_FRAMEWORK_FLAG_NO_DSO     = 4,
+    MCA_BASE_FRAMEWORK_FLAG_NO_DSO = 4,
     /** Internal. Don't set outside mca_base_framework.h */
-    MCA_BASE_FRAMEWORK_FLAG_OPEN       = 8,
+    MCA_BASE_FRAMEWORK_FLAG_OPEN = 8,
     /**
      * The upper 16 bits are reserved for project specific flags.
      */
@@ -128,40 +127,39 @@ typedef enum {
 
 typedef struct mca_base_framework_t {
     /** Project name for this component (ex "opal") */
-    char                                    *framework_project;
+    char *framework_project;
     /** Framework name */
-    char                                    *framework_name;
+    char *framework_name;
     /** Description of this framework or NULL */
-    const char                              *framework_description;
+    const char *framework_description;
     /** Framework register function or NULL if the framework
         and all its components have nothing to register */
-    mca_base_framework_register_params_fn_t  framework_register;
+    mca_base_framework_register_params_fn_t framework_register;
     /** Framework open function or NULL */
-    mca_base_framework_open_fn_t             framework_open;
+    mca_base_framework_open_fn_t framework_open;
     /** Framework close function or NULL */
-    mca_base_framework_close_fn_t            framework_close;
+    mca_base_framework_close_fn_t framework_close;
     /** Framework flags (future use) set to 0 */
-    mca_base_framework_flags_t               framework_flags;
+    mca_base_framework_flags_t framework_flags;
     /** Framework open count */
-    int                                      framework_refcnt;
+    int framework_refcnt;
     /** List of static components */
-    const mca_base_component_t             **framework_static_components;
+    const mca_base_component_t **framework_static_components;
     /** Component selection. This will be registered with the MCA
         variable system and should be either NULL (all components) or
         a heap allocated, comma-delimited list of components. */
-    char                                    *framework_selection;
+    char *framework_selection;
     /** Verbosity level (0-100) */
-    int                                      framework_verbose;
+    int framework_verbose;
     /** Opal output for this framework (or -1) */
-    int                                      framework_output;
+    int framework_output;
     /** List of selected components (filled in by mca_base_framework_register()
         or mca_base_framework_open() */
-    opal_list_t                              framework_components;
+    opal_list_t framework_components;
     /** List of components that failed to load */
-    opal_list_t                              framework_failed_components;
+    opal_list_t framework_failed_components;
 } mca_base_framework_t;
 
-
 /**
  * Register a framework with MCA.
  *
@@ -172,8 +170,8 @@ typedef struct mca_base_framework_t {
  *
  * Call a framework's register function.
  */
-OPAL_DECLSPEC int mca_base_framework_register (mca_base_framework_t *framework,
-                                               mca_base_register_flag_t flags);
+OPAL_DECLSPEC int mca_base_framework_register(mca_base_framework_t *framework,
+                                              mca_base_register_flag_t flags);
 
 /**
  * Register frameworks with MCA.
@@ -186,8 +184,8 @@ OPAL_DECLSPEC int mca_base_framework_register (mca_base_framework_t *framework,
  * Call the MCA variable registration functions of each framework in the
  * frameworks array.
  */
-OPAL_DECLSPEC int mca_base_framework_register_list (mca_base_framework_t **frameworks,
-                                                    mca_base_register_flag_t flags);
+OPAL_DECLSPEC int mca_base_framework_register_list(mca_base_framework_t **frameworks,
+                                                   mca_base_register_flag_t flags);
 
 /**
  * Open a framework
@@ -199,8 +197,8 @@ OPAL_DECLSPEC int mca_base_framework_register_list (mca_base_framework_t **frame
  *
  * Call a framework's open function.
  */
-OPAL_DECLSPEC int mca_base_framework_open (mca_base_framework_t *framework,
-                                           mca_base_open_flag_t flags);
+OPAL_DECLSPEC int mca_base_framework_open(mca_base_framework_t *framework,
+                                          mca_base_open_flag_t flags);
 
 /**
  * Open frameworks
@@ -212,8 +210,8 @@ OPAL_DECLSPEC int mca_base_framework_open (mca_base_framework_t *framework,
  *
  * Call the open function on multiple frameworks
  */
-OPAL_DECLSPEC int mca_base_framework_open_list (mca_base_framework_t **frameworks,
-                                                mca_base_open_flag_t flags);
+OPAL_DECLSPEC int mca_base_framework_open_list(mca_base_framework_t **frameworks,
+                                               mca_base_open_flag_t flags);
 
 /**
  * Close a framework
@@ -225,7 +223,7 @@ OPAL_DECLSPEC int mca_base_framework_open_list (mca_base_framework_t **framework
  *
  * Call a framework's close function.
  */
-OPAL_DECLSPEC int mca_base_framework_close (mca_base_framework_t *framework);
+OPAL_DECLSPEC int mca_base_framework_close(mca_base_framework_t *framework);
 
 /**
  * Close frameworks
@@ -237,7 +235,7 @@ OPAL_DECLSPEC int mca_base_framework_close (mca_base_framework_t *framework);
  *
  * Call the close function on multiple frameworks
  */
-OPAL_DECLSPEC int mca_base_framework_close_list (mca_base_framework_t **frameworks);
+OPAL_DECLSPEC int mca_base_framework_close_list(mca_base_framework_t **frameworks);
 
 /**
  * Check if a framework is already registered
@@ -247,8 +245,7 @@ OPAL_DECLSPEC int mca_base_framework_close_list (mca_base_framework_t **framewor
  * @retval true if the framework's mca variables are registered
  * @retval false if not
  */
-OPAL_DECLSPEC bool mca_base_framework_is_registered (struct mca_base_framework_t *framework);
-
+OPAL_DECLSPEC bool mca_base_framework_is_registered(struct mca_base_framework_t *framework);
 
 /**
  * Check if a framework is already open
@@ -258,28 +255,29 @@ OPAL_DECLSPEC bool mca_base_framework_is_registered (struct mca_base_framework_t
  * @retval true if the framework is open
  * @retval false if not
  */
-OPAL_DECLSPEC bool mca_base_framework_is_open (struct mca_base_framework_t *framework);
-
+OPAL_DECLSPEC bool mca_base_framework_is_open(struct mca_base_framework_t *framework);
 
 /**
  * Macro to declare an MCA framework
  *
  * Example:
- *  MCA_BASE_FRAMEWORK_DECLARE(opal, foo, NULL, opal_foo_open, opal_foo_close, MCA_BASE_FRAMEWORK_FLAG_LAZY)
+ *  MCA_BASE_FRAMEWORK_DECLARE(opal, foo, NULL, opal_foo_open, opal_foo_close,
+ * MCA_BASE_FRAMEWORK_FLAG_LAZY)
  */
-#define MCA_BASE_FRAMEWORK_DECLARE(project, name, description, registerfn, openfn, closefn, static_components, flags) \
-    mca_base_framework_t project##_##name##_base_framework = {   \
-        .framework_project           = #project,                        \
-        .framework_name              = #name,                           \
-        .framework_description       = description,                     \
-        .framework_register          = registerfn,                      \
-        .framework_open              = openfn,                          \
-        .framework_close             = closefn,                         \
-        .framework_flags             = flags,                           \
-        .framework_refcnt            = 0,                               \
-        .framework_static_components = static_components,               \
-        .framework_selection         = NULL,                            \
-        .framework_verbose           = 0,                               \
-        .framework_output            = -1}
+#    define MCA_BASE_FRAMEWORK_DECLARE(project, name, description, registerfn, openfn, closefn, \
+                                       static_components, flags)                                \
+        mca_base_framework_t project##_##name##_base_framework                                  \
+            = {.framework_project = #project,                                                   \
+               .framework_name = #name,                                                         \
+               .framework_description = description,                                            \
+               .framework_register = registerfn,                                                \
+               .framework_open = openfn,                                                        \
+               .framework_close = closefn,                                                      \
+               .framework_flags = flags,                                                        \
+               .framework_refcnt = 0,                                                           \
+               .framework_static_components = static_components,                                \
+               .framework_selection = NULL,                                                     \
+               .framework_verbose = 0,                                                          \
+               .framework_output = -1}
 
 #endif /* OPAL_MCA_BASE_FRAMEWORK_H */
diff --git a/opal/mca/base/mca_base_list.c b/opal/mca/base/mca_base_list.c
index 0c97e69a0ce..9a233949a16 100644
--- a/opal/mca/base/mca_base_list.c
+++ b/opal/mca/base/mca_base_list.c
@@ -21,44 +21,37 @@
 #include "opal/class/opal_list.h"
 #include "opal/mca/base/base.h"
 
-
 /*
  * Local functions
  */
 static void cl_constructor(opal_object_t *obj);
 static void cpl_constructor(opal_object_t *obj);
 
-
 /*
  * Class instance of the mca_base_component_list_item_t class
  */
-OBJ_CLASS_INSTANCE(mca_base_component_list_item_t,
-                   opal_list_item_t, cl_constructor, NULL);
-
+OBJ_CLASS_INSTANCE(mca_base_component_list_item_t, opal_list_item_t, cl_constructor, NULL);
 
 /*
  * Class instance of the mca_base_component_priority_list_item_t class
  */
-OBJ_CLASS_INSTANCE(mca_base_component_priority_list_item_t,
-                   mca_base_component_list_item_t, cpl_constructor, NULL);
-
+OBJ_CLASS_INSTANCE(mca_base_component_priority_list_item_t, mca_base_component_list_item_t,
+                   cpl_constructor, NULL);
 
 /*
- * Just do basic sentinel intialization
+ * Just do basic sentinel initialization
  */
 static void cl_constructor(opal_object_t *obj)
 {
-  mca_base_component_list_item_t *cli = (mca_base_component_list_item_t *) obj;
-  cli->cli_component = NULL;
+    mca_base_component_list_item_t *cli = (mca_base_component_list_item_t *) obj;
+    cli->cli_component = NULL;
 }
 
-
 /*
- * Just do basic sentinel intialization
+ * Just do basic sentinel initialization
  */
 static void cpl_constructor(opal_object_t *obj)
 {
-  mca_base_component_priority_list_item_t *cpli =
-    (mca_base_component_priority_list_item_t *) obj;
-  cpli->cpli_priority = -1;
+    mca_base_component_priority_list_item_t *cpli = (mca_base_component_priority_list_item_t *) obj;
+    cpli->cpli_priority = -1;
 }
diff --git a/opal/mca/base/mca_base_open.c b/opal/mca/base/mca_base_open.c
index 282cfd26424..32f7890a781 100644
--- a/opal/mca/base/mca_base_open.c
+++ b/opal/mca/base/mca_base_open.c
@@ -10,7 +10,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2011-2017 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2011-2022 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2017      IBM Corporation.  All rights reserved.
@@ -29,21 +29,21 @@
 #include <stdio.h>
 #include <string.h>
 #ifdef HAVE_SYSLOG_H
-#include <syslog.h>
+#    include <syslog.h>
 #endif
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 
-#include "opal/runtime/opal.h"
-#include "opal/mca/installdirs/installdirs.h"
-#include "opal/util/output.h"
-#include "opal/util/printf.h"
-#include "opal/mca/mca.h"
+#include "opal/constants.h"
 #include "opal/mca/base/base.h"
 #include "opal/mca/base/mca_base_component_repository.h"
-#include "opal/constants.h"
+#include "opal/mca/installdirs/installdirs.h"
+#include "opal/mca/mca.h"
+#include "opal/runtime/opal.h"
 #include "opal/util/opal_environ.h"
+#include "opal/util/output.h"
+#include "opal/util/printf.h"
 
 /*
  * Public variables
@@ -52,8 +52,7 @@ char *mca_base_component_path = NULL;
 int mca_base_opened = 0;
 char *mca_base_system_default_path = NULL;
 char *mca_base_user_default_path = NULL;
-bool mca_base_component_show_load_errors =
-    (bool) OPAL_SHOW_LOAD_ERRORS_DEFAULT;
+char *mca_base_component_show_load_errors = NULL;
 bool mca_base_component_track_load_errors = false;
 bool mca_base_component_disable_dlopen = false;
 
@@ -65,7 +64,6 @@ static char *mca_base_verbose = NULL;
 static void set_defaults(opal_output_stream_t *lds);
 static void parse_verbose(char *e, opal_output_stream_t *lds);
 
-
 /*
  * Main MCA initialization.
  */
@@ -83,7 +81,8 @@ int mca_base_open(void)
     /* define the system and user default paths */
 #if OPAL_WANT_HOME_CONFIG_FILES
     mca_base_system_default_path = strdup(opal_install_dirs.opallibdir);
-    opal_asprintf(&mca_base_user_default_path, "%s"OPAL_PATH_SEP".openmpi"OPAL_PATH_SEP"components", opal_home_directory());
+    opal_asprintf(&mca_base_user_default_path,
+                  "%s" OPAL_PATH_SEP ".openmpi" OPAL_PATH_SEP "components", opal_home_directory());
 #else
     opal_asprintf(&mca_base_system_default_path, "%s", opal_install_dirs.opallibdir);
 #endif
@@ -92,47 +91,50 @@ int mca_base_open(void)
     if (NULL == mca_base_user_default_path) {
         value = strdup(mca_base_system_default_path);
     } else {
-        opal_asprintf(&value, "%s%c%s", mca_base_system_default_path,
-                 OPAL_ENV_SEP, mca_base_user_default_path);
+        opal_asprintf(&value, "%s%c%s", mca_base_system_default_path, OPAL_ENV_SEP,
+                      mca_base_user_default_path);
     }
 
     mca_base_component_path = value;
     var_id = mca_base_var_register("opal", "mca", "base", "component_path",
                                    "Path where to look for additional components",
-                                   MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
-                                   OPAL_INFO_LVL_9,
-                                   MCA_BASE_VAR_SCOPE_READONLY,
-                                   &mca_base_component_path);
+                                   MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_9,
+                                   MCA_BASE_VAR_SCOPE_READONLY, &mca_base_component_path);
     (void) mca_base_var_register_synonym(var_id, "opal", "mca", NULL, "component_path",
                                          MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
     free(value);
 
-    mca_base_component_show_load_errors =
-        (bool) OPAL_SHOW_LOAD_ERRORS_DEFAULT;
-    var_id = mca_base_var_register("opal", "mca", "base", "component_show_load_errors",
-                                   "Whether to show errors for components that failed to load or not",
-                                   MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
-                                   OPAL_INFO_LVL_9,
-                                   MCA_BASE_VAR_SCOPE_READONLY,
-                                   &mca_base_component_show_load_errors);
+    mca_base_component_show_load_errors = OPAL_SHOW_LOAD_ERRORS_DEFAULT;
+    var_id
+        = mca_base_var_register("opal", "mca", "base",
+                                "component_show_load_errors",
+                                "Whether to show warnings for components that fail to load or not.  Valid values are \"all\" (meaning: all load failures are reported), \"none\" (no load failures are reported), or a comma-delimited list of items, each of which can be a framework/component pair or a framework name (only load failures from the specifically-listed items are reported).  If the comma-delimited list is prefixed with \"^\", then orientation of the list is negated: warn about all load failures *except* for the listed items.",
+                                MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
+                                OPAL_INFO_LVL_9,
+                                MCA_BASE_VAR_SCOPE_READONLY,
+                                &mca_base_component_show_load_errors);
     (void) mca_base_var_register_synonym(var_id, "opal", "mca", NULL, "component_show_load_errors",
                                          MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
 
+    // Parse the mca_base_component_show_load_errors value
+    int ret = mca_base_show_load_errors_init();
+    if (OPAL_SUCCESS != ret) {
+        return ret;
+    }
+
     mca_base_component_track_load_errors = false;
-    var_id = mca_base_var_register("opal", "mca", "base", "component_track_load_errors",
-                                   "Whether to track errors for components that failed to load or not",
-                                   MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
-                                   OPAL_INFO_LVL_9,
-                                   MCA_BASE_VAR_SCOPE_READONLY,
-                                   &mca_base_component_track_load_errors);
+    var_id
+        = mca_base_var_register("opal", "mca", "base", "component_track_load_errors",
+                                "Whether to track errors for components that failed to load or not",
+                                MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9,
+                                MCA_BASE_VAR_SCOPE_READONLY, &mca_base_component_track_load_errors);
 
     mca_base_component_disable_dlopen = false;
-    var_id = mca_base_var_register("opal", "mca", "base", "component_disable_dlopen",
-                                   "Whether to attempt to disable opening dynamic components or not",
-                                   MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
-                                   OPAL_INFO_LVL_9,
-                                   MCA_BASE_VAR_SCOPE_READONLY,
-                                   &mca_base_component_disable_dlopen);
+    var_id
+        = mca_base_var_register("opal", "mca", "base", "component_disable_dlopen",
+                                "Whether to attempt to disable opening dynamic components or not",
+                                MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9,
+                                MCA_BASE_VAR_SCOPE_READONLY, &mca_base_component_disable_dlopen);
     (void) mca_base_var_register_synonym(var_id, "opal", "mca", NULL, "component_disable_dlopen",
                                          MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
 
@@ -140,16 +142,19 @@ int mca_base_open(void)
     char *str = getenv("OPAL_OUTPUT_INTERNAL_TO_STDOUT");
     if (NULL != str && str[0] == '1') {
         mca_base_verbose = "stdout";
-    }
-    else {
+    } else {
         mca_base_verbose = "stderr";
     }
-    var_id = mca_base_var_register("opal", "mca", "base", "verbose",
-                                   "Specifies where the default error output stream goes (this is separate from distinct help messages).  Accepts a comma-delimited list of: stderr, stdout, syslog, syslogpri:<notice|info|debug>, syslogid:<str> (where str is the prefix string for all syslog notices), file[:filename] (if filename is not specified, a default filename is used), fileappend (if not specified, the file is opened for truncation), level[:N] (if specified, integer verbose level; otherwise, 0 is implied)",
-                                   MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
-                                   OPAL_INFO_LVL_9,
-                                   MCA_BASE_VAR_SCOPE_READONLY,
-                                   &mca_base_verbose);
+    var_id = mca_base_var_register(
+        "opal", "mca", "base", "verbose",
+        "Specifies where the default error output stream goes (this is separate from distinct help "
+        "messages).  Accepts a comma-delimited list of: stderr, stdout, syslog, "
+        "syslogpri:<notice|info|debug>, syslogid:<str> (where str is the prefix string for all "
+        "syslog notices), file[:filename] (if filename is not specified, a default filename is "
+        "used), fileappend (if not specified, the file is opened for truncation), level[:N] (if "
+        "specified, integer verbose level; otherwise, 0 is implied)",
+        MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+        &mca_base_verbose);
     (void) mca_base_var_register_synonym(var_id, "opal", "mca", NULL, "verbose",
                                          MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
 
@@ -162,17 +167,16 @@ int mca_base_open(void)
     hostname = opal_gethostname();
     opal_asprintf(&lds.lds_prefix, "[%s:%05d] ", hostname, getpid());
     opal_output_reopen(0, &lds);
-    opal_output_verbose (MCA_BASE_VERBOSE_COMPONENT, 0, "mca: base: opening components");
+    opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT, 0, "mca: base: opening components");
     free(lds.lds_prefix);
 
     /* Open up the component repository */
 
-    opal_finalize_register_cleanup (mca_base_close);
+    opal_finalize_register_cleanup(mca_base_close);
 
     return mca_base_component_repository_init();
 }
 
-
 /*
  * Set sane default values for the lds
  */
@@ -189,7 +193,6 @@ static void set_defaults(opal_output_stream_t *lds)
     lds->lds_want_stderr = true;
 }
 
-
 /*
  * Parse the value of an environment variable describing verbosity
  */
@@ -220,28 +223,28 @@ static void parse_verbose(char *e, opal_output_stream_t *lds)
             have_output = true;
 #else
             opal_output(0, "syslog support requested but not available on this system");
-#endif  /* defined(HAVE_SYSLOG) && defined(HAVE_SYSLOG_H) */
-        }
-        else if (strncasecmp(ptr, "syslogpri:", 10) == 0) {
+#endif /* defined(HAVE_SYSLOG) && defined(HAVE_SYSLOG_H) */
+        } else if (strncasecmp(ptr, "syslogpri:", 10) == 0) {
 #if defined(HAVE_SYSLOG) && defined(HAVE_SYSLOG_H)
             lds->lds_want_syslog = true;
             have_output = true;
-            if (strcasecmp(ptr + 10, "notice") == 0)
+            if (strcasecmp(ptr + 10, "notice") == 0) {
                 lds->lds_syslog_priority = LOG_NOTICE;
-            else if (strcasecmp(ptr + 10, "INFO") == 0)
+            } else if (strcasecmp(ptr + 10, "INFO") == 0) {
                 lds->lds_syslog_priority = LOG_INFO;
-            else if (strcasecmp(ptr + 10, "DEBUG") == 0)
+            } else if (strcasecmp(ptr + 10, "DEBUG") == 0) {
                 lds->lds_syslog_priority = LOG_DEBUG;
+            }
 #else
             opal_output(0, "syslog support requested but not available on this system");
-#endif  /* defined(HAVE_SYSLOG) && defined(HAVE_SYSLOG_H) */
+#endif /* defined(HAVE_SYSLOG) && defined(HAVE_SYSLOG_H) */
         } else if (strncasecmp(ptr, "syslogid:", 9) == 0) {
 #if defined(HAVE_SYSLOG) && defined(HAVE_SYSLOG_H)
             lds->lds_want_syslog = true;
             lds->lds_syslog_ident = ptr + 9;
 #else
             opal_output(0, "syslog support requested but not available on this system");
-#endif  /* defined(HAVE_SYSLOG) && defined(HAVE_SYSLOG_H) */
+#endif /* defined(HAVE_SYSLOG) && defined(HAVE_SYSLOG_H) */
         }
 
         else if (strcasecmp(ptr, "stdout") == 0) {
@@ -267,8 +270,9 @@ static void parse_verbose(char *e, opal_output_stream_t *lds)
 
         else if (strncasecmp(ptr, "level", 5) == 0) {
             lds->lds_verbose_level = 0;
-            if (ptr[5] == OPAL_ENV_SEP)
+            if (ptr[5] == OPAL_ENV_SEP) {
                 lds->lds_verbose_level = atoi(ptr + 6);
+            }
         }
 
         if (NULL == next) {
diff --git a/opal/mca/base/mca_base_parse_paramfile.c b/opal/mca/base/mca_base_parse_paramfile.c
index db19d143a2d..81561818885 100644
--- a/opal/mca/base/mca_base_parse_paramfile.c
+++ b/opal/mca/base/mca_base_parse_paramfile.c
@@ -25,19 +25,19 @@
 #include <string.h>
 
 #include "opal/class/opal_list.h"
-#include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
 #include "opal/mca/base/mca_base_vari.h"
+#include "opal/mca/mca.h"
 #include "opal/util/keyval_parse.h"
 #include "opal/util/output.h"
 static void save_value(const char *name, const char *value);
 
-static char * file_being_read;
-static opal_list_t * _param_list;
+static char *file_being_read;
+static opal_list_t *_param_list;
 
 int mca_base_parse_paramfile(const char *paramfile, opal_list_t *list)
 {
-    file_being_read = (char*)paramfile;
+    file_being_read = (char *) paramfile;
     _param_list = list;
 
     return opal_util_keyval_parse(paramfile, save_value);
@@ -57,10 +57,10 @@ static void save_value(const char *name, const char *value)
        already have a param of this name.  If we do, just replace the
        value. */
 
-    OPAL_LIST_FOREACH(fv, _param_list, mca_base_var_file_value_t) {
+    OPAL_LIST_FOREACH (fv, _param_list, mca_base_var_file_value_t) {
         if (0 == strcmp(name, fv->mbvfv_var)) {
             if (NULL != fv->mbvfv_value) {
-                free (fv->mbvfv_value);
+                free(fv->mbvfv_value);
             }
             found = true;
             break;
@@ -79,6 +79,6 @@ static void save_value(const char *name, const char *value)
     }
 
     fv->mbvfv_value = value ? strdup(value) : NULL;
-    fv->mbvfv_file  = file_being_read;
+    fv->mbvfv_file = file_being_read;
     fv->mbvfv_lineno = opal_util_keyval_parse_lineno;
 }
diff --git a/opal/mca/base/mca_base_pvar.c b/opal/mca/base/mca_base_pvar.c
index 1bc71161743..2b1f4dc0c38 100644
--- a/opal/mca/base/mca_base_pvar.c
+++ b/opal/mca/base/mca_base_pvar.c
@@ -20,11 +20,11 @@
 #include "opal/mca/base/mca_base_vari.h"
 
 #include <stddef.h>
-#include <sys/time.h>
 #include <sys/resource.h>
+#include <sys/time.h>
 
-#include "opal/class/opal_pointer_array.h"
 #include "opal/class/opal_hash_table.h"
+#include "opal/class/opal_pointer_array.h"
 #include "opal/util/minmax.h"
 #include "opal/util/printf.h"
 
@@ -33,23 +33,14 @@ static opal_pointer_array_t registered_pvars;
 static bool mca_base_pvar_initialized = false;
 static int pvar_count = 0;
 
-static int mca_base_pvar_get_internal (int index, mca_base_pvar_t **pvar, bool invalidok);
+static int mca_base_pvar_get_internal(int index, mca_base_pvar_t **pvar, bool invalidok);
 
 /* string representations of class names */
-static const char *pvar_class_names[] = {
-    "state",
-    "level",
-    "size",
-    "percentage",
-    "high watermark",
-    "low watermark",
-    "counter",
-    "aggregate",
-    "timer",
-    "generic"
-};
-
-int mca_base_pvar_init (void)
+static const char *pvar_class_names[] = {"state",          "level",         "size",    "percentage",
+                                         "high watermark", "low watermark", "counter", "aggregate",
+                                         "timer",          "generic"};
+
+int mca_base_pvar_init(void)
 {
     int ret = OPAL_SUCCESS;
 
@@ -60,7 +51,7 @@ int mca_base_pvar_init (void)
         opal_pointer_array_init(&registered_pvars, 128, 2048, 128);
 
         OBJ_CONSTRUCT(&mca_base_pvar_index_hash, opal_hash_table_t);
-        ret = opal_hash_table_init (&mca_base_pvar_index_hash, 1024);
+        ret = opal_hash_table_init(&mca_base_pvar_index_hash, 1024);
         if (OPAL_SUCCESS != ret) {
             mca_base_pvar_initialized = false;
             OBJ_DESTRUCT(&registered_pvars);
@@ -71,37 +62,38 @@ int mca_base_pvar_init (void)
     return ret;
 }
 
-int mca_base_pvar_find (const char *project, const char *framework, const char *component, const char *name)
+int mca_base_pvar_find(const char *project, const char *framework, const char *component,
+                       const char *name)
 {
     char *full_name;
     int ret, index;
 
-    ret = mca_base_var_generate_full_name4 (NULL, framework, component, name, &full_name);
+    ret = mca_base_var_generate_full_name4(NULL, framework, component, name, &full_name);
     if (OPAL_SUCCESS != ret) {
         return OPAL_ERROR;
     }
 
-    ret = mca_base_pvar_find_by_name (full_name, MCA_BASE_PVAR_CLASS_ANY, &index);
-    free (full_name);
+    ret = mca_base_pvar_find_by_name(full_name, MCA_BASE_PVAR_CLASS_ANY, &index);
+    free(full_name);
 
     /* NTH: should we verify the name components match the returned variable? */
 
     return (OPAL_SUCCESS != ret) ? ret : index;
 }
 
-int mca_base_pvar_find_by_name (const char *full_name, int var_class, int *index)
+int mca_base_pvar_find_by_name(const char *full_name, int var_class, int *index)
 {
     mca_base_pvar_t *pvar;
     void *tmp;
     int rc;
 
-    rc = opal_hash_table_get_value_ptr (&mca_base_pvar_index_hash, full_name, strlen (full_name),
-                                        &tmp);
+    rc = opal_hash_table_get_value_ptr(&mca_base_pvar_index_hash, full_name, strlen(full_name),
+                                       &tmp);
     if (OPAL_SUCCESS != rc) {
         return rc;
     }
 
-    rc = mca_base_pvar_get_internal ((int)(uintptr_t) tmp, &pvar, false);
+    rc = mca_base_pvar_get_internal((int) (uintptr_t) tmp, &pvar, false);
     if (OPAL_SUCCESS != rc) {
         return rc;
     }
@@ -110,20 +102,20 @@ int mca_base_pvar_find_by_name (const char *full_name, int var_class, int *index
         return OPAL_ERR_NOT_FOUND;
     }
 
-    *index = (int)(uintptr_t) tmp;
+    *index = (int) (uintptr_t) tmp;
 
     return OPAL_SUCCESS;
 }
 
-int mca_base_pvar_finalize (void)
+int mca_base_pvar_finalize(void)
 {
     int i;
 
-    if (mca_base_pvar_initialized)  {
+    if (mca_base_pvar_initialized) {
         mca_base_pvar_initialized = false;
 
-        for (i = 0 ; i < pvar_count ; ++i) {
-            mca_base_pvar_t *pvar = opal_pointer_array_get_item (&registered_pvars, i);
+        for (i = 0; i < pvar_count; ++i) {
+            mca_base_pvar_t *pvar = opal_pointer_array_get_item(&registered_pvars, i);
             if (pvar) {
                 OBJ_RELEASE(pvar);
             }
@@ -138,33 +130,36 @@ int mca_base_pvar_finalize (void)
     return OPAL_SUCCESS;
 }
 
-int mca_base_pvar_get_count (int *count)
+int mca_base_pvar_get_count(int *count)
 {
     *count = pvar_count;
     return OPAL_SUCCESS;
 }
 
-static int mca_base_pvar_default_get_value (const mca_base_pvar_t *pvar, void *value, void *obj_handle)
+static int mca_base_pvar_default_get_value(const mca_base_pvar_t *pvar, void *value,
+                                           void *obj_handle)
 {
     /* not used */
     (void) obj_handle;
 
-    memmove (value, pvar->ctx, ompi_var_type_sizes[pvar->type]);
+    memmove(value, pvar->ctx, ompi_var_type_sizes[pvar->type]);
 
     return OPAL_SUCCESS;
 }
 
-static int mca_base_pvar_default_set_value (mca_base_pvar_t *pvar, const void *value, void *obj_handle)
+static int mca_base_pvar_default_set_value(mca_base_pvar_t *pvar, const void *value,
+                                           void *obj_handle)
 {
     /* not used */
     (void) obj_handle;
 
-    memmove (pvar->ctx, value, ompi_var_type_sizes[pvar->type]);
+    memmove(pvar->ctx, value, ompi_var_type_sizes[pvar->type]);
 
     return OPAL_SUCCESS;
 }
 
-static int mca_base_pvar_notify_ignore (mca_base_pvar_t *pvar, mca_base_pvar_event_t event, void *obj_handle, int *count)
+static int mca_base_pvar_notify_ignore(mca_base_pvar_t *pvar, mca_base_pvar_event_t event,
+                                       void *obj_handle, int *count)
 {
     /* silence compiler warnings */
     (void) pvar;
@@ -178,11 +173,13 @@ static int mca_base_pvar_notify_ignore (mca_base_pvar_t *pvar, mca_base_pvar_eve
     return OPAL_SUCCESS;
 }
 
-int mca_base_pvar_register (const char *project, const char *framework, const char *component, const char *name,
-                            const char *description, mca_base_var_info_lvl_t verbosity,
-                            int var_class, mca_base_var_type_t type, mca_base_var_enum_t *enumerator,
-                            int bind, mca_base_pvar_flag_t flags, mca_base_get_value_fn_t get_value,
-                            mca_base_set_value_fn_t set_value, mca_base_notify_fn_t notify, void *ctx)
+int mca_base_pvar_register(const char *project, const char *framework, const char *component,
+                           const char *name, const char *description,
+                           mca_base_var_info_lvl_t verbosity, int var_class,
+                           mca_base_var_type_t type, mca_base_var_enum_t *enumerator, int bind,
+                           mca_base_pvar_flag_t flags, mca_base_get_value_fn_t get_value,
+                           mca_base_set_value_fn_t set_value, mca_base_notify_fn_t notify,
+                           void *ctx)
 {
     int ret, group_index, pvar_index;
     mca_base_pvar_t *pvar;
@@ -193,7 +190,7 @@ int mca_base_pvar_register (const char *project, const char *framework, const ch
     }
 
     /* ensure the caller did not set an invalid flag */
-    assert (!(flags & 0x3f));
+    assert(!(flags & 0x3f));
 
     flags &= ~MCA_BASE_PVAR_FLAG_INVALID;
 
@@ -217,10 +214,8 @@ int mca_base_pvar_register (const char *project, const char *framework, const ch
     case MCA_BASE_PVAR_CLASS_LOWWATERMARK:
     case MCA_BASE_PVAR_CLASS_AGGREGATE:
     case MCA_BASE_PVAR_CLASS_TIMER:
-        if (MCA_BASE_VAR_TYPE_UNSIGNED_INT != type &&
-            MCA_BASE_VAR_TYPE_UNSIGNED_LONG != type &&
-            MCA_BASE_VAR_TYPE_UNSIGNED_LONG_LONG != type &&
-            MCA_BASE_VAR_TYPE_DOUBLE != type) {
+        if (MCA_BASE_VAR_TYPE_UNSIGNED_INT != type && MCA_BASE_VAR_TYPE_UNSIGNED_LONG != type
+            && MCA_BASE_VAR_TYPE_UNSIGNED_LONG_LONG != type && MCA_BASE_VAR_TYPE_DOUBLE != type) {
             return OPAL_ERR_BAD_PARAM;
         }
         break;
@@ -239,12 +234,12 @@ int mca_base_pvar_register (const char *project, const char *framework, const ch
     }
 
     /* update this assert if more MPIT verbosity levels are added */
-    assert (verbosity >= OPAL_INFO_LVL_1 && verbosity <= OPAL_INFO_LVL_9);
+    assert(verbosity >= OPAL_INFO_LVL_1 && verbosity <= OPAL_INFO_LVL_9);
 
     /* check if this variable is already registered */
-    ret = mca_base_pvar_find (project, framework, component, name);
+    ret = mca_base_pvar_find(project, framework, component, name);
     if (OPAL_SUCCESS <= ret) {
-        ret = mca_base_pvar_get_internal (ret, &pvar, true);
+        ret = mca_base_pvar_get_internal(ret, &pvar, true);
         if (OPAL_SUCCESS != ret) {
             /* inconsistent internal state */
             return OPAL_ERROR;
@@ -255,7 +250,7 @@ int mca_base_pvar_register (const char *project, const char *framework, const ch
         }
     } else {
         /* find/register an MCA parameter group for this performance variable */
-        group_index = mca_base_var_group_register (project, framework, component, NULL);
+        group_index = mca_base_var_group_register(project, framework, component, NULL);
         if (-1 > group_index) {
             return group_index;
         }
@@ -268,7 +263,7 @@ int mca_base_pvar_register (const char *project, const char *framework, const ch
 
         do {
             /* generate the variable's full name */
-            ret = mca_base_var_generate_full_name4 (NULL, framework, component, name, &pvar->name);
+            ret = mca_base_var_generate_full_name4(NULL, framework, component, name, &pvar->name);
             if (OPAL_SUCCESS != ret) {
                 ret = OPAL_ERR_OUT_OF_RESOURCE;
                 break;
@@ -282,7 +277,7 @@ int mca_base_pvar_register (const char *project, const char *framework, const ch
                 }
             }
 
-            pvar_index = opal_pointer_array_add (&registered_pvars, pvar);
+            pvar_index = opal_pointer_array_add(&registered_pvars, pvar);
             if (0 > pvar_index) {
                 break;
             }
@@ -290,15 +285,15 @@ int mca_base_pvar_register (const char *project, const char *framework, const ch
 
             /* add this performance variable to the MCA variable group */
             if (0 <= group_index) {
-                ret = mca_base_var_group_add_pvar (group_index, pvar_index);
+                ret = mca_base_var_group_add_pvar(group_index, pvar_index);
                 if (0 > ret) {
                     break;
                 }
             }
 
             pvar->pvar_index = pvar_count;
-            opal_hash_table_set_value_ptr (&mca_base_pvar_index_hash, pvar->name, strlen (pvar->name),
-                                           (void *)(uintptr_t) pvar->pvar_index);
+            opal_hash_table_set_value_ptr(&mca_base_pvar_index_hash, pvar->name, strlen(pvar->name),
+                                          (void *) (uintptr_t) pvar->pvar_index);
 
             pvar_count++;
             ret = OPAL_SUCCESS;
@@ -312,10 +307,10 @@ int mca_base_pvar_register (const char *project, const char *framework, const ch
         pvar->group_index = group_index;
     }
 
-    pvar->verbosity   = verbosity;
-    pvar->var_class   = var_class;
-    pvar->type        = type;
-    pvar->enumerator  = enumerator;
+    pvar->verbosity = verbosity;
+    pvar->var_class = var_class;
+    pvar->type = type;
+    pvar->enumerator = enumerator;
     if (enumerator) {
         OBJ_RETAIN(enumerator);
     }
@@ -330,33 +325,36 @@ int mca_base_pvar_register (const char *project, const char *framework, const ch
         pvar->set_value = set_value ? set_value : mca_base_pvar_default_set_value;
     }
 
-    pvar->ctx        = ctx;
+    pvar->ctx = ctx;
 
     return pvar->pvar_index;
 }
 
-int mca_base_component_pvar_register (const mca_base_component_t *component, const char *name,
-                                      const char *description, mca_base_var_info_lvl_t verbosity,
-                                      int var_class, mca_base_var_type_t type, mca_base_var_enum_t *enumerator,
-                                      int bind, mca_base_pvar_flag_t flags, mca_base_get_value_fn_t get_value,
-                                      mca_base_set_value_fn_t set_value, mca_base_notify_fn_t notify, void *ctx)
+int mca_base_component_pvar_register(const mca_base_component_t *component, const char *name,
+                                     const char *description, mca_base_var_info_lvl_t verbosity,
+                                     int var_class, mca_base_var_type_t type,
+                                     mca_base_var_enum_t *enumerator, int bind,
+                                     mca_base_pvar_flag_t flags, mca_base_get_value_fn_t get_value,
+                                     mca_base_set_value_fn_t set_value, mca_base_notify_fn_t notify,
+                                     void *ctx)
 {
     /* invalidate this variable if the component's group is deregistered */
-    return mca_base_pvar_register(component->mca_project_name, component->mca_type_name, component->mca_component_name,
-                                  name, description, verbosity, var_class, type, enumerator, bind,
-                                  flags | MCA_BASE_PVAR_FLAG_IWG, get_value, set_value, notify, ctx);
+    return mca_base_pvar_register(component->mca_project_name, component->mca_type_name,
+                                  component->mca_component_name, name, description, verbosity,
+                                  var_class, type, enumerator, bind, flags | MCA_BASE_PVAR_FLAG_IWG,
+                                  get_value, set_value, notify, ctx);
 }
 
-static int mca_base_pvar_get_internal (int index, mca_base_pvar_t **pvar, bool invalidok)
+static int mca_base_pvar_get_internal(int index, mca_base_pvar_t **pvar, bool invalidok)
 {
     if (index >= pvar_count) {
         return OPAL_ERR_VALUE_OUT_OF_BOUNDS;
     }
 
-    *pvar = opal_pointer_array_get_item (&registered_pvars, index);
+    *pvar = opal_pointer_array_get_item(&registered_pvars, index);
 
     /* variables should never be removed per MPI 3.0 § 14.3.7 */
-    assert (*pvar);
+    assert(*pvar);
 
     if (((*pvar)->flags & MCA_BASE_PVAR_FLAG_INVALID) && !invalidok) {
         *pvar = NULL;
@@ -366,17 +364,17 @@ static int mca_base_pvar_get_internal (int index, mca_base_pvar_t **pvar, bool i
     return OPAL_SUCCESS;
 }
 
-int mca_base_pvar_get (int index, const mca_base_pvar_t **pvar)
+int mca_base_pvar_get(int index, const mca_base_pvar_t **pvar)
 {
-    return mca_base_pvar_get_internal (index, (mca_base_pvar_t **) pvar, false);
+    return mca_base_pvar_get_internal(index, (mca_base_pvar_t **) pvar, false);
 }
 
-int mca_base_pvar_mark_invalid (int index)
+int mca_base_pvar_mark_invalid(int index)
 {
     mca_base_pvar_t *pvar;
     int ret;
 
-    ret = mca_base_pvar_get_internal (index, &pvar, false);
+    ret = mca_base_pvar_get_internal(index, &pvar, false);
     if (OPAL_SUCCESS != ret) {
         return ret;
     }
@@ -386,47 +384,48 @@ int mca_base_pvar_mark_invalid (int index)
     return OPAL_SUCCESS;
 }
 
-int mca_base_pvar_notify (mca_base_pvar_handle_t *handle, mca_base_pvar_event_t event, int *count)
+int mca_base_pvar_notify(mca_base_pvar_handle_t *handle, mca_base_pvar_event_t event, int *count)
 {
-    if (mca_base_pvar_is_invalid (handle->pvar)) {
+    if (mca_base_pvar_is_invalid(handle->pvar)) {
         return OPAL_ERR_NOT_BOUND;
     }
 
-    return handle->pvar->notify (handle->pvar, event, handle->obj_handle, count);
+    return handle->pvar->notify(handle->pvar, event, handle->obj_handle, count);
 }
 
-int mca_base_pvar_update_all_handles (int index, const void *obj)
+int mca_base_pvar_update_all_handles(int index, const void *obj)
 {
     mca_base_pvar_handle_t *handle, *next;
     mca_base_pvar_t *pvar;
     int ret;
 
-    ret = mca_base_pvar_get_internal (index, &pvar, false);
+    ret = mca_base_pvar_get_internal(index, &pvar, false);
     if (OPAL_SUCCESS != ret) {
         return ret;
     }
 
-    if (0 == opal_list_get_size (&pvar->bound_handles)) {
+    if (0 == opal_list_get_size(&pvar->bound_handles)) {
         /* nothing to do */
         return OPAL_SUCCESS;
     }
 
     /* TODO -- probably need to add a handle/variable lock */
-    OPAL_LIST_FOREACH_SAFE(handle, next, &pvar->bound_handles, mca_base_pvar_handle_t) {
-        handle = (mca_base_pvar_handle_t *)((char *) handle - offsetof (mca_base_pvar_handle_t, list2));
+    OPAL_LIST_FOREACH_SAFE (handle, next, &pvar->bound_handles, mca_base_pvar_handle_t) {
+        handle = (mca_base_pvar_handle_t *) ((char *) handle
+                                             - offsetof(mca_base_pvar_handle_t, list2));
 
         if (handle->obj_handle != obj) {
             continue;
         }
 
-        (void) mca_base_pvar_handle_update (handle);
+        (void) mca_base_pvar_handle_update(handle);
     }
 
     return OPAL_SUCCESS;
 }
 
-int mca_base_pvar_handle_alloc (mca_base_pvar_session_t *session, int index, void *obj_handle,
-                                mca_base_pvar_handle_t **handle, int *count)
+int mca_base_pvar_handle_alloc(mca_base_pvar_session_t *session, int index, void *obj_handle,
+                               mca_base_pvar_handle_t **handle, int *count)
 {
     mca_base_pvar_handle_t *pvar_handle = NULL;
     size_t datatype_size;
@@ -435,7 +434,7 @@ int mca_base_pvar_handle_alloc (mca_base_pvar_session_t *session, int index, voi
 
     do {
         /* find the requested performance variable */
-        ret = mca_base_pvar_get_internal (index, &pvar, false);
+        ret = mca_base_pvar_get_internal(index, &pvar, false);
         if (OPAL_SUCCESS != ret) {
             break;
         }
@@ -456,7 +455,7 @@ int mca_base_pvar_handle_alloc (mca_base_pvar_session_t *session, int index, voi
             break;
         }
 
-        pvar_handle->obj_handle = (NULL == obj_handle ? NULL : *(void**)obj_handle);
+        pvar_handle->obj_handle = (NULL == obj_handle ? NULL : *(void **) obj_handle);
         pvar_handle->pvar = pvar;
 
         *handle = pvar_handle;
@@ -464,7 +463,7 @@ int mca_base_pvar_handle_alloc (mca_base_pvar_session_t *session, int index, voi
         /* notify the variable that a handle has been bound and determine
            how many values this handle has. NTH: finding the count should
            probably be pushed into a separate function. */
-        ret = mca_base_pvar_notify (pvar_handle, MCA_BASE_PVAR_HANDLE_BIND, count);
+        ret = mca_base_pvar_notify(pvar_handle, MCA_BASE_PVAR_HANDLE_BIND, count);
         if (0 > ret) {
             ret = OPAL_ERROR;
             break;
@@ -480,30 +479,30 @@ int mca_base_pvar_handle_alloc (mca_base_pvar_session_t *session, int index, voi
             break;
         }
 
-        if (!mca_base_pvar_is_continuous (pvar) || mca_base_pvar_is_sum (pvar) ||
-            mca_base_pvar_is_watermark (pvar)) {
+        if (!mca_base_pvar_is_continuous(pvar) || mca_base_pvar_is_sum(pvar)
+            || mca_base_pvar_is_watermark(pvar)) {
             /* if a variable is not continuous we will need to keep track of its last value
                to support start->stop->read correctly. use calloc to initialize the current
                value to 0. */
-            pvar_handle->current_value = calloc (*count, datatype_size);
+            pvar_handle->current_value = calloc(*count, datatype_size);
             if (NULL == pvar_handle->current_value) {
                 ret = OPAL_ERR_OUT_OF_RESOURCE;
                 break;
             }
         }
 
-        if (mca_base_pvar_is_sum (pvar) || mca_base_pvar_is_watermark (pvar)) {
+        if (mca_base_pvar_is_sum(pvar) || mca_base_pvar_is_watermark(pvar)) {
             /* for sums (counters, timers, etc) we need to keep track of
                what the last value of the underlying counter was. this allows
                us to push the computation of handle values from the event(s)
                (which could be in a critical path) to pvar read/stop/reset/etc */
-            pvar_handle->tmp_value = calloc (*count, datatype_size);
+            pvar_handle->tmp_value = calloc(*count, datatype_size);
             if (NULL == pvar_handle->tmp_value) {
                 ret = OPAL_ERR_OUT_OF_RESOURCE;
                 break;
             }
 
-            pvar_handle->last_value = calloc (*count, datatype_size);
+            pvar_handle->last_value = calloc(*count, datatype_size);
             if (NULL == pvar_handle->last_value) {
                 ret = OPAL_ERR_OUT_OF_RESOURCE;
                 break;
@@ -512,12 +511,13 @@ int mca_base_pvar_handle_alloc (mca_base_pvar_session_t *session, int index, voi
             /* get the current value of the performance variable if this is a
                continuous sum or watermark. if this variable needs to be started first the
                current value is not relevant. */
-            if (mca_base_pvar_is_continuous (pvar)) {
-                if (mca_base_pvar_is_sum (pvar)) {
-                    ret = pvar->get_value (pvar, pvar_handle->last_value, pvar_handle->obj_handle);
+            if (mca_base_pvar_is_continuous(pvar)) {
+                if (mca_base_pvar_is_sum(pvar)) {
+                    ret = pvar->get_value(pvar, pvar_handle->last_value, pvar_handle->obj_handle);
                 } else {
                     /* the initial value of a watermark is the current value of the variable */
-                    ret = pvar->get_value (pvar, pvar_handle->current_value, pvar_handle->obj_handle);
+                    ret = pvar->get_value(pvar, pvar_handle->current_value,
+                                          pvar_handle->obj_handle);
                 }
 
                 if (OPAL_SUCCESS != ret) {
@@ -529,10 +529,10 @@ int mca_base_pvar_handle_alloc (mca_base_pvar_session_t *session, int index, voi
         pvar_handle->session = session;
 
         /* the handle is ready. add it to the appropriate lists */
-        opal_list_append (&session->handles, &pvar_handle->super);
-        opal_list_append (&pvar->bound_handles, &pvar_handle->list2);
+        opal_list_append(&session->handles, &pvar_handle->super);
+        opal_list_append(&pvar->bound_handles, &pvar_handle->list2);
 
-        if (mca_base_pvar_is_continuous (pvar)) {
+        if (mca_base_pvar_is_continuous(pvar)) {
             /* mark this variable as started */
             pvar_handle->started = true;
         }
@@ -547,52 +547,56 @@ int mca_base_pvar_handle_alloc (mca_base_pvar_session_t *session, int index, voi
     return ret;
 }
 
-int mca_base_pvar_handle_free (mca_base_pvar_handle_t *handle)
+int mca_base_pvar_handle_free(mca_base_pvar_handle_t *handle)
 {
     OBJ_RELEASE(handle);
 
     return OPAL_SUCCESS;
 }
 
-int mca_base_pvar_handle_update (mca_base_pvar_handle_t *handle)
+int mca_base_pvar_handle_update(mca_base_pvar_handle_t *handle)
 {
     int i, ret;
     void *tmp;
 
-    if (mca_base_pvar_is_invalid (handle->pvar)) {
+    if (mca_base_pvar_is_invalid(handle->pvar)) {
         return OPAL_ERR_NOT_BOUND;
     }
 
-    if (!mca_base_pvar_handle_is_running (handle)) {
+    if (!mca_base_pvar_handle_is_running(handle)) {
         return OPAL_SUCCESS;
     }
 
-    if (mca_base_pvar_is_sum (handle->pvar) || mca_base_pvar_is_watermark (handle->pvar)) {
-        ret = handle->pvar->get_value (handle->pvar, handle->tmp_value, handle->obj_handle);
+    if (mca_base_pvar_is_sum(handle->pvar) || mca_base_pvar_is_watermark(handle->pvar)) {
+        ret = handle->pvar->get_value(handle->pvar, handle->tmp_value, handle->obj_handle);
         if (OPAL_SUCCESS != ret) {
             return OPAL_ERROR;
         }
 
-        if (mca_base_pvar_is_sum (handle->pvar)) {
-            for (i = 0 ; i < handle->count ; ++i) {
-                /* the instance started at 0. need to subract the initial value off the
+        if (mca_base_pvar_is_sum(handle->pvar)) {
+            for (i = 0; i < handle->count; ++i) {
+                /* the instance started at 0. need to subtract the initial value off the
                    result. */
                 switch (handle->pvar->type) {
                 case MCA_BASE_VAR_TYPE_UNSIGNED_INT:
-                    ((unsigned *) handle->current_value)[i] += ((unsigned *) handle->tmp_value)[i] -
-                        ((unsigned *) handle->last_value)[i];
+                    ((unsigned *) handle->current_value)[i] += ((unsigned *) handle->tmp_value)[i]
+                                                               - ((unsigned *)
+                                                                      handle->last_value)[i];
                     break;
                 case MCA_BASE_VAR_TYPE_UNSIGNED_LONG:
-                    ((unsigned long *) handle->current_value)[i] += ((unsigned long *) handle->tmp_value)[i] -
-                        ((unsigned long *) handle->last_value)[i];
+                    ((unsigned long *) handle->current_value)[i] += ((unsigned long *)
+                                                                         handle->tmp_value)[i]
+                                                                    - ((unsigned long *)
+                                                                           handle->last_value)[i];
                     break;
                 case MCA_BASE_VAR_TYPE_UNSIGNED_LONG_LONG:
-                    ((unsigned long long *) handle->current_value)[i] += ((unsigned long long *) handle->tmp_value)[i] -
-                        ((unsigned long long *) handle->last_value)[i];
+                    ((unsigned long long *) handle->current_value)[i]
+                        += ((unsigned long long *) handle->tmp_value)[i]
+                           - ((unsigned long long *) handle->last_value)[i];
                     break;
                 case MCA_BASE_VAR_TYPE_DOUBLE:
-                    ((double *) handle->current_value)[i] += ((double *) handle->tmp_value)[i] -
-                        ((double *) handle->last_value)[i];
+                    ((double *) handle->current_value)[i] += ((double *) handle->tmp_value)[i]
+                                                             - ((double *) handle->last_value)[i];
                     break;
                 default:
                     /* shouldn't happen */
@@ -604,24 +608,28 @@ int mca_base_pvar_handle_update (mca_base_pvar_handle_t *handle)
             handle->tmp_value = handle->last_value;
             handle->last_value = tmp;
         } else {
-            for (i = 0 ; i < handle->count ; ++i) {
+            for (i = 0; i < handle->count; ++i) {
                 if (MCA_BASE_PVAR_CLASS_LOWWATERMARK == handle->pvar->var_class) {
                     switch (handle->pvar->type) {
                     case MCA_BASE_VAR_TYPE_UNSIGNED_INT:
-                        ((unsigned *) handle->current_value)[i] = opal_min(((unsigned *) handle->tmp_value)[i],
-                                                                           ((unsigned *) handle->current_value)[i]);
+                        ((unsigned *) handle->current_value)[i]
+                            = opal_min(((unsigned *) handle->tmp_value)[i],
+                                       ((unsigned *) handle->current_value)[i]);
                         break;
                     case MCA_BASE_VAR_TYPE_UNSIGNED_LONG:
-                        ((unsigned long *) handle->current_value)[i] = opal_min(((unsigned long *) handle->tmp_value)[i],
-                                                                                ((unsigned long *) handle->current_value)[i]);
+                        ((unsigned long *) handle->current_value)[i]
+                            = opal_min(((unsigned long *) handle->tmp_value)[i],
+                                       ((unsigned long *) handle->current_value)[i]);
                         break;
                     case MCA_BASE_VAR_TYPE_UNSIGNED_LONG_LONG:
-                        ((unsigned long long *) handle->current_value)[i] = opal_min(((unsigned long long *) handle->tmp_value)[i],
-                                                                                     ((unsigned long long *) handle->current_value)[i]);
+                        ((unsigned long long *) handle->current_value)[i]
+                            = opal_min(((unsigned long long *) handle->tmp_value)[i],
+                                       ((unsigned long long *) handle->current_value)[i]);
                         break;
                     case MCA_BASE_VAR_TYPE_DOUBLE:
-                        ((double *) handle->current_value)[i] = opal_min(((double *) handle->tmp_value)[i],
-                                                                         ((double *) handle->current_value)[i]);
+                        ((double *) handle->current_value)[i]
+                            = opal_min(((double *) handle->tmp_value)[i],
+                                       ((double *) handle->current_value)[i]);
                         break;
                     default:
                         /* shouldn't happen */
@@ -630,20 +638,24 @@ int mca_base_pvar_handle_update (mca_base_pvar_handle_t *handle)
                 } else {
                     switch (handle->pvar->type) {
                     case MCA_BASE_VAR_TYPE_UNSIGNED_INT:
-                        ((unsigned *) handle->current_value)[i] = opal_max(((unsigned *) handle->tmp_value)[i],
-                                                                           ((unsigned *) handle->current_value)[i]);
+                        ((unsigned *) handle->current_value)[i]
+                            = opal_max(((unsigned *) handle->tmp_value)[i],
+                                       ((unsigned *) handle->current_value)[i]);
                         break;
                     case MCA_BASE_VAR_TYPE_UNSIGNED_LONG:
-                        ((unsigned long *) handle->current_value)[i] = opal_max(((unsigned long *) handle->tmp_value)[i],
-                                                                                ((unsigned long *) handle->current_value)[i]);
+                        ((unsigned long *) handle->current_value)[i]
+                            = opal_max(((unsigned long *) handle->tmp_value)[i],
+                                       ((unsigned long *) handle->current_value)[i]);
                         break;
                     case MCA_BASE_VAR_TYPE_UNSIGNED_LONG_LONG:
-                        ((unsigned long long *) handle->current_value)[i] = opal_max(((unsigned long long *) handle->tmp_value)[i],
-                                                                                     ((unsigned long long *) handle->current_value)[i]);
+                        ((unsigned long long *) handle->current_value)[i]
+                            = opal_max(((unsigned long long *) handle->tmp_value)[i],
+                                       ((unsigned long long *) handle->current_value)[i]);
                         break;
                     case MCA_BASE_VAR_TYPE_DOUBLE:
-                        ((double *) handle->current_value)[i] = opal_max(((double *) handle->tmp_value)[i],
-                                                                         ((double *) handle->current_value)[i]);
+                        ((double *) handle->current_value)[i]
+                            = opal_max(((double *) handle->tmp_value)[i],
+                                       ((double *) handle->current_value)[i]);
                         break;
                     default:
                         /* shouldn't happen */
@@ -652,100 +664,101 @@ int mca_base_pvar_handle_update (mca_base_pvar_handle_t *handle)
                 }
             }
         }
-    } else if (!mca_base_pvar_is_continuous (handle->pvar)) {
+    } else if (!mca_base_pvar_is_continuous(handle->pvar)) {
         /* cache the current value */
-        ret = handle->pvar->get_value (handle->pvar, handle->current_value, handle->obj_handle);
+        ret = handle->pvar->get_value(handle->pvar, handle->current_value, handle->obj_handle);
         if (OPAL_SUCCESS != ret) {
             return ret;
         }
     }
 
-    /* XXX -- TODO -- For watermarks this function will have to be invoked for each handle whenever the underlying value is updated. */
+    /* XXX -- TODO -- For watermarks this function will have to be invoked for each handle whenever
+     * the underlying value is updated. */
 
     return OPAL_SUCCESS;
 }
 
-int mca_base_pvar_handle_read_value (mca_base_pvar_handle_t *handle, void *value)
+int mca_base_pvar_handle_read_value(mca_base_pvar_handle_t *handle, void *value)
 {
     int ret;
 
-    if (mca_base_pvar_is_invalid (handle->pvar)) {
+    if (mca_base_pvar_is_invalid(handle->pvar)) {
         return OPAL_ERR_NOT_BOUND;
     }
 
     /* ensure this handle's value is up to date. */
-    ret = mca_base_pvar_handle_update (handle);
+    ret = mca_base_pvar_handle_update(handle);
     if (OPAL_SUCCESS != ret) {
         return ret;
     }
 
-    if (mca_base_pvar_is_sum (handle->pvar) || mca_base_pvar_is_watermark (handle->pvar) ||
-        !mca_base_pvar_handle_is_running (handle)) {
+    if (mca_base_pvar_is_sum(handle->pvar) || mca_base_pvar_is_watermark(handle->pvar)
+        || !mca_base_pvar_handle_is_running(handle)) {
         /* read the value cached in the handle. */
-        memmove (value, handle->current_value, handle->count * ompi_var_type_sizes[handle->pvar->type]);
+        memmove(value, handle->current_value,
+                handle->count * ompi_var_type_sizes[handle->pvar->type]);
     } else {
         /* read the value directly from the variable. */
-        ret = handle->pvar->get_value (handle->pvar, value, handle->obj_handle);
+        ret = handle->pvar->get_value(handle->pvar, value, handle->obj_handle);
     }
 
     return ret;
 }
 
-int mca_base_pvar_handle_write_value (mca_base_pvar_handle_t *handle, const void *value)
+int mca_base_pvar_handle_write_value(mca_base_pvar_handle_t *handle, const void *value)
 {
     int ret;
 
-    if (mca_base_pvar_is_invalid (handle->pvar)) {
+    if (mca_base_pvar_is_invalid(handle->pvar)) {
         return OPAL_ERR_NOT_BOUND;
     }
 
-    if (mca_base_pvar_is_readonly (handle->pvar)) {
+    if (mca_base_pvar_is_readonly(handle->pvar)) {
         return OPAL_ERR_PERM;
     }
 
     /* write the value directly from the variable. */
-    ret = handle->pvar->set_value (handle->pvar, value, handle->obj_handle);
+    ret = handle->pvar->set_value(handle->pvar, value, handle->obj_handle);
 
-    ret = mca_base_pvar_handle_update (handle);
+    ret = mca_base_pvar_handle_update(handle);
     if (OPAL_SUCCESS != ret) {
         return ret;
     }
 
-    memmove (handle->current_value, value, handle->count * ompi_var_type_sizes[handle->pvar->type]);
+    memmove(handle->current_value, value, handle->count * ompi_var_type_sizes[handle->pvar->type]);
     /* read the value directly from the variable. */
-    ret = handle->pvar->set_value (handle->pvar, value, handle->obj_handle);
+    ret = handle->pvar->set_value(handle->pvar, value, handle->obj_handle);
 
     return OPAL_SUCCESS;
 }
 
-int mca_base_pvar_handle_start (mca_base_pvar_handle_t *handle)
+int mca_base_pvar_handle_start(mca_base_pvar_handle_t *handle)
 {
     int ret;
 
     /* Can't start a continuous or an already started variable */
-    if ((handle->pvar->flags & MCA_BASE_PVAR_FLAG_CONTINUOUS) ||
-        handle->started) {
+    if ((handle->pvar->flags & MCA_BASE_PVAR_FLAG_CONTINUOUS) || handle->started) {
         return OPAL_ERR_NOT_SUPPORTED;
     }
 
     /* Notify the variable that a handle has started */
-    ret = mca_base_pvar_notify (handle, MCA_BASE_PVAR_HANDLE_START, NULL);
+    ret = mca_base_pvar_notify(handle, MCA_BASE_PVAR_HANDLE_START, NULL);
     if (OPAL_SUCCESS != ret) {
         return ret;
     }
 
     handle->started = true;
 
-    if (mca_base_pvar_is_sum (handle->pvar)) {
+    if (mca_base_pvar_is_sum(handle->pvar)) {
         /* Keep track of the counter value from when this counter started. */
-        ret = handle->pvar->get_value (handle->pvar, handle->last_value, handle->obj_handle);
+        ret = handle->pvar->get_value(handle->pvar, handle->last_value, handle->obj_handle);
         if (OPAL_SUCCESS != ret) {
             return ret;
         }
-    } else if (mca_base_pvar_is_watermark (handle->pvar)) {
-        /* Find the current watermark. is this correct in the case where a watermark is started, stopped,
-           then restarted? Probably will need to add a check. */
-        ret = handle->pvar->get_value (handle->pvar, handle->current_value, handle->obj_handle);
+    } else if (mca_base_pvar_is_watermark(handle->pvar)) {
+        /* Find the current watermark. is this correct in the case where a watermark is started,
+           stopped, then restarted? Probably will need to add a check. */
+        ret = handle->pvar->get_value(handle->pvar, handle->current_value, handle->obj_handle);
         if (OPAL_SUCCESS != ret) {
             return ret;
         }
@@ -754,26 +767,26 @@ int mca_base_pvar_handle_start (mca_base_pvar_handle_t *handle)
     return OPAL_SUCCESS;
 }
 
-int mca_base_pvar_handle_stop (mca_base_pvar_handle_t *handle)
+int mca_base_pvar_handle_stop(mca_base_pvar_handle_t *handle)
 {
     int ret;
 
-    if (mca_base_pvar_is_invalid (handle->pvar)) {
+    if (mca_base_pvar_is_invalid(handle->pvar)) {
         return OPAL_ERR_NOT_BOUND;
     }
 
     /* Can't stop a continuous or an already stopped variable */
-    if (!mca_base_pvar_handle_is_running (handle) || mca_base_pvar_is_continuous (handle->pvar)) {
+    if (!mca_base_pvar_handle_is_running(handle) || mca_base_pvar_is_continuous(handle->pvar)) {
         return OPAL_ERR_NOT_SUPPORTED;
     }
 
-    ret = mca_base_pvar_handle_update (handle);
+    ret = mca_base_pvar_handle_update(handle);
     if (OPAL_SUCCESS != ret) {
         return ret;
     }
 
     /* Notify the variable that a handle has stopped */
-    (void) mca_base_pvar_notify (handle, MCA_BASE_PVAR_HANDLE_STOP, NULL);
+    (void) mca_base_pvar_notify(handle, MCA_BASE_PVAR_HANDLE_STOP, NULL);
 
     /* Handle is stopped */
     handle->started = false;
@@ -781,27 +794,28 @@ int mca_base_pvar_handle_stop (mca_base_pvar_handle_t *handle)
     return OPAL_SUCCESS;
 }
 
-int mca_base_pvar_handle_reset (mca_base_pvar_handle_t *handle)
+int mca_base_pvar_handle_reset(mca_base_pvar_handle_t *handle)
 {
     int ret = OPAL_SUCCESS;
 
-    if (mca_base_pvar_is_invalid (handle->pvar)) {
+    if (mca_base_pvar_is_invalid(handle->pvar)) {
         return OPAL_ERR_NOT_BOUND;
     }
 
-    /* reset this handle to a state analagous to when it was created */
-    if (mca_base_pvar_is_sum (handle->pvar)) {
+    /* reset this handle to a state analogous to when it was created */
+    if (mca_base_pvar_is_sum(handle->pvar)) {
         /* reset the running sum to 0 */
-        memset (handle->current_value, 0, handle->count * ompi_var_type_sizes[handle->pvar->type]);
+        memset(handle->current_value, 0, handle->count * ompi_var_type_sizes[handle->pvar->type]);
 
-        if (mca_base_pvar_handle_is_running (handle)) {
-            ret = handle->pvar->get_value (handle->pvar, handle->last_value, handle->obj_handle);
+        if (mca_base_pvar_handle_is_running(handle)) {
+            ret = handle->pvar->get_value(handle->pvar, handle->last_value, handle->obj_handle);
         }
-    } else if (mca_base_pvar_handle_is_running (handle) && mca_base_pvar_is_watermark (handle->pvar)) {
-            /* watermarks should get set to the current value if runnning. */
+    } else if (mca_base_pvar_handle_is_running(handle)
+               && mca_base_pvar_is_watermark(handle->pvar)) {
+        /* watermarks should get set to the current value if running. */
 
-        ret = handle->pvar->get_value (handle->pvar, handle->current_value, handle->obj_handle);
-    } else if (mca_base_pvar_is_readonly (handle->pvar)) {
+        ret = handle->pvar->get_value(handle->pvar, handle->current_value, handle->obj_handle);
+    } else if (mca_base_pvar_is_readonly(handle->pvar)) {
         return OPAL_ERR_PERM;
     }
     /* NTH: TODO -- Actually write the value for variable of other types */
@@ -818,12 +832,12 @@ int mca_base_pvar_dump(int index, char ***out, mca_base_var_dump_type_t output_t
     int ret, enum_count = 0;
     char *tmp;
 
-    ret = mca_base_pvar_get (index, &pvar);
+    ret = mca_base_pvar_get(index, &pvar);
     if (OPAL_SUCCESS != ret) {
         return ret;
     }
 
-    ret = mca_base_var_group_get_internal (pvar->group_index, &group, true);
+    ret = mca_base_var_group_get_internal(pvar->group_index, &group, true);
     if (OPAL_SUCCESS != ret) {
         return ret;
     }
@@ -839,62 +853,66 @@ int mca_base_pvar_dump(int index, char ***out, mca_base_var_dump_type_t output_t
     if (MCA_BASE_VAR_DUMP_PARSABLE == output_type) {
         line_count = 5 + !!(pvar->description) + enum_count;
 
-        *out = (char **) calloc (line_count + 1, sizeof (char *));
+        *out = (char **) calloc(line_count + 1, sizeof(char *));
         if (NULL == *out) {
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
 
         /* build the message*/
-        (void)opal_asprintf(&tmp, "mca:%s:%s:pvar:%s:", framework, component, full_name);
+        (void) opal_asprintf(&tmp, "mca:%s:%s:pvar:%s:", framework, component, full_name);
 
-        (void)opal_asprintf(out[0] + line++, "%sclass:%s", tmp, pvar_class_names[pvar->var_class]);
-        (void)opal_asprintf(out[0] + line++, "%sread-only:%s", tmp, mca_base_pvar_is_readonly(pvar) ? "true" : "false");
-        (void)opal_asprintf(out[0] + line++, "%scontinuous:%s", tmp, mca_base_pvar_is_continuous(pvar) ? "true" : "false");
-        (void)opal_asprintf(out[0] + line++, "%satomic:%s", tmp, mca_base_pvar_is_atomic(pvar) ? "true" : "false");
+        (void) opal_asprintf(out[0] + line++, "%sclass:%s", tmp, pvar_class_names[pvar->var_class]);
+        (void) opal_asprintf(out[0] + line++, "%sread-only:%s", tmp,
+                             mca_base_pvar_is_readonly(pvar) ? "true" : "false");
+        (void) opal_asprintf(out[0] + line++, "%scontinuous:%s", tmp,
+                             mca_base_pvar_is_continuous(pvar) ? "true" : "false");
+        (void) opal_asprintf(out[0] + line++, "%satomic:%s", tmp,
+                             mca_base_pvar_is_atomic(pvar) ? "true" : "false");
 
         /* if it has a help message, output the help message */
         if (pvar->description) {
-            (void)opal_asprintf(out[0] + line++, "%shelp:%s", tmp, pvar->description);
+            (void) opal_asprintf(out[0] + line++, "%shelp:%s", tmp, pvar->description);
         }
 
         if (NULL != pvar->enumerator) {
-            for (i = 0 ; i < enum_count ; ++i) {
+            for (i = 0; i < enum_count; ++i) {
                 const char *enum_string = NULL;
                 int enum_value;
 
-                ret = pvar->enumerator->get_value(pvar->enumerator, i, &enum_value,
-                                                     &enum_string);
+                ret = pvar->enumerator->get_value(pvar->enumerator, i, &enum_value, &enum_string);
                 if (OPAL_SUCCESS != ret) {
                     continue;
                 }
 
-                (void)opal_asprintf(out[0] + line++, "%senumerator:value:%d:%s", tmp, enum_value, enum_string);
+                (void) opal_asprintf(out[0] + line++, "%senumerator:value:%d:%s", tmp, enum_value,
+                                     enum_string);
             }
         }
 
-        (void)opal_asprintf(out[0] + line++, "%stype:%s", tmp, ompi_var_type_names[pvar->type]);
-        free(tmp);  // release tmp storage
+        (void) opal_asprintf(out[0] + line++, "%stype:%s", tmp, ompi_var_type_names[pvar->type]);
+        free(tmp); // release tmp storage
     } else {
         /* there will be at most three lines in the pretty print case */
-        *out = (char **) calloc (3, sizeof (char *));
+        *out = (char **) calloc(3, sizeof(char *));
         if (NULL == *out) {
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
 
-        (void)opal_asprintf (out[0] + line++, "performance \"%s\" (type: %s, class: %s)", full_name,
-                        ompi_var_type_names[pvar->type], pvar_class_names[pvar->var_class]);
+        (void) opal_asprintf(out[0] + line++, "performance \"%s\" (type: %s, class: %s)", full_name,
+                             ompi_var_type_names[pvar->type], pvar_class_names[pvar->var_class]);
 
         if (pvar->description) {
-            (void)opal_asprintf(out[0] + line++, "%s", pvar->description);
+            (void) opal_asprintf(out[0] + line++, "%s", pvar->description);
         }
 
         if (NULL != pvar->enumerator) {
             char *values;
 
-            ret = pvar->enumerator->dump(pvar->enumerator, &values);
+            ret = pvar->enumerator->dump(pvar->enumerator, &values,
+                MCA_BASE_VAR_DUMP_TYPE_TO_ENUM_DUMP_TYPE(output_type));
             if (OPAL_SUCCESS == ret) {
-                (void)opal_asprintf (out[0] + line++, "Values: %s", values);
-                free (values);
+                (void) opal_asprintf(out[0] + line++, "Values: %s", values);
+                free(values);
             }
         }
     }
@@ -903,20 +921,20 @@ int mca_base_pvar_dump(int index, char ***out, mca_base_var_dump_type_t output_t
 }
 
 /* mca_base_pvar_t class */
-static void mca_base_pvar_contructor (mca_base_pvar_t *pvar)
+static void mca_base_pvar_contructor(mca_base_pvar_t *pvar)
 {
-    memset ((char *) pvar + sizeof (pvar->super), 0, sizeof (*pvar) - sizeof (pvar->super));
+    memset((char *) pvar + sizeof(pvar->super), 0, sizeof(*pvar) - sizeof(pvar->super));
     OBJ_CONSTRUCT(&pvar->bound_handles, opal_list_t);
 }
 
-static void mca_base_pvar_destructor (mca_base_pvar_t *pvar)
+static void mca_base_pvar_destructor(mca_base_pvar_t *pvar)
 {
     if (pvar->name) {
-        free (pvar->name);
+        free(pvar->name);
     }
 
     if (pvar->description) {
-        free (pvar->description);
+        free(pvar->description);
     }
 
     if (NULL != pvar->enumerator) {
@@ -926,22 +944,23 @@ static void mca_base_pvar_destructor (mca_base_pvar_t *pvar)
     OBJ_DESTRUCT(&pvar->bound_handles);
 }
 
-OBJ_CLASS_INSTANCE(mca_base_pvar_t, opal_object_t, mca_base_pvar_contructor, mca_base_pvar_destructor);
+OBJ_CLASS_INSTANCE(mca_base_pvar_t, opal_object_t, mca_base_pvar_contructor,
+                   mca_base_pvar_destructor);
 
 /* mca_base_pvar_session_t class */
-static void opal_mpi_pvar_session_constructor (mca_base_pvar_session_t *session)
+static void opal_mpi_pvar_session_constructor(mca_base_pvar_session_t *session)
 {
     OBJ_CONSTRUCT(&session->handles, opal_list_t);
 }
 
-static void opal_mpi_pvar_session_destructor (mca_base_pvar_session_t *session)
+static void opal_mpi_pvar_session_destructor(mca_base_pvar_session_t *session)
 {
     mca_base_pvar_handle_t *handle, *next;
 
     /* it is likely a user error if there are any allocated handles when the session
      * is freed. clean it up anyway. The handle destructor will remove the handle from
      * the session's handle list. */
-    OPAL_LIST_FOREACH_SAFE(handle, next, &session->handles, mca_base_pvar_handle_t) {
+    OPAL_LIST_FOREACH_SAFE (handle, next, &session->handles, mca_base_pvar_handle_t) {
         OBJ_DESTRUCT(handle);
     }
 
@@ -952,41 +971,41 @@ OBJ_CLASS_INSTANCE(mca_base_pvar_session_t, opal_object_t, opal_mpi_pvar_session
                    opal_mpi_pvar_session_destructor);
 
 /* mca_base_pvar_handle_t class */
-static void mca_base_pvar_handle_constructor (mca_base_pvar_handle_t *handle)
+static void mca_base_pvar_handle_constructor(mca_base_pvar_handle_t *handle)
 {
-    memset ((char *) handle + sizeof (handle->super), 0, sizeof (*handle) - sizeof (handle->super));
+    memset((char *) handle + sizeof(handle->super), 0, sizeof(*handle) - sizeof(handle->super));
 
     OBJ_CONSTRUCT(&handle->list2, opal_list_item_t);
 }
 
-static void mca_base_pvar_handle_destructor (mca_base_pvar_handle_t *handle)
+static void mca_base_pvar_handle_destructor(mca_base_pvar_handle_t *handle)
 {
     if (handle->pvar) {
-        (void) mca_base_pvar_notify (handle, MCA_BASE_PVAR_HANDLE_UNBIND, NULL);
+        (void) mca_base_pvar_notify(handle, MCA_BASE_PVAR_HANDLE_UNBIND, NULL);
     }
 
     if (NULL != handle->last_value) {
-        free (handle->last_value);
+        free(handle->last_value);
     }
 
     if (NULL != handle->current_value) {
-        free (handle->current_value);
+        free(handle->current_value);
     }
 
     if (NULL != handle->tmp_value) {
-        free (handle->tmp_value);
+        free(handle->tmp_value);
     }
 
     /* remove this handle from the pvar list */
     if (handle->pvar) {
-        opal_list_remove_item (&handle->pvar->bound_handles, &handle->list2);
+        opal_list_remove_item(&handle->pvar->bound_handles, &handle->list2);
     }
 
     OBJ_DESTRUCT(&handle->list2);
 
     /* remove this handle from the session */
     if (handle->session) {
-        opal_list_remove_item (&handle->session->handles, &handle->super);
+        opal_list_remove_item(&handle->session->handles, &handle->super);
     }
 }
 
diff --git a/opal/mca/base/mca_base_pvar.h b/opal/mca/base/mca_base_pvar.h
index 03ff53b98f8..f0bdb1b747c 100644
--- a/opal/mca/base/mca_base_pvar.h
+++ b/opal/mca/base/mca_base_pvar.h
@@ -12,9 +12,9 @@
  */
 
 #if !defined(OPAL_MPIT_PVAR_H)
-#define OPAL_MPIT_PVAR_H
+#    define OPAL_MPIT_PVAR_H
 
-#include "opal/mca/base/mca_base_var.h"
+#    include "opal/mca/base/mca_base_var.h"
 
 /*
  * These flags are used when registering a new pvar.
@@ -27,20 +27,20 @@ typedef enum {
         manually when you register a variable with
         mca_base_pvar_register().  Analogous to the
         MCA_BASE_VAR_FLAG_DWG flag. */
-    MCA_BASE_PVAR_FLAG_IWG        = 0x040,
+    MCA_BASE_PVAR_FLAG_IWG = 0x040,
     /** This variable can not be written. Will be ignored for counter,
         timer, and aggregate variables. These variable handles will be
         updated relative to the value reported by the get_value()
         function provided at registration time. */
-    MCA_BASE_PVAR_FLAG_READONLY   = 0x080,
+    MCA_BASE_PVAR_FLAG_READONLY = 0x080,
     /** This variable runs continuously after being bound to a handle. */
     MCA_BASE_PVAR_FLAG_CONTINUOUS = 0x100,
     /** This variable can be updated atomically. This flag is ignored
         by mca_base_pvar_register() at this time. */
-    MCA_BASE_PVAR_FLAG_ATOMIC     = 0x200,
+    MCA_BASE_PVAR_FLAG_ATOMIC = 0x200,
     /** This variable has been marked as invalid. This flag is ignored
         by mca_base_pvar_register(). */
-    MCA_BASE_PVAR_FLAG_INVALID    = 0x400,
+    MCA_BASE_PVAR_FLAG_INVALID = 0x400,
 } mca_base_pvar_flag_t;
 
 /*
@@ -54,7 +54,7 @@ typedef enum {
         return value should be the size of the bound communicator. */
     MCA_BASE_PVAR_HANDLE_BIND,
     /** A handle associated with this variable has been started. It is
-        recommended that any computation that might affect perfomance
+        recommended that any computation that might affect performance
         only be performed when a bound handle has been started. */
     MCA_BASE_PVAR_HANDLE_START,
     /** A handle associated with this variable has been stopped */
@@ -82,7 +82,7 @@ enum {
     /** Variable describes the low-watermark of the utilization
        of a resource */
     MCA_BASE_PVAR_CLASS_LOWWATERMARK,
-    /** Variable counts the number of occurences of a specific event */
+    /** Variable counts the number of occurrences of a specific event */
     MCA_BASE_PVAR_CLASS_COUNTER,
     /** Variable represents a sum of arguments processed during a
        specific event */
@@ -94,7 +94,7 @@ enum {
     MCA_BASE_PVAR_CLASS_GENERIC
 };
 
-#define MCA_BASE_PVAR_CLASS_ANY -1
+#    define MCA_BASE_PVAR_CLASS_ANY -1
 
 /*
  * Reserved bindings; passed when registering a new pvar. OPAL will
@@ -128,7 +128,7 @@ struct mca_base_pvar_t;
  * pointer will be large enough to hold the datatype and count specified when this
  * variable was created and bound.
  */
-typedef int (*mca_base_get_value_fn_t) (const struct mca_base_pvar_t *pvar, void *value, void *obj);
+typedef int (*mca_base_get_value_fn_t)(const struct mca_base_pvar_t *pvar, void *value, void *obj);
 
 /**
  * Function to set the current value of a variable.
@@ -142,12 +142,12 @@ typedef int (*mca_base_get_value_fn_t) (const struct mca_base_pvar_t *pvar, void
  * variable was created and bound. Read-only variables are not expected to provide
  * this function.
  */
-typedef int (*mca_base_set_value_fn_t) (struct mca_base_pvar_t *pvar, const void *value, void *obj);
+typedef int (*mca_base_set_value_fn_t)(struct mca_base_pvar_t *pvar, const void *value, void *obj);
 
 /**
  * Function to notify of a pvar handle event.
  *
- * @param[in]  pvar  Performance variable the handle is assocaited with
+ * @param[in]  pvar  Performance variable the handle is associated with
  * @param[in]  event Event that has occurred. See mca_base_pvar_event_t.
  * @param[in]  obj   Bound object
  * @param[out] count Value count for this object (on MCA_BASE_PVAR_HANDLE_BIND)
@@ -158,7 +158,8 @@ typedef int (*mca_base_set_value_fn_t) (struct mca_base_pvar_t *pvar, const void
  * On MCA_BASE_PVAR_HANDLE_START: enable the performance variable.
  * On MCA_BASE_PVAR_HANDLE_STOP: XXX -- TODO -- finish me
  */
-typedef int (*mca_base_notify_fn_t) (struct mca_base_pvar_t *pvar, mca_base_pvar_event_t event, void *obj, int *count);
+typedef int (*mca_base_notify_fn_t)(struct mca_base_pvar_t *pvar, mca_base_pvar_event_t event,
+                                    void *obj, int *count);
 
 /**
  * Structure representing an OPAL performance variable.
@@ -311,14 +312,15 @@ OBJ_CLASS_DECLARATION(mca_base_pvar_handle_t);
  * Note: if used incorrectly this function may fail an assert(); see
  * MPI 3.0 14.3 to see acceptable values for datatype given the class.
  */
-OPAL_DECLSPEC int mca_base_pvar_register (const char *project, const char *framework, const char *component, const char *name,
-                            const char *description, mca_base_var_info_lvl_t verbosity,
-                            int var_class, mca_base_var_type_t type, mca_base_var_enum_t *enumerator,
-                            int bind, mca_base_pvar_flag_t flags, mca_base_get_value_fn_t get_value,
-                            mca_base_set_value_fn_t set_value, mca_base_notify_fn_t notify, void *ctx);
+OPAL_DECLSPEC int
+mca_base_pvar_register(const char *project, const char *framework, const char *component,
+                       const char *name, const char *description, mca_base_var_info_lvl_t verbosity,
+                       int var_class, mca_base_var_type_t type, mca_base_var_enum_t *enumerator,
+                       int bind, mca_base_pvar_flag_t flags, mca_base_get_value_fn_t get_value,
+                       mca_base_set_value_fn_t set_value, mca_base_notify_fn_t notify, void *ctx);
 
 /**
- * Convinience function for registering a performance variable
+ * Convenience function for registering a performance variable
  * associated with a component.
  *
  * While quite similar to mca_base_pvar_register(), there is one key
@@ -326,12 +328,12 @@ OPAL_DECLSPEC int mca_base_pvar_register (const char *project, const char *frame
  * be unregistered / made unavailable when that component is closed by
  * its framework.
  */
-OPAL_DECLSPEC int mca_base_component_pvar_register (const mca_base_component_t *component, const char *name,
-                            const char *description, mca_base_var_info_lvl_t verbosity, int var_class,
-                            mca_base_var_type_t type, mca_base_var_enum_t *enumerator, int bind,
-                            mca_base_pvar_flag_t flags, mca_base_get_value_fn_t get_value,
-                            mca_base_set_value_fn_t set_value, mca_base_notify_fn_t notify, void *ctx);
-
+OPAL_DECLSPEC int mca_base_component_pvar_register(
+    const mca_base_component_t *component, const char *name, const char *description,
+    mca_base_var_info_lvl_t verbosity, int var_class, mca_base_var_type_t type,
+    mca_base_var_enum_t *enumerator, int bind, mca_base_pvar_flag_t flags,
+    mca_base_get_value_fn_t get_value, mca_base_set_value_fn_t set_value,
+    mca_base_notify_fn_t notify, void *ctx);
 
 /**
  * Find the index for an MCA performance variable based on its names.
@@ -351,7 +353,8 @@ OPAL_DECLSPEC int mca_base_component_pvar_register (const mca_base_component_t *
  * mca_base_pvar_get(), mca_base_pvar_handle_alloc(), and
  * mca_base_pvar_dump().
  */
-OPAL_DECLSPEC int mca_base_pvar_find (const char *project, const char *framework, const char *component, const char *name);
+OPAL_DECLSPEC int mca_base_pvar_find(const char *project, const char *framework,
+                                     const char *component, const char *name);
 
 /**
  * Find the index for a performance variable based on its full name
@@ -361,7 +364,7 @@ OPAL_DECLSPEC int mca_base_pvar_find (const char *project, const char *framework
  *
  * See mca_base_pvar_find().
  */
-OPAL_DECLSPEC int mca_base_pvar_find_by_name (const char *full_name, int var_class, int *index);
+OPAL_DECLSPEC int mca_base_pvar_find_by_name(const char *full_name, int var_class, int *index);
 
 /****************************************************************************
  * The following functions are the back-end to the MPI_T API functions
@@ -376,7 +379,7 @@ OPAL_DECLSPEC int mca_base_pvar_find_by_name (const char *full_name, int var_cla
  * This function can be called before mca_base_pvar_init() and after
  * mca_base_pvar_finalize().
  */
-OPAL_DECLSPEC int mca_base_pvar_get_count (int *count);
+OPAL_DECLSPEC int mca_base_pvar_get_count(int *count);
 
 /**
  * Update the handles associated with the specified performance variable and MPI object
@@ -389,7 +392,7 @@ OPAL_DECLSPEC int mca_base_pvar_get_count (int *count);
  * is recommended this function not be called from within any critical code path. Calling
  * this function should only be necessary to update watermarks.
  */
-OPAL_DECLSPEC int mca_base_pvar_update_all_handles (int index, const void *obj);
+OPAL_DECLSPEC int mca_base_pvar_update_all_handles(int index, const void *obj);
 
 /**
  * Get the variable at an index
@@ -400,7 +403,7 @@ OPAL_DECLSPEC int mca_base_pvar_update_all_handles (int index, const void *obj);
  * @returns OPAL_SUCCESS on success
  * @returns OPAL_ERR_VALUE_OUT_OF_BOUNDS on if index is out of range
  */
-OPAL_DECLSPEC int mca_base_pvar_get (int index, const mca_base_pvar_t **pvar);
+OPAL_DECLSPEC int mca_base_pvar_get(int index, const mca_base_pvar_t **pvar);
 
 /**
  * Dump strings describing the performance variable at an index
@@ -423,40 +426,40 @@ OPAL_DECLSPEC int mca_base_pvar_dump(int index, char ***out, mca_base_var_dump_t
  * A performance variable that has been marked as invalid will not be available. To
  * restore a performance variable it has to be re-registered using mca_base_pvar_register().
  */
-int mca_base_pvar_mark_invalid (int index);
+int mca_base_pvar_mark_invalid(int index);
 
 /**
- * Convienience functions for performance variables
+ * Convenience functions for performance variables
  */
-static inline bool mca_base_pvar_is_sum (const mca_base_pvar_t *pvar)
+static inline bool mca_base_pvar_is_sum(const mca_base_pvar_t *pvar)
 {
-    return (MCA_BASE_PVAR_CLASS_COUNTER == pvar->var_class ||
-            MCA_BASE_PVAR_CLASS_TIMER == pvar->var_class ||
-            MCA_BASE_PVAR_CLASS_AGGREGATE == pvar->var_class);
+    return (MCA_BASE_PVAR_CLASS_COUNTER == pvar->var_class
+            || MCA_BASE_PVAR_CLASS_TIMER == pvar->var_class
+            || MCA_BASE_PVAR_CLASS_AGGREGATE == pvar->var_class);
 }
 
-static inline bool mca_base_pvar_is_watermark (const mca_base_pvar_t *pvar)
+static inline bool mca_base_pvar_is_watermark(const mca_base_pvar_t *pvar)
 {
-    return (MCA_BASE_PVAR_CLASS_HIGHWATERMARK == pvar->var_class ||
-            MCA_BASE_PVAR_CLASS_LOWWATERMARK == pvar->var_class);
+    return (MCA_BASE_PVAR_CLASS_HIGHWATERMARK == pvar->var_class
+            || MCA_BASE_PVAR_CLASS_LOWWATERMARK == pvar->var_class);
 }
 
-static inline bool mca_base_pvar_is_readonly (const mca_base_pvar_t *pvar)
+static inline bool mca_base_pvar_is_readonly(const mca_base_pvar_t *pvar)
 {
     return !!(pvar->flags & MCA_BASE_PVAR_FLAG_READONLY);
 }
 
-static inline bool mca_base_pvar_is_continuous (const mca_base_pvar_t *pvar)
+static inline bool mca_base_pvar_is_continuous(const mca_base_pvar_t *pvar)
 {
     return !!(pvar->flags & MCA_BASE_PVAR_FLAG_CONTINUOUS);
 }
 
-static inline bool mca_base_pvar_is_atomic (const mca_base_pvar_t *pvar)
+static inline bool mca_base_pvar_is_atomic(const mca_base_pvar_t *pvar)
 {
     return !!(pvar->flags & MCA_BASE_PVAR_FLAG_ATOMIC);
 }
 
-static inline bool mca_base_pvar_is_invalid (const mca_base_pvar_t *pvar)
+static inline bool mca_base_pvar_is_invalid(const mca_base_pvar_t *pvar)
 {
     return !!(pvar->flags & MCA_BASE_PVAR_FLAG_INVALID);
 }
@@ -479,8 +482,9 @@ static inline bool mca_base_pvar_is_invalid (const mca_base_pvar_t *pvar)
  * hold \count values of the type of the performance variable specified by
  * \index.
  */
-OPAL_DECLSPEC int mca_base_pvar_handle_alloc (mca_base_pvar_session_t *session, int index, void *obj_handle,
-                                mca_base_pvar_handle_t **handle, int *count);
+OPAL_DECLSPEC int mca_base_pvar_handle_alloc(mca_base_pvar_session_t *session, int index,
+                                             void *obj_handle, mca_base_pvar_handle_t **handle,
+                                             int *count);
 
 /**
  * Free an allocated performance variable handle
@@ -489,7 +493,7 @@ OPAL_DECLSPEC int mca_base_pvar_handle_alloc (mca_base_pvar_session_t *session,
  *
  * After calling this function the performance variable will no longer be valid.
  */
-OPAL_DECLSPEC int mca_base_pvar_handle_free (mca_base_pvar_handle_t *handle);
+OPAL_DECLSPEC int mca_base_pvar_handle_free(mca_base_pvar_handle_t *handle);
 
 /**
  * Update a performance variable handle.
@@ -502,7 +506,7 @@ OPAL_DECLSPEC int mca_base_pvar_handle_free (mca_base_pvar_handle_t *handle);
  * the new value will be the greater/lesser of the current handle value and the
  * current variable value. This call does not update other types of handles.
  */
-OPAL_DECLSPEC int mca_base_pvar_handle_update (mca_base_pvar_handle_t *handle);
+OPAL_DECLSPEC int mca_base_pvar_handle_update(mca_base_pvar_handle_t *handle);
 
 /**
  * Read the current value of a handle
@@ -514,7 +518,7 @@ OPAL_DECLSPEC int mca_base_pvar_handle_update (mca_base_pvar_handle_t *handle);
  * and return it in the buffer specified by value. The buffer must be large enough to
  * hold the correct number and type of this handle's value (see mca_base_pvar_handle_update()).
  */
-OPAL_DECLSPEC int mca_base_pvar_handle_read_value (mca_base_pvar_handle_t *handle, void *value);
+OPAL_DECLSPEC int mca_base_pvar_handle_read_value(mca_base_pvar_handle_t *handle, void *value);
 
 /**
  * Write a value to a read-write handle
@@ -524,16 +528,18 @@ OPAL_DECLSPEC int mca_base_pvar_handle_read_value (mca_base_pvar_handle_t *handl
  *
  * If the underlying variable is read-only this function will fail with OPAL_ERR_PERM.
  */
-OPAL_DECLSPEC int mca_base_pvar_handle_write_value (mca_base_pvar_handle_t *handle, const void *value);
+OPAL_DECLSPEC int mca_base_pvar_handle_write_value(mca_base_pvar_handle_t *handle,
+                                                   const void *value);
 
 /**
- * Convienience function for sending notification of a handle change
+ * Convenience function for sending notification of a handle change
  *
  * @param[in]  handle Handle event occurred on
  * @param[in]  event  Event that occurred
  * @param[out] count  Value count returned when binding a handle
  */
-OPAL_DECLSPEC int mca_base_pvar_notify (mca_base_pvar_handle_t *handle, mca_base_pvar_event_t event, int *count);
+OPAL_DECLSPEC int mca_base_pvar_notify(mca_base_pvar_handle_t *handle, mca_base_pvar_event_t event,
+                                       int *count);
 
 /**
  * Start a performance variable handle
@@ -543,7 +549,7 @@ OPAL_DECLSPEC int mca_base_pvar_notify (mca_base_pvar_handle_t *handle, mca_base
  * @returns OPAL_SUCCESS on success
  * @returns OPAL_ERR_NOT_SUPPORTED if the handle could not be started
  */
-OPAL_DECLSPEC int mca_base_pvar_handle_start (mca_base_pvar_handle_t *handle);
+OPAL_DECLSPEC int mca_base_pvar_handle_start(mca_base_pvar_handle_t *handle);
 
 /**
  * Stop a performance variable handle
@@ -553,7 +559,7 @@ OPAL_DECLSPEC int mca_base_pvar_handle_start (mca_base_pvar_handle_t *handle);
  * @return OPAL_SUCCESS on success
  * @returns OPAL_ERR_NOT_SUPPORTED if the handle could not be started
  */
-OPAL_DECLSPEC int mca_base_pvar_handle_stop (mca_base_pvar_handle_t *handle);
+OPAL_DECLSPEC int mca_base_pvar_handle_stop(mca_base_pvar_handle_t *handle);
 
 /**
  * Reset a performance variable handle
@@ -562,9 +568,9 @@ OPAL_DECLSPEC int mca_base_pvar_handle_stop (mca_base_pvar_handle_t *handle);
  *
  * Reset the handle to a value equivalent to when the handle was first allocated.
  */
-OPAL_DECLSPEC int mca_base_pvar_handle_reset (mca_base_pvar_handle_t *handle);
+OPAL_DECLSPEC int mca_base_pvar_handle_reset(mca_base_pvar_handle_t *handle);
 
-static inline bool mca_base_pvar_handle_is_running (mca_base_pvar_handle_t *handle)
+static inline bool mca_base_pvar_handle_is_running(mca_base_pvar_handle_t *handle)
 {
     return handle->started || !!(handle->pvar->flags & MCA_BASE_PVAR_FLAG_CONTINUOUS);
 }
diff --git a/opal/mca/base/mca_base_var.c b/opal/mca/base/mca_base_var.c
index 71203421d7d..b9860c6ed1e 100644
--- a/opal/mca/base/mca_base_var.c
+++ b/opal/mca/base/mca_base_var.c
@@ -3,14 +3,14 @@
  * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2012 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2008-2018 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2008-2021 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
@@ -22,6 +22,9 @@
  *                         reserved.
  * Copyright (c) 2020      Google, LLC. All rights reserved.
  * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2022      Computer Architecture and VLSI Systems (CARV)
+ *                         Laboratory, ICS Forth. All rights reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -32,30 +35,31 @@
 #include "opal_config.h"
 
 #include <stdio.h>
-#include <string.h>
 #include <stdlib.h>
+#include <string.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 #ifdef HAVE_SYS_PARAM_H
-#include <sys/param.h>
+#    include <sys/param.h>
 #endif
 #include <errno.h>
 
+#include "opal/constants.h"
 #include "opal/include/opal_stdint.h"
+#include "opal/mca/base/mca_base_alias.h"
+#include "opal/mca/base/mca_base_vari.h"
 #include "opal/mca/installdirs/installdirs.h"
+#include "opal/mca/mca.h"
+#include "opal/runtime/opal.h"
+#include "opal/runtime/opal_params_core.h"
+#include "opal/util/argv.h"
+#include "opal/util/opal_environ.h"
 #include "opal/util/os_path.h"
+#include "opal/util/output.h"
 #include "opal/util/path.h"
-#include "opal/util/show_help.h"
 #include "opal/util/printf.h"
-#include "opal/util/argv.h"
-#include "opal/mca/mca.h"
-#include "opal/mca/base/mca_base_vari.h"
-#include "opal/mca/base/mca_base_alias.h"
-#include "opal/constants.h"
-#include "opal/util/output.h"
-#include "opal/util/opal_environ.h"
-#include "opal/runtime/opal.h"
+#include "opal/util/show_help.h"
 
 /*
  * local variables
@@ -63,9 +67,9 @@
 static opal_pointer_array_t mca_base_vars;
 static const char *mca_prefix = OPAL_MCA_PREFIX;
 static char *home = NULL;
-static char *cwd  = NULL;
+static char *cwd = NULL;
 bool mca_base_var_initialized = false;
-static char * force_agg_path = NULL;
+static char *force_agg_path = NULL;
 static char *mca_base_var_files = NULL;
 static char *mca_base_envar_files = NULL;
 static char **mca_base_var_file_list = NULL;
@@ -85,92 +89,56 @@ static int mca_base_var_count = 0;
 
 static opal_hash_table_t mca_base_var_index_hash;
 
-#define OPAL_MCA_VAR_MBV_ENUMERATOR_FREE(mbv_enumerator) {       \
-    if(mbv_enumerator && !mbv_enumerator->enum_is_static) { \
-        OBJ_RELEASE(mbv_enumerator);                        \
-    }                                                       \
-}
+#define OPAL_MCA_VAR_MBV_ENUMERATOR_FREE(mbv_enumerator)         \
+    {                                                            \
+        if (mbv_enumerator && !mbv_enumerator->enum_is_static) { \
+            OBJ_RELEASE(mbv_enumerator);                         \
+        }                                                        \
+    }
 
 const char *ompi_var_type_names[] = {
-    "int",
-    "unsigned_int",
-    "unsigned_long",
-    "unsigned_long_long",
-    "size_t",
-    "string",
-    "version_string",
-    "bool",
-    "double",
-    "long",
-    "int32_t",
-    "uint32_t",
-    "int64_t",
-    "uint64_t",
+    "int",     "unsigned_int", "unsigned_long",  "unsigned_long_long",
+    "size_t",  "string",       "version_string", "bool",
+    "double",  "long",         "int32_t",        "uint32_t",
+    "int64_t", "uint64_t",
 };
 
 const size_t ompi_var_type_sizes[] = {
-    sizeof (int),
-    sizeof (unsigned),
-    sizeof (unsigned long),
-    sizeof (unsigned long long),
-    sizeof (size_t),
-    sizeof (char),
-    sizeof (char),
-    sizeof (bool),
-    sizeof (double),
-    sizeof (long),
-    sizeof (int32_t),
-    sizeof (uint32_t),
-    sizeof (int64_t),
-    sizeof (uint64_t),
-};
-
-static const char *var_source_names[] = {
-    "default",
-    "command line",
-    "environment",
-    "file",
-    "set",
-    "override"
+    sizeof(int),     sizeof(unsigned), sizeof(unsigned long), sizeof(unsigned long long),
+    sizeof(size_t),  sizeof(char),     sizeof(char),          sizeof(bool),
+    sizeof(double),  sizeof(long),     sizeof(int32_t),       sizeof(uint32_t),
+    sizeof(int64_t), sizeof(uint64_t),
 };
 
+static const char *var_source_names[] = {"default", "command line", "environment",
+                                         "file",    "set",          "override"};
 
-static const char *info_lvl_strings[] = {
-    "user/basic",
-    "user/detail",
-    "user/all",
-    "tuner/basic",
-    "tuner/detail",
-    "tuner/all",
-    "dev/basic",
-    "dev/detail",
-    "dev/all"
-};
+static const char *info_lvl_strings[] = {"user/basic",  "user/detail",  "user/all",
+                                         "tuner/basic", "tuner/detail", "tuner/all",
+                                         "dev/basic",   "dev/detail",   "dev/all"};
 
 /*
  * local functions
  */
-static int fixup_files(char **file_list, char * path, bool rel_path_search, char sep);
-static int read_files (char *file_list, opal_list_t *file_values, char sep);
-static int var_set_initial (mca_base_var_t *var, mca_base_var_t *original);
-static int var_get (int vari, mca_base_var_t **var_out, bool original);
-static int var_value_string (mca_base_var_t *var, char **value_string);
-static void mca_base_var_finalize (void);
+static int fixup_files(char **file_list, char *path, bool rel_path_search, char sep);
+static int read_files(char *file_list, opal_list_t *file_values, char sep);
+static int var_set_initial(mca_base_var_t *var, mca_base_var_t *original);
+static int var_get(int vari, mca_base_var_t **var_out, bool original);
+static int var_value_string(mca_base_var_t *var, char **value_string);
+static void mca_base_var_finalize(void);
 
 /*
  * classes
  */
-static void var_constructor (mca_base_var_t *p);
-static void var_destructor (mca_base_var_t *p);
-OBJ_CLASS_INSTANCE(mca_base_var_t, opal_object_t,
-                   var_constructor, var_destructor);
+static void var_constructor(mca_base_var_t *p);
+static void var_destructor(mca_base_var_t *p);
+OBJ_CLASS_INSTANCE(mca_base_var_t, opal_object_t, var_constructor, var_destructor);
 
-static void fv_constructor (mca_base_var_file_value_t *p);
-static void fv_destructor (mca_base_var_file_value_t *p);
-OBJ_CLASS_INSTANCE(mca_base_var_file_value_t, opal_list_item_t,
-                   fv_constructor, fv_destructor);
+static void fv_constructor(mca_base_var_file_value_t *p);
+static void fv_destructor(mca_base_var_file_value_t *p);
+OBJ_CLASS_INSTANCE(mca_base_var_file_value_t, opal_list_item_t, fv_constructor, fv_destructor);
 
-static const char *mca_base_var_source_file (const mca_base_var_t *var)
+static const char *mca_base_var_source_file(const mca_base_var_t *var)
 {
     mca_base_var_file_value_t *fv = (mca_base_var_file_value_t *) var->mbv_file_value;
 
@@ -188,34 +156,34 @@ static const char *mca_base_var_source_file (const mca_base_var_t *var)
 /*
  * Generate a full name from three names
  */
-int mca_base_var_generate_full_name4 (const char *project, const char *framework, const char *component,
-                                      const char *variable, char **full_name)
+int mca_base_var_generate_full_name4(const char *project, const char *framework,
+                                     const char *component, const char *variable, char **full_name)
 {
-    const char * const names[] = {project, framework, component, variable};
+    const char *const names[] = {project, framework, component, variable};
     char *name, *tmp;
     size_t i, len;
 
     *full_name = NULL;
 
-    for (i = 0, len = 0 ; i < 4 ; ++i) {
+    for (i = 0, len = 0; i < 4; ++i) {
         if (NULL != names[i]) {
             /* Add space for the string + _ or \0 */
-            len += strlen (names[i]) + 1;
+            len += strlen(names[i]) + 1;
         }
     }
 
-    name = calloc (1, len);
+    name = calloc(1, len);
     if (NULL == name) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
-    for (i = 0, tmp = name ; i < 4 ; ++i) {
+    for (i = 0, tmp = name; i < 4; ++i) {
         if (NULL != names[i]) {
             if (name != tmp) {
                 *tmp++ = '_';
             }
-            strncat (name, names[i], len - (size_t)(uintptr_t)(tmp - name));
-            tmp += strlen (names[i]);
+            strncat(name, names[i], len - (size_t)(uintptr_t)(tmp - name));
+            tmp += strlen(names[i]);
         }
     }
 
@@ -223,14 +191,14 @@ int mca_base_var_generate_full_name4 (const char *project, const char *framework
     return OPAL_SUCCESS;
 }
 
-static int compare_strings (const char *str1, const char *str2) {
-    if ((NULL != str1 && 0 == strcmp (str1, "*")) ||
-        (NULL == str1 && NULL == str2)) {
+static int compare_strings(const char *str1, const char *str2)
+{
+    if ((NULL != str1 && 0 == strcmp(str1, "*")) || (NULL == str1 && NULL == str2)) {
         return 0;
     }
 
     if (NULL != str1 && NULL != str2) {
-        return strcmp (str1, str2);
+        return strcmp(str1, str2);
     }
 
     return 1;
@@ -249,7 +217,7 @@ static char *append_filename_to_list(const char *filename)
     count = opal_argv_count(mca_base_var_file_list);
 
     for (i = count - 1; i >= 0; --i) {
-        if (0 == strcmp (mca_base_var_file_list[i], filename)) {
+        if (0 == strcmp(mca_base_var_file_list[i], filename)) {
             return mca_base_var_file_list[i];
         }
     }
@@ -270,7 +238,7 @@ int mca_base_var_init(void)
 
         OBJ_CONSTRUCT(&mca_base_vars, opal_pointer_array_t);
         /* These values are arbitrary */
-        ret = opal_pointer_array_init (&mca_base_vars, 128, 16384, 128);
+        ret = opal_pointer_array_init(&mca_base_vars, 128, 16384, 128);
         if (OPAL_SUCCESS != ret) {
             return ret;
         }
@@ -284,29 +252,29 @@ int mca_base_var_init(void)
         OBJ_CONSTRUCT(&mca_base_var_override_values, opal_list_t);
         OBJ_CONSTRUCT(&mca_base_var_index_hash, opal_hash_table_t);
 
-        ret = opal_hash_table_init (&mca_base_var_index_hash, 1024);
+        ret = opal_hash_table_init(&mca_base_var_index_hash, 1024);
         if (OPAL_SUCCESS != ret) {
             return ret;
         }
 
-        ret = mca_base_var_group_init ();
-        if  (OPAL_SUCCESS != ret) {
+        ret = mca_base_var_group_init();
+        if (OPAL_SUCCESS != ret) {
             return ret;
         }
 
-        ret = mca_base_pvar_init ();
+        ret = mca_base_pvar_init();
         if (OPAL_SUCCESS != ret) {
             return ret;
         }
 
         /* We may need this later */
-        home = (char*)opal_home_directory();
+        home = (char *) opal_home_directory();
         if (NULL == home) {
             opal_output(0, "Error: Unable to get the user home directory\n");
             return OPAL_ERROR;
         }
 
-        if( NULL == (cwd = getcwd(NULL, 0) )) {
+        if (NULL == (cwd = getcwd(NULL, 0))) {
             opal_output(0, "Error: Unable to get the current working directory\n");
             cwd = strdup(".");
         }
@@ -314,35 +282,34 @@ int mca_base_var_init(void)
         /* Set this before we register the parameter, below */
 
         mca_base_var_initialized = true;
-
     }
 
-    opal_finalize_register_cleanup (mca_base_var_finalize);
+    opal_finalize_register_cleanup(mca_base_var_finalize);
 
     return OPAL_SUCCESS;
 }
 
 static void process_env_list(char *env_list, char ***argv, char sep)
 {
-    char** tokens;
+    char **tokens;
     char *ptr, *value;
 
-    tokens = opal_argv_split(env_list, (int)sep);
+    tokens = opal_argv_split(env_list, (int) sep);
     if (NULL == tokens) {
         return;
     }
 
-    for (int i = 0 ; NULL != tokens[i] ; ++i) {
+    for (int i = 0; NULL != tokens[i]; ++i) {
         if (NULL == (ptr = strchr(tokens[i], '='))) {
             value = getenv(tokens[i]);
             if (NULL == value) {
-                opal_show_help("help-mca-var.txt", "incorrect-env-list-param",
-                               true, tokens[i], env_list);
+                opal_show_help("help-mca-var.txt", "incorrect-env-list-param", true, tokens[i],
+                               env_list);
                 break;
             }
 
             /* duplicate the value to silence tainted string coverity issue */
-            value = strdup (value);
+            value = strdup(value);
             if (NULL == value) {
                 /* out of memory */
                 break;
@@ -355,7 +322,7 @@ static void process_env_list(char *env_list, char ***argv, char sep)
                 opal_setenv(tokens[i], value, true, argv);
             }
 
-            free (value);
+            free(value);
         } else {
             *ptr = '\0';
             opal_setenv(tokens[i], ptr + 1, true, argv);
@@ -374,8 +341,8 @@ int mca_base_var_process_env_list(char *list, char ***argv)
         if (1 == strlen(mca_base_env_list_sep)) {
             sep = mca_base_env_list_sep[0];
         } else {
-            opal_show_help("help-mca-var.txt", "incorrect-env-list-sep",
-                    true, mca_base_env_list_sep);
+            opal_show_help("help-mca-var.txt", "incorrect-env-list-sep", true,
+                           mca_base_env_list_sep);
             return OPAL_SUCCESS;
         }
     }
@@ -396,33 +363,35 @@ int mca_base_var_process_env_list_from_file(char ***argv)
     return OPAL_SUCCESS;
 }
 
-static void resolve_relative_paths(char **file_prefix, char *file_path, bool rel_path_search, char **files, char sep)
+static void resolve_relative_paths(char **file_prefix, char *file_path, bool rel_path_search,
+                                   char **files, char sep)
 {
     char *tmp_str;
     /*
      * Resolve all relative paths.
      * the file list returned will contain only absolute paths
      */
-    if( OPAL_SUCCESS != fixup_files(file_prefix, file_path, rel_path_search, sep) ) {
+    if (OPAL_SUCCESS != fixup_files(file_prefix, file_path, rel_path_search, sep)) {
 #if 0
         /* JJH We need to die! */
         abort();
 #else
         ;
 #endif
-    }
-    else {
+    } else {
         /* Prepend the files to the search list */
         opal_asprintf(&tmp_str, "%s%c%s", *file_prefix, sep, *files);
-        free (*files);
+        free(*files);
         *files = tmp_str;
     }
 }
 
-int mca_base_var_load_extra_files(char* files, bool rel_path_search) {
-    char *tmp1=strdup(files);
-    resolve_relative_paths(&tmp1, mca_base_param_file_path, rel_path_search, &mca_base_var_files, OPAL_ENV_SEP);
-    read_files (tmp1, &mca_base_var_file_values, ',');
+int mca_base_var_load_extra_files(char *files, bool rel_path_search)
+{
+    char *tmp1 = strdup(files);
+    resolve_relative_paths(&tmp1, mca_base_param_file_path, rel_path_search, &mca_base_var_files,
+                           OPAL_ENV_SEP);
+    read_files(tmp1, &mca_base_var_file_values, ',');
     free(tmp1);
     return OPAL_SUCCESS;
 }
@@ -434,19 +403,18 @@ int mca_base_var_cache_files(bool rel_path_search)
 
 #if OPAL_WANT_HOME_CONFIG_FILES
     if (NULL == getenv("OPAL_USER_PARAMS_GIVEN")) {
-        opal_asprintf(&tmp, "%s"OPAL_PATH_SEP".openmpi" OPAL_PATH_SEP
-                 "mca-params.conf", home);
+        opal_asprintf(&tmp, "%s" OPAL_PATH_SEP ".openmpi" OPAL_PATH_SEP "mca-params.conf", home);
     }
 #endif
 
     if (NULL == getenv("OPAL_SYS_PARAMS_GIVEN")) {
         if (NULL != tmp) {
-            opal_asprintf(&mca_base_var_files, "%s,%s" OPAL_PATH_SEP "openmpi-mca-params.conf",
-                     tmp, opal_install_dirs.sysconfdir);
+            opal_asprintf(&mca_base_var_files, "%s,%s" OPAL_PATH_SEP "openmpi-mca-params.conf", tmp,
+                          opal_install_dirs.sysconfdir);
             free(tmp);
         } else {
             opal_asprintf(&mca_base_var_files, "%s" OPAL_PATH_SEP "openmpi-mca-params.conf",
-                     opal_install_dirs.sysconfdir);
+                          opal_install_dirs.sysconfdir);
         }
     } else {
         mca_base_var_files = strdup("none");
@@ -455,48 +423,51 @@ int mca_base_var_cache_files(bool rel_path_search)
     /* Initialize a parameter that says where MCA param files can be found.
        We may change this value so set the scope to MCA_BASE_VAR_SCOPE_READONLY */
     tmp = mca_base_var_files;
-    ret = mca_base_var_register ("opal", "mca", "base", "param_files", "Path for MCA "
-                                 "configuration files containing variable values",
-                                 MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_2,
-                                 MCA_BASE_VAR_SCOPE_READONLY, &mca_base_var_files);
-    free (tmp);
+    ret = mca_base_var_register("opal", "mca", "base", "param_files",
+                                "Path for MCA "
+                                "configuration files containing variable values",
+                                MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_2,
+                                MCA_BASE_VAR_SCOPE_READONLY, &mca_base_var_files);
+    free(tmp);
     if (0 > ret) {
         return ret;
     }
 
     mca_base_envar_files = strdup(mca_base_var_files);
 
-    (void) mca_base_var_register_synonym (ret, "opal", "mca", NULL, "param_files",
-                                          MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
+    (void) mca_base_var_register_synonym(ret, "opal", "mca", NULL, "param_files",
+                                         MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
 
-    ret = opal_asprintf(&mca_base_var_override_file, "%s" OPAL_PATH_SEP "openmpi-mca-params-override.conf",
-                   opal_install_dirs.sysconfdir);
+    ret = opal_asprintf(&mca_base_var_override_file,
+                        "%s" OPAL_PATH_SEP "openmpi-mca-params-override.conf",
+                        opal_install_dirs.sysconfdir);
     if (0 > ret) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
     tmp = mca_base_var_override_file;
-    ret = mca_base_var_register ("opal", "mca", "base", "override_param_file",
-                                 "Variables set in this file will override any value set in"
-                                 "the environment or another configuration file",
-                                 MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_DEFAULT_ONLY,
-                                 OPAL_INFO_LVL_2, MCA_BASE_VAR_SCOPE_CONSTANT,
-                                 &mca_base_var_override_file);
-    free (tmp);
+    ret = mca_base_var_register("opal", "mca", "base", "override_param_file",
+                                "Variables set in this file will override any value set in"
+                                "the environment or another configuration file",
+                                MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_DEFAULT_ONLY,
+                                OPAL_INFO_LVL_2, MCA_BASE_VAR_SCOPE_CONSTANT,
+                                &mca_base_var_override_file);
+    free(tmp);
     if (0 > ret) {
         return ret;
     }
 
     /* Disable reading MCA parameter files. */
-    if (0 == strcmp (mca_base_var_files, "none")) {
+    if (0 == strcmp(mca_base_var_files, "none")) {
         return OPAL_SUCCESS;
     }
 
     mca_base_var_suppress_override_warning = false;
-    ret = mca_base_var_register ("opal", "mca", "base", "suppress_override_warning",
-                                 "Suppress warnings when attempting to set an overridden value (default: false)",
-                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_2,
-                                 MCA_BASE_VAR_SCOPE_LOCAL, &mca_base_var_suppress_override_warning);
+    ret = mca_base_var_register(
+        "opal", "mca", "base", "suppress_override_warning",
+        "Suppress warnings when attempting to set an overridden value (default: false)",
+        MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_2, MCA_BASE_VAR_SCOPE_LOCAL,
+        &mca_base_var_suppress_override_warning);
     if (0 > ret) {
         return ret;
     }
@@ -506,44 +477,44 @@ int mca_base_var_cache_files(bool rel_path_search)
      * requests that do not specify an absolute path
      */
     mca_base_var_file_prefix = NULL;
-    ret = mca_base_var_register ("opal", "mca", "base", "param_file_prefix",
-                                 "Aggregate MCA parameter file sets",
-                                 MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_3,
-                                 MCA_BASE_VAR_SCOPE_READONLY, &mca_base_var_file_prefix);
+    ret = mca_base_var_register("opal", "mca", "base", "param_file_prefix",
+                                "Aggregate MCA parameter file sets", MCA_BASE_VAR_TYPE_STRING, NULL,
+                                0, 0, OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_READONLY,
+                                &mca_base_var_file_prefix);
     if (0 > ret) {
         return ret;
     }
 
     mca_base_envar_file_prefix = NULL;
-    ret = mca_base_var_register ("opal", "mca", "base", "envar_file_prefix",
-                                 "Aggregate MCA parameter file set for env variables",
-                                 MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_3,
-                                 MCA_BASE_VAR_SCOPE_READONLY, &mca_base_envar_file_prefix);
+    ret = mca_base_var_register("opal", "mca", "base", "envar_file_prefix",
+                                "Aggregate MCA parameter file set for env variables",
+                                MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_3,
+                                MCA_BASE_VAR_SCOPE_READONLY, &mca_base_envar_file_prefix);
     if (0 > ret) {
         return ret;
     }
 
     ret = opal_asprintf(&mca_base_param_file_path, "%s" OPAL_PATH_SEP "amca-param-sets%c%s",
-                   opal_install_dirs.opaldatadir, OPAL_ENV_SEP, cwd);
+                        opal_install_dirs.opaldatadir, OPAL_ENV_SEP, cwd);
     if (0 > ret) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
     tmp = mca_base_param_file_path;
-    ret = mca_base_var_register ("opal", "mca", "base", "param_file_path",
-                                 "Aggregate MCA parameter Search path",
-                                 MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_3,
-                                 MCA_BASE_VAR_SCOPE_READONLY, &mca_base_param_file_path);
-    free (tmp);
+    ret = mca_base_var_register("opal", "mca", "base", "param_file_path",
+                                "Aggregate MCA parameter Search path", MCA_BASE_VAR_TYPE_STRING,
+                                NULL, 0, 0, OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_READONLY,
+                                &mca_base_param_file_path);
+    free(tmp);
     if (0 > ret) {
         return ret;
     }
 
     force_agg_path = NULL;
-    ret = mca_base_var_register ("opal", "mca", "base", "param_file_path_force",
-                                 "Forced Aggregate MCA parameter Search path",
-                                 MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_3,
-                                 MCA_BASE_VAR_SCOPE_READONLY, &force_agg_path);
+    ret = mca_base_var_register("opal", "mca", "base", "param_file_path_force",
+                                "Forced Aggregate MCA parameter Search path",
+                                MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_3,
+                                MCA_BASE_VAR_SCOPE_READONLY, &force_agg_path);
     if (0 > ret) {
         return ret;
     }
@@ -552,7 +523,8 @@ int mca_base_var_cache_files(bool rel_path_search)
         if (NULL != mca_base_param_file_path) {
             char *tmp_str = mca_base_param_file_path;
 
-            opal_asprintf(&mca_base_param_file_path, "%s%c%s", force_agg_path, OPAL_ENV_SEP, tmp_str);
+            opal_asprintf(&mca_base_param_file_path, "%s%c%s", force_agg_path, OPAL_ENV_SEP,
+                          tmp_str);
             free(tmp_str);
         } else {
             mca_base_param_file_path = strdup(force_agg_path);
@@ -560,17 +532,19 @@ int mca_base_var_cache_files(bool rel_path_search)
     }
 
     if (NULL != mca_base_var_file_prefix) {
-       resolve_relative_paths(&mca_base_var_file_prefix, mca_base_param_file_path, rel_path_search, &mca_base_var_files, OPAL_ENV_SEP);
+        resolve_relative_paths(&mca_base_var_file_prefix, mca_base_param_file_path, rel_path_search,
+                               &mca_base_var_files, OPAL_ENV_SEP);
     }
-    read_files (mca_base_var_files, &mca_base_var_file_values, ',');
+    read_files(mca_base_var_files, &mca_base_var_file_values, ',');
 
     if (NULL != mca_base_envar_file_prefix) {
-       resolve_relative_paths(&mca_base_envar_file_prefix, mca_base_param_file_path, rel_path_search, &mca_base_envar_files, ',');
+        resolve_relative_paths(&mca_base_envar_file_prefix, mca_base_param_file_path,
+                               rel_path_search, &mca_base_envar_files, ',');
     }
-    read_files (mca_base_envar_files, &mca_base_envar_file_values, ',');
+    read_files(mca_base_envar_files, &mca_base_envar_file_values, ',');
 
     if (0 == access(mca_base_var_override_file, F_OK)) {
-        read_files (mca_base_var_override_file, &mca_base_var_override_values, OPAL_ENV_SEP);
+        read_files(mca_base_var_override_file, &mca_base_var_override_values, OPAL_ENV_SEP);
     }
 
     return OPAL_SUCCESS;
@@ -579,15 +553,14 @@ int mca_base_var_cache_files(bool rel_path_search)
 /*
  * Look up an integer MCA parameter.
  */
-int mca_base_var_get_value (int vari, const void *value,
-                            mca_base_var_source_t *source,
-                            const char **source_file)
+int mca_base_var_get_value(int vari, const void *value, mca_base_var_source_t *source,
+                           const char **source_file)
 {
     mca_base_var_t *var;
     void **tmp = (void **) value;
     int ret;
 
-    ret = var_get (vari, &var, true);
+    ret = var_get(vari, &var, true);
     if (OPAL_SUCCESS != ret) {
         return ret;
     }
@@ -597,7 +570,7 @@ int mca_base_var_get_value (int vari, const void *value,
     }
 
     if (NULL != value) {
-        /* Return a poiner to our backing store (either a char **, int *,
+        /* Return a pointer to our backing store (either a char **, int *,
            or bool *) */
         *tmp = var->mbv_storage;
     }
@@ -607,55 +580,54 @@ int mca_base_var_get_value (int vari, const void *value,
     }
 
     if (NULL != source_file) {
-        *source_file = mca_base_var_source_file (var);
+        *source_file = mca_base_var_source_file(var);
     }
 
     return OPAL_SUCCESS;
 }
 
-static int var_set_string (mca_base_var_t *var, char *value)
+static int var_set_string(mca_base_var_t *var, char *value)
 {
     char *tmp;
     int ret;
 
     if (NULL != var->mbv_storage->stringval) {
-        free (var->mbv_storage->stringval);
+        free(var->mbv_storage->stringval);
     }
 
     var->mbv_storage->stringval = NULL;
 
-    if (NULL == value || 0 == strlen (value)) {
+    if (NULL == value || 0 == strlen(value)) {
         return OPAL_SUCCESS;
     }
 
     /* Replace all instances of ~/ in a path-style string with the
        user's home directory. This may be handled by the enumerator
        in the future. */
-    if (0 == strncmp (value, "~/", 2)) {
+    if (0 == strncmp(value, "~/", 2)) {
         if (NULL != home) {
-            ret = opal_asprintf (&value, "%s/%s", home, value + 2);
+            ret = opal_asprintf(&value, "%s/%s", home, value + 2);
             if (0 > ret) {
                 return OPAL_ERROR;
             }
         } else {
-            value = strdup (value + 2);
+            value = strdup(value + 2);
         }
     } else {
-        value = strdup (value);
+        value = strdup(value);
     }
 
     if (NULL == value) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
-    while (NULL != (tmp = strstr (value, ":~/"))) {
+    while (NULL != (tmp = strstr(value, ":~/"))) {
         tmp[0] = '\0';
         tmp += 3;
 
-        ret = opal_asprintf (&tmp, "%s:%s%s%s", value,
-                        home ? home : "", home ? "/" : "", tmp);
+        ret = opal_asprintf(&tmp, "%s:%s%s%s", value, home ? home : "", home ? "/" : "", tmp);
 
-        free (value);
+        free(value);
 
         if (0 > ret) {
             return OPAL_ERR_OUT_OF_RESOURCE;
@@ -675,7 +647,7 @@ static int int_from_string(const char *src, mca_base_var_enum_t *enumerator, uin
     bool is_int;
     char *tmp;
 
-    if (NULL == src || 0 == strlen (src)) {
+    if (NULL == src || 0 == strlen(src)) {
         if (NULL == enumerator) {
             *value_out = 0;
         }
@@ -695,7 +667,7 @@ static int int_from_string(const char *src, mca_base_var_enum_t *enumerator, uin
     }
 
     /* Check for an integer value */
-    value = strtoull (src, &tmp, 0);
+    value = strtoull(src, &tmp, 0);
     is_int = tmp[0] == '\0';
 
     if (!is_int && tmp != src) {
@@ -720,7 +692,7 @@ static int int_from_string(const char *src, mca_base_var_enum_t *enumerator, uin
     return OPAL_SUCCESS;
 }
 
-static int var_set_from_string (mca_base_var_t *var, char *src)
+static int var_set_from_string(mca_base_var_t *var, char *src)
 {
     mca_base_var_storage_t *dst = var->mbv_storage;
     uint64_t int_value = 0;
@@ -739,59 +711,60 @@ static int var_set_from_string (mca_base_var_t *var, char *src)
     case MCA_BASE_VAR_TYPE_BOOL:
     case MCA_BASE_VAR_TYPE_SIZE_T:
         ret = int_from_string(src, var->mbv_enumerator, &int_value);
-        if (OPAL_SUCCESS != ret ||
-            (MCA_BASE_VAR_TYPE_INT == var->mbv_type && ((int) int_value != (int64_t) int_value)) ||
-            (MCA_BASE_VAR_TYPE_UNSIGNED_INT == var->mbv_type && ((unsigned int) int_value != int_value))) {
+        if (OPAL_SUCCESS != ret
+            || (MCA_BASE_VAR_TYPE_INT == var->mbv_type && ((int) int_value != (int64_t) int_value))
+            || (MCA_BASE_VAR_TYPE_UNSIGNED_INT == var->mbv_type
+                && ((unsigned int) int_value != int_value))) {
             if (var->mbv_enumerator) {
                 char *valid_values;
-                (void) var->mbv_enumerator->dump(var->mbv_enumerator, &valid_values);
-                opal_show_help("help-mca-var.txt", "invalid-value-enum",
-                               true, var->mbv_full_name, src, valid_values);
+                (void) var->mbv_enumerator->dump(var->mbv_enumerator, &valid_values,
+                    MCA_BASE_VAR_ENUM_DUMP_READABLE);
+                opal_show_help("help-mca-var.txt", "invalid-value-enum", true, var->mbv_full_name,
+                               src, valid_values);
                 free(valid_values);
             } else {
-                opal_show_help("help-mca-var.txt", "invalid-value",
-                               true, var->mbv_full_name, src);
+                opal_show_help("help-mca-var.txt", "invalid-value", true, var->mbv_full_name, src);
             }
 
             return OPAL_ERR_VALUE_OUT_OF_BOUNDS;
         }
 
-        if (MCA_BASE_VAR_TYPE_INT == var->mbv_type ||
-            MCA_BASE_VAR_TYPE_UNSIGNED_INT == var->mbv_type) {
-            int *castme = (int*) var->mbv_storage;
+        if (MCA_BASE_VAR_TYPE_INT == var->mbv_type
+            || MCA_BASE_VAR_TYPE_UNSIGNED_INT == var->mbv_type) {
+            int *castme = (int *) var->mbv_storage;
             *castme = int_value;
-        } else if (MCA_BASE_VAR_TYPE_INT32_T == var->mbv_type ||
-            MCA_BASE_VAR_TYPE_UINT32_T == var->mbv_type) {
+        } else if (MCA_BASE_VAR_TYPE_INT32_T == var->mbv_type
+                   || MCA_BASE_VAR_TYPE_UINT32_T == var->mbv_type) {
             int32_t *castme = (int32_t *) var->mbv_storage;
             *castme = int_value;
-        } else if (MCA_BASE_VAR_TYPE_INT64_T == var->mbv_type ||
-            MCA_BASE_VAR_TYPE_UINT64_T == var->mbv_type) {
+        } else if (MCA_BASE_VAR_TYPE_INT64_T == var->mbv_type
+                   || MCA_BASE_VAR_TYPE_UINT64_T == var->mbv_type) {
             int64_t *castme = (int64_t *) var->mbv_storage;
             *castme = int_value;
         } else if (MCA_BASE_VAR_TYPE_LONG == var->mbv_type) {
-            long *castme = (long*) var->mbv_storage;
+            long *castme = (long *) var->mbv_storage;
             *castme = (long) int_value;
         } else if (MCA_BASE_VAR_TYPE_UNSIGNED_LONG == var->mbv_type) {
-            unsigned long *castme = (unsigned long*) var->mbv_storage;
+            unsigned long *castme = (unsigned long *) var->mbv_storage;
             *castme = (unsigned long) int_value;
         } else if (MCA_BASE_VAR_TYPE_UNSIGNED_LONG_LONG == var->mbv_type) {
-            unsigned long long *castme = (unsigned long long*) var->mbv_storage;
+            unsigned long long *castme = (unsigned long long *) var->mbv_storage;
             *castme = (unsigned long long) int_value;
         } else if (MCA_BASE_VAR_TYPE_SIZE_T == var->mbv_type) {
-            size_t *castme = (size_t*) var->mbv_storage;
+            size_t *castme = (size_t *) var->mbv_storage;
             *castme = (size_t) int_value;
         } else if (MCA_BASE_VAR_TYPE_BOOL == var->mbv_type) {
-            bool *castme = (bool*) var->mbv_storage;
+            bool *castme = (bool *) var->mbv_storage;
             *castme = !!int_value;
         }
 
         return ret;
     case MCA_BASE_VAR_TYPE_DOUBLE:
-        dst->lfval = strtod (src, NULL);
+        dst->lfval = strtod(src, NULL);
         break;
     case MCA_BASE_VAR_TYPE_STRING:
     case MCA_BASE_VAR_TYPE_VERSION_STRING:
-        var_set_string (var, src);
+        var_set_string(var, src);
         break;
     case MCA_BASE_VAR_TYPE_MAX:
         return OPAL_ERROR;
@@ -803,13 +776,13 @@ static int var_set_from_string (mca_base_var_t *var, char *src)
 /*
  * Set a variable
  */
-int mca_base_var_set_value (int vari, const void *value, size_t size, mca_base_var_source_t source,
-                            const char *source_file)
+int mca_base_var_set_value(int vari, const void *value, size_t size, mca_base_var_source_t source,
+                           const char *source_file)
 {
     mca_base_var_t *var;
     int ret;
 
-    ret = var_get (vari, &var, true);
+    ret = var_get(vari, &var, true);
     if (OPAL_SUCCESS != ret) {
         return ret;
     }
@@ -824,17 +797,17 @@ int mca_base_var_set_value (int vari, const void *value, size_t size, mca_base_v
 
     if (NULL != var->mbv_enumerator) {
         /* Validate */
-        ret = var->mbv_enumerator->string_from_value(var->mbv_enumerator,
-                                                     ((int *) value)[0], NULL);
+        ret = var->mbv_enumerator->string_from_value(var->mbv_enumerator, ((int *) value)[0], NULL);
         if (OPAL_SUCCESS != ret) {
             return ret;
         }
     }
 
-    if (MCA_BASE_VAR_TYPE_STRING != var->mbv_type && MCA_BASE_VAR_TYPE_VERSION_STRING != var->mbv_type) {
-        memmove (var->mbv_storage, value, ompi_var_type_sizes[var->mbv_type]);
+    if (MCA_BASE_VAR_TYPE_STRING != var->mbv_type
+        && MCA_BASE_VAR_TYPE_VERSION_STRING != var->mbv_type) {
+        memmove(var->mbv_storage, value, ompi_var_type_sizes[var->mbv_type]);
     } else {
-        var_set_string (var, (char *) value);
+        var_set_string(var, (char *) value);
     }
 
     var->mbv_source = source;
@@ -855,7 +828,7 @@ int mca_base_var_deregister(int vari)
     mca_base_var_t *var;
     int ret;
 
-    ret = var_get (vari, &var, false);
+    ret = var_get(vari, &var, false);
     if (OPAL_SUCCESS != ret) {
         return ret;
     }
@@ -874,12 +847,13 @@ int mca_base_var_deregister(int vari)
     }
 
     /* Release the current value if it is a string. */
-    if ((MCA_BASE_VAR_TYPE_STRING == var->mbv_type || MCA_BASE_VAR_TYPE_VERSION_STRING == var->mbv_type) &&
-        var->mbv_storage->stringval) {
-        free (var->mbv_storage->stringval);
+    if ((MCA_BASE_VAR_TYPE_STRING == var->mbv_type
+         || MCA_BASE_VAR_TYPE_VERSION_STRING == var->mbv_type)
+        && var->mbv_storage->stringval) {
+        free(var->mbv_storage->stringval);
         var->mbv_storage->stringval = NULL;
     } else {
-        OPAL_MCA_VAR_MBV_ENUMERATOR_FREE(var -> mbv_enumerator);
+        OPAL_MCA_VAR_MBV_ENUMERATOR_FREE(var->mbv_enumerator);
     }
 
     var->mbv_enumerator = NULL;
@@ -889,7 +863,7 @@ int mca_base_var_deregister(int vari)
     return OPAL_SUCCESS;
 }
 
-static int var_get (int vari, mca_base_var_t **var_out, bool original)
+static int var_get(int vari, mca_base_var_t **var_out, bool original)
 {
     mca_base_var_t *var;
 
@@ -906,7 +880,7 @@ static int var_get (int vari, mca_base_var_t **var_out, bool original)
         return OPAL_ERR_BAD_PARAM;
     }
 
-    var = opal_pointer_array_get_item (&mca_base_vars, vari);
+    var = opal_pointer_array_get_item(&mca_base_vars, vari);
     if (NULL == var) {
         return OPAL_ERR_BAD_PARAM;
     }
@@ -922,12 +896,11 @@ static int var_get (int vari, mca_base_var_t **var_out, bool original)
     return OPAL_SUCCESS;
 }
 
-int mca_base_var_env_name(const char *param_name,
-                          char **env_name)
+int mca_base_var_env_name(const char *param_name, char **env_name)
 {
     int ret;
 
-    assert (NULL != env_name);
+    assert(NULL != env_name);
 
     ret = opal_asprintf(env_name, "%s%s", mca_prefix, param_name);
     if (0 > ret) {
@@ -940,37 +913,36 @@ int mca_base_var_env_name(const char *param_name,
 /*
  * Find the index for an MCA parameter based on its names.
  */
-static int var_find_by_name (const char *full_name, int *vari, bool invalidok)
+static int var_find_by_name(const char *full_name, int *vari, bool invalidok)
 {
     mca_base_var_t *var = NULL;
     void *tmp;
     int rc;
 
-    rc = opal_hash_table_get_value_ptr (&mca_base_var_index_hash, full_name, strlen (full_name),
-                                        &tmp);
+    rc = opal_hash_table_get_value_ptr(&mca_base_var_index_hash, full_name, strlen(full_name),
+                                       &tmp);
     if (OPAL_SUCCESS != rc) {
         return rc;
     }
 
-    (void) var_get ((int)(uintptr_t) tmp, &var, false);
+    (void) var_get((int) (uintptr_t) tmp, &var, false);
 
     if (invalidok || (var && VAR_IS_VALID(var[0]))) {
-        *vari = (int)(uintptr_t) tmp;
+        *vari = (int) (uintptr_t) tmp;
         return OPAL_SUCCESS;
     }
 
     return OPAL_ERR_NOT_FOUND;
 }
 
-static int var_find (const char *project_name, const char *framework_name,
-                     const char *component_name, const char *variable_name,
-                     bool invalidok)
+static int var_find(const char *project_name, const char *framework_name,
+                    const char *component_name, const char *variable_name, bool invalidok)
 {
     char *full_name;
     int ret, vari;
 
-    ret = mca_base_var_generate_full_name4 (NULL, framework_name, component_name,
-                                            variable_name, &full_name);
+    ret = mca_base_var_generate_full_name4(NULL, framework_name, component_name, variable_name,
+                                           &full_name);
     if (OPAL_SUCCESS != ret) {
         return OPAL_ERROR;
     }
@@ -979,7 +951,7 @@ static int var_find (const char *project_name, const char *framework_name,
 
     /* NTH: should we verify the name components match? */
 
-    free (full_name);
+    free(full_name);
 
     if (OPAL_SUCCESS != ret) {
         return ret;
@@ -991,26 +963,26 @@ static int var_find (const char *project_name, const char *framework_name,
 /*
  * Find the index for an MCA parameter based on its name components.
  */
-int mca_base_var_find (const char *project_name, const char *framework_name,
-                       const char *component_name, const char *variable_name)
+int mca_base_var_find(const char *project_name, const char *framework_name,
+                      const char *component_name, const char *variable_name)
 {
-    return var_find (project_name, framework_name, component_name, variable_name, false);
+    return var_find(project_name, framework_name, component_name, variable_name, false);
 }
 
 /*
  * Find the index for an MCA parameter based on full name.
  */
-int mca_base_var_find_by_name (const char *full_name, int *vari)
+int mca_base_var_find_by_name(const char *full_name, int *vari)
 {
-    return var_find_by_name (full_name, vari, false);
+    return var_find_by_name(full_name, vari, false);
 }
 
-int mca_base_var_set_flag (int vari, mca_base_var_flag_t flag, bool set)
+int mca_base_var_set_flag(int vari, mca_base_var_flag_t flag, bool set)
 {
     mca_base_var_t *var;
     int ret;
 
-    ret = var_get (vari, &var, true);
+    ret = var_get(vari, &var, true);
     if (OPAL_SUCCESS != ret || VAR_IS_SYNONYM(var[0])) {
         return OPAL_ERR_BAD_PARAM;
     }
@@ -1024,10 +996,10 @@ int mca_base_var_set_flag (int vari, mca_base_var_flag_t flag, bool set)
 /*
  * Return info on a parameter at an index
  */
-int mca_base_var_get (int vari, const mca_base_var_t **var)
+int mca_base_var_get(int vari, const mca_base_var_t **var)
 {
     int ret;
-    ret = var_get (vari, (mca_base_var_t **) var, false);
+    ret = var_get(vari, (mca_base_var_t **) var, false);
 
     if (OPAL_SUCCESS != ret) {
         return ret;
@@ -1062,31 +1034,31 @@ int mca_base_var_build_env(char ***env, int *num_env, bool internal)
         char *value_string;
         char *str = NULL;
 
-        var = opal_pointer_array_get_item (&mca_base_vars, i);
+        var = opal_pointer_array_get_item(&mca_base_vars, i);
         if (NULL == var) {
             continue;
         }
 
         /* Don't output default values or internal variables (unless
            requested) */
-        if (MCA_BASE_VAR_SOURCE_DEFAULT == var->mbv_source ||
-            (!internal && VAR_IS_INTERNAL(var[0]))) {
+        if (MCA_BASE_VAR_SOURCE_DEFAULT == var->mbv_source
+            || (!internal && VAR_IS_INTERNAL(var[0]))) {
             continue;
         }
 
-        if ((MCA_BASE_VAR_TYPE_STRING == var->mbv_type || MCA_BASE_VAR_TYPE_VERSION_STRING == var->mbv_type) &&
-            NULL == var->mbv_storage->stringval) {
+        if ((MCA_BASE_VAR_TYPE_STRING == var->mbv_type
+             || MCA_BASE_VAR_TYPE_VERSION_STRING == var->mbv_type)
+            && NULL == var->mbv_storage->stringval) {
             continue;
         }
 
-        ret = var_value_string (var, &value_string);
+        ret = var_value_string(var, &value_string);
         if (OPAL_SUCCESS != ret) {
             goto cleanup;
         }
 
-        ret = opal_asprintf (&str, "%s%s=%s", mca_prefix, var->mbv_full_name,
-                        value_string);
-        free (value_string);
+        ret = opal_asprintf(&str, "%s%s=%s", mca_prefix, var->mbv_full_name, value_string);
+        free(value_string);
         if (0 > ret) {
             goto cleanup;
         }
@@ -1097,11 +1069,11 @@ int mca_base_var_build_env(char ***env, int *num_env, bool internal)
         switch (var->mbv_source) {
         case MCA_BASE_VAR_SOURCE_FILE:
         case MCA_BASE_VAR_SOURCE_OVERRIDE:
-            opal_asprintf (&str, "%sSOURCE_%s=FILE:%s", mca_prefix, var->mbv_full_name,
-                      mca_base_var_source_file (var));
+            opal_asprintf(&str, "%sSOURCE_%s=FILE:%s", mca_prefix, var->mbv_full_name,
+                          mca_base_var_source_file(var));
             break;
         case MCA_BASE_VAR_SOURCE_COMMAND_LINE:
-            opal_asprintf (&str, "%sSOURCE_%s=COMMAND_LINE", mca_prefix, var->mbv_full_name);
+            opal_asprintf(&str, "%sSOURCE_%s=COMMAND_LINE", mca_prefix, var->mbv_full_name);
             break;
         case MCA_BASE_VAR_SOURCE_ENV:
         case MCA_BASE_VAR_SOURCE_SET:
@@ -1124,7 +1096,7 @@ int mca_base_var_build_env(char ***env, int *num_env, bool internal)
 
     /* Error condition */
 
- cleanup:
+cleanup:
     if (*num_env > 0) {
         opal_argv_free(*env);
         *num_env = 0;
@@ -1137,7 +1109,7 @@ int mca_base_var_build_env(char ***env, int *num_env, bool internal)
  * Shut down the MCA parameter system (normally only invoked by the
  * MCA framework itself).
  */
-static void mca_base_var_finalize (void)
+static void mca_base_var_finalize(void)
 {
     opal_object_t *object;
     opal_list_item_t *item;
@@ -1145,33 +1117,30 @@ static void mca_base_var_finalize (void)
 
     if (mca_base_var_initialized) {
         size = opal_pointer_array_get_size(&mca_base_vars);
-        for (i = 0 ; i < size ; ++i) {
-            object = opal_pointer_array_get_item (&mca_base_vars, i);
+        for (i = 0; i < size; ++i) {
+            object = opal_pointer_array_get_item(&mca_base_vars, i);
             if (NULL != object) {
                 OBJ_RELEASE(object);
             }
         }
         OBJ_DESTRUCT(&mca_base_vars);
 
-        while (NULL !=
-               (item = opal_list_remove_first(&mca_base_var_file_values))) {
+        while (NULL != (item = opal_list_remove_first(&mca_base_var_file_values))) {
             OBJ_RELEASE(item);
         }
         OBJ_DESTRUCT(&mca_base_var_file_values);
 
-        while (NULL !=
-               (item = opal_list_remove_first(&mca_base_envar_file_values))) {
+        while (NULL != (item = opal_list_remove_first(&mca_base_envar_file_values))) {
             OBJ_RELEASE(item);
         }
         OBJ_DESTRUCT(&mca_base_envar_file_values);
 
-        while (NULL !=
-               (item = opal_list_remove_first(&mca_base_var_override_values))) {
+        while (NULL != (item = opal_list_remove_first(&mca_base_var_override_values))) {
             OBJ_RELEASE(item);
         }
         OBJ_DESTRUCT(&mca_base_var_override_values);
 
-        if( NULL != cwd ) {
+        if (NULL != cwd) {
             free(cwd);
             cwd = NULL;
         }
@@ -1184,23 +1153,23 @@ static void mca_base_var_finalize (void)
         }
         mca_base_var_file_list = NULL;
 
-        (void) mca_base_var_group_finalize ();
-        (void) mca_base_pvar_finalize ();
+        (void) mca_base_var_group_finalize();
+        (void) mca_base_pvar_finalize();
 
         OBJ_DESTRUCT(&mca_base_var_index_hash);
 
-        free (mca_base_envar_files);
+        free(mca_base_envar_files);
         mca_base_envar_files = NULL;
     }
 }
 
-
 /*************************************************************************/
-static int fixup_files(char **file_list, char * path, bool rel_path_search, char sep) {
+static int fixup_files(char **file_list, char *path, bool rel_path_search, char sep)
+{
     int exit_status = OPAL_SUCCESS;
     char **files = NULL;
     char **search_path = NULL;
-    char * tmp_file = NULL;
+    char *tmp_file = NULL;
     char **argv = NULL;
     char *rel_path;
     int mode = R_OK; /* The file exists, and we can read it */
@@ -1213,7 +1182,7 @@ static int fixup_files(char **file_list, char * path, bool rel_path_search, char
     rel_path = force_agg_path ? force_agg_path : cwd;
 
     /* Read in reverse order, so we can preserve the original ordering */
-    for (i = 0 ; i < count; ++i) {
+    for (i = 0; i < count; ++i) {
         char *msg_path = path;
         if (opal_path_is_absolute(files[i])) {
             /* Absolute paths preserved */
@@ -1233,12 +1202,12 @@ static int fixup_files(char **file_list, char * path, bool rel_path_search, char
              *    - if found and readable, use it
              *    - otherwise, warn/error
              */
-            tmp_file = opal_path_find (files[i], search_path, mode, NULL);
+            tmp_file = opal_path_find(files[i], search_path, mode, NULL);
         }
 
         if (NULL == tmp_file) {
-            opal_show_help("help-mca-var.txt", "missing-param-file",
-                           true, getpid(), files[i], msg_path);
+            opal_show_help("help-mca-var.txt", "missing-param-file", true, getpid(), files[i],
+                           msg_path);
             exit_status = OPAL_ERROR;
             break;
         }
@@ -1254,17 +1223,17 @@ static int fixup_files(char **file_list, char * path, bool rel_path_search, char
         *file_list = opal_argv_join(argv, sep);
     }
 
-    if( NULL != files ) {
+    if (NULL != files) {
         opal_argv_free(files);
         files = NULL;
     }
 
-    if( NULL != argv ) {
+    if (NULL != argv) {
         opal_argv_free(argv);
         argv = NULL;
     }
 
-    if( NULL != search_path ) {
+    if (NULL != search_path) {
         opal_argv_free(search_path);
         search_path = NULL;
     }
@@ -1288,11 +1257,11 @@ static int read_files(char *file_list, opal_list_t *file_values, char sep)
        the entries farthest to the left get precedence) */
 
     for (i = count - 1; i >= 0; --i) {
-        char *file_name = append_filename_to_list (tmp[i]);
+        char *file_name = append_filename_to_list(tmp[i]);
         mca_base_parse_paramfile(file_name, file_values);
     }
 
-    opal_argv_free (tmp);
+    opal_argv_free(tmp);
 
     mca_base_internal_env_store();
 
@@ -1300,36 +1269,32 @@ static int read_files(char *file_list, opal_list_t *file_values, char sep)
 }
 
 /******************************************************************************/
-static int register_variable (const char *project_name, const char *framework_name,
-                              const char *component_name, const char *variable_name,
-                              const char *description, mca_base_var_type_t type,
-                              mca_base_var_enum_t *enumerator, int bind,
-                              mca_base_var_flag_t flags, mca_base_var_info_lvl_t info_lvl,
-                              mca_base_var_scope_t scope, int synonym_for,
-                              void *storage)
+static int register_variable(const char *project_name, const char *framework_name,
+                             const char *component_name, const char *variable_name,
+                             const char *description, mca_base_var_type_t type,
+                             mca_base_var_enum_t *enumerator, int bind, mca_base_var_flag_t flags,
+                             mca_base_var_info_lvl_t info_lvl, mca_base_var_scope_t scope,
+                             int synonym_for, void *storage)
 {
     int ret, var_index, group_index, tmp;
     mca_base_var_group_t *group;
     mca_base_var_t *var, *original = NULL;
 
     /* Developer error. Storage can not be NULL and type must exist */
-    assert (((flags & MCA_BASE_VAR_FLAG_SYNONYM) || NULL != storage) && type >= 0 && type < MCA_BASE_VAR_TYPE_MAX);
+    assert(((flags & MCA_BASE_VAR_FLAG_SYNONYM) || NULL != storage) && type >= 0
+           && type < MCA_BASE_VAR_TYPE_MAX);
 
     /* Developer error: check max length of strings */
-    if (NULL != project_name &&
-        strlen(project_name) > MCA_BASE_MAX_PROJECT_NAME_LEN) {
+    if (NULL != project_name && strlen(project_name) > MCA_BASE_MAX_PROJECT_NAME_LEN) {
         return OPAL_ERR_BAD_PARAM;
     }
-    if (NULL != framework_name &&
-        strlen(framework_name) > MCA_BASE_MAX_TYPE_NAME_LEN) {
+    if (NULL != framework_name && strlen(framework_name) > MCA_BASE_MAX_TYPE_NAME_LEN) {
         return OPAL_ERR_BAD_PARAM;
     }
-    if (NULL != component_name &&
-        strlen(component_name) > MCA_BASE_MAX_COMPONENT_NAME_LEN) {
+    if (NULL != component_name && strlen(component_name) > MCA_BASE_MAX_COMPONENT_NAME_LEN) {
         return OPAL_ERR_BAD_PARAM;
     }
-    if (NULL != variable_name &&
-        strlen(variable_name) > MCA_BASE_MAX_VARIABLE_NAME_LEN) {
+    if (NULL != variable_name && strlen(variable_name) > MCA_BASE_MAX_VARIABLE_NAME_LEN) {
         return OPAL_ERR_BAD_PARAM;
     }
 
@@ -1384,11 +1349,11 @@ static int register_variable (const char *project_name, const char *framework_na
 #endif
 
     if (flags & MCA_BASE_VAR_FLAG_SYNONYM) {
-        original = opal_pointer_array_get_item (&mca_base_vars, synonym_for);
+        original = opal_pointer_array_get_item(&mca_base_vars, synonym_for);
         if (NULL == original) {
             /* Attempting to create a synonym for a non-existent variable. probably a
              * developer error. */
-            assert (NULL != original);
+            assert(NULL != original);
             return OPAL_ERR_NOT_FOUND;
         }
     }
@@ -1402,13 +1367,12 @@ static int register_variable (const char *project_name, const char *framework_na
     }
 
     /* See if this entry is already in the array */
-    var_index = var_find (project_name, framework_name, component_name, variable_name,
-                          true);
+    var_index = var_find(project_name, framework_name, component_name, variable_name, true);
 
     if (0 > var_index) {
         /* Create a new parameter entry */
-        group_index = mca_base_var_group_register (project_name, framework_name, component_name,
-                                                   NULL);
+        group_index = mca_base_var_group_register(project_name, framework_name, component_name,
+                                                  NULL);
         if (-1 > group_index) {
             return group_index;
         }
@@ -1416,8 +1380,8 @@ static int register_variable (const char *project_name, const char *framework_na
         /* Read-only and constant variables can't be settable */
         if (scope < MCA_BASE_VAR_SCOPE_LOCAL || (flags & MCA_BASE_VAR_FLAG_DEFAULT_ONLY)) {
             if ((flags & MCA_BASE_VAR_FLAG_DEFAULT_ONLY) && (flags & MCA_BASE_VAR_FLAG_SETTABLE)) {
-                opal_show_help("help-mca-var.txt", "invalid-flag-combination",
-                               true, "MCA_BASE_VAR_FLAG_DEFAULT_ONLY", "MCA_BASE_VAR_FLAG_SETTABLE");
+                opal_show_help("help-mca-var.txt", "invalid-flag-combination", true,
+                               "MCA_BASE_VAR_FLAG_DEFAULT_ONLY", "MCA_BASE_VAR_FLAG_SETTABLE");
                 return OPAL_ERROR;
             }
 
@@ -1427,20 +1391,20 @@ static int register_variable (const char *project_name, const char *framework_na
 
         var = OBJ_NEW(mca_base_var_t);
 
-        var->mbv_type        = type;
-        var->mbv_flags       = flags;
+        var->mbv_type = type;
+        var->mbv_flags = flags;
         var->mbv_group_index = group_index;
-        var->mbv_info_lvl    = info_lvl;
-        var->mbv_scope       = scope;
+        var->mbv_info_lvl = info_lvl;
+        var->mbv_scope = scope;
         var->mbv_synonym_for = synonym_for;
-        var->mbv_bind        = bind;
+        var->mbv_bind = bind;
 
         if (NULL != description) {
             var->mbv_description = strdup(description);
         }
 
-        ret = mca_base_var_generate_full_name4 (project_name, framework_name, component_name,
-                                                variable_name, &var->mbv_long_name);
+        ret = mca_base_var_generate_full_name4(project_name, framework_name, component_name,
+                                               variable_name, &var->mbv_long_name);
         if (OPAL_SUCCESS != ret) {
             OBJ_RELEASE(var);
             return OPAL_ERROR;
@@ -1449,19 +1413,20 @@ static int register_variable (const char *project_name, const char *framework_na
          * so instead of allocating them we can just point into the var mbv_long_name
          * at the right location.
          */
-        var->mbv_full_name = var->mbv_long_name +
-                             (NULL == project_name ? 0 : (strlen(project_name)+1)); /* 1 for _ */
-        if( NULL != variable_name ) {
-            var->mbv_variable_name = var->mbv_full_name +
-                                     (NULL == framework_name ? 0 : (strlen(framework_name)+1)) +
-                                     (NULL == component_name ? 0 : (strlen(component_name)+1));
+        var->mbv_full_name = var->mbv_long_name
+                             + (NULL == project_name ? 0
+                                                     : (strlen(project_name) + 1)); /* 1 for _ */
+        if (NULL != variable_name) {
+            var->mbv_variable_name = var->mbv_full_name
+                                     + (NULL == framework_name ? 0 : (strlen(framework_name) + 1))
+                                     + (NULL == component_name ? 0 : (strlen(component_name) + 1));
         }
 
         /* Add it to the array.  Note that we copy the mca_var_t by value,
            so the entire contents of the struct is copied.  The synonym list
            will always be empty at this point, so there's no need for an
            extra RETAIN or RELEASE. */
-        var_index = opal_pointer_array_add (&mca_base_vars, var);
+        var_index = opal_pointer_array_add(&mca_base_vars, var);
         if (0 > var_index) {
             OBJ_RELEASE(var);
             return OPAL_ERROR;
@@ -1470,25 +1435,26 @@ static int register_variable (const char *project_name, const char *framework_na
         var->mbv_index = var_index;
 
         if (0 <= group_index) {
-            mca_base_var_group_add_var (group_index, var_index);
+            mca_base_var_group_add_var(group_index, var_index);
         }
 
         mca_base_var_count++;
-        if (0 <= var_find_by_name (var->mbv_full_name, &tmp, 0)) {
-            /* XXX --- FIXME: variable overshadows an existing variable. this is difficult to support */
-            assert (0);
+        if (0 <= var_find_by_name(var->mbv_full_name, &tmp, 0)) {
+            /* XXX --- FIXME: variable overshadows an existing variable. this is difficult to
+             * support */
+            assert(0);
         }
 
-        opal_hash_table_set_value_ptr (&mca_base_var_index_hash, var->mbv_full_name, strlen (var->mbv_full_name),
-                                       (void *)(uintptr_t) var_index);
+        opal_hash_table_set_value_ptr(&mca_base_var_index_hash, var->mbv_full_name,
+                                      strlen(var->mbv_full_name), (void *) (uintptr_t) var_index);
     } else {
-        ret = var_get (var_index, &var, false);
+        ret = var_get(var_index, &var, false);
         if (OPAL_SUCCESS != ret) {
             /* Shouldn't ever happen */
             return OPAL_ERROR;
         }
 
-        ret = mca_base_var_group_get_internal (var->mbv_group_index, &group, true);
+        ret = mca_base_var_group_get_internal(var->mbv_group_index, &group, true);
         if (OPAL_SUCCESS != ret) {
             /* Shouldn't ever happen */
             return OPAL_ERROR;
@@ -1499,24 +1465,21 @@ static int register_variable (const char *project_name, const char *framework_na
         }
 
         /* Verify the name components match */
-        if (0 != compare_strings(framework_name, group->group_framework) ||
-            0 != compare_strings(component_name, group->group_component) ||
-            0 != compare_strings(variable_name, var->mbv_variable_name)) {
-            opal_show_help("help-mca-var.txt", "var-name-conflict",
-                           true, var->mbv_full_name, framework_name,
-                           component_name, variable_name,
-                           group->group_framework, group->group_component,
-                           var->mbv_variable_name);
+        if (0 != compare_strings(framework_name, group->group_framework)
+            || 0 != compare_strings(component_name, group->group_component)
+            || 0 != compare_strings(variable_name, var->mbv_variable_name)) {
+            opal_show_help("help-mca-var.txt", "var-name-conflict", true, var->mbv_full_name,
+                           framework_name, component_name, variable_name, group->group_framework,
+                           group->group_component, var->mbv_variable_name);
             /* This is developer error. abort! */
-            assert (0);
+            assert(0);
             return OPAL_ERROR;
         }
 
         if (var->mbv_type != type) {
 #if OPAL_ENABLE_DEBUG
-            opal_show_help("help-mca-var.txt",
-                           "re-register-with-different-type",
-                           true, var->mbv_full_name);
+            opal_show_help("help-mca-var.txt", "re-register-with-different-type", true,
+                           var->mbv_full_name);
 #endif
             return OPAL_ERR_VALUE_OUT_OF_BOUNDS;
         }
@@ -1537,8 +1500,9 @@ static int register_variable (const char *project_name, const char *framework_na
         var->mbv_storage = storage;
 
         /* make a copy of the default string value */
-        if ((MCA_BASE_VAR_TYPE_STRING == type || MCA_BASE_VAR_TYPE_VERSION_STRING == type) && NULL != ((char **)storage)[0]) {
-            ((char **)storage)[0] = strdup (((char **)storage)[0]);
+        if ((MCA_BASE_VAR_TYPE_STRING == type || MCA_BASE_VAR_TYPE_VERSION_STRING == type)
+            && NULL != ((char **) storage)[0]) {
+            ((char **) storage)[0] = strdup(((char **) storage)[0]);
         }
     } else {
         /* synonym variable */
@@ -1548,7 +1512,7 @@ static int register_variable (const char *project_name, const char *framework_na
     /* go ahead and mark this variable as valid */
     var->mbv_flags |= MCA_BASE_VAR_FLAG_VALID;
 
-    ret = var_set_initial (var, original);
+    ret = var_set_initial(var, original);
     if (OPAL_SUCCESS != ret) {
         return ret;
     }
@@ -1557,81 +1521,78 @@ static int register_variable (const char *project_name, const char *framework_na
     return var_index;
 }
 
-int mca_base_var_register (const char *project_name, const char *framework_name,
-                           const char *component_name, const char *variable_name,
-                           const char *description, mca_base_var_type_t type,
-                           mca_base_var_enum_t *enumerator, int bind,
-                           mca_base_var_flag_t flags,
-                           mca_base_var_info_lvl_t info_lvl,
-                           mca_base_var_scope_t scope, void *storage)
+int mca_base_var_register(const char *project_name, const char *framework_name,
+                          const char *component_name, const char *variable_name,
+                          const char *description, mca_base_var_type_t type,
+                          mca_base_var_enum_t *enumerator, int bind, mca_base_var_flag_t flags,
+                          mca_base_var_info_lvl_t info_lvl, mca_base_var_scope_t scope,
+                          void *storage)
 {
     int ret;
     /* Only integer variables can have enumerator */
-    assert (NULL == enumerator || (MCA_BASE_VAR_TYPE_INT == type || MCA_BASE_VAR_TYPE_UNSIGNED_INT == type));
+    assert(NULL == enumerator
+           || (MCA_BASE_VAR_TYPE_INT == type || MCA_BASE_VAR_TYPE_UNSIGNED_INT == type));
 
-    ret = register_variable (project_name, framework_name, component_name,
-                             variable_name, description, type, enumerator,
-                             bind, flags, info_lvl, scope, -1, storage);
+    ret = register_variable(project_name, framework_name, component_name, variable_name,
+                            description, type, enumerator, bind, flags, info_lvl, scope, -1,
+                            storage);
     if (OPAL_UNLIKELY(0 > ret)) {
         return ret;
     }
 
     /* Register aliases if any exist */
-    const mca_base_alias_t *alias = mca_base_alias_lookup (project_name, framework_name, component_name);
+    const mca_base_alias_t *alias = mca_base_alias_lookup(project_name, framework_name,
+                                                          component_name);
     if (NULL == alias) {
         return ret;
     }
 
     OPAL_LIST_FOREACH_DECL(alias_item, &alias->component_aliases, mca_base_alias_item_t) {
-        mca_base_var_syn_flag_t flags = 0;
+        mca_base_var_syn_flag_t flags_derived = 0;
         if (alias_item->alias_flags & MCA_BASE_ALIAS_FLAG_DEPRECATED) {
-            flags = MCA_BASE_VAR_SYN_FLAG_DEPRECATED;
+            flags_derived = MCA_BASE_VAR_SYN_FLAG_DEPRECATED;
         }
-        (void) mca_base_var_register_synonym (ret, project_name, framework_name, alias_item->component_alias,
-                                              variable_name, flags);
+        (void) mca_base_var_register_synonym(ret, project_name, framework_name,
+                                             alias_item->component_alias, variable_name, flags_derived);
     }
 
     return ret;
 }
 
-int mca_base_component_var_register (const mca_base_component_t *component,
-                                     const char *variable_name, const char *description,
-                                     mca_base_var_type_t type, mca_base_var_enum_t *enumerator,
-                                     int bind, mca_base_var_flag_t flags,
-                                     mca_base_var_info_lvl_t info_lvl,
-                                     mca_base_var_scope_t scope, void *storage)
+int mca_base_component_var_register(const mca_base_component_t *component,
+                                    const char *variable_name, const char *description,
+                                    mca_base_var_type_t type, mca_base_var_enum_t *enumerator,
+                                    int bind, mca_base_var_flag_t flags,
+                                    mca_base_var_info_lvl_t info_lvl, mca_base_var_scope_t scope,
+                                    void *storage)
 {
-    return mca_base_var_register (component->mca_project_name, component->mca_type_name,
-                                  component->mca_component_name,
-                                  variable_name, description, type, enumerator,
-                                  bind, flags | MCA_BASE_VAR_FLAG_DWG,
-                                  info_lvl, scope, storage);
+    return mca_base_var_register(component->mca_project_name, component->mca_type_name,
+                                 component->mca_component_name, variable_name, description, type,
+                                 enumerator, bind, flags | MCA_BASE_VAR_FLAG_DWG, info_lvl, scope,
+                                 storage);
 }
 
-int mca_base_framework_var_register (const mca_base_framework_t *framework,
-                                     const char *variable_name,
-                                     const char *help_msg, mca_base_var_type_t type,
-                                     mca_base_var_enum_t *enumerator, int bind,
-                                     mca_base_var_flag_t flags,
-                                     mca_base_var_info_lvl_t info_level,
-                                     mca_base_var_scope_t scope, void *storage)
+int mca_base_framework_var_register(const mca_base_framework_t *framework,
+                                    const char *variable_name, const char *help_msg,
+                                    mca_base_var_type_t type, mca_base_var_enum_t *enumerator,
+                                    int bind, mca_base_var_flag_t flags,
+                                    mca_base_var_info_lvl_t info_level, mca_base_var_scope_t scope,
+                                    void *storage)
 {
-    return mca_base_var_register (framework->framework_project, framework->framework_name,
-                                  "base", variable_name, help_msg, type, enumerator, bind,
-                                  flags | MCA_BASE_VAR_FLAG_DWG, info_level, scope, storage);
+    return mca_base_var_register(framework->framework_project, framework->framework_name, "base",
+                                 variable_name, help_msg, type, enumerator, bind,
+                                 flags | MCA_BASE_VAR_FLAG_DWG, info_level, scope, storage);
 }
 
-int mca_base_var_register_synonym (int synonym_for, const char *project_name,
-                                   const char *framework_name,
-                                   const char *component_name,
-                                   const char *synonym_name,
-                                   mca_base_var_syn_flag_t flags)
+int mca_base_var_register_synonym(int synonym_for, const char *project_name,
+                                  const char *framework_name, const char *component_name,
+                                  const char *synonym_name, mca_base_var_syn_flag_t flags)
 {
     mca_base_var_flag_t var_flags = (mca_base_var_flag_t) MCA_BASE_VAR_FLAG_SYNONYM;
     mca_base_var_t *var;
     int ret;
 
-    ret = var_get (synonym_for, &var, false);
+    ret = var_get(synonym_for, &var, false);
     if (OPAL_SUCCESS != ret || VAR_IS_SYNONYM(var[0])) {
         return OPAL_ERR_BAD_PARAM;
     }
@@ -1643,17 +1604,16 @@ int mca_base_var_register_synonym (int synonym_for, const char *project_name,
         var_flags |= MCA_BASE_VAR_FLAG_INTERNAL;
     }
 
-    return register_variable (project_name, framework_name, component_name,
-                              synonym_name, var->mbv_description, var->mbv_type, var->mbv_enumerator,
-                              var->mbv_bind, var_flags, var->mbv_info_lvl, var->mbv_scope,
-                              synonym_for, NULL);
+    return register_variable(project_name, framework_name, component_name, synonym_name,
+                             var->mbv_description, var->mbv_type, var->mbv_enumerator,
+                             var->mbv_bind, var_flags, var->mbv_info_lvl, var->mbv_scope,
+                             synonym_for, NULL);
 }
 
-static int var_get_env (mca_base_var_t *var, const char *name, char **source, char **value)
+static int var_get_env(mca_base_var_t *var, const char *name, char **source, char **value)
 {
     const char source_prefix[] = "SOURCE_";
-    const int max_len = strlen(mca_prefix) + strlen(source_prefix) +
-        strlen(name) + 1;
+    const int max_len = strlen(mca_prefix) + strlen(source_prefix) + strlen(name) + 1;
     char *envvar = alloca(max_len);
     if (NULL == envvar) {
         return OPAL_ERR_OUT_OF_RESOURCE;
@@ -1665,14 +1625,13 @@ static int var_get_env (mca_base_var_t *var, const char *name, char **source, ch
         return OPAL_ERROR;
     }
     *value = getenv(envvar);
-    if( NULL == *value ) {
+    if (NULL == *value) {
         *source = NULL;
         return OPAL_ERR_NOT_FOUND;
     }
 
-    ret = snprintf(envvar, max_len, "%s%s%s", mca_prefix,
-                   source_prefix, name);
-    if( 0 > ret ) {
+    ret = snprintf(envvar, max_len, "%s%s%s", mca_prefix, source_prefix, name);
+    if (0 > ret) {
         return OPAL_ERROR;
     }
     *source = getenv(envvar);
@@ -1683,7 +1642,7 @@ static int var_get_env (mca_base_var_t *var, const char *name, char **source, ch
 /*
  * Lookup a param in the environment
  */
-static int var_set_from_env (mca_base_var_t *var, mca_base_var_t *original)
+static int var_set_from_env(mca_base_var_t *var, mca_base_var_t *original)
 {
     const char *var_full_name = var->mbv_full_name;
     const char *var_long_name = var->mbv_long_name;
@@ -1692,9 +1651,9 @@ static int var_set_from_env (mca_base_var_t *var, mca_base_var_t *original)
     char *source_env, *value_env;
     int ret;
 
-    ret = var_get_env (var, var_long_name, &source_env, &value_env);
+    ret = var_get_env(var, var_long_name, &source_env, &value_env);
     if (OPAL_SUCCESS != ret) {
-        ret = var_get_env (var, var_full_name, &source_env, &value_env);
+        ret = var_get_env(var, var_full_name, &source_env, &value_env);
     }
 
     if (OPAL_SUCCESS != ret) {
@@ -1704,16 +1663,14 @@ static int var_set_from_env (mca_base_var_t *var, mca_base_var_t *original)
     /* we found an environment variable but this variable is default-only. print
        a warning. */
     if (VAR_IS_DEFAULT_ONLY(original[0])) {
-        opal_show_help("help-mca-var.txt", "default-only-param-set",
-                       true, var_full_name);
+        opal_show_help("help-mca-var.txt", "default-only-param-set", true, var_full_name);
 
         return OPAL_ERR_NOT_FOUND;
     }
 
     if (MCA_BASE_VAR_SOURCE_OVERRIDE == original->mbv_source) {
         if (!mca_base_var_suppress_override_warning) {
-            opal_show_help("help-mca-var.txt", "overridden-param-set",
-                           true, var_full_name);
+            opal_show_help("help-mca-var.txt", "overridden-param-set", true, var_full_name);
         }
 
         return OPAL_ERR_NOT_FOUND;
@@ -1722,14 +1679,14 @@ static int var_set_from_env (mca_base_var_t *var, mca_base_var_t *original)
     original->mbv_source = MCA_BASE_VAR_SOURCE_ENV;
 
     if (NULL != source_env) {
-        if (0 == strncasecmp (source_env, "file:", 5)) {
+        if (0 == strncasecmp(source_env, "file:", 5)) {
             original->mbv_source_file = append_filename_to_list(source_env + 5);
-            if (0 == strcmp (var->mbv_source_file, mca_base_var_override_file)) {
+            if (0 == strcmp(var->mbv_source_file, mca_base_var_override_file)) {
                 original->mbv_source = MCA_BASE_VAR_SOURCE_OVERRIDE;
             } else {
                 original->mbv_source = MCA_BASE_VAR_SOURCE_FILE;
             }
-        } else if (0 == strcasecmp (source_env, "command")) {
+        } else if (0 == strcasecmp(source_env, "command")) {
             var->mbv_source = MCA_BASE_VAR_SOURCE_COMMAND_LINE;
         }
     }
@@ -1743,18 +1700,17 @@ static int var_set_from_env (mca_base_var_t *var, mca_base_var_t *original)
 
         switch (var->mbv_source) {
         case MCA_BASE_VAR_SOURCE_ENV:
-            opal_show_help("help-mca-var.txt", "deprecated-mca-env",
-                           true, var_full_name, new_variable);
+            opal_show_help("help-mca-var.txt", "deprecated-mca-env", true, var_full_name,
+                           new_variable);
             break;
         case MCA_BASE_VAR_SOURCE_COMMAND_LINE:
-            opal_show_help("help-mca-var.txt", "deprecated-mca-cli",
-                           true, var_full_name, new_variable);
+            opal_show_help("help-mca-var.txt", "deprecated-mca-cli", true, var_full_name,
+                           new_variable);
             break;
         case MCA_BASE_VAR_SOURCE_FILE:
         case MCA_BASE_VAR_SOURCE_OVERRIDE:
-            opal_show_help("help-mca-var.txt", "deprecated-mca-file",
-                           true, var_full_name, mca_base_var_source_file (var),
-                           new_variable);
+            opal_show_help("help-mca-var.txt", "deprecated-mca-file", true, var_full_name,
+                           mca_base_var_source_file(var), new_variable);
             break;
 
         case MCA_BASE_VAR_SOURCE_DEFAULT:
@@ -1765,13 +1721,14 @@ static int var_set_from_env (mca_base_var_t *var, mca_base_var_t *original)
         }
     }
 
-    return var_set_from_string (original, value_env);
+    return var_set_from_string(original, value_env);
 }
 
 /*
  * Lookup a param in the files
  */
-static int var_set_from_file (mca_base_var_t *var, mca_base_var_t *original, opal_list_t *file_values)
+static int var_set_from_file(mca_base_var_t *var, mca_base_var_t *original,
+                             opal_list_t *file_values)
 {
     const char *var_full_name = var->mbv_full_name;
     const char *var_long_name = var->mbv_long_name;
@@ -1783,32 +1740,29 @@ static int var_set_from_file (mca_base_var_t *var, mca_base_var_t *original, opa
        find a match.  If we do, cache it on the param (for future
        lookups) and save it in the storage. */
 
-    OPAL_LIST_FOREACH(fv, file_values, mca_base_var_file_value_t) {
-        if (0 != strcmp(fv->mbvfv_var, var_full_name) &&
-            0 != strcmp(fv->mbvfv_var, var_long_name)) {
+    OPAL_LIST_FOREACH (fv, file_values, mca_base_var_file_value_t) {
+        if (0 != strcmp(fv->mbvfv_var, var_full_name)
+            && 0 != strcmp(fv->mbvfv_var, var_long_name)) {
             continue;
         }
 
         /* found it */
         if (VAR_IS_DEFAULT_ONLY(var[0])) {
-            opal_show_help("help-mca-var.txt", "default-only-param-set",
-                           true, var_full_name);
+            opal_show_help("help-mca-var.txt", "default-only-param-set", true, var_full_name);
 
             return OPAL_ERR_NOT_FOUND;
         }
 
         if (MCA_BASE_VAR_FLAG_ENVIRONMENT_ONLY & original->mbv_flags) {
-            opal_show_help("help-mca-var.txt", "environment-only-param",
-                           true, var_full_name, fv->mbvfv_value,
-                           fv->mbvfv_file);
+            opal_show_help("help-mca-var.txt", "environment-only-param", true, var_full_name,
+                           fv->mbvfv_value, fv->mbvfv_file);
 
             return OPAL_ERR_NOT_FOUND;
         }
 
         if (MCA_BASE_VAR_SOURCE_OVERRIDE == original->mbv_source) {
             if (!mca_base_var_suppress_override_warning) {
-                opal_show_help("help-mca-var.txt", "overridden-param-set",
-                               true, var_full_name);
+                opal_show_help("help-mca-var.txt", "overridden-param-set", true, var_full_name);
             }
 
             return OPAL_ERR_NOT_FOUND;
@@ -1821,9 +1775,8 @@ static int var_set_from_file (mca_base_var_t *var, mca_base_var_t *original, opa
                 new_variable = original->mbv_full_name;
             }
 
-            opal_show_help("help-mca-var.txt", "deprecated-mca-file",
-                           true, var_full_name, fv->mbvfv_file,
-                           new_variable);
+            opal_show_help("help-mca-var.txt", "deprecated-mca-file", true, var_full_name,
+                           fv->mbvfv_file, new_variable);
         }
 
         original->mbv_file_value = (void *) fv;
@@ -1833,7 +1786,7 @@ static int var_set_from_file (mca_base_var_t *var, mca_base_var_t *original, opa
             var->mbv_source = MCA_BASE_VAR_SOURCE_FILE;
         }
 
-        return var_set_from_string (original, fv->mbvfv_value);
+        return var_set_from_string(original, fv->mbvfv_value);
     }
 
     return OPAL_ERR_NOT_FOUND;
@@ -1842,7 +1795,7 @@ static int var_set_from_file (mca_base_var_t *var, mca_base_var_t *original, opa
 /*
  * Lookup the initial value for a parameter
  */
-static int var_set_initial (mca_base_var_t *var, mca_base_var_t *original)
+static int var_set_initial(mca_base_var_t *var, mca_base_var_t *original)
 {
     int ret;
 
@@ -1860,23 +1813,24 @@ static int var_set_initial (mca_base_var_t *var, mca_base_var_t *original)
        order. If the default only flag is set the user will get a
        warning if they try to set a value from the environment or a
        file. */
-    ret = var_set_from_file (var, original, &mca_base_var_override_values);
+    ret = var_set_from_file(var, original, &mca_base_var_override_values);
     if (OPAL_SUCCESS == ret) {
-        var->mbv_flags = ~MCA_BASE_VAR_FLAG_SETTABLE & (var->mbv_flags | MCA_BASE_VAR_FLAG_OVERRIDE);
+        var->mbv_flags = ~MCA_BASE_VAR_FLAG_SETTABLE
+                         & (var->mbv_flags | MCA_BASE_VAR_FLAG_OVERRIDE);
         var->mbv_source = MCA_BASE_VAR_SOURCE_OVERRIDE;
     }
 
-    ret = var_set_from_env (var, original);
+    ret = var_set_from_env(var, original);
     if (OPAL_ERR_NOT_FOUND != ret) {
         return ret;
     }
 
-    ret = var_set_from_file (var, original, &mca_base_envar_file_values);
+    ret = var_set_from_file(var, original, &mca_base_envar_file_values);
     if (OPAL_ERR_NOT_FOUND != ret) {
         return ret;
     }
 
-    ret = var_set_from_file (var, original, &mca_base_var_file_values);
+    ret = var_set_from_file(var, original, &mca_base_var_file_values);
     if (OPAL_ERR_NOT_FOUND != ret) {
         return ret;
     }
@@ -1889,24 +1843,22 @@ static int var_set_initial (mca_base_var_t *var, mca_base_var_t *original)
  */
 static void var_constructor(mca_base_var_t *var)
 {
-    memset ((char *) var + sizeof (var->super), 0, sizeof (*var) - sizeof (var->super));
+    memset((char *) var + sizeof(var->super), 0, sizeof(*var) - sizeof(var->super));
 
     var->mbv_type = MCA_BASE_VAR_TYPE_MAX;
     OBJ_CONSTRUCT(&var->mbv_synonyms, opal_value_array_t);
-    opal_value_array_init (&var->mbv_synonyms, sizeof (int));
+    opal_value_array_init(&var->mbv_synonyms, sizeof(int));
 }
 
-
 /*
  * Free all the contents of a param container
  */
 static void var_destructor(mca_base_var_t *var)
 {
-    if ((MCA_BASE_VAR_TYPE_STRING == var->mbv_type ||
-                MCA_BASE_VAR_TYPE_VERSION_STRING == var->mbv_type) &&
-        NULL != var->mbv_storage &&
-        NULL != var->mbv_storage->stringval) {
-        free (var->mbv_storage->stringval);
+    if ((MCA_BASE_VAR_TYPE_STRING == var->mbv_type
+         || MCA_BASE_VAR_TYPE_VERSION_STRING == var->mbv_type)
+        && NULL != var->mbv_storage && NULL != var->mbv_storage->stringval) {
+        free(var->mbv_storage->stringval);
         var->mbv_storage->stringval = NULL;
     }
 
@@ -1931,17 +1883,15 @@ static void var_destructor(mca_base_var_t *var)
 
 #if OPAL_ENABLE_DEBUG
     /* Cheap trick to reset everything to NULL */
-    memset ((char *) var + sizeof (var->super), 0, sizeof (*var) - sizeof (var->super));
+    memset((char *) var + sizeof(var->super), 0, sizeof(*var) - sizeof(var->super));
 #endif
 }
 
-
 static void fv_constructor(mca_base_var_file_value_t *f)
 {
-    memset ((char *) f + sizeof (f->super), 0, sizeof (*f) - sizeof (f->super));
+    memset((char *) f + sizeof(f->super), 0, sizeof(*f) - sizeof(f->super));
 }
 
-
 static void fv_destructor(mca_base_var_file_value_t *f)
 {
     if (NULL != f->mbvfv_var) {
@@ -1958,7 +1908,8 @@ static char *source_name(mca_base_var_t *var)
 {
     char *ret;
 
-    if (MCA_BASE_VAR_SOURCE_FILE == var->mbv_source || MCA_BASE_VAR_SOURCE_OVERRIDE == var->mbv_source) {
+    if (MCA_BASE_VAR_SOURCE_FILE == var->mbv_source
+        || MCA_BASE_VAR_SOURCE_OVERRIDE == var->mbv_source) {
         struct mca_base_var_file_value_t *fv = var->mbv_file_value;
         int rc;
 
@@ -1968,30 +1919,31 @@ static char *source_name(mca_base_var_t *var)
             rc = opal_asprintf(&ret, "file (%s)", var->mbv_source_file);
         }
 
-        /* some compilers will warn if the return code of opal_asprintf is not checked (even if it is cast to void) */
+        /* some compilers will warn if the return code of opal_asprintf is not checked (even if it
+         * is cast to void) */
         if (0 > rc) {
             return NULL;
         }
         return ret;
     } else if (MCA_BASE_VAR_SOURCE_MAX <= var->mbv_source) {
-        return strdup ("unknown(!!)");
+        return strdup("unknown(!!)");
     }
 
-    return strdup (var_source_names[var->mbv_source]);
+    return strdup(var_source_names[var->mbv_source]);
 }
 
-static int var_value_string (mca_base_var_t *var, char **value_string)
+static int var_value_string(mca_base_var_t *var, char **value_string)
 {
-    const mca_base_var_storage_t *value=NULL;
+    const mca_base_var_storage_t *value = NULL;
     int ret;
 
-    assert (MCA_BASE_VAR_TYPE_MAX > var->mbv_type);
+    assert(MCA_BASE_VAR_TYPE_MAX > var->mbv_type);
 
     /** Parameters with MCA_BASE_VAR_FLAG_DEF_UNSET flag should be shown
      * as "unset" by default. */
-    if ((var->mbv_flags & MCA_BASE_VAR_FLAG_DEF_UNSET) &&
-        (MCA_BASE_VAR_SOURCE_DEFAULT == var->mbv_source)){
-        opal_asprintf (value_string, "%s", "unset");
+    if ((var->mbv_flags & MCA_BASE_VAR_FLAG_DEF_UNSET)
+        && (MCA_BASE_VAR_SOURCE_DEFAULT == var->mbv_source)) {
+        opal_asprintf(value_string, "%s", "unset");
         return OPAL_SUCCESS;
     }
 
@@ -2003,45 +1955,44 @@ static int var_value_string (mca_base_var_t *var, char **value_string)
     if (NULL == var->mbv_enumerator) {
         switch (var->mbv_type) {
         case MCA_BASE_VAR_TYPE_INT:
-            ret = opal_asprintf (value_string, "%d", value->intval);
+            ret = opal_asprintf(value_string, "%d", value->intval);
             break;
         case MCA_BASE_VAR_TYPE_INT32_T:
-            ret = opal_asprintf (value_string, "%" PRId32, value->int32tval);
+            ret = opal_asprintf(value_string, "%" PRId32, value->int32tval);
             break;
         case MCA_BASE_VAR_TYPE_UINT32_T:
-            ret = opal_asprintf (value_string, "%" PRIu32, value->uint32tval);
+            ret = opal_asprintf(value_string, "%" PRIu32, value->uint32tval);
             break;
         case MCA_BASE_VAR_TYPE_INT64_T:
-            ret = opal_asprintf (value_string, "%" PRId64, value->int64tval);
+            ret = opal_asprintf(value_string, "%" PRId64, value->int64tval);
             break;
         case MCA_BASE_VAR_TYPE_UINT64_T:
-            ret = opal_asprintf (value_string, "%" PRIu64, value->uint64tval);
+            ret = opal_asprintf(value_string, "%" PRIu64, value->uint64tval);
             break;
         case MCA_BASE_VAR_TYPE_LONG:
-            ret = opal_asprintf (value_string, "%ld", value->longval);
+            ret = opal_asprintf(value_string, "%ld", value->longval);
             break;
         case MCA_BASE_VAR_TYPE_UNSIGNED_INT:
-            ret = opal_asprintf (value_string, "%u", value->uintval);
+            ret = opal_asprintf(value_string, "%u", value->uintval);
             break;
         case MCA_BASE_VAR_TYPE_UNSIGNED_LONG:
-            ret = opal_asprintf (value_string, "%lu", value->ulval);
+            ret = opal_asprintf(value_string, "%lu", value->ulval);
             break;
         case MCA_BASE_VAR_TYPE_UNSIGNED_LONG_LONG:
-            ret = opal_asprintf (value_string, "%llu", value->ullval);
+            ret = opal_asprintf(value_string, "%llu", value->ullval);
             break;
         case MCA_BASE_VAR_TYPE_SIZE_T:
-            ret = opal_asprintf (value_string, "%" PRIsize_t, value->sizetval);
+            ret = opal_asprintf(value_string, "%" PRIsize_t, value->sizetval);
             break;
         case MCA_BASE_VAR_TYPE_STRING:
         case MCA_BASE_VAR_TYPE_VERSION_STRING:
-            ret = opal_asprintf (value_string, "%s",
-                            value->stringval ? value->stringval : "");
+            ret = opal_asprintf(value_string, "%s", value->stringval ? value->stringval : "");
             break;
         case MCA_BASE_VAR_TYPE_BOOL:
-            ret = opal_asprintf (value_string, "%d", value->boolval);
+            ret = opal_asprintf(value_string, "%d", value->boolval);
             break;
         case MCA_BASE_VAR_TYPE_DOUBLE:
-            ret = opal_asprintf (value_string, "%lf", value->lfval);
+            ret = opal_asprintf(value_string, "%lf", value->lfval);
             break;
         default:
             ret = -1;
@@ -2052,22 +2003,20 @@ static int var_value_string (mca_base_var_t *var, char **value_string)
     } else {
         /* we use an enumerator to handle string->bool and bool->string conversion */
         if (MCA_BASE_VAR_TYPE_BOOL == var->mbv_type) {
-            ret = var->mbv_enumerator->string_from_value(var->mbv_enumerator, value->boolval, value_string);
+            ret = var->mbv_enumerator->string_from_value(var->mbv_enumerator, value->boolval,
+                                                         value_string);
         } else {
-            ret = var->mbv_enumerator->string_from_value(var->mbv_enumerator, value->intval, value_string);
+            ret = var->mbv_enumerator->string_from_value(var->mbv_enumerator, value->intval,
+                                                         value_string);
         }
     }
 
     return ret;
 }
 
-int mca_base_var_check_exclusive (const char *project,
-                                  const char *type_a,
-                                  const char *component_a,
-                                  const char *param_a,
-                                  const char *type_b,
-                                  const char *component_b,
-                                  const char *param_b)
+int mca_base_var_check_exclusive(const char *project, const char *type_a, const char *component_a,
+                                 const char *param_a, const char *type_b, const char *component_b,
+                                 const char *param_b)
 {
     mca_base_var_t *var_a = NULL, *var_b = NULL;
     int var_ai, var_bi;
@@ -2075,20 +2024,20 @@ int mca_base_var_check_exclusive (const char *project,
     /* XXX -- Remove me once the project name is in the componennt */
     project = NULL;
 
-    var_ai = mca_base_var_find (project, type_a, component_a, param_a);
-    var_bi = mca_base_var_find (project, type_b, component_b, param_b);
+    var_ai = mca_base_var_find(project, type_a, component_a, param_a);
+    var_bi = mca_base_var_find(project, type_b, component_b, param_b);
     if (var_bi < 0 || var_ai < 0) {
         return OPAL_ERR_NOT_FOUND;
     }
 
-    (void) var_get (var_ai, &var_a, true);
-    (void) var_get (var_bi, &var_b, true);
+    (void) var_get(var_ai, &var_a, true);
+    (void) var_get(var_bi, &var_b, true);
     if (NULL == var_a || NULL == var_b) {
         return OPAL_ERR_NOT_FOUND;
     }
 
-    if (MCA_BASE_VAR_SOURCE_DEFAULT != var_a->mbv_source &&
-        MCA_BASE_VAR_SOURCE_DEFAULT != var_b->mbv_source) {
+    if (MCA_BASE_VAR_SOURCE_DEFAULT != var_a->mbv_source
+        && MCA_BASE_VAR_SOURCE_DEFAULT != var_b->mbv_source) {
         char *str_a, *str_b;
 
         /* Form cosmetic string names for A */
@@ -2098,11 +2047,8 @@ int mca_base_var_check_exclusive (const char *project,
         str_b = source_name(var_b);
 
         /* Print it all out */
-        opal_show_help("help-mca-var.txt",
-                       "mutually-exclusive-vars",
-                       true, var_a->mbv_full_name,
-                       str_a, var_b->mbv_full_name,
-                       str_b);
+        opal_show_help("help-mca-var.txt", "mutually-exclusive-vars", true, var_a->mbv_full_name,
+                       str_a, var_b->mbv_full_name, str_b);
 
         /* Free the temp strings */
         free(str_a);
@@ -2114,7 +2060,7 @@ int mca_base_var_check_exclusive (const char *project,
     return OPAL_SUCCESS;
 }
 
-int mca_base_var_get_count (void)
+int mca_base_var_get_count(void)
 {
     return mca_base_var_count;
 }
@@ -2125,7 +2071,7 @@ int mca_base_var_dump(int vari, char ***out, mca_base_var_dump_type_t output_typ
     int i, line_count, line = 0, enum_count = 0;
     char *value_string, *source_string, *tmp;
     int synonym_count, ret, *synonyms = NULL;
-    mca_base_var_t *var, *original=NULL;
+    mca_base_var_t *var, *original = NULL;
     mca_base_var_group_t *group;
 
     ret = var_get(vari, &var, false);
@@ -2158,14 +2104,14 @@ int mca_base_var_dump(int vari, char ***out, mca_base_var_dump_type_t output_typ
         synonyms = OPAL_VALUE_ARRAY_GET_BASE(&var->mbv_synonyms, int);
     }
 
-    ret = var_value_string (var, &value_string);
+    ret = var_value_string(var, &value_string);
     if (OPAL_SUCCESS != ret) {
         return ret;
     }
 
     source_string = source_name(var);
     if (NULL == source_string) {
-        free (value_string);
+        free(value_string);
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
@@ -2174,19 +2120,18 @@ int mca_base_var_dump(int vari, char ***out, mca_base_var_dump_type_t output_typ
             (void) var->mbv_enumerator->get_count(var->mbv_enumerator, &enum_count);
         }
 
-        line_count = 8 + (var->mbv_description ? 1 : 0) + (VAR_IS_SYNONYM(var[0]) ? 1 : synonym_count) +
-            enum_count;
+        line_count = 8 + (var->mbv_description ? 1 : 0)
+                     + (VAR_IS_SYNONYM(var[0]) ? 1 : synonym_count) + enum_count;
 
-        *out = (char **) calloc (line_count + 1, sizeof (char *));
+        *out = (char **) calloc(line_count + 1, sizeof(char *));
         if (NULL == *out) {
-            free (value_string);
-            free (source_string);
+            free(value_string);
+            free(source_string);
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
 
         /* build the message*/
-        opal_asprintf(&tmp, "mca:%s:%s:param:%s:", framework, component,
-                 full_name);
+        opal_asprintf(&tmp, "mca:%s:%s:param:%s:", framework, component, full_name);
 
         /* Output the value */
         char *colon = strchr(value_string, ':');
@@ -2203,7 +2148,7 @@ int mca_base_var_dump(int vari, char ***out, mca_base_var_dump_type_t output_typ
         opal_asprintf(out[0] + line++, "%sstatus:%s", tmp,
                       VAR_IS_SETTABLE(var[0]) ? "writeable" : "read-only");
 
-        /* Output the info level of this parametere */
+        /* Output the info level of this parameter */
         opal_asprintf(out[0] + line++, "%slevel:%d", tmp, var->mbv_info_lvl + 1);
 
         /* If it has a help message, output the help message */
@@ -2212,7 +2157,7 @@ int mca_base_var_dump(int vari, char ***out, mca_base_var_dump_type_t output_typ
         }
 
         if (NULL != var->mbv_enumerator) {
-            for (i = 0 ; i < enum_count ; ++i) {
+            for (i = 0; i < enum_count; ++i) {
                 const char *enum_string = NULL;
                 int enum_value;
 
@@ -2222,12 +2167,14 @@ int mca_base_var_dump(int vari, char ***out, mca_base_var_dump_type_t output_typ
                     continue;
                 }
 
-                opal_asprintf(out[0] + line++, "%senumerator:value:%d:%s", tmp, enum_value, enum_string);
+                opal_asprintf(out[0] + line++, "%senumerator:value:%d:%s", tmp, enum_value,
+                              enum_string);
             }
         }
 
         /* Is this variable deprecated? */
-        opal_asprintf(out[0] + line++, "%sdeprecated:%s", tmp, VAR_IS_DEPRECATED(var[0]) ? "yes" : "no");
+        opal_asprintf(out[0] + line++, "%sdeprecated:%s", tmp,
+                      VAR_IS_DEPRECATED(var[0]) ? "yes" : "no");
 
         opal_asprintf(out[0] + line++, "%stype:%s", tmp, ompi_var_type_names[var->mbv_type]);
 
@@ -2235,7 +2182,7 @@ int mca_base_var_dump(int vari, char ***out, mca_base_var_dump_type_t output_typ
         if (VAR_IS_SYNONYM(var[0])) {
             opal_asprintf(out[0] + line++, "%ssynonym_of:name:%s", tmp, original->mbv_full_name);
         } else if (opal_value_array_get_size(&var->mbv_synonyms)) {
-            for (i = 0 ; i < synonym_count ; ++i) {
+            for (i = 0; i < synonym_count; ++i) {
                 mca_base_var_t *synonym;
 
                 ret = var_get(synonyms[i], &synonym, false);
@@ -2247,37 +2194,50 @@ int mca_base_var_dump(int vari, char ***out, mca_base_var_dump_type_t output_typ
             }
         }
 
-        free (tmp);
-    } else if (MCA_BASE_VAR_DUMP_READABLE == output_type) {
+        free(tmp);
+    } else if (MCA_BASE_VAR_DUMP_READABLE == output_type
+        || MCA_BASE_VAR_DUMP_READABLE_COLOR == output_type) {
+
+        char *color_name = "", *color_value = "", *color_reset = "";
+
         /* There will be at most three lines in the pretty print case */
-        *out = (char **) calloc (4, sizeof (char *));
+        *out = (char **) calloc(4, sizeof(char *));
         if (NULL == *out) {
-            free (value_string);
-            free (source_string);
+            free(value_string);
+            free(source_string);
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
 
-        opal_asprintf (out[0], "%s \"%s\" (current value: \"%s\", data source: %s, level: %d %s, type: %s",
-                  VAR_IS_DEFAULT_ONLY(var[0]) ? "informational" : "parameter",
-                  full_name, value_string, source_string, var->mbv_info_lvl + 1,
-                  info_lvl_strings[var->mbv_info_lvl], ompi_var_type_names[var->mbv_type]);
+        if (MCA_BASE_VAR_DUMP_READABLE_COLOR == output_type) {
+            color_name = opal_var_dump_color[OPAL_VAR_DUMP_COLOR_VAR_NAME];
+            color_value = opal_var_dump_color[OPAL_VAR_DUMP_COLOR_VAR_VALUE];
+            color_reset = "\033[0m";
+        }
+
+        opal_asprintf(out[0],
+                      "%s %s\"%s\"%s (current value: %s\"%s\"%s, data source: %s, level: %d %s, type: %s",
+                      VAR_IS_DEFAULT_ONLY(var[0]) ? "informational" : "parameter",
+                      color_name, full_name, color_reset,
+                      color_value, value_string, color_reset,
+                      source_string, var->mbv_info_lvl + 1,
+                      info_lvl_strings[var->mbv_info_lvl], ompi_var_type_names[var->mbv_type]);
 
         tmp = out[0][0];
         if (VAR_IS_DEPRECATED(var[0])) {
-            opal_asprintf (out[0], "%s, deprecated", tmp);
-            free (tmp);
+            opal_asprintf(out[0], "%s, deprecated", tmp);
+            free(tmp);
             tmp = out[0][0];
         }
 
         /* Does this parameter have any synonyms or is it a synonym? */
         if (VAR_IS_SYNONYM(var[0])) {
             opal_asprintf(out[0], "%s, synonym of: %s)", tmp, original->mbv_full_name);
-            free (tmp);
+            free(tmp);
         } else if (synonym_count) {
             opal_asprintf(out[0], "%s, synonyms: ", tmp);
-            free (tmp);
+            free(tmp);
 
-            for (i = 0 ; i < synonym_count ; ++i) {
+            for (i = 0; i < synonym_count; ++i) {
                 mca_base_var_t *synonym;
 
                 ret = var_get(synonyms[i], &synonym, false);
@@ -2286,7 +2246,7 @@ int mca_base_var_dump(int vari, char ***out, mca_base_var_dump_type_t output_typ
                 }
 
                 tmp = out[0][0];
-                if (synonym_count == i+1) {
+                if (synonym_count == i + 1) {
                     opal_asprintf(out[0], "%s%s)", tmp, synonym->mbv_full_name);
                 } else {
                     opal_asprintf(out[0], "%s%s, ", tmp, synonym->mbv_full_name);
@@ -2307,26 +2267,26 @@ int mca_base_var_dump(int vari, char ***out, mca_base_var_dump_type_t output_typ
         if (NULL != var->mbv_enumerator) {
             char *values;
 
-            ret = var->mbv_enumerator->dump(var->mbv_enumerator, &values);
+            ret = var->mbv_enumerator->dump(var->mbv_enumerator, &values,
+                MCA_BASE_VAR_DUMP_TYPE_TO_ENUM_DUMP_TYPE(output_type));
             if (OPAL_SUCCESS == ret) {
-                opal_asprintf (out[0] + line++, "Valid values: %s", values);
-                free (values);
+                opal_asprintf(out[0] + line++, "Valid values: %s", values);
+                free(values);
             }
         }
     } else if (MCA_BASE_VAR_DUMP_SIMPLE == output_type) {
-        *out = (char **) calloc (2, sizeof (char *));
+        *out = (char **) calloc(2, sizeof(char *));
         if (NULL == *out) {
-            free (value_string);
-            free (source_string);
+            free(value_string);
+            free(source_string);
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
 
         opal_asprintf(out[0], "%s=%s (%s)", var->mbv_full_name, value_string, source_string);
     }
 
-    free (value_string);
-    free (source_string);
+    free(value_string);
+    free(source_string);
 
     return OPAL_SUCCESS;
 }
-
diff --git a/opal/mca/base/mca_base_var.h b/opal/mca/base/mca_base_var.h
index 5719266cfd2..5f6e4ced3f1 100644
--- a/opal/mca/base/mca_base_var.h
+++ b/opal/mca/base/mca_base_var.h
@@ -17,6 +17,8 @@
  * Copyright (c) 2017      IBM Corporation. All rights reserved.
  * Copyright (c) 2018      Triad National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      Computer Architecture and VLSI Systems (CARV)
+ *                         Laboratory, ICS Forth. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -55,7 +57,7 @@
  * - If nothing else was found, use the variable's default value.
  *
  * Note that there is a second header file (mca_base_vari.h)
- * that contains several internal type delcarations for the variable
+ * that contains several internal type declarations for the variable
  * system.  The internal file is only used within the variable system
  * itself; it should not be required by any other Open MPI entities.
  */
@@ -67,9 +69,9 @@
 
 #include "opal/class/opal_list.h"
 #include "opal/class/opal_value_array.h"
+#include "opal/mca/base/mca_base_framework.h"
 #include "opal/mca/base/mca_base_var_enum.h"
 #include "opal/mca/base/mca_base_var_group.h"
-#include "opal/mca/base/mca_base_framework.h"
 #include "opal/mca/mca.h"
 
 /**
@@ -136,7 +138,7 @@ typedef enum {
  * MCA variable scopes
  *
  * Equivalent to MPI_T scopes with the same base name (e.g.,
- * MCA_BASE_VAR_SCOPE_CONSTANT corresponts to MPI_T_SCOPE_CONSTANT).
+ * MCA_BASE_VAR_SCOPE_CONSTANT corresponds to MPI_T_SCOPE_CONSTANT).
  */
 typedef enum {
     /** The value of this variable will not change after it is
@@ -180,21 +182,21 @@ typedef enum {
 
 typedef enum {
     MCA_BASE_VAR_SYN_FLAG_DEPRECATED = 0x0001,
-    MCA_BASE_VAR_SYN_FLAG_INTERNAL   = 0x0002
+    MCA_BASE_VAR_SYN_FLAG_INTERNAL = 0x0002
 } mca_base_var_syn_flag_t;
 
 typedef enum {
     /** Variable is internal (hidden from *_info/MPIT) */
-    MCA_BASE_VAR_FLAG_INTERNAL     = 0x0001,
+    MCA_BASE_VAR_FLAG_INTERNAL = 0x0001,
     /** Variable will always be the default value. Implies
         !MCA_BASE_VAR_FLAG_SETTABLE */
     MCA_BASE_VAR_FLAG_DEFAULT_ONLY = 0x0002,
     /** Variable can be set with mca_base_var_set() */
-    MCA_BASE_VAR_FLAG_SETTABLE     = 0x0004,
+    MCA_BASE_VAR_FLAG_SETTABLE = 0x0004,
     /** Variable is deprecated */
-    MCA_BASE_VAR_FLAG_DEPRECATED   = 0x0008,
+    MCA_BASE_VAR_FLAG_DEPRECATED = 0x0008,
     /** Variable has been overridden */
-    MCA_BASE_VAR_FLAG_OVERRIDE     = 0x0010,
+    MCA_BASE_VAR_FLAG_OVERRIDE = 0x0010,
     /** Variable may not be set from a file */
     MCA_BASE_VAR_FLAG_ENVIRONMENT_ONLY = 0x0020,
     /** Variable should be deregistered when the group is deregistered
@@ -204,13 +206,12 @@ typedef enum {
         manually when you register a variable with
         mca_base_var_register().  Analogous to the
         MCA_BASE_PVAR_FLAG_IWG. */
-    MCA_BASE_VAR_FLAG_DWG          = 0x0040,
+    MCA_BASE_VAR_FLAG_DWG = 0x0040,
     /** Variable has a default value of "unset". Meaning to only
      * be set when the user explicitly asks for it */
-    MCA_BASE_VAR_FLAG_DEF_UNSET    = 0x0080,
+    MCA_BASE_VAR_FLAG_DEF_UNSET = 0x0080,
 } mca_base_var_flag_t;
 
-
 /**
  * Types for MCA parameters.
  */
@@ -243,7 +244,6 @@ typedef union {
     double lfval;
 } mca_base_var_storage_t;
 
-
 /**
  * Entry for holding information about an MCA variable.
  */
@@ -263,7 +263,7 @@ struct mca_base_var_t {
     mca_base_var_info_lvl_t mbv_info_lvl;
 
     /** Enum indicating the type of the variable (integer, string, boolean) */
-     mca_base_var_type_t mbv_type;
+    mca_base_var_type_t mbv_type;
 
     /** String of the variable name */
     char *mbv_variable_name;
@@ -282,10 +282,10 @@ struct mca_base_var_t {
     opal_value_array_t mbv_synonyms;
 
     /** Variable flags */
-    mca_base_var_flag_t  mbv_flags;
+    mca_base_var_flag_t mbv_flags;
 
     /** Variable scope */
-    mca_base_var_scope_t  mbv_scope;
+    mca_base_var_scope_t mbv_scope;
 
     /** Source of the current value */
     mca_base_var_source_t mbv_source;
@@ -323,7 +323,7 @@ typedef struct mca_base_var_t mca_base_var_t;
 BEGIN_C_DECLS
 
 /**
- * Object declarayion for mca_base_var_t
+ * Object declaration for mca_base_var_t
  */
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(mca_base_var_t);
 
@@ -332,7 +332,7 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(mca_base_var_t);
  *
  * @retval OPAL_SUCCESS
  *
- * This function initalizes the MCA variable system.  It is
+ * This function initializes the MCA variable system.  It is
  * invoked internally (by mca_base_open()) and is only documented
  * here for completeness.
  */
@@ -427,12 +427,12 @@ OPAL_DECLSPEC int mca_base_var_init(void);
  * the caller may free the original string after this function returns
  * successfully.
  */
-OPAL_DECLSPEC int mca_base_var_register (const char *project_name, const char *framework_name,
-                                         const char *component_name, const char *variable_name,
-                                         const char *description, mca_base_var_type_t type,
-                                         mca_base_var_enum_t *enumerator, int bind, mca_base_var_flag_t flags,
-                                         mca_base_var_info_lvl_t info_lvl,
-                                         mca_base_var_scope_t scope, void *storage);
+OPAL_DECLSPEC int mca_base_var_register(const char *project_name, const char *framework_name,
+                                        const char *component_name, const char *variable_name,
+                                        const char *description, mca_base_var_type_t type,
+                                        mca_base_var_enum_t *enumerator, int bind,
+                                        mca_base_var_flag_t flags, mca_base_var_info_lvl_t info_lvl,
+                                        mca_base_var_scope_t scope, void *storage);
 
 /**
  * Convenience function for registering a variable associated with a
@@ -443,25 +443,20 @@ OPAL_DECLSPEC int mca_base_var_register (const char *project_name, const char *f
  * be unregistered / made unavailable when that component is closed by
  * its framework.
  */
-OPAL_DECLSPEC int mca_base_component_var_register (const mca_base_component_t *component,
-                                                   const char *variable_name, const char *description,
-                                                   mca_base_var_type_t type, mca_base_var_enum_t *enumerator,
-                                                   int bind, mca_base_var_flag_t flags,
-                                                   mca_base_var_info_lvl_t info_lvl,
-                                                   mca_base_var_scope_t scope, void *storage);
+OPAL_DECLSPEC int mca_base_component_var_register(
+    const mca_base_component_t *component, const char *variable_name, const char *description,
+    mca_base_var_type_t type, mca_base_var_enum_t *enumerator, int bind, mca_base_var_flag_t flags,
+    mca_base_var_info_lvl_t info_lvl, mca_base_var_scope_t scope, void *storage);
 
 /**
  * Convenience function for registering a variable associated with a framework. This
  * function is equivalent to mca_base_var_register with component_name = "base" and
  * with the MCA_BASE_VAR_FLAG_DWG set. See mca_base_var_register().
  */
-OPAL_DECLSPEC int mca_base_framework_var_register (const mca_base_framework_t *framework,
-                                     const char *variable_name,
-                                     const char *help_msg, mca_base_var_type_t type,
-                                     mca_base_var_enum_t *enumerator, int bind,
-                                     mca_base_var_flag_t flags,
-                                     mca_base_var_info_lvl_t info_level,
-                                     mca_base_var_scope_t scope, void *storage);
+OPAL_DECLSPEC int mca_base_framework_var_register(
+    const mca_base_framework_t *framework, const char *variable_name, const char *help_msg,
+    mca_base_var_type_t type, mca_base_var_enum_t *enumerator, int bind, mca_base_var_flag_t flags,
+    mca_base_var_info_lvl_t info_level, mca_base_var_scope_t scope, void *storage);
 
 /**
  * Register a synonym name for an MCA variable.
@@ -487,7 +482,7 @@ OPAL_DECLSPEC int mca_base_framework_var_register (const mca_base_framework_t *f
  * creates a new name that by which the same variable value is
  * accessible.
  *
- * Note that the original variable name has precendence over all
+ * Note that the original variable name has precedence over all
  * synonyms.  For example, consider the case if variable is
  * originally registered under the name "A" and is later
  * registered with synonyms "B" and "C".  If the user sets values
@@ -498,11 +493,11 @@ OPAL_DECLSPEC int mca_base_framework_var_register (const mca_base_framework_t *f
  * variable names "B" and "C" (and does *not* set a value for
  * "A"), it is undefined as to which value will be used.
  */
-OPAL_DECLSPEC int mca_base_var_register_synonym (int synonym_for, const char *project_name,
-                                                 const char *framework_name,
-                                                 const char *component_name,
-                                                 const char *synonym_name,
-                                                 mca_base_var_syn_flag_t flags);
+OPAL_DECLSPEC int mca_base_var_register_synonym(int synonym_for, const char *project_name,
+                                                const char *framework_name,
+                                                const char *component_name,
+                                                const char *synonym_name,
+                                                mca_base_var_syn_flag_t flags);
 
 /**
  * Deregister a MCA variable or synonym
@@ -510,7 +505,7 @@ OPAL_DECLSPEC int mca_base_var_register_synonym (int synonym_for, const char *pr
  * @param vari Index returned from mca_base_var_register() or
  * mca_base_var_register_synonym().
  *
- * Deregistering a variable does not free the variable or any memory assoicated
+ * Deregistering a variable does not free the variable or any memory associated
  * with it. All memory will be freed and the variable index released when
  * mca_base_var_finalize() is called.
  *
@@ -518,7 +513,6 @@ OPAL_DECLSPEC int mca_base_var_register_synonym (int synonym_for, const char *pr
  */
 OPAL_DECLSPEC int mca_base_var_deregister(int vari);
 
-
 /**
  * Get the current value of an MCA variable.
  *
@@ -537,9 +531,8 @@ OPAL_DECLSPEC int mca_base_var_deregister(int vari);
  * Note: The value can be changed by the registering code without using
  * the mca_base_var_* interface so the source may be incorrect.
  */
-OPAL_DECLSPEC int mca_base_var_get_value (int vari, const void *value,
-                                          mca_base_var_source_t *source,
-                                          const char **source_file);
+OPAL_DECLSPEC int mca_base_var_get_value(int vari, const void *value, mca_base_var_source_t *source,
+                                         const char **source_file);
 
 /**
  * Sets an "override" value for an integer MCA variable.
@@ -562,9 +555,8 @@ OPAL_DECLSPEC int mca_base_var_get_value (int vari, const void *value,
  * a synonym the variable the synonym represents) if the value is
  * settable.
  */
-OPAL_DECLSPEC int mca_base_var_set_value (int vari, const void *value, size_t size,
-                                          mca_base_var_source_t source,
-                                          const char *source_file);
+OPAL_DECLSPEC int mca_base_var_set_value(int vari, const void *value, size_t size,
+                                         mca_base_var_source_t source, const char *source_file);
 
 /**
  * Get the string name corresponding to the MCA variable
@@ -579,8 +571,7 @@ OPAL_DECLSPEC int mca_base_var_set_value (int vari, const void *value, size_t si
  * The string that is returned is owned by the caller; if
  * appropriate, it must be eventually freed by the caller.
  */
-OPAL_DECLSPEC int mca_base_var_env_name(const char *param_name,
-                                        char **env_name);
+OPAL_DECLSPEC int mca_base_var_env_name(const char *param_name, char **env_name);
 
 /**
  * Find the index for an MCA variable based on its names.
@@ -599,10 +590,8 @@ OPAL_DECLSPEC int mca_base_var_env_name(const char *param_name,
  * of any registered variable.  The returned index can be used with
  * mca_base_var_get() and mca_base_var_get_value().
  */
-OPAL_DECLSPEC int mca_base_var_find (const char *project_name,
-                                     const char *type_name,
-                                     const char *component_name,
-                                     const char *param_name);
+OPAL_DECLSPEC int mca_base_var_find(const char *project_name, const char *type_name,
+                                    const char *component_name, const char *param_name);
 
 /**
  * Find the index for a variable based on its full name
@@ -612,7 +601,7 @@ OPAL_DECLSPEC int mca_base_var_find (const char *project_name,
  *
  * See mca_base_var_find().
  */
-OPAL_DECLSPEC int mca_base_var_find_by_name (const char *full_name, int *vari);
+OPAL_DECLSPEC int mca_base_var_find_by_name(const char *full_name, int *vari);
 
 /**
  * Check that two MCA variables were not both set to non-default
@@ -642,13 +631,10 @@ OPAL_DECLSPEC int mca_base_var_find_by_name (const char *full_name, int *vari);
  * are not MCA_BASE_VAR_SOURCE_DEFAULT.
  * @returns OPAL_SUCCESS otherwise.
  */
-OPAL_DECLSPEC int mca_base_var_check_exclusive (const char *project,
-                                                const char *type_a,
-                                                const char *component_a,
-                                                const char *param_a,
-                                                const char *type_b,
-                                                const char *component_b,
-                                                const char *param_b);
+OPAL_DECLSPEC int mca_base_var_check_exclusive(const char *project, const char *type_a,
+                                               const char *component_a, const char *param_a,
+                                               const char *type_b, const char *component_b,
+                                               const char *param_b);
 
 /**
  * Set or unset a flag on a variable.
@@ -661,8 +647,7 @@ OPAL_DECLSPEC int mca_base_var_check_exclusive (const char *project,
  * @returns OPAL_ERR_BAD_PARAM If the variable is not registered.
  * @returns OPAL_ERROR Otherwise
  */
-OPAL_DECLSPEC int mca_base_var_set_flag(int vari, mca_base_var_flag_t flag,
-                                        bool set);
+OPAL_DECLSPEC int mca_base_var_set_flag(int vari, mca_base_var_flag_t flag, bool set);
 
 /**
  * Obtain basic info on a single variable (name, help message, etc)
@@ -676,7 +661,7 @@ OPAL_DECLSPEC int mca_base_var_set_flag(int vari, mca_base_var_flag_t flag,
  * The returned pointer belongs to the MCA variable system. Do not
  * modify/free/retain the pointer.
  */
-OPAL_DECLSPEC int mca_base_var_get (int vari, const mca_base_var_t **var);
+OPAL_DECLSPEC int mca_base_var_get(int vari, const mca_base_var_t **var);
 
 /**
  * Obtain the number of variables that have been registered.
@@ -689,10 +674,10 @@ OPAL_DECLSPEC int mca_base_var_get (int vari, const mca_base_var_t **var);
  * returned is equal to the number of calls to mca_base_var_register with
  * unique names. ie. two calls with the same name will not affect the count.
  */
-OPAL_DECLSPEC int mca_base_var_get_count (void);
+OPAL_DECLSPEC int mca_base_var_get_count(void);
 
 /**
- * Obtain a list of enironment variables describing the all
+ * Obtain a list of environment variables describing the all
  * valid (non-default) MCA variables and their sources.
  *
  * @param[out] env A pointer to an argv-style array of key=value
@@ -708,8 +693,7 @@ OPAL_DECLSPEC int mca_base_var_get_count (void);
  * its output is in terms of an argv-style array of key=value
  * strings, suitable for using in an environment.
  */
-OPAL_DECLSPEC int mca_base_var_build_env(char ***env, int *num_env,
-                                         bool internal);
+OPAL_DECLSPEC int mca_base_var_build_env(char ***env, int *num_env, bool internal);
 
 typedef enum {
     /* Dump human-readable strings */
@@ -717,7 +701,9 @@ typedef enum {
     /* Dump easily parsable strings */
     MCA_BASE_VAR_DUMP_PARSABLE = 1,
     /* Dump simple name=value string */
-    MCA_BASE_VAR_DUMP_SIMPLE   = 2
+    MCA_BASE_VAR_DUMP_SIMPLE = 2,
+    /* Dump human-readable strings, with color where supported */
+    MCA_BASE_VAR_DUMP_READABLE_COLOR = 3
 } mca_base_var_dump_type_t;
 
 /**
@@ -733,7 +719,7 @@ typedef enum {
 OPAL_DECLSPEC int mca_base_var_dump(int vari, char ***out, mca_base_var_dump_type_t output_type);
 
 #define MCA_COMPILETIME_VER "print_compiletime_version"
-#define MCA_RUNTIME_VER "print_runtime_version"
+#define MCA_RUNTIME_VER     "print_runtime_version"
 
 /*
  * Parse a provided list of envars and add their local value, or
@@ -746,8 +732,7 @@ OPAL_DECLSPEC int mca_base_var_process_env_list_from_file(char ***argv);
  * Initialize any file-based params
  */
 OPAL_DECLSPEC int mca_base_var_cache_files(bool rel_path_search);
-OPAL_DECLSPEC int mca_base_var_load_extra_files(char* files, bool rel_path_search);
-
+OPAL_DECLSPEC int mca_base_var_load_extra_files(char *files, bool rel_path_search);
 
 extern char *mca_base_env_list;
 #define MCA_BASE_ENV_LIST_SEP_DEFAULT ";"
diff --git a/opal/mca/base/mca_base_var_enum.c b/opal/mca/base/mca_base_var_enum.c
index 67aa875f5f9..a6d89da5f20 100644
--- a/opal/mca/base/mca_base_var_enum.c
+++ b/opal/mca/base/mca_base_var_enum.c
@@ -15,8 +15,11 @@
  *                         reserved.
  * Copyright (c) 2017      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2017-2022 IBM Corporation. All rights reserved.
  * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2022      Computer Architecture and VLSI Systems (CARV)
+ *                         Laboratory, ICS Forth. All rights reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -26,38 +29,40 @@
 
 #include "opal_config.h"
 
+#include "opal/mca/base/base.h"
 #include "opal/mca/base/mca_base_var_enum.h"
 #include "opal/mca/base/mca_base_vari.h"
-#include "opal/mca/base/base.h"
 #include "opal/util/argv.h"
 #include "opal/util/printf.h"
 
+#include <errno.h>
 #include <stdio.h>
 #include <string.h>
-#include <errno.h>
 
-static void mca_base_var_enum_constructor (mca_base_var_enum_t *enumerator);
-static void mca_base_var_enum_destructor (mca_base_var_enum_t *enumerator);
+static void mca_base_var_enum_constructor(mca_base_var_enum_t *enumerator);
+static void mca_base_var_enum_destructor(mca_base_var_enum_t *enumerator);
 OBJ_CLASS_INSTANCE(mca_base_var_enum_t, opal_object_t, mca_base_var_enum_constructor,
                    mca_base_var_enum_destructor);
 
-static void mca_base_var_enum_flag_constructor (mca_base_var_enum_flag_t *enumerator);
-static void mca_base_var_enum_flag_destructor (mca_base_var_enum_flag_t *enumerator);
-static OBJ_CLASS_INSTANCE(mca_base_var_enum_flag_t, opal_object_t, mca_base_var_enum_flag_constructor,
-                   mca_base_var_enum_flag_destructor);
+static void mca_base_var_enum_flag_constructor(mca_base_var_enum_flag_t *enumerator);
+static void mca_base_var_enum_flag_destructor(mca_base_var_enum_flag_t *enumerator);
+static OBJ_CLASS_INSTANCE(mca_base_var_enum_flag_t, opal_object_t,
+                          mca_base_var_enum_flag_constructor, mca_base_var_enum_flag_destructor);
 
-static int enum_dump (mca_base_var_enum_t *self, char **out);
-static int enum_get_count (mca_base_var_enum_t *self, int *count);
-static int enum_get_value (mca_base_var_enum_t *self, int index, int *value, const char **string_value);
+static int enum_dump(mca_base_var_enum_t *self, char **out,
+                     mca_base_var_enum_dump_type_t output_type);
+static int enum_get_count(mca_base_var_enum_t *self, int *count);
+static int enum_get_value(mca_base_var_enum_t *self, int index, int *value,
+                          const char **string_value);
 
-static int mca_base_var_enum_bool_get_count (mca_base_var_enum_t *enumerator, int *count)
+static int mca_base_var_enum_bool_get_count(mca_base_var_enum_t *enumerator, int *count)
 {
     *count = 2;
     return OPAL_SUCCESS;
 }
 
-static int mca_base_var_enum_bool_get_value (mca_base_var_enum_t *self, int index,
-                                             int *value, const char **string_value)
+static int mca_base_var_enum_bool_get_value(mca_base_var_enum_t *self, int index, int *value,
+                                            const char **string_value)
 {
     if (1 < index) {
         return OPAL_ERR_VALUE_OUT_OF_BOUNDS;
@@ -69,24 +74,24 @@ static int mca_base_var_enum_bool_get_value (mca_base_var_enum_t *self, int inde
     return OPAL_SUCCESS;
 }
 
-static int mca_base_var_enum_bool_vfs (mca_base_var_enum_t *self, const char *string_value,
-                                       int *value)
+static int mca_base_var_enum_bool_vfs(mca_base_var_enum_t *self, const char *string_value,
+                                      int *value)
 {
     char *tmp;
     long v;
 
     /* skip whitespace */
-    string_value += strspn (string_value, " \t\n\v\f\r");
+    string_value += strspn(string_value, " \t\n\v\f\r");
 
-    v = strtol (string_value, &tmp, 10);
+    v = strtol(string_value, &tmp, 10);
     if (*tmp != '\0') {
-        if (0 == strcmp (string_value, "true") || 0 == strcmp (string_value, "t") ||
-            0 == strcmp (string_value, "enabled") || 0 == strcmp (string_value, "yes") ||
-            0 == strcmp (string_value, "y")) {
+        if (0 == strcmp(string_value, "true") || 0 == strcmp(string_value, "t")
+            || 0 == strcmp(string_value, "enabled") || 0 == strcmp(string_value, "yes")
+            || 0 == strcmp(string_value, "y")) {
             v = 1;
-        } else if (0 == strcmp (string_value, "false") || 0 == strcmp (string_value, "f") ||
-                   0 == strcmp (string_value, "disabled") || 0 == strcmp (string_value, "no") ||
-                   0 == strcmp (string_value, "n")) {
+        } else if (0 == strcmp(string_value, "false") || 0 == strcmp(string_value, "f")
+                   || 0 == strcmp(string_value, "disabled") || 0 == strcmp(string_value, "no")
+                   || 0 == strcmp(string_value, "n")) {
             v = 0;
         } else {
             return OPAL_ERR_VALUE_OUT_OF_BOUNDS;
@@ -98,41 +103,66 @@ static int mca_base_var_enum_bool_vfs (mca_base_var_enum_t *self, const char *st
     return OPAL_SUCCESS;
 }
 
-static int mca_base_var_enum_bool_sfv (mca_base_var_enum_t *self, const int value,
-                                       char **string_value)
+static int mca_base_var_enum_bool_sfv(mca_base_var_enum_t *self, const int value,
+                                      char **string_value)
 {
     if (string_value) {
-        *string_value = strdup (value ? "true" : "false");
+        *string_value = strdup(value ? "true" : "false");
     }
 
     return OPAL_SUCCESS;
 }
 
-static int mca_base_var_enum_bool_dump (mca_base_var_enum_t *self, char **out)
+static int mca_base_var_enum_bool_dump(mca_base_var_enum_t *self, char **out,
+                                       mca_base_var_enum_dump_type_t output_type)
 {
-    *out = strdup ("0: f|false|disabled|no|n, 1: t|true|enabled|yes|y");
-    return *out ? OPAL_SUCCESS : OPAL_ERR_OUT_OF_RESOURCE;
+    int ret;
+    char *color_vv = "", *color_reset = "";
+
+    if (MCA_BASE_VAR_ENUM_DUMP_READABLE_COLOR == output_type) {
+        color_vv = opal_var_dump_color[OPAL_VAR_DUMP_COLOR_VALID_VALUES];
+        color_reset = "\033[0m";
+    }
+
+    ret = opal_asprintf(out, "%s0%s|%sf%s|%sfalse%s|%sdisabled%s|%sno%s|%sn%s, "
+                        "%s1%s|%st%s|%strue%s|%senabled%s|%syes%s|%sy%s",
+                        color_vv, color_reset,
+                        color_vv, color_reset,
+                        color_vv, color_reset,
+                        color_vv, color_reset,
+                        color_vv, color_reset,
+                        color_vv, color_reset,
+                        color_vv, color_reset,
+                        color_vv, color_reset,
+                        color_vv, color_reset,
+                        color_vv, color_reset,
+                        color_vv, color_reset,
+                        color_vv, color_reset);
+    if (ret < 0) {
+        *out = NULL;
+        return OPAL_ERR_OUT_OF_RESOURCE;
+    }
+
+    return OPAL_SUCCESS;
 }
 
-mca_base_var_enum_t mca_base_var_enum_bool = {
-    .super     = OPAL_OBJ_STATIC_INIT(opal_object_t),
-    .enum_is_static = true,
-    .enum_name = "boolean",
-    .get_count = mca_base_var_enum_bool_get_count,
-    .get_value = mca_base_var_enum_bool_get_value,
-    .value_from_string = mca_base_var_enum_bool_vfs,
-    .string_from_value = mca_base_var_enum_bool_sfv,
-    .dump      = mca_base_var_enum_bool_dump
-};
+mca_base_var_enum_t mca_base_var_enum_bool = {.super = OPAL_OBJ_STATIC_INIT(opal_object_t),
+                                              .enum_is_static = true,
+                                              .enum_name = "boolean",
+                                              .get_count = mca_base_var_enum_bool_get_count,
+                                              .get_value = mca_base_var_enum_bool_get_value,
+                                              .value_from_string = mca_base_var_enum_bool_vfs,
+                                              .string_from_value = mca_base_var_enum_bool_sfv,
+                                              .dump = mca_base_var_enum_bool_dump};
 
-static int mca_base_var_enum_auto_bool_get_count (mca_base_var_enum_t *enumerator, int *count)
+static int mca_base_var_enum_auto_bool_get_count(mca_base_var_enum_t *enumerator, int *count)
 {
     *count = 3;
     return OPAL_SUCCESS;
 }
 
-static int mca_base_var_enum_auto_bool_get_value (mca_base_var_enum_t *self, int index,
-                                                  int *value, const char **string_value)
+static int mca_base_var_enum_auto_bool_get_value(mca_base_var_enum_t *self, int index, int *value,
+                                                 const char **string_value)
 {
     const int values[3] = {0, 1, -1};
     const char *strings[3] = {"false", "true", "auto"};
@@ -147,26 +177,26 @@ static int mca_base_var_enum_auto_bool_get_value (mca_base_var_enum_t *self, int
     return OPAL_SUCCESS;
 }
 
-static int mca_base_var_enum_auto_bool_vfs (mca_base_var_enum_t *self, const char *string_value,
-                                            int *value)
+static int mca_base_var_enum_auto_bool_vfs(mca_base_var_enum_t *self, const char *string_value,
+                                           int *value)
 {
     char *tmp;
     long v;
 
     /* skip whitespace */
-    string_value += strspn (string_value, " \t\n\v\f\r");
+    string_value += strspn(string_value, " \t\n\v\f\r");
 
-    v = strtol (string_value, &tmp, 10);
+    v = strtol(string_value, &tmp, 10);
     if (*tmp != '\0') {
-        if (0 == strcasecmp (string_value, "true") || 0 == strcasecmp (string_value, "t") ||
-            0 == strcasecmp (string_value, "enabled") || 0 == strcasecmp (string_value, "yes") ||
-            0 == strcasecmp (string_value, "y")) {
+        if (0 == strcasecmp(string_value, "true") || 0 == strcasecmp(string_value, "t")
+            || 0 == strcasecmp(string_value, "enabled") || 0 == strcasecmp(string_value, "yes")
+            || 0 == strcasecmp(string_value, "y")) {
             v = 1;
-        } else if (0 == strcasecmp (string_value, "false") || 0 == strcasecmp (string_value, "f") ||
-                   0 == strcasecmp (string_value, "disabled") || 0 == strcasecmp (string_value, "no") ||
-                   0 == strcasecmp (string_value, "n")) {
+        } else if (0 == strcasecmp(string_value, "false") || 0 == strcasecmp(string_value, "f")
+                   || 0 == strcasecmp(string_value, "disabled")
+                   || 0 == strcasecmp(string_value, "no") || 0 == strcasecmp(string_value, "n")) {
             v = 0;
-        } else if (0 == strcasecmp (string_value, "auto")) {
+        } else if (0 == strcasecmp(string_value, "auto")) {
             v = -1;
         } else {
             return OPAL_ERR_VALUE_OUT_OF_BOUNDS;
@@ -184,65 +214,92 @@ static int mca_base_var_enum_auto_bool_vfs (mca_base_var_enum_t *self, const cha
     return OPAL_SUCCESS;
 }
 
-static int mca_base_var_enum_auto_bool_sfv (mca_base_var_enum_t *self, const int value,
-                                            char **string_value)
+static int mca_base_var_enum_auto_bool_sfv(mca_base_var_enum_t *self, const int value,
+                                           char **string_value)
 {
     if (string_value) {
         if (value < 0) {
-            *string_value = strdup ("auto");
+            *string_value = strdup("auto");
         } else if (value > 0) {
-            *string_value = strdup ("true");
+            *string_value = strdup("true");
         } else {
-            *string_value = strdup ("false");
+            *string_value = strdup("false");
         }
     }
 
     return OPAL_SUCCESS;
 }
 
-static int mca_base_var_enum_auto_bool_dump (mca_base_var_enum_t *self, char **out)
+static int mca_base_var_enum_auto_bool_dump(mca_base_var_enum_t *self, char **out,
+                                            mca_base_var_enum_dump_type_t output_type)
 {
-    *out = strdup ("-1: auto, 0: f|false|disabled|no|n, 1: t|true|enabled|yes|y");
-    return *out ? OPAL_SUCCESS : OPAL_ERR_OUT_OF_RESOURCE;
+    int ret;
+    char *color_vv = "", *color_reset = "";
+
+    if (MCA_BASE_VAR_ENUM_DUMP_READABLE_COLOR == output_type) {
+        color_vv = opal_var_dump_color[OPAL_VAR_DUMP_COLOR_VALID_VALUES];
+        color_reset = "\033[0m";
+    }
+
+    ret = opal_asprintf(out, "%s-1%s|%sauto%s, "
+                        "%s0%s|%sf%s|%sfalse%s|%sdisabled%s|%sno%s|%sn%s, "
+                        "%s1%s|%st%s|%strue%s|%senabled%s|%syes%s|%sy%s",
+                        color_vv, color_reset,
+                        color_vv, color_reset,
+                        color_vv, color_reset,
+                        color_vv, color_reset,
+                        color_vv, color_reset,
+                        color_vv, color_reset,
+                        color_vv, color_reset,
+                        color_vv, color_reset,
+                        color_vv, color_reset,
+                        color_vv, color_reset,
+                        color_vv, color_reset,
+                        color_vv, color_reset,
+                        color_vv, color_reset,
+                        color_vv, color_reset);
+    if (ret < 0) {
+        *out = NULL;
+        return OPAL_ERR_OUT_OF_RESOURCE;
+    }
+
+    return OPAL_SUCCESS;
 }
 
-mca_base_var_enum_t mca_base_var_enum_auto_bool = {
-    .super     = OPAL_OBJ_STATIC_INIT(opal_object_t),
-    .enum_is_static = true,
-    .enum_name = "auto_boolean",
-    .get_count = mca_base_var_enum_auto_bool_get_count,
-    .get_value = mca_base_var_enum_auto_bool_get_value,
-    .value_from_string = mca_base_var_enum_auto_bool_vfs,
-    .string_from_value = mca_base_var_enum_auto_bool_sfv,
-    .dump      = mca_base_var_enum_auto_bool_dump
-};
+mca_base_var_enum_t mca_base_var_enum_auto_bool
+    = {.super = OPAL_OBJ_STATIC_INIT(opal_object_t),
+       .enum_is_static = true,
+       .enum_name = "auto_boolean",
+       .get_count = mca_base_var_enum_auto_bool_get_count,
+       .get_value = mca_base_var_enum_auto_bool_get_value,
+       .value_from_string = mca_base_var_enum_auto_bool_vfs,
+       .string_from_value = mca_base_var_enum_auto_bool_sfv,
+       .dump = mca_base_var_enum_auto_bool_dump};
 
 /* verbosity enumerator */
-static mca_base_var_enum_value_t verbose_values[] = {
-    {MCA_BASE_VERBOSE_NONE,      "none"},
-    {MCA_BASE_VERBOSE_ERROR,     "error"},
-    {MCA_BASE_VERBOSE_COMPONENT, "component"},
-    {MCA_BASE_VERBOSE_WARN,      "warn"},
-    {MCA_BASE_VERBOSE_INFO,      "info"},
-    {MCA_BASE_VERBOSE_TRACE,     "trace"},
-    {MCA_BASE_VERBOSE_DEBUG,     "debug"},
-    {MCA_BASE_VERBOSE_MAX,       "max"},
-    {-1, NULL}
-};
-
-static int mca_base_var_enum_verbose_vfs (mca_base_var_enum_t *self, const char *string_value,
-                                          int *value)
+static mca_base_var_enum_value_t verbose_values[] = {{MCA_BASE_VERBOSE_NONE, "none"},
+                                                     {MCA_BASE_VERBOSE_ERROR, "error"},
+                                                     {MCA_BASE_VERBOSE_COMPONENT, "component"},
+                                                     {MCA_BASE_VERBOSE_WARN, "warn"},
+                                                     {MCA_BASE_VERBOSE_INFO, "info"},
+                                                     {MCA_BASE_VERBOSE_TRACE, "trace"},
+                                                     {MCA_BASE_VERBOSE_DEBUG, "debug"},
+                                                     {MCA_BASE_VERBOSE_MAX, "max"},
+                                                     {-1, NULL}};
+
+static int mca_base_var_enum_verbose_vfs(mca_base_var_enum_t *self, const char *string_value,
+                                         int *value)
 {
     char *tmp;
     int v;
 
     /* skip whitespace */
-    string_value += strspn (string_value, " \t\n\v\f\r");
+    string_value += strspn(string_value, " \t\n\v\f\r");
 
-    v = strtol (string_value, &tmp, 10);
+    v = strtol(string_value, &tmp, 10);
     if (*tmp != '\0') {
-        for (int i = 0 ; verbose_values[i].string ; ++i) {
-            if (0 == strcmp (verbose_values[i].string, string_value)) {
+        for (int i = 0; verbose_values[i].string; ++i) {
+            if (0 == strcmp(verbose_values[i].string, string_value)) {
                 *value = verbose_values[i].value;
                 return OPAL_SUCCESS;
             }
@@ -260,8 +317,8 @@ static int mca_base_var_enum_verbose_vfs (mca_base_var_enum_t *self, const char
     return OPAL_SUCCESS;
 }
 
-static int mca_base_var_enum_verbose_sfv (mca_base_var_enum_t *self, const int value,
-                                          char **string_value)
+static int mca_base_var_enum_verbose_sfv(mca_base_var_enum_t *self, const int value,
+                                         char **string_value)
 {
     int ret;
 
@@ -269,17 +326,17 @@ static int mca_base_var_enum_verbose_sfv (mca_base_var_enum_t *self, const int v
         return OPAL_ERR_VALUE_OUT_OF_BOUNDS;
     }
 
-    for (int i = 0 ; verbose_values[i].string ; ++i) {
+    for (int i = 0; verbose_values[i].string; ++i) {
         if (verbose_values[i].value == value) {
             if (string_value) {
-                *string_value = strdup (verbose_values[i].string);
+                *string_value = strdup(verbose_values[i].string);
             }
             return OPAL_SUCCESS;
         }
     }
 
     if (string_value) {
-        ret = opal_asprintf (string_value, "%d", value);
+        ret = opal_asprintf(string_value, "%d", value);
         if (0 > ret) {
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
@@ -288,18 +345,28 @@ static int mca_base_var_enum_verbose_sfv (mca_base_var_enum_t *self, const int v
     return OPAL_SUCCESS;
 }
 
-static int mca_base_var_enum_verbose_dump (mca_base_var_enum_t *self, char **out)
+static int mca_base_var_enum_verbose_dump(mca_base_var_enum_t *self, char **out,
+                                          mca_base_var_enum_dump_type_t output_type)
 {
     char *tmp;
     int ret;
 
-    ret = enum_dump (self, out);
+    char *color_vv = "", *color_reset = "";
+
+    ret = enum_dump(self, out, output_type);
     if (OPAL_SUCCESS != ret) {
         return ret;
     }
 
-    ret = opal_asprintf (&tmp, "%s, 0 - 100", *out);
-    free (*out);
+    if (MCA_BASE_VAR_ENUM_DUMP_READABLE_COLOR == output_type) {
+        color_vv = opal_var_dump_color[OPAL_VAR_DUMP_COLOR_VALID_VALUES];
+        color_reset = "\033[0m";
+    }
+
+    ret = opal_asprintf(&tmp, "%s, %s0%s-%s100%s", *out,
+                        color_vv, color_reset,
+                        color_vv, color_reset);
+    free(*out);
     if (0 > ret) {
         *out = NULL;
         return OPAL_ERR_OUT_OF_RESOURCE;
@@ -311,20 +378,20 @@ static int mca_base_var_enum_verbose_dump (mca_base_var_enum_t *self, char **out
 }
 
 mca_base_var_enum_t mca_base_var_enum_verbose = {
-    .super     = OPAL_OBJ_STATIC_INIT(opal_object_t),
+    .super = OPAL_OBJ_STATIC_INIT(opal_object_t),
     .enum_is_static = true,
     .enum_name = "verbosity",
     .get_count = enum_get_count,
     .get_value = enum_get_value,
     .value_from_string = mca_base_var_enum_verbose_vfs,
     .string_from_value = mca_base_var_enum_verbose_sfv,
-    .dump      = mca_base_var_enum_verbose_dump,
+    .dump = mca_base_var_enum_verbose_dump,
     .enum_value_count = 8,
     .enum_values = verbose_values,
 };
 
-
-int mca_base_var_enum_create (const char *name, const mca_base_var_enum_value_t *values, mca_base_var_enum_t **enumerator)
+int mca_base_var_enum_create(const char *name, const mca_base_var_enum_value_t *values,
+                             mca_base_var_enum_t **enumerator)
 {
     mca_base_var_enum_t *new_enum;
     int i;
@@ -336,24 +403,25 @@ int mca_base_var_enum_create (const char *name, const mca_base_var_enum_value_t
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
-    new_enum->enum_name = strdup (name);
+    new_enum->enum_name = strdup(name);
     if (NULL == new_enum->enum_name) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
-    for (i = 0 ; values[i].string ; ++i);
+    for (i = 0; values[i].string; ++i) {
+    }
     new_enum->enum_value_count = i;
 
     /* make a copy of the values */
-    new_enum->enum_values = calloc (new_enum->enum_value_count + 1, sizeof (*new_enum->enum_values));
+    new_enum->enum_values = calloc(new_enum->enum_value_count + 1, sizeof(*new_enum->enum_values));
     if (NULL == new_enum->enum_values) {
         OBJ_RELEASE(new_enum);
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
-    for (i = 0 ; i < new_enum->enum_value_count ; ++i) {
+    for (i = 0; i < new_enum->enum_value_count; ++i) {
         new_enum->enum_values[i].value = values[i].value;
-        new_enum->enum_values[i].string = strdup (values[i].string);
+        new_enum->enum_values[i].string = strdup(values[i].string);
     }
 
     *enumerator = new_enum;
@@ -361,10 +429,12 @@ int mca_base_var_enum_create (const char *name, const mca_base_var_enum_value_t
     return OPAL_SUCCESS;
 }
 
-int mca_base_var_enum_create_flag (const char *name, const mca_base_var_enum_value_flag_t *flags, mca_base_var_enum_flag_t **enumerator)
+int mca_base_var_enum_create_flag(const char *name, const mca_base_var_enum_value_flag_t *flags,
+                                  mca_base_var_enum_flag_t **enumerator)
 {
     mca_base_var_enum_flag_t *new_enum;
     int i;
+    int __opal_attribute_unused__ all_flags = 0;
 
     *enumerator = NULL;
 
@@ -373,32 +443,34 @@ int mca_base_var_enum_create_flag (const char *name, const mca_base_var_enum_val
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
-    new_enum->super.enum_name = strdup (name);
+    new_enum->super.enum_name = strdup(name);
     if (NULL == new_enum->super.enum_name) {
+        OBJ_RELEASE(new_enum);
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
-    for (i = 0 ; flags[i].string ; ++i);
+    for (i = 0; flags[i].string; ++i) {
+    }
     new_enum->super.enum_value_count = i;
 
     /* make a copy of the values */
-    new_enum->enum_flags = calloc (new_enum->super.enum_value_count + 1, sizeof (*new_enum->enum_flags));
+    new_enum->enum_flags = calloc(new_enum->super.enum_value_count + 1,
+                                  sizeof(*new_enum->enum_flags));
     if (NULL == new_enum->enum_flags) {
         OBJ_RELEASE(new_enum);
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
-    int all_flags = 0;
-    for (i = 0 ; i < new_enum->super.enum_value_count ; ++i) {
+    for (i = 0; i < new_enum->super.enum_value_count; ++i) {
         new_enum->enum_flags[i].flag = flags[i].flag;
-        new_enum->enum_flags[i].string = strdup (flags[i].string);
+        new_enum->enum_flags[i].string = strdup(flags[i].string);
         new_enum->enum_flags[i].conflicting_flag = flags[i].conflicting_flag;
         /* ensure flags are only set a single bit, doesn't conflict with itself, and
          * hasn't already been specified. */
-        assert (!(flags[i].flag & (flags[i].flag - 1)));
-        assert (!(flags[i].flag & flags[i].conflicting_flag));
-        assert (!(all_flags & flags[i].flag));
-        assert (flags[i].flag);
+        assert(!(flags[i].flag & (flags[i].flag - 1)));
+        assert(!(flags[i].flag & flags[i].conflicting_flag));
+        assert(!(all_flags & flags[i].flag));
+        assert(flags[i].flag);
         all_flags |= flags[i].flag;
     }
 
@@ -407,23 +479,35 @@ int mca_base_var_enum_create_flag (const char *name, const mca_base_var_enum_val
     return OPAL_SUCCESS;
 }
 
-static int enum_dump (mca_base_var_enum_t *self, char **out)
+static int enum_dump(mca_base_var_enum_t *self, char **out,
+                     mca_base_var_enum_dump_type_t output_type)
 {
     int i;
     char *tmp;
     int ret;
 
+    char *color_vv = "", *color_reset = "";
+
     *out = NULL;
 
     if (NULL == self) {
         return OPAL_ERROR;
     }
 
+    if (MCA_BASE_VAR_ENUM_DUMP_READABLE_COLOR == output_type) {
+        color_vv = opal_var_dump_color[OPAL_VAR_DUMP_COLOR_VALID_VALUES];
+        color_reset = "\033[0m";
+    }
+
     tmp = NULL;
-    for (i = 0; i < self->enum_value_count && self->enum_values[i].string ; ++i) {
-        ret = opal_asprintf (out, "%s%s%d:\"%s\"", tmp ? tmp : "", tmp ? ", " : "", self->enum_values[i].value,
-                        self->enum_values[i].string);
-        if (tmp) free (tmp);
+    for (i = 0; i < self->enum_value_count && self->enum_values[i].string; ++i) {
+        ret = opal_asprintf(out, "%s%s%s%d%s|%s%s%s",
+                            tmp ? tmp : "", tmp ? ", " : "",
+                            color_vv, self->enum_values[i].value, color_reset,
+                            color_vv, self->enum_values[i].string, color_reset);
+        if (tmp) {
+            free(tmp);
+        }
         if (0 > ret) {
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
@@ -433,13 +517,14 @@ static int enum_dump (mca_base_var_enum_t *self, char **out)
     return OPAL_SUCCESS;
 }
 
-static int enum_get_count (mca_base_var_enum_t *self, int *count)
+static int enum_get_count(mca_base_var_enum_t *self, int *count)
 {
     *count = self->enum_value_count;
     return OPAL_SUCCESS;
 }
 
-static int enum_get_value (mca_base_var_enum_t *self, int index, int *value, const char **string_value)
+static int enum_get_value(mca_base_var_enum_t *self, int index, int *value,
+                          const char **string_value)
 {
     int count, ret;
 
@@ -457,13 +542,15 @@ static int enum_get_value (mca_base_var_enum_t *self, int index, int *value, con
     }
 
     if (string_value) {
-        *string_value = strdup (self->enum_values[index].string);
+        *string_value = strdup(self->enum_values[index].string);
     }
 
     return OPAL_SUCCESS;
 }
 
-static int enum_value_from_string(mca_base_var_enum_t *self, const char *string_value, int *value_out) {
+static int enum_value_from_string(mca_base_var_enum_t *self, const char *string_value,
+                                  int *value_out)
+{
     int value, count, ret, i;
     bool is_int;
     char *tmp;
@@ -478,9 +565,9 @@ static int enum_value_from_string(mca_base_var_enum_t *self, const char *string_
     /* Check if the string is an integer */
     is_int = tmp[0] == '\0';
 
-    for (i = 0 ; i < count ; ++i) {
-        if ((is_int && value == self->enum_values[i].value) ||
-            0 == strcasecmp (string_value, self->enum_values[i].string)) {
+    for (i = 0; i < count; ++i) {
+        if ((is_int && value == self->enum_values[i].value)
+            || 0 == strcasecmp(string_value, self->enum_values[i].string)) {
             break;
         }
     }
@@ -494,7 +581,8 @@ static int enum_value_from_string(mca_base_var_enum_t *self, const char *string_
     return OPAL_SUCCESS;
 }
 
-static int enum_string_from_value(mca_base_var_enum_t *self, const int value, char **string_value) {
+static int enum_string_from_value(mca_base_var_enum_t *self, const int value, char **string_value)
+{
     int count, ret, i;
 
     ret = self->get_count(self, &count);
@@ -502,7 +590,7 @@ static int enum_string_from_value(mca_base_var_enum_t *self, const int value, ch
         return ret;
     }
 
-    for (i = 0 ; i < count ; ++i) {
+    for (i = 0; i < count; ++i) {
         if (value == self->enum_values[i].value) {
             break;
         }
@@ -513,40 +601,42 @@ static int enum_string_from_value(mca_base_var_enum_t *self, const int value, ch
     }
 
     if (string_value) {
-        *string_value = strdup (self->enum_values[i].string);
+        *string_value = strdup(self->enum_values[i].string);
     }
 
     return OPAL_SUCCESS;
 }
 
-static void mca_base_var_enum_constructor (mca_base_var_enum_t *enumerator)
+static void mca_base_var_enum_constructor(mca_base_var_enum_t *enumerator)
 {
-    memset ((char *) enumerator + sizeof (enumerator->super), 0 , sizeof(*enumerator) - sizeof(enumerator->super));
+    memset((char *) enumerator + sizeof(enumerator->super), 0,
+           sizeof(*enumerator) - sizeof(enumerator->super));
 
     enumerator->get_value = enum_get_value;
     enumerator->get_count = enum_get_count;
     enumerator->value_from_string = enum_value_from_string;
     enumerator->string_from_value = enum_string_from_value;
-    enumerator->dump      = enum_dump;
+    enumerator->dump = enum_dump;
     enumerator->enum_is_static = false;
 }
 
-static void mca_base_var_enum_destructor (mca_base_var_enum_t *enumerator)
+static void mca_base_var_enum_destructor(mca_base_var_enum_t *enumerator)
 {
     if (enumerator->enum_name) {
-        free (enumerator->enum_name);
+        free(enumerator->enum_name);
     }
 
     /* release the copy of the values */
     if (enumerator->enum_values) {
-        for (int i = 0 ; i < enumerator->enum_value_count ; ++i) {
-            free ((void *) enumerator->enum_values[i].string);
+        for (int i = 0; i < enumerator->enum_value_count; ++i) {
+            free((void *) enumerator->enum_values[i].string);
         }
-        free (enumerator->enum_values);
+        free(enumerator->enum_values);
     }
 }
 
-static int enum_get_value_flag (mca_base_var_enum_t *self, int index, int *value, const char **string_value)
+static int enum_get_value_flag(mca_base_var_enum_t *self, int index, int *value,
+                               const char **string_value)
 {
     mca_base_var_enum_flag_t *flag_enum = (mca_base_var_enum_flag_t *) self;
     int count, ret;
@@ -565,13 +655,15 @@ static int enum_get_value_flag (mca_base_var_enum_t *self, int index, int *value
     }
 
     if (string_value) {
-        *string_value = strdup (flag_enum->enum_flags[index].string);
+        *string_value = strdup(flag_enum->enum_flags[index].string);
     }
 
     return OPAL_SUCCESS;
 }
 
-static int enum_value_from_string_flag (mca_base_var_enum_t *self, const char *string_value, int *value_out) {
+static int enum_value_from_string_flag(mca_base_var_enum_t *self, const char *string_value,
+                                       int *value_out)
+{
     mca_base_var_enum_flag_t *flag_enum = (mca_base_var_enum_flag_t *) self;
     int value, count, ret, flag;
     char **flags;
@@ -583,21 +675,21 @@ static int enum_value_from_string_flag (mca_base_var_enum_t *self, const char *s
         return ret;
     }
 
-    flags = opal_argv_split (string_value, ',');
+    flags = opal_argv_split(string_value, ',');
     if (NULL == flags) {
         return OPAL_ERR_BAD_PARAM;
     }
 
     flag = 0;
 
-    for (int i = 0 ; flags[i] ; ++i) {
-        value = strtol (flags[i], &tmp, 0);
+    for (int i = 0; flags[i]; ++i) {
+        value = strtol(flags[i], &tmp, 0);
         is_int = tmp[0] == '\0';
 
         bool found = false, conflict = false;
-        for (int j = 0 ; j < count ; ++j) {
-            if ((is_int && (value & flag_enum->enum_flags[j].flag)) ||
-                0 == strcasecmp (flags[i], flag_enum->enum_flags[j].string)) {
+        for (int j = 0; j < count; ++j) {
+            if ((is_int && (value & flag_enum->enum_flags[j].flag))
+                || 0 == strcasecmp(flags[i], flag_enum->enum_flags[j].string)) {
                 found = true;
 
                 if (flag & flag_enum->enum_flags[j].conflicting_flag) {
@@ -618,19 +710,21 @@ static int enum_value_from_string_flag (mca_base_var_enum_t *self, const char *s
         }
 
         if (!found || conflict || (is_int && value)) {
-            opal_argv_free (flags);
+            opal_argv_free(flags);
             return !found ? OPAL_ERR_VALUE_OUT_OF_BOUNDS : OPAL_ERR_BAD_PARAM;
         }
     }
 
-    opal_argv_free (flags);
+    opal_argv_free(flags);
 
     *value_out = flag;
 
     return OPAL_SUCCESS;
 }
 
-static int enum_string_from_value_flag (mca_base_var_enum_t *self, const int value, char **string_value) {
+static int enum_string_from_value_flag(mca_base_var_enum_t *self, const int value,
+                                       char **string_value)
+{
     mca_base_var_enum_flag_t *flag_enum = (mca_base_var_enum_flag_t *) self;
     int count, ret, current;
     char *out = NULL, *tmp;
@@ -641,22 +735,23 @@ static int enum_string_from_value_flag (mca_base_var_enum_t *self, const int val
     }
 
     current = value;
-    for (int i = 0 ; i < count ; ++i) {
+    for (int i = 0; i < count; ++i) {
         if (!(flag_enum->enum_flags[i].flag & current)) {
             continue;
         }
 
         tmp = out;
 
-        ret = opal_asprintf (&out, "%s%s%s", tmp ? tmp : "", tmp ? "," : "", flag_enum->enum_flags[i].string);
-        free (tmp);
+        ret = opal_asprintf(&out, "%s%s%s", tmp ? tmp : "", tmp ? "," : "",
+                            flag_enum->enum_flags[i].string);
+        free(tmp);
 
         if (0 > ret) {
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
 
         if (value & flag_enum->enum_flags[i].conflicting_flag) {
-            free (out);
+            free(out);
             return OPAL_ERR_BAD_PARAM;
         }
 
@@ -664,42 +759,52 @@ static int enum_string_from_value_flag (mca_base_var_enum_t *self, const int val
     }
 
     if (current) {
-        free (out);
+        free(out);
         return OPAL_ERR_VALUE_OUT_OF_BOUNDS;
     }
 
     if (string_value) {
-        *string_value = out ? out : strdup ("");
+        *string_value = out ? out : strdup("");
     } else {
-        free (out);
+        free(out);
     }
 
     return OPAL_SUCCESS;
 }
 
-static int enum_dump_flag (mca_base_var_enum_t *self, char **out)
+static int enum_dump_flag(mca_base_var_enum_t *self, char **out,
+                          mca_base_var_enum_dump_type_t output_type)
 {
     mca_base_var_enum_flag_t *flag_enum = (mca_base_var_enum_flag_t *) self;
     char *tmp;
     int ret;
 
+    char *color_vv = "", *color_reset = "";
+
     *out = NULL;
 
     if (NULL == self) {
         return OPAL_ERROR;
     }
 
-    *out = strdup ("Comma-delimited list of: ");
+    if (MCA_BASE_VAR_ENUM_DUMP_READABLE_COLOR == output_type) {
+        color_vv = opal_var_dump_color[OPAL_VAR_DUMP_COLOR_VALID_VALUES];
+        color_reset = "\033[0m";
+    }
+
+    *out = strdup("Comma-delimited list of: ");
     if (NULL == *out) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
-    for (int i = 0; i < self->enum_value_count ; ++i) {
+    for (int i = 0; i < self->enum_value_count; ++i) {
         tmp = *out;
 
-        ret = opal_asprintf (out, "%s%s0x%x:\"%s\"", tmp, i ? ", " : " ", flag_enum->enum_flags[i].flag,
-                        flag_enum->enum_flags[i].string);
-        free (tmp);
+        ret = opal_asprintf(out, "%s%s%s0x%x%s|%s%s%s",
+                            tmp, i ? ", " : " ",
+                            color_vv, flag_enum->enum_flags[i].flag, color_reset,
+                            color_vv, flag_enum->enum_flags[i].string, color_reset);
+        free(tmp);
         if (0 > ret) {
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
@@ -708,7 +813,7 @@ static int enum_dump_flag (mca_base_var_enum_t *self, char **out)
     return OPAL_SUCCESS;
 }
 
-static void mca_base_var_enum_flag_constructor (mca_base_var_enum_flag_t *enumerator)
+static void mca_base_var_enum_flag_constructor(mca_base_var_enum_flag_t *enumerator)
 {
     enumerator->enum_flags = NULL;
     enumerator->super.get_value = enum_get_value_flag;
@@ -720,38 +825,36 @@ static void mca_base_var_enum_flag_constructor (mca_base_var_enum_flag_t *enumer
     enumerator->super.enum_name = NULL;
 }
 
-static void mca_base_var_enum_flag_destructor (mca_base_var_enum_flag_t *enumerator)
+static void mca_base_var_enum_flag_destructor(mca_base_var_enum_flag_t *enumerator)
 {
     /* release the copy of the values */
     if (enumerator->enum_flags) {
-        for (int i = 0 ; i < enumerator->super.enum_value_count ; ++i) {
-            free ((void *) enumerator->enum_flags[i].string);
+        for (int i = 0; i < enumerator->super.enum_value_count; ++i) {
+            free((void *) enumerator->enum_flags[i].string);
         }
-        free (enumerator->enum_flags);
+        free(enumerator->enum_flags);
     }
     if (NULL != enumerator->super.enum_name) {
-        free (enumerator->super.enum_name);
+        free(enumerator->super.enum_name);
     }
 }
 
 int mca_base_var_enum_register(const char *project_name, const char *framework_name,
-                               const char *component_name, const char *enum_name,
-                               void *storage)
+                               const char *component_name, const char *enum_name, void *storage)
 {
     int group_index;
 
     /* Developer error. Storage can not be NULL */
-    assert (NULL != storage);
+    assert(NULL != storage);
 
     /* Create a new parameter entry */
-    group_index = mca_base_var_group_register (project_name, framework_name, component_name,
-                                               NULL);
+    group_index = mca_base_var_group_register(project_name, framework_name, component_name, NULL);
     if (-1 > group_index) {
         return group_index;
     }
 
     if (0 <= group_index) {
-        mca_base_var_group_add_enum (group_index, storage);
+        mca_base_var_group_add_enum(group_index, storage);
     }
 
     return OPAL_SUCCESS;
diff --git a/opal/mca/base/mca_base_var_enum.h b/opal/mca/base/mca_base_var_enum.h
index 52ff045fc48..7fdfa8e64f0 100644
--- a/opal/mca/base/mca_base_var_enum.h
+++ b/opal/mca/base/mca_base_var_enum.h
@@ -15,6 +15,8 @@
  *                         reserved.
  * Copyright (c) 2017      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      Computer Architecture and VLSI Systems (CARV)
+ *                         Laboratory, ICS Forth. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -23,12 +25,27 @@
  */
 
 #if !defined(MCA_BASE_VAR_ENUM_H)
-#define MCA_BASE_VAR_ENUM_H
+#    define MCA_BASE_VAR_ENUM_H
 
-#include "opal_config.h"
+#    include "opal_config.h"
 
-#include "opal/class/opal_object.h"
-#include "opal/constants.h"
+#    include "opal/class/opal_object.h"
+#    include "opal/constants.h"
+#    include "opal/runtime/opal_params_core.h"
+
+/* Output mode for dumping var enumerators.
+ * Caution: Not 1:1 with mca_base_var_dump_type_t, appropriate
+ * conversion is required, see MCA_BASE_VAR_DUMP_TYPE_TO_ENUM_DUMP_TYPE() */
+typedef enum {
+    /* Dump human-readable strings */
+    MCA_BASE_VAR_ENUM_DUMP_READABLE = 0,
+    /* Dump human-readable strings, with color where supported */
+    MCA_BASE_VAR_ENUM_DUMP_READABLE_COLOR = 1
+} mca_base_var_enum_dump_type_t;
+
+#define MCA_BASE_VAR_DUMP_TYPE_TO_ENUM_DUMP_TYPE(var_dump_type) \
+    (MCA_BASE_VAR_DUMP_READABLE_COLOR == (var_dump_type) ? \
+    MCA_BASE_VAR_ENUM_DUMP_READABLE_COLOR : MCA_BASE_VAR_ENUM_DUMP_READABLE)
 
 typedef struct mca_base_var_enum_t mca_base_var_enum_t;
 
@@ -48,8 +65,8 @@ typedef int (*mca_base_var_enum_get_count_fn_t)(mca_base_var_enum_t *self, int *
  * @param[out] value integer value
  * @param[out] string_value string value
  */
-typedef int (*mca_base_var_enum_get_value_fn_t)(mca_base_var_enum_t *self, int index,
-                                                int *value, const char **string_value);
+typedef int (*mca_base_var_enum_get_value_fn_t)(mca_base_var_enum_t *self, int index, int *value,
+                                                const char **string_value);
 
 /**
  * Look up the integer value of a string
@@ -69,11 +86,13 @@ typedef int (*mca_base_var_enum_vfs_fn_t)(mca_base_var_enum_t *self, const char
  *
  * @param[in] self the enumerator
  * @param[out] out the string representation
+ * @param[in] output_type the type of output desired
  *
  * @retval OPAL_SUCCESS on success
  * @retval opal error on error
  */
-typedef int (*mca_base_var_enum_dump_fn_t)(mca_base_var_enum_t *self, char **out);
+typedef int (*mca_base_var_enum_dump_fn_t)(mca_base_var_enum_t *self, char **out,
+                                           mca_base_var_enum_dump_type_t output_type);
 
 /**
  * Get the string representation for an enumerator value
@@ -85,7 +104,7 @@ typedef int (*mca_base_var_enum_dump_fn_t)(mca_base_var_enum_t *self, char **out
  * @retval OPAL_SUCCESS on success
  * @retval OPAL_ERR_VALUE_OUT_OF_BOUNDS if not found
  *
- * @long This function returns the string value for a given interger value in the
+ * @long This function returns the string value for a given integer value in the
  * {string_value} parameter. The {string_value} parameter may be NULL in which case
  * no string is returned. If a string is returned in {string_value} the caller
  * must free the string with free().
@@ -140,7 +159,6 @@ struct mca_base_var_enum_t {
     mca_base_var_enum_value_t *enum_values;
 };
 
-
 /**
  * The default flag enumerator class takes in a list of integer-string pairs. If a
  * string is read from an environment variable or a file value the matching
@@ -199,8 +217,9 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(mca_base_var_enum_t);
  * strings passed in values[] after mca_base_var_enum_create()
  * returns.
  */
-OPAL_DECLSPEC int mca_base_var_enum_create (const char *name, const mca_base_var_enum_value_t values[],
-                                            mca_base_var_enum_t **enumerator);
+OPAL_DECLSPEC int mca_base_var_enum_create(const char *name,
+                                           const mca_base_var_enum_value_t values[],
+                                           mca_base_var_enum_t **enumerator);
 
 /**
  * Create a new default flag enumerator
@@ -226,8 +245,9 @@ OPAL_DECLSPEC int mca_base_var_enum_create (const char *name, const mca_base_var
  * strings passed in values[] after mca_base_var_enum_create()
  * returns.
  */
-OPAL_DECLSPEC int mca_base_var_enum_create_flag (const char *name, const mca_base_var_enum_value_flag_t flags[],
-                                                 mca_base_var_enum_flag_t **enumerator);
+OPAL_DECLSPEC int mca_base_var_enum_create_flag(const char *name,
+                                                const mca_base_var_enum_value_flag_t flags[],
+                                                mca_base_var_enum_flag_t **enumerator);
 
 OPAL_DECLSPEC int mca_base_var_enum_register(const char *project_name, const char *framework_name,
                                              const char *component_name, const char *enum_name,
diff --git a/opal/mca/base/mca_base_var_group.c b/opal/mca/base/mca_base_var_group.c
index d009e19c2a2..732df663e98 100644
--- a/opal/mca/base/mca_base_var_group.c
+++ b/opal/mca/base/mca_base_var_group.c
@@ -15,6 +15,8 @@
  *                         reserved.
  * Copyright (c) 2017      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2024      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -25,25 +27,25 @@
 #include "opal_config.h"
 
 #include <stdio.h>
-#include <string.h>
 #include <stdlib.h>
+#include <string.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 #ifdef HAVE_SYS_PARAM_H
-#include <sys/param.h>
+#    include <sys/param.h>
 #endif
 #include <errno.h>
 
+#include "opal/constants.h"
 #include "opal/include/opal_stdint.h"
-#include "opal/util/show_help.h"
-#include "opal/mca/mca.h"
-#include "opal/mca/base/mca_base_vari.h"
 #include "opal/mca/base/mca_base_pvar.h"
-#include "opal/constants.h"
-#include "opal/util/output.h"
-#include "opal/util/opal_environ.h"
+#include "opal/mca/base/mca_base_vari.h"
+#include "opal/mca/mca.h"
 #include "opal/runtime/opal.h"
+#include "opal/util/opal_environ.h"
+#include "opal/util/output.h"
+#include "opal/util/show_help.h"
 
 static opal_pointer_array_t mca_base_var_groups;
 static opal_hash_table_t mca_base_var_group_index_hash;
@@ -51,13 +53,12 @@ static int mca_base_var_group_count = 0;
 static int mca_base_var_groups_timestamp = 0;
 static bool mca_base_var_group_initialized = false;
 
-static void mca_base_var_group_constructor (mca_base_var_group_t *group);
-static void mca_base_var_group_destructor (mca_base_var_group_t *group);
-OBJ_CLASS_INSTANCE(mca_base_var_group_t, opal_object_t,
-                   mca_base_var_group_constructor,
+static void mca_base_var_group_constructor(mca_base_var_group_t *group);
+static void mca_base_var_group_destructor(mca_base_var_group_t *group);
+OBJ_CLASS_INSTANCE(mca_base_var_group_t, opal_object_t, mca_base_var_group_constructor,
                    mca_base_var_group_destructor);
 
-int mca_base_var_group_init (void)
+int mca_base_var_group_init(void)
 {
     int ret;
 
@@ -65,13 +66,13 @@ int mca_base_var_group_init (void)
         OBJ_CONSTRUCT(&mca_base_var_groups, opal_pointer_array_t);
 
         /* These values are arbitrary */
-        ret = opal_pointer_array_init (&mca_base_var_groups, 128, 16384, 128);
+        ret = opal_pointer_array_init(&mca_base_var_groups, 128, 16384, 128);
         if (OPAL_SUCCESS != ret) {
             return ret;
         }
 
         OBJ_CONSTRUCT(&mca_base_var_group_index_hash, opal_hash_table_t);
-        ret = opal_hash_table_init (&mca_base_var_group_index_hash, 256);
+        ret = opal_hash_table_init(&mca_base_var_group_index_hash, 256);
         if (OPAL_SUCCESS != ret) {
             return ret;
         }
@@ -83,15 +84,15 @@ int mca_base_var_group_init (void)
     return OPAL_SUCCESS;
 }
 
-int mca_base_var_group_finalize (void)
+int mca_base_var_group_finalize(void)
 {
     opal_object_t *object;
     int size, i;
 
     if (mca_base_var_group_initialized) {
         size = opal_pointer_array_get_size(&mca_base_var_groups);
-        for (i = 0 ; i < size ; ++i) {
-            object = opal_pointer_array_get_item (&mca_base_var_groups, i);
+        for (i = 0; i < size; ++i) {
+            object = opal_pointer_array_get_item(&mca_base_var_groups, i);
             if (NULL != object) {
                 OBJ_RELEASE(object);
             }
@@ -105,14 +106,15 @@ int mca_base_var_group_finalize (void)
     return OPAL_SUCCESS;
 }
 
-int mca_base_var_group_get_internal (const int group_index, mca_base_var_group_t **group, bool invalidok)
+int mca_base_var_group_get_internal(const int group_index, mca_base_var_group_t **group,
+                                    bool invalidok)
 {
     if (group_index < 0) {
         return OPAL_ERR_NOT_FOUND;
     }
 
-    *group = (mca_base_var_group_t *) opal_pointer_array_get_item (&mca_base_var_groups,
-                                                                   group_index);
+    *group = (mca_base_var_group_t *) opal_pointer_array_get_item(&mca_base_var_groups,
+                                                                  group_index);
     if (NULL == *group || (!invalidok && !(*group)->group_isvalid)) {
         *group = NULL;
         return OPAL_ERR_NOT_FOUND;
@@ -121,58 +123,58 @@ int mca_base_var_group_get_internal (const int group_index, mca_base_var_group_t
     return OPAL_SUCCESS;
 }
 
-static int group_find_by_name (const char *full_name, int *index, bool invalidok)
+static int group_find_by_name(const char *full_name, int *index, bool invalidok)
 {
     mca_base_var_group_t *group;
     void *tmp;
     int rc;
 
-    rc = opal_hash_table_get_value_ptr (&mca_base_var_group_index_hash, full_name,
-                                        strlen (full_name), &tmp);
+    rc = opal_hash_table_get_value_ptr(&mca_base_var_group_index_hash, full_name, strlen(full_name),
+                                       &tmp);
     if (OPAL_SUCCESS != rc) {
         return rc;
     }
 
-    rc = mca_base_var_group_get_internal ((int)(uintptr_t) tmp, &group, invalidok);
+    rc = mca_base_var_group_get_internal((int) (uintptr_t) tmp, &group, invalidok);
     if (OPAL_SUCCESS != rc) {
         return rc;
     }
 
     if (invalidok || group->group_isvalid) {
-        *index = (int)(uintptr_t) tmp;
+        *index = (int) (uintptr_t) tmp;
         return OPAL_SUCCESS;
     }
 
     return OPAL_ERR_NOT_FOUND;
 }
 
-static bool compare_strings (const char *str1, const char *str2) {
-    if ((NULL != str1 && 0 == strcmp (str1, "*")) ||
-        (NULL == str1 && NULL == str2)) {
+static bool compare_strings(const char *str1, const char *str2)
+{
+    if ((NULL != str1 && 0 == strcmp(str1, "*")) || (NULL == str1 && NULL == str2)) {
         return true;
     }
 
     if (NULL != str1 && NULL != str2) {
-        return 0 == strcmp (str1, str2);
+        return 0 == strcmp(str1, str2);
     }
 
     return false;
 }
 
-static int group_find_linear (const char *project_name, const char *framework_name,
-                              const char *component_name, bool invalidok)
+static int group_find_linear(const char *project_name, const char *framework_name,
+                             const char *component_name, bool invalidok)
 {
-    for (int i = 0 ; i < mca_base_var_group_count ; ++i) {
+    for (int i = 0; i < mca_base_var_group_count; ++i) {
         mca_base_var_group_t *group;
 
-        int rc = mca_base_var_group_get_internal (i, &group, invalidok);
+        int rc = mca_base_var_group_get_internal(i, &group, invalidok);
         if (OPAL_SUCCESS != rc) {
             continue;
         }
 
-        if (compare_strings (project_name, group->group_project) &&
-            compare_strings (framework_name, group->group_framework) &&
-            compare_strings (component_name, group->group_component)) {
+        if (compare_strings(project_name, group->group_project)
+            && compare_strings(framework_name, group->group_framework)
+            && compare_strings(component_name, group->group_component)) {
             return i;
         }
     }
@@ -180,36 +182,36 @@ static int group_find_linear (const char *project_name, const char *framework_na
     return OPAL_ERR_NOT_FOUND;
 }
 
-static int group_find (const char *project_name, const char *framework_name,
-                       const char *component_name, bool invalidok)
+static int group_find(const char *project_name, const char *framework_name,
+                      const char *component_name, bool invalidok)
 {
     char *full_name;
-    int ret, index=0;
+    int ret, index = 0;
 
     if (!mca_base_var_initialized) {
         return OPAL_ERR_NOT_FOUND;
     }
 
     /* check for wildcards */
-    if ((project_name && '*' == project_name[0]) || (framework_name && '*' == framework_name[0]) ||
-        (component_name && '*' == component_name[0])) {
-        return group_find_linear (project_name, framework_name, component_name, invalidok);
+    if ((project_name && '*' == project_name[0]) || (framework_name && '*' == framework_name[0])
+        || (component_name && '*' == component_name[0])) {
+        return group_find_linear(project_name, framework_name, component_name, invalidok);
     }
 
-    ret = mca_base_var_generate_full_name4(project_name, framework_name, component_name,
-                                           NULL, &full_name);
+    ret = mca_base_var_generate_full_name4(project_name, framework_name, component_name, NULL,
+                                           &full_name);
     if (OPAL_SUCCESS != ret) {
         return OPAL_ERROR;
     }
 
     ret = group_find_by_name(full_name, &index, invalidok);
-    free (full_name);
+    free(full_name);
 
     return (0 > ret) ? ret : index;
 }
 
-static int group_register (const char *project_name, const char *framework_name,
-                           const char *component_name, const char *description)
+static int group_register(const char *project_name, const char *framework_name,
+                          const char *component_name, const char *description)
 {
     mca_base_var_group_t *group;
     int group_id, parent_id = -1;
@@ -221,17 +223,17 @@ static int group_register (const char *project_name, const char *framework_name,
     }
 
     /* avoid groups of the form opal_opal, ompi_ompi, etc */
-    if (NULL != project_name && NULL != framework_name &&
-        (0 == strcmp (project_name, framework_name))) {
+    if (NULL != project_name && NULL != framework_name
+        && (0 == strcmp(project_name, framework_name))) {
         project_name = NULL;
     }
 
-    group_id = group_find (project_name, framework_name, component_name, true);
+    group_id = group_find(project_name, framework_name, component_name, true);
     if (0 <= group_id) {
-        ret = mca_base_var_group_get_internal (group_id, &group, true);
+        ret = mca_base_var_group_get_internal(group_id, &group, true);
         if (OPAL_SUCCESS != ret) {
             /* something went horribly wrong */
-            assert (NULL != group);
+            assert(NULL != group);
             return ret;
         }
         group->group_isvalid = true;
@@ -246,28 +248,28 @@ static int group_register (const char *project_name, const char *framework_name,
     group->group_isvalid = true;
 
     if (NULL != project_name) {
-        group->group_project = strdup (project_name);
+        group->group_project = strdup(project_name);
         if (NULL == group->group_project) {
             OBJ_RELEASE(group);
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
     }
     if (NULL != framework_name) {
-        group->group_framework = strdup (framework_name);
+        group->group_framework = strdup(framework_name);
         if (NULL == group->group_framework) {
             OBJ_RELEASE(group);
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
     }
     if (NULL != component_name) {
-        group->group_component = strdup (component_name);
+        group->group_component = strdup(component_name);
         if (NULL == group->group_component) {
             OBJ_RELEASE(group);
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
     }
     if (NULL != description) {
-        group->group_description = strdup (description);
+        group->group_description = strdup(description);
         if (NULL == group->group_description) {
             OBJ_RELEASE(group);
             return OPAL_ERR_OUT_OF_RESOURCE;
@@ -276,28 +278,28 @@ static int group_register (const char *project_name, const char *framework_name,
 
     if (NULL != framework_name && NULL != component_name) {
         if (component_name) {
-            parent_id = group_register (project_name, framework_name, NULL, NULL);
+            parent_id = group_register(project_name, framework_name, NULL, NULL);
         } else if (framework_name && project_name) {
-            parent_id = group_register (project_name, NULL, NULL, NULL);
+            parent_id = group_register(project_name, NULL, NULL, NULL);
         }
     }
 
     /* build the group name */
-    ret = mca_base_var_generate_full_name4 (NULL, project_name, framework_name, component_name,
-                                            &group->group_full_name);
+    ret = mca_base_var_generate_full_name4(NULL, project_name, framework_name, component_name,
+                                           &group->group_full_name);
     if (OPAL_SUCCESS != ret) {
         OBJ_RELEASE(group);
         return ret;
     }
 
-    group_id = opal_pointer_array_add (&mca_base_var_groups, group);
+    group_id = opal_pointer_array_add(&mca_base_var_groups, group);
     if (0 > group_id) {
         OBJ_RELEASE(group);
         return OPAL_ERROR;
     }
 
-    opal_hash_table_set_value_ptr (&mca_base_var_group_index_hash, group->group_full_name,
-                                   strlen (group->group_full_name), (void *)(uintptr_t) group_id);
+    opal_hash_table_set_value_ptr(&mca_base_var_group_index_hash, group->group_full_name,
+                                  strlen(group->group_full_name), (void *) (uintptr_t) group_id);
 
     mca_base_var_group_count++;
     mca_base_var_groups_timestamp++;
@@ -306,34 +308,33 @@ static int group_register (const char *project_name, const char *framework_name,
         mca_base_var_group_t *parent_group;
 
         (void) mca_base_var_group_get_internal(parent_id, &parent_group, false);
-        opal_value_array_append_item (&parent_group->group_subgroups, &group_id);
+        opal_value_array_append_item(&parent_group->group_subgroups, &group_id);
     }
 
     return group_id;
 }
 
-int mca_base_var_group_register (const char *project_name, const char *framework_name,
-                                 const char *component_name, const char *description)
+int mca_base_var_group_register(const char *project_name, const char *framework_name,
+                                const char *component_name, const char *description)
 {
-    return group_register (project_name, framework_name, component_name, description);
+    return group_register(project_name, framework_name, component_name, description);
 }
 
-int mca_base_var_group_component_register (const mca_base_component_t *component,
-                                           const char *description)
+int mca_base_var_group_component_register(const mca_base_component_t *component,
+                                          const char *description)
 {
-    return group_register (component->mca_project_name, component->mca_type_name,
-                           component->mca_component_name, description);
+    return group_register(component->mca_project_name, component->mca_type_name,
+                          component->mca_component_name, description);
 }
 
-
-int mca_base_var_group_deregister (int group_index)
+int mca_base_var_group_deregister(int group_index)
 {
     mca_base_var_group_t *group;
     int size, ret;
     int *params, *subgroups;
-    opal_object_t ** enums;
+    opal_object_t **enums;
 
-    ret = mca_base_var_group_get_internal (group_index, &group, false);
+    ret = mca_base_var_group_get_internal(group_index, &group, false);
     if (OPAL_SUCCESS != ret) {
         return ret;
     }
@@ -344,42 +345,44 @@ int mca_base_var_group_deregister (int group_index)
     size = opal_value_array_get_size(&group->group_vars);
     params = OPAL_VALUE_ARRAY_GET_BASE(&group->group_vars, int);
 
-    for (int i = 0 ; i < size ; ++i) {
+    for (int i = 0; i < size; ++i) {
         const mca_base_var_t *var;
 
-        ret = mca_base_var_get (params[i], &var);
+        ret = mca_base_var_get(params[i], &var);
         if (OPAL_SUCCESS != ret || !(var->mbv_flags & MCA_BASE_VAR_FLAG_DWG)) {
             continue;
         }
 
-        (void) mca_base_var_deregister (params[i]);
+        (void) mca_base_var_deregister(params[i]);
     }
 
     /* invalidate all associated mca performance variables */
     size = opal_value_array_get_size(&group->group_pvars);
     params = OPAL_VALUE_ARRAY_GET_BASE(&group->group_pvars, int);
 
-    for (int i = 0 ; i < size ; ++i) {
+    for (int i = 0; i < size; ++i) {
         const mca_base_pvar_t *var;
 
-        ret = mca_base_pvar_get (params[i], &var);
+        ret = mca_base_pvar_get(params[i], &var);
         if (OPAL_SUCCESS != ret || !(var->flags & MCA_BASE_PVAR_FLAG_IWG)) {
             continue;
         }
 
-        (void) mca_base_pvar_mark_invalid (params[i]);
+        (void) mca_base_pvar_mark_invalid(params[i]);
     }
 
     size = opal_value_array_get_size(&group->group_enums);
     enums = OPAL_VALUE_ARRAY_GET_BASE(&group->group_enums, opal_object_t *);
-    for (int i = 0 ; i < size ; ++i) {
-        OBJ_RELEASE (enums[i]);
+    for (int i = 0; i < size; ++i) {
+        OBJ_RELEASE(enums[i]);
     }
+    opal_value_array_set_size(&group->group_enums, 0);
+
 
     size = opal_value_array_get_size(&group->group_subgroups);
     subgroups = OPAL_VALUE_ARRAY_GET_BASE(&group->group_subgroups, int);
-    for (int i = 0 ; i < size ; ++i) {
-        (void) mca_base_var_group_deregister (subgroups[i]);
+    for (int i = 0; i < size; ++i) {
+        (void) mca_base_var_group_deregister(subgroups[i]);
     }
     /* ordering of variables and subgroups must be the same if the
      * group is re-registered */
@@ -389,118 +392,114 @@ int mca_base_var_group_deregister (int group_index)
     return OPAL_SUCCESS;
 }
 
-int mca_base_var_group_find (const char *project_name,
-                             const char *framework_name,
-                             const char *component_name)
+int mca_base_var_group_find(const char *project_name, const char *framework_name,
+                            const char *component_name)
 {
-    return group_find (project_name, framework_name, component_name, false);
+    return group_find(project_name, framework_name, component_name, false);
 }
 
-int mca_base_var_group_find_by_name (const char *full_name, int *index)
+int mca_base_var_group_find_by_name(const char *full_name, int *index)
 {
-    return group_find_by_name (full_name, index, false);
+    return group_find_by_name(full_name, index, false);
 }
 
-int mca_base_var_group_add_var (const int group_index, const int param_index)
+int mca_base_var_group_add_var(const int group_index, const int param_index)
 {
     mca_base_var_group_t *group;
     int size, i, ret;
     int *params;
 
-    ret = mca_base_var_group_get_internal (group_index, &group, false);
+    ret = mca_base_var_group_get_internal(group_index, &group, false);
     if (OPAL_SUCCESS != ret) {
         return ret;
     }
 
     size = opal_value_array_get_size(&group->group_vars);
     params = OPAL_VALUE_ARRAY_GET_BASE(&group->group_vars, int);
-    for (i = 0 ; i < size ; ++i) {
+    for (i = 0; i < size; ++i) {
         if (params[i] == param_index) {
             return i;
         }
     }
 
-    if (OPAL_SUCCESS !=
-        (ret = opal_value_array_append_item (&group->group_vars, &param_index))) {
+    if (OPAL_SUCCESS != (ret = opal_value_array_append_item(&group->group_vars, &param_index))) {
         return ret;
     }
 
     mca_base_var_groups_timestamp++;
 
     /* return the group index */
-    return (int) opal_value_array_get_size (&group->group_vars) - 1;
+    return (int) opal_value_array_get_size(&group->group_vars) - 1;
 }
 
-int mca_base_var_group_add_pvar (const int group_index, const int param_index)
+int mca_base_var_group_add_pvar(const int group_index, const int param_index)
 {
     mca_base_var_group_t *group;
     int size, i, ret;
     int *params;
 
-    ret = mca_base_var_group_get_internal (group_index, &group, false);
+    ret = mca_base_var_group_get_internal(group_index, &group, false);
     if (OPAL_SUCCESS != ret) {
         return ret;
     }
 
     size = opal_value_array_get_size(&group->group_pvars);
     params = OPAL_VALUE_ARRAY_GET_BASE(&group->group_pvars, int);
-    for (i = 0 ; i < size ; ++i) {
+    for (i = 0; i < size; ++i) {
         if (params[i] == param_index) {
             return i;
         }
     }
 
-    if (OPAL_SUCCESS !=
-        (ret = opal_value_array_append_item (&group->group_pvars, &param_index))) {
+    if (OPAL_SUCCESS != (ret = opal_value_array_append_item(&group->group_pvars, &param_index))) {
         return ret;
     }
 
     mca_base_var_groups_timestamp++;
 
     /* return the group index */
-    return (int) opal_value_array_get_size (&group->group_pvars) - 1;
+    return (int) opal_value_array_get_size(&group->group_pvars) - 1;
 }
 
-int mca_base_var_group_add_enum (const int group_index, const void * storage)
+int mca_base_var_group_add_enum(const int group_index, const void *storage)
 {
     mca_base_var_group_t *group;
     int size, i, ret;
     void **params;
 
-    ret = mca_base_var_group_get_internal (group_index, &group, false);
+    ret = mca_base_var_group_get_internal(group_index, &group, false);
     if (OPAL_SUCCESS != ret) {
         return ret;
     }
 
     size = opal_value_array_get_size(&group->group_enums);
     params = OPAL_VALUE_ARRAY_GET_BASE(&group->group_enums, void *);
-    for (i = 0 ; i < size ; ++i) {
+    for (i = 0; i < size; ++i) {
         if (params[i] == storage) {
             return i;
         }
     }
 
-    if (OPAL_SUCCESS !=
-        (ret = opal_value_array_append_item (&group->group_enums, storage))) {
+    if (OPAL_SUCCESS != (ret = opal_value_array_append_item(&group->group_enums, storage))) {
         return ret;
     }
 
     /* return the group index */
-    return (int) opal_value_array_get_size (&group->group_enums) - 1;
+    return (int) opal_value_array_get_size(&group->group_enums) - 1;
 }
 
-int mca_base_var_group_get (const int group_index, const mca_base_var_group_t **group)
+int mca_base_var_group_get(const int group_index, const mca_base_var_group_t **group)
 {
-    return mca_base_var_group_get_internal (group_index, (mca_base_var_group_t **) group, false);
+    return mca_base_var_group_get_internal(group_index, (mca_base_var_group_t **) group, false);
 }
 
-int mca_base_var_group_set_var_flag (const int group_index, int flags, bool set)
+int mca_base_var_group_set_var_flag(const int group_index, int flags, bool set)
 {
     mca_base_var_group_t *group;
     int size, i, ret;
     int *vars;
 
-    ret = mca_base_var_group_get_internal (group_index, &group, false);
+    ret = mca_base_var_group_get_internal(group_index, &group, false);
     if (OPAL_SUCCESS != ret) {
         return ret;
     }
@@ -509,48 +508,47 @@ int mca_base_var_group_set_var_flag (const int group_index, int flags, bool set)
     size = opal_value_array_get_size(&group->group_vars);
     vars = OPAL_VALUE_ARRAY_GET_BASE(&group->group_vars, int);
 
-    for (i = 0 ; i < size ; ++i) {
+    for (i = 0; i < size; ++i) {
         if (0 <= vars[i]) {
-            (void) mca_base_var_set_flag (vars[i], flags, set);
+            (void) mca_base_var_set_flag(vars[i], flags, set);
         }
     }
 
     return OPAL_SUCCESS;
 }
 
-
-static void mca_base_var_group_constructor (mca_base_var_group_t *group)
+static void mca_base_var_group_constructor(mca_base_var_group_t *group)
 {
-    memset ((char *) group + sizeof (group->super), 0, sizeof (*group) - sizeof (group->super));
+    memset((char *) group + sizeof(group->super), 0, sizeof(*group) - sizeof(group->super));
 
     OBJ_CONSTRUCT(&group->group_subgroups, opal_value_array_t);
-    opal_value_array_init (&group->group_subgroups, sizeof (int));
+    opal_value_array_init(&group->group_subgroups, sizeof(int));
 
     OBJ_CONSTRUCT(&group->group_vars, opal_value_array_t);
-    opal_value_array_init (&group->group_vars, sizeof (int));
+    opal_value_array_init(&group->group_vars, sizeof(int));
 
     OBJ_CONSTRUCT(&group->group_pvars, opal_value_array_t);
-    opal_value_array_init (&group->group_pvars, sizeof (int));
+    opal_value_array_init(&group->group_pvars, sizeof(int));
 
     OBJ_CONSTRUCT(&group->group_enums, opal_value_array_t);
-    opal_value_array_init (&group->group_enums, sizeof(void *));
+    opal_value_array_init(&group->group_enums, sizeof(void *));
 }
 
-static void mca_base_var_group_destructor (mca_base_var_group_t *group)
+static void mca_base_var_group_destructor(mca_base_var_group_t *group)
 {
-    free (group->group_full_name);
+    free(group->group_full_name);
     group->group_full_name = NULL;
 
-    free (group->group_description);
+    free(group->group_description);
     group->group_description = NULL;
 
-    free (group->group_project);
+    free(group->group_project);
     group->group_project = NULL;
 
-    free (group->group_framework);
+    free(group->group_framework);
     group->group_framework = NULL;
 
-    free (group->group_component);
+    free(group->group_component);
     group->group_component = NULL;
 
     OBJ_DESTRUCT(&group->group_subgroups);
@@ -559,12 +557,12 @@ static void mca_base_var_group_destructor (mca_base_var_group_t *group)
     OBJ_DESTRUCT(&group->group_enums);
 }
 
-int mca_base_var_group_get_count (void)
+int mca_base_var_group_get_count(void)
 {
     return mca_base_var_group_count;
 }
 
-int mca_base_var_group_get_stamp (void)
+int mca_base_var_group_get_stamp(void)
 {
     return mca_base_var_groups_timestamp;
 }
diff --git a/opal/mca/base/mca_base_var_group.h b/opal/mca/base/mca_base_var_group.h
index e0cdbae5e6e..71261b16940 100644
--- a/opal/mca/base/mca_base_var_group.h
+++ b/opal/mca/base/mca_base_var_group.h
@@ -72,7 +72,7 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(mca_base_var_group_t);
  * @param[in] project_name Project name for this group.
  * @param[in] framework_name Framework name for this group.
  * @param[in] component_name Component name for this group.
- * @param[in] descrition Description of this group.
+ * @param[in] description Description of this group.
  *
  * @retval index Unique group index
  * @return opal error code on Error
@@ -80,10 +80,8 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(mca_base_var_group_t);
  * Create an MCA variable group. If the group already exists
  * this call is equivalent to mca_base_ver_find_group().
  */
-OPAL_DECLSPEC int mca_base_var_group_register(const char *project_name,
-                                              const char *framework_name,
-                                              const char *component_name,
-                                              const char *description);
+OPAL_DECLSPEC int mca_base_var_group_register(const char *project_name, const char *framework_name,
+                                              const char *component_name, const char *description);
 
 /**
  * Register an MCA variable group for a component
@@ -95,8 +93,8 @@ OPAL_DECLSPEC int mca_base_var_group_register(const char *project_name,
  * @retval index Unique group index
  * @return opal error code on Error
  */
-OPAL_DECLSPEC int mca_base_var_group_component_register (const mca_base_component_t *component,
-                                                         const char *description);
+OPAL_DECLSPEC int mca_base_var_group_component_register(const mca_base_component_t *component,
+                                                        const char *description);
 
 /**
  * Deregister an MCA param group
@@ -106,7 +104,7 @@ OPAL_DECLSPEC int mca_base_var_group_component_register (const mca_base_componen
  *
  * This call deregisters all associated variables and subgroups.
  */
-OPAL_DECLSPEC int mca_base_var_group_deregister (int group_index);
+OPAL_DECLSPEC int mca_base_var_group_deregister(int group_index);
 
 /**
  * Find an MCA group
@@ -118,9 +116,8 @@ OPAL_DECLSPEC int mca_base_var_group_deregister (int group_index);
  * @returns OPAL_SUCCESS if found
  * @returns OPAL_ERR_NOT_FOUND if not found
  */
-OPAL_DECLSPEC int mca_base_var_group_find (const char *project_name,
-                                           const char *framework_name,
-                                           const char *component_name);
+OPAL_DECLSPEC int mca_base_var_group_find(const char *project_name, const char *framework_name,
+                                          const char *component_name);
 
 /**
  * Find an MCA group by its full name
@@ -131,7 +128,7 @@ OPAL_DECLSPEC int mca_base_var_group_find (const char *project_name,
  * @returns OPAL_SUCCESS if found
  * @returns OPAL_ERR_NOT_FOUND if not found
  */
-OPAL_DECLSPEC int mca_base_var_group_find_by_name (const char *full_name, int *index);
+OPAL_DECLSPEC int mca_base_var_group_find_by_name(const char *full_name, int *index);
 
 /**
  * Get the group at a specified index
@@ -145,8 +142,7 @@ OPAL_DECLSPEC int mca_base_var_group_find_by_name (const char *full_name, int *i
  * The returned pointer belongs to the MCA variable system. Do not modify/release/retain
  * the pointer.
  */
-OPAL_DECLSPEC int mca_base_var_group_get (const int group_index,
-                                          const mca_base_var_group_t **group);
+OPAL_DECLSPEC int mca_base_var_group_get(const int group_index, const mca_base_var_group_t **group);
 
 /**
  * Set/unset a flags for all variables in a group.
@@ -157,15 +153,14 @@ OPAL_DECLSPEC int mca_base_var_group_get (const int group_index,
  *
  * Set a flag for every variable in a group. See mca_base_var_set_flag() for more info.
  */
-OPAL_DECLSPEC int mca_base_var_group_set_var_flag (const int group_index, int flags,
-                                                   bool set);
+OPAL_DECLSPEC int mca_base_var_group_set_var_flag(const int group_index, int flags, bool set);
 
 /**
  * Get the number of registered MCA groups
  *
  * @retval count Number of registered MCA groups
  */
-OPAL_DECLSPEC int mca_base_var_group_get_count (void);
+OPAL_DECLSPEC int mca_base_var_group_get_count(void);
 
 /**
  * Get a relative timestamp for the MCA group system
@@ -174,6 +169,6 @@ OPAL_DECLSPEC int mca_base_var_group_get_count (void);
  *
  * This value will change if groups or variables are either added or removed.
  */
-OPAL_DECLSPEC int mca_base_var_group_get_stamp (void);
+OPAL_DECLSPEC int mca_base_var_group_get_stamp(void);
 
 #endif /* OPAL_MCA_BASE_VAR_GROUP_H */
diff --git a/opal/mca/base/mca_base_vari.h b/opal/mca/base/mca_base_vari.h
index 51f879dfda9..cc89842fdd8 100644
--- a/opal/mca/base/mca_base_vari.h
+++ b/opal/mca/base/mca_base_vari.h
@@ -41,13 +41,13 @@
 
 #include "opal_config.h"
 
-#include "opal/class/opal_object.h"
+#include "opal/class/opal_hash_table.h"
 #include "opal/class/opal_list.h"
-#include "opal/class/opal_value_array.h"
+#include "opal/class/opal_object.h"
 #include "opal/class/opal_pointer_array.h"
-#include "opal/class/opal_hash_table.h"
-#include "opal/mca/base/mca_base_var.h"
+#include "opal/class/opal_value_array.h"
 #include "opal/mca/base/mca_base_pvar.h"
+#include "opal/mca/base/mca_base_var.h"
 
 BEGIN_C_DECLS
 
@@ -56,7 +56,7 @@ BEGIN_C_DECLS
 
 typedef enum {
     /** Variable is valid */
-    MCA_BASE_VAR_FLAG_VALID   = 0x00010000,
+    MCA_BASE_VAR_FLAG_VALID = 0x00010000,
     /** Variable is a synonym */
     MCA_BASE_VAR_FLAG_SYNONYM = 0x00020000,
     /** mbv_source_file needs to be freed */
@@ -65,12 +65,12 @@ typedef enum {
 
 #define VAR_FLAG_ISSET(var, flag) (!!((var).mbp_flags & (flag)))
 
-#define VAR_IS_VALID(var) (!!((var).mbv_flags & MCA_BASE_VAR_FLAG_VALID))
-#define VAR_IS_SYNONYM(var) (!!((var).mbv_flags & MCA_BASE_VAR_FLAG_SYNONYM))
-#define VAR_IS_INTERNAL(var) (!!((var).mbv_flags & MCA_BASE_VAR_FLAG_INTERNAL))
+#define VAR_IS_VALID(var)        (!!((var).mbv_flags & MCA_BASE_VAR_FLAG_VALID))
+#define VAR_IS_SYNONYM(var)      (!!((var).mbv_flags & MCA_BASE_VAR_FLAG_SYNONYM))
+#define VAR_IS_INTERNAL(var)     (!!((var).mbv_flags & MCA_BASE_VAR_FLAG_INTERNAL))
 #define VAR_IS_DEFAULT_ONLY(var) (!!((var).mbv_flags & MCA_BASE_VAR_FLAG_DEFAULT_ONLY))
-#define VAR_IS_SETTABLE(var) (!!((var).mbv_flags & MCA_BASE_VAR_FLAG_SETTABLE))
-#define VAR_IS_DEPRECATED(var) (!!((var).mbv_flags & MCA_BASE_VAR_FLAG_DEPRECATED))
+#define VAR_IS_SETTABLE(var)     (!!((var).mbv_flags & MCA_BASE_VAR_FLAG_SETTABLE))
+#define VAR_IS_DEPRECATED(var)   (!!((var).mbv_flags & MCA_BASE_VAR_FLAG_DEPRECATED))
 
 extern const char *ompi_var_type_names[];
 extern const size_t ompi_var_type_sizes[];
@@ -116,7 +116,8 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(mca_base_var_file_value_t);
  * @param[out] group       Returned group if it exists
  * @param[in]  invalidok   Return group even if it has been deregistered
  */
-OPAL_DECLSPEC int mca_base_var_group_get_internal (const int group_index, mca_base_var_group_t **group, bool invalidok);
+OPAL_DECLSPEC int mca_base_var_group_get_internal(const int group_index,
+                                                  mca_base_var_group_t **group, bool invalidok);
 
 /**
  * \internal
@@ -130,30 +131,30 @@ OPAL_DECLSPEC int mca_base_parse_paramfile(const char *paramfile, opal_list_t *l
  *
  * Add a variable to a group
  */
-OPAL_DECLSPEC int mca_base_var_group_add_var (const int group_index, const int param_index);
+OPAL_DECLSPEC int mca_base_var_group_add_var(const int group_index, const int param_index);
 
 /**
  * \internal
  *
  * Add a performance variable to a group
  */
-OPAL_DECLSPEC int mca_base_var_group_add_pvar (const int group_index, const int param_index);
+OPAL_DECLSPEC int mca_base_var_group_add_pvar(const int group_index, const int param_index);
 
 /**
  * \internal
  *
  * Add an enum to a group
  */
-OPAL_DECLSPEC int mca_base_var_group_add_enum (const int group_index, const void *storage);
+OPAL_DECLSPEC int mca_base_var_group_add_enum(const int group_index, const void *storage);
 
 /**
  * \internal
  *
  * Generate a full name with _ between all of the non-NULL arguments
  */
-OPAL_DECLSPEC int mca_base_var_generate_full_name4 (const char *project, const char *framework,
-                                                    const char *component, const char *variable,
-                                                    char **full_name);
+OPAL_DECLSPEC int mca_base_var_generate_full_name4(const char *project, const char *framework,
+                                                   const char *component, const char *variable,
+                                                   char **full_name);
 
 /**
  * \internal
@@ -167,16 +168,16 @@ OPAL_DECLSPEC int mca_base_internal_env_store(void);
  *
  * Initialize/finalize MCA variable groups
  */
-OPAL_DECLSPEC int mca_base_var_group_init (void);
-OPAL_DECLSPEC int mca_base_var_group_finalize (void);
+OPAL_DECLSPEC int mca_base_var_group_init(void);
+OPAL_DECLSPEC int mca_base_var_group_finalize(void);
 
 /**
  * \internal
  *
  * Initialize MCA performance variables
  */
-OPAL_DECLSPEC int mca_base_pvar_init (void);
-OPAL_DECLSPEC int mca_base_pvar_finalize (void);
+OPAL_DECLSPEC int mca_base_pvar_init(void);
+OPAL_DECLSPEC int mca_base_pvar_finalize(void);
 
 END_C_DECLS
 
diff --git a/opal/mca/btl/base/Makefile.am b/opal/mca/btl/base/Makefile.am
index 84b375f8ac9..7019bc1983a 100644
--- a/opal/mca/btl/base/Makefile.am
+++ b/opal/mca/btl/base/Makefile.am
@@ -11,6 +11,7 @@
 #                         All rights reserved.
 # Copyright (c) 2021      Triad National Security, LLC. All rights reserved.
 # Copyright (c) 2021      Google, LLC. All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -18,7 +19,7 @@
 # $HEADER$
 #
 
-dist_opaldata_DATA = base/help-mpi-btl-base.txt
+EXTRA_DIST = base/help-mpi-btl-base.txt
 
 headers += \
         base/base.h \
diff --git a/opal/mca/btl/base/base.h b/opal/mca/btl/base/base.h
index 76d3d8b9b9c..09c4502ef85 100644
--- a/opal/mca/btl/base/base.h
+++ b/opal/mca/btl/base/base.h
@@ -26,8 +26,8 @@
 
 #include "opal_config.h"
 #include "opal/class/opal_list.h"
-#include "opal/mca/mca.h"
 #include "opal/mca/btl/btl.h"
+#include "opal/mca/mca.h"
 
 #define MCA_BTL_BASE_TAG_RDMA      MCA_BTL_TAG_BTL_BASE
 #define MCA_BTL_BASE_TAG_ATOMIC    (MCA_BTL_TAG_BTL_BASE + 1)
@@ -36,43 +36,39 @@
 BEGIN_C_DECLS
 
 struct mca_btl_base_selected_module_t {
-  opal_list_item_t super;
-  mca_btl_base_component_t *btl_component;
-  mca_btl_base_module_t *btl_module;
+    opal_list_item_t super;
+    mca_btl_base_component_t *btl_component;
+    mca_btl_base_module_t *btl_module;
 };
 typedef struct mca_btl_base_selected_module_t mca_btl_base_selected_module_t;
 
-
 /* holds the recv call back function to be called by the btl on
  * a receive.
  */
 struct mca_btl_base_recv_reg_t {
     mca_btl_base_module_recv_cb_fn_t cbfunc;
-    void* cbdata;
+    void *cbdata;
 };
 typedef struct mca_btl_base_recv_reg_t mca_btl_base_recv_reg_t;
 
-
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(mca_btl_base_selected_module_t);
 
 /*
  * Global functions for MCA: overall BTL open and close
  */
 
-OPAL_DECLSPEC  int mca_btl_base_select(bool enable_progress_threads, bool enable_mpi_threads);
-OPAL_DECLSPEC  void mca_btl_base_dump(
-    struct mca_btl_base_module_t*,
-    struct mca_btl_base_endpoint_t*,
-    int verbose);
-OPAL_DECLSPEC  int mca_btl_base_param_register(mca_base_component_t *version,
-        mca_btl_base_module_t *module);
-OPAL_DECLSPEC  int mca_btl_base_param_verify(mca_btl_base_module_t *module);
+OPAL_DECLSPEC int mca_btl_base_select(bool enable_progress_threads, bool enable_mpi_threads);
+OPAL_DECLSPEC void mca_btl_base_dump(struct mca_btl_base_module_t *,
+                                     struct mca_btl_base_endpoint_t *, int verbose);
+OPAL_DECLSPEC int mca_btl_base_param_register(mca_base_component_t *version,
+                                              mca_btl_base_module_t *module);
+OPAL_DECLSPEC int mca_btl_base_param_verify(mca_btl_base_module_t *module);
 
 /*
  * Globals
  */
-extern char* mca_btl_base_include;
-extern char* mca_btl_base_exclude;
+extern char *mca_btl_base_include;
+extern char *mca_btl_base_exclude;
 extern int mca_btl_base_warn_component_unused;
 extern int mca_btl_base_already_opened;
 OPAL_DECLSPEC extern opal_list_t mca_btl_base_modules_initialized;
diff --git a/opal/mca/btl/base/btl_base_am_rdma.c b/opal/mca/btl/base/btl_base_am_rdma.c
index 80b6af6add5..2d8dd41f764 100644
--- a/opal/mca/btl/base/btl_base_am_rdma.c
+++ b/opal/mca/btl/base/btl_base_am_rdma.c
@@ -3,6 +3,9 @@
  * Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2020-2021 Google, LLC. All rights reserved.
+ * Copyright (c) 2021-2022 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -10,18 +13,27 @@
  * $HEADER$
  */
 
+#include "opal_config.h"
+
+#include <assert.h>
+
 #include "opal/mca/btl/base/btl_base_am_rdma.h"
 #include "opal/mca/btl/base/base.h"
 #include "opal/mca/btl/base/btl_base_error.h"
 #include "opal/mca/threads/mutex.h"
+#include "opal/util/minmax.h"
+
 
 /**
- * @brief data for active-message atomics
+ * @brief global data for active message wrapper
  *
- * There is currently only one module but it is defined to allow
- * moving the data pointer into the associated BTL module.
+ * While individual entries in queued_responses and
+ * queued_initiator_descriptors are module-specific (ie, per BTL
+ * module), they are progressed in a common progress function.  It is
+ * much more efficient to have one list of work to do, rather than
+ * having to poll through all active btls to find the work to do.
  */
-struct mca_btl_base_am_rdma_module_t {
+struct am_rdma_component_t {
     opal_object_t super;
     /** provides protection for multi-threaded access to module members */
     opal_mutex_t mutex;
@@ -30,29 +42,36 @@ struct mca_btl_base_am_rdma_module_t {
     /** queued initiator descriptors */
     opal_list_t queued_initiator_descriptors;
 };
-typedef struct mca_btl_base_am_rdma_module_t mca_btl_base_am_rdma_module_t;
+typedef struct am_rdma_component_t am_rdma_component_t;
 
-static void mca_btl_base_am_rdma_module_init (mca_btl_base_am_rdma_module_t *module)
+static am_rdma_component_t default_component;
+
+static void am_rdma_component_init(am_rdma_component_t *component)
 {
-    OBJ_CONSTRUCT (&module->mutex, opal_mutex_t);
-    OBJ_CONSTRUCT (&module->queued_responses, opal_list_t);
-    OBJ_CONSTRUCT (&module->queued_initiator_descriptors, opal_list_t);
+    OBJ_CONSTRUCT(&component->mutex, opal_mutex_t);
+    OBJ_CONSTRUCT(&component->queued_responses, opal_list_t);
+    OBJ_CONSTRUCT(&component->queued_initiator_descriptors, opal_list_t);
 }
 
-static void mca_btl_base_am_rdma_module_fini (mca_btl_base_am_rdma_module_t *module)
+static void am_rdma_component_fini(am_rdma_component_t *component)
 {
-    OBJ_DESTRUCT (&module->mutex);
-    OBJ_DESTRUCT (&module->queued_responses);
-    OBJ_DESTRUCT (&module->queued_initiator_descriptors);
+    OBJ_DESTRUCT(&component->mutex);
+    OBJ_DESTRUCT(&component->queued_responses);
+    OBJ_DESTRUCT(&component->queued_initiator_descriptors);
 }
 
-static OBJ_CLASS_INSTANCE (mca_btl_base_am_rdma_module_t, opal_object_t,
-                           mca_btl_base_am_rdma_module_init, mca_btl_base_am_rdma_module_fini);
+static OBJ_CLASS_INSTANCE(am_rdma_component_t, opal_object_t,
+                          am_rdma_component_init, am_rdma_component_fini);
+
+
+OBJ_CLASS_INSTANCE(mca_btl_base_am_rdma_module_t, opal_object_t,
+                   NULL, NULL);
+
 
 /**
  * @brief response header for an active-message RDMA/atomic operation
  */
-struct mca_btl_base_rdma_response_hdr_t {
+struct am_rdma_response_hdr_t {
     /** context for the response */
     uint64_t context;
     /** initiator address */
@@ -62,12 +81,13 @@ struct mca_btl_base_rdma_response_hdr_t {
     /** response data may follow. the size is implied by the size of the incoming
      * descriptor */
 };
-typedef struct mca_btl_base_rdma_response_hdr_t mca_btl_base_rdma_response_hdr_t;
+typedef struct am_rdma_response_hdr_t am_rdma_response_hdr_t;
+
 
 /**
  * @brief type of active-message RDMA/atomic operation
  */
-enum mca_btl_base_rdma_type_t {
+enum am_rdma_type_t {
     /** active-message put. May be implemented with send/recv or RDMA get
      * depending on the functions that the BTL implements. */
     MCA_BTL_BASE_AM_PUT,
@@ -79,12 +99,13 @@ enum mca_btl_base_rdma_type_t {
     /** compare-and-swap */
     MCA_BTL_BASE_AM_CAS,
 };
-typedef enum mca_btl_base_rdma_type_t mca_btl_base_rdma_type_t;
+typedef enum am_rdma_type_t am_rdma_type_t;
+
 
 /**
  * @brief origin-side operation context for an active-message RDMA/atomic operation
  */
-struct mca_btl_base_rdma_context_t {
+struct am_rdma_context_t {
     opal_object_t super;
     /** operation type */
     uint8_t type;
@@ -109,35 +130,37 @@ struct mca_btl_base_rdma_context_t {
     /** local handle for this request */
     struct mca_btl_base_registration_handle_t *local_handle;
 };
-typedef struct mca_btl_base_rdma_context_t mca_btl_base_rdma_context_t;
+typedef struct am_rdma_context_t am_rdma_context_t;
 
-static void mca_btl_base_rdma_context_init (mca_btl_base_rdma_context_t *context)
+static void am_rdma_context_init(am_rdma_context_t *context)
 {
-    context->sent         = 0;
+    context->sent = 0;
     context->acknowledged = 0;
-    context->descriptor   = NULL;
+    context->descriptor = NULL;
 }
 
-static OBJ_CLASS_INSTANCE (mca_btl_base_rdma_context_t, opal_object_t,
-                           mca_btl_base_rdma_context_init, NULL);
+static OBJ_CLASS_INSTANCE(am_rdma_context_t, opal_object_t,
+                          am_rdma_context_init, NULL);
+
 
 /**
  * @brief queued initiator descriptor
  */
-struct mca_btl_base_am_rdma_queued_descriptor_t {
+struct am_rdma_queued_descriptor_t {
     opal_list_item_t super;
-    mca_btl_base_module_t *btl;
+    mca_btl_base_am_rdma_module_t *am_module;
     struct mca_btl_base_endpoint_t *endpoint;
     mca_btl_base_descriptor_t *descriptor;
 };
-typedef struct mca_btl_base_am_rdma_queued_descriptor_t mca_btl_base_am_rdma_queued_descriptor_t;
+typedef struct am_rdma_queued_descriptor_t am_rdma_queued_descriptor_t;
+
+static OBJ_CLASS_INSTANCE(am_rdma_queued_descriptor_t, opal_list_item_t, NULL, NULL);
 
-static OBJ_CLASS_INSTANCE (mca_btl_base_am_rdma_queued_descriptor_t, opal_list_item_t, NULL, NULL);
 
 /**
  * @brief header for an active-message atomic/RDMA operation
  */
-struct mca_btl_base_rdma_hdr_t {
+struct am_rdma_hdr_t {
     /** type of operation requested. */
     uint8_t type;
     uint8_t padding[3];
@@ -175,16 +198,13 @@ struct mca_btl_base_rdma_hdr_t {
     /* the following fields are not used on the target and are only relevant
      * to the initiator */
     uint64_t context;
-
-    /* registration handles (if required) */
-    uint8_t handle_data[];
 };
-typedef struct mca_btl_base_rdma_hdr_t mca_btl_base_rdma_hdr_t;
+typedef struct am_rdma_hdr_t am_rdma_hdr_t;
 
 /**
  * @brief target-side RDMA/atomic operation
  */
-struct mca_btl_base_rdma_operation_t {
+struct am_rdma_operation_t {
     /** these may be stored in lists */
     opal_list_item_t super;
     /** btl module associated with this operation */
@@ -192,11 +212,11 @@ struct mca_btl_base_rdma_operation_t {
     /** endpoint used to communicate with the origin */
     struct mca_btl_base_endpoint_t *endpoint;
     /** response descriptor (if allocated). this will be stored
-     * if the send operation was unsuccessfull and the response
+     * if the send operation was unsuccessful and the response
      * needs to be retried. */
     mca_btl_base_descriptor_t *descriptor;
     /** incoming operation header */
-    mca_btl_base_rdma_hdr_t hdr;
+    am_rdma_hdr_t hdr;
     /** local memory handle (if using RDMA) */
     uint8_t local_handle_data[MCA_BTL_REG_HANDLE_MAX_SIZE];
     /** remote memory handle (if using RMDA) */
@@ -208,44 +228,32 @@ struct mca_btl_base_rdma_operation_t {
     /** rdma operation was completed (waiting response) */
     bool is_completed;
 };
-typedef struct mca_btl_base_rdma_operation_t mca_btl_base_rdma_operation_t;
-
-static OBJ_CLASS_INSTANCE (mca_btl_base_rdma_operation_t, opal_list_item_t, NULL, NULL);
-
-static inline size_t size_t_min (size_t a, size_t b) { return (a < b) ? a : b; }
+typedef struct am_rdma_operation_t am_rdma_operation_t;
 
-static inline size_t size_t_max (size_t a, size_t b) { return (a > b) ? a : b; }
+static OBJ_CLASS_INSTANCE(am_rdma_operation_t, opal_list_item_t, NULL, NULL);
 
-static mca_btl_base_am_rdma_module_t default_module;
 
-static inline bool mca_btl_base_rdma_use_rdma_get (mca_btl_base_module_t *btl)
+static inline bool am_rdma_is_atomic(am_rdma_type_t type)
 {
-    return !!(btl->btl_flags & MCA_BTL_FLAGS_GET);
+    return (MCA_BTL_BASE_AM_PUT != type && MCA_BTL_BASE_AM_GET != type);
 }
 
-static inline bool mca_btl_base_rdma_use_rdma_put (mca_btl_base_module_t *btl)
-{
-    return !!(btl->btl_flags & MCA_BTL_FLAGS_PUT);
-}
 
-static inline bool mca_btl_base_rdma_is_atomic (mca_btl_base_rdma_type_t type)
+static inline size_t am_rdma_operation_size(mca_btl_base_am_rdma_module_t *am_module,
+                                            am_rdma_type_t type,
+                                            size_t remaining)
 {
-    return (MCA_BTL_BASE_AM_PUT != type && MCA_BTL_BASE_AM_GET != type);
-}
+    mca_btl_base_module_t *btl = am_module->btl;
 
-static inline size_t mca_btl_base_rdma_operation_size (mca_btl_base_module_t *btl,
-                                                       mca_btl_base_rdma_type_t type,
-                                                       size_t remaining)
-{
     switch (type) {
     case MCA_BTL_BASE_AM_PUT:
-        if (mca_btl_base_rdma_use_rdma_get (btl)) {
-            return size_t_min (remaining, btl->btl_get_limit);
+        if (am_module->use_rdma_get) {
+            return opal_min(remaining, btl->btl_get_limit);
         }
         break;
     case MCA_BTL_BASE_AM_GET:
-        if (mca_btl_base_rdma_use_rdma_put (btl)) {
-            return size_t_min (remaining, btl->btl_put_limit);
+        if (am_module->use_rdma_put) {
+            return opal_min(remaining, btl->btl_put_limit);
         }
         break;
     case MCA_BTL_BASE_AM_ATOMIC:
@@ -254,12 +262,13 @@ static inline size_t mca_btl_base_rdma_operation_size (mca_btl_base_module_t *bt
         return remaining;
     }
 
-    return size_t_min (remaining, btl->btl_max_send_size - sizeof (mca_btl_base_rdma_hdr_t));
+    return opal_min(remaining, btl->btl_max_send_size - sizeof(am_rdma_hdr_t));
 }
 
-static inline int mca_btl_base_rdma_tag (mca_btl_base_rdma_type_t type)
+
+static inline int am_rdma_tag(am_rdma_type_t type)
 {
-    (void)type;
+    (void) type;
     switch (type) {
     case MCA_BTL_BASE_AM_PUT:
     case MCA_BTL_BASE_AM_GET:
@@ -271,7 +280,12 @@ static inline int mca_btl_base_rdma_tag (mca_btl_base_rdma_type_t type)
     return MCA_BTL_BASE_TAG_RDMA_RESP;
 }
 
-static inline int mca_btl_base_rdma_resp_tag (void) { return MCA_BTL_BASE_TAG_RDMA_RESP; }
+
+static inline int am_rdma_resp_tag(void)
+{
+    return MCA_BTL_BASE_TAG_RDMA_RESP;
+}
+
 
 /**
  * @brief copy data from a segment to a local address
@@ -281,29 +295,30 @@ static inline int mca_btl_base_rdma_resp_tag (void) { return MCA_BTL_BASE_TAG_RD
  * @in segments      segments to copy data from
  * @in segment_count number of segments
  */
-static void mca_btl_base_copy_from_segments (uint64_t addr, size_t skip_bytes,
-                                             const mca_btl_base_segment_t *segments,
-                                             size_t segment_count)
+static void am_rdma_copy_from_segments(uint64_t addr, size_t skip_bytes,
+                                       const mca_btl_base_segment_t *segments,
+                                       size_t segment_count)
 {
-    const void *seg0_data = (const void *)((uintptr_t)segments[0].seg_addr.pval + skip_bytes);
-    size_t seg0_len       = segments[0].seg_len - skip_bytes;
+    const void *seg0_data = (const void *) ((uintptr_t) segments[0].seg_addr.pval + skip_bytes);
+    size_t seg0_len = segments[0].seg_len - skip_bytes;
 
     if (seg0_len > 0) {
-        BTL_VERBOSE (
-            ("unpacking %" PRIsize_t " bytes from segment 0 to %p", seg0_len, (void *)addr));
-        memcpy ((void *)addr, seg0_data, seg0_len);
+        BTL_VERBOSE(
+            ("unpacking %" PRIsize_t " bytes from segment 0 to %p", seg0_len, (void *) addr));
+        memcpy((void *) addr, seg0_data, seg0_len);
         addr += seg0_len;
     }
 
     for (size_t i = 1; i < segment_count; ++i) {
         size_t seg_len = segments[i].seg_len;
-        BTL_VERBOSE (("unpacking %" PRIsize_t " bytes from segment %" PRIsize_t " to %p", seg_len,
-                      i, (void *)addr));
-        memcpy ((void *)addr, segments[i].seg_addr.pval, seg_len);
+        BTL_VERBOSE(("unpacking %" PRIsize_t " bytes from segment %" PRIsize_t " to %p", seg_len, i,
+                     (void *) addr));
+        memcpy((void *) addr, segments[i].seg_addr.pval, seg_len);
         addr += seg_len;
     }
 }
 
+
 /**
  * @brief copy data from a local address into a segment
  *
@@ -312,28 +327,28 @@ static void mca_btl_base_copy_from_segments (uint64_t addr, size_t skip_bytes,
  * @in segments      segments to copy data to
  * @in segment_count number of segments
  */
-static void mca_btl_base_copy_to_segments (uint64_t addr, size_t max_len, size_t skip_bytes,
-                                           mca_btl_base_segment_t *segments, size_t segment_count)
+static void am_rdma_copy_to_segments(uint64_t addr, size_t max_len, size_t skip_bytes,
+                                     mca_btl_base_segment_t *segments, size_t segment_count)
 {
-    void *seg0_data = (void *)((uintptr_t)segments[0].seg_addr.pval + skip_bytes);
-    size_t seg0_len = size_t_min (max_len, segments[0].seg_len - skip_bytes);
+    void *seg0_data = (void *) ((uintptr_t) segments[0].seg_addr.pval + skip_bytes);
+    size_t seg0_len = opal_min(max_len, segments[0].seg_len - skip_bytes);
 
     if (seg0_len > 0) {
-        BTL_VERBOSE (
+        BTL_VERBOSE(
             ("packing %" PRIsize_t " bytes from 0x%" PRIx64 " to segment 0", seg0_len, addr));
-        memcpy (seg0_data, (const void *)addr, seg0_len);
+        memcpy(seg0_data, (const void *) addr, seg0_len);
         addr += seg0_len;
         max_len -= seg0_len;
         segments[0].seg_len = seg0_len + skip_bytes;
     }
 
     for (size_t i = 1; i < segment_count && max_len; ++i) {
-        size_t seg_len = size_t_min (segments[i].seg_len, max_len);
+        size_t seg_len = opal_min(segments[i].seg_len, max_len);
 
-        BTL_VERBOSE (("packing %" PRIsize_t " bytes from 0x%" PRIx64 " to segment %" PRIsize_t,
-                      seg_len, addr, i));
+        BTL_VERBOSE(("packing %" PRIsize_t " bytes from 0x%" PRIx64 " to segment %" PRIsize_t,
+                     seg_len, addr, i));
 
-        memcpy (segments[i].seg_addr.pval, (const void *)addr, seg_len);
+        memcpy(segments[i].seg_addr.pval, (const void *) addr, seg_len);
         segments[i].seg_len = seg_len;
 
         addr += seg_len;
@@ -341,50 +356,55 @@ static void mca_btl_base_copy_to_segments (uint64_t addr, size_t max_len, size_t
     }
 }
 
-static void mca_btl_base_am_queue_initiator_descriptor (mca_btl_base_module_t *btl,
-                                                        struct mca_btl_base_endpoint_t *endpoint,
-                                                        mca_btl_base_descriptor_t *descriptor)
+
+static void am_rdma_queue_initiator_descriptor(mca_btl_base_am_rdma_module_t *am_module,
+                                               struct mca_btl_base_endpoint_t *endpoint,
+                                               mca_btl_base_descriptor_t *descriptor)
 {
-    mca_btl_base_am_rdma_queued_descriptor_t *queued_descriptor =
-        OBJ_NEW (mca_btl_base_am_rdma_queued_descriptor_t);
+    am_rdma_queued_descriptor_t *queued_descriptor = OBJ_NEW(am_rdma_queued_descriptor_t);
 
-    queued_descriptor->btl        = btl;
-    queued_descriptor->endpoint   = endpoint;
+    queued_descriptor->am_module = am_module;
+    queued_descriptor->endpoint = endpoint;
     queued_descriptor->descriptor = descriptor;
 
-    OPAL_THREAD_SCOPED_LOCK (
-        &default_module.mutex,
-        opal_list_append (&default_module.queued_initiator_descriptors, &queued_descriptor->super));
+    OPAL_THREAD_SCOPED_LOCK(&default_component.mutex,
+                            opal_list_append(&default_component.queued_initiator_descriptors,
+                                             &queued_descriptor->super));
 }
 
-static inline int mca_btl_base_am_rdma_advance (mca_btl_base_module_t *btl,
-                                                struct mca_btl_base_endpoint_t *endpoint,
-                                                mca_btl_base_rdma_context_t *context,
-                                                bool send_descriptor)
+
+static inline int am_rdma_advance(mca_btl_base_am_rdma_module_t *am_module,
+                                  struct mca_btl_base_endpoint_t *endpoint,
+                                  am_rdma_context_t *context,
+                                  bool send_descriptor)
 {
+    mca_btl_base_module_t *btl = am_module->btl;
+    int ret;
     const size_t remaining = context->total_size - context->sent;
 
     if (0 == remaining) {
         if (context->descriptor) {
-            btl->btl_free (btl, context->descriptor);
+            btl->btl_free(btl, context->descriptor);
             context->descriptor = NULL;
         }
 
         /* release the reference retained at context creation */
-        OBJ_RELEASE (context);
+        OBJ_RELEASE(context);
         /* nothing more needs to be done */
         return OPAL_SUCCESS;
     }
 
     mca_btl_base_descriptor_t *descriptor = context->descriptor;
-    mca_btl_base_rdma_hdr_t *hdr =
-        (mca_btl_base_rdma_hdr_t *)descriptor->des_segments[0].seg_addr.pval;
-    const size_t packet_size = mca_btl_base_rdma_operation_size (btl, hdr->type, remaining);
+    am_rdma_hdr_t *hdr = (am_rdma_hdr_t *) descriptor->des_segments[0]
+                                       .seg_addr.pval;
+    const size_t packet_size = am_rdma_operation_size(am_module, hdr->type, remaining);
 
-    if (!mca_btl_base_rdma_is_atomic (hdr->type)) {
-        hdr->data.rdma.size              = packet_size;
-        hdr->data.rdma.initiator_address = (uint64_t)context->local_address + context->sent;
+    if (!am_rdma_is_atomic(hdr->type)) {
+        hdr->data.rdma.size = packet_size;
+        hdr->data.rdma.initiator_address = (uint64_t) context->local_address + context->sent;
     } else {
+        /* atomics today are single datatype entries */
+        assert(packet_size < UINT8_MAX);
         hdr->data.atomic.size = packet_size;
     }
 
@@ -394,98 +414,98 @@ static inline int mca_btl_base_am_rdma_advance (mca_btl_base_module_t *btl,
 
     if (MCA_BTL_BASE_AM_PUT == hdr->type && !hdr->data.rdma.use_rdma) {
         /* copy the next block into the fragment buffer */
-        mca_btl_base_copy_to_segments (hdr->data.rdma.initiator_address, packet_size, sizeof (*hdr),
-                                       descriptor->des_segments, descriptor->des_segment_count);
+        am_rdma_copy_to_segments(hdr->data.rdma.initiator_address, packet_size, sizeof(*hdr),
+                                 descriptor->des_segments, descriptor->des_segment_count);
     }
 
     if (send_descriptor) {
-        return btl->btl_send (btl, endpoint, descriptor, mca_btl_base_rdma_tag (hdr->type));
+        assert(0 != (descriptor->des_flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK));
+        ret = btl->btl_send(btl, endpoint, descriptor, am_rdma_tag(hdr->type));
+        if (ret == 1) {
+            ret = OPAL_SUCCESS;
+        }
+        return ret;
     }
 
     /* queue for later to avoid btl_send in callback */
-    mca_btl_base_am_queue_initiator_descriptor (btl, endpoint, descriptor);
+    am_rdma_queue_initiator_descriptor(am_module, endpoint, descriptor);
 
     return OPAL_SUCCESS;
 }
 
-static void mca_btl_base_am_descriptor_complete (mca_btl_base_module_t *btl,
-                                                 struct mca_btl_base_endpoint_t *endpoint,
-                                                 mca_btl_base_descriptor_t *descriptor, int status)
+
+static void am_rdma_descriptor_complete(mca_btl_base_module_t *btl,
+                                        struct mca_btl_base_endpoint_t *endpoint,
+                                        mca_btl_base_descriptor_t *descriptor, int status)
 {
-    mca_btl_base_rdma_context_t *context = (mca_btl_base_rdma_context_t *)descriptor->des_context;
+    mca_btl_base_am_rdma_module_t *am_module = (mca_btl_base_am_rdma_module_t *)descriptor->des_cbdata;
 
-    (void)mca_btl_base_am_rdma_advance (btl, endpoint,
-                                        (mca_btl_base_rdma_context_t *)descriptor->des_context,
-                                        /*send_descriptor=*/false);
+    (void) am_rdma_advance(am_module, endpoint,
+                           (am_rdma_context_t *) descriptor->des_context,
+                           /*send_descriptor=*/false);
 }
 
-static inline int
-mca_btl_base_rdma_start (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
-                         int type, uint64_t operand1, uint64_t operand2,
-                         mca_btl_base_atomic_op_t op, int order, int flags, size_t size,
-                         void *local_address, mca_btl_base_registration_handle_t *local_handle,
-                         int64_t remote_address, mca_btl_base_registration_handle_t *remote_handle,
-                         mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
+
+static inline int am_rdma_start(mca_btl_base_am_rdma_module_t *am_module, struct mca_btl_base_endpoint_t *endpoint,
+                                int type, uint64_t operand1, uint64_t operand2, mca_btl_base_atomic_op_t op,
+                                int order, int flags, size_t size, void *local_address,
+                                mca_btl_base_registration_handle_t *local_handle, int64_t remote_address,
+                                mca_btl_base_registration_handle_t *remote_handle,
+                                mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
 {
-    mca_btl_base_rdma_hdr_t *hdr;
-    size_t packet_size = sizeof (*hdr);
+    mca_btl_base_module_t *btl = am_module->btl;
+    am_rdma_hdr_t *hdr;
+    size_t packet_size = sizeof(*hdr);
     mca_btl_base_descriptor_t *descriptor;
-    mca_btl_base_rdma_context_t *context = OBJ_NEW (mca_btl_base_rdma_context_t);
+    am_rdma_context_t *context = OBJ_NEW(am_rdma_context_t);
 
-    if (OPAL_UNLIKELY (NULL == context)) {
+    if (OPAL_UNLIKELY(NULL == context)) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
-    context->type           = type;
-    context->cbfunc         = cbfunc;
-    context->cbcontext      = cbcontext;
-    context->cbdata         = cbdata;
-    context->local_address  = local_address;
+    context->type = type;
+    context->cbfunc = cbfunc;
+    context->cbcontext = cbcontext;
+    context->cbdata = cbdata;
+    context->local_address = local_address;
     context->remote_address = remote_address;
-    context->local_handle   = local_handle;
-    context->total_size     = size;
+    context->local_handle = local_handle;
+    context->total_size = size;
 
-    size_t send_size = 0;
-    size_t recv_size = 0;
-    bool use_rdma    = false;
+    bool use_rdma = false;
 
     if (MCA_BTL_BASE_AM_PUT == type) {
-        if (sizeof (*hdr) + size <= btl->btl_eager_limit) {
+        if (sizeof(*hdr) + size <= btl->btl_eager_limit) {
             /* just go ahead and send the data */
-            send_size = size;
-        } else if (!mca_btl_base_rdma_use_rdma_get (btl)) {
-            send_size = size_t_min (size, btl->btl_max_send_size - sizeof (*hdr));
+            packet_size += size;
+        } else if (!am_module->use_rdma_get) {
+            packet_size += opal_min (size, btl->btl_max_send_size - sizeof (*hdr));
         } else {
             use_rdma = true;
         }
     } else if (MCA_BTL_BASE_AM_GET == type) {
-        if (sizeof (mca_btl_base_rdma_response_hdr_t) + size <= btl->btl_eager_limit) {
-            recv_size = size;
-        } else if (!mca_btl_base_rdma_use_rdma_put (btl)) {
-            recv_size = size_t_min (size, btl->btl_max_send_size -
-                                              sizeof (mca_btl_base_rdma_response_hdr_t));
+        if (!am_module->use_rdma_put) {
+            packet_size += opal_min(size, btl->btl_max_send_size - sizeof(*hdr));
         } else {
             use_rdma = true;
         }
     } else {
         /* fetching atomic. always recv result via active message */
-        recv_size = size;
+        packet_size += size;
     }
 
     if (use_rdma && btl->btl_register_mem) {
         packet_size += 2 * btl->btl_registration_handle_size;
     }
 
-    packet_size += send_size;
-
-    BTL_VERBOSE (("Initiating RDMA operation. context=%p, size=%" PRIsize_t
-                  ", packet_size=%" PRIsize_t,
-                  context, size, packet_size));
+    BTL_VERBOSE(("Initiating RDMA operation. context=%p, size=%" PRIsize_t
+                 ", packet_size=%" PRIsize_t,
+                 (void*) context, size, packet_size));
 
-    descriptor = btl->btl_alloc (btl, endpoint, order, packet_size,
-                                 MCA_BTL_DES_SEND_ALWAYS_CALLBACK | MCA_BTL_DES_FLAGS_SIGNAL);
-    if (OPAL_UNLIKELY (NULL == descriptor)) {
-        OBJ_RELEASE (context);
+    descriptor = btl->btl_alloc(btl, endpoint, order, packet_size,
+                                MCA_BTL_DES_SEND_ALWAYS_CALLBACK | MCA_BTL_DES_FLAGS_SIGNAL);
+    if (OPAL_UNLIKELY(NULL == descriptor)) {
+        OBJ_RELEASE(context);
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
@@ -493,107 +513,111 @@ mca_btl_base_rdma_start (mca_btl_base_module_t *btl, struct mca_btl_base_endpoin
 
     /* keep a reference around until the descriptor callback is complete. the initial reference may
      * be released on response before the descriptor callback has completed. */
-    OBJ_RETAIN (context);
+    OBJ_RETAIN(context);
 
-    descriptor->des_cbfunc  = mca_btl_base_am_descriptor_complete;
-    descriptor->des_cbdata  = local_handle;
+    descriptor->des_cbfunc = am_rdma_descriptor_complete;
+    descriptor->des_cbdata = am_module;
     descriptor->des_context = context;
 
-    hdr       = (mca_btl_base_rdma_hdr_t *)descriptor->des_segments[0].seg_addr.pval;
+    hdr = (am_rdma_hdr_t *) descriptor->des_segments[0].seg_addr.pval;
     hdr->type = type;
 
-    if (!mca_btl_base_rdma_is_atomic (type)) {
+    if (!am_rdma_is_atomic(type)) {
         hdr->data.rdma.use_rdma = use_rdma;
     } else {
-        hdr->data.atomic.op         = op;
+        hdr->data.atomic.op = op;
         hdr->data.atomic.operand[0] = operand1;
         hdr->data.atomic.operand[1] = operand2;
     }
 
-    hdr->context = (uintptr_t)context;
+    hdr->context = (uintptr_t) context;
 
     if (use_rdma && btl->btl_register_mem) {
-        uint8_t *handle_buffer = (uint8_t *)(hdr + 1);
-        memcpy (handle_buffer, local_handle, btl->btl_registration_handle_size);
+        uint8_t *handle_buffer = (uint8_t *) (hdr + 1);
+        memcpy(handle_buffer, local_handle, btl->btl_registration_handle_size);
         handle_buffer += btl->btl_registration_handle_size;
-        memcpy (handle_buffer, remote_handle, btl->btl_registration_handle_size);
+        memcpy(handle_buffer, remote_handle, btl->btl_registration_handle_size);
     }
 
-    return mca_btl_base_am_rdma_advance (btl, endpoint, context, /*send_descriptor=*/true);
+    return am_rdma_advance(am_module, endpoint, context, /*send_descriptor=*/true);
 }
 
-static mca_btl_base_rdma_operation_t *mca_btl_base_rdma_alloc_operation (
-    mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
-    mca_btl_base_descriptor_t *descriptor, const mca_btl_base_rdma_hdr_t *hdr)
+
+static am_rdma_operation_t *am_rdma_alloc_operation(mca_btl_base_module_t *btl,
+                                                    struct mca_btl_base_endpoint_t *endpoint,
+                                                    mca_btl_base_descriptor_t *descriptor,
+                                                    const am_rdma_hdr_t *hdr)
 {
-    mca_btl_base_rdma_operation_t *operation = OBJ_NEW (mca_btl_base_rdma_operation_t);
+    am_rdma_operation_t *operation = OBJ_NEW(am_rdma_operation_t);
     if (NULL == operation) {
         return NULL;
     }
 
-    operation->btl          = btl;
-    operation->endpoint     = endpoint;
-    operation->descriptor   = descriptor;
+    operation->btl = btl;
+    operation->endpoint = endpoint;
+    operation->descriptor = descriptor;
     operation->is_completed = false;
-    operation->is_queued    = false;
-    memcpy (&operation->hdr, hdr, sizeof (*hdr));
+    operation->is_queued = false;
+    memcpy(&operation->hdr, hdr, sizeof(*hdr));
 
-    if (!mca_btl_base_rdma_is_atomic (hdr->type) && hdr->data.rdma.use_rdma &&
-        btl->btl_register_mem) {
-        const uint8_t *handle_data = (const uint8_t *)(hdr + 1);
+    if (!am_rdma_is_atomic(hdr->type) && hdr->data.rdma.use_rdma
+        && btl->btl_register_mem) {
+        const uint8_t *handle_data = (const uint8_t *) (hdr + 1);
         /* the initiator packs these in order of their local and then remote. */
-        memcpy (operation->remote_handle_data, handle_data, btl->btl_registration_handle_size);
+        memcpy(operation->remote_handle_data, handle_data, btl->btl_registration_handle_size);
         handle_data += btl->btl_registration_handle_size;
-        memcpy (operation->local_handle_data, handle_data, btl->btl_registration_handle_size);
+        memcpy(operation->local_handle_data, handle_data, btl->btl_registration_handle_size);
     }
 
     return operation;
 }
 
-static void mca_btl_base_rdma_queue_operation (mca_btl_base_module_t *btl,
-                                               struct mca_btl_base_endpoint_t *endpoint,
-                                               mca_btl_base_descriptor_t *descriptor,
-                                               uint64_t atomic_response,
-                                               const mca_btl_base_rdma_hdr_t *hdr,
-                                               mca_btl_base_rdma_operation_t *operation)
+
+static void am_rdma_queue_operation(mca_btl_base_module_t *btl,
+                                    struct mca_btl_base_endpoint_t *endpoint,
+                                    mca_btl_base_descriptor_t *descriptor,
+                                    uint64_t atomic_response,
+                                    const am_rdma_hdr_t *hdr,
+                                    am_rdma_operation_t *operation)
 {
     if (NULL == operation) {
-        operation = mca_btl_base_rdma_alloc_operation (btl, endpoint, descriptor, hdr);
+        operation = am_rdma_alloc_operation(btl, endpoint, descriptor, hdr);
         if (NULL == operation) {
             /* couldn't even allocate a small amount of memory. not much else can be done. */
-            BTL_ERROR (("could not allocate memory to queue active-message RDMA operation"));
-            abort ();
+            BTL_ERROR(("could not allocate memory to queue active-message RDMA operation"));
+            abort();
         }
     }
 
-    operation->is_queued       = true;
+    operation->is_queued = true;
     operation->atomic_response = atomic_response;
-    OPAL_THREAD_SCOPED_LOCK (
-        &default_module.mutex,
-        opal_list_append (&default_module.queued_responses, &operation->super));
+    OPAL_THREAD_SCOPED_LOCK(&default_component.mutex,
+                            opal_list_append(&default_component.queued_responses, &operation->super));
 }
 
-static int mca_btl_base_am_rdma_respond (mca_btl_base_module_t *btl,
-                                         struct mca_btl_base_endpoint_t *endpoint,
-                                         mca_btl_base_descriptor_t **descriptor, void *addr,
-                                         const mca_btl_base_rdma_hdr_t *hdr)
+
+static int am_rdma_respond(mca_btl_base_module_t *btl,
+                           struct mca_btl_base_endpoint_t *endpoint,
+                           mca_btl_base_descriptor_t **descriptor, void *addr,
+                           const am_rdma_hdr_t *hdr)
 {
     mca_btl_base_descriptor_t *send_descriptor = *descriptor;
-    *descriptor                                = NULL;
+    *descriptor = NULL;
 
     if (NULL == send_descriptor) {
-        mca_btl_base_rdma_response_hdr_t *resp_hdr;
-        size_t data_size =
-            mca_btl_base_rdma_is_atomic (hdr->type) ? hdr->data.atomic.size : hdr->data.rdma.size;
-        size_t packet_size = sizeof (*resp_hdr) + (addr ? data_size : 0);
-        send_descriptor    = btl->btl_alloc (btl, endpoint, MCA_BTL_NO_ORDER, packet_size,
-                                          MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
+        assert(NULL != hdr);
+        am_rdma_response_hdr_t *resp_hdr;
+        size_t data_size = am_rdma_is_atomic(hdr->type) ? hdr->data.atomic.size
+                                                                  : hdr->data.rdma.size;
+        size_t packet_size = sizeof(*resp_hdr) + (addr ? data_size : 0);
+        send_descriptor = btl->btl_alloc(btl, endpoint, MCA_BTL_NO_ORDER, packet_size,
+                                         MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
         if (NULL == send_descriptor) {
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
 
-        resp_hdr =
-            (mca_btl_base_rdma_response_hdr_t *)send_descriptor->des_segments[0].seg_addr.pval;
+        resp_hdr = (am_rdma_response_hdr_t *) send_descriptor->des_segments[0]
+                       .seg_addr.pval;
         resp_hdr->context = hdr->context;
         if (MCA_BTL_BASE_AM_GET == hdr->type) {
             resp_hdr->initiator_address = hdr->data.rdma.initiator_address;
@@ -604,222 +628,244 @@ static int mca_btl_base_am_rdma_respond (mca_btl_base_module_t *btl,
         resp_hdr->response_size = data_size;
 
         if (NULL != addr) {
-            mca_btl_base_copy_to_segments ((uint64_t) (uintptr_t)addr, packet_size,
-                                           sizeof (*resp_hdr), send_descriptor->des_segments,
-                                           send_descriptor->des_segment_count);
+            am_rdma_copy_to_segments((uint64_t)(uintptr_t) addr, packet_size,
+                                     sizeof(*resp_hdr), send_descriptor->des_segments,
+                                     send_descriptor->des_segment_count);
         }
     }
 
-    BTL_VERBOSE (("sending descriptor %p", send_descriptor));
+    BTL_VERBOSE(("sending descriptor %p", (void*) send_descriptor));
 
     send_descriptor->des_cbfunc = NULL;
 
-    int ret = btl->btl_send (btl, endpoint, send_descriptor, mca_btl_base_rdma_resp_tag ());
-    if (OPAL_UNLIKELY (OPAL_SUCCESS != ret)) {
+    /* There is no callback for the response descriptor, therefore it is
+     * safe to treat 0 and 1 return codes the same
+     */
+    int ret = btl->btl_send(btl, endpoint, send_descriptor, am_rdma_resp_tag());
+    if (ret == 1) {
+        ret = OPAL_SUCCESS;
+    }
+
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != ret)) {
         *descriptor = send_descriptor;
     }
     return ret;
 }
 
+
 static void
-mca_btl_base_am_rmda_rdma_complete (mca_btl_base_module_t *btl,
-                                    struct mca_btl_base_endpoint_t *endpoint, void *local_address,
-                                    struct mca_btl_base_registration_handle_t *local_handle,
-                                    void *context, void *cbdata, int status)
+am_rdma_rdma_complete(mca_btl_base_module_t *btl,
+                      struct mca_btl_base_endpoint_t *endpoint, void *local_address,
+                      struct mca_btl_base_registration_handle_t *local_handle,
+                      void *context, void *cbdata, int status)
 {
-    mca_btl_base_rdma_operation_t *operation = (mca_btl_base_rdma_operation_t *)context;
+    am_rdma_operation_t *operation = (am_rdma_operation_t *) context;
 
-    BTL_VERBOSE (("BTL RDMA operation complete. status=%d", status));
+    BTL_VERBOSE(("BTL RDMA operation complete. status=%d", status));
 
-    assert (OPAL_SUCCESS == status);
+    assert(OPAL_SUCCESS == status);
 
     operation->is_completed = true;
-    int ret                 = mca_btl_base_am_rdma_respond (operation->btl, operation->endpoint,
-                                            &operation->descriptor, NULL, &operation->hdr);
-    if (OPAL_UNLIKELY (OPAL_SUCCESS != ret)) {
-        BTL_VERBOSE (
+    int ret = am_rdma_respond(operation->btl, operation->endpoint,
+                              &operation->descriptor, NULL, &operation->hdr);
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != ret)) {
+        BTL_VERBOSE(
             ("could not send a response. queueing the response for later. endpoint=%p, ret=%d",
-             endpoint, ret));
-        mca_btl_base_rdma_queue_operation (btl, NULL, NULL, 0, NULL, operation);
+             (void*) endpoint, ret));
+        am_rdma_queue_operation(btl, NULL, NULL, 0, NULL, operation);
     }
 
-    OBJ_RELEASE (operation);
+    OBJ_RELEASE(operation);
 }
 
-static int mca_btl_base_am_rdma_target_get (mca_btl_base_module_t *btl,
-                                            struct mca_btl_base_endpoint_t *endpoint,
-                                            mca_btl_base_descriptor_t **descriptor,
-                                            void *target_address,
-                                            const mca_btl_base_rdma_hdr_t *hdr,
-                                            mca_btl_base_rdma_operation_t **operation)
+
+static int am_rdma_target_put(mca_btl_base_module_t *btl,
+                              struct mca_btl_base_endpoint_t *endpoint,
+                              mca_btl_base_descriptor_t **descriptor,
+                              const mca_btl_base_segment_t *segments,
+                              size_t segment_count, void *target_address,
+                              const am_rdma_hdr_t *hdr,
+                              am_rdma_operation_t **operation)
 {
     if (hdr->data.rdma.use_rdma) {
         if (NULL == *operation) {
-            *operation = mca_btl_base_rdma_alloc_operation (btl, endpoint, *descriptor, hdr);
+            *operation = am_rdma_alloc_operation(btl, endpoint, *descriptor, hdr);
             if (NULL == *operation) {
                 return OPAL_ERR_OUT_OF_RESOURCE;
             }
         }
 
-        /* btl supports put but not get. emulating get with put */
-        OBJ_RETAIN (*operation);
-        int ret = btl->btl_put (
+        /* btl supports get but not put. emulating put with get */
+        OBJ_RETAIN(*operation);
+        int ret = btl->btl_get(
             btl, endpoint, target_address, hdr->data.rdma.initiator_address,
-            (struct mca_btl_base_registration_handle_t *)(*operation)->local_handle_data,
-            (struct mca_btl_base_registration_handle_t *)(*operation)->remote_handle_data,
-            hdr->data.rdma.size, /*flags=*/0, MCA_BTL_NO_ORDER, mca_btl_base_am_rmda_rdma_complete,
+            (struct mca_btl_base_registration_handle_t *) (*operation)->local_handle_data,
+            (struct mca_btl_base_registration_handle_t *) (*operation)->remote_handle_data,
+            hdr->data.rdma.size, /*flags=*/0, MCA_BTL_NO_ORDER, am_rdma_rdma_complete,
             *operation, NULL);
         if (OPAL_SUCCESS != ret) {
-            OBJ_RELEASE (*operation);
+            OBJ_RELEASE(*operation);
         }
+
         if (OPAL_ERR_NOT_AVAILABLE != ret) {
             return ret;
         }
+    } else if (NULL != segments) {
+        am_rdma_copy_from_segments(hdr->target_address, sizeof(*hdr), segments, segment_count);
     }
 
-    return mca_btl_base_am_rdma_respond (btl, endpoint, descriptor, target_address, hdr);
+    return am_rdma_respond(btl, endpoint, descriptor, NULL, hdr);
 }
 
-static int mca_btl_base_am_rdma_target_put (mca_btl_base_module_t *btl,
-                                            struct mca_btl_base_endpoint_t *endpoint,
-                                            mca_btl_base_descriptor_t **descriptor,
-                                            const mca_btl_base_segment_t *segments,
-                                            size_t segment_count, void *target_address,
-                                            const mca_btl_base_rdma_hdr_t *hdr,
-                                            mca_btl_base_rdma_operation_t **operation)
+
+static int am_rdma_target_get(mca_btl_base_module_t *btl,
+                              struct mca_btl_base_endpoint_t *endpoint,
+                              mca_btl_base_descriptor_t **descriptor,
+                              void *target_address, const am_rdma_hdr_t *hdr,
+                              am_rdma_operation_t **operation)
 {
     if (hdr->data.rdma.use_rdma) {
         if (NULL == *operation) {
-            *operation = mca_btl_base_rdma_alloc_operation (btl, endpoint, *descriptor, hdr);
+            *operation = am_rdma_alloc_operation(btl, endpoint, *descriptor, hdr);
             if (NULL == *operation) {
                 return OPAL_ERR_OUT_OF_RESOURCE;
             }
         }
 
         /* btl supports put but not get. emulating get with put */
-        OBJ_RETAIN (*operation);
-        int ret = btl->btl_get (
+        OBJ_RETAIN(*operation);
+        int ret = btl->btl_put(
             btl, endpoint, target_address, hdr->data.rdma.initiator_address,
-            (struct mca_btl_base_registration_handle_t *)(*operation)->local_handle_data,
-            (struct mca_btl_base_registration_handle_t *)(*operation)->remote_handle_data,
-            hdr->data.rdma.size, /*flags=*/0, MCA_BTL_NO_ORDER, mca_btl_base_am_rmda_rdma_complete,
-            operation, NULL);
+            (struct mca_btl_base_registration_handle_t *) (*operation)->local_handle_data,
+            (struct mca_btl_base_registration_handle_t *) (*operation)->remote_handle_data,
+            hdr->data.rdma.size, /*flags=*/0, MCA_BTL_NO_ORDER, am_rdma_rdma_complete,
+            *operation, NULL);
         if (OPAL_SUCCESS != ret) {
-            OBJ_RELEASE (*operation);
+            OBJ_RELEASE(*operation);
         }
-
         if (OPAL_ERR_NOT_AVAILABLE != ret) {
             return ret;
         }
-    } else if (NULL != segments) {
-        mca_btl_base_copy_from_segments (hdr->target_address, sizeof (*hdr), segments,
-                                         segment_count);
     }
 
-    return mca_btl_base_am_rdma_respond (btl, endpoint, descriptor, NULL, hdr);
+    return am_rdma_respond(btl, endpoint, descriptor, target_address, hdr);
 }
 
-static void mca_btl_base_rdma_retry_operation (mca_btl_base_rdma_operation_t *operation)
+
+static void am_rdma_retry_operation(am_rdma_operation_t *operation)
 {
-    mca_btl_base_module_t *btl = operation->btl;
-    void *target_address       = (void *)(intptr_t)operation->hdr.target_address;
-    int ret;
+    void *target_address = (void *) (intptr_t) operation->hdr.target_address;
+    int ret = OPAL_SUCCESS;
 
     if (!operation->descriptor && !operation->is_completed) {
         switch (operation->hdr.type) {
         case MCA_BTL_BASE_AM_GET:
-            ret = mca_btl_base_am_rdma_target_get (operation->btl, operation->endpoint,
-                                                   &operation->descriptor, target_address,
-                                                   &operation->hdr, &operation);
+            ret = am_rdma_target_get(operation->btl, operation->endpoint,
+                                     &operation->descriptor, target_address,
+                                     &operation->hdr, &operation);
             break;
         case MCA_BTL_BASE_AM_PUT:
-            ret = mca_btl_base_am_rdma_target_put (
-                operation->btl, operation->endpoint, &operation->descriptor,
-                /*segments=*/NULL,
-                /*segment_count=*/0, target_address, &operation->hdr, &operation);
+            ret = am_rdma_target_put(operation->btl, operation->endpoint,
+                                     &operation->descriptor,
+                                     /*segments=*/NULL,
+                                     /*segment_count=*/0, target_address,
+                                     &operation->hdr, &operation);
             break;
         case MCA_BTL_BASE_AM_ATOMIC:
             /* atomic operation was completed */
-            ret = mca_btl_base_am_rdma_respond (operation->btl, operation->endpoint,
-                                                &operation->descriptor, &operation->atomic_response,
-                                                &operation->hdr);
+            ret = am_rdma_respond(operation->btl, operation->endpoint,
+                                  &operation->descriptor, &operation->atomic_response,
+                                  &operation->hdr);
             break;
         }
     } else {
-        ret = mca_btl_base_am_rdma_respond (operation->btl, operation->endpoint,
-                                            &operation->descriptor,
-                                            /*addr=*/NULL, /*hdr=*/NULL);
+        ret = am_rdma_respond(operation->btl, operation->endpoint,
+                              &operation->descriptor,
+                              /*addr=*/NULL, &operation->hdr);
     }
 
     if (OPAL_SUCCESS == ret) {
         if (operation->is_queued) {
-            opal_list_remove_item (&default_module.queued_responses, &operation->super);
+            opal_list_remove_item(&default_component.queued_responses, &operation->super);
         }
 
-        OBJ_RELEASE (operation);
+        OBJ_RELEASE(operation);
     }
 }
 
-static int mca_btl_base_am_rdma_progress (void)
+
+static int am_rdma_progress(void)
 {
-    if (0 == opal_list_get_size (&default_module.queued_responses) &&
-        0 == opal_list_get_size (&default_module.queued_initiator_descriptors)) {
+    if (0 == opal_list_get_size(&default_component.queued_responses)
+        && 0 == opal_list_get_size(&default_component.queued_initiator_descriptors)) {
         return 0;
     }
 
-    OPAL_THREAD_SCOPED_LOCK (&default_module.mutex, ({
-        mca_btl_base_rdma_operation_t *operation, *next;
-        OPAL_LIST_FOREACH_SAFE (operation, next, &default_module.queued_responses,
-                                mca_btl_base_rdma_operation_t)
-        {
-            mca_btl_base_rdma_retry_operation (operation);
-        }
-    }));
-
-    OPAL_THREAD_SCOPED_LOCK (&default_module.mutex, ({
-        mca_btl_base_am_rdma_queued_descriptor_t *descriptor, *next;
-        OPAL_LIST_FOREACH_SAFE (descriptor, next, &default_module.queued_initiator_descriptors,
-                                mca_btl_base_am_rdma_queued_descriptor_t)
-        {
-            mca_btl_base_rdma_context_t *context =
-                (mca_btl_base_rdma_context_t *)descriptor->descriptor->des_context;
-            int ret = descriptor->btl->btl_send (descriptor->btl, descriptor->endpoint,
-                                                 descriptor->descriptor,
-                                                 mca_btl_base_rdma_tag (context->type));
-            if (OPAL_SUCCESS == ret) {
-                opal_list_remove_item (&default_module.queued_initiator_descriptors,
-                                       &descriptor->super);
-            }
-        }
-    }));
+    // It's a little cleaner, stylistically, to make the multi-line
+    // ACTION argument to OPAL_THREAD_SCOPED_LOCK be a macro itself
+    // (vs. using continuation characters in the use of
+    // OPAL_THREAD_SCOPED_LOCK).
+#define ACTION1                                                         \
+    am_rdma_operation_t *operation, *next;                              \
+    OPAL_LIST_FOREACH_SAFE(operation, next,                             \
+                           &default_component.queued_responses,         \
+                           am_rdma_operation_t) {                       \
+        am_rdma_retry_operation(operation);                             \
+    }
+
+    OPAL_THREAD_SCOPED_LOCK(&default_component.mutex, ACTION1);
+
+#define ACTION2                                                         \
+    am_rdma_queued_descriptor_t *descriptor, *next;                     \
+    OPAL_LIST_FOREACH_SAFE(descriptor, next,                            \
+                           &default_component.queued_initiator_descriptors, \
+                           am_rdma_queued_descriptor_t) {               \
+        am_rdma_context_t *context =                                    \
+            (am_rdma_context_t *) descriptor->descriptor->des_context;  \
+        mca_btl_base_module_t *btl = descriptor->am_module->btl;        \
+        assert(0 != (descriptor->descriptor->des_flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK)); \
+        int ret = btl->btl_send(btl,                                    \
+                                descriptor->endpoint,                   \
+                                descriptor->descriptor,                 \
+                                am_rdma_tag(context->type));            \
+        if (OPAL_SUCCESS == ret || 1 == ret) {                          \
+            opal_list_remove_item(&default_component.queued_initiator_descriptors, \
+                                  &descriptor->super);                  \
+        }                                                               \
+    }
+
+    OPAL_THREAD_SCOPED_LOCK(&default_component.mutex, ACTION2);
+
+    return 0;
 }
 
-static int mca_btl_base_am_atomic_64 (int64_t *operand, opal_atomic_int64_t *addr,
-                                      mca_btl_base_atomic_op_t op)
+
+static int am_rdma_atomic_64(int64_t *operand, opal_atomic_int64_t *addr,
+                             mca_btl_base_atomic_op_t op)
 {
     int64_t result = 0;
 
     switch (op) {
     case MCA_BTL_ATOMIC_ADD:
-        result = opal_atomic_fetch_add_64 (addr, *operand);
+        result = opal_atomic_fetch_add_64(addr, *operand);
         break;
     case MCA_BTL_ATOMIC_AND:
-        result = opal_atomic_fetch_and_64 (addr, *operand);
+        result = opal_atomic_fetch_and_64(addr, *operand);
         break;
     case MCA_BTL_ATOMIC_OR:
-        result = opal_atomic_fetch_or_64 (addr, *operand);
+        result = opal_atomic_fetch_or_64(addr, *operand);
         break;
     case MCA_BTL_ATOMIC_XOR:
-        result = opal_atomic_fetch_xor_64 (addr, *operand);
+        result = opal_atomic_fetch_xor_64(addr, *operand);
         break;
     case MCA_BTL_ATOMIC_SWAP:
-        result = opal_atomic_swap_64 (addr, *operand);
+        result = opal_atomic_swap_64(addr, *operand);
         break;
     case MCA_BTL_ATOMIC_MIN:
-        result = opal_atomic_fetch_min_64 (addr, *operand);
+        result = opal_atomic_fetch_min_64(addr, *operand);
         break;
     case MCA_BTL_ATOMIC_MAX:
-        result = opal_atomic_fetch_max_64 (addr, *operand);
+        result = opal_atomic_fetch_max_64(addr, *operand);
         break;
     default:
         return OPAL_ERR_BAD_PARAM;
@@ -829,32 +875,33 @@ static int mca_btl_base_am_atomic_64 (int64_t *operand, opal_atomic_int64_t *add
     return OPAL_SUCCESS;
 }
 
-static int mca_btl_base_am_atomic_32 (int32_t *operand, opal_atomic_int32_t *addr,
-                                      mca_btl_base_atomic_op_t op)
+
+static int am_rdma_atomic_32(int32_t *operand, opal_atomic_int32_t *addr,
+                             mca_btl_base_atomic_op_t op)
 {
     int32_t result = 0;
 
     switch (op) {
     case MCA_BTL_ATOMIC_ADD:
-        result = opal_atomic_fetch_add_32 (addr, *operand);
+        result = opal_atomic_fetch_add_32(addr, *operand);
         break;
     case MCA_BTL_ATOMIC_AND:
-        result = opal_atomic_fetch_and_32 (addr, *operand);
+        result = opal_atomic_fetch_and_32(addr, *operand);
         break;
     case MCA_BTL_ATOMIC_OR:
-        result = opal_atomic_fetch_or_32 (addr, *operand);
+        result = opal_atomic_fetch_or_32(addr, *operand);
         break;
     case MCA_BTL_ATOMIC_XOR:
-        result = opal_atomic_fetch_xor_32 (addr, *operand);
+        result = opal_atomic_fetch_xor_32(addr, *operand);
         break;
     case MCA_BTL_ATOMIC_SWAP:
-        result = opal_atomic_swap_32 (addr, *operand);
+        result = opal_atomic_swap_32(addr, *operand);
         break;
     case MCA_BTL_ATOMIC_MIN:
-        result = opal_atomic_fetch_min_32 (addr, *operand);
+        result = opal_atomic_fetch_min_32(addr, *operand);
         break;
     case MCA_BTL_ATOMIC_MAX:
-        result = opal_atomic_fetch_max_32 (addr, *operand);
+        result = opal_atomic_fetch_max_32(addr, *operand);
         break;
     default:
         return OPAL_ERR_BAD_PARAM;
@@ -864,272 +911,501 @@ static int mca_btl_base_am_atomic_32 (int32_t *operand, opal_atomic_int32_t *add
     return OPAL_SUCCESS;
 }
 
-static void mca_btl_base_am_rdma_response (mca_btl_base_module_t *btl,
-                                           const mca_btl_base_receive_descriptor_t *desc)
+
+static void am_rdma_response(mca_btl_base_module_t *btl,
+                             const mca_btl_base_receive_descriptor_t *desc)
 {
-    mca_btl_base_rdma_response_hdr_t *resp_hdr =
-        (mca_btl_base_rdma_response_hdr_t *)desc->des_segments[0].seg_addr.pval;
+    am_rdma_response_hdr_t *resp_hdr = (am_rdma_response_hdr_t *) desc
+                                                     ->des_segments[0]
+                                                     .seg_addr.pval;
 
-    assert (desc->des_segments[0].seg_len >= sizeof (*resp_hdr));
+    assert(desc->des_segments[0].seg_len >= sizeof(*resp_hdr));
 
-    mca_btl_base_rdma_context_t *context =
-        (mca_btl_base_rdma_context_t *)(uintptr_t)resp_hdr->context;
+    am_rdma_context_t *context = (am_rdma_context_t *) (uintptr_t)
+                                               resp_hdr->context;
 
-    BTL_VERBOSE (("received response for RDMA operation. context=%p, size=%" PRIu64, context,
-                  resp_hdr->response_size));
+    BTL_VERBOSE(("received response for RDMA operation. context=%p, size=%" PRIu64, (void*) context,
+                 resp_hdr->response_size));
 
     if (MCA_BTL_BASE_AM_PUT != context->type) {
         uint64_t local_address = resp_hdr->initiator_address ? resp_hdr->initiator_address
-                                                             : (uintptr_t)context->local_address;
+                                                             : (uintptr_t) context->local_address;
         if (local_address && MCA_BTL_BASE_AM_PUT != context->type) {
-            BTL_VERBOSE (("unpacking response from packet"));
+            BTL_VERBOSE(("unpacking response from packet"));
             /* if there is a result copy it out of the incoming buffer. if RDMA is being used
              * (get/put or put/get) then the header should be the only thing in the incoming
              * message. */
-            mca_btl_base_copy_from_segments (local_address, sizeof (*resp_hdr), desc->des_segments,
-                                             desc->des_segment_count);
+            am_rdma_copy_from_segments(local_address, sizeof(*resp_hdr), desc->des_segments,
+                                       desc->des_segment_count);
         }
     }
 
-    if (context->total_size ==
-        opal_atomic_add_fetch_64 (&context->acknowledged, resp_hdr->response_size)) {
-        context->cbfunc (btl, desc->endpoint, context->local_address, context->local_handle,
-                         context->cbcontext, context->cbdata, OPAL_SUCCESS);
-        OBJ_RELEASE (context);
+    if (context->total_size
+        == (uint64_t) opal_atomic_add_fetch_64(&context->acknowledged, resp_hdr->response_size)) {
+        context->cbfunc(btl, desc->endpoint, context->local_address, context->local_handle,
+                        context->cbcontext, context->cbdata, OPAL_SUCCESS);
+        OBJ_RELEASE(context);
     }
 }
 
-static void mca_btl_base_am_process_rdma (mca_btl_base_module_t *btl,
-                                          const mca_btl_base_receive_descriptor_t *desc)
+
+static void am_rdma_process_rdma(mca_btl_base_module_t *btl,
+                                 const mca_btl_base_receive_descriptor_t *desc)
 {
     /* not all btls work with these active message atomics. at this time
      * all of the affected btls already have atomic support so there is
      * no need to attempt to get the endpoint here. */
     if (NULL == desc->endpoint) {
-        BTL_ERROR (("BTL is not compatible with active-message RDMA"));
-        abort ();
+        BTL_ERROR(("BTL is not compatible with active-message RDMA"));
+        abort();
     }
 
-    const mca_btl_base_rdma_hdr_t *hdr =
-        (mca_btl_base_rdma_hdr_t *)desc->des_segments[0].seg_addr.pval;
-    void *target_address                     = (void *)(intptr_t)hdr->target_address;
-    mca_btl_base_descriptor_t *descriptor    = NULL;
-    mca_btl_base_rdma_operation_t *operation = NULL;
+    const am_rdma_hdr_t *hdr = (am_rdma_hdr_t *) desc->des_segments[0]
+                                             .seg_addr.pval;
+    void *target_address = (void *) (intptr_t) hdr->target_address;
+    mca_btl_base_descriptor_t *descriptor = NULL;
+    am_rdma_operation_t *operation = NULL;
     int ret;
 
-    BTL_VERBOSE (("got active-message \"RDMA\" request. hdr->context=0x%" PRIx64
-                  ", target_address=%p, "
-                  "segment 0 size=%" PRIu64,
-                  hdr->context, target_address, desc->des_segments[0].seg_len));
+    BTL_VERBOSE(("got active-message \"RDMA\" request. hdr->context=0x%" PRIx64
+                 ", target_address=%p, "
+                 "segment 0 size=%" PRIu64,
+                 hdr->context, target_address, desc->des_segments[0].seg_len));
 
     if (MCA_BTL_BASE_AM_PUT == hdr->type) {
-        ret = mca_btl_base_am_rdma_target_put (btl, desc->endpoint, &descriptor, desc->des_segments,
-                                               desc->des_segment_count, target_address, hdr,
-                                               &operation);
+        ret = am_rdma_target_put(btl, desc->endpoint, &descriptor, desc->des_segments,
+                                              desc->des_segment_count, target_address, hdr,
+                                              &operation);
     } else if (MCA_BTL_BASE_AM_GET == hdr->type) {
-        ret = mca_btl_base_am_rdma_target_get (btl, desc->endpoint, &descriptor, target_address,
-                                               hdr, &operation);
+        ret = am_rdma_target_get(btl, desc->endpoint, &descriptor, target_address, hdr,
+                                              &operation);
     } else {
-        BTL_ERROR (("Unexpected tag when processing active-message RDMA request"));
-        abort ();
+        BTL_ERROR(("Unexpected tag when processing active-message RDMA request"));
+        abort();
     }
 
     if (OPAL_SUCCESS != ret) {
-        mca_btl_base_rdma_queue_operation (btl, desc->endpoint, descriptor, 0, hdr, operation);
+        am_rdma_queue_operation(btl, desc->endpoint, descriptor, 0, hdr, operation);
     }
 }
 
-static void mca_btl_base_am_process_atomic (mca_btl_base_module_t *btl,
-                                            const mca_btl_base_receive_descriptor_t *desc)
+
+static void am_rdma_process_atomic(mca_btl_base_module_t *btl,
+                                   const mca_btl_base_receive_descriptor_t *desc)
 {
     /* not all btls work with these active message atomics. at this time
      * all of the affected btls already have atomic support so there is
      * no need to attempt to get the endpoint here. */
     if (NULL == desc->endpoint) {
-        BTL_ERROR (("BTL is not compatible with active-message RDMA"));
-        abort ();
+        BTL_ERROR(("BTL is not compatible with active-message RDMA"));
+        abort();
     }
 
-    const mca_btl_base_rdma_hdr_t *hdr =
-        (mca_btl_base_rdma_hdr_t *)desc->des_segments[0].seg_addr.pval;
-    void *target_address     = (void *)(intptr_t)hdr->target_address;
+    const am_rdma_hdr_t *hdr = (am_rdma_hdr_t *) desc->des_segments[0]
+                                             .seg_addr.pval;
     uint64_t atomic_response = hdr->data.atomic.operand[0];
 
     if (4 != hdr->data.atomic.size && 8 != hdr->data.atomic.size) {
-        BTL_ERROR (("Unexpected atomic operation size: %hu", hdr->data.atomic.size));
-        abort ();
+        BTL_ERROR(("Unexpected atomic operation size: %hu", hdr->data.atomic.size));
+        abort();
     }
 
-    BTL_VERBOSE (("got active-message atomic request. hdr->context=0x%" PRIx64
-                  ", target_address=%p, "
-                  "segment 0 size=%" PRIu64,
-                  hdr->context, target_address, desc->des_segments[0].seg_len));
+    BTL_VERBOSE(("got active-message atomic request. hdr->context=0x%" PRIx64
+                 ", target_address=%p, "
+                 "segment 0 size=%" PRIu64,
+                 hdr->context, (void *)(intptr_t)hdr->target_address, desc->des_segments[0].seg_len));
 
     switch (hdr->type) {
     case MCA_BTL_BASE_AM_ATOMIC:
         if (4 == hdr->data.atomic.size) {
-            uint32_t tmp = (uint32_t)atomic_response;
-            mca_btl_base_am_atomic_32 (&tmp, (opal_atomic_int32_t *)(uintptr_t)hdr->target_address,
-                                       hdr->data.atomic.op);
+            int32_t tmp = (int32_t) atomic_response;
+            am_rdma_atomic_32(&tmp, (opal_atomic_int32_t *) hdr->target_address,
+                              hdr->data.atomic.op);
+            atomic_response = tmp;
+        } else if (8 == hdr->data.atomic.size) {
+            int64_t tmp = (int64_t) atomic_response;
+            am_rdma_atomic_64(&tmp,
+                              (opal_atomic_int64_t *) hdr->target_address,
+                              hdr->data.atomic.op);
             atomic_response = tmp;
-        }
-        if (8 == hdr->data.atomic.size) {
-            mca_btl_base_am_atomic_64 ((int64_t *)hdr->target_address,
-                                       (opal_atomic_int64_t *)(uintptr_t)hdr->target_address,
-                                       hdr->data.atomic.op);
         }
         break;
     case MCA_BTL_BASE_AM_CAS:
         if (4 == hdr->data.atomic.size) {
-            int32_t tmp = (int32_t)atomic_response;
-            opal_atomic_compare_exchange_strong_32 ((opal_atomic_int32_t *)hdr->target_address,
-                                                    &tmp, (int32_t)hdr->data.atomic.operand[1]);
+            int32_t tmp = (int32_t) atomic_response;
+            opal_atomic_compare_exchange_strong_32((opal_atomic_int32_t *) hdr->target_address,
+                                                   &tmp, (int32_t) hdr->data.atomic.operand[1]);
+            atomic_response = tmp;
+        } else if (8 == hdr->data.atomic.size) {
+            int64_t tmp = (int64_t) atomic_response;
+            opal_atomic_compare_exchange_strong_64((opal_atomic_int64_t *) hdr->target_address,
+                                                   &tmp, hdr->data.atomic.operand[1]);
             atomic_response = tmp;
-        }
-        if (8 == hdr->data.atomic.size) {
-            opal_atomic_compare_exchange_strong_64 ((opal_atomic_int64_t *)hdr->target_address,
-                                                    &atomic_response, hdr->data.atomic.operand[1]);
         }
         break;
     default:
-        BTL_ERROR (("Unexpected AM atomic request type"));
-        abort ();
+        BTL_ERROR(("Unexpected AM atomic request type"));
+        abort();
     }
 
     mca_btl_base_descriptor_t *descriptor = NULL;
-    int ret =
-        mca_btl_base_am_rdma_respond (btl, desc->endpoint, &descriptor, &atomic_response, hdr);
+    int ret = am_rdma_respond(btl, desc->endpoint, &descriptor, &atomic_response, hdr);
     if (OPAL_SUCCESS != ret) {
-        mca_btl_base_rdma_queue_operation (btl, desc->endpoint, descriptor, atomic_response, hdr,
-                                           NULL);
+        am_rdma_queue_operation(btl, desc->endpoint, descriptor, atomic_response, hdr,
+                                          NULL);
     }
 }
 
-void mca_btl_sm_sc_emu_init (void)
+
+static int am_rdma_put(mca_btl_base_am_rdma_module_t *am_module,
+                       struct mca_btl_base_endpoint_t *endpoint, void *local_address,
+                       uint64_t remote_address,
+                       struct mca_btl_base_registration_handle_t *local_handle,
+                       struct mca_btl_base_registration_handle_t *remote_handle,
+                       size_t size, int flags, int order,
+                       mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext,
+                       void *cbdata)
 {
-    mca_btl_base_active_message_trigger[MCA_BTL_BASE_TAG_RDMA].cbfunc =
-        mca_btl_base_am_process_rdma;
-    mca_btl_base_active_message_trigger[MCA_BTL_BASE_TAG_ATOMIC].cbfunc =
-        mca_btl_base_am_process_atomic;
-    mca_btl_base_active_message_trigger[MCA_BTL_BASE_TAG_RDMA_RESP].cbfunc =
-        mca_btl_base_am_rdma_response;
+    return am_rdma_start(am_module, endpoint, MCA_BTL_BASE_AM_PUT, 0, 0, 0, order, flags, size,
+                         local_address, local_handle, remote_address, remote_handle,
+                         cbfunc, cbcontext, cbdata);
 }
 
-static int mca_btl_base_am_fop (struct mca_btl_base_module_t *btl,
-                                struct mca_btl_base_endpoint_t *endpoint, void *local_address,
-                                uint64_t remote_address,
-                                mca_btl_base_registration_handle_t *local_handle,
-                                mca_btl_base_registration_handle_t *remote_handle,
-                                mca_btl_base_atomic_op_t op, uint64_t operand, int flags, int order,
-                                mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext,
-                                void *cbdata)
+
+static int am_rdma_get(mca_btl_base_am_rdma_module_t *am_module,
+                       struct mca_btl_base_endpoint_t *endpoint, void *local_address,
+                       uint64_t remote_address,
+                       struct mca_btl_base_registration_handle_t *local_handle,
+                       struct mca_btl_base_registration_handle_t *remote_handle,
+                       size_t size, int flags, int order,
+                       mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext,
+                       void *cbdata)
+{
+    return am_rdma_start(am_module, endpoint, MCA_BTL_BASE_AM_GET, 0, 0, 0, order, flags, size,
+                         local_address, local_handle, remote_address, remote_handle,
+                         cbfunc, cbcontext, cbdata);
+}
+
+
+static int am_rdma_fop(mca_btl_base_am_rdma_module_t *am_module,
+                       struct mca_btl_base_endpoint_t *endpoint, void *local_address,
+                       uint64_t remote_address,
+                       mca_btl_base_registration_handle_t *local_handle,
+                       mca_btl_base_registration_handle_t *remote_handle,
+                       mca_btl_base_atomic_op_t op, uint64_t operand, int flags, int order,
+                       mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext,
+                       void *cbdata)
 {
     size_t size = (flags & MCA_BTL_ATOMIC_FLAG_32BIT) ? 4 : 8;
-    return mca_btl_base_rdma_start (btl, endpoint, MCA_BTL_BASE_AM_ATOMIC, operand, 0, op, order,
-                                    flags, size, local_address, local_handle, remote_address,
-                                    remote_handle, cbfunc, cbcontext, cbdata);
+
+    return am_rdma_start(am_module, endpoint, MCA_BTL_BASE_AM_ATOMIC, operand, 0, op, order,
+                         flags, size, local_address, local_handle, remote_address,
+                         remote_handle, cbfunc, cbcontext, cbdata);
 }
 
-static int mca_btl_base_am_cswap (
-    struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
-    void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-    mca_btl_base_registration_handle_t *remote_handle, uint64_t compare, uint64_t value, int flags,
-    int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
+
+static int am_rdma_cswap(mca_btl_base_am_rdma_module_t *am_module,
+                         struct mca_btl_base_endpoint_t *endpoint,
+                         void *local_address, uint64_t remote_address,
+                         mca_btl_base_registration_handle_t *local_handle,
+                         mca_btl_base_registration_handle_t *remote_handle,
+                         uint64_t compare, uint64_t value, int flags, int order,
+                         mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext,
+                         void *cbdata)
 {
     size_t size = (flags & MCA_BTL_ATOMIC_FLAG_32BIT) ? 4 : 8;
-    return mca_btl_base_rdma_start (btl, endpoint, MCA_BTL_BASE_AM_CAS, compare, value, 0, order,
-                                    flags, size, local_address, local_handle, remote_address,
-                                    remote_handle, cbfunc, cbcontext, cbdata);
+
+    return am_rdma_start(am_module, endpoint, MCA_BTL_BASE_AM_CAS, compare, value, 0, order,
+                         flags, size, local_address, local_handle, remote_address,
+                         remote_handle, cbfunc, cbcontext, cbdata);
 }
 
-static int mca_btl_base_am_rdma_get (struct mca_btl_base_module_t *btl,
-                                     struct mca_btl_base_endpoint_t *endpoint, void *local_address,
-                                     uint64_t remote_address,
-                                     struct mca_btl_base_registration_handle_t *local_handle,
-                                     struct mca_btl_base_registration_handle_t *remote_handle,
-                                     size_t size, int flags, int order,
-                                     mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext,
-                                     void *cbdata)
+
+static mca_btl_base_am_rdma_module_t *am_rdma_get_module(struct mca_btl_base_module_t *btl)
 {
-    return mca_btl_base_rdma_start (btl, endpoint, MCA_BTL_BASE_AM_GET, 0, 0, 0, order, flags, size,
-                                    local_address, local_handle, remote_address, remote_handle,
-                                    cbfunc, cbcontext, cbdata);
+    assert(NULL != btl->btl_am_data);
+    return (mca_btl_base_am_rdma_module_t *)btl->btl_am_data;
 }
 
-static int mca_btl_base_am_rdma_put (struct mca_btl_base_module_t *btl,
-                                     struct mca_btl_base_endpoint_t *endpoint, void *local_address,
-                                     uint64_t remote_address,
-                                     struct mca_btl_base_registration_handle_t *local_handle,
-                                     struct mca_btl_base_registration_handle_t *remote_handle,
-                                     size_t size, int flags, int order,
-                                     mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext,
-                                     void *cbdata)
+
+static int am_rdma_put_wrapper(struct mca_btl_base_module_t *btl,
+                               struct mca_btl_base_endpoint_t *endpoint, void *local_address,
+                               uint64_t remote_address,
+                               struct mca_btl_base_registration_handle_t *local_handle,
+                               struct mca_btl_base_registration_handle_t *remote_handle,
+                               size_t size, int flags, int order,
+                               mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext,
+                               void *cbdata)
 {
-    return mca_btl_base_rdma_start (btl, endpoint, MCA_BTL_BASE_AM_PUT, 0, 0, 0, order, flags, size,
-                                    local_address, local_handle, remote_address, remote_handle,
-                                    cbfunc, cbcontext, cbdata);
+    mca_btl_base_am_rdma_module_t *am_module = am_rdma_get_module(btl);
+
+    return am_rdma_start(am_module, endpoint, MCA_BTL_BASE_AM_PUT, 0, 0, 0, order, flags, size,
+                         local_address, local_handle, remote_address, remote_handle,
+                         cbfunc, cbcontext, cbdata);
 }
 
-int mca_btl_base_am_rdma_init (mca_btl_base_module_t *btl)
+
+static int am_rdma_get_wrapper(struct mca_btl_base_module_t *btl,
+                               struct mca_btl_base_endpoint_t *endpoint, void *local_address,
+                               uint64_t remote_address,
+                               struct mca_btl_base_registration_handle_t *local_handle,
+                               struct mca_btl_base_registration_handle_t *remote_handle,
+                               size_t size, int flags, int order,
+                               mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext,
+                               void *cbdata)
 {
-    static bool progress_registered = false;
+    mca_btl_base_am_rdma_module_t *am_module = am_rdma_get_module(btl);
 
-    if ((btl->btl_flags & (MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_ATOMIC_FOPS)) ==
-        (MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_ATOMIC_FOPS)) {
-        /* nothing to do */
-        return OPAL_SUCCESS;
+    return am_rdma_start(am_module, endpoint, MCA_BTL_BASE_AM_GET, 0, 0, 0, order, flags, size,
+                         local_address, local_handle, remote_address, remote_handle,
+                         cbfunc, cbcontext, cbdata);
+}
+
+
+static int am_rdma_fop_wrapper(struct mca_btl_base_module_t *btl,
+                               struct mca_btl_base_endpoint_t *endpoint, void *local_address,
+                               uint64_t remote_address,
+                               mca_btl_base_registration_handle_t *local_handle,
+                               mca_btl_base_registration_handle_t *remote_handle,
+                               mca_btl_base_atomic_op_t op, uint64_t operand, int flags, int order,
+                               mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext,
+                               void *cbdata)
+{
+    mca_btl_base_am_rdma_module_t *am_module = am_rdma_get_module(btl);
+    size_t size = (flags & MCA_BTL_ATOMIC_FLAG_32BIT) ? 4 : 8;
+
+    return am_rdma_start(am_module, endpoint, MCA_BTL_BASE_AM_ATOMIC, operand, 0, op, order,
+                         flags, size, local_address, local_handle, remote_address,
+                         remote_handle, cbfunc, cbcontext, cbdata);
+}
+
+
+static int am_rdma_cswap_wrapper(struct mca_btl_base_module_t *btl,
+                                 struct mca_btl_base_endpoint_t *endpoint,
+                                 void *local_address, uint64_t remote_address,
+                                 mca_btl_base_registration_handle_t *local_handle,
+                                 mca_btl_base_registration_handle_t *remote_handle,
+                                 uint64_t compare, uint64_t value, int flags, int order,
+                                 mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext,
+                                 void *cbdata)
+{
+    mca_btl_base_am_rdma_module_t *am_module = am_rdma_get_module(btl);
+    size_t size = (flags & MCA_BTL_ATOMIC_FLAG_32BIT) ? 4 : 8;
+
+    return am_rdma_start(am_module, endpoint, MCA_BTL_BASE_AM_CAS, compare, value, 0, order,
+                         flags, size, local_address, local_handle, remote_address,
+                         remote_handle, cbfunc, cbcontext, cbdata);
+}
+
+
+static void am_rdma_register_callbacks(void)
+{
+    mca_btl_base_active_message_trigger[MCA_BTL_BASE_TAG_RDMA].cbfunc
+        = am_rdma_process_rdma;
+    mca_btl_base_active_message_trigger[MCA_BTL_BASE_TAG_ATOMIC].cbfunc
+        = am_rdma_process_atomic;
+    mca_btl_base_active_message_trigger[MCA_BTL_BASE_TAG_RDMA_RESP].cbfunc
+        = am_rdma_response;
+}
+
+
+static int am_rdma_internal_init(mca_btl_base_module_t *btl,
+                                 uint32_t flags_requested,
+                                 bool no_memory_registration,
+                                 mca_btl_base_am_rdma_module_t **new_module)
+{
+    static bool initialized = false;
+    static opal_mutex_t initialized_mutex = OPAL_MUTEX_STATIC_INIT;
+    mca_btl_base_am_rdma_module_t *module;
+    size_t max_operation_size;
+    size_t operation_alignment;
+
+    opal_mutex_lock(&initialized_mutex);
+    if (!initialized) {
+        initialized = true;
+        OBJ_CONSTRUCT(&default_component, am_rdma_component_t);
+        opal_progress_register(am_rdma_progress);
+        am_rdma_register_callbacks();
     }
+    opal_mutex_unlock(&initialized_mutex);
 
-    size_t max_operation_size  = btl->btl_max_send_size;
-    size_t operation_alignment = 1;
-    if (mca_btl_base_rdma_use_rdma_get (btl)) {
-        /* implement put over get. */
-        max_operation_size  = btl->btl_get_limit;
+    module = OBJ_NEW(mca_btl_base_am_rdma_module_t);
+    if (NULL == module) {
+        return OPAL_ERR_TEMP_OUT_OF_RESOURCE;
+    }
+
+    module->btl = btl;
+    module->use_rdma_put = !!(btl->btl_flags & MCA_BTL_FLAGS_PUT);
+    module->use_rdma_get = !!(btl->btl_flags & MCA_BTL_FLAGS_GET);
+
+    /* if the requester asked for remote completion and the btl does
+     * not provide remove completion, we can not use put.
+     */
+    if (!(btl->btl_flags & MCA_BTL_FLAGS_RDMA_REMOTE_COMPLETION)) {
+        module->use_rdma_put = false;
+    }
+
+    /* if the requester does not want to do memory registration and
+     * the BTL requires memory registration, disable the use of RDMA.
+     */
+    if (no_memory_registration && NULL != btl->btl_register_mem) {
+        module->use_rdma_put = false;
+        module->use_rdma_get = false;
+    }
+
+    if (module->use_rdma_get) {
+        /* implement operations over get. */
+        max_operation_size = btl->btl_get_limit;
         operation_alignment = btl->btl_get_alignment;
-    } else if (mca_btl_base_rdma_use_rdma_put (btl)) {
-        /* implement get over put. */
-        max_operation_size  = btl->btl_put_limit;
+        BTL_VERBOSE(("am_rdma_init: btl %p using get.  operation size %zu, alignment %zu",
+                     (void *)btl, max_operation_size, operation_alignment));
+    } else if (module->use_rdma_put) {
+        /* implement operations over put. */
+        max_operation_size = btl->btl_put_limit;
         operation_alignment = btl->btl_put_alignment;
+        BTL_VERBOSE(("am_rdma_init: btl %p using put.  operation size %zu, alignment %zu",
+                     (void *)btl, max_operation_size, operation_alignment));
+    } else {
+        /* implement operations over send. */
+        max_operation_size = btl->btl_max_send_size;
+        operation_alignment = 1;
+        BTL_VERBOSE(("am_rdma_init: btl %p using send.  operation size %zu, alignment %zu",
+                     (void *)btl, max_operation_size, operation_alignment));
+    }
+
+    module->am_btl_put_limit = max_operation_size - sizeof(am_rdma_hdr_t);
+    module->am_btl_put_alignment = operation_alignment;
+    module->am_btl_get_limit = max_operation_size - sizeof(am_rdma_response_hdr_t);
+    module->am_btl_get_alignment = operation_alignment;
+
+    module->am_btl_put = am_rdma_put;
+    module->am_btl_get = am_rdma_get;
+    module->am_btl_atomic_fop = am_rdma_fop;
+    module->am_btl_atomic_cswap = am_rdma_cswap;
+
+    *new_module = module;
+
+    return OPAL_SUCCESS;
+}
+
+
+static int am_rdma_internal_fini(mca_btl_base_am_rdma_module_t *am_rdma_module)
+{
+    OBJ_RELEASE(am_rdma_module);
+
+    return OPAL_SUCCESS;
+}
+
+
+int mca_btl_base_am_rdma_init(mca_btl_base_module_t *btl)
+{
+    mca_btl_base_am_rdma_module_t *am_module;
+    int ret;
+
+    BTL_VERBOSE(("am_rdma_init: called for btl %s (%p)",
+                 btl->btl_component->btl_version.mca_component_name, (void *)btl));
+
+    if ((btl->btl_flags & (MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_ATOMIC_FOPS))
+        == (MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_ATOMIC_FOPS)) {
+        BTL_VERBOSE(("am_rdma_init: btl %p already supports rdma", (void *)btl));
+        return OPAL_SUCCESS;
     }
 
+    /*
+     * note that it is not safe to access any am rdma functionality
+     * (even default_component global data) until internal_init returns
+     * successfully.
+     */
+    ret = am_rdma_internal_init(btl, 0, false, &am_module);
+    if (OPAL_SUCCESS != ret) {
+        BTL_VERBOSE(("am_rdma_init: btl %p internal_init failure %d",
+                     (void *)btl, ret));
+        return ret;
+    }
+
+    /*
+     * we can't lock any field on the BTL structure (because it's not
+     * ours to poke at), so take the global am rdma lock.  I suppose we
+     * could do a cswap of the btl_am_data pointer to the same result,
+     * but that seems too cute for something that should be a relatively
+     * rare event.
+     */
+    opal_mutex_lock(&default_component.mutex);
+    if (NULL != btl->btl_am_data) {
+        BTL_VERBOSE(("am_rdma_init: btl %p already initialized", (void *)btl));
+        am_rdma_internal_fini(am_module);
+        opal_mutex_unlock(&default_component.mutex);
+        return OPAL_SUCCESS;
+    }
+    opal_mutex_unlock(&default_component.mutex);
+
+    btl->btl_am_data = am_module;
+
+    /* TODO: Ideally, we would swap the BTL's flush for our own
+     * implementation which completed all outstanding transactions on
+     * that BTL and then called the underlying flush().  Given the
+     * work and the lack of use case today, we instead just remove
+     * flush support from the underlying BTL. */
+    btl->btl_flush = NULL;
+
     if (!(btl->btl_flags & MCA_BTL_FLAGS_PUT)) {
-        BTL_VERBOSE (("Enabling AM-based RDMA put for BTL %p", btl));
         btl->btl_flags |= MCA_BTL_FLAGS_PUT_AM;
-        btl->btl_put_limit     = max_operation_size;
-        btl->btl_put_alignment = operation_alignment;
-        btl->btl_put           = mca_btl_base_am_rdma_put;
+        btl->btl_put_limit = am_module->am_btl_put_limit;
+        btl->btl_put_alignment = am_module->am_btl_put_alignment;
+        btl->btl_put = am_rdma_put_wrapper;
+        BTL_VERBOSE(("am_rdma_init: Enabling AM-based RDMA put for BTL %p. max put = %zu", (void*)btl, btl->btl_put_limit));
     }
 
     if (!(btl->btl_flags & MCA_BTL_FLAGS_GET)) {
-        BTL_VERBOSE (("Enabling AM-based RDMA get for BTL %p", btl));
         btl->btl_flags |= MCA_BTL_FLAGS_GET_AM;
-        btl->btl_get_limit     = max_operation_size;
-        btl->btl_get_alignment = operation_alignment;
-        btl->btl_get           = mca_btl_base_am_rdma_get;
+        btl->btl_get_limit = am_module->am_btl_get_limit;
+        btl->btl_get_alignment = am_module->am_btl_get_alignment;
+        btl->btl_get = am_rdma_get_wrapper;
+        BTL_VERBOSE(("Enabling AM-based RDMA get for BTL %p. max get = %zu", (void*)btl, btl->btl_get_limit));
     }
 
     if (!(btl->btl_flags & MCA_BTL_FLAGS_ATOMIC_FOPS)) {
-        BTL_VERBOSE (("Enabling AM-based FOPs get for BTL %p", btl));
         btl->btl_flags |= MCA_BTL_FLAGS_ATOMIC_AM_FOP;
 
-        btl->btl_atomic_fop   = mca_btl_base_am_fop;
-        btl->btl_atomic_cswap = mca_btl_base_am_cswap;
+        btl->btl_atomic_fop = am_rdma_fop_wrapper;
+        btl->btl_atomic_cswap = am_rdma_cswap_wrapper;
 
         /* emulated RDMA atomics can support the full range of atomics. for
          * now only a handful are supported. */
-        btl->btl_atomic_flags = MCA_BTL_ATOMIC_SUPPORTS_GLOB | MCA_BTL_ATOMIC_SUPPORTS_CSWAP |
-                                MCA_BTL_ATOMIC_SUPPORTS_32BIT | MCA_BTL_ATOMIC_SUPPORTS_ADD |
-                                MCA_BTL_ATOMIC_SUPPORTS_AND | MCA_BTL_ATOMIC_SUPPORTS_OR |
-                                MCA_BTL_ATOMIC_SUPPORTS_XOR | MCA_BTL_ATOMIC_SUPPORTS_SWAP |
-                                MCA_BTL_ATOMIC_SUPPORTS_MIN | MCA_BTL_ATOMIC_SUPPORTS_MAX;
+        btl->btl_atomic_flags = MCA_BTL_ATOMIC_SUPPORTS_GLOB | MCA_BTL_ATOMIC_SUPPORTS_CSWAP
+                                | MCA_BTL_ATOMIC_SUPPORTS_32BIT | MCA_BTL_ATOMIC_SUPPORTS_ADD
+                                | MCA_BTL_ATOMIC_SUPPORTS_AND | MCA_BTL_ATOMIC_SUPPORTS_OR
+                                | MCA_BTL_ATOMIC_SUPPORTS_XOR | MCA_BTL_ATOMIC_SUPPORTS_SWAP
+                                | MCA_BTL_ATOMIC_SUPPORTS_MIN | MCA_BTL_ATOMIC_SUPPORTS_MAX;
+        BTL_VERBOSE(("Enabling AM-based FOPs get for BTL %p", (void*)btl));
     }
 
-    if (!progress_registered) {
-        progress_registered = true;
-        opal_progress_register (mca_btl_base_am_rdma_progress);
-        mca_btl_sm_sc_emu_init ();
-        OBJ_CONSTRUCT (&default_module, mca_btl_base_am_rdma_module_t);
+    return OPAL_SUCCESS;
+}
+
+
+int opal_btl_base_am_rdma_create(mca_btl_base_module_t *btl,
+                                 uint32_t flags_requested,
+                                 bool no_memory_registration,
+                                 mca_btl_base_am_rdma_module_t **am_module)
+{
+    int ret;
+
+    BTL_VERBOSE(("am_rdma_create: called for btl %s (%p)",
+                 btl->btl_component->btl_version.mca_component_name, (void *)btl));
+
+    ret = am_rdma_internal_init(btl, flags_requested, no_memory_registration, am_module);
+    if (OPAL_SUCCESS != ret) {
+        BTL_VERBOSE(("am_rdma_create: btl %p internal_init failure %d",
+                     (void *)btl, ret));
+        return ret;
     }
 
     return OPAL_SUCCESS;
 }
+
+
+int opal_btl_base_am_rdma_destroy(mca_btl_base_am_rdma_module_t *am_module)
+{
+    return am_rdma_internal_fini(am_module);
+}
diff --git a/opal/mca/btl/base/btl_base_am_rdma.h b/opal/mca/btl/base/btl_base_am_rdma.h
index 140f858d488..bd1641f1cc2 100644
--- a/opal/mca/btl/base/btl_base_am_rdma.h
+++ b/opal/mca/btl/base/btl_base_am_rdma.h
@@ -3,6 +3,8 @@
  * Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2020      Google, LLC. All rights reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -13,29 +15,137 @@
 /**
  * This file provides support for active-message (send/recv) based RDMA.
  * It can be used with any btl that provides a minimum of send support but
- * can also be used with partial-RDMA BTLs (put only, get only, etc). It
- * will provide support for any RDMA or atomic operation not currently
- * supported by the supplied BTL. For more info see the description of
- * mca_btl_base_am_rdma_init.
+ * can also be used with partial-RDMA BTLs (put only, get only, etc)
+ * to provide a complete RDMA interface.
+ *
+ * There are two modes of using this interface, depending on your
+ * requirements:
+ *
+ * First, this interface can be used to provide a complete
+ * put/get/atomic interface for BTLs which do not natively provide
+ * such an interface.  In this mode, active message rdma functions are
+ * only used if the underlying implementation does not already provide
+ * the required functionality.  For example, if a BTL natively
+ * supports put but not get, the interface would provide an emulated
+ * get.  The registration, completion and atomicity semantics of the
+ * BTL remain the native interface's capabilities.  That is, if the
+ * native interface does not provide remote completion or atomics that
+ * are atomic with processor atomics, neither will the interface after
+ * initializing the am rdma interface for that BTL.  This mode will
+ * likely give better performance than the second mode for transfers
+ * that fit within the BTL's native semantics.  In this mode, the BTL
+ * interface is updated so that the btl_{put, get, atomic_fop,
+ * atomic_cswap} function pointers are usage.  However, the btl
+ * capability flags will not be updated to indicate native support of
+ * the emulated functionality (for example, if btl_get() is emulated,
+ * MCA_BTL_FLAGS_GET will not be set).  Instead, the emulated flags
+ * will be set (MCA_BTL_FLAGS_PUT_AM, MCA_BTL_FLAGS_GET_AM,
+ * MCA_BTL_FLAGS_ATOMIC_AM_FOP, etc.).
+ *
+ * Second, this interface can be used to provide different
+ * semantics than a BTL natively provides.  This mode is not
+ * transparent to the caller (unlike the first mode).  Instead, the
+ * caller must manage calling the active message put/get/atomic
+ * interface directly (rather than through the BTL function pointers).
+ * For interfaces which require strict remote completion or require
+ * implicit memory registration, this can greatly simplify the code,
+ * in return for marginally more management complexity and lower
+ * performance.
+ *
+ * While the calling convention and initialization are different, the
+ * communication routines uses by the active message rdma
+ * implementation are identical in both modes of operation.
  */
 
-#include "opal/mca/btl/btl.h"
 #include "opal_config.h"
+#include "opal/mca/btl/btl.h"
 
 #if !defined(OPAL_MCA_BTL_BASE_AM_RDMA_H)
-#define OPAL_MCA_BTL_BASE_AM_RDMA_H
+#    define OPAL_MCA_BTL_BASE_AM_RDMA_H
 
 /**
  * @brief initialize active-message RDMA/atomic support
  *
- * @inout btl   btl module to augment
+ * @param  btl[in,out]  btl module to augment
+ *
+ * @retval OPAL_SUCCESS  btl successfully updated, btl already
+ *                       updated, or btl has all available
+ *                       functionality natively.
+ * @retval OPAL_ERR_TEMP_OUT_OF_RESOURCE Allocating BTL-level data
+ *                       structure failed.
  *
  * This function adds functionality to the btl for any missing RDMA/atomic
  * operation. Atomic operations are entirely emulated using send/recv and
  * work best with a btl that also has async-progress enabled. Put/get
  * support will use either send/recv or get (for put)/put (for get) (if
  * available).
+ *
+ * Note that calling this function will change the BTL interface.
+ * Care must be taken to not call this function outside of early
+ * initialization routines.
  */
-int mca_btl_base_am_rdma_init (mca_btl_base_module_t *btl);
+int mca_btl_base_am_rdma_init(mca_btl_base_module_t *btl);
+
+struct mca_btl_base_am_rdma_module_t;
+
+typedef int (*mca_btl_base_am_rdma_module_put_fn_t)(
+    struct mca_btl_base_am_rdma_module_t *am_btl, struct mca_btl_base_endpoint_t *endpoint,
+    void *local_address, uint64_t remote_address,
+    struct mca_btl_base_registration_handle_t *local_handle,
+    struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags, int order,
+    mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
+
+typedef int (*mca_btl_base_am_rdma_module_get_fn_t)(
+    struct mca_btl_base_am_rdma_module_t *am_btl, struct mca_btl_base_endpoint_t *endpoint,
+    void *local_address, uint64_t remote_address,
+    struct mca_btl_base_registration_handle_t *local_handle,
+    struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags, int order,
+    mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
+
+typedef int (*mca_btl_base_am_rdma_module_atomic_fop64_fn_t)(
+    struct mca_btl_base_am_rdma_module_t *am_btl, struct mca_btl_base_endpoint_t *endpoint,
+    void *local_address, uint64_t remote_address,
+    struct mca_btl_base_registration_handle_t *local_handle,
+    struct mca_btl_base_registration_handle_t *remote_handle, mca_btl_base_atomic_op_t op,
+    uint64_t operand, int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
+    void *cbcontext, void *cbdata);
+
+typedef int (*mca_btl_base_am_rdma_module_atomic_cswap64_fn_t)(
+    struct mca_btl_base_am_rdma_module_t *am_btl, struct mca_btl_base_endpoint_t *endpoint,
+    void *local_address, uint64_t remote_address,
+    struct mca_btl_base_registration_handle_t *local_handle,
+    struct mca_btl_base_registration_handle_t *remote_handle, uint64_t compare, uint64_t value,
+    int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
+
+struct mca_btl_base_am_rdma_module_t {
+    opal_object_t super;
+    mca_btl_base_module_t *btl;
+    bool use_rdma_put;
+    bool use_rdma_get;
+
+    size_t am_btl_put_limit;
+    size_t am_btl_put_alignment;
+    size_t am_btl_get_limit;
+    size_t am_btl_get_alignment;
+
+    mca_btl_base_am_rdma_module_put_fn_t am_btl_put;
+    mca_btl_base_am_rdma_module_get_fn_t am_btl_get;
+    mca_btl_base_am_rdma_module_atomic_fop64_fn_t am_btl_atomic_fop;
+    mca_btl_base_am_rdma_module_atomic_cswap64_fn_t am_btl_atomic_cswap;
+};
+typedef struct mca_btl_base_am_rdma_module_t mca_btl_base_am_rdma_module_t;
+
+OPAL_DECLSPEC OBJ_CLASS_DECLARATION(mca_btl_base_am_rdma_module_t);
+
+
+/**
+ * @brief create active-message RDMA/atomics functions
+ */
+int opal_btl_base_am_rdma_create(mca_btl_base_module_t *btl,
+                                 uint32_t flags_requested,
+                                 bool no_memory_registration,
+                                 mca_btl_base_am_rdma_module_t **am_module);
+
+int opal_btl_base_am_rdma_destroy(mca_btl_base_am_rdma_module_t *am_module);
 
 #endif /* OPAL_MCA_BTL_BASE_AM_RDMA_H */
diff --git a/opal/mca/btl/base/btl_base_error.c b/opal/mca/btl/base/btl_base_error.c
index 3be2a3a8e04..da7363b8b98 100644
--- a/opal/mca/btl/base/btl_base_error.c
+++ b/opal/mca/btl/base/btl_base_error.c
@@ -21,21 +21,20 @@
  * $HEADER$
  */
 
-
 #include "opal_config.h"
-#include <stdio.h>
 #include <stdarg.h>
+#include <stdio.h>
 
-#include "opal/util/show_help.h"
-#include "opal/util/proc.h"
 #include "opal/util/printf.h"
+#include "opal/util/proc.h"
+#include "opal/util/show_help.h"
 
 #include "base.h"
 #include "btl_base_error.h"
 
 int mca_btl_base_verbose = -1;
 
-int mca_btl_base_err(const char* fmt, ...)
+int mca_btl_base_err(const char *fmt, ...)
 {
     va_list list;
     int ret;
@@ -46,8 +45,7 @@ int mca_btl_base_err(const char* fmt, ...)
     return ret;
 }
 
-
-int mca_btl_base_out(const char* fmt, ...)
+int mca_btl_base_out(const char *fmt, ...)
 {
     va_list list;
     int ret;
@@ -58,26 +56,20 @@ int mca_btl_base_out(const char* fmt, ...)
     return ret;
 }
 
-
-void mca_btl_base_error_no_nics(const char* transport,
-                                const char* nic_name)
+void mca_btl_base_error_no_nics(const char *transport, const char *nic_name)
 {
     char *procid;
     if (mca_btl_base_warn_component_unused) {
         /* print out no-nic warning if user told us to */
         opal_asprintf(&procid, "%s", OPAL_NAME_PRINT(OPAL_PROC_MY_NAME));
 
-        opal_show_help("help-mpi-btl-base.txt", "btl:no-nics",
-                       true, procid, transport, opal_process_info.nodename,
-                       nic_name);
+        opal_show_help("help-mpi-btl-base.txt", "btl:no-nics", true, procid, transport,
+                       opal_process_info.nodename, nic_name);
         free(procid);
     }
 }
 
-
-void mca_btl_base_dump(
-    struct mca_btl_base_module_t* btl,
-    struct mca_btl_base_endpoint_t* endpoint,
-    int verbose)
+void mca_btl_base_dump(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+                       int verbose)
 {
 }
diff --git a/opal/mca/btl/base/btl_base_error.h b/opal/mca/btl/base/btl_base_error.h
index fcebc6000f4..e589092dc53 100644
--- a/opal/mca/btl/base/btl_base_error.h
+++ b/opal/mca/btl/base/btl_base_error.h
@@ -22,84 +22,74 @@
  */
 
 #ifndef MCA_BTL_BASE_ERROR_H
-#define MCA_BTL_BASE_ERROR_H
+#    define MCA_BTL_BASE_ERROR_H
 
-#include "opal_config.h"
+#    include "opal_config.h"
 
-#include <errno.h>
-#include <stdio.h>
+#    include <errno.h>
+#    include <stdio.h>
 
-#include "opal/util/proc.h"
+#    include "opal/util/proc.h"
 
 OPAL_DECLSPEC extern int mca_btl_base_verbose;
 OPAL_DECLSPEC extern int mca_btl_base_warn_peer_error;
 
-OPAL_DECLSPEC extern int mca_btl_base_err(const char*, ...) __opal_attribute_format__(__printf__, 1, 2);
-OPAL_DECLSPEC extern int mca_btl_base_out(const char*, ...) __opal_attribute_format__(__printf__, 1, 2);
-
-#define BTL_OUTPUT(args)                                        \
-    do {                                                        \
-        mca_btl_base_out("[%s]%s[%s:%d:%s] ",                   \
-                         opal_process_info.nodename,            \
-                         OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),    \
-                         __FILE__, __LINE__, __func__);         \
-        mca_btl_base_out args;                                  \
-        mca_btl_base_out("\n");                                 \
-    } while(0);
-
-
-#define BTL_ERROR(args)                                         \
-    do {                                                        \
-        mca_btl_base_err("[%s]%s[%s:%d:%s] ",                   \
-                         opal_process_info.nodename,            \
-                         OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),    \
-                         __FILE__, __LINE__, __func__);         \
-        mca_btl_base_err args;                                  \
-        mca_btl_base_err("\n");                                 \
-    } while(0);
-
-#define BTL_PEER_ERROR(proc, args)                              \
-    do {                                                        \
-        if( mca_btl_base_warn_peer_error                        \
-         || mca_btl_base_verbose > 0 ) { /* warn if verbose */  \
-            char *errhost;                                      \
-            mca_btl_base_err("[%s]%s[%s:%d:%s] ",               \
-                         opal_process_info.nodename,            \
-                         OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),    \
-                         __FILE__, __LINE__, __func__);         \
-            if (proc) {                                         \
-                errhost = opal_get_proc_hostname(proc);         \
-                mca_btl_base_err("peer: %s ", errhost);         \
-                free(errhost);                                  \
-            }                                                   \
-            mca_btl_base_err args;                              \
-            mca_btl_base_err("\n");                             \
-        }                                                       \
-    } while(0);
-
-
-#if OPAL_ENABLE_DEBUG
-#define BTL_VERBOSE(args)                                               \
-    do {                                                                \
-        if(mca_btl_base_verbose > 0) {                                  \
-            mca_btl_base_err("[%s]%s[%s:%d:%s] ",                       \
-                             opal_process_info.nodename,                \
-                             OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),        \
-                             __FILE__, __LINE__, __func__);             \
-            mca_btl_base_err args;                                      \
-            mca_btl_base_err("\n");                                     \
-        }                                                               \
-    } while(0);
-#else
-#define BTL_VERBOSE(args)
-#endif
-
+OPAL_DECLSPEC extern int mca_btl_base_err(const char *, ...)
+    __opal_attribute_format__(__printf__, 1, 2);
+OPAL_DECLSPEC extern int mca_btl_base_out(const char *, ...)
+    __opal_attribute_format__(__printf__, 1, 2);
+
+#    define BTL_OUTPUT(args)                                                                    \
+        do {                                                                                    \
+            mca_btl_base_out("[%s]%s[%s:%d:%s] ", opal_process_info.nodename,                   \
+                             OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), __FILE__, __LINE__, __func__); \
+            mca_btl_base_out args;                                                              \
+            mca_btl_base_out("\n");                                                             \
+        } while (0);
+
+#    define BTL_ERROR(args)                                                                     \
+        do {                                                                                    \
+            mca_btl_base_err("[%s]%s[%s:%d:%s] ", opal_process_info.nodename,                   \
+                             OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), __FILE__, __LINE__, __func__); \
+            mca_btl_base_err args;                                                              \
+            mca_btl_base_err("\n");                                                             \
+        } while (0);
+
+#    define BTL_PEER_ERROR(proc, args)                                                            \
+        do {                                                                                      \
+            if (mca_btl_base_warn_peer_error || mca_btl_base_verbose > 0) { /* warn if verbose */ \
+                char *btl_peer_errhost;                                                           \
+                mca_btl_base_err("[%s]%s[%s:%d:%s] ", opal_process_info.nodename,                 \
+                                 OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), __FILE__, __LINE__,          \
+                                 __func__);                                                       \
+                if (proc) {                                                                       \
+                    btl_peer_errhost= opal_get_proc_hostname(proc);                               \
+                    mca_btl_base_err("peer: %s ", btl_peer_errhost);                              \
+                    free(btl_peer_errhost);                                                       \
+                }                                                                                 \
+                mca_btl_base_err args;                                                            \
+                mca_btl_base_err("\n");                                                           \
+            }                                                                                     \
+        } while (0);
+
+#    if OPAL_ENABLE_DEBUG
+#        define BTL_VERBOSE(args)                                                            \
+            do {                                                                             \
+                if (mca_btl_base_verbose > 0) {                                              \
+                    mca_btl_base_err("[%s]%s[%s:%d:%s] ", opal_process_info.nodename,        \
+                                     OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), __FILE__, __LINE__, \
+                                     __func__);                                              \
+                    mca_btl_base_err args;                                                   \
+                    mca_btl_base_err("\n");                                                  \
+                }                                                                            \
+            } while (0);
+#    else
+#        define BTL_VERBOSE(args)
+#    endif
 #endif
 
-
 BEGIN_C_DECLS
 
-OPAL_DECLSPEC extern void mca_btl_base_error_no_nics(const char* transport,
-                                                     const char* nic_name);
+OPAL_DECLSPEC extern void mca_btl_base_error_no_nics(const char *transport, const char *nic_name);
 
 END_C_DECLS
diff --git a/opal/mca/btl/base/btl_base_frame.c b/opal/mca/btl/base/btl_base_frame.c
index 7be1c5dcbd3..4341d78f6ec 100644
--- a/opal/mca/btl/base/btl_base_frame.c
+++ b/opal/mca/btl/base/btl_base_frame.c
@@ -24,59 +24,56 @@
  * $HEADER$
  */
 
-
 #include "opal_config.h"
 #include <stdio.h>
 
-#include "opal/mca/mca.h"
-#include "opal/util/output.h"
 #include "opal/mca/base/base.h"
 #include "opal/mca/base/mca_base_alias.h"
-#include "opal/mca/btl/btl.h"
 #include "opal/mca/btl/base/base.h"
+#include "opal/mca/btl/btl.h"
+#include "opal/mca/mca.h"
+#include "opal/util/output.h"
 
 mca_base_var_enum_flag_t *mca_btl_base_flag_enum = NULL;
 mca_base_var_enum_flag_t *mca_btl_base_atomic_enum = NULL;
 
-mca_base_var_enum_value_flag_t mca_btl_base_flag_enum_flags[] = {
-    {MCA_BTL_FLAGS_SEND, "send", 0},
-    {MCA_BTL_FLAGS_PUT, "put", MCA_BTL_FLAGS_PUT_AM},
-    {MCA_BTL_FLAGS_GET, "get", MCA_BTL_FLAGS_GET_AM},
-    {MCA_BTL_FLAGS_SEND_INPLACE, "inplace", 0},
-    {MCA_BTL_FLAGS_SIGNALED, "signaled", 0},
-    {MCA_BTL_FLAGS_ATOMIC_OPS, "atomics", MCA_BTL_FLAGS_ATOMIC_AM_FOP},
-    {MCA_BTL_FLAGS_ATOMIC_FOPS, "fetching-atomics", MCA_BTL_FLAGS_ATOMIC_AM_FOP},
-    {MCA_BTL_FLAGS_SINGLE_ADD_PROCS, "static", 0},
-    {MCA_BTL_FLAGS_CUDA_PUT, "cuda-put", 0},
-    {MCA_BTL_FLAGS_CUDA_GET, "cuda-get", 0},
-    {MCA_BTL_FLAGS_CUDA_COPY_ASYNC_SEND, "cuda-async-send", 0},
-    {MCA_BTL_FLAGS_CUDA_COPY_ASYNC_RECV, "cuda-async-recv", 0},
-    {MCA_BTL_FLAGS_FAILOVER_SUPPORT, "failover", 0},
-    {MCA_BTL_FLAGS_NEED_ACK, "need-ack", 0},
-    {MCA_BTL_FLAGS_NEED_CSUM, "need-csum", 0},
-    {MCA_BTL_FLAGS_HETEROGENEOUS_RDMA, "hetero-rdma", 0},
-    {MCA_BTL_FLAGS_RDMA_FLUSH, "rdma-flush", 0},
-    {MCA_BTL_FLAGS_PUT_AM, "put-am", MCA_BTL_FLAGS_PUT},
-    {MCA_BTL_FLAGS_GET_AM, "get_am", MCA_BTL_FLAGS_GET},
-    {MCA_BTL_FLAGS_ATOMIC_AM_FOP, "atomic-am", MCA_BTL_FLAGS_ATOMIC_FOPS},
-    {0, NULL, 0}
-};
-
-mca_base_var_enum_value_flag_t mca_btl_base_atomic_enum_flags[] = {
-  {MCA_BTL_ATOMIC_SUPPORTS_ADD, "add", 0},
-  {MCA_BTL_ATOMIC_SUPPORTS_AND, "and", 0},
-  {MCA_BTL_ATOMIC_SUPPORTS_OR, "or", 0},
-  {MCA_BTL_ATOMIC_SUPPORTS_XOR, "xor", 0},
-  {MCA_BTL_ATOMIC_SUPPORTS_LAND, "land", 0},
-  {MCA_BTL_ATOMIC_SUPPORTS_LOR, "lor", 0},
-  {MCA_BTL_ATOMIC_SUPPORTS_LXOR, "lxor", 0},
-  {MCA_BTL_ATOMIC_SUPPORTS_SWAP, "swap", 0},
-  {MCA_BTL_ATOMIC_SUPPORTS_MIN, "min", 0},
-  {MCA_BTL_ATOMIC_SUPPORTS_MAX, "max", 0},
-  {MCA_BTL_ATOMIC_SUPPORTS_CSWAP, "compare-and-swap", 0},
-  {MCA_BTL_ATOMIC_SUPPORTS_GLOB, "global"},
-  {0, NULL, 0}
-};
+mca_base_var_enum_value_flag_t mca_btl_base_flag_enum_flags[]
+    = {{MCA_BTL_FLAGS_SEND, "send", 0},
+       {MCA_BTL_FLAGS_PUT, "put", MCA_BTL_FLAGS_PUT_AM},
+       {MCA_BTL_FLAGS_GET, "get", MCA_BTL_FLAGS_GET_AM},
+       {MCA_BTL_FLAGS_SEND_INPLACE, "inplace", 0},
+       {MCA_BTL_FLAGS_SIGNALED, "signaled", 0},
+       {MCA_BTL_FLAGS_ATOMIC_OPS, "atomics", MCA_BTL_FLAGS_ATOMIC_AM_FOP},
+       {MCA_BTL_FLAGS_ATOMIC_FOPS, "fetching-atomics", MCA_BTL_FLAGS_ATOMIC_AM_FOP},
+       {MCA_BTL_FLAGS_SINGLE_ADD_PROCS, "static", 0},
+       {MCA_BTL_FLAGS_ACCELERATOR_PUT, "accelerator-put", 0},
+       {MCA_BTL_FLAGS_ACCELERATOR_GET, "accelerator-get", 0},
+       {MCA_BTL_FLAGS_ACCELERATOR_COPY_ASYNC_SEND, "accelerator-async-send", 0},
+       {MCA_BTL_FLAGS_ACCELERATOR_COPY_ASYNC_RECV, "accelerator-async-recv", 0},
+       {MCA_BTL_FLAGS_FAILOVER_SUPPORT, "failover", 0},
+       {MCA_BTL_FLAGS_NEED_ACK, "need-ack", 0},
+       {MCA_BTL_FLAGS_NEED_CSUM, "need-csum", 0},
+       {MCA_BTL_FLAGS_HETEROGENEOUS_RDMA, "hetero-rdma", 0},
+       {MCA_BTL_FLAGS_RDMA_FLUSH, "rdma-flush", 0},
+       {MCA_BTL_FLAGS_PUT_AM, "put-am", MCA_BTL_FLAGS_PUT},
+       {MCA_BTL_FLAGS_GET_AM, "get_am", MCA_BTL_FLAGS_GET},
+       {MCA_BTL_FLAGS_ATOMIC_AM_FOP, "atomic-am", MCA_BTL_FLAGS_ATOMIC_FOPS},
+       {0, NULL, 0}};
+
+mca_base_var_enum_value_flag_t mca_btl_base_atomic_enum_flags[]
+    = {{MCA_BTL_ATOMIC_SUPPORTS_ADD, "add", 0},
+       {MCA_BTL_ATOMIC_SUPPORTS_AND, "and", 0},
+       {MCA_BTL_ATOMIC_SUPPORTS_OR, "or", 0},
+       {MCA_BTL_ATOMIC_SUPPORTS_XOR, "xor", 0},
+       {MCA_BTL_ATOMIC_SUPPORTS_LAND, "land", 0},
+       {MCA_BTL_ATOMIC_SUPPORTS_LOR, "lor", 0},
+       {MCA_BTL_ATOMIC_SUPPORTS_LXOR, "lxor", 0},
+       {MCA_BTL_ATOMIC_SUPPORTS_SWAP, "swap", 0},
+       {MCA_BTL_ATOMIC_SUPPORTS_MIN, "min", 0},
+       {MCA_BTL_ATOMIC_SUPPORTS_MAX, "max", 0},
+       {MCA_BTL_ATOMIC_SUPPORTS_CSWAP, "compare-and-swap", 0},
+       {MCA_BTL_ATOMIC_SUPPORTS_GLOB, "global"},
+       {0, NULL, 0}};
 
 mca_btl_active_message_callback_t mca_btl_base_active_message_trigger[MCA_BTL_TAG_MAX] = {{0}};
 
@@ -84,7 +81,7 @@ mca_btl_active_message_callback_t mca_btl_base_active_message_trigger[MCA_BTL_TA
  *  mca_btl_base_descriptor_t
  */
 
-static void mca_btl_base_descriptor_constructor(mca_btl_base_descriptor_t* des)
+static void mca_btl_base_descriptor_constructor(mca_btl_base_descriptor_t *des)
 {
     des->des_segments = NULL;
     des->des_segment_count = 0;
@@ -93,16 +90,12 @@ static void mca_btl_base_descriptor_constructor(mca_btl_base_descriptor_t* des)
     des->des_flags = 0;
 }
 
-static void mca_btl_base_descriptor_destructor(mca_btl_base_descriptor_t* des)
+static void mca_btl_base_descriptor_destructor(mca_btl_base_descriptor_t *des)
 {
 }
 
-OBJ_CLASS_INSTANCE(
-    mca_btl_base_descriptor_t,
-    opal_list_item_t,
-    mca_btl_base_descriptor_constructor,
-    mca_btl_base_descriptor_destructor);
-
+OBJ_CLASS_INSTANCE(mca_btl_base_descriptor_t, opal_list_item_t, mca_btl_base_descriptor_constructor,
+                   mca_btl_base_descriptor_destructor);
 
 /*
  * The following file was created by configure.  It contains extern
@@ -110,14 +103,14 @@ OBJ_CLASS_INSTANCE(
  * component's public mca_base_component_t struct.
  */
 
-#include "opal/mca/btl/base/static-components.h"
 #include "btl_base_error.h"
+#include "opal/mca/btl/base/static-components.h"
 
 /*
  * Global variables
  */
-char* mca_btl_base_include = NULL;
-char* mca_btl_base_exclude = NULL;
+char *mca_btl_base_include = NULL;
+char *mca_btl_base_exclude = NULL;
 int mca_btl_base_warn_component_unused = 1;
 int mca_btl_base_warn_peer_error = true;
 opal_list_t mca_btl_base_modules_initialized = {{0}};
@@ -128,43 +121,37 @@ static int mca_btl_base_register(mca_base_register_flag_t flags)
     /* Override the per-BTL "don't run if THREAD_MULTIPLE selected"
        embargo? */
     mca_btl_base_thread_multiple_override = false;
-    (void) mca_base_var_register("opal", "btl", "base", "thread_multiple_override",
-                                 "Enable BTLs that are not normally enabled when MPI_THREAD_MULTIPLE is enabled (THIS IS FOR DEVELOPERS ONLY!  SHOULD NOT BE USED BY END USERS!)",
-                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0,
-                                 MCA_BASE_VAR_FLAG_INTERNAL,
-                                 OPAL_INFO_LVL_9,
-                                 MCA_BASE_VAR_SCOPE_READONLY,
-                                 &mca_btl_base_thread_multiple_override);
-
-    (void) mca_base_var_register("opal", "btl", "base", "include", NULL,
-                                 MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
-                                 OPAL_INFO_LVL_9,
-                                 MCA_BASE_VAR_SCOPE_READONLY,
+    (void) mca_base_var_register(
+        "opal", "btl", "base", "thread_multiple_override",
+        "Enable BTLs that are not normally enabled when MPI_THREAD_MULTIPLE is enabled (THIS IS "
+        "FOR DEVELOPERS ONLY!  SHOULD NOT BE USED BY END USERS!)",
+        MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_INTERNAL, OPAL_INFO_LVL_9,
+        MCA_BASE_VAR_SCOPE_READONLY, &mca_btl_base_thread_multiple_override);
+
+    (void) mca_base_var_register("opal", "btl", "base", "include", NULL, MCA_BASE_VAR_TYPE_STRING,
+                                 NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
                                  &mca_btl_base_include);
-    (void) mca_base_var_register("opal", "btl", "base", "exclude", NULL,
-                                 MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
-                                 OPAL_INFO_LVL_9,
-                                 MCA_BASE_VAR_SCOPE_READONLY,
+    (void) mca_base_var_register("opal", "btl", "base", "exclude", NULL, MCA_BASE_VAR_TYPE_STRING,
+                                 NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
                                  &mca_btl_base_exclude);
-    (void) mca_base_var_register("opal", "btl", "base", "warn_peer_error",
-                                 "This parameter is used to turn on warning messages when peers disconnect unexpectedly",
-                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
-                                 OPAL_INFO_LVL_9,
-                                 MCA_BASE_VAR_SCOPE_READONLY,
-                                 &mca_btl_base_warn_peer_error);
-    (void) mca_base_var_register("opal", "btl", "base", "warn_component_unused",
-                                 "This parameter is used to turn on warning messages when certain NICs are not used",
-                                 MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                 OPAL_INFO_LVL_9,
-                                 MCA_BASE_VAR_SCOPE_READONLY,
-                                 &mca_btl_base_warn_component_unused);
-
-    (void) mca_base_var_enum_create_flag ("btl_flags", mca_btl_base_flag_enum_flags, &mca_btl_base_flag_enum);
-    (void) mca_base_var_enum_register("opal", "btl", "base", "btl_flags",
-                                 &mca_btl_base_flag_enum);
-    (void) mca_base_var_enum_create_flag ("btl_atomic_flags", mca_btl_base_atomic_enum_flags, &mca_btl_base_atomic_enum);
+    (void) mca_base_var_register(
+        "opal", "btl", "base", "warn_peer_error",
+        "This parameter is used to turn on warning messages when peers disconnect unexpectedly",
+        MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+        &mca_btl_base_warn_peer_error);
+    (void) mca_base_var_register(
+        "opal", "btl", "base", "warn_component_unused",
+        "This parameter is used to turn on warning messages when certain NICs are not used",
+        MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+        &mca_btl_base_warn_component_unused);
+
+    (void) mca_base_var_enum_create_flag("btl_flags", mca_btl_base_flag_enum_flags,
+                                         &mca_btl_base_flag_enum);
+    (void) mca_base_var_enum_register("opal", "btl", "base", "btl_flags", &mca_btl_base_flag_enum);
+    (void) mca_base_var_enum_create_flag("btl_atomic_flags", mca_btl_base_atomic_enum_flags,
+                                         &mca_btl_base_atomic_enum);
     (void) mca_base_var_enum_register("opal", "btl", "base", "btl_atomic_flags",
-                                 &mca_btl_base_atomic_enum);
+                                      &mca_btl_base_atomic_enum);
 
     /* Note that we break abstraction rules here by listing two
        specific BTLs here in the base.  This is necessary, however,
@@ -187,7 +174,7 @@ static int mca_btl_base_register(mca_base_register_flag_t flags)
 
        This is why we tolerate this abstraction break up here in the
        BTL component base. */
-    (void) mca_base_alias_register ("opal", "btl", "sm", "vader", MCA_BASE_ALIAS_FLAG_NONE);
+    (void) mca_base_alias_register("opal", "btl", "sm", "vader", MCA_BASE_ALIAS_FLAG_NONE);
 
     return OPAL_SUCCESS;
 }
@@ -202,22 +189,22 @@ static int mca_btl_base_open(mca_base_open_flag_t flags)
 
     /* Open up all available components */
 
-    if (OPAL_SUCCESS !=
-        (ret = mca_base_framework_components_open(&opal_btl_base_framework, flags))) {
+    if (OPAL_SUCCESS
+        != (ret = mca_base_framework_components_open(&opal_btl_base_framework, flags))) {
         return ret;
     }
 
-  /* Initialize the list so that in mca_btl_base_close(), we can
-     iterate over it (even if it's empty, as in the case of
-     opal_info) */
+    /* Initialize the list so that in mca_btl_base_close(), we can
+       iterate over it (even if it's empty, as in the case of
+       opal_info) */
 
-  OBJ_CONSTRUCT(&mca_btl_base_modules_initialized, opal_list_t);
+    OBJ_CONSTRUCT(&mca_btl_base_modules_initialized, opal_list_t);
 
-  /* get the verbosity so that BTL_VERBOSE will work */
-  mca_btl_base_verbose = opal_output_get_verbosity(opal_btl_base_framework.framework_output);
+    /* get the verbosity so that BTL_VERBOSE will work */
+    mca_btl_base_verbose = opal_output_get_verbosity(opal_btl_base_framework.framework_output);
 
-  /* All done */
-  return OPAL_SUCCESS;
+    /* All done */
+    return OPAL_SUCCESS;
 }
 
 static int mca_btl_base_close(void)
@@ -230,8 +217,9 @@ static int mca_btl_base_close(void)
 #endif
     /* Finalize all the btl components and free their list items */
 
-    OPAL_LIST_FOREACH_SAFE(sm, next, &mca_btl_base_modules_initialized, mca_btl_base_selected_module_t) {
-        /* Blatently ignore the return code (what would we do to recover,
+    OPAL_LIST_FOREACH_SAFE (sm, next, &mca_btl_base_modules_initialized,
+                            mca_btl_base_selected_module_t) {
+        /* Blatantly ignore the return code (what would we do to recover,
            anyway?  This component is going away, so errors don't matter
            anymore) */
 
diff --git a/opal/mca/btl/base/btl_base_mca.c b/opal/mca/btl/base/btl_base_mca.c
index c65c0a3b5c2..a5b29cf02dd 100644
--- a/opal/mca/btl/base/btl_base_mca.c
+++ b/opal/mca/btl/base/btl_base_mca.c
@@ -28,138 +28,158 @@
 
 #include <stdio.h>
 
-#include "opal/util/output.h"
 #include "opal/constants.h"
-#include "opal/mca/btl/btl.h"
 #include "opal/mca/btl/base/base.h"
+#include "opal/mca/btl/btl.h"
+#include "opal/util/output.h"
 
-int mca_btl_base_param_register(mca_base_component_t *version,
-                                mca_btl_base_module_t *module)
+int mca_btl_base_param_register(mca_base_component_t *version, mca_btl_base_module_t *module)
 {
     /* If this is ever triggered change the uint32_ts in mca_btl_base_module_t to unsigned ints */
     assert(sizeof(unsigned int) == sizeof(uint32_t));
 
-    (void) mca_base_component_var_register(version, "exclusivity",
-                                           "BTL exclusivity (must be >= 0)",
+    (void) mca_base_component_var_register(version, "exclusivity", "BTL exclusivity (must be >= 0)",
                                            MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_7,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           OPAL_INFO_LVL_7, MCA_BASE_VAR_SCOPE_READONLY,
                                            &module->btl_exclusivity);
 
-    (void) mca_base_component_var_register(version, "flags", "BTL bit flags (general flags: send, put, get, in-place, hetero-rdma, "
-                                           "atomics, fetching-atomics)", MCA_BASE_VAR_TYPE_UNSIGNED_INT,
-                                           &mca_btl_base_flag_enum->super, 0, 0, OPAL_INFO_LVL_5,
-                                           MCA_BASE_VAR_SCOPE_READONLY, &module->btl_flags);
-
-    (void) mca_base_component_var_register(version, "atomic_flags", "BTL atomic support flags", MCA_BASE_VAR_TYPE_UNSIGNED_INT,
-                                           &mca_btl_base_atomic_enum->super, 0, MCA_BASE_VAR_FLAG_DEFAULT_ONLY, OPAL_INFO_LVL_5,
+    (void) mca_base_component_var_register(
+        version, "flags",
+        "BTL bit flags (general flags: send, put, get, in-place, hetero-rdma, "
+        "atomics, fetching-atomics)",
+        MCA_BASE_VAR_TYPE_UNSIGNED_INT, &mca_btl_base_flag_enum->super, 0, 0, OPAL_INFO_LVL_5,
+        MCA_BASE_VAR_SCOPE_READONLY, &module->btl_flags);
+
+    (void) mca_base_component_var_register(version, "atomic_flags", "BTL atomic support flags",
+                                           MCA_BASE_VAR_TYPE_UNSIGNED_INT,
+                                           &mca_btl_base_atomic_enum->super, 0,
+                                           MCA_BASE_VAR_FLAG_DEFAULT_ONLY, OPAL_INFO_LVL_5,
                                            MCA_BASE_VAR_SCOPE_CONSTANT, &module->btl_atomic_flags);
 
-    (void) mca_base_component_var_register(version, "rndv_eager_limit", "Size (in bytes, including header) of \"phase 1\" fragment sent for all large messages (must be >= 0 and <= eager_limit)",
-                                           MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0,
-                                           OPAL_INFO_LVL_4,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
-                                           &module->btl_rndv_eager_limit);
+    (void) mca_base_component_var_register(
+        version, "rndv_eager_limit",
+        "Size (in bytes, including header) of \"phase 1\" fragment sent for all large messages "
+        "(must be >= 0 and <= eager_limit)",
+        MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_4, MCA_BASE_VAR_SCOPE_READONLY,
+        &module->btl_rndv_eager_limit);
 
-    (void) mca_base_component_var_register(version, "eager_limit", "Maximum size (in bytes, including header) of \"short\" messages (must be >= 1).",
-                                           MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0,
-                                           OPAL_INFO_LVL_4,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
-                                           &module->btl_eager_limit);
+    (void) mca_base_component_var_register(
+        version, "eager_limit",
+        "Maximum size (in bytes, including header) of \"short\" messages (must be >= 1).",
+        MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_4, MCA_BASE_VAR_SCOPE_READONLY,
+        &module->btl_eager_limit);
 
     if ((module->btl_flags & MCA_BTL_FLAGS_GET) && module->btl_get) {
-	if (0 == module->btl_get_limit) {
-	    module->btl_get_limit = SIZE_MAX;
-	}
-
-	(void) mca_base_component_var_register(version, "get_limit", "Maximum size (in bytes) for btl get",
-					       MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_4,
-					       MCA_BASE_VAR_SCOPE_READONLY, &module->btl_get_limit);
-
-	/* Allow the user to set the alignment. The BTL should double-check the alignment in its open
-	 * function. */
-	(void) mca_base_component_var_register(version, "get_alignment", "Alignment required for btl get",
-					       MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_6,
-					       MCA_BASE_VAR_SCOPE_CONSTANT, &module->btl_get_alignment);
+        if (0 == module->btl_get_limit) {
+            module->btl_get_limit = SIZE_MAX;
+        }
+
+        (void) mca_base_component_var_register(version, "get_limit",
+                                               "Maximum size (in bytes) for btl get",
+                                               MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0,
+                                               OPAL_INFO_LVL_4, MCA_BASE_VAR_SCOPE_READONLY,
+                                               &module->btl_get_limit);
+
+        /* Allow the user to set the alignment. The BTL should double-check the alignment in its
+         * open function. */
+        (void) mca_base_component_var_register(version, "get_alignment",
+                                               "Alignment required for btl get",
+                                               MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0,
+                                               OPAL_INFO_LVL_6, MCA_BASE_VAR_SCOPE_CONSTANT,
+                                               &module->btl_get_alignment);
     }
 
     if ((module->btl_flags & MCA_BTL_FLAGS_PUT) && module->btl_put) {
-	if (0 == module->btl_put_limit) {
-	    module->btl_put_limit = SIZE_MAX;
-	}
-	(void) mca_base_component_var_register(version, "put_limit", "Maximum size (in bytes) for btl put",
-					       MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_4,
-					       MCA_BASE_VAR_SCOPE_READONLY, &module->btl_put_limit);
-
-	/* Allow the user to set the alignment. The BTL should double-check the alignment in its open
-	 * function. */
-	(void) mca_base_component_var_register(version, "put_alignment", "Alignment required for btl put",
-					       MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_6,
-					       MCA_BASE_VAR_SCOPE_CONSTANT, &module->btl_put_alignment);
+        if (0 == module->btl_put_limit) {
+            module->btl_put_limit = SIZE_MAX;
+        }
+        (void) mca_base_component_var_register(version, "put_limit",
+                                               "Maximum size (in bytes) for btl put",
+                                               MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0,
+                                               OPAL_INFO_LVL_4, MCA_BASE_VAR_SCOPE_READONLY,
+                                               &module->btl_put_limit);
+
+        /* Allow the user to set the alignment. The BTL should double-check the alignment in its
+         * open function. */
+        (void) mca_base_component_var_register(version, "put_alignment",
+                                               "Alignment required for btl put",
+                                               MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0,
+                                               OPAL_INFO_LVL_6, MCA_BASE_VAR_SCOPE_CONSTANT,
+                                               &module->btl_put_alignment);
     }
 
-
 #if OPAL_CUDA_GDR_SUPPORT
     /* If no CUDA RDMA support, zero them out */
-    if (!(MCA_BTL_FLAGS_CUDA_GET & module->btl_flags)) {
-        module->btl_cuda_eager_limit = 0;
-        module->btl_cuda_rdma_limit = SIZE_MAX;
+    if (!(MCA_BTL_FLAGS_ACCELERATOR_GET & module->btl_flags)) {
+        module->btl_accelerator_eager_limit = 0;
+        module->btl_accelerator_rdma_limit = SIZE_MAX;
     }
-    (void) mca_base_component_var_register(version, "cuda_eager_limit", "Maximum size (in bytes, including header) of \"GPU short\" messages (must be >= 1).",
-                                           MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0,
-                                           OPAL_INFO_LVL_5,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
-                                           &module->btl_cuda_eager_limit);
-    (void) mca_base_component_var_register(version, "cuda_rdma_limit", "Size (in bytes, including header) of GPU buffer when switch to rndv protocol and pipeline.",
-                                           MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0,
-                                           OPAL_INFO_LVL_5,
+    (void) mca_base_component_var_register(
+        version, "accelerator_eager_limit",
+        "Maximum size (in bytes, including header) of \"GPU short\" messages (must be >= 1).",
+        MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_READONLY,
+        &module->btl_accelerator_eager_limit);
+    (void) mca_base_component_var_register(version, "accelerator_rdma_limit",
+                                           "Size (in bytes, including header) of GPU buffer when "
+                                           "switch to rndv protocol and pipeline.",
+                                           MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_5,
                                            MCA_BASE_VAR_SCOPE_READONLY,
-                                           &module->btl_cuda_rdma_limit);
+                                           &module->btl_accelerator_rdma_limit);
 #endif /* OPAL_CUDA_GDR_SUPPORT */
-#if OPAL_CUDA_SUPPORT
-    module->btl_cuda_max_send_size = 0;
-    (void) mca_base_component_var_register(version, "cuda_max_send_size", "Maximum size (in bytes) of a single GPU \"phase 2\" fragment of a long message when using the pipeline protocol (must be >= 1) (only valid on smcuda btl)",
-                                           MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0,
-                                           OPAL_INFO_LVL_4,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
-                                           &module->btl_cuda_max_send_size);
-#endif /* OPAL_CUDA_SUPPORT */
-
-    (void) mca_base_component_var_register(version, "max_send_size", "Maximum size (in bytes) of a single \"phase 2\" fragment of a long message when using the pipeline protocol (must be >= 1)",
-                                           MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0,
-                                           OPAL_INFO_LVL_4,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
-                                           &module->btl_max_send_size);
+    module->btl_accelerator_max_send_size = 0;
+    (void) mca_base_component_var_register(
+        version, "accelerator_max_send_size",
+        "Maximum size (in bytes) of a single GPU \"phase 2\" fragment of a long message when using "
+        "the pipeline protocol (must be >= 1) (only valid on smcuda btl)",
+        MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_4, MCA_BASE_VAR_SCOPE_READONLY,
+        &module->btl_accelerator_max_send_size);
+
+    (void) mca_base_component_var_register(
+        version, "max_send_size",
+        "Maximum size (in bytes) of a single \"phase 2\" fragment of a long message when using the "
+        "pipeline protocol (must be >= 1)",
+        MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_4, MCA_BASE_VAR_SCOPE_READONLY,
+        &module->btl_max_send_size);
 
     if (NULL != module->btl_put) {
-      (void) mca_base_component_var_register(version, "rdma_pipeline_send_length", "Length of the \"phase 2\" portion of a large message (in bytes) when using the pipeline protocol.  This part of the message will be split into fragments of size max_send_size and sent using send/receive semantics (must be >= 0; only relevant when the PUT flag is set)",
-                                             MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0,
-                                             OPAL_INFO_LVL_4,
-                                             MCA_BASE_VAR_SCOPE_READONLY,
-                                             &module->btl_rdma_pipeline_send_length);
-
-      (void) mca_base_component_var_register(version, "rdma_pipeline_frag_size", "Maximum size (in bytes) of a single \"phase 3\" fragment from a long message when using the pipeline protocol.  These fragments will be sent using RDMA semantics (must be >= 1; only relevant when the PUT flag is set)",
-                                             MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0,
-                                             OPAL_INFO_LVL_4,
-                                             MCA_BASE_VAR_SCOPE_READONLY,
-                                             &module->btl_rdma_pipeline_frag_size);
-
-      (void) mca_base_component_var_register(version, "min_rdma_pipeline_size", "Messages smaller than this size (in bytes) will not use the RDMA pipeline protocol.  Instead, they will be split into fragments of max_send_size and sent using send/receive semantics (must be >=0, and is automatically adjusted up to at least (eager_limit+btl_rdma_pipeline_send_length); only relevant when the PUT flag is set)",
-                                             MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0,
-                                             OPAL_INFO_LVL_4,
-                                             MCA_BASE_VAR_SCOPE_READONLY,
-                                             &module->btl_min_rdma_pipeline_size);
-
-      (void) mca_base_component_var_register(version, "latency", "Approximate latency of interconnect (0 = auto-detect value at run-time [not supported in all BTL modules], >= 1 = latency in microseconds)",
-                                             MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, 0,
-                                             OPAL_INFO_LVL_5,
-                                             MCA_BASE_VAR_SCOPE_READONLY,
-                                             &module->btl_latency);
-      (void) mca_base_component_var_register(version, "bandwidth", "Approximate maximum bandwidth of interconnect (0 = auto-detect value at run-time [not supported in all BTL modules], >= 1 = bandwidth in Mbps)",
-                                             MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, 0,
-                                             OPAL_INFO_LVL_5,
-                                             MCA_BASE_VAR_SCOPE_READONLY,
-                                             &module->btl_bandwidth);
+        (void) mca_base_component_var_register(
+            version, "rdma_pipeline_send_length",
+            "Length of the \"phase 2\" portion of a large message (in bytes) when using the "
+            "pipeline protocol.  This part of the message will be split into fragments of size "
+            "max_send_size and sent using send/receive semantics (must be >= 0; only relevant when "
+            "the PUT flag is set)",
+            MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_4, MCA_BASE_VAR_SCOPE_READONLY,
+            &module->btl_rdma_pipeline_send_length);
+
+        (void) mca_base_component_var_register(
+            version, "rdma_pipeline_frag_size",
+            "Maximum size (in bytes) of a single \"phase 3\" fragment from a long message when "
+            "using the pipeline protocol.  These fragments will be sent using RDMA semantics (must "
+            "be >= 1; only relevant when the PUT flag is set)",
+            MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_4, MCA_BASE_VAR_SCOPE_READONLY,
+            &module->btl_rdma_pipeline_frag_size);
+
+        (void) mca_base_component_var_register(
+            version, "min_rdma_pipeline_size",
+            "Messages smaller than this size (in bytes) will not use the RDMA pipeline protocol.  "
+            "Instead, they will be split into fragments of max_send_size and sent using "
+            "send/receive semantics (must be >=0, and is automatically adjusted up to at least "
+            "(eager_limit+btl_rdma_pipeline_send_length); only relevant when the PUT flag is set)",
+            MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_4, MCA_BASE_VAR_SCOPE_READONLY,
+            &module->btl_min_rdma_pipeline_size);
+
+        (void) mca_base_component_var_register(
+            version, "latency",
+            "Approximate latency of interconnect (0 = auto-detect value at run-time [not supported "
+            "in all BTL modules], >= 1 = latency in microseconds)",
+            MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
+            MCA_BASE_VAR_SCOPE_READONLY, &module->btl_latency);
+        (void) mca_base_component_var_register(
+            version, "bandwidth",
+            "Approximate maximum bandwidth of interconnect (0 = auto-detect value at run-time [not "
+            "supported in all BTL modules], >= 1 = bandwidth in Mbps)",
+            MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
+            MCA_BASE_VAR_SCOPE_READONLY, &module->btl_bandwidth);
     }
 
     return mca_btl_base_param_verify(module);
@@ -168,10 +188,10 @@ int mca_btl_base_param_register(mca_base_component_t *version,
 /* Verify btl parameters make sense */
 int mca_btl_base_param_verify(mca_btl_base_module_t *module)
 {
-    if (module->btl_min_rdma_pipeline_size <
-        (module->btl_eager_limit + module->btl_rdma_pipeline_send_length)) {
-        module->btl_min_rdma_pipeline_size =
-            module->btl_eager_limit + module->btl_rdma_pipeline_send_length;
+    if (module->btl_min_rdma_pipeline_size
+        < (module->btl_eager_limit + module->btl_rdma_pipeline_send_length)) {
+        module->btl_min_rdma_pipeline_size = module->btl_eager_limit
+                                             + module->btl_rdma_pipeline_send_length;
     }
 
     if (NULL == module->btl_put) {
@@ -191,11 +211,11 @@ int mca_btl_base_param_verify(mca_btl_base_module_t *module)
     }
 
     if (0 == module->btl_get_limit) {
-	module->btl_get_limit = SIZE_MAX;
+        module->btl_get_limit = SIZE_MAX;
     }
 
     if (0 == module->btl_put_limit) {
-	module->btl_put_limit = SIZE_MAX;
+        module->btl_put_limit = SIZE_MAX;
     }
 
     return OPAL_SUCCESS;
diff --git a/opal/mca/btl/base/btl_base_select.c b/opal/mca/btl/base/btl_base_select.c
index 642901fecdb..028655f0011 100644
--- a/opal/mca/btl/base/btl_base_select.c
+++ b/opal/mca/btl/base/btl_base_select.c
@@ -25,21 +25,18 @@
 
 #include "opal_config.h"
 
-#include "opal/util/argv.h"
-#include "opal/util/output.h"
-#include "opal/util/show_help.h"
-#include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
 #include "opal/mca/base/mca_base_component_repository.h"
-#include "opal/runtime/opal.h"
-#include "opal/mca/btl/btl.h"
-#include "opal/mca/btl/base/btl_base_error.h"
 #include "opal/mca/btl/base/base.h"
+#include "opal/mca/btl/base/btl_base_error.h"
+#include "opal/mca/btl/btl.h"
+#include "opal/mca/mca.h"
+#include "opal/runtime/opal.h"
+#include "opal/util/argv.h"
+#include "opal/util/output.h"
+#include "opal/util/show_help.h"
 
-OBJ_CLASS_INSTANCE( mca_btl_base_selected_module_t,
-                    opal_list_item_t,
-                    NULL,
-                    NULL );
+OBJ_CLASS_INSTANCE(mca_btl_base_selected_module_t, opal_list_item_t, NULL, NULL);
 
 /**
  * Function for weeding out btl components that don't want to run.
@@ -49,8 +46,7 @@ OBJ_CLASS_INSTANCE( mca_btl_base_selected_module_t,
  * components will be closed and unloaded.  The selected modules will
  * be returned to the caller in a opal_list_t.
  */
-int mca_btl_base_select(bool enable_progress_threads,
-                        bool enable_mpi_threads)
+int mca_btl_base_select(bool enable_progress_threads, bool enable_mpi_threads)
 {
     int i, num_btls;
     mca_base_component_list_item_t *cli, *next;
@@ -58,42 +54,44 @@ int mca_btl_base_select(bool enable_progress_threads,
     mca_btl_base_module_t **modules;
     mca_btl_base_selected_module_t *sm;
 
-    char** include = opal_argv_split(mca_btl_base_include, ',');
-    char** exclude = opal_argv_split(mca_btl_base_exclude, ',');
+    char **include = opal_argv_split(mca_btl_base_include, ',');
+    char **exclude = opal_argv_split(mca_btl_base_exclude, ',');
 
     /* Traverse the list of opened modules; call their init
        functions. */
 
-    OPAL_LIST_FOREACH_SAFE(cli, next, &opal_btl_base_framework.framework_components, mca_base_component_list_item_t) {
+    OPAL_LIST_FOREACH_SAFE (cli, next, &opal_btl_base_framework.framework_components,
+                            mca_base_component_list_item_t) {
         component = (mca_btl_base_component_t *) cli->cli_component;
 
         /* if there is an include list - item must be in the list to be included */
-        if ( NULL != include ) {
-            char** argv = include;
+        if (NULL != include) {
+            char **argv = include;
             bool found = false;
-            while(argv && *argv) {
-                if(strcmp(component->btl_version.mca_component_name,*argv) == 0) {
+            while (argv && *argv) {
+                if (strcmp(component->btl_version.mca_component_name, *argv) == 0) {
                     found = true;
                     break;
                 }
                 argv++;
             }
-            if(found == false) {
+            if (found == false) {
                 continue;
             }
 
-            /* otherwise - check the exclude list to see if this item has been specifically excluded */
-        } else if ( NULL != exclude ) {
-            char** argv = exclude;
+            /* otherwise - check the exclude list to see if this item has been specifically excluded
+             */
+        } else if (NULL != exclude) {
+            char **argv = exclude;
             bool found = false;
-            while(argv && *argv) {
-                if(strcmp(component->btl_version.mca_component_name,*argv) == 0) {
+            while (argv && *argv) {
+                if (strcmp(component->btl_version.mca_component_name, *argv) == 0) {
                     found = true;
                     break;
                 }
                 argv++;
             }
-            if(found == true) {
+            if (found == true) {
                 continue;
             }
         }
@@ -107,8 +105,7 @@ int mca_btl_base_select(bool enable_progress_threads,
                                 "select: no init function; ignoring component %s",
                                 component->btl_version.mca_component_name);
         } else {
-            modules = component->btl_init(&num_btls, enable_progress_threads,
-                                          enable_mpi_threads);
+            modules = component->btl_init(&num_btls, enable_progress_threads, enable_mpi_threads);
 
             /* If the component didn't initialize, remove it from the opened
                list and remove it from the component repository */
@@ -134,8 +131,10 @@ int mca_btl_base_select(bool enable_progress_threads,
                 for (i = 0; i < num_btls; ++i) {
                     /* If modules[i] is NULL, it's a developer error */
                     if (NULL == modules[i]) {
-                        BTL_ERROR(("BTL module init of %s returned a NULL -- this should never happen, and is a developer error.  Contact the Open MPI developers.",
-                                   component->btl_version.mca_component_name));
+                        BTL_ERROR(
+                            ("BTL module init of %s returned a NULL -- this should never happen, "
+                             "and is a developer error.  Contact the Open MPI developers.",
+                             component->btl_version.mca_component_name));
                         exit(1);
                     }
                     sm = OBJ_NEW(mca_btl_base_selected_module_t);
@@ -150,8 +149,7 @@ int mca_btl_base_select(bool enable_progress_threads,
                     }
                     sm->btl_component = component;
                     sm->btl_module = modules[i];
-                    opal_list_append(&mca_btl_base_modules_initialized,
-                                     (opal_list_item_t*) sm);
+                    opal_list_append(&mca_btl_base_modules_initialized, (opal_list_item_t *) sm);
                 }
                 free(modules);
             }
@@ -168,11 +166,8 @@ int mca_btl_base_select(bool enable_progress_threads,
     /* Finished querying all components.  Check for the bozo case. */
 
     if (0 == opal_list_get_size(&mca_btl_base_modules_initialized)) {
-        opal_show_help("help-mca-base.txt", "find-available:none found",
-                       true,
-                       "btl",
-                       opal_process_info.nodename,
-                       "btl");
+        opal_show_help("help-mca-base.txt", "find-available:none found", true, "btl",
+                       opal_process_info.nodename, "btl");
         return OPAL_ERROR;
     }
     return OPAL_SUCCESS;
diff --git a/opal/mca/btl/btl.h b/opal/mca/btl/btl.h
index d86ab6b10f1..c66ca1ad909 100644
--- a/opal/mca/btl/btl.h
+++ b/opal/mca/btl/btl.h
@@ -19,6 +19,8 @@
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2020      Intel, Inc.  All rights reserved.
  * Copyright (c) 2020-2021 Google, LLC. All rights reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -42,7 +44,7 @@
  * transport are not available.
  *
  * The mca_btl_base_component_init_fn_t() is then called for each of the
- * components that are succesfully opened. The component init function may
+ * components that are successfully opened. The component init function may
  * return either:
  *
  * (1) a NULL list of BTL modules if the transport is not available,
@@ -118,13 +120,13 @@
 #define OPAL_MCA_BTL_H
 
 #include "opal_config.h"
-#include "opal/types.h"
-#include "opal/prefetch.h" /* For OPAL_LIKELY */
 #include "opal/class/opal_bitmap.h"
 #include "opal/datatype/opal_convertor.h"
 #include "opal/mca/mca.h"
 #include "opal/mca/mpool/mpool.h"
 #include "opal/mca/rcache/rcache.h"
+#include "opal/prefetch.h" /* For OPAL_LIKELY */
+#include "opal/types.h"
 
 BEGIN_C_DECLS
 
@@ -152,14 +154,13 @@ struct opal_proc_t;
 struct mca_btl_base_registration_handle_t;
 typedef struct mca_btl_base_registration_handle_t mca_btl_base_registration_handle_t;
 
-
 /* Wildcard endpoint for use in the register_mem function */
 #define MCA_BTL_ENDPOINT_ANY (struct mca_btl_base_endpoint_t *) -1
 
 /* send/recv operations require tag matching */
 typedef uint8_t mca_btl_base_tag_t;
 
-#define MCA_BTL_NO_ORDER       255
+#define MCA_BTL_NO_ORDER 255
 
 /*
  * Communication specific defines. There are a number of active message ID
@@ -175,60 +176,60 @@ typedef uint8_t mca_btl_base_tag_t;
  * information about how these framework ID are defined, take a look in the
  * header file associated with the framework.
  */
-#define MCA_BTL_AM_FRAMEWORK_MASK   0xD0
-#define MCA_BTL_TAG_BTL_BASE        0x10
-#define MCA_BTL_TAG_BTL             0x20
+#define MCA_BTL_AM_FRAMEWORK_MASK 0xD0
+#define MCA_BTL_TAG_BTL_BASE      0x10
+#define MCA_BTL_TAG_BTL           0x20
 #if OPAL_ENABLE_FT_MPI
-#define MCA_BTL_TAG_FT_RBCAST       0x30
-#define MCA_BTL_TAG_FT_AGREE        0x31
+#    define MCA_BTL_TAG_FT_RBCAST 0x30
+#    define MCA_BTL_TAG_FT_AGREE  0x31
 #endif
-#define MCA_BTL_TAG_PML             0x40
-#define MCA_BTL_TAG_OSC_RDMA        0x60
-#define MCA_BTL_TAG_USR             0x80
-#define MCA_BTL_TAG_MAX             255 /* 1 + highest allowed tag num */
+#define MCA_BTL_TAG_PML      0x40
+#define MCA_BTL_TAG_OSC_RDMA 0x60
+#define MCA_BTL_TAG_USR      0x80
+#define MCA_BTL_TAG_MAX      255 /* 1 + highest allowed tag num */
 
 /*
  * Reserved tags for specific BTLs. As multiple BTLs can be active
  * simultaneously, their tags should not collide.
  */
-#define MCA_BTL_TAG_IB                (MCA_BTL_TAG_BTL + 0)
-#define MCA_BTL_TAG_UDAPL             (MCA_BTL_TAG_BTL + 1)
-#define MCA_BTL_TAG_SMCUDA            (MCA_BTL_TAG_BTL + 2)
-#define MCA_BTL_TAG_SM                (MCA_BTL_TAG_BTL + 3)
-
-/* prefered protocol */
-#define MCA_BTL_FLAGS_SEND            0x0001
-#define MCA_BTL_FLAGS_PUT             0x0002
-#define MCA_BTL_FLAGS_GET             0x0004
+#define MCA_BTL_TAG_IB     (MCA_BTL_TAG_BTL + 0)
+#define MCA_BTL_TAG_UDAPL  (MCA_BTL_TAG_BTL + 1)
+#define MCA_BTL_TAG_SMCUDA (MCA_BTL_TAG_BTL + 2)
+#define MCA_BTL_TAG_SM     (MCA_BTL_TAG_BTL + 3)
+
+/* preferred protocol */
+#define MCA_BTL_FLAGS_SEND 0x0001
+#define MCA_BTL_FLAGS_PUT  0x0002
+#define MCA_BTL_FLAGS_GET  0x0004
 /* btls that set the MCA_BTL_FLAGS_RDMA will always get added to the BML
  * rdma_btls list. This allows the updated one-sided component to
  * use btls that are not otherwise used for send/recv. */
-#define MCA_BTL_FLAGS_RDMA (MCA_BTL_FLAGS_GET|MCA_BTL_FLAGS_PUT)
+#define MCA_BTL_FLAGS_RDMA (MCA_BTL_FLAGS_GET | MCA_BTL_FLAGS_PUT)
 
 /* btl can send directly from user buffer w/out registration */
-#define MCA_BTL_FLAGS_SEND_INPLACE    0x0008
+#define MCA_BTL_FLAGS_SEND_INPLACE 0x0008
 
 /* btl transport reliability flags - currently used only by the DR PML */
-#define MCA_BTL_FLAGS_NEED_ACK        0x0010
-#define MCA_BTL_FLAGS_NEED_CSUM       0x0020
+#define MCA_BTL_FLAGS_NEED_ACK  0x0010
+#define MCA_BTL_FLAGS_NEED_CSUM 0x0020
 
 /** deprecated (BTL 3.0) */
-#define MCA_BTL_FLAGS_RDMA_MATCHED    0x0040
+#define MCA_BTL_FLAGS_RDMA_MATCHED 0x0040
 
 /* btl needs local rdma completion */
 #define MCA_BTL_FLAGS_RDMA_COMPLETION 0x0080
 
- /* btl can do heterogeneous rdma operations on byte buffers */
+/* btl can do heterogeneous rdma operations on byte buffers */
 #define MCA_BTL_FLAGS_HETEROGENEOUS_RDMA 0x0100
 
 /* btl can support failover if enabled */
 #define MCA_BTL_FLAGS_FAILOVER_SUPPORT 0x0200
 
-#define MCA_BTL_FLAGS_CUDA_PUT        0x0400
-#define MCA_BTL_FLAGS_CUDA_GET        0x0800
-#define MCA_BTL_FLAGS_CUDA_RDMA (MCA_BTL_FLAGS_CUDA_GET|MCA_BTL_FLAGS_CUDA_PUT)
-#define MCA_BTL_FLAGS_CUDA_COPY_ASYNC_SEND 0x1000
-#define MCA_BTL_FLAGS_CUDA_COPY_ASYNC_RECV 0x2000
+#define MCA_BTL_FLAGS_ACCELERATOR_PUT             0x0400
+#define MCA_BTL_FLAGS_ACCELERATOR_GET             0x0800
+#define MCA_BTL_FLAGS_ACCELERATOR_RDMA            (MCA_BTL_FLAGS_ACCELERATOR_GET | MCA_BTL_FLAGS_ACCELERATOR_PUT)
+#define MCA_BTL_FLAGS_ACCELERATOR_COPY_ASYNC_SEND 0x1000
+#define MCA_BTL_FLAGS_ACCELERATOR_COPY_ASYNC_RECV 0x2000
 
 /* btl can support signaled operations. BTLs that support this flag are
  * expected to provide a mechanism for asynchronous progress on descriptors
@@ -236,12 +237,12 @@ typedef uint8_t mca_btl_base_tag_t;
  * (and probably will) turn this flag on and off using the MCA variable
  * system.
  */
-#define MCA_BTL_FLAGS_SIGNALED        0x4000
+#define MCA_BTL_FLAGS_SIGNALED 0x4000
 
 /** The BTL supports network atomic operations */
-#define MCA_BTL_FLAGS_ATOMIC_OPS      0x08000
+#define MCA_BTL_FLAGS_ATOMIC_OPS 0x08000
 /** The BTL supports fetching network atomic operations */
-#define MCA_BTL_FLAGS_ATOMIC_FOPS     0x10000
+#define MCA_BTL_FLAGS_ATOMIC_FOPS 0x10000
 
 /** The BTL requires add_procs to be with all procs including non-local. Shared-memory
  * BTLs should not set this flag. */
@@ -251,26 +252,36 @@ typedef uint8_t mca_btl_base_tag_t;
 #define MCA_BTL_FLAGS_BTL_PROGRESS_THREAD_ENABLED 0x40000
 
 /* The BTL supports RMDA flush */
-#define MCA_BTL_FLAGS_RDMA_FLUSH      0x80000
+#define MCA_BTL_FLAGS_RDMA_FLUSH 0x80000
 
 /* The BTL has an active-message based put */
-#define MCA_BTL_FLAGS_PUT_AM          0x100000
+#define MCA_BTL_FLAGS_PUT_AM 0x100000
 
 /* The BTL has an active-message based get */
-#define MCA_BTL_FLAGS_GET_AM          0x200000
+#define MCA_BTL_FLAGS_GET_AM 0x200000
 
 /* The BTL has active-message based atomics */
-#define MCA_BTL_FLAGS_ATOMIC_AM_FOP   0x400000
+#define MCA_BTL_FLAGS_ATOMIC_AM_FOP 0x400000
+
+/** The BTL's RDMA/atomics operation supports remote completion.
+ * When the BTL reported the completion of a RDMA/atomic operation
+ * on the initiator side, the operation also finished on the target side.
+ *
+ * Note, this flag is for put and atomic write operations. Operations
+ * like get, atomic fetch and atomic swap support remote
+ * completion by nature.
+ */
+#define MCA_BTL_FLAGS_RDMA_REMOTE_COMPLETION 0x800000
 
 /* Default exclusivity levels */
-#define MCA_BTL_EXCLUSIVITY_HIGH     (64*1024) /* internal loopback */
-#define MCA_BTL_EXCLUSIVITY_DEFAULT  1024      /* GM/IB/etc. */
-#define MCA_BTL_EXCLUSIVITY_LOW      0         /* TCP used as a last resort */
+#define MCA_BTL_EXCLUSIVITY_HIGH    (64 * 1024) /* internal loopback */
+#define MCA_BTL_EXCLUSIVITY_DEFAULT 1024 /* GM/IB/etc. */
+#define MCA_BTL_EXCLUSIVITY_LOW     0 /* TCP used as a last resort */
 
 /* error callback flags */
-#define MCA_BTL_ERROR_FLAGS_FATAL 0x1
-#define MCA_BTL_ERROR_FLAGS_NONFATAL 0x2
-#define MCA_BTL_ERROR_FLAGS_ADD_CUDA_IPC 0x4
+#define MCA_BTL_ERROR_FLAGS_FATAL        0x1
+#define MCA_BTL_ERROR_FLAGS_NONFATAL     0x2
+#define MCA_BTL_ERROR_FLAGS_ADD_ACCELERATOR_IPC 0x4
 
 /** registration flags. the access flags are a 1-1 mapping with the mpool
  * access flags. */
@@ -278,15 +289,15 @@ enum {
     /** Allow local write on the registered region. If a region is registered
      * with this flag the registration can be used as the local handle for a
      * btl_get operation. */
-    MCA_BTL_REG_FLAG_LOCAL_WRITE   = MCA_RCACHE_ACCESS_LOCAL_WRITE,
+    MCA_BTL_REG_FLAG_LOCAL_WRITE = MCA_RCACHE_ACCESS_LOCAL_WRITE,
     /** Allow remote read on the registered region. If a region is registered
      * with this flag the registration can be used as the remote handle for a
      * btl_get operation. */
-    MCA_BTL_REG_FLAG_REMOTE_READ   = MCA_RCACHE_ACCESS_REMOTE_READ,
+    MCA_BTL_REG_FLAG_REMOTE_READ = MCA_RCACHE_ACCESS_REMOTE_READ,
     /** Allow remote write on the registered region. If a region is registered
      * with this flag the registration can be used as the remote handle for a
      * btl_put operation. */
-    MCA_BTL_REG_FLAG_REMOTE_WRITE  = MCA_RCACHE_ACCESS_REMOTE_WRITE,
+    MCA_BTL_REG_FLAG_REMOTE_WRITE = MCA_RCACHE_ACCESS_REMOTE_WRITE,
     /** Allow remote atomic operations on the registered region. If a region is
      * registered with this flag the registration can be used as the remote
      * handle for a btl_atomic_op or btl_atomic_fop operation. */
@@ -294,53 +305,53 @@ enum {
     /** Allow any btl operation on the registered region. If a region is registered
      * with this flag the registration can be used as the local or remote handle for
      * any btl operation. */
-    MCA_BTL_REG_FLAG_ACCESS_ANY    = MCA_RCACHE_ACCESS_ANY,
+    MCA_BTL_REG_FLAG_ACCESS_ANY = MCA_RCACHE_ACCESS_ANY,
 #if OPAL_CUDA_GDR_SUPPORT
     /** Region is in GPU memory */
-    MCA_BTL_REG_FLAG_CUDA_GPU_MEM  = 0x00010000,
+    MCA_BTL_REG_FLAG_CUDA_GPU_MEM = 0x00010000,
 #endif
 };
 
 /** supported atomic operations */
 enum {
     /** The btl supports atomic add */
-    MCA_BTL_ATOMIC_SUPPORTS_ADD    = 0x00000001,
+    MCA_BTL_ATOMIC_SUPPORTS_ADD = 0x00000001,
     /** The btl supports atomic bitwise and */
-    MCA_BTL_ATOMIC_SUPPORTS_AND    = 0x00000200,
+    MCA_BTL_ATOMIC_SUPPORTS_AND = 0x00000200,
     /** The btl supports atomic bitwise or */
-    MCA_BTL_ATOMIC_SUPPORTS_OR     = 0x00000400,
+    MCA_BTL_ATOMIC_SUPPORTS_OR = 0x00000400,
     /** The btl supports atomic bitwise exclusive or */
-    MCA_BTL_ATOMIC_SUPPORTS_XOR    = 0x00000800,
+    MCA_BTL_ATOMIC_SUPPORTS_XOR = 0x00000800,
 
     /** The btl supports logical and */
-    MCA_BTL_ATOMIC_SUPPORTS_LAND   = 0x00001000,
+    MCA_BTL_ATOMIC_SUPPORTS_LAND = 0x00001000,
     /** The btl supports logical or */
-    MCA_BTL_ATOMIC_SUPPORTS_LOR    = 0x00002000,
+    MCA_BTL_ATOMIC_SUPPORTS_LOR = 0x00002000,
     /** The btl supports logical exclusive or */
-    MCA_BTL_ATOMIC_SUPPORTS_LXOR   = 0x00004000,
+    MCA_BTL_ATOMIC_SUPPORTS_LXOR = 0x00004000,
 
     /** The btl supports atomic swap */
-    MCA_BTL_ATOMIC_SUPPORTS_SWAP   = 0x00010000,
+    MCA_BTL_ATOMIC_SUPPORTS_SWAP = 0x00010000,
 
     /** The btl supports atomic min */
-    MCA_BTL_ATOMIC_SUPPORTS_MIN    = 0x00100000,
+    MCA_BTL_ATOMIC_SUPPORTS_MIN = 0x00100000,
     /** The btl supports atomic min */
-    MCA_BTL_ATOMIC_SUPPORTS_MAX    = 0x00200000,
+    MCA_BTL_ATOMIC_SUPPORTS_MAX = 0x00200000,
 
     /** The btl supports 32-bit integer operations. Keep in mind the btl may
      * support only a subset of the available atomics. */
-    MCA_BTL_ATOMIC_SUPPORTS_32BIT  = 0x01000000,
+    MCA_BTL_ATOMIC_SUPPORTS_32BIT = 0x01000000,
 
     /** The btl supports floating-point operations. Keep in mind the btl may
      * support only a subset of the available atomics and may not support
      * both 64 or 32-bit floating point. */
-    MCA_BTL_ATOMIC_SUPPORTS_FLOAT  = 0x02000000,
+    MCA_BTL_ATOMIC_SUPPORTS_FLOAT = 0x02000000,
 
     /** The btl supports atomic compare-and-swap */
-    MCA_BTL_ATOMIC_SUPPORTS_CSWAP  = 0x10000000,
+    MCA_BTL_ATOMIC_SUPPORTS_CSWAP = 0x10000000,
 
     /** The btl guarantees global atomicity (can mix btl atomics with cpu atomics) */
-    MCA_BTL_ATOMIC_SUPPORTS_GLOB   = 0x20000000,
+    MCA_BTL_ATOMIC_SUPPORTS_GLOB = 0x20000000,
 };
 
 enum {
@@ -356,7 +367,7 @@ enum mca_btl_base_atomic_op_t {
     /** Atomic and: (*remote_address) = (*remote_address) & operand */
     MCA_BTL_ATOMIC_AND = 0x0011,
     /** Atomic or: (*remote_address) = (*remote_address) | operand */
-    MCA_BTL_ATOMIC_OR  = 0x0012,
+    MCA_BTL_ATOMIC_OR = 0x0012,
     /** Atomic xor: (*remote_address) = (*remote_address) ^ operand */
     MCA_BTL_ATOMIC_XOR = 0x0014,
     /** Atomic logical and: (*remote_address) = (*remote_address) && operand */
@@ -388,12 +399,10 @@ typedef enum mca_btl_base_atomic_op_t mca_btl_base_atomic_op_t;
  * @param[IN] descriptor  the BTL descriptor
  *
  */
-typedef void (*mca_btl_base_completion_fn_t)(
-    struct mca_btl_base_module_t* module,
-    struct mca_btl_base_endpoint_t* endpoint,
-    struct mca_btl_base_descriptor_t* descriptor,
-    int status);
-
+typedef void (*mca_btl_base_completion_fn_t)(struct mca_btl_base_module_t *module,
+                                             struct mca_btl_base_endpoint_t *endpoint,
+                                             struct mca_btl_base_descriptor_t *descriptor,
+                                             int status);
 
 /**
  * Asynchronous callback function on completion of an rdma or atomic operation.
@@ -411,14 +420,9 @@ typedef void (*mca_btl_base_completion_fn_t)(
  *
  */
 typedef void (*mca_btl_base_rdma_completion_fn_t)(
-    struct mca_btl_base_module_t* module,
-    struct mca_btl_base_endpoint_t* endpoint,
-    void *local_address,
-    struct mca_btl_base_registration_handle_t *local_handle,
-    void *context,
-    void *cbdata,
-    int status);
-
+    struct mca_btl_base_module_t *module, struct mca_btl_base_endpoint_t *endpoint,
+    void *local_address, struct mca_btl_base_registration_handle_t *local_handle, void *context,
+    void *cbdata, int status);
 
 /**
  * Describes a region/segment of memory that is addressable
@@ -440,26 +444,25 @@ typedef void (*mca_btl_base_rdma_completion_fn_t)(
 struct mca_btl_base_segment_t {
     /** Address of the memory */
     opal_ptr_t seg_addr;
-     /** Length in bytes */
-    uint64_t   seg_len;
+    /** Length in bytes */
+    uint64_t seg_len;
 };
 typedef struct mca_btl_base_segment_t mca_btl_base_segment_t;
 
-
 #if OPAL_ENABLE_HETEROGENEOUS_SUPPORT && !defined(WORDS_BIGENDIAN)
-#define MCA_BTL_BASE_SEGMENT_HTON(s)                   \
-    do {                                               \
-        (s).seg_addr.lval = hton64((s).seg_addr.lval); \
-        (s).seg_len = hton64((s).seg_len);             \
-    } while(0)
-#define MCA_BTL_BASE_SEGMENT_NTOH(s)                   \
-    do {                                               \
-        (s).seg_addr.lval = ntoh64((s).seg_addr.lval); \
-        (s).seg_len = ntoh64((s).seg_len);             \
-    } while(0)
+#    define MCA_BTL_BASE_SEGMENT_HTON(s)                   \
+        do {                                               \
+            (s).seg_addr.lval = hton64((s).seg_addr.lval); \
+            (s).seg_len = hton64((s).seg_len);             \
+        } while (0)
+#    define MCA_BTL_BASE_SEGMENT_NTOH(s)                   \
+        do {                                               \
+            (s).seg_addr.lval = ntoh64((s).seg_addr.lval); \
+            (s).seg_len = ntoh64((s).seg_len);             \
+        } while (0)
 #else
-#define MCA_BTL_BASE_SEGMENT_HTON(s)
-#define MCA_BTL_BASE_SEGMENT_NTOH(s)
+#    define MCA_BTL_BASE_SEGMENT_HTON(s)
+#    define MCA_BTL_BASE_SEGMENT_NTOH(s)
 #endif
 /**
  * @brief Descriptor to hold outgoing send messages
@@ -470,12 +473,12 @@ typedef struct mca_btl_base_segment_t mca_btl_base_segment_t;
  */
 struct mca_btl_base_descriptor_t {
     opal_free_list_item_t super;
-    mca_btl_base_segment_t *des_segments;     /**< local segments */
-    size_t des_segment_count;                 /**< number of local segments */
-    mca_btl_base_completion_fn_t des_cbfunc;  /**< local callback function */
-    void* des_cbdata;                         /**< opaque callback data */
-    void* des_context;                        /**< more opaque callback data */
-    uint32_t des_flags;                       /**< hints to BTL */
+    mca_btl_base_segment_t *des_segments;    /**< local segments */
+    size_t des_segment_count;                /**< number of local segments */
+    mca_btl_base_completion_fn_t des_cbfunc; /**< local callback function */
+    void *des_cbdata;                        /**< opaque callback data */
+    void *des_context;                       /**< more opaque callback data */
+    uint32_t des_flags;                      /**< hints to BTL */
     /** order value, this is only
         valid in the local completion callback
         and may be used in subsequent calls to
@@ -489,25 +492,25 @@ typedef struct mca_btl_base_descriptor_t mca_btl_base_descriptor_t;
 
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(mca_btl_base_descriptor_t);
 
-#define MCA_BTL_DES_FLAGS_PRIORITY          0x0001
+#define MCA_BTL_DES_FLAGS_PRIORITY 0x0001
 /* Allow the BTL to dispose the descriptor once the callback
  * associated was triggered.
  */
-#define MCA_BTL_DES_FLAGS_BTL_OWNERSHIP     0x0002
+#define MCA_BTL_DES_FLAGS_BTL_OWNERSHIP 0x0002
 /* Allow the BTL to avoid calling the descriptor callback
- * if the send succeded in the btl_send (i.e in the fast path).
+ * if the send succeeded in the btl_send (i.e in the fast path).
  */
-#define MCA_BTL_DES_SEND_ALWAYS_CALLBACK    0x0004
+#define MCA_BTL_DES_SEND_ALWAYS_CALLBACK 0x0004
 
 /* Tell the PML that the copy is being done asynchronously
  */
-#define MCA_BTL_DES_FLAGS_CUDA_COPY_ASYNC   0x0008
+#define MCA_BTL_DES_FLAGS_ACCELERATOR_COPY_ASYNC 0x0008
 
 /* Ask the BTL to wake the remote process (send/sendi) or local process
  * (put/get) to handle this message. The BTL may ignore this flag if
  * signaled operations are not supported.
  */
-#define MCA_BTL_DES_FLAGS_SIGNAL            0x0040
+#define MCA_BTL_DES_FLAGS_SIGNAL 0x0040
 
 /**
  * Maximum number of allowed segments in src/dst fields of a descriptor.
@@ -546,11 +549,10 @@ struct mca_btl_base_receive_descriptor_t {
 };
 typedef struct mca_btl_base_receive_descriptor_t mca_btl_base_receive_descriptor_t;
 
-
 /*
  *  BTL base header, stores the tag at a minimum
  */
-struct mca_btl_base_header_t{
+struct mca_btl_base_header_t {
     mca_btl_base_tag_t tag;
 };
 typedef struct mca_btl_base_header_t mca_btl_base_header_t;
@@ -566,33 +568,27 @@ typedef struct mca_btl_base_header_t mca_btl_base_header_t;
  * MCA->BTL Initializes the BTL component and creates specific BTL
  * module(s).
  *
- * @param num_btls (OUT) Returns the number of btl modules created, or 0
- *                       if the transport is not available.
- *
- * @param enable_progress_threads (IN) Whether this component is
- * allowed to run a hidden/progress thread or not.
- *
- * @param enable_mpi_threads (IN) Whether support for multiple MPI
- * threads is enabled or not (i.e., MPI_THREAD_MULTIPLE), which
- * indicates whether multiple threads may invoke this component
- * simultaneously or not.
- *
- * @return Array of pointers to BTL modules, or NULL if the transport
- *         is not available.
- *
  * During component initialization, the BTL component should discover
  * the physical devices that are available for the given transport,
  * and create a BTL module to represent each device. Any addressing
  * information required by peers to reach the device should be published
  * during this function via the modex_send() interface.
  *
+ * @param[OUT] num_btls Returns the number of btl modules created, or 0
+ *                      if the transport is not available.
+ * @param[IN] enable_progress_threads Whether this component is
+ *                      allowed to run a hidden/progress thread or not.
+ * @param[IN] enable_mpi_threads Whether support for multiple MPI
+ *                      threads is enabled or not (i.e.,
+ *                      MPI_THREAD_MULTIPLE), which indicates whether
+ *                      multiple threads may invoke this component
+ *                      simultaneously or not.
+ *
+ * @return Array of pointers to BTL modules, or NULL if the transport
+ *         is not available.
  */
-
-typedef struct mca_btl_base_module_t** (*mca_btl_base_component_init_fn_t)(
-    int *num_btls,
-    bool enable_progress_threads,
-    bool enable_mpi_threads
-);
+typedef struct mca_btl_base_module_t **(*mca_btl_base_component_init_fn_t)(
+    int *num_btls, bool enable_progress_threads, bool enable_mpi_threads);
 
 /**
  * MCA->BTL Called to progress outstanding requests for
@@ -602,10 +598,8 @@ typedef struct mca_btl_base_module_t** (*mca_btl_base_component_init_fn_t)(
  *                   how many items where completed in the call
  *                   to progress.
  */
-
 typedef int (*mca_btl_base_component_progress_fn_t)(void);
 
-
 /**
  * Callback function that is called asynchronously on receipt
  * of data by the transport layer.
@@ -621,16 +615,15 @@ typedef int (*mca_btl_base_component_progress_fn_t)(void);
  */
 
 typedef void (*mca_btl_base_module_recv_cb_fn_t)(
-    struct mca_btl_base_module_t *btl,
-    const mca_btl_base_receive_descriptor_t *descriptor);
+    struct mca_btl_base_module_t *btl, const mca_btl_base_receive_descriptor_t *descriptor);
 
 typedef struct mca_btl_active_message_callback_t {
     mca_btl_base_module_recv_cb_fn_t cbfunc;
-    void* cbdata;
+    void *cbdata;
 } mca_btl_active_message_callback_t;
 
-OPAL_DECLSPEC extern
-mca_btl_active_message_callback_t mca_btl_base_active_message_trigger[MCA_BTL_TAG_MAX];
+OPAL_DECLSPEC extern mca_btl_active_message_callback_t
+    mca_btl_base_active_message_trigger[MCA_BTL_TAG_MAX];
 
 /**
  *  BTL component descriptor. Contains component version information
@@ -638,10 +631,10 @@ mca_btl_active_message_callback_t mca_btl_base_active_message_trigger[MCA_BTL_TA
  */
 
 struct mca_btl_base_component_3_0_0_t {
-  mca_base_component_t btl_version;
-  mca_base_component_data_t btl_data;
-  mca_btl_base_component_init_fn_t btl_init;
-  mca_btl_base_component_progress_fn_t btl_progress;
+    mca_base_component_t btl_version;
+    mca_base_component_data_t btl_data;
+    mca_btl_base_component_init_fn_t btl_init;
+    mca_btl_base_component_progress_fn_t btl_progress;
 };
 typedef struct mca_btl_base_component_3_0_0_t mca_btl_base_component_3_0_0_t;
 typedef struct mca_btl_base_component_3_0_0_t mca_btl_base_component_t;
@@ -654,7 +647,6 @@ typedef struct mca_btl_base_component_3_0_0_t mca_btl_base_component_t;
  */
 typedef struct mca_btl_base_component_3_0_0_t mca_btl_base_component_2_0_0_t;
 
-
 /*
  * BTL module interface functions and datatype.
  */
@@ -663,29 +655,20 @@ typedef struct mca_btl_base_component_3_0_0_t mca_btl_base_component_2_0_0_t;
  * MCA->BTL Clean up any resources held by BTL module
  * before the module is unloaded.
  *
- * @param btl (IN)   BTL module.
- * @return           OPAL_SUCCESS or error status on failure.
- *
  * Prior to unloading a BTL module, the MCA framework will call
  * the BTL finalize method of the module. Any resources held by
  * the BTL should be released and if required the memory corresponding
  * to the BTL module freed.
  *
+ * @param[IN] btl    BTL module.
+ *
+ * @return           OPAL_SUCCESS or error status on failure.
  */
-typedef int (*mca_btl_base_module_finalize_fn_t)(
-    struct mca_btl_base_module_t* btl
-);
+typedef int (*mca_btl_base_module_finalize_fn_t)(struct mca_btl_base_module_t *btl);
 
 /**
  * BML->BTL notification of change in the process list.
  *
- * @param btl (IN)            BTL module
- * @param nprocs (IN)         Number of processes
- * @param procs (IN)          Array of processes
- * @param endpoint (OUT)      Array of mca_btl_base_endpoint_t structures by BTL.
- * @param reachable (OUT)     Bitmask indicating set of peer processes that are reachable by this BTL.
- * @return                    OPAL_SUCCESS or error status on failure.
- *
  * The mca_btl_base_module_add_procs_fn_t() is called by the BML to
  * determine the set of BTLs that should be used to reach each process.
  * Any addressing information exported by the peer via the modex_send()
@@ -702,34 +685,37 @@ typedef int (*mca_btl_base_module_finalize_fn_t)(
  * functions (e.g btl_send). This may be used by the BTL to cache any
  * addressing or connection information (e.g. TCP socket, IB queue
  * pair).
+ *
+ * @param[IN] btl             BTL module
+ * @param[IN] nprocs          Number of processes
+ * @param[IN] procs           Array of processes
+ * @param[OUT] endpoint       Array of mca_btl_base_endpoint_t structures by BTL.
+ * @param[OUT] reachable      Bitmask indicating set of peer processes that
+ *                            are reachable by this BTL.
+ * @return                    OPAL_SUCCESS or error status on failure.
  */
-typedef int (*mca_btl_base_module_add_procs_fn_t)(
-    struct mca_btl_base_module_t* btl,
-    size_t nprocs,
-    struct opal_proc_t** procs,
-    struct mca_btl_base_endpoint_t** endpoints,
-    struct opal_bitmap_t* reachable
-);
+typedef int (*mca_btl_base_module_add_procs_fn_t)(struct mca_btl_base_module_t *btl, size_t nprocs,
+                                                  struct opal_proc_t **procs,
+                                                  struct mca_btl_base_endpoint_t **endpoints,
+                                                  struct opal_bitmap_t *reachable);
 
 /**
  * Notification of change to the process list.
  *
- * @param btl (IN)     BTL module
- * @param nprocs (IN)  Number of processes
- * @param proc (IN)    Set of processes
- * @param peer (IN)    Set of peer addressing information.
- * @return             Status indicating if cleanup was successful
- *
  * When the process list changes, the BML notifies the BTL of the
  * change, to provide the opportunity to cleanup or release any
  * resources associated with the peer.
+ *
+ * @param[IN] btl     BTL module
+ * @param[IN] nprocs  Number of processes
+ * @param[IN] proc    Set of processes
+ * @param[IN] peer    Set of peer addressing information.
+ *
+ * @return             Status indicating if cleanup was successful
  */
-typedef int (*mca_btl_base_module_del_procs_fn_t)(
-    struct mca_btl_base_module_t* btl,
-    size_t nprocs,
-    struct opal_proc_t** procs,
-    struct mca_btl_base_endpoint_t** peer
-);
+typedef int (*mca_btl_base_module_del_procs_fn_t)(struct mca_btl_base_module_t *btl, size_t nprocs,
+                                                  struct opal_proc_t **procs,
+                                                  struct mca_btl_base_endpoint_t **peer);
 
 /**
  * Register a callback function that is called on receipt
@@ -741,17 +727,14 @@ typedef int (*mca_btl_base_module_del_procs_fn_t)(
  * @param[IN] cbfunc   The callback function
  * @param[IN] cbdata   Opaque callback data
  *
- * @return OPAL_SUCCESS The callback was registered successfully
- * @return OPAL_ERROR   The callback was NOT registered successfully
+ * @retval OPAL_SUCCESS The callback was registered successfully
+ * @retval OPAL_ERROR   The callback was NOT registered successfully
  *
  */
-typedef int (*mca_btl_base_module_register_fn_t)(
-    struct mca_btl_base_module_t* btl,
-    mca_btl_base_tag_t tag,
-    mca_btl_base_module_recv_cb_fn_t cbfunc,
-    void* cbdata
-);
-
+typedef int (*mca_btl_base_module_register_fn_t)(struct mca_btl_base_module_t *btl,
+                                                 mca_btl_base_tag_t tag,
+                                                 mca_btl_base_module_recv_cb_fn_t cbfunc,
+                                                 void *cbdata);
 
 /**
  * Callback function that is called asynchronously on receipt
@@ -762,14 +745,8 @@ typedef int (*mca_btl_base_module_register_fn_t)(
  * @param[IN] errproc process that had an error
  * @param[IN] btlinfo descriptive string from the BTL
  */
-
-typedef void (*mca_btl_base_module_error_cb_fn_t)(
-        struct mca_btl_base_module_t* btl,
-        int32_t flags,
-        struct opal_proc_t* errproc,
-        char* btlinfo
-);
-
+typedef void (*mca_btl_base_module_error_cb_fn_t)(struct mca_btl_base_module_t *btl, int32_t flags,
+                                                  struct opal_proc_t *errproc, char *btlinfo);
 
 /**
  * Register a callback function that is called on receipt
@@ -778,15 +755,12 @@ typedef void (*mca_btl_base_module_error_cb_fn_t)(
  * @param[IN] btl       BTL module
  * @param[IN] cbfunc    The callback function
  *
- * @return OPAL_SUCCESS The callback was registered successfully
- * @return OPAL_ERROR   The callback was NOT registered successfully
+ * @retval OPAL_SUCCESS The callback was registered successfully
+ * @retval OPAL_ERROR   The callback was NOT registered successfully
  *
  */
-typedef int (*mca_btl_base_module_register_error_fn_t)(
-    struct mca_btl_base_module_t* btl,
-    mca_btl_base_module_error_cb_fn_t cbfunc
-);
-
+typedef int (*mca_btl_base_module_register_error_fn_t)(struct mca_btl_base_module_t *btl,
+                                                       mca_btl_base_module_error_cb_fn_t cbfunc);
 
 /**
  * Allocate a descriptor with a segment of the requested size.
@@ -798,33 +772,25 @@ typedef int (*mca_btl_base_module_register_error_fn_t)(
  * local completion callback function called and the order tag of
  * that descriptor is only valid upon the local completion callback function.
  *
- *
- * @param btl (IN)      BTL module
- * @param size (IN)     Request segment size.
- * @param order (IN)    The ordering tag (may be MCA_BTL_NO_ORDER)
+ * @param[IN] btl       BTL module
+ * @param[IN] size      Request segment size.
+ * @param[IN] order     The ordering tag (may be MCA_BTL_NO_ORDER)
  */
 
-typedef mca_btl_base_descriptor_t* (*mca_btl_base_module_alloc_fn_t)(
-    struct mca_btl_base_module_t* btl,
-    struct mca_btl_base_endpoint_t* endpoint,
-    uint8_t order,
-    size_t size,
-    uint32_t flags
-);
+typedef mca_btl_base_descriptor_t *(*mca_btl_base_module_alloc_fn_t)(
+    struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, uint8_t order,
+    size_t size, uint32_t flags);
 
 /**
  * Return a descriptor allocated from this BTL via alloc/prepare.
  * A descriptor can only be deallocated after its local completion
  * callback function has called for all send/put/get operations.
  *
- * @param btl (IN)      BTL module
- * @param segment (IN)  Descriptor allocated from the BTL
+ * @param[IN] btl       BTL module
+ * @param[IN] segment   Descriptor allocated from the BTL
  */
-typedef int (*mca_btl_base_module_free_fn_t)(
-    struct mca_btl_base_module_t* btl,
-    mca_btl_base_descriptor_t* descriptor
-);
-
+typedef int (*mca_btl_base_module_free_fn_t)(struct mca_btl_base_module_t *btl,
+                                             mca_btl_base_descriptor_t *descriptor);
 
 /**
  * Prepare a descriptor for send using the supplied convertor. If the convertor
@@ -839,38 +805,23 @@ typedef int (*mca_btl_base_module_free_fn_t)(
  * called and the order tag of that descriptor is only valid upon the local
  * completion callback function.
  *
- * @param btl (IN)          BTL module
- * @param endpoint (IN)     BTL peer addressing
- * @param registration (IN) Memory registration
- * @param convertor (IN)    Data type convertor
- * @param order (IN)        The ordering tag (may be MCA_BTL_NO_ORDER)
- * @param reserve (IN)      Additional bytes requested by upper layer to precede user data
- * @param size (IN/OUT)     Number of bytes to prepare (IN),
+ * @param[IN] btl           BTL module
+ * @param[IN] endpoint      BTL peer addressing
+ * @param[IN] registration  Memory registration
+ * @param[IN] convertor     Data type convertor
+ * @param[IN] order         The ordering tag (may be MCA_BTL_NO_ORDER)
+ * @param[IN] reserve       Additional bytes requested by upper layer to precede user data
+ * @param[IN,OUT] size      Number of bytes to prepare (IN),
  *                          number of bytes actually prepared (OUT)
- *
  */
-typedef struct mca_btl_base_descriptor_t* (*mca_btl_base_module_prepare_fn_t)(
-    struct mca_btl_base_module_t* btl,
-    struct mca_btl_base_endpoint_t* endpoint,
-    struct opal_convertor_t* convertor,
-    uint8_t order,
-    size_t reserve,
-    size_t* size,
-    uint32_t flags
-);
+typedef struct mca_btl_base_descriptor_t *(*mca_btl_base_module_prepare_fn_t)(
+    struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+    struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size,
+    uint32_t flags);
 
 /**
  * @brief Register a memory region for put/get/atomic operations.
  *
- * @param btl (IN)         BTL module
- * @param endpoint(IN)     BTL addressing information (or NULL for all endpoints)
- * @param base (IN)        Pointer to start of region
- * @param size (IN)        Size of region
- * @param flags (IN)       Flags including access permissions
- *
- * @returns a memory registration handle valid for both local and remote operations
- * @returns NULL if the region could not be registered
- *
  * This function registers the specified region with the hardware for use with
  * the btl_put, btl_get, btl_atomic_cas, btl_atomic_op, and btl_atomic_fop
  * functions. Care should be taken to not hold an excessive number of registrations
@@ -880,17 +831,23 @@ typedef struct mca_btl_base_descriptor_t* (*mca_btl_base_module_prepare_fn_t)(
  * mca_btl_base_registration_handle_t*) is passed to the caller.  The
  * BTL module cannot free or reuse the handle until it is returned via
  * the mca_btl_base_module_deregister_mem_fn_t function.
+ *
+ * @param[IN] btl          BTL module
+ * @param[IN] endpoint     BTL addressing information (or NULL for all endpoints)
+ * @param[IN] base         Pointer to start of region
+ * @param[IN] size         Size of region
+ * @param[IN] flags        Flags including access permissions
+ *
+ * @returns a memory registration handle valid for both local and
+ * remote operations or NULL if the region could not be registered.
  */
 typedef struct mca_btl_base_registration_handle_t *(*mca_btl_base_module_register_mem_fn_t)(
-    struct mca_btl_base_module_t* btl, struct mca_btl_base_endpoint_t *endpoint, void *base,
+    struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, void *base,
     size_t size, uint32_t flags);
 
 /**
  * @brief Deregister a memory region
  *
- * @param btl (IN)         BTL module region was registered with
- * @param handle (IN)      BTL registration handle to deregister
- *
  * This function deregisters the memory region associated with the specified handle. Care
  * should be taken to not perform any RDMA or atomic operation on this memory region
  * after it is deregistered. It is erroneous to specify a memory handle associated with
@@ -900,53 +857,62 @@ typedef struct mca_btl_base_registration_handle_t *(*mca_btl_base_module_registe
  * mca_btl_base_module_register_mem_fn_t function.  Ownership of the
  * memory pointed to by handle passes to the BTL module; this function
  * is now is allowed to free the memory, return it to a freelist, etc.
+ *
+ * @param[IN] btl         BTL module region was registered with
+ * @param[IN] handle      BTL registration handle to deregister
  */
 typedef int (*mca_btl_base_module_deregister_mem_fn_t)(
-    struct mca_btl_base_module_t* btl, struct mca_btl_base_registration_handle_t *handle);
+    struct mca_btl_base_module_t *btl, struct mca_btl_base_registration_handle_t *handle);
 
 /**
  * Initiate an asynchronous send.
- * Completion Semantics: the descriptor has been queued for a send operation
- *                       the BTL now controls the descriptor until local
- *                       completion callback is made on the descriptor
  *
  * All BTLs allow multiple concurrent asynchronous send operations on a descriptor
  *
- * @param btl (IN)         BTL module
- * @param endpoint (IN)    BTL addressing information
- * @param descriptor (IN)  Description of the data to be transfered
- * @param tag (IN)         The tag value used to notify the peer.
- *
+ * Completion Semantics: If OPAL_SUCCESS is returned, the descriptor
+ * has been queued for a send operation the BTL now controls the
+ * descriptor until local completion callback is made on the
+ * descriptor.  If 1 is returned, the descriptor has been sent and the
+ * operation is complete.  The local completion callback will not be
+ * made unless the MCA_BTL_DES_SEND_ALWAYS_CALLBACK flag was set on
+ * the descriptor.
+ *
+ * @param[IN] btl          BTL module
+ * @param[IN] endpoint     BTL addressing information
+ * @param[IN] descriptor   Description of the data to be transferred
+ * @param[IN] tag          The tag value used to notify the peer.
+ *
+ * @retval 1               The descriptor was successfully sent (see
+ *                         completion semantics above).
  * @retval OPAL_SUCCESS    The descriptor was successfully queued for a send
  * @retval OPAL_ERROR      The descriptor was NOT successfully queued for a send
  * @retval OPAL_ERR_UNREACH The endpoint is not reachable
  */
-typedef int (*mca_btl_base_module_send_fn_t)(
-    struct mca_btl_base_module_t* btl,
-    struct mca_btl_base_endpoint_t* endpoint,
-    struct mca_btl_base_descriptor_t* descriptor,
-    mca_btl_base_tag_t tag
-);
+typedef int (*mca_btl_base_module_send_fn_t)(struct mca_btl_base_module_t *btl,
+                                             struct mca_btl_base_endpoint_t *endpoint,
+                                             struct mca_btl_base_descriptor_t *descriptor,
+                                             mca_btl_base_tag_t tag);
 
 /**
  * Initiate an immediate blocking send.
- * Completion Semantics: the BTL will make a best effort
- *  to send the header and "size" bytes from the datatype using the convertor.
- *  The header is guaranteed to be delivered entirely in the first segment.
- *  Should the BTL be unable to deliver the data due to resource constraints
- *  the BTL will return a descriptor (via the OUT param)
- *  of size "payload_size + header_size".
- *
- * @param btl (IN)             BTL module
- * @param endpoint (IN)        BTL addressing information
- * @param convertor (IN)       Data type convertor
- * @param header (IN)          Pointer to header.
- * @param header_size (IN)     Size of header.
- * @param payload_size (IN)    Size of payload (from convertor).
- * @param order (IN)           The ordering tag (may be MCA_BTL_NO_ORDER)
- * @param flags (IN)           Flags.
- * @param tag (IN)             The tag value used to notify the peer.
- * @param descriptor (OUT)     The descriptor to be returned unable to be sent immediately
+ *
+ * Completion Semantics: the BTL will make a best effort to send the
+ * header and "size" bytes from the datatype using the convertor.  The
+ * header is guaranteed to be delivered entirely in the first segment.
+ * Should the BTL be unable to deliver the data due to resource
+ * constraints the BTL will return a descriptor (via the OUT param) of
+ * size "payload_size + header_size".
+ *
+ * @param[IN] btl              BTL module
+ * @param[IN] endpoint         BTL addressing information
+ * @param[IN] convertor        Data type convertor
+ * @param[IN] header           Pointer to header.
+ * @param[IN] header_size      Size of header.
+ * @param[IN] payload_size     Size of payload (from convertor).
+ * @param[IN] order            The ordering tag (may be MCA_BTL_NO_ORDER)
+ * @param[IN] flags            Flags.
+ * @param[IN] tag              The tag value used to notify the peer.
+ * @param[OUT] descriptor      The descriptor to be returned unable to be sent immediately
  *                             (may be NULL).
  *
  * @retval OPAL_SUCCESS           The send was successfully queued
@@ -955,22 +921,16 @@ typedef int (*mca_btl_base_module_send_fn_t)(
  * @retval OPAL_ERR_RESOURCE_BUSY The BTL is busy a descriptor will be returned
  *                                (via the OUT param) if descriptors are available
  */
-
-typedef int (*mca_btl_base_module_sendi_fn_t)(
-    struct mca_btl_base_module_t* btl,
-    struct mca_btl_base_endpoint_t* endpoint,
-    struct opal_convertor_t* convertor,
-    void* header,
-    size_t header_size,
-    size_t payload_size,
-    uint8_t order,
-    uint32_t flags,
-    mca_btl_base_tag_t tag,
-    mca_btl_base_descriptor_t** descriptor
- );
+typedef int (*mca_btl_base_module_sendi_fn_t)(struct mca_btl_base_module_t *btl,
+                                              struct mca_btl_base_endpoint_t *endpoint,
+                                              struct opal_convertor_t *convertor, void *header,
+                                              size_t header_size, size_t payload_size,
+                                              uint8_t order, uint32_t flags, mca_btl_base_tag_t tag,
+                                              mca_btl_base_descriptor_t **descriptor);
 
 /**
  * Initiate an asynchronous put.
+ *
  * Completion Semantics: if this function returns a 1 then the operation
  *                       is complete. a return of OPAL_SUCCESS indicates
  *                       the put operation has been queued with the
@@ -978,20 +938,20 @@ typedef int (*mca_btl_base_module_sendi_fn_t)(
  *                       until all outstanding operations on that handle
  *                       have been completed.
  *
- * @param btl (IN)            BTL module
- * @param endpoint (IN)       BTL addressing information
- * @param local_address (IN)  Local address to put from (registered)
- * @param remote_address (IN) Remote address to put to (registered remotely)
- * @param local_handle (IN)   Registration handle for region containing
+ * @param[IN] btl             BTL module
+ * @param[IN] endpoint        BTL addressing information
+ * @param[IN] local_address   Local address to put from (registered)
+ * @param[IN] remote_address  Remote address to put to (registered remotely)
+ * @param[IN] local_handle    Registration handle for region containing
  *                            (local_address, local_address + size)
- * @param remote_handle (IN)  Remote registration handle for region containing
+ * @param[IN] remote_handle   Remote registration handle for region containing
  *                            (remote_address, remote_address + size)
- * @param size (IN)           Number of bytes to put
- * @param flags (IN)          Flags for this put operation
- * @param order (IN)          Ordering
- * @param cbfunc (IN)         Function to call on completion (if queued)
- * @param cbcontext (IN)      Context for the callback
- * @param cbdata (IN)         Data for callback
+ * @param[IN] size            Number of bytes to put
+ * @param[IN] flags           Flags for this put operation
+ * @param[IN] order           Ordering
+ * @param[IN] cbfunc          Function to call on completion (if queued)
+ * @param[IN] cbcontext       Context for the callback
+ * @param[IN] cbdata          Data for callback
  *
  * @retval OPAL_SUCCESS    The descriptor was successfully queued for a put
  * @retval OPAL_ERROR      The descriptor was NOT successfully queued for a put
@@ -1000,11 +960,12 @@ typedef int (*mca_btl_base_module_sendi_fn_t)(
  * @retval OPAL_ERR_NOT_AVAILABLE  Put can not be performed due to size or
  *                         alignment restrictions.
  */
-typedef int (*mca_btl_base_module_put_fn_t) (struct mca_btl_base_module_t *btl,
-    struct mca_btl_base_endpoint_t *endpoint, void *local_address,
-    uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle,
-    struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-    int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
+typedef int (*mca_btl_base_module_put_fn_t)(
+    struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+    void *local_address, uint64_t remote_address,
+    struct mca_btl_base_registration_handle_t *local_handle,
+    struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags, int order,
+    mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
 
 /**
  * Initiate an asynchronous get.
@@ -1015,20 +976,20 @@ typedef int (*mca_btl_base_module_put_fn_t) (struct mca_btl_base_module_t *btl,
  *                       until all outstanding operations on that handle
  *                       have been completed.
  *
- * @param btl (IN)            BTL module
- * @param endpoint (IN)       BTL addressing information
- * @param local_address (IN)  Local address to put from (registered)
- * @param remote_address (IN) Remote address to put to (registered remotely)
- * @param local_handle (IN)   Registration handle for region containing
+ * @param[IN] btl             BTL module
+ * @param[IN] endpoint        BTL addressing information
+ * @param[IN] local_address   Local address to put from (registered)
+ * @param[IN] remote_address  Remote address to put to (registered remotely)
+ * @param[IN] local_handle    Registration handle for region containing
  *                            (local_address, local_address + size)
- * @param remote_handle (IN)  Remote registration handle for region containing
+ * @param[IN] remote_handle   Remote registration handle for region containing
  *                            (remote_address, remote_address + size)
- * @param size (IN)           Number of bytes to put
- * @param flags (IN)          Flags for this put operation
- * @param order (IN)          Ordering
- * @param cbfunc (IN)         Function to call on completion (if queued)
- * @param cbcontext (IN)      Context for the callback
- * @param cbdata (IN)         Data for callback
+ * @param[IN] size            Number of bytes to put
+ * @param[IN] flags           Flags for this put operation
+ * @param[IN] order           Ordering
+ * @param[IN] cbfunc          Function to call on completion (if queued)
+ * @param[IN] cbcontext       Context for the callback
+ * @param[IN] cbdata          Data for callback
  *
  * @retval OPAL_SUCCESS    The descriptor was successfully queued for a put
  * @retval OPAL_ERROR      The descriptor was NOT successfully queued for a put
@@ -1037,11 +998,12 @@ typedef int (*mca_btl_base_module_put_fn_t) (struct mca_btl_base_module_t *btl,
  * @retval OPAL_ERR_NOT_AVAILABLE  Put can not be performed due to size or
  *                         alignment restrictions.
  */
-typedef int (*mca_btl_base_module_get_fn_t) (struct mca_btl_base_module_t *btl,
-    struct mca_btl_base_endpoint_t *endpoint, void *local_address,
-    uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle,
-    struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-    int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
+typedef int (*mca_btl_base_module_get_fn_t)(
+    struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+    void *local_address, uint64_t remote_address,
+    struct mca_btl_base_registration_handle_t *local_handle,
+    struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags, int order,
+    mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
 
 /**
  * Initiate an asynchronous atomic operation.
@@ -1050,18 +1012,24 @@ typedef int (*mca_btl_base_module_get_fn_t) (struct mca_btl_base_module_t *btl,
  *                       the atomic operation has been queued with the
  *                       network.
  *
- * @param btl (IN)            BTL module
- * @param endpoint (IN)       BTL addressing information
- * @param remote_address (IN) Remote address to put to (registered remotely)
- * @param remote_handle (IN)  Remote registration handle for region containing
+ * After the operation is complete the remote address specified by {remote_address} and
+ * {remote_handle} will be updated with (*remote_address) = (*remote_address) op operand.
+ * The btl will guarantee consistency of atomic operations performed via the btl. Note,
+ * however, that not all btls will provide consistency between btl atomic operations and
+ * cpu or other btl atomics.
+ *
+ * @param[IN] btl             BTL module
+ * @param[IN] endpoint        BTL addressing information
+ * @param[IN] remote_address  Remote address to put to (registered remotely)
+ * @param[IN] remote_handle   Remote registration handle for region containing
  *                            (remote_address, remote_address + 8)
- * @param op (IN)             Operation to perform
- * @param operand (IN)        Operand for the operation
- * @param flags (IN)          Flags for this atomic operation
- * @param order (IN)          Ordering
- * @param cbfunc (IN)         Function to call on completion (if queued)
- * @param cbcontext (IN)      Context for the callback
- * @param cbdata (IN)         Data for callback
+ * @param[IN] op              Operation to perform
+ * @param[IN] operand         Operand for the operation
+ * @param[IN] flags           Flags for this atomic operation
+ * @param[IN] order           Ordering
+ * @param[IN] cbfunc          Function to call on completion (if queued)
+ * @param[IN] cbcontext       Context for the callback
+ * @param[IN] cbdata          Data for callback
  *
  * @retval OPAL_SUCCESS    The operation was successfully queued
  * @retval 1               The operation is complete
@@ -1071,18 +1039,12 @@ typedef int (*mca_btl_base_module_get_fn_t) (struct mca_btl_base_module_t *btl,
  * @retval OPAL_ERR_NOT_AVAILABLE  Atomic operation can not be performed due to
  *                         alignment restrictions or the operation {op} is not supported
  *                         by the hardware.
- *
- * After the operation is complete the remote address specified by {remote_address} and
- * {remote_handle} will be updated with (*remote_address) = (*remote_address) op operand.
- * The btl will guarantee consistency of atomic operations performed via the btl. Note,
- * however, that not all btls will provide consistency between btl atomic operations and
- * cpu or other btl atomics.
  */
-typedef int (*mca_btl_base_module_atomic_op64_fn_t) (struct mca_btl_base_module_t *btl,
-    struct mca_btl_base_endpoint_t *endpoint, uint64_t remote_address,
-    struct mca_btl_base_registration_handle_t *remote_handle, mca_btl_base_atomic_op_t op,
-    uint64_t operand, int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
-    void *cbcontext, void *cbdata);
+typedef int (*mca_btl_base_module_atomic_op64_fn_t)(
+    struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+    uint64_t remote_address, struct mca_btl_base_registration_handle_t *remote_handle,
+    mca_btl_base_atomic_op_t op, uint64_t operand, int flags, int order,
+    mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
 
 /**
  * Initiate an asynchronous fetching atomic operation.
@@ -1091,21 +1053,28 @@ typedef int (*mca_btl_base_module_atomic_op64_fn_t) (struct mca_btl_base_module_
  *                       the atomic operation has been queued with the
  *                       network.
  *
- * @param btl (IN)            BTL module
- * @param endpoint (IN)       BTL addressing information
- * @param local_address (OUT) Local address to store the result in
- * @param remote_address (IN) Remote address perfom operation on to (registered remotely)
- * @param local_handle (IN)   Local registration handle for region containing
+ * After the operation is complete the remote address specified by {remote_address} and
+ * {remote_handle} will be updated with (*remote_address) = (*remote_address) op operand.
+ * {local_address} will be updated with the previous value stored in {remote_address}.
+ * The btl will guarantee consistency of atomic operations performed via the btl. Note,
+ * however, that not all btls will provide consistency between btl atomic operations and
+ * cpu or other btl atomics.
+ *
+ * @param[IN] btl             BTL module
+ * @param[IN] endpoint        BTL addressing information
+ * @param[IN] local_address (OUT) Local address to store the result in
+ * @param[IN] remote_address  Remote address perform operation on to (registered remotely)
+ * @param[IN] local_handle    Local registration handle for region containing
  *                            (local_address, local_address + 8)
- * @param remote_handle (IN)  Remote registration handle for region containing
+ * @param[IN] remote_handle   Remote registration handle for region containing
  *                            (remote_address, remote_address + 8)
- * @param op (IN)             Operation to perform
- * @param operand (IN)        Operand for the operation
- * @param flags (IN)          Flags for this atomic operation
- * @param order (IN)          Ordering
- * @param cbfunc (IN)         Function to call on completion (if queued)
- * @param cbcontext (IN)      Context for the callback
- * @param cbdata (IN)         Data for callback
+ * @param[IN] op              Operation to perform
+ * @param[IN] operand         Operand for the operation
+ * @param[IN] flags           Flags for this atomic operation
+ * @param[IN] order           Ordering
+ * @param[IN] cbfunc          Function to call on completion (if queued)
+ * @param[IN] cbcontext       Context for the callback
+ * @param[IN] cbdata          Data for callback
  *
  * @retval OPAL_SUCCESS    The operation was successfully queued
  * @retval 1               The operation is complete
@@ -1115,16 +1084,10 @@ typedef int (*mca_btl_base_module_atomic_op64_fn_t) (struct mca_btl_base_module_
  * @retval OPAL_ERR_NOT_AVAILABLE  Atomic operation can not be performed due to
  *                         alignment restrictions or the operation {op} is not supported
  *                         by the hardware.
- *
- * After the operation is complete the remote address specified by {remote_address} and
- * {remote_handle} will be updated with (*remote_address) = (*remote_address) op operand.
- * {local_address} will be updated with the previous value stored in {remote_address}.
- * The btl will guarantee consistency of atomic operations performed via the btl. Note,
- * however, that not all btls will provide consistency between btl atomic operations and
- * cpu or other btl atomics.
  */
-typedef int (*mca_btl_base_module_atomic_fop64_fn_t) (struct mca_btl_base_module_t *btl,
-    struct mca_btl_base_endpoint_t *endpoint, void *local_address, uint64_t remote_address,
+typedef int (*mca_btl_base_module_atomic_fop64_fn_t)(
+    struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+    void *local_address, uint64_t remote_address,
     struct mca_btl_base_registration_handle_t *local_handle,
     struct mca_btl_base_registration_handle_t *remote_handle, mca_btl_base_atomic_op_t op,
     uint64_t operand, int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
@@ -1137,21 +1100,28 @@ typedef int (*mca_btl_base_module_atomic_fop64_fn_t) (struct mca_btl_base_module
  *                       the atomic operation has been queued with the
  *                       network.
  *
- * @param btl (IN)            BTL module
- * @param endpoint (IN)       BTL addressing information
- * @param local_address (OUT) Local address to store the result in
- * @param remote_address (IN) Remote address perfom operation on to (registered remotely)
- * @param local_handle (IN)   Local registration handle for region containing
+ * After the operation is complete the remote address specified by {remote_address} and
+ * {remote_handle} will be updated with {value} if *remote_address == compare.
+ * {local_address} will be updated with the previous value stored in {remote_address}.
+ * The btl will guarantee consistency of atomic operations performed via the btl. Note,
+ * however, that not all btls will provide consistency between btl atomic operations and
+ * cpu atomics.
+ *
+ * @param[IN] btl             BTL module
+ * @param[IN] endpoint        BTL addressing information
+ * @param[OUT] local_address  Local address to store the result in
+ * @param[IN] remote_address  Remote address perform operation on to (registered remotely)
+ * @param[IN] local_handle    Local registration handle for region containing
  *                            (local_address, local_address + 8)
- * @param remote_handle (IN)  Remote registration handle for region containing
+ * @param[IN] remote_handle   Remote registration handle for region containing
  *                            (remote_address, remote_address + 8)
- * @param compare (IN)        Operand for the operation
- * @param value (IN)          Value to store on success
- * @param flags (IN)          Flags for this atomic operation
- * @param order (IN)          Ordering
- * @param cbfunc (IN)         Function to call on completion (if queued)
- * @param cbcontext (IN)      Context for the callback
- * @param cbdata (IN)         Data for callback
+ * @param[IN] compare         Operand for the operation
+ * @param[IN] value           Value to store on success
+ * @param[IN] flags           Flags for this atomic operation
+ * @param[IN] order           Ordering
+ * @param[IN] cbfunc          Function to call on completion (if queued)
+ * @param[IN] cbcontext       Context for the callback
+ * @param[IN] cbdata          Data for callback
  *
  * @retval OPAL_SUCCESS    The operation was successfully queued
  * @retval 1               The operation is complete
@@ -1161,40 +1131,31 @@ typedef int (*mca_btl_base_module_atomic_fop64_fn_t) (struct mca_btl_base_module
  * @retval OPAL_ERR_NOT_AVAILABLE  Atomic operation can not be performed due to
  *                         alignment restrictions or the operation {op} is not supported
  *                         by the hardware.
- *
- * After the operation is complete the remote address specified by {remote_address} and
- * {remote_handle} will be updated with {value} if *remote_address == compare.
- * {local_address} will be updated with the previous value stored in {remote_address}.
- * The btl will guarantee consistency of atomic operations performed via the btl. Note,
- * however, that not all btls will provide consistency between btl atomic operations and
- * cpu atomics.
  */
-typedef int (*mca_btl_base_module_atomic_cswap64_fn_t) (struct mca_btl_base_module_t *btl,
-    struct mca_btl_base_endpoint_t *endpoint, void *local_address, uint64_t remote_address,
+typedef int (*mca_btl_base_module_atomic_cswap64_fn_t)(
+    struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+    void *local_address, uint64_t remote_address,
     struct mca_btl_base_registration_handle_t *local_handle,
-    struct mca_btl_base_registration_handle_t *remote_handle, uint64_t compare,
-    uint64_t value, int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
-    void *cbcontext, void *cbdata);
+    struct mca_btl_base_registration_handle_t *remote_handle, uint64_t compare, uint64_t value,
+    int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
 
 /**
  * Diagnostic dump of btl state.
  *
- * @param btl (IN)         BTL module
- * @param endpoint (IN)    BTL endpoint
- * @param verbose (IN)     Verbosity level
+ * @param[IN] btl          BTL module
+ * @param[IN] endpoint     BTL endpoint
+ * @param[IN] verbose      Verbosity level
  */
 
-typedef void (*mca_btl_base_module_dump_fn_t)(
-    struct mca_btl_base_module_t* btl,
-    struct mca_btl_base_endpoint_t* endpoint,
-    int verbose
-);
+typedef void (*mca_btl_base_module_dump_fn_t)(struct mca_btl_base_module_t *btl,
+                                              struct mca_btl_base_endpoint_t *endpoint,
+                                              int verbose);
 
 /**
  * Flush all outstanding RDMA operations on an endpoint or all endpoints.
  *
- * @param btl (IN)         BTL module
- * @param endpoint (IN)    Endpoint to flush (NULL == all)
+ * @param[IN] btl          BTL module
+ * @param[IN] endpoint     Endpoint to flush (NULL == all)
  *
  * This function returns when all outstanding RDMA (put, get, atomic) operations
  * that were started prior to the flush call have completed. This call does
@@ -1202,7 +1163,8 @@ typedef void (*mca_btl_base_module_dump_fn_t)(
  *
  * The BTL is allowed to ignore the endpoint parameter and flush *all* endpoints.
  */
-typedef int (*mca_btl_base_module_flush_fn_t) (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint);
+typedef int (*mca_btl_base_module_flush_fn_t)(struct mca_btl_base_module_t *btl,
+                                              struct mca_btl_base_endpoint_t *endpoint);
 
 /**
  * BTL module interface functions and attributes.
@@ -1210,69 +1172,79 @@ typedef int (*mca_btl_base_module_flush_fn_t) (struct mca_btl_base_module_t *btl
 struct mca_btl_base_module_t {
 
     /* BTL common attributes */
-    mca_btl_base_component_t* btl_component; /**< pointer back to the BTL component structure */
-    size_t      btl_eager_limit;      /**< maximum size of first fragment -- eager send */
-    size_t      btl_rndv_eager_limit;    /**< the size of a data sent in a first fragment of rendezvous protocol */
-    size_t      btl_max_send_size;    /**< maximum send fragment size supported by the BTL */
-    size_t      btl_rdma_pipeline_send_length; /**< amount of bytes that should be send by pipeline protocol */
-    size_t      btl_rdma_pipeline_frag_size; /**< maximum rdma fragment size supported by the BTL */
-    size_t      btl_min_rdma_pipeline_size; /**< minimum packet size for pipeline protocol  */
-    uint32_t    btl_exclusivity;      /**< indicates this BTL should be used exclusively */
-    uint32_t    btl_latency;          /**< relative ranking of latency used to prioritize btls */
-    uint32_t    btl_bandwidth;        /**< bandwidth (Mbytes/sec) supported by each endpoint */
-    uint32_t    btl_flags;            /**< flags (put/get...) */
-    uint32_t    btl_atomic_flags;     /**< atomic operations supported (add, and, xor, etc) */
-    size_t      btl_registration_handle_size; /**< size of the BTLs registration handles */
+    mca_btl_base_component_t *btl_component; /**< pointer back to the BTL component structure */
+    size_t btl_eager_limit;                  /**< maximum size of first fragment -- eager send */
+    size_t btl_rndv_eager_limit; /**< the size of a data sent in a first fragment of rendezvous
+                                    protocol */
+    size_t btl_max_send_size;    /**< maximum send fragment size supported by the BTL */
+    size_t btl_rdma_pipeline_send_length; /**< amount of bytes that should be send by pipeline
+                                             protocol */
+    size_t btl_rdma_pipeline_frag_size;   /**< maximum rdma fragment size supported by the BTL */
+    size_t btl_min_rdma_pipeline_size;    /**< minimum packet size for pipeline protocol  */
+    uint32_t btl_exclusivity;             /**< indicates this BTL should be used exclusively */
+    uint32_t btl_latency;                /**< relative ranking of latency used to prioritize btls */
+    uint32_t btl_bandwidth;              /**< bandwidth (Mbytes/sec) supported by each endpoint */
+    uint32_t btl_flags;                  /**< flags (put/get...) */
+    uint32_t btl_atomic_flags;           /**< atomic operations supported (add, and, xor, etc) */
+    size_t btl_registration_handle_size; /**< size of the BTLs registration handles */
 
     /* One-sided limitations (0 for no alignment, SIZE_MAX for no limit ) */
-    size_t      btl_get_limit;        /**< maximum size supported by the btl_get function */
-    size_t      btl_get_alignment;    /**< minimum alignment/size needed by btl_get (power of 2) */
-    size_t      btl_put_limit;        /**< maximum size supported by the btl_put function */
-    size_t      btl_put_alignment;    /**< minimum alignment/size needed by btl_put (power of 2) */
+    size_t btl_get_limit;     /**< maximum size supported by the btl_get function */
+    size_t btl_get_alignment; /**< minimum alignment/size needed by btl_get (power of 2) */
+    size_t btl_put_limit;     /**< maximum size supported by the btl_put function */
+    size_t btl_put_alignment; /**< minimum alignment/size needed by btl_put (power of 2) */
 
     /* minimum transaction sizes for which registration is required for local memory */
-    size_t      btl_get_local_registration_threshold;
-    size_t      btl_put_local_registration_threshold;
+    size_t btl_get_local_registration_threshold;
+    size_t btl_put_local_registration_threshold;
 
     /* BTL function table */
-    mca_btl_base_module_add_procs_fn_t      btl_add_procs;
-    mca_btl_base_module_del_procs_fn_t      btl_del_procs;
-    mca_btl_base_module_register_fn_t       btl_register;
-    mca_btl_base_module_finalize_fn_t       btl_finalize;
-
-    mca_btl_base_module_alloc_fn_t          btl_alloc;
-    mca_btl_base_module_free_fn_t           btl_free;
-    mca_btl_base_module_prepare_fn_t        btl_prepare_src;
-    mca_btl_base_module_send_fn_t           btl_send;
-    mca_btl_base_module_sendi_fn_t          btl_sendi;
-    mca_btl_base_module_put_fn_t            btl_put;
-    mca_btl_base_module_get_fn_t            btl_get;
-    mca_btl_base_module_dump_fn_t           btl_dump;
+    mca_btl_base_module_add_procs_fn_t btl_add_procs;
+    mca_btl_base_module_del_procs_fn_t btl_del_procs;
+    mca_btl_base_module_register_fn_t btl_register;
+    mca_btl_base_module_finalize_fn_t btl_finalize;
+
+    mca_btl_base_module_alloc_fn_t btl_alloc;
+    mca_btl_base_module_free_fn_t btl_free;
+    mca_btl_base_module_prepare_fn_t btl_prepare_src;
+    mca_btl_base_module_send_fn_t btl_send;
+    mca_btl_base_module_sendi_fn_t btl_sendi;
+    mca_btl_base_module_put_fn_t btl_put;
+    mca_btl_base_module_get_fn_t btl_get;
+    mca_btl_base_module_dump_fn_t btl_dump;
 
     /* atomic operations */
-    mca_btl_base_module_atomic_op64_fn_t    btl_atomic_op;
-    mca_btl_base_module_atomic_fop64_fn_t   btl_atomic_fop;
+    mca_btl_base_module_atomic_op64_fn_t btl_atomic_op;
+    mca_btl_base_module_atomic_fop64_fn_t btl_atomic_fop;
     mca_btl_base_module_atomic_cswap64_fn_t btl_atomic_cswap;
 
     /* new memory registration functions */
-    mca_btl_base_module_register_mem_fn_t   btl_register_mem;   /**< memory registration function (NULL if not needed) */
-    mca_btl_base_module_deregister_mem_fn_t btl_deregister_mem; /**< memory deregistration function (NULL if not needed) */
+    mca_btl_base_module_register_mem_fn_t
+        btl_register_mem; /**< memory registration function (NULL if not needed) */
+    mca_btl_base_module_deregister_mem_fn_t
+        btl_deregister_mem; /**< memory deregistration function (NULL if not needed) */
 
     /** the mpool associated with this btl (optional) */
-    mca_mpool_base_module_t*             btl_mpool;
+    mca_mpool_base_module_t *btl_mpool;
     /** register a default error handler */
     mca_btl_base_module_register_error_fn_t btl_register_error;
 #if OPAL_CUDA_GDR_SUPPORT
-    size_t      btl_cuda_eager_limit;  /**< switch from eager to RDMA */
-    size_t      btl_cuda_rdma_limit;   /**< switch from RDMA to rndv pipeline */
+    size_t btl_accelerator_eager_limit; /**< switch from eager to RDMA */
+    size_t btl_accelerator_rdma_limit;  /**< switch from RDMA to rndv pipeline */
 #endif /* OPAL_CUDA_GDR_SUPPORT */
-#if OPAL_CUDA_SUPPORT
-    size_t      btl_cuda_max_send_size;   /**< set if CUDA max send_size is different from host max send size */
-#endif /* OPAL_CUDA_SUPPORT */
+    size_t btl_accelerator_max_send_size; /**< set if CUDA max send_size is different from host max send
+                                      size */
 
     mca_btl_base_module_flush_fn_t btl_flush; /**< flush all previous operations on an endpoint */
 
-    unsigned char padding[256]; /**< padding to future-proof the btl module */
+
+    union {
+        struct {
+            void *btl_am_data;
+        };
+        unsigned char padding[256]; /**< padding to future-proof the
+                                       btl module */
+    };
 };
 typedef struct mca_btl_base_module_t mca_btl_base_module_t;
 
@@ -1283,17 +1255,14 @@ typedef struct mca_btl_base_module_t mca_btl_base_module_t;
 /*
  * Macro for use in modules that are of type btl v3.2.0
  */
-#define MCA_BTL_BASE_VERSION_3_3_0                          \
-    OPAL_MCA_BASE_VERSION_2_1_0("btl",                      \
-                                MCA_BTL_BASE_MAJOR_VERSION, \
-                                MCA_BTL_BASE_MINOR_VERSION, \
+#define MCA_BTL_BASE_VERSION_3_3_0                                                             \
+    OPAL_MCA_BASE_VERSION_2_1_0("btl", MCA_BTL_BASE_MAJOR_VERSION, MCA_BTL_BASE_MINOR_VERSION, \
                                 MCA_BTL_BASE_PATCH_VERSION)
 
-#define MCA_BTL_DEFAULT_VERSION(name)                       \
-    MCA_BTL_BASE_VERSION_3_3_0,                             \
-    .mca_component_name = name,                             \
-    MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION, \
-                          OPAL_RELEASE_VERSION)
+#define MCA_BTL_DEFAULT_VERSION(name)                                                \
+    MCA_BTL_BASE_VERSION_3_3_0, .mca_component_name = name,                          \
+                                MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, \
+                                                      OPAL_MINOR_VERSION, OPAL_RELEASE_VERSION)
 
 /**
  * Convenience macro for detecting the BTL interface version.
diff --git a/opal/mca/btl/ofi/Makefile.am b/opal/mca/btl/ofi/Makefile.am
index dd0c7caae90..271a93bed54 100644
--- a/opal/mca/btl/ofi/Makefile.am
+++ b/opal/mca/btl/ofi/Makefile.am
@@ -15,6 +15,8 @@
 # Copyright (c) 2018      Intel, inc. All rights reserved
 # Copyright (c) 2020      Triad National Security, LLC. All rights
 #                         reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -22,9 +24,7 @@
 # $HEADER$
 #
 
-#dist_opaldata_DATA = help-mpi-btl-ofi.txt
-
-AM_CPPFLAGS = $(opal_ofi_CPPFLAGS)
+AM_CPPFLAGS = $(btl_ofi_CPPFLAGS)
 sources = \
     btl_ofi.h \
     btl_ofi_component.c \
@@ -58,12 +58,12 @@ mcacomponentdir = $(opallibdir)
 mcacomponent_LTLIBRARIES = $(component)
 mca_btl_ofi_la_SOURCES = $(component_sources)
 mca_btl_ofi_la_LDFLAGS = -module -avoid-version \
-                         $(opal_ofi_LDFLAGS)
-mca_btl_ofi_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
-    $(OPAL_TOP_BUILDDIR)/opal/mca/common/ofi/lib@OPAL_LIB_PREFIX@mca_common_ofi.la \
-    $(opal_ofi_LIBS)
+                         $(btl_ofi_LDFLAGS)
+mca_btl_ofi_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
+    $(OPAL_TOP_BUILDDIR)/opal/mca/common/ofi/lib@OPAL_LIB_NAME@mca_common_ofi.la \
+    $(btl_ofi_LIBS)
 
 noinst_LTLIBRARIES = $(lib)
 libmca_btl_ofi_la_SOURCES = $(lib_sources)
-libmca_btl_ofi_la_LDFLAGS = -module -avoid-version $(opal_ofi_LDFLAGS)
-libmca_btl_ofi_la_LIBS = $(opal_ofi_LIBS)
+libmca_btl_ofi_la_LDFLAGS = -module -avoid-version $(btl_ofi_LDFLAGS)
+libmca_btl_ofi_la_LIBS = $(btl_ofi_LIBS)
diff --git a/opal/mca/btl/ofi/README.md b/opal/mca/btl/ofi/README.md
index bab4ef1c1bf..7c82f578e1a 100644
--- a/opal/mca/btl/ofi/README.md
+++ b/opal/mca/btl/ofi/README.md
@@ -5,7 +5,7 @@ enable RDMA with multiple vendor hardware through one interface. Most
 of the operations are managed by upper layer (osc/rdma). This BTL is
 mostly doing the low level work.
 
-Tested providers: sockets,psm2,ugni
+Tested providers: sockets,psm2
 
 ## Component
 
@@ -102,7 +102,7 @@ advised that the number of context should be equal to number of
 physical core for optimal performance.
 
 User can disable scalable endpoint by MCA parameter
-`btl_ofi_disable_sep`.  With scalable endpoint disbled, the BTL will
+`btl_ofi_disable_sep`.  With scalable endpoint disabled, the BTL will
 alias OFI endpoint to both tx and rx context.
 
 ## Two sided communication
diff --git a/opal/mca/btl/ofi/btl_ofi.h b/opal/mca/btl/ofi/btl_ofi.h
index 61e654c6772..0019065ecfe 100644
--- a/opal/mca/btl/ofi/btl_ofi.h
+++ b/opal/mca/btl/ofi/btl_ofi.h
@@ -15,6 +15,8 @@
  * Copyright (c) 2018-2019 Intel, Inc.  All rights reserved.
  * Copyright (c) 2020      Amazon.com, Inc. or its affiliates.
  *                         All Rights reserved.
+ * Copyright (c) 2022      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -28,38 +30,38 @@
 #define MCA_BTL_OFI_H
 
 #include "opal_config.h"
-#include <sys/types.h>
 #include <string.h>
+#include <sys/types.h>
 
 /* Open MPI includes */
-#include "opal/util/event.h"
-#include "opal/mca/btl/btl.h"
 #include "opal/mca/btl/base/base.h"
-#include "opal/mca/mpool/mpool.h"
 #include "opal/mca/btl/base/btl_base_error.h"
-#include "opal/mca/rcache/base/base.h"
+#include "opal/mca/btl/btl.h"
+#include "opal/mca/mpool/mpool.h"
 #include "opal/mca/pmix/pmix-internal.h"
+#include "opal/mca/rcache/base/base.h"
+#include "opal/util/event.h"
 
 #include "opal/class/opal_hash_table.h"
 
 #include <rdma/fabric.h>
-#include <rdma/fi_domain.h>
-#include <rdma/fi_errno.h>
 #include <rdma/fi_cm.h>
+#include <rdma/fi_domain.h>
 #include <rdma/fi_endpoint.h>
+#include <rdma/fi_errno.h>
 #include <rdma/fi_rma.h>
 
 BEGIN_C_DECLS
-#define MCA_BTL_OFI_MAX_MODULES         16
-#define MCA_BTL_OFI_NUM_CQE_READ        64
+#define MCA_BTL_OFI_MAX_MODULES  16
+#define MCA_BTL_OFI_NUM_CQE_READ 64
 
-#define MCA_BTL_OFI_DEFAULT_RD_NUM              10
-#define MCA_BTL_OFI_DEFAULT_MAX_CQE             128
-#define MCA_BTL_OFI_DEFAULT_PROGRESS_THRESHOLD  64
+#define MCA_BTL_OFI_DEFAULT_RD_NUM             10
+#define MCA_BTL_OFI_DEFAULT_MAX_CQE            128
+#define MCA_BTL_OFI_DEFAULT_PROGRESS_THRESHOLD 64
 
-#define MCA_BTL_OFI_ABORT(args)     mca_btl_ofi_exit(args)
+#define MCA_BTL_OFI_ABORT(args) mca_btl_ofi_exit(args)
 
-#define TWO_SIDED_ENABLED           mca_btl_ofi_component.two_sided_enabled
+#define TWO_SIDED_ENABLED mca_btl_ofi_component.two_sided_enabled
 
 enum mca_btl_ofi_mode {
     MCA_BTL_OFI_MODE_ONE_SIDED = 0,
@@ -124,6 +126,7 @@ struct mca_btl_ofi_module_t {
     bool initialized;
     bool use_virt_addr;
     bool is_scalable_ep;
+    bool use_fi_mr_bind;
 
     opal_atomic_int64_t outstanding_rdma;
     opal_atomic_int64_t outstanding_send;
@@ -137,6 +140,9 @@ struct mca_btl_ofi_module_t {
 
     /** registration cache */
     mca_rcache_base_module_t *rcache;
+    /* If the underlying OFI provider has its own cache, we want to bypass
+     * rcache registration */
+    bool bypass_cache;
 };
 typedef struct mca_btl_ofi_module_t mca_btl_ofi_module_t;
 
@@ -146,7 +152,7 @@ extern mca_btl_ofi_module_t mca_btl_ofi_module_template;
  * @brief OFI BTL component
  */
 struct mca_btl_ofi_component_t {
-    mca_btl_base_component_3_0_0_t super;  /**< base BTL component */
+    mca_btl_base_component_3_0_0_t super; /**< base BTL component */
 
     /** number of TL modules */
     int module_count;
@@ -159,13 +165,18 @@ struct mca_btl_ofi_component_t {
 
     size_t namelen;
 
+    /** Maximum inject size */
+    size_t max_inject_size;
+    bool disable_inject;
+
+    bool disable_hmem;
+
     /** All BTL OFI modules (1 per tl) */
     mca_btl_ofi_module_t *modules[MCA_BTL_OFI_MAX_MODULES];
-
 };
 typedef struct mca_btl_ofi_component_t mca_btl_ofi_component_t;
 
-OPAL_MODULE_DECLSPEC extern mca_btl_ofi_component_t mca_btl_ofi_component;
+OPAL_DECLSPEC extern mca_btl_ofi_component_t mca_btl_ofi_component;
 
 struct mca_btl_base_registration_handle_t {
     uint64_t rkey;
@@ -205,9 +216,8 @@ typedef struct mca_btl_ofi_base_frag_t mca_btl_ofi_base_frag_t;
 
 OBJ_CLASS_DECLARATION(mca_btl_ofi_base_frag_t);
 
-
 struct mca_btl_ofi_completion_context_t {
-    struct fi_context ctx;
+    struct fi_context2 ctx;
     void *comp;
 };
 
@@ -283,11 +293,12 @@ OBJ_CLASS_DECLARATION(mca_btl_ofi_frag_completion_t);
  * @retval OPAL_ERR_NOT_AVAILABLE  Put can not be performed due to size or
  *                         alignment restrictions.
  */
-int mca_btl_ofi_put (struct mca_btl_base_module_t *btl,
-    struct mca_btl_base_endpoint_t *endpoint, void *local_address,
-    uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle,
-    struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-    int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
+int mca_btl_ofi_put(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+                    void *local_address, uint64_t remote_address,
+                    struct mca_btl_base_registration_handle_t *local_handle,
+                    struct mca_btl_base_registration_handle_t *remote_handle, size_t size,
+                    int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext,
+                    void *cbdata);
 
 /**
  * Initiate an asynchronous get.
@@ -320,54 +331,59 @@ int mca_btl_ofi_put (struct mca_btl_base_module_t *btl,
  * @retval OPAL_ERR_NOT_AVAILABLE  Put can not be performed due to size or
  *                         alignment restrictions.
  */
-int mca_btl_ofi_get (struct mca_btl_base_module_t *btl,
-    struct mca_btl_base_endpoint_t *endpoint, void *local_address,
-    uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle,
-    struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-    int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
-
-int mca_btl_ofi_aop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
-                     uint64_t remote_address, mca_btl_base_registration_handle_t *remote_handle,
-                     mca_btl_base_atomic_op_t op, uint64_t operand, int flags, int order,
+int mca_btl_ofi_get(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+                    void *local_address, uint64_t remote_address,
+                    struct mca_btl_base_registration_handle_t *local_handle,
+                    struct mca_btl_base_registration_handle_t *remote_handle, size_t size,
+                    int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext,
+                    void *cbdata);
+
+int mca_btl_ofi_aop(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+                    uint64_t remote_address, mca_btl_base_registration_handle_t *remote_handle,
+                    mca_btl_base_atomic_op_t op, uint64_t operand, int flags, int order,
+                    mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
+
+int mca_btl_ofi_afop(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+                     void *local_address, uint64_t remote_address,
+                     mca_btl_base_registration_handle_t *local_handle,
+                     mca_btl_base_registration_handle_t *remote_handle, mca_btl_base_atomic_op_t op,
+                     uint64_t operand, int flags, int order,
                      mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
 
-int mca_btl_ofi_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
-                      void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                      mca_btl_base_registration_handle_t *remote_handle, mca_btl_base_atomic_op_t op,
-                      uint64_t operand, int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
-                      void *cbcontext, void *cbdata);
-
-int mca_btl_ofi_acswap (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
-                        void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                        mca_btl_base_registration_handle_t *remote_handle, uint64_t compare, uint64_t value, int flags,
-                        int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
-
+int mca_btl_ofi_acswap(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+                       void *local_address, uint64_t remote_address,
+                       mca_btl_base_registration_handle_t *local_handle,
+                       mca_btl_base_registration_handle_t *remote_handle, uint64_t compare,
+                       uint64_t value, int flags, int order,
+                       mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
 
-int mca_btl_ofi_flush (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint);
+int mca_btl_ofi_flush(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint);
 
-int mca_btl_ofi_finalize (mca_btl_base_module_t *btl);
+int mca_btl_ofi_finalize(mca_btl_base_module_t *btl);
 
-void mca_btl_ofi_rcache_init (mca_btl_ofi_module_t *module);
-int mca_btl_ofi_reg_mem (void *reg_data, void *base, size_t size,
-                         mca_rcache_base_registration_t *reg);
-int mca_btl_ofi_dereg_mem (void *reg_data, mca_rcache_base_registration_t *reg);
+void mca_btl_ofi_rcache_init(mca_btl_ofi_module_t *module);
+int mca_btl_ofi_reg_mem(void *reg_data, void *base, size_t size,
+                        mca_rcache_base_registration_t *reg);
+int mca_btl_ofi_dereg_mem(void *reg_data, mca_rcache_base_registration_t *reg);
 
 int mca_btl_ofi_context_progress(mca_btl_ofi_context_t *context);
 
-mca_btl_ofi_module_t * mca_btl_ofi_module_alloc (int mode);
+mca_btl_ofi_module_t *mca_btl_ofi_module_alloc(int mode);
 
-int mca_btl_ofi_post_recvs(mca_btl_base_module_t* module, mca_btl_ofi_context_t *context, int count);
+int mca_btl_ofi_post_recvs(mca_btl_base_module_t *module, mca_btl_ofi_context_t *context,
+                           int count);
 void mca_btl_ofi_exit(void);
 
 /* thread atomics */
-static inline bool mca_btl_ofi_context_trylock (mca_btl_ofi_context_t *context)
+static inline bool mca_btl_ofi_context_trylock(mca_btl_ofi_context_t *context)
 {
     return (context->lock || OPAL_ATOMIC_SWAP_32(&context->lock, 1));
 }
 
 static inline void mca_btl_ofi_context_lock(mca_btl_ofi_context_t *context)
 {
-    while (mca_btl_ofi_context_trylock(context));
+    while (mca_btl_ofi_context_trylock(context))
+        ;
 }
 
 static inline void mca_btl_ofi_context_unlock(mca_btl_ofi_context_t *context)
diff --git a/opal/mca/btl/ofi/btl_ofi_atomics.c b/opal/mca/btl/ofi/btl_ofi_atomics.c
index f9b2447130b..3dd129f4c42 100644
--- a/opal/mca/btl/ofi/btl_ofi_atomics.c
+++ b/opal/mca/btl/ofi/btl_ofi_atomics.c
@@ -11,8 +11,8 @@
  * $HEADER$
  */
 
-#include <rdma/fi_atomic.h>
 #include "btl_ofi_rdma.h"
+#include <rdma/fi_atomic.h>
 
 static inline int to_fi_op(mca_btl_base_atomic_op_t op)
 {
@@ -21,6 +21,23 @@ static inline int to_fi_op(mca_btl_base_atomic_op_t op)
         return FI_SUM;
     case MCA_BTL_ATOMIC_SWAP:
         return FI_ATOMIC_WRITE;
+    case MCA_BTL_ATOMIC_MAX:
+        return FI_MAX;
+    case MCA_BTL_ATOMIC_MIN:
+        return FI_MIN;
+    case MCA_BTL_ATOMIC_LAND:
+        return FI_LAND;
+    case MCA_BTL_ATOMIC_AND:
+        return FI_BAND;
+    case MCA_BTL_ATOMIC_LOR:
+        return FI_LOR;
+    case MCA_BTL_ATOMIC_OR:
+        return FI_BOR;
+    case MCA_BTL_ATOMIC_LXOR:
+        return FI_LXOR;
+    case MCA_BTL_ATOMIC_XOR:
+        return FI_BXOR;
+
     default:
         BTL_ERROR(("Unknown or unsupported atomic op."));
         MCA_BTL_OFI_ABORT();
@@ -30,21 +47,23 @@ static inline int to_fi_op(mca_btl_base_atomic_op_t op)
     }
 }
 
-int mca_btl_ofi_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
-                      void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                      mca_btl_base_registration_handle_t *remote_handle, mca_btl_base_atomic_op_t op,
-                      uint64_t operand, int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
-                      void *cbcontext, void *cbdata)
+int mca_btl_ofi_afop(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+                     void *local_address, uint64_t remote_address,
+                     mca_btl_base_registration_handle_t *local_handle,
+                     mca_btl_base_registration_handle_t *remote_handle, mca_btl_base_atomic_op_t op,
+                     uint64_t operand, int flags, int order,
+                     mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
 {
     int rc;
     int fi_datatype = FI_UINT64;
     int fi_op;
 
     mca_btl_ofi_module_t *ofi_btl = (mca_btl_ofi_module_t *) btl;
-    mca_btl_ofi_endpoint_t *btl_endpoint = (mca_btl_ofi_endpoint_t*) endpoint;
+    mca_btl_ofi_endpoint_t *btl_endpoint = (mca_btl_ofi_endpoint_t *) endpoint;
     mca_btl_ofi_rdma_completion_t *comp = NULL;
     mca_btl_ofi_context_t *ofi_context;
 
+    MCA_BTL_OFI_NUM_RDMA_INC(ofi_btl);
     ofi_context = get_ofi_context(ofi_btl);
 
     if (flags & MCA_BTL_ATOMIC_FLAG_32BIT) {
@@ -53,11 +72,8 @@ int mca_btl_ofi_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_end
 
     fi_op = to_fi_op(op);
 
-    comp = mca_btl_ofi_rdma_completion_alloc(btl, endpoint,
-                                             ofi_context,
-                                             local_address,
-                                             local_handle,
-                                             cbfunc, cbcontext, cbdata,
+    comp = mca_btl_ofi_rdma_completion_alloc(btl, endpoint, ofi_context, local_address,
+                                             local_handle, cbfunc, cbcontext, cbdata,
                                              MCA_BTL_OFI_TYPE_AFOP);
 
     /* copy the operand because it might get freed from upper layer */
@@ -65,41 +81,41 @@ int mca_btl_ofi_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_end
 
     remote_address = (remote_address - (uint64_t) remote_handle->base_addr);
 
-    rc = fi_fetch_atomic(ofi_context->tx_ctx,
-                         (void*) &comp->operand, 1, NULL,       /* operand */
-                         local_address, local_handle->desc,     /* results */
-                         btl_endpoint->peer_addr,               /* remote addr */
-                         remote_address, remote_handle->rkey,   /* remote buffer */
+    rc = fi_fetch_atomic(ofi_context->tx_ctx, (void *) &comp->operand, 1, NULL, /* operand */
+                         local_address, local_handle->desc,                     /* results */
+                         btl_endpoint->peer_addr,                               /* remote addr */
+                         remote_address, remote_handle->rkey,                   /* remote buffer */
                          fi_datatype, fi_op, &comp->comp_ctx);
 
     if (rc == -FI_EAGAIN) {
-        opal_free_list_return(comp->base.my_list, (opal_free_list_item_t*) comp);
+        MCA_BTL_OFI_NUM_RDMA_DEC(ofi_btl);
+        opal_free_list_return(comp->base.my_list, (opal_free_list_item_t *) comp);
         return OPAL_ERR_OUT_OF_RESOURCE;
     } else if (rc < 0) {
-        opal_free_list_return(comp->base.my_list, (opal_free_list_item_t*) comp);
+        MCA_BTL_OFI_NUM_RDMA_DEC(ofi_btl);
+        opal_free_list_return(comp->base.my_list, (opal_free_list_item_t *) comp);
         BTL_ERROR(("fi_fetch_atomic failed with rc=%d (%s)", rc, fi_strerror(-rc)));
         MCA_BTL_OFI_ABORT();
     }
 
-    MCA_BTL_OFI_NUM_RDMA_INC(ofi_btl);
-
     return OPAL_SUCCESS;
 }
 
-int mca_btl_ofi_aop (struct mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
-                     uint64_t remote_address, mca_btl_base_registration_handle_t *remote_handle,
-                     mca_btl_base_atomic_op_t op, uint64_t operand, int flags, int order,
-                     mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
+int mca_btl_ofi_aop(struct mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
+                    uint64_t remote_address, mca_btl_base_registration_handle_t *remote_handle,
+                    mca_btl_base_atomic_op_t op, uint64_t operand, int flags, int order,
+                    mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
 {
     int rc;
     int fi_datatype = FI_UINT64;
     int fi_op;
 
     mca_btl_ofi_module_t *ofi_btl = (mca_btl_ofi_module_t *) btl;
-    mca_btl_ofi_endpoint_t *btl_endpoint = (mca_btl_ofi_endpoint_t*) endpoint;
+    mca_btl_ofi_endpoint_t *btl_endpoint = (mca_btl_ofi_endpoint_t *) endpoint;
     mca_btl_ofi_rdma_completion_t *comp = NULL;
     mca_btl_ofi_context_t *ofi_context;
 
+    MCA_BTL_OFI_NUM_RDMA_INC(ofi_btl);
     ofi_context = get_ofi_context(ofi_btl);
 
     if (flags & MCA_BTL_ATOMIC_FLAG_32BIT) {
@@ -108,42 +124,39 @@ int mca_btl_ofi_aop (struct mca_btl_base_module_t *btl, mca_btl_base_endpoint_t
 
     fi_op = to_fi_op(op);
 
-    comp = mca_btl_ofi_rdma_completion_alloc(btl, endpoint,
-                                             ofi_context,
-                                             NULL,
-                                             NULL,
-                                             cbfunc, cbcontext, cbdata,
-                                             MCA_BTL_OFI_TYPE_AOP);
+    comp = mca_btl_ofi_rdma_completion_alloc(btl, endpoint, ofi_context, NULL, NULL, cbfunc,
+                                             cbcontext, cbdata, MCA_BTL_OFI_TYPE_AOP);
 
     /* copy the operand because it might get freed from upper layer */
     comp->operand = (uint64_t) operand;
 
     remote_address = (remote_address - (uint64_t) remote_handle->base_addr);
 
-    rc = fi_atomic(ofi_context->tx_ctx,
-                   (void*) &comp->operand, 1, NULL,       /* operand */
-                   btl_endpoint->peer_addr,               /* remote addr */
-                   remote_address, remote_handle->rkey,   /* remote buffer */
+    rc = fi_atomic(ofi_context->tx_ctx, (void *) &comp->operand, 1, NULL, /* operand */
+                   btl_endpoint->peer_addr,                               /* remote addr */
+                   remote_address, remote_handle->rkey,                   /* remote buffer */
                    fi_datatype, fi_op, &comp->comp_ctx);
 
     if (rc == -FI_EAGAIN) {
-        opal_free_list_return(comp->base.my_list, (opal_free_list_item_t*) comp);
+        MCA_BTL_OFI_NUM_RDMA_DEC(ofi_btl);
+        opal_free_list_return(comp->base.my_list, (opal_free_list_item_t *) comp);
         return OPAL_ERR_OUT_OF_RESOURCE;
     } else if (rc < 0) {
-        opal_free_list_return(comp->base.my_list, (opal_free_list_item_t*) comp);
+        MCA_BTL_OFI_NUM_RDMA_DEC(ofi_btl);
+        opal_free_list_return(comp->base.my_list, (opal_free_list_item_t *) comp);
         BTL_ERROR(("fi_atomic failed with rc=%d (%s)", rc, fi_strerror(-rc)));
         MCA_BTL_OFI_ABORT();
     }
 
-    MCA_BTL_OFI_NUM_RDMA_INC(ofi_btl);
-
     return OPAL_SUCCESS;
 }
 
-int mca_btl_ofi_acswap (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
-                        void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                        mca_btl_base_registration_handle_t *remote_handle, uint64_t compare, uint64_t value, int flags,
-                        int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
+int mca_btl_ofi_acswap(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+                       void *local_address, uint64_t remote_address,
+                       mca_btl_base_registration_handle_t *local_handle,
+                       mca_btl_base_registration_handle_t *remote_handle, uint64_t compare,
+                       uint64_t value, int flags, int order,
+                       mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
 {
     int rc;
     int fi_datatype = FI_UINT64;
@@ -151,20 +164,18 @@ int mca_btl_ofi_acswap (struct mca_btl_base_module_t *btl, struct mca_btl_base_e
     mca_btl_ofi_rdma_completion_t *comp = NULL;
 
     mca_btl_ofi_module_t *ofi_btl = (mca_btl_ofi_module_t *) btl;
-    mca_btl_ofi_endpoint_t *btl_endpoint = (mca_btl_ofi_endpoint_t*) endpoint;
+    mca_btl_ofi_endpoint_t *btl_endpoint = (mca_btl_ofi_endpoint_t *) endpoint;
     mca_btl_ofi_context_t *ofi_context;
 
+    MCA_BTL_OFI_NUM_RDMA_INC(ofi_btl);
     ofi_context = get_ofi_context(ofi_btl);
 
     if (flags & MCA_BTL_ATOMIC_FLAG_32BIT) {
         fi_datatype = FI_UINT32;
     }
 
-    comp = mca_btl_ofi_rdma_completion_alloc(btl, endpoint,
-                                             ofi_context,
-                                             local_address,
-                                             local_handle,
-                                             cbfunc, cbcontext, cbdata,
+    comp = mca_btl_ofi_rdma_completion_alloc(btl, endpoint, ofi_context, local_address,
+                                             local_handle, cbfunc, cbcontext, cbdata,
                                              MCA_BTL_OFI_TYPE_CSWAP);
 
     /* copy the operand because it might get freed from upper layer */
@@ -174,26 +185,21 @@ int mca_btl_ofi_acswap (struct mca_btl_base_module_t *btl, struct mca_btl_base_e
     remote_address = (remote_address - (uint64_t) remote_handle->base_addr);
 
     /* perform atomic */
-    rc = fi_compare_atomic(ofi_context->tx_ctx,
-                           (void*) &comp->operand, 1, NULL,
-                           (void*) &comp->compare, NULL,
-                           local_address, local_handle->desc,
-                           btl_endpoint->peer_addr,
-                           remote_address, remote_handle->rkey,
-                           fi_datatype,
-                           FI_CSWAP,
-                           &comp->comp_ctx);
+    rc = fi_compare_atomic(ofi_context->tx_ctx, (void *) &comp->operand, 1, NULL,
+                           (void *) &comp->compare, NULL, local_address, local_handle->desc,
+                           btl_endpoint->peer_addr, remote_address, remote_handle->rkey,
+                           fi_datatype, FI_CSWAP, &comp->comp_ctx);
 
     if (rc == -FI_EAGAIN) {
-        opal_free_list_return(comp->base.my_list, (opal_free_list_item_t*) comp);
+        MCA_BTL_OFI_NUM_RDMA_DEC(ofi_btl);
+        opal_free_list_return(comp->base.my_list, (opal_free_list_item_t *) comp);
         return OPAL_ERR_OUT_OF_RESOURCE;
     } else if (rc < 0) {
-        opal_free_list_return(comp->base.my_list, (opal_free_list_item_t*) comp);
+        MCA_BTL_OFI_NUM_RDMA_DEC(ofi_btl);
+        opal_free_list_return(comp->base.my_list, (opal_free_list_item_t *) comp);
         BTL_ERROR(("fi_compare_atomic failed with rc=%d (%s)", rc, fi_strerror(-rc)));
         MCA_BTL_OFI_ABORT();
     }
 
-    MCA_BTL_OFI_NUM_RDMA_INC(ofi_btl);
-
     return OPAL_SUCCESS;
 }
diff --git a/opal/mca/btl/ofi/btl_ofi_component.c b/opal/mca/btl/ofi/btl_ofi_component.c
index 326d4fe69c2..3c3ff6da4e1 100644
--- a/opal/mca/btl/ofi/btl_ofi_component.c
+++ b/opal/mca/btl/ofi/btl_ofi_component.c
@@ -14,8 +14,8 @@
  *                         reserved.
  * Copyright (c) 2018-2019 Intel, Inc.  All rights reserved.
  *
- * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
- * Copyright (c) 2020      Triad National Security, LLC. All rights
+ * Copyright (c) 2018-2021 Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2020-2023 Triad National Security, LLC. All rights
  *                         reserved.
  * $COPYRIGHT$
  *
@@ -24,28 +24,27 @@
  * $HEADER$
  */
 
-
 #include "opal_config.h"
 
-#include "opal/util/printf.h"
 #include "opal/util/argv.h"
+#include "opal/util/printf.h"
 
-#include "opal/mca/btl/btl.h"
 #include "opal/mca/btl/base/base.h"
-#include "opal/mca/hwloc/base/base.h"
+#include "opal/mca/btl/btl.h"
 #include "opal/mca/common/ofi/common_ofi.h"
+#include "opal/mca/hwloc/base/base.h"
 
 #include <string.h>
 
 #include "btl_ofi.h"
 #include "btl_ofi_endpoint.h"
-#include "btl_ofi_rdma.h"
 #include "btl_ofi_frag.h"
+#include "btl_ofi_rdma.h"
 
-#define MCA_BTL_OFI_ONE_SIDED_REQUIRED_CAPS       (FI_RMA | FI_ATOMIC)
-#define MCA_BTL_OFI_TWO_SIDED_REQUIRED_CAPS       (FI_MSG)
+#define MCA_BTL_OFI_ONE_SIDED_REQUIRED_CAPS (FI_RMA | FI_ATOMIC)
+#define MCA_BTL_OFI_TWO_SIDED_REQUIRED_CAPS (FI_MSG)
 
-#define MCA_BTL_OFI_REQUESTED_MR_MODE   (FI_MR_ALLOCATED | FI_MR_PROV_KEY | FI_MR_VIRT_ADDR)
+#define MCA_BTL_OFI_REQUESTED_MR_MODE (FI_MR_ALLOCATED | FI_MR_PROV_KEY | FI_MR_VIRT_ADDR | FI_MR_ENDPOINT)
 
 static char *ofi_progress_mode;
 static bool disable_sep;
@@ -53,22 +52,21 @@ static int mca_btl_ofi_init_device(struct fi_info *info);
 
 /* validate information returned from fi_getinfo().
  * return OPAL_ERROR if we dont have what we need. */
-static int validate_info(struct fi_info *info, uint64_t required_caps,
-                         char **include_list, char **exclude_list)
+static int validate_info(struct fi_info *info, uint64_t required_caps, char **include_list,
+                         char **exclude_list)
 {
     int mr_mode;
 
-    if (NULL != include_list && !opal_common_ofi_is_in_list(include_list, info->fabric_attr->prov_name)) {
+    if (NULL != include_list
+        && !opal_common_ofi_is_in_list(include_list, info->fabric_attr->prov_name)) {
         opal_output_verbose(1, opal_common_ofi.output,
-                            "%s:%d: btl:ofi: \"%s\" not in include list\n",
-                            __FILE__, __LINE__,
+                            "%s:%d: btl:ofi: \"%s\" not in include list\n", __FILE__, __LINE__,
                             info->fabric_attr->prov_name);
         return OPAL_ERROR;
-    } else if (NULL != exclude_list && opal_common_ofi_is_in_list(exclude_list, info->fabric_attr->prov_name)) {
-        opal_output_verbose(1, opal_common_ofi.output,
-                            "%s:%d: btl:ofi: \"%s\" in exclude list\n",
-                            __FILE__, __LINE__,
-                            info->fabric_attr->prov_name);
+    } else if (NULL != exclude_list
+               && opal_common_ofi_is_in_list(exclude_list, info->fabric_attr->prov_name)) {
+        opal_output_verbose(1, opal_common_ofi.output, "%s:%d: btl:ofi: \"%s\" in exclude list\n",
+                            __FILE__, __LINE__, info->fabric_attr->prov_name);
         return OPAL_ERROR;
     }
 
@@ -80,7 +78,7 @@ static int validate_info(struct fi_info *info, uint64_t required_caps,
      * Thus, FI_VERSION(112,0) corresponds to libfabric 1.12.0
      */
     if (!strncasecmp(info->fabric_attr->prov_name, "efa", 3)
-        && FI_VERSION_LT(info->fabric_attr->prov_version, FI_VERSION(112,0))) {
+        && FI_VERSION_LT(info->fabric_attr->prov_version, FI_VERSION(112, 0))) {
         BTL_VERBOSE(("unsupported libfabric efa version"));
         return OPAL_ERROR;
     }
@@ -107,8 +105,12 @@ static int validate_info(struct fi_info *info, uint64_t required_caps,
 
     mr_mode = info->domain_attr->mr_mode;
 
-    if (!(mr_mode == FI_MR_BASIC || mr_mode == FI_MR_SCALABLE ||
-         (mr_mode & ~(FI_MR_VIRT_ADDR | FI_MR_ALLOCATED | FI_MR_PROV_KEY)) == 0)) {
+    if (!(mr_mode == FI_MR_BASIC || mr_mode == FI_MR_SCALABLE
+#if defined(FI_MR_HMEM)
+          || (mr_mode & ~(FI_MR_VIRT_ADDR | FI_MR_ALLOCATED | FI_MR_PROV_KEY | FI_MR_ENDPOINT | FI_MR_HMEM)) == 0)) {
+#else
+          || (mr_mode & ~(FI_MR_VIRT_ADDR | FI_MR_ALLOCATED | FI_MR_PROV_KEY | FI_MR_ENDPOINT)) == 0)) {
+#endif
         BTL_VERBOSE(("unsupported MR mode"));
         return OPAL_ERROR;
     }
@@ -125,101 +127,104 @@ static int validate_info(struct fi_info *info, uint64_t required_caps,
 /* Register the MCA parameters */
 static int mca_btl_ofi_component_register(void)
 {
+    int ret;
     char *msg;
     mca_btl_ofi_module_t *module = &mca_btl_ofi_module_template;
 
-    opal_asprintf(&msg, "BTL OFI mode of operation. Valid values are: %d = One-Sided only, %d=Two-Sided only, "
-                   "%d = Both one and two sided. BTL OFI is only optimized for one-sided communication",
-                   MCA_BTL_OFI_MODE_ONE_SIDED,
-                   MCA_BTL_OFI_MODE_TWO_SIDED,
-                   MCA_BTL_OFI_MODE_FULL_SUPPORT);
+    opal_asprintf(
+        &msg,
+        "BTL OFI mode of operation. Valid values are: %d = One-Sided only, %d=Two-Sided only, "
+        "%d = Both one and two sided. BTL OFI is only optimized for one-sided communication",
+        MCA_BTL_OFI_MODE_ONE_SIDED, MCA_BTL_OFI_MODE_TWO_SIDED, MCA_BTL_OFI_MODE_FULL_SUPPORT);
     if (NULL == msg) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
     mca_btl_ofi_component.mode = MCA_BTL_OFI_MODE_ONE_SIDED;
-    (void)mca_base_component_var_register(&mca_btl_ofi_component.super.btl_version,
-                                          "mode",
-                                          msg,
-                                          MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                          OPAL_INFO_LVL_5,
-                                          MCA_BASE_VAR_SCOPE_READONLY,
-                                          &mca_btl_ofi_component.mode);
-
+    (void) mca_base_component_var_register(&mca_btl_ofi_component.super.btl_version, "mode", msg,
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           &mca_btl_ofi_component.mode);
 
     mca_btl_ofi_component.num_cqe_read = MCA_BTL_OFI_NUM_CQE_READ;
-    (void) mca_base_component_var_register(&mca_btl_ofi_component.super.btl_version,
-                                          "num_cq_read",
-                                          "Number of completion entries to read from a single cq_read. ",
-                                          MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                          OPAL_INFO_LVL_5,
-                                          MCA_BASE_VAR_SCOPE_READONLY,
-                                          &mca_btl_ofi_component.num_cqe_read);
+    (void) mca_base_component_var_register(
+        &mca_btl_ofi_component.super.btl_version, "num_cq_read",
+        "Number of completion entries to read from a single cq_read. ", MCA_BASE_VAR_TYPE_INT, NULL,
+        0, 0, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_READONLY, &mca_btl_ofi_component.num_cqe_read);
 
     ofi_progress_mode = "unspec";
-    (void) mca_base_component_var_register(&mca_btl_ofi_component.super.btl_version,
-                                          "progress_mode",
-                                          "requested provider progress mode. [unspec, auto, manual]"
-                                          "(default: unspec)",
-                                          MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
-                                          OPAL_INFO_LVL_5,
-                                          MCA_BASE_VAR_SCOPE_READONLY,
-                                          &ofi_progress_mode);
+    (void)
+        mca_base_component_var_register(&mca_btl_ofi_component.super.btl_version, "progress_mode",
+                                        "requested provider progress mode. [unspec, auto, manual]"
+                                        "(default: unspec)",
+                                        MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                        MCA_BASE_VAR_SCOPE_READONLY, &ofi_progress_mode);
 
     mca_btl_ofi_component.num_contexts_per_module = 1;
-    (void) mca_base_component_var_register(&mca_btl_ofi_component.super.btl_version,
-                                          "num_contexts_per_module",
-                                          "number of communication context per module to create. "
-                                          "This should increase multithreaded performance but it is "
-                                          "advised that this number should be lower than total cores.",
-                                          MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                          OPAL_INFO_LVL_5,
-                                          MCA_BASE_VAR_SCOPE_READONLY,
-                                          &mca_btl_ofi_component.num_contexts_per_module);
+    (void) mca_base_component_var_register(
+        &mca_btl_ofi_component.super.btl_version, "num_contexts_per_module",
+        "number of communication context per module to create. "
+        "This should increase multithreaded performance but it is "
+        "advised that this number should be lower than total cores.",
+        MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_READONLY,
+        &mca_btl_ofi_component.num_contexts_per_module);
 
     disable_sep = false;
-    (void) mca_base_component_var_register(&mca_btl_ofi_component.super.btl_version,
-                                          "disable_sep",
-                                          "force btl/ofi to never use scalable endpoint.",
-                                          MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
-                                          OPAL_INFO_LVL_5,
-                                          MCA_BASE_VAR_SCOPE_READONLY,
-                                          &disable_sep);
+    (void) mca_base_component_var_register(&mca_btl_ofi_component.super.btl_version, "disable_sep",
+                                           "force btl/ofi to never use scalable endpoint.",
+                                           MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                           MCA_BASE_VAR_SCOPE_READONLY, &disable_sep);
 
     mca_btl_ofi_component.progress_threshold = MCA_BTL_OFI_DEFAULT_PROGRESS_THRESHOLD;
-    (void) mca_base_component_var_register(&mca_btl_ofi_component.super.btl_version,
-                                          "progress_threshold",
-                                          "number of outstanding operation before btl will progress "
-                                          "automatically. Tuning this might improve performance on "
-                                          "certain type of application.",
-                                          MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                          OPAL_INFO_LVL_5,
-                                          MCA_BASE_VAR_SCOPE_READONLY,
-                                          &mca_btl_ofi_component.progress_threshold);
+    (void)
+        mca_base_component_var_register(&mca_btl_ofi_component.super.btl_version,
+                                        "progress_threshold",
+                                        "number of outstanding operation before btl will progress "
+                                        "automatically. Tuning this might improve performance on "
+                                        "certain type of application.",
+                                        MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                        MCA_BASE_VAR_SCOPE_READONLY,
+                                        &mca_btl_ofi_component.progress_threshold);
 
     mca_btl_ofi_component.rd_num = MCA_BTL_OFI_DEFAULT_RD_NUM;
-    (void) mca_base_component_var_register(&mca_btl_ofi_component.super.btl_version,
-                                          "rd_num",
-                                          "Number of receive descriptor posted per context.",
-                                          MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                          OPAL_INFO_LVL_5,
-                                          MCA_BASE_VAR_SCOPE_READONLY,
-                                          &mca_btl_ofi_component.rd_num);
+    (void) mca_base_component_var_register(&mca_btl_ofi_component.super.btl_version, "rd_num",
+                                           "Number of receive descriptor posted per context.",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           &mca_btl_ofi_component.rd_num);
+
+    mca_btl_ofi_component.disable_inject = false;
+    (void) mca_base_component_var_register(&mca_btl_ofi_component.super.btl_version, "disable_inject",
+                                           "disable use of fi_inject for short messages.",
+                                           MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           &mca_btl_ofi_component.disable_inject);
+
+    mca_btl_ofi_component.disable_hmem = false;
+    mca_base_component_var_register(&mca_btl_ofi_component.super.btl_version,
+                                    "disable_hmem",
+                                    "Disable HMEM usage",
+                                    MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
+                                    OPAL_INFO_LVL_5,
+                                    MCA_BASE_VAR_SCOPE_READONLY,
+                                    &mca_btl_ofi_component.disable_hmem);
 
 
     /* for now we want this component to lose to the MTL. */
     module->super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_HIGH - 50;
 
-    opal_common_ofi_register_mca_variables(&mca_btl_ofi_component.super.btl_version);
+    ret = opal_common_ofi_mca_register(&mca_btl_ofi_component.super.btl_version);
+    if (OPAL_SUCCESS != ret) {
+        return ret;
+    }
 
-    return mca_btl_base_param_register (&mca_btl_ofi_component.super.btl_version,
-                                        &module->super);
+    return mca_btl_base_param_register(&mca_btl_ofi_component.super.btl_version, &module->super);
 }
 
 static int mca_btl_ofi_component_open(void)
 {
     mca_btl_ofi_component.module_count = 0;
-    return OPAL_SUCCESS;
+    return opal_common_ofi_open();
 }
 
 /*
@@ -227,10 +232,11 @@ static int mca_btl_ofi_component_open(void)
  */
 static int mca_btl_ofi_component_close(void)
 {
-    opal_common_ofi_mca_deregister();
+    int ret;
+    ret = opal_common_ofi_close();
     /* If we don't sleep, sockets provider freaks out. Ummm this is a scary comment */
     sleep(1);
-    return OPAL_SUCCESS;
+    return ret;
 }
 
 void mca_btl_ofi_exit(void)
@@ -245,14 +251,16 @@ void mca_btl_ofi_exit(void)
  *   then create a BTL instance for selected interfaces
  */
 
-static mca_btl_base_module_t **mca_btl_ofi_component_init (int *num_btl_modules, bool enable_progress_threads,
-                                                           bool enable_mpi_threads)
+static mca_btl_base_module_t **mca_btl_ofi_component_init(int *num_btl_modules,
+                                                          bool enable_progress_threads,
+                                                          bool enable_mpi_threads)
 {
-    /* for this BTL to be useful the interface needs to support RDMA and certain atomic operations */
+    /* for this BTL to be useful the interface needs to support RDMA and certain atomic operations
+     */
     int rc;
     uint64_t progress_mode;
     unsigned resource_count = 0;
-    struct mca_btl_base_module_t **base_modules;
+    struct mca_btl_base_module_t **base_modules = NULL;
     char **include_list = NULL, **exclude_list = NULL;
 
     BTL_VERBOSE(("initializing ofi btl"));
@@ -262,12 +270,12 @@ static mca_btl_base_module_t **mca_btl_ofi_component_init (int *num_btl_modules,
     libfabric_api = fi_version();
 
     /* bail if OFI version is less than 1.5. */
-    if (libfabric_api < FI_VERSION(1, 5)) {
-        BTL_VERBOSE(("ofi btl disqualified because OFI version < 1.5."));
+    if (libfabric_api < FI_VERSION(1, 9)) {
+        BTL_VERBOSE(("ofi btl disqualified because OFI version < 1.9."));
         return NULL;
     }
 
-    struct fi_info *info, *info_list, *selected_info;
+    struct fi_info *info, *info_list = NULL, *selected_info = NULL;
     struct fi_info hints = {0};
     struct fi_ep_attr ep_attr = {0};
     struct fi_rx_attr rx_attr = {0};
@@ -276,34 +284,29 @@ static mca_btl_base_module_t **mca_btl_ofi_component_init (int *num_btl_modules,
     struct fi_domain_attr domain_attr = {0};
     uint64_t required_caps;
 
-    opal_common_ofi_mca_register();
-
     switch (mca_btl_ofi_component.mode) {
 
-        case MCA_BTL_OFI_MODE_TWO_SIDED:
-            mca_btl_ofi_component.two_sided_enabled = true;
-            required_caps = MCA_BTL_OFI_TWO_SIDED_REQUIRED_CAPS;
-            break;
+    case MCA_BTL_OFI_MODE_TWO_SIDED:
+        mca_btl_ofi_component.two_sided_enabled = true;
+        required_caps = MCA_BTL_OFI_TWO_SIDED_REQUIRED_CAPS;
+        break;
 
-        case MCA_BTL_OFI_MODE_FULL_SUPPORT:
-            mca_btl_ofi_component.two_sided_enabled = true;
-            required_caps = MCA_BTL_OFI_ONE_SIDED_REQUIRED_CAPS |
-                            MCA_BTL_OFI_TWO_SIDED_REQUIRED_CAPS;
-            break;
+    case MCA_BTL_OFI_MODE_FULL_SUPPORT:
+        mca_btl_ofi_component.two_sided_enabled = true;
+        required_caps = MCA_BTL_OFI_ONE_SIDED_REQUIRED_CAPS | MCA_BTL_OFI_TWO_SIDED_REQUIRED_CAPS;
+        break;
 
-        default:
-            /* default to only one sided. */
-            required_caps = MCA_BTL_OFI_ONE_SIDED_REQUIRED_CAPS;
-            break;
+    default:
+        /* default to only one sided. */
+        required_caps = MCA_BTL_OFI_ONE_SIDED_REQUIRED_CAPS;
+        break;
     }
 
     fabric_attr.prov_name = NULL;
 
-    opal_output_verbose(1, opal_common_ofi.output,
-                        "%s:%d: btl:ofi:provider_include = \"%s\"\n",
+    opal_output_verbose(1, opal_common_ofi.output, "%s:%d: btl:ofi:provider_include = \"%s\"\n",
                         __FILE__, __LINE__, *opal_common_ofi.prov_include);
-    opal_output_verbose(1, opal_common_ofi.output,
-                        "%s:%d: btl:ofi:provider_exclude = \"%s\"\n",
+    opal_output_verbose(1, opal_common_ofi.output, "%s:%d: btl:ofi:provider_exclude = \"%s\"\n",
                         __FILE__, __LINE__, *opal_common_ofi.prov_exclude);
 
     if (NULL != *opal_common_ofi.prov_include) {
@@ -332,10 +335,9 @@ static mca_btl_base_module_t **mca_btl_ofi_component_init (int *num_btl_modules,
     /* ask for capabilities */
     /* TODO: catch the caps here. */
     hints.caps = required_caps;
-    hints.mode = FI_CONTEXT;
 
     /* Ask for completion context */
-    hints.mode = FI_CONTEXT;
+    hints.mode = FI_CONTEXT | FI_CONTEXT2;
 
     hints.fabric_attr = &fabric_attr;
     hints.domain_attr = &domain_attr;
@@ -351,20 +353,67 @@ static mca_btl_base_module_t **mca_btl_ofi_component_init (int *num_btl_modules,
 
     mca_btl_ofi_component.module_count = 0;
 
-    /* do the query. */
-    rc = fi_getinfo(FI_VERSION(1, 5), NULL, NULL, 0, &hints, &info_list);
-    if (0 != rc) {
-        BTL_VERBOSE(("fi_getinfo failed with code %d: %s",rc, fi_strerror(-rc)));
-        if (NULL != include_list) {
-            opal_argv_free(include_list);
+#if defined(FI_HMEM)
+    /* Request device transfer capabilities, separate from required_caps */
+    if (false == mca_btl_ofi_component.disable_hmem) {
+        hints.caps |= FI_HMEM;
+        hints.domain_attr->mr_mode |= FI_MR_HMEM;
+    }
+no_hmem:
+#endif
+
+    /* Do the query. The earliest version that supports FI_HMEM hints is 1.9.
+     * The earliest version the explictly allow provider to call CUDA API is 1.18  */
+    rc = fi_getinfo(FI_VERSION(1, 18), NULL, NULL, 0, &hints, &info_list);
+    if (FI_ENOSYS == -rc) {
+        rc = fi_getinfo(FI_VERSION(1, 9), NULL, NULL, 0, &hints, &info_list);
+    }
+
+    if ((FI_ENODATA == -rc)
+        || (0 == rc && include_list
+            && 0 == opal_common_ofi_count_providers_in_list(info_list, include_list))
+        || (0 == rc && !include_list && exclude_list
+            && opal_common_ofi_providers_subset_of_list(info_list, exclude_list))) {
+#if defined(FI_HMEM)
+        /* Attempt selecting a provider without FI_HMEM hints */
+        if (hints.caps & FI_HMEM) {
+            hints.caps &= ~FI_HMEM;
+            hints.domain_attr->mr_mode &= ~FI_MR_HMEM;
+            if (info_list) {
+                (void) fi_freeinfo(info_list);
+                info_list = NULL;
+            }
+            goto no_hmem;
         }
-        return NULL;
+#endif
+        /* It is not an error if no information is returned. */
+        goto out;
+    } else if (0 != rc) {
+        BTL_VERBOSE(("fi_getinfo failed with code %d: %s", rc, fi_strerror(-rc)));
+        goto out;
     }
 
+#if defined(FI_HMEM)
+    /* If we get to this point with FI_HMEM hint set, we want it to be a
+     * required capability
+     */
+    if (hints.caps & FI_HMEM) {
+        /* The EFA provider has a bug where it incorrectly advertises FI_HMEM +
+         * FI_ATOMIC capability without being able to provide that support in
+         * versions before libfabric 1.18.0
+         */
+        if (libfabric_api < FI_VERSION(1, 18) && !strncasecmp(info_list->fabric_attr->prov_name, "efa", 3)) {
+            hints.caps &= ~FI_HMEM;
+            hints.domain_attr->mr_mode &= ~FI_MR_HMEM;
+            goto no_hmem;
+        }
+        required_caps |= FI_HMEM;
+    }
+#endif
 
     /* count the number of resources/ */
     info = info_list;
-    while(info) {
+    while (info) {
         resource_count++;
         info = info->next;
     }
@@ -372,7 +421,7 @@ static mca_btl_base_module_t **mca_btl_ofi_component_init (int *num_btl_modules,
 
     info = info_list;
 
-    while(info) {
+    while (info) {
         rc = validate_info(info, required_caps, include_list, exclude_list);
         if (OPAL_SUCCESS == rc) {
             /* Device passed sanity check, let's make a module.
@@ -389,9 +438,9 @@ static mca_btl_base_module_t **mca_btl_ofi_component_init (int *num_btl_modules,
              * however some may have multiple info objects with different
              * attributes for the same NIC. The initial provider attributes
              * are used to ensure that all NICs we return provide the same
-             * capabilities as the inital one.
+             * capabilities as the initial one.
              */
-            selected_info = opal_mca_common_ofi_select_provider(info, &opal_process_info);
+            selected_info = opal_common_ofi_select_provider(info, &opal_process_info);
             rc = mca_btl_ofi_init_device(selected_info);
             if (OPAL_SUCCESS == rc) {
                 info = selected_info;
@@ -401,26 +450,36 @@ static mca_btl_base_module_t **mca_btl_ofi_component_init (int *num_btl_modules,
         info = info->next;
     }
 
-    /* We are done with the returned info. */
-    fi_freeinfo(info_list);
-    if (NULL != include_list) {
-        opal_argv_free(include_list);
+    if (NULL == info) {
+        BTL_VERBOSE(("No provider is selected"));
+        goto out;
     }
 
     /* pass module array back to caller */
-    base_modules = calloc (mca_btl_ofi_component.module_count, sizeof (*base_modules));
+    base_modules = calloc(mca_btl_ofi_component.module_count, sizeof(*base_modules));
     if (NULL == base_modules) {
-        return NULL;
+        goto out;
     }
 
     memcpy(base_modules, mca_btl_ofi_component.modules,
-           mca_btl_ofi_component.module_count *sizeof (mca_btl_ofi_component.modules[0]));
+           mca_btl_ofi_component.module_count * sizeof(mca_btl_ofi_component.modules[0]));
 
     BTL_VERBOSE(("ofi btl initialization complete. found %d suitable transports",
                  mca_btl_ofi_component.module_count));
 
     *num_btl_modules = mca_btl_ofi_component.module_count;
 
+out:
+    if (include_list) {
+        opal_argv_free(include_list);
+    }
+    if (exclude_list) {
+        opal_argv_free(exclude_list);
+    }
+    if (info_list) {
+        (void) fi_freeinfo(info_list);
+    }
+
     return base_modules;
 }
 
@@ -432,7 +491,7 @@ static int mca_btl_ofi_init_device(struct fi_info *info)
     size_t num_contexts_to_create;
 
     char *linux_device_name;
-    char ep_name[FI_NAME_MAX];
+    void *ep_name = NULL;
 
     struct fi_info *ofi_info;
     struct fi_ep_attr *ep_attr;
@@ -453,7 +512,7 @@ static int mca_btl_ofi_init_device(struct fi_info *info)
 
     /* If the user ask for two sided support, something bad is happening
      * to the MTL, so we will take maximum priority to supersede the MTL. */
-    module->super.btl_exclusivity    = MCA_BTL_EXCLUSIVITY_DEFAULT;
+    module->super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_DEFAULT;
 
     /* make a copy of the given info to store on the module */
     ofi_info = fi_dupinfo(info);
@@ -466,39 +525,47 @@ static int mca_btl_ofi_init_device(struct fi_info *info)
      * to prevent races. */
     mca_btl_ofi_rcache_init(module);
 
+    /* for similar reasons to the rcache call, this must be called
+     * during single threaded part of the code and before Libfabric
+     * configures its memory monitors.  Easiest to do that before
+     * domain open.  Silently ignore not-supported errors, as they
+     * are not critical to program correctness, but only indicate
+     * that LIbfabric will have to pick a different, possibly less
+     * optimal, monitor. */
+    rc = opal_common_ofi_export_memory_monitor();
+    if (0 != rc && -FI_ENOSYS != rc) {
+        BTL_VERBOSE(("Failed to inject Libfabric memory monitor: %s",
+                     fi_strerror(-rc)));
+    }
+
     linux_device_name = info->domain_attr->name;
-    BTL_VERBOSE(("initializing dev:%s provider:%s",
-                    linux_device_name,
-                    info->fabric_attr->prov_name));
+    BTL_VERBOSE(
+        ("initializing dev:%s provider:%s", linux_device_name, info->fabric_attr->prov_name));
 
     /* fabric */
     rc = fi_fabric(ofi_info->fabric_attr, &fabric, NULL);
     if (0 != rc) {
-        BTL_VERBOSE(("%s failed fi_fabric with err=%s",
-                        linux_device_name,
-                        fi_strerror(-rc)
-                        ));
+        BTL_VERBOSE(("%s failed fi_fabric with err=%s", linux_device_name, fi_strerror(-rc)));
         goto fail;
     }
 
     /* domain */
     rc = fi_domain(fabric, ofi_info, &domain, NULL);
     if (0 != rc) {
-        BTL_VERBOSE(("%s failed fi_domain with err=%s",
-                        linux_device_name,
-                        fi_strerror(-rc)
-                        ));
+        BTL_VERBOSE(("%s failed fi_domain with err=%s", linux_device_name, fi_strerror(-rc)));
         goto fail;
     }
 
+    /**
+     * Save the maximum sizes.
+     */
+    mca_btl_ofi_component.max_inject_size = ofi_info->tx_attr->inject_size;
+
     /* AV */
     av_attr.type = FI_AV_MAP;
     rc = fi_av_open(domain, &av_attr, &av, NULL);
     if (0 != rc) {
-        BTL_VERBOSE(("%s failed fi_av_open with err=%s",
-                        linux_device_name,
-                        fi_strerror(-rc)
-                        ));
+        BTL_VERBOSE(("%s failed fi_av_open with err=%s", linux_device_name, fi_strerror(-rc)));
         goto fail;
     }
 
@@ -511,10 +578,9 @@ static int mca_btl_ofi_init_device(struct fi_info *info)
 
         if (num_contexts_to_create > domain_attr->max_ep_tx_ctx) {
             BTL_VERBOSE(("cannot create requested %u contexts. (node max=%zu)",
-                            module->num_contexts,
-                            domain_attr->max_ep_tx_ctx));
+                         module->num_contexts, domain_attr->max_ep_tx_ctx));
             goto fail;
-         }
+        }
 
         /* modify the info to let the provider know we are creating x contexts */
         ep_attr->tx_ctx_cnt = num_contexts_to_create;
@@ -523,10 +589,8 @@ static int mca_btl_ofi_init_device(struct fi_info *info)
         /* create scalable endpoint */
         rc = fi_scalable_ep(domain, ofi_info, &ep, NULL);
         if (0 != rc) {
-            BTL_VERBOSE(("%s failed fi_scalable_ep with err=%s",
-                            linux_device_name,
-                            fi_strerror(-rc)
-                            ));
+            BTL_VERBOSE(
+                ("%s failed fi_scalable_ep with err=%s", linux_device_name, fi_strerror(-rc)));
             goto fail;
         }
 
@@ -534,26 +598,22 @@ static int mca_btl_ofi_init_device(struct fi_info *info)
         module->is_scalable_ep = true;
 
         /* create contexts */
-        module->contexts = mca_btl_ofi_context_alloc_scalable(ofi_info,
-                                domain, ep, av,
-                                num_contexts_to_create);
+        module->contexts = mca_btl_ofi_context_alloc_scalable(ofi_info, domain, ep, av,
+                                                              num_contexts_to_create);
 
-   } else {
+    } else {
         /* warn the user if they want more than 1 context */
         if (num_contexts_to_create > 1) {
             BTL_ERROR(("cannot create %zu contexts as the provider does not support "
-                        "scalable endpoint. Falling back to single context endpoint.",
-                        num_contexts_to_create));
+                       "scalable endpoint. Falling back to single context endpoint.",
+                       num_contexts_to_create));
         }
 
         BTL_VERBOSE(("btl/ofi using normal endpoint."));
 
         rc = fi_endpoint(domain, ofi_info, &ep, NULL);
         if (0 != rc) {
-            BTL_VERBOSE(("%s failed fi_endpoint with err=%s",
-                            linux_device_name,
-                            fi_strerror(-rc)
-                            ));
+            BTL_VERBOSE(("%s failed fi_endpoint with err=%s", linux_device_name, fi_strerror(-rc)));
             goto fail;
         }
 
@@ -561,8 +621,7 @@ static int mca_btl_ofi_init_device(struct fi_info *info)
         module->is_scalable_ep = false;
 
         /* create contexts */
-        module->contexts = mca_btl_ofi_context_alloc_normal(ofi_info,
-                                                            domain, ep, av);
+        module->contexts = mca_btl_ofi_context_alloc_normal(ofi_info, domain, ep, av);
     }
 
     if (NULL == module->contexts) {
@@ -573,10 +632,7 @@ static int mca_btl_ofi_init_device(struct fi_info *info)
     /* enable the endpoint for using */
     rc = fi_enable(ep);
     if (0 != rc) {
-        BTL_VERBOSE(("%s failed fi_enable with err=%s",
-                        linux_device_name,
-                        fi_strerror(-rc)
-                        ));
+        BTL_VERBOSE(("%s failed fi_enable with err=%s", linux_device_name, fi_strerror(-rc)));
         goto fail;
     }
 
@@ -590,42 +646,55 @@ static int mca_btl_ofi_init_device(struct fi_info *info)
     module->linux_device_name = linux_device_name;
     module->outstanding_rdma = 0;
     module->use_virt_addr = false;
+    module->use_fi_mr_bind = false;
+    module->bypass_cache = false;
+
+#if defined(FI_HMEM)
+    if (ofi_info->caps & FI_HMEM) {
+        module->super.btl_flags |= MCA_BTL_FLAGS_ACCELERATOR_RDMA;
+    }
+#endif
 
-    if (ofi_info->domain_attr->mr_mode == FI_MR_BASIC ||
-        ofi_info->domain_attr->mr_mode & FI_MR_VIRT_ADDR) {
+    if (ofi_info->domain_attr->mr_mode == FI_MR_BASIC
+        || ofi_info->domain_attr->mr_mode & FI_MR_VIRT_ADDR) {
         module->use_virt_addr = true;
     }
 
+    if (ofi_info->domain_attr->mr_mode & FI_MR_ENDPOINT) {
+        module->use_fi_mr_bind = true;
+    }
+
+    /* Currently there is no API to query whether the libfabric provider
+     * uses an underlying registration cache. For now, just check for known
+     * providers that use registration caching. */
+    if (!strncasecmp(info->fabric_attr->prov_name, "efa", 3)) {
+        module->bypass_cache = true;
+    }
+
     /* create endpoint list */
     OBJ_CONSTRUCT(&module->endpoints, opal_list_t);
     OBJ_CONSTRUCT(&module->module_lock, opal_mutex_t);
     OBJ_CONSTRUCT(&module->id_to_endpoint, opal_hash_table_t);
 
-    rc = opal_hash_table_init (&module->id_to_endpoint, 512);
+    rc = opal_hash_table_init(&module->id_to_endpoint, 512);
     if (OPAL_SUCCESS != rc) {
         BTL_ERROR(("error initializing hash table."));
         goto fail;
     }
 
-    /* create and send the modex for this device */
-    namelen = sizeof(ep_name);
-    rc = fi_getname((fid_t)ep, &ep_name[0], &namelen);
-    if (0 != rc) {
-        BTL_VERBOSE(("%s failed fi_getname with err=%s",
-                        linux_device_name,
-                        fi_strerror(-rc)
-                        ));
+    rc = opal_common_ofi_fi_getname((fid_t)ep,
+                                     &ep_name,
+                                     &namelen);
+    if (OPAL_SUCCESS != rc) {
+        BTL_VERBOSE(("%s failed opal_common_ofi_fi_getname  with err=%d", linux_device_name, rc));
         goto fail;
     }
 
-
-    /* If we have two-sided support. */
     if (TWO_SIDED_ENABLED) {
 
         /* post wildcard recvs */
-        for (int i=0; i < module->num_contexts; i++) {
-            rc = mca_btl_ofi_post_recvs((mca_btl_base_module_t*) module,
-                                        &module->contexts[i],
+        for (int i = 0; i < module->num_contexts; i++) {
+            rc = mca_btl_ofi_post_recvs((mca_btl_base_module_t *) module, &module->contexts[i],
                                         mca_btl_ofi_component.rd_num);
             if (OPAL_SUCCESS != rc) {
                 goto fail;
@@ -634,12 +703,9 @@ static int mca_btl_ofi_init_device(struct fi_info *info)
     }
 
     /* post our endpoint name so peer can use it to connect to us */
-    OPAL_MODEX_SEND(rc,
-                    PMIX_GLOBAL,
-                    &mca_btl_ofi_component.super.btl_version,
-                    &ep_name,
-                    namelen);
+    OPAL_MODEX_SEND(rc, PMIX_GLOBAL, &mca_btl_ofi_component.super.btl_version, ep_name, namelen);
     mca_btl_ofi_component.namelen = namelen;
+    free(ep_name);
 
     /* add this module to the list */
     mca_btl_ofi_component.modules[(*module_count)++] = module;
@@ -657,8 +723,10 @@ static int mca_btl_ofi_init_device(struct fi_info *info)
 
     /* if the contexts have not been initiated, num_contexts should
      * be zero and we skip this. */
-    for (int i=0; i < module->num_contexts; i++) {
-        mca_btl_ofi_context_finalize(&module->contexts[i], module->is_scalable_ep);
+    if (NULL != module->contexts) {
+        for (int i = 0; i < module->num_contexts; i++) {
+            mca_btl_ofi_context_finalize(&module->contexts[i], module->is_scalable_ep);
+        }
     }
     free(module->contexts);
 
@@ -681,6 +749,10 @@ static int mca_btl_ofi_init_device(struct fi_info *info)
     }
     free(module);
 
+    if (NULL != ep_name) {
+        free(ep_name);
+    }
+
     /* not really a failure. just skip this device. */
     return OPAL_ERR_OUT_OF_RESOURCE;
 }
@@ -688,14 +760,14 @@ static int mca_btl_ofi_init_device(struct fi_info *info)
 /**
  * @brief OFI BTL progress function
  *
- * This function explictly progresses all workers.
+ * This function explicitly progresses all workers.
  */
-static int mca_btl_ofi_component_progress (void)
+static int mca_btl_ofi_component_progress(void)
 {
     int events = 0;
     mca_btl_ofi_context_t *context;
 
-    for (int i = 0 ; i < mca_btl_ofi_component.module_count ; ++i) {
+    for (int i = 0; i < mca_btl_ofi_component.module_count; ++i) {
         mca_btl_ofi_module_t *module = mca_btl_ofi_component.modules[i];
 
         /* progress context we own first. */
@@ -708,7 +780,7 @@ static int mca_btl_ofi_component_progress (void)
 
         /* if there is nothing to do, try progress other's. */
         if (events == 0) {
-            for (int j = 0 ; j < module->num_contexts ; j++ ) {
+            for (int j = 0; j < module->num_contexts; j++) {
 
                 context = get_ofi_context_rr(module);
 
@@ -731,19 +803,20 @@ static int mca_btl_ofi_component_progress (void)
 
 /** OFI btl component */
 mca_btl_ofi_component_t mca_btl_ofi_component = {
-    .super = {
-        .btl_version = {
-            MCA_BTL_DEFAULT_VERSION("ofi"),
-            .mca_open_component = mca_btl_ofi_component_open,
-            .mca_close_component = mca_btl_ofi_component_close,
-            .mca_register_component_params = mca_btl_ofi_component_register,
-        },
-        .btl_data = {
-            /* The component is not checkpoint ready */
-            .param_field = MCA_BASE_METADATA_PARAM_NONE
+    .super =
+        {
+            .btl_version =
+                {
+                    MCA_BTL_DEFAULT_VERSION("ofi"),
+                    .mca_open_component = mca_btl_ofi_component_open,
+                    .mca_close_component = mca_btl_ofi_component_close,
+                    .mca_register_component_params = mca_btl_ofi_component_register,
+                },
+            .btl_data =
+                {/* The component is not checkpoint ready */
+                 .param_field = MCA_BASE_METADATA_PARAM_NONE},
+
+            .btl_init = mca_btl_ofi_component_init,
+            .btl_progress = mca_btl_ofi_component_progress,
         },
-
-        .btl_init = mca_btl_ofi_component_init,
-        .btl_progress = mca_btl_ofi_component_progress,
-    },
 };
diff --git a/opal/mca/btl/ofi/btl_ofi_context.c b/opal/mca/btl/ofi/btl_ofi_context.c
index bd399c62280..2b9a5fb6905 100644
--- a/opal/mca/btl/ofi/btl_ofi_context.c
+++ b/opal/mca/btl/ofi/btl_ofi_context.c
@@ -2,6 +2,8 @@
 /*
  * $COPYRIGHT$
  * Copyright (c) 2018      Intel Inc. All rights reserved
+ * Copyright (c) 2025      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -14,27 +16,16 @@
 #include "btl_ofi_rdma.h"
 
 #if OPAL_HAVE_THREAD_LOCAL
-opal_thread_local mca_btl_ofi_context_t *my_context = NULL;
+static opal_thread_local mca_btl_ofi_context_t *my_context = NULL;
 #endif /* OPAL_HAVE_THREAD_LOCAL */
 
-int init_context_freelists(mca_btl_ofi_context_t *context)
+static int init_context_freelists(mca_btl_ofi_context_t *context)
 {
     int rc;
     OBJ_CONSTRUCT(&context->rdma_comp_list, opal_free_list_t);
-    rc = opal_free_list_init(&context->rdma_comp_list,
-                             sizeof(mca_btl_ofi_rdma_completion_t),
-                             opal_cache_line_size,
-                             OBJ_CLASS(mca_btl_ofi_rdma_completion_t),
-                             0,
-                             0,
-                             512,
-                             -1,
-                             512,
-                             NULL,
-                             0,
-                             NULL,
-                             NULL,
-                             NULL);
+    rc = opal_free_list_init(&context->rdma_comp_list, sizeof(mca_btl_ofi_rdma_completion_t),
+                             opal_cache_line_size, OBJ_CLASS(mca_btl_ofi_rdma_completion_t), 0, 0,
+                             512, -1, 512, NULL, 0, NULL, NULL, NULL);
     if (rc != OPAL_SUCCESS) {
         BTL_VERBOSE(("cannot allocate completion freelist"));
         return rc;
@@ -42,20 +33,9 @@ int init_context_freelists(mca_btl_ofi_context_t *context)
 
     if (TWO_SIDED_ENABLED) {
         OBJ_CONSTRUCT(&context->frag_comp_list, opal_free_list_t);
-        rc = opal_free_list_init(&context->frag_comp_list,
-                                 sizeof(mca_btl_ofi_frag_completion_t),
-                                 opal_cache_line_size,
-                                 OBJ_CLASS(mca_btl_ofi_frag_completion_t),
-                                 0,
-                                 0,
-                                 512,
-                                 -1,
-                                 512,
-                                 NULL,
-                                 0,
-                                 NULL,
-                                 NULL,
-                                 NULL);
+        rc = opal_free_list_init(&context->frag_comp_list, sizeof(mca_btl_ofi_frag_completion_t),
+                                 opal_cache_line_size, OBJ_CLASS(mca_btl_ofi_frag_completion_t), 0,
+                                 0, 512, -1, 512, NULL, 0, NULL, NULL, NULL);
         if (rc != OPAL_SUCCESS) {
             BTL_VERBOSE(("cannot allocate completion freelist"));
             return rc;
@@ -64,20 +44,9 @@ int init_context_freelists(mca_btl_ofi_context_t *context)
         /* Initialize frag pool */
         OBJ_CONSTRUCT(&context->frag_list, opal_free_list_t);
         rc = opal_free_list_init(&context->frag_list,
-                                 sizeof(mca_btl_ofi_base_frag_t) +
-                                    MCA_BTL_OFI_FRAG_SIZE,
-                                 opal_cache_line_size,
-                                 OBJ_CLASS(mca_btl_ofi_base_frag_t),
-                                 0,
-                                 0,
-                                 1024,
-                                 -1,
-                                 1024,
-                                 NULL,
-                                 0,
-                                 NULL,
-                                 NULL,
-                                 NULL);
+                                 sizeof(mca_btl_ofi_base_frag_t) + MCA_BTL_OFI_FRAG_SIZE,
+                                 opal_cache_line_size, OBJ_CLASS(mca_btl_ofi_base_frag_t), 0, 0,
+                                 1024, -1, 1024, NULL, 0, NULL, NULL, NULL);
         if (OPAL_SUCCESS != rc) {
             BTL_VERBOSE(("failed to init frag pool (free_list)"));
         }
@@ -93,8 +62,7 @@ int init_context_freelists(mca_btl_ofi_context_t *context)
  * USE WITH NORMAL ENDPOINT ONLY */
 mca_btl_ofi_context_t *mca_btl_ofi_context_alloc_normal(struct fi_info *info,
                                                         struct fid_domain *domain,
-                                                        struct fid_ep *ep,
-                                                        struct fid_av *av)
+                                                        struct fid_ep *ep, struct fid_av *av)
 {
     int rc;
     uint32_t cq_flags = FI_TRANSMIT | FI_SEND | FI_RECV;
@@ -104,7 +72,7 @@ mca_btl_ofi_context_t *mca_btl_ofi_context_alloc_normal(struct fi_info *info,
 
     mca_btl_ofi_context_t *context;
 
-    context = (mca_btl_ofi_context_t*) calloc(1, sizeof(*context));
+    context = (mca_btl_ofi_context_t *) calloc(1, sizeof(*context));
     if (NULL == context) {
         BTL_VERBOSE(("cannot allocate context"));
         return NULL;
@@ -122,28 +90,20 @@ mca_btl_ofi_context_t *mca_btl_ofi_context_alloc_normal(struct fi_info *info,
     cq_attr.wait_obj = FI_WAIT_NONE;
     rc = fi_cq_open(domain, &cq_attr, &context->cq, NULL);
     if (0 != rc) {
-        BTL_VERBOSE(("%s failed fi_cq_open with err=%s",
-                        linux_device_name,
-                        fi_strerror(-rc)
-                        ));
+        BTL_VERBOSE(("%s failed fi_cq_open with err=%s", linux_device_name, fi_strerror(-rc)));
         goto single_fail;
     }
 
-    rc = fi_ep_bind(ep, (fid_t)av, 0);
+    rc = fi_ep_bind(ep, (fid_t) av, 0);
     if (0 != rc) {
-        BTL_VERBOSE(("%s failed fi_ep_bind with err=%s",
-                        linux_device_name,
-                        fi_strerror(-rc)
-                        ));
+        BTL_VERBOSE(("%s failed fi_ep_bind with err=%s", linux_device_name, fi_strerror(-rc)));
         goto single_fail;
     }
 
-    rc = fi_ep_bind(ep, (fid_t)context->cq, cq_flags);
+    rc = fi_ep_bind(ep, (fid_t) context->cq, cq_flags);
     if (0 != rc) {
-        BTL_VERBOSE(("%s failed fi_scalable_ep_bind with err=%s",
-                        linux_device_name,
-                        fi_strerror(-rc)
-                        ));
+        BTL_VERBOSE(
+            ("%s failed fi_scalable_ep_bind with err=%s", linux_device_name, fi_strerror(-rc)));
         goto single_fail;
     }
 
@@ -155,6 +115,7 @@ mca_btl_ofi_context_t *mca_btl_ofi_context_alloc_normal(struct fi_info *info,
     context->tx_ctx = ep;
     context->rx_ctx = ep;
     context->context_id = 0;
+    my_context = NULL;
 
     return context;
 
@@ -170,8 +131,7 @@ mca_btl_ofi_context_t *mca_btl_ofi_context_alloc_normal(struct fi_info *info,
  * USE WITH SCALABLE ENDPOINT ONLY */
 mca_btl_ofi_context_t *mca_btl_ofi_context_alloc_scalable(struct fi_info *info,
                                                           struct fid_domain *domain,
-                                                          struct fid_ep *sep,
-                                                          struct fid_av *av,
+                                                          struct fid_ep *sep, struct fid_av *av,
                                                           size_t num_contexts)
 {
     BTL_VERBOSE(("creating %zu contexts", num_contexts));
@@ -187,7 +147,7 @@ mca_btl_ofi_context_t *mca_btl_ofi_context_alloc_scalable(struct fi_info *info,
     mca_btl_ofi_context_t *contexts;
     tx_attr.op_flags = FI_DELIVERY_COMPLETE;
 
-    contexts = (mca_btl_ofi_context_t*) calloc(num_contexts, sizeof(*contexts));
+    contexts = (mca_btl_ofi_context_t *) calloc(num_contexts, sizeof(*contexts));
     if (NULL == contexts) {
         BTL_VERBOSE(("cannot allocate communication contexts."));
         return NULL;
@@ -201,23 +161,19 @@ mca_btl_ofi_context_t *mca_btl_ofi_context_alloc_scalable(struct fi_info *info,
         goto scalable_fail;
     }
 
-     /* bind AV to endpoint */
-    rc = fi_scalable_ep_bind(sep, (fid_t)av, 0);
+    /* bind AV to endpoint */
+    rc = fi_scalable_ep_bind(sep, (fid_t) av, 0);
     if (0 != rc) {
-        BTL_VERBOSE(("%s failed fi_scalable_ep_bind with err=%s",
-                        linux_device_name,
-                        fi_strerror(-rc)
-                        ));
+        BTL_VERBOSE(
+            ("%s failed fi_scalable_ep_bind with err=%s", linux_device_name, fi_strerror(-rc)));
         goto scalable_fail;
     }
 
-    for (i=0; i < num_contexts; i++) {
+    for (i = 0; i < num_contexts; i++) {
         rc = fi_tx_context(sep, i, &tx_attr, &contexts[i].tx_ctx, NULL);
         if (0 != rc) {
-            BTL_VERBOSE(("%s failed fi_tx_context with err=%s",
-                            linux_device_name,
-                            fi_strerror(-rc)
-                            ));
+            BTL_VERBOSE(
+                ("%s failed fi_tx_context with err=%s", linux_device_name, fi_strerror(-rc)));
             goto scalable_fail;
         }
 
@@ -226,10 +182,8 @@ mca_btl_ofi_context_t *mca_btl_ofi_context_alloc_scalable(struct fi_info *info,
          * also nice to have equal number of tx/rx context. */
         rc = fi_rx_context(sep, i, &rx_attr, &contexts[i].rx_ctx, NULL);
         if (0 != rc) {
-            BTL_VERBOSE(("%s failed fi_rx_context with err=%s",
-                            linux_device_name,
-                            fi_strerror(-rc)
-                            ));
+            BTL_VERBOSE(
+                ("%s failed fi_rx_context with err=%s", linux_device_name, fi_strerror(-rc)));
             goto scalable_fail;
         }
 
@@ -238,31 +192,23 @@ mca_btl_ofi_context_t *mca_btl_ofi_context_alloc_scalable(struct fi_info *info,
         cq_attr.wait_obj = FI_WAIT_NONE;
         rc = fi_cq_open(domain, &cq_attr, &contexts[i].cq, NULL);
         if (0 != rc) {
-            BTL_VERBOSE(("%s failed fi_cq_open with err=%s",
-                            linux_device_name,
-                            fi_strerror(-rc)
-                            ));
+            BTL_VERBOSE(("%s failed fi_cq_open with err=%s", linux_device_name, fi_strerror(-rc)));
             goto scalable_fail;
         }
 
         /* bind cq to transmit context */
-        rc = fi_ep_bind(contexts[i].tx_ctx, (fid_t)contexts[i].cq, FI_TRANSMIT);
+        rc = fi_ep_bind(contexts[i].tx_ctx, (fid_t) contexts[i].cq, FI_TRANSMIT);
         if (0 != rc) {
-            BTL_VERBOSE(("%s failed fi_ep_bind with err=%s",
-                            linux_device_name,
-                            fi_strerror(-rc)
-                            ));
+            BTL_VERBOSE(("%s failed fi_ep_bind with err=%s", linux_device_name, fi_strerror(-rc)));
             goto scalable_fail;
         }
 
         /* bind cq to receiving  context */
         if (TWO_SIDED_ENABLED) {
-            rc = fi_ep_bind(contexts[i].rx_ctx, (fid_t)contexts[i].cq, FI_RECV);
+            rc = fi_ep_bind(contexts[i].rx_ctx, (fid_t) contexts[i].cq, FI_RECV);
             if (0 != rc) {
-                BTL_VERBOSE(("%s failed fi_ep_bind with err=%s",
-                                linux_device_name,
-                                fi_strerror(-rc)
-                                ));
+                BTL_VERBOSE(
+                    ("%s failed fi_ep_bind with err=%s", linux_device_name, fi_strerror(-rc)));
                 goto scalable_fail;
             }
         }
@@ -270,19 +216,13 @@ mca_btl_ofi_context_t *mca_btl_ofi_context_alloc_scalable(struct fi_info *info,
         /* enable the context. */
         rc = fi_enable(contexts[i].tx_ctx);
         if (0 != rc) {
-            BTL_VERBOSE(("%s failed fi_enable with err=%s",
-                            linux_device_name,
-                            fi_strerror(-rc)
-                            ));
+            BTL_VERBOSE(("%s failed fi_enable with err=%s", linux_device_name, fi_strerror(-rc)));
             goto scalable_fail;
         }
 
         rc = fi_enable(contexts[i].rx_ctx);
         if (0 != rc) {
-            BTL_VERBOSE(("%s failed fi_enable with err=%s",
-                            linux_device_name,
-                            fi_strerror(-rc)
-                            ));
+            BTL_VERBOSE(("%s failed fi_enable with err=%s", linux_device_name, fi_strerror(-rc)));
             goto scalable_fail;
         }
 
@@ -300,7 +240,7 @@ mca_btl_ofi_context_t *mca_btl_ofi_context_alloc_scalable(struct fi_info *info,
 
 scalable_fail:
     /* close and free */
-    for(i=0; i < num_contexts; i++) {
+    for (i = 0; i < num_contexts; i++) {
         mca_btl_ofi_context_finalize(&contexts[i], true);
     }
     free(contexts);
@@ -308,7 +248,8 @@ mca_btl_ofi_context_t *mca_btl_ofi_context_alloc_scalable(struct fi_info *info,
     return NULL;
 }
 
-void mca_btl_ofi_context_finalize(mca_btl_ofi_context_t *context, bool scalable_ep) {
+void mca_btl_ofi_context_finalize(mca_btl_ofi_context_t *context, bool scalable_ep)
+{
 
     /* if it is a scalable ep, we have to close all contexts. */
     if (scalable_ep) {
@@ -321,12 +262,14 @@ void mca_btl_ofi_context_finalize(mca_btl_ofi_context_t *context, bool scalable_
         }
     }
 
-    if( NULL != context->cq) {
+    if (NULL != context->cq) {
         fi_close(&context->cq->fid);
     }
 
     /* Can we destruct the object that hasn't been constructed? */
-    OBJ_DESTRUCT(&context->rdma_comp_list);
+    if (context->rdma_comp_list.fl_num_allocated != 0){
+        OBJ_DESTRUCT(&context->rdma_comp_list);
+    }
 
     if (TWO_SIDED_ENABLED) {
         OBJ_DESTRUCT(&context->frag_comp_list);
@@ -347,12 +290,12 @@ mca_btl_ofi_context_t *get_ofi_context(mca_btl_ofi_module_t *btl)
         OPAL_THREAD_LOCK(&btl->module_lock);
 
         my_context = &btl->contexts[cur_num];
-        cur_num = (cur_num + 1) %btl->num_contexts;
+        cur_num = (cur_num + 1) % btl->num_contexts;
 
         OPAL_THREAD_UNLOCK(&btl->module_lock);
     }
 
-    assert (my_context);
+    assert(my_context);
     return my_context;
 #else
     return get_ofi_context_rr(btl);
@@ -364,10 +307,11 @@ mca_btl_ofi_context_t *get_ofi_context(mca_btl_ofi_module_t *btl)
 mca_btl_ofi_context_t *get_ofi_context_rr(mca_btl_ofi_module_t *btl)
 {
     static volatile uint64_t rr_num = 0;
-    return &btl->contexts[rr_num++%btl->num_contexts];
+    return &btl->contexts[rr_num++ % btl->num_contexts];
 }
 
-int mca_btl_ofi_context_progress(mca_btl_ofi_context_t *context) {
+int mca_btl_ofi_context_progress(mca_btl_ofi_context_t *context)
+{
 
     int ret = 0;
     int events_read;
@@ -388,12 +332,12 @@ int mca_btl_ofi_context_progress(mca_btl_ofi_context_t *context) {
             if (NULL != cq_entry[i].op_context) {
                 ++events;
 
-                c_ctx = (mca_btl_ofi_completion_context_t*) cq_entry[i].op_context;
+                c_ctx = (mca_btl_ofi_completion_context_t *) cq_entry[i].op_context;
 
                 /* We are casting to every type  here just for simplicity. */
-                comp = (mca_btl_ofi_base_completion_t*) c_ctx->comp;
-                frag_comp = (mca_btl_ofi_frag_completion_t*) c_ctx->comp;
-                rdma_comp = (mca_btl_ofi_rdma_completion_t*) c_ctx->comp;
+                comp = (mca_btl_ofi_base_completion_t *) c_ctx->comp;
+                frag_comp = (mca_btl_ofi_frag_completion_t *) c_ctx->comp;
+                rdma_comp = (mca_btl_ofi_rdma_completion_t *) c_ctx->comp;
 
                 switch (comp->type) {
                 case MCA_BTL_OFI_TYPE_GET:
@@ -403,22 +347,22 @@ int mca_btl_ofi_context_progress(mca_btl_ofi_context_t *context) {
                 case MCA_BTL_OFI_TYPE_CSWAP:
                     /* call the callback */
                     if (rdma_comp->cbfunc) {
-                        rdma_comp->cbfunc (comp->btl, comp->endpoint,
-                                           rdma_comp->local_address, rdma_comp->local_handle,
-                                           rdma_comp->cbcontext, rdma_comp->cbdata, OPAL_SUCCESS);
+                        rdma_comp->cbfunc(comp->btl, comp->endpoint, rdma_comp->local_address,
+                                          rdma_comp->local_handle, rdma_comp->cbcontext,
+                                          rdma_comp->cbdata, OPAL_SUCCESS);
                     }
 
-                    MCA_BTL_OFI_NUM_RDMA_DEC((mca_btl_ofi_module_t*) comp->btl);
+                    MCA_BTL_OFI_NUM_RDMA_DEC((mca_btl_ofi_module_t *) comp->btl);
                     break;
 
                 case MCA_BTL_OFI_TYPE_RECV:
-                    mca_btl_ofi_recv_frag((mca_btl_ofi_module_t*)  comp->btl,
-                                          (mca_btl_ofi_endpoint_t*) comp->endpoint,
-                                          context, frag_comp->frag);
+                    mca_btl_ofi_recv_frag((mca_btl_ofi_module_t *) comp->btl,
+                                          (mca_btl_ofi_endpoint_t *) comp->endpoint, context,
+                                          frag_comp->frag);
                     break;
 
                 case MCA_BTL_OFI_TYPE_SEND:
-                    MCA_BTL_OFI_NUM_SEND_DEC((mca_btl_ofi_module_t*) comp->btl);
+                    MCA_BTL_OFI_NUM_SEND_DEC((mca_btl_ofi_module_t *) comp->btl);
                     mca_btl_ofi_frag_complete(frag_comp->frag, OPAL_SUCCESS);
                     break;
 
@@ -429,7 +373,7 @@ int mca_btl_ofi_context_progress(mca_btl_ofi_context_t *context) {
                 }
 
                 /* return the completion handler */
-                opal_free_list_return(comp->my_list, (opal_free_list_item_t*) comp);
+                opal_free_list_return(comp->my_list, (opal_free_list_item_t *) comp);
             }
         }
     } else if (OPAL_UNLIKELY(ret == -FI_EAVAIL)) {
@@ -437,16 +381,15 @@ int mca_btl_ofi_context_progress(mca_btl_ofi_context_t *context) {
 
         /* cq readerr failed!? */
         if (0 > ret) {
-            BTL_ERROR(("%s:%d: Error returned from fi_cq_readerr: %s(%d)",
-                       __FILE__, __LINE__, fi_strerror(-ret), ret));
+            BTL_ERROR(("%s:%d: Error returned from fi_cq_readerr: %s(%d)", __FILE__, __LINE__,
+                       fi_strerror(-ret), ret));
         } else {
-            BTL_ERROR(("fi_cq_readerr: (provider err_code = %d)\n",
-                       cqerr.prov_errno));
+            BTL_ERROR(("fi_cq_readerr: (provider err_code = %d)\n", cqerr.prov_errno));
         }
         MCA_BTL_OFI_ABORT();
     }
 #ifdef FI_EINTR
-    /* sometimes, sockets provider complain about interupt. We do nothing. */
+    /* sometimes, sockets provider complain about interrupt. We do nothing. */
     else if (OPAL_UNLIKELY(ret == -FI_EINTR)) {
 
     }
@@ -459,5 +402,3 @@ int mca_btl_ofi_context_progress(mca_btl_ofi_context_t *context) {
 
     return events;
 }
-
-
diff --git a/opal/mca/btl/ofi/btl_ofi_endpoint.c b/opal/mca/btl/ofi/btl_ofi_endpoint.c
index a374946cb00..f56a8741049 100644
--- a/opal/mca/btl/ofi/btl_ofi_endpoint.c
+++ b/opal/mca/btl/ofi/btl_ofi_endpoint.c
@@ -11,17 +11,17 @@
  * $HEADER$
  */
 
-#include "btl_ofi.h"
 #include "btl_ofi_endpoint.h"
+#include "btl_ofi.h"
 #include "opal/util/proc.h"
 
-static void mca_btl_ofi_endpoint_construct (mca_btl_ofi_endpoint_t *endpoint)
+static void mca_btl_ofi_endpoint_construct(mca_btl_ofi_endpoint_t *endpoint)
 {
     endpoint->peer_addr = 0;
     OBJ_CONSTRUCT(&endpoint->ep_lock, opal_mutex_t);
 }
 
-static void mca_btl_ofi_endpoint_destruct (mca_btl_ofi_endpoint_t *endpoint)
+static void mca_btl_ofi_endpoint_destruct(mca_btl_ofi_endpoint_t *endpoint)
 {
     endpoint->peer_addr = 0;
 
@@ -31,11 +31,10 @@ static void mca_btl_ofi_endpoint_destruct (mca_btl_ofi_endpoint_t *endpoint)
     OBJ_DESTRUCT(&endpoint->ep_lock);
 }
 
-OBJ_CLASS_INSTANCE(mca_btl_ofi_endpoint_t, opal_list_item_t,
-                   mca_btl_ofi_endpoint_construct,
+OBJ_CLASS_INSTANCE(mca_btl_ofi_endpoint_t, opal_list_item_t, mca_btl_ofi_endpoint_construct,
                    mca_btl_ofi_endpoint_destruct);
 
-mca_btl_base_endpoint_t *mca_btl_ofi_endpoint_create (opal_proc_t *proc, struct fid_ep *ep)
+mca_btl_base_endpoint_t *mca_btl_ofi_endpoint_create(opal_proc_t *proc, struct fid_ep *ep)
 {
     mca_btl_ofi_endpoint_t *endpoint = OBJ_NEW(mca_btl_ofi_endpoint_t);
 
diff --git a/opal/mca/btl/ofi/btl_ofi_endpoint.h b/opal/mca/btl/ofi/btl_ofi_endpoint.h
index af8a08d7795..f6b420273af 100644
--- a/opal/mca/btl/ofi/btl_ofi_endpoint.h
+++ b/opal/mca/btl/ofi/btl_ofi_endpoint.h
@@ -15,6 +15,8 @@
  * Copyright (c) 2018      Intel, Inc, All rights reserved
  * Copyright (c) 2020      Amazon.com, Inc. or its affiliates.
  *                         All Rights reserved.
+ * Copyright (c) 2025      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -32,10 +34,6 @@
 
 BEGIN_C_DECLS
 
-#if OPAL_HAVE_THREAD_LOCAL
-extern opal_thread_local  mca_btl_ofi_context_t *my_context;
-#endif /* OPAL_HAVE_THREAD_LOCAL */
-
 struct mca_btl_base_endpoint_t {
     opal_list_item_t super;
 
@@ -53,21 +51,17 @@ typedef struct mca_btl_base_endpoint_t mca_btl_base_endpoint_t;
 typedef mca_btl_base_endpoint_t mca_btl_ofi_endpoint_t;
 OBJ_CLASS_DECLARATION(mca_btl_ofi_endpoint_t);
 
-int init_context_freelists(mca_btl_ofi_context_t *context);
-
-mca_btl_base_endpoint_t *mca_btl_ofi_endpoint_create (opal_proc_t *proc, struct fid_ep *ep);
+mca_btl_base_endpoint_t *mca_btl_ofi_endpoint_create(opal_proc_t *proc, struct fid_ep *ep);
 
 /* contexts */
 mca_btl_ofi_context_t *mca_btl_ofi_context_alloc_scalable(struct fi_info *info,
                                                           struct fid_domain *domain,
-                                                          struct fid_ep *sep,
-                                                          struct fid_av *av,
+                                                          struct fid_ep *sep, struct fid_av *av,
                                                           size_t num_contexts);
 
 mca_btl_ofi_context_t *mca_btl_ofi_context_alloc_normal(struct fi_info *info,
                                                         struct fid_domain *domain,
-                                                        struct fid_ep *ep,
-                                                        struct fid_av *av);
+                                                        struct fid_ep *ep, struct fid_av *av);
 void mca_btl_ofi_context_finalize(mca_btl_ofi_context_t *context, bool scalable_ep);
 
 mca_btl_ofi_context_t *get_ofi_context(mca_btl_ofi_module_t *btl);
diff --git a/opal/mca/btl/ofi/btl_ofi_frag.c b/opal/mca/btl/ofi/btl_ofi_frag.c
index 203da3197d1..25433c0b6a3 100644
--- a/opal/mca/btl/ofi/btl_ofi_frag.c
+++ b/opal/mca/btl/ofi/btl_ofi_frag.c
@@ -12,44 +12,37 @@
  * $HEADER$
  */
 
-#include "btl_ofi.h"
 #include "btl_ofi_frag.h"
-#include "btl_ofi_rdma.h"
+#include "btl_ofi.h"
 #include "btl_ofi_endpoint.h"
+#include "btl_ofi_rdma.h"
 
-static void mca_btl_ofi_base_frag_constructor (mca_btl_ofi_base_frag_t *frag)
+static void mca_btl_ofi_base_frag_constructor(mca_btl_ofi_base_frag_t *frag)
 {
     /* zero everything out */
-    memset ((char *) frag + sizeof (frag->base), 0, sizeof (*frag) - sizeof (frag->base));
+    memset((char *) frag + sizeof(frag->base), 0, sizeof(*frag) - sizeof(frag->base));
 
     frag->base.des_segments = frag->segments;
     frag->base.des_segment_count = 1;
 }
 
-static void mca_btl_ofi_base_frag_destructor (mca_btl_ofi_base_frag_t *frag)
+static void mca_btl_ofi_base_frag_destructor(mca_btl_ofi_base_frag_t *frag)
 {
-
 }
 
-OBJ_CLASS_INSTANCE(mca_btl_ofi_base_frag_t,
-                   mca_btl_base_descriptor_t,
-                   mca_btl_ofi_base_frag_constructor,
-                   mca_btl_ofi_base_frag_destructor);
-
-OBJ_CLASS_INSTANCE(mca_btl_ofi_frag_completion_t,
-                   opal_free_list_item_t,
-                   NULL,
-                   NULL);
-
-mca_btl_ofi_frag_completion_t *mca_btl_ofi_frag_completion_alloc
-                                        (mca_btl_base_module_t *btl,
-                                         mca_btl_ofi_context_t *context,
-                                         mca_btl_ofi_base_frag_t *frag,
-                                         int type)
+OBJ_CLASS_INSTANCE(mca_btl_ofi_base_frag_t, mca_btl_base_descriptor_t,
+                   mca_btl_ofi_base_frag_constructor, mca_btl_ofi_base_frag_destructor);
+
+OBJ_CLASS_INSTANCE(mca_btl_ofi_frag_completion_t, opal_free_list_item_t, NULL, NULL);
+
+mca_btl_ofi_frag_completion_t *mca_btl_ofi_frag_completion_alloc(mca_btl_base_module_t *btl,
+                                                                 mca_btl_ofi_context_t *context,
+                                                                 mca_btl_ofi_base_frag_t *frag,
+                                                                 int type)
 {
     mca_btl_ofi_frag_completion_t *comp;
 
-    comp = (mca_btl_ofi_frag_completion_t*) opal_free_list_get(&context->frag_comp_list);
+    comp = (mca_btl_ofi_frag_completion_t *) opal_free_list_get(&context->frag_comp_list);
     comp->base.btl = btl;
     comp->base.my_context = context;
     comp->base.my_list = &context->frag_comp_list;
@@ -61,13 +54,11 @@ mca_btl_ofi_frag_completion_t *mca_btl_ofi_frag_completion_alloc
     return comp;
 }
 
-
-mca_btl_base_descriptor_t *mca_btl_ofi_alloc(
-                                mca_btl_base_module_t *btl,
-                                mca_btl_base_endpoint_t *endpoint,
-                                uint8_t order, size_t size, uint32_t flags)
+mca_btl_base_descriptor_t *mca_btl_ofi_alloc(mca_btl_base_module_t *btl,
+                                             mca_btl_base_endpoint_t *endpoint, uint8_t order,
+                                             size_t size, uint32_t flags)
 {
-    mca_btl_ofi_module_t *ofi_btl = (mca_btl_ofi_module_t*) btl;
+    mca_btl_ofi_module_t *ofi_btl = (mca_btl_ofi_module_t *) btl;
     mca_btl_ofi_base_frag_t *frag = NULL;
     mca_btl_ofi_context_t *context = get_ofi_context(ofi_btl);
 
@@ -80,30 +71,29 @@ mca_btl_base_descriptor_t *mca_btl_ofi_alloc(
         frag->base.des_segment_count = 1;
         frag->base.des_segments = &frag->segments[0];
         frag->base.des_flags = flags;
-        frag->base.order     = order;
-        frag->hdr.len        = size;
+        frag->base.order = order;
+        frag->hdr.len = size;
     }
 
-    return (mca_btl_base_descriptor_t*) frag;
+    return (mca_btl_base_descriptor_t *) frag;
 }
 
-int mca_btl_ofi_free (mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des)
+int mca_btl_ofi_free(mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des)
 {
     /* return the frag to the free list. */
-    mca_btl_ofi_frag_return ((mca_btl_ofi_base_frag_t*) des);
+    mca_btl_ofi_frag_return((mca_btl_ofi_base_frag_t *) des);
     return OPAL_SUCCESS;
 }
 
-int mca_btl_ofi_send (mca_btl_base_module_t *btl,
-                      mca_btl_base_endpoint_t *endpoint,
-                      mca_btl_base_descriptor_t *descriptor,
-                      mca_btl_base_tag_t tag)
+int mca_btl_ofi_send(mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
+                     mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag)
 {
     int rc = 0;
+    size_t msg_sz;
     mca_btl_ofi_context_t *context;
-    mca_btl_ofi_module_t *ofi_btl = (mca_btl_ofi_module_t*) btl;
-    mca_btl_ofi_endpoint_t *ofi_ep = (mca_btl_ofi_endpoint_t*) endpoint;
-    mca_btl_ofi_base_frag_t *frag = (mca_btl_ofi_base_frag_t*) descriptor;
+    mca_btl_ofi_module_t *ofi_btl = (mca_btl_ofi_module_t *) btl;
+    mca_btl_ofi_endpoint_t *ofi_ep = (mca_btl_ofi_endpoint_t *) endpoint;
+    mca_btl_ofi_base_frag_t *frag = (mca_btl_ofi_base_frag_t *) descriptor;
     mca_btl_ofi_frag_completion_t *comp;
 
     frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
@@ -111,19 +101,40 @@ int mca_btl_ofi_send (mca_btl_base_module_t *btl,
     /* This tag is the active message tag for the remote side */
     frag->hdr.tag = tag;
 
-    /* create completion context */
     context = get_ofi_context(ofi_btl);
-    comp = mca_btl_ofi_frag_completion_alloc(btl, context, frag,
-                                             MCA_BTL_OFI_TYPE_SEND);
+    msg_sz = sizeof(mca_btl_ofi_header_t) + frag->hdr.len;
+
+    /*
+     * If supported by the provider and not disabled by
+     * the btl_ofi_disable_inject  MCA parameter,
+     * try to use the inject path for short messages.
+     * fi_inject can return -FI_EAGAIN if the provider is
+     * unable to buffer the message, so one needs to fall
+     * back to the fi_send path when that error code is
+     * returned. 
+     */
+    if (msg_sz <= mca_btl_ofi_component.max_inject_size &&
+        false == mca_btl_ofi_component.disable_inject) {
+        rc = fi_inject(context->tx_ctx, &frag->hdr, msg_sz, ofi_ep->peer_addr);
+        if (FI_SUCCESS == rc) {
+            mca_btl_ofi_frag_complete(frag, OPAL_SUCCESS);
+            return OPAL_SUCCESS;
+        }
+        /*
+         * -FI_EAGAIN is okay but any other error is a problem
+         */
+        if (-FI_EAGAIN != rc) {
+            return OPAL_ERROR;
+        }
+    }
+
+    /* create completion context */
+     comp = mca_btl_ofi_frag_completion_alloc(btl, context, frag, MCA_BTL_OFI_TYPE_SEND);
 
     /* send the frag. Note that we start sending from BTL header + payload
      * because we need the other side to have this header information. */
-    rc = fi_send(context->tx_ctx,
-                 &frag->hdr,
-                 sizeof(mca_btl_ofi_header_t) +  frag->hdr.len,
-                 NULL,
-                 ofi_ep->peer_addr,
-                 &comp->comp_ctx);
+    rc = fi_send(context->tx_ctx, &frag->hdr, sizeof(mca_btl_ofi_header_t) + frag->hdr.len, NULL,
+                 ofi_ep->peer_addr, &comp->comp_ctx);
 
     if (OPAL_UNLIKELY(FI_SUCCESS != rc)) {
         return OPAL_ERR_OUT_OF_RESOURCE;
@@ -133,14 +144,12 @@ int mca_btl_ofi_send (mca_btl_base_module_t *btl,
     return OPAL_SUCCESS;
 }
 
-int mca_btl_ofi_recv_frag (mca_btl_ofi_module_t *ofi_btl,
-                           mca_btl_base_endpoint_t *endpoint,
-                           mca_btl_ofi_context_t *context,
-                           mca_btl_ofi_base_frag_t *frag)
+int mca_btl_ofi_recv_frag(mca_btl_ofi_module_t *ofi_btl, mca_btl_base_endpoint_t *endpoint,
+                          mca_btl_ofi_context_t *context, mca_btl_ofi_base_frag_t *frag)
 {
     int rc;
     mca_btl_active_message_callback_t *reg = mca_btl_base_active_message_trigger + frag->hdr.tag;
-    mca_btl_base_segment_t segment = {.seg_addr.pval = (void *)(frag + 1),
+    mca_btl_base_segment_t segment = {.seg_addr.pval = (void *) (frag + 1),
                                       .seg_len = frag->hdr.len};
     /* Tell PML where the payload is */
     mca_btl_base_receive_descriptor_t recv_desc = {.endpoint = endpoint,
@@ -150,11 +159,11 @@ int mca_btl_ofi_recv_frag (mca_btl_ofi_module_t *ofi_btl,
                                                    .cbdata = reg->cbdata};
 
     /* call the callback */
-    reg->cbfunc (&ofi_btl->super, &recv_desc);
+    reg->cbfunc(&ofi_btl->super, &recv_desc);
     mca_btl_ofi_frag_complete(frag, OPAL_SUCCESS);
 
     /* repost the recv */
-    rc = mca_btl_ofi_post_recvs((mca_btl_base_module_t*) ofi_btl, context, 1);
+    rc = mca_btl_ofi_post_recvs((mca_btl_base_module_t *) ofi_btl, context, 1);
     if (OPAL_SUCCESS != rc) {
         /* might not be that bad but let's just fail here. */
         BTL_ERROR(("failed reposting receive."));
@@ -164,12 +173,11 @@ int mca_btl_ofi_recv_frag (mca_btl_ofi_module_t *ofi_btl,
     return OPAL_SUCCESS;
 }
 
-struct mca_btl_base_descriptor_t *mca_btl_ofi_prepare_src (
-                                                mca_btl_base_module_t *btl,
-                                                mca_btl_base_endpoint_t *endpoint,
-                                                opal_convertor_t *convertor,
-                                                uint8_t order, size_t reserve,
-                                                size_t *size, uint32_t flags)
+struct mca_btl_base_descriptor_t *mca_btl_ofi_prepare_src(mca_btl_base_module_t *btl,
+                                                          mca_btl_base_endpoint_t *endpoint,
+                                                          opal_convertor_t *convertor,
+                                                          uint8_t order, size_t reserve,
+                                                          size_t *size, uint32_t flags)
 {
     struct iovec iov;
     size_t length;
@@ -177,15 +185,15 @@ struct mca_btl_base_descriptor_t *mca_btl_ofi_prepare_src (
     mca_btl_ofi_base_frag_t *frag;
 
     /* allocate the frag with reserve. */
-    frag = (mca_btl_ofi_base_frag_t*) mca_btl_ofi_alloc(btl, endpoint,
-                                                        order, reserve, flags);
+    frag = (mca_btl_ofi_base_frag_t *) mca_btl_ofi_alloc(btl, endpoint, order, reserve, flags);
     if (OPAL_UNLIKELY(NULL == frag)) {
         return NULL;
     }
 
     /* pack the data after the reserve */
     iov.iov_len = *size;
-    iov.iov_base = (IOVBASE_TYPE*)(((unsigned char*)(frag->segments[0].seg_addr.pval)) + reserve);
+    iov.iov_base = (IOVBASE_TYPE *) (((unsigned char *) (frag->segments[0].seg_addr.pval))
+                                     + reserve);
     opal_convertor_pack(convertor, &iov, &iov_count, &length);
 
     /* pass on frag information */
diff --git a/opal/mca/btl/ofi/btl_ofi_frag.h b/opal/mca/btl/ofi/btl_ofi_frag.h
index 7e87a895b87..3afa8866265 100644
--- a/opal/mca/btl/ofi/btl_ofi_frag.h
+++ b/opal/mca/btl/ofi/btl_ofi_frag.h
@@ -10,59 +10,52 @@
  */
 
 #if !defined(MCA_BTL_OFI_FRAG_H)
-#define MCA_BTL_OFI_FRAG_H
+#    define MCA_BTL_OFI_FRAG_H
 
-#include "btl_ofi.h"
-#include "btl_ofi_endpoint.h"
+#    include "btl_ofi.h"
+#    include "btl_ofi_endpoint.h"
 
+#    define MCA_BTL_OFI_HDR_SIZE  sizeof(mca_btl_ofi_header_t)
+#    define MCA_BTL_OFI_FRAG_SIZE 4096
+#    define MCA_BTL_OFI_RECV_SIZE MCA_BTL_OFI_FRAG_SIZE + MCA_BTL_OFI_HDR_SIZE
 
-#define MCA_BTL_OFI_HDR_SIZE        sizeof(mca_btl_ofi_header_t)
-#define MCA_BTL_OFI_FRAG_SIZE       4096
-#define MCA_BTL_OFI_RECV_SIZE       MCA_BTL_OFI_FRAG_SIZE + MCA_BTL_OFI_HDR_SIZE
+#    define MCA_BTL_OFI_NUM_SEND_INC(module)                                       \
+        OPAL_ATOMIC_ADD_FETCH64(&(module)->outstanding_send, 1);                   \
+        if (module->outstanding_send > mca_btl_ofi_component.progress_threshold) { \
+            mca_btl_ofi_component.super.btl_progress();                            \
+        }
 
-#define MCA_BTL_OFI_NUM_SEND_INC(module)                                          \
-    OPAL_ATOMIC_ADD_FETCH64(&(module)->outstanding_send, 1);                      \
-    if (module->outstanding_send > mca_btl_ofi_component.progress_threshold) {    \
-        mca_btl_ofi_component.super.btl_progress();                               \
-    }
+#    define MCA_BTL_OFI_NUM_SEND_DEC(module) \
+        OPAL_ATOMIC_ADD_FETCH64(&(module)->outstanding_send, -1);
+
+mca_btl_base_descriptor_t *mca_btl_ofi_alloc(mca_btl_base_module_t *btl,
+                                             mca_btl_base_endpoint_t *endpoint, uint8_t order,
+                                             size_t size, uint32_t flags);
+
+int mca_btl_ofi_free(mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des);
+
+int mca_btl_ofi_send(mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
+                     mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag);
 
-#define MCA_BTL_OFI_NUM_SEND_DEC(module)                                          \
-    OPAL_ATOMIC_ADD_FETCH64(&(module)->outstanding_send, -1);
-
-mca_btl_base_descriptor_t *mca_btl_ofi_alloc(
-                                mca_btl_base_module_t *btl,
-                                mca_btl_base_endpoint_t *endpoint,
-                                uint8_t order, size_t size, uint32_t flags);
-
-int mca_btl_ofi_free (mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des);
-
-int mca_btl_ofi_send (mca_btl_base_module_t *btl,
-                      mca_btl_base_endpoint_t *endpoint,
-                      mca_btl_base_descriptor_t *descriptor,
-                      mca_btl_base_tag_t tag);
-
-int mca_btl_ofi_recv_frag (mca_btl_ofi_module_t *ofi_btl,
-                           mca_btl_base_endpoint_t *endpoint,
-                           mca_btl_ofi_context_t *context,
-                           mca_btl_ofi_base_frag_t *frag);
-
-struct mca_btl_base_descriptor_t *mca_btl_ofi_prepare_src (
-                                                mca_btl_base_module_t *btl,
-                                                mca_btl_base_endpoint_t *endpoint,
-                                                opal_convertor_t *convertor,
-                                                uint8_t order, size_t reserve,
-                                                size_t *size, uint32_t flags);
-
-mca_btl_ofi_frag_completion_t *mca_btl_ofi_frag_completion_alloc
-                                        (mca_btl_base_module_t *btl,
-                                         mca_btl_ofi_context_t *context,
-                                         mca_btl_ofi_base_frag_t *frag,
-                                         int type);
-
-static inline mca_btl_ofi_base_frag_t *mca_btl_ofi_frag_alloc (mca_btl_ofi_module_t *ofi_btl, opal_free_list_t *fl,
-                                                               mca_btl_base_endpoint_t *endpoint)
+int mca_btl_ofi_recv_frag(mca_btl_ofi_module_t *ofi_btl, mca_btl_base_endpoint_t *endpoint,
+                          mca_btl_ofi_context_t *context, mca_btl_ofi_base_frag_t *frag);
+
+struct mca_btl_base_descriptor_t *mca_btl_ofi_prepare_src(mca_btl_base_module_t *btl,
+                                                          mca_btl_base_endpoint_t *endpoint,
+                                                          opal_convertor_t *convertor,
+                                                          uint8_t order, size_t reserve,
+                                                          size_t *size, uint32_t flags);
+
+mca_btl_ofi_frag_completion_t *mca_btl_ofi_frag_completion_alloc(mca_btl_base_module_t *btl,
+                                                                 mca_btl_ofi_context_t *context,
+                                                                 mca_btl_ofi_base_frag_t *frag,
+                                                                 int type);
+
+static inline mca_btl_ofi_base_frag_t *mca_btl_ofi_frag_alloc(mca_btl_ofi_module_t *ofi_btl,
+                                                              opal_free_list_t *fl,
+                                                              mca_btl_base_endpoint_t *endpoint)
 {
-    mca_btl_ofi_base_frag_t *frag = (mca_btl_ofi_base_frag_t *) opal_free_list_get (fl);
+    mca_btl_ofi_base_frag_t *frag = (mca_btl_ofi_base_frag_t *) opal_free_list_get(fl);
 
     if (OPAL_LIKELY(NULL != frag)) {
         frag->free_list = fl;
@@ -73,12 +66,13 @@ static inline mca_btl_ofi_base_frag_t *mca_btl_ofi_frag_alloc (mca_btl_ofi_modul
     return frag;
 }
 
-static inline void mca_btl_ofi_frag_return (mca_btl_ofi_base_frag_t *frag)
+static inline void mca_btl_ofi_frag_return(mca_btl_ofi_base_frag_t *frag)
 {
-    opal_free_list_return (frag->free_list, &frag->base.super);
+    opal_free_list_return(frag->free_list, &frag->base.super);
 }
 
-static inline void mca_btl_ofi_frag_complete (mca_btl_ofi_base_frag_t *frag, int rc) {
+static inline void mca_btl_ofi_frag_complete(mca_btl_ofi_base_frag_t *frag, int rc)
+{
     mca_btl_ofi_module_t *ofi_btl = frag->btl;
 
     /* call the local callback if specified */
@@ -88,7 +82,7 @@ static inline void mca_btl_ofi_frag_complete (mca_btl_ofi_base_frag_t *frag, int
 
     /* If the BTL has ownership, return it to the free list, */
     if (OPAL_LIKELY(frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP)) {
-        mca_btl_ofi_frag_return (frag);
+        mca_btl_ofi_frag_return(frag);
     }
 }
 
diff --git a/opal/mca/btl/ofi/btl_ofi_module.c b/opal/mca/btl/ofi/btl_ofi_module.c
index b32a1f106a7..e213d5b1865 100644
--- a/opal/mca/btl/ofi/btl_ofi_module.c
+++ b/opal/mca/btl/ofi/btl_ofi_module.c
@@ -16,6 +16,8 @@
  *
  * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * Copyright (c) 2020      Google, LLC. All rights reserved.
+ * Copyright (c) 2022-2024 Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -24,22 +26,24 @@
  */
 
 #include "opal_config.h"
-#include <string.h>
 #include "opal/class/opal_bitmap.h"
-#include "opal/util/printf.h"
-#include "opal/mca/btl/btl.h"
 #include "opal/datatype/opal_convertor.h"
+#include "opal/mca/accelerator/accelerator.h"
+#include "opal/mca/accelerator/base/base.h"
+#include "opal/mca/btl/btl.h"
+#include "opal/mca/common/ofi/common_ofi.h"
 #include "opal/mca/mpool/base/base.h"
 #include "opal/mca/mpool/mpool.h"
+#include "opal/util/printf.h"
+#include <string.h>
 
 #include "btl_ofi.h"
 #include "btl_ofi_endpoint.h"
 #include "btl_ofi_frag.h"
 
-static int mca_btl_ofi_add_procs (mca_btl_base_module_t *btl,
-                                  size_t nprocs, opal_proc_t **opal_procs,
-                                  mca_btl_base_endpoint_t **peers,
-                                  opal_bitmap_t *reachable)
+static int mca_btl_ofi_add_procs(mca_btl_base_module_t *btl, size_t nprocs,
+                                 opal_proc_t **opal_procs, mca_btl_base_endpoint_t **peers,
+                                 opal_bitmap_t *reachable)
 {
     int rc;
     int count;
@@ -51,48 +55,51 @@ static int mca_btl_ofi_add_procs (mca_btl_base_module_t *btl,
 
     mca_btl_ofi_module_t *ofi_btl = (mca_btl_ofi_module_t *) btl;
 
-    for (size_t i = 0 ; i < nprocs ; ++i) {
+    for (size_t i = 0; i < nprocs; ++i) {
 
         proc = opal_procs[i];
 
         /* See if we already have an endpoint for this proc. */
-        rc = opal_hash_table_get_value_uint64 (&ofi_btl->id_to_endpoint, (intptr_t) proc, (void **) &ep);
+        rc = opal_hash_table_get_value_uint64(&ofi_btl->id_to_endpoint, (intptr_t) proc,
+                                              (void **) &ep);
 
         if (OPAL_SUCCESS == rc) {
-            BTL_VERBOSE(("returning existing endpoint for proc %s", OPAL_NAME_PRINT(proc->proc_name)));
+            BTL_VERBOSE(
+                ("returning existing endpoint for proc %s", OPAL_NAME_PRINT(proc->proc_name)));
             peers[i] = ep;
 
         } else {
             /* We don't have this endpoint yet, create one */
-            peers[i] = mca_btl_ofi_endpoint_create (proc, ofi_btl->ofi_endpoint);
-            BTL_VERBOSE(("creating peer %p", (void*) peers[i]));
+            peers[i] = mca_btl_ofi_endpoint_create(proc, ofi_btl->ofi_endpoint);
+            BTL_VERBOSE(("creating peer %p", (void *) peers[i]));
 
             if (OPAL_UNLIKELY(NULL == peers[i])) {
                 return OPAL_ERR_OUT_OF_RESOURCE;
             }
 
             /* Add this endpoint to the lookup table */
-            (void) opal_hash_table_set_value_uint64 (&ofi_btl->id_to_endpoint, (intptr_t) proc, (void**) &ep);
+            (void) opal_hash_table_set_value_uint64(&ofi_btl->id_to_endpoint, (intptr_t) proc,
+                                                    (void **) &ep);
         }
 
-        OPAL_MODEX_RECV(rc, &mca_btl_ofi_component.super.btl_version,
-                        &peers[i]->ep_proc->proc_name, (void **)&ep_name, &namelen);
+        OPAL_MODEX_RECV(rc, &mca_btl_ofi_component.super.btl_version, &peers[i]->ep_proc->proc_name,
+                        (void **) &ep_name, &namelen);
         if (OPAL_SUCCESS != rc) {
             BTL_ERROR(("error receiving modex"));
             MCA_BTL_OFI_ABORT();
         }
 
         /* get peer fi_addr */
-        count = fi_av_insert(ofi_btl->av,      /* Address vector to insert */
-                             ep_name,          /* peer name */
-                             1,                /* amount to insert */
+        count = fi_av_insert(ofi_btl->av,          /* Address vector to insert */
+                             ep_name,              /* peer name */
+                             1,                    /* amount to insert */
                              &peers[i]->peer_addr, /* return peer address here */
-                             0,                /* flags */
-                             NULL);            /* context */
+                             0,                    /* flags */
+                             NULL);                /* context */
 
         /* if succeed, add this proc and mark reachable */
         if (count == 1) { /* we inserted 1 address. */
-            opal_list_append (&ofi_btl->endpoints, &peers[i]->super);
+            opal_list_append(&ofi_btl->endpoints, &peers[i]->super);
             opal_bitmap_set_bit(reachable, i);
         } else {
             BTL_VERBOSE(("fi_av_insert failed with rc = %d", count));
@@ -103,16 +110,17 @@ static int mca_btl_ofi_add_procs (mca_btl_base_module_t *btl,
     return OPAL_SUCCESS;
 }
 
-static int mca_btl_ofi_del_procs (mca_btl_base_module_t *btl, size_t nprocs,
-                                  opal_proc_t **procs, mca_btl_base_endpoint_t **peers)
+static int mca_btl_ofi_del_procs(mca_btl_base_module_t *btl, size_t nprocs, opal_proc_t **procs,
+                                 mca_btl_base_endpoint_t **peers)
 {
     int rc;
     mca_btl_ofi_module_t *ofi_btl = (mca_btl_ofi_module_t *) btl;
     mca_btl_base_endpoint_t *ep;
 
-    for (size_t i = 0 ; i < nprocs ; ++i) {
+    for (size_t i = 0; i < nprocs; ++i) {
         if (peers[i]) {
-            rc = opal_hash_table_get_value_uint64 (&ofi_btl->id_to_endpoint, (intptr_t) procs[i], (void **) &ep);
+            rc = opal_hash_table_get_value_uint64(&ofi_btl->id_to_endpoint, (intptr_t) procs[i],
+                                                  (void **) &ep);
 
             if (OPAL_SUCCESS == rc) {
                 /* remove the address from AV. */
@@ -120,37 +128,37 @@ static int mca_btl_ofi_del_procs (mca_btl_base_module_t *btl, size_t nprocs,
                 if (rc < 0) {
                     /* remove failed. this should not happen. */
                     /* Lets not crash because we failed to remove an address. */
-                    BTL_ERROR(("fi_av_remove failed with error %d:%s",
-                                    rc, fi_strerror(-rc)));
+                    BTL_ERROR(("fi_av_remove failed with error %d:%s", rc, fi_strerror(-rc)));
                 }
 
                 /* remove and free MPI endpoint from the list. */
-                opal_list_remove_item (&ofi_btl->endpoints, &peers[i]->super);
-                (void) opal_hash_table_remove_value_uint64 (&ofi_btl->id_to_endpoint, (intptr_t) procs[i]);
+                opal_list_remove_item(&ofi_btl->endpoints, &peers[i]->super);
+                (void) opal_hash_table_remove_value_uint64(&ofi_btl->id_to_endpoint,
+                                                           (intptr_t) procs[i]);
                 OBJ_RELEASE(peers[i]);
-           }
+            }
         }
     }
 
     return OPAL_SUCCESS;
 }
 
-void mca_btl_ofi_rcache_init (mca_btl_ofi_module_t *module)
+void mca_btl_ofi_rcache_init(mca_btl_ofi_module_t *module)
 {
     if (!module->initialized) {
         mca_rcache_base_resources_t rcache_resources;
         char *tmp;
 
-        (void) opal_asprintf (&tmp, "ofi.%s", module->linux_device_name);
+        (void) opal_asprintf(&tmp, "ofi.%s", module->linux_device_name);
 
-        rcache_resources.cache_name     = tmp;
-        rcache_resources.reg_data       = (void *) module;
-        rcache_resources.sizeof_reg     = sizeof (mca_btl_ofi_reg_t);
-        rcache_resources.register_mem   = mca_btl_ofi_reg_mem;
+        rcache_resources.cache_name = tmp;
+        rcache_resources.reg_data = (void *) module;
+        rcache_resources.sizeof_reg = sizeof(mca_btl_ofi_reg_t);
+        rcache_resources.register_mem = mca_btl_ofi_reg_mem;
         rcache_resources.deregister_mem = mca_btl_ofi_dereg_mem;
 
-        module->rcache = mca_rcache_base_module_create ("grdma", module, &rcache_resources);
-        free (tmp);
+        module->rcache = mca_rcache_base_module_create("grdma", module, &rcache_resources);
+        free(tmp);
 
         if (NULL == module->rcache) {
             /* something when horribly wrong */
@@ -162,7 +170,6 @@ void mca_btl_ofi_rcache_init (mca_btl_ofi_module_t *module)
     }
 }
 
-
 /**
  * @brief Register a memory region for put/get/atomic operations.
  *
@@ -182,16 +189,21 @@ void mca_btl_ofi_rcache_init (mca_btl_ofi_module_t *module)
  * as they may use limited system/NIC resources.
  */
 static struct mca_btl_base_registration_handle_t *
-mca_btl_ofi_register_mem (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, void *base,
-                          size_t size, uint32_t flags)
+mca_btl_ofi_register_mem(struct mca_btl_base_module_t *btl,
+                         struct mca_btl_base_endpoint_t *endpoint, void *base, size_t size,
+                         uint32_t flags)
 {
     mca_btl_ofi_module_t *ofi_module = (mca_btl_ofi_module_t *) btl;
     mca_btl_ofi_reg_t *reg;
     int access_flags = flags & MCA_BTL_REG_FLAG_ACCESS_ANY;
     int rc;
+    uint32_t cache_flags = 0;
+    if (ofi_module->bypass_cache) {
+	   cache_flags |= MCA_RCACHE_FLAGS_CACHE_BYPASS;
+    }
 
-    rc = ofi_module->rcache->rcache_register (ofi_module->rcache, base, size, 0, access_flags,
-                                              (mca_rcache_base_registration_t **) &reg);
+    rc = ofi_module->rcache->rcache_register(ofi_module->rcache, base, size, cache_flags, access_flags,
+                                             (mca_rcache_base_registration_t **) &reg);
     if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
         return NULL;
     }
@@ -210,31 +222,83 @@ mca_btl_ofi_register_mem (struct mca_btl_base_module_t *btl, struct mca_btl_base
  * after it is deregistered. It is erroneous to specify a memory handle associated with
  * a remote node.
  */
-static int mca_btl_ofi_deregister_mem (mca_btl_base_module_t *btl, mca_btl_base_registration_handle_t *handle)
+static int mca_btl_ofi_deregister_mem(mca_btl_base_module_t *btl,
+                                      mca_btl_base_registration_handle_t *handle)
 {
     mca_btl_ofi_module_t *ofi_module = (mca_btl_ofi_module_t *) btl;
-    mca_btl_ofi_reg_t *reg =
-        (mca_btl_ofi_reg_t *)((intptr_t) handle - offsetof (mca_btl_ofi_reg_t, handle));
+    mca_btl_ofi_reg_t *reg = (mca_btl_ofi_reg_t *) ((intptr_t) handle
+                                                    - offsetof(mca_btl_ofi_reg_t, handle));
 
-    (void) ofi_module->rcache->rcache_deregister (ofi_module->rcache, &reg->base);
+    (void) ofi_module->rcache->rcache_deregister(ofi_module->rcache, &reg->base);
 
     return OPAL_SUCCESS;
 }
 
-int mca_btl_ofi_reg_mem (void *reg_data, void *base, size_t size, mca_rcache_base_registration_t *reg)
+int mca_btl_ofi_reg_mem(void *reg_data, void *base, size_t size,
+                        mca_rcache_base_registration_t *reg)
 {
-    int rc;
+    int rc, dev_id;
+    uint64_t flags;
     static uint64_t access_flags = FI_REMOTE_WRITE | FI_REMOTE_READ | FI_READ | FI_WRITE;
+    struct fi_mr_attr attr = {0};
+    struct iovec iov = {0};
+
+    mca_btl_ofi_module_t *btl = (mca_btl_ofi_module_t *) reg_data;
+    mca_btl_ofi_reg_t *ur = (mca_btl_ofi_reg_t *) reg;
+
+    iov.iov_base = base;
+    iov.iov_len = size;
+    attr.mr_iov = &iov;
+    attr.iov_count = 1;
+    attr.access = access_flags;
+    attr.offset = 0;
+    attr.context = NULL;
+    attr.requested_key = (uint64_t) reg;
+
+#if  OPAL_OFI_HAVE_FI_MR_IFACE
+    if (OPAL_LIKELY(NULL != base)) {
+        rc = opal_accelerator.check_addr(base, &dev_id, &flags);
+        if (rc < 0) {
+            return rc;
+        } else if (rc > 0 ) {
+            if (0 == strcmp(opal_accelerator_base_selected_component.base_version.mca_component_name, "cuda")) {
+                attr.iface = FI_HMEM_CUDA;
+                opal_accelerator.get_device(&attr.device.cuda);
+#if OPAL_OFI_HAVE_FI_HMEM_ROCR
+	    } else if (0 == strcmp(opal_accelerator_base_selected_component.base_version.mca_component_name, "rocm")) {
+                attr.iface = FI_HMEM_ROCR;
+                opal_accelerator.get_device(&attr.device.cuda);
+#endif
+#if OPAL_OFI_HAVE_FI_HMEM_ZE
+            } else if (0 == strcmp(opal_accelerator_base_selected_component.base_version.mca_component_name, "ze")) {
+                attr.iface = FI_HMEM_ZE;
+                opal_accelerator.get_device(&attr.device.ze);
+#endif
+            } else {
+                return OPAL_ERROR;
+            }
+        }
+    }
+#endif
 
-    mca_btl_ofi_module_t *btl = (mca_btl_ofi_module_t*) reg_data;
-    mca_btl_ofi_reg_t *ur = (mca_btl_ofi_reg_t*) reg;
-
-    rc = fi_mr_reg(btl->domain, base, size, access_flags, 0,
-                   (uint64_t) reg, 0, &ur->ur_mr, NULL);
+    rc = fi_mr_regattr(btl->domain, &attr, 0, &ur->ur_mr);
     if (0 != rc) {
+        ur->ur_mr = NULL;
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
+    if (btl->use_fi_mr_bind) {
+        BTL_VERBOSE(("binding mr to endpoint"));
+        rc = fi_mr_bind(ur->ur_mr, &btl->ofi_endpoint->fid, 0ULL);
+        if (FI_SUCCESS != rc) {
+            return OPAL_ERR_OUT_OF_RESOURCE;
+        }
+        rc = fi_mr_enable(ur->ur_mr);
+        if (FI_SUCCESS != rc) {
+            return OPAL_ERR_OUT_OF_RESOURCE;
+        }
+    }
+
     ur->handle.rkey = fi_mr_key(ur->ur_mr);
     ur->handle.desc = fi_mr_desc(ur->ur_mr);
 
@@ -250,14 +314,14 @@ int mca_btl_ofi_reg_mem (void *reg_data, void *base, size_t size, mca_rcache_bas
     return OPAL_SUCCESS;
 }
 
-int mca_btl_ofi_dereg_mem (void *reg_data, mca_rcache_base_registration_t *reg)
+int mca_btl_ofi_dereg_mem(void *reg_data, mca_rcache_base_registration_t *reg)
 {
-    mca_btl_ofi_reg_t *ur = (mca_btl_ofi_reg_t*)reg;
+    mca_btl_ofi_reg_t *ur = (mca_btl_ofi_reg_t *) reg;
 
     if (ur->ur_mr != NULL) {
         if (0 != fi_close(&ur->ur_mr->fid)) {
-            BTL_ERROR(("%s: error unpinning memory mr=%p: %s",
-                       __func__, (void*) ur->ur_mr, strerror(errno)));
+            BTL_ERROR(("%s: error unpinning memory mr=%p: %s", __func__, (void *) ur->ur_mr,
+                       strerror(errno)));
             return OPAL_ERROR;
         }
     }
@@ -269,7 +333,7 @@ int mca_btl_ofi_dereg_mem (void *reg_data, mca_rcache_base_registration_t *reg)
  * Cleanup/release module resources.
  */
 
-int mca_btl_ofi_finalize (mca_btl_base_module_t* btl)
+int mca_btl_ofi_finalize(mca_btl_base_module_t *btl)
 {
     int i;
     mca_btl_ofi_module_t *ofi_btl = (mca_btl_ofi_module_t *) btl;
@@ -279,7 +343,7 @@ int mca_btl_ofi_finalize (mca_btl_base_module_t* btl)
 
     /* clear the rcache */
     if (ofi_btl->rcache) {
-        mca_rcache_base_module_destroy (ofi_btl->rcache);
+        mca_rcache_base_module_destroy(ofi_btl->rcache);
         ofi_btl->rcache = NULL;
     }
 
@@ -290,7 +354,7 @@ int mca_btl_ofi_finalize (mca_btl_base_module_t* btl)
     }
 
     /* loop over all the contexts */
-    for (i=0; i < ofi_btl->num_contexts; i++) {
+    for (i = 0; i < ofi_btl->num_contexts; i++) {
         mca_btl_ofi_context_finalize(&ofi_btl->contexts[i], ofi_btl->is_scalable_ep);
     }
     free(ofi_btl->contexts);
@@ -317,8 +381,8 @@ int mca_btl_ofi_finalize (mca_btl_base_module_t* btl)
     }
 
     /* clean up any leftover endpoints */
-    OPAL_LIST_FOREACH_SAFE(endpoint, next, &ofi_btl->endpoints, mca_btl_ofi_endpoint_t) {
-        opal_list_remove_item (&ofi_btl->endpoints, &endpoint->super);
+    OPAL_LIST_FOREACH_SAFE (endpoint, next, &ofi_btl->endpoints, mca_btl_ofi_endpoint_t) {
+        opal_list_remove_item(&ofi_btl->endpoints, &endpoint->super);
         OBJ_RELEASE(endpoint);
     }
 
@@ -326,40 +390,31 @@ int mca_btl_ofi_finalize (mca_btl_base_module_t* btl)
     OBJ_DESTRUCT(&ofi_btl->id_to_endpoint);
     OBJ_DESTRUCT(&ofi_btl->module_lock);
 
-    free (btl);
+    free(btl);
 
     return OPAL_SUCCESS;
 }
 
 /* Post wildcard recvs on the rx context. */
-int mca_btl_ofi_post_recvs (mca_btl_base_module_t *module,
-                            mca_btl_ofi_context_t *context,
-                            int count)
+int mca_btl_ofi_post_recvs(mca_btl_base_module_t *module, mca_btl_ofi_context_t *context, int count)
 {
     int i;
     int rc;
     mca_btl_ofi_base_frag_t *frag;
     mca_btl_ofi_frag_completion_t *comp;
 
-    for (i=0; i < count; i++) {
-        frag = (mca_btl_ofi_base_frag_t*) mca_btl_ofi_alloc(module,
-                                                     NULL,
-                                                     0,
-                                                     MCA_BTL_OFI_FRAG_SIZE,
-                                                     MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
+    for (i = 0; i < count; i++) {
+        frag = (mca_btl_ofi_base_frag_t *) mca_btl_ofi_alloc(module, NULL, 0, MCA_BTL_OFI_FRAG_SIZE,
+                                                             MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
         if (NULL == frag) {
             BTL_ERROR(("cannot allocate recv frag."));
             return OPAL_ERROR;
         }
 
-        comp = mca_btl_ofi_frag_completion_alloc (module,
-                                                  context,
-                                                  frag,
-                                                  MCA_BTL_OFI_TYPE_RECV);
-
-        rc = fi_recv (context->rx_ctx, &frag->hdr, MCA_BTL_OFI_RECV_SIZE,
-                      NULL, FI_ADDR_UNSPEC, &comp->comp_ctx);
+        comp = mca_btl_ofi_frag_completion_alloc(module, context, frag, MCA_BTL_OFI_TYPE_RECV);
 
+        OFI_RETRY_UNTIL_DONE(fi_recv(context->rx_ctx, &frag->hdr, MCA_BTL_OFI_RECV_SIZE, NULL, FI_ADDR_UNSPEC,
+                             &comp->comp_ctx), rc);
         if (FI_SUCCESS != rc) {
             BTL_ERROR(("cannot post recvs"));
             return OPAL_ERROR;
@@ -369,12 +424,12 @@ int mca_btl_ofi_post_recvs (mca_btl_base_module_t *module,
 }
 
 /* Allocate and fill out the module capabilities according to operation mode. */
-mca_btl_ofi_module_t * mca_btl_ofi_module_alloc (int mode)
+mca_btl_ofi_module_t *mca_btl_ofi_module_alloc(int mode)
 {
     mca_btl_ofi_module_t *module;
 
     /* allocate module */
-    module = (mca_btl_ofi_module_t*) calloc(1, sizeof(mca_btl_ofi_module_t));
+    module = (mca_btl_ofi_module_t *) calloc(1, sizeof(mca_btl_ofi_module_t));
     if (NULL == module) {
         return NULL;
     }
@@ -384,24 +439,24 @@ mca_btl_ofi_module_t * mca_btl_ofi_module_alloc (int mode)
 
     if (mode == MCA_BTL_OFI_MODE_ONE_SIDED || mode == MCA_BTL_OFI_MODE_FULL_SUPPORT) {
 
-        module->super.btl_put            = mca_btl_ofi_put;
-        module->super.btl_get            = mca_btl_ofi_get;
-        module->super.btl_atomic_op      = mca_btl_ofi_aop;
-        module->super.btl_atomic_fop     = mca_btl_ofi_afop;
-        module->super.btl_atomic_cswap   = mca_btl_ofi_acswap;
-        module->super.btl_flush          = mca_btl_ofi_flush;
+        module->super.btl_put = mca_btl_ofi_put;
+        module->super.btl_get = mca_btl_ofi_get;
+        module->super.btl_atomic_op = mca_btl_ofi_aop;
+        module->super.btl_atomic_fop = mca_btl_ofi_afop;
+        module->super.btl_atomic_cswap = mca_btl_ofi_acswap;
+        module->super.btl_flush = mca_btl_ofi_flush;
 
-        module->super.btl_register_mem   = mca_btl_ofi_register_mem;
+        module->super.btl_register_mem = mca_btl_ofi_register_mem;
         module->super.btl_deregister_mem = mca_btl_ofi_deregister_mem;
 
-        module->super.btl_flags         |= MCA_BTL_FLAGS_ATOMIC_FOPS |
-                                           MCA_BTL_FLAGS_ATOMIC_OPS |
-                                           MCA_BTL_FLAGS_RDMA;
+        /* btl/ofi support remote completion because it required FI_DELIVERY_COMPLETE capability
+         */
+        module->super.btl_flags |= MCA_BTL_FLAGS_ATOMIC_FOPS | MCA_BTL_FLAGS_ATOMIC_OPS
+                                   | MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_RDMA_REMOTE_COMPLETION;
 
-        module->super.btl_atomic_flags   = MCA_BTL_ATOMIC_SUPPORTS_ADD |
-                                           MCA_BTL_ATOMIC_SUPPORTS_SWAP |
-                                           MCA_BTL_ATOMIC_SUPPORTS_CSWAP |
-                                           MCA_BTL_ATOMIC_SUPPORTS_32BIT ;
+        module->super.btl_atomic_flags = MCA_BTL_ATOMIC_SUPPORTS_ADD | MCA_BTL_ATOMIC_SUPPORTS_SWAP
+                                         | MCA_BTL_ATOMIC_SUPPORTS_CSWAP
+                                         | MCA_BTL_ATOMIC_SUPPORTS_32BIT;
 
         module->super.btl_put_limit = 1 << 23;
         module->super.btl_put_alignment = 0;
@@ -409,26 +464,25 @@ mca_btl_ofi_module_t * mca_btl_ofi_module_alloc (int mode)
         module->super.btl_get_limit = 1 << 23;
         module->super.btl_get_alignment = 0;
 
-        module->super.btl_registration_handle_size =
-                                sizeof(mca_btl_base_registration_handle_t);
+        module->super.btl_registration_handle_size = sizeof(mca_btl_base_registration_handle_t);
     }
 
     if (mode == MCA_BTL_OFI_MODE_TWO_SIDED || mode == MCA_BTL_OFI_MODE_FULL_SUPPORT) {
 
-        module->super.btl_alloc          = mca_btl_ofi_alloc;
-        module->super.btl_free           = mca_btl_ofi_free;
-        module->super.btl_prepare_src    = mca_btl_ofi_prepare_src;
+        module->super.btl_alloc = mca_btl_ofi_alloc;
+        module->super.btl_free = mca_btl_ofi_free;
+        module->super.btl_prepare_src = mca_btl_ofi_prepare_src;
 
-        module->super.btl_send           = mca_btl_ofi_send;
+        module->super.btl_send = mca_btl_ofi_send;
 
-        module->super.btl_flags         |= MCA_BTL_FLAGS_SEND;
-        module->super.btl_eager_limit    = MCA_BTL_OFI_FRAG_SIZE;
-        module->super.btl_max_send_size  = MCA_BTL_OFI_FRAG_SIZE;
+        module->super.btl_flags |= MCA_BTL_FLAGS_SEND;
+        module->super.btl_eager_limit = MCA_BTL_OFI_FRAG_SIZE;
+        module->super.btl_max_send_size = MCA_BTL_OFI_FRAG_SIZE;
         module->super.btl_rndv_eager_limit = MCA_BTL_OFI_FRAG_SIZE;
 
         /* If two sided is enabled, we expected that the user knows exactly what
          * they want. We bump the priority to maximum, making this BTL the default. */
-        module->super.btl_exclusivity    = MCA_BTL_EXCLUSIVITY_HIGH;
+        module->super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_HIGH;
     }
 
     if (mode == MCA_BTL_OFI_MODE_FULL_SUPPORT) {
@@ -441,9 +495,8 @@ mca_btl_ofi_module_t * mca_btl_ofi_module_alloc (int mode)
 
 mca_btl_ofi_module_t mca_btl_ofi_module_template = {
     .super = {
-        .btl_component      = &mca_btl_ofi_component.super,
-        .btl_add_procs      = mca_btl_ofi_add_procs,
-        .btl_del_procs      = mca_btl_ofi_del_procs,
-        .btl_finalize       = mca_btl_ofi_finalize,
-   }
-};
+        .btl_component = &mca_btl_ofi_component.super,
+        .btl_add_procs = mca_btl_ofi_add_procs,
+        .btl_del_procs = mca_btl_ofi_del_procs,
+        .btl_finalize = mca_btl_ofi_finalize,
+    }};
diff --git a/opal/mca/btl/ofi/btl_ofi_rdma.c b/opal/mca/btl/ofi/btl_ofi_rdma.c
index f2728e36dc0..5e7d0b082ed 100644
--- a/opal/mca/btl/ofi/btl_ofi_rdma.c
+++ b/opal/mca/btl/ofi/btl_ofi_rdma.c
@@ -3,6 +3,9 @@
  * Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2018      Intel, Inc, All rights reserved
+ * Copyright (c) 2021      Triad National Security, LLC. All rights
+ *                         reserved.
+ *
  *
  * $COPYRIGHT$
  *
@@ -13,20 +16,14 @@
 
 #include "btl_ofi_rdma.h"
 
-OBJ_CLASS_INSTANCE(mca_btl_ofi_rdma_completion_t,
-                   opal_free_list_item_t,
-                   NULL,
-                   NULL);
-
-mca_btl_ofi_rdma_completion_t *mca_btl_ofi_rdma_completion_alloc (
-                                         mca_btl_base_module_t *btl,
-                                         mca_btl_base_endpoint_t *endpoint,
-                                         mca_btl_ofi_context_t *ofi_context,
-                                         void *local_address,
-                                         mca_btl_base_registration_handle_t *local_handle,
-                                         mca_btl_base_rdma_completion_fn_t cbfunc,
-                                         void *cbcontext, void *cbdata,
-                                         int type)
+OBJ_CLASS_INSTANCE(mca_btl_ofi_rdma_completion_t, opal_free_list_item_t, NULL, NULL);
+
+mca_btl_ofi_rdma_completion_t *
+mca_btl_ofi_rdma_completion_alloc(mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
+                                  mca_btl_ofi_context_t *ofi_context, void *local_address,
+                                  mca_btl_base_registration_handle_t *local_handle,
+                                  mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext,
+                                  void *cbdata, int type)
 {
     assert(btl);
     assert(endpoint);
@@ -34,7 +31,7 @@ mca_btl_ofi_rdma_completion_t *mca_btl_ofi_rdma_completion_alloc (
 
     mca_btl_ofi_rdma_completion_t *comp;
 
-    comp = (mca_btl_ofi_rdma_completion_t*) opal_free_list_get(&ofi_context->rdma_comp_list);
+    comp = (mca_btl_ofi_rdma_completion_t *) opal_free_list_get(&ofi_context->rdma_comp_list);
     assert(comp);
 
     comp->base.btl = btl;
@@ -54,108 +51,107 @@ mca_btl_ofi_rdma_completion_t *mca_btl_ofi_rdma_completion_alloc (
     return comp;
 }
 
-int mca_btl_ofi_get (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
-                      uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                      mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-                      int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
+int mca_btl_ofi_get(mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
+                    void *local_address, uint64_t remote_address,
+                    mca_btl_base_registration_handle_t *local_handle,
+                    mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
+                    int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext,
+                    void *cbdata)
 {
 
     int rc;
     mca_btl_ofi_rdma_completion_t *comp;
 
     mca_btl_ofi_module_t *ofi_btl = (mca_btl_ofi_module_t *) btl;
-    mca_btl_ofi_endpoint_t *btl_endpoint = (mca_btl_ofi_endpoint_t*) endpoint;
+    mca_btl_ofi_endpoint_t *btl_endpoint = (mca_btl_ofi_endpoint_t *) endpoint;
     mca_btl_ofi_context_t *ofi_context;
 
+    MCA_BTL_OFI_NUM_RDMA_INC(ofi_btl);
+
     ofi_context = get_ofi_context(ofi_btl);
 
     /* create completion context */
-    comp = mca_btl_ofi_rdma_completion_alloc(btl, endpoint,
-                                             ofi_context,
-                                             local_address,
-                                             local_handle,
-                                             cbfunc, cbcontext, cbdata,
+    comp = mca_btl_ofi_rdma_completion_alloc(btl, endpoint, ofi_context, local_address,
+                                             local_handle, cbfunc, cbcontext, cbdata,
                                              MCA_BTL_OFI_TYPE_GET);
 
     remote_address = (remote_address - (uint64_t) remote_handle->base_addr);
 
     /* Remote write data across the wire */
-    rc = fi_read(ofi_context->tx_ctx,
-                local_address, size,   /* payload */
-                local_handle->desc,
-                btl_endpoint->peer_addr,
-                remote_address, remote_handle->rkey,
-                &comp->comp_ctx);       /* completion context */
+    rc = fi_read(ofi_context->tx_ctx, local_address, size, /* payload */
+                 (NULL == local_handle ? NULL : local_handle->desc), 
+                 btl_endpoint->peer_addr, 
+                 remote_address, 
+                 remote_handle->rkey,
+                 &comp->comp_ctx); /* completion context */
 
     if (-FI_EAGAIN == rc) {
-        opal_free_list_return(comp->base.my_list, (opal_free_list_item_t*) comp);
+        MCA_BTL_OFI_NUM_RDMA_DEC(ofi_btl);
+        opal_free_list_return(comp->base.my_list, (opal_free_list_item_t *) comp);
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
     if (0 != rc) {
-        opal_free_list_return(comp->base.my_list, (opal_free_list_item_t*) comp);
+        MCA_BTL_OFI_NUM_RDMA_DEC(ofi_btl);
+        opal_free_list_return(comp->base.my_list, (opal_free_list_item_t *) comp);
         BTL_ERROR(("fi_read failed with %d:%s", rc, fi_strerror(-rc)));
         MCA_BTL_OFI_ABORT();
     }
 
-    MCA_BTL_OFI_NUM_RDMA_INC(ofi_btl);
 
     return OPAL_SUCCESS;
 }
 
-int mca_btl_ofi_put (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
-                      uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                      mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-                      int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
+int mca_btl_ofi_put(mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
+                    void *local_address, uint64_t remote_address,
+                    mca_btl_base_registration_handle_t *local_handle,
+                    mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
+                    int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext,
+                    void *cbdata)
 {
     int rc;
     mca_btl_ofi_module_t *ofi_btl = (mca_btl_ofi_module_t *) btl;
-    mca_btl_ofi_endpoint_t *btl_endpoint = (mca_btl_ofi_endpoint_t*) endpoint;
+    mca_btl_ofi_endpoint_t *btl_endpoint = (mca_btl_ofi_endpoint_t *) endpoint;
     mca_btl_ofi_context_t *ofi_context;
 
+    MCA_BTL_OFI_NUM_RDMA_INC(ofi_btl);
+
     ofi_context = get_ofi_context(ofi_btl);
 
     /* create completion context */
     mca_btl_ofi_rdma_completion_t *comp;
-    comp = mca_btl_ofi_rdma_completion_alloc(btl, endpoint,
-                                             ofi_context,
-                                             local_address,
-                                             local_handle,
-                                             cbfunc, cbcontext, cbdata,
+    comp = mca_btl_ofi_rdma_completion_alloc(btl, endpoint, ofi_context, local_address,
+                                             local_handle, cbfunc, cbcontext, cbdata,
                                              MCA_BTL_OFI_TYPE_PUT);
 
     remote_address = (remote_address - (uint64_t) remote_handle->base_addr);
 
     /* Remote write data across the wire */
-    rc = fi_write(ofi_context->tx_ctx,
-                  local_address, size,   /* payload */
-                  local_handle->desc,
-                  btl_endpoint->peer_addr,
-                  remote_address, remote_handle->rkey,
-                  &comp->comp_ctx);       /* completion context */
+    rc = fi_write(ofi_context->tx_ctx, local_address, size, /* payload */
+                  local_handle->desc, btl_endpoint->peer_addr, remote_address, remote_handle->rkey,
+                  &comp->comp_ctx); /* completion context */
 
     if (-FI_EAGAIN == rc) {
-        opal_free_list_return(comp->base.my_list, (opal_free_list_item_t*) comp);
+        MCA_BTL_OFI_NUM_RDMA_DEC(ofi_btl);
+        opal_free_list_return(comp->base.my_list, (opal_free_list_item_t *) comp);
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
     if (0 != rc) {
-        opal_free_list_return(comp->base.my_list, (opal_free_list_item_t*) comp);
+        MCA_BTL_OFI_NUM_RDMA_DEC(ofi_btl);
+        opal_free_list_return(comp->base.my_list, (opal_free_list_item_t *) comp);
         BTL_ERROR(("fi_write failed with %d:%s", rc, fi_strerror(-rc)));
         MCA_BTL_OFI_ABORT();
     }
 
-    MCA_BTL_OFI_NUM_RDMA_INC(ofi_btl);
-
     return OPAL_SUCCESS;
-
 }
 
-int mca_btl_ofi_flush (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint)
+int mca_btl_ofi_flush(mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint)
 {
     mca_btl_ofi_module_t *ofi_btl = (mca_btl_ofi_module_t *) btl;
 
-    while(ofi_btl->outstanding_rdma > 0) {
+    while (ofi_btl->outstanding_rdma > 0) {
         (void) mca_btl_ofi_component.super.btl_progress();
     }
 
diff --git a/opal/mca/btl/ofi/btl_ofi_rdma.h b/opal/mca/btl/ofi/btl_ofi_rdma.h
index b4e1965081e..0091f7d398a 100644
--- a/opal/mca/btl/ofi/btl_ofi_rdma.h
+++ b/opal/mca/btl/ofi/btl_ofi_rdma.h
@@ -19,24 +19,19 @@
 #include "btl_ofi.h"
 #include "btl_ofi_endpoint.h"
 
-mca_btl_ofi_rdma_completion_t *mca_btl_ofi_rdma_completion_alloc (
-                                         mca_btl_base_module_t *btl,
-                                         mca_btl_base_endpoint_t *endpoint,
-                                         mca_btl_ofi_context_t *ofi_context,
-                                         void *local_address,
-                                         mca_btl_base_registration_handle_t *local_handle,
-                                         mca_btl_base_rdma_completion_fn_t cbfunc,
-                                         void *cbcontext, void *cbdata,
-                                         int type);
+mca_btl_ofi_rdma_completion_t *
+mca_btl_ofi_rdma_completion_alloc(mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
+                                  mca_btl_ofi_context_t *ofi_context, void *local_address,
+                                  mca_btl_base_registration_handle_t *local_handle,
+                                  mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext,
+                                  void *cbdata, int type);
 
-#define MCA_BTL_OFI_NUM_RDMA_INC(module)                                                \
-            OPAL_THREAD_ADD_FETCH64(&(module)->outstanding_rdma, 1);                    \
-            if (module->outstanding_rdma > mca_btl_ofi_component.progress_threshold){   \
-                mca_btl_ofi_component.super.btl_progress();                             \
-            }
+#define MCA_BTL_OFI_NUM_RDMA_INC(module)                                       \
+    OPAL_THREAD_ADD_FETCH64(&(module)->outstanding_rdma, 1);                   \
+    if (module->outstanding_rdma > mca_btl_ofi_component.progress_threshold) { \
+        mca_btl_ofi_component.super.btl_progress();                            \
+    }
 
-#define MCA_BTL_OFI_NUM_RDMA_DEC(module)                            \
-            OPAL_THREAD_ADD_FETCH64(&(module)->outstanding_rdma, -1);
+#define MCA_BTL_OFI_NUM_RDMA_DEC(module) OPAL_THREAD_ADD_FETCH64(&(module)->outstanding_rdma, -1);
 
 #endif /* !defined(BTL_OFI_RDMA_H) */
-
diff --git a/opal/mca/btl/ofi/configure.m4 b/opal/mca/btl/ofi/configure.m4
index 23c7370e48e..18c090112ad 100644
--- a/opal/mca/btl/ofi/configure.m4
+++ b/opal/mca/btl/ofi/configure.m4
@@ -1,4 +1,4 @@
-# -*- shell-script -*-
+# -*- autoconf -*-
 #
 # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
 #                         University Research and Technology
@@ -15,7 +15,7 @@
 # Copyright (c) 2011-2018 Los Alamos National Security, LLC.
 #                         All rights reserved.
 # Copyright (c) 2018      Intel, inc. All rights reserved
-#
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -30,23 +30,33 @@
 # support, otherwise executes action-if-not-found
 
 AC_DEFUN([MCA_opal_btl_ofi_CONFIG],[
-    OPAL_VAR_SCOPE_PUSH([opal_btl_ofi_happy CPPFLAGS_save])
+    OPAL_VAR_SCOPE_PUSH([btl_ofi_happy CPPFLAGS_save])
 
     AC_CONFIG_FILES([opal/mca/btl/ofi/Makefile])
 
     # Check for OFI
-    OPAL_CHECK_OFI
+    OPAL_CHECK_OFI([btl_ofi], [btl_ofi_happy=1], [btl_ofi_happy=0])
 
-    opal_btl_ofi_happy=0
-    AS_IF([test "$opal_ofi_happy" = "yes"],
-          [CPPFLAGS_save=$CPPFLAGS
-           CPPFLAGS="$opal_ofi_CPPFLAGS $CPPFLAGS"
-           AC_CHECK_DECL([FI_MR_VIRT_ADDR], [opal_btl_ofi_happy=1], [],
+    AS_IF([test ${btl_ofi_happy} -eq 1],
+          [CPPFLAGS_save=${CPPFLAGS}
+           OPAL_FLAGS_APPEND_UNIQ([CPPFLAGS], [${btl_ofi_CPPFLAGS}])
+           AC_CHECK_DECL([FI_MR_VIRT_ADDR], [], [btl_ofi_happy=0],
                          [#include <rdma/fabric.h>])
-           CPPFLAGS=$CPPFLAGS_save])
-    AS_IF([test $opal_btl_ofi_happy -eq 1],
+           CPPFLAGS=${CPPFLAGS_save}])
+
+    AS_IF([test ${btl_ofi_happy} -eq 1],
+          [OPAL_MCA_CHECK_DEPENDENCY([opal], [btl], [ofi], [opal], [common], [ofi])])
+
+    AS_IF([test ${btl_ofi_happy} -eq 1],
           [$1],
-          [$2])
+          [AS_IF([test -n "${with_ofi}" -a "${with_ofi}" != "no"],
+                 [AC_MSG_WARN([OFI libfabric support requested (via --with-ofi or --with-libfabric), but not found.])
+                  AC_MSG_ERROR([Cannot continue.])])
+           $2])
+
+    AC_SUBST([btl_ofi_CPPFLAGS])
+    AC_SUBST([btl_ofi_LDFLAGS])
+    AC_SUBST([btl_ofi_LIBS])
 
     OPAL_VAR_SCOPE_POP
 ])dnl
diff --git a/opal/mca/btl/portals4/Makefile.am b/opal/mca/btl/portals4/Makefile.am
index f6c9351e336..7d449e0ef4d 100644
--- a/opal/mca/btl/portals4/Makefile.am
+++ b/opal/mca/btl/portals4/Makefile.am
@@ -48,7 +48,7 @@ local_sources = \
 mcacomponentdir = $(opallibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_btl_portals4_la_SOURCES = $(local_sources)
-mca_btl_portals4_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
+mca_btl_portals4_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
     $(btl_portals4_LIBS)
 mca_btl_portals4_la_LDFLAGS = -module -avoid-version $(btl_portals4_LDFLAGS)
 
diff --git a/opal/mca/btl/portals4/btl_portals4.c b/opal/mca/btl/portals4/btl_portals4.c
index bf63bef119f..d71c69a8943 100644
--- a/opal/mca/btl/portals4/btl_portals4.c
+++ b/opal/mca/btl/portals4/btl_portals4.c
@@ -24,119 +24,114 @@
 
 #include "opal_config.h"
 
+#include "opal_stdint.h"
+#include <stdio.h>
 #include <sys/types.h>
 #include <unistd.h>
-#include <stdio.h>
-#include "opal_stdint.h"
 
 #include "opal/class/opal_bitmap.h"
 #include "opal/constants.h"
-#include "opal/mca/btl/btl.h"
 #include "opal/datatype/opal_convertor.h"
-#include "opal/util/proc.h"
+#include "opal/mca/btl/btl.h"
 #include "opal/mca/pmix/pmix-internal.h"
+#include "opal/util/proc.h"
 
 #include "btl_portals4.h"
+#include "btl_portals4_frag.h"
 #include "btl_portals4_recv.h"
 
-
-mca_btl_base_registration_handle_t *
-mca_btl_portals4_register_mem(mca_btl_base_module_t *btl,
-                              mca_btl_base_endpoint_t *endpoint,
-                              void *base,
-                              size_t size,
-                              uint32_t flags);
+mca_btl_base_registration_handle_t *mca_btl_portals4_register_mem(mca_btl_base_module_t *btl,
+                                                                  mca_btl_base_endpoint_t *endpoint,
+                                                                  void *base, size_t size,
+                                                                  uint32_t flags);
 
 int mca_btl_portals4_deregister_mem(mca_btl_base_module_t *btl,
                                     mca_btl_base_registration_handle_t *handle);
 
 mca_btl_portals4_module_t mca_btl_portals4_module = {
-    .super = {
-        .btl_component = &mca_btl_portals4_component.super,
-
-        /* NOTE: All the default values are set in
-           component_open() */
-
-        .btl_add_procs = mca_btl_portals4_add_procs,
-        .btl_del_procs = mca_btl_portals4_del_procs,
-        .btl_finalize = mca_btl_portals4_finalize,
-        .btl_alloc = mca_btl_portals4_alloc,
-        .btl_free = mca_btl_portals4_free,
-        .btl_prepare_src = mca_btl_portals4_prepare_src,
-        .btl_register_mem = mca_btl_portals4_register_mem,
-        .btl_deregister_mem = mca_btl_portals4_deregister_mem,
-        .btl_send = mca_btl_portals4_send,
-        .btl_get = mca_btl_portals4_get,
-        .btl_dump = mca_btl_base_dump,
-    },
+    .super =
+        {
+            .btl_component = &mca_btl_portals4_component.super,
+
+            /* NOTE: All the default values are set in
+               component_open() */
+
+            .btl_add_procs = mca_btl_portals4_add_procs,
+            .btl_del_procs = mca_btl_portals4_del_procs,
+            .btl_finalize = mca_btl_portals4_finalize,
+            .btl_alloc = mca_btl_portals4_alloc,
+            .btl_free = mca_btl_portals4_free,
+            .btl_prepare_src = mca_btl_portals4_prepare_src,
+            .btl_register_mem = mca_btl_portals4_register_mem,
+            .btl_deregister_mem = mca_btl_portals4_deregister_mem,
+            .btl_send = mca_btl_portals4_send,
+            .btl_get = mca_btl_portals4_get,
+            .btl_dump = mca_btl_base_dump,
+        },
 };
 
-static int
-btl_portals4_init_interface(void)
+static int btl_portals4_init_interface(void)
 {
     mca_btl_portals4_module_t *portals4_btl;
     unsigned int ret, interface;
     ptl_md_t md;
     ptl_me_t me;
 
-// The initialisation of EQ, PT and ME must be done after the SetMap !
-    for (interface=0; interface<mca_btl_portals4_component.num_btls; interface++) {
+    // The initialisation of EQ, PT and ME must be done after the SetMap !
+    for (interface = 0; interface < mca_btl_portals4_component.num_btls; interface++) {
         portals4_btl = mca_btl_portals4_component.btls[interface];
 
         /* create event queue */
-        ret = PtlEQAlloc(portals4_btl->portals_ni_h,
-                     mca_btl_portals4_component.recv_queue_size,
-                     &portals4_btl->recv_eq_h);
+        ret = PtlEQAlloc(portals4_btl->portals_ni_h, mca_btl_portals4_component.recv_queue_size,
+                         &portals4_btl->recv_eq_h);
         if (PTL_OK != ret) {
             opal_output_verbose(1, opal_btl_base_framework.framework_output,
-                            "%s:%d: PtlEQAlloc failed for NI %d: %d",
-                            __FILE__, __LINE__, interface, ret);
+                                "%s:%d: PtlEQAlloc failed for NI %d: %d", __FILE__, __LINE__,
+                                interface, ret);
             goto error;
         }
         mca_btl_portals4_component.eqs_h[interface] = portals4_btl->recv_eq_h;
         OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
-            "PtlEQAlloc (recv_eq=%d) OK for NI %d\n", portals4_btl->recv_eq_h, interface));
+                             "PtlEQAlloc (recv_eq=%d) OK for NI %d\n", portals4_btl->recv_eq_h,
+                             interface));
 
         /* Create recv_idx portal table entry */
-        ret = PtlPTAlloc(portals4_btl->portals_ni_h,
-                     PTL_PT_ONLY_TRUNCATE,
-                     portals4_btl->recv_eq_h,
-                     REQ_BTL_TABLE_ID,
-                     &portals4_btl->recv_idx);
+        ret = PtlPTAlloc(portals4_btl->portals_ni_h, PTL_PT_ONLY_TRUNCATE, portals4_btl->recv_eq_h,
+                         REQ_BTL_TABLE_ID, &portals4_btl->recv_idx);
         if (PTL_OK != ret) {
             opal_output_verbose(1, opal_btl_base_framework.framework_output,
-                            "%s:%d: PtlPTAlloc failed for NI %d: %d",
-                            __FILE__, __LINE__, interface, ret);
+                                "%s:%d: PtlPTAlloc failed for NI %d: %d", __FILE__, __LINE__,
+                                interface, ret);
             goto error;
         }
         OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
-            "PtlPTAlloc (recv_idx) OK for NI %d recv_idx=%d", interface, portals4_btl->recv_idx));
+                             "PtlPTAlloc (recv_idx) OK for NI %d recv_idx=%d", interface,
+                             portals4_btl->recv_idx));
 
         if (portals4_btl->recv_idx != REQ_BTL_TABLE_ID) {
             opal_output_verbose(1, opal_btl_base_framework.framework_output,
-                            "%s:%d: PtlPTAlloc did not allocate the requested PT: %d",
-                           __FILE__, __LINE__, portals4_btl->recv_idx);
+                                "%s:%d: PtlPTAlloc did not allocate the requested PT: %d", __FILE__,
+                                __LINE__, portals4_btl->recv_idx);
             goto error;
         }
 
         /* bind zero-length md for sending acks */
-        md.start     = NULL;
-        md.length    = 0;
-        md.options   = 0;
+        md.start = NULL;
+        md.length = 0;
+        md.options = 0;
         md.eq_handle = PTL_EQ_NONE;
         md.ct_handle = PTL_CT_NONE;
 
-        ret = PtlMDBind(portals4_btl->portals_ni_h,
-                    &md,
-                    &portals4_btl->zero_md_h);
+        ret = PtlMDBind(portals4_btl->portals_ni_h, &md, &portals4_btl->zero_md_h);
         if (PTL_OK != ret) {
             opal_output_verbose(1, opal_btl_base_framework.framework_output,
-                            "%s:%d: PtlMDBind failed for NI %d: %d",
-                            __FILE__, __LINE__, interface, ret);
+                                "%s:%d: PtlMDBind failed for NI %d: %d", __FILE__, __LINE__,
+                                interface, ret);
             goto error;
         }
         OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
-            "PtlMDBind (zero-length md=%d) OK for NI %d", portals4_btl->zero_md_h, interface));
+                             "PtlMDBind (zero-length md=%d) OK for NI %d", portals4_btl->zero_md_h,
+                             interface));
 
         /* Bind MD across all memory */
         md.start = 0;
@@ -145,13 +140,11 @@ btl_portals4_init_interface(void)
         md.eq_handle = portals4_btl->recv_eq_h;
         md.ct_handle = PTL_CT_NONE;
 
-        ret = PtlMDBind(portals4_btl->portals_ni_h,
-                    &md,
-                    &portals4_btl->send_md_h);
+        ret = PtlMDBind(portals4_btl->portals_ni_h, &md, &portals4_btl->send_md_h);
         if (PTL_OK != ret) {
             opal_output_verbose(1, opal_btl_base_framework.framework_output,
-                            "%s:%d: PtlMDBind failed for NI %d: %d\n",
-                            __FILE__, __LINE__, interface, ret);
+                                "%s:%d: PtlMDBind failed for NI %d: %d\n", __FILE__, __LINE__,
+                                interface, ret);
             goto error;
         }
 
@@ -161,10 +154,8 @@ btl_portals4_init_interface(void)
         me.ct_handle = PTL_CT_NONE;
         me.min_free = 0;
         me.uid = PTL_UID_ANY;
-        me.options = PTL_ME_OP_PUT |
-            PTL_ME_EVENT_LINK_DISABLE |
-            PTL_ME_EVENT_COMM_DISABLE |
-            PTL_ME_EVENT_UNLINK_DISABLE;
+        me.options = PTL_ME_OP_PUT | PTL_ME_EVENT_LINK_DISABLE | PTL_ME_EVENT_COMM_DISABLE
+                     | PTL_ME_EVENT_UNLINK_DISABLE;
         if (mca_btl_portals4_component.use_logical) {
             me.match_id.rank = PTL_RANK_ANY;
         } else {
@@ -172,81 +163,80 @@ btl_portals4_init_interface(void)
             me.match_id.phys.pid = PTL_PID_ANY;
         }
         me.match_bits = BTL_PORTALS4_LONG_MSG;
-        me.ignore_bits = BTL_PORTALS4_CONTEXT_MASK |
-            BTL_PORTALS4_SOURCE_MASK |
-            BTL_PORTALS4_TAG_MASK;
-        ret = PtlMEAppend(portals4_btl->portals_ni_h,
-                      portals4_btl->recv_idx,
-                      &me,
-                      PTL_OVERFLOW_LIST,
-                      NULL,
-                      &portals4_btl->long_overflow_me_h);
+        me.ignore_bits = BTL_PORTALS4_CONTEXT_MASK | BTL_PORTALS4_SOURCE_MASK
+                         | BTL_PORTALS4_TAG_MASK;
+        ret = PtlMEAppend(portals4_btl->portals_ni_h, portals4_btl->recv_idx, &me,
+                          PTL_OVERFLOW_LIST, NULL, &portals4_btl->long_overflow_me_h);
         if (PTL_OK != ret) {
             opal_output_verbose(1, opal_btl_base_framework.framework_output,
-                            "%s:%d: PtlMEAppend failed for NI %d: %d",
-                            __FILE__, __LINE__, interface, ret);
+                                "%s:%d: PtlMEAppend failed for NI %d: %d", __FILE__, __LINE__,
+                                interface, ret);
             goto error;
         }
-        OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "PtlMEAppend (overflow list) OK for NI %d", interface));
-    }
+        OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
+                             "PtlMEAppend (overflow list) OK for NI %d", interface));
 
-    ret = mca_btl_portals4_recv_enable(portals4_btl);
-    if (PTL_OK != ret) {
-        opal_output_verbose(1, opal_btl_base_framework.framework_output,
-                            "%s:%d: Initialization of recv buffer failed: %d",
-                            __FILE__, __LINE__, ret);
-        goto error;
+        ret = mca_btl_portals4_recv_enable(portals4_btl);
+        if (PTL_OK != ret) {
+            opal_output_verbose(1, opal_btl_base_framework.framework_output,
+                                "%s:%d: Initialization of recv buffer failed: %d", __FILE__, __LINE__,
+                                ret);
+            goto error;
+        }
     }
 
     return OPAL_SUCCESS;
 
- error:
-    opal_output_verbose(1, opal_btl_base_framework.framework_output, "Error in btl_portals4_init_interface");
+error:
+    opal_output_verbose(1, opal_btl_base_framework.framework_output,
+                        "Error in btl_portals4_init_interface");
 
-    for (interface=0; interface<mca_btl_portals4_component.num_btls; interface++) {
+    for (interface = 0; interface < mca_btl_portals4_component.num_btls; interface++) {
         portals4_btl = mca_btl_portals4_component.btls[interface];
-        if (NULL != portals4_btl) mca_btl_portals4_free_module(portals4_btl);
+        if (NULL != portals4_btl)
+            mca_btl_portals4_free_module(portals4_btl);
     }
     mca_btl_portals4_component.num_btls = 0;
-    if (NULL != mca_btl_portals4_component.btls)  free(mca_btl_portals4_component.btls);
-    if (NULL != mca_btl_portals4_component.eqs_h) free(mca_btl_portals4_component.eqs_h);
+    if (NULL != mca_btl_portals4_component.btls)
+        free(mca_btl_portals4_component.btls);
+    if (NULL != mca_btl_portals4_component.eqs_h)
+        free(mca_btl_portals4_component.eqs_h);
     mca_btl_portals4_component.btls = NULL;
     mca_btl_portals4_component.eqs_h = NULL;
 
     return OPAL_ERROR;
 }
 
-static int
-create_endpoint(int                       interface,
-                opal_proc_t              *proc,
-                mca_btl_base_endpoint_t **endpoint)
+static int create_endpoint(int interface, opal_proc_t *proc, mca_btl_base_endpoint_t **endpoint)
 {
     int ret;
     size_t size;
     ptl_process_t *id;
 
-    OPAL_MODEX_RECV(ret, &mca_btl_portals4_component.super.btl_version,
-                    &proc->proc_name, (void**) &id, &size);
+    OPAL_MODEX_RECV(ret, &mca_btl_portals4_component.super.btl_version, &proc->proc_name,
+                    (void **) &id, &size);
 
     if (OPAL_ERR_NOT_FOUND == ret) {
         OPAL_OUTPUT_VERBOSE((30, opal_btl_base_framework.framework_output,
-            "btl/portals4: Portals 4 BTL not available on peer: %s", opal_strerror(ret)));
+                             "btl/portals4: Portals 4 BTL not available on peer: %s",
+                             opal_strerror(ret)));
         return ret;
     } else if (OPAL_SUCCESS != ret) {
         opal_output_verbose(0, opal_btl_base_framework.framework_output,
-            "btl/portals4: opal_modex_recv failed: %s", opal_strerror(ret));
+                            "btl/portals4: opal_modex_recv failed: %s", opal_strerror(ret));
         return ret;
     }
-    if (size < sizeof(ptl_process_t)) {  /* no available connection */
+    if (size < sizeof(ptl_process_t)) { /* no available connection */
         return OPAL_ERROR;
     }
     if ((size % sizeof(ptl_process_t)) != 0) {
         opal_output_verbose(0, opal_btl_base_framework.framework_output,
-            "btl/portals4: invalid format in modex");
+                            "btl/portals4: invalid format in modex");
         return OPAL_ERROR;
     }
     OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
-        "btl/portals4: %d NI(s) declared in the modex", (int) (size/sizeof(ptl_process_t))));
+                         "btl/portals4: %d NI(s) declared in the modex",
+                         (int) (size / sizeof(ptl_process_t))));
 
     *endpoint = malloc(sizeof(mca_btl_base_endpoint_t));
     if (NULL == *endpoint) {
@@ -258,38 +248,37 @@ create_endpoint(int                       interface,
     return OPAL_SUCCESS;
 }
 
-static int
-create_peer_and_endpoint(int                       interface,
-                         opal_proc_t              *proc,
-                         ptl_process_t            *phys_peer,
-                         mca_btl_base_endpoint_t **endpoint)
+static int create_peer_and_endpoint(int interface, opal_proc_t *proc, ptl_process_t *phys_peer,
+                                    mca_btl_base_endpoint_t **endpoint)
 {
     int ret;
     size_t size;
     ptl_process_t *id;
 
-    OPAL_MODEX_RECV(ret, &mca_btl_portals4_component.super.btl_version,
-                    &proc->proc_name, (void**) &id, &size);
+    OPAL_MODEX_RECV(ret, &mca_btl_portals4_component.super.btl_version, &proc->proc_name,
+                    (void **) &id, &size);
 
     if (OPAL_ERR_NOT_FOUND == ret) {
         OPAL_OUTPUT_VERBOSE((30, opal_btl_base_framework.framework_output,
-            "btl/portals4: Portals 4 BTL not available on peer: %s", opal_strerror(ret)));
+                             "btl/portals4: Portals 4 BTL not available on peer: %s",
+                             opal_strerror(ret)));
         return ret;
     } else if (OPAL_SUCCESS != ret) {
         opal_output_verbose(0, opal_btl_base_framework.framework_output,
-            "btl/portals4: opal_modex_recv failed: %s", opal_strerror(ret));
+                            "btl/portals4: opal_modex_recv failed: %s", opal_strerror(ret));
         return ret;
     }
-    if (size < sizeof(ptl_process_t)) {  /* no available connection */
+    if (size < sizeof(ptl_process_t)) { /* no available connection */
         return OPAL_ERROR;
     }
     if ((size % sizeof(ptl_process_t)) != 0) {
         opal_output_verbose(0, opal_btl_base_framework.framework_output,
-            "btl/portals4: invalid format in modex");
+                            "btl/portals4: invalid format in modex");
         return OPAL_ERROR;
     }
     OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
-        "btl/portals4: %d NI(s) declared in the modex", (int) (size/sizeof(ptl_process_t))));
+                         "btl/portals4: %d NI(s) declared in the modex",
+                         (int) (size / sizeof(ptl_process_t))));
 
     /*
      * check if create_endpoint() already created the endpoint.
@@ -310,30 +299,26 @@ create_peer_and_endpoint(int                       interface,
     phys_peer->phys.pid = id[interface].phys.pid;
     phys_peer->phys.nid = id[interface].phys.nid;
     opal_output_verbose(50, opal_btl_base_framework.framework_output,
-        "logical: global rank=%d pid=%d nid=%d\n",
-        proc->proc_name.vpid, phys_peer->phys.pid, phys_peer->phys.nid);
+                        "logical: global rank=%d pid=%d nid=%d\n", proc->proc_name.vpid,
+                        phys_peer->phys.pid, phys_peer->phys.nid);
 
     return OPAL_SUCCESS;
 }
 
-static int
-create_maptable(struct mca_btl_portals4_module_t *portals4_btl,
-                size_t                            nprocs,
-                opal_proc_t                     **procs,
-                mca_btl_base_endpoint_t         **endpoint)
+static int create_maptable(struct mca_btl_portals4_module_t *portals4_btl, size_t nprocs,
+                           opal_proc_t **procs, mca_btl_base_endpoint_t **endpoint)
 {
     int ret;
     ptl_process_t *maptable;
 
     maptable = malloc(sizeof(ptl_process_t) * nprocs);
     if (NULL == maptable) {
-        opal_output_verbose(1, opal_btl_base_framework.framework_output,
-                            "%s:%d: malloc failed\n",
+        opal_output_verbose(1, opal_btl_base_framework.framework_output, "%s:%d: malloc failed\n",
                             __FILE__, __LINE__);
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
-    for (uint32_t i = 0 ; i < nprocs ; i++) {
+    for (uint32_t i = 0; i < nprocs; i++) {
         struct opal_proc_t *curr_proc;
 
         curr_proc = procs[i];
@@ -343,15 +328,13 @@ create_maptable(struct mca_btl_portals4_module_t *portals4_btl,
             opal_output_verbose(1, opal_btl_base_framework.framework_output,
                                 "Portals 4 BTL does not support heterogeneous operations.");
             opal_output_verbose(1, opal_btl_base_framework.framework_output,
-                "Proc %s architecture %x, mine %x.",
-                OPAL_NAME_PRINT(curr_proc->proc_name),
-                curr_proc->proc_arch, opal_proc_local_get()->proc_arch);
+                                "Proc %s architecture %x, mine %x.",
+                                OPAL_NAME_PRINT(curr_proc->proc_name), curr_proc->proc_arch,
+                                opal_proc_local_get()->proc_arch);
             return OPAL_ERR_NOT_SUPPORTED;
         }
 
-        ret = create_peer_and_endpoint(portals4_btl->interface_num,
-                                       curr_proc,
-                                       &maptable[i],
+        ret = create_peer_and_endpoint(portals4_btl->interface_num, curr_proc, &maptable[i],
                                        &endpoint[i]);
         if (OPAL_SUCCESS != ret) {
             opal_output_verbose(1, opal_btl_base_framework.framework_output,
@@ -361,17 +344,13 @@ create_maptable(struct mca_btl_portals4_module_t *portals4_btl,
         }
     }
 
-    ret = PtlSetMap(portals4_btl->portals_ni_h,
-                    nprocs,
-                    maptable);
+    ret = PtlSetMap(portals4_btl->portals_ni_h, nprocs, maptable);
     if (OPAL_SUCCESS != ret) {
         opal_output_verbose(1, opal_btl_base_framework.framework_output,
-                            "%s:%d: logical mapping failed: %d\n",
-                            __FILE__, __LINE__, ret);
+                            "%s:%d: logical mapping failed: %d\n", __FILE__, __LINE__, ret);
         return ret;
     }
-    opal_output_verbose(90, opal_btl_base_framework.framework_output,
-        "logical mapping OK\n");
+    opal_output_verbose(90, opal_btl_base_framework.framework_output, "logical mapping OK\n");
     free(maptable);
 
     return OPAL_SUCCESS;
@@ -379,28 +358,24 @@ create_maptable(struct mca_btl_portals4_module_t *portals4_btl,
 
 #define NEED_ALL_PROCS (mca_btl_portals4_component.use_logical)
 
-int
-mca_btl_portals4_add_procs(struct mca_btl_base_module_t* btl_base,
-                          size_t nprocs,
-                          struct opal_proc_t **procs,
-                          struct mca_btl_base_endpoint_t** btl_peer_data,
-                          opal_bitmap_t* reachable)
+int mca_btl_portals4_add_procs(struct mca_btl_base_module_t *btl_base, size_t nprocs,
+                               struct opal_proc_t **procs,
+                               struct mca_btl_base_endpoint_t **btl_peer_data,
+                               opal_bitmap_t *reachable)
 {
-    struct mca_btl_portals4_module_t* portals4_btl = (struct mca_btl_portals4_module_t*) btl_base;
+    struct mca_btl_portals4_module_t *portals4_btl = (struct mca_btl_portals4_module_t *) btl_base;
     int ret;
     size_t i;
 
     opal_output_verbose(50, opal_btl_base_framework.framework_output,
-                        "mca_btl_portals4_add_procs: Adding %d procs (%d) for NI %d",
-                        (int) nprocs,
-                        (int) portals4_btl->portals_num_procs,
-                        portals4_btl->interface_num);
+                        "mca_btl_portals4_add_procs: Adding %d procs (%d) for NI %d", (int) nprocs,
+                        (int) portals4_btl->portals_num_procs, portals4_btl->interface_num);
 
     /*
      * The PML handed us a list of procs that need Portals4
      * peer info.  Complete those procs here.
      */
-    for (i = 0 ; i < nprocs ; ++i) {
+    for (i = 0; i < nprocs; ++i) {
         struct opal_proc_t *curr_proc = procs[i];
 
         /* portals doesn't support heterogeneous yet... */
@@ -408,35 +383,32 @@ mca_btl_portals4_add_procs(struct mca_btl_base_module_t* btl_base,
             opal_output_verbose(1, opal_btl_base_framework.framework_output,
                                 "Portals 4 BTL does not support heterogeneous operations.");
             opal_output_verbose(1, opal_btl_base_framework.framework_output,
-                "Proc %s architecture %x, mine %x.",
-                OPAL_NAME_PRINT(curr_proc->proc_name),
-                curr_proc->proc_arch, opal_proc_local_get()->proc_arch);
+                                "Proc %s architecture %x, mine %x.",
+                                OPAL_NAME_PRINT(curr_proc->proc_name), curr_proc->proc_arch,
+                                opal_proc_local_get()->proc_arch);
             return OPAL_ERR_NOT_SUPPORTED;
         }
 
-        ret = create_endpoint(portals4_btl->interface_num,
-                              curr_proc,
-                              &btl_peer_data[i]);
+        ret = create_endpoint(portals4_btl->interface_num, curr_proc, &btl_peer_data[i]);
 
         OPAL_THREAD_ADD_FETCH32(&portals4_btl->portals_num_procs, 1);
         /* and here we can reach */
         opal_bitmap_set_bit(reachable, i);
 
         OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
-            "add_procs: rank=%lx nid=%x pid=%x for NI %d",
-            i,
-            btl_peer_data[i]->ptl_proc.phys.nid,
-            btl_peer_data[i]->ptl_proc.phys.pid,
-            portals4_btl->interface_num));
+                             "add_procs: rank=%lx nid=%x pid=%x for NI %d", i,
+                             btl_peer_data[i]->ptl_proc.phys.nid,
+                             btl_peer_data[i]->ptl_proc.phys.pid, portals4_btl->interface_num));
     }
 
     if (mca_btl_portals4_component.need_init && portals4_btl->portals_num_procs > 0) {
         if (mca_btl_portals4_component.use_logical) {
             ret = create_maptable(portals4_btl, nprocs, procs, btl_peer_data);
             if (OPAL_SUCCESS != ret) {
-                opal_output_verbose(1, opal_btl_base_framework.framework_output,
-                                    "%s:%d: mca_btl_portals4_add_procs::create_maptable() failed: %d\n",
-                                    __FILE__, __LINE__, ret);
+                opal_output_verbose(
+                    1, opal_btl_base_framework.framework_output,
+                    "%s:%d: mca_btl_portals4_add_procs::create_maptable() failed: %d\n", __FILE__,
+                    __LINE__, ret);
                 return ret;
             }
         }
@@ -444,8 +416,8 @@ mca_btl_portals4_add_procs(struct mca_btl_base_module_t* btl_base,
         ret = btl_portals4_init_interface();
         if (OPAL_SUCCESS != ret) {
             opal_output_verbose(1, opal_btl_base_framework.framework_output,
-                                "%s:%d: portals4 interface initialization failed: %d",
-                                __FILE__, __LINE__, ret);
+                                "%s:%d: portals4 interface initialization failed: %d", __FILE__,
+                                __LINE__, ret);
             return ret;
         }
         mca_btl_portals4_component.need_init = 0;
@@ -454,14 +426,11 @@ mca_btl_portals4_add_procs(struct mca_btl_base_module_t* btl_base,
     return OPAL_SUCCESS;
 }
 
-
-int
-mca_btl_portals4_del_procs(struct mca_btl_base_module_t *btl,
-			  size_t nprocs,
-			  struct opal_proc_t **procs,
-			  struct mca_btl_base_endpoint_t **btl_peer_data)
+int mca_btl_portals4_del_procs(struct mca_btl_base_module_t *btl, size_t nprocs,
+                               struct opal_proc_t **procs,
+                               struct mca_btl_base_endpoint_t **btl_peer_data)
 {
-    struct mca_btl_portals4_module_t* portals4_btl = (struct mca_btl_portals4_module_t*) btl;
+    struct mca_btl_portals4_module_t *portals4_btl = (struct mca_btl_portals4_module_t *) btl;
     size_t i;
 
     opal_output_verbose(50, opal_btl_base_framework.framework_output,
@@ -470,7 +439,7 @@ mca_btl_portals4_del_procs(struct mca_btl_base_module_t *btl,
 
     /* See comment in btl_portals4_endpoint.h about why we look at the
        portals4 entry in proc_endpoints instead of the peer_data */
-    for (i = 0 ; i < nprocs ; ++i) {
+    for (i = 0; i < nprocs; ++i) {
         free(btl_peer_data[i]);
         OPAL_THREAD_ADD_FETCH32(&portals4_btl->portals_num_procs, -1);
     }
@@ -478,26 +447,25 @@ mca_btl_portals4_del_procs(struct mca_btl_base_module_t *btl,
     return OPAL_SUCCESS;
 }
 
-mca_btl_base_descriptor_t*
-mca_btl_portals4_alloc(struct mca_btl_base_module_t* btl_base,
-                      struct mca_btl_base_endpoint_t* endpoint,
-                      uint8_t order,
-                      size_t size,
-                      uint32_t flags)
+mca_btl_base_descriptor_t *mca_btl_portals4_alloc(struct mca_btl_base_module_t *btl_base,
+                                                  struct mca_btl_base_endpoint_t *endpoint,
+                                                  uint8_t order, size_t size, uint32_t flags)
 {
-    struct mca_btl_portals4_module_t* portals4_btl = (struct mca_btl_portals4_module_t*) btl_base;
-    mca_btl_portals4_frag_t* frag;
+    struct mca_btl_portals4_module_t *portals4_btl = (struct mca_btl_portals4_module_t *) btl_base;
+    mca_btl_portals4_frag_t *frag;
 
     if (size <= portals4_btl->super.btl_eager_limit) {
         OPAL_BTL_PORTALS4_FRAG_ALLOC_EAGER(portals4_btl, frag);
-        if (NULL == frag) return NULL;
+        if (NULL == frag)
+            return NULL;
         frag->segments[0].base.seg_len = size;
     } else {
         OPAL_BTL_PORTALS4_FRAG_ALLOC_MAX(portals4_btl, frag);
-        if (NULL == frag) return NULL;
-        frag->segments[0].base.seg_len =
-            size <= portals4_btl->super.btl_max_send_size ?
-            size : portals4_btl->super.btl_max_send_size ;
+        if (NULL == frag)
+            return NULL;
+        frag->segments[0].base.seg_len = size <= portals4_btl->super.btl_max_send_size
+                                             ? size
+                                             : portals4_btl->super.btl_max_send_size;
     }
 
     frag->base.des_segment_count = 1;
@@ -505,34 +473,33 @@ mca_btl_portals4_alloc(struct mca_btl_base_module_t* btl_base,
     frag->base.order = MCA_BTL_NO_ORDER;
 
     OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
-	"mca_btl_portals4_alloc: %p\n", (void *) &frag->base));
+                         "mca_btl_portals4_alloc: %p\n", (void *) &frag->base));
     return &frag->base;
 }
 
-int
-mca_btl_portals4_free(struct mca_btl_base_module_t* btl_base,
-                      mca_btl_base_descriptor_t* des)
+int mca_btl_portals4_free(struct mca_btl_base_module_t *btl_base, mca_btl_base_descriptor_t *des)
 {
-    struct mca_btl_portals4_module_t* portals4_btl = (struct mca_btl_portals4_module_t*) btl_base;
-    mca_btl_portals4_frag_t* frag = (mca_btl_portals4_frag_t*) des;
+    struct mca_btl_portals4_module_t *portals4_btl = (struct mca_btl_portals4_module_t *) btl_base;
+    mca_btl_portals4_frag_t *frag = (mca_btl_portals4_frag_t *) des;
 
     if (BTL_PORTALS4_FRAG_TYPE_EAGER == frag->type) {
         /* don't ever unlink eager frags */
         OPAL_BTL_PORTALS4_FRAG_RETURN_EAGER(portals4_btl, frag);
 
     } else if (BTL_PORTALS4_FRAG_TYPE_MAX == frag->type) {
-        if (frag->me_h != PTL_INVALID_HANDLE) {
+        if (!PtlHandleIsEqual(frag->me_h,PTL_INVALID_HANDLE)) {
             frag->me_h = PTL_INVALID_HANDLE;
         }
         OPAL_BTL_PORTALS4_FRAG_RETURN_MAX(portals4_btl, frag);
 
     } else if (BTL_PORTALS4_FRAG_TYPE_USER == frag->type) {
-        if (frag->me_h != PTL_INVALID_HANDLE) {
+        if (!PtlHandleIsEqual(frag->me_h,PTL_INVALID_HANDLE)) {
             frag->me_h = PTL_INVALID_HANDLE;
         }
         OPAL_THREAD_ADD_FETCH32(&portals4_btl->portals_outstanding_ops, -1);
         OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
-            "mca_btl_portals4_free: Decrementing portals_outstanding_ops=%d\n", portals4_btl->portals_outstanding_ops));
+                             "mca_btl_portals4_free: Decrementing portals_outstanding_ops=%d\n",
+                             portals4_btl->portals_outstanding_ops));
         OPAL_BTL_PORTALS4_FRAG_RETURN_USER(portals4_btl, frag);
     } else {
         return OPAL_ERR_BAD_PARAM;
@@ -549,29 +516,31 @@ mca_btl_portals4_free(struct mca_btl_base_module_t* btl_base,
  * @param peer (IN)     BTL peer addressing
  */
 
-mca_btl_base_descriptor_t*
-mca_btl_portals4_prepare_src(struct mca_btl_base_module_t* btl_base,
-                            struct mca_btl_base_endpoint_t* peer,
-                            struct opal_convertor_t* convertor,
-                            uint8_t order,
-                            size_t reserve,
-                            size_t* size,
-                            uint32_t flags)
+mca_btl_base_descriptor_t *mca_btl_portals4_prepare_src(struct mca_btl_base_module_t *btl_base,
+                                                        struct mca_btl_base_endpoint_t *peer,
+                                                        struct opal_convertor_t *convertor,
+                                                        uint8_t order, size_t reserve, size_t *size,
+                                                        uint32_t flags)
 {
-    struct mca_btl_portals4_module_t* portals4_btl = (struct mca_btl_portals4_module_t*) btl_base;
-    mca_btl_portals4_frag_t* frag;
+    mca_btl_portals4_frag_t *frag = NULL;
     size_t max_data = *size;
     struct iovec iov;
     uint32_t iov_count = 1;
     int ret;
 
     OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
-        "mca_btl_portals4_prepare_src NI=%d reserve=%ld size=%ld max_data=%ld\n", portals4_btl->interface_num, reserve, *size, max_data));
+                         "mca_btl_portals4_prepare_src NI=%d reserve=%ld size=%ld max_data=%ld\n",
+                         ((struct mca_btl_portals4_module_t *) btl_base)->interface_num,
+                         reserve, *size, max_data));
 
-    if (0 != reserve || 0 != opal_convertor_need_buffers(convertor)) {
-        OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "mca_btl_portals4_prepare_src NEED BUFFERS or RESERVE\n"));
-        frag = (mca_btl_portals4_frag_t*) mca_btl_portals4_alloc(btl_base, peer, MCA_BTL_NO_ORDER, max_data + reserve, flags);
-        if (NULL == frag)  {
+    if (0 != reserve ||
+        0 != opal_convertor_need_buffers(convertor) ||
+        0 != opal_convertor_on_device(convertor)) {
+        OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
+                             "mca_btl_portals4_prepare_src NEED BUFFERS or RESERVE\n"));
+        frag = (mca_btl_portals4_frag_t *) mca_btl_portals4_alloc(btl_base, peer, MCA_BTL_NO_ORDER,
+                                                                  max_data + reserve, flags);
+        if (NULL == frag) {
             return NULL;
         }
 
@@ -579,10 +548,10 @@ mca_btl_portals4_prepare_src(struct mca_btl_base_module_t* btl_base,
             max_data = frag->size - reserve;
         }
         iov.iov_len = max_data;
-        iov.iov_base = (unsigned char*) frag->segments[0].base.seg_addr.pval + reserve;
-        ret = opal_convertor_pack(convertor, &iov, &iov_count, &max_data );
+        iov.iov_base = (unsigned char *) frag->segments[0].base.seg_addr.pval + reserve;
+        ret = opal_convertor_pack(convertor, &iov, &iov_count, &max_data);
 
-        *size  = max_data;
+        *size = max_data;
         if (ret < 0) {
             mca_btl_portals4_free(btl_base, (mca_btl_base_descriptor_t *) frag);
             return NULL;
@@ -598,19 +567,18 @@ mca_btl_portals4_prepare_src(struct mca_btl_base_module_t* btl_base,
     return &frag->base;
 }
 
-mca_btl_base_registration_handle_t *
-mca_btl_portals4_register_mem(mca_btl_base_module_t *btl_base,
-                              mca_btl_base_endpoint_t *endpoint,
-                              void *base,
-                              size_t size,
-                              uint32_t flags)
+mca_btl_base_registration_handle_t *mca_btl_portals4_register_mem(mca_btl_base_module_t *btl_base,
+                                                                  mca_btl_base_endpoint_t *endpoint,
+                                                                  void *base, size_t size,
+                                                                  uint32_t flags)
 {
-    struct mca_btl_portals4_module_t   *portals4_btl = (struct mca_btl_portals4_module_t*) btl_base;
+    struct mca_btl_portals4_module_t *portals4_btl = (struct mca_btl_portals4_module_t *) btl_base;
     mca_btl_base_registration_handle_t *handle = NULL;
     ptl_me_t me;
     int ret;
 
-    handle = (mca_btl_base_registration_handle_t *)malloc(sizeof(mca_btl_base_registration_handle_t));
+    handle = (mca_btl_base_registration_handle_t *) malloc(
+        sizeof(mca_btl_base_registration_handle_t));
     if (!handle) {
         return NULL;
     }
@@ -618,9 +586,10 @@ mca_btl_portals4_register_mem(mca_btl_base_module_t *btl_base,
     handle->key = OPAL_THREAD_ADD_FETCH64(&(portals4_btl->portals_rdma_key), 1);
     handle->remote_offset = 0;
 
-    OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
-        "mca_btl_portals4_register_mem NI=%d base=%p size=%ld handle=%p key=%ld flags=%d",
-        portals4_btl->interface_num, base, size, (void *)handle, handle->key, flags));
+    OPAL_OUTPUT_VERBOSE(
+        (90, opal_btl_base_framework.framework_output,
+         "mca_btl_portals4_register_mem NI=%d base=%p size=%ld handle=%p key=%ld flags=%d",
+         portals4_btl->interface_num, base, size, (void *) handle, handle->key, flags));
 
     /* create a match entry */
     me.start = base;
@@ -628,10 +597,8 @@ mca_btl_portals4_register_mem(mca_btl_base_module_t *btl_base,
     me.ct_handle = PTL_CT_NONE;
     me.min_free = 0;
     me.uid = PTL_UID_ANY;
-    me.options = PTL_ME_OP_GET |
-        PTL_ME_EVENT_LINK_DISABLE |
-        PTL_ME_EVENT_COMM_DISABLE |
-        PTL_ME_EVENT_UNLINK_DISABLE;
+    me.options = PTL_ME_OP_GET | PTL_ME_EVENT_LINK_DISABLE | PTL_ME_EVENT_COMM_DISABLE
+                 | PTL_ME_EVENT_UNLINK_DISABLE;
 
     if (mca_btl_portals4_component.use_logical) {
         me.match_id.rank = endpoint->ptl_proc.rank;
@@ -640,47 +607,41 @@ mca_btl_portals4_register_mem(mca_btl_base_module_t *btl_base,
         me.match_id.phys.pid = endpoint->ptl_proc.phys.pid;
     }
     me.match_bits = handle->key;
-    me.ignore_bits = BTL_PORTALS4_PROTOCOL_MASK |
-        BTL_PORTALS4_CONTEXT_MASK |
-        BTL_PORTALS4_SOURCE_MASK;
+    me.ignore_bits = BTL_PORTALS4_PROTOCOL_MASK | BTL_PORTALS4_CONTEXT_MASK
+                     | BTL_PORTALS4_SOURCE_MASK;
     me.ignore_bits = 0;
 
-    ret = PtlMEAppend(portals4_btl->portals_ni_h,
-                      portals4_btl->recv_idx,
-                      &me,
-                      PTL_PRIORITY_LIST,
-                      handle,
-                      &(handle->me_h));
+    ret = PtlMEAppend(portals4_btl->portals_ni_h, portals4_btl->recv_idx, &me, PTL_PRIORITY_LIST,
+                      handle, &(handle->me_h));
     if (PTL_OK != ret) {
         opal_output_verbose(1, opal_btl_base_framework.framework_output,
-                            "%s:%d: PtlMEAppend failed: %d\n",
-                            __FILE__, __LINE__, ret);
+                            "%s:%d: PtlMEAppend failed: %d\n", __FILE__, __LINE__, ret);
         OPAL_THREAD_ADD_FETCH32(&portals4_btl->portals_outstanding_ops, -1);
         return NULL;
     }
     OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
-        "PtlMEAppend (mca_btl_portals4_register_mem) handle=%p, me_h=%d start=%p length=%ld rank=%x nid=%x pid=%x match_bits=%lx\n",
-        (void *)handle, handle->me_h, me.start, me.length,
-        me.match_id.rank, me.match_id.phys.nid, me.match_id.phys.pid, me.match_bits));
+                         "PtlMEAppend (mca_btl_portals4_register_mem) handle=%p, me_h=%d start=%p "
+                         "length=%ld rank=%x nid=%x pid=%x match_bits=%lx\n",
+                         (void *) handle, handle->me_h, me.start, me.length, me.match_id.rank,
+                         me.match_id.phys.nid, me.match_id.phys.pid, me.match_bits));
     return handle;
 }
 
-int
-mca_btl_portals4_deregister_mem(mca_btl_base_module_t *btl_base,
-                                mca_btl_base_registration_handle_t *handle)
+int mca_btl_portals4_deregister_mem(mca_btl_base_module_t *btl_base,
+                                    mca_btl_base_registration_handle_t *handle)
 {
     int ret;
-    struct mca_btl_portals4_module_t   *portals4_btl = (struct mca_btl_portals4_module_t*) btl_base;
 
     OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
-        "mca_btl_portals4_deregister_mem NI=%d handle=%p key=%ld me_h=%d\n",
-        portals4_btl->interface_num, (void *)handle, handle->key, handle->me_h));
+                         "mca_btl_portals4_deregister_mem NI=%d handle=%p key=%ld me_h=%d\n",
+                         ((struct mca_btl_portals4_module_t *) btl_base)->interface_num,
+                         (void *) handle, handle->key, handle->me_h));
 
     if (!PtlHandleIsEqual(handle->me_h, PTL_INVALID_HANDLE)) {
         ret = PtlMEUnlink(handle->me_h);
-        if (PTL_OK !=  ret) {
+        if (PTL_OK != ret) {
             opal_output_verbose(1, opal_btl_base_framework.framework_output,
-                "%s:%d: PtlMEUnlink failed: %d\n",__FILE__, __LINE__, ret);
+                                "%s:%d: PtlMEUnlink failed: %d\n", __FILE__, __LINE__, ret);
             return OPAL_ERROR;
         }
         handle->me_h = PTL_INVALID_HANDLE;
@@ -691,10 +652,9 @@ mca_btl_portals4_deregister_mem(mca_btl_base_module_t *btl_base,
     return OPAL_SUCCESS;
 }
 
-int
-mca_btl_portals4_finalize(struct mca_btl_base_module_t *btl)
+int mca_btl_portals4_finalize(struct mca_btl_base_module_t *btl)
 {
-    struct mca_btl_portals4_module_t* portals4_btl = (struct mca_btl_portals4_module_t*) btl;
+    struct mca_btl_portals4_module_t *portals4_btl = (struct mca_btl_portals4_module_t *) btl;
 
     mca_btl_portals4_free_module(portals4_btl);
 
@@ -706,7 +666,7 @@ mca_btl_portals4_finalize(struct mca_btl_base_module_t *btl)
     free(portals4_btl);
 
     OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
-        "mca_btl_portals4_finalize NI %d: OK\n", portals4_btl->interface_num));
+                         "mca_btl_portals4_finalize NI %d: OK\n", portals4_btl->interface_num));
 
     return OPAL_SUCCESS;
 }
@@ -716,18 +676,20 @@ void mca_btl_portals4_free_module(mca_btl_portals4_module_t *portals4_btl)
     int ret;
 
     OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
-         "mca_btl_portals4_free_module portals_outstanding_ops=%d\n", portals4_btl->portals_outstanding_ops));
+                         "mca_btl_portals4_free_module portals_outstanding_ops=%d\n",
+                         portals4_btl->portals_outstanding_ops));
 
     /* sanity check */
-    assert(portals4_btl->portals_outstanding_ops  >= 0);
+    assert(portals4_btl->portals_outstanding_ops >= 0);
 
     /* finalize all communication */
     while (portals4_btl->portals_outstanding_ops > 0) {
         OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
-                            "mca_btl_portals4_free_module portals_outstanding_ops: %d",
-                            portals4_btl->portals_outstanding_ops));
+                             "mca_btl_portals4_free_module portals_outstanding_ops: %d",
+                             portals4_btl->portals_outstanding_ops));
 
-        OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "Call to mca_btl_portals4_component_progress (3)\n"));
+        OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
+                             "Call to mca_btl_portals4_component_progress (3)\n"));
         mca_btl_portals4_component_progress();
     }
 
@@ -747,25 +709,33 @@ void mca_btl_portals4_free_module(mca_btl_portals4_module_t *portals4_btl)
 
     if ((ptl_pt_index_t) ~0UL != mca_btl_portals4_module.recv_idx) {
         PtlPTFree(portals4_btl->portals_ni_h, portals4_btl->recv_idx);
-        portals4_btl->recv_idx= (ptl_pt_index_t) ~0UL;
+        portals4_btl->recv_idx = (ptl_pt_index_t) ~0UL;
     }
 
     if (PTL_EQ_NONE != portals4_btl->recv_eq_h) {
         ret = PtlEQFree(portals4_btl->recv_eq_h);
-        if (PTL_OK != ret) OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "Error freeing EQ recv: %d", ret));
-        OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "PtlEQFree: recv_eq_h=%d portals4_btl=%p",
-            portals4_btl->recv_eq_h, (void*)portals4_btl));
+        if (PTL_OK != ret)
+            OPAL_OUTPUT_VERBOSE(
+                (90, opal_btl_base_framework.framework_output, "Error freeing EQ recv: %d", ret));
+        OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
+                             "PtlEQFree: recv_eq_h=%d portals4_btl=%p", portals4_btl->recv_eq_h,
+                             (void *) portals4_btl));
 
         portals4_btl->recv_eq_h = PTL_EQ_NONE;
     }
     if (!PtlHandleIsEqual(portals4_btl->portals_ni_h, PTL_INVALID_HANDLE)) {
         ret = PtlNIFini(portals4_btl->portals_ni_h);
-        if (PTL_OK != ret) OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "Error returned by PtlNIFini: %d\n", ret));
-        OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "PtlNIFini: portals_ni_h=%d portals4_btl=%p",
-            portals4_btl->portals_ni_h, (void*)portals4_btl));
+        if (PTL_OK != ret)
+            OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
+                                 "Error returned by PtlNIFini: %d\n", ret));
+        OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
+                             "PtlNIFini: portals_ni_h=%d portals4_btl=%p",
+                             portals4_btl->portals_ni_h, (void *) portals4_btl));
 
         portals4_btl->portals_ni_h = PTL_INVALID_HANDLE;
     }
     ret = mca_btl_portals4_recv_disable(portals4_btl);
-    if (PTL_OK != ret) OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "Error freeing recv list: %d", ret));
+    if (PTL_OK != ret)
+        OPAL_OUTPUT_VERBOSE(
+            (90, opal_btl_base_framework.framework_output, "Error freeing recv list: %d", ret));
 }
diff --git a/opal/mca/btl/portals4/btl_portals4.h b/opal/mca/btl/portals4/btl_portals4.h
index 41e7da7a743..dd86a36d9fc 100644
--- a/opal/mca/btl/portals4/btl_portals4.h
+++ b/opal/mca/btl/portals4/btl_portals4.h
@@ -24,15 +24,17 @@
 #ifndef BTL_PORTALS_H_HAS_BEEN_INCLUDED
 #define BTL_PORTALS_H_HAS_BEEN_INCLUDED
 
-#include <portals4.h>
-#include <btl_portals4_frag.h>
+#include "opal_config.h"
 
 #include "opal/class/opal_free_list.h"
 #include "opal/class/opal_list.h"
 #include "opal/datatype/opal_convertor.h"
-#include "opal/mca/btl/btl.h"
+
 #include "opal/mca/btl/base/base.h"
 #include "opal/mca/btl/base/btl_base_error.h"
+#include "opal/mca/btl/btl.h"
+
+#include <portals4.h>
 
 BEGIN_C_DECLS
 
@@ -46,12 +48,13 @@ struct mca_btl_portals4_component_t {
     unsigned int num_btls;
     unsigned int max_btls; /* Maximum number of accepted Portals4 cards */
 
-    struct mca_btl_portals4_module_t** btls; /* array of available BTL modules */
+    struct mca_btl_portals4_module_t **btls; /* array of available BTL modules */
 
-    /* add_procs() can get called multiple times.  this prevents multiple calls to portals4_init_interface(). */
+    /* add_procs() can get called multiple times.  this prevents multiple calls to
+     * portals4_init_interface(). */
     int need_init;
 
-    /* Use the logical to physical table to accelerate portals4 adressing: 1 (true) : 0 (false) */
+    /* Use the logical to physical table to accelerate portals4 addressing: 1 (true) : 0 (false) */
     int use_logical;
 
     /* initial size of free lists */
@@ -145,111 +148,86 @@ typedef struct mca_btl_portals4_module_t mca_btl_portals4_module_t;
  * +---- protocol
  */
 
-#define BTL_PORTALS4_PROTOCOL_MASK  0xF000000000000000ULL
-#define BTL_PORTALS4_CONTEXT_MASK   0x0FFF000000000000ULL
-#define BTL_PORTALS4_SOURCE_MASK    0x0000FFFF00000000ULL
-#define BTL_PORTALS4_TAG_MASK       0x00000000FFFFFFFFULL
+#define BTL_PORTALS4_PROTOCOL_MASK 0xF000000000000000ULL
+#define BTL_PORTALS4_CONTEXT_MASK  0x0FFF000000000000ULL
+#define BTL_PORTALS4_SOURCE_MASK   0x0000FFFF00000000ULL
+#define BTL_PORTALS4_TAG_MASK      0x00000000FFFFFFFFULL
 
-#define BTL_PORTALS4_PROTOCOL_IGNR  BTL_PORTALS4_PROTOCOL_MASK
-#define BTL_PORTALS4_CONTEXT_IGNR   BTL_PORTALS4_CONTEXT_MASK
-#define BTL_PORTALS4_SOURCE_IGNR    BTL_PORTALS4_SOURCE_MASK
-#define BTL_PORTALS4_TAG_IGNR       0x000000007FFFFFFFULL
+#define BTL_PORTALS4_PROTOCOL_IGNR BTL_PORTALS4_PROTOCOL_MASK
+#define BTL_PORTALS4_CONTEXT_IGNR  BTL_PORTALS4_CONTEXT_MASK
+#define BTL_PORTALS4_SOURCE_IGNR   BTL_PORTALS4_SOURCE_MASK
+#define BTL_PORTALS4_TAG_IGNR      0x000000007FFFFFFFULL
 
-#define BTL_PORTALS4_SHORT_MSG      0x1000000000000000ULL
-#define BTL_PORTALS4_LONG_MSG       0x2000000000000000ULL
+#define BTL_PORTALS4_SHORT_MSG 0x1000000000000000ULL
+#define BTL_PORTALS4_LONG_MSG  0x2000000000000000ULL
 
 /* send posting */
 #define BTL_PORTALS4_SET_SEND_BITS(match_bits, contextid, source, tag, type) \
-    {                                                                   \
-        match_bits = contextid;                                         \
-        match_bits = (match_bits << 16);                                \
-        match_bits |= source;                                           \
-        match_bits = (match_bits << 32);                                \
-        match_bits |= (BTL_PORTALS4_TAG_MASK & tag) | type;             \
+    {                                                                        \
+        match_bits = contextid;                                              \
+        match_bits = (match_bits << 16);                                     \
+        match_bits |= source;                                                \
+        match_bits = (match_bits << 32);                                     \
+        match_bits |= (BTL_PORTALS4_TAG_MASK & tag) | type;                  \
     }
 
-#define BTL_PORTALS4_SET_HDR_DATA(hdr_data, opcount, length, sync)   \
-    {                                                                \
-        hdr_data = (sync) ? 1 : 0;                                   \
-        hdr_data = (hdr_data << 15);                                 \
-        hdr_data |= opcount & 0x7FFFULL;                             \
-        hdr_data = (hdr_data << 48);                                 \
-        hdr_data |= (length & 0xFFFFFFFFFFFFULL);                    \
+#define BTL_PORTALS4_SET_HDR_DATA(hdr_data, opcount, length, sync) \
+    {                                                              \
+        hdr_data = (sync) ? 1 : 0;                                 \
+        hdr_data = (hdr_data << 15);                               \
+        hdr_data |= opcount & 0x7FFFULL;                           \
+        hdr_data = (hdr_data << 48);                               \
+        hdr_data |= (length & 0xFFFFFFFFFFFFULL);                  \
     }
 
-#define REQ_BTL_TABLE_ID	2
+#define REQ_BTL_TABLE_ID 2
 
 int mca_btl_portals4_component_progress(void);
 void mca_btl_portals4_free_module(mca_btl_portals4_module_t *portals4_btl);
 
 /* BTL interface functions */
-int mca_btl_portals4_finalize(struct mca_btl_base_module_t* btl_base);
-
-
-int mca_btl_portals4_add_procs(struct mca_btl_base_module_t* btl_base,
-                              size_t nprocs,
-                              struct opal_proc_t **procs,
-                              struct mca_btl_base_endpoint_t** peers,
-                              opal_bitmap_t* reachable);
-
-int mca_btl_portals4_del_procs(struct mca_btl_base_module_t* btl_base,
-                              size_t nprocs,
-                              struct opal_proc_t **procs,
-                              struct mca_btl_base_endpoint_t** peers);
-
-mca_btl_base_descriptor_t*
-mca_btl_portals4_alloc(struct mca_btl_base_module_t* btl_base,
-                      struct mca_btl_base_endpoint_t* endpoint,
-                      uint8_t order,
-                      size_t size,
-                      uint32_t flags);
-
-int mca_btl_portals4_free(struct mca_btl_base_module_t* btl_base,
-                         mca_btl_base_descriptor_t* des);
-
-mca_btl_base_descriptor_t*
-mca_btl_portals4_prepare_src(struct mca_btl_base_module_t* btl_base,
-                            struct mca_btl_base_endpoint_t* peer,
-                            struct opal_convertor_t* convertor,
-                            uint8_t order,
-                            size_t reserve,
-                            size_t* size,
-                            uint32_t flags);
-
-int mca_btl_portals4_send(struct mca_btl_base_module_t* btl_base,
-                         struct mca_btl_base_endpoint_t* btl_peer,
-                         struct mca_btl_base_descriptor_t* descriptor,
-                         mca_btl_base_tag_t tag);
-
-
-int mca_btl_portals4_sendi(struct mca_btl_base_module_t* btl_base,
-                          struct mca_btl_base_endpoint_t* endpoint,
-                          struct opal_convertor_t* convertor,
-                          void* header,
-                          size_t header_size,
-                          size_t payload_size,
-                          uint8_t order,
-                          uint32_t flags,
-                          mca_btl_base_tag_t tag,
-                          mca_btl_base_descriptor_t** des);
-
-int mca_btl_portals4_put(struct mca_btl_base_module_t* btl_base,
-                        struct mca_btl_base_endpoint_t* btl_peer,
-                        struct mca_btl_base_descriptor_t* decriptor);
-
-
-int mca_btl_portals4_get(struct mca_btl_base_module_t* btl_base,
-                        struct mca_btl_base_endpoint_t* btl_peer,
-                        void *local_address,
-                        uint64_t remote_address,
-                        struct mca_btl_base_registration_handle_t *local_handle,
-                        struct mca_btl_base_registration_handle_t *remote_handle,
-                        size_t size,
-                        int flags,
-                        int order,
-                        mca_btl_base_rdma_completion_fn_t cbfunc,
-                        void *cbcontext,
-                        void *cbdata);
+int mca_btl_portals4_finalize(struct mca_btl_base_module_t *btl_base);
+
+int mca_btl_portals4_add_procs(struct mca_btl_base_module_t *btl_base, size_t nprocs,
+                               struct opal_proc_t **procs, struct mca_btl_base_endpoint_t **peers,
+                               opal_bitmap_t *reachable);
+
+int mca_btl_portals4_del_procs(struct mca_btl_base_module_t *btl_base, size_t nprocs,
+                               struct opal_proc_t **procs, struct mca_btl_base_endpoint_t **peers);
+
+mca_btl_base_descriptor_t *mca_btl_portals4_alloc(struct mca_btl_base_module_t *btl_base,
+                                                  struct mca_btl_base_endpoint_t *endpoint,
+                                                  uint8_t order, size_t size, uint32_t flags);
+
+int mca_btl_portals4_free(struct mca_btl_base_module_t *btl_base, mca_btl_base_descriptor_t *des);
+
+mca_btl_base_descriptor_t *mca_btl_portals4_prepare_src(struct mca_btl_base_module_t *btl_base,
+                                                        struct mca_btl_base_endpoint_t *peer,
+                                                        struct opal_convertor_t *convertor,
+                                                        uint8_t order, size_t reserve, size_t *size,
+                                                        uint32_t flags);
+
+int mca_btl_portals4_send(struct mca_btl_base_module_t *btl_base,
+                          struct mca_btl_base_endpoint_t *btl_peer,
+                          struct mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag);
+
+int mca_btl_portals4_sendi(struct mca_btl_base_module_t *btl_base,
+                           struct mca_btl_base_endpoint_t *endpoint,
+                           struct opal_convertor_t *convertor, void *header, size_t header_size,
+                           size_t payload_size, uint8_t order, uint32_t flags,
+                           mca_btl_base_tag_t tag, mca_btl_base_descriptor_t **des);
+
+int mca_btl_portals4_put(struct mca_btl_base_module_t *btl_base,
+                         struct mca_btl_base_endpoint_t *btl_peer,
+                         struct mca_btl_base_descriptor_t *decriptor);
+
+int mca_btl_portals4_get(struct mca_btl_base_module_t *btl_base,
+                         struct mca_btl_base_endpoint_t *btl_peer, void *local_address,
+                         uint64_t remote_address,
+                         struct mca_btl_base_registration_handle_t *local_handle,
+                         struct mca_btl_base_registration_handle_t *remote_handle, size_t size,
+                         int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
+                         void *cbcontext, void *cbdata);
 
 int mca_btl_portals4_get_error(int ptl_error);
 
@@ -265,7 +243,7 @@ struct mca_btl_base_registration_handle_t {
 /*
  * global structures
  */
-OPAL_MODULE_DECLSPEC extern mca_btl_portals4_component_t mca_btl_portals4_component;
+OPAL_DECLSPEC extern mca_btl_portals4_component_t mca_btl_portals4_component;
 extern mca_btl_portals4_module_t mca_btl_portals4_module;
 
 /**
@@ -281,4 +259,4 @@ typedef struct mca_btl_base_endpoint_t mca_btl_base_endpoint_t;
 
 END_C_DECLS
 
-#endif  /* BTL_PORTALS_H_HAS_BEEN_INCLUDED */
+#endif /* BTL_PORTALS_H_HAS_BEEN_INCLUDED */
diff --git a/opal/mca/btl/portals4/btl_portals4_component.c b/opal/mca/btl/portals4/btl_portals4_component.c
index 06b1312aed6..36671327c96 100644
--- a/opal/mca/btl/portals4/btl_portals4_component.c
+++ b/opal/mca/btl/portals4/btl_portals4_component.c
@@ -27,15 +27,14 @@
 
 #include "opal_config.h"
 
+#include "opal/mca/btl/base/base.h"
+#include "opal/mca/btl/btl.h"
+#include "opal/mca/mpool/base/base.h"
+#include "opal/mca/pmix/pmix-internal.h"
 #include "opal/util/event.h"
 #include "opal/util/output.h"
-#include "opal/mca/pmix/pmix-internal.h"
 #include "opal/util/show_help.h"
-#include "opal/mca/btl/btl.h"
-#include "opal/mca/btl/base/base.h"
-#include "opal/mca/mpool/base/base.h"
 
-#include "portals4.h"
 #include "btl_portals4.h"
 #include "btl_portals4_frag.h"
 #include "btl_portals4_recv.h"
@@ -43,186 +42,132 @@
 static int mca_btl_portals4_component_register(void);
 static int mca_btl_portals4_component_open(void);
 static int mca_btl_portals4_component_close(void);
-static mca_btl_base_module_t** mca_btl_portals4_component_init(int *num_btls,
-                                                       bool enable_progress_threads,
-                                                       bool enable_mpi_threads);
+static mca_btl_base_module_t **mca_btl_portals4_component_init(int *num_btls,
+                                                               bool enable_progress_threads,
+                                                               bool enable_mpi_threads);
 int mca_btl_portals4_component_progress(void);
 
-OPAL_MODULE_DECLSPEC extern mca_btl_portals4_component_t mca_btl_portals4_component;
-
-mca_btl_portals4_component_t mca_btl_portals4_component = {
-    {
-      /* First, the mca_base_module_t struct containing meta
-         information about the module itself */
-      .btl_version = {
-	MCA_BTL_DEFAULT_VERSION("portals4"),
-        .mca_open_component = mca_btl_portals4_component_open,
-        .mca_close_component = mca_btl_portals4_component_close,
-        .mca_register_component_params = mca_btl_portals4_component_register,
-      },
-      .btl_data = {
-          /* The component is not checkpoint ready */
-          .param_field = MCA_BASE_METADATA_PARAM_NONE
-      },
-
-      .btl_init = mca_btl_portals4_component_init,
-      .btl_progress = mca_btl_portals4_component_progress,
-    }
-};
-
-static int
-mca_btl_portals4_component_register(void)
+OPAL_DECLSPEC extern mca_btl_portals4_component_t mca_btl_portals4_component;
+
+mca_btl_portals4_component_t mca_btl_portals4_component = {{
+    /* First, the mca_base_module_t struct containing meta
+       information about the module itself */
+    .btl_version =
+        {
+            MCA_BTL_DEFAULT_VERSION("portals4"),
+            .mca_open_component = mca_btl_portals4_component_open,
+            .mca_close_component = mca_btl_portals4_component_close,
+            .mca_register_component_params = mca_btl_portals4_component_register,
+        },
+    .btl_data =
+        {/* The component is not checkpoint ready */
+         .param_field = MCA_BASE_METADATA_PARAM_NONE},
+
+    .btl_init = mca_btl_portals4_component_init,
+    .btl_progress = mca_btl_portals4_component_progress,
+}};
+
+static int mca_btl_portals4_component_register(void)
 {
     mca_btl_portals4_component.use_logical = 0;
-    (void) mca_base_component_var_register(&mca_btl_portals4_component.super.btl_version,
-                           "use_logical",
-                           "Use the logical to physical table to accelerate portals4 adressing: 1 (true) : 0 (false)",
-                           MCA_BASE_VAR_TYPE_INT,
-                           NULL,
-                           0,
-                           0,
-                           OPAL_INFO_LVL_5,
-                           MCA_BASE_VAR_SCOPE_READONLY,
-                           &mca_btl_portals4_component.use_logical);
+    (void) mca_base_component_var_register(
+        &mca_btl_portals4_component.super.btl_version, "use_logical",
+        "Use the logical to physical table to accelerate portals4 addressing: 1 (true) : 0 (false)",
+        MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_READONLY,
+        &mca_btl_portals4_component.use_logical);
 
     mca_btl_portals4_component.max_btls = 1;
     (void) mca_base_component_var_register(&mca_btl_portals4_component.super.btl_version,
-                           "max_btls",
-                           "Maximum number of accepted Portals4 cards",
-                           MCA_BASE_VAR_TYPE_UNSIGNED_INT,
-                           NULL,
-                           0,
-                           0,
-                           OPAL_INFO_LVL_5,
-                           MCA_BASE_VAR_SCOPE_READONLY,
-                           &mca_btl_portals4_component.max_btls);
+                                           "max_btls", "Maximum number of accepted Portals4 cards",
+                                           MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, 0,
+                                           OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_READONLY,
+                                           &mca_btl_portals4_component.max_btls);
 
     mca_btl_portals4_component.portals_free_list_init_num = 16;
-    (void) mca_base_component_var_register(&mca_btl_portals4_component.super.btl_version,
-                           "free_list_init_num",
-                           "Initial number of elements to initialize in free lists",
-                           MCA_BASE_VAR_TYPE_INT,
-                           NULL,
-                           0,
-                           0,
-                           OPAL_INFO_LVL_5,
-                           MCA_BASE_VAR_SCOPE_READONLY,
-                           &(mca_btl_portals4_component.portals_free_list_init_num));
+    (void)
+        mca_base_component_var_register(&mca_btl_portals4_component.super.btl_version,
+                                        "free_list_init_num",
+                                        "Initial number of elements to initialize in free lists",
+                                        MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                        MCA_BASE_VAR_SCOPE_READONLY,
+                                        &(mca_btl_portals4_component.portals_free_list_init_num));
 
     mca_btl_portals4_component.portals_free_list_max_num = 1024;
     (void) mca_base_component_var_register(&mca_btl_portals4_component.super.btl_version,
-                           "free_list_max_num",
-                           "Max number of elements to initialize in free lists",
-                           MCA_BASE_VAR_TYPE_INT,
-                           NULL,
-                           0,
-                           0,
-                           OPAL_INFO_LVL_5,
-                           MCA_BASE_VAR_SCOPE_READONLY,
-                           &(mca_btl_portals4_component.portals_free_list_max_num));
+                                           "free_list_max_num",
+                                           "Max number of elements to initialize in free lists",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           &(mca_btl_portals4_component.portals_free_list_max_num));
 
     mca_btl_portals4_component.portals_free_list_inc_num = 16;
     (void) mca_base_component_var_register(&mca_btl_portals4_component.super.btl_version,
-                           "free_list_inc_num",
-                           "Increment count for free lists",
-                           MCA_BASE_VAR_TYPE_INT,
-                           NULL,
-                           0,
-                           0,
-                           OPAL_INFO_LVL_5,
-                           MCA_BASE_VAR_SCOPE_READONLY,
-                           &(mca_btl_portals4_component.portals_free_list_inc_num));
+                                           "free_list_inc_num", "Increment count for free lists",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           &(mca_btl_portals4_component.portals_free_list_inc_num));
 
     mca_btl_portals4_component.portals_free_list_eager_max_num = 32;
     (void) mca_base_component_var_register(&mca_btl_portals4_component.super.btl_version,
-                           "eager_frag_limit",
-                           "Maximum number of pre-pinned eager fragments",
-                           MCA_BASE_VAR_TYPE_INT,
-                           NULL,
-                           0,
-                           0,
-                           OPAL_INFO_LVL_5,
-                           MCA_BASE_VAR_SCOPE_READONLY,
-                           &(mca_btl_portals4_component.portals_free_list_eager_max_num));
-
-    mca_btl_portals4_component.portals_need_ack =  1; /* default to true.. */
+                                           "eager_frag_limit",
+                                           "Maximum number of pre-pinned eager fragments",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           &(mca_btl_portals4_component
+                                                 .portals_free_list_eager_max_num));
+
+    mca_btl_portals4_component.portals_need_ack = 1; /* default to true.. */
     (void) mca_base_component_var_register(&mca_btl_portals4_component.super.btl_version,
-                           "needs_ack",
-                           "Require a portals level ACK",
-                           MCA_BASE_VAR_TYPE_INT,
-                           NULL,
-                           0,
-                           0,
-                           OPAL_INFO_LVL_5,
-                           MCA_BASE_VAR_SCOPE_READONLY,
-                           &(mca_btl_portals4_component.portals_need_ack));
+                                           "needs_ack", "Require a portals level ACK",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           &(mca_btl_portals4_component.portals_need_ack));
 
     mca_btl_portals4_component.recv_queue_size = 4 * 1024;
     (void) mca_base_component_var_register(&mca_btl_portals4_component.super.btl_version,
-                           "eq_recv_size",
-                           "Size of the receive event queue",
-                           MCA_BASE_VAR_TYPE_INT,
-                           NULL,
-                           0,
-                           0,
-                           OPAL_INFO_LVL_5,
-                           MCA_BASE_VAR_SCOPE_READONLY,
-                           &(mca_btl_portals4_component.recv_queue_size));
+                                           "eq_recv_size", "Size of the receive event queue",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           &(mca_btl_portals4_component.recv_queue_size));
 
     mca_btl_portals4_component.portals_max_outstanding_ops = 8 * 1024;
-    (void) mca_base_component_var_register(&mca_btl_portals4_component.super.btl_version,
-                           "max_pending_ops",
-                           "Maximum number of pending send/rdma frags",
-                           MCA_BASE_VAR_TYPE_INT,
-                           NULL,
-                           0,
-                           0,
-                           OPAL_INFO_LVL_5,
-                           MCA_BASE_VAR_SCOPE_READONLY,
-                           &(mca_btl_portals4_component.portals_max_outstanding_ops));
+    (void)
+        mca_base_component_var_register(&mca_btl_portals4_component.super.btl_version,
+                                        "max_pending_ops",
+                                        "Maximum number of pending send/rdma frags",
+                                        MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                        MCA_BASE_VAR_SCOPE_READONLY,
+                                        &(mca_btl_portals4_component.portals_max_outstanding_ops));
 
     mca_btl_portals4_component.portals_recv_mds_num = 8;
     (void) mca_base_component_var_register(&mca_btl_portals4_component.super.btl_version,
-                           "recv_md_num",
-                           "Number of send frag receive descriptors",
-                           MCA_BASE_VAR_TYPE_INT,
-                           NULL,
-                           0,
-                           0,
-                           OPAL_INFO_LVL_5,
-                           MCA_BASE_VAR_SCOPE_READONLY,
-                           &(mca_btl_portals4_component.portals_recv_mds_num));
+                                           "recv_md_num", "Number of send frag receive descriptors",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           &(mca_btl_portals4_component.portals_recv_mds_num));
 
     mca_btl_portals4_component.portals_recv_mds_size = 256 * 1024;
     (void) mca_base_component_var_register(&mca_btl_portals4_component.super.btl_version,
-                           "recv_md_size",
-                           "Size of send frag receive descriptors",
-                           MCA_BASE_VAR_TYPE_INT,
-                           NULL,
-                           0,
-                           0,
-                           OPAL_INFO_LVL_5,
-                           MCA_BASE_VAR_SCOPE_READONLY,
-                           &(mca_btl_portals4_component.portals_recv_mds_size));
+                                           "recv_md_size", "Size of send frag receive descriptors",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           &(mca_btl_portals4_component.portals_recv_mds_size));
 
     mca_btl_portals4_component.portals_max_msg_size = PTL_SIZE_MAX;
     (void) mca_base_component_var_register(&mca_btl_portals4_component.super.btl_version,
-                           "max_msg_size",
-                           "Max size supported by portals4 (above that, a message is cut into messages less than that size)",
-                           MCA_BASE_VAR_TYPE_UNSIGNED_LONG,
-                           NULL,
-                           0,
-                           0,
-                           OPAL_INFO_LVL_5,
-                           MCA_BASE_VAR_SCOPE_READONLY,
-                           &(mca_btl_portals4_component.portals_max_msg_size));
+                                           "max_msg_size",
+                                           "Max size supported by portals4 (above that, a message "
+                                           "is cut into messages less than that size)",
+                                           MCA_BASE_VAR_TYPE_UNSIGNED_LONG, NULL, 0, 0,
+                                           OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_READONLY,
+                                           &(mca_btl_portals4_component.portals_max_msg_size));
     return OPAL_SUCCESS;
 }
 
-static int
-mca_btl_portals4_component_open(void)
+static int mca_btl_portals4_component_open(void)
 {
-    OPAL_OUTPUT_VERBOSE((1, opal_btl_base_framework.framework_output, "mca_btl_portals4_component_open\n"));
+    OPAL_OUTPUT_VERBOSE(
+        (1, opal_btl_base_framework.framework_output, "mca_btl_portals4_component_open\n"));
 
     /*
      * fill default module state
@@ -231,22 +176,25 @@ mca_btl_portals4_component_open(void)
     mca_btl_portals4_module.super.btl_eager_limit = 32 * 1024;
     mca_btl_portals4_module.super.btl_rndv_eager_limit = 32 * 1024;
     mca_btl_portals4_module.super.btl_max_send_size = 64 * 1024;
-    if (mca_btl_portals4_module.super.btl_max_send_size > mca_btl_portals4_component.portals_max_msg_size)
-        mca_btl_portals4_module.super.btl_max_send_size = mca_btl_portals4_component.portals_max_msg_size;
+    if (mca_btl_portals4_module.super.btl_max_send_size
+        > mca_btl_portals4_component.portals_max_msg_size)
+        mca_btl_portals4_module.super.btl_max_send_size = mca_btl_portals4_component
+                                                              .portals_max_msg_size;
     mca_btl_portals4_module.super.btl_rdma_pipeline_send_length = 64 * 1024;
     mca_btl_portals4_module.super.btl_rdma_pipeline_frag_size = INT_MAX;
     mca_btl_portals4_module.super.btl_min_rdma_pipeline_size = 0;
-    mca_btl_portals4_module.super.btl_flags =
-        MCA_BTL_FLAGS_RDMA |
-        MCA_BTL_FLAGS_RDMA_MATCHED |
-        MCA_BTL_FLAGS_SEND;
+    mca_btl_portals4_module.super.btl_flags = MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_RDMA_MATCHED
+                                              | MCA_BTL_FLAGS_SEND;
 
-    mca_btl_portals4_module.super.btl_registration_handle_size = sizeof (mca_btl_base_registration_handle_t);
+    mca_btl_portals4_module.super.btl_registration_handle_size = sizeof(
+        mca_btl_base_registration_handle_t);
 
     mca_btl_portals4_module.super.btl_get_limit = SIZE_MAX;
-    if (mca_btl_portals4_module.super.btl_get_limit > mca_btl_portals4_component.portals_max_msg_size)
-         mca_btl_portals4_module.super.btl_get_limit = mca_btl_portals4_component.portals_max_msg_size;
-    mca_btl_portals4_module.super.btl_put_limit = 0;        /* not implemented */
+    if (mca_btl_portals4_module.super.btl_get_limit
+        > mca_btl_portals4_component.portals_max_msg_size)
+        mca_btl_portals4_module.super.btl_get_limit = mca_btl_portals4_component
+                                                          .portals_max_msg_size;
+    mca_btl_portals4_module.super.btl_put_limit = 0; /* not implemented */
     mca_btl_portals4_module.super.btl_get_alignment = 0;
     mca_btl_portals4_module.super.btl_put_alignment = 0;
 
@@ -256,7 +204,8 @@ mca_btl_portals4_component_open(void)
     mca_btl_portals4_module.super.btl_bandwidth = 1000;
     mca_btl_portals4_module.super.btl_latency = 0;
 
-    mca_btl_base_param_register(&mca_btl_portals4_component.super.btl_version, &mca_btl_portals4_module.super);
+    mca_btl_base_param_register(&mca_btl_portals4_component.super.btl_version,
+                                &mca_btl_portals4_module.super);
 
     mca_btl_portals4_module.portals_num_procs = 0;
 
@@ -283,19 +232,20 @@ mca_btl_portals4_component_open(void)
     return OPAL_SUCCESS;
 }
 
-
-static int
-mca_btl_portals4_component_close(void)
+static int mca_btl_portals4_component_close(void)
 {
-    opal_output_verbose(50, opal_btl_base_framework.framework_output, "mca_btl_portals4_component_close\n");
+    opal_output_verbose(50, opal_btl_base_framework.framework_output,
+                        "mca_btl_portals4_component_close\n");
 
     /* release resources */
     /* close debugging stream */
     opal_output_close(opal_btl_base_framework.framework_output);
     opal_btl_base_framework.framework_output = -1;
 
-    if (NULL != mca_btl_portals4_component.btls)  free(mca_btl_portals4_component.btls);
-    if (NULL != mca_btl_portals4_component.eqs_h) free(mca_btl_portals4_component.eqs_h);
+    if (NULL != mca_btl_portals4_component.btls)
+        free(mca_btl_portals4_component.btls);
+    if (NULL != mca_btl_portals4_component.eqs_h)
+        free(mca_btl_portals4_component.eqs_h);
     mca_btl_portals4_component.btls = NULL;
     mca_btl_portals4_component.eqs_h = NULL;
 
@@ -304,18 +254,19 @@ mca_btl_portals4_component_close(void)
     return OPAL_SUCCESS;
 }
 
-static mca_btl_base_module_t** mca_btl_portals4_component_init(int *num_btls,
-                                                       bool enable_progress_threads,
-                                                       bool enable_mpi_threads)
+static mca_btl_base_module_t **mca_btl_portals4_component_init(int *num_btls,
+                                                               bool enable_progress_threads,
+                                                               bool enable_mpi_threads)
 {
     mca_btl_portals4_module_t *portals4_btl = NULL;
     mca_btl_base_module_t **btls = NULL;
     unsigned int ret, interface;
     ptl_handle_ni_t *portals4_nis_h = NULL;
-    ptl_ni_limits_t portals4_ni_limits ;
+    ptl_ni_limits_t portals4_ni_limits;
     ptl_process_t *ptl_process_ids = NULL;
 
-    opal_output_verbose(50, opal_btl_base_framework.framework_output, "mca_btl_portals4_component_init\n");
+    opal_output_verbose(50, opal_btl_base_framework.framework_output,
+                        "mca_btl_portals4_component_init\n");
 
     if (enable_mpi_threads && !mca_btl_base_thread_multiple_override) {
         opal_output_verbose(1, opal_btl_base_framework.framework_output,
@@ -327,8 +278,7 @@ static mca_btl_base_module_t** mca_btl_portals4_component_init(int *num_btls,
     ret = PtlInit();
     if (PTL_OK != ret) {
         opal_output_verbose(1, opal_btl_base_framework.framework_output,
-                            "%s:%d: PtlInit failed: %d\n",
-                            __FILE__, __LINE__, ret);
+                            "%s:%d: PtlInit failed: %d\n", __FILE__, __LINE__, ret);
         goto error;
     }
     OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "PtlInit OK\n"));
@@ -338,55 +288,62 @@ static mca_btl_base_module_t** mca_btl_portals4_component_init(int *num_btls,
      */
     *num_btls = 0;
     portals4_nis_h = malloc(mca_btl_portals4_component.max_btls * sizeof(ptl_handle_ni_t));
-    for (interface=0; interface<mca_btl_portals4_component.max_btls; interface++) {
+    for (interface = 0; interface < mca_btl_portals4_component.max_btls; interface++) {
 
         if (mca_btl_portals4_component.use_logical)
-            ret = PtlNIInit((1 == mca_btl_portals4_component.max_btls) ? PTL_IFACE_DEFAULT : interface,
-                    PTL_NI_LOGICAL | PTL_NI_MATCHING,
-                    PTL_PID_ANY,       /* let library assign our pid */
-                    NULL,              /* no desired limits */
-                    &portals4_ni_limits, /* actual limits */
-                    &portals4_nis_h[*num_btls] /* our interface handle */
-                    );
-        else ret = PtlNIInit((1 == mca_btl_portals4_component.max_btls) ? PTL_IFACE_DEFAULT : interface,
-                    PTL_NI_PHYSICAL | PTL_NI_MATCHING,
-                    PTL_PID_ANY,       /* let library assign our pid */
-                    NULL,              /* no desired limits */
-                    &portals4_ni_limits, /* actual limits */
-                    &portals4_nis_h[*num_btls] /* our interface handle */
-                    );
+            ret = PtlNIInit((1 == mca_btl_portals4_component.max_btls) ? PTL_IFACE_DEFAULT
+                                                                       : interface,
+                            PTL_NI_LOGICAL | PTL_NI_MATCHING,
+                            PTL_PID_ANY,               /* let library assign our pid */
+                            NULL,                      /* no desired limits */
+                            &portals4_ni_limits,       /* actual limits */
+                            &portals4_nis_h[*num_btls] /* our interface handle */
+            );
+        else
+            ret = PtlNIInit((1 == mca_btl_portals4_component.max_btls) ? PTL_IFACE_DEFAULT
+                                                                       : interface,
+                            PTL_NI_PHYSICAL | PTL_NI_MATCHING,
+                            PTL_PID_ANY,               /* let library assign our pid */
+                            NULL,                      /* no desired limits */
+                            &portals4_ni_limits,       /* actual limits */
+                            &portals4_nis_h[*num_btls] /* our interface handle */
+            );
         if (PTL_OK != ret) {
             opal_output_verbose(90, opal_btl_base_framework.framework_output,
-                            "%s:%d: PtlNIInit failed for NI %d: %d\n", __FILE__, __LINE__, interface, ret);
-        }
-        else {
+                                "%s:%d: PtlNIInit failed for NI %d: %d\n", __FILE__, __LINE__,
+                                interface, ret);
+        } else {
             if (mca_btl_portals4_component.portals_max_msg_size > portals4_ni_limits.max_msg_size)
                 mca_btl_portals4_component.portals_max_msg_size = portals4_ni_limits.max_msg_size;
             if (mca_btl_portals4_module.super.btl_max_send_size > portals4_ni_limits.max_msg_size)
                 mca_btl_portals4_module.super.btl_max_send_size = portals4_ni_limits.max_msg_size;
             if (mca_btl_portals4_module.super.btl_get_limit > portals4_ni_limits.max_msg_size)
                 mca_btl_portals4_module.super.btl_get_limit = portals4_ni_limits.max_msg_size;
-            OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "PtlNIInit OK for NI %d max_msg_size=%ld",
-                                 *num_btls, mca_btl_portals4_component.portals_max_msg_size));
+            OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
+                                 "PtlNIInit OK for NI %d max_msg_size=%ld", *num_btls,
+                                 mca_btl_portals4_component.portals_max_msg_size));
 
             (*num_btls)++;
         }
     }
-    if (0 == *num_btls) goto error;
+    if (0 == *num_btls)
+        goto error;
 
     /*
      * Configure the different network interfaces and the associated btl modules
      */
     mca_btl_portals4_component.num_btls = *num_btls;
-    mca_btl_portals4_component.btls = malloc(mca_btl_portals4_component.num_btls * sizeof(mca_btl_portals4_module_t*) );
-    mca_btl_portals4_component.eqs_h = malloc(mca_btl_portals4_component.num_btls * sizeof(ptl_handle_eq_t));
-    ptl_process_ids = malloc(mca_btl_portals4_component.num_btls * sizeof(ptl_process_t) );
+    mca_btl_portals4_component.btls = malloc(mca_btl_portals4_component.num_btls
+                                             * sizeof(mca_btl_portals4_module_t *));
+    mca_btl_portals4_component.eqs_h = malloc(mca_btl_portals4_component.num_btls
+                                              * sizeof(ptl_handle_eq_t));
+    ptl_process_ids = malloc(mca_btl_portals4_component.num_btls * sizeof(ptl_process_t));
 
-    for (interface=0; interface<mca_btl_portals4_component.num_btls; interface++) {
+    for (interface = 0; interface < mca_btl_portals4_component.num_btls; interface++) {
         mca_btl_portals4_component.btls[interface] = NULL;
         mca_btl_portals4_component.eqs_h[interface] = PTL_EQ_NONE;
     }
-    for (interface=0; interface<mca_btl_portals4_component.num_btls; interface++) {
+    for (interface = 0; interface < mca_btl_portals4_component.num_btls; interface++) {
         portals4_btl = malloc(sizeof(mca_btl_portals4_module_t));
         mca_btl_portals4_component.btls[interface] = portals4_btl;
 
@@ -395,46 +352,43 @@ static mca_btl_base_module_t** mca_btl_portals4_component_init(int *num_btls,
 
         portals4_btl->interface_num = interface;
         portals4_btl->portals_ni_h = portals4_nis_h[interface];
-        portals4_btl->portals_max_outstanding_ops = mca_btl_portals4_component.portals_max_outstanding_ops;
+        portals4_btl->portals_max_outstanding_ops = mca_btl_portals4_component
+                                                        .portals_max_outstanding_ops;
 
         OBJ_CONSTRUCT(&(portals4_btl->portals_frag_eager), opal_free_list_t);
         OBJ_CONSTRUCT(&(portals4_btl->portals_frag_max), opal_free_list_t);
         OBJ_CONSTRUCT(&(portals4_btl->portals_frag_user), opal_free_list_t);
 
         /* eager frags */
-        opal_free_list_init (&(portals4_btl->portals_frag_eager),
-                        sizeof(mca_btl_portals4_frag_eager_t) +
-                        portals4_btl->super.btl_eager_limit,
-                        opal_cache_line_size,
-                        OBJ_CLASS(mca_btl_portals4_frag_eager_t),
-                        0,opal_cache_line_size,
-                        mca_btl_portals4_component.portals_free_list_init_num,
-                        mca_btl_portals4_component.portals_free_list_eager_max_num,
-                        mca_btl_portals4_component.portals_free_list_inc_num,
-                        NULL, 0, NULL, NULL, NULL);
+        opal_free_list_init(&(portals4_btl->portals_frag_eager),
+                            sizeof(mca_btl_portals4_frag_eager_t)
+                                + portals4_btl->super.btl_eager_limit,
+                            opal_cache_line_size, OBJ_CLASS(mca_btl_portals4_frag_eager_t), 0,
+                            opal_cache_line_size,
+                            mca_btl_portals4_component.portals_free_list_init_num,
+                            mca_btl_portals4_component.portals_free_list_eager_max_num,
+                            mca_btl_portals4_component.portals_free_list_inc_num, NULL, 0, NULL,
+                            NULL, NULL);
 
         /* send frags */
-        opal_free_list_init (&(portals4_btl->portals_frag_max),
-                        sizeof(mca_btl_portals4_frag_max_t) +
-                        portals4_btl->super.btl_max_send_size,
-                        opal_cache_line_size,
-                        OBJ_CLASS(mca_btl_portals4_frag_max_t),
-                        0,opal_cache_line_size,
-                        mca_btl_portals4_component.portals_free_list_init_num,
-                        mca_btl_portals4_component.portals_free_list_max_num,
-                        mca_btl_portals4_component.portals_free_list_inc_num,
-                        NULL, 0, NULL, NULL, NULL);
+        opal_free_list_init(&(portals4_btl->portals_frag_max),
+                            sizeof(mca_btl_portals4_frag_max_t)
+                                + portals4_btl->super.btl_max_send_size,
+                            opal_cache_line_size, OBJ_CLASS(mca_btl_portals4_frag_max_t), 0,
+                            opal_cache_line_size,
+                            mca_btl_portals4_component.portals_free_list_init_num,
+                            mca_btl_portals4_component.portals_free_list_max_num,
+                            mca_btl_portals4_component.portals_free_list_inc_num, NULL, 0, NULL,
+                            NULL, NULL);
 
         /* user frags */
-        opal_free_list_init (&(portals4_btl->portals_frag_user),
-                        sizeof(mca_btl_portals4_frag_user_t),
-                        opal_cache_line_size,
-                        OBJ_CLASS(mca_btl_portals4_frag_user_t),
-                        0,opal_cache_line_size,
-                        mca_btl_portals4_component.portals_free_list_init_num,
-                        mca_btl_portals4_component.portals_free_list_max_num,
-                        mca_btl_portals4_component.portals_free_list_inc_num,
-                        NULL, 0, NULL, NULL, NULL);
+        opal_free_list_init(&(portals4_btl->portals_frag_user),
+                            sizeof(mca_btl_portals4_frag_user_t), opal_cache_line_size,
+                            OBJ_CLASS(mca_btl_portals4_frag_user_t), 0, opal_cache_line_size,
+                            mca_btl_portals4_component.portals_free_list_init_num,
+                            mca_btl_portals4_component.portals_free_list_max_num,
+                            mca_btl_portals4_component.portals_free_list_inc_num, NULL, 0, NULL,
+                            NULL, NULL);
 
         /* receive block list */
         OBJ_CONSTRUCT(&(portals4_btl->portals_recv_blocks), opal_list_t);
@@ -443,70 +397,73 @@ static mca_btl_base_module_t** mca_btl_portals4_component_init(int *num_btls,
     portals4_nis_h = NULL;
 
     /* Publish our NID(s)/PID(s) in the modex */
-    for (interface=0; interface<mca_btl_portals4_component.num_btls; interface++) {
+    for (interface = 0; interface < mca_btl_portals4_component.num_btls; interface++) {
         portals4_btl = mca_btl_portals4_component.btls[interface];
 
-        ret = PtlGetPhysId(portals4_btl->portals_ni_h ,&ptl_process_ids[interface]);
+        ret = PtlGetPhysId(portals4_btl->portals_ni_h, &ptl_process_ids[interface]);
         if (PTL_OK != ret) {
             opal_output_verbose(1, opal_btl_base_framework.framework_output,
-                            "%s:%d: PtlGetPhysId for NI %d failed: %d\n",
-                            __FILE__, __LINE__, interface, ret);
+                                "%s:%d: PtlGetPhysId for NI %d failed: %d\n", __FILE__, __LINE__,
+                                interface, ret);
             goto error;
         }
 
         OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
-                  "PtlGetPhysId NI number %d: ni_h=%d rank=%x nid=%x pid=%x\n",
-                  interface, portals4_btl->portals_ni_h,
-                  ptl_process_ids[interface].rank,
-                  ptl_process_ids[interface].phys.nid, ptl_process_ids[interface].phys.pid));
+                             "PtlGetPhysId NI number %d: ni_h=%d rank=%x nid=%x pid=%x\n",
+                             interface, portals4_btl->portals_ni_h, ptl_process_ids[interface].rank,
+                             ptl_process_ids[interface].phys.nid,
+                             ptl_process_ids[interface].phys.pid));
     }
-    OPAL_MODEX_SEND(ret, PMIX_GLOBAL,
-                    &mca_btl_portals4_component.super.btl_version,
+    OPAL_MODEX_SEND(ret, PMIX_GLOBAL, &mca_btl_portals4_component.super.btl_version,
                     ptl_process_ids, mca_btl_portals4_component.num_btls * sizeof(ptl_process_t));
     if (OPAL_SUCCESS != ret) {
         opal_output_verbose(1, opal_btl_base_framework.framework_output,
-                        "%s:%d: opal_modex_send failed: %d\n",
-                        __FILE__, __LINE__, ret);
+                            "%s:%d: opal_modex_send failed: %d\n", __FILE__, __LINE__, ret);
         goto error;
     }
     free(ptl_process_ids);
     ptl_process_ids = NULL;
 
-    btls = malloc(mca_btl_portals4_component.num_btls * sizeof(mca_btl_portals4_module_t*) );
-    memcpy(btls , mca_btl_portals4_component.btls,
-            mca_btl_portals4_component.num_btls*sizeof(mca_btl_portals4_module_t*) );
+    btls = malloc(mca_btl_portals4_component.num_btls * sizeof(mca_btl_portals4_module_t *));
+    memcpy(btls, mca_btl_portals4_component.btls,
+           mca_btl_portals4_component.num_btls * sizeof(mca_btl_portals4_module_t *));
 
-    opal_output_verbose(1, opal_btl_base_framework.framework_output, "The btl portals4 component has been initialized and uses %d NI(s)",
-        mca_btl_portals4_component.num_btls);
+    opal_output_verbose(1, opal_btl_base_framework.framework_output,
+                        "The btl portals4 component has been initialized and uses %d NI(s)",
+                        mca_btl_portals4_component.num_btls);
 
     mca_btl_portals4_component.need_init = 1;
 
     return btls;
 
- error:
-    opal_output_verbose(1, opal_btl_base_framework.framework_output, "Error in mca_btl_portals4_component_init\n");
+error:
+    opal_output_verbose(1, opal_btl_base_framework.framework_output,
+                        "Error in mca_btl_portals4_component_init\n");
 
     if (*num_btls) {
-        if (NULL != portals4_nis_h) free(portals4_nis_h);
-        if (NULL != ptl_process_ids) free(ptl_process_ids);
+        if (NULL != portals4_nis_h)
+            free(portals4_nis_h);
+        if (NULL != ptl_process_ids)
+            free(ptl_process_ids);
 
-        for (interface=0; interface<mca_btl_portals4_component.num_btls; interface++) {
+        for (interface = 0; interface < mca_btl_portals4_component.num_btls; interface++) {
             portals4_btl = mca_btl_portals4_component.btls[interface];
-            if (NULL != portals4_btl) mca_btl_portals4_free_module(portals4_btl);
+            if (NULL != portals4_btl)
+                mca_btl_portals4_free_module(portals4_btl);
         }
         mca_btl_portals4_component.num_btls = 0;
         *num_btls = 0;
-        if (NULL != mca_btl_portals4_component.btls)  free(mca_btl_portals4_component.btls);
-        if (NULL != mca_btl_portals4_component.eqs_h) free(mca_btl_portals4_component.eqs_h);
+        if (NULL != mca_btl_portals4_component.btls)
+            free(mca_btl_portals4_component.btls);
+        if (NULL != mca_btl_portals4_component.eqs_h)
+            free(mca_btl_portals4_component.eqs_h);
         mca_btl_portals4_component.btls = NULL;
         mca_btl_portals4_component.eqs_h = NULL;
-
     }
- return NULL;
+    return NULL;
 }
 
-int
-mca_btl_portals4_get_error(int ptl_error)
+int mca_btl_portals4_get_error(int ptl_error)
 {
     int ret;
 
@@ -564,8 +521,7 @@ mca_btl_portals4_get_error(int ptl_error)
     return ret;
 }
 
-int
-mca_btl_portals4_component_progress(void)
+int mca_btl_portals4_component_progress(void)
 {
     mca_btl_portals4_module_t *portals4_btl;
     int num_progressed = 0;
@@ -574,38 +530,41 @@ mca_btl_portals4_component_progress(void)
     mca_btl_base_tag_t tag;
     static ptl_event_t ev;
     unsigned int which;
-    mca_btl_active_message_callback_t* reg;
+    mca_btl_active_message_callback_t *reg;
     mca_btl_base_segment_t seg[1];
-    mca_btl_base_receive_descriptor_t recv_descriptor = {.des_segments = seg, .des_segment_count = 1};
+    mca_btl_base_receive_descriptor_t recv_descriptor = {.des_segments = seg,
+                                                         .des_segment_count = 1};
 
     while (true) {
-        ret = PtlEQPoll(mca_btl_portals4_component.eqs_h, mca_btl_portals4_component.num_btls, 0, &ev, &which);
+        ret = PtlEQPoll(mca_btl_portals4_component.eqs_h, mca_btl_portals4_component.num_btls, 0,
+                        &ev, &which);
 
         if (PTL_OK == ret) {
-            OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "PtlEQPoll Event received: %d (fail=%d) on NI %d\n",
-                ev.type, ev.ni_fail_type, which));
+            OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
+                                 "PtlEQPoll Event received: %d (fail=%d) on NI %d\n", ev.type,
+                                 ev.ni_fail_type, which));
             num_progressed++;
             portals4_btl = mca_btl_portals4_component.btls[which];
 
             switch (ev.type) {
 
-            case PTL_EVENT_SEND:   /* generated on source (origin) when put stops sending */
+            case PTL_EVENT_SEND: /* generated on source (origin) when put stops sending */
 
                 frag = ev.user_ptr;
                 if (NULL == frag) {
-                    opal_output(opal_btl_base_framework.framework_output, "btl/portals4: PTL_EVENT_SEND event with NULL user_ptr");
+                    opal_output(opal_btl_base_framework.framework_output,
+                                "btl/portals4: PTL_EVENT_SEND event with NULL user_ptr");
                     break;
                 }
                 btl_ownership = (frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
 
                 if (!mca_btl_portals4_component.portals_need_ack) {
                     /* my part's done, in portals we trust! */
-                    if( MCA_BTL_DES_SEND_ALWAYS_CALLBACK & frag->base.des_flags ){
+                    if (MCA_BTL_DES_SEND_ALWAYS_CALLBACK & frag->base.des_flags) {
                         OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
-                            "PTL_EVENT_SEND: Direct call to des_cbfunc: %lx\n", (uint64_t)frag->base.des_cbfunc));
-                        frag->base.des_cbfunc(&portals4_btl->super,
-                                              frag->endpoint,
-                                              &frag->base,
+                                             "PTL_EVENT_SEND: Direct call to des_cbfunc: %lx\n",
+                                             (uint64_t) frag->base.des_cbfunc));
+                        frag->base.des_cbfunc(&portals4_btl->super, frag->endpoint, &frag->base,
                                               OPAL_SUCCESS);
                     }
                     if (btl_ownership) {
@@ -613,35 +572,39 @@ mca_btl_portals4_component_progress(void)
                     }
                     if (0 != frag->size) {
                         OPAL_THREAD_ADD_FETCH32(&portals4_btl->portals_outstanding_ops, -1);
-                        OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
-                            "PTL_EVENT_SEND: Decrementing portals_outstanding_ops=%d (1)\n",
-                            portals4_btl->portals_outstanding_ops));
+                        OPAL_OUTPUT_VERBOSE(
+                            (90, opal_btl_base_framework.framework_output,
+                             "PTL_EVENT_SEND: Decrementing portals_outstanding_ops=%d (1)\n",
+                             portals4_btl->portals_outstanding_ops));
                     }
                 }
 
                 goto done;
                 break;
 
-            case PTL_EVENT_ACK:   /* Ack that a put as completed on other side. We just call the callback function */
+            case PTL_EVENT_ACK: /* Ack that a put as completed on other side. We just call the
+                                   callback function */
 
                 frag = ev.user_ptr;
                 if (NULL == frag) {
-                    opal_output(opal_btl_base_framework.framework_output, "btl/portals4: PTL_EVENT_ACK event with NULL user_ptr");
+                    opal_output(opal_btl_base_framework.framework_output,
+                                "btl/portals4: PTL_EVENT_ACK event with NULL user_ptr");
                     break;
                 }
-                OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
-                    "PTL_EVENT_ACK received rlength=%ld mlength=%ld des_flags=%d\n", ev.rlength, ev.mlength, frag->base.des_flags));
+                OPAL_OUTPUT_VERBOSE(
+                    (90, opal_btl_base_framework.framework_output,
+                     "PTL_EVENT_ACK received rlength=%ld mlength=%ld des_flags=%d\n", ev.rlength,
+                     ev.mlength, frag->base.des_flags));
                 btl_ownership = (frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
 
                 /* other side received the message.  should have
                    received entire thing */
                 /* let the PML know we're done */
-                if (MCA_BTL_DES_SEND_ALWAYS_CALLBACK & frag->base.des_flags ) {
+                if (MCA_BTL_DES_SEND_ALWAYS_CALLBACK & frag->base.des_flags) {
                     OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
-                        "PTL_EVENT_ACK: Call to des_cbfunc %lx\n", (uint64_t)frag->base.des_cbfunc));
-                    frag->base.des_cbfunc(&portals4_btl->super,
-                                          frag->endpoint,
-                                          &frag->base,
+                                         "PTL_EVENT_ACK: Call to des_cbfunc %lx\n",
+                                         (uint64_t) frag->base.des_cbfunc));
+                    frag->base.des_cbfunc(&portals4_btl->super, frag->endpoint, &frag->base,
                                           OPAL_SUCCESS);
                 }
                 if (btl_ownership) {
@@ -650,14 +613,16 @@ mca_btl_portals4_component_progress(void)
 
                 if (0 != frag->size) {
                     OPAL_THREAD_ADD_FETCH32(&portals4_btl->portals_outstanding_ops, -1);
-                    OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
-                        "PTL_EVENT_ACK: Decrementing portals_outstanding_ops=%d (2)\n", portals4_btl->portals_outstanding_ops));
+                    OPAL_OUTPUT_VERBOSE(
+                        (90, opal_btl_base_framework.framework_output,
+                         "PTL_EVENT_ACK: Decrementing portals_outstanding_ops=%d (2)\n",
+                         portals4_btl->portals_outstanding_ops));
                 }
 
                 goto done;
                 break;
 
-            case PTL_EVENT_PUT:   /* Generated on destination (target) when a put into memory ends */
+            case PTL_EVENT_PUT: /* Generated on destination (target) when a put into memory ends */
 
                 tag = (unsigned char) (ev.hdr_data);
 
@@ -671,7 +636,8 @@ mca_btl_portals4_component_progress(void)
                 recv_descriptor.cbdata = reg->cbdata;
 
                 OPAL_OUTPUT_VERBOSE((50, opal_btl_base_framework.framework_output,
-                    "PTL_EVENT_PUT: tag=%x base_descriptor=%p cbfunc: %lx\n", tag, (void*)&btl_base_descriptor, (uint64_t)reg->cbfunc));
+                                     "PTL_EVENT_PUT: tag=%x recv_descriptor=%p cbfunc: %lx\n", tag,
+                                     (void *) &recv_descriptor, (uint64_t) reg->cbfunc));
                 reg->cbfunc(&portals4_btl->super, &recv_descriptor);
 
                 goto done;
@@ -680,7 +646,7 @@ mca_btl_portals4_component_progress(void)
             case PTL_EVENT_PUT_OVERFLOW:
                 /* */
                 OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
-                    "PTL_EVENT_OVERFLOW received\n"));
+                                     "PTL_EVENT_OVERFLOW received\n"));
                 goto done;
                 break;
 
@@ -688,7 +654,8 @@ mca_btl_portals4_component_progress(void)
                 /* */
                 frag = ev.user_ptr;
                 if (NULL == frag) {
-                    opal_output(opal_btl_base_framework.framework_output, "btl/portals4: PTL_EVENT_LINK event with NULL user_ptr");
+                    opal_output(opal_btl_base_framework.framework_output,
+                                "btl/portals4: PTL_EVENT_LINK event with NULL user_ptr");
                     break;
                 }
                 goto done;
@@ -696,7 +663,8 @@ mca_btl_portals4_component_progress(void)
 
             case PTL_EVENT_AUTO_UNLINK:
                 /* */
-                /* The Priority List is used, so PTL_EVENT_AUTO_FREE will never be received. So, we have to reactivate the block here */
+                /* The Priority List is used, so PTL_EVENT_AUTO_FREE will never be received. So, we
+                 * have to reactivate the block here */
                 mca_btl_portals4_activate_block(ev.user_ptr);
                 goto done;
                 break;
@@ -706,10 +674,11 @@ mca_btl_portals4_component_progress(void)
                 goto done;
                 break;
 
-            case PTL_EVENT_GET:   /* Generated on source (target) when a get from memory ends */
+            case PTL_EVENT_GET: /* Generated on source (target) when a get from memory ends */
                 /* */
                 OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
-                    "PTL_EVENT_GET received at target rlength=%ld mlength=%ld\n", ev.rlength, ev.mlength));
+                                     "PTL_EVENT_GET received at target rlength=%ld mlength=%ld\n",
+                                     ev.rlength, ev.mlength));
                 goto done;
                 break;
 
@@ -718,46 +687,45 @@ mca_btl_portals4_component_progress(void)
                 frag = ev.user_ptr;
 
                 if (PTL_NI_PERM_VIOLATION == ev.ni_fail_type) {
-                        opal_output_verbose(1, opal_btl_base_framework.framework_output,
-                            "Warning : PTL_EVENT_REPLY with PTL_NI_PERM_VIOLATION received, try to re-issue a PtlGet");
+                    opal_output_verbose(1, opal_btl_base_framework.framework_output,
+                                        "Warning : PTL_EVENT_REPLY with PTL_NI_PERM_VIOLATION "
+                                        "received, try to re-issue a PtlGet");
 
-                    /* The distant PtlMEAppend is not finished (distant PTL_EVENT_LINK not received) */
+                    /* The distant PtlMEAppend is not finished (distant PTL_EVENT_LINK not received)
+                     */
                     /* Re-issue the PtlGet (see btl_portals4_rdma.c) */
-                    ret = PtlGet(portals4_btl->send_md_h,
-                                 (ptl_size_t) frag->addr,
-                                 frag->length,
-                                 frag->peer_proc,
-                                 portals4_btl->recv_idx,
+                    ret = PtlGet(portals4_btl->send_md_h, (ptl_size_t) frag->addr, frag->length,
+                                 frag->peer_proc, portals4_btl->recv_idx,
                                  frag->match_bits, /* match bits */
                                  0, // Warning : should be  ev.remote_offset but it is not defined,
                                  frag);
                     if (OPAL_UNLIKELY(PTL_OK != ret)) {
                         opal_output_verbose(1, opal_btl_base_framework.framework_output,
-                                            "%s:%d: Re-issued PtlGet failed: %d",
-                                            __FILE__, __LINE__, ret);
+                                            "%s:%d: Re-issued PtlGet failed: %d", __FILE__,
+                                            __LINE__, ret);
                         return OPAL_ERROR;
                     }
 
                     OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
-                        "Re-issued PtlGet length=%ld recv_idx=%d rank=%x pid=%x nid=%x match_bits=%lx\n",
-                        frag->length, portals4_btl->recv_idx,
-                        frag->peer_proc.rank, frag->peer_proc.phys.pid, frag->peer_proc.phys.nid, frag->match_bits));
-                }
-                else {
+                                         "Re-issued PtlGet length=%ld recv_idx=%d rank=%x pid=%x "
+                                         "nid=%x match_bits=%lx\n",
+                                         frag->length, portals4_btl->recv_idx, frag->peer_proc.rank,
+                                         frag->peer_proc.phys.pid, frag->peer_proc.phys.nid,
+                                         frag->match_bits));
+                } else {
                     OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
-                        "PTL_EVENT_REPLY: Call to rdma_cbfunc=%lx\n", (uint64_t)frag->rdma_cb.func));
-                    frag->rdma_cb.func(&portals4_btl->super,
-                                 frag->endpoint,
-                                 ev.start,
-                                 frag->rdma_cb.local_handle,
-                                 frag->rdma_cb.context,
-                                 frag->rdma_cb.data,
-                                 OPAL_SUCCESS);
+                                         "PTL_EVENT_REPLY: Call to rdma_cbfunc=%lx\n",
+                                         (uint64_t) frag->rdma_cb.func));
+                    frag->rdma_cb.func(&portals4_btl->super, frag->endpoint, ev.start,
+                                       frag->rdma_cb.local_handle, frag->rdma_cb.context,
+                                       frag->rdma_cb.data, OPAL_SUCCESS);
 
                     OPAL_BTL_PORTALS4_FRAG_RETURN_USER(&portals4_btl->super, frag);
                     OPAL_THREAD_ADD_FETCH32(&portals4_btl->portals_outstanding_ops, -1);
-                    OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
-                        "PTL_EVENT_REPLY: Decrementing portals_outstanding_ops=%d\n", portals4_btl->portals_outstanding_ops));
+                    OPAL_OUTPUT_VERBOSE(
+                        (90, opal_btl_base_framework.framework_output,
+                         "PTL_EVENT_REPLY: Decrementing portals_outstanding_ops=%d\n",
+                         portals4_btl->portals_outstanding_ops));
                     goto done;
                 }
                 break;
@@ -785,6 +753,6 @@ mca_btl_portals4_component_progress(void)
             break;
         }
     }
- done:
+done:
     return num_progressed;
 }
diff --git a/opal/mca/btl/portals4/btl_portals4_endpoint.h b/opal/mca/btl/portals4/btl_portals4_endpoint.h
index 53c50c14079..4dfad610da9 100644
--- a/opal/mca/btl/portals4/btl_portals4_endpoint.h
+++ b/opal/mca/btl/portals4/btl_portals4_endpoint.h
@@ -21,6 +21,10 @@
 #ifndef OPAL_BTL_PORTALS4_ENDPOINT_H
 #define OPAL_BTL_PORTALS4_ENDPOINT_H
 
+#include "opal_config.h"
+
+#include "opal/mca/btl/btl.h"
+
 #include "btl_portals4.h"
 
 BEGIN_C_DECLS
diff --git a/opal/mca/btl/portals4/btl_portals4_frag.c b/opal/mca/btl/portals4/btl_portals4_frag.c
index 6aa20a14af4..56df617a2c6 100644
--- a/opal/mca/btl/portals4/btl_portals4_frag.c
+++ b/opal/mca/btl/portals4/btl_portals4_frag.c
@@ -23,8 +23,7 @@
 #include "btl_portals4.h"
 #include "btl_portals4_frag.h"
 
-static void
-mca_btl_portals4_frag_common_send_constructor(mca_btl_portals4_frag_t* frag)
+static void mca_btl_portals4_frag_common_send_constructor(mca_btl_portals4_frag_t *frag)
 {
     frag->base.des_flags = 0;
     frag->base.des_segments = &frag->segments[0].base;
@@ -37,59 +36,42 @@ mca_btl_portals4_frag_common_send_constructor(mca_btl_portals4_frag_t* frag)
     frag->me_h = PTL_INVALID_HANDLE;
 }
 
-static void
-mca_btl_portals4_frag_eager_constructor(mca_btl_portals4_frag_t* frag)
+static void mca_btl_portals4_frag_eager_constructor(mca_btl_portals4_frag_t *frag)
 {
     frag->size = mca_btl_portals4_module.super.btl_eager_limit;
     mca_btl_portals4_frag_common_send_constructor(frag);
     frag->type = BTL_PORTALS4_FRAG_TYPE_EAGER;
 }
 
-static void
-mca_btl_portals4_frag_eager_destructor(mca_btl_portals4_frag_t* frag)
+static void mca_btl_portals4_frag_eager_destructor(mca_btl_portals4_frag_t *frag)
 {
-    if (PTL_INVALID_HANDLE != frag->me_h) {
+    if (!PtlHandleIsEqual(frag->me_h, PTL_INVALID_HANDLE)) {
         PtlMEUnlink(frag->me_h);
         frag->me_h = PTL_INVALID_HANDLE;
     }
 }
 
-static void
-mca_btl_portals4_frag_max_constructor(mca_btl_portals4_frag_t* frag)
+static void mca_btl_portals4_frag_max_constructor(mca_btl_portals4_frag_t *frag)
 {
     frag->size = mca_btl_portals4_module.super.btl_max_send_size;
     mca_btl_portals4_frag_common_send_constructor(frag);
     frag->type = BTL_PORTALS4_FRAG_TYPE_MAX;
 }
 
-static void
-mca_btl_portals4_frag_user_constructor(mca_btl_portals4_frag_t* frag)
+static void mca_btl_portals4_frag_user_constructor(mca_btl_portals4_frag_t *frag)
 {
     frag->base.des_flags = 0;
     frag->size = 0;
     frag->type = BTL_PORTALS4_FRAG_TYPE_USER;
 }
 
-OBJ_CLASS_INSTANCE(
-    mca_btl_portals4_frag_t,
-    mca_btl_base_descriptor_t,
-    NULL,
-    NULL);
+OBJ_CLASS_INSTANCE(mca_btl_portals4_frag_t, mca_btl_base_descriptor_t, NULL, NULL);
 
-OBJ_CLASS_INSTANCE(
-    mca_btl_portals4_frag_eager_t,
-    mca_btl_base_descriptor_t,
-    mca_btl_portals4_frag_eager_constructor,
-    mca_btl_portals4_frag_eager_destructor);
+OBJ_CLASS_INSTANCE(mca_btl_portals4_frag_eager_t, mca_btl_base_descriptor_t,
+                   mca_btl_portals4_frag_eager_constructor, mca_btl_portals4_frag_eager_destructor);
 
-OBJ_CLASS_INSTANCE(
-    mca_btl_portals4_frag_max_t,
-    mca_btl_base_descriptor_t,
-    mca_btl_portals4_frag_max_constructor,
-    NULL);
+OBJ_CLASS_INSTANCE(mca_btl_portals4_frag_max_t, mca_btl_base_descriptor_t,
+                   mca_btl_portals4_frag_max_constructor, NULL);
 
-OBJ_CLASS_INSTANCE(
-    mca_btl_portals4_frag_user_t,
-    mca_btl_base_descriptor_t,
-    mca_btl_portals4_frag_user_constructor,
-    NULL);
+OBJ_CLASS_INSTANCE(mca_btl_portals4_frag_user_t, mca_btl_base_descriptor_t,
+                   mca_btl_portals4_frag_user_constructor, NULL);
diff --git a/opal/mca/btl/portals4/btl_portals4_frag.h b/opal/mca/btl/portals4/btl_portals4_frag.h
index 8f3c6e49ac3..e192304bd8c 100644
--- a/opal/mca/btl/portals4/btl_portals4_frag.h
+++ b/opal/mca/btl/portals4/btl_portals4_frag.h
@@ -23,13 +23,15 @@
 #ifndef OPAL_BTL_PORTALS4_FRAG_H
 #define OPAL_BTL_PORTALS4_FRAG_H
 
-#include "opal/mca/btl/btl.h"
+#include "opal_config.h"
+
+#include "btl_portals4.h"
 
 BEGIN_C_DECLS
 
 struct mca_btl_portals4_segment_t {
-  mca_btl_base_segment_t base;
-  ptl_match_bits_t key;
+    mca_btl_base_segment_t base;
+    ptl_match_bits_t key;
 };
 typedef struct mca_btl_portals4_segment_t mca_btl_portals4_segment_t;
 
@@ -40,7 +42,7 @@ struct mca_btl_portals4_frag_t {
     mca_btl_base_descriptor_t base;
     mca_btl_portals4_segment_t segments[1];
     /* needed for retransmit case */
-   struct mca_btl_base_endpoint_t *endpoint;
+    struct mca_btl_base_endpoint_t *endpoint;
     /* needed for retransmit case */
     mca_btl_base_header_t hdr;
     /* handle to use for communication */
@@ -64,10 +66,12 @@ struct mca_btl_portals4_frag_t {
         mca_btl_base_registration_handle_t *local_handle;
     } rdma_cb;
 
-    enum { BTL_PORTALS4_FRAG_TYPE_EAGER,
-           BTL_PORTALS4_FRAG_TYPE_MAX,
-           BTL_PORTALS4_FRAG_TYPE_USER } type;
-    unsigned char  data[16];
+    enum {
+        BTL_PORTALS4_FRAG_TYPE_EAGER,
+        BTL_PORTALS4_FRAG_TYPE_MAX,
+        BTL_PORTALS4_FRAG_TYPE_USER
+    } type;
+    unsigned char data[16];
 };
 typedef struct mca_btl_portals4_frag_t mca_btl_portals4_frag_t;
 OBJ_CLASS_DECLARATION(mca_btl_portals4_frag_t);
@@ -85,54 +89,48 @@ OBJ_CLASS_DECLARATION(mca_btl_portals4_frag_user_t);
  * Macros to allocate/return descriptors from module specific
  * free list(s).
  */
-#define OPAL_BTL_PORTALS4_FRAG_ALLOC_EAGER(btl_macro, frag)                                \
-{                                                                                          \
-    frag = (mca_btl_portals4_frag_t *)                                                     \
-        opal_free_list_get (&((mca_btl_portals4_module_t*)btl_macro)->portals_frag_eager); \
-    if (NULL == frag) {                                                                    \
-        OPAL_BTL_PORTALS4_FRAG_ALLOC_MAX(btl_macro, frag);                                 \
-    }                                                                                      \
-}
-
-
-#define OPAL_BTL_PORTALS4_FRAG_RETURN_EAGER(btl_macro, frag)                            \
-{                                                                                       \
-    assert(BTL_PORTALS4_FRAG_TYPE_EAGER == frag->type);                                 \
-    opal_free_list_return (&((mca_btl_portals4_module_t*)btl_macro)->portals_frag_eager, \
-        (opal_free_list_item_t*)(frag));                                                \
-}
-
-
-#define OPAL_BTL_PORTALS4_FRAG_ALLOC_MAX(btl_macro, frag)                                           \
-{                                                                                                   \
-    frag = (mca_btl_portals4_frag_t*)                                                               \
-        opal_free_list_get (&((mca_btl_portals4_module_t*)btl_macro)->portals_frag_max);            \
-}
-
-
-#define OPAL_BTL_PORTALS4_FRAG_RETURN_MAX(btl_macro, frag)                            \
-{                                                                                     \
-    assert(BTL_PORTALS4_FRAG_TYPE_MAX == frag->type);                                 \
-    opal_free_list_return (&((mca_btl_portals4_module_t*)btl_macro)->portals_frag_max, \
-        (opal_free_list_item_t*)(frag));                                              \
-}
-
-
-#define OPAL_BTL_PORTALS4_FRAG_ALLOC_USER(btl_macro, frag)                                      \
-{                                                                                               \
-    frag = (mca_btl_portals4_frag_t*)                                                           \
-        opal_free_list_get (&((mca_btl_portals4_module_t*)btl_macro)->portals_frag_user);       \
-    frag->base.des_cbfunc = NULL;                                                               \
-}
-
-
-#define OPAL_BTL_PORTALS4_FRAG_RETURN_USER(btl_macro, frag)                            \
-{                                                                                      \
-    assert(BTL_PORTALS4_FRAG_TYPE_USER == frag->type);                                 \
-    opal_free_list_return (&((mca_btl_portals4_module_t*)btl_macro)->portals_frag_user, \
-        (opal_free_list_item_t*)(frag));                                               \
-}
-
+#define OPAL_BTL_PORTALS4_FRAG_ALLOC_EAGER(btl_macro, frag)                  \
+    {                                                                        \
+        frag = (mca_btl_portals4_frag_t *) opal_free_list_get(               \
+            &((mca_btl_portals4_module_t *) btl_macro)->portals_frag_eager); \
+        if (NULL == frag) {                                                  \
+            OPAL_BTL_PORTALS4_FRAG_ALLOC_MAX(btl_macro, frag);               \
+        }                                                                    \
+    }
+
+#define OPAL_BTL_PORTALS4_FRAG_RETURN_EAGER(btl_macro, frag)                                  \
+    {                                                                                         \
+        assert(BTL_PORTALS4_FRAG_TYPE_EAGER == frag->type);                                   \
+        opal_free_list_return(&((mca_btl_portals4_module_t *) btl_macro)->portals_frag_eager, \
+                              (opal_free_list_item_t *) (frag));                              \
+    }
+
+#define OPAL_BTL_PORTALS4_FRAG_ALLOC_MAX(btl_macro, frag)                  \
+    {                                                                      \
+        frag = (mca_btl_portals4_frag_t *) opal_free_list_get(             \
+            &((mca_btl_portals4_module_t *) btl_macro)->portals_frag_max); \
+    }
+
+#define OPAL_BTL_PORTALS4_FRAG_RETURN_MAX(btl_macro, frag)                                  \
+    {                                                                                       \
+        assert(BTL_PORTALS4_FRAG_TYPE_MAX == frag->type);                                   \
+        opal_free_list_return(&((mca_btl_portals4_module_t *) btl_macro)->portals_frag_max, \
+                              (opal_free_list_item_t *) (frag));                            \
+    }
+
+#define OPAL_BTL_PORTALS4_FRAG_ALLOC_USER(btl_macro, frag)                  \
+    {                                                                       \
+        frag = (mca_btl_portals4_frag_t *) opal_free_list_get(              \
+            &((mca_btl_portals4_module_t *) btl_macro)->portals_frag_user); \
+        frag->base.des_cbfunc = NULL;                                       \
+    }
+
+#define OPAL_BTL_PORTALS4_FRAG_RETURN_USER(btl_macro, frag)                                  \
+    {                                                                                        \
+        assert(BTL_PORTALS4_FRAG_TYPE_USER == frag->type);                                   \
+        opal_free_list_return(&((mca_btl_portals4_module_t *) btl_macro)->portals_frag_user, \
+                              (opal_free_list_item_t *) (frag));                             \
+    }
 
 END_C_DECLS
 #endif
diff --git a/opal/mca/btl/portals4/btl_portals4_rdma.c b/opal/mca/btl/portals4/btl_portals4_rdma.c
index 9237b30fce2..ea797c4eefb 100644
--- a/opal/mca/btl/portals4/btl_portals4_rdma.c
+++ b/opal/mca/btl/portals4/btl_portals4_rdma.c
@@ -19,13 +19,15 @@
  */
 
 #include "opal_config.h"
+
 #include "opal/constants.h"
+
 #include "btl_portals4.h"
+#include "btl_portals4_frag.h"
 
-int
-mca_btl_portals4_put(struct mca_btl_base_module_t* btl_base,
-                    struct mca_btl_base_endpoint_t* btl_peer,
-                    struct mca_btl_base_descriptor_t* descriptor)
+int mca_btl_portals4_put(struct mca_btl_base_module_t *btl_base,
+                         struct mca_btl_base_endpoint_t *btl_peer,
+                         struct mca_btl_base_descriptor_t *descriptor)
 {
     opal_output(opal_btl_base_framework.framework_output, "mca_btl_portals4_put not implemented\n");
 
@@ -33,45 +35,39 @@ mca_btl_portals4_put(struct mca_btl_base_module_t* btl_base,
     return OPAL_SUCCESS;
 }
 
-
-int
-mca_btl_portals4_get(struct mca_btl_base_module_t* btl_base,
-                    struct mca_btl_base_endpoint_t* btl_peer,
-                    void *local_address,
-                    uint64_t remote_address,
-                    struct mca_btl_base_registration_handle_t *local_handle,
-                    struct mca_btl_base_registration_handle_t *remote_handle,
-                    size_t size,
-                    int flags,
-                    int order,
-                    mca_btl_base_rdma_completion_fn_t cbfunc,
-                    void *cbcontext,
-                    void *cbdata)
+int mca_btl_portals4_get(struct mca_btl_base_module_t *btl_base,
+                         struct mca_btl_base_endpoint_t *btl_peer, void *local_address,
+                         uint64_t remote_address,
+                         struct mca_btl_base_registration_handle_t *local_handle,
+                         struct mca_btl_base_registration_handle_t *remote_handle, size_t size,
+                         int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
+                         void *cbcontext, void *cbdata)
 {
     mca_btl_portals4_module_t *portals4_btl = (mca_btl_portals4_module_t *) btl_base;
-    mca_btl_portals4_frag_t   *frag         = NULL;
+    mca_btl_portals4_frag_t *frag = NULL;
     int ret;
 
     /* reserve space in the event queue for rdma operations immediately */
-    while (OPAL_THREAD_ADD_FETCH32(&portals4_btl->portals_outstanding_ops, 1) >
-           portals4_btl->portals_max_outstanding_ops) {
+    while (OPAL_THREAD_ADD_FETCH32(&portals4_btl->portals_outstanding_ops, 1)
+           > portals4_btl->portals_max_outstanding_ops) {
         OPAL_THREAD_ADD_FETCH32(&portals4_btl->portals_outstanding_ops, -1);
-        OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "Call to mca_btl_portals4_component_progress (1)\n"));
+        OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
+                             "Call to mca_btl_portals4_component_progress (1)\n"));
         mca_btl_portals4_component_progress();
     }
 
     OPAL_BTL_PORTALS4_FRAG_ALLOC_USER(portals4_btl, frag);
-    if (NULL == frag){
+    if (NULL == frag) {
         OPAL_THREAD_ADD_FETCH32(&portals4_btl->portals_outstanding_ops, -1);
         return OPAL_ERROR;
     }
     OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
-        "mca_btl_portals4_get: Incrementing portals_outstanding_ops=%d frag=%p",
-        portals4_btl->portals_outstanding_ops, (void *)frag));
+                         "mca_btl_portals4_get: Incrementing portals_outstanding_ops=%d frag=%p",
+                         portals4_btl->portals_outstanding_ops, (void *) frag));
 
-    frag->rdma_cb.func         = cbfunc;
-    frag->rdma_cb.context      = cbcontext;
-    frag->rdma_cb.data         = cbdata;
+    frag->rdma_cb.func = cbfunc;
+    frag->rdma_cb.context = cbcontext;
+    frag->rdma_cb.data = cbdata;
     frag->rdma_cb.local_handle = local_handle;
 
     frag->endpoint = btl_peer;
@@ -82,20 +78,17 @@ mca_btl_portals4_get(struct mca_btl_base_module_t* btl_base,
     frag->length = size;
     frag->peer_proc = btl_peer->ptl_proc;
 
-    OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "PtlGet offset=%p length=%ld remote_offset=%p nid=%x pid=%x match_bits=%lx",
-        local_address, size, (void*)local_handle->remote_offset, btl_peer->ptl_proc.phys.nid, btl_peer->ptl_proc.phys.pid, frag->match_bits));
+    OPAL_OUTPUT_VERBOSE(
+        (90, opal_btl_base_framework.framework_output,
+         "PtlGet offset=%p length=%ld remote_offset=%p nid=%x pid=%x match_bits=%lx", local_address,
+         size, (void *) local_handle->remote_offset, btl_peer->ptl_proc.phys.nid,
+         btl_peer->ptl_proc.phys.pid, frag->match_bits));
 
-    ret = PtlGet(portals4_btl->send_md_h,
-                 (ptl_size_t) local_address,
-                 size,
-                 btl_peer->ptl_proc,
-                 portals4_btl->recv_idx,
-                 frag->match_bits, /* match bits */
-                 local_handle->remote_offset,
-                 frag);
+    ret = PtlGet(portals4_btl->send_md_h, (ptl_size_t) local_address, size, btl_peer->ptl_proc,
+                 portals4_btl->recv_idx, frag->match_bits, /* match bits */
+                 local_handle->remote_offset, frag);
     if (OPAL_UNLIKELY(PTL_OK != ret)) {
-        opal_output_verbose(1, opal_btl_base_framework.framework_output,
-                            "%s:%d: PtlGet failed: %d",
+        opal_output_verbose(1, opal_btl_base_framework.framework_output, "%s:%d: PtlGet failed: %d",
                             __FILE__, __LINE__, ret);
         return OPAL_ERROR;
     }
diff --git a/opal/mca/btl/portals4/btl_portals4_recv.c b/opal/mca/btl/portals4/btl_portals4_recv.c
index c8dbfa3cc26..1bd9e4279ee 100644
--- a/opal/mca/btl/portals4/btl_portals4_recv.c
+++ b/opal/mca/btl/portals4/btl_portals4_recv.c
@@ -17,50 +17,40 @@
  * $HEADER$
  */
 
-
 #include "opal_config.h"
 
 #include "opal/constants.h"
 
 #include "btl_portals4.h"
-#include "btl_portals4_recv.h"
 #include "btl_portals4_frag.h"
+#include "btl_portals4_recv.h"
 
+OBJ_CLASS_INSTANCE(mca_btl_portals4_recv_block_t, opal_list_item_t, NULL, NULL);
 
-OBJ_CLASS_INSTANCE(mca_btl_portals4_recv_block_t,
-                   opal_list_item_t,
-                   NULL, NULL);
-
-int
-mca_btl_portals4_recv_enable(mca_btl_portals4_module_t *btl)
+int mca_btl_portals4_recv_enable(mca_btl_portals4_module_t *btl)
 {
     int i;
 
     /* create the recv blocks */
-    for (i = 0 ; i < mca_btl_portals4_component.portals_recv_mds_num ; ++i) {
-        mca_btl_portals4_recv_block_t *block =
-            mca_btl_portals4_recv_block_init(btl);
+    for (i = 0; i < mca_btl_portals4_component.portals_recv_mds_num; ++i) {
+        mca_btl_portals4_recv_block_t *block = mca_btl_portals4_recv_block_init(btl);
         if (NULL == block) {
             mca_btl_portals4_recv_disable(btl);
             return OPAL_ERROR;
         }
-        opal_list_append(&(btl->portals_recv_blocks),
-                         (opal_list_item_t*) block);
+        opal_list_append(&(btl->portals_recv_blocks), (opal_list_item_t *) block);
         mca_btl_portals4_activate_block(block);
     }
     return OPAL_SUCCESS;
 }
 
-int
-mca_btl_portals4_recv_disable(mca_btl_portals4_module_t *btl)
+int mca_btl_portals4_recv_disable(mca_btl_portals4_module_t *btl)
 {
     opal_list_item_t *item;
 
     if (opal_list_get_size(&btl->portals_recv_blocks) > 0) {
-        while (NULL !=
-               (item = opal_list_remove_first(&btl->portals_recv_blocks))) {
-            mca_btl_portals4_recv_block_t *block =
-                (mca_btl_portals4_recv_block_t*) item;
+        while (NULL != (item = opal_list_remove_first(&btl->portals_recv_blocks))) {
+            mca_btl_portals4_recv_block_t *block = (mca_btl_portals4_recv_block_t *) item;
             mca_btl_portals4_recv_block_free(block);
         }
     }
@@ -68,8 +58,7 @@ mca_btl_portals4_recv_disable(mca_btl_portals4_module_t *btl)
     return OPAL_SUCCESS;
 }
 
-mca_btl_portals4_recv_block_t*
-mca_btl_portals4_recv_block_init(mca_btl_portals4_module_t *btl)
+mca_btl_portals4_recv_block_t *mca_btl_portals4_recv_block_init(mca_btl_portals4_module_t *btl)
 {
     mca_btl_portals4_recv_block_t *block;
 
@@ -77,7 +66,8 @@ mca_btl_portals4_recv_block_init(mca_btl_portals4_module_t *btl)
     block->btl = btl;
     block->length = mca_btl_portals4_component.portals_recv_mds_size;
     block->start = malloc(block->length);
-    if (block->start == NULL) return NULL;
+    if (block->start == NULL)
+        return NULL;
 
     block->me_h = PTL_INVALID_HANDLE;
 
@@ -87,9 +77,7 @@ mca_btl_portals4_recv_block_init(mca_btl_portals4_module_t *btl)
     return block;
 }
 
-
-int
-mca_btl_portals4_recv_block_free(mca_btl_portals4_recv_block_t *block)
+int mca_btl_portals4_recv_block_free(mca_btl_portals4_recv_block_t *block)
 {
     if (NULL != block->start) {
         free(block->start);
diff --git a/opal/mca/btl/portals4/btl_portals4_recv.h b/opal/mca/btl/portals4/btl_portals4_recv.h
index be7e98ca7cf..c8144b112c5 100644
--- a/opal/mca/btl/portals4/btl_portals4_recv.h
+++ b/opal/mca/btl/portals4/btl_portals4_recv.h
@@ -20,7 +20,9 @@
 #ifndef OPAL_BTL_PORTALS4_RECV_H
 #define OPAL_BTL_PORTALS4_RECV_H
 
-#include "btl_portals4_frag.h"
+#include "opal_config.h"
+
+#include "btl_portals4.h"
 
 struct mca_btl_portals4_recv_block_t {
     opal_list_item_t base;
@@ -37,7 +39,6 @@ struct mca_btl_portals4_recv_block_t {
 typedef struct mca_btl_portals4_recv_block_t mca_btl_portals4_recv_block_t;
 OBJ_CLASS_DECLARATION(mca_btl_portals4_recv_block_t);
 
-
 int mca_btl_portals4_recv_enable(mca_btl_portals4_module_t *btl);
 
 int mca_btl_portals4_recv_disable(mca_btl_portals4_module_t *btl);
@@ -55,16 +56,14 @@ int mca_btl_portals4_recv_block_free(mca_btl_portals4_recv_block_t *block);
  *
  * Module lock must be held before calling this function
  */
-mca_btl_portals4_recv_block_t*
-mca_btl_portals4_recv_block_init(mca_btl_portals4_module_t *btl);
+mca_btl_portals4_recv_block_t *mca_btl_portals4_recv_block_init(mca_btl_portals4_module_t *btl);
 
 /**
  * activate a block.  Blocks that are full (have gone inactive) can be
  * re-activated with this call.  There is no need to hold the lock
  * before calling this function
  */
-static inline int
-mca_btl_portals4_activate_block(mca_btl_portals4_recv_block_t *block)
+static inline int mca_btl_portals4_activate_block(mca_btl_portals4_recv_block_t *block)
 {
     int ret;
     ptl_me_t me;
@@ -72,7 +71,8 @@ mca_btl_portals4_activate_block(mca_btl_portals4_recv_block_t *block)
     ptl_match_bits_t match_bits, ignore_bits;
     mca_btl_portals4_module_t *btl = block->btl;
 
-    if (NULL == block->start) return OPAL_ERROR;
+    if (NULL == block->start)
+        return OPAL_ERROR;
 
     ignore_bits = BTL_PORTALS4_CONTEXT_MASK | BTL_PORTALS4_SOURCE_MASK | BTL_PORTALS4_TAG_MASK;
     match_bits = BTL_PORTALS4_SHORT_MSG;
@@ -82,11 +82,7 @@ mca_btl_portals4_activate_block(mca_btl_portals4_recv_block_t *block)
     me.ct_handle = PTL_CT_NONE;
     me.min_free = btl->super.btl_eager_limit;
     me.uid = PTL_UID_ANY;
-    me.options =
-        PTL_ME_OP_PUT |
-        PTL_ME_MANAGE_LOCAL |
-        PTL_ME_EVENT_LINK_DISABLE  |
-        PTL_ME_MAY_ALIGN;
+    me.options = PTL_ME_OP_PUT | PTL_ME_MANAGE_LOCAL | PTL_ME_EVENT_LINK_DISABLE | PTL_ME_MAY_ALIGN;
 
     if (mca_btl_portals4_component.use_logical) {
         remote_proc.rank = PTL_RANK_ANY;
@@ -103,20 +99,18 @@ mca_btl_portals4_activate_block(mca_btl_portals4_recv_block_t *block)
     block->full = false;
     opal_atomic_mb();
 
-    ret = PtlMEAppend(btl->portals_ni_h,
-                      btl->recv_idx,
-                      &me,
-                      PTL_PRIORITY_LIST,
-                      block,
+    ret = PtlMEAppend(btl->portals_ni_h, btl->recv_idx, &me, PTL_PRIORITY_LIST, block,
                       &block->me_h);
     if (OPAL_UNLIKELY(PTL_OK != ret)) {
         opal_output_verbose(1, opal_btl_base_framework.framework_output,
-                            "%s:%d: PtlMEAppend failed on NI %d: %d",
-                            __FILE__, __LINE__, btl->interface_num, ret);
+                            "%s:%d: PtlMEAppend failed on NI %d: %d", __FILE__, __LINE__,
+                            btl->interface_num, ret);
         return OPAL_ERROR;
     }
-    OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "PtlMEAppend (recv) block=%p me_h=%d start=%p len=%x NI=%d\n",
-        (void *)block, block->me_h, block->start, (unsigned int) block->length, btl->interface_num));
+    OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
+                         "PtlMEAppend (recv) block=%p me_h=%d start=%p len=%x NI=%d\n",
+                         (void *) block, block->me_h, block->start, (unsigned int) block->length,
+                         btl->interface_num));
 
     return OPAL_SUCCESS;
 }
diff --git a/opal/mca/btl/portals4/btl_portals4_send.c b/opal/mca/btl/portals4/btl_portals4_send.c
index 218ed877803..eaca685f18a 100644
--- a/opal/mca/btl/portals4/btl_portals4_send.c
+++ b/opal/mca/btl/portals4/btl_portals4_send.c
@@ -22,18 +22,19 @@
  */
 
 #include "opal_config.h"
+
 #include "opal/constants.h"
 #include "opal/datatype/opal_convertor.h"
 
 #include "btl_portals4.h"
+#include "btl_portals4_frag.h"
 
-int mca_btl_portals4_send(struct mca_btl_base_module_t* btl_base,
-                     struct mca_btl_base_endpoint_t* endpoint,
-                     struct mca_btl_base_descriptor_t* descriptor,
-                     mca_btl_base_tag_t tag)
+int mca_btl_portals4_send(struct mca_btl_base_module_t *btl_base,
+                          struct mca_btl_base_endpoint_t *endpoint,
+                          struct mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag)
 {
-    struct mca_btl_portals4_module_t* portals4_btl = (struct mca_btl_portals4_module_t*) btl_base;
-    mca_btl_portals4_frag_t *frag = (mca_btl_portals4_frag_t*) descriptor;
+    struct mca_btl_portals4_module_t *portals4_btl = (struct mca_btl_portals4_module_t *) btl_base;
+    mca_btl_portals4_frag_t *frag = (mca_btl_portals4_frag_t *) descriptor;
     ptl_match_bits_t match_bits, msglen_type;
     ptl_size_t put_length;
     int ret;
@@ -41,16 +42,17 @@ int mca_btl_portals4_send(struct mca_btl_base_module_t* btl_base,
     frag->endpoint = endpoint;
     frag->hdr.tag = tag;
 
-    put_length       = frag->segments[0].base.seg_len;
+    put_length = frag->segments[0].base.seg_len;
     if (put_length > portals4_btl->super.btl_eager_limit)
-         msglen_type = BTL_PORTALS4_LONG_MSG;
-    else msglen_type = BTL_PORTALS4_SHORT_MSG;
+        msglen_type = BTL_PORTALS4_LONG_MSG;
+    else
+        msglen_type = BTL_PORTALS4_SHORT_MSG;
 
     BTL_PORTALS4_SET_SEND_BITS(match_bits, 0, 0, tag, msglen_type);
 
     /* reserve space in the event queue for rdma operations immediately */
-    while (OPAL_THREAD_ADD_FETCH32(&portals4_btl->portals_outstanding_ops, 1) >
-           portals4_btl->portals_max_outstanding_ops) {
+    while (OPAL_THREAD_ADD_FETCH32(&portals4_btl->portals_outstanding_ops, 1)
+           > portals4_btl->portals_max_outstanding_ops) {
         OPAL_THREAD_ADD_FETCH32(&portals4_btl->portals_outstanding_ops, -1);
         OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
                              "Call to mca_btl_portals4_component_progress (4)\n"));
@@ -58,48 +60,43 @@ int mca_btl_portals4_send(struct mca_btl_base_module_t* btl_base,
     }
     OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
                          "mca_btl_portals4_send: Incrementing portals_outstanding_ops=%d\n",
-        portals4_btl->portals_outstanding_ops));
+                         portals4_btl->portals_outstanding_ops));
 
     OPAL_OUTPUT_VERBOSE((50, opal_btl_base_framework.framework_output,
                          "PtlPut frag=%p rank=%x pid=%x tag=%x len=%ld match_bits=%lx\n",
-                         (void*)frag,  endpoint->ptl_proc.rank, endpoint->ptl_proc.phys.pid, tag,
-                         put_length, (uint64_t)match_bits));
+                         (void *) frag, endpoint->ptl_proc.rank, endpoint->ptl_proc.phys.pid, tag,
+                         put_length, (uint64_t) match_bits));
 
-    ret = PtlPut(portals4_btl->send_md_h,
-                 (ptl_size_t) frag->segments[0].base.seg_addr.pval,
+    ret = PtlPut(portals4_btl->send_md_h, (ptl_size_t) frag->segments[0].base.seg_addr.pval,
                  put_length, /* fragment length */
                  (mca_btl_portals4_component.portals_need_ack ? PTL_ACK_REQ : PTL_NO_ACK_REQ),
-                 endpoint->ptl_proc,
-                 portals4_btl->recv_idx,
-                 match_bits,                     /* match bits */
-                 0,                              /* remote offset - not used */
-                 (void *) frag,                  /* user ptr */
-                 tag);                           /* hdr_data: tag */
+                 endpoint->ptl_proc, portals4_btl->recv_idx, match_bits, /* match bits */
+                 0,             /* remote offset - not used */
+                 (void *) frag, /* user ptr */
+                 tag);          /* hdr_data: tag */
     if (ret != PTL_OK) {
-        opal_output(opal_btl_base_framework.framework_output, "mca_btl_portals4_send: PtlPut failed with error %d", ret);
+        opal_output(opal_btl_base_framework.framework_output,
+                    "mca_btl_portals4_send: PtlPut failed with error %d", ret);
         return OPAL_ERROR;
     }
     OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
-            "PtlPut frag=%p rank=%x pid=%x tag=%x addr=%p len=%ld match_bits=%lx",
-            (void*)frag,  endpoint->ptl_proc.rank, endpoint->ptl_proc.phys.pid, tag,
-            (void *)frag->segments[0].base.seg_addr.pval, put_length, (uint64_t)match_bits));
+                         "PtlPut frag=%p rank=%x pid=%x tag=%x addr=%p len=%ld match_bits=%lx",
+                         (void *) frag, endpoint->ptl_proc.rank, endpoint->ptl_proc.phys.pid, tag,
+                         (void *) frag->segments[0].base.seg_addr.pval, put_length,
+                         (uint64_t) match_bits));
 
     return OPAL_SUCCESS;
 }
 
 /* NOT IMPLEMENTED */
-int mca_btl_portals4_sendi(struct mca_btl_base_module_t* btl_base,
-                          struct mca_btl_base_endpoint_t* endpoint,
-                          struct opal_convertor_t* convertor,
-                          void* header,
-                          size_t header_size,
-                          size_t payload_size,
-                          uint8_t order,
-                          uint32_t flags,
-                          mca_btl_base_tag_t tag,
-                          mca_btl_base_descriptor_t** des)
+int mca_btl_portals4_sendi(struct mca_btl_base_module_t *btl_base,
+                           struct mca_btl_base_endpoint_t *endpoint,
+                           struct opal_convertor_t *convertor, void *header, size_t header_size,
+                           size_t payload_size, uint8_t order, uint32_t flags,
+                           mca_btl_base_tag_t tag, mca_btl_base_descriptor_t **des)
 {
-    opal_output(opal_btl_base_framework.framework_output, "mca_btl_portals_sendi is not implemented");
+    opal_output(opal_btl_base_framework.framework_output,
+                "mca_btl_portals_sendi is not implemented");
     BTL_ERROR(("mca_btl_portals_sendi is not implemented"));
     return OPAL_SUCCESS;
 }
diff --git a/opal/mca/btl/portals4/configure.m4 b/opal/mca/btl/portals4/configure.m4
index 4d526f91c14..7f5fcf27402 100644
--- a/opal/mca/btl/portals4/configure.m4
+++ b/opal/mca/btl/portals4/configure.m4
@@ -36,7 +36,7 @@ AC_DEFUN([MCA_opal_btl_portals4_CONFIG],[
            $1],
           [$2])
 
-    # need to propogate CPPFLAGS to all of OPAL
+    # need to propagate CPPFLAGS to all of OPAL
     AS_IF([test "$DIRECT_btl" = "portals4"],
           [CPPFLAGS="$CPPFLAGS $btl_portals4_CPPFLAGS"])
 
diff --git a/opal/mca/btl/self/Makefile.am b/opal/mca/btl/self/Makefile.am
index 3bafcb2e6ee..9e3097a6322 100644
--- a/opal/mca/btl/self/Makefile.am
+++ b/opal/mca/btl/self/Makefile.am
@@ -41,7 +41,7 @@ mcacomponentdir = $(opallibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_btl_self_la_SOURCES = $(libmca_btl_self_la_sources)
 mca_btl_self_la_LDFLAGS = -module -avoid-version
-mca_btl_self_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+mca_btl_self_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 
 noinst_LTLIBRARIES = $(component_noinst)
 libmca_btl_self_la_SOURCES = $(libmca_btl_self_la_sources)
diff --git a/opal/mca/btl/self/btl_self.c b/opal/mca/btl/self/btl_self.c
index 1b247a78480..1790d576780 100644
--- a/opal/mca/btl/self/btl_self.c
+++ b/opal/mca/btl/self/btl_self.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2013 The University of Tennessee and The University
+ * Copyright (c) 2004-2022 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -25,13 +25,13 @@
 
 #include "opal_config.h"
 
-#include <string.h>
 #include <stdlib.h>
+#include <string.h>
 
-#include "opal/class/opal_bitmap.h"
-#include "opal/datatype/opal_convertor.h"
 #include "btl_self.h"
 #include "btl_self_frag.h"
+#include "opal/class/opal_bitmap.h"
+#include "opal/datatype/opal_convertor.h"
 #include "opal/util/proc.h"
 
 /**
@@ -47,17 +47,17 @@
  * @return     OPAL_SUCCESS or error status on failure.
  *
  */
-static int mca_btl_self_add_procs (struct mca_btl_base_module_t *btl, size_t nprocs,
-                                   struct opal_proc_t **procs,
-                                   struct mca_btl_base_endpoint_t **peers,
-                                   opal_bitmap_t* reachability)
+static int mca_btl_self_add_procs(struct mca_btl_base_module_t *btl, size_t nprocs,
+                                  struct opal_proc_t **procs,
+                                  struct mca_btl_base_endpoint_t **peers,
+                                  opal_bitmap_t *reachability)
 {
-    for (int i = 0; i < (int)nprocs; i++ ) {
-        if( 0 == opal_compare_proc(procs[i]->proc_name, OPAL_PROC_MY_NAME) ) {
-            opal_bitmap_set_bit( reachability, i );
+    for (int i = 0; i < (int) nprocs; i++) {
+        if (0 == opal_compare_proc(procs[i]->proc_name, OPAL_PROC_MY_NAME)) {
+            opal_bitmap_set_bit(reachability, i);
             /* need to return something to keep the bml from ignoring us */
             peers[i] = (struct mca_btl_base_endpoint_t *) 1;
-            break;  /* there will always be only one ... */
+            break; /* there will always be only one ... */
         }
     }
 
@@ -73,14 +73,13 @@ static int mca_btl_self_add_procs (struct mca_btl_base_module_t *btl, size_t npr
  * @return             Status indicating if cleanup was successful
  *
  */
-static int mca_btl_self_del_procs (struct mca_btl_base_module_t *btl, size_t nprocs,
-                                   struct opal_proc_t **procs,
-                                   struct mca_btl_base_endpoint_t **peers)
+static int mca_btl_self_del_procs(struct mca_btl_base_module_t *btl, size_t nprocs,
+                                  struct opal_proc_t **procs,
+                                  struct mca_btl_base_endpoint_t **peers)
 {
     return OPAL_SUCCESS;
 }
 
-
 /**
  * MCA->BTL Clean up any resources held by BTL module
  * before the module is unloaded.
@@ -94,21 +93,20 @@ static int mca_btl_self_del_procs (struct mca_btl_base_module_t *btl, size_t npr
  *
  */
 
-static int mca_btl_self_finalize(struct mca_btl_base_module_t* btl)
+static int mca_btl_self_finalize(struct mca_btl_base_module_t *btl)
 {
     return OPAL_SUCCESS;
 }
 
-
 /**
  * Allocate a segment.
  *
  * @param btl (IN)      BTL module
  * @param size (IN)     Request segment size.
  */
-static mca_btl_base_descriptor_t *mca_btl_self_alloc (struct mca_btl_base_module_t *btl,
-                                                      struct mca_btl_base_endpoint_t *endpoint,
-                                                      uint8_t order, size_t size, uint32_t flags)
+static mca_btl_base_descriptor_t *mca_btl_self_alloc(struct mca_btl_base_module_t *btl,
+                                                     struct mca_btl_base_endpoint_t *endpoint,
+                                                     uint8_t order, size_t size, uint32_t flags)
 {
     mca_btl_self_frag_t *frag = NULL;
 
@@ -120,13 +118,13 @@ static mca_btl_base_descriptor_t *mca_btl_self_alloc (struct mca_btl_base_module
         MCA_BTL_SELF_FRAG_ALLOC_SEND(frag);
     }
 
-    if( OPAL_UNLIKELY(NULL == frag) ) {
+    if (OPAL_UNLIKELY(NULL == frag)) {
         return NULL;
     }
 
     frag->segments[0].seg_len = size;
     frag->base.des_segment_count = 1;
-    frag->base.des_flags       = flags;
+    frag->base.des_flags = flags;
 
     return &frag->base;
 }
@@ -137,44 +135,42 @@ static mca_btl_base_descriptor_t *mca_btl_self_alloc (struct mca_btl_base_module
  * @param btl (IN)      BTL module
  * @param segment (IN)  Allocated segment.
  */
-static int mca_btl_self_free (struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des)
+static int mca_btl_self_free(struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des)
 {
     MCA_BTL_SELF_FRAG_RETURN((mca_btl_self_frag_t *) des);
 
     return OPAL_SUCCESS;
 }
 
-
 /**
  * Prepare data for send
  *
  * @param btl (IN)      BTL module
  */
-static struct mca_btl_base_descriptor_t *mca_btl_self_prepare_src (struct mca_btl_base_module_t* btl,
-                                                                   struct mca_btl_base_endpoint_t *endpoint,
-                                                                   struct opal_convertor_t *convertor,
-                                                                   uint8_t order, size_t reserve,
-                                                                   size_t *size, uint32_t flags)
+static struct mca_btl_base_descriptor_t *mca_btl_self_prepare_src(
+    struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+    struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size, uint32_t flags)
 {
-    bool inline_send = !opal_convertor_need_buffers(convertor);
+    bool inline_send = !(opal_convertor_need_buffers(convertor) || opal_convertor_on_device(convertor));
     size_t buffer_len = reserve + (inline_send ? 0 : *size);
     mca_btl_self_frag_t *frag;
 
-    frag = (mca_btl_self_frag_t *) mca_btl_self_alloc (btl, endpoint, order, buffer_len, flags);
+    frag = (mca_btl_self_frag_t *) mca_btl_self_alloc(btl, endpoint, order, buffer_len, flags);
     if (OPAL_UNLIKELY(NULL == frag)) {
         return NULL;
     }
 
-    /* non-contigous data */
+    /* non-contiguous data */
     if (OPAL_UNLIKELY(!inline_send)) {
-        struct iovec iov = {.iov_len = *size, .iov_base = (IOVBASE_TYPE *) ((uintptr_t) frag->data + reserve)};
+        struct iovec iov = {.iov_len = *size,
+                            .iov_base = (IOVBASE_TYPE *) ((uintptr_t) frag->data + reserve)};
         size_t max_data = *size;
         uint32_t iov_count = 1;
         int rc;
 
-        rc = opal_convertor_pack (convertor, &iov, &iov_count, &max_data);
-        if(rc < 0) {
-            mca_btl_self_free (btl, &frag->base);
+        rc = opal_convertor_pack(convertor, &iov, &iov_count, &max_data);
+        if (rc < 0) {
+            mca_btl_self_free(btl, &frag->base);
             return NULL;
         }
 
@@ -183,7 +179,7 @@ static struct mca_btl_base_descriptor_t *mca_btl_self_prepare_src (struct mca_bt
     } else {
         void *data_ptr;
 
-        opal_convertor_get_current_pointer (convertor, &data_ptr);
+        opal_convertor_get_current_pointer(convertor, &data_ptr);
 
         frag->segments[1].seg_addr.pval = data_ptr;
         frag->segments[1].seg_len = *size;
@@ -200,10 +196,9 @@ static struct mca_btl_base_descriptor_t *mca_btl_self_prepare_src (struct mca_bt
  * @param peer (IN)     BTL peer addressing
  */
 
-static int mca_btl_self_send (struct mca_btl_base_module_t *btl,
-                              struct mca_btl_base_endpoint_t *endpoint,
-                              struct mca_btl_base_descriptor_t *des,
-                              mca_btl_base_tag_t tag)
+static int mca_btl_self_send(struct mca_btl_base_module_t *btl,
+                             struct mca_btl_base_endpoint_t *endpoint,
+                             struct mca_btl_base_descriptor_t *des, mca_btl_base_tag_t tag)
 {
     mca_btl_active_message_callback_t *reg = mca_btl_base_active_message_trigger + tag;
     mca_btl_base_receive_descriptor_t recv_desc = {.endpoint = endpoint,
@@ -214,88 +209,96 @@ static int mca_btl_self_send (struct mca_btl_base_module_t *btl,
     int btl_ownership = (des->des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
 
     /* upcall */
-    reg->cbfunc (btl, &recv_desc);
+    reg->cbfunc(btl, &recv_desc);
 
     /* send completion */
-    if( des->des_flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK ) {
-        des->des_cbfunc( btl, endpoint, des, OPAL_SUCCESS );
+    if (des->des_flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK) {
+        des->des_cbfunc(btl, endpoint, des, OPAL_SUCCESS);
     }
-    if( btl_ownership ) {
-        mca_btl_self_free( btl, des );
+    if (btl_ownership) {
+        mca_btl_self_free(btl, des);
     }
     return 1;
 }
 
-static int mca_btl_self_sendi (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
-                               struct opal_convertor_t *convertor, void *header, size_t header_size,
-                               size_t payload_size, uint8_t order, uint32_t flags, mca_btl_base_tag_t tag,
-                               mca_btl_base_descriptor_t **descriptor)
+static int mca_btl_self_sendi(struct mca_btl_base_module_t *btl,
+                              struct mca_btl_base_endpoint_t *endpoint,
+                              struct opal_convertor_t *convertor, void *header, size_t header_size,
+                              size_t payload_size, uint8_t order, uint32_t flags,
+                              mca_btl_base_tag_t tag, mca_btl_base_descriptor_t **descriptor)
 {
     mca_btl_base_descriptor_t *frag;
 
-    if (!payload_size || !opal_convertor_need_buffers(convertor)) {
+    if (!payload_size ||
+        !(opal_convertor_need_buffers(convertor) ||
+          opal_convertor_on_device(convertor))) {
         void *data_ptr = NULL;
         if (payload_size) {
-            opal_convertor_get_current_pointer (convertor, &data_ptr);
+            opal_convertor_get_current_pointer(convertor, &data_ptr);
         }
 
         mca_btl_base_segment_t segments[2] = {{.seg_addr.pval = header, .seg_len = header_size},
                                               {.seg_addr.pval = data_ptr, .seg_len = payload_size}};
-        mca_btl_base_descriptor_t des = {.des_segments = segments, .des_segment_count = payload_size ? 2 : 1,
+        mca_btl_base_descriptor_t des = {.des_segments = segments,
+                                         .des_segment_count = payload_size ? 2 : 1,
                                          .des_flags = 0};
 
-        (void) mca_btl_self_send (btl, endpoint, &des, tag);
+        (void) mca_btl_self_send(btl, endpoint, &des, tag);
         return OPAL_SUCCESS;
     }
 
-    frag = mca_btl_self_prepare_src (btl, endpoint, convertor, order, header_size, &payload_size,
-                                     flags | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
+    frag = mca_btl_self_prepare_src(btl, endpoint, convertor, order, header_size, &payload_size,
+                                    flags | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
     if (NULL == frag) {
-        *descriptor = NULL;
+        if( NULL != descriptor ) {
+            *descriptor = NULL;
+        }
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
-    memcpy (frag->des_segments[0].seg_addr.pval, header, header_size);
-    (void) mca_btl_self_send (btl, endpoint, frag, tag);
+    memcpy(frag->des_segments[0].seg_addr.pval, header, header_size);
+    (void) mca_btl_self_send(btl, endpoint, frag, tag);
     return OPAL_SUCCESS;
 }
 
-static int mca_btl_self_put (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, void *local_address,
-                             uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                             mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-                             int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
+static int mca_btl_self_put(mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+                            void *local_address, uint64_t remote_address,
+                            mca_btl_base_registration_handle_t *local_handle,
+                            mca_btl_base_registration_handle_t *remote_handle, size_t size,
+                            int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
+                            void *cbcontext, void *cbdata)
 {
-    memcpy ((void *)(intptr_t) remote_address, local_address, size);
+    memcpy((void *) (intptr_t) remote_address, local_address, size);
 
-    cbfunc (btl, endpoint, local_address, NULL, cbcontext, cbdata, OPAL_SUCCESS);
+    cbfunc(btl, endpoint, local_address, NULL, cbcontext, cbdata, OPAL_SUCCESS);
 
     return OPAL_SUCCESS;
 }
 
-static int mca_btl_self_get (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, void *local_address,
-                             uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                             mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-                             int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
+static int mca_btl_self_get(mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+                            void *local_address, uint64_t remote_address,
+                            mca_btl_base_registration_handle_t *local_handle,
+                            mca_btl_base_registration_handle_t *remote_handle, size_t size,
+                            int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
+                            void *cbcontext, void *cbdata)
 {
-    memcpy (local_address, (void *)(intptr_t) remote_address, size);
+    memcpy(local_address, (void *) (intptr_t) remote_address, size);
 
-    cbfunc (btl, endpoint, local_address, NULL, cbcontext, cbdata, OPAL_SUCCESS);
+    cbfunc(btl, endpoint, local_address, NULL, cbcontext, cbdata, OPAL_SUCCESS);
 
     return OPAL_SUCCESS;
 }
 
 /* btl self module */
-mca_btl_base_module_t mca_btl_self = {
-    .btl_component = &mca_btl_self_component.super,
-    .btl_add_procs = mca_btl_self_add_procs,
-    .btl_del_procs = mca_btl_self_del_procs,
-    .btl_finalize = mca_btl_self_finalize,
-    .btl_alloc = mca_btl_self_alloc,
-    .btl_free = mca_btl_self_free,
-    .btl_prepare_src = mca_btl_self_prepare_src,
-    .btl_send = mca_btl_self_send,
-    .btl_sendi = mca_btl_self_sendi,
-    .btl_put = mca_btl_self_put,
-    .btl_get = mca_btl_self_get,
-    .btl_dump = mca_btl_base_dump
-};
+mca_btl_base_module_t mca_btl_self = {.btl_component = &mca_btl_self_component.super,
+                                      .btl_add_procs = mca_btl_self_add_procs,
+                                      .btl_del_procs = mca_btl_self_del_procs,
+                                      .btl_finalize = mca_btl_self_finalize,
+                                      .btl_alloc = mca_btl_self_alloc,
+                                      .btl_free = mca_btl_self_free,
+                                      .btl_prepare_src = mca_btl_self_prepare_src,
+                                      .btl_send = mca_btl_self_send,
+                                      .btl_sendi = mca_btl_self_sendi,
+                                      .btl_put = mca_btl_self_put,
+                                      .btl_get = mca_btl_self_get,
+                                      .btl_dump = mca_btl_base_dump};
diff --git a/opal/mca/btl/self/btl_self.h b/opal/mca/btl/self/btl_self.h
index ac4cab6eb6a..2bf87aa6793 100644
--- a/opal/mca/btl/self/btl_self.h
+++ b/opal/mca/btl/self/btl_self.h
@@ -28,11 +28,11 @@
 
 #include <stdlib.h>
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif  /* HAVE_SYS_TYPES_H */
+#    include <sys/types.h>
+#endif /* HAVE_SYS_TYPES_H */
 
-#include "opal/mca/btl/btl.h"
 #include "opal/mca/btl/base/base.h"
+#include "opal/mca/btl/btl.h"
 
 BEGIN_C_DECLS
 
@@ -42,20 +42,19 @@ BEGIN_C_DECLS
  * Shared Memory (SELF) BTL module.
  */
 struct mca_btl_self_component_t {
-    mca_btl_base_component_3_0_0_t super;  /**< base BTL component */
-    int free_list_num;                     /**< initial size of free lists */
-    int free_list_max;                     /**< maximum size of free lists */
-    int free_list_inc;                     /**< number of elements to alloc when growing free lists */
-    opal_free_list_t self_frags_eager;     /**< free list of self first */
-    opal_free_list_t self_frags_send;      /**< free list of self second */
-    opal_free_list_t self_frags_rdma;      /**< free list of self second */
+    mca_btl_base_component_3_0_0_t super; /**< base BTL component */
+    int free_list_num;                    /**< initial size of free lists */
+    int free_list_max;                    /**< maximum size of free lists */
+    int free_list_inc;                 /**< number of elements to alloc when growing free lists */
+    opal_free_list_t self_frags_eager; /**< free list of self first */
+    opal_free_list_t self_frags_send;  /**< free list of self second */
+    opal_free_list_t self_frags_rdma;  /**< free list of self second */
 };
 typedef struct mca_btl_self_component_t mca_btl_self_component_t;
-OPAL_MODULE_DECLSPEC extern mca_btl_self_component_t mca_btl_self_component;
+OPAL_DECLSPEC extern mca_btl_self_component_t mca_btl_self_component;
 
 extern mca_btl_base_module_t mca_btl_self;
 
 END_C_DECLS
 
 #endif
-
diff --git a/opal/mca/btl/self/btl_self_component.c b/opal/mca/btl/self/btl_self_component.c
index 271d28215e6..875fd066bb7 100644
--- a/opal/mca/btl/self/btl_self_component.c
+++ b/opal/mca/btl/self/btl_self_component.c
@@ -32,35 +32,39 @@ static int mca_btl_self_component_close(void);
  * SELF module initialization.
  *
  * @param num_btls (OUT)                  Number of BTLs returned in BTL array.
- * @param enable_progress_threads (IN)    Flag indicating whether BTL is allowed to have progress threads
- * @param enable_mpi_threads (IN)         Flag indicating whether BTL must support multilple simultaneous invocations from different threads
+ * @param enable_progress_threads (IN)    Flag indicating whether BTL is allowed to have progress
+ * threads
+ * @param enable_mpi_threads (IN)         Flag indicating whether BTL must support multilple
+ * simultaneous invocations from different threads
  *
  */
-static mca_btl_base_module_t **mca_btl_self_component_init (int *num_btls,
-                                                            bool enable_progress_threads,
-                                                            bool enable_mpi_threads);
+static mca_btl_base_module_t **
+mca_btl_self_component_init(int *num_btls, bool enable_progress_threads, bool enable_mpi_threads);
 
 /*
  * Shared Memory (SELF) component instance.
  */
 
 mca_btl_self_component_t mca_btl_self_component = {
-    .super = {
-        /* First, the mca_base_component_t struct containing meta information
-          about the component itself */
-        .btl_version = {
-            MCA_BTL_DEFAULT_VERSION("self"),
-            .mca_open_component = mca_btl_self_component_open,
-            .mca_close_component = mca_btl_self_component_close,
-            .mca_register_component_params = mca_btl_self_component_register,
-        },
-        .btl_data = {
-            /* The component is checkpoint ready */
-            .param_field = MCA_BASE_METADATA_PARAM_CHECKPOINT,
-        },
-
-        .btl_init = mca_btl_self_component_init,
-    }  /* end super */
+    .super =
+        {
+            /* First, the mca_base_component_t struct containing meta information
+              about the component itself */
+            .btl_version =
+                {
+                    MCA_BTL_DEFAULT_VERSION("self"),
+                    .mca_open_component = mca_btl_self_component_open,
+                    .mca_close_component = mca_btl_self_component_close,
+                    .mca_register_component_params = mca_btl_self_component_register,
+                },
+            .btl_data =
+                {
+                    /* The component is checkpoint ready */
+                    .param_field = MCA_BASE_METADATA_PARAM_CHECKPOINT,
+                },
+
+            .btl_init = mca_btl_self_component_init,
+        } /* end super */
 };
 
 /*
@@ -75,25 +79,23 @@ static int mca_btl_self_component_register(void)
 
     /* register SELF component parameters */
     mca_btl_self_component.free_list_num = 0;
-    (void) mca_base_component_var_register(&mca_btl_self_component.super.btl_version, "free_list_num",
-                                           "Number of fragments by default",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
+    (void) mca_base_component_var_register(&mca_btl_self_component.super.btl_version,
+                                           "free_list_num", "Number of fragments by default",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9,
                                            MCA_BASE_VAR_SCOPE_READONLY,
                                            &mca_btl_self_component.free_list_num);
-    /* NTH: free list buffers are not released until we tear down so DO NOT make them unlimited here */
+    /* NTH: free list buffers are not released until we tear down so DO NOT make them unlimited here
+     */
     mca_btl_self_component.free_list_max = 64;
-    (void) mca_base_component_var_register(&mca_btl_self_component.super.btl_version, "free_list_max",
-                                           "Maximum number of fragments",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
+    (void) mca_base_component_var_register(&mca_btl_self_component.super.btl_version,
+                                           "free_list_max", "Maximum number of fragments",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9,
                                            MCA_BASE_VAR_SCOPE_READONLY,
                                            &mca_btl_self_component.free_list_max);
     mca_btl_self_component.free_list_inc = 8;
-    (void) mca_base_component_var_register(&mca_btl_self_component.super.btl_version, "free_list_inc",
-                                           "Increment by this number of fragments",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
+    (void) mca_base_component_var_register(&mca_btl_self_component.super.btl_version,
+                                           "free_list_inc", "Increment by this number of fragments",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9,
                                            MCA_BASE_VAR_SCOPE_READONLY,
                                            &mca_btl_self_component.free_list_inc);
 
@@ -105,9 +107,11 @@ static int mca_btl_self_component_register(void)
     mca_btl_self.btl_rdma_pipeline_frag_size = INT_MAX;
     mca_btl_self.btl_min_rdma_pipeline_size = 0;
     mca_btl_self.btl_flags = MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_SEND;
+    /* for self, remote completion is local completion */
+    mca_btl_self.btl_flags |= MCA_BTL_FLAGS_RDMA_REMOTE_COMPLETION;
     mca_btl_self.btl_bandwidth = 100;
     mca_btl_self.btl_latency = 0;
-    mca_btl_base_param_register (&mca_btl_self_component.super.btl_version, &mca_btl_self);
+    mca_btl_base_param_register(&mca_btl_self_component.super.btl_version, &mca_btl_self);
 
     return OPAL_SUCCESS;
 }
@@ -122,7 +126,6 @@ static int mca_btl_self_component_open(void)
     return OPAL_SUCCESS;
 }
 
-
 /*
  * component cleanup - sanity checking of queue lengths
  */
@@ -138,49 +141,45 @@ static int mca_btl_self_component_close(void)
 /*
  *  SELF component initialization
  */
-static mca_btl_base_module_t **mca_btl_self_component_init (int *num_btls,
-                                                            bool enable_progress_threads,
-                                                            bool enable_mpi_threads)
+static mca_btl_base_module_t **
+mca_btl_self_component_init(int *num_btls, bool enable_progress_threads, bool enable_mpi_threads)
 {
     mca_btl_base_module_t **btls = NULL;
     int ret;
 
     /* initialize free lists */
-    ret = opal_free_list_init (&mca_btl_self_component.self_frags_eager,
-                               sizeof (mca_btl_self_frag_eager_t) + mca_btl_self.btl_eager_limit,
-                               opal_cache_line_size, OBJ_CLASS(mca_btl_self_frag_eager_t), 0,
-                               opal_cache_line_size, mca_btl_self_component.free_list_num,
-                               mca_btl_self_component.free_list_max,
-                               mca_btl_self_component.free_list_inc,
-                               NULL, 0, NULL, NULL, NULL);
+    ret = opal_free_list_init(&mca_btl_self_component.self_frags_eager,
+                              sizeof(mca_btl_self_frag_eager_t) + mca_btl_self.btl_eager_limit,
+                              opal_cache_line_size, OBJ_CLASS(mca_btl_self_frag_eager_t), 0,
+                              opal_cache_line_size, mca_btl_self_component.free_list_num,
+                              mca_btl_self_component.free_list_max,
+                              mca_btl_self_component.free_list_inc, NULL, 0, NULL, NULL, NULL);
     if (OPAL_SUCCESS != ret) {
         return NULL;
     }
 
-    ret = opal_free_list_init (&mca_btl_self_component.self_frags_send,
-                               sizeof (mca_btl_self_frag_send_t) + mca_btl_self.btl_max_send_size,
-                               opal_cache_line_size, OBJ_CLASS(mca_btl_self_frag_send_t), 0,
-                               opal_cache_line_size, mca_btl_self_component.free_list_num,
-                               mca_btl_self_component.free_list_max,
-                               mca_btl_self_component.free_list_inc,
-                               NULL, 0, NULL, NULL, NULL);
+    ret = opal_free_list_init(&mca_btl_self_component.self_frags_send,
+                              sizeof(mca_btl_self_frag_send_t) + mca_btl_self.btl_max_send_size,
+                              opal_cache_line_size, OBJ_CLASS(mca_btl_self_frag_send_t), 0,
+                              opal_cache_line_size, mca_btl_self_component.free_list_num,
+                              mca_btl_self_component.free_list_max,
+                              mca_btl_self_component.free_list_inc, NULL, 0, NULL, NULL, NULL);
     if (OPAL_SUCCESS != ret) {
         return NULL;
     }
 
-    ret = opal_free_list_init (&mca_btl_self_component.self_frags_rdma,
-                               sizeof (mca_btl_self_frag_rdma_t) + MCA_BTL_SELF_MAX_INLINE_SIZE,
-                               opal_cache_line_size, OBJ_CLASS(mca_btl_self_frag_rdma_t), 0,
-                               opal_cache_line_size, mca_btl_self_component.free_list_num,
-                               mca_btl_self_component.free_list_max,
-                               mca_btl_self_component.free_list_inc,
-                               NULL, 0, NULL, NULL, NULL);
+    ret = opal_free_list_init(&mca_btl_self_component.self_frags_rdma,
+                              sizeof(mca_btl_self_frag_rdma_t) + MCA_BTL_SELF_MAX_INLINE_SIZE,
+                              opal_cache_line_size, OBJ_CLASS(mca_btl_self_frag_rdma_t), 0,
+                              opal_cache_line_size, mca_btl_self_component.free_list_num,
+                              mca_btl_self_component.free_list_max,
+                              mca_btl_self_component.free_list_inc, NULL, 0, NULL, NULL, NULL);
     if (OPAL_SUCCESS != ret) {
         return NULL;
     }
 
     /* get pointer to the btls */
-    btls = (mca_btl_base_module_t **) malloc (sizeof (mca_btl_base_module_t *));
+    btls = (mca_btl_base_module_t **) malloc(sizeof(mca_btl_base_module_t *));
     if (NULL == btls) {
         return NULL;
     }
@@ -190,4 +189,3 @@ static mca_btl_base_module_t **mca_btl_self_component_init (int *num_btls,
 
     return btls;
 }
-
diff --git a/opal/mca/btl/self/btl_self_frag.c b/opal/mca/btl/self/btl_self_frag.c
index eff0fbc4bec..00f2e62528a 100644
--- a/opal/mca/btl/self/btl_self_frag.c
+++ b/opal/mca/btl/self/btl_self_frag.c
@@ -22,7 +22,7 @@
 #include "opal_config.h"
 #include "btl_self_frag.h"
 
-static inline void mca_btl_self_frag_constructor(mca_btl_self_frag_t* frag)
+static inline void mca_btl_self_frag_constructor(mca_btl_self_frag_t *frag)
 {
     frag->base.des_flags = 0;
     frag->segments[0].seg_addr.pval = (void *) frag->data;
@@ -31,38 +31,32 @@ static inline void mca_btl_self_frag_constructor(mca_btl_self_frag_t* frag)
     frag->base.des_segment_count = 1;
 }
 
-static void mca_btl_self_frag_eager_constructor(mca_btl_self_frag_t* frag)
+static void mca_btl_self_frag_eager_constructor(mca_btl_self_frag_t *frag)
 {
     frag->list = &mca_btl_self_component.self_frags_eager;
     frag->size = mca_btl_self.btl_eager_limit;
     mca_btl_self_frag_constructor(frag);
 }
 
-static void mca_btl_self_frag_send_constructor(mca_btl_self_frag_t* frag)
+static void mca_btl_self_frag_send_constructor(mca_btl_self_frag_t *frag)
 {
     frag->list = &mca_btl_self_component.self_frags_send;
     frag->size = mca_btl_self.btl_max_send_size;
     mca_btl_self_frag_constructor(frag);
 }
 
-static void mca_btl_self_frag_rdma_constructor(mca_btl_self_frag_t* frag)
+static void mca_btl_self_frag_rdma_constructor(mca_btl_self_frag_t *frag)
 {
     frag->list = &mca_btl_self_component.self_frags_rdma;
     frag->size = MCA_BTL_SELF_MAX_INLINE_SIZE;
     mca_btl_self_frag_constructor(frag);
 }
 
-OBJ_CLASS_INSTANCE( mca_btl_self_frag_eager_t,
-                    mca_btl_base_descriptor_t,
-                    mca_btl_self_frag_eager_constructor,
-                    NULL );
+OBJ_CLASS_INSTANCE(mca_btl_self_frag_eager_t, mca_btl_base_descriptor_t,
+                   mca_btl_self_frag_eager_constructor, NULL);
 
-OBJ_CLASS_INSTANCE( mca_btl_self_frag_send_t,
-                    mca_btl_base_descriptor_t,
-                    mca_btl_self_frag_send_constructor,
-                    NULL );
+OBJ_CLASS_INSTANCE(mca_btl_self_frag_send_t, mca_btl_base_descriptor_t,
+                   mca_btl_self_frag_send_constructor, NULL);
 
-OBJ_CLASS_INSTANCE( mca_btl_self_frag_rdma_t,
-                    mca_btl_base_descriptor_t,
-                    mca_btl_self_frag_rdma_constructor,
-                    NULL );
+OBJ_CLASS_INSTANCE(mca_btl_self_frag_rdma_t, mca_btl_base_descriptor_t,
+                   mca_btl_self_frag_rdma_constructor, NULL);
diff --git a/opal/mca/btl/self/btl_self_frag.h b/opal/mca/btl/self/btl_self_frag.h
index 38236f1d7f5..9f59d6ed2b8 100644
--- a/opal/mca/btl/self/btl_self_frag.h
+++ b/opal/mca/btl/self/btl_self_frag.h
@@ -24,9 +24,8 @@
 #ifndef MCA_BTL_SELF_SEND_FRAG_H
 #define MCA_BTL_SELF_SEND_FRAG_H
 
-#include <sys/types.h>
 #include "btl_self.h"
-
+#include <sys/types.h>
 
 /**
  * shared memory send fragment derived type.
@@ -48,28 +47,27 @@ OBJ_CLASS_DECLARATION(mca_btl_self_frag_eager_t);
 OBJ_CLASS_DECLARATION(mca_btl_self_frag_send_t);
 OBJ_CLASS_DECLARATION(mca_btl_self_frag_rdma_t);
 
-#define MCA_BTL_SELF_FRAG_ALLOC_EAGER(frag)                             \
-    {                                                                   \
-        frag = (mca_btl_self_frag_t *)                                  \
-            opal_free_list_get (&mca_btl_self_component.self_frags_eager); \
+#define MCA_BTL_SELF_FRAG_ALLOC_EAGER(frag)                \
+    {                                                      \
+        frag = (mca_btl_self_frag_t *) opal_free_list_get( \
+            &mca_btl_self_component.self_frags_eager);     \
     }
 
-
-#define MCA_BTL_SELF_FRAG_ALLOC_RDMA(frag)                              \
-    {                                                                   \
-        frag = (mca_btl_self_frag_t *)                                  \
-            opal_free_list_get (&mca_btl_self_component.self_frags_rdma); \
+#define MCA_BTL_SELF_FRAG_ALLOC_RDMA(frag)                 \
+    {                                                      \
+        frag = (mca_btl_self_frag_t *) opal_free_list_get( \
+            &mca_btl_self_component.self_frags_rdma);      \
     }
 
-#define MCA_BTL_SELF_FRAG_ALLOC_SEND(frag)                              \
-    {                                                                   \
-        frag = (mca_btl_self_frag_t *)                                  \
-            opal_free_list_get (&mca_btl_self_component.self_frags_send); \
+#define MCA_BTL_SELF_FRAG_ALLOC_SEND(frag)                 \
+    {                                                      \
+        frag = (mca_btl_self_frag_t *) opal_free_list_get( \
+            &mca_btl_self_component.self_frags_send);      \
     }
 
-#define MCA_BTL_SELF_FRAG_RETURN(frag)                                  \
-    {                                                                   \
-        opal_free_list_return ((frag)->list, (opal_free_list_item_t*)(frag)); \
+#define MCA_BTL_SELF_FRAG_RETURN(frag)                                         \
+    {                                                                          \
+        opal_free_list_return((frag)->list, (opal_free_list_item_t *) (frag)); \
     }
 
 #endif /* MCA_BTL_SELF_SEND_FRAG_H */
diff --git a/opal/mca/btl/sm/Makefile.am b/opal/mca/btl/sm/Makefile.am
index 31123347e1d..367df73c132 100644
--- a/opal/mca/btl/sm/Makefile.am
+++ b/opal/mca/btl/sm/Makefile.am
@@ -9,11 +9,12 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2009-2014 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2009-2022 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights
 #                         reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
-# Copyright (c) 2020      Google, LLC. All rights reserved.
+# Copyright (c) 2020-2021 Google, LLC. All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -21,15 +22,10 @@
 # $HEADER$
 #
 
-AM_CPPFLAGS = $(btl_sm_CPPFLAGS)
-
-dist_opaldata_DATA = help-btl-sm.txt
-
 libmca_btl_sm_la_sources = \
     btl_sm_module.c \
     btl_sm.h \
     btl_sm_component.c \
-    btl_sm_endpoint.h \
     btl_sm_fifo.h \
     btl_sm_frag.c \
     btl_sm_frag.h \
@@ -38,11 +34,9 @@ libmca_btl_sm_la_sources = \
     btl_sm_fbox.h \
     btl_sm_get.c \
     btl_sm_put.c \
-    btl_sm_xpmem.c \
-    btl_sm_xpmem.h \
-    btl_sm_knem.c \
-    btl_sm_knem.h \
-    btl_sm_atomic.c
+    btl_sm_types.h \
+    btl_sm_virtual.h
+
 
 # Make the output library in this directory, and name it either
 # mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
@@ -59,11 +53,10 @@ endif
 mcacomponentdir = $(opallibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_btl_sm_la_SOURCES = $(libmca_btl_sm_la_sources)
-mca_btl_sm_la_LDFLAGS = -module -avoid-version $(btl_sm_LDFLAGS)
-mca_btl_sm_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
-	$(btl_sm_LIBS)
+mca_btl_sm_la_LDFLAGS = -module -avoid-version
+mca_btl_sm_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 
 noinst_LTLIBRARIES = $(component_noinst)
 libmca_btl_sm_la_SOURCES = $(libmca_btl_sm_la_sources)
-libmca_btl_sm_la_LIBADD = $(btl_sm_LIBS)
-libmca_btl_sm_la_LDFLAGS = -module -avoid-version $(btl_sm_LDFLAGS)
+libmca_btl_sm_la_LIBADD =
+libmca_btl_sm_la_LDFLAGS = -module -avoid-version
diff --git a/opal/mca/btl/sm/btl_sm.h b/opal/mca/btl/sm/btl_sm.h
index 416c2269fa5..9c05788f05b 100644
--- a/opal/mca/btl/sm/btl_sm.h
+++ b/opal/mca/btl/sm/btl_sm.h
@@ -38,142 +38,41 @@
 #include <stdlib.h>
 #include <string.h>
 
-# include <stdint.h>
+#include <stdint.h>
 #ifdef HAVE_SCHED_H
-# include <sched.h>
-#endif  /* HAVE_SCHED_H */
+#    include <sched.h>
+#endif /* HAVE_SCHED_H */
 #ifdef HAVE_UNISTD_H
-# include <unistd.h>
+#    include <unistd.h>
 #endif /* HAVE_UNISTD_H */
 
 #include "opal/mca/shmem/base/base.h"
 
-#include "opal/class/opal_free_list.h"
-#include "opal/sys/atomic.h"
-#include "opal/mca/btl/btl.h"
-#include "opal/mca/rcache/rcache.h"
-#include "opal/mca/rcache/base/rcache_base_vma.h"
 #include "opal/mca/btl/base/base.h"
-#include "opal/mca/rcache/rcache.h"
-#include "opal/mca/rcache/base/base.h"
 #include "opal/mca/btl/base/btl_base_error.h"
+#include "opal/mca/btl/btl.h"
+#include "opal/mca/btl/sm/btl_sm_types.h"
 #include "opal/mca/mpool/base/base.h"
+#include "opal/mca/rcache/base/base.h"
+#include "opal/mca/rcache/base/rcache_base_vma.h"
+#include "opal/mca/rcache/rcache.h"
+#include "opal/sys/atomic.h"
 #include "opal/util/proc.h"
-#include "btl_sm_endpoint.h"
 
 #include "opal/mca/pmix/pmix-internal.h"
 
-#include "btl_sm_xpmem.h"
-#include "btl_sm_knem.h"
-
 BEGIN_C_DECLS
 
-#define min(a,b) ((a) < (b) ? (a) : (b))
+#define min(a, b) ((a) < (b) ? (a) : (b))
 
 /*
- * Shared Memory resource managment
+ * Shared Memory resource management
  */
 
 struct sm_fifo_t;
 
-/*
- * Modex data
- */
-union sm_modex_t {
-#if OPAL_BTL_SM_HAVE_XPMEM
-    struct sm_modex_xpmem_t {
-        xpmem_segid_t seg_id;
-        void *segment_base;
-        uintptr_t address_max;
-    } xpmem;
-#endif
-    struct sm_modex_other_t {
-        ino_t user_ns_id;
-        int seg_ds_size;
-        /* seg_ds needs to be the last element */
-        opal_shmem_ds_t seg_ds;
-    } other;
-};
-
-/**
- * Single copy mechanisms
- */
-enum {
-      MCA_BTL_SM_XPMEM = 0,
-      MCA_BTL_SM_CMA   = 1,
-      MCA_BTL_SM_KNEM  = 2,
-      MCA_BTL_SM_NONE  = 3,
-};
-
-/**
- * Shared Memory (SM) BTL module.
- */
-struct mca_btl_sm_component_t {
-    mca_btl_base_component_3_0_0_t super;   /**< base BTL component */
-    int sm_free_list_num;                /**< initial size of free lists */
-    int sm_free_list_max;                /**< maximum size of free lists */
-    int sm_free_list_inc;                /**< number of elements to alloc when growing free lists */
-#if OPAL_BTL_SM_HAVE_XPMEM
-    xpmem_segid_t my_seg_id;                /**< this rank's xpmem segment id */
-    uintptr_t my_address_max;               /**< largest address */
-    mca_rcache_base_vma_module_t *vma_module; /**< registration cache for xpmem segments */
-#endif
-    opal_shmem_ds_t seg_ds;                 /**< this rank's shared memory segment (when not using xpmem) */
-
-    opal_mutex_t lock;                      /**< lock to protect concurrent updates to this structure's members */
-    char *my_segment;                       /**< this rank's base pointer */
-    size_t segment_size;                    /**< size of my_segment */
-    int32_t num_smp_procs;                  /**< current number of smp procs on this host */
-    opal_free_list_t sm_frags_eager;     /**< free list of sm send frags */
-    opal_free_list_t sm_frags_max_send;  /**< free list of sm max send frags (large fragments) */
-    opal_free_list_t sm_frags_user;      /**< free list of small inline frags */
-    opal_free_list_t sm_fboxes;          /**< free list of available fast-boxes */
-
-    unsigned int fbox_threshold;            /**< number of sends required before we setup a send fast box for a peer */
-    unsigned int fbox_max;                  /**< maximum number of send fast boxes to allocate */
-    unsigned int fbox_size;                 /**< size of each peer fast box allocation */
-
-    int single_copy_mechanism;              /**< single copy mechanism to use */
-
-    int memcpy_limit;                       /**< Limit where we switch from memmove to memcpy */
-    int log_attach_align;                   /**< Log of the alignment for xpmem segments */
-    unsigned int max_inline_send;           /**< Limit for copy-in-copy-out fragments */
-
-    mca_btl_base_endpoint_t *endpoints;     /**< array of local endpoints (one for each local peer including myself) */
-    mca_btl_base_endpoint_t **fbox_in_endpoints; /**< array of fast box in endpoints */
-    unsigned int num_fbox_in_endpoints;     /**< number of fast boxes to poll */
-    struct sm_fifo_t *my_fifo;           /**< pointer to the local fifo */
-
-    opal_list_t pending_endpoints;          /**< list of endpoints with pending fragments */
-    opal_list_t pending_fragments;          /**< fragments pending remote completion */
-
-    char *backing_directory;                /**< directory to place shared memory backing files */
-
-    /* knem stuff */
-#if OPAL_BTL_SM_HAVE_KNEM
-    unsigned int knem_dma_min;              /**< minimum size to enable DMA for knem transfers (0 disables) */
-#endif
-    mca_mpool_base_module_t *mpool;
-};
-typedef struct mca_btl_sm_component_t mca_btl_sm_component_t;
-OPAL_MODULE_DECLSPEC extern mca_btl_sm_component_t mca_btl_sm_component;
-
-/**
- * SM BTL Interface
- */
-struct mca_btl_sm_t {
-    mca_btl_base_module_t  super;       /**< base BTL interface */
-    bool btl_inited;  /**< flag indicating if btl has been inited */
-    mca_btl_base_module_error_cb_fn_t error_cb;
-#if OPAL_BTL_SM_HAVE_KNEM
-    int knem_fd;
-
-    /* registration cache */
-    mca_rcache_base_module_t *knem_rcache;
-#endif
-};
-typedef struct mca_btl_sm_t mca_btl_sm_t;
-OPAL_MODULE_DECLSPEC extern mca_btl_sm_t mca_btl_sm;
+OPAL_DECLSPEC extern mca_btl_sm_component_t mca_btl_sm_component;
+OPAL_DECLSPEC extern mca_btl_sm_t mca_btl_sm;
 
 /* number of peers on the node (not including self) */
 #define MCA_BTL_SM_NUM_LOCAL_PEERS opal_process_info.num_local_peers
@@ -183,12 +82,12 @@ OPAL_MODULE_DECLSPEC extern mca_btl_sm_t mca_btl_sm;
 
 /* memcpy is faster at larger sizes but is undefined if the
    pointers are aliased (TODO -- readd alias check) */
-static inline void sm_memmove (void *dst, void *src, size_t size)
+static inline void sm_memmove(void *dst, void *src, size_t size)
 {
     if (size >= (size_t) mca_btl_sm_component.memcpy_limit) {
-        memcpy (dst, src, size);
+        memcpy(dst, src, size);
     } else {
-        memmove (dst, src, size);
+        memmove(dst, src, size);
     }
 }
 
@@ -198,10 +97,8 @@ static inline void sm_memmove (void *dst, void *src, size_t size)
  * @param btl (IN)      BTL module
  * @param peer (IN)     BTL peer addressing
  */
-int mca_btl_sm_send(struct mca_btl_base_module_t *btl,
-                    struct mca_btl_base_endpoint_t *endpoint,
-                    struct mca_btl_base_descriptor_t *descriptor,
-                    mca_btl_base_tag_t tag);
+int mca_btl_sm_send(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+                    struct mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag);
 
 /**
  * Initiate an inline send to the peer.
@@ -209,13 +106,10 @@ int mca_btl_sm_send(struct mca_btl_base_module_t *btl,
  * @param btl (IN)      BTL module
  * @param peer (IN)     BTL peer addressing
  */
-int mca_btl_sm_sendi (struct mca_btl_base_module_t *btl,
-                      struct mca_btl_base_endpoint_t *endpoint,
-                      struct opal_convertor_t *convertor,
-                      void *header, size_t header_size,
-                      size_t payload_size, uint8_t order,
-                      uint32_t flags, mca_btl_base_tag_t tag,
-                      mca_btl_base_descriptor_t **descriptor);
+int mca_btl_sm_sendi(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+                     struct opal_convertor_t *convertor, void *header, size_t header_size,
+                     size_t payload_size, uint8_t order, uint32_t flags, mca_btl_base_tag_t tag,
+                     mca_btl_base_descriptor_t **descriptor);
 
 /**
  * Initiate an synchronous put.
@@ -224,26 +118,12 @@ int mca_btl_sm_sendi (struct mca_btl_base_module_t *btl,
  * @param endpoint (IN)    BTL addressing information
  * @param descriptor (IN)  Description of the data to be transferred
  */
-#if OPAL_BTL_SM_HAVE_XPMEM
-int mca_btl_sm_put_xpmem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
-                          uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                          mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-                          int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
-#endif
-
-#if OPAL_BTL_SM_HAVE_CMA
-int mca_btl_sm_put_cma (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
-                        uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                        mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-                        int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
-#endif
-
-#if OPAL_BTL_SM_HAVE_KNEM
-int mca_btl_sm_put_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
-                         uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                         mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-                         int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
-#endif
+int mca_btl_sm_put(mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
+                   void *local_address, uint64_t remote_address,
+                   mca_btl_base_registration_handle_t *local_handle,
+                   mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
+                   int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext,
+                   void *cbdata);
 
 /**
  * Initiate an synchronous get.
@@ -252,28 +132,12 @@ int mca_btl_sm_put_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *en
  * @param endpoint (IN)    BTL addressing information
  * @param descriptor (IN)  Description of the data to be transferred
  */
-#if OPAL_BTL_SM_HAVE_XPMEM
-int mca_btl_sm_get_xpmem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
-                          uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                          mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-                          int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
-#endif
-
-#if OPAL_BTL_SM_HAVE_CMA
-int mca_btl_sm_get_cma (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
-                        uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                        mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-                        int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
-#endif
-
-#if OPAL_BTL_SM_HAVE_KNEM
-int mca_btl_sm_get_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
-                         uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                         mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-                         int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
-#endif
-
-ino_t mca_btl_sm_get_user_ns_id(void);
+int mca_btl_sm_get(mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
+                   void *local_address, uint64_t remote_address,
+                   mca_btl_base_registration_handle_t *local_handle,
+                   mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
+                   int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext,
+                   void *cbdata);
 
 /**
  * Allocate a segment.
@@ -281,9 +145,9 @@ ino_t mca_btl_sm_get_user_ns_id(void);
  * @param btl (IN)      BTL module
  * @param size (IN)     Request segment size.
  */
-mca_btl_base_descriptor_t* mca_btl_sm_alloc (struct mca_btl_base_module_t* btl,
-                                             struct mca_btl_base_endpoint_t* endpoint,
-                                             uint8_t order, size_t size, uint32_t flags);
+mca_btl_base_descriptor_t *mca_btl_sm_alloc(struct mca_btl_base_module_t *btl,
+                                            struct mca_btl_base_endpoint_t *endpoint, uint8_t order,
+                                            size_t size, uint32_t flags);
 
 /**
  * Return a segment allocated by this BTL.
@@ -291,8 +155,11 @@ mca_btl_base_descriptor_t* mca_btl_sm_alloc (struct mca_btl_base_module_t* btl,
  * @param btl (IN)      BTL module
  * @param segment (IN)  Allocated segment.
  */
-int mca_btl_sm_free (struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des);
+int mca_btl_sm_free(struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des);
 
+static inline bool mca_btl_is_self_endpoint(mca_btl_base_endpoint_t *endpoint) {
+    return endpoint->peer_smp_rank == MCA_BTL_SM_LOCAL_RANK;
+}
 
 END_C_DECLS
 
diff --git a/opal/mca/btl/sm/btl_sm_atomic.c b/opal/mca/btl/sm/btl_sm_atomic.c
deleted file mode 100644
index edd579dec87..00000000000
--- a/opal/mca/btl/sm/btl_sm_atomic.c
+++ /dev/null
@@ -1,49 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2010-2017 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2019-2020  Google, Inc. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "opal_config.h"
-
-#include "btl_sm.h"
-#include "btl_sm_frag.h"
-#include "btl_sm_endpoint.h"
-#include "btl_sm_xpmem.h"
-
-int mca_btl_sm_emu_aop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
-                        uint64_t remote_address, mca_btl_base_registration_handle_t *remote_handle,
-                        mca_btl_base_atomic_op_t op, uint64_t operand, int flags, int order,
-                        mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
-{
-    size_t size = (flags & MCA_BTL_ATOMIC_FLAG_32BIT) ? 4 : 8;
-    return mca_btl_sm_rdma_frag_start (btl, endpoint, MCA_BTL_SM_OP_ATOMIC, operand, 0, op, order, flags,
-                                       size, NULL, remote_address, cbfunc, cbcontext, cbdata);
-}
-
-int mca_btl_sm_emu_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
-                            void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                         mca_btl_base_registration_handle_t *remote_handle, mca_btl_base_atomic_op_t op,
-                         uint64_t operand, int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
-                         void *cbcontext, void *cbdata)
-{
-    size_t size = (flags & MCA_BTL_ATOMIC_FLAG_32BIT) ? 4 : 8;
-    return mca_btl_sm_rdma_frag_start (btl, endpoint, MCA_BTL_SM_OP_ATOMIC, operand, 0, op, order, flags,
-                                       size, local_address, remote_address, cbfunc, cbcontext, cbdata);
-}
-
-int mca_btl_sm_emu_acswap (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
-                           void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                           mca_btl_base_registration_handle_t *remote_handle, uint64_t compare, uint64_t value, int flags,
-                           int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
-{
-    size_t size = (flags & MCA_BTL_ATOMIC_FLAG_32BIT) ? 4 : 8;
-    return mca_btl_sm_rdma_frag_start (btl, endpoint, MCA_BTL_SM_OP_CSWAP, compare, value, 0, order,
-                                       flags, size, local_address, remote_address, cbfunc, cbcontext, cbdata);
-}
diff --git a/opal/mca/btl/sm/btl_sm_component.c b/opal/mca/btl/sm/btl_sm_component.c
index 671323e5bcb..a0ec85e1f40 100644
--- a/opal/mca/btl/sm/btl_sm_component.c
+++ b/opal/mca/btl/sm/btl_sm_component.c
@@ -11,7 +11,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2006-2007 Voltaire. All rights reserved.
- * Copyright (c) 2009-2010 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2009-2022 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2010-2018 Los Alamos National Security, LLC.
  *                         All rights reserved.
  * Copyright (c) 2011      NVIDIA Corporation.  All rights reserved.
@@ -21,7 +21,11 @@
  * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * Copyright (c) 2018      Triad National Security, LLC. All rights
  *                         reserved.
- * Copyright (c) 2019-2020 Google, Inc. All rights reserved.
+ * Copyright (c) 2019-2025 Google, Inc. All rights reserved.
+ * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2022      Computer Architecture and VLSI Systems (CARV)
+ *                         Laboratory, ICS Forth. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -30,139 +34,114 @@
  */
 #include "opal_config.h"
 
+#include "opal/mca/btl/base/btl_base_error.h"
+#include "opal/mca/threads/mutex.h"
+#include "opal/util/bit_ops.h"
 #include "opal/util/output.h"
-#include "opal/util/show_help.h"
 #include "opal/util/printf.h"
-#include "opal/mca/threads/mutex.h"
-#include "opal/mca/btl/base/btl_base_error.h"
 
-#include "btl_sm.h"
-#include "btl_sm_frag.h"
-#include "btl_sm_fifo.h"
-#include "btl_sm_fbox.h"
-#include "btl_sm_xpmem.h"
+#include "opal/mca/btl/sm/btl_sm.h"
+#include "opal/mca/btl/sm/btl_sm_fbox.h"
+#include "opal/mca/btl/sm/btl_sm_fifo.h"
+#include "opal/mca/btl/sm/btl_sm_frag.h"
+#include "opal/mca/smsc/smsc.h"
 
 #ifdef HAVE_SYS_STAT_H
-#include <sys/stat.h>
+#    include <sys/stat.h>
 #endif
 
-#include <sys/mman.h>
 #include <fcntl.h>
+#include <sys/mman.h>
 
 #ifdef HAVE_SYS_PRCTL_H
-#include <sys/prctl.h>
+#    include <sys/prctl.h>
 #endif
 
 /* NTH: OS X does not define MAP_ANONYMOUS */
 #if !defined(MAP_ANONYMOUS)
-#define MAP_ANONYMOUS MAP_ANON
+#    define MAP_ANONYMOUS MAP_ANON
 #endif
 
-static int mca_btl_sm_component_progress (void);
+static int mca_btl_sm_component_progress(void);
 static int mca_btl_sm_component_open(void);
 static int mca_btl_sm_component_close(void);
 static int mca_btl_sm_component_register(void);
-static mca_btl_base_module_t** mca_btl_sm_component_init(int *num_btls,
-                                                         bool enable_progress_threads,
-                                                         bool enable_mpi_threads);
-
-/* This enumeration is in order of preference */
-static mca_base_var_enum_value_t single_copy_mechanisms[] = {
-#if OPAL_BTL_SM_HAVE_XPMEM
-                                                             {.value = MCA_BTL_SM_XPMEM, .string = "xpmem"},
-#endif
-#if OPAL_BTL_SM_HAVE_CMA
-                                                             {.value = MCA_BTL_SM_CMA, .string = "cma"},
-#endif
-#if OPAL_BTL_SM_HAVE_KNEM
-                                                             {.value = MCA_BTL_SM_KNEM, .string = "knem"},
-#endif
-                                                             {.value = MCA_BTL_SM_NONE, .string = "none"},
-                                                             {.value = 0, .string = NULL}
-};
+static mca_btl_base_module_t **
+mca_btl_sm_component_init(int *num_btls, bool enable_progress_threads, bool enable_mpi_threads);
 
 /*
  * Shared Memory (SM) component instance.
  */
 mca_btl_sm_component_t mca_btl_sm_component = {
-                                               .super = {
-                                                         /* First, the mca_base_component_t struct containing meta information
-                                                            about the component itself */
-                                                         .btl_version = {
-                                                                         MCA_BTL_DEFAULT_VERSION("sm"),
-                                                                         .mca_open_component = mca_btl_sm_component_open,
-                                                                         .mca_close_component = mca_btl_sm_component_close,
-                                                                         .mca_register_component_params = mca_btl_sm_component_register,
-                                                                         },
-                                                         .btl_data = {
-                                                                      /* The component is checkpoint ready */
-                                                                      .param_field = MCA_BASE_METADATA_PARAM_CHECKPOINT
-                                                                      },
-
-                                                         .btl_init = mca_btl_sm_component_init,
-                                                         .btl_progress = mca_btl_sm_component_progress,
-    }  /* end super */
+    .super =
+        {
+            /* First, the mca_base_component_t struct containing meta information
+               about the component itself */
+            .btl_version =
+                {
+                    MCA_BTL_DEFAULT_VERSION("sm"),
+                    .mca_open_component = mca_btl_sm_component_open,
+                    .mca_close_component = mca_btl_sm_component_close,
+                    .mca_register_component_params = mca_btl_sm_component_register,
+                },
+            .btl_data =
+                {/* The component is checkpoint ready */
+                 .param_field = MCA_BASE_METADATA_PARAM_CHECKPOINT},
+
+            .btl_init = mca_btl_sm_component_init,
+            .btl_progress = mca_btl_sm_component_progress,
+        } /* end super */
 };
 
-static int mca_btl_sm_component_register (void)
+static int mca_btl_sm_component_register(void)
 {
-    mca_base_var_enum_t *new_enum;
-
     (void) mca_base_var_group_component_register(&mca_btl_sm_component.super.btl_version,
                                                  "Enhanced shared memory byte transport later");
 
     /* register SM component variables */
     mca_btl_sm_component.sm_free_list_num = 8;
-    (void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version,
-                                           "free_list_num", "Initial number of fragments "
+    (void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version, "free_list_num",
+                                           "Initial number of fragments "
                                            "to allocate for shared memory communication.",
                                            MCA_BASE_VAR_TYPE_INT, NULL, 0,
                                            MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9,
                                            MCA_BASE_VAR_SCOPE_LOCAL,
                                            &mca_btl_sm_component.sm_free_list_num);
     mca_btl_sm_component.sm_free_list_max = 512;
-    (void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version,
-                                           "free_list_max", "Maximum number of fragments "
+    (void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version, "free_list_max",
+                                           "Maximum number of fragments "
                                            "to allocate for shared memory communication.",
                                            MCA_BASE_VAR_TYPE_INT, NULL, 0,
                                            MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9,
                                            MCA_BASE_VAR_SCOPE_LOCAL,
                                            &mca_btl_sm_component.sm_free_list_max);
     mca_btl_sm_component.sm_free_list_inc = 64;
-    (void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version,
-                                           "free_list_inc", "Number of fragments to create "
-                                           "on each allocation.", MCA_BASE_VAR_TYPE_INT, NULL, 0,
+    (void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version, "free_list_inc",
+                                           "Number of fragments to create "
+                                           "on each allocation.",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0,
                                            MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9,
                                            MCA_BASE_VAR_SCOPE_LOCAL,
                                            &mca_btl_sm_component.sm_free_list_inc);
 
     mca_btl_sm_component.memcpy_limit = 524288;
-    (void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version,
-                                           "memcpy_limit", "Message size to switch from using "
-                                           "memove to memcpy. The relative speed of these two "
-                                           "routines can vary by size.", MCA_BASE_VAR_TYPE_INT,
-                                           NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5,
-                                           MCA_BASE_VAR_SCOPE_LOCAL,
-                                           &mca_btl_sm_component.memcpy_limit);
-#if OPAL_BTL_SM_HAVE_XPMEM
-    mca_btl_sm_component.log_attach_align = 21;
-    (void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version,
-                                           "log_align", "Log base 2 of the alignment to use for xpmem "
-                                           "segments (default: 21, minimum: 12, maximum: 25)",
+    (void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version, "memcpy_limit",
+                                           "Message size to switch from using "
+                                           "memmove to memcpy. The relative speed of these two "
+                                           "routines can vary by size.",
                                            MCA_BASE_VAR_TYPE_INT, NULL, 0,
                                            MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5,
                                            MCA_BASE_VAR_SCOPE_LOCAL,
-                                           &mca_btl_sm_component.log_attach_align);
-#endif
+                                           &mca_btl_sm_component.memcpy_limit);
 
-#if OPAL_BTL_SM_HAVE_XPMEM && 64 == MCA_BTL_SM_BITNESS
+#if 64 == MCA_BTL_SM_BITNESS
     mca_btl_sm_component.segment_size = 1 << 24;
 #else
     mca_btl_sm_component.segment_size = 1 << 22;
 #endif
-    (void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version,
-                                           "segment_size", "Maximum size of all shared "
-#if OPAL_BTL_SM_HAVE_XPMEM && 64 == MCA_BTL_SM_BITNESS
+    (void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version, "segment_size",
+                                           "Maximum size of all shared "
+#if 64 == MCA_BTL_SM_BITNESS
                                            "memory buffers (default: 16M)",
 #else
                                            "memory buffers (default: 4M)",
@@ -174,7 +153,8 @@ static int mca_btl_sm_component_register (void)
 
     mca_btl_sm_component.max_inline_send = 256;
     (void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version,
-                                           "max_inline_send", "Maximum size to transfer "
+                                           "max_inline_send",
+                                           "Maximum size to transfer "
                                            "using copy-in copy-out semantics",
                                            MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0,
                                            MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5,
@@ -183,90 +163,62 @@ static int mca_btl_sm_component_register (void)
 
     mca_btl_sm_component.fbox_threshold = 16;
     (void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version,
-                                           "fbox_threshold", "Number of sends required "
+                                           "fbox_threshold",
+                                           "Number of sends required "
                                            "before an eager send buffer is setup for a peer "
-                                           "(default: 16)", MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL,
-                                           0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5,
-                                           MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_sm_component.fbox_threshold);
+                                           "(default: 16)",
+                                           MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0,
+                                           MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5,
+                                           MCA_BASE_VAR_SCOPE_LOCAL,
+                                           &mca_btl_sm_component.fbox_threshold);
 
     mca_btl_sm_component.fbox_max = 32;
-    (void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version,
-                                           "fbox_max", "Maximum number of eager send buffers "
-                                           "to allocate (default: 32)", MCA_BASE_VAR_TYPE_UNSIGNED_INT,
-                                           NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5,
-                                           MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_sm_component.fbox_max);
+    (void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version, "fbox_max",
+                                           "Maximum number of eager send buffers "
+                                           "to allocate (default: 32)",
+                                           MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0,
+                                           MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5,
+                                           MCA_BASE_VAR_SCOPE_LOCAL,
+                                           &mca_btl_sm_component.fbox_max);
 
     mca_btl_sm_component.fbox_size = 4096;
-    (void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version,
-                                           "fbox_size", "Size of per-peer fast transfer buffers (default: 4k)",
-                                           MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-                                           OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_sm_component.fbox_size);
-
-    (void) mca_base_var_enum_create ("btl_sm_single_copy_mechanisms", single_copy_mechanisms, &new_enum);
-
-    /* Default to the best available mechanism (see the enumerator for ordering) */
-    mca_btl_sm_component.single_copy_mechanism = single_copy_mechanisms[0].value;
-    (void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version,
-                                           "single_copy_mechanism", "Single copy mechanism to use (defaults to best available)",
-                                           MCA_BASE_VAR_TYPE_INT, new_enum, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-                                           OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_GROUP, &mca_btl_sm_component.single_copy_mechanism);
-    OBJ_RELEASE(new_enum);
+    (void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version, "fbox_size",
+                                           "Size of per-peer fast transfer buffers. Must be a power of two (default: 4k)",
+                                           MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0,
+                                           MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5,
+                                           MCA_BASE_VAR_SCOPE_LOCAL,
+                                           &mca_btl_sm_component.fbox_size);
 
-    if (0 == access ("/dev/shm", W_OK)) {
+    if (0 == access("/dev/shm", W_OK)) {
         mca_btl_sm_component.backing_directory = "/dev/shm";
     } else {
         mca_btl_sm_component.backing_directory = opal_process_info.job_session_dir;
     }
-    (void) mca_base_component_var_register (&mca_btl_sm_component.super.btl_version, "backing_directory",
-                                            "Directory to place backing files for shared memory communication. "
-                                            "This directory should be on a local filesystem such as /tmp or "
-                                            "/dev/shm (default: (linux) /dev/shm, (others) session directory)",
-                                            MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_3,
-                                            MCA_BASE_VAR_SCOPE_READONLY, &mca_btl_sm_component.backing_directory);
-
-
-#if OPAL_BTL_SM_HAVE_KNEM
-    /* Currently disabling DMA mode by default; it's not clear that this is useful in all applications and architectures. */
-    mca_btl_sm_component.knem_dma_min = 0;
-    (void) mca_base_component_var_register(&mca_btl_sm_component.super.btl_version, "knem_dma_min",
-                                           "Minimum message size (in bytes) to use the knem DMA mode; "
-                                           "ignored if knem does not support DMA mode (0 = do not use the "
-                                           "knem DMA mode, default: 0)", MCA_BASE_VAR_TYPE_UNSIGNED_INT,
-                                           NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
-                                           &mca_btl_sm_component.knem_dma_min);
-#endif
+    (void) mca_base_component_var_register(
+        &mca_btl_sm_component.super.btl_version, "backing_directory",
+        "Directory to place backing files for shared memory communication. "
+        "This directory should be on a local filesystem such as /tmp or "
+        "/dev/shm (default: (linux) /dev/shm, (others) session directory)",
+        MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_READONLY,
+        &mca_btl_sm_component.backing_directory);
 
-    mca_btl_sm.super.btl_exclusivity               = MCA_BTL_EXCLUSIVITY_HIGH;
+    mca_btl_sm.super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_HIGH;
 
-    if (MCA_BTL_SM_XPMEM == mca_btl_sm_component.single_copy_mechanism) {
-        mca_btl_sm.super.btl_eager_limit               = 32 * 1024;
-        mca_btl_sm.super.btl_rndv_eager_limit          = mca_btl_sm.super.btl_eager_limit;
-        mca_btl_sm.super.btl_max_send_size             = mca_btl_sm.super.btl_eager_limit;
-        mca_btl_sm.super.btl_min_rdma_pipeline_size    = INT_MAX;
-    } else {
-        mca_btl_sm.super.btl_eager_limit               = 4 * 1024;
-        mca_btl_sm.super.btl_rndv_eager_limit          = 32 * 1024;
-        mca_btl_sm.super.btl_max_send_size             = 32 * 1024;
-        mca_btl_sm.super.btl_min_rdma_pipeline_size    = INT_MAX;
-    }
+    mca_btl_sm.super.btl_eager_limit = 4 * 1024;
+    mca_btl_sm.super.btl_rndv_eager_limit = 32 * 1024;
+    mca_btl_sm.super.btl_max_send_size = 32 * 1024;
+    mca_btl_sm.super.btl_min_rdma_pipeline_size = INT_MAX;
 
     mca_btl_sm.super.btl_rdma_pipeline_send_length = mca_btl_sm.super.btl_eager_limit;
-    mca_btl_sm.super.btl_rdma_pipeline_frag_size   = mca_btl_sm.super.btl_eager_limit;
+    mca_btl_sm.super.btl_rdma_pipeline_frag_size = mca_btl_sm.super.btl_eager_limit;
 
     mca_btl_sm.super.btl_flags = MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_SEND;
 
-    if (MCA_BTL_SM_NONE != mca_btl_sm_component.single_copy_mechanism) {
-        /* True single copy mechanisms should provide better bandwidth */
-        mca_btl_sm.super.btl_bandwidth = 40000; /* Mbs */
-    } else {
-        mca_btl_sm.super.btl_bandwidth = 10000; /* Mbs */
-    }
-
-    mca_btl_sm.super.btl_latency   = 1;     /* Microsecs */
+    mca_btl_sm.super.btl_bandwidth = 20000; /* Mbs */
+    mca_btl_sm.super.btl_latency = 1; /* Microsecs */
 
     /* Call the BTL based to register its MCA params */
-    mca_btl_base_param_register(&mca_btl_sm_component.super.btl_version,
-                                &mca_btl_sm.super);
+    mca_btl_base_param_register(&mca_btl_sm_component.super.btl_version, &mca_btl_sm.super);
 
     return OPAL_SUCCESS;
 }
@@ -286,14 +238,10 @@ static int mca_btl_sm_component_open(void)
     OBJ_CONSTRUCT(&mca_btl_sm_component.lock, opal_mutex_t);
     OBJ_CONSTRUCT(&mca_btl_sm_component.pending_endpoints, opal_list_t);
     OBJ_CONSTRUCT(&mca_btl_sm_component.pending_fragments, opal_list_t);
-#if OPAL_BTL_SM_HAVE_KNEM
-    mca_btl_sm.knem_fd = -1;
-#endif
 
     return OPAL_SUCCESS;
 }
 
-
 /*
  * component cleanup - sanity checking of queue lengths
  */
@@ -308,186 +256,52 @@ static int mca_btl_sm_component_close(void)
     OBJ_DESTRUCT(&mca_btl_sm_component.pending_endpoints);
     OBJ_DESTRUCT(&mca_btl_sm_component.pending_fragments);
 
-    if (MCA_BTL_SM_XPMEM == mca_btl_sm_component.single_copy_mechanism &&
-        NULL != mca_btl_sm_component.my_segment) {
-        munmap (mca_btl_sm_component.my_segment, mca_btl_sm_component.segment_size);
-    }
-
     mca_btl_sm_component.my_segment = NULL;
 
-#if OPAL_BTL_SM_HAVE_KNEM
-    mca_btl_sm_knem_fini ();
-#endif
-
     if (mca_btl_sm_component.mpool) {
-        mca_btl_sm_component.mpool->mpool_finalize (mca_btl_sm_component.mpool);
+        mca_btl_sm_component.mpool->mpool_finalize(mca_btl_sm_component.mpool);
         mca_btl_sm_component.mpool = NULL;
     }
 
     return OPAL_SUCCESS;
 }
 
-/*
- * mca_btl_sm_parse_proc_ns_user() tries to get the user namespace ID
- * of the current process.
- * Returns the ID of the user namespace. In the case of an error '0' is returned.
- */
-ino_t mca_btl_sm_get_user_ns_id(void)
+static int mca_btl_base_sm_modex_send(void)
 {
-    struct stat buf;
-
-    if (0 > stat("/proc/self/ns/user", &buf)) {
-        /*
-         * Something went wrong, probably an old kernel that does not support namespaces
-         * simply assume all processes are in the same user namespace and return 0
-         */
-        return 0;
-    }
+    mca_btl_sm_modex_t modex;
+    int modex_size;
 
-    return buf.st_ino;
-}
-static int mca_btl_base_sm_modex_send (void)
-{
-    union sm_modex_t modex;
-    int modex_size, rc;
+    modex_size = sizeof(modex) - sizeof(modex.seg_ds);
 
-#if OPAL_BTL_SM_HAVE_XPMEM
-    if (MCA_BTL_SM_XPMEM == mca_btl_sm_component.single_copy_mechanism) {
-        modex.xpmem.seg_id = mca_btl_sm_component.my_seg_id;
-        modex.xpmem.segment_base = mca_btl_sm_component.my_segment;
-        modex.xpmem.address_max = mca_btl_sm_component.my_address_max;
+        modex.seg_ds_size = opal_shmem_sizeof_shmem_ds(&mca_btl_sm_component.seg_ds);
+        memmove(&modex.seg_ds, &mca_btl_sm_component.seg_ds, modex.seg_ds_size);
+        modex_size += modex.seg_ds_size;
 
-        modex_size = sizeof (modex.xpmem);
-    } else {
-#endif
-        modex.other.seg_ds_size = opal_shmem_sizeof_shmem_ds (&mca_btl_sm_component.seg_ds);
-        memmove (&modex.other.seg_ds, &mca_btl_sm_component.seg_ds, modex.other.seg_ds_size);
-        modex.other.user_ns_id = mca_btl_sm_get_user_ns_id();
-        /*
-         * If modex.other.user_ns_id is '0' something did not work out
-         * during user namespace detection. Assuming there are no
-         * namespaces available it will return '0' for all processes and
-         * the check later will see '0' everywhere and not disable CMA.
-         */
-        modex_size = sizeof (modex.other);
-
-#if OPAL_BTL_SM_HAVE_XPMEM
-    }
-#endif
-
-    OPAL_MODEX_SEND(rc, PMIX_LOCAL,
-                    &mca_btl_sm_component.super.btl_version, &modex, modex_size);
+    int rc;
+    OPAL_MODEX_SEND(rc, PMIX_LOCAL, &mca_btl_sm_component.super.btl_version, &modex, modex_size);
 
     return rc;
 }
 
-#if OPAL_BTL_SM_HAVE_XPMEM || OPAL_BTL_SM_HAVE_CMA || OPAL_BTL_SM_HAVE_KNEM
-static void mca_btl_sm_select_next_single_copy_mechanism (void)
+static mca_btl_base_registration_handle_t *
+mca_btl_sm_register_mem(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+                        void *base, size_t size, uint32_t flags)
 {
-    for (int i = 0 ; single_copy_mechanisms[i].value != MCA_BTL_SM_NONE ; ++i) {
-        if (single_copy_mechanisms[i].value == mca_btl_sm_component.single_copy_mechanism) {
-            mca_btl_sm_component.single_copy_mechanism = single_copy_mechanisms[i+1].value;
-            return;
-        }
-    }
+    return (mca_btl_base_registration_handle_t *) MCA_SMSC_CALL(register_region, base, size);
 }
-#endif
 
-static void mca_btl_sm_check_single_copy (void)
+static int mca_btl_sm_deregister_mem_knem(struct mca_btl_base_module_t *btl,
+                                          struct mca_btl_base_registration_handle_t *handle)
 {
-#if OPAL_BTL_SM_HAVE_XPMEM || OPAL_BTL_SM_HAVE_CMA || OPAL_BTL_SM_HAVE_KNEM
-    int initial_mechanism = mca_btl_sm_component.single_copy_mechanism;
-#endif
-
-#if OPAL_BTL_SM_HAVE_XPMEM
-    if (MCA_BTL_SM_XPMEM == mca_btl_sm_component.single_copy_mechanism) {
-        /* try to create an xpmem segment for the entire address space */
-        int rc = mca_btl_sm_xpmem_init ();
-        if (OPAL_SUCCESS != rc) {
-            if (MCA_BTL_SM_XPMEM == initial_mechanism) {
-                opal_show_help("help-btl-sm.txt", "xpmem-make-failed",
-                               true, opal_process_info.nodename, errno,
-                               strerror(errno));
-            }
-
-            mca_btl_sm_select_next_single_copy_mechanism ();
-        }
-    }
-#endif
-
-#if OPAL_BTL_SM_HAVE_CMA
-    if (MCA_BTL_SM_CMA == mca_btl_sm_component.single_copy_mechanism) {
-        /* Check if we have the proper permissions for CMA */
-        char buffer = '0';
-        bool cma_happy = false;
-        int fd;
-
-        /* check system setting for current ptrace scope */
-        fd = open ("/proc/sys/kernel/yama/ptrace_scope", O_RDONLY);
-        if (0 <= fd) {
-            read (fd, &buffer, 1);
-            close (fd);
-        }
-
-        /* ptrace scope 0 will allow an attach from any of the process owner's
-         * processes. ptrace scope 1 limits attachers to the process tree
-         * starting at the parent of this process. */
-        if ('0' != buffer) {
-#if defined PR_SET_PTRACER
-            /* try setting the ptrace scope to allow attach */
-            int ret = prctl (PR_SET_PTRACER, PR_SET_PTRACER_ANY, 0, 0, 0);
-            if (0 == ret) {
-                cma_happy = true;
-            }
-#endif
-        } else {
-            cma_happy = true;
-        }
-
-        if (!cma_happy) {
-            mca_btl_sm_select_next_single_copy_mechanism ();
-
-            if (MCA_BTL_SM_CMA == initial_mechanism) {
-                opal_show_help("help-btl-sm.txt", "cma-permission-denied",
-                               true, opal_process_info.nodename);
-            }
-        } else {
-            /* ptrace_scope will allow CMA */
-            mca_btl_sm.super.btl_get = mca_btl_sm_get_cma;
-            mca_btl_sm.super.btl_put = mca_btl_sm_put_cma;
-        }
-    }
-#endif
-
-#if OPAL_BTL_SM_HAVE_KNEM
-    if (MCA_BTL_SM_KNEM == mca_btl_sm_component.single_copy_mechanism) {
-        /* mca_btl_sm_knem_init will set the appropriate get/put functions */
-        int rc = mca_btl_sm_knem_init ();
-        if (OPAL_SUCCESS != rc) {
-            if (MCA_BTL_SM_KNEM == initial_mechanism) {
-                opal_show_help("help-btl-sm.txt", "knem requested but not available",
-                               true, opal_process_info.nodename);
-            }
-
-            /* disable single copy */
-            mca_btl_sm_select_next_single_copy_mechanism ();
-        }
-    }
-#endif
-
-    if (MCA_BTL_SM_NONE == mca_btl_sm_component.single_copy_mechanism) {
-        mca_btl_sm.super.btl_flags &= ~MCA_BTL_FLAGS_RDMA;
-        mca_btl_sm.super.btl_get = NULL;
-        mca_btl_sm.super.btl_put = NULL;
-    }
+    MCA_SMSC_CALL(deregister_region, (void *) handle);
+    return OPAL_SUCCESS;
 }
 
 /*
  *  SM component initialization
  */
-static mca_btl_base_module_t **mca_btl_sm_component_init (int *num_btls,
-                                                          bool enable_progress_threads,
-                                                          bool enable_mpi_threads)
+static mca_btl_base_module_t **
+mca_btl_sm_component_init(int *num_btls, bool enable_progress_threads, bool enable_mpi_threads)
 {
     mca_btl_sm_component_t *component = &mca_btl_sm_component;
     mca_btl_base_module_t **btls = NULL;
@@ -501,16 +315,7 @@ static mca_btl_base_module_t **mca_btl_sm_component_init (int *num_btls,
         return NULL;
     }
 
-#if OPAL_BTL_SM_HAVE_XPMEM
-    /* limit segment alignment to be between 4k and 16M */
-    if (component->log_attach_align < 12) {
-        component->log_attach_align = 12;
-    } else if (component->log_attach_align > 25) {
-        component->log_attach_align = 25;
-    }
-#endif
-
-    btls = (mca_btl_base_module_t **) calloc (1, sizeof (mca_btl_base_module_t *));
+    btls = (mca_btl_base_module_t **) calloc(1, sizeof(mca_btl_base_module_t *));
     if (NULL == btls) {
         return NULL;
     }
@@ -520,7 +325,10 @@ static mca_btl_base_module_t **mca_btl_sm_component_init (int *num_btls,
         component->segment_size = (2 << 20);
     }
 
-    component->fbox_size = (component->fbox_size + MCA_BTL_SM_FBOX_ALIGNMENT_MASK) & ~MCA_BTL_SM_FBOX_ALIGNMENT_MASK;
+    if (component->fbox_size & (component->fbox_size - 1)) {
+        BTL_VERBOSE(("fast box size must be a power of two, rounding up to next power of two."));
+        component->fbox_size = opal_next_poweroftwo_inclusive(component->fbox_size);
+    }
 
     if (component->segment_size > (1ul << MCA_BTL_SM_OFFSET_BITS)) {
         component->segment_size = 2ul << MCA_BTL_SM_OFFSET_BITS;
@@ -529,47 +337,74 @@ static mca_btl_base_module_t **mca_btl_sm_component_init (int *num_btls,
     /* no fast boxes allocated initially */
     component->num_fbox_in_endpoints = 0;
 
-    mca_btl_sm_check_single_copy ();
-
-    if (MCA_BTL_SM_XPMEM != mca_btl_sm_component.single_copy_mechanism) {
-        char *sm_file;
+    bool have_smsc = (NULL != mca_smsc);
+    if (have_smsc) {
+        mca_btl_sm.super.btl_flags |= MCA_BTL_FLAGS_RDMA;
+        mca_btl_sm.super.btl_get = mca_btl_sm_get;
+        mca_btl_sm.super.btl_put = mca_btl_sm_put;
 
-        rc = opal_asprintf(&sm_file, "%s" OPAL_PATH_SEP "sm_segment.%s.%x.%d", mca_btl_sm_component.backing_directory,
-                           opal_process_info.nodename, OPAL_PROC_MY_NAME.jobid, MCA_BTL_SM_LOCAL_RANK);
-        if (0 > rc) {
-            free (btls);
-            return NULL;
-        }
-        opal_pmix_register_cleanup (sm_file, false, false, false);
-
-        rc = opal_shmem_segment_create (&component->seg_ds, sm_file, component->segment_size);
-        free (sm_file);
-        if (OPAL_SUCCESS != rc) {
-            BTL_VERBOSE(("Could not create shared memory segment"));
-            free (btls);
-            return NULL;
+        if (mca_smsc_base_has_feature(MCA_SMSC_FEATURE_ACCELERATOR)) {
+            mca_btl_sm.super.btl_flags |= MCA_BTL_FLAGS_ACCELERATOR_GET;
         }
 
-        component->my_segment = opal_shmem_segment_attach (&component->seg_ds);
-        if (NULL == component->my_segment) {
-            BTL_VERBOSE(("Could not attach to just created shared memory segment"));
-            goto failed;
+        mca_btl_sm.super.btl_bandwidth = 40000; /* Mbs */
+
+        if (mca_smsc_base_has_feature(MCA_SMSC_FEATURE_CAN_MAP)) {
+            mca_btl_sm.super.btl_eager_limit = 32 * 1024;
+            mca_btl_sm.super.btl_rndv_eager_limit = mca_btl_sm.super.btl_eager_limit;
+            mca_btl_sm.super.btl_max_send_size = mca_btl_sm.super.btl_eager_limit;
+            mca_btl_sm.super.btl_min_rdma_pipeline_size = INT_MAX;
         }
-    } else {
-        /* when using xpmem it is safe to use an anonymous segment */
-        component->my_segment = mmap (NULL, component->segment_size, PROT_READ |
-                                      PROT_WRITE, MAP_ANONYMOUS | MAP_SHARED, -1, 0);
-        if ((void *)-1 == component->my_segment) {
-            BTL_VERBOSE(("Could not create anonymous memory segment"));
-            free (btls);
-            return NULL;
+        if (mca_smsc_base_has_feature(MCA_SMSC_FEATURE_REQUIRE_REGISTRATION)) {
+            ssize_t handle_size = mca_smsc_base_registration_data_size();
+            if (handle_size > 0) {
+                mca_btl_sm.super.btl_registration_handle_size = (size_t) handle_size;
+                mca_btl_sm.super.btl_register_mem = mca_btl_sm_register_mem;
+                mca_btl_sm.super.btl_deregister_mem = mca_btl_sm_deregister_mem_knem;
+            } else {
+                BTL_ERROR(("single-copy component requires registration but could not provide the "
+                           "registration handle size"));
+                have_smsc = false;
+            }
         }
     }
+    if (!have_smsc) {
+        mca_btl_sm.super.btl_flags &= ~MCA_BTL_FLAGS_RDMA;
+        mca_btl_sm.super.btl_get = NULL;
+        mca_btl_sm.super.btl_put = NULL;
+    }
+
+    char *sm_file;
+
+    // Note: Use the node_rank not the local_rank for the backing file.
+    // This makes the file unique even when recovering from failures.
+    rc = opal_asprintf(&sm_file, "%s" OPAL_PATH_SEP "sm_segment.%s.%u.%x.%d",
+                       mca_btl_sm_component.backing_directory, opal_process_info.nodename,
+                       geteuid(), OPAL_PROC_MY_NAME.jobid, opal_process_info.my_node_rank);
+    if (0 > rc) {
+        free(btls);
+        return NULL;
+    }
+    opal_pmix_register_cleanup(sm_file, false, false, false);
+
+    rc = opal_shmem_segment_create(&component->seg_ds, sm_file, component->segment_size);
+    free(sm_file);
+    if (OPAL_SUCCESS != rc) {
+        BTL_VERBOSE(("Could not create shared memory segment"));
+        free(btls);
+        return NULL;
+    }
+
+    component->my_segment = opal_shmem_segment_attach(&component->seg_ds);
+    if (NULL == component->my_segment) {
+        BTL_VERBOSE(("Could not attach to just created shared memory segment"));
+        goto failed;
+    }
 
     /* initialize my fifo */
-    sm_fifo_init ((struct sm_fifo_t *) component->my_segment);
+    sm_fifo_init((struct sm_fifo_t *) component->my_segment);
 
-    rc = mca_btl_base_sm_modex_send ();
+    rc = mca_btl_base_sm_modex_send();
     if (OPAL_SUCCESS != rc) {
         BTL_VERBOSE(("Error sending modex"));
         goto failed;
@@ -584,74 +419,71 @@ static mca_btl_base_module_t **mca_btl_sm_component_init (int *num_btls,
     mca_btl_sm.btl_inited = false;
 
     return btls;
- failed:
-#if OPAL_BTL_SM_HAVE_XPMEM
-    if (MCA_BTL_SM_XPMEM == mca_btl_sm_component.single_copy_mechanism) {
-        munmap (component->my_segment, component->segment_size);
-    } else
-#endif
-        opal_shmem_unlink (&component->seg_ds);
+failed:
+    opal_shmem_unlink(&component->seg_ds);
 
     if (btls) {
-        free (btls);
+        free(btls);
     }
 
     return NULL;
 }
 
-void mca_btl_sm_poll_handle_frag (mca_btl_sm_hdr_t *hdr, struct mca_btl_base_endpoint_t *endpoint)
+void mca_btl_sm_poll_handle_frag(mca_btl_sm_hdr_t *hdr, struct mca_btl_base_endpoint_t *endpoint)
 {
     if (hdr->flags & MCA_BTL_SM_FLAG_COMPLETE) {
-        mca_btl_sm_frag_complete (hdr->frag);
+        mca_btl_sm_frag_complete(hdr->frag);
         return;
     }
 
     const mca_btl_active_message_callback_t *reg = mca_btl_base_active_message_trigger + hdr->tag;
-    mca_btl_base_segment_t segments[2] = {[0] = {.seg_addr.pval = (void *) (hdr + 1), .seg_len = hdr->len}};
-    mca_btl_base_receive_descriptor_t frag = {.endpoint = endpoint, .des_segments = segments,
-                                              .des_segment_count = 1, .tag = hdr->tag,
+    mca_btl_base_segment_t segments[2] = {
+        [0] = {.seg_addr.pval = (void *) (hdr + 1), .seg_len = hdr->len}};
+    mca_btl_base_receive_descriptor_t frag = {.endpoint = endpoint,
+                                              .des_segments = segments,
+                                              .des_segment_count = 1,
+                                              .tag = hdr->tag,
                                               .cbdata = reg->cbdata};
 
     if (hdr->flags & MCA_BTL_SM_FLAG_SINGLE_COPY) {
-        mca_rcache_base_registration_t *xpmem_reg;
-
-        xpmem_reg = sm_get_registation (endpoint, hdr->sc_iov.iov_base,
-                                        hdr->sc_iov.iov_len, 0,
-                                        &segments[1].seg_addr.pval);
-        assert (NULL != xpmem_reg);
+        void *ctx = MCA_SMSC_CALL(map_peer_region, endpoint->smsc_endpoint,
+                                  MCA_RCACHE_FLAGS_PERSIST, hdr->sc_iov.iov_base,
+                                  hdr->sc_iov.iov_len, &segments[1].seg_addr.pval);
+        assert(NULL != ctx);
 
         segments[1].seg_len = hdr->sc_iov.iov_len;
         frag.des_segment_count = 2;
 
         /* recv upcall */
         reg->cbfunc(&mca_btl_sm.super, &frag);
-        sm_return_registration (xpmem_reg, endpoint);
+        MCA_SMSC_CALL(unmap_peer_region, ctx);
     } else {
         reg->cbfunc(&mca_btl_sm.super, &frag);
     }
 
     if (OPAL_UNLIKELY(MCA_BTL_SM_FLAG_SETUP_FBOX & hdr->flags)) {
-        mca_btl_sm_endpoint_setup_fbox_recv (endpoint, relative2virtual(hdr->fbox_base));
-        mca_btl_sm_component.fbox_in_endpoints[mca_btl_sm_component.num_fbox_in_endpoints++] = endpoint;
+        mca_btl_sm_endpoint_setup_fbox_recv(endpoint, relative2virtual(hdr->fbox_base));
+        mca_btl_sm_component.fbox_in_endpoints[mca_btl_sm_component.num_fbox_in_endpoints++]
+            = endpoint;
     }
 
     hdr->flags = MCA_BTL_SM_FLAG_COMPLETE;
-    sm_fifo_write_back (hdr, endpoint);
+    sm_fifo_write_back(hdr, endpoint);
 }
 
-static int mca_btl_sm_poll_fifo (void)
+static int mca_btl_sm_poll_fifo(void)
 {
     struct mca_btl_base_endpoint_t *endpoint;
     mca_btl_sm_hdr_t *hdr;
 
     /* poll the fifo until it is empty or a limit has been hit (8 is arbitrary) */
-    for (int fifo_count = 0 ; fifo_count < 31 ; ++fifo_count) {
-        hdr = sm_fifo_read (mca_btl_sm_component.my_fifo, &endpoint);
+    for (int fifo_count = 0; fifo_count < 31; ++fifo_count) {
+        hdr = sm_fifo_read(mca_btl_sm_component.my_fifo, &endpoint);
         if (NULL == hdr) {
             return fifo_count;
         }
 
-        mca_btl_sm_poll_handle_frag (hdr, endpoint);
+        mca_btl_sm_poll_handle_frag(hdr, endpoint);
     }
 
     return 1;
@@ -663,9 +495,9 @@ static int mca_btl_sm_poll_fifo (void)
  * @param ep (IN)       Sm BTL endpoint
  *
  * This is called with the component lock held so the component lock does
- * not need to be aquired before modifying the pending_endpoints list.
+ * not need to be acquired before modifying the pending_endpoints list.
  */
-static void mca_btl_sm_progress_waiting (mca_btl_base_endpoint_t *ep)
+static void mca_btl_sm_progress_waiting(mca_btl_base_endpoint_t *ep)
 {
     mca_btl_sm_frag_t *frag, *next;
     int ret = 1;
@@ -675,18 +507,18 @@ static void mca_btl_sm_progress_waiting (mca_btl_base_endpoint_t *ep)
     }
 
     OPAL_THREAD_LOCK(&ep->pending_frags_lock);
-    OPAL_LIST_FOREACH_SAFE(frag, next, &ep->pending_frags, mca_btl_sm_frag_t) {
-        ret = sm_fifo_write_ep (frag->hdr, ep);
+    OPAL_LIST_FOREACH_SAFE (frag, next, &ep->pending_frags, mca_btl_sm_frag_t) {
+        ret = sm_fifo_write_ep(frag->hdr, ep);
         if (!ret) {
             OPAL_THREAD_UNLOCK(&ep->pending_frags_lock);
             return;
         }
 
-        (void) opal_list_remove_first (&ep->pending_frags);
+        (void) opal_list_remove_first(&ep->pending_frags);
     }
 
     ep->waiting = false;
-    opal_list_remove_item (&mca_btl_sm_component.pending_endpoints, &ep->super);
+    opal_list_remove_item(&mca_btl_sm_component.pending_endpoints, &ep->super);
 
     OPAL_THREAD_UNLOCK(&ep->pending_frags_lock);
 }
@@ -696,48 +528,49 @@ static void mca_btl_sm_progress_waiting (mca_btl_base_endpoint_t *ep)
  *
  * @param ep (IN)       Sm BTL endpoint
  */
-static void mca_btl_sm_progress_endpoints (void)
+static void mca_btl_sm_progress_endpoints(void)
 {
     mca_btl_base_endpoint_t *ep, *next;
     int count;
 
-    count = opal_list_get_size (&mca_btl_sm_component.pending_endpoints);
+    count = opal_list_get_size(&mca_btl_sm_component.pending_endpoints);
     if (OPAL_LIKELY(0 == count)) {
         return;
     }
 
     OPAL_THREAD_LOCK(&mca_btl_sm_component.lock);
-    OPAL_LIST_FOREACH_SAFE(ep, next, &mca_btl_sm_component.pending_endpoints, mca_btl_base_endpoint_t) {
-        mca_btl_sm_progress_waiting (ep);
+    OPAL_LIST_FOREACH_SAFE (ep, next, &mca_btl_sm_component.pending_endpoints,
+                            mca_btl_base_endpoint_t) {
+        mca_btl_sm_progress_waiting(ep);
     }
     OPAL_THREAD_UNLOCK(&mca_btl_sm_component.lock);
 }
 
-static int mca_btl_sm_component_progress (void)
+static int mca_btl_sm_component_progress(void)
 {
     static opal_atomic_int32_t lock = 0;
     int count = 0;
 
     if (opal_using_threads()) {
-        if (opal_atomic_swap_32 (&lock, 1)) {
+        if (opal_atomic_swap_32(&lock, 1)) {
             return 0;
         }
     }
 
     /* check for messages in fast boxes */
     if (mca_btl_sm_component.num_fbox_in_endpoints) {
-        count = mca_btl_sm_check_fboxes ();
+        count = mca_btl_sm_check_fboxes();
     }
 
-    mca_btl_sm_progress_endpoints ();
+    mca_btl_sm_progress_endpoints();
 
     if (SM_FIFO_FREE == mca_btl_sm_component.my_fifo->fifo_head) {
         lock = 0;
         return count;
     }
 
-    count += mca_btl_sm_poll_fifo ();
-    opal_atomic_mb ();
+    count += mca_btl_sm_poll_fifo();
+    opal_atomic_mb();
     lock = 0;
 
     return count;
diff --git a/opal/mca/btl/sm/btl_sm_endpoint.h b/opal/mca/btl/sm/btl_sm_endpoint.h
deleted file mode 100644
index 94dcf01d1b9..00000000000
--- a/opal/mca/btl/sm/btl_sm_endpoint.h
+++ /dev/null
@@ -1,127 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2007 Voltaire. All rights reserved.
- * Copyright (c) 2012-2016 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2018      Triad National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2020      Google, LLC. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-/**
- * @file
- */
-
-#ifndef MCA_BTL_SM_ENDPOINT_H
-#define MCA_BTL_SM_ENDPOINT_H
-
-#include "opal_config.h"
-#include "btl_sm_xpmem.h"
-
-#define MCA_BTL_SM_FBOX_ALIGNMENT      32
-#define MCA_BTL_SM_FBOX_ALIGNMENT_MASK (MCA_BTL_SM_FBOX_ALIGNMENT - 1)
-
-struct sm_fifo_t;
-
-/**
- *  An abstraction that represents a connection to a endpoint process.
- *  An instance of mca_ptl_base_endpoint_t is associated w/ each process
- *  and BTL pair at startup.
- */
-
-struct mca_btl_sm_fbox_t;
-
-typedef struct mca_btl_base_endpoint_t {
-    opal_list_item_t super;
-
-    /* per peer buffers */
-    struct {
-        unsigned char *buffer; /**< starting address of peer's fast box out */
-        uint32_t *startp;
-        unsigned int start;
-        uint16_t seq;
-    } fbox_in;
-
-    struct {
-        unsigned char *buffer; /**< starting address of peer's fast box in */
-        uint32_t *startp;      /**< pointer to location storing start offset */
-        unsigned int start, end;
-        uint16_t seq;
-        opal_free_list_item_t *fbox; /**< fast-box free list item */
-    } fbox_out;
-
-    uint16_t peer_smp_rank; /**< my peer's SMP process rank.  Used for accessing
-                             *   SMP specfic data structures. */
-    opal_atomic_size_t send_count;    /**< number of fragments sent to this peer */
-    char *segment_base;     /**< start of the peer's segment (in the address space
-                             *   of this process) */
-
-    struct sm_fifo_t *fifo; /**< */
-
-    opal_mutex_t lock;      /**< lock to protect endpoint structures from concurrent
-                             *   access */
-
-    union {
-#if OPAL_BTL_SM_HAVE_XPMEM
-        struct {
-            xpmem_apid_t    apid;       /**< xpmem apid for remote peer */
-            uintptr_t       address_max; /**< largest address that can be attached */
-        } xpmem;
-#endif
-        struct {
-            pid_t           pid;        /**< pid of remote peer (used for CMA) */
-            opal_shmem_ds_t *seg_ds;    /**< stored segment information for detach */
-        } other;
-    } segment_data;
-
-    opal_mutex_t pending_frags_lock; /**< protect pending_frags */
-    opal_list_t pending_frags; /**< fragments pending fast box space */
-    bool waiting;           /**< endpoint is on the component wait list */
-} mca_btl_base_endpoint_t;
-
-typedef mca_btl_base_endpoint_t mca_btl_sm_endpoint_t;
-
-OBJ_CLASS_DECLARATION(mca_btl_sm_endpoint_t);
-
-static inline void mca_btl_sm_endpoint_setup_fbox_recv (struct mca_btl_base_endpoint_t *endpoint, void *base)
-{
-    endpoint->fbox_in.startp = (uint32_t *) base;
-    endpoint->fbox_in.start = MCA_BTL_SM_FBOX_ALIGNMENT;
-    endpoint->fbox_in.seq = 0;
-    opal_atomic_wmb ();
-    endpoint->fbox_in.buffer = base;
-}
-
-static inline void mca_btl_sm_endpoint_setup_fbox_send (struct mca_btl_base_endpoint_t *endpoint, opal_free_list_item_t *fbox)
-{
-    void *base = fbox->ptr;
-
-    endpoint->fbox_out.start = MCA_BTL_SM_FBOX_ALIGNMENT;
-    endpoint->fbox_out.end = MCA_BTL_SM_FBOX_ALIGNMENT;
-    endpoint->fbox_out.startp = (uint32_t *) base;
-    endpoint->fbox_out.startp[0] = MCA_BTL_SM_FBOX_ALIGNMENT;
-    endpoint->fbox_out.seq = 0;
-    endpoint->fbox_out.fbox = fbox;
-
-    /* zero out the first header in the fast box */
-    memset ((char *) base + MCA_BTL_SM_FBOX_ALIGNMENT, 0, MCA_BTL_SM_FBOX_ALIGNMENT);
-
-    opal_atomic_wmb ();
-    endpoint->fbox_out.buffer = base;
-}
-
-#endif /* MCA_BTL_SM_ENDPOINT_H */
diff --git a/opal/mca/btl/sm/btl_sm_fbox.h b/opal/mca/btl/sm/btl_sm_fbox.h
index 981852ece09..b81bb951950 100644
--- a/opal/mca/btl/sm/btl_sm_fbox.h
+++ b/opal/mca/btl/sm/btl_sm_fbox.h
@@ -4,7 +4,7 @@
  *                         reserved.
  * Copyright (c) 2018      Triad National Security, LLC. All rights
  *                         reserved.
- * Copyright (c) 2020      Google, LLC. All rights reserveed.
+ * Copyright (c) 2020-2025 Google, LLC. All rights reserveed.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -12,12 +12,16 @@
  * $HEADER$
  */
 
-#if !defined(MCA_BTL_SM_FBOX_H)
+#ifndef MCA_BTL_SM_FBOX_H
 #define MCA_BTL_SM_FBOX_H
 
-#include "btl_sm.h"
+#include "opal/mca/btl/sm/btl_sm_types.h"
+#include "opal/mca/btl/sm/btl_sm_virtual.h"
+#include "opal/util/minmax.h"
 
-#define MCA_BTL_SM_POLL_COUNT 31
+#define MCA_BTL_SM_POLL_COUNT          31
+#define MCA_BTL_SM_FBOX_ALIGNMENT      32
+#define MCA_BTL_SM_FBOX_ALIGNMENT_MASK (MCA_BTL_SM_FBOX_ALIGNMENT - 1)
 
 typedef union mca_btl_sm_fbox_hdr_t {
     struct {
@@ -26,11 +30,11 @@ typedef union mca_btl_sm_fbox_hdr_t {
          * and the tag is never read before seq put them in the same 32-bits of the
          * header. */
         /** message size */
-        uint32_t  size;
+        uint32_t size;
         /** message tag */
-        uint16_t  tag;
+        uint16_t tag;
         /** sequence number */
-        uint16_t  seq;
+        uint16_t seq;
     } data;
     struct {
         uint32_t value0;
@@ -39,251 +43,311 @@ typedef union mca_btl_sm_fbox_hdr_t {
     uint64_t ival;
 } mca_btl_sm_fbox_hdr_t;
 
-#define MCA_BTL_SM_FBOX_HDR(x) ((mca_btl_sm_fbox_hdr_t *) (x))
+/**
+ *  An abstraction that represents a connection to a endpoint process.
+ *  An instance of mca_ptl_base_endpoint_t is associated w/ each process
+ *  and BTL pair at startup.
+ */
+
+static inline void mca_btl_sm_endpoint_setup_fbox_recv(struct mca_btl_base_endpoint_t *endpoint,
+                                                       void *base)
+{
+    endpoint->fbox_in.metadata = (mca_btl_sm_fbox_metadata_t *) base;
+    endpoint->fbox_in.start = endpoint->fbox_in.metadata->start;
+    endpoint->fbox_in.seq = 0;
+    endpoint->fbox_in.buffer = (unsigned char *)(endpoint->fbox_in.metadata + 1);
+}
+
+static inline void mca_btl_sm_endpoint_setup_fbox_send(struct mca_btl_base_endpoint_t *endpoint,
+                                                       opal_free_list_item_t *fbox)
+{
+    void *base = fbox->ptr;
 
-#define MCA_BTL_SM_FBOX_OFFSET_MASK 0x7fffffff
-#define MCA_BTL_SM_FBOX_HB_MASK     0x80000000
+    endpoint->fbox_out.start = 0;
+    endpoint->fbox_out.end = 0;
+    endpoint->fbox_out.seq = 0;
+    endpoint->fbox_out.fbox = fbox;
+
+    endpoint->fbox_out.metadata = (mca_btl_sm_fbox_metadata_t *) base;
+    endpoint->fbox_out.metadata->start = 0;
+
+    endpoint->fbox_out.buffer = (unsigned char *)(endpoint->fbox_out.metadata + 1);
+
+    /* zero out the first header in the fast box */
+    ((mca_btl_sm_fbox_hdr_t *)endpoint->fbox_out.buffer)->ival = 0;
+
+    opal_atomic_wmb();
+}
+
+#define MCA_BTL_SM_FBOX_HDR(x) ((mca_btl_sm_fbox_hdr_t *) (x))
 
-/* if the two offsets are equal and the high bit matches the buffer is empty else the buffer is full.
- * note that start will never be end - 1 so this simplified conditional will always produce the correct
- * result */
-#define BUFFER_FREE(s,e,hbm,size) (((s + !hbm) > (e)) ? (s) - (e) : (size - (e)))
+static inline unsigned int mca_btl_sm_fbox_out_free(mca_btl_sm_fbox_out_t *fbox_out, unsigned int fbox_size) {
+    unsigned int fbox_offset_mask = fbox_size - 1;
+    unsigned int masked_end = fbox_out->end & fbox_offset_mask;
+    unsigned int free;
+
+    /* start pointer will always trail the end pointer. check for rollover of the end pointer where
+     * the start pointer has not yet rolled over. */
+    if (OPAL_UNLIKELY(fbox_out->end <= fbox_size && fbox_out->start > fbox_size)) {
+        unsigned int masked_start = fbox_out->start & fbox_offset_mask;
+        free = masked_start - masked_end;
+    } else {
+        free = fbox_size - (fbox_out->end - fbox_out->start);
+    }
 
-/** macro for checking if the high bit is set */
-#define MCA_BTL_SM_FBOX_OFFSET_HBS(v) (!!((v) & MCA_BTL_SM_FBOX_HB_MASK))
+    return opal_min(free, fbox_size - masked_end);
+}
 
-void mca_btl_sm_poll_handle_frag (mca_btl_sm_hdr_t *hdr, mca_btl_base_endpoint_t *ep);
+void mca_btl_sm_poll_handle_frag(mca_btl_sm_hdr_t *hdr, mca_btl_base_endpoint_t *endpoint);
 
-static inline void mca_btl_sm_fbox_set_header (mca_btl_sm_fbox_hdr_t *hdr, uint16_t tag,
-                                               uint16_t seq, uint32_t size)
+static inline void mca_btl_sm_fbox_set_header(mca_btl_sm_fbox_hdr_t *hdr, uint16_t tag,
+                                              uint16_t seq, uint32_t size)
 {
     mca_btl_sm_fbox_hdr_t tmp = {.data = {.tag = tag, .seq = seq, .size = size}};
-    /* clear out existing tag/seq */
-    hdr->data_i32.value1 = 0;
-    opal_atomic_wmb ();
-    hdr->data_i32.value0 = size;
-    opal_atomic_wmb ();
-    hdr->data_i32.value1 = tmp.data_i32.value1;
+    opal_atomic_wmb();
+    hdr->ival = tmp.ival;
 }
 
-static inline mca_btl_sm_fbox_hdr_t mca_btl_sm_fbox_read_header (mca_btl_sm_fbox_hdr_t *hdr)
+static inline mca_btl_sm_fbox_hdr_t mca_btl_sm_fbox_read_header(mca_btl_sm_fbox_hdr_t *hdr)
 {
-    mca_btl_sm_fbox_hdr_t tmp = {.data_i32 = {.value1 = hdr->data_i32.value1}};;
-    opal_atomic_rmb ();
+    mca_btl_sm_fbox_hdr_t tmp = {.data_i32 = {.value1 = hdr->data_i32.value1}};
+
+    opal_atomic_rmb();
     tmp.data_i32.value0 = hdr->data_i32.value0;
     return tmp;
 }
 
-/* attempt to reserve a contiguous segment from the remote ep */
-static inline bool mca_btl_sm_fbox_sendi (mca_btl_base_endpoint_t *ep, unsigned char tag,
-                                          void * restrict header, const size_t header_size,
-                                          void * restrict payload, const size_t payload_size)
-{
+static inline unsigned int mca_btl_sm_fbox_align(unsigned int size) {
+    return (size + MCA_BTL_SM_FBOX_ALIGNMENT_MASK) & ~MCA_BTL_SM_FBOX_ALIGNMENT_MASK;
+}
+
+static inline unsigned char *mca_btl_sm_fbox_reserve_locked(mca_btl_base_endpoint_t *ep, unsigned int data_size) {
     const unsigned int fbox_size = mca_btl_sm_component.fbox_size;
-    size_t size = header_size + payload_size;
-    unsigned int start, end, buffer_free;
-    size_t data_size = size;
-    unsigned char *dst, *data;
-    bool hbs, hbm;
-
-    /* don't try to use the per-peer buffer for messages that will fill up more than 25% of the buffer */
-    if (OPAL_UNLIKELY(NULL == ep->fbox_out.buffer || size > (fbox_size >> 2))) {
-        return false;
+    const unsigned int fbox_offset_mask = fbox_size - 1;
+    unsigned int buffer_free;
+    unsigned char *dst;
+    size_t aligned_entry_size;
+
+    /* don't try to use the per-peer buffer for messages that will fill up more than 25% of the
+     * buffer */
+    if (OPAL_UNLIKELY(NULL == ep->fbox_out.buffer || data_size > (fbox_size >> 2))) {
+        return NULL;
     }
 
-    OPAL_THREAD_LOCK(&ep->lock);
-
-    /* the high bit helps determine if the buffer is empty or full */
-    hbs = MCA_BTL_SM_FBOX_OFFSET_HBS(ep->fbox_out.end);
-    hbm = MCA_BTL_SM_FBOX_OFFSET_HBS(ep->fbox_out.start) == hbs;
+    assert ((fbox_size & fbox_offset_mask) == 0);
 
-    /* read current start and end offsets and check for free space */
-    start = ep->fbox_out.start & MCA_BTL_SM_FBOX_OFFSET_MASK;
-    end = ep->fbox_out.end & MCA_BTL_SM_FBOX_OFFSET_MASK;
-    buffer_free = BUFFER_FREE(start, end, hbm, fbox_size);
+    buffer_free = mca_btl_sm_fbox_out_free(&ep->fbox_out, fbox_size);
 
     /* need space for the fragment + the header */
-    size = (size + sizeof (mca_btl_sm_fbox_hdr_t) + MCA_BTL_SM_FBOX_ALIGNMENT_MASK) & ~MCA_BTL_SM_FBOX_ALIGNMENT_MASK;
+    aligned_entry_size = mca_btl_sm_fbox_align(data_size + sizeof(mca_btl_sm_fbox_hdr_t));
 
-    dst = ep->fbox_out.buffer + end;
+    dst = ep->fbox_out.buffer + (ep->fbox_out.end & fbox_offset_mask);
 
-    if (OPAL_UNLIKELY(buffer_free < size)) {
+    if (OPAL_UNLIKELY(buffer_free < aligned_entry_size)) {
         /* check if we need to free up space for this fragment */
-        BTL_VERBOSE(("not enough room for a fragment of size %u. in use buffer segment: {start: %x, end: %x, high bit matches: %d}",
-                     (unsigned) size, start, end, (int) hbm));
+        BTL_VERBOSE(("not enough room for a fragment of size %u. in use buffer segment: {start: "
+                     "%x, end: %x}",
+                     (unsigned) aligned_entry_size, ep->fbox_out.start, ep->fbox_out.end));
+
+        /* read the current start pointer from the remote peer and recalculate the available buffer
+         * space */
+        ep->fbox_out.start = ep->fbox_out.metadata->start;
+        opal_atomic_rmb();
+
+        buffer_free = mca_btl_sm_fbox_out_free(&ep->fbox_out, fbox_size);
+
+        /* if this is the end of the buffer and the fragment doesn't fit then mark the remaining
+         * buffer space to be skipped and check if the fragment can be written at the beginning of
+         * the buffer. */
+        if (OPAL_UNLIKELY(buffer_free > 0 && buffer_free < aligned_entry_size &&
+                          ((ep->fbox_out.end + buffer_free) & fbox_offset_mask) == 0)) {
+#if OPAL_ENABLE_DEBUG
+            unsigned int old_end = ep->fbox_out.end;
+#endif
+            unsigned int remaining = buffer_free;
+
+            BTL_VERBOSE(("space needed for message: %" PRIsize_t ", remaining space in buffer: "
+                         "%u, checking for space at beginning of buffer",
+                         aligned_entry_size, remaining));
+
+            ep->fbox_out.end += remaining;
+            buffer_free = mca_btl_sm_fbox_out_free(&ep->fbox_out, fbox_size);
+            if (OPAL_UNLIKELY(buffer_free < aligned_entry_size)) {
+                /* not writing the skip token so give this space back */
+                ep->fbox_out.end -= remaining;
+                return NULL;
+            }
 
-        /* read the current start pointer from the remote peer and recalculate the available buffer space */
-        start = ep->fbox_out.start = ep->fbox_out.startp[0];
+            MCA_BTL_SM_FBOX_HDR(ep->fbox_out.buffer)->ival = 0;
+            opal_atomic_wmb();
 
-        /* recalculate how much buffer space is available */
-        start &= MCA_BTL_SM_FBOX_OFFSET_MASK;
-        hbm = MCA_BTL_SM_FBOX_OFFSET_HBS(ep->fbox_out.start) == hbs;
-        buffer_free = BUFFER_FREE(start, end, hbm, fbox_size);
+            BTL_VERBOSE(("writing a skip token at offset %u", old_end));
+            /* space is available. go ahead and mark remaining space to skip */
+            mca_btl_sm_fbox_set_header(MCA_BTL_SM_FBOX_HDR(dst), 0xff, ep->fbox_out.seq++,
+                                       remaining - sizeof(mca_btl_sm_fbox_hdr_t));
+            dst = ep->fbox_out.buffer;
+        }
+
+        if (buffer_free < aligned_entry_size) {
+            return NULL;
+        }
+    }
 
-        opal_atomic_rmb ();
+    BTL_VERBOSE(("writing fragment of size %u {start: 0x%x, end: 0x%x} of "
+                 "peer's buffer. free = %u",
+                 (unsigned int) aligned_entry_size, ep->fbox_out.start, ep->fbox_out.end, buffer_free));
 
-        /* if this is the end of the buffer and the fragment doesn't fit then mark the remaining buffer space to
-         * be skipped and check if the fragment can be written at the beginning of the buffer. */
-        if (OPAL_UNLIKELY(buffer_free > 0 && buffer_free < size && start <= end)) {
-            BTL_VERBOSE(("message will not fit in remaining buffer space. skipping to beginning"));
+    ep->fbox_out.end += aligned_entry_size;
+    
+    /* zero-out the next */
+    if (buffer_free > aligned_entry_size) {
+        MCA_BTL_SM_FBOX_HDR(ep->fbox_out.buffer + (ep->fbox_out.end & fbox_offset_mask))->ival = 0;
+        opal_atomic_wmb();
+    }
 
-            mca_btl_sm_fbox_set_header (MCA_BTL_SM_FBOX_HDR(dst), 0xff, ep->fbox_out.seq++,
-                                        buffer_free - sizeof (mca_btl_sm_fbox_hdr_t));
+    return dst;
+}
 
-            end = MCA_BTL_SM_FBOX_ALIGNMENT;
-            /* toggle the high bit */
-            hbs = !hbs;
-            /* toggle the high bit match */
-            buffer_free = BUFFER_FREE(start, end, !hbm, fbox_size);
-            dst = ep->fbox_out.buffer + end;
-        }
+/* attempt to reserve a contiguous segment from the remote ep */
+static inline bool mca_btl_sm_fbox_sendi(mca_btl_base_endpoint_t *ep, unsigned char tag,
+                                         void *restrict header, const size_t header_size,
+                                         void *restrict payload, const size_t payload_size)
+{
+    size_t data_size = header_size + payload_size;
 
-        if (OPAL_UNLIKELY(buffer_free < size)) {
-            ep->fbox_out.end = (hbs << 31) | end;
-            opal_atomic_wmb ();
-            OPAL_THREAD_UNLOCK(&ep->lock);
-            return false;
-        }
+    OPAL_THREAD_LOCK(&ep->lock);
+    unsigned char *dst = mca_btl_sm_fbox_reserve_locked(ep, (unsigned int) data_size);
+    OPAL_THREAD_UNLOCK(&ep->lock);
+    if (OPAL_UNLIKELY(NULL == dst)) {
+        return false;
     }
 
-    BTL_VERBOSE(("writing fragment of size %u to offset %u {start: 0x%x, end: 0x%x (hbs: %d)} of peer's buffer. free = %u",
-                 (unsigned int) size, end, start, end, hbs, buffer_free));
-
-    data = dst + sizeof (mca_btl_sm_fbox_hdr_t);
+    unsigned char *data = dst + sizeof(mca_btl_sm_fbox_hdr_t);
 
-    memcpy (data, header, header_size);
+    memcpy(data, header, header_size);
     if (payload) {
         /* inline sends are typically just pml headers (due to MCA_BTL_FLAGS_SEND_INPLACE) */
-        memcpy (data + header_size, payload, payload_size);
+        memcpy(data + header_size, payload, payload_size);
     }
 
-    end += size;
+    opal_atomic_wmb();
+    /* write out part of the header now. the tag will be written when the data is available */
+    mca_btl_sm_fbox_set_header(MCA_BTL_SM_FBOX_HDR(dst), tag, ep->fbox_out.seq++,
+                               (uint32_t) data_size);
+
+    return true;
+}
+
+static inline bool mca_btl_sm_poll_fbox(mca_btl_base_endpoint_t *ep)
+{
+    const unsigned int fbox_offset_mask = mca_btl_sm_component.fbox_size - 1;
+    unsigned int start_offset = ep->fbox_in.start & fbox_offset_mask;
+    const mca_btl_sm_fbox_hdr_t hdr = mca_btl_sm_fbox_read_header(
+        MCA_BTL_SM_FBOX_HDR(ep->fbox_in.buffer + start_offset));
 
-    if (OPAL_UNLIKELY(fbox_size == end)) {
-        /* toggle the high bit */
-        hbs = !hbs;
-        /* reset the end pointer to the beginning of the buffer */
-        end = MCA_BTL_SM_FBOX_ALIGNMENT;
-    } else if (buffer_free > size) {
-        MCA_BTL_SM_FBOX_HDR(ep->fbox_out.buffer + end)->ival = 0;
+    /* check for a valid tag a sequence number */
+    if (0 == hdr.data.tag || hdr.data.seq != ep->fbox_in.seq) {
+        return false;
     }
 
-    /* write out part of the header now. the tag will be written when the data is available */
-    mca_btl_sm_fbox_set_header (MCA_BTL_SM_FBOX_HDR(dst), tag, ep->fbox_out.seq++, data_size);
+    ++ep->fbox_in.seq;
+
+    /* force all prior reads to complete before continuing */
+    opal_atomic_rmb();
+
+    BTL_VERBOSE(
+        ("got frag from %d with header {.tag = %d, .size = %d, .seq = %u} from offset %u",
+         ep->peer_smp_rank, hdr.data.tag, hdr.data.size, hdr.data.seq, start_offset));
+
+    /* the 0xff tag indicates we should skip the rest of the buffer */
+    if (OPAL_LIKELY((0xfe & hdr.data.tag) != 0xfe)) {
+        mca_btl_base_segment_t segment;
+        const mca_btl_active_message_callback_t *reg = mca_btl_base_active_message_trigger
+            + hdr.data.tag;
+        mca_btl_base_receive_descriptor_t desc = {.endpoint = ep,
+                                                  .des_segments = &segment,
+                                                  .des_segment_count = 1,
+                                                  .tag = hdr.data.tag,
+                                                  .cbdata = reg->cbdata};
+
+        /* fragment fits entirely in the remaining buffer space. some
+         * btl users do not handle fragmented data so we can't split
+         * the fragment without introducing another copy here. this
+         * limitation has not appeared to cause any performance
+         * degradation. */
+        segment.seg_len = hdr.data.size;
+        segment.seg_addr.pval = (void *) (ep->fbox_in.buffer + start_offset + sizeof(hdr));
+
+        /* call the registered callback function */
+        reg->cbfunc(&mca_btl_sm.super, &desc);
+    } else if (OPAL_LIKELY(0xfe == hdr.data.tag)) {
+        /* process fragment header */
+        fifo_value_t *value = (fifo_value_t *) (ep->fbox_in.buffer + start_offset + sizeof(hdr));
+        mca_btl_sm_hdr_t *sm_hdr = relative2virtual(*value);
+        mca_btl_sm_poll_handle_frag(sm_hdr, ep);
+    }
 
-    /* align the buffer */
-    ep->fbox_out.end = ((uint32_t) hbs << 31) | end;
-    opal_atomic_wmb ();
-    OPAL_THREAD_UNLOCK(&ep->lock);
+    ep->fbox_in.start += mca_btl_sm_fbox_align(hdr.data.size + sizeof(hdr));
 
     return true;
 }
 
-static inline bool mca_btl_sm_check_fboxes (void)
+static inline int mca_btl_sm_check_fboxes(void)
 {
-    const unsigned int fbox_size = mca_btl_sm_component.fbox_size;
-    bool processed = false;
+    int total_processed = 0;
 
-    for (unsigned int i = 0 ; i < mca_btl_sm_component.num_fbox_in_endpoints ; ++i) {
+    for (unsigned int i = 0; i < mca_btl_sm_component.num_fbox_in_endpoints; ++i) {
         mca_btl_base_endpoint_t *ep = mca_btl_sm_component.fbox_in_endpoints[i];
-        unsigned int start = ep->fbox_in.start & MCA_BTL_SM_FBOX_OFFSET_MASK;
-
-        /* save the current high bit state */
-        bool hbs = MCA_BTL_SM_FBOX_OFFSET_HBS(ep->fbox_in.start);
-        int poll_count;
 
-        for (poll_count = 0 ; poll_count <= MCA_BTL_SM_POLL_COUNT ; ++poll_count) {
-            const mca_btl_sm_fbox_hdr_t hdr = mca_btl_sm_fbox_read_header (MCA_BTL_SM_FBOX_HDR(ep->fbox_in.buffer + start));
-
-            /* check for a valid tag a sequence number */
-            if (0 == hdr.data.tag || hdr.data.seq != ep->fbox_in.seq) {
+        int frag_count = 0;
+        for (int j = 0 ; j < MCA_BTL_SM_POLL_COUNT ; ++j) {
+            if (!mca_btl_sm_poll_fbox(ep)) {
                 break;
             }
-
-            ++ep->fbox_in.seq;
-
-            /* force all prior reads to complete before continuing */
-            opal_atomic_rmb ();
-
-            BTL_VERBOSE(("got frag from %d with header {.tag = %d, .size = %d, .seq = %u} from offset %u",
-                         ep->peer_smp_rank, hdr.data.tag, hdr.data.size, hdr.data.seq, start));
-
-            /* the 0xff tag indicates we should skip the rest of the buffer */
-            if (OPAL_LIKELY((0xfe & hdr.data.tag) != 0xfe)) {
-                mca_btl_base_segment_t segment;
-                const mca_btl_active_message_callback_t *reg =
-                    mca_btl_base_active_message_trigger + hdr.data.tag;
-                mca_btl_base_receive_descriptor_t desc = {.endpoint = ep, .des_segments = &segment,
-                                                          .des_segment_count = 1, .tag = hdr.data.tag,
-                                                          .cbdata = reg->cbdata};
-
-                /* fragment fits entirely in the remaining buffer space. some
-                 * btl users do not handle fragmented data so we can't split
-                 * the fragment without introducing another copy here. this
-                 * limitation has not appeared to cause any performance
-                 * degradation. */
-                segment.seg_len = hdr.data.size;
-                segment.seg_addr.pval = (void *) (ep->fbox_in.buffer + start + sizeof (hdr));
-
-                /* call the registered callback function */
-                reg->cbfunc(&mca_btl_sm.super, &desc);
-            } else if (OPAL_LIKELY(0xfe == hdr.data.tag)) {
-                /* process fragment header */
-                fifo_value_t *value = (fifo_value_t *)(ep->fbox_in.buffer + start + sizeof (hdr));
-                mca_btl_sm_hdr_t *hdr = relative2virtual(*value);
-                mca_btl_sm_poll_handle_frag (hdr, ep);
-            }
-
-            start = (start + hdr.data.size + sizeof (hdr) + MCA_BTL_SM_FBOX_ALIGNMENT_MASK) & ~MCA_BTL_SM_FBOX_ALIGNMENT_MASK;
-            if (OPAL_UNLIKELY(fbox_size == start)) {
-                /* jump to the beginning of the buffer */
-                start = MCA_BTL_SM_FBOX_ALIGNMENT;
-                /* toggle the high bit */
-                hbs = !hbs;
-            }
+            ++frag_count;
         }
 
-        if (poll_count) {
-            BTL_VERBOSE(("left off at offset %u (hbs: %d)", start, hbs));
+        if (frag_count) {
+            BTL_VERBOSE(("finished processing at offset %x", ep->fbox_in.start));
 
-            /* save where we left off */
             /* let the sender know where we stopped */
-            opal_atomic_mb ();
-            ep->fbox_in.start = ep->fbox_in.startp[0] = ((uint32_t) hbs << 31) | start;
-            processed = true;
+            opal_atomic_mb();
+            ep->fbox_in.metadata->start = ep->fbox_in.start;
+            total_processed += frag_count;
         }
     }
 
-    /* return the number of fragments processed */
-    return processed;
+    return total_processed;
 }
 
-static inline void mca_btl_sm_try_fbox_setup (mca_btl_base_endpoint_t *ep, mca_btl_sm_hdr_t *hdr)
+static inline void mca_btl_sm_try_fbox_setup(mca_btl_base_endpoint_t *ep, mca_btl_sm_hdr_t *hdr)
 {
-    if (OPAL_UNLIKELY(NULL == ep->fbox_out.buffer && mca_btl_sm_component.fbox_threshold == OPAL_THREAD_ADD_FETCH_SIZE_T (&ep->send_count, 1))) {
+    if (OPAL_UNLIKELY(NULL == ep->fbox_out.buffer
+                      && mca_btl_sm_component.fbox_threshold
+                             == OPAL_THREAD_ADD_FETCH_SIZE_T(&ep->send_count, 1))) {
         /* protect access to mca_btl_sm_component.segment_offset */
         OPAL_THREAD_LOCK(&mca_btl_sm_component.lock);
 
         /* verify the remote side will accept another fbox */
-        if (0 <= opal_atomic_add_fetch_32 (&ep->fifo->fbox_available, -1)) {
-            opal_free_list_item_t *fbox = opal_free_list_get (&mca_btl_sm_component.sm_fboxes);
+        if (0 <= opal_atomic_add_fetch_32(&ep->fifo->fbox_available, -1)) {
+            opal_free_list_item_t *fbox = opal_free_list_get(&mca_btl_sm_component.sm_fboxes);
 
             if (NULL != fbox) {
                 /* zero out the fast box */
-                memset (fbox->ptr, 0, mca_btl_sm_component.fbox_size);
-                mca_btl_sm_endpoint_setup_fbox_send (ep, fbox);
+                memset(fbox->ptr, 0, mca_btl_sm_component.fbox_size);
+                mca_btl_sm_endpoint_setup_fbox_send(ep, fbox);
 
                 hdr->flags |= MCA_BTL_SM_FLAG_SETUP_FBOX;
-                hdr->fbox_base = virtual2relative((char *) ep->fbox_out.buffer);
+                hdr->fbox_base = virtual2relative((char *) ep->fbox_out.metadata);
             } else {
-                opal_atomic_add_fetch_32 (&ep->fifo->fbox_available, 1);
+                opal_atomic_add_fetch_32(&ep->fifo->fbox_available, 1);
             }
 
-            opal_atomic_wmb ();
+            opal_atomic_wmb();
         }
 
         OPAL_THREAD_UNLOCK(&mca_btl_sm_component.lock);
     }
 }
 
-#endif /* !defined(MCA_BTL_SM_FBOX_H) */
+#endif /* MCA_BTL_SM_FBOX_H */
diff --git a/opal/mca/btl/sm/btl_sm_fifo.h b/opal/mca/btl/sm/btl_sm_fifo.h
index 3156d31024b..36f497a5dd9 100644
--- a/opal/mca/btl/sm/btl_sm_fifo.h
+++ b/opal/mca/btl/sm/btl_sm_fifo.h
@@ -27,30 +27,17 @@
 #ifndef MCA_BTL_SM_FIFO_H
 #define MCA_BTL_SM_FIFO_H
 
-#include "btl_sm.h"
-#include "btl_sm_endpoint.h"
-#include "btl_sm_frag.h"
+#include "opal_config.h"
 
-#define sm_item_compare_exchange(x, y, z) opal_atomic_compare_exchange_strong_ptr ((opal_atomic_intptr_t *) (x), (intptr_t *) (y), (intptr_t) (z))
+#include "opal/mca/btl/sm/btl_sm_fbox.h"
+#include "opal/mca/btl/sm/btl_sm_types.h"
+#include "opal/mca/btl/sm/btl_sm_virtual.h"
 
-#if SIZEOF_VOID_P == 8
-#define sm_item_swap(x, y)      opal_atomic_swap_64((opal_atomic_int64_t *)(x), (int64_t)(y))
+#define sm_item_compare_exchange(x, y, z)                                                   \
+    opal_atomic_compare_exchange_strong_ptr((opal_atomic_intptr_t *) (x), (intptr_t *) (y), \
+                                            (intptr_t)(z))
 
-#define MCA_BTL_SM_OFFSET_MASK 0xffffffffll
-#define MCA_BTL_SM_OFFSET_BITS 32
-#define MCA_BTL_SM_BITNESS     64
-#else
-#define sm_item_swap(x, y)      opal_atomic_swap_32((opal_atomic_int32_t *)(x), (int32_t)(y))
-
-#define MCA_BTL_SM_OFFSET_MASK 0x00ffffffl
-#define MCA_BTL_SM_OFFSET_BITS 24
-#define MCA_BTL_SM_BITNESS     32
-#endif
-
-typedef opal_atomic_intptr_t atomic_fifo_value_t;
-typedef intptr_t fifo_value_t;
-
-#define SM_FIFO_FREE  ((fifo_value_t)-2)
+#define SM_FIFO_FREE ((fifo_value_t) -2)
 
 /*
  * Shared Memory FIFOs
@@ -66,43 +53,9 @@ typedef intptr_t fifo_value_t;
  * We introduce some padding at the end of the structure but it is probably unnecessary.
  */
 
-/* lock free fifo */
-typedef struct sm_fifo_t {
-    atomic_fifo_value_t fifo_head;
-    atomic_fifo_value_t fifo_tail;
-    opal_atomic_int32_t fbox_available;
-} sm_fifo_t;
-
 /* large enough to ensure the fifo is on its own cache line */
 #define MCA_BTL_SM_FIFO_SIZE 128
 
-/***
- * One or more FIFO components may be a pointer that must be
- * accessed by multiple processes.  Since the shared region may
- * be mmapped differently into each process's address space,
- * these pointers will be relative to some base address.  Here,
- * we define inline functions to translate between relative
- * addresses and virtual addresses.
- */
-
-/* This only works for finding the relative address for a pointer within my_segment */
-static inline fifo_value_t virtual2relative (char *addr)
-{
-    return (fifo_value_t) ((intptr_t) (addr - mca_btl_sm_component.my_segment)) | ((fifo_value_t)MCA_BTL_SM_LOCAL_RANK << MCA_BTL_SM_OFFSET_BITS);
-}
-
-static inline fifo_value_t virtual2relativepeer (struct mca_btl_base_endpoint_t *endpoint, char *addr)
-{
-    return (fifo_value_t) ((intptr_t) (addr - endpoint->segment_base)) | ((fifo_value_t)endpoint->peer_smp_rank << MCA_BTL_SM_OFFSET_BITS);
-}
-
-static inline void *relative2virtual (fifo_value_t offset)
-{
-    return (void *)(intptr_t)((offset & MCA_BTL_SM_OFFSET_MASK) + mca_btl_sm_component.endpoints[offset >> MCA_BTL_SM_OFFSET_BITS].segment_base);
-}
-
-#include "btl_sm_fbox.h"
-
 /**
  * sm_fifo_read:
  *
@@ -115,7 +68,7 @@ static inline void *relative2virtual (fifo_value_t offset)
  *
  * This function does not currently support multiple readers.
  */
-static inline mca_btl_sm_hdr_t *sm_fifo_read (sm_fifo_t *fifo, struct mca_btl_base_endpoint_t **ep)
+static inline mca_btl_sm_hdr_t *sm_fifo_read(sm_fifo_t *fifo, struct mca_btl_base_endpoint_t **ep)
 {
     mca_btl_sm_hdr_t *hdr;
     fifo_value_t value;
@@ -124,23 +77,23 @@ static inline mca_btl_sm_hdr_t *sm_fifo_read (sm_fifo_t *fifo, struct mca_btl_ba
         return NULL;
     }
 
-    opal_atomic_rmb ();
+    opal_atomic_rmb();
 
     value = fifo->fifo_head;
 
     *ep = &mca_btl_sm_component.endpoints[value >> MCA_BTL_SM_OFFSET_BITS];
-    hdr = (mca_btl_sm_hdr_t *) relative2virtual (value);
+    hdr = (mca_btl_sm_hdr_t *) relative2virtual(value);
 
     fifo->fifo_head = SM_FIFO_FREE;
 
-    assert (hdr->next != value);
+    assert(hdr->next != value);
 
     if (OPAL_UNLIKELY(SM_FIFO_FREE == hdr->next)) {
         opal_atomic_rmb();
 
-        if (!sm_item_compare_exchange (&fifo->fifo_tail, &value, SM_FIFO_FREE)) {
+        if (!sm_item_compare_exchange(&fifo->fifo_tail, &value, SM_FIFO_FREE)) {
             while (SM_FIFO_FREE == hdr->next) {
-                opal_atomic_rmb ();
+                opal_atomic_rmb();
             }
 
             fifo->fifo_head = hdr->next;
@@ -149,11 +102,11 @@ static inline mca_btl_sm_hdr_t *sm_fifo_read (sm_fifo_t *fifo, struct mca_btl_ba
         fifo->fifo_head = hdr->next;
     }
 
-    opal_atomic_wmb ();
+    opal_atomic_wmb();
     return hdr;
 }
 
-static inline void sm_fifo_init (sm_fifo_t *fifo)
+static inline void sm_fifo_init(sm_fifo_t *fifo)
 {
     /* due to a compiler bug in Oracle C 5.15 the following line was broken into two. Not
      * ideal but oh well. See #5814 */
@@ -164,24 +117,24 @@ static inline void sm_fifo_init (sm_fifo_t *fifo)
     mca_btl_sm_component.my_fifo = fifo;
 }
 
-static inline void sm_fifo_write (sm_fifo_t *fifo, fifo_value_t value)
+static inline void sm_fifo_write(sm_fifo_t *fifo, fifo_value_t value)
 {
     fifo_value_t prev;
 
-    opal_atomic_wmb ();
-    prev = sm_item_swap (&fifo->fifo_tail, value);
-    opal_atomic_rmb ();
+    opal_atomic_wmb();
+    prev = opal_atomic_swap_ptr(&fifo->fifo_tail, value);
+    opal_atomic_rmb();
 
-    assert (prev != value);
+    assert(prev != value);
 
     if (OPAL_LIKELY(SM_FIFO_FREE != prev)) {
-        mca_btl_sm_hdr_t *hdr = (mca_btl_sm_hdr_t *) relative2virtual (prev);
+        mca_btl_sm_hdr_t *hdr = (mca_btl_sm_hdr_t *) relative2virtual(prev);
         hdr->next = value;
     } else {
         fifo->fifo_head = value;
     }
 
-    opal_atomic_wmb ();
+    opal_atomic_wmb();
 }
 
 /**
@@ -195,18 +148,18 @@ static inline void sm_fifo_write (sm_fifo_t *fifo, fifo_value_t value)
  * This function is used to send a fragment to a remote peer. {hdr} must belong
  * to the current process.
  */
-static inline bool sm_fifo_write_ep (mca_btl_sm_hdr_t *hdr, struct mca_btl_base_endpoint_t *ep)
+static inline bool sm_fifo_write_ep(mca_btl_sm_hdr_t *hdr, struct mca_btl_base_endpoint_t *ep)
 {
-    fifo_value_t rhdr = virtual2relative ((char *) hdr);
+    fifo_value_t rhdr = virtual2relative((char *) hdr);
     if (ep->fbox_out.buffer) {
         /* if there is a fast box for this peer then use the fast box to send the fragment header.
          * this is done to ensure fragment ordering */
-        opal_atomic_wmb ();
-        return mca_btl_sm_fbox_sendi (ep, 0xfe, &rhdr, sizeof (rhdr), NULL, 0);
+        opal_atomic_wmb();
+        return mca_btl_sm_fbox_sendi(ep, 0xfe, &rhdr, sizeof(rhdr), NULL, 0);
     }
-    mca_btl_sm_try_fbox_setup (ep, hdr);
+    mca_btl_sm_try_fbox_setup(ep, hdr);
     hdr->next = SM_FIFO_FREE;
-    sm_fifo_write (ep->fifo, rhdr);
+    sm_fifo_write(ep->fifo, rhdr);
 
     return true;
 }
@@ -219,13 +172,13 @@ static inline bool sm_fifo_write_ep (mca_btl_sm_hdr_t *hdr, struct mca_btl_base_
  * @param[in]  hdr - fragment header to write
  * @param[in]  ep  - endpoint the fragment belongs to
  *
- * This function is used to return a fragment to the sending process. It differs from sm_fifo_write_ep
- * in that it uses the {ep} to produce the relative address.
+ * This function is used to return a fragment to the sending process. It differs from
+ * sm_fifo_write_ep in that it uses the {ep} to produce the relative address.
  */
-static inline void sm_fifo_write_back (mca_btl_sm_hdr_t *hdr, struct mca_btl_base_endpoint_t *ep)
+static inline void sm_fifo_write_back(mca_btl_sm_hdr_t *hdr, struct mca_btl_base_endpoint_t *ep)
 {
     hdr->next = SM_FIFO_FREE;
-    sm_fifo_write(ep->fifo, virtual2relativepeer (ep, (char *) hdr));
+    sm_fifo_write(ep->fifo, virtual2relativepeer(ep, (char *) hdr));
 }
 
 #endif /* MCA_BTL_SM_FIFO_H */
diff --git a/opal/mca/btl/sm/btl_sm_frag.c b/opal/mca/btl/sm/btl_sm_frag.c
index 960495c3a5b..b43f34b4ded 100644
--- a/opal/mca/btl/sm/btl_sm_frag.c
+++ b/opal/mca/btl/sm/btl_sm_frag.c
@@ -28,20 +28,20 @@
 #include "btl_sm.h"
 #include "btl_sm_frag.h"
 
-static inline void mca_btl_sm_frag_constructor (mca_btl_sm_frag_t *frag)
+static inline void mca_btl_sm_frag_constructor(mca_btl_sm_frag_t *frag)
 {
-    frag->hdr = (mca_btl_sm_hdr_t*)frag->base.super.ptr;
-    if(frag->hdr != NULL) {
+    frag->hdr = (mca_btl_sm_hdr_t *) frag->base.super.ptr;
+    if (frag->hdr != NULL) {
         frag->hdr->frag = frag;
         frag->hdr->flags = 0;
-        frag->segments[0].seg_addr.pval = (char *)(frag->hdr + 1);
+        frag->segments[0].seg_addr.pval = (char *) (frag->hdr + 1);
     }
 
-    frag->base.des_segments      = frag->segments;
+    frag->base.des_segments = frag->segments;
     frag->base.des_segment_count = 1;
 }
 
-int mca_btl_sm_frag_init (opal_free_list_item_t *item, void *ctx)
+int mca_btl_sm_frag_init(opal_free_list_item_t *item, void *ctx)
 {
     mca_btl_sm_frag_t *frag = (mca_btl_sm_frag_t *) item;
 
@@ -51,5 +51,4 @@ int mca_btl_sm_frag_init (opal_free_list_item_t *item, void *ctx)
     return OPAL_SUCCESS;
 }
 
-OBJ_CLASS_INSTANCE(mca_btl_sm_frag_t, mca_btl_base_descriptor_t,
-                   mca_btl_sm_frag_constructor, NULL);
+OBJ_CLASS_INSTANCE(mca_btl_sm_frag_t, mca_btl_base_descriptor_t, mca_btl_sm_frag_constructor, NULL);
diff --git a/opal/mca/btl/sm/btl_sm_frag.h b/opal/mca/btl/sm/btl_sm_frag.h
index 2d31be34fbf..7d31fa8ed7d 100644
--- a/opal/mca/btl/sm/btl_sm_frag.h
+++ b/opal/mca/btl/sm/btl_sm_frag.h
@@ -27,125 +27,49 @@
 
 #include "opal_config.h"
 
-enum {
-      MCA_BTL_SM_FLAG_INLINE      = 0,
-      MCA_BTL_SM_FLAG_SINGLE_COPY = 1,
-      MCA_BTL_SM_FLAG_COMPLETE    = 2,
-      MCA_BTL_SM_FLAG_SETUP_FBOX  = 4,
-};
-
-struct mca_btl_sm_frag_t;
-struct mca_btl_sm_fbox_t;
-
-enum mca_btl_sm_sc_emu_type_t {
-                               MCA_BTL_SM_OP_PUT,
-                               MCA_BTL_SM_OP_GET,
-                               MCA_BTL_SM_OP_ATOMIC,
-                               MCA_BTL_SM_OP_CSWAP,
-};
-typedef enum mca_btl_sm_sc_emu_type_t mca_btl_sm_sc_emu_type_t;
-
-struct mca_btl_sm_sc_emu_hdr_t {
-    mca_btl_sm_sc_emu_type_t type;
-    uint64_t addr;
-    mca_btl_base_atomic_op_t op;
-    int flags;
-    int64_t operand[2];
-};
-typedef struct mca_btl_sm_sc_emu_hdr_t mca_btl_sm_sc_emu_hdr_t;
-
-/**
- * FIFO fragment header
- */
-struct mca_btl_sm_hdr_t {
-    /** next item in fifo. many peers may touch this */
-    volatile intptr_t next;
-    /** pointer back the the fragment */
-    struct mca_btl_sm_frag_t *frag;
-    /** tag associated with this fragment (used to lookup callback) */
-    mca_btl_base_tag_t tag;
-    /** sm send flags (inline, complete, setup fbox, etc) */
-    uint8_t flags;
-    /** length of data following this header */
-    int32_t len;
-    /** io vector containing pointer to single-copy data */
-    struct iovec sc_iov;
-    /** if the fragment indicates to setup a fast box the base is stored here */
-    intptr_t fbox_base;
-};
-typedef struct mca_btl_sm_hdr_t mca_btl_sm_hdr_t;
-
-/**
- * shared memory send fragment derived type.
- */
-struct mca_btl_sm_frag_t {
-    /** base object */
-    mca_btl_base_descriptor_t base;
-    /** storage for segment data (max 2) */
-    mca_btl_base_segment_t segments[2];
-    /** endpoint this fragment is active on */
-    struct mca_btl_base_endpoint_t *endpoint;
-    /** fragment header (in the shared memory region) */
-    mca_btl_sm_hdr_t *hdr;
-    /** free list this fragment was allocated within */
-    opal_free_list_t *my_list;
-    /** rdma callback data */
-    struct mca_btl_sm_rdma_cbdata_t {
-        void *local_address;
-        uint64_t remote_address;
-        mca_btl_base_rdma_completion_fn_t cbfunc;
-        void *context;
-        void *cbdata;
-        size_t remaining;
-        size_t sent;
-    } rdma;
-};
-
-typedef struct mca_btl_sm_frag_t mca_btl_sm_frag_t;
-
-static inline int mca_btl_sm_frag_alloc (mca_btl_sm_frag_t **frag, opal_free_list_t *list,
-                                         struct mca_btl_base_endpoint_t *endpoint) {
-    *frag = (mca_btl_sm_frag_t *) opal_free_list_get (list);
-    if (OPAL_LIKELY(NULL != *frag)) {
-        (*frag)->endpoint = endpoint;
+static inline mca_btl_sm_frag_t *mca_btl_sm_frag_alloc(opal_free_list_t *list,
+                                                       struct mca_btl_base_endpoint_t *endpoint)
+{
+    mca_btl_sm_frag_t *frag = (mca_btl_sm_frag_t *) opal_free_list_get(list);
+    if (OPAL_UNLIKELY(NULL == frag)) {
+        return NULL;
     }
 
-    return OPAL_SUCCESS;
+    frag->endpoint = endpoint;
+    return frag;
 }
 
-static inline void mca_btl_sm_frag_return (mca_btl_sm_frag_t *frag)
+static inline void mca_btl_sm_frag_return(mca_btl_sm_frag_t *frag)
 {
     if (frag->hdr) {
         frag->hdr->flags = 0;
     }
 
-    frag->segments[0].seg_addr.pval = (char *)(frag->hdr + 1);
+    frag->segments[0].seg_addr.pval = (char *) (frag->hdr + 1);
     frag->base.des_segment_count = 1;
 
-    opal_free_list_return (frag->my_list, (opal_free_list_item_t *)frag);
+    opal_free_list_return(frag->my_list, (opal_free_list_item_t *) frag);
 }
 
-OBJ_CLASS_DECLARATION(mca_btl_sm_frag_t);
-
-#define MCA_BTL_SM_FRAG_ALLOC_EAGER(frag, endpoint)                     \
-    mca_btl_sm_frag_alloc (&(frag), &mca_btl_sm_component.sm_frags_eager, endpoint)
+#define MCA_BTL_SM_FRAG_ALLOC_EAGER(frag, endpoint) \
+    (frag) = mca_btl_sm_frag_alloc(&mca_btl_sm_component.sm_frags_eager, endpoint)
 
-#define MCA_BTL_SM_FRAG_ALLOC_MAX(frag, endpoint)                       \
-    mca_btl_sm_frag_alloc (&(frag), &mca_btl_sm_component.sm_frags_max_send, endpoint)
+#define MCA_BTL_SM_FRAG_ALLOC_MAX(frag, endpoint) \
+    (frag) = mca_btl_sm_frag_alloc(&mca_btl_sm_component.sm_frags_max_send, endpoint)
 
-#define MCA_BTL_SM_FRAG_ALLOC_USER(frag, endpoint)                      \
-    mca_btl_sm_frag_alloc (&(frag), &mca_btl_sm_component.sm_frags_user, endpoint)
+#define MCA_BTL_SM_FRAG_ALLOC_USER(frag, endpoint) \
+    (frag) = mca_btl_sm_frag_alloc(&mca_btl_sm_component.sm_frags_user, endpoint)
 
 #define MCA_BTL_SM_FRAG_RETURN(frag) mca_btl_sm_frag_return(frag)
 
-
-static inline void mca_btl_sm_frag_complete (mca_btl_sm_frag_t *frag) {
+static inline void mca_btl_sm_frag_complete(mca_btl_sm_frag_t *frag)
+{
     /* save the descriptor flags since the callback is allowed to free the frag */
     int des_flags = frag->base.des_flags;
 
     if (OPAL_UNLIKELY(MCA_BTL_DES_SEND_ALWAYS_CALLBACK & des_flags)) {
         /* completion callback */
-        frag->base.des_cbfunc (&mca_btl_sm.super, frag->endpoint, &frag->base, OPAL_SUCCESS);
+        frag->base.des_cbfunc(&mca_btl_sm.super, frag->endpoint, &frag->base, OPAL_SUCCESS);
     }
 
     if (OPAL_LIKELY(des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP)) {
@@ -153,14 +77,15 @@ static inline void mca_btl_sm_frag_complete (mca_btl_sm_frag_t *frag) {
     }
 }
 
-int mca_btl_sm_frag_init (opal_free_list_item_t *item, void *ctx);
+int mca_btl_sm_frag_init(opal_free_list_item_t *item, void *ctx);
 
-static inline void mca_btl_sm_rdma_frag_advance (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
-                                                 mca_btl_sm_frag_t *frag, int status)
+static inline void mca_btl_sm_rdma_frag_advance(mca_btl_base_module_t *btl,
+                                                mca_btl_base_endpoint_t *endpoint,
+                                                mca_btl_sm_frag_t *frag, int status)
 {
     mca_btl_sm_sc_emu_hdr_t *hdr = (mca_btl_sm_sc_emu_hdr_t *) frag->segments[0].seg_addr.pval;
     mca_btl_base_rdma_completion_fn_t cbfunc = frag->rdma.cbfunc;
-    size_t hdr_size = sizeof (*hdr);
+    size_t hdr_size = sizeof(*hdr);
     size_t len = frag->rdma.sent ? frag->segments[0].seg_len - hdr_size : 0;
     void *context = frag->rdma.context;
     void *cbdata = frag->rdma.cbdata;
@@ -168,9 +93,9 @@ static inline void mca_btl_sm_rdma_frag_advance (mca_btl_base_module_t *btl, mca
 
     if (frag->rdma.sent) {
         if (MCA_BTL_SM_OP_GET == hdr->type) {
-            memcpy (frag->rdma.local_address, data, len);
-        } else if ((MCA_BTL_SM_OP_ATOMIC == hdr->type || MCA_BTL_SM_OP_CSWAP == hdr->type) &&
-                   frag->rdma.local_address) {
+            memcpy(frag->rdma.local_address, data, len);
+        } else if ((MCA_BTL_SM_OP_ATOMIC == hdr->type || MCA_BTL_SM_OP_CSWAP == hdr->type)
+                   && frag->rdma.local_address) {
             if (8 == len) {
                 *((int64_t *) frag->rdma.local_address) = hdr->operand[0];
             } else {
@@ -180,50 +105,53 @@ static inline void mca_btl_sm_rdma_frag_advance (mca_btl_base_module_t *btl, mca
     }
 
     if (frag->rdma.remaining) {
-        size_t packet_size = (frag->rdma.remaining + hdr_size) <= mca_btl_sm.super.btl_max_send_size ?
-            frag->rdma.remaining : mca_btl_sm.super.btl_max_send_size - hdr_size;
+        size_t packet_size = (frag->rdma.remaining + hdr_size) <= mca_btl_sm.super.btl_max_send_size
+                                 ? frag->rdma.remaining
+                                 : mca_btl_sm.super.btl_max_send_size - hdr_size;
 
         /* advance the local and remote pointers */
-        frag->rdma.local_address = (void *)((uintptr_t) frag->rdma.local_address + len);
+        frag->rdma.local_address = (void *) ((uintptr_t) frag->rdma.local_address + len);
         frag->rdma.remote_address += len;
 
         if (MCA_BTL_SM_OP_PUT == hdr->type) {
             /* copy the next block into the fragment buffer */
-            memcpy ((void *) (hdr + 1), frag->rdma.local_address, packet_size);
+            memcpy((void *) (hdr + 1), frag->rdma.local_address, packet_size);
         }
 
         hdr->addr = frag->rdma.remote_address;
         /* clear out the complete flag before sending the fragment again */
         frag->hdr->flags &= ~MCA_BTL_SM_FLAG_COMPLETE;
-        frag->segments[0].seg_len = packet_size + sizeof (*hdr);
+        frag->segments[0].seg_len = packet_size + sizeof(*hdr);
         frag->rdma.sent += packet_size;
         frag->rdma.remaining -= packet_size;
 
         /* send is always successful */
-        (void) mca_btl_sm_send (btl, endpoint, &frag->base, MCA_BTL_TAG_SM);
+        (void) mca_btl_sm_send(btl, endpoint, &frag->base, MCA_BTL_TAG_SM);
         return;
     }
 
     /* return the fragment before calling the callback */
     MCA_BTL_SM_FRAG_RETURN(frag);
-    cbfunc (btl, endpoint, (void *)((uintptr_t) frag->rdma.local_address - frag->rdma.sent), NULL,
-            context, cbdata, status);
+    cbfunc(btl, endpoint, (void *) ((uintptr_t) frag->rdma.local_address - frag->rdma.sent), NULL,
+           context, cbdata, status);
 }
 
 static inline int
-mca_btl_sm_rdma_frag_start (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, int type,
-                            uint64_t operand1, uint64_t operand2, mca_btl_base_atomic_op_t op, int order,
-                            int flags, size_t size, void *local_address, int64_t remote_address,
-                            mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
+mca_btl_sm_rdma_frag_start(mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, int type,
+                           uint64_t operand1, uint64_t operand2, mca_btl_base_atomic_op_t op,
+                           int order, int flags, size_t size, void *local_address,
+                           int64_t remote_address, mca_btl_base_rdma_completion_fn_t cbfunc,
+                           void *cbcontext, void *cbdata)
 {
     mca_btl_sm_sc_emu_hdr_t *hdr;
-    size_t hdr_size = sizeof (*hdr);
-    size_t packet_size = (size + hdr_size) <= mca_btl_sm.super.btl_max_send_size ? size :
-        mca_btl_sm.super.btl_max_send_size - hdr_size;
+    size_t hdr_size = sizeof(*hdr);
+    size_t packet_size = (size + hdr_size) <= mca_btl_sm.super.btl_max_send_size
+                             ? size
+                             : mca_btl_sm.super.btl_max_send_size - hdr_size;
     mca_btl_sm_frag_t *frag;
 
-    frag = (mca_btl_sm_frag_t *) mca_btl_sm_alloc (btl, endpoint, order, packet_size + hdr_size,
-                                                   MCA_BTL_DES_SEND_ALWAYS_CALLBACK);
+    frag = (mca_btl_sm_frag_t *) mca_btl_sm_alloc(btl, endpoint, order, packet_size + hdr_size,
+                                                  MCA_BTL_DES_SEND_ALWAYS_CALLBACK);
     if (OPAL_UNLIKELY(NULL == frag)) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
@@ -246,7 +174,7 @@ mca_btl_sm_rdma_frag_start (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t
     hdr->operand[0] = operand1;
     hdr->operand[1] = operand2;
 
-    mca_btl_sm_rdma_frag_advance (btl, endpoint, frag, OPAL_SUCCESS);
+    mca_btl_sm_rdma_frag_advance(btl, endpoint, frag, OPAL_SUCCESS);
     return OPAL_SUCCESS;
 }
 
diff --git a/opal/mca/btl/sm/btl_sm_get.c b/opal/mca/btl/sm/btl_sm_get.c
index 9bc1e3ae291..10ed71ca165 100644
--- a/opal/mca/btl/sm/btl_sm_get.c
+++ b/opal/mca/btl/sm/btl_sm_get.c
@@ -4,7 +4,7 @@
  *                         reserved.
  * Copyright (c) 2018      Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2019      Google, Inc. All rights reserved.
+ * Copyright (c) 2019-2022 Google, Inc. All rights reserved.
  * Copyright (c) 2020      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
@@ -17,20 +17,9 @@
 
 #include "opal_config.h"
 
-#include "btl_sm.h"
-#include "btl_sm_frag.h"
-#include "btl_sm_endpoint.h"
-#include "btl_sm_xpmem.h"
-
-#if OPAL_BTL_SM_HAVE_CMA
-#include <sys/uio.h>
-
-#if OPAL_CMA_NEED_SYSCALL_DEFS
-#include "opal/sys/cma.h"
-#endif /* OPAL_CMA_NEED_SYSCALL_DEFS */
-
-
-#endif
+#include "opal/mca/btl/sm/btl_sm.h"
+#include "opal/mca/btl/sm/btl_sm_frag.h"
+#include "opal/mca/smsc/smsc.h"
 
 /**
  * Initiate an synchronous get.
@@ -39,128 +28,26 @@
  * @param endpoint (IN)    BTL addressing information
  * @param descriptor (IN)  Description of the data to be transferred
  */
-#if OPAL_BTL_SM_HAVE_XPMEM
-int mca_btl_sm_get_xpmem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
-                             uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                             mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-                             int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
-{
-    mca_rcache_base_registration_t *reg;
-    void *rem_ptr;
-
-    /* silence warning about unused arguments */
-    (void) local_handle;
-    (void) remote_handle;
-
-    reg = sm_get_registation (endpoint, (void *)(intptr_t) remote_address, size, 0, &rem_ptr);
-    if (OPAL_UNLIKELY(NULL == rem_ptr)) {
-        return OPAL_ERROR;
-    }
-
-    sm_memmove (local_address, rem_ptr, size);
-
-    sm_return_registration (reg, endpoint);
-
-    /* always call the callback function */
-    cbfunc (btl, endpoint, local_address, local_handle, cbcontext, cbdata, OPAL_SUCCESS);
-
-    return OPAL_SUCCESS;
-}
-#endif
 
-#if OPAL_BTL_SM_HAVE_CMA
-int mca_btl_sm_get_cma (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
-                           uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                           mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-                           int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
+int mca_btl_sm_get(mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
+                   void *local_address, uint64_t remote_address,
+                   mca_btl_base_registration_handle_t *local_handle,
+                   mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
+                   int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext,
+                   void *cbdata)
 {
-    struct iovec src_iov = {.iov_base = (void *)(intptr_t) remote_address, .iov_len = size};
-    struct iovec dst_iov = {.iov_base = local_address, .iov_len = size};
-    ssize_t ret;
-
-    /*
-     * According to the man page :
-     * "On success, process_vm_readv() returns the number of bytes read and
-     * process_vm_writev() returns the number of bytes written.  This return
-     * value may be less than the total number of requested bytes, if a
-     * partial read/write occurred.  (Partial transfers apply at the
-     * granularity of iovec elements.  These system calls won't perform a
-     * partial transfer that splits a single iovec element.)".
-     * So since we use a single iovec element, the returned size should either
-     * be 0 or size, and the do loop should not be needed here.
-     * We tried on various Linux kernels with size > 2 GB, and surprisingly,
-     * the returned value is always 0x7ffff000 (fwiw, it happens to be the size
-     * of the larger number of pages that fits a signed 32 bits integer).
-     * We do not know whether this is a bug from the kernel, the libc or even
-     * the man page, but for the time being, we do as is process_vm_readv() could
-     * return any value.
-     */
-    do {
-        ret = process_vm_readv (endpoint->segment_data.other.seg_ds->seg_cpid, &dst_iov, 1, &src_iov, 1, 0);
-        if (0 > ret) {
-            if (ESRCH == errno) {
-                BTL_PEER_ERROR(NULL,
-                        ("CMA read %ld, expected %lu, errno = %d\n", (long)ret, (unsigned long)size, errno));
-                return OPAL_ERROR;
-            }
-            BTL_ERROR(("CMA read %ld, expected %lu, errno = %d\n", (long)ret, (unsigned long)size, errno));
-            return OPAL_ERROR;
+    if (!mca_btl_is_self_endpoint(endpoint)) {
+        int ret = MCA_SMSC_CALL(copy_from, endpoint->smsc_endpoint, local_address,
+                                (void *) (intptr_t) remote_address, size, remote_handle);
+        if (OPAL_UNLIKELY(OPAL_SUCCESS != ret)) {
+            return ret;
         }
-        src_iov.iov_base = (void *)((char *)src_iov.iov_base + ret);
-        src_iov.iov_len -= ret;
-        dst_iov.iov_base = (void *)((char *)dst_iov.iov_base + ret);
-        dst_iov.iov_len -= ret;
-    } while (0 < src_iov.iov_len);
-
-    /* always call the callback function */
-    cbfunc (btl, endpoint, local_address, local_handle, cbcontext, cbdata, OPAL_SUCCESS);
-
-    return OPAL_SUCCESS;
-}
-#endif
-
-#if OPAL_BTL_SM_HAVE_KNEM
-int mca_btl_sm_get_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
-                            uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                            mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-                            int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
-{
-    struct knem_cmd_param_iovec recv_iovec;
-    struct knem_cmd_inline_copy icopy;
-
-    /* Fill in the ioctl data fields.  There's no async completion, so
-       we don't need to worry about getting a slot, etc. */
-    recv_iovec.base = (uintptr_t) local_address;
-    recv_iovec.len = size;
-    icopy.local_iovec_array = (uintptr_t) &recv_iovec;
-    icopy.local_iovec_nr    = 1;
-    icopy.remote_cookie     = remote_handle->cookie;
-    icopy.remote_offset     = remote_address - remote_handle->base_addr;
-    icopy.write             = 0;
-    icopy.flags             = 0;
-
-    /* Use the DMA flag if knem supports it *and* the segment length
-     * is greater than the cutoff. Not that if DMA is not supported
-     * or the user specified 0 for knem_dma_min the knem_dma_min was
-     * set to UINT_MAX in mca_btl_sm_knem_init. */
-    if (mca_btl_sm_component.knem_dma_min <= size) {
-        icopy.flags = KNEM_FLAG_DMA;
-    }
-    /* synchronous flags only, no need to specify icopy.async_status_index */
-
-    /* When the ioctl returns, the transfer is done and we can invoke
-       the btl callback and return the frag */
-    if (OPAL_UNLIKELY(0 != ioctl (mca_btl_sm.knem_fd, KNEM_CMD_INLINE_COPY, &icopy))) {
-        return OPAL_ERROR;
-    }
-
-    if (KNEM_STATUS_FAILED == icopy.current_status) {
-        return OPAL_ERROR;
+    } else {
+        memcpy(local_address, (void *)(uintptr_t) remote_address, size);
     }
 
     /* always call the callback function */
-    cbfunc (btl, endpoint, local_address, local_handle, cbcontext, cbdata, OPAL_SUCCESS);
+    cbfunc(btl, endpoint, local_address, local_handle, cbcontext, cbdata, OPAL_SUCCESS);
 
     return OPAL_SUCCESS;
 }
-#endif
diff --git a/opal/mca/btl/sm/btl_sm_knem.c b/opal/mca/btl/sm/btl_sm_knem.c
deleted file mode 100644
index ee22a9cae78..00000000000
--- a/opal/mca/btl/sm/btl_sm_knem.c
+++ /dev/null
@@ -1,202 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "btl_sm.h"
-
-#if OPAL_BTL_SM_HAVE_KNEM
-
-#include <stdio.h>
-#include <fcntl.h>
-#include <sys/stat.h>
-#include <unistd.h>
-
-#include "opal/util/show_help.h"
-
-OBJ_CLASS_INSTANCE(mca_btl_sm_registration_handle_t, mca_rcache_base_registration_t, NULL, NULL);
-
-static int mca_btl_sm_knem_reg (void *reg_data, void *base, size_t size,
-                                   mca_rcache_base_registration_t *reg)
-{
-    mca_btl_sm_registration_handle_t *knem_reg = (mca_btl_sm_registration_handle_t *) reg;
-    struct knem_cmd_create_region knem_cr;
-    struct knem_cmd_param_iovec knem_iov;
-
-    knem_iov.base = (uintptr_t) base;
-    knem_iov.len = size;
-
-    knem_cr.iovec_array = (uintptr_t) &knem_iov;
-    knem_cr.iovec_nr = 1;
-    knem_cr.protection = 0;
-
-    if (reg->access_flags & (MCA_RCACHE_ACCESS_LOCAL_WRITE | MCA_RCACHE_ACCESS_REMOTE_WRITE)) {
-        knem_cr.protection |= PROT_WRITE;
-    }
-
-    if (reg->access_flags & MCA_RCACHE_ACCESS_REMOTE_READ) {
-        knem_cr.protection |= PROT_READ;
-    }
-
-    /* Sm will explicitly destroy this cookie */
-    knem_cr.flags = 0;
-    if (OPAL_UNLIKELY(ioctl(mca_btl_sm.knem_fd, KNEM_CMD_CREATE_REGION, &knem_cr) < 0)) {
-        return OPAL_ERROR;
-    }
-
-    knem_reg->btl_handle.cookie = knem_cr.cookie;
-    knem_reg->btl_handle.base_addr = (intptr_t) base;
-
-    return OPAL_SUCCESS;
-}
-
-static int mca_btl_sm_knem_dereg (void *reg_data, mca_rcache_base_registration_t *reg)
-{
-    mca_btl_sm_registration_handle_t *knem_reg = (mca_btl_sm_registration_handle_t *) reg;
-
-    /* NTH: explicity ignore the return code. Don't care about this cookie anymore anyway. */
-    (void) ioctl(mca_btl_sm.knem_fd, KNEM_CMD_DESTROY_REGION, &knem_reg->btl_handle.cookie);
-
-    return OPAL_SUCCESS;
-}
-
-static mca_btl_base_registration_handle_t *
-mca_btl_sm_register_mem_knem (struct mca_btl_base_module_t* btl,
-                                 struct mca_btl_base_endpoint_t *endpoint,
-                                 void *base, size_t size, uint32_t flags)
-{
-    mca_btl_sm_t *sm_module = (mca_btl_sm_t *) btl;
-    mca_btl_sm_registration_handle_t *reg = NULL;
-    int access_flags = flags & MCA_BTL_REG_FLAG_ACCESS_ANY;
-    int rc;
-
-    rc = sm_module->knem_rcache->rcache_register (sm_module->knem_rcache, base, size, 0,
-                                                     access_flags,
-                                                     (mca_rcache_base_registration_t **) &reg);
-    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-        return NULL;
-    }
-
-    return &reg->btl_handle;
-}
-
-static int
-mca_btl_sm_deregister_mem_knem (struct mca_btl_base_module_t *btl, struct mca_btl_base_registration_handle_t *handle)
-{
-    mca_btl_sm_t *sm_module = (mca_btl_sm_t *) btl;
-    mca_btl_sm_registration_handle_t *reg =
-        (mca_btl_sm_registration_handle_t *)((intptr_t) handle - offsetof (mca_btl_sm_registration_handle_t, btl_handle));
-
-    sm_module->knem_rcache->rcache_deregister (sm_module->knem_rcache, &reg->base);
-
-    return OPAL_SUCCESS;
-}
-
-int mca_btl_sm_knem_init (void)
-{
-    mca_rcache_base_resources_t rcache_resources = {
-        .cache_name = "sm", .reg_data = NULL,
-        .sizeof_reg = sizeof (mca_btl_sm_registration_handle_t),
-        .register_mem = mca_btl_sm_knem_reg,
-        .deregister_mem = mca_btl_sm_knem_dereg
-    };
-    struct knem_cmd_info knem_info;
-    int rc;
-
-    /* Open the knem device.  Try to print a helpful message if we
-       fail to open it. */
-    mca_btl_sm.knem_fd = open("/dev/knem", O_RDWR);
-    if (mca_btl_sm.knem_fd < 0) {
-	if (EACCES == errno) {
-	    struct stat sbuf;
-	    if (0 != stat("/dev/knem", &sbuf)) {
-		sbuf.st_mode = 0;
-	    }
-	    opal_show_help("help-btl-sm.txt", "knem permission denied",
-			   true, opal_process_info.nodename, sbuf.st_mode);
-	} else {
-	    opal_show_help("help-btl-sm.txt", "knem fail open",
-			   true, opal_process_info.nodename, errno,
-			   strerror(errno));
-	}
-
-	return OPAL_ERR_NOT_AVAILABLE;
-    }
-
-    do {
-	/* Check that the ABI if kernel module running is the same
-	 * as what we were compiled against. */
-        memset (&knem_info, 0, sizeof (knem_info));
-	rc = ioctl(mca_btl_sm.knem_fd, KNEM_CMD_GET_INFO, &knem_info);
-	if (rc < 0) {
-	    opal_show_help("help-btl-sm.txt", "knem get ABI fail",
-			   true, opal_process_info.nodename, errno,
-			   strerror(errno));
-	    break;
-	}
-
-	if (KNEM_ABI_VERSION != knem_info.abi) {
-	    opal_show_help("help-btl-sm.txt", "knem ABI mismatch",
-			   true, opal_process_info.nodename, KNEM_ABI_VERSION,
-			   knem_info.abi);
-	    break;
-	}
-
-	if (!(mca_btl_sm_component.knem_dma_min && (knem_info.features & KNEM_FEATURE_DMA))) {
-	    /* disable DMA */
-	    mca_btl_sm_component.knem_dma_min = UINT_MAX;
-	}
-
-	/* TODO: add async support */
-
-	/* knem set up successfully */
-	mca_btl_sm.super.btl_get = mca_btl_sm_get_knem;
-	mca_btl_sm.super.btl_put = mca_btl_sm_put_knem;
-
-        /* knem requires registration */
-        mca_btl_sm.super.btl_register_mem = mca_btl_sm_register_mem_knem;
-        mca_btl_sm.super.btl_deregister_mem = mca_btl_sm_deregister_mem_knem;
-        mca_btl_sm.super.btl_registration_handle_size = sizeof (mca_btl_base_registration_handle_t);
-
-        mca_btl_sm.knem_rcache = mca_rcache_base_module_create ("grdma", NULL,
-                                                                   &rcache_resources);
-        if (NULL == mca_btl_sm.knem_rcache) {
-            return OPAL_ERR_OUT_OF_RESOURCE;
-        }
-
-	return OPAL_SUCCESS;
-    } while (0);
-
-    mca_btl_sm_knem_fini ();
-
-    return OPAL_ERR_NOT_AVAILABLE;;
-}
-
-int mca_btl_sm_knem_fini (void)
-{
-    if (-1 != mca_btl_sm.knem_fd) {
-	close (mca_btl_sm.knem_fd);
-	mca_btl_sm.knem_fd = -1;
-    }
-
-    if (mca_btl_sm.knem_rcache) {
-        (void) mca_rcache_base_module_destroy (mca_btl_sm.knem_rcache);
-        mca_btl_sm.knem_rcache = NULL;
-    }
-
-    return OPAL_SUCCESS;
-}
-
-int mca_btl_sm_knem_progress (void)
-{
-    /* NTH: does nothing until async support is added */
-    return OPAL_SUCCESS;
-}
-
-#endif
diff --git a/opal/mca/btl/sm/btl_sm_knem.h b/opal/mca/btl/sm/btl_sm_knem.h
deleted file mode 100644
index 7f5edfea100..00000000000
--- a/opal/mca/btl/sm/btl_sm_knem.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2014-2015 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#if !defined(BTL_SM_KNEM_H)
-#define BTL_SM_KNEM_H
-
-#if OPAL_BTL_SM_HAVE_KNEM
-
-#include <knem_io.h>
-#include <sys/mman.h>
-
-/* At this time only knem requires a registration of "RDMA" buffers */
-struct mca_btl_base_registration_handle_t {
-    uint64_t cookie;
-    intptr_t base_addr;
-};
-
-struct mca_btl_sm_registration_handle_t {
-    mca_rcache_base_registration_t base;
-    mca_btl_base_registration_handle_t btl_handle;
-};
-typedef struct mca_btl_sm_registration_handle_t mca_btl_sm_registration_handle_t;
-
-int mca_btl_sm_knem_init (void);
-int mca_btl_sm_knem_fini (void);
-int mca_btl_sm_knem_progress (void);
-
-#endif /* OPAL_BTL_SM_HAVE_KNEM */
-
-#endif /* defined(BTL_SM_KNEM_H) */
diff --git a/opal/mca/btl/sm/btl_sm_module.c b/opal/mca/btl/sm/btl_sm_module.c
index 6e53e59b795..3cb69c8d3da 100644
--- a/opal/mca/btl/sm/btl_sm_module.c
+++ b/opal/mca/btl/sm/btl_sm_module.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2011 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2009 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2007 High Performance Computing Center Stuttgart,
@@ -11,7 +11,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2006-2007 Voltaire. All rights reserved.
- * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2009-2022 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2010-2015 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
@@ -19,7 +19,7 @@
  *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2018-2019 Triad National Security, LLC. All rights
  *                         reserved.
- * Copyright (c) 2020-2021 Google, LLC. All rights reserved.
+ * Copyright (c) 2020-2022 Google, LLC. All rights reserved.
  * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
  * $COPYRIGHT$
  *
@@ -29,77 +29,38 @@
  */
 
 #include "opal_config.h"
-#include "opal/util/show_help.h"
 
-#include "btl_sm.h"
-#include "btl_sm_endpoint.h"
-#include "btl_sm_fifo.h"
-#include "btl_sm_fbox.h"
-#include "btl_sm_xpmem.h"
+#include "opal/mca/btl/sm/btl_sm.h"
+#include "opal/mca/btl/sm/btl_sm_fbox.h"
+#include "opal/mca/btl/sm/btl_sm_fifo.h"
+#include "opal/mca/btl/sm/btl_sm_frag.h"
+#include "opal/mca/smsc/smsc.h"
 
 #include <string.h>
 
-static int sm_del_procs (struct mca_btl_base_module_t *btl,
-                            size_t nprocs, struct opal_proc_t **procs,
-                            struct mca_btl_base_endpoint_t **peers);
+static int sm_del_procs(struct mca_btl_base_module_t *btl, size_t nprocs,
+                        struct opal_proc_t **procs, struct mca_btl_base_endpoint_t **peers);
 
-static int sm_register_error_cb (struct mca_btl_base_module_t* btl,
-                                    mca_btl_base_module_error_cb_fn_t cbfunc);
+static int sm_register_error_cb(struct mca_btl_base_module_t *btl,
+                                mca_btl_base_module_error_cb_fn_t cbfunc);
 
-static int sm_finalize (struct mca_btl_base_module_t* btl);
+static int sm_finalize(struct mca_btl_base_module_t *btl);
 
-static struct mca_btl_base_descriptor_t *sm_prepare_src (
-                                                            struct mca_btl_base_module_t *btl,
-                                                            struct mca_btl_base_endpoint_t *endpoint,
-                                                            struct opal_convertor_t *convertor,
-                                                            uint8_t order,
-                                                            size_t reserve,
-                                                            size_t *size,
-                                                            uint32_t flags
-                                                            );
+static struct mca_btl_base_descriptor_t *sm_prepare_src(struct mca_btl_base_module_t *btl,
+                                                        struct mca_btl_base_endpoint_t *endpoint,
+                                                        struct opal_convertor_t *convertor,
+                                                        uint8_t order, size_t reserve, size_t *size,
+                                                        uint32_t flags);
 
-static int sm_add_procs(struct mca_btl_base_module_t* btl,
-                           size_t nprocs, struct opal_proc_t **procs,
-                           struct mca_btl_base_endpoint_t** peers,
-                           struct opal_bitmap_t* reachability);
+static int sm_add_procs(struct mca_btl_base_module_t *btl, size_t nprocs,
+                        struct opal_proc_t **procs, struct mca_btl_base_endpoint_t **peers,
+                        struct opal_bitmap_t *reachability);
 
 mca_btl_sm_t mca_btl_sm = {
-    {
-        &mca_btl_sm_component.super,
-        .btl_add_procs = sm_add_procs,
-        .btl_del_procs = sm_del_procs,
-        .btl_finalize = sm_finalize,
-        .btl_alloc = mca_btl_sm_alloc,
-        .btl_free = mca_btl_sm_free,
-        .btl_prepare_src = sm_prepare_src,
-        .btl_send = mca_btl_sm_send,
-        .btl_sendi = mca_btl_sm_sendi,
-        .btl_dump = mca_btl_base_dump,
-        .btl_register_error = sm_register_error_cb
-    }
-};
-
-/*
- * Exit function copied from btl_usnic_util.c
- *
- * The following comment tells Coverity that this function does not return.
- * See https://scan.coverity.com/tune.
- */
-
-/* coverity[+kill] */
-static void sm_btl_exit(mca_btl_sm_t *btl)
-{
-    if (NULL != btl && NULL != btl->error_cb) {
-        btl->error_cb(&btl->super, MCA_BTL_ERROR_FLAGS_FATAL,
-                      (opal_proc_t*) opal_proc_local_get(),
-                      "The sm BTL is aborting the MPI job (via PML error callback).");
-    }
-
-    /* If the PML error callback returns (or if there wasn't one), just exit. Shrug. */
-    fprintf(stderr, "*** The Open MPI sm BTL is aborting the MPI job (via exit(3)).\n");
-    fflush(stderr);
-    exit(1);
-}
+    {&mca_btl_sm_component.super, .btl_add_procs = sm_add_procs, .btl_del_procs = sm_del_procs,
+     .btl_finalize = sm_finalize, .btl_alloc = mca_btl_sm_alloc, .btl_free = mca_btl_sm_free,
+     .btl_prepare_src = sm_prepare_src, .btl_send = mca_btl_sm_send, .btl_sendi = mca_btl_sm_sendi,
+     .btl_dump = mca_btl_base_dump, .btl_register_error = sm_register_error_cb}};
 
 static int sm_btl_first_time_init(mca_btl_sm_t *sm_btl, int n)
 {
@@ -107,73 +68,73 @@ static int sm_btl_first_time_init(mca_btl_sm_t *sm_btl, int n)
     int rc;
 
     /* generate the endpoints */
-    component->endpoints = (struct mca_btl_base_endpoint_t *) calloc (n + 1, sizeof (struct mca_btl_base_endpoint_t));
+    component->endpoints = (struct mca_btl_base_endpoint_t *)
+        calloc(n + 1, sizeof(struct mca_btl_base_endpoint_t));
     if (NULL == component->endpoints) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
     component->endpoints[n].peer_smp_rank = -1;
 
-    component->fbox_in_endpoints = calloc (n + 1, sizeof (void *));
+    component->fbox_in_endpoints = calloc(n + 1, sizeof(void *));
     if (NULL == component->fbox_in_endpoints) {
         free(component->endpoints);
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
-    component->mpool = mca_mpool_basic_create ((void *) (component->my_segment + MCA_BTL_SM_FIFO_SIZE),
-                                               (unsigned long) (mca_btl_sm_component.segment_size - MCA_BTL_SM_FIFO_SIZE), 64);
+    component->mpool = mca_mpool_basic_create((void *) (component->my_segment
+                                                        + MCA_BTL_SM_FIFO_SIZE),
+                                              (unsigned long) (mca_btl_sm_component.segment_size
+                                                               - MCA_BTL_SM_FIFO_SIZE),
+                                              64);
     if (NULL == component->mpool) {
-        free (component->endpoints);
+        free(component->endpoints);
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
-    rc = opal_free_list_init (&component->sm_fboxes, sizeof (opal_free_list_item_t), 8,
-                              OBJ_CLASS(opal_free_list_item_t), mca_btl_sm_component.fbox_size,
-                              opal_cache_line_size, 0, mca_btl_sm_component.fbox_max, 4,
-                              component->mpool, 0, NULL, NULL, NULL);
+    /* Fast box buffers are prepended with a metadata section. */
+    rc = opal_free_list_init(&component->sm_fboxes, sizeof(opal_free_list_item_t), 8,
+                             OBJ_CLASS(opal_free_list_item_t), mca_btl_sm_component.fbox_size +
+                             sizeof (mca_btl_sm_fbox_metadata_t),
+                             opal_cache_line_size, 0, mca_btl_sm_component.fbox_max, 4,
+                             component->mpool, 0, NULL, NULL, NULL);
     if (OPAL_SUCCESS != rc) {
         return rc;
     }
 
     /* initialize fragment descriptor free lists */
     /* initialize free list for small send and inline fragments */
-    rc = opal_free_list_init (&component->sm_frags_user,
-                              sizeof(mca_btl_sm_frag_t),
-                              opal_cache_line_size, OBJ_CLASS(mca_btl_sm_frag_t),
-                              mca_btl_sm_component.max_inline_send + sizeof (mca_btl_sm_frag_t),
-                              opal_cache_line_size, component->sm_free_list_num,
-                              component->sm_free_list_max,
-                              component->sm_free_list_inc,
-                              component->mpool, 0, NULL, mca_btl_sm_frag_init,
-                              &component->sm_frags_user);
+    rc = opal_free_list_init(&component->sm_frags_user, sizeof(mca_btl_sm_frag_t),
+                             opal_cache_line_size, OBJ_CLASS(mca_btl_sm_frag_t),
+                             mca_btl_sm_component.max_inline_send + sizeof(mca_btl_sm_frag_t),
+                             opal_cache_line_size, component->sm_free_list_num,
+                             component->sm_free_list_max, component->sm_free_list_inc,
+                             component->mpool, 0, NULL, mca_btl_sm_frag_init,
+                             &component->sm_frags_user);
     if (OPAL_SUCCESS != rc) {
         return rc;
     }
 
     /* initialize free list for buffered send fragments */
-    rc = opal_free_list_init (&component->sm_frags_eager,
-                              sizeof (mca_btl_sm_frag_t),
-                              opal_cache_line_size, OBJ_CLASS(mca_btl_sm_frag_t),
-                              mca_btl_sm.super.btl_eager_limit + sizeof (mca_btl_sm_frag_t),
-                              opal_cache_line_size, component->sm_free_list_num,
-                              component->sm_free_list_max,
-                              component->sm_free_list_inc,
-                              component->mpool, 0, NULL, mca_btl_sm_frag_init,
-                              &component->sm_frags_eager);
+    rc = opal_free_list_init(&component->sm_frags_eager, sizeof(mca_btl_sm_frag_t),
+                             opal_cache_line_size, OBJ_CLASS(mca_btl_sm_frag_t),
+                             mca_btl_sm.super.btl_eager_limit + sizeof(mca_btl_sm_frag_t),
+                             opal_cache_line_size, component->sm_free_list_num,
+                             component->sm_free_list_max, component->sm_free_list_inc,
+                             component->mpool, 0, NULL, mca_btl_sm_frag_init,
+                             &component->sm_frags_eager);
     if (OPAL_SUCCESS != rc) {
         return rc;
     }
 
-    if (MCA_BTL_SM_XPMEM != mca_btl_sm_component.single_copy_mechanism) {
+    if (!mca_smsc_base_has_feature(MCA_SMSC_FEATURE_CAN_MAP)) {
         /* initialize free list for buffered send fragments */
-        rc = opal_free_list_init (&component->sm_frags_max_send,
-                                  sizeof (mca_btl_sm_frag_t),
-                                  opal_cache_line_size, OBJ_CLASS(mca_btl_sm_frag_t),
-                                  mca_btl_sm.super.btl_max_send_size + sizeof (mca_btl_sm_frag_t),
-                                  opal_cache_line_size, component->sm_free_list_num,
-                                  component->sm_free_list_max,
-                                  component->sm_free_list_inc,
-                                  component->mpool, 0, NULL, mca_btl_sm_frag_init,
-                                  &component->sm_frags_max_send);
+        rc = opal_free_list_init(&component->sm_frags_max_send, sizeof(mca_btl_sm_frag_t),
+                                 opal_cache_line_size, OBJ_CLASS(mca_btl_sm_frag_t),
+                                 mca_btl_sm.super.btl_max_send_size + sizeof(mca_btl_sm_frag_t),
+                                 opal_cache_line_size, component->sm_free_list_num,
+                                 component->sm_free_list_max, component->sm_free_list_inc,
+                                 component->mpool, 0, NULL, mca_btl_sm_frag_init,
+                                 &component->sm_frags_max_send);
         if (OPAL_SUCCESS != rc) {
             return rc;
         }
@@ -182,27 +143,19 @@ static int sm_btl_first_time_init(mca_btl_sm_t *sm_btl, int n)
     /* set flag indicating btl has been inited */
     sm_btl->btl_inited = true;
 
-#if OPAL_BTL_SM_HAVE_XPMEM
-    if (MCA_BTL_SM_XPMEM == mca_btl_sm_component.single_copy_mechanism) {
-        mca_btl_sm_component.vma_module = mca_rcache_base_vma_module_alloc ();
-    }
-#endif
-
     return OPAL_SUCCESS;
 }
 
-
-static int init_sm_endpoint (struct mca_btl_base_endpoint_t **ep_out, struct opal_proc_t *proc) {
+static int init_sm_endpoint(struct mca_btl_base_endpoint_t **ep_out, struct opal_proc_t *proc)
+{
     mca_btl_sm_component_t *component = &mca_btl_sm_component;
-    union sm_modex_t *modex;
-    ino_t my_user_ns_id;
+    mca_btl_sm_modex_t *modex;
     size_t msg_size;
     int rc;
 
     uint16_t peer_local_rank;
     uint16_t *ptr = &peer_local_rank;
-    OPAL_MODEX_RECV_VALUE(rc, PMIX_LOCAL_RANK, &proc->proc_name,
-                          &ptr, PMIX_UINT16);
+    OPAL_MODEX_RECV_VALUE(rc, PMIX_LOCAL_RANK, &proc->proc_name, &ptr, PMIX_UINT16);
     if (OPAL_SUCCESS != rc) {
         BTL_VERBOSE(("could not read the local rank for peer. rc=%d", rc));
         return rc;
@@ -215,78 +168,40 @@ static int init_sm_endpoint (struct mca_btl_base_endpoint_t **ep_out, struct opa
 
     ep->peer_smp_rank = peer_local_rank;
 
-    if (peer_local_rank != MCA_BTL_SM_LOCAL_RANK) {
-        OPAL_MODEX_RECV_IMMEDIATE(rc, &component->super.btl_version,
-                                  &proc->proc_name, (void **) &modex, &msg_size);
+    if (!mca_btl_is_self_endpoint(ep)) {
+        OPAL_MODEX_RECV_IMMEDIATE(rc, &component->super.btl_version, &proc->proc_name,
+                                  (void **) &modex, &msg_size);
         if (OPAL_SUCCESS != rc) {
             return rc;
         }
 
         /* attach to the remote segment */
-#if OPAL_BTL_SM_HAVE_XPMEM
-        if (MCA_BTL_SM_XPMEM == mca_btl_sm_component.single_copy_mechanism) {
-            /* always use xpmem if it is available */
-            ep->segment_data.xpmem.apid = xpmem_get (modex->xpmem.seg_id, XPMEM_RDWR, XPMEM_PERMIT_MODE, (void *) 0666);
-            ep->segment_data.xpmem.address_max = modex->xpmem.address_max;
-            (void) sm_get_registation (ep, modex->xpmem.segment_base, mca_btl_sm_component.segment_size,
-                                          MCA_RCACHE_FLAGS_PERSIST, (void **) &ep->segment_base);
-        } else {
-#endif
+        ep->smsc_endpoint = NULL;  /* assume no one sided support */
+        if( NULL != mca_smsc ) {
+            ep->smsc_endpoint = MCA_SMSC_CALL(get_endpoint, proc);
+        }
+        if (NULL == ep->smsc_endpoint) {
+            /* disable RDMA */
+            mca_btl_sm.super.btl_get = NULL;
+            mca_btl_sm.super.btl_put = NULL;
+            mca_btl_sm.super.btl_flags &= ~MCA_BTL_FLAGS_RDMA;
+        }
             /* store a copy of the segment information for detach */
-            ep->segment_data.other.seg_ds = malloc (modex->other.seg_ds_size);
-            if (NULL == ep->segment_data.other.seg_ds) {
+            ep->seg_ds = malloc(modex->seg_ds_size);
+            if (NULL == ep->seg_ds) {
                 return OPAL_ERR_OUT_OF_RESOURCE;
             }
 
-            memcpy (ep->segment_data.other.seg_ds, &modex->other.seg_ds, modex->other.seg_ds_size);
+            memcpy(ep->seg_ds, &modex->seg_ds, modex->seg_ds_size);
 
-            ep->segment_base = opal_shmem_segment_attach (ep->segment_data.other.seg_ds);
+            ep->segment_base = opal_shmem_segment_attach(ep->seg_ds);
             if (NULL == ep->segment_base) {
                 return OPAL_ERROR;
             }
 
-            if (MCA_BTL_SM_CMA == mca_btl_sm_component.single_copy_mechanism) {
-                my_user_ns_id = mca_btl_sm_get_user_ns_id();
-                if (my_user_ns_id != modex->other.user_ns_id) {
-                    mca_base_var_source_t source;
-                    int vari;
-                    rc = mca_base_var_find_by_name("btl_sm_single_copy_mechanism", &vari);
-                    if (OPAL_ERROR == rc) {
-                        return OPAL_ERROR;
-                    }
-                    rc = mca_base_var_get_value(vari, NULL, &source, NULL);
-                    if (OPAL_ERROR == rc) {
-                        return OPAL_ERROR;
-                    }
-                    /*
-                     * CMA is not possible as different user namespaces are in use.
-                     * Currently the kernel does not allow * process_vm_{read,write}v()
-                     * for processes running in different user namespaces even if
-                     * all involved user IDs are mapped to the same user ID.
-                     */
-                    if (MCA_BASE_VAR_SOURCE_DEFAULT != source) {
-                        /* If CMA has been explicitly selected we want to error out */
-                        opal_show_help("help-btl-sm.txt", "cma-different-user-namespace-error",
-                                       true, opal_process_info.nodename);
-                        sm_btl_exit(&mca_btl_sm);
-                    }
-                    /*
-                     * If CMA has been selected because it is the default or
-                     * some fallback, this falls back even further.
-                     */
-                    opal_show_help("help-btl-sm.txt", "cma-different-user-namespace-warning",
-                                   true, opal_process_info.nodename);
-                    mca_btl_sm_component.single_copy_mechanism = MCA_BTL_SM_NONE;
-                    mca_btl_sm.super.btl_get = NULL;
-                    mca_btl_sm.super.btl_put = NULL;
-                }
-            }
-#if OPAL_BTL_SM_HAVE_XPMEM
-        }
-#endif
         OBJ_CONSTRUCT(&ep->lock, opal_mutex_t);
 
-        free (modex);
+        free(modex);
     } else {
         /* set up the segment base so we can calculate a virtual to real for local pointers */
         ep->segment_base = component->my_segment;
@@ -297,7 +212,7 @@ static int init_sm_endpoint (struct mca_btl_base_endpoint_t **ep_out, struct opa
     return OPAL_SUCCESS;
 }
 
-static int fini_sm_endpoint (struct mca_btl_base_endpoint_t *ep)
+static int fini_sm_endpoint(struct mca_btl_base_endpoint_t *ep)
 {
     /* check if the endpoint is initialized. avoids a double-destruct */
     if (ep->fifo) {
@@ -321,10 +236,9 @@ static int fini_sm_endpoint (struct mca_btl_base_endpoint_t *ep)
  *
  */
 
-static int sm_add_procs (struct mca_btl_base_module_t* btl,
-                            size_t nprocs, struct opal_proc_t **procs,
-                            struct mca_btl_base_endpoint_t **peers,
-                            opal_bitmap_t *reachability)
+static int sm_add_procs(struct mca_btl_base_module_t *btl, size_t nprocs,
+                        struct opal_proc_t **procs, struct mca_btl_base_endpoint_t **peers,
+                        opal_bitmap_t *reachability)
 {
     mca_btl_sm_t *sm_btl = (mca_btl_sm_t *) btl;
     const opal_proc_t *my_proc;
@@ -342,9 +256,8 @@ static int sm_add_procs (struct mca_btl_base_module_t* btl,
         return OPAL_SUCCESS;
     }
 
-
     if (!sm_btl->btl_inited) {
-        rc = sm_btl_first_time_init (sm_btl, 1 + MCA_BTL_SM_NUM_LOCAL_PEERS);
+        rc = sm_btl_first_time_init(sm_btl, 1 + MCA_BTL_SM_NUM_LOCAL_PEERS);
         if (rc != OPAL_SUCCESS) {
             return rc;
         }
@@ -353,22 +266,22 @@ static int sm_add_procs (struct mca_btl_base_module_t* btl,
     for (int32_t proc = 0; proc < (int32_t) nprocs; ++proc) {
         /* check to see if this proc can be reached via shmem (i.e.,
            if they're on my local host and in my job) */
-        if (procs[proc]->proc_name.jobid != my_proc->proc_name.jobid ||
-            !OPAL_PROC_ON_LOCAL_NODE(procs[proc]->proc_flags)) {
+        if (procs[proc]->proc_name.jobid != my_proc->proc_name.jobid
+            || !OPAL_PROC_ON_LOCAL_NODE(procs[proc]->proc_flags)) {
             peers[proc] = NULL;
             continue;
         }
 
         if (my_proc != procs[proc] && NULL != reachability) {
             /* add this proc to shared memory accessibility list */
-            rc = opal_bitmap_set_bit (reachability, proc);
-            if(OPAL_SUCCESS != rc) {
+            rc = opal_bitmap_set_bit(reachability, proc);
+            if (OPAL_SUCCESS != rc) {
                 return rc;
             }
         }
 
         /* setup endpoint */
-        rc = init_sm_endpoint (peers + proc, procs[proc]);
+        rc = init_sm_endpoint(peers + proc, procs[proc]);
         if (OPAL_SUCCESS != rc) {
             break;
         }
@@ -387,13 +300,12 @@ static int sm_add_procs (struct mca_btl_base_module_t* btl,
  *
  */
 
-static int sm_del_procs(struct mca_btl_base_module_t *btl,
-                           size_t nprocs, struct opal_proc_t **procs,
-                           struct mca_btl_base_endpoint_t **peers)
+static int sm_del_procs(struct mca_btl_base_module_t *btl, size_t nprocs,
+                        struct opal_proc_t **procs, struct mca_btl_base_endpoint_t **peers)
 {
-    for (size_t i = 0 ; i < nprocs ; ++i) {
+    for (size_t i = 0; i < nprocs; ++i) {
         if (peers[i]) {
-            fini_sm_endpoint (peers[i]);
+            fini_sm_endpoint(peers[i]);
             peers[i] = NULL;
         }
     }
@@ -401,7 +313,6 @@ static int sm_del_procs(struct mca_btl_base_module_t *btl,
     return OPAL_SUCCESS;
 }
 
-
 /**
  * MCA->BTL Clean up any resources held by BTL module
  * before the module is unloaded.
@@ -424,33 +335,24 @@ static int sm_finalize(struct mca_btl_base_module_t *btl)
         return OPAL_SUCCESS;
     }
 
-    for (int i = 0 ; i < (int)(1 + MCA_BTL_SM_NUM_LOCAL_PEERS) ; ++i) {
-        fini_sm_endpoint (component->endpoints + i);
+    for (int i = 0; i < (int) (1 + MCA_BTL_SM_NUM_LOCAL_PEERS); ++i) {
+        fini_sm_endpoint(component->endpoints + i);
     }
 
-    free (component->endpoints);
+    free(component->endpoints);
     component->endpoints = NULL;
 
     sm_btl->btl_inited = false;
 
-    free (component->fbox_in_endpoints);
+    free(component->fbox_in_endpoints);
     component->fbox_in_endpoints = NULL;
 
-    if (MCA_BTL_SM_XPMEM != mca_btl_sm_component.single_copy_mechanism) {
-        opal_shmem_unlink (&mca_btl_sm_component.seg_ds);
-        opal_shmem_segment_detach (&mca_btl_sm_component.seg_ds);
-    }
-
-#if OPAL_BTL_SM_HAVE_XPMEM
-    if (NULL != mca_btl_sm_component.vma_module) {
-        OBJ_RELEASE(mca_btl_sm_component.vma_module);
-    }
-#endif
+    opal_shmem_unlink(&mca_btl_sm_component.seg_ds);
+    opal_shmem_segment_detach(&mca_btl_sm_component.seg_ds);
 
     return OPAL_SUCCESS;
 }
 
-
 /**
  * Register a callback function that is called on error..
  *
@@ -458,10 +360,10 @@ static int sm_finalize(struct mca_btl_base_module_t *btl)
  * @param cbfunc (IN)  function to call on error
  * @return             Status indicating if cleanup was successful
  */
-static int sm_register_error_cb(struct mca_btl_base_module_t* btl,
-                                   mca_btl_base_module_error_cb_fn_t cbfunc)
+static int sm_register_error_cb(struct mca_btl_base_module_t *btl,
+                                mca_btl_base_module_error_cb_fn_t cbfunc)
 {
-    ((mca_btl_sm_t *)btl)->error_cb = cbfunc;
+    ((mca_btl_sm_t *) btl)->error_cb = cbfunc;
     return OPAL_SUCCESS;
 }
 
@@ -472,25 +374,25 @@ static int sm_register_error_cb(struct mca_btl_base_module_t* btl,
  * @param size (IN)     Request segment size.
  */
 mca_btl_base_descriptor_t *mca_btl_sm_alloc(struct mca_btl_base_module_t *btl,
-                                               struct mca_btl_base_endpoint_t *endpoint,
-                                               uint8_t order, size_t size, uint32_t flags)
+                                            struct mca_btl_base_endpoint_t *endpoint, uint8_t order,
+                                            size_t size, uint32_t flags)
 {
     mca_btl_sm_frag_t *frag = NULL;
 
     if (size <= (size_t) mca_btl_sm_component.max_inline_send) {
-        (void) MCA_BTL_SM_FRAG_ALLOC_USER(frag, endpoint);
+        MCA_BTL_SM_FRAG_ALLOC_USER(frag, endpoint);
     } else if (size <= mca_btl_sm.super.btl_eager_limit) {
-        (void) MCA_BTL_SM_FRAG_ALLOC_EAGER(frag, endpoint);
-    } else if (MCA_BTL_SM_XPMEM != mca_btl_sm_component.single_copy_mechanism &&
-               size <= mca_btl_sm.super.btl_max_send_size) {
-        (void) MCA_BTL_SM_FRAG_ALLOC_MAX(frag, endpoint);
+        MCA_BTL_SM_FRAG_ALLOC_EAGER(frag, endpoint);
+    } else if (!mca_smsc_base_has_feature(MCA_SMSC_FEATURE_CAN_MAP)
+               && size <= mca_btl_sm.super.btl_max_send_size) {
+        MCA_BTL_SM_FRAG_ALLOC_MAX(frag, endpoint);
     }
 
     if (OPAL_LIKELY(frag != NULL)) {
-        frag->segments[0].seg_len  = size;
+        frag->segments[0].seg_len = size;
 
-        frag->base.des_flags   = flags;
-        frag->base.order       = order;
+        frag->base.des_flags = flags;
+        frag->base.order = order;
     }
 
     return (mca_btl_base_descriptor_t *) frag;
@@ -502,7 +404,7 @@ mca_btl_base_descriptor_t *mca_btl_sm_alloc(struct mca_btl_base_module_t *btl,
  * @param btl (IN)      BTL module
  * @param segment (IN)  Allocated segment.
  */
-int mca_btl_sm_free (struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des)
+int mca_btl_sm_free(struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des)
 {
     MCA_BTL_SM_FRAG_RETURN((mca_btl_sm_frag_t *) des);
 
@@ -514,42 +416,44 @@ int mca_btl_sm_free (struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_
  *
  * @param btl (IN)      BTL module
  */
-static struct mca_btl_base_descriptor_t *sm_prepare_src (struct mca_btl_base_module_t *btl,
-                                                            struct mca_btl_base_endpoint_t *endpoint,
-                                                            struct opal_convertor_t *convertor,
-                                                            uint8_t order, size_t reserve, size_t *size,
-                                                            uint32_t flags)
+static struct mca_btl_base_descriptor_t *sm_prepare_src(struct mca_btl_base_module_t *btl,
+                                                        struct mca_btl_base_endpoint_t *endpoint,
+                                                        struct opal_convertor_t *convertor,
+                                                        uint8_t order, size_t reserve, size_t *size,
+                                                        uint32_t flags)
 {
     const size_t total_size = reserve + *size;
     mca_btl_sm_frag_t *frag;
     void *data_ptr;
     int rc;
 
-    opal_convertor_get_current_pointer (convertor, &data_ptr);
-    assert (NULL != data_ptr);
+    opal_convertor_get_current_pointer(convertor, &data_ptr);
+    assert(NULL != data_ptr);
 
     /* in place send fragment */
-    if (OPAL_UNLIKELY(opal_convertor_need_buffers(convertor))) {
+    if (OPAL_UNLIKELY(opal_convertor_need_buffers(convertor) ||
+                      opal_convertor_on_discrete_device(convertor) ||
+                      (opal_convertor_on_unified_device(convertor) &&
+                       total_size > (size_t) mca_btl_sm_component.max_inline_send))) {
         uint32_t iov_count = 1;
         struct iovec iov;
 
         /* non-contiguous data requires the convertor */
-        if (MCA_BTL_SM_XPMEM != mca_btl_sm_component.single_copy_mechanism &&
-            total_size > mca_btl_sm.super.btl_eager_limit) {
-            (void) MCA_BTL_SM_FRAG_ALLOC_MAX(frag, endpoint);
-        } else
-            (void) MCA_BTL_SM_FRAG_ALLOC_EAGER(frag, endpoint);
+        if (!mca_smsc_base_has_feature(MCA_SMSC_FEATURE_CAN_MAP)
+            && total_size > mca_btl_sm.super.btl_eager_limit) {
+            MCA_BTL_SM_FRAG_ALLOC_MAX(frag, endpoint);
+        } else {
+            MCA_BTL_SM_FRAG_ALLOC_EAGER(frag, endpoint);
+        }
 
         if (OPAL_UNLIKELY(NULL == frag)) {
             return NULL;
         }
 
         iov.iov_len = *size;
-        iov.iov_base =
-            (IOVBASE_TYPE *)(((uintptr_t)(frag->segments[0].seg_addr.pval)) +
-                             reserve);
+        iov.iov_base = (IOVBASE_TYPE *) (((uintptr_t)(frag->segments[0].seg_addr.pval)) + reserve);
 
-        rc = opal_convertor_pack (convertor, &iov, &iov_count, size);
+        rc = opal_convertor_pack(convertor, &iov, &iov_count, size);
         if (OPAL_UNLIKELY(rc < 0)) {
             MCA_BTL_SM_FRAG_RETURN(frag);
             return NULL;
@@ -557,51 +461,49 @@ static struct mca_btl_base_descriptor_t *sm_prepare_src (struct mca_btl_base_mod
 
         frag->segments[0].seg_len = *size + reserve;
     } else {
-        if (MCA_BTL_SM_XPMEM != mca_btl_sm_component.single_copy_mechanism) {
+        if (!mca_smsc_base_has_feature(MCA_SMSC_FEATURE_CAN_MAP)) {
             if (OPAL_LIKELY(total_size <= mca_btl_sm.super.btl_eager_limit)) {
-                (void) MCA_BTL_SM_FRAG_ALLOC_EAGER(frag, endpoint);
+                MCA_BTL_SM_FRAG_ALLOC_EAGER(frag, endpoint);
             } else {
-                (void) MCA_BTL_SM_FRAG_ALLOC_MAX(frag, endpoint);
+                MCA_BTL_SM_FRAG_ALLOC_MAX(frag, endpoint);
             }
-        } else
-            (void) MCA_BTL_SM_FRAG_ALLOC_USER(frag, endpoint);
+        } else {
+            MCA_BTL_SM_FRAG_ALLOC_USER(frag, endpoint);
+        }
 
         if (OPAL_UNLIKELY(NULL == frag)) {
             return NULL;
         }
 
-#if OPAL_BTL_SM_HAVE_XPMEM
-        /* use xpmem to send this segment if it is above the max inline send size */
-        if (OPAL_UNLIKELY(MCA_BTL_SM_XPMEM == mca_btl_sm_component.single_copy_mechanism &&
-                          total_size > (size_t) mca_btl_sm_component.max_inline_send)) {
+        /* use single-copy to send this segment if it is above the max inline send size */
+        if (mca_smsc_base_has_feature(MCA_SMSC_FEATURE_CAN_MAP)
+            && total_size > (size_t) mca_btl_sm_component.max_inline_send) {
             /* single copy send */
             frag->hdr->flags = MCA_BTL_SM_FLAG_SINGLE_COPY;
 
             /* set up single copy io vector */
             frag->hdr->sc_iov.iov_base = data_ptr;
-            frag->hdr->sc_iov.iov_len  = *size;
+            frag->hdr->sc_iov.iov_len = *size;
 
             frag->segments[0].seg_len = reserve;
             frag->segments[1].seg_len = *size;
             frag->segments[1].seg_addr.pval = data_ptr;
             frag->base.des_segment_count = 2;
         } else {
-#endif
             /* NTH: the covertor adds some latency so we bypass it here */
-            memcpy ((void *)((uintptr_t)frag->segments[0].seg_addr.pval + reserve), data_ptr, *size);
+            memcpy((void *) ((uintptr_t) frag->segments[0].seg_addr.pval + reserve), data_ptr,
+                   *size);
             frag->segments[0].seg_len = total_size;
-#if OPAL_BTL_SM_HAVE_XPMEM
         }
-#endif
     }
 
-    frag->base.order       = order;
-    frag->base.des_flags   = flags;
+    frag->base.order = order;
+    frag->base.des_flags = flags;
 
     return &frag->base;
 }
 
-static void mca_btl_sm_endpoint_constructor (mca_btl_sm_endpoint_t *ep)
+static void mca_btl_sm_endpoint_constructor(mca_btl_sm_endpoint_t *ep)
 {
     OBJ_CONSTRUCT(&ep->pending_frags, opal_list_t);
     OBJ_CONSTRUCT(&ep->pending_frags_lock, opal_mutex_t);
@@ -609,34 +511,32 @@ static void mca_btl_sm_endpoint_constructor (mca_btl_sm_endpoint_t *ep)
     ep->fbox_out.fbox = NULL;
 }
 
-#if OPAL_BTL_SM_HAVE_XPMEM
-#endif
-
-static void mca_btl_sm_endpoint_destructor (mca_btl_sm_endpoint_t *ep)
+static void mca_btl_sm_endpoint_destructor(mca_btl_sm_endpoint_t *ep)
 {
     OBJ_DESTRUCT(&ep->pending_frags);
     OBJ_DESTRUCT(&ep->pending_frags_lock);
 
-#if OPAL_BTL_SM_HAVE_XPMEM
-    if (MCA_BTL_SM_XPMEM == mca_btl_sm_component.single_copy_mechanism) {
-        mca_btl_sm_xpmem_cleanup_endpoint (ep);
-    } else
-#endif
-    if (ep->segment_data.other.seg_ds) {
+    if (ep->seg_ds) {
         opal_shmem_ds_t seg_ds;
 
         /* opal_shmem_segment_detach expects a opal_shmem_ds_t and will
          * stomp past the end of the seg_ds if it is too small (which
          * ep->seg_ds probably is) */
-        memcpy (&seg_ds, ep->segment_data.other.seg_ds, opal_shmem_sizeof_shmem_ds (ep->segment_data.other.seg_ds));
-        free (ep->segment_data.other.seg_ds);
-        ep->segment_data.other.seg_ds = NULL;
+        memcpy(&seg_ds, ep->seg_ds, opal_shmem_sizeof_shmem_ds(ep->seg_ds));
+        free(ep->seg_ds);
+        ep->seg_ds = NULL;
 
         /* disconnect from the peer's segment */
-        opal_shmem_segment_detach (&seg_ds);
+        opal_shmem_segment_detach(&seg_ds);
     }
+
     if (ep->fbox_out.fbox) {
-        opal_free_list_return (&mca_btl_sm_component.sm_fboxes, ep->fbox_out.fbox);
+        opal_free_list_return(&mca_btl_sm_component.sm_fboxes, ep->fbox_out.fbox);
+    }
+
+    if (ep->smsc_endpoint) {
+        MCA_SMSC_CALL(return_endpoint, ep->smsc_endpoint);
+        ep->smsc_endpoint = NULL;
     }
 
     ep->fbox_in.buffer = ep->fbox_out.buffer = NULL;
@@ -645,4 +545,5 @@ static void mca_btl_sm_endpoint_destructor (mca_btl_sm_endpoint_t *ep)
     ep->fifo = NULL;
 }
 
-OBJ_CLASS_INSTANCE(mca_btl_sm_endpoint_t, opal_list_item_t, mca_btl_sm_endpoint_constructor, mca_btl_sm_endpoint_destructor);
+OBJ_CLASS_INSTANCE(mca_btl_sm_endpoint_t, opal_list_item_t, mca_btl_sm_endpoint_constructor,
+                   mca_btl_sm_endpoint_destructor);
diff --git a/opal/mca/btl/sm/btl_sm_put.c b/opal/mca/btl/sm/btl_sm_put.c
index de72fe38b65..6b7588ea622 100644
--- a/opal/mca/btl/sm/btl_sm_put.c
+++ b/opal/mca/btl/sm/btl_sm_put.c
@@ -4,7 +4,7 @@
  *                         reserved.
  * Copyright (c) 2014-2018 Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2019      Google, Inc. All rights reserved.
+ * Copyright (c) 2019-2022 Google, Inc. All rights reserved.
  * Copyright (c) 2020      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
@@ -17,19 +17,9 @@
 
 #include "opal_config.h"
 
-#include "btl_sm.h"
-#include "btl_sm_frag.h"
-#include "btl_sm_endpoint.h"
-#include "btl_sm_xpmem.h"
-
-#if OPAL_BTL_SM_HAVE_CMA
-#include <sys/uio.h>
-
-#if OPAL_CMA_NEED_SYSCALL_DEFS
-#include "opal/sys/cma.h"
-#endif /* OPAL_CMA_NEED_SYSCALL_DEFS */
-
-#endif
+#include "opal/mca/btl/sm/btl_sm.h"
+#include "opal/mca/btl/sm/btl_sm_frag.h"
+#include "opal/mca/smsc/smsc.h"
 
 /**
  * Initiate an synchronous put.
@@ -38,108 +28,26 @@
  * @param endpoint (IN)    BTL addressing information
  * @param descriptor (IN)  Description of the data to be transferred
  */
-#if OPAL_BTL_SM_HAVE_XPMEM
-int mca_btl_sm_put_xpmem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
-                             uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                             mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-                             int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
-{
-    mca_rcache_base_registration_t *reg;
-    void *rem_ptr;
-
-    reg = sm_get_registation (endpoint, (void *)(intptr_t) remote_address, size, 0, &rem_ptr);
-    if (OPAL_UNLIKELY(NULL == reg)) {
-        return OPAL_ERROR;
-    }
-
-    sm_memmove (rem_ptr, local_address, size);
 
-    sm_return_registration (reg, endpoint);
-
-    /* always call the callback function */
-    cbfunc (btl, endpoint, local_address, local_handle, cbcontext, cbdata, OPAL_SUCCESS);
-
-    return OPAL_SUCCESS;
-}
-#endif
-
-#if OPAL_BTL_SM_HAVE_CMA
-int mca_btl_sm_put_cma (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
-                           uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                           mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-                           int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
+int mca_btl_sm_put(mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
+                   void *local_address, uint64_t remote_address,
+                   mca_btl_base_registration_handle_t *local_handle,
+                   mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
+                   int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext,
+                   void *cbdata)
 {
-    struct iovec src_iov = {.iov_base = local_address, .iov_len = size};
-    struct iovec dst_iov = {.iov_base = (void *)(intptr_t) remote_address, .iov_len = size};
-    ssize_t ret;
-
-    /* This should not be needed, see the rationale in mca_btl_sm_get_cma() */
-    do {
-        ret = process_vm_writev (endpoint->segment_data.other.seg_ds->seg_cpid, &src_iov, 1, &dst_iov, 1, 0);
-        if (0 > ret) {
-            if (ESRCH == errno) {
-                BTL_PEER_ERROR(NULL,
-                        ("CMA wrote %ld, expected %lu, errno = %d\n", (long)ret, (unsigned long)size, errno));
-                return OPAL_ERROR;
-            }
-            BTL_ERROR(("CMA wrote %ld, expected %lu, errno = %d\n", (long)ret, (unsigned long)size, errno));
-            return OPAL_ERROR;
+    if (!mca_btl_is_self_endpoint(endpoint)) {
+        int ret = MCA_SMSC_CALL(copy_to, endpoint->smsc_endpoint, local_address,
+                                (void *) (intptr_t) remote_address, size, remote_handle);
+        if (OPAL_UNLIKELY(OPAL_SUCCESS != ret)) {
+            return ret;
         }
-        src_iov.iov_base = (void *)((char *)src_iov.iov_base + ret);
-        src_iov.iov_len -= ret;
-        dst_iov.iov_base = (void *)((char *)dst_iov.iov_base + ret);
-        dst_iov.iov_len -= ret;
-    } while (0 < src_iov.iov_len);
-
-    /* always call the callback function */
-    cbfunc (btl, endpoint, local_address, local_handle, cbcontext, cbdata, OPAL_SUCCESS);
-
-    return OPAL_SUCCESS;
-}
-#endif
-
-#if OPAL_BTL_SM_HAVE_KNEM
-int mca_btl_sm_put_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
-                            uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                            mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-                            int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
-{
-    struct knem_cmd_param_iovec send_iovec;
-    struct knem_cmd_inline_copy icopy;
-
-    /* Fill in the ioctl data fields.  There's no async completion, so
-       we don't need to worry about getting a slot, etc. */
-    send_iovec.base = (uintptr_t) local_address;
-    send_iovec.len = size;
-    icopy.local_iovec_array = (uintptr_t) &send_iovec;
-    icopy.local_iovec_nr    = 1;
-    icopy.remote_cookie     = remote_handle->cookie;
-    icopy.remote_offset     = remote_address - remote_handle->base_addr;
-    icopy.write             = 1;
-    icopy.flags             = 0;
-
-    /* Use the DMA flag if knem supports it *and* the segment length
-     * is greater than the cutoff. Not that if DMA is not supported
-     * or the user specified 0 for knem_dma_min the knem_dma_min was
-     * set to UINT_MAX in mca_btl_sm_knem_init. */
-    if (mca_btl_sm_component.knem_dma_min <= size) {
-        icopy.flags = KNEM_FLAG_DMA;
-    }
-    /* synchronous flags only, no need to specify icopy.async_status_index */
-
-    /* When the ioctl returns, the transfer is done and we can invoke
-       the btl callback and return the frag */
-    if (OPAL_UNLIKELY(0 != ioctl (mca_btl_sm.knem_fd, KNEM_CMD_INLINE_COPY, &icopy))) {
-        return OPAL_ERROR;
-    }
-
-    if (KNEM_STATUS_FAILED == icopy.current_status) {
-        return OPAL_ERROR;
+    } else {
+        memcpy((void *)(uintptr_t) remote_address, local_address, size);
     }
 
     /* always call the callback function */
-    cbfunc (btl, endpoint, local_address, local_handle, cbcontext, cbdata, OPAL_SUCCESS);
+    cbfunc(btl, endpoint, local_address, local_handle, cbcontext, cbdata, OPAL_SUCCESS);
 
     return OPAL_SUCCESS;
 }
-#endif
diff --git a/opal/mca/btl/sm/btl_sm_send.c b/opal/mca/btl/sm/btl_sm_send.c
index 8beff20a1fb..34ac9295d7b 100644
--- a/opal/mca/btl/sm/btl_sm_send.c
+++ b/opal/mca/btl/sm/btl_sm_send.c
@@ -24,10 +24,10 @@
 
 #include "opal_config.h"
 
-#include "btl_sm.h"
-#include "btl_sm_frag.h"
-#include "btl_sm_fifo.h"
-#include "btl_sm_fbox.h"
+#include "opal/mca/btl/sm/btl_sm.h"
+#include "opal/mca/btl/sm/btl_sm_fbox.h"
+#include "opal/mca/btl/sm/btl_sm_fifo.h"
+#include "opal/mca/btl/sm/btl_sm_frag.h"
 
 /**
  * Initiate a send to the peer.
@@ -35,10 +35,8 @@
  * @param btl (IN)      BTL module
  * @param peer (IN)     BTL peer addressing
  */
-int mca_btl_sm_send (struct mca_btl_base_module_t *btl,
-                        struct mca_btl_base_endpoint_t *endpoint,
-                        struct mca_btl_base_descriptor_t *descriptor,
-                        mca_btl_base_tag_t tag)
+int mca_btl_sm_send(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+                    struct mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag)
 {
     mca_btl_sm_frag_t *frag = (mca_btl_sm_frag_t *) descriptor;
     const size_t total_size = frag->segments[0].seg_len;
@@ -58,15 +56,15 @@ int mca_btl_sm_send (struct mca_btl_base_module_t *btl,
     frag->hdr->flags &= ~MCA_BTL_SM_FLAG_COMPLETE;
 
     /* post the relative address of the descriptor into the peer's fifo */
-    if (opal_list_get_size (&endpoint->pending_frags) || !sm_fifo_write_ep (frag->hdr, endpoint)) {
+    if (opal_list_get_size(&endpoint->pending_frags) || !sm_fifo_write_ep(frag->hdr, endpoint)) {
         if (frag->base.des_cbfunc) {
             frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
         }
         OPAL_THREAD_LOCK(&endpoint->pending_frags_lock);
-        opal_list_append (&endpoint->pending_frags, (opal_list_item_t *) frag);
+        opal_list_append(&endpoint->pending_frags, (opal_list_item_t *) frag);
         if (!endpoint->waiting) {
             OPAL_THREAD_LOCK(&mca_btl_sm_component.lock);
-            opal_list_append (&mca_btl_sm_component.pending_endpoints, &endpoint->super);
+            opal_list_append(&mca_btl_sm_component.pending_endpoints, &endpoint->super);
             OPAL_THREAD_UNLOCK(&mca_btl_sm_component.lock);
             endpoint->waiting = true;
         }
@@ -75,18 +73,4 @@ int mca_btl_sm_send (struct mca_btl_base_module_t *btl,
     }
 
     return OPAL_SUCCESS;
-
-#if 0
-    if (((frag->hdr->flags & MCA_BTL_SM_FLAG_SINGLE_COPY) ||
-        !(frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP)) &&
-        frag->base.des_cbfunc) {
-        frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
-
-        return OPAL_SUCCESS;
-    }
-
-    /* data is gone (from the pml's perspective). frag callback/release will
-       happen later */
-    return 1;
-#endif
 }
diff --git a/opal/mca/btl/sm/btl_sm_sendi.c b/opal/mca/btl/sm/btl_sm_sendi.c
index a0cea566744..ed05405f295 100644
--- a/opal/mca/btl/sm/btl_sm_sendi.c
+++ b/opal/mca/btl/sm/btl_sm_sendi.c
@@ -25,11 +25,11 @@
 
 #include "opal_config.h"
 
-#include "btl_sm.h"
-#include "btl_sm_frag.h"
-#include "btl_sm_fifo.h"
+#include "opal/mca/btl/sm/btl_sm.h"
+#include "opal/mca/btl/sm/btl_sm_fifo.h"
+#include "opal/mca/btl/sm/btl_sm_frag.h"
 
-#include "btl_sm_fbox.h"
+#include "opal/mca/btl/sm/btl_sm_fbox.h"
 
 /**
  * Initiate an inline send to the peer.
@@ -37,20 +37,17 @@
  * @param btl (IN)      BTL module
  * @param peer (IN)     BTL peer addressing
  */
-int mca_btl_sm_sendi (struct mca_btl_base_module_t *btl,
-                         struct mca_btl_base_endpoint_t *endpoint,
-                         struct opal_convertor_t *convertor,
-                         void *header, size_t header_size,
-                         size_t payload_size, uint8_t order,
-                         uint32_t flags, mca_btl_base_tag_t tag,
-                         mca_btl_base_descriptor_t **descriptor)
+int mca_btl_sm_sendi(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+                     struct opal_convertor_t *convertor, void *header, size_t header_size,
+                     size_t payload_size, uint8_t order, uint32_t flags, mca_btl_base_tag_t tag,
+                     mca_btl_base_descriptor_t **descriptor)
 {
     mca_btl_sm_frag_t *frag;
     void *data_ptr = NULL;
     size_t length;
 
     /* don't attempt sendi if there are pending fragments on the endpoint */
-    if (OPAL_UNLIKELY(opal_list_get_size (&endpoint->pending_frags))) {
+    if (OPAL_UNLIKELY(opal_list_get_size(&endpoint->pending_frags))) {
         if (descriptor) {
             *descriptor = NULL;
         }
@@ -59,19 +56,19 @@ int mca_btl_sm_sendi (struct mca_btl_base_module_t *btl,
     }
 
     if (payload_size) {
-        opal_convertor_get_current_pointer (convertor, &data_ptr);
+        opal_convertor_get_current_pointer(convertor, &data_ptr);
     }
 
-    if (!(payload_size && opal_convertor_need_buffers (convertor)) &&
-        mca_btl_sm_fbox_sendi (endpoint, tag, header, header_size, data_ptr, payload_size)) {
+    if (!(payload_size && (opal_convertor_need_buffers(convertor) || opal_convertor_on_device(convertor)))
+        && mca_btl_sm_fbox_sendi(endpoint, tag, header, header_size, data_ptr, payload_size)) {
         return OPAL_SUCCESS;
     }
 
     length = header_size + payload_size;
 
     /* allocate a fragment, giving up if we can't get one */
-    frag = (mca_btl_sm_frag_t *) mca_btl_sm_alloc (btl, endpoint, order, length,
-                                                         flags | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
+    frag = (mca_btl_sm_frag_t *) mca_btl_sm_alloc(btl, endpoint, order, length,
+                                                  flags | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
     if (OPAL_UNLIKELY(NULL == frag)) {
         if (descriptor) {
             *descriptor = NULL;
@@ -85,7 +82,7 @@ int mca_btl_sm_sendi (struct mca_btl_base_module_t *btl,
     frag->hdr->tag = tag;
 
     /* write the match header (with MPI comm/tag/etc. info) */
-    memcpy (frag->segments[0].seg_addr.pval, header, header_size);
+    memcpy(frag->segments[0].seg_addr.pval, header, header_size);
 
     /* write the message data if there is any */
     /* we can't use single-copy semantics here since as caller will consider the send
@@ -95,20 +92,21 @@ int mca_btl_sm_sendi (struct mca_btl_base_module_t *btl,
         struct iovec iov;
 
         /* pack the data into the supplied buffer */
-        iov.iov_base = (IOVBASE_TYPE *)((uintptr_t)frag->segments[0].seg_addr.pval + header_size);
-        iov.iov_len  = length = payload_size;
+        iov.iov_base = (IOVBASE_TYPE *) ((uintptr_t) frag->segments[0].seg_addr.pval + header_size);
+        iov.iov_len = length = payload_size;
 
-        (void) opal_convertor_pack (convertor, &iov, &iov_count, &length);
+        (void) opal_convertor_pack(convertor, &iov, &iov_count, &length);
 
-        assert (length == payload_size);
+        assert(length == payload_size);
     }
 
-    /* write the fragment pointer to peer's the FIFO. the progress function will return the fragment */
-    if (!sm_fifo_write_ep (frag->hdr, endpoint)) {
+    /* write the fragment pointer to peer's the FIFO. the progress function will return the fragment
+     */
+    if (!sm_fifo_write_ep(frag->hdr, endpoint)) {
         if (descriptor) {
             *descriptor = &frag->base;
         } else {
-            mca_btl_sm_free (btl, &frag->base);
+            mca_btl_sm_free(btl, &frag->base);
         }
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
diff --git a/opal/mca/btl/sm/btl_sm_types.h b/opal/mca/btl/sm/btl_sm_types.h
new file mode 100644
index 00000000000..35953ca7b34
--- /dev/null
+++ b/opal/mca/btl/sm/btl_sm_types.h
@@ -0,0 +1,246 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2009 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 Voltaire. All rights reserved.
+ * Copyright (c) 2009-2010 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2018      Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2020-2025 Google, LLC. All rights reserved.
+ *
+ * Copyright (c) 2019-2020 Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+/**
+ * @file
+ */
+#ifndef MCA_BTL_SM_TYPES_H
+#define MCA_BTL_SM_TYPES_H
+
+#include "opal_config.h"
+#include "opal/class/opal_free_list.h"
+#include "opal/mca/btl/btl.h"
+#include "opal/mca/smsc/smsc.h"
+
+/*
+ * Modex data
+ */
+struct mca_btl_sm_modex_t {
+    uint64_t segment_base;
+    int seg_ds_size;
+    /* seg_ds needs to be the last element */
+    opal_shmem_ds_t seg_ds;
+};
+
+typedef struct mca_btl_sm_modex_t mca_btl_sm_modex_t;
+
+typedef struct mca_btl_sm_fbox_metadata {
+    uint32_t start;
+    uint8_t  padding[26];
+} mca_btl_sm_fbox_metadata_t;
+
+typedef struct mca_btl_sm_fbox_out {
+    unsigned char *buffer; /**< starting address of peer's fast box in */
+    mca_btl_sm_fbox_metadata_t *metadata;
+    unsigned int start, end;
+    uint16_t seq;
+    opal_free_list_item_t *fbox; /**< fast-box free list item */
+} mca_btl_sm_fbox_out_t;
+
+typedef struct mca_btl_sm_fbox_in {
+    unsigned char *buffer; /**< starting address of peer's fast box out */
+    mca_btl_sm_fbox_metadata_t *metadata;
+    unsigned int start;
+    uint16_t seq;
+} mca_btl_sm_fbox_in_t;
+
+typedef struct mca_btl_base_endpoint_t {
+    opal_list_item_t super;
+
+    /* per peer buffers */
+    mca_btl_sm_fbox_in_t fbox_in;
+    mca_btl_sm_fbox_out_t fbox_out;
+
+    uint16_t peer_smp_rank;        /**< my peer's SMP process rank.  Used for accessing
+                                    *   SMP specific data structures. */
+    opal_atomic_size_t send_count; /**< number of fragments sent to this peer */
+    char *segment_base;            /**< start of the peer's segment (in the address space
+                                    *   of this process) */
+
+    struct sm_fifo_t *fifo; /**< */
+
+    opal_mutex_t lock; /**< lock to protect endpoint structures from concurrent
+                        *   access */
+
+    mca_smsc_endpoint_t *smsc_endpoint;
+    void *smsc_map_context;
+    opal_shmem_ds_t *seg_ds; /**< stored segment information for detach */
+
+    opal_mutex_t pending_frags_lock; /**< protect pending_frags */
+    opal_list_t pending_frags;       /**< fragments pending fast box space */
+    bool waiting;                    /**< endpoint is on the component wait list */
+} mca_btl_base_endpoint_t;
+
+typedef mca_btl_base_endpoint_t mca_btl_sm_endpoint_t;
+
+OBJ_CLASS_DECLARATION(mca_btl_sm_endpoint_t);
+
+/**
+ * Shared Memory (SM) BTL module.
+ */
+struct mca_btl_sm_component_t {
+    mca_btl_base_component_3_0_0_t super; /**< base BTL component */
+    int sm_free_list_num;                 /**< initial size of free lists */
+    int sm_free_list_max;                 /**< maximum size of free lists */
+    int sm_free_list_inc; /**< number of elements to alloc when growing free lists */
+    opal_shmem_ds_t seg_ds; /**< this rank's shared memory segment (when not using xpmem) */
+
+    opal_mutex_t lock;     /**< lock to protect concurrent updates to this structure's members */
+    char *my_segment;      /**< this rank's base pointer */
+    size_t segment_size;   /**< size of my_segment */
+    int32_t num_smp_procs; /**< current number of smp procs on this host */
+    opal_free_list_t sm_frags_eager;    /**< free list of sm send frags */
+    opal_free_list_t sm_frags_max_send; /**< free list of sm max send frags (large fragments) */
+    opal_free_list_t sm_frags_user;     /**< free list of small inline frags */
+    opal_free_list_t sm_fboxes;         /**< free list of available fast-boxes */
+
+    unsigned int
+        fbox_threshold; /**< number of sends required before we setup a send fast box for a peer */
+    unsigned int fbox_max;  /**< maximum number of send fast boxes to allocate */
+    unsigned int fbox_size; /**< size of each peer fast box allocation */
+
+    int single_copy_mechanism; /**< single copy mechanism to use */
+
+    int memcpy_limit;             /**< Limit where we switch from memmove to memcpy */
+    unsigned int max_inline_send; /**< Limit for copy-in-copy-out fragments */
+
+    mca_btl_base_endpoint_t
+        *endpoints; /**< array of local endpoints (one for each local peer including myself) */
+    mca_btl_base_endpoint_t **fbox_in_endpoints; /**< array of fast box in endpoints */
+    unsigned int num_fbox_in_endpoints;          /**< number of fast boxes to poll */
+    struct sm_fifo_t *my_fifo;                   /**< pointer to the local fifo */
+
+    opal_list_t pending_endpoints; /**< list of endpoints with pending fragments */
+    opal_list_t pending_fragments; /**< fragments pending remote completion */
+
+    char *backing_directory; /**< directory to place shared memory backing files */
+
+    mca_mpool_base_module_t *mpool;
+};
+typedef struct mca_btl_sm_component_t mca_btl_sm_component_t;
+
+/**
+ * SM BTL Interface
+ */
+struct mca_btl_sm_t {
+    mca_btl_base_module_t super; /**< base BTL interface */
+    bool btl_inited;             /**< flag indicating if btl has been inited */
+    mca_btl_base_module_error_cb_fn_t error_cb;
+};
+typedef struct mca_btl_sm_t mca_btl_sm_t;
+
+enum {
+    MCA_BTL_SM_FLAG_INLINE = 0,
+    MCA_BTL_SM_FLAG_SINGLE_COPY = 1,
+    MCA_BTL_SM_FLAG_COMPLETE = 2,
+    MCA_BTL_SM_FLAG_SETUP_FBOX = 4,
+};
+
+struct mca_btl_sm_frag_t;
+struct mca_btl_sm_fbox_t;
+
+enum mca_btl_sm_sc_emu_type_t {
+    MCA_BTL_SM_OP_PUT,
+    MCA_BTL_SM_OP_GET,
+    MCA_BTL_SM_OP_ATOMIC,
+    MCA_BTL_SM_OP_CSWAP,
+};
+typedef enum mca_btl_sm_sc_emu_type_t mca_btl_sm_sc_emu_type_t;
+
+struct mca_btl_sm_sc_emu_hdr_t {
+    mca_btl_sm_sc_emu_type_t type;
+    uint64_t addr;
+    mca_btl_base_atomic_op_t op;
+    int flags;
+    int64_t operand[2];
+};
+typedef struct mca_btl_sm_sc_emu_hdr_t mca_btl_sm_sc_emu_hdr_t;
+
+/**
+ * FIFO fragment header
+ */
+struct mca_btl_sm_hdr_t {
+    /** next item in fifo. many peers may touch this */
+    volatile intptr_t next;
+    /** pointer back the the fragment */
+    struct mca_btl_sm_frag_t *frag;
+    /** tag associated with this fragment (used to lookup callback) */
+    mca_btl_base_tag_t tag;
+    /** sm send flags (inline, complete, setup fbox, etc) */
+    uint8_t flags;
+    /** length of data following this header */
+    int32_t len;
+    /** io vector containing pointer to single-copy data */
+    struct iovec sc_iov;
+    /** if the fragment indicates to setup a fast box the base is stored here */
+    intptr_t fbox_base;
+};
+typedef struct mca_btl_sm_hdr_t mca_btl_sm_hdr_t;
+
+/**
+ * shared memory send fragment derived type.
+ */
+struct mca_btl_sm_frag_t {
+    /** base object */
+    mca_btl_base_descriptor_t base;
+    /** storage for segment data (max 2) */
+    mca_btl_base_segment_t segments[2];
+    /** endpoint this fragment is active on */
+    struct mca_btl_base_endpoint_t *endpoint;
+    /** fragment header (in the shared memory region) */
+    mca_btl_sm_hdr_t *hdr;
+    /** free list this fragment was allocated within */
+    opal_free_list_t *my_list;
+    /** rdma callback data */
+    struct mca_btl_sm_rdma_cbdata_t {
+        void *local_address;
+        uint64_t remote_address;
+        mca_btl_base_rdma_completion_fn_t cbfunc;
+        void *context;
+        void *cbdata;
+        size_t remaining;
+        size_t sent;
+    } rdma;
+};
+typedef struct mca_btl_sm_frag_t mca_btl_sm_frag_t;
+
+OBJ_CLASS_DECLARATION(mca_btl_sm_frag_t);
+
+/** FIFO types **/
+typedef opal_atomic_intptr_t atomic_fifo_value_t;
+typedef intptr_t fifo_value_t;
+
+/* lock free fifo */
+struct sm_fifo_t {
+    atomic_fifo_value_t fifo_head;
+    atomic_fifo_value_t fifo_tail;
+    opal_atomic_int32_t fbox_available;
+};
+typedef struct sm_fifo_t sm_fifo_t;
+
+#endif /* MCA_BTL_SM_TYPES_H */
diff --git a/opal/mca/btl/sm/btl_sm_virtual.h b/opal/mca/btl/sm/btl_sm_virtual.h
new file mode 100644
index 00000000000..320e6f69a6c
--- /dev/null
+++ b/opal/mca/btl/sm/btl_sm_virtual.h
@@ -0,0 +1,74 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2009 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 Voltaire. All rights reserved.
+ * Copyright (c) 2009-2010 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2018 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2020      Google, LLC. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+/**
+ * @file
+ */
+#ifndef MCA_BTL_SM_VIRTUAL_H
+#define MCA_BTL_SM_VIRTUAL_H
+
+#include "opal_config.h"
+
+#include "opal/mca/btl/sm/btl_sm_types.h"
+
+#if SIZEOF_VOID_P == 8
+#    define MCA_BTL_SM_OFFSET_MASK 0xffffffffll
+#    define MCA_BTL_SM_OFFSET_BITS 32
+#    define MCA_BTL_SM_BITNESS     64
+#else
+#    define MCA_BTL_SM_OFFSET_MASK 0x00ffffffl
+#    define MCA_BTL_SM_OFFSET_BITS 24
+#    define MCA_BTL_SM_BITNESS     32
+#endif
+
+/***
+ * One or more FIFO components may be a pointer that must be
+ * accessed by multiple processes.  Since the shared region may
+ * be mmapped differently into each process's address space,
+ * these pointers will be relative to some base address.  Here,
+ * we define inline functions to translate between relative
+ * addresses and virtual addresses.
+ */
+
+/* This only works for finding the relative address for a pointer within my_segment */
+static inline fifo_value_t virtual2relative(char *addr)
+{
+    return (fifo_value_t)((intptr_t)(addr - mca_btl_sm_component.my_segment))
+           | ((fifo_value_t) MCA_BTL_SM_LOCAL_RANK << MCA_BTL_SM_OFFSET_BITS);
+}
+
+static inline fifo_value_t virtual2relativepeer(struct mca_btl_base_endpoint_t *endpoint,
+                                                char *addr)
+{
+    return (fifo_value_t)((intptr_t)(addr - endpoint->segment_base))
+           | ((fifo_value_t) endpoint->peer_smp_rank << MCA_BTL_SM_OFFSET_BITS);
+}
+
+static inline void *relative2virtual(fifo_value_t offset)
+{
+    return (void *) (intptr_t)(
+        (offset & MCA_BTL_SM_OFFSET_MASK)
+        + mca_btl_sm_component.endpoints[offset >> MCA_BTL_SM_OFFSET_BITS].segment_base);
+}
+
+#endif /* MCA_BTL_SM_VIRTUAL_H */
diff --git a/opal/mca/btl/sm/btl_sm_xpmem.c b/opal/mca/btl/sm/btl_sm_xpmem.c
deleted file mode 100644
index 805f5a1d639..00000000000
--- a/opal/mca/btl/sm/btl_sm_xpmem.c
+++ /dev/null
@@ -1,247 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2014      The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2020      Google, LLC. All rights reserved.
- * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "btl_sm.h"
-
-#include "opal/include/opal/align.h"
-#include "opal/mca/memchecker/base/base.h"
-
-#if OPAL_BTL_SM_HAVE_XPMEM
-
-int mca_btl_sm_xpmem_init (void)
-{
-    /* Any attachment that goes past the Linux TASK_SIZE will always fail. To prevent this we need to
-     * determine the value of TASK_SIZE. On x86_64 the value was hard-coded in sm to be
-     * 0x7ffffffffffful but this approach does not work with AARCH64 (and possibly other architectures).
-     * Since there is really no way to directly determine the value we can (in all cases?) look through
-     * the mapping for this process to determine what the largest address is. This should be the top
-     * of the stack. No heap allocations should be larger than this value. Since the largest address
-     * may differ between processes the value must be shared as part of the modex and stored in the
-     * endpoint. */
-    FILE *fh = fopen("/proc/self/maps", "r");
-    if (NULL == fh) {
-        BTL_ERROR(("could not open /proc/self/maps for reading. disabling XPMEM"));
-        return OPAL_ERR_NOT_AVAILABLE;
-    }
-
-    char buffer[1024];
-    uintptr_t address_max = 0;
-    while (fgets(buffer, sizeof(buffer), fh)) {
-        uintptr_t low, high;
-        char *tmp;
-        /* each line of /proc/self/maps starts with low-high in hexidecimal (without a 0x) */
-        low = strtoul(buffer, &tmp, 16);
-        high = strtoul(tmp+1, NULL, 16);
-        if (address_max < high) {
-            address_max = high;
-        }
-    }
-
-    fclose (fh);
-
-    if (0 == address_max) {
-        BTL_ERROR(("could not determine the address max"));
-        return OPAL_ERR_NOT_AVAILABLE;
-    }
-
-    /* save the calcuated maximum */
-    mca_btl_sm_component.my_address_max = address_max - 1;
-
-    /* it is safe to use XPMEM_MAXADDR_SIZE here (which is always (size_t)-1 even though
-     * it is not safe for attach */
-    mca_btl_sm_component.my_seg_id = xpmem_make (0, XPMEM_MAXADDR_SIZE, XPMEM_PERMIT_MODE,
-                                                    (void *)0666);
-    if (-1 == mca_btl_sm_component.my_seg_id) {
-        return OPAL_ERR_NOT_AVAILABLE;
-    }
-
-    mca_btl_sm.super.btl_get = mca_btl_sm_get_xpmem;
-    mca_btl_sm.super.btl_put = mca_btl_sm_put_xpmem;
-
-    return OPAL_SUCCESS;
-}
-
-struct sm_check_reg_ctx_t {
-    mca_btl_base_endpoint_t *ep;
-    mca_rcache_base_registration_t **reg;
-    uintptr_t base;
-    uintptr_t bound;
-};
-typedef struct sm_check_reg_ctx_t sm_check_reg_ctx_t;
-
-static int sm_check_reg (mca_rcache_base_registration_t *reg, void *ctx)
-{
-    sm_check_reg_ctx_t *sm_ctx = (sm_check_reg_ctx_t *) ctx;
-
-    if ((intptr_t) reg->alloc_base != sm_ctx->ep->peer_smp_rank) {
-        /* ignore this registration */
-        return OPAL_SUCCESS;
-    }
-
-    sm_ctx->reg[0] = reg;
-
-    if (sm_ctx->bound <= (uintptr_t) reg->bound && sm_ctx->base >= (uintptr_t) reg->base) {
-        if (0 == opal_atomic_fetch_add_32 (&reg->ref_count, 1)) {
-            /* registration is being deleted by a thread in sm_return_registration. the
-             * VMA tree implementation will block in mca_rcache_delete until we finish
-             * iterating over the VMA tree so it is safe to just ignore this registration
-             * and continue. */
-            sm_ctx->reg[0] = NULL;
-            return OPAL_SUCCESS;
-        }
-        return 1;
-    }
-
-    if (MCA_RCACHE_FLAGS_INVALID & opal_atomic_fetch_or_32(&reg->flags, MCA_RCACHE_FLAGS_INVALID)) {
-        /* another thread has already marked this registration as invalid. ignore and continue. */
-        sm_ctx->reg[0] = NULL;
-        return OPAL_SUCCESS;
-    }
-
-    /* let the caller know we found an overlapping registration that can be coalesced into
-     * the requested interval. the caller will remove the last reference and delete the
-     * registration. */
-    return 2;
-}
-
-void sm_return_registration (mca_rcache_base_registration_t *reg, struct mca_btl_base_endpoint_t *ep)
-{
-    mca_rcache_base_vma_module_t *vma_module =  mca_btl_sm_component.vma_module;
-    int32_t ref_count;
-
-    ref_count = opal_atomic_add_fetch_32 (&reg->ref_count, -1);
-    if (OPAL_UNLIKELY(0 == ref_count && !(reg->flags & MCA_RCACHE_FLAGS_PERSIST))) {
-#if OPAL_ENABLE_DEBUG
-        int ret = mca_rcache_base_vma_delete (vma_module, reg);
-        assert (OPAL_SUCCESS == ret);
-#else
-        (void) mca_rcache_base_vma_delete (vma_module, reg);
-#endif
-        opal_memchecker_base_mem_noaccess (reg->rcache_context, (uintptr_t)(reg->bound - reg->base));
-        (void)xpmem_detach (reg->rcache_context);
-        OBJ_RELEASE (reg);
-    }
-}
-
-/* look up the remote pointer in the peer rcache and attach if
- * necessary */
-mca_rcache_base_registration_t *sm_get_registation (struct mca_btl_base_endpoint_t *ep, void *rem_ptr,
-                                                       size_t size, int flags, void **local_ptr)
-{
-    mca_rcache_base_vma_module_t *vma_module = mca_btl_sm_component.vma_module;
-    uint64_t attach_align = 1 << mca_btl_sm_component.log_attach_align;
-    mca_rcache_base_registration_t *reg = NULL;
-    sm_check_reg_ctx_t check_ctx = {.ep = ep, .reg = &reg};
-    xpmem_addr_t xpmem_addr;
-    uintptr_t base, bound;
-    int rc;
-
-    base = OPAL_DOWN_ALIGN((uintptr_t) rem_ptr, attach_align, uintptr_t);
-    bound = OPAL_ALIGN((uintptr_t) rem_ptr + size - 1, attach_align, uintptr_t) + 1;
-    if (OPAL_UNLIKELY(bound > ep->segment_data.xpmem.address_max)) {
-        bound = ep->segment_data.xpmem.address_max;
-    }
-
-    check_ctx.base = base;
-    check_ctx.bound = bound;
-
-    /* several segments may match the base pointer */
-    rc = mca_rcache_base_vma_iterate (vma_module, (void *) base, bound - base, true, sm_check_reg, &check_ctx);
-    if (2 == rc) {
-        bound = bound < (uintptr_t) reg->bound ? (uintptr_t) reg->bound : bound;
-        base = base > (uintptr_t) reg->base ? (uintptr_t) reg->base : base;
-        sm_return_registration(reg, ep);
-        reg = NULL;
-    }
-
-    if (NULL == reg) {
-        reg = OBJ_NEW(mca_rcache_base_registration_t);
-        if (OPAL_LIKELY(NULL != reg)) {
-            /* stick around for awhile */
-            reg->ref_count = 2;
-            reg->base  = (unsigned char *) base;
-            reg->bound = (unsigned char *) bound;
-            reg->flags = flags;
-            reg->alloc_base = (void *) (intptr_t) ep->peer_smp_rank;
-
-#if defined(HAVE_SN_XPMEM_H)
-            xpmem_addr.id     = ep->segment_data.xpmem.apid;
-#else
-            xpmem_addr.apid   = ep->segment_data.xpmem.apid;
-#endif
-            xpmem_addr.offset = base;
-
-            reg->rcache_context = xpmem_attach (xpmem_addr, bound - base, NULL);
-            if (OPAL_UNLIKELY((void *)-1 == reg->rcache_context)) {
-                OBJ_RELEASE(reg);
-                return NULL;
-            }
-
-            opal_memchecker_base_mem_defined (reg->rcache_context, bound - base);
-
-            if (!(flags & MCA_RCACHE_FLAGS_PERSIST)) {
-                mca_rcache_base_vma_insert (vma_module, reg, 0);
-            }
-        }
-    }
-
-    opal_atomic_wmb ();
-    *local_ptr = (void *) ((uintptr_t) reg->rcache_context +
-                           (ptrdiff_t)((uintptr_t) rem_ptr - (uintptr_t) reg->base));
-
-    return reg;
-}
-
-struct sm_cleanup_reg_ctx {
-    mca_btl_sm_endpoint_t *ep;
-    opal_list_t *registrations;
-};
-
-static int mca_btl_sm_endpoint_xpmem_rcache_cleanup (mca_rcache_base_registration_t *reg, void *ctx)
-{
-    struct sm_cleanup_reg_ctx *cleanup_ctx = (struct sm_cleanup_reg_ctx *) ctx;
-    if ((intptr_t) reg->alloc_base == cleanup_ctx->ep->peer_smp_rank) {
-        opal_list_append(cleanup_ctx->registrations, &reg->super.super);
-    }
-
-    return OPAL_SUCCESS;
-}
-
-void mca_btl_sm_xpmem_cleanup_endpoint (struct mca_btl_base_endpoint_t *ep)
-{
-    mca_rcache_base_registration_t *reg;
-    opal_list_t registrations;
-    struct sm_cleanup_reg_ctx cleanup_ctx = {.ep = ep, .registrations = &registrations};
-
-    OBJ_CONSTRUCT(&registrations, opal_list_t);
-
-    /* clean out the registration cache */
-    (void) mca_rcache_base_vma_iterate (mca_btl_sm_component.vma_module,
-                                        NULL, (size_t) -1, true,
-                                        mca_btl_sm_endpoint_xpmem_rcache_cleanup,
-                                        (void *) &cleanup_ctx);
-    while (NULL != (reg = (mca_rcache_base_registration_t *) opal_list_remove_first(&registrations))) {
-        sm_return_registration (reg, ep);
-    }
-    OBJ_DESTRUCT(&registrations);
-
-    if (ep->segment_base) {
-        xpmem_release (ep->segment_data.xpmem.apid);
-        ep->segment_data.xpmem.apid = 0;
-    }
-}
-
-#endif /* OPAL_BTL_SM_HAVE_XPMEM */
diff --git a/opal/mca/btl/sm/btl_sm_xpmem.h b/opal/mca/btl/sm/btl_sm_xpmem.h
deleted file mode 100644
index 422964c7fb6..00000000000
--- a/opal/mca/btl/sm/btl_sm_xpmem.h
+++ /dev/null
@@ -1,67 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2013-2014 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2016      ARM, Inc. All rights reserved.
- * Copyright (c) 2020      Google, LLC. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#if !defined(MCA_BTL_SM_XPMEM_H)
-#define MCA_BTL_SM_XPMEM_H
-
-#if OPAL_BTL_SM_HAVE_XPMEM
-
-#if defined(HAVE_XPMEM_H)
-  #include <xpmem.h>
-
-  typedef struct xpmem_addr xpmem_addr_t;
-#elif defined(HAVE_SN_XPMEM_H)
-  #include <sn/xpmem.h>
-
-  typedef int64_t xpmem_segid_t;
-  typedef int64_t xpmem_apid_t;
-#endif
-
-#include "opal/mca/rcache/base/rcache_base_vma.h"
-#include "opal/mca/rcache/rcache.h"
-
-/* look up the remote pointer in the peer rcache and attach if
- * necessary */
-
-struct mca_btl_base_endpoint_t;
-
-int mca_btl_sm_xpmem_init (void);
-
-mca_rcache_base_registration_t *sm_get_registation (struct mca_btl_base_endpoint_t *endpoint, void *rem_ptr,
-                                                       size_t size, int flags, void **local_ptr);
-
-void sm_return_registration (mca_rcache_base_registration_t *reg, struct mca_btl_base_endpoint_t *endpoint);
-void mca_btl_sm_xpmem_cleanup_endpoint (struct mca_btl_base_endpoint_t *ep);
-
-#else
-
-static inline mca_rcache_base_registration_t *sm_get_registation (struct mca_btl_base_endpoint_t *endpoint, void *rem_ptr,
-                                                                     size_t size, int flags, void **local_ptr)
-{
-    (void) endpoint;
-    (void) rem_ptr;
-    (void) size;
-    (void) flags;
-    (void) local_ptr;
-    return NULL;
-}
-
-static inline void sm_return_registration (mca_rcache_base_registration_t *reg, struct mca_btl_base_endpoint_t *endpoint)
-{
-    (void) reg;
-    (void) endpoint;
-}
-
-#endif /* OPAL_BTL_SM_HAVE_XPMEM */
-
-#endif
diff --git a/opal/mca/btl/sm/configure.m4 b/opal/mca/btl/sm/configure.m4
index f1eea2710dc..2e649810574 100644
--- a/opal/mca/btl/sm/configure.m4
+++ b/opal/mca/btl/sm/configure.m4
@@ -8,6 +8,7 @@
 #                         reserved.
 # Copyright (c) 2015      Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -21,33 +22,8 @@
 AC_DEFUN([MCA_opal_btl_sm_CONFIG],[
     AC_CONFIG_FILES([opal/mca/btl/sm/Makefile])
 
-    OPAL_VAR_SCOPE_PUSH([btl_sm_xpmem_happy btl_sm_cma_happy btl_sm_knem_happy])
-
-    # Check for single-copy APIs
-
-    OPAL_CHECK_XPMEM([btl_sm], [btl_sm_xpmem_happy=1], [btl_sm_xpmem_happy=0])
-    OPAL_CHECK_KNEM([btl_sm], [btl_sm_knem_happy=1],[btl_sm_knem_happy=0])
-    OPAL_CHECK_CMA([btl_sm], [AC_CHECK_HEADER([sys/prctl.h]) btl_sm_cma_happy=1], [btl_sm_cma_happy=0])
-
-    AC_DEFINE_UNQUOTED([OPAL_BTL_SM_HAVE_XPMEM], [$btl_sm_xpmem_happy],
-        [If XPMEM support can be enabled within sm])
-
-    AC_DEFINE_UNQUOTED([OPAL_BTL_SM_HAVE_CMA], [$btl_sm_cma_happy],
-        [If CMA support can be enabled within sm])
-
-    AC_DEFINE_UNQUOTED([OPAL_BTL_SM_HAVE_KNEM], [$btl_sm_knem_happy],
-	[If KNEM support can be enabled within sm])
-
-    OPAL_VAR_SCOPE_POP
-
     # always happy
-    [$1]
+    $1
 
     OPAL_SUMMARY_ADD([[Transports]],[[Shared memory/copy in+copy out]],[$1],[yes])
-
-    # substitute in the things needed to build with XPMEM support
-    AC_SUBST([btl_sm_CFLAGS])
-    AC_SUBST([btl_sm_CPPFLAGS])
-    AC_SUBST([btl_sm_LDFLAGS])
-    AC_SUBST([btl_sm_LIBS])
 ])dnl
diff --git a/opal/mca/btl/sm/help-btl-sm.txt b/opal/mca/btl/sm/help-btl-sm.txt
deleted file mode 100644
index 84e00c36d1c..00000000000
--- a/opal/mca/btl/sm/help-btl-sm.txt
+++ /dev/null
@@ -1,156 +0,0 @@
-# -*- text -*-
-#
-# Copyright (c) 2004-2009 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2006-2014 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2012-2014 Los Alamos National Security, LLC.
-#                         All rights reserved.
-# Copyright (c) 2014      Research Organization for Information Science
-#                         and Technology (RIST). All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-# This is the US/English help file for Open MPI's shared memory support.
-#
-[sys call fail]
-A system call failed during sm shared memory BTL initialization
-that should not have.  It is likely that your MPI job will now either
-abort or experience performance degradation.
-
-  System call: %s
-  Error:       %s (errno %d)
-#
-[no locality]
-WARNING: Missing locality information required for sm shared memory
-BTL initialization. Continuing without shared memory support.
-#
-[knem permission denied]
-WARING: Open MPI failed to open the /dev/knem device due to a
-permissions problem.  Please check with your system administrator to
-get the permissions fixed, or set the btl_sm_single_copy_mechanism
-MCA variable to none to silence this warning and run without knem
-support.
-
-  Local host:            %s
-  /dev/knem permissions: 0%o
-#
-[knem fail open]
-WARNING: Open MPI failed to open the /dev/knem device due to a local
-error. Please check with your system administrator to get the problem
-fixed, or set the btl_sm_single_copy_mechanism MCA variable to none
-to silence this warning and run without knem support.
-
-The sm shared memory BTL will fall back on another single-copy
-mechanism if one is available. This may result in lower performance.
-
-  Local host: %s
-  Errno:      %d (%s)
-#
-[knem get ABI fail]
-WARNING: Open MPI failed to retrieve the ABI version from the
-/dev/knem device due to a local error.  This usually indicates an
-error in your knem installation; please check with your system
-administrator, or set the btl_sm_single_copy_mechanism MCA variable
-to none to silence this warning and run without knem support.
-
-The sm shared memory BTL will fall back on another single-copy
-mechanism if one is available. This may result in lower performance.
-
-  Local host: %s
-  Errno:      %d (%s)
-#
-[knem ABI mismatch]
-WARNING: Open MPI was compiled with support for one version of the
-knem kernel module, but it discovered a different version running in
-/dev/knem. Open MPI needs to be installed with support for the same
-version of knem as is in the running Linux kernel. Please check with
-your system administrator, or set the btl_sm_single_copy_mechanism
-MCA variable to none to silence this warning and run without knem
-support.
-
-The sm shared memory BTL will fall back on another single-copy
-mechanism if one is available. This may result in lower performance.
-
-  Local host:              %s
-  Open MPI's knem version: 0x%x
-  /dev/knem's version:     0x%x
-#
-[knem mmap fail]
-Open MPI failed to map support from the knem Linux kernel module; this
-shouldn't happen. Please check with your system administrator, or set
-the btl_sm_single_copy_mechanism MCA variable to none to silence
-this warning and run without knem support.
-
-The sm shared memory BTL will fall back on another single-copy
-mechanism if one is available. This may result in lower performance.
-
-  Local host:  %s
-  System call: mmap()
-  Errno:       %d (%s)
-#
-[knem init error]
-Open MPI encountered an error during the knem initialization. Please
-check with your system administrator, or set the
-btl_sm_single_copy_mechanism MCA variable to none to silence this
-warning and run without knem support.
-
-The sm shared memory BTL will fall back on another single-copy
-mechanism if one is available. This may result in lower performance.
-
-  Local host:  %s
-  System call: %s
-  Errno:       %d (%s)
-#
-[knem requested but not available]
-WARNING: Linux kernel knem support was requested via the
-btl_sm_single_copy_mechanism MCA parameter, but Knem support was either not
-compiled into this Open MPI installation, or Knem support was unable
-to be activated in this process.
-
-The sm BTL will fall back on another single-copy mechanism if one
-is available. This may result in lower performance.
-
-  Local host: %s
-#
-[cma-permission-denied]
-WARNING: Linux kernel CMA support was requested via the
-btl_sm_single_copy_mechanism MCA variable, but CMA support is
-not available due to restrictive ptrace settings.
-
-The sm shared memory BTL will fall back on another single-copy
-mechanism if one is available. This may result in lower performance.
-
-  Local host: %s
-#
-[cma-different-user-namespace-error]
-ERROR: Linux kernel CMA support was requested via the
-btl_sm_single_copy_mechanism MCA variable, but CMA support is
-not available due to different user namespaces.
-
-Your MPI job will abort now. Please select another value for
-btl_sm_single_copy_mechanism.
-
-  Local host: %s
-#
-[cma-different-user-namespace-warning]
-WARNING: The default btl_sm_single_copy_mechanism CMA is
-not available due to different user namespaces.
-
-The sm shared memory BTL will fall back on another single-copy
-mechanism if one is available. This may result in lower performance.
-
-  Local host: %s
-#
-[xpmem-make-failed]
-WARNING: Could not generate an xpmem segment id for this process'
-address space.
-
-The sm shared memory BTL will fall back on another single-copy
-mechanism if one is available. This may result in lower performance.
-
-  Local host: %s
-  Error code: %d (%s)
diff --git a/opal/mca/btl/smcuda/Makefile.am b/opal/mca/btl/smcuda/Makefile.am
index 733965596fd..c0cdf788e8d 100644
--- a/opal/mca/btl/smcuda/Makefile.am
+++ b/opal/mca/btl/smcuda/Makefile.am
@@ -12,6 +12,8 @@
 # Copyright (c) 2009-2014 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2012      NVIDIA Corporation.  All rights reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -19,7 +21,7 @@
 # $HEADER$
 #
 
-dist_opaldata_DATA = help-mpi-btl-smcuda.txt
+EXTRA_DIST = help-mpi-btl-smcuda.txt
 
 libmca_btl_smcuda_la_sources = \
     btl_smcuda.c \
@@ -28,7 +30,9 @@ libmca_btl_smcuda_la_sources = \
     btl_smcuda_endpoint.h \
     btl_smcuda_fifo.h \
     btl_smcuda_frag.c \
-    btl_smcuda_frag.h
+    btl_smcuda_frag.h \
+    btl_smcuda_accelerator.c \
+    btl_smcuda_accelerator.h
 
 # Make the output library in this directory, and name it either
 # mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
@@ -48,16 +52,14 @@ endif
 mcacomponentdir = $(opallibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_btl_smcuda_la_SOURCES = $(libmca_btl_smcuda_la_sources)
-mca_btl_smcuda_la_LDFLAGS = -module -avoid-version
-mca_btl_smcuda_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
-    $(OPAL_TOP_BUILDDIR)/opal/mca/common/sm/lib@OPAL_LIB_PREFIX@mca_common_sm.la
+mca_btl_smcuda_la_LDFLAGS = -module -avoid-version $(btl_smcuda_LDFLAGS)
+mca_btl_smcuda_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
+    $(OPAL_TOP_BUILDDIR)/opal/mca/common/sm/lib@OPAL_LIB_NAME@mca_common_sm.la \
+    $(btl_smcuda_LIBS)
 mca_btl_smcuda_la_CPPFLAGS = $(btl_smcuda_CPPFLAGS)
-if OPAL_cuda_support
-mca_btl_smcuda_la_LIBADD += \
-    $(OPAL_TOP_BUILDDIR)/opal/mca/common/cuda/lib@OPAL_LIB_PREFIX@mca_common_cuda.la
-endif
 
 noinst_LTLIBRARIES = $(component_noinst)
 libmca_btl_smcuda_la_SOURCES = $(libmca_btl_smcuda_la_sources)
-libmca_btl_smcuda_la_LDFLAGS = -module -avoid-version
+libmca_btl_smcuda_la_LDFLAGS = -module -avoid-version $(btl_smcuda_LDFLAGS)
 libmca_btl_smcuda_la_CPPFLAGS = $(btl_smcuda_CPPFLAGS)
+libmca_btl_smcuda_la_LIBADD = $(btl_smcuda_LIBS)
diff --git a/opal/mca/btl/smcuda/btl_smcuda.c b/opal/mca/btl/smcuda/btl_smcuda.c
index ad155ae7f69..c4389d422f2 100644
--- a/opal/mca/btl/smcuda/btl_smcuda.c
+++ b/opal/mca/btl/smcuda/btl_smcuda.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2011 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2014 The University of Tennessee and The University
+ * Copyright (c) 2004-2022 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2007 High Performance Computing Center Stuttgart,
@@ -14,11 +14,16 @@
  * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2010-2017 Los Alamos National Security, LLC. All rights
  *                         reserved.
- * Copyright (c) 2012-2015 NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2012-2024 NVIDIA Corporation.  All rights reserved.
  * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
  * Copyright (c) 2014-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2015-2019 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
+ * Copyright (c) 2023      Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2023      Advanced Micro Devices, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -28,88 +33,83 @@
 
 #include "opal_config.h"
 
-#include <sys/types.h>
 #include <sys/stat.h>
+#include <sys/types.h>
 #ifdef HAVE_FCNTL_H
-#include <fcntl.h>
-#endif  /* HAVE_FCNTL_H */
+#    include <fcntl.h>
+#endif /* HAVE_FCNTL_H */
 #include <errno.h>
 #ifdef HAVE_SYS_MMAN_H
-#include <sys/mman.h>
-#endif  /* HAVE_SYS_MMAN_H */
+#    include <sys/mman.h>
+#endif /* HAVE_SYS_MMAN_H */
 
 #ifdef OPAL_BTL_SM_CMA_NEED_SYSCALL_DEFS
-#include "opal/sys/cma.h"
+#    include "opal/sys/cma.h"
 #endif /* OPAL_BTL_SM_CMA_NEED_SYSCALL_DEFS */
 
-#include "opal/sys/atomic.h"
 #include "opal/class/opal_bitmap.h"
-#include "opal/util/output.h"
-#include "opal/util/show_help.h"
-#include "opal/util/printf.h"
+#include "opal/datatype/opal_convertor.h"
+#include "opal/mca/btl/btl.h"
 #include "opal/mca/hwloc/base/base.h"
 #include "opal/mca/pmix/base/base.h"
 #include "opal/mca/shmem/base/base.h"
 #include "opal/mca/shmem/shmem.h"
-#include "opal/datatype/opal_convertor.h"
-#include "opal/mca/btl/btl.h"
+#include "opal/sys/atomic.h"
+#include "opal/util/output.h"
+#include "opal/util/printf.h"
+#include "opal/util/show_help.h"
 
 #include "opal/mca/common/sm/common_sm_mpool.h"
 
-#if OPAL_CUDA_SUPPORT
-#include "opal/mca/common/cuda/common_cuda.h"
-#endif /* OPAL_CUDA_SUPPORT */
+#include "opal/mca/accelerator/accelerator.h"
+#include "opal/mca/accelerator/base/base.h"
 #include "opal/mca/mpool/base/base.h"
 #include "opal/mca/rcache/base/base.h"
 
 #include "btl_smcuda.h"
 #include "btl_smcuda_endpoint.h"
-#include "btl_smcuda_frag.h"
 #include "btl_smcuda_fifo.h"
-
-#if OPAL_CUDA_SUPPORT
-static struct mca_btl_base_registration_handle_t *mca_btl_smcuda_register_mem (
-    struct mca_btl_base_module_t* btl, struct mca_btl_base_endpoint_t *endpoint, void *base,
-    size_t size, uint32_t flags);
-
-static int mca_btl_smcuda_deregister_mem (struct mca_btl_base_module_t* btl,
-                                          struct mca_btl_base_registration_handle_t *handle);
-#endif
-
-mca_btl_smcuda_t mca_btl_smcuda = {
-    .super = {
-        .btl_component = &mca_btl_smcuda_component.super,
-        .btl_add_procs = mca_btl_smcuda_add_procs,
-        .btl_del_procs = mca_btl_smcuda_del_procs,
-        .btl_finalize = mca_btl_smcuda_finalize,
-        .btl_alloc = mca_btl_smcuda_alloc,
-        .btl_free = mca_btl_smcuda_free,
-        .btl_prepare_src = mca_btl_smcuda_prepare_src,
-#if OPAL_CUDA_SUPPORT
-        .btl_register_mem = mca_btl_smcuda_register_mem,
-        .btl_deregister_mem = mca_btl_smcuda_deregister_mem,
-#endif /* OPAL_CUDA_SUPPORT */
-        .btl_send = mca_btl_smcuda_send,
-        .btl_sendi = mca_btl_smcuda_sendi,
-        .btl_dump = mca_btl_smcuda_dump,
-        .btl_register_error = mca_btl_smcuda_register_error_cb,
-    }
-};
-
-#if OPAL_CUDA_SUPPORT
-static void mca_btl_smcuda_send_cuda_ipc_request(struct mca_btl_base_module_t* btl,
-                                                 struct mca_btl_base_endpoint_t* endpoint);
-#endif /* OPAL_CUDA_SUPPORT */
+#include "btl_smcuda_frag.h"
+#include "btl_smcuda_accelerator.h"
+
+#include "opal/include/opal/opal_gpu.h"
+
+static struct mca_btl_base_registration_handle_t *
+mca_btl_smcuda_register_mem(struct mca_btl_base_module_t *btl,
+                            struct mca_btl_base_endpoint_t *endpoint, void *base, size_t size,
+                            uint32_t flags);
+
+static int mca_btl_smcuda_deregister_mem(struct mca_btl_base_module_t *btl,
+                                         struct mca_btl_base_registration_handle_t *handle);
+
+mca_btl_smcuda_t mca_btl_smcuda = {.super = {
+                                       .btl_component = &mca_btl_smcuda_component.super,
+                                       .btl_add_procs = mca_btl_smcuda_add_procs,
+                                       .btl_del_procs = mca_btl_smcuda_del_procs,
+                                       .btl_finalize = mca_btl_smcuda_finalize,
+                                       .btl_alloc = mca_btl_smcuda_alloc,
+                                       .btl_free = mca_btl_smcuda_free,
+                                       .btl_prepare_src = mca_btl_smcuda_prepare_src,
+                                       .btl_register_mem = mca_btl_smcuda_register_mem,
+                                       .btl_deregister_mem = mca_btl_smcuda_deregister_mem,
+                                       .btl_send = mca_btl_smcuda_send,
+                                       .btl_sendi = mca_btl_smcuda_sendi,
+                                       .btl_dump = mca_btl_smcuda_dump,
+                                       .btl_register_error = mca_btl_smcuda_register_error_cb,
+                                   }};
+
+static void mca_btl_smcuda_send_cuda_ipc_request(struct mca_btl_base_module_t *btl,
+                                                 struct mca_btl_base_endpoint_t *endpoint);
 /*
  * calculate offset of an address from the beginning of a shared memory segment
  */
-#define ADDR2OFFSET(ADDR, BASE) ((char*)(ADDR) - (char*)(BASE))
+#define ADDR2OFFSET(ADDR, BASE) ((char *) (ADDR) - (char *) (BASE))
 
 /*
  * calculate an absolute address in a local address space given an offset and
  * a base address of a shared memory segment
  */
-#define OFFSET2ADDR(OFFSET, BASE) ((ptrdiff_t)(OFFSET) + (char*)(BASE))
+#define OFFSET2ADDR(OFFSET, BASE) ((ptrdiff_t)(OFFSET) + (char *) (BASE))
 
 static void *mpool_calloc(size_t nmemb, size_t size)
 {
@@ -126,9 +126,8 @@ static void *mpool_calloc(size_t nmemb, size_t size)
     return buf;
 }
 
-static int
-setup_mpool_base_resources(mca_btl_smcuda_component_t *comp_ptr,
-                           mca_common_sm_mpool_resources_t *out_res)
+static int setup_mpool_base_resources(mca_btl_smcuda_component_t *comp_ptr,
+                                      mca_common_sm_mpool_resources_t *out_res)
 {
     int rc = OPAL_SUCCESS;
     int fd = -1;
@@ -136,39 +135,37 @@ setup_mpool_base_resources(mca_btl_smcuda_component_t *comp_ptr,
 
     if (-1 == (fd = open(comp_ptr->sm_mpool_rndv_file_name, O_RDONLY))) {
         int err = errno;
-        opal_show_help("help-mpi-btl-smcuda.txt", "sys call fail", true,
-                       "open(2)", strerror(err), err);
+        opal_show_help("help-mpi-btl-smcuda.txt", "sys call fail", true, "open(2)", strerror(err),
+                       err);
         rc = OPAL_ERR_IN_ERRNO;
         goto out;
     }
-    if ((ssize_t)sizeof(opal_shmem_ds_t) != (bread =
-        read(fd, &out_res->bs_meta_buf, sizeof(opal_shmem_ds_t)))) {
-        opal_output(0, "setup_mpool_base_resources: "
+    if ((ssize_t) sizeof(opal_shmem_ds_t)
+        != (bread = read(fd, &out_res->bs_meta_buf, sizeof(opal_shmem_ds_t)))) {
+        opal_output(0,
+                    "setup_mpool_base_resources: "
                     "Read inconsistency -- read: %lu, but expected: %lu!\n",
-                    (unsigned long)bread,
-                    (unsigned long)sizeof(opal_shmem_ds_t));
+                    (unsigned long) bread, (unsigned long) sizeof(opal_shmem_ds_t));
         rc = OPAL_ERROR;
         goto out;
     }
-    if ((ssize_t)sizeof(out_res->size) != (bread =
-        read(fd, &out_res->size, sizeof(size_t)))) {
-        opal_output(0, "setup_mpool_base_resources: "
+    if ((ssize_t) sizeof(out_res->size) != (bread = read(fd, &out_res->size, sizeof(size_t)))) {
+        opal_output(0,
+                    "setup_mpool_base_resources: "
                     "Read inconsistency -- read: %lu, but expected: %lu!\n",
-                    (unsigned long)bread,
-                    (unsigned long)sizeof(opal_shmem_ds_t));
+                    (unsigned long) bread, (unsigned long) sizeof(opal_shmem_ds_t));
         rc = OPAL_ERROR;
         goto out;
     }
 
 out:
     if (-1 != fd) {
-        (void)close(fd);
+        (void) close(fd);
     }
     return rc;
 }
 
-static int
-sm_segment_attach(mca_btl_smcuda_component_t *comp_ptr)
+static int sm_segment_attach(mca_btl_smcuda_component_t *comp_ptr)
 {
     int rc = OPAL_SUCCESS;
     int fd = -1;
@@ -180,34 +177,34 @@ sm_segment_attach(mca_btl_smcuda_component_t *comp_ptr)
     }
     if (-1 == (fd = open(comp_ptr->sm_rndv_file_name, O_RDONLY))) {
         int err = errno;
-        opal_show_help("help-mpi-btl-smcuda.txt", "sys call fail", true,
-                       "open(2)", strerror(err), err);
+        opal_show_help("help-mpi-btl-smcuda.txt", "sys call fail", true, "open(2)", strerror(err),
+                       err);
         rc = OPAL_ERR_IN_ERRNO;
         goto out;
     }
-    if ((ssize_t)sizeof(opal_shmem_ds_t) != (bread =
-        read(fd, tmp_shmem_ds, sizeof(opal_shmem_ds_t)))) {
-        opal_output(0, "sm_segment_attach: "
+    if ((ssize_t) sizeof(opal_shmem_ds_t)
+        != (bread = read(fd, tmp_shmem_ds, sizeof(opal_shmem_ds_t)))) {
+        opal_output(0,
+                    "sm_segment_attach: "
                     "Read inconsistency -- read: %lu, but expected: %lu!\n",
-                    (unsigned long)bread,
-                    (unsigned long)sizeof(opal_shmem_ds_t));
+                    (unsigned long) bread, (unsigned long) sizeof(opal_shmem_ds_t));
         rc = OPAL_ERROR;
         goto out;
     }
-    if (NULL == (comp_ptr->sm_seg =
-                 mca_common_sm_module_attach(tmp_shmem_ds,
-                                             sizeof(mca_common_sm_seg_header_t),
-                                             opal_cache_line_size))) {
+    if (NULL
+        == (comp_ptr->sm_seg = mca_common_sm_module_attach(tmp_shmem_ds,
+                                                           sizeof(mca_common_sm_seg_header_t),
+                                                           opal_cache_line_size))) {
         /* don't have to detach here, because module_attach cleans up after
          * itself on failure. */
         opal_output(0, "sm_segment_attach: "
-                    "mca_common_sm_module_attach failure!\n");
+                       "mca_common_sm_module_attach failure!\n");
         rc = OPAL_ERROR;
     }
 
 out:
     if (-1 != fd) {
-        (void)close(fd);
+        (void) close(fd);
     }
     if (tmp_shmem_ds) {
         free(tmp_shmem_ds);
@@ -215,39 +212,33 @@ sm_segment_attach(mca_btl_smcuda_component_t *comp_ptr)
     return rc;
 }
 
-static int
-smcuda_btl_first_time_init(mca_btl_smcuda_t *smcuda_btl,
-                       int32_t my_smp_rank,
-                       int n)
+static int smcuda_btl_first_time_init(mca_btl_smcuda_t *smcuda_btl, int32_t my_smp_rank, int n)
 {
     size_t length, length_payload;
     sm_fifo_t *my_fifos;
-    int my_mem_node, num_mem_nodes, i, rc;
     mca_common_sm_mpool_resources_t *res = NULL;
-    mca_btl_smcuda_component_t* m = &mca_btl_smcuda_component;
     char *loc, *mynuma;
     opal_process_name_t wildcard_rank;
+    int rc;
 
     /* Assume we don't have hwloc support and fill in dummy info */
-    mca_btl_smcuda_component.mem_node = my_mem_node = 0;
-    mca_btl_smcuda_component.num_mem_nodes = num_mem_nodes = 1;
+    mca_btl_smcuda_component.mem_node = -1;
+    mca_btl_smcuda_component.num_mem_nodes = 1;
 
     /* see if we were given a topology signature */
     wildcard_rank.jobid = OPAL_PROC_MY_NAME.jobid;
     wildcard_rank.vpid = OPAL_VPID_WILDCARD;
-    OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_TOPOLOGY_SIGNATURE,
-                                   &wildcard_rank, &loc, PMIX_STRING);
+    OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_TOPOLOGY_SIGNATURE, &wildcard_rank, &loc, PMIX_STRING);
     if (OPAL_SUCCESS == rc) {
         /* the number of NUMA nodes is right at the front */
-        mca_btl_smcuda_component.num_mem_nodes = num_mem_nodes = strtoul(loc, NULL, 10);
+        mca_btl_smcuda_component.num_mem_nodes = strtoul(loc, NULL, 10);
         free(loc);
     } else {
         /* If we have hwloc support, then get accurate information */
         loc = NULL;
         if (OPAL_SUCCESS == opal_hwloc_base_get_topology()) {
-            i = opal_hwloc_base_get_nbobjs_by_type(opal_hwloc_topology,
-                                                   HWLOC_OBJ_NODE, 0,
-                                                   OPAL_HWLOC_AVAILABLE);
+            rc = opal_hwloc_base_get_nbobjs_by_type(opal_hwloc_topology, HWLOC_OBJ_NODE, 0,
+                                                    OPAL_HWLOC_AVAILABLE);
 
             /* JMS This tells me how many numa nodes are *available*,
                but it's not how many are being used *by this job*.
@@ -255,28 +246,22 @@ smcuda_btl_first_time_init(mca_btl_smcuda_t *smcuda_btl,
                the previous carto-based implementation), but it really
                should be improved to be how many NUMA nodes are being
                used *in this job*. */
-            mca_btl_smcuda_component.num_mem_nodes = num_mem_nodes = i;
+            mca_btl_smcuda_component.num_mem_nodes = rc;
         }
     }
     /* see if we were given our location */
-    OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_LOCALITY_STRING,
-                                   &OPAL_PROC_MY_NAME, &loc, PMIX_STRING);
+    OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_LOCALITY_STRING, &OPAL_PROC_MY_NAME, &loc, PMIX_STRING);
     if (OPAL_SUCCESS == rc) {
-        if (NULL == loc) {
-            mca_btl_smcuda_component.mem_node = my_mem_node = -1;
-        } else {
+        if (NULL != loc) {
             /* get our NUMA location */
             mynuma = opal_hwloc_base_get_location(loc, HWLOC_OBJ_NODE, 0);
-            if (NULL == mynuma ||
-                NULL != strchr(mynuma, ',') ||
-                NULL != strchr(mynuma, '-')) {
+            if (NULL == mynuma || NULL != strchr(mynuma, ',') || NULL != strchr(mynuma, '-')) {
                 /* we either have no idea what NUMA we are on, or we
                  * are on multiple NUMA nodes */
-                mca_btl_smcuda_component.mem_node = my_mem_node = -1;
+                mca_btl_smcuda_component.mem_node = -1;
             } else {
                 /* we are bound to a single NUMA node */
-                my_mem_node = strtoul(mynuma, NULL, 10);
-                mca_btl_smcuda_component.mem_node = my_mem_node;
+                mca_btl_smcuda_component.mem_node = strtoul(mynuma, NULL, 10);
             }
             if (NULL != mynuma) {
                 free(mynuma);
@@ -285,17 +270,17 @@ smcuda_btl_first_time_init(mca_btl_smcuda_t *smcuda_btl,
         }
     } else {
         /* If we have hwloc support, then get accurate information */
-        if (OPAL_SUCCESS == opal_hwloc_base_get_topology() &&
-            num_mem_nodes > 0 && NULL != opal_process_info.cpuset) {
-            int numa=0, w;
-            unsigned n_bound=0;
+        if (OPAL_SUCCESS == opal_hwloc_base_get_topology() && mca_btl_smcuda_component.num_mem_nodes > 0
+            && NULL != opal_process_info.cpuset) {
+            int numa = 0, w;
+            unsigned n_bound = 0;
             hwloc_obj_t obj;
 
             /* count the number of NUMA nodes to which we are bound */
-            for (w=0; w < i; w++) {
-                if (NULL == (obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology,
-                                                                   HWLOC_OBJ_NODE, 0, w,
-                                                                   OPAL_HWLOC_AVAILABLE))) {
+            for (w = 0; w < mca_btl_smcuda_component.num_mem_nodes; w++) {
+                if (NULL
+                    == (obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology, HWLOC_OBJ_NODE,
+                                                              0, w, OPAL_HWLOC_AVAILABLE))) {
                     continue;
                 }
                 /* see if we intersect with that NUMA node's cpus */
@@ -308,33 +293,40 @@ smcuda_btl_first_time_init(mca_btl_smcuda_t *smcuda_btl,
              * a NUMA we are on, then not much we can do
              */
             if (1 == n_bound) {
-                mca_btl_smcuda_component.mem_node = my_mem_node = numa;
-            } else {
-                mca_btl_smcuda_component.mem_node = my_mem_node = -1;
+                mca_btl_smcuda_component.mem_node = numa;
             }
         }
     }
+    /* sanity check: do we have the NUMA node info ? */
+    if( mca_btl_smcuda_component.mem_node < 0 ||
+        mca_btl_smcuda_component.num_mem_nodes < 1) {
+        opal_output_verbose(10, opal_btl_base_framework.framework_output,
+                            "btl:smcuda: %s unable to find topological information mem_node=%d, num_mem_nodes=%d",
+                            OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),
+                            mca_btl_smcuda_component.mem_node, mca_btl_smcuda_component.num_mem_nodes);
+        mca_btl_smcuda_component.mem_node = 0;
+        mca_btl_smcuda_component.num_mem_nodes = 1;
+    }
 
     if (NULL == (res = calloc(1, sizeof(*res)))) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
     /* lookup shared memory pool */
-    mca_btl_smcuda_component.sm_mpools =
-        (mca_mpool_base_module_t **)calloc(num_mem_nodes,
-                                           sizeof(mca_mpool_base_module_t *));
+    mca_btl_smcuda_component.sm_mpools = (mca_mpool_base_module_t **)
+        calloc(mca_btl_smcuda_component.num_mem_nodes, sizeof(mca_mpool_base_module_t *));
 
     /* Disable memory binding, because each MPI process will claim pages in the
      * mpool for their local NUMA node */
     res->mem_node = -1;
     res->allocator = mca_btl_smcuda_component.allocator;
 
-    if (OPAL_SUCCESS != (rc = setup_mpool_base_resources(m, res))) {
+    if (OPAL_SUCCESS != (rc = setup_mpool_base_resources(&mca_btl_smcuda_component, res))) {
         free(res);
         return rc;
     }
     /* now that res is fully populated, create the thing */
-    mca_btl_smcuda_component.sm_mpools[0] = common_sm_mpool_create (res);
+    mca_btl_smcuda_component.sm_mpools[0] = opal_btl_smcuda_common_sm_mpool_create(res);
     /* Sanity check to ensure that we found it */
     if (NULL == mca_btl_smcuda_component.sm_mpools[0]) {
         free(res);
@@ -343,12 +335,12 @@ smcuda_btl_first_time_init(mca_btl_smcuda_t *smcuda_btl,
 
     mca_btl_smcuda_component.sm_mpool = mca_btl_smcuda_component.sm_mpools[0];
 
-    mca_btl_smcuda_component.sm_mpool_base =
-        mca_btl_smcuda_component.sm_mpools[0]->mpool_base(mca_btl_smcuda_component.sm_mpools[0]);
+    mca_btl_smcuda_component.sm_mpool_base = mca_btl_smcuda_component.sm_mpools[0]->mpool_base(
+        mca_btl_smcuda_component.sm_mpools[0]);
 
     /* create a list of peers */
-    mca_btl_smcuda_component.sm_peers = (struct mca_btl_base_endpoint_t**)
-        calloc(n, sizeof(struct mca_btl_base_endpoint_t*));
+    mca_btl_smcuda_component.sm_peers = (struct mca_btl_base_endpoint_t **)
+        calloc(n, sizeof(struct mca_btl_base_endpoint_t *));
     if (NULL == mca_btl_smcuda_component.sm_peers) {
         free(res);
         return OPAL_ERR_OUT_OF_RESOURCE;
@@ -356,20 +348,27 @@ smcuda_btl_first_time_init(mca_btl_smcuda_t *smcuda_btl,
 
     /* remember that node rank zero is already attached */
     if (0 != my_smp_rank) {
-        if (OPAL_SUCCESS != (rc = sm_segment_attach(m))) {
+        if (OPAL_SUCCESS != (rc = sm_segment_attach(&mca_btl_smcuda_component))) {
             free(res);
             return rc;
         }
     }
-#if OPAL_CUDA_SUPPORT
     /* Register the entire shared memory region with the CUDA library which will
-     * force it to be pinned.  This aproach was chosen as there is no way for this
+     * force it to be pinned.  This approach was chosen as there is no way for this
      * local process to know which parts of the memory are being utilized by a
      * remote process. */
     opal_output_verbose(10, opal_btl_base_framework.framework_output,
-                        "btl:smcuda: CUDA cuMemHostRegister address=%p, size=%d",
-                        mca_btl_smcuda_component.sm_mpool_base, (int)res->size);
-    mca_common_cuda_register(mca_btl_smcuda_component.sm_mpool_base, res->size, "smcuda");
+                        "btl:smcuda: host_register address=%p, size=%d",
+                        mca_btl_smcuda_component.sm_mpool_base, (int) res->size);
+    if (0 != strcmp(opal_accelerator_base_selected_component.base_version.mca_component_name, "null")) {
+        rc = opal_accelerator.host_register(MCA_ACCELERATOR_NO_DEVICE_ID, mca_btl_smcuda_component.sm_mpool_base, res->size);
+        if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
+            /* If registering the memory fails, print a message and continue.
+             * This is not a fatal error. */
+            opal_output_verbose(10, opal_btl_base_framework.framework_output,
+                                "btl:smcuda: host_register failed");
+        }
+    }
 
     /* Create a local memory pool that sends handles to the remote
      * side.  Note that the res argument is not really used, but
@@ -379,97 +378,91 @@ smcuda_btl_first_time_init(mca_btl_smcuda_t *smcuda_btl,
     if (NULL == smcuda_btl->rcache) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
-#endif /* OPAL_CUDA_SUPPORT */
 
     /* it is now safe to free the mpool resources */
     free(res);
 
     /* check to make sure number of local procs is within the
      * specified limits */
-    if(mca_btl_smcuda_component.sm_max_procs > 0 &&
-       mca_btl_smcuda_component.num_smp_procs + n >
-       mca_btl_smcuda_component.sm_max_procs) {
+    if (mca_btl_smcuda_component.sm_max_procs > 0
+        && mca_btl_smcuda_component.num_smp_procs + n > mca_btl_smcuda_component.sm_max_procs) {
         return OPAL_ERROR;
     }
 
-    mca_btl_smcuda_component.shm_fifo = (volatile sm_fifo_t **)mca_btl_smcuda_component.sm_seg->module_data_addr;
-    mca_btl_smcuda_component.shm_bases = (char**)(mca_btl_smcuda_component.shm_fifo + n);
-    mca_btl_smcuda_component.shm_mem_nodes = (uint16_t*)(mca_btl_smcuda_component.shm_bases + n);
+    mca_btl_smcuda_component.shm_fifo = (volatile sm_fifo_t **)
+                                            mca_btl_smcuda_component.sm_seg->module_data_addr;
+    mca_btl_smcuda_component.shm_bases = (char **) (mca_btl_smcuda_component.shm_fifo + n);
+    mca_btl_smcuda_component.shm_mem_nodes = (uint16_t *) (mca_btl_smcuda_component.shm_bases + n);
 
     /* set the base of the shared memory segment */
-    mca_btl_smcuda_component.shm_bases[mca_btl_smcuda_component.my_smp_rank] =
-        (char*)mca_btl_smcuda_component.sm_mpool_base;
-    mca_btl_smcuda_component.shm_mem_nodes[mca_btl_smcuda_component.my_smp_rank] =
-        (uint16_t)my_mem_node;
+    mca_btl_smcuda_component.shm_bases[mca_btl_smcuda_component.my_smp_rank]
+        = (char *) mca_btl_smcuda_component.sm_mpool_base;
+    mca_btl_smcuda_component.shm_mem_nodes[mca_btl_smcuda_component.my_smp_rank] = (uint16_t)
+        mca_btl_smcuda_component.mem_node;
 
     /* initialize the array of fifo's "owned" by this process */
-    if(NULL == (my_fifos = (sm_fifo_t*)mpool_calloc(FIFO_MAP_NUM(n), sizeof(sm_fifo_t))))
+    if (NULL == (my_fifos = (sm_fifo_t *) mpool_calloc(FIFO_MAP_NUM(n), sizeof(sm_fifo_t))))
         return OPAL_ERR_OUT_OF_RESOURCE;
 
     mca_btl_smcuda_component.shm_fifo[mca_btl_smcuda_component.my_smp_rank] = my_fifos;
 
     /* cache the pointer to the 2d fifo array.  These addresses
      * are valid in the current process space */
-    mca_btl_smcuda_component.fifo = (sm_fifo_t**)malloc(sizeof(sm_fifo_t*) * n);
+    mca_btl_smcuda_component.fifo = (sm_fifo_t **) malloc(sizeof(sm_fifo_t *) * n);
 
-    if(NULL == mca_btl_smcuda_component.fifo)
+    if (NULL == mca_btl_smcuda_component.fifo)
         return OPAL_ERR_OUT_OF_RESOURCE;
 
     mca_btl_smcuda_component.fifo[mca_btl_smcuda_component.my_smp_rank] = my_fifos;
 
     mca_btl_smcuda_component.mem_nodes = (uint16_t *) malloc(sizeof(uint16_t) * n);
-    if(NULL == mca_btl_smcuda_component.mem_nodes)
+    if (NULL == mca_btl_smcuda_component.mem_nodes)
         return OPAL_ERR_OUT_OF_RESOURCE;
 
     /* initialize fragment descriptor free lists */
 
     /* allocation will be for the fragment descriptor and payload buffer */
     length = sizeof(mca_btl_smcuda_frag1_t);
-    length_payload =
-        sizeof(mca_btl_smcuda_hdr_t) + mca_btl_smcuda_component.eager_limit;
-    i = opal_free_list_init (&mca_btl_smcuda_component.sm_frags_eager, length,
-                             opal_cache_line_size, OBJ_CLASS(mca_btl_smcuda_frag1_t),
-                             length_payload, opal_cache_line_size,
+    length_payload = sizeof(mca_btl_smcuda_hdr_t) + mca_btl_smcuda_component.eager_limit;
+    rc = opal_free_list_init(&mca_btl_smcuda_component.sm_frags_eager, length, opal_cache_line_size,
+                             OBJ_CLASS(mca_btl_smcuda_frag1_t), length_payload, opal_cache_line_size,
                              mca_btl_smcuda_component.sm_free_list_num,
                              mca_btl_smcuda_component.sm_free_list_max,
                              mca_btl_smcuda_component.sm_free_list_inc,
                              mca_btl_smcuda_component.sm_mpool, 0, NULL, NULL, NULL);
-    if ( OPAL_SUCCESS != i )
-        return i;
+    if (OPAL_SUCCESS != rc)
+        return rc;
 
     length = sizeof(mca_btl_smcuda_frag2_t);
-    length_payload =
-        sizeof(mca_btl_smcuda_hdr_t) + mca_btl_smcuda_component.max_frag_size;
-    i = opal_free_list_init (&mca_btl_smcuda_component.sm_frags_max, length,
-                             opal_cache_line_size, OBJ_CLASS(mca_btl_smcuda_frag2_t),
-                             length_payload, opal_cache_line_size,
+    length_payload = sizeof(mca_btl_smcuda_hdr_t) + mca_btl_smcuda_component.max_frag_size;
+    rc = opal_free_list_init(&mca_btl_smcuda_component.sm_frags_max, length, opal_cache_line_size,
+                             OBJ_CLASS(mca_btl_smcuda_frag2_t), length_payload, opal_cache_line_size,
                              mca_btl_smcuda_component.sm_free_list_num,
                              mca_btl_smcuda_component.sm_free_list_max,
                              mca_btl_smcuda_component.sm_free_list_inc,
                              mca_btl_smcuda_component.sm_mpool, 0, NULL, NULL, NULL);
-    if ( OPAL_SUCCESS != i )
-        return i;
+    if (OPAL_SUCCESS != rc)
+        return rc;
 
-    i = opal_free_list_init (&mca_btl_smcuda_component.sm_frags_user,
-                             sizeof(mca_btl_smcuda_user_t),
+    rc = opal_free_list_init(&mca_btl_smcuda_component.sm_frags_user, sizeof(mca_btl_smcuda_user_t),
                              opal_cache_line_size, OBJ_CLASS(mca_btl_smcuda_user_t),
                              sizeof(mca_btl_smcuda_hdr_t), opal_cache_line_size,
                              mca_btl_smcuda_component.sm_free_list_num,
                              mca_btl_smcuda_component.sm_free_list_max,
                              mca_btl_smcuda_component.sm_free_list_inc,
                              mca_btl_smcuda_component.sm_mpool, 0, NULL, NULL, NULL);
-    if ( OPAL_SUCCESS != i )
-            return i;
+    if (OPAL_SUCCESS != rc)
+        return rc;
 
     mca_btl_smcuda_component.num_outstanding_frags = 0;
 
     mca_btl_smcuda_component.num_pending_sends = 0;
-    i = opal_free_list_init (&mca_btl_smcuda_component.pending_send_fl,
+    rc = opal_free_list_init(&mca_btl_smcuda_component.pending_send_fl,
                              sizeof(btl_smcuda_pending_send_item_t), 8,
-                             OBJ_CLASS(opal_free_list_item_t),
-                             0, 0, 16, -1, 32, NULL, 0, NULL, NULL, NULL);
-    if ( OPAL_SUCCESS != i )
-        return i;
+                             OBJ_CLASS(opal_free_list_item_t), 0, 0, 16, -1, 32, NULL, 0, NULL, NULL,
+                             NULL);
+    if (OPAL_SUCCESS != rc)
+        return rc;
 
     /* set flag indicating btl has been inited */
     smcuda_btl->btl_inited = true;
@@ -477,53 +470,46 @@ smcuda_btl_first_time_init(mca_btl_smcuda_t *smcuda_btl,
     return OPAL_SUCCESS;
 }
 
-static struct mca_btl_base_endpoint_t *
-create_sm_endpoint(int local_proc, struct opal_proc_t *proc)
+static struct mca_btl_base_endpoint_t *create_sm_endpoint(int local_proc, struct opal_proc_t *proc)
 {
     struct mca_btl_base_endpoint_t *ep;
 
 #if OPAL_ENABLE_PROGRESS_THREADS == 1
-    char path[PATH_MAX];
+    char path[OPAL_PATH_MAX];
 #endif
 
-    ep = (struct mca_btl_base_endpoint_t*)
-        malloc(sizeof(struct mca_btl_base_endpoint_t));
-    if(NULL == ep)
+    ep = (struct mca_btl_base_endpoint_t *) malloc(sizeof(struct mca_btl_base_endpoint_t));
+    if (NULL == ep)
         return NULL;
     ep->peer_smp_rank = local_proc + mca_btl_smcuda_component.num_smp_procs;
 
     OBJ_CONSTRUCT(&ep->pending_sends, opal_list_t);
     OBJ_CONSTRUCT(&ep->endpoint_lock, opal_mutex_t);
 #if OPAL_ENABLE_PROGRESS_THREADS == 1
-    sprintf(path, "%s"OPAL_PATH_SEP"sm_fifo.%lu",
-            opal_process_info.job_session_dir,
-            (unsigned long)proc->proc_name);
+    sprintf(path, "%s" OPAL_PATH_SEP "sm_fifo.%lu", opal_process_info.job_session_dir,
+            (unsigned long) proc->proc_name);
     ep->fifo_fd = open(path, O_WRONLY);
-    if(ep->fifo_fd < 0) {
-        opal_output(0, "mca_btl_smcuda_add_procs: open(%s) failed with errno=%d\n",
-                    path, errno);
+    if (ep->fifo_fd < 0) {
+        opal_output(0, "mca_btl_smcuda_add_procs: open(%s) failed with errno=%d\n", path, errno);
         free(ep);
         return NULL;
     }
 #endif
-#if OPAL_CUDA_SUPPORT
+
     /* Create a remote memory pool on the endpoint. The rgpusm component
      * does not take any resources. They are filled in internally. */
-    ep->rcache = mca_rcache_base_module_create ("rgpusm", NULL, NULL);
-#endif /* OPAL_CUDA_SUPPORT */
+    ep->rcache = mca_rcache_base_module_create("rgpusm", NULL, NULL);
+
     return ep;
 }
 
-int mca_btl_smcuda_add_procs(
-    struct mca_btl_base_module_t* btl,
-    size_t nprocs,
-    struct opal_proc_t **procs,
-    struct mca_btl_base_endpoint_t **peers,
-    opal_bitmap_t* reachability)
+int mca_btl_smcuda_add_procs(struct mca_btl_base_module_t *btl, size_t nprocs,
+                             struct opal_proc_t **procs, struct mca_btl_base_endpoint_t **peers,
+                             opal_bitmap_t *reachability)
 {
     int return_code = OPAL_SUCCESS;
     int32_t n_local_procs = 0, proc, j, my_smp_rank = -1;
-    const opal_proc_t* my_proc; /* pointer to caller's proc structure */
+    const opal_proc_t *my_proc; /* pointer to caller's proc structure */
     mca_btl_smcuda_t *smcuda_btl;
     bool have_connected_peer = false;
     char **bases;
@@ -532,26 +518,26 @@ int mca_btl_smcuda_add_procs(
 
     /* initializion */
 
-    smcuda_btl = (mca_btl_smcuda_t *)btl;
+    smcuda_btl = (mca_btl_smcuda_t *) btl;
 
     /* get pointer to my proc structure */
-    if(NULL == (my_proc = opal_proc_local_get()))
+    if (NULL == (my_proc = opal_proc_local_get()))
         return OPAL_ERR_OUT_OF_RESOURCE;
 
     /* Get unique host identifier for each process in the list,
      * and idetify procs that are on this host.  Add procs on this
      * host to shared memory reachbility list.  Also, get number
      * of local procs in the procs list. */
-    for (proc = 0; proc < (int32_t)nprocs; proc++) {
+    for (proc = 0; proc < (int32_t) nprocs; proc++) {
         /* check to see if this proc can be reached via shmem (i.e.,
            if they're on my local host and in my job) */
-        if (procs[proc]->proc_name.jobid != my_proc->proc_name.jobid ||
-            !OPAL_PROC_ON_LOCAL_NODE(procs[proc]->proc_flags)) {
+        if (procs[proc]->proc_name.jobid != my_proc->proc_name.jobid
+            || !OPAL_PROC_ON_LOCAL_NODE(procs[proc]->proc_flags)) {
             peers[proc] = NULL;
             continue;
         }
         /* check to see if this is me */
-        if(my_proc == procs[proc]) {
+        if (my_proc == procs[proc]) {
             my_smp_rank = mca_btl_smcuda_component.my_smp_rank = n_local_procs++;
             continue;
         }
@@ -559,20 +545,20 @@ int mca_btl_smcuda_add_procs(
         /* we have someone to talk to */
         have_connected_peer = true;
 
-        if(!(peers[proc] = create_sm_endpoint(n_local_procs, procs[proc]))) {
+        if (!(peers[proc] = create_sm_endpoint(n_local_procs, procs[proc]))) {
             return_code = OPAL_ERROR;
             goto CLEANUP;
         }
-#if OPAL_CUDA_SUPPORT
+
         peers[proc]->proc_opal = procs[proc];
         peers[proc]->ipcstate = IPC_INIT;
         peers[proc]->ipctries = 0;
-#endif /* OPAL_CUDA_SUPPORT */
+
         n_local_procs++;
 
         /* add this proc to shared memory accessibility list */
         return_code = opal_bitmap_set_bit(reachability, proc);
-        if(OPAL_SUCCESS != return_code)
+        if (OPAL_SUCCESS != return_code)
             goto CLEANUP;
     }
 
@@ -587,9 +573,8 @@ int mca_btl_smcuda_add_procs(
     }
 
     if (!smcuda_btl->btl_inited) {
-        return_code =
-            smcuda_btl_first_time_init(smcuda_btl, my_smp_rank,
-                                   mca_btl_smcuda_component.sm_max_procs);
+        return_code = smcuda_btl_first_time_init(smcuda_btl, my_smp_rank,
+                                                 mca_btl_smcuda_component.sm_max_procs);
         if (return_code != OPAL_SUCCESS) {
             goto CLEANUP;
         }
@@ -597,15 +582,15 @@ int mca_btl_smcuda_add_procs(
 
     /* set local proc's smp rank in the peers structure for
      * rapid access and calculate reachability */
-    for(proc = 0; proc < (int32_t)nprocs; proc++) {
-        if(NULL == peers[proc])
+    for (proc = 0; proc < (int32_t) nprocs; proc++) {
+        if (NULL == peers[proc])
             continue;
         mca_btl_smcuda_component.sm_peers[peers[proc]->peer_smp_rank] = peers[proc];
         peers[proc]->my_smp_rank = my_smp_rank;
     }
 
     bases = mca_btl_smcuda_component.shm_bases;
-    sm_mpool_modp = (mca_common_sm_mpool_module_t *)mca_btl_smcuda_component.sm_mpool;
+    sm_mpool_modp = (mca_common_sm_mpool_module_t *) mca_btl_smcuda_component.sm_mpool;
 
     /* initialize own FIFOs */
     /*
@@ -613,14 +598,14 @@ int mca_btl_smcuda_add_procs(
      * be allocated near the receiver.  Nothing will be local to
      * "the sender" since there will be many senders.
      */
-    for(j = mca_btl_smcuda_component.num_smp_procs;
-        j < mca_btl_smcuda_component.num_smp_procs + FIFO_MAP_NUM(n_local_procs); j++) {
+    for (j = mca_btl_smcuda_component.num_smp_procs;
+         j < mca_btl_smcuda_component.num_smp_procs + FIFO_MAP_NUM(n_local_procs); j++) {
 
-        return_code = sm_fifo_init( mca_btl_smcuda_component.fifo_size,
-                                    mca_btl_smcuda_component.sm_mpool,
+        return_code = sm_fifo_init(mca_btl_smcuda_component.fifo_size,
+                                   mca_btl_smcuda_component.sm_mpool,
                                    &mca_btl_smcuda_component.fifo[my_smp_rank][j],
-                                    mca_btl_smcuda_component.fifo_lazy_free);
-        if(return_code != OPAL_SUCCESS)
+                                   mca_btl_smcuda_component.fifo_lazy_free);
+        if (return_code != OPAL_SUCCESS)
             goto CLEANUP;
     }
 
@@ -629,9 +614,8 @@ int mca_btl_smcuda_add_procs(
     /* Sync with other local procs. Force the FIFO initialization to always
      * happens before the readers access it.
      */
-    (void)opal_atomic_add_fetch_32(&mca_btl_smcuda_component.sm_seg->module_seg->seg_inited, 1);
-    while( n_local_procs >
-           mca_btl_smcuda_component.sm_seg->module_seg->seg_inited) {
+    (void) opal_atomic_add_fetch_32(&mca_btl_smcuda_component.sm_seg->module_seg->seg_inited, 1);
+    while (n_local_procs > mca_btl_smcuda_component.sm_seg->module_seg->seg_inited) {
         opal_progress();
         opal_atomic_rmb();
     }
@@ -639,8 +623,7 @@ int mca_btl_smcuda_add_procs(
     /* it is now safe to unlink the shared memory segment. only one process
      * needs to do this, so just let smp rank zero take care of it. */
     if (0 == my_smp_rank) {
-        if (OPAL_SUCCESS !=
-            mca_common_sm_module_unlink(mca_btl_smcuda_component.sm_seg)) {
+        if (OPAL_SUCCESS != mca_common_sm_module_unlink(mca_btl_smcuda_component.sm_seg)) {
             /* it is "okay" if this fails at this point. we have gone this far,
              * so just warn about the failure and continue. this is probably
              * only triggered by a programming error. */
@@ -651,8 +634,7 @@ int mca_btl_smcuda_add_procs(
 
         /* at this point, all processes have attached to the mpool segment. so
          * it is safe to unlink it here. */
-        if (OPAL_SUCCESS !=
-            mca_common_sm_module_unlink(sm_mpool_modp->sm_common_module)) {
+        if (OPAL_SUCCESS != mca_common_sm_module_unlink(sm_mpool_modp->sm_common_module)) {
             opal_output(0, "WARNING: common_sm_module_unlink failed.\n");
         }
         if (-1 == unlink(mca_btl_smcuda_component.sm_mpool_rndv_file_name)) {
@@ -672,14 +654,14 @@ int mca_btl_smcuda_add_procs(
     free(mca_btl_smcuda_component.sm_rndv_file_name);
 
     /* coordinate with other processes */
-    for(j = mca_btl_smcuda_component.num_smp_procs;
-        j < mca_btl_smcuda_component.num_smp_procs + n_local_procs; j++) {
+    for (j = mca_btl_smcuda_component.num_smp_procs;
+         j < mca_btl_smcuda_component.num_smp_procs + n_local_procs; j++) {
         ptrdiff_t diff;
 
         /* spin until this element is allocated */
         /* doesn't really wait for that process... FIFO might be allocated, but not initialized */
         opal_atomic_rmb();
-        while(NULL == mca_btl_smcuda_component.shm_fifo[j]) {
+        while (NULL == mca_btl_smcuda_component.shm_fifo[j]) {
             opal_progress();
             opal_atomic_rmb();
         }
@@ -688,8 +670,8 @@ int mca_btl_smcuda_add_procs(
         diff = ADDR2OFFSET(bases[my_smp_rank], bases[j]);
 
         /* store local address of remote fifos */
-        mca_btl_smcuda_component.fifo[j] =
-            (sm_fifo_t*)OFFSET2ADDR(diff, mca_btl_smcuda_component.shm_fifo[j]);
+        mca_btl_smcuda_component.fifo[j] = (sm_fifo_t *)
+            OFFSET2ADDR(diff, mca_btl_smcuda_component.shm_fifo[j]);
 
         /* cache local copy of peer memory node number */
         mca_btl_smcuda_component.mem_nodes[j] = mca_btl_smcuda_component.shm_mem_nodes[j];
@@ -699,8 +681,8 @@ int mca_btl_smcuda_add_procs(
     mca_btl_smcuda_component.num_smp_procs += n_local_procs;
 
     /* make sure we have enough eager fragmnents for each process */
-    return_code = opal_free_list_resize_mt (&mca_btl_smcuda_component.sm_frags_eager,
-                                            mca_btl_smcuda_component.num_smp_procs * 2);
+    return_code = opal_free_list_resize_mt(&mca_btl_smcuda_component.sm_frags_eager,
+                                           mca_btl_smcuda_component.num_smp_procs * 2);
     if (OPAL_SUCCESS != return_code)
         goto CLEANUP;
 
@@ -708,15 +690,12 @@ int mca_btl_smcuda_add_procs(
     return return_code;
 }
 
-int mca_btl_smcuda_del_procs(
-    struct mca_btl_base_module_t* btl,
-    size_t nprocs,
-    struct opal_proc_t **procs,
-    struct mca_btl_base_endpoint_t **peers)
+int mca_btl_smcuda_del_procs(struct mca_btl_base_module_t *btl, size_t nprocs,
+                             struct opal_proc_t **procs, struct mca_btl_base_endpoint_t **peers)
 {
-    for (size_t i = 0 ; i < nprocs ; ++i) {
+    for (size_t i = 0; i < nprocs; ++i) {
         if (peers[i]->rcache) {
-            mca_rcache_base_module_destroy (peers[i]->rcache);
+            mca_rcache_base_module_destroy(peers[i]->rcache);
             peers[i]->rcache = NULL;
         }
     }
@@ -724,7 +703,6 @@ int mca_btl_smcuda_del_procs(
     return OPAL_SUCCESS;
 }
 
-
 /**
  * MCA->BTL Clean up any resources held by BTL module
  * before the module is unloaded.
@@ -738,20 +716,18 @@ int mca_btl_smcuda_del_procs(
  *
  */
 
-int mca_btl_smcuda_finalize(struct mca_btl_base_module_t* btl)
+int mca_btl_smcuda_finalize(struct mca_btl_base_module_t *btl)
 {
     return OPAL_SUCCESS;
 }
 
-
 /*
  * Register callback function for error handling..
  */
-int mca_btl_smcuda_register_error_cb(
-        struct mca_btl_base_module_t* btl,
-        mca_btl_base_module_error_cb_fn_t cbfunc)
+int mca_btl_smcuda_register_error_cb(struct mca_btl_base_module_t *btl,
+                                     mca_btl_base_module_error_cb_fn_t cbfunc)
 {
-    mca_btl_smcuda_t *smcuda_btl = (mca_btl_smcuda_t *)btl;
+    mca_btl_smcuda_t *smcuda_btl = (mca_btl_smcuda_t *) btl;
     smcuda_btl->error_cb = cbfunc;
     return OPAL_SUCCESS;
 }
@@ -762,15 +738,12 @@ int mca_btl_smcuda_register_error_cb(
  * @param btl (IN)      BTL module
  * @param size (IN)     Request segment size.
  */
-extern mca_btl_base_descriptor_t* mca_btl_smcuda_alloc(
-    struct mca_btl_base_module_t* btl,
-    struct mca_btl_base_endpoint_t* endpoint,
-    uint8_t order,
-    size_t size,
-    uint32_t flags)
+extern mca_btl_base_descriptor_t *mca_btl_smcuda_alloc(struct mca_btl_base_module_t *btl,
+                                                       struct mca_btl_base_endpoint_t *endpoint,
+                                                       uint8_t order, size_t size, uint32_t flags)
 {
-    mca_btl_smcuda_frag_t* frag = NULL;
-    if(size <= mca_btl_smcuda_component.eager_limit) {
+    mca_btl_smcuda_frag_t *frag = NULL;
+    if (size <= mca_btl_smcuda_component.eager_limit) {
         MCA_BTL_SMCUDA_FRAG_ALLOC_EAGER(frag);
     } else if (size <= mca_btl_smcuda_component.max_frag_size) {
         MCA_BTL_SMCUDA_FRAG_ALLOC_MAX(frag);
@@ -780,7 +753,7 @@ extern mca_btl_base_descriptor_t* mca_btl_smcuda_alloc(
         frag->segment.seg_len = size;
         frag->base.des_flags = flags;
     }
-    return (mca_btl_base_descriptor_t*)frag;
+    return (mca_btl_base_descriptor_t *) frag;
 }
 
 /**
@@ -789,55 +762,46 @@ extern mca_btl_base_descriptor_t* mca_btl_smcuda_alloc(
  * @param btl (IN)      BTL module
  * @param segment (IN)  Allocated segment.
  */
-extern int mca_btl_smcuda_free(
-    struct mca_btl_base_module_t* btl,
-    mca_btl_base_descriptor_t* des)
+extern int mca_btl_smcuda_free(struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des)
 {
-    mca_btl_smcuda_frag_t* frag = (mca_btl_smcuda_frag_t*)des;
+    mca_btl_smcuda_frag_t *frag = (mca_btl_smcuda_frag_t *) des;
     MCA_BTL_SMCUDA_FRAG_RETURN(frag);
 
     return OPAL_SUCCESS;
 }
 
-
 /**
  * Pack data
  *
  * @param btl (IN)      BTL module
  */
-struct mca_btl_base_descriptor_t* mca_btl_smcuda_prepare_src(
-    struct mca_btl_base_module_t* btl,
-    struct mca_btl_base_endpoint_t* endpoint,
-    struct opal_convertor_t* convertor,
-    uint8_t order,
-    size_t reserve,
-    size_t* size,
-    uint32_t flags)
+struct mca_btl_base_descriptor_t *mca_btl_smcuda_prepare_src(
+    struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+    struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size, uint32_t flags)
 {
-    mca_btl_smcuda_frag_t* frag;
+    mca_btl_smcuda_frag_t *frag;
     struct iovec iov;
     uint32_t iov_count = 1;
     size_t max_data = *size;
     int rc;
 
-    if ( reserve + max_data <= mca_btl_smcuda_component.eager_limit ) {
+    if (reserve + max_data <= mca_btl_smcuda_component.eager_limit) {
         MCA_BTL_SMCUDA_FRAG_ALLOC_EAGER(frag);
     } else {
         MCA_BTL_SMCUDA_FRAG_ALLOC_MAX(frag);
     }
-    if( OPAL_UNLIKELY(NULL == frag) ) {
+    if (OPAL_UNLIKELY(NULL == frag)) {
         return NULL;
     }
 
-    if( OPAL_UNLIKELY(reserve + max_data > frag->size) ) {
+    if (OPAL_UNLIKELY(reserve + max_data > frag->size)) {
         max_data = frag->size - reserve;
     }
     iov.iov_len = max_data;
-    iov.iov_base =
-        (IOVBASE_TYPE*)(((unsigned char*)(frag->segment.seg_addr.pval)) + reserve);
+    iov.iov_base = (IOVBASE_TYPE *) (((unsigned char *) (frag->segment.seg_addr.pval)) + reserve);
 
-    rc = opal_convertor_pack(convertor, &iov, &iov_count, &max_data );
-    if( OPAL_UNLIKELY(rc < 0) ) {
+    rc = opal_convertor_pack(convertor, &iov, &iov_count, &max_data);
+    if (OPAL_UNLIKELY(rc < 0)) {
         MCA_BTL_SMCUDA_FRAG_RETURN(frag);
         return NULL;
     }
@@ -852,34 +816,46 @@ struct mca_btl_base_descriptor_t* mca_btl_smcuda_prepare_src(
 }
 
 #if 0
-#define MCA_BTL_SMCUDA_TOUCH_DATA_TILL_CACHELINE_BOUNDARY(sm_frag)          \
-    do {                                                                \
-        char* _memory = (char*)(sm_frag)->segment.seg_addr.pval +       \
-            (sm_frag)->segment.seg_len;                                 \
-        int* _intmem;                                                   \
-        size_t align = (intptr_t)_memory & 0xFUL;                       \
-        switch( align & 0x3 ) {                                         \
-        case 3: *_memory = 0; _memory++;                                \
-        case 2: *_memory = 0; _memory++;                                \
-        case 1: *_memory = 0; _memory++;                                \
-        }                                                               \
-        align >>= 2;                                                    \
-        _intmem = (int*)_memory;                                        \
-        switch( align ) {                                               \
-        case 3: *_intmem = 0; _intmem++;                                \
-        case 2: *_intmem = 0; _intmem++;                                \
-        case 1: *_intmem = 0; _intmem++;                                \
-        }                                                               \
-    } while(0)
+#    define MCA_BTL_SMCUDA_TOUCH_DATA_TILL_CACHELINE_BOUNDARY(sm_frag) \
+        do {                                                           \
+            char *_memory = (char *) (sm_frag)->segment.seg_addr.pval  \
+                            + (sm_frag)->segment.seg_len;              \
+            int *_intmem;                                              \
+            size_t align = (intptr_t) _memory & 0xFUL;                 \
+            switch (align & 0x3) {                                     \
+            case 3:                                                    \
+                *_memory = 0;                                          \
+                _memory++;                                             \
+            case 2:                                                    \
+                *_memory = 0;                                          \
+                _memory++;                                             \
+            case 1:                                                    \
+                *_memory = 0;                                          \
+                _memory++;                                             \
+            }                                                          \
+            align >>= 2;                                               \
+            _intmem = (int *) _memory;                                 \
+            switch (align) {                                           \
+            case 3:                                                    \
+                *_intmem = 0;                                          \
+                _intmem++;                                             \
+            case 2:                                                    \
+                *_intmem = 0;                                          \
+                _intmem++;                                             \
+            case 1:                                                    \
+                *_intmem = 0;                                          \
+                _intmem++;                                             \
+            }                                                          \
+        } while (0)
 #else
-#define MCA_BTL_SMCUDA_TOUCH_DATA_TILL_CACHELINE_BOUNDARY(sm_frag)
+#    define MCA_BTL_SMCUDA_TOUCH_DATA_TILL_CACHELINE_BOUNDARY(sm_frag)
 #endif
 
 #if 0
         if( OPAL_LIKELY(align > 0) ) {                                  \
             align = 0xFUL - align;                                      \
             memset( _memory, 0, align );                                \
-        }                                                               \
+        }
 
 #endif
 
@@ -889,74 +865,69 @@ struct mca_btl_base_descriptor_t* mca_btl_smcuda_prepare_src(
  * @param btl (IN)      BTL module
  * @param peer (IN)     BTL peer addressing
  */
-int mca_btl_smcuda_sendi( struct mca_btl_base_module_t* btl,
-                      struct mca_btl_base_endpoint_t* endpoint,
-                      struct opal_convertor_t* convertor,
-                      void* header,
-                      size_t header_size,
-                      size_t payload_size,
-                      uint8_t order,
-                      uint32_t flags,
-                      mca_btl_base_tag_t tag,
-                      mca_btl_base_descriptor_t** descriptor )
+int mca_btl_smcuda_sendi(struct mca_btl_base_module_t *btl,
+                         struct mca_btl_base_endpoint_t *endpoint,
+                         struct opal_convertor_t *convertor, void *header, size_t header_size,
+                         size_t payload_size, uint8_t order, uint32_t flags, mca_btl_base_tag_t tag,
+                         mca_btl_base_descriptor_t **descriptor)
 {
     size_t length = (header_size + payload_size);
-    mca_btl_smcuda_frag_t* frag;
+    mca_btl_smcuda_frag_t *frag;
     int rc;
 
-    if ( mca_btl_smcuda_component.num_outstanding_frags * 2 > (int) mca_btl_smcuda_component.fifo_size ) {
+    if (mca_btl_smcuda_component.num_outstanding_frags * 2
+        > (int) mca_btl_smcuda_component.fifo_size) {
         mca_btl_smcuda_component_progress();
     }
-#if OPAL_CUDA_SUPPORT
     /* Initiate setting up CUDA IPC support. */
-    if (mca_common_cuda_enabled && (IPC_INIT == endpoint->ipcstate) && mca_btl_smcuda_component.use_cuda_ipc) {
+
+    if (0 != strcmp(opal_accelerator_base_selected_component.base_version.mca_component_name, "null") && (IPC_INIT == endpoint->ipcstate)
+        && mca_btl_smcuda_component.use_cuda_ipc) {
         mca_btl_smcuda_send_cuda_ipc_request(btl, endpoint);
     }
     /* We do not want to use this path when we have CUDA IPC support */
-    if ((convertor->flags & CONVERTOR_CUDA) && (IPC_ACKED == endpoint->ipcstate)) {
-        if (NULL != descriptor) {
-            *descriptor = mca_btl_smcuda_alloc(btl, endpoint, order, payload_size+header_size, flags);
-        }
-        return OPAL_ERR_RESOURCE_BUSY;
+    if ((NULL != convertor) && (convertor->flags & CONVERTOR_ACCELERATOR) && (IPC_ACKED == endpoint->ipcstate)) {
+        goto return_resource_busy;
     }
-#endif /* OPAL_CUDA_SUPPORT */
 
     /* this check should be unnecessary... turn into an assertion? */
-    if( length < mca_btl_smcuda_component.eager_limit ) {
+    if (length < mca_btl_smcuda_component.eager_limit) {
 
         /* allocate a fragment, giving up if we can't get one */
         /* note that frag==NULL is equivalent to rc returning an error code */
         MCA_BTL_SMCUDA_FRAG_ALLOC_EAGER(frag);
-        if( OPAL_UNLIKELY(NULL == frag) ) {
-            *descriptor = NULL;
-            return OPAL_ERR_OUT_OF_RESOURCE;
+        if (OPAL_UNLIKELY(NULL == frag)) {
+            goto return_resource_busy;
         }
 
         /* fill in fragment fields */
         frag->segment.seg_len = length;
-        frag->hdr->len        = length;
-        assert( 0 == (flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK) );
-        frag->base.des_flags = flags | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP;   /* why do any flags matter here other than OWNERSHIP? */
+        frag->hdr->len = length;
+        assert(0 == (flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK));
+        frag->base.des_flags = flags
+                               | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP; /* why do any flags matter here
+                                                                     other than OWNERSHIP? */
         frag->hdr->tag = tag;
         frag->endpoint = endpoint;
 
         /* write the match header (with MPI comm/tag/etc. info) */
-        memcpy( frag->segment.seg_addr.pval, header, header_size );
+        memcpy(frag->segment.seg_addr.pval, header, header_size);
 
         /* write the message data if there is any */
         /*
           We can add MEMCHECKER calls before and after the packing.
         */
-        if( payload_size ) {
+        if (payload_size) {
             size_t max_data;
             struct iovec iov;
             uint32_t iov_count;
             /* pack the data into the supplied buffer */
-            iov.iov_base = (IOVBASE_TYPE*)((unsigned char*)frag->segment.seg_addr.pval + header_size);
-            iov.iov_len  = max_data = payload_size;
-            iov_count    = 1;
+            iov.iov_base = (IOVBASE_TYPE *) ((unsigned char *) frag->segment.seg_addr.pval
+                                             + header_size);
+            iov.iov_len = max_data = payload_size;
+            iov_count = 1;
 
-            (void)opal_convertor_pack( convertor, &iov, &iov_count, &max_data);
+            (void) opal_convertor_pack(convertor, &iov, &iov_count, &max_data);
 
             assert(max_data == payload_size);
         }
@@ -970,15 +941,16 @@ int mca_btl_smcuda_sendi( struct mca_btl_base_module_t* btl,
          * our point of view:  it has been posted to a "pending send" queue.
          */
         OPAL_THREAD_ADD_FETCH32(&mca_btl_smcuda_component.num_outstanding_frags, +1);
-        MCA_BTL_SMCUDA_FIFO_WRITE(endpoint, endpoint->my_smp_rank,
-                              endpoint->peer_smp_rank, (void *) VIRTUAL2RELATIVE(frag->hdr), false, true, rc);
-        (void)rc; /* this is safe to ignore as the message is requeued till success */
+        MCA_BTL_SMCUDA_FIFO_WRITE(endpoint, endpoint->my_smp_rank, endpoint->peer_smp_rank,
+                                  (void *) VIRTUAL2RELATIVE(frag->hdr), false, true, rc);
+        (void) rc; /* this is safe to ignore as the message is requeued till success */
         return OPAL_SUCCESS;
     }
 
-    /* presumably, this code path will never get executed */
-    *descriptor = mca_btl_smcuda_alloc( btl, endpoint, order,
-                                    payload_size + header_size, flags);
+  return_resource_busy:
+    if (NULL != descriptor) {
+        *descriptor = mca_btl_smcuda_alloc(btl, endpoint, order, length, flags);
+    }
     return OPAL_ERR_RESOURCE_BUSY;
 }
 
@@ -988,23 +960,21 @@ int mca_btl_smcuda_sendi( struct mca_btl_base_module_t* btl,
  * @param btl (IN)      BTL module
  * @param peer (IN)     BTL peer addressing
  */
-int mca_btl_smcuda_send( struct mca_btl_base_module_t* btl,
-                     struct mca_btl_base_endpoint_t* endpoint,
-                     struct mca_btl_base_descriptor_t* descriptor,
-                     mca_btl_base_tag_t tag )
+int mca_btl_smcuda_send(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+                        struct mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag)
 {
-    mca_btl_smcuda_frag_t* frag = (mca_btl_smcuda_frag_t*)descriptor;
+    mca_btl_smcuda_frag_t *frag = (mca_btl_smcuda_frag_t *) descriptor;
     int rc;
 
-    if ( mca_btl_smcuda_component.num_outstanding_frags * 2 > (int) mca_btl_smcuda_component.fifo_size ) {
+    if (mca_btl_smcuda_component.num_outstanding_frags * 2
+        > (int) mca_btl_smcuda_component.fifo_size) {
         mca_btl_smcuda_component_progress();
     }
-#if OPAL_CUDA_SUPPORT
     /* Initiate setting up CUDA IPC support */
-    if (mca_common_cuda_enabled && (IPC_INIT == endpoint->ipcstate) && mca_btl_smcuda_component.use_cuda_ipc) {
+    if (0 != strcmp(opal_accelerator_base_selected_component.base_version.mca_component_name, "null") && (IPC_INIT == endpoint->ipcstate)
+        && mca_btl_smcuda_component.use_cuda_ipc) {
         mca_btl_smcuda_send_cuda_ipc_request(btl, endpoint);
     }
-#endif /* OPAL_CUDA_SUPPORT */
 
     /* available header space */
     frag->hdr->len = frag->segment.seg_len;
@@ -1020,10 +990,10 @@ int mca_btl_smcuda_send( struct mca_btl_base_module_t* btl,
      * address
      */
     OPAL_THREAD_ADD_FETCH32(&mca_btl_smcuda_component.num_outstanding_frags, +1);
-    MCA_BTL_SMCUDA_FIFO_WRITE(endpoint, endpoint->my_smp_rank,
-                          endpoint->peer_smp_rank, (void *) VIRTUAL2RELATIVE(frag->hdr), false, true, rc);
-    if( OPAL_LIKELY(0 == rc) ) {
-        return 1;  /* the data is completely gone */
+    MCA_BTL_SMCUDA_FIFO_WRITE(endpoint, endpoint->my_smp_rank, endpoint->peer_smp_rank,
+                              (void *) VIRTUAL2RELATIVE(frag->hdr), false, true, rc);
+    if (OPAL_LIKELY(0 == rc)) {
+        return 1; /* the data is completely gone */
     }
     frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
     /* not yet gone, but pending. Let the upper level knows that
@@ -1032,22 +1002,23 @@ int mca_btl_smcuda_send( struct mca_btl_base_module_t* btl,
     return 0;
 }
 
-#if OPAL_CUDA_SUPPORT
-static struct mca_btl_base_registration_handle_t *mca_btl_smcuda_register_mem (
-    struct mca_btl_base_module_t* btl, struct mca_btl_base_endpoint_t *endpoint, void *base,
-    size_t size, uint32_t flags)
+static struct mca_btl_base_registration_handle_t *
+mca_btl_smcuda_register_mem(struct mca_btl_base_module_t *btl,
+                            struct mca_btl_base_endpoint_t *endpoint, void *base, size_t size,
+                            uint32_t flags)
 {
     mca_btl_smcuda_t *smcuda_module = (mca_btl_smcuda_t *) btl;
-    mca_rcache_common_cuda_reg_t *reg;
+    mca_opal_gpu_reg_t *reg;
     int access_flags = flags & MCA_BTL_REG_FLAG_ACCESS_ANY;
     int rcache_flags = 0;
 
+#if OPAL_CUDA_GDR_SUPPORT
     if (MCA_BTL_REG_FLAG_CUDA_GPU_MEM & flags) {
-        rcache_flags |= MCA_RCACHE_FLAGS_CUDA_GPU_MEM;
+        rcache_flags |= MCA_RCACHE_FLAGS_ACCELERATOR_MEM;
     }
-
-    smcuda_module->rcache->rcache_register (smcuda_module->rcache, base, size, rcache_flags,
-                                            access_flags, (mca_rcache_base_registration_t **) &reg);
+#endif
+    smcuda_module->rcache->rcache_register(smcuda_module->rcache, base, size, rcache_flags,
+                                           access_flags, (mca_rcache_base_registration_t **) &reg);
     if (OPAL_UNLIKELY(NULL == reg)) {
         return NULL;
     }
@@ -1055,34 +1026,92 @@ static struct mca_btl_base_registration_handle_t *mca_btl_smcuda_register_mem (
     return (mca_btl_base_registration_handle_t *) &reg->data;
 }
 
-static int mca_btl_smcuda_deregister_mem (struct mca_btl_base_module_t* btl,
-                                          struct mca_btl_base_registration_handle_t *handle)
+static int mca_btl_smcuda_deregister_mem(struct mca_btl_base_module_t *btl,
+                                         struct mca_btl_base_registration_handle_t *handle)
 {
     mca_btl_smcuda_t *smcuda_module = (mca_btl_smcuda_t *) btl;
-    mca_rcache_common_cuda_reg_t *reg = (mca_rcache_common_cuda_reg_t *)
-        ((intptr_t) handle - offsetof (mca_rcache_common_cuda_reg_t, data));
+    mca_opal_gpu_reg_t *reg = (mca_opal_gpu_reg_t *) ((intptr_t) handle
+                                                      - offsetof(mca_opal_gpu_reg_t, data));
 
-    smcuda_module->rcache->rcache_deregister (smcuda_module->rcache, &reg->base);
+    smcuda_module->rcache->rcache_deregister(smcuda_module->rcache, &reg->base);
 
     return OPAL_SUCCESS;
 }
 
-int mca_btl_smcuda_get_cuda (struct mca_btl_base_module_t *btl,
-    struct mca_btl_base_endpoint_t *ep, void *local_address,
-    uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle,
-    struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-    int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
+/*
+ * Put remote event on stream to ensure that the the start of the
+ * copy does not start until the completion of the event.
+ */
+static void mca_btl_smcuda_wait_stream_synchronize(mca_opal_gpu_reg_t *rget_reg)
+{
+    opal_accelerator_ipc_event_handle_t evtHandle;
+    opal_accelerator_event_t event;
+    int result;
+
+    if (opal_accelerator_use_sync_memops) {
+        /* No need for any of this with SYNC_MEMOPS feature */
+        return;
+    }
+
+    result = opal_accelerator.import_ipc_event_handle(rget_reg->data.ipcEventHandle.handle, &evtHandle);
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != result)) {
+        opal_output_verbose(10, mca_btl_smcuda_component.cuda_ipc_output,
+                            "import_ipc_event_handle failed");
+        return;
+    }
+
+    result = opal_accelerator.open_ipc_event_handle(&evtHandle, &event);
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != result)) {
+        opal_output_verbose(10, mca_btl_smcuda_component.cuda_ipc_output,
+                            "open_ipc_event_handle failed");
+        return;
+    }
+
+#if 0
+    /* BEGIN of Workaround to deal with a bug in an early CUDA releases  (4.1. an older)
+     * Need to record an event on the stream, even though
+     * it is not used, to make sure we do not short circuit our way
+     * out of the cuStreamWaitEvent test.
+     */
+    result = opal_accelerator.record_event(MCA_ACCELERATOR_NO_DEVICE_ID, &event, MCA_ACCELERATOR_STREAM_DEFAULT);
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != result)) {
+        opal_output_verbose(10, mca_btl_smcuda_component.cuda_ipc_output,
+                            "record_event failed");
+        return;
+    }
+    /* END of Workaround */
+#endif
+
+    result = opal_accelerator.wait_event(MCA_ACCELERATOR_NO_DEVICE_ID, &event, MCA_ACCELERATOR_STREAM_DEFAULT);
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != result)) {
+        opal_output_verbose(10, mca_btl_smcuda_component.cuda_ipc_output,
+                            "wait_event failed");
+        return;
+    }
+
+    // ipc event are assumed to be static, hence no OBJ_RELEASE
+    // but OBJ_DESTRUCT here.
+    OBJ_DESTRUCT(&event);
+    return;
+}
+
+int mca_btl_smcuda_get_cuda(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *ep,
+                            void *local_address, uint64_t remote_address,
+                            struct mca_btl_base_registration_handle_t *local_handle,
+                            struct mca_btl_base_registration_handle_t *remote_handle, size_t size,
+                            int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
+                            void *cbcontext, void *cbdata)
 {
-    mca_rcache_common_cuda_reg_t rget_reg;
-    mca_rcache_common_cuda_reg_t *reg_ptr = &rget_reg;
-    int rc, done;
+    mca_opal_gpu_reg_t rget_reg;
+    mca_opal_gpu_reg_t *reg_ptr = &rget_reg;
+    int rc;
     void *remote_memory_address;
     size_t offset;
     mca_btl_smcuda_frag_t *frag;
 
     /* NTH: copied from old prepare_dst function */
     MCA_BTL_SMCUDA_FRAG_ALLOC_USER(frag);
-    if(OPAL_UNLIKELY(NULL == frag)) {
+    if (OPAL_UNLIKELY(NULL == frag)) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
@@ -1098,17 +1127,18 @@ int mca_btl_smcuda_get_cuda (struct mca_btl_base_module_t *btl,
     frag->local_handle = local_handle;
 
     /* Set to 0 for debugging since it is a list item but I am not
-     * intializing it properly and it is annoying to see all the
+     * initializing it properly and it is annoying to see all the
      * garbage in the debugger.  */
 
     memset(&rget_reg, 0, sizeof(rget_reg));
-    memcpy(&rget_reg.data.memHandle, remote_handle->reg_data.memHandle,
-           sizeof(remote_handle->reg_data.memHandle));
-#if !OPAL_CUDA_SYNC_MEMOPS
-    /* Only need the remote event handle when syncing with remote events */
-    memcpy(&rget_reg.data.evtHandle, remote_handle->reg_data.evtHandle,
-           sizeof(remote_handle->reg_data.evtHandle));
-#endif
+    memcpy(&rget_reg.data.ipcHandle.handle, remote_handle->reg_data.ipcHandle.handle,
+           sizeof(remote_handle->reg_data.ipcHandle.handle));
+
+    if (!opal_accelerator_use_sync_memops) {
+        /* Only need the remote event handle when syncing with remote events */
+        memcpy(&rget_reg.data.ipcEventHandle.handle, remote_handle->reg_data.ipcEventHandle.handle,
+               sizeof(remote_handle->reg_data.ipcEventHandle.handle));
+    }
 
     /* Open the memory handle to the remote memory.  If it is cached, then
      * we just retrieve it from cache and avoid a call to open the handle.  That
@@ -1117,16 +1147,16 @@ int mca_btl_smcuda_get_cuda (struct mca_btl_base_module_t *btl,
      * remote memory which may lie somewhere in the middle. This is taken care of
      * a few lines down. Note that we hand in the peer rank just for debugging
      * support. */
-    rc = ep->rcache->rcache_register (ep->rcache, remote_handle->reg_data.memh_seg_addr.pval,
-                                      remote_handle->reg_data.memh_seg_len, ep->peer_smp_rank,
-                                      MCA_RCACHE_ACCESS_LOCAL_WRITE,
-                                      (mca_rcache_base_registration_t **)&reg_ptr);
+    rc = ep->rcache->rcache_register(ep->rcache, remote_handle->reg_data.memh_seg_addr.pval,
+                                     remote_handle->reg_data.memh_seg_len, ep->peer_smp_rank,
+                                     MCA_RCACHE_ACCESS_LOCAL_WRITE,
+                                     (mca_rcache_base_registration_t **) &reg_ptr);
 
     if (OPAL_SUCCESS != rc) {
         opal_output(0, "Failed to register remote memory, rc=%d", rc);
         return rc;
     }
-    frag->registration = (mca_rcache_base_registration_t *)reg_ptr;
+    frag->registration = (mca_rcache_base_registration_t *) reg_ptr;
     frag->endpoint = ep;
 
     /* The registration has given us back the memory block that this
@@ -1134,10 +1164,10 @@ int mca_btl_smcuda_get_cuda (struct mca_btl_base_module_t *btl,
      * not equal the address that was used to retrieve the block.
      * Therefore, compute the offset and add it to the address of the
      * memory handle. */
-    offset = (size_t) ((intptr_t) remote_address - (intptr_t) reg_ptr->base.base);
-    remote_memory_address = (unsigned char *)reg_ptr->base.alloc_base + offset;
+    offset = (size_t)((intptr_t) remote_address - (intptr_t) reg_ptr->base.base);
+    remote_memory_address = (unsigned char *) reg_ptr->base.alloc_base + offset;
     if (0 != offset) {
-        opal_output(-1, "OFFSET=%d", (int)offset);
+        opal_output(-1, "OFFSET=%d", (int) offset);
     }
 
     /* The remote side posted an IPC event to make sure we do not start our
@@ -1145,11 +1175,10 @@ int mca_btl_smcuda_get_cuda (struct mca_btl_base_module_t *btl,
      * is available in the sender's GPU buffer.  Therefore, do a stream synchronize
      * on the IPC event that we received.  Note that we pull it from
      * rget_reg, not reg_ptr, as we do not cache the event. */
-    mca_common_wait_stream_synchronize(&rget_reg);
+    mca_btl_smcuda_wait_stream_synchronize(&rget_reg);
 
-    rc = mca_common_cuda_memcpy(local_address, remote_memory_address, size,
-                                "mca_btl_smcuda_get", (mca_btl_base_descriptor_t *)frag,
-                                &done);
+    rc = mca_btl_smcuda_memcpy(local_address, remote_memory_address, size, "mca_btl_smcuda_get",
+                                (mca_btl_base_descriptor_t *) frag);
     if (OPAL_SUCCESS != rc) {
         /* Out of resources can be handled by upper layers. */
         if (OPAL_ERR_OUT_OF_RESOURCE != rc) {
@@ -1158,13 +1187,7 @@ int mca_btl_smcuda_get_cuda (struct mca_btl_base_module_t *btl,
         return rc;
     }
 
-    if (OPAL_UNLIKELY(1 == done)) {
-        cbfunc (btl, ep, local_address, local_handle, cbcontext, cbdata, OPAL_SUCCESS);
-        mca_btl_smcuda_free(btl, (mca_btl_base_descriptor_t *)frag);
-    }
-
     return OPAL_SUCCESS;
-
 }
 
 /**
@@ -1176,11 +1199,11 @@ int mca_btl_smcuda_get_cuda (struct mca_btl_base_module_t *btl,
  * @param btl (IN)      BTL module
  * @param peer (IN)     BTL peer addressing
  */
-#define MAXTRIES 5
-static void mca_btl_smcuda_send_cuda_ipc_request(struct mca_btl_base_module_t* btl,
-                                                 struct mca_btl_base_endpoint_t* endpoint)
+#    define MAXTRIES 5
+static void mca_btl_smcuda_send_cuda_ipc_request(struct mca_btl_base_module_t *btl,
+                                                 struct mca_btl_base_endpoint_t *endpoint)
 {
-    mca_btl_smcuda_frag_t* frag;
+    mca_btl_smcuda_frag_t *frag;
     int rc, mydevnum, res;
     ctrlhdr_t ctrlhdr;
 
@@ -1201,11 +1224,12 @@ static void mca_btl_smcuda_send_cuda_ipc_request(struct mca_btl_base_module_t* b
     endpoint->ipcstate = IPC_SENT;
     OPAL_THREAD_UNLOCK(&endpoint->endpoint_lock);
 
-    if ( mca_btl_smcuda_component.num_outstanding_frags * 2 > (int) mca_btl_smcuda_component.fifo_size ) {
+    if (mca_btl_smcuda_component.num_outstanding_frags * 2
+        > (int) mca_btl_smcuda_component.fifo_size) {
         mca_btl_smcuda_component_progress();
     }
 
-    if (0 != (res = mca_common_cuda_get_device(&mydevnum))) {
+    if (0 != (res = opal_accelerator.get_device(&mydevnum))) {
         opal_output(0, "Cannot determine device.  IPC cannot be set.");
         endpoint->ipcstate = IPC_BAD;
         return;
@@ -1213,7 +1237,7 @@ static void mca_btl_smcuda_send_cuda_ipc_request(struct mca_btl_base_module_t* b
 
     /* allocate a fragment, giving up if we can't get one */
     MCA_BTL_SMCUDA_FRAG_ALLOC_EAGER(frag);
-    if( OPAL_UNLIKELY(NULL == frag) ) {
+    if (OPAL_UNLIKELY(NULL == frag)) {
         endpoint->ipcstate = IPC_BAD;
         return;
     }
@@ -1235,37 +1259,31 @@ static void mca_btl_smcuda_send_cuda_ipc_request(struct mca_btl_base_module_t* b
      */
     OPAL_THREAD_ADD_FETCH32(&mca_btl_smcuda_component.num_outstanding_frags, +1);
     opal_output_verbose(10, mca_btl_smcuda_component.cuda_ipc_output,
-                        "Sending CUDA IPC REQ (try=%d): myrank=%d, mydev=%d, peerrank=%d",
-                        endpoint->ipctries,
-                        mca_btl_smcuda_component.my_smp_rank,
-                        mydevnum, endpoint->peer_smp_rank);
+                        "Sending IPC REQ (try=%d): myrank=%d, mydev=%d, peerrank=%d",
+                        endpoint->ipctries, mca_btl_smcuda_component.my_smp_rank, mydevnum,
+                        endpoint->peer_smp_rank);
 
-    MCA_BTL_SMCUDA_FIFO_WRITE(endpoint, endpoint->my_smp_rank,
-                              endpoint->peer_smp_rank, (void *) VIRTUAL2RELATIVE(frag->hdr), false, true, rc);
+    MCA_BTL_SMCUDA_FIFO_WRITE(endpoint, endpoint->my_smp_rank, endpoint->peer_smp_rank,
+                              (void *) VIRTUAL2RELATIVE(frag->hdr), false, true, rc);
+    (void)rc;
     return;
-
 }
 
-#endif /* OPAL_CUDA_SUPPORT */
-
 /**
  *
  */
-void mca_btl_smcuda_dump(struct mca_btl_base_module_t* btl,
-                     struct mca_btl_base_endpoint_t* endpoint,
-                     int verbose)
+void mca_btl_smcuda_dump(struct mca_btl_base_module_t *btl,
+                         struct mca_btl_base_endpoint_t *endpoint, int verbose)
 {
-    mca_btl_smcuda_frag_t* frag;
+    mca_btl_smcuda_frag_t *frag;
 
-    mca_btl_base_err("BTL SM %p endpoint %p [smp_rank %d] [peer_rank %d]\n",
-                     (void*) btl, (void*) endpoint,
-                     endpoint->my_smp_rank, endpoint->peer_smp_rank);
-    if( NULL != endpoint ) {
-        OPAL_LIST_FOREACH(frag, &endpoint->pending_sends, mca_btl_smcuda_frag_t) {
+    mca_btl_base_err("BTL SM %p endpoint %p [smp_rank %d] [peer_rank %d]\n", (void *) btl,
+                     (void *) endpoint, endpoint->my_smp_rank, endpoint->peer_smp_rank);
+    if (NULL != endpoint) {
+        OPAL_LIST_FOREACH (frag, &endpoint->pending_sends, mca_btl_smcuda_frag_t) {
             mca_btl_base_err(" |  frag %p size %lu (hdr frag %p len %lu rank %d tag %d)\n",
-                             (void*) frag, frag->size, (void*) frag->hdr->frag,
-                             frag->hdr->len, frag->hdr->my_smp_rank,
-                             frag->hdr->tag);
+                             (void *) frag, frag->size, (void *) frag->hdr->frag, frag->hdr->len,
+                             frag->hdr->my_smp_rank, frag->hdr->tag);
         }
     }
 }
diff --git a/opal/mca/btl/smcuda/btl_smcuda.h b/opal/mca/btl/smcuda/btl_smcuda.h
index e930e15381a..812e04ea7a6 100644
--- a/opal/mca/btl/smcuda/btl_smcuda.h
+++ b/opal/mca/btl/smcuda/btl_smcuda.h
@@ -14,7 +14,8 @@
  * Copyright (c) 2009-2010 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2010-2015 Los Alamos National Security, LLC.
  *                         All rights reserved.
- * Copyright (c) 2012-2013 NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2012-2023 NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2024      Advanced Micro Devices, Inc. All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -29,17 +30,17 @@
 
 #include "opal_config.h"
 #include <stddef.h>
+#include <stdint.h>
 #include <stdlib.h>
 #include <string.h>
-#include <stdint.h>
 #ifdef HAVE_SCHED_H
-#include <sched.h>
-#endif  /* HAVE_SCHED_H */
+#    include <sched.h>
+#endif /* HAVE_SCHED_H */
 
-#include "opal/util/bit_ops.h"
 #include "opal/class/opal_free_list.h"
 #include "opal/mca/btl/btl.h"
 #include "opal/mca/common/sm/common_sm.h"
+#include "opal/util/bit_ops.h"
 
 BEGIN_C_DECLS
 
@@ -71,7 +72,7 @@ BEGIN_C_DECLS
  * cachelines.
  */
 
-#define SM_FIFO_FREE  (void *) (-2)
+#define SM_FIFO_FREE (void *) (-2)
 /* We can't use opal_cache_line_size here because we need a
    compile-time constant for padding the struct.  We can't really have
    a compile-time constant that is portable, either (e.g., compile on
@@ -98,83 +99,81 @@ struct sm_fifo_t {
     volatile int tail;
     int num_to_clear;
     int lazy_free;
-    char pad4[SM_CACHE_LINE_PAD - sizeof(void **) -
-              sizeof(opal_atomic_lock_t) -
-              sizeof(int) * 3];
+    char pad4[SM_CACHE_LINE_PAD - sizeof(void **) - sizeof(opal_atomic_lock_t) - sizeof(int) * 3];
 };
 typedef struct sm_fifo_t sm_fifo_t;
 
 /*
- * Shared Memory resource managment
+ * Shared Memory resource management
  */
 
 #if OPAL_ENABLE_PROGRESS_THREADS == 1
-#define DATA (char)0
-#define DONE (char)1
+#    define DATA (char) 0
+#    define DONE (char) 1
 #endif
 
 typedef struct mca_btl_smcuda_mem_node_t {
-    mca_mpool_base_module_t* sm_mpool; /**< shared memory pool */
+    mca_mpool_base_module_t *sm_mpool; /**< shared memory pool */
 } mca_btl_smcuda_mem_node_t;
 
 /**
  * Shared Memory (SM) BTL module.
  */
 struct mca_btl_smcuda_component_t {
-    mca_btl_base_component_2_0_0_t super;  /**< base BTL component */
-    int sm_free_list_num;              /**< initial size of free lists */
-    int sm_free_list_max;              /**< maximum size of free lists */
-    int sm_free_list_inc;              /**< number of elements to alloc when growing free lists */
-    int sm_max_procs;                  /**< upper limit on the number of processes using the shared memory pool */
-    int sm_extra_procs;                /**< number of extra procs to allow */
-    char* sm_mpool_name;               /**< name of shared memory pool module */
+    mca_btl_base_component_2_0_0_t super; /**< base BTL component */
+    int sm_free_list_num;                 /**< initial size of free lists */
+    int sm_free_list_max;                 /**< maximum size of free lists */
+    int sm_free_list_inc; /**< number of elements to alloc when growing free lists */
+    int sm_max_procs;    /**< upper limit on the number of processes using the shared memory pool */
+    int sm_extra_procs;  /**< number of extra procs to allow */
+    char *sm_mpool_name; /**< name of shared memory pool module */
     mca_mpool_base_module_t **sm_mpools; /**< shared memory pools (one for each memory node) */
-    mca_mpool_base_module_t *sm_mpool; /**< mpool on local node */
-    void* sm_mpool_base;               /**< base address of shared memory pool */
-    size_t eager_limit;                /**< first fragment size */
-    size_t max_frag_size;              /**< maximum (second and beyone) fragment size */
+    mca_mpool_base_module_t *sm_mpool;   /**< mpool on local node */
+    void *sm_mpool_base;                 /**< base address of shared memory pool */
+    size_t eager_limit;                  /**< first fragment size */
+    size_t max_frag_size;                /**< maximum (second and beyond) fragment size */
     opal_mutex_t sm_lock;
-    mca_common_sm_module_t *sm_seg;   /**< description of shared memory segment */
-    volatile sm_fifo_t **shm_fifo;     /**< pointer to fifo 2D array in shared memory */
-    char **shm_bases;                  /**< pointer to base pointers in shared memory */
-    uint16_t *shm_mem_nodes;           /**< pointer to mem noded in shared memory */
-    sm_fifo_t **fifo;                  /**< cached copy of the pointer to the 2D
-                                          fifo array.  The address in the shared
-                                          memory segment sm_ctl_header is a relative,
-                                          but this one, in process private memory, is
-                                          a real virtual address */
-    uint16_t *mem_nodes;               /**< cached copy of mem nodes of each local rank */
-    unsigned int fifo_size;            /**< number of FIFO queue entries */
-    unsigned int fifo_lazy_free;       /**< number of reads before lazy fifo free is triggered */
-    int nfifos;                        /**< number of FIFOs per receiver */
-    int32_t num_smp_procs;             /**< current number of smp procs on this host */
-    int32_t my_smp_rank;               /**< My SMP process rank.  Used for accessing
-                                        *   SMP specfic data structures. */
-    opal_free_list_t sm_frags_eager;   /**< free list of sm first */
-    opal_free_list_t sm_frags_max;     /**< free list of sm second */
+    mca_common_sm_module_t *sm_seg;  /**< description of shared memory segment */
+    volatile sm_fifo_t **shm_fifo;   /**< pointer to fifo 2D array in shared memory */
+    char **shm_bases;                /**< pointer to base pointers in shared memory */
+    uint16_t *shm_mem_nodes;         /**< pointer to mem noded in shared memory */
+    sm_fifo_t **fifo;                /**< cached copy of the pointer to the 2D
+                                        fifo array.  The address in the shared
+                                        memory segment sm_ctl_header is a relative,
+                                        but this one, in process private memory, is
+                                        a real virtual address */
+    uint16_t *mem_nodes;             /**< cached copy of mem nodes of each local rank */
+    unsigned int fifo_size;          /**< number of FIFO queue entries */
+    unsigned int fifo_lazy_free;     /**< number of reads before lazy fifo free is triggered */
+    int nfifos;                      /**< number of FIFOs per receiver */
+    int32_t num_smp_procs;           /**< current number of smp procs on this host */
+    int32_t my_smp_rank;             /**< My SMP process rank.  Used for accessing
+                                      *   SMP specific data structures. */
+    opal_free_list_t sm_frags_eager; /**< free list of sm first */
+    opal_free_list_t sm_frags_max;   /**< free list of sm second */
     opal_free_list_t sm_frags_user;
-    opal_free_list_t sm_first_frags_to_progress;  /**< list of first
-                                                    fragments that are
-                                                    awaiting resources */
+    opal_free_list_t sm_first_frags_to_progress; /**< list of first
+                                                   fragments that are
+                                                   awaiting resources */
     struct mca_btl_base_endpoint_t **sm_peers;
 
     opal_free_list_t pending_send_fl;
-    opal_atomic_int32_t num_outstanding_frags;         /**< number of fragments sent but not yet returned to free list */
-    opal_atomic_int32_t num_pending_sends;             /**< total number on all of my pending-send queues */
+    opal_atomic_int32_t
+        num_outstanding_frags; /**< number of fragments sent but not yet returned to free list */
+    opal_atomic_int32_t num_pending_sends; /**< total number on all of my pending-send queues */
     int mem_node;
     int num_mem_nodes;
 
 #if OPAL_ENABLE_PROGRESS_THREADS == 1
-    char sm_fifo_path[PATH_MAX];   /**< path to fifo used to signal this process */
-    int  sm_fifo_fd;               /**< file descriptor corresponding to opened fifo */
+    char sm_fifo_path[OPAL_PATH_MAX]; /**< path to fifo used to signal this process */
+    int sm_fifo_fd;              /**< file descriptor corresponding to opened fifo */
     opal_thread_t sm_fifo_thread;
 #endif
-    struct mca_btl_smcuda_t      **sm_btls;
+    struct mca_btl_smcuda_t **sm_btls;
     struct mca_btl_smcuda_frag_t **table;
     size_t sm_num_btls;
     size_t sm_max_btls;
 
-
     /** MCA: should we be using knem or not?  neg=try but continue if
         not available, 0=don't try, 1=try and fail if not available */
     int use_knem;
@@ -200,32 +199,34 @@ struct mca_btl_smcuda_component_t {
     char *sm_mpool_rndv_file_name;
     char *sm_ctl_file_name;
     char *sm_rndv_file_name;
-#if OPAL_CUDA_SUPPORT
+
     int cuda_ipc_verbose;
     int cuda_ipc_output;
     int use_cuda_ipc;
     int use_cuda_ipc_same_gpu;
-#endif /* OPAL_CUDA_SUPPORT */
+
+    int accelerator_delayed_ipc_init;
+    int accelerator_max_ipc_events;
+
     unsigned long mpool_min_size;
     char *allocator;
 };
 typedef struct mca_btl_smcuda_component_t mca_btl_smcuda_component_t;
-OPAL_MODULE_DECLSPEC extern mca_btl_smcuda_component_t mca_btl_smcuda_component;
+OPAL_DECLSPEC extern mca_btl_smcuda_component_t mca_btl_smcuda_component;
 
 /**
  * SM BTL Interface
  */
 struct mca_btl_smcuda_t {
-    mca_btl_base_module_t  super;       /**< base BTL interface */
-    bool btl_inited;  /**< flag indicating if btl has been inited */
+    mca_btl_base_module_t super; /**< base BTL interface */
+    bool btl_inited;             /**< flag indicating if btl has been inited */
     mca_btl_base_module_error_cb_fn_t error_cb;
     mca_rcache_base_module_t *rcache;
 };
 typedef struct mca_btl_smcuda_t mca_btl_smcuda_t;
-OPAL_MODULE_DECLSPEC extern mca_btl_smcuda_t mca_btl_smcuda;
+OPAL_DECLSPEC extern mca_btl_smcuda_t mca_btl_smcuda;
 
-struct btl_smcuda_pending_send_item_t
-{
+struct btl_smcuda_pending_send_item_t {
     opal_free_list_item_t super;
     void *data;
 };
@@ -243,26 +244,30 @@ typedef struct btl_smcuda_pending_send_item_t btl_smcuda_pending_send_item_t;
  * we define macros to translate between relative addresses and
  * virtual addresses.
  */
-#define VIRTUAL2RELATIVE(VADDR ) ((long)(VADDR)  - (long)mca_btl_smcuda_component.shm_bases[mca_btl_smcuda_component.my_smp_rank])
-#define RELATIVE2VIRTUAL(OFFSET) ((long)(OFFSET) + (long)mca_btl_smcuda_component.shm_bases[mca_btl_smcuda_component.my_smp_rank])
-
-static inline int sm_fifo_init(int fifo_size, mca_mpool_base_module_t *mpool,
-                               sm_fifo_t *fifo, int lazy_free)
+#define VIRTUAL2RELATIVE(VADDR) \
+    ((long) (VADDR)             \
+     - (long) mca_btl_smcuda_component.shm_bases[mca_btl_smcuda_component.my_smp_rank])
+#define RELATIVE2VIRTUAL(OFFSET) \
+    ((long) (OFFSET)             \
+     + (long) mca_btl_smcuda_component.shm_bases[mca_btl_smcuda_component.my_smp_rank])
+
+static inline int sm_fifo_init(int fifo_size, mca_mpool_base_module_t *mpool, sm_fifo_t *fifo,
+                               int lazy_free)
 {
     int i, qsize;
 
     /* figure out the queue size (a power of two that is at least 1) */
-    qsize = opal_next_poweroftwo_inclusive (fifo_size);
+    qsize = opal_next_poweroftwo_inclusive(fifo_size);
 
     /* allocate the queue in the receiver's address space */
-    fifo->queue_recv = (volatile void **)mpool->mpool_alloc(
-            mpool, sizeof(void *) * qsize, opal_cache_line_size, 0);
-    if(NULL == fifo->queue_recv) {
+    fifo->queue_recv = (volatile void **) mpool->mpool_alloc(mpool, sizeof(void *) * qsize,
+                                                             opal_cache_line_size, 0);
+    if (NULL == fifo->queue_recv) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
     /* initialize the queue */
-    for ( i = 0; i < qsize; i++ )
+    for (i = 0; i < qsize; i++)
         fifo->queue_recv[i] = SM_FIFO_FREE;
 
     /* shift queue address to be relative */
@@ -271,8 +276,8 @@ static inline int sm_fifo_init(int fifo_size, mca_mpool_base_module_t *mpool,
     /* initialize the locks */
     opal_atomic_lock_init(&(fifo->head_lock), OPAL_ATOMIC_LOCK_UNLOCKED);
     opal_atomic_lock_init(&(fifo->tail_lock), OPAL_ATOMIC_LOCK_UNLOCKED);
-    opal_atomic_unlock(&(fifo->head_lock));  /* should be unnecessary */
-    opal_atomic_unlock(&(fifo->tail_lock));  /* should be unnecessary */
+    opal_atomic_unlock(&(fifo->head_lock)); /* should be unnecessary */
+    opal_atomic_unlock(&(fifo->tail_lock)); /* should be unnecessary */
 
     /* other initializations */
     fifo->head = 0;
@@ -284,14 +289,13 @@ static inline int sm_fifo_init(int fifo_size, mca_mpool_base_module_t *mpool,
     return OPAL_SUCCESS;
 }
 
-
 static inline int sm_fifo_write(void *value, sm_fifo_t *fifo)
 {
     volatile void **q = (volatile void **) RELATIVE2VIRTUAL(fifo->queue);
 
     /* if there is no free slot to write, report exhausted resource */
     opal_atomic_rmb();
-    if ( SM_FIFO_FREE != q[fifo->head] )
+    if (SM_FIFO_FREE != q[fifo->head])
         return OPAL_ERR_OUT_OF_RESOURCE;
 
     /* otherwise, write to the slot and advance the head index */
@@ -301,7 +305,6 @@ static inline int sm_fifo_write(void *value, sm_fifo_t *fifo)
     return OPAL_SUCCESS;
 }
 
-
 static inline void *sm_fifo_read(sm_fifo_t *fifo)
 {
     void *value;
@@ -312,18 +315,18 @@ static inline void *sm_fifo_read(sm_fifo_t *fifo)
     opal_atomic_rmb();
 
     /* if you read a non-empty slot, advance the tail pointer */
-    if ( SM_FIFO_FREE != value ) {
+    if (SM_FIFO_FREE != value) {
 
-        fifo->tail = ( fifo->tail + 1 ) & fifo->mask;
+        fifo->tail = (fifo->tail + 1) & fifo->mask;
         fifo->num_to_clear += 1;
 
         /* check if it's time to free slots, which we do lazily */
-        if ( fifo->num_to_clear >= fifo->lazy_free ) {
-            int i = (fifo->tail - fifo->num_to_clear ) & fifo->mask;
+        if (fifo->num_to_clear >= fifo->lazy_free) {
+            int i = (fifo->tail - fifo->num_to_clear) & fifo->mask;
 
-            while ( fifo->num_to_clear > 0 ) {
+            while (fifo->num_to_clear > 0) {
                 fifo->queue_recv[i] = SM_FIFO_FREE;
-                i = (i+1) & fifo->mask;
+                i = (i + 1) & fifo->mask;
                 fifo->num_to_clear -= 1;
             }
             opal_atomic_wmb();
@@ -338,8 +341,6 @@ static inline void *sm_fifo_read(sm_fifo_t *fifo)
  */
 extern int mca_btl_smcuda_component_progress(void);
 
-
-
 /**
  * Register a callback function that is called on error..
  *
@@ -347,10 +348,8 @@ extern int mca_btl_smcuda_component_progress(void);
  * @return             Status indicating if cleanup was successful
  */
 
-int mca_btl_smcuda_register_error_cb(
-    struct mca_btl_base_module_t* btl,
-    mca_btl_base_module_error_cb_fn_t cbfunc
-);
+int mca_btl_smcuda_register_error_cb(struct mca_btl_base_module_t *btl,
+                                     mca_btl_base_module_error_cb_fn_t cbfunc);
 
 /**
  * Cleanup any resources held by the BTL.
@@ -359,10 +358,7 @@ int mca_btl_smcuda_register_error_cb(
  * @return     OPAL_SUCCESS or error status on failure.
  */
 
-extern int mca_btl_smcuda_finalize(
-    struct mca_btl_base_module_t* btl
-);
-
+extern int mca_btl_smcuda_finalize(struct mca_btl_base_module_t *btl);
 
 /**
  * PML->BTL notification of change in the process list.
@@ -378,14 +374,10 @@ extern int mca_btl_smcuda_finalize(
  *
  */
 
-extern int mca_btl_smcuda_add_procs(
-    struct mca_btl_base_module_t* btl,
-    size_t nprocs,
-    struct opal_proc_t **procs,
-    struct mca_btl_base_endpoint_t** peers,
-    struct opal_bitmap_t* reachability
-);
-
+extern int mca_btl_smcuda_add_procs(struct mca_btl_base_module_t *btl, size_t nprocs,
+                                    struct opal_proc_t **procs,
+                                    struct mca_btl_base_endpoint_t **peers,
+                                    struct opal_bitmap_t *reachability);
 
 /**
  * PML->BTL notification of change in the process list.
@@ -396,13 +388,9 @@ extern int mca_btl_smcuda_add_procs(
  * @return             Status indicating if cleanup was successful
  *
  */
-extern int mca_btl_smcuda_del_procs(
-    struct mca_btl_base_module_t* btl,
-    size_t nprocs,
-    struct opal_proc_t **procs,
-    struct mca_btl_base_endpoint_t **peers
-);
-
+extern int mca_btl_smcuda_del_procs(struct mca_btl_base_module_t *btl, size_t nprocs,
+                                    struct opal_proc_t **procs,
+                                    struct mca_btl_base_endpoint_t **peers);
 
 /**
  * Allocate a segment.
@@ -410,13 +398,9 @@ extern int mca_btl_smcuda_del_procs(
  * @param btl (IN)      BTL module
  * @param size (IN)     Request segment size.
  */
-extern mca_btl_base_descriptor_t* mca_btl_smcuda_alloc(
-    struct mca_btl_base_module_t* btl,
-    struct mca_btl_base_endpoint_t* endpoint,
-    uint8_t order,
-    size_t size,
-    uint32_t flags
-);
+extern mca_btl_base_descriptor_t *mca_btl_smcuda_alloc(struct mca_btl_base_module_t *btl,
+                                                       struct mca_btl_base_endpoint_t *endpoint,
+                                                       uint8_t order, size_t size, uint32_t flags);
 
 /**
  * Return a segment allocated by this BTL.
@@ -424,11 +408,8 @@ extern mca_btl_base_descriptor_t* mca_btl_smcuda_alloc(
  * @param btl (IN)      BTL module
  * @param segment (IN)  Allocated segment.
  */
-extern int mca_btl_smcuda_free(
-    struct mca_btl_base_module_t* btl,
-    mca_btl_base_descriptor_t* segment
-);
-
+extern int mca_btl_smcuda_free(struct mca_btl_base_module_t *btl,
+                               mca_btl_base_descriptor_t *segment);
 
 /**
  * Pack data
@@ -436,16 +417,11 @@ extern int mca_btl_smcuda_free(
  * @param btl (IN)      BTL module
  * @param peer (IN)     BTL peer addressing
  */
-struct mca_btl_base_descriptor_t* mca_btl_smcuda_prepare_src(
-    struct mca_btl_base_module_t* btl,
-    struct mca_btl_base_endpoint_t* endpoint,
-    struct opal_convertor_t* convertor,
-    uint8_t order,
-    size_t reserve,
-    size_t* size,
-    uint32_t flags
-);
-
+struct mca_btl_base_descriptor_t *
+mca_btl_smcuda_prepare_src(struct mca_btl_base_module_t *btl,
+                           struct mca_btl_base_endpoint_t *endpoint,
+                           struct opal_convertor_t *convertor, uint8_t order, size_t reserve,
+                           size_t *size, uint32_t flags);
 
 /**
  * Initiate an inlined send to the peer or return a descriptor.
@@ -453,16 +429,12 @@ struct mca_btl_base_descriptor_t* mca_btl_smcuda_prepare_src(
  * @param btl (IN)      BTL module
  * @param peer (IN)     BTL peer addressing
  */
-extern int mca_btl_smcuda_sendi( struct mca_btl_base_module_t* btl,
-                             struct mca_btl_base_endpoint_t* endpoint,
-                             struct opal_convertor_t* convertor,
-                             void* header,
-                             size_t header_size,
-                             size_t payload_size,
-                             uint8_t order,
-                             uint32_t flags,
-                             mca_btl_base_tag_t tag,
-                             mca_btl_base_descriptor_t** descriptor );
+extern int mca_btl_smcuda_sendi(struct mca_btl_base_module_t *btl,
+                                struct mca_btl_base_endpoint_t *endpoint,
+                                struct opal_convertor_t *convertor, void *header,
+                                size_t header_size, size_t payload_size, uint8_t order,
+                                uint32_t flags, mca_btl_base_tag_t tag,
+                                mca_btl_base_descriptor_t **descriptor);
 
 /**
  * Initiate a send to the peer.
@@ -470,22 +442,21 @@ extern int mca_btl_smcuda_sendi( struct mca_btl_base_module_t* btl,
  * @param btl (IN)      BTL module
  * @param peer (IN)     BTL peer addressing
  */
-extern int mca_btl_smcuda_send(
-    struct mca_btl_base_module_t* btl,
-    struct mca_btl_base_endpoint_t* endpoint,
-    struct mca_btl_base_descriptor_t* descriptor,
-    mca_btl_base_tag_t tag
-);
-
-#if OPAL_CUDA_SUPPORT
+extern int mca_btl_smcuda_send(struct mca_btl_base_module_t *btl,
+                               struct mca_btl_base_endpoint_t *endpoint,
+                               struct mca_btl_base_descriptor_t *descriptor,
+                               mca_btl_base_tag_t tag);
+
+
 /**
  * Remote get using device memory.
  */
-int mca_btl_smcuda_get_cuda (struct mca_btl_base_module_t *btl,
-    struct mca_btl_base_endpoint_t *ep, void *local_address,
-    uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle,
-    struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-    int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
+int mca_btl_smcuda_get_cuda(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *ep,
+                            void *local_address, uint64_t remote_address,
+                            struct mca_btl_base_registration_handle_t *local_handle,
+                            struct mca_btl_base_registration_handle_t *remote_handle, size_t size,
+                            int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
+                            void *cbcontext, void *cbdata);
 
 /* CUDA IPC control message tags */
 enum ipcCtrlMsg {
@@ -496,44 +467,32 @@ enum ipcCtrlMsg {
 
 /* CUDA IPC control message */
 typedef struct ctrlhdr_st {
-        enum ipcCtrlMsg ctag;
-        int cudev;
+    enum ipcCtrlMsg ctag;
+    int cudev;
 } ctrlhdr_t;
 
 /* State of setting up CUDA IPC on an endpoint */
-enum ipcState {
-    IPC_INIT = 1,
-    IPC_SENT,
-    IPC_ACKING,
-    IPC_ACKED,
-    IPC_OK,
-    IPC_BAD
-};
-
-#endif /* OPAL_CUDA_SUPPORT */
-
+enum ipcState { IPC_INIT = 1, IPC_SENT, IPC_ACKING, IPC_ACKED, IPC_OK, IPC_BAD };
 
-extern void mca_btl_smcuda_dump(struct mca_btl_base_module_t* btl,
-                            struct mca_btl_base_endpoint_t* endpoint,
-                            int verbose);
+extern void mca_btl_smcuda_dump(struct mca_btl_base_module_t *btl,
+                                struct mca_btl_base_endpoint_t *endpoint, int verbose);
 
 #if OPAL_ENABLE_PROGRESS_THREADS == 1
-void mca_btl_smcuda_component_event_thread(opal_object_t*);
+void mca_btl_smcuda_component_event_thread(opal_object_t *);
 #endif
 
 #if OPAL_ENABLE_PROGRESS_THREADS == 1
-#define MCA_BTL_SMCUDA_SIGNAL_PEER(peer) \
-{ \
-    unsigned char cmd = DATA; \
-    if(write(peer->fifo_fd, &cmd, sizeof(cmd)) != sizeof(cmd)) { \
-        opal_output(0, "mca_btl_smcuda_send: write fifo failed: errno=%d\n", errno); \
-    } \
-}
+#    define MCA_BTL_SMCUDA_SIGNAL_PEER(peer)                                                 \
+        {                                                                                    \
+            unsigned char cmd = DATA;                                                        \
+            if (write(peer->fifo_fd, &cmd, sizeof(cmd)) != sizeof(cmd)) {                    \
+                opal_output(0, "mca_btl_smcuda_send: write fifo failed: errno=%d\n", errno); \
+            }                                                                                \
+        }
 #else
-#define MCA_BTL_SMCUDA_SIGNAL_PEER(peer)
+#    define MCA_BTL_SMCUDA_SIGNAL_PEER(peer)
 #endif
 
 END_C_DECLS
 
 #endif
-
diff --git a/opal/mca/btl/smcuda/btl_smcuda_accelerator.c b/opal/mca/btl/smcuda/btl_smcuda_accelerator.c
new file mode 100644
index 00000000000..56245879dd8
--- /dev/null
+++ b/opal/mca/btl/smcuda/btl_smcuda_accelerator.c
@@ -0,0 +1,275 @@
+/* 
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2023      Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2024      Advanced Micro Devices, Inc. All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "opal/mca/btl/btl.h"
+#include "ompi/constants.h"
+#include "btl_smcuda.h"
+
+#include "btl_smcuda_accelerator.h"
+#include "opal/mca/accelerator/base/base.h"
+#include "opal/mca/accelerator/accelerator.h"
+
+static opal_mutex_t btl_smcuda_accelerator_ipc_lock;
+static opal_accelerator_stream_t *ipc_stream = NULL;
+
+opal_accelerator_event_t **accelerator_event_ipc_array = NULL;
+static struct mca_btl_base_descriptor_t **accelerator_event_ipc_frag_array = NULL;
+
+/* First free/available location in accelerator_event_status_array */
+static int accelerator_event_ipc_first_avail;
+
+/* First currently-being used location in the accelerator_event_status_array */
+static int accelerator_event_ipc_first_used;
+
+/* Number of status items currently in use */
+static volatile int accelerator_event_ipc_num_used;
+
+/* Size of array holding events */
+static int accelerator_event_ipc_most = 0;
+static bool smcuda_accelerator_initialized = false;
+
+void mca_btl_smcuda_accelerator_fini(void);
+
+/* Initialize the internal ipc stream and the events (s&e) */
+static int mca_btl_smcuda_accelerator_ipc_init(void)
+{
+    int rc = OPAL_SUCCESS;
+    int i;
+    int device_id;
+
+    rc = opal_accelerator.get_device(&device_id);
+    if (OPAL_SUCCESS != rc) {
+        opal_output_verbose(1, mca_btl_smcuda_component.cuda_ipc_output, "Failed to retrieve current device.");
+        return OPAL_ERROR;
+    }
+
+    rc = opal_accelerator.create_stream(device_id, &ipc_stream);
+    if (OPAL_SUCCESS != rc) {
+        opal_output_verbose(1, mca_btl_smcuda_component.cuda_ipc_output, "Failed to create accelerator ipc_stream stream.");
+        return OPAL_ERROR;
+    }
+
+    /* Create the events since they can be reused. */
+    for (i = 0; i < mca_btl_smcuda_component.accelerator_max_ipc_events; i++) {
+        rc = opal_accelerator.create_event(device_id, &accelerator_event_ipc_array[i], opal_accelerator_use_sync_memops ? false : true);
+        if (OPAL_SUCCESS != rc) {
+            opal_output_verbose(1, mca_btl_smcuda_component.cuda_ipc_output, "Accelerator create event failed.");
+            return OPAL_ERROR;
+        }
+    }
+
+    return OPAL_SUCCESS;
+}
+
+int mca_btl_smcuda_accelerator_init(void)
+{
+    int rc = OPAL_SUCCESS;
+    int i;
+
+    OBJ_CONSTRUCT(&btl_smcuda_accelerator_ipc_lock, opal_mutex_t);
+
+    accelerator_event_ipc_num_used = 0;
+    accelerator_event_ipc_first_avail = 0;
+    accelerator_event_ipc_first_used = 0;
+
+    accelerator_event_ipc_array = calloc(mca_btl_smcuda_component.accelerator_max_ipc_events, sizeof(opal_accelerator_event_t *));
+    if (NULL == accelerator_event_ipc_array) {
+        opal_output_verbose(1, mca_btl_smcuda_component.cuda_ipc_output, "No memory.");
+        rc = OPAL_ERROR;
+        goto cleanup_and_error;
+    }
+
+    /* The first available status index is 0.  Make an empty frag
+       array. */
+    accelerator_event_ipc_frag_array = (struct mca_btl_base_descriptor_t **) malloc(sizeof(struct mca_btl_base_descriptor_t *) *
+                                                                                    mca_btl_smcuda_component.accelerator_max_ipc_events);
+    if (NULL == accelerator_event_ipc_frag_array) {
+        opal_output_verbose(1, mca_btl_smcuda_component.cuda_ipc_output, "No memory.");
+        rc = OPAL_ERROR;
+        goto cleanup_and_error;
+    }
+
+    if (!mca_btl_smcuda_component.accelerator_delayed_ipc_init) {
+        mca_btl_smcuda_accelerator_ipc_init();
+    }
+
+    smcuda_accelerator_initialized = true;
+
+cleanup_and_error:
+    if (OPAL_SUCCESS != rc) {
+        if (NULL != accelerator_event_ipc_array) {
+            for (i = 0; i < mca_btl_smcuda_component.accelerator_max_ipc_events; i++) {
+                if (NULL != accelerator_event_ipc_array[i]) {
+                    OBJ_RELEASE(accelerator_event_ipc_array[i]);
+                }
+            }
+            free(accelerator_event_ipc_array);
+        }
+        if (NULL != accelerator_event_ipc_frag_array) {
+            free(accelerator_event_ipc_frag_array);
+        }
+        if (NULL != ipc_stream) {
+            OBJ_RELEASE(ipc_stream);
+        }
+        OBJ_DESTRUCT(&btl_smcuda_accelerator_ipc_lock);
+    }
+
+    return rc;
+}
+
+void mca_btl_smcuda_accelerator_fini(void)
+{
+    int i;
+
+    if (0 == strcmp(opal_accelerator_base_selected_component.base_version.mca_component_name, "null") ||
+        false == smcuda_accelerator_initialized) {
+        return;
+    }
+
+    if (NULL != accelerator_event_ipc_array) {
+        for (i = 0; i < mca_btl_smcuda_component.accelerator_max_ipc_events; i++) {
+            if (NULL != accelerator_event_ipc_array[i]) {
+                OBJ_RELEASE(accelerator_event_ipc_array[i]);
+            }
+        }
+        free(accelerator_event_ipc_array);
+    }
+
+    if (NULL != accelerator_event_ipc_frag_array) {
+        free(accelerator_event_ipc_frag_array);
+    }
+
+    if (NULL != ipc_stream) {
+        OBJ_RELEASE(ipc_stream);
+    }
+
+    OBJ_DESTRUCT(&btl_smcuda_accelerator_ipc_lock);
+    smcuda_accelerator_initialized = false;
+}
+
+/*
+ * Function is called every time progress is called with the sm BTL.  If there
+ * are outstanding events, check to see if one has completed.  If so, hand
+ * back the fragment for further processing.
+ */
+int mca_btl_smcuda_progress_one_ipc_event(struct mca_btl_base_descriptor_t **frag)
+{   
+    int result;
+    
+    if (OPAL_LIKELY(0 == accelerator_event_ipc_num_used))
+        return 0;
+    
+    OPAL_THREAD_LOCK(&btl_smcuda_accelerator_ipc_lock);
+    if (accelerator_event_ipc_num_used > 0) {
+        opal_output_verbose(20, mca_btl_smcuda_component.cuda_ipc_output,
+                            "smcuda: progress_one_accelerator_ipc_event, outstanding_events=%d",
+                            accelerator_event_ipc_num_used);
+        result = opal_accelerator.query_event(MCA_ACCELERATOR_NO_DEVICE_ID, accelerator_event_ipc_array[accelerator_event_ipc_first_used]);
+        
+        /* We found an event that is not ready, so return. */
+        if (OPAL_ERR_RESOURCE_BUSY == result) {
+            opal_output_verbose(20, mca_btl_smcuda_component.cuda_ipc_output,
+                                "smcuda: event query returned not ready");
+            *frag = NULL;
+            OPAL_THREAD_UNLOCK(&btl_smcuda_accelerator_ipc_lock);
+            return 0;
+        } else if (OPAL_SUCCESS != result) {
+            opal_output_verbose(1, mca_btl_smcuda_component.cuda_ipc_output,
+                            "smcuda: event query failed: %d", result);
+            *frag = NULL;
+            OPAL_THREAD_UNLOCK(&btl_smcuda_accelerator_ipc_lock);
+            return OPAL_ERROR;
+        }
+        
+        *frag = accelerator_event_ipc_frag_array[accelerator_event_ipc_first_used];
+        opal_output_verbose(10, mca_btl_smcuda_component.cuda_ipc_output, "smcuda: event query returned %d", result);
+        
+        /* Bump counters, loop around the circular buffer if necessary */
+        --accelerator_event_ipc_num_used;
+        ++accelerator_event_ipc_first_used;
+        if (accelerator_event_ipc_first_used >= mca_btl_smcuda_component.accelerator_max_ipc_events) {
+            accelerator_event_ipc_first_used = 0;
+        }
+        /* A return value of 1 indicates an event completed and a frag was returned */
+        OPAL_THREAD_UNLOCK(&btl_smcuda_accelerator_ipc_lock);
+        return 1;
+    }
+    OPAL_THREAD_UNLOCK(&btl_smcuda_accelerator_ipc_lock);
+    return 0;
+}
+
+/*
+ * Start the asynchronous copy.  Then record and save away an event that will
+ * be queried to indicate the copy has completed.
+ */
+int mca_btl_smcuda_memcpy(void *dst, void *src, size_t amount, char *msg,
+                           struct mca_btl_base_descriptor_t *frag)
+{
+    int result;
+    OPAL_THREAD_LOCK(&btl_smcuda_accelerator_ipc_lock);
+
+    if (NULL == ipc_stream) {
+        result = mca_btl_smcuda_accelerator_ipc_init();
+        if (OPAL_SUCCESS != result) {
+            return result;
+        }
+    }
+
+    /* First make sure there is room to store the event.  If not, then
+     * return an error.  The error message will tell the user to try and
+     * run again, but with a larger array for storing events. */
+    if (accelerator_event_ipc_num_used == mca_btl_smcuda_component.accelerator_max_ipc_events) {
+        opal_output_verbose(1, mca_btl_smcuda_component.cuda_ipc_output, "smcuda: Out of event handles");
+        OPAL_THREAD_UNLOCK(&btl_smcuda_accelerator_ipc_lock);
+        return OPAL_ERR_OUT_OF_RESOURCE;
+    }
+
+    if (accelerator_event_ipc_num_used > accelerator_event_ipc_most) {
+        accelerator_event_ipc_most = accelerator_event_ipc_num_used;
+        /* Just print multiples of 10 */
+        if (0 == (accelerator_event_ipc_most % 10)) {
+            opal_output_verbose(20, mca_btl_smcuda_component.cuda_ipc_output, "smcuda: Maximum ipc events used is now %d",
+                                accelerator_event_ipc_most);
+        }
+    }
+
+    result = opal_accelerator.mem_copy_async(MCA_ACCELERATOR_NO_DEVICE_ID, MCA_ACCELERATOR_NO_DEVICE_ID,
+                                             dst, src, amount, ipc_stream, MCA_ACCELERATOR_TRANSFER_DTOD);
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != result)) {
+        opal_output_verbose(1, mca_btl_smcuda_component.cuda_ipc_output, "smcuda: memcpy async failed: %d",
+                            result);
+        OPAL_THREAD_UNLOCK(&btl_smcuda_accelerator_ipc_lock);
+        return OPAL_ERROR;
+    } else {
+        opal_output_verbose(20, mca_btl_smcuda_component.cuda_ipc_output,
+                            "smcuda: mem_copy_async passed: dst=%p, src=%p, size=%d", dst, src,
+                            (int) amount);
+    }
+
+    result = opal_accelerator.record_event(MCA_ACCELERATOR_NO_DEVICE_ID, accelerator_event_ipc_array[accelerator_event_ipc_first_avail], ipc_stream);
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != result)) {
+        opal_output_verbose(1, mca_btl_smcuda_component.cuda_ipc_output, "Event Record failed.");
+        OPAL_THREAD_UNLOCK(&btl_smcuda_accelerator_ipc_lock);
+        return OPAL_ERROR;
+    }
+    accelerator_event_ipc_frag_array[accelerator_event_ipc_first_avail] = frag;
+
+    /* Bump up the first available slot and number used by 1 */
+    accelerator_event_ipc_first_avail++;
+    if (accelerator_event_ipc_first_avail >= mca_btl_smcuda_component.accelerator_max_ipc_events) {
+        accelerator_event_ipc_first_avail = 0;
+    }
+    accelerator_event_ipc_num_used++;
+
+    OPAL_THREAD_UNLOCK(&btl_smcuda_accelerator_ipc_lock);
+    return OPAL_SUCCESS;
+}
diff --git a/opal/mca/btl/smcuda/btl_smcuda_accelerator.h b/opal/mca/btl/smcuda/btl_smcuda_accelerator.h
new file mode 100644
index 00000000000..0f2ec7392a1
--- /dev/null
+++ b/opal/mca/btl/smcuda/btl_smcuda_accelerator.h
@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights
+ *                         reserved.
+ * Copyright (c) 2023      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/* Implements a progress engine based accelerator asynchronous copy implementation */
+
+#ifndef MCA_BTL_SMCUDA_ACCELERATOR_H
+#define MCA_BTL_SMCUDA_ACCELERATOR_H
+
+#include "opal/mca/accelerator/accelerator.h"
+#include "opal/mca/btl/btl.h"
+
+OPAL_DECLSPEC int mca_btl_smcuda_accelerator_init(void);
+OPAL_DECLSPEC int mca_btl_smcuda_progress_one_ipc_event(struct mca_btl_base_descriptor_t **frag);
+OPAL_DECLSPEC int mca_btl_smcuda_memcpy(void *dst, void *src, size_t amount, char *msg,
+                           struct mca_btl_base_descriptor_t *frag);
+OPAL_DECLSPEC void mca_btl_smcuda_accelerator_fini(void);
+
+#endif /* MCA_BTL_SMCUDA_ACCELERATOR_H */
diff --git a/opal/mca/btl/smcuda/btl_smcuda_component.c b/opal/mca/btl/smcuda/btl_smcuda_component.c
index 9fa7db4430f..a3662e2db0e 100644
--- a/opal/mca/btl/smcuda/btl_smcuda_component.c
+++ b/opal/mca/btl/smcuda/btl_smcuda_component.c
@@ -16,7 +16,12 @@
  *                         reserved.
  * Copyright (c) 2011-2015 NVIDIA Corporation.  All rights reserved.
  * Copyright (c) 2014      Intel, Inc. All rights reserved.
- * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2018-2022 Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2023      Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2024      Advanced Micro Devices, Inc. All Rights reserved.
+ * Copyright (c) 2024      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -26,51 +31,49 @@
 #include "opal_config.h"
 #include <errno.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif  /* HAVE_UNISTD_H */
+#    include <unistd.h>
+#endif /* HAVE_UNISTD_H */
 #include <string.h>
 #ifdef HAVE_FCNTL_H
-#include <fcntl.h>
-#endif  /* HAVE_FCNTL_H */
+#    include <fcntl.h>
+#endif /* HAVE_FCNTL_H */
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif  /* HAVE_SYS_TYPES_H */
+#    include <sys/types.h>
+#endif /* HAVE_SYS_TYPES_H */
 #ifdef HAVE_SYS_MMAN_H
-#include <sys/mman.h>
-#endif  /* HAVE_SYS_MMAN_H */
+#    include <sys/mman.h>
+#endif /* HAVE_SYS_MMAN_H */
 #ifdef HAVE_SYS_STAT_H
-#include <sys/stat.h>  /* for mkfifo */
-#endif  /* HAVE_SYS_STAT_H */
+#    include <sys/stat.h> /* for mkfifo */
+#endif                    /* HAVE_SYS_STAT_H */
 
+#include "opal/mca/accelerator/accelerator.h"
+#include "opal/mca/accelerator/base/base.h"
 #include "opal/mca/hwloc/base/base.h"
 #include "opal/mca/shmem/base/base.h"
 #include "opal/mca/shmem/shmem.h"
 #include "opal/util/bit_ops.h"
 #include "opal/util/output.h"
-#include "opal/util/show_help.h"
 #include "opal/util/printf.h"
+#include "opal/util/show_help.h"
 
-#include "opal/mca/mpool/base/base.h"
-#include "opal/mca/common/sm/common_sm.h"
 #include "opal/mca/btl/base/btl_base_error.h"
+#include "opal/mca/common/sm/common_sm.h"
+#include "opal/mca/mpool/base/base.h"
 #include "opal/runtime/opal_params.h"
 
-#if OPAL_CUDA_SUPPORT
-#include "opal/mca/common/cuda/common_cuda.h"
-#endif /* OPAL_CUDA_SUPPORT */
-
 #include "btl_smcuda.h"
-#include "btl_smcuda_frag.h"
 #include "btl_smcuda_fifo.h"
+#include "btl_smcuda_frag.h"
+#include "btl_smcuda_accelerator.h"
 
 static int mca_btl_smcuda_component_open(void);
 static int mca_btl_smcuda_component_close(void);
 static int smcuda_register(void);
-static mca_btl_base_module_t** mca_btl_smcuda_component_init(
-    int *num_btls,
-    bool enable_progress_threads,
-    bool enable_mpi_threads
-);
+static mca_btl_base_module_t **
+mca_btl_smcuda_component_init(int *num_btls, bool enable_progress_threads, bool enable_mpi_threads);
+
+static void mca_btl_smcuda_component_fini(void);
 
 typedef enum {
     MCA_BTL_SM_RNDV_MOD_SM = 0,
@@ -81,67 +84,61 @@ typedef enum {
  * Shared Memory (SM) component instance.
  */
 mca_btl_smcuda_component_t mca_btl_smcuda_component = {
-    .super = {
-        /* First, the mca_base_component_t struct containing meta information
-          about the component itself */
-        .btl_version = {
-            MCA_BTL_DEFAULT_VERSION("smcuda"),
-            .mca_open_component = mca_btl_smcuda_component_open,
-            .mca_close_component = mca_btl_smcuda_component_close,
-            .mca_register_component_params = smcuda_register,
-        },
-        .btl_data = {
-            /* The component is checkpoint ready */
-            .param_field = MCA_BASE_METADATA_PARAM_CHECKPOINT
-        },
-
-        .btl_init = mca_btl_smcuda_component_init,
-        .btl_progress = mca_btl_smcuda_component_progress,
-    }  /* end super */
+    .super =
+        {
+            /* First, the mca_base_component_t struct containing meta information
+              about the component itself */
+            .btl_version =
+                {
+                    MCA_BTL_DEFAULT_VERSION("smcuda"),
+                    .mca_open_component = mca_btl_smcuda_component_open,
+                    .mca_close_component = mca_btl_smcuda_component_close,
+                    .mca_register_component_params = smcuda_register,
+                },
+            .btl_data =
+                {/* The component is checkpoint ready */
+                 .param_field = MCA_BASE_METADATA_PARAM_CHECKPOINT},
+
+            .btl_init = mca_btl_smcuda_component_init,
+            .btl_progress = mca_btl_smcuda_component_progress,
+        } /* end super */
 };
 
-
 /*
  * utility routines for parameter registration
  */
 
-static inline int mca_btl_smcuda_param_register_int(
-    const char* param_name,
-    int default_value,
-    int level,
-    int *storage)
+static inline int mca_btl_smcuda_param_register_int(const char *param_name, int default_value,
+                                                    int level, int *storage)
 {
     *storage = default_value;
-    (void) mca_base_component_var_register (&mca_btl_smcuda_component.super.btl_version,
-                                            param_name, NULL, MCA_BASE_VAR_TYPE_INT,
-                                            NULL, 0, 0, level,
-                                            MCA_BASE_VAR_SCOPE_READONLY, storage);
+    (void) mca_base_component_var_register(&mca_btl_smcuda_component.super.btl_version, param_name,
+                                           NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, level,
+                                           MCA_BASE_VAR_SCOPE_READONLY, storage);
     return *storage;
 }
 
-static inline unsigned int mca_btl_smcuda_param_register_uint(
-    const char* param_name,
-    unsigned int default_value,
-    int level,
-    unsigned int *storage)
+static inline unsigned int mca_btl_smcuda_param_register_uint(const char *param_name,
+                                                              unsigned int default_value, int level,
+                                                              unsigned int *storage)
 {
     *storage = default_value;
-    (void) mca_base_component_var_register (&mca_btl_smcuda_component.super.btl_version,
-                                            param_name, NULL, MCA_BASE_VAR_TYPE_UNSIGNED_INT,
-                                            NULL, 0, 0, level,
-                                            MCA_BASE_VAR_SCOPE_READONLY, storage);
+    (void) mca_base_component_var_register(&mca_btl_smcuda_component.super.btl_version, param_name,
+                                           NULL, MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, 0, level,
+                                           MCA_BASE_VAR_SCOPE_READONLY, storage);
     return *storage;
 }
 
 static int mca_btl_smcuda_component_verify(void)
 {
     /* We canot support async memcpy right now */
-    if( (mca_btl_smcuda.super.btl_flags & MCA_BTL_FLAGS_CUDA_COPY_ASYNC_RECV) ||
-        (mca_btl_smcuda.super.btl_flags & MCA_BTL_FLAGS_CUDA_COPY_ASYNC_SEND) ) {
+    if ((mca_btl_smcuda.super.btl_flags & MCA_BTL_FLAGS_ACCELERATOR_COPY_ASYNC_RECV)
+        || (mca_btl_smcuda.super.btl_flags & MCA_BTL_FLAGS_ACCELERATOR_COPY_ASYNC_SEND)) {
         opal_output_verbose(10, opal_btl_base_framework.framework_output,
                             "btl: smcuda: disable all asynchronous memcpy support");
     }
-    mca_btl_smcuda.super.btl_flags &= ~(MCA_BTL_FLAGS_CUDA_COPY_ASYNC_RECV | MCA_BTL_FLAGS_CUDA_COPY_ASYNC_SEND);
+    mca_btl_smcuda.super.btl_flags &= ~(MCA_BTL_FLAGS_ACCELERATOR_COPY_ASYNC_RECV
+                                        | MCA_BTL_FLAGS_ACCELERATOR_COPY_ASYNC_SEND);
     return mca_btl_base_param_verify(&mca_btl_smcuda.super);
 }
 
@@ -155,67 +152,89 @@ static int smcuda_register(void)
                                            OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
                                            &mca_btl_smcuda_component.mpool_min_size);
 
-    mca_btl_smcuda_param_register_int("free_list_num", 8, OPAL_INFO_LVL_5, &mca_btl_smcuda_component.sm_free_list_num);
-    mca_btl_smcuda_param_register_int("free_list_max", -1, OPAL_INFO_LVL_5, &mca_btl_smcuda_component.sm_free_list_max);
-    mca_btl_smcuda_param_register_int("free_list_inc", 64, OPAL_INFO_LVL_5, &mca_btl_smcuda_component.sm_free_list_inc);
-    mca_btl_smcuda_param_register_int("max_procs", -1, OPAL_INFO_LVL_5, &mca_btl_smcuda_component.sm_max_procs);
+    mca_btl_smcuda_param_register_int("free_list_num", 8, OPAL_INFO_LVL_5,
+                                      &mca_btl_smcuda_component.sm_free_list_num);
+    mca_btl_smcuda_param_register_int("free_list_max", -1, OPAL_INFO_LVL_5,
+                                      &mca_btl_smcuda_component.sm_free_list_max);
+    mca_btl_smcuda_param_register_int("free_list_inc", 64, OPAL_INFO_LVL_5,
+                                      &mca_btl_smcuda_component.sm_free_list_inc);
+    mca_btl_smcuda_param_register_int("max_procs", -1, OPAL_INFO_LVL_5,
+                                      &mca_btl_smcuda_component.sm_max_procs);
     /* there is no practical use for the mpool name parameter since mpool resources differ
        between components */
-    mca_btl_smcuda_component.sm_mpool_name = "sm";
-    mca_btl_smcuda_param_register_uint("fifo_size", 4096, OPAL_INFO_LVL_4, &mca_btl_smcuda_component.fifo_size);
-    mca_btl_smcuda_param_register_int("num_fifos", 1, OPAL_INFO_LVL_4, &mca_btl_smcuda_component.nfifos);
-
-    mca_btl_smcuda_param_register_uint("fifo_lazy_free", 120, OPAL_INFO_LVL_5, &mca_btl_smcuda_component.fifo_lazy_free);
+    mca_btl_smcuda_component.sm_mpool_name = "smgpu";
+    mca_btl_smcuda_param_register_uint("fifo_size", 4096, OPAL_INFO_LVL_4,
+                                       &mca_btl_smcuda_component.fifo_size);
+    mca_btl_smcuda_param_register_int("num_fifos", 1, OPAL_INFO_LVL_4,
+                                      &mca_btl_smcuda_component.nfifos);
+
+    mca_btl_smcuda_param_register_uint("fifo_lazy_free", 120, OPAL_INFO_LVL_5,
+                                       &mca_btl_smcuda_component.fifo_lazy_free);
+
+    /* Delay the creation of the IPC stream and events. This has the advantage of also
+     * working in scenarios where the user did not set the accelerator device
+     * before MPI_Init AND the stream/event has internally some reference to the device
+     * used at that time */
+    mca_btl_smcuda_component.accelerator_delayed_ipc_init = 1;
+    (void) mca_base_component_var_register(&mca_btl_smcuda_component.super.btl_version, "delayed_stream_init",
+                                           "Delay the initialization of the ipc stream and internal events",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                           MCA_BASE_VAR_SCOPE_READONLY, &mca_btl_smcuda_component.accelerator_delayed_ipc_init);
+
+    mca_btl_smcuda_component.accelerator_max_ipc_events = 400;
+    (void) mca_base_component_var_register(&mca_btl_smcuda_component.super.btl_version, "max_ipc_events",
+                                           "Number of events created by the smcuda components internally",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                           MCA_BASE_VAR_SCOPE_READONLY, &mca_btl_smcuda_component.accelerator_max_ipc_events);
 
     /* default number of extra procs to allow for future growth */
-    mca_btl_smcuda_param_register_int("sm_extra_procs", 0, OPAL_INFO_LVL_9, &mca_btl_smcuda_component.sm_extra_procs);
+    mca_btl_smcuda_param_register_int("sm_extra_procs", 0, OPAL_INFO_LVL_9,
+                                      &mca_btl_smcuda_component.sm_extra_procs);
 
     mca_btl_smcuda_component.allocator = "bucket";
-    (void) mca_base_component_var_register (&mca_btl_smcuda_component.super.btl_version, "allocator",
-                                            "Name of allocator component to use for btl/smcuda allocations",
-                                            MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_9,
-                                            MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_smcuda_component.allocator);
+    (void) mca_base_component_var_register(
+        &mca_btl_smcuda_component.super.btl_version, "allocator",
+        "Name of allocator component to use for btl/smcuda allocations", MCA_BASE_VAR_TYPE_STRING,
+        NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_smcuda_component.allocator);
 
-#if OPAL_CUDA_SUPPORT
     /* Lower priority when CUDA support is not requested */
-    if (opal_cuda_support) {
-        mca_btl_smcuda.super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_HIGH+1;
+    if (0 != strcmp(opal_accelerator_base_selected_component.base_version.mca_component_name, "null")) {
+        mca_btl_smcuda.super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_DEFAULT;
     } else {
         mca_btl_smcuda.super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_LOW;
     }
-    mca_btl_smcuda_param_register_int("use_cuda_ipc", 1, OPAL_INFO_LVL_4, &mca_btl_smcuda_component.use_cuda_ipc);
-    mca_btl_smcuda_param_register_int("use_cuda_ipc_same_gpu", 1, OPAL_INFO_LVL_4,&mca_btl_smcuda_component.use_cuda_ipc_same_gpu);
-    mca_btl_smcuda_param_register_int("cuda_ipc_verbose", 0, OPAL_INFO_LVL_4, &mca_btl_smcuda_component.cuda_ipc_verbose);
+    mca_btl_smcuda_param_register_int("use_cuda_ipc", 1, OPAL_INFO_LVL_4,
+                                      &mca_btl_smcuda_component.use_cuda_ipc);
+    mca_btl_smcuda_param_register_int("use_cuda_ipc_same_gpu", 1, OPAL_INFO_LVL_4,
+                                      &mca_btl_smcuda_component.use_cuda_ipc_same_gpu);
+    mca_btl_smcuda_param_register_int("cuda_ipc_verbose", 0, OPAL_INFO_LVL_4,
+                                      &mca_btl_smcuda_component.cuda_ipc_verbose);
     mca_btl_smcuda_component.cuda_ipc_output = opal_output_open(NULL);
-    opal_output_set_verbosity(mca_btl_smcuda_component.cuda_ipc_output, mca_btl_smcuda_component.cuda_ipc_verbose);
-#else /* OPAL_CUDA_SUPPORT */
-    mca_btl_smcuda.super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_LOW;
-#endif /* OPAL_CUDA_SUPPORT */
-    mca_btl_smcuda.super.btl_eager_limit = 4*1024;
-    mca_btl_smcuda.super.btl_rndv_eager_limit = 4*1024;
-    mca_btl_smcuda.super.btl_max_send_size = 32*1024;
-    mca_btl_smcuda.super.btl_rdma_pipeline_send_length = 64*1024;
-    mca_btl_smcuda.super.btl_rdma_pipeline_frag_size = 64*1024;
-    mca_btl_smcuda.super.btl_min_rdma_pipeline_size = 64*1024;
+    opal_output_set_verbosity(mca_btl_smcuda_component.cuda_ipc_output,
+                              mca_btl_smcuda_component.cuda_ipc_verbose);
+    mca_btl_smcuda.super.btl_eager_limit = 4 * 1024;
+    mca_btl_smcuda.super.btl_rndv_eager_limit = 4 * 1024;
+    mca_btl_smcuda.super.btl_max_send_size = 32 * 1024;
+    mca_btl_smcuda.super.btl_rdma_pipeline_send_length = 64 * 1024;
+    mca_btl_smcuda.super.btl_rdma_pipeline_frag_size = 64 * 1024;
+    mca_btl_smcuda.super.btl_min_rdma_pipeline_size = 64 * 1024;
     mca_btl_smcuda.super.btl_flags = MCA_BTL_FLAGS_SEND;
-    mca_btl_smcuda.super.btl_registration_handle_size = sizeof (mca_btl_base_registration_handle_t);
-    mca_btl_smcuda.super.btl_bandwidth = 9000;  /* Mbs */
-    mca_btl_smcuda.super.btl_latency   = 1;     /* Microsecs */
+    mca_btl_smcuda.super.btl_registration_handle_size = sizeof(mca_btl_base_registration_handle_t);
+    mca_btl_smcuda.super.btl_bandwidth = 9000; /* Mbs */
+    mca_btl_smcuda.super.btl_latency = 1;      /* Microsecs */
 
     /* Call the BTL based to register its MCA params */
-    mca_btl_base_param_register(&mca_btl_smcuda_component.super.btl_version,
-                                &mca_btl_smcuda.super);
-#if OPAL_CUDA_SUPPORT
+    mca_btl_base_param_register(&mca_btl_smcuda_component.super.btl_version, &mca_btl_smcuda.super);
     /* If user has not set the value, then set to the defalt */
-    if (0 == mca_btl_smcuda.super.btl_cuda_max_send_size) {
-        mca_btl_smcuda.super.btl_cuda_max_send_size = 128*1024;
+    if (0 == mca_btl_smcuda.super.btl_accelerator_max_send_size) {
+        mca_btl_smcuda.super.btl_accelerator_max_send_size = 128 * 1024;
     }
-    /* If user has not set the value, then set to magic number which will be converted to the minimum
-     * size needed to fit the PML header (see pml_ob1.c) */
-    if (0 == mca_btl_smcuda.super.btl_cuda_eager_limit) {
-        mca_btl_smcuda.super.btl_cuda_eager_limit = SIZE_MAX; /* magic number */
+#if OPAL_CUDA_GDR_SUPPORT
+    /* If user has not set the value, then set to magic number which will be converted to the
+     * minimum size needed to fit the PML header (see pml_ob1.c) */
+    if (0 == mca_btl_smcuda.super.btl_accelerator_eager_limit) {
+        mca_btl_smcuda.super.btl_accelerator_eager_limit = SIZE_MAX; /* magic number */
     }
-    mca_common_cuda_register_mca_variables();
 #endif /* OPAL_CUDA_SUPPORT */
     return mca_btl_smcuda_component_verify();
 }
@@ -234,27 +253,27 @@ static int mca_btl_smcuda_component_open(void)
     mca_btl_smcuda_component.sm_max_btls = 1;
 
     /* make sure the number of fifos is a power of 2 */
-    mca_btl_smcuda_component.nfifos = opal_next_poweroftwo_inclusive (mca_btl_smcuda_component.nfifos);
+    mca_btl_smcuda_component.nfifos = opal_next_poweroftwo_inclusive(
+        mca_btl_smcuda_component.nfifos);
 
     /* make sure that queue size and lazy free parameter are compatible */
-    if (mca_btl_smcuda_component.fifo_lazy_free >= (mca_btl_smcuda_component.fifo_size >> 1) )
-        mca_btl_smcuda_component.fifo_lazy_free  = (mca_btl_smcuda_component.fifo_size >> 1);
+    if (mca_btl_smcuda_component.fifo_lazy_free >= (mca_btl_smcuda_component.fifo_size >> 1))
+        mca_btl_smcuda_component.fifo_lazy_free = (mca_btl_smcuda_component.fifo_size >> 1);
     if (mca_btl_smcuda_component.fifo_lazy_free <= 0)
-        mca_btl_smcuda_component.fifo_lazy_free  = 1;
+        mca_btl_smcuda_component.fifo_lazy_free = 1;
 
     mca_btl_smcuda_component.max_frag_size = mca_btl_smcuda.super.btl_max_send_size;
     mca_btl_smcuda_component.eager_limit = mca_btl_smcuda.super.btl_eager_limit;
 
-#if OPAL_CUDA_SUPPORT
     /* Possibly adjust max_frag_size if the cuda size is bigger */
-    if (mca_btl_smcuda.super.btl_cuda_max_send_size > mca_btl_smcuda.super.btl_max_send_size) {
-        mca_btl_smcuda_component.max_frag_size = mca_btl_smcuda.super.btl_cuda_max_send_size;
+    if (mca_btl_smcuda.super.btl_accelerator_max_send_size > mca_btl_smcuda.super.btl_max_send_size) {
+        mca_btl_smcuda_component.max_frag_size = mca_btl_smcuda.super.btl_accelerator_max_send_size;
     }
     opal_output_verbose(10, opal_btl_base_framework.framework_output,
                         "btl: smcuda: cuda_max_send_size=%d, max_send_size=%d, max_frag_size=%d",
-                        (int)mca_btl_smcuda.super.btl_cuda_max_send_size, (int)mca_btl_smcuda.super.btl_max_send_size,
-                        (int)mca_btl_smcuda_component.max_frag_size);
-#endif /* OPAL_CUDA_SUPPORT */
+                        (int) mca_btl_smcuda.super.btl_accelerator_max_send_size,
+                        (int) mca_btl_smcuda.super.btl_max_send_size,
+                        (int) mca_btl_smcuda_component.max_frag_size);
 
     /* initialize objects */
     OBJ_CONSTRUCT(&mca_btl_smcuda_component.sm_lock, opal_mutex_t);
@@ -262,18 +281,24 @@ static int mca_btl_smcuda_component_open(void)
     OBJ_CONSTRUCT(&mca_btl_smcuda_component.sm_frags_max, opal_free_list_t);
     OBJ_CONSTRUCT(&mca_btl_smcuda_component.sm_frags_user, opal_free_list_t);
     OBJ_CONSTRUCT(&mca_btl_smcuda_component.pending_send_fl, opal_free_list_t);
+
     return OPAL_SUCCESS;
 }
 
+static int mca_btl_smcuda_component_close(void)
+{
+    return OPAL_SUCCESS;
+}
 
 /*
  * component cleanup - sanity checking of queue lengths
  */
 
-static int mca_btl_smcuda_component_close(void)
+static void mca_btl_smcuda_component_fini(void)
 {
-    int return_value = OPAL_SUCCESS;
+    int rc;
 
+    mca_btl_smcuda_accelerator_fini();
 
     OBJ_DESTRUCT(&mca_btl_smcuda_component.sm_lock);
     /**
@@ -285,12 +310,11 @@ static int mca_btl_smcuda_component_close(void)
     /*OBJ_DESTRUCT(&mca_btl_smcuda_component.sm_frags_max);*/
 
     /* unmap the shared memory control structure */
-    if(mca_btl_smcuda_component.sm_seg != NULL) {
-        return_value = mca_common_sm_fini( mca_btl_smcuda_component.sm_seg );
-        if( OPAL_SUCCESS != return_value ) {
-            return_value = OPAL_ERROR;
-            opal_output(0," mca_common_sm_fini failed\n");
-            goto CLEANUP;
+    if (mca_btl_smcuda_component.sm_seg != NULL) {
+        rc = mca_common_sm_fini(mca_btl_smcuda_component.sm_seg);
+        if (OPAL_SUCCESS != rc) {
+            opal_output(0, " mca_common_sm_fini failed\n");
+            return;
         }
 
         /* unlink file, so that it will be deleted when all references
@@ -303,51 +327,38 @@ static int mca_btl_smcuda_component_close(void)
 
 #if OPAL_ENABLE_PROGRESS_THREADS == 1
     /* close/cleanup fifo create for event notification */
-    if(mca_btl_smcuda_component.sm_fifo_fd > 0) {
+    if (mca_btl_smcuda_component.sm_fifo_fd > 0) {
         /* write a done message down the pipe */
         unsigned char cmd = DONE;
-        if( write(mca_btl_smcuda_component.sm_fifo_fd,&cmd,sizeof(cmd)) !=
-                sizeof(cmd)){
-            opal_output(0, "mca_btl_smcuda_component_close: write fifo failed: errno=%d\n",
-                    errno);
+        if (write(mca_btl_smcuda_component.sm_fifo_fd, &cmd, sizeof(cmd)) != sizeof(cmd)) {
+            opal_output(0, "mca_btl_smcuda_component_close: write fifo failed: errno=%d\n", errno);
         }
         opal_thread_join(&mca_btl_smcuda_component.sm_fifo_thread, NULL);
         close(mca_btl_smcuda_component.sm_fifo_fd);
         unlink(mca_btl_smcuda_component.sm_fifo_path);
     }
 #endif
-
-CLEANUP:
-
-#if OPAL_CUDA_SUPPORT
-    mca_common_cuda_fini();
-#endif /* OPAL_CUDA_SUPPORT */
-
-    /* return */
-    return return_value;
+    return;
 }
 
 /*
  * Returns the number of processes on the node.
  */
-static inline int
-get_num_local_procs(void)
+static inline int get_num_local_procs(void)
 {
     /* num_local_peers does not include us in
      * its calculation, so adjust for that */
-    return (int)(1 + opal_process_info.num_local_peers);
+    return (int) (1 + opal_process_info.num_local_peers);
 }
 
-static void
-calc_sm_max_procs(int n)
+static void calc_sm_max_procs(int n)
 {
     /* see if need to allocate space for extra procs */
     if (0 > mca_btl_smcuda_component.sm_max_procs) {
         /* no limit */
         if (0 <= mca_btl_smcuda_component.sm_extra_procs) {
             /* limit */
-            mca_btl_smcuda_component.sm_max_procs =
-                n + mca_btl_smcuda_component.sm_extra_procs;
+            mca_btl_smcuda_component.sm_max_procs = n + mca_btl_smcuda_component.sm_extra_procs;
         } else {
             /* no limit */
             mca_btl_smcuda_component.sm_max_procs = 2 * n;
@@ -355,30 +366,24 @@ calc_sm_max_procs(int n)
     }
 }
 
-static int
-create_and_attach(mca_btl_smcuda_component_t *comp_ptr,
-                  size_t size,
-                  char *file_name,
-                  size_t size_ctl_structure,
-                  size_t data_seg_alignment,
-                  mca_common_sm_module_t **out_modp)
+static int create_and_attach(mca_btl_smcuda_component_t *comp_ptr, size_t size, char *file_name,
+                             size_t size_ctl_structure, size_t data_seg_alignment,
+                             mca_common_sm_module_t **out_modp)
 
 {
-    if (NULL == (*out_modp =
-        mca_common_sm_module_create_and_attach(size, file_name,
-                                               size_ctl_structure,
-                                               data_seg_alignment))) {
-        opal_output(0, "create_and_attach: unable to create shared memory "
-                    "BTL coordinating strucure :: size %lu \n",
-                    (unsigned long)size);
+    if (NULL
+        == (*out_modp = mca_common_sm_module_create_and_attach(size, file_name, size_ctl_structure,
+                                                               data_seg_alignment))) {
+        opal_output(0,
+                    "create_and_attach: unable to create shared memory "
+                    "BTL coordinating structure :: size %lu \n",
+                    (unsigned long) size);
         return OPAL_ERROR;
     }
     return OPAL_SUCCESS;
 }
 
-static int
-get_mpool_res_size(int32_t max_procs,
-                   size_t *out_res_size)
+static int get_mpool_res_size(int32_t max_procs, size_t *out_res_size)
 {
     size_t size = 0;
 
@@ -396,13 +401,13 @@ get_mpool_res_size(int32_t max_procs,
      * "opal_cache_line_size" additions to account for some
      * padding and edge effects that may lie in the allocator.
      */
-    size = FIFO_MAP_NUM(max_procs) *
-           (sizeof(sm_fifo_t) + sizeof(void *) *
-            mca_btl_smcuda_component.fifo_size + 4 * opal_cache_line_size) +
-           (2 * max_procs + mca_btl_smcuda_component.sm_free_list_inc) *
-           (mca_btl_smcuda_component.eager_limit + 2 * opal_cache_line_size) +
-           mca_btl_smcuda_component.sm_free_list_num *
-           (mca_btl_smcuda_component.max_frag_size + 2 * opal_cache_line_size);
+    size = FIFO_MAP_NUM(max_procs)
+               * (sizeof(sm_fifo_t) + sizeof(void *) * mca_btl_smcuda_component.fifo_size
+                  + 4 * opal_cache_line_size)
+           + (2 * max_procs + mca_btl_smcuda_component.sm_free_list_inc)
+                 * (mca_btl_smcuda_component.eager_limit + 2 * opal_cache_line_size)
+           + mca_btl_smcuda_component.sm_free_list_num
+                 * (mca_btl_smcuda_component.max_frag_size + 2 * opal_cache_line_size);
 
     /* add something for the control structure */
     size += sizeof(mca_common_sm_module_t);
@@ -413,19 +418,17 @@ get_mpool_res_size(int32_t max_procs,
      * mpool_sm_component.c when sizeof(mca_common_sm_module_t) is
      * added.
      */
-    if (((double)size) * max_procs > LONG_MAX - 4096) {
+    if (((double) size) * max_procs > (double) (LONG_MAX - 4096)) {
         return OPAL_ERR_VALUE_OUT_OF_BOUNDS;
     }
-    size *= (size_t)max_procs;
+    size *= (size_t) max_procs;
     *out_res_size = size;
     return OPAL_SUCCESS;
 }
 
-
 /* Generates all the unique paths for the shared-memory segments that this BTL
  * needs along with other file paths used to share "connection information". */
-static int
-set_uniq_paths_for_init_rndv(mca_btl_smcuda_component_t *comp_ptr)
+static int set_uniq_paths_for_init_rndv(mca_btl_smcuda_component_t *comp_ptr)
 {
     int rc = OPAL_ERR_OUT_OF_RESOURCE;
 
@@ -436,30 +439,30 @@ set_uniq_paths_for_init_rndv(mca_btl_smcuda_component_t *comp_ptr)
     comp_ptr->sm_rndv_file_name = NULL;
 
     if (opal_asprintf(&comp_ptr->sm_mpool_ctl_file_name,
-                 "%s"OPAL_PATH_SEP"shared_mem_cuda_pool.%s",
-                 opal_process_info.job_session_dir,
-                 opal_process_info.nodename) < 0) {
+                      "%s" OPAL_PATH_SEP "shared_mem_cuda_pool.%s",
+                      opal_process_info.job_session_dir, opal_process_info.nodename)
+        < 0) {
         /* rc set */
         goto out;
     }
     if (opal_asprintf(&comp_ptr->sm_mpool_rndv_file_name,
-                 "%s"OPAL_PATH_SEP"shared_mem_cuda_pool_rndv.%s",
-                 opal_process_info.job_session_dir,
-                 opal_process_info.nodename) < 0) {
+                      "%s" OPAL_PATH_SEP "shared_mem_cuda_pool_rndv.%s",
+                      opal_process_info.job_session_dir, opal_process_info.nodename)
+        < 0) {
         /* rc set */
         goto out;
     }
     if (opal_asprintf(&comp_ptr->sm_ctl_file_name,
-                 "%s"OPAL_PATH_SEP"shared_mem_cuda_btl_module.%s",
-                 opal_process_info.job_session_dir,
-                 opal_process_info.nodename) < 0) {
+                      "%s" OPAL_PATH_SEP "shared_mem_cuda_btl_module.%s",
+                      opal_process_info.job_session_dir, opal_process_info.nodename)
+        < 0) {
         /* rc set */
         goto out;
     }
     if (opal_asprintf(&comp_ptr->sm_rndv_file_name,
-                 "%s"OPAL_PATH_SEP"shared_mem_cuda_btl_rndv.%s",
-                 opal_process_info.job_session_dir,
-                 opal_process_info.nodename) < 0) {
+                      "%s" OPAL_PATH_SEP "shared_mem_cuda_btl_rndv.%s",
+                      opal_process_info.job_session_dir, opal_process_info.nodename)
+        < 0) {
         /* rc set */
         goto out;
     }
@@ -484,9 +487,8 @@ set_uniq_paths_for_init_rndv(mca_btl_smcuda_component_t *comp_ptr)
     return rc;
 }
 
-static int
-create_rndv_file(mca_btl_smcuda_component_t *comp_ptr,
-                  mca_btl_sm_rndv_module_type_t type)
+static int create_rndv_file(mca_btl_smcuda_component_t *comp_ptr,
+                            mca_btl_sm_rndv_module_type_t type)
 {
     size_t size = 0;
     int rc = OPAL_SUCCESS;
@@ -497,8 +499,7 @@ create_rndv_file(mca_btl_smcuda_component_t *comp_ptr,
 
     if (MCA_BTL_SM_RNDV_MOD_MPOOL == type) {
         /* get the segment size for the sm mpool. */
-        if (OPAL_SUCCESS != (rc = get_mpool_res_size(comp_ptr->sm_max_procs,
-                                                     &size))) {
+        if (OPAL_SUCCESS != (rc = get_mpool_res_size(comp_ptr->sm_max_procs, &size))) {
             /* rc is already set */
             goto out;
         }
@@ -508,36 +509,32 @@ create_rndv_file(mca_btl_smcuda_component_t *comp_ptr,
             size = mca_btl_smcuda_component.mpool_min_size;
         }
 
-        /* we only need the shmem_ds info at this point. initilization will be
+        /* we only need the shmem_ds info at this point. initialization will be
          * completed in the mpool module code. the idea is that we just need this
          * info so we can populate the rndv file (or modex when we have it). */
-        if (OPAL_SUCCESS != (rc =
-            create_and_attach(comp_ptr, size, comp_ptr->sm_mpool_ctl_file_name,
-                              sizeof(mca_common_sm_module_t), 8, &tmp_modp))) {
+        if (OPAL_SUCCESS
+            != (rc = create_and_attach(comp_ptr, size, comp_ptr->sm_mpool_ctl_file_name,
+                                       sizeof(mca_common_sm_module_t), 8, &tmp_modp))) {
             /* rc is set */
             goto out;
         }
         fname = comp_ptr->sm_mpool_rndv_file_name;
-    }
-    else if (MCA_BTL_SM_RNDV_MOD_SM == type) {
+    } else if (MCA_BTL_SM_RNDV_MOD_SM == type) {
         /* calculate the segment size. */
-        size = sizeof(mca_common_sm_seg_header_t) +
-               comp_ptr->sm_max_procs *
-               (sizeof(sm_fifo_t *) +
-                sizeof(char *) + sizeof(uint16_t)) +
-               opal_cache_line_size;
-
-        if (OPAL_SUCCESS != (rc =
-            create_and_attach(comp_ptr, size, comp_ptr->sm_ctl_file_name,
-                              sizeof(mca_common_sm_seg_header_t),
-                              opal_cache_line_size, &comp_ptr->sm_seg))) {
+        size = sizeof(mca_common_sm_seg_header_t)
+               + comp_ptr->sm_max_procs * (sizeof(sm_fifo_t *) + sizeof(char *) + sizeof(uint16_t))
+               + opal_cache_line_size;
+
+        if (OPAL_SUCCESS
+            != (rc = create_and_attach(comp_ptr, size, comp_ptr->sm_ctl_file_name,
+                                       sizeof(mca_common_sm_seg_header_t), opal_cache_line_size,
+                                       &comp_ptr->sm_seg))) {
             /* rc is set */
             goto out;
         }
         fname = comp_ptr->sm_rndv_file_name;
         tmp_modp = comp_ptr->sm_seg;
-    }
-    else {
+    } else {
         return OPAL_ERR_BAD_PARAM;
     }
 
@@ -548,24 +545,24 @@ create_rndv_file(mca_btl_smcuda_component_t *comp_ptr,
      * sizeof(opal_shmem_ds_t), so we know where the mpool_res_size starts. */
     if (-1 == (fd = open(fname, O_CREAT | O_RDWR, 0600))) {
         int err = errno;
-        opal_show_help("help-mpi-btl-smcuda.txt", "sys call fail", true,
-                       "open(2)", strerror(err), err);
+        opal_show_help("help-mpi-btl-smcuda.txt", "sys call fail", true, "open(2)", strerror(err),
+                       err);
         rc = OPAL_ERR_IN_ERRNO;
         goto out;
     }
-    if ((ssize_t)sizeof(opal_shmem_ds_t) != write(fd, &(tmp_modp->shmem_ds),
-                                                  sizeof(opal_shmem_ds_t))) {
+    if ((ssize_t) sizeof(opal_shmem_ds_t)
+        != write(fd, &(tmp_modp->shmem_ds), sizeof(opal_shmem_ds_t))) {
         int err = errno;
-        opal_show_help("help-mpi-btl-smcuda.txt", "sys call fail", true,
-                       "write(2)", strerror(err), err);
+        opal_show_help("help-mpi-btl-smcuda.txt", "sys call fail", true, "write(2)", strerror(err),
+                       err);
         rc = OPAL_ERR_IN_ERRNO;
         goto out;
     }
     if (MCA_BTL_SM_RNDV_MOD_MPOOL == type) {
-        if ((ssize_t)sizeof(size) != write(fd, &size, sizeof(size))) {
+        if ((ssize_t) sizeof(size) != write(fd, &size, sizeof(size))) {
             int err = errno;
-            opal_show_help("help-mpi-btl-smcuda.txt", "sys call fail", true,
-                           "write(2)", strerror(err), err);
+            opal_show_help("help-mpi-btl-smcuda.txt", "sys call fail", true, "write(2)",
+                           strerror(err), err);
             rc = OPAL_ERR_IN_ERRNO;
             goto out;
         }
@@ -575,7 +572,7 @@ create_rndv_file(mca_btl_smcuda_component_t *comp_ptr,
 
 out:
     if (-1 != fd) {
-        (void)close(fd);
+        (void) close(fd);
     }
     return rc;
 }
@@ -583,9 +580,7 @@ create_rndv_file(mca_btl_smcuda_component_t *comp_ptr,
 /*
  * Creates information required for the sm modex and modex sends it.
  */
-static int
-backing_store_init(mca_btl_smcuda_component_t *comp_ptr,
-                   uint32_t local_rank)
+static int backing_store_init(mca_btl_smcuda_component_t *comp_ptr, uint32_t local_rank)
 {
     int rc = OPAL_SUCCESS;
 
@@ -595,13 +590,11 @@ backing_store_init(mca_btl_smcuda_component_t *comp_ptr,
     /* only let the lowest rank setup the metadata */
     if (0 == local_rank) {
         /* === sm mpool === */
-        if (OPAL_SUCCESS != (rc =
-            create_rndv_file(comp_ptr, MCA_BTL_SM_RNDV_MOD_MPOOL))) {
+        if (OPAL_SUCCESS != (rc = create_rndv_file(comp_ptr, MCA_BTL_SM_RNDV_MOD_MPOOL))) {
             goto out;
         }
         /* === sm === */
-        if (OPAL_SUCCESS != (rc =
-            create_rndv_file(comp_ptr, MCA_BTL_SM_RNDV_MOD_SM))) {
+        if (OPAL_SUCCESS != (rc = create_rndv_file(comp_ptr, MCA_BTL_SM_RNDV_MOD_SM))) {
             goto out;
         }
     }
@@ -610,8 +603,6 @@ backing_store_init(mca_btl_smcuda_component_t *comp_ptr,
     return rc;
 }
 
-#if OPAL_CUDA_SUPPORT
-
 /**
  * Send a CUDA IPC ACK or NOTREADY message back to the peer.
  *
@@ -619,20 +610,21 @@ backing_store_init(mca_btl_smcuda_component_t *comp_ptr,
  * @param peer (IN)     BTL peer addressing
  * @param peer (IN)     If ready, then send ACK
  */
-static void mca_btl_smcuda_send_cuda_ipc_ack(struct mca_btl_base_module_t* btl,
-                                             struct mca_btl_base_endpoint_t* endpoint, int ready)
+static void mca_btl_smcuda_send_cuda_ipc_ack(struct mca_btl_base_module_t *btl,
+                                             struct mca_btl_base_endpoint_t *endpoint, int ready)
 {
-    mca_btl_smcuda_frag_t* frag;
+    mca_btl_smcuda_frag_t *frag;
     ctrlhdr_t ctrlhdr;
     int rc;
 
-    if ( mca_btl_smcuda_component.num_outstanding_frags * 2 > (int) mca_btl_smcuda_component.fifo_size ) {
+    if (mca_btl_smcuda_component.num_outstanding_frags * 2
+        > (int) mca_btl_smcuda_component.fifo_size) {
         mca_btl_smcuda_component_progress();
     }
 
     /* allocate a fragment, giving up if we can't get one */
     MCA_BTL_SMCUDA_FRAG_ALLOC_EAGER(frag);
-    if( OPAL_UNLIKELY(NULL == frag) ) {
+    if (OPAL_UNLIKELY(NULL == frag)) {
         endpoint->ipcstate = IPC_BAD;
         return;
     }
@@ -657,8 +649,9 @@ static void mca_btl_smcuda_send_cuda_ipc_ack(struct mca_btl_base_module_t* btl,
      */
     OPAL_THREAD_ADD_FETCH32(&mca_btl_smcuda_component.num_outstanding_frags, +1);
 
-    MCA_BTL_SMCUDA_FIFO_WRITE(endpoint, endpoint->my_smp_rank,
-                              endpoint->peer_smp_rank, (void *) VIRTUAL2RELATIVE(frag->hdr), false, true, rc);
+    MCA_BTL_SMCUDA_FIFO_WRITE(endpoint, endpoint->my_smp_rank, endpoint->peer_smp_rank,
+                              (void *) VIRTUAL2RELATIVE(frag->hdr), false, true, rc);
+    (void)rc;
 
     /* Set state now that we have sent message */
     if (ready) {
@@ -668,19 +661,18 @@ static void mca_btl_smcuda_send_cuda_ipc_ack(struct mca_btl_base_module_t* btl,
     }
 
     return;
-
 }
 /* This function is utilized to set up CUDA IPC support within the smcuda
  * BTL.  It handles smcuda specific control messages that are triggered
  * when GPU memory transfers are initiated. */
-static void btl_smcuda_control(mca_btl_base_module_t* btl,
+static void btl_smcuda_control(mca_btl_base_module_t *btl,
                                const mca_btl_base_receive_descriptor_t *descriptor)
 {
     int mydevnum, ipcaccess, res;
     ctrlhdr_t ctrlhdr;
     opal_proc_t *ep_proc;
-    mca_btl_smcuda_t *smcuda_btl = (mca_btl_smcuda_t *)btl;
-    const mca_btl_base_segment_t* segments = descriptor->des_segments;
+    mca_btl_smcuda_t *smcuda_btl = (mca_btl_smcuda_t *) btl;
+    const mca_btl_base_segment_t *segments = descriptor->des_segments;
     struct mca_btl_base_endpoint_t *endpoint = descriptor->endpoint;
 
     ep_proc = endpoint->proc_opal;
@@ -699,24 +691,16 @@ static void btl_smcuda_control(mca_btl_base_module_t* btl,
          * accordingly.  Otherwise, drop the request and let the other
          * side continue the handshake. */
         OPAL_THREAD_LOCK(&endpoint->endpoint_lock);
-        if ((IPC_INIT == endpoint->ipcstate) ||
-            ((IPC_SENT == endpoint->ipcstate) && (endpoint->my_smp_rank > endpoint->peer_smp_rank))) {
-            endpoint->ipcstate = IPC_ACKING; /* Move into new state to prevent any new connection attempts */
+        if ((IPC_INIT == endpoint->ipcstate)
+            || ((IPC_SENT == endpoint->ipcstate)
+                && (endpoint->my_smp_rank > endpoint->peer_smp_rank))) {
+            endpoint->ipcstate = IPC_ACKING; /* Move into new state to prevent any new connection
+                                                attempts */
             OPAL_THREAD_UNLOCK(&endpoint->endpoint_lock);
 
-            /* If not yet CUDA ready, send a NOTREADY message back. */
-            if (!mca_common_cuda_enabled) {
-                opal_output_verbose(10, mca_btl_smcuda_component.cuda_ipc_output,
-                                    "Sending CUDA IPC NOTREADY: myrank=%d, peerrank=%d",
-                                    mca_btl_smcuda_component.my_smp_rank,
-                                    endpoint->peer_smp_rank);
-                mca_btl_smcuda_send_cuda_ipc_ack(btl, endpoint, 0);
-                return;
-            }
-
             /* Get my current device.  If this fails, move this endpoint state into
              * bad state.  No need to send a reply.  */
-            res = mca_common_cuda_get_device(&mydevnum);
+            res = opal_accelerator.get_device(&mydevnum);
             if (0 != res) {
                 endpoint->ipcstate = IPC_BAD;
                 return;
@@ -726,36 +710,37 @@ static void btl_smcuda_control(mca_btl_base_module_t* btl,
              * same device and use_cuda_ipc_same_gpu is 1 (default),
              * then assume CUDA IPC is possible.  This could be a
              * device running in DEFAULT mode or running under MPS.
-             * Otherwise, check peer acces to determine CUDA IPC
+             * Otherwise, check peer access to determine CUDA IPC
              * support.  If the CUDA API call fails, then just move
              * endpoint into bad state.  No need to send a reply. */
             if (mydevnum == ctrlhdr.cudev) {
                 if (mca_btl_smcuda_component.use_cuda_ipc_same_gpu) {
                     ipcaccess = 1;
                 } else {
-                    opal_output_verbose(10, mca_btl_smcuda_component.cuda_ipc_output,
-                                        "Analyzed CUDA IPC request: myrank=%d, mydev=%d, peerrank=%d, "
-                                        "peerdev=%d --> Access is disabled by btl_smcuda_use_cuda_ipc_same_gpu",
-                                        endpoint->my_smp_rank, mydevnum, endpoint->peer_smp_rank,
-                                        ctrlhdr.cudev);
+                    opal_output_verbose(
+                        10, mca_btl_smcuda_component.cuda_ipc_output,
+                        "Analyzed GPU IPC request: myrank=%d, mydev=%d, peerrank=%d, "
+                        "peerdev=%d --> Access is disabled by btl_smcuda_use_cuda_ipc_same_gpu",
+                        endpoint->my_smp_rank, mydevnum, endpoint->peer_smp_rank, ctrlhdr.cudev);
                     endpoint->ipcstate = IPC_BAD;
                     return;
                 }
             } else {
-                res = mca_common_cuda_device_can_access_peer(&ipcaccess, mydevnum, ctrlhdr.cudev);
+                res = opal_accelerator.device_can_access_peer(&ipcaccess, mydevnum, ctrlhdr.cudev);
                 if (0 != res) {
-                    opal_output_verbose(10, mca_btl_smcuda_component.cuda_ipc_output,
-                                        "Analyzed CUDA IPC request: myrank=%d, mydev=%d, peerrank=%d, "
-                                        "peerdev=%d --> Access is disabled because peer check failed with err=%d",
-                                        endpoint->my_smp_rank, mydevnum, endpoint->peer_smp_rank,
-                                        ctrlhdr.cudev, res);
+                    opal_output_verbose(
+                        10, mca_btl_smcuda_component.cuda_ipc_output,
+                        "Analyzed GPU IPC request: myrank=%d, mydev=%d, peerrank=%d, "
+                        "peerdev=%d --> Access is disabled because peer check failed with err=%d",
+                        endpoint->my_smp_rank, mydevnum, endpoint->peer_smp_rank, ctrlhdr.cudev,
+                        res);
                     endpoint->ipcstate = IPC_BAD;
                     return;
                 }
             }
 
             opal_output_verbose(10, mca_btl_smcuda_component.cuda_ipc_output,
-                                "Analyzed CUDA IPC request: myrank=%d, mydev=%d, peerrank=%d, "
+                                "Analyzed GPU IPC request: myrank=%d, mydev=%d, peerrank=%d, "
                                 "peerdev=%d --> ACCESS=%d",
                                 endpoint->my_smp_rank, mydevnum, endpoint->peer_smp_rank,
                                 ctrlhdr.cudev, ipcaccess);
@@ -763,32 +748,32 @@ static void btl_smcuda_control(mca_btl_base_module_t* btl,
             if (0 == ipcaccess) {
                 /* No CUDA IPC support */
                 opal_output_verbose(10, mca_btl_smcuda_component.cuda_ipc_output,
-                                    "Not sending CUDA IPC ACK, no P2P support");
+                                    "Not sending GPU IPC ACK, no P2P support");
                 endpoint->ipcstate = IPC_BAD;
             } else {
                 /* CUDA IPC works */
-                smcuda_btl->error_cb(&smcuda_btl->super, MCA_BTL_ERROR_FLAGS_ADD_CUDA_IPC,
-                                     ep_proc, (char *)&mca_btl_smcuda_component.cuda_ipc_output);
-                opal_output_verbose(10, mca_btl_smcuda_component.cuda_ipc_output,
-                                    "Sending CUDA IPC ACK:  myrank=%d, mydev=%d, peerrank=%d, peerdev=%d",
-                                    endpoint->my_smp_rank, mydevnum, endpoint->peer_smp_rank,
-                                    ctrlhdr.cudev);
+                smcuda_btl->error_cb(&smcuda_btl->super, MCA_BTL_ERROR_FLAGS_ADD_ACCELERATOR_IPC, ep_proc,
+                                     (char *) &mca_btl_smcuda_component.cuda_ipc_output);
+                opal_output_verbose(
+                    10, mca_btl_smcuda_component.cuda_ipc_output,
+                    "Sending GPU IPC ACK:  myrank=%d, mydev=%d, peerrank=%d, peerdev=%d",
+                    endpoint->my_smp_rank, mydevnum, endpoint->peer_smp_rank, ctrlhdr.cudev);
                 mca_btl_smcuda_send_cuda_ipc_ack(btl, endpoint, 1);
             }
         } else {
             OPAL_THREAD_UNLOCK(&endpoint->endpoint_lock);
             opal_output_verbose(10, mca_btl_smcuda_component.cuda_ipc_output,
-                                "Not sending CUDA IPC ACK because request already initiated");
+                                "Not sending GPU IPC ACK because request already initiated");
         }
         break;
 
     case IPC_ACK:
         opal_output_verbose(10, mca_btl_smcuda_component.cuda_ipc_output,
-                            "Received CUDA IPC ACK, notifying PML: myrank=%d, peerrank=%d",
+                            "Received GPU IPC ACK, notifying PML: myrank=%d, peerrank=%d",
                             endpoint->my_smp_rank, endpoint->peer_smp_rank);
 
-        smcuda_btl->error_cb(&smcuda_btl->super, MCA_BTL_ERROR_FLAGS_ADD_CUDA_IPC,
-                             ep_proc, (char *)&mca_btl_smcuda_component.cuda_ipc_output);
+        smcuda_btl->error_cb(&smcuda_btl->super, MCA_BTL_ERROR_FLAGS_ADD_ACCELERATOR_IPC, ep_proc,
+                             (char *) &mca_btl_smcuda_component.cuda_ipc_output);
         assert(endpoint->ipcstate == IPC_SENT);
         endpoint->ipcstate = IPC_ACKED;
         break;
@@ -797,7 +782,7 @@ static void btl_smcuda_control(mca_btl_base_module_t* btl,
         /* The remote side is not ready.  Reset state to initialized so next
          * send call will try again to set up connection. */
         opal_output_verbose(10, mca_btl_smcuda_component.cuda_ipc_output,
-                            "Received CUDA IPC NOTREADY, reset state to allow another attempt: "
+                            "Received GPU IPC NOTREADY, reset state to allow another attempt: "
                             "myrank=%d, peerrank=%d",
                             endpoint->my_smp_rank, endpoint->peer_smp_rank);
         OPAL_THREAD_LOCK(&endpoint->endpoint_lock);
@@ -808,19 +793,15 @@ static void btl_smcuda_control(mca_btl_base_module_t* btl,
         break;
 
     default:
-        opal_output(0, "Received UNKNOWN CUDA IPC control message. This should not happen.");
+        opal_output(0, "Received UNKNOWN GPU IPC control message. This should not happen.");
     }
 }
 
-#endif /* OPAL_CUDA_SUPPORT */
-
 /*
  *  SM component initialization
  */
 static mca_btl_base_module_t **
-mca_btl_smcuda_component_init(int *num_btls,
-                          bool enable_progress_threads,
-                          bool enable_mpi_threads)
+mca_btl_smcuda_component_init(int *num_btls, bool enable_progress_threads, bool enable_mpi_threads)
 {
     int num_local_procs = 0;
     mca_btl_base_module_t **btls = NULL;
@@ -831,15 +812,15 @@ mca_btl_smcuda_component_init(int *num_btls,
     mca_btl_smcuda_component.sm_mpool = NULL;
     mca_btl_smcuda_component.sm_mpool_base = NULL;
 
-#if OPAL_CUDA_SUPPORT
-    mca_common_cuda_stage_one_init();
-#endif /* OPAL_CUDA_SUPPORT */
+    if (OPAL_SUCCESS != mca_btl_smcuda_accelerator_init()) {
+        return NULL;
+    }
 
     /* if no session directory was created, then we cannot be used */
     if (NULL == opal_process_info.job_session_dir) {
-    /* SKG - this isn't true anymore. Some backing facilities don't require a
-     * file-backed store. Extend shmem to provide this info one day. Especially
-     * when we use a proper modex for init. */
+        /* SKG - this isn't true anymore. Some backing facilities don't require a
+         * file-backed store. Extend shmem to provide this info one day. Especially
+         * when we use a proper modex for init. */
         return NULL;
     }
     /* if we don't have locality information, then we cannot be used because we
@@ -848,8 +829,7 @@ mca_btl_smcuda_component_init(int *num_btls,
      * the spawn case we need to designate a metadata creator rank within the
      * set of processes that are initializing the btl, and my_local_rank seems
      * to provide that for us. */
-    if (UINT32_MAX ==
-        (my_local_rank = opal_process_info.my_local_rank)) {
+    if (UINT32_MAX == (my_local_rank = opal_process_info.my_local_rank)) {
         opal_show_help("help-mpi-btl-smcuda.txt", "no locality", true);
         return NULL;
     }
@@ -872,88 +852,97 @@ mca_btl_smcuda_component_init(int *num_btls,
      * 0 create a rendezvous file containing the backing store info, so the
      * other local procs can read from it during add_procs. The rest will just
      * stash the known paths for use later in init. */
-    if (OPAL_SUCCESS != backing_store_init(&mca_btl_smcuda_component,
-                                           my_local_rank)) {
+    if (OPAL_SUCCESS != backing_store_init(&mca_btl_smcuda_component, my_local_rank)) {
         return NULL;
     }
 
 #if OPAL_ENABLE_PROGRESS_THREADS == 1
     /* create a named pipe to receive events  */
-    sprintf( mca_btl_smcuda_component.sm_fifo_path,
-             "%s"OPAL_PATH_SEP"sm_fifo.%lu", opal_process_info.job_session_dir,
-             (unsigned long)OPAL_PROC_MY_NAME->vpid );
-    if(mkfifo(mca_btl_smcuda_component.sm_fifo_path, 0660) < 0) {
-        opal_output(0, "mca_btl_smcuda_component_init: mkfifo failed with errno=%d\n",errno);
+    sprintf(mca_btl_smcuda_component.sm_fifo_path, "%s" OPAL_PATH_SEP "sm_fifo.%lu",
+            opal_process_info.job_session_dir, (unsigned long) OPAL_PROC_MY_NAME->vpid);
+    if (mkfifo(mca_btl_smcuda_component.sm_fifo_path, 0660) < 0) {
+        opal_output(0, "mca_btl_smcuda_component_init: mkfifo failed with errno=%d\n", errno);
         return NULL;
     }
-    mca_btl_smcuda_component.sm_fifo_fd = open(mca_btl_smcuda_component.sm_fifo_path,
-                                           O_RDWR);
-    if(mca_btl_smcuda_component.sm_fifo_fd < 0) {
-        opal_output(0, "mca_btl_smcuda_component_init: "
-                   "open(%s) failed with errno=%d\n",
+    mca_btl_smcuda_component.sm_fifo_fd = open(mca_btl_smcuda_component.sm_fifo_path, O_RDWR);
+    if (mca_btl_smcuda_component.sm_fifo_fd < 0) {
+        opal_output(0,
+                    "mca_btl_smcuda_component_init: "
+                    "open(%s) failed with errno=%d\n",
                     mca_btl_smcuda_component.sm_fifo_path, errno);
         return NULL;
     }
 
     OBJ_CONSTRUCT(&mca_btl_smcuda_component.sm_fifo_thread, opal_thread_t);
-    mca_btl_smcuda_component.sm_fifo_thread.t_run =
-        (opal_thread_fn_t)mca_btl_smcuda_component_event_thread;
+    mca_btl_smcuda_component.sm_fifo_thread.t_run = (opal_thread_fn_t)
+        mca_btl_smcuda_component_event_thread;
     opal_thread_start(&mca_btl_smcuda_component.sm_fifo_thread);
 #endif
 
-    mca_btl_smcuda_component.sm_btls =
-        (mca_btl_smcuda_t **)malloc(mca_btl_smcuda_component.sm_max_btls *
-                                sizeof(mca_btl_smcuda_t *));
+    mca_btl_smcuda_component.sm_btls = (mca_btl_smcuda_t **) malloc(
+        mca_btl_smcuda_component.sm_max_btls * sizeof(mca_btl_smcuda_t *));
     if (NULL == mca_btl_smcuda_component.sm_btls) {
         return NULL;
     }
 
     /* allocate the Shared Memory BTL */
     *num_btls = 1;
-    btls = (mca_btl_base_module_t**)malloc(sizeof(mca_btl_base_module_t*));
+    btls = (mca_btl_base_module_t **) malloc(sizeof(mca_btl_base_module_t *));
     if (NULL == btls) {
         return NULL;
     }
 
     /* get pointer to the btls */
-    btls[0] = (mca_btl_base_module_t*)(&(mca_btl_smcuda));
-    mca_btl_smcuda_component.sm_btls[0] = (mca_btl_smcuda_t*)(&(mca_btl_smcuda));
+    btls[0] = (mca_btl_base_module_t *) (&(mca_btl_smcuda));
+    mca_btl_smcuda_component.sm_btls[0] = (mca_btl_smcuda_t *) (&(mca_btl_smcuda));
 
     /* initialize some BTL data */
     /* start with no SM procs */
     mca_btl_smcuda_component.num_smp_procs = 0;
-    mca_btl_smcuda_component.my_smp_rank   = -1;  /* not defined */
-    mca_btl_smcuda_component.sm_num_btls   = 1;
+    mca_btl_smcuda_component.my_smp_rank = -1; /* not defined */
+    mca_btl_smcuda_component.sm_num_btls = 1;
     /* set flag indicating btl not inited */
     mca_btl_smcuda.btl_inited = false;
 
-#if OPAL_CUDA_SUPPORT
     /* Assume CUDA GET works. */
     mca_btl_smcuda.super.btl_get = mca_btl_smcuda_get_cuda;
     /* Register a smcuda control function to help setup IPC support */
     mca_btl_base_active_message_trigger[MCA_BTL_TAG_SMCUDA].cbfunc = btl_smcuda_control;
     mca_btl_base_active_message_trigger[MCA_BTL_TAG_SMCUDA].cbdata = NULL;
-#endif /* OPAL_CUDA_SUPPORT */
 
-    return btls;
+    /*
+     * add smcuda component fini code to opal's list of cleanup functions.
+     * Cleanups are called before all the MCA frameworks are closed, so by
+     * of the closing of the BTL framework with the accelerator framework, etc. etc.
+     * We add it here in the btl_init routine as its possible under
+     * certain scenarios that one of the steps above in this routine will fail,
+     * resulting in a NULL return value, and the btl component selector to close
+     * the btl.  This can also happen in normal operation, for instance for singleton
+     * where the smcuda is closed during mpi initialization.  We don't want
+     * to add a cleanup callback if no btls were returned.
+     */
 
-}
+    if (NULL !=  btls) {
+        opal_finalize_register_cleanup(mca_btl_smcuda_component_fini);
+    }
 
+    return btls;
+}
 
 /*
  *  SM component progress.
  */
 
 #if OPAL_ENABLE_PROGRESS_THREADS == 1
-void mca_btl_smcuda_component_event_thread(opal_object_t* thread)
+void mca_btl_smcuda_component_event_thread(opal_object_t *thread)
 {
-    while(1) {
+    while (1) {
         unsigned char cmd;
-        if(read(mca_btl_smcuda_component.sm_fifo_fd, &cmd, sizeof(cmd)) != sizeof(cmd)) {
+        if (read(mca_btl_smcuda_component.sm_fifo_fd, &cmd, sizeof(cmd)) != sizeof(cmd)) {
             /* error condition */
             return;
         }
-        if( DONE == cmd ){
+        if (DONE == cmd) {
             /* return when done message received */
             return;
         }
@@ -967,25 +956,27 @@ void btl_smcuda_process_pending_sends(struct mca_btl_base_endpoint_t *ep)
     btl_smcuda_pending_send_item_t *si;
     int rc;
 
-    while ( 0 < opal_list_get_size(&ep->pending_sends) ) {
+    while (0 < opal_list_get_size(&ep->pending_sends)) {
         /* Note that we access the size of ep->pending_sends unlocked
            as it doesn't really matter if the result is wrong as
            opal_list_remove_first is called with a lock and we handle it
            not finding an item to process */
         OPAL_THREAD_LOCK(&ep->endpoint_lock);
-        si = (btl_smcuda_pending_send_item_t*)opal_list_remove_first(&ep->pending_sends);
+        si = (btl_smcuda_pending_send_item_t *) opal_list_remove_first(&ep->pending_sends);
         OPAL_THREAD_UNLOCK(&ep->endpoint_lock);
 
-        if(NULL == si) return; /* Another thread got in before us. Thats ok. */
+        if (NULL == si)
+            return; /* Another thread got in before us. That's ok. */
 
         OPAL_THREAD_ADD_FETCH32(&mca_btl_smcuda_component.num_pending_sends, -1);
 
-        MCA_BTL_SMCUDA_FIFO_WRITE(ep, ep->my_smp_rank, ep->peer_smp_rank, si->data,
-                          true, false, rc);
+        MCA_BTL_SMCUDA_FIFO_WRITE(ep, ep->my_smp_rank, ep->peer_smp_rank, si->data, true, false,
+                                  rc);
 
-        opal_free_list_return (&mca_btl_smcuda_component.pending_send_fl, (opal_free_list_item_t*)si);
+        opal_free_list_return(&mca_btl_smcuda_component.pending_send_fl,
+                              (opal_free_list_item_t *) si);
 
-        if ( OPAL_SUCCESS != rc )
+        if (OPAL_SUCCESS != rc)
             return;
     }
 }
@@ -1002,98 +993,99 @@ int mca_btl_smcuda_component_progress(void)
 
     /* first, deal with any pending sends */
     /* This check should be fast since we only need to check one variable. */
-    if ( 0 < mca_btl_smcuda_component.num_pending_sends ) {
+    if (0 < mca_btl_smcuda_component.num_pending_sends) {
 
         /* perform a loop to find the endpoints that have pending sends */
         /* This can take a while longer if there are many endpoints to check. */
-        for ( peer_smp_rank = 0; peer_smp_rank < mca_btl_smcuda_component.num_smp_procs; peer_smp_rank++) {
-            struct mca_btl_base_endpoint_t* endpoint;
-            if ( peer_smp_rank == my_smp_rank )
+        for (peer_smp_rank = 0; peer_smp_rank < mca_btl_smcuda_component.num_smp_procs;
+             peer_smp_rank++) {
+            struct mca_btl_base_endpoint_t *endpoint;
+            if (peer_smp_rank == my_smp_rank)
                 continue;
             endpoint = mca_btl_smcuda_component.sm_peers[peer_smp_rank];
-            if ( 0 < opal_list_get_size(&endpoint->pending_sends) )
+            if (0 < opal_list_get_size(&endpoint->pending_sends))
                 btl_smcuda_process_pending_sends(endpoint);
         }
     }
 
     /* poll each fifo */
-    for(j = 0; j < FIFO_MAP_NUM(mca_btl_smcuda_component.num_smp_procs); j++) {
+    for (j = 0; j < FIFO_MAP_NUM(mca_btl_smcuda_component.num_smp_procs); j++) {
         fifo = &(mca_btl_smcuda_component.fifo[my_smp_rank][j]);
-      recheck_peer:
-        /* aquire thread lock */
-        if(opal_using_threads()) {
+    recheck_peer:
+        /* acquire thread lock */
+        if (opal_using_threads()) {
             opal_atomic_lock(&(fifo->tail_lock));
         }
 
-        hdr = (mca_btl_smcuda_hdr_t *)sm_fifo_read(fifo);
+        hdr = (mca_btl_smcuda_hdr_t *) sm_fifo_read(fifo);
 
         /* release thread lock */
-        if(opal_using_threads()) {
+        if (opal_using_threads()) {
             opal_atomic_unlock(&(fifo->tail_lock));
         }
 
-        if(SM_FIFO_FREE == hdr) {
+        if (SM_FIFO_FREE == hdr) {
             continue;
         }
 
         nevents++;
         /* dispatch fragment by type */
-        switch(((uintptr_t)hdr) & MCA_BTL_SMCUDA_FRAG_TYPE_MASK) {
-            case MCA_BTL_SMCUDA_FRAG_SEND:
-            {
-                /* change the address from address relative to the shared
-                 * memory address, to a true virtual address */
-                hdr = (mca_btl_smcuda_hdr_t *) RELATIVE2VIRTUAL(hdr);
-                peer_smp_rank = hdr->my_smp_rank;
+        switch (((uintptr_t) hdr) & MCA_BTL_SMCUDA_FRAG_TYPE_MASK) {
+        case MCA_BTL_SMCUDA_FRAG_SEND: {
+            /* change the address from address relative to the shared
+             * memory address, to a true virtual address */
+            hdr = (mca_btl_smcuda_hdr_t *) RELATIVE2VIRTUAL(hdr);
+            peer_smp_rank = hdr->my_smp_rank;
 #if OPAL_ENABLE_DEBUG
-                if ( FIFO_MAP(peer_smp_rank) != j ) {
-                    opal_output(0, "mca_btl_smcuda_component_progress: "
-                                "rank %d got %d on FIFO %d, but this sender should send to FIFO %d\n",
-                                my_smp_rank, peer_smp_rank, j, FIFO_MAP(peer_smp_rank));
-                }
+            if (FIFO_MAP(peer_smp_rank) != j) {
+                opal_output(0,
+                            "mca_btl_smcuda_component_progress: "
+                            "rank %d got %d on FIFO %d, but this sender should send to FIFO %d\n",
+                            my_smp_rank, peer_smp_rank, j, FIFO_MAP(peer_smp_rank));
+            }
 #endif
-                seg.seg_addr.pval = ((char *)hdr) + sizeof(mca_btl_smcuda_hdr_t);
-                seg.seg_len = hdr->len;
-
-                mca_btl_active_message_callback_t *reg = mca_btl_base_active_message_trigger + hdr->tag;
-                mca_btl_base_receive_descriptor_t recv_desc = {.endpoint = mca_btl_smcuda_component.sm_peers[peer_smp_rank],
-                                                               .des_segments = &seg,
-                                                               .des_segment_count = 1,
-                                                               .tag = hdr->tag,
-                                                               .cbdata = reg->cbdata};
-                reg->cbfunc(&mca_btl_smcuda.super, &recv_desc);
-                /* return the fragment */
-                MCA_BTL_SMCUDA_FIFO_WRITE(
-                        mca_btl_smcuda_component.sm_peers[peer_smp_rank],
-                        my_smp_rank, peer_smp_rank, hdr->frag, false, true, rc);
-                break;
+            seg.seg_addr.pval = ((char *) hdr) + sizeof(mca_btl_smcuda_hdr_t);
+            seg.seg_len = hdr->len;
+
+            mca_btl_active_message_callback_t *reg = mca_btl_base_active_message_trigger + hdr->tag;
+            mca_btl_base_receive_descriptor_t recv_desc = {.endpoint = mca_btl_smcuda_component
+                                                                           .sm_peers[peer_smp_rank],
+                                                           .des_segments = &seg,
+                                                           .des_segment_count = 1,
+                                                           .tag = hdr->tag,
+                                                           .cbdata = reg->cbdata};
+            reg->cbfunc(&mca_btl_smcuda.super, &recv_desc);
+            /* return the fragment */
+            MCA_BTL_SMCUDA_FIFO_WRITE(mca_btl_smcuda_component.sm_peers[peer_smp_rank], my_smp_rank,
+                                      peer_smp_rank, hdr->frag, false, true, rc);
+            (void)rc;
+            break;
+        }
+        case MCA_BTL_SMCUDA_FRAG_ACK: {
+            int status = (uintptr_t) hdr & MCA_BTL_SMCUDA_FRAG_STATUS_MASK;
+            int btl_ownership;
+            struct mca_btl_base_endpoint_t *endpoint;
+
+            frag = (mca_btl_smcuda_frag_t *) ((
+                char *) ((uintptr_t) hdr
+                         & (~(MCA_BTL_SMCUDA_FRAG_TYPE_MASK | MCA_BTL_SMCUDA_FRAG_STATUS_MASK))));
+
+            endpoint = frag->endpoint;
+            btl_ownership = (frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
+            if (MCA_BTL_DES_SEND_ALWAYS_CALLBACK & frag->base.des_flags) {
+                /* completion callback */
+                frag->base.des_cbfunc(&mca_btl_smcuda.super, frag->endpoint, &frag->base,
+                                      status ? OPAL_ERROR : OPAL_SUCCESS);
             }
-        case MCA_BTL_SMCUDA_FRAG_ACK:
-            {
-                int status = (uintptr_t)hdr & MCA_BTL_SMCUDA_FRAG_STATUS_MASK;
-                int btl_ownership;
-                struct mca_btl_base_endpoint_t* endpoint;
-
-                frag = (mca_btl_smcuda_frag_t *)((char*)((uintptr_t)hdr &
-                                                     (~(MCA_BTL_SMCUDA_FRAG_TYPE_MASK |
-                                                        MCA_BTL_SMCUDA_FRAG_STATUS_MASK))));
-
-                endpoint = frag->endpoint;
-                btl_ownership = (frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
-                if( MCA_BTL_DES_SEND_ALWAYS_CALLBACK & frag->base.des_flags ) {
-                    /* completion callback */
-                    frag->base.des_cbfunc(&mca_btl_smcuda.super, frag->endpoint,
-                                          &frag->base, status?OPAL_ERROR:OPAL_SUCCESS);
-                }
-                if( btl_ownership ) {
-                    MCA_BTL_SMCUDA_FRAG_RETURN(frag);
-                }
-                OPAL_THREAD_ADD_FETCH32(&mca_btl_smcuda_component.num_outstanding_frags, -1);
-                if ( 0 < opal_list_get_size(&endpoint->pending_sends) ) {
-                    btl_smcuda_process_pending_sends(endpoint);
-                }
-                goto recheck_peer;
+            if (btl_ownership) {
+                MCA_BTL_SMCUDA_FRAG_RETURN(frag);
             }
+            OPAL_THREAD_ADD_FETCH32(&mca_btl_smcuda_component.num_outstanding_frags, -1);
+            if (0 < opal_list_get_size(&endpoint->pending_sends)) {
+                btl_smcuda_process_pending_sends(endpoint);
+            }
+            goto recheck_peer;
+        }
         default:
             /* unknown */
             /*
@@ -1109,35 +1101,34 @@ int mca_btl_smcuda_component_progress(void)
             opal_output(0, "mca_btl_smcuda_component_progress read an unknown type of header");
             hdr = (mca_btl_smcuda_hdr_t *) RELATIVE2VIRTUAL(hdr);
             peer_smp_rank = hdr->my_smp_rank;
-            hdr = (mca_btl_smcuda_hdr_t*)((uintptr_t)hdr->frag |
-                                          MCA_BTL_SMCUDA_FRAG_STATUS_MASK);
-            MCA_BTL_SMCUDA_FIFO_WRITE(
-                    mca_btl_smcuda_component.sm_peers[peer_smp_rank],
-                    my_smp_rank, peer_smp_rank, hdr, false, true, rc);
+            hdr = (mca_btl_smcuda_hdr_t *) ((uintptr_t) hdr->frag
+                                            | MCA_BTL_SMCUDA_FRAG_STATUS_MASK);
+            MCA_BTL_SMCUDA_FIFO_WRITE(mca_btl_smcuda_component.sm_peers[peer_smp_rank], my_smp_rank,
+                                      peer_smp_rank, hdr, false, true, rc);
+            (void)rc;
             break;
         }
     }
-    (void)rc; /* this is safe to ignore as the message is requeued till success */
+    (void) rc; /* this is safe to ignore as the message is requeued till success */
 
-#if OPAL_CUDA_SUPPORT
     /* Check to see if there are any outstanding CUDA events that have
      * completed.  If so, issue the PML callbacks on the fragments.
      */
-    while (1 == progress_one_cuda_ipc_event((mca_btl_base_descriptor_t **)&frag)) {
-        mca_btl_base_rdma_completion_fn_t cbfunc = (mca_btl_base_rdma_completion_fn_t) frag->base.des_cbfunc;
+    while (1 == mca_btl_smcuda_progress_one_ipc_event((mca_btl_base_descriptor_t **) &frag)) {
+        mca_btl_base_rdma_completion_fn_t cbfunc = (mca_btl_base_rdma_completion_fn_t)
+                                                       frag->base.des_cbfunc;
 
-        cbfunc (&mca_btl_smcuda.super, frag->endpoint, frag->segment.seg_addr.pval,
-                frag->local_handle, frag->base.des_context, frag->base.des_cbdata,
-                OPAL_SUCCESS);
+        cbfunc(&mca_btl_smcuda.super, frag->endpoint, frag->segment.seg_addr.pval,
+               frag->local_handle, frag->base.des_context, frag->base.des_cbdata, OPAL_SUCCESS);
 
-        if(frag->registration != NULL) {
-            frag->endpoint->rcache->rcache_deregister (frag->endpoint->rcache,
-                                                       (mca_rcache_base_registration_t*)frag->registration);
+        if (frag->registration != NULL) {
+            frag->endpoint->rcache->rcache_deregister(frag->endpoint->rcache,
+                                                      (mca_rcache_base_registration_t *)
+                                                          frag->registration);
             frag->registration = NULL;
             MCA_BTL_SMCUDA_FRAG_RETURN(frag);
         }
         nevents++;
     }
-#endif /* OPAL_CUDA_SUPPORT */
     return nevents;
 }
diff --git a/opal/mca/btl/smcuda/btl_smcuda_endpoint.h b/opal/mca/btl/smcuda/btl_smcuda_endpoint.h
index 1dfb359e17f..d5d9506ed15 100644
--- a/opal/mca/btl/smcuda/btl_smcuda_endpoint.h
+++ b/opal/mca/btl/smcuda/btl_smcuda_endpoint.h
@@ -33,26 +33,22 @@
  */
 
 struct mca_btl_base_endpoint_t {
-    int my_smp_rank;    /**< My SMP process rank.  Used for accessing
-                         *   SMP specfic data structures. */
-    int peer_smp_rank;  /**< My peer's SMP process rank.  Used for accessing
-                         *   SMP specfic data structures. */
-#if OPAL_CUDA_SUPPORT
+    int my_smp_rank;   /**< My SMP process rank.  Used for accessing
+                        *   SMP specific data structures. */
+    int peer_smp_rank; /**< My peer's SMP process rank.  Used for accessing
+                        *   SMP specific data structures. */
     mca_rcache_base_module_t *rcache; /**< rcache for remotely registered memory */
-#endif /* OPAL_CUDA_SUPPORT */
 #if OPAL_ENABLE_PROGRESS_THREADS == 1
-    int fifo_fd;        /**< pipe/fifo used to signal endpoint that data is queued */
+    int fifo_fd; /**< pipe/fifo used to signal endpoint that data is queued */
 #endif
     opal_list_t pending_sends; /**< pending data to send */
 
     /** lock for concurrent access to endpoint state */
     opal_mutex_t endpoint_lock;
 
-#if OPAL_CUDA_SUPPORT
-    opal_proc_t *proc_opal;  /**< Needed for adding CUDA IPC support dynamically */
-    enum ipcState ipcstate;  /**< CUDA IPC connection status */
-    int ipctries;            /**< Number of times CUDA IPC connect was sent */
-#endif /* OPAL_CUDA_SUPPORT */
+    opal_proc_t *proc_opal; /**< Needed for adding CUDA IPC support dynamically */
+    enum ipcState ipcstate; /**< CUDA IPC connection status */
+    int ipctries;           /**< Number of times CUDA IPC connect was sent */
 };
 
 void btl_smcuda_process_pending_sends(struct mca_btl_base_endpoint_t *ep);
diff --git a/opal/mca/btl/smcuda/btl_smcuda_fifo.h b/opal/mca/btl/smcuda/btl_smcuda_fifo.h
index c4db00d10a8..278cbeafbf7 100644
--- a/opal/mca/btl/smcuda/btl_smcuda_fifo.h
+++ b/opal/mca/btl/smcuda/btl_smcuda_fifo.h
@@ -27,17 +27,16 @@
 #include "btl_smcuda.h"
 #include "btl_smcuda_endpoint.h"
 
-static void
-add_pending(struct mca_btl_base_endpoint_t *ep, void *data, bool resend)
+static void add_pending(struct mca_btl_base_endpoint_t *ep, void *data, bool resend)
 {
     btl_smcuda_pending_send_item_t *si;
     opal_free_list_item_t *i;
-    i = opal_free_list_get (&mca_btl_smcuda_component.pending_send_fl);
+    i = opal_free_list_get(&mca_btl_smcuda_component.pending_send_fl);
 
     /* don't handle error for now */
     assert(i != NULL);
 
-    si = (btl_smcuda_pending_send_item_t*)i;
+    si = (btl_smcuda_pending_send_item_t *) i;
     si->data = data;
 
     OPAL_THREAD_ADD_FETCH32(&mca_btl_smcuda_component.num_pending_sends, +1);
@@ -46,9 +45,9 @@ add_pending(struct mca_btl_base_endpoint_t *ep, void *data, bool resend)
      * minimize reordering */
     OPAL_THREAD_LOCK(&ep->endpoint_lock);
     if (resend)
-        opal_list_prepend(&ep->pending_sends, (opal_list_item_t*)si);
+        opal_list_prepend(&ep->pending_sends, (opal_list_item_t *) si);
     else
-        opal_list_append(&ep->pending_sends, (opal_list_item_t*)si);
+        opal_list_append(&ep->pending_sends, (opal_list_item_t *) si);
     OPAL_THREAD_UNLOCK(&ep->endpoint_lock);
 }
 
@@ -79,31 +78,33 @@ add_pending(struct mca_btl_base_endpoint_t *ep, void *data, bool resend)
  *    nfifos == 1:  In this case, all senders use the same
  *       FIFO and each receiver has just one FIFO for all senders.
  */
-#define FIFO_MAP(x)     ((x) & (mca_btl_smcuda_component.nfifos - 1))
-#define FIFO_MAP_NUM(n) ( (mca_btl_smcuda_component.nfifos) < (n) ? (mca_btl_smcuda_component.nfifos) : (n) )
+#define FIFO_MAP(x) ((x) & (mca_btl_smcuda_component.nfifos - 1))
+#define FIFO_MAP_NUM(n) \
+    ((mca_btl_smcuda_component.nfifos) < (n) ? (mca_btl_smcuda_component.nfifos) : (n))
 
-
-#define MCA_BTL_SMCUDA_FIFO_WRITE(endpoint_peer, my_smp_rank,               \
-                              peer_smp_rank, hdr, resend, retry_pending_sends, rc)        \
-do {                                                                    \
-    sm_fifo_t* fifo = &(mca_btl_smcuda_component.fifo[peer_smp_rank][FIFO_MAP(my_smp_rank)]); \
-                                                                        \
-    if ( retry_pending_sends ) {                                        \
-        if ( 0 < opal_list_get_size(&endpoint_peer->pending_sends) ) {  \
-            btl_smcuda_process_pending_sends(endpoint_peer);                \
-        }                                                               \
-    }                                                                   \
-                                                                        \
-    opal_atomic_lock(&(fifo->head_lock));                               \
-    /* post fragment */                                                 \
-    if(sm_fifo_write(hdr, fifo) != OPAL_SUCCESS) {                      \
-        add_pending(endpoint_peer, hdr, resend);                        \
-        rc = OPAL_ERR_RESOURCE_BUSY;                                    \
-    } else {                                                            \
-        MCA_BTL_SMCUDA_SIGNAL_PEER(endpoint_peer);                          \
-        rc = OPAL_SUCCESS;                                              \
-    }                                                                   \
-    opal_atomic_unlock(&(fifo->head_lock));                             \
-} while(0)
+#define MCA_BTL_SMCUDA_FIFO_WRITE(endpoint_peer, my_smp_rank, peer_smp_rank, hdr, resend,         \
+                                  retry_pending_sends, rc)                                        \
+    do {                                                                                          \
+        /* memory barrier: ensure writes to the hdr have completed */                             \
+        opal_atomic_wmb();                                                                        \
+        sm_fifo_t *_fifo = &(mca_btl_smcuda_component.fifo[peer_smp_rank][FIFO_MAP(my_smp_rank)]);\
+                                                                                                  \
+        if (retry_pending_sends) {                                                                \
+            if (0 < opal_list_get_size(&endpoint_peer->pending_sends)) {                          \
+                btl_smcuda_process_pending_sends(endpoint_peer);                                  \
+            }                                                                                     \
+        }                                                                                         \
+                                                                                                  \
+        opal_atomic_lock(&(_fifo->head_lock));                                                    \
+        /* post fragment */                                                                       \
+        if (sm_fifo_write(hdr, _fifo) != OPAL_SUCCESS) {                                          \
+            add_pending(endpoint_peer, hdr, resend);                                              \
+            rc = OPAL_ERR_RESOURCE_BUSY;                                                          \
+        } else {                                                                                  \
+            MCA_BTL_SMCUDA_SIGNAL_PEER(endpoint_peer);                                            \
+            rc = OPAL_SUCCESS;                                                                    \
+        }                                                                                         \
+        opal_atomic_unlock(&(_fifo->head_lock));                                                  \
+    } while (0)
 
 #endif
diff --git a/opal/mca/btl/smcuda/btl_smcuda_frag.c b/opal/mca/btl/smcuda/btl_smcuda_frag.c
index 3f28e7cdbaf..e0647a2e5e4 100644
--- a/opal/mca/btl/smcuda/btl_smcuda_frag.c
+++ b/opal/mca/btl/smcuda/btl_smcuda_frag.c
@@ -23,61 +23,47 @@
 #include "opal_config.h"
 #include "btl_smcuda_frag.h"
 
-
-static inline void mca_btl_smcuda_frag_common_constructor(mca_btl_smcuda_frag_t* frag)
+static inline void mca_btl_smcuda_frag_common_constructor(mca_btl_smcuda_frag_t *frag)
 {
-    frag->hdr = (mca_btl_smcuda_hdr_t*)frag->base.super.ptr;
-    if(frag->hdr != NULL) {
-        frag->hdr->frag = (mca_btl_smcuda_frag_t*)((uintptr_t)frag |
-            MCA_BTL_SMCUDA_FRAG_ACK);
-        frag->segment.seg_addr.pval = ((char*)frag->hdr) +
-            sizeof(mca_btl_smcuda_hdr_t);
+    frag->hdr = (mca_btl_smcuda_hdr_t *) frag->base.super.ptr;
+    if (frag->hdr != NULL) {
+        frag->hdr->frag = (mca_btl_smcuda_frag_t *) ((uintptr_t) frag | MCA_BTL_SMCUDA_FRAG_ACK);
+        frag->segment.seg_addr.pval = ((char *) frag->hdr) + sizeof(mca_btl_smcuda_hdr_t);
         frag->hdr->my_smp_rank = mca_btl_smcuda_component.my_smp_rank;
     }
     frag->segment.seg_len = frag->size;
     frag->base.des_segments = &frag->segment;
     frag->base.des_segment_count = 1;
     frag->base.des_flags = 0;
-#if OPAL_CUDA_SUPPORT
     frag->registration = NULL;
-#endif /* OPAL_CUDA_SUPPORT */
 }
 
-static void mca_btl_smcuda_frag1_constructor(mca_btl_smcuda_frag_t* frag)
+static void mca_btl_smcuda_frag1_constructor(mca_btl_smcuda_frag_t *frag)
 {
     frag->size = mca_btl_smcuda_component.eager_limit;
     frag->my_list = &mca_btl_smcuda_component.sm_frags_eager;
     mca_btl_smcuda_frag_common_constructor(frag);
 }
 
-static void mca_btl_smcuda_frag2_constructor(mca_btl_smcuda_frag_t* frag)
+static void mca_btl_smcuda_frag2_constructor(mca_btl_smcuda_frag_t *frag)
 {
     frag->size = mca_btl_smcuda_component.max_frag_size;
     frag->my_list = &mca_btl_smcuda_component.sm_frags_max;
     mca_btl_smcuda_frag_common_constructor(frag);
 }
 
-static void mca_btl_smcuda_user_constructor(mca_btl_smcuda_frag_t* frag)
+static void mca_btl_smcuda_user_constructor(mca_btl_smcuda_frag_t *frag)
 {
-	frag->size = 0;
-	frag->my_list = &mca_btl_smcuda_component.sm_frags_user;
-	mca_btl_smcuda_frag_common_constructor(frag);
+    frag->size = 0;
+    frag->my_list = &mca_btl_smcuda_component.sm_frags_user;
+    mca_btl_smcuda_frag_common_constructor(frag);
 }
 
-OBJ_CLASS_INSTANCE(
-    mca_btl_smcuda_frag1_t,
-    mca_btl_base_descriptor_t,
-    mca_btl_smcuda_frag1_constructor,
-    NULL);
+OBJ_CLASS_INSTANCE(mca_btl_smcuda_frag1_t, mca_btl_base_descriptor_t,
+                   mca_btl_smcuda_frag1_constructor, NULL);
 
-OBJ_CLASS_INSTANCE(
-    mca_btl_smcuda_frag2_t,
-    mca_btl_base_descriptor_t,
-    mca_btl_smcuda_frag2_constructor,
-    NULL);
+OBJ_CLASS_INSTANCE(mca_btl_smcuda_frag2_t, mca_btl_base_descriptor_t,
+                   mca_btl_smcuda_frag2_constructor, NULL);
 
-OBJ_CLASS_INSTANCE(
-    mca_btl_smcuda_user_t,
-    mca_btl_base_descriptor_t,
-    mca_btl_smcuda_user_constructor,
-    NULL);
+OBJ_CLASS_INSTANCE(mca_btl_smcuda_user_t, mca_btl_base_descriptor_t,
+                   mca_btl_smcuda_user_constructor, NULL);
diff --git a/opal/mca/btl/smcuda/btl_smcuda_frag.h b/opal/mca/btl/smcuda/btl_smcuda_frag.h
index 78cc9c39012..13b1e222958 100644
--- a/opal/mca/btl/smcuda/btl_smcuda_frag.h
+++ b/opal/mca/btl/smcuda/btl_smcuda_frag.h
@@ -15,6 +15,7 @@
  * Copyright (c) 2012      NVIDIA Corporation.  All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -30,17 +31,15 @@
 #include "opal_config.h"
 #include "btl_smcuda.h"
 
-#if OPAL_CUDA_SUPPORT
-#include "opal/mca/common/cuda/common_cuda.h"
-#endif
+#include "opal/include/opal/opal_gpu.h"
 
-#define MCA_BTL_SMCUDA_FRAG_TYPE_MASK ((uintptr_t)0x3)
-#define MCA_BTL_SMCUDA_FRAG_SEND ((uintptr_t)0x0)
-#define MCA_BTL_SMCUDA_FRAG_ACK ((uintptr_t)0x1)
-#define MCA_BTL_SMCUDA_FRAG_PUT ((uintptr_t)0x2)
-#define MCA_BTL_SMCUDA_FRAG_GET ((uintptr_t)0x3)
+#define MCA_BTL_SMCUDA_FRAG_TYPE_MASK ((uintptr_t) 0x3)
+#define MCA_BTL_SMCUDA_FRAG_SEND      ((uintptr_t) 0x0)
+#define MCA_BTL_SMCUDA_FRAG_ACK       ((uintptr_t) 0x1)
+#define MCA_BTL_SMCUDA_FRAG_PUT       ((uintptr_t) 0x2)
+#define MCA_BTL_SMCUDA_FRAG_GET       ((uintptr_t) 0x3)
 
-#define MCA_BTL_SMCUDA_FRAG_STATUS_MASK ((uintptr_t)0x4)
+#define MCA_BTL_SMCUDA_FRAG_STATUS_MASK ((uintptr_t) 0x4)
 
 struct mca_btl_smcuda_frag_t;
 
@@ -52,21 +51,17 @@ struct mca_btl_smcuda_hdr_t {
 };
 typedef struct mca_btl_smcuda_hdr_t mca_btl_smcuda_hdr_t;
 
-#if OPAL_CUDA_SUPPORT
 struct mca_btl_base_registration_handle_t {
-    mca_rcache_common_cuda_reg_data_t reg_data;
+    mca_opal_gpu_reg_data_t reg_data;
 };
-#endif
 
 struct mca_btl_smcuda_segment_t {
     mca_btl_base_segment_t base;
-#if OPAL_CUDA_SUPPORT
     uint8_t key[128]; /* 64 bytes for CUDA mem handle, 64 bytes for CUDA event handle */
     /** Address of the entire memory handle */
     opal_ptr_t memh_seg_addr;
-     /** Length in bytes of entire memory handle */
+    /** Length in bytes of entire memory handle */
     uint32_t memh_seg_len;
-#endif /* OPAL_CUDA_SUPPORT */
 };
 typedef struct mca_btl_smcuda_segment_t mca_btl_smcuda_segment_t;
 
@@ -77,46 +72,43 @@ struct mca_btl_smcuda_frag_t {
     mca_btl_base_descriptor_t base;
     mca_btl_base_segment_t segment;
     struct mca_btl_base_endpoint_t *endpoint;
-#if OPAL_CUDA_SUPPORT
     struct mca_rcache_base_registration_t *registration;
     struct mca_btl_base_registration_handle_t *local_handle;
-#endif /* OPAL_CUDA_SUPPORT */
     size_t size;
     /* pointer written to the FIFO, this is the base of the shared memory region */
     mca_btl_smcuda_hdr_t *hdr;
-    opal_free_list_t* my_list;
+    opal_free_list_t *my_list;
 };
 typedef struct mca_btl_smcuda_frag_t mca_btl_smcuda_frag_t;
 typedef struct mca_btl_smcuda_frag_t mca_btl_smcuda_frag1_t;
 typedef struct mca_btl_smcuda_frag_t mca_btl_smcuda_frag2_t;
 typedef struct mca_btl_smcuda_frag_t mca_btl_smcuda_user_t;
 
-
 OBJ_CLASS_DECLARATION(mca_btl_smcuda_frag_t);
 OBJ_CLASS_DECLARATION(mca_btl_smcuda_frag1_t);
 OBJ_CLASS_DECLARATION(mca_btl_smcuda_frag2_t);
 OBJ_CLASS_DECLARATION(mca_btl_smcuda_user_t);
 
-#define MCA_BTL_SMCUDA_FRAG_ALLOC_EAGER(frag)                           \
-{                                                                       \
-    frag = (mca_btl_smcuda_frag_t *)                                    \
-        opal_free_list_get (&mca_btl_smcuda_component.sm_frags_eager);  \
-}
+#define MCA_BTL_SMCUDA_FRAG_ALLOC_EAGER(frag)                \
+    {                                                        \
+        frag = (mca_btl_smcuda_frag_t *) opal_free_list_get( \
+            &mca_btl_smcuda_component.sm_frags_eager);       \
+    }
 
-#define MCA_BTL_SMCUDA_FRAG_ALLOC_MAX(frag)                             \
-{                                                                       \
-    frag = (mca_btl_smcuda_frag_t *)                                    \
-        opal_free_list_get (&mca_btl_smcuda_component.sm_frags_max);    \
-}
+#define MCA_BTL_SMCUDA_FRAG_ALLOC_MAX(frag)                  \
+    {                                                        \
+        frag = (mca_btl_smcuda_frag_t *) opal_free_list_get( \
+            &mca_btl_smcuda_component.sm_frags_max);         \
+    }
 
-#define MCA_BTL_SMCUDA_FRAG_ALLOC_USER(frag)                            \
-{                                                                       \
-    frag = (mca_btl_smcuda_frag_t *)                                    \
-        opal_free_list_get (&mca_btl_smcuda_component.sm_frags_user);   \
-}
+#define MCA_BTL_SMCUDA_FRAG_ALLOC_USER(frag)                 \
+    {                                                        \
+        frag = (mca_btl_smcuda_frag_t *) opal_free_list_get( \
+            &mca_btl_smcuda_component.sm_frags_user);        \
+    }
 
-#define MCA_BTL_SMCUDA_FRAG_RETURN(frag)                                \
-{                                                                       \
-    opal_free_list_return (frag->my_list, (opal_free_list_item_t*)(frag)); \
-}
+#define MCA_BTL_SMCUDA_FRAG_RETURN(frag)                                        \
+    {                                                                           \
+        opal_free_list_return(frag->my_list, (opal_free_list_item_t *) (frag)); \
+    }
 #endif
diff --git a/opal/mca/btl/smcuda/configure.m4 b/opal/mca/btl/smcuda/configure.m4
deleted file mode 100644
index 016f691944f..00000000000
--- a/opal/mca/btl/smcuda/configure.m4
+++ /dev/null
@@ -1,29 +0,0 @@
-# -*- shell-script -*-
-#
-# Copyright (c) 2009-2013 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2009-2010 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2012-2015 NVIDIA Corporation.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-# MCA_btl_smcuda_CONFIG([action-if-can-compile],
-#                   [action-if-cant-compile])
-# ------------------------------------------------
-AC_DEFUN([MCA_opal_btl_smcuda_CONFIG],[
-    AC_CONFIG_FILES([opal/mca/btl/smcuda/Makefile])
-
-    # make sure that CUDA-aware checks have been done
-    AC_REQUIRE([OPAL_CHECK_CUDA])
-
-    # Only build if CUDA support is available
-    AS_IF([test "x$CUDA_SUPPORT" = "x1"],
-          [$1],
-          [$2])
-
-])dnl
diff --git a/opal/mca/btl/tcp/Makefile.am b/opal/mca/btl/tcp/Makefile.am
index 5c73d46061c..a2de9e77883 100644
--- a/opal/mca/btl/tcp/Makefile.am
+++ b/opal/mca/btl/tcp/Makefile.am
@@ -12,6 +12,7 @@
 # Copyright (c) 2009-2014 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2013      NVIDIA Corporation.  All rights reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -19,7 +20,7 @@
 # $HEADER$
 #
 
-dist_opaldata_DATA = help-mpi-btl-tcp.txt
+EXTRA_DIST = help-mpi-btl-tcp.txt
 
 sources = \
     btl_tcp.c \
@@ -54,10 +55,6 @@ mcacomponentdir = $(opallibdir)
 mcacomponent_LTLIBRARIES = $(component)
 mca_btl_tcp_la_SOURCES = $(component_sources)
 mca_btl_tcp_la_LDFLAGS = -module -avoid-version
-if OPAL_cuda_support
-mca_btl_tcp_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
-    $(OPAL_TOP_BUILDDIR)/opal/mca/common/cuda/lib@OPAL_LIB_PREFIX@mca_common_cuda.la
-endif
 
 noinst_LTLIBRARIES = $(lib)
 libmca_btl_tcp_la_SOURCES = $(lib_sources)
diff --git a/opal/mca/btl/tcp/btl_tcp.c b/opal/mca/btl/tcp/btl_tcp.c
index d800c563f47..976e5ae5835 100644
--- a/opal/mca/btl/tcp/btl_tcp.c
+++ b/opal/mca/btl/tcp/btl_tcp.c
@@ -26,44 +26,44 @@
  */
 
 #include "opal_config.h"
-#include <string.h>
 #include "opal/class/opal_bitmap.h"
-#include "opal/mca/btl/btl.h"
 #include "opal/datatype/opal_convertor.h"
+#include "opal/mca/btl/base/btl_base_error.h"
+#include "opal/mca/btl/btl.h"
 #include "opal/mca/mpool/base/base.h"
 #include "opal/mca/mpool/mpool.h"
-#include "opal/mca/btl/base/btl_base_error.h"
 #include "opal/opal_socket_errno.h"
+#include <string.h>
 
 #include "btl_tcp.h"
+#include "btl_tcp_endpoint.h"
 #include "btl_tcp_frag.h"
 #include "btl_tcp_proc.h"
-#include "btl_tcp_endpoint.h"
 
-static int mca_btl_tcp_register_error_cb(struct mca_btl_base_module_t* btl,
+static int mca_btl_tcp_register_error_cb(struct mca_btl_base_module_t *btl,
                                          mca_btl_base_module_error_cb_fn_t cbfunc);
 
-mca_btl_tcp_module_t mca_btl_tcp_module = {
-    .super = {
-        .btl_component = &mca_btl_tcp_component.super,
-        .btl_add_procs = mca_btl_tcp_add_procs,
-        .btl_del_procs = mca_btl_tcp_del_procs,
-        .btl_finalize = mca_btl_tcp_finalize,
-        .btl_alloc = mca_btl_tcp_alloc,
-        .btl_free = mca_btl_tcp_free,
-        .btl_prepare_src = mca_btl_tcp_prepare_src,
-        .btl_send = mca_btl_tcp_send,
-        .btl_put = mca_btl_tcp_put,
-        .btl_dump = mca_btl_base_dump,
-        .btl_register_error = mca_btl_tcp_register_error_cb, /* register error */
-    },
-    .tcp_endpoints_mutex = OPAL_MUTEX_STATIC_INIT
-};
-
-static int mca_btl_tcp_register_error_cb(struct mca_btl_base_module_t* btl,
+mca_btl_tcp_module_t mca_btl_tcp_module =
+    {.super =
+         {
+             .btl_component = &mca_btl_tcp_component.super,
+             .btl_add_procs = mca_btl_tcp_add_procs,
+             .btl_del_procs = mca_btl_tcp_del_procs,
+             .btl_finalize = mca_btl_tcp_finalize,
+             .btl_alloc = mca_btl_tcp_alloc,
+             .btl_free = mca_btl_tcp_free,
+             .btl_prepare_src = mca_btl_tcp_prepare_src,
+             .btl_send = mca_btl_tcp_send,
+             .btl_put = mca_btl_tcp_put,
+             .btl_dump = mca_btl_base_dump,
+             .btl_register_error = mca_btl_tcp_register_error_cb, /* register error */
+         },
+     .tcp_endpoints_mutex = OPAL_MUTEX_STATIC_INIT};
+
+static int mca_btl_tcp_register_error_cb(struct mca_btl_base_module_t *btl,
                                          mca_btl_base_module_error_cb_fn_t cbfunc)
 {
-    mca_btl_tcp_module_t* tcp_btl = (mca_btl_tcp_module_t*)btl;
+    mca_btl_tcp_module_t *tcp_btl = (mca_btl_tcp_module_t *) btl;
     tcp_btl->tcp_error_cb = cbfunc;
     return OPAL_SUCCESS;
 }
@@ -72,39 +72,38 @@ static int mca_btl_tcp_register_error_cb(struct mca_btl_base_module_t* btl,
  *
  */
 
-int mca_btl_tcp_add_procs( struct mca_btl_base_module_t* btl,
-                           size_t nprocs,
-                           struct opal_proc_t **procs,
-                           struct mca_btl_base_endpoint_t** peers,
-                           opal_bitmap_t* reachable )
+int mca_btl_tcp_add_procs(struct mca_btl_base_module_t *btl, size_t nprocs,
+                          struct opal_proc_t **procs, struct mca_btl_base_endpoint_t **peers,
+                          opal_bitmap_t *reachable)
 {
-    mca_btl_tcp_module_t* tcp_btl = (mca_btl_tcp_module_t*)btl;
-    const opal_proc_t* my_proc; /* pointer to caller's proc structure */
+    mca_btl_tcp_module_t *tcp_btl = (mca_btl_tcp_module_t *) btl;
+    const opal_proc_t *my_proc; /* pointer to caller's proc structure */
     int i, rc;
 
     /* get pointer to my proc structure */
-    if( NULL == (my_proc = opal_proc_local_get()) )
+    if (NULL == (my_proc = opal_proc_local_get())) {
         return OPAL_ERR_OUT_OF_RESOURCE;
+    }
 
-    for(i = 0; i < (int) nprocs; i++) {
+    for (i = 0; i < (int) nprocs; i++) {
 
-        struct opal_proc_t* opal_proc = procs[i];
-        mca_btl_tcp_proc_t* tcp_proc;
-        mca_btl_base_endpoint_t* tcp_endpoint;
+        struct opal_proc_t *opal_proc = procs[i];
+        mca_btl_tcp_proc_t *tcp_proc;
+        mca_btl_base_endpoint_t *tcp_endpoint;
         bool existing_found = false;
 
         /* Do not create loopback TCP connections */
-        if( my_proc == opal_proc ) {
+        if (my_proc == opal_proc) {
             continue;
         }
 
-        if(NULL == (tcp_proc = mca_btl_tcp_proc_create(opal_proc))) {
+        if (NULL == (tcp_proc = mca_btl_tcp_proc_create(opal_proc))) {
             continue;
         }
 
         OPAL_THREAD_LOCK(&tcp_proc->proc_lock);
 
-        for (uint32_t j = 0 ; j < (uint32_t)tcp_proc->proc_endpoint_count ; ++j) {
+        for (uint32_t j = 0; j < (uint32_t) tcp_proc->proc_endpoint_count; ++j) {
             tcp_endpoint = tcp_proc->proc_endpoints[j];
             if (tcp_endpoint->endpoint_btl == tcp_btl) {
                 existing_found = true;
@@ -118,21 +117,21 @@ int mca_btl_tcp_add_procs( struct mca_btl_base_module_t* btl,
              * Cache the peer instance on the btl_proc.
              */
             tcp_endpoint = OBJ_NEW(mca_btl_tcp_endpoint_t);
-            if(NULL == tcp_endpoint) {
+            if (NULL == tcp_endpoint) {
                 OPAL_THREAD_UNLOCK(&tcp_proc->proc_lock);
                 return OPAL_ERR_OUT_OF_RESOURCE;
             }
 
             tcp_endpoint->endpoint_btl = tcp_btl;
             rc = mca_btl_tcp_proc_insert(tcp_proc, tcp_endpoint);
-            if(rc != OPAL_SUCCESS) {
+            if (rc != OPAL_SUCCESS) {
                 OPAL_THREAD_UNLOCK(&tcp_proc->proc_lock);
                 OBJ_RELEASE(tcp_endpoint);
                 continue;
             }
 
             OPAL_THREAD_LOCK(&tcp_btl->tcp_endpoints_mutex);
-            opal_list_append(&tcp_btl->tcp_endpoints, (opal_list_item_t*)tcp_endpoint);
+            opal_list_append(&tcp_btl->tcp_endpoints, (opal_list_item_t *) tcp_endpoint);
             OPAL_THREAD_UNLOCK(&tcp_btl->tcp_endpoints_mutex);
         }
 
@@ -148,25 +147,22 @@ int mca_btl_tcp_add_procs( struct mca_btl_base_module_t* btl,
     return OPAL_SUCCESS;
 }
 
-int mca_btl_tcp_del_procs(struct mca_btl_base_module_t* btl,
-                          size_t nprocs,
-                          struct opal_proc_t **procs,
-                          struct mca_btl_base_endpoint_t ** endpoints)
+int mca_btl_tcp_del_procs(struct mca_btl_base_module_t *btl, size_t nprocs,
+                          struct opal_proc_t **procs, struct mca_btl_base_endpoint_t **endpoints)
 {
-    mca_btl_tcp_module_t* tcp_btl = (mca_btl_tcp_module_t*)btl;
+    mca_btl_tcp_module_t *tcp_btl = (mca_btl_tcp_module_t *) btl;
     size_t i;
 
     OPAL_THREAD_LOCK(&tcp_btl->tcp_endpoints_mutex);
-    for( i = 0; i < nprocs; i++ ) {
-        mca_btl_tcp_endpoint_t* tcp_endpoint = endpoints[i];
-        opal_list_remove_item(&tcp_btl->tcp_endpoints, (opal_list_item_t*)tcp_endpoint);
+    for (i = 0; i < nprocs; i++) {
+        mca_btl_tcp_endpoint_t *tcp_endpoint = endpoints[i];
+        opal_list_remove_item(&tcp_btl->tcp_endpoints, (opal_list_item_t *) tcp_endpoint);
         OBJ_RELEASE(tcp_endpoint);
     }
     OPAL_THREAD_UNLOCK(&tcp_btl->tcp_endpoints_mutex);
     return OPAL_SUCCESS;
 }
 
-
 /**
  * Allocate a segment.
  *
@@ -174,45 +170,39 @@ int mca_btl_tcp_del_procs(struct mca_btl_base_module_t* btl,
  * @param size (IN)     Request segment size.
  */
 
-mca_btl_base_descriptor_t* mca_btl_tcp_alloc(
-    struct mca_btl_base_module_t* btl,
-    struct mca_btl_base_endpoint_t* endpoint,
-    uint8_t order,
-    size_t size,
-    uint32_t flags)
+mca_btl_base_descriptor_t *mca_btl_tcp_alloc(struct mca_btl_base_module_t *btl,
+                                             struct mca_btl_base_endpoint_t *endpoint,
+                                             uint8_t order, size_t size, uint32_t flags)
 {
-    mca_btl_tcp_frag_t* frag = NULL;
+    mca_btl_tcp_frag_t *frag = NULL;
 
-    if(size <= btl->btl_eager_limit) {
+    if (size <= btl->btl_eager_limit) {
         MCA_BTL_TCP_FRAG_ALLOC_EAGER(frag);
     } else if (size <= btl->btl_max_send_size) {
         MCA_BTL_TCP_FRAG_ALLOC_MAX(frag);
     }
-    if( OPAL_UNLIKELY(NULL == frag) ) {
+    if (OPAL_UNLIKELY(NULL == frag)) {
         return NULL;
     }
 
     frag->segments[0].seg_len = size;
-    frag->segments[0].seg_addr.pval = frag+1;
+    frag->segments[0].seg_addr.pval = frag + 1;
 
     frag->base.des_segments = frag->segments;
     frag->base.des_segment_count = 1;
     frag->base.des_flags = flags;
     frag->base.order = MCA_BTL_NO_ORDER;
-    frag->btl = (mca_btl_tcp_module_t*)btl;
-    return (mca_btl_base_descriptor_t*)frag;
+    frag->btl = (mca_btl_tcp_module_t *) btl;
+    return (mca_btl_base_descriptor_t *) frag;
 }
 
-
 /**
  * Return a segment
  */
 
-int mca_btl_tcp_free(
-    struct mca_btl_base_module_t* btl,
-    mca_btl_base_descriptor_t* des)
+int mca_btl_tcp_free(struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des)
 {
-    mca_btl_tcp_frag_t* frag = (mca_btl_tcp_frag_t*)des;
+    mca_btl_tcp_frag_t *frag = (mca_btl_tcp_frag_t *) des;
     MCA_BTL_TCP_FRAG_RETURN(frag);
     return OPAL_SUCCESS;
 }
@@ -224,29 +214,26 @@ int mca_btl_tcp_free(
  * @param btl (IN)      BTL module
  * @param peer (IN)     BTL peer addressing
  */
-mca_btl_base_descriptor_t* mca_btl_tcp_prepare_src(
-    struct mca_btl_base_module_t* btl,
-    struct mca_btl_base_endpoint_t* endpoint,
-    struct opal_convertor_t* convertor,
-    uint8_t order,
-    size_t reserve,
-    size_t* size,
-    uint32_t flags)
+mca_btl_base_descriptor_t *mca_btl_tcp_prepare_src(struct mca_btl_base_module_t *btl,
+                                                   struct mca_btl_base_endpoint_t *endpoint,
+                                                   struct opal_convertor_t *convertor,
+                                                   uint8_t order, size_t reserve, size_t *size,
+                                                   uint32_t flags)
 {
-    mca_btl_tcp_frag_t* frag;
+    mca_btl_tcp_frag_t *frag;
     struct iovec iov;
     uint32_t iov_count = 1;
     size_t max_data = *size;
     int rc;
 
-    if( OPAL_UNLIKELY(max_data > UINT32_MAX) ) {  /* limit the size to what we support */
-        max_data = (size_t)UINT32_MAX;
+    if (OPAL_UNLIKELY(max_data > UINT32_MAX)) { /* limit the size to what we support */
+        max_data = (size_t) UINT32_MAX;
     }
     /*
      * if we aren't pinning the data and the requested size is less
      * than the eager limit pack into a fragment from the eager pool
      */
-    if (max_data+reserve <= btl->btl_eager_limit) {
+    if (max_data + reserve <= btl->btl_eager_limit) {
         MCA_BTL_TCP_FRAG_ALLOC_EAGER(frag);
     } else {
         /*
@@ -255,7 +242,7 @@ mca_btl_base_descriptor_t* mca_btl_tcp_prepare_src(
          */
         MCA_BTL_TCP_FRAG_ALLOC_MAX(frag);
     }
-    if( OPAL_UNLIKELY(NULL == frag) ) {
+    if (OPAL_UNLIKELY(NULL == frag)) {
         return NULL;
     }
 
@@ -263,16 +250,17 @@ mca_btl_base_descriptor_t* mca_btl_tcp_prepare_src(
     frag->segments[0].seg_len = reserve;
 
     frag->base.des_segment_count = 1;
-    if(opal_convertor_need_buffers(convertor)) {
+    if (opal_convertor_need_buffers(convertor) || opal_convertor_on_device(convertor)) {
 
         if (max_data + reserve > frag->size) {
             max_data = frag->size - reserve;
         }
         iov.iov_len = max_data;
-        iov.iov_base = (IOVBASE_TYPE*)(((unsigned char*)(frag->segments[0].seg_addr.pval)) + reserve);
+        iov.iov_base = (IOVBASE_TYPE *) (((unsigned char *) (frag->segments[0].seg_addr.pval))
+                                         + reserve);
 
-        rc = opal_convertor_pack(convertor, &iov, &iov_count, &max_data );
-        if( OPAL_UNLIKELY(rc < 0) ) {
+        rc = opal_convertor_pack(convertor, &iov, &iov_count, &max_data);
+        if (OPAL_UNLIKELY(rc < 0)) {
             mca_btl_tcp_free(btl, &frag->base);
             return NULL;
         }
@@ -284,8 +272,8 @@ mca_btl_base_descriptor_t* mca_btl_tcp_prepare_src(
         iov.iov_len = max_data;
         iov.iov_base = NULL;
 
-        rc = opal_convertor_pack(convertor, &iov, &iov_count, &max_data );
-        if( OPAL_UNLIKELY(rc < 0) ) {
+        rc = opal_convertor_pack(convertor, &iov, &iov_count, &max_data);
+        if (OPAL_UNLIKELY(rc < 0)) {
             mca_btl_tcp_free(btl, &frag->base);
             return NULL;
         }
@@ -307,17 +295,15 @@ mca_btl_base_descriptor_t* mca_btl_tcp_prepare_src(
  *
  * @param btl (IN)         BTL module
  * @param endpoint (IN)    BTL addressing information
- * @param descriptor (IN)  Description of the data to be transfered
+ * @param descriptor (IN)  Description of the data to be transferred
  * @param tag (IN)         The tag value used to notify the peer.
  */
 
-int mca_btl_tcp_send( struct mca_btl_base_module_t* btl,
-                      struct mca_btl_base_endpoint_t* endpoint,
-                      struct mca_btl_base_descriptor_t* descriptor,
-                      mca_btl_base_tag_t tag )
+int mca_btl_tcp_send(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+                     struct mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag)
 {
-    mca_btl_tcp_module_t* tcp_btl = (mca_btl_tcp_module_t*) btl;
-    mca_btl_tcp_frag_t* frag = (mca_btl_tcp_frag_t*)descriptor;
+    mca_btl_tcp_module_t *tcp_btl = (mca_btl_tcp_module_t *) btl;
+    mca_btl_tcp_frag_t *frag = (mca_btl_tcp_frag_t *) descriptor;
     int i;
 
     frag->btl = tcp_btl;
@@ -326,46 +312,50 @@ int mca_btl_tcp_send( struct mca_btl_base_module_t* btl,
     frag->iov_idx = 0;
     frag->iov_cnt = 1;
     frag->iov_ptr = frag->iov;
-    frag->iov[0].iov_base = (IOVBASE_TYPE*)&frag->hdr;
+    frag->iov[0].iov_base = (IOVBASE_TYPE *) &frag->hdr;
     frag->iov[0].iov_len = sizeof(frag->hdr);
     frag->hdr.size = 0;
-    for( i = 0; i < (int)frag->base.des_segment_count; i++) {
+    for (i = 0; i < (int) frag->base.des_segment_count; i++) {
         frag->hdr.size += frag->segments[i].seg_len;
-        frag->iov[i+1].iov_len = frag->segments[i].seg_len;
-        frag->iov[i+1].iov_base = (IOVBASE_TYPE*)frag->segments[i].seg_addr.pval;
+        frag->iov[i + 1].iov_len = frag->segments[i].seg_len;
+        frag->iov[i + 1].iov_base = (IOVBASE_TYPE *) frag->segments[i].seg_addr.pval;
         frag->iov_cnt++;
     }
     frag->hdr.base.tag = tag;
     frag->hdr.type = MCA_BTL_TCP_HDR_TYPE_SEND;
     frag->hdr.count = 0;
-    if (endpoint->endpoint_nbo) MCA_BTL_TCP_HDR_HTON(frag->hdr);
-    return mca_btl_tcp_endpoint_send(endpoint,frag);
+    if (endpoint->endpoint_nbo) {
+        MCA_BTL_TCP_HDR_HTON(frag->hdr);
+    }
+    return mca_btl_tcp_endpoint_send(endpoint, frag);
 }
 
-static void fake_rdma_complete (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
-                                mca_btl_base_descriptor_t *desc, int rc)
+static void fake_rdma_complete(mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
+                               mca_btl_base_descriptor_t *desc, int rc)
 {
     mca_btl_tcp_frag_t *frag = (mca_btl_tcp_frag_t *) desc;
 
-    frag->cb.func (btl, endpoint, frag->segments[0].seg_addr.pval, NULL, frag->cb.context, frag->cb.data,
-                   rc);
+    frag->cb.func(btl, endpoint, frag->segments[0].seg_addr.pval, NULL, frag->cb.context,
+                  frag->cb.data, rc);
 }
 
 /**
  * Initiate an asynchronous put.
  */
 
-int mca_btl_tcp_put (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, void *local_address,
-		     uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-		     mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-		     int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
+int mca_btl_tcp_put(mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+                    void *local_address, uint64_t remote_address,
+                    mca_btl_base_registration_handle_t *local_handle,
+                    mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
+                    int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext,
+                    void *cbdata)
 {
-    mca_btl_tcp_module_t* tcp_btl = (mca_btl_tcp_module_t*) btl;
+    mca_btl_tcp_module_t *tcp_btl = (mca_btl_tcp_module_t *) btl;
     mca_btl_tcp_frag_t *frag = NULL;
     int i;
 
     MCA_BTL_TCP_FRAG_ALLOC_USER(frag);
-    if( OPAL_UNLIKELY(NULL == frag) ) {
+    if (OPAL_UNLIKELY(NULL == frag)) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
@@ -383,7 +373,9 @@ int mca_btl_tcp_put (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t
 
     frag->segments[1].seg_addr.lval = remote_address;
     frag->segments[1].seg_len = size;
-    if (endpoint->endpoint_nbo) MCA_BTL_BASE_SEGMENT_HTON(frag->segments[1]);
+    if (endpoint->endpoint_nbo) {
+        MCA_BTL_BASE_SEGMENT_HTON(frag->segments[1]);
+    }
 
     frag->base.des_flags = MCA_BTL_DES_FLAGS_BTL_OWNERSHIP | MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
     frag->base.des_cbfunc = fake_rdma_complete;
@@ -399,40 +391,44 @@ int mca_btl_tcp_put (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t
     frag->hdr.size = 0;
     frag->iov_cnt = 2;
     frag->iov_ptr = frag->iov;
-    frag->iov[0].iov_base = (IOVBASE_TYPE*)&frag->hdr;
+    frag->iov[0].iov_base = (IOVBASE_TYPE *) &frag->hdr;
     frag->iov[0].iov_len = sizeof(frag->hdr);
-    frag->iov[1].iov_base = (IOVBASE_TYPE*) (frag->segments + 1);
+    frag->iov[1].iov_base = (IOVBASE_TYPE *) (frag->segments + 1);
     frag->iov[1].iov_len = sizeof(mca_btl_base_segment_t);
-    for( i = 0; i < (int)frag->base.des_segment_count; i++ ) {
+    for (i = 0; i < (int) frag->base.des_segment_count; i++) {
         frag->hdr.size += frag->segments[i].seg_len;
-        frag->iov[i+2].iov_len = frag->segments[i].seg_len;
-        frag->iov[i+2].iov_base = (IOVBASE_TYPE*)frag->segments[i].seg_addr.pval;
+        frag->iov[i + 2].iov_len = frag->segments[i].seg_len;
+        frag->iov[i + 2].iov_base = (IOVBASE_TYPE *) frag->segments[i].seg_addr.pval;
         frag->iov_cnt++;
     }
     frag->hdr.base.tag = MCA_BTL_TAG_BTL;
     frag->hdr.type = MCA_BTL_TCP_HDR_TYPE_PUT;
     frag->hdr.count = 1;
-    if (endpoint->endpoint_nbo) MCA_BTL_TCP_HDR_HTON(frag->hdr);
-    return ((i = mca_btl_tcp_endpoint_send(endpoint,frag)) >= 0 ? OPAL_SUCCESS : i);
+    if (endpoint->endpoint_nbo) {
+        MCA_BTL_TCP_HDR_HTON(frag->hdr);
+    }
+    return ((i = mca_btl_tcp_endpoint_send(endpoint, frag)) >= 0 ? OPAL_SUCCESS : i);
 }
 
-
 /**
  * Initiate an asynchronous get.
  */
 
-int mca_btl_tcp_get (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, void *local_address,
-		     uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-		     mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-		     int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
+int mca_btl_tcp_get(mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+                    void *local_address, uint64_t remote_address,
+                    mca_btl_base_registration_handle_t *local_handle,
+                    mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
+                    int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext,
+                    void *cbdata)
 {
-    mca_btl_tcp_module_t* tcp_btl = (mca_btl_tcp_module_t*) btl;
-    mca_btl_tcp_frag_t* frag = NULL;
+    mca_btl_tcp_module_t *tcp_btl = (mca_btl_tcp_module_t *) btl;
+    mca_btl_tcp_frag_t *frag = NULL;
     int rc;
 
     MCA_BTL_TCP_FRAG_ALLOC_USER(frag);
-    if( OPAL_UNLIKELY(NULL == frag) ) {
-        return OPAL_ERR_OUT_OF_RESOURCE;;
+    if (OPAL_UNLIKELY(NULL == frag)) {
+        return OPAL_ERR_OUT_OF_RESOURCE;
+        ;
     }
 
     frag->endpoint = endpoint;
@@ -466,103 +462,101 @@ int mca_btl_tcp_get (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t
     frag->hdr.size = 0;
     frag->iov_cnt = 2;
     frag->iov_ptr = frag->iov;
-    frag->iov[0].iov_base = (IOVBASE_TYPE*)&frag->hdr;
+    frag->iov[0].iov_base = (IOVBASE_TYPE *) &frag->hdr;
     frag->iov[0].iov_len = sizeof(frag->hdr);
-    frag->iov[1].iov_base = (IOVBASE_TYPE*) &frag->segments[1];
+    frag->iov[1].iov_base = (IOVBASE_TYPE *) &frag->segments[1];
     frag->iov[1].iov_len = sizeof(mca_btl_base_segment_t);
     frag->hdr.base.tag = MCA_BTL_TAG_BTL;
     frag->hdr.type = MCA_BTL_TCP_HDR_TYPE_GET;
     frag->hdr.count = 1;
-    if (endpoint->endpoint_nbo) MCA_BTL_TCP_HDR_HTON(frag->hdr);
-    return ((rc = mca_btl_tcp_endpoint_send(endpoint,frag)) >= 0 ? OPAL_SUCCESS : rc);
+    if (endpoint->endpoint_nbo) {
+        MCA_BTL_TCP_HDR_HTON(frag->hdr);
+    }
+    return ((rc = mca_btl_tcp_endpoint_send(endpoint, frag)) >= 0 ? OPAL_SUCCESS : rc);
 }
 
-
 /*
  * Cleanup/release module resources.
  */
 
-int mca_btl_tcp_finalize(struct mca_btl_base_module_t* btl)
+int mca_btl_tcp_finalize(struct mca_btl_base_module_t *btl)
 {
-    mca_btl_tcp_module_t* tcp_btl = (mca_btl_tcp_module_t*) btl;
-    opal_list_item_t* item;
+    mca_btl_tcp_module_t *tcp_btl = (mca_btl_tcp_module_t *) btl;
+    opal_list_item_t *item;
 
     /* Don't lock the tcp_endpoints_mutex, at this point a single
      * thread should be active.
      */
-    for( item = opal_list_remove_first(&tcp_btl->tcp_endpoints);
-         item != NULL;
+    for (item = opal_list_remove_first(&tcp_btl->tcp_endpoints); item != NULL;
          item = opal_list_remove_first(&tcp_btl->tcp_endpoints)) {
-        mca_btl_tcp_endpoint_t *endpoint = (mca_btl_tcp_endpoint_t*)item;
+        mca_btl_tcp_endpoint_t *endpoint = (mca_btl_tcp_endpoint_t *) item;
         OBJ_RELEASE(endpoint);
     }
     free(tcp_btl);
     return OPAL_SUCCESS;
 }
 
-void mca_btl_tcp_dump(struct mca_btl_base_module_t* base_btl,
-                      struct mca_btl_base_endpoint_t* endpoint,
-                      int verbose)
+void mca_btl_tcp_dump(struct mca_btl_base_module_t *base_btl,
+                      struct mca_btl_base_endpoint_t *endpoint, int verbose)
 {
-    mca_btl_tcp_module_t* btl = (mca_btl_tcp_module_t*)base_btl;
+    mca_btl_tcp_module_t *btl = (mca_btl_tcp_module_t *) base_btl;
     mca_btl_base_err("%s TCP %p kernel_id %d\n"
 #if MCA_BTL_TCP_STATISTICS
                      " |   statistics: sent %lu recv %lu\n"
-#endif  /* MCA_BTL_TCP_STATISTICS */
+#endif /* MCA_BTL_TCP_STATISTICS */
                      " |   latency %u bandwidth %u\n",
-                     OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), (void*)btl, btl->tcp_ifkindex,
+                     OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), (void *) btl, btl->tcp_ifkindex,
 #if MCA_BTL_TCP_STATISTICS
                      btl->tcp_bytes_sent, btl->btl_bytes_recv,
-#endif  /* MCA_BTL_TCP_STATISTICS */
+#endif /* MCA_BTL_TCP_STATISTICS */
                      btl->super.btl_latency, btl->super.btl_bandwidth);
 #if OPAL_ENABLE_DEBUG && WANT_PEER_DUMP
-    if( NULL != endpoint ) {
+    if (NULL != endpoint) {
         MCA_BTL_TCP_ENDPOINT_DUMP(10, endpoint, false, "TCP");
 
-    } else if( verbose ) {
+    } else if (verbose) {
         opal_list_item_t *item;
 
         OPAL_THREAD_LOCK(&btl->tcp_endpoints_mutex);
-        for(item =  opal_list_get_first(&btl->tcp_endpoints);
-            item != opal_list_get_end(&btl->tcp_endpoints);
-            item = opal_list_get_next(item)) {
-            MCA_BTL_TCP_ENDPOINT_DUMP(10, (mca_btl_base_endpoint_t*)item, false, "TCP");
+        for (item = opal_list_get_first(&btl->tcp_endpoints);
+             item != opal_list_get_end(&btl->tcp_endpoints); item = opal_list_get_next(item)) {
+            MCA_BTL_TCP_ENDPOINT_DUMP(10, (mca_btl_base_endpoint_t *) item, false, "TCP");
         }
         OPAL_THREAD_UNLOCK(&btl->tcp_endpoints_mutex);
     }
 #endif /* OPAL_ENABLE_DEBUG && WANT_PEER_DUMP */
 }
 
-
 /*
- * A blocking recv for both blocking and non-blocking socket. 
- * Used to receive the small amount of connection information 
+ * A blocking recv for both blocking and non-blocking socket.
+ * Used to receive the small amount of connection information
  * that identifies the endpoints
- * 
- * when the socket is blocking (the caller introduces timeout) 
- * which happens during initial handshake otherwise socket is 
+ *
+ * when the socket is blocking (the caller introduces timeout)
+ * which happens during initial handshake otherwise socket is
  * non-blocking most of the time.
  */
 
-int mca_btl_tcp_recv_blocking(int sd, void* data, size_t size)
+int mca_btl_tcp_recv_blocking(int sd, void *data, size_t size)
 {
-    unsigned char* ptr = (unsigned char*)data;
+    unsigned char *ptr = (unsigned char *) data;
     size_t cnt = 0;
     while (cnt < size) {
-        int retval = recv(sd, ((char *)ptr) + cnt, size - cnt, 0);
+        int retval = recv(sd, ((char *) ptr) + cnt, size - cnt, 0);
         /* remote closed connection */
         if (0 == retval) {
             OPAL_OUTPUT_VERBOSE((100, opal_btl_base_framework.framework_output,
-                                 "remote peer unexpectedly closed connection while I was waiting for a blocking message"));
+                                 "remote peer unexpectedly closed connection while I was waiting "
+                                 "for a blocking message"));
             break;
         }
 
         /* socket is non-blocking so handle errors */
         if (retval < 0) {
-            if (opal_socket_errno != EINTR &&
-                opal_socket_errno != EAGAIN &&
-                opal_socket_errno != EWOULDBLOCK) {
-                BTL_VERBOSE(("recv(%d) failed: %s (%d)", sd, strerror(opal_socket_errno), opal_socket_errno));
+            if (opal_socket_errno != EINTR && opal_socket_errno != EAGAIN
+                && opal_socket_errno != EWOULDBLOCK) {
+                BTL_VERBOSE(("recv(%d) failed: %s (%d)", sd, strerror(opal_socket_errno),
+                             opal_socket_errno));
                 break;
             }
             continue;
@@ -572,24 +566,23 @@ int mca_btl_tcp_recv_blocking(int sd, void* data, size_t size)
     return cnt;
 }
 
-
 /*
  * A blocking send on a non-blocking socket. Used to send the small
  * amount of connection information used during the initial handshake
  * (magic string plus process guid)
  */
 
-int mca_btl_tcp_send_blocking(int sd, const void* data, size_t size)
+int mca_btl_tcp_send_blocking(int sd, const void *data, size_t size)
 {
-    unsigned char* ptr = (unsigned char*)data;
+    unsigned char *ptr = (unsigned char *) data;
     size_t cnt = 0;
-    while(cnt < size) {
-        int retval = send(sd, ((const char *)ptr) + cnt, size - cnt, 0);
+    while (cnt < size) {
+        int retval = send(sd, ((const char *) ptr) + cnt, size - cnt, 0);
         if (retval < 0) {
-            if (opal_socket_errno != EINTR &&
-                opal_socket_errno != EAGAIN &&
-                opal_socket_errno != EWOULDBLOCK) {
-                BTL_VERBOSE(("send() failed: %s (%d)", strerror(opal_socket_errno), opal_socket_errno));
+            if (opal_socket_errno != EINTR && opal_socket_errno != EAGAIN
+                && opal_socket_errno != EWOULDBLOCK) {
+                BTL_VERBOSE(
+                    ("send() failed: %s (%d)", strerror(opal_socket_errno), opal_socket_errno));
                 return -1;
             }
             continue;
diff --git a/opal/mca/btl/tcp/btl_tcp.h b/opal/mca/btl/tcp/btl_tcp.h
index a73970a8550..f8f1e49d25a 100644
--- a/opal/mca/btl/tcp/btl_tcp.h
+++ b/opal/mca/btl/tcp/btl_tcp.h
@@ -31,42 +31,42 @@
 
 #include "opal_config.h"
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 #ifdef HAVE_SYS_SOCKET_H
-#include <sys/socket.h>
+#    include <sys/socket.h>
 #endif
 #ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
+#    include <netinet/in.h>
 #endif
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 
 /* Open MPI includes */
-#include "opal/util/event.h"
 #include "opal/class/opal_free_list.h"
-#include "opal/mca/btl/btl.h"
+#include "opal/class/opal_hash_table.h"
 #include "opal/mca/btl/base/base.h"
+#include "opal/mca/btl/btl.h"
 #include "opal/mca/mpool/mpool.h"
-#include "opal/class/opal_hash_table.h"
+#include "opal/util/event.h"
 #include "opal/util/fd.h"
 
 #define MCA_BTL_TCP_STATISTICS 0
 BEGIN_C_DECLS
 
-extern opal_event_base_t* mca_btl_tcp_event_base;
+extern opal_event_base_t *mca_btl_tcp_event_base;
 
-#define MCA_BTL_TCP_COMPLETE_FRAG_SEND(frag)                            \
-    do {                                                                \
-        int btl_ownership = (frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP); \
-        if( frag->base.des_flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK ) { \
+#define MCA_BTL_TCP_COMPLETE_FRAG_SEND(frag)                                            \
+    do {                                                                                \
+        int btl_ownership = (frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);   \
+        if (frag->base.des_flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK) {                  \
             frag->base.des_cbfunc(&frag->endpoint->endpoint_btl->super, frag->endpoint, \
-                                  &frag->base, frag->rc);               \
-        }                                                               \
-        if( btl_ownership ) {                                           \
-            MCA_BTL_TCP_FRAG_RETURN(frag);                              \
-        }                                                               \
+                                  &frag->base, frag->rc);                               \
+        }                                                                               \
+        if (btl_ownership) {                                                            \
+            MCA_BTL_TCP_FRAG_RETURN(frag);                                              \
+        }                                                                               \
     } while (0)
 
 extern opal_list_t mca_btl_tcp_ready_frag_pending_queue;
@@ -74,21 +74,18 @@ extern opal_mutex_t mca_btl_tcp_ready_frag_mutex;
 extern int mca_btl_tcp_pipe_to_progress[2];
 extern int mca_btl_tcp_progress_thread_trigger;
 
-#define MCA_BTL_TCP_CRITICAL_SECTION_ENTER(name) \
-    opal_mutex_atomic_lock((name))
-#define MCA_BTL_TCP_CRITICAL_SECTION_LEAVE(name) \
-    opal_mutex_atomic_unlock((name))
-
-#define MCA_BTL_TCP_ACTIVATE_EVENT(event, value)                        \
-    do {                                                                \
-        if(0 < mca_btl_tcp_progress_thread_trigger) {                   \
-            opal_event_t* _event = (opal_event_t*)(event);                  \
-            (void) opal_fd_write( mca_btl_tcp_pipe_to_progress[1], sizeof(opal_event_t*), \
-                           &_event);                                        \
-        }                                                                   \
-        else {                                                          \
-            opal_event_add(event, (value));                             \
-        }                                                               \
+#define MCA_BTL_TCP_CRITICAL_SECTION_ENTER(name) opal_mutex_atomic_lock((name))
+#define MCA_BTL_TCP_CRITICAL_SECTION_LEAVE(name) opal_mutex_atomic_unlock((name))
+
+#define MCA_BTL_TCP_ACTIVATE_EVENT(event, value)                                          \
+    do {                                                                                  \
+        if (0 < mca_btl_tcp_progress_thread_trigger) {                                    \
+            opal_event_t *_event = (opal_event_t *) (event);                              \
+            (void) opal_fd_write(mca_btl_tcp_pipe_to_progress[1], sizeof(opal_event_t *), \
+                                 &_event);                                                \
+        } else {                                                                          \
+            opal_event_add(event, (value));                                               \
+        }                                                                                 \
     } while (0)
 
 /**
@@ -96,53 +93,53 @@ extern int mca_btl_tcp_progress_thread_trigger;
  */
 
 struct mca_btl_tcp_component_t {
-    mca_btl_base_component_3_0_0_t super;   /**< base BTL component */
-    uint32_t tcp_addr_count;                /**< total number of addresses */
-    uint32_t tcp_num_btls;                  /**< number of interfaces available to the TCP component */
-    unsigned int tcp_num_links;             /**< number of logical links per physical device */
+    mca_btl_base_component_3_0_0_t super; /**< base BTL component */
+    uint32_t tcp_addr_count;              /**< total number of addresses */
+    uint32_t tcp_num_btls;      /**< number of interfaces available to the TCP component */
+    unsigned int tcp_num_links; /**< number of logical links per physical device */
     struct mca_btl_tcp_module_t **tcp_btls; /**< array of available BTL modules */
-    opal_list_t local_ifs;		    /**< opal list of local opal_if_t interfaces */
+    opal_list_t local_ifs;                  /**< opal list of local opal_if_t interfaces */
     int tcp_free_list_num;                  /**< initial size of free lists */
     int tcp_free_list_max;                  /**< maximum size of free lists */
-    int tcp_free_list_inc;                  /**< number of elements to alloc when growing free lists */
-    int tcp_endpoint_cache;                 /**< amount of cache on each endpoint */
-    opal_proc_table_t tcp_procs;            /**< hash table of tcp proc structures */
-    opal_mutex_t tcp_lock;                  /**< lock for accessing module state */
+    int tcp_free_list_inc;       /**< number of elements to alloc when growing free lists */
+    int tcp_endpoint_cache;      /**< amount of cache on each endpoint */
+    opal_proc_table_t tcp_procs; /**< hash table of tcp proc structures */
+    opal_mutex_t tcp_lock;       /**< lock for accessing module state */
     opal_list_t tcp_events;
 
-    opal_event_t tcp_recv_event;            /**< recv event for IPv4 listen socket */
-    int tcp_listen_sd;                      /**< IPv4 listen socket for incoming connection requests */
-    unsigned short tcp_listen_port;         /**< IPv4 listen port */
-    int tcp_port_min;                       /**< IPv4 minimum port */
-    int tcp_port_range;                     /**< IPv4 port range */
+    opal_event_t tcp_recv_event;    /**< recv event for IPv4 listen socket */
+    int tcp_listen_sd;              /**< IPv4 listen socket for incoming connection requests */
+    unsigned short tcp_listen_port; /**< IPv4 listen port */
+    int tcp_port_min;               /**< IPv4 minimum port */
+    int tcp_port_range;             /**< IPv4 port range */
 #if OPAL_ENABLE_IPV6
-    opal_event_t tcp6_recv_event;           /**< recv event for IPv6 listen socket */
-    int tcp6_listen_sd;                     /**< IPv6 listen socket for incoming connection requests */
-    unsigned short tcp6_listen_port;        /**< IPv6 listen port */
-    int tcp6_port_min;                      /**< IPv4 minimum port */
-    int tcp6_port_range;                    /**< IPv4 port range */
+    opal_event_t tcp6_recv_event;    /**< recv event for IPv6 listen socket */
+    int tcp6_listen_sd;              /**< IPv6 listen socket for incoming connection requests */
+    unsigned short tcp6_listen_port; /**< IPv6 listen port */
+    int tcp6_port_min;               /**< IPv4 minimum port */
+    int tcp6_port_range;             /**< IPv4 port range */
 #endif
     /* Port range restriction */
 
-    char*  tcp_if_include;                  /**< comma seperated list of interface to include */
-    char*  tcp_if_exclude;                  /**< comma seperated list of interface to exclude */
-    int    tcp_sndbuf;                      /**< socket sndbuf size */
-    int    tcp_rcvbuf;                      /**< socket rcvbuf size */
-    int    tcp_disable_family;              /**< disabled AF_family */
+    char *tcp_if_include;   /**< comma separated list of interface to include */
+    char *tcp_if_exclude;   /**< comma separated list of interface to exclude */
+    int tcp_sndbuf;         /**< socket sndbuf size */
+    int tcp_rcvbuf;         /**< socket rcvbuf size */
+    int tcp_disable_family; /**< disabled AF_family */
 
     /* free list of fragment descriptors */
     opal_free_list_t tcp_frag_eager;
     opal_free_list_t tcp_frag_max;
     opal_free_list_t tcp_frag_user;
 
-    int tcp_enable_progress_thread;         /** Support for tcp progress thread flag */
+    int tcp_enable_progress_thread; /** Support for tcp progress thread flag */
 
     opal_event_t tcp_recv_thread_async_event;
     opal_mutex_t tcp_frag_eager_mutex;
     opal_mutex_t tcp_frag_max_mutex;
     opal_mutex_t tcp_frag_user_mutex;
     /* Do we want to use TCP_NODELAY? */
-    int    tcp_not_use_nodelay;
+    int tcp_not_use_nodelay;
 
     /* do we want to warn on all excluded interfaces
      * that are not found?
@@ -151,28 +148,28 @@ struct mca_btl_tcp_component_t {
 };
 typedef struct mca_btl_tcp_component_t mca_btl_tcp_component_t;
 
-OPAL_MODULE_DECLSPEC extern mca_btl_tcp_component_t mca_btl_tcp_component;
+OPAL_DECLSPEC extern mca_btl_tcp_component_t mca_btl_tcp_component;
 
 /**
  * BTL Module Interface
  */
 struct mca_btl_tcp_module_t {
-    mca_btl_base_module_t  super;  /**< base BTL interface */
-    uint32_t           btl_index;  /**< Local BTL module index, used for vertex
-                                        data and used as a hash key when
-                                        solving module matching problem */
-    uint16_t           tcp_ifkindex; /** <BTL kernel interface index */
-    struct sockaddr_storage tcp_ifaddr;   /**< First address
-                                             discovered for this
-                                             interface, bound as
-                                             sending address for this
-                                             BTL */
-    uint32_t           tcp_ifmask;  /**< BTL interface netmask */
-
-    opal_mutex_t       tcp_endpoints_mutex;
-    opal_list_t        tcp_endpoints;
-
-    mca_btl_base_module_error_cb_fn_t tcp_error_cb;  /**< Upper layer error callback */
+    mca_btl_base_module_t super;        /**< base BTL interface */
+    uint32_t btl_index;                 /**< Local BTL module index, used for vertex
+                                             data and used as a hash key when
+                                             solving module matching problem */
+    uint16_t tcp_ifkindex;              /** <BTL kernel interface index */
+    struct sockaddr_storage tcp_ifaddr; /**< First address
+                                           discovered for this
+                                           interface, bound as
+                                           sending address for this
+                                           BTL */
+    uint32_t tcp_ifmask;                /**< BTL interface netmask */
+
+    opal_mutex_t tcp_endpoints_mutex;
+    opal_list_t tcp_endpoints;
+
+    mca_btl_base_module_error_cb_fn_t tcp_error_cb; /**< Upper layer error callback */
 #if MCA_BTL_TCP_STATISTICS
     size_t tcp_bytes_sent;
     size_t tcp_bytes_recv;
@@ -182,21 +179,24 @@ struct mca_btl_tcp_module_t {
 typedef struct mca_btl_tcp_module_t mca_btl_tcp_module_t;
 extern mca_btl_tcp_module_t mca_btl_tcp_module;
 
-#define CLOSE_THE_SOCKET(socket)   {OPAL_OUTPUT_VERBOSE((20, opal_btl_base_framework.framework_output, "CLOSE FD %d at %s:%d", socket, __FILE__, __LINE__)); (void)shutdown(socket, SHUT_RDWR); (void)close(socket);}
+#define CLOSE_THE_SOCKET(socket)                                                                   \
+    {                                                                                              \
+        OPAL_OUTPUT_VERBOSE((20, opal_btl_base_framework.framework_output, "CLOSE FD %d at %s:%d", \
+                             socket, __FILE__, __LINE__));                                         \
+        (void) shutdown(socket, SHUT_RDWR);                                                        \
+        (void) close(socket);                                                                      \
+    }
 
 /**
  * TCP component initialization.
  *
  * @param num_btl_modules (OUT)           Number of BTLs returned in BTL array.
- * @param allow_multi_user_threads (OUT)  Flag indicating wether BTL supports user threads (TRUE)
- * @param have_hidden_threads (OUT)       Flag indicating wether BTL uses threads (TRUE)
+ * @param allow_multi_user_threads (OUT)  Flag indicating whether BTL supports user threads (TRUE)
+ * @param have_hidden_threads (OUT)       Flag indicating whether BTL uses threads (TRUE)
  */
-extern mca_btl_base_module_t** mca_btl_tcp_component_init(
-    int *num_btl_modules,
-    bool allow_multi_user_threads,
-    bool have_hidden_threads
-);
-
+extern mca_btl_base_module_t **mca_btl_tcp_component_init(int *num_btl_modules,
+                                                          bool allow_multi_user_threads,
+                                                          bool have_hidden_threads);
 
 /**
  * Cleanup any resources held by the BTL.
@@ -205,10 +205,7 @@ extern mca_btl_base_module_t** mca_btl_tcp_component_init(
  * @return     OPAL_SUCCESS or error status on failure.
  */
 
-extern int mca_btl_tcp_finalize(
-    struct mca_btl_base_module_t* btl
-);
-
+extern int mca_btl_tcp_finalize(struct mca_btl_base_module_t *btl);
 
 /**
  * PML->BTL notification of change in the process list.
@@ -222,13 +219,9 @@ extern int mca_btl_tcp_finalize(
  *
  */
 
-extern int mca_btl_tcp_add_procs(
-    struct mca_btl_base_module_t* btl,
-    size_t nprocs,
-    struct opal_proc_t **procs,
-    struct mca_btl_base_endpoint_t** peers,
-    opal_bitmap_t* reachable
-);
+extern int mca_btl_tcp_add_procs(struct mca_btl_base_module_t *btl, size_t nprocs,
+                                 struct opal_proc_t **procs, struct mca_btl_base_endpoint_t **peers,
+                                 opal_bitmap_t *reachable);
 
 /**
  * PML->BTL notification of change in the process list.
@@ -241,49 +234,44 @@ extern int mca_btl_tcp_add_procs(
  *
  */
 
-extern int mca_btl_tcp_del_procs(
-    struct mca_btl_base_module_t* btl,
-    size_t nprocs,
-    struct opal_proc_t **procs,
-    struct mca_btl_base_endpoint_t** peers
-);
-
+extern int mca_btl_tcp_del_procs(struct mca_btl_base_module_t *btl, size_t nprocs,
+                                 struct opal_proc_t **procs,
+                                 struct mca_btl_base_endpoint_t **peers);
 
 /**
  * Initiate an asynchronous send.
  *
  * @param btl (IN)         BTL module
  * @param endpoint (IN)    BTL addressing information
- * @param descriptor (IN)  Description of the data to be transfered
+ * @param descriptor (IN)  Description of the data to be transferred
  * @param tag (IN)         The tag value used to notify the peer.
  */
 
-extern int mca_btl_tcp_send(
-    struct mca_btl_base_module_t* btl,
-    struct mca_btl_base_endpoint_t* btl_peer,
-    struct mca_btl_base_descriptor_t* descriptor,
-    mca_btl_base_tag_t tag
-);
-
+extern int mca_btl_tcp_send(struct mca_btl_base_module_t *btl,
+                            struct mca_btl_base_endpoint_t *btl_peer,
+                            struct mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag);
 
 /**
  * Initiate an asynchronous put.
  */
 
-int mca_btl_tcp_put (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, void *local_address,
-                     uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                     mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-                     int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
-
+int mca_btl_tcp_put(mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+                    void *local_address, uint64_t remote_address,
+                    mca_btl_base_registration_handle_t *local_handle,
+                    mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
+                    int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext,
+                    void *cbdata);
 
 /**
  * Initiate an asynchronous get.
  */
 
-int mca_btl_tcp_get (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, void *local_address,
-                     uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                     mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-                     int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
+int mca_btl_tcp_get(mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+                    void *local_address, uint64_t remote_address,
+                    mca_btl_base_registration_handle_t *local_handle,
+                    mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
+                    int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext,
+                    void *cbdata);
 
 /**
  * Allocate a descriptor with a segment of the requested size.
@@ -294,13 +282,9 @@ int mca_btl_tcp_get (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t
  * @param size (IN)     Request segment size.
  */
 
-extern mca_btl_base_descriptor_t* mca_btl_tcp_alloc(
-    struct mca_btl_base_module_t* btl,
-    struct mca_btl_base_endpoint_t* endpoint,
-    uint8_t order,
-    size_t size,
-    uint32_t flags);
-
+extern mca_btl_base_descriptor_t *mca_btl_tcp_alloc(struct mca_btl_base_module_t *btl,
+                                                    struct mca_btl_base_endpoint_t *endpoint,
+                                                    uint8_t order, size_t size, uint32_t flags);
 
 /**
  * Return a segment allocated by this BTL.
@@ -309,14 +293,11 @@ extern mca_btl_base_descriptor_t* mca_btl_tcp_alloc(
  * @param descriptor (IN)  Allocated descriptor.
  */
 
-extern int mca_btl_tcp_free(
-    struct mca_btl_base_module_t* btl,
-    mca_btl_base_descriptor_t* des);
-
+extern int mca_btl_tcp_free(struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des);
 
 /**
  * Prepare a descriptor for send/rdma using the supplied
- * convertor. If the convertor references data that is contigous,
+ * convertor. If the convertor references data that is contiguous,
  * the descriptor may simply point to the user buffer. Otherwise,
  * this routine is responsible for allocating buffer space and
  * packing if required.
@@ -326,29 +307,23 @@ extern int mca_btl_tcp_free(
  * @param convertor (IN)    Data type convertor
  * @param reserve (IN)      Additional bytes requested by upper layer to precede user data
  * @param size (IN/OUT)     Number of bytes to prepare (IN), number of bytes actually prepared (OUT)
-*/
-
-mca_btl_base_descriptor_t* mca_btl_tcp_prepare_src(
-    struct mca_btl_base_module_t* btl,
-    struct mca_btl_base_endpoint_t* peer,
-    struct opal_convertor_t* convertor,
-    uint8_t order,
-    size_t reserve,
-    size_t* size,
-    uint32_t flags
-);
-
-extern void
-mca_btl_tcp_dump(struct mca_btl_base_module_t* btl,
-                 struct mca_btl_base_endpoint_t* endpoint,
-                 int verbose);
+ */
+
+mca_btl_base_descriptor_t *mca_btl_tcp_prepare_src(struct mca_btl_base_module_t *btl,
+                                                   struct mca_btl_base_endpoint_t *peer,
+                                                   struct opal_convertor_t *convertor,
+                                                   uint8_t order, size_t reserve, size_t *size,
+                                                   uint32_t flags);
+
+extern void mca_btl_tcp_dump(struct mca_btl_base_module_t *btl,
+                             struct mca_btl_base_endpoint_t *endpoint, int verbose);
 
 /*
  * A blocking send on a non-blocking socket. Used to send the small
  * amount of connection information that identifies the endpoints
  * endpoint.
  */
-int mca_btl_tcp_send_blocking(int sd, const void* data, size_t size);
+int mca_btl_tcp_send_blocking(int sd, const void *data, size_t size);
 
 /*
  * A blocking recv for both blocking and non-blocking socket.
@@ -359,7 +334,7 @@ int mca_btl_tcp_send_blocking(int sd, const void* data, size_t size);
  * which happens during initial handshake otherwise socket is
  * non-blocking most of the time.
  */
-int mca_btl_tcp_recv_blocking(int sd, void* data, size_t size);
+int mca_btl_tcp_recv_blocking(int sd, void *data, size_t size);
 
 END_C_DECLS
 #endif
diff --git a/opal/mca/btl/tcp/btl_tcp_addr.h b/opal/mca/btl/tcp/btl_tcp_addr.h
index d714df36bce..f456970a318 100644
--- a/opal/mca/btl/tcp/btl_tcp_addr.h
+++ b/opal/mca/btl/tcp/btl_tcp_addr.h
@@ -25,13 +25,13 @@
 #define MCA_BTL_TCP_ADDR_H
 
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 #ifdef HAVE_SYS_SOCKET_H
-#include <sys/socket.h>
+#    include <sys/socket.h>
 #endif
 #ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
+#    include <netinet/in.h>
 #endif
 #include <assert.h>
 
@@ -46,19 +46,19 @@
  * structure.
  */
 struct mca_btl_tcp_modex_addr_t {
-    uint8_t     addr[16];       /* endpoint address.  for addr_family
-                                   of MCA_BTL_TCP_AF_INET, only the
-                                   first 4 bytes have meaning. */
-    uint32_t    addr_ifkindex;  /* endpoint kernel index */
-    uint32_t    addr_mask;      /* ip mask */
-    uint32_t    addr_bandwidth; /* interface bandwidth */
-    uint16_t    addr_port;      /* endpoint listen port */
-    uint8_t     addr_family;    /* endpoint address family.  Note that
-                                   this is
-                                   MCA_BTL_TCP_AF_{INET,INET6}, not
-                                   the traditional
-                                   AF_INET/AF_INET6. */
-    uint8_t     padding[1];     /* pad out to an 8-byte word */
+    uint8_t addr[16];        /* endpoint address.  for addr_family
+                                of MCA_BTL_TCP_AF_INET, only the
+                                first 4 bytes have meaning. */
+    uint32_t addr_ifkindex;  /* endpoint kernel index */
+    uint32_t addr_mask;      /* ip mask */
+    uint32_t addr_bandwidth; /* interface bandwidth */
+    uint16_t addr_port;      /* endpoint listen port */
+    uint8_t addr_family;     /* endpoint address family.  Note that
+                                this is
+                                MCA_BTL_TCP_AF_{INET,INET6}, not
+                                the traditional
+                                AF_INET/AF_INET6. */
+    uint8_t padding[1];      /* pad out to an 8-byte word */
 };
 typedef struct mca_btl_tcp_modex_addr_t mca_btl_tcp_modex_addr_t;
 
@@ -75,20 +75,19 @@ _Static_assert(sizeof(struct mca_btl_tcp_modex_addr_t) == 32, "mca_btl_tcp_modex
  */
 struct mca_btl_tcp_addr_t {
     union {
-        struct in_addr  addr_inet;      /* IPv6 listen address */
+        struct in_addr addr_inet; /* IPv6 listen address */
 #if OPAL_ENABLE_IPV6
-        struct in6_addr addr_inet6;     /* IPv6 listen address */
+        struct in6_addr addr_inet6; /* IPv6 listen address */
 #endif
     } addr_union;
-    in_port_t   addr_port;     /**< listen port */
-    int         addr_ifkindex; /**< remote interface index assigned with
-                                    this address */
-    uint8_t     addr_family;   /**< AF_INET or AF_INET6 */
+    in_port_t addr_port; /**< listen port */
+    int addr_ifkindex;   /**< remote interface index assigned with
+                              this address */
+    uint8_t addr_family; /**< AF_INET or AF_INET6 */
 };
 typedef struct mca_btl_tcp_addr_t mca_btl_tcp_addr_t;
 
-#define MCA_BTL_TCP_AF_INET     0
-#define MCA_BTL_TCP_AF_INET6    1
+#define MCA_BTL_TCP_AF_INET  0
+#define MCA_BTL_TCP_AF_INET6 1
 
 #endif
-
diff --git a/opal/mca/btl/tcp/btl_tcp_component.c b/opal/mca/btl/tcp/btl_tcp_component.c
index fe08760a957..131d9c1fe53 100644
--- a/opal/mca/btl/tcp/btl_tcp_component.c
+++ b/opal/mca/btl/tcp/btl_tcp_component.c
@@ -19,8 +19,8 @@
  * Copyright (c) 2014-2019 Intel, Inc.  All rights reserved.
  * Copyright (c) 2014-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2018-2020 Amazon.com, Inc. or its affiliates.  All Rights
- *                         reserved.
+ * Copyright (c) 2018-2022 Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -33,69 +33,65 @@
 
 #include "opal/opal_socket_errno.h"
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
-#include <string.h>
 #include <fcntl.h>
+#include <string.h>
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 #ifdef HAVE_SYS_SOCKET_H
-#include <sys/socket.h>
+#    include <sys/socket.h>
 #endif
 #ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
+#    include <netinet/in.h>
 #endif
 #ifdef HAVE_ARPA_INET_H
-#include <arpa/inet.h>
+#    include <arpa/inet.h>
 #endif
 #if OPAL_ENABLE_IPV6
-#  ifdef HAVE_NETDB_H
-#  include <netdb.h>
-#  endif
+#    ifdef HAVE_NETDB_H
+#        include <netdb.h>
+#    endif
 #endif
 #include <ctype.h>
 #include <limits.h>
 #ifdef HAVE_SYS_TIME_H
-#include <sys/time.h>
+#    include <sys/time.h>
 #endif
 
-#include "opal/util/event.h"
+#include "opal/constants.h"
+#include "opal/mca/btl/base/base.h"
+#include "opal/mca/btl/base/btl_base_error.h"
+#include "opal/mca/btl/btl.h"
+#include "opal/mca/mpool/base/base.h"
+#include "opal/mca/pmix/pmix-internal.h"
+#include "opal/mca/reachable/base/base.h"
+#include "opal/mca/threads/threads.h"
+#include "opal/util/argv.h"
 #include "opal/util/ethtool.h"
+#include "opal/util/event.h"
+#include "opal/util/fd.h"
 #include "opal/util/if.h"
-#include "opal/util/output.h"
-#include "opal/util/argv.h"
 #include "opal/util/net.h"
+#include "opal/util/output.h"
+#include "opal/util/printf.h"
 #include "opal/util/proc.h"
-#include "opal/util/net.h"
-#include "opal/util/fd.h"
 #include "opal/util/show_help.h"
 #include "opal/util/string_copy.h"
-#include "opal/util/printf.h"
-#include "opal/constants.h"
-#include "opal/mca/btl/btl.h"
-#include "opal/mca/btl/base/base.h"
-#include "opal/mca/mpool/base/base.h"
-#include "opal/mca/btl/base/btl_base_error.h"
-#include "opal/mca/reachable/base/base.h"
-#include "opal/mca/pmix/pmix-internal.h"
-#include "opal/mca/threads/threads.h"
 
-#include "opal/constants.h"
-#include "opal/mca/btl/btl.h"
-#include "opal/mca/btl/base/base.h"
-#include "opal/mca/btl/base/btl_base_error.h"
 #include "btl_tcp.h"
 #include "btl_tcp_addr.h"
-#include "btl_tcp_proc.h"
-#include "btl_tcp_frag.h"
 #include "btl_tcp_endpoint.h"
-#if OPAL_CUDA_SUPPORT
-#include "opal/mca/common/cuda/common_cuda.h"
-#endif /* OPAL_CUDA_SUPPORT */
+#include "btl_tcp_frag.h"
+#include "btl_tcp_proc.h"
+#include "opal/constants.h"
+#include "opal/mca/btl/base/base.h"
+#include "opal/mca/btl/base/btl_base_error.h"
+#include "opal/mca/btl/btl.h"
 
 #define MCA_BTL_TCP_BTL_BANDWIDTH 100
-#define MCA_BTL_TCP_BTL_LATENCY 100
+#define MCA_BTL_TCP_BTL_LATENCY   100
 
 /*
  * Local functions
@@ -104,11 +100,11 @@ static int mca_btl_tcp_component_register(void);
 static int mca_btl_tcp_component_open(void);
 static int mca_btl_tcp_component_close(void);
 
-opal_event_base_t* mca_btl_tcp_event_base = NULL;
+opal_event_base_t *mca_btl_tcp_event_base = NULL;
 int mca_btl_tcp_progress_thread_trigger = -1;
-int mca_btl_tcp_pipe_to_progress[2] = { -1, -1 };
-static opal_thread_t mca_btl_tcp_progress_thread = { { 0 } };
-opal_list_t mca_btl_tcp_ready_frag_pending_queue = { { 0 } };
+int mca_btl_tcp_pipe_to_progress[2] = {-1, -1};
+static opal_thread_t mca_btl_tcp_progress_thread = {{0}};
+opal_list_t mca_btl_tcp_ready_frag_pending_queue = {{0}};
 opal_mutex_t mca_btl_tcp_ready_frag_mutex = OPAL_MUTEX_STATIC_INIT;
 
 mca_btl_tcp_component_t mca_btl_tcp_component = {
@@ -116,72 +112,59 @@ mca_btl_tcp_component_t mca_btl_tcp_component = {
         /* First, the mca_base_component_t struct containing meta information
            about the component itself */
 
-        .btl_version = {
-            MCA_BTL_DEFAULT_VERSION("tcp"),
-            .mca_open_component = mca_btl_tcp_component_open,
-            .mca_close_component = mca_btl_tcp_component_close,
-            .mca_register_component_params = mca_btl_tcp_component_register,
-        },
-        .btl_data = {
-            /* The component is checkpoint ready */
-            .param_field = MCA_BASE_METADATA_PARAM_CHECKPOINT
-        },
+        .btl_version =
+            {
+                MCA_BTL_DEFAULT_VERSION("tcp"),
+                .mca_open_component = mca_btl_tcp_component_open,
+                .mca_close_component = mca_btl_tcp_component_close,
+                .mca_register_component_params = mca_btl_tcp_component_register,
+            },
+        .btl_data =
+            {/* The component is checkpoint ready */
+             .param_field = MCA_BASE_METADATA_PARAM_CHECKPOINT},
 
         .btl_init = mca_btl_tcp_component_init,
         .btl_progress = NULL,
-    }
-};
+    }};
 
 /*
  * utility routines for parameter registration
  */
 
-static inline char* mca_btl_tcp_param_register_string(
-        const char* param_name,
-        const char* help_string,
-        const char* default_value,
-        int level,
-        char **storage)
+static inline char *mca_btl_tcp_param_register_string(const char *param_name,
+                                                      const char *help_string,
+                                                      const char *default_value, int level,
+                                                      char **storage)
 {
     *storage = (char *) default_value;
-    (void) mca_base_component_var_register(&mca_btl_tcp_component.super.btl_version,
-                                           param_name, help_string, MCA_BASE_VAR_TYPE_STRING,
-                                           NULL, 0, 0, level,
+    (void) mca_base_component_var_register(&mca_btl_tcp_component.super.btl_version, param_name,
+                                           help_string, MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, level,
                                            MCA_BASE_VAR_SCOPE_READONLY, storage);
     return *storage;
 }
 
-static inline int mca_btl_tcp_param_register_int(
-        const char* param_name,
-        const char* help_string,
-        int default_value,
-        int level,
-        int *storage)
+static inline int mca_btl_tcp_param_register_int(const char *param_name, const char *help_string,
+                                                 int default_value, int level, int *storage)
 {
     *storage = default_value;
-    (void) mca_base_component_var_register(&mca_btl_tcp_component.super.btl_version,
-                                           param_name, help_string, MCA_BASE_VAR_TYPE_INT,
-                                           NULL, 0, 0, level,
+    (void) mca_base_component_var_register(&mca_btl_tcp_component.super.btl_version, param_name,
+                                           help_string, MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, level,
                                            MCA_BASE_VAR_SCOPE_READONLY, storage);
     return *storage;
 }
 
-static inline unsigned int mca_btl_tcp_param_register_uint(
-        const char* param_name,
-        const char* help_string,
-        unsigned int default_value,
-        int level,
-        unsigned int *storage)
+static inline unsigned int mca_btl_tcp_param_register_uint(const char *param_name,
+                                                           const char *help_string,
+                                                           unsigned int default_value, int level,
+                                                           unsigned int *storage)
 {
     *storage = default_value;
-    (void) mca_base_component_var_register(&mca_btl_tcp_component.super.btl_version,
-                                           param_name, help_string, MCA_BASE_VAR_TYPE_UNSIGNED_INT,
-                                           NULL, 0, 0, level,
-                                           MCA_BASE_VAR_SCOPE_READONLY, storage);
+    (void) mca_base_component_var_register(&mca_btl_tcp_component.super.btl_version, param_name,
+                                           help_string, MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, 0,
+                                           level, MCA_BASE_VAR_SCOPE_READONLY, storage);
     return *storage;
 }
 
-
 /*
  * Data structure for accepting connections.
  */
@@ -192,46 +175,40 @@ struct mca_btl_tcp_event_t {
 };
 typedef struct mca_btl_tcp_event_t mca_btl_tcp_event_t;
 
-static void mca_btl_tcp_event_construct(mca_btl_tcp_event_t* event)
+static void mca_btl_tcp_event_construct(mca_btl_tcp_event_t *event)
 {
     MCA_BTL_TCP_CRITICAL_SECTION_ENTER(&mca_btl_tcp_component.tcp_lock);
     opal_list_append(&mca_btl_tcp_component.tcp_events, &event->item);
     MCA_BTL_TCP_CRITICAL_SECTION_LEAVE(&mca_btl_tcp_component.tcp_lock);
 }
 
-static void mca_btl_tcp_event_destruct(mca_btl_tcp_event_t* event)
+static void mca_btl_tcp_event_destruct(mca_btl_tcp_event_t *event)
 {
     MCA_BTL_TCP_CRITICAL_SECTION_ENTER(&mca_btl_tcp_component.tcp_lock);
     opal_list_remove_item(&mca_btl_tcp_component.tcp_events, &event->item);
     MCA_BTL_TCP_CRITICAL_SECTION_LEAVE(&mca_btl_tcp_component.tcp_lock);
 }
 
-OBJ_CLASS_INSTANCE(
-    mca_btl_tcp_event_t,
-    opal_list_item_t,
-    mca_btl_tcp_event_construct,
-    mca_btl_tcp_event_destruct);
-
+OBJ_CLASS_INSTANCE(mca_btl_tcp_event_t, opal_list_item_t, mca_btl_tcp_event_construct,
+                   mca_btl_tcp_event_destruct);
 
 /*
  * functions for receiving event callbacks
  */
-static void mca_btl_tcp_component_recv_handler(int, short, void*);
-static void mca_btl_tcp_component_accept_handler(int, short, void*);
+static void mca_btl_tcp_component_recv_handler(int, short, void *);
+static void mca_btl_tcp_component_accept_handler(int, short, void *);
 
 static int mca_btl_tcp_component_verify(void)
 {
-    if( mca_btl_tcp_component.tcp_port_min > USHRT_MAX ) {
-        opal_show_help("help-mpi-btl-tcp.txt", "invalid minimum port",
-                       true, "v4", opal_process_info.nodename,
-                       mca_btl_tcp_component.tcp_port_min );
+    if (mca_btl_tcp_component.tcp_port_min > USHRT_MAX) {
+        opal_show_help("help-mpi-btl-tcp.txt", "invalid minimum port", true, "v4",
+                       opal_process_info.nodename, mca_btl_tcp_component.tcp_port_min);
         mca_btl_tcp_component.tcp_port_min = 1024;
     }
 #if OPAL_ENABLE_IPV6
-    if( mca_btl_tcp_component.tcp6_port_min > USHRT_MAX ) {
-        opal_show_help("help-mpi-btl-tcp.txt", "invalid minimum port",
-                       true, "v6", opal_process_info.nodename,
-                       mca_btl_tcp_component.tcp6_port_min );
+    if (mca_btl_tcp_component.tcp6_port_min > USHRT_MAX) {
+        opal_show_help("help-mpi-btl-tcp.txt", "invalid minimum port", true, "v6",
+                       opal_process_info.nodename, mca_btl_tcp_component.tcp6_port_min);
         mca_btl_tcp_component.tcp6_port_min = 1024;
     }
 #endif
@@ -246,96 +223,111 @@ static int mca_btl_tcp_component_verify(void)
 
 static int mca_btl_tcp_component_register(void)
 {
-    char* message;
+    char *message;
 
     /* register TCP component parameters */
-    mca_btl_tcp_param_register_uint("links", NULL, 1, OPAL_INFO_LVL_4, &mca_btl_tcp_component.tcp_num_links);
-    mca_btl_tcp_param_register_string("if_include", "Comma-delimited list of devices and/or CIDR notation of networks to use for MPI communication (e.g., \"eth0,192.168.0.0/16\").  Mutually exclusive with btl_tcp_if_exclude.", "", OPAL_INFO_LVL_1, &mca_btl_tcp_component.tcp_if_include);
-    mca_btl_tcp_param_register_string("if_exclude", "Comma-delimited list of devices and/or CIDR notation of networks to NOT use for MPI communication -- all devices not matching these specifications will be used (e.g., \"eth0,192.168.0.0/16\").  If set to a non-default value, it is mutually exclusive with btl_tcp_if_include.",
-                                      "127.0.0.1/8,sppp",
-                                      OPAL_INFO_LVL_1, &mca_btl_tcp_component.tcp_if_exclude);
-
-    mca_btl_tcp_param_register_int ("free_list_num", NULL, 8, OPAL_INFO_LVL_5,  &mca_btl_tcp_component.tcp_free_list_num);
-    mca_btl_tcp_param_register_int ("free_list_max", NULL, -1, OPAL_INFO_LVL_5,  &mca_btl_tcp_component.tcp_free_list_max);
-    mca_btl_tcp_param_register_int ("free_list_inc", NULL, 32, OPAL_INFO_LVL_5,  &mca_btl_tcp_component.tcp_free_list_inc);
-    mca_btl_tcp_param_register_int ("sndbuf",
-                                    "The size of the send buffer socket option for each connection.  "
-                                    "Modern TCP stacks generally are smarter than a fixed size and in some "
-                                    "situations setting a buffer size explicitly can actually lower "
-                                    "performance.  0 means the tcp btl will not try to set a send buffer "
-                                    "size.",
-                                    0, OPAL_INFO_LVL_4, &mca_btl_tcp_component.tcp_sndbuf);
-    mca_btl_tcp_param_register_int ("rcvbuf",
-                                    "The size of the receive buffer socket option for each connection.  "
-                                    "Modern TCP stacks generally are smarter than a fixed size and in some "
-                                    "situations setting a buffer size explicitly can actually lower "
-                                    "performance.  0 means the tcp btl will not try to set a receive buffer "
-                                    "size.",
-                                    0, OPAL_INFO_LVL_4, &mca_btl_tcp_component.tcp_rcvbuf);
-    mca_btl_tcp_param_register_int ("endpoint_cache",
+    mca_btl_tcp_param_register_uint("links", NULL, 1, OPAL_INFO_LVL_4,
+                                    &mca_btl_tcp_component.tcp_num_links);
+    mca_btl_tcp_param_register_string(
+        "if_include",
+        "Comma-delimited list of devices and/or CIDR notation of networks to use for MPI "
+        "communication (e.g., \"eth0,192.168.0.0/16\").  Mutually exclusive with "
+        "btl_tcp_if_exclude.",
+        "", OPAL_INFO_LVL_1, &mca_btl_tcp_component.tcp_if_include);
+    mca_btl_tcp_param_register_string(
+        "if_exclude",
+        "Comma-delimited list of devices and/or CIDR notation of networks to NOT use for MPI "
+        "communication -- all devices not matching these specifications will be used (e.g., "
+        "\"eth0,192.168.0.0/16\").  If set to a non-default value, it is mutually exclusive with "
+        "btl_tcp_if_include.",
+        "127.0.0.1/8,sppp", OPAL_INFO_LVL_1, &mca_btl_tcp_component.tcp_if_exclude);
+
+    mca_btl_tcp_param_register_int("free_list_num", NULL, 8, OPAL_INFO_LVL_5,
+                                   &mca_btl_tcp_component.tcp_free_list_num);
+    mca_btl_tcp_param_register_int("free_list_max", NULL, -1, OPAL_INFO_LVL_5,
+                                   &mca_btl_tcp_component.tcp_free_list_max);
+    mca_btl_tcp_param_register_int("free_list_inc", NULL, 32, OPAL_INFO_LVL_5,
+                                   &mca_btl_tcp_component.tcp_free_list_inc);
+    mca_btl_tcp_param_register_int(
+        "sndbuf",
+        "The size of the send buffer socket option for each connection.  "
+        "Modern TCP stacks generally are smarter than a fixed size and in some "
+        "situations setting a buffer size explicitly can actually lower "
+        "performance.  0 means the tcp btl will not try to set a send buffer "
+        "size.",
+        0, OPAL_INFO_LVL_4, &mca_btl_tcp_component.tcp_sndbuf);
+    mca_btl_tcp_param_register_int(
+        "rcvbuf",
+        "The size of the receive buffer socket option for each connection.  "
+        "Modern TCP stacks generally are smarter than a fixed size and in some "
+        "situations setting a buffer size explicitly can actually lower "
+        "performance.  0 means the tcp btl will not try to set a receive buffer "
+        "size.",
+        0, OPAL_INFO_LVL_4, &mca_btl_tcp_component.tcp_rcvbuf);
+    mca_btl_tcp_param_register_int(
+        "endpoint_cache",
         "The size of the internal cache for each TCP connection. This cache is"
         " used to reduce the number of syscalls, by replacing them with memcpy."
         " Every read will read the expected data plus the amount of the"
-                                    " endpoint_cache", 30*1024, OPAL_INFO_LVL_4, &mca_btl_tcp_component.tcp_endpoint_cache);
-    mca_btl_tcp_param_register_int ("use_nagle", "Whether to use Nagle's algorithm or not (using Nagle's algorithm may increase short message latency)",
-                                    0, OPAL_INFO_LVL_4, &mca_btl_tcp_component.tcp_not_use_nodelay);
-    mca_btl_tcp_param_register_int( "port_min_v4",
-                                    "The minimum port where the TCP BTL will try to bind (default 1024)",
-                                    1024, OPAL_INFO_LVL_2, &mca_btl_tcp_component.tcp_port_min);
-
-    opal_asprintf( &message,
-              "The number of ports where the TCP BTL will try to bind (default %d)."
-              " This parameter together with the port min, define a range of ports"
-              " where Open MPI will open sockets.",
-              (0x1 << 16) - mca_btl_tcp_component.tcp_port_min - 1 );
-    mca_btl_tcp_param_register_int( "port_range_v4", message,
-                                    (0x1 << 16) - mca_btl_tcp_component.tcp_port_min - 1,
-                                    OPAL_INFO_LVL_2, &mca_btl_tcp_component.tcp_port_range);
+        " endpoint_cache",
+        30 * 1024, OPAL_INFO_LVL_4, &mca_btl_tcp_component.tcp_endpoint_cache);
+    mca_btl_tcp_param_register_int("use_nagle",
+                                   "Whether to use Nagle's algorithm or not (using Nagle's "
+                                   "algorithm may increase short message latency)",
+                                   0, OPAL_INFO_LVL_4, &mca_btl_tcp_component.tcp_not_use_nodelay);
+    mca_btl_tcp_param_register_int(
+        "port_min_v4", "The minimum port where the TCP BTL will try to bind (default 1024)", 1024,
+        OPAL_INFO_LVL_2, &mca_btl_tcp_component.tcp_port_min);
+
+    opal_asprintf(&message,
+                  "The number of ports where the TCP BTL will try to bind (default %d)."
+                  " This parameter together with the port min, define a range of ports"
+                  " where Open MPI will open sockets.",
+                  (0x1 << 16) - mca_btl_tcp_component.tcp_port_min - 1);
+    mca_btl_tcp_param_register_int("port_range_v4", message,
+                                   (0x1 << 16) - mca_btl_tcp_component.tcp_port_min - 1,
+                                   OPAL_INFO_LVL_2, &mca_btl_tcp_component.tcp_port_range);
     free(message);
 #if OPAL_ENABLE_IPV6
-    mca_btl_tcp_param_register_int( "port_min_v6",
-                                    "The minimum port where the TCP BTL will try to bind (default 1024)", 1024,
-                                    OPAL_INFO_LVL_2, & mca_btl_tcp_component.tcp6_port_min );
-    opal_asprintf( &message,
-              "The number of ports where the TCP BTL will try to bind (default %d)."
-              " This parameter together with the port min, define a range of ports"
-              " where Open MPI will open sockets.",
-              (0x1 << 16) - mca_btl_tcp_component.tcp6_port_min - 1 );
-    mca_btl_tcp_param_register_int( "port_range_v6", message,
-                                    (0x1 << 16) - mca_btl_tcp_component.tcp6_port_min - 1,
-                                    OPAL_INFO_LVL_2, &mca_btl_tcp_component.tcp6_port_range );
+    mca_btl_tcp_param_register_int(
+        "port_min_v6", "The minimum port where the TCP BTL will try to bind (default 1024)", 1024,
+        OPAL_INFO_LVL_2, &mca_btl_tcp_component.tcp6_port_min);
+    opal_asprintf(&message,
+                  "The number of ports where the TCP BTL will try to bind (default %d)."
+                  " This parameter together with the port min, define a range of ports"
+                  " where Open MPI will open sockets.",
+                  (0x1 << 16) - mca_btl_tcp_component.tcp6_port_min - 1);
+    mca_btl_tcp_param_register_int("port_range_v6", message,
+                                   (0x1 << 16) - mca_btl_tcp_component.tcp6_port_min - 1,
+                                   OPAL_INFO_LVL_2, &mca_btl_tcp_component.tcp6_port_range);
     free(message);
 #endif
 
     /* Check if we should support async progress */
-    mca_btl_tcp_param_register_int ("progress_thread", NULL, 0, OPAL_INFO_LVL_1,
-                                     &mca_btl_tcp_component.tcp_enable_progress_thread);
+    mca_btl_tcp_param_register_int("progress_thread", NULL, 0, OPAL_INFO_LVL_1,
+                                   &mca_btl_tcp_component.tcp_enable_progress_thread);
     mca_btl_tcp_component.report_all_unfound_interfaces = false;
-    (void) mca_base_component_var_register(&mca_btl_tcp_component.super.btl_version,
-                                           "warn_all_unfound_interfaces",
-                                           "Issue a warning for all unfound interfaces included in if_exclude",
-                                           MCA_BASE_VAR_TYPE_BOOL,
-                                           NULL, 0, 0, OPAL_INFO_LVL_2,
-                                           MCA_BASE_VAR_SCOPE_READONLY, &mca_btl_tcp_component.report_all_unfound_interfaces);
-
-    mca_btl_tcp_module.super.btl_exclusivity =  MCA_BTL_EXCLUSIVITY_LOW + 100;
-    mca_btl_tcp_module.super.btl_eager_limit = 64*1024;
-    mca_btl_tcp_module.super.btl_rndv_eager_limit = 64*1024;
-    mca_btl_tcp_module.super.btl_max_send_size = 128*1024;
-    mca_btl_tcp_module.super.btl_rdma_pipeline_send_length = 128*1024;
+    (void) mca_base_component_var_register(
+        &mca_btl_tcp_component.super.btl_version, "warn_all_unfound_interfaces",
+        "Issue a warning for all unfound interfaces included in if_exclude", MCA_BASE_VAR_TYPE_BOOL,
+        NULL, 0, 0, OPAL_INFO_LVL_2, MCA_BASE_VAR_SCOPE_READONLY,
+        &mca_btl_tcp_component.report_all_unfound_interfaces);
+
+    mca_btl_tcp_module.super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_LOW + 100;
+    mca_btl_tcp_module.super.btl_eager_limit = 64 * 1024;
+    mca_btl_tcp_module.super.btl_rndv_eager_limit = 64 * 1024;
+    mca_btl_tcp_module.super.btl_max_send_size = 128 * 1024;
+    mca_btl_tcp_module.super.btl_rdma_pipeline_send_length = 128 * 1024;
     /* Some OSes have hard coded limits on how many bytes can be manipulated
      * by each writev operation.  Force a reasonable limit, to prevent overflowing
      * a signed 32-bit integer (limit comes from BSD and OS X). We remove 1k to
      * make some room for our internal headers.
      */
-    mca_btl_tcp_module.super.btl_rdma_pipeline_frag_size = ((1UL<<31) - 1024);
+    mca_btl_tcp_module.super.btl_rdma_pipeline_frag_size = ((1UL << 31) - 1024);
     mca_btl_tcp_module.super.btl_min_rdma_pipeline_size = 0;
-    mca_btl_tcp_module.super.btl_flags = MCA_BTL_FLAGS_PUT |
-                                       MCA_BTL_FLAGS_SEND_INPLACE |
-                                       MCA_BTL_FLAGS_NEED_CSUM |
-                                       MCA_BTL_FLAGS_NEED_ACK |
-                                       MCA_BTL_FLAGS_HETEROGENEOUS_RDMA |
-                                       MCA_BTL_FLAGS_SEND;
+    mca_btl_tcp_module.super.btl_flags = MCA_BTL_FLAGS_PUT | MCA_BTL_FLAGS_SEND_INPLACE
+                                         | MCA_BTL_FLAGS_NEED_CSUM | MCA_BTL_FLAGS_NEED_ACK
+                                         | MCA_BTL_FLAGS_HETEROGENEOUS_RDMA | MCA_BTL_FLAGS_SEND;
 
     /* Bandwidth and latency initially set to 0. May be overridden during
      * mca_btl_tcp_create().
@@ -345,12 +337,13 @@ static int mca_btl_tcp_component_register(void)
 
     mca_btl_base_param_register(&mca_btl_tcp_component.super.btl_version,
                                 &mca_btl_tcp_module.super);
-    if (mca_btl_tcp_module.super.btl_rdma_pipeline_frag_size > ((1UL<<31) - 1024) ) {
+    if (mca_btl_tcp_module.super.btl_rdma_pipeline_frag_size > ((1UL << 31) - 1024)) {
         /* Assume a hard limit. A test in configure would be a better solution, but until then
          * kicking-in the pipeline RDMA for extremely large data is good enough. */
-        mca_btl_tcp_module.super.btl_rdma_pipeline_frag_size = ((1UL<<31) - 1024);
+        mca_btl_tcp_module.super.btl_rdma_pipeline_frag_size = ((1UL << 31) - 1024);
     }
-    mca_btl_tcp_param_register_int ("disable_family", NULL, 0, OPAL_INFO_LVL_2,  &mca_btl_tcp_component.tcp_disable_family);
+    mca_btl_tcp_param_register_int("disable_family", NULL, 0, OPAL_INFO_LVL_2,
+                                   &mca_btl_tcp_component.tcp_disable_family);
 
     return mca_btl_tcp_component_verify();
 }
@@ -387,14 +380,14 @@ static int mca_btl_tcp_component_open(void)
     OBJ_CONSTRUCT(&mca_btl_tcp_ready_frag_pending_queue, opal_list_t);
 
     /* if_include and if_exclude need to be mutually exclusive */
-    if (OPAL_SUCCESS !=
-        mca_base_var_check_exclusive("opal",
-        mca_btl_tcp_component.super.btl_version.mca_type_name,
-        mca_btl_tcp_component.super.btl_version.mca_component_name,
-        "if_include",
-        mca_btl_tcp_component.super.btl_version.mca_type_name,
-        mca_btl_tcp_component.super.btl_version.mca_component_name,
-        "if_exclude")) {
+    if (OPAL_SUCCESS
+        != mca_base_var_check_exclusive("opal",
+                                        mca_btl_tcp_component.super.btl_version.mca_type_name,
+                                        mca_btl_tcp_component.super.btl_version.mca_component_name,
+                                        "if_include",
+                                        mca_btl_tcp_component.super.btl_version.mca_type_name,
+                                        mca_btl_tcp_component.super.btl_version.mca_component_name,
+                                        "if_exclude")) {
         /* Return ERR_NOT_AVAILABLE so that a warning message about
            "open" failing is not printed */
         return OPAL_ERR_NOT_AVAILABLE;
@@ -403,7 +396,6 @@ static int mca_btl_tcp_component_open(void)
     return OPAL_SUCCESS;
 }
 
-
 /*
  * module cleanup - sanity checking of queue lengths
  */
@@ -416,28 +408,27 @@ static int mca_btl_tcp_component_close(void)
      * If we have a progress thread we should shut it down before
      * moving forward with the TCP tearing down process.
      */
-    if( (NULL != mca_btl_tcp_event_base) &&
-        (mca_btl_tcp_event_base != opal_sync_event_base) ) {
+    if ((NULL != mca_btl_tcp_event_base) && (mca_btl_tcp_event_base != opal_sync_event_base)) {
         /* Turn of the progress thread before moving forward */
-        if( -1 != mca_btl_tcp_progress_thread_trigger ) {
-            void* ret = NULL;  /* not currently used */
+        if (-1 != mca_btl_tcp_progress_thread_trigger) {
+            void *ret = NULL; /* not currently used */
 
             mca_btl_tcp_progress_thread_trigger = 0;
             /* Let the progress thread know that we're going away */
-            if( -1 != mca_btl_tcp_pipe_to_progress[1] ) {
+            if (-1 != mca_btl_tcp_pipe_to_progress[1]) {
                 close(mca_btl_tcp_pipe_to_progress[1]);
                 mca_btl_tcp_pipe_to_progress[1] = -1;
             }
             /* wait until the TCP progress thread completes */
             opal_thread_join(&mca_btl_tcp_progress_thread, &ret);
-            assert( -1 == mca_btl_tcp_progress_thread_trigger );
+            assert(-1 == mca_btl_tcp_progress_thread_trigger);
         }
         opal_event_del(&mca_btl_tcp_component.tcp_recv_thread_async_event);
         opal_event_base_free(mca_btl_tcp_event_base);
         mca_btl_tcp_event_base = NULL;
 
         /* Close the remaining pipes */
-        if( -1 != mca_btl_tcp_pipe_to_progress[0] ) {
+        if (-1 != mca_btl_tcp_pipe_to_progress[0]) {
             close(mca_btl_tcp_pipe_to_progress[0]);
             mca_btl_tcp_pipe_to_progress[0] = -1;
         }
@@ -468,12 +459,13 @@ static int mca_btl_tcp_component_close(void)
 
     /* remove all pending events. Do not lock the tcp_events list as
        the event themselves will unregister during the destructor. */
-    OPAL_LIST_FOREACH_SAFE(event, next, &mca_btl_tcp_component.tcp_events, mca_btl_tcp_event_t) {
+    OPAL_LIST_FOREACH_SAFE (event, next, &mca_btl_tcp_component.tcp_events, mca_btl_tcp_event_t) {
         opal_event_del(&event->event);
         OBJ_RELEASE(event);
     }
 
-    opal_proc_table_remove_value(&mca_btl_tcp_component.tcp_procs, opal_proc_local_get()->proc_name);
+    opal_proc_table_remove_value(&mca_btl_tcp_component.tcp_procs,
+                                 opal_proc_local_get()->proc_name);
 
     /* release resources */
     OBJ_DESTRUCT(&mca_btl_tcp_component.tcp_procs);
@@ -481,23 +473,18 @@ static int mca_btl_tcp_component_close(void)
     OBJ_DESTRUCT(&mca_btl_tcp_component.tcp_frag_max);
     OBJ_DESTRUCT(&mca_btl_tcp_component.tcp_frag_user);
     OBJ_DESTRUCT(&mca_btl_tcp_component.tcp_lock);
-    OBJ_DESTRUCT(&mca_btl_tcp_component.local_ifs);
-
-#if OPAL_CUDA_SUPPORT
-    mca_common_cuda_fini();
-#endif /* OPAL_CUDA_SUPPORT */
+    OPAL_LIST_DESTRUCT(&mca_btl_tcp_component.local_ifs);
 
     return OPAL_SUCCESS;
 }
 
-
 /*
  *  Create a btl instance and add to modules list.
  */
 
-static int mca_btl_tcp_create(const int if_kindex, const char* if_name)
+static int mca_btl_tcp_create(const int if_kindex, const char *if_name)
 {
-    struct mca_btl_tcp_module_t* btl;
+    struct mca_btl_tcp_module_t *btl;
     opal_if_t *copied_interface, *selected_interface;
     char param[256];
     int i, if_index;
@@ -515,28 +502,26 @@ static int mca_btl_tcp_create(const int if_kindex, const char* if_name)
      * split_and_resolve and pass the address used to select the
      * device into mca_btl_tcp_create().  This is a cleanup of the
      * logic that's been in use for years, but the case it doesn't
-     * cover is (say) only specifing mca_btl_if_include 10.0.0.0/16
+     * cover is (say) only specifying mca_btl_if_include 10.0.0.0/16
      * when the interface has addresses of both 10.0.0.1 and 10.1.0.1;
      * there's absolutely nothing that keeps this code from picking
      * 10.1.0.1 as the one that is published in the modex and used for
      * connection.
      */
-    OPAL_LIST_FOREACH(selected_interface, &opal_if_list, opal_if_t) {
+    OPAL_LIST_FOREACH (selected_interface, &opal_if_list, opal_if_t) {
         if (if_kindex != selected_interface->if_kernel_index) {
             continue;
         }
 
         if_index = selected_interface->if_index;
 
-        memcpy((struct sockaddr*)&addr, &selected_interface->if_addr,
+        memcpy((struct sockaddr *) &addr, &selected_interface->if_addr,
                MIN(sizeof(struct sockaddr_storage), sizeof(selected_interface->if_addr)));
 
-        if (addr.ss_family == AF_INET &&
-            4 != mca_btl_tcp_component.tcp_disable_family) {
+        if (addr.ss_family == AF_INET && 4 != mca_btl_tcp_component.tcp_disable_family) {
             found = true;
             break;
-        } else if (addr.ss_family == AF_INET6 &&
-                   6 != mca_btl_tcp_component.tcp_disable_family) {
+        } else if (addr.ss_family == AF_INET6 && 6 != mca_btl_tcp_component.tcp_disable_family) {
             found = true;
             break;
         }
@@ -547,10 +532,11 @@ static int mca_btl_tcp_create(const int if_kindex, const char* if_name)
         return OPAL_SUCCESS;
     }
 
-    for( i = 0; i < (int)mca_btl_tcp_component.tcp_num_links; i++ ) {
-        btl = (struct mca_btl_tcp_module_t *)malloc(sizeof(mca_btl_tcp_module_t));
-        if(NULL == btl)
+    for (i = 0; i < (int) mca_btl_tcp_component.tcp_num_links; i++) {
+        btl = (struct mca_btl_tcp_module_t *) malloc(sizeof(mca_btl_tcp_module_t));
+        if (NULL == btl) {
             return OPAL_ERR_OUT_OF_RESOURCE;
+        }
         copied_interface = OBJ_NEW(opal_if_t);
         if (NULL == copied_interface) {
             free(btl);
@@ -576,23 +562,27 @@ static int mca_btl_tcp_create(const int if_kindex, const char* if_name)
 
         /* allow user to specify interface bandwidth */
         sprintf(param, "bandwidth_%s", if_name);
-        mca_btl_tcp_param_register_uint(param, NULL, btl->super.btl_bandwidth, OPAL_INFO_LVL_5, &btl->super.btl_bandwidth);
+        mca_btl_tcp_param_register_uint(param, NULL, btl->super.btl_bandwidth, OPAL_INFO_LVL_5,
+                                        &btl->super.btl_bandwidth);
 
         /* allow user to override/specify latency ranking */
         sprintf(param, "latency_%s", if_name);
-        mca_btl_tcp_param_register_uint(param, NULL, btl->super.btl_latency, OPAL_INFO_LVL_5,  &btl->super.btl_latency);
-        if( i > 0 ) {
+        mca_btl_tcp_param_register_uint(param, NULL, btl->super.btl_latency, OPAL_INFO_LVL_5,
+                                        &btl->super.btl_latency);
+        if (i > 0) {
             btl->super.btl_bandwidth >>= 1;
-            btl->super.btl_latency   <<= 1;
+            btl->super.btl_latency <<= 1;
         }
 
         /* allow user to specify interface bandwidth */
         sprintf(param, "bandwidth_%s:%d", if_name, i);
-        mca_btl_tcp_param_register_uint(param, NULL, btl->super.btl_bandwidth, OPAL_INFO_LVL_5, &btl->super.btl_bandwidth);
+        mca_btl_tcp_param_register_uint(param, NULL, btl->super.btl_bandwidth, OPAL_INFO_LVL_5,
+                                        &btl->super.btl_bandwidth);
 
         /* allow user to override/specify latency ranking */
         sprintf(param, "latency_%s:%d", if_name, i);
-        mca_btl_tcp_param_register_uint(param, NULL, btl->super.btl_latency, OPAL_INFO_LVL_5, &btl->super.btl_latency);
+        mca_btl_tcp_param_register_uint(param, NULL, btl->super.btl_latency, OPAL_INFO_LVL_5,
+                                        &btl->super.btl_latency);
 
         /* Only attempt to auto-detect bandwidth and/or latency if it is 0.
          *
@@ -624,15 +614,16 @@ static int mca_btl_tcp_create(const int if_kindex, const char* if_name)
         memcpy(&copied_interface->if_addr, &btl->tcp_ifaddr, sizeof(struct sockaddr_storage));
         copied_interface->if_mask = selected_interface->if_mask;
         copied_interface->if_bandwidth = btl->super.btl_bandwidth;
-        memcpy(&copied_interface->if_mac, &selected_interface->if_mac, sizeof(copied_interface->if_mac));
+        memcpy(&copied_interface->if_mac, &selected_interface->if_mac,
+               sizeof(copied_interface->if_mac));
         copied_interface->ifmtu = selected_interface->ifmtu;
 
         opal_list_append(&mca_btl_tcp_component.local_ifs, &(copied_interface->super));
 
         opal_output_verbose(5, opal_btl_base_framework.framework_output,
                             "btl:tcp: %p: if %s kidx %d cnt %i addr %s %s bw %d lt %d\n",
-                            (void*)btl, if_name, (int) btl->tcp_ifkindex, i,
-                            opal_net_get_hostname((struct sockaddr*)&btl->tcp_ifaddr),
+                            (void *) btl, if_name, (int) btl->tcp_ifkindex, i,
+                            opal_net_get_hostname((struct sockaddr *) &btl->tcp_ifaddr),
                             (btl->tcp_ifaddr.ss_family == AF_INET) ? "IPv4" : "IPv6",
                             btl->super.btl_bandwidth, btl->super.btl_latency);
     }
@@ -646,8 +637,8 @@ static int mca_btl_tcp_create(const int if_kindex, const char* if_name)
  */
 static char **split_and_resolve(char **orig_str, char *name, bool reqd)
 {
-    int i, ret, save, if_index;
-    char **argv, *str, *tmp;
+    int i, n, ret, if_index, match_count, interface_count;
+    char **argv, **interfaces, *str, *tmp;
     char if_name[OPAL_IF_NAMESIZE];
     struct sockaddr_storage argv_inaddr, if_inaddr;
     uint32_t argv_prefix;
@@ -661,9 +652,23 @@ static char **split_and_resolve(char **orig_str, char *name, bool reqd)
     if (NULL == argv) {
         return NULL;
     }
-    for (save = i = 0; NULL != argv[i]; ++i) {
+    interface_count = 0;
+    interfaces = NULL;
+    for (i = 0; NULL != argv[i]; ++i) {
         if (isalpha(argv[i][0])) {
-            argv[save++] = argv[i];
+            /* This is an interface name. If not already in the interfaces array, add it */
+            for (n = 0; n < interface_count; n++) {
+                if (0 == strcmp(argv[i], interfaces[n])) {
+                    break;
+                }
+            }
+            if (n == interface_count) {
+                opal_output_verbose(20,
+                                    opal_btl_base_framework.framework_output,
+                                    "btl: tcp: Using interface: %s ", argv[i]);
+                opal_argv_append(&interface_count, &interfaces, argv[i]);
+            }
+            free(argv[i]);
             continue;
         }
 
@@ -673,9 +678,9 @@ static char **split_and_resolve(char **orig_str, char *name, bool reqd)
         tmp = strdup(argv[i]);
         str = strchr(argv[i], '/');
         if (NULL == str) {
-            opal_show_help("help-mpi-btl-tcp.txt", "invalid if_inexclude",
-                           true, name, opal_process_info.nodename,
-                           tmp, "Invalid specification (missing \"/\")");
+            opal_show_help("help-mpi-btl-tcp.txt", "invalid if_inexclude", true, name,
+                           opal_process_info.nodename, tmp,
+                           "Invalid specification (missing \"/\")");
             free(argv[i]);
             free(tmp);
             continue;
@@ -684,25 +689,23 @@ static char **split_and_resolve(char **orig_str, char *name, bool reqd)
         argv_prefix = atoi(str + 1);
 
         /* Now convert the IPv4 address */
-        ((struct sockaddr*) &argv_inaddr)->sa_family = AF_INET;
-        ret = inet_pton(AF_INET, argv[i],
-                        &((struct sockaddr_in*) &argv_inaddr)->sin_addr);
+        ((struct sockaddr *) &argv_inaddr)->sa_family = AF_INET;
+        ret = inet_pton(AF_INET, argv[i], &((struct sockaddr_in *) &argv_inaddr)->sin_addr);
         free(argv[i]);
 
         if (1 != ret) {
-            opal_show_help("help-mpi-btl-tcp.txt", "invalid if_inexclude",
-                           true, name, opal_process_info.nodename, tmp,
+            opal_show_help("help-mpi-btl-tcp.txt", "invalid if_inexclude", true, name,
+                           opal_process_info.nodename, tmp,
                            "Invalid specification (inet_pton() failed)");
             free(tmp);
             continue;
         }
         opal_output_verbose(20, opal_btl_base_framework.framework_output,
-                            "btl: tcp: Searching for %s address+prefix: %s / %u",
-                            name,
-                            opal_net_get_hostname((struct sockaddr*) &argv_inaddr),
-                            argv_prefix);
+                            "btl: tcp: Searching for %s address+prefix: %s / %u", name,
+                            opal_net_get_hostname((struct sockaddr *) &argv_inaddr), argv_prefix);
 
         /* Go through all interfaces and see if we can find a match */
+        match_count = 0;
         for (if_index = opal_ifbegin(); if_index >= 0;
              if_index = opal_ifnext(if_index)) {
             opal_ifindextoaddr(if_index,
@@ -711,41 +714,50 @@ static char **split_and_resolve(char **orig_str, char *name, bool reqd)
             if (opal_net_samenetwork((struct sockaddr*) &argv_inaddr,
                                      (struct sockaddr*) &if_inaddr,
                                      argv_prefix)) {
-                break;
+                /* We found a match. If it's not already in the interfaces array,
+                   add it. If it's already in the array, treat it as a match */
+                match_count = match_count + 1;
+                opal_ifindextoname(if_index, if_name, sizeof(if_name));
+                for (n = 0; n < interface_count; n++) {
+                    if (0 == strcmp(if_name, interfaces[n])) {
+                        break;
+                    }
+                }
+                if (n == interface_count) {
+                    opal_output_verbose(20,
+                                        opal_btl_base_framework.framework_output,
+                                        "btl: tcp: Found match: %s (%s)",
+                                        opal_net_get_hostname((struct sockaddr*) &if_inaddr),
+                                        if_name);
+                    opal_argv_append(&interface_count, &interfaces, if_name);
+                }
             }
         }
 
         /* If we didn't find a match, keep trying */
-        if (if_index < 0) {
+        if (0 == match_count) {
             if (reqd || mca_btl_tcp_component.report_all_unfound_interfaces) {
-                opal_show_help("help-mpi-btl-tcp.txt", "invalid if_inexclude",
-                               true, name, opal_process_info.nodename, tmp,
+                opal_show_help("help-mpi-btl-tcp.txt", "invalid if_inexclude", true, name,
+                               opal_process_info.nodename, tmp,
                                "Did not find interface matching this subnet");
             }
             free(tmp);
             continue;
         }
 
-        /* We found a match; get the name and replace it in the
-           argv */
-        opal_ifindextoname(if_index, if_name, sizeof(if_name));
-        opal_output_verbose(20, opal_btl_base_framework.framework_output,
-                            "btl: tcp: Found match: %s (%s)",
-                            opal_net_get_hostname((struct sockaddr*) &if_inaddr),
-                            if_name);
-        argv[save++] = strdup(if_name);
         free(tmp);
     }
 
-    /* The list may have been compressed if there were invalid
-       entries, so ensure we end it with a NULL entry */
-    argv[save] = NULL;
+    /* Mark the end of the interface name array with NULL */
+    if (NULL != interfaces) {
+        interfaces[interface_count] = NULL;
+    }
+    free(argv);
     free(*orig_str);
-    *orig_str = opal_argv_join(argv, ',');
-    return argv;
+    *orig_str = opal_argv_join(interfaces, ',');
+    return interfaces;
 }
 
-
 /*
  * Create a TCP BTL instance for either:
  * (1) all interfaces specified by the user
@@ -764,7 +776,7 @@ static int mca_btl_tcp_component_create_instances(void)
     char **argv;
     int ret = OPAL_SUCCESS;
 
-    if(if_count <= 0) {
+    if (if_count <= 0) {
         return OPAL_ERROR;
     }
 
@@ -779,13 +791,13 @@ static int mca_btl_tcp_component_create_instances(void)
 
         /* initialize array to 0. Assumption: 0 isn't a valid kernel
            index */
-        memset (kindexes, 0, sizeof(int) * if_count);
+        memset(kindexes, 0, sizeof(int) * if_count);
 
         /* assign the corresponding kernel indexes for all opal_list
          * indexes (loop over all addresses)
          */
-        for(if_index = opal_ifbegin(); if_index >= 0; if_index = opal_ifnext(if_index)){
-            int index = opal_ifindextokindex (if_index);
+        for (if_index = opal_ifbegin(); if_index >= 0; if_index = opal_ifnext(if_index)) {
+            int index = opal_ifindextokindex(if_index);
 
             if (index > 0) {
                 bool want_this_if = true;
@@ -806,9 +818,9 @@ static int mca_btl_tcp_component_create_instances(void)
     }
 
     /* allocate memory for btls */
-    mca_btl_tcp_component.tcp_btls = (mca_btl_tcp_module_t**)malloc(mca_btl_tcp_component.tcp_num_links *
-                                                                    kif_count * sizeof(mca_btl_tcp_module_t*));
-    if(NULL == mca_btl_tcp_component.tcp_btls) {
+    mca_btl_tcp_component.tcp_btls = (mca_btl_tcp_module_t **) malloc(
+        mca_btl_tcp_component.tcp_num_links * kif_count * sizeof(mca_btl_tcp_module_t *));
+    if (NULL == mca_btl_tcp_component.tcp_btls) {
         ret = OPAL_ERR_OUT_OF_RESOURCE;
         goto cleanup;
     }
@@ -816,19 +828,17 @@ static int mca_btl_tcp_component_create_instances(void)
     mca_btl_tcp_component.tcp_addr_count = if_count;
 
     /* if the user specified an interface list - use these exclusively */
-    argv = include = split_and_resolve(&mca_btl_tcp_component.tcp_if_include,
-                                       "include", true);
-    while(argv && *argv) {
-        char* if_name = *argv;
-        int if_index = opal_ifnametokindex(if_name);
-        if(if_index < 0) {
-            opal_show_help("help-mpi-btl-tcp.txt", "invalid if_inexclude",
-                           true, "include", opal_process_info.nodename,
-                           if_name, "Unknown interface name");
+    argv = include = split_and_resolve(&mca_btl_tcp_component.tcp_if_include, "include", true);
+    while (argv && *argv) {
+        char *if_name = *argv;
+        int idx = opal_ifnametokindex(if_name);
+        if (idx < 0) {
+            opal_show_help("help-mpi-btl-tcp.txt", "invalid if_inexclude", true, "include",
+                           opal_process_info.nodename, if_name, "Unknown interface name");
             ret = OPAL_ERR_NOT_FOUND;
             goto cleanup;
         }
-        mca_btl_tcp_create(if_index, if_name);
+        mca_btl_tcp_create(idx, if_name);
         argv++;
     }
 
@@ -841,12 +851,11 @@ static int mca_btl_tcp_component_create_instances(void)
 
     /* if the interface list was not specified by the user, create
      * a BTL for each interface that was not excluded.
-    */
-    exclude = split_and_resolve(&mca_btl_tcp_component.tcp_if_exclude,
-                                "exclude", false);
+     */
+    exclude = split_and_resolve(&mca_btl_tcp_component.tcp_if_exclude, "exclude", false);
     {
         int i;
-        for(i = 0; i < kif_count; i++) {
+        for (i = 0; i < kif_count; i++) {
             /* OPAL_IF_NAMESIZE is defined in opal/util/if.h */
             char if_name[OPAL_IF_NAMESIZE];
             if_index = kindexes[i];
@@ -855,19 +864,20 @@ static int mca_btl_tcp_component_create_instances(void)
 
             /* check to see if this interface exists in the exclude list */
             argv = exclude;
-            while(argv && *argv) {
-                if(strncmp(*argv,if_name,strlen(*argv)) == 0)
+            while (argv && *argv) {
+                if (strncmp(*argv, if_name, strlen(*argv)) == 0) {
                     break;
+                }
                 argv++;
             }
             /* if this interface was not found in the excluded list, create a BTL */
-            if(argv == 0 || *argv == 0) {
+            if (argv == 0 || *argv == 0) {
                 mca_btl_tcp_create(if_index, if_name);
             }
         }
     }
 
- cleanup:
+cleanup:
     if (NULL != include) {
         opal_argv_free(include);
     }
@@ -880,28 +890,28 @@ static int mca_btl_tcp_component_create_instances(void)
     return ret;
 }
 
-static void* mca_btl_tcp_progress_thread_engine(opal_object_t *obj)
+static void *mca_btl_tcp_progress_thread_engine(opal_object_t *obj)
 {
-    opal_thread_t* current_thread = (opal_thread_t*)obj;
+    opal_thread_t *current_thread = (opal_thread_t *) obj;
 
-    while( 1 == (*((int*)current_thread->t_arg)) ) {
+    while (1 == (*((int *) current_thread->t_arg))) {
         opal_event_loop(mca_btl_tcp_event_base, OPAL_EVLOOP_ONCE);
     }
-    (*((int*)current_thread->t_arg)) = -1;
+    (*((int *) current_thread->t_arg)) = -1;
     return NULL;
 }
 
 static void mca_btl_tcp_component_event_async_handler(int fd, short unused, void *context)
 {
-    opal_event_t* event;
+    opal_event_t *event;
     int rc;
 
-    rc = read(fd, (void*)&event, sizeof(opal_event_t*));
-    assert( fd == mca_btl_tcp_pipe_to_progress[0] );
-    if( 0 == rc ) {
+    rc = read(fd, (void *) &event, sizeof(opal_event_t *));
+    assert(fd == mca_btl_tcp_pipe_to_progress[0]);
+    if (0 == rc) {
         /* The main thread closed the pipe to trigger the shutdown procedure */
-        opal_thread_t* current_thread = (opal_thread_t*)context;
-        (*((int*)current_thread->t_arg)) = 0;
+        opal_thread_t *current_thread = (opal_thread_t *) context;
+        (*((int *) current_thread->t_arg)) = 0;
     } else {
         opal_event_add(event, 0);
     }
@@ -919,10 +929,9 @@ static int mca_btl_tcp_component_create_listen(uint16_t af_family)
 
     /* create a listen socket for incoming connections */
     sd = socket(af_family, SOCK_STREAM, 0);
-    if(sd < 0) {
+    if (sd < 0) {
         if (EAFNOSUPPORT != opal_socket_errno) {
-            BTL_ERROR(("socket() failed: %s (%d)",
-                       strerror(opal_socket_errno), opal_socket_errno));
+            BTL_ERROR(("socket() failed: %s (%d)", strerror(opal_socket_errno), opal_socket_errno));
         }
         return OPAL_ERR_IN_ERRNO;
     }
@@ -933,24 +942,22 @@ static int mca_btl_tcp_component_create_listen(uint16_t af_family)
     {
         struct addrinfo hints, *res = NULL;
 
-        memset (&hints, 0, sizeof(hints));
+        memset(&hints, 0, sizeof(hints));
         hints.ai_family = af_family;
         hints.ai_socktype = SOCK_STREAM;
         hints.ai_flags = AI_PASSIVE;
 
         if ((rc = getaddrinfo(NULL, "0", &hints, &res))) {
-            opal_output (0,
-                         "mca_btl_tcp_create_listen: unable to resolve. %s\n",
-                         gai_strerror (rc));
+            opal_output(0, "mca_btl_tcp_create_listen: unable to resolve. %s\n", gai_strerror(rc));
             CLOSE_THE_SOCKET(sd);
             return OPAL_ERROR;
         }
 
-        memcpy (&inaddr, res->ai_addr, res->ai_addrlen);
+        memcpy(&inaddr, res->ai_addr, res->ai_addrlen);
         addrlen = res->ai_addrlen;
-        freeaddrinfo (res);
+        freeaddrinfo(res);
 
-#ifdef IPV6_V6ONLY
+#    ifdef IPV6_V6ONLY
         /* If this OS supports the "IPV6_V6ONLY" constant, then set it
            on this socket.  It specifies that *only* V6 connections
            should be accepted on this socket (vs. allowing incoming
@@ -960,22 +967,21 @@ static int mca_btl_tcp_component_create_listen(uint16_t af_family)
            more details. */
         if (AF_INET6 == af_family) {
             int flg = 1;
-            if (setsockopt (sd, IPPROTO_IPV6, IPV6_V6ONLY,
-                            (char *) &flg, sizeof (flg)) < 0) {
+            if (setsockopt(sd, IPPROTO_IPV6, IPV6_V6ONLY, (char *) &flg, sizeof(flg)) < 0) {
                 BTL_ERROR(("mca_btl_tcp_create_listen: unable to set IPV6_V6ONLY\n"));
             }
         }
-#endif /* IPV6_V6ONLY */
+#    endif /* IPV6_V6ONLY */
     }
 #else
-    ((struct sockaddr_in*) &inaddr)->sin_family = AF_INET;
-    ((struct sockaddr_in*) &inaddr)->sin_addr.s_addr = INADDR_ANY;
+    ((struct sockaddr_in *) &inaddr)->sin_family = AF_INET;
+    ((struct sockaddr_in *) &inaddr)->sin_addr.s_addr = INADDR_ANY;
     addrlen = sizeof(struct sockaddr_in);
 #endif
 
-    {  /* Don't reuse ports */
+    { /* Don't reuse ports */
         int flg = 0;
-        if (setsockopt (sd, SOL_SOCKET, SO_REUSEADDR, (const char *)&flg, sizeof (flg)) < 0) {
+        if (setsockopt(sd, SOL_SOCKET, SO_REUSEADDR, (const char *) &flg, sizeof(flg)) < 0) {
             BTL_ERROR(("mca_btl_tcp_create_listen: unable to unset the "
                        "SO_REUSEADDR option (%s:%d)\n",
                        strerror(opal_socket_errno), opal_socket_errno));
@@ -992,35 +998,33 @@ static int mca_btl_tcp_component_create_listen(uint16_t af_family)
             range = mca_btl_tcp_component.tcp6_port_range;
             port = mca_btl_tcp_component.tcp6_port_min;
         } else
-#endif  /* OPAL_ENABLE_IPV6 */
+#endif /* OPAL_ENABLE_IPV6 */
         {
             range = mca_btl_tcp_component.tcp_port_range;
             port = mca_btl_tcp_component.tcp_port_min;
         }
 
-        for( index = 0;  index < range; index++ ) {
+        for (index = 0; index < range; index++) {
 #if OPAL_ENABLE_IPV6
-            ((struct sockaddr_in6*) &inaddr)->sin6_port = htons(port + index);
+            ((struct sockaddr_in6 *) &inaddr)->sin6_port = htons(port + index);
 #else
-            ((struct sockaddr_in*) &inaddr)->sin_port = htons(port + index);
-#endif  /* OPAL_ENABLE_IPV6 */
+            ((struct sockaddr_in *) &inaddr)->sin_port = htons(port + index);
+#endif /* OPAL_ENABLE_IPV6 */
             opal_output_verbose(30, opal_btl_base_framework.framework_output,
                                 "btl:tcp: Attempting to bind to %s port %d",
-                                (AF_INET == af_family) ? "AF_INET" : "AF_INET6",
-                                port + index);
-            if(bind(sd, (struct sockaddr*)&inaddr, addrlen) < 0) {
-                if( (EADDRINUSE == opal_socket_errno) || (EADDRNOTAVAIL == opal_socket_errno) ) {
+                                (AF_INET == af_family) ? "AF_INET" : "AF_INET6", port + index);
+            if (bind(sd, (struct sockaddr *) &inaddr, addrlen) < 0) {
+                if ((EADDRINUSE == opal_socket_errno) || (EADDRNOTAVAIL == opal_socket_errno)) {
                     continue;
                 }
-                BTL_ERROR(("bind() failed: %s (%d)",
-                          strerror(opal_socket_errno), opal_socket_errno));
+                BTL_ERROR(
+                    ("bind() failed: %s (%d)", strerror(opal_socket_errno), opal_socket_errno));
                 CLOSE_THE_SOCKET(sd);
                 return OPAL_ERROR;
             }
             opal_output_verbose(30, opal_btl_base_framework.framework_output,
                                 "btl:tcp: Successfully bound to %s port %d",
-                                (AF_INET == af_family) ? "AF_INET" : "AF_INET6",
-                                port + index);
+                                (AF_INET == af_family) ? "AF_INET" : "AF_INET6", port + index);
             goto socket_binded;
         }
 #if OPAL_ENABLE_IPV6
@@ -1029,7 +1033,7 @@ static int mca_btl_tcp_component_create_listen(uint16_t af_family)
                        mca_btl_tcp_component.tcp6_port_min,
                        mca_btl_tcp_component.tcp6_port_min + range));
         } else
-#endif  /* OPAL_ENABLE_IPV6 */
+#endif /* OPAL_ENABLE_IPV6 */
         {
             BTL_ERROR(("bind() failed: no port available in the range [%d..%d]",
                        mca_btl_tcp_component.tcp_port_min,
@@ -1038,18 +1042,18 @@ static int mca_btl_tcp_component_create_listen(uint16_t af_family)
         CLOSE_THE_SOCKET(sd);
         return OPAL_ERROR;
     }
-  socket_binded:
-    /* resolve system assignend port */
-    if(getsockname(sd, (struct sockaddr*)&inaddr, &addrlen) < 0) {
-        BTL_ERROR(("getsockname() failed: %s (%d)",
-                   strerror(opal_socket_errno), opal_socket_errno));
+socket_binded:
+    /* resolve system assigned port */
+    if (getsockname(sd, (struct sockaddr *) &inaddr, &addrlen) < 0) {
+        BTL_ERROR(
+            ("getsockname() failed: %s (%d)", strerror(opal_socket_errno), opal_socket_errno));
         CLOSE_THE_SOCKET(sd);
         return OPAL_ERROR;
     }
 
 #if OPAL_ENABLE_IPV6
     if (AF_INET6 == af_family) {
-        mca_btl_tcp_component.tcp6_listen_port = ((struct sockaddr_in6*) &inaddr)->sin6_port;
+        mca_btl_tcp_component.tcp6_listen_port = ((struct sockaddr_in6 *) &inaddr)->sin6_port;
         mca_btl_tcp_component.tcp6_listen_sd = sd;
         opal_output_verbose(30, opal_btl_base_framework.framework_output,
                             "btl:tcp: my listening v6 socket port is %d",
@@ -1058,50 +1062,47 @@ static int mca_btl_tcp_component_create_listen(uint16_t af_family)
 #endif
     {
         char str[16];
-        mca_btl_tcp_component.tcp_listen_port = ((struct sockaddr_in*) &inaddr)->sin_port;
+        mca_btl_tcp_component.tcp_listen_port = ((struct sockaddr_in *) &inaddr)->sin_port;
         mca_btl_tcp_component.tcp_listen_sd = sd;
-        inet_ntop(AF_INET, &(((struct sockaddr_in*)&inaddr)->sin_addr), str, sizeof(str));
+        inet_ntop(AF_INET, &(((struct sockaddr_in *) &inaddr)->sin_addr), str, sizeof(str));
         opal_output_verbose(30, opal_btl_base_framework.framework_output,
-                            "btl:tcp: my listening v4 socket is %s:%u",
-                            str, ntohs(mca_btl_tcp_component.tcp_listen_port));
+                            "btl:tcp: my listening v4 socket is %s:%u", str,
+                            ntohs(mca_btl_tcp_component.tcp_listen_port));
     }
 
     /* setup listen backlog to maximum allowed by kernel */
-    if(listen(sd, SOMAXCONN) < 0) {
-        BTL_ERROR(("listen() failed: %s (%d)",
-                   strerror(opal_socket_errno), opal_socket_errno));
+    if (listen(sd, SOMAXCONN) < 0) {
+        BTL_ERROR(("listen() failed: %s (%d)", strerror(opal_socket_errno), opal_socket_errno));
         CLOSE_THE_SOCKET(sd);
         return OPAL_ERROR;
     }
 
     /* set socket up to be non-blocking, otherwise accept could block */
-    if((flags = fcntl(sd, F_GETFL, 0)) < 0) {
-        opal_show_help("help-mpi-btl-tcp.txt", "socket flag fail",
-                       true, opal_process_info.nodename,
-                       getpid(), "fcntl(sd, F_GETFL, 0)",
-                       strerror(opal_socket_errno), opal_socket_errno);
+    if ((flags = fcntl(sd, F_GETFL, 0)) < 0) {
+        opal_show_help("help-mpi-btl-tcp.txt", "socket flag fail", true, opal_process_info.nodename,
+                       getpid(), "fcntl(sd, F_GETFL, 0)", strerror(opal_socket_errno),
+                       opal_socket_errno);
         CLOSE_THE_SOCKET(sd);
         return OPAL_ERROR;
     } else {
         flags |= O_NONBLOCK;
-        if(fcntl(sd, F_SETFL, flags) < 0) {
-            opal_show_help("help-mpi-btl-tcp.txt", "socket flag fail",
-                           true, opal_process_info.nodename,
-                           getpid(),
-                           "fcntl(sd, F_SETFL, flags & O_NONBLOCK)",
-                           strerror(opal_socket_errno), opal_socket_errno);
+        if (fcntl(sd, F_SETFL, flags) < 0) {
+            opal_show_help("help-mpi-btl-tcp.txt", "socket flag fail", true,
+                           opal_process_info.nodename, getpid(),
+                           "fcntl(sd, F_SETFL, flags & O_NONBLOCK)", strerror(opal_socket_errno),
+                           opal_socket_errno);
             CLOSE_THE_SOCKET(sd);
             return OPAL_ERROR;
         }
     }
 
-    if(mca_btl_tcp_component.tcp_enable_progress_thread){
+    if (mca_btl_tcp_component.tcp_enable_progress_thread) {
         /* Declare our intent to use threads. */
         opal_event_use_threads();
-        if( NULL == mca_btl_tcp_event_base ) {
+        if (NULL == mca_btl_tcp_event_base) {
             /* fall back to only one event base (the one shared by the entire Open MPI framework) */
 
-            if( NULL == (mca_btl_tcp_event_base = opal_event_base_create()) ) {
+            if (NULL == (mca_btl_tcp_event_base = opal_event_base_create())) {
                 BTL_ERROR(("BTL TCP failed to create progress event base"));
                 goto move_forward_with_no_thread;
             }
@@ -1115,65 +1116,62 @@ static int mca_btl_tcp_component_create_listen(uint16_t af_family)
              */
             if (0 != pipe(mca_btl_tcp_pipe_to_progress)) {
                 opal_event_base_free(mca_btl_tcp_event_base);
-                /* fall back to only one event base (the one shared by the entire Open MPI framework */
+                /* fall back to only one event base (the one shared by the entire Open MPI framework
+                 */
                 mca_btl_tcp_event_base = opal_sync_event_base;
-                mca_btl_tcp_progress_thread_trigger = -1;  /* thread not started */
+                mca_btl_tcp_progress_thread_trigger = -1; /* thread not started */
                 goto move_forward_with_no_thread;
             }
             /* setup the receiving end of the pipe as non-blocking */
-            if((flags = fcntl(mca_btl_tcp_pipe_to_progress[0], F_GETFL, 0)) < 0) {
-                BTL_ERROR(("fcntl(F_GETFL) failed: %s (%d)",
-                           strerror(opal_socket_errno), opal_socket_errno));
+            if ((flags = fcntl(mca_btl_tcp_pipe_to_progress[0], F_GETFL, 0)) < 0) {
+                BTL_ERROR(("fcntl(F_GETFL) failed: %s (%d)", strerror(opal_socket_errno),
+                           opal_socket_errno));
             } else {
                 flags |= O_NONBLOCK;
-                if(fcntl(mca_btl_tcp_pipe_to_progress[0], F_SETFL, flags) < 0)
-                    BTL_ERROR(("fcntl(F_SETFL) failed: %s (%d)",
-                               strerror(opal_socket_errno), opal_socket_errno));
+                if (fcntl(mca_btl_tcp_pipe_to_progress[0], F_SETFL, flags) < 0)
+                    BTL_ERROR(("fcntl(F_SETFL) failed: %s (%d)", strerror(opal_socket_errno),
+                               opal_socket_errno));
             }
             /* Progress thread event */
-            opal_event_set(mca_btl_tcp_event_base, &mca_btl_tcp_component.tcp_recv_thread_async_event,
-                           mca_btl_tcp_pipe_to_progress[0],
-                           OPAL_EV_READ|OPAL_EV_PERSIST,
-                           mca_btl_tcp_component_event_async_handler,
-                           &mca_btl_tcp_progress_thread );
+            opal_event_set(mca_btl_tcp_event_base,
+                           &mca_btl_tcp_component.tcp_recv_thread_async_event,
+                           mca_btl_tcp_pipe_to_progress[0], OPAL_EV_READ | OPAL_EV_PERSIST,
+                           mca_btl_tcp_component_event_async_handler, &mca_btl_tcp_progress_thread);
             opal_event_add(&mca_btl_tcp_component.tcp_recv_thread_async_event, 0);
 
             /* fork off a thread to progress it */
             mca_btl_tcp_progress_thread.t_run = mca_btl_tcp_progress_thread_engine;
             mca_btl_tcp_progress_thread.t_arg = &mca_btl_tcp_progress_thread_trigger;
-            mca_btl_tcp_progress_thread_trigger = 1;  /* thread up and running */
-            if( OPAL_SUCCESS != (rc = opal_thread_start(&mca_btl_tcp_progress_thread)) ) {
+            mca_btl_tcp_progress_thread_trigger = 1; /* thread up and running */
+            if (OPAL_SUCCESS != (rc = opal_thread_start(&mca_btl_tcp_progress_thread))) {
                 BTL_ERROR(("BTL TCP progress thread initialization failed (%d)", rc));
                 opal_event_base_free(mca_btl_tcp_event_base);
-                /* fall back to only one event base (the one shared by the entire Open MPI framework */
+                /* fall back to only one event base (the one shared by the entire Open MPI framework
+                 */
                 mca_btl_tcp_event_base = opal_sync_event_base;
-                mca_btl_tcp_progress_thread_trigger = -1;  /* thread not started */
+                mca_btl_tcp_progress_thread_trigger = -1; /* thread not started */
                 goto move_forward_with_no_thread;
             }
-            /* We have async progress, the rest of the library should now protect itself against races */
+            /* We have async progress, the rest of the library should now protect itself against
+             * races */
             opal_set_using_threads(true);
         }
-    }
-    else {
-        move_forward_with_no_thread:
-            mca_btl_tcp_event_base = opal_sync_event_base;
+    } else {
+    move_forward_with_no_thread:
+        mca_btl_tcp_event_base = opal_sync_event_base;
     }
 
     if (AF_INET == af_family) {
         opal_event_set(mca_btl_tcp_event_base, &mca_btl_tcp_component.tcp_recv_event,
-                       mca_btl_tcp_component.tcp_listen_sd,
-                       OPAL_EV_READ|OPAL_EV_PERSIST,
-                       mca_btl_tcp_component_accept_handler,
-                       0 );
+                       mca_btl_tcp_component.tcp_listen_sd, OPAL_EV_READ | OPAL_EV_PERSIST,
+                       mca_btl_tcp_component_accept_handler, 0);
         MCA_BTL_TCP_ACTIVATE_EVENT(&mca_btl_tcp_component.tcp_recv_event, 0);
     }
 #if OPAL_ENABLE_IPV6
     if (AF_INET6 == af_family) {
         opal_event_set(mca_btl_tcp_event_base, &mca_btl_tcp_component.tcp6_recv_event,
-                       mca_btl_tcp_component.tcp6_listen_sd,
-                       OPAL_EV_READ|OPAL_EV_PERSIST,
-                       mca_btl_tcp_component_accept_handler,
-                       0 );
+                       mca_btl_tcp_component.tcp6_listen_sd, OPAL_EV_READ | OPAL_EV_PERSIST,
+                       mca_btl_tcp_component_accept_handler, 0);
         MCA_BTL_TCP_ACTIVATE_EVENT(&mca_btl_tcp_component.tcp6_recv_event, 0);
     }
 #endif
@@ -1187,68 +1185,62 @@ static int mca_btl_tcp_component_create_listen(uint16_t af_family)
 
 static int mca_btl_tcp_component_exchange(void)
 {
-     int rc;
-     size_t i;
-     size_t num_btls = mca_btl_tcp_component.tcp_num_btls;
-     size_t size = num_btls * sizeof(mca_btl_tcp_modex_addr_t);
-     mca_btl_tcp_modex_addr_t *addrs;
-
-     if (num_btls <= 0) {
-         return 0;
-     }
-
-     addrs = (mca_btl_tcp_modex_addr_t*)malloc(size);
-     if (NULL == addrs) {
-         return OPAL_ERR_OUT_OF_RESOURCE;
-     }
-     memset(addrs, 0, size);
-
-     for (i = 0 ; i < num_btls ; i++) {
-         struct mca_btl_tcp_module_t *btl = mca_btl_tcp_component.tcp_btls[i];
-         struct sockaddr *addr = (struct sockaddr*)&(btl->tcp_ifaddr);
+    int rc;
+    size_t i;
+    size_t num_btls = mca_btl_tcp_component.tcp_num_btls;
+    size_t size = num_btls * sizeof(mca_btl_tcp_modex_addr_t);
+    mca_btl_tcp_modex_addr_t *addrs;
+
+    if (num_btls <= 0) {
+        return 0;
+    }
+
+    addrs = (mca_btl_tcp_modex_addr_t *) malloc(size);
+    if (NULL == addrs) {
+        return OPAL_ERR_OUT_OF_RESOURCE;
+    }
+    memset(addrs, 0, size);
+
+    for (i = 0; i < num_btls; i++) {
+        struct mca_btl_tcp_module_t *btl = mca_btl_tcp_component.tcp_btls[i];
+        struct sockaddr *addr = (struct sockaddr *) &(btl->tcp_ifaddr);
 
 #if OPAL_ENABLE_IPV6
-         if (AF_INET6 == addr->sa_family) {
-             struct sockaddr_in6 *inaddr6 = (struct sockaddr_in6*)addr;
-
-             memcpy(&addrs[i].addr, &(inaddr6->sin6_addr),
-                    sizeof(struct in6_addr));
-             addrs[i].addr_port = mca_btl_tcp_component.tcp6_listen_port;
-             addrs[i].addr_family = MCA_BTL_TCP_AF_INET6;
-             opal_output_verbose(5, opal_btl_base_framework.framework_output,
-                                 "btl: tcp: exchange: %d %d IPv6 %s",
-                                 (int)i, btl->tcp_ifkindex,
-                                 opal_net_get_hostname(addr));
-         } else
+        if (AF_INET6 == addr->sa_family) {
+            struct sockaddr_in6 *inaddr6 = (struct sockaddr_in6 *) addr;
+
+            memcpy(&addrs[i].addr, &(inaddr6->sin6_addr), sizeof(struct in6_addr));
+            addrs[i].addr_port = mca_btl_tcp_component.tcp6_listen_port;
+            addrs[i].addr_family = MCA_BTL_TCP_AF_INET6;
+            opal_output_verbose(5, opal_btl_base_framework.framework_output,
+                                "btl: tcp: exchange: %d %d IPv6 %s", (int) i, btl->tcp_ifkindex,
+                                opal_net_get_hostname(addr));
+        } else
 #endif
-         if (AF_INET == addr->sa_family) {
-             struct sockaddr_in *inaddr = (struct sockaddr_in*)addr;
-
-             memcpy(&addrs[i].addr, &(inaddr->sin_addr),
-                    sizeof(struct in_addr));
-             addrs[i].addr_port = mca_btl_tcp_component.tcp_listen_port;
-             addrs[i].addr_family = MCA_BTL_TCP_AF_INET;
-             opal_output_verbose(5, opal_btl_base_framework.framework_output,
-                                 "btl: tcp: exchange: %d %d IPv4 %s",
-                                 (int)i, btl->tcp_ifkindex,
-                                 opal_net_get_hostname(addr));
-         } else {
-             BTL_ERROR(("Unexpected address family: %d", addr->sa_family));
-             free(addrs);
-             return OPAL_ERR_BAD_PARAM;
-         }
-
-         addrs[i].addr_ifkindex = btl->tcp_ifkindex;
-         addrs[i].addr_mask = btl->tcp_ifmask;
-         addrs[i].addr_bandwidth =  btl->super.btl_bandwidth;
-     }
-
-     OPAL_MODEX_SEND(rc, PMIX_GLOBAL,
-                     &mca_btl_tcp_component.super.btl_version,
-                     addrs, size);
-     free(addrs);
-
-     return rc;
+            if (AF_INET == addr->sa_family) {
+            struct sockaddr_in *inaddr = (struct sockaddr_in *) addr;
+
+            memcpy(&addrs[i].addr, &(inaddr->sin_addr), sizeof(struct in_addr));
+            addrs[i].addr_port = mca_btl_tcp_component.tcp_listen_port;
+            addrs[i].addr_family = MCA_BTL_TCP_AF_INET;
+            opal_output_verbose(5, opal_btl_base_framework.framework_output,
+                                "btl: tcp: exchange: %d %d IPv4 %s", (int) i, btl->tcp_ifkindex,
+                                opal_net_get_hostname(addr));
+        } else {
+            BTL_ERROR(("Unexpected address family: %d", addr->sa_family));
+            free(addrs);
+            return OPAL_ERR_BAD_PARAM;
+        }
+
+        addrs[i].addr_ifkindex = btl->tcp_ifkindex;
+        addrs[i].addr_mask = btl->tcp_ifmask;
+        addrs[i].addr_bandwidth = btl->super.btl_bandwidth;
+    }
+
+    OPAL_MODEX_SEND(rc, PMIX_GLOBAL, &mca_btl_tcp_component.super.btl_version, addrs, size);
+    free(addrs);
+
+    return rc;
 }
 
 /*
@@ -1258,7 +1250,7 @@ static int mca_btl_tcp_component_exchange(void)
  *  (2) setup TCP listen socket for incoming connection attempts
  *  (3) register BTL parameters with the MCA
  */
-mca_btl_base_module_t** mca_btl_tcp_component_init(int *num_btl_modules,
+mca_btl_base_module_t **mca_btl_tcp_component_init(int *num_btl_modules,
                                                    bool enable_progress_threads,
                                                    bool enable_mpi_threads)
 {
@@ -1268,75 +1260,63 @@ mca_btl_base_module_t** mca_btl_tcp_component_init(int *num_btl_modules,
     *num_btl_modules = 0;
 
     /* initialize free lists */
-    opal_free_list_init( &mca_btl_tcp_component.tcp_frag_eager,
-                         sizeof (mca_btl_tcp_frag_eager_t) +
-                         mca_btl_tcp_module.super.btl_eager_limit,
-                         opal_cache_line_size,
-                         OBJ_CLASS (mca_btl_tcp_frag_eager_t),
-                         0,opal_cache_line_size,
-                         mca_btl_tcp_component.tcp_free_list_num,
-                         mca_btl_tcp_component.tcp_free_list_max,
-                         mca_btl_tcp_component.tcp_free_list_inc,
-                         NULL, 0, NULL, NULL, NULL );
-
-    opal_free_list_init( &mca_btl_tcp_component.tcp_frag_max,
-                         sizeof (mca_btl_tcp_frag_max_t) +
-                         mca_btl_tcp_module.super.btl_max_send_size,
-                         opal_cache_line_size,
-                         OBJ_CLASS (mca_btl_tcp_frag_max_t),
-                         0,opal_cache_line_size,
-                         mca_btl_tcp_component.tcp_free_list_num,
-                         mca_btl_tcp_component.tcp_free_list_max,
-                         mca_btl_tcp_component.tcp_free_list_inc,
-                         NULL, 0, NULL, NULL, NULL );
-
-    opal_free_list_init( &mca_btl_tcp_component.tcp_frag_user,
-                         sizeof (mca_btl_tcp_frag_user_t),
-                         opal_cache_line_size,
-                         OBJ_CLASS (mca_btl_tcp_frag_user_t),
-                         0,opal_cache_line_size,
-                         mca_btl_tcp_component.tcp_free_list_num,
-                         mca_btl_tcp_component.tcp_free_list_max,
-                         mca_btl_tcp_component.tcp_free_list_inc,
-                         NULL, 0, NULL, NULL, NULL );
+    opal_free_list_init(&mca_btl_tcp_component.tcp_frag_eager,
+                        sizeof(mca_btl_tcp_frag_eager_t) + mca_btl_tcp_module.super.btl_eager_limit,
+                        opal_cache_line_size, OBJ_CLASS(mca_btl_tcp_frag_eager_t), 0,
+                        opal_cache_line_size, mca_btl_tcp_component.tcp_free_list_num,
+                        mca_btl_tcp_component.tcp_free_list_max,
+                        mca_btl_tcp_component.tcp_free_list_inc, NULL, 0, NULL, NULL, NULL);
+
+    opal_free_list_init(&mca_btl_tcp_component.tcp_frag_max,
+                        sizeof(mca_btl_tcp_frag_max_t) + mca_btl_tcp_module.super.btl_max_send_size,
+                        opal_cache_line_size, OBJ_CLASS(mca_btl_tcp_frag_max_t), 0,
+                        opal_cache_line_size, mca_btl_tcp_component.tcp_free_list_num,
+                        mca_btl_tcp_component.tcp_free_list_max,
+                        mca_btl_tcp_component.tcp_free_list_inc, NULL, 0, NULL, NULL, NULL);
+
+    opal_free_list_init(&mca_btl_tcp_component.tcp_frag_user, sizeof(mca_btl_tcp_frag_user_t),
+                        opal_cache_line_size, OBJ_CLASS(mca_btl_tcp_frag_user_t), 0,
+                        opal_cache_line_size, mca_btl_tcp_component.tcp_free_list_num,
+                        mca_btl_tcp_component.tcp_free_list_max,
+                        mca_btl_tcp_component.tcp_free_list_inc, NULL, 0, NULL, NULL, NULL);
 
     /* create a BTL TCP module for selected interfaces */
-    if(OPAL_SUCCESS != (ret = mca_btl_tcp_component_create_instances() )) {
+    if (OPAL_SUCCESS != (ret = mca_btl_tcp_component_create_instances())) {
         return 0;
     }
 
     /* create a TCP listen socket for incoming connection attempts */
-    if(OPAL_SUCCESS != (ret = mca_btl_tcp_component_create_listen(AF_INET) )) {
+    if (OPAL_SUCCESS != (ret = mca_btl_tcp_component_create_listen(AF_INET))) {
         return 0;
     }
 #if OPAL_ENABLE_IPV6
-    if((ret = mca_btl_tcp_component_create_listen(AF_INET6)) != OPAL_SUCCESS) {
-        if (!(OPAL_ERR_IN_ERRNO == ret &&
-              EAFNOSUPPORT == opal_socket_errno)) {
-            opal_output (0, "mca_btl_tcp_component: IPv6 listening socket failed\n");
+    if ((ret = mca_btl_tcp_component_create_listen(AF_INET6)) != OPAL_SUCCESS) {
+        if (!(OPAL_ERR_IN_ERRNO == ret && EAFNOSUPPORT == opal_socket_errno)) {
+            opal_output(0, "mca_btl_tcp_component: IPv6 listening socket failed\n");
             return 0;
         }
     }
 #endif
 
     /* publish TCP parameters with the MCA framework */
-    if(OPAL_SUCCESS != (ret = mca_btl_tcp_component_exchange() )) {
+    if (OPAL_SUCCESS != (ret = mca_btl_tcp_component_exchange())) {
         return 0;
     }
-    btls = (mca_btl_base_module_t **)malloc(mca_btl_tcp_component.tcp_num_btls *
-                  sizeof(mca_btl_base_module_t*));
-    if(NULL == btls) {
+    btls = (mca_btl_base_module_t **) malloc(mca_btl_tcp_component.tcp_num_btls
+                                             * sizeof(mca_btl_base_module_t *));
+    if (NULL == btls) {
         return NULL;
     }
 
     /* Register the btl to support the progress_thread */
     if (0 < mca_btl_tcp_progress_thread_trigger) {
-        for( i = 0; i < mca_btl_tcp_component.tcp_num_btls; i++) {
-            mca_btl_tcp_component.tcp_btls[i]->super.btl_flags |= MCA_BTL_FLAGS_BTL_PROGRESS_THREAD_ENABLED;
+        for (i = 0; i < mca_btl_tcp_component.tcp_num_btls; i++) {
+            mca_btl_tcp_component.tcp_btls[i]->super.btl_flags
+                |= MCA_BTL_FLAGS_BTL_PROGRESS_THREAD_ENABLED;
         }
     }
 
-    /* Avoid a race in wire-up when using threads (progess or user)
+    /* Avoid a race in wire-up when using threads (progress or user)
        and multiple BTL modules.  The details of the race are in
        https://github.com/open-mpi/ompi/issues/3035#issuecomment-429500032,
        but the summary is that the lookup code in
@@ -1347,33 +1327,27 @@ mca_btl_base_module_t** mca_btl_tcp_component_init(int *num_btl_modules,
        isn't a proper fix, but will solve the "dropped connection"
        problem until we can come up with a more complete fix to how we
        initialize procs, endpoints, and modules in the TCP BTL. */
-    if (mca_btl_tcp_component.tcp_num_btls > 1 &&
-        (enable_mpi_threads || 0 < mca_btl_tcp_progress_thread_trigger)) {
-        for( i = 0; i < mca_btl_tcp_component.tcp_num_btls; i++) {
+    if (mca_btl_tcp_component.tcp_num_btls > 1
+        && (enable_mpi_threads || 0 < mca_btl_tcp_progress_thread_trigger)) {
+        for (i = 0; i < mca_btl_tcp_component.tcp_num_btls; i++) {
             mca_btl_tcp_component.tcp_btls[i]->super.btl_flags |= MCA_BTL_FLAGS_SINGLE_ADD_PROCS;
         }
     }
 
-#if OPAL_CUDA_SUPPORT
-    mca_common_cuda_stage_one_init();
-#endif /* OPAL_CUDA_SUPPORT */
-
-    memcpy(btls, mca_btl_tcp_component.tcp_btls, mca_btl_tcp_component.tcp_num_btls*sizeof(mca_btl_tcp_module_t*));
+    memcpy(btls, mca_btl_tcp_component.tcp_btls,
+           mca_btl_tcp_component.tcp_num_btls * sizeof(mca_btl_tcp_module_t *));
     *num_btl_modules = mca_btl_tcp_component.tcp_num_btls;
     return btls;
 }
 
-
 /**
  * Called by the event engine when the listening socket has
  * a connection event. Accept the incoming connection request
  * and queue them for completion of the connection handshake.
  */
-static void mca_btl_tcp_component_accept_handler( int incoming_sd,
-                                                  short ignored,
-                                                  void* unused )
+static void mca_btl_tcp_component_accept_handler(int incoming_sd, short ignored, void *unused)
 {
-    while(true) {
+    while (true) {
 #if OPAL_ENABLE_IPV6
         struct sockaddr_in6 addr;
 #else
@@ -1382,65 +1356,61 @@ static void mca_btl_tcp_component_accept_handler( int incoming_sd,
         opal_socklen_t addrlen = sizeof(addr);
 
         mca_btl_tcp_event_t *event;
-        int sd = accept(incoming_sd, (struct sockaddr*)&addr, &addrlen);
-        if(sd < 0) {
-            if(opal_socket_errno == EINTR)
+        int sd = accept(incoming_sd, (struct sockaddr *) &addr, &addrlen);
+        if (sd < 0) {
+            if (opal_socket_errno == EINTR) {
                 continue;
-            if (opal_socket_errno != EAGAIN &&
-                opal_socket_errno != EWOULDBLOCK) {
-                opal_show_help("help-mpi-btl-tcp.txt", "accept failed",
-                               true, opal_process_info.nodename,
-                               getpid(),
-                               opal_socket_errno,
+            }
+            if (opal_socket_errno != EAGAIN && opal_socket_errno != EWOULDBLOCK) {
+                opal_show_help("help-mpi-btl-tcp.txt", "accept failed", true,
+                               opal_process_info.nodename, getpid(), opal_socket_errno,
                                strerror(opal_socket_errno));
             }
             return;
         }
         mca_btl_tcp_set_socket_options(sd);
 
-        assert( NULL != mca_btl_tcp_event_base );
+        assert(NULL != mca_btl_tcp_event_base);
         /* wait for receipt of peers process identifier to complete this connection */
         event = OBJ_NEW(mca_btl_tcp_event_t);
-        opal_event_set(mca_btl_tcp_event_base, &(event->event), sd,
-                       OPAL_EV_READ, mca_btl_tcp_component_recv_handler, event);
+        opal_event_set(mca_btl_tcp_event_base, &(event->event), sd, OPAL_EV_READ,
+                       mca_btl_tcp_component_recv_handler, event);
         opal_event_add(&event->event, 0);
     }
 }
 
-
 /**
  * Event callback when there is data available on the registered
  * socket to recv. This callback is triggered only once per lifetime
  * for any socket, in the beginning when we setup the handshake
  * protocol.
  */
-static void mca_btl_tcp_component_recv_handler(int sd, short flags, void* user)
+static void mca_btl_tcp_component_recv_handler(int sd, short flags, void *user)
 {
-    mca_btl_tcp_event_t *event = (mca_btl_tcp_event_t *)user;
+    mca_btl_tcp_event_t *event = (mca_btl_tcp_event_t *) user;
     opal_process_name_t guid;
     struct sockaddr_storage addr;
     opal_socklen_t addr_len = sizeof(addr);
-    mca_btl_tcp_proc_t* btl_proc;
+    mca_btl_tcp_proc_t *btl_proc;
     bool sockopt = true;
     size_t retval, len = strlen(mca_btl_tcp_magic_id_string);
     mca_btl_tcp_endpoint_hs_msg_t hs_msg;
     struct timeval save, tv;
     socklen_t rcvtimeo_save_len = sizeof(save);
 
-    /* Note, Socket will be in blocking mode during intial handshake
+    /* Note, Socket will be in blocking mode during initial handshake
      * hence setting SO_RCVTIMEO to say 2 seconds here to avoid waiting
      * forever when connecting to older versions (that reply to the
      * handshake with only the guid) or when the remote side isn't OMPI
      */
 
     /* get the current timeout value so we can reset to it */
-    if (0 != getsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, (void*)&save, &rcvtimeo_save_len)) {
+    if (0 != getsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, (void *) &save, &rcvtimeo_save_len)) {
         if (ENOPROTOOPT == errno || EOPNOTSUPP == errno) {
             sockopt = false;
         } else {
-            opal_show_help("help-mpi-btl-tcp.txt", "socket flag fail",
-                           true, opal_process_info.nodename,
-                           getpid(),
+            opal_show_help("help-mpi-btl-tcp.txt", "socket flag fail", true,
+                           opal_process_info.nodename, getpid(),
                            "getsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, ...)",
                            strerror(opal_socket_errno), opal_socket_errno);
             return;
@@ -1449,17 +1419,16 @@ static void mca_btl_tcp_component_recv_handler(int sd, short flags, void* user)
         tv.tv_sec = 2;
         tv.tv_usec = 0;
         if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv))) {
-            opal_show_help("help-mpi-btl-tcp.txt", "socket flag fail",
-                           true, opal_process_info.nodename,
-                           getpid(),
+            opal_show_help("help-mpi-btl-tcp.txt", "socket flag fail", true,
+                           opal_process_info.nodename, getpid(),
                            "setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, ...)",
                            strerror(opal_socket_errno), opal_socket_errno);
-           return;
-       }
+            return;
+        }
     }
 
     OBJ_RELEASE(event);
-    retval = mca_btl_tcp_recv_blocking(sd, (void *)&hs_msg, sizeof(hs_msg));
+    retval = mca_btl_tcp_recv_blocking(sd, (void *) &hs_msg, sizeof(hs_msg));
 
     /* If we get a zero-length message back, it's likely that we
        connected to Open MPI peer process X simultaneously, and the
@@ -1471,13 +1440,15 @@ static void mca_btl_tcp_component_recv_handler(int sd, short flags, void* user)
        because the peer closed the socket.  So just close it and move
        on. */
     if (retval < sizeof(hs_msg)) {
-         const char *peer = opal_fd_get_peer_name(sd);
-         opal_output_verbose(20, opal_btl_base_framework.framework_output,
-                             "Peer %s closed socket without sending BTL TCP magic ID handshake (we received %d bytes out of the expected %d) -- closing/ignoring this connection",
-                             peer, (int) retval, (int) sizeof(hs_msg));
-         free((char*) peer);
-         CLOSE_THE_SOCKET(sd);
-         return;
+        const char *peer = opal_fd_get_peer_name(sd);
+        opal_output_verbose(
+            20, opal_btl_base_framework.framework_output,
+            "Peer %s closed socket without sending BTL TCP magic ID handshake (we received %d "
+            "bytes out of the expected %d) -- closing/ignoring this connection",
+            peer, (int) retval, (int) sizeof(hs_msg));
+        free((char *) peer);
+        CLOSE_THE_SOCKET(sd);
+        return;
     }
 
     /* Open MPI uses a "magic" string to trivially verify that the
@@ -1485,13 +1456,13 @@ static void mca_btl_tcp_component_recv_handler(int sd, short flags, void* user)
        the correct magic string. */
     guid = hs_msg.guid;
     if (0 != strncmp(hs_msg.magic_id, mca_btl_tcp_magic_id_string, len)) {
-         const char *peer = opal_fd_get_peer_name(sd);
-         opal_output_verbose(20, opal_btl_base_framework.framework_output,
-                             "Peer %s send us an incorrect Open MPI magic ID string (i.e., this was not a connection from the same version of Open MPI; expected \"%s\", received \"%s\")",
-                             peer,
-                             mca_btl_tcp_magic_id_string,
-                             hs_msg.magic_id);
-         free((char*) peer);
+        const char *peer = opal_fd_get_peer_name(sd);
+        opal_output_verbose(
+            20, opal_btl_base_framework.framework_output,
+            "Peer %s send us an incorrect Open MPI magic ID string (i.e., this was not a "
+            "connection from the same version of Open MPI; expected \"%s\", received \"%s\")",
+            peer, mca_btl_tcp_magic_id_string, hs_msg.magic_id);
+        free((char *) peer);
 
         /* The other side probably isn't OMPI, so just hang up */
         CLOSE_THE_SOCKET(sd);
@@ -1499,68 +1470,61 @@ static void mca_btl_tcp_component_recv_handler(int sd, short flags, void* user)
     }
 
     if (sockopt) {
-       /* reset RECVTIMEO option to its original state */
-       if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &save, sizeof(save))) {
-           opal_show_help("help-mpi-btl-tcp.txt", "socket flag fail",
-                          true, opal_process_info.nodename,
-                          getpid(),
-                          "setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, ...)",
-                          strerror(opal_socket_errno), opal_socket_errno);
-          return;
-       }
+        /* reset RECVTIMEO option to its original state */
+        if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &save, sizeof(save))) {
+            opal_show_help("help-mpi-btl-tcp.txt", "socket flag fail", true,
+                           opal_process_info.nodename, getpid(),
+                           "setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, ...)",
+                           strerror(opal_socket_errno), opal_socket_errno);
+            return;
+        }
     }
 
     OPAL_PROCESS_NAME_NTOH(guid);
 
     /* now set socket up to be non-blocking */
-    if((flags = fcntl(sd, F_GETFL, 0)) < 0) {
-        opal_show_help("help-mpi-btl-tcp.txt", "socket flag fail",
-                       true, opal_process_info.nodename,
-                       getpid(), "fcntl(sd, F_GETFL, 0)",
-                       strerror(opal_socket_errno), opal_socket_errno);
+    if ((flags = fcntl(sd, F_GETFL, 0)) < 0) {
+        opal_show_help("help-mpi-btl-tcp.txt", "socket flag fail", true, opal_process_info.nodename,
+                       getpid(), "fcntl(sd, F_GETFL, 0)", strerror(opal_socket_errno),
+                       opal_socket_errno);
         CLOSE_THE_SOCKET(sd);
     } else {
         flags |= O_NONBLOCK;
-        if(fcntl(sd, F_SETFL, flags) < 0) {
-            opal_show_help("help-mpi-btl-tcp.txt", "socket flag fail",
-                           true, opal_process_info.nodename,
-                           getpid(),
-                           "fcntl(sd, F_SETFL, flags & O_NONBLOCK)",
-                           strerror(opal_socket_errno), opal_socket_errno);
+        if (fcntl(sd, F_SETFL, flags) < 0) {
+            opal_show_help("help-mpi-btl-tcp.txt", "socket flag fail", true,
+                           opal_process_info.nodename, getpid(),
+                           "fcntl(sd, F_SETFL, flags & O_NONBLOCK)", strerror(opal_socket_errno),
+                           opal_socket_errno);
             CLOSE_THE_SOCKET(sd);
         }
     }
 
     /* lookup the corresponding process */
     btl_proc = mca_btl_tcp_proc_lookup(&guid);
-    if(NULL == btl_proc) {
-        opal_show_help("help-mpi-btl-tcp.txt",
-                       "server accept cannot find guid",
-                       true, opal_process_info.nodename,
-                       getpid());
+    if (NULL == btl_proc) {
+        opal_show_help("help-mpi-btl-tcp.txt", "server accept cannot find guid", true,
+                       opal_process_info.nodename, getpid());
         CLOSE_THE_SOCKET(sd);
         return;
     }
 
     /* lookup peer address */
-    if(getpeername(sd, (struct sockaddr*)&addr, &addr_len) != 0) {
+    if (getpeername(sd, (struct sockaddr *) &addr, &addr_len) != 0) {
         if (ENOTCONN != opal_socket_errno) {
-            opal_show_help("help-mpi-btl-tcp.txt",
-                           "server getpeername failed",
-                           true, opal_process_info.nodename,
-                           getpid(),
-                           strerror(opal_socket_errno), opal_socket_errno);
+            opal_show_help("help-mpi-btl-tcp.txt", "server getpeername failed", true,
+                           opal_process_info.nodename, getpid(), strerror(opal_socket_errno),
+                           opal_socket_errno);
         }
         CLOSE_THE_SOCKET(sd);
         return;
     }
 
     /* are there any existing peer instances willing to accept this connection */
-    (void)mca_btl_tcp_proc_accept(btl_proc, (struct sockaddr*)&addr, sd);
+    (void) mca_btl_tcp_proc_accept(btl_proc, (struct sockaddr *) &addr, sd);
 
     const char *str = opal_fd_get_peer_name(sd);
     opal_output_verbose(10, opal_btl_base_framework.framework_output,
                         "btl:tcp: now connected to %s, process %s", str,
                         OPAL_NAME_PRINT(btl_proc->proc_opal->proc_name));
-    free((char*) str);
+    free((char *) str);
 }
diff --git a/opal/mca/btl/tcp/btl_tcp_endpoint.c b/opal/mca/btl/tcp/btl_tcp_endpoint.c
index f5ad7f51b81..28138a6b43f 100644
--- a/opal/mca/btl/tcp/btl_tcp_endpoint.c
+++ b/opal/mca/btl/tcp/btl_tcp_endpoint.c
@@ -11,7 +11,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2007-2008 Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2013-2018 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2013-2022 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2014      Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -30,42 +30,42 @@
 #include <stdlib.h>
 #include <string.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 #include "opal/opal_socket_errno.h"
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 #ifdef HAVE_FCNTL_H
-#include <fcntl.h>
+#    include <fcntl.h>
 #endif
 #ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
+#    include <netinet/in.h>
 #endif
 #ifdef HAVE_NETINET_TCP_H
-#include <netinet/tcp.h>
+#    include <netinet/tcp.h>
 #endif
 #ifdef HAVE_ARPA_INET_H
-#include <arpa/inet.h>
+#    include <arpa/inet.h>
 #endif
 #ifdef HAVE_SYS_TIME_H
-#include <sys/time.h>
-#endif  /* HAVE_SYS_TIME_H */
+#    include <sys/time.h>
+#endif /* HAVE_SYS_TIME_H */
 #include <time.h>
 
+#include "opal/mca/btl/base/btl_base_error.h"
 #include "opal/util/event.h"
 #include "opal/util/net.h"
-#include "opal/util/show_help.h"
-#include "opal/util/proc.h"
 #include "opal/util/printf.h"
+#include "opal/util/proc.h"
+#include "opal/util/show_help.h"
 #include "opal/util/string_copy.h"
-#include "opal/mca/btl/base/btl_base_error.h"
 
 #include "btl_tcp.h"
+#include "btl_tcp_addr.h"
 #include "btl_tcp_endpoint.h"
-#include "btl_tcp_proc.h"
 #include "btl_tcp_frag.h"
-#include "btl_tcp_addr.h"
+#include "btl_tcp_proc.h"
 
 /*
  * Magic ID string send during connect/accept handshake
@@ -77,7 +77,7 @@ const char mca_btl_tcp_magic_id_string[MCA_BTL_TCP_MAGIC_STRING_LENGTH] = "OPAL-
  * Initialize state of the endpoint instance.
  *
  */
-static void mca_btl_tcp_endpoint_construct(mca_btl_tcp_endpoint_t* endpoint)
+static void mca_btl_tcp_endpoint_construct(mca_btl_tcp_endpoint_t *endpoint)
 {
     endpoint->endpoint_btl = NULL;
     endpoint->endpoint_proc = NULL;
@@ -90,10 +90,10 @@ static void mca_btl_tcp_endpoint_construct(mca_btl_tcp_endpoint_t* endpoint)
     endpoint->endpoint_retries = 0;
     endpoint->endpoint_nbo = false;
 #if MCA_BTL_TCP_ENDPOINT_CACHE
-    endpoint->endpoint_cache        = NULL;
-    endpoint->endpoint_cache_pos    = NULL;
+    endpoint->endpoint_cache = NULL;
+    endpoint->endpoint_cache_pos = NULL;
     endpoint->endpoint_cache_length = 0;
-#endif  /* MCA_BTL_TCP_ENDPOINT_CACHE */
+#endif /* MCA_BTL_TCP_ENDPOINT_CACHE */
     OBJ_CONSTRUCT(&endpoint->endpoint_frags, opal_list_t);
     OBJ_CONSTRUCT(&endpoint->endpoint_send_lock, opal_mutex_t);
     OBJ_CONSTRUCT(&endpoint->endpoint_recv_lock, opal_mutex_t);
@@ -103,7 +103,7 @@ static void mca_btl_tcp_endpoint_construct(mca_btl_tcp_endpoint_t* endpoint)
  * Destroy a endpoint
  *
  */
-static void mca_btl_tcp_endpoint_destruct(mca_btl_tcp_endpoint_t* endpoint)
+static void mca_btl_tcp_endpoint_destruct(mca_btl_tcp_endpoint_t *endpoint)
 {
     mca_btl_tcp_endpoint_close(endpoint);
     mca_btl_tcp_proc_remove(endpoint->endpoint_proc, endpoint);
@@ -112,19 +112,15 @@ static void mca_btl_tcp_endpoint_destruct(mca_btl_tcp_endpoint_t* endpoint)
     OBJ_DESTRUCT(&endpoint->endpoint_recv_lock);
 }
 
-OBJ_CLASS_INSTANCE(
-    mca_btl_tcp_endpoint_t,
-    opal_list_item_t,
-    mca_btl_tcp_endpoint_construct,
-    mca_btl_tcp_endpoint_destruct);
+OBJ_CLASS_INSTANCE(mca_btl_tcp_endpoint_t, opal_list_item_t, mca_btl_tcp_endpoint_construct,
+                   mca_btl_tcp_endpoint_destruct);
 
-
-static void mca_btl_tcp_endpoint_construct(mca_btl_base_endpoint_t* btl_endpoint);
-static void mca_btl_tcp_endpoint_destruct(mca_btl_base_endpoint_t* btl_endpoint);
-static int  mca_btl_tcp_endpoint_start_connect(mca_btl_base_endpoint_t*);
-static void mca_btl_tcp_endpoint_connected(mca_btl_base_endpoint_t*);
-static void mca_btl_tcp_endpoint_recv_handler(int sd, short flags, void* user);
-static void mca_btl_tcp_endpoint_send_handler(int sd, short flags, void* user);
+static void mca_btl_tcp_endpoint_construct(mca_btl_base_endpoint_t *btl_endpoint);
+static void mca_btl_tcp_endpoint_destruct(mca_btl_base_endpoint_t *btl_endpoint);
+static int mca_btl_tcp_endpoint_start_connect(mca_btl_base_endpoint_t *);
+static void mca_btl_tcp_endpoint_connected(mca_btl_base_endpoint_t *);
+static void mca_btl_tcp_endpoint_recv_handler(int sd, short flags, void *user);
+static void mca_btl_tcp_endpoint_send_handler(int sd, short flags, void *user);
 
 /*
  * diagnostics
@@ -132,165 +128,187 @@ static void mca_btl_tcp_endpoint_send_handler(int sd, short flags, void* user);
 
 #if OPAL_ENABLE_DEBUG && WANT_PEER_DUMP
 
-#define DEBUG_LENGTH  1024
+#    define DEBUG_LENGTH 1024
 /**
  * The lack of protection in the mca_btl_tcp_endpoint_dump function is voluntary
  * so that it can be called regardless of the state of the mutexes. As a result,
  * when multiple threads work on the same endpoint not only the information
- * displayed might be inacurate, but when we manipulate the pending fragments we
+ * displayed might be inaccurate, but when we manipulate the pending fragments we
  * might access freed memory. Thus, the caller should lock the endpoint prior
  * to the call.
  */
-void
-mca_btl_tcp_endpoint_dump(int level,
-                          const char* fname,
-                          int lineno,
-                          const char* funcname,
-                          mca_btl_base_endpoint_t* btl_endpoint,
-                          bool full_info,
-                          const char* msg)
+void mca_btl_tcp_endpoint_dump(int level, const char *fname, int lineno, const char *funcname,
+                               mca_btl_base_endpoint_t *btl_endpoint, bool full_info,
+                               const char *msg)
 {
     char outmsg[DEBUG_LENGTH];
     int sndbuf, rcvbuf, nodelay, flags, used = 0;
-#if OPAL_ENABLE_IPV6
+#    if OPAL_ENABLE_IPV6
     struct sockaddr_storage inaddr;
-#else
+#    else
     struct sockaddr_in inaddr;
-#endif
+#    endif
     opal_socklen_t obtlen;
     opal_socklen_t addrlen = sizeof(inaddr);
-    mca_btl_tcp_frag_t* item;
+    mca_btl_tcp_frag_t *item;
 
     used += snprintf(&outmsg[used], DEBUG_LENGTH - used, "%s: ", msg);
-    if (used >= DEBUG_LENGTH) goto out;
+    if (used >= DEBUG_LENGTH)
+        goto out;
 
-    getsockname(btl_endpoint->endpoint_sd, (struct sockaddr*)&inaddr, &addrlen);
-#if OPAL_ENABLE_IPV6
+    getsockname(btl_endpoint->endpoint_sd, (struct sockaddr *) &inaddr, &addrlen);
+#    if OPAL_ENABLE_IPV6
     {
         char *address;
-        address = (char *) opal_net_get_hostname((struct sockaddr*) &inaddr);
+        address = (char *) opal_net_get_hostname((struct sockaddr *) &inaddr);
         if (NULL != address) {
             used += snprintf(&outmsg[used], DEBUG_LENGTH - used, "%s -", address);
-            if (used >= DEBUG_LENGTH) goto out;
+            if (used >= DEBUG_LENGTH)
+                goto out;
         }
     }
-#else
+#    else
     used += snprintf(&outmsg[used], DEBUG_LENGTH - used, "%s -", inet_ntoa(inaddr.sin_addr));
-    if (used >= DEBUG_LENGTH) goto out;
-#endif
-    getpeername(btl_endpoint->endpoint_sd, (struct sockaddr*)&inaddr, &addrlen);
-#if OPAL_ENABLE_IPV6
+    if (used >= DEBUG_LENGTH)
+        goto out;
+#    endif
+    getpeername(btl_endpoint->endpoint_sd, (struct sockaddr *) &inaddr, &addrlen);
+#    if OPAL_ENABLE_IPV6
     {
         char *address;
-        address = (char *) opal_net_get_hostname ((struct sockaddr*) &inaddr);
+        address = (char *) opal_net_get_hostname((struct sockaddr *) &inaddr);
         if (NULL != address) {
             used += snprintf(&outmsg[used], DEBUG_LENGTH - used, " %s", address);
-            if (used >= DEBUG_LENGTH) goto out;
+            if (used >= DEBUG_LENGTH)
+                goto out;
         }
     }
-#else
+#    else
     used += snprintf(&outmsg[used], DEBUG_LENGTH - used, " %s", inet_ntoa(inaddr.sin_addr));
-    if (used >= DEBUG_LENGTH) goto out;
-#endif
+    if (used >= DEBUG_LENGTH)
+        goto out;
+#    endif
 
     used = snprintf(outmsg, DEBUG_LENGTH, "[%d", btl_endpoint->endpoint_sd);
-    if (used >= DEBUG_LENGTH) goto out;
-    switch(btl_endpoint->endpoint_state) {
+    if (used >= DEBUG_LENGTH)
+        goto out;
+    switch (btl_endpoint->endpoint_state) {
     case MCA_BTL_TCP_CONNECTING:
         used += snprintf(&outmsg[used], DEBUG_LENGTH - used, ":%s]", "connecting");
-        if (used >= DEBUG_LENGTH) goto out;
+        if (used >= DEBUG_LENGTH)
+            goto out;
         break;
     case MCA_BTL_TCP_CONNECT_ACK:
         used += snprintf(&outmsg[used], DEBUG_LENGTH - used, ":%s]", "ack");
-        if (used >= DEBUG_LENGTH) goto out;
+        if (used >= DEBUG_LENGTH)
+            goto out;
         break;
     case MCA_BTL_TCP_CLOSED:
         used += snprintf(&outmsg[used], DEBUG_LENGTH - used, ":%s]", "close");
-        if (used >= DEBUG_LENGTH) goto out;
+        if (used >= DEBUG_LENGTH)
+            goto out;
         break;
     case MCA_BTL_TCP_FAILED:
         used += snprintf(&outmsg[used], DEBUG_LENGTH - used, ":%s]", "failed");
-        if (used >= DEBUG_LENGTH) goto out;
+        if (used >= DEBUG_LENGTH)
+            goto out;
         break;
     case MCA_BTL_TCP_CONNECTED:
         used += snprintf(&outmsg[used], DEBUG_LENGTH - used, ":%s]", "connected");
-        if (used >= DEBUG_LENGTH) goto out;
+        if (used >= DEBUG_LENGTH)
+            goto out;
         break;
     default:
         used += snprintf(&outmsg[used], DEBUG_LENGTH - used, ":%s]", "unknown");
-        if (used >= DEBUG_LENGTH) goto out;
+        if (used >= DEBUG_LENGTH)
+            goto out;
         break;
     }
 
-    if( full_info ) {
-        if((flags = fcntl(btl_endpoint->endpoint_sd, F_GETFL, 0)) < 0) {
-            BTL_ERROR(("fcntl(F_GETFL) failed: %s (%d)",
-                       strerror(opal_socket_errno), opal_socket_errno));
+    if (full_info) {
+        if ((flags = fcntl(btl_endpoint->endpoint_sd, F_GETFL, 0)) < 0) {
+            BTL_ERROR(
+                ("fcntl(F_GETFL) failed: %s (%d)", strerror(opal_socket_errno), opal_socket_errno));
         }
 
-#if defined(SO_SNDBUF)
+#    if defined(SO_SNDBUF)
         obtlen = sizeof(sndbuf);
-        if(getsockopt(btl_endpoint->endpoint_sd, SOL_SOCKET, SO_SNDBUF, (char *)&sndbuf, &obtlen) < 0) {
-            BTL_ERROR(("SO_SNDBUF option: %s (%d)",
-                       strerror(opal_socket_errno), opal_socket_errno));
+        if (getsockopt(btl_endpoint->endpoint_sd, SOL_SOCKET, SO_SNDBUF, (char *) &sndbuf, &obtlen)
+            < 0) {
+            BTL_ERROR(
+                ("SO_SNDBUF option: %s (%d)", strerror(opal_socket_errno), opal_socket_errno));
         }
-#else
+#    else
         sndbuf = -1;
-#endif
-#if defined(SO_RCVBUF)
+#    endif
+#    if defined(SO_RCVBUF)
         obtlen = sizeof(rcvbuf);
-        if(getsockopt(btl_endpoint->endpoint_sd, SOL_SOCKET, SO_RCVBUF, (char *)&rcvbuf, &obtlen) < 0) {
-            BTL_ERROR(("SO_RCVBUF option: %s (%d)",
-                       strerror(opal_socket_errno), opal_socket_errno));
+        if (getsockopt(btl_endpoint->endpoint_sd, SOL_SOCKET, SO_RCVBUF, (char *) &rcvbuf, &obtlen)
+            < 0) {
+            BTL_ERROR(
+                ("SO_RCVBUF option: %s (%d)", strerror(opal_socket_errno), opal_socket_errno));
         }
-#else
+#    else
         rcvbuf = -1;
-#endif
-#if defined(TCP_NODELAY)
+#    endif
+#    if defined(TCP_NODELAY)
         obtlen = sizeof(nodelay);
-        if(getsockopt(btl_endpoint->endpoint_sd, IPPROTO_TCP, TCP_NODELAY, (char *)&nodelay, &obtlen) < 0) {
-            BTL_ERROR(("TCP_NODELAY option: %s (%d)",
-                       strerror(opal_socket_errno), opal_socket_errno));
+        if (getsockopt(btl_endpoint->endpoint_sd, IPPROTO_TCP, TCP_NODELAY, (char *) &nodelay,
+                       &obtlen)
+            < 0) {
+            BTL_ERROR(
+                ("TCP_NODELAY option: %s (%d)", strerror(opal_socket_errno), opal_socket_errno));
         }
-#else
+#    else
         nodelay = 0;
-#endif
-        used += snprintf(&outmsg[used], DEBUG_LENGTH - used, " nodelay %d sndbuf %d rcvbuf %d flags %08x",
-                         nodelay, sndbuf, rcvbuf, flags);
-        if (used >= DEBUG_LENGTH) goto out;
-#if MCA_BTL_TCP_ENDPOINT_CACHE
+#    endif
+        used += snprintf(&outmsg[used], DEBUG_LENGTH - used,
+                         " nodelay %d sndbuf %d rcvbuf %d flags %08x", nodelay, sndbuf, rcvbuf,
+                         flags);
+        if (used >= DEBUG_LENGTH)
+            goto out;
+#    if MCA_BTL_TCP_ENDPOINT_CACHE
         used += snprintf(&outmsg[used], DEBUG_LENGTH - used, "\n\t[cache %p used %lu/%lu]",
-                         (void*)btl_endpoint->endpoint_cache, btl_endpoint->endpoint_cache_pos - btl_endpoint->endpoint_cache,
+                         (void *) btl_endpoint->endpoint_cache,
+                         btl_endpoint->endpoint_cache_pos - btl_endpoint->endpoint_cache,
                          btl_endpoint->endpoint_cache_length);
-        if (used >= DEBUG_LENGTH) goto out;
-#endif  /* MCA_BTL_TCP_ENDPOINT_CACHE */
+        if (used >= DEBUG_LENGTH)
+            goto out;
+#    endif /* MCA_BTL_TCP_ENDPOINT_CACHE */
         used += snprintf(&outmsg[used], DEBUG_LENGTH - used, "{%s - retries %d}",
-                         (btl_endpoint->endpoint_nbo ? "NBO" : ""), (int)btl_endpoint->endpoint_retries);
-        if (used >= DEBUG_LENGTH) goto out;
+                         (btl_endpoint->endpoint_nbo ? "NBO" : ""),
+                         (int) btl_endpoint->endpoint_retries);
+        if (used >= DEBUG_LENGTH)
+            goto out;
     }
     used += snprintf(&outmsg[used], DEBUG_LENGTH - used, "\n");
-    if (used >= DEBUG_LENGTH) goto out;
+    if (used >= DEBUG_LENGTH)
+        goto out;
 
-    if( NULL != btl_endpoint->endpoint_recv_frag )
+    if (NULL != btl_endpoint->endpoint_recv_frag)
         used += mca_btl_tcp_frag_dump(btl_endpoint->endpoint_recv_frag, "active recv",
                                       &outmsg[used], DEBUG_LENGTH - used);
-    if (used >= DEBUG_LENGTH) goto out;
-
-    if( NULL != btl_endpoint->endpoint_send_frag )
-        used += mca_btl_tcp_frag_dump(btl_endpoint->endpoint_send_frag, "active send (inaccurate iov)",
-                                      &outmsg[used], DEBUG_LENGTH - used);
-    if (used >= DEBUG_LENGTH) goto out;
-    OPAL_LIST_FOREACH(item, &btl_endpoint->endpoint_frags, mca_btl_tcp_frag_t) {
+    if (used >= DEBUG_LENGTH)
+        goto out;
+
+    if (NULL != btl_endpoint->endpoint_send_frag)
+        used += mca_btl_tcp_frag_dump(btl_endpoint->endpoint_send_frag,
+                                      "active send (inaccurate iov)", &outmsg[used],
+                                      DEBUG_LENGTH - used);
+    if (used >= DEBUG_LENGTH)
+        goto out;
+    OPAL_LIST_FOREACH (item, &btl_endpoint->endpoint_frags, mca_btl_tcp_frag_t) {
         used += mca_btl_tcp_frag_dump(item, "pending send", &outmsg[used], DEBUG_LENGTH - used);
-        if (used >= DEBUG_LENGTH) goto out;
+        if (used >= DEBUG_LENGTH)
+            goto out;
     }
 out:
-    outmsg[ used >= DEBUG_LENGTH ? (DEBUG_LENGTH-1) : used ] = '\0';
-    opal_output_verbose(level, opal_btl_base_framework.framework_output,
-                        "[%s:%d:%s][%s -> %s] %s",
-                        fname, lineno, funcname,
-                        OPAL_NAME_PRINT(opal_proc_local_get()->proc_name),
-                        (NULL != btl_endpoint->endpoint_proc ? OPAL_NAME_PRINT(btl_endpoint->endpoint_proc->proc_opal->proc_name) : "unknown remote"),
+    outmsg[used >= DEBUG_LENGTH ? (DEBUG_LENGTH - 1) : used] = '\0';
+    opal_output_verbose(level, opal_btl_base_framework.framework_output, "[%s:%d:%s][%s -> %s] %s",
+                        fname, lineno, funcname, OPAL_NAME_PRINT(opal_proc_local_get()->proc_name),
+                        (NULL != btl_endpoint->endpoint_proc
+                             ? OPAL_NAME_PRINT(btl_endpoint->endpoint_proc->proc_opal->proc_name)
+                             : "unknown remote"),
                         outmsg);
 }
 #endif /* OPAL_ENABLE_DEBUG && WANT_PEER_DUMP */
@@ -299,79 +317,78 @@ mca_btl_tcp_endpoint_dump(int level,
  * Initialize events to be used by the endpoint instance for TCP select/poll callbacks.
  */
 
-static inline void mca_btl_tcp_endpoint_event_init(mca_btl_base_endpoint_t* btl_endpoint)
+static inline void mca_btl_tcp_endpoint_event_init(mca_btl_base_endpoint_t *btl_endpoint)
 {
 #if MCA_BTL_TCP_ENDPOINT_CACHE
     assert(NULL == btl_endpoint->endpoint_cache);
-    btl_endpoint->endpoint_cache     = (char*)malloc(mca_btl_tcp_component.tcp_endpoint_cache);
+    btl_endpoint->endpoint_cache = (char *) malloc(mca_btl_tcp_component.tcp_endpoint_cache);
     btl_endpoint->endpoint_cache_pos = btl_endpoint->endpoint_cache;
-#endif  /* MCA_BTL_TCP_ENDPOINT_CACHE */
+#endif /* MCA_BTL_TCP_ENDPOINT_CACHE */
 
     opal_event_set(mca_btl_tcp_event_base, &btl_endpoint->endpoint_recv_event,
-                    btl_endpoint->endpoint_sd,
-                    OPAL_EV_READ | OPAL_EV_PERSIST,
-                    mca_btl_tcp_endpoint_recv_handler,
-                    btl_endpoint );
+                   btl_endpoint->endpoint_sd, OPAL_EV_READ | OPAL_EV_PERSIST,
+                   mca_btl_tcp_endpoint_recv_handler, btl_endpoint);
     /**
      * In the multi-threaded case, the send event must be persistent in order
      * to avoid missing the connection notification in send_handler due to
      * a local handling of the peer process (which holds the lock).
      */
     opal_event_set(mca_btl_tcp_event_base, &btl_endpoint->endpoint_send_event,
-                    btl_endpoint->endpoint_sd,
-                    OPAL_EV_WRITE | OPAL_EV_PERSIST,
-                    mca_btl_tcp_endpoint_send_handler,
-                    btl_endpoint);
+                   btl_endpoint->endpoint_sd, OPAL_EV_WRITE | OPAL_EV_PERSIST,
+                   mca_btl_tcp_endpoint_send_handler, btl_endpoint);
 }
 
-
 /*
  * Attempt to send a fragment using a given endpoint. If the endpoint is not connected,
  * queue the fragment and start the connection as required.
  */
 
-int mca_btl_tcp_endpoint_send(mca_btl_base_endpoint_t* btl_endpoint, mca_btl_tcp_frag_t* frag)
+int mca_btl_tcp_endpoint_send(mca_btl_base_endpoint_t *btl_endpoint, mca_btl_tcp_frag_t *frag)
 {
     int rc = OPAL_SUCCESS;
 
     OPAL_THREAD_LOCK(&btl_endpoint->endpoint_send_lock);
-    switch(btl_endpoint->endpoint_state) {
+    switch (btl_endpoint->endpoint_state) {
     case MCA_BTL_TCP_CONNECTING:
     case MCA_BTL_TCP_CONNECT_ACK:
     case MCA_BTL_TCP_CLOSED:
-        opal_list_append(&btl_endpoint->endpoint_frags, (opal_list_item_t*)frag);
+        opal_list_append(&btl_endpoint->endpoint_frags, (opal_list_item_t *) frag);
         frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
-        if(btl_endpoint->endpoint_state == MCA_BTL_TCP_CLOSED)
+        if (btl_endpoint->endpoint_state == MCA_BTL_TCP_CLOSED) {
             rc = mca_btl_tcp_endpoint_start_connect(btl_endpoint);
+        }
         break;
     case MCA_BTL_TCP_FAILED:
         rc = OPAL_ERR_UNREACH;
         break;
     case MCA_BTL_TCP_CONNECTED:
         if (NULL == btl_endpoint->endpoint_send_frag) {
-            if(frag->base.des_flags & MCA_BTL_DES_FLAGS_PRIORITY &&
-               mca_btl_tcp_frag_send(frag, btl_endpoint->endpoint_sd)) {
+            if (frag->base.des_flags & MCA_BTL_DES_FLAGS_PRIORITY
+                && mca_btl_tcp_frag_send(frag, btl_endpoint->endpoint_sd)) {
                 int btl_ownership = (frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
 
                 OPAL_THREAD_UNLOCK(&btl_endpoint->endpoint_send_lock);
-                if( frag->base.des_flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK ) {
+                if (frag->base.des_flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK) {
                     frag->base.des_cbfunc(&frag->btl->super, frag->endpoint, &frag->base, frag->rc);
                 }
-                if( btl_ownership ) {
+                if (btl_ownership) {
                     MCA_BTL_TCP_FRAG_RETURN(frag);
                 }
-                MCA_BTL_TCP_ENDPOINT_DUMP(50, btl_endpoint, true, "complete send fragment [endpoint_send]");
+                MCA_BTL_TCP_ENDPOINT_DUMP(50, btl_endpoint, true,
+                                          "complete send fragment [endpoint_send]");
                 return 1;
             } else {
                 btl_endpoint->endpoint_send_frag = frag;
-                MCA_BTL_TCP_ENDPOINT_DUMP(10, btl_endpoint, true, "event_add(send) [endpoint_send]");
+                MCA_BTL_TCP_ENDPOINT_DUMP(10, btl_endpoint, true,
+                                          "event_add(send) [endpoint_send]");
                 frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
                 MCA_BTL_TCP_ACTIVATE_EVENT(&btl_endpoint->endpoint_send_event, 0);
             }
         } else {
-            MCA_BTL_TCP_ENDPOINT_DUMP(10, btl_endpoint, true, "send fragment enqueued [endpoint_send]");
+            MCA_BTL_TCP_ENDPOINT_DUMP(10, btl_endpoint, true,
+                                      "send fragment enqueued [endpoint_send]");
             frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
-            opal_list_append(&btl_endpoint->endpoint_frags, (opal_list_item_t*)frag);
+            opal_list_append(&btl_endpoint->endpoint_frags, (opal_list_item_t *) frag);
         }
         break;
     }
@@ -379,14 +396,12 @@ int mca_btl_tcp_endpoint_send(mca_btl_base_endpoint_t* btl_endpoint, mca_btl_tcp
     return rc;
 }
 
-
 /*
  * A blocking send on a non-blocking socket. Used to send the small
  * amount of connection information that identifies the endpoints endpoint.
  */
-static int
-mca_btl_tcp_endpoint_send_blocking(mca_btl_base_endpoint_t* btl_endpoint,
-                                   const void* data, size_t size)
+static int mca_btl_tcp_endpoint_send_blocking(mca_btl_base_endpoint_t *btl_endpoint,
+                                              const void *data, size_t size)
 {
     int ret = mca_btl_tcp_send_blocking(btl_endpoint->endpoint_sd, data, size);
     if (ret < 0) {
@@ -402,68 +417,64 @@ mca_btl_tcp_endpoint_send_blocking(mca_btl_base_endpoint_t* btl_endpoint,
  * Send the globally unique identifier for this process to a endpoint on
  * a newly connected socket.
  */
-static int
-mca_btl_tcp_endpoint_send_connect_ack(mca_btl_base_endpoint_t* btl_endpoint)
+static int mca_btl_tcp_endpoint_send_connect_ack(mca_btl_base_endpoint_t *btl_endpoint)
 {
     opal_process_name_t guid = opal_proc_local_get()->proc_name;
     OPAL_PROCESS_NAME_HTON(guid);
 
     mca_btl_tcp_endpoint_hs_msg_t hs_msg;
-    opal_string_copy(hs_msg.magic_id, mca_btl_tcp_magic_id_string,
-                     sizeof(hs_msg.magic_id));
+    opal_string_copy(hs_msg.magic_id, mca_btl_tcp_magic_id_string, sizeof(hs_msg.magic_id));
     hs_msg.guid = guid;
 
-    if(sizeof(hs_msg) !=
-       mca_btl_tcp_endpoint_send_blocking(btl_endpoint,
-                                          &hs_msg, sizeof(hs_msg))) {
-         opal_show_help("help-mpi-btl-tcp.txt", "client handshake fail",
-                       true, opal_process_info.nodename,
-                       sizeof(hs_msg),
+    if (sizeof(hs_msg)
+        != mca_btl_tcp_endpoint_send_blocking(btl_endpoint, &hs_msg, sizeof(hs_msg))) {
+        opal_show_help("help-mpi-btl-tcp.txt", "client handshake fail", true,
+                       opal_process_info.nodename, sizeof(hs_msg),
                        "connect ACK failed to send magic-id and guid");
-          return OPAL_ERR_UNREACH;
+        return OPAL_ERR_UNREACH;
     }
     return OPAL_SUCCESS;
 }
 
 static void *mca_btl_tcp_endpoint_complete_accept(int fd, int flags, void *context)
 {
-    mca_btl_base_endpoint_t* btl_endpoint = (mca_btl_base_endpoint_t*)context;
+    mca_btl_base_endpoint_t *btl_endpoint = (mca_btl_base_endpoint_t *) context;
     struct timeval now = {0, 0};
     int cmpval;
 
-    if( OPAL_THREAD_TRYLOCK(&btl_endpoint->endpoint_recv_lock) ) {
+    if (OPAL_THREAD_TRYLOCK(&btl_endpoint->endpoint_recv_lock)) {
         opal_event_add(&btl_endpoint->endpoint_accept_event, &now);
         return NULL;
     }
-    if( OPAL_THREAD_TRYLOCK(&btl_endpoint->endpoint_send_lock) ) {
+    if (OPAL_THREAD_TRYLOCK(&btl_endpoint->endpoint_send_lock)) {
         OPAL_THREAD_UNLOCK(&btl_endpoint->endpoint_recv_lock);
         opal_event_add(&btl_endpoint->endpoint_accept_event, &now);
         return NULL;
     }
 
-    if(NULL == btl_endpoint->endpoint_addr) {
-        CLOSE_THE_SOCKET(btl_endpoint->endpoint_sd_next); /* No further use of this socket. Close it */
+    if (NULL == btl_endpoint->endpoint_addr) {
+        CLOSE_THE_SOCKET(
+            btl_endpoint->endpoint_sd_next); /* No further use of this socket. Close it */
         btl_endpoint->endpoint_sd_next = -1;
         OPAL_THREAD_UNLOCK(&btl_endpoint->endpoint_send_lock);
         OPAL_THREAD_UNLOCK(&btl_endpoint->endpoint_recv_lock);
-        if( NULL != btl_endpoint->endpoint_btl->tcp_error_cb ) {
-            btl_endpoint->endpoint_btl->tcp_error_cb(
-                &btl_endpoint->endpoint_btl->super, MCA_BTL_ERROR_FLAGS_NONFATAL,
-                btl_endpoint->endpoint_proc->proc_opal,
-                "The endpoint addr is set to NULL (unsettling)");
+        if (NULL != btl_endpoint->endpoint_btl->tcp_error_cb) {
+            btl_endpoint->endpoint_btl
+                ->tcp_error_cb(&btl_endpoint->endpoint_btl->super, MCA_BTL_ERROR_FLAGS_NONFATAL,
+                               btl_endpoint->endpoint_proc->proc_opal,
+                               "The endpoint addr is set to NULL (unsettling)");
         }
         return NULL;
     }
 
     cmpval = opal_compare_proc(btl_endpoint->endpoint_proc->proc_opal->proc_name,
                                opal_proc_local_get()->proc_name);
-    if((btl_endpoint->endpoint_sd < 0) ||
-       (btl_endpoint->endpoint_state != MCA_BTL_TCP_CONNECTED &&
-        cmpval < 0)) {
+    if ((btl_endpoint->endpoint_sd < 0)
+        || (btl_endpoint->endpoint_state != MCA_BTL_TCP_CONNECTED && cmpval < 0)) {
         mca_btl_tcp_endpoint_close(btl_endpoint);
         btl_endpoint->endpoint_sd = btl_endpoint->endpoint_sd_next;
         btl_endpoint->endpoint_sd_next = -1;
-        if(mca_btl_tcp_endpoint_send_connect_ack(btl_endpoint) != OPAL_SUCCESS) {
+        if (mca_btl_tcp_endpoint_send_connect_ack(btl_endpoint) != OPAL_SUCCESS) {
             MCA_BTL_TCP_ENDPOINT_DUMP(1, btl_endpoint, true, " [endpoint_accept]");
             btl_endpoint->endpoint_state = MCA_BTL_TCP_FAILED;
             mca_btl_tcp_endpoint_close(btl_endpoint);
@@ -472,7 +483,7 @@ static void *mca_btl_tcp_endpoint_complete_accept(int fd, int flags, void *conte
         mca_btl_tcp_endpoint_event_init(btl_endpoint);
         MCA_BTL_TCP_ENDPOINT_DUMP(10, btl_endpoint, true, "event_add(recv) [endpoint_accept]");
         opal_event_add(&btl_endpoint->endpoint_recv_event, 0);
-        if( mca_btl_tcp_event_base == opal_sync_event_base ) {
+        if (mca_btl_tcp_event_base == opal_sync_event_base) {
             /* If no progress thread then raise the awarness of the default progress engine */
             opal_progress_event_users_increment();
         }
@@ -483,7 +494,7 @@ static void *mca_btl_tcp_endpoint_complete_accept(int fd, int flags, void *conte
     }
     CLOSE_THE_SOCKET(btl_endpoint->endpoint_sd_next); /* No further use of this socket. Close it */
     btl_endpoint->endpoint_sd_next = -1;
-  unlock_and_return:
+unlock_and_return:
     OPAL_THREAD_UNLOCK(&btl_endpoint->endpoint_send_lock);
     OPAL_THREAD_UNLOCK(&btl_endpoint->endpoint_recv_lock);
     return NULL;
@@ -498,8 +509,8 @@ static void *mca_btl_tcp_endpoint_complete_accept(int fd, int flags, void *conte
  * otherwise, reject the connection and continue with the current connection
  */
 
-void mca_btl_tcp_endpoint_accept(mca_btl_base_endpoint_t* btl_endpoint,
-                                 struct sockaddr* addr, int sd)
+void mca_btl_tcp_endpoint_accept(mca_btl_base_endpoint_t *btl_endpoint, struct sockaddr *addr,
+                                 int sd)
 {
     struct timeval now = {0, 0};
 
@@ -511,21 +522,21 @@ void mca_btl_tcp_endpoint_accept(mca_btl_base_endpoint_t* btl_endpoint,
     opal_event_add(&btl_endpoint->endpoint_accept_event, &now);
 }
 
-
 /*
  * Remove any event registrations associated with the socket
  * and update the endpoint state to reflect the connection has
  * been closed.
  */
-void mca_btl_tcp_endpoint_close(mca_btl_base_endpoint_t* btl_endpoint)
+void mca_btl_tcp_endpoint_close(mca_btl_base_endpoint_t *btl_endpoint)
 {
     MCA_BTL_TCP_ENDPOINT_DUMP(1, btl_endpoint, false, "[close]");
-    if(btl_endpoint->endpoint_sd < 0)
+    if (btl_endpoint->endpoint_sd < 0) {
         return;
+    }
     btl_endpoint->endpoint_retries++;
     MCA_BTL_TCP_ENDPOINT_DUMP(1, btl_endpoint, false, "event_del(recv) [close]");
     opal_event_del(&btl_endpoint->endpoint_recv_event);
-    if( mca_btl_tcp_event_base == opal_sync_event_base ) {
+    if (mca_btl_tcp_event_base == opal_sync_event_base) {
         /* If no progress thread then lower the awarness of the default progress engine */
         opal_progress_event_users_decrement();
     }
@@ -533,23 +544,22 @@ void mca_btl_tcp_endpoint_close(mca_btl_base_endpoint_t* btl_endpoint)
     opal_event_del(&btl_endpoint->endpoint_send_event);
 
 #if MCA_BTL_TCP_ENDPOINT_CACHE
-    free( btl_endpoint->endpoint_cache );
-    btl_endpoint->endpoint_cache        = NULL;
-    btl_endpoint->endpoint_cache_pos    = NULL;
+    free(btl_endpoint->endpoint_cache);
+    btl_endpoint->endpoint_cache = NULL;
+    btl_endpoint->endpoint_cache_pos = NULL;
     btl_endpoint->endpoint_cache_length = 0;
-#endif  /* MCA_BTL_TCP_ENDPOINT_CACHE */
+#endif /* MCA_BTL_TCP_ENDPOINT_CACHE */
 
     /* send a message before closing to differentiate between failures and
      * clean disconnect during finalize */
-    if( MCA_BTL_TCP_CONNECTED == btl_endpoint->endpoint_state ) {
+    if (MCA_BTL_TCP_CONNECTED == btl_endpoint->endpoint_state) {
         mca_btl_tcp_hdr_t fin_msg = {
             .base.tag = 0,
             .type = MCA_BTL_TCP_HDR_TYPE_FIN,
             .count = 0,
             .size = 0,
         };
-        mca_btl_tcp_endpoint_send_blocking(btl_endpoint,
-                                           &fin_msg, sizeof(fin_msg));
+        mca_btl_tcp_endpoint_send_blocking(btl_endpoint, &fin_msg, sizeof(fin_msg));
     }
 
     CLOSE_THE_SOCKET(btl_endpoint->endpoint_sd);
@@ -560,22 +570,24 @@ void mca_btl_tcp_endpoint_close(mca_btl_base_endpoint_t* btl_endpoint)
      * reporting the error. The upper layer has then the opportunity to
      * re-route or re-schedule the fragments.
      */
-    if( MCA_BTL_TCP_FAILED == btl_endpoint->endpoint_state ) {
-        mca_btl_tcp_frag_t* frag = btl_endpoint->endpoint_send_frag;
-        if( NULL == frag )
-            frag = (mca_btl_tcp_frag_t*)opal_list_remove_first(&btl_endpoint->endpoint_frags);
-        while(NULL != frag) {
+    if (MCA_BTL_TCP_FAILED == btl_endpoint->endpoint_state) {
+        mca_btl_tcp_frag_t *frag = btl_endpoint->endpoint_send_frag;
+        if (NULL == frag) {
+            frag = (mca_btl_tcp_frag_t *) opal_list_remove_first(&btl_endpoint->endpoint_frags);
+        }
+        while (NULL != frag) {
             frag->base.des_cbfunc(&frag->btl->super, frag->endpoint, &frag->base, OPAL_ERR_UNREACH);
-            if( frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP ) {
+            if (frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP) {
                 MCA_BTL_TCP_FRAG_RETURN(frag);
             }
-            frag = (mca_btl_tcp_frag_t*)opal_list_remove_first(&btl_endpoint->endpoint_frags);
+            frag = (mca_btl_tcp_frag_t *) opal_list_remove_first(&btl_endpoint->endpoint_frags);
         }
         btl_endpoint->endpoint_send_frag = NULL;
         /* Let's report the error upstream */
-        if(NULL != btl_endpoint->endpoint_btl->tcp_error_cb) {
-            btl_endpoint->endpoint_btl->tcp_error_cb((mca_btl_base_module_t*)btl_endpoint->endpoint_btl, 0,
-                                                      btl_endpoint->endpoint_proc->proc_opal, "Socket closed");
+        if (NULL != btl_endpoint->endpoint_btl->tcp_error_cb) {
+            btl_endpoint->endpoint_btl
+                ->tcp_error_cb((mca_btl_base_module_t *) btl_endpoint->endpoint_btl, 0,
+                               btl_endpoint->endpoint_proc->proc_opal, "Socket closed");
         }
     } else {
         btl_endpoint->endpoint_state = MCA_BTL_TCP_CLOSED;
@@ -588,24 +600,24 @@ void mca_btl_tcp_endpoint_close(mca_btl_base_endpoint_t* btl_endpoint)
  *  send lock locked.
  */
 
-static void mca_btl_tcp_endpoint_connected(mca_btl_base_endpoint_t* btl_endpoint)
+static void mca_btl_tcp_endpoint_connected(mca_btl_base_endpoint_t *btl_endpoint)
 {
     /* setup socket options */
-    assert( MCA_BTL_TCP_CONNECTED != btl_endpoint->endpoint_state );
+    assert(MCA_BTL_TCP_CONNECTED != btl_endpoint->endpoint_state);
     btl_endpoint->endpoint_state = MCA_BTL_TCP_CONNECTED;
     btl_endpoint->endpoint_retries = 0;
     MCA_BTL_TCP_ENDPOINT_DUMP(1, btl_endpoint, true, "READY [endpoint_connected]");
 
-    if(opal_list_get_size(&btl_endpoint->endpoint_frags) > 0) {
-        if(NULL == btl_endpoint->endpoint_send_frag)
-            btl_endpoint->endpoint_send_frag = (mca_btl_tcp_frag_t*)
-                opal_list_remove_first(&btl_endpoint->endpoint_frags);
+    if (opal_list_get_size(&btl_endpoint->endpoint_frags) > 0) {
+        if (NULL == btl_endpoint->endpoint_send_frag) {
+            btl_endpoint->endpoint_send_frag = (mca_btl_tcp_frag_t *) opal_list_remove_first(
+                &btl_endpoint->endpoint_frags);
+        }
         MCA_BTL_TCP_ENDPOINT_DUMP(10, btl_endpoint, true, "event_add(send) [endpoint_connected]");
         opal_event_add(&btl_endpoint->endpoint_send_event, 0);
     }
 }
 
-
 /*
  *  Receive the endpoints globally unique process identification from a newly
  *  connected socket and verify the expected response. If so, move the
@@ -615,10 +627,10 @@ static void mca_btl_tcp_endpoint_connected(mca_btl_base_endpoint_t* btl_endpoint
  *  mca_btl_tcp_endpoint_recv_handler().  Don't change them here
  *  without also changing the handling in _recv_handler()!
  */
-static int mca_btl_tcp_endpoint_recv_connect_ack(mca_btl_base_endpoint_t* btl_endpoint)
+static int mca_btl_tcp_endpoint_recv_connect_ack(mca_btl_base_endpoint_t *btl_endpoint)
 {
-    size_t retval, len = strlen(mca_btl_tcp_magic_id_string);;
-    mca_btl_tcp_proc_t* btl_proc = btl_endpoint->endpoint_proc;
+    size_t retval, len = strlen(mca_btl_tcp_magic_id_string);
+    mca_btl_tcp_proc_t *btl_proc = btl_endpoint->endpoint_proc;
     opal_process_name_t guid;
 
     mca_btl_tcp_endpoint_hs_msg_t hs_msg;
@@ -633,15 +645,14 @@ static int mca_btl_tcp_endpoint_recv_connect_ack(mca_btl_base_endpoint_t* btl_en
                upstream. */
             return OPAL_ERROR;
         }
-        opal_show_help("help-mpi-btl-tcp.txt", "client handshake fail",
-                       true, opal_process_info.nodename,
-                       getpid(), "did not receive entire connect ACK from peer");
+        opal_show_help("help-mpi-btl-tcp.txt", "client handshake fail", true,
+                       opal_process_info.nodename, getpid(),
+                       "did not receive entire connect ACK from peer");
         return OPAL_ERR_BAD_PARAM;
     }
     if (0 != strncmp(hs_msg.magic_id, mca_btl_tcp_magic_id_string, len)) {
-        opal_show_help("help-mpi-btl-tcp.txt", "server did not receive magic string",
-                       true, opal_process_info.nodename,
-                       getpid(), "client", hs_msg.magic_id,
+        opal_show_help("help-mpi-btl-tcp.txt", "server did not receive magic string", true,
+                       opal_process_info.nodename, getpid(), "client", hs_msg.magic_id,
                        "string value");
         return OPAL_ERR_BAD_PARAM;
     }
@@ -663,29 +674,32 @@ static int mca_btl_tcp_endpoint_recv_connect_ack(mca_btl_base_endpoint_t* btl_en
     return OPAL_SUCCESS;
 }
 
-
 void mca_btl_tcp_set_socket_options(int sd)
 {
 #if defined(TCP_NODELAY)
     int optval;
     optval = !mca_btl_tcp_component.tcp_not_use_nodelay;
-    if(setsockopt(sd, IPPROTO_TCP, TCP_NODELAY, (char *)&optval, sizeof(optval)) < 0) {
-        BTL_ERROR(("setsockopt(TCP_NODELAY) failed: %s (%d)",
-                   strerror(opal_socket_errno), opal_socket_errno));
+    if (setsockopt(sd, IPPROTO_TCP, TCP_NODELAY, (char *) &optval, sizeof(optval)) < 0) {
+        BTL_ERROR(("setsockopt(TCP_NODELAY) failed: %s (%d)", strerror(opal_socket_errno),
+                   opal_socket_errno));
     }
 #endif
 #if defined(SO_SNDBUF)
-    if(mca_btl_tcp_component.tcp_sndbuf > 0 &&
-       setsockopt(sd, SOL_SOCKET, SO_SNDBUF, (char *)&mca_btl_tcp_component.tcp_sndbuf, sizeof(int)) < 0) {
-        BTL_ERROR(("setsockopt(SO_SNDBUF) failed: %s (%d)",
-                   strerror(opal_socket_errno), opal_socket_errno));
+    if (mca_btl_tcp_component.tcp_sndbuf > 0
+        && setsockopt(sd, SOL_SOCKET, SO_SNDBUF, (char *) &mca_btl_tcp_component.tcp_sndbuf,
+                      sizeof(int))
+               < 0) {
+        BTL_ERROR(("setsockopt(SO_SNDBUF) failed: %s (%d)", strerror(opal_socket_errno),
+                   opal_socket_errno));
     }
 #endif
 #if defined(SO_RCVBUF)
-    if(mca_btl_tcp_component.tcp_rcvbuf > 0 &&
-       setsockopt(sd, SOL_SOCKET, SO_RCVBUF, (char *)&mca_btl_tcp_component.tcp_rcvbuf, sizeof(int)) < 0) {
-        BTL_ERROR(("setsockopt(SO_RCVBUF) failed: %s (%d)",
-                   strerror(opal_socket_errno), opal_socket_errno));
+    if (mca_btl_tcp_component.tcp_rcvbuf > 0
+        && setsockopt(sd, SOL_SOCKET, SO_RCVBUF, (char *) &mca_btl_tcp_component.tcp_rcvbuf,
+                      sizeof(int))
+               < 0) {
+        BTL_ERROR(("setsockopt(SO_RCVBUF) failed: %s (%d)", strerror(opal_socket_errno),
+                   opal_socket_errno));
     }
 #endif
 #if defined(SO_NOSIGPIPE)
@@ -694,15 +708,13 @@ void mca_btl_tcp_set_socket_options(int sd)
      * the endpoint.
      */
     int optval2 = 1;
-    if(setsockopt(sd, SOL_SOCKET, SO_NOSIGPIPE, (char *)&optval2, sizeof(optval2)) < 0) {
-        BTL_ERROR(("setsockopt(SO_NOSIGPIPE) failed: %s (%d)",
-                   strerror(opal_socket_errno), opal_socket_errno));
+    if (setsockopt(sd, SOL_SOCKET, SO_NOSIGPIPE, (char *) &optval2, sizeof(optval2)) < 0) {
+        BTL_ERROR(("setsockopt(SO_NOSIGPIPE) failed: %s (%d)", strerror(opal_socket_errno),
+                   opal_socket_errno));
     }
 #endif
 }
 
-
-
 /*
  *  Start a connection to the endpoint. This will likely not complete,
  *  as the socket is set to non-blocking, so register for event
@@ -710,9 +722,9 @@ void mca_btl_tcp_set_socket_options(int sd)
  *  globally unique process identifier to the endpoint and wait for
  *  the endpoint response.
  */
-static int mca_btl_tcp_endpoint_start_connect(mca_btl_base_endpoint_t* btl_endpoint)
+static int mca_btl_tcp_endpoint_start_connect(mca_btl_base_endpoint_t *btl_endpoint)
 {
-    int rc,flags;
+    int rc, flags;
     struct sockaddr_storage endpoint_addr;
     /* By default consider a IPv4 connection */
     uint16_t af_family = AF_INET;
@@ -721,10 +733,10 @@ static int mca_btl_tcp_endpoint_start_connect(mca_btl_base_endpoint_t* btl_endpo
 #if OPAL_ENABLE_IPV6
     if (AF_INET6 == btl_endpoint->endpoint_addr->addr_family) {
         af_family = AF_INET6;
-        addrlen = sizeof (struct sockaddr_in6);
+        addrlen = sizeof(struct sockaddr_in6);
     }
 #endif
-    assert( btl_endpoint->endpoint_sd < 0 );
+    assert(btl_endpoint->endpoint_sd < 0);
     btl_endpoint->endpoint_sd = socket(af_family, SOCK_STREAM, 0);
     if (btl_endpoint->endpoint_sd < 0) {
         btl_endpoint->endpoint_retries++;
@@ -738,21 +750,19 @@ static int mca_btl_tcp_endpoint_start_connect(mca_btl_base_endpoint_t* btl_endpo
     mca_btl_tcp_endpoint_event_init(btl_endpoint);
 
     /* setup the socket as non-blocking */
-    if((flags = fcntl(btl_endpoint->endpoint_sd, F_GETFL, 0)) < 0) {
-        opal_show_help("help-mpi-btl-tcp.txt", "socket flag fail",
-                       true, opal_process_info.nodename,
-                       getpid(), "fcntl(sd, F_GETFL, 0)",
-                       strerror(opal_socket_errno), opal_socket_errno);
+    if ((flags = fcntl(btl_endpoint->endpoint_sd, F_GETFL, 0)) < 0) {
+        opal_show_help("help-mpi-btl-tcp.txt", "socket flag fail", true, opal_process_info.nodename,
+                       getpid(), "fcntl(sd, F_GETFL, 0)", strerror(opal_socket_errno),
+                       opal_socket_errno);
         /* Upper layer will handler the error */
         return OPAL_ERR_UNREACH;
     } else {
         flags |= O_NONBLOCK;
-        if(fcntl(btl_endpoint->endpoint_sd, F_SETFL, flags) < 0) {
-            opal_show_help("help-mpi-btl-tcp.txt", "socket flag fail",
-                           true, opal_process_info.nodename,
-                           getpid(),
-                           "fcntl(sd, F_SETFL, flags & O_NONBLOCK)",
-                           strerror(opal_socket_errno), opal_socket_errno);
+        if (fcntl(btl_endpoint->endpoint_sd, F_SETFL, flags) < 0) {
+            opal_show_help("help-mpi-btl-tcp.txt", "socket flag fail", true,
+                           opal_process_info.nodename, getpid(),
+                           "fcntl(sd, F_SETFL, flags & O_NONBLOCK)", strerror(opal_socket_errno),
+                           opal_socket_errno);
             /* Upper layer will handler the error */
             return OPAL_ERR_UNREACH;
         }
@@ -768,12 +778,15 @@ static int mca_btl_tcp_endpoint_start_connect(mca_btl_base_endpoint_t* btl_endpo
      * might do something unexpected with routing */
     if (endpoint_addr.ss_family == AF_INET) {
         assert(NULL != &btl_endpoint->endpoint_btl->tcp_ifaddr);
-        if (bind(btl_endpoint->endpoint_sd, (struct sockaddr*) &btl_endpoint->endpoint_btl->tcp_ifaddr,
-                 sizeof(struct sockaddr_in)) < 0) {
-            BTL_ERROR(("bind on local address (%s:%d) failed: %s (%d)",
-                       opal_net_get_hostname((struct sockaddr*) &btl_endpoint->endpoint_btl->tcp_ifaddr),
-                       htons(((struct sockaddr_in*)&btl_endpoint->endpoint_btl->tcp_ifaddr)->sin_port),
-                       strerror(opal_socket_errno), opal_socket_errno));
+        if (bind(btl_endpoint->endpoint_sd,
+                 (struct sockaddr *) &btl_endpoint->endpoint_btl->tcp_ifaddr,
+                 sizeof(struct sockaddr_in))
+            < 0) {
+            BTL_ERROR(
+                ("bind on local address (%s:%d) failed: %s (%d)",
+                 opal_net_get_hostname((struct sockaddr *) &btl_endpoint->endpoint_btl->tcp_ifaddr),
+                 htons(((struct sockaddr_in *) &btl_endpoint->endpoint_btl->tcp_ifaddr)->sin_port),
+                 strerror(opal_socket_errno), opal_socket_errno));
 
             CLOSE_THE_SOCKET(btl_endpoint->endpoint_sd);
             return OPAL_ERROR;
@@ -782,12 +795,16 @@ static int mca_btl_tcp_endpoint_start_connect(mca_btl_base_endpoint_t* btl_endpo
 #if OPAL_ENABLE_IPV6
     if (endpoint_addr.ss_family == AF_INET6) {
         assert(NULL != &btl_endpoint->endpoint_btl->tcp_ifaddr);
-        if (bind(btl_endpoint->endpoint_sd, (struct sockaddr*) &btl_endpoint->endpoint_btl->tcp_ifaddr,
-                 sizeof(struct sockaddr_in6)) < 0) {
-            BTL_ERROR(("bind on local address (%s:%d) failed: %s (%d)",
-                       opal_net_get_hostname((struct sockaddr*) &btl_endpoint->endpoint_btl->tcp_ifaddr),
-                       htons(((struct sockaddr_in6*)&btl_endpoint->endpoint_btl->tcp_ifaddr)->sin6_port),
-                       strerror(opal_socket_errno), opal_socket_errno));
+        if (bind(btl_endpoint->endpoint_sd,
+                 (struct sockaddr *) &btl_endpoint->endpoint_btl->tcp_ifaddr,
+                 sizeof(struct sockaddr_in6))
+            < 0) {
+            BTL_ERROR(
+                ("bind on local address (%s:%d) failed: %s (%d)",
+                 opal_net_get_hostname((struct sockaddr *) &btl_endpoint->endpoint_btl->tcp_ifaddr),
+                 htons(
+                     ((struct sockaddr_in6 *) &btl_endpoint->endpoint_btl->tcp_ifaddr)->sin6_port),
+                 strerror(opal_socket_errno), opal_socket_errno));
 
             CLOSE_THE_SOCKET(btl_endpoint->endpoint_sd);
             return OPAL_ERROR;
@@ -797,30 +814,31 @@ static int mca_btl_tcp_endpoint_start_connect(mca_btl_base_endpoint_t* btl_endpo
     opal_output_verbose(10, opal_btl_base_framework.framework_output,
                         "btl: tcp: attempting to connect() to %s address %s on port %d",
                         OPAL_NAME_PRINT(btl_endpoint->endpoint_proc->proc_opal->proc_name),
-                        opal_net_get_hostname((struct sockaddr*) &endpoint_addr),
+                        opal_net_get_hostname((struct sockaddr *) &endpoint_addr),
                         ntohs(btl_endpoint->endpoint_addr->addr_port));
 
-    if(0 == connect(btl_endpoint->endpoint_sd, (struct sockaddr*)&endpoint_addr, addrlen)) {
+    if (0 == connect(btl_endpoint->endpoint_sd, (struct sockaddr *) &endpoint_addr, addrlen)) {
         opal_output_verbose(10, opal_btl_base_framework.framework_output,
                             "btl:tcp: connect() to %s:%d completed",
-                            opal_net_get_hostname((struct sockaddr*) &endpoint_addr),
-                            ntohs(((struct sockaddr_in*) &endpoint_addr)->sin_port));
+                            opal_net_get_hostname((struct sockaddr *) &endpoint_addr),
+                            ntohs(((struct sockaddr_in *) &endpoint_addr)->sin_port));
         /* send our globally unique process identifier to the endpoint */
-        if((rc = mca_btl_tcp_endpoint_send_connect_ack(btl_endpoint)) == OPAL_SUCCESS) {
+        if ((rc = mca_btl_tcp_endpoint_send_connect_ack(btl_endpoint)) == OPAL_SUCCESS) {
             btl_endpoint->endpoint_state = MCA_BTL_TCP_CONNECT_ACK;
             MCA_BTL_TCP_ENDPOINT_DUMP(10, btl_endpoint, true, "event_add(recv) [start_connect]");
             opal_event_add(&btl_endpoint->endpoint_recv_event, 0);
-            if( mca_btl_tcp_event_base == opal_sync_event_base ) {
+            if (mca_btl_tcp_event_base == opal_sync_event_base) {
                 /* If no progress thread then raise the awarness of the default progress engine */
                 opal_progress_event_users_increment();
             }
             return OPAL_SUCCESS;
         }
-        /* We connected to the peer, but he close the socket before we got a chance to send our guid */
+        /* We connected to the peer, but he close the socket before we got a chance to send our guid
+         */
         MCA_BTL_TCP_ENDPOINT_DUMP(1, btl_endpoint, true, "dropped connection [start_connect]");
     } else {
         /* non-blocking so wait for completion */
-        if(opal_socket_errno == EINPROGRESS || opal_socket_errno == EWOULDBLOCK) {
+        if (opal_socket_errno == EINPROGRESS || opal_socket_errno == EWOULDBLOCK) {
             btl_endpoint->endpoint_state = MCA_BTL_TCP_CONNECTING;
             MCA_BTL_TCP_ENDPOINT_DUMP(10, btl_endpoint, true, "event_add(send) [start_connect]");
             MCA_BTL_TCP_ACTIVATE_EVENT(&btl_endpoint->endpoint_send_event, 0);
@@ -832,24 +850,23 @@ static int mca_btl_tcp_endpoint_start_connect(mca_btl_base_endpoint_t* btl_endpo
 
     {
         char *address;
-        address = opal_net_get_hostname((struct sockaddr*) &endpoint_addr);
-        BTL_PEER_ERROR( btl_endpoint->endpoint_proc->proc_opal,
-                      ( "Unable to connect to the peer %s on port %d: %s\n",
-                        address,
-                        ntohs(btl_endpoint->endpoint_addr->addr_port), strerror(opal_socket_errno) ) );
+        address = opal_net_get_hostname((struct sockaddr *) &endpoint_addr);
+        BTL_PEER_ERROR(btl_endpoint->endpoint_proc->proc_opal,
+                       ("Unable to connect to the peer %s on port %d: %s\n", address,
+                        ntohs(btl_endpoint->endpoint_addr->addr_port),
+                        strerror(opal_socket_errno)));
     }
     btl_endpoint->endpoint_state = MCA_BTL_TCP_FAILED;
     mca_btl_tcp_endpoint_close(btl_endpoint);
     return OPAL_ERR_UNREACH;
 }
 
-
 /*
  * Check the status of the connection. If the connection failed, will retry
  * later. Otherwise, send this processes identifier to the endpoint on the
  * newly connected socket.
  */
-static int mca_btl_tcp_endpoint_complete_connect(mca_btl_base_endpoint_t* btl_endpoint)
+static int mca_btl_tcp_endpoint_complete_connect(mca_btl_base_endpoint_t *btl_endpoint)
 {
     int so_error = 0;
     opal_socklen_t so_length = sizeof(so_error);
@@ -864,33 +881,30 @@ static int mca_btl_tcp_endpoint_complete_connect(mca_btl_base_endpoint_t* btl_en
     mca_btl_tcp_proc_tosocks(btl_endpoint->endpoint_addr, &endpoint_addr);
 
     /* check connect completion status */
-    if(getsockopt(btl_endpoint->endpoint_sd, SOL_SOCKET, SO_ERROR, (char *)&so_error, &so_length) < 0) {
-        opal_show_help("help-mpi-btl-tcp.txt", "socket flag fail",
-                       true, opal_process_info.nodename,
-                       getpid(), "fcntl(sd, F_GETFL, 0)",
-                       strerror(opal_socket_errno), opal_socket_errno);
+    if (getsockopt(btl_endpoint->endpoint_sd, SOL_SOCKET, SO_ERROR, (char *) &so_error, &so_length)
+        < 0) {
+        opal_show_help("help-mpi-btl-tcp.txt", "socket flag fail", true, opal_process_info.nodename,
+                       getpid(), "fcntl(sd, F_GETFL, 0)", strerror(opal_socket_errno),
+                       opal_socket_errno);
         BTL_ERROR(("getsockopt() to %s:%d failed: %s (%d)",
-                   opal_net_get_hostname((struct sockaddr*) &endpoint_addr),
-                   ((struct sockaddr_in*) &endpoint_addr)->sin_port,
-                   strerror(opal_socket_errno), opal_socket_errno));
+                   opal_net_get_hostname((struct sockaddr *) &endpoint_addr),
+                   ((struct sockaddr_in *) &endpoint_addr)->sin_port, strerror(opal_socket_errno),
+                   opal_socket_errno));
         btl_endpoint->endpoint_state = MCA_BTL_TCP_FAILED;
         mca_btl_tcp_endpoint_close(btl_endpoint);
         return OPAL_ERROR;
     }
-    if(so_error == EINPROGRESS || so_error == EWOULDBLOCK) {
+    if (so_error == EINPROGRESS || so_error == EWOULDBLOCK) {
         return OPAL_SUCCESS;
     }
-    if(so_error != 0) {
-        if( mca_btl_base_warn_peer_error
-         || mca_btl_base_verbose > 0 ) {
+    if (so_error != 0) {
+        if (mca_btl_base_warn_peer_error || mca_btl_base_verbose > 0) {
             char *msg;
             opal_asprintf(&msg, "connect() to %s:%d failed",
-                 opal_net_get_hostname((struct sockaddr*) &endpoint_addr),
-                 ntohs(((struct sockaddr_in*) &endpoint_addr)->sin_port));
-            opal_show_help("help-mpi-btl-tcp.txt", "client connect fail",
-                       true, opal_process_info.nodename,
-                       getpid(), msg,
-                       strerror(so_error), so_error);
+                          opal_net_get_hostname((struct sockaddr *) &endpoint_addr),
+                          ntohs(((struct sockaddr_in *) &endpoint_addr)->sin_port));
+            opal_show_help("help-mpi-btl-tcp.txt", "client connect fail", true,
+                           opal_process_info.nodename, getpid(), msg, strerror(so_error), so_error);
             free(msg);
         }
         btl_endpoint->endpoint_state = MCA_BTL_TCP_FAILED;
@@ -898,15 +912,16 @@ static int mca_btl_tcp_endpoint_complete_connect(mca_btl_base_endpoint_t* btl_en
         return OPAL_ERROR;
     }
 
-    opal_output_verbose(10, opal_btl_base_framework.framework_output,
-                        "btl:tcp: connect() to %s:%d completed (complete_connect), sending connect ACK",
-                        opal_net_get_hostname((struct sockaddr*) &endpoint_addr),
-                        ntohs(((struct sockaddr_in*) &endpoint_addr)->sin_port));
+    opal_output_verbose(
+        10, opal_btl_base_framework.framework_output,
+        "btl:tcp: connect() to %s:%d completed (complete_connect), sending connect ACK",
+        opal_net_get_hostname((struct sockaddr *) &endpoint_addr),
+        ntohs(((struct sockaddr_in *) &endpoint_addr)->sin_port));
 
-    if(mca_btl_tcp_endpoint_send_connect_ack(btl_endpoint) == OPAL_SUCCESS) {
+    if (mca_btl_tcp_endpoint_send_connect_ack(btl_endpoint) == OPAL_SUCCESS) {
         btl_endpoint->endpoint_state = MCA_BTL_TCP_CONNECT_ACK;
         opal_event_add(&btl_endpoint->endpoint_recv_event, 0);
-        if( mca_btl_tcp_event_base == opal_sync_event_base ) {
+        if (mca_btl_tcp_event_base == opal_sync_event_base) {
             /* If no progress thread then raise the awarness of the default progress engine */
             opal_progress_event_users_increment();
         }
@@ -919,21 +934,21 @@ static int mca_btl_tcp_endpoint_complete_connect(mca_btl_base_endpoint_t* btl_en
     return OPAL_ERROR;
 }
 
-
 /*
  * A file descriptor is available/ready for recv. Check the state
  * of the socket and take the appropriate action.
  */
 
-static void mca_btl_tcp_endpoint_recv_handler(int sd, short flags, void* user)
+static void mca_btl_tcp_endpoint_recv_handler(int sd, short flags, void *user)
 {
-    mca_btl_base_endpoint_t* btl_endpoint = (mca_btl_base_endpoint_t *)user;
+    mca_btl_base_endpoint_t *btl_endpoint = (mca_btl_base_endpoint_t *) user;
 
     /* Make sure we don't have a race between a thread that remove the
      * recv event, and one event already scheduled.
      */
-    if( sd != btl_endpoint->endpoint_sd )
+    if (sd != btl_endpoint->endpoint_sd) {
         return;
+    }
 
     /**
      * There is an extremely rare race condition here, that can only be
@@ -949,105 +964,102 @@ static void mca_btl_tcp_endpoint_recv_handler(int sd, short flags, void* user)
      * event_base lock (while holding the endpoint_recv lock).
      *
      * If we can't lock this mutex, it is OK to cancel the receive operation, it
-     * will be eventually triggered again shorthly.
+     * will be eventually triggered again shortly.
      */
-    if( OPAL_THREAD_TRYLOCK(&btl_endpoint->endpoint_recv_lock) )
+    if (OPAL_THREAD_TRYLOCK(&btl_endpoint->endpoint_recv_lock)) {
         return;
+    }
 
-    switch(btl_endpoint->endpoint_state) {
-    case MCA_BTL_TCP_CONNECT_ACK:
-        {
-            int rc = mca_btl_tcp_endpoint_recv_connect_ack(btl_endpoint);
-            if( OPAL_SUCCESS == rc ) {
-                /* we are now connected. Start sending the data */
-                OPAL_THREAD_LOCK(&btl_endpoint->endpoint_send_lock);
-                mca_btl_tcp_endpoint_connected(btl_endpoint);
-                OPAL_THREAD_UNLOCK(&btl_endpoint->endpoint_send_lock);
-                MCA_BTL_TCP_ENDPOINT_DUMP(10, btl_endpoint, true, "connected");
-            }
-            else if (OPAL_ERR_BAD_PARAM == rc
-                  || OPAL_ERROR == rc) {
-                /* If we get a BAD_PARAM, it means that it probably wasn't
-                   an OMPI process on the other end of the socket (e.g.,
-                   the magic string ID failed). recv_connect_ack already cleaned
-                   up the socket. */
-                /* If we get OPAL_ERROR, the other end closed the connection
-                 * because it has initiated a symetrical connexion on its end.
-                 * recv_connect_ack already cleaned up the socket. */
-            }
-            else {
-                /* Otherwise, it probably *was* an OMPI peer process on
-                   the other end, and something bad has probably
-                   happened.  */
-                mca_btl_tcp_module_t *m = btl_endpoint->endpoint_btl;
-
-                /* Fail up to the PML */
-                if (NULL != m->tcp_error_cb) {
-                    m->tcp_error_cb((mca_btl_base_module_t*) m, MCA_BTL_ERROR_FLAGS_FATAL,
-                        btl_endpoint->endpoint_proc->proc_opal,
-                        "TCP ACK is neither SUCCESS nor ERR (something bad has probably happened)");
-                }
+    switch (btl_endpoint->endpoint_state) {
+    case MCA_BTL_TCP_CONNECT_ACK: {
+        int rc = mca_btl_tcp_endpoint_recv_connect_ack(btl_endpoint);
+        if (OPAL_SUCCESS == rc) {
+            /* we are now connected. Start sending the data */
+            OPAL_THREAD_LOCK(&btl_endpoint->endpoint_send_lock);
+            mca_btl_tcp_endpoint_connected(btl_endpoint);
+            OPAL_THREAD_UNLOCK(&btl_endpoint->endpoint_send_lock);
+            MCA_BTL_TCP_ENDPOINT_DUMP(10, btl_endpoint, true, "connected");
+        } else if (OPAL_ERR_BAD_PARAM == rc || OPAL_ERROR == rc) {
+            /* If we get a BAD_PARAM, it means that it probably wasn't
+               an OMPI process on the other end of the socket (e.g.,
+               the magic string ID failed). recv_connect_ack already cleaned
+               up the socket. */
+            /* If we get OPAL_ERROR, the other end closed the connection
+             * because it has initiated a symmetrical connexion on its end.
+             * recv_connect_ack already cleaned up the socket. */
+        } else {
+            /* Otherwise, it probably *was* an OMPI peer process on
+               the other end, and something bad has probably
+               happened.  */
+            mca_btl_tcp_module_t *m = btl_endpoint->endpoint_btl;
+
+            /* Fail up to the PML */
+            if (NULL != m->tcp_error_cb) {
+                m->tcp_error_cb(
+                    (mca_btl_base_module_t *) m, MCA_BTL_ERROR_FLAGS_FATAL,
+                    btl_endpoint->endpoint_proc->proc_opal,
+                    "TCP ACK is neither SUCCESS nor ERR (something bad has probably happened)");
             }
-            OPAL_THREAD_UNLOCK(&btl_endpoint->endpoint_recv_lock);
-            return;
         }
-    case MCA_BTL_TCP_CONNECTED:
-        {
-            mca_btl_tcp_frag_t* frag;
-
-            frag = btl_endpoint->endpoint_recv_frag;
-            if(NULL == frag) {
-                if(mca_btl_tcp_module.super.btl_max_send_size >
-                   mca_btl_tcp_module.super.btl_eager_limit) {
-                    MCA_BTL_TCP_FRAG_ALLOC_MAX(frag);
-                } else {
-                    MCA_BTL_TCP_FRAG_ALLOC_EAGER(frag);
-                }
+        OPAL_THREAD_UNLOCK(&btl_endpoint->endpoint_recv_lock);
+        return;
+    }
+    case MCA_BTL_TCP_CONNECTED: {
+        mca_btl_tcp_frag_t *frag;
+
+        frag = btl_endpoint->endpoint_recv_frag;
+        if (NULL == frag) {
+            if (mca_btl_tcp_module.super.btl_max_send_size
+                > mca_btl_tcp_module.super.btl_eager_limit) {
+                MCA_BTL_TCP_FRAG_ALLOC_MAX(frag);
+            } else {
+                MCA_BTL_TCP_FRAG_ALLOC_EAGER(frag);
+            }
 
-                if(NULL == frag) {
-                    OPAL_THREAD_UNLOCK(&btl_endpoint->endpoint_recv_lock);
-                    return;
-                }
-                MCA_BTL_TCP_FRAG_INIT_DST(frag, btl_endpoint);
+            if (NULL == frag) {
+                OPAL_THREAD_UNLOCK(&btl_endpoint->endpoint_recv_lock);
+                return;
             }
+            MCA_BTL_TCP_FRAG_INIT_DST(frag, btl_endpoint);
+        }
 
 #if MCA_BTL_TCP_ENDPOINT_CACHE
-            assert( 0 == btl_endpoint->endpoint_cache_length );
-        data_still_pending_on_endpoint:
-#endif  /* MCA_BTL_TCP_ENDPOINT_CACHE */
-            /* check for completion of non-blocking recv on the current fragment */
-            if(mca_btl_tcp_frag_recv(frag, btl_endpoint->endpoint_sd) == false) {
-                btl_endpoint->endpoint_recv_frag = frag;
-            } else {
-                btl_endpoint->endpoint_recv_frag = NULL;
-                if( MCA_BTL_TCP_HDR_TYPE_SEND == frag->hdr.type ) {
-                    mca_btl_active_message_callback_t *reg =
-                      mca_btl_base_active_message_trigger + frag->hdr.base.tag;
-                    const mca_btl_base_receive_descriptor_t desc =
-                      {.endpoint = btl_endpoint,
+        assert(0 == btl_endpoint->endpoint_cache_length);
+    data_still_pending_on_endpoint:
+#endif /* MCA_BTL_TCP_ENDPOINT_CACHE */
+        /* check for completion of non-blocking recv on the current fragment */
+        if (mca_btl_tcp_frag_recv(frag, btl_endpoint->endpoint_sd) == false) {
+            btl_endpoint->endpoint_recv_frag = frag;
+        } else {
+            btl_endpoint->endpoint_recv_frag = NULL;
+            if (MCA_BTL_TCP_HDR_TYPE_SEND == frag->hdr.type) {
+                mca_btl_active_message_callback_t *reg = mca_btl_base_active_message_trigger
+                                                         + frag->hdr.base.tag;
+                const mca_btl_base_receive_descriptor_t desc
+                    = {.endpoint = btl_endpoint,
                        .des_segments = frag->base.des_segments,
                        .des_segment_count = frag->base.des_segment_count,
                        .tag = frag->hdr.base.tag,
                        .cbdata = reg->cbdata};
-                    reg->cbfunc(&frag->btl->super, &desc);
-                }
-#if MCA_BTL_TCP_ENDPOINT_CACHE
-                if( 0 != btl_endpoint->endpoint_cache_length ) {
-                    /* If the cache still contain some data we can reuse the same fragment
-                     * until we flush it completly.
-                     */
-                    MCA_BTL_TCP_FRAG_INIT_DST(frag, btl_endpoint);
-                    goto data_still_pending_on_endpoint;
-                }
-#endif  /* MCA_BTL_TCP_ENDPOINT_CACHE */
-                MCA_BTL_TCP_FRAG_RETURN(frag);
+                reg->cbfunc(&frag->btl->super, &desc);
             }
 #if MCA_BTL_TCP_ENDPOINT_CACHE
-            assert( 0 == btl_endpoint->endpoint_cache_length );
-#endif  /* MCA_BTL_TCP_ENDPOINT_CACHE */
-            OPAL_THREAD_UNLOCK(&btl_endpoint->endpoint_recv_lock);
-            break;
+            if (0 != btl_endpoint->endpoint_cache_length) {
+                /* If the cache still contain some data we can reuse the same fragment
+                 * until we flush it completely.
+                 */
+                MCA_BTL_TCP_FRAG_INIT_DST(frag, btl_endpoint);
+                goto data_still_pending_on_endpoint;
+            }
+#endif /* MCA_BTL_TCP_ENDPOINT_CACHE */
+            MCA_BTL_TCP_FRAG_RETURN(frag);
         }
+#if MCA_BTL_TCP_ENDPOINT_CACHE
+        assert(0 == btl_endpoint->endpoint_cache_length);
+#endif /* MCA_BTL_TCP_ENDPOINT_CACHE */
+        OPAL_THREAD_UNLOCK(&btl_endpoint->endpoint_recv_lock);
+        break;
+    }
     case MCA_BTL_TCP_CLOSED:
         /* This is a thread-safety issue. As multiple threads are allowed
          * to generate events (in the lib event) we endup with several
@@ -1066,68 +1078,72 @@ static void mca_btl_tcp_endpoint_recv_handler(int sd, short flags, void* user)
     }
 }
 
-
 /*
  * A file descriptor is available/ready for send. Check the state
  * of the socket and take the appropriate action.
  */
 
-static void mca_btl_tcp_endpoint_send_handler(int sd, short flags, void* user)
+static void mca_btl_tcp_endpoint_send_handler(int sd, short flags, void *user)
 {
-    mca_btl_tcp_endpoint_t* btl_endpoint = (mca_btl_tcp_endpoint_t *)user;
+    mca_btl_tcp_endpoint_t *btl_endpoint = (mca_btl_tcp_endpoint_t *) user;
 
     /* if another thread is already here, give up */
-    if( OPAL_THREAD_TRYLOCK(&btl_endpoint->endpoint_send_lock) )
+    if (OPAL_THREAD_TRYLOCK(&btl_endpoint->endpoint_send_lock)) {
         return;
+    }
 
-    switch(btl_endpoint->endpoint_state) {
+    switch (btl_endpoint->endpoint_state) {
     case MCA_BTL_TCP_CONNECTING:
         mca_btl_tcp_endpoint_complete_connect(btl_endpoint);
         break;
     case MCA_BTL_TCP_CONNECTED:
         /* complete the current send */
         while (NULL != btl_endpoint->endpoint_send_frag) {
-            mca_btl_tcp_frag_t* frag = btl_endpoint->endpoint_send_frag;
+            mca_btl_tcp_frag_t *frag = btl_endpoint->endpoint_send_frag;
             int btl_ownership = (frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
 
             assert(btl_endpoint->endpoint_state == MCA_BTL_TCP_CONNECTED);
-            if(mca_btl_tcp_frag_send(frag, btl_endpoint->endpoint_sd) == false) {
+            if (mca_btl_tcp_frag_send(frag, btl_endpoint->endpoint_sd) == false) {
                 break;
             }
             /* progress any pending sends */
-            btl_endpoint->endpoint_send_frag = (mca_btl_tcp_frag_t*)
-                opal_list_remove_first(&btl_endpoint->endpoint_frags);
+            btl_endpoint->endpoint_send_frag = (mca_btl_tcp_frag_t *) opal_list_remove_first(
+                &btl_endpoint->endpoint_frags);
 
             /* if required - update request status and release fragment */
             OPAL_THREAD_UNLOCK(&btl_endpoint->endpoint_send_lock);
-            assert( frag->base.des_flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK );
+            assert(frag->base.des_flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK);
             if (NULL != frag->base.des_cbfunc) {
                 frag->base.des_cbfunc(&frag->btl->super, frag->endpoint, &frag->base, frag->rc);
             }
-            if( btl_ownership ) {
+            if (btl_ownership) {
                 MCA_BTL_TCP_FRAG_RETURN(frag);
             }
             /* if we fail to take the lock simply return. In the worst case the
              * send_handler will be triggered once more, and as there will be
              * nothing to send the handler will be deleted.
              */
-            if( OPAL_THREAD_TRYLOCK(&btl_endpoint->endpoint_send_lock) )
+            if (OPAL_THREAD_TRYLOCK(&btl_endpoint->endpoint_send_lock)) {
                 return;
+            }
         }
 
         /* if nothing else to do unregister for send event notifications */
-        if(NULL == btl_endpoint->endpoint_send_frag) {
-            MCA_BTL_TCP_ENDPOINT_DUMP(10, btl_endpoint, false, "event_del(send) [endpoint_send_handler]");
+        if (NULL == btl_endpoint->endpoint_send_frag) {
+            MCA_BTL_TCP_ENDPOINT_DUMP(10, btl_endpoint, false,
+                                      "event_del(send) [endpoint_send_handler]");
             opal_event_del(&btl_endpoint->endpoint_send_event);
         }
         break;
     case MCA_BTL_TCP_FAILED:
-        MCA_BTL_TCP_ENDPOINT_DUMP(1, btl_endpoint, true, "event_del(send) [endpoint_send_handler:error]");
+        MCA_BTL_TCP_ENDPOINT_DUMP(1, btl_endpoint, true,
+                                  "event_del(send) [endpoint_send_handler:error]");
         opal_event_del(&btl_endpoint->endpoint_send_event);
         break;
     default:
         BTL_ERROR(("invalid connection state (%d)", btl_endpoint->endpoint_state));
-        MCA_BTL_TCP_ENDPOINT_DUMP(1, btl_endpoint, true, "event_del(send) [endpoint_send_handler:error]");
+        MCA_BTL_TCP_ENDPOINT_DUMP(1, btl_endpoint, true,
+                                  "event_del(send) [endpoint_send_handler:error]");
         opal_event_del(&btl_endpoint->endpoint_send_event);
         break;
     }
diff --git a/opal/mca/btl/tcp/btl_tcp_endpoint.h b/opal/mca/btl/tcp/btl_tcp_endpoint.h
index 05318f393d6..8eaff79e8f3 100644
--- a/opal/mca/btl/tcp/btl_tcp_endpoint.h
+++ b/opal/mca/btl/tcp/btl_tcp_endpoint.h
@@ -21,13 +21,13 @@
 #ifndef MCA_BTL_TCP_ENDPOINT_H
 #define MCA_BTL_TCP_ENDPOINT_H
 
+#include "btl_tcp.h"
+#include "btl_tcp_frag.h"
 #include "opal/class/opal_list.h"
 #include "opal/util/event.h"
-#include "btl_tcp_frag.h"
-#include "btl_tcp.h"
 BEGIN_C_DECLS
 
-#define MCA_BTL_TCP_ENDPOINT_CACHE 1
+#define MCA_BTL_TCP_ENDPOINT_CACHE      1
 #define MCA_BTL_TCP_MAGIC_STRING_LENGTH 16
 /**
  * State of TCP endpoint connection.
@@ -49,47 +49,48 @@ typedef enum {
  */
 
 struct mca_btl_base_endpoint_t {
-    opal_list_item_t                super;
-    struct mca_btl_tcp_module_t*    endpoint_btl;          /**< BTL instance that created this connection */
-    struct mca_btl_tcp_proc_t*      endpoint_proc;         /**< proc structure corresponding to endpoint */
-    struct mca_btl_tcp_addr_t*      endpoint_addr;         /**< address of endpoint */
-    int                             endpoint_sd;           /**< socket connection to endpoint */
-    int                             endpoint_sd_next;      /**< deadlock avoidance: socket connection to endpoint to set once the endpoint_sd has been correctly closed */
+    opal_list_item_t super;
+    struct mca_btl_tcp_module_t *endpoint_btl; /**< BTL instance that created this connection */
+    struct mca_btl_tcp_proc_t *endpoint_proc;  /**< proc structure corresponding to endpoint */
+    struct mca_btl_tcp_addr_t *endpoint_addr;  /**< address of endpoint */
+    int endpoint_sd;                           /**< socket connection to endpoint */
+    int endpoint_sd_next; /**< deadlock avoidance: socket connection to endpoint to set once the
+                             endpoint_sd has been correctly closed */
 #if MCA_BTL_TCP_ENDPOINT_CACHE
-    char*                           endpoint_cache;        /**< cache for the recv (reduce the number of recv syscall) */
-    char*                           endpoint_cache_pos;    /**< current position in the cache */
-    size_t                          endpoint_cache_length; /**< length of the data in the cache */
-#endif  /* MCA_BTL_TCP_ENDPOINT_CACHE */
-    struct mca_btl_tcp_frag_t*      endpoint_send_frag;    /**< current send frag being processed */
-    struct mca_btl_tcp_frag_t*      endpoint_recv_frag;    /**< current recv frag being processed */
-    mca_btl_tcp_state_t             endpoint_state;        /**< current state of the connection */
-    uint32_t                        endpoint_retries;      /**< number of connection retries attempted */
-    opal_list_t                     endpoint_frags;        /**< list of pending frags to send */
-    opal_mutex_t                    endpoint_send_lock;    /**< lock for concurrent access to endpoint state */
-    opal_mutex_t                    endpoint_recv_lock;    /**< lock for concurrent access to endpoint state */
-    opal_event_t                    endpoint_accept_event;   /**< event for async processing of accept requests */
-    opal_event_t                    endpoint_send_event;   /**< event for async processing of send frags */
-    opal_event_t                    endpoint_recv_event;   /**< event for async processing of recv frags */
-    bool                            endpoint_nbo;          /**< convert headers to network byte order? */
+    char *endpoint_cache;         /**< cache for the recv (reduce the number of recv syscall) */
+    char *endpoint_cache_pos;     /**< current position in the cache */
+    size_t endpoint_cache_length; /**< length of the data in the cache */
+#endif /* MCA_BTL_TCP_ENDPOINT_CACHE */
+    struct mca_btl_tcp_frag_t *endpoint_send_frag; /**< current send frag being processed */
+    struct mca_btl_tcp_frag_t *endpoint_recv_frag; /**< current recv frag being processed */
+    mca_btl_tcp_state_t endpoint_state;            /**< current state of the connection */
+    uint32_t endpoint_retries;                     /**< number of connection retries attempted */
+    opal_list_t endpoint_frags;                    /**< list of pending frags to send */
+    opal_mutex_t endpoint_send_lock;    /**< lock for concurrent access to endpoint state */
+    opal_mutex_t endpoint_recv_lock;    /**< lock for concurrent access to endpoint state */
+    opal_event_t endpoint_accept_event; /**< event for async processing of accept requests */
+    opal_event_t endpoint_send_event;   /**< event for async processing of send frags */
+    opal_event_t endpoint_recv_event;   /**< event for async processing of recv frags */
+    bool endpoint_nbo;                  /**< convert headers to network byte order? */
 };
 
 typedef struct mca_btl_base_endpoint_t mca_btl_base_endpoint_t;
-typedef mca_btl_base_endpoint_t  mca_btl_tcp_endpoint_t;
+typedef mca_btl_base_endpoint_t mca_btl_tcp_endpoint_t;
 OBJ_CLASS_DECLARATION(mca_btl_tcp_endpoint_t);
 
 /* Magic socket handshake string */
 extern const char mca_btl_tcp_magic_id_string[MCA_BTL_TCP_MAGIC_STRING_LENGTH];
 
-typedef struct { 
+typedef struct {
     opal_process_name_t guid;
     char magic_id[MCA_BTL_TCP_MAGIC_STRING_LENGTH];
 } mca_btl_tcp_endpoint_hs_msg_t;
 
 void mca_btl_tcp_set_socket_options(int sd);
-void mca_btl_tcp_endpoint_close(mca_btl_base_endpoint_t*);
-int  mca_btl_tcp_endpoint_send(mca_btl_base_endpoint_t*, struct mca_btl_tcp_frag_t*);
-void mca_btl_tcp_endpoint_accept(mca_btl_base_endpoint_t*, struct sockaddr*, int);
-void mca_btl_tcp_endpoint_shutdown(mca_btl_base_endpoint_t*);
+void mca_btl_tcp_endpoint_close(mca_btl_base_endpoint_t *);
+int mca_btl_tcp_endpoint_send(mca_btl_base_endpoint_t *, struct mca_btl_tcp_frag_t *);
+void mca_btl_tcp_endpoint_accept(mca_btl_base_endpoint_t *, struct sockaddr *, int);
+void mca_btl_tcp_endpoint_shutdown(mca_btl_base_endpoint_t *);
 
 /*
  * Diagnostics: change this to "1" to enable the function
@@ -98,12 +99,14 @@ void mca_btl_tcp_endpoint_shutdown(mca_btl_base_endpoint_t*);
 #define WANT_PEER_DUMP 0
 
 #if OPAL_ENABLE_DEBUG && WANT_PEER_DUMP
-#define MCA_BTL_TCP_ENDPOINT_DUMP(LEVEL, ENDPOINT, INFO, MSG) mca_btl_tcp_endpoint_dump((LEVEL), __FILE__, __LINE__, __func__, (ENDPOINT), (INFO), (MSG))
-void mca_btl_tcp_endpoint_dump(int level, const char* fname, int lineno, const char* funcname,
-                               mca_btl_base_endpoint_t* btl_endpoint, bool full_info, const char* msg);
+#    define MCA_BTL_TCP_ENDPOINT_DUMP(LEVEL, ENDPOINT, INFO, MSG) \
+        mca_btl_tcp_endpoint_dump((LEVEL), __FILE__, __LINE__, __func__, (ENDPOINT), (INFO), (MSG))
+void mca_btl_tcp_endpoint_dump(int level, const char *fname, int lineno, const char *funcname,
+                               mca_btl_base_endpoint_t *btl_endpoint, bool full_info,
+                               const char *msg);
 #else
-#define MCA_BTL_TCP_ENDPOINT_DUMP(LEVEL, ENDPOINT, INFO, MSG)
-#endif  /* OPAL_ENABLE_DEBUG && WANT_PEER_DUMP */
+#    define MCA_BTL_TCP_ENDPOINT_DUMP(LEVEL, ENDPOINT, INFO, MSG)
+#endif /* OPAL_ENABLE_DEBUG && WANT_PEER_DUMP */
 
 END_C_DECLS
 
diff --git a/opal/mca/btl/tcp/btl_tcp_frag.c b/opal/mca/btl/tcp/btl_tcp_frag.c
index 6e0ead7a386..36c01537895 100644
--- a/opal/mca/btl/tcp/btl_tcp_frag.c
+++ b/opal/mca/btl/tcp/btl_tcp_frag.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
+ * Copyright (c) 2004-2023 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -31,165 +31,161 @@
 #include "opal_config.h"
 
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 #ifdef HAVE_SYS_UIO_H
-#include <sys/uio.h>
+#    include <sys/uio.h>
 #endif
 #ifdef HAVE_NET_UIO_H
-#include <net/uio.h>
+#    include <net/uio.h>
 #endif
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif  /* HAVE_UNISTD_H */
+#    include <unistd.h>
+#endif /* HAVE_UNISTD_H */
 
-#include "opal/opal_socket_errno.h"
 #include "opal/mca/btl/base/btl_base_error.h"
+#include "opal/opal_socket_errno.h"
 #include "opal/util/proc.h"
 #include "opal/util/show_help.h"
 
-#include "btl_tcp_frag.h"
-#include "btl_tcp_proc.h"
 #include "btl_tcp_endpoint.h"
+#include "btl_tcp_frag.h"
 #include "btl_tcp_proc.h"
 
-
-static void mca_btl_tcp_frag_eager_constructor(mca_btl_tcp_frag_t* frag)
+static void mca_btl_tcp_frag_eager_constructor(mca_btl_tcp_frag_t *frag)
 {
     frag->size = mca_btl_tcp_module.super.btl_eager_limit;
     frag->my_list = &mca_btl_tcp_component.tcp_frag_eager;
 }
 
-static void mca_btl_tcp_frag_max_constructor(mca_btl_tcp_frag_t* frag)
+static void mca_btl_tcp_frag_max_constructor(mca_btl_tcp_frag_t *frag)
 {
     frag->size = mca_btl_tcp_module.super.btl_max_send_size;
     frag->my_list = &mca_btl_tcp_component.tcp_frag_max;
 }
 
-static void mca_btl_tcp_frag_user_constructor(mca_btl_tcp_frag_t* frag)
+static void mca_btl_tcp_frag_user_constructor(mca_btl_tcp_frag_t *frag)
 {
     frag->size = 0;
     frag->my_list = &mca_btl_tcp_component.tcp_frag_user;
 }
 
+OBJ_CLASS_INSTANCE(mca_btl_tcp_frag_t, mca_btl_base_descriptor_t, NULL, NULL);
 
-OBJ_CLASS_INSTANCE(
-    mca_btl_tcp_frag_t,
-    mca_btl_base_descriptor_t,
-    NULL,
-    NULL);
+OBJ_CLASS_INSTANCE(mca_btl_tcp_frag_eager_t, mca_btl_base_descriptor_t,
+                   mca_btl_tcp_frag_eager_constructor, NULL);
 
-OBJ_CLASS_INSTANCE(
-    mca_btl_tcp_frag_eager_t,
-    mca_btl_base_descriptor_t,
-    mca_btl_tcp_frag_eager_constructor,
-    NULL);
+OBJ_CLASS_INSTANCE(mca_btl_tcp_frag_max_t, mca_btl_base_descriptor_t,
+                   mca_btl_tcp_frag_max_constructor, NULL);
 
-OBJ_CLASS_INSTANCE(
-    mca_btl_tcp_frag_max_t,
-    mca_btl_base_descriptor_t,
-    mca_btl_tcp_frag_max_constructor,
-    NULL);
+OBJ_CLASS_INSTANCE(mca_btl_tcp_frag_user_t, mca_btl_base_descriptor_t,
+                   mca_btl_tcp_frag_user_constructor, NULL);
 
-OBJ_CLASS_INSTANCE(
-    mca_btl_tcp_frag_user_t,
-    mca_btl_base_descriptor_t,
-    mca_btl_tcp_frag_user_constructor,
-    NULL);
-
-size_t mca_btl_tcp_frag_dump(mca_btl_tcp_frag_t* frag, char* msg, char* buf, size_t length)
+size_t mca_btl_tcp_frag_dump(mca_btl_tcp_frag_t *frag, char *msg, char *buf, size_t length)
 {
     int i, used;
 
-    used = snprintf(buf, length, "%s frag %p iov_cnt %d iov_idx %d size %lu\n",
-                    msg, (void*)frag, (int)frag->iov_cnt, (int)frag->iov_idx, frag->size);
-    if ((size_t)used >= length) return length;
-    for( i = 0; i < (int)frag->iov_cnt; i++ ) {
+    used = snprintf(buf, length, "%s frag %p iov_cnt %d iov_idx %d size %lu\n", msg, (void *) frag,
+                    (int) frag->iov_cnt, (int) frag->iov_idx, frag->size);
+    if ((size_t) used >= length) {
+        return length;
+    }
+    for (i = 0; i < (int) frag->iov_cnt; i++) {
         used += snprintf(&buf[used], length - used, "[%s%p:%lu] ",
-                         (i < (int)frag->iov_idx ? "*" : ""),
-                         frag->iov[i].iov_base, frag->iov[i].iov_len);
-        if ((size_t)used >= length) return length;
+                         (i < (int) frag->iov_idx ? "*" : ""), frag->iov[i].iov_base,
+                         frag->iov[i].iov_len);
+        if ((size_t) used >= length) {
+            return length;
+        }
     }
     return used;
 }
 
-bool mca_btl_tcp_frag_send(mca_btl_tcp_frag_t* frag, int sd)
+bool mca_btl_tcp_frag_send(mca_btl_tcp_frag_t *frag, int sd)
 {
     ssize_t cnt;
     size_t i, num_vecs;
+    struct msghdr msg = {
+                     .msg_iov = frag->iov_ptr,
+                     .msg_iovlen = frag->iov_cnt };
+    int msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL;
 
-    /* non-blocking write, but continue if interrupted */
+    /* non-blocking write, continue if interrupted */
     do {
-        cnt = writev(sd, frag->iov_ptr, frag->iov_cnt);
-        if(cnt < 0) {
-            switch(opal_socket_errno) {
+        /* Use sendmsg to avoid issues with SIGPIPE as described in
+         * https://blog.erratasec.com/2018/10/tcpip-sockets-and-sigpipe.html#
+         */
+        cnt = sendmsg(sd, &msg, msg_flags);
+        if (cnt < 0) {
+            switch (opal_socket_errno) {
             case EINTR:
                 continue;
             case EWOULDBLOCK:
                 return false;
             case EFAULT:
-                BTL_ERROR(("mca_btl_tcp_frag_send: writev error (%p, %lu)\n\t%s(%lu)\n",
-                    frag->iov_ptr[0].iov_base, (unsigned long) frag->iov_ptr[0].iov_len,
-                    strerror(opal_socket_errno), (unsigned long) frag->iov_cnt));
+                BTL_ERROR(("mca_btl_tcp_frag_send: sendmsg error (%p, %lu)\n\t%s(%lu)\n",
+                           frag->iov_ptr[0].iov_base, (unsigned long) frag->iov_ptr[0].iov_len,
+                           strerror(opal_socket_errno), (unsigned long) frag->iov_cnt));
                 /* send_lock held by caller */
                 frag->endpoint->endpoint_state = MCA_BTL_TCP_FAILED;
                 mca_btl_tcp_endpoint_close(frag->endpoint);
                 return false;
             default:
-                BTL_PEER_ERROR(frag->endpoint->endpoint_proc->proc_opal, ("mca_btl_tcp_frag_send: writev failed: %s (%d)",
-                           strerror(opal_socket_errno),
-                           opal_socket_errno));
+                BTL_PEER_ERROR(frag->endpoint->endpoint_proc->proc_opal,
+                               ("mca_btl_tcp_frag_send: sendmsg failed: %s (%d)",
+                                strerror(opal_socket_errno), opal_socket_errno));
                 /* send_lock held by caller */
                 frag->endpoint->endpoint_state = MCA_BTL_TCP_FAILED;
                 mca_btl_tcp_endpoint_close(frag->endpoint);
                 return false;
             }
         }
-    } while(cnt < 0);
+    } while (cnt < 0);
 
     /* if the write didn't complete - update the iovec state */
     num_vecs = frag->iov_cnt;
-    for( i = 0; i < num_vecs; i++) {
-        if(cnt >= (ssize_t)frag->iov_ptr->iov_len) {
+    for (i = 0; i < num_vecs; i++) {
+        if (cnt >= (ssize_t) frag->iov_ptr->iov_len) {
             cnt -= frag->iov_ptr->iov_len;
             frag->iov_ptr++;
             frag->iov_idx++;
             frag->iov_cnt--;
         } else {
-            frag->iov_ptr->iov_base = (opal_iov_base_ptr_t)
-                (((unsigned char*)frag->iov_ptr->iov_base) + cnt);
+            frag->iov_ptr->iov_base = (opal_iov_base_ptr_t)(
+                ((unsigned char *) frag->iov_ptr->iov_base) + cnt);
             frag->iov_ptr->iov_len -= cnt;
             OPAL_OUTPUT_VERBOSE((100, opal_btl_base_framework.framework_output,
-                                 "%s:%d write %ld bytes on socket %d\n",
-                                 __FILE__, __LINE__, cnt, sd));
+                                 "%s:%d write %ld bytes on socket %d\n", __FILE__, __LINE__, cnt,
+                                 sd));
             break;
         }
     }
     return (frag->iov_cnt == 0);
 }
 
-bool mca_btl_tcp_frag_recv(mca_btl_tcp_frag_t* frag, int sd)
+bool mca_btl_tcp_frag_recv(mca_btl_tcp_frag_t *frag, int sd)
 {
-    mca_btl_base_endpoint_t* btl_endpoint = frag->endpoint;
+    mca_btl_base_endpoint_t *btl_endpoint = frag->endpoint;
     ssize_t cnt;
     int32_t i, num_vecs, dont_copy_data = 0;
     char *errhost;
 
- repeat:
+repeat:
     num_vecs = frag->iov_cnt;
 #if MCA_BTL_TCP_ENDPOINT_CACHE
-    if( 0 != btl_endpoint->endpoint_cache_length ) {
+    if (0 != btl_endpoint->endpoint_cache_length) {
         size_t length;
         /* It's strange at the first look but cnt have to be set to the full amount of data
          * available. After going to advance_iov_position we will use cnt to detect if there
          * is still some data pending.
          */
         cnt = length = btl_endpoint->endpoint_cache_length;
-        for( i = 0; i < (int)frag->iov_cnt; i++ ) {
-            if( length > frag->iov_ptr[i].iov_len )
+        for (i = 0; i < (int) frag->iov_cnt; i++) {
+            if (length > frag->iov_ptr[i].iov_len) {
                 length = frag->iov_ptr[i].iov_len;
-            if( (0 == dont_copy_data) || (length < frag->iov_ptr[i].iov_len) ) {
-                memcpy( frag->iov_ptr[i].iov_base, btl_endpoint->endpoint_cache_pos, length );
+            }
+            if ((0 == dont_copy_data) || (length < frag->iov_ptr[i].iov_len)) {
+                memcpy(frag->iov_ptr[i].iov_base, btl_endpoint->endpoint_cache_pos, length);
             } else {
                 frag->segments[0].seg_addr.pval = btl_endpoint->endpoint_cache_pos;
                 frag->iov_ptr[i].iov_base = btl_endpoint->endpoint_cache_pos;
@@ -197,7 +193,7 @@ bool mca_btl_tcp_frag_recv(mca_btl_tcp_frag_t* frag, int sd)
             btl_endpoint->endpoint_cache_pos += length;
             btl_endpoint->endpoint_cache_length -= length;
             length = btl_endpoint->endpoint_cache_length;
-            if( 0 == length ) {
+            if (0 == length) {
                 btl_endpoint->endpoint_cache_pos = btl_endpoint->endpoint_cache;
                 break;
             }
@@ -208,24 +204,27 @@ bool mca_btl_tcp_frag_recv(mca_btl_tcp_frag_t* frag, int sd)
      * iovec for the caching in the fragment structure (the +1).
      */
     frag->iov_ptr[num_vecs].iov_base = btl_endpoint->endpoint_cache_pos;
-    frag->iov_ptr[num_vecs].iov_len  =
-        mca_btl_tcp_component.tcp_endpoint_cache - btl_endpoint->endpoint_cache_length;
+    frag->iov_ptr[num_vecs].iov_len = mca_btl_tcp_component.tcp_endpoint_cache
+                                      - btl_endpoint->endpoint_cache_length;
     num_vecs++;
-#endif  /* MCA_BTL_TCP_ENDPOINT_CACHE */
+#endif /* MCA_BTL_TCP_ENDPOINT_CACHE */
 
     /* non-blocking read, but continue if interrupted */
     do {
         cnt = readv(sd, frag->iov_ptr, num_vecs);
-        if( 0 < cnt ) goto advance_iov_position;
-        if( cnt == 0 ) {
+        if (0 < cnt) {
+            goto advance_iov_position;
+        }
+        if (cnt == 0) {
             OPAL_THREAD_LOCK(&btl_endpoint->endpoint_send_lock);
-            if(MCA_BTL_TCP_CONNECTED == btl_endpoint->endpoint_state)
+            if (MCA_BTL_TCP_CONNECTED == btl_endpoint->endpoint_state) {
                 btl_endpoint->endpoint_state = MCA_BTL_TCP_FAILED;
+            }
             mca_btl_tcp_endpoint_close(btl_endpoint);
             OPAL_THREAD_UNLOCK(&btl_endpoint->endpoint_send_lock);
             return false;
         }
-        switch(opal_socket_errno) {
+        switch (opal_socket_errno) {
         case EINTR:
             continue;
         case EWOULDBLOCK:
@@ -236,19 +235,17 @@ bool mca_btl_tcp_frag_recv(mca_btl_tcp_frag_t* frag, int sd)
                        strerror(opal_socket_errno), (unsigned long) frag->iov_cnt));
             break;
         case ECONNRESET:
-            if( mca_btl_base_warn_peer_error
-             || mca_btl_base_verbose > 0 ) {
+            if (mca_btl_base_warn_peer_error || mca_btl_base_verbose > 0) {
                 errhost = opal_get_proc_hostname(btl_endpoint->endpoint_proc->proc_opal);
-                opal_show_help("help-mpi-btl-tcp.txt", "peer hung up",
-                           true, opal_process_info.nodename,
-                           getpid(), errhost);
+                opal_show_help("help-mpi-btl-tcp.txt", "peer hung up", true,
+                               opal_process_info.nodename, getpid(), errhost);
                 free(errhost);
             }
             break;
         default:
-            BTL_PEER_ERROR(frag->endpoint->endpoint_proc->proc_opal, ("mca_btl_tcp_frag_recv: readv failed: %s (%d)",
-                       strerror(opal_socket_errno),
-                       opal_socket_errno));
+            BTL_PEER_ERROR(frag->endpoint->endpoint_proc->proc_opal,
+                           ("mca_btl_tcp_frag_recv: readv failed: %s (%d)",
+                            strerror(opal_socket_errno), opal_socket_errno));
             break;
         }
         OPAL_THREAD_LOCK(&btl_endpoint->endpoint_send_lock);
@@ -256,15 +253,15 @@ bool mca_btl_tcp_frag_recv(mca_btl_tcp_frag_t* frag, int sd)
         mca_btl_tcp_endpoint_close(btl_endpoint);
         OPAL_THREAD_UNLOCK(&btl_endpoint->endpoint_send_lock);
         return false;
-    } while( cnt < 0 );
+    } while (cnt < 0);
 
- advance_iov_position:
+advance_iov_position:
     /* if the read didn't complete - update the iovec state */
     num_vecs = frag->iov_cnt;
-    for( i = 0; i < num_vecs; i++ ) {
-        if( cnt < (ssize_t)frag->iov_ptr->iov_len ) {
-            frag->iov_ptr->iov_base = (opal_iov_base_ptr_t)
-                (((unsigned char*)frag->iov_ptr->iov_base) + cnt);
+    for (i = 0; i < num_vecs; i++) {
+        if (cnt < (ssize_t) frag->iov_ptr->iov_len) {
+            frag->iov_ptr->iov_base = (opal_iov_base_ptr_t)(
+                ((unsigned char *) frag->iov_ptr->iov_base) + cnt);
             frag->iov_ptr->iov_len -= cnt;
             cnt = 0;
             break;
@@ -276,37 +273,41 @@ bool mca_btl_tcp_frag_recv(mca_btl_tcp_frag_t* frag, int sd)
     }
 #if MCA_BTL_TCP_ENDPOINT_CACHE
     btl_endpoint->endpoint_cache_length = cnt;
-#endif  /* MCA_BTL_TCP_ENDPOINT_CACHE */
+#endif /* MCA_BTL_TCP_ENDPOINT_CACHE */
 
     /* read header */
-    if(frag->iov_cnt == 0) {
-        if (btl_endpoint->endpoint_nbo && frag->iov_idx == 1) MCA_BTL_TCP_HDR_NTOH(frag->hdr);
-        switch(frag->hdr.type) {
+    if (frag->iov_cnt == 0) {
+        if (btl_endpoint->endpoint_nbo && frag->iov_idx == 1) {
+            MCA_BTL_TCP_HDR_NTOH(frag->hdr);
+        }
+        switch (frag->hdr.type) {
         case MCA_BTL_TCP_HDR_TYPE_FIN:
             frag->endpoint->endpoint_state = MCA_BTL_TCP_CLOSED;
             mca_btl_tcp_endpoint_close(frag->endpoint);
             break;
         case MCA_BTL_TCP_HDR_TYPE_SEND:
-            if(frag->iov_idx == 1 && frag->hdr.size) {
-                frag->segments[0].seg_addr.pval = frag+1;
+            if (frag->iov_idx == 1 && frag->hdr.size) {
+                frag->segments[0].seg_addr.pval = frag + 1;
                 frag->segments[0].seg_len = frag->hdr.size;
-                frag->iov[1].iov_base = (IOVBASE_TYPE*)(frag->segments[0].seg_addr.pval);
+                frag->iov[1].iov_base = (IOVBASE_TYPE *) (frag->segments[0].seg_addr.pval);
                 frag->iov[1].iov_len = frag->hdr.size;
                 frag->iov_cnt++;
                 goto repeat;
             }
             break;
         case MCA_BTL_TCP_HDR_TYPE_PUT:
-            if(frag->iov_idx == 1) {
-                frag->iov[1].iov_base = (IOVBASE_TYPE*)frag->segments;
+            if (frag->iov_idx == 1) {
+                frag->iov[1].iov_base = (IOVBASE_TYPE *) frag->segments;
                 frag->iov[1].iov_len = frag->hdr.count * sizeof(mca_btl_base_segment_t);
                 frag->iov_cnt++;
                 goto repeat;
             } else if (frag->iov_idx == 2) {
-                for( i = 0; i < frag->hdr.count; i++ ) {
-                    if (btl_endpoint->endpoint_nbo) MCA_BTL_BASE_SEGMENT_NTOH(frag->segments[i]);
-                    frag->iov[i+2].iov_base = (IOVBASE_TYPE*)frag->segments[i].seg_addr.pval;
-                    frag->iov[i+2].iov_len = frag->segments[i].seg_len;
+                for (i = 0; i < frag->hdr.count; i++) {
+                    if (btl_endpoint->endpoint_nbo) {
+                        MCA_BTL_BASE_SEGMENT_NTOH(frag->segments[i]);
+                    }
+                    frag->iov[i + 2].iov_base = (IOVBASE_TYPE *) frag->segments[i].seg_addr.pval;
+                    frag->iov[i + 2].iov_len = frag->segments[i].seg_len;
                 }
                 frag->iov_cnt += frag->hdr.count;
                 goto repeat;
@@ -320,4 +321,3 @@ bool mca_btl_tcp_frag_recv(mca_btl_tcp_frag_t* frag, int sd)
     }
     return false;
 }
-
diff --git a/opal/mca/btl/tcp/btl_tcp_frag.h b/opal/mca/btl/tcp/btl_tcp_frag.h
index e1b068502f9..d26abbfbe08 100644
--- a/opal/mca/btl/tcp/btl_tcp_frag.h
+++ b/opal/mca/btl/tcp/btl_tcp_frag.h
@@ -22,17 +22,16 @@
 #ifndef MCA_BTL_TCP_FRAG_H
 #define MCA_BTL_TCP_FRAG_H
 
-
 #include "opal_config.h"
 
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 #ifdef HAVE_SYS_UIO_H
-#include <sys/uio.h>
+#    include <sys/uio.h>
 #endif
 #ifdef HAVE_NET_UIO_H
-#include <net/uio.h>
+#    include <net/uio.h>
 #endif
 
 #include "btl_tcp.h"
@@ -40,7 +39,7 @@
 
 BEGIN_C_DECLS
 
-#define MCA_BTL_TCP_FRAG_IOVEC_NUMBER  4
+#define MCA_BTL_TCP_FRAG_IOVEC_NUMBER 4
 
 /**
  * TCP fragment derived type.
@@ -49,7 +48,7 @@ struct mca_btl_tcp_frag_t {
     mca_btl_base_descriptor_t base;
     mca_btl_base_segment_t segments[2];
     struct mca_btl_base_endpoint_t *endpoint;
-    struct mca_btl_tcp_module_t* btl;
+    struct mca_btl_tcp_module_t *btl;
     mca_btl_tcp_hdr_t hdr;
     struct iovec iov[MCA_BTL_TCP_FRAG_IOVEC_NUMBER + 1];
     struct iovec *iov_ptr;
@@ -58,7 +57,7 @@ struct mca_btl_tcp_frag_t {
     size_t size;
     uint16_t next_step;
     int rc;
-    opal_free_list_t* my_list;
+    opal_free_list_t *my_list;
     /* fake rdma completion */
     struct {
         mca_btl_base_rdma_completion_fn_t func;
@@ -81,52 +80,47 @@ typedef struct mca_btl_tcp_frag_t mca_btl_tcp_frag_user_t;
 
 OBJ_CLASS_DECLARATION(mca_btl_tcp_frag_user_t);
 
-
 /*
  * Macros to allocate/return descriptors from module specific
  * free list(s).
  */
 
-#define MCA_BTL_TCP_FRAG_ALLOC_EAGER(frag)                              \
-{                                                                       \
-    frag = (mca_btl_tcp_frag_t*)                                        \
-        opal_free_list_get (&mca_btl_tcp_component.tcp_frag_eager);     \
-}
-
-#define MCA_BTL_TCP_FRAG_ALLOC_MAX(frag)                                \
-{                                                                       \
-    frag = (mca_btl_tcp_frag_t*)                                        \
-        opal_free_list_get (&mca_btl_tcp_component.tcp_frag_max);       \
-}
-
-#define MCA_BTL_TCP_FRAG_ALLOC_USER(frag)                               \
-{                                                                       \
-    frag = (mca_btl_tcp_frag_t*)                                        \
-        opal_free_list_get (&mca_btl_tcp_component.tcp_frag_user);      \
-}
-
-#define MCA_BTL_TCP_FRAG_RETURN(frag)                                      \
-{                                                                          \
-    opal_free_list_return (frag->my_list, (opal_free_list_item_t*)(frag)); \
-}
-
-#define MCA_BTL_TCP_FRAG_INIT_DST(frag,ep)                                 \
-do {                                                                       \
-    frag->rc = 0;                                                          \
-    frag->btl = ep->endpoint_btl;                                          \
-    frag->endpoint = ep;                                                   \
-    frag->iov[0].iov_len = sizeof(frag->hdr);                              \
-    frag->iov[0].iov_base = (IOVBASE_TYPE*)&frag->hdr;                     \
-    frag->iov_cnt = 1;                                                     \
-    frag->iov_idx = 0;                                                     \
-    frag->iov_ptr = frag->iov;                                             \
-    frag->base.des_segments = frag->segments;                                 \
-    frag->base.des_segment_count = 1;                                        \
-} while(0)
-
-
-bool mca_btl_tcp_frag_send(mca_btl_tcp_frag_t*, int sd);
-bool mca_btl_tcp_frag_recv(mca_btl_tcp_frag_t*, int sd);
-size_t mca_btl_tcp_frag_dump(mca_btl_tcp_frag_t* frag, char* msg, char* buf, size_t length);
+#define MCA_BTL_TCP_FRAG_ALLOC_EAGER(frag)                                                       \
+    {                                                                                            \
+        frag = (mca_btl_tcp_frag_t *) opal_free_list_get(&mca_btl_tcp_component.tcp_frag_eager); \
+    }
+
+#define MCA_BTL_TCP_FRAG_ALLOC_MAX(frag)                                                       \
+    {                                                                                          \
+        frag = (mca_btl_tcp_frag_t *) opal_free_list_get(&mca_btl_tcp_component.tcp_frag_max); \
+    }
+
+#define MCA_BTL_TCP_FRAG_ALLOC_USER(frag)                                                       \
+    {                                                                                           \
+        frag = (mca_btl_tcp_frag_t *) opal_free_list_get(&mca_btl_tcp_component.tcp_frag_user); \
+    }
+
+#define MCA_BTL_TCP_FRAG_RETURN(frag)                                           \
+    {                                                                           \
+        opal_free_list_return(frag->my_list, (opal_free_list_item_t *) (frag)); \
+    }
+
+#define MCA_BTL_TCP_FRAG_INIT_DST(frag, ep)                  \
+    do {                                                     \
+        frag->rc = 0;                                        \
+        frag->btl = ep->endpoint_btl;                        \
+        frag->endpoint = ep;                                 \
+        frag->iov[0].iov_len = sizeof(frag->hdr);            \
+        frag->iov[0].iov_base = (IOVBASE_TYPE *) &frag->hdr; \
+        frag->iov_cnt = 1;                                   \
+        frag->iov_idx = 0;                                   \
+        frag->iov_ptr = frag->iov;                           \
+        frag->base.des_segments = frag->segments;            \
+        frag->base.des_segment_count = 1;                    \
+    } while (0)
+
+bool mca_btl_tcp_frag_send(mca_btl_tcp_frag_t *, int sd);
+bool mca_btl_tcp_frag_recv(mca_btl_tcp_frag_t *, int sd);
+size_t mca_btl_tcp_frag_dump(mca_btl_tcp_frag_t *frag, char *msg, char *buf, size_t length);
 END_C_DECLS
 #endif
diff --git a/opal/mca/btl/tcp/btl_tcp_hdr.h b/opal/mca/btl/tcp/btl_tcp_hdr.h
index cd907cb3b99..69ec22f6fdf 100644
--- a/opal/mca/btl/tcp/btl_tcp_hdr.h
+++ b/opal/mca/btl/tcp/btl_tcp_hdr.h
@@ -19,10 +19,9 @@
 #ifndef MCA_BTL_TCP_HDR_H
 #define MCA_BTL_TCP_HDR_H
 
-
 #include "opal_config.h"
-#include "opal/mca/btl/base/base.h"
 #include "btl_tcp.h"
+#include "opal/mca/btl/base/base.h"
 
 BEGIN_C_DECLS
 
@@ -48,7 +47,7 @@ BEGIN_C_DECLS
 
 struct mca_btl_tcp_hdr_t {
     mca_btl_base_header_t base;
-    uint8_t  type;
+    uint8_t type;
     uint16_t count;
     uint32_t size;
 };
diff --git a/opal/mca/btl/tcp/btl_tcp_proc.c b/opal/mca/btl/tcp/btl_tcp_proc.c
index 20c7e22d85b..fc038fde92a 100644
--- a/opal/mca/btl/tcp/btl_tcp_proc.c
+++ b/opal/mca/btl/tcp/btl_tcp_proc.c
@@ -31,44 +31,41 @@
 #include "opal_config.h"
 
 #ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
+#    include <netinet/in.h>
 #endif
 #ifdef HAVE_ARPA_INET_H
-#include <arpa/inet.h>
+#    include <arpa/inet.h>
 #endif
 
 #include "opal/class/opal_hash_table.h"
 #include "opal/mca/btl/base/btl_base_error.h"
-#include "opal/mca/reachable/base/base.h"
 #include "opal/mca/pmix/pmix-internal.h"
+#include "opal/mca/reachable/base/base.h"
 #include "opal/util/arch.h"
 #include "opal/util/argv.h"
+#include "opal/util/bipartite_graph.h"
 #include "opal/util/if.h"
 #include "opal/util/net.h"
-#include "opal/util/proc.h"
-#include "opal/util/show_help.h"
 #include "opal/util/printf.h"
 #include "opal/util/proc.h"
+#include "opal/util/show_help.h"
 #include "opal/util/string_copy.h"
-#include "opal/util/bipartite_graph.h"
 
 #include "btl_tcp.h"
 #include "btl_tcp_proc.h"
 
-static void mca_btl_tcp_proc_construct(mca_btl_tcp_proc_t* proc);
-static void mca_btl_tcp_proc_destruct(mca_btl_tcp_proc_t* proc);
+static void mca_btl_tcp_proc_construct(mca_btl_tcp_proc_t *proc);
+static void mca_btl_tcp_proc_destruct(mca_btl_tcp_proc_t *proc);
 
-OBJ_CLASS_INSTANCE( mca_btl_tcp_proc_t,
-                    opal_list_item_t,
-                    mca_btl_tcp_proc_construct,
-                    mca_btl_tcp_proc_destruct );
+OBJ_CLASS_INSTANCE(mca_btl_tcp_proc_t, opal_list_item_t, mca_btl_tcp_proc_construct,
+                   mca_btl_tcp_proc_destruct);
 
-void mca_btl_tcp_proc_construct(mca_btl_tcp_proc_t* tcp_proc)
+void mca_btl_tcp_proc_construct(mca_btl_tcp_proc_t *tcp_proc)
 {
-    tcp_proc->proc_opal           = NULL;
-    tcp_proc->proc_addrs          = NULL;
-    tcp_proc->proc_addr_count     = 0;
-    tcp_proc->proc_endpoints      = NULL;
+    tcp_proc->proc_opal = NULL;
+    tcp_proc->proc_addrs = NULL;
+    tcp_proc->proc_addr_count = 0;
+    tcp_proc->proc_endpoints = NULL;
     tcp_proc->proc_endpoint_count = 0;
     OBJ_CONSTRUCT(&tcp_proc->proc_lock, opal_mutex_t);
     OBJ_CONSTRUCT(&tcp_proc->btl_index_to_endpoint, opal_hash_table_t);
@@ -79,9 +76,9 @@ void mca_btl_tcp_proc_construct(mca_btl_tcp_proc_t* tcp_proc)
  * Cleanup ib proc instance
  */
 
-void mca_btl_tcp_proc_destruct(mca_btl_tcp_proc_t* tcp_proc)
+void mca_btl_tcp_proc_destruct(mca_btl_tcp_proc_t *tcp_proc)
 {
-    if( NULL != tcp_proc->proc_opal ) {
+    if (NULL != tcp_proc->proc_opal) {
         /* remove from list of all proc instances */
         OPAL_THREAD_LOCK(&mca_btl_tcp_component.tcp_lock);
         opal_proc_table_remove_value(&mca_btl_tcp_component.tcp_procs,
@@ -91,18 +88,17 @@ void mca_btl_tcp_proc_destruct(mca_btl_tcp_proc_t* tcp_proc)
         tcp_proc->proc_opal = NULL;
     }
     /* release resources */
-    if(NULL != tcp_proc->proc_endpoints) {
+    if (NULL != tcp_proc->proc_endpoints) {
         free(tcp_proc->proc_endpoints);
     }
-    if(NULL != tcp_proc->proc_addrs) {
+    if (NULL != tcp_proc->proc_addrs) {
         free(tcp_proc->proc_addrs);
     }
     OBJ_DESTRUCT(&tcp_proc->btl_index_to_endpoint);
     OBJ_DESTRUCT(&tcp_proc->proc_lock);
 }
 
-static inline int mca_btl_tcp_proc_is_proc_left(opal_process_name_t a,
-                                                opal_process_name_t b)
+static inline int mca_btl_tcp_proc_is_proc_left(opal_process_name_t a, opal_process_name_t b)
 {
     if (a.jobid != b.jobid) {
         return (a.jobid < b.jobid);
@@ -111,7 +107,7 @@ static inline int mca_btl_tcp_proc_is_proc_left(opal_process_name_t a,
     }
 }
 
-#define MCA_BTL_TCP_PROC_LOCAL_VERTEX(index) (index)
+#define MCA_BTL_TCP_PROC_LOCAL_VERTEX(index)  (index)
 #define MCA_BTL_TCP_PROC_REMOTE_VERTEX(index) (index + mca_btl_tcp_component.tcp_num_btls)
 
 /* This function builds a graph to match local and remote interfaces
@@ -142,8 +138,8 @@ static inline int mca_btl_tcp_proc_is_proc_left(opal_process_name_t a,
  * This allows us to pass the correct pointers to the vertex data for storage.
  *
  */
-static int mca_btl_tcp_proc_create_interface_graph(mca_btl_tcp_proc_t* btl_proc,
-                                                   mca_btl_tcp_modex_addr_t* remote_addrs,
+static int mca_btl_tcp_proc_create_interface_graph(mca_btl_tcp_proc_t *btl_proc,
+                                                   mca_btl_tcp_modex_addr_t *remote_addrs,
                                                    int local_proc_is_left,
                                                    opal_bp_graph_t **graph_out)
 {
@@ -162,7 +158,7 @@ static int mca_btl_tcp_proc_create_interface_graph(mca_btl_tcp_proc_t* btl_proc,
 
     /* the modex and proc structures differ slightly, so copy the
        fields needed in the proc version */
-    for (i = 0 ; i < btl_proc->proc_addr_count ; i++) {
+    for (i = 0; i < btl_proc->proc_addr_count; i++) {
         /* Construct opal_if_t objects for the remote interfaces */
         opal_if_t *interface = OBJ_NEW(opal_if_t);
         if (NULL == interface) {
@@ -171,32 +167,31 @@ static int mca_btl_tcp_proc_create_interface_graph(mca_btl_tcp_proc_t* btl_proc,
         }
 
         if (MCA_BTL_TCP_AF_INET == remote_addrs[i].addr_family) {
-            memcpy(&btl_proc->proc_addrs[i].addr_union.addr_inet,
-                   remote_addrs[i].addr, sizeof(struct in_addr));
+            memcpy(&btl_proc->proc_addrs[i].addr_union.addr_inet, remote_addrs[i].addr,
+                   sizeof(struct in_addr));
             btl_proc->proc_addrs[i].addr_family = AF_INET;
 
-            memcpy(&((struct sockaddr_in *)&(interface->if_addr))->sin_addr,
-                   remote_addrs[i].addr, sizeof(struct in_addr));
-           ((struct sockaddr *)&(interface->if_addr))->sa_family = AF_INET;
-           interface->af_family = AF_INET;
+            memcpy(&((struct sockaddr_in *) &(interface->if_addr))->sin_addr, remote_addrs[i].addr,
+                   sizeof(struct in_addr));
+            ((struct sockaddr *) &(interface->if_addr))->sa_family = AF_INET;
+            interface->af_family = AF_INET;
         } else if (MCA_BTL_TCP_AF_INET6 == remote_addrs[i].addr_family) {
 #if OPAL_ENABLE_IPV6
-            memcpy(&btl_proc->proc_addrs[i].addr_union.addr_inet6,
-                   remote_addrs[i].addr, sizeof(struct in6_addr));
+            memcpy(&btl_proc->proc_addrs[i].addr_union.addr_inet6, remote_addrs[i].addr,
+                   sizeof(struct in6_addr));
             btl_proc->proc_addrs[i].addr_family = AF_INET6;
 
-            memcpy(&((struct sockaddr_in6 *)&(interface->if_addr))->sin6_addr,
+            memcpy(&((struct sockaddr_in6 *) &(interface->if_addr))->sin6_addr,
                    remote_addrs[i].addr, sizeof(struct in6_addr));
-           ((struct sockaddr *)&(interface->if_addr))->sa_family = AF_INET6;
-           interface->af_family = AF_INET6;
+            ((struct sockaddr *) &(interface->if_addr))->sa_family = AF_INET6;
+            interface->af_family = AF_INET6;
 #else
             rc = OPAL_ERR_NOT_SUPPORTED;
             OBJ_RELEASE(interface);
             goto out;
 #endif
         } else {
-            BTL_ERROR(("Unexpected address family %d",
-                       (int)remote_addrs[i].addr_family));
+            BTL_ERROR(("Unexpected address family %d", (int) remote_addrs[i].addr_family));
             rc = OPAL_ERR_BAD_PARAM;
             OBJ_RELEASE(interface);
             goto out;
@@ -223,7 +218,8 @@ static int mca_btl_tcp_proc_create_interface_graph(mca_btl_tcp_proc_t* btl_proc,
 
     /* Add vertices for each local node. These will store the btl index */
     for (x = 0; x < results->num_local; x++) {
-        rc = opal_bp_graph_add_vertex(graph, &mca_btl_tcp_component.tcp_btls[x]->btl_index, &v_index);
+        rc = opal_bp_graph_add_vertex(graph, &mca_btl_tcp_component.tcp_btls[x]->btl_index,
+                                      &v_index);
         if (OPAL_SUCCESS != rc) {
             goto err_graph;
         }
@@ -292,45 +288,45 @@ static int mca_btl_tcp_proc_create_interface_graph(mca_btl_tcp_proc_t* btl_proc,
  * a pointer to a mca_btl_tcp_addr_t struct.
  */
 static int mca_btl_tcp_proc_store_matched_interfaces(mca_btl_tcp_proc_t *btl_proc,
-                                                     int local_proc_is_left,
-                                                     opal_bp_graph_t *graph,
+                                                     int local_proc_is_left, opal_bp_graph_t *graph,
                                                      int num_matched, int *matched_edges)
 {
     int rc = OPAL_SUCCESS;
     int i, left, right;
-    uint32_t* local_index;
+    uint32_t *local_index;
     struct mca_btl_tcp_addr_t *remote_addr;
 
     for (i = 0; i < num_matched; i++) {
-        left  = matched_edges[2 * i + 0];
+        left = matched_edges[2 * i + 0];
         right = matched_edges[2 * i + 1];
         if (local_proc_is_left) {
-            rc = opal_bp_graph_get_vertex_data(graph, left, (void *)&local_index);
+            rc = opal_bp_graph_get_vertex_data(graph, left, (void *) &local_index);
             if (OPAL_SUCCESS != rc) {
                 goto out;
             }
-            rc = opal_bp_graph_get_vertex_data(graph, right, (void *)&remote_addr);
+            rc = opal_bp_graph_get_vertex_data(graph, right, (void *) &remote_addr);
             if (OPAL_SUCCESS != rc) {
                 goto out;
             }
         } else {
-            rc = opal_bp_graph_get_vertex_data(graph, right, (void *)&local_index);
+            rc = opal_bp_graph_get_vertex_data(graph, right, (void *) &local_index);
             if (OPAL_SUCCESS != rc) {
                 goto out;
             }
-            rc = opal_bp_graph_get_vertex_data(graph, left, (void *)&remote_addr);
+            rc = opal_bp_graph_get_vertex_data(graph, left, (void *) &remote_addr);
             if (OPAL_SUCCESS != rc) {
                 goto out;
             }
         }
-        opal_hash_table_set_value_uint32(&btl_proc->btl_index_to_endpoint, *local_index, (void *)remote_addr);
+        opal_hash_table_set_value_uint32(&btl_proc->btl_index_to_endpoint, *local_index,
+                                         (void *) remote_addr);
     }
 out:
     return rc;
 }
 
-static int mca_btl_tcp_proc_handle_modex_addresses(mca_btl_tcp_proc_t* btl_proc,
-                                                   mca_btl_tcp_modex_addr_t* remote_addrs,
+static int mca_btl_tcp_proc_handle_modex_addresses(mca_btl_tcp_proc_t *btl_proc,
+                                                   mca_btl_tcp_modex_addr_t *remote_addrs,
                                                    int local_proc_is_left)
 {
     opal_bp_graph_t *graph = NULL;
@@ -338,7 +334,8 @@ static int mca_btl_tcp_proc_handle_modex_addresses(mca_btl_tcp_proc_t* btl_proc,
     int num_matched = 0;
     int *matched_edges = NULL;
 
-    rc = mca_btl_tcp_proc_create_interface_graph(btl_proc, remote_addrs, local_proc_is_left, &graph);
+    rc = mca_btl_tcp_proc_create_interface_graph(btl_proc, remote_addrs, local_proc_is_left,
+                                                 &graph);
     if (rc) {
         goto cleanup;
     }
@@ -348,16 +345,14 @@ static int mca_btl_tcp_proc_handle_modex_addresses(mca_btl_tcp_proc_t* btl_proc,
         goto cleanup;
     }
 
-    rc = mca_btl_tcp_proc_store_matched_interfaces(btl_proc, local_proc_is_left,
-                                                   graph, num_matched, matched_edges);
-    if (rc) {
-        goto cleanup;
-    }
+    rc = mca_btl_tcp_proc_store_matched_interfaces(btl_proc, local_proc_is_left, graph, num_matched,
+                                                   matched_edges);
 
 cleanup:
     if (NULL != graph) {
         opal_bp_graph_free(graph);
     }
+    free(matched_edges);
     return rc;
 }
 
@@ -369,16 +364,16 @@ static int mca_btl_tcp_proc_handle_modex_addresses(mca_btl_tcp_proc_t* btl_proc,
  * datastructure.
  */
 
-mca_btl_tcp_proc_t* mca_btl_tcp_proc_create(opal_proc_t* proc)
+mca_btl_tcp_proc_t *mca_btl_tcp_proc_create(opal_proc_t *proc)
 {
-    mca_btl_tcp_proc_t* btl_proc;
+    mca_btl_tcp_proc_t *btl_proc;
     int rc, local_proc_is_left;
     mca_btl_tcp_modex_addr_t *remote_addrs = NULL;
     size_t size;
 
     OPAL_THREAD_LOCK(&mca_btl_tcp_component.tcp_lock);
-    rc = opal_proc_table_get_value(&mca_btl_tcp_component.tcp_procs,
-                                   proc->proc_name, (void**)&btl_proc);
+    rc = opal_proc_table_get_value(&mca_btl_tcp_component.tcp_procs, proc->proc_name,
+                                   (void **) &btl_proc);
     if (OPAL_SUCCESS == rc) {
         OPAL_THREAD_UNLOCK(&mca_btl_tcp_component.tcp_lock);
         return btl_proc;
@@ -398,8 +393,8 @@ mca_btl_tcp_proc_t* mca_btl_tcp_proc_create(opal_proc_t* proc)
     OBJ_RETAIN(proc);
 
     /* lookup tcp parameters exported by this proc */
-    OPAL_MODEX_RECV(rc, &mca_btl_tcp_component.super.btl_version,
-                    &proc->proc_name, (uint8_t**)&remote_addrs, &size);
+    OPAL_MODEX_RECV(rc, &mca_btl_tcp_component.super.btl_version, &proc->proc_name,
+                    (uint8_t **) &remote_addrs, &size);
     if (OPAL_SUCCESS != rc) {
         if (OPAL_ERR_NOT_FOUND != rc) {
             BTL_ERROR(("opal_modex_recv: failed with return value=%d", rc));
@@ -408,16 +403,14 @@ mca_btl_tcp_proc_t* mca_btl_tcp_proc_create(opal_proc_t* proc)
     }
 
     if (0 != (size % sizeof(mca_btl_tcp_modex_addr_t))) {
-        BTL_ERROR(("opal_modex_recv: invalid size %lu: btl-size: %lu\n",
-                   (unsigned long)size,
-                   (unsigned long)sizeof(mca_btl_tcp_modex_addr_t)));
+        BTL_ERROR(("opal_modex_recv: invalid size %lu: btl-size: %lu\n", (unsigned long) size,
+                   (unsigned long) sizeof(mca_btl_tcp_modex_addr_t)));
         rc = OPAL_ERROR;
         goto cleanup;
     }
 
     btl_proc->proc_addr_count = size / sizeof(mca_btl_tcp_modex_addr_t);
-    btl_proc->proc_addrs = malloc(btl_proc->proc_addr_count *
-                                  sizeof(mca_btl_tcp_addr_t));
+    btl_proc->proc_addrs = malloc(btl_proc->proc_addr_count * sizeof(mca_btl_tcp_addr_t));
     if (NULL == btl_proc->proc_addrs) {
         rc = OPAL_ERR_OUT_OF_RESOURCE;
         goto cleanup;
@@ -432,7 +425,8 @@ mca_btl_tcp_proc_t* mca_btl_tcp_proc_create(opal_proc_t* proc)
      * The concept of mirroring the local and remote sides is borrowed
      * from the usnic btl implementation of its bipartite assignment solver.
      */
-    local_proc_is_left = mca_btl_tcp_proc_is_proc_left(proc->proc_name, opal_proc_local_get()->proc_name);
+    local_proc_is_left = mca_btl_tcp_proc_is_proc_left(proc->proc_name,
+                                                       opal_proc_local_get()->proc_name);
     rc = mca_btl_tcp_proc_handle_modex_addresses(btl_proc, remote_addrs, local_proc_is_left);
 
     if (OPAL_SUCCESS != rc) {
@@ -440,9 +434,8 @@ mca_btl_tcp_proc_t* mca_btl_tcp_proc_create(opal_proc_t* proc)
     }
 
     /* allocate space for endpoint array - one for each exported address */
-    btl_proc->proc_endpoints = (mca_btl_base_endpoint_t**)
-        malloc((1 + btl_proc->proc_addr_count) *
-               sizeof(mca_btl_base_endpoint_t*));
+    btl_proc->proc_endpoints = (mca_btl_base_endpoint_t **) malloc(
+        (1 + btl_proc->proc_addr_count) * sizeof(mca_btl_base_endpoint_t *));
     if (NULL == btl_proc->proc_endpoints) {
         rc = OPAL_ERR_OUT_OF_RESOURCE;
         goto cleanup;
@@ -450,14 +443,13 @@ mca_btl_tcp_proc_t* mca_btl_tcp_proc_create(opal_proc_t* proc)
 
 cleanup:
     if (OPAL_SUCCESS == rc) {
-        btl_proc->proc_opal = proc;  /* link with the proc */
+        btl_proc->proc_opal = proc; /* link with the proc */
         /* add to hash table of all proc instance. */
-        opal_proc_table_set_value(&mca_btl_tcp_component.tcp_procs,
-                                  proc->proc_name, btl_proc);
+        opal_proc_table_set_value(&mca_btl_tcp_component.tcp_procs, proc->proc_name, btl_proc);
     } else {
         if (btl_proc) {
-            OBJ_RELEASE(btl_proc);  /* release the local proc */
-            OBJ_RELEASE(proc);      /* and the ref on the OMPI proc */
+            OBJ_RELEASE(btl_proc); /* release the local proc */
+            OBJ_RELEASE(proc);     /* and the ref on the OMPI proc */
             btl_proc = NULL;
         }
     }
@@ -476,19 +468,18 @@ mca_btl_tcp_proc_t* mca_btl_tcp_proc_create(opal_proc_t* proc)
  * already held.  Insert a btl instance into the proc array and assign
  * it an address.
  */
-int mca_btl_tcp_proc_insert(mca_btl_tcp_proc_t* btl_proc,
-                            mca_btl_base_endpoint_t* btl_endpoint)
+int mca_btl_tcp_proc_insert(mca_btl_tcp_proc_t *btl_proc, mca_btl_base_endpoint_t *btl_endpoint)
 {
-    mca_btl_tcp_module_t* tcp_btl = btl_endpoint->endpoint_btl;
+    mca_btl_tcp_module_t *tcp_btl = btl_endpoint->endpoint_btl;
     mca_btl_tcp_addr_t *remote_addr;
     int rc = OPAL_SUCCESS;
 
-    rc = opal_hash_table_get_value_uint32(&btl_proc->btl_index_to_endpoint, tcp_btl->btl_index, (void **)&remote_addr);
+    rc = opal_hash_table_get_value_uint32(&btl_proc->btl_index_to_endpoint, tcp_btl->btl_index,
+                                          (void **) &remote_addr);
     if (OPAL_SUCCESS != rc) {
         if (9 < opal_output_get_verbosity(opal_btl_base_framework.framework_output)) {
             char *proc_hostname = opal_get_proc_hostname(btl_proc->proc_opal);
-            opal_output(0, "btl:tcp: host %s, process %s UNREACHABLE",
-                        proc_hostname,
+            opal_output(0, "btl:tcp: host %s, process %s UNREACHABLE", proc_hostname,
                         OPAL_NAME_PRINT(btl_proc->proc_opal->proc_name));
             free(proc_hostname);
         }
@@ -520,16 +511,17 @@ int mca_btl_tcp_proc_insert(mca_btl_tcp_proc_t* btl_proc,
  * no longer in use.
  */
 
-int mca_btl_tcp_proc_remove(mca_btl_tcp_proc_t* btl_proc, mca_btl_base_endpoint_t* btl_endpoint)
+int mca_btl_tcp_proc_remove(mca_btl_tcp_proc_t *btl_proc, mca_btl_base_endpoint_t *btl_endpoint)
 {
     size_t i;
     if (NULL != btl_proc) {
         OPAL_THREAD_LOCK(&btl_proc->proc_lock);
-        for(i = 0; i < btl_proc->proc_endpoint_count; i++) {
-            if(btl_proc->proc_endpoints[i] == btl_endpoint) {
-                memmove(btl_proc->proc_endpoints+i, btl_proc->proc_endpoints+i+1,
-                        (btl_proc->proc_endpoint_count-i-1)*sizeof(mca_btl_base_endpoint_t*));
-                if(--btl_proc->proc_endpoint_count == 0) {
+        for (i = 0; i < btl_proc->proc_endpoint_count; i++) {
+            if (btl_proc->proc_endpoints[i] == btl_endpoint) {
+                memmove(btl_proc->proc_endpoints + i, btl_proc->proc_endpoints + i + 1,
+                        (btl_proc->proc_endpoint_count - i - 1)
+                            * sizeof(mca_btl_base_endpoint_t *));
+                if (--btl_proc->proc_endpoint_count == 0) {
                     OPAL_THREAD_UNLOCK(&btl_proc->proc_lock);
                     OBJ_RELEASE(btl_proc);
                     return OPAL_SUCCESS;
@@ -546,31 +538,29 @@ int mca_btl_tcp_proc_remove(mca_btl_tcp_proc_t* btl_proc, mca_btl_base_endpoint_
  * Look for an existing TCP process instance based on the globally unique
  * process identifier.
  */
-mca_btl_tcp_proc_t* mca_btl_tcp_proc_lookup(const opal_process_name_t *name)
+mca_btl_tcp_proc_t *mca_btl_tcp_proc_lookup(const opal_process_name_t *name)
 {
-    mca_btl_tcp_proc_t* proc = NULL;
+    mca_btl_tcp_proc_t *proc = NULL;
 
     OPAL_THREAD_LOCK(&mca_btl_tcp_component.tcp_lock);
-    opal_proc_table_get_value(&mca_btl_tcp_component.tcp_procs,
-                              *name, (void**)&proc);
+    opal_proc_table_get_value(&mca_btl_tcp_component.tcp_procs, *name, (void **) &proc);
     OPAL_THREAD_UNLOCK(&mca_btl_tcp_component.tcp_lock);
     if (OPAL_UNLIKELY(NULL == proc)) {
         mca_btl_base_endpoint_t *endpoint;
         opal_proc_t *opal_proc;
 
-        BTL_VERBOSE(("adding tcp proc for peer {%s}",
-                     OPAL_NAME_PRINT(*name)));
+        BTL_VERBOSE(("adding tcp proc for peer {%s}", OPAL_NAME_PRINT(*name)));
 
-        opal_proc = opal_proc_for_name (*name);
+        opal_proc = opal_proc_for_name(*name);
         if (NULL == opal_proc) {
             return NULL;
         }
 
         /* try adding this proc to each btl until */
-        for( uint32_t i = 0; i < mca_btl_tcp_component.tcp_num_btls; ++i ) {
+        for (uint32_t i = 0; i < mca_btl_tcp_component.tcp_num_btls; ++i) {
             endpoint = NULL;
-            (void) mca_btl_tcp_add_procs (&mca_btl_tcp_component.tcp_btls[i]->super, 1, &opal_proc,
-                                          &endpoint, NULL);
+            (void) mca_btl_tcp_add_procs(&mca_btl_tcp_component.tcp_btls[i]->super, 1, &opal_proc,
+                                         &endpoint, NULL);
             if (NULL != endpoint && NULL == proc) {
                 /* construct all the endpoints and get the proc */
                 proc = endpoint->endpoint_proc;
@@ -585,78 +575,84 @@ mca_btl_tcp_proc_t* mca_btl_tcp_proc_lookup(const opal_process_name_t *name)
  * loop through all available BTLs for one matching the source address
  * of the request.
  */
-void mca_btl_tcp_proc_accept(mca_btl_tcp_proc_t* btl_proc, struct sockaddr* addr, int sd)
+void mca_btl_tcp_proc_accept(mca_btl_tcp_proc_t *btl_proc, struct sockaddr *addr, int sd)
 {
     OPAL_THREAD_LOCK(&btl_proc->proc_lock);
     int found_match = 0;
-    mca_btl_base_endpoint_t* match_btl_endpoint;
+    mca_btl_base_endpoint_t *match_btl_endpoint;
 
-    for( size_t i = 0; i < btl_proc->proc_endpoint_count; i++ ) {
-        mca_btl_base_endpoint_t* btl_endpoint = btl_proc->proc_endpoints[i];
+    for (size_t i = 0; i < btl_proc->proc_endpoint_count; i++) {
+        mca_btl_base_endpoint_t *btl_endpoint = btl_proc->proc_endpoints[i];
         /* We are not here to make a decision about what is good socket
          * and what is not. We simply check that this socket fit the endpoint
          * end we prepare for the real decision function mca_btl_tcp_endpoint_accept. */
-        if( btl_endpoint->endpoint_addr->addr_family != addr->sa_family) {
+        if (btl_endpoint->endpoint_addr->addr_family != addr->sa_family) {
             continue;
         }
         switch (addr->sa_family) {
         case AF_INET:
-            if( memcmp( &btl_endpoint->endpoint_addr->addr_union.addr_inet,
-                        &(((struct sockaddr_in*)addr)->sin_addr),
-                        sizeof(struct in_addr) ) ) {
+            if (memcmp(&btl_endpoint->endpoint_addr->addr_union.addr_inet,
+                       &(((struct sockaddr_in *) addr)->sin_addr), sizeof(struct in_addr))) {
                 char tmp[2][16];
                 opal_output_verbose(20, opal_btl_base_framework.framework_output,
-                                    "btl: tcp: Match incoming connection from %s %s with locally known IP %s failed (iface %d/%d)!\n",
+                                    "btl: tcp: Match incoming connection from %s %s with locally "
+                                    "known IP %s failed (iface %d/%d)!\n",
                                     OPAL_NAME_PRINT(btl_proc->proc_opal->proc_name),
-                                    inet_ntop(AF_INET, (void*)&((struct sockaddr_in*)addr)->sin_addr,
+                                    inet_ntop(AF_INET,
+                                              (void *) &((struct sockaddr_in *) addr)->sin_addr,
                                               tmp[0], 16),
-                                    inet_ntop(AF_INET, (void*)(struct in_addr*)&btl_endpoint->endpoint_addr->addr_union.addr_inet,
+                                    inet_ntop(AF_INET,
+                                              (void *) (struct in_addr *) &btl_endpoint
+                                                  ->endpoint_addr->addr_union.addr_inet,
                                               tmp[1], 16),
-                                    (int)i, (int)btl_proc->proc_endpoint_count);
+                                    (int) i, (int) btl_proc->proc_endpoint_count);
                 continue;
             } else if (btl_endpoint->endpoint_state != MCA_BTL_TCP_CLOSED) {
-                 found_match = 1;
-                 match_btl_endpoint = btl_endpoint;
-                 continue;
+                found_match = 1;
+                match_btl_endpoint = btl_endpoint;
+                continue;
             }
             break;
 #if OPAL_ENABLE_IPV6
         case AF_INET6:
-            if( memcmp( &btl_endpoint->endpoint_addr->addr_union.addr_inet,
-                        &(((struct sockaddr_in6*)addr)->sin6_addr),
-                        sizeof(struct in6_addr) ) ) {
+            if (memcmp(&btl_endpoint->endpoint_addr->addr_union.addr_inet,
+                       &(((struct sockaddr_in6 *) addr)->sin6_addr), sizeof(struct in6_addr))) {
                 char tmp[2][INET6_ADDRSTRLEN];
                 opal_output_verbose(20, opal_btl_base_framework.framework_output,
-                                    "btl: tcp: Match incoming connection from %s %s with locally known IP %s failed (iface %d/%d)!\n",
+                                    "btl: tcp: Match incoming connection from %s %s with locally "
+                                    "known IP %s failed (iface %d/%d)!\n",
                                     OPAL_NAME_PRINT(btl_proc->proc_opal->proc_name),
-                                    inet_ntop(AF_INET6, (void*)&((struct sockaddr_in6*)addr)->sin6_addr,
+                                    inet_ntop(AF_INET6,
+                                              (void *) &((struct sockaddr_in6 *) addr)->sin6_addr,
                                               tmp[0], INET6_ADDRSTRLEN),
-                                    inet_ntop(AF_INET6, (void*)(struct in6_addr*)&btl_endpoint->endpoint_addr->addr_union.addr_inet,
+                                    inet_ntop(AF_INET6,
+                                              (void *) (struct in6_addr *) &btl_endpoint
+                                                  ->endpoint_addr->addr_union.addr_inet,
                                               tmp[1], INET6_ADDRSTRLEN),
-                                    (int)i, (int)btl_proc->proc_endpoint_count);
+                                    (int) i, (int) btl_proc->proc_endpoint_count);
                 continue;
             } else if (btl_endpoint->endpoint_state != MCA_BTL_TCP_CLOSED) {
-                 found_match = 1;
-                 match_btl_endpoint = btl_endpoint;
-                 continue;
+                found_match = 1;
+                match_btl_endpoint = btl_endpoint;
+                continue;
             }
             break;
 #endif
-        default:
-            ;
+        default:;
         }
 
-        /* Set state to CONNECTING to ensure that subsequent conenctions do not attempt to re-use endpoint in the num_links > 1 case*/
+        /* Set state to CONNECTING to ensure that subsequent connections do not attempt to re-use
+         * endpoint in the num_links > 1 case*/
         btl_endpoint->endpoint_state = MCA_BTL_TCP_CONNECTING;
-        (void)mca_btl_tcp_endpoint_accept(btl_endpoint, addr, sd);
+        (void) mca_btl_tcp_endpoint_accept(btl_endpoint, addr, sd);
         OPAL_THREAD_UNLOCK(&btl_proc->proc_lock);
         return;
     }
-    /* In this case the connection was inbound to an address exported, but was not in a CLOSED state.
-     * mca_btl_tcp_endpoint_accept() has logic to deal with the race condition that has likely caused this
-     * scenario, so call it here.*/
+    /* In this case the connection was inbound to an address exported, but was not in a CLOSED
+     * state. mca_btl_tcp_endpoint_accept() has logic to deal with the race condition that has
+     * likely caused this scenario, so call it here.*/
     if (found_match) {
-        (void)mca_btl_tcp_endpoint_accept(match_btl_endpoint, addr, sd);
+        (void) mca_btl_tcp_endpoint_accept(match_btl_endpoint, addr, sd);
         OPAL_THREAD_UNLOCK(&btl_proc->proc_lock);
         return;
     }
@@ -668,13 +664,13 @@ void mca_btl_tcp_proc_accept(mca_btl_tcp_proc_t* btl_proc, struct sockaddr* addr
         ip[sizeof(ip) - 1] = '\0';
 
         for (size_t i = 0; i < btl_proc->proc_endpoint_count; i++) {
-            mca_btl_base_endpoint_t* btl_endpoint = btl_proc->proc_endpoints[i];
+            mca_btl_base_endpoint_t *btl_endpoint = btl_proc->proc_endpoints[i];
             if (btl_endpoint->endpoint_addr->addr_family != addr->sa_family) {
                 continue;
             }
             inet_ntop(btl_endpoint->endpoint_addr->addr_family,
-                      (void*) &(btl_endpoint->endpoint_addr->addr_union.addr_inet),
-                      ip, sizeof(ip) - 1);
+                      (void *) &(btl_endpoint->endpoint_addr->addr_union.addr_inet), ip,
+                      sizeof(ip) - 1);
             if (NULL == addr_str) {
                 opal_asprintf(&tmp, "\n\t%s", ip);
             } else {
@@ -684,13 +680,11 @@ void mca_btl_tcp_proc_accept(mca_btl_tcp_proc_t* btl_proc, struct sockaddr* addr
             addr_str = tmp;
         }
         tmp = opal_get_proc_hostname(btl_proc->proc_opal);
-        opal_show_help("help-mpi-btl-tcp.txt", "dropped inbound connection",
-                       true, opal_process_info.nodename,
-                       getpid(), tmp,
+        opal_show_help("help-mpi-btl-tcp.txt", "dropped inbound connection", true,
+                       opal_process_info.nodename, getpid(), tmp,
                        OPAL_NAME_PRINT(btl_proc->proc_opal->proc_name),
-                       opal_net_get_hostname((struct sockaddr*)addr),
-                       btl_proc->proc_endpoint_count,
-                       (NULL == addr_str) ? "NONE" : addr_str);
+                       opal_net_get_hostname((struct sockaddr *) addr),
+                       btl_proc->proc_endpoint_count, (NULL == addr_str) ? "NONE" : addr_str);
         free(tmp);
         if (NULL != addr_str) {
             free(addr_str);
@@ -703,35 +697,31 @@ void mca_btl_tcp_proc_accept(mca_btl_tcp_proc_t* btl_proc, struct sockaddr* addr
  * convert internal data structure (mca_btl_tcp_addr_t) to sockaddr_storage
  *
  */
-bool mca_btl_tcp_proc_tosocks(mca_btl_tcp_addr_t* proc_addr,
-                              struct sockaddr_storage* output)
+bool mca_btl_tcp_proc_tosocks(mca_btl_tcp_addr_t *proc_addr, struct sockaddr_storage *output)
 {
-    memset(output, 0, sizeof (*output));
+    memset(output, 0, sizeof(*output));
     switch (proc_addr->addr_family) {
     case AF_INET:
         output->ss_family = AF_INET;
-        memcpy(&((struct sockaddr_in*)output)->sin_addr,
-               &proc_addr->addr_union.addr_inet, sizeof(struct in_addr));
-        ((struct sockaddr_in*)output)->sin_port = proc_addr->addr_port;
+        memcpy(&((struct sockaddr_in *) output)->sin_addr, &proc_addr->addr_union.addr_inet,
+               sizeof(struct in_addr));
+        ((struct sockaddr_in *) output)->sin_port = proc_addr->addr_port;
         break;
 #if OPAL_ENABLE_IPV6
-    case AF_INET6:
-        {
-            struct sockaddr_in6* inaddr = (struct sockaddr_in6*)output;
-            output->ss_family = AF_INET6;
-            memcpy(&inaddr->sin6_addr, &proc_addr->addr_union.addr_inet6,
-                   sizeof (proc_addr->addr_union.addr_inet6));
-            inaddr->sin6_port = proc_addr->addr_port;
-            inaddr->sin6_scope_id = 0;
-            inaddr->sin6_flowinfo = 0;
-        }
-        break;
+    case AF_INET6: {
+        struct sockaddr_in6 *inaddr = (struct sockaddr_in6 *) output;
+        output->ss_family = AF_INET6;
+        memcpy(&inaddr->sin6_addr, &proc_addr->addr_union.addr_inet6,
+               sizeof(proc_addr->addr_union.addr_inet6));
+        inaddr->sin6_port = proc_addr->addr_port;
+        inaddr->sin6_scope_id = 0;
+        inaddr->sin6_flowinfo = 0;
+    } break;
 #endif
     default:
-        opal_output( 0, "mca_btl_tcp_proc: unknown af_family received: %d\n",
-                     proc_addr->addr_family );
+        opal_output(0, "mca_btl_tcp_proc: unknown af_family received: %d\n",
+                    proc_addr->addr_family);
         return false;
     }
     return true;
 }
-
diff --git a/opal/mca/btl/tcp/btl_tcp_proc.h b/opal/mca/btl/tcp/btl_tcp_proc.h
index abbb9baf864..45bd3849bdb 100644
--- a/opal/mca/btl/tcp/btl_tcp_proc.h
+++ b/opal/mca/btl/tcp/btl_tcp_proc.h
@@ -22,11 +22,11 @@
 #ifndef MCA_BTL_TCP_PROC_H
 #define MCA_BTL_TCP_PROC_H
 
-#include "opal/class/opal_object.h"
-#include "opal/util/proc.h"
 #include "btl_tcp.h"
 #include "btl_tcp_addr.h"
 #include "btl_tcp_endpoint.h"
+#include "opal/class/opal_object.h"
+#include "opal/util/proc.h"
 
 BEGIN_C_DECLS
 
@@ -43,7 +43,7 @@ struct mca_btl_tcp_proc_t {
     opal_proc_t *proc_opal;
     /**< pointer to corresponding opal_proc_t */
 
-    struct mca_btl_tcp_addr_t* proc_addrs;
+    struct mca_btl_tcp_addr_t *proc_addrs;
     /**< array of addresses exported by peer */
 
     size_t proc_addr_count;
@@ -64,12 +64,12 @@ struct mca_btl_tcp_proc_t {
 typedef struct mca_btl_tcp_proc_t mca_btl_tcp_proc_t;
 OBJ_CLASS_DECLARATION(mca_btl_tcp_proc_t);
 
-mca_btl_tcp_proc_t* mca_btl_tcp_proc_create(opal_proc_t* proc);
-mca_btl_tcp_proc_t* mca_btl_tcp_proc_lookup(const opal_process_name_t* name);
-int  mca_btl_tcp_proc_insert(mca_btl_tcp_proc_t*, mca_btl_base_endpoint_t*);
-int  mca_btl_tcp_proc_remove(mca_btl_tcp_proc_t*, mca_btl_base_endpoint_t*);
-void mca_btl_tcp_proc_accept(mca_btl_tcp_proc_t*, struct sockaddr*, int);
-bool mca_btl_tcp_proc_tosocks(mca_btl_tcp_addr_t*, struct sockaddr_storage*);
+mca_btl_tcp_proc_t *mca_btl_tcp_proc_create(opal_proc_t *proc);
+mca_btl_tcp_proc_t *mca_btl_tcp_proc_lookup(const opal_process_name_t *name);
+int mca_btl_tcp_proc_insert(mca_btl_tcp_proc_t *, mca_btl_base_endpoint_t *);
+int mca_btl_tcp_proc_remove(mca_btl_tcp_proc_t *, mca_btl_base_endpoint_t *);
+void mca_btl_tcp_proc_accept(mca_btl_tcp_proc_t *, struct sockaddr *, int);
+bool mca_btl_tcp_proc_tosocks(mca_btl_tcp_addr_t *, struct sockaddr_storage *);
 
 END_C_DECLS
 #endif
diff --git a/opal/mca/btl/tcp/configure.m4 b/opal/mca/btl/tcp/configure.m4
index 9eb66631ce1..c8cfb571336 100644
--- a/opal/mca/btl/tcp/configure.m4
+++ b/opal/mca/btl/tcp/configure.m4
@@ -13,6 +13,7 @@
 # Copyright (c) 2010-2016 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2016      Los Alamos National Security, LLC. All rights
 #                         reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -36,5 +37,5 @@ AC_DEFUN([MCA_opal_btl_tcp_CONFIG],[
 #include <netinet/in.h>
 #endif
 		   ])
-    OPAL_SUMMARY_ADD([[Transports]],[[TCP]],[[btl_tcp]],[$opal_btl_tcp_happy])
+    OPAL_SUMMARY_ADD([Transports], [TCP], [], [$opal_btl_tcp_happy])
 ])dnl
diff --git a/opal/mca/btl/tcp/help-mpi-btl-tcp.txt b/opal/mca/btl/tcp/help-mpi-btl-tcp.txt
index b8788206cf1..0afedce447f 100644
--- a/opal/mca/btl/tcp/help-mpi-btl-tcp.txt
+++ b/opal/mca/btl/tcp/help-mpi-btl-tcp.txt
@@ -1,6 +1,6 @@
 # -*- text -*-
 #
-# Copyright (c) 2009-2018 Cisco Systems, Inc.  All rights reserved
+# Copyright (c) 2009-2022 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2015-2016 The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
@@ -113,17 +113,6 @@ aborting your job.
   Flag:       %s
   Error:      %s (%d)
 #
-[server did not get guid]
-WARNING: Open MPI accepted a TCP connection from what appears to be a
-another Open MPI process but the peer process did not complete the
-initial handshake properly.  This should not happen.
-
-This attempted connection will be ignored; your MPI job may or may not
-continue properly.
-
-  Local host: %s
-  PID:        %d
-#
 [server accept cannot find guid]
 WARNING: Open MPI accepted a TCP connection from what appears to be a
 another Open MPI process but cannot find a corresponding process
@@ -146,13 +135,14 @@ continue properly.
   PID:        %d
   Error:      %s (%d)
 #
-[server cannot find endpoint]
-WARNING: Open MPI accepted a TCP connection from what appears to be a
-valid peer Open MPI process but cannot find a corresponding endpoint
-entry for that peer.  This should not happen.
+[server did not receive magic string]
+WARNING: Open MPI accepted a TCP connection but did not receive the
+expected network identification indicating that the peer is an Open
+MPI process.
 
-This attempted connection will be ignored; your MPI job may or may not
-continue properly.
+This attempted connection will be closed and ignored.
 
   Local host: %s
   PID:        %d
+  Peer:       %s
+  Identifier: %s (%s)
diff --git a/opal/mca/btl/template/Makefile.am b/opal/mca/btl/template/Makefile.am
index 176a5958984..bc9f4b304a3 100644
--- a/opal/mca/btl/template/Makefile.am
+++ b/opal/mca/btl/template/Makefile.am
@@ -52,7 +52,7 @@ mcacomponentdir = $(opallibdir)
 mcacomponent_LTLIBRARIES = $(component)
 mca_btl_template_la_SOURCES = $(component_sources)
 mca_btl_template_la_LDFLAGS = -module -avoid-version
-mca_btl_template_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+mca_btl_template_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 
 noinst_LTLIBRARIES = $(lib)
 libmca_btl_template_la_SOURCES = $(lib_sources)
diff --git a/opal/mca/btl/template/btl_template.c b/opal/mca/btl/template/btl_template.c
index 1d0bd1183ca..832f1e16e69 100644
--- a/opal/mca/btl/template/btl_template.c
+++ b/opal/mca/btl/template/btl_template.c
@@ -21,18 +21,17 @@
  */
 
 #include "opal_config.h"
-#include <string.h>
 #include "opal/class/opal_bitmap.h"
-#include "opal/mca/btl/btl.h"
 #include "opal/datatype/opal_convertor.h"
+#include "opal/mca/btl/btl.h"
 #include "opal/mca/mpool/base/base.h"
 #include "opal/mca/mpool/mpool.h"
+#include <string.h>
 
 #include "btl_template.h"
+#include "btl_template_endpoint.h"
 #include "btl_template_frag.h"
 #include "btl_template_proc.h"
-#include "btl_template_endpoint.h"
-
 
 mca_btl_template_module_t mca_btl_template_module = {
     .super = {
@@ -49,28 +48,24 @@ mca_btl_template_module_t mca_btl_template_module = {
         .btl_get = mca_btl_template_get,
         .btl_register_mem = mca_btl_template_register_mem,
         .btl_deregister_mem = mca_btl_template_deregister_mem,
-    }
-};
+    }};
 
 /**
  *
  */
 
-int mca_btl_template_add_procs(
-    struct mca_btl_base_module_t* btl,
-    size_t nprocs,
-    struct opal_proc_t **opal_procs,
-    struct mca_btl_base_endpoint_t** peers,
-    opal_bitmap_t* reachable)
+int mca_btl_template_add_procs(struct mca_btl_base_module_t *btl, size_t nprocs,
+                               struct opal_proc_t **opal_procs,
+                               struct mca_btl_base_endpoint_t **peers, opal_bitmap_t *reachable)
 {
-    mca_btl_template_module_t* template_btl = (mca_btl_template_module_t*)btl;
+    mca_btl_template_module_t *template_btl = (mca_btl_template_module_t *) btl;
     int i, rc;
 
-    for(i = 0; i < (int) nprocs; i++) {
+    for (i = 0; i < (int) nprocs; i++) {
 
-        struct opal_proc_t* opal_proc = opal_procs[i];
-        mca_btl_template_proc_t* template_proc;
-        mca_btl_base_endpoint_t* template_endpoint;
+        struct opal_proc_t *opal_proc = opal_procs[i];
+        mca_btl_template_proc_t *template_proc;
+        mca_btl_base_endpoint_t *template_endpoint;
 
 #if 0
         /* If the BTL doesn't support heterogeneous operations, be
@@ -80,7 +75,7 @@ int mca_btl_template_add_procs(
         }
 #endif
 
-        if(NULL == (template_proc = mca_btl_template_proc_create(opal_proc))) {
+        if (NULL == (template_proc = mca_btl_template_proc_create(opal_proc))) {
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
 
@@ -97,14 +92,14 @@ int mca_btl_template_add_procs(
          * Cache the peer instance on the btl_proc.
          */
         template_endpoint = OBJ_NEW(mca_btl_template_endpoint_t);
-        if(NULL == template_endpoint) {
+        if (NULL == template_endpoint) {
             OPAL_THREAD_UNLOCK(&template_proc->proc_lock);
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
 
         template_endpoint->endpoint_btl = template_btl;
         rc = mca_btl_template_proc_insert(template_proc, template_endpoint);
-        if(rc != OPAL_SUCCESS) {
+        if (rc != OPAL_SUCCESS) {
             OBJ_RELEASE(template_endpoint);
             OPAL_THREAD_UNLOCK(&template_proc->proc_lock);
             continue;
@@ -118,30 +113,23 @@ int mca_btl_template_add_procs(
     return OPAL_SUCCESS;
 }
 
-int mca_btl_template_del_procs(struct mca_btl_base_module_t* btl,
-        size_t nprocs,
-        struct opal_proc_t **procs,
-        struct mca_btl_base_endpoint_t ** peers)
+int mca_btl_template_del_procs(struct mca_btl_base_module_t *btl, size_t nprocs,
+                               struct opal_proc_t **procs, struct mca_btl_base_endpoint_t **peers)
 {
     /* TODO */
     return OPAL_SUCCESS;
 }
 
-
 /**
  * Register callback function to support send/recv semantics
  */
 
-int mca_btl_template_register(
-                        struct mca_btl_base_module_t* btl,
-                        mca_btl_base_tag_t tag,
-                        mca_btl_base_module_recv_cb_fn_t cbfunc,
-                        void* cbdata)
+int mca_btl_template_register(struct mca_btl_base_module_t *btl, mca_btl_base_tag_t tag,
+                              mca_btl_base_module_recv_cb_fn_t cbfunc, void *cbdata)
 {
     return OPAL_SUCCESS;
 }
 
-
 /**
  * Allocate a segment.
  *
@@ -149,50 +137,44 @@ int mca_btl_template_register(
  * @param size (IN)     Request segment size.
  */
 
-mca_btl_base_descriptor_t* mca_btl_template_alloc(
-    struct mca_btl_base_module_t* btl,
-    struct mca_btl_base_endpoint_t* endpoint,
-    uint8_t order,
-    size_t size,
-    uint32_t flags)
+mca_btl_base_descriptor_t *mca_btl_template_alloc(struct mca_btl_base_module_t *btl,
+                                                  struct mca_btl_base_endpoint_t *endpoint,
+                                                  uint8_t order, size_t size, uint32_t flags)
 {
-    mca_btl_template_module_t* template_btl = (mca_btl_template_module_t*) btl;
-    mca_btl_template_frag_t* frag = NULL;
+    mca_btl_template_module_t *template_btl = (mca_btl_template_module_t *) btl;
+    mca_btl_template_frag_t *frag = NULL;
 
-    if(size <= btl->btl_eager_limit){
+    if (size <= btl->btl_eager_limit) {
         MCA_BTL_TEMPLATE_FRAG_ALLOC_EAGER(template_btl, frag);
     } else {
         MCA_BTL_TEMPLATE_FRAG_ALLOC_MAX(template_btl, frag);
     }
-    if( OPAL_UNLIKELY(NULL != frag) ) {
+    if (OPAL_UNLIKELY(NULL != frag)) {
         return NULL;
     }
 
     frag->segment.seg_len = size;
     frag->base.des_flags = 0;
-    return (mca_btl_base_descriptor_t*)frag;
+    return (mca_btl_base_descriptor_t *) frag;
 }
 
-
 /**
  * Return a segment
  */
 
-int mca_btl_template_free(
-    struct mca_btl_base_module_t* btl,
-    mca_btl_base_descriptor_t* des)
+int mca_btl_template_free(struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des)
 {
-    mca_btl_template_frag_t* frag = (mca_btl_template_frag_t*)des;
-    if(frag->size == 0) {
+    mca_btl_template_frag_t *frag = (mca_btl_template_frag_t *) des;
+    if (frag->size == 0) {
 #if MCA_BTL_HAS_MPOOL
         OBJ_RELEASE(frag->registration);
 #endif
         MCA_BTL_TEMPLATE_FRAG_RETURN_USER(btl, frag);
-    } else if(frag->size == btl->btl_eager_limit){
+    } else if (frag->size == btl->btl_eager_limit) {
         MCA_BTL_TEMPLATE_FRAG_RETURN_EAGER(btl, frag);
-    } else if(frag->size == btl->btl_max_send_size) {
+    } else if (frag->size == btl->btl_max_send_size) {
         MCA_BTL_TEMPLATE_FRAG_RETURN_EAGER(btl, frag);
-    }  else {
+    } else {
         return OPAL_ERR_BAD_PARAM;
     }
     return OPAL_SUCCESS;
@@ -205,40 +187,35 @@ int mca_btl_template_free(
  * @param btl (IN)      BTL module
  * @param peer (IN)     BTL peer addressing
  */
-mca_btl_base_descriptor_t* mca_btl_template_prepare_src(
-    struct mca_btl_base_module_t* btl,
-    struct mca_btl_base_endpoint_t* endpoint,
-    struct opal_convertor_t* convertor,
-    uint8_t order,
-    size_t reserve,
-    size_t* size,
-    uint32_t flags
-)
+mca_btl_base_descriptor_t *mca_btl_template_prepare_src(struct mca_btl_base_module_t *btl,
+                                                        struct mca_btl_base_endpoint_t *endpoint,
+                                                        struct opal_convertor_t *convertor,
+                                                        uint8_t order, size_t reserve, size_t *size,
+                                                        uint32_t flags)
 {
-    mca_btl_template_frag_t* frag;
+    mca_btl_template_frag_t *frag;
     struct iovec iov;
     uint32_t iov_count = 1;
     size_t max_data = *size;
     int rc;
 
-
     /*
      * if we aren't pinning the data and the requested size is less
      * than the eager limit pack into a fragment from the eager pool
-    */
-    if (max_data+reserve <= btl->btl_eager_limit) {
+     */
+    if (max_data + reserve <= btl->btl_eager_limit) {
 
         MCA_BTL_TEMPLATE_FRAG_ALLOC_EAGER(btl, frag);
-        if(OPAL_UNLIKELY(NULL == frag)) {
+        if (OPAL_UNLIKELY(NULL == frag)) {
             return NULL;
         }
 
         iov.iov_len = max_data;
-        iov.iov_base = (unsigned char*) frag->segment.seg_addr.pval + reserve;
+        iov.iov_base = (unsigned char *) frag->segment.seg_addr.pval + reserve;
 
-        rc = opal_convertor_pack(convertor, &iov, &iov_count, &max_data );
-        *size  = max_data;
-        if( rc < 0 ) {
+        rc = opal_convertor_pack(convertor, &iov, &iov_count, &max_data);
+        *size = max_data;
+        if (rc < 0) {
             MCA_BTL_TEMPLATE_FRAG_RETURN_EAGER(btl, frag);
             return NULL;
         }
@@ -252,19 +229,19 @@ mca_btl_base_descriptor_t* mca_btl_template_prepare_src(
     else {
 
         MCA_BTL_TEMPLATE_FRAG_ALLOC_MAX(btl, frag);
-        if(OPAL_UNLIKELY(NULL == frag)) {
+        if (OPAL_UNLIKELY(NULL == frag)) {
             return NULL;
         }
-        if(max_data + reserve > frag->size){
+        if (max_data + reserve > frag->size) {
             max_data = frag->size - reserve;
         }
         iov.iov_len = max_data;
-        iov.iov_base = (unsigned char*) frag->segment.seg_addr.pval + reserve;
+        iov.iov_base = (unsigned char *) frag->segment.seg_addr.pval + reserve;
 
-        rc = opal_convertor_pack(convertor, &iov, &iov_count, &max_data );
-        *size  = max_data;
+        rc = opal_convertor_pack(convertor, &iov, &iov_count, &max_data);
+        *size = max_data;
 
-        if( rc < 0 ) {
+        if (rc < 0) {
             MCA_BTL_TEMPLATE_FRAG_RETURN_MAX(btl, frag);
             return NULL;
         }
@@ -277,31 +254,27 @@ mca_btl_base_descriptor_t* mca_btl_template_prepare_src(
     return &frag->base;
 }
 
-
 /**
  * Initiate an asynchronous send.
  *
  * @param btl (IN)         BTL module
  * @param endpoint (IN)    BTL addressing information
- * @param descriptor (IN)  Description of the data to be transfered
+ * @param descriptor (IN)  Description of the data to be transferred
  * @param tag (IN)         The tag value used to notify the peer.
  */
 
-int mca_btl_template_send(
-    struct mca_btl_base_module_t* btl,
-    struct mca_btl_base_endpoint_t* endpoint,
-    struct mca_btl_base_descriptor_t* descriptor,
-    mca_btl_base_tag_t tag)
+int mca_btl_template_send(struct mca_btl_base_module_t *btl,
+                          struct mca_btl_base_endpoint_t *endpoint,
+                          struct mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag)
 
 {
     /* mca_btl_template_module_t* template_btl = (mca_btl_template_module_t*) btl; */
-    mca_btl_template_frag_t* frag = (mca_btl_template_frag_t*)descriptor;
+    mca_btl_template_frag_t *frag = (mca_btl_template_frag_t *) descriptor;
     frag->endpoint = endpoint;
     /* TODO */
     return OPAL_ERR_NOT_IMPLEMENTED;
 }
 
-
 /**
  * Initiate an asynchronous put.
  *
@@ -310,18 +283,19 @@ int mca_btl_template_send(
  * @param descriptor (IN)  Description of the data to be transferred
  */
 
-int mca_btl_template_put (struct mca_btl_base_module_t *btl,
-    struct mca_btl_base_endpoint_t *endpoint, void *local_address,
-    uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle,
-    struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-    int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
+int mca_btl_template_put(struct mca_btl_base_module_t *btl,
+                         struct mca_btl_base_endpoint_t *endpoint, void *local_address,
+                         uint64_t remote_address,
+                         struct mca_btl_base_registration_handle_t *local_handle,
+                         struct mca_btl_base_registration_handle_t *remote_handle, size_t size,
+                         int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
+                         void *cbcontext, void *cbdata)
 {
     /* mca_btl_template_module_t* template_btl = (mca_btl_template_module_t*) btl; */
     /* TODO */
     return OPAL_ERR_NOT_IMPLEMENTED;
 }
 
-
 /**
  * Initiate an asynchronous get.
  *
@@ -331,11 +305,13 @@ int mca_btl_template_put (struct mca_btl_base_module_t *btl,
  *
  */
 
-int mca_btl_template_get (struct mca_btl_base_module_t *btl,
-    struct mca_btl_base_endpoint_t *endpoint, void *local_address,
-    uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle,
-    struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-    int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
+int mca_btl_template_get(struct mca_btl_base_module_t *btl,
+                         struct mca_btl_base_endpoint_t *endpoint, void *local_address,
+                         uint64_t remote_address,
+                         struct mca_btl_base_registration_handle_t *local_handle,
+                         struct mca_btl_base_registration_handle_t *remote_handle, size_t size,
+                         int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
+                         void *cbcontext, void *cbdata)
 {
     /* mca_btl_template_module_t* template_btl = (mca_btl_template_module_t*) btl; */
     /* TODO */
@@ -360,9 +336,10 @@ int mca_btl_template_get (struct mca_btl_base_module_t *btl,
  * functions. Care should be taken to not hold an excessive number of registrations
  * as they may use limited system/NIC resources.
  */
-struct mca_btl_base_registration_handle_t *mca_btl_template_register_mem (
-    struct mca_btl_base_module_t* btl, struct mca_btl_base_endpoint_t *endpoint, void *base,
-    size_t size, uint32_t flags)
+struct mca_btl_base_registration_handle_t *
+mca_btl_template_register_mem(struct mca_btl_base_module_t *btl,
+                              struct mca_btl_base_endpoint_t *endpoint, void *base, size_t size,
+                              uint32_t flags)
 {
     /* mca_btl_template_module_t* template_btl = (mca_btl_template_module_t*) btl; */
     /* TODO */
@@ -380,22 +357,21 @@ struct mca_btl_base_registration_handle_t *mca_btl_template_register_mem (
  * after it is deregistered. It is erroneous to specify a memory handle associated with
  * a remote node.
  */
-int mca_btl_template_deregister_mem (struct mca_btl_base_module_t* btl,
-				     struct mca_btl_base_registration_handle_t *handle)
+int mca_btl_template_deregister_mem(struct mca_btl_base_module_t *btl,
+                                    struct mca_btl_base_registration_handle_t *handle)
 {
     /* mca_btl_template_module_t* template_btl = (mca_btl_template_module_t*) btl; */
     /* TODO */
     return OPAL_ERR_NOT_IMPLEMENTED;
 }
 
-
 /*
  * Cleanup/release module resources.
  */
 
-int mca_btl_template_finalize(struct mca_btl_base_module_t* btl)
+int mca_btl_template_finalize(struct mca_btl_base_module_t *btl)
 {
-    mca_btl_template_module_t* template_btl = (mca_btl_template_module_t*) btl;
+    mca_btl_template_module_t *template_btl = (mca_btl_template_module_t *) btl;
     OBJ_DESTRUCT(&template_btl->template_lock);
     OBJ_DESTRUCT(&template_btl->template_frag_eager);
     OBJ_DESTRUCT(&template_btl->template_frag_max);
diff --git a/opal/mca/btl/template/btl_template.h b/opal/mca/btl/template/btl_template.h
index 260f972b48d..11baf7285d0 100644
--- a/opal/mca/btl/template/btl_template.h
+++ b/opal/mca/btl/template/btl_template.h
@@ -28,14 +28,14 @@
 #define MCA_BTL_TEMPLATE_H
 
 #include "opal_config.h"
-#include <sys/types.h>
 #include <string.h>
+#include <sys/types.h>
 
 /* Open MPI includes */
-#include "opal/util/event.h"
-#include "opal/mca/btl/btl.h"
 #include "opal/mca/btl/base/base.h"
+#include "opal/mca/btl/btl.h"
 #include "opal/mca/mpool/mpool.h"
+#include "opal/util/event.h"
 
 BEGIN_C_DECLS
 
@@ -46,12 +46,12 @@ BEGIN_C_DECLS
  */
 
 struct mca_btl_template_component_t {
-    mca_btl_base_component_3_0_0_t          super;  /**< base BTL component */
+    mca_btl_base_component_3_0_0_t super; /**< base BTL component */
 
-    uint32_t                                template_num_btls;
+    uint32_t template_num_btls;
     /**< number of hcas available to the TEMPLATE component */
 
-    struct mca_btl_template_module_t       *template_btls;
+    struct mca_btl_template_module_t *template_btls;
     /**< array of available BTL modules */
 
     int template_free_list_num;
@@ -69,7 +69,7 @@ struct mca_btl_template_component_t {
     opal_mutex_t template_lock;
     /**< lock for accessing module state */
 
-    char* template_mpool_name;
+    char *template_mpool_name;
     /**< name of memory pool */
 
     bool leave_pinned;
@@ -77,13 +77,13 @@ struct mca_btl_template_component_t {
 };
 typedef struct mca_btl_template_component_t mca_btl_template_component_t;
 
-OPAL_MODULE_DECLSPEC extern mca_btl_template_component_t mca_btl_template_component;
+OPAL_DECLSPEC extern mca_btl_template_component_t mca_btl_template_component;
 
 /**
  * BTL Module Interface
  */
 struct mca_btl_template_module_t {
-    mca_btl_base_module_t  super;  /**< base BTL interface */
+    mca_btl_base_module_t super; /**< base BTL interface */
 
     /* free list of fragment descriptors */
     opal_free_list_t template_frag_eager;
@@ -94,34 +94,28 @@ struct mca_btl_template_module_t {
     opal_mutex_t template_lock;
 
 #if MCA_BTL_HAS_MPOOL
-    struct mca_mpool_base_module_t* template_mpool;
+    struct mca_mpool_base_module_t *template_mpool;
 #endif
 };
 typedef struct mca_btl_template_module_t mca_btl_template_module_t;
 extern mca_btl_template_module_t mca_btl_template_module;
 
-
 /**
  * TEMPLATE component initialization.
  *
  * @param num_btl_modules (OUT)           Number of BTLs returned in BTL array.
- * @param allow_multi_user_threads (OUT)  Flag indicating wether BTL supports user threads (TRUE)
- * @param have_hidden_threads (OUT)       Flag indicating wether BTL uses threads (TRUE)
+ * @param allow_multi_user_threads (OUT)  Flag indicating whether BTL supports user threads (TRUE)
+ * @param have_hidden_threads (OUT)       Flag indicating whether BTL uses threads (TRUE)
  */
-extern mca_btl_base_module_t** mca_btl_template_component_init(
-    int *num_btl_modules,
-    bool allow_multi_user_threads,
-    bool have_hidden_threads
-);
-
+extern mca_btl_base_module_t **mca_btl_template_component_init(int *num_btl_modules,
+                                                               bool allow_multi_user_threads,
+                                                               bool have_hidden_threads);
 
 /**
  * TEMPLATE component progress.
  */
 extern int mca_btl_template_component_progress(void);
 
-
-
 /**
  * Cleanup any resources held by the BTL.
  *
@@ -129,10 +123,7 @@ extern int mca_btl_template_component_progress(void);
  * @return     OPAL_SUCCESS or error status on failure.
  */
 
-extern int mca_btl_template_finalize(
-    struct mca_btl_base_module_t* btl
-);
-
+extern int mca_btl_template_finalize(struct mca_btl_base_module_t *btl);
 
 /**
  * PML->BTL notification of change in the process list.
@@ -146,13 +137,10 @@ extern int mca_btl_template_finalize(
  *
  */
 
-extern int mca_btl_template_add_procs(
-    struct mca_btl_base_module_t* btl,
-    size_t nprocs,
-    struct opal_proc_t **procs,
-    struct mca_btl_base_endpoint_t** peers,
-    opal_bitmap_t* reachable
-);
+extern int mca_btl_template_add_procs(struct mca_btl_base_module_t *btl, size_t nprocs,
+                                      struct opal_proc_t **procs,
+                                      struct mca_btl_base_endpoint_t **peers,
+                                      opal_bitmap_t *reachable);
 
 /**
  * PML->BTL notification of change in the process list.
@@ -165,30 +153,23 @@ extern int mca_btl_template_add_procs(
  *
  */
 
-extern int mca_btl_template_del_procs(
-    struct mca_btl_base_module_t* btl,
-    size_t nprocs,
-    struct opal_proc_t **procs,
-    struct mca_btl_base_endpoint_t** peers
-);
-
+extern int mca_btl_template_del_procs(struct mca_btl_base_module_t *btl, size_t nprocs,
+                                      struct opal_proc_t **procs,
+                                      struct mca_btl_base_endpoint_t **peers);
 
 /**
  * Initiate an asynchronous send.
  *
  * @param btl (IN)         BTL module
  * @param endpoint (IN)    BTL addressing information
- * @param descriptor (IN)  Description of the data to be transfered
+ * @param descriptor (IN)  Description of the data to be transferred
  * @param tag (IN)         The tag value used to notify the peer.
  */
 
-extern int mca_btl_template_send(
-    struct mca_btl_base_module_t* btl,
-    struct mca_btl_base_endpoint_t* btl_peer,
-    struct mca_btl_base_descriptor_t* descriptor,
-    mca_btl_base_tag_t tag
-);
-
+extern int mca_btl_template_send(struct mca_btl_base_module_t *btl,
+                                 struct mca_btl_base_endpoint_t *btl_peer,
+                                 struct mca_btl_base_descriptor_t *descriptor,
+                                 mca_btl_base_tag_t tag);
 
 /**
  * Initiate an asynchronous put.
@@ -221,11 +202,13 @@ extern int mca_btl_template_send(
  * @retval OPAL_ERR_NOT_AVAILABLE  Put can not be performed due to size or
  *                         alignment restrictions.
  */
-int mca_btl_template_put (struct mca_btl_base_module_t *btl,
-    struct mca_btl_base_endpoint_t *endpoint, void *local_address,
-    uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle,
-    struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-    int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
+int mca_btl_template_put(struct mca_btl_base_module_t *btl,
+                         struct mca_btl_base_endpoint_t *endpoint, void *local_address,
+                         uint64_t remote_address,
+                         struct mca_btl_base_registration_handle_t *local_handle,
+                         struct mca_btl_base_registration_handle_t *remote_handle, size_t size,
+                         int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
+                         void *cbcontext, void *cbdata);
 
 /**
  * Initiate an asynchronous get.
@@ -258,11 +241,13 @@ int mca_btl_template_put (struct mca_btl_base_module_t *btl,
  * @retval OPAL_ERR_NOT_AVAILABLE  Put can not be performed due to size or
  *                         alignment restrictions.
  */
-int mca_btl_template_get (struct mca_btl_base_module_t *btl,
-    struct mca_btl_base_endpoint_t *endpoint, void *local_address,
-    uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle,
-    struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-    int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
+int mca_btl_template_get(struct mca_btl_base_module_t *btl,
+                         struct mca_btl_base_endpoint_t *endpoint, void *local_address,
+                         uint64_t remote_address,
+                         struct mca_btl_base_registration_handle_t *local_handle,
+                         struct mca_btl_base_registration_handle_t *remote_handle, size_t size,
+                         int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
+                         void *cbcontext, void *cbdata);
 
 /**
  * @brief Register a memory region for put/get/atomic operations.
@@ -282,9 +267,10 @@ int mca_btl_template_get (struct mca_btl_base_module_t *btl,
  * functions. Care should be taken to not hold an excessive number of registrations
  * as they may use limited system/NIC resources.
  */
-struct mca_btl_base_registration_handle_t *mca_btl_template_register_mem (
-    struct mca_btl_base_module_t* btl, struct mca_btl_base_endpoint_t *endpoint, void *base,
-    size_t size, uint32_t flags);
+struct mca_btl_base_registration_handle_t *
+mca_btl_template_register_mem(struct mca_btl_base_module_t *btl,
+                              struct mca_btl_base_endpoint_t *endpoint, void *base, size_t size,
+                              uint32_t flags);
 
 /**
  * @brief Deregister a memory region
@@ -297,8 +283,8 @@ struct mca_btl_base_registration_handle_t *mca_btl_template_register_mem (
  * after it is deregistered. It is erroneous to specify a memory handle associated with
  * a remote node.
  */
-int mca_btl_template_deregister_mem (struct mca_btl_base_module_t* btl,
-                                     struct mca_btl_base_registration_handle_t *handle);
+int mca_btl_template_deregister_mem(struct mca_btl_base_module_t *btl,
+                                    struct mca_btl_base_registration_handle_t *handle);
 
 /**
  * Register a callback function that is called on receipt
@@ -309,11 +295,8 @@ int mca_btl_template_deregister_mem (struct mca_btl_base_module_t* btl,
  *
  */
 
-extern int mca_btl_template_register(
-    struct mca_btl_base_module_t* btl,
-    mca_btl_base_tag_t tag,
-    mca_btl_base_module_recv_cb_fn_t cbfunc,
-    void* cbdata);
+extern int mca_btl_template_register(struct mca_btl_base_module_t *btl, mca_btl_base_tag_t tag,
+                                     mca_btl_base_module_recv_cb_fn_t cbfunc, void *cbdata);
 
 /**
  * Allocate a descriptor with a segment of the requested size.
@@ -324,13 +307,10 @@ extern int mca_btl_template_register(
  * @param size (IN)     Request segment size.
  */
 
-extern mca_btl_base_descriptor_t* mca_btl_template_alloc(
-    struct mca_btl_base_module_t* btl,
-    struct mca_btl_base_endpoint_t* endpoint,
-    uint8_t order,
-    size_t size,
-    uint32_t flags);
-
+extern mca_btl_base_descriptor_t *mca_btl_template_alloc(struct mca_btl_base_module_t *btl,
+                                                         struct mca_btl_base_endpoint_t *endpoint,
+                                                         uint8_t order, size_t size,
+                                                         uint32_t flags);
 
 /**
  * Return a segment allocated by this BTL.
@@ -339,14 +319,11 @@ extern mca_btl_base_descriptor_t* mca_btl_template_alloc(
  * @param descriptor (IN)  Allocated descriptor.
  */
 
-extern int mca_btl_template_free(
-    struct mca_btl_base_module_t* btl,
-    mca_btl_base_descriptor_t* des);
-
+extern int mca_btl_template_free(struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des);
 
 /**
  * Prepare a descriptor for send/rdma using the supplied
- * convertor. If the convertor references data that is contigous,
+ * convertor. If the convertor references data that is contiguous,
  * the descriptor may simply point to the user buffer. Otherwise,
  * this routine is responsible for allocating buffer space and
  * packing if required.
@@ -356,17 +333,13 @@ extern int mca_btl_template_free(
  * @param convertor (IN)    Data type convertor
  * @param reserve (IN)      Additional bytes requested by upper layer to precede user data
  * @param size (IN/OUT)     Number of bytes to prepare (IN), number of bytes actually prepared (OUT)
-*/
-
-mca_btl_base_descriptor_t* mca_btl_template_prepare_src(
-    struct mca_btl_base_module_t* btl,
-    struct mca_btl_base_endpoint_t* peer,
-    struct opal_convertor_t* convertor,
-    uint8_t order,
-    size_t reserve,
-    size_t* size,
-    uint32_t flags
-);
+ */
+
+mca_btl_base_descriptor_t *mca_btl_template_prepare_src(struct mca_btl_base_module_t *btl,
+                                                        struct mca_btl_base_endpoint_t *peer,
+                                                        struct opal_convertor_t *convertor,
+                                                        uint8_t order, size_t reserve, size_t *size,
+                                                        uint32_t flags);
 
 END_C_DECLS
 #endif
diff --git a/opal/mca/btl/template/btl_template_component.c b/opal/mca/btl/template/btl_template_component.c
index 3b09fc9ca46..af71db179a0 100644
--- a/opal/mca/btl/template/btl_template_component.c
+++ b/opal/mca/btl/template/btl_template_component.c
@@ -14,6 +14,7 @@
  *                         reserved.
  * Copyright (c) 2020      Amazon.com, Inc. or its affiliates.
  *                         All Rights reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -21,17 +22,16 @@
  * $HEADER$
  */
 
-
 #include "opal_config.h"
 #include "opal/constants.h"
-#include "opal/util/event.h"
+#include "opal/mca/btl/base/base.h"
 #include "opal/mca/btl/btl.h"
 #include "opal/mca/mpool/base/base.h"
-#include "opal/mca/btl/base/base.h"
+#include "opal/util/event.h"
 
 #include "btl_template.h"
-#include "btl_template_frag.h"
 #include "btl_template_endpoint.h"
+#include "btl_template_frag.h"
 
 /**
  * Register any MCA parameters associated with this component
@@ -48,27 +48,25 @@ static int mca_btl_template_component_open(void);
  */
 static int mca_btl_template_component_close(void);
 
-
 mca_btl_template_component_t mca_btl_template_component = {
     .super = {
         /* First, the mca_base_component_t struct containing meta information
            about the component itself */
 
-        .btl_version = {
-            MCA_BTL_DEFAULT_VERSION("template"),
-            .mca_open_component = mca_btl_template_component_open,
-            .mca_close_component = mca_btl_template_component_close,
-            .mca_register_component_params = mca_btl_template_component_register,
-        },
-        .btl_data = {
-            /* The component is not checkpoint ready */
-            .param_field = MCA_BASE_METADATA_PARAM_NONE
-        },
+        .btl_version =
+            {
+                MCA_BTL_DEFAULT_VERSION("template"),
+                .mca_open_component = mca_btl_template_component_open,
+                .mca_close_component = mca_btl_template_component_close,
+                .mca_register_component_params = mca_btl_template_component_register,
+            },
+        .btl_data =
+            {/* The component is not checkpoint ready */
+             .param_field = MCA_BASE_METADATA_PARAM_NONE},
 
         .btl_init = mca_btl_template_component_init,
         .btl_progress = mca_btl_template_component_progress,
-    }
-};
+    }};
 
 static int mca_btl_template_component_open(void)
 {
@@ -78,8 +76,8 @@ static int mca_btl_template_component_open(void)
 static int mca_btl_template_component_register(void)
 {
     /* initialize state */
-    mca_btl_template_component.template_num_btls=0;
-    mca_btl_template_component.template_btls=NULL;
+    mca_btl_template_component.template_num_btls = 0;
+    mca_btl_template_component.template_btls = NULL;
 
     /* initialize objects */
     OBJ_CONSTRUCT(&mca_btl_template_component.template_procs, opal_list_t);
@@ -87,36 +85,32 @@ static int mca_btl_template_component_register(void)
     /* register TEMPLATE component parameters */
     mca_btl_template_component.template_free_list_num = 8;
     (void) mca_base_component_var_register(&mca_btl_template_component.super.btl_version,
-                                           "free_list_num", NULL, MCA_BASE_VAR_TYPE_INT,
-                                           NULL, 0, 0, OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           "free_list_num", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
                                            &mca_btl_template_component.template_free_list_num);
     (void) mca_base_component_var_register(&mca_btl_template_component.super.btl_version,
-                                           "free_list_max", NULL, MCA_BASE_VAR_TYPE_INT,
-                                           NULL, 0, 0, OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           "free_list_max", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
                                            &mca_btl_template_component.template_free_list_max);
     (void) mca_base_component_var_register(&mca_btl_template_component.super.btl_version,
-                                           "free_list_inc", NULL, MCA_BASE_VAR_TYPE_INT,
-                                           NULL, 0, 0, OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           "free_list_inc", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
                                            &mca_btl_template_component.template_free_list_inc);
 
     mca_btl_template_component.template_mpool_name = "grdma";
-    (void) mca_base_component_var_register(&mca_btl_template_component.super.btl_version,
-                                           "mpool", NULL, MCA_BASE_VAR_TYPE_STRING,
-                                           NULL, 0, 0, OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
+    (void) mca_base_component_var_register(&mca_btl_template_component.super.btl_version, "mpool",
+                                           NULL, MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
+                                           OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
                                            &mca_btl_template_component.template_mpool_name);
 
     mca_btl_template_module.super.btl_exclusivity = 0;
-    mca_btl_template_module.super.btl_eager_limit = 64*1024;
-    mca_btl_template_module.super.btl_rndv_eager_limit = 64*1024;
-    mca_btl_template_module.super.btl_max_send_size = 128*1024;
-    mca_btl_template_module.super.btl_min_rdma_pipeline_size = 1024*1024;
-    mca_btl_template_module.super.btl_rdma_pipeline_frag_size = 1024*1024;
-    mca_btl_template_module.super.btl_rdma_pipeline_send_length = 1024*1024;
-    mca_btl_template_module.super.btl_flags  = MCA_BTL_FLAGS_PUT;
+    mca_btl_template_module.super.btl_eager_limit = 64 * 1024;
+    mca_btl_template_module.super.btl_rndv_eager_limit = 64 * 1024;
+    mca_btl_template_module.super.btl_max_send_size = 128 * 1024;
+    mca_btl_template_module.super.btl_min_rdma_pipeline_size = 1024 * 1024;
+    mca_btl_template_module.super.btl_rdma_pipeline_frag_size = 1024 * 1024;
+    mca_btl_template_module.super.btl_rdma_pipeline_send_length = 1024 * 1024;
+    mca_btl_template_module.super.btl_flags = MCA_BTL_FLAGS_PUT;
 
     return mca_btl_base_param_register(&mca_btl_template_component.super.btl_version,
                                        &mca_btl_template_module.super);
@@ -139,9 +133,9 @@ static int mca_btl_template_component_close(void)
  *  (3) register BTL parameters with the MCA
  */
 
-mca_btl_base_module_t** mca_btl_template_component_init(int *num_btl_modules,
-                                                  bool enable_progress_threads,
-                                                  bool enable_mpi_threads)
+mca_btl_base_module_t **mca_btl_template_component_init(int *num_btl_modules,
+                                                        bool enable_progress_threads,
+                                                        bool enable_mpi_threads)
 {
     return NULL;
 }
@@ -150,9 +144,7 @@ mca_btl_base_module_t** mca_btl_template_component_init(int *num_btl_modules,
  *  TEMPLATE component progress.
  */
 
-
-int mca_btl_template_component_progress()
+int mca_btl_template_component_progress(void)
 {
     return 0;
 }
-
diff --git a/opal/mca/btl/template/btl_template_endpoint.c b/opal/mca/btl/template/btl_template_endpoint.c
index 71d205de07e..faa3b55d775 100644
--- a/opal/mca/btl/template/btl_template_endpoint.c
+++ b/opal/mca/btl/template/btl_template_endpoint.c
@@ -16,22 +16,20 @@
  * $HEADER$
  */
 
-
 #include "opal_config.h"
-#include <sys/time.h>
-#include <time.h>
-#include "btl_template.h"
 #include "btl_template_endpoint.h"
-#include "btl_template_proc.h"
+#include "btl_template.h"
 #include "btl_template_frag.h"
-
+#include "btl_template_proc.h"
+#include <sys/time.h>
+#include <time.h>
 
 /*
  * Initialize state of the endpoint instance.
  *
  */
 
-static void mca_btl_template_endpoint_construct(mca_btl_base_endpoint_t* endpoint)
+static void mca_btl_template_endpoint_construct(mca_btl_base_endpoint_t *endpoint)
 {
     endpoint->endpoint_btl = 0;
     endpoint->endpoint_proc = 0;
@@ -42,14 +40,9 @@ static void mca_btl_template_endpoint_construct(mca_btl_base_endpoint_t* endpoin
  *
  */
 
-static void mca_btl_template_endpoint_destruct(mca_btl_base_endpoint_t* endpoint)
+static void mca_btl_template_endpoint_destruct(mca_btl_base_endpoint_t *endpoint)
 {
 }
 
-
-OBJ_CLASS_INSTANCE(
-    mca_btl_template_endpoint_t,
-    opal_list_item_t,
-    mca_btl_template_endpoint_construct,
-    mca_btl_template_endpoint_destruct);
-
+OBJ_CLASS_INSTANCE(mca_btl_template_endpoint_t, opal_list_item_t,
+                   mca_btl_template_endpoint_construct, mca_btl_template_endpoint_destruct);
diff --git a/opal/mca/btl/template/btl_template_endpoint.h b/opal/mca/btl/template/btl_template_endpoint.h
index 5ab253b2ab8..9828e56d814 100644
--- a/opal/mca/btl/template/btl_template_endpoint.h
+++ b/opal/mca/btl/template/btl_template_endpoint.h
@@ -21,10 +21,10 @@
 #ifndef MCA_BTL_TEMPLATE_ENDPOINT_H
 #define MCA_BTL_TEMPLATE_ENDPOINT_H
 
+#include "btl_template.h"
+#include "btl_template_frag.h"
 #include "opal/class/opal_list.h"
 #include "opal/util/event.h"
-#include "btl_template_frag.h"
-#include "btl_template.h"
 BEGIN_C_DECLS
 
 /**
@@ -35,17 +35,17 @@ BEGIN_C_DECLS
  */
 
 struct mca_btl_base_endpoint_t {
-    opal_list_item_t            super;
+    opal_list_item_t super;
 
-    struct mca_btl_template_module_t* endpoint_btl;
+    struct mca_btl_template_module_t *endpoint_btl;
     /**< BTL instance that created this connection */
 
-    struct mca_btl_template_proc_t*   endpoint_proc;
+    struct mca_btl_template_proc_t *endpoint_proc;
     /**< proc structure corresponding to endpoint */
 };
 
 typedef struct mca_btl_base_endpoint_t mca_btl_base_endpoint_t;
-typedef mca_btl_base_endpoint_t  mca_btl_template_endpoint_t;
+typedef mca_btl_base_endpoint_t mca_btl_template_endpoint_t;
 OBJ_CLASS_DECLARATION(mca_btl_template_endpoint_t);
 
 END_C_DECLS
diff --git a/opal/mca/btl/template/btl_template_frag.c b/opal/mca/btl/template/btl_template_frag.c
index b22d38138ce..1859ecb903f 100644
--- a/opal/mca/btl/template/btl_template_frag.c
+++ b/opal/mca/btl/template/btl_template_frag.c
@@ -1,47 +1,30 @@
 #include "btl_template_frag.h"
 
-
-
-static void mca_btl_template_frag_eager_constructor(mca_btl_template_frag_t* frag)
+static void mca_btl_template_frag_eager_constructor(mca_btl_template_frag_t *frag)
 {
     frag->registration = NULL;
     frag->size = mca_btl_template_module.super.btl_eager_limit;
 }
 
-static void mca_btl_template_frag_max_constructor(mca_btl_template_frag_t* frag)
+static void mca_btl_template_frag_max_constructor(mca_btl_template_frag_t *frag)
 {
     frag->registration = NULL;
     frag->size = mca_btl_template_module.super.btl_max_send_size;
 }
 
-static void mca_btl_template_frag_user_constructor(mca_btl_template_frag_t* frag)
+static void mca_btl_template_frag_user_constructor(mca_btl_template_frag_t *frag)
 {
     frag->size = 0;
     frag->registration = NULL;
 }
 
+OBJ_CLASS_INSTANCE(mca_btl_template_frag_t, mca_btl_base_descriptor_t, NULL, NULL);
 
-OBJ_CLASS_INSTANCE(
-    mca_btl_template_frag_t,
-    mca_btl_base_descriptor_t,
-    NULL,
-    NULL);
-
-OBJ_CLASS_INSTANCE(
-    mca_btl_template_frag_eager_t,
-    mca_btl_base_descriptor_t,
-    mca_btl_template_frag_eager_constructor,
-    NULL);
-
-OBJ_CLASS_INSTANCE(
-    mca_btl_template_frag_max_t,
-    mca_btl_base_descriptor_t,
-    mca_btl_template_frag_max_constructor,
-    NULL);
+OBJ_CLASS_INSTANCE(mca_btl_template_frag_eager_t, mca_btl_base_descriptor_t,
+                   mca_btl_template_frag_eager_constructor, NULL);
 
-OBJ_CLASS_INSTANCE(
-    mca_btl_template_frag_user_t,
-    mca_btl_base_descriptor_t,
-    mca_btl_template_frag_user_constructor,
-    NULL);
+OBJ_CLASS_INSTANCE(mca_btl_template_frag_max_t, mca_btl_base_descriptor_t,
+                   mca_btl_template_frag_max_constructor, NULL);
 
+OBJ_CLASS_INSTANCE(mca_btl_template_frag_user_t, mca_btl_base_descriptor_t,
+                   mca_btl_template_frag_user_constructor, NULL);
diff --git a/opal/mca/btl/template/btl_template_frag.h b/opal/mca/btl/template/btl_template_frag.h
index aab1f1502ca..5f53e01dc74 100644
--- a/opal/mca/btl/template/btl_template_frag.h
+++ b/opal/mca/btl/template/btl_template_frag.h
@@ -22,7 +22,6 @@
 #ifndef MCA_BTL_TEMPLATE_FRAG_H
 #define MCA_BTL_TEMPLATE_FRAG_H
 
-
 #define MCA_BTL_TEMPLATE_FRAG_ALIGN (8)
 #include "opal_config.h"
 #include "btl_template.h"
@@ -39,7 +38,7 @@ struct mca_btl_template_frag_t {
     mca_btl_base_header_t *hdr;
     size_t size;
 #if MCA_BTL_HAS_MPOOL
-    struct mca_mpool_base_registration_t* registration;
+    struct mca_mpool_base_registration_t *registration;
 #endif
 };
 typedef struct mca_btl_template_frag_t mca_btl_template_frag_t;
@@ -57,50 +56,46 @@ typedef struct mca_btl_template_frag_t mca_btl_template_frag_user_t;
 
 OBJ_CLASS_DECLARATION(mca_btl_template_frag_user_t);
 
-
 /*
  * Macros to allocate/return descriptors from module specific
  * free list(s).
  */
 
 #define MCA_BTL_TEMPLATE_FRAG_ALLOC_EAGER(btl, frag)                    \
-{                                                                       \
-    frag = (mca_btl_template_frag_t *)                                  \
-        opal_free_list_get (&((mca_btl_template_module_t*)btl)->template_frag_eager); \
-}
-
-#define MCA_BTL_TEMPLATE_FRAG_RETURN_EAGER(btl, frag)                   \
-{                                                                       \
-    opal_free_list_return (&((mca_btl_template_module_t*)btl)->template_frag_eager, \
-                           (opal_free_list_item_t*)(frag));             \
-}
-
-#define MCA_BTL_TEMPLATE_FRAG_ALLOC_MAX(btl, frag)                      \
-{                                                                       \
-    frag = (mca_btl_template_frag_t *)                                  \
-        opal_free_list_get (&((mca_btl_template_module_t*)btl)->template_frag_max); \
-}
-
-#define MCA_BTL_TEMPLATE_FRAG_RETURN_MAX(btl, frag)                     \
-{                                                                       \
-    opal_free_list_return (&((mca_btl_template_module_t*)btl)->template_frag_max, \
-                           (opal_free_list_item_t*)(frag));             \
-}
-
-
-#define MCA_BTL_TEMPLATE_FRAG_ALLOC_USER(btl, frag)                     \
-{                                                                       \
-    frag = (mca_btl_template_frag_t*)                                   \
-            opal_free_list_get (&((mca_btl_template_module_t*)btl)->template_frag_user); \
-}
-
-#define MCA_BTL_TEMPLATE_FRAG_RETURN_USER(btl, frag)                    \
-{                                                                       \
-    opal_free_list_return (&((mca_btl_template_module_t*)btl)->template_frag_user, \
-                           (opal_free_list_item_t*)(frag));             \
-}
-
-
+    {                                                                   \
+        frag = (mca_btl_template_frag_t *) opal_free_list_get(          \
+            &((mca_btl_template_module_t *) btl)->template_frag_eager); \
+    }
+
+#define MCA_BTL_TEMPLATE_FRAG_RETURN_EAGER(btl, frag)                                    \
+    {                                                                                    \
+        opal_free_list_return(&((mca_btl_template_module_t *) btl)->template_frag_eager, \
+                              (opal_free_list_item_t *) (frag));                         \
+    }
+
+#define MCA_BTL_TEMPLATE_FRAG_ALLOC_MAX(btl, frag)                    \
+    {                                                                 \
+        frag = (mca_btl_template_frag_t *) opal_free_list_get(        \
+            &((mca_btl_template_module_t *) btl)->template_frag_max); \
+    }
+
+#define MCA_BTL_TEMPLATE_FRAG_RETURN_MAX(btl, frag)                                    \
+    {                                                                                  \
+        opal_free_list_return(&((mca_btl_template_module_t *) btl)->template_frag_max, \
+                              (opal_free_list_item_t *) (frag));                       \
+    }
+
+#define MCA_BTL_TEMPLATE_FRAG_ALLOC_USER(btl, frag)                    \
+    {                                                                  \
+        frag = (mca_btl_template_frag_t *) opal_free_list_get(         \
+            &((mca_btl_template_module_t *) btl)->template_frag_user); \
+    }
+
+#define MCA_BTL_TEMPLATE_FRAG_RETURN_USER(btl, frag)                                    \
+    {                                                                                   \
+        opal_free_list_return(&((mca_btl_template_module_t *) btl)->template_frag_user, \
+                              (opal_free_list_item_t *) (frag));                        \
+    }
 
 END_C_DECLS
 #endif
diff --git a/opal/mca/btl/template/btl_template_proc.c b/opal/mca/btl/template/btl_template_proc.c
index 76f6f52f677..8c70cd3ffe9 100644
--- a/opal/mca/btl/template/btl_template_proc.c
+++ b/opal/mca/btl/template/btl_template_proc.c
@@ -18,18 +18,16 @@
 
 #include "opal_config.h"
 
-
 #include "btl_template.h"
 #include "btl_template_proc.h"
 
-static void mca_btl_template_proc_construct(mca_btl_template_proc_t* proc);
-static void mca_btl_template_proc_destruct(mca_btl_template_proc_t* proc);
+static void mca_btl_template_proc_construct(mca_btl_template_proc_t *proc);
+static void mca_btl_template_proc_destruct(mca_btl_template_proc_t *proc);
 
-OBJ_CLASS_INSTANCE(mca_btl_template_proc_t,
-        opal_list_item_t, mca_btl_template_proc_construct,
-        mca_btl_template_proc_destruct);
+OBJ_CLASS_INSTANCE(mca_btl_template_proc_t, opal_list_item_t, mca_btl_template_proc_construct,
+                   mca_btl_template_proc_destruct);
 
-void mca_btl_template_proc_construct(mca_btl_template_proc_t* template_proc)
+void mca_btl_template_proc_construct(mca_btl_template_proc_t *template_proc)
 {
     template_proc->proc_opal = 0;
     template_proc->proc_addr_count = 0;
@@ -46,43 +44,41 @@ void mca_btl_template_proc_construct(mca_btl_template_proc_t* template_proc)
  * Cleanup ib proc instance
  */
 
-void mca_btl_template_proc_destruct(mca_btl_template_proc_t* template_proc)
+void mca_btl_template_proc_destruct(mca_btl_template_proc_t *template_proc)
 {
     /* remove from list of all proc instances */
     OPAL_THREAD_LOCK(&mca_btl_template_component.template_lock);
-    opal_list_remove_item(&mca_btl_template_component.template_procs,
-                          &template_proc->super);
+    opal_list_remove_item(&mca_btl_template_component.template_procs, &template_proc->super);
     OPAL_THREAD_UNLOCK(&mca_btl_template_component.template_lock);
 
     /* release resources */
-    if(NULL != template_proc->proc_endpoints) {
+    if (NULL != template_proc->proc_endpoints) {
         free(template_proc->proc_endpoints);
     }
     OBJ_DESTRUCT(&template_proc->proc_lock);
 }
 
-
 /*
  * Look for an existing TEMPLATE process instances based on the associated
  * opal_proc_t instance.
  */
-static mca_btl_template_proc_t* mca_btl_template_proc_lookup_opal(opal_proc_t* opal_proc)
+static mca_btl_template_proc_t *mca_btl_template_proc_lookup_opal(opal_proc_t *opal_proc)
 {
-    mca_btl_template_proc_t* template_proc;
+    mca_btl_template_proc_t *template_proc;
 
     OPAL_THREAD_LOCK(&mca_btl_template_component.template_lock);
 
-    for(template_proc = (mca_btl_template_proc_t*)
-            opal_list_get_first(&mca_btl_template_component.template_procs);
-            template_proc != (mca_btl_template_proc_t*)
-            opal_list_get_end(&mca_btl_template_component.template_procs);
-            template_proc  = (mca_btl_template_proc_t*)opal_list_get_next(template_proc)) {
+    for (template_proc = (mca_btl_template_proc_t *) opal_list_get_first(
+             &mca_btl_template_component.template_procs);
+         template_proc
+         != (mca_btl_template_proc_t *) opal_list_get_end(
+             &mca_btl_template_component.template_procs);
+         template_proc = (mca_btl_template_proc_t *) opal_list_get_next(template_proc)) {
 
-        if(template_proc->proc_opal == opal_proc) {
+        if (template_proc->proc_opal == opal_proc) {
             OPAL_THREAD_UNLOCK(&mca_btl_template_component.template_lock);
             return template_proc;
         }
-
     }
 
     OPAL_THREAD_UNLOCK(&mca_btl_template_component.template_lock);
@@ -98,15 +94,15 @@ static mca_btl_template_proc_t* mca_btl_template_proc_lookup_opal(opal_proc_t* o
  * datastructure.
  */
 
-mca_btl_template_proc_t* mca_btl_template_proc_create(opal_proc_t* opal_proc)
+mca_btl_template_proc_t *mca_btl_template_proc_create(opal_proc_t *opal_proc)
 {
-    mca_btl_template_proc_t* module_proc = NULL;
+    mca_btl_template_proc_t *module_proc = NULL;
 
     /* Check if we have already created a TEMPLATE proc
      * structure for this process */
     module_proc = mca_btl_template_proc_lookup_opal(opal_proc);
 
-    if(module_proc != NULL) {
+    if (module_proc != NULL) {
 
         /* Gotcha! */
         return module_proc;
@@ -131,24 +127,23 @@ mca_btl_template_proc_t* mca_btl_template_proc_create(opal_proc_t* opal_proc)
      * mca_btl_template_proc_t to allow on demand increasing of
      * number of endpoints for this proc */
 
-    module_proc->proc_endpoints = (mca_btl_base_endpoint_t**)
-        malloc(module_proc->proc_addr_count * sizeof(mca_btl_base_endpoint_t*));
+    module_proc->proc_endpoints = (mca_btl_base_endpoint_t **) malloc(
+        module_proc->proc_addr_count * sizeof(mca_btl_base_endpoint_t *));
 
-    if(NULL == module_proc->proc_endpoints) {
+    if (NULL == module_proc->proc_endpoints) {
         OBJ_RELEASE(module_proc);
         return NULL;
     }
     return module_proc;
 }
 
-
 /*
  * Note that this routine must be called with the lock on the process
  * already held.  Insert a btl instance into the proc array and assign
  * it an address.
  */
-int mca_btl_template_proc_insert(mca_btl_template_proc_t* module_proc,
-        mca_btl_base_endpoint_t* module_endpoint)
+int mca_btl_template_proc_insert(mca_btl_template_proc_t *module_proc,
+                                 mca_btl_base_endpoint_t *module_endpoint)
 {
     /* insert into endpoint array */
     module_endpoint->endpoint_proc = module_proc;
diff --git a/opal/mca/btl/template/btl_template_proc.h b/opal/mca/btl/template/btl_template_proc.h
index 5e19d8400c8..35669c7ea46 100644
--- a/opal/mca/btl/template/btl_template_proc.h
+++ b/opal/mca/btl/template/btl_template_proc.h
@@ -19,10 +19,10 @@
 #ifndef MCA_BTL_TEMPLATE_PROC_H
 #define MCA_BTL_TEMPLATE_PROC_H
 
-#include "opal/class/opal_object.h"
-#include "opal/util/proc.h"
 #include "btl_template.h"
 #include "btl_template_endpoint.h"
+#include "opal/class/opal_object.h"
+#include "opal/util/proc.h"
 
 BEGIN_C_DECLS
 
@@ -54,8 +54,8 @@ struct mca_btl_template_proc_t {
 typedef struct mca_btl_template_proc_t mca_btl_template_proc_t;
 OBJ_CLASS_DECLARATION(mca_btl_template_proc_t);
 
-mca_btl_template_proc_t* mca_btl_template_proc_create(opal_proc_t* proc);
-int mca_btl_template_proc_insert(mca_btl_template_proc_t*, mca_btl_base_endpoint_t*);
+mca_btl_template_proc_t *mca_btl_template_proc_create(opal_proc_t *proc);
+int mca_btl_template_proc_insert(mca_btl_template_proc_t *, mca_btl_base_endpoint_t *);
 
 END_C_DECLS
 #endif
diff --git a/opal/mca/btl/uct/Makefile.am b/opal/mca/btl/uct/Makefile.am
index e1015f2823e..df548cc66ff 100644
--- a/opal/mca/btl/uct/Makefile.am
+++ b/opal/mca/btl/uct/Makefile.am
@@ -61,7 +61,7 @@ mcacomponentdir = $(opallibdir)
 mcacomponent_LTLIBRARIES = $(component)
 mca_btl_uct_la_SOURCES = $(component_sources)
 mca_btl_uct_la_LDFLAGS = -module -avoid-version $(btl_uct_LDFLAGS)
-mca_btl_uct_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la $(btl_uct_LIBS)
+mca_btl_uct_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la $(btl_uct_LIBS)
 
 noinst_LTLIBRARIES = $(lib)
 libmca_btl_uct_la_SOURCES = $(lib_sources)
diff --git a/opal/mca/btl/uct/btl_uct.h b/opal/mca/btl/uct/btl_uct.h
index 2a38138c00a..65bc69fddb2 100644
--- a/opal/mca/btl/uct/btl_uct.h
+++ b/opal/mca/btl/uct/btl_uct.h
@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights
  *                         reserved.
- * Copyright (c) 2019      Google, LLC. All rights reserved.
+ * Copyright (c) 2019-2025 Google, LLC. All rights reserved.
  * Copyright (c) 2019      Intel, Inc.  All rights reserved.
  * Copyright (c) 2020      Amazon.com, Inc. or its affiliates.
  *                         All Rights reserved.
@@ -29,19 +29,21 @@
 #define MCA_BTL_UCT_H
 
 #include "opal_config.h"
-#include <sys/types.h>
 #include <string.h>
+#include <sys/types.h>
 
 /* Open MPI includes */
-#include "opal/util/event.h"
-#include "opal/mca/btl/base/base.h"
-#include "opal/mca/mpool/mpool.h"
-#include "opal/mca/btl/base/btl_base_error.h"
-#include "opal/mca/rcache/base/base.h"
 #include "opal/class/opal_fifo.h"
 #include "opal/class/opal_hash_table.h"
+#include "opal/mca/btl/base/base.h"
+#include "opal/mca/btl/base/btl_base_error.h"
+#include "opal/mca/mpool/mpool.h"
 #include "opal/mca/pmix/pmix-internal.h"
+#include "opal/mca/rcache/base/base.h"
+#include "opal/mca/threads/condition.h"
+#include "opal/mca/threads/mutex.h"
 #include "opal/mca/threads/tsd.h"
+#include "opal/util/event.h"
 #include <uct/api/uct.h>
 
 #include "btl_uct_types.h"
@@ -50,9 +52,9 @@ BEGIN_C_DECLS
 
 /* detection for old vs new atomic flags */
 #if defined(UCT_IFACE_FLAG_ATOMIC_ADD32)
-#define OPAL_HAVE_UCT_EP_ATOMIC64_POST 0
+#    define OPAL_HAVE_UCT_EP_ATOMIC64_POST 0
 #else
-#define OPAL_HAVE_UCT_EP_ATOMIC64_POST 1
+#    define OPAL_HAVE_UCT_EP_ATOMIC64_POST 1
 #endif
 
 /**
@@ -153,10 +155,13 @@ struct mca_btl_uct_component_t {
 
     /** disable UCX memory hooks */
     bool disable_ucx_memory_hooks;
+
+    /** connection retry timeout */
+    unsigned int connection_retry_timeout;
 };
 typedef struct mca_btl_uct_component_t mca_btl_uct_component_t;
 
-OPAL_MODULE_DECLSPEC extern mca_btl_uct_component_t mca_btl_uct_component;
+OPAL_DECLSPEC extern mca_btl_uct_component_t mca_btl_uct_component;
 
 struct mca_btl_base_registration_handle_t {
     /** The packed memory handle. The size of this field is defined by UCT. */
@@ -176,7 +181,8 @@ typedef struct mca_btl_uct_reg_t mca_btl_uct_reg_t;
 
 OBJ_CLASS_DECLARATION(mca_btl_uct_reg_t);
 
-#define MCA_BTL_UCT_REG_REMOTE_TO_LOCAL(reg) ((mca_btl_uct_reg_t *)((intptr_t) (reg) - offsetof (mca_btl_uct_reg_t, handle)))
+#define MCA_BTL_UCT_REG_REMOTE_TO_LOCAL(reg) \
+    ((mca_btl_uct_reg_t *) ((intptr_t)(reg) -offsetof(mca_btl_uct_reg_t, handle)))
 
 /**
  * Initiate an asynchronous put.
@@ -209,11 +215,12 @@ OBJ_CLASS_DECLARATION(mca_btl_uct_reg_t);
  * @retval OPAL_ERR_NOT_AVAILABLE  Put can not be performed due to size or
  *                         alignment restrictions.
  */
-int mca_btl_uct_put (struct mca_btl_base_module_t *btl,
-    struct mca_btl_base_endpoint_t *endpoint, void *local_address,
-    uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle,
-    struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-    int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
+int mca_btl_uct_put(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+                    void *local_address, uint64_t remote_address,
+                    struct mca_btl_base_registration_handle_t *local_handle,
+                    struct mca_btl_base_registration_handle_t *remote_handle, size_t size,
+                    int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext,
+                    void *cbdata);
 
 /**
  * Initiate an asynchronous get.
@@ -246,61 +253,70 @@ int mca_btl_uct_put (struct mca_btl_base_module_t *btl,
  * @retval OPAL_ERR_NOT_AVAILABLE  Put can not be performed due to size or
  *                         alignment restrictions.
  */
-int mca_btl_uct_get (struct mca_btl_base_module_t *btl,
-    struct mca_btl_base_endpoint_t *endpoint, void *local_address,
-    uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle,
-    struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-    int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
-
-int mca_btl_uct_aop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
-                     uint64_t remote_address, mca_btl_base_registration_handle_t *remote_handle,
-                     mca_btl_base_atomic_op_t op, uint64_t operand, int flags, int order,
+int mca_btl_uct_get(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+                    void *local_address, uint64_t remote_address,
+                    struct mca_btl_base_registration_handle_t *local_handle,
+                    struct mca_btl_base_registration_handle_t *remote_handle, size_t size,
+                    int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext,
+                    void *cbdata);
+
+int mca_btl_uct_aop(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+                    uint64_t remote_address, mca_btl_base_registration_handle_t *remote_handle,
+                    mca_btl_base_atomic_op_t op, uint64_t operand, int flags, int order,
+                    mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
+
+int mca_btl_uct_afop(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+                     void *local_address, uint64_t remote_address,
+                     mca_btl_base_registration_handle_t *local_handle,
+                     mca_btl_base_registration_handle_t *remote_handle, mca_btl_base_atomic_op_t op,
+                     uint64_t operand, int flags, int order,
                      mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
 
-int mca_btl_uct_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
-                      void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                      mca_btl_base_registration_handle_t *remote_handle, mca_btl_base_atomic_op_t op,
-                      uint64_t operand, int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
-                      void *cbcontext, void *cbdata);
-
-int mca_btl_uct_acswap (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
-                        void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                        mca_btl_base_registration_handle_t *remote_handle, uint64_t compare, uint64_t value, int flags,
-                        int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
-
+int mca_btl_uct_acswap(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+                       void *local_address, uint64_t remote_address,
+                       mca_btl_base_registration_handle_t *local_handle,
+                       mca_btl_base_registration_handle_t *remote_handle, uint64_t compare,
+                       uint64_t value, int flags, int order,
+                       mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
 
-int mca_btl_uct_flush (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint);
-int mca_btl_uct_flush_thread (mca_btl_base_module_t *btl);
+int mca_btl_uct_flush(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint);
+int mca_btl_uct_flush_thread(mca_btl_base_module_t *btl);
 
-int mca_btl_uct_finalize (mca_btl_base_module_t *btl);
+int mca_btl_uct_finalize(mca_btl_base_module_t *btl);
 
-int mca_btl_uct_reg_mem (void *reg_data, void *base, size_t size, mca_rcache_base_registration_t *reg);
-int mca_btl_uct_dereg_mem (void *reg_data, mca_rcache_base_registration_t *reg);
+int mca_btl_uct_reg_mem(void *reg_data, void *base, size_t size,
+                        mca_rcache_base_registration_t *reg);
+int mca_btl_uct_dereg_mem(void *reg_data, mca_rcache_base_registration_t *reg);
 
-ucs_status_t mca_btl_uct_am_handler (void *arg, void *data, size_t length, unsigned flags);
+ucs_status_t mca_btl_uct_am_handler(void *arg, void *data, size_t length, unsigned flags);
 
-struct mca_btl_base_endpoint_t *mca_btl_uct_get_ep (struct mca_btl_base_module_t *module, opal_proc_t *proc);
+struct mca_btl_base_endpoint_t *mca_btl_uct_get_ep(struct mca_btl_base_module_t *module,
+                                                   opal_proc_t *proc);
 
-int mca_btl_uct_query_tls (mca_btl_uct_module_t *module, mca_btl_uct_md_t *md, uct_tl_resource_desc_t *tl_descs, unsigned tl_count);
-int mca_btl_uct_process_connection_request (mca_btl_uct_module_t *module, mca_btl_uct_conn_req_t *req);
+int mca_btl_uct_query_tls(mca_btl_uct_module_t *module, mca_btl_uct_md_t *md,
+                          uct_tl_resource_desc_t *tl_descs, unsigned tl_count);
+int mca_btl_uct_process_connection_request(mca_btl_uct_module_t *module,
+                                           mca_btl_uct_conn_req_t *req);
 
 /**
  * @brief Checks if a tl is suitable for using for RDMA
  *
  * @param[in] tl  btl/uct tl pointer
  */
-static inline bool mca_btl_uct_tl_supports_rdma (mca_btl_uct_tl_t *tl)
+static inline bool mca_btl_uct_tl_supports_rdma(mca_btl_uct_tl_t *tl)
 {
-    return (MCA_BTL_UCT_TL_ATTR(tl, 0).cap.flags & (UCT_IFACE_FLAG_PUT_ZCOPY | UCT_IFACE_FLAG_GET_ZCOPY)) ==
-        (UCT_IFACE_FLAG_PUT_ZCOPY | UCT_IFACE_FLAG_GET_ZCOPY);
+    return (MCA_BTL_UCT_TL_ATTR(tl, 0).cap.flags
+            & (UCT_IFACE_FLAG_PUT_ZCOPY | UCT_IFACE_FLAG_GET_ZCOPY))
+           == (UCT_IFACE_FLAG_PUT_ZCOPY | UCT_IFACE_FLAG_GET_ZCOPY);
 }
 
 /**
  * @brief Checks if a tl is suitable for using for active messaging
  */
-static inline bool mca_btl_uct_tl_support_am (mca_btl_uct_tl_t *tl)
+static inline bool mca_btl_uct_tl_support_am(mca_btl_uct_tl_t *tl)
 {
-    return (MCA_BTL_UCT_TL_ATTR(tl, 0).cap.flags & (UCT_IFACE_FLAG_AM_SHORT | UCT_IFACE_FLAG_AM_BCOPY | UCT_IFACE_FLAG_AM_ZCOPY));
+    return (MCA_BTL_UCT_TL_ATTR(tl, 0).cap.flags
+            & (UCT_IFACE_FLAG_AM_SHORT | UCT_IFACE_FLAG_AM_BCOPY | UCT_IFACE_FLAG_AM_ZCOPY));
 }
 
 /**
@@ -308,10 +324,11 @@ static inline bool mca_btl_uct_tl_support_am (mca_btl_uct_tl_t *tl)
  *
  * @param[in] tl  btl/uct tl pointer
  */
-static inline bool mca_btl_uct_tl_supports_conn (mca_btl_uct_tl_t *tl)
+static inline bool mca_btl_uct_tl_supports_conn(mca_btl_uct_tl_t *tl)
 {
-    return (MCA_BTL_UCT_TL_ATTR(tl, 0).cap.flags & (UCT_IFACE_FLAG_AM_SHORT | UCT_IFACE_FLAG_CONNECT_TO_IFACE)) ==
-        (UCT_IFACE_FLAG_AM_SHORT | UCT_IFACE_FLAG_CONNECT_TO_IFACE);
+    return (MCA_BTL_UCT_TL_ATTR(tl, 0).cap.flags
+            & (UCT_IFACE_FLAG_AM_SHORT | UCT_IFACE_FLAG_CONNECT_TO_IFACE))
+           == (UCT_IFACE_FLAG_AM_SHORT | UCT_IFACE_FLAG_CONNECT_TO_IFACE);
 }
 
 /**
@@ -319,7 +336,7 @@ static inline bool mca_btl_uct_tl_supports_conn (mca_btl_uct_tl_t *tl)
  *
  * @param[in] tl  btl/uct tl pointer
  */
-static inline bool mca_btl_uct_tl_requires_connection_tl (mca_btl_uct_tl_t *tl)
+static inline bool mca_btl_uct_tl_requires_connection_tl(mca_btl_uct_tl_t *tl)
 {
     return !(MCA_BTL_UCT_TL_ATTR(tl, 0).cap.flags & UCT_IFACE_FLAG_CONNECT_TO_IFACE);
 }
diff --git a/opal/mca/btl/uct/btl_uct_am.c b/opal/mca/btl/uct/btl_uct_am.c
index 08f2b21ba8a..1aae456842c 100644
--- a/opal/mca/btl/uct/btl_uct_am.c
+++ b/opal/mca/btl/uct/btl_uct_am.c
@@ -10,8 +10,8 @@
  */
 
 #include "btl_uct_am.h"
-#include "btl_uct_rdma.h"
 #include "btl_uct_device_context.h"
+#include "btl_uct_rdma.h"
 
 /**
  * Allocate a segment.
@@ -19,26 +19,27 @@
  * @param btl (IN)      BTL module
  * @param size (IN)     Request segment size.
  */
-mca_btl_base_descriptor_t *mca_btl_uct_alloc (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
-                                              uint8_t order, size_t size, uint32_t flags)
+mca_btl_base_descriptor_t *mca_btl_uct_alloc(mca_btl_base_module_t *btl,
+                                             mca_btl_base_endpoint_t *endpoint, uint8_t order,
+                                             size_t size, uint32_t flags)
 {
     mca_btl_uct_module_t *uct_btl = (mca_btl_uct_module_t *) btl;
     mca_btl_uct_base_frag_t *frag = NULL;
 
     if (size <= (size_t) MCA_BTL_UCT_TL_ATTR(uct_btl->am_tl, 0).cap.am.max_short) {
-        frag = mca_btl_uct_frag_alloc_short (uct_btl, endpoint);
+        frag = mca_btl_uct_frag_alloc_short(uct_btl, endpoint);
     } else if (size <= uct_btl->super.btl_eager_limit) {
-        frag = mca_btl_uct_frag_alloc_eager (uct_btl, endpoint);
+        frag = mca_btl_uct_frag_alloc_eager(uct_btl, endpoint);
     } else {
-        frag = mca_btl_uct_frag_alloc_max (uct_btl, endpoint);
+        frag = mca_btl_uct_frag_alloc_max(uct_btl, endpoint);
     }
 
     if (OPAL_LIKELY(frag != NULL)) {
-        frag->segments[0].seg_len  = size;
+        frag->segments[0].seg_len = size;
 
         frag->base.des_segment_count = 1;
-        frag->base.des_flags   = flags;
-        frag->base.order       = order;
+        frag->base.des_flags = flags;
+        frag->base.order = order;
         frag->uct_iov.length = size;
         if (NULL != frag->base.super.registration) {
             /* zero-copy fragments will need callbacks */
@@ -49,32 +50,30 @@ mca_btl_base_descriptor_t *mca_btl_uct_alloc (mca_btl_base_module_t *btl, mca_bt
     return (mca_btl_base_descriptor_t *) frag;
 }
 
-static inline void _mca_btl_uct_send_pack (void *data, void *header, size_t header_size, opal_convertor_t *convertor,
-                                           size_t payload_size)
+static inline void _mca_btl_uct_send_pack(void *data, void *header, size_t header_size,
+                                          opal_convertor_t *convertor, size_t* payload_size)
 {
     uint32_t iov_count = 1;
     struct iovec iov;
     size_t length;
 
     if (header_size > 0) {
-        assert (NULL != header);
-        memcpy (data, header, header_size);
+        assert(NULL != header);
+        memcpy(data, header, header_size);
     }
 
     /* pack the data into the supplied buffer */
     iov.iov_base = (IOVBASE_TYPE *) ((intptr_t) data + header_size);
-    iov.iov_len  = length = payload_size;
-
-    (void) opal_convertor_pack (convertor, &iov, &iov_count, &length);
+    iov.iov_len = *payload_size;
 
-    assert (length == payload_size);
+    (void) opal_convertor_pack(convertor, &iov, &iov_count, payload_size);
 }
 
-struct mca_btl_base_descriptor_t *mca_btl_uct_prepare_src (mca_btl_base_module_t *btl,
-                                                           mca_btl_base_endpoint_t *endpoint,
-                                                           opal_convertor_t *convertor,
-                                                           uint8_t order, size_t reserve,
-                                                           size_t *size, uint32_t flags)
+struct mca_btl_base_descriptor_t *mca_btl_uct_prepare_src(mca_btl_base_module_t *btl,
+                                                          mca_btl_base_endpoint_t *endpoint,
+                                                          opal_convertor_t *convertor,
+                                                          uint8_t order, size_t reserve,
+                                                          size_t *size, uint32_t flags)
 {
     mca_btl_uct_module_t *uct_btl = (mca_btl_uct_module_t *) btl;
     const size_t total_size = reserve + *size;
@@ -82,26 +81,31 @@ struct mca_btl_base_descriptor_t *mca_btl_uct_prepare_src (mca_btl_base_module_t
     void *data_ptr;
 
     /* in place send fragment */
-    if (OPAL_UNLIKELY(opal_convertor_need_buffers(convertor) || total_size > uct_btl->super.btl_eager_limit)) {
-        frag = (mca_btl_uct_base_frag_t *) mca_btl_uct_alloc (btl, endpoint, order, total_size, flags);
+    if (OPAL_UNLIKELY(opal_convertor_need_buffers(convertor)
+                      || total_size > uct_btl->super.btl_eager_limit)) {
+        frag = (mca_btl_uct_base_frag_t *) mca_btl_uct_alloc(btl, endpoint, order, total_size,
+                                                             flags);
         if (OPAL_UNLIKELY(NULL == frag)) {
             return NULL;
         }
 
-        _mca_btl_uct_send_pack ((void *) ((intptr_t) frag->uct_iov.buffer + reserve), NULL, 0,
-                                convertor, *size);
+        _mca_btl_uct_send_pack((void *) ((intptr_t) frag->uct_iov.buffer + reserve), NULL, 0,
+                               convertor, size);
+        /* update the length of the fragment according to the convertor packed data */
+        frag->segments[0].seg_len = reserve + *size;
+        frag->uct_iov.length = frag->segments[0].seg_len;
     } else {
-        opal_convertor_get_current_pointer (convertor, &data_ptr);
-        assert (NULL != data_ptr);
+        opal_convertor_get_current_pointer(convertor, &data_ptr);
+        assert(NULL != data_ptr);
 
-        frag = mca_btl_uct_frag_alloc_short (uct_btl, endpoint);
+        frag = mca_btl_uct_frag_alloc_short(uct_btl, endpoint);
         if (OPAL_UNLIKELY(NULL == frag)) {
             return NULL;
         }
 
-        frag->uct_iov.length   = total_size;
-        frag->base.order       = order;
-        frag->base.des_flags   = flags;
+        frag->uct_iov.length = total_size;
+        frag->base.order = order;
+        frag->base.des_flags = flags;
         if (total_size > (size_t) MCA_BTL_UCT_TL_ATTR(uct_btl->am_tl, 0).cap.am.max_short) {
             frag->segments[0].seg_len = reserve;
             frag->segments[1].seg_len = *size;
@@ -109,7 +113,8 @@ struct mca_btl_base_descriptor_t *mca_btl_uct_prepare_src (mca_btl_base_module_t
             frag->base.des_segment_count = 2;
         } else {
             frag->segments[0].seg_len = total_size;
-            memcpy ((void *)((intptr_t) frag->segments[0].seg_addr.pval + reserve), data_ptr, *size);
+            memcpy((void *) ((intptr_t) frag->segments[0].seg_addr.pval + reserve), data_ptr,
+                   *size);
             frag->base.des_segment_count = 1;
         }
     }
@@ -123,42 +128,43 @@ struct mca_btl_base_descriptor_t *mca_btl_uct_prepare_src (mca_btl_base_module_t
  * @param btl (IN)      BTL module
  * @param segment (IN)  Allocated segment.
  */
-int mca_btl_uct_free (mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des)
+int mca_btl_uct_free(mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des)
 {
-    mca_btl_uct_frag_return ((mca_btl_uct_base_frag_t *) des);
+    mca_btl_uct_frag_return((mca_btl_uct_base_frag_t *) des);
     return OPAL_SUCCESS;
 }
 
-static size_t mca_btl_uct_send_frag_pack (void *data, void *arg)
+static size_t mca_btl_uct_send_frag_pack(void *data, void *arg)
 {
     mca_btl_uct_base_frag_t *frag = (mca_btl_uct_base_frag_t *) arg;
     size_t length = 8;
 
-    memcpy (data, &frag->header, sizeof (frag->header));
-    data = (void *)((intptr_t) data + 8);
+    memcpy(data, &frag->header, sizeof(frag->header));
+    data = (void *) ((intptr_t) data + 8);
 
     /* this function should only ever get called with fragments with two segments */
-    for (size_t i = 0 ; i < frag->base.des_segment_count ; ++i) {
+    for (size_t i = 0; i < frag->base.des_segment_count; ++i) {
         const size_t seg_len = frag->segments[i].seg_len;
-        memcpy (data, frag->segments[i].seg_addr.pval, seg_len);
-        data = (void *)((intptr_t) data + seg_len);
+        memcpy(data, frag->segments[i].seg_addr.pval, seg_len);
+        data = (void *) ((intptr_t) data + seg_len);
         length += seg_len;
     }
 
     return length;
 }
 
-static void mca_btl_uct_append_pending_frag (mca_btl_uct_module_t *uct_btl, mca_btl_uct_base_frag_t *frag,
-                                             mca_btl_uct_device_context_t *context, bool ready)
+static void mca_btl_uct_append_pending_frag(mca_btl_uct_module_t *uct_btl,
+                                            mca_btl_uct_base_frag_t *frag,
+                                            mca_btl_uct_device_context_t *context, bool ready)
 {
     frag->ready = ready;
     frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
-    opal_atomic_wmb ();
+    opal_atomic_wmb();
 
-    opal_list_append (&uct_btl->pending_frags, (opal_list_item_t *) frag);
+    opal_list_append(&uct_btl->pending_frags, (opal_list_item_t *) frag);
 }
 
-int mca_btl_uct_send_frag (mca_btl_uct_module_t *uct_btl, mca_btl_uct_base_frag_t *frag, bool append)
+int mca_btl_uct_send_frag(mca_btl_uct_module_t *uct_btl, mca_btl_uct_base_frag_t *frag, bool append)
 {
     mca_btl_uct_device_context_t *context = frag->context;
     const ssize_t msg_size = frag->uct_iov.length + 8;
@@ -167,55 +173,59 @@ int mca_btl_uct_send_frag (mca_btl_uct_module_t *uct_btl, mca_btl_uct_base_frag_
     uct_ep_h ep_handle = NULL;
 
     /* if we get here then we must have an endpoint handle for this context/endpoint pair */
-    (void) mca_btl_uct_endpoint_test_am (uct_btl, frag->endpoint, frag->context, &ep_handle);
-    assert (NULL != ep_handle);
+    (void) mca_btl_uct_endpoint_test_am(uct_btl, frag->endpoint, frag->context, &ep_handle);
+    assert(NULL != ep_handle);
 
     /* if another thread set this we really don't care too much as this flag is only meant
      * to protect against deep recursion */
     if (!context->in_am_callback) {
-        mca_btl_uct_context_lock (context);
+        mca_btl_uct_context_lock(context);
         /* attempt to post the fragment */
-        if (NULL != frag->base.super.registration &&
-            (context->uct_iface_attr.cap.flags & UCT_IFACE_FLAG_AM_ZCOPY)) {
+        if (NULL != frag->base.super.registration
+            && (context->uct_iface_attr.cap.flags & UCT_IFACE_FLAG_AM_ZCOPY)) {
             frag->comp.dev_context = context;
-            ucs_status = uct_ep_am_zcopy (ep_handle, MCA_BTL_UCT_FRAG, &frag->header, sizeof (frag->header),
-                                          &frag->uct_iov, 1, 0, &frag->comp.uct_comp);
+            ucs_status = uct_ep_am_zcopy(ep_handle, MCA_BTL_UCT_FRAG, &frag->header,
+                                         sizeof(frag->header), &frag->uct_iov, 1, 0,
+                                         &frag->comp.uct_comp);
 
             if (OPAL_LIKELY(UCS_INPROGRESS == ucs_status)) {
-                uct_worker_progress (context->uct_worker);
-                mca_btl_uct_context_unlock (context);
+                uct_worker_progress(context->uct_worker);
+                mca_btl_uct_context_unlock(context);
                 return OPAL_SUCCESS;
             }
         } else {
             /* short message */
-            if (1 == frag->base.des_segment_count && (frag->uct_iov.length + 8) < MCA_BTL_UCT_TL_ATTR(uct_btl->am_tl, 0).cap.am.max_short) {
-                ucs_status = uct_ep_am_short (ep_handle, MCA_BTL_UCT_FRAG, frag->header.value, frag->uct_iov.buffer,
-                                              frag->uct_iov.length);
+            if (1 == frag->base.des_segment_count
+                && (frag->uct_iov.length + 8)
+                       < MCA_BTL_UCT_TL_ATTR(uct_btl->am_tl, 0).cap.am.max_short) {
+                ucs_status = uct_ep_am_short(ep_handle, MCA_BTL_UCT_FRAG, frag->header.value,
+                                             frag->uct_iov.buffer, frag->uct_iov.length);
 
                 if (OPAL_LIKELY(UCS_OK == ucs_status)) {
-                    uct_worker_progress (context->uct_worker);
-                    mca_btl_uct_context_unlock (context);
+                    uct_worker_progress(context->uct_worker);
+                    mca_btl_uct_context_unlock(context);
                     /* send is complete */
-                    mca_btl_uct_frag_complete (frag, OPAL_SUCCESS);
+                    mca_btl_uct_frag_complete(frag, OPAL_SUCCESS);
                     return 1;
                 }
             }
 
-            size = uct_ep_am_bcopy (ep_handle, MCA_BTL_UCT_FRAG, mca_btl_uct_send_frag_pack, frag, 0);
+            size = uct_ep_am_bcopy(ep_handle, MCA_BTL_UCT_FRAG, mca_btl_uct_send_frag_pack, frag,
+                                   0);
             if (OPAL_LIKELY(size == msg_size)) {
-                uct_worker_progress (context->uct_worker);
-                mca_btl_uct_context_unlock (context);
+                uct_worker_progress(context->uct_worker);
+                mca_btl_uct_context_unlock(context);
                 /* send is complete */
-                mca_btl_uct_frag_complete (frag, OPAL_SUCCESS);
+                mca_btl_uct_frag_complete(frag, OPAL_SUCCESS);
                 return 1;
             }
         }
 
         /* wait for something to happen */
-        uct_worker_progress (context->uct_worker);
-        mca_btl_uct_context_unlock (context);
+        uct_worker_progress(context->uct_worker);
+        mca_btl_uct_context_unlock(context);
 
-        mca_btl_uct_device_handle_completions (context);
+        mca_btl_uct_device_handle_completions(context);
     }
 
     if (!append) {
@@ -223,41 +233,41 @@ int mca_btl_uct_send_frag (mca_btl_uct_module_t *uct_btl, mca_btl_uct_base_frag_
     }
 
     OPAL_THREAD_LOCK(&uct_btl->lock);
-    mca_btl_uct_append_pending_frag (uct_btl, frag, context, true);
+    mca_btl_uct_append_pending_frag(uct_btl, frag, context, true);
     OPAL_THREAD_UNLOCK(&uct_btl->lock);
 
     return OPAL_SUCCESS;
 }
 
-int mca_btl_uct_send (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, mca_btl_base_descriptor_t *descriptor,
-                      mca_btl_base_tag_t tag)
+int mca_btl_uct_send(mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
+                     mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag)
 {
     mca_btl_uct_module_t *uct_btl = (mca_btl_uct_module_t *) btl;
-    mca_btl_uct_device_context_t *context = mca_btl_uct_module_get_am_context (uct_btl);
+    mca_btl_uct_device_context_t *context = mca_btl_uct_module_get_am_context(uct_btl);
     mca_btl_uct_base_frag_t *frag = (mca_btl_uct_base_frag_t *) descriptor;
     uct_ep_h ep_handle;
     int rc;
 
-    BTL_VERBOSE(("btl/uct sending descriptor %p from %d -> %d. length = %" PRIu64, (void *)descriptor,
-                 OPAL_PROC_MY_NAME.vpid, endpoint->ep_proc->proc_name.vpid, frag->uct_iov.length));
-
+    BTL_VERBOSE(("btl/uct sending descriptor %p from %d -> %d. length = %" PRIu64,
+                 (void *) descriptor, OPAL_PROC_MY_NAME.vpid, endpoint->ep_proc->proc_name.vpid,
+                 frag->uct_iov.length));
 
     frag->header.data.tag = tag;
     frag->context = context;
 
-    rc = mca_btl_uct_endpoint_check_am (uct_btl, endpoint, context, &ep_handle);
+    rc = mca_btl_uct_endpoint_check_am(uct_btl, endpoint, context, &ep_handle);
     if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
         OPAL_THREAD_LOCK(&uct_btl->lock);
         /* check one more time in case another thread is completing the connection now */
-        if (OPAL_SUCCESS != mca_btl_uct_endpoint_test_am (uct_btl, endpoint, context, &ep_handle)) {
-            mca_btl_uct_append_pending_frag (uct_btl, frag, context, false);
+        if (OPAL_SUCCESS != mca_btl_uct_endpoint_test_am(uct_btl, endpoint, context, &ep_handle)) {
+            mca_btl_uct_append_pending_frag(uct_btl, frag, context, false);
             OPAL_THREAD_UNLOCK(&uct_btl->lock);
             return OPAL_SUCCESS;
         }
         OPAL_THREAD_UNLOCK(&uct_btl->lock);
     }
 
-    return mca_btl_uct_send_frag (uct_btl, frag, true);
+    return mca_btl_uct_send_frag(uct_btl, frag, true);
 }
 
 struct mca_btl_uct_sendi_pack_args_t {
@@ -270,28 +280,29 @@ struct mca_btl_uct_sendi_pack_args_t {
 
 typedef struct mca_btl_uct_sendi_pack_args_t mca_btl_uct_sendi_pack_args_t;
 
-static size_t mca_btl_uct_sendi_pack (void *data, void *arg)
+static size_t mca_btl_uct_sendi_pack(void *data, void *arg)
 {
     mca_btl_uct_sendi_pack_args_t *args = (mca_btl_uct_sendi_pack_args_t *) arg;
     mca_btl_uct_am_header_t *am_header = (mca_btl_uct_am_header_t *) data;
 
     am_header->value = args->am_header;
-    _mca_btl_uct_send_pack ((void *)((intptr_t)data + 8), args->header, args->header_size, args->convertor,
-                            args->payload_size);
+    _mca_btl_uct_send_pack((void *) ((intptr_t) data + 8), args->header, args->header_size,
+                           args->convertor, &args->payload_size);
     return args->header_size + args->payload_size + 8;
 }
 
-static inline size_t mca_btl_uct_max_sendi (mca_btl_uct_module_t *uct_btl, int context_id)
+static inline size_t mca_btl_uct_max_sendi(mca_btl_uct_module_t *uct_btl, int context_id)
 {
     return MCA_BTL_UCT_TL_ATTR(uct_btl->am_tl, context_id).cap.am.max_bcopy;
 }
 
-int mca_btl_uct_sendi (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, opal_convertor_t *convertor,
-                       void *header, size_t header_size, size_t payload_size, uint8_t order, uint32_t flags,
-                       mca_btl_base_tag_t tag, mca_btl_base_descriptor_t **descriptor)
+int mca_btl_uct_sendi(mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
+                      opal_convertor_t *convertor, void *header, size_t header_size,
+                      size_t payload_size, uint8_t order, uint32_t flags, mca_btl_base_tag_t tag,
+                      mca_btl_base_descriptor_t **descriptor)
 {
     mca_btl_uct_module_t *uct_btl = (mca_btl_uct_module_t *) btl;
-    mca_btl_uct_device_context_t *context = mca_btl_uct_module_get_am_context (uct_btl);
+    mca_btl_uct_device_context_t *context = mca_btl_uct_module_get_am_context(uct_btl);
     const size_t total_size = header_size + payload_size;
     /* message with header */
     const size_t msg_size = total_size + 8;
@@ -300,10 +311,11 @@ int mca_btl_uct_sendi (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endp
     uct_ep_h ep_handle;
     int rc;
 
-    rc = mca_btl_uct_endpoint_check_am (uct_btl, endpoint, context, &ep_handle);
-    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc || msg_size > mca_btl_uct_max_sendi (uct_btl, context->context_id))) {
+    rc = mca_btl_uct_endpoint_check_am(uct_btl, endpoint, context, &ep_handle);
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc
+                      || msg_size > mca_btl_uct_max_sendi(uct_btl, context->context_id))) {
         if (descriptor) {
-            *descriptor = mca_btl_uct_alloc (btl, endpoint, order, total_size, flags);
+            *descriptor = mca_btl_uct_alloc(btl, endpoint, order, total_size, flags);
         }
 
         return OPAL_ERR_OUT_OF_RESOURCE;
@@ -311,30 +323,45 @@ int mca_btl_uct_sendi (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endp
 
     am_header.data.tag = tag;
 
-    mca_btl_uct_context_lock (context);
+    mca_btl_uct_context_lock(context);
     if (0 == payload_size) {
-        ucs_status = uct_ep_am_short (ep_handle, MCA_BTL_UCT_FRAG, am_header.value, header, header_size);
-    } else if (msg_size < (size_t) MCA_BTL_UCT_TL_ATTR(uct_btl->am_tl, context->context_id).cap.am.max_short) {
-        int8_t *data = alloca (total_size);
-        _mca_btl_uct_send_pack (data, header, header_size, convertor, payload_size);
-        ucs_status = uct_ep_am_short (ep_handle, MCA_BTL_UCT_FRAG, am_header.value, data, total_size);
+        ucs_status = uct_ep_am_short(ep_handle, MCA_BTL_UCT_FRAG, am_header.value, header,
+                                     header_size);
+    } else if (msg_size < (size_t) MCA_BTL_UCT_TL_ATTR(uct_btl->am_tl, context->context_id)
+                              .cap.am.max_short) {
+        int8_t *data = alloca(total_size);
+        size_t packed_payload_size = payload_size;
+        _mca_btl_uct_send_pack(data, header, header_size, convertor, &packed_payload_size);
+        if (packed_payload_size != payload_size) {
+            /* This should never happen as the packed data should go in a single pack. But
+               in case it does, fallback onto a descriptor allocation and let the caller
+               send the data.
+             */
+            ucs_status = UCS_ERR_NO_RESOURCE;
+        } else {
+            ucs_status = uct_ep_am_short(ep_handle, MCA_BTL_UCT_FRAG, am_header.value, data,
+                                         total_size);
+        }
     } else {
         ssize_t size;
 
-        size = uct_ep_am_bcopy (ep_handle, MCA_BTL_UCT_FRAG, mca_btl_uct_sendi_pack,
-                                &(mca_btl_uct_sendi_pack_args_t) {.am_header = am_header.value,
-                                        .header = header, .header_size = header_size,
-                                        .convertor = convertor, .payload_size = payload_size}, 0);
+        size = uct_ep_am_bcopy(ep_handle, MCA_BTL_UCT_FRAG, mca_btl_uct_sendi_pack,
+                               &(mca_btl_uct_sendi_pack_args_t){.am_header = am_header.value,
+                                                                .header = header,
+                                                                .header_size = header_size,
+                                                                .convertor = convertor,
+                                                                .payload_size = payload_size},
+                               0);
         if (OPAL_LIKELY(size == (ssize_t) msg_size)) {
             ucs_status = UCS_OK;
         }
     }
 
-    mca_btl_uct_context_unlock (context);
+    mca_btl_uct_context_unlock(context);
 
     if (OPAL_UNLIKELY(UCS_OK != ucs_status)) {
         if (descriptor) {
-            *descriptor = mca_btl_uct_alloc (btl, endpoint, order, total_size, flags);
+            *descriptor = mca_btl_uct_alloc(btl, endpoint, order, total_size, flags);
         }
 
         return OPAL_ERR_OUT_OF_RESOURCE;
diff --git a/opal/mca/btl/uct/btl_uct_am.h b/opal/mca/btl/uct/btl_uct_am.h
index 9035540e710..695cc0b5e0a 100644
--- a/opal/mca/btl/uct/btl_uct_am.h
+++ b/opal/mca/btl/uct/btl_uct_am.h
@@ -10,29 +10,31 @@
  */
 
 #if !defined(MCA_BTL_UCT_AM_H)
-#define MCA_BTL_UCT_AM_H
+#    define MCA_BTL_UCT_AM_H
 
-#include "btl_uct_frag.h"
+#    include "btl_uct_frag.h"
 
-struct mca_btl_base_descriptor_t *mca_btl_uct_prepare_src (mca_btl_base_module_t *btl,
-                                                           mca_btl_base_endpoint_t *endpoint,
-                                                           opal_convertor_t *convertor,
-                                                           uint8_t order, size_t reserve,
-                                                           size_t *size, uint32_t flags);
+struct mca_btl_base_descriptor_t *mca_btl_uct_prepare_src(mca_btl_base_module_t *btl,
+                                                          mca_btl_base_endpoint_t *endpoint,
+                                                          opal_convertor_t *convertor,
+                                                          uint8_t order, size_t reserve,
+                                                          size_t *size, uint32_t flags);
 
-int mca_btl_uct_sendi (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, opal_convertor_t *convertor,
-                       void *header, size_t header_size, size_t payload_size, uint8_t order, uint32_t flags,
-                       mca_btl_base_tag_t tag, mca_btl_base_descriptor_t **descriptor);
+int mca_btl_uct_sendi(mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
+                      opal_convertor_t *convertor, void *header, size_t header_size,
+                      size_t payload_size, uint8_t order, uint32_t flags, mca_btl_base_tag_t tag,
+                      mca_btl_base_descriptor_t **descriptor);
 
-int mca_btl_uct_send (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, mca_btl_base_descriptor_t *descriptor,
-                      mca_btl_base_tag_t tag);
+int mca_btl_uct_send(mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
+                     mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag);
 
-int mca_btl_uct_send_frag (mca_btl_uct_module_t *uct_btl, mca_btl_uct_base_frag_t *frag, bool append);
+int mca_btl_uct_send_frag(mca_btl_uct_module_t *uct_btl, mca_btl_uct_base_frag_t *frag,
+                          bool append);
 
-mca_btl_base_descriptor_t *mca_btl_uct_alloc (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
-                                              uint8_t order, size_t size, uint32_t flags);
-
-int mca_btl_uct_free (mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des);
+mca_btl_base_descriptor_t *mca_btl_uct_alloc(mca_btl_base_module_t *btl,
+                                             mca_btl_base_endpoint_t *endpoint, uint8_t order,
+                                             size_t size, uint32_t flags);
 
+int mca_btl_uct_free(mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des);
 
 #endif /* !defined(MCA_BTL_UCT_AM_H) */
diff --git a/opal/mca/btl/uct/btl_uct_amo.c b/opal/mca/btl/uct/btl_uct_amo.c
index 72398ce7369..01e195274b2 100644
--- a/opal/mca/btl/uct/btl_uct_amo.c
+++ b/opal/mca/btl/uct/btl_uct_amo.c
@@ -15,22 +15,21 @@
 /* we add 1 to the ops to differentiate between unsupported and supported ops since
  * UCT_ATOMIC_OP_ADD == 0. otherwise we would have to fill in this table completely. */
 static int mca_btl_uct_btl_to_uct_atomic[MCA_BTL_ATOMIC_LAST] = {
-    [MCA_BTL_ATOMIC_ADD] = UCT_ATOMIC_OP_ADD + 1,
-    [MCA_BTL_ATOMIC_AND] = UCT_ATOMIC_OP_AND + 1,
-    [MCA_BTL_ATOMIC_OR]  = UCT_ATOMIC_OP_OR + 1,
-    [MCA_BTL_ATOMIC_XOR] = UCT_ATOMIC_OP_XOR + 1,
+    [MCA_BTL_ATOMIC_ADD] = UCT_ATOMIC_OP_ADD + 1,   [MCA_BTL_ATOMIC_AND] = UCT_ATOMIC_OP_AND + 1,
+    [MCA_BTL_ATOMIC_OR] = UCT_ATOMIC_OP_OR + 1,     [MCA_BTL_ATOMIC_XOR] = UCT_ATOMIC_OP_XOR + 1,
     [MCA_BTL_ATOMIC_SWAP] = UCT_ATOMIC_OP_SWAP + 1,
 };
 #endif
 
-int mca_btl_uct_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
-                      void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                      mca_btl_base_registration_handle_t *remote_handle, mca_btl_base_atomic_op_t op,
-                      uint64_t operand, int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
-                      void *cbcontext, void *cbdata)
+int mca_btl_uct_afop(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+                     void *local_address, uint64_t remote_address,
+                     mca_btl_base_registration_handle_t *local_handle,
+                     mca_btl_base_registration_handle_t *remote_handle, mca_btl_base_atomic_op_t op,
+                     uint64_t operand, int flags, int order,
+                     mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
 {
     mca_btl_uct_module_t *uct_btl = (mca_btl_uct_module_t *) btl;
-    mca_btl_uct_device_context_t *context = mca_btl_uct_module_get_rdma_context (uct_btl);
+    mca_btl_uct_device_context_t *context = mca_btl_uct_module_get_rdma_context(uct_btl);
     mca_btl_uct_uct_completion_t *comp = NULL;
     ucs_status_t ucs_status;
     uct_rkey_bundle_t rkey;
@@ -50,75 +49,77 @@ int mca_btl_uct_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_end
 #endif
 
     if (cbfunc) {
-        comp = mca_btl_uct_uct_completion_alloc (uct_btl, endpoint, local_address, local_handle, context,
-                                                 cbfunc, cbcontext, cbdata);
+        comp = mca_btl_uct_uct_completion_alloc(uct_btl, endpoint, local_address, local_handle,
+                                                context, cbfunc, cbcontext, cbdata);
         if (OPAL_UNLIKELY(NULL == comp)) {
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
     }
 
-    rc = mca_btl_uct_get_rkey (uct_btl, context, endpoint, remote_handle, &rkey, &ep_handle);
+    rc = mca_btl_uct_get_rkey(uct_btl, context, endpoint, remote_handle, &rkey, &ep_handle);
     if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-        mca_btl_uct_uct_completion_release (comp);
+        mca_btl_uct_uct_completion_release(comp);
         return rc;
     }
 
-    mca_btl_uct_context_lock (context);
+    mca_btl_uct_context_lock(context);
 
 #if OPAL_HAVE_UCT_EP_ATOMIC64_POST
     if (flags & MCA_BTL_ATOMIC_FLAG_32BIT) {
-        ucs_status = uct_ep_atomic32_fetch (ep_handle, uct_op, operand, (uint32_t *) local_address, remote_address,
-	                                    rkey.rkey, &comp->uct_comp);
+        ucs_status = uct_ep_atomic32_fetch(ep_handle, uct_op, operand, (uint32_t *) local_address,
+                                           remote_address, rkey.rkey, &comp->uct_comp);
     } else {
-        ucs_status = uct_ep_atomic64_fetch (ep_handle, uct_op, operand, (uint64_t *) local_address, remote_address,
-	                                    rkey.rkey, &comp->uct_comp);
+        ucs_status = uct_ep_atomic64_fetch(ep_handle, uct_op, operand, (uint64_t *) local_address,
+                                           remote_address, rkey.rkey, &comp->uct_comp);
     }
 #else
     if (MCA_BTL_ATOMIC_ADD == op) {
         if (flags & MCA_BTL_ATOMIC_FLAG_32BIT) {
-            ucs_status = uct_ep_atomic_fadd32 (ep_handle, (uint32_t) operand, remote_address,
-                                               rkey.rkey, (uint32_t *) local_address, &comp->uct_comp);
+            ucs_status = uct_ep_atomic_fadd32(ep_handle, (uint32_t) operand, remote_address,
+                                              rkey.rkey, (uint32_t *) local_address,
+                                              &comp->uct_comp);
         } else {
-            ucs_status = uct_ep_atomic_fadd64 (ep_handle, operand, remote_address, rkey.rkey,
-                                               (uint64_t *) local_address, &comp->uct_comp);
+            ucs_status = uct_ep_atomic_fadd64(ep_handle, operand, remote_address, rkey.rkey,
+                                              (uint64_t *) local_address, &comp->uct_comp);
         }
     } else {
         if (flags & MCA_BTL_ATOMIC_FLAG_32BIT) {
-            ucs_status = uct_ep_atomic_swap32 (ep_handle, (uint32_t) operand, remote_address,
-                                               rkey.rkey, (uint32_t *) local_address, &comp->uct_comp);
+            ucs_status = uct_ep_atomic_swap32(ep_handle, (uint32_t) operand, remote_address,
+                                              rkey.rkey, (uint32_t *) local_address,
+                                              &comp->uct_comp);
         } else {
-            ucs_status = uct_ep_atomic_swap64 (ep_handle, operand, remote_address, rkey.rkey,
-                                               (uint64_t *) local_address, &comp->uct_comp);
+            ucs_status = uct_ep_atomic_swap64(ep_handle, operand, remote_address, rkey.rkey,
+                                              (uint64_t *) local_address, &comp->uct_comp);
         }
     }
 #endif
 
     /* go ahead and progress the worker while we have the lock */
-    (void) uct_worker_progress (context->uct_worker);
+    (void) uct_worker_progress(context->uct_worker);
 
-    mca_btl_uct_context_unlock (context);
+    mca_btl_uct_context_unlock(context);
 
-    mca_btl_uct_device_handle_completions (context);
+    mca_btl_uct_device_handle_completions(context);
 
     if (UCS_INPROGRESS == ucs_status) {
         rc = OPAL_SUCCESS;
     } else if (UCS_OK == ucs_status) {
         rc = 1;
-        mca_btl_uct_uct_completion_release (comp);
+        mca_btl_uct_uct_completion_release(comp);
     } else {
         rc = OPAL_ERR_OUT_OF_RESOURCE;
-        mca_btl_uct_uct_completion_release (comp);
+        mca_btl_uct_uct_completion_release(comp);
     }
 
-    mca_btl_uct_rkey_release (uct_btl, &rkey);
+    mca_btl_uct_rkey_release(uct_btl, &rkey);
 
     return rc;
 }
 
-int mca_btl_uct_aop (struct mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
-                     uint64_t remote_address, mca_btl_base_registration_handle_t *remote_handle,
-                     mca_btl_base_atomic_op_t op, uint64_t operand, int flags, int order,
-                     mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
+int mca_btl_uct_aop(struct mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
+                    uint64_t remote_address, mca_btl_base_registration_handle_t *remote_handle,
+                    mca_btl_base_atomic_op_t op, uint64_t operand, int flags, int order,
+                    mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
 {
     /* this is static so it survives after this function returns. we don't care about the result */
     static uint64_t result;
@@ -126,17 +127,19 @@ int mca_btl_uct_aop (struct mca_btl_base_module_t *btl, mca_btl_base_endpoint_t
     /* just use the fetching ops for now. there probably is a performance benefit to using
      * the non-fetching on some platforms but this is easier to implement quickly and it
      * guarantees remote completion. */
-    return mca_btl_uct_afop (btl, endpoint, &result, remote_address, NULL, remote_handle, op,
-                             operand, flags, order, cbfunc, cbcontext, cbdata);
+    return mca_btl_uct_afop(btl, endpoint, &result, remote_address, NULL, remote_handle, op,
+                            operand, flags, order, cbfunc, cbcontext, cbdata);
 }
 
-int mca_btl_uct_acswap (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
-                        void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                        mca_btl_base_registration_handle_t *remote_handle, uint64_t compare, uint64_t value, int flags,
-                        int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
+int mca_btl_uct_acswap(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
+                       void *local_address, uint64_t remote_address,
+                       mca_btl_base_registration_handle_t *local_handle,
+                       mca_btl_base_registration_handle_t *remote_handle, uint64_t compare,
+                       uint64_t value, int flags, int order,
+                       mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
 {
     mca_btl_uct_module_t *uct_btl = (mca_btl_uct_module_t *) btl;
-    mca_btl_uct_device_context_t *context = mca_btl_uct_module_get_rdma_context (uct_btl);
+    mca_btl_uct_device_context_t *context = mca_btl_uct_module_get_rdma_context(uct_btl);
     mca_btl_uct_uct_completion_t *comp = NULL;
     ucs_status_t ucs_status;
     uct_rkey_bundle_t rkey;
@@ -144,47 +147,48 @@ int mca_btl_uct_acswap (struct mca_btl_base_module_t *btl, struct mca_btl_base_e
     int rc;
 
     if (cbfunc) {
-        comp = mca_btl_uct_uct_completion_alloc (uct_btl, endpoint, local_address, local_handle, context,
-                                                 cbfunc, cbcontext, cbdata);
+        comp = mca_btl_uct_uct_completion_alloc(uct_btl, endpoint, local_address, local_handle,
+                                                context, cbfunc, cbcontext, cbdata);
         if (OPAL_UNLIKELY(NULL == comp)) {
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
     }
 
-    rc = mca_btl_uct_get_rkey (uct_btl, context, endpoint, remote_handle, &rkey, &ep_handle);
+    rc = mca_btl_uct_get_rkey(uct_btl, context, endpoint, remote_handle, &rkey, &ep_handle);
     if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-        mca_btl_uct_uct_completion_release (comp);
+        mca_btl_uct_uct_completion_release(comp);
         return rc;
     }
 
-    mca_btl_uct_context_lock (context);
+    mca_btl_uct_context_lock(context);
 
     if (flags & MCA_BTL_ATOMIC_FLAG_32BIT) {
-        ucs_status = uct_ep_atomic_cswap32 (ep_handle, (uint32_t) compare, (uint32_t) value, remote_address,
-                                            rkey.rkey, (uint32_t *) local_address, &comp->uct_comp);
+        ucs_status = uct_ep_atomic_cswap32(ep_handle, (uint32_t) compare, (uint32_t) value,
+                                           remote_address, rkey.rkey, (uint32_t *) local_address,
+                                           &comp->uct_comp);
     } else {
-        ucs_status = uct_ep_atomic_cswap64 (ep_handle, compare, value, remote_address, rkey.rkey,
-                                            (uint64_t *) local_address, &comp->uct_comp);
+        ucs_status = uct_ep_atomic_cswap64(ep_handle, compare, value, remote_address, rkey.rkey,
+                                           (uint64_t *) local_address, &comp->uct_comp);
     }
 
     /* go ahead and progress the worker while we have the lock */
-    (void) uct_worker_progress (context->uct_worker);
+    (void) uct_worker_progress(context->uct_worker);
 
-    mca_btl_uct_context_unlock (context);
+    mca_btl_uct_context_unlock(context);
 
-    mca_btl_uct_device_handle_completions (context);
+    mca_btl_uct_device_handle_completions(context);
 
     if (UCS_INPROGRESS == ucs_status) {
         rc = OPAL_SUCCESS;
     } else if (UCS_OK == ucs_status) {
         rc = 1;
-        mca_btl_uct_uct_completion_release (comp);
+        mca_btl_uct_uct_completion_release(comp);
     } else {
         rc = OPAL_ERR_OUT_OF_RESOURCE;
-        mca_btl_uct_uct_completion_release (comp);
+        mca_btl_uct_uct_completion_release(comp);
     }
 
-    mca_btl_uct_rkey_release (uct_btl, &rkey);
+    mca_btl_uct_rkey_release(uct_btl, &rkey);
 
     return rc;
 }
diff --git a/opal/mca/btl/uct/btl_uct_component.c b/opal/mca/btl/uct/btl_uct_component.c
index baf4e11f124..5eec97ec487 100644
--- a/opal/mca/btl/uct/btl_uct_component.c
+++ b/opal/mca/btl/uct/btl_uct_component.c
@@ -15,10 +15,11 @@
  * Copyright (c) 2018      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
- * Copyright (c) 2018      Triad National Security, LLC. All rights
+ * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
  *                         reserved.
- * Copyright (c) 2019-2020 Google, LLC. All rights reserved.
+ * Copyright (c) 2019-2025 Google, LLC. All rights reserved.
  * Copyright (c) 2019      Intel, Inc.  All rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -26,81 +27,96 @@
  * $HEADER$
  */
 
-
 #include "opal_config.h"
 
-#include "opal/mca/btl/btl.h"
 #include "opal/mca/btl/base/base.h"
+#include "opal/mca/btl/btl.h"
 #include "opal/mca/hwloc/base/base.h"
-#include "opal/util/argv.h"
-#include "opal/memoryhooks/memory.h"
 #include "opal/mca/memory/base/base.h"
+#include "opal/memoryhooks/memory.h"
+#include "opal/util/argv.h"
 #include <ucm/api/ucm.h>
 
 #include "opal/util/printf.h"
 
 #include <string.h>
 
-#include "btl_uct_device_context.h"
 #include "btl_uct_am.h"
+#include "btl_uct_device_context.h"
 
 static int mca_btl_uct_component_register(void)
 {
     mca_btl_uct_module_t *module = &mca_btl_uct_module_template;
 
-    mca_btl_uct_component.memory_domains = "none";
-    (void) mca_base_component_var_register(&mca_btl_uct_component.super.btl_version,
-                                           "memory_domains", "Comma-delimited list of memory domains of the form "
-                                           "to use for communication. Memory domains MUST provide transports that "
-                                           "support put, get, and amos. Special values: all (all available), none."
-                                           " (default: none)", MCA_BASE_VAR_TYPE_STRING, NULL, 0,
-                                           MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_LOCAL,
-                                           &mca_btl_uct_component.memory_domains);
+    mca_btl_uct_component.memory_domains = "mlx5_0,mlx4_0,rocep0s4";
+    (void) mca_base_component_var_register(
+        &mca_btl_uct_component.super.btl_version, "memory_domains",
+        "Comma-delimited list of memory domains of the form "
+        "to use for communication. Memory domains MUST provide transports that "
+        "support put, get, and amos. Special values: all (all available), none."
+        " (default: mlx5_0,mlx4_0,rocep0s4)",
+        MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
+        MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_uct_component.memory_domains);
 
     mca_btl_uct_component.allowed_transports = "dc_mlx5,rc_mlx5,ud,ugni_rdma,ugni_smsg,any";
-    (void) mca_base_component_var_register(&mca_btl_uct_component.super.btl_version,
-                                           "transports", "Comma-delimited list of transports to use sorted by increasing "
-                                           "priority. The list of transports available can be queried using ucx_info. Special"
-                                           "values: any (any available) (default: dc_mlx5,rc_mlx5,ud,any)",
-                                           MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
-                                           MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_uct_component.allowed_transports);
+    (void) mca_base_component_var_register(
+        &mca_btl_uct_component.super.btl_version, "transports",
+        "Comma-delimited list of transports to use sorted by increasing "
+        "priority. The list of transports available can be queried using ucx_info. Special"
+        "values: any (any available) (default: dc_mlx5,rc_mlx5,ud,any)",
+        MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
+        MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_uct_component.allowed_transports);
 
     mca_btl_uct_component.num_contexts_per_module = 0;
-    (void) mca_base_component_var_register(&mca_btl_uct_component.super.btl_version,
-                                           "num_contexts_per_module", "Number of UCT worker contexts "
-                                           "to create for each BTL module. Larger numbers will improve "
-                                           "multi-threaded performance but may increase memory usage. "
-                                           "A good rule of thumb is one context per application thread "
-                                           "that will be calling into MPI. (default: 0 -- autoselect "
-                                           "based on the number of cores)", MCA_BASE_VAR_TYPE_INT,
-                                           NULL, 0 ,MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
-                                           MCA_BASE_VAR_SCOPE_ALL, &mca_btl_uct_component.num_contexts_per_module);
+    (void) mca_base_component_var_register(
+        &mca_btl_uct_component.super.btl_version, "num_contexts_per_module",
+        "Number of UCT worker contexts "
+        "to create for each BTL module. Larger numbers will improve "
+        "multi-threaded performance but may increase memory usage. "
+        "A good rule of thumb is one context per application thread "
+        "that will be calling into MPI. (default: 0 -- autoselect "
+        "based on the number of cores)",
+        MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
+        MCA_BASE_VAR_SCOPE_ALL, &mca_btl_uct_component.num_contexts_per_module);
 
     mca_btl_uct_component.disable_ucx_memory_hooks = true;
-    (void) mca_base_component_var_register(&mca_btl_uct_component.super.btl_version,
-                                           "disable_ucx_memory_hooks", "Disable the munmap memory hook "
-                                           "inside UCX. These hooks are not necessary when using the "
-                                           "uct btl and tend to cause performance problems when using "
-                                           "multiple threads (default: true)", MCA_BASE_VAR_TYPE_BOOL,
-                                           NULL, 0 ,MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
-                                           MCA_BASE_VAR_SCOPE_ALL, &mca_btl_uct_component.disable_ucx_memory_hooks);
-
+    (void)
+        mca_base_component_var_register(&mca_btl_uct_component.super.btl_version,
+                                        "disable_ucx_memory_hooks",
+                                        "Disable the munmap memory hook "
+                                        "inside UCX. These hooks are not necessary when using the "
+                                        "uct btl and tend to cause performance problems when using "
+                                        "multiple threads (default: true)",
+                                        MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
+                                        OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_ALL,
+                                        &mca_btl_uct_component.disable_ucx_memory_hooks);
 
 #if OPAL_C_HAVE__THREAD_LOCAL
     mca_btl_uct_component.bind_threads_to_contexts = true;
-    (void) mca_base_component_var_register(&mca_btl_uct_component.super.btl_version,
-                                           "bind_threads_to_contexts", "Bind threads to device contexts. "
-                                           "In general this should improve the multi-threaded performance "
-                                           "when threads are used. (default: true)", MCA_BASE_VAR_TYPE_BOOL,
-                                           NULL, 0 ,MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
-                                           MCA_BASE_VAR_SCOPE_ALL, &mca_btl_uct_component.bind_threads_to_contexts);
+    (void) mca_base_component_var_register(
+        &mca_btl_uct_component.super.btl_version, "bind_threads_to_contexts",
+        "Bind threads to device contexts. "
+        "In general this should improve the multi-threaded performance "
+        "when threads are used. (default: true)",
+        MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
+        MCA_BASE_VAR_SCOPE_ALL, &mca_btl_uct_component.bind_threads_to_contexts);
 #endif
 
+    /* timeout between connection message attempts in µs */
+    mca_btl_uct_component.connection_retry_timeout = 2000;
+    (void) mca_base_component_var_register(
+        &mca_btl_uct_component.super.btl_version, "connection_retry_timeout",
+        "Timeout between attempts to send connection messages for connect-to-"
+        "endpoint connections. The timeout is measured in µs and is only"
+        "necessary when using unreliable transports for connections (ex: UD). "
+        "(default: 2000µs)",
+        MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_4,
+        MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_uct_component.connection_retry_timeout);
+
     /* for now we want this component to lose to btl/ugni and btl/vader */
-    module->super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_HIGH;
+    module->super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_HIGH - 1;
 
-    return mca_btl_base_param_register (&mca_btl_uct_component.super.btl_version,
-                                        &module->super);
+    return mca_btl_base_param_register(&mca_btl_uct_component.super.btl_version, &module->super);
 }
 
 static void mca_btl_uct_mem_release_cb(void *buf, size_t length, void *cbdata, bool from_alloc)
@@ -115,15 +131,19 @@ static int mca_btl_uct_component_open(void)
          * how many UCT workers to create */
         int core_count = 36;
 
-        (void) opal_hwloc_base_get_topology ();
-        core_count = hwloc_get_nbobjs_by_type (opal_hwloc_topology, HWLOC_OBJ_CORE);
+        (void) opal_hwloc_base_get_topology();
+        if (0 > (core_count = hwloc_get_nbobjs_by_type(opal_hwloc_topology, HWLOC_OBJ_CORE))) {
+            return OPAL_ERROR;
+        }
 
-        if (core_count <= opal_process_info.num_local_peers || !opal_using_threads()) {
+        if ((uint32_t)core_count <= opal_process_info.num_local_peers || !opal_using_threads()) {
             /* there is probably no benefit to using multiple device contexts when not
              * using threads or oversubscribing the node with mpi processes. */
             mca_btl_uct_component.num_contexts_per_module = 1;
         } else {
-            mca_btl_uct_component.num_contexts_per_module = core_count / (opal_process_info.num_local_peers + 1);
+            mca_btl_uct_component.num_contexts_per_module = core_count
+                                                            / (opal_process_info.num_local_peers
+                                                               + 1);
         }
     }
 
@@ -131,10 +151,10 @@ static int mca_btl_uct_component_open(void)
         mca_btl_uct_component.num_contexts_per_module = MCA_BTL_UCT_MAX_WORKERS;
     }
 
-    if (mca_btl_uct_component.disable_ucx_memory_hooks &&
-        ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) ==
-        ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) &
-         opal_mem_hooks_support_level()))) {
+    if (mca_btl_uct_component.disable_ucx_memory_hooks
+        && ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT)
+            == ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT)
+                & opal_mem_hooks_support_level()))) {
         ucm_set_external_event(UCM_EVENT_VM_UNMAPPED);
         opal_mem_hooks_register_release(mca_btl_uct_mem_release_cb, NULL);
     }
@@ -142,60 +162,61 @@ static int mca_btl_uct_component_open(void)
     return OPAL_SUCCESS;
 }
 
-
 /*
  * component cleanup - sanity checking of queue lengths
  */
 static int mca_btl_uct_component_close(void)
 {
     if (mca_btl_uct_component.disable_ucx_memory_hooks) {
-        opal_mem_hooks_unregister_release (mca_btl_uct_mem_release_cb);
+        opal_mem_hooks_unregister_release(mca_btl_uct_mem_release_cb);
     }
 
     return OPAL_SUCCESS;
 }
 
-static size_t mca_btl_uct_tl_modex_size (mca_btl_uct_tl_t *tl)
+static size_t mca_btl_uct_tl_modex_size(mca_btl_uct_tl_t *tl)
 {
-    const size_t size = strlen (tl->uct_tl_name) + 1;
+    const size_t size = strlen(tl->uct_tl_name) + 1;
 
     if (MCA_BTL_UCT_TL_ATTR(tl, 0).cap.flags & UCT_IFACE_FLAG_CONNECT_TO_IFACE) {
         /* pad out to a multiple of 4 bytes */
-        return (4 + 3 + size + MCA_BTL_UCT_TL_ATTR(tl, 0).device_addr_len + MCA_BTL_UCT_TL_ATTR(tl, 0).iface_addr_len) & ~3;
+        return (4 + 3 + size + MCA_BTL_UCT_TL_ATTR(tl, 0).device_addr_len
+                + MCA_BTL_UCT_TL_ATTR(tl, 0).iface_addr_len)
+               & ~3;
     }
 
     return (4 + 3 + size + MCA_BTL_UCT_TL_ATTR(tl, 0).device_addr_len) & ~3;
 }
 
-static size_t mca_btl_uct_module_modex_size (mca_btl_uct_module_t *module)
+static size_t mca_btl_uct_module_modex_size(mca_btl_uct_module_t *module)
 {
-    size_t modex_size = 4 + strlen (module->md_name) + 1;
+    size_t modex_size = 4 + strlen(module->md_name) + 1;
 
     if (module->rdma_tl) {
-        modex_size += mca_btl_uct_tl_modex_size (module->rdma_tl);
+        modex_size += mca_btl_uct_tl_modex_size(module->rdma_tl);
     }
 
     if (module->am_tl && module->am_tl != module->rdma_tl) {
-        modex_size += mca_btl_uct_tl_modex_size (module->am_tl);
+        modex_size += mca_btl_uct_tl_modex_size(module->am_tl);
     }
 
     if (module->conn_tl && module->conn_tl != module->rdma_tl && module->conn_tl != module->am_tl) {
-        modex_size += mca_btl_uct_tl_modex_size (module->conn_tl);
+        modex_size += mca_btl_uct_tl_modex_size(module->conn_tl);
     }
 
     return modex_size;
 }
 
-static size_t mca_btl_uct_tl_modex_pack (mca_btl_uct_tl_t *tl, uint8_t *modex_data)
+static size_t mca_btl_uct_tl_modex_pack(mca_btl_uct_tl_t *tl, uint8_t *modex_data)
 {
     mca_btl_uct_device_context_t *dev_context = tl->uct_dev_contexts[0];
-    size_t modex_size = mca_btl_uct_tl_modex_size (tl);
+    size_t modex_size = mca_btl_uct_tl_modex_size(tl);
 
     *((uint32_t *) modex_data) = (uint32_t) modex_size;
     modex_data += 4;
 
-    strcpy ((char *) modex_data, tl->uct_tl_name);
-    modex_data += strlen (tl->uct_tl_name) + 1;
+    strcpy((char *) modex_data, tl->uct_tl_name);
+    modex_data += strlen(tl->uct_tl_name) + 1;
 
     /* NTH: only the first context is available. i assume the device addresses of the
      * contexts will be the same but they will have different iface addresses. i also
@@ -204,54 +225,55 @@ static size_t mca_btl_uct_tl_modex_pack (mca_btl_uct_tl_t *tl, uint8_t *modex_da
      * wrong then we can't delay creating the other contexts and must include their
      * information in the modex. */
     if (MCA_BTL_UCT_TL_ATTR(tl, 0).cap.flags & UCT_IFACE_FLAG_CONNECT_TO_IFACE) {
-        uct_iface_get_address (dev_context->uct_iface, (uct_iface_addr_t *) modex_data);
+        uct_iface_get_address(dev_context->uct_iface, (uct_iface_addr_t *) modex_data);
         modex_data += MCA_BTL_UCT_TL_ATTR(tl, 0).iface_addr_len;
     }
 
-    uct_iface_get_device_address (dev_context->uct_iface, (uct_device_addr_t *) modex_data);
+    uct_iface_get_device_address(dev_context->uct_iface, (uct_device_addr_t *) modex_data);
     modex_data += MCA_BTL_UCT_TL_ATTR(tl, 0).device_addr_len;
 
     return modex_size;
 }
 
-static int mca_btl_uct_modex_send (void)
+static int mca_btl_uct_modex_send(void)
 {
-    size_t modex_size = sizeof (mca_btl_uct_modex_t);
+    size_t modex_size = sizeof(mca_btl_uct_modex_t);
     mca_btl_uct_modex_t *modex;
     uint8_t *modex_data;
     int rc;
 
-    for (int i = 0 ; i < mca_btl_uct_component.module_count ; ++i) {
-        modex_size += mca_btl_uct_module_modex_size (mca_btl_uct_component.modules[i]);
+    for (int i = 0; i < mca_btl_uct_component.module_count; ++i) {
+        modex_size += mca_btl_uct_module_modex_size(mca_btl_uct_component.modules[i]);
     }
 
-    modex = alloca (modex_size);
+    modex = alloca(modex_size);
     modex_data = modex->data;
 
     modex->module_count = mca_btl_uct_component.module_count;
 
-    for (int i = 0 ; i < mca_btl_uct_component.module_count ; ++i) {
+    for (int i = 0; i < mca_btl_uct_component.module_count; ++i) {
         mca_btl_uct_module_t *module = mca_btl_uct_component.modules[i];
-        size_t name_len = strlen (module->md_name);
+        size_t name_len = strlen(module->md_name);
 
         /* pack the size */
-        *((uint32_t *) modex_data) = (uint32_t) mca_btl_uct_module_modex_size (module);
+        *((uint32_t *) modex_data) = (uint32_t) mca_btl_uct_module_modex_size(module);
 
         modex_data += 4;
 
-        strcpy ((char *) modex_data, module->md_name);
+        strcpy((char *) modex_data, module->md_name);
         modex_data += name_len + 1;
 
         if (module->rdma_tl) {
-            modex_data += mca_btl_uct_tl_modex_pack (module->rdma_tl, modex_data);
+            modex_data += mca_btl_uct_tl_modex_pack(module->rdma_tl, modex_data);
         }
 
         if (module->am_tl && module->am_tl != module->rdma_tl) {
-            modex_data += mca_btl_uct_tl_modex_pack (module->am_tl, modex_data);
+            modex_data += mca_btl_uct_tl_modex_pack(module->am_tl, modex_data);
         }
 
-        if (module->conn_tl && module->conn_tl != module->rdma_tl && module->conn_tl != module->am_tl) {
-            modex_data += mca_btl_uct_tl_modex_pack (module->conn_tl, modex_data);
+        if (module->conn_tl && module->conn_tl != module->rdma_tl
+            && module->conn_tl != module->am_tl) {
+            modex_data += mca_btl_uct_tl_modex_pack(module->conn_tl, modex_data);
         }
     }
 
@@ -259,13 +281,13 @@ static int mca_btl_uct_modex_send (void)
     return rc;
 }
 
-static mca_btl_uct_module_t *mca_btl_uct_alloc_module (const char *md_name, mca_btl_uct_md_t *md,
-                                                       size_t registration_size)
+static mca_btl_uct_module_t *mca_btl_uct_alloc_module(const char *md_name, mca_btl_uct_md_t *md,
+                                                      size_t registration_size)
 {
     mca_btl_uct_module_t *module;
     ucs_status_t ucs_status;
 
-    module = malloc (sizeof (*module));
+    module = malloc(sizeof(*module));
     if (NULL == module) {
         return NULL;
     }
@@ -283,43 +305,49 @@ static mca_btl_uct_module_t *mca_btl_uct_alloc_module (const char *md_name, mca_
     OBJ_CONSTRUCT(&module->pending_connection_reqs, opal_fifo_t);
 
     module->md = md;
-    module->md_name = strdup (md_name);
+    module->md_name = strdup(md_name);
     module->super.btl_registration_handle_size = registration_size;
 
-    ucs_status = ucs_async_context_create (UCS_ASYNC_MODE_THREAD, &module->ucs_async);
+    ucs_status = ucs_async_context_create(UCS_ASYNC_MODE_THREAD, &module->ucs_async);
     if (UCS_OK != ucs_status) {
         BTL_VERBOSE(("Could not create a UCT async context"));
-        mca_btl_uct_finalize (&module->super);
+        mca_btl_uct_finalize(&module->super);
         return NULL;
     }
 
     return module;
 }
 
-ucs_status_t mca_btl_uct_am_handler (void *arg, void *data, size_t length, unsigned flags)
+ucs_status_t mca_btl_uct_am_handler(void *arg, void *data, size_t length, unsigned flags)
 {
     mca_btl_uct_device_context_t *tl_context = (mca_btl_uct_device_context_t *) arg;
     mca_btl_uct_module_t *uct_btl = tl_context->uct_btl;
     mca_btl_uct_am_header_t *header = (mca_btl_uct_am_header_t *) data;
     mca_btl_active_message_callback_t *reg = mca_btl_base_active_message_trigger + header->data.tag;
-    mca_btl_base_segment_t seg = {.seg_addr = {.pval = (void *) ((intptr_t) data + sizeof (*header))},
-                                  .seg_len = length - sizeof (*header)};
-    mca_btl_base_receive_descriptor_t desc = {.endpoint = NULL, .des_segments = &seg, .des_segment_count = 1,
-                                              .tag = header->data.tag, .cbdata = reg->cbdata};
+    mca_btl_base_segment_t seg = {.seg_addr = {.pval = (void *) ((intptr_t) data
+                                                                 + sizeof(*header))},
+                                  .seg_len = length - sizeof(*header)};
+    mca_btl_base_receive_descriptor_t desc = {.endpoint = NULL,
+                                              .des_segments = &seg,
+                                              .des_segment_count = 1,
+                                              .tag = header->data.tag,
+                                              .cbdata = reg->cbdata};
 
     /* prevent recursion */
     tl_context->in_am_callback = true;
-    reg->cbfunc (&uct_btl->super, &desc);
+    reg->cbfunc(&uct_btl->super, &desc);
     tl_context->in_am_callback = false;
 
     return UCS_OK;
 }
 
 #if UCT_API >= UCT_VERSION(1, 7)
-static int mca_btl_uct_component_process_uct_md (uct_component_h component, uct_md_resource_desc_t *md_desc,
-                                                 char **allowed_ifaces)
+static int mca_btl_uct_component_process_uct_md(uct_component_h component,
+                                                uct_md_resource_desc_t *md_desc,
+                                                char **allowed_ifaces)
 #else
-static int mca_btl_uct_component_process_uct_md (uct_md_resource_desc_t *md_desc, char **allowed_ifaces)
+static int mca_btl_uct_component_process_uct_md(uct_md_resource_desc_t *md_desc,
+                                                char **allowed_ifaces)
 #endif
 {
     mca_rcache_base_resources_t rcache_resources;
@@ -331,6 +359,7 @@ static int mca_btl_uct_component_process_uct_md (uct_md_resource_desc_t *md_desc
     bool found = false;
     unsigned num_tls;
     char *tmp;
+    ucs_status_t ucs_status;
 
     if (MCA_BTL_UCT_MAX_MODULES == mca_btl_uct_component.module_count) {
         BTL_VERBOSE(("created the maximum number of allowable modules"));
@@ -339,9 +368,9 @@ static int mca_btl_uct_component_process_uct_md (uct_md_resource_desc_t *md_desc
 
     BTL_VERBOSE(("processing memory domain %s", md_desc->md_name));
 
-    for (int j = 0 ; allowed_ifaces[j] ; ++j) {
-        if (0 == strncmp (allowed_ifaces[j], md_desc->md_name, strlen (md_desc->md_name)) ||
-            0 == strcmp (allowed_ifaces[j], "all")) {
+    for (int j = 0; allowed_ifaces[j]; ++j) {
+        if (0 == strncmp(allowed_ifaces[j], md_desc->md_name, strlen(md_desc->md_name))
+            || 0 == strcmp(allowed_ifaces[j], "all")) {
             found = true;
             break;
         }
@@ -354,36 +383,59 @@ static int mca_btl_uct_component_process_uct_md (uct_md_resource_desc_t *md_desc
 
     md = OBJ_NEW(mca_btl_uct_md_t);
 
-
 #if UCT_API >= UCT_VERSION(1, 7)
-    uct_md_config_read (component, NULL, NULL, &uct_config);
-    uct_md_open (component, md_desc->md_name, uct_config, &md->uct_md);
+    ucs_status = uct_md_config_read(component, NULL, NULL, &uct_config);
+    if (UCS_OK != ucs_status) {
+        BTL_VERBOSE(("uct_md_config_read failed %d (%s)", ucs_status, ucs_status_string(ucs_status)));
+        return OPAL_ERR_NOT_AVAILABLE;
+    }
+    ucs_status = uct_md_open(component, md_desc->md_name, uct_config, &md->uct_md);
+    if (UCS_OK != ucs_status) {
+        BTL_VERBOSE(("uct_md_open failed %d (%s)", ucs_status, ucs_status_string(ucs_status)));
+        return OPAL_ERR_NOT_AVAILABLE;
+    }
 #else
-    uct_md_config_read (md_desc->md_name, NULL, NULL, &uct_config);
-    uct_md_open (md_desc->md_name, uct_config, &md->uct_md);
+    ucs_status = uct_md_config_read(md_desc->md_name, NULL, NULL, &uct_config);
+    if (UCS_OK != ucs_status) {
+        BTL_VERBOSE(("uct_md_config_read failed %d (%s)", ucs_status, ucs_status_string(ucs_status)));
+        return OPAL_ERR_NOT_AVAILABLE;
+    }
+    ucs_status = uct_md_open(md_desc->md_name, uct_config, &md->uct_md);
+    if (UCS_OK != ucs_status) {
+        BTL_VERBOSE(("uct_md_open failed %d (%s)", ucs_status, ucs_status_string(ucs_status)));
+        return OPAL_ERR_NOT_AVAILABLE;
+    }
 #endif
-    uct_config_release (uct_config);
+    uct_config_release(uct_config);
 
-    uct_md_query (md->uct_md, &md_attr);
-    uct_md_query_tl_resources (md->uct_md, &tl_desc, &num_tls);
+    ucs_status = uct_md_query(md->uct_md, &md_attr);
+    if (UCS_OK != ucs_status) {
+        BTL_VERBOSE(("uct_config_release failed %d (%s)", ucs_status, ucs_status_string(ucs_status)));
+        return OPAL_ERR_NOT_AVAILABLE;
+    }
+    ucs_status = uct_md_query_tl_resources(md->uct_md, &tl_desc, &num_tls);
+    if (UCS_OK != ucs_status) {
+        BTL_VERBOSE(("uct_config_release failed %d (%s)", ucs_status, ucs_status_string(ucs_status)));
+        return OPAL_ERR_NOT_AVAILABLE;
+    }
 
-    module = mca_btl_uct_alloc_module (md_desc->md_name, md, md_attr.rkey_packed_size);
+    module = mca_btl_uct_alloc_module(md_desc->md_name, md, md_attr.rkey_packed_size);
     if (NULL == module) {
-        uct_release_tl_resource_list (tl_desc);
+        uct_release_tl_resource_list(tl_desc);
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
-    (void) mca_btl_uct_query_tls (module, md, tl_desc, num_tls);
+    (void) mca_btl_uct_query_tls(module, md, tl_desc, num_tls);
 
-    uct_release_tl_resource_list (tl_desc);
+    uct_release_tl_resource_list(tl_desc);
 
-    /* release the initial reference to the md object. if any modules were created the UCT md will remain
-     * open until those modules are finalized. */
+    /* release the initial reference to the md object. if any modules were created the UCT md will
+     * remain open until those modules are finalized. */
     OBJ_RELEASE(md);
 
     if (NULL == module->am_tl && NULL == module->rdma_tl) {
         BTL_VERBOSE(("uct memory domain %s does not have any appropriate tls", md_desc->md_name));
-        mca_btl_uct_finalize (&module->super);
+        mca_btl_uct_finalize(&module->super);
         return OPAL_ERR_NOT_AVAILABLE;
     }
 
@@ -396,20 +448,21 @@ static int mca_btl_uct_component_process_uct_md (uct_md_resource_desc_t *md_desc
     /* NTH: a registration cache shouldn't be necessary when using UCT but there are measurable
      * performance benefits to using rcache/grdma instead of assuming UCT will do the right
      * thing. */
-    (void) opal_asprintf (&tmp, "uct.%s", module->md_name);
+    (void) opal_asprintf(&tmp, "uct.%s", module->md_name);
 
-    rcache_resources.cache_name     = tmp;
-    rcache_resources.reg_data       = (void *) module;
-    rcache_resources.sizeof_reg     = sizeof (mca_btl_uct_reg_t) + module->super.btl_registration_handle_size;
-    rcache_resources.register_mem   = mca_btl_uct_reg_mem;
+    rcache_resources.cache_name = tmp;
+    rcache_resources.reg_data = (void *) module;
+    rcache_resources.sizeof_reg = sizeof(mca_btl_uct_reg_t)
+                                  + module->super.btl_registration_handle_size;
+    rcache_resources.register_mem = mca_btl_uct_reg_mem;
     rcache_resources.deregister_mem = mca_btl_uct_dereg_mem;
 
-    module->rcache = mca_rcache_base_module_create ("grdma", module, &rcache_resources);
-    free (tmp);
+    module->rcache = mca_rcache_base_module_create("grdma", module, &rcache_resources);
+    free(tmp);
     if (NULL == module->rcache) {
         /* something when horribly wrong */
         BTL_VERBOSE(("could not allocate a registration cache for this btl module"));
-        mca_btl_uct_finalize (&module->super);
+        mca_btl_uct_finalize(&module->super);
         return OPAL_ERROR;
     }
 
@@ -417,36 +470,36 @@ static int mca_btl_uct_component_process_uct_md (uct_md_resource_desc_t *md_desc
 }
 
 #if UCT_API >= UCT_VERSION(1, 7)
-static int mca_btl_uct_component_process_uct_component (uct_component_h component, char **allowed_ifaces)
+static int mca_btl_uct_component_process_uct_component(uct_component_h component,
+                                                       char **allowed_ifaces)
 {
-    uct_component_attr_t attr = {.field_mask = UCT_COMPONENT_ATTR_FIELD_NAME |
-                                 UCT_COMPONENT_ATTR_FIELD_MD_RESOURCE_COUNT};
+    uct_component_attr_t attr = {.field_mask = UCT_COMPONENT_ATTR_FIELD_NAME
+                                               | UCT_COMPONENT_ATTR_FIELD_MD_RESOURCE_COUNT};
     ucs_status_t ucs_status;
     int rc;
 
-    ucs_status = uct_component_query (component, &attr);
+    ucs_status = uct_component_query(component, &attr);
     if (UCS_OK != ucs_status) {
         return OPAL_ERROR;
     }
 
     BTL_VERBOSE(("processing uct component %s", attr.name));
 
-    attr.md_resources = calloc (attr.md_resource_count, sizeof (*attr.md_resources));
+    attr.md_resources = calloc(attr.md_resource_count, sizeof(*attr.md_resources));
     attr.field_mask |= UCT_COMPONENT_ATTR_FIELD_MD_RESOURCES;
-    ucs_status = uct_component_query (component, &attr);
+    ucs_status = uct_component_query(component, &attr);
     if (UCS_OK != ucs_status) {
         return OPAL_ERROR;
     }
 
-    for (int i = 0 ; i < attr.md_resource_count ; ++i) {
-        rc = mca_btl_uct_component_process_uct_md (component, attr.md_resources + i,
-                                                   allowed_ifaces);
+    for (unsigned i = 0; i < attr.md_resource_count; ++i) {
+        rc = mca_btl_uct_component_process_uct_md(component, attr.md_resources + i, allowed_ifaces);
         if (OPAL_SUCCESS != rc) {
             break;
         }
     }
 
-    free (attr.md_resources);
+    free(attr.md_resources);
 
     return OPAL_SUCCESS;
 }
@@ -460,26 +513,27 @@ static int mca_btl_uct_component_process_uct_component (uct_component_h componen
  *  (3) register BTL parameters with the MCA
  */
 
-static mca_btl_base_module_t **mca_btl_uct_component_init (int *num_btl_modules, bool enable_progress_threads,
-                                                           bool enable_mpi_threads)
+static mca_btl_base_module_t **mca_btl_uct_component_init(int *num_btl_modules,
+                                                          bool enable_progress_threads,
+                                                          bool enable_mpi_threads)
 {
-    /* for this BTL to be useful the interface needs to support RDMA and certain atomic operations */
+    /* for this BTL to be useful the interface needs to support RDMA and certain atomic operations
+     */
     struct mca_btl_base_module_t **base_modules;
-    uct_md_resource_desc_t *resources;
-    unsigned resource_count;
     ucs_status_t ucs_status;
     char **allowed_ifaces;
     int rc;
 
     BTL_VERBOSE(("initializing uct btl"));
 
-    if (NULL == mca_btl_uct_component.memory_domains || 0 == strlen (mca_btl_uct_component.memory_domains) ||
-        0 == strcmp (mca_btl_uct_component.memory_domains, "none")) {
+    if (NULL == mca_btl_uct_component.memory_domains
+        || 0 == strlen(mca_btl_uct_component.memory_domains)
+        || 0 == strcmp(mca_btl_uct_component.memory_domains, "none")) {
         BTL_VERBOSE(("no uct memory domains specified"));
         return NULL;
     }
 
-    allowed_ifaces = opal_argv_split (mca_btl_uct_component.memory_domains, ',');
+    allowed_ifaces = opal_argv_split(mca_btl_uct_component.memory_domains, ',');
     if (NULL == allowed_ifaces) {
         return NULL;
     }
@@ -497,42 +551,44 @@ static mca_btl_base_module_t **mca_btl_uct_component_init (int *num_btl_modules,
     }
 
     /* generate all suitable btl modules */
-    for (unsigned i = 0 ; i < num_components ; ++i) {
-        rc = mca_btl_uct_component_process_uct_component (components[i], allowed_ifaces);
+    for (unsigned i = 0; i < num_components; ++i) {
+        rc = mca_btl_uct_component_process_uct_component(components[i], allowed_ifaces);
         if (OPAL_SUCCESS != rc) {
             break;
         }
     }
 
-    uct_release_component_list (components);
+    uct_release_component_list(components);
 
 #else /* UCT 1.6 and older */
+    uct_md_resource_desc_t *resources;
+    unsigned resource_count;
 
-    uct_query_md_resources (&resources, &resource_count);
+    uct_query_md_resources(&resources, &resource_count);
 
     /* generate all suitable btl modules */
-    for (unsigned i = 0 ; i < resource_count ; ++i) {
-        rc = mca_btl_uct_component_process_uct_md (resources + i, allowed_ifaces);
+    for (unsigned i = 0; i < resource_count; ++i) {
+        rc = mca_btl_uct_component_process_uct_md(resources + i, allowed_ifaces);
         if (OPAL_SUCCESS != rc) {
             break;
         }
     }
 
-    uct_release_md_resource_list (resources);
+    uct_release_md_resource_list(resources);
 
 #endif /* UCT_API >= UCT_VERSION(1, 7) */
 
-    opal_argv_free (allowed_ifaces);
-    mca_btl_uct_modex_send ();
+    opal_argv_free(allowed_ifaces);
+    mca_btl_uct_modex_send();
 
     /* pass module array back to caller */
-    base_modules = calloc (mca_btl_uct_component.module_count, sizeof (*base_modules));
+    base_modules = calloc(mca_btl_uct_component.module_count, sizeof(*base_modules));
     if (NULL == base_modules) {
         return NULL;
     }
 
-    memcpy (base_modules, mca_btl_uct_component.modules, mca_btl_uct_component.module_count *
-            sizeof (mca_btl_uct_component.modules[0]));
+    memcpy(base_modules, mca_btl_uct_component.modules,
+           mca_btl_uct_component.module_count * sizeof(mca_btl_uct_component.modules[0]));
 
     *num_btl_modules = mca_btl_uct_component.module_count;
 
@@ -542,7 +598,7 @@ static mca_btl_base_module_t **mca_btl_uct_component_init (int *num_btl_modules,
     return base_modules;
 }
 
-static int mca_btl_uct_tl_progress (mca_btl_uct_tl_t *tl, int starting_index)
+static int mca_btl_uct_tl_progress(mca_btl_uct_tl_t *tl, int starting_index)
 {
     unsigned int ret = 0;
 
@@ -550,35 +606,35 @@ static int mca_btl_uct_tl_progress (mca_btl_uct_tl_t *tl, int starting_index)
         return 0;
     }
 
-    for (int j = 0 ; j < tl->max_device_contexts ; ++j) {
+    for (int j = 0; j < tl->max_device_contexts; ++j) {
         if (tl->uct_dev_contexts[j]) {
-            ret += mca_btl_uct_context_progress (tl->uct_dev_contexts[j]);
+            ret += mca_btl_uct_context_progress(tl->uct_dev_contexts[j]);
         }
     }
 
     return ret;
 }
 
-static int mca_btl_uct_component_progress_pending (mca_btl_uct_module_t *uct_btl)
+static int mca_btl_uct_component_progress_pending(mca_btl_uct_module_t *uct_btl)
 {
     mca_btl_uct_base_frag_t *frag, *next;
     int completed = 0;
     size_t count;
 
-    if (0 == (count = opal_list_get_size (&uct_btl->pending_frags))) {
+    if (0 == (count = opal_list_get_size(&uct_btl->pending_frags))) {
         return 0;
     }
 
     OPAL_THREAD_LOCK(&uct_btl->lock);
-    OPAL_LIST_FOREACH_SAFE(frag, next, &uct_btl->pending_frags, mca_btl_uct_base_frag_t) {
+    OPAL_LIST_FOREACH_SAFE (frag, next, &uct_btl->pending_frags, mca_btl_uct_base_frag_t) {
         if (!frag->ready) {
             continue;
         }
 
-        opal_list_remove_item (&uct_btl->pending_frags, (opal_list_item_t *) frag);
+        opal_list_remove_item(&uct_btl->pending_frags, (opal_list_item_t *) frag);
 
-        if (OPAL_SUCCESS > mca_btl_uct_send_frag (uct_btl, frag, false)) {
-            opal_list_prepend (&uct_btl->pending_frags, (opal_list_item_t *) frag);
+        if (OPAL_SUCCESS > mca_btl_uct_send_frag(uct_btl, frag, false)) {
+            opal_list_prepend(&uct_btl->pending_frags, (opal_list_item_t *) frag);
         } else {
             completed++;
         }
@@ -591,39 +647,42 @@ static int mca_btl_uct_component_progress_pending (mca_btl_uct_module_t *uct_btl
 /**
  * @brief UCT BTL progress function
  *
- * This function explictly progresses all workers.
+ * This function explicitly progresses all workers.
  */
-static int mca_btl_uct_component_progress (void)
+static int mca_btl_uct_component_progress(void)
 {
-    int starting_index = mca_btl_uct_get_context_index ();
+    int starting_index = mca_btl_uct_get_context_index();
     unsigned ret = 0;
 
-    for (int i = 0 ; i < mca_btl_uct_component.module_count ; ++i) {
+    for (int i = 0; i < mca_btl_uct_component.module_count; ++i) {
         mca_btl_uct_module_t *module = mca_btl_uct_component.modules[i];
 
         /* unlike ucp, uct actually tells us something useful! its almost like it was "inspired"
          * by the btl progress functions.... */
-        ret += mca_btl_uct_tl_progress (module->rdma_tl, starting_index);
+        ret += mca_btl_uct_tl_progress(module->rdma_tl, starting_index);
 
         if (module->am_tl != module->rdma_tl) {
-            ret += mca_btl_uct_tl_progress (module->am_tl, starting_index);
+            ret += mca_btl_uct_tl_progress(module->am_tl, starting_index);
         }
 
         if (module->conn_tl) {
             mca_btl_uct_pending_connection_request_t *request;
 
             if (module->conn_tl != module->am_tl && module->conn_tl != module->rdma_tl) {
-                ret += mca_btl_uct_tl_progress (module->conn_tl, 0);
+                ret += mca_btl_uct_tl_progress(module->conn_tl, 0);
             }
 
-            while (NULL != (request = (mca_btl_uct_pending_connection_request_t *) opal_fifo_pop_atomic (&module->pending_connection_reqs))) {
-                mca_btl_uct_process_connection_request (module, (mca_btl_uct_conn_req_t *) request->request_data);
+            while (NULL
+                   != (request = (mca_btl_uct_pending_connection_request_t *) opal_fifo_pop_atomic(
+                           &module->pending_connection_reqs))) {
+                mca_btl_uct_process_connection_request(module, (mca_btl_uct_conn_req_t *)
+                                                                   request->request_data);
                 OBJ_RELEASE(request);
             }
         }
 
-        if (0 != opal_list_get_size (&module->pending_frags)) {
-            mca_btl_uct_component_progress_pending (module);
+        if (0 != opal_list_get_size(&module->pending_frags)) {
+            mca_btl_uct_component_progress_pending(module);
         }
     }
 
@@ -633,18 +692,22 @@ static int mca_btl_uct_component_progress (void)
 /** UCT btl component */
 mca_btl_uct_component_t mca_btl_uct_component = {
     .super = {
-        .btl_version = {
-            MCA_BTL_DEFAULT_VERSION("uct"),
-            .mca_open_component = mca_btl_uct_component_open,
-            .mca_close_component = mca_btl_uct_component_close,
-            .mca_register_component_params = mca_btl_uct_component_register,
-        },
-        .btl_data = {
-            /* The component is not checkpoint ready */
-            .param_field = MCA_BASE_METADATA_PARAM_NONE
-        },
+        .btl_version =
+            {
+                MCA_BTL_DEFAULT_VERSION("uct"),
+                .mca_open_component = mca_btl_uct_component_open,
+                .mca_close_component = mca_btl_uct_component_close,
+                .mca_register_component_params = mca_btl_uct_component_register,
+            },
+        .btl_data =
+            {/* The component is not checkpoint ready */
+             .param_field = MCA_BASE_METADATA_PARAM_NONE},
 
         .btl_init = mca_btl_uct_component_init,
         .btl_progress = mca_btl_uct_component_progress,
-    }
-};
+    }};
+
+static void safety_valve(void) __opal_attribute_destructor__;
+void safety_valve(void) {
+    opal_mem_hooks_unregister_release(mca_btl_uct_mem_release_cb);
+}
diff --git a/opal/mca/btl/uct/btl_uct_device_context.h b/opal/mca/btl/uct/btl_uct_device_context.h
index f88e4314710..7e25e0bef19 100644
--- a/opal/mca/btl/uct/btl_uct_device_context.h
+++ b/opal/mca/btl/uct/btl_uct_device_context.h
@@ -10,11 +10,11 @@
  */
 
 #if !defined(BTL_UCT_DEVICE_CONTEXT_H)
-#define BTL_UCT_DEVICE_CONTEXT_H
+#    define BTL_UCT_DEVICE_CONTEXT_H
 
-#include "btl_uct.h"
-#include "btl_uct_rdma.h"
-#include "btl_uct_frag.h"
+#    include "btl_uct.h"
+#    include "btl_uct_frag.h"
+#    include "btl_uct_rdma.h"
 
 /**
  * @brief Create a new device context for the given transport
@@ -23,68 +23,73 @@
  * @param[in] tl         btl uct tl pointer
  * @param[in] context_id identifier for this context (0..MCA_BTL_UCT_MAX_WORKERS-1)
  */
-mca_btl_uct_device_context_t *mca_btl_uct_context_create (mca_btl_uct_module_t *module, mca_btl_uct_tl_t *tl, int context_id, bool enable_progress);
+mca_btl_uct_device_context_t *mca_btl_uct_context_create(mca_btl_uct_module_t *module,
+                                                         mca_btl_uct_tl_t *tl, int context_id,
+                                                         bool enable_progress);
 
 /**
  * @brief Destroy a device context and release all resources
  *
  * @param[in] context   btl uct device context
  *
- * This call frees a device context and all assoicated resources. It is not
+ * This call frees a device context and all associated resources. It is not
  * valid to use the device context after this returns.
  */
-void mca_btl_uct_context_destroy (mca_btl_uct_device_context_t *context);
+void mca_btl_uct_context_destroy(mca_btl_uct_device_context_t *context);
 
-static inline bool mca_btl_uct_context_trylock (mca_btl_uct_device_context_t *context)
+static inline bool mca_btl_uct_context_trylock(mca_btl_uct_device_context_t *context)
 {
     return OPAL_THREAD_TRYLOCK(&context->mutex);
 }
 
-static inline void mca_btl_uct_context_lock (mca_btl_uct_device_context_t *context)
+static inline void mca_btl_uct_context_lock(mca_btl_uct_device_context_t *context)
 {
-    OPAL_THREAD_LOCK (&context->mutex);
+    OPAL_THREAD_LOCK(&context->mutex);
 }
 
-static inline void mca_btl_uct_context_unlock (mca_btl_uct_device_context_t *context)
+static inline void mca_btl_uct_context_unlock(mca_btl_uct_device_context_t *context)
 {
-    OPAL_THREAD_UNLOCK (&context->mutex);
+    OPAL_THREAD_UNLOCK(&context->mutex);
 }
 
-#define MCA_BTL_UCT_CONTEXT_SERIALIZE(context,code)     \
-    do {                                                \
-        mca_btl_uct_context_lock (context);             \
-        code;                                           \
-        mca_btl_uct_context_unlock(context);            \
-    } while (0);
+#    define MCA_BTL_UCT_CONTEXT_SERIALIZE(context, code) \
+        do {                                             \
+            mca_btl_uct_context_lock(context);           \
+            code;                                        \
+            mca_btl_uct_context_unlock(context);         \
+        } while (0);
 
-static inline int mca_btl_uct_get_context_index (void)
+static inline int mca_btl_uct_get_context_index(void)
 {
     static opal_atomic_uint32_t next_uct_index = 0;
     int context_id;
 
-#if OPAL_C_HAVE__THREAD_LOCAL
+#    if OPAL_C_HAVE__THREAD_LOCAL
     if (mca_btl_uct_component.bind_threads_to_contexts) {
         static _Thread_local int uct_index = -1;
 
         context_id = uct_index;
         if (OPAL_UNLIKELY(-1 == context_id)) {
-            context_id = uct_index = opal_atomic_fetch_add_32 ((opal_atomic_int32_t *) &next_uct_index, 1) %
-                mca_btl_uct_component.num_contexts_per_module;
+            context_id = uct_index = opal_atomic_fetch_add_32((opal_atomic_int32_t
+                                                                   *) &next_uct_index,
+                                                              1)
+                                     % mca_btl_uct_component.num_contexts_per_module;
         }
     } else {
-#endif
-        /* avoid using atomics in this. i doubt it improves performance to ensure atomicity on the next
-         * index in this case. */
+#    endif
+        /* avoid using atomics in this. i doubt it improves performance to ensure atomicity on the
+         * next index in this case. */
         context_id = next_uct_index++ % mca_btl_uct_component.num_contexts_per_module;
-#if OPAL_C_HAVE__THREAD_LOCAL
+#    if OPAL_C_HAVE__THREAD_LOCAL
     }
-#endif
+#    endif
 
     return context_id;
 }
 
 static inline mca_btl_uct_device_context_t *
-mca_btl_uct_module_get_tl_context_specific (mca_btl_uct_module_t *module, mca_btl_uct_tl_t *tl, int context_id)
+mca_btl_uct_module_get_tl_context_specific(mca_btl_uct_module_t *module, mca_btl_uct_tl_t *tl,
+                                           int context_id)
 {
     mca_btl_uct_device_context_t *context = tl->uct_dev_contexts[context_id];
 
@@ -92,7 +97,9 @@ mca_btl_uct_module_get_tl_context_specific (mca_btl_uct_module_t *module, mca_bt
         OPAL_THREAD_LOCK(&module->lock);
         context = tl->uct_dev_contexts[context_id];
         if (OPAL_UNLIKELY(NULL == context)) {
-            context = tl->uct_dev_contexts[context_id] = mca_btl_uct_context_create (module, tl, context_id, true);
+            context = tl->uct_dev_contexts[context_id] = mca_btl_uct_context_create(module, tl,
+                                                                                    context_id,
+                                                                                    true);
         }
         OPAL_THREAD_UNLOCK(&module->lock);
     }
@@ -100,32 +107,39 @@ mca_btl_uct_module_get_tl_context_specific (mca_btl_uct_module_t *module, mca_bt
     return context;
 }
 
-static inline mca_btl_uct_device_context_t *mca_btl_uct_module_get_rdma_context (mca_btl_uct_module_t *module)
+static inline mca_btl_uct_device_context_t *
+mca_btl_uct_module_get_rdma_context(mca_btl_uct_module_t *module)
 {
-    return mca_btl_uct_module_get_tl_context_specific (module, module->rdma_tl, mca_btl_uct_get_context_index ());
+    return mca_btl_uct_module_get_tl_context_specific(module, module->rdma_tl,
+                                                      mca_btl_uct_get_context_index());
 }
 
-static inline mca_btl_uct_device_context_t *mca_btl_uct_module_get_rdma_context_specific (mca_btl_uct_module_t *module, int context_id)
+static inline mca_btl_uct_device_context_t *
+mca_btl_uct_module_get_rdma_context_specific(mca_btl_uct_module_t *module, int context_id)
 {
-    return mca_btl_uct_module_get_tl_context_specific (module, module->rdma_tl, context_id);
+    return mca_btl_uct_module_get_tl_context_specific(module, module->rdma_tl, context_id);
 }
 
-static inline mca_btl_uct_device_context_t *mca_btl_uct_module_get_am_context (mca_btl_uct_module_t *module)
+static inline mca_btl_uct_device_context_t *
+mca_btl_uct_module_get_am_context(mca_btl_uct_module_t *module)
 {
-    return mca_btl_uct_module_get_tl_context_specific (module, module->am_tl, mca_btl_uct_get_context_index ());
+    return mca_btl_uct_module_get_tl_context_specific(module, module->am_tl,
+                                                      mca_btl_uct_get_context_index());
 }
 
-static inline void mca_btl_uct_device_handle_completions (mca_btl_uct_device_context_t *dev_context)
+static inline void mca_btl_uct_device_handle_completions(mca_btl_uct_device_context_t *dev_context)
 {
     mca_btl_uct_uct_completion_t *comp;
 
-    while (NULL != (comp = (mca_btl_uct_uct_completion_t *) opal_fifo_pop (&dev_context->completion_fifo))) {
+    while (
+        NULL
+        != (comp = (mca_btl_uct_uct_completion_t *) opal_fifo_pop(&dev_context->completion_fifo))) {
         int rc = UCS_OK == comp->status ? OPAL_SUCCESS : OPAL_ERROR;
 
         if (comp->frag) {
             /* reset the count */
             comp->uct_comp.count = 1;
-            mca_btl_uct_frag_complete (comp->frag, rc);
+            mca_btl_uct_frag_complete(comp->frag, rc);
 
             continue;
         }
@@ -133,13 +147,13 @@ static inline void mca_btl_uct_device_handle_completions (mca_btl_uct_device_con
         /* we may be calling the callback before remote completion. this is in violation of the
          * btl interface specification but should not hurt in non-ob1 use cases. if this ever
          * becomes a problem we can look at possible solutions. */
-        comp->cbfunc (comp->btl, comp->endpoint, comp->local_address, comp->local_handle,
-                      comp->cbcontext, comp->cbdata, rc);
-        mca_btl_uct_uct_completion_release (comp);
+        comp->cbfunc(comp->btl, comp->endpoint, comp->local_address, comp->local_handle,
+                     comp->cbcontext, comp->cbdata, rc);
+        mca_btl_uct_uct_completion_release(comp);
     }
 }
 
-static inline int mca_btl_uct_context_progress (mca_btl_uct_device_context_t *context)
+static inline int mca_btl_uct_context_progress(mca_btl_uct_device_context_t *context)
 {
     int ret = 0;
 
@@ -147,11 +161,11 @@ static inline int mca_btl_uct_context_progress (mca_btl_uct_device_context_t *co
         return 0;
     }
 
-    if (!mca_btl_uct_context_trylock (context)) {
-        ret = uct_worker_progress (context->uct_worker);
-        mca_btl_uct_context_unlock (context);
+    if (!mca_btl_uct_context_trylock(context)) {
+        ret = uct_worker_progress(context->uct_worker);
+        mca_btl_uct_context_unlock(context);
 
-        mca_btl_uct_device_handle_completions (context);
+        mca_btl_uct_device_handle_completions(context);
     }
 
     return ret;
diff --git a/opal/mca/btl/uct/btl_uct_endpoint.c b/opal/mca/btl/uct/btl_uct_endpoint.c
index ccdbd4511a2..695fd754aa2 100644
--- a/opal/mca/btl/uct/btl_uct_endpoint.c
+++ b/opal/mca/btl/uct/btl_uct_endpoint.c
@@ -4,7 +4,7 @@
  *                         reserved.
  * Copyright (c) 2018      Triad National Security, LLC. All rights
  *                         reserved.
- * Copyright (c) 2019      Google, LLC. All rights reserved.
+ * Copyright (c) 2019-2025 Google, LLC. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -12,40 +12,46 @@
  * $HEADER$
  */
 
-#include "btl_uct.h"
 #include "btl_uct_endpoint.h"
-#include "btl_uct_device_context.h"
+#include "btl_uct.h"
 #include "btl_uct_am.h"
+#include "btl_uct_device_context.h"
+#include "opal/mca/timer/base/base.h"
 #include "opal/util/proc.h"
 
-static void mca_btl_uct_endpoint_construct (mca_btl_uct_endpoint_t *endpoint)
+static void mca_btl_uct_endpoint_construct(mca_btl_uct_endpoint_t *endpoint)
 {
-    memset (endpoint->uct_eps, 0, sizeof (endpoint->uct_eps[0]) * mca_btl_uct_component.num_contexts_per_module);
+    memset(endpoint->uct_eps, 0,
+           sizeof(endpoint->uct_eps[0]) * mca_btl_uct_component.num_contexts_per_module);
     endpoint->conn_ep = NULL;
     OBJ_CONSTRUCT(&endpoint->ep_lock, opal_recursive_mutex_t);
 }
 
-static void mca_btl_uct_endpoint_destruct (mca_btl_uct_endpoint_t *endpoint)
+static void mca_btl_uct_endpoint_destruct(mca_btl_uct_endpoint_t *endpoint)
 {
-    for (int tl_index = 0 ; tl_index < 2 ; ++tl_index) {
-        for (int i = 0 ; i < mca_btl_uct_component.num_contexts_per_module ; ++i) {
+    for (int tl_index = 0; tl_index < 2; ++tl_index) {
+        for (int i = 0; i < mca_btl_uct_component.num_contexts_per_module; ++i) {
             if (NULL != endpoint->uct_eps[i][tl_index].uct_ep) {
-                uct_ep_destroy (endpoint->uct_eps[i][tl_index].uct_ep);
+                uct_ep_destroy(endpoint->uct_eps[i][tl_index].uct_ep);
             }
         }
     }
 
     OBJ_DESTRUCT(&endpoint->ep_lock);
+    if (endpoint->conn_ep) {
+        OBJ_RELEASE(endpoint->conn_ep);
+    }
 }
 
-OBJ_CLASS_INSTANCE(mca_btl_uct_endpoint_t, opal_object_t,
-                   mca_btl_uct_endpoint_construct,
+OBJ_CLASS_INSTANCE(mca_btl_uct_endpoint_t, opal_object_t, mca_btl_uct_endpoint_construct,
                    mca_btl_uct_endpoint_destruct);
 
-mca_btl_base_endpoint_t *mca_btl_uct_endpoint_create (opal_proc_t *proc)
+mca_btl_base_endpoint_t *mca_btl_uct_endpoint_create(opal_proc_t *proc)
 {
-    mca_btl_uct_endpoint_t *endpoint = calloc (1, sizeof (*endpoint) + sizeof (endpoint->uct_eps[0]) *
-                                               mca_btl_uct_component.num_contexts_per_module);
+    mca_btl_uct_endpoint_t *endpoint = calloc(1, sizeof(*endpoint)
+                                                     + sizeof(endpoint->uct_eps[0])
+                                                           * mca_btl_uct_component
+                                                                 .num_contexts_per_module);
 
     if (OPAL_UNLIKELY(NULL == endpoint)) {
         return NULL;
@@ -57,24 +63,25 @@ mca_btl_base_endpoint_t *mca_btl_uct_endpoint_create (opal_proc_t *proc)
     return (mca_btl_base_endpoint_t *) endpoint;
 }
 
-static unsigned char *mca_btl_uct_process_modex_tl (unsigned char *modex_data)
+static unsigned char *mca_btl_uct_process_modex_tl(unsigned char *modex_data)
 {
-    BTL_VERBOSE(("processing modex for tl %s. size: %u", modex_data + 4, *((uint32_t *) modex_data)));
+    BTL_VERBOSE(
+        ("processing modex for tl %s. size: %u", modex_data + 4, *((uint32_t *) modex_data)));
 
     /* skip size and name */
-    return modex_data + 4 + strlen ((char *) modex_data + 4) + 1;
+    return modex_data + 4 + strlen((char *) modex_data + 4) + 1;
 }
 
-static void mca_btl_uct_process_modex (mca_btl_uct_module_t *uct_btl, unsigned char *modex_data,
-                                       unsigned char **rdma_tl_data, unsigned char **am_tl_data,
-                                       unsigned char **conn_tl_data)
+static void mca_btl_uct_process_modex(mca_btl_uct_module_t *uct_btl, unsigned char *modex_data,
+                                      unsigned char **rdma_tl_data, unsigned char **am_tl_data,
+                                      unsigned char **conn_tl_data)
 {
     BTL_VERBOSE(("processing remote modex data"));
 
     if (uct_btl->rdma_tl) {
         BTL_VERBOSE(("modex contains RDMA data"));
         if (rdma_tl_data) {
-            *rdma_tl_data = mca_btl_uct_process_modex_tl (modex_data);
+            *rdma_tl_data = mca_btl_uct_process_modex_tl(modex_data);
         }
         modex_data += *((uint32_t *) modex_data);
     } else if (rdma_tl_data) {
@@ -84,17 +91,18 @@ static void mca_btl_uct_process_modex (mca_btl_uct_module_t *uct_btl, unsigned c
     if (uct_btl->am_tl && uct_btl->am_tl != uct_btl->rdma_tl) {
         BTL_VERBOSE(("modex contains active message data"));
         if (am_tl_data) {
-            *am_tl_data = mca_btl_uct_process_modex_tl (modex_data);
+            *am_tl_data = mca_btl_uct_process_modex_tl(modex_data);
         }
         modex_data += *((uint32_t *) modex_data);
     } else if (am_tl_data) {
         *am_tl_data = NULL;
     }
 
-    if (uct_btl->conn_tl && uct_btl->conn_tl != uct_btl->rdma_tl && uct_btl->conn_tl != uct_btl->am_tl) {
+    if (uct_btl->conn_tl && uct_btl->conn_tl != uct_btl->rdma_tl
+        && uct_btl->conn_tl != uct_btl->am_tl) {
         BTL_VERBOSE(("modex contains connection data"));
         if (conn_tl_data) {
-            *conn_tl_data = mca_btl_uct_process_modex_tl (modex_data);
+            *conn_tl_data = mca_btl_uct_process_modex_tl(modex_data);
         }
         modex_data += *((uint32_t *) modex_data);
     } else if (conn_tl_data) {
@@ -102,31 +110,38 @@ static void mca_btl_uct_process_modex (mca_btl_uct_module_t *uct_btl, unsigned c
     }
 }
 
-static inline ucs_status_t mca_btl_uct_ep_create_connected_compat (uct_iface_h iface, uct_device_addr_t *device_addr,
-                                                                   uct_iface_addr_t *iface_addr, uct_ep_h *uct_ep)
+static inline ucs_status_t mca_btl_uct_ep_create_connected_compat(uct_iface_h iface,
+                                                                  uct_device_addr_t *device_addr,
+                                                                  uct_iface_addr_t *iface_addr,
+                                                                  uct_ep_h *uct_ep)
 {
 #if UCT_API >= UCT_VERSION(1, 6)
-    uct_ep_params_t ep_params = {.field_mask = UCT_EP_PARAM_FIELD_IFACE | UCT_EP_PARAM_FIELD_DEV_ADDR | UCT_EP_PARAM_FIELD_IFACE_ADDR,
-                                 .iface = iface, .dev_addr = device_addr, .iface_addr = iface_addr};
-    return uct_ep_create (&ep_params, uct_ep);
+    uct_ep_params_t ep_params = {.field_mask = UCT_EP_PARAM_FIELD_IFACE
+                                               | UCT_EP_PARAM_FIELD_DEV_ADDR
+                                               | UCT_EP_PARAM_FIELD_IFACE_ADDR,
+                                 .iface = iface,
+                                 .dev_addr = device_addr,
+                                 .iface_addr = iface_addr};
+    return uct_ep_create(&ep_params, uct_ep);
 #else
-    return uct_ep_create_connected (iface, device_addr, iface_addr, uct_ep);
+    return uct_ep_create_connected(iface, device_addr, iface_addr, uct_ep);
 #endif
 }
 
-static inline ucs_status_t mca_btl_uct_ep_create_compat (uct_iface_h iface, uct_ep_h *uct_ep)
+static inline ucs_status_t mca_btl_uct_ep_create_compat(uct_iface_h iface, uct_ep_h *uct_ep)
 {
 #if UCT_API >= UCT_VERSION(1, 6)
     uct_ep_params_t ep_params = {.field_mask = UCT_EP_PARAM_FIELD_IFACE, .iface = iface};
-    return uct_ep_create (&ep_params, uct_ep);
+    return uct_ep_create(&ep_params, uct_ep);
 #else
-    return uct_ep_create (iface, uct_ep);
+    return uct_ep_create(iface, uct_ep);
 #endif
 }
 
-static int mca_btl_uct_endpoint_connect_iface (mca_btl_uct_module_t *uct_btl, mca_btl_uct_tl_t *tl,
-                                               mca_btl_uct_device_context_t *tl_context,
-                                               mca_btl_uct_tl_endpoint_t *tl_endpoint, uint8_t *tl_data)
+static int mca_btl_uct_endpoint_connect_iface(mca_btl_uct_module_t *uct_btl, mca_btl_uct_tl_t *tl,
+                                              mca_btl_uct_device_context_t *tl_context,
+                                              mca_btl_uct_tl_endpoint_t *tl_endpoint,
+                                              uint8_t *tl_data)
 {
     uct_device_addr_t *device_addr = NULL;
     uct_iface_addr_t *iface_addr;
@@ -134,27 +149,30 @@ static int mca_btl_uct_endpoint_connect_iface (mca_btl_uct_module_t *uct_btl, mc
 
     /* easy case. just connect to the interface */
     iface_addr = (uct_iface_addr_t *) tl_data;
-    device_addr = (uct_device_addr_t *) ((uintptr_t) iface_addr + MCA_BTL_UCT_TL_ATTR(tl, tl_context->context_id).iface_addr_len);
+    device_addr = (uct_device_addr_t *) ((uintptr_t) iface_addr
+                                         + MCA_BTL_UCT_TL_ATTR(tl, tl_context->context_id)
+                                               .iface_addr_len);
 
     BTL_VERBOSE(("connecting endpoint to interface"));
 
-    mca_btl_uct_context_lock (tl_context);
-    ucs_status = mca_btl_uct_ep_create_connected_compat (tl_context->uct_iface, device_addr, iface_addr, &tl_endpoint->uct_ep);
+    mca_btl_uct_context_lock(tl_context);
+    ucs_status = mca_btl_uct_ep_create_connected_compat(tl_context->uct_iface, device_addr,
+                                                        iface_addr, &tl_endpoint->uct_ep);
     tl_endpoint->flags = MCA_BTL_UCT_ENDPOINT_FLAG_CONN_READY;
-    mca_btl_uct_context_unlock (tl_context);
+    mca_btl_uct_context_unlock(tl_context);
 
     return (UCS_OK == ucs_status) ? OPAL_SUCCESS : OPAL_ERROR;
 }
 
-static void mca_btl_uct_connection_ep_construct (mca_btl_uct_connection_ep_t *ep)
+static void mca_btl_uct_connection_ep_construct(mca_btl_uct_connection_ep_t *ep)
 {
     ep->uct_ep = NULL;
 }
 
-static void mca_btl_uct_connection_ep_destruct (mca_btl_uct_connection_ep_t *ep)
+static void mca_btl_uct_connection_ep_destruct(mca_btl_uct_connection_ep_t *ep)
 {
     if (ep->uct_ep) {
-        uct_ep_destroy (ep->uct_ep);
+        uct_ep_destroy(ep->uct_ep);
         ep->uct_ep = NULL;
     }
 }
@@ -168,35 +186,45 @@ struct mca_btl_uct_conn_completion_t {
 };
 typedef struct mca_btl_uct_conn_completion_t mca_btl_uct_conn_completion_t;
 
-static void mca_btl_uct_endpoint_flush_complete (uct_completion_t *self, ucs_status_t status)
+#if UCT_API >= ((1L<<UCT_MAJOR_BIT)|(10L << UCT_MINOR_BIT))
+static void mca_btl_uct_endpoint_flush_complete(uct_completion_t *self)
 {
     mca_btl_uct_conn_completion_t *completion = (mca_btl_uct_conn_completion_t *) self;
     BTL_VERBOSE(("connection flush complete"));
     completion->complete = true;
 }
+#else
+static void mca_btl_uct_endpoint_flush_complete(uct_completion_t *self, ucs_status_t status)
+{
+    mca_btl_uct_conn_completion_t *completion = (mca_btl_uct_conn_completion_t *) self;
+    (void) status;
+    BTL_VERBOSE(("connection flush complete"));
+    completion->complete = true;
+}
+#endif
 
-static int mca_btl_uct_endpoint_send_conn_req (mca_btl_uct_module_t *uct_btl, mca_btl_base_endpoint_t *endpoint,
-                                               mca_btl_uct_device_context_t *conn_tl_context,
-                                               mca_btl_uct_conn_req_t *request, size_t request_length)
+static int mca_btl_uct_endpoint_send_conn_req(mca_btl_uct_module_t *uct_btl,
+                                              mca_btl_base_endpoint_t *endpoint,
+                                              mca_btl_uct_device_context_t *conn_tl_context,
+                                              mca_btl_uct_conn_req_t *request,
+                                              size_t request_length)
 {
-    mca_btl_uct_connection_ep_t *conn_ep = endpoint->conn_ep;
-    mca_btl_uct_conn_completion_t completion = {.super = {.count = 1, .func = mca_btl_uct_endpoint_flush_complete},
-                                                .complete = false};
+    mca_btl_uct_conn_completion_t completion
+        = {.super = {.count = 1, .func = mca_btl_uct_endpoint_flush_complete}, .complete = false};
     ucs_status_t ucs_status;
 
-    BTL_VERBOSE(("sending connection request to peer. context id: %d, type: %d, length: %" PRIsize_t,
-                 request->context_id, request->type, request_length));
-
-    OBJ_RETAIN(endpoint->conn_ep);
+    BTL_VERBOSE(
+        ("sending connection request to peer. context id: %d, type: %d, length: %" PRIsize_t,
+         request->context_id, request->type, request_length));
 
     /* need to drop the lock to avoid hold-and-wait */
-    opal_mutex_unlock (&endpoint->ep_lock);
+    opal_mutex_unlock(&endpoint->ep_lock);
 
     do {
         MCA_BTL_UCT_CONTEXT_SERIALIZE(conn_tl_context, {
-                ucs_status = uct_ep_am_short (conn_ep->uct_ep, MCA_BTL_UCT_CONNECT_RDMA, request->type, request,
-                                              request_length);
-            });
+            ucs_status = uct_ep_am_short(endpoint->conn_ep->uct_ep, MCA_BTL_UCT_CONNECT_RDMA,
+                                         request->type, request, request_length);
+        });
         if (OPAL_LIKELY(UCS_OK == ucs_status)) {
             break;
         }
@@ -206,88 +234,122 @@ static int mca_btl_uct_endpoint_send_conn_req (mca_btl_uct_module_t *uct_btl, mc
         }
 
         /* some TLs (UD for example) need to be progressed to get resources */
-        mca_btl_uct_context_progress (conn_tl_context);
+        mca_btl_uct_context_progress(conn_tl_context);
     } while (1);
 
     /* for now we just wait for the connection request to complete before continuing */
-    ucs_status = uct_ep_flush (conn_ep->uct_ep, 0, &completion.super);
+    ucs_status = uct_ep_flush(endpoint->conn_ep->uct_ep, 0, &completion.super);
     if (UCS_OK != ucs_status && UCS_INPROGRESS != ucs_status) {
         /* NTH: I don't know if this path is needed. For some networks we must use a completion. */
         do {
-            ucs_status = uct_ep_flush (conn_ep->uct_ep, 0, NULL);
-            mca_btl_uct_context_progress (conn_tl_context);
+            ucs_status = uct_ep_flush(endpoint->conn_ep->uct_ep, 0, NULL);
+            mca_btl_uct_context_progress(conn_tl_context);
         } while (UCS_INPROGRESS == ucs_status);
     } else {
         do {
-            mca_btl_uct_context_progress (conn_tl_context);
+            mca_btl_uct_context_progress(conn_tl_context);
         } while (!completion.complete);
     }
 
-    opal_mutex_lock (&endpoint->ep_lock);
-
-    OBJ_RELEASE(endpoint->conn_ep);
+    opal_mutex_lock(&endpoint->ep_lock);
 
     return OPAL_SUCCESS;
 }
 
-static int mca_btl_uct_endpoint_connect_endpoint (mca_btl_uct_module_t *uct_btl, mca_btl_base_endpoint_t *endpoint,
-                                                  mca_btl_uct_tl_t *tl, mca_btl_uct_device_context_t *tl_context,
-                                                  mca_btl_uct_tl_endpoint_t *tl_endpoint, uint8_t *tl_data,
-                                                  uint8_t *conn_tl_data, void *ep_addr)
+static int mca_btl_uct_endpoint_send_connection_data(
+    mca_btl_uct_module_t *uct_btl, mca_btl_base_endpoint_t *endpoint, mca_btl_uct_tl_t *tl,
+    mca_btl_uct_device_context_t *tl_context, mca_btl_uct_tl_endpoint_t *tl_endpoint,
+    uint8_t *conn_tl_data, int request_type)
 {
-    size_t request_length = sizeof (mca_btl_uct_conn_req_t) + MCA_BTL_UCT_TL_ATTR(tl, tl_context->context_id).ep_addr_len;
-    mca_btl_uct_connection_ep_t *conn_ep = endpoint->conn_ep;
     mca_btl_uct_tl_t *conn_tl = uct_btl->conn_tl;
     mca_btl_uct_device_context_t *conn_tl_context = conn_tl->uct_dev_contexts[0];
-    mca_btl_uct_conn_req_t *request = alloca (request_length);
     uct_device_addr_t *device_addr = NULL;
     uct_iface_addr_t *iface_addr;
     ucs_status_t ucs_status;
-    int rc;
 
-    assert (NULL != conn_tl);
+    assert(NULL != conn_tl);
 
     BTL_VERBOSE(("connecting endpoint to remote endpoint"));
 
-    if (NULL == conn_ep) {
+    if (NULL == endpoint->conn_ep) {
         BTL_VERBOSE(("creating a temporary endpoint for handling connections to %p",
-                     opal_process_name_print (endpoint->ep_proc->proc_name)));
+                     opal_process_name_print(endpoint->ep_proc->proc_name)));
 
         iface_addr = (uct_iface_addr_t *) conn_tl_data;
-        device_addr = (uct_device_addr_t *) ((uintptr_t) conn_tl_data + MCA_BTL_UCT_TL_ATTR(conn_tl, 0).iface_addr_len);
+        device_addr = (uct_device_addr_t *) ((uintptr_t) conn_tl_data
+                                             + MCA_BTL_UCT_TL_ATTR(conn_tl, 0).iface_addr_len);
 
-        endpoint->conn_ep = conn_ep = OBJ_NEW(mca_btl_uct_connection_ep_t);
-        if (OPAL_UNLIKELY(NULL == conn_ep)) {
+        endpoint->conn_ep = OBJ_NEW(mca_btl_uct_connection_ep_t);
+        if (OPAL_UNLIKELY(NULL == endpoint->conn_ep)) {
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
 
         /* create a temporary endpoint for setting up the rdma endpoint */
         MCA_BTL_UCT_CONTEXT_SERIALIZE(conn_tl_context, {
-                ucs_status = mca_btl_uct_ep_create_connected_compat (conn_tl_context->uct_iface, device_addr, iface_addr,
-                                                                     &conn_ep->uct_ep);
-            });
+            ucs_status = mca_btl_uct_ep_create_connected_compat(conn_tl_context->uct_iface,
+                                                                device_addr, iface_addr,
+                                                                &endpoint->conn_ep->uct_ep);
+        });
         if (UCS_OK != ucs_status) {
-            BTL_VERBOSE(("could not create an endpoint for forming connection to remote peer. code = %d",
-                         ucs_status));
+            BTL_VERBOSE(
+                ("could not create an endpoint for forming connection to remote peer. code = %d",
+                 ucs_status));
             return OPAL_ERROR;
         }
     } else {
-        OBJ_RETAIN(conn_ep);
+        OBJ_RETAIN(endpoint->conn_ep);
     }
 
+    size_t request_length = sizeof(mca_btl_uct_conn_req_t)
+                            + MCA_BTL_UCT_TL_ATTR(tl, tl_context->context_id).ep_addr_len;
+    mca_btl_uct_conn_req_t *request = alloca(request_length);
+
     /* fill in common request parameters */
     request->proc_name = OPAL_PROC_MY_NAME;
     request->context_id = tl_context->context_id;
     request->tl_index = tl->tl_index;
-    request->type = !!(ep_addr);
+    request->type = request_type;
+
+    /* fill in connection request */
+    ucs_status = uct_ep_get_address(tl_endpoint->uct_ep, (uct_ep_addr_t *) request->ep_addr);
+    if (UCS_OK != ucs_status) {
+        /* this is a fatal a fatal error */
+        OBJ_RELEASE(endpoint->conn_ep);
+        uct_ep_destroy(tl_endpoint->uct_ep);
+        tl_endpoint->uct_ep = NULL;
+        return OPAL_ERROR;
+    }
+
+    /* let the remote side know that the connection has been established and
+     * wait for the message to be sent */
+    int rc = mca_btl_uct_endpoint_send_conn_req(uct_btl, endpoint, conn_tl_context, request,
+                                                request_length);
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
+        OBJ_RELEASE(endpoint->conn_ep);
+        uct_ep_destroy(tl_endpoint->uct_ep);
+        tl_endpoint->uct_ep = NULL;
+        return OPAL_ERROR;
+    }
+
+    tl_endpoint->last_connection_req = opal_timer_base_get_usec();
+
+    return OPAL_SUCCESS;
+}
+
+static int mca_btl_uct_endpoint_connect_endpoint(
+    mca_btl_uct_module_t *uct_btl, mca_btl_base_endpoint_t *endpoint, mca_btl_uct_tl_t *tl,
+    mca_btl_uct_device_context_t *tl_context, mca_btl_uct_tl_endpoint_t *tl_endpoint,
+    uint8_t *tl_data, uint8_t *conn_tl_data, void *ep_addr)
+{
+    ucs_status_t ucs_status;
 
     if (NULL == tl_endpoint->uct_ep) {
         BTL_VERBOSE(("allocating endpoint for peer %s and sending connection data",
-                     opal_process_name_print (endpoint->ep_proc->proc_name)));
+                     opal_process_name_print(endpoint->ep_proc->proc_name)));
 
         MCA_BTL_UCT_CONTEXT_SERIALIZE(tl_context, {
-                ucs_status = mca_btl_uct_ep_create_compat (tl_context->uct_iface, &tl_endpoint->uct_ep);
-            });
+            ucs_status = mca_btl_uct_ep_create_compat(tl_context->uct_iface, &tl_endpoint->uct_ep);
+        });
         if (UCS_OK != ucs_status) {
             OBJ_RELEASE(endpoint->conn_ep);
             return OPAL_ERROR;
@@ -295,137 +357,125 @@ static int mca_btl_uct_endpoint_connect_endpoint (mca_btl_uct_module_t *uct_btl,
     }
 
     if (ep_addr) {
-        BTL_VERBOSE(("using remote endpoint address to connect endpoint for tl %s, index %d. ep_addr = %p",
-                     tl->uct_tl_name, tl_context->context_id, ep_addr));
+        BTL_VERBOSE(
+            ("using remote endpoint address to connect endpoint for tl %s, index %d. ep_addr = %p",
+             tl->uct_tl_name, tl_context->context_id, ep_addr));
 
         /* NTH: there is no need to lock the device context in this case */
-        ucs_status = uct_ep_connect_to_ep (tl_endpoint->uct_ep, (uct_device_addr_t *) tl_data, ep_addr);
+        ucs_status = uct_ep_connect_to_ep(tl_endpoint->uct_ep, (uct_device_addr_t *) tl_data,
+                                          ep_addr);
         if (UCS_OK != ucs_status) {
             return OPAL_ERROR;
         }
-    }
 
-    /* fill in connection request */
-    ucs_status = uct_ep_get_address (tl_endpoint->uct_ep, (uct_ep_addr_t *) request->ep_addr);
-    if (UCS_OK != ucs_status) {
-        /* this is a fatal a fatal error */
-        OBJ_RELEASE(endpoint->conn_ep);
-        uct_ep_destroy (tl_endpoint->uct_ep);
-        tl_endpoint->uct_ep = NULL;
-        return OPAL_ERROR;
+        mca_btl_uct_endpoint_set_flag(uct_btl, endpoint, tl_context->context_id, tl_endpoint,
+                                      MCA_BTL_UCT_ENDPOINT_FLAG_EP_CONNECTED);
     }
 
-    /* let the remote side know that the connection has been established and
-     * wait for the message to be sent */
-    rc = mca_btl_uct_endpoint_send_conn_req (uct_btl, endpoint, conn_tl_context, request, request_length);
-    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-        OBJ_RELEASE(endpoint->conn_ep);
-        uct_ep_destroy (tl_endpoint->uct_ep);
-        tl_endpoint->uct_ep = NULL;
-        return OPAL_ERROR;
+    opal_timer_t now = opal_timer_base_get_usec();
+    if ((now - tl_endpoint->last_connection_req) < mca_btl_uct_component.connection_retry_timeout && !ep_addr) {
+        return (tl_endpoint->flags & MCA_BTL_UCT_ENDPOINT_FLAG_CONN_READY) ? OPAL_SUCCESS
+            : OPAL_ERR_OUT_OF_RESOURCE;
     }
 
-    return (tl_endpoint->flags & MCA_BTL_UCT_ENDPOINT_FLAG_CONN_READY) ? OPAL_SUCCESS : OPAL_ERR_OUT_OF_RESOURCE;
+    int rc = mca_btl_uct_endpoint_send_connection_data(uct_btl, endpoint, tl, tl_context, tl_endpoint,
+                                                       conn_tl_data, /*request_type=*/!!ep_addr);
+    return (OPAL_SUCCESS == rc) ? OPAL_ERR_OUT_OF_RESOURCE : rc;
 }
 
-int mca_btl_uct_endpoint_connect (mca_btl_uct_module_t *uct_btl, mca_btl_uct_endpoint_t *endpoint, int context_id,
-                                  void *ep_addr, int tl_index)
+int mca_btl_uct_endpoint_connect(mca_btl_uct_module_t *uct_btl, mca_btl_uct_endpoint_t *endpoint,
+                                 int context_id, void *ep_addr, int tl_index)
 {
     mca_btl_uct_tl_endpoint_t *tl_endpoint = endpoint->uct_eps[context_id] + tl_index;
-    mca_btl_uct_tl_t *tl = (uct_btl->rdma_tl && tl_index == uct_btl->rdma_tl->tl_index) ?
-        uct_btl->rdma_tl : uct_btl->am_tl;
-    mca_btl_uct_device_context_t *tl_context = mca_btl_uct_module_get_tl_context_specific (uct_btl, tl, context_id);
+    mca_btl_uct_tl_t *tl = (uct_btl->rdma_tl && tl_index == uct_btl->rdma_tl->tl_index)
+                               ? uct_btl->rdma_tl
+                               : uct_btl->am_tl;
+    mca_btl_uct_device_context_t *tl_context
+        = mca_btl_uct_module_get_tl_context_specific(uct_btl, tl, context_id);
     uint8_t *rdma_tl_data = NULL, *conn_tl_data = NULL, *am_tl_data = NULL, *tl_data;
-    mca_btl_uct_connection_ep_t *conn_ep = NULL;
     mca_btl_uct_modex_t *modex;
     uint8_t *modex_data;
     size_t msg_size;
     int rc;
 
     /* only two types of endpoints at this time */
-    assert (tl_index < 2);
+    assert(tl_index < 2);
 
     if (OPAL_UNLIKELY(NULL == tl)) {
         return OPAL_ERR_UNREACH;
     }
 
-    BTL_VERBOSE(("checking endpoint %p with context id %d. cached uct ep: %p, ready: %d", (void *) endpoint, context_id,
-                 (void *) tl_endpoint->uct_ep, !!(MCA_BTL_UCT_ENDPOINT_FLAG_CONN_READY & tl_endpoint->flags)));
+    BTL_VERBOSE(("checking endpoint %p with context id %d. cached uct ep: %p, ready: %d",
+                 (void *) endpoint, context_id, (void *) tl_endpoint->uct_ep,
+                 !!(MCA_BTL_UCT_ENDPOINT_FLAG_CONN_READY & tl_endpoint->flags)));
 
-    opal_mutex_lock (&endpoint->ep_lock);
+    opal_mutex_lock(&endpoint->ep_lock);
     if (MCA_BTL_UCT_ENDPOINT_FLAG_CONN_READY & tl_endpoint->flags) {
-        opal_mutex_unlock (&endpoint->ep_lock);
+        opal_mutex_unlock(&endpoint->ep_lock);
         /* nothing more to do. someone else completed the connection */
         return OPAL_SUCCESS;
     }
 
     /* dumpicate connection request. nothing to do until the endpoint data is received */
     if (NULL != tl_endpoint->uct_ep && NULL == ep_addr) {
-        opal_mutex_unlock (&endpoint->ep_lock);
+        opal_mutex_unlock(&endpoint->ep_lock);
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
     do {
         /* read the modex. this is done both to start the connection and to process endpoint data */
-        OPAL_MODEX_RECV(rc, &mca_btl_uct_component.super.btl_version,
-                        &endpoint->ep_proc->proc_name, (void **)&modex, &msg_size);
+        OPAL_MODEX_RECV(rc, &mca_btl_uct_component.super.btl_version, &endpoint->ep_proc->proc_name,
+                        (void **) &modex, &msg_size);
         if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
             BTL_ERROR(("error receiving modex"));
             break;
         }
 
-        BTL_VERBOSE(("received modex of size %lu for proc %s. module count %d", (unsigned long) msg_size,
-                     OPAL_NAME_PRINT(endpoint->ep_proc->proc_name), modex->module_count));
+        BTL_VERBOSE(("received modex of size %lu for proc %s. module count %d",
+                     (unsigned long) msg_size, OPAL_NAME_PRINT(endpoint->ep_proc->proc_name),
+                     modex->module_count));
         modex_data = modex->data;
 
         /* look for matching transport in the modex */
-        for (int i = 0 ; i < modex->module_count ; ++i) {
+        for (int i = 0; i < modex->module_count; ++i) {
             uint32_t modex_size = *((uint32_t *) modex_data);
 
-            BTL_VERBOSE(("found modex for md %s, searching for %s", modex_data + 4, uct_btl->md_name));
+            BTL_VERBOSE(
+                ("found modex for md %s, searching for %s", modex_data + 4, uct_btl->md_name));
 
             modex_data += 4;
 
-            if (0 != strcmp ((char *) modex_data, uct_btl->md_name)) {
+            if (0 != strcmp((char *) modex_data, uct_btl->md_name)) {
                 /* modex belongs to a different module, skip it and continue */
                 modex_data += modex_size - 4;
                 continue;
             }
 
-            modex_data += strlen ((char *) modex_data) + 1;
+            modex_data += strlen((char *) modex_data) + 1;
 
-            mca_btl_uct_process_modex (uct_btl, modex_data, &rdma_tl_data, &am_tl_data, &conn_tl_data);
+            mca_btl_uct_process_modex(uct_btl, modex_data, &rdma_tl_data, &am_tl_data,
+                                      &conn_tl_data);
             break;
         }
 
         tl_data = (tl == uct_btl->rdma_tl) ? rdma_tl_data : am_tl_data;
 
         if (NULL == tl_data) {
-            opal_mutex_unlock (&endpoint->ep_lock);
+            opal_mutex_unlock(&endpoint->ep_lock);
             return OPAL_ERR_UNREACH;
         }
 
         /* connect the endpoint */
-        if (!mca_btl_uct_tl_requires_connection_tl (tl)) {
-            rc = mca_btl_uct_endpoint_connect_iface (uct_btl, tl, tl_context, tl_endpoint, tl_data);
+        if (!mca_btl_uct_tl_requires_connection_tl(tl)) {
+            rc = mca_btl_uct_endpoint_connect_iface(uct_btl, tl, tl_context, tl_endpoint, tl_data);
         } else {
-            rc = mca_btl_uct_endpoint_connect_endpoint (uct_btl, endpoint, tl, tl_context, tl_endpoint,
-                                                        tl_data, conn_tl_data, ep_addr);
+            rc = mca_btl_uct_endpoint_connect_endpoint(uct_btl, endpoint, tl, tl_context,
+                                                       tl_endpoint, tl_data, conn_tl_data, ep_addr);
         }
 
     } while (0);
 
-    /* to avoid a possible hold-and wait deadlock. destroy the endpoint after dropping the endpoint lock. */
-    if (endpoint->conn_ep && 1 == endpoint->conn_ep->super.obj_reference_count) {
-        conn_ep = endpoint->conn_ep;
-        endpoint->conn_ep = NULL;
-    }
-
-    opal_mutex_unlock (&endpoint->ep_lock);
-
-    if (conn_ep) {
-        OBJ_RELEASE(conn_ep);
-    }
+    opal_mutex_unlock(&endpoint->ep_lock);
 
     BTL_VERBOSE(("endpoint%s ready for use", (OPAL_ERR_OUT_OF_RESOURCE != rc) ? "" : " not yet"));
 
diff --git a/opal/mca/btl/uct/btl_uct_endpoint.h b/opal/mca/btl/uct/btl_uct_endpoint.h
index 12d93b4192d..0c4bae20050 100644
--- a/opal/mca/btl/uct/btl_uct_endpoint.h
+++ b/opal/mca/btl/uct/btl_uct_endpoint.h
@@ -14,6 +14,7 @@
  *                         reserved.
  * Copyright (c) 2020      Amazon.com, Inc. or its affiliates.
  *                         All Rights reserved.
+ * Copyright (c) 2025      Google, LLC. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -24,22 +25,26 @@
 #ifndef MCA_BTL_UCT_ENDPOINT_H
 #define MCA_BTL_UCT_ENDPOINT_H
 
+#include "btl_uct.h"
 #include "opal/class/opal_list.h"
 #include "opal/util/event.h"
-#include "btl_uct.h"
 
 BEGIN_C_DECLS
 
-mca_btl_base_endpoint_t *mca_btl_uct_endpoint_create (opal_proc_t *proc);
-int mca_btl_uct_endpoint_connect (mca_btl_uct_module_t *module, mca_btl_uct_endpoint_t *endpoint, int ep_index, void *ep_addr, int tl_index);
+mca_btl_base_endpoint_t *mca_btl_uct_endpoint_create(opal_proc_t *proc);
+int mca_btl_uct_endpoint_connect(mca_btl_uct_module_t *module, mca_btl_uct_endpoint_t *endpoint,
+                                 int ep_index, void *ep_addr, int tl_index);
 
-static inline int mca_btl_uct_endpoint_test_am (mca_btl_uct_module_t *module, mca_btl_uct_endpoint_t *endpoint,
-                                                mca_btl_uct_device_context_t *context, uct_ep_h *ep_handle)
+static inline int mca_btl_uct_endpoint_test_am(mca_btl_uct_module_t *module,
+                                               mca_btl_uct_endpoint_t *endpoint,
+                                               mca_btl_uct_device_context_t *context,
+                                               uct_ep_h *ep_handle)
 {
     int tl_index = module->am_tl->tl_index;
     int ep_index = context->context_id;
 
-    if (OPAL_LIKELY(MCA_BTL_UCT_ENDPOINT_FLAG_CONN_READY & endpoint->uct_eps[ep_index][tl_index].flags)) {
+    if (OPAL_LIKELY(MCA_BTL_UCT_ENDPOINT_FLAG_CONN_READY
+                    & endpoint->uct_eps[ep_index][tl_index].flags)) {
         *ep_handle = endpoint->uct_eps[ep_index][tl_index].uct_ep;
         return OPAL_SUCCESS;
     }
@@ -60,37 +65,70 @@ static inline int mca_btl_uct_endpoint_test_am (mca_btl_uct_module_t *module, mc
  * @returns OPAL_ERR_RESOURCE_BUSY if the connection is underway
  * @returns OPAL_ERROR otherwise
  */
-static inline int mca_btl_uct_endpoint_check (mca_btl_uct_module_t *module, mca_btl_uct_endpoint_t *endpoint,
-                                              mca_btl_uct_device_context_t *context, uct_ep_h *ep_handle,
-                                              const int tl_index)
+static inline int mca_btl_uct_endpoint_check(mca_btl_uct_module_t *module,
+                                             mca_btl_uct_endpoint_t *endpoint,
+                                             mca_btl_uct_device_context_t *context,
+                                             uct_ep_h *ep_handle, const int tl_index)
 {
     int ep_index = context->context_id;
     int rc;
 
-    if (OPAL_LIKELY(MCA_BTL_UCT_ENDPOINT_FLAG_CONN_READY & endpoint->uct_eps[ep_index][tl_index].flags)) {
+    if (OPAL_LIKELY(MCA_BTL_UCT_ENDPOINT_FLAG_CONN_READY
+                    & endpoint->uct_eps[ep_index][tl_index].flags)) {
         *ep_handle = endpoint->uct_eps[ep_index][tl_index].uct_ep;
         return OPAL_SUCCESS;
     }
 
-    rc = mca_btl_uct_endpoint_connect (module, endpoint, ep_index, NULL, tl_index);
+    rc = mca_btl_uct_endpoint_connect(module, endpoint, ep_index, NULL, tl_index);
     *ep_handle = endpoint->uct_eps[ep_index][tl_index].uct_ep;
-    BTL_VERBOSE(("mca_btl_uct_endpoint_connect returned %d. context id = %d, flags = 0x%x", rc, ep_index,
-                 MCA_BTL_UCT_ENDPOINT_FLAG_CONN_READY & endpoint->uct_eps[ep_index][tl_index].flags));
+    BTL_VERBOSE(
+        ("mca_btl_uct_endpoint_connect returned %d. context id = %d, flags = 0x%x", rc, ep_index,
+         MCA_BTL_UCT_ENDPOINT_FLAG_CONN_READY & endpoint->uct_eps[ep_index][tl_index].flags));
     return rc;
 }
 
-static inline int mca_btl_uct_endpoint_check_rdma (mca_btl_uct_module_t *module, mca_btl_uct_endpoint_t *endpoint,
-                                                   mca_btl_uct_device_context_t *context, uct_ep_h *ep_handle)
+static inline int mca_btl_uct_endpoint_check_rdma(mca_btl_uct_module_t *module,
+                                                  mca_btl_uct_endpoint_t *endpoint,
+                                                  mca_btl_uct_device_context_t *context,
+                                                  uct_ep_h *ep_handle)
 {
-    assert (NULL != module->rdma_tl);
-    return mca_btl_uct_endpoint_check (module, endpoint, context, ep_handle, module->rdma_tl->tl_index);
+    assert(NULL != module->rdma_tl);
+    return mca_btl_uct_endpoint_check(module, endpoint, context, ep_handle,
+                                      module->rdma_tl->tl_index);
 }
 
-static inline int mca_btl_uct_endpoint_check_am (mca_btl_uct_module_t *module, mca_btl_uct_endpoint_t *endpoint,
-                                                 mca_btl_uct_device_context_t *context, uct_ep_h *ep_handle)
+static inline int mca_btl_uct_endpoint_check_am(mca_btl_uct_module_t *module,
+                                                mca_btl_uct_endpoint_t *endpoint,
+                                                mca_btl_uct_device_context_t *context,
+                                                uct_ep_h *ep_handle)
 {
-    assert (NULL != module->am_tl);
-    return mca_btl_uct_endpoint_check (module, endpoint, context, ep_handle, module->am_tl->tl_index);
+    assert(NULL != module->am_tl);
+    return mca_btl_uct_endpoint_check(module, endpoint, context, ep_handle,
+                                      module->am_tl->tl_index);
+}
+
+// Requires that the endpoint lock is held.
+static inline void mca_btl_uct_endpoint_set_flag(mca_btl_uct_module_t *module, mca_btl_uct_endpoint_t *endpoint,
+                                                 int context_id, mca_btl_uct_tl_endpoint_t *tl_endpoint, int32_t flag) {
+    opal_atomic_wmb();
+    int32_t flag_value = opal_atomic_or_fetch_32(&tl_endpoint->flags, flag);
+    if ((flag_value & (MCA_BTL_UCT_ENDPOINT_FLAG_EP_CONNECTED | MCA_BTL_UCT_ENDPOINT_FLAG_CONN_REM_READY)) ==
+        (MCA_BTL_UCT_ENDPOINT_FLAG_EP_CONNECTED | MCA_BTL_UCT_ENDPOINT_FLAG_CONN_REM_READY)) {
+	opal_atomic_fetch_or_32(&tl_endpoint->flags, MCA_BTL_UCT_ENDPOINT_FLAG_CONN_READY);
+
+        opal_atomic_wmb();
+
+        mca_btl_uct_base_frag_t *frag;
+        OPAL_LIST_FOREACH (frag, &module->pending_frags, mca_btl_uct_base_frag_t) {
+            if (frag->context->context_id == context_id && endpoint == frag->endpoint) {
+                frag->ready = true;
+            }
+        }
+
+        if (endpoint->conn_ep) {
+            OBJ_RELEASE(endpoint->conn_ep);
+        }
+    }
 }
 
 END_C_DECLS
diff --git a/opal/mca/btl/uct/btl_uct_frag.c b/opal/mca/btl/uct/btl_uct_frag.c
index 3e5622cac45..37eee126654 100644
--- a/opal/mca/btl/uct/btl_uct_frag.c
+++ b/opal/mca/btl/uct/btl_uct_frag.c
@@ -2,6 +2,7 @@
 /*
  * Copyright (c) 2018      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2025      Google, LLC. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -11,32 +12,19 @@
 
 #include "btl_uct_frag.h"
 
-static void mca_btl_uct_frag_completion (uct_completion_t *uct_comp, ucs_status_t status)
-{
-    mca_btl_uct_uct_completion_t *comp = (mca_btl_uct_uct_completion_t *) ((uintptr_t) uct_comp - offsetof (mca_btl_uct_uct_completion_t, uct_comp));
-
-    BTL_VERBOSE(("frag operation complete. frag = %p. status = %d", (void *) comp->frag, status));
-
-    comp->status = status;
-    opal_fifo_push (&comp->dev_context->completion_fifo, &comp->super.super);
-}
-
-static void mca_btl_uct_base_frag_constructor (mca_btl_uct_base_frag_t *frag)
+static void mca_btl_uct_base_frag_constructor(mca_btl_uct_base_frag_t *frag)
 {
     mca_btl_uct_reg_t *reg = (mca_btl_uct_reg_t *) frag->base.super.registration;
 
     /* zero everything out */
-    memset ((char *) frag + sizeof (frag->base), 0, sizeof (*frag) - sizeof (frag->base));
+    memset((char *) frag + sizeof(frag->base), 0, sizeof(*frag) - sizeof(frag->base));
 
     OBJ_CONSTRUCT(&frag->comp, mca_btl_uct_uct_completion_t);
+    frag->comp.frag = frag;
 
     frag->base.des_segments = frag->segments;
     frag->base.des_segment_count = 1;
 
-    frag->comp.uct_comp.func = mca_btl_uct_frag_completion;
-    frag->comp.uct_comp.count = 1;
-    frag->comp.frag = frag;
-
     frag->segments[0].seg_addr.pval = frag->base.super.ptr;
     frag->uct_iov.buffer = frag->base.super.ptr;
     frag->uct_iov.stride = 0;
@@ -46,7 +34,7 @@ static void mca_btl_uct_base_frag_constructor (mca_btl_uct_base_frag_t *frag)
     }
 }
 
-static void mca_btl_uct_base_frag_destructor (mca_btl_uct_base_frag_t *frag)
+static void mca_btl_uct_base_frag_destructor(mca_btl_uct_base_frag_t *frag)
 {
     OBJ_DESTRUCT(&frag->comp);
 }
diff --git a/opal/mca/btl/uct/btl_uct_frag.h b/opal/mca/btl/uct/btl_uct_frag.h
index 8aa8789d0e3..7907774cf42 100644
--- a/opal/mca/btl/uct/btl_uct_frag.h
+++ b/opal/mca/btl/uct/btl_uct_frag.h
@@ -10,14 +10,15 @@
  */
 
 #if !defined(MCA_BTL_UCT_FRAG_H)
-#define MCA_BTL_UCT_FRAG_H
+#    define MCA_BTL_UCT_FRAG_H
 
-#include "btl_uct.h"
+#    include "btl_uct.h"
 
-static inline mca_btl_uct_base_frag_t *mca_btl_uct_frag_alloc (mca_btl_uct_module_t *uct_btl, opal_free_list_t *fl,
-                                                               mca_btl_base_endpoint_t *endpoint)
+static inline mca_btl_uct_base_frag_t *mca_btl_uct_frag_alloc(mca_btl_uct_module_t *uct_btl,
+                                                              opal_free_list_t *fl,
+                                                              mca_btl_base_endpoint_t *endpoint)
 {
-    mca_btl_uct_base_frag_t *frag = (mca_btl_uct_base_frag_t *) opal_free_list_get (fl);
+    mca_btl_uct_base_frag_t *frag = (mca_btl_uct_base_frag_t *) opal_free_list_get(fl);
     if (OPAL_LIKELY(NULL != frag)) {
         frag->free_list = fl;
         frag->endpoint = endpoint;
@@ -27,12 +28,13 @@ static inline mca_btl_uct_base_frag_t *mca_btl_uct_frag_alloc (mca_btl_uct_modul
     return frag;
 }
 
-static inline void mca_btl_uct_frag_return (mca_btl_uct_base_frag_t *frag)
+static inline void mca_btl_uct_frag_return(mca_btl_uct_base_frag_t *frag)
 {
-    opal_free_list_return (frag->free_list, &frag->base.super);
+    opal_free_list_return(frag->free_list, &frag->base.super);
 }
 
-static inline void mca_btl_uct_frag_complete (mca_btl_uct_base_frag_t *frag, int rc) {
+static inline void mca_btl_uct_frag_complete(mca_btl_uct_base_frag_t *frag, int rc)
+{
     mca_btl_uct_module_t *uct_btl = frag->btl;
 
     /* call callback if specified */
@@ -41,23 +43,26 @@ static inline void mca_btl_uct_frag_complete (mca_btl_uct_base_frag_t *frag, int
     }
 
     if (OPAL_LIKELY(frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP)) {
-        mca_btl_uct_frag_return (frag);
+        mca_btl_uct_frag_return(frag);
     }
 }
 
-static inline mca_btl_uct_base_frag_t *mca_btl_uct_frag_alloc_short (mca_btl_uct_module_t *uct_btl, mca_btl_base_endpoint_t *endpoint)
+static inline mca_btl_uct_base_frag_t *
+mca_btl_uct_frag_alloc_short(mca_btl_uct_module_t *uct_btl, mca_btl_base_endpoint_t *endpoint)
 {
-    return mca_btl_uct_frag_alloc (uct_btl, &uct_btl->short_frags, endpoint);
+    return mca_btl_uct_frag_alloc(uct_btl, &uct_btl->short_frags, endpoint);
 }
 
-static inline mca_btl_uct_base_frag_t *mca_btl_uct_frag_alloc_eager (mca_btl_uct_module_t *uct_btl, mca_btl_base_endpoint_t *endpoint)
+static inline mca_btl_uct_base_frag_t *
+mca_btl_uct_frag_alloc_eager(mca_btl_uct_module_t *uct_btl, mca_btl_base_endpoint_t *endpoint)
 {
-    return mca_btl_uct_frag_alloc (uct_btl, &uct_btl->eager_frags, endpoint);
+    return mca_btl_uct_frag_alloc(uct_btl, &uct_btl->eager_frags, endpoint);
 }
 
-static inline mca_btl_uct_base_frag_t *mca_btl_uct_frag_alloc_max (mca_btl_uct_module_t *uct_btl, mca_btl_base_endpoint_t *endpoint)
+static inline mca_btl_uct_base_frag_t *mca_btl_uct_frag_alloc_max(mca_btl_uct_module_t *uct_btl,
+                                                                  mca_btl_base_endpoint_t *endpoint)
 {
-    return mca_btl_uct_frag_alloc (uct_btl, &uct_btl->max_frags, endpoint);
+    return mca_btl_uct_frag_alloc(uct_btl, &uct_btl->max_frags, endpoint);
 }
 
 #endif /* !defined(MCA_BTL_UCT_FRAG_H) */
diff --git a/opal/mca/btl/uct/btl_uct_module.c b/opal/mca/btl/uct/btl_uct_module.c
index 53d1df3302f..9577d615b92 100644
--- a/opal/mca/btl/uct/btl_uct_module.c
+++ b/opal/mca/btl/uct/btl_uct_module.c
@@ -21,34 +21,37 @@
  */
 
 #include "opal_config.h"
-#include <string.h>
 #include "opal/class/opal_bitmap.h"
-#include "opal/mca/btl/btl.h"
 #include "opal/datatype/opal_convertor.h"
+#include "opal/mca/btl/btl.h"
 #include "opal/mca/mpool/base/base.h"
 #include "opal/mca/mpool/mpool.h"
+#include <string.h>
 
 #include "btl_uct.h"
-#include "btl_uct_endpoint.h"
 #include "btl_uct_am.h"
+#include "btl_uct_endpoint.h"
 
-struct mca_btl_base_endpoint_t *mca_btl_uct_get_ep (struct mca_btl_base_module_t *module, opal_proc_t *proc)
+struct mca_btl_base_endpoint_t *mca_btl_uct_get_ep(struct mca_btl_base_module_t *module,
+                                                   opal_proc_t *proc)
 {
     mca_btl_uct_module_t *uct_module = (mca_btl_uct_module_t *) module;
     mca_btl_base_endpoint_t *ep;
     int rc;
 
-    opal_mutex_lock (&uct_module->endpoint_lock);
+    opal_mutex_lock(&uct_module->endpoint_lock);
 
     do {
-        rc = opal_hash_table_get_value_uint64 (&uct_module->id_to_endpoint, (intptr_t) proc, (void **) &ep);
+        rc = opal_hash_table_get_value_uint64(&uct_module->id_to_endpoint, (intptr_t) proc,
+                                              (void **) &ep);
         if (OPAL_SUCCESS == rc) {
-            BTL_VERBOSE(("returning existing endpoint for proc %s", OPAL_NAME_PRINT(proc->proc_name)));
+            BTL_VERBOSE(
+                ("returning existing endpoint for proc %s", OPAL_NAME_PRINT(proc->proc_name)));
             break;
         }
 
         /*  Create and Init endpoints */
-        ep = mca_btl_uct_endpoint_create (proc);
+        ep = mca_btl_uct_endpoint_create(proc);
         if (OPAL_UNLIKELY(NULL == ep)) {
             BTL_ERROR(("btl/uct error initializing endpoint"));
             break;
@@ -57,18 +60,17 @@ struct mca_btl_base_endpoint_t *mca_btl_uct_get_ep (struct mca_btl_base_module_t
         BTL_VERBOSE(("endpoint initialized. new endpoint: %p", (void *) ep));
 
         /* add this endpoint to the connection lookup table */
-        (void) opal_hash_table_set_value_uint64 (&uct_module->id_to_endpoint, (intptr_t) proc, ep);
+        (void) opal_hash_table_set_value_uint64(&uct_module->id_to_endpoint, (intptr_t) proc, ep);
     } while (0);
 
-    opal_mutex_unlock (&uct_module->endpoint_lock);
+    opal_mutex_unlock(&uct_module->endpoint_lock);
 
     return ep;
 }
 
-static int mca_btl_uct_add_procs (mca_btl_base_module_t *btl,
-                                  size_t nprocs, opal_proc_t **opal_procs,
-                                  mca_btl_base_endpoint_t **peers,
-                                  opal_bitmap_t *reachable)
+static int mca_btl_uct_add_procs(mca_btl_base_module_t *btl, size_t nprocs,
+                                 opal_proc_t **opal_procs, mca_btl_base_endpoint_t **peers,
+                                 opal_bitmap_t *reachable)
 {
     mca_btl_uct_module_t *uct_module = (mca_btl_uct_module_t *) btl;
     int rc;
@@ -79,35 +81,35 @@ static int mca_btl_uct_add_procs (mca_btl_base_module_t *btl,
         /* NTH: might want to vary this size based off the universe size (if
          * one exists). the table is only used for connection lookup and
          * endpoint removal. */
-        rc = opal_hash_table_init (&uct_module->id_to_endpoint, 512);
+        rc = opal_hash_table_init(&uct_module->id_to_endpoint, 512);
         if (OPAL_SUCCESS != rc) {
             BTL_ERROR(("error initializing the endpoint hash. rc = %d", rc));
             return rc;
         }
 
         if (am_tl) {
-            rc = opal_free_list_init (&uct_module->short_frags, sizeof (mca_btl_uct_base_frag_t),
-                                      opal_cache_line_size, OBJ_CLASS(mca_btl_uct_base_frag_t),
-                                      MCA_BTL_UCT_TL_ATTR(am_tl, 0).cap.am.max_short, opal_cache_line_size,
-                                      0, 1024, 64, NULL, 0, NULL, NULL, NULL);
-
-            rc = opal_free_list_init (&uct_module->eager_frags, sizeof (mca_btl_uct_base_frag_t),
-                                      opal_cache_line_size, OBJ_CLASS(mca_btl_uct_base_frag_t),
-                                      btl->btl_eager_limit, opal_cache_line_size,
-                                      0, 1024, 64, NULL, 0, uct_module->rcache, NULL, NULL);
-
-            rc = opal_free_list_init (&uct_module->max_frags, sizeof (mca_btl_uct_base_frag_t),
-                                      opal_cache_line_size, OBJ_CLASS(mca_btl_uct_base_frag_t),
-                                      btl->btl_max_send_size, opal_cache_line_size, 0, 128, 8,
-                                      NULL, 0, uct_module->rcache, NULL, NULL);
+            rc = opal_free_list_init(&uct_module->short_frags, sizeof(mca_btl_uct_base_frag_t),
+                                     opal_cache_line_size, OBJ_CLASS(mca_btl_uct_base_frag_t),
+                                     MCA_BTL_UCT_TL_ATTR(am_tl, 0).cap.am.max_short,
+                                     opal_cache_line_size, 0, 1024, 64, NULL, 0, NULL, NULL, NULL);
+
+            rc = opal_free_list_init(&uct_module->eager_frags, sizeof(mca_btl_uct_base_frag_t),
+                                     opal_cache_line_size, OBJ_CLASS(mca_btl_uct_base_frag_t),
+                                     btl->btl_eager_limit, opal_cache_line_size, 0, 1024, 64, NULL,
+                                     0, uct_module->rcache, NULL, NULL);
+
+            rc = opal_free_list_init(&uct_module->max_frags, sizeof(mca_btl_uct_base_frag_t),
+                                     opal_cache_line_size, OBJ_CLASS(mca_btl_uct_base_frag_t),
+                                     btl->btl_max_send_size, opal_cache_line_size, 0, 128, 8, NULL,
+                                     0, uct_module->rcache, NULL, NULL);
         }
 
         uct_module->initialized = true;
     }
 
-    for (size_t i = 0 ; i < nprocs ; ++i) {
+    for (size_t i = 0; i < nprocs; ++i) {
         /* all endpoints are reachable for uct */
-        peers[i] = mca_btl_uct_get_ep (btl, opal_procs[i]);
+        peers[i] = mca_btl_uct_get_ep(btl, opal_procs[i]);
         if (OPAL_UNLIKELY(NULL == peers[i])) {
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
@@ -118,31 +120,32 @@ static int mca_btl_uct_add_procs (mca_btl_base_module_t *btl,
     return OPAL_SUCCESS;
 }
 
-static int mca_btl_uct_del_procs (mca_btl_base_module_t *btl, size_t nprocs,
-                                  opal_proc_t **procs, mca_btl_base_endpoint_t **peers)
+static int mca_btl_uct_del_procs(mca_btl_base_module_t *btl, size_t nprocs, opal_proc_t **procs,
+                                 mca_btl_base_endpoint_t **peers)
 {
     mca_btl_uct_module_t *uct_module = (mca_btl_uct_module_t *) btl;
     mca_btl_base_endpoint_t *ep;
     int rc;
 
-    for (size_t i = 0 ; i < nprocs ; ++i) {
+    for (size_t i = 0; i < nprocs; ++i) {
         if (NULL == procs[i]) {
             continue;
         }
 
-        rc = opal_hash_table_get_value_uint64 (&uct_module->id_to_endpoint, (intptr_t) procs[i], (void **) &ep);
+        rc = opal_hash_table_get_value_uint64(&uct_module->id_to_endpoint, (intptr_t) procs[i],
+                                              (void **) &ep);
         if (OPAL_SUCCESS != rc) {
             continue;
         }
 
-        (void) opal_hash_table_remove_value_uint64 (&uct_module->id_to_endpoint, (intptr_t) procs[i]);
+        (void) opal_hash_table_remove_value_uint64(&uct_module->id_to_endpoint,
+                                                   (intptr_t) procs[i]);
         OBJ_RELEASE(ep);
     }
 
     return OPAL_SUCCESS;
 }
 
-
 /**
  * @brief Register a memory region for put/get/atomic operations.
  *
@@ -162,16 +165,17 @@ static int mca_btl_uct_del_procs (mca_btl_base_module_t *btl, size_t nprocs,
  * as they may use limited system/NIC resources.
  */
 static struct mca_btl_base_registration_handle_t *
-mca_btl_uct_register_mem (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, void *base,
-                          size_t size, uint32_t flags)
+mca_btl_uct_register_mem(struct mca_btl_base_module_t *btl,
+                         struct mca_btl_base_endpoint_t *endpoint, void *base, size_t size,
+                         uint32_t flags)
 {
     mca_btl_uct_module_t *uct_module = (mca_btl_uct_module_t *) btl;
     mca_btl_uct_reg_t *reg;
     int access_flags = flags & MCA_BTL_REG_FLAG_ACCESS_ANY;
     int rc;
 
-    rc = uct_module->rcache->rcache_register (uct_module->rcache, base, size, 0, access_flags,
-                                              (mca_rcache_base_registration_t **) &reg);
+    rc = uct_module->rcache->rcache_register(uct_module->rcache, base, size, 0, access_flags,
+                                             (mca_rcache_base_registration_t **) &reg);
     if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
         return NULL;
     }
@@ -190,18 +194,20 @@ mca_btl_uct_register_mem (struct mca_btl_base_module_t *btl, struct mca_btl_base
  * after it is deregistered. It is erroneous to specify a memory handle associated with
  * a remote node.
  */
-static int mca_btl_uct_deregister_mem (mca_btl_base_module_t *btl, mca_btl_base_registration_handle_t *handle)
+static int mca_btl_uct_deregister_mem(mca_btl_base_module_t *btl,
+                                      mca_btl_base_registration_handle_t *handle)
 {
     mca_btl_uct_module_t *uct_module = (mca_btl_uct_module_t *) btl;
-    mca_btl_uct_reg_t *reg =
-        (mca_btl_uct_reg_t *)((intptr_t) handle - offsetof (mca_btl_uct_reg_t, handle));
+    mca_btl_uct_reg_t *reg = (mca_btl_uct_reg_t *) ((intptr_t) handle
+                                                    - offsetof(mca_btl_uct_reg_t, handle));
 
-    (void) uct_module->rcache->rcache_deregister (uct_module->rcache, &reg->base);
+    (void) uct_module->rcache->rcache_deregister(uct_module->rcache, &reg->base);
 
     return OPAL_SUCCESS;
 }
 
-int mca_btl_uct_reg_mem (void *reg_data, void *base, size_t size, mca_rcache_base_registration_t *reg)
+int mca_btl_uct_reg_mem(void *reg_data, void *base, size_t size,
+                        mca_rcache_base_registration_t *reg)
 {
     mca_btl_uct_module_t *uct_module = (mca_btl_uct_module_t *) reg_data;
     mca_btl_uct_reg_t *uct_reg = (mca_btl_uct_reg_t *) reg;
@@ -225,19 +231,22 @@ int mca_btl_uct_reg_mem (void *reg_data, void *base, size_t size, mca_rcache_bas
         uct_flags = UCT_MD_MEM_ACCESS_ALL;
     }
 
-    ucs_status = uct_md_mem_reg (uct_module->md->uct_md, base, size, uct_flags, &uct_reg->uct_memh);
+    ucs_status = uct_md_mem_reg(uct_module->md->uct_md, base, size, uct_flags, &uct_reg->uct_memh);
     if (UCS_OK != ucs_status) {
         BTL_VERBOSE(("Error registering memory with UCT. code: %d", ucs_status));
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
-    if (reg->access_flags & (MCA_BTL_REG_FLAG_REMOTE_READ | MCA_BTL_REG_FLAG_REMOTE_WRITE | MCA_BTL_REG_FLAG_REMOTE_ATOMIC)) {
+    if (reg->access_flags
+        & (MCA_BTL_REG_FLAG_REMOTE_READ | MCA_BTL_REG_FLAG_REMOTE_WRITE
+           | MCA_BTL_REG_FLAG_REMOTE_ATOMIC)) {
         /* requested registration may be used by a remote process so go ahead and pack
          * the registration handle */
-        ucs_status = uct_md_mkey_pack (uct_module->md->uct_md, uct_reg->uct_memh, uct_reg->handle.packed_handle);
+        ucs_status = uct_md_mkey_pack(uct_module->md->uct_md, uct_reg->uct_memh,
+                                      uct_reg->handle.packed_handle);
         if (OPAL_UNLIKELY(UCS_OK != ucs_status)) {
             BTL_VERBOSE(("Could not pack remote key. code: %d", ucs_status));
-            uct_md_mem_dereg (uct_module->md->uct_md, uct_reg->uct_memh);
+            uct_md_mem_dereg(uct_module->md->uct_md, uct_reg->uct_memh);
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
     }
@@ -245,29 +254,28 @@ int mca_btl_uct_reg_mem (void *reg_data, void *base, size_t size, mca_rcache_bas
     return OPAL_SUCCESS;
 }
 
-int mca_btl_uct_dereg_mem (void *reg_data, mca_rcache_base_registration_t *reg)
+int mca_btl_uct_dereg_mem(void *reg_data, mca_rcache_base_registration_t *reg)
 {
     mca_btl_uct_module_t *uct_module = (mca_btl_uct_module_t *) reg_data;
     mca_btl_uct_reg_t *uct_reg = (mca_btl_uct_reg_t *) reg;
 
-    uct_md_mem_dereg (uct_module->md->uct_md, uct_reg->uct_memh);
+    uct_md_mem_dereg(uct_module->md->uct_md, uct_reg->uct_memh);
 
     return OPAL_SUCCESS;
 }
 
-
 /*
  * Cleanup/release module resources.
  */
 
-int mca_btl_uct_finalize (mca_btl_base_module_t* btl)
+int mca_btl_uct_finalize(mca_btl_base_module_t *btl)
 {
     mca_btl_uct_module_t *uct_module = (mca_btl_uct_module_t *) btl;
     mca_btl_uct_endpoint_t *endpoint;
     uint64_t key;
 
     /* clean up any leftover endpoints */
-    OPAL_HASH_TABLE_FOREACH(key, uint64, endpoint, &uct_module->id_to_endpoint) {
+    OPAL_HASH_TABLE_FOREACH (key, uint64, endpoint, &uct_module->id_to_endpoint) {
         OBJ_RELEASE(endpoint);
     }
     OBJ_DESTRUCT(&uct_module->id_to_endpoint);
@@ -279,7 +287,7 @@ int mca_btl_uct_finalize (mca_btl_base_module_t* btl)
     OBJ_DESTRUCT(&uct_module->pending_connection_reqs);
 
     if (uct_module->rcache) {
-        mca_rcache_base_module_destroy (uct_module->rcache);
+        mca_rcache_base_module_destroy(uct_module->rcache);
     }
 
     if (NULL != uct_module->am_tl) {
@@ -294,12 +302,12 @@ int mca_btl_uct_finalize (mca_btl_base_module_t* btl)
         OBJ_RELEASE(uct_module->rdma_tl);
     }
 
-    ucs_async_context_destroy (uct_module->ucs_async);
+    ucs_async_context_destroy(uct_module->ucs_async);
 
     OBJ_DESTRUCT(&uct_module->endpoint_lock);
 
-    free (uct_module->md_name);
-    free (uct_module);
+    free(uct_module->md_name);
+    free(uct_module);
 
     return OPAL_SUCCESS;
 }
@@ -308,58 +316,59 @@ mca_btl_uct_module_t mca_btl_uct_module_template = {
     .super = {
         /* initialize functions. this btl only support RDMA and atomics
          * for now so it does not provide prepare_src, alloc, free, or send */
-        .btl_component      = &mca_btl_uct_component.super,
-        .btl_add_procs      = mca_btl_uct_add_procs,
-        .btl_del_procs      = mca_btl_uct_del_procs,
-        .btl_finalize       = mca_btl_uct_finalize,
-        .btl_put            = mca_btl_uct_put,
-        .btl_get            = mca_btl_uct_get,
-        .btl_register_mem   = mca_btl_uct_register_mem,
+        .btl_component = &mca_btl_uct_component.super,
+        .btl_add_procs = mca_btl_uct_add_procs,
+        .btl_del_procs = mca_btl_uct_del_procs,
+        .btl_finalize = mca_btl_uct_finalize,
+        .btl_put = mca_btl_uct_put,
+        .btl_get = mca_btl_uct_get,
+        .btl_register_mem = mca_btl_uct_register_mem,
         .btl_deregister_mem = mca_btl_uct_deregister_mem,
-        .btl_atomic_op      = mca_btl_uct_aop,
-        .btl_atomic_fop     = mca_btl_uct_afop,
-        .btl_atomic_cswap   = mca_btl_uct_acswap,
-        .btl_flush          = mca_btl_uct_flush,
+        .btl_atomic_op = mca_btl_uct_aop,
+        .btl_atomic_fop = mca_btl_uct_afop,
+        .btl_atomic_cswap = mca_btl_uct_acswap,
+        .btl_flush = mca_btl_uct_flush,
 
-        .btl_sendi          = mca_btl_uct_sendi,
-        .btl_prepare_src    = mca_btl_uct_prepare_src,
-        .btl_send           = mca_btl_uct_send,
-        .btl_alloc          = mca_btl_uct_alloc,
-        .btl_free           = mca_btl_uct_free,
+        .btl_sendi = mca_btl_uct_sendi,
+        .btl_prepare_src = mca_btl_uct_prepare_src,
+        .btl_send = mca_btl_uct_send,
+        .btl_alloc = mca_btl_uct_alloc,
+        .btl_free = mca_btl_uct_free,
 
         /* set the default flags for this btl. uct provides us with rdma and both
          * fetching and non-fetching atomics (though limited to add and cswap) */
-        .btl_flags          = MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_ATOMIC_FOPS | MCA_BTL_FLAGS_ATOMIC_OPS,
-        .btl_atomic_flags   = MCA_BTL_ATOMIC_SUPPORTS_ADD | MCA_BTL_ATOMIC_SUPPORTS_CSWAP |
-                              MCA_BTL_ATOMIC_SUPPORTS_SWAP | MCA_BTL_ATOMIC_SUPPORTS_32BIT,
+        .btl_flags = MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_ATOMIC_FOPS | MCA_BTL_FLAGS_ATOMIC_OPS
+                     | MCA_BTL_FLAGS_RDMA_REMOTE_COMPLETION,
+        .btl_atomic_flags = MCA_BTL_ATOMIC_SUPPORTS_ADD | MCA_BTL_ATOMIC_SUPPORTS_CSWAP
+                            | MCA_BTL_ATOMIC_SUPPORTS_SWAP | MCA_BTL_ATOMIC_SUPPORTS_32BIT,
 
         /* set the default limits on put and get */
-        .btl_put_limit      = 1 << 23,
-        .btl_put_alignment  = 0,
-        .btl_get_limit      = 1 << 23,
-        .btl_get_alignment  = 0,
+        .btl_put_limit = 1 << 23,
+        .btl_put_alignment = 0,
+        .btl_get_limit = 1 << 23,
+        .btl_get_alignment = 0,
 
         .btl_rndv_eager_limit = 8192,
         .btl_rdma_pipeline_frag_size = 4 * 1024 * 1024,
         .btl_rdma_pipeline_send_length = 8192,
-        .btl_eager_limit    = 8192,
-        .btl_max_send_size  = 65536,
-   }
-};
+        .btl_eager_limit = 8192,
+        .btl_max_send_size = 65536,
+    }};
 
 OBJ_CLASS_INSTANCE(mca_btl_uct_reg_t, opal_free_list_item_t, NULL, NULL);
 
-static void mca_btl_uct_md_construct (mca_btl_uct_md_t *md)
+static void mca_btl_uct_md_construct(mca_btl_uct_md_t *md)
 {
     md->uct_md = NULL;
 }
 
-static void mca_btl_uct_md_destruct (mca_btl_uct_md_t *md)
+static void mca_btl_uct_md_destruct(mca_btl_uct_md_t *md)
 {
     if (md->uct_md) {
-        uct_md_close (md->uct_md);
+        uct_md_close(md->uct_md);
         md->uct_md = NULL;
     }
 }
 
-OBJ_CLASS_INSTANCE(mca_btl_uct_md_t, opal_object_t, mca_btl_uct_md_construct, mca_btl_uct_md_destruct);
+OBJ_CLASS_INSTANCE(mca_btl_uct_md_t, opal_object_t, mca_btl_uct_md_construct,
+                   mca_btl_uct_md_destruct);
diff --git a/opal/mca/btl/uct/btl_uct_rdma.c b/opal/mca/btl/uct/btl_uct_rdma.c
index 9ee9530f260..d4210e4631c 100644
--- a/opal/mca/btl/uct/btl_uct_rdma.c
+++ b/opal/mca/btl/uct/btl_uct_rdma.c
@@ -2,6 +2,8 @@
 /*
  * Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2025      Google, LLC. All rights reserved.
+ * Copyright (c) 2025      Triad National Security, LLC. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -11,33 +13,56 @@
 
 #include "btl_uct_device_context.h"
 
-void mca_btl_uct_uct_completion (uct_completion_t *uct_comp, ucs_status_t status)
+static void mca_btl_uct_uct_completion_compat(uct_completion_t *uct_comp, ucs_status_t status)
 {
-    mca_btl_uct_uct_completion_t *comp = (mca_btl_uct_uct_completion_t *) ((uintptr_t) uct_comp - offsetof (mca_btl_uct_uct_completion_t, uct_comp));
+    mca_btl_uct_uct_completion_t *comp = (mca_btl_uct_uct_completion_t
+                                              *) ((uintptr_t) uct_comp
+                                                  - offsetof(mca_btl_uct_uct_completion_t,
+                                                             uct_comp));
 
-    BTL_VERBOSE(("network operation complete. status = %d", status));
+    BTL_VERBOSE(("network operation complete. frag = %p, status = %d", (void *) comp->frag, status));
 
     comp->status = status;
-    opal_fifo_push (&comp->dev_context->completion_fifo, &comp->super.super);
+    opal_fifo_push(&comp->dev_context->completion_fifo, &comp->super.super);
 }
 
+#if UCT_API >= ((1L<<UCT_MAJOR_BIT)|(10L << UCT_MINOR_BIT))
+static void mca_btl_uct_uct_completion(uct_completion_t *uct_comp) {
+    ucs_status_t status = uct_comp->status;
+    /* reset the status now as the completion can not be accessed after calling
+     * mca_btl_uct_uct_completion_compat (may get returned) */
+    uct_comp->status = UCS_OK;
 
-static void mca_btl_uct_uct_completion_construct (mca_btl_uct_uct_completion_t *comp)
+    mca_btl_uct_uct_completion_compat(uct_comp, status);
+    /* do not access the completion struture here */
+}
+#else
+static void mca_btl_uct_uct_completion(uct_completion_t *uct_comp, ucs_status_t status) {
+     mca_btl_uct_uct_completion_compat(uct_comp, status);
+}
+#endif
+
+static void mca_btl_uct_uct_completion_construct(mca_btl_uct_uct_completion_t *comp)
 {
     comp->frag = NULL;
     comp->uct_comp.func = mca_btl_uct_uct_completion;
-}
+    comp->uct_comp.count = 1;
 
-OBJ_CLASS_INSTANCE(mca_btl_uct_uct_completion_t, opal_free_list_item_t, mca_btl_uct_uct_completion_construct, NULL);
+#if UCT_API >= ((1L<<UCT_MAJOR_BIT)|(10L << UCT_MINOR_BIT))
+    comp->uct_comp.status = UCS_OK;
+#endif
+}
 
+OBJ_CLASS_INSTANCE(mca_btl_uct_uct_completion_t, opal_free_list_item_t,
+                   mca_btl_uct_uct_completion_construct, NULL);
 
-mca_btl_uct_uct_completion_t *
-mca_btl_uct_uct_completion_alloc (mca_btl_uct_module_t *uct_btl, mca_btl_base_endpoint_t *endpoint,
-                                  void *local_address, mca_btl_base_registration_handle_t *local_handle,
-                                  mca_btl_uct_device_context_t *dev_context, mca_btl_base_rdma_completion_fn_t cbfunc,
-                                  void *cbcontext, void *cbdata)
+mca_btl_uct_uct_completion_t *mca_btl_uct_uct_completion_alloc(
+    mca_btl_uct_module_t *uct_btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
+    mca_btl_base_registration_handle_t *local_handle, mca_btl_uct_device_context_t *dev_context,
+    mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
 {
-    mca_btl_uct_uct_completion_t *comp = (mca_btl_uct_uct_completion_t *) opal_free_list_get (&dev_context->rdma_completions);
+    mca_btl_uct_uct_completion_t *comp = (mca_btl_uct_uct_completion_t *) opal_free_list_get(
+        &dev_context->rdma_completions);
     if (OPAL_LIKELY(NULL != comp)) {
         comp->uct_comp.count = 1;
         comp->btl = &uct_btl->super;
@@ -53,86 +78,93 @@ mca_btl_uct_uct_completion_alloc (mca_btl_uct_module_t *uct_btl, mca_btl_base_en
     return comp;
 }
 
-void mca_btl_uct_uct_completion_release (mca_btl_uct_uct_completion_t *comp)
+void mca_btl_uct_uct_completion_release(mca_btl_uct_uct_completion_t *comp)
 {
     if (comp) {
-        opal_free_list_return (&comp->dev_context->rdma_completions, &comp->super);
+        opal_free_list_return(&comp->dev_context->rdma_completions, &comp->super);
     }
 }
 
-static void mca_btl_uct_get_unpack (void *arg, const void *data, size_t length)
+static void mca_btl_uct_get_unpack(void *arg, const void *data, size_t length)
 {
-    memcpy (arg, data, length);
+    memcpy(arg, data, length);
 }
 
-int mca_btl_uct_get (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
-                      uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                      mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-                      int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
+int mca_btl_uct_get(mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
+                    void *local_address, uint64_t remote_address,
+                    mca_btl_base_registration_handle_t *local_handle,
+                    mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
+                    int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext,
+                    void *cbdata)
 {
     mca_btl_uct_module_t *uct_btl = (mca_btl_uct_module_t *) btl;
-    mca_btl_uct_device_context_t *context = mca_btl_uct_module_get_rdma_context (uct_btl);
+    mca_btl_uct_device_context_t *context = mca_btl_uct_module_get_rdma_context(uct_btl);
     mca_btl_uct_uct_completion_t *comp = NULL;
     ucs_status_t ucs_status;
     uct_rkey_bundle_t rkey;
     uct_ep_h ep_handle;
     int rc;
 
-    BTL_VERBOSE(("performing get operation. local address: %p, length: %lu", local_address, (unsigned long) size));
+    BTL_VERBOSE(("performing get operation. local address: %p, length: %lu", local_address,
+                 (unsigned long) size));
 
     if (cbfunc) {
-        comp = mca_btl_uct_uct_completion_alloc (uct_btl, endpoint, local_address, local_handle, context,
-                                                 cbfunc, cbcontext, cbdata);
+        comp = mca_btl_uct_uct_completion_alloc(uct_btl, endpoint, local_address, local_handle,
+                                                context, cbfunc, cbcontext, cbdata);
         if (OPAL_UNLIKELY(NULL == comp)) {
             BTL_VERBOSE(("culd not allocate completion structure"));
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
     }
 
-    rc = mca_btl_uct_get_rkey (uct_btl, context, endpoint, remote_handle, &rkey, &ep_handle);
+    rc = mca_btl_uct_get_rkey(uct_btl, context, endpoint, remote_handle, &rkey, &ep_handle);
     if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
         BTL_VERBOSE(("mca_btl_uct_get_rkey returned %d", rc));
-        mca_btl_uct_uct_completion_release (comp);
+        mca_btl_uct_uct_completion_release(comp);
         return rc;
     }
 
-    mca_btl_uct_context_lock (context);
+    mca_btl_uct_context_lock(context);
 
     if (size <= MCA_BTL_UCT_TL_ATTR(uct_btl->rdma_tl, context->context_id).cap.get.max_bcopy) {
-        ucs_status = uct_ep_get_bcopy (ep_handle, mca_btl_uct_get_unpack, local_address, size, remote_address,
-                                       rkey.rkey, &comp->uct_comp);
+        ucs_status = uct_ep_get_bcopy(ep_handle, mca_btl_uct_get_unpack, local_address, size,
+                                      remote_address, rkey.rkey, &comp->uct_comp);
     } else {
-        uct_iov_t iov = {.buffer = local_address, .length = size, .stride = 0, .count = 1,
+        uct_iov_t iov = {.buffer = local_address,
+                         .length = size,
+                         .stride = 0,
+                         .count = 1,
                          .memh = MCA_BTL_UCT_REG_REMOTE_TO_LOCAL(local_handle)->uct_memh};
-        ucs_status = uct_ep_get_zcopy (ep_handle, &iov, 1, remote_address, rkey.rkey, &comp->uct_comp);
+        ucs_status = uct_ep_get_zcopy(ep_handle, &iov, 1, remote_address, rkey.rkey,
+                                      &comp->uct_comp);
     }
 
     /* go ahead and progress the worker while we have the lock (if we are not in an AM callback) */
     if (!context->in_am_callback) {
-        (void) uct_worker_progress (context->uct_worker);
+        (void) uct_worker_progress(context->uct_worker);
     }
 
-    mca_btl_uct_context_unlock (context);
+    mca_btl_uct_context_unlock(context);
 
     if (!context->in_am_callback) {
-        mca_btl_uct_device_handle_completions (context);
+        mca_btl_uct_device_handle_completions(context);
     }
 
     if (UCS_OK == ucs_status && cbfunc) {
         /* if UCS_OK is returned the callback will never fire so we have to make the callback
          * ourselves */
-        cbfunc (btl, endpoint, local_address, local_handle, cbcontext, cbdata, OPAL_SUCCESS);
+        cbfunc(btl, endpoint, local_address, local_handle, cbcontext, cbdata, OPAL_SUCCESS);
     }
 
     if (UCS_INPROGRESS == ucs_status) {
         ucs_status = UCS_OK;
     } else {
-        mca_btl_uct_uct_completion_release (comp);
+        mca_btl_uct_uct_completion_release(comp);
     }
 
     BTL_VERBOSE(("get issued. status = %d", ucs_status));
 
-    mca_btl_uct_rkey_release (uct_btl, &rkey);
+    mca_btl_uct_rkey_release(uct_btl, &rkey);
 
     return OPAL_LIKELY(UCS_OK == ucs_status) ? OPAL_SUCCESS : OPAL_ERR_RESOURCE_BUSY;
 }
@@ -144,21 +176,23 @@ struct mca_btl_uct_put_pack_args_t {
 
 typedef struct mca_btl_uct_put_pack_args_t mca_btl_uct_put_pack_args_t;
 
-static size_t mca_btl_uct_put_pack (void *dest, void *arg)
+static size_t mca_btl_uct_put_pack(void *dest, void *arg)
 {
     mca_btl_uct_put_pack_args_t *args = (mca_btl_uct_put_pack_args_t *) arg;
 
-    memcpy (dest, args->local_address, args->size);
+    memcpy(dest, args->local_address, args->size);
     return args->size;
 }
 
-int mca_btl_uct_put (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
-                      uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                      mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-                      int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
+int mca_btl_uct_put(mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
+                    void *local_address, uint64_t remote_address,
+                    mca_btl_base_registration_handle_t *local_handle,
+                    mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
+                    int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext,
+                    void *cbdata)
 {
     mca_btl_uct_module_t *uct_btl = (mca_btl_uct_module_t *) btl;
-    mca_btl_uct_device_context_t *context = mca_btl_uct_module_get_rdma_context (uct_btl);
+    mca_btl_uct_device_context_t *context = mca_btl_uct_module_get_rdma_context(uct_btl);
     mca_btl_uct_uct_completion_t *comp = NULL;
     ucs_status_t ucs_status;
     uct_rkey_bundle_t rkey;
@@ -167,82 +201,90 @@ int mca_btl_uct_put (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoi
     bool use_bcopy = false;
     int rc;
 
-    BTL_VERBOSE(("performing put operation. local address: %p, length: %lu", local_address, (unsigned long) size));
+    BTL_VERBOSE(("performing put operation. local address: %p, length: %lu", local_address,
+                 (unsigned long) size));
 
     if (size > uct_btl->super.btl_put_local_registration_threshold && cbfunc) {
-        comp = mca_btl_uct_uct_completion_alloc (uct_btl, endpoint, local_address, local_handle, context,
-                                                 cbfunc, cbcontext, cbdata);
+        comp = mca_btl_uct_uct_completion_alloc(uct_btl, endpoint, local_address, local_handle,
+                                                context, cbfunc, cbcontext, cbdata);
         if (OPAL_UNLIKELY(NULL == comp)) {
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
     }
 
-    rc = mca_btl_uct_get_rkey (uct_btl, context, endpoint, remote_handle, &rkey, &ep_handle);
+    rc = mca_btl_uct_get_rkey(uct_btl, context, endpoint, remote_handle, &rkey, &ep_handle);
     if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-        mca_btl_uct_uct_completion_release (comp);
+        mca_btl_uct_uct_completion_release(comp);
         return rc;
     }
 
-    mca_btl_uct_context_lock (context);
+    mca_btl_uct_context_lock(context);
 
     /* determine what UCT prototol should be used */
     if (size <= uct_btl->super.btl_put_local_registration_threshold) {
-        use_short = size <= MCA_BTL_UCT_TL_ATTR(uct_btl->rdma_tl, context->context_id).cap.put.max_short;
+        use_short = size
+                    <= MCA_BTL_UCT_TL_ATTR(uct_btl->rdma_tl, context->context_id).cap.put.max_short;
         use_bcopy = !use_short;
     }
 
     do {
         if (use_short) {
-            ucs_status = uct_ep_put_short (ep_handle, local_address, size, remote_address, rkey.rkey);
+            ucs_status = uct_ep_put_short(ep_handle, local_address, size, remote_address,
+                                          rkey.rkey);
         } else if (use_bcopy) {
-            ssize_t tmp = uct_ep_put_bcopy (ep_handle, mca_btl_uct_put_pack,
-                                            &(mca_btl_uct_put_pack_args_t) {.local_address = local_address,
-                                                    .size = size},
-                                            remote_address, rkey.rkey);
+            ssize_t tmp = uct_ep_put_bcopy(
+                ep_handle, mca_btl_uct_put_pack,
+                &(mca_btl_uct_put_pack_args_t){.local_address = local_address, .size = size},
+                remote_address, rkey.rkey);
             ucs_status = (tmp == (ssize_t) size) ? UCS_OK : UCS_ERR_NO_RESOURCE;
         } else {
-            uct_iov_t iov = {.buffer = local_address, .length = size, .stride = 0, .count = 1,
-                         .memh = MCA_BTL_UCT_REG_REMOTE_TO_LOCAL(local_handle)->uct_memh};
-
-            ucs_status = uct_ep_put_zcopy (ep_handle, &iov, 1, remote_address, rkey.rkey, &comp->uct_comp);
+            uct_iov_t iov = {.buffer = local_address,
+                             .length = size,
+                             .stride = 0,
+                             .count = 1,
+                             .memh = MCA_BTL_UCT_REG_REMOTE_TO_LOCAL(local_handle)->uct_memh};
+
+            ucs_status = uct_ep_put_zcopy(ep_handle, &iov, 1, remote_address, rkey.rkey,
+                                          &comp->uct_comp);
         }
 
         /* go ahead and progress the worker while we have the lock */
         if (UCS_ERR_NO_RESOURCE != ucs_status || context->in_am_callback) {
             if (!context->in_am_callback) {
-                (void) uct_worker_progress (context->uct_worker);
+                (void) uct_worker_progress(context->uct_worker);
             }
 
             break;
         }
 
         /* wait for something to complete */
-        while (!uct_worker_progress (context->uct_worker));
+        while (!uct_worker_progress(context->uct_worker))
+            ;
     } while (1);
 
-    mca_btl_uct_context_unlock (context);
+    mca_btl_uct_context_unlock(context);
 
-    mca_btl_uct_device_handle_completions (context);
+    mca_btl_uct_device_handle_completions(context);
 
     if (UCS_OK == ucs_status && cbfunc) {
         /* if UCS_OK is returned the callback will never fire so we have to make the callback
          * ourselves. this callback is possibly being made before the data is visible to the
          * remote process. */
-        cbfunc (btl, endpoint, local_address, local_handle, cbcontext, cbdata, OPAL_SUCCESS);
+        cbfunc(btl, endpoint, local_address, local_handle, cbcontext, cbdata, OPAL_SUCCESS);
     }
 
     if (UCS_INPROGRESS == ucs_status) {
         ucs_status = UCS_OK;
     } else {
-        mca_btl_uct_uct_completion_release (comp);
+        mca_btl_uct_uct_completion_release(comp);
     }
 
-    mca_btl_uct_rkey_release (uct_btl, &rkey);
+    mca_btl_uct_rkey_release(uct_btl, &rkey);
 
     return OPAL_LIKELY(UCS_OK == ucs_status) ? OPAL_SUCCESS : OPAL_ERR_RESOURCE_BUSY;
 }
 
-int mca_btl_uct_flush (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint)
+int mca_btl_uct_flush(mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint)
 {
     mca_btl_uct_module_t *uct_btl = (mca_btl_uct_module_t *) btl;
     const int tl_index = uct_btl->rdma_tl->tl_index;
@@ -251,37 +293,38 @@ int mca_btl_uct_flush (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endp
 
     BTL_VERBOSE(("mca_btl_uct_flush starting"));
 
-    for (int i = 0 ; i < context_count ; ++i) {
+    for (int i = 0; i < context_count; ++i) {
         mca_btl_uct_device_context_t *context = uct_btl->rdma_tl->uct_dev_contexts[i];
 
         if (NULL == context) {
             continue;
         }
 
-        mca_btl_uct_context_lock (context);
+        mca_btl_uct_context_lock(context);
         /* this loop is here because at least some of the TLs do no support a
          * completion callback. its a real PIA but has to be done for now. */
         do {
-            uct_worker_progress (context->uct_worker);
+            uct_worker_progress(context->uct_worker);
 
             if (NULL != endpoint && endpoint->uct_eps[context->context_id][tl_index].uct_ep) {
-                ucs_status = uct_ep_flush (endpoint->uct_eps[context->context_id][tl_index].uct_ep, 0, NULL);
+                ucs_status = uct_ep_flush(endpoint->uct_eps[context->context_id][tl_index].uct_ep,
+                                          0, NULL);
             } else {
-                ucs_status = uct_iface_flush (context->uct_iface, 0, NULL);
+                ucs_status = uct_iface_flush(context->uct_iface, 0, NULL);
             }
         } while (UCS_INPROGRESS == ucs_status);
 
-        mca_btl_uct_context_unlock (context);
-        mca_btl_uct_device_handle_completions (context);
+        mca_btl_uct_context_unlock(context);
+        mca_btl_uct_device_handle_completions(context);
     }
 
     return OPAL_SUCCESS;
 }
 
-int mca_btl_uct_flush_thread (mca_btl_base_module_t *btl)
+int mca_btl_uct_flush_thread(mca_btl_base_module_t *btl)
 {
     mca_btl_uct_module_t *uct_btl = (mca_btl_uct_module_t *) btl;
-    const int context_id = mca_btl_uct_get_context_index ();
+    const int context_id = mca_btl_uct_get_context_index();
     mca_btl_uct_device_context_t *context = uct_btl->rdma_tl->uct_dev_contexts[context_id];
     ucs_status_t ucs_status;
 
@@ -291,18 +334,18 @@ int mca_btl_uct_flush_thread (mca_btl_base_module_t *btl)
         return OPAL_SUCCESS;
     }
 
-    mca_btl_uct_context_lock (context);
+    mca_btl_uct_context_lock(context);
 
     /* this loop is here because at least some of the TLs do no support a
      * completion callback. its a real PIA but has to be done for now. */
     do {
-        uct_worker_progress (context->uct_worker);
-        ucs_status = uct_iface_flush (context->uct_iface, 0, NULL);
+        uct_worker_progress(context->uct_worker);
+        ucs_status = uct_iface_flush(context->uct_iface, 0, NULL);
     } while (UCS_INPROGRESS == ucs_status);
 
-    mca_btl_uct_context_unlock (context);
+    mca_btl_uct_context_unlock(context);
 
-    mca_btl_uct_device_handle_completions (context);
+    mca_btl_uct_device_handle_completions(context);
 
     return OPAL_SUCCESS;
 }
diff --git a/opal/mca/btl/uct/btl_uct_rdma.h b/opal/mca/btl/uct/btl_uct_rdma.h
index ab790371afe..0438106b2c8 100644
--- a/opal/mca/btl/uct/btl_uct_rdma.h
+++ b/opal/mca/btl/uct/btl_uct_rdma.h
@@ -10,25 +10,23 @@
  */
 
 #if !defined(BTL_UCT_RDMA_H)
-#define BTL_UCT_RDMA_H
+#    define BTL_UCT_RDMA_H
 
-#include "btl_uct.h"
-#include "btl_uct_endpoint.h"
-#include "btl_uct_frag.h"
+#    include "btl_uct.h"
+#    include "btl_uct_endpoint.h"
+#    include "btl_uct_frag.h"
 
 /**
  * @brief allocate a callback structure
  */
-mca_btl_uct_uct_completion_t *mca_btl_uct_uct_completion_alloc (mca_btl_uct_module_t *btl, mca_btl_base_endpoint_t *endpoint,
-                                                                void *local_address, mca_btl_base_registration_handle_t *local_handle,
-                                                                mca_btl_uct_device_context_t *dev_context, mca_btl_base_rdma_completion_fn_t cbfunc,
-                                                                void *cbcontext, void *cbdata);
+mca_btl_uct_uct_completion_t *mca_btl_uct_uct_completion_alloc(
+    mca_btl_uct_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
+    mca_btl_base_registration_handle_t *local_handle, mca_btl_uct_device_context_t *dev_context,
+    mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
 /**
  * @brief release a callback structure
  */
-void mca_btl_uct_uct_completion_release (mca_btl_uct_uct_completion_t *comp);
-
-void mca_btl_uct_uct_completion (uct_completion_t *uct_comp, ucs_status_t status);
+void mca_btl_uct_uct_completion_release(mca_btl_uct_uct_completion_t *comp);
 
 /**
  * @brief unpack the registration key and ensure the endpoint is connected
@@ -40,37 +38,36 @@ void mca_btl_uct_uct_completion (uct_completion_t *uct_comp, ucs_status_t status
  * @param[inout] rkey          uct registration key bundle
  * @param[out]   ep_handle     uct endpoint handle
  */
-static inline int mca_btl_uct_get_rkey (mca_btl_uct_module_t *module,
-                                        mca_btl_uct_device_context_t *context,
-                                        mca_btl_base_endpoint_t *endpoint,
-					mca_btl_base_registration_handle_t *remote_handle,
-                                        uct_rkey_bundle_t *rkey,
-                                        uct_ep_h *ep_handle)
+static inline int mca_btl_uct_get_rkey(mca_btl_uct_module_t *module,
+                                       mca_btl_uct_device_context_t *context,
+                                       mca_btl_base_endpoint_t *endpoint,
+                                       mca_btl_base_registration_handle_t *remote_handle,
+                                       uct_rkey_bundle_t *rkey, uct_ep_h *ep_handle)
 {
     ucs_status_t ucs_status;
     int rc;
 
-    rc = mca_btl_uct_endpoint_check_rdma (module, endpoint, context, ep_handle);
+    rc = mca_btl_uct_endpoint_check_rdma(module, endpoint, context, ep_handle);
     if (OPAL_SUCCESS != rc) {
         return rc;
     }
 
-#if UCT_API >= UCT_VERSION(1, 7)
-    ucs_status = uct_rkey_unpack (module->uct_component, (void *) remote_handle, rkey);
-#else
-    ucs_status = uct_rkey_unpack ((void *) remote_handle, rkey);
-#endif
+#    if UCT_API >= UCT_VERSION(1, 7)
+    ucs_status = uct_rkey_unpack(module->uct_component, (void *) remote_handle, rkey);
+#    else
+    ucs_status = uct_rkey_unpack((void *) remote_handle, rkey);
+#    endif
     return (UCS_OK == ucs_status) ? OPAL_SUCCESS : OPAL_ERROR;
 }
 
-static inline void mca_btl_uct_rkey_release (mca_btl_uct_module_t *uct_btl, uct_rkey_bundle_t *rkey)
+static inline void mca_btl_uct_rkey_release(mca_btl_uct_module_t *uct_btl, uct_rkey_bundle_t *rkey)
 {
-#if UCT_API >= UCT_VERSION(1, 7)
-    uct_rkey_release (uct_btl->uct_component, rkey);
-#else
+#    if UCT_API >= UCT_VERSION(1, 7)
+    uct_rkey_release(uct_btl->uct_component, rkey);
+#    else
     (void) uct_btl;
-    uct_rkey_release (rkey);
-#endif
+    uct_rkey_release(rkey);
+#    endif
 }
 
 #endif /* !defined(BTL_UCT_RDMA_H) */
diff --git a/opal/mca/btl/uct/btl_uct_tl.c b/opal/mca/btl/uct/btl_uct_tl.c
index 502fbba7567..c1ef4c6d727 100644
--- a/opal/mca/btl/uct/btl_uct_tl.c
+++ b/opal/mca/btl/uct/btl_uct_tl.c
@@ -6,7 +6,7 @@
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2018      Triad National Security, LLC. All rights
  *                         reserved.
- * Copyright (c) 2019      Google, LLC. All rights reserved.
+ * Copyright (c) 2019-2025 Google, LLC. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -14,15 +14,15 @@
  * $HEADER$
  */
 
-#include "btl_uct_device_context.h"
 #include "btl_uct_am.h"
-#include "opal/util/bit_ops.h"
+#include "btl_uct_device_context.h"
 #include "opal/util/argv.h"
+#include "opal/util/bit_ops.h"
 
 #if HAVE_DECL_UCT_CB_FLAG_SYNC
-#define MCA_BTL_UCT_CB_FLAG_SYNC UCT_CB_FLAG_SYNC
+#    define MCA_BTL_UCT_CB_FLAG_SYNC UCT_CB_FLAG_SYNC
 #else
-#define MCA_BTL_UCT_CB_FLAG_SYNC 0
+#    define MCA_BTL_UCT_CB_FLAG_SYNC 0
 #endif
 
 /**
@@ -32,7 +32,7 @@ static uint64_t mca_btl_uct_cap_to_btl_flag[][2] = {
     {UCT_IFACE_FLAG_AM_SHORT, MCA_BTL_FLAGS_SEND},
     {UCT_IFACE_FLAG_PUT_ZCOPY, MCA_BTL_FLAGS_PUT},
     {UCT_IFACE_FLAG_GET_ZCOPY, MCA_BTL_FLAGS_GET},
-    {0,0},
+    {0, 0},
 };
 
 /**
@@ -42,11 +42,11 @@ static uint64_t mca_btl_uct_cap_to_btl_flag[][2] = {
  *
  * @returns equivalent BTL flags
  */
-static int32_t mca_btl_uct_module_flags (uint64_t cap_flags)
+static int32_t mca_btl_uct_module_flags(uint64_t cap_flags)
 {
     uint32_t flags = 0;
 
-    for (int i = 0 ; mca_btl_uct_cap_to_btl_flag[i][0] > 0 ; ++i) {
+    for (int i = 0; mca_btl_uct_cap_to_btl_flag[i][0] > 0; ++i) {
         if (cap_flags & mca_btl_uct_cap_to_btl_flag[i][0]) {
             flags |= (uint32_t) mca_btl_uct_cap_to_btl_flag[i][1];
         }
@@ -65,10 +65,12 @@ static uint64_t mca_btl_uct_cap_to_btl_atomic_flag[][2] = {
     {UCS_BIT(UCT_ATOMIC_OP_XOR), MCA_BTL_ATOMIC_SUPPORTS_XOR},
     {UCS_BIT(UCT_ATOMIC_OP_SWAP), MCA_BTL_ATOMIC_SUPPORTS_SWAP},
     {UCS_BIT(UCT_ATOMIC_OP_CSWAP), MCA_BTL_ATOMIC_SUPPORTS_CSWAP},
-    {0, },
+    {
+        0,
+    },
 };
 
-static void mca_btl_uct_module_set_atomic_flags (mca_btl_uct_module_t *module, mca_btl_uct_tl_t *tl)
+static void mca_btl_uct_module_set_atomic_flags(mca_btl_uct_module_t *module, mca_btl_uct_tl_t *tl)
 {
     uint64_t cap_flags = MCA_BTL_UCT_TL_ATTR(tl, 0).cap.flags;
 
@@ -76,16 +78,15 @@ static void mca_btl_uct_module_set_atomic_flags (mca_btl_uct_module_t *module, m
     uint64_t atomic_flags32 = MCA_BTL_UCT_TL_ATTR(tl, 0).cap.atomic32.fop_flags;
     uint64_t atomic_flags64 = MCA_BTL_UCT_TL_ATTR(tl, 0).cap.atomic64.fop_flags;
 
-    /* NTH: don't really have a way to seperate 32-bit and 64-bit right now */
-    uint64_t all_flags = atomic_flags32 & atomic_flags64;
-
-    module->super.btl_atomic_flags = 0;
+    uint64_t all_flags = atomic_flags64 | atomic_flags32;
 
+    module->super.btl_atomic_flags = (0 != atomic_flags32) ? MCA_BTL_ATOMIC_SUPPORTS_32BIT : 0;
+    
     if (cap_flags & UCT_IFACE_FLAG_ATOMIC_CPU) {
         module->super.btl_atomic_flags |= MCA_BTL_ATOMIC_SUPPORTS_GLOB;
     }
 
-    for (int i = 0 ; mca_btl_uct_cap_to_btl_atomic_flag[i][0] ; ++i) {
+    for (int i = 0; mca_btl_uct_cap_to_btl_atomic_flag[i][0]; ++i) {
         if (all_flags & mca_btl_uct_cap_to_btl_atomic_flag[i][0]) {
             module->super.btl_atomic_flags |= mca_btl_uct_cap_to_btl_atomic_flag[i][1];
         }
@@ -107,7 +108,9 @@ static uint64_t mca_btl_uct_cap_to_btl_atomic_flag[][2] = {
     {UCT_IFACE_FLAG_ATOMIC_CSWAP64, MCA_BTL_ATOMIC_SUPPORTS_CSWAP},
     {UCT_IFACE_FLAG_ATOMIC_SWAP64, MCA_BTL_ATOMIC_SUPPORTS_SWAP},
     {UCT_IFACE_FLAG_ATOMIC_CPU, MCA_BTL_ATOMIC_SUPPORTS_GLOB},
-    {0, },
+    {
+        0,
+    },
 };
 
 /**
@@ -117,13 +120,13 @@ static uint64_t mca_btl_uct_cap_to_btl_atomic_flag[][2] = {
  *
  * @returns equivalent BTL atomic flags
  */
-static void mca_btl_uct_module_set_atomic_flags (mca_btl_uct_module_t *module, mca_btl_uct_tl_t *tl)
+static void mca_btl_uct_module_set_atomic_flags(mca_btl_uct_module_t *module, mca_btl_uct_tl_t *tl)
 {
     uint64_t cap_flags = MCA_BTL_UCT_TL_ATTR(tl, 0).cap.flags;
 
     module->super.btl_atomic_flags = 0;
 
-    for (int i = 0 ; mca_btl_uct_cap_to_btl_atomic_flag[i][0] > 0 ; ++i) {
+    for (int i = 0; mca_btl_uct_cap_to_btl_atomic_flag[i][0] > 0; ++i) {
         if (cap_flags & mca_btl_uct_cap_to_btl_atomic_flag[i][0]) {
             module->super.btl_atomic_flags |= (uint32_t) mca_btl_uct_cap_to_btl_atomic_flag[i][1];
         }
@@ -137,19 +140,19 @@ static void mca_btl_uct_module_set_atomic_flags (mca_btl_uct_module_t *module, m
 
 #endif
 
-static void mca_btl_uct_tl_constructor (mca_btl_uct_tl_t *tl)
+static void mca_btl_uct_tl_constructor(mca_btl_uct_tl_t *tl)
 {
-    memset ((void *)((uintptr_t) tl + sizeof (tl->super)), 0, sizeof (*tl) - sizeof (tl->super));
+    memset((void *) ((uintptr_t) tl + sizeof(tl->super)), 0, sizeof(*tl) - sizeof(tl->super));
     OBJ_CONSTRUCT(&tl->tl_lock, opal_mutex_t);
 }
 
-static void mca_btl_uct_tl_destructor (mca_btl_uct_tl_t *tl)
+static void mca_btl_uct_tl_destructor(mca_btl_uct_tl_t *tl)
 {
-    assert (((opal_object_t *) tl)->obj_reference_count == 0);
+    assert(((opal_object_t *) tl)->obj_reference_count == 0);
 
-    for (int context_id = 0 ; context_id < MCA_BTL_UCT_MAX_WORKERS ; ++context_id) {
+    for (int context_id = 0; context_id < MCA_BTL_UCT_MAX_WORKERS; ++context_id) {
         if (NULL != tl->uct_dev_contexts[context_id]) {
-            mca_btl_uct_context_destroy (tl->uct_dev_contexts[context_id]);
+            mca_btl_uct_context_destroy(tl->uct_dev_contexts[context_id]);
         }
     }
 
@@ -157,58 +160,61 @@ static void mca_btl_uct_tl_destructor (mca_btl_uct_tl_t *tl)
         OBJ_RELEASE(tl->uct_md);
     }
 
-    free (tl->uct_dev_contexts);
-    free (tl->uct_tl_name);
-    free (tl->uct_dev_name);
+    free(tl->uct_dev_contexts);
+    free(tl->uct_tl_name);
+    free(tl->uct_dev_name);
 
     if (NULL != tl->uct_tl_config) {
-        uct_config_release (tl->uct_tl_config);
+        uct_config_release(tl->uct_tl_config);
     }
 
     OBJ_DESTRUCT(&tl->tl_lock);
 }
 
-OBJ_CLASS_INSTANCE(mca_btl_uct_tl_t, opal_list_item_t, mca_btl_uct_tl_constructor, mca_btl_uct_tl_destructor);
+OBJ_CLASS_INSTANCE(mca_btl_uct_tl_t, opal_list_item_t, mca_btl_uct_tl_constructor,
+                   mca_btl_uct_tl_destructor);
 
-static ucs_status_t mca_btl_uct_conn_req_cb (void *arg, void *data, size_t length, unsigned flags)
+static ucs_status_t mca_btl_uct_conn_req_cb(void *arg, void *data, size_t length, unsigned flags)
 {
     mca_btl_uct_module_t *module = (mca_btl_uct_module_t *) arg;
-    mca_btl_uct_pending_connection_request_t *request = calloc (1, length + sizeof (request->super));
+    mca_btl_uct_pending_connection_request_t *request = calloc(1, length + sizeof(request->super));
 
     /* it is not safe to process the connection request from the callback so just save it for
      * later processing */
     OBJ_CONSTRUCT(request, mca_btl_uct_pending_connection_request_t);
-    memcpy (&request->request_data, (void *) ((intptr_t) data + 8), length);
-    opal_fifo_push_atomic (&module->pending_connection_reqs, &request->super);
+    memcpy(&request->request_data, (void *) ((intptr_t) data + 8), length);
+    opal_fifo_push_atomic(&module->pending_connection_reqs, &request->super);
 
     return UCS_OK;
 }
 
 OBJ_CLASS_INSTANCE(mca_btl_uct_pending_connection_request_t, opal_list_item_t, NULL, NULL);
 
-int mca_btl_uct_process_connection_request (mca_btl_uct_module_t *module, mca_btl_uct_conn_req_t *req)
+int mca_btl_uct_process_connection_request(mca_btl_uct_module_t *module,
+                                           mca_btl_uct_conn_req_t *req)
 {
-    struct opal_proc_t *remote_proc = opal_proc_for_name (req->proc_name);
-    mca_btl_base_endpoint_t *endpoint = mca_btl_uct_get_ep (&module->super, remote_proc);
+    struct opal_proc_t *remote_proc = opal_proc_for_name(req->proc_name);
+    mca_btl_base_endpoint_t *endpoint = mca_btl_uct_get_ep(&module->super, remote_proc);
     mca_btl_uct_tl_endpoint_t *tl_endpoint = endpoint->uct_eps[req->context_id] + req->tl_index;
     int32_t ep_flags;
     int rc;
 
-    BTL_VERBOSE(("got connection request for endpoint %p. type = %d. context id = %d",
-                 (void *) endpoint, req->type, req->context_id));
-
     if (NULL == endpoint) {
         BTL_ERROR(("could not create endpoint for connection request"));
         return UCS_ERR_UNREACHABLE;
     }
 
-    assert (req->type < 2);
+    assert(req->type < 2);
 
-    ep_flags = opal_atomic_fetch_or_32 (&tl_endpoint->flags, MCA_BTL_UCT_ENDPOINT_FLAG_CONN_REC);
+    ep_flags = opal_atomic_fetch_or_32(&tl_endpoint->flags, MCA_BTL_UCT_ENDPOINT_FLAG_CONN_REC);
+
+    BTL_VERBOSE(("got connection request for endpoint %p. type = %d. context id = %d. ep_flags = %x",
+                 (void *) endpoint, req->type, req->context_id, ep_flags));
 
     if (!(ep_flags & MCA_BTL_UCT_ENDPOINT_FLAG_CONN_REC)) {
         /* create any necessary resources */
-        rc = mca_btl_uct_endpoint_connect (module, endpoint, req->context_id, req->ep_addr, req->tl_index);
+        rc = mca_btl_uct_endpoint_connect(module, endpoint, req->context_id, req->ep_addr,
+                                          req->tl_index);
         if (OPAL_SUCCESS != rc && OPAL_ERR_OUT_OF_RESOURCE != rc) {
             BTL_ERROR(("could not setup rdma endpoint. rc = %d", rc));
             return rc;
@@ -219,27 +225,20 @@ int mca_btl_uct_process_connection_request (mca_btl_uct_module_t *module, mca_bt
      * message. this might be overkill but there is little documentation at the UCT level on when
      * an endpoint can be used. */
     if (req->type == 1) {
-        /* remote side is ready */
-        mca_btl_uct_base_frag_t *frag;
-
+        /* remote side is connected */
         /* to avoid a race with send adding pending frags grab the lock here */
-        OPAL_THREAD_SCOPED_LOCK(&endpoint->ep_lock,{
-                BTL_VERBOSE(("connection ready. sending %" PRIsize_t " frags", opal_list_get_size (&module->pending_frags)));
-                (void) opal_atomic_or_fetch_32 (&tl_endpoint->flags, MCA_BTL_UCT_ENDPOINT_FLAG_CONN_READY);
-                opal_atomic_wmb ();
-
-                OPAL_LIST_FOREACH(frag, &module->pending_frags, mca_btl_uct_base_frag_t) {
-                    if (frag->context->context_id == req->context_id && endpoint == frag->endpoint) {
-                        frag->ready = true;
-                    }
-                }
-            });
+        OPAL_THREAD_SCOPED_LOCK(&endpoint->ep_lock, {
+            BTL_VERBOSE(("connection ready. sending %" PRIsize_t " frags",
+                         opal_list_get_size(&module->pending_frags)));
+            mca_btl_uct_endpoint_set_flag(module, endpoint, req->context_id, tl_endpoint,
+                                          MCA_BTL_UCT_ENDPOINT_FLAG_CONN_REM_READY);
+        });
     }
 
     return OPAL_SUCCESS;
 }
 
-static int mca_btl_uct_setup_connection_tl (mca_btl_uct_module_t *module)
+static int mca_btl_uct_setup_connection_tl(mca_btl_uct_module_t *module)
 {
     ucs_status_t ucs_status;
 
@@ -247,8 +246,9 @@ static int mca_btl_uct_setup_connection_tl (mca_btl_uct_module_t *module)
         return OPAL_ERR_NOT_SUPPORTED;
     }
 
-    ucs_status = uct_iface_set_am_handler (module->conn_tl->uct_dev_contexts[0]->uct_iface, MCA_BTL_UCT_CONNECT_RDMA,
-                                           mca_btl_uct_conn_req_cb, module, UCT_CB_FLAG_ASYNC);
+    ucs_status = uct_iface_set_am_handler(module->conn_tl->uct_dev_contexts[0]->uct_iface,
+                                          MCA_BTL_UCT_CONNECT_RDMA, mca_btl_uct_conn_req_cb, module,
+                                          UCT_CB_FLAG_ASYNC);
     if (UCS_OK != ucs_status) {
         BTL_ERROR(("could not set active message handler for uct tl"));
     }
@@ -256,30 +256,35 @@ static int mca_btl_uct_setup_connection_tl (mca_btl_uct_module_t *module)
     return UCS_OK == ucs_status ? OPAL_SUCCESS : OPAL_ERROR;
 }
 
-static void mca_btl_uct_context_enable_progress (mca_btl_uct_device_context_t *context)
+static void mca_btl_uct_context_enable_progress(mca_btl_uct_device_context_t *context)
 {
     if (!context->progress_enabled) {
 #if HAVE_DECL_UCT_PROGRESS_THREAD_SAFE
-        uct_iface_progress_enable (context->uct_iface, UCT_PROGRESS_THREAD_SAFE | UCT_PROGRESS_SEND |
-                                   UCT_PROGRESS_RECV);
+        uct_iface_progress_enable(context->uct_iface,
+                                  UCT_PROGRESS_THREAD_SAFE | UCT_PROGRESS_SEND | UCT_PROGRESS_RECV);
 #else
-        uct_iface_progress_enable (context->uct_iface, UCT_PROGRESS_SEND | UCT_PROGRESS_RECV);
+        uct_iface_progress_enable(context->uct_iface, UCT_PROGRESS_SEND | UCT_PROGRESS_RECV);
 #endif
         context->progress_enabled = true;
     }
 }
 
-mca_btl_uct_device_context_t *mca_btl_uct_context_create (mca_btl_uct_module_t *module, mca_btl_uct_tl_t *tl, int context_id, bool enable_progress)
+mca_btl_uct_device_context_t *mca_btl_uct_context_create(mca_btl_uct_module_t *module,
+                                                         mca_btl_uct_tl_t *tl, int context_id,
+                                                         bool enable_progress)
 {
 #if UCT_API >= UCT_VERSION(1, 6)
-    uct_iface_params_t iface_params = {.field_mask = UCT_IFACE_PARAM_FIELD_OPEN_MODE |
-                                                     UCT_IFACE_PARAM_FIELD_DEVICE,
+    uct_iface_params_t iface_params = {.field_mask = UCT_IFACE_PARAM_FIELD_OPEN_MODE
+                                                     | UCT_IFACE_PARAM_FIELD_DEVICE,
                                        .open_mode = UCT_IFACE_OPEN_MODE_DEVICE,
                                        .mode = {.device = {.tl_name = tl->uct_tl_name,
                                                            .dev_name = tl->uct_dev_name}}};
 #else
-    uct_iface_params_t iface_params = {.rndv_cb = NULL, .eager_cb = NULL, .stats_root = NULL,
-                                       .rx_headroom = 0, .open_mode = UCT_IFACE_OPEN_MODE_DEVICE,
+    uct_iface_params_t iface_params = {.rndv_cb = NULL,
+                                       .eager_cb = NULL,
+                                       .stats_root = NULL,
+                                       .rx_headroom = 0,
+                                       .open_mode = UCT_IFACE_OPEN_MODE_DEVICE,
                                        .mode = {.device = {.tl_name = tl->uct_tl_name,
                                                            .dev_name = tl->uct_dev_name}}};
 #endif
@@ -287,7 +292,7 @@ mca_btl_uct_device_context_t *mca_btl_uct_context_create (mca_btl_uct_module_t *
     ucs_status_t ucs_status;
     int rc;
 
-    context = calloc (1, sizeof (*context));
+    context = calloc(1, sizeof(*context));
     if (OPAL_UNLIKELY(NULL == context)) {
         return NULL;
     }
@@ -298,12 +303,11 @@ mca_btl_uct_device_context_t *mca_btl_uct_context_create (mca_btl_uct_module_t *
     OBJ_CONSTRUCT(&context->mutex, opal_recursive_mutex_t);
     OBJ_CONSTRUCT(&context->rdma_completions, opal_free_list_t);
 
-    rc = opal_free_list_init (&context->rdma_completions, sizeof (mca_btl_uct_uct_completion_t),
-                              opal_cache_line_size, OBJ_CLASS(mca_btl_uct_uct_completion_t),
-                              0, opal_cache_line_size, 0, 4096, 128, NULL, 0, NULL, NULL,
-                              NULL);
+    rc = opal_free_list_init(&context->rdma_completions, sizeof(mca_btl_uct_uct_completion_t),
+                             opal_cache_line_size, OBJ_CLASS(mca_btl_uct_uct_completion_t), 0,
+                             opal_cache_line_size, 0, 4096, 128, NULL, 0, NULL, NULL, NULL);
     if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-        mca_btl_uct_context_destroy (context);
+        mca_btl_uct_context_destroy(context);
         return NULL;
     }
 
@@ -311,61 +315,61 @@ mca_btl_uct_device_context_t *mca_btl_uct_context_create (mca_btl_uct_module_t *
      * use our own locks just go ahead and use UCS_THREAD_MODE_SINGLE. if they ever fix their
      * api then change this back to UCS_THREAD_MODE_MULTI and remove the locks around the
      * various UCT calls. */
-    ucs_status = uct_worker_create (module->ucs_async, UCS_THREAD_MODE_SINGLE, &context->uct_worker);
+    ucs_status = uct_worker_create(module->ucs_async, UCS_THREAD_MODE_SINGLE, &context->uct_worker);
     if (OPAL_UNLIKELY(UCS_OK != ucs_status)) {
         BTL_VERBOSE(("could not create a UCT worker"));
-        mca_btl_uct_context_destroy (context);
+        mca_btl_uct_context_destroy(context);
         return NULL;
     }
 
-    ucs_status = uct_iface_open (tl->uct_md->uct_md, context->uct_worker, &iface_params,
-                                 tl->uct_tl_config, &context->uct_iface);
+    ucs_status = uct_iface_open(tl->uct_md->uct_md, context->uct_worker, &iface_params,
+                                tl->uct_tl_config, &context->uct_iface);
     if (OPAL_UNLIKELY(UCS_OK != ucs_status)) {
         BTL_VERBOSE(("could not open UCT interface. error code: %d", ucs_status));
-        mca_btl_uct_context_destroy (context);
+        mca_btl_uct_context_destroy(context);
         return NULL;
     }
 
     /* only need to query one of the interfaces to get the attributes */
-    ucs_status = uct_iface_query (context->uct_iface, &context->uct_iface_attr);
+    ucs_status = uct_iface_query(context->uct_iface, &context->uct_iface_attr);
     if (UCS_OK != ucs_status) {
         BTL_VERBOSE(("Error querying UCT interface"));
-        mca_btl_uct_context_destroy (context);
+        mca_btl_uct_context_destroy(context);
         return NULL;
     }
 
     if (context_id > 0 && tl == module->am_tl) {
-      BTL_VERBOSE(("installing AM handler for tl %p context id %d", (void *) tl, context_id));
-      uct_iface_set_am_handler (context->uct_iface, MCA_BTL_UCT_FRAG, mca_btl_uct_am_handler,
-				context, MCA_BTL_UCT_CB_FLAG_SYNC);
+        BTL_VERBOSE(("installing AM handler for tl %p context id %d", (void *) tl, context_id));
+        uct_iface_set_am_handler(context->uct_iface, MCA_BTL_UCT_FRAG, mca_btl_uct_am_handler,
+                                 context, MCA_BTL_UCT_CB_FLAG_SYNC);
     }
 
     if (enable_progress) {
         BTL_VERBOSE(("enabling progress for tl %p context id %d", (void *) tl, context_id));
-        mca_btl_uct_context_enable_progress (context);
+        mca_btl_uct_context_enable_progress(context);
     }
 
     return context;
 }
 
-void mca_btl_uct_context_destroy (mca_btl_uct_device_context_t *context)
+void mca_btl_uct_context_destroy(mca_btl_uct_device_context_t *context)
 {
     if (context->uct_iface) {
-        uct_iface_close (context->uct_iface);
+        uct_iface_close(context->uct_iface);
         context->uct_iface = NULL;
     }
 
     if (context->uct_worker) {
-        uct_worker_destroy (context->uct_worker);
+        uct_worker_destroy(context->uct_worker);
         context->uct_worker = NULL;
     }
 
     OBJ_DESTRUCT(&context->completion_fifo);
     OBJ_DESTRUCT(&context->rdma_completions);
-    free (context);
+    free(context);
 }
 
-static int tl_compare (opal_list_item_t **a, opal_list_item_t **b)
+static int tl_compare(opal_list_item_t **a, opal_list_item_t **b)
 {
     mca_btl_uct_tl_t *tl_a = (mca_btl_uct_tl_t *) *a;
     mca_btl_uct_tl_t *tl_b = (mca_btl_uct_tl_t *) *b;
@@ -373,7 +377,8 @@ static int tl_compare (opal_list_item_t **a, opal_list_item_t **b)
     return tl_a->priority - tl_b->priority;
 }
 
-static mca_btl_uct_tl_t *mca_btl_uct_create_tl (mca_btl_uct_module_t *module, mca_btl_uct_md_t *md, uct_tl_resource_desc_t *tl_desc, int priority)
+static mca_btl_uct_tl_t *mca_btl_uct_create_tl(mca_btl_uct_module_t *module, mca_btl_uct_md_t *md,
+                                               uct_tl_resource_desc_t *tl_desc, int priority)
 {
     mca_btl_uct_tl_t *tl = OBJ_NEW(mca_btl_uct_tl_t);
 
@@ -385,20 +390,20 @@ static mca_btl_uct_tl_t *mca_btl_uct_create_tl (mca_btl_uct_module_t *module, mc
     tl->uct_md = md;
     OBJ_RETAIN(md);
 
-    tl->uct_tl_name = strdup (tl_desc->tl_name);
-    tl->uct_dev_name = strdup (tl_desc->dev_name);
+    tl->uct_tl_name = strdup(tl_desc->tl_name);
+    tl->uct_dev_name = strdup(tl_desc->dev_name);
     tl->priority = priority;
 
-    tl->uct_dev_contexts = calloc (MCA_BTL_UCT_MAX_WORKERS, sizeof (tl->uct_dev_contexts[0]));
+    tl->uct_dev_contexts = calloc(MCA_BTL_UCT_MAX_WORKERS, sizeof(tl->uct_dev_contexts[0]));
     if (NULL == tl->uct_dev_contexts) {
         OBJ_RELEASE(tl);
         return NULL;
     }
 
-    (void) uct_md_iface_config_read (md->uct_md, tl_desc->tl_name, NULL, NULL, &tl->uct_tl_config);
+    (void) uct_md_iface_config_read(md->uct_md, tl_desc->tl_name, NULL, NULL, &tl->uct_tl_config);
 
     /* always create a 0 context (needed to query) */
-    tl->uct_dev_contexts[0] = mca_btl_uct_context_create (module, tl, 0, false);
+    tl->uct_dev_contexts[0] = mca_btl_uct_context_create(module, tl, 0, false);
     if (NULL == tl->uct_dev_contexts[0]) {
         BTL_VERBOSE(("could not create a uct device context"));
         OBJ_RELEASE(tl);
@@ -411,26 +416,30 @@ static mca_btl_uct_tl_t *mca_btl_uct_create_tl (mca_btl_uct_module_t *module, mc
     return tl;
 }
 
-static void mca_btl_uct_set_tl_rdma (mca_btl_uct_module_t *module, mca_btl_uct_tl_t *tl)
+static void mca_btl_uct_set_tl_rdma(mca_btl_uct_module_t *module, mca_btl_uct_tl_t *tl)
 {
     BTL_VERBOSE(("tl %s is suitable for RDMA", tl->uct_tl_name));
 
-    mca_btl_uct_module_set_atomic_flags (module, tl);
+    mca_btl_uct_module_set_atomic_flags(module, tl);
 
     module->super.btl_get_limit = MCA_BTL_UCT_TL_ATTR(tl, 0).cap.get.max_zcopy;
     if (MCA_BTL_UCT_TL_ATTR(tl, 0).cap.get.max_bcopy) {
         module->super.btl_get_alignment = 0;
-        module->super.btl_get_local_registration_threshold = MCA_BTL_UCT_TL_ATTR(tl, 0).cap.get.max_bcopy;
+        module->super.btl_get_local_registration_threshold = MCA_BTL_UCT_TL_ATTR(tl, 0)
+                                                                 .cap.get.max_bcopy;
     } else {
-        /* this is overkill in terms of alignment but we have no way to enforce a minimum get size */
-        module->super.btl_get_alignment = opal_next_poweroftwo_inclusive (MCA_BTL_UCT_TL_ATTR(tl, 0).cap.get.min_zcopy);
+        /* this is overkill in terms of alignment but we have no way to enforce a minimum get size
+         */
+        module->super.btl_get_alignment = opal_next_poweroftwo_inclusive(
+            MCA_BTL_UCT_TL_ATTR(tl, 0).cap.get.min_zcopy);
     }
 
     module->super.btl_put_limit = MCA_BTL_UCT_TL_ATTR(tl, 0).cap.put.max_zcopy;
     module->super.btl_put_alignment = 0;
 
     /* no registration needed when using short/bcopy put */
-    module->super.btl_put_local_registration_threshold = MCA_BTL_UCT_TL_ATTR(tl, 0).cap.put.max_bcopy;
+    module->super.btl_put_local_registration_threshold = MCA_BTL_UCT_TL_ATTR(tl, 0)
+                                                             .cap.put.max_bcopy;
 
     module->rdma_tl = tl;
     OBJ_RETAIN(tl);
@@ -438,47 +447,47 @@ static void mca_btl_uct_set_tl_rdma (mca_btl_uct_module_t *module, mca_btl_uct_t
     tl->tl_index = (module->am_tl && tl != module->am_tl) ? 1 : 0;
     module->comm_tls[tl->tl_index] = tl;
     if (tl->max_device_contexts <= 1) {
-	tl->max_device_contexts = mca_btl_uct_component.num_contexts_per_module;
+        tl->max_device_contexts = mca_btl_uct_component.num_contexts_per_module;
     }
 }
 
-static void mca_btl_uct_set_tl_am (mca_btl_uct_module_t *module, mca_btl_uct_tl_t *tl)
+static void mca_btl_uct_set_tl_am(mca_btl_uct_module_t *module, mca_btl_uct_tl_t *tl)
 {
     BTL_VERBOSE(("tl %s is suitable for active-messaging", tl->uct_tl_name));
 
     if (module->rdma_tl == tl) {
-	module->shared_endpoints = true;
+        module->shared_endpoints = true;
     }
     module->am_tl = tl;
     OBJ_RETAIN(tl);
 
-    uct_iface_set_am_handler (tl->uct_dev_contexts[0]->uct_iface, MCA_BTL_UCT_FRAG,
-                              mca_btl_uct_am_handler, tl->uct_dev_contexts[0], UCT_CB_FLAG_ASYNC);
+    uct_iface_set_am_handler(tl->uct_dev_contexts[0]->uct_iface, MCA_BTL_UCT_FRAG,
+                             mca_btl_uct_am_handler, tl->uct_dev_contexts[0], UCT_CB_FLAG_ASYNC);
 
     tl->tl_index = (module->rdma_tl && tl != module->rdma_tl) ? 1 : 0;
     module->comm_tls[tl->tl_index] = tl;
     if (tl->max_device_contexts <= 1) {
-	tl->max_device_contexts = mca_btl_uct_component.num_contexts_per_module;
+        tl->max_device_contexts = mca_btl_uct_component.num_contexts_per_module;
     }
 
-    module->super.btl_eager_limit = MCA_BTL_UCT_TL_ATTR(tl, 0).cap.am.max_bcopy -
-        sizeof (mca_btl_uct_am_header_t);
+    module->super.btl_eager_limit = MCA_BTL_UCT_TL_ATTR(tl, 0).cap.am.max_bcopy
+                                    - sizeof(mca_btl_uct_am_header_t);
     if (MCA_BTL_UCT_TL_ATTR(tl, 0).cap.flags & UCT_IFACE_FLAG_AM_ZCOPY) {
-        module->super.btl_max_send_size = MCA_BTL_UCT_TL_ATTR(tl, 0).cap.am.max_zcopy -
-            sizeof (mca_btl_uct_am_header_t);
+        module->super.btl_max_send_size = MCA_BTL_UCT_TL_ATTR(tl, 0).cap.am.max_zcopy
+                                          - sizeof(mca_btl_uct_am_header_t);
     } else {
         module->super.btl_max_send_size = module->super.btl_eager_limit;
     }
 }
 
-static int mca_btl_uct_set_tl_conn (mca_btl_uct_module_t *module, mca_btl_uct_tl_t *tl)
+static int mca_btl_uct_set_tl_conn(mca_btl_uct_module_t *module, mca_btl_uct_tl_t *tl)
 {
     int rc;
 
     BTL_VERBOSE(("tl %s is suitable for making connections", tl->uct_tl_name));
 
     module->conn_tl = tl;
-    rc = mca_btl_uct_setup_connection_tl (module);
+    rc = mca_btl_uct_setup_connection_tl(module);
     if (OPAL_SUCCESS != rc) {
         return rc;
     }
@@ -486,29 +495,29 @@ static int mca_btl_uct_set_tl_conn (mca_btl_uct_module_t *module, mca_btl_uct_tl
     OBJ_RETAIN(tl);
 
     if (!tl->max_device_contexts) {
-	/* if a tl is only being used to create connections do not bother with multiple
-	 * contexts */
-	tl->max_device_contexts = 1;
+        /* if a tl is only being used to create connections do not bother with multiple
+         * contexts */
+        tl->max_device_contexts = 1;
     }
 
     return OPAL_SUCCESS;
 }
 
-static int mca_btl_uct_evaluate_tl (mca_btl_uct_module_t *module, mca_btl_uct_tl_t *tl)
+static int mca_btl_uct_evaluate_tl(mca_btl_uct_module_t *module, mca_btl_uct_tl_t *tl)
 {
     int rc;
 
     BTL_VERBOSE(("evaluating tl %s", tl->uct_tl_name));
-    if (NULL == module->rdma_tl && mca_btl_uct_tl_supports_rdma (tl)) {
-	mca_btl_uct_set_tl_rdma (module, tl);
+    if (NULL == module->rdma_tl && mca_btl_uct_tl_supports_rdma(tl)) {
+        mca_btl_uct_set_tl_rdma(module, tl);
     }
 
-    if (NULL == module->am_tl && mca_btl_uct_tl_support_am (tl)) {
-	mca_btl_uct_set_tl_am (module, tl);
+    if (NULL == module->am_tl && mca_btl_uct_tl_support_am(tl)) {
+        mca_btl_uct_set_tl_am(module, tl);
     }
 
-    if (NULL == module->conn_tl && mca_btl_uct_tl_supports_conn (tl)) {
-	rc = mca_btl_uct_set_tl_conn (module, tl);
+    if (NULL == module->conn_tl && mca_btl_uct_tl_supports_conn(tl)) {
+        rc = mca_btl_uct_set_tl_conn(module, tl);
         if (OPAL_SUCCESS != rc) {
             return rc;
         }
@@ -516,32 +525,35 @@ static int mca_btl_uct_evaluate_tl (mca_btl_uct_module_t *module, mca_btl_uct_tl
 
     if (tl == module->rdma_tl || tl == module->am_tl) {
         BTL_VERBOSE(("tl has flags 0x%" PRIx64, MCA_BTL_UCT_TL_ATTR(tl, 0).cap.flags));
-        module->super.btl_flags |= mca_btl_uct_module_flags (MCA_BTL_UCT_TL_ATTR(tl, 0).cap.flags);
+        module->super.btl_flags |= mca_btl_uct_module_flags(MCA_BTL_UCT_TL_ATTR(tl, 0).cap.flags);
 
-	/* the bandwidth and latency numbers relate to both rdma and active messages. need to
-	 * come up with a better estimate. */
+        /* the bandwidth and latency numbers relate to both rdma and active messages. need to
+         * come up with a better estimate. */
 
-	/* UCT bandwidth is in bytes/sec, BTL is in MB/sec */
+        /* UCT bandwidth is in bytes/sec, BTL is in MB/sec */
 #if UCT_API >= UCT_VERSION(1, 7)
-	module->super.btl_bandwidth = (uint32_t) ((MCA_BTL_UCT_TL_ATTR(tl, 0).bandwidth.dedicated +
-                                                   MCA_BTL_UCT_TL_ATTR(tl, 0).bandwidth.shared /
-                                                   (opal_process_info.num_local_peers + 1)) / 1048576.0);
+        module->super.btl_bandwidth = (uint32_t)((MCA_BTL_UCT_TL_ATTR(tl, 0).bandwidth.dedicated
+                                                  + MCA_BTL_UCT_TL_ATTR(tl, 0).bandwidth.shared
+                                                        / (opal_process_info.num_local_peers + 1))
+                                                 / 1048576.0);
 #else
-	module->super.btl_bandwidth = (uint32_t) (MCA_BTL_UCT_TL_ATTR(tl, 0).bandwidth / 1048576.0);
+        module->super.btl_bandwidth = (uint32_t)(MCA_BTL_UCT_TL_ATTR(tl, 0).bandwidth / 1048576.0);
 #endif
-	/* TODO -- figure out how to translate UCT latency to us */
-	module->super.btl_latency = 1;
+        /* TODO -- figure out how to translate UCT latency to us */
+        module->super.btl_latency = 1;
     }
 
     if (tl == module->rdma_tl || tl == module->am_tl || tl == module->conn_tl) {
-        /* make sure progress is enabled on the default context now that we know this TL will be used */
-        mca_btl_uct_context_enable_progress (tl->uct_dev_contexts[0]);
+        /* make sure progress is enabled on the default context now that we know this TL will be
+         * used */
+        mca_btl_uct_context_enable_progress(tl->uct_dev_contexts[0]);
     }
 
     return OPAL_SUCCESS;
 }
 
-int mca_btl_uct_query_tls (mca_btl_uct_module_t *module, mca_btl_uct_md_t *md, uct_tl_resource_desc_t *tl_descs, unsigned tl_count)
+int mca_btl_uct_query_tls(mca_btl_uct_module_t *module, mca_btl_uct_md_t *md,
+                          uct_tl_resource_desc_t *tl_descs, unsigned tl_count)
 {
     bool include = true, any = false;
     mca_btl_uct_tl_t *tl;
@@ -551,20 +563,20 @@ int mca_btl_uct_query_tls (mca_btl_uct_module_t *module, mca_btl_uct_md_t *md, u
 
     OBJ_CONSTRUCT(&tl_list, opal_list_t);
 
-    tl_filter = opal_argv_split (mca_btl_uct_component.allowed_transports, ',');
+    tl_filter = opal_argv_split(mca_btl_uct_component.allowed_transports, ',');
 
     if ('^' == tl_filter[0][0]) {
-	/* user has negated the include list */
-	char *tmp = strdup (tl_filter[0] + 1);
+        /* user has negated the include list */
+        char *tmp = strdup(tl_filter[0] + 1);
 
-	free (tl_filter[0]);
-	tl_filter[0] = tmp;
-	include = false;
+        free(tl_filter[0]);
+        tl_filter[0] = tmp;
+        include = false;
     }
 
     /* check for the any keyword */
-    for (unsigned j = 0 ; tl_filter[j] ; ++j) {
-        if (0 == strcmp (tl_filter[j], "any")) {
+    for (unsigned j = 0; tl_filter[j]; ++j) {
+        if (0 == strcmp(tl_filter[j], "any")) {
             any_priority = j;
             any = true;
             break;
@@ -572,90 +584,92 @@ int mca_btl_uct_query_tls (mca_btl_uct_module_t *module, mca_btl_uct_md_t *md, u
     }
 
     if (any && !include) {
-        opal_argv_free (tl_filter);
+        opal_argv_free(tl_filter);
         return OPAL_ERR_NOT_AVAILABLE;
     }
 
-    for (unsigned i = 0 ; i < tl_count ; ++i) {
-	bool try_tl = any;
-	int priority = any_priority;
+    for (unsigned i = 0; i < tl_count; ++i) {
+        bool try_tl = any;
+        int priority = any_priority;
 
-	for (unsigned j = 0 ; tl_filter[j] ; ++j) {
-            if (0 == strcmp (tl_filter[j], tl_descs[i].tl_name)) {
+        for (unsigned j = 0; tl_filter[j]; ++j) {
+            if (0 == strcmp(tl_filter[j], tl_descs[i].tl_name)) {
                 try_tl = include;
                 priority = j;
                 break;
             }
-	}
+        }
 
-        BTL_VERBOSE(("tl filter: tl_name = %s, use = %d, priority = %d", tl_descs[i].tl_name, try_tl, priority));
+        BTL_VERBOSE(("tl filter: tl_name = %s, use = %d, priority = %d", tl_descs[i].tl_name,
+                     try_tl, priority));
 
-	if (!try_tl) {
-	    continue;
-	}
+        if (!try_tl) {
+            continue;
+        }
 
-        if (0 == strcmp (tl_descs[i].tl_name, "ud")) {
+        if (0 == strcmp(tl_descs[i].tl_name, "ud")) {
             /* ud looks like any normal transport but we do not want to use it for anything other
              * than connection management so ensure it gets evaluated last */
             priority = INT_MAX;
         }
 
-	tl = mca_btl_uct_create_tl (module, md, tl_descs + i, priority);
+        tl = mca_btl_uct_create_tl(module, md, tl_descs + i, priority);
 
-	if (tl) {
-	    opal_list_append (&tl_list, &tl->super);
-	}
+        if (tl) {
+            opal_list_append(&tl_list, &tl->super);
+        }
     }
 
-    opal_argv_free (tl_filter);
+    opal_argv_free(tl_filter);
 
-    if (0 == opal_list_get_size (&tl_list)) {
-	BTL_VERBOSE(("no suitable tls match filter: %s", mca_btl_uct_component.allowed_transports));
-	OBJ_DESTRUCT(&tl_list);
-	return OPAL_ERR_NOT_AVAILABLE;
+    if (0 == opal_list_get_size(&tl_list)) {
+        BTL_VERBOSE(("no suitable tls match filter: %s", mca_btl_uct_component.allowed_transports));
+        OBJ_DESTRUCT(&tl_list);
+        return OPAL_ERR_NOT_AVAILABLE;
     }
 
-    opal_list_sort (&tl_list, tl_compare);
+    opal_list_sort(&tl_list, tl_compare);
 
-    OPAL_LIST_FOREACH(tl, &tl_list, mca_btl_uct_tl_t) {
-	mca_btl_uct_evaluate_tl (module, tl);
-	if (NULL != module->am_tl && NULL != module->rdma_tl &&
-	    (NULL != module->conn_tl || !(mca_btl_uct_tl_requires_connection_tl (module->am_tl) ||
-					  mca_btl_uct_tl_requires_connection_tl (module->rdma_tl)))) {
-	    /* all done */
-	    break;
-	}
+    OPAL_LIST_FOREACH (tl, &tl_list, mca_btl_uct_tl_t) {
+        mca_btl_uct_evaluate_tl(module, tl);
+        if (NULL != module->am_tl && NULL != module->rdma_tl
+            && (NULL != module->conn_tl
+                || !(mca_btl_uct_tl_requires_connection_tl(module->am_tl)
+                     || mca_btl_uct_tl_requires_connection_tl(module->rdma_tl)))) {
+            /* all done */
+            break;
+        }
     }
 
     if (NULL == module->rdma_tl) {
-	/* no rdma tls */
-	BTL_VERBOSE(("no rdma tl matched supplied filter. disabling RDMA support"));
+        /* no rdma tls */
+        BTL_VERBOSE(("no rdma tl matched supplied filter. disabling RDMA support"));
 
         module->super.btl_flags &= ~MCA_BTL_FLAGS_RDMA;
-	module->super.btl_put = NULL;
-	module->super.btl_get = NULL;
-	module->super.btl_atomic_fop = NULL;
-	module->super.btl_atomic_op = NULL;
+        module->super.btl_put = NULL;
+        module->super.btl_get = NULL;
+        module->super.btl_atomic_fop = NULL;
+        module->super.btl_atomic_op = NULL;
     }
 
     if (NULL == module->am_tl) {
-	/* no active message tls == no send/recv */
-	BTL_VERBOSE(("no active message tl matched supplied filter. disabling send/recv support"));
+        /* no active message tls == no send/recv */
+        BTL_VERBOSE(("no active message tl matched supplied filter. disabling send/recv support"));
 
-	module->super.btl_send = NULL;
-	module->super.btl_sendi = NULL;
-	module->super.btl_alloc = NULL;
-	module->super.btl_free = NULL;
+        module->super.btl_send = NULL;
+        module->super.btl_sendi = NULL;
+        module->super.btl_alloc = NULL;
+        module->super.btl_free = NULL;
     }
 
     OPAL_LIST_DESTRUCT(&tl_list);
 
-    if (!(NULL != module->am_tl && mca_btl_uct_tl_requires_connection_tl (module->am_tl)) &&
-	!(NULL != module->rdma_tl && mca_btl_uct_tl_requires_connection_tl (module->rdma_tl)) &&
-	module->conn_tl) {
-	/* no connection tl needed for selected transports */
-	OBJ_RELEASE(module->conn_tl);
-	module->conn_tl = NULL;
+    if (!(NULL != module->am_tl && mca_btl_uct_tl_requires_connection_tl(module->am_tl))
+        && !(NULL != module->rdma_tl && mca_btl_uct_tl_requires_connection_tl(module->rdma_tl))
+        && module->conn_tl) {
+        /* no connection tl needed for selected transports */
+        OBJ_RELEASE(module->conn_tl);
+        module->conn_tl = NULL;
     } else if (NULL == module->conn_tl) {
         BTL_VERBOSE(("a connection tl is required but no tls match the filter %s",
                      mca_btl_uct_component.allowed_transports));
diff --git a/opal/mca/btl/uct/btl_uct_types.h b/opal/mca/btl/uct/btl_uct_types.h
index 4b0df21e2a7..b2bac61be61 100644
--- a/opal/mca/btl/uct/btl_uct_types.h
+++ b/opal/mca/btl/uct/btl_uct_types.h
@@ -2,6 +2,7 @@
 /*
  * Copyright (c) 2018      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2025      Google, LLC. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -10,9 +11,11 @@
  */
 
 #if !defined(BTL_UCT_TYPES_H)
-#define BTL_UCT_TYPES_H
+#    define BTL_UCT_TYPES_H
 
-#include "opal/mca/btl/btl.h"
+#    include "opal/mca/btl/btl.h"
+
+#include "opal/mca/timer/base/base.h"
 
 /* forward declarations */
 struct mca_btl_uct_module_t;
@@ -21,22 +24,24 @@ struct mca_btl_uct_base_frag_t;
 
 /* TL endpoint flags */
 /** connection data was received */
-#define MCA_BTL_UCT_ENDPOINT_FLAG_CONN_REC       0x1
+#    define MCA_BTL_UCT_ENDPOINT_FLAG_CONN_REC 0x1
 /** remote endpoint read */
-#define MCA_BTL_UCT_ENDPOINT_FLAG_CONN_REM_READY 0x2
+#    define MCA_BTL_UCT_ENDPOINT_FLAG_CONN_REM_READY 0x2
+/** local UCT endpoint connected */
+#    define MCA_BTL_UCT_ENDPOINT_FLAG_EP_CONNECTED 0x4
 /** connection was established */
-#define MCA_BTL_UCT_ENDPOINT_FLAG_CONN_READY     0x4
+#    define MCA_BTL_UCT_ENDPOINT_FLAG_CONN_READY 0x8
 
 /* AM tags */
 /** BTL fragment */
-#define MCA_BTL_UCT_FRAG         0x0d
+#    define MCA_BTL_UCT_FRAG 0x0d
 /** connection request */
-#define MCA_BTL_UCT_CONNECT_RDMA 0x0e
+#    define MCA_BTL_UCT_CONNECT_RDMA 0x0e
 
 /** maximum number of modules supported by the btl component */
-#define MCA_BTL_UCT_MAX_MODULES 16
+#    define MCA_BTL_UCT_MAX_MODULES 16
 /** maximum number of UCT workers */
-#define MCA_BTL_UCT_MAX_WORKERS 64
+#    define MCA_BTL_UCT_MAX_WORKERS 64
 
 /**
  * @brief MODEx data
@@ -69,7 +74,6 @@ typedef struct mca_btl_uct_md_t mca_btl_uct_md_t;
 
 OBJ_CLASS_DECLARATION(mca_btl_uct_md_t);
 
-
 /**
  * @brief Connection request structure
  */
@@ -93,7 +97,7 @@ struct mca_btl_uct_conn_req_t {
 typedef struct mca_btl_uct_conn_req_t mca_btl_uct_conn_req_t;
 
 /**
- * @brief Transport endpoint stucture
+ * @brief Transport endpoint structure
  */
 struct mca_btl_uct_tl_endpoint_t {
     /** current flags (connected, requested, etc) */
@@ -101,6 +105,9 @@ struct mca_btl_uct_tl_endpoint_t {
 
     /** UCT endpoint handle */
     uct_ep_h uct_ep;
+
+    /** Time of last connection message. */
+    opal_timer_t last_connection_req;
 };
 
 typedef struct mca_btl_uct_tl_endpoint_t mca_btl_uct_tl_endpoint_t;
@@ -169,11 +176,11 @@ typedef struct mca_btl_uct_device_context_t mca_btl_uct_device_context_t;
 union mca_btl_uct_am_header_t {
     /** active message header data */
     struct mca_btl_uct_am_header_data_t {
-	/** callback tag */
+        /** callback tag */
         mca_btl_base_tag_t tag;
 
-	/** padding */
-        uint8_t            padding[7];
+        /** padding */
+        uint8_t padding[7];
     } data;
 
     /** header value. this is 64-bits to support using this with uct_ep_am_short */
@@ -316,7 +323,7 @@ struct mca_btl_uct_tl_t {
     /** device name for this tl (used for creating device contexts) */
     char *uct_dev_name;
 
-    /** maxiumum number of device contexts that can be created */
+    /** maximum number of device contexts that can be created */
     int max_device_contexts;
 
     /** array of device contexts */
@@ -330,7 +337,7 @@ struct mca_btl_uct_tl_t {
 typedef struct mca_btl_uct_tl_t mca_btl_uct_tl_t;
 OBJ_CLASS_DECLARATION(mca_btl_uct_tl_t);
 
-#define MCA_BTL_UCT_TL_ATTR(tl, context_id) (tl)->uct_dev_contexts[(context_id)]->uct_iface_attr
+#    define MCA_BTL_UCT_TL_ATTR(tl, context_id) (tl)->uct_dev_contexts[(context_id)]->uct_iface_attr
 
 struct mca_btl_uct_pending_connection_request_t {
     opal_list_item_t super;
diff --git a/opal/mca/btl/uct/configure.m4 b/opal/mca/btl/uct/configure.m4
index 5e7ad3838d9..ac444db251b 100644
--- a/opal/mca/btl/uct/configure.m4
+++ b/opal/mca/btl/uct/configure.m4
@@ -51,22 +51,23 @@ dnl
         AC_MSG_RESULT([no])
     fi
 
-    max_allowed_uct_major=1
-    max_allowed_uct_minor=9
+    min_allowed_uct_major=1
+    min_allowed_uct_minor=9
     if test "$btl_uct_happy" = "yes" && test "$enable_uct_version_check" != "no"; then
         AC_MSG_CHECKING([UCT version compatibility])
         OPAL_VAR_SCOPE_PUSH([CPPFLAGS_save])
         CPPFLAGS_save="$CPPFLAGS"
         CPPFLAGS="$CPPFLAGS $btl_uct_CPPFLAGS"
         AC_PREPROC_IFELSE([AC_LANG_PROGRAM([#include <uct/api/version.h>
-                                            #if (UCT_VERNO_MAJOR > $max_allowed_uct_major)
-                                            #error "UCT MAJOR VERNO > $max_allowed_uct_major"
+                                            #if (UCT_VERNO_MAJOR < $min_allowed_uct_major)
+                                            #error "UCT MAJOR VERNO < $min_allowed_uct_major"
                                             #endif
-                                            #if (UCT_VERNO_MINOR > $max_allowed_uct_minor)
-                                            #error "UCT MINOR VERNO > $max_allowed_uct_minor"
+                                            #if (UCT_VERNO_MAJOR == $min_allowed_uct_major) &&\
+                                                (UCT_VERNO_MINOR <  $min_allowed_uct_minor)
+                                            #error "UCT MINOR VERNO < $min_allowed_uct_minor"
                                             #endif], [])],
                            [AC_MSG_RESULT([UCT version compatible])],
-                           [AC_MSG_RESULT([UCT version not compatible - need UCX $max_allowed_uct_major.$max_allowed_uct_minor or older])
+                           [AC_MSG_RESULT([UCT version not compatible - need UCX $min_allowed_uct_major.$min_allowed_uct_minor or newer])
                             btl_uct_happy="no"])
         CPPFLAGS="$CPPFLAGS_save"
         OPAL_VAR_SCOPE_POP
diff --git a/opal/mca/btl/ugni/Makefile.am b/opal/mca/btl/ugni/Makefile.am
deleted file mode 100644
index 958e12aec58..00000000000
--- a/opal/mca/btl/ugni/Makefile.am
+++ /dev/null
@@ -1,59 +0,0 @@
-# -*- indent-tabs-mode:nil -*-
-#
-# Copyright (c) 2011-2017 Los Alamos National Security, LLC. All rights
-#                         reserved.
-# Copyright (c) 2011      UT-Battelle, LLC. All rights reserved.
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-# Make the output library in this directory, and name it either
-# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
-# (for static builds).
-
-AM_CPPFLAGS = $(btl_ugni_CPPFLAGS)
-
-if MCA_BUILD_opal_btl_ugni_DSO
-component_noinst =
-component_install = mca_btl_ugni.la
-else
-component_noinst = libmca_btl_ugni.la
-component_install =
-endif
-
-ugni_SOURCES = \
-        btl_ugni_component.c \
-        btl_ugni_module.c \
-        btl_ugni_add_procs.c \
-        btl_ugni_endpoint.h \
-        btl_ugni_endpoint.c \
-        btl_ugni_frag.c \
-        btl_ugni_frag.h \
-        btl_ugni_rdma.h \
-        btl_ugni_send.c \
-        btl_ugni_put.c \
-        btl_ugni_get.c \
-        btl_ugni.h \
-        btl_ugni_smsg.h \
-        btl_ugni_smsg.c \
-        btl_ugni_progress_thread.c \
-        btl_ugni_prepare.h \
-        btl_ugni_atomic.c \
-        btl_ugni_init.c \
-        btl_ugni_device.h
-
-mcacomponentdir = $(opallibdir)
-mcacomponent_LTLIBRARIES = $(component_install)
-mca_btl_ugni_la_SOURCES = $(ugni_SOURCES)
-nodist_mca_btl_ugni_la_SOURCES = $(ugni_nodist_SOURCES)
-mca_btl_ugni_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
-	$(btl_ugni_LIBS)
-mca_btl_ugni_la_LDFLAGS = -module -avoid-version $(btl_ugni_LDFLAGS)
-
-noinst_LTLIBRARIES = $(component_noinst)
-libmca_btl_ugni_la_SOURCES = $(ugni_SOURCES)
-nodist_libmca_btl_ugni_la_SOURCES = $(ugni_nodist_SOURCES)
-libmca_btl_ugni_la_LIBADD = $(btl_ugni_LIBS)
-libmca_btl_ugni_la_LDFLAGS = -module -avoid-version $(btl_ugni_LDFLAGS)
diff --git a/opal/mca/btl/ugni/btl_ugni.h b/opal/mca/btl/ugni/btl_ugni.h
deleted file mode 100644
index eaf5c4e5e24..00000000000
--- a/opal/mca/btl/ugni/btl_ugni.h
+++ /dev/null
@@ -1,620 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2011      UT-Battelle, LLC. All rights reserved.
- * Copyright (c) 2014      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-/*
- * The ugni btl is implemented with native Cray Gemini.
- *
- * Known issues with ugni:
- *  -
- */
-
-#ifndef MCA_BTL_UGNI_H
-#define MCA_BTL_UGNI_H
-
-#include "opal_config.h"
-
-#include "opal/mca/mpool/mpool.h"
-#include "opal/mca/mpool/base/base.h"
-#include "opal/mca/rcache/base/base.h"
-#include "opal/mca/rcache/udreg/rcache_udreg.h"
-#include "opal/util/output.h"
-#include "opal_stdint.h"
-#include "opal/mca/btl/btl.h"
-#include "opal/mca/btl/base/base.h"
-#include "opal/mca/btl/base/btl_base_error.h"
-#include "opal/class/opal_hash_table.h"
-#include "opal/class/opal_free_list.h"
-#include "opal/class/opal_pointer_array.h"
-#include "opal/mca/pmix/pmix-internal.h"
-
-#include <errno.h>
-#include <stdint.h>
-#include <sys/types.h>
-#include <assert.h>
-#include <sys/time.h>
-#include <gni_pub.h>
-
-/* datagram message ids */
-#define MCA_BTL_UGNI_CONNECT_WILDCARD_ID 0x0000000000000000ull
-#define MCA_BTL_UGNI_CONNECT_DIRECTED_ID 0x8000000000000000ull
-#define MCA_BTL_UGNI_DATAGRAM_MASK       0x8000000000000000ull
-
-/** maximum number of supported virtual devices */
-#define MCA_BTL_UGNI_MAX_DEV_HANDLES 128
-
-/** number of rdma completion queue items to remove per progress loop */
-#define MCA_BTL_UGNI_COMPLETIONS_PER_LOOP 32
-
-/** how often to check for connection requests */
-#define MCA_BTL_UGNI_CONNECT_USEC 10
-
-/**
- * Modex data
- */
-struct mca_btl_ugni_modex_t {
-    /** GNI NIC address */
-    uint32_t addr;
-    /** CDM identifier (base) */
-    int      id;
-};
-typedef struct mca_btl_ugni_modex_t mca_btl_ugni_modex_t;
-
-/* ompi and smsg endpoint attributes */
-typedef struct mca_btl_ugni_endpoint_attr_t {
-    opal_process_name_t proc_name;
-    uint32_t index;
-    gni_smsg_attr_t smsg_attr;
-    gni_mem_handle_t rmt_irq_mem_hndl;
-} mca_btl_ugni_endpoint_attr_t;
-
-enum {
-    MCA_BTL_UGNI_RCACHE_UDREG,
-    MCA_BTL_UGNI_RCACHE_GRDMA
-};
-
-enum mca_btl_ugni_free_list_id_t {
-    /* eager fragment list (registered) */
-    MCA_BTL_UGNI_LIST_EAGER_SEND,
-    MCA_BTL_UGNI_LIST_EAGER_RECV,
-    /* SMSG fragment list (unregistered) */
-    MCA_BTL_UGNI_LIST_SMSG,
-    /* RDMA fragment list */
-    MCA_BTL_UGNI_LIST_RDMA,
-    MCA_BTL_UGNI_LIST_RDMA_INT,
-    MCA_BTL_UGNI_LIST_MAX,
-};
-
-struct mca_btl_ugni_cq_t {
-    /** ugni CQ handle */
-    gni_cq_handle_t gni_handle;
-    /** number of completions expected on the CQ */
-    volatile int32_t active_operations;
-};
-typedef struct mca_btl_ugni_cq_t mca_btl_ugni_cq_t;
-
-/**
- * GNI virtual device
- */
-struct mca_btl_ugni_device_t {
-    /** Communication domain handle */
-    gni_cdm_handle_t dev_cd_handle;
-
-    /** protection for ugni access */
-    volatile int32_t lock;
-
-    /** Index of device in module devices array */
-    int dev_index;
-
-    /** number of SMSG connections */
-    opal_atomic_int32_t smsg_connections;
-
-    /** boolean indicating that the device was recently flushed */
-    volatile bool flushed;
-
-    /** uGNI device handle */
-    gni_nic_handle_t dev_handle;
-
-    /** uGNI rdma completion queue */
-    mca_btl_ugni_cq_t dev_rdma_local_cq;
-
-    /** local rdma completion queue (async) */
-    mca_btl_ugni_cq_t dev_rdma_local_irq_cq;
-
-    /** local SMSG completion queue */
-    mca_btl_ugni_cq_t dev_smsg_local_cq;
-
-    /** IRQ memory handle for this device */
-    gni_mem_handle_t smsg_irq_mhndl;
-
-    /** RDMA endpoint free list */
-    opal_free_list_t rdma_descs;
-};
-typedef struct mca_btl_ugni_device_t mca_btl_ugni_device_t;
-
-typedef intptr_t (*mca_btl_ugni_device_serialize_fn_t) (mca_btl_ugni_device_t *device, void *arg);
-
-typedef struct mca_btl_ugni_module_t {
-    mca_btl_base_module_t super;
-
-    bool initialized;
-
-    mca_btl_ugni_device_t devices[MCA_BTL_UGNI_MAX_DEV_HANDLES];
-
-    opal_mutex_t endpoint_lock;
-    size_t endpoint_count;
-    opal_pointer_array_t endpoints;
-    opal_hash_table_t id_to_endpoint;
-
-    /* lock for this list */
-    opal_mutex_t     failed_frags_lock;
-    /** rdma frags waiting to be reposted */
-    opal_list_t failed_frags;
-
-    /** lock for the eager_get_pending list */
-    opal_mutex_t eager_get_pending_lock;
-    opal_list_t eager_get_pending;
-
-    mca_mpool_base_module_t *mpool;
-    opal_free_list_t         smsg_mboxes;
-
-    gni_ep_handle_t wildcard_ep;
-    struct mca_btl_base_endpoint_t *local_ep;
-
-    opal_atomic_int32_t active_datagrams;
-    opal_event_t connection_event;
-
-    struct mca_btl_ugni_endpoint_attr_t wc_remote_attr, wc_local_attr;
-
-    gni_cq_handle_t smsg_remote_cq;
-    gni_cq_handle_t smsg_remote_irq_cq;
-
-    /** fragment free lists (see enum mca_btl_ugni_free_list_id_t) */
-    opal_free_list_t frags_lists[MCA_BTL_UGNI_LIST_MAX];
-
-    /* lock for this list */
-    opal_mutex_t     ep_wait_list_lock;
-    /* endpoints waiting on credits */
-    opal_list_t      ep_wait_list;
-
-    /* fragment id bounce buffer (smsg msg ids are only 32 bits) */
-    opal_pointer_array_t pending_smsg_frags_bb;
-
-    int32_t reg_max;
-    opal_atomic_int32_t reg_count;
-
-    /* used to calculate the fraction of registered memory resources
-     * this rank should be limited too */
-    int nlocal_procs;
-
-    opal_atomic_int32_t active_rdma_count;
-
-    mca_rcache_base_module_t *rcache;
-} mca_btl_ugni_module_t;
-
-typedef struct mca_btl_ugni_component_t {
-    /* base BTL component */
-    mca_btl_base_component_3_0_0_t super;
-
-    /* maximum supported btls. hardcoded to 1 for now */
-    uint32_t ugni_max_btls;
-    /* Maximum number of entries a completion queue can hold */
-    uint32_t remote_cq_size;
-    uint32_t local_cq_size;
-    uint32_t local_rdma_cq_size;
-    /* There is a hardware limitation that hurts BTE performance
-     * if we submit too many BTE requests. This acts as a throttle. */
-    int32_t active_rdma_threshold;
-
-    /* number of ugni modules */
-    uint32_t ugni_num_btls;
-    /* ugni modules */
-    mca_btl_ugni_module_t *modules;
-
-    size_t smsg_max_data;
-
-    /* After this message size switch to BTE protocols */
-    long int ugni_fma_limit;
-    /** FMA switchover for get */
-    long int ugni_fma_get_limit;
-    /** FMA switchover for put */
-    long int ugni_fma_put_limit;
-
-#if OPAL_C_HAVE__THREAD_LOCAL
-    bool bind_threads_to_devices;
-#endif
-
-    /* Switch to get when sending above this size */
-    size_t ugni_smsg_limit;
-
-    /* RDMA/SMSG free list settings */
-    int ugni_free_list_num;
-    int ugni_free_list_max;
-    int ugni_free_list_inc;
-
-    /* eager free list settings */
-    int ugni_eager_num;
-    int ugni_eager_max;
-    int ugni_eager_inc;
-
-    int smsg_max_retries;
-    /* number of times to retry a post */
-    int rdma_max_retries;
-
-    /* Maximum number of outstanding eager messages */
-    int smsg_max_credits;
-    /* mailbox size (computed) */
-    int smsg_mbox_size;
-
-    /* Maximum number of memory registrations per process */
-    int max_mem_reg;
-
-    /* Page size to use for SMSG allocations (udreg mpool) */
-    unsigned int smsg_page_size;
-
-    /* rcache type (grdma or udreg) */
-    int rcache_type;
-
-    /* memory pool hints */
-    char *mpool_hints;
-
-    /* Number of mailboxes to allocate in each block */
-    unsigned int mbox_increment;
-
-    /* Indicate whether progress thread requested */
-    bool progress_thread_requested;
-
-    /* Indicate whether progress thread allowed */
-    bool progress_thread_enabled;
-
-    /** Number of ugni device contexts to create per GNI device */
-    int virtual_device_count;
-
-    /** Protection tag */
-    uint8_t ptag;
-
-    /** Unique id for this process assigned by the system */
-    uint32_t cookie;
-
-    /** Starting value of communication identifier */
-    uint32_t cdm_id_base;
-
-    /** GNI CDM flags */
-    uint32_t cdm_flags;
-
-    /** NIC address */
-    uint32_t dev_addr;
-
-    /** MCA variable identifier for the cdm_flags variable */
-    int cdm_flags_id;
-} mca_btl_ugni_component_t;
-
-/* Global structures */
-
-OPAL_MODULE_DECLSPEC extern mca_btl_ugni_component_t mca_btl_ugni_component;
-OPAL_MODULE_DECLSPEC extern mca_btl_ugni_module_t mca_btl_ugni_module;
-
-static inline uint32_t mca_btl_ugni_ep_get_device_index (mca_btl_ugni_module_t *ugni_module)
-{
-    static opal_atomic_int64_t device_index = 0;
-
-    /* don't really care if the device index is atomically updated */
-    return (uint32_t) (opal_atomic_fetch_add_64 (&device_index, 1) % mca_btl_ugni_component.virtual_device_count);
-}
-
-/**
- * Get a virtual device for communication
- */
-static inline mca_btl_ugni_device_t *mca_btl_ugni_ep_get_device (mca_btl_ugni_module_t *ugni_module)
-{
-    return ugni_module->devices + mca_btl_ugni_ep_get_device_index (ugni_module);
-}
-
-static inline int mca_btl_rc_ugni_to_opal (gni_return_t rc)
-{
-    static int codes[] = {OPAL_SUCCESS,
-                          OPAL_ERR_RESOURCE_BUSY,
-                          OPAL_ERR_BAD_PARAM,
-                          OPAL_ERR_OUT_OF_RESOURCE,
-                          OPAL_ERR_TIMEOUT,
-                          OPAL_ERR_PERM,
-                          OPAL_ERROR,
-                          OPAL_ERR_BAD_PARAM,
-                          OPAL_ERR_BAD_PARAM,
-                          OPAL_ERR_NOT_FOUND,
-                          OPAL_ERR_VALUE_OUT_OF_BOUNDS,
-                          OPAL_ERROR,
-                          OPAL_ERR_NOT_SUPPORTED,
-                          OPAL_ERR_OUT_OF_RESOURCE};
-    return codes[rc];
-}
-
-
-int mca_btl_ugni_flush (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint);
-
-/**
- * BML->BTL notification of change in the process list.
- *
- * location: btl_ugni_add_procs.c
- *
- * @param btl (IN)            BTL module
- * @param nprocs (IN)         Number of processes
- * @param procs (IN)          Array of processes
- * @param endpoint (OUT)      Array of mca_btl_base_endpoint_t structures by BTL.
- * @param reachable (OUT)     Bitmask indicating set of peer processes that are reachable by this BTL.
- * @return                    OPAL_SUCCESS or error status on failure.
- */
-int
-mca_btl_ugni_add_procs (struct mca_btl_base_module_t* btl,
-                        size_t nprocs,
-                        struct opal_proc_t **procs,
-                        struct mca_btl_base_endpoint_t **peers,
-                        opal_bitmap_t *reachable);
-
-/**
- * Notification of change to the process list.
- *
- * location: btl_ugni_add_procs.c
- *
- * @param btl (IN)     BTL module
- * @param nprocs (IN)  Number of processes
- * @param proc (IN)    Set of processes
- * @param peer (IN)    Set of peer addressing information.
- * @return             Status indicating if cleanup was successful
- */
-int
-mca_btl_ugni_del_procs (struct mca_btl_base_module_t *btl,
-                        size_t nprocs,
-                        struct opal_proc_t **procs,
-                        struct mca_btl_base_endpoint_t **peers);
-
-struct mca_btl_base_endpoint_t *mca_btl_ugni_get_ep (struct mca_btl_base_module_t *module, opal_proc_t *proc);
-
-/**
- * Initiate an asynchronous send.
- *
- * location: btl_ugni_send.c
- *
- * @param btl (IN)         BTL module
- * @param endpoint (IN)    BTL addressing information
- * @param descriptor (IN)  Description of the data to be transfered
- * @param tag (IN)         The tag value used to notify the peer.
- */
-int
-mca_btl_ugni_send (struct mca_btl_base_module_t *btl,
-                   struct mca_btl_base_endpoint_t *btl_peer,
-                   struct mca_btl_base_descriptor_t *descriptor,
-                   mca_btl_base_tag_t tag);
-
-/**
- * Initiate an immediate blocking send.
- *
- * location: btl_ugni_sendi.c
- *
- * @param btl (IN)             BTL module
- * @param endpoint (IN)        BTL addressing information
- * @param convertor (IN)       Data type convertor
- * @param header (IN)          Pointer to header.
- * @param header_size (IN)     Size of header.
- * @param payload_size (IN)    Size of payload (from convertor).
- * @param order (IN)           The ordering tag (may be MCA_BTL_NO_ORDER)
- * @param flags (IN)           Flags.
- * @param tag (IN)             The tag value used to notify the peer.
- * @param descriptor (OUT)     The descriptor to be returned unable to be sent immediately
- */
-int
-mca_btl_ugni_sendi (struct mca_btl_base_module_t *btl,
-                    struct mca_btl_base_endpoint_t *endpoint,
-                    struct opal_convertor_t *convertor,
-                    void *header, size_t header_size,
-                    size_t payload_size, uint8_t order,
-                    uint32_t flags, mca_btl_base_tag_t tag,
-                    mca_btl_base_descriptor_t **descriptor);
-
-int mca_btl_ugni_get (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, void *local_address,
-                      uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                      mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-                      int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
-
-int mca_btl_ugni_put (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, void *local_address,
-                      uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                      mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-                      int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
-
-int mca_btl_ugni_aop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
-                      uint64_t remote_address, mca_btl_base_registration_handle_t *remote_handle,
-                      mca_btl_base_atomic_op_t op, uint64_t operand, int flags, int order,
-                      mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
-
-int mca_btl_ugni_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
-                       void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                       mca_btl_base_registration_handle_t *remote_handle, mca_btl_base_atomic_op_t op,
-                       uint64_t operand, int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
-                       void *cbcontext, void *cbdata);
-
-int mca_btl_ugni_acswap (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
-                         void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                         mca_btl_base_registration_handle_t *remote_handle, uint64_t compare, uint64_t value,
-                         int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
-
-int mca_btl_ugni_progress_send_wait_list (struct mca_btl_base_endpoint_t *endpoint);
-int mca_btl_ugni_progress_datagram (mca_btl_ugni_device_t *device);
-
-mca_btl_base_descriptor_t *
-mca_btl_ugni_alloc(struct mca_btl_base_module_t *btl,
-                   struct mca_btl_base_endpoint_t *endpoint,
-                   uint8_t order, size_t size, uint32_t flags);
-
-struct mca_btl_base_registration_handle_t {
-    /** uGNI memory handle */
-    gni_mem_handle_t gni_handle;
-};
-
-typedef struct mca_btl_ugni_reg_t {
-    mca_rcache_base_registration_t base;
-    mca_btl_base_registration_handle_t handle;
-} mca_btl_ugni_reg_t;
-
-/**
- * Initialize uGNI support.
- */
-int mca_btl_ugni_init (void);
-
-/**
- * Finalize uGNI support.
- */
-int mca_btl_ugni_fini (void);
-
-int mca_btl_ugni_module_init (mca_btl_ugni_module_t *ugni_module);
-
-/**
- * Intialize a virtual device for device index 0.
- *
- * @param[inout] device         Device to initialize
- * @param[in] virtual_device_id Virtual device identified (up to max handles)
- */
-int mca_btl_ugni_device_init (mca_btl_ugni_device_t *device, int virtual_device_id);
-
-/**
- * Finalize a virtual device.
- *
- * @param[in] device Device to finalize
- */
-int mca_btl_ugni_device_fini (mca_btl_ugni_device_t *dev);
-
-/* Get a unique 64-bit id for the process name */
-static inline uint64_t mca_btl_ugni_proc_name_to_id (opal_process_name_t name) {
-    /* Throw away the top bit of the jobid for the datagram type */
-    return ((uint64_t) (name.jobid & 0x7fffffff) << 32 | name.vpid);
-}
-
-int mca_btl_ugni_spawn_progress_thread(struct mca_btl_base_module_t* btl);
-int mca_btl_ugni_kill_progress_thread(void);
-
-struct mca_btl_ugni_post_descriptor_t;
-
-void btl_ugni_dump_post_desc (struct mca_btl_ugni_post_descriptor_t *desc);
-
-
-struct mca_btl_ugni_post_descriptor_t;
-
-void mca_btl_ugni_handle_rdma_completions (mca_btl_ugni_module_t *ugni_module, mca_btl_ugni_device_t *device,
-                                           struct mca_btl_ugni_post_descriptor_t *post_desc, const int count);
-
-/**
- * Try to lock a uGNI device for exclusive access
- */
-static inline int mca_btl_ugni_device_trylock (mca_btl_ugni_device_t *device)
-{
-    /* checking the lock non-atomically first can reduce the number of
-     * unnecessary atomic operations. */
-    return (device->lock || opal_atomic_swap_32 (&device->lock, 1));
-}
-
-/**
- * Lock a uGNI device for exclusive access
- */
-static inline void mca_btl_ugni_device_lock (mca_btl_ugni_device_t *device)
-{
-    while (mca_btl_ugni_device_trylock (device));
-}
-
-/**
- * Release exclusive access to the device
- */
-static inline void mca_btl_ugni_device_unlock (mca_btl_ugni_device_t *device)
-{
-    opal_atomic_wmb ();
-    device->lock = 0;
-}
-
-/**
- * Serialize an operation on a uGNI device
- *
- * @params[in] device ugni device
- * @params[in] fn     function to serialize
- * @params[in] arg    function argument
- */
-static inline intptr_t mca_btl_ugni_device_serialize (mca_btl_ugni_device_t *device,
-                                                      mca_btl_ugni_device_serialize_fn_t fn, void *arg)
-{
-    intptr_t rc;
-
-    if (!opal_using_threads ()) {
-        return fn (device, arg);
-    }
-
-    /* NTH: for now the device is just protected by a spin lock but this will change in the future */
-    mca_btl_ugni_device_lock (device);
-    rc = fn (device, arg);
-    mca_btl_ugni_device_unlock (device);
-    return rc;
-}
-
-static inline intptr_t mca_btl_ugni_device_serialize_any (mca_btl_ugni_module_t *ugni_module,
-                                                          mca_btl_ugni_device_serialize_fn_t fn, void *arg)
-{
-    mca_btl_ugni_device_t *device;
-    intptr_t rc;
-
-    if (!opal_using_threads ()) {
-        return fn (ugni_module->devices, arg);
-    }
-
-#if OPAL_C_HAVE__THREAD_LOCAL
-    if (mca_btl_ugni_component.bind_threads_to_devices) {
-        /* NTH: if we have C11 _Thread_local just go ahead and assign the devices round-robin to each
-         * thread. in testing this give much better performance than just picking any device */
-        static _Thread_local mca_btl_ugni_device_t *device_local = NULL;
-
-        device = device_local;
-        if (OPAL_UNLIKELY(NULL == device)) {
-            /* assign device contexts round-robin */
-            device_local = device = mca_btl_ugni_ep_get_device (ugni_module);
-        }
-
-        mca_btl_ugni_device_lock (device);
-    } else {
-#endif
-        /* get the next starting index */
-        uint32_t device_index = mca_btl_ugni_ep_get_device_index (ugni_module);
-        const int device_count = mca_btl_ugni_component.virtual_device_count;
-
-        for (int i = 0 ; i < device_count ; ++i) {
-            device = ugni_module->devices + ((device_index + i) % device_count);
-            if (!mca_btl_ugni_device_trylock (device)) {
-                break;
-            }
-
-            device = NULL;
-        }
-
-        if (NULL == device) {
-            device = mca_btl_ugni_ep_get_device (ugni_module);
-            mca_btl_ugni_device_lock (device);
-        }
-#if OPAL_C_HAVE__THREAD_LOCAL
-    }
-#endif
-
-    rc = fn (device, arg);
-    mca_btl_ugni_device_unlock (device);
-
-    return rc;
-}
-
-
-/** Number of times the progress thread has woken up */
-extern unsigned int mca_btl_ugni_progress_thread_wakeups;
-
-#endif
diff --git a/opal/mca/btl/ugni/btl_ugni_add_procs.c b/opal/mca/btl/ugni/btl_ugni_add_procs.c
deleted file mode 100644
index 664271c17a2..00000000000
--- a/opal/mca/btl/ugni/btl_ugni_add_procs.c
+++ /dev/null
@@ -1,532 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2011-2017 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2011      UT-Battelle, LLC. All rights reserved.
- * Copyright (c) 2014-2019 Intel, Inc.  All rights reserved.
- * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "opal_config.h"
-
-#include "btl_ugni.h"
-#include "btl_ugni_frag.h"
-#include "btl_ugni_smsg.h"
-
-#include "opal/include/opal/align.h"
-#include "opal/mca/pmix/pmix-internal.h"
-
-#define INITIAL_GNI_EPS 1024
-
-static int
-mca_btl_ugni_setup_mpools (mca_btl_ugni_module_t *ugni_module);
-static void
-mca_btl_ugni_module_set_max_reg (mca_btl_ugni_module_t *ugni_module, int nlocal_procs);
-static int mca_btl_ugni_smsg_setup (int nprocs);
-
-int mca_btl_ugni_add_procs (struct mca_btl_base_module_t* btl, size_t nprocs,
-                            struct opal_proc_t **procs,
-                            struct mca_btl_base_endpoint_t **peers,
-                            opal_bitmap_t *reachable) {
-    mca_btl_ugni_module_t *ugni_module = (mca_btl_ugni_module_t *) btl;
-    int rc;
-    void *mmap_start_addr;
-    struct timeval tv = {.tv_sec = 0, .tv_usec = MCA_BTL_UGNI_CONNECT_USEC};
-
-    if (false == ugni_module->initialized) {
-
-        /* TODO: need to think of something more elegant than this max array */
-
-        rc = opal_pointer_array_init (&ugni_module->endpoints, INITIAL_GNI_EPS, 1 << 24, 512);
-        if (OPAL_SUCCESS != rc) {
-            BTL_ERROR(("error inializing the endpoint array. rc = %d", rc));
-            return rc;
-        }
-
-
-        /* NTH: might want to vary this size based off the universe size (if
-         * one exists). the table is only used for connection lookup and
-         * endpoint removal. */
-        rc = opal_hash_table_init (&ugni_module->id_to_endpoint, INITIAL_GNI_EPS);
-        if (OPAL_SUCCESS != rc) {
-            BTL_ERROR(("error initializing the endpoint hash. rc = %d", rc));
-            return rc;
-        }
-    }
-
-    for (size_t i = 0 ; i < nprocs ; ++i) {
-        peers[i] = mca_btl_ugni_get_ep (btl, procs[i]);
-        if (NULL == peers[i]) {
-            continue;
-        }
-
-        if (procs[i] == opal_proc_local_get ()) {
-            ugni_module->local_ep = peers[i];
-        }
-
-        /* Set the reachable bit if necessary */
-        if (reachable) {
-            (void) opal_bitmap_set_bit (reachable, i);
-        }
-    }
-
-    mca_btl_ugni_module_set_max_reg (ugni_module, ugni_module->nlocal_procs);
-
-    if (false == ugni_module->initialized) {
-        for (int i = 0 ; i < mca_btl_ugni_component.virtual_device_count ; ++i) {
-            mca_btl_ugni_device_t *device = ugni_module->devices + i;
-            rc = GNI_CqCreate (device->dev_handle, mca_btl_ugni_component.local_rdma_cq_size, 0,
-                               GNI_CQ_NOBLOCK, NULL, NULL, &device->dev_rdma_local_cq.gni_handle);
-            if (GNI_RC_SUCCESS != rc) {
-                BTL_ERROR(("error creating local BTE/FMA CQ"));
-                return mca_btl_rc_ugni_to_opal (rc);
-            }
-
-            rc = GNI_CqCreate (device->dev_handle, mca_btl_ugni_component.local_cq_size,
-                               0, GNI_CQ_NOBLOCK, NULL, NULL, &device->dev_smsg_local_cq.gni_handle);
-            if (GNI_RC_SUCCESS != rc) {
-                BTL_ERROR(("error creating local SMSG CQ"));
-                return mca_btl_rc_ugni_to_opal (rc);
-            }
-
-            if (mca_btl_ugni_component.progress_thread_enabled) {
-                rc = GNI_CqCreate (device->dev_handle, mca_btl_ugni_component.local_rdma_cq_size,
-                                   0, GNI_CQ_BLOCKING, NULL, NULL, &device->dev_rdma_local_irq_cq.gni_handle);
-                if (GNI_RC_SUCCESS != rc) {
-                    BTL_ERROR(("error creating local BTE/FMA CQ"));
-                    return mca_btl_rc_ugni_to_opal (rc);
-                }
-            }
-        }
-
-        rc = GNI_CqCreate (ugni_module->devices[0].dev_handle, mca_btl_ugni_component.remote_cq_size,
-                           0, GNI_CQ_NOBLOCK, NULL, NULL, &ugni_module->smsg_remote_cq);
-        if (GNI_RC_SUCCESS != rc) {
-            BTL_ERROR(("error creating remote SMSG CQ"));
-            return mca_btl_rc_ugni_to_opal (rc);
-        }
-
-        if (mca_btl_ugni_component.progress_thread_enabled) {
-            rc = GNI_CqCreate (ugni_module->devices[0].dev_handle, mca_btl_ugni_component.remote_cq_size,
-                               0, GNI_CQ_BLOCKING, NULL, NULL, &ugni_module->smsg_remote_irq_cq);
-            if (GNI_RC_SUCCESS != rc) {
-                BTL_ERROR(("error creating remote SMSG CQ"));
-                return mca_btl_rc_ugni_to_opal (rc);
-            }
-        }
-
-        rc = mca_btl_ugni_setup_mpools (ugni_module);
-        if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-            BTL_ERROR(("btl/ugni error setting up mpools/free lists"));
-            return rc;
-        }
-
-        rc = mca_btl_ugni_smsg_init (ugni_module);
-        if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-            BTL_ERROR(("btl/ugni error initializing SMSG"));
-            return rc;
-        }
-
-        /*
-         * If progress thread enabled, registered a page of memory
-         * with the smsg_remote_irq_cq.  This memory handle is passed
-         * to ranks which want to communicate with this rank. A rank which
-         * posts a GNI_PostCqWrite targeting this memory handle generates
-         * an IRQ at the target node, which ultimately causes the progress
-         * thread in the target rank to become schedulable.
-         */
-        if (mca_btl_ugni_component.progress_thread_enabled) {
-            mmap_start_addr = mmap(NULL, 4096, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
-            if (NULL == mmap_start_addr) {
-                BTL_ERROR(("btl/ugni mmap returned error"));
-                return OPAL_ERR_OUT_OF_RESOURCE;
-            }
-
-            rc = GNI_MemRegister(ugni_module->devices[0].dev_handle,
-                                     (unsigned long)mmap_start_addr,
-                                     4096,
-                                     ugni_module->smsg_remote_irq_cq,
-                                     GNI_MEM_READWRITE,
-                                     -1,
-                                     &ugni_module->devices[0].smsg_irq_mhndl);
-
-            mca_btl_ugni_spawn_progress_thread(btl);
-        }
-
-        opal_event_evtimer_add (&ugni_module->connection_event, &tv);
-
-        ugni_module->initialized = true;
-    }
-
-    return OPAL_SUCCESS;
-}
-
-int mca_btl_ugni_del_procs (struct mca_btl_base_module_t *btl,
-                            size_t nprocs, struct opal_proc_t **procs,
-                            struct mca_btl_base_endpoint_t **peers) {
-    mca_btl_ugni_module_t *ugni_module = (mca_btl_ugni_module_t *) btl;
-
-    OPAL_THREAD_LOCK(&ugni_module->endpoint_lock);
-
-    for (size_t i = 0 ; i < nprocs ; ++i) {
-        struct opal_proc_t *opal_proc = procs[i];
-        uint64_t proc_id = mca_btl_ugni_proc_name_to_id(opal_proc->proc_name);
-        mca_btl_base_endpoint_t *ep = NULL;
-
-        /* lookup this proc in the hash table */
-        (void) opal_hash_table_get_value_uint64 (&ugni_module->id_to_endpoint, proc_id, (void **) &ep);
-
-        BTL_VERBOSE(("deleting endpoint with proc id 0x%" PRIx64 ", ptr: %p", proc_id, (void *) ep));
-
-        if (NULL != ep) {
-            mca_btl_ugni_release_ep (ep);
-            --ugni_module->endpoint_count;
-        }
-
-        if (OPAL_PROC_ON_LOCAL_NODE(opal_proc->proc_flags)) {
-            --ugni_module->nlocal_procs;
-        }
-
-        /* remote the endpoint from the hash table */
-        opal_hash_table_set_value_uint64 (&ugni_module->id_to_endpoint, proc_id, NULL);
-    }
-
-    OPAL_THREAD_UNLOCK(&ugni_module->endpoint_lock);
-
-    mca_btl_ugni_module_set_max_reg (ugni_module, ugni_module->nlocal_procs);
-
-    return OPAL_SUCCESS;
-}
-
-
-struct mca_btl_base_endpoint_t *mca_btl_ugni_get_ep (struct mca_btl_base_module_t *module, opal_proc_t *proc)
-{
-    mca_btl_ugni_module_t *ugni_module = (mca_btl_ugni_module_t *) module;
-    uint64_t proc_id = mca_btl_ugni_proc_name_to_id(proc->proc_name);
-    mca_btl_base_endpoint_t *ep;
-    int rc;
-
-    OPAL_THREAD_LOCK(&ugni_module->endpoint_lock);
-
-    do {
-        rc = opal_hash_table_get_value_uint64 (&ugni_module->id_to_endpoint, proc_id, (void **) &ep);
-        if (OPAL_SUCCESS == rc) {
-            BTL_VERBOSE(("returning existing endpoint for proc %s", OPAL_NAME_PRINT(proc->proc_name)));
-            break;
-        }
-
-        BTL_VERBOSE(("initialized uGNI endpoint for proc id: 0x%" PRIx64 " ptr: %p", proc_id, (void *) proc));
-
-        /*  Create and Init endpoints */
-        rc = mca_btl_ugni_init_ep (ugni_module, &ep, ugni_module, proc);
-        if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-            BTL_ERROR(("btl/ugni error initializing endpoint"));
-            break;
-        }
-
-        /* ugni is allowed on local processes to provide support for network atomic operations */
-        if (OPAL_PROC_ON_LOCAL_NODE(proc->proc_flags)) {
-            ++ugni_module->nlocal_procs;
-        }
-        ++ugni_module->endpoint_count;
-
-        /* add this endpoint to the connection lookup table */
-        opal_hash_table_set_value_uint64 (&ugni_module->id_to_endpoint, proc_id, ep);
-    } while (0);
-
-    OPAL_THREAD_UNLOCK(&ugni_module->endpoint_lock);
-
-    return ep;
-}
-
-
-static int ugni_reg_mem (void *reg_data, void *base, size_t size,
-                         mca_rcache_base_registration_t *reg)
-{
-    mca_btl_ugni_module_t *ugni_module = (mca_btl_ugni_module_t *) reg_data;
-    gni_cq_handle_t cq = 0;
-    int flags, rc;
-
-    if (ugni_module->reg_count >= ugni_module->reg_max) {
-        return OPAL_ERR_OUT_OF_RESOURCE;
-    }
-
-    if (reg->access_flags & (MCA_RCACHE_ACCESS_REMOTE_WRITE | MCA_RCACHE_ACCESS_LOCAL_WRITE |
-                             MCA_RCACHE_ACCESS_REMOTE_ATOMIC)) {
-        flags = GNI_MEM_READWRITE;
-    } else {
-        flags = GNI_MEM_READ_ONLY;
-    }
-
-    if (!(reg->flags & MCA_RCACHE_FLAGS_SO_MEM)) {
-        flags |= GNI_MEM_RELAXED_PI_ORDERING;
-    }
-
-    if (reg->flags & MCA_RCACHE_FLAGS_RESV0) {
-        cq = ugni_module->smsg_remote_cq;
-    }
-
-    rc = mca_btl_ugni_reg_mem (ugni_module, base, size, (mca_btl_ugni_reg_t *) reg, cq, flags);
-    if (OPAL_LIKELY(OPAL_SUCCESS == rc)) {
-        opal_atomic_add_fetch_32(&ugni_module->reg_count,1);
-    }
-
-    return rc;
-}
-
-static int
-ugni_dereg_mem (void *reg_data, mca_rcache_base_registration_t *reg)
-{
-    mca_btl_ugni_module_t *ugni_module = (mca_btl_ugni_module_t *) reg_data;
-    int rc;
-
-    rc = mca_btl_ugni_dereg_mem (ugni_module, (mca_btl_ugni_reg_t *) reg);
-    if (OPAL_LIKELY(OPAL_SUCCESS == rc)) {
-        opal_atomic_add_fetch_32(&ugni_module->reg_count,-1);
-    }
-
-    return rc;
-}
-
-static int
-mca_btl_ugni_setup_mpools (mca_btl_ugni_module_t *ugni_module)
-{
-    mca_rcache_udreg_resources_t rcache_resources;
-    unsigned int mbox_increment;
-    uint32_t nprocs, *u32;
-    char *rcache_name;
-    int rc;
-
-    rc = opal_pointer_array_init (&ugni_module->pending_smsg_frags_bb, 0,
-                                  1 << 30, 32768);
-    if (OPAL_SUCCESS != rc) {
-        return rc;
-    }
-
-    /* determine how many procs are in the job (might want to check universe size here) */
-    u32 = &nprocs;
-    OPAL_MODEX_RECV_VALUE(rc, PMIX_UNIV_SIZE, &OPAL_PROC_MY_NAME,
-                          &u32, PMIX_UINT32);
-    if (OPAL_SUCCESS != rc) {
-        /* take a wild conservative guess */
-        nprocs = 512;
-    }
-
-    rc = mca_btl_ugni_smsg_setup (nprocs);
-    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-        BTL_ERROR(("error setting up smsg"));
-        return rc;
-    }
-
-    rc = opal_free_list_init (ugni_module->frags_lists + MCA_BTL_UGNI_LIST_SMSG,
-                              sizeof (mca_btl_ugni_smsg_frag_t),
-                              opal_cache_line_size, OBJ_CLASS(mca_btl_ugni_smsg_frag_t),
-                              mca_btl_ugni_component.ugni_smsg_limit,
-                              opal_cache_line_size,
-                              mca_btl_ugni_component.ugni_free_list_num,
-                              mca_btl_ugni_component.ugni_free_list_max,
-                              mca_btl_ugni_component.ugni_free_list_inc,
-                              NULL, 0, NULL, (opal_free_list_item_init_fn_t) mca_btl_ugni_frag_init,
-                              (void *) (intptr_t) MCA_BTL_UGNI_LIST_SMSG);
-    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-        BTL_ERROR(("error creating smsg fragment free list"));
-        return rc;
-    }
-
-    rc = opal_free_list_init (ugni_module->frags_lists + MCA_BTL_UGNI_LIST_RDMA,
-                              sizeof (mca_btl_ugni_rdma_frag_t), 64,
-                              OBJ_CLASS(mca_btl_ugni_rdma_frag_t),
-                              0, opal_cache_line_size,
-                              mca_btl_ugni_component.ugni_free_list_num,
-                              mca_btl_ugni_component.ugni_free_list_max,
-                              mca_btl_ugni_component.ugni_free_list_inc,
-                              NULL, 0, NULL, (opal_free_list_item_init_fn_t) mca_btl_ugni_frag_init,
-                              (void *) (intptr_t) MCA_BTL_UGNI_LIST_RDMA);
-    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-        return rc;
-    }
-
-    rc = opal_free_list_init (ugni_module->frags_lists + MCA_BTL_UGNI_LIST_RDMA_INT,
-                              sizeof (mca_btl_ugni_rdma_frag_t), 8,
-                              OBJ_CLASS(mca_btl_ugni_rdma_frag_t),
-                              0, opal_cache_line_size, 0, -1, 64,
-                              NULL, 0, NULL, (opal_free_list_item_init_fn_t) mca_btl_ugni_frag_init,
-                              (void *) (intptr_t) MCA_BTL_UGNI_LIST_RDMA_INT);
-    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-        return rc;
-    }
-
-    ugni_module->super.btl_mpool = mca_mpool_base_module_lookup (mca_btl_ugni_component.mpool_hints);
-    if (NULL == ugni_module->super.btl_mpool) {
-        BTL_ERROR(("could not find mpool matching hints %s", mca_btl_ugni_component.mpool_hints));
-        return OPAL_ERROR;
-    }
-
-    rcache_resources.base.cache_name     = "ompi.ugni";
-    rcache_resources.base.reg_data       = (void *) ugni_module;
-    rcache_resources.base.sizeof_reg     = sizeof (mca_btl_ugni_reg_t);
-    rcache_resources.base.register_mem   = ugni_reg_mem;
-    rcache_resources.base.deregister_mem = ugni_dereg_mem;
-
-    if (MCA_BTL_UGNI_RCACHE_UDREG == mca_btl_ugni_component.rcache_type) {
-        /* additional settings for the udreg mpool */
-        /* 4k should be large enough for any Gemini/Ares system */
-        rcache_resources.max_entries       = 4096;
-        rcache_resources.use_kernel_cache  = true;
-
-        rcache_resources.use_evict_w_unreg = false;
-        rcache_name = "udreg";
-    } else {
-        rcache_name = "grdma";
-    }
-
-    ugni_module->rcache =
-        mca_rcache_base_module_create (rcache_name, ugni_module->devices, &rcache_resources.base);
-
-    if (NULL == ugni_module->rcache) {
-        BTL_ERROR(("error creating registration cache"));
-        return OPAL_ERROR;
-    }
-
-    rc = opal_free_list_init (ugni_module->frags_lists + MCA_BTL_UGNI_LIST_EAGER_SEND,
-                              sizeof (mca_btl_ugni_eager_frag_t), 8,
-                              OBJ_CLASS(mca_btl_ugni_eager_frag_t),
-                              ugni_module->super.btl_eager_limit, 64,
-                              mca_btl_ugni_component.ugni_eager_num,
-                              mca_btl_ugni_component.ugni_eager_max,
-                              mca_btl_ugni_component.ugni_eager_inc,
-                              ugni_module->super.btl_mpool, 0, ugni_module->rcache,
-                              (opal_free_list_item_init_fn_t) mca_btl_ugni_frag_init,
-                              (void *) (intptr_t) MCA_BTL_UGNI_LIST_EAGER_SEND);
-    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-        BTL_ERROR(("error creating eager send fragment free list"));
-        return rc;
-    }
-
-    rc = opal_free_list_init (ugni_module->frags_lists + MCA_BTL_UGNI_LIST_EAGER_RECV,
-                              sizeof (mca_btl_ugni_eager_frag_t), 8,
-                              OBJ_CLASS(mca_btl_ugni_eager_frag_t),
-                              ugni_module->super.btl_eager_limit, 64,
-                              mca_btl_ugni_component.ugni_eager_num,
-                              mca_btl_ugni_component.ugni_eager_max,
-                              mca_btl_ugni_component.ugni_eager_inc,
-                              ugni_module->super.btl_mpool, 0, ugni_module->rcache,
-                              (opal_free_list_item_init_fn_t) mca_btl_ugni_frag_init,
-                              (void *) (intptr_t) MCA_BTL_UGNI_LIST_EAGER_RECV);
-    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-        BTL_ERROR(("error creating eager receive fragment free list"));
-        return rc;
-    }
-
-    if (0 == mca_btl_ugni_component.mbox_increment) {
-        /* limit mailbox allocations to either 12.5% of available registrations
-           or 2MiB per allocation */
-        mbox_increment = (unsigned int) (2097152.0 / (float)mca_btl_ugni_component.smsg_mbox_size);
-
-        /* we may end up using more */
-        if (nprocs/mbox_increment > (unsigned int) ugni_module->reg_max / 8) {
-            mbox_increment = nprocs / (ugni_module->reg_max >> 3);
-        }
-    } else {
-        mbox_increment = mca_btl_ugni_component.mbox_increment;
-    }
-
-    /* use the MCA_RCACHE_FLAGS_RESV0 to signal this is smsg memory */
-    rc = opal_free_list_init (&ugni_module->smsg_mboxes,
-                              sizeof (mca_btl_ugni_smsg_mbox_t), 8,
-                              OBJ_CLASS(mca_btl_ugni_smsg_mbox_t),
-                              mca_btl_ugni_component.smsg_mbox_size, 128,
-                              32, -1, mbox_increment, ugni_module->super.btl_mpool,
-                              MCA_RCACHE_FLAGS_SO_MEM | MCA_RCACHE_FLAGS_RESV0,
-                              ugni_module->rcache, NULL, NULL);
-    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-        BTL_ERROR(("error creating smsg mailbox free list"));
-        return rc;
-    }
-
-    return OPAL_SUCCESS;
-}
-
-static void
-mca_btl_ugni_module_set_max_reg (mca_btl_ugni_module_t *ugni_module, int nlocal_procs)
-{
-    if (0 == mca_btl_ugni_component.max_mem_reg) {
-#if defined(HAVE_GNI_GETJOBRESINFO)
-        gni_job_res_desc_t res_des;
-        gni_return_t grc;
-        int fuzz = 20;
-
-        grc = GNI_GetJobResInfo (0, mca_btl_ugni_component.ptag,
-                                 GNI_JOB_RES_MDD, &res_des);
-        if (GNI_RC_SUCCESS == grc) {
-            if (nlocal_procs) {
-                ugni_module->reg_max = (res_des.limit - fuzz) / nlocal_procs;
-            } else {
-                ugni_module->reg_max = 0;
-            }
-        }
-#else
-        /* no way to determine the maximum registration count */
-        if (nlocal_procs) {
-            ugni_module->reg_max = 1200 / nlocal_procs;
-        } else {
-            ugni_module->reg_max = 0;
-        }
-#endif
-    } else if (-1 == mca_btl_ugni_component.max_mem_reg) {
-        ugni_module->reg_max = INT_MAX;
-    } else {
-        ugni_module->reg_max = mca_btl_ugni_component.max_mem_reg;
-    }
-
-    ugni_module->reg_count = 0;
-}
-
-static int mca_btl_ugni_smsg_setup (int nprocs)
-{
-    gni_smsg_attr_t tmp_smsg_attrib;
-    unsigned int mbox_size;
-    gni_return_t grc;
-
-    if (0 == mca_btl_ugni_component.ugni_smsg_limit) {
-        /* auto-set the smsg limit based on the number of ranks */
-        if (nprocs <= 512) {
-            mca_btl_ugni_component.ugni_smsg_limit = 8192;
-        } else if (nprocs <= 1024) {
-            mca_btl_ugni_component.ugni_smsg_limit = 2048;
-        } else if (nprocs <= 8192) {
-            mca_btl_ugni_component.ugni_smsg_limit = 1024;
-        } else if (nprocs <= 16384) {
-            mca_btl_ugni_component.ugni_smsg_limit = 512;
-        } else {
-            mca_btl_ugni_component.ugni_smsg_limit = 256;
-        }
-    }
-
-    mca_btl_ugni_component.smsg_max_data = mca_btl_ugni_component.ugni_smsg_limit -
-        sizeof (mca_btl_ugni_send_frag_hdr_t);
-
-    if (mca_btl_ugni_component.ugni_smsg_limit == mca_btl_ugni_module.super.btl_eager_limit) {
-        mca_btl_ugni_module.super.btl_eager_limit = mca_btl_ugni_component.smsg_max_data;
-    }
-
-    /* calculate mailbox size */
-    tmp_smsg_attrib.msg_type       = GNI_SMSG_TYPE_MBOX_AUTO_RETRANSMIT;
-    tmp_smsg_attrib.msg_maxsize    = mca_btl_ugni_component.ugni_smsg_limit;
-    tmp_smsg_attrib.mbox_maxcredit = mca_btl_ugni_component.smsg_max_credits;
-
-    grc = GNI_SmsgBufferSizeNeeded (&tmp_smsg_attrib, &mbox_size);
-    if (OPAL_UNLIKELY(GNI_RC_SUCCESS != grc)) {
-        BTL_ERROR(("error in GNI_SmsgBufferSizeNeeded"));
-        return mca_btl_rc_ugni_to_opal (grc);
-    }
-
-    mca_btl_ugni_component.smsg_mbox_size = OPAL_ALIGN(mbox_size, 64, unsigned int);
-
-    return OPAL_SUCCESS;
-}
diff --git a/opal/mca/btl/ugni/btl_ugni_atomic.c b/opal/mca/btl/ugni/btl_ugni_atomic.c
deleted file mode 100644
index a4396837e2c..00000000000
--- a/opal/mca/btl/ugni/btl_ugni_atomic.c
+++ /dev/null
@@ -1,162 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2019      Triad National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2019      Google, LLC. All rights reserved.
- * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "btl_ugni_rdma.h"
-
-static gni_fma_cmd_type_t amo_cmds[][MCA_BTL_ATOMIC_LAST] = {
-    [PMIX_INT32] = {
-        [MCA_BTL_ATOMIC_ADD] = GNI_FMA_ATOMIC2_IADD_S,
-        [MCA_BTL_ATOMIC_LAND] = GNI_FMA_ATOMIC2_AND_S,
-        [MCA_BTL_ATOMIC_LOR] = GNI_FMA_ATOMIC2_OR_S,
-        [MCA_BTL_ATOMIC_LXOR] = GNI_FMA_ATOMIC2_XOR_S,
-        [MCA_BTL_ATOMIC_SWAP] = GNI_FMA_ATOMIC2_SWAP_S,
-        [MCA_BTL_ATOMIC_MIN] = GNI_FMA_ATOMIC2_IMIN_S,
-        [MCA_BTL_ATOMIC_MAX] = GNI_FMA_ATOMIC2_IMAX_S,
-    },
-    [PMIX_INT64] = {
-        [MCA_BTL_ATOMIC_ADD] = GNI_FMA_ATOMIC_ADD,
-        [MCA_BTL_ATOMIC_AND] = GNI_FMA_ATOMIC_AND,
-        [MCA_BTL_ATOMIC_OR] = GNI_FMA_ATOMIC_OR,
-        [MCA_BTL_ATOMIC_XOR] = GNI_FMA_ATOMIC_XOR,
-        [MCA_BTL_ATOMIC_SWAP] = GNI_FMA_ATOMIC2_SWAP,
-        [MCA_BTL_ATOMIC_MIN] = GNI_FMA_ATOMIC2_IMIN,
-        [MCA_BTL_ATOMIC_MAX] = GNI_FMA_ATOMIC2_IMAX,
-    },
-    [PMIX_FLOAT] = {
-        [MCA_BTL_ATOMIC_ADD] = GNI_FMA_ATOMIC2_FPADD_S,
-        [MCA_BTL_ATOMIC_MIN] = GNI_FMA_ATOMIC2_FPMIN_S,
-        [MCA_BTL_ATOMIC_MAX] = GNI_FMA_ATOMIC2_FPMAX_S,
-    },
-    [PMIX_DOUBLE] = {
-        [MCA_BTL_ATOMIC_ADD] = GNI_FMA_ATOMIC2_FPADD,
-        [MCA_BTL_ATOMIC_MIN] = GNI_FMA_ATOMIC2_FPMIN,
-        [MCA_BTL_ATOMIC_MAX] = GNI_FMA_ATOMIC2_FPMAX,
-    },
-};
-
-static gni_fma_cmd_type_t famo_cmds[][MCA_BTL_ATOMIC_LAST] = {
-    [PMIX_INT32] = {
-        [MCA_BTL_ATOMIC_ADD] = GNI_FMA_ATOMIC2_FIADD_S,
-        [MCA_BTL_ATOMIC_LAND] = GNI_FMA_ATOMIC2_FAND_S,
-        [MCA_BTL_ATOMIC_LOR] = GNI_FMA_ATOMIC2_FOR_S,
-        [MCA_BTL_ATOMIC_LXOR] = GNI_FMA_ATOMIC2_FXOR_S,
-        [MCA_BTL_ATOMIC_SWAP] = GNI_FMA_ATOMIC2_FSWAP_S,
-        [MCA_BTL_ATOMIC_MIN] = GNI_FMA_ATOMIC2_FIMIN_S,
-        [MCA_BTL_ATOMIC_MAX] = GNI_FMA_ATOMIC2_FIMAX_S,
-    },
-    [PMIX_INT64] = {
-        [MCA_BTL_ATOMIC_ADD] = GNI_FMA_ATOMIC_FADD,
-        [MCA_BTL_ATOMIC_AND] = GNI_FMA_ATOMIC_FAND,
-        [MCA_BTL_ATOMIC_OR] = GNI_FMA_ATOMIC_FOR,
-        [MCA_BTL_ATOMIC_XOR] = GNI_FMA_ATOMIC_FXOR,
-        [MCA_BTL_ATOMIC_SWAP] = GNI_FMA_ATOMIC2_FSWAP,
-        [MCA_BTL_ATOMIC_MIN] = GNI_FMA_ATOMIC2_FIMIN,
-        [MCA_BTL_ATOMIC_MAX] = GNI_FMA_ATOMIC2_FIMAX,
-    },
-    [PMIX_FLOAT] = {
-        [MCA_BTL_ATOMIC_ADD] = GNI_FMA_ATOMIC2_FFPADD_S,
-        [MCA_BTL_ATOMIC_MIN] = GNI_FMA_ATOMIC2_FFPMIN_S,
-        [MCA_BTL_ATOMIC_MAX] = GNI_FMA_ATOMIC2_FFPMAX_S,
-    },
-    [PMIX_DOUBLE] = {
-        [MCA_BTL_ATOMIC_ADD] = GNI_FMA_ATOMIC2_FFPADD,
-        [MCA_BTL_ATOMIC_MIN] = GNI_FMA_ATOMIC2_FFPMIN,
-        [MCA_BTL_ATOMIC_MAX] = GNI_FMA_ATOMIC2_FFPMAX,
-    },
-};
-
-int mca_btl_ugni_aop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
-                      uint64_t remote_address, mca_btl_base_registration_handle_t *remote_handle,
-                      mca_btl_base_atomic_op_t op, uint64_t operand, int flags, int order,
-                      mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
-{
-    gni_mem_handle_t dummy = {0, 0};
-    mca_btl_ugni_post_descriptor_t post_desc;
-    int gni_op, type;
-    size_t size;
-
-    size = (MCA_BTL_ATOMIC_FLAG_32BIT & flags) ? 4 : 8;
-    if (MCA_BTL_ATOMIC_FLAG_FLOAT & flags) {
-        type = (MCA_BTL_ATOMIC_FLAG_32BIT & flags) ? PMIX_FLOAT : PMIX_DOUBLE;
-    } else {
-        type = (MCA_BTL_ATOMIC_FLAG_32BIT & flags) ? PMIX_INT32 : PMIX_INT64;
-    }
-
-    gni_op = amo_cmds[type][op];
-    if (0 == gni_op) {
-        return OPAL_ERR_NOT_SUPPORTED;
-    }
-
-    init_post_desc (&post_desc, endpoint, order, GNI_POST_AMO, 0, dummy, remote_address,
-                    remote_handle->gni_handle, size, 0, cbfunc, cbcontext, cbdata,
-                    NULL);
-    post_desc.gni_desc.amo_cmd = gni_op;
-    post_desc.gni_desc.first_operand = operand;
-
-    return mca_btl_ugni_endpoint_post_fma (endpoint, &post_desc);
-}
-
-int mca_btl_ugni_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
-                       void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                       mca_btl_base_registration_handle_t *remote_handle, mca_btl_base_atomic_op_t op,
-                       uint64_t operand, int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
-                       void *cbcontext, void *cbdata)
-{
-    mca_btl_ugni_post_descriptor_t post_desc;
-    int gni_op, type;
-    size_t size;
-
-    size = (MCA_BTL_ATOMIC_FLAG_32BIT & flags) ? 4 : 8;
-    if (MCA_BTL_ATOMIC_FLAG_FLOAT & flags) {
-        type = (MCA_BTL_ATOMIC_FLAG_32BIT & flags) ? PMIX_FLOAT : PMIX_DOUBLE;
-    } else {
-        type = (MCA_BTL_ATOMIC_FLAG_32BIT & flags) ? PMIX_INT32 : PMIX_INT64;
-    }
-
-    gni_op = famo_cmds[type][op];
-    if (0 == gni_op) {
-        return OPAL_ERR_NOT_SUPPORTED;
-    }
-
-    init_post_desc (&post_desc, endpoint, order, GNI_POST_AMO, (intptr_t) local_address,
-                    local_handle->gni_handle, remote_address, remote_handle->gni_handle,
-                    size, 0, cbfunc, cbcontext, cbdata, local_handle);
-    post_desc.gni_desc.amo_cmd = gni_op;
-    post_desc.gni_desc.first_operand = operand;
-
-    return mca_btl_ugni_endpoint_post_fma (endpoint, &post_desc);
-}
-
-int mca_btl_ugni_acswap (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
-                         void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                         mca_btl_base_registration_handle_t *remote_handle, uint64_t compare, uint64_t value, int flags,
-                         int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
-{
-    mca_btl_ugni_post_descriptor_t post_desc;
-    size_t size;
-    int gni_op;
-
-    gni_op = (MCA_BTL_ATOMIC_FLAG_32BIT & flags) ? GNI_FMA_ATOMIC2_FCSWAP_S : GNI_FMA_ATOMIC_CSWAP;
-    size = (MCA_BTL_ATOMIC_FLAG_32BIT & flags) ? 4 : 8;
-
-    init_post_desc (&post_desc, endpoint, order, GNI_POST_AMO, (intptr_t) local_address,
-                    local_handle->gni_handle, remote_address, remote_handle->gni_handle, size, 0,
-                    cbfunc, cbcontext, cbdata, local_handle);
-    post_desc.gni_desc.amo_cmd = gni_op;
-    post_desc.gni_desc.first_operand = compare;
-    post_desc.gni_desc.second_operand = value;
-
-    return mca_btl_ugni_endpoint_post_fma (endpoint, &post_desc);
-}
diff --git a/opal/mca/btl/ugni/btl_ugni_component.c b/opal/mca/btl/ugni/btl_ugni_component.c
deleted file mode 100644
index a974f29f03b..00000000000
--- a/opal/mca/btl/ugni/btl_ugni_component.c
+++ /dev/null
@@ -1,783 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2011      UT-Battelle, LLC. All rights reserved.
- * Copyright (c) 2017      Intel, Inc.  All rights reserved.
- * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "btl_ugni.h"
-#include "btl_ugni_frag.h"
-#include "btl_ugni_rdma.h"
-#include "btl_ugni_smsg.h"
-
-#include "opal/util/sys_limits.h"
-#include "opal/util/printf.h"
-
-#include <stdlib.h>
-#include <fcntl.h>
-#include <ctype.h>
-
-#include "opal/memoryhooks/memory.h"
-#include "opal/runtime/opal_params.h"
-
-#include "opal/mca/base/mca_base_pvar.h"
-
-static int btl_ugni_component_register(void);
-static int btl_ugni_component_open(void);
-static int btl_ugni_component_close(void);
-static mca_btl_base_module_t **mca_btl_ugni_component_init(int *, bool, bool);
-static int mca_btl_ugni_component_progress(void);
-static unsigned long mca_btl_ugni_ugni_page_size = 0;
-
-mca_btl_ugni_component_t mca_btl_ugni_component = {
-    .super = {
-        /* First, the mca_base_component_t struct containing meta information
-           about the component itself */
-        .btl_version = {
-            MCA_BTL_DEFAULT_VERSION("ugni"),
-            .mca_open_component = btl_ugni_component_open,
-            .mca_close_component = btl_ugni_component_close,
-            .mca_register_component_params = btl_ugni_component_register,
-        },
-        .btl_data = {
-            .param_field = MCA_BASE_METADATA_PARAM_CHECKPOINT
-        },
-        .btl_init = mca_btl_ugni_component_init,
-        .btl_progress = mca_btl_ugni_component_progress,
-    }
-};
-
-mca_base_var_enum_value_t rcache_values[] = {
-    {MCA_BTL_UGNI_RCACHE_UDREG, "udreg"},
-    {MCA_BTL_UGNI_RCACHE_GRDMA, "grdma"},
-    {-1, NULL} /* sentinal */
-};
-
-mca_base_var_enum_value_flag_t cdm_flags[] = {
-    {.flag = GNI_CDM_MODE_FORK_NOCOPY, .string = "fork-no-copy", .conflicting_flag = GNI_CDM_MODE_FORK_FULLCOPY | GNI_CDM_MODE_FORK_PARTCOPY},
-    {.flag = GNI_CDM_MODE_FORK_FULLCOPY, .string = "fork-full-copy", .conflicting_flag = GNI_CDM_MODE_FORK_NOCOPY | GNI_CDM_MODE_FORK_PARTCOPY},
-    {.flag = GNI_CDM_MODE_FORK_PARTCOPY, .string = "fork-part-copy", .conflicting_flag = GNI_CDM_MODE_FORK_NOCOPY | GNI_CDM_MODE_FORK_FULLCOPY},
-    {.flag = GNI_CDM_MODE_ERR_NO_KILL, .string = "err-no-kill", .conflicting_flag = GNI_CDM_MODE_ERR_ALL_KILL},
-    {.flag = GNI_CDM_MODE_ERR_ALL_KILL, .string = "err-all-kill", .conflicting_flag = GNI_CDM_MODE_ERR_NO_KILL},
-    {.flag = GNI_CDM_MODE_FAST_DATAGRAM_POLL, .string = "fast-datagram-poll", .conflicting_flag = 0},
-    {.flag = GNI_CDM_MODE_BTE_SINGLE_CHANNEL, .string = "bte-single-channel", .conflicting_flag = 0},
-    {.flag = GNI_CDM_MODE_USE_PCI_IOMMU, .string = "use-pci-iommu", .conflicting_flag = 0},
-    {.flag = GNI_CDM_MODE_MDD_DEDICATED, .string = "mdd-dedicated", .conflicting_flag = GNI_CDM_MODE_MDD_SHARED},
-    {.flag = GNI_CDM_MODE_MDD_SHARED, .string = "mdd-shared", .conflicting_flag = GNI_CDM_MODE_MDD_DEDICATED},
-    {.flag = GNI_CDM_MODE_FMA_DEDICATED, .string = "fma-dedicated", .conflicting_flag = GNI_CDM_MODE_FMA_SHARED},
-    {.flag = GNI_CDM_MODE_FMA_SHARED, .string = "fma-shared", .conflicting_flag = GNI_CDM_MODE_FMA_DEDICATED},
-    {.flag = GNI_CDM_MODE_CACHED_AMO_ENABLED, .string = "cached-amo-enabled", .conflicting_flag = 0},
-    {.flag = GNI_CDM_MODE_CQ_NIC_LOCAL_PLACEMENT, .string = "cq-nic-placement", .conflicting_flag = 0},
-    {.flag = GNI_CDM_MODE_FMA_SMALL_WINDOW, .string = "fma-small-window", .conflicting_flag = 0},
-    {.string = NULL}
-};
-
-static inline int mca_btl_ugni_get_stat (const mca_base_pvar_t *pvar, void *value, void *obj)
-{
-    gni_statistic_t statistic = (gni_statistic_t) (intptr_t) pvar->ctx;
-    gni_return_t rc = GNI_RC_SUCCESS;
-
-    for (int i = 0 ; i < mca_btl_ugni_component.virtual_device_count ; ++i) {
-        rc = GNI_GetNicStat (mca_btl_ugni_component.modules[0].devices[i].dev_handle, statistic,
-                             ((unsigned int *) value) + i);
-    }
-
-    return mca_btl_rc_ugni_to_opal (rc);
-}
-
-static inline int mca_btl_ugni_notify_stat (mca_base_pvar_t *pvar, mca_base_pvar_event_t event, void *obj, int *count)
-{
-    if (MCA_BASE_PVAR_HANDLE_BIND == event) {
-        /* one value for each virtual device handle */
-        *count = mca_btl_ugni_component.virtual_device_count;
-    }
-
-    return OPAL_SUCCESS;
-}
-
-static int btl_ugni_component_register(void)
-{
-    mca_base_var_enum_t *new_enum;
-    gni_nic_device_t device_type;
-    char *mpool_hints_tmp = NULL;
-    int rc;
-
-    (void) mca_base_var_group_component_register(&mca_btl_ugni_component.super.btl_version,
-                                                 "uGNI byte transport layer");
-
-    mca_btl_ugni_component.ugni_free_list_num = 8;
-    (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version,
-                                           "free_list_num", NULL, MCA_BASE_VAR_TYPE_INT,
-                                           NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-                                           OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL,
-                                           &mca_btl_ugni_component.ugni_free_list_num);
-    mca_btl_ugni_component.ugni_free_list_max = 4096;
-    (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version,
-                                           "free_list_max", NULL, MCA_BASE_VAR_TYPE_INT,
-                                           NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-                                           OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL,
-                                           &mca_btl_ugni_component.ugni_free_list_max);
-    mca_btl_ugni_component.ugni_free_list_inc = 64;
-    (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version,
-                                           "free_list_inc", NULL, MCA_BASE_VAR_TYPE_INT,
-                                           NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-                                           OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL,
-                                           &mca_btl_ugni_component.ugni_free_list_inc);
-
-    mca_btl_ugni_component.ugni_eager_num = 16;
-    (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version,
-                                           "eager_num", NULL, MCA_BASE_VAR_TYPE_INT,
-                                           NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-                                           OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL,
-                                           &mca_btl_ugni_component.ugni_eager_num);
-    mca_btl_ugni_component.ugni_eager_max = 128;
-    (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version,
-                                           "eager_max", NULL, MCA_BASE_VAR_TYPE_INT,
-                                           NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-                                           OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL,
-                                           &mca_btl_ugni_component.ugni_eager_max);
-    mca_btl_ugni_component.ugni_eager_inc = 16;
-    (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version,
-                                           "eager_inc", NULL, MCA_BASE_VAR_TYPE_INT,
-                                           NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-                                           OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL,
-                                           &mca_btl_ugni_component.ugni_eager_inc);
-
-    mca_btl_ugni_component.remote_cq_size = 40000;
-    (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version,
-                                           "remote_cq_size", "Remote SMSG completion queue "
-                                           "size (default 40000)", MCA_BASE_VAR_TYPE_INT,
-                                           NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-                                           OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL,
-                                           &mca_btl_ugni_component.remote_cq_size);
-
-    mca_btl_ugni_component.local_cq_size = 8192;
-    (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version,
-                                           "local_cq_size", "Local SMSG completion queue size "
-                                           "(default 8k)", MCA_BASE_VAR_TYPE_INT,
-                                           NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-                                           OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL,
-                                           &mca_btl_ugni_component.local_cq_size);
-
-    mca_btl_ugni_component.local_rdma_cq_size = 1024;
-    (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version,
-                                           "local_rdma_cq_size", "Local FMA/RDMA completion queue size "
-                                           "(default: 1024)",MCA_BASE_VAR_TYPE_INT, NULL, 0,
-                                           MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5,
-                                           MCA_BASE_VAR_SCOPE_LOCAL,
-                                           &mca_btl_ugni_component.local_rdma_cq_size);
-
-    mca_btl_ugni_component.ugni_smsg_limit = 0;
-    (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version,
-                                           "smsg_limit", "Maximum size message that "
-                                           "will be sent using the SMSG/MSGQ protocol "
-                                           "(0 - autoselect(default), 16k max)",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0,
-                                           MCA_BASE_VAR_FLAG_SETTABLE,
-                                           OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL,
-                                           &mca_btl_ugni_component.ugni_smsg_limit);
-
-    mca_btl_ugni_component.smsg_max_credits = 32;
-    (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version,
-                                           "smsg_max_credits", "Maximum number of "
-                                           "outstanding SMSG/MSGQ message (default 32)",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0,
-                                           MCA_BASE_VAR_FLAG_SETTABLE,
-                                           OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL,
-                                           &mca_btl_ugni_component.smsg_max_credits);
-
-#if OPAL_C_HAVE__THREAD_LOCAL
-    mca_btl_ugni_component.bind_threads_to_devices = true;
-
-    (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version,
-                                           "bind_devices", "Bind threads to virtual "
-                                           "devices. In general this should improve "
-                                           "RDMA performance (default: true)",
-                                           MCA_BASE_VAR_TYPE_BOOL, NULL, 0,
-                                           MCA_BASE_VAR_FLAG_SETTABLE,
-                                           OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL,
-                                           &mca_btl_ugni_component.bind_threads_to_devices);
-#endif
-
-    mca_btl_ugni_component.ugni_fma_limit = -1;
-    (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version,
-                                           "fma_limit", "Default maximum size message that "
-                                           "will be sent using the FMA (Fast Memory "
-                                           "Access) protocol (default: -1 (don't use), 64k max)",
-                                           MCA_BASE_VAR_TYPE_LONG, NULL, 0,
-                                           MCA_BASE_VAR_FLAG_SETTABLE | MCA_BASE_VAR_FLAG_DEPRECATED,
-                                           OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL,
-                                           &mca_btl_ugni_component.ugni_fma_limit);
-
-    mca_btl_ugni_component.ugni_fma_get_limit = 2048;
-    (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version,
-                                           "fma_get_limit", "Maximum size message that "
-                                           "will be sent using the FMA (Fast Memory "
-                                           "Access) protocol for get (default 2k, "
-                                           "64k max)",
-                                           MCA_BASE_VAR_TYPE_LONG, NULL, 0,
-                                           MCA_BASE_VAR_FLAG_SETTABLE,
-                                           OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL,
-                                           &mca_btl_ugni_component.ugni_fma_get_limit);
-
-    mca_btl_ugni_component.ugni_fma_put_limit = 4096;
-    (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version,
-                                           "fma_put_limit", "Maximum size message that "
-                                           "will be sent using the FMA (Fast Memory "
-                                           "Access) protocol for put (default: 4k, "
-                                           "64k max)",
-                                           MCA_BASE_VAR_TYPE_LONG, NULL, 0,
-                                           MCA_BASE_VAR_FLAG_SETTABLE,
-                                           OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL,
-                                           &mca_btl_ugni_component.ugni_fma_put_limit);
-
-    mca_btl_ugni_component.rdma_max_retries = 16;
-    (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version,
-                                           "rdma_max_retries", NULL, MCA_BASE_VAR_TYPE_INT,
-                                           NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-                                           OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL,
-                                           &mca_btl_ugni_component.rdma_max_retries);
-
-    mca_btl_ugni_component.smsg_max_retries = 16;
-    (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version,
-                                           "smsg_max_retries", NULL, MCA_BASE_VAR_TYPE_INT,
-                                           NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-                                           OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL,
-                                           &mca_btl_ugni_component.smsg_max_retries);
-
-    mca_btl_ugni_component.max_mem_reg = 0;
-    (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version,
-                                           "max_mem_reg", "Maximum number of "
-                                           "memory registrations a process can "
-                                           "hold (0 - autoselect, -1 - unlimited)"
-                                           " (default 0)", MCA_BASE_VAR_TYPE_INT,
-                                           NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-                                           OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_LOCAL,
-                                           &mca_btl_ugni_component.max_mem_reg);
-
-    mca_btl_ugni_component.mbox_increment = 0;
-    (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version,
-                                           "mbox_inc", "Number of SMSG mailboxes to "
-                                           "allocate in each block (0 - autoselect(default))",
-                                           MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0,
-                                           MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
-                                           MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.mbox_increment);
-
-    /* communication domain flags */
-    rc = mca_base_var_enum_create_flag ("btl_ugni_cdm_flags", cdm_flags, (mca_base_var_enum_flag_t **) &new_enum);
-    if (OPAL_SUCCESS != rc) {
-        return rc;
-    }
-
-    mca_btl_ugni_component.cdm_flags = GNI_CDM_MODE_FORK_PARTCOPY | GNI_CDM_MODE_ERR_NO_KILL | GNI_CDM_MODE_FAST_DATAGRAM_POLL |
-        GNI_CDM_MODE_MDD_SHARED | GNI_CDM_MODE_FMA_SHARED | GNI_CDM_MODE_FMA_SMALL_WINDOW;
-    mca_btl_ugni_component.cdm_flags_id = mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version,
-                                           "cdm_flags", "Flags to set when creating a communication domain "
-                                           " (default: fork-full-copy,cached-amo-enabled,err-no-kill,fast-datagram-poll,"
-                                           "fma-shared,fma-small-window)",
-                                           MCA_BASE_VAR_TYPE_UNSIGNED_INT, new_enum, 0,
-                                           MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
-                                           MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.cdm_flags);
-    OBJ_RELEASE(new_enum);
-
-    mca_btl_ugni_component.virtual_device_count = 0;
-    (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version,
-                                           "virtual_device_count", "Number of virtual devices to create. Higher numbers may "
-                                           "result in better performance when using threads. (default: 0 (auto), max: 128)",
-                                           MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0,
-                                           MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
-                                           MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.virtual_device_count);
-
-    /* determine if there are get alignment restrictions */
-    GNI_GetDeviceType (&device_type);
-
-
-    mca_btl_ugni_component.smsg_page_size = 2 << 20;
-    if (GNI_DEVICE_GEMINI == device_type) {
-        if (access ("/sys/class/gemini/ghal0/mrt", R_OK)) {
-            int fd = open ("/sys/class/gemini/ghal0/mrt", O_RDONLY);
-            char buffer[10];
-
-            if (0 <= fd) {
-                memset (buffer, 0, sizeof (buffer));
-                read (fd, buffer, sizeof (buffer) - 1);
-                close (fd);
-                mca_btl_ugni_ugni_page_size = strtol (buffer, NULL, 10) * 1024;
-                mca_btl_ugni_component.smsg_page_size = mca_btl_ugni_ugni_page_size;
-            }
-        }
-    }
-
-    (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version,
-                                           "smsg_page_size", "Page size to use for SMSG mailbox allocation (default: detect)",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
-                                           MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.smsg_page_size);
-
-    mca_btl_ugni_component.progress_thread_requested = 0;
-    (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version,
-                                           "request_progress_thread",
-                                           "Enable to request ugni btl progress thread - requires MPI_THREAD_MULTIPLE support",
-                                           MCA_BASE_VAR_TYPE_BOOL, NULL, 0,
-                                           MCA_BASE_VAR_FLAG_SETTABLE,
-                                           OPAL_INFO_LVL_3,
-                                           MCA_BASE_VAR_SCOPE_LOCAL,
-                                           &mca_btl_ugni_component.progress_thread_requested);
-
-    /* performance variables */
-    mca_btl_ugni_progress_thread_wakeups = 0;
-    (void) mca_base_component_pvar_register(&mca_btl_ugni_component.super.btl_version,
-                                            "progress_thread_wakeups", "Number of times the progress thread "
-                                            "has been woken", OPAL_INFO_LVL_9, MCA_BASE_PVAR_CLASS_COUNTER,
-                                            MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, MCA_BASE_VAR_BIND_NO_OBJECT,
-                                            MCA_BASE_PVAR_FLAG_READONLY | MCA_BASE_PVAR_FLAG_CONTINUOUS, NULL,
-                                            NULL, NULL, &mca_btl_ugni_progress_thread_wakeups);
-
-    /* register network statistics as performance variables */
-    for (int i = 0 ; i < GNI_NUM_STATS ; ++i) {
-        char name[128], desc[128];
-        size_t str_len = strlen (gni_statistic_str[i]);
-
-        assert (str_len < sizeof (name));
-
-        /* we can get an all-caps string for the variable from gni_statistic_str. need to make it lowercase
-         * to match ompi standards */
-        for (size_t j = 0 ; j < str_len ; ++j) {
-            name[j] = tolower (gni_statistic_str[i][j]);
-            desc[j] = ('_' == name[j]) ? ' ' : name[j];
-        }
-
-        name[str_len] = '\0';
-        desc[str_len] = '\0';
-
-        (void) mca_base_component_pvar_register (&mca_btl_ugni_component.super.btl_version, name, desc,
-                                                 OPAL_INFO_LVL_4, MCA_BASE_PVAR_CLASS_COUNTER,
-                                                 MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, MCA_BASE_VAR_BIND_NO_OBJECT,
-                                                 MCA_BASE_PVAR_FLAG_READONLY | MCA_BASE_PVAR_FLAG_CONTINUOUS,
-                                                 mca_btl_ugni_get_stat, NULL, mca_btl_ugni_notify_stat,
-                                                 (void *) (intptr_t) i);
-    }
-
-    /* btl/ugni can only support only a fixed set of rcache components (these rcache components have compatible resource
-     * structures) */
-    rc = mca_base_var_enum_create ("btl_ugni_rcache", rcache_values, &new_enum);
-    if (OPAL_SUCCESS != rc) {
-        return rc;
-    }
-
-    /* NTH: there are known *serious* performance issues with udreg. if they are ever resolved it is the preferred rcache */
-    mca_btl_ugni_component.rcache_type = MCA_BTL_UGNI_RCACHE_GRDMA;
-    (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version,
-                                           "rcache", "registration cache to use (default: grdma)", MCA_BASE_VAR_TYPE_INT, new_enum,
-                                           0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
-                                           MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.rcache_type);
-    OBJ_RELEASE(new_enum);
-
-    if (mca_btl_ugni_ugni_page_size) {
-        rc = opal_asprintf (&mpool_hints_tmp, "page_size=%lu", mca_btl_ugni_ugni_page_size);
-        if (rc < 0) {
-            return OPAL_ERR_OUT_OF_RESOURCE;
-        }
-
-        mca_btl_ugni_component.mpool_hints = mpool_hints_tmp;
-    } else {
-        mca_btl_ugni_component.mpool_hints = "page_size=2M";
-    }
-
-    (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version,
-                                           "mpool_hints", "hints to use when selecting a memory pool (default: "
-                                           "\"page_size=2M\")", MCA_BASE_VAR_TYPE_STRING, NULL, 0,
-                                           MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
-                                           MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.mpool_hints);
-    free (mpool_hints_tmp);
-
-    /* ensure we loose send exclusivity to sm and vader if they are enabled */
-    mca_btl_ugni_module.super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_HIGH - 2;
-
-    /* smsg threshold */
-    mca_btl_ugni_module.super.btl_eager_limit               = 8 * 1024;
-    mca_btl_ugni_module.super.btl_rndv_eager_limit          = 8 * 1024;
-    mca_btl_ugni_module.super.btl_rdma_pipeline_frag_size   = 4 * 1024 * 1024;
-    mca_btl_ugni_module.super.btl_max_send_size             = 8 * 1024;
-    mca_btl_ugni_module.super.btl_rdma_pipeline_send_length = 8 * 1024;
-
-    mca_btl_ugni_module.super.btl_get_limit = 1 * 1024 * 1024;
-
-    /*
-     * see def. of ALIGNMENT_MASK to figure this one out
-     */
-    /* both gemini and aries have a 4-byte alignment requirement on remote addresses */
-    mca_btl_ugni_module.super.btl_get_alignment = 4;
-
-    /* threshold for put */
-    mca_btl_ugni_module.super.btl_min_rdma_pipeline_size    = 8 * 1024;
-
-    mca_btl_ugni_module.super.btl_flags = MCA_BTL_FLAGS_SEND |
-        MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_ATOMIC_OPS |
-        MCA_BTL_FLAGS_ATOMIC_FOPS;
-    mca_btl_ugni_module.super.btl_atomic_flags = MCA_BTL_ATOMIC_SUPPORTS_ADD |
-        MCA_BTL_ATOMIC_SUPPORTS_AND | MCA_BTL_ATOMIC_SUPPORTS_OR | MCA_BTL_ATOMIC_SUPPORTS_XOR |
-        MCA_BTL_ATOMIC_SUPPORTS_CSWAP;
-
-    if (GNI_DEVICE_ARIES == device_type) {
-        /* aries supports additional atomic operations */
-        mca_btl_ugni_module.super.btl_atomic_flags |= MCA_BTL_ATOMIC_SUPPORTS_MIN | MCA_BTL_ATOMIC_SUPPORTS_MAX |
-            MCA_BTL_ATOMIC_SUPPORTS_LAND | MCA_BTL_ATOMIC_SUPPORTS_LOR | MCA_BTL_ATOMIC_SUPPORTS_LXOR |
-            MCA_BTL_ATOMIC_SUPPORTS_32BIT | MCA_BTL_ATOMIC_SUPPORTS_FLOAT;
-    }
-
-    mca_btl_ugni_module.super.btl_registration_handle_size = sizeof (mca_btl_base_registration_handle_t);
-
-    mca_btl_ugni_module.super.btl_bandwidth = 40000; /* Mbs */
-    mca_btl_ugni_module.super.btl_latency   = 2;     /* Microsecs */
-
-    mca_btl_ugni_module.super.btl_get_local_registration_threshold = 0;
-    mca_btl_ugni_module.super.btl_put_local_registration_threshold = mca_btl_ugni_component.ugni_fma_put_limit;
-
-    /* Call the BTL based to register its MCA params */
-    mca_btl_base_param_register(&mca_btl_ugni_component.super.btl_version,
-                                &mca_btl_ugni_module.super);
-
-    return OPAL_SUCCESS;
-}
-
-static int
-btl_ugni_component_open(void)
-{
-    mca_btl_ugni_component.ugni_num_btls = 0;
-    mca_btl_ugni_component.modules = NULL;
-
-    return OPAL_SUCCESS;
-}
-
-/*
- * component cleanup - sanity checking of queue lengths
- */
-static int
-btl_ugni_component_close(void)
-{
-    mca_btl_ugni_fini ();
-
-    free (mca_btl_ugni_component.modules);
-    mca_btl_ugni_component.modules = NULL;
-
-    return OPAL_SUCCESS;
-}
-
-static mca_btl_base_module_t **
-mca_btl_ugni_component_init (int *num_btl_modules,
-                             bool enable_progress_threads,
-                             bool enable_mpi_threads)
-{
-    struct mca_btl_base_module_t **base_modules;
-    mca_btl_ugni_module_t *ugni_modules;
-    int rc;
-
-    if (16384 < mca_btl_ugni_component.ugni_smsg_limit) {
-        mca_btl_ugni_component.ugni_smsg_limit = 16384;
-    }
-
-    if (65536 < mca_btl_ugni_component.ugni_fma_limit) {
-        mca_btl_ugni_component.ugni_fma_limit = 65536;
-    }
-
-    if (-1 != mca_btl_ugni_component.ugni_fma_limit) {
-        mca_btl_ugni_component.ugni_fma_get_limit = mca_btl_ugni_component.ugni_fma_limit;
-    } else if (65536 < mca_btl_ugni_component.ugni_fma_get_limit) {
-        mca_btl_ugni_component.ugni_fma_get_limit = 65536;
-    }
-
-    if (-1 != mca_btl_ugni_component.ugni_fma_limit) {
-        mca_btl_ugni_component.ugni_fma_put_limit = mca_btl_ugni_component.ugni_fma_limit;
-    } else if (65536 < mca_btl_ugni_component.ugni_fma_put_limit) {
-        mca_btl_ugni_component.ugni_fma_put_limit = 65536;
-    }
-
-    mca_btl_ugni_module.super.btl_put_local_registration_threshold = mca_btl_ugni_component.ugni_fma_put_limit;
-
-    /* limit the number of outstanding RDMA operations over all devices */
-    mca_btl_ugni_component.active_rdma_threshold = mca_btl_ugni_component.local_rdma_cq_size;
-
-    if (enable_mpi_threads && mca_btl_ugni_component.progress_thread_requested) {
-        mca_btl_ugni_component.progress_thread_enabled = 1;
-    }
-
-    /* Initialize ugni library and create communication domain */
-    rc = mca_btl_ugni_init();
-    if (OPAL_SUCCESS != rc) {
-        return NULL;
-    }
-
-    /* For now only create a single BTL module */
-    mca_btl_ugni_component.ugni_num_btls = 1;
-
-    BTL_VERBOSE(("btl/ugni initializing"));
-
-    ugni_modules = mca_btl_ugni_component.modules = (mca_btl_ugni_module_t *)
-        calloc (mca_btl_ugni_component.ugni_num_btls, sizeof (mca_btl_ugni_module_t));
-
-    if (OPAL_UNLIKELY(NULL == mca_btl_ugni_component.modules)) {
-        BTL_ERROR(("Failed malloc: %s:%d", __FILE__, __LINE__));
-        return NULL;
-    }
-
-    base_modules = (struct mca_btl_base_module_t **)
-        calloc (mca_btl_ugni_component.ugni_num_btls,
-                sizeof (struct mca_btl_base_module_t *));
-    if (OPAL_UNLIKELY(NULL == base_modules)) {
-        BTL_ERROR(("Malloc failed : %s:%d", __FILE__, __LINE__));
-        return NULL;
-    }
-
-    if (mca_btl_ugni_component.smsg_page_size != (unsigned long) opal_getpagesize ()) {
-        if (mca_btl_ugni_ugni_page_size > mca_btl_ugni_component.smsg_page_size) {
-            mca_btl_ugni_component.smsg_page_size = mca_btl_ugni_ugni_page_size;
-        }
-    }
-
-    mca_btl_ugni_module.super.btl_rdma_pipeline_send_length = mca_btl_ugni_module.super.btl_eager_limit;
-
-    rc = mca_btl_ugni_module_init (ugni_modules);
-    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-        BTL_ERROR(("Failed to initialize uGNI module @ %s:%d", __FILE__,
-                   __LINE__));
-        return NULL;
-    }
-
-    *base_modules = (mca_btl_base_module_t *) ugni_modules;
-
-    *num_btl_modules = mca_btl_ugni_component.ugni_num_btls;
-
-    BTL_VERBOSE(("btl/ugni done initializing %d module(s)", *num_btl_modules));
-
-    return base_modules;
-}
-
-int mca_btl_ugni_progress_datagram (mca_btl_ugni_device_t *device)
-{
-    mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_component.modules;
-    mca_btl_base_endpoint_t *ep = NULL;
-    gni_ep_handle_t handle;
-    int count = 0, rc;
-
-    rc = mca_btl_ugni_get_datagram (ugni_module, device, &handle, &ep);
-    if (1 != rc) {
-        return rc;
-    }
-
-    BTL_VERBOSE(("remote datagram completion on handle %p", (void*)handle));
-
-    /* if this is a wildcard endpoint lookup the remote peer by the proc id we received */
-    if (handle == ugni_module->wildcard_ep) {
-        struct opal_proc_t *remote_proc = opal_proc_for_name (ugni_module->wc_remote_attr.proc_name);
-
-        BTL_VERBOSE(("received connection attempt on wildcard endpoint from proc: %s",
-                     OPAL_NAME_PRINT(ugni_module->wc_remote_attr.proc_name)));
-
-        ep = mca_btl_ugni_get_ep (&ugni_module->super, remote_proc);
-        if (OPAL_UNLIKELY(NULL == ep)) {
-            /* there is no way to recover from this error so just abort() */
-            BTL_ERROR(("could not find/allocate a btl endpoint for peer %s",
-                       OPAL_NAME_PRINT(ugni_module->wc_remote_attr.proc_name)));
-            abort ();
-            return OPAL_ERR_NOT_FOUND;
-        }
-    }
-
-    /* should not have gotten a NULL endpoint */
-    assert (NULL != ep);
-
-    BTL_VERBOSE(("got a datagram completion: ep = %p. wc = %d", (void *) ep, handle == ugni_module->wildcard_ep));
-
-    /* NTH: TODO -- error handling */
-    opal_mutex_lock (&ep->lock);
-    if (handle != ugni_module->wildcard_ep) {
-        /* directed post complete */
-        BTL_VERBOSE(("directed datagram complete for endpoint %p", (void *) ep));
-
-        ep->dg_posted = false;
-        (void) opal_atomic_add_fetch_32 (&ugni_module->active_datagrams, -1);
-    }
-
-    (void) mca_btl_ugni_ep_connect_progress (ep);
-    opal_mutex_unlock (&ep->lock);
-
-    if (MCA_BTL_UGNI_EP_STATE_CONNECTED == ep->state) {
-        /*  process messages waiting in the endpoint's smsg mailbox */
-        count = mca_btl_ugni_smsg_process (ep);
-    }
-
-    /* repost the wildcard datagram */
-    if (handle == ugni_module->wildcard_ep) {
-        mca_btl_ugni_wildcard_ep_post (ugni_module);
-    }
-
-    return count;
-}
-
-void mca_btl_ugni_handle_rdma_completions (mca_btl_ugni_module_t *ugni_module, mca_btl_ugni_device_t *device,
-                                           struct mca_btl_ugni_post_descriptor_t *post_desc, const int count)
-{
-    int bte_complete = 0;
-
-    for (int i = 0 ; i < count ; ++i) {
-        BTL_VERBOSE(("post descriptor complete. status: %d", post_desc[i].rc));
-
-        if (OPAL_UNLIKELY(OPAL_SUCCESS != post_desc[i].rc)) {
-            /* dump the post descriptor if in a debug build */
-            btl_ugni_dump_post_desc (post_desc + i);
-        }
-
-        bte_complete += post_desc[i].use_bte == true;
-
-        mca_btl_ugni_post_desc_complete (ugni_module, post_desc + i, post_desc[i].rc);
-    }
-
-    if (bte_complete > 0)  {
-        (void) OPAL_THREAD_FETCH_ADD32 (&ugni_module->active_rdma_count, -bte_complete);
-    }
-}
-
-static inline int mca_btl_ugni_progress_rdma (mca_btl_ugni_module_t *ugni_module, mca_btl_ugni_device_t *device,
-                                              mca_btl_ugni_cq_t *cq)
-{
-    mca_btl_ugni_post_descriptor_t post_desc[MCA_BTL_UGNI_COMPLETIONS_PER_LOOP];
-    int rc;
-
-    rc = mca_btl_ugni_cq_get_completed_desc (device, cq, post_desc, MCA_BTL_UGNI_COMPLETIONS_PER_LOOP);
-    if (0 >= rc) {
-        return rc;
-    }
-
-    BTL_VERBOSE(("got %d completed rdma descriptors", rc));
-
-    mca_btl_ugni_handle_rdma_completions (ugni_module, device, post_desc, rc);
-
-    return rc;
-}
-
-static inline int
-mca_btl_ugni_progress_wait_list (mca_btl_ugni_module_t *ugni_module)
-{
-    int rc = OPAL_SUCCESS;
-    opal_list_t tmplist;
-    opal_list_t *waitlist = &ugni_module->ep_wait_list;
-    mca_btl_base_endpoint_t *endpoint = NULL;
-    int count;
-
-    /* check the count before taking the lock to avoid unnecessary locking */
-    count = opal_list_get_size(waitlist);
-    if (0 == count) {
-        return 0;
-    }
-
-    /* Don't hold the wait-list lock while processing the list as that may lead
-     * to a deadlock.
-     * Instead, move the wait_list elements into a temporary list and work on that.*/
-    OBJ_CONSTRUCT(&tmplist, opal_list_t);
-    OPAL_THREAD_LOCK(&ugni_module->ep_wait_list_lock);
-    opal_list_join(&tmplist, opal_list_get_end(&tmplist), waitlist);
-    OPAL_THREAD_UNLOCK(&ugni_module->ep_wait_list_lock);
-    count = opal_list_get_size(&tmplist);
-    do {
-        endpoint = (mca_btl_base_endpoint_t *) opal_list_remove_first (&tmplist);
-        if (endpoint != NULL) {
-            rc = mca_btl_ugni_progress_send_wait_list (endpoint);
-
-            if (OPAL_SUCCESS != rc) {
-                opal_list_append (&tmplist, &endpoint->super);
-            } else {
-                endpoint->wait_listed = false;
-            }
-        }
-    } while (endpoint != NULL && --count > 0) ;
-
-    /* reinsert unfinished elements into the wait-list */
-    count = opal_list_get_size(&tmplist);
-    if (0 < count) {
-        OPAL_THREAD_LOCK(&ugni_module->ep_wait_list_lock);
-        opal_list_join(waitlist, opal_list_get_end(waitlist), &tmplist);
-        OPAL_THREAD_UNLOCK(&ugni_module->ep_wait_list_lock);
-    }
-    OBJ_DESTRUCT(&tmplist);
-
-    return rc;
-}
-
-static int mca_btl_ugni_component_progress (void)
-{
-    mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_component.modules;
-    int count = 0;
-
-    count += mca_btl_ugni_progress_remote_smsg (ugni_module);
-
-    if (ugni_module->active_datagrams) {
-        count += mca_btl_ugni_progress_datagram (ugni_module->devices);
-    }
-
-    for (int i = 0 ; i < mca_btl_ugni_component.virtual_device_count ; ++i) {
-        mca_btl_ugni_device_t *device = ugni_module->devices + i;
-
-        if (device->smsg_connections) {
-            count += mca_btl_ugni_progress_local_smsg (ugni_module, device);
-            mca_btl_ugni_progress_wait_list (ugni_module);
-        }
-
-        if (device->dev_rdma_local_cq.active_operations) {
-            count += mca_btl_ugni_progress_rdma (ugni_module, device, &device->dev_rdma_local_cq);
-        }
-
-        if (mca_btl_ugni_component.progress_thread_enabled && device->dev_rdma_local_irq_cq.active_operations) {
-            count += mca_btl_ugni_progress_rdma (ugni_module, device, &device->dev_rdma_local_irq_cq);
-        }
-    }
-
-    return count;
-}
-
-int mca_btl_ugni_flush (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint)
-{
-    mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_component.modules;
-
-    for (int i = 0 ; i < mca_btl_ugni_component.virtual_device_count ; ++i) {
-        mca_btl_ugni_device_t *device = ugni_module->devices + i;
-        /* spin on progress until all active operations are complete. it is tempting to
-         * take an initial count then wait until that many operations have been completed
-         * but it is impossible to tell if those are the operations the caller is waiting
-         * on. */
-        while (device->dev_rdma_local_cq.active_operations) {
-            (void) mca_btl_ugni_progress_rdma (ugni_module, device, &device->dev_rdma_local_cq);
-        }
-
-        /* mark that the device was recently flushed */
-        device->flushed = true;
-    }
-
-    return OPAL_SUCCESS;
-}
-
-void btl_ugni_dump_post_desc (mca_btl_ugni_post_descriptor_t *desc)
-{
-
-    fprintf (stderr, "desc->gni_desc.post_id          = %" PRIx64 "\n", desc->gni_desc.post_id);
-    fprintf (stderr, "desc->gni_desc.status           = %" PRIx64 "\n", desc->gni_desc.status);
-    fprintf (stderr, "desc->gni_desc.cq_mode_complete = %hu\n", desc->gni_desc.cq_mode_complete);
-    fprintf (stderr, "desc->gni_desc.type             = %d\n", desc->gni_desc.type);
-    fprintf (stderr, "desc->gni_desc.cq_mode          = %hu\n", desc->gni_desc.cq_mode);
-    fprintf (stderr, "desc->gni_desc.dlvr_mode        = %hu\n", desc->gni_desc.dlvr_mode);
-    fprintf (stderr, "desc->gni_desc.local_addr       = %" PRIx64 "\n", desc->gni_desc.local_addr);
-    fprintf (stderr, "desc->gni_desc.local_mem_hndl   = {%" PRIx64 ", %" PRIx64 "}\n", desc->gni_desc.local_mem_hndl.qword1,
-             desc->gni_desc.local_mem_hndl.qword2);
-    fprintf (stderr, "desc->gni_desc.remote_addr      = %" PRIx64 "\n", desc->gni_desc.remote_addr);
-    fprintf (stderr, "desc->gni_desc.remote_mem_hndl  = {%" PRIx64 ", %" PRIx64 "}\n", desc->gni_desc.remote_mem_hndl.qword1,
-             desc->gni_desc.remote_mem_hndl.qword2);
-    fprintf (stderr, "desc->gni_desc.length           = %" PRIu64 "\n", desc->gni_desc.length);
-    fprintf (stderr, "desc->gni_desc.rdma_mode        = %hu\n", desc->gni_desc.rdma_mode);
-    fprintf (stderr, "desc->gni_desc.amo_cmd          = %d\n", desc->gni_desc.amo_cmd);
-}
diff --git a/opal/mca/btl/ugni/btl_ugni_device.h b/opal/mca/btl/ugni/btl_ugni_device.h
deleted file mode 100644
index 0f58b06205a..00000000000
--- a/opal/mca/btl/ugni/btl_ugni_device.h
+++ /dev/null
@@ -1,579 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2011      UT-Battelle, LLC. All rights reserved.
- * Copyright (c) 2014      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      Intel, Inc.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-/**
- * @file This file contains wrappers for uGNI functionality. These wrappers are thread-safe
- * and intended to provide a way to measure various different ways to handle mutual exclusion
- * into the uGNI library (which is not thread safe). These functions are all defined to be
- * inline to limit the cost to non-threaded users.
- */
-
-#if !defined(BTL_UGNI_DEVICE_H)
-#define BTL_UGNI_DEVICE_H
-
-#include "btl_ugni_endpoint.h"
-#include "btl_ugni_frag.h"
-
-/* helper functions */
-/**
- * @brief Output an error message on CQ or completion error.
- *
- * @param[in] grc        GNI error from GNI_CqGetEvent or GNI_GetCompleted
- * @param[in] event_data event data from GNI_CqGetEvent
- *
- * This is a small function to print out an error if an error
- * was detected on a CQ event.
- */
-int mca_btl_ugni_event_fatal_error (gni_return_t grc, gni_cq_entry_t event_data);
-
-/**
- * @brief Attempt to re-post an rdma descriptor
- *
- * @param[in] rdma_desc  RDMA descriptor that failed
- * @param[in] event_data CQ event data
- *
- * @returns OPAL_SUCCESS if the descriptor was re-posted
- * @returns OPAL_ERROR otherwise
- *
- * This function checks if the error is recoverable and re-posts the
- * descriptor if possible. The device lock MUST be held when this
- * function is called.
- */
-int mca_btl_ugni_device_handle_event_error (struct mca_btl_ugni_rdma_desc_t *rdma_desc, gni_cq_entry_t event_data);
-
-typedef struct mca_btl_ugni_smsg_send_wtag_arg_t {
-    gni_ep_handle_t ep_handle;
-    void *hdr;
-    size_t hdr_len;
-    void *payload;
-    size_t payload_len;
-    uint32_t msg_id;
-    int tag;
-} mca_btl_ugni_smsg_send_wtag_arg_t;
-
-static inline int mca_btl_ugni_smsg_send_wtag_device (mca_btl_ugni_device_t *device, void *arg)
-{
-    mca_btl_ugni_smsg_send_wtag_arg_t *args = (mca_btl_ugni_smsg_send_wtag_arg_t *) arg;
-    gni_return_t grc;
-
-    grc = GNI_SmsgSendWTag (args->ep_handle, args->hdr, args->hdr_len, args->payload,
-                            args->payload_len, args->msg_id, args->tag);
-    device->dev_smsg_local_cq.active_operations += (GNI_RC_SUCCESS == grc);
-    return grc;
-}
-
-typedef struct mca_btl_ugni_smsg_get_next_wtag_arg_t {
-    gni_ep_handle_t ep_handle;
-    uintptr_t *data_ptr;
-    uint8_t *tag;
-} mca_btl_ugni_smsg_get_next_wtag_arg_t;
-
-static inline intptr_t mca_btl_ugni_smsg_get_next_wtag_device (mca_btl_ugni_device_t *device, void *arg)
-{
-    mca_btl_ugni_smsg_get_next_wtag_arg_t *args = (mca_btl_ugni_smsg_get_next_wtag_arg_t *) arg;
-    return GNI_SmsgGetNextWTag(args->ep_handle, (void **) args->data_ptr, args->tag);
-}
-
-static inline intptr_t mca_btl_ugni_smsg_release_device (mca_btl_ugni_device_t *device, void *arg)
-{
-    mca_btl_ugni_endpoint_handle_t *ep_handle = (mca_btl_ugni_endpoint_handle_t *) arg;
-
-    return GNI_SmsgRelease (ep_handle->gni_handle);
-}
-
-typedef struct mca_btl_ugni_cq_get_event_args_t {
-    mca_btl_ugni_cq_t *cq;
-    gni_cq_entry_t *event_data;
-} mca_btl_ugni_cq_get_event_args_t;
-
-static inline intptr_t mca_btl_ugni_cq_get_event_device (mca_btl_ugni_device_t *device, void *arg)
-{
-    mca_btl_ugni_cq_get_event_args_t *args = (mca_btl_ugni_cq_get_event_args_t *) arg;
-    gni_return_t rc;
-
-    rc = GNI_CqGetEvent (args->cq->gni_handle, args->event_data);
-    args->cq->active_operations -= (GNI_RC_NOT_DONE != rc);
-    return rc;
-}
-
-static inline intptr_t mca_btl_ugni_cq_clear_device (mca_btl_ugni_device_t *device, void *arg)
-{
-    gni_cq_handle_t cq = (gni_cq_handle_t) (intptr_t) arg;
-    gni_cq_entry_t event_data;
-    int rc;
-
-    do {
-        rc = GNI_CqGetEvent (cq, &event_data);
-    } while (GNI_RC_NOT_DONE != rc);
-
-    return OPAL_SUCCESS;
-}
-
-typedef struct mca_btl_ugni_gni_cq_get_event_args_t {
-    gni_cq_handle_t cq;
-    gni_cq_entry_t *event_data;
-} mca_btl_ugni_gni_cq_get_event_args_t;
-
-static inline intptr_t mca_btl_ugni_gni_cq_get_event_device (mca_btl_ugni_device_t *device, void *arg)
-{
-    mca_btl_ugni_gni_cq_get_event_args_t *args = (mca_btl_ugni_gni_cq_get_event_args_t *) arg;
-
-    return GNI_CqGetEvent (args->cq, args->event_data);
-}
-
-typedef struct mca_btl_ugni_cq_get_completed_desc_arg_t {
-    mca_btl_ugni_cq_t *cq;
-    mca_btl_ugni_post_descriptor_t *post_desc;
-    int count;
-} mca_btl_ugni_cq_get_completed_desc_arg_t;
-
-__opal_attribute_always_inline__
-static inline int _mca_btl_ugni_repost_rdma_desc_device (mca_btl_ugni_device_t *device, mca_btl_ugni_rdma_desc_t *rdma_desc)
-{
-    mca_btl_ugni_post_descriptor_t *post_desc = &rdma_desc->btl_ugni_desc;
-    int rc;
-
-    if (post_desc->use_bte) {
-        rc = GNI_PostRdma (rdma_desc->gni_handle, &post_desc->gni_desc);
-    } else {
-        rc = GNI_PostFma (rdma_desc->gni_handle, &post_desc->gni_desc);
-    }
-
-    return mca_btl_rc_ugni_to_opal (rc);
-}
-
-static inline intptr_t _mca_btl_ugni_cq_get_completed_desc_device (mca_btl_ugni_device_t *device, mca_btl_ugni_cq_t *cq,
-                                                                   mca_btl_ugni_post_descriptor_t *post_desc,
-                                                                   const int count, bool block)
-{
-    mca_btl_ugni_rdma_desc_t *rdma_desc;
-    gni_post_descriptor_t *desc;
-    gni_cq_entry_t event_data;
-    int rc, desc_index = 0;
-
-    for (desc_index = 0 ; desc_index < count && cq->active_operations ; ) {
-        int desc_rc = OPAL_SUCCESS;
-
-        rc = GNI_CqGetEvent (cq->gni_handle, &event_data);
-        if (GNI_RC_NOT_DONE == rc) {
-	    if (block) {
-		/* try again */
-		continue;
-	    }
-            break;
-        }
-
-	block = false;
-
-        rc = GNI_GetCompleted (cq->gni_handle, event_data, &desc);
-        if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc && GNI_RC_TRANSACTION_ERROR != rc)) {
-            return mca_btl_ugni_event_fatal_error (rc, event_data);
-        }
-
-        rdma_desc = MCA_BTL_UGNI_GNI_DESC_TO_RDMA_DESC(desc);
-
-        if (OPAL_UNLIKELY(!GNI_CQ_STATUS_OK(event_data))) {
-            desc_rc = mca_btl_ugni_device_handle_event_error (rdma_desc, event_data);
-            if (OPAL_LIKELY(OPAL_SUCCESS == desc_rc)) {
-                /* descriptor was re-posted */
-                continue;
-            }
-        }
-
-        /* copy back the descriptor only if additional processing is needed. in this case more processing
-         * is needed if a user callback is specified or the bte was in use. */
-        if (rdma_desc->btl_ugni_desc.cbfunc || rdma_desc->btl_ugni_desc.use_bte || OPAL_SUCCESS != desc_rc) {
-            post_desc[desc_index] = rdma_desc->btl_ugni_desc;
-            post_desc[desc_index++].rc = desc_rc;
-        }
-
-        /* return the descriptor while we have the lock. this is done so we can avoid using the
-         * free list atomics (as both push and pop are done with the lock) */
-        mca_btl_ugni_return_rdma_desc (rdma_desc);
-        --cq->active_operations;
-    }
-
-    return desc_index;
-}
-
-static inline intptr_t mca_btl_ugni_cq_get_completed_desc_device (mca_btl_ugni_device_t *device, void *arg0)
-{
-    mca_btl_ugni_cq_get_completed_desc_arg_t *args = (mca_btl_ugni_cq_get_completed_desc_arg_t *) arg0;
-
-    return _mca_btl_ugni_cq_get_completed_desc_device (device, args->cq, args->post_desc, args->count, false);
-}
-
-/* NTH: When posting FMA or RDMA descriptors it makes sense to try and clear out a completion
- * event after posting the descriptor. This probably gives us a couple of things:
- *   1) Good locality on the associated data structures (especially with FMA which may
- *      complete fairly quickly).
- *   2) Since we are already holding the lock it could mean fewer attempts to
- *      lock the device over the course of the program.
- *
- * As far as I can tell there is not reason to try and clear out more than a couple
- * completiong events. The code has been written to allow us to easily modify the
- * number reaped if we determine that there is a benefit to clearing a different
- * number of events. */
-
-/**
- * @brief Number of events to clear after posting a descriptor
- */
-#define MCA_BTL_UGNI_DEVICE_REAP_COUNT 4
-
-struct mca_btl_ugni_post_device_args_t {
-    mca_btl_ugni_post_descriptor_t *desc;
-    mca_btl_ugni_device_t *device;
-    int count;
-    mca_btl_ugni_post_descriptor_t completed[MCA_BTL_UGNI_DEVICE_REAP_COUNT];
-};
-
-static inline mca_btl_ugni_rdma_desc_t *
-mca_btl_ugni_get_rdma_desc_device (mca_btl_ugni_device_t *device, struct mca_btl_ugni_post_device_args_t *args, bool use_bte)
-{
-    mca_btl_ugni_post_descriptor_t *desc = args->desc;
-    mca_btl_ugni_rdma_desc_t *rdma_desc;
-
-    args->device = device;
-    args->count = 0;
-
-    do {
-        rdma_desc = mca_btl_ugni_alloc_rdma_desc (device, desc, use_bte);
-	if (OPAL_LIKELY(NULL != rdma_desc)) {
-	    return rdma_desc;
-	}
-
-        if (OPAL_LIKELY(NULL == rdma_desc && !args->count)) {
-	    args->count = _mca_btl_ugni_cq_get_completed_desc_device (device, &device->dev_rdma_local_cq,
-								      args->completed, MCA_BTL_UGNI_DEVICE_REAP_COUNT,
-								      true);
-	    continue;
-        }
-
-	return NULL;
-    } while (1);
-}
-
-
-static inline intptr_t mca_btl_ugni_post_fma_device (mca_btl_ugni_device_t *device, void *arg)
-{
-    struct mca_btl_ugni_post_device_args_t *args = (struct mca_btl_ugni_post_device_args_t *) arg;
-    mca_btl_ugni_rdma_desc_t *rdma_desc;
-    int rc;
-
-    rdma_desc = mca_btl_ugni_get_rdma_desc_device (device, args, false);
-    if (OPAL_UNLIKELY(NULL == rdma_desc)) {
-	return OPAL_ERR_TEMP_OUT_OF_RESOURCE;
-    }
-
-    BTL_VERBOSE(("Posting FMA descriptor %p with op_type %d, amo %d, remote_addr 0x%lx, "
-                 "length %lu", (void*)rdma_desc, rdma_desc->btl_ugni_desc.gni_desc.type, rdma_desc->btl_ugni_desc.gni_desc.amo_cmd,
-                 rdma_desc->btl_ugni_desc.gni_desc.remote_addr, rdma_desc->btl_ugni_desc.gni_desc.length));
-
-    rc = GNI_PostFma (rdma_desc->gni_handle, &rdma_desc->btl_ugni_desc.gni_desc);
-    if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc)) {
-        mca_btl_ugni_return_rdma_desc (rdma_desc);
-        return mca_btl_rc_ugni_to_opal (rc);
-    }
-
-    ++device->dev_rdma_local_cq.active_operations;
-
-    /* to improve bandwidth and latency it is ideal for all posting threads to also reap completions from
-     * the rdma completion queue. there are two optimizations here. 1) for bandwidth we only want to
-     * reap what is available now so more messages can be posted quickly, and 2) for latency (single
-     * put/get before flushing) we want to ensure the operation is complete. To some degree this is
-     * gaming the benchmark but it may benefit some application communication patterns without really
-     * hurting others (in theory). */
-    if (opal_using_threads ()) {
-	int count = args->count;
-	args->count += _mca_btl_ugni_cq_get_completed_desc_device (device, &device->dev_rdma_local_cq,
-								   args->completed + count,
-								   MCA_BTL_UGNI_DEVICE_REAP_COUNT - count,
-								   device->flushed);
-	device->flushed = false;
-    }
-
-    return OPAL_SUCCESS;
-}
-
-static inline intptr_t mca_btl_ugni_post_rdma_device (mca_btl_ugni_device_t *device, void *arg)
-{
-    struct mca_btl_ugni_post_device_args_t *args = (struct mca_btl_ugni_post_device_args_t *) arg;
-    mca_btl_ugni_rdma_desc_t *rdma_desc;
-    int rc;
-
-    rdma_desc = mca_btl_ugni_get_rdma_desc_device (device, args, true);
-    if (OPAL_UNLIKELY(NULL == rdma_desc)) {
-	return OPAL_ERR_TEMP_OUT_OF_RESOURCE;
-    }
-
-    /* pick the appropriate CQ */
-    rdma_desc->btl_ugni_desc.cq = mca_btl_ugni_component.progress_thread_enabled ? &device->dev_rdma_local_irq_cq :
-        &device->dev_rdma_local_cq;
-
-    BTL_VERBOSE(("Posting RDMA descriptor %p with op_type %d, amo %d, remote_addr 0x%lx, "
-                 "length %lu", (void*)rdma_desc, rdma_desc->btl_ugni_desc.gni_desc.type, rdma_desc->btl_ugni_desc.gni_desc.amo_cmd,
-                 rdma_desc->btl_ugni_desc.gni_desc.remote_addr, rdma_desc->btl_ugni_desc.gni_desc.length));
-
-    rc = GNI_PostRdma (rdma_desc->gni_handle, &rdma_desc->btl_ugni_desc.gni_desc);
-    if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc)) {
-        mca_btl_ugni_return_rdma_desc (rdma_desc);
-        return mca_btl_rc_ugni_to_opal (rc);
-    }
-
-    ++rdma_desc->btl_ugni_desc.cq->active_operations;
-
-    /* to improve bandwidth and latency it is ideal for all posting threads to also reap completions from
-     * the rdma completion queue. there are two optimizations here. 1) for bandwidth we only want to
-     * reap what is available now so more messages can be posted quickly, and 2) for latency (single
-     * put/get before flushing) we want to ensure the operation is complete. To some degree this is
-     * gaming the benchmark but it may benefit some application communication patterns without really
-     * hurting others (in theory). */
-    if (opal_using_threads ()) {
-	int count = args->count;
-	args->count += _mca_btl_ugni_cq_get_completed_desc_device (device, &device->dev_rdma_local_cq,
-								   args->completed + count,
-								   MCA_BTL_UGNI_DEVICE_REAP_COUNT - count,
-								   device->flushed);
-	device->flushed = false;
-    }
-
-    return OPAL_SUCCESS;
-}
-
-static inline intptr_t mca_btl_ugni_post_cqwrite_device (mca_btl_ugni_device_t *device, void *arg)
-{
-    mca_btl_ugni_post_descriptor_t *desc = (mca_btl_ugni_post_descriptor_t *) arg;
-    mca_btl_ugni_rdma_desc_t *rdma_desc;
-    int rc;
-
-    desc->gni_desc.src_cq_hndl = device->dev_rdma_local_cq.gni_handle;
-
-    rdma_desc = mca_btl_ugni_alloc_rdma_desc (device, desc, false);
-    if (OPAL_UNLIKELY(NULL == rdma_desc)) {
-        return OPAL_ERR_OUT_OF_RESOURCE;
-    }
-
-    rc = GNI_PostCqWrite (rdma_desc->gni_handle, &rdma_desc->btl_ugni_desc.gni_desc);
-    if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc)) {
-        mca_btl_ugni_return_rdma_desc (rdma_desc);
-    }
-
-    return mca_btl_rc_ugni_to_opal (rc);
-}
-
-typedef struct mca_btl_ugni_get_datagram_args_t {
-    mca_btl_ugni_module_t *ugni_module;
-    gni_ep_handle_t *handle;
-    mca_btl_base_endpoint_t **ep;
-} mca_btl_ugni_get_datagram_args_t;
-
-static inline intptr_t mca_btl_ugni_get_datagram_device (mca_btl_ugni_device_t *device, void *arg0)
-{
-    mca_btl_ugni_get_datagram_args_t *args = (mca_btl_ugni_get_datagram_args_t *) arg0;
-    uint32_t remote_addr, remote_id;
-    uint64_t datagram_id;
-    gni_post_state_t post_state;
-    gni_return_t grc;
-    uint64_t data;
-
-    grc = GNI_PostDataProbeById (device->dev_handle, &datagram_id);
-    if (OPAL_LIKELY(GNI_RC_SUCCESS != grc)) {
-        return 0;
-    }
-
-    data = datagram_id & ~(MCA_BTL_UGNI_DATAGRAM_MASK);
-
-    BTL_VERBOSE(("rc: %d, datgram_id: %" PRIx64 ", mask: %" PRIx64, grc, datagram_id, (uint64_t) (datagram_id & MCA_BTL_UGNI_DATAGRAM_MASK)));
-
-    if ((datagram_id & MCA_BTL_UGNI_DATAGRAM_MASK) == MCA_BTL_UGNI_CONNECT_DIRECTED_ID) {
-        *(args->ep) = (mca_btl_base_endpoint_t *) opal_pointer_array_get_item (&args->ugni_module->endpoints, data);
-        *(args->handle) = (*args->ep)->smsg_ep_handle.gni_handle;
-    } else {
-        *(args->handle) = args->ugni_module->wildcard_ep;
-    }
-
-    /* wait for the incoming datagram to complete (in case it isn't) */
-    grc = GNI_EpPostDataWaitById (*args->handle, datagram_id, -1, &post_state,
-                                  &remote_addr, &remote_id);
-    if (GNI_RC_SUCCESS != grc) {
-        BTL_ERROR(("GNI_EpPostDataWaitById failed with rc = %d", grc));
-        return mca_btl_rc_ugni_to_opal (grc);
-    }
-
-    BTL_VERBOSE(("handled datagram completion. post_state: %d, remote_addr: %u, remote_id: %u, directed?: %d",
-                 post_state, remote_addr, remote_id, (datagram_id & MCA_BTL_UGNI_DATAGRAM_MASK) == MCA_BTL_UGNI_CONNECT_DIRECTED_ID));
-
-    return 1;
-}
-
-typedef struct mca_btl_ugni_reg_mem_args_t {
-    mca_btl_ugni_module_t *ugni_module;
-    void *base;
-    size_t size;
-    mca_btl_ugni_reg_t *ugni_reg;
-    gni_cq_handle_t cq;
-    int flags;
-} mca_btl_ugni_reg_mem_args_t;
-
-static intptr_t mca_btl_ugni_reg_mem_device (mca_btl_ugni_device_t *device, void *arg)
-{
-    mca_btl_ugni_reg_mem_args_t *args = (mca_btl_ugni_reg_mem_args_t *) arg;
-    gni_return_t rc;
-
-    rc = GNI_MemRegister (device->dev_handle, (uint64_t) args->base, args->size, args->cq,
-                          args->flags, -1, &args->ugni_reg->handle.gni_handle);
-    if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc)) {
-        return OPAL_ERR_OUT_OF_RESOURCE;
-    }
-
-    return OPAL_SUCCESS;
-}
-
-typedef struct mca_btl_ugni_dereg_mem_arg_t {
-    mca_btl_ugni_module_t *ugni_module;
-    mca_btl_ugni_reg_t *ugni_reg;
-} mca_btl_ugni_dereg_mem_arg_t;
-
-static intptr_t mca_btl_ugni_dereg_mem_device (mca_btl_ugni_device_t *device, void *arg)
-{
-    mca_btl_ugni_dereg_mem_arg_t *args = (mca_btl_ugni_dereg_mem_arg_t *) arg;
-    gni_return_t rc;
-
-    rc = GNI_MemDeregister (device->dev_handle, &args->ugni_reg->handle.gni_handle);
-    return mca_btl_rc_ugni_to_opal (rc);
-}
-
-/* multi-thread safe interface to uGNI */
-
-static inline int mca_btl_ugni_endpoint_smsg_send_wtag (mca_btl_base_endpoint_t *endpoint, void *hdr, size_t hdr_len,
-                                                        void *payload, size_t payload_len, uint32_t msg_id, int tag)
-{
-    mca_btl_ugni_smsg_send_wtag_arg_t args = {.ep_handle = endpoint->smsg_ep_handle.gni_handle,
-                                              .hdr = hdr, .hdr_len = hdr_len, .payload = payload,
-                                              .payload_len = payload_len, .msg_id = msg_id,
-                                              .tag = tag};
-    mca_btl_ugni_device_t *device = endpoint->smsg_ep_handle.device;
-    return (int) mca_btl_ugni_device_serialize (device, (mca_btl_ugni_device_serialize_fn_t) mca_btl_ugni_smsg_send_wtag_device, &args);
-}
-
-static inline int mca_btl_ugni_smsg_get_next_wtag (mca_btl_ugni_endpoint_handle_t *ep_handle, uintptr_t *data_ptr, uint8_t *tag)
-{
-    mca_btl_ugni_device_t *device = ep_handle->device;
-    mca_btl_ugni_smsg_get_next_wtag_arg_t args = {.ep_handle = ep_handle->gni_handle, .data_ptr = data_ptr, .tag = tag};
-
-    return (int) mca_btl_ugni_device_serialize (device, (mca_btl_ugni_device_serialize_fn_t) mca_btl_ugni_smsg_get_next_wtag_device, &args);
-}
-
-static inline int mca_btl_ugni_smsg_release (mca_btl_ugni_endpoint_handle_t *ep_handle)
-{
-    mca_btl_ugni_device_t *device = ep_handle->device;
-
-    return (int) mca_btl_ugni_device_serialize (device, (mca_btl_ugni_device_serialize_fn_t) mca_btl_ugni_smsg_release_device, ep_handle);
-}
-
-static inline void mca_btl_ugni_cq_clear (mca_btl_ugni_device_t *device, gni_cq_handle_t cq)
-{
-    (void) mca_btl_ugni_device_serialize (device, (mca_btl_ugni_device_serialize_fn_t) mca_btl_ugni_cq_clear_device, (void *) (intptr_t) cq);
-}
-
-static inline int mca_btl_ugni_cq_get_event (mca_btl_ugni_device_t *device, mca_btl_ugni_cq_t *cq, gni_cq_entry_t *event_data)
-{
-    mca_btl_ugni_cq_get_event_args_t args = {.cq = cq, .event_data = event_data};
-    /* NTH: normally there would be a check for any outstanding CQ operations but there seems
-     * to be a reason to check the local SMSG completion queue anyway. since this function
-     * only handled the SMSG local completion queue not checking here should be fine and
-     * should not impact performance. */
-    return (int) mca_btl_ugni_device_serialize (device, (mca_btl_ugni_device_serialize_fn_t) mca_btl_ugni_cq_get_event_device, &args);
-}
-
-static inline int mca_btl_ugni_gni_cq_get_event (mca_btl_ugni_device_t *device, gni_cq_handle_t cq, gni_cq_entry_t *event_data)
-{
-    mca_btl_ugni_gni_cq_get_event_args_t args = {.cq = cq, .event_data = event_data};
-    return (int) mca_btl_ugni_device_serialize (device, (mca_btl_ugni_device_serialize_fn_t) mca_btl_ugni_gni_cq_get_event_device, &args);
-}
-
-__opal_attribute_always_inline__
-static inline int mca_btl_ugni_endpoint_post (mca_btl_ugni_endpoint_t *endpoint, mca_btl_ugni_post_descriptor_t *desc,
-                                              mca_btl_ugni_device_serialize_fn_t post_fn)
-{
-    struct mca_btl_ugni_post_device_args_t args = {.desc = desc};
-    mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl (endpoint);
-    int rc;
-
-    /* use serialize_any as it is responsible for binding devices to threads (if enabled). this generally
-     * gives better performance as it reduces contention on any individual device. */
-    rc = mca_btl_ugni_device_serialize_any (ugni_module, post_fn, &args);
-    if (args.count) {
-        mca_btl_ugni_handle_rdma_completions (ugni_module, args.device, args.completed, args.count);
-    }
-
-    return rc;
-}
-
-__opal_attribute_always_inline__
-static inline int mca_btl_ugni_endpoint_post_fma (mca_btl_ugni_endpoint_t *endpoint, mca_btl_ugni_post_descriptor_t *desc)
-{
-    return mca_btl_ugni_endpoint_post (endpoint, desc, (mca_btl_ugni_device_serialize_fn_t) mca_btl_ugni_post_fma_device);
-}
-
-__opal_attribute_always_inline__
-static inline int mca_btl_ugni_endpoint_post_rdma (mca_btl_ugni_endpoint_t *endpoint, mca_btl_ugni_post_descriptor_t *desc)
-{
-    return mca_btl_ugni_endpoint_post (endpoint, desc, (mca_btl_ugni_device_serialize_fn_t) mca_btl_ugni_post_rdma_device);
-}
-
-static inline int mca_btl_ugni_endpoint_post_cqwrite (mca_btl_ugni_endpoint_t *endpoint, mca_btl_ugni_post_descriptor_t *desc)
-{
-    mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl (endpoint);
-    mca_btl_ugni_device_t *device = ugni_module->devices;
-    return (int) mca_btl_ugni_device_serialize (device, (mca_btl_ugni_device_serialize_fn_t) mca_btl_ugni_post_cqwrite_device, desc);
-}
-
-__opal_attribute_always_inline__
-static inline int mca_btl_ugni_cq_get_completed_desc (mca_btl_ugni_device_t *device, mca_btl_ugni_cq_t *cq,
-                                                      mca_btl_ugni_post_descriptor_t *post_desc,
-                                                      int count)
-{
-    mca_btl_ugni_cq_get_completed_desc_arg_t args = {.cq = cq, .post_desc = post_desc, .count = count};
-    if (0 == cq->active_operations) {
-        return 0;
-    }
-
-    return (int) mca_btl_ugni_device_serialize (device, (mca_btl_ugni_device_serialize_fn_t) mca_btl_ugni_cq_get_completed_desc_device, &args);
-}
-
-static inline int mca_btl_ugni_get_datagram (mca_btl_ugni_module_t *ugni_module, mca_btl_ugni_device_t *device, gni_ep_handle_t *gni_handle,
-                                             mca_btl_base_endpoint_t **ep)
-{
-    mca_btl_ugni_get_datagram_args_t args = {.ugni_module = ugni_module, .ep = ep, .handle = gni_handle};
-    return (int) mca_btl_ugni_device_serialize (device, (mca_btl_ugni_device_serialize_fn_t) mca_btl_ugni_get_datagram_device, &args);
-}
-
-static inline int mca_btl_ugni_reg_mem (mca_btl_ugni_module_t *ugni_module, void *base, size_t size, mca_btl_ugni_reg_t *ugni_reg,
-                                 gni_cq_handle_t cq, int flags)
-{
-    mca_btl_ugni_reg_mem_args_t args = {.ugni_module = ugni_module, .base = base, .size = size,
-                                        .ugni_reg = ugni_reg, .cq = cq, .flags = flags};
-    mca_btl_ugni_device_t *device = ugni_module->devices;
-    return (int) mca_btl_ugni_device_serialize (device, (mca_btl_ugni_device_serialize_fn_t) mca_btl_ugni_reg_mem_device, &args);
-}
-
-static inline int mca_btl_ugni_dereg_mem (mca_btl_ugni_module_t *ugni_module, mca_btl_ugni_reg_t *ugni_reg)
-{
-    mca_btl_ugni_dereg_mem_arg_t args = {.ugni_module = ugni_module, .ugni_reg = ugni_reg};
-    mca_btl_ugni_device_t *device = ugni_module->devices;
-    return (int) mca_btl_ugni_device_serialize (device, (mca_btl_ugni_device_serialize_fn_t) mca_btl_ugni_dereg_mem_device, &args);
-}
-
-#endif /* BTL_UGNI_DEVICE_H */
diff --git a/opal/mca/btl/ugni/btl_ugni_endpoint.c b/opal/mca/btl/ugni/btl_ugni_endpoint.c
deleted file mode 100644
index 98e3ca90c8f..00000000000
--- a/opal/mca/btl/ugni/btl_ugni_endpoint.c
+++ /dev/null
@@ -1,409 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2011-2013 UT-Battelle, LLC. All rights reserved.
- * Copyright (c) 2017-2019 Intel, Inc.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "btl_ugni_endpoint.h"
-#include "btl_ugni_smsg.h"
-#include "opal/mca/pmix/pmix-internal.h"
-
-static void mca_btl_ugni_ep_construct (mca_btl_base_endpoint_t *ep)
-{
-    memset ((char *) ep + sizeof(ep->super), 0, sizeof (*ep) - sizeof (ep->super));
-    OBJ_CONSTRUCT(&ep->frag_wait_list, opal_list_t);
-    OBJ_CONSTRUCT(&ep->lock, opal_recursive_mutex_t);
-}
-
-static void mca_btl_ugni_ep_destruct (mca_btl_base_endpoint_t *ep)
-{
-    OBJ_DESTRUCT(&ep->frag_wait_list);
-    OBJ_DESTRUCT(&ep->lock);
-    free (ep->remote_attr);
-}
-
-OBJ_CLASS_INSTANCE(mca_btl_ugni_endpoint_t, opal_list_item_t,
-                   mca_btl_ugni_ep_construct, mca_btl_ugni_ep_destruct);
-
-static int mca_btl_ugni_endpoint_get_modex (mca_btl_base_endpoint_t *ep)
-{
-    mca_btl_ugni_modex_t *modex;
-    size_t msg_size;
-    int rc;
-
-    assert (NULL != ep && NULL != ep->peer_proc);
-
-    /* Receive the modex */
-    OPAL_MODEX_RECV(rc, &mca_btl_ugni_component.super.btl_version,
-                    &ep->peer_proc->proc_name, (void **)&modex, &msg_size);
-    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-        BTL_ERROR(("error receiving modex"));
-        return rc;
-    }
-
-    ep->ep_rem_addr = modex->addr;
-    ep->ep_rem_id = modex->id;
-
-
-    BTL_VERBOSE(("received modex for ep %p. addr: %d, id: %d",  (void*)ep, ep->ep_rem_addr, ep->ep_rem_id));
-
-    free (modex);
-
-    return OPAL_SUCCESS;
-}
-
-int mca_btl_ugni_init_ep (mca_btl_ugni_module_t *ugni_module, mca_btl_ugni_endpoint_t **ep,
-                          mca_btl_ugni_module_t *btl, opal_proc_t *peer_proc)
-{
-    mca_btl_ugni_endpoint_t *endpoint;
-    int rc;
-
-    endpoint = OBJ_NEW(mca_btl_ugni_endpoint_t);
-    assert (endpoint != NULL);
-
-    endpoint->smsg_progressing = 0;
-    endpoint->state = MCA_BTL_UGNI_EP_STATE_INIT;
-    endpoint->peer_proc = peer_proc;
-
-    /* get the modex info for this endpoint and setup a ugni endpoint. this call may lead
-     * to re-entry through opal_progress(). */
-    rc = mca_btl_ugni_endpoint_get_modex (endpoint);
-    if (OPAL_SUCCESS != rc) {
-        assert (0);
-        return rc;
-    }
-
-    /* add this endpoint to the pointer array */
-    endpoint->index = opal_pointer_array_add (&ugni_module->endpoints, endpoint);
-
-    *ep = endpoint;
-
-    return OPAL_SUCCESS;
-}
-
-void mca_btl_ugni_release_ep (mca_btl_ugni_endpoint_t *ep)
-{
-    mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl (ep);
-    int rc;
-
-    opal_mutex_lock (&ep->lock);
-
-    rc = mca_btl_ugni_ep_disconnect (ep, false);
-    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-        BTL_VERBOSE(("btl/ugni error disconnecting endpoint"));
-    }
-
-    /* TODO -- Clear space at the end of the endpoint array */
-    opal_pointer_array_set_item (&ugni_module->endpoints, ep->index, NULL);
-
-    opal_mutex_unlock (&ep->lock);
-
-    OBJ_RELEASE(ep);
-}
-
-static inline int mca_btl_ugni_ep_smsg_get_mbox (mca_btl_base_endpoint_t *ep) {
-    mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl (ep);
-    opal_free_list_item_t *mbox;
-
-    assert (NULL == ep->mailbox);
-
-    mbox = opal_free_list_get (&ugni_module->smsg_mboxes);
-    if (OPAL_UNLIKELY(NULL == mbox)) {
-        return OPAL_ERR_OUT_OF_RESOURCE;
-    }
-
-    ep->mailbox = (mca_btl_ugni_smsg_mbox_t *) mbox;
-    ep->mailbox->attr.index = ep->index;
-
-    /* per ugni spec we need to zero mailbox data before connecting */
-    memset ((char *)ep->mailbox->attr.smsg_attr.msg_buffer + ep->mailbox->attr.smsg_attr.mbox_offset, 0,
-            ep->mailbox->attr.smsg_attr.buff_size);
-    return OPAL_SUCCESS;
-}
-
-static int mca_btl_ugni_ep_send_disconnect (mca_btl_base_endpoint_t *ep)
-{
-    int rc;
-
-    do {
-        rc = mca_btl_ugni_endpoint_smsg_send_wtag (ep, NULL, 0, NULL, 0, -1, MCA_BTL_UGNI_TAG_DISCONNECT);
-        if (OPAL_LIKELY(GNI_RC_NOT_DONE != rc)) {
-            break;
-        }
-
-        /* most likely got here because we are out of credits. check the remote CQ to get credit return */
-        (void) mca_btl_ugni_progress_remote_smsg (mca_btl_ugni_ep_btl (ep));
-    } while (1);
-
-    return mca_btl_rc_ugni_to_opal (rc);
-}
-
-int mca_btl_ugni_ep_disconnect (mca_btl_base_endpoint_t *ep, bool send_disconnect)
-{
-    mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl (ep);
-    mca_btl_ugni_device_t *device;
-    int rc;
-
-    if (MCA_BTL_UGNI_EP_STATE_INIT == ep->state) {
-        /* nothing to do */
-        return OPAL_SUCCESS;
-    }
-
-    device = ep->smsg_ep_handle.device;
-
-    while (device->dev_smsg_local_cq.active_operations) {
-        /* ensure all sends are complete before removing and procs */
-        rc = mca_btl_ugni_progress_local_smsg (ugni_module, device);
-        if (OPAL_SUCCESS != rc) {
-            break;
-        }
-    }
-
-    if (MCA_BTL_UGNI_EP_STATE_CONNECTED == ep->state && send_disconnect) {
-        rc = mca_btl_ugni_ep_send_disconnect (ep);
-        if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-            BTL_VERBOSE(("could not send disconnect message to peer"));
-        }
-
-        /* wait for the disconnect messagse to go */
-        do {
-            /* ensure all sends are complete before removing and procs */
-            rc = mca_btl_ugni_progress_local_smsg (ugni_module, device);
-            if (OPAL_SUCCESS != rc) {
-                break;
-            }
-        } while (device->dev_smsg_local_cq.active_operations);
-
-        (void) opal_atomic_add_fetch_32 (&ep->smsg_ep_handle.device->smsg_connections, -1);
-    }
-
-    mca_btl_ugni_device_lock (device);
-
-    /* NTH: this call may not need the device lock. seems to work without it but
-     * the lock is here to be safe. */
-    (void) mca_btl_ugni_ep_handle_cleanup (&ep->smsg_ep_handle);
-
-    mca_btl_ugni_device_unlock (device);
-
-    if (ep->mailbox) {
-        opal_free_list_return (&ugni_module->smsg_mboxes, ((opal_free_list_item_t *) ep->mailbox));
-        ep->mailbox = NULL;
-    }
-
-    ep->state = MCA_BTL_UGNI_EP_STATE_INIT;
-
-    return OPAL_SUCCESS;
-}
-
-static inline int mca_btl_ugni_ep_connect_start (mca_btl_base_endpoint_t *ep) {
-    mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl (ep);
-    mca_btl_ugni_device_t *device = ugni_module->devices;
-    int rc;
-
-    /* protect against re-entry from opal_progress */
-    if (OPAL_UNLIKELY(MCA_BTL_UGNI_EP_STATE_CONNECTING == ep->state)) {
-        return OPAL_ERR_RESOURCE_BUSY;
-    }
-
-    ep->state = MCA_BTL_UGNI_EP_STATE_CONNECTING;
-
-    BTL_VERBOSE(("initiating connection to remote peer with address: %u id: %u proc: %p",
-                 ep->ep_rem_addr, ep->ep_rem_id, (void *)ep->peer_proc));
-
-    /* bind endpoint to remote address */
-    /* we bind two endpoints to seperate out local smsg completion and local fma completion */
-    mca_btl_ugni_device_lock (device);
-    rc = mca_btl_ugni_ep_handle_init (ep, device->dev_smsg_local_cq.gni_handle, device, &ep->smsg_ep_handle);
-    mca_btl_ugni_device_unlock (device);
-    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-        return rc;
-    }
-
-    /* build connection data */
-    rc = mca_btl_ugni_ep_smsg_get_mbox (ep);
-    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-        return rc;
-    }
-
-    ep->remote_attr = calloc (1, sizeof (*ep->remote_attr));
-    if (OPAL_UNLIKELY(NULL == ep->remote_attr)) {
-        return OPAL_ERR_OUT_OF_RESOURCE;
-    }
-
-    BTL_VERBOSE(("btl/ugni connection to remote peer initiated"));
-
-    return OPAL_SUCCESS;
-}
-
-static inline int mca_btl_ugni_ep_connect_finish (mca_btl_base_endpoint_t *ep) {
-    mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl (ep);
-    gni_return_t grc;
-    int rc;
-
-    BTL_VERBOSE(("finishing connection. remote attributes: msg_type = %d, msg_buffer = %p, buff_size = %d, "
-                 "mem_hndl = {qword1 = %" PRIu64 ", qword2 = %" PRIu64 "}, mbox = %d, mbox_maxcredit = %d, "
-                 "msg_maxsize = %d", ep->remote_attr->smsg_attr.msg_type, ep->remote_attr->smsg_attr.msg_buffer,
-                 ep->remote_attr->smsg_attr.buff_size, ep->remote_attr->smsg_attr.mem_hndl.qword1,
-                 ep->remote_attr->smsg_attr.mem_hndl.qword2, ep->remote_attr->smsg_attr.mbox_offset,
-                 ep->remote_attr->smsg_attr.mbox_maxcredit, ep->remote_attr->smsg_attr.msg_maxsize));
-
-    BTL_VERBOSE(("finishing connection. local attributes: msg_type = %d, msg_buffer = %p, buff_size = %d, "
-                 "mem_hndl = {qword1 = %" PRIu64 ", qword2 = %" PRIu64 "}, mbox = %d, mbox_maxcredit = %d, "
-                 "msg_maxsize = %d", ep->mailbox->attr.smsg_attr.msg_type, ep->mailbox->attr.smsg_attr.msg_buffer,
-                 ep->mailbox->attr.smsg_attr.buff_size, ep->mailbox->attr.smsg_attr.mem_hndl.qword1,
-                 ep->mailbox->attr.smsg_attr.mem_hndl.qword2, ep->mailbox->attr.smsg_attr.mbox_offset,
-                 ep->mailbox->attr.smsg_attr.mbox_maxcredit, ep->mailbox->attr.smsg_attr.msg_maxsize));
-
-    grc = GNI_SmsgInit (ep->smsg_ep_handle.gni_handle, &ep->mailbox->attr.smsg_attr,
-                        &ep->remote_attr->smsg_attr);
-    if (OPAL_UNLIKELY(GNI_RC_SUCCESS != grc)) {
-        BTL_ERROR(("error initializing SMSG protocol. rc = %d", grc));
-
-        return mca_btl_rc_ugni_to_opal (grc);
-    }
-
-    /* set the local event data to the local index and the remote event data to my
-     * index on the remote peer. This makes lookup of endpoints on completion take
-     * a single lookup in the endpoints array. we will not be able to change the
-     * remote peer's index in the endpoint's array after this point. */
-    GNI_EpSetEventData (ep->smsg_ep_handle.gni_handle, ep->index, ep->remote_attr->index);
-
-    ep->rmt_irq_mem_hndl = ep->remote_attr->rmt_irq_mem_hndl;
-    ep->state = MCA_BTL_UGNI_EP_STATE_CONNECTED;
-    (void) opal_atomic_add_fetch_32 (&ep->smsg_ep_handle.device->smsg_connections, 1);
-
-    /* send all pending messages */
-    BTL_VERBOSE(("endpoint connected. posting %u sends", (unsigned int) opal_list_get_size (&ep->frag_wait_list)));
-
-    rc = mca_btl_ugni_progress_send_wait_list (ep);
-    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-        OPAL_THREAD_LOCK(&ugni_module->ep_wait_list_lock);
-        if (false == ep->wait_listed) {
-            opal_list_append (&ugni_module->ep_wait_list, &ep->super);
-            ep->wait_listed = true;
-        }
-        OPAL_THREAD_UNLOCK(&ugni_module->ep_wait_list_lock);
-    }
-
-    free (ep->remote_attr);
-    ep->remote_attr = NULL;
-
-    return OPAL_SUCCESS;
-}
-
-static int mca_btl_ugni_directed_ep_post (mca_btl_base_endpoint_t *ep)
-{
-    mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl (ep);
-    gni_return_t rc;
-
-    BTL_VERBOSE(("posting directed datagram to remote id: %d for endpoint %p", ep->ep_rem_id, (void *)ep));
-    /* the irq cq is associated with only the first device */
-    ep->mailbox->attr.rmt_irq_mem_hndl = ugni_module->devices->smsg_irq_mhndl;
-
-    rc = GNI_EpPostDataWId (ep->smsg_ep_handle.gni_handle, &ep->mailbox->attr, sizeof (ep->mailbox->attr),
-                            ep->remote_attr, sizeof (*ep->remote_attr),
-                            MCA_BTL_UGNI_CONNECT_DIRECTED_ID | ep->index);
-    if (OPAL_LIKELY(GNI_RC_SUCCESS == rc)) {
-        (void) opal_atomic_add_fetch_32 (&ugni_module->active_datagrams, 1);
-    }
-
-    return mca_btl_rc_ugni_to_opal (rc);
-}
-
-int mca_btl_ugni_wildcard_ep_post (mca_btl_ugni_module_t *ugni_module)
-{
-    gni_return_t rc;
-
-    BTL_VERBOSE(("posting wildcard datagram"));
-
-    memset (&ugni_module->wc_local_attr, 0, sizeof (ugni_module->wc_local_attr));
-    memset (&ugni_module->wc_remote_attr, 0, sizeof (ugni_module->wc_remote_attr));
-    rc = GNI_EpPostDataWId (ugni_module->wildcard_ep, &ugni_module->wc_local_attr,
-                            sizeof (ugni_module->wc_local_attr), &ugni_module->wc_remote_attr,
-                            sizeof (ugni_module->wc_remote_attr), MCA_BTL_UGNI_CONNECT_WILDCARD_ID);
-
-    return mca_btl_rc_ugni_to_opal (rc);
-}
-
-
-int mca_btl_ugni_ep_connect_progress (mca_btl_base_endpoint_t *ep)
-{
-    int rc;
-
-    BTL_VERBOSE(("progressing connection for endpoint %p with state %d", (void *)ep, ep->state));
-
-    if (MCA_BTL_UGNI_EP_STATE_CONNECTED == ep->state) {
-        return OPAL_SUCCESS;
-    }
-
-    if (MCA_BTL_UGNI_EP_STATE_INIT == ep->state) {
-        rc = mca_btl_ugni_ep_connect_start (ep);
-        if (OPAL_SUCCESS != rc) {
-            return rc;
-        }
-    }
-
-    BTL_VERBOSE(("ep->remote_attr->smsg_attr = {.msg_type = %d, .msg_buffer = %p}", ep->remote_attr->smsg_attr.msg_type,
-                 (void*)ep->remote_attr->smsg_attr.msg_buffer));
-
-    if (GNI_SMSG_TYPE_INVALID == ep->remote_attr->smsg_attr.msg_type) {
-        /* use datagram to exchange connection information with the remote peer */
-        if (!ep->dg_posted) {
-            rc = mca_btl_ugni_directed_ep_post (ep);
-            if (OPAL_SUCCESS == rc) {
-                ep->dg_posted = true;
-                rc = OPAL_ERR_RESOURCE_BUSY;
-            }
-
-            return rc;
-        }
-
-        return OPAL_SUCCESS;
-    }
-
-    return mca_btl_ugni_ep_connect_finish (ep);
-}
-
-int mca_btl_ugni_ep_handle_init (mca_btl_ugni_endpoint_t *ep, gni_cq_handle_t cq,
-                                 mca_btl_ugni_device_t *device, mca_btl_ugni_endpoint_handle_t *ep_handle)
-{
-    gni_return_t grc;
-
-    ep_handle->device = device;
-
-    /* create a uGNI endpoint handle and bind it to the remote peer */
-    grc = GNI_EpCreate (device->dev_handle, cq, &ep_handle->gni_handle);
-    if (OPAL_LIKELY(GNI_RC_SUCCESS == grc)) {
-        grc = GNI_EpBind (ep_handle->gni_handle, ep->ep_rem_addr, ep->ep_rem_id);
-    }
-
-    return mca_btl_rc_ugni_to_opal (grc);
-}
-
-int mca_btl_ugni_ep_handle_cleanup (mca_btl_ugni_endpoint_handle_t *ep_handle)
-{
-    int rc;
-
-    if (0 == ep_handle->gni_handle) {
-        return OPAL_SUCCESS;
-    }
-
-    /* TODO: need to fix, may be outstanding tx's, etc. */
-    rc = GNI_EpUnbind (ep_handle->gni_handle);
-    if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc)) {
-        /* should warn */
-    } else {
-        (void) GNI_EpDestroy (ep_handle->gni_handle);
-    }
-
-    ep_handle->gni_handle = 0;
-
-    return OPAL_SUCCESS;
-}
diff --git a/opal/mca/btl/ugni/btl_ugni_endpoint.h b/opal/mca/btl/ugni/btl_ugni_endpoint.h
deleted file mode 100644
index 71bfcca9016..00000000000
--- a/opal/mca/btl/ugni/btl_ugni_endpoint.h
+++ /dev/null
@@ -1,145 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2011      UT-Battelle, LLC. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef MCA_BTL_UGNI_ENDPOINT_H
-#define MCA_BTL_UGNI_ENDPOINT_H
-
-#include "btl_ugni.h"
-
-enum mca_btl_ugni_endpoint_state_t {
-    MCA_BTL_UGNI_EP_STATE_INIT = 0,
-    MCA_BTL_UGNI_EP_STATE_CONNECTING,
-    MCA_BTL_UGNI_EP_STATE_CONNECTED,
-};
-typedef enum mca_btl_ugni_endpoint_state_t mca_btl_ugni_endpoint_state_t;
-
-struct mca_btl_ugni_smsg_mbox_t;
-
-struct mca_btl_ugni_endpoint_handle_t {
-    mca_btl_ugni_device_t *device;
-    gni_ep_handle_t gni_handle;
-};
-
-typedef struct mca_btl_ugni_endpoint_handle_t mca_btl_ugni_endpoint_handle_t;
-
-typedef struct mca_btl_base_endpoint_t {
-    opal_list_item_t super;
-
-    opal_proc_t *peer_proc;
-
-    /** may need to lock recursively as the modex lookup could call opal_progress
-     * and hence our progress function. if this changes modify this mutex to not
-     * be recursive. also need to update the constructor function. */
-    opal_recursive_mutex_t lock;
-    mca_btl_ugni_endpoint_state_t state;
-
-    /** Remote NIC address */
-    uint32_t ep_rem_addr;
-
-    /** Remote CDM identifier (base) */
-    uint32_t ep_rem_id;
-
-    /** endpoint to use for SMSG messages */
-    mca_btl_ugni_endpoint_handle_t smsg_ep_handle;
-
-    /** temporary space to store the remote SMSG attributes */
-    mca_btl_ugni_endpoint_attr_t *remote_attr;
-
-    /** SMSG mailbox assigned to this endpoint */
-    struct mca_btl_ugni_smsg_mbox_t *mailbox;
-
-    /** Remote IRQ handle (for async completion) */
-    gni_mem_handle_t rmt_irq_mem_hndl;
-
-    /** frags waiting for SMSG credits */
-    opal_list_t frag_wait_list;
-
-    /** endpoint is currently wait-listed for SMSG progress */
-    bool wait_listed;
-
-    /** protect against race on connection */
-    bool dg_posted;
-
-    /** protect against re-entry to SMSG */
-    opal_atomic_int32_t smsg_progressing;
-
-    int index;
-} mca_btl_base_endpoint_t;
-
-typedef mca_btl_base_endpoint_t  mca_btl_ugni_endpoint_t;
-OBJ_CLASS_DECLARATION(mca_btl_ugni_endpoint_t);
-
-int mca_btl_ugni_ep_connect_progress (mca_btl_ugni_endpoint_t *ep);
-int mca_btl_ugni_ep_disconnect (mca_btl_ugni_endpoint_t *ep, bool send_disconnect);
-int mca_btl_ugni_wildcard_ep_post (mca_btl_ugni_module_t *ugni_module);
-void mca_btl_ugni_release_ep (mca_btl_ugni_endpoint_t *ep);
-int mca_btl_ugni_init_ep (mca_btl_ugni_module_t *ugni_module, mca_btl_ugni_endpoint_t **ep,
-                          mca_btl_ugni_module_t *btl, opal_proc_t *peer_proc);
-
-static inline int mca_btl_ugni_check_endpoint_state (mca_btl_ugni_endpoint_t *ep) {
-    int rc;
-
-    if (OPAL_LIKELY(MCA_BTL_UGNI_EP_STATE_CONNECTED == ep->state)) {
-        return OPAL_SUCCESS;
-    }
-
-    opal_mutex_lock (&ep->lock);
-
-    switch (ep->state) {
-    case MCA_BTL_UGNI_EP_STATE_INIT:
-        rc = mca_btl_ugni_ep_connect_progress (ep);
-        if (OPAL_SUCCESS != rc) {
-            break;
-        }
-    case MCA_BTL_UGNI_EP_STATE_CONNECTING:
-        rc = OPAL_ERR_RESOURCE_BUSY;
-        break;
-    default:
-        rc = OPAL_SUCCESS;
-    }
-
-    opal_mutex_unlock (&ep->lock);
-
-    return rc;
-}
-
-/**
- * Accessor function for endpoint btl
- *
- * @param[in] ep   endpoint to query
- *
- * This helper function exists to make it easy to switch between using a single
- * and multiple ugni modules. Currently there is only one so we just use the
- * pointer in the component structure. This saves 4-8 bytes in the endpoint
- * structure.
- */
-static inline mca_btl_ugni_module_t *mca_btl_ugni_ep_btl (mca_btl_ugni_endpoint_t *ep)
-{
-    /* there is only one ugni module at this time. if that changes add a btl pointer back
-     * to the endpoint structure. */
-    return mca_btl_ugni_component.modules;
-}
-
-/**
- * Initialize and bind an endpoint handle
- *
- * @param[in]  ep                 BTL endpoint
- * @param[in]  cq                 completion queue
- * @param[in]  device             device to bind with
- * @param[in]  ep_handle          endpoint handle to initialize and bind
- */
-int mca_btl_ugni_ep_handle_init (mca_btl_ugni_endpoint_t *ep, gni_cq_handle_t cq,
-                                 mca_btl_ugni_device_t *device, mca_btl_ugni_endpoint_handle_t *ep_handle);
-
-int mca_btl_ugni_ep_handle_cleanup (mca_btl_ugni_endpoint_handle_t *ep_handle);
-
-#endif /* MCA_BTL_UGNI_ENDPOINT_H */
diff --git a/opal/mca/btl/ugni/btl_ugni_frag.c b/opal/mca/btl/ugni/btl_ugni_frag.c
deleted file mode 100644
index 5f49e3e3369..00000000000
--- a/opal/mca/btl/ugni/btl_ugni_frag.c
+++ /dev/null
@@ -1,91 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2011      UT-Battelle, LLC. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "btl_ugni.h"
-#include "btl_ugni_frag.h"
-
-static inline void mca_btl_ugni_base_frag_constructor (mca_btl_ugni_base_frag_t *frag)
-{
-    memset ((char *) frag + sizeof (frag->base), 0, sizeof (*frag) - sizeof (frag->base));
-    frag->segments[0].seg_addr.pval = frag->base.super.ptr;
-}
-
-static inline void mca_btl_ugni_eager_frag_constructor (mca_btl_ugni_base_frag_t *frag)
-{
-    struct mca_btl_ugni_reg_t *reg =
-        (struct mca_btl_ugni_reg_t *) frag->base.super.registration;
-
-    mca_btl_ugni_base_frag_constructor (frag);
-
-    frag->memory_handle = reg->handle;
-}
-
-OBJ_CLASS_INSTANCE(mca_btl_ugni_smsg_frag_t, mca_btl_base_descriptor_t,
-                   mca_btl_ugni_base_frag_constructor, NULL);
-
-OBJ_CLASS_INSTANCE(mca_btl_ugni_rdma_frag_t, mca_btl_base_descriptor_t,
-                   mca_btl_ugni_base_frag_constructor, NULL);
-
-OBJ_CLASS_INSTANCE(mca_btl_ugni_eager_frag_t, mca_btl_base_descriptor_t,
-                   mca_btl_ugni_eager_frag_constructor, NULL);
-
-static void mca_btl_ugni_post_descriptor_constructor (mca_btl_ugni_post_descriptor_t *desc)
-{
-    desc->cq = NULL;
-}
-
-OBJ_CLASS_INSTANCE(mca_btl_ugni_post_descriptor_t, opal_free_list_item_t,
-                   mca_btl_ugni_post_descriptor_constructor, NULL);
-
-static void mca_btl_ugni_rdma_desc_constructor (mca_btl_ugni_rdma_desc_t *desc)
-{
-    desc->device = NULL;
-    desc->gni_handle = 0;
-    desc->tries = 0;
-}
-
-static void mca_btl_ugni_rdma_desc_destructor (mca_btl_ugni_rdma_desc_t *desc)
-{
-    if (0 != desc->gni_handle) {
-        (void) GNI_EpDestroy (desc->gni_handle);
-        desc->gni_handle = 0;
-    }
-}
-
-int mca_btl_ugni_rdma_desc_init (opal_free_list_item_t *item, void *ctx)
-{
-    mca_btl_ugni_rdma_desc_t *rdma_desc = (mca_btl_ugni_rdma_desc_t *) item;
-    mca_btl_ugni_device_t *device = (mca_btl_ugni_device_t *) ctx;
-    gni_return_t grc;
-
-    grc = GNI_EpCreate (device->dev_handle, device->dev_rdma_local_cq.gni_handle, &rdma_desc->gni_handle);
-    rdma_desc->device = device;
-    return mca_btl_rc_ugni_to_opal (grc);
-}
-
-
-OBJ_CLASS_INSTANCE(mca_btl_ugni_rdma_desc_t, opal_free_list_item_t,
-                   mca_btl_ugni_rdma_desc_constructor, mca_btl_ugni_rdma_desc_destructor);
-
-int mca_btl_ugni_frag_init (mca_btl_ugni_base_frag_t *frag, void *id)
-{
-    /* NTH: the id is a combination of the module id and the free list id. for now there
-     * is only ever one module so the module id is ignored. if this changes the code
-     * here and btl_ugni_add_procs.c (opal_free_list_init calls) needs to be updated */
-    intptr_t free_list_id = (intptr_t) id & 0xff;
-    mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_component.modules;
-
-    frag->msg_id = opal_pointer_array_add (&ugni_module->pending_smsg_frags_bb, (void *) frag);
-    frag->my_list = ugni_module->frags_lists + free_list_id;
-
-    return OPAL_SUCCESS;
-}
diff --git a/opal/mca/btl/ugni/btl_ugni_frag.h b/opal/mca/btl/ugni/btl_ugni_frag.h
deleted file mode 100644
index c8e91568c97..00000000000
--- a/opal/mca/btl/ugni/btl_ugni_frag.h
+++ /dev/null
@@ -1,298 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2011-2017 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2011      UT-Battelle, LLC. All rights reserved.
- * Copyright (c) 2013      The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#if !defined(MCA_BTL_UGNI_FRAG_H)
-#define MCA_BTL_UGNI_FRAG_H
-
-#include "btl_ugni.h"
-#include "btl_ugni_endpoint.h"
-
-#include <string.h>
-
-typedef struct mca_btl_ugni_send_frag_hdr_t {
-    uint32_t lag;
-} mca_btl_ugni_send_frag_hdr_t;
-
-typedef struct mca_btl_ugni_send_ex_frag_hdr_t {
-    mca_btl_ugni_send_frag_hdr_t send;
-    uint8_t                      pml_header[128];
-} mca_btl_ugni_send_ex_frag_hdr_t;
-
-typedef struct mca_btl_ugni_rdma_frag_hdr_t {
-    void *ctx;
-} mca_btl_ugni_rdma_frag_hdr_t;
-
-typedef struct mca_btl_ugni_eager_frag_hdr_t {
-    mca_btl_ugni_send_frag_hdr_t send;
-    uint32_t size;
-    uint64_t address;
-    mca_btl_base_registration_handle_t memory_handle;
-    void *ctx;
-} mca_btl_ugni_eager_frag_hdr_t;
-
-typedef struct mca_btl_ugni_eager_ex_frag_hdr_t {
-    mca_btl_ugni_eager_frag_hdr_t eager;
-    uint8_t                       pml_header[128];
-} mca_btl_ugni_eager_ex_frag_hdr_t;
-
-typedef union mca_btl_ugni_frag_hdr_t {
-    mca_btl_ugni_send_frag_hdr_t     send;
-    mca_btl_ugni_send_ex_frag_hdr_t  send_ex;
-    mca_btl_ugni_rdma_frag_hdr_t     rdma;
-    mca_btl_ugni_eager_frag_hdr_t    eager;
-    mca_btl_ugni_eager_ex_frag_hdr_t eager_ex;
-} mca_btl_ugni_frag_hdr_t;
-
-enum {
-    MCA_BTL_UGNI_FRAG_BUFFERED      = 1,  /* frag data is buffered */
-    MCA_BTL_UGNI_FRAG_COMPLETE      = 2,  /* smsg complete for frag */
-    MCA_BTL_UGNI_FRAG_EAGER         = 4,  /* eager get frag */
-    MCA_BTL_UGNI_FRAG_IGNORE        = 8,  /* ignore local smsg completion */
-    MCA_BTL_UGNI_FRAG_SMSG_COMPLETE = 16, /* SMSG has completed for this message */
-    MCA_BTL_UGNI_FRAG_RESPONSE      = 32,
-};
-
-struct mca_btl_ugni_base_frag_t;
-
-typedef struct mca_btl_ugni_base_frag_t {
-    mca_btl_base_descriptor_t    base;
-    opal_atomic_int32_t          ref_cnt;
-    uint32_t                     msg_id;
-    uint16_t                     hdr_size;
-    uint16_t                     flags;
-    mca_btl_ugni_frag_hdr_t      hdr;
-    mca_btl_base_segment_t       segments[2];
-    gni_post_descriptor_t        post_desc;
-    mca_btl_base_endpoint_t     *endpoint;
-    mca_btl_ugni_reg_t          *registration;
-    opal_free_list_t            *my_list;
-    mca_btl_base_registration_handle_t memory_handle;
-} mca_btl_ugni_base_frag_t;
-
-typedef struct mca_btl_ugni_base_frag_t mca_btl_ugni_smsg_frag_t;
-typedef struct mca_btl_ugni_base_frag_t mca_btl_ugni_rdma_frag_t;
-typedef struct mca_btl_ugni_base_frag_t mca_btl_ugni_eager_frag_t;
-
-typedef struct mca_btl_ugni_post_descriptor_t {
-    /** endpoint currently associated with this desctiptor */
-    mca_btl_base_endpoint_t *endpoint;
-    /** local memory handle (for callback) */
-    mca_btl_base_registration_handle_t *local_handle;
-    /** currently associated completion queue */
-    mca_btl_ugni_cq_t *cq;
-    /** user callback function */
-    mca_btl_base_rdma_completion_fn_t cbfunc;
-    /** user callback data */
-    void *cbdata;
-    /** user callback context */
-    void *ctx;
-    /** opal status of this descriptor. filled in by
-     * mca_btl_ugni_cq_get_completed_desc_device() */
-    int rc;
-    /** true if posted with the BTE. false if FMA. this is used as part
-     * of the BTE throttling code. */
-    bool use_bte;
-    /** uGNI library post descriptor. this is last in this structure
-     * to try to keep it hot in the cache after copying this descriptor
-     * into the allocated descritor. (post follows almost immediately
-     * after allocate. */
-    gni_post_descriptor_t gni_desc;
-} mca_btl_ugni_post_descriptor_t;
-
-OBJ_CLASS_DECLARATION(mca_btl_ugni_post_descriptor_t);
-
-typedef struct mca_btl_ugni_rdma_desc_t {
-    opal_free_list_item_t super;
-    mca_btl_ugni_post_descriptor_t btl_ugni_desc;
-    mca_btl_ugni_device_t *device;
-    gni_ep_handle_t gni_handle;
-    int tries;
-} mca_btl_ugni_rdma_desc_t;
-
-OBJ_CLASS_DECLARATION(mca_btl_ugni_rdma_desc_t);
-
-#define MCA_BTL_UGNI_GNI_DESC_TO_RDMA_DESC(desc) \
-    ((mca_btl_ugni_rdma_desc_t *) ((uintptr_t)(desc) - offsetof (mca_btl_ugni_rdma_desc_t, btl_ugni_desc) - offsetof (mca_btl_ugni_post_descriptor_t, gni_desc)))
-
-/**
- * Initialize a RDMA descriptor
- *
- * @param[in]    item    free list item (must be of class mca_btl_ugni_rdma_desc_t)
- * @param[in]    ctx     pointer to ugni device context
- *
- * This function initializes a mca_btl_ugni_rdma_desc_t for use. It allocates
- * resources from the ugni library. This must be called before a RDMA
- * descriptor can be used. Usually this is passed as an argument to
- * opal_free_list_init().
- */
-int mca_btl_ugni_rdma_desc_init (opal_free_list_item_t *item, void *ctx);
-
-/**
- * @brief get an endpoint handle from a device's free list
- *
- * @param[in] ep      btl endpoint
- * @param[in] device  btl device to use
- * @param[in] use_bte whether this descriptor will be used with the BTE
- *
- * This function MUST be called with the device lock held. This was done over using
- * the atomic free list to avoid unnecessary atomics in the critical path.
- */
-static inline mca_btl_ugni_rdma_desc_t *
-mca_btl_ugni_alloc_rdma_desc (mca_btl_ugni_device_t *device, mca_btl_ugni_post_descriptor_t *ugni_desc, const bool use_bte)
-{
-    mca_btl_ugni_rdma_desc_t *desc = (mca_btl_ugni_rdma_desc_t *) opal_free_list_get_st (&device->rdma_descs);
-    mca_btl_ugni_endpoint_t *ep = ugni_desc->endpoint;
-    gni_return_t grc;
-
-    if (OPAL_LIKELY(NULL != desc)) {
-        grc = GNI_EpBind (desc->gni_handle, ep->ep_rem_addr, ep->ep_rem_id | device->dev_index);
-        if (OPAL_UNLIKELY(GNI_RC_SUCCESS != grc)) {
-            opal_free_list_return_st (&device->rdma_descs, &desc->super);
-            return NULL;
-        }
-
-        desc->device = device;
-        desc->tries = 0;
-        desc->btl_ugni_desc = *ugni_desc;
-        desc->btl_ugni_desc.use_bte = use_bte;
-    }
-
-    return desc;
-}
-
-static inline void mca_btl_ugni_return_rdma_desc (mca_btl_ugni_rdma_desc_t *desc)
-{
-    (void) GNI_EpUnbind (desc->gni_handle);
-    opal_free_list_return_st (&desc->device->rdma_descs, &desc->super);
-}
-
-static inline void mca_btl_ugni_post_desc_complete (mca_btl_ugni_module_t *module, mca_btl_ugni_post_descriptor_t *desc, int rc)
-{
-    BTL_VERBOSE(("RDMA/FMA/ATOMIC operation complete for post descriptor %p. rc = %d", (void *) desc, rc));
-
-    if (NULL != desc->cbfunc) {
-        /* call the user's callback function */
-        desc->cbfunc (&module->super, desc->endpoint, (void *)(intptr_t) desc->gni_desc.local_addr,
-                      desc->local_handle, desc->ctx, desc->cbdata, rc);
-    }
-}
-
-OBJ_CLASS_DECLARATION(mca_btl_ugni_smsg_frag_t);
-OBJ_CLASS_DECLARATION(mca_btl_ugni_rdma_frag_t);
-OBJ_CLASS_DECLARATION(mca_btl_ugni_eager_frag_t);
-
-int mca_btl_ugni_frag_init (mca_btl_ugni_base_frag_t *frag, void *id);
-
-static inline mca_btl_ugni_base_frag_t *mca_btl_ugni_frag_alloc (mca_btl_base_endpoint_t *ep,
-                                                                 opal_free_list_t *list)
-{
-    mca_btl_ugni_base_frag_t *frag = (mca_btl_ugni_base_frag_t *) opal_free_list_get (list);
-    if (OPAL_LIKELY(NULL != frag)) {
-        frag->endpoint = ep;
-        frag->ref_cnt = 1;
-    }
-
-    return frag;
-}
-
-static inline int mca_btl_ugni_frag_return (mca_btl_ugni_base_frag_t *frag)
-{
-    mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl (frag->endpoint);
-    if (frag->registration) {
-        ugni_module->rcache->rcache_deregister (ugni_module->rcache,
-                                                (mca_rcache_base_registration_t *) frag->registration);
-        frag->registration = NULL;
-    }
-
-    frag->flags = 0;
-
-    opal_free_list_return (frag->my_list, (opal_free_list_item_t *) frag);
-
-    return OPAL_SUCCESS;
-}
-
-static inline bool mca_btl_ugni_frag_del_ref (mca_btl_ugni_base_frag_t *frag, int rc) {
-    mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl (frag->endpoint);
-    /* save the descriptor flags since the callback is allowed to free the frag */
-    int des_flags = frag->base.des_flags;
-    int32_t ref_cnt;
-
-    opal_atomic_mb ();
-
-    ref_cnt = OPAL_THREAD_ADD_FETCH32(&frag->ref_cnt, -1);
-    if (ref_cnt) {
-        assert (ref_cnt > 0);
-        return false;
-    }
-
-    /* call callback if specified */
-    if (des_flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK) {
-        frag->base.des_cbfunc(&ugni_module->super, frag->endpoint, &frag->base, rc);
-    }
-
-    if (des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP) {
-        mca_btl_ugni_frag_return (frag);
-    }
-
-    return true;
-}
-
-static inline void mca_btl_ugni_frag_complete (mca_btl_ugni_base_frag_t *frag, int rc) {
-    BTL_VERBOSE(("frag complete. flags = %d", frag->base.des_flags));
-
-    frag->flags |= MCA_BTL_UGNI_FRAG_COMPLETE;
-
-    mca_btl_ugni_frag_del_ref (frag, rc);
-}
-
-static inline bool mca_btl_ugni_frag_check_complete (mca_btl_ugni_base_frag_t *frag) {
-    return !!(MCA_BTL_UGNI_FRAG_COMPLETE & frag->flags);
-}
-
-
-void mca_btl_ugni_wait_list_append (mca_btl_ugni_module_t *ugni_module, mca_btl_base_endpoint_t *endpoint,
-                                    mca_btl_ugni_base_frag_t *frag);
-
-static inline mca_btl_ugni_base_frag_t *mca_btl_ugni_frag_alloc_smsg (mca_btl_base_endpoint_t *ep)
-{
-    mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl (ep);
-    return mca_btl_ugni_frag_alloc (ep, ugni_module->frags_lists + MCA_BTL_UGNI_LIST_SMSG);
-}
-
-static inline mca_btl_ugni_base_frag_t *mca_btl_ugni_frag_alloc_rdma (mca_btl_base_endpoint_t *ep)
-{
-    mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl (ep);
-    return mca_btl_ugni_frag_alloc (ep, ugni_module->frags_lists + MCA_BTL_UGNI_LIST_RDMA);
-}
-
-static inline mca_btl_ugni_base_frag_t *mca_btl_ugni_frag_alloc_rdma_int (mca_btl_base_endpoint_t *ep)
-{
-    mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl (ep);
-    return mca_btl_ugni_frag_alloc (ep, ugni_module->frags_lists + MCA_BTL_UGNI_LIST_RDMA_INT);
-}
-
-static inline mca_btl_ugni_base_frag_t *mca_btl_ugni_frag_alloc_eager_send (mca_btl_base_endpoint_t *ep)
-{
-    mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl (ep);
-    return mca_btl_ugni_frag_alloc (ep, ugni_module->frags_lists + MCA_BTL_UGNI_LIST_EAGER_SEND);
-}
-
-static inline mca_btl_ugni_base_frag_t *mca_btl_ugni_frag_alloc_eager_recv (mca_btl_base_endpoint_t *ep)
-{
-    mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl (ep);
-    return mca_btl_ugni_frag_alloc (ep, ugni_module->frags_lists + MCA_BTL_UGNI_LIST_EAGER_RECV);
-}
-
-#endif /* MCA_BTL_UGNI_FRAG_H */
diff --git a/opal/mca/btl/ugni/btl_ugni_get.c b/opal/mca/btl/ugni/btl_ugni_get.c
deleted file mode 100644
index e5189c0d2e1..00000000000
--- a/opal/mca/btl/ugni/btl_ugni_get.c
+++ /dev/null
@@ -1,176 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2011-2017 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2011      UT-Battelle, LLC. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "btl_ugni_rdma.h"
-#include "btl_ugni_smsg.h"
-
-/* 
- * taken from osc_rdma_comm.h, ugh.
- */
-
-#define ALIGNMENT_MASK(x) ((x) ? (x) - 1 : 0)
-
-int mca_btl_ugni_get (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, void *local_address,
-                      uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                      mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-                      int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
-{
-    bool check;
-
-    /* Check if the get is aligned/sized on a multiple of 4 */
-    check = !!((remote_address | (uint64_t)(intptr_t) local_address | size) &
-                 ALIGNMENT_MASK(mca_btl_ugni_module.super.btl_get_alignment));
-
-    if (OPAL_UNLIKELY(check || size > mca_btl_ugni_module.super.btl_get_limit)) {
-        BTL_VERBOSE(("RDMA/FMA Get not available due to size or alignment restrictions"));
-
-        /* notify the caller that get is not available */
-        return OPAL_ERR_NOT_AVAILABLE;
-    }
-
-    BTL_VERBOSE(("Using RDMA/FMA Get %lu bytes to local address %p to remote address %" PRIx64,
-                 (unsigned long) size, local_address, remote_address));
-
-    return mca_btl_ugni_post (endpoint, true, size, local_address, remote_address, local_handle,
-                              remote_handle, order, cbfunc, cbcontext, cbdata);
-}
-
-/* eager get */
-static void mca_btl_ugni_callback_eager_get_progress_pending (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
-                                                              struct mca_btl_base_descriptor_t *desc, int rc)
-{
-    mca_btl_ugni_module_t *ugni_module = (mca_btl_ugni_module_t *) btl;
-    mca_btl_ugni_base_frag_t *pending_frag, *frag = (mca_btl_ugni_base_frag_t *) desc;
-
-    memset (&frag->hdr, 0, sizeof (frag->hdr));
-
-    OPAL_THREAD_LOCK(&ugni_module->eager_get_pending_lock);
-    pending_frag = (mca_btl_ugni_base_frag_t *) opal_list_remove_first (&ugni_module->eager_get_pending);
-    OPAL_THREAD_UNLOCK(&ugni_module->eager_get_pending_lock);
-
-    if (NULL != pending_frag) {
-        /* copy the relevant data out of the pending fragment */
-        frag->endpoint = pending_frag->endpoint;
-
-        assert (frag != pending_frag);
-
-        /* start the next eager get using this fragment */
-        (void) mca_btl_ugni_start_eager_get (frag->endpoint, pending_frag->hdr.eager_ex, frag);
-
-        /* return the temporary fragment */
-        mca_btl_ugni_frag_return (pending_frag);
-    } else {
-        /* not needed anymore */
-        mca_btl_ugni_frag_return (frag);
-    }
-}
-
-static void mca_btl_ugni_callback_eager_get (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
-                                             void *local_address, mca_btl_base_registration_handle_t *local_handle,
-                                             void *context, void *cbdata, int status)
-{
-    mca_btl_ugni_module_t *ugni_module = (mca_btl_ugni_module_t *) btl;
-    mca_btl_ugni_base_frag_t *frag = (mca_btl_ugni_base_frag_t *) context;
-    uint32_t len = frag->hdr.eager.send.lag & 0x00ffffff;
-    uint8_t tag = frag->hdr.eager.send.lag >> 24;
-    size_t payload_len = frag->hdr.eager.size;
-    size_t hdr_len = len - payload_len;
-    mca_btl_active_message_callback_t *reg = mca_btl_base_active_message_trigger + tag;
-    mca_btl_base_segment_t segs[2];
-    const mca_btl_base_receive_descriptor_t tmp = {.endpoint = endpoint, .des_segments = segs,
-                                                   .des_segment_count = hdr_len ? 2 : 1,
-                                                   .tag = tag, .cbdata = reg->cbdata};
-    int rc;
-
-    BTL_VERBOSE(("eager get for rem_ctx %p complete", frag->hdr.eager.ctx))
-
-    if (hdr_len) {
-        segs[0].seg_addr.pval = frag->hdr.eager_ex.pml_header;
-        segs[0].seg_len       = hdr_len;
-        segs[1].seg_addr.pval = local_address;
-        segs[1].seg_len       = payload_len;
-    } else {
-        segs[0].seg_addr.pval = local_address;
-        segs[0].seg_len       = payload_len;
-    }
-
-    reg->cbfunc(&ugni_module->super, &tmp);
-
-    /* fill in the response header */
-    frag->hdr.rdma.ctx = frag->hdr.eager.ctx;
-    frag->flags = MCA_BTL_UGNI_FRAG_RESPONSE;
-    frag->ref_cnt = 1;
-
-    frag->ref_cnt = 1;
-
-    /* once complete use this fragment for a pending eager get if any exist */
-    frag->base.des_cbfunc = mca_btl_ugni_callback_eager_get_progress_pending;
-
-    /* tell the remote peer the operation is complete */
-    rc = opal_mca_btl_ugni_smsg_send (frag, &frag->hdr.rdma, sizeof (frag->hdr.rdma),
-                                      NULL, 0, MCA_BTL_UGNI_TAG_RDMA_COMPLETE);
-    if (OPAL_UNLIKELY(0 > rc)) {
-        /* queue fragment */
-        mca_btl_ugni_wait_list_append (ugni_module, endpoint, frag);
-    }
-}
-
-int mca_btl_ugni_start_eager_get (mca_btl_base_endpoint_t *endpoint,
-                                  mca_btl_ugni_eager_ex_frag_hdr_t hdr,
-                                  mca_btl_ugni_base_frag_t *frag)
-{
-    mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl (endpoint);
-    size_t size;
-    int rc = OPAL_SUCCESS;
-
-    BTL_VERBOSE(("starting eager get for remote ctx: %p", hdr.eager.ctx));
-
-    do {
-        if (NULL == frag) {
-            /* try to allocate a registered buffer */
-            frag = mca_btl_ugni_frag_alloc_eager_recv (endpoint);
-            if (OPAL_UNLIKELY(NULL == frag)) {
-                /* no registered buffers available. try again later */
-                frag = mca_btl_ugni_frag_alloc_rdma_int (endpoint);
-
-                /* not much can be done if a small fragment can not be allocated. abort! */
-                assert (NULL != frag);
-                frag->hdr.eager_ex = hdr;
-                break;
-            }
-        }
-
-        frag->flags = 0;
-
-        frag->hdr.eager_ex = hdr;
-
-        /* increase size to a multiple of 4 bytes (required for get on Gemini) */
-        size = (hdr.eager.size + 3) & ~3;
-
-        /* set up callback for get completion */
-        frag->base.des_flags  = MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
-
-        /* start the get */
-        rc = mca_btl_ugni_post (endpoint, true, size, frag->base.super.ptr, hdr.eager.address,
-                                &frag->memory_handle, &hdr.eager.memory_handle,
-                                MCA_BTL_NO_ORDER, mca_btl_ugni_callback_eager_get, frag, NULL);
-        if (OPAL_UNLIKELY(OPAL_SUCCESS == rc)) {
-            return OPAL_SUCCESS;
-        }
-    } while (0);
-
-    OPAL_THREAD_LOCK(&ugni_module->eager_get_pending_lock);
-    opal_list_append (&ugni_module->eager_get_pending, (opal_list_item_t *) frag);
-    OPAL_THREAD_UNLOCK(&ugni_module->eager_get_pending_lock);
-
-    return rc;
-}
diff --git a/opal/mca/btl/ugni/btl_ugni_init.c b/opal/mca/btl/ugni/btl_ugni_init.c
deleted file mode 100644
index e29d3da2510..00000000000
--- a/opal/mca/btl/ugni/btl_ugni_init.c
+++ /dev/null
@@ -1,324 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2011      UT-Battelle, LLC. All rights reserved.
- * Copyright (c) 2014-2019 Intel, Inc.  All rights reserved.
- * Copyright (c) 2014      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-
-#include "btl_ugni.h"
-#include "btl_ugni_endpoint.h"
-#include "btl_ugni_frag.h"
-
-#include "opal/class/opal_list.h"
-#include "opal/mca/pmix/pmix-internal.h"
-#include "opal/util/bit_ops.h"
-#include "opal/mca/hwloc/base/base.h"
-
-static inline int get_ptag(uint8_t *out_ptag)
-{
-    /* TODO no need for tmp */
-    char *ptr;
-    uint8_t tmp_ptag;
-
-    if (NULL == (ptr = getenv("PMI_GNI_PTAG"))) {
-        /* TODO add err msg - better rc? */
-        return OPAL_ERR_NOT_FOUND;
-    }
-    errno = 0;
-    tmp_ptag = (uint8_t)strtoul (ptr, (char **)NULL, 10);
-    if (0 != errno) {
-        /* TODO add err msg - better rc? */
-        return OPAL_ERR_VALUE_OUT_OF_BOUNDS;
-    }
-    *out_ptag = tmp_ptag;
-    return OPAL_SUCCESS;
-}
-
-static inline int get_cookie (uint32_t *out_cookie)
-{
-    /* TODO no need for tmp */
-    char *ptr;
-    uint32_t tmp_cookie;
-
-    if (NULL == (ptr = getenv("PMI_GNI_COOKIE"))) {
-        /* TODO add err msg - better rc? */
-        return OPAL_ERR_NOT_FOUND;
-    }
-    errno = 0;
-    tmp_cookie = (uint32_t) strtoul (ptr, NULL, 10);
-    if (0 != errno) {
-        /* TODO add err msg - better rc? */
-        return OPAL_ERR_VALUE_OUT_OF_BOUNDS;
-    }
-
-    *out_cookie = tmp_cookie;
-    return OPAL_SUCCESS;
-}
-
-static unsigned int mca_btl_ugni_get_nic_address(int device_id)
-{
-    unsigned int address, cpu_id;
-    gni_return_t status;
-    int i, alps_dev_id = -1;
-    char *token,*p_ptr;
-
-    p_ptr = getenv("PMI_GNI_DEV_ID");
-    if (!p_ptr) {
-        status = GNI_CdmGetNicAddress(device_id, &address, &cpu_id);
-        if(status != GNI_RC_SUCCESS) {
-            opal_output (0, "FAILED:GNI_CdmGetNicAddress returned error %d", status);
-            return (unsigned int)-1;
-        }
-        return address;
-    }
-
-    while (NULL != (token = strtok(p_ptr, ":"))) {
-        alps_dev_id = atoi(token);
-        if (alps_dev_id == device_id) {
-            break;
-        }
-        p_ptr = NULL;
-    }
-
-    if (OPAL_UNLIKELY(-1 == alps_dev_id)) {
-        return (unsigned int)-1;
-    }
-
-    p_ptr = getenv("PMI_GNI_LOC_ADDR");
-    if (OPAL_UNLIKELY(NULL == p_ptr)) {
-        return (unsigned int)-1;
-    }
-
-    i = 0;
-    while (NULL != (token = strtok(p_ptr, ":"))) {
-        if (i == alps_dev_id) {
-            return strtoul (token, NULL, 10);
-        }
-        p_ptr = NULL;
-        ++i;
-    }
-
-    return (unsigned int)-1;
-}
-
-int mca_btl_ugni_device_init (mca_btl_ugni_device_t *device, int virtual_device_id)
-{
-    uint32_t dev_pe_addr;
-    int rc;
-
-    OBJ_CONSTRUCT(&device->rdma_descs, opal_free_list_t);
-
-    /* create a communication domain */
-    rc = GNI_CdmCreate (mca_btl_ugni_component.cdm_id_base | virtual_device_id, mca_btl_ugni_component.ptag,
-                        mca_btl_ugni_component.cookie, mca_btl_ugni_component.cdm_flags, &device->dev_cd_handle);
-    if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc)) {
-        /* this REALLY is an error but under alps + mapn we may not get any credentials */
-        BTL_VERBOSE(("Error: Creating communication domain %d for virtual device %d", rc, virtual_device_id));
-        return mca_btl_rc_ugni_to_opal (rc);
-    }
-
-    device->dev_index = virtual_device_id;
-
-    /* Create a NIC Adress */
-    OPAL_OUTPUT((-1, "Got NIC Addr: 0x%08x, CPU ID: %d", mca_btl_ugni_component.dev_addr, 0));
-
-    /* Attach device to the communication domain */
-    rc = GNI_CdmAttach (device->dev_cd_handle, 0, &dev_pe_addr, &device->dev_handle);
-    if (GNI_RC_SUCCESS != rc) {
-        BTL_VERBOSE(("Error: Attaching to communication domain. rc = %d, virtual device = %d", rc, virtual_device_id));
-        return mca_btl_rc_ugni_to_opal (rc);
-    }
-
-    rc = opal_free_list_init (&device->rdma_descs, sizeof (mca_btl_ugni_rdma_desc_t),
-                              64, OBJ_CLASS(mca_btl_ugni_rdma_desc_t), 0, 8, 0,
-                              mca_btl_ugni_component.local_rdma_cq_size, 32,
-                              NULL, 0, NULL, mca_btl_ugni_rdma_desc_init, (void *) device);
-    if (OPAL_SUCCESS != rc) {
-        OBJ_DESTRUCT(&device->rdma_descs);
-        return rc;
-    }
-
-    device->lock = 0;
-    device->dev_rdma_local_cq.gni_handle = 0;
-    device->dev_rdma_local_cq.active_operations = 0;
-    device->dev_rdma_local_irq_cq.gni_handle = 0;
-    device->dev_rdma_local_irq_cq.active_operations = 0;
-    device->dev_smsg_local_cq.gni_handle = 0;
-    device->dev_smsg_local_cq.active_operations = 0;
-    device->flushed = true;
-
-    return OPAL_SUCCESS;
-}
-
-int mca_btl_ugni_device_fini (mca_btl_ugni_device_t *dev)
-{
-    int rc;
-
-    OBJ_DESTRUCT(&dev->rdma_descs);
-
-    if (0 != dev->dev_rdma_local_cq.gni_handle) {
-        GNI_CqDestroy (dev->dev_rdma_local_cq.gni_handle);
-        dev->dev_rdma_local_cq.gni_handle = 0;
-    }
-
-    if (0 != dev->dev_rdma_local_irq_cq.gni_handle) {
-        GNI_CqDestroy (dev->dev_rdma_local_irq_cq.gni_handle);
-        dev->dev_rdma_local_irq_cq.gni_handle = 0;
-    }
-
-    if (0 != dev->dev_smsg_local_cq.gni_handle) {
-        GNI_CqDestroy (dev->dev_smsg_local_cq.gni_handle);
-        dev->dev_smsg_local_cq.gni_handle = 0;
-    }
-
-    rc = GNI_CdmDestroy (dev->dev_cd_handle);
-    if (GNI_RC_SUCCESS != rc) {
-        BTL_VERBOSE(("error destroying cdm handle"));
-    }
-
-    return OPAL_SUCCESS;
-}
-
-/*
- * Send local device information and other information
- * required for setup
- */
-static int mca_btl_ugni_send_modex (void)
-{
-    struct mca_btl_ugni_modex_t modex;
-    uint32_t modex_size;
-    char *modex_msg;
-    int rc;
-
-    modex_size = sizeof (struct mca_btl_ugni_modex_t);
-
-    modex_msg = (char *) malloc (modex_size);
-    if (NULL == modex_msg) {
-        OPAL_OUTPUT((-1, "Error allocating memory for modex @ %s:%d",
-                     __FILE__, __LINE__));
-        return OPAL_ERR_OUT_OF_RESOURCE;
-    }
-
-    modex.addr = mca_btl_ugni_component.dev_addr;
-    modex.id   = mca_btl_ugni_component.cdm_id_base;
-
-    BTL_VERBOSE(("sending modex. addr: %d, id: %d", modex.addr, modex.id));
-
-    memcpy ((void *) modex_msg, (void *) &modex, modex_size);
-
-    /*
-     * need global for edge cases like MPI_Comm_spawn support with
-     * new ranks started on the same nodes as the spawnee ranks, etc.
-     */
-
-    OPAL_MODEX_SEND(rc, PMIX_GLOBAL,
-                    &mca_btl_ugni_component.super.btl_version,
-                    modex_msg, modex_size);
-
-    free (modex_msg);
-
-    return rc;
-}
-
-int mca_btl_ugni_fini (void)
-{
-    return OPAL_SUCCESS;
-}
-
-int mca_btl_ugni_init (void)
-{
-    int32_t pid_max = 32768;
-    int rc, bit;
-    FILE *fh;
-
-    if (0 == mca_btl_ugni_component.virtual_device_count) {
-        int core_count;
-
-        (void) opal_hwloc_base_get_topology ();
-        core_count = hwloc_get_nbobjs_by_type (opal_hwloc_topology, HWLOC_OBJ_CORE);
-
-        if (core_count <= opal_process_info.num_local_peers || !opal_using_threads()) {
-            /* there is probably no benefit to using multiple device contexts when not
-             * using threads. */
-            mca_btl_ugni_component.virtual_device_count = 1;
-        } else {
-            mca_btl_ugni_component.virtual_device_count = core_count / (opal_process_info.num_local_peers + 1);
-        }
-    }
-
-    if (MCA_BTL_UGNI_MAX_DEV_HANDLES < mca_btl_ugni_component.virtual_device_count) {
-        mca_btl_ugni_component.virtual_device_count = MCA_BTL_UGNI_MAX_DEV_HANDLES;
-    }
-
-    if (0 == mca_btl_ugni_component.local_rdma_cq_size) {
-        if (1 == mca_btl_ugni_component.virtual_device_count) {
-            mca_btl_ugni_component.local_rdma_cq_size = 2048;
-        } else {
-            mca_btl_ugni_component.local_rdma_cq_size = 256;
-        }
-    }
-
-    if ((mca_btl_ugni_component.virtual_device_count * (1 + opal_process_info.num_local_peers)) < 122) {
-        /* if there are fewer total devices than FMA descriptors it makes sense to turn off FMA sharing.
-         * *DO NOT* override a user requested flag. */
-        mca_base_var_source_t source = MCA_BASE_VAR_SOURCE_DEFAULT;
-
-        mca_base_var_get_value (mca_btl_ugni_component.cdm_flags_id, NULL, &source, NULL);
-        if (MCA_BASE_VAR_SOURCE_DEFAULT == source) {
-            BTL_VERBOSE(("disabling shared FMA sharing"));
-
-            mca_btl_ugni_component.cdm_flags &= ~GNI_CDM_MODE_FMA_SHARED;
-            mca_btl_ugni_component.cdm_flags |= GNI_CDM_MODE_FMA_DEDICATED;
-        }
-    }
-
-    fh = fopen ("/proc/sys/kernel/pid_max", "r");
-    if (NULL != fh) {
-        fscanf (fh, "%d", &pid_max);
-        fclose (fh);
-    }
-
-    /* Use pid to generate the cdm_id.  Although its not stated in the uGNI
-     * documentation, the cdm_id only needs to be unique within a node for a
-     * given ptag/cookie tuple */
-    bit = opal_hibit (pid_max, 31);
-    if (bit >= 31) {
-        mca_btl_ugni_component.virtual_device_count = 1;
-        mca_btl_ugni_component.cdm_id_base = getpid();
-    } else if (bit >= 30 && mca_btl_ugni_component.virtual_device_count > 2) {
-        mca_btl_ugni_component.virtual_device_count = 2;
-        mca_btl_ugni_component.cdm_id_base = getpid() << 1;
-    } else {
-        mca_btl_ugni_component.cdm_id_base = getpid() << 8;
-    }
-
-    /* Create a communication domain */
-    /* collect uGNI information */
-    rc = get_ptag(&mca_btl_ugni_component.ptag);
-    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-        return rc;
-    }
-
-    rc = get_cookie(&mca_btl_ugni_component.cookie);
-    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-        return rc;
-    }
-
-    /* get the device address of the NIC */
-    mca_btl_ugni_component.dev_addr = mca_btl_ugni_get_nic_address (0);
-
-    /* send ugni modex */
-    mca_btl_ugni_send_modex ();
-
-    return OPAL_SUCCESS;
-}
diff --git a/opal/mca/btl/ugni/btl_ugni_module.c b/opal/mca/btl/ugni/btl_ugni_module.c
deleted file mode 100644
index 3993e4c2524..00000000000
--- a/opal/mca/btl/ugni/btl_ugni_module.c
+++ /dev/null
@@ -1,362 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2011      UT-Battelle, LLC. All rights reserved.
- * Copyright (c) 2014-2016 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      Intel, Inc.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "opal_config.h"
-
-#include "btl_ugni.h"
-#include "btl_ugni_frag.h"
-#include "btl_ugni_endpoint.h"
-#include "btl_ugni_prepare.h"
-#include "btl_ugni_smsg.h"
-
-static int
-mca_btl_ugni_free (struct mca_btl_base_module_t *btl,
-                   mca_btl_base_descriptor_t *des);
-
-static int
-mca_btl_ugni_module_finalize (struct mca_btl_base_module_t* btl);
-
-static struct mca_btl_base_descriptor_t *
-mca_btl_ugni_prepare_src (struct mca_btl_base_module_t *btl,
-                          struct mca_btl_base_endpoint_t *endpoint,
-                          struct opal_convertor_t *convertor,
-                          uint8_t order, size_t reserve, size_t *size,
-                          uint32_t flags);
-
-static mca_btl_base_registration_handle_t *
-mca_btl_ugni_register_mem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *base,
-                           size_t size, uint32_t flags);
-
-static int mca_btl_ugni_deregister_mem (mca_btl_base_module_t *btl, mca_btl_base_registration_handle_t *handle);
-
-mca_btl_ugni_module_t mca_btl_ugni_module = {
-    .super = {
-        .btl_component      = &mca_btl_ugni_component.super,
-        .btl_add_procs      = mca_btl_ugni_add_procs,
-        .btl_del_procs      = mca_btl_ugni_del_procs,
-        .btl_finalize       = mca_btl_ugni_module_finalize,
-        .btl_alloc          = mca_btl_ugni_alloc,
-        .btl_free           = mca_btl_ugni_free,
-        .btl_prepare_src    = mca_btl_ugni_prepare_src,
-        .btl_send           = mca_btl_ugni_send,
-        .btl_sendi          = mca_btl_ugni_sendi,
-        .btl_put            = mca_btl_ugni_put,
-        .btl_get            = mca_btl_ugni_get,
-        .btl_register_mem   = mca_btl_ugni_register_mem,
-        .btl_deregister_mem = mca_btl_ugni_deregister_mem,
-        .btl_atomic_op      = mca_btl_ugni_aop,
-        .btl_atomic_fop     = mca_btl_ugni_afop,
-        .btl_atomic_cswap   = mca_btl_ugni_acswap,
-        .btl_flush          = mca_btl_ugni_flush,
-    }
-};
-
-static void mca_btl_ugni_datagram_event (int foo, short bar, void *arg)
-{
-    mca_btl_ugni_module_t *ugni_module = (mca_btl_ugni_module_t *) arg;
-    mca_btl_ugni_device_t *device = ugni_module->devices;
-    struct timeval tv = {.tv_sec = 0, .tv_usec = MCA_BTL_UGNI_CONNECT_USEC};
-
-    mca_btl_ugni_progress_datagram (device);
-
-    opal_event_evtimer_add (&ugni_module->connection_event, &tv);
-}
-
-int
-mca_btl_ugni_module_init (mca_btl_ugni_module_t *ugni_module)
-{
-    int rc;
-
-    BTL_VERBOSE(("binding module %p to device 0", (void *) ugni_module));
-
-    /* copy module defaults (and function pointers) */
-    memmove (ugni_module, &mca_btl_ugni_module, sizeof (mca_btl_ugni_module));
-
-    ugni_module->initialized = false;
-    ugni_module->nlocal_procs = 0;
-    ugni_module->active_datagrams = 0;
-    ugni_module->active_rdma_count = 0;
-
-    opal_event_evtimer_set (opal_sync_event_base, &ugni_module->connection_event,
-                            mca_btl_ugni_datagram_event, ugni_module);
-
-    OBJ_CONSTRUCT(&ugni_module->failed_frags, opal_list_t);
-    OBJ_CONSTRUCT(&ugni_module->failed_frags_lock, opal_mutex_t);
-
-    OBJ_CONSTRUCT(&ugni_module->eager_get_pending, opal_list_t);
-    OBJ_CONSTRUCT(&ugni_module->eager_get_pending_lock,opal_mutex_t);
-
-    for (int i = 0 ; i < MCA_BTL_UGNI_LIST_MAX ; ++i) {
-        OBJ_CONSTRUCT(ugni_module->frags_lists + i, opal_free_list_t);
-    }
-
-    OBJ_CONSTRUCT(&ugni_module->pending_smsg_frags_bb, opal_pointer_array_t);
-    OBJ_CONSTRUCT(&ugni_module->ep_wait_list_lock,opal_mutex_t);
-    OBJ_CONSTRUCT(&ugni_module->ep_wait_list, opal_list_t);
-    OBJ_CONSTRUCT(&ugni_module->endpoint_lock, opal_mutex_t);
-    OBJ_CONSTRUCT(&ugni_module->endpoints, opal_pointer_array_t);
-    OBJ_CONSTRUCT(&ugni_module->id_to_endpoint, opal_hash_table_t);
-    OBJ_CONSTRUCT(&ugni_module->smsg_mboxes, opal_free_list_t);
-    OBJ_CONSTRUCT(&ugni_module->eager_get_pending, opal_list_t);
-
-    /* set up virtual device handles */
-    for (int i = 0 ; i < mca_btl_ugni_component.virtual_device_count ; ++i) {
-        rc = mca_btl_ugni_device_init (ugni_module->devices + i, i);
-        if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-            BTL_VERBOSE(("error initializing uGNI device handle"));
-            return rc;
-        }
-    }
-
-    /* create wildcard endpoint on first device to listen for connections.
-     * there is no need to bind this endpoint. We are single threaded
-     * here so there is no need for a device lock. */
-    rc = GNI_EpCreate (ugni_module->devices[0].dev_handle, NULL,
-                       &ugni_module->wildcard_ep);
-    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-        BTL_ERROR(("error creating wildcard ugni endpoint"));
-        return mca_btl_rc_ugni_to_opal (rc);
-    }
-
-    /* post wildcard datagram */
-    rc = mca_btl_ugni_wildcard_ep_post (ugni_module);
-    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-        BTL_ERROR(("error posting wildcard datagram"));
-        return rc;
-    }
-
-    return OPAL_SUCCESS;
-}
-
-static int
-mca_btl_ugni_module_finalize (struct mca_btl_base_module_t *btl)
-{
-    mca_btl_ugni_module_t *ugni_module = (mca_btl_ugni_module_t *)btl;
-    mca_btl_base_endpoint_t *ep;
-    uint64_t key;
-    int rc;
-
-    if (ugni_module->initialized) {
-        /* close all open connections and release endpoints */
-        OPAL_HASH_TABLE_FOREACH(key, uint64, ep, &ugni_module->id_to_endpoint) {
-            if (NULL != ep) {
-                mca_btl_ugni_release_ep (ep);
-            }
-        }
-
-        if (mca_btl_ugni_component.progress_thread_enabled) {
-            mca_btl_ugni_kill_progress_thread();
-        }
-
-        /* destroy all cqs */
-        rc = GNI_CqDestroy (ugni_module->smsg_remote_cq);
-        if (GNI_RC_SUCCESS != rc) {
-            BTL_ERROR(("error tearing down RX SMSG CQ - %s",gni_err_str[rc]));
-        }
-
-        if (mca_btl_ugni_component.progress_thread_enabled) {
-            rc = GNI_CqDestroy (ugni_module->smsg_remote_irq_cq);
-            if (GNI_RC_SUCCESS != rc) {
-                BTL_ERROR(("error tearing down remote SMSG CQ - %s",gni_err_str[rc]));
-            }
-        }
-
-        /* cancel wildcard post */
-        rc = GNI_EpPostDataCancelById (ugni_module->wildcard_ep,
-                                       MCA_BTL_UGNI_CONNECT_WILDCARD_ID |
-                                       OPAL_PROC_MY_NAME.vpid);
-        if (GNI_RC_SUCCESS != rc) {
-            BTL_VERBOSE(("btl/ugni error cancelling wildcard post"));
-        }
-
-        /* tear down wildcard endpoint */
-        rc = GNI_EpDestroy (ugni_module->wildcard_ep);
-        if (GNI_RC_SUCCESS != rc) {
-            BTL_VERBOSE(("btl/ugni error destroying endpoint - %s",gni_err_str[rc]));
-        }
-
-        opal_event_del (&ugni_module->connection_event);
-    }
-
-    for (int i = 0 ; i < MCA_BTL_UGNI_LIST_MAX ; ++i) {
-        OBJ_DESTRUCT(ugni_module->frags_lists + i);
-    }
-
-    OBJ_DESTRUCT(&ugni_module->ep_wait_list);
-    OBJ_DESTRUCT(&ugni_module->smsg_mboxes);
-    OBJ_DESTRUCT(&ugni_module->pending_smsg_frags_bb);
-    OBJ_DESTRUCT(&ugni_module->id_to_endpoint);
-    OBJ_DESTRUCT(&ugni_module->endpoint_lock);
-    OBJ_DESTRUCT(&ugni_module->endpoints);
-
-    OBJ_DESTRUCT(&ugni_module->eager_get_pending);
-    OBJ_DESTRUCT(&ugni_module->eager_get_pending_lock);
-
-    if (ugni_module->rcache) {
-        mca_rcache_base_module_destroy (ugni_module->rcache);
-    }
-
-    for (int i = 0 ; i < mca_btl_ugni_component.virtual_device_count ; ++i) {
-        mca_btl_ugni_device_fini (ugni_module->devices + i);
-    }
-
-    ugni_module->initialized = false;
-
-    return OPAL_SUCCESS;
-}
-
-
-mca_btl_base_descriptor_t *
-mca_btl_ugni_alloc(struct mca_btl_base_module_t *btl,
-                   struct mca_btl_base_endpoint_t *endpoint,
-                   uint8_t order, size_t size, uint32_t flags)
-{
-    mca_btl_ugni_base_frag_t *frag = NULL;
-
-    /* do not allocate a fragment unless the wait list is relatively small. this
-     * reduces the potential for resource exhaustion. note the wait list only exists
-     * because we have no way to notify the sender that credits are available. */
-    if (OPAL_UNLIKELY(opal_list_get_size (&endpoint->frag_wait_list) > 32)) {
-        return NULL;
-    }
-
-    if (size <= mca_btl_ugni_component.smsg_max_data) {
-        frag = mca_btl_ugni_frag_alloc_smsg (endpoint);
-    } else if (size <= btl->btl_eager_limit) {
-        frag = mca_btl_ugni_frag_alloc_eager_send (endpoint);
-    }
-
-    if (OPAL_UNLIKELY(NULL == frag)) {
-        return NULL;
-    }
-
-    BTL_VERBOSE(("btl/ugni_module allocated frag of size: %u, flags: %x. frag = %p",
-                 (unsigned int)size, flags, (void *) frag));
-
-    frag->base.des_flags = flags;
-    frag->base.order = order;
-    frag->base.des_segments = &frag->segments[1];
-    frag->base.des_segment_count = 1;
-
-    frag->segments[0].seg_addr.pval = NULL;
-    frag->segments[0].seg_len       = 0;
-    frag->segments[1].seg_addr.pval = frag->base.super.ptr;
-    frag->segments[1].seg_len       = size;
-
-    frag->flags = MCA_BTL_UGNI_FRAG_BUFFERED;
-    if (size > mca_btl_ugni_component.smsg_max_data) {
-        mca_btl_ugni_reg_t *registration;
-
-        frag->hdr_size = sizeof (frag->hdr.eager);
-        frag->flags    |= MCA_BTL_UGNI_FRAG_EAGER | MCA_BTL_UGNI_FRAG_IGNORE;
-
-        registration = (mca_btl_ugni_reg_t *) frag->base.super.registration;
-
-        frag->hdr.eager.memory_handle = registration->handle;
-    } else {
-        frag->hdr_size = sizeof (frag->hdr.send);
-    }
-
-    return &frag->base;
-}
-
-static int
-mca_btl_ugni_free (struct mca_btl_base_module_t *btl,
-                   mca_btl_base_descriptor_t *des)
-{
-    return mca_btl_ugni_frag_return ((mca_btl_ugni_base_frag_t *) des);
-}
-
-static struct mca_btl_base_descriptor_t *
-mca_btl_ugni_prepare_src (struct mca_btl_base_module_t *btl,
-                          mca_btl_base_endpoint_t *endpoint,
-                          struct opal_convertor_t *convertor,
-                          uint8_t order, size_t reserve, size_t *size,
-                          uint32_t flags)
-{
-    /* do not allocate a fragment unless the wait list is relatively small. this
-     * reduces the potential for resource exhaustion. note the wait list only exists
-     * because we have no way to notify the sender that credits are available. */
-    if (OPAL_UNLIKELY(opal_list_get_size (&endpoint->frag_wait_list) > 32)) {
-        return NULL;
-    }
-
-    return mca_btl_ugni_prepare_src_send (btl, endpoint, convertor,
-                                          order, reserve, size, flags);
-}
-
-static mca_btl_base_registration_handle_t *
-mca_btl_ugni_register_mem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *base,
-                           size_t size, uint32_t flags)
-{
-    mca_btl_ugni_module_t *ugni_module = (mca_btl_ugni_module_t *) btl;
-    mca_btl_ugni_reg_t *reg;
-    int access_flags = flags & MCA_BTL_REG_FLAG_ACCESS_ANY;
-    int rc;
-
-    rc = ugni_module->rcache->rcache_register (ugni_module->rcache, base, size, 0, access_flags,
-                                               (mca_rcache_base_registration_t **) &reg);
-    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-        return NULL;
-    }
-
-    return &reg->handle;
-}
-
-static int mca_btl_ugni_deregister_mem (mca_btl_base_module_t *btl, mca_btl_base_registration_handle_t *handle)
-{
-    mca_btl_ugni_module_t *ugni_module = (mca_btl_ugni_module_t *) btl;
-    mca_btl_ugni_reg_t *reg =
-        (mca_btl_ugni_reg_t *)((intptr_t) handle - offsetof (mca_btl_ugni_reg_t, handle));
-
-    (void) ugni_module->rcache->rcache_deregister (ugni_module->rcache, &reg->base);
-
-    return OPAL_SUCCESS;
-}
-
-int mca_btl_ugni_event_fatal_error (gni_return_t grc, gni_cq_entry_t event_data)
-{
-    /* combined error check for get event and get completed. we might miss exactly
-     * what happened but it is unrecoverable anyway. fwiw, this error path has
-     * never been seen in production. */
-    if (GNI_CQ_OVERRUN(event_data)) {
-        /* TODO -- need to handle overrun -- how do we do this without an event?
-           will the event eventually come back? Ask Cray */
-        BTL_ERROR(("CQ overrun detected in RDMA event data. can not recover"));
-    } else {
-        BTL_ERROR(("Error in GNI_GetComplete %s", gni_err_str[grc]));
-    }
-
-    return mca_btl_rc_ugni_to_opal (grc);
-}
-
-int mca_btl_ugni_device_handle_event_error (mca_btl_ugni_rdma_desc_t *rdma_desc, gni_cq_entry_t event_data)
-{
-    mca_btl_ugni_device_t *device = rdma_desc->device;
-    uint32_t recoverable = 1;
-
-    (void) GNI_CqErrorRecoverable (event_data, &recoverable);
-
-    if (OPAL_UNLIKELY(++rdma_desc->tries >= mca_btl_ugni_component.rdma_max_retries || !recoverable)) {
-        char char_buffer[1024];
-        GNI_CqErrorStr (event_data, char_buffer, sizeof (char_buffer));
-
-        BTL_ERROR(("giving up on desciptor %p, recoverable %d: %s", (void *) rdma_desc, recoverable, char_buffer));
-
-        return OPAL_ERROR;
-    }
-
-    return _mca_btl_ugni_repost_rdma_desc_device (device, rdma_desc);
-}
diff --git a/opal/mca/btl/ugni/btl_ugni_prepare.h b/opal/mca/btl/ugni/btl_ugni_prepare.h
deleted file mode 100644
index 9d2da1954cf..00000000000
--- a/opal/mca/btl/ugni/btl_ugni_prepare.h
+++ /dev/null
@@ -1,200 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2011-2015 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2011      UT-Battelle, LLC. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#if !defined(MCA_BTL_UGNI_PREPARE_H)
-#define MCA_BTL_UGNI_PREPARE_H
-
-#include "opal_config.h"
-
-#include "btl_ugni.h"
-#include "btl_ugni_frag.h"
-
-static inline struct mca_btl_base_descriptor_t *
-mca_btl_ugni_prepare_src_send_nodata (struct mca_btl_base_module_t *btl,
-                                      mca_btl_base_endpoint_t *endpoint,
-                                      uint8_t order, size_t reserve,
-                                      uint32_t flags)
-{
-    mca_btl_ugni_base_frag_t *frag = NULL;
-
-    frag = mca_btl_ugni_frag_alloc_rdma (endpoint);
-    if (OPAL_UNLIKELY(NULL == frag)) {
-        return NULL;
-    }
-
-    BTL_VERBOSE(("preparing src for send fragment. size = %u", (unsigned int) reserve));
-
-    frag->hdr_size = reserve + sizeof (frag->hdr.send);
-
-    frag->segments[0].seg_addr.pval = frag->hdr.send_ex.pml_header;
-    frag->segments[0].seg_len       = reserve;
-
-    frag->segments[1].seg_addr.pval = NULL;
-    frag->segments[1].seg_len       = 0;
-
-    frag->base.des_segments      = frag->segments;
-    frag->base.des_segment_count = 1;
-    frag->base.order           = order;
-    frag->base.des_flags       = flags;
-
-    return &frag->base;
-}
-
-static inline struct mca_btl_base_descriptor_t *
-mca_btl_ugni_prepare_src_send_inplace (struct mca_btl_base_module_t *btl,
-                                       mca_btl_base_endpoint_t *endpoint,
-                                       struct opal_convertor_t *convertor,
-                                       uint8_t order, size_t reserve, size_t *size,
-                                       uint32_t flags)
-{
-    bool use_eager_get = (*size + reserve) > mca_btl_ugni_component.smsg_max_data;
-    mca_btl_ugni_module_t *ugni_module = (mca_btl_ugni_module_t *) btl;
-    mca_btl_ugni_base_frag_t *frag = NULL;
-    mca_btl_ugni_reg_t *registration = NULL;
-    void *data_ptr;
-    int rc;
-
-    opal_convertor_get_current_pointer (convertor, &data_ptr);
-
-    frag = mca_btl_ugni_frag_alloc_rdma (endpoint);
-    if (OPAL_UNLIKELY(NULL == frag)) {
-        return NULL;
-    }
-
-    BTL_VERBOSE(("preparing src for send fragment. size = %u",
-                 (unsigned int)(*size + reserve)));
-
-    if (OPAL_UNLIKELY(true == use_eager_get)) {
-        rc = ugni_module->rcache->rcache_register (ugni_module->rcache, data_ptr, *size, 0,
-                                                   MCA_RCACHE_ACCESS_REMOTE_READ,
-                                                   (mca_rcache_base_registration_t **)&registration);
-        if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-            mca_btl_ugni_frag_return (frag);
-            return NULL;
-        }
-
-        frag->flags = MCA_BTL_UGNI_FRAG_EAGER | MCA_BTL_UGNI_FRAG_IGNORE;
-
-        frag->registration = registration;
-        frag->hdr.eager.memory_handle = registration->handle;;
-
-        frag->hdr_size = reserve + sizeof (frag->hdr.eager);
-        frag->segments[0].seg_addr.pval = frag->hdr.eager_ex.pml_header;
-    } else {
-        frag->hdr_size = reserve + sizeof (frag->hdr.send);
-        frag->segments[0].seg_addr.pval = frag->hdr.send_ex.pml_header;
-    }
-
-    frag->segments[0].seg_len       = reserve;
-
-    frag->segments[1].seg_addr.pval = data_ptr;
-    frag->segments[1].seg_len       = *size;
-
-    frag->base.des_segments       = frag->segments;
-    frag->base.des_segment_count = 2;
-    frag->base.order           = order;
-    frag->base.des_flags       = flags;
-
-    return &frag->base;
-}
-
-static inline struct mca_btl_base_descriptor_t *
-mca_btl_ugni_prepare_src_send_buffered (struct mca_btl_base_module_t *btl,
-                                        mca_btl_base_endpoint_t *endpoint,
-                                        struct opal_convertor_t *convertor,
-                                        uint8_t order, size_t reserve, size_t *size,
-                                        uint32_t flags)
-{
-    bool use_eager_get = (*size + reserve) > mca_btl_ugni_component.smsg_max_data;
-    mca_btl_ugni_reg_t *registration = NULL;
-    mca_btl_ugni_base_frag_t *frag = NULL;
-    uint32_t iov_count = 1;
-    struct iovec iov;
-    int rc;
-
-    if (OPAL_UNLIKELY(true == use_eager_get)) {
-        frag = mca_btl_ugni_frag_alloc_eager_send (endpoint);
-        if (OPAL_UNLIKELY(NULL == frag)) {
-            return NULL;
-        }
-
-        frag->flags = MCA_BTL_UGNI_FRAG_EAGER | MCA_BTL_UGNI_FRAG_IGNORE;
-
-        registration = (mca_btl_ugni_reg_t *) frag->base.super.registration;
-
-        frag->hdr.eager.memory_handle = registration->handle;
-        frag->hdr_size = reserve + sizeof (frag->hdr.eager);
-        frag->segments[0].seg_addr.pval = frag->hdr.eager_ex.pml_header;
-    } else {
-        frag = mca_btl_ugni_frag_alloc_smsg (endpoint);
-        if (OPAL_UNLIKELY(NULL == frag)) {
-            return NULL;
-        }
-
-        frag->hdr_size = reserve + sizeof (frag->hdr.send);
-        frag->segments[0].seg_addr.pval = frag->hdr.send_ex.pml_header;
-    }
-
-    frag->flags |= MCA_BTL_UGNI_FRAG_BUFFERED;
-
-    iov.iov_len  = *size;
-    iov.iov_base = (IOVBASE_TYPE *) frag->base.super.ptr;
-
-    rc = opal_convertor_pack (convertor, &iov, &iov_count, size);
-    if (OPAL_UNLIKELY(rc < 0)) {
-        mca_btl_ugni_frag_return (frag);
-        return NULL;
-    }
-
-    frag->segments[0].seg_len       = reserve;
-
-    frag->segments[1].seg_addr.pval = frag->base.super.ptr;
-    frag->segments[1].seg_len       = *size;
-
-    frag->base.des_segments       = frag->segments;
-    frag->base.des_segment_count = 2;
-    frag->base.order           = order;
-    frag->base.des_flags       = flags;
-
-    return &frag->base;
-}
-
-static inline struct mca_btl_base_descriptor_t *
-mca_btl_ugni_prepare_src_send (struct mca_btl_base_module_t *btl,
-                               mca_btl_base_endpoint_t *endpoint,
-                               struct opal_convertor_t *convertor,
-                               uint8_t order, size_t reserve, size_t *size,
-                               uint32_t flags)
-{
-    bool use_eager_get = (*size + reserve) > mca_btl_ugni_component.smsg_max_data;
-    bool send_in_place;
-    void *data_ptr;
-
-    if (!(*size)) {
-        return mca_btl_ugni_prepare_src_send_nodata (btl, endpoint, order, reserve, flags);
-    }
-
-    opal_convertor_get_current_pointer (convertor, &data_ptr);
-
-    send_in_place = (btl->btl_flags & MCA_BTL_FLAGS_SEND_INPLACE) && !(opal_convertor_need_buffers(convertor) ||
-                                                                       (use_eager_get && ((uintptr_t)data_ptr & 3)));
-
-    if (send_in_place) {
-        return mca_btl_ugni_prepare_src_send_inplace (btl, endpoint, convertor, order,
-                                                      reserve, size, flags);
-    } else {
-        return mca_btl_ugni_prepare_src_send_buffered (btl, endpoint, convertor, order,
-                                                       reserve, size, flags);
-    }
-}
-
-#endif
diff --git a/opal/mca/btl/ugni/btl_ugni_progress_thread.c b/opal/mca/btl/ugni/btl_ugni_progress_thread.c
deleted file mode 100644
index 3fa03911409..00000000000
--- a/opal/mca/btl/ugni/btl_ugni_progress_thread.c
+++ /dev/null
@@ -1,132 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2011-2017 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2011      UT-Battelle, LLC. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "opal_config.h"
-
-#include "btl_ugni.h"
-#include "btl_ugni_frag.h"
-#include "btl_ugni_smsg.h"
-
-#include "opal/include/opal/align.h"
-
-
-static pthread_t mca_btl_ugni_progress_thread_id;
-
-static volatile int stop_progress_thread = 0;
-
-unsigned int mca_btl_ugni_progress_thread_wakeups = 0;
-
-static void *mca_btl_ugni_prog_thread_fn(void * data)
-{
-    uint32_t which;
-    gni_return_t status;
-    gni_cq_handle_t cq_vec[1 + MCA_BTL_UGNI_MAX_DEV_HANDLES];
-
-    struct mca_btl_ugni_module_t *btl = (mca_btl_ugni_module_t *)data;
-    int cq_count = 1 + mca_btl_ugni_component.virtual_device_count;
-
-    /*
-     * need to block signals
-     */
-
-    cq_vec[0] = btl->smsg_remote_irq_cq;
-    for (int i = 0 ; i < mca_btl_ugni_component.virtual_device_count ; ++i) {
-        cq_vec[i + 1] = btl->devices[i].dev_rdma_local_irq_cq.gni_handle;
-    }
-
-    while (stop_progress_thread == 0) {
-
-        /*
-         * this ugni call doesn't need a lock
-         */
-
-        status = GNI_CqVectorMonitor(cq_vec,
-                                     cq_count,
-                                     -1,
-                                     &which);
-
-        if (status == GNI_RC_NOT_DONE) continue;
-
-        if ((status == GNI_RC_SUCCESS) && (stop_progress_thread == 0)) {
-            mca_btl_ugni_progress_thread_wakeups++;
-            opal_progress();
-        }
-    }
-
-    return (void *) (intptr_t) OPAL_SUCCESS;
-}
-
-int mca_btl_ugni_spawn_progress_thread(struct mca_btl_base_module_t *btl)
-{
-    int rc, ret=OPAL_SUCCESS;
-    pthread_attr_t attr;
-
-    pthread_attr_init(&attr);
-    rc = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
-    if (0 != rc) {
-        BTL_ERROR(("btl/ugni pthread_attr_setdetachstate returned %s ",strerror(rc)));
-        ret = OPAL_ERROR;
-        goto fn_exit;
-    }
-
-    rc = pthread_create(&mca_btl_ugni_progress_thread_id,
-                        &attr, mca_btl_ugni_prog_thread_fn, (void *)btl);
-    if (0 != rc) {
-        BTL_ERROR(("btl/ugni pthread_create returned %s ",strerror(rc)));
-        ret = OPAL_ERROR;
-        goto fn_exit;
-    }
-
-    rc = pthread_attr_destroy(&attr);
-    if (0 != rc) {
-        BTL_ERROR(("btl/ugni pthread_attr_destory returned %s ",strerror(rc)));
-        ret = OPAL_ERROR;
-    }
-
-   fn_exit:
-    return ret;
-}
-
-int mca_btl_ugni_kill_progress_thread(void)
-{
-    int ret=OPAL_SUCCESS;
-    void *thread_rc;
-
-    stop_progress_thread = 1;
-
-    /*
-     * post a CQ to myself to wake my thread up
-     */
-
-    ret = mca_btl_ugni_post_cqwrite (mca_btl_ugni_component.modules[0].local_ep,
-                                     &mca_btl_ugni_component.modules[0].devices[0].dev_rdma_local_cq,
-                                     mca_btl_ugni_component.modules[0].devices[0].smsg_irq_mhndl,
-                                     0xdead, NULL, NULL, NULL);
-    /*
-     * TODO: if error returned, need to kill off thread manually
-     */
-    if (OPAL_SUCCESS != ret) {
-        /* force the thread to exit */
-        pthread_cancel (mca_btl_ugni_progress_thread_id);
-        goto fn_exit;
-    }
-
-    pthread_join (mca_btl_ugni_progress_thread_id, &thread_rc);
-    if (0 != (intptr_t) thread_rc) {
-        BTL_ERROR(("btl/ugni error returned from progress thread: %d", (int) (intptr_t) thread_rc));
-        ret = (int)(intptr_t) thread_rc;
-    }
-
-   fn_exit:
-    return ret;
-}
-
diff --git a/opal/mca/btl/ugni/btl_ugni_put.c b/opal/mca/btl/ugni/btl_ugni_put.c
deleted file mode 100644
index 71ab146f040..00000000000
--- a/opal/mca/btl/ugni/btl_ugni_put.c
+++ /dev/null
@@ -1,27 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2011-2017 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2011      UT-Battelle, LLC. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-#include "opal_config.h"
-#include "opal/include/opal_stdint.h"
-
-#include "btl_ugni_rdma.h"
-
-int mca_btl_ugni_put (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, void *local_address,
-                      uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                      mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-                      int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
-{
-    BTL_VERBOSE(("Using RDMA/FMA Put %lu bytes from local address %p to remote address %" PRIx64,
-                 (unsigned long) size, local_address, remote_address));
-
-    return mca_btl_ugni_post (endpoint, false, size, local_address, remote_address, local_handle,
-                              remote_handle, order, cbfunc, cbcontext, cbdata);
-}
diff --git a/opal/mca/btl/ugni/btl_ugni_rdma.h b/opal/mca/btl/ugni/btl_ugni_rdma.h
deleted file mode 100644
index 31096d2fcb4..00000000000
--- a/opal/mca/btl/ugni/btl_ugni_rdma.h
+++ /dev/null
@@ -1,149 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2011      UT-Battelle, LLC. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#if !defined(MCA_BTL_UGNI_RDMA_H)
-#define MCA_BTL_UGNI_RDMA_H
-
-#include "btl_ugni.h"
-#include "btl_ugni_frag.h"
-#include "btl_ugni_device.h"
-
-int mca_btl_ugni_start_eager_get (mca_btl_base_endpoint_t *ep,
-                                  mca_btl_ugni_eager_ex_frag_hdr_t hdr,
-                                  mca_btl_ugni_base_frag_t *frag);
-
-static inline void init_post_desc (mca_btl_ugni_post_descriptor_t *post_desc,
-                                   mca_btl_base_endpoint_t *endpoint, int order,
-                                   gni_post_type_t op_type, uint64_t lcl_addr,
-                                   gni_mem_handle_t lcl_mdh, uint64_t rem_addr,
-                                   gni_mem_handle_t rem_mdh, uint64_t bufsize,
-                                   gni_cq_handle_t cq_hndl, mca_btl_base_rdma_completion_fn_t cbfunc,
-                                   void *cbcontext, void *cbdata,
-                                   mca_btl_base_registration_handle_t *local_handle) {
-    post_desc->endpoint = endpoint;
-    post_desc->cbfunc = cbfunc;
-    post_desc->ctx = cbcontext;
-    post_desc->cbdata = cbdata;
-    post_desc->local_handle = local_handle;
-    post_desc->gni_desc.type            = op_type;
-    post_desc->gni_desc.cq_mode         = GNI_CQMODE_GLOBAL_EVENT;
-    if (MCA_BTL_NO_ORDER == order) {
-        post_desc->gni_desc.dlvr_mode       = GNI_DLVMODE_PERFORMANCE;
-    } else {
-        post_desc->gni_desc.dlvr_mode       = GNI_DLVMODE_NO_ADAPT;
-    }
-    post_desc->gni_desc.local_addr      = (uint64_t) lcl_addr;
-    post_desc->gni_desc.local_mem_hndl  = lcl_mdh;
-    post_desc->gni_desc.remote_addr     = (uint64_t) rem_addr;
-    post_desc->gni_desc.remote_mem_hndl = rem_mdh;
-    post_desc->gni_desc.length          = bufsize;
-    post_desc->gni_desc.rdma_mode       = 0;
-    post_desc->gni_desc.src_cq_hndl     = cq_hndl;
-}
-
-__opal_attribute_always_inline__
-static inline int mca_btl_ugni_post_fma (struct mca_btl_base_endpoint_t *endpoint, gni_post_type_t op_type,
-                                         size_t size, void *local_address, uint64_t remote_address,
-                                         mca_btl_base_registration_handle_t *local_handle,
-                                         mca_btl_base_registration_handle_t *remote_handle,
-                                         int order, mca_btl_base_rdma_completion_fn_t cbfunc,
-                                         void *cbcontext, void *cbdata)
-{
-    mca_btl_ugni_post_descriptor_t post_desc;
-    gni_mem_handle_t local_gni_handle = {0, 0};
-
-    if (local_handle) {
-        local_gni_handle = local_handle->gni_handle;
-    }
-
-    /* Post descriptor (CQ is ignored for FMA transactions) -- The CQ associated with the endpoint
-     * is used. */
-    init_post_desc (&post_desc, endpoint, order, op_type, (intptr_t) local_address, local_gni_handle,
-                    remote_address, remote_handle->gni_handle, size, 0, cbfunc, cbcontext, cbdata,
-                    local_handle);
-
-    return mca_btl_ugni_endpoint_post_fma (endpoint, &post_desc);
-}
-
-static inline int mca_btl_ugni_post_bte (mca_btl_base_endpoint_t *endpoint, gni_post_type_t op_type,
-                                         size_t size, void *local_address, uint64_t remote_address,
-                                         mca_btl_base_registration_handle_t *local_handle,
-                                         mca_btl_base_registration_handle_t *remote_handle,
-                                         int order, mca_btl_base_rdma_completion_fn_t cbfunc,
-                                         void *cbcontext, void *cbdata)
-{
-    mca_btl_ugni_module_t *module = mca_btl_ugni_ep_btl (endpoint);
-    mca_btl_ugni_post_descriptor_t post_desc;
-    int rc;
-
-    /* There is a performance benefit to throttling the total number of active BTE tranactions. Not sure
-     * what the optimium is but the limit is inforced as a soft limit. */
-    if (module->active_rdma_count >= mca_btl_ugni_component.active_rdma_threshold) {
-        return OPAL_ERR_OUT_OF_RESOURCE;
-    }
-
-    (void) OPAL_THREAD_FETCH_ADD32 (&module->active_rdma_count, 1);
-
-    /* Post descriptor */
-    init_post_desc (&post_desc, endpoint, order, op_type, (intptr_t) local_address, local_handle->gni_handle,
-                    remote_address, remote_handle->gni_handle, size, 0, cbfunc, cbcontext, cbdata,
-                    local_handle);
-
-    rc = mca_btl_ugni_endpoint_post_rdma (endpoint, &post_desc);
-    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-        (void) OPAL_THREAD_FETCH_ADD32 (&module->active_rdma_count, -1);
-    }
-
-    return rc;
-}
-
-static inline int mca_btl_ugni_post_cqwrite (mca_btl_base_endpoint_t *endpoint, mca_btl_ugni_cq_t *cq,
-                                             gni_mem_handle_t irq_mhndl, uint64_t value,
-                                             mca_btl_base_rdma_completion_fn_t cbfunc,
-                                             void *cbcontext, void *cbdata)
-{
-    mca_btl_ugni_post_descriptor_t post_desc;
-
-    post_desc.gni_desc.type = GNI_POST_CQWRITE;
-    post_desc.gni_desc.cqwrite_value = value;   /* up to 48 bytes here, not used for now */
-    post_desc.gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT;
-    post_desc.gni_desc.dlvr_mode = GNI_DLVMODE_IN_ORDER;
-    post_desc.gni_desc.src_cq_hndl = cq->gni_handle;
-    post_desc.gni_desc.remote_mem_hndl = irq_mhndl;
-    post_desc.cq = cq;
-
-    return mca_btl_ugni_endpoint_post_cqwrite (endpoint, &post_desc);
-}
-
-__opal_attribute_always_inline__
-static inline int mca_btl_ugni_post (mca_btl_base_endpoint_t *endpoint, int get, size_t size,
-                                     void *local_address, uint64_t remote_address,
-                                     mca_btl_base_registration_handle_t *local_handle,
-                                     mca_btl_base_registration_handle_t *remote_handle,
-                                     int order, mca_btl_base_rdma_completion_fn_t cbfunc,
-                                     void *cbcontext, void *cbdata)
-{
-    const gni_post_type_t fma_ops[2] = {GNI_POST_FMA_PUT, GNI_POST_FMA_GET};
-    const gni_post_type_t rdma_ops[2] = {GNI_POST_RDMA_PUT, GNI_POST_RDMA_GET};
-    const size_t fma_limit = (size_t) (get ? mca_btl_ugni_component.ugni_fma_get_limit :
-                                       mca_btl_ugni_component.ugni_fma_put_limit);
-
-    if (size <= fma_limit) {
-        return mca_btl_ugni_post_fma (endpoint, fma_ops[get], size, local_address, remote_address,
-                                      local_handle, remote_handle, order, cbfunc, cbcontext, cbdata);
-    }
-
-    return mca_btl_ugni_post_bte (endpoint, rdma_ops[get], size, local_address, remote_address,
-                                  local_handle, remote_handle, order, cbfunc, cbcontext, cbdata);
-}
-
-#endif /* MCA_BTL_UGNI_RDMA_H */
diff --git a/opal/mca/btl/ugni/btl_ugni_send.c b/opal/mca/btl/ugni/btl_ugni_send.c
deleted file mode 100644
index 5b120b75965..00000000000
--- a/opal/mca/btl/ugni/btl_ugni_send.c
+++ /dev/null
@@ -1,199 +0,0 @@
-/* -*- Mode: C; c-basic-offset:3 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2011-2017 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2011      UT-Battelle, LLC. All rights reserved.
- * Copyright (c) 2014      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      Intel, Inc.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "btl_ugni.h"
-#include "btl_ugni_frag.h"
-#include "btl_ugni_smsg.h"
-#include "btl_ugni_prepare.h"
-
-void mca_btl_ugni_wait_list_append (mca_btl_ugni_module_t *ugni_module, mca_btl_base_endpoint_t *endpoint,
-                                    mca_btl_ugni_base_frag_t *frag)
-{
-    BTL_VERBOSE(("wait-listing fragment %p to %s. endpoint state %d\n", (void*)frag, OPAL_NAME_PRINT(endpoint->peer_proc->proc_name), endpoint->state));
-
-    frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
-
-    /* queue up request */
-    OPAL_THREAD_LOCK(&endpoint->lock);
-
-    opal_list_append (&endpoint->frag_wait_list, (opal_list_item_t *) frag);
-
-    OPAL_THREAD_UNLOCK(&endpoint->lock);
-
-    if (false == endpoint->wait_listed && MCA_BTL_UGNI_EP_STATE_CONNECTED == endpoint->state) {
-        OPAL_THREAD_LOCK(&ugni_module->ep_wait_list_lock);
-        if (false == endpoint->wait_listed) {
-            opal_list_append (&ugni_module->ep_wait_list, &endpoint->super);
-            endpoint->wait_listed = true;
-        }
-        OPAL_THREAD_UNLOCK(&ugni_module->ep_wait_list_lock);
-    }
-}
-
-int mca_btl_ugni_send (struct mca_btl_base_module_t *btl,
-                       struct mca_btl_base_endpoint_t *endpoint,
-                       struct mca_btl_base_descriptor_t *descriptor,
-                       mca_btl_base_tag_t tag)
-{
-    mca_btl_ugni_base_frag_t *frag = (mca_btl_ugni_base_frag_t *) descriptor;
-    size_t size = frag->segments[0].seg_len + frag->segments[1].seg_len;
-    mca_btl_ugni_module_t *ugni_module = (mca_btl_ugni_module_t *) btl;
-    int rc;
-
-    /* tag and len are at the same location in eager and smsg frag hdrs */
-    frag->hdr.send.lag = (tag << 24) | size;
-
-    BTL_VERBOSE(("btl/ugni sending descriptor %p from %d -> %d. length = %" PRIu64, (void *)descriptor,
-                 OPAL_PROC_MY_NAME.vpid, endpoint->peer_proc->proc_name.vpid, size));
-
-    rc = mca_btl_ugni_check_endpoint_state (endpoint);
-    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc || opal_list_get_size (&endpoint->frag_wait_list))) {
-        mca_btl_ugni_wait_list_append (ugni_module, endpoint, frag);
-        return OPAL_SUCCESS;
-    }
-
-    /* add a reference to prevent the fragment from being returned until after the
-     * completion flag is checked. */
-    ++frag->ref_cnt;
-    frag->flags &= ~MCA_BTL_UGNI_FRAG_COMPLETE;
-
-    rc = mca_btl_ugni_send_frag (endpoint, frag);
-    if (OPAL_LIKELY(mca_btl_ugni_frag_check_complete (frag))) {
-        /* fast path: remote side has received the frag */
-        (void) mca_btl_ugni_frag_del_ref (frag, OPAL_SUCCESS);
-
-        return 1;
-    }
-
-    if ((OPAL_SUCCESS == rc) && (frag->flags & MCA_BTL_UGNI_FRAG_BUFFERED) && (frag->flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP)) {
-        /* fast(ish) path: btl owned buffered frag. report send as complete */
-        bool call_callback = !!(frag->flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK);
-        frag->flags &= ~MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
-
-        if (call_callback) {
-            frag->base.des_cbfunc(&ugni_module->super, frag->endpoint, &frag->base, rc);
-        }
-
-        (void) mca_btl_ugni_frag_del_ref (frag, OPAL_SUCCESS);
-
-        return 1;
-    }
-
-    /* slow(ish) path: remote side hasn't received the frag. call the frag's callback when
-       we get the local smsg/msgq or remote rdma completion */
-    frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
-
-    mca_btl_ugni_frag_del_ref (frag, OPAL_SUCCESS);
-
-    if (OPAL_UNLIKELY(OPAL_ERR_OUT_OF_RESOURCE == rc)) {
-        /* queue up request */
-        mca_btl_ugni_wait_list_append (ugni_module, endpoint, frag);
-        rc = OPAL_SUCCESS;
-    }
-
-    return rc;
-}
-
-int mca_btl_ugni_sendi (struct mca_btl_base_module_t *btl,
-                        struct mca_btl_base_endpoint_t *endpoint,
-                        struct opal_convertor_t *convertor,
-                        void *header, size_t header_size,
-                        size_t payload_size, uint8_t order,
-                        uint32_t flags, mca_btl_base_tag_t tag,
-                        mca_btl_base_descriptor_t **descriptor)
-{
-    size_t total_size = header_size + payload_size;
-    mca_btl_ugni_base_frag_t *frag = NULL;
-    size_t packed_size = payload_size;
-    int rc;
-
-    if (OPAL_UNLIKELY(opal_list_get_size (&endpoint->frag_wait_list))) {
-        if (NULL != descriptor) {
-            *descriptor = NULL;
-        }
-        return OPAL_ERR_OUT_OF_RESOURCE;
-    }
-
-    do {
-        BTL_VERBOSE(("btl/ugni isend sending fragment from %d -> %d. length = %" PRIu64
-                     " endoint state %d", OPAL_PROC_MY_NAME.vpid, endpoint->peer_proc->proc_name.vpid,
-                     payload_size + header_size, endpoint->state));
-
-        flags |= MCA_BTL_DES_FLAGS_BTL_OWNERSHIP;
-
-        if (0 == payload_size) {
-            frag = (mca_btl_ugni_base_frag_t *) mca_btl_ugni_prepare_src_send_nodata (btl, endpoint, order, header_size,
-                                                                                      flags);
-        } else {
-            frag = (mca_btl_ugni_base_frag_t *) mca_btl_ugni_prepare_src_send_buffered (btl, endpoint, convertor, order,
-                                                                                        header_size, &packed_size, flags);
-        }
-
-        assert (packed_size == payload_size);
-        if (OPAL_UNLIKELY(NULL == frag || OPAL_SUCCESS != mca_btl_ugni_check_endpoint_state (endpoint))) {
-            break;
-        }
-
-        frag->hdr.send.lag = (tag << 24) | total_size;
-        memcpy (frag->segments[0].seg_addr.pval, header, header_size);
-
-        rc = mca_btl_ugni_send_frag (endpoint, frag);
-        if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-            break;
-        }
-
-        return OPAL_SUCCESS;
-    } while (0);
-
-    if (NULL != descriptor) {
-        *descriptor = &frag->base;
-    }
-
-    return OPAL_ERR_OUT_OF_RESOURCE;
-}
-
-int mca_btl_ugni_progress_send_wait_list (mca_btl_base_endpoint_t *endpoint)
-{
-    mca_btl_ugni_base_frag_t *frag=NULL;
-    int rc;
-
-    do {
-        OPAL_THREAD_LOCK(&endpoint->lock);
-        frag = (mca_btl_ugni_base_frag_t *) opal_list_remove_first (&endpoint->frag_wait_list);
-        OPAL_THREAD_UNLOCK(&endpoint->lock);
-        if (NULL == frag) {
-            break;
-        }
-        if (OPAL_LIKELY(!(frag->flags & MCA_BTL_UGNI_FRAG_RESPONSE))) {
-            rc = mca_btl_ugni_send_frag (endpoint, frag);
-        } else {
-            rc = opal_mca_btl_ugni_smsg_send (frag, &frag->hdr.rdma, sizeof (frag->hdr.rdma),
-                                              NULL, 0, MCA_BTL_UGNI_TAG_RDMA_COMPLETE);
-        }
-
-        if (OPAL_UNLIKELY(OPAL_SUCCESS > rc)) {
-            if (OPAL_LIKELY(OPAL_ERR_OUT_OF_RESOURCE == rc)) {
-                OPAL_THREAD_LOCK(&endpoint->lock);
-                opal_list_prepend (&endpoint->frag_wait_list, (opal_list_item_t *) frag);
-                OPAL_THREAD_UNLOCK(&endpoint->lock);
-            } else {
-                mca_btl_ugni_frag_complete (frag, rc);
-            }
-            return rc;
-        }
-    } while(1);
-
-    return OPAL_SUCCESS;
-}
diff --git a/opal/mca/btl/ugni/btl_ugni_smsg.c b/opal/mca/btl/ugni/btl_ugni_smsg.c
deleted file mode 100644
index c04586b0924..00000000000
--- a/opal/mca/btl/ugni/btl_ugni_smsg.c
+++ /dev/null
@@ -1,242 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2011-2017 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2011      UT-Battelle, LLC. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "btl_ugni_smsg.h"
-#include "btl_ugni_rdma.h"
-
-static void mca_btl_ugni_smsg_mbox_construct (mca_btl_ugni_smsg_mbox_t *mbox) {
-    struct mca_btl_ugni_reg_t *ugni_reg =
-        (struct mca_btl_ugni_reg_t *) mbox->super.registration;
-    mca_rcache_base_registration_t *base_reg =
-        (mca_rcache_base_registration_t *) ugni_reg;
-
-    /* initialize mailbox attributes */
-    mbox->attr.smsg_attr.msg_type       = GNI_SMSG_TYPE_MBOX_AUTO_RETRANSMIT;
-    mbox->attr.smsg_attr.msg_maxsize    = mca_btl_ugni_component.ugni_smsg_limit;
-    mbox->attr.smsg_attr.mbox_maxcredit = mca_btl_ugni_component.smsg_max_credits;
-    mbox->attr.smsg_attr.mbox_offset    = (uintptr_t) mbox->super.ptr - (uintptr_t) base_reg->base;
-    mbox->attr.smsg_attr.msg_buffer     = base_reg->base;
-    mbox->attr.smsg_attr.buff_size      = mca_btl_ugni_component.smsg_mbox_size;
-    mbox->attr.smsg_attr.mem_hndl       = ugni_reg->handle.gni_handle;
-    mbox->attr.proc_name = OPAL_PROC_MY_NAME;
-    mbox->attr.rmt_irq_mem_hndl = mca_btl_ugni_component.modules[0].devices[0].smsg_irq_mhndl;
-}
-
-OBJ_CLASS_INSTANCE(mca_btl_ugni_smsg_mbox_t, opal_free_list_item_t,
-                   mca_btl_ugni_smsg_mbox_construct, NULL);
-
-
-int mca_btl_ugni_smsg_init (mca_btl_ugni_module_t *ugni_module)
-{
-    gni_return_t rc;
-
-    for (int i = 0 ; i < mca_btl_ugni_component.virtual_device_count ; ++i) {
-        rc = GNI_SmsgSetMaxRetrans (ugni_module->devices[i].dev_handle,
-                                    mca_btl_ugni_component.smsg_max_retries);
-        if (GNI_RC_SUCCESS != rc) {
-            BTL_ERROR(("error setting maximum SMSG retries %s",gni_err_str[rc]));
-            return mca_btl_rc_ugni_to_opal (rc);
-        }
-    }
-
-    return OPAL_SUCCESS;
-}
-
-/* progress */
-int mca_btl_ugni_smsg_process (mca_btl_base_endpoint_t *ep)
-{
-    mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl (ep);
-    mca_btl_active_message_callback_t *reg;
-    mca_btl_ugni_base_frag_t frag;
-    bool disconnect = false;
-    int32_t _tmp_value = 0;
-    uintptr_t data_ptr;
-    gni_return_t rc;
-    uint32_t len;
-    int count = 0;
-
-    if (!opal_atomic_compare_exchange_strong_32 (&ep->smsg_progressing, &_tmp_value, 1)) {
-        /* already progressing (we can't support reentry here) */
-        return 0;
-    }
-
-    /* per uGNI documentation we loop until the mailbox is empty */
-    do {
-        uint8_t tag = GNI_SMSG_ANY_TAG;
-
-        rc = mca_btl_ugni_smsg_get_next_wtag (&ep->smsg_ep_handle, &data_ptr, &tag);
-        if (GNI_RC_SUCCESS != rc) {
-            if (OPAL_LIKELY(GNI_RC_NOT_DONE == rc)) {
-                BTL_VERBOSE(("no smsg message waiting. rc = %s", gni_err_str[rc]));
-
-                ep->smsg_progressing = 0;
-                return count;
-            }
-
-            BTL_ERROR(("unhandled GNI_SmsgGetNextWTag error"));
-            return OPAL_ERROR;
-        }
-
-        assert (0 != data_ptr);
-
-        count++;
-
-        BTL_VERBOSE(("got smsg fragment. tag = %d\n", tag));
-
-        switch (tag) {
-        case MCA_BTL_UGNI_TAG_SEND:
-            
-            frag.hdr.send = ((mca_btl_ugni_send_frag_hdr_t *) data_ptr)[0];
-
-            tag = frag.hdr.send.lag >> 24;
-            len = frag.hdr.send.lag & 0x00ffffff;
-
-            BTL_VERBOSE(("received smsg fragment. hdr = {len = %u, tag = %d}", len, tag));
-
-            reg = mca_btl_base_active_message_trigger + tag;
-
-            mca_btl_base_segment_t seg =
-                {.seg_addr.pval = (void *)((uintptr_t)data_ptr + sizeof (mca_btl_ugni_send_frag_hdr_t)),
-                 seg.seg_len = len};
-            const mca_btl_base_receive_descriptor_t desc = {.endpoint = ep, .des_segments = &seg,
-                                                            .des_segment_count = 1, .tag = tag,
-                                                            .cbdata = reg->cbdata};
-            reg->cbfunc(&ugni_module->super, &desc);
-
-            break;
-        case MCA_BTL_UGNI_TAG_GET_INIT:
-            frag.hdr.eager_ex = ((mca_btl_ugni_eager_ex_frag_hdr_t *) data_ptr)[0];
-
-            mca_btl_ugni_start_eager_get (ep, frag.hdr.eager_ex, NULL);
-            break;
-        case MCA_BTL_UGNI_TAG_RDMA_COMPLETE:
-            frag.hdr.rdma = ((mca_btl_ugni_rdma_frag_hdr_t *) data_ptr)[0];
-
-            if (((mca_btl_ugni_base_frag_t *)frag.hdr.rdma.ctx)->flags & MCA_BTL_UGNI_FRAG_SMSG_COMPLETE) {
-                mca_btl_ugni_frag_complete (frag.hdr.rdma.ctx, OPAL_SUCCESS);
-            } else {
-                /* let the local smsg completion finish this frag */
-                ((mca_btl_ugni_base_frag_t *)frag.hdr.rdma.ctx)->flags &= ~MCA_BTL_UGNI_FRAG_IGNORE;
-            }
-            break;
-        case MCA_BTL_UGNI_TAG_DISCONNECT:
-            /* remote endpoint has disconnected */
-            disconnect = true;
-            break;
-        default:
-            BTL_ERROR(("unknown tag %d\n", tag));
-            break;
-        }
-
-        rc = mca_btl_ugni_smsg_release (&ep->smsg_ep_handle);
-        if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc)) {
-            BTL_ERROR(("Smsg release failed! rc = %d", rc));
-            return OPAL_ERROR;
-        }
-    } while (!disconnect);
-
-    ep->smsg_progressing = 0;
-
-    /* disconnect if we get here */
-    opal_mutex_lock (&ep->lock);
-
-    mca_btl_ugni_ep_disconnect (ep, false);
-
-    opal_mutex_unlock (&ep->lock);
-
-    return count;
-}
-
-static inline int
-mca_btl_ugni_handle_remote_smsg_overrun (mca_btl_ugni_module_t *btl)
-{
-    size_t endpoint_count;
-    unsigned int ep_index;
-    int count, rc;
-
-    BTL_VERBOSE(("btl/ugni_component detected SMSG CQ overrun. "
-                 "processing message backlog..."));
-
-    /* we don't know which endpoint lost an smsg completion. clear the
-       smsg remote cq and check all mailboxes */
-
-    /* clear out remote cq */
-    mca_btl_ugni_cq_clear (btl->devices, btl->smsg_remote_cq);
-
-    endpoint_count = opal_pointer_array_get_size (&btl->endpoints);
-
-    for (ep_index = 0, count = 0 ; ep_index < endpoint_count ; ++ep_index) {
-        mca_btl_base_endpoint_t *ep;
-
-        ep = (mca_btl_base_endpoint_t *) opal_pointer_array_get_item (&btl->endpoints,
-                                                                      ep_index);
-
-        if (NULL == ep || MCA_BTL_UGNI_EP_STATE_CONNECTED != ep->state) {
-            continue;
-        }
-
-        /* clear out smsg mailbox */
-        rc = mca_btl_ugni_smsg_process (ep);
-        if (OPAL_LIKELY(rc >= 0)) {
-            count += rc;
-        }
-    }
-
-    return count;
-}
-
-int mca_btl_ugni_progress_remote_smsg (mca_btl_ugni_module_t *btl)
-{
-    mca_btl_base_endpoint_t *ep;
-    gni_cq_entry_t event_data;
-    gni_return_t grc;
-    uint64_t inst_id;
-
-    grc = mca_btl_ugni_gni_cq_get_event (btl->devices, btl->smsg_remote_cq, &event_data);
-    if (GNI_RC_NOT_DONE == grc) {
-        return 0;
-    }
-
-    if (OPAL_UNLIKELY(GNI_RC_SUCCESS != grc || !GNI_CQ_STATUS_OK(event_data) ||
-                      GNI_CQ_OVERRUN(event_data))) {
-        if (GNI_RC_ERROR_RESOURCE == grc ||
-            (GNI_RC_SUCCESS == grc && GNI_CQ_OVERRUN(event_data))) {
-            /* recover from smsg cq overrun */
-            return mca_btl_ugni_handle_remote_smsg_overrun (btl);
-        }
-
-        BTL_ERROR(("unhandled error in GNI_CqGetEvent"));
-
-        /* unhandled error: crash */
-        assert (0);
-        return mca_btl_rc_ugni_to_opal (grc);
-    }
-
-    BTL_VERBOSE(("REMOTE CQ: Got event 0x%" PRIx64 ". msg id = %" PRIu64
-                 ". ok = %d, type = %" PRIu64, (uint64_t) event_data,
-                 GNI_CQ_GET_INST_ID(event_data), GNI_CQ_STATUS_OK(event_data),
-                 GNI_CQ_GET_TYPE(event_data)));
-
-    inst_id = GNI_CQ_GET_INST_ID(event_data);
-
-    ep = (mca_btl_base_endpoint_t *) opal_pointer_array_get_item (&btl->endpoints, inst_id);
-
-    if (OPAL_UNLIKELY(MCA_BTL_UGNI_EP_STATE_CONNECTED != ep->state)) {
-        /* due to the nature of datagrams we may get a smsg completion before
-           we get mailbox info from the peer */
-        BTL_VERBOSE(("event occurred on an unconnected endpoint! ep state = %d", ep->state));
-        return 0;
-    }
-
-    return mca_btl_ugni_smsg_process (ep);
-}
-
diff --git a/opal/mca/btl/ugni/btl_ugni_smsg.h b/opal/mca/btl/ugni/btl_ugni_smsg.h
deleted file mode 100644
index 82e54d55cb8..00000000000
--- a/opal/mca/btl/ugni/btl_ugni_smsg.h
+++ /dev/null
@@ -1,128 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2011      UT-Battelle, LLC. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#if !defined(MCA_BTL_UGNI_SMSG_H)
-#define MCA_BTL_UGNI_SMSG_H
-
-#include "btl_ugni.h"
-#include "btl_ugni_endpoint.h"
-#include "btl_ugni_frag.h"
-#include "btl_ugni_rdma.h"
-
-typedef enum {
-    MCA_BTL_UGNI_TAG_SEND,
-    MCA_BTL_UGNI_TAG_DISCONNECT,
-    MCA_BTL_UGNI_TAG_GET_INIT,
-    MCA_BTL_UGNI_TAG_RDMA_COMPLETE
-} mca_btl_ugni_smsg_tag_t;
-
-typedef struct mca_btl_ugni_smsg_mbox_t {
-    opal_free_list_item_t super;
-    mca_btl_ugni_endpoint_attr_t attr;
-} mca_btl_ugni_smsg_mbox_t;
-
-OBJ_CLASS_DECLARATION(mca_btl_ugni_smsg_mbox_t);
-
-int mca_btl_ugni_smsg_init (mca_btl_ugni_module_t *ugni_module);
-int mca_btl_ugni_smsg_process (mca_btl_base_endpoint_t *ep);
-int mca_btl_ugni_progress_remote_smsg (mca_btl_ugni_module_t *btl);
-
-static inline int mca_btl_ugni_progress_local_smsg (mca_btl_ugni_module_t *ugni_module, mca_btl_ugni_device_t *device)
-{
-    mca_btl_ugni_base_frag_t *frag;
-    gni_cq_entry_t event_data;
-    gni_return_t grc;
-
-    grc = mca_btl_ugni_cq_get_event (device, &device->dev_smsg_local_cq, &event_data);
-    if (GNI_RC_NOT_DONE == grc) {
-        return OPAL_SUCCESS;
-    }
-
-    if (OPAL_UNLIKELY((GNI_RC_SUCCESS != grc && !event_data) || GNI_CQ_OVERRUN(event_data))) {
-        /* TODO -- need to handle overrun -- how do we do this without an event?
-         * will the event eventually come back? Ask Cray */
-        return mca_btl_ugni_event_fatal_error (grc, event_data);
-    }
-
-    assert (GNI_CQ_GET_TYPE(event_data) == GNI_CQ_EVENT_TYPE_SMSG);
-
-    frag = (mca_btl_ugni_base_frag_t *) opal_pointer_array_get_item (&ugni_module->pending_smsg_frags_bb,
-                                                                     GNI_CQ_GET_MSG_ID(event_data));
-    if (OPAL_UNLIKELY(NULL == frag)) {
-        assert (0);
-        return OPAL_ERROR;
-    }
-
-    frag->flags |= MCA_BTL_UGNI_FRAG_SMSG_COMPLETE;
-
-    if (!(frag->flags & MCA_BTL_UGNI_FRAG_IGNORE)) {
-        mca_btl_ugni_frag_complete (frag, OPAL_SUCCESS);
-    }
-
-    return 1;
-}
-
-static inline int opal_mca_btl_ugni_smsg_send (mca_btl_ugni_base_frag_t *frag,
-                                               void *hdr, size_t hdr_len,
-                                               void *payload, size_t payload_len,
-                                               mca_btl_ugni_smsg_tag_t tag)
-{
-    mca_btl_base_endpoint_t *endpoint = frag->endpoint;
-    mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl (endpoint);
-    gni_return_t grc;
-
-    grc = mca_btl_ugni_endpoint_smsg_send_wtag (endpoint, hdr, hdr_len, payload, payload_len,
-                                                frag->msg_id, tag);
-    if (OPAL_LIKELY(GNI_RC_SUCCESS == grc)) {
-        if (mca_btl_ugni_component.progress_thread_enabled) {
-            if (frag->base.des_flags & MCA_BTL_DES_FLAGS_SIGNAL) {
-                /* errors for PostCqWrite treated as non-fatal */
-                (void) mca_btl_ugni_post_cqwrite (endpoint, &ugni_module->devices[0].dev_rdma_local_cq,
-                                                  endpoint->rmt_irq_mem_hndl, 0xdead, NULL, NULL, NULL);
-            }
-        }
-
-        (void) mca_btl_ugni_progress_local_smsg (ugni_module, endpoint->smsg_ep_handle.device);
-        return OPAL_SUCCESS;
-    }
-
-    if (OPAL_LIKELY(GNI_RC_NOT_DONE == grc)) {
-        BTL_VERBOSE(("out of credits"));
-
-        return OPAL_ERR_OUT_OF_RESOURCE;
-    }
-
-    BTL_ERROR(("GNI_SmsgSendWTag failed with rc = %d. handle = %lu, hdr_len = %d, payload_len = %d",
-               grc, (uintptr_t) frag->endpoint->smsg_ep_handle.gni_handle, (int) hdr_len,
-               (int) payload_len));
-
-    return OPAL_ERROR;
-}
-
-static inline int mca_btl_ugni_send_frag (struct mca_btl_base_endpoint_t *btl_peer,
-                                          mca_btl_ugni_base_frag_t *frag) {
-    if (OPAL_LIKELY(!(frag->flags & MCA_BTL_UGNI_FRAG_EAGER))) {
-        return opal_mca_btl_ugni_smsg_send (frag, &frag->hdr.send, frag->hdr_size,
-                                            frag->segments[1].seg_addr.pval,
-                                            frag->segments[1].seg_len,
-                                            MCA_BTL_UGNI_TAG_SEND);
-    }
-
-    frag->hdr.eager.size    = frag->segments[1].seg_len;
-    frag->hdr.eager.address = frag->segments[1].seg_addr.lval;
-    frag->hdr.eager.ctx     = (void *) frag;
-
-    return opal_mca_btl_ugni_smsg_send (frag, &frag->hdr.eager, frag->hdr_size,
-                                        NULL, 0, MCA_BTL_UGNI_TAG_GET_INIT);
-}
-
-#endif /* MCA_BTL_UGNI_SMSG_H */
diff --git a/opal/mca/btl/ugni/configure.m4 b/opal/mca/btl/ugni/configure.m4
deleted file mode 100644
index bb08cbb7d20..00000000000
--- a/opal/mca/btl/ugni/configure.m4
+++ /dev/null
@@ -1,54 +0,0 @@
-# -*- shell-script -*-
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2006 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2006      QLogic Corp. All rights reserved.
-# Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2011      Los Alamos National Security, LLC.
-#                         All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-# OPAL_CHECK_UGNI(prefix, [action-if-found], [action-if-not-found])
-# --------------------------------------------------------
-# check if GNI support can be found.  sets prefix_{CPPFLAGS,
-# LDFLAGS, LIBS} as needed and runs action-if-found if there is
-# support, otherwise executes action-if-not-found
-#
-# NOTES
-# on Cray XE6 systems, the GNI development header (gni_pub.h) is in a
-# completely different place than the ugni library (libugni).
-#
-# EXAMPLE CONFIGURE USAGE:
-# --with-ugni=/base/path/to/libugni --with-ugni-includedir=/path/to/gni_pub.h
-#
-# --with-ugni=/opt/cray/ugni/default --with-ugni-includedir=/opt/cray/gni-headers/default/include
-
-AC_DEFUN([MCA_opal_btl_ugni_CONFIG],[
-    AC_CONFIG_FILES([opal/mca/btl/ugni/Makefile])
-
-    OPAL_CHECK_UGNI([btl_ugni],
-                     [btl_ugni_happy="yes"],
-                     [btl_ugni_happy="no"])
-
-    AS_IF([test "$btl_ugni_happy" = "yes"],
-          [$1],
-          [$2])
-
-    # substitute in the things needed to build ugni
-    AC_SUBST([btl_ugni_CPPFLAGS])
-    AC_SUBST([btl_ugni_LDFLAGS])
-    AC_SUBST([btl_ugni_LIBS])
-])dnl
diff --git a/opal/mca/btl/ugni/owner.txt b/opal/mca/btl/ugni/owner.txt
deleted file mode 100644
index 4918816bc9e..00000000000
--- a/opal/mca/btl/ugni/owner.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-#
-# owner/status file
-# owner: institution that is responsible for this package
-# status: e.g. active, maintenance, unmaintained
-#
-owner:LANL
-status:active
diff --git a/opal/mca/btl/usnic/Makefile.am b/opal/mca/btl/usnic/Makefile.am
index 48ec0bb751e..182ba93655b 100644
--- a/opal/mca/btl/usnic/Makefile.am
+++ b/opal/mca/btl/usnic/Makefile.am
@@ -18,6 +18,8 @@
 #                         reserved.
 # Copyright (c) 2019      Research Organization for Information Science
 #                         and Technology (RIST).  All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -25,11 +27,10 @@
 # $HEADER$
 #
 
-AM_CPPFLAGS = $(opal_ofi_CPPFLAGS) -DOMPI_LIBMPI_NAME=\"$(OMPI_LIBMPI_NAME)\"
+AM_CPPFLAGS = $(btl_usnic_CPPFLAGS) -DOMPI_LIBMPI_NAME=\"$(OMPI_LIBMPI_NAME)\"
 
-EXTRA_DIST = README.md README.test
-
-dist_opaldata_DATA = \
+EXTRA_DIST = \
+    README.md README.test \
     help-mpi-btl-usnic.txt
 
 test_sources = \
@@ -90,18 +91,18 @@ mcacomponent_LTLIBRARIES = $(component)
 mca_btl_usnic_la_SOURCES = $(component_sources)
 mca_btl_usnic_la_LDFLAGS = \
         $(opal_btl_usnic_LDFLAGS) \
-        $(opal_ofi_LDFLAGS) \
+        $(btl_usnic_LDFLAGS) \
         -module -avoid-version
-mca_btl_usnic_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
-        $(opal_ofi_LIBS)
+mca_btl_usnic_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
+        $(btl_usnic_LIBS)
 
 noinst_LTLIBRARIES = $(lib)
 libmca_btl_usnic_la_SOURCES = $(lib_sources)
 libmca_btl_usnic_la_LDFLAGS = \
         $(opal_btl_usnic_LDFLAGS) \
-        $(opal_ofi_LDFLAGS) \
+        $(btl_usnic_LDFLAGS) \
         -module -avoid-version
-libmca_btl_usnic_la_LIBADD = $(opal_ofi_LIBS)
+libmca_btl_usnic_la_LIBADD = $(btl_usnic_LIBS)
 
 if OPAL_BTL_USNIC_BUILD_UNIT_TESTS
 usnic_btl_run_tests_CPPFLAGS = $(AM_CPPFLAGS) \
diff --git a/opal/mca/btl/usnic/README.md b/opal/mca/btl/usnic/README.md
index a1875a818f1..eae79b8f82b 100644
--- a/opal/mca/btl/usnic/README.md
+++ b/opal/mca/btl/usnic/README.md
@@ -194,12 +194,12 @@ is a receive on the priority queue, then it is handled by a routine
 called `opal_btl_usnic_recv_fast()` which does nothing but validates
 that the packet is OK to be received (sequence number OK and not a
 DUP) and then delivers it to the PML.  This packet is recorded in the
-channel structure, and all bookeeping for the packet is deferred until
+channel structure, and all bookkeeping for the packet is deferred until
 the next time `component_progress` is called again.
 
 This fast path cannot be taken every time we pass through
 `component_progress` because there will be other completions that need
-processing, and the receive bookeeping for one fast receive must be
+processing, and the receive bookkeeping for one fast receive must be
 complete before allowing another fast receive to occur, as only one
 recv segment can be saved for deferred processing at a time.  This is
 handled by maintaining a variable in `opal_btl_usnic_recv_fast()`
diff --git a/opal/mca/btl/usnic/README.test b/opal/mca/btl/usnic/README.test
index 2dd6de0896f..6ae2afaf254 100644
--- a/opal/mca/btl/usnic/README.test
+++ b/opal/mca/btl/usnic/README.test
@@ -23,7 +23,7 @@ Goals
 Anti-Goals
 ----------
 * Testing the low level networking API (e.g., libfabric).
-* Testing inter-process interaction, such as ORTE-related functionality.
+* Testing inter-process interaction, such as PRTE-related functionality.
 
 Constraints
 -----------
diff --git a/opal/mca/btl/usnic/btl_usnic.h b/opal/mca/btl/usnic/btl_usnic.h
index 5a3a94103cd..eeddcdfbb40 100644
--- a/opal/mca/btl/usnic/btl_usnic.h
+++ b/opal/mca/btl/usnic/btl_usnic.h
@@ -32,15 +32,15 @@
 #include "opal_config.h"
 #include <sys/types.h>
 
-#include "opal_stdint.h"
-#include "opal/util/alfg.h"
-#include "opal/class/opal_hash_table.h"
 #include "opal/class/opal_hash_table.h"
+#include "opal/util/alfg.h"
 #include "opal/util/event.h"
+#include "opal/util/misc.h"
+#include "opal_stdint.h"
 
-#include "opal/mca/btl/btl.h"
-#include "opal/mca/btl/base/btl_base_error.h"
 #include "opal/mca/btl/base/base.h"
+#include "opal/mca/btl/base/btl_base_error.h"
+#include "opal/mca/btl/btl.h"
 #include "opal/mca/rcache/rcache.h"
 
 #include "btl_usnic_compat.h"
@@ -58,8 +58,7 @@ extern uint64_t opal_btl_usnic_ticks;
 /* Lock for MPU_THREAD_MULTIPLE support */
 extern opal_recursive_mutex_t btl_usnic_lock;
 
-static inline uint64_t
-get_ticks(void)
+static inline uint64_t get_ticks(void)
 {
     return opal_btl_usnic_ticks;
 }
@@ -67,36 +66,35 @@ get_ticks(void)
 /* RNG buffer declaration */
 extern opal_rng_buff_t opal_btl_usnic_rand_buff;
 
-#ifndef container_of
-#define container_of(ptr, type, member) ( \
-        (type *)( ((char *)(ptr)) - offsetof(type,member) ))
-#endif
-
 #ifndef max
-#define max(a, b) (((a) > (b)) ? (a) : (b))
+#    define max(a, b) (((a) > (b)) ? (a) : (b))
 #endif
 
 /* MSGDEBUG2 prints 1 line at each BTL entry point */
-#define MSGDEBUG2 (MSGDEBUG1||0)
+#define MSGDEBUG2 (MSGDEBUG1 || 0)
 /* MSGDEBUG1 prints more info about arguments and internal functions */
 #define MSGDEBUG1 0
 
 /* output macros to declutter source */
 #if MSGDEBUG1
-#define MSGDEBUG1_OUT(...) opal_output(0, __VA_ARGS__)
+#    define MSGDEBUG1_OUT(...) opal_output(0, __VA_ARGS__)
 #else
-#define MSGDEBUG1_OUT(...) do {} while (0)
+#    define MSGDEBUG1_OUT(...) \
+        do {                   \
+        } while (0)
 #endif
 #if MSGDEBUG2
-#define MSGDEBUG2_OUT(...) opal_output(0, __VA_ARGS__)
+#    define MSGDEBUG2_OUT(...) opal_output(0, __VA_ARGS__)
 #else
-#define MSGDEBUG2_OUT(...) do {} while (0)
+#    define MSGDEBUG2_OUT(...) \
+        do {                   \
+        } while (0)
 #endif
 
 /* Set to >0 to randomly drop received frags.  The higher the number,
    the more frequent the drops. */
 #define WANT_RECV_DROPS 0
-/* Set to >0 to randomly fail to send an ACK, mimicing a lost ACK.
+/* Set to >0 to randomly fail to send an ACK, mimicking a lost ACK.
    The higher the number, the more frequent the failed-to-send-ACK. */
 #define WANT_FAIL_TO_SEND_ACK 0
 /* Set to >0 to randomly fail to resend a frag (causing it to be
@@ -105,24 +103,24 @@ extern opal_rng_buff_t opal_btl_usnic_rand_buff;
 #define WANT_FAIL_TO_RESEND_FRAG 0
 
 #if WANT_RECV_DROPS > 0
-#define FAKE_RECV_DROP (opal_rand(&opal_btl_usnic_rand_buff) < WANT_RECV_DROPS)
+#    define FAKE_RECV_DROP (opal_rand(&opal_btl_usnic_rand_buff) < WANT_RECV_DROPS)
 #else
-#define FAKE_RECV_DROP 0
+#    define FAKE_RECV_DROP 0
 #endif
 
 #if WANT_FAIL_TO_SEND_ACK > 0
-#define FAKE_FAIL_TO_SEND_ACK (opal_rand(&opal_btl_usnic_rand_buff) < WANT_FAIL_TO_SEND_ACK)
+#    define FAKE_FAIL_TO_SEND_ACK (opal_rand(&opal_btl_usnic_rand_buff) < WANT_FAIL_TO_SEND_ACK)
 #else
-#define FAKE_FAIL_TO_SEND_ACK 0
+#    define FAKE_FAIL_TO_SEND_ACK 0
 #endif
 
 #if WANT_FAIL_TO_RESEND_FRAG > 0
-#define FAKE_FAIL_TO_RESEND_FRAG (opal_rand(&opal_btl_usnic_rand_buff) < WANT_FAIL_TO_RESEND_FRAG)
+#    define FAKE_FAIL_TO_RESEND_FRAG \
+        (opal_rand(&opal_btl_usnic_rand_buff) < WANT_FAIL_TO_RESEND_FRAG)
 #else
-#define FAKE_FAIL_TO_RESEND_FRAG 0
+#    define FAKE_FAIL_TO_RESEND_FRAG 0
 #endif
 
-
 /**
  * usnic BTL component
  */
@@ -143,9 +141,9 @@ typedef struct opal_btl_usnic_component_t {
     uint64_t my_hashed_rte_name;
 
     /** array of possible BTLs (>= num_modules elements) */
-    struct opal_btl_usnic_module_t* usnic_all_modules;
+    struct opal_btl_usnic_module_t *usnic_all_modules;
     /** array of pointers to active BTLs (num_modules elements) */
-    struct opal_btl_usnic_module_t** usnic_active_modules;
+    struct opal_btl_usnic_module_t **usnic_active_modules;
 
     /** convertor packing threshold */
     int pack_lazy_threshold;
@@ -156,7 +154,7 @@ typedef struct opal_btl_usnic_component_t {
     opal_list_t usnic_procs;
 
     /** memory pool hints */
-    char* usnic_mpool_hints;
+    char *usnic_mpool_hints;
 
     /** registration cache name */
     char *usnic_rcache_name;
@@ -179,7 +177,7 @@ typedef struct opal_btl_usnic_component_t {
     /** max receive descriptors per module */
     int32_t rd_num;
 
-    /** max send/receive desriptors for priority channel */
+    /** max send/receive descriptors for priority channel */
     int32_t prio_sd_num;
     int32_t prio_rd_num;
 
@@ -240,7 +238,7 @@ typedef struct opal_btl_usnic_component_t {
     opal_event_base_t *opal_evbase;
 } opal_btl_usnic_component_t;
 
-OPAL_MODULE_DECLSPEC extern opal_btl_usnic_component_t mca_btl_usnic_component;
+OPAL_DECLSPEC extern opal_btl_usnic_component_t mca_btl_usnic_component;
 
 typedef mca_btl_base_recv_reg_t opal_btl_usnic_recv_reg_t;
 
@@ -255,11 +253,11 @@ typedef uint16_t opal_btl_usnic_seq_t;
  * Relies on the fact that sequence numbers should be relatively close
  * together as compared to (1<<31)
  */
-#define SEQ_DIFF(A,B) ((int16_t)((A)-(B)))
-#define SEQ_LT(A,B) (SEQ_DIFF(A,B) < 0)
-#define SEQ_LE(A,B) (SEQ_DIFF(A,B) <= 0)
-#define SEQ_GT(A,B) (SEQ_DIFF(A,B) > 0)
-#define SEQ_GE(A,B) (SEQ_DIFF(A,B) >= 0)
+#define SEQ_DIFF(A, B) ((int16_t)((A) - (B)))
+#define SEQ_LT(A, B)   (SEQ_DIFF(A, B) < 0)
+#define SEQ_LE(A, B)   (SEQ_DIFF(A, B) <= 0)
+#define SEQ_GT(A, B)   (SEQ_DIFF(A, B) > 0)
+#define SEQ_GE(A, B)   (SEQ_DIFF(A, B) >= 0)
 
 /**
  * Register the usnic BTL MCA params
diff --git a/opal/mca/btl/usnic/btl_usnic_ack.c b/opal/mca/btl/usnic/btl_usnic_ack.c
index b7415e427d7..6bf02764c5a 100644
--- a/opal/mca/btl/usnic/btl_usnic_ack.c
+++ b/opal/mca/btl/usnic/btl_usnic_ack.c
@@ -13,26 +13,24 @@
 #include <string.h>
 #include <unistd.h>
 
-#include "opal/util/output.h"
 #include "opal/class/opal_hotel.h"
+#include "opal/util/output.h"
 
 #include "btl_usnic.h"
-#include "btl_usnic_frag.h"
+#include "btl_usnic_ack.h"
+#include "btl_usnic_connectivity.h"
 #include "btl_usnic_endpoint.h"
+#include "btl_usnic_frag.h"
 #include "btl_usnic_module.h"
-#include "btl_usnic_ack.h"
-#include "btl_usnic_util.h"
 #include "btl_usnic_send.h"
-#include "btl_usnic_connectivity.h"
+#include "btl_usnic_util.h"
 
 /*
  * Special case: we know exactly which segment is missing at the
  * receive; explicitly force retrans of that segment.
  */
-static void
-opal_btl_usnic_fast_retrans(
-    opal_btl_usnic_endpoint_t *endpoint,
-    opal_btl_usnic_seq_t ack_seq)
+static void opal_btl_usnic_fast_retrans(opal_btl_usnic_endpoint_t *endpoint,
+                                        opal_btl_usnic_seq_t ack_seq)
 {
     opal_btl_usnic_send_segment_t *sseg;
     int is;
@@ -59,15 +57,11 @@ opal_btl_usnic_fast_retrans(
     ++endpoint->endpoint_module->stats.num_fast_retrans;
 }
 
-
 /*
  * We have received an ACK for a given sequence number (either standalone
  * or via piggy-back on a regular send)
  */
-void
-opal_btl_usnic_handle_ack(
-    opal_btl_usnic_endpoint_t *endpoint,
-    opal_btl_usnic_seq_t ack_seq)
+void opal_btl_usnic_handle_ack(opal_btl_usnic_endpoint_t *endpoint, opal_btl_usnic_seq_t ack_seq)
 {
     opal_btl_usnic_seq_t is;
     opal_btl_usnic_send_segment_t *sseg;
@@ -80,8 +74,8 @@ opal_btl_usnic_handle_ack(
     /* ignore if this is an old ACK */
     if (SEQ_LT(ack_seq, endpoint->endpoint_ack_seq_rcvd)) {
 #if MSGDEBUG1
-        opal_output(0, "Got OLD DUP ACK seq %"UDSEQ" < %"UDSEQ"\n",
-                ack_seq, endpoint->endpoint_ack_seq_rcvd);
+        opal_output(0, "Got OLD DUP ACK seq %" UDSEQ " < %" UDSEQ "\n", ack_seq,
+                    endpoint->endpoint_ack_seq_rcvd);
 #endif
         ++module->stats.num_old_dup_acks;
         return;
@@ -102,18 +96,20 @@ opal_btl_usnic_handle_ack(
         sseg = endpoint->endpoint_sent_segs[WINDOW_SIZE_MOD(is)];
 
 #if MSGDEBUG1
-        opal_output(0, "  Checking ACK/sent_segs window %p, index %lu, seq %lu, occupied=%p, seg_room=%d",
-            (void*) endpoint->endpoint_sent_segs,
-            WINDOW_SIZE_MOD(is), is, (void*)sseg, (sseg?sseg->ss_hotel_room:-2));
+        opal_output(
+            0, "  Checking ACK/sent_segs window %p, index %lu, seq %lu, occupied=%p, seg_room=%d",
+            (void *) endpoint->endpoint_sent_segs, WINDOW_SIZE_MOD(is), is, (void *) sseg,
+            (sseg ? sseg->ss_hotel_room : -2));
 #endif
 
         assert(sseg != NULL);
         assert(sseg->ss_base.us_btl_header->pkt_seq == is);
 #if MSGDEBUG1
         if (sseg->ss_hotel_room == -1) {
-            opal_output(0, "=== ACKed frag in sent_frags array is not in hotel/enqueued, module %p, endpoint %p, seg %p, seq %" UDSEQ ", slot %lu",
-                        (void*) module, (void*) endpoint,
-                        (void*) sseg, is, WINDOW_SIZE_MOD(is));
+            opal_output(0,
+                        "=== ACKed frag in sent_frags array is not in hotel/enqueued, module %p, "
+                        "endpoint %p, seg %p, seq %" UDSEQ ", slot %lu",
+                        (void *) module, (void *) endpoint, (void *) sseg, is, WINDOW_SIZE_MOD(is));
         }
 #endif
 
@@ -127,8 +123,7 @@ opal_btl_usnic_handle_ack(
         }
         /* hotel_room == -1 means queued for resend, remove it */
         else {
-            opal_list_remove_item((&module->pending_resend_segs),
-                    &sseg->ss_base.us_list.super);
+            opal_list_remove_item((&module->pending_resend_segs), &sseg->ss_base.us_list.super);
         }
 
         /* update the owning fragment */
@@ -136,11 +131,12 @@ opal_btl_usnic_handle_ack(
         frag = sseg->ss_parent_frag;
 
 #if MSGDEBUG1
-        opal_output(0, "   ACKED seg %p frag %p ack_bytes=%"PRIu32" left=%zd dst_seg[0].seg_addr=%p des_flags=0x%x\n",
-                (void*)sseg, (void*)frag, bytes_acked,
-                frag->sf_ack_bytes_left - bytes_acked,
-                frag->sf_base.uf_local_seg[0].seg_addr.pval,
-                frag->sf_base.uf_base.des_flags);
+        opal_output(0,
+                    "   ACKED seg %p frag %p ack_bytes=%" PRIu32
+                    " left=%zd dst_seg[0].seg_addr=%p des_flags=0x%x\n",
+                    (void *) sseg, (void *) frag, bytes_acked,
+                    frag->sf_ack_bytes_left - bytes_acked,
+                    frag->sf_base.uf_local_seg[0].seg_addr.pval, frag->sf_base.uf_base.des_flags);
 #endif
 
         /* If all ACKs received, and this is a put or a regular send
@@ -158,14 +154,12 @@ opal_btl_usnic_handle_ack(
 #if BTL_VERSION == 30
             if (frag->sf_base.uf_remote_seg[0].seg_addr.pval != NULL) {
                 OPAL_BTL_USNIC_DO_PUT_FRAG_CB(module, frag, "put completion");
-            } else if (frag->sf_base.uf_base.des_flags &
-                       MCA_BTL_DES_SEND_ALWAYS_CALLBACK) {
+            } else if (frag->sf_base.uf_base.des_flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK) {
                 OPAL_BTL_USNIC_DO_SEND_FRAG_CB(module, frag, "send completion");
             }
 #else
-            if ((frag->sf_base.uf_remote_seg[0].seg_addr.pval != NULL) ||
-                (frag->sf_base.uf_base.des_flags &
-                 MCA_BTL_DES_SEND_ALWAYS_CALLBACK)) {
+            if ((frag->sf_base.uf_remote_seg[0].seg_addr.pval != NULL)
+                || (frag->sf_base.uf_base.des_flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK)) {
                 OPAL_BTL_USNIC_DO_SEND_FRAG_CB(module, frag, "send completion");
             }
 #endif
@@ -198,10 +192,7 @@ opal_btl_usnic_handle_ack(
 /*
  * Send an ACK
  */
-int
-opal_btl_usnic_ack_send(
-    opal_btl_usnic_module_t *module,
-    opal_btl_usnic_endpoint_t *endpoint)
+int opal_btl_usnic_ack_send(opal_btl_usnic_module_t *module, opal_btl_usnic_endpoint_t *endpoint)
 {
     opal_btl_usnic_ack_segment_t *ack;
 
@@ -221,25 +212,18 @@ opal_btl_usnic_ack_send(
 
     /* send the seq of the lowest item in the window that
        we've received */
-    ack->ss_base.us_btl_header->ack_seq =
-        SEQ_DIFF(endpoint->endpoint_next_contig_seq_to_recv, 1);
+    ack->ss_base.us_btl_header->ack_seq = SEQ_DIFF(endpoint->endpoint_next_contig_seq_to_recv, 1);
     ack->ss_len = sizeof(opal_btl_usnic_btl_header_t);
 
 #if MSGDEBUG1
     {
         char remote_ip[IPV4STRADDRLEN];
-        struct opal_btl_usnic_modex_t *modex =
-            &endpoint->endpoint_remote_modex;
-        opal_btl_usnic_snprintf_ipv4_addr(remote_ip, sizeof(remote_ip),
-                                          modex->ipv4_addr,
+        struct opal_btl_usnic_modex_t *modex = &endpoint->endpoint_remote_modex;
+        opal_btl_usnic_snprintf_ipv4_addr(remote_ip, sizeof(remote_ip), modex->ipv4_addr,
                                           modex->netmask);
 
-
-        opal_output(0, "--> Sending ACK, length %d, seq %" UDSEQ " to %s, port %u",
-                    ack->ss_len,
-                    ack->ss_base.us_btl_header->ack_seq,
-                    remote_ip,
-                    modex->ports[ack->ss_channel]);
+        opal_output(0, "--> Sending ACK, length %d, seq %" UDSEQ " to %s, port %u", ack->ss_len,
+                    ack->ss_base.us_btl_header->ack_seq, remote_ip, modex->ports[ack->ss_channel]);
     }
 #endif
 
@@ -260,9 +244,7 @@ opal_btl_usnic_ack_send(
 /*
  * Sending an ACK has completed, return the segment to the free list
  */
-void
-opal_btl_usnic_ack_complete(opal_btl_usnic_module_t *module,
-                                   opal_btl_usnic_ack_segment_t *ack)
+void opal_btl_usnic_ack_complete(opal_btl_usnic_module_t *module, opal_btl_usnic_ack_segment_t *ack)
 {
     ++module->mod_channels[USNIC_PRIORITY_CHANNEL].credits;
     opal_btl_usnic_ack_segment_return(module, ack);
@@ -275,25 +257,20 @@ opal_btl_usnic_ack_complete(opal_btl_usnic_module_t *module,
  * Callback for when a send times out without receiving a
  * corresponding ACK.
  */
-void
-opal_btl_usnic_ack_timeout(
-    opal_hotel_t *hotel,
-    int room_num,
-    void *occupant)
+void opal_btl_usnic_ack_timeout(opal_hotel_t *hotel, int room_num, void *occupant)
 {
     opal_btl_usnic_send_segment_t *seg;
     opal_btl_usnic_endpoint_t *endpoint;
     opal_btl_usnic_module_t *module;
 
-    seg = (opal_btl_usnic_send_segment_t*) occupant;
+    seg = (opal_btl_usnic_send_segment_t *) occupant;
     endpoint = seg->ss_parent_frag->sf_endpoint;
     module = endpoint->endpoint_module;
 
 #if MSGDEBUG1
     {
-        opal_output(0, "Send timeout!  seg %p, room %d, seq %" UDSEQ "\n",
-                    (void*)seg, seg->ss_hotel_room,
-                    seg->ss_base.us_btl_header->pkt_seq);
+        opal_output(0, "Send timeout!  seg %p, room %d, seq %" UDSEQ "\n", (void *) seg,
+                    seg->ss_hotel_room, seg->ss_base.us_btl_header->pkt_seq);
     }
 #endif
 
@@ -301,8 +278,7 @@ opal_btl_usnic_ack_timeout(
     seg->ss_hotel_room = -1;
 
     /* Queue up this frag to be resent */
-    opal_list_append(&(module->pending_resend_segs),
-                     &(seg->ss_base.us_list.super));
+    opal_list_append(&(module->pending_resend_segs), &(seg->ss_base.us_list.super));
 
     /* Stats */
     ++module->stats.num_timeout_retrans;
diff --git a/opal/mca/btl/usnic/btl_usnic_ack.h b/opal/mca/btl/usnic/btl_usnic_ack.h
index e1d4fe6fc87..7dbfb3ce0eb 100644
--- a/opal/mca/btl/usnic/btl_usnic_ack.h
+++ b/opal/mca/btl/usnic/btl_usnic_ack.h
@@ -15,93 +15,78 @@
 #include "opal/class/opal_hotel.h"
 
 #include "btl_usnic.h"
-#include "btl_usnic_frag.h"
-#include "btl_usnic_endpoint.h"
 #include "btl_usnic_compat.h"
+#include "btl_usnic_endpoint.h"
+#include "btl_usnic_frag.h"
 
 /* Invoke the descriptor callback for a (non-PUT) send frag, updating
  * stats and clearing the _CALLBACK flag in the process. */
-#define OPAL_BTL_USNIC_DO_SEND_FRAG_CB(module, send_frag, comment)            \
-    do {                                                                      \
-        MSGDEBUG1_OUT("%s:%d: %s SEND callback for module=%p frag=%p\n",      \
-                      __func__, __LINE__,                                     \
-                      (comment), (void *)(module), (void *)(send_frag));      \
-        (send_frag)->sf_base.uf_base.des_cbfunc(                              \
-            &(module)->super,                                                 \
-            (send_frag)->sf_endpoint,                                         \
-            &(send_frag)->sf_base.uf_base,                                    \
-            OPAL_SUCCESS);                                                    \
-        frag->sf_base.uf_base.des_flags &= ~MCA_BTL_DES_SEND_ALWAYS_CALLBACK; \
-        ++((module)->stats.pml_send_callbacks);                               \
+#define OPAL_BTL_USNIC_DO_SEND_FRAG_CB(module, send_frag, comment)                            \
+    do {                                                                                      \
+        MSGDEBUG1_OUT("%s:%d: %s SEND callback for module=%p frag=%p\n", __func__, __LINE__,  \
+                      (comment), (void *) (module), (void *) (send_frag));                    \
+        (send_frag)->sf_base.uf_base.des_cbfunc(&(module)->super, (send_frag)->sf_endpoint,   \
+                                                &(send_frag)->sf_base.uf_base, OPAL_SUCCESS); \
+        frag->sf_base.uf_base.des_flags &= ~MCA_BTL_DES_SEND_ALWAYS_CALLBACK;                 \
+        ++((module)->stats.pml_send_callbacks);                                               \
     } while (0)
 
 #if BTL_VERSION == 30
 /* Invoke the descriptor callback for a send frag that was a PUT,
  * updating stats and clearing the _CALLBACK flag in the process. */
-#define OPAL_BTL_USNIC_DO_PUT_FRAG_CB(module, send_frag, comment)             \
-    do {                                                                      \
-        MSGDEBUG1_OUT("%s:%d: %s PUT callback for module=%p frag=%p\n",       \
-                      __func__, __LINE__,                                     \
-                      (comment), (void *)(module), (void *)(send_frag));      \
-        mca_btl_base_rdma_completion_fn_t func =                        \
-            (mca_btl_base_rdma_completion_fn_t)                         \
-            (send_frag)->sf_base.uf_base.des_cbfunc;                    \
-        func(&(module)->super,                                          \
-             (send_frag)->sf_endpoint,                                  \
-             (send_frag)->sf_base.uf_local_seg[0].seg_addr.pval,        \
-             NULL,                                                      \
-             (send_frag)->sf_base.uf_base.des_context,                  \
-             (send_frag)->sf_base.uf_base.des_cbdata,                   \
-             OPAL_SUCCESS);                                             \
-        ++((module)->stats.pml_send_callbacks);                         \
-    } while (0)
+#    define OPAL_BTL_USNIC_DO_PUT_FRAG_CB(module, send_frag, comment)                           \
+        do {                                                                                    \
+            MSGDEBUG1_OUT("%s:%d: %s PUT callback for module=%p frag=%p\n", __func__, __LINE__, \
+                          (comment), (void *) (module), (void *) (send_frag));                  \
+            mca_btl_base_rdma_completion_fn_t func = (mca_btl_base_rdma_completion_fn_t)(       \
+                                                         send_frag)                             \
+                                                         ->sf_base.uf_base.des_cbfunc;          \
+            func(&(module)->super, (send_frag)->sf_endpoint,                                    \
+                 (send_frag)->sf_base.uf_local_seg[0].seg_addr.pval, NULL,                      \
+                 (send_frag)->sf_base.uf_base.des_context,                                      \
+                 (send_frag)->sf_base.uf_base.des_cbdata, OPAL_SUCCESS);                        \
+            ++((module)->stats.pml_send_callbacks);                                             \
+        } while (0)
 #endif
 
 /*
  * Reap an ACK send that is complete
  */
 void opal_btl_usnic_ack_complete(opal_btl_usnic_module_t *module,
-                                   opal_btl_usnic_ack_segment_t *ack);
-
+                                 opal_btl_usnic_ack_segment_t *ack);
 
 /*
  * Send an ACK
  */
-int opal_btl_usnic_ack_send(opal_btl_usnic_module_t *module,
-                               opal_btl_usnic_endpoint_t *endpoint);
+int opal_btl_usnic_ack_send(opal_btl_usnic_module_t *module, opal_btl_usnic_endpoint_t *endpoint);
 
 /*
  * Callback for when a send times out without receiving a
  * corresponding ACK
  */
-void opal_btl_usnic_ack_timeout(opal_hotel_t *hotel, int room_num,
-                                  void *occupant);
+void opal_btl_usnic_ack_timeout(opal_hotel_t *hotel, int room_num, void *occupant);
 
 /*
  * Handle an incoming ACK
  */
-void opal_btl_usnic_handle_ack(opal_btl_usnic_endpoint_t *endpoint,
-                               opal_btl_usnic_seq_t ack_seq);
+void opal_btl_usnic_handle_ack(opal_btl_usnic_endpoint_t *endpoint, opal_btl_usnic_seq_t ack_seq);
 
-static inline void
-opal_btl_usnic_piggyback_ack(
-    opal_btl_usnic_endpoint_t *endpoint,
-    opal_btl_usnic_send_segment_t *sseg)
+static inline void opal_btl_usnic_piggyback_ack(opal_btl_usnic_endpoint_t *endpoint,
+                                                opal_btl_usnic_send_segment_t *sseg)
 {
     /* If ACK is needed, piggy-back it here and send it on */
     if (endpoint->endpoint_ack_needed) {
         opal_btl_usnic_remove_from_endpoints_needing_ack(endpoint);
-        sseg->ss_base.us_btl_header->ack_seq =
-            SEQ_DIFF(endpoint->endpoint_next_contig_seq_to_recv, 1);
+        sseg->ss_base.us_btl_header->ack_seq = SEQ_DIFF(endpoint->endpoint_next_contig_seq_to_recv,
+                                                        1);
         sseg->ss_base.us_btl_header->ack_present = 1;
 #if MSGDEBUG1
-        opal_output(0, "Piggy-backing ACK for sequence %"UDSEQ"\n",
-                sseg->ss_base.us_btl_header->ack_seq);
+        opal_output(0, "Piggy-backing ACK for sequence %" UDSEQ "\n",
+                    sseg->ss_base.us_btl_header->ack_seq);
 #endif
     } else {
         sseg->ss_base.us_btl_header->ack_present = 0;
     }
 }
 
-
 #endif /* BTL_USNIC_ACK_H */
diff --git a/opal/mca/btl/usnic/btl_usnic_cagent.c b/opal/mca/btl/usnic/btl_usnic_cagent.c
index 105d5dfcfe5..e55aa02d603 100644
--- a/opal/mca/btl/usnic/btl_usnic_cagent.c
+++ b/opal/mca/btl/usnic/btl_usnic_cagent.c
@@ -13,23 +13,23 @@
 #include "opal_config.h"
 
 #include <assert.h>
-#include <sys/types.h>
 #include <sys/socket.h>
+#include <sys/types.h>
 #include <sys/un.h>
 #include <unistd.h>
 #ifdef HAVE_ALLOCA_H
-#include <alloca.h>
+#    include <alloca.h>
 #endif
 
-#include "opal_stdint.h"
 #include "opal/mca/threads/mutex.h"
-#include "opal/util/event.h"
-#include "opal/util/show_help.h"
 #include "opal/types.h"
-#include "opal/util/output.h"
+#include "opal/util/event.h"
 #include "opal/util/fd.h"
-#include "opal/util/string_copy.h"
+#include "opal/util/output.h"
 #include "opal/util/printf.h"
+#include "opal/util/show_help.h"
+#include "opal/util/string_copy.h"
+#include "opal_stdint.h"
 
 #include "btl_usnic.h"
 #include "btl_usnic_connectivity.h"
@@ -47,14 +47,13 @@ static opal_event_t ipc_event;
 static struct timeval ack_timeout;
 static opal_list_t udp_port_listeners;
 static opal_list_t ipc_listeners;
-static volatile int ipc_accepts = 0;
+static volatile uint32_t ipc_accepts = 0;
 /* JMS The pings_pending and ping_results should probably both be hash
    tables for more efficient lookups */
 static opal_list_t pings_pending;
 static opal_list_t ping_results;
 static volatile bool agent_initialized = false;
 
-
 /*
  * Holds all the information about a UDP port that the agent thread is
  * listening on (for incoming PINGs and ACKs).
@@ -95,14 +94,11 @@ typedef struct {
 
 OBJ_CLASS_DECLARATION(agent_ipc_listener_t);
 
-typedef enum {
-    AGENT_MSG_TYPE_PING = 17,
-    AGENT_MSG_TYPE_ACK
-} agent_udp_message_type_t;
+typedef enum { AGENT_MSG_TYPE_PING = 17, AGENT_MSG_TYPE_ACK } agent_udp_message_type_t;
 
 // Arbitrary 64 bit numbers
 #define MAGIC_ORIGINATOR 0x9a9e2fbce63a11e5
-#define MAGIC_TARGET 0x60735c68f368aace
+#define MAGIC_TARGET     0x60735c68f368aace
 
 /*
  * Ping and ACK messages
@@ -159,18 +155,14 @@ typedef struct {
 
 OBJ_CLASS_DECLARATION(agent_ping_t);
 
-
 /**************************************************************************
  * Utility functions, constructors, destructors
  **************************************************************************/
 
 static void udp_port_listener_zero(agent_udp_port_listener_t *obj)
 {
-    obj->ipv4_addr =
-        obj->netmask =
-        obj->max_msg_size = 0;
-    obj->nodename =
-        obj->usnic_name = NULL;
+    obj->ipv4_addr = obj->netmask = obj->max_msg_size = 0;
+    obj->nodename = obj->usnic_name = NULL;
     memset(obj->ipv4_addr_str, 0, sizeof(obj->ipv4_addr_str));
 
     obj->fd = -1;
@@ -190,7 +182,7 @@ static void udp_port_listener_destructor(agent_udp_port_listener_t *obj)
     /* Find any pings that are pending on this listener and delete
        them */
     agent_ping_t *ap, *apnext;
-    OPAL_LIST_FOREACH_SAFE(ap, apnext, &pings_pending, agent_ping_t) {
+    OPAL_LIST_FOREACH_SAFE (ap, apnext, &pings_pending, agent_ping_t) {
         if (ap->src_ipv4_addr == obj->ipv4_addr) {
             opal_list_remove_item(&pings_pending, &ap->super);
             OBJ_RELEASE(ap);
@@ -220,9 +212,7 @@ static void udp_port_listener_destructor(agent_udp_port_listener_t *obj)
     udp_port_listener_zero(obj);
 }
 
-OBJ_CLASS_INSTANCE(agent_udp_port_listener_t,
-                   opal_list_item_t,
-                   udp_port_listener_constructor,
+OBJ_CLASS_INSTANCE(agent_udp_port_listener_t, opal_list_item_t, udp_port_listener_constructor,
                    udp_port_listener_destructor);
 
 static void ipc_listener_zero(agent_ipc_listener_t *obj)
@@ -252,9 +242,7 @@ static void ipc_listener_destructor(agent_ipc_listener_t *obj)
     ipc_listener_zero(obj);
 }
 
-OBJ_CLASS_INSTANCE(agent_ipc_listener_t,
-                   opal_list_item_t,
-                   ipc_listener_constructor,
+OBJ_CLASS_INSTANCE(agent_ipc_listener_t, opal_list_item_t, ipc_listener_constructor,
                    ipc_listener_destructor);
 
 static void agent_ping_result_zero(agent_ping_t *obj)
@@ -293,16 +281,13 @@ static void agent_ping_result_destructor(agent_ping_t *obj)
     agent_ping_result_zero(obj);
 }
 
-OBJ_CLASS_INSTANCE(agent_ping_t,
-                   opal_list_item_t,
-                   agent_ping_result_constructor,
+OBJ_CLASS_INSTANCE(agent_ping_t, opal_list_item_t, agent_ping_result_constructor,
                    agent_ping_result_destructor);
 
 /*
  * Wrapper around sendto() loop
  */
-static void agent_sendto(int fd, char *buffer, ssize_t numbytes,
-                         struct sockaddr *addr)
+static void agent_sendto(int fd, char *buffer, ssize_t numbytes, struct sockaddr *addr)
 {
     ssize_t rc;
     while (1) {
@@ -321,8 +306,7 @@ static void agent_sendto(int fd, char *buffer, ssize_t numbytes,
             }
 
             char *msg;
-            opal_asprintf(&msg, "Unexpected sendto() error: errno=%d (%s)",
-                     errno, strerror(errno));
+            opal_asprintf(&msg, "Unexpected sendto() error: errno=%d (%s)", errno, strerror(errno));
             ABORT(msg);
             /* Will not return */
         }
@@ -342,20 +326,21 @@ static void agent_sendto(int fd, char *buffer, ssize_t numbytes,
 /*
  * Handle an incoming PING message (send an ACK)
  */
-static void agent_thread_handle_ping(agent_udp_port_listener_t *listener,
-                                     ssize_t numbytes, struct sockaddr *from)
+static void agent_thread_handle_ping(agent_udp_port_listener_t *listener, ssize_t numbytes,
+                                     struct sockaddr *from)
 {
     /* If the size we received isn't equal to what the sender says it
        sent, do the simple thing: just don't send an ACK */
-    agent_udp_message_t *msg = (agent_udp_message_t*) listener->buffer;
-    struct sockaddr_in *src_addr_in = (struct sockaddr_in*) from;
+    agent_udp_message_t *msg = (agent_udp_message_t *) listener->buffer;
+    struct sockaddr_in *src_addr_in = (struct sockaddr_in *) from;
     if (msg->size != numbytes) {
         char str[INET_ADDRSTRLEN];
         inet_ntop(AF_INET, &src_addr_in->sin_addr, str, sizeof(str));
 
-        opal_output_verbose(20, USNIC_OUT,
-                            "usNIC connectivity got bad ping: %d bytes from %s, expected %d (discarded)",
-                            (int) numbytes, str, (int) msg->size);
+        opal_output_verbose(
+            20, USNIC_OUT,
+            "usNIC connectivity got bad ping: %d bytes from %s, expected %d (discarded)",
+            (int) numbytes, str, (int) msg->size);
         return;
     }
 
@@ -367,32 +352,32 @@ static void agent_thread_handle_ping(agent_udp_port_listener_t *listener,
     char msg_ipv4_addr_str[IPV4STRADDRLEN];
     char real_ipv4_addr_str[IPV4STRADDRLEN];
 
-    opal_btl_usnic_snprintf_ipv4_addr(msg_ipv4_addr_str,
-                                      sizeof(msg_ipv4_addr_str),
+    opal_btl_usnic_snprintf_ipv4_addr(msg_ipv4_addr_str, sizeof(msg_ipv4_addr_str),
                                       msg->src_ipv4_addr, 0);
-    opal_btl_usnic_snprintf_ipv4_addr(real_ipv4_addr_str,
-                                      sizeof(real_ipv4_addr_str),
+    opal_btl_usnic_snprintf_ipv4_addr(real_ipv4_addr_str, sizeof(real_ipv4_addr_str),
                                       src_addr_in->sin_addr.s_addr, 0);
 
     if (msg->src_ipv4_addr != src_addr_in->sin_addr.s_addr) {
-        opal_output_verbose(20, USNIC_OUT,
-                            "usNIC connectivity got bad ping (from unexpected address: %s != %s, discarded)",
-                            msg_ipv4_addr_str, real_ipv4_addr_str);
+        opal_output_verbose(
+            20, USNIC_OUT,
+            "usNIC connectivity got bad ping (from unexpected address: %s != %s, discarded)",
+            msg_ipv4_addr_str, real_ipv4_addr_str);
         return;
     }
 
     if (msg->magic_number != MAGIC_ORIGINATOR) {
         opal_output_verbose(20, USNIC_OUT,
-                            "usNIC connectivity got bad ping (magic number: %" PRIu64 ", discarded)",
+                            "usNIC connectivity got bad ping (magic number: %" PRIu64
+                            ", discarded)",
                             msg->magic_number);
         return;
     }
-    if (msg->major_version != OPAL_MAJOR_VERSION ||
-        msg->minor_version != OPAL_MINOR_VERSION) {
+    if (msg->major_version != OPAL_MAJOR_VERSION || msg->minor_version != OPAL_MINOR_VERSION) {
         opal_output_verbose(20, USNIC_OUT,
-                            "usNIC connectivity got bad ping (originator version: %d.%d, expected %d.%d, discarded)",
-                            msg->major_version, msg->minor_version,
-                            OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION);
+                            "usNIC connectivity got bad ping (originator version: %d.%d, expected "
+                            "%d.%d, discarded)",
+                            msg->major_version, msg->minor_version, OPAL_MAJOR_VERSION,
+                            OPAL_MINOR_VERSION);
         return;
     }
 
@@ -401,8 +386,8 @@ static void agent_thread_handle_ping(agent_udp_port_listener_t *listener,
        expected. */
 
     opal_output_verbose(20, USNIC_OUT,
-                        "usNIC connectivity got PING (size=%ld) from %s; sending ACK",
-                        numbytes, msg_ipv4_addr_str);
+                        "usNIC connectivity got PING (size=%ld) from %s; sending ACK", numbytes,
+                        msg_ipv4_addr_str);
 
     /* Send back an ACK.  No need to allocate a new buffer; just
        re-use the same buffer we just got.  Note that msg->size is
@@ -412,26 +397,27 @@ static void agent_thread_handle_ping(agent_udp_port_listener_t *listener,
     msg->message_type = AGENT_MSG_TYPE_ACK;
     msg->magic_number = MAGIC_TARGET;
 
-    agent_sendto(listener->fd, (char*) listener->buffer, sizeof(*msg), from);
+    agent_sendto(listener->fd, (char *) listener->buffer, sizeof(*msg), from);
 }
 
 /*
  * Handle an incoming ACK message
  */
-static void agent_thread_handle_ack(agent_udp_port_listener_t *listener,
-                                    ssize_t numbytes, struct sockaddr *from)
+static void agent_thread_handle_ack(agent_udp_port_listener_t *listener, ssize_t numbytes,
+                                    struct sockaddr *from)
 {
     char str[INET_ADDRSTRLEN];
-    struct sockaddr_in *src_addr_in = (struct sockaddr_in*) from;
+    struct sockaddr_in *src_addr_in = (struct sockaddr_in *) from;
     inet_ntop(AF_INET, &src_addr_in->sin_addr, str, sizeof(str));
 
     /* If we got a wonky ACK message that is the wrong length, just
        return */
-    agent_udp_message_t *msg = (agent_udp_message_t*) listener->buffer;
+    agent_udp_message_t *msg = (agent_udp_message_t *) listener->buffer;
     if (numbytes != sizeof(*msg)) {
-        opal_output_verbose(20, USNIC_OUT,
-                            "usNIC connectivity got bad ACK: %d bytes from %s, expected %d (discarded)",
-                            (int) numbytes, str, (int) sizeof(*msg));
+        opal_output_verbose(
+            20, USNIC_OUT,
+            "usNIC connectivity got bad ACK: %d bytes from %s, expected %d (discarded)",
+            (int) numbytes, str, (int) sizeof(*msg));
         return;
     }
     if (msg->magic_number != MAGIC_TARGET) {
@@ -445,11 +431,9 @@ static void agent_thread_handle_ack(agent_udp_port_listener_t *listener,
        If we don't find a match, we'll drop it. */
     agent_ping_t *ap;
     uint32_t src_in_port = ntohs(src_addr_in->sin_port);
-    OPAL_LIST_FOREACH(ap, &pings_pending, agent_ping_t) {
-        if (ap->dest_ipv4_addr == src_addr_in->sin_addr.s_addr &&
-            ap->dest_udp_port == src_in_port &&
-            ap->src_ipv4_addr == msg->src_ipv4_addr &&
-            ap->src_udp_port == msg->src_udp_port) {
+    OPAL_LIST_FOREACH (ap, &pings_pending, agent_ping_t) {
+        if (ap->dest_ipv4_addr == src_addr_in->sin_addr.s_addr && ap->dest_udp_port == src_in_port
+            && ap->src_ipv4_addr == msg->src_ipv4_addr && ap->src_udp_port == msg->src_udp_port) {
             /* Found it -- indicate that it has been acked */
             for (int i = 0; i < NUM_PING_SIZES; ++i) {
                 if (ap->sizes[i] == msg->size) {
@@ -472,19 +456,18 @@ static void agent_thread_handle_ack(agent_udp_port_listener_t *listener,
  */
 static void agent_thread_receive_ping(int fd, short flags, void *context)
 {
-    agent_udp_port_listener_t *listener =
-        (agent_udp_port_listener_t *) context;
+    agent_udp_port_listener_t *listener = (agent_udp_port_listener_t *) context;
     assert(NULL != listener);
 
     /* Receive the message */
     ssize_t numbytes;
     struct sockaddr src_addr;
-    struct sockaddr_in *src_addr_in = (struct sockaddr_in*) &src_addr;
+    struct sockaddr_in *src_addr_in = (struct sockaddr_in *) &src_addr;
     socklen_t addrlen = sizeof(src_addr);
 
     while (1) {
-        numbytes = recvfrom(listener->fd, listener->buffer, listener->max_msg_size, 0,
-                            &src_addr, &addrlen);
+        numbytes = recvfrom(listener->fd, listener->buffer, listener->max_msg_size, 0, &src_addr,
+                            &addrlen);
         if (numbytes > 0) {
             break;
         } else if (numbytes < 0) {
@@ -517,11 +500,10 @@ static void agent_thread_receive_ping(int fd, short flags, void *context)
     }
 }
 
-static agent_udp_port_listener_t *
-agent_thread_find_listener(uint32_t ipv4_addr, uint32_t *udp_port)
+static agent_udp_port_listener_t *agent_thread_find_listener(uint32_t ipv4_addr, uint32_t *udp_port)
 {
     agent_udp_port_listener_t *listener;
-    OPAL_LIST_FOREACH(listener, &udp_port_listeners, agent_udp_port_listener_t) {
+    OPAL_LIST_FOREACH (listener, &udp_port_listeners, agent_udp_port_listener_t) {
         if (listener->ipv4_addr == ipv4_addr) {
             *udp_port = listener->udp_port;
             return listener;
@@ -535,16 +517,13 @@ agent_thread_find_listener(uint32_t ipv4_addr, uint32_t *udp_port)
  * Send reply back from the LISTEN command: send back the IP address
  * and UDP port that we're listening on.
  */
-static int agent_thread_cmd_listen_reply(int fd,
-                                         uint32_t addr, int32_t udp_port)
+static int agent_thread_cmd_listen_reply(int fd, uint32_t addr, int32_t udp_port)
 {
     int ret;
 
-    opal_btl_usnic_connectivity_cmd_listen_reply_t cmd = {
-        .cmd = CONNECTIVITY_AGENT_CMD_LISTEN,
-        .ipv4_addr = addr,
-        .udp_port = udp_port
-    };
+    opal_btl_usnic_connectivity_cmd_listen_reply_t cmd = {.cmd = CONNECTIVITY_AGENT_CMD_LISTEN,
+                                                          .ipv4_addr = addr,
+                                                          .udp_port = udp_port};
 
     ret = opal_fd_write(fd, sizeof(cmd), &cmd);
     if (OPAL_SUCCESS != ret) {
@@ -587,8 +566,7 @@ static void agent_thread_cmd_listen(agent_ipc_listener_t *ipc_listener)
         if (NULL == udp_listener->module) {
             udp_listener->module = cmd.module;
         }
-        agent_thread_cmd_listen_reply(ipc_listener->client_fd,
-                                      cmd.ipv4_addr, udp_port);
+        agent_thread_cmd_listen_reply(ipc_listener->client_fd, cmd.ipv4_addr, udp_port);
         return;
     }
 
@@ -611,8 +589,8 @@ static void agent_thread_cmd_listen(agent_ipc_listener_t *ipc_listener)
        address in sockaddr_in form, it's not worth using
        inet_ntop() */
     opal_btl_usnic_snprintf_ipv4_addr(udp_listener->ipv4_addr_str,
-                                      sizeof(udp_listener->ipv4_addr_str),
-                                      cmd.ipv4_addr, cmd.netmask);
+                                      sizeof(udp_listener->ipv4_addr_str), cmd.ipv4_addr,
+                                      cmd.netmask);
 
     udp_listener->buffer = malloc(udp_listener->max_msg_size);
     if (NULL == udp_listener->buffer) {
@@ -636,7 +614,7 @@ static void agent_thread_cmd_listen(agent_ipc_listener_t *ipc_listener)
     inaddr.sin_addr.s_addr = cmd.ipv4_addr;
     inaddr.sin_port = htons(0);
 
-    ret = bind(udp_listener->fd, (struct sockaddr*) &inaddr, sizeof(inaddr));
+    ret = bind(udp_listener->fd, (struct sockaddr *) &inaddr, sizeof(inaddr));
     if (ret < 0) {
         OPAL_ERROR_LOG(ret);
         ABORT("Could not bind listening socket");
@@ -645,7 +623,7 @@ static void agent_thread_cmd_listen(agent_ipc_listener_t *ipc_listener)
 
     /* Find out the port we got */
     opal_socklen_t addrlen = sizeof(struct sockaddr_in);
-    ret = getsockname(udp_listener->fd, (struct sockaddr*) &inaddr, &addrlen);
+    ret = getsockname(udp_listener->fd, (struct sockaddr *) &inaddr, &addrlen);
     if (ret < 0) {
         OPAL_ERROR_LOG(ret);
         ABORT("Could not get UDP port number from listening socket");
@@ -653,10 +631,8 @@ static void agent_thread_cmd_listen(agent_ipc_listener_t *ipc_listener)
     }
     udp_listener->udp_port = ntohs(inaddr.sin_port);
 
-    opal_output_verbose(20, USNIC_OUT,
-                        "usNIC connectivity agent listening on %s:%d, (%s)",
-                        udp_listener->ipv4_addr_str,
-                        udp_listener->udp_port,
+    opal_output_verbose(20, USNIC_OUT, "usNIC connectivity agent listening on %s:%d, (%s)",
+                        udp_listener->ipv4_addr_str, udp_listener->udp_port,
                         udp_listener->usnic_name);
 
     /* Set the "don't fragment" bit on outgoing frames because we
@@ -664,8 +640,7 @@ static void agent_thread_cmd_listen(agent_ipc_listener_t *ipc_listener)
        peer, or fail if they have to fragment because of an MTU
        mismatch somewhere enroute */
     int val = IP_PMTUDISC_DO;
-    ret = setsockopt(udp_listener->fd, IPPROTO_IP, IP_MTU_DISCOVER,
-                     &val, sizeof(val));
+    ret = setsockopt(udp_listener->fd, IPPROTO_IP, IP_MTU_DISCOVER, &val, sizeof(val));
     if (0 != ret) {
         OPAL_ERROR_LOG(ret);
         ABORT("Unable to set \"do not fragment\" on UDP socket");
@@ -675,20 +650,16 @@ static void agent_thread_cmd_listen(agent_ipc_listener_t *ipc_listener)
     /* Set the send and receive buffer sizes to our MTU size */
     int temp;
     temp = (int) udp_listener->max_msg_size;
-    if ((ret = setsockopt(udp_listener->fd, SOL_SOCKET, SO_RCVBUF,
-                          &temp, sizeof(temp))) < 0 ||
-        (ret = setsockopt(udp_listener->fd, SOL_SOCKET, SO_SNDBUF,
-                          &temp, sizeof(temp))) < 0) {
+    if ((ret = setsockopt(udp_listener->fd, SOL_SOCKET, SO_RCVBUF, &temp, sizeof(temp))) < 0
+        || (ret = setsockopt(udp_listener->fd, SOL_SOCKET, SO_SNDBUF, &temp, sizeof(temp))) < 0) {
         OPAL_ERROR_LOG(ret);
         ABORT("Could not set socket buffer sizes");
         /* Will not return */
     }
 
     /* Create a listening event */
-    opal_event_set(mca_btl_usnic_component.opal_evbase,
-                   &udp_listener->event, udp_listener->fd,
-                   OPAL_EV_READ | OPAL_EV_PERSIST,
-                   agent_thread_receive_ping, udp_listener);
+    opal_event_set(mca_btl_usnic_component.opal_evbase, &udp_listener->event, udp_listener->fd,
+                   OPAL_EV_READ | OPAL_EV_PERSIST, agent_thread_receive_ping, udp_listener);
     opal_event_add(&udp_listener->event, 0);
 
     /* Save this listener on the list of udp_port_listeners */
@@ -697,8 +668,8 @@ static void agent_thread_cmd_listen(agent_ipc_listener_t *ipc_listener)
     udp_listener->active = true;
 
     /* Return the port number to the sender */
-    ret = agent_thread_cmd_listen_reply(ipc_listener->client_fd,
-                                        cmd.ipv4_addr, udp_listener->udp_port);
+    ret = agent_thread_cmd_listen_reply(ipc_listener->client_fd, cmd.ipv4_addr,
+                                        udp_listener->udp_port);
 
     /* All done! */
     return;
@@ -709,12 +680,11 @@ static void agent_thread_cmd_listen(agent_ipc_listener_t *ipc_listener)
  */
 static void agent_thread_send_ping(int fd, short flags, void *context)
 {
-    agent_ping_t *ap = (agent_ping_t*) context;
+    agent_ping_t *ap = (agent_ping_t *) context;
     ap->timer_active = false;
 
     char dest_ipv4_addr_str[IPV4STRADDRLEN];
-    opal_btl_usnic_snprintf_ipv4_addr(dest_ipv4_addr_str,
-                                      sizeof(dest_ipv4_addr_str),
+    opal_btl_usnic_snprintf_ipv4_addr(dest_ipv4_addr_str, sizeof(dest_ipv4_addr_str),
                                       ap->dest_ipv4_addr, ap->dest_netmask);
 
     /* If we got all the ACKs for this ping, then move this ping from
@@ -725,10 +695,8 @@ static void agent_thread_send_ping(int fd, short flags, void *context)
         opal_list_remove_item(&pings_pending, &ap->super);
         opal_list_append(&ping_results, &ap->super);
 
-        opal_output_verbose(20, USNIC_OUT,
-                            "usNIC connectivity GOOD between %s <--> %s",
-                            ap->listener->ipv4_addr_str,
-                            dest_ipv4_addr_str);
+        opal_output_verbose(20, USNIC_OUT, "usNIC connectivity GOOD between %s <--> %s",
+                            ap->listener->ipv4_addr_str, dest_ipv4_addr_str);
 
         for (int i = 0; i < 2; ++i) {
             if (NULL != ap->buffers[i]) {
@@ -758,40 +726,29 @@ static void agent_thread_send_ping(int fd, short flags, void *context)
         }
 
         char ipv4_addr_str[IPV4STRADDRLEN];
-        opal_btl_usnic_snprintf_ipv4_addr(ipv4_addr_str, sizeof(ipv4_addr_str),
-                                          ap->dest_ipv4_addr,
+        opal_btl_usnic_snprintf_ipv4_addr(ipv4_addr_str, sizeof(ipv4_addr_str), ap->dest_ipv4_addr,
                                           ap->dest_netmask);
-        opal_show_help("help-mpi-btl-usnic.txt", topic, true,
-                       opal_process_info.nodename,
-                       ap->listener->ipv4_addr_str,
-                       ap->listener->usnic_name,
-                       ap->dest_nodename,
-                       ipv4_addr_str,
-                       ap->sizes[0],
-                       ap->sizes[1]);
+        opal_show_help("help-mpi-btl-usnic.txt", topic, true, opal_process_info.nodename,
+                       ap->listener->ipv4_addr_str, ap->listener->usnic_name, ap->dest_nodename,
+                       ipv4_addr_str, ap->sizes[0], ap->sizes[1]);
         opal_btl_usnic_exit(NULL);
         /* Will not return */
     }
 
     time_t t = time(NULL);
-    opal_output_verbose(20, USNIC_OUT,
-                        "usNIC connectivity pinging %s:%d (%s) from %s (%s) at %s",
-                        dest_ipv4_addr_str,
-                        ntohs(ap->dest_sockaddr.sin_port),
-                        ap->dest_nodename,
-                        ap->listener->ipv4_addr_str,
-                        ap->listener->usnic_name,
-                        ctime(&t));
+    opal_output_verbose(20, USNIC_OUT, "usNIC connectivity pinging %s:%d (%s) from %s (%s) at %s",
+                        dest_ipv4_addr_str, ntohs(ap->dest_sockaddr.sin_port), ap->dest_nodename,
+                        ap->listener->ipv4_addr_str, ap->listener->usnic_name, ctime(&t));
 
     /* Send the ping messages to the peer */
     for (int i = 0; i < NUM_PING_SIZES; ++i) {
-        agent_sendto(ap->listener->fd, (char*) ap->buffers[i], ap->sizes[i],
-                     (struct sockaddr*) &ap->dest_sockaddr);
+        agent_sendto(ap->listener->fd, (char *) ap->buffers[i], ap->sizes[i],
+                     (struct sockaddr *) &ap->dest_sockaddr);
     }
 
     /* Set a timer to check if these pings are ACKed */
-    opal_event_set(mca_btl_usnic_component.opal_evbase, &ap->timer,
-                   -1, 0, agent_thread_send_ping, ap);
+    opal_event_set(mca_btl_usnic_component.opal_evbase, &ap->timer, -1, 0, agent_thread_send_ping,
+                   ap);
     opal_event_add(&ap->timer, &ack_timeout);
     ap->timer_active = true;
 
@@ -817,9 +774,8 @@ static void agent_thread_cmd_ping(agent_ipc_listener_t *ipc_listener)
 
     /* Have we already pinged this IP address / port? */
     agent_ping_t *ap;
-    OPAL_LIST_FOREACH(ap, &ping_results, agent_ping_t) {
-        if (ap->dest_ipv4_addr == cmd.dest_ipv4_addr &&
-            ap->dest_udp_port == cmd.dest_udp_port) {
+    OPAL_LIST_FOREACH (ap, &ping_results, agent_ping_t) {
+        if (ap->dest_ipv4_addr == cmd.dest_ipv4_addr && ap->dest_udp_port == cmd.dest_udp_port) {
             /* We already have results from pinging this IP address /
                port, so there's no need for further action */
             return;
@@ -827,9 +783,8 @@ static void agent_thread_cmd_ping(agent_ipc_listener_t *ipc_listener)
     }
 
     /* Are we in the middle of pinging this IP address / port? */
-    OPAL_LIST_FOREACH(ap, &pings_pending, agent_ping_t) {
-        if (ap->dest_ipv4_addr == cmd.dest_ipv4_addr &&
-            ap->dest_udp_port == cmd.dest_udp_port) {
+    OPAL_LIST_FOREACH (ap, &pings_pending, agent_ping_t) {
+        if (ap->dest_ipv4_addr == cmd.dest_ipv4_addr && ap->dest_udp_port == cmd.dest_udp_port) {
             /* We're already in the middle of pinging this IP address
                / port, so there's no need for further action */
             return;
@@ -840,8 +795,7 @@ static void agent_thread_cmd_ping(agent_ipc_listener_t *ipc_listener)
        ipv4 address */
     bool found = false;
     agent_udp_port_listener_t *udp_listener;
-    OPAL_LIST_FOREACH(udp_listener, &udp_port_listeners,
-                      agent_udp_port_listener_t) {
+    OPAL_LIST_FOREACH (udp_listener, &udp_port_listeners, agent_udp_port_listener_t) {
         if (udp_listener->ipv4_addr == cmd.src_ipv4_addr) {
             found = true;
             break;
@@ -895,7 +849,7 @@ static void agent_thread_cmd_ping(agent_ipc_listener_t *ipc_listener)
         }
 
         /* Fill in the message with return addressing information */
-        msg = (agent_udp_message_t*) ap->buffers[i];
+        msg = (agent_udp_message_t *) ap->buffers[i];
         msg->message_type = AGENT_MSG_TYPE_PING;
         msg->src_ipv4_addr = ap->src_ipv4_addr;
         msg->src_udp_port = ap->src_udp_port;
@@ -947,7 +901,7 @@ static void agent_thread_cmd_unlisten(agent_ipc_listener_t *ipc_listener)
 static void agent_thread_ipc_receive(int fd, short flags, void *context)
 {
     int32_t command;
-    agent_ipc_listener_t *ipc_listener = (agent_ipc_listener_t*) context;
+    agent_ipc_listener_t *ipc_listener = (agent_ipc_listener_t *) context;
 
     /* Read the command */
     command = -1;
@@ -962,9 +916,8 @@ static void agent_thread_ipc_receive(int fd, short flags, void *context)
         /* Will not return */
     }
 
-    assert(CONNECTIVITY_AGENT_CMD_LISTEN == command ||
-           CONNECTIVITY_AGENT_CMD_PING == command ||
-           CONNECTIVITY_AGENT_CMD_UNLISTEN == command);
+    assert(CONNECTIVITY_AGENT_CMD_LISTEN == command || CONNECTIVITY_AGENT_CMD_PING == command
+           || CONNECTIVITY_AGENT_CMD_UNLISTEN == command);
 
     switch (command) {
     case CONNECTIVITY_AGENT_CMD_LISTEN:
@@ -1014,8 +967,9 @@ static void agent_thread_accept(int fd, short flags, void *context)
         /* Will not return */
     }
     if (0 != memcmp(msg, CONNECTIVITY_MAGIC_TOKEN, tlen)) {
-        opal_output_verbose(20, USNIC_OUT,
-                            "usNIC connectivity got bad IPC client (wrong magic token); disconnected");
+        opal_output_verbose(
+            20, USNIC_OUT,
+            "usNIC connectivity got bad IPC client (wrong magic token); disconnected");
         close(client_fd);
         return;
     }
@@ -1029,18 +983,15 @@ static void agent_thread_accept(int fd, short flags, void *context)
 
     /* Write back the magic token to ACK that we got the peer's
        magic token and all is kosher */
-    if (OPAL_SUCCESS != opal_fd_write(client_fd, tlen,
-                                      CONNECTIVITY_MAGIC_TOKEN)) {
+    if (OPAL_SUCCESS != opal_fd_write(client_fd, tlen, CONNECTIVITY_MAGIC_TOKEN)) {
         OPAL_ERROR_LOG(OPAL_ERR_OUT_OF_RESOURCE);
         ABORT("usnic connectivity agent IPC read failed");
         /* Will not return */
     }
 
     /* Add this IPC listener to the event base */
-    opal_event_set(mca_btl_usnic_component.opal_evbase,
-                   &listener->event, client_fd,
-                   OPAL_EV_READ | OPAL_EV_PERSIST,
-                   agent_thread_ipc_receive, listener);
+    opal_event_set(mca_btl_usnic_component.opal_evbase, &listener->event, client_fd,
+                   OPAL_EV_READ | OPAL_EV_PERSIST, agent_thread_ipc_receive, listener);
     opal_event_add(&listener->event, 0);
 
     /* Save this listener on the list of ipc_listeners */
@@ -1078,25 +1029,23 @@ static void agent_thread_finalize(int fd, short flags, void *context)
         first = false;
     }
 
-    if (ipc_accepts < opal_process_info.num_local_peers &&
-        time(NULL) < timestamp + 10) {
-        opal_output_verbose(20, USNIC_OUT,
-                            "usNIC connectivity agent delaying shutdown until all clients connect...");
+    if (ipc_accepts < opal_process_info.num_local_peers && time(NULL) < timestamp + 10) {
+        opal_output_verbose(
+            20, USNIC_OUT,
+            "usNIC connectivity agent delaying shutdown until all clients connect...");
 
         opal_event_t *ev = calloc(sizeof(*ev), 1);
-        struct timeval finalize_retry = {
-            .tv_sec = 0,
-            .tv_usec = 10000
-        };
+        struct timeval finalize_retry = {.tv_sec = 0, .tv_usec = 10000};
 
-        opal_event_set(mca_btl_usnic_component.opal_evbase,
-                       ev, -1, 0, agent_thread_finalize, ev);
+        opal_event_set(mca_btl_usnic_component.opal_evbase, ev, -1, 0, agent_thread_finalize, ev);
         opal_event_add(ev, &finalize_retry);
         return;
     }
     if (ipc_accepts < opal_process_info.num_local_peers) {
         opal_output_verbose(20, USNIC_OUT,
-                            "usNIC connectivity agent: only %d of %d clients connected, but timeout has expired -- exiting anyway", ipc_accepts, opal_process_info.num_local_peers);
+                            "usNIC connectivity agent: only %d of %d clients connected, but "
+                            "timeout has expired -- exiting anyway",
+                            ipc_accepts, opal_process_info.num_local_peers);
     }
 
     /* Remove the agent listening event from the opal async event
@@ -1105,27 +1054,25 @@ static void agent_thread_finalize(int fd, short flags, void *context)
 
     /* Shut down all active udp_port_listeners */
     agent_udp_port_listener_t *udp_listener, *ulnext;
-    OPAL_LIST_FOREACH_SAFE(udp_listener, ulnext, &udp_port_listeners,
-                           agent_udp_port_listener_t) {
+    OPAL_LIST_FOREACH_SAFE (udp_listener, ulnext, &udp_port_listeners, agent_udp_port_listener_t) {
         OBJ_RELEASE(udp_listener);
     }
 
     /* Destroy the pending pings and ping results */
     agent_ping_t *request, *pnext;
-    OPAL_LIST_FOREACH_SAFE(request, pnext, &pings_pending, agent_ping_t) {
+    OPAL_LIST_FOREACH_SAFE (request, pnext, &pings_pending, agent_ping_t) {
         opal_list_remove_item(&pings_pending, &request->super);
         OBJ_RELEASE(request);
     }
 
-    OPAL_LIST_FOREACH_SAFE(request, pnext, &ping_results, agent_ping_t) {
+    OPAL_LIST_FOREACH_SAFE (request, pnext, &ping_results, agent_ping_t) {
         opal_list_remove_item(&ping_results, &request->super);
         OBJ_RELEASE(request);
     }
 
     /* Shut down all active ipc_listeners */
     agent_ipc_listener_t *ipc_listener, *inext;
-    OPAL_LIST_FOREACH_SAFE(ipc_listener, inext, &ipc_listeners,
-                           agent_ipc_listener_t) {
+    OPAL_LIST_FOREACH_SAFE (ipc_listener, inext, &ipc_listeners, agent_ipc_listener_t) {
         OBJ_RELEASE(ipc_listener);
     }
 
@@ -1154,10 +1101,8 @@ int opal_btl_usnic_connectivity_agent_init(void)
     /* Make a struct timeval for use with timer events.  Note that the
        MCA param is expressed in terms of *milli*seconds, but the
        timeval timeout is expressed in terms of *micro*seconds. */
-    ack_timeout.tv_sec =
-        mca_btl_usnic_component.connectivity_ack_timeout / 1000;
-    ack_timeout.tv_usec =
-        1000 * (mca_btl_usnic_component.connectivity_ack_timeout % 1000);
+    ack_timeout.tv_sec = mca_btl_usnic_component.connectivity_ack_timeout / 1000;
+    ack_timeout.tv_usec = 1000 * (mca_btl_usnic_component.connectivity_ack_timeout % 1000);
 
     /* Create lists */
     OBJ_CONSTRUCT(&udp_port_listeners, opal_list_t);
@@ -1178,8 +1123,8 @@ int opal_btl_usnic_connectivity_agent_init(void)
         /* Will not return */
     }
 
-    opal_asprintf(&ipc_filename, "%s/%s",
-             opal_process_info.job_session_dir, CONNECTIVITY_SOCK_NAME);
+    opal_asprintf(&ipc_filename, "%s/%s", opal_process_info.job_session_dir,
+                  CONNECTIVITY_SOCK_NAME);
     if (NULL == ipc_filename) {
         OPAL_ERROR_LOG(OPAL_ERR_IN_ERRNO);
         ABORT("Out of memory");
@@ -1194,8 +1139,7 @@ int opal_btl_usnic_connectivity_agent_init(void)
     address.sun_family = AF_UNIX;
     opal_string_copy(address.sun_path, ipc_filename, sizeof(address.sun_path));
 
-    if (bind(ipc_accept_fd, (struct sockaddr *) &address,
-             sizeof(struct sockaddr_un)) != 0) {
+    if (bind(ipc_accept_fd, (struct sockaddr *) &address, sizeof(struct sockaddr_un)) != 0) {
         OPAL_ERROR_LOG(OPAL_ERR_IN_ERRNO);
         ABORT("bind() failed");
         /* Will not return */
@@ -1212,14 +1156,11 @@ int opal_btl_usnic_connectivity_agent_init(void)
     }
 
     /* Add the socket to the event base */
-    opal_event_set(mca_btl_usnic_component.opal_evbase,
-                   &ipc_event, ipc_accept_fd,
-                   OPAL_EV_READ | OPAL_EV_PERSIST,
-                   agent_thread_accept, NULL);
+    opal_event_set(mca_btl_usnic_component.opal_evbase, &ipc_event, ipc_accept_fd,
+                   OPAL_EV_READ | OPAL_EV_PERSIST, agent_thread_accept, NULL);
     opal_event_add(&ipc_event, 0);
 
-    opal_output_verbose(20, USNIC_OUT,
-                        "usNIC connectivity agent initialized");
+    opal_output_verbose(20, USNIC_OUT, "usNIC connectivity agent initialized");
     agent_initialized = true;
     return OPAL_SUCCESS;
 }
@@ -1238,16 +1179,13 @@ int opal_btl_usnic_connectivity_agent_finalize(void)
        the usNIC events.  See the rationale for doing this in the
        comment in the agent_thread_finalize() function. */
     opal_event_t *ev = calloc(sizeof(*ev), 1);
-    opal_event_set(mca_btl_usnic_component.opal_evbase,
-                   ev, -1, OPAL_EV_WRITE, agent_thread_finalize, ev);
+    opal_event_set(mca_btl_usnic_component.opal_evbase, ev, -1, OPAL_EV_WRITE,
+                   agent_thread_finalize, ev);
     opal_event_active(ev, OPAL_EV_WRITE, 1);
 
     /* Wait for the event to fire and complete */
     while (agent_initialized) {
-        struct timespec tp = {
-            .tv_sec  = 0,
-            .tv_nsec = 1000
-        };
+        struct timespec tp = {.tv_sec = 0, .tv_nsec = 1000};
         nanosleep(&tp, NULL);
     }
 
@@ -1262,7 +1200,6 @@ int opal_btl_usnic_connectivity_agent_finalize(void)
         ipc_filename = NULL;
     }
 
-    opal_output_verbose(20, USNIC_OUT,
-                        "usNIC connectivity client finalized");
+    opal_output_verbose(20, USNIC_OUT, "usNIC connectivity client finalized");
     return OPAL_SUCCESS;
 }
diff --git a/opal/mca/btl/usnic/btl_usnic_cclient.c b/opal/mca/btl/usnic/btl_usnic_cclient.c
index 3de1b46763c..5f7b5534512 100644
--- a/opal/mca/btl/usnic/btl_usnic_cclient.c
+++ b/opal/mca/btl/usnic/btl_usnic_cclient.c
@@ -13,28 +13,28 @@
 #include "opal_config.h"
 
 #include <assert.h>
-#include <sys/stat.h>
+#include <string.h>
 #include <sys/socket.h>
-#include <sys/un.h>
+#include <sys/stat.h>
 #include <sys/types.h>
-#include <string.h>
+#include <sys/un.h>
 #include <unistd.h>
 #ifdef HAVE_ALLOCA_H
-#include <alloca.h>
+#    include <alloca.h>
 #endif
 #include <time.h>
 
-#include "opal_stdint.h"
 #include "opal/mca/threads/mutex.h"
 #include "opal/util/event.h"
-#include "opal/util/output.h"
 #include "opal/util/fd.h"
-#include "opal/util/string_copy.h"
+#include "opal/util/output.h"
 #include "opal/util/printf.h"
+#include "opal/util/string_copy.h"
+#include "opal_stdint.h"
 
 #include "btl_usnic.h"
-#include "btl_usnic_module.h"
 #include "btl_usnic_connectivity.h"
+#include "btl_usnic_module.h"
 
 /**************************************************************************
  * Client-side data and methods
@@ -43,7 +43,6 @@
 static bool initialized = false;
 static int agent_fd = -1;
 
-
 /*
  * Startup the agent and share our MCA param values with the it.
  */
@@ -64,8 +63,8 @@ int opal_btl_usnic_connectivity_client_init(void)
     }
 
     char *ipc_filename = NULL;
-    opal_asprintf(&ipc_filename, "%s/%s",
-             opal_process_info.job_session_dir, CONNECTIVITY_SOCK_NAME);
+    opal_asprintf(&ipc_filename, "%s/%s", opal_process_info.job_session_dir,
+                  CONNECTIVITY_SOCK_NAME);
     if (NULL == ipc_filename) {
         OPAL_ERROR_LOG(OPAL_ERR_IN_ERRNO);
         ABORT("Out of memory");
@@ -110,8 +109,7 @@ int opal_btl_usnic_connectivity_client_init(void)
 
     int count = 0;
     while (1) {
-        int ret = connect(agent_fd, (struct sockaddr*) &address,
-                          sizeof(address));
+        int ret = connect(agent_fd, (struct sockaddr *) &address, sizeof(address));
         if (0 == ret) {
             break;
         }
@@ -132,8 +130,7 @@ int opal_btl_usnic_connectivity_client_init(void)
 
     /* Send the magic token */
     int tlen = strlen(CONNECTIVITY_MAGIC_TOKEN);
-    if (OPAL_SUCCESS != opal_fd_write(agent_fd, tlen,
-                                      CONNECTIVITY_MAGIC_TOKEN)) {
+    if (OPAL_SUCCESS != opal_fd_write(agent_fd, tlen, CONNECTIVITY_MAGIC_TOKEN)) {
         OPAL_ERROR_LOG(OPAL_ERR_IN_ERRNO);
         ABORT("usnic connectivity client IPC connect write failed");
         /* Will not return */
@@ -159,12 +156,10 @@ int opal_btl_usnic_connectivity_client_init(void)
 
     /* All done */
     initialized = true;
-    opal_output_verbose(20, USNIC_OUT,
-                        "usNIC connectivity client initialized");
+    opal_output_verbose(20, USNIC_OUT, "usNIC connectivity client initialized");
     return OPAL_SUCCESS;
 }
 
-
 /*
  * Send a listen command to the agent
  */
@@ -185,12 +180,11 @@ int opal_btl_usnic_connectivity_listen(opal_btl_usnic_module_t *module)
     }
 
     /* Send the LISTEN command parameters */
-    opal_btl_usnic_connectivity_cmd_listen_t cmd = {
-        .module = NULL,
-        .ipv4_addr = module->local_modex.ipv4_addr,
-        .netmask = module->local_modex.netmask,
-        .max_msg_size = module->local_modex.max_msg_size
-    };
+    opal_btl_usnic_connectivity_cmd_listen_t cmd = {.module = NULL,
+                                                    .ipv4_addr = module->local_modex.ipv4_addr,
+                                                    .netmask = module->local_modex.netmask,
+                                                    .max_msg_size = module->local_modex
+                                                                        .max_msg_size};
     /* Only the MPI process who is also the agent will send the
        pointer value (it doesn't make sense otherwise) */
     if (0 == opal_process_info.my_local_rank) {
@@ -198,10 +192,8 @@ int opal_btl_usnic_connectivity_listen(opal_btl_usnic_module_t *module)
     }
 
     /* Ensure to NULL-terminate the passed strings */
-    opal_string_copy(cmd.nodename, opal_process_info.nodename,
-            CONNECTIVITY_NODENAME_LEN);
-    opal_string_copy(cmd.usnic_name, module->linux_device_name,
-            CONNECTIVITY_IFNAME_LEN);
+    opal_string_copy(cmd.nodename, opal_process_info.nodename, CONNECTIVITY_NODENAME_LEN);
+    opal_string_copy(cmd.usnic_name, module->linux_device_name, CONNECTIVITY_IFNAME_LEN);
 
     if (OPAL_SUCCESS != opal_fd_write(agent_fd, sizeof(cmd), &cmd)) {
         OPAL_ERROR_LOG(OPAL_ERR_IN_ERRNO);
@@ -225,11 +217,8 @@ int opal_btl_usnic_connectivity_listen(opal_btl_usnic_module_t *module)
     return OPAL_SUCCESS;
 }
 
-
-int opal_btl_usnic_connectivity_ping(uint32_t src_ipv4_addr, int src_port,
-                                     uint32_t dest_ipv4_addr,
-                                     uint32_t dest_netmask, int dest_port,
-                                     char *dest_nodename,
+int opal_btl_usnic_connectivity_ping(uint32_t src_ipv4_addr, int src_port, uint32_t dest_ipv4_addr,
+                                     uint32_t dest_netmask, int dest_port, char *dest_nodename,
                                      size_t max_msg_size)
 {
     /* If connectivity checking is not enabled, do nothing */
@@ -249,14 +238,12 @@ int opal_btl_usnic_connectivity_ping(uint32_t src_ipv4_addr, int src_port,
     }
 
     /* Send the PING command parameters */
-    opal_btl_usnic_connectivity_cmd_ping_t cmd = {
-        .src_ipv4_addr = src_ipv4_addr,
-        .src_udp_port = src_port,
-        .dest_ipv4_addr = dest_ipv4_addr,
-        .dest_netmask = dest_netmask,
-        .dest_udp_port = dest_port,
-        .max_msg_size = max_msg_size
-    };
+    opal_btl_usnic_connectivity_cmd_ping_t cmd = {.src_ipv4_addr = src_ipv4_addr,
+                                                  .src_udp_port = src_port,
+                                                  .dest_ipv4_addr = dest_ipv4_addr,
+                                                  .dest_netmask = dest_netmask,
+                                                  .dest_udp_port = dest_port,
+                                                  .max_msg_size = max_msg_size};
     /* Ensure to NULL-terminate the passed string */
     opal_string_copy(cmd.dest_nodename, dest_nodename, CONNECTIVITY_NODENAME_LEN);
 
@@ -272,7 +259,6 @@ int opal_btl_usnic_connectivity_ping(uint32_t src_ipv4_addr, int src_port,
     return OPAL_SUCCESS;
 }
 
-
 /*
  * Send an unlisten command to the agent
  */
@@ -310,7 +296,6 @@ int opal_btl_usnic_connectivity_unlisten(opal_btl_usnic_module_t *module)
     return OPAL_SUCCESS;
 }
 
-
 /*
  * Shut down the connectivity client
  */
diff --git a/opal/mca/btl/usnic/btl_usnic_compat.c b/opal/mca/btl/usnic/btl_usnic_compat.c
index b1dc6900c81..c4ed935c343 100644
--- a/opal/mca/btl/usnic/btl_usnic_compat.c
+++ b/opal/mca/btl/usnic/btl_usnic_compat.c
@@ -15,30 +15,23 @@
 #include "opal_stdint.h"
 
 #include "btl_usnic_compat.h"
-#include "btl_usnic_frag.h"
-#include "btl_usnic_endpoint.h"
 #include "btl_usnic_connectivity.h"
+#include "btl_usnic_endpoint.h"
+#include "btl_usnic_frag.h"
 #include "btl_usnic_send.h"
 
 #include "opal/util/proc.h"
 
-void usnic_compat_modex_send(int *rc,
-                             mca_base_component_t *component,
-                             opal_btl_usnic_modex_t *modexes,
-                             size_t size)
+void usnic_compat_modex_send(int *rc, mca_base_component_t *component,
+                             opal_btl_usnic_modex_t *modexes, size_t size)
 {
-    OPAL_MODEX_SEND(*rc, PMIX_REMOTE, component,
-                    modexes, size);
+    OPAL_MODEX_SEND(*rc, PMIX_REMOTE, component, modexes, size);
 }
 
-void usnic_compat_modex_recv(int *rc,
-                             mca_base_component_t *component,
-                             opal_proc_t *proc,
-                             opal_btl_usnic_modex_t **modexes,
-                             size_t *size)
+void usnic_compat_modex_recv(int *rc, mca_base_component_t *component, opal_proc_t *proc,
+                             opal_btl_usnic_modex_t **modexes, size_t *size)
 {
-    OPAL_MODEX_RECV(*rc, component, &proc->proc_name,
-                    (uint8_t**) modexes, size);
+    OPAL_MODEX_RECV(*rc, component, &proc->proc_name, (uint8_t **) modexes, size);
 }
 
 uint64_t usnic_compat_rte_hash_name(opal_process_name_t *pname)
@@ -61,14 +54,9 @@ const char *usnic_compat_proc_name_print(opal_process_name_t *pname)
  * requested if the given convertor cannot process the entire (*size).
  */
 static inline opal_btl_usnic_send_frag_t *
-prepare_src_small(
-    struct opal_btl_usnic_module_t* module,
-    struct mca_btl_base_endpoint_t* endpoint,
-    struct opal_convertor_t* convertor,
-    uint8_t order,
-    size_t reserve,
-    size_t* size,
-    uint32_t flags)
+prepare_src_small(struct opal_btl_usnic_module_t *module, struct mca_btl_base_endpoint_t *endpoint,
+                  struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size,
+                  uint32_t flags)
 {
     opal_btl_usnic_send_frag_t *frag;
     opal_btl_usnic_small_send_frag_t *sfrag;
@@ -103,14 +91,13 @@ prepare_src_small(
      * we might still use INLINE for the send, and in that case we do not want
      * to copy the data at all.
      */
-    if (OPAL_UNLIKELY(opal_convertor_need_buffers(convertor))) {
+    if (OPAL_UNLIKELY(opal_convertor_need_buffers(convertor) || opal_convertor_on_device(convertor))) {
         /* put user data just after end of 1st seg (upper layer header) */
         assert(payload_len <= module->max_frag_payload);
-        usnic_convertor_pack_simple(
-                convertor,
-                (IOVBASE_TYPE*)(intptr_t)(frag->sf_base.uf_local_seg[0].seg_addr.lval + reserve),
-                *size,
-                size);
+        usnic_convertor_pack_simple(convertor,
+                                    (IOVBASE_TYPE *) (intptr_t)(
+                                        frag->sf_base.uf_local_seg[0].seg_addr.lval + reserve),
+                                    *size, size);
         payload_len = reserve + *size;
         frag->sf_base.uf_base.USNIC_SEND_LOCAL_COUNT = 1;
         /* PML will copy header into beginning of segment */
@@ -129,14 +116,11 @@ prepare_src_small(
     return frag;
 }
 
-static void *
-pack_chunk_seg_chain_with_reserve(
-    struct opal_btl_usnic_module_t* module,
-    opal_btl_usnic_large_send_frag_t *lfrag,
-    size_t reserve_len,
-    opal_convertor_t *convertor,
-    size_t max_convertor_bytes,
-    size_t *convertor_bytes_packed)
+static void *pack_chunk_seg_chain_with_reserve(struct opal_btl_usnic_module_t *module,
+                                               opal_btl_usnic_large_send_frag_t *lfrag,
+                                               size_t reserve_len, opal_convertor_t *convertor,
+                                               size_t max_convertor_bytes,
+                                               size_t *convertor_bytes_packed)
 {
     opal_btl_usnic_chunk_segment_t *seg;
     void *ret_ptr = NULL;
@@ -154,12 +138,10 @@ pack_chunk_seg_chain_with_reserve(
     *convertor_bytes_packed = 0;
 
     first_pass = true;
-    while (*convertor_bytes_packed < max_convertor_bytes ||
-           first_pass) {
+    while (*convertor_bytes_packed < max_convertor_bytes || first_pass) {
         seg = opal_btl_usnic_chunk_segment_alloc(module);
         if (OPAL_UNLIKELY(NULL == seg)) {
-            opal_btl_usnic_util_abort("chunk segment allocation error",
-                                      __FILE__, __LINE__);
+            opal_btl_usnic_util_abort("chunk segment allocation error", __FILE__, __LINE__);
         }
         ++n_segs;
 
@@ -185,8 +167,8 @@ pack_chunk_seg_chain_with_reserve(
             *convertor_bytes_packed += max_data;
 
             /* If unable to pack any of the remaining bytes, release the
-            * most recently allocated segment and finish processing.
-            */
+             * most recently allocated segment and finish processing.
+             */
             if (seg_space == module->max_chunk_payload) {
                 assert(max_data == 0); /* only way this can happen */
                 opal_btl_usnic_chunk_segment_return(module, seg);
@@ -204,9 +186,8 @@ pack_chunk_seg_chain_with_reserve(
         opal_list_append(&lfrag->lsf_seg_chain, &seg->ss_base.us_list.super);
 
 #if MSGDEBUG1
-        opal_output(0, "%s: appending seg=%p, frag=%p, payload=%zd\n",
-                    __func__, (void *)seg, (void *)lfrag,
-                    (module->max_chunk_payload - seg_space));
+        opal_output(0, "%s: appending seg=%p, frag=%p, payload=%zd\n", __func__, (void *) seg,
+                    (void *) lfrag, (module->max_chunk_payload - seg_space));
 #endif
 
         first_pass = false;
@@ -219,15 +200,11 @@ pack_chunk_seg_chain_with_reserve(
  * in the same manner as btl_prepare_src.  Must return a smaller amount than
  * requested if the given convertor cannot process the entire (*size).
  */
-static opal_btl_usnic_send_frag_t *
-prepare_src_large(
-    struct opal_btl_usnic_module_t* module,
-    struct mca_btl_base_endpoint_t* endpoint,
-    struct opal_convertor_t* convertor,
-    uint8_t order,
-    size_t reserve,
-    size_t* size,
-    uint32_t flags)
+static opal_btl_usnic_send_frag_t *prepare_src_large(struct opal_btl_usnic_module_t *module,
+                                                     struct mca_btl_base_endpoint_t *endpoint,
+                                                     struct opal_convertor_t *convertor,
+                                                     uint8_t order, size_t reserve, size_t *size,
+                                                     uint32_t flags)
 {
     opal_btl_usnic_send_frag_t *frag;
     opal_btl_usnic_large_send_frag_t *lfrag;
@@ -250,11 +227,11 @@ prepare_src_large(
     /* make sure upper header small enough */
     assert(reserve <= sizeof(lfrag->lsf_ompi_header));
 
-    if (OPAL_UNLIKELY(opal_convertor_need_buffers(convertor))) {
+    if (OPAL_UNLIKELY(opal_convertor_need_buffers(convertor) || opal_convertor_on_device(convertor))) {
         /* threshold == -1 means always pack eagerly */
-        if (mca_btl_usnic_component.pack_lazy_threshold >= 0 &&
-            *size >= (size_t)mca_btl_usnic_component.pack_lazy_threshold) {
-            MSGDEBUG1_OUT("packing frag %p on the fly", (void *)frag);
+        if (mca_btl_usnic_component.pack_lazy_threshold >= 0
+            && *size >= (size_t) mca_btl_usnic_component.pack_lazy_threshold) {
+            MSGDEBUG1_OUT("packing frag %p on the fly", (void *) frag);
             lfrag->lsf_pack_on_the_fly = true;
 
             /* tell the PML we will absorb as much as possible while still
@@ -266,16 +243,13 @@ prepare_src_large(
             rc = opal_convertor_clone(convertor, &frag->sf_convertor,
                                       /*copy_stack=*/true);
             if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-                opal_btl_usnic_util_abort("unexpected convertor clone error",
-                                          __FILE__, __LINE__);
+                opal_btl_usnic_util_abort("unexpected convertor clone error", __FILE__, __LINE__);
             }
-        }
-        else {
+        } else {
             /* pack everything in the convertor into a chain of segments now,
              * leaving space for the PML header in the first segment */
-            lfrag->lsf_base.sf_base.uf_local_seg[0].seg_addr.pval =
-                pack_chunk_seg_chain_with_reserve(module, lfrag, reserve,
-                                                  convertor, *size, size);
+            lfrag->lsf_base.sf_base.uf_local_seg[0].seg_addr.pval
+                = pack_chunk_seg_chain_with_reserve(module, lfrag, reserve, convertor, *size, size);
         }
 
         /* We set SG[1] to {NULL,bytes_packed} so that various calculations
@@ -287,8 +261,7 @@ prepare_src_large(
     } else {
         /* convertor not needed, just save the payload pointer in SG[1] */
         lfrag->lsf_pack_on_the_fly = true;
-        opal_convertor_get_current_pointer(convertor,
-                                           &frag->sf_base.uf_local_seg[1].seg_addr.pval);
+        opal_convertor_get_current_pointer(convertor, &frag->sf_base.uf_local_seg[1].seg_addr.pval);
         frag->sf_base.uf_local_seg[1].seg_len = *size;
     }
 
@@ -327,16 +300,11 @@ prepare_src_large(
  * NOTE that the *only* reason this routine is allowed to return a size smaller
  * than was requested is if the convertor cannot process the entire amount.
  */
-struct mca_btl_base_descriptor_t *
-opal_btl_usnic_prepare_src(struct mca_btl_base_module_t *base_module,
-                           struct mca_btl_base_endpoint_t *endpoint,
-                           struct opal_convertor_t *convertor,
-                           uint8_t order,
-                           size_t reserve,
-                           size_t *size,
-                           uint32_t flags)
+struct mca_btl_base_descriptor_t *opal_btl_usnic_prepare_src(
+    struct mca_btl_base_module_t *base_module, struct mca_btl_base_endpoint_t *endpoint,
+    struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size, uint32_t flags)
 {
-    opal_btl_usnic_module_t *module = (opal_btl_usnic_module_t*) base_module;
+    opal_btl_usnic_module_t *module = (opal_btl_usnic_module_t *) base_module;
     opal_btl_usnic_send_frag_t *frag;
     uint32_t payload_len;
 #if MSGDEBUG2
@@ -353,30 +321,27 @@ opal_btl_usnic_prepare_src(struct mca_btl_base_module_t *base_module,
      */
     payload_len = *size + reserve;
     if (payload_len <= module->max_frag_payload) {
-        frag = prepare_src_small(module, endpoint, convertor,
-                                 order, reserve, size, flags);
+        frag = prepare_src_small(module, endpoint, convertor, order, reserve, size, flags);
     } else {
-        frag = prepare_src_large(module, endpoint, convertor,
-                                 order, reserve, size, flags);
+        frag = prepare_src_large(module, endpoint, convertor, order, reserve, size, flags);
     }
 
 #if MSGDEBUG2
     opal_output(0, "prep_src: %s %s frag %p, size=%d+%u (was %u), conv=%p\n",
                 module->linux_device_name,
-                (reserve + *size) <= module->max_frag_payload?"small":"large",
-                (void *)frag, (int)reserve, (unsigned)*size, (unsigned)osize,
-                (void *)convertor);
-#if MSGDEBUG1
+                (reserve + *size) <= module->max_frag_payload ? "small" : "large", (void *) frag,
+                (int) reserve, (unsigned) *size, (unsigned) osize, (void *) convertor);
+#    if MSGDEBUG1
     {
         unsigned i;
         mca_btl_base_descriptor_t *desc = &frag->sf_base.uf_base;
-        for (i=0; i<desc->USNIC_SEND_LOCAL_COUNT; ++i) {
+        for (i = 0; i < desc->USNIC_SEND_LOCAL_COUNT; ++i) {
             opal_output(0, "  %d: ptr:%p len:%d\n", i,
-                        (void *)desc->USNIC_SEND_LOCAL[i].seg_addr.pval,
+                        (void *) desc->USNIC_SEND_LOCAL[i].seg_addr.pval,
                         desc->USNIC_SEND_LOCAL[i].seg_len);
         }
     }
-#endif
+#    endif
 #endif
 
     return &frag->sf_base.uf_base;
@@ -391,18 +356,16 @@ opal_btl_usnic_prepare_src(struct mca_btl_base_module_t *base_module,
  * Note that this function is only ever called with contiguous
  * buffers, so a convertor is not necessary.
  */
-int
-opal_btl_usnic_put(struct mca_btl_base_module_t *base_module,
-                   struct mca_btl_base_endpoint_t *endpoint,
-                   void *local_address, uint64_t remote_address,
-                   struct mca_btl_base_registration_handle_t *local_handle,
-                   struct mca_btl_base_registration_handle_t *remote_handle,
-                   size_t size, int flags, int order,
-                   mca_btl_base_rdma_completion_fn_t cbfunc,
-                   void *cbcontext, void *cbdata)
+int opal_btl_usnic_put(struct mca_btl_base_module_t *base_module,
+                       struct mca_btl_base_endpoint_t *endpoint, void *local_address,
+                       uint64_t remote_address,
+                       struct mca_btl_base_registration_handle_t *local_handle,
+                       struct mca_btl_base_registration_handle_t *remote_handle, size_t size,
+                       int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
+                       void *cbcontext, void *cbdata)
 {
     opal_btl_usnic_send_frag_t *sfrag;
-    opal_btl_usnic_module_t *module = (opal_btl_usnic_module_t*) base_module;
+    opal_btl_usnic_module_t *module = (opal_btl_usnic_module_t *) base_module;
 
     /* At least for the moment, continue to make a descriptor, like we
        used to in BTL 2.0 */
@@ -439,8 +402,7 @@ opal_btl_usnic_put(struct mca_btl_base_module_t *base_module,
     frag->uf_local_seg[0].seg_len = size;
     frag->uf_local_seg[0].seg_addr.pval = local_address;
     frag->uf_remote_seg[0].seg_len = size;
-    frag->uf_remote_seg[0].seg_addr.pval =
-        (void *)(uintptr_t) remote_address;
+    frag->uf_remote_seg[0].seg_addr.pval = (void *) (uintptr_t) remote_address;
 
     mca_btl_base_descriptor_t *desc;
     desc = &frag->uf_base;
@@ -455,9 +417,7 @@ opal_btl_usnic_put(struct mca_btl_base_module_t *base_module,
     desc->order = order;
 
     int rc;
-    rc = opal_btl_usnic_finish_put_or_send(module,
-                                           (opal_btl_usnic_endpoint_t *)endpoint,
-                                           sfrag,
+    rc = opal_btl_usnic_finish_put_or_send(module, (opal_btl_usnic_endpoint_t *) endpoint, sfrag,
                                            /*tag=*/MCA_BTL_NO_ORDER);
     return rc;
 }
diff --git a/opal/mca/btl/usnic/btl_usnic_compat.h b/opal/mca/btl/usnic/btl_usnic_compat.h
index f2a6a486a96..1a0f5b14e9d 100644
--- a/opal/mca/btl/usnic/btl_usnic_compat.h
+++ b/opal/mca/btl/usnic/btl_usnic_compat.h
@@ -17,8 +17,8 @@
 #ifndef BTL_USNIC_COMPAT_H
 #define BTL_USNIC_COMPAT_H
 
-#include "opal/mca/rcache/rcache.h"
 #include "opal/mca/btl/btl.h"
+#include "opal/mca/rcache/rcache.h"
 
 /************************************************************************/
 
@@ -34,14 +34,14 @@
 /* Free lists are unified into OPAL free lists */
 #include "opal/class/opal_free_list.h"
 
-/* Inclue the progress thread stuff */
+/* Include the progress thread stuff */
 #include "opal/runtime/opal_progress_threads.h"
 
 #define USNIC_OUT opal_btl_base_framework.framework_output
 /* JMS Really want to be able to get the job size somehow...  But for
    now, so that we can compile, just set it to a constant :-( */
-#define USNIC_MCW_SIZE 2
-#define proc_bound() (NULL != opal_process_info.cpuset ? 1 : 0)
+#define USNIC_MCW_SIZE                  2
+#define proc_bound()                    (NULL != opal_process_info.cpuset ? 1 : 0)
 #define USNIC_BTL_DEFAULT_VERSION(name) MCA_BTL_DEFAULT_VERSION(name)
 
 #define USNIC_SEND_LOCAL        des_segments
@@ -54,24 +54,20 @@
 #define USNIC_RECV_REMOTE       des_segments
 #define USNIC_RECV_REMOTE_COUNT des_segment_count
 
-#define USNIC_PUT_LOCAL         des_segments
-#define USNIC_PUT_LOCAL_COUNT   des_segment_count
-#define USNIC_PUT_REMOTE        des_segments
-#define USNIC_PUT_REMOTE_COUNT  des_segments_count
+#define USNIC_PUT_LOCAL        des_segments
+#define USNIC_PUT_LOCAL_COUNT  des_segment_count
+#define USNIC_PUT_REMOTE       des_segments
+#define USNIC_PUT_REMOTE_COUNT des_segments_count
 
-#define USNIC_COMPAT_FREE_LIST_GET(list, item) \
-    (item) = opal_free_list_get((list))
-#define USNIC_COMPAT_FREE_LIST_RETURN(list, item) \
-    opal_free_list_return((list), (item))
+#define USNIC_COMPAT_FREE_LIST_GET(list, item)    (item) = opal_free_list_get((list))
+#define USNIC_COMPAT_FREE_LIST_RETURN(list, item) opal_free_list_return((list), (item))
 
 #define usnic_compat_free_list_init opal_free_list_init
 
 /*
  * Performance critical; needs to be inline
  */
-static inline int
-usnic_compat_proc_name_compare(opal_process_name_t a,
-                               opal_process_name_t b)
+static inline int usnic_compat_proc_name_compare(opal_process_name_t a, opal_process_name_t b)
 {
     return (bool) (a.jobid == b.jobid && a.vpid == b.vpid);
 }
@@ -81,16 +77,11 @@ usnic_compat_proc_name_compare(opal_process_name_t a,
 /* Forward declare to avoid #include ordering complications */
 struct opal_btl_usnic_modex_t;
 
-void usnic_compat_modex_send(int *rc,
-                             mca_base_component_t *component,
-                             struct opal_btl_usnic_modex_t *modexes,
-                             size_t size);
+void usnic_compat_modex_send(int *rc, mca_base_component_t *component,
+                             struct opal_btl_usnic_modex_t *modexes, size_t size);
 
-void usnic_compat_modex_recv(int *rc,
-                             mca_base_component_t *component,
-                             opal_proc_t *proc,
-                             struct opal_btl_usnic_modex_t **modexes,
-                             size_t *size);
+void usnic_compat_modex_recv(int *rc, mca_base_component_t *component, opal_proc_t *proc,
+                             struct opal_btl_usnic_modex_t **modexes, size_t *size);
 
 uint64_t usnic_compat_rte_hash_name(opal_process_name_t *pname);
 const char *usnic_compat_proc_name_print(opal_process_name_t *pname);
@@ -103,20 +94,15 @@ struct mca_btl_base_endpoint_t;
 struct mca_btl_base_descriptor_t *
 opal_btl_usnic_prepare_src(struct mca_btl_base_module_t *base_module,
                            struct mca_btl_base_endpoint_t *endpoint,
-                           struct opal_convertor_t *convertor,
-                           uint8_t order,
-                           size_t reserve,
-                           size_t *size,
-                           uint32_t flags);
-
-int
-opal_btl_usnic_put(struct mca_btl_base_module_t *base_module,
-                   struct mca_btl_base_endpoint_t *endpoint,
-                   void *local_address, uint64_t remote_address,
-                   struct mca_btl_base_registration_handle_t *local_handle,
-                   struct mca_btl_base_registration_handle_t *remote_handle,
-                   size_t size, int flags, int order,
-                   mca_btl_base_rdma_completion_fn_t cbfunc,
-                   void *cbcontext, void *cbdata);
+                           struct opal_convertor_t *convertor, uint8_t order, size_t reserve,
+                           size_t *size, uint32_t flags);
+
+int opal_btl_usnic_put(struct mca_btl_base_module_t *base_module,
+                       struct mca_btl_base_endpoint_t *endpoint, void *local_address,
+                       uint64_t remote_address,
+                       struct mca_btl_base_registration_handle_t *local_handle,
+                       struct mca_btl_base_registration_handle_t *remote_handle, size_t size,
+                       int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
+                       void *cbcontext, void *cbdata);
 
 #endif /* BTL_USNIC_COMPAT_H */
diff --git a/opal/mca/btl/usnic/btl_usnic_component.c b/opal/mca/btl/usnic/btl_usnic_component.c
index ea796e8e3bb..84f96938be9 100644
--- a/opal/mca/btl/usnic/btl_usnic_component.c
+++ b/opal/mca/btl/usnic/btl_usnic_component.c
@@ -38,52 +38,52 @@
 
 #include "opal_config.h"
 
-#include <string.h>
 #include <ctype.h>
 #include <errno.h>
-#include <unistd.h>
+#include <fcntl.h>
 #include <stdlib.h>
-#include <sys/time.h>
+#include <string.h>
 #include <sys/resource.h>
-#include <sys/types.h>
 #include <sys/stat.h>
-#include <fcntl.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <unistd.h>
 
 #include <rdma/fabric.h>
 
-#include "opal_stdint.h"
-#include "opal/prefetch.h"
+#include "opal/constants.h"
+#include "opal/mca/base/mca_base_var.h"
+#include "opal/mca/memchecker/base/base.h"
 #include "opal/mca/timer/base/base.h"
+#include "opal/prefetch.h"
 #include "opal/util/argv.h"
-#include "opal/util/net.h"
 #include "opal/util/if.h"
+#include "opal/util/net.h"
 #include "opal/util/printf.h"
-#include "opal/mca/base/mca_base_var.h"
-#include "opal/mca/memchecker/base/base.h"
 #include "opal/util/show_help.h"
-#include "opal/constants.h"
+#include "opal_stdint.h"
 
-#include "opal/mca/btl/btl.h"
 #include "opal/mca/btl/base/base.h"
+#include "opal/mca/btl/btl.h"
 #include "opal/util/proc.h"
 
 #include "btl_usnic.h"
+#include "btl_usnic_ack.h"
 #include "btl_usnic_connectivity.h"
-#include "btl_usnic_frag.h"
 #include "btl_usnic_endpoint.h"
+#include "btl_usnic_frag.h"
 #include "btl_usnic_module.h"
-#include "btl_usnic_stats.h"
-#include "btl_usnic_util.h"
-#include "btl_usnic_ack.h"
-#include "btl_usnic_send.h"
-#include "btl_usnic_recv.h"
 #include "btl_usnic_proc.h"
+#include "btl_usnic_recv.h"
+#include "btl_usnic_send.h"
+#include "btl_usnic_stats.h"
 #include "btl_usnic_test.h"
+#include "btl_usnic_util.h"
 
 #define OPAL_BTL_USNIC_NUM_COMPLETIONS 500
 
 /* MPI_THREAD_MULTIPLE_SUPPORT */
-opal_recursive_mutex_t btl_usnic_lock =  OPAL_RECURSIVE_MUTEX_STATIC_INIT;
+opal_recursive_mutex_t btl_usnic_lock;  /* recursive mutexes must be dynamically initialized */
 
 /* RNG buffer definition */
 opal_rng_buff_t opal_btl_usnic_rand_buff = {{0}};
@@ -102,8 +102,7 @@ static volatile bool dump_bitvectors = false;
 static int usnic_component_open(void);
 static int usnic_component_close(void);
 static mca_btl_base_module_t **
-usnic_component_init(int* num_btl_modules, bool want_progress_threads,
-                       bool want_mpi_threads);
+usnic_component_init(int *num_btl_modules, bool want_progress_threads, bool want_mpi_threads);
 static int usnic_component_progress(void);
 
 /* Types for filtering interfaces */
@@ -123,34 +122,29 @@ typedef struct usnic_if_filter_t {
     filter_elt_t *elts;
 } usnic_if_filter_t;
 
-static bool filter_module(opal_btl_usnic_module_t *module,
-                          usnic_if_filter_t *filter,
+static bool filter_module(opal_btl_usnic_module_t *module, usnic_if_filter_t *filter,
                           bool filter_incl);
-static usnic_if_filter_t *parse_ifex_str(const char *orig_str,
-                                         const char *name);
+static usnic_if_filter_t *parse_ifex_str(const char *orig_str, const char *name);
 static void free_filter(usnic_if_filter_t *filter);
 
-
 opal_btl_usnic_component_t mca_btl_usnic_component = {
     .super = {
         /* First, the mca_base_component_t struct containing meta information
            about the component itself */
-        .btl_version = {
-            USNIC_BTL_DEFAULT_VERSION("usnic"),
-            .mca_open_component = usnic_component_open,
-            .mca_close_component = usnic_component_close,
-            .mca_register_component_params = opal_btl_usnic_component_register,
-        },
-        .btl_data = {
-            /* The component is not checkpoint ready */
-            .param_field = MCA_BASE_METADATA_PARAM_NONE
-        },
+        .btl_version =
+            {
+                USNIC_BTL_DEFAULT_VERSION("usnic"),
+                .mca_open_component = usnic_component_open,
+                .mca_close_component = usnic_component_close,
+                .mca_register_component_params = opal_btl_usnic_component_register,
+            },
+        .btl_data =
+            {/* The component is not checkpoint ready */
+             .param_field = MCA_BASE_METADATA_PARAM_NONE},
 
         .btl_init = usnic_component_init,
         .btl_progress = usnic_component_progress,
-    }
-};
-
+    }};
 
 /*
  *  Called by MCA framework to open the component
@@ -166,17 +160,17 @@ static int usnic_component_open(void)
 
     /* initialize objects */
     OBJ_CONSTRUCT(&mca_btl_usnic_component.usnic_procs, opal_list_t);
+    OBJ_CONSTRUCT(&btl_usnic_lock, opal_recursive_mutex_t);
+
 
     /* Sanity check: if_include and if_exclude need to be mutually
        exclusive */
-    if (OPAL_SUCCESS !=
-        mca_base_var_check_exclusive("opal",
-            mca_btl_usnic_component.super.btl_version.mca_type_name,
-            mca_btl_usnic_component.super.btl_version.mca_component_name,
-            "if_include",
+    if (OPAL_SUCCESS
+        != mca_base_var_check_exclusive(
+            "opal", mca_btl_usnic_component.super.btl_version.mca_type_name,
+            mca_btl_usnic_component.super.btl_version.mca_component_name, "if_include",
             mca_btl_usnic_component.super.btl_version.mca_type_name,
-            mca_btl_usnic_component.super.btl_version.mca_component_name,
-            "if_exclude")) {
+            mca_btl_usnic_component.super.btl_version.mca_component_name, "if_exclude")) {
         /* Return ERR_NOT_AVAILABLE so that a warning message about
            "open" failing is not printed */
         return OPAL_ERR_NOT_AVAILABLE;
@@ -185,7 +179,6 @@ static int usnic_component_open(void)
     return OPAL_SUCCESS;
 }
 
-
 /*
  * Component cleanup
  */
@@ -226,7 +219,6 @@ static int usnic_component_close(void)
     return OPAL_SUCCESS;
 }
 
-
 /*
  * Register address information.  The modex will make this available
  * to all peers.
@@ -236,22 +228,20 @@ static int usnic_modex_send(void)
     int rc;
     int i;
     size_t size;
-    opal_btl_usnic_modex_t* modexes = NULL;
+    opal_btl_usnic_modex_t *modexes = NULL;
 
     if (0 == mca_btl_usnic_component.num_modules) {
         return OPAL_SUCCESS;
     }
 
-    size = mca_btl_usnic_component.num_modules *
-        sizeof(opal_btl_usnic_modex_t);
-    modexes = (opal_btl_usnic_modex_t*) malloc(size);
+    size = mca_btl_usnic_component.num_modules * sizeof(opal_btl_usnic_modex_t);
+    modexes = (opal_btl_usnic_modex_t *) malloc(size);
     if (NULL == modexes) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
     for (i = 0; i < mca_btl_usnic_component.num_modules; i++) {
-        opal_btl_usnic_module_t* module =
-            mca_btl_usnic_component.usnic_active_modules[i];
+        opal_btl_usnic_module_t *module = mca_btl_usnic_component.usnic_active_modules[i];
         modexes[i] = module->local_modex;
         opal_output_verbose(5, USNIC_OUT,
                             "btl:usnic: "
@@ -259,21 +249,18 @@ static int usnic_modex_send(void)
                             "modex_send data port:%d, "
                             "%s",
                             modexes[i].ports[USNIC_PRIORITY_CHANNEL],
-                            modexes[i].ports[USNIC_DATA_CHANNEL],
-                            module->if_ipv4_addr_str);
+                            modexes[i].ports[USNIC_DATA_CHANNEL], module->if_ipv4_addr_str);
     }
 
-    usnic_compat_modex_send(&rc, &mca_btl_usnic_component.super.btl_version,
-                            modexes, size);
+    usnic_compat_modex_send(&rc, &mca_btl_usnic_component.super.btl_version, modexes, size);
     free(modexes);
 
     return rc;
 }
 
-
 /*
  * See if our memlock limit is >64K.  64K is the RHEL default memlock
- * limit; this check is a first-line-of-defense hueristic to see if
+ * limit; this check is a first-line-of-defense heuristic to see if
  * the user has set the memlock limit to *something*.
  *
  * We have other checks elsewhere (e.g., to ensure that QPs are able
@@ -290,35 +277,30 @@ static int check_reg_mem_basics(void)
 
     ret = getrlimit(RLIMIT_MEMLOCK, &limit);
     if (0 == ret) {
-        if ((long) limit.rlim_cur > (64 * 1024) ||
-            limit.rlim_cur == RLIM_INFINITY) {
+        if ((long) limit.rlim_cur > (64 * 1024) || limit.rlim_cur == RLIM_INFINITY) {
             return OPAL_SUCCESS;
         } else {
-            opal_asprintf(&str_limit, "%ld", (long)limit.rlim_cur);
+            opal_asprintf(&str_limit, "%ld", (long) limit.rlim_cur);
         }
     } else {
         opal_asprintf(&str_limit, "Unknown");
     }
 
-    opal_show_help("help-mpi-btl-usnic.txt", "check_reg_mem_basics fail",
-                   true,
-                   opal_process_info.nodename,
-                   str_limit);
+    opal_show_help("help-mpi-btl-usnic.txt", "check_reg_mem_basics fail", true,
+                   opal_process_info.nodename, str_limit);
 
     return OPAL_ERR_OUT_OF_RESOURCE;
 #else
     /* If we don't have RLIMIT_MEMLOCK, then just bypass this
-       safety/hueristic check. */
+       safety/heuristic check. */
     return OPAL_SUCCESS;
 #endif
 }
 
-
 /*
  * Basic sanity checking for usNIC VFs / resources.
  */
-static int check_usnic_config(opal_btl_usnic_module_t *module,
-        int num_local_procs)
+static int check_usnic_config(opal_btl_usnic_module_t *module, int num_local_procs)
 {
     char str[128];
     unsigned unlp;
@@ -344,40 +326,33 @@ static int check_usnic_config(opal_btl_usnic_module_t *module,
           (to ensure that each MPI process will be able to get the
           number of CQs that it needs) */
     if (uip->ui.v1.ui_num_vf < unlp) {
-        snprintf(str, sizeof(str), "Not enough usNICs (found %d, need %d)",
-                 uip->ui.v1.ui_num_vf, unlp);
+        snprintf(str, sizeof(str), "Not enough usNICs (found %d, need %d)", uip->ui.v1.ui_num_vf,
+                 unlp);
         goto error;
     }
 
     if (uip->ui.v1.ui_qp_per_vf < USNIC_NUM_CHANNELS) {
-        snprintf(str, sizeof(str), "Not enough transmit/receive queues per usNIC (found %d, need %d)",
-                 uip->ui.v1.ui_qp_per_vf,
-                 USNIC_NUM_CHANNELS);
+        snprintf(str, sizeof(str),
+                 "Not enough transmit/receive queues per usNIC (found %d, need %d)",
+                 uip->ui.v1.ui_qp_per_vf, USNIC_NUM_CHANNELS);
         goto error;
     }
     if (uip->ui.v1.ui_cq_per_vf < USNIC_NUM_CHANNELS) {
-        snprintf(str, sizeof(str),
-                 "Not enough completion queues per usNIC (found %d, need %d)",
-                 uip->ui.v1.ui_cq_per_vf,
-                 USNIC_NUM_CHANNELS);
+        snprintf(str, sizeof(str), "Not enough completion queues per usNIC (found %d, need %d)",
+                 uip->ui.v1.ui_cq_per_vf, USNIC_NUM_CHANNELS);
         goto error;
     }
 
     /* All is good! */
     return OPAL_SUCCESS;
 
- error:
+error:
     /* Sad panda */
-    opal_show_help("help-mpi-btl-usnic.txt",
-                   "not enough usnic resources",
-                   true,
-                   opal_process_info.nodename,
-                   module->linux_device_name,
-                   str);
+    opal_show_help("help-mpi-btl-usnic.txt", "not enough usnic resources", true,
+                   opal_process_info.nodename, module->linux_device_name, str);
     return OPAL_ERROR;
 }
 
-
 static void usnic_clock_callback(int fd, short flags, void *timeout)
 {
     /* Increase by so many ticks that we will definitely force sending
@@ -390,7 +365,6 @@ static void usnic_clock_callback(int fd, short flags, void *timeout)
     opal_event_add(&usnic_clock_timer_event, timeout);
 }
 
-
 /* Parse a string which is a comma-separated list containing a mix of
  * interface names and IPv4 CIDR-format netmasks.
  *
@@ -402,8 +376,7 @@ static void usnic_clock_callback(int fd, short flags, void *timeout)
  * of filter elements, and any strings in it (can do this via
  * free_filter()).
  */
-static usnic_if_filter_t *parse_ifex_str(const char *orig_str,
-                                         const char *name)
+static usnic_if_filter_t *parse_ifex_str(const char *orig_str, const char *name)
 {
     int i, ret;
     char **argv, *str, *tmp;
@@ -450,8 +423,8 @@ static usnic_if_filter_t *parse_ifex_str(const char *orig_str,
             filter->elts[filter->n_elt].is_netmask = false;
             filter->elts[filter->n_elt].if_name = strdup(argv[i]);
             opal_output_verbose(20, USNIC_OUT,
-                                "btl:usnic:parse_ifex_str: parsed %s device name: %s",
-                                name, filter->elts[filter->n_elt].if_name);
+                                "btl:usnic:parse_ifex_str: parsed %s device name: %s", name,
+                                filter->elts[filter->n_elt].if_name);
 
             ++filter->n_elt;
             continue;
@@ -463,51 +436,44 @@ static usnic_if_filter_t *parse_ifex_str(const char *orig_str,
         tmp = strdup(argv[i]);
         str = strchr(argv[i], '/');
         if (NULL == str) {
-            opal_show_help("help-mpi-btl-usnic.txt", "invalid if_inexclude",
-                           true, name, opal_process_info.nodename,
-                           tmp, "Invalid specification (missing \"/\")");
+            opal_show_help("help-mpi-btl-usnic.txt", "invalid if_inexclude", true, name,
+                           opal_process_info.nodename, tmp,
+                           "Invalid specification (missing \"/\")");
             free(tmp);
             continue;
         }
         *str = '\0';
         argv_prefix = atoi(str + 1);
         if (argv_prefix < 1 || argv_prefix > 32) {
-            opal_show_help("help-mpi-btl-usnic.txt", "invalid if_inexclude",
-                           true, name, opal_process_info.nodename,
-                           tmp, "Invalid specification (prefix < 1 or prefix >32)");
+            opal_show_help("help-mpi-btl-usnic.txt", "invalid if_inexclude", true, name,
+                           opal_process_info.nodename, tmp,
+                           "Invalid specification (prefix < 1 or prefix >32)");
             free(tmp);
             continue;
         }
 
         /* Now convert the IPv4 address */
-        ((struct sockaddr*) &argv_inaddr)->sa_family = AF_INET;
-        ret = inet_pton(AF_INET, argv[i],
-                        &((struct sockaddr_in*) &argv_inaddr)->sin_addr);
+        ((struct sockaddr *) &argv_inaddr)->sa_family = AF_INET;
+        ret = inet_pton(AF_INET, argv[i], &((struct sockaddr_in *) &argv_inaddr)->sin_addr);
         if (1 != ret) {
-            opal_show_help("help-mpi-btl-usnic.txt", "invalid if_inexclude",
-                           true, name, opal_process_info.nodename, tmp,
+            opal_show_help("help-mpi-btl-usnic.txt", "invalid if_inexclude", true, name,
+                           opal_process_info.nodename, tmp,
                            "Invalid specification (inet_pton() failed)");
             free(tmp);
             continue;
         }
         opal_output_verbose(20, USNIC_OUT,
-                            "btl:usnic:parse_ifex_str: parsed %s address+prefix: %s / %u",
-                            name,
-                            opal_net_get_hostname((struct sockaddr*) &argv_inaddr),
-                            argv_prefix);
+                            "btl:usnic:parse_ifex_str: parsed %s address+prefix: %s / %u", name,
+                            opal_net_get_hostname((struct sockaddr *) &argv_inaddr), argv_prefix);
 
-        memcpy(&addr,
-               &((struct sockaddr_in*) &argv_inaddr)->sin_addr,
-               sizeof(addr));
+        memcpy(&addr, &((struct sockaddr_in *) &argv_inaddr)->sin_addr, sizeof(addr));
 
         /* be helpful: if the user passed A.B.C.D/24 instead of A.B.C.0/24,
          * also normalize the netmask */
         filter->elts[filter->n_elt].is_netmask = true;
         filter->elts[filter->n_elt].if_name = NULL;
-        filter->elts[filter->n_elt].netmask_be =
-            usnic_cidrlen_to_netmask(argv_prefix);
-        filter->elts[filter->n_elt].addr_be = addr &
-            filter->elts[filter->n_elt].netmask_be;
+        filter->elts[filter->n_elt].netmask_be = usnic_cidrlen_to_netmask(argv_prefix);
+        filter->elts[filter->n_elt].addr_be = addr & filter->elts[filter->n_elt].netmask_be;
         ++filter->n_elt;
 
         free(tmp);
@@ -528,8 +494,7 @@ static usnic_if_filter_t *parse_ifex_str(const char *orig_str,
 /*
  * Check this module to see if should be kept or not.
  */
-static bool filter_module(opal_btl_usnic_module_t *module,
-                          usnic_if_filter_t *filter,
+static bool filter_module(opal_btl_usnic_module_t *module, usnic_if_filter_t *filter,
                           bool filter_incl)
 {
     int i;
@@ -549,13 +514,12 @@ static bool filter_module(opal_btl_usnic_module_t *module,
     for (i = 0; i < filter->n_elt; ++i) {
         if (filter->elts[i].is_netmask) {
             /* conservative: we also require the netmask to match */
-            if (filter->elts[i].netmask_be == uip->ui.v1.ui_netmask_be &&
-                filter->elts[i].addr_be == module_mask) {
+            if (filter->elts[i].netmask_be == uip->ui.v1.ui_netmask_be
+                && filter->elts[i].addr_be == module_mask) {
                 match = true;
                 break;
             }
-        }
-        else {
+        } else {
             if (strcmp(filter->elts[i].if_name, linux_device_name) == 0) {
                 match = true;
                 break;
@@ -594,9 +558,8 @@ static void free_filter(usnic_if_filter_t *filter)
  *  (2) post OOB receive for incoming connection attempts
  *  (3) register BTL parameters with the MCA
  */
-static mca_btl_base_module_t** usnic_component_init(int* num_btl_modules,
-                                                    bool want_progress_threads,
-                                                    bool want_mpi_threads)
+static mca_btl_base_module_t **
+usnic_component_init(int *num_btl_modules, bool want_progress_threads, bool want_mpi_threads)
 {
     mca_btl_base_module_t **btls = NULL;
     int i, j, num_final_modules;
@@ -616,19 +579,16 @@ static mca_btl_base_module_t** usnic_component_init(int* num_btl_modules,
 
     /* MPI_THREAD_MULTIPLE is only supported in 2.0+ */
     if (want_mpi_threads && !mca_btl_base_thread_multiple_override) {
-	if (OPAL_MAJOR_VERSION >= 2) {
+        if (OPAL_MAJOR_VERSION >= 2) {
             opal_output_verbose(5, USNIC_OUT,
                                 "btl:usnic: MPI_THREAD_MULTIPLE support is in testing phase.");
-	}
-	else {
+        } else {
             opal_output_verbose(5, USNIC_OUT,
                                 "btl:usnic: MPI_THREAD_MULTIPLE is not supported in version < 2.");
-	    return NULL;
-	}
+            return NULL;
+        }
     }
 
-    OBJ_CONSTRUCT(&btl_usnic_lock, opal_recursive_mutex_t);
-
     /* There are multiple dimensions to consider when requesting an
        API version number from libfabric:
 
@@ -687,8 +647,10 @@ static mca_btl_base_module_t** usnic_component_init(int* num_btl_modules,
     uint32_t libfabric_api;
     libfabric_api = fi_version();
     if (libfabric_api < FI_VERSION(1, 3)) {
-        opal_output_verbose(5, USNIC_OUT,
-                            "btl:usnic: disqualifiying myself because Libfabric does not support v1.3 of the API (v1.3 is *required* for correct usNIC functionality).");
+        opal_output_verbose(
+            5, USNIC_OUT,
+            "btl:usnic: disqualifiying myself because Libfabric does not support v1.3 of the API "
+            "(v1.3 is *required* for correct usNIC functionality).");
         return NULL;
     }
 
@@ -721,8 +683,9 @@ static mca_btl_base_module_t** usnic_component_init(int* num_btl_modules,
 
     ret = fi_getinfo(libfabric_api, NULL, 0, 0, &hints, &info_list);
     if (0 != ret) {
-        opal_output_verbose(5, USNIC_OUT,
-                            "btl:usnic: disqualifiying myself due to fi_getinfo(3) failure: %s (%d)", strerror(-ret), ret);
+        opal_output_verbose(
+            5, USNIC_OUT, "btl:usnic: disqualifiying myself due to fi_getinfo(3) failure: %s (%d)",
+            strerror(-ret), ret);
         return NULL;
     }
 
@@ -732,7 +695,7 @@ static mca_btl_base_module_t** usnic_component_init(int* num_btl_modules,
     }
     if (0 == num_devs) {
         opal_output_verbose(5, USNIC_OUT,
-            "btl:usnic: disqualifiying myself due to lack of libfabric providers");
+                            "btl:usnic: disqualifiying myself due to lack of libfabric providers");
         return NULL;
     }
 
@@ -750,38 +713,35 @@ static mca_btl_base_module_t** usnic_component_init(int* num_btl_modules,
      * we fail.
      ************************************************************************/
 
-    opal_output_verbose(5, USNIC_OUT,
-                        "btl:usnic: usNIC fabrics found");
+    opal_output_verbose(5, USNIC_OUT, "btl:usnic: usNIC fabrics found");
 
     opal_proc_t *me = opal_proc_local_get();
     opal_process_name_t *name = &(me->proc_name);
-    mca_btl_usnic_component.my_hashed_rte_name =
-        usnic_compat_rte_hash_name(name);
-    MSGDEBUG1_OUT("%s: my_hashed_rte_name=0x%" PRIx64,
-                   __func__, mca_btl_usnic_component.my_hashed_rte_name);
+    mca_btl_usnic_component.my_hashed_rte_name = usnic_compat_rte_hash_name(name);
+    MSGDEBUG1_OUT("%s: my_hashed_rte_name=0x%" PRIx64, __func__,
+                  mca_btl_usnic_component.my_hashed_rte_name);
 
     opal_srand(&opal_btl_usnic_rand_buff, ((uint32_t) getpid()));
 
     /* Setup an array of pointers to point to each module (which we'll
        return upstream) */
     mca_btl_usnic_component.num_modules = num_devs;
-    btls = (struct mca_btl_base_module_t**)
-        malloc(mca_btl_usnic_component.num_modules *
-               sizeof(opal_btl_usnic_module_t*));
+    btls = (struct mca_btl_base_module_t **) malloc(mca_btl_usnic_component.num_modules
+                                                    * sizeof(opal_btl_usnic_module_t *));
     if (NULL == btls) {
         OPAL_ERROR_LOG(OPAL_ERR_OUT_OF_RESOURCE);
         goto send_modex;
     }
 
     /* Allocate space for btl module instances */
-    mca_btl_usnic_component.usnic_all_modules =
-        calloc(mca_btl_usnic_component.num_modules,
-               sizeof(*mca_btl_usnic_component.usnic_all_modules));
-    mca_btl_usnic_component.usnic_active_modules =
-        calloc(mca_btl_usnic_component.num_modules,
-               sizeof(*mca_btl_usnic_component.usnic_active_modules));
-    if (NULL == mca_btl_usnic_component.usnic_all_modules ||
-        NULL == mca_btl_usnic_component.usnic_active_modules) {
+    mca_btl_usnic_component.usnic_all_modules = calloc(mca_btl_usnic_component.num_modules,
+                                                       sizeof(*mca_btl_usnic_component
+                                                                   .usnic_all_modules));
+    mca_btl_usnic_component.usnic_active_modules = calloc(mca_btl_usnic_component.num_modules,
+                                                          sizeof(*mca_btl_usnic_component
+                                                                      .usnic_active_modules));
+    if (NULL == mca_btl_usnic_component.usnic_all_modules
+        || NULL == mca_btl_usnic_component.usnic_active_modules) {
         OPAL_ERROR_LOG(OPAL_ERR_OUT_OF_RESOURCE);
         goto error;
     }
@@ -791,15 +751,13 @@ static mca_btl_base_module_t** usnic_component_init(int* num_btl_modules,
      * so don't bother checking that here)
      */
     if (NULL != mca_btl_usnic_component.if_include) {
-        opal_output_verbose(20, USNIC_OUT,
-                            "btl:usnic:filter_module: if_include=%s",
+        opal_output_verbose(20, USNIC_OUT, "btl:usnic:filter_module: if_include=%s",
                             mca_btl_usnic_component.if_include);
 
         filter_incl = true;
         filter = parse_ifex_str(mca_btl_usnic_component.if_include, "include");
     } else if (NULL != mca_btl_usnic_component.if_exclude) {
-        opal_output_verbose(20, USNIC_OUT,
-                            "btl:usnic:filter_module: if_exclude=%s",
+        opal_output_verbose(20, USNIC_OUT, "btl:usnic:filter_module: if_exclude=%s",
                             mca_btl_usnic_component.if_exclude);
 
         filter_incl = false;
@@ -811,10 +769,10 @@ static mca_btl_base_module_t** usnic_component_init(int* num_btl_modules,
     /* Go through the list of devices and determine if we want it or
        not.  Create a module for each one that we want. */
     info = info_list;
-    for (j = i = 0; i < num_devs &&
-             (0 == mca_btl_usnic_component.max_modules ||
-              i < mca_btl_usnic_component.max_modules);
-             ++i, info = info->next) {
+    for (j = i = 0;
+         i < num_devs
+         && (0 == mca_btl_usnic_component.max_modules || i < mca_btl_usnic_component.max_modules);
+         ++i, info = info->next) {
 
         // The fabric/domain names changed at libfabric API v1.4 (see above).
         char *linux_device_name;
@@ -826,42 +784,29 @@ static mca_btl_base_module_t** usnic_component_init(int* num_btl_modules,
 
         ret = fi_fabric(info->fabric_attr, &fabric, NULL);
         if (0 != ret) {
-            opal_show_help("help-mpi-btl-usnic.txt",
-                           "libfabric API failed",
-                           true,
-                           opal_process_info.nodename,
-                           linux_device_name,
-                           "fi_fabric()", __FILE__, __LINE__,
-                           ret,
-                           strerror(-ret));
+            opal_show_help("help-mpi-btl-usnic.txt", "libfabric API failed", true,
+                           opal_process_info.nodename, linux_device_name, "fi_fabric()", __FILE__,
+                           __LINE__, ret, strerror(-ret));
             continue;
         }
         opal_memchecker_base_mem_defined(&fabric, sizeof(fabric));
 
         ret = fi_domain(fabric, info, &domain, NULL);
         if (0 != ret) {
-            opal_show_help("help-mpi-btl-usnic.txt",
-                           "libfabric API failed",
-                           true,
-                           opal_process_info.nodename,
-                           linux_device_name,
-                           "fi_domain()", __FILE__, __LINE__,
-                           ret,
-                           strerror(-ret));
+            opal_show_help("help-mpi-btl-usnic.txt", "libfabric API failed", true,
+                           opal_process_info.nodename, linux_device_name, "fi_domain()", __FILE__,
+                           __LINE__, ret, strerror(-ret));
             continue;
         }
         opal_memchecker_base_mem_defined(&domain, sizeof(domain));
 
-        opal_output_verbose(5, USNIC_OUT,
-                            "btl:usnic: found: usNIC device %s",
-                            linux_device_name);
+        opal_output_verbose(5, USNIC_OUT, "btl:usnic: found: usNIC device %s", linux_device_name);
 
         /* Save a little info on the module that we have already
            gathered.  The rest of the module will be filled in
            later. */
         module = &(mca_btl_usnic_component.usnic_all_modules[j]);
-        memcpy(module, &opal_btl_usnic_module_template,
-               sizeof(opal_btl_usnic_module_t));
+        memcpy(module, &opal_btl_usnic_module_template, sizeof(opal_btl_usnic_module_t));
         module->fabric = fabric;
         module->domain = domain;
         module->fabric_info = info;
@@ -876,45 +821,38 @@ static mca_btl_base_module_t** usnic_component_init(int* num_btl_modules,
            doesn't come in the normal fi_getinfo(). This allows us to
            do filtering, later. */
         ret = fi_open_ops(&fabric->fid, FI_USNIC_FABRIC_OPS_1, 0,
-                (void **)&module->usnic_fabric_ops, NULL);
+                          (void **) &module->usnic_fabric_ops, NULL);
         if (ret != 0) {
-            opal_output_verbose(5, USNIC_OUT,
-                        "btl:usnic: device %s fabric_open_ops failed %d (%s)",
-                        module->linux_device_name, ret, fi_strerror(-ret));
+            opal_output_verbose(5, USNIC_OUT, "btl:usnic: device %s fabric_open_ops failed %d (%s)",
+                                module->linux_device_name, ret, fi_strerror(-ret));
             fi_close(&domain->fid);
             fi_close(&fabric->fid);
             continue;
         }
 
-        ret =
-            module->usnic_fabric_ops->getinfo(1,
-                                            fabric,
-                                            &module->usnic_info);
+        ret = module->usnic_fabric_ops->getinfo(1, fabric, &module->usnic_info);
         if (ret != 0) {
-            opal_output_verbose(5, USNIC_OUT,
-                        "btl:usnic: device %s usnic_getinfo failed %d (%s)",
-                        module->linux_device_name, ret, fi_strerror(-ret));
+            opal_output_verbose(5, USNIC_OUT, "btl:usnic: device %s usnic_getinfo failed %d (%s)",
+                                module->linux_device_name, ret, fi_strerror(-ret));
             fi_close(&domain->fid);
             fi_close(&fabric->fid);
             continue;
         }
         opal_output_verbose(5, USNIC_OUT,
-                            "btl:usnic: device %s usnic_info: link speed=%d, netmask=0x%x, ifname=%s, num_vf=%d, qp/vf=%d, cq/vf=%d",
+                            "btl:usnic: device %s usnic_info: link speed=%d, netmask=0x%x, "
+                            "ifname=%s, num_vf=%d, qp/vf=%d, cq/vf=%d",
                             module->linux_device_name,
                             (unsigned int) module->usnic_info.ui.v1.ui_link_speed,
                             (unsigned int) module->usnic_info.ui.v1.ui_netmask_be,
-                            module->usnic_info.ui.v1.ui_ifname,
-                            module->usnic_info.ui.v1.ui_num_vf,
+                            module->usnic_info.ui.v1.ui_ifname, module->usnic_info.ui.v1.ui_num_vf,
                             module->usnic_info.ui.v1.ui_qp_per_vf,
                             module->usnic_info.ui.v1.ui_cq_per_vf);
 
         /* respect if_include/if_exclude subnets/ifaces from the user */
         if (filter != NULL) {
             keep_module = filter_module(module, filter, filter_incl);
-            opal_output_verbose(5, USNIC_OUT,
-                                "btl:usnic: %s %s due to %s",
-                                (keep_module ? "keeping" : "skipping"),
-                                module->linux_device_name,
+            opal_output_verbose(5, USNIC_OUT, "btl:usnic: %s %s due to %s",
+                                (keep_module ? "keeping" : "skipping"), module->linux_device_name,
                                 (filter_incl ? "if_include" : "if_exclude"));
             if (!keep_module) {
                 fi_close(&domain->fid);
@@ -928,11 +866,11 @@ static mca_btl_base_module_t** usnic_component_init(int* num_btl_modules,
            probes (these are VIC-wide settings -- they don't change
            for each module we create, so we only need to check
            once). */
-        if (0 == j &&
-            check_usnic_config(module, num_local_procs) != OPAL_SUCCESS) {
-            opal_output_verbose(5, USNIC_OUT,
-                                "btl:usnic: device %s is not provisioned with enough resources -- skipping",
-                                module->linux_device_name);
+        if (0 == j && check_usnic_config(module, num_local_procs) != OPAL_SUCCESS) {
+            opal_output_verbose(
+                5, USNIC_OUT,
+                "btl:usnic: device %s is not provisioned with enough resources -- skipping",
+                module->linux_device_name);
             fi_close(&domain->fid);
             fi_close(&fabric->fid);
 
@@ -944,8 +882,7 @@ static mca_btl_base_module_t** usnic_component_init(int* num_btl_modules,
         /* Below this point, we know we want this device */
         /*************************************************/
 
-        opal_output_verbose(5, USNIC_OUT,
-                            "btl:usnic: device %s looks good!",
+        opal_output_verbose(5, USNIC_OUT, "btl:usnic: device %s looks good!",
                             module->linux_device_name);
 
         /* Let this module advance to the next round! */
@@ -961,11 +898,10 @@ static mca_btl_base_module_t** usnic_component_init(int* num_btl_modules,
 
     /* If we actually have some modules, setup the connectivity
        checking agent and client. */
-    if (mca_btl_usnic_component.num_modules > 0 &&
-        mca_btl_usnic_component.connectivity_enabled) {
+    if (mca_btl_usnic_component.num_modules > 0 && mca_btl_usnic_component.connectivity_enabled) {
         mca_btl_usnic_component.opal_evbase = opal_progress_thread_init(NULL);
-        if (OPAL_SUCCESS != opal_btl_usnic_connectivity_agent_init() ||
-            OPAL_SUCCESS != opal_btl_usnic_connectivity_client_init()) {
+        if (OPAL_SUCCESS != opal_btl_usnic_connectivity_agent_init()
+            || OPAL_SUCCESS != opal_btl_usnic_connectivity_client_init()) {
             opal_progress_thread_finalize(NULL);
             return NULL;
         }
@@ -974,14 +910,12 @@ static mca_btl_base_module_t** usnic_component_init(int* num_btl_modules,
     /* Now that we know how many modules there are, let the modules
        initialize themselves (it's useful to know how many modules
        there are before doing this). */
-    for (num_final_modules = i = 0;
-         i < mca_btl_usnic_component.num_modules; ++i) {
-        module = (opal_btl_usnic_module_t*) btls[i];
+    for (num_final_modules = i = 0; i < mca_btl_usnic_component.num_modules; ++i) {
+        module = (opal_btl_usnic_module_t *) btls[i];
 
         /* Let the module initialize itself */
         if (OPAL_SUCCESS != opal_btl_usnic_module_init(module)) {
-            opal_output_verbose(5, USNIC_OUT,
-                                "btl:usnic: failed to init module for %s",
+            opal_output_verbose(5, USNIC_OUT, "btl:usnic: failed to init module for %s",
                                 module->if_ipv4_addr_str);
             continue;
         }
@@ -999,35 +933,20 @@ static mca_btl_base_module_t** usnic_component_init(int* num_btl_modules,
         const char *devname = module->linux_device_name;
         opal_output_verbose(5, USNIC_OUT,
                             "btl:usnic: %s num sqe=%d, num rqe=%d, num cqe=%d, num aveqe=%d",
-                            devname,
-                            module->sd_num,
-                            module->rd_num,
-                            module->cq_num,
+                            devname, module->sd_num, module->rd_num, module->cq_num,
                             module->av_eq_num);
-        opal_output_verbose(5, USNIC_OUT,
-                            "btl:usnic: %s priority MTU = %" PRIsize_t,
-                            devname,
+        opal_output_verbose(5, USNIC_OUT, "btl:usnic: %s priority MTU = %" PRIsize_t, devname,
                             module->max_tiny_msg_size);
-        opal_output_verbose(5, USNIC_OUT,
-                            "btl:usnic: %s priority limit = %" PRIsize_t,
-                            devname,
+        opal_output_verbose(5, USNIC_OUT, "btl:usnic: %s priority limit = %" PRIsize_t, devname,
                             module->max_tiny_payload);
-        opal_output_verbose(5, USNIC_OUT,
-                            "btl:usnic: %s eager limit = %" PRIsize_t,
-                            devname,
+        opal_output_verbose(5, USNIC_OUT, "btl:usnic: %s eager limit = %" PRIsize_t, devname,
                             module->super.btl_eager_limit);
-        opal_output_verbose(5, USNIC_OUT,
-                            "btl:usnic: %s eager rndv limit = %" PRIsize_t,
-                            devname,
+        opal_output_verbose(5, USNIC_OUT, "btl:usnic: %s eager rndv limit = %" PRIsize_t, devname,
                             module->super.btl_rndv_eager_limit);
         opal_output_verbose(5, USNIC_OUT,
-                            "btl:usnic: %s max send size= %" PRIsize_t
-                            " (not overrideable)",
-                            devname,
-                            module->super.btl_max_send_size);
-        opal_output_verbose(5, USNIC_OUT,
-                            "btl:usnic: %s exclusivity = %d",
-                            devname,
+                            "btl:usnic: %s max send size= %" PRIsize_t " (not overridable)",
+                            devname, module->super.btl_max_send_size);
+        opal_output_verbose(5, USNIC_OUT, "btl:usnic: %s exclusivity = %d", devname,
                             module->super.btl_exclusivity);
     }
 
@@ -1044,15 +963,13 @@ static mca_btl_base_module_t** usnic_component_init(int* num_btl_modules,
        That being said, if we ended up with zero acceptable devices,
        then free everything. */
     if (0 == num_final_modules) {
-        opal_output_verbose(5, USNIC_OUT,
-                            "btl:usnic: returning 0 modules");
+        opal_output_verbose(5, USNIC_OUT, "btl:usnic: returning 0 modules");
         goto error;
     }
 
     /* we have a nonzero number of modules, so save a copy of the btls array
      * for later use */
-    memcpy(mca_btl_usnic_component.usnic_active_modules, btls,
-           num_final_modules * sizeof(*btls));
+    memcpy(mca_btl_usnic_component.usnic_active_modules, btls, num_final_modules * sizeof(*btls));
 
     /* Loop over the modules and find the minimum value for
        module->numa_distance.  For every module that has a
@@ -1061,13 +978,13 @@ static mca_btl_base_module_t** usnic_component_init(int* num_btl_modules,
        messages over "near" modules. */
     min_distance = 9999999;
     for (i = 0; i < mca_btl_usnic_component.num_modules; ++i) {
-        module = (opal_btl_usnic_module_t*) btls[i];
+        module = (opal_btl_usnic_module_t *) btls[i];
         if (module->numa_distance < min_distance) {
             min_distance = module->numa_distance;
         }
     }
     for (i = 0; i < mca_btl_usnic_component.num_modules; ++i) {
-        module = (opal_btl_usnic_module_t*) btls[i];
+        module = (opal_btl_usnic_module_t *) btls[i];
         if (module->numa_distance > min_distance) {
             ++module->super.btl_latency;
             opal_output_verbose(5, USNIC_OUT,
@@ -1077,8 +994,7 @@ static mca_btl_base_module_t** usnic_component_init(int* num_btl_modules,
     }
 
     /* start timer to guarantee synthetic clock advances */
-    opal_event_set(opal_sync_event_base, &usnic_clock_timer_event,
-                   -1, 0, usnic_clock_callback,
+    opal_event_set(opal_sync_event_base, &usnic_clock_timer_event, -1, 0, usnic_clock_callback,
                    &usnic_clock_timeout);
     usnic_clock_timer_event_set = true;
 
@@ -1092,14 +1008,13 @@ static mca_btl_base_module_t** usnic_component_init(int* num_btl_modules,
 
     /* All done */
     *num_btl_modules = mca_btl_usnic_component.num_modules;
-    opal_output_verbose(5, USNIC_OUT,
-                        "btl:usnic: returning %d modules", *num_btl_modules);
+    opal_output_verbose(5, USNIC_OUT, "btl:usnic: returning %d modules", *num_btl_modules);
 
- send_modex:
+send_modex:
     usnic_modex_send();
     return btls;
 
- error:
+error:
     /* clean up as much allocated memory as possible */
     free(btls);
     btls = NULL;
@@ -1127,18 +1042,19 @@ static mca_btl_base_module_t** usnic_component_init(int* num_btl_modules,
  * immediately after one packet to avoid starvation, "fastpath_ok" is
  * used for this.
  */
-static int usnic_handle_completion(opal_btl_usnic_module_t* module,
-    opal_btl_usnic_channel_t *channel, struct fi_cq_entry *completion);
+static int usnic_handle_completion(opal_btl_usnic_module_t *module,
+                                   opal_btl_usnic_channel_t *channel,
+                                   struct fi_cq_entry *completion);
 static int usnic_component_progress_2(bool check_priority);
-static void usnic_handle_cq_error(opal_btl_usnic_module_t* module,
-    opal_btl_usnic_channel_t *channel, int cq_ret);
+static void usnic_handle_cq_error(opal_btl_usnic_module_t *module,
+                                  opal_btl_usnic_channel_t *channel, int cq_ret);
 
 static int usnic_component_progress(void)
 {
     int i;
     int count;
-    opal_btl_usnic_recv_segment_t* rseg;
-    opal_btl_usnic_module_t* module;
+    opal_btl_usnic_recv_segment_t *rseg;
+    opal_btl_usnic_module_t *module;
     struct fi_cq_entry completion;
     opal_btl_usnic_channel_t *channel;
     static bool fastpath_ok = true;
@@ -1155,19 +1071,16 @@ static int usnic_component_progress(void)
             int ret = fi_cq_read(channel->cq, &completion, 1);
             assert(0 != ret);
             if (OPAL_LIKELY(1 == ret)) {
-                opal_memchecker_base_mem_defined(&completion,
-                                                 sizeof(completion));
-                rseg = (opal_btl_usnic_recv_segment_t*) completion.op_context;
-                if (OPAL_LIKELY(OPAL_BTL_USNIC_SEG_RECV ==
-                            rseg->rs_base.us_type)) {
+                opal_memchecker_base_mem_defined(&completion, sizeof(completion));
+                rseg = (opal_btl_usnic_recv_segment_t *) completion.op_context;
+                if (OPAL_LIKELY(OPAL_BTL_USNIC_SEG_RECV == rseg->rs_base.us_type)) {
                     opal_btl_usnic_recv_fast(module, rseg, channel);
                     ++module->stats.num_seg_total_completions;
                     ++module->stats.num_seg_recv_completions;
-                    fastpath_ok = false;    /* prevent starvation */
+                    fastpath_ok = false; /* prevent starvation */
                     return 1;
                 } else {
-                    count += usnic_handle_completion(module, channel,
-                                                     &completion);
+                    count += usnic_handle_completion(module, channel, &completion);
                 }
             } else if (OPAL_LIKELY(-FI_EAGAIN == ret)) {
                 continue;
@@ -1182,16 +1095,15 @@ static int usnic_component_progress(void)
     return count + usnic_component_progress_2(check_priority);
 }
 
-static int usnic_handle_completion(
-    opal_btl_usnic_module_t* module,
-    opal_btl_usnic_channel_t *channel,
-    struct fi_cq_entry *completion)
+static int usnic_handle_completion(opal_btl_usnic_module_t *module,
+                                   opal_btl_usnic_channel_t *channel,
+                                   struct fi_cq_entry *completion)
 {
-    opal_btl_usnic_segment_t* seg;
-    opal_btl_usnic_recv_segment_t* rseg;
+    opal_btl_usnic_segment_t *seg;
+    opal_btl_usnic_recv_segment_t *rseg;
 
-    seg = (opal_btl_usnic_segment_t*)completion->op_context;
-    rseg = (opal_btl_usnic_recv_segment_t*)seg;
+    seg = (opal_btl_usnic_segment_t *) completion->op_context;
+    rseg = (opal_btl_usnic_recv_segment_t *) seg;
 
     ++module->stats.num_seg_total_completions;
 
@@ -1201,29 +1113,26 @@ static int usnic_handle_completion(
     OPAL_THREAD_LOCK(&btl_usnic_lock);
 
     /* Handle work completions */
-    switch(seg->us_type) {
+    switch (seg->us_type) {
 
     /**** Send ACK completions ****/
     case OPAL_BTL_USNIC_SEG_ACK:
         ++module->stats.num_seg_ack_completions;
-        opal_btl_usnic_ack_complete(module,
-                (opal_btl_usnic_ack_segment_t *)seg);
+        opal_btl_usnic_ack_complete(module, (opal_btl_usnic_ack_segment_t *) seg);
         break;
 
     /**** Send of frag segment completion (i.e., the MPI message's
           one-and-only segment has completed sending) ****/
     case OPAL_BTL_USNIC_SEG_FRAG:
         ++module->stats.num_seg_frag_completions;
-        opal_btl_usnic_frag_send_complete(module,
-                (opal_btl_usnic_frag_segment_t*)seg);
+        opal_btl_usnic_frag_send_complete(module, (opal_btl_usnic_frag_segment_t *) seg);
         break;
 
     /**** Send of chunk segment completion (i.e., part of a large MPI
           message is done sending) ****/
     case OPAL_BTL_USNIC_SEG_CHUNK:
         ++module->stats.num_seg_chunk_completions;
-        opal_btl_usnic_chunk_send_complete(module,
-                (opal_btl_usnic_chunk_segment_t*)seg);
+        opal_btl_usnic_chunk_send_complete(module, (opal_btl_usnic_chunk_segment_t *) seg);
         break;
 
     /**** Receive completions ****/
@@ -1241,18 +1150,16 @@ static int usnic_handle_completion(
     return 1;
 }
 
-static void
-usnic_handle_cq_error(opal_btl_usnic_module_t* module,
-    opal_btl_usnic_channel_t *channel, int cq_ret)
+static void usnic_handle_cq_error(opal_btl_usnic_module_t *module,
+                                  opal_btl_usnic_channel_t *channel, int cq_ret)
 {
     int rc;
     struct fi_cq_err_entry err_entry;
-    opal_btl_usnic_recv_segment_t* rseg;
+    opal_btl_usnic_recv_segment_t *rseg;
 
     if (cq_ret != -FI_EAVAIL) {
-        BTL_ERROR(("%s: cq_read ret = %d (%s)",
-               module->linux_device_name, cq_ret,
-               fi_strerror(-cq_ret)));
+        BTL_ERROR(
+            ("%s: cq_read ret = %d (%s)", module->linux_device_name, cq_ret, fi_strerror(-cq_ret)));
         channel->chan_error = true;
     }
 
@@ -1260,23 +1167,18 @@ usnic_handle_cq_error(opal_btl_usnic_module_t* module,
     if (rc == -FI_EAGAIN) {
         return;
     } else if (rc != 1) {
-        BTL_ERROR(("%s: cq_readerr ret = %d (expected 1)",
-                   module->linux_device_name, rc));
+        BTL_ERROR(("%s: cq_readerr ret = %d (expected 1)", module->linux_device_name, rc));
         channel->chan_error = true;
     }
 
     /* Silently count CRC errors.  Truncation errors are usually a
        different symptom of a CRC error. */
-    else if (FI_ECRC == err_entry.prov_errno ||
-             FI_ETRUNC == err_entry.prov_errno) {
+    else if (FI_ECRC == err_entry.prov_errno || FI_ETRUNC == err_entry.prov_errno) {
 #if MSGDEBUG1
         static int once = 0;
         if (once++ == 0) {
-            BTL_ERROR(("%s: Channel %d, %s",
-                       module->linux_device_name,
-                       channel->chan_index,
-                       FI_ECRC == err_entry.prov_errno ?
-                       "CRC error" : "message truncation"));
+            BTL_ERROR(("%s: Channel %d, %s", module->linux_device_name, channel->chan_index,
+                       FI_ECRC == err_entry.prov_errno ? "CRC error" : "message truncation"));
         }
 #endif
 
@@ -1293,8 +1195,7 @@ usnic_handle_cq_error(opal_btl_usnic_module_t* module,
             channel->repost_recv_head = rseg;
         }
     } else {
-        BTL_ERROR(("%s: CQ[%d] prov_err = %d",
-                   module->linux_device_name, channel->chan_index,
+        BTL_ERROR(("%s: CQ[%d] prov_err = %d", module->linux_device_name, channel->chan_index,
                    err_entry.prov_errno));
         channel->chan_error = true;
     }
@@ -1303,7 +1204,7 @@ usnic_handle_cq_error(opal_btl_usnic_module_t* module,
 static int usnic_component_progress_2(bool check_priority)
 {
     int i, j, count = 0, num_events, ret;
-    opal_btl_usnic_module_t* module;
+    opal_btl_usnic_module_t *module;
     static struct fi_cq_entry completions[OPAL_BTL_USNIC_NUM_COMPLETIONS];
     opal_btl_usnic_channel_t *channel;
     int rc;
@@ -1320,18 +1221,16 @@ static int usnic_component_progress_2(bool check_priority)
         module = mca_btl_usnic_component.usnic_active_modules[i];
 
         /* poll each channel */
-        for (c=c_start; c<USNIC_NUM_CHANNELS; ++c) {
+        for (c = c_start; c < USNIC_NUM_CHANNELS; ++c) {
             channel = &module->mod_channels[c];
 
             if (channel->chan_deferred_recv != NULL) {
-                (void) opal_btl_usnic_recv_frag_bookkeeping(module,
-                        channel->chan_deferred_recv, channel);
+                (void) opal_btl_usnic_recv_frag_bookkeeping(module, channel->chan_deferred_recv,
+                                                            channel);
                 channel->chan_deferred_recv = NULL;
             }
 
-            num_events = ret =
-                fi_cq_read(channel->cq, completions,
-                           OPAL_BTL_USNIC_NUM_COMPLETIONS);
+            num_events = ret = fi_cq_read(channel->cq, completions, OPAL_BTL_USNIC_NUM_COMPLETIONS);
             assert(0 != ret);
             opal_memchecker_base_mem_defined(&ret, sizeof(ret));
             if (OPAL_UNLIKELY(ret < 0 && -FI_EAGAIN != ret)) {
@@ -1341,13 +1240,10 @@ static int usnic_component_progress_2(bool check_priority)
                 num_events = 0;
             }
 
-            opal_memchecker_base_mem_defined(completions,
-                                             sizeof(completions[0]) *
-                                             num_events);
+            opal_memchecker_base_mem_defined(completions, sizeof(completions[0]) * num_events);
             /* Handle each event */
             for (j = 0; j < num_events; j++) {
-                count += usnic_handle_completion(module, channel,
-                                                 &completions[j]);
+                count += usnic_handle_completion(module, channel, &completions[j]);
             }
 
             /* return error if detected - this may be slightly deferred
@@ -1389,71 +1285,69 @@ static void dump_endpoint(opal_btl_usnic_endpoint_t *endpoint)
     ia.s_addr = endpoint->endpoint_remote_modex.ipv4_addr;
     inet_ntop(AF_INET, &ia, ep_addr_str, sizeof(ep_addr_str));
 
-    opal_output(0, "    endpoint %p, %s job=%u, rank=%u rts=%s s_credits=%"PRIi32"\n",
-                (void *)endpoint, ep_addr_str,
-                endpoint->endpoint_proc->proc_opal->proc_name.jobid,
+    opal_output(0, "    endpoint %p, %s job=%u, rank=%u rts=%s s_credits=%" PRIi32 "\n",
+                (void *) endpoint, ep_addr_str, endpoint->endpoint_proc->proc_opal->proc_name.jobid,
                 endpoint->endpoint_proc->proc_opal->proc_name.vpid,
                 (endpoint->endpoint_ready_to_send ? "true" : "false"),
                 endpoint->endpoint_send_credits);
     opal_output(0, "      endpoint->frag_send_queue:\n");
 
-    OPAL_LIST_FOREACH(frag, &endpoint->endpoint_frag_send_queue,
-                      opal_btl_usnic_frag_t) {
+    OPAL_LIST_FOREACH (frag, &endpoint->endpoint_frag_send_queue, opal_btl_usnic_frag_t) {
         opal_btl_usnic_small_send_frag_t *ssfrag;
         opal_btl_usnic_large_send_frag_t *lsfrag;
 
-        snprintf(str, sizeof(str), "      --> frag %p, %s", (void *)frag,
+        snprintf(str, sizeof(str), "      --> frag %p, %s", (void *) frag,
                  usnic_frag_type(frag->uf_type));
         switch (frag->uf_type) {
-            case OPAL_BTL_USNIC_FRAG_LARGE_SEND:
-                lsfrag = (opal_btl_usnic_large_send_frag_t *)frag;
-                snprintf(tmp, sizeof(tmp), " tag=%"PRIu8" id=%"PRIu32" offset=%llu/%llu post_cnt=%"PRIu32" ack_bytes_left=%llu\n",
-                        lsfrag->lsf_tag,
-                        lsfrag->lsf_frag_id,
-                        (unsigned long long)lsfrag->lsf_cur_offset,
-                        (unsigned long long)lsfrag->lsf_base.sf_size,
-                        lsfrag->lsf_base.sf_seg_post_cnt,
-                        (unsigned long long)lsfrag->lsf_base.sf_ack_bytes_left);
-                strncat(str, tmp, sizeof(str) - strlen(str) - 1);
-                opal_output(0, "%s", str);
-
-                OPAL_LIST_FOREACH(sseg, &lsfrag->lsf_seg_chain,
-                                  opal_btl_usnic_send_segment_t) {
-                    /* chunk segs are just typedefs to send segs */
-                    opal_output(0, "        chunk seg %p, chan=%s hotel=%d times_posted=%"PRIu32" pending=%s\n",
-                                (void *)sseg,
-                                (USNIC_PRIORITY_CHANNEL == sseg->ss_channel ?
-                                "prio" : "data"),
-                                sseg->ss_hotel_room,
-                                sseg->ss_send_posted,
-                                (sseg->ss_ack_pending ? "true" : "false"));
-                }
+        case OPAL_BTL_USNIC_FRAG_LARGE_SEND:
+            lsfrag = (opal_btl_usnic_large_send_frag_t *) frag;
+            snprintf(tmp, sizeof(tmp),
+                     " tag=%" PRIu8 " id=%" PRIu32 " offset=%llu/%llu post_cnt=%" PRIu32
+                     " ack_bytes_left=%llu\n",
+                     lsfrag->lsf_tag, lsfrag->lsf_frag_id,
+                     (unsigned long long) lsfrag->lsf_cur_offset,
+                     (unsigned long long) lsfrag->lsf_base.sf_size,
+                     lsfrag->lsf_base.sf_seg_post_cnt,
+                     (unsigned long long) lsfrag->lsf_base.sf_ack_bytes_left);
+            strncat(str, tmp, sizeof(str) - strlen(str) - 1);
+            opal_output(0, "%s", str);
+
+            OPAL_LIST_FOREACH (sseg, &lsfrag->lsf_seg_chain, opal_btl_usnic_send_segment_t) {
+                /* chunk segs are just typedefs to send segs */
+                opal_output(0,
+                            "        chunk seg %p, chan=%s hotel=%d times_posted=%" PRIu32
+                            " pending=%s\n",
+                            (void *) sseg,
+                            (USNIC_PRIORITY_CHANNEL == sseg->ss_channel ? "prio" : "data"),
+                            sseg->ss_hotel_room, sseg->ss_send_posted,
+                            (sseg->ss_ack_pending ? "true" : "false"));
+            }
             break;
 
-            case OPAL_BTL_USNIC_FRAG_SMALL_SEND:
-                ssfrag = (opal_btl_usnic_small_send_frag_t *)frag;
-                snprintf(tmp, sizeof(tmp), " sf_size=%llu post_cnt=%"PRIu32" ack_bytes_left=%llu\n",
-                        (unsigned long long)ssfrag->ssf_base.sf_size,
-                        ssfrag->ssf_base.sf_seg_post_cnt,
-                        (unsigned long long)ssfrag->ssf_base.sf_ack_bytes_left);
-                strncat(str, tmp, sizeof(str) - strlen(str) - 1);
-                opal_output(0, "%s", str);
-
-                sseg = &ssfrag->ssf_segment;
-                opal_output(0, "        small seg %p, chan=%s hotel=%d times_posted=%"PRIu32" pending=%s\n",
-                    (void *)sseg,
-                    (USNIC_PRIORITY_CHANNEL == sseg->ss_channel ?
-                        "prio" : "data"),
-                    sseg->ss_hotel_room,
-                    sseg->ss_send_posted,
-                    (sseg->ss_ack_pending ? "true" : "false"));
+        case OPAL_BTL_USNIC_FRAG_SMALL_SEND:
+            ssfrag = (opal_btl_usnic_small_send_frag_t *) frag;
+            snprintf(tmp, sizeof(tmp), " sf_size=%llu post_cnt=%" PRIu32 " ack_bytes_left=%llu\n",
+                     (unsigned long long) ssfrag->ssf_base.sf_size,
+                     ssfrag->ssf_base.sf_seg_post_cnt,
+                     (unsigned long long) ssfrag->ssf_base.sf_ack_bytes_left);
+            strncat(str, tmp, sizeof(str) - strlen(str) - 1);
+            opal_output(0, "%s", str);
+
+            sseg = &ssfrag->ssf_segment;
+            opal_output(0,
+                        "        small seg %p, chan=%s hotel=%d times_posted=%" PRIu32
+                        " pending=%s\n",
+                        (void *) sseg,
+                        (USNIC_PRIORITY_CHANNEL == sseg->ss_channel ? "prio" : "data"),
+                        sseg->ss_hotel_room, sseg->ss_send_posted,
+                        (sseg->ss_ack_pending ? "true" : "false"));
             break;
 
-            case OPAL_BTL_USNIC_FRAG_PUT_DEST:
-                /* put_dest frags are just a typedef to generic frags */
-                snprintf(tmp, sizeof(tmp), " put_addr=%p\n", frag->uf_remote_seg[0].seg_addr.pval);
-                strncat(str, tmp, sizeof(str) - strlen(str) - 1);
-                opal_output(0, "%s", str);
+        case OPAL_BTL_USNIC_FRAG_PUT_DEST:
+            /* put_dest frags are just a typedef to generic frags */
+            snprintf(tmp, sizeof(tmp), " put_addr=%p\n", frag->uf_remote_seg[0].seg_addr.pval);
+            strncat(str, tmp, sizeof(str) - strlen(str) - 1);
+            opal_output(0, "%s", str);
             break;
         }
     }
@@ -1463,33 +1357,30 @@ static void dump_endpoint(opal_btl_usnic_endpoint_t *endpoint)
      * eventually this should be done through some sort of debug or iteration
      * interface in the hotel code. */
     opal_output(0, "      endpoint->endpoint_sent_segs (%p):\n",
-           (void *)endpoint->endpoint_sent_segs);
+                (void *) endpoint->endpoint_sent_segs);
     for (i = 0; i < WINDOW_SIZE; ++i) {
         sseg = endpoint->endpoint_sent_segs[i];
         if (NULL != sseg) {
-            opal_output(0, "        [%d] sseg=%p %s chan=%s hotel=%d times_posted=%"PRIu32" pending=%s\n",
-                   i,
-                   (void *)sseg,
-                   usnic_seg_type_str(sseg->ss_base.us_type),
-                   (USNIC_PRIORITY_CHANNEL == sseg->ss_channel ?
-                    "prio" : "data"),
-                   sseg->ss_hotel_room,
-                   sseg->ss_send_posted,
-                   (sseg->ss_ack_pending ? "true" : "false"));
+            opal_output(0,
+                        "        [%d] sseg=%p %s chan=%s hotel=%d times_posted=%" PRIu32
+                        " pending=%s\n",
+                        i, (void *) sseg, usnic_seg_type_str(sseg->ss_base.us_type),
+                        (USNIC_PRIORITY_CHANNEL == sseg->ss_channel ? "prio" : "data"),
+                        sseg->ss_hotel_room, sseg->ss_send_posted,
+                        (sseg->ss_ack_pending ? "true" : "false"));
         }
     }
 
-    opal_output(0, "      ack_needed=%s n_t=%"UDSEQ" n_a=%"UDSEQ" n_r=%"UDSEQ" n_s=%"UDSEQ" rfstart=%"PRIu32"\n",
-                (endpoint->endpoint_ack_needed?"true":"false"),
-                endpoint->endpoint_next_seq_to_send,
-                endpoint->endpoint_ack_seq_rcvd,
-                endpoint->endpoint_next_contig_seq_to_recv,
-                endpoint->endpoint_highest_seq_rcvd,
+    opal_output(0,
+                "      ack_needed=%s n_t=%" UDSEQ " n_a=%" UDSEQ " n_r=%" UDSEQ " n_s=%" UDSEQ
+                " rfstart=%" PRIu32 "\n",
+                (endpoint->endpoint_ack_needed ? "true" : "false"),
+                endpoint->endpoint_next_seq_to_send, endpoint->endpoint_ack_seq_rcvd,
+                endpoint->endpoint_next_contig_seq_to_recv, endpoint->endpoint_highest_seq_rcvd,
                 endpoint->endpoint_rfstart);
 
     if (dump_bitvectors) {
-        opal_btl_usnic_snprintf_bool_array(str, sizeof(str),
-                                           endpoint->endpoint_rcvd_segs,
+        opal_btl_usnic_snprintf_bool_array(str, sizeof(str), endpoint->endpoint_rcvd_segs,
                                            WINDOW_SIZE);
         opal_output(0, "      rcvd_segs 0x%s", str);
     }
@@ -1506,24 +1397,22 @@ void opal_btl_usnic_component_debug(void)
 
     opal_output(0, "*** dumping usnic state for MPI_COMM_WORLD rank %u ***\n",
                 proc->proc_name.vpid);
-    for (i = 0; i < (int)mca_btl_usnic_component.num_modules; ++i) {
+    for (i = 0; i < (int) mca_btl_usnic_component.num_modules; ++i) {
         module = mca_btl_usnic_component.usnic_active_modules[i];
 
-        opal_output(0, "active_modules[%d]=%p %s max{frag,chunk,tiny}=%llu,%llu,%llu\n",
-               i, (void *)module, module->linux_device_name,
-               (unsigned long long)module->max_frag_payload,
-               (unsigned long long)module->max_chunk_payload,
-               (unsigned long long)module->max_tiny_payload);
+        opal_output(0, "active_modules[%d]=%p %s max{frag,chunk,tiny}=%llu,%llu,%llu\n", i,
+                    (void *) module, module->linux_device_name,
+                    (unsigned long long) module->max_frag_payload,
+                    (unsigned long long) module->max_chunk_payload,
+                    (unsigned long long) module->max_tiny_payload);
 
         opal_output(0, "  endpoints_with_sends:\n");
-        OPAL_LIST_FOREACH(endpoint, &module->endpoints_with_sends,
-                          opal_btl_usnic_endpoint_t) {
+        OPAL_LIST_FOREACH (endpoint, &module->endpoints_with_sends, opal_btl_usnic_endpoint_t) {
             dump_endpoint(endpoint);
         }
 
         opal_output(0, "  endpoints_that_need_acks:\n");
-        OPAL_LIST_FOREACH(endpoint, &module->endpoints_that_need_acks,
-                          opal_btl_usnic_endpoint_t) {
+        OPAL_LIST_FOREACH (endpoint, &module->endpoints_that_need_acks, opal_btl_usnic_endpoint_t) {
             dump_endpoint(endpoint);
         }
 
@@ -1532,17 +1421,15 @@ void opal_btl_usnic_component_debug(void)
         opal_mutex_lock(&module->all_endpoints_lock);
         item = opal_list_get_first(&module->all_endpoints);
         while (item != opal_list_get_end(&module->all_endpoints)) {
-            endpoint = container_of(item, mca_btl_base_endpoint_t,
-                                    endpoint_endpoint_li);
+            endpoint = container_of(item, mca_btl_base_endpoint_t, endpoint_endpoint_li);
             item = opal_list_get_next(item);
             dump_endpoint(endpoint);
         }
         opal_mutex_unlock(&module->all_endpoints_lock);
 
         opal_output(0, "  pending_resend_segs:\n");
-        OPAL_LIST_FOREACH(sseg, &module->pending_resend_segs,
-                          opal_btl_usnic_send_segment_t) {
-            opal_output(0, "    sseg %p\n", (void *)sseg);
+        OPAL_LIST_FOREACH (sseg, &module->pending_resend_segs, opal_btl_usnic_send_segment_t) {
+            opal_output(0, "    sseg %p\n", (void *) sseg);
         }
 
         opal_btl_usnic_print_stats(module, "  manual", /*reset=*/false);
diff --git a/opal/mca/btl/usnic/btl_usnic_connectivity.h b/opal/mca/btl/usnic/btl_usnic_connectivity.h
index 7bde19f81fd..a2678030e94 100644
--- a/opal/mca/btl/usnic/btl_usnic_connectivity.h
+++ b/opal/mca/btl/usnic/btl_usnic_connectivity.h
@@ -22,7 +22,6 @@
 #include "btl_usnic_proc.h"
 #include "btl_usnic_util.h"
 
-
 /**
  * Agent-based service to verify UDP connectivity between two peers.
  *
@@ -104,7 +103,7 @@ enum {
 };
 
 #define CONNECTIVITY_NODENAME_LEN 128
-#define CONNECTIVITY_IFNAME_LEN 32
+#define CONNECTIVITY_IFNAME_LEN   32
 
 /*
  * Unix domain socket name
@@ -217,10 +216,8 @@ int opal_btl_usnic_connectivity_listen(struct opal_btl_usnic_module_t *module);
  * It is safe to call this function even if the connectivity check is
  * disabled; it will be a no-op in this case.
  */
-int opal_btl_usnic_connectivity_ping(uint32_t src_ipv4_addr, int src_port,
-                                     uint32_t dest_ipv4_addr,
-                                     uint32_t dest_netmask, int dest_port,
-                                     char *dest_nodename,
+int opal_btl_usnic_connectivity_ping(uint32_t src_ipv4_addr, int src_port, uint32_t dest_ipv4_addr,
+                                     uint32_t dest_netmask, int dest_port, char *dest_nodename,
                                      size_t max_msg_size);
 
 /**
@@ -268,25 +265,22 @@ int opal_btl_usnic_connectivity_agent_init(void);
  */
 int opal_btl_usnic_connectivity_agent_finalize(void);
 
-
 /**
  * Helper function invoked in the BTL that will invoke a ping, if the
  * ping hasn't already been invoked.
  */
-static inline void
-opal_btl_usnic_check_connectivity(opal_btl_usnic_module_t *module,
-                                  opal_btl_usnic_endpoint_t *endpoint)
+static inline void opal_btl_usnic_check_connectivity(opal_btl_usnic_module_t *module,
+                                                     opal_btl_usnic_endpoint_t *endpoint)
 {
-    if (OPAL_LIKELY(mca_btl_usnic_component.connectivity_enabled) &&
-        OPAL_UNLIKELY(!endpoint->endpoint_connectivity_checked)) {
+    if (OPAL_LIKELY(mca_btl_usnic_component.connectivity_enabled)
+        && OPAL_UNLIKELY(!endpoint->endpoint_connectivity_checked)) {
         char *host = opal_get_proc_hostname(endpoint->endpoint_proc->proc_opal);
         opal_btl_usnic_connectivity_ping(module->local_modex.ipv4_addr,
                                          module->local_modex.connectivity_udp_port,
                                          endpoint->endpoint_remote_modex.ipv4_addr,
                                          endpoint->endpoint_remote_modex.netmask,
                                          endpoint->endpoint_remote_modex.connectivity_udp_port,
-                                         host,
-                                         endpoint->endpoint_remote_modex.max_msg_size);
+                                         host, endpoint->endpoint_remote_modex.max_msg_size);
         free(host);
         endpoint->endpoint_connectivity_checked = true;
     }
diff --git a/opal/mca/btl/usnic/btl_usnic_endpoint.c b/opal/mca/btl/usnic/btl_usnic_endpoint.c
index 40a1faee1bb..65317f8006b 100644
--- a/opal/mca/btl/usnic/btl_usnic_endpoint.c
+++ b/opal/mca/btl/usnic/btl_usnic_endpoint.c
@@ -24,8 +24,8 @@
 
 #include "opal_config.h"
 
-#include <stdio.h>
 #include <errno.h>
+#include <stdio.h>
 #include <string.h>
 #include <sys/types.h>
 #include <unistd.h>
@@ -34,18 +34,18 @@
 #include "opal/types.h"
 
 #include "btl_usnic.h"
+#include "btl_usnic_ack.h"
 #include "btl_usnic_endpoint.h"
-#include "btl_usnic_module.h"
 #include "btl_usnic_frag.h"
+#include "btl_usnic_module.h"
 #include "btl_usnic_proc.h"
-#include "btl_usnic_util.h"
-#include "btl_usnic_ack.h"
 #include "btl_usnic_send.h"
+#include "btl_usnic_util.h"
 
 /*
  * Construct/destruct an endpoint structure.
  */
-static void endpoint_construct(mca_btl_base_endpoint_t* endpoint)
+static void endpoint_construct(mca_btl_base_endpoint_t *endpoint)
 {
     int i;
 
@@ -74,10 +74,8 @@ static void endpoint_construct(mca_btl_base_endpoint_t* endpoint)
     endpoint->endpoint_ack_needed = false;
 
     /* clear sent/received sequence number array */
-    memset(endpoint->endpoint_sent_segs, 0,
-           sizeof(endpoint->endpoint_sent_segs));
-    memset(endpoint->endpoint_rcvd_segs, 0,
-           sizeof(endpoint->endpoint_rcvd_segs));
+    memset(endpoint->endpoint_sent_segs, 0, sizeof(endpoint->endpoint_sent_segs));
+    memset(endpoint->endpoint_rcvd_segs, 0, sizeof(endpoint->endpoint_rcvd_segs));
 
     /*
      * Make a new OPAL hotel for this module
@@ -85,12 +83,8 @@ static void endpoint_construct(mca_btl_base_endpoint_t* endpoint)
      * due to timeout
      */
     OBJ_CONSTRUCT(&endpoint->endpoint_hotel, opal_hotel_t);
-    opal_hotel_init(&endpoint->endpoint_hotel,
-                    WINDOW_SIZE,
-                    opal_sync_event_base,
-                    mca_btl_usnic_component.retrans_timeout,
-                    0,
-                    opal_btl_usnic_ack_timeout);
+    opal_hotel_init(&endpoint->endpoint_hotel, WINDOW_SIZE, opal_sync_event_base,
+                    mca_btl_usnic_component.retrans_timeout, 0, opal_btl_usnic_ack_timeout);
 
     /* Setup this endpoint's list links */
     OBJ_CONSTRUCT(&(endpoint->endpoint_ack_li), opal_list_item_t);
@@ -98,8 +92,8 @@ static void endpoint_construct(mca_btl_base_endpoint_t* endpoint)
     endpoint->endpoint_ack_needed = false;
 
     /* fragment reassembly info */
-    endpoint->endpoint_rx_frag_info =
-        calloc(sizeof(struct opal_btl_usnic_rx_frag_info_t), MAX_ACTIVE_FRAGS);
+    endpoint->endpoint_rx_frag_info = calloc(sizeof(struct opal_btl_usnic_rx_frag_info_t),
+                                             MAX_ACTIVE_FRAGS);
     assert(NULL != endpoint->endpoint_rx_frag_info);
     if (OPAL_UNLIKELY(endpoint->endpoint_rx_frag_info == NULL)) {
         BTL_ERROR(("calloc returned NULL -- this should not happen!"));
@@ -108,7 +102,7 @@ static void endpoint_construct(mca_btl_base_endpoint_t* endpoint)
     }
 }
 
-static void endpoint_destruct(mca_btl_base_endpoint_t* endpoint)
+static void endpoint_destruct(mca_btl_base_endpoint_t *endpoint)
 {
     opal_btl_usnic_proc_t *proc;
 
@@ -121,8 +115,7 @@ static void endpoint_destruct(mca_btl_base_endpoint_t* endpoint)
     opal_btl_usnic_module_t *module = endpoint->endpoint_module;
     opal_mutex_lock(&module->all_endpoints_lock);
     if (endpoint->endpoint_on_all_endpoints) {
-        opal_list_remove_item(&module->all_endpoints,
-                              &endpoint->endpoint_endpoint_li);
+        opal_list_remove_item(&module->all_endpoints, &endpoint->endpoint_endpoint_li);
         endpoint->endpoint_on_all_endpoints = false;
     }
     opal_mutex_unlock(&module->all_endpoints_lock);
@@ -144,25 +137,21 @@ static void endpoint_destruct(mca_btl_base_endpoint_t* endpoint)
     free(endpoint->endpoint_rx_frag_info);
 }
 
-OBJ_CLASS_INSTANCE(opal_btl_usnic_endpoint_t,
-                   opal_list_item_t,
-                   endpoint_construct,
+OBJ_CLASS_INSTANCE(opal_btl_usnic_endpoint_t, opal_list_item_t, endpoint_construct,
                    endpoint_destruct);
 
 /*
  * Forcibly drain all pending output on an endpoint, without waiting for
  * actual completion.
  */
-void
-opal_btl_usnic_flush_endpoint(
-    opal_btl_usnic_endpoint_t *endpoint)
+void opal_btl_usnic_flush_endpoint(opal_btl_usnic_endpoint_t *endpoint)
 {
     opal_btl_usnic_send_frag_t *frag;
 
     /* First, free all pending fragments */
     while (!opal_list_is_empty(&endpoint->endpoint_frag_send_queue)) {
-        frag = (opal_btl_usnic_send_frag_t *)opal_list_remove_first(
-                &endpoint->endpoint_frag_send_queue);
+        frag = (opal_btl_usnic_send_frag_t *) opal_list_remove_first(
+            &endpoint->endpoint_frag_send_queue);
 
         /* _cond still needs to check ownership, but make sure the
          * fragment is marked as done.
@@ -173,5 +162,5 @@ opal_btl_usnic_flush_endpoint(
     }
 
     /* Now, ACK everything that is pending */
-    opal_btl_usnic_handle_ack(endpoint, endpoint->endpoint_next_seq_to_send-1);
+    opal_btl_usnic_handle_ack(endpoint, endpoint->endpoint_next_seq_to_send - 1);
 }
diff --git a/opal/mca/btl/usnic/btl_usnic_endpoint.h b/opal/mca/btl/usnic/btl_usnic_endpoint.h
index ff66550a66f..2346c8fa98c 100644
--- a/opal/mca/btl/usnic/btl_usnic_endpoint.h
+++ b/opal/mca/btl/usnic/btl_usnic_endpoint.h
@@ -26,8 +26,8 @@
 
 #include <rdma/fabric.h>
 
-#include "opal/class/opal_list.h"
 #include "opal/class/opal_hotel.h"
+#include "opal/class/opal_list.h"
 #include "opal/util/event.h"
 
 #include "btl_usnic.h"
@@ -44,18 +44,16 @@ struct opal_btl_usnic_send_segment_t;
  * Have the window size as a compile-time constant that is a power of
  * two so that we can take advantage of fast bit operations.
  */
-#define WINDOW_SIZE 4096
+#define WINDOW_SIZE        4096
 #define WINDOW_SIZE_MOD(a) (((a) & (WINDOW_SIZE - 1)))
-#define WINDOW_OPEN(E) (SEQ_LT((E)->endpoint_next_seq_to_send, \
-        ((E)->endpoint_ack_seq_rcvd + WINDOW_SIZE)))
-#define WINDOW_EMPTY(E) ((E)->endpoint_ack_seq_rcvd == \
-        ((E)->endpoint_next_seq_to_send-1))
+#define WINDOW_OPEN(E) \
+    (SEQ_LT((E)->endpoint_next_seq_to_send, ((E)->endpoint_ack_seq_rcvd + WINDOW_SIZE)))
+#define WINDOW_EMPTY(E) ((E)->endpoint_ack_seq_rcvd == ((E)->endpoint_next_seq_to_send - 1))
 
 /*
  * Returns true when an endpoint has nothing left to send
  */
-#define ENDPOINT_DRAINED(E) (WINDOW_EMPTY(E) && \
-        opal_list_is_empty(&(E)->endpoint_frag_send_queue))
+#define ENDPOINT_DRAINED(E) (WINDOW_EMPTY(E) && opal_list_is_empty(&(E)->endpoint_frag_send_queue))
 
 /*
  * Channel IDs
@@ -99,15 +97,15 @@ struct opal_btl_usnic_proc_t;
  * and waiting for retrans is just fine in this hypothetical hyper-pathological
  * case, which is what we'll do.
  */
-#define MAX_ACTIVE_FRAGS (WINDOW_SIZE/2)
+#define MAX_ACTIVE_FRAGS (WINDOW_SIZE / 2)
 typedef struct opal_btl_usnic_rx_frag_info_t {
-    uint32_t    rfi_frag_id;    /* ID for this fragment */
-    uint32_t    rfi_frag_size;  /* bytes in this fragment */
-    uint32_t    rfi_bytes_left; /* bytes remaining to RX in fragment */
-    bool        rfi_data_in_pool; /* data in data_pool if true, else malloced */
-    int         rfi_data_pool;  /* if <0, data malloced, else rx buf pool */
-    char       *rfi_data;       /* pointer to assembly area */
-    opal_free_list_item_t *rfi_fl_elt; /* free list elemement from buf pool
+    uint32_t rfi_frag_id;              /* ID for this fragment */
+    uint32_t rfi_frag_size;            /* bytes in this fragment */
+    uint32_t rfi_bytes_left;           /* bytes remaining to RX in fragment */
+    bool rfi_data_in_pool;             /* data in data_pool if true, else malloced */
+    int rfi_data_pool;                 /* if <0, data malloced, else rx buf pool */
+    char *rfi_data;                    /* pointer to assembly area */
+    opal_free_list_item_t *rfi_fl_elt; /* free list element from buf pool
                                           when rfi_data_pool is nonzero */
 } opal_btl_usnic_rx_frag_info_t;
 
@@ -126,7 +124,7 @@ typedef struct mca_btl_base_endpoint_t {
 
     /** proc that owns this endpoint */
     struct opal_btl_usnic_proc_t *endpoint_proc;
-    int endpoint_proc_index;    /* index in owning proc's endpoint array */
+    int endpoint_proc_index; /* index in owning proc's endpoint array */
 
     /** True when proc has been deleted, but still have sends that need ACKs */
     bool endpoint_exiting;
@@ -160,7 +158,7 @@ typedef struct mca_btl_base_endpoint_t {
     /* Values for the current proc to send to this endpoint on the
        peer proc */
     opal_btl_usnic_seq_t endpoint_next_seq_to_send; /* n_t */
-    opal_btl_usnic_seq_t endpoint_ack_seq_rcvd; /* n_a */
+    opal_btl_usnic_seq_t endpoint_ack_seq_rcvd;     /* n_a */
 
     /* Table where sent segments sit while waiting for their ACKs.
        When a segment is ACKed, it is removed from this table. */
@@ -176,7 +174,7 @@ typedef struct mca_btl_base_endpoint_t {
     uint64_t endpoint_acktime;
 
     opal_btl_usnic_seq_t endpoint_next_contig_seq_to_recv; /* n_r */
-    opal_btl_usnic_seq_t endpoint_highest_seq_rcvd; /* n_s */
+    opal_btl_usnic_seq_t endpoint_highest_seq_rcvd;        /* n_s */
 
     bool endpoint_rcvd_segs[WINDOW_SIZE];
     uint32_t endpoint_rfstart;
@@ -189,7 +187,7 @@ typedef mca_btl_base_endpoint_t opal_btl_usnic_endpoint_t;
 OBJ_CLASS_DECLARATION(opal_btl_usnic_endpoint_t);
 
 /*
- * Helper struct for the asynchornous creation of fi_addr array
+ * Helper struct for the asynchronous creation of fi_addr array
  */
 typedef struct {
     opal_btl_usnic_endpoint_t *endpoint;
@@ -199,9 +197,7 @@ typedef struct {
 /*
  * Flush all pending sends and resends from and endpoint
  */
-void
-opal_btl_usnic_flush_endpoint(
-    opal_btl_usnic_endpoint_t *endpoint);
+void opal_btl_usnic_flush_endpoint(opal_btl_usnic_endpoint_t *endpoint);
 
 END_C_DECLS
 #endif
diff --git a/opal/mca/btl/usnic/btl_usnic_frag.c b/opal/mca/btl/usnic/btl_usnic_frag.c
index 6a8129a3f7e..4c0beaedc2f 100644
--- a/opal/mca/btl/usnic/btl_usnic_frag.c
+++ b/opal/mca/btl/usnic/btl_usnic_frag.c
@@ -24,13 +24,12 @@
 #include <string.h>
 
 #include "btl_usnic.h"
+#include "btl_usnic_ack.h"
 #include "btl_usnic_endpoint.h"
-#include "btl_usnic_module.h"
 #include "btl_usnic_frag.h"
-#include "btl_usnic_ack.h"
+#include "btl_usnic_module.h"
 
-static void
-common_send_seg_helper(opal_btl_usnic_send_segment_t *seg)
+static void common_send_seg_helper(opal_btl_usnic_send_segment_t *seg)
 {
     opal_btl_usnic_segment_t *bseg;
 
@@ -43,57 +42,51 @@ common_send_seg_helper(opal_btl_usnic_send_segment_t *seg)
     /* Offset the BTL header by (prefix_send_offset) bytes into the
        raw buffer */
     bseg = &seg->ss_base;
-    bseg->us_btl_header = (opal_btl_usnic_btl_header_t *)
-        (seg->ss_ptr + mca_btl_usnic_component.prefix_send_offset);
+    bseg->us_btl_header = (opal_btl_usnic_btl_header_t
+                               *) (seg->ss_ptr + mca_btl_usnic_component.prefix_send_offset);
     bseg->us_btl_header->sender = mca_btl_usnic_component.my_hashed_rte_name;
 }
 
-static void
-chunk_seg_constructor(
-    opal_btl_usnic_chunk_segment_t *cseg)
+static void chunk_seg_constructor(opal_btl_usnic_chunk_segment_t *cseg)
 {
     opal_btl_usnic_segment_t *bseg;
 
     bseg = &cseg->ss_base;
     bseg->us_type = OPAL_BTL_USNIC_SEG_CHUNK;
 
-    /* some more common initializaiton */
+    /* some more common initialization */
     common_send_seg_helper(cseg);
 
     /* payload starts next byte beyond BTL chunk header */
-    bseg->us_payload.raw = (uint8_t *)(bseg->us_btl_chunk_header + 1);
+    bseg->us_payload.raw = (uint8_t *) (bseg->us_btl_chunk_header + 1);
 
     bseg->us_btl_header->payload_type = OPAL_BTL_USNIC_PAYLOAD_TYPE_CHUNK;
 }
 
-static void
-frag_seg_constructor(
-    opal_btl_usnic_frag_segment_t *fseg)
+static void frag_seg_constructor(opal_btl_usnic_frag_segment_t *fseg)
 {
     opal_btl_usnic_segment_t *bseg;
 
     bseg = &fseg->ss_base;
     bseg->us_type = OPAL_BTL_USNIC_SEG_FRAG;
 
-    /* some more common initializaiton */
+    /* some more common initialization */
     common_send_seg_helper(fseg);
 
     /* payload starts next byte beyond BTL header */
-    bseg->us_payload.raw = (uint8_t *)(bseg->us_btl_header + 1);
+    bseg->us_payload.raw = (uint8_t *) (bseg->us_btl_header + 1);
 
     bseg->us_btl_header->payload_type = OPAL_BTL_USNIC_PAYLOAD_TYPE_FRAG;
 }
 
-static void
-ack_seg_constructor(
-    opal_btl_usnic_ack_segment_t *ack)
+static void ack_seg_constructor(opal_btl_usnic_ack_segment_t *ack)
 {
     opal_btl_usnic_segment_t *bseg;
 
     bseg = &ack->ss_base;
     bseg->us_type = OPAL_BTL_USNIC_SEG_ACK;
 
-    /* some more common initializaiton */
+    /* some more common initialization */
     common_send_seg_helper(ack);
 
     /* ACK value embedded in BTL header */
@@ -104,10 +97,7 @@ ack_seg_constructor(
     ack->ss_len = sizeof(bseg->us_btl_header);
 }
 
-
-static void
-recv_seg_constructor(
-    opal_btl_usnic_recv_segment_t *seg)
+static void recv_seg_constructor(opal_btl_usnic_recv_segment_t *seg)
 {
     opal_btl_usnic_segment_t *bseg;
 
@@ -116,9 +106,9 @@ recv_seg_constructor(
 
     /* on receive, BTL header starts after protocol header */
     seg->rs_protocol_header = bseg->us_list.ptr;
-    bseg->us_btl_header = (opal_btl_usnic_btl_header_t *)(
-        ((char *) seg->rs_protocol_header) +
-        mca_btl_usnic_component.transport_header_len);
+    bseg->us_btl_header = (opal_btl_usnic_btl_header_t *) (((char *) seg->rs_protocol_header)
+                                                           + mca_btl_usnic_component
+                                                                 .transport_header_len);
 
     /* initialize descriptor */
     /* JMS Initializing RECV_REMOTE for receive frags is unnecessary
@@ -138,13 +128,11 @@ recv_seg_constructor(
      * OPAL_BTL_USNIC_PAYLOAD_TYPE_FRAG, but that's the only time
      * we ever give segment directly to upper layer, so its OK
      */
-    bseg->us_payload.ompi_header = (mca_btl_base_header_t *)
-        (bseg->us_btl_header+1);
+    bseg->us_payload.ompi_header = (mca_btl_base_header_t *) (bseg->us_btl_header + 1);
     seg->rs_segment.seg_addr.pval = bseg->us_payload.ompi_header;
 }
 
-static void
-send_frag_constructor(opal_btl_usnic_send_frag_t *frag)
+static void send_frag_constructor(opal_btl_usnic_send_frag_t *frag)
 {
     mca_btl_base_descriptor_t *desc;
 
@@ -172,8 +160,7 @@ send_frag_constructor(opal_btl_usnic_send_frag_t *frag)
     frag->sf_seg_post_cnt = 0;
 }
 
-static void
-send_frag_destructor(opal_btl_usnic_send_frag_t *frag)
+static void send_frag_destructor(opal_btl_usnic_send_frag_t *frag)
 {
 #if OPAL_ENABLE_DEBUG
     /* make sure nobody twiddled these values after the constructor */
@@ -189,8 +176,7 @@ send_frag_destructor(opal_btl_usnic_send_frag_t *frag)
     OBJ_DESTRUCT(&frag->sf_convertor);
 }
 
-static void
-small_send_frag_constructor(opal_btl_usnic_small_send_frag_t *frag)
+static void small_send_frag_constructor(opal_btl_usnic_small_send_frag_t *frag)
 {
     opal_btl_usnic_frag_segment_t *fseg;
 
@@ -201,44 +187,38 @@ small_send_frag_constructor(opal_btl_usnic_small_send_frag_t *frag)
     OBJ_CONSTRUCT(fseg, opal_btl_usnic_frag_segment_t);
 
     /* set us as parent in dedicated frag */
-    fseg->ss_parent_frag = (struct opal_btl_usnic_send_frag_t *)frag;
+    fseg->ss_parent_frag = (struct opal_btl_usnic_send_frag_t *) frag;
 
     frag->ssf_base.sf_base.uf_type = OPAL_BTL_USNIC_FRAG_SMALL_SEND;
 
     /* save data pointer for PML */
-    frag->ssf_base.sf_base.uf_local_seg[0].seg_addr.pval =
-        fseg->ss_base.us_payload.raw;
+    frag->ssf_base.sf_base.uf_local_seg[0].seg_addr.pval = fseg->ss_base.us_payload.raw;
 }
 
-static void
-small_send_frag_destructor(opal_btl_usnic_small_send_frag_t *frag)
+static void small_send_frag_destructor(opal_btl_usnic_small_send_frag_t *frag)
 {
     opal_btl_usnic_frag_segment_t *fseg;
 
     fseg = &frag->ssf_segment;
-    assert(fseg->ss_parent_frag == (struct opal_btl_usnic_send_frag_t *)frag);
+    assert(fseg->ss_parent_frag == (struct opal_btl_usnic_send_frag_t *) frag);
     assert(frag->ssf_base.sf_base.uf_type == OPAL_BTL_USNIC_FRAG_SMALL_SEND);
-    assert(frag->ssf_base.sf_base.uf_local_seg[0].seg_addr.pval ==
-           fseg->ss_base.us_payload.raw);
+    assert(frag->ssf_base.sf_base.uf_local_seg[0].seg_addr.pval == fseg->ss_base.us_payload.raw);
     OBJ_DESTRUCT(fseg);
 }
 
-static void
-large_send_frag_constructor(opal_btl_usnic_large_send_frag_t *lfrag)
+static void large_send_frag_constructor(opal_btl_usnic_large_send_frag_t *lfrag)
 {
     lfrag->lsf_base.sf_base.uf_type = OPAL_BTL_USNIC_FRAG_LARGE_SEND;
 
     /* save data pointer for upper layer */
-    lfrag->lsf_base.sf_base.uf_local_seg[0].seg_addr.pval =
-                    &lfrag->lsf_ompi_header;
+    lfrag->lsf_base.sf_base.uf_local_seg[0].seg_addr.pval = &lfrag->lsf_ompi_header;
 
     lfrag->lsf_buffer = NULL;
     OBJ_CONSTRUCT(&lfrag->lsf_seg_chain, opal_list_t);
     lfrag->lsf_pack_on_the_fly = false;
 }
 
-static void
-put_dest_frag_constructor(opal_btl_usnic_put_dest_frag_t *pfrag)
+static void put_dest_frag_constructor(opal_btl_usnic_put_dest_frag_t *pfrag)
 {
     pfrag->uf_type = OPAL_BTL_USNIC_FRAG_PUT_DEST;
 
@@ -247,67 +227,41 @@ put_dest_frag_constructor(opal_btl_usnic_put_dest_frag_t *pfrag)
     pfrag->uf_base.USNIC_PUT_LOCAL_COUNT = 1;
 }
 
-static void
-put_dest_frag_destructor(opal_btl_usnic_put_dest_frag_t *pfrag)
+static void put_dest_frag_destructor(opal_btl_usnic_put_dest_frag_t *pfrag)
 {
     assert(pfrag->uf_base.USNIC_PUT_LOCAL == pfrag->uf_remote_seg);
     assert(1 == pfrag->uf_base.USNIC_PUT_LOCAL_COUNT);
 }
 
-OBJ_CLASS_INSTANCE(opal_btl_usnic_segment_t,
-                   opal_free_list_item_t,
-                   NULL,
-                   NULL);
+OBJ_CLASS_INSTANCE(opal_btl_usnic_segment_t, opal_free_list_item_t, NULL, NULL);
 
-OBJ_CLASS_INSTANCE(opal_btl_usnic_frag_segment_t,
-                   opal_btl_usnic_segment_t,
-                   frag_seg_constructor,
+OBJ_CLASS_INSTANCE(opal_btl_usnic_frag_segment_t, opal_btl_usnic_segment_t, frag_seg_constructor,
                    NULL);
 
-OBJ_CLASS_INSTANCE(opal_btl_usnic_chunk_segment_t,
-                   opal_btl_usnic_segment_t,
-                   chunk_seg_constructor,
+OBJ_CLASS_INSTANCE(opal_btl_usnic_chunk_segment_t, opal_btl_usnic_segment_t, chunk_seg_constructor,
                    NULL);
 
-OBJ_CLASS_INSTANCE(opal_btl_usnic_recv_segment_t,
-                   opal_btl_usnic_segment_t,
-                   recv_seg_constructor,
+OBJ_CLASS_INSTANCE(opal_btl_usnic_recv_segment_t, opal_btl_usnic_segment_t, recv_seg_constructor,
                    NULL);
 
-OBJ_CLASS_INSTANCE(opal_btl_usnic_ack_segment_t,
-                   opal_btl_usnic_segment_t,
-                   ack_seg_constructor,
+OBJ_CLASS_INSTANCE(opal_btl_usnic_ack_segment_t, opal_btl_usnic_segment_t, ack_seg_constructor,
                    NULL);
 
 /*
  * Fragments
  */
-OBJ_CLASS_INSTANCE(opal_btl_usnic_frag_t,
-                   mca_btl_base_descriptor_t,
-                   NULL,
-                   NULL);
+OBJ_CLASS_INSTANCE(opal_btl_usnic_frag_t, mca_btl_base_descriptor_t, NULL, NULL);
 
-OBJ_CLASS_INSTANCE(opal_btl_usnic_send_frag_t,
-                   opal_btl_usnic_frag_t,
-                   send_frag_constructor,
+OBJ_CLASS_INSTANCE(opal_btl_usnic_send_frag_t, opal_btl_usnic_frag_t, send_frag_constructor,
                    send_frag_destructor);
 
-OBJ_CLASS_INSTANCE(opal_btl_usnic_large_send_frag_t,
-                   opal_btl_usnic_send_frag_t,
-                   large_send_frag_constructor,
-                   NULL);
+OBJ_CLASS_INSTANCE(opal_btl_usnic_large_send_frag_t, opal_btl_usnic_send_frag_t,
+                   large_send_frag_constructor, NULL);
 
-OBJ_CLASS_INSTANCE(opal_btl_usnic_small_send_frag_t,
-                   opal_btl_usnic_send_frag_t,
-                   small_send_frag_constructor,
-                   small_send_frag_destructor);
+OBJ_CLASS_INSTANCE(opal_btl_usnic_small_send_frag_t, opal_btl_usnic_send_frag_t,
+                   small_send_frag_constructor, small_send_frag_destructor);
 
-OBJ_CLASS_INSTANCE(opal_btl_usnic_put_dest_frag_t,
-                   opal_btl_usnic_frag_t,
-                   put_dest_frag_constructor,
+OBJ_CLASS_INSTANCE(opal_btl_usnic_put_dest_frag_t, opal_btl_usnic_frag_t, put_dest_frag_constructor,
                    put_dest_frag_destructor);
 
-OBJ_CLASS_INSTANCE(opal_btl_usnic_rx_buf_t,
-                   opal_free_list_item_t,
-                   NULL,
-                   NULL);
+OBJ_CLASS_INSTANCE(opal_btl_usnic_rx_buf_t, opal_free_list_item_t, NULL, NULL);
diff --git a/opal/mca/btl/usnic/btl_usnic_frag.h b/opal/mca/btl/usnic/btl_usnic_frag.h
index e9bdd8223c8..dc307c746be 100644
--- a/opal/mca/btl/usnic/btl_usnic_frag.h
+++ b/opal/mca/btl/usnic/btl_usnic_frag.h
@@ -44,7 +44,7 @@ struct opal_btl_usnic_module_t;
 
 /**
  * Fragment types
- * The upper layer may give us very large "fragements" to send, larger than
+ * The upper layer may give us very large "fragments" to send, larger than
  * an MTU.  We break fragments into segments for sending, a segment being
  * defined to fit within an MTU.
  */
@@ -54,14 +54,17 @@ typedef enum {
     OPAL_BTL_USNIC_FRAG_PUT_DEST
 } opal_btl_usnic_frag_type_t;
 
-static inline const char *
-usnic_frag_type(opal_btl_usnic_frag_type_t t)
+static inline const char *usnic_frag_type(opal_btl_usnic_frag_type_t t)
 {
     switch (t) {
-    case OPAL_BTL_USNIC_FRAG_LARGE_SEND: return "large";
-    case OPAL_BTL_USNIC_FRAG_SMALL_SEND: return "small";
-    case OPAL_BTL_USNIC_FRAG_PUT_DEST: return "put dest";
-    default: return "unknown";
+    case OPAL_BTL_USNIC_FRAG_LARGE_SEND:
+        return "large";
+    case OPAL_BTL_USNIC_FRAG_SMALL_SEND:
+        return "small";
+    case OPAL_BTL_USNIC_FRAG_PUT_DEST:
+        return "put dest";
+    default:
+        return "unknown";
     }
 }
 
@@ -72,19 +75,22 @@ typedef enum {
     OPAL_BTL_USNIC_SEG_RECV
 } opal_btl_usnic_seg_type_t;
 
-static inline const char *
-usnic_seg_type_str(opal_btl_usnic_seg_type_t t)
+static inline const char *usnic_seg_type_str(opal_btl_usnic_seg_type_t t)
 {
     switch (t) {
-    case OPAL_BTL_USNIC_SEG_ACK:   return "ACK";
-    case OPAL_BTL_USNIC_SEG_FRAG:  return "FRAG";
-    case OPAL_BTL_USNIC_SEG_CHUNK: return "CHUNK";
-    case OPAL_BTL_USNIC_SEG_RECV:  return "RECV";
-    default:                       return "unknown";
+    case OPAL_BTL_USNIC_SEG_ACK:
+        return "ACK";
+    case OPAL_BTL_USNIC_SEG_FRAG:
+        return "FRAG";
+    case OPAL_BTL_USNIC_SEG_CHUNK:
+        return "CHUNK";
+    case OPAL_BTL_USNIC_SEG_RECV:
+        return "RECV";
+    default:
+        return "unknown";
     }
 }
 
-
 /*
  * usnic registration handle (passed over the network to peers as a
  * cookie).
@@ -109,14 +115,13 @@ typedef struct opal_btl_usnic_reg_t {
     struct fid_mr *ur_mr;
 } opal_btl_usnic_reg_t;
 
-
 /**
  * usnic header type
  */
 typedef enum {
     OPAL_BTL_USNIC_PAYLOAD_TYPE_ACK = 1,
-    OPAL_BTL_USNIC_PAYLOAD_TYPE_FRAG = 2,       /* an entire fragment */
-    OPAL_BTL_USNIC_PAYLOAD_TYPE_CHUNK = 3       /* one chunk of fragment */
+    OPAL_BTL_USNIC_PAYLOAD_TYPE_FRAG = 2, /* an entire fragment */
+    OPAL_BTL_USNIC_PAYLOAD_TYPE_CHUNK = 3 /* one chunk of fragment */
 } opal_btl_usnic_payload_type_t;
 
 /**
@@ -131,10 +136,10 @@ typedef struct {
 
     /* Sliding window sequence number (echoed back in an ACK). */
     opal_btl_usnic_seq_t pkt_seq;
-    opal_btl_usnic_seq_t ack_seq;       /* for piggy-backing ACKs */
+    opal_btl_usnic_seq_t ack_seq; /* for piggy-backing ACKs */
 
-    /* payload legnth (in bytes).  We unfortunately have to include
-       this in our header because the L2 layer may artifically inflate
+    /* payload length (in bytes).  We unfortunately have to include
+       this in our header because the L2 layer may artificially inflate
        the length of the packet to meet a minimum size */
     uint16_t payload_len;
 
@@ -163,9 +168,9 @@ typedef struct {
 typedef struct {
     opal_btl_usnic_btl_header_t ch_hdr;
 
-    uint32_t ch_frag_id;        /* ID for collecting segments of same frag */
-    uint32_t ch_frag_size;      /* total frag len */
-    uint32_t ch_frag_offset;    /* where in fragment this goes */
+    uint32_t ch_frag_id;     /* ID for collecting segments of same frag */
+    uint32_t ch_frag_size;   /* total frag len */
+    uint32_t ch_frag_offset; /* where in fragment this goes */
 } opal_btl_usnic_btl_chunk_header_t;
 
 /**
@@ -182,7 +187,7 @@ typedef struct opal_btl_usnic_segment_t {
         opal_btl_usnic_btl_header_t *uus_btl_header;
         opal_btl_usnic_btl_chunk_header_t *uus_btl_chunk_header;
     } us_hdr;
-#define us_btl_header us_hdr.uus_btl_header
+#define us_btl_header       us_hdr.uus_btl_header
 #define us_btl_chunk_header us_hdr.uus_btl_chunk_header
 
     union {
@@ -226,11 +231,11 @@ typedef struct opal_btl_usnic_send_segment_t {
     opal_btl_usnic_channel_id_t ss_channel;
 
     struct opal_btl_usnic_send_frag_t *ss_parent_frag;
-    int ss_hotel_room;          /* current retrans room, or -1 if none */
+    int ss_hotel_room; /* current retrans room, or -1 if none */
 
     /* How many times is this frag on a hardware queue? */
     uint32_t ss_send_posted;
-    bool ss_ack_pending;        /* true until this segment is ACKed */
+    bool ss_ack_pending; /* true until this segment is ACKed */
 
 } opal_btl_usnic_send_segment_t;
 
@@ -262,13 +267,13 @@ typedef struct opal_btl_usnic_send_frag_t {
 
     struct mca_btl_base_endpoint_t *sf_endpoint;
 
-    size_t sf_size;             /* total_fragment size (upper + user payload) */
+    size_t sf_size; /* total_fragment size (upper + user payload) */
 
     struct opal_convertor_t sf_convertor; /* copy of original message data if
                                              convertor required */
 
-    uint32_t sf_seg_post_cnt;   /* total segs currently posted for this frag */
-    size_t sf_ack_bytes_left;   /* bytes remaining to be ACKed */
+    uint32_t sf_seg_post_cnt; /* total segs currently posted for this frag */
+    size_t sf_ack_bytes_left; /* bytes remaining to be ACKed */
 
     struct opal_btl_usnic_send_frag_t *sf_next;
 } opal_btl_usnic_send_frag_t;
@@ -284,7 +289,7 @@ typedef struct opal_btl_usnic_large_send_frag_t {
     char lsf_ompi_header[64];   /* space for upper layer header */
     mca_btl_base_tag_t lsf_tag; /* save tag */
 
-    uint32_t lsf_frag_id;       /* fragment ID for reassembly */
+    uint32_t lsf_frag_id; /* fragment ID for reassembly */
 
     size_t lsf_cur_offset;      /* next byte offset to be enqueued on the
                                    endpoint (incl. any convertor payload) */
@@ -296,16 +301,16 @@ typedef struct opal_btl_usnic_large_send_frag_t {
     int lsf_cur_sge;
     size_t lsf_bytes_left_in_sge;
 
-    uint8_t *lsf_buffer;        /* attached storage for usnic_alloc() */
+    uint8_t *lsf_buffer; /* attached storage for usnic_alloc() */
 
-    opal_list_t lsf_seg_chain;  /* chain of segments for converted data */
+    opal_list_t lsf_seg_chain; /* chain of segments for converted data */
 
-    bool lsf_pack_on_the_fly;   /* true if we are packing on the fly */
+    bool lsf_pack_on_the_fly; /* true if we are packing on the fly */
 } opal_btl_usnic_large_send_frag_t;
 
 /* Shortcut member macros.  Access uf_src_seg array instead of the descriptor's
  * des_src ptr to save a deref. */
-#define lsf_des_src       lsf_base.sf_base.uf_local_seg
+#define lsf_des_src lsf_base.sf_base.uf_local_seg
 
 /**
  * small send fragment
@@ -372,7 +377,7 @@ opal_btl_usnic_small_send_frag_alloc(opal_btl_usnic_module_t *module)
         return NULL;
     }
 
-    frag = (opal_btl_usnic_small_send_frag_t*) item;
+    frag = (opal_btl_usnic_small_send_frag_t *) item;
 
     /* this belongs in constructor... */
     frag->ssf_base.sf_base.uf_freelist = &(module->small_send_frags);
@@ -395,7 +400,7 @@ opal_btl_usnic_large_send_frag_alloc(opal_btl_usnic_module_t *module)
         return NULL;
     }
 
-    frag = (opal_btl_usnic_large_send_frag_t*) item;
+    frag = (opal_btl_usnic_large_send_frag_t *) item;
 
     /* this belongs in constructor... */
     frag->lsf_base.sf_base.uf_freelist = &(module->large_send_frags);
@@ -407,8 +412,7 @@ opal_btl_usnic_large_send_frag_alloc(opal_btl_usnic_module_t *module)
 }
 
 static inline opal_btl_usnic_put_dest_frag_t *
-opal_btl_usnic_put_dest_frag_alloc(
-    struct opal_btl_usnic_module_t *module)
+opal_btl_usnic_put_dest_frag_alloc(struct opal_btl_usnic_module_t *module)
 {
     opal_free_list_item_t *item;
     opal_btl_usnic_put_dest_frag_t *frag;
@@ -418,7 +422,7 @@ opal_btl_usnic_put_dest_frag_alloc(
         return NULL;
     }
 
-    frag = (opal_btl_usnic_put_dest_frag_t*) item;
+    frag = (opal_btl_usnic_put_dest_frag_t *) item;
 
     /* this belongs in constructor... */
     frag->uf_freelist = &(module->put_dest_frags);
@@ -439,31 +443,24 @@ opal_btl_usnic_put_dest_frag_alloc(
  *    b) all of its segments have been ACKed
  *    c) it is owned by the BTL
  */
-static inline bool
-opal_btl_usnic_send_frag_ok_to_return(
-    opal_btl_usnic_module_t *module,
-    opal_btl_usnic_send_frag_t *frag)
+static inline bool opal_btl_usnic_send_frag_ok_to_return(opal_btl_usnic_module_t *module,
+                                                         opal_btl_usnic_send_frag_t *frag)
 {
     assert(frag);
 
-    if (OPAL_LIKELY(frag->sf_base.uf_base.des_flags &
-                MCA_BTL_DES_FLAGS_BTL_OWNERSHIP) &&
-        0 == frag->sf_ack_bytes_left &&
-        0 == frag->sf_seg_post_cnt) {
+    if (OPAL_LIKELY(frag->sf_base.uf_base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP)
+        && 0 == frag->sf_ack_bytes_left && 0 == frag->sf_seg_post_cnt) {
         return true;
     }
 
     return false;
 }
 
-static inline void
-opal_btl_usnic_frag_return(
-    struct opal_btl_usnic_module_t *module,
-    opal_btl_usnic_frag_t *frag)
+static inline void opal_btl_usnic_frag_return(struct opal_btl_usnic_module_t *module,
+                                              opal_btl_usnic_frag_t *frag)
 {
 #if MSGDEBUG1
-    opal_output(0, "freeing frag %p, type %s\n", (void *)frag,
-            usnic_frag_type(frag->uf_type));
+    opal_output(0, "freeing frag %p, type %s\n", (void *) frag, usnic_frag_type(frag->uf_type));
 #endif
     frag->uf_local_seg[0].seg_len = 0;
     frag->uf_local_seg[1].seg_len = 0;
@@ -473,7 +470,7 @@ opal_btl_usnic_frag_return(
      */
     if (frag->uf_type == OPAL_BTL_USNIC_FRAG_LARGE_SEND) {
         opal_btl_usnic_large_send_frag_t *lfrag;
-        lfrag = (opal_btl_usnic_large_send_frag_t *)frag;
+        lfrag = (opal_btl_usnic_large_send_frag_t *) frag;
         if (lfrag->lsf_buffer != NULL) {
             free(lfrag->lsf_buffer);
             lfrag->lsf_buffer = NULL;
@@ -481,8 +478,8 @@ opal_btl_usnic_frag_return(
         lfrag->lsf_pack_on_the_fly = false;
 
         /* JMS This should never happen any more, right? */
-        if (2 == lfrag->lsf_base.sf_base.uf_base.USNIC_SEND_LOCAL_COUNT &&
-            NULL == lfrag->lsf_des_src[1].seg_addr.pval) {
+        if (2 == lfrag->lsf_base.sf_base.uf_base.USNIC_SEND_LOCAL_COUNT
+            && NULL == lfrag->lsf_des_src[1].seg_addr.pval) {
             opal_convertor_cleanup(&lfrag->lsf_base.sf_convertor);
         }
     }
@@ -501,10 +498,8 @@ opal_btl_usnic_frag_return(
 /*
  * Return a send frag if it's all done and owned by BTL
  */
-static inline void
-opal_btl_usnic_send_frag_return_cond(
-    struct opal_btl_usnic_module_t *module,
-    opal_btl_usnic_send_frag_t *frag)
+static inline void opal_btl_usnic_send_frag_return_cond(struct opal_btl_usnic_module_t *module,
+                                                        opal_btl_usnic_send_frag_t *frag)
 {
     if (opal_btl_usnic_send_frag_ok_to_return(module, frag)) {
         opal_btl_usnic_frag_return(module, &frag->sf_base);
@@ -517,25 +512,20 @@ opal_btl_usnic_send_frag_return_cond(
  * a send frag, there are other conditions, so use the specific send frag
  * return checker.
  */
-static inline void
-opal_btl_usnic_frag_return_cond(
-    struct opal_btl_usnic_module_t *module,
-    opal_btl_usnic_frag_t *frag)
+static inline void opal_btl_usnic_frag_return_cond(struct opal_btl_usnic_module_t *module,
+                                                   opal_btl_usnic_frag_t *frag)
 {
     if (OPAL_BTL_USNIC_FRAG_PUT_DEST == frag->uf_type) {
-        if (OPAL_LIKELY(frag->uf_base.des_flags &
-                                    MCA_BTL_DES_FLAGS_BTL_OWNERSHIP)) {
+        if (OPAL_LIKELY(frag->uf_base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP)) {
             opal_btl_usnic_frag_return(module, frag);
         }
     } else {
-        opal_btl_usnic_send_frag_return_cond(module,
-                (opal_btl_usnic_send_frag_t *)frag);
+        opal_btl_usnic_send_frag_return_cond(module, (opal_btl_usnic_send_frag_t *) frag);
     }
 }
 
 static inline opal_btl_usnic_chunk_segment_t *
-opal_btl_usnic_chunk_segment_alloc(
-    opal_btl_usnic_module_t *module)
+opal_btl_usnic_chunk_segment_alloc(opal_btl_usnic_module_t *module)
 {
     opal_free_list_item_t *item;
     opal_btl_usnic_send_segment_t *seg;
@@ -545,7 +535,7 @@ opal_btl_usnic_chunk_segment_alloc(
         return NULL;
     }
 
-    seg = (opal_btl_usnic_send_segment_t*) item;
+    seg = (opal_btl_usnic_send_segment_t *) item;
     seg->ss_channel = USNIC_DATA_CHANNEL;
 
     assert(seg);
@@ -554,10 +544,8 @@ opal_btl_usnic_chunk_segment_alloc(
     return seg;
 }
 
-static inline void
-opal_btl_usnic_chunk_segment_return(
-    opal_btl_usnic_module_t *module,
-    opal_btl_usnic_chunk_segment_t *seg)
+static inline void opal_btl_usnic_chunk_segment_return(opal_btl_usnic_module_t *module,
+                                                       opal_btl_usnic_chunk_segment_t *seg)
 {
     assert(seg);
     assert(OPAL_BTL_USNIC_SEG_CHUNK == seg->ss_base.us_type);
@@ -579,7 +567,7 @@ opal_btl_usnic_ack_segment_alloc(opal_btl_usnic_module_t *module)
         return NULL;
     }
 
-    ack = (opal_btl_usnic_ack_segment_t*) item;
+    ack = (opal_btl_usnic_ack_segment_t *) item;
     ack->ss_channel = USNIC_PRIORITY_CHANNEL;
 
     assert(ack);
@@ -591,10 +579,8 @@ opal_btl_usnic_ack_segment_alloc(opal_btl_usnic_module_t *module)
 /*
  * Return an ACK segment
  */
-static inline void
-opal_btl_usnic_ack_segment_return(
-    opal_btl_usnic_module_t *module,
-    opal_btl_usnic_ack_segment_t *ack)
+static inline void opal_btl_usnic_ack_segment_return(opal_btl_usnic_module_t *module,
+                                                     opal_btl_usnic_ack_segment_t *ack)
 {
     assert(ack);
     assert(OPAL_BTL_USNIC_SEG_ACK == ack->ss_base.us_type);
@@ -605,8 +591,7 @@ opal_btl_usnic_ack_segment_return(
 /* Compute and set the proper value for sfrag->sf_size.  This must not be used
  * during usnic_alloc, since the PML might change the segment size after
  * usnic_alloc returns. */
-static inline void
-opal_btl_usnic_compute_sf_size(opal_btl_usnic_send_frag_t *sfrag)
+static inline void opal_btl_usnic_compute_sf_size(opal_btl_usnic_send_frag_t *sfrag)
 {
     opal_btl_usnic_frag_t *frag;
 
diff --git a/opal/mca/btl/usnic/btl_usnic_hwloc.c b/opal/mca/btl/usnic/btl_usnic_hwloc.c
index 88768f4116b..4d67e423255 100644
--- a/opal/mca/btl/usnic/btl_usnic_hwloc.c
+++ b/opal/mca/btl/usnic/btl_usnic_hwloc.c
@@ -10,8 +10,8 @@
 
 #include "opal_config.h"
 
-#include "opal/mca/hwloc/base/base.h"
 #include "opal/constants.h"
+#include "opal/mca/hwloc/base/base.h"
 
 #include "opal/mca/btl/base/base.h"
 
@@ -37,15 +37,15 @@ static int get_distance_matrix(void)
      * responsible for freeing it. */
 
     if (NULL == matrix) {
-        matrix = hwloc_get_whole_distance_matrix_by_type(opal_hwloc_topology,
-                                                         HWLOC_OBJ_NODE);
+        matrix = hwloc_get_whole_distance_matrix_by_type(opal_hwloc_topology, HWLOC_OBJ_NODE);
     }
 
     return (NULL == matrix) ? OPAL_ERROR : OPAL_SUCCESS;
 #else
-    if (0 != hwloc_distances_get_by_type(opal_hwloc_topology, HWLOC_OBJ_NODE,
-                                         &matrix_nr, &matrix,
-                                         HWLOC_DISTANCES_KIND_MEANS_LATENCY, 0) || 0 == matrix_nr) {
+    if (0
+            != hwloc_distances_get_by_type(opal_hwloc_topology, HWLOC_OBJ_NODE, &matrix_nr, &matrix,
+                                           HWLOC_DISTANCES_KIND_MEANS_LATENCY, 0)
+        || 0 == matrix_nr) {
         return OPAL_ERROR;
     }
     return OPAL_SUCCESS;
@@ -59,28 +59,27 @@ static hwloc_obj_t find_numa_node(hwloc_bitmap_t cpuset)
 {
     hwloc_obj_t obj;
 
-    obj =
-        hwloc_get_first_largest_obj_inside_cpuset(opal_hwloc_topology, cpuset);
+    obj = hwloc_get_first_largest_obj_inside_cpuset(opal_hwloc_topology, cpuset);
 
     /* Go upwards until we hit the NUMA node or run out of parents */
-    while (obj->type > HWLOC_OBJ_NODE &&
-           NULL != obj->parent) {
+    while (obj->type > HWLOC_OBJ_NODE && NULL != obj->parent) {
         obj = obj->parent;
     }
 
     /* Make sure we ended up on the NUMA node */
     if (obj->type != HWLOC_OBJ_NODE) {
         opal_output_verbose(5, USNIC_OUT,
-                            "btl:usnic:filter_numa: could not find NUMA node where this process is bound; filtering by NUMA distance not possible");
+                            "btl:usnic:filter_numa: could not find NUMA node where this process is "
+                            "bound; filtering by NUMA distance not possible");
         return NULL;
     }
 
     /* Finally, make sure that our cpuset doesn't span more than 1
        NUMA node */
-    if (hwloc_get_nbobjs_inside_cpuset_by_type(opal_hwloc_topology,
-                                               cpuset, HWLOC_OBJ_NODE) > 1) {
+    if (hwloc_get_nbobjs_inside_cpuset_by_type(opal_hwloc_topology, cpuset, HWLOC_OBJ_NODE) > 1) {
         opal_output_verbose(5, USNIC_OUT,
-                            "btl:usnic:filter_numa: this process is bound to more than 1 NUMA node; filtering by NUMA distance not possible");
+                            "btl:usnic:filter_numa: this process is bound to more than 1 NUMA "
+                            "node; filtering by NUMA distance not possible");
         return NULL;
     }
 
@@ -124,10 +123,8 @@ static int find_my_numa_node(void)
 
     /* Happiness */
     my_numa_node = obj;
-    num_numa_nodes = hwloc_get_nbobjs_by_type(opal_hwloc_topology,
-                                              HWLOC_OBJ_NODE);
+    num_numa_nodes = hwloc_get_nbobjs_by_type(opal_hwloc_topology, HWLOC_OBJ_NODE);
     return OPAL_SUCCESS;
-
 }
 
 /*
@@ -162,15 +159,15 @@ static hwloc_obj_t find_device_numa(opal_btl_usnic_module_t *module)
 
     /* Search upwards to find the device's NUMA node */
     /* Go upwards until we hit the NUMA node or run out of parents */
-    while (obj->type > HWLOC_OBJ_NODE &&
-           NULL != obj->parent) {
+    while (obj->type > HWLOC_OBJ_NODE && NULL != obj->parent) {
         obj = obj->parent;
     }
 
     /* Make sure we ended up on the NUMA node */
     if (obj->type != HWLOC_OBJ_NODE) {
         opal_output_verbose(5, USNIC_OUT,
-                            "btl:usnic:filter_numa: could not find NUMA node for %s; filtering by NUMA distance not possible",
+                            "btl:usnic:filter_numa: could not find NUMA node for %s; filtering by "
+                            "NUMA distance not possible",
                             module->linux_device_name);
         return NULL;
     }
@@ -192,18 +189,19 @@ int opal_btl_usnic_hwloc_distance(opal_btl_usnic_module_t *module)
 
     /* Is this process bound? */
     if (!proc_bound()) {
-        opal_output_verbose(5, USNIC_OUT,
-                            "btl:usnic:filter_numa: not sorting devices by NUMA distance (process not bound)");
+        opal_output_verbose(
+            5, USNIC_OUT,
+            "btl:usnic:filter_numa: not sorting devices by NUMA distance (process not bound)");
         return OPAL_SUCCESS;
     }
 
-    opal_output_verbose(5, USNIC_OUT,
-                        "btl:usnic:filter_numa: filtering devices by NUMA distance");
+    opal_output_verbose(5, USNIC_OUT, "btl:usnic:filter_numa: filtering devices by NUMA distance");
 
     /* ensure we have the topology */
     if (OPAL_SUCCESS != opal_hwloc_base_get_topology()) {
-        opal_output_verbose(5, USNIC_OUT,
-                            "btl:usnic:filter_numa: not sorting devices by NUMA distance (topology not available)");
+        opal_output_verbose(
+            5, USNIC_OUT,
+            "btl:usnic:filter_numa: not sorting devices by NUMA distance (topology not available)");
         return OPAL_SUCCESS;
     }
 
@@ -229,21 +227,18 @@ int opal_btl_usnic_hwloc_distance(opal_btl_usnic_module_t *module)
        the device */
 #if HWLOC_API_VERSION < 0x20000
     if (NULL != dev_numa) {
-        module->numa_distance =
-            matrix->latency[dev_numa->logical_index * num_numa_nodes +
-                            my_numa_node->logical_index];
+        module->numa_distance = matrix->latency[dev_numa->logical_index * num_numa_nodes
+                                                + my_numa_node->logical_index];
 
-        opal_output_verbose(5, USNIC_OUT,
-                            "btl:usnic:filter_numa: %s is distance %d from me",
-                            module->linux_device_name,
-                            module->numa_distance);
+        opal_output_verbose(5, USNIC_OUT, "btl:usnic:filter_numa: %s is distance %d from me",
+                            module->linux_device_name, module->numa_distance);
     }
 #else
     if (NULL != dev_numa) {
         int myindex, devindex;
         unsigned int j;
         myindex = -1;
-        for (j=0; j < matrix_nr; j++) {
+        for (j = 0; j < matrix_nr; j++) {
             if (my_numa_node == matrix->objs[j]) {
                 myindex = j;
                 break;
@@ -253,7 +248,7 @@ int opal_btl_usnic_hwloc_distance(opal_btl_usnic_module_t *module)
             return OPAL_SUCCESS;
         }
         devindex = -1;
-        for (j=0; j < matrix_nr; j++) {
+        for (j = 0; j < matrix_nr; j++) {
             if (dev_numa == matrix->objs[j]) {
                 devindex = j;
                 break;
@@ -263,13 +258,10 @@ int opal_btl_usnic_hwloc_distance(opal_btl_usnic_module_t *module)
             return OPAL_SUCCESS;
         }
 
-        module->numa_distance =
-            matrix->values[(devindex * num_numa_nodes) + myindex];
+        module->numa_distance = matrix->values[(devindex * num_numa_nodes) + myindex];
 
-        opal_output_verbose(5, USNIC_OUT,
-                            "btl:usnic:filter_numa: %s is distance %d from me",
-                            module->linux_device_name,
-                            module->numa_distance);
+        opal_output_verbose(5, USNIC_OUT, "btl:usnic:filter_numa: %s is distance %d from me",
+                            module->linux_device_name, module->numa_distance);
     }
 #endif
 
diff --git a/opal/mca/btl/usnic/btl_usnic_hwloc.h b/opal/mca/btl/usnic/btl_usnic_hwloc.h
index 80bc2c7ac77..87ac494ddca 100644
--- a/opal/mca/btl/usnic/btl_usnic_hwloc.h
+++ b/opal/mca/btl/usnic/btl_usnic_hwloc.h
@@ -14,7 +14,6 @@
 
 #include "btl_usnic_module.h"
 
-
 int opal_btl_usnic_hwloc_distance(opal_btl_usnic_module_t *module);
 
 #endif /* BTL_USNIC_HWLOC_H */
diff --git a/opal/mca/btl/usnic/btl_usnic_map.c b/opal/mca/btl/usnic/btl_usnic_map.c
index b9d074efdf3..948b639a23c 100644
--- a/opal/mca/btl/usnic/btl_usnic_map.c
+++ b/opal/mca/btl/usnic/btl_usnic_map.c
@@ -2,6 +2,7 @@
  * Copyright (c) 2013-2016 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2023      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -15,25 +16,24 @@
 #include <string.h>
 #include <unistd.h>
 
-#include "opal/util/show_help.h"
 #include "opal/util/printf.h"
+#include "opal/util/show_help.h"
 
-#include "btl_usnic_compat.h"
 #include "btl_usnic.h"
+#include "btl_usnic_compat.h"
 #include "btl_usnic_module.h"
-#include "btl_usnic_util.h"
 #include "btl_usnic_proc.h"
+#include "btl_usnic_util.h"
 
 /*
  * qsort helper: compare modules by fabric name
  */
 static int map_compare_modules(const void *aa, const void *bb)
 {
-    opal_btl_usnic_module_t *a = *((opal_btl_usnic_module_t**) aa);
-    opal_btl_usnic_module_t *b = *((opal_btl_usnic_module_t**) bb);
+    opal_btl_usnic_module_t *a = *((opal_btl_usnic_module_t **) aa);
+    opal_btl_usnic_module_t *b = *((opal_btl_usnic_module_t **) bb);
 
-    return strcmp(a->linux_device_name,
-                  b->linux_device_name);
+    return strcmp(a->linux_device_name, b->linux_device_name);
 }
 
 /*
@@ -53,17 +53,15 @@ static int map_output_modules(FILE *fp)
 
     /* First, we must sort the modules (by device name) so that
        they're always output in a repeatable order. */
-    size = mca_btl_usnic_component.num_modules *
-        sizeof(opal_btl_usnic_module_t*);
+    size = mca_btl_usnic_component.num_modules * sizeof(opal_btl_usnic_module_t *);
     modules = calloc(1, size);
     if (NULL == modules) {
         return OPAL_ERR_IN_ERRNO;
     }
 
     memcpy(modules, mca_btl_usnic_component.usnic_active_modules, size);
-    qsort(modules, mca_btl_usnic_component.num_modules,
-          sizeof(opal_btl_usnic_module_t*), map_compare_modules);
-
+    qsort(modules, mca_btl_usnic_component.num_modules, sizeof(opal_btl_usnic_module_t *),
+          map_compare_modules);
 
     /* Loop over and print the sorted module device information */
     for (i = 0; i < mca_btl_usnic_component.num_modules; ++i) {
@@ -71,13 +69,10 @@ static int map_output_modules(FILE *fp)
         sin = modules[i]->fabric_info->src_addr;
         prefix_len = usnic_netmask_to_cidrlen(uip->ui.v1.ui_netmask_be);
 
-        opal_btl_usnic_snprintf_ipv4_addr(ipv4, IPV4STRADDRLEN,
-                                        sin->sin_addr.s_addr,
-                                        prefix_len);
+        opal_btl_usnic_snprintf_ipv4_addr(ipv4, IPV4STRADDRLEN, sin->sin_addr.s_addr, prefix_len);
 
-        fprintf(fp, "device=%s,ip=%s,mss=%" PRIsize_t "\n",
-                modules[i]->linux_device_name,
-                ipv4, modules[i]->fabric_info->ep_attr->max_msg_size);
+        fprintf(fp, "device=%s,ip=%s,mss=%" PRIsize_t "\n", modules[i]->linux_device_name, ipv4,
+                modules[i]->fabric_info->ep_attr->max_msg_size);
     }
 
     /* Free the temp array */
@@ -93,8 +88,8 @@ static int map_output_modules(FILE *fp)
  */
 static int map_compare_endpoints(const void *aa, const void *bb)
 {
-    opal_btl_usnic_endpoint_t *a = *((opal_btl_usnic_endpoint_t**) aa);
-    opal_btl_usnic_endpoint_t *b = *((opal_btl_usnic_endpoint_t**) bb);
+    opal_btl_usnic_endpoint_t *a = *((opal_btl_usnic_endpoint_t **) aa);
+    opal_btl_usnic_endpoint_t *b = *((opal_btl_usnic_endpoint_t **) bb);
 
     if (NULL == a && NULL == b) {
         return 0;
@@ -104,8 +99,7 @@ static int map_compare_endpoints(const void *aa, const void *bb)
         return -1;
     }
 
-    return strcmp(a->endpoint_module->linux_device_name,
-                  b->endpoint_module->linux_device_name);
+    return strcmp(a->endpoint_module->linux_device_name, b->endpoint_module->linux_device_name);
 }
 
 /*
@@ -131,8 +125,7 @@ static int map_output_endpoints(FILE *fp, opal_btl_usnic_proc_t *proc)
     }
 
     memcpy(eps, proc->proc_endpoints, size);
-    qsort(eps, proc->proc_endpoint_count,
-          sizeof(opal_btl_usnic_endpoint_t*),
+    qsort(eps, proc->proc_endpoint_count, sizeof(opal_btl_usnic_endpoint_t *),
           map_compare_endpoints);
 
     /* Loop over and print the sorted endpoint information, ignoring
@@ -149,9 +142,7 @@ static int map_output_endpoints(FILE *fp, opal_btl_usnic_proc_t *proc)
                                           eps[i]->endpoint_remote_modex.ipv4_addr,
                                           eps[i]->endpoint_remote_modex.netmask);
 
-        fprintf(fp, "device=%s@peer_ip=%s",
-                eps[i]->endpoint_module->linux_device_name,
-                ipv4);
+        fprintf(fp, "device=%s@peer_ip=%s", eps[i]->endpoint_module->linux_device_name, ipv4);
         ++num_output;
     }
     fprintf(fp, "\n");
@@ -169,8 +160,8 @@ static int map_output_endpoints(FILE *fp, opal_btl_usnic_proc_t *proc)
  */
 static int map_compare_procs(const void *aa, const void *bb)
 {
-    opal_btl_usnic_proc_t *a = *((opal_btl_usnic_proc_t**) aa);
-    opal_btl_usnic_proc_t *b = *((opal_btl_usnic_proc_t**) bb);
+    opal_btl_usnic_proc_t *a = *((opal_btl_usnic_proc_t **) aa);
+    opal_btl_usnic_proc_t *b = *((opal_btl_usnic_proc_t **) bb);
     opal_process_name_t *an = &(a->proc_opal->proc_name);
     opal_process_name_t *bn = &(b->proc_opal->proc_name);
 
@@ -198,19 +189,17 @@ static int map_output_procs(FILE *fp)
     /* First, we must sort the procs by MCW rank so that they're
        always output in a repeatable order. */
     num_procs = opal_list_get_size(&mca_btl_usnic_component.usnic_procs);
-    procs = calloc(num_procs, sizeof(opal_btl_usnic_proc_t*));
+    procs = calloc(num_procs, sizeof(opal_btl_usnic_proc_t *));
     if (NULL == procs) {
         return OPAL_ERR_IN_ERRNO;
     }
 
     i = 0;
-    OPAL_LIST_FOREACH(pitem, &mca_btl_usnic_component.usnic_procs,
-                      opal_btl_usnic_proc_t) {
+    OPAL_LIST_FOREACH (pitem, &mca_btl_usnic_component.usnic_procs, opal_btl_usnic_proc_t) {
         procs[i] = pitem;
         ++i;
     }
-    qsort(procs, num_procs, sizeof(opal_btl_usnic_proc_t*),
-          map_compare_procs);
+    qsort(procs, num_procs, sizeof(opal_btl_usnic_proc_t *), map_compare_procs);
 
     /* Loop over and print the sorted module device information */
     int ret = OPAL_SUCCESS;
@@ -247,11 +236,9 @@ void opal_btl_usnic_connectivity_map(void)
     /* Filename is of the form: <prefix>-<hostname>.<pid>.<job>.<MCW
        rank>.txt */
     opal_asprintf(&filename, "%s-%s.pid%d.job%d.mcwrank%d.txt",
-             mca_btl_usnic_component.connectivity_map_prefix,
-             opal_process_info.nodename,
-             getpid(),
-             opal_proc_local_get()->proc_name.jobid,
-             opal_proc_local_get()->proc_name.vpid);
+                  mca_btl_usnic_component.connectivity_map_prefix, opal_process_info.nodename,
+                  getpid(), opal_proc_local_get()->proc_name.jobid,
+                  opal_proc_local_get()->proc_name.vpid);
     if (NULL == filename) {
         /* JMS abort? */
         return;
@@ -259,15 +246,11 @@ void opal_btl_usnic_connectivity_map(void)
 
     fp = fopen(filename, "w");
     if (NULL == fp) {
-        char dirname[PATH_MAX];
+        char dirname[OPAL_PATH_MAX];
         getcwd(dirname, sizeof(dirname));
         dirname[sizeof(dirname) - 1] = '\0';
-        opal_show_help("help-mpi-btl-usnic.txt", "cannot write to map file",
-                       true,
-                       opal_process_info.nodename,
-                       filename,
-                       dirname,
-                       strerror(errno), errno);
+        opal_show_help("help-mpi-btl-usnic.txt", "cannot write to map file", true,
+                       opal_process_info.nodename, filename, dirname, strerror(errno), errno);
         return;
     }
 
diff --git a/opal/mca/btl/usnic/btl_usnic_mca.c b/opal/mca/btl/usnic/btl_usnic_mca.c
index 76ff5595512..387eaa70e57 100644
--- a/opal/mca/btl/usnic/btl_usnic_mca.c
+++ b/opal/mca/btl/usnic/btl_usnic_mca.c
@@ -25,23 +25,22 @@
 
 #include "opal_config.h"
 
-#include <string.h>
 #include <errno.h>
+#include <string.h>
 
 #include "opal/mca/base/mca_base_var.h"
 #include "opal/util/argv.h"
 
 #include "opal/constants.h"
 
-#include "opal/mca/btl/btl.h"
 #include "opal/mca/btl/base/base.h"
+#include "opal/mca/btl/btl.h"
 
 #include "btl_usnic.h"
-#include "btl_usnic_frag.h"
 #include "btl_usnic_endpoint.h"
+#include "btl_usnic_frag.h"
 #include "btl_usnic_module.h"
 
-
 /*
  * Local flags
  */
@@ -54,99 +53,69 @@ enum {
     REGINT_MAX = 0x88
 };
 
-
 enum {
     REGSTR_EMPTY_OK = 0x01,
 
     REGSTR_MAX = 0x88
 };
 
-
 /*
  * utility routine for string parameter registration
  */
-static int reg_string(const char* param_name,
-                      const char* help_string,
-                      const char* default_value, char **storage,
-                      int flags, int level)
+static int reg_string(const char *param_name, const char *help_string, const char *default_value,
+                      char **storage, int flags, int level)
 {
-    *storage = (char*) default_value;
-    mca_base_component_var_register(&mca_btl_usnic_component.super.btl_version,
-                                    param_name, help_string,
-                                    MCA_BASE_VAR_TYPE_STRING,
-                                    NULL,
-                                    0,
-                                    0,
-                                    level,
-                                    MCA_BASE_VAR_SCOPE_READONLY,
-                                    storage);
-
-    if (0 == (flags & REGSTR_EMPTY_OK) &&
-        (NULL == *storage || 0 == strlen(*storage))) {
-        opal_output(0, "Bad parameter value for parameter \"%s\"",
-                    param_name);
+    *storage = (char *) default_value;
+    mca_base_component_var_register(&mca_btl_usnic_component.super.btl_version, param_name,
+                                    help_string, MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, level,
+                                    MCA_BASE_VAR_SCOPE_READONLY, storage);
+
+    if (0 == (flags & REGSTR_EMPTY_OK) && (NULL == *storage || 0 == strlen(*storage))) {
+        opal_output(0, "Bad parameter value for parameter \"%s\"", param_name);
         return OPAL_ERR_BAD_PARAM;
     }
 
     return OPAL_SUCCESS;
 }
 
-
 /*
  * utility routine for integer parameter registration
  */
-static int reg_int(const char* param_name,
-                   const char* help_string,
-                   int default_value, int *storage, int flags, int level)
+static int reg_int(const char *param_name, const char *help_string, int default_value, int *storage,
+                   int flags, int level)
 {
     *storage = default_value;
-    mca_base_component_var_register(&mca_btl_usnic_component.super.btl_version,
-                                    param_name, help_string,
-                                    MCA_BASE_VAR_TYPE_INT,
-                                    NULL,
-                                    0,
-                                    0,
-                                    level,
-                                    MCA_BASE_VAR_SCOPE_READONLY,
-                                    storage);
+    mca_base_component_var_register(&mca_btl_usnic_component.super.btl_version, param_name,
+                                    help_string, MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, level,
+                                    MCA_BASE_VAR_SCOPE_READONLY, storage);
 
     if (0 != (flags & REGINT_NEG_ONE_OK) && -1 == *storage) {
         return OPAL_SUCCESS;
     }
-    if ((0 != (flags & REGINT_GE_ZERO) && *storage < 0) ||
-        (0 != (flags & REGINT_GE_ONE) && *storage < 1) ||
-        (0 != (flags & REGINT_NONZERO) && 0 == *storage)) {
-        opal_output(0, "Bad parameter value for parameter \"%s\"",
-                    param_name);
+    if ((0 != (flags & REGINT_GE_ZERO) && *storage < 0)
+        || (0 != (flags & REGINT_GE_ONE) && *storage < 1)
+        || (0 != (flags & REGINT_NONZERO) && 0 == *storage)) {
+        opal_output(0, "Bad parameter value for parameter \"%s\"", param_name);
         return OPAL_ERR_BAD_PARAM;
     }
 
     return OPAL_SUCCESS;
 }
 
-
 /*
  * utility routine for integer parameter registration
  */
-static int reg_bool(const char* param_name,
-                    const char* help_string,
-                    bool default_value, bool *storage, int level)
+static int reg_bool(const char *param_name, const char *help_string, bool default_value,
+                    bool *storage, int level)
 {
     *storage = default_value;
-    mca_base_component_var_register(&mca_btl_usnic_component.super.btl_version,
-                                    param_name, help_string,
-                                    MCA_BASE_VAR_TYPE_BOOL,
-                                    NULL,
-                                    0,
-                                    0,
-                                    level,
-                                    MCA_BASE_VAR_SCOPE_READONLY,
-                                    storage);
+    mca_base_component_var_register(&mca_btl_usnic_component.super.btl_version, param_name,
+                                    help_string, MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, level,
+                                    MCA_BASE_VAR_SCOPE_READONLY, storage);
 
     return OPAL_SUCCESS;
 }
 
-
 int opal_btl_usnic_component_register(void)
 {
     int tmp, ret = 0;
@@ -166,119 +135,144 @@ int opal_btl_usnic_component_register(void)
     static int pack_lazy_threshold;
     static int max_short_packets;
 
-#define CHECK(expr) do {\
-        tmp = (expr); \
-        if (OPAL_SUCCESS != tmp) ret = tmp; \
-     } while (0)
+#define CHECK(expr)              \
+    do {                         \
+        tmp = (expr);            \
+        if (OPAL_SUCCESS != tmp) \
+            ret = tmp;           \
+    } while (0)
 
     CHECK(reg_int("max_btls",
-                  "Maximum number of usNICs to use (default: 0 = as many as are available)",
-                  0, &max_modules,
-                  REGINT_GE_ZERO, OPAL_INFO_LVL_2));
+                  "Maximum number of usNICs to use (default: 0 = as many as are available)", 0,
+                  &max_modules, REGINT_GE_ZERO, OPAL_INFO_LVL_2));
     mca_btl_usnic_component.max_modules = (size_t) max_modules;
 
     CHECK(reg_string("if_include",
-                     "Comma-delimited list of usNIC devices/networks to be used (e.g. \"eth3,usnic_0,10.10.0.0/16\"; empty value means to use all available usNICs).  Mutually exclusive with btl_usnic_if_exclude.",
-                     NULL, &mca_btl_usnic_component.if_include,
-                     REGSTR_EMPTY_OK, OPAL_INFO_LVL_1));
+                     "Comma-delimited list of usNIC devices/networks to be used (e.g. "
+                     "\"eth3,usnic_0,10.10.0.0/16\"; empty value means to use all available "
+                     "usNICs).  Mutually exclusive with btl_usnic_if_exclude.",
+                     NULL, &mca_btl_usnic_component.if_include, REGSTR_EMPTY_OK, OPAL_INFO_LVL_1));
 
-    CHECK(reg_string("if_exclude",
-                     "Comma-delimited list of usNIC devices/networks to be excluded (empty value means to not exclude any usNICs).  Mutually exclusive with btl_usnic_if_include.",
-                     NULL, &mca_btl_usnic_component.if_exclude,
-                     REGSTR_EMPTY_OK, OPAL_INFO_LVL_1));
+    CHECK(reg_string(
+        "if_exclude",
+        "Comma-delimited list of usNIC devices/networks to be excluded (empty value means to not "
+        "exclude any usNICs).  Mutually exclusive with btl_usnic_if_include.",
+        NULL, &mca_btl_usnic_component.if_exclude, REGSTR_EMPTY_OK, OPAL_INFO_LVL_1));
 
     CHECK(reg_int("stats",
-                  "A non-negative integer specifying the frequency at which each usnic BTL will output statistics (default: 0 seconds, meaning that statistics are disabled)",
-                  0, &mca_btl_usnic_component.stats_frequency, 0,
-                  OPAL_INFO_LVL_4));
-    mca_btl_usnic_component.stats_enabled =
-        (bool) (mca_btl_usnic_component.stats_frequency > 0);
+                  "A non-negative integer specifying the frequency at which each usnic BTL will "
+                  "output statistics (default: 0 seconds, meaning that statistics are disabled)",
+                  0, &mca_btl_usnic_component.stats_frequency, 0, OPAL_INFO_LVL_4));
+    mca_btl_usnic_component.stats_enabled = (bool) (mca_btl_usnic_component.stats_frequency > 0);
 
     CHECK(reg_int("stats_relative",
-                  "If stats are enabled, output relative stats between the timestamps (vs. cumulative stats since the beginning of the job) (default: 0 -- i.e., absolute)",
+                  "If stats are enabled, output relative stats between the timestamps (vs. "
+                  "cumulative stats since the beginning of the job) (default: 0 -- i.e., absolute)",
                   0, &stats_relative, 0, OPAL_INFO_LVL_4));
     mca_btl_usnic_component.stats_relative = (bool) stats_relative;
 
-    CHECK(reg_string("mpool_hints", "Hints to use when selecting mpool",
-                     NULL, &mca_btl_usnic_component.usnic_mpool_hints,
-                     REGSTR_EMPTY_OK,
-                     OPAL_INFO_LVL_5));
+    CHECK(reg_string("mpool_hints", "Hints to use when selecting mpool", NULL,
+                     &mca_btl_usnic_component.usnic_mpool_hints, REGSTR_EMPTY_OK, OPAL_INFO_LVL_5));
 
-    CHECK(reg_string("rcache", "Name of the registration cache to be used",
-                     "grdma", &mca_btl_usnic_component.usnic_rcache_name, 0,
-                     OPAL_INFO_LVL_5));
+    CHECK(reg_string("rcache", "Name of the registration cache to be used", "grdma",
+                     &mca_btl_usnic_component.usnic_rcache_name, 0, OPAL_INFO_LVL_5));
 
     want_numa_device_assignment = 1;
-    CHECK(reg_int("want_numa_device_assignment",
-                  "If 1, use only Cisco VIC ports thare are a minimum NUMA distance from the MPI process for short messages.  If 0, use all available Cisco VIC ports for short messages.  This parameter is meaningless (and ignored) unless MPI proceses are bound to processor cores.  Defaults to 1 if NUMA support is included in Open MPI; -1 otherwise.",
-                  want_numa_device_assignment,
-                  &want_numa_device_assignment,
-                  0, OPAL_INFO_LVL_5));
-    mca_btl_usnic_component.want_numa_device_assignment =
-        (1 == want_numa_device_assignment) ? true : false;
-
-    CHECK(reg_int("sd_num", "Maximum send descriptors to post (-1 = pre-set defaults; depends on number and type of devices available)",
+    CHECK(reg_int(
+        "want_numa_device_assignment",
+        "If 1, use only Cisco VIC ports there are a minimum NUMA distance from the MPI process for "
+        "short messages.  If 0, use all available Cisco VIC ports for short messages.  This "
+        "parameter is meaningless (and ignored) unless MPI processes are bound to processor cores.  "
+        "Defaults to 1 if NUMA support is included in Open MPI; -1 otherwise.",
+        want_numa_device_assignment, &want_numa_device_assignment, 0, OPAL_INFO_LVL_5));
+    mca_btl_usnic_component.want_numa_device_assignment = (1 == want_numa_device_assignment)
+                                                              ? true
+                                                              : false;
+
+    CHECK(reg_int("sd_num",
+                  "Maximum send descriptors to post (-1 = pre-set defaults; depends on number and "
+                  "type of devices available)",
                   -1, &sd_num, REGINT_NEG_ONE_OK, OPAL_INFO_LVL_5));
     mca_btl_usnic_component.sd_num = (int32_t) sd_num;
 
-    CHECK(reg_int("rd_num", "Number of pre-posted receive buffers (-1 = pre-set defaults; depends on number and type of devices available)",
+    CHECK(reg_int("rd_num",
+                  "Number of pre-posted receive buffers (-1 = pre-set defaults; depends on number "
+                  "and type of devices available)",
                   -1, &rd_num, REGINT_NEG_ONE_OK, OPAL_INFO_LVL_5));
     mca_btl_usnic_component.rd_num = (int32_t) rd_num;
 
-    CHECK(reg_int("prio_sd_num", "Maximum priority send descriptors to post (-1 = pre-set defaults; depends on number and type of devices available)",
+    CHECK(reg_int("prio_sd_num",
+                  "Maximum priority send descriptors to post (-1 = pre-set defaults; depends on "
+                  "number and type of devices available)",
                   -1, &prio_sd_num, REGINT_NEG_ONE_OK, OPAL_INFO_LVL_5));
     mca_btl_usnic_component.prio_sd_num = (int32_t) prio_sd_num;
 
-    CHECK(reg_int("prio_rd_num", "Number of pre-posted priority receive buffers (-1 = pre-set defaults; depends on number and type of devices available)",
+    CHECK(reg_int("prio_rd_num",
+                  "Number of pre-posted priority receive buffers (-1 = pre-set defaults; depends "
+                  "on number and type of devices available)",
                   -1, &prio_rd_num, REGINT_NEG_ONE_OK, OPAL_INFO_LVL_5));
     mca_btl_usnic_component.prio_rd_num = (int32_t) prio_rd_num;
 
-    CHECK(reg_int("cq_num", "Number of completion queue entries (-1 = pre-set defaults; depends on number and type of devices available; will error if (sd_num+rd_num)>cq_num)",
+    CHECK(reg_int("cq_num",
+                  "Number of completion queue entries (-1 = pre-set defaults; depends on number "
+                  "and type of devices available; will error if (sd_num+rd_num)>cq_num)",
                   -1, &cq_num, REGINT_NEG_ONE_OK, OPAL_INFO_LVL_5));
     mca_btl_usnic_component.cq_num = (int32_t) cq_num;
 
-    CHECK(reg_int("av_eq_num", "Number of event queue entries for peer address resolution",
-                  1024, &av_eq_num, REGINT_GE_ONE, OPAL_INFO_LVL_5));
+    CHECK(reg_int("av_eq_num", "Number of event queue entries for peer address resolution", 1024,
+                  &av_eq_num, REGINT_GE_ONE, OPAL_INFO_LVL_5));
     mca_btl_usnic_component.av_eq_num = (int32_t) av_eq_num;
 
-    CHECK(reg_int("base_udp_port", "Base UDP port to use for usNIC communications.  If 0, system will pick the port number.  If non-zero, it will be added to each process' local rank to obtain the final port number (default: 0)",
+    CHECK(reg_int("base_udp_port",
+                  "Base UDP port to use for usNIC communications.  If 0, system will pick the port "
+                  "number.  If non-zero, it will be added to each process' local rank to obtain "
+                  "the final port number (default: 0)",
                   0, &udp_port_base, REGINT_GE_ZERO, OPAL_INFO_LVL_5));
     mca_btl_usnic_component.udp_port_base = (int) udp_port_base;
 
-    CHECK(reg_int("retrans_timeout", "Number of microseconds before retransmitting a frame",
-                  5000, &mca_btl_usnic_component.retrans_timeout,
-                  REGINT_GE_ONE, OPAL_INFO_LVL_5));
-
-    CHECK(reg_int("max_resends_per_iteration", "Maximum number of frames to resend in a single iteration through usNIC component progress",
-                  16, &mca_btl_usnic_component.max_resends_per_iteration,
-                  REGINT_GE_ONE, OPAL_INFO_LVL_5));
-
-    CHECK(reg_int("ack_iteration_delay", "Minimum number of times through usNIC \"progress\" function before checking to see if standalone ACKs need to be sent",
-                  4, &mca_btl_usnic_component.ack_iteration_delay,
-                  REGINT_GE_ZERO, OPAL_INFO_LVL_5));
-
-    CHECK(reg_int("priority_limit", "Max size of \"priority\" messages (0 = use pre-set defaults; depends on number and type of devices available)",
-                  0, &max_tiny_msg_size,
-                  REGINT_GE_ZERO, OPAL_INFO_LVL_5));
-    opal_btl_usnic_module_template.max_tiny_msg_size =
-        (size_t) max_tiny_msg_size;
-
-    CHECK(reg_int("eager_limit", "Eager send limit (0 = use pre-set defaults; depends on number and type of devices available)",
+    CHECK(reg_int("retrans_timeout", "Number of microseconds before retransmitting a frame", 5000,
+                  &mca_btl_usnic_component.retrans_timeout, REGINT_GE_ONE, OPAL_INFO_LVL_5));
+
+    CHECK(reg_int(
+        "max_resends_per_iteration",
+        "Maximum number of frames to resend in a single iteration through usNIC component progress",
+        16, &mca_btl_usnic_component.max_resends_per_iteration, REGINT_GE_ONE, OPAL_INFO_LVL_5));
+
+    CHECK(reg_int("ack_iteration_delay",
+                  "Minimum number of times through usNIC \"progress\" function before checking to "
+                  "see if standalone ACKs need to be sent",
+                  4, &mca_btl_usnic_component.ack_iteration_delay, REGINT_GE_ZERO,
+                  OPAL_INFO_LVL_5));
+
+    CHECK(reg_int("priority_limit",
+                  "Max size of \"priority\" messages (0 = use pre-set defaults; depends on number "
+                  "and type of devices available)",
+                  0, &max_tiny_msg_size, REGINT_GE_ZERO, OPAL_INFO_LVL_5));
+    opal_btl_usnic_module_template.max_tiny_msg_size = (size_t) max_tiny_msg_size;
+
+    CHECK(reg_int("eager_limit",
+                  "Eager send limit (0 = use pre-set defaults; depends on number and type of "
+                  "devices available)",
                   0, &eager_limit, REGINT_GE_ZERO, OPAL_INFO_LVL_5));
     opal_btl_usnic_module_template.super.btl_eager_limit = eager_limit;
 
-    CHECK(reg_int("rndv_eager_limit", "Eager rendezvous limit (0 = use pre-set defaults; depends on number and type of devices available)",
+    CHECK(reg_int("rndv_eager_limit",
+                  "Eager rendezvous limit (0 = use pre-set defaults; depends on number and type of "
+                  "devices available)",
                   0, &rndv_eager_limit, REGINT_GE_ZERO, OPAL_INFO_LVL_5));
-    opal_btl_usnic_module_template.super.btl_rndv_eager_limit =
-        rndv_eager_limit;
+    opal_btl_usnic_module_template.super.btl_rndv_eager_limit = rndv_eager_limit;
 
-    CHECK(reg_int("pack_lazy_threshold", "Convertor packing on-the-fly threshold (-1 = always pack eagerly, 0 = always pack lazily, otherwise will pack on the fly if fragment size is > limit)",
-                  USNIC_DFLT_PACK_LAZY_THRESHOLD, &pack_lazy_threshold, REGINT_NEG_ONE_OK, OPAL_INFO_LVL_5));
+    CHECK(reg_int("pack_lazy_threshold",
+                  "Convertor packing on-the-fly threshold (-1 = always pack eagerly, 0 = always "
+                  "pack lazily, otherwise will pack on the fly if fragment size is > limit)",
+                  USNIC_DFLT_PACK_LAZY_THRESHOLD, &pack_lazy_threshold, REGINT_NEG_ONE_OK,
+                  OPAL_INFO_LVL_5));
     mca_btl_usnic_component.pack_lazy_threshold = pack_lazy_threshold;
 
-    CHECK(reg_int("max_short_packets", "Number of abnormally-short packets received before outputting a warning (0 = never show the warning)",
-                  25, &max_short_packets,
-                  REGINT_GE_ZERO, OPAL_INFO_LVL_5));
+    CHECK(reg_int("max_short_packets",
+                  "Number of abnormally-short packets received before outputting a warning (0 = "
+                  "never show the warning)",
+                  25, &max_short_packets, REGINT_GE_ZERO, OPAL_INFO_LVL_5));
     mca_btl_usnic_component.max_short_packets = max_short_packets;
 
     /* Default to bandwidth auto-detection */
@@ -288,39 +282,43 @@ int opal_btl_usnic_component_register(void)
     /* Show "cannot find route" warnings? */
     mca_btl_usnic_component.show_route_failures = true;
     CHECK(reg_bool("show_route_failures",
-                   "Whether to show a warning when route failures between MPI process peers are detected (default = 1, enabled; 0 = disabled)",
+                   "Whether to show a warning when route failures between MPI process peers are "
+                   "detected (default = 1, enabled; 0 = disabled)",
                    mca_btl_usnic_component.show_route_failures,
-                   &mca_btl_usnic_component.show_route_failures,
-                   OPAL_INFO_LVL_3));
+                   &mca_btl_usnic_component.show_route_failures, OPAL_INFO_LVL_3));
 
     /* Connectivity verification */
     mca_btl_usnic_component.connectivity_enabled = true;
     CHECK(reg_bool("connectivity_check",
-                   "Whether to enable the usNIC connectivity check upon first send (default = 1, enabled; 0 = disabled)",
+                   "Whether to enable the usNIC connectivity check upon first send (default = 1, "
+                   "enabled; 0 = disabled)",
                    mca_btl_usnic_component.connectivity_enabled,
-                   &mca_btl_usnic_component.connectivity_enabled,
-                   OPAL_INFO_LVL_3));
+                   &mca_btl_usnic_component.connectivity_enabled, OPAL_INFO_LVL_3));
 
     mca_btl_usnic_component.connectivity_ack_timeout = 250;
-    CHECK(reg_int("connectivity_ack_timeout",
-                  "Timeout, in milliseconds, while waiting for an ACK while verification connectivity between usNIC interfaces.  If 0, the connectivity check is disabled (must be >=0).",
-                  mca_btl_usnic_component.connectivity_ack_timeout,
-                  &mca_btl_usnic_component.connectivity_ack_timeout,
-                  REGINT_GE_ZERO, OPAL_INFO_LVL_3));
+    CHECK(reg_int(
+        "connectivity_ack_timeout",
+        "Timeout, in milliseconds, while waiting for an ACK while verification connectivity "
+        "between usNIC interfaces.  If 0, the connectivity check is disabled (must be >=0).",
+        mca_btl_usnic_component.connectivity_ack_timeout,
+        &mca_btl_usnic_component.connectivity_ack_timeout, REGINT_GE_ZERO, OPAL_INFO_LVL_3));
 
     mca_btl_usnic_component.connectivity_num_retries = 40;
     CHECK(reg_int("connectivity_error_num_retries",
-                  "Number of times to retry usNIC connectivity verification before aborting the MPI job (must be >0).",
+                  "Number of times to retry usNIC connectivity verification before aborting the "
+                  "MPI job (must be >0).",
                   mca_btl_usnic_component.connectivity_num_retries,
-                  &mca_btl_usnic_component.connectivity_num_retries,
-                  REGINT_GE_ONE, OPAL_INFO_LVL_3));
+                  &mca_btl_usnic_component.connectivity_num_retries, REGINT_GE_ONE,
+                  OPAL_INFO_LVL_3));
 
     mca_btl_usnic_component.connectivity_map_prefix = NULL;
-    CHECK(reg_string("connectivity_map",
-                     "Write a per-process file containing the usNIC connectivity map.  If this parameter is specified, it is the filename prefix emitted by each MPI process.  The full filename emitted by each process is of the form: <prefix>-<hostname>.<pid>.<jobid>.<MCW rank>.txt.",
-                     mca_btl_usnic_component.connectivity_map_prefix,
-                     &mca_btl_usnic_component.connectivity_map_prefix,
-                     REGSTR_EMPTY_OK, OPAL_INFO_LVL_3));
+    CHECK(reg_string(
+        "connectivity_map",
+        "Write a per-process file containing the usNIC connectivity map.  If this parameter is "
+        "specified, it is the filename prefix emitted by each MPI process.  The full filename "
+        "emitted by each process is of the form: <prefix>-<hostname>.<pid>.<jobid>.<MCW rank>.txt.",
+        mca_btl_usnic_component.connectivity_map_prefix,
+        &mca_btl_usnic_component.connectivity_map_prefix, REGSTR_EMPTY_OK, OPAL_INFO_LVL_3));
 
     return ret;
 }
diff --git a/opal/mca/btl/usnic/btl_usnic_module.c b/opal/mca/btl/usnic/btl_usnic_module.c
index e65d0093d12..7ae08e6fb67 100644
--- a/opal/mca/btl/usnic/btl_usnic_module.c
+++ b/opal/mca/btl/usnic/btl_usnic_module.c
@@ -27,59 +27,54 @@
 #include "opal_config.h"
 
 #include <errno.h>
-#include <string.h>
-#include <unistd.h>
 #include <stdlib.h>
+#include <string.h>
 #include <time.h>
+#include <unistd.h>
 
-#include "opal_stdint.h"
 #include "opal/class/opal_bitmap.h"
+#include "opal/datatype/opal_convertor.h"
+#include "opal/mca/memchecker/base/base.h"
 #include "opal/prefetch.h"
 #include "opal/util/output.h"
-#include "opal/datatype/opal_convertor.h"
-#include "opal/util/show_help.h"
 #include "opal/util/printf.h"
-#include "opal/mca/memchecker/base/base.h"
+#include "opal/util/show_help.h"
+#include "opal_stdint.h"
 
-#include "opal/mca/btl/btl.h"
 #include "opal/mca/btl/base/btl_base_error.h"
+#include "opal/mca/btl/btl.h"
 #include "opal/mca/mpool/base/base.h"
 #include "opal/mca/mpool/mpool.h"
 #include "opal/mca/rcache/base/base.h"
 #include "opal/mca/rcache/rcache.h"
 
-#include "btl_usnic_compat.h"
 #include "btl_usnic.h"
+#include "btl_usnic_ack.h"
+#include "btl_usnic_compat.h"
 #include "btl_usnic_connectivity.h"
-#include "btl_usnic_frag.h"
-#include "btl_usnic_proc.h"
 #include "btl_usnic_endpoint.h"
+#include "btl_usnic_frag.h"
+#include "btl_usnic_hwloc.h"
 #include "btl_usnic_module.h"
-#include "btl_usnic_util.h"
+#include "btl_usnic_proc.h"
 #include "btl_usnic_send.h"
-#include "btl_usnic_ack.h"
-#include "btl_usnic_hwloc.h"
 #include "btl_usnic_stats.h"
+#include "btl_usnic_util.h"
 
 static void finalize_one_channel(opal_btl_usnic_module_t *module,
                                  struct opal_btl_usnic_channel_t *channel);
 
-static int channel_addr2str(opal_btl_usnic_module_t *module, int channel,
-                            char *str, size_t len_param)
+static int channel_addr2str(opal_btl_usnic_module_t *module, int channel, char *str,
+                            size_t len_param)
 {
     size_t len;
 
     len = len_param;
-    fi_av_straddr(module->av, module->mod_channels[channel].info->src_addr,
-                  str, &len);
+    fi_av_straddr(module->av, module->mod_channels[channel].info->src_addr, str, &len);
     if (len > len_param) {
-        opal_show_help("help-mpi-btl-usnic.txt",
-                       "libfabric API failed",
-                       true,
-                       opal_process_info.nodename,
-                       module->linux_device_name,
-                       "fi_av_straddr", __FILE__, __LINE__,
-                       FI_ENODATA,
+        opal_show_help("help-mpi-btl-usnic.txt", "libfabric API failed", true,
+                       opal_process_info.nodename, module->linux_device_name, "fi_av_straddr",
+                       __FILE__, __LINE__, FI_ENODATA,
                        "Failed to convert address to string: buffer too short");
 
         return OPAL_ERR_OUT_OF_RESOURCE;
@@ -88,19 +83,16 @@ static int channel_addr2str(opal_btl_usnic_module_t *module, int channel,
     return OPAL_SUCCESS;
 }
 
-
 /*
  * Loop over a block of procs sent to us in add_procs and see if we
  * want to add a proc/endpoint for them.
  */
-static int add_procs_block_create_endpoints(opal_btl_usnic_module_t *module,
-                                            size_t block_offset,
-                                            size_t block_len,
-                                            opal_proc_t **procs,
+static int add_procs_block_create_endpoints(opal_btl_usnic_module_t *module, size_t block_offset,
+                                            size_t block_len, opal_proc_t **procs,
                                             mca_btl_base_endpoint_t **endpoints)
 {
     int rc;
-    opal_proc_t* my_proc;
+    opal_proc_t *my_proc;
     size_t num_created = 0;
     char *errhost;
 
@@ -112,16 +104,15 @@ static int add_procs_block_create_endpoints(opal_btl_usnic_module_t *module,
 
     /* Loop over a block in the procs we were given */
     for (size_t i = block_offset; i < (block_offset + block_len); i++) {
-        struct opal_proc_t* opal_proc = procs[i];
-        opal_btl_usnic_proc_t* usnic_proc;
-        mca_btl_base_endpoint_t* usnic_endpoint;
+        struct opal_proc_t *opal_proc = procs[i];
+        opal_btl_usnic_proc_t *usnic_proc;
+        mca_btl_base_endpoint_t *usnic_endpoint;
 
         endpoints[i] = NULL;
 
         /* Do not create loopback usnic connections */
         if (opal_proc == my_proc) {
-            opal_output_verbose(75, USNIC_OUT,
-                                "btl:usnic:add_procs:%s: not connecting to self",
+            opal_output_verbose(75, USNIC_OUT, "btl:usnic:add_procs:%s: not connecting to self",
                                 module->linux_device_name);
             continue;
         }
@@ -144,12 +135,13 @@ static int add_procs_block_create_endpoints(opal_btl_usnic_module_t *module,
         if (OPAL_ERR_UNREACH == rc) {
             /* If the peer doesn't have usnic modex info, then we just
                skip it */
-          if (74 < opal_output_get_verbosity(USNIC_OUT)) {
+            if (74 < opal_output_get_verbosity(USNIC_OUT)) {
                 errhost = opal_get_proc_hostname(opal_proc);
-                opal_output(0, "btl:usnic:add_procs:%s: peer %s on %s does not have usnic modex info; skipping",
+                opal_output(0,
+                            "btl:usnic:add_procs:%s: peer %s on %s does not have usnic modex info; "
+                            "skipping",
                             module->linux_device_name,
-                            usnic_compat_proc_name_print(&opal_proc->proc_name),
-                            errhost);
+                            usnic_compat_proc_name_print(&opal_proc->proc_name), errhost);
                 free(errhost);
             }
             continue;
@@ -160,15 +152,13 @@ static int add_procs_block_create_endpoints(opal_btl_usnic_module_t *module,
         /* Create the endpoint for this proc/module combination.  If we cannot
          * reach this proc via this module, move on to the next proc. */
         usnic_endpoint = NULL;
-        rc = opal_btl_usnic_create_endpoint(module, usnic_proc,
-                                            &usnic_endpoint);
+        rc = opal_btl_usnic_create_endpoint(module, usnic_proc, &usnic_endpoint);
         if (OPAL_SUCCESS != rc) {
-          if (4 < opal_output_get_verbosity(USNIC_OUT)) {
+            if (4 < opal_output_get_verbosity(USNIC_OUT)) {
                 errhost = opal_get_proc_hostname(opal_proc);
                 opal_output(0, "btl:usnic:add_procs:%s: unable to create endpoint to peer %s on %s",
                             module->linux_device_name,
-                            usnic_compat_proc_name_print(&opal_proc->proc_name),
-                            errhost);
+                            usnic_compat_proc_name_print(&opal_proc->proc_name), errhost);
                 free(errhost);
             }
             OBJ_RELEASE(usnic_proc);
@@ -179,43 +169,35 @@ static int add_procs_block_create_endpoints(opal_btl_usnic_module_t *module,
         opal_pointer_array_add(&module->all_procs, usnic_proc);
 
         char str[IPV4STRADDRLEN];
-        struct opal_btl_usnic_modex_t *modex =
-            &usnic_endpoint->endpoint_remote_modex;
-        opal_btl_usnic_snprintf_ipv4_addr(str, sizeof(str),
-                                          modex->ipv4_addr,
-                                          modex->netmask);
+        struct opal_btl_usnic_modex_t *modex = &usnic_endpoint->endpoint_remote_modex;
+        opal_btl_usnic_snprintf_ipv4_addr(str, sizeof(str), modex->ipv4_addr, modex->netmask);
 
         char local_pri_addr[64] = {0};
-        rc = channel_addr2str(module, USNIC_PRIORITY_CHANNEL,
-                              local_pri_addr, sizeof(local_pri_addr));
+        rc = channel_addr2str(module, USNIC_PRIORITY_CHANNEL, local_pri_addr,
+                              sizeof(local_pri_addr));
         if (OPAL_SUCCESS != rc) {
             OBJ_RELEASE(usnic_proc);
             continue;
         }
 
         char local_data_addr[64] = {0};
-        rc = channel_addr2str(module, USNIC_DATA_CHANNEL,
-                              local_data_addr, sizeof(local_data_addr));
+        rc = channel_addr2str(module, USNIC_DATA_CHANNEL, local_data_addr, sizeof(local_data_addr));
         if (OPAL_SUCCESS != rc) {
             OBJ_RELEASE(usnic_proc);
             continue;
         }
 
         opal_output_verbose(5, USNIC_OUT,
-                            "btl:usnic:add_procs:%s: new usnic peer endpoint: pri=%s:%d, data=%s:%d (local: pri=%s, data=%s)",
-                            module->linux_device_name,
-                            str, modex->ports[USNIC_PRIORITY_CHANNEL],
-                            str, modex->ports[USNIC_DATA_CHANNEL],
-                            local_pri_addr,
-                            local_data_addr);
+                            "btl:usnic:add_procs:%s: new usnic peer endpoint: pri=%s:%d, "
+                            "data=%s:%d (local: pri=%s, data=%s)",
+                            module->linux_device_name, str, modex->ports[USNIC_PRIORITY_CHANNEL],
+                            str, modex->ports[USNIC_DATA_CHANNEL], local_pri_addr, local_data_addr);
 
         endpoints[i] = usnic_endpoint;
         ++num_created;
     }
 
-    opal_output_verbose(5, USNIC_OUT,
-                        "btl:usnic: made %" PRIsize_t " endpoints",
-                        num_created);
+    opal_output_verbose(5, USNIC_OUT, "btl:usnic: made %" PRIsize_t " endpoints", num_created);
     return OPAL_SUCCESS;
 }
 
@@ -240,30 +222,21 @@ static void add_procs_warn_unreachable(opal_btl_usnic_module_t *module,
                                       endpoint->endpoint_remote_modex.ipv4_addr,
                                       endpoint->endpoint_remote_modex.netmask);
 
-    opal_output_verbose(15, USNIC_OUT,
-                        "btl:usnic: %s (which is %s) couldn't reach peer %s",
-                        module->linux_device_name,
-                        module->if_ipv4_addr_str,
-                        remote);
+    opal_output_verbose(15, USNIC_OUT, "btl:usnic: %s (which is %s) couldn't reach peer %s",
+                        module->linux_device_name, module->if_ipv4_addr_str, remote);
     errhost = opal_get_proc_hostname(endpoint->endpoint_proc->proc_opal);
-    opal_show_help("help-mpi-btl-usnic.txt", "unreachable peer IP",
-                   true,
-                   opal_process_info.nodename,
-                   module->if_ipv4_addr_str,
-                   module->linux_device_name,
-                   errhost,
-                   remote);
+    opal_show_help("help-mpi-btl-usnic.txt", "unreachable peer IP", true,
+                   opal_process_info.nodename, module->if_ipv4_addr_str, module->linux_device_name,
+                   errhost, remote);
     free(errhost);
 }
 
 /* A bunch of calls to fi_av_insert() were previously
  * invoked.  Go reap them all.
  */
-static int
-add_procs_block_reap_fi_av_inserts(opal_btl_usnic_module_t *module,
-                                   size_t block_offset,
-                                   size_t block_len,
-                                   struct mca_btl_base_endpoint_t **endpoints)
+static int add_procs_block_reap_fi_av_inserts(opal_btl_usnic_module_t *module, size_t block_offset,
+                                              size_t block_len,
+                                              struct mca_btl_base_endpoint_t **endpoints)
 {
     int ret = OPAL_SUCCESS;
     int num_left;
@@ -319,8 +292,7 @@ add_procs_block_reap_fi_av_inserts(opal_btl_usnic_module_t *module,
                    that we couldn't find a route to that peer (e.g., the
                    networking is hosed between us).  So just mark that we
                    can't reach this peer, and print a pretty warning. */
-                if (EADDRNOTAVAIL == err_entry.err ||
-                     EHOSTUNREACH == err_entry.err) {
+                if (EADDRNOTAVAIL == err_entry.err || EHOSTUNREACH == err_entry.err) {
 
                     /* Note that endpoint was passed in a context in
                        USNIC_NUM_CHANNELS fi_av_insert() calls.
@@ -333,8 +305,7 @@ add_procs_block_reap_fi_av_inserts(opal_btl_usnic_module_t *module,
                        the *first* time it is reported. */
                     for (i = block_offset; i < (block_offset + block_len); ++i) {
                         if (endpoints[i] == context->endpoint) {
-                            add_procs_warn_unreachable(module,
-                                                       context->endpoint);
+                            add_procs_warn_unreachable(module, context->endpoint);
                             OBJ_RELEASE(context->endpoint);
                             endpoints[i] = NULL;
                             break;
@@ -346,13 +317,9 @@ add_procs_block_reap_fi_av_inserts(opal_btl_usnic_module_t *module,
                 /* Got some other kind of error -- give up on this
                    interface. */
                 else {
-                    opal_show_help("help-mpi-btl-usnic.txt",
-                                   "libfabric API failed",
-                                   true,
-                                   opal_process_info.nodename,
-                                   module->linux_device_name,
-                                   "async insertion result", __FILE__, __LINE__,
-                                   err_entry.err,
+                    opal_show_help("help-mpi-btl-usnic.txt", "libfabric API failed", true,
+                                   opal_process_info.nodename, module->linux_device_name,
+                                   "async insertion result", __FILE__, __LINE__, err_entry.err,
                                    "Failed to insert address to AV");
                     ret = OPAL_ERR_OUT_OF_RESOURCE;
                     error_occurred = true;
@@ -370,13 +337,9 @@ add_procs_block_reap_fi_av_inserts(opal_btl_usnic_module_t *module,
                    badly, which means something has gone tremendously
                    wrong.  Probably the only safe thing to do here is
                    exit. */
-                opal_show_help("help-mpi-btl-usnic.txt",
-                               "internal error during init",
-                               true,
-                               opal_process_info.nodename,
-                               module->linux_device_name,
-                               "fi_eq_readerr()", __FILE__, __LINE__,
-                               ret,
+                opal_show_help("help-mpi-btl-usnic.txt", "internal error during init", true,
+                               opal_process_info.nodename, module->linux_device_name,
+                               "fi_eq_readerr()", __FILE__, __LINE__, ret,
                                "Returned != sizeof(err_entry)");
                 ret = OPAL_ERR_OUT_OF_RESOURCE;
                 error_occurred = true;
@@ -391,14 +354,9 @@ add_procs_block_reap_fi_av_inserts(opal_btl_usnic_module_t *module,
                Given that we're potentially not even all the way
                through MPI_INIT yet, the only sane thing to do here is
                exit. */
-            opal_show_help("help-mpi-btl-usnic.txt",
-                           "internal error during init",
-                           true,
-                           opal_process_info.nodename,
-                           module->linux_device_name,
-                           "fi_eq_sread()", __FILE__, __LINE__,
-                           ret,
-                           "Returned != (sizeof(entry) or -FI_EAVAIL)");
+            opal_show_help("help-mpi-btl-usnic.txt", "internal error during init", true,
+                           opal_process_info.nodename, module->linux_device_name, "fi_eq_sread()",
+                           __FILE__, __LINE__, ret, "Returned != (sizeof(entry) or -FI_EAVAIL)");
             ret = OPAL_ERR_OUT_OF_RESOURCE;
             error_occurred = true;
 
@@ -422,8 +380,7 @@ add_procs_block_reap_fi_av_inserts(opal_btl_usnic_module_t *module,
                 happy = false;
             } else {
                 for (channel = 0; channel < USNIC_NUM_CHANNELS; ++channel) {
-                    if (FI_ADDR_NOTAVAIL ==
-                            endpoints[i]->endpoint_remote_addrs[channel]) {
+                    if (FI_ADDR_NOTAVAIL == endpoints[i]->endpoint_remote_addrs[channel]) {
                         happy = false;
                         break;
                     }
@@ -441,8 +398,7 @@ add_procs_block_reap_fi_av_inserts(opal_btl_usnic_module_t *module,
 
     /* All done */
     opal_output_verbose(5, USNIC_OUT,
-                        "btl:usnic: created destinations for %" PRIsize_t
-                        " endpoints",
+                        "btl:usnic: created destinations for %" PRIsize_t " endpoints",
                         num_endpoints_created);
     return ret;
 }
@@ -450,10 +406,9 @@ add_procs_block_reap_fi_av_inserts(opal_btl_usnic_module_t *module,
 /*
  * Create endpoints for the procs we were given in add_procs.
  */
-static int add_procs_create_endpoints(struct opal_btl_usnic_module_t* module,
-                                      size_t nprocs,
+static int add_procs_create_endpoints(struct opal_btl_usnic_module_t *module, size_t nprocs,
                                       struct opal_proc_t **procs,
-                                      struct mca_btl_base_endpoint_t** endpoints)
+                                      struct mca_btl_base_endpoint_t **endpoints)
 {
     /* We need to ensure that we don't overrun the libfabric AV EQ.
        Divide up all the peer address resolutions we need to do into a
@@ -464,11 +419,8 @@ static int add_procs_create_endpoints(struct opal_btl_usnic_module_t* module,
 
     /* Leave a few empty slots in the AV EQ, just for good measure */
     if (module->av_eq_size < 8) {
-        opal_show_help("help-mpi-btl-usnic.txt", "fi_av_eq too small",
-                       true,
-                       opal_process_info.nodename,
-                       module->av_eq_size,
-                       8);
+        opal_show_help("help-mpi-btl-usnic.txt", "fi_av_eq too small", true,
+                       opal_process_info.nodename, module->av_eq_size, 8);
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
@@ -497,9 +449,7 @@ static int add_procs_create_endpoints(struct opal_btl_usnic_module_t* module,
 
         /* First, create endpoints (and procs, if they're not already
            created) for the usnic-reachable procs we were given. */
-        rc = add_procs_block_create_endpoints(module,
-                                              block_offset, block_len,
-                                              procs, endpoints);
+        rc = add_procs_block_create_endpoints(module, block_offset, block_len, procs, endpoints);
         if (OPAL_SUCCESS != rc) {
             return rc;
         }
@@ -509,9 +459,7 @@ static int add_procs_create_endpoints(struct opal_btl_usnic_module_t* module,
            those.  This will be the final determination of whether we
            can use the endpoint or not because we'll find out if each
            endpoint is reachable or not. */
-        rc = add_procs_block_reap_fi_av_inserts(module,
-                                                block_offset, block_len,
-                                                endpoints);
+        rc = add_procs_block_reap_fi_av_inserts(module, block_offset, block_len, endpoints);
         if (OPAL_SUCCESS != rc) {
             return rc;
         }
@@ -537,13 +485,11 @@ static int add_procs_create_endpoints(struct opal_btl_usnic_module_t* module,
  * peers (i.e., we should just mark those peers as unreachable and not
  * add a proc or endpoint for them).
  */
-static int usnic_add_procs(struct mca_btl_base_module_t* base_module,
-                             size_t nprocs,
-                             struct opal_proc_t **procs,
-                             struct mca_btl_base_endpoint_t** endpoints,
-                             opal_bitmap_t* reachable)
+static int usnic_add_procs(struct mca_btl_base_module_t *base_module, size_t nprocs,
+                           struct opal_proc_t **procs, struct mca_btl_base_endpoint_t **endpoints,
+                           opal_bitmap_t *reachable)
 {
-    opal_btl_usnic_module_t* module = (opal_btl_usnic_module_t*) base_module;
+    opal_btl_usnic_module_t *module = (opal_btl_usnic_module_t *) base_module;
     int rc;
 
     /* Go create the endpoints (including all relevant address
@@ -559,8 +505,7 @@ static int usnic_add_procs(struct mca_btl_base_module_t* base_module,
         if (NULL != endpoints[i]) {
             bool happy = true;
             for (int channel = 0; channel < USNIC_NUM_CHANNELS; ++channel) {
-                if (FI_ADDR_NOTAVAIL ==
-                        endpoints[i]->endpoint_remote_addrs[channel]) {
+                if (FI_ADDR_NOTAVAIL == endpoints[i]->endpoint_remote_addrs[channel]) {
                     happy = false;
                     break;
                 }
@@ -581,14 +526,13 @@ static int usnic_add_procs(struct mca_btl_base_module_t* base_module,
        call the function to output the map. */
     static int num_times_add_procs_called = 0;
     ++num_times_add_procs_called;
-    if (0 == (num_times_add_procs_called %
-              mca_btl_usnic_component.num_modules)) {
+    if (0 == (num_times_add_procs_called % mca_btl_usnic_component.num_modules)) {
         opal_btl_usnic_connectivity_map();
     }
 
     return OPAL_SUCCESS;
 
- fail:
+fail:
     /* If we get here, it means something went terribly wrong.  Scorch
        the earth: destroy all endpoints and say that nothing was
        reachable. */
@@ -608,21 +552,18 @@ static int usnic_add_procs(struct mca_btl_base_module_t* base_module,
  * multiple times to remove each proc.  The OBJ reference counts
  * will make all the details work out.
  */
-static int usnic_del_procs(struct mca_btl_base_module_t *base_module,
-                             size_t nprocs,
-                             struct opal_proc_t **procs,
-                             struct mca_btl_base_endpoint_t **peers)
+static int usnic_del_procs(struct mca_btl_base_module_t *base_module, size_t nprocs,
+                           struct opal_proc_t **procs, struct mca_btl_base_endpoint_t **peers)
 {
     size_t i, j;
     opal_btl_usnic_module_t *module;
     opal_btl_usnic_endpoint_t *endpoint;
     int index;
 
-    module = (struct opal_btl_usnic_module_t *)base_module;
+    module = (struct opal_btl_usnic_module_t *) base_module;
 
     for (i = 0; i < nprocs; i++) {
-        opal_btl_usnic_proc_t* proc =
-            opal_btl_usnic_proc_lookup_ompi(procs[i]);
+        opal_btl_usnic_proc_t *proc = opal_btl_usnic_proc_lookup_ompi(procs[i]);
         if (NULL != proc) {
 
             /* find endpoint for this module */
@@ -645,17 +586,15 @@ static int usnic_del_procs(struct mca_btl_base_module_t *base_module,
                     /* We're all done with this endpoint */
                     OBJ_RELEASE(endpoint);
 
-                    break;  /* done once we found match */
+                    break; /* done once we found match */
                 }
             }
 
             /* remove proc from this module, and decrement its refcount */
             for (index = 0; index < module->all_procs.size; ++index) {
-                if (opal_pointer_array_get_item(&module->all_procs, index) ==
-                        proc) {
+                if (opal_pointer_array_get_item(&module->all_procs, index) == proc) {
                     OBJ_RELEASE(proc);
-                    opal_pointer_array_set_item(&module->all_procs, index,
-                            NULL);
+                    opal_pointer_array_set_item(&module->all_procs, index, NULL);
                     break;
                 }
             }
@@ -665,14 +604,13 @@ static int usnic_del_procs(struct mca_btl_base_module_t *base_module,
     return OPAL_SUCCESS;
 }
 
-
 /*
  * Let the PML register a callback function with me
  */
-static int usnic_register_pml_err_cb(struct mca_btl_base_module_t* btl,
+static int usnic_register_pml_err_cb(struct mca_btl_base_module_t *btl,
                                      mca_btl_base_module_error_cb_fn_t cbfunc)
 {
-    opal_btl_usnic_module_t *module = (opal_btl_usnic_module_t*) btl;
+    opal_btl_usnic_module_t *module = (opal_btl_usnic_module_t *) btl;
 
     module->pml_error_callback = cbfunc;
 
@@ -686,18 +624,14 @@ static int usnic_register_pml_err_cb(struct mca_btl_base_module_t* btl,
  * alloc.
  *  --> Contraction in the btl.h documentation.
  */
-static mca_btl_base_descriptor_t*
-usnic_alloc(struct mca_btl_base_module_t* btl,
-              struct mca_btl_base_endpoint_t* endpoint,
-              uint8_t order,
-              size_t size,
-              uint32_t flags)
+static mca_btl_base_descriptor_t *usnic_alloc(struct mca_btl_base_module_t *btl,
+                                              struct mca_btl_base_endpoint_t *endpoint,
+                                              uint8_t order, size_t size, uint32_t flags)
 {
     opal_btl_usnic_send_frag_t *frag;
-    opal_btl_usnic_module_t *module = (opal_btl_usnic_module_t*) btl;
+    opal_btl_usnic_module_t *module = (opal_btl_usnic_module_t *) btl;
     mca_btl_base_descriptor_t *desc;
 
-
     /* small is easy, just allocate a small segment */
     if (OPAL_LIKELY(size <= module->max_frag_payload)) {
         opal_btl_usnic_small_send_frag_t *sfrag;
@@ -708,10 +642,10 @@ usnic_alloc(struct mca_btl_base_module_t* btl,
         }
         frag = &sfrag->ssf_base;
 
-    /* between MTU and eager limit, we need to allocate a buffer
-     * which can hold the data.  We will allocate a
-     * large fragment, and attach the buffer to it.
-     */
+        /* between MTU and eager limit, we need to allocate a buffer
+         * which can hold the data.  We will allocate a
+         * large fragment, and attach the buffer to it.
+         */
     } else {
         opal_btl_usnic_large_send_frag_t *lfrag;
 
@@ -734,17 +668,16 @@ usnic_alloc(struct mca_btl_base_module_t* btl,
         }
 
         /* pointer to buffer for caller */
-        frag->sf_base.uf_base.USNIC_SEND_LOCAL[0].seg_addr.pval =
-            lfrag->lsf_buffer;
+        frag->sf_base.uf_base.USNIC_SEND_LOCAL[0].seg_addr.pval = lfrag->lsf_buffer;
 
-        MSGDEBUG1_OUT("usnic_alloc: packing frag %p on the fly", (void *)frag);
+        MSGDEBUG1_OUT("usnic_alloc: packing frag %p on the fly", (void *) frag);
         lfrag->lsf_pack_on_the_fly = true;
     }
 
 #if MSGDEBUG2
-        opal_output(0, "usnic_alloc: %s frag=%p, size=%d, flags=0x%x\n",
-                (size <= module->max_frag_payload)?"small":"large",
-                (void *)frag, (int)size, flags);
+    opal_output(0, "usnic_alloc: %s frag=%p, size=%d, flags=0x%x\n",
+                (size <= module->max_frag_payload) ? "small" : "large", (void *) frag, (int) size,
+                flags);
 #endif
 
     /* set endpoint */
@@ -759,28 +692,24 @@ usnic_alloc(struct mca_btl_base_module_t* btl,
     return desc;
 }
 
-
 /**
  * Return an allocated fragment
  *
  * Return the send fragment to the appropriate list
  */
-static int usnic_free(struct mca_btl_base_module_t* btl,
-                        mca_btl_base_descriptor_t* des)
+static int usnic_free(struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des)
 {
-    opal_btl_usnic_frag_t* frag = (opal_btl_usnic_frag_t*)des;
+    opal_btl_usnic_frag_t *frag = (opal_btl_usnic_frag_t *) des;
 
 #if MSGDEBUG2
-    opal_output(0, "usnic_free: %p (%s)\n", (void*)frag,
-            usnic_frag_type(frag->uf_type));
+    opal_output(0, "usnic_free: %p (%s)\n", (void *) frag, usnic_frag_type(frag->uf_type));
 #endif
     /* calling free routine gives us ownership - we need to make sure
      * the flag is set for lower layers.
      */
     frag->uf_base.des_flags |= MCA_BTL_DES_FLAGS_BTL_OWNERSHIP;
 
-    opal_btl_usnic_frag_return_cond((struct opal_btl_usnic_module_t *)btl,
-            frag);
+    opal_btl_usnic_frag_return_cond((struct opal_btl_usnic_module_t *) btl, frag);
 
     return OPAL_SUCCESS;
 }
@@ -791,11 +720,9 @@ static int usnic_free(struct mca_btl_base_module_t* btl,
  * the convertor contained in the frag.
  *
  * The frag's bookkeeping data will be updated appropriately. */
-static
-opal_btl_usnic_chunk_segment_t *
-pack_chunk_seg_from_frag(
-    struct opal_btl_usnic_module_t* module,
-    opal_btl_usnic_large_send_frag_t *lfrag)
+static opal_btl_usnic_chunk_segment_t *
+pack_chunk_seg_from_frag(struct opal_btl_usnic_module_t *module,
+                         opal_btl_usnic_large_send_frag_t *lfrag)
 {
     opal_btl_usnic_chunk_segment_t *seg;
     uint8_t *copyptr;
@@ -812,8 +739,7 @@ pack_chunk_seg_from_frag(
         /* TODO look at ways to deal with this case more gracefully, possibly as
          * part of capping the overall BTL memory consumption.  Watch out for
          * possible MPI-layer deadlock. */
-        opal_btl_usnic_util_abort("chunk segment allocation error",
-                                  __FILE__, __LINE__);
+        opal_btl_usnic_util_abort("chunk segment allocation error", __FILE__, __LINE__);
     }
 
     seg_space = module->max_chunk_payload;
@@ -822,9 +748,7 @@ pack_chunk_seg_from_frag(
     /* Keep copying in as long as we have space, there is data to be copied, and
      * we aren't using a convertor (SG[1] will be NULL if we have a convertor).
      */
-    while (seg_space > 0 &&
-           lfrag->lsf_pack_bytes_left > 0 &&
-           NULL != lfrag->lsf_cur_ptr) {
+    while (seg_space > 0 && lfrag->lsf_pack_bytes_left > 0 && NULL != lfrag->lsf_cur_ptr) {
         if (seg_space > lfrag->lsf_bytes_left_in_sge) {
             copylen = lfrag->lsf_bytes_left_in_sge;
         } else {
@@ -840,10 +764,8 @@ pack_chunk_seg_from_frag(
             lfrag->lsf_cur_ptr += copylen;
         } else {
             ++lfrag->lsf_cur_sge;
-            lfrag->lsf_cur_ptr =
-                lfrag->lsf_des_src[lfrag->lsf_cur_sge].seg_addr.pval;
-            lfrag->lsf_bytes_left_in_sge =
-                lfrag->lsf_des_src[lfrag->lsf_cur_sge].seg_len;
+            lfrag->lsf_cur_ptr = lfrag->lsf_des_src[lfrag->lsf_cur_sge].seg_addr.pval;
+            lfrag->lsf_bytes_left_in_sge = lfrag->lsf_des_src[lfrag->lsf_cur_sge].seg_len;
         }
     }
 
@@ -856,16 +778,14 @@ pack_chunk_seg_from_frag(
         if (copylen > seg_space) {
             copylen = seg_space;
         }
-        usnic_convertor_pack_simple(&lfrag->lsf_base.sf_convertor, copyptr,
-                                    copylen, &max_data);
+        usnic_convertor_pack_simple(&lfrag->lsf_base.sf_convertor, copyptr, copylen, &max_data);
         seg_space -= max_data;
         lfrag->lsf_bytes_left_in_sge -= max_data;
         lfrag->lsf_pack_bytes_left -= max_data;
     }
 
-    MSGDEBUG1_OUT("%s: packed seg=%p, frag=%p, payload=%zd\n",
-                  __func__, (void *)seg, (void *)lfrag,
-                  (module->max_chunk_payload - seg_space));
+    MSGDEBUG1_OUT("%s: packed seg=%p, frag=%p, payload=%zd\n", __func__, (void *) seg,
+                  (void *) lfrag, (module->max_chunk_payload - seg_space));
 
     assert(lfrag->lsf_cur_sge <= 2);
     assert(seg_space < module->max_chunk_payload); /* must make progress */
@@ -876,9 +796,9 @@ pack_chunk_seg_from_frag(
     return seg;
 }
 
-static int usnic_finalize(struct mca_btl_base_module_t* btl)
+static int usnic_finalize(struct mca_btl_base_module_t *btl)
 {
-    opal_btl_usnic_module_t* module = (opal_btl_usnic_module_t*)btl;
+    opal_btl_usnic_module_t *module = (opal_btl_usnic_module_t *) btl;
 
     if (module->device_async_event_active) {
         opal_event_del(&(module->device_async_event));
@@ -887,10 +807,8 @@ static int usnic_finalize(struct mca_btl_base_module_t* btl)
 
     opal_btl_usnic_connectivity_unlisten(module);
 
-    finalize_one_channel(module,
-                         &module->mod_channels[USNIC_DATA_CHANNEL]);
-    finalize_one_channel(module,
-                         &module->mod_channels[USNIC_PRIORITY_CHANNEL]);
+    finalize_one_channel(module, &module->mod_channels[USNIC_DATA_CHANNEL]);
+    finalize_one_channel(module, &module->mod_channels[USNIC_PRIORITY_CHANNEL]);
 
     /* Shutdown the stats on this module */
     opal_btl_usnic_stats_finalize(module);
@@ -952,15 +870,12 @@ static int usnic_finalize(struct mca_btl_base_module_t* btl)
     return OPAL_SUCCESS;
 }
 
-static inline unsigned
-get_send_credits(struct opal_btl_usnic_channel_t *chan)
+static inline unsigned get_send_credits(struct opal_btl_usnic_channel_t *chan)
 {
     return chan->credits;
 }
 
-static void
-usnic_do_resends(
-    opal_btl_usnic_module_t *module)
+static void usnic_do_resends(opal_btl_usnic_module_t *module)
 {
     opal_btl_usnic_send_segment_t *sseg;
     opal_btl_usnic_endpoint_t *endpoint;
@@ -970,15 +885,15 @@ usnic_do_resends(
     data_channel = &module->mod_channels[USNIC_DATA_CHANNEL];
 
     count = mca_btl_usnic_component.max_resends_per_iteration;
-    while (count > 0 && (get_send_credits(data_channel) > 1) &&
-           !opal_list_is_empty(&module->pending_resend_segs)) {
+    while (count > 0 && (get_send_credits(data_channel) > 1)
+           && !opal_list_is_empty(&module->pending_resend_segs)) {
 
         /*
          * If a segment is on the re-send list, it will not
          * be in the retransmit hotel.  Post the segment, then check it in.
          */
-        sseg = (opal_btl_usnic_send_segment_t *)
-            opal_list_remove_first(&module->pending_resend_segs);
+        sseg = (opal_btl_usnic_send_segment_t *) opal_list_remove_first(
+            &module->pending_resend_segs);
         endpoint = sseg->ss_parent_frag->sf_endpoint;
 
         /* clobber any stale piggy-backed ACK */
@@ -1006,8 +921,7 @@ usnic_do_resends(
         }
 
         /* restart the retrans timer */
-        ret = opal_hotel_checkin(&endpoint->endpoint_hotel,
-                sseg, &sseg->ss_hotel_room);
+        ret = opal_hotel_checkin(&endpoint->endpoint_hotel, sseg, &sseg->ss_hotel_room);
         if (OPAL_UNLIKELY(OPAL_SUCCESS != ret)) {
             opal_btl_usnic_util_abort("hotel checkin failed\n", __FILE__, __LINE__);
         }
@@ -1025,11 +939,9 @@ usnic_do_resends(
  * ASSUMES THAT THE CALLER HAS ALREADY CHECKED TO SEE IF WE HAVE
  * A SEND CREDIT!
  */
-static void
-usnic_handle_large_send(
-    opal_btl_usnic_module_t *module,
-    opal_btl_usnic_endpoint_t *endpoint,
-    opal_btl_usnic_send_frag_t *frag)
+static void usnic_handle_large_send(opal_btl_usnic_module_t *module,
+                                    opal_btl_usnic_endpoint_t *endpoint,
+                                    opal_btl_usnic_send_frag_t *frag)
 {
     opal_btl_usnic_large_send_frag_t *lfrag;
     opal_btl_usnic_btl_chunk_header_t *chp;
@@ -1037,7 +949,7 @@ usnic_handle_large_send(
     size_t payload_len;
 
     assert(frag->sf_base.uf_type == OPAL_BTL_USNIC_FRAG_LARGE_SEND);
-    lfrag = (opal_btl_usnic_large_send_frag_t *)frag;
+    lfrag = (opal_btl_usnic_large_send_frag_t *) frag;
     if (lfrag->lsf_cur_offset == 0) {
         /* assign a fragment ID */
         do {
@@ -1052,8 +964,7 @@ usnic_handle_large_send(
         sseg = pack_chunk_seg_from_frag(module, lfrag);
     } else {
         /* data was pre-packed in prepare_src */
-        sseg = (opal_btl_usnic_send_segment_t *)
-            opal_list_remove_first(&lfrag->lsf_seg_chain);
+        sseg = (opal_btl_usnic_send_segment_t *) opal_list_remove_first(&lfrag->lsf_seg_chain);
     }
 
     assert(NULL != sseg);
@@ -1085,9 +996,8 @@ usnic_handle_large_send(
     lfrag->lsf_cur_offset += payload_len;
 
 #if MSGDEBUG1
-    opal_output(0, "%s: payload_len=%zd, bytes_left=%zd on_the_fly=%s\n",
-                __func__, payload_len, lfrag->lsf_bytes_left,
-                lfrag->lsf_pack_on_the_fly?"true":"false");
+    opal_output(0, "%s: payload_len=%zd, bytes_left=%zd on_the_fly=%s\n", __func__, payload_len,
+                lfrag->lsf_bytes_left, lfrag->lsf_pack_on_the_fly ? "true" : "false");
 #endif
     /* done with fragment? */
     if (lfrag->lsf_bytes_left == 0) {
@@ -1096,14 +1006,13 @@ usnic_handle_large_send(
          * decide to put it on some other list in the callback
          */
         opal_list_remove_item(&endpoint->endpoint_frag_send_queue,
-                &frag->sf_base.uf_base.super.super);
+                              &frag->sf_base.uf_base.super.super);
 
         /* only callback now if this was not a PUT and we own the fragment,
          * otherwise we need to wait until last byte is ACKed
          */
-        if (frag->sf_base.uf_remote_seg[0].seg_addr.pval == NULL &&
-                (frag->sf_base.uf_base.des_flags &
-                 MCA_BTL_DES_FLAGS_BTL_OWNERSHIP)) {
+        if (frag->sf_base.uf_remote_seg[0].seg_addr.pval == NULL
+            && (frag->sf_base.uf_base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP)) {
 
             OPAL_BTL_USNIC_DO_SEND_FRAG_CB(module, frag, "large");
         }
@@ -1115,9 +1024,7 @@ usnic_handle_large_send(
  * Should only ever be called from usnic_component_progress() to
  * avoid re-entrancy issues.
  */
-void
-opal_btl_usnic_module_progress_sends(
-    opal_btl_usnic_module_t *module)
+void opal_btl_usnic_module_progress_sends(opal_btl_usnic_module_t *module)
 {
     opal_btl_usnic_send_frag_t *frag;
     opal_btl_usnic_send_segment_t *sseg;
@@ -1145,8 +1052,8 @@ opal_btl_usnic_module_progress_sends(
     /*
      * Keep sending as long as there are WQEs and something to do
      */
-    while ((get_send_credits(data_channel) > 1) &&
-           !opal_list_is_empty(&module->endpoints_with_sends)) {
+    while ((get_send_credits(data_channel) > 1)
+           && !opal_list_is_empty(&module->endpoints_with_sends)) {
         opal_btl_usnic_small_send_frag_t *sfrag;
         size_t payload_len;
 
@@ -1157,10 +1064,9 @@ opal_btl_usnic_module_progress_sends(
          * credits.
          */
 
-        endpoint = (opal_btl_usnic_endpoint_t *)
-            opal_list_get_first(&module->endpoints_with_sends);
-        frag = (opal_btl_usnic_send_frag_t *)
-            opal_list_get_first(&endpoint->endpoint_frag_send_queue);
+        endpoint = (opal_btl_usnic_endpoint_t *) opal_list_get_first(&module->endpoints_with_sends);
+        frag = (opal_btl_usnic_send_frag_t *) opal_list_get_first(
+            &endpoint->endpoint_frag_send_queue);
 
         /*
          * small send?  (fragment fits in one segment)
@@ -1172,9 +1078,9 @@ opal_btl_usnic_module_progress_sends(
              * decide to put it on some other list in the callback
              */
             opal_list_remove_item(&endpoint->endpoint_frag_send_queue,
-                    &frag->sf_base.uf_base.super.super);
+                                  &frag->sf_base.uf_base.super.super);
 
-            sfrag = (opal_btl_usnic_small_send_frag_t *)frag;
+            sfrag = (opal_btl_usnic_small_send_frag_t *) frag;
             sseg = &sfrag->ssf_segment;
 
             /* get payload len from segment */
@@ -1182,12 +1088,11 @@ opal_btl_usnic_module_progress_sends(
             sseg->ss_base.us_btl_header->payload_len = payload_len;
 
 #if MSGDEBUG1
-            opal_output(0, "progress send small, frag=%p, ptr=%p, payload=%zd, len=%"PRIu32", ep=%p, tag=%d\n",
-                    (void *)frag,
-                    (void *)sseg->ss_ptr, payload_len,
-                    sseg->ss_len,
-                    (void *)frag->sf_endpoint,
-                    sseg->ss_base.us_btl_header->tag);
+            opal_output(0,
+                        "progress send small, frag=%p, ptr=%p, payload=%zd, len=%" PRIu32
+                        ", ep=%p, tag=%d\n",
+                        (void *) frag, (void *) sseg->ss_ptr, payload_len, sseg->ss_len,
+                        (void *) frag->sf_endpoint, sseg->ss_base.us_btl_header->tag);
 #endif
 
             /* post the send (we have a send credit available) */
@@ -1201,16 +1106,14 @@ opal_btl_usnic_module_progress_sends(
                  * because we are not done with the segment inside.
                  * (ACK not received yet)
                  */
-                if ((frag->sf_base.uf_base.des_flags &
-                        (MCA_BTL_DES_SEND_ALWAYS_CALLBACK |
-                         MCA_BTL_DES_FLAGS_BTL_OWNERSHIP)) ==
-                        (MCA_BTL_DES_SEND_ALWAYS_CALLBACK |
-                         MCA_BTL_DES_FLAGS_BTL_OWNERSHIP)) {
+                if ((frag->sf_base.uf_base.des_flags
+                     & (MCA_BTL_DES_SEND_ALWAYS_CALLBACK | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP))
+                    == (MCA_BTL_DES_SEND_ALWAYS_CALLBACK | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP)) {
                     OPAL_BTL_USNIC_DO_SEND_FRAG_CB(module, frag, "small");
                 }
             }
 
-        /* Large sends... */
+            /* Large sends... */
         } else {
             usnic_handle_large_send(module, endpoint, frag);
         }
@@ -1218,12 +1121,10 @@ opal_btl_usnic_module_progress_sends(
         /* If no more sends or endpoint send window is closed,
          * or no more send credits, remove from send list
          */
-        if (opal_list_is_empty(&endpoint->endpoint_frag_send_queue) ||
-            endpoint->endpoint_send_credits <= 0 ||
-            !WINDOW_OPEN(endpoint)) {
+        if (opal_list_is_empty(&endpoint->endpoint_frag_send_queue)
+            || endpoint->endpoint_send_credits <= 0 || !WINDOW_OPEN(endpoint)) {
 
-            opal_list_remove_item(&module->endpoints_with_sends,
-                    &endpoint->super);
+            opal_list_remove_item(&module->endpoints_with_sends, &endpoint->super);
             endpoint->endpoint_ready_to_send = false;
         }
     }
@@ -1239,8 +1140,7 @@ opal_btl_usnic_module_progress_sends(
         next_endpoint = opal_btl_usnic_get_next_endpoint_needing_ack(endpoint);
 
         /* Is it time to send ACK? */
-        if (endpoint->endpoint_acktime == 0 ||
-            endpoint->endpoint_acktime <= get_ticks()) {
+        if (endpoint->endpoint_acktime == 0 || endpoint->endpoint_acktime <= get_ticks()) {
             if (OPAL_LIKELY(opal_btl_usnic_ack_send(module, endpoint) == OPAL_SUCCESS)) {
                 opal_btl_usnic_remove_from_endpoints_needing_ack(endpoint);
             } else {
@@ -1273,12 +1173,9 @@ opal_btl_usnic_module_progress_sends(
  *  until the last of the data has been copied out by routines called
  *  from opal_btl_usnic_progress_sends()
  */
-static int
-usnic_send(
-    struct mca_btl_base_module_t* base_module,
-    struct mca_btl_base_endpoint_t* base_endpoint,
-    struct mca_btl_base_descriptor_t* descriptor,
-    mca_btl_base_tag_t tag)
+static int usnic_send(struct mca_btl_base_module_t *base_module,
+                      struct mca_btl_base_endpoint_t *base_endpoint,
+                      struct mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag)
 {
     int rc;
     opal_btl_usnic_send_frag_t *frag;
@@ -1288,30 +1185,30 @@ usnic_send(
     opal_btl_usnic_send_segment_t *sseg;
 
     OPAL_THREAD_LOCK(&btl_usnic_lock);
-    endpoint = (opal_btl_usnic_endpoint_t *)base_endpoint;
-    module = (opal_btl_usnic_module_t *)base_module;
-    frag = (opal_btl_usnic_send_frag_t*) descriptor;
+    endpoint = (opal_btl_usnic_endpoint_t *) base_endpoint;
+    module = (opal_btl_usnic_module_t *) base_module;
+    frag = (opal_btl_usnic_send_frag_t *) descriptor;
 
     assert(frag->sf_endpoint == endpoint);
-    frag->sf_base.uf_remote_seg[0].seg_addr.pval = NULL;      /* not a PUT */
+    frag->sf_base.uf_remote_seg[0].seg_addr.pval = NULL; /* not a PUT */
 
     opal_btl_usnic_compute_sf_size(frag);
     frag->sf_ack_bytes_left = frag->sf_size;
 
 #if MSGDEBUG2
-    opal_output(0, "usnic_send: frag=%p, endpoint=%p, tag=%d, sf_size=%d\n",
-            (void *)frag, (void *)endpoint,
-            tag, (int)frag->sf_size);
-#if MSGDEBUG1
-    { unsigned i;
+    opal_output(0, "usnic_send: frag=%p, endpoint=%p, tag=%d, sf_size=%d\n", (void *) frag,
+                (void *) endpoint, tag, (int) frag->sf_size);
+#    if MSGDEBUG1
+    {
+        unsigned i;
         opal_output(0, "  descriptor->des_flags=0x%x\n", descriptor->des_flags);
-        for (i=0; i<descriptor->USNIC_SEND_LOCAL_COUNT; ++i) {
+        for (i = 0; i < descriptor->USNIC_SEND_LOCAL_COUNT; ++i) {
             opal_output(0, "  %d: ptr:%p len:%d\n", i,
-                    descriptor->USNIC_SEND_LOCAL[i].seg_addr.pval,
-                    descriptor->USNIC_SEND_LOCAL[i].seg_len);
+                        descriptor->USNIC_SEND_LOCAL[i].seg_addr.pval,
+                        descriptor->USNIC_SEND_LOCAL[i].seg_len);
         }
     }
-#endif
+#    endif
 #endif
 
     /*
@@ -1319,31 +1216,28 @@ usnic_send(
      * and we have enough send WQEs,
      * then inline and fastpath it
      */
-    if (frag->sf_base.uf_type == OPAL_BTL_USNIC_FRAG_SMALL_SEND &&
-            frag->sf_ack_bytes_left < module->max_tiny_payload &&
-            WINDOW_OPEN(endpoint) &&
-            (get_send_credits(&module->mod_channels[USNIC_DATA_CHANNEL]) >=
-             module->mod_channels[USNIC_DATA_CHANNEL].fastsend_wqe_thresh)) {
+    if (frag->sf_base.uf_type == OPAL_BTL_USNIC_FRAG_SMALL_SEND
+        && frag->sf_ack_bytes_left < module->max_tiny_payload && WINDOW_OPEN(endpoint)
+        && (get_send_credits(&module->mod_channels[USNIC_DATA_CHANNEL])
+            >= module->mod_channels[USNIC_DATA_CHANNEL].fastsend_wqe_thresh)) {
         size_t payload_len;
 
-        sfrag = (opal_btl_usnic_small_send_frag_t *)frag;
+        sfrag = (opal_btl_usnic_small_send_frag_t *) frag;
         sseg = &sfrag->ssf_segment;
 
         payload_len = frag->sf_ack_bytes_left;
         sseg->ss_base.us_btl_header->payload_len = payload_len;
 
-
         /* copy the 2nd SGE into the segment */
         if (frag->sf_base.uf_base.USNIC_SEND_LOCAL_COUNT > 1) {
-            memcpy(((char *)(intptr_t)frag->sf_base.uf_local_seg[0].seg_addr.lval +
-                     frag->sf_base.uf_local_seg[0].seg_len),
-                    frag->sf_base.uf_local_seg[1].seg_addr.pval,
-                    frag->sf_base.uf_local_seg[1].seg_len);
+            memcpy(((char *) (intptr_t) frag->sf_base.uf_local_seg[0].seg_addr.lval
+                    + frag->sf_base.uf_local_seg[0].seg_len),
+                   frag->sf_base.uf_local_seg[1].seg_addr.pval,
+                   frag->sf_base.uf_local_seg[1].seg_len);
 
             /* update 1st segment length */
             frag->sf_base.uf_base.USNIC_SEND_LOCAL_COUNT = 1;
-            frag->sf_base.uf_local_seg[0].seg_len +=
-                frag->sf_base.uf_local_seg[1].seg_len;
+            frag->sf_base.uf_local_seg[0].seg_len += frag->sf_base.uf_local_seg[1].seg_len;
         }
 
         /* assign length */
@@ -1352,7 +1246,7 @@ usnic_send(
         sseg->ss_channel = USNIC_DATA_CHANNEL;
         sseg->ss_base.us_btl_header->tag = tag;
 #if MSGDEBUG1
-        opal_output(0, "INLINE send, sseg=%p", (void *)sseg);
+        opal_output(0, "INLINE send, sseg=%p", (void *) sseg);
 #endif
 
         /* post the segment now (we have a send credit available) */
@@ -1369,14 +1263,14 @@ usnic_send(
                 rc = 0;
             } else {
 #if MSGDEBUG1
-                opal_output(0, "skipping callback for frag %p, returning 1\n", (void *)frag);
+                opal_output(0, "skipping callback for frag %p, returning 1\n", (void *) frag);
 #endif
                 rc = 1;
-                ++module->stats.pml_send_callbacks;   /* returning "1" is an implicit CB */
+                ++module->stats.pml_send_callbacks; /* returning "1" is an implicit CB */
             }
         } else {
 #if MSGDEBUG1
-            opal_output(0, "don't own descriptor, defer callback for frag %p\n", (void *)frag);
+            opal_output(0, "don't own descriptor, defer callback for frag %p\n", (void *) frag);
 #endif
             descriptor->des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
             rc = 0;
@@ -1420,15 +1314,14 @@ static int usnic_sendi(struct mca_btl_base_module_t* btl,
 }
 #endif
 
-
 /*
  * RDMA Memory Pool (de)register callbacks
  */
-static int usnic_reg_mr(void* reg_data, void* base, size_t size,
-                        mca_rcache_base_registration_t* reg)
+static int usnic_reg_mr(void *reg_data, void *base, size_t size,
+                        mca_rcache_base_registration_t *reg)
 {
-    opal_btl_usnic_module_t* mod = (opal_btl_usnic_module_t*)reg_data;
-    opal_btl_usnic_reg_t* ur = (opal_btl_usnic_reg_t*)reg;
+    opal_btl_usnic_module_t *mod = (opal_btl_usnic_module_t *) reg_data;
+    opal_btl_usnic_reg_t *ur = (opal_btl_usnic_reg_t *) reg;
     int rc;
 
     rc = fi_mr_reg(mod->domain, base, size, 0, 0, 0, 0, &ur->ur_mr, NULL);
@@ -1439,15 +1332,14 @@ static int usnic_reg_mr(void* reg_data, void* base, size_t size,
     return OPAL_SUCCESS;
 }
 
-static int usnic_dereg_mr(void* reg_data,
-                          mca_rcache_base_registration_t* reg)
+static int usnic_dereg_mr(void *reg_data, mca_rcache_base_registration_t *reg)
 {
-    opal_btl_usnic_reg_t* ur = (opal_btl_usnic_reg_t*)reg;
+    opal_btl_usnic_reg_t *ur = (opal_btl_usnic_reg_t *) reg;
 
     if (ur->ur_mr != NULL) {
         if (0 != fi_close(&ur->ur_mr->fid)) {
-            opal_output(0, "%s: error unpinning USD memory mr=%p: %s\n",
-                        __func__, (void*) ur->ur_mr, strerror(errno));
+            opal_output(0, "%s: error unpinning USD memory mr=%p: %s\n", __func__,
+                        (void *) ur->ur_mr, strerror(errno));
             return OPAL_ERROR;
         }
     }
@@ -1456,7 +1348,6 @@ static int usnic_dereg_mr(void* reg_data,
     return OPAL_SUCCESS;
 }
 
-
 /*
  * Called back by libevent if an async event occurs on the device
  */
@@ -1464,7 +1355,7 @@ static void module_async_event_callback(int fd, short flags, void *arg)
 {
     char *str = NULL;
     bool fatal = false;
-    opal_btl_usnic_module_t *module = (opal_btl_usnic_module_t*) arg;
+    opal_btl_usnic_module_t *module = (opal_btl_usnic_module_t *) arg;
     uint32_t event;
     struct fi_eq_entry entry;
 
@@ -1476,13 +1367,9 @@ static void module_async_event_callback(int fd, short flags, void *arg)
     }
 
     else if (ret != 0) {
-        opal_show_help("help-mpi-btl-usnic.txt", "libfabric API failed",
-                       true,
-                       opal_process_info.nodename,
-                       module->linux_device_name,
-                       "fi_eq_read()", __FILE__, __LINE__,
-                       ret,
-                       "Failed to get domain event");
+        opal_show_help("help-mpi-btl-usnic.txt", "libfabric API failed", true,
+                       opal_process_info.nodename, module->linux_device_name, "fi_eq_read()",
+                       __FILE__, __LINE__, ret, "Failed to get domain event");
         fatal = true;
     }
 
@@ -1490,18 +1377,17 @@ static void module_async_event_callback(int fd, short flags, void *arg)
         opal_memchecker_base_mem_defined(&event, sizeof(event));
         opal_memchecker_base_mem_defined(&entry, sizeof(entry));
         switch (entry.data) {
-            case 0: // USD_EVENT_LINK_UP:
+        case 0: // USD_EVENT_LINK_UP:
             /* This event should never happen, because OMPI will
                ignore ports that are down, and we should only get this
                event if a port *was* down and is now *up*.  But if we
                ever do get it, it should be a harmless event -- just
                ignore it. */
-            opal_output_verbose(10, USNIC_OUT,
-                                "btl:usnic: got LINK_UP on %s",
+            opal_output_verbose(10, USNIC_OUT, "btl:usnic: got LINK_UP on %s",
                                 module->linux_device_name);
             break;
 
-            case 1: // USD_EVENT_LINK_DOWN:
+        case 1: // USD_EVENT_LINK_DOWN:
             str = "link down";
             /* Fall through */
 
@@ -1514,11 +1400,8 @@ static void module_async_event_callback(int fd, short flags, void *arg)
                will report to us.  However, they're only listed here
                for completeness.  We currently abort if any async
                event other than LINK_UP occurs. */
-            opal_show_help("help-mpi-btl-usnic.txt", "async event",
-                           true,
-                           opal_process_info.nodename,
-                           module->linux_device_name,
-                           str, entry.data);
+            opal_show_help("help-mpi-btl-usnic.txt", "async event", true,
+                           opal_process_info.nodename, module->linux_device_name, str, entry.data);
             fatal = true;
         }
     }
@@ -1534,8 +1417,7 @@ static void module_async_event_callback(int fd, short flags, void *arg)
 /*
  * Create a single libfabric endpoint
  */
-static int create_ep(opal_btl_usnic_module_t* module,
-                            struct opal_btl_usnic_channel_t *channel)
+static int create_ep(opal_btl_usnic_module_t *module, struct opal_btl_usnic_channel_t *channel)
 {
     int rc;
     struct sockaddr_in *sin;
@@ -1544,13 +1426,9 @@ static int create_ep(opal_btl_usnic_module_t* module,
 
     hint = fi_dupinfo(module->fabric_info);
     if (NULL == hint) {
-        opal_show_help("help-mpi-btl-usnic.txt",
-                       "internal error during init",
-                       true,
-                       opal_process_info.nodename,
-                       module->linux_device_name,
-                       "fi_dupinfo() failed", __FILE__, __LINE__,
-                       -1, "Unknown");
+        opal_show_help("help-mpi-btl-usnic.txt", "internal error during init", true,
+                       opal_process_info.nodename, module->linux_device_name, "fi_dupinfo() failed",
+                       __FILE__, __LINE__, -1, "Unknown");
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
@@ -1562,20 +1440,16 @@ static int create_ep(opal_btl_usnic_module_t* module,
     if (0 == mca_btl_usnic_component.udp_port_base) {
         sin->sin_port = 0;
     } else {
-        sin->sin_port = htons(mca_btl_usnic_component.udp_port_base +
-            opal_process_info.my_local_rank);
+        sin->sin_port = htons(mca_btl_usnic_component.udp_port_base
+                              + opal_process_info.my_local_rank);
     }
 
     rc = fi_getinfo(module->libfabric_api, NULL, 0, 0, hint, &channel->info);
     fi_freeinfo(hint);
     if (0 != rc) {
-        opal_show_help("help-mpi-btl-usnic.txt",
-                       "internal error during init",
-                       true,
-                       opal_process_info.nodename,
-                       module->linux_device_name,
-                       "fi_getinfo() failed", __FILE__, __LINE__,
-                       rc, fi_strerror(-rc));
+        opal_show_help("help-mpi-btl-usnic.txt", "internal error during init", true,
+                       opal_process_info.nodename, module->linux_device_name, "fi_getinfo() failed",
+                       __FILE__, __LINE__, rc, fi_strerror(-rc));
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
     if (channel->chan_index != USNIC_PRIORITY_CHANNEL) {
@@ -1588,16 +1462,16 @@ static int create_ep(opal_btl_usnic_module_t* module,
     /* all of the OMPI code assumes IPv4, but some versions of libfabric will
      * return FI_SOCKADDR instead of FI_SOCKADDR_IN, so we need to do a little
      * bit of sanity checking */
-    assert(FI_SOCKADDR_IN == channel->info->addr_format ||
-           FI_SOCKADDR == channel->info->addr_format);
+    assert(FI_SOCKADDR_IN == channel->info->addr_format
+           || FI_SOCKADDR == channel->info->addr_format);
     if (FI_SOCKADDR == channel->info->addr_format) {
         struct sockaddr *sa;
-        sa = (struct sockaddr *)channel->info->src_addr;
+        sa = (struct sockaddr *) channel->info->src_addr;
         assert(AF_INET == sa->sa_family);
     }
 #endif
 
-    sin = (struct sockaddr_in *)channel->info->src_addr;
+    sin = (struct sockaddr_in *) channel->info->src_addr;
     assert(sizeof(struct sockaddr_in) == channel->info->src_addrlen);
 
     /* no matter the version of libfabric, this should hold */
@@ -1605,13 +1479,9 @@ static int create_ep(opal_btl_usnic_module_t* module,
 
     rc = fi_endpoint(module->domain, channel->info, &channel->ep, NULL);
     if (0 != rc || NULL == channel->ep) {
-        opal_show_help("help-mpi-btl-usnic.txt",
-                       "internal error during init",
-                       true,
-                       opal_process_info.nodename,
-                       module->linux_device_name,
-                       "fi_endpoint() failed", __FILE__, __LINE__,
-                       rc, fi_strerror(-rc));
+        opal_show_help("help-mpi-btl-usnic.txt", "internal error during init", true,
+                       opal_process_info.nodename, module->linux_device_name,
+                       "fi_endpoint() failed", __FILE__, __LINE__, rc, fi_strerror(-rc));
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
@@ -1619,72 +1489,50 @@ static int create_ep(opal_btl_usnic_module_t* module,
        long as we asked for */
     if ((int) channel->info->rx_attr->size < channel->chan_rd_num) {
         rc = FI_ETOOSMALL;
-        opal_show_help("help-mpi-btl-usnic.txt",
-                       "internal error during init",
-                       true,
-                       opal_process_info.nodename,
-                       module->linux_device_name,
-                       "endpoint RX queue length is too short", __FILE__, __LINE__,
-                       rc, fi_strerror(rc));
+        opal_show_help("help-mpi-btl-usnic.txt", "internal error during init", true,
+                       opal_process_info.nodename, module->linux_device_name,
+                       "endpoint RX queue length is too short", __FILE__, __LINE__, rc,
+                       fi_strerror(rc));
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
     if ((int) channel->info->tx_attr->size < channel->chan_sd_num) {
         rc = FI_ETOOSMALL;
-        opal_show_help("help-mpi-btl-usnic.txt",
-                       "internal error during init",
-                       true,
-                       opal_process_info.nodename,
-                       module->linux_device_name,
-                       "endpoint TX queue length is too short", __FILE__, __LINE__,
-                       rc, fi_strerror(rc));
+        opal_show_help("help-mpi-btl-usnic.txt", "internal error during init", true,
+                       opal_process_info.nodename, module->linux_device_name,
+                       "endpoint TX queue length is too short", __FILE__, __LINE__, rc,
+                       fi_strerror(rc));
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
     /* attach CQ to EP */
     rc = fi_ep_bind(channel->ep, &channel->cq->fid, FI_SEND);
     if (0 != rc) {
-        opal_show_help("help-mpi-btl-usnic.txt",
-                       "internal error during init",
-                       true,
-                       opal_process_info.nodename,
-                       module->linux_device_name,
-                       "fi_ep_bind() SCQ to EP failed", __FILE__, __LINE__,
-                       rc, fi_strerror(-rc));
+        opal_show_help("help-mpi-btl-usnic.txt", "internal error during init", true,
+                       opal_process_info.nodename, module->linux_device_name,
+                       "fi_ep_bind() SCQ to EP failed", __FILE__, __LINE__, rc, fi_strerror(-rc));
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
     rc = fi_ep_bind(channel->ep, &channel->cq->fid, FI_RECV);
     if (0 != rc) {
-        opal_show_help("help-mpi-btl-usnic.txt",
-                       "internal error during init",
-                       true,
-                       opal_process_info.nodename,
-                       module->linux_device_name,
-                       "fi_ep_bind() RCQ to EP failed", __FILE__, __LINE__,
-                       rc, fi_strerror(-rc));
+        opal_show_help("help-mpi-btl-usnic.txt", "internal error during init", true,
+                       opal_process_info.nodename, module->linux_device_name,
+                       "fi_ep_bind() RCQ to EP failed", __FILE__, __LINE__, rc, fi_strerror(-rc));
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
     rc = fi_ep_bind(channel->ep, &module->av->fid, 0);
     if (0 != rc) {
-        opal_show_help("help-mpi-btl-usnic.txt",
-                       "internal error during init",
-                       true,
-                       opal_process_info.nodename,
-                       module->linux_device_name,
-                       "fi_ep_bind() AV to EP failed", __FILE__, __LINE__,
-                       rc, fi_strerror(-rc));
+        opal_show_help("help-mpi-btl-usnic.txt", "internal error during init", true,
+                       opal_process_info.nodename, module->linux_device_name,
+                       "fi_ep_bind() AV to EP failed", __FILE__, __LINE__, rc, fi_strerror(-rc));
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
     /* Enable the endpoint */
     rc = fi_enable(channel->ep);
     if (0 != rc) {
-        opal_show_help("help-mpi-btl-usnic.txt",
-                       "internal error during init",
-                       true,
-                       opal_process_info.nodename,
-                       module->linux_device_name,
-                       "fi_enable() failed", __FILE__, __LINE__,
-                       rc, fi_strerror(-rc));
+        opal_show_help("help-mpi-btl-usnic.txt", "internal error during init", true,
+                       opal_process_info.nodename, module->linux_device_name, "fi_enable() failed",
+                       __FILE__, __LINE__, rc, fi_strerror(-rc));
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
@@ -1700,13 +1548,9 @@ static int create_ep(opal_btl_usnic_module_t* module,
         addrlen = sizeof(struct sockaddr_in);
         rc = fi_getname(&channel->ep->fid, channel->info->src_addr, &addrlen);
         if (0 != rc) {
-            opal_show_help("help-mpi-btl-usnic.txt",
-                           "internal error during init",
-                           true,
-                           opal_process_info.nodename,
-                           module->linux_device_name,
-                           "fi_getname() failed", __FILE__, __LINE__,
-                           rc, fi_strerror(-rc));
+            opal_show_help("help-mpi-btl-usnic.txt", "internal error during init", true,
+                           opal_process_info.nodename, module->linux_device_name,
+                           "fi_getname() failed", __FILE__, __LINE__, rc, fi_strerror(-rc));
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
         assert(0 != sin->sin_port);
@@ -1722,15 +1566,12 @@ static int create_ep(opal_btl_usnic_module_t* module,
     }
     opal_output_verbose(15, USNIC_OUT,
                         "btl:usnic:create_ep:%s: new usnic local endpoint channel %s: %s:%d",
-                        module->linux_device_name,
-                        str,
-                        inet_ntoa(sin->sin_addr),
+                        module->linux_device_name, str, inet_ntoa(sin->sin_addr),
                         ntohs(sin->sin_port));
 
     return OPAL_SUCCESS;
 }
 
-
 /*
  * finalize channel - release all associated resources
  */
@@ -1765,18 +1606,14 @@ static void finalize_one_channel(opal_btl_usnic_module_t *module,
 /*
  * Initialize a channel
  */
-static int init_one_channel(opal_btl_usnic_module_t *module,
-                            int index,
-                            int max_msg_size,
-                            int rd_num,
-                            int sd_num,
-                            int cq_num)
+static int init_one_channel(opal_btl_usnic_module_t *module, int index, int max_msg_size,
+                            int rd_num, int sd_num, int cq_num)
 {
     int i;
     int rc;
     uint32_t segsize;
     opal_btl_usnic_recv_segment_t *rseg;
-    opal_free_list_item_t* item;
+    opal_free_list_item_t *item;
     struct opal_btl_usnic_channel_t *channel;
     struct fi_cq_attr cq_attr;
 
@@ -1803,13 +1640,9 @@ static int init_one_channel(opal_btl_usnic_module_t *module,
     cq_attr.size = cq_num;
     rc = fi_cq_open(module->domain, &cq_attr, &channel->cq, NULL);
     if (0 != rc) {
-        opal_show_help("help-mpi-btl-usnic.txt",
-                       "internal error during init",
-                       true,
-                       opal_process_info.nodename,
-                       module->linux_device_name,
-                       "failed to create CQ", __FILE__, __LINE__,
-                       rc, fi_strerror(-rc));
+        opal_show_help("help-mpi-btl-usnic.txt", "internal error during init", true,
+                       opal_process_info.nodename, module->linux_device_name, "failed to create CQ",
+                       __FILE__, __LINE__, rc, fi_strerror(-rc));
         goto error;
     }
 
@@ -1817,13 +1650,9 @@ static int init_one_channel(opal_btl_usnic_module_t *module,
        for */
     if ((int) cq_attr.size < cq_num) {
         rc = FI_ETOOSMALL;
-        opal_show_help("help-mpi-btl-usnic.txt",
-                       "internal error during init",
-                       true,
-                       opal_process_info.nodename,
-                       module->linux_device_name,
-                       "created CQ is too small", __FILE__, __LINE__,
-                       rc, fi_strerror(rc));
+        opal_show_help("help-mpi-btl-usnic.txt", "internal error during init", true,
+                       opal_process_info.nodename, module->linux_device_name,
+                       "created CQ is too small", __FILE__, __LINE__, rc, fi_strerror(rc));
         goto error;
     }
 
@@ -1833,16 +1662,15 @@ static int init_one_channel(opal_btl_usnic_module_t *module,
         goto error;
     }
 
-    assert(channel->info->ep_attr->msg_prefix_size ==
-           (uint32_t) mca_btl_usnic_component.transport_header_len);
+    assert(channel->info->ep_attr->msg_prefix_size
+           == (uint32_t) mca_btl_usnic_component.transport_header_len);
 
     opal_output_verbose(15, USNIC_OUT,
-                        "btl:usnic:init_one_channel:%s: channel %s, rx queue size=%" PRIsize_t ", tx queue size=%" PRIsize_t ", cq size=%" PRIsize_t ", send credits=%d",
+                        "btl:usnic:init_one_channel:%s: channel %s, rx queue size=%" PRIsize_t
+                        ", tx queue size=%" PRIsize_t ", cq size=%" PRIsize_t ", send credits=%d",
                         module->linux_device_name,
                         (index == USNIC_PRIORITY_CHANNEL) ? "priority" : "data",
-                        channel->info->rx_attr->size,
-                        channel->info->tx_attr->size,
-                        cq_attr.size,
+                        channel->info->rx_attr->size, channel->info->tx_attr->size, cq_attr.size,
                         channel->credits);
 
     /*
@@ -1850,24 +1678,18 @@ static int init_one_channel(opal_btl_usnic_module_t *module,
      * line size so that each segment is guaranteed to start on a
      * cache line boundary.
      */
-    segsize = (max_msg_size + channel->info->ep_attr->msg_prefix_size +
-            opal_cache_line_size - 1) & ~(opal_cache_line_size - 1);
+    segsize = (max_msg_size + channel->info->ep_attr->msg_prefix_size + opal_cache_line_size - 1)
+              & ~(opal_cache_line_size - 1);
     OBJ_CONSTRUCT(&channel->recv_segs, opal_free_list_t);
-    rc =
-        usnic_compat_free_list_init(&channel->recv_segs,
-                                    sizeof(opal_btl_usnic_recv_segment_t) /* frag size */,
-                                    opal_cache_line_size /* frag alignment */,
-                                    OBJ_CLASS(opal_btl_usnic_recv_segment_t),
-                                    segsize /* payload buffer size */,
-                                    opal_cache_line_size /* payload alignmt */,
-                                    rd_num /* num erorments to alloc */,
-                                    rd_num /* max elements to alloc */,
-                                    rd_num /* num elements per alloc */,
-                                    module->super.btl_mpool /* mpool for (1.x, 2.0: reg, 2.1+: allocation) */,
-                                    0 /* mpool reg flags */,
-                                    module->rcache /* registration cache for 2.1+ */,
-                                    NULL /* item_init */,
-                                    NULL /* item_init_context */);
+    rc = usnic_compat_free_list_init(
+        &channel->recv_segs, sizeof(opal_btl_usnic_recv_segment_t) /* frag size */,
+        opal_cache_line_size /* frag alignment */, OBJ_CLASS(opal_btl_usnic_recv_segment_t),
+        segsize /* payload buffer size */, opal_cache_line_size /* payload alignmt */,
+        rd_num /* num erorments to alloc */, rd_num /* max elements to alloc */,
+        rd_num /* num elements per alloc */,
+        module->super.btl_mpool /* mpool for (1.x, 2.0: reg, 2.1+: allocation) */,
+        0 /* mpool reg flags */, module->rcache /* registration cache for 2.1+ */,
+        NULL /* item_init */, NULL /* item_init_context */);
     channel->recv_segs.ctx = module; /* must come after
                                         free_list_init,
                                         otherwise ctx gets
@@ -1880,32 +1702,23 @@ static int init_one_channel(opal_btl_usnic_module_t *module,
     for (i = 0; i < rd_num; i++) {
         USNIC_COMPAT_FREE_LIST_GET(&channel->recv_segs, item);
         assert(NULL != item);
-        rseg = (opal_btl_usnic_recv_segment_t*)item;
+        rseg = (opal_btl_usnic_recv_segment_t *) item;
 
         if (NULL == rseg) {
-            opal_show_help("help-mpi-btl-usnic.txt",
-                           "internal error during init",
-                           true,
-                           opal_process_info.nodename,
-                           module->linux_device_name,
-                           "Failed to get receive buffer from freelist",
-                           __FILE__, __LINE__);
+            opal_show_help("help-mpi-btl-usnic.txt", "internal error during init", true,
+                           opal_process_info.nodename, module->linux_device_name,
+                           "Failed to get receive buffer from freelist", __FILE__, __LINE__);
             goto error;
         }
 
         /* cannot find length from constructor, set it now */
         rseg->rs_len = segsize;
 
-        rc = fi_recv(channel->ep, rseg->rs_protocol_header, segsize,
-                     NULL, FI_ADDR_UNSPEC, rseg);
+        rc = fi_recv(channel->ep, rseg->rs_protocol_header, segsize, NULL, FI_ADDR_UNSPEC, rseg);
         if (0 != rc) {
-            opal_show_help("help-mpi-btl-usnic.txt",
-                           "internal error during init",
-                           true,
-                           opal_process_info.nodename,
-                           module->linux_device_name,
-                           "Failed to post receive buffer",
-                           __FILE__, __LINE__);
+            opal_show_help("help-mpi-btl-usnic.txt", "internal error during init", true,
+                           opal_process_info.nodename, module->linux_device_name,
+                           "Failed to post receive buffer", __FILE__, __LINE__);
             goto error;
         }
     }
@@ -1920,12 +1733,11 @@ static int init_one_channel(opal_btl_usnic_module_t *module,
 /*
  * generate initial send sequence number
  */
-static opal_btl_usnic_seq_t
-get_initial_seq_no(void)
+static opal_btl_usnic_seq_t get_initial_seq_no(void)
 {
     opal_btl_usnic_seq_t isn;
 
-    isn = (opal_btl_usnic_seq_t)opal_rand(&opal_btl_usnic_rand_buff);
+    isn = (opal_btl_usnic_seq_t) opal_rand(&opal_btl_usnic_rand_buff);
 
     return isn;
 }
@@ -1943,7 +1755,6 @@ static void init_module_globals(opal_btl_usnic_module_t *module)
     OBJ_CONSTRUCT(&module->all_endpoints_lock, opal_mutex_t);
 }
 
-
 /*
  * Initialize our local modex entry from the device attributes
  */
@@ -1958,20 +1769,16 @@ static void init_local_modex_part1(opal_btl_usnic_module_t *module)
     struct sockaddr_in *sin;
 
     sin = info->src_addr;
-    modex->ipv4_addr =       sin->sin_addr.s_addr;
-    modex->netmask =         uip->ui.v1.ui_netmask_be;
-    modex->max_msg_size =    info->ep_attr->max_msg_size;
+    modex->ipv4_addr = sin->sin_addr.s_addr;
+    modex->netmask = uip->ui.v1.ui_netmask_be;
+    modex->max_msg_size = info->ep_attr->max_msg_size;
     modex->link_speed_mbps = uip->ui.v1.ui_link_speed;
 
-    opal_btl_usnic_snprintf_ipv4_addr(module->if_ipv4_addr_str,
-                                      sizeof(module->if_ipv4_addr_str),
-                                      modex->ipv4_addr,
-                                      modex->netmask);
+    opal_btl_usnic_snprintf_ipv4_addr(module->if_ipv4_addr_str, sizeof(module->if_ipv4_addr_str),
+                                      modex->ipv4_addr, modex->netmask);
 
-    opal_output_verbose(5, USNIC_OUT,
-                        "btl:usnic: %s IP charactertics: %s, %u Mbps",
-                        module->linux_device_name,
-                        module->if_ipv4_addr_str,
+    opal_output_verbose(5, USNIC_OUT, "btl:usnic: %s IP charactertics: %s, %u Mbps",
+                        module->linux_device_name, module->if_ipv4_addr_str,
                         modex->link_speed_mbps);
 }
 
@@ -1984,10 +1791,8 @@ static void init_local_modex_part1(opal_btl_usnic_module_t *module)
  */
 static void init_find_transport_header_len(opal_btl_usnic_module_t *module)
 {
-    mca_btl_usnic_component.transport_header_len =
-        module->fabric_info->ep_attr->msg_prefix_size;
-    mca_btl_usnic_component.transport_protocol =
-        module->fabric_info->ep_attr->protocol;
+    mca_btl_usnic_component.transport_header_len = module->fabric_info->ep_attr->msg_prefix_size;
+    mca_btl_usnic_component.transport_protocol = module->fabric_info->ep_attr->protocol;
 
     /* The usnic provider in libfabric v1.0.0 (i.e., API v1.0) treated
        FI_MSG_PREFIX inconsistently between senders and receivers.  It
@@ -2008,12 +1813,10 @@ static void init_find_transport_header_len(opal_btl_usnic_module_t *module)
        attribute to the same value.  So it's ok. */
     uint32_t libfabric_api;
     libfabric_api = fi_version();
-    if (1 == FI_MAJOR(libfabric_api) &&
-        0 == FI_MINOR(libfabric_api)) {
+    if (1 == FI_MAJOR(libfabric_api) && 0 == FI_MINOR(libfabric_api)) {
         mca_btl_usnic_component.prefix_send_offset = 0;
     } else {
-        mca_btl_usnic_component.prefix_send_offset =
-            module->fabric_info->ep_attr->msg_prefix_size;
+        mca_btl_usnic_component.prefix_send_offset = module->fabric_info->ep_attr->msg_prefix_size;
     }
 }
 
@@ -2056,20 +1859,17 @@ static void init_queue_lengths(opal_btl_usnic_module_t *module)
     } else {
         module->prio_sd_num = mca_btl_usnic_component.prio_sd_num;
     }
-    if (module->prio_sd_num > 0 &&
-        (unsigned) module->prio_sd_num >
-         module->fabric_info->tx_attr->size) {
+    if (module->prio_sd_num > 0
+        && (unsigned) module->prio_sd_num > module->fabric_info->tx_attr->size) {
         module->prio_sd_num = module->fabric_info->tx_attr->size;
     }
     if (-1 == mca_btl_usnic_component.prio_rd_num) {
-        module->prio_rd_num =
-            max(128, 32 * USNIC_MCW_SIZE) - 1;
+        module->prio_rd_num = max(128, 32 * USNIC_MCW_SIZE) - 1;
     } else {
         module->prio_rd_num = mca_btl_usnic_component.prio_rd_num;
     }
-    if (module->prio_rd_num > 0 &&
-        (unsigned) module->prio_rd_num >
-         module->fabric_info->rx_attr->size) {
+    if (module->prio_rd_num > 0
+        && (unsigned) module->prio_rd_num > module->fabric_info->rx_attr->size) {
         module->prio_rd_num = module->fabric_info->rx_attr->size;
     }
     if (cq_is_sum) {
@@ -2082,24 +1882,21 @@ static void init_queue_lengths(opal_btl_usnic_module_t *module)
 static void init_payload_lengths(opal_btl_usnic_module_t *module)
 {
     /* Find the max payload this port can handle */
-    module->max_frag_payload =
-        module->local_modex.max_msg_size - /* start with the MTU */
-        sizeof(opal_btl_usnic_btl_header_t) - /* subtract size of
-                                                 the BTL header */
-        mca_btl_usnic_component.prefix_send_offset;
+    module->max_frag_payload = module->local_modex.max_msg_size -    /* start with the MTU */
+                               sizeof(opal_btl_usnic_btl_header_t) - /* subtract size of
+                                                                        the BTL header */
+                               mca_btl_usnic_component.prefix_send_offset;
 
     /* same, but use chunk header */
-    module->max_chunk_payload =
-        module->local_modex.max_msg_size -
-        sizeof(opal_btl_usnic_btl_chunk_header_t) -
-        mca_btl_usnic_component.prefix_send_offset;
+    module->max_chunk_payload = module->local_modex.max_msg_size
+                                - sizeof(opal_btl_usnic_btl_chunk_header_t)
+                                - mca_btl_usnic_component.prefix_send_offset;
 
     /* Priorirty queue MTU and max size */
     if (0 == module->max_tiny_msg_size) {
         module->max_tiny_msg_size = 768;
     }
-    module->max_tiny_payload = module->max_tiny_msg_size -
-        sizeof(opal_btl_usnic_btl_header_t);
+    module->max_tiny_payload = module->max_tiny_msg_size - sizeof(opal_btl_usnic_btl_header_t);
 }
 
 static void init_pml_values(opal_btl_usnic_module_t *module)
@@ -2115,22 +1912,18 @@ static void init_pml_values(opal_btl_usnic_module_t *module)
     if (0 == module->super.btl_eager_limit) {
         /* 150k for 1 module, 25k for >1 module */
         if (1 == mca_btl_usnic_component.num_modules) {
-            module->super.btl_eager_limit =
-                USNIC_DFLT_EAGER_LIMIT_1DEVICE;
+            module->super.btl_eager_limit = USNIC_DFLT_EAGER_LIMIT_1DEVICE;
         } else {
-            module->super.btl_eager_limit =
-                USNIC_DFLT_EAGER_LIMIT_NDEVICES;
+            module->super.btl_eager_limit = USNIC_DFLT_EAGER_LIMIT_NDEVICES;
         }
     }
 
     /* Since we emulate PUT, max_send_size can be same as
        eager_limit */
-    module->super.btl_max_send_size =
-        module->super.btl_eager_limit;
+    module->super.btl_max_send_size = module->super.btl_eager_limit;
 
 #if BTL_VERSION == 30
-    module->super.btl_put_limit =
-        module->super.btl_eager_limit;
+    module->super.btl_put_limit = module->super.btl_eager_limit;
 #endif
 }
 
@@ -2151,8 +1944,8 @@ static void init_connectivity_checker(opal_btl_usnic_module_t *module)
     int rc = opal_btl_usnic_connectivity_listen(module);
     if (OPAL_SUCCESS != rc) {
         OPAL_ERROR_LOG(rc);
-        opal_btl_usnic_util_abort("Failed to notify connectivity agent to listen",
-                                  __FILE__, __LINE__);
+        opal_btl_usnic_util_abort("Failed to notify connectivity agent to listen", __FILE__,
+                                  __LINE__);
     }
 }
 
@@ -2164,7 +1957,8 @@ static void init_hwloc(opal_btl_usnic_module_t *module)
         opal_btl_usnic_hwloc_distance(module);
     } else {
         opal_output_verbose(5, USNIC_OUT,
-                            "btl:usnic: not sorting devices by NUMA distance (MCA btl_usnic_want_numa_device_assignment)");
+                            "btl:usnic: not sorting devices by NUMA distance (MCA "
+                            "btl_usnic_want_numa_device_assignment)");
     }
 }
 
@@ -2183,32 +1977,25 @@ static int init_mpool(opal_btl_usnic_module_t *module)
 {
     struct mca_rcache_base_resources_t rcache_resources;
 
-    rcache_resources.reg_data = (void*)module;
+    rcache_resources.reg_data = (void *) module;
     rcache_resources.sizeof_reg = sizeof(opal_btl_usnic_reg_t);
     rcache_resources.register_mem = usnic_reg_mr;
     rcache_resources.deregister_mem = usnic_dereg_mr;
     rcache_resources.cache_name = mca_btl_usnic_component.usnic_rcache_name;
-    module->rcache =
-        mca_rcache_base_module_create (mca_btl_usnic_component.usnic_rcache_name,
-                                       &module->super, &rcache_resources);
+    module->rcache = mca_rcache_base_module_create(mca_btl_usnic_component.usnic_rcache_name,
+                                                   &module->super, &rcache_resources);
     if (NULL == module->rcache) {
-        opal_show_help("help-mpi-btl-usnic.txt",
-                       "internal error during init",
-                       true,
-                       opal_process_info.nodename,
-                       module->linux_device_name,
-                       "create rcache", __FILE__, __LINE__);
+        opal_show_help("help-mpi-btl-usnic.txt", "internal error during init", true,
+                       opal_process_info.nodename, module->linux_device_name, "create rcache",
+                       __FILE__, __LINE__);
         return OPAL_ERROR;
     }
-    module->super.btl_mpool =
-        mca_mpool_base_module_lookup (mca_btl_usnic_component.usnic_mpool_hints);
+    module->super.btl_mpool = mca_mpool_base_module_lookup(
+        mca_btl_usnic_component.usnic_mpool_hints);
     if (NULL == module->super.btl_mpool) {
-        opal_show_help("help-mpi-btl-usnic.txt",
-                       "internal error during init",
-                       true,
-                       opal_process_info.nodename,
-                       module->linux_device_name,
-                       "create mpool", __FILE__, __LINE__);
+        opal_show_help("help-mpi-btl-usnic.txt", "internal error during init", true,
+                       opal_process_info.nodename, module->linux_device_name, "create mpool",
+                       __FILE__, __LINE__);
         return OPAL_ERROR;
     }
 
@@ -2221,10 +2008,8 @@ static int init_channels(opal_btl_usnic_module_t *module)
     struct fi_av_attr av_attr;
     struct fi_eq_attr eq_attr;
 
-    memset(&module->mod_channels[0], 0,
-           sizeof(module->mod_channels[0]));
-    memset(&module->mod_channels[1], 0,
-           sizeof(module->mod_channels[1]));
+    memset(&module->mod_channels[0], 0, sizeof(module->mod_channels[0]));
+    memset(&module->mod_channels[1], 0, sizeof(module->mod_channels[1]));
 
     memset(&av_attr, 0, sizeof(av_attr));
     av_attr.type = FI_AV_MAP;
@@ -2234,8 +2019,7 @@ static int init_channels(opal_btl_usnic_module_t *module)
         goto destroy;
     }
 
-    rc = fi_open_ops(&module->av->fid, FI_USNIC_AV_OPS_1, 0,
-            (void **)&module->usnic_av_ops, NULL);
+    rc = fi_open_ops(&module->av->fid, FI_USNIC_AV_OPS_1, 0, (void **) &module->usnic_av_ops, NULL);
     if (rc != OPAL_SUCCESS) {
         goto destroy;
     }
@@ -2267,28 +2051,22 @@ static int init_channels(opal_btl_usnic_module_t *module)
     }
 
     /* initialize data and priority channels */
-    rc = init_one_channel(module,
-            USNIC_PRIORITY_CHANNEL,
-            module->max_tiny_msg_size,
-            module->prio_rd_num, module->prio_sd_num, module->prio_cq_num);
+    rc = init_one_channel(module, USNIC_PRIORITY_CHANNEL, module->max_tiny_msg_size,
+                          module->prio_rd_num, module->prio_sd_num, module->prio_cq_num);
     if (rc != OPAL_SUCCESS) {
         goto destroy;
     }
-    rc = init_one_channel(module,
-            USNIC_DATA_CHANNEL,
-            module->fabric_info->ep_attr->max_msg_size,
-            module->rd_num, module->sd_num, module->cq_num);
+    rc = init_one_channel(module, USNIC_DATA_CHANNEL, module->fabric_info->ep_attr->max_msg_size,
+                          module->rd_num, module->sd_num, module->cq_num);
     if (rc != OPAL_SUCCESS) {
         goto destroy;
     }
 
     return OPAL_SUCCESS;
 
- destroy:
-    finalize_one_channel(module,
-                         &module->mod_channels[USNIC_DATA_CHANNEL]);
-    finalize_one_channel(module,
-                         &module->mod_channels[USNIC_PRIORITY_CHANNEL]);
+destroy:
+    finalize_one_channel(module, &module->mod_channels[USNIC_DATA_CHANNEL]);
+    finalize_one_channel(module, &module->mod_channels[USNIC_PRIORITY_CHANNEL]);
 
     return rc;
 }
@@ -2316,22 +2094,16 @@ static void init_async_event(opal_btl_usnic_module_t *module)
 
     ret = fi_control(&module->dom_eq->fid, FI_GETWAIT, &fd);
     if (ret != 0) {
-        opal_show_help("help-mpi-btl-usnic.txt",
-                    "libfabric API failed",
-                   true,
-                   opal_process_info.nodename,
-                   module->linux_device_name,
-                   "fi_control(eq, FI_GETWAIT)", __FILE__, __LINE__,
-                   ret,
-                   fi_strerror(-ret));
+        opal_show_help("help-mpi-btl-usnic.txt", "libfabric API failed", true,
+                       opal_process_info.nodename, module->linux_device_name,
+                       "fi_control(eq, FI_GETWAIT)", __FILE__, __LINE__, ret, fi_strerror(-ret));
         return;
     }
 
     /* Get the fd to receive events on this device.  Keep this in the
        sync event base (not the async event base) */
     opal_event_set(opal_sync_event_base, &(module->device_async_event), fd,
-                   OPAL_EV_READ | OPAL_EV_PERSIST,
-                   module_async_event_callback, module);
+                   OPAL_EV_READ | OPAL_EV_PERSIST, module_async_event_callback, module);
     opal_event_add(&(module->device_async_event), NULL);
     module->device_async_event_active = true;
 }
@@ -2357,101 +2129,66 @@ static void init_freelists(opal_btl_usnic_module_t *module)
     int rc __opal_attribute_unused__;
     uint32_t segsize;
 
-    segsize = (module->local_modex.max_msg_size +
-               mca_btl_usnic_component.prefix_send_offset +
-               opal_cache_line_size - 1) &
-        ~(opal_cache_line_size - 1);
+    segsize = (module->local_modex.max_msg_size + mca_btl_usnic_component.prefix_send_offset
+               + opal_cache_line_size - 1)
+              & ~(opal_cache_line_size - 1);
 
     /* Send frags freelists */
     OBJ_CONSTRUCT(&module->small_send_frags, opal_free_list_t);
     rc = usnic_compat_free_list_init(&module->small_send_frags,
-                             sizeof(opal_btl_usnic_small_send_frag_t),
-                             opal_cache_line_size,
-                             OBJ_CLASS(opal_btl_usnic_small_send_frag_t),
-                             segsize,
-                             opal_cache_line_size,
-                             module->sd_num * 4,
-                             -1,
-                             module->sd_num / 2,
-                             module->super.btl_mpool,
-                             0 /* mpool reg flags */,
-                             module->rcache,
-                             NULL /* item_init */,
-                             NULL /* item_init_context */);
+                                     sizeof(opal_btl_usnic_small_send_frag_t), opal_cache_line_size,
+                                     OBJ_CLASS(opal_btl_usnic_small_send_frag_t), segsize,
+                                     opal_cache_line_size, module->sd_num * 4, -1,
+                                     module->sd_num / 2, module->super.btl_mpool,
+                                     0 /* mpool reg flags */, module->rcache, NULL /* item_init */,
+                                     NULL /* item_init_context */);
     assert(OPAL_SUCCESS == rc);
 
     OBJ_CONSTRUCT(&module->large_send_frags, opal_free_list_t);
     rc = usnic_compat_free_list_init(&module->large_send_frags,
-                             sizeof(opal_btl_usnic_large_send_frag_t),
-                             opal_cache_line_size,
-                             OBJ_CLASS(opal_btl_usnic_large_send_frag_t),
-                             0,  /* payload size */
-                             0,  /* payload align */
-                             module->sd_num / 8,
-                             -1,
-                             module->sd_num / 8,
-                             NULL,
-                             0 /* mpool reg flags */,
-                             NULL /* unused0 */,
-                             NULL /* item_init */,
-                             NULL /* item_init_context */);
+                                     sizeof(opal_btl_usnic_large_send_frag_t), opal_cache_line_size,
+                                     OBJ_CLASS(opal_btl_usnic_large_send_frag_t),
+                                     0, /* payload size */
+                                     0, /* payload align */
+                                     module->sd_num / 8, -1, module->sd_num / 8, NULL,
+                                     0 /* mpool reg flags */, NULL /* unused0 */,
+                                     NULL /* item_init */, NULL /* item_init_context */);
     assert(OPAL_SUCCESS == rc);
 
     OBJ_CONSTRUCT(&module->put_dest_frags, opal_free_list_t);
     rc = usnic_compat_free_list_init(&module->put_dest_frags,
-                             sizeof(opal_btl_usnic_put_dest_frag_t),
-                             opal_cache_line_size,
-                             OBJ_CLASS(opal_btl_usnic_put_dest_frag_t),
-                             0,  /* payload size */
-                             0,  /* payload align */
-                             module->sd_num / 8,
-                             -1,
-                             module->sd_num / 8,
-                             NULL,
-                             0 /* mpool reg flags */,
-                             NULL /* unused0 */,
-                             NULL /* item_init */,
-                             NULL /* item_init_context */);
+                                     sizeof(opal_btl_usnic_put_dest_frag_t), opal_cache_line_size,
+                                     OBJ_CLASS(opal_btl_usnic_put_dest_frag_t),
+                                     0, /* payload size */
+                                     0, /* payload align */
+                                     module->sd_num / 8, -1, module->sd_num / 8, NULL,
+                                     0 /* mpool reg flags */, NULL /* unused0 */,
+                                     NULL /* item_init */, NULL /* item_init_context */);
     assert(OPAL_SUCCESS == rc);
 
     /* list of segments to use for sending */
     OBJ_CONSTRUCT(&module->chunk_segs, opal_free_list_t);
-    rc = usnic_compat_free_list_init(&module->chunk_segs,
-                             sizeof(opal_btl_usnic_chunk_segment_t),
-                             opal_cache_line_size,
-                             OBJ_CLASS(opal_btl_usnic_chunk_segment_t),
-                             segsize,
-                             opal_cache_line_size,
-                             module->sd_num * 4,
-                             -1,
-                             module->sd_num / 2,
-                             module->super.btl_mpool,
-                             0 /* mpool reg flags */,
-                             module->rcache,
-                             NULL /* item_init */,
-                             NULL /* item_init_context */);
+    rc = usnic_compat_free_list_init(&module->chunk_segs, sizeof(opal_btl_usnic_chunk_segment_t),
+                                     opal_cache_line_size,
+                                     OBJ_CLASS(opal_btl_usnic_chunk_segment_t), segsize,
+                                     opal_cache_line_size, module->sd_num * 4, -1,
+                                     module->sd_num / 2, module->super.btl_mpool,
+                                     0 /* mpool reg flags */, module->rcache, NULL /* item_init */,
+                                     NULL /* item_init_context */);
     assert(OPAL_SUCCESS == rc);
 
     /* ACK segments freelist */
     uint32_t ack_segment_len;
-    ack_segment_len = (sizeof(opal_btl_usnic_btl_header_t) +
-                       mca_btl_usnic_component.prefix_send_offset +
-            opal_cache_line_size - 1) & ~(opal_cache_line_size - 1);
+    ack_segment_len = (sizeof(opal_btl_usnic_btl_header_t)
+                       + mca_btl_usnic_component.prefix_send_offset + opal_cache_line_size - 1)
+                      & ~(opal_cache_line_size - 1);
     OBJ_CONSTRUCT(&module->ack_segs, opal_free_list_t);
-    rc = usnic_compat_free_list_init(&module->ack_segs,
-                             sizeof(opal_btl_usnic_ack_segment_t),
-                             opal_cache_line_size,
-                             OBJ_CLASS(opal_btl_usnic_ack_segment_t),
-                             ack_segment_len,
-                             opal_cache_line_size,
-                             module->sd_num * 4,
-                             -1,
-                             module->sd_num / 2,
-                             module->super.btl_mpool,
-                             0 /* mpool reg flags */,
-                             module->rcache,
-                             NULL /* item_init */,
-                             NULL /* item_init_context */);
+    rc = usnic_compat_free_list_init(&module->ack_segs, sizeof(opal_btl_usnic_ack_segment_t),
+                                     opal_cache_line_size, OBJ_CLASS(opal_btl_usnic_ack_segment_t),
+                                     ack_segment_len, opal_cache_line_size, module->sd_num * 4, -1,
+                                     module->sd_num / 2, module->super.btl_mpool,
+                                     0 /* mpool reg flags */, module->rcache, NULL /* item_init */,
+                                     NULL /* item_init_context */);
     assert(OPAL_SUCCESS == rc);
 
     /*
@@ -2461,27 +2198,22 @@ static void init_freelists(opal_btl_usnic_module_t *module)
      * pools simply won't be used in that case.
      */
     module->first_pool = 16; /* 64 kiB */
-    module->last_pool = usnic_fls(module->super.btl_eager_limit-1);
-    module->module_recv_buffers = calloc(module->last_pool+1,
-            sizeof(opal_free_list_t));
+    module->last_pool = usnic_fls(module->super.btl_eager_limit - 1);
+    module->module_recv_buffers = calloc(module->last_pool + 1, sizeof(opal_free_list_t));
     assert(module->module_recv_buffers != NULL);
     for (int i = module->first_pool; i <= module->last_pool; ++i) {
         size_t elt_size = sizeof(opal_btl_usnic_rx_buf_t) - 1 + (1 << i);
         OBJ_CONSTRUCT(&module->module_recv_buffers[i], opal_free_list_t);
-        rc = usnic_compat_free_list_init(&module->module_recv_buffers[i],
-                                 elt_size,
-                                 opal_cache_line_size,
-                                 OBJ_CLASS(opal_btl_usnic_rx_buf_t),
-                                 0,   /* payload size */
-                                 0,   /* payload align */
-                                 128,   /* init elts to alloc */
-                                 128, /* max elts to alloc */
-                                 128,   /* num elts per alloc */
-                                 NULL /* mpool */,
-                                 0 /* mpool reg flags */,
-                                 NULL /* unused0 */,
-                                 NULL /* item_init */,
-                                 NULL /* item_init_context */);
+        rc = usnic_compat_free_list_init(&module->module_recv_buffers[i], elt_size,
+                                         opal_cache_line_size, OBJ_CLASS(opal_btl_usnic_rx_buf_t),
+                                         0,   /* payload size */
+                                         0,   /* payload align */
+                                         128, /* init elts to alloc */
+                                         128, /* max elts to alloc */
+                                         128, /* num elts per alloc */
+                                         NULL /* mpool */, 0 /* mpool reg flags */,
+                                         NULL /* unused0 */, NULL /* item_init */,
+                                         NULL /* item_init_context */);
         assert(OPAL_SUCCESS == rc);
     }
 }
@@ -2504,9 +2236,9 @@ int opal_btl_usnic_module_init(opal_btl_usnic_module_t *module)
     init_procs(module);
 
     int ret;
-    if (OPAL_SUCCESS != (ret = init_mpool(module)) ||
-        OPAL_SUCCESS != (ret = init_channels(module))) {
-        mca_rcache_base_module_destroy (module->rcache);
+    if (OPAL_SUCCESS != (ret = init_mpool(module))
+        || OPAL_SUCCESS != (ret = init_channels(module))) {
+        mca_rcache_base_module_destroy(module->rcache);
         return ret;
     }
 
@@ -2522,8 +2254,8 @@ int opal_btl_usnic_module_init(opal_btl_usnic_module_t *module)
         int rc = opal_btl_usnic_connectivity_listen(module);
         if (OPAL_SUCCESS != rc) {
             OPAL_ERROR_LOG(rc);
-            opal_btl_usnic_util_abort("Failed to notify connectivity agent to listen",
-                                      __FILE__, __LINE__);
+            opal_btl_usnic_util_abort("Failed to notify connectivity agent to listen", __FILE__,
+                                      __LINE__);
         }
     } else {
         /* If we're not doing a connectivity check, just set the port
@@ -2534,7 +2266,6 @@ int opal_btl_usnic_module_init(opal_btl_usnic_module_t *module)
     return OPAL_SUCCESS;
 }
 
-
 opal_btl_usnic_module_t opal_btl_usnic_module_template = {
     .super = {
         .btl_component = &mca_btl_usnic_component.super,
@@ -2557,13 +2288,11 @@ opal_btl_usnic_module_t opal_btl_usnic_module_template = {
 #endif
 
         .btl_exclusivity = MCA_BTL_EXCLUSIVITY_DEFAULT,
-        .btl_flags =
-            MCA_BTL_FLAGS_SEND |
-            MCA_BTL_FLAGS_SEND_INPLACE |
-            /* Need to set FLAGS_SINGLE_ADD_PROCS until
-               btl_recv.h:lookup_sender() can handle an incoming
-               message with an unknown sender. */
-            MCA_BTL_FLAGS_SINGLE_ADD_PROCS,
+        .btl_flags = MCA_BTL_FLAGS_SEND | MCA_BTL_FLAGS_SEND_INPLACE |
+                     /* Need to set FLAGS_SINGLE_ADD_PROCS until
+                        btl_recv.h:lookup_sender() can handle an incoming
+                        message with an unknown sender. */
+                     MCA_BTL_FLAGS_SINGLE_ADD_PROCS,
 
         .btl_add_procs = usnic_add_procs,
         .btl_del_procs = usnic_del_procs,
@@ -2581,5 +2310,4 @@ opal_btl_usnic_module_t opal_btl_usnic_module_template = {
 
         .btl_mpool = NULL,
         .btl_register_error = usnic_register_pml_err_cb,
-    }
-};
+    }};
diff --git a/opal/mca/btl/usnic/btl_usnic_module.h b/opal/mca/btl/usnic/btl_usnic_module.h
index 17de3d090be..4b6ff2ed877 100644
--- a/opal/mca/btl/usnic/btl_usnic_module.h
+++ b/opal/mca/btl/usnic/btl_usnic_module.h
@@ -29,8 +29,8 @@
 
 #include <rdma/fabric.h>
 #include <rdma/fi_cm.h>
-#include <rdma/fi_eq.h>
 #include <rdma/fi_endpoint.h>
+#include <rdma/fi_eq.h>
 #include <rdma/fi_errno.h>
 #include <rdma/fi_ext_usnic.h>
 
@@ -46,10 +46,10 @@
  * These values obtained from empirical testing on Intel E5-2690
  * machines with Sereno/Lexington cards through an N3546 switch.
  */
-#define USNIC_DFLT_EAGER_LIMIT_1DEVICE (150 * 1024)
+#define USNIC_DFLT_EAGER_LIMIT_1DEVICE  (150 * 1024)
 #define USNIC_DFLT_EAGER_LIMIT_NDEVICES (25 * 1024)
-#define USNIC_DFLT_RNDV_EAGER_LIMIT 500
-#define USNIC_DFLT_PACK_LAZY_THRESHOLD (16 * 1024)
+#define USNIC_DFLT_RNDV_EAGER_LIMIT     500
+#define USNIC_DFLT_PACK_LAZY_THRESHOLD  (16 * 1024)
 
 BEGIN_C_DECLS
 
@@ -89,7 +89,7 @@ typedef struct opal_btl_usnic_channel_t {
     /** receive segments & buffers */
     opal_free_list_t recv_segs;
 
-    bool chan_error;    /* set when error detected on channel */
+    bool chan_error; /* set when error detected on channel */
 
     /* statistics */
     uint32_t num_channel_sends;
@@ -146,9 +146,9 @@ typedef struct opal_btl_usnic_module_t {
      * to fragment reassembly info.
      */
     size_t max_tiny_msg_size;
-    size_t max_frag_payload;    /* most that fits in a frag segment */
-    size_t max_chunk_payload;   /* most that can fit in chunk segment */
-    size_t max_tiny_payload;    /* threshold for using inline send */
+    size_t max_frag_payload;  /* most that fits in a frag segment */
+    size_t max_chunk_payload; /* most that can fit in chunk segment */
+    size_t max_tiny_payload;  /* threshold for using inline send */
 
     /** Hash table to keep track of senders */
     opal_hash_table_t senders;
@@ -217,8 +217,7 @@ extern opal_btl_usnic_module_t opal_btl_usnic_module_template;
 
 /* get first endpoint needing ACK */
 static inline opal_btl_usnic_endpoint_t *
-opal_btl_usnic_get_first_endpoint_needing_ack(
-    opal_btl_usnic_module_t *module)
+opal_btl_usnic_get_first_endpoint_needing_ack(opal_btl_usnic_module_t *module)
 {
     opal_list_item_t *item;
     opal_btl_usnic_endpoint_t *endpoint;
@@ -234,8 +233,7 @@ opal_btl_usnic_get_first_endpoint_needing_ack(
 
 /* get next item in chain */
 static inline opal_btl_usnic_endpoint_t *
-opal_btl_usnic_get_next_endpoint_needing_ack(
-    opal_btl_usnic_endpoint_t *endpoint)
+opal_btl_usnic_get_next_endpoint_needing_ack(opal_btl_usnic_endpoint_t *endpoint)
 {
     opal_list_item_t *item;
     opal_btl_usnic_module_t *module;
@@ -252,36 +250,31 @@ opal_btl_usnic_get_next_endpoint_needing_ack(
 }
 
 static inline void
-opal_btl_usnic_remove_from_endpoints_needing_ack(
-    opal_btl_usnic_endpoint_t *endpoint)
+opal_btl_usnic_remove_from_endpoints_needing_ack(opal_btl_usnic_endpoint_t *endpoint)
 {
-    opal_list_remove_item(
-            &(endpoint->endpoint_module->endpoints_that_need_acks),
-            &endpoint->endpoint_ack_li);
+    opal_list_remove_item(&(endpoint->endpoint_module->endpoints_that_need_acks),
+                          &endpoint->endpoint_ack_li);
     endpoint->endpoint_ack_needed = false;
     endpoint->endpoint_acktime = 0;
 #if MSGDEBUG1
-    opal_output(0, "clear ack_needed on %p\n", (void*)endpoint);
+    opal_output(0, "clear ack_needed on %p\n", (void *) endpoint);
 #endif
 }
 
-static inline void
-opal_btl_usnic_add_to_endpoints_needing_ack(
-    opal_btl_usnic_endpoint_t *endpoint)
+static inline void opal_btl_usnic_add_to_endpoints_needing_ack(opal_btl_usnic_endpoint_t *endpoint)
 {
     opal_list_append(&(endpoint->endpoint_module->endpoints_that_need_acks),
-            &endpoint->endpoint_ack_li);
+                     &endpoint->endpoint_ack_li);
     endpoint->endpoint_ack_needed = true;
 #if MSGDEBUG1
-    opal_output(0, "set ack_needed on %p\n", (void*)endpoint);
+    opal_output(0, "set ack_needed on %p\n", (void *) endpoint);
 #endif
 }
 
 /*
  * Initialize a module
  */
-int opal_btl_usnic_module_init(opal_btl_usnic_module_t* module);
-
+int opal_btl_usnic_module_init(opal_btl_usnic_module_t *module);
 
 /*
  * Progress pending sends on a module
@@ -289,10 +282,8 @@ int opal_btl_usnic_module_init(opal_btl_usnic_module_t* module);
 void opal_btl_usnic_module_progress_sends(opal_btl_usnic_module_t *module);
 
 /* opal_output statistics that are useful for debugging */
-void opal_btl_usnic_print_stats(
-    opal_btl_usnic_module_t *module,
-    const char *prefix,
-    bool reset_stats);
+void opal_btl_usnic_print_stats(opal_btl_usnic_module_t *module, const char *prefix,
+                                bool reset_stats);
 
 END_C_DECLS
 #endif
diff --git a/opal/mca/btl/usnic/btl_usnic_proc.c b/opal/mca/btl/usnic/btl_usnic_proc.c
index a2878c5b509..4a87a729f1d 100644
--- a/opal/mca/btl/usnic/btl_usnic_proc.c
+++ b/opal/mca/btl/usnic/btl_usnic_proc.c
@@ -23,34 +23,32 @@
 
 #include "opal_config.h"
 
-#include "opal_stdint.h"
-#include "opal/util/arch.h"
-#include "opal/util/show_help.h"
 #include "opal/constants.h"
+#include "opal/util/arch.h"
 #include "opal/util/bipartite_graph.h"
+#include "opal/util/show_help.h"
 #include "opal/util/string_copy.h"
+#include "opal_stdint.h"
 
-#include "btl_usnic_compat.h"
 #include "btl_usnic.h"
-#include "btl_usnic_proc.h"
+#include "btl_usnic_compat.h"
 #include "btl_usnic_endpoint.h"
 #include "btl_usnic_module.h"
+#include "btl_usnic_proc.h"
 #include "btl_usnic_util.h"
 
 /* larger weight values are more desirable (i.e., worth, not cost) */
-enum {
-    WEIGHT_UNREACHABLE = -1
-};
+enum { WEIGHT_UNREACHABLE = -1 };
 
 /* Helper macros for "match_modex" and friends for translating between array
  * indices and vertex IDs.  Module vertices always come first in the graph,
  * followed by proc (endpoint) vertices. */
-#define PROC_VERTEX(modex_idx) (mca_btl_usnic_component.num_modules + modex_idx)
-#define MODULE_VERTEX(module_idx) (module_idx)
-#define PROC_INDEX(proc_vertex) ((proc_vertex) - mca_btl_usnic_component.num_modules)
+#define PROC_VERTEX(modex_idx)      (mca_btl_usnic_component.num_modules + modex_idx)
+#define MODULE_VERTEX(module_idx)   (module_idx)
+#define PROC_INDEX(proc_vertex)     ((proc_vertex) -mca_btl_usnic_component.num_modules)
 #define MODULE_INDEX(module_vertex) (module_vertex)
 
-static void proc_construct(opal_btl_usnic_proc_t* proc)
+static void proc_construct(opal_btl_usnic_proc_t *proc)
 {
     proc->proc_opal = 0;
     proc->proc_modex = NULL;
@@ -65,8 +63,7 @@ static void proc_construct(opal_btl_usnic_proc_t* proc)
     opal_list_append(&mca_btl_usnic_component.usnic_procs, &proc->super);
 }
 
-
-static void proc_destruct(opal_btl_usnic_proc_t* proc)
+static void proc_destruct(opal_btl_usnic_proc_t *proc)
 {
     /* remove from list of all proc instances */
     opal_list_remove_item(&mca_btl_usnic_component.usnic_procs, &proc->super);
@@ -94,27 +91,21 @@ static void proc_destruct(opal_btl_usnic_proc_t* proc)
     }
 }
 
-
-OBJ_CLASS_INSTANCE(opal_btl_usnic_proc_t,
-                   opal_list_item_t,
-                   proc_construct,
-                   proc_destruct);
+OBJ_CLASS_INSTANCE(opal_btl_usnic_proc_t, opal_list_item_t, proc_construct, proc_destruct);
 
 /*
  * Look for an existing usnic process instance based on the
  * associated opal_proc_t instance.
  */
-opal_btl_usnic_proc_t *
-opal_btl_usnic_proc_lookup_ompi(opal_proc_t* opal_proc)
+opal_btl_usnic_proc_t *opal_btl_usnic_proc_lookup_ompi(opal_proc_t *opal_proc)
 {
-    opal_btl_usnic_proc_t* usnic_proc;
-
-    for (usnic_proc = (opal_btl_usnic_proc_t*)
-             opal_list_get_first(&mca_btl_usnic_component.usnic_procs);
-         usnic_proc != (opal_btl_usnic_proc_t*)
-             opal_list_get_end(&mca_btl_usnic_component.usnic_procs);
-         usnic_proc  = (opal_btl_usnic_proc_t*)
-             opal_list_get_next(usnic_proc)) {
+    opal_btl_usnic_proc_t *usnic_proc;
+
+    for (usnic_proc = (opal_btl_usnic_proc_t *) opal_list_get_first(
+             &mca_btl_usnic_component.usnic_procs);
+         usnic_proc
+         != (opal_btl_usnic_proc_t *) opal_list_get_end(&mca_btl_usnic_component.usnic_procs);
+         usnic_proc = (opal_btl_usnic_proc_t *) opal_list_get_next(usnic_proc)) {
         if (usnic_proc->proc_opal == opal_proc) {
             return usnic_proc;
         }
@@ -123,38 +114,32 @@ opal_btl_usnic_proc_lookup_ompi(opal_proc_t* opal_proc)
     return NULL;
 }
 
-
 /*
  * Look for an existing usnic proc based on a hashed RTE process
  * name.
  */
-opal_btl_usnic_endpoint_t *
-opal_btl_usnic_proc_lookup_endpoint(opal_btl_usnic_module_t *receiver,
-                                    uint64_t sender_proc_name)
+opal_btl_usnic_endpoint_t *opal_btl_usnic_proc_lookup_endpoint(opal_btl_usnic_module_t *receiver,
+                                                               uint64_t sender_proc_name)
 {
     opal_btl_usnic_proc_t *proc;
     opal_btl_usnic_endpoint_t *endpoint;
     opal_list_item_t *item;
 
-    MSGDEBUG1_OUT("lookup_endpoint: recvmodule=%p sendhash=0x%" PRIx64,
-                  (void *)receiver, sender_proc_name);
+    MSGDEBUG1_OUT("lookup_endpoint: recvmodule=%p sendhash=0x%" PRIx64, (void *) receiver,
+                  sender_proc_name);
 
     opal_mutex_lock(&receiver->all_endpoints_lock);
     for (item = opal_list_get_first(&receiver->all_endpoints);
-         item != opal_list_get_end(&receiver->all_endpoints);
-         item = opal_list_get_next(item)) {
-        endpoint = container_of(item, opal_btl_usnic_endpoint_t,
-                                endpoint_endpoint_li);
+         item != opal_list_get_end(&receiver->all_endpoints); item = opal_list_get_next(item)) {
+        endpoint = container_of(item, opal_btl_usnic_endpoint_t, endpoint_endpoint_li);
         proc = endpoint->endpoint_proc;
         /* Note that this works today because opal_proc_t->proc_name
            is unique across the universe.  George is potentially
            working to give handles instead of proc names, and then
            have a function pointer to perform comparisons.  This would
            be bad here in the critical path, though... */
-        if (usnic_compat_rte_hash_name(&(proc->proc_opal->proc_name)) ==
-            sender_proc_name) {
-            MSGDEBUG1_OUT("lookup_endpoint: matched endpoint=%p",
-                          (void *)endpoint);
+        if (usnic_compat_rte_hash_name(&(proc->proc_opal->proc_name)) == sender_proc_name) {
+            MSGDEBUG1_OUT("lookup_endpoint: matched endpoint=%p", (void *) endpoint);
             opal_mutex_unlock(&receiver->all_endpoints_lock);
             return endpoint;
         }
@@ -172,8 +157,7 @@ opal_btl_usnic_proc_lookup_endpoint(opal_btl_usnic_module_t *receiver,
  * Returns OPAL_ERR_UNREACH if we can't reach the peer (i.e., we can't
  * find their modex data).
  */
-static int create_proc(opal_proc_t *opal_proc,
-                       opal_btl_usnic_proc_t **usnic_proc)
+static int create_proc(opal_proc_t *opal_proc, opal_btl_usnic_proc_t **usnic_proc)
 {
     opal_btl_usnic_proc_t *proc = NULL;
     size_t size;
@@ -192,8 +176,8 @@ static int create_proc(opal_proc_t *opal_proc,
     proc->proc_opal = opal_proc;
 
     /* query for the peer address info */
-    usnic_compat_modex_recv(&rc, &mca_btl_usnic_component.super.btl_version,
-                            opal_proc, &proc->proc_modex, &size);
+    usnic_compat_modex_recv(&rc, &mca_btl_usnic_component.super.btl_version, opal_proc,
+                            &proc->proc_modex, &size);
 
     /* If this proc simply doesn't have this key, then they're not
        running the usnic BTL -- just ignore them.  Otherwise, show an
@@ -202,13 +186,9 @@ static int create_proc(opal_proc_t *opal_proc,
         OBJ_RELEASE(proc);
         return OPAL_ERR_UNREACH;
     } else if (OPAL_SUCCESS != rc) {
-        opal_show_help("help-mpi-btl-usnic.txt",
-                       "internal error during init",
-                       true,
-                       opal_process_info.nodename,
-                       "<none>", "<none>",
-                       "opal_modex_recv() failed", __FILE__, __LINE__,
-                       opal_strerror(rc));
+        opal_show_help("help-mpi-btl-usnic.txt", "internal error during init", true,
+                       opal_process_info.nodename, "<none>", "<none>", "opal_modex_recv() failed",
+                       __FILE__, __LINE__, opal_strerror(rc));
         OBJ_RELEASE(proc);
         return OPAL_ERROR;
     }
@@ -216,16 +196,12 @@ static int create_proc(opal_proc_t *opal_proc,
     if ((size % sizeof(opal_btl_usnic_modex_t)) != 0) {
         char msg[1024];
 
-        snprintf(msg, sizeof(msg),
-                 "sizeof(modex for peer %s data) == %d, expected multiple of %d",
-                 usnic_compat_proc_name_print(&opal_proc->proc_name),
-                 (int) size, (int) sizeof(opal_btl_usnic_modex_t));
-        opal_show_help("help-mpi-btl-usnic.txt", "internal error during init",
-                       true,
-                       opal_process_info.nodename,
-                       "<none>", 0,
-                       "invalid modex data", __FILE__, __LINE__,
-                       msg);
+        snprintf(msg, sizeof(msg), "sizeof(modex for peer %s data) == %d, expected multiple of %d",
+                 usnic_compat_proc_name_print(&opal_proc->proc_name), (int) size,
+                 (int) sizeof(opal_btl_usnic_modex_t));
+        opal_show_help("help-mpi-btl-usnic.txt", "internal error during init", true,
+                       opal_process_info.nodename, "<none>", 0, "invalid modex data", __FILE__,
+                       __LINE__, msg);
 
         OBJ_RELEASE(proc);
         return OPAL_ERR_VALUE_OUT_OF_BOUNDS;
@@ -233,22 +209,16 @@ static int create_proc(opal_proc_t *opal_proc,
 
     /* See if the peer has the same underlying wire protocol as me.
        If not, then print an error and ignore this peer. */
-// RFXXX - things are weird when i force this to fail
-    if (mca_btl_usnic_component.transport_protocol !=
-        proc->proc_modex->protocol) {
+    // RFXXX - things are weird when i force this to fail
+    if (mca_btl_usnic_component.transport_protocol != proc->proc_modex->protocol) {
         uint64_t proto;
         char protostr[32];
         proto = mca_btl_usnic_component.transport_protocol;
         memset(protostr, 0, sizeof(protostr));
-        opal_string_copy(protostr, fi_tostr(&proto, FI_TYPE_PROTOCOL),
-                sizeof(protostr));
+        opal_string_copy(protostr, fi_tostr(&proto, FI_TYPE_PROTOCOL), sizeof(protostr));
         proto = proc->proc_modex->protocol;
-        opal_show_help("help-mpi-btl-usnic.txt",
-                       "transport mismatch",
-                       true,
-                       opal_process_info.nodename,
-                       protostr,
-                       "peer",
+        opal_show_help("help-mpi-btl-usnic.txt", "transport mismatch", true,
+                       opal_process_info.nodename, protostr, "peer",
                        fi_tostr(&proto, FI_TYPE_PROTOCOL));
 
         OBJ_RELEASE(proc);
@@ -262,16 +232,15 @@ static int create_proc(opal_proc_t *opal_proc,
         return OPAL_ERR_UNREACH;
     }
 
-    proc->proc_modex_claimed = (bool*)
-        calloc(proc->proc_modex_count, sizeof(bool));
+    proc->proc_modex_claimed = (bool *) calloc(proc->proc_modex_count, sizeof(bool));
     if (NULL == proc->proc_modex_claimed) {
         OPAL_ERROR_LOG(OPAL_ERR_OUT_OF_RESOURCE);
         OBJ_RELEASE(proc);
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
-    proc->proc_endpoints = (mca_btl_base_endpoint_t**)
-        calloc(proc->proc_modex_count, sizeof(mca_btl_base_endpoint_t*));
+    proc->proc_endpoints = (mca_btl_base_endpoint_t **) calloc(proc->proc_modex_count,
+                                                               sizeof(mca_btl_base_endpoint_t *));
     if (NULL == proc->proc_endpoints) {
         OPAL_ERROR_LOG(OPAL_ERR_OUT_OF_RESOURCE);
         OBJ_RELEASE(proc);
@@ -285,9 +254,8 @@ static int create_proc(opal_proc_t *opal_proc,
 /* Compare the addresses of the local interface corresponding to module and the
  * remote interface corresponding to proc_modex_addr.  Returns a weight value
  * (higher values indicate more desirable connections). */
-static uint64_t compute_weight(
-    opal_btl_usnic_module_t *module,
-    opal_btl_usnic_modex_t *proc_modex_addr)
+static uint64_t compute_weight(opal_btl_usnic_module_t *module,
+                               opal_btl_usnic_modex_t *proc_modex_addr)
 {
     char my_ip_string[INET_ADDRSTRLEN], peer_ip_string[INET_ADDRSTRLEN];
     struct sockaddr_in sin;
@@ -300,10 +268,8 @@ static uint64_t compute_weight(
 
     uip = &module->usnic_info;
     sinp = module->fabric_info->src_addr;
-    inet_ntop(AF_INET, &sinp->sin_addr,
-              my_ip_string, sizeof(my_ip_string));
-    inet_ntop(AF_INET, &proc_modex_addr->ipv4_addr,
-              peer_ip_string, sizeof(peer_ip_string));
+    inet_ntop(AF_INET, &sinp->sin_addr, my_ip_string, sizeof(my_ip_string));
+    inet_ntop(AF_INET, &proc_modex_addr->ipv4_addr, peer_ip_string, sizeof(peer_ip_string));
 
     /* Just compare the CIDR-masked IP address to see if they're on
        the same network.  If so, we're good. */
@@ -311,14 +277,11 @@ static uint64_t compute_weight(
     peernet = proc_modex_addr->ipv4_addr & proc_modex_addr->netmask;
     opal_output_verbose(5, USNIC_OUT,
                         "btl:usnic:%s: checking my IP address/subnet (%s/%d) vs. peer (%s/%d): %s",
-                        __func__, my_ip_string,
-                        usnic_netmask_to_cidrlen(uip->ui.v1.ui_netmask_be),
-                        peer_ip_string,
-                        usnic_netmask_to_cidrlen(proc_modex_addr->netmask),
+                        __func__, my_ip_string, usnic_netmask_to_cidrlen(uip->ui.v1.ui_netmask_be),
+                        peer_ip_string, usnic_netmask_to_cidrlen(proc_modex_addr->netmask),
                         (mynet == peernet ? "match" : "DO NOT match"));
 
-    min_link_speed_gbps = MIN(module->super.btl_bandwidth,
-                              proc_modex_addr->link_speed_mbps) / 1000;
+    min_link_speed_gbps = MIN(module->super.btl_bandwidth, proc_modex_addr->link_speed_mbps) / 1000;
 
     /* Returned metric is:
      *    0 - same VLAN
@@ -332,8 +295,7 @@ static uint64_t compute_weight(
     err = module->usnic_av_ops->get_distance(module->av, &sin, &metric);
     if (0 != err || (0 == err && -1 == metric)) {
         return 0; /* no connectivity */
-    }
-    else {
+    } else {
         /* Format in binary    MSB                             LSB
          * most sig. 32-bits:  00000000 0000000A BBBBBBBB 00000001
          * least sig. 32-bits: CCCCCCCC CCCCCCCC CCCCCCCC CCCCCCCC
@@ -359,10 +321,8 @@ static uint64_t compute_weight(
                                 min_link_speed_gbps);
             min_link_speed_gbps = 0xff;
         }
-        return ((uint64_t)(mynet == peernet) << 48) |
-               ((uint64_t)(min_link_speed_gbps & 0xff) << 40) |
-               ((uint64_t)0x1 << 32) |
-               (/*metric=*/0);
+        return ((uint64_t)(mynet == peernet) << 48) | ((uint64_t)(min_link_speed_gbps & 0xff) << 40)
+               | ((uint64_t) 0x1 << 32) | (/*metric=*/0);
     }
 }
 
@@ -371,23 +331,20 @@ static uint64_t compute_weight(
  * (DJG: this unfortunately knows a bit too much about the internals of
  * "match_modex")
  */
-static void edge_pairs_to_match_table(
-    opal_btl_usnic_proc_t *proc,
-    bool proc_is_left,
-    int nme,
-    int *me)
+static void edge_pairs_to_match_table(opal_btl_usnic_proc_t *proc, bool proc_is_left, int nme,
+                                      int *me)
 {
     int i;
     int left, right;
     int module_idx, proc_idx;
     int num_modules;
 
-    num_modules = (int)mca_btl_usnic_component.num_modules;
+    num_modules = (int) mca_btl_usnic_component.num_modules;
 
     assert(nme >= 0);
     for (i = 0; i < nme; ++i) {
-        left  = me[2*i+0];
-        right = me[2*i+1];
+        left = me[2 * i + 0];
+        right = me[2 * i + 1];
 
         if (proc_is_left) {
             proc_idx = PROC_INDEX(left);
@@ -397,7 +354,7 @@ static void edge_pairs_to_match_table(
             proc_idx = PROC_INDEX(right);
         }
         assert(module_idx >= 0 && module_idx < num_modules);
-        assert(proc_idx >= 0 && proc_idx < (int)proc->proc_modex_count);
+        assert(proc_idx >= 0 && proc_idx < (int) proc->proc_modex_count);
         proc->proc_ep_match_table[module_idx] = proc_idx;
         proc->proc_match_exists = true;
     }
@@ -405,17 +362,15 @@ static void edge_pairs_to_match_table(
     /* emit match summary for debugging purposes */
     for (i = 0; i < num_modules; ++i) {
         if (-1 != proc->proc_ep_match_table[i]) {
-            opal_output_verbose(5, USNIC_OUT,
-                                "btl:usnic:%s: module[%d] (%p) should claim endpoint[%d] on proc %p",
-                                __func__, i,
-                                (void *)mca_btl_usnic_component.usnic_active_modules[i],
-                                proc->proc_ep_match_table[i], (void *)proc);
+            opal_output_verbose(
+                5, USNIC_OUT, "btl:usnic:%s: module[%d] (%p) should claim endpoint[%d] on proc %p",
+                __func__, i, (void *) mca_btl_usnic_component.usnic_active_modules[i],
+                proc->proc_ep_match_table[i], (void *) proc);
         } else {
-            opal_output_verbose(5, USNIC_OUT,
-                                "btl:usnic:%s: module[%d] (%p) will NOT claim an endpoint on proc %p",
-                                __func__, i,
-                                (void *)mca_btl_usnic_component.usnic_active_modules[i],
-                                (void *)proc);
+            opal_output_verbose(
+                5, USNIC_OUT, "btl:usnic:%s: module[%d] (%p) will NOT claim an endpoint on proc %p",
+                __func__, i, (void *) mca_btl_usnic_component.usnic_active_modules[i],
+                (void *) proc);
         }
     }
 }
@@ -425,10 +380,8 @@ static void edge_pairs_to_match_table(
  * remote interfaces.  The resulting vertices will always have the module
  * vertices appear before the proc vertices.
  */
-static int create_proc_module_graph(
-    opal_btl_usnic_proc_t *proc,
-    bool proc_is_left,
-    opal_bp_graph_t **g_out)
+static int create_proc_module_graph(opal_btl_usnic_proc_t *proc, bool proc_is_left,
+                                    opal_bp_graph_t **g_out)
 {
     int err;
     int i, j;
@@ -441,7 +394,7 @@ static int create_proc_module_graph(
     }
     *g_out = NULL;
 
-    num_modules = (int)mca_btl_usnic_component.num_modules;
+    num_modules = (int) mca_btl_usnic_component.num_modules;
 
     /* Construct a bipartite graph with remote interfaces on the one side and
      * local interfaces (modules) on the other. */
@@ -454,28 +407,26 @@ static int create_proc_module_graph(
     /* create vertices for each interface (local and remote) */
     for (i = 0; i < num_modules; ++i) {
         int idx = -1;
-        err = opal_bp_graph_add_vertex(g,
-				       mca_btl_usnic_component.usnic_active_modules[i],
-				       &idx);
+        err = opal_bp_graph_add_vertex(g, mca_btl_usnic_component.usnic_active_modules[i], &idx);
         if (OPAL_SUCCESS != err) {
             OPAL_ERROR_LOG(err);
             goto out_free_graph;
         }
         assert(idx == MODULE_VERTEX(i));
     }
-    for (i = 0; i < (int)proc->proc_modex_count; ++i) {
+    for (i = 0; i < (int) proc->proc_modex_count; ++i) {
         int idx = -1;
         err = opal_bp_graph_add_vertex(g, &proc->proc_modex[i], &idx);
         if (OPAL_SUCCESS != err) {
             OPAL_ERROR_LOG(err);
             goto out_free_graph;
         }
-        assert(idx == (int)PROC_VERTEX(i));
+        assert(idx == (int) PROC_VERTEX(i));
     }
 
     /* now add edges between interfaces that can communicate */
     for (i = 0; i < num_modules; ++i) {
-        for (j = 0; j < (int)proc->proc_modex_count; ++j) {
+        for (j = 0; j < (int) proc->proc_modex_count; ++j) {
             int64_t weight, cost;
 
             /* assumption: compute_weight returns the same weight on the
@@ -484,11 +435,11 @@ static int create_proc_module_graph(
                                     &proc->proc_modex[j]);
 
             opal_output_verbose(20, USNIC_OUT,
-                                "btl:usnic:%s: weight=0x%016" PRIx64 " for edge module[%d] (%p) <--> endpoint[%d] on proc %p",
-                                __func__,
-                                weight, i,
-                                (void *)mca_btl_usnic_component.usnic_active_modules[i],
-                                j, (void *)proc);
+                                "btl:usnic:%s: weight=0x%016" PRIx64
+                                " for edge module[%d] (%p) <--> endpoint[%d] on proc %p",
+                                __func__, weight, i,
+                                (void *) mca_btl_usnic_component.usnic_active_modules[i], j,
+                                (void *) proc);
 
             if (WEIGHT_UNREACHABLE == weight) {
                 continue;
@@ -508,11 +459,12 @@ static int create_proc_module_graph(
                 v = PROC_VERTEX(j);
             }
             opal_output_verbose(20, USNIC_OUT,
-                                "btl:usnic:%s: adding edge (%d,%d) with cost=%" PRIi64 " for edge module[%d] <--> endpoint[%d]",
+                                "btl:usnic:%s: adding edge (%d,%d) with cost=%" PRIi64
+                                " for edge module[%d] <--> endpoint[%d]",
                                 __func__, u, v, cost, i, j);
             err = opal_bp_graph_add_edge(g, u, v, cost,
-					 /*capacity=*/1,
-					 /*e_data=*/NULL);
+                                         /*capacity=*/1,
+                                         /*e_data=*/NULL);
             if (OPAL_SUCCESS != err) {
                 OPAL_ERROR_LOG(err);
                 goto out_free_graph;
@@ -541,9 +493,7 @@ static int create_proc_module_graph(
  * canonicalize this match ordering somehow, probably by (jobid,vpid) pair or
  * by the interface MAC or IP address.
  */
-static int match_modex(opal_btl_usnic_module_t *module,
-                       opal_btl_usnic_proc_t *proc,
-                       int *index_out)
+static int match_modex(opal_btl_usnic_module_t *module, opal_btl_usnic_proc_t *proc, int *index_out)
 {
     int err = OPAL_SUCCESS;
     size_t i;
@@ -559,15 +509,14 @@ static int match_modex(opal_btl_usnic_module_t *module,
     num_modules = mca_btl_usnic_component.num_modules;
 
     opal_output_verbose(20, USNIC_OUT, "btl:usnic:%s: module=%p proc=%p with dimensions %d x %d",
-                        __func__, (void *)module, (void *)proc,
-                        num_modules, (int)proc->proc_modex_count);
+                        __func__, (void *) module, (void *) proc, num_modules,
+                        (int) proc->proc_modex_count);
 
     /* We compute an interface match-up table once for each (module,proc) pair
      * and cache it in the proc.  Store per-proc instead of per-module, since
      * MPI dynamic process routines can add procs but not new modules. */
     if (NULL == proc->proc_ep_match_table) {
-        proc->proc_ep_match_table = malloc(num_modules *
-                                       sizeof(*proc->proc_ep_match_table));
+        proc->proc_ep_match_table = malloc(num_modules * sizeof(*proc->proc_ep_match_table));
         if (NULL == proc->proc_ep_match_table) {
             OPAL_ERROR_LOG(OPAL_ERR_OUT_OF_RESOURCE);
             return OPAL_ERR_OUT_OF_RESOURCE;
@@ -588,9 +537,8 @@ static int match_modex(opal_btl_usnic_module_t *module,
         proc_is_left = (proc->proc_opal->proc_name <
                         opal_proc_local_get()->proc_name);
 #else
-        proc_is_left =
-            usnic_compat_proc_name_compare(proc->proc_opal->proc_name,
-                                           opal_proc_local_get()->proc_name);
+        proc_is_left = usnic_compat_proc_name_compare(proc->proc_opal->proc_name,
+                                                      opal_proc_local_get()->proc_name);
 #endif
 
         err = create_proc_module_graph(proc, proc_is_left, &g);
@@ -616,11 +564,10 @@ static int match_modex(opal_btl_usnic_module_t *module,
         }
     }
 
-
     if (!proc->proc_match_exists) {
-        opal_output_verbose(5, USNIC_OUT, "btl:usnic:%s: unable to find any valid interface pairs for proc %s",
-                            __func__,
-                            usnic_compat_proc_name_print(&proc->proc_opal->proc_name));
+        opal_output_verbose(5, USNIC_OUT,
+                            "btl:usnic:%s: unable to find any valid interface pairs for proc %s",
+                            __func__, usnic_compat_proc_name_print(&proc->proc_opal->proc_name));
         return OPAL_ERR_NOT_FOUND;
     }
 
@@ -638,17 +585,13 @@ static int match_modex(opal_btl_usnic_module_t *module,
     /* TODO with UDP, do we still want to enforce this restriction or just take
      * the min of the two MTUs?  Another choice is to disqualify this pairing
      * before running the matching algorithm on it. */
-    if (*index_out >= 0 &&
-        proc->proc_modex[*index_out].max_msg_size !=
-        (uint16_t) module->fabric_info->ep_attr->max_msg_size) {
+    if (*index_out >= 0
+        && proc->proc_modex[*index_out].max_msg_size
+               != (uint16_t) module->fabric_info->ep_attr->max_msg_size) {
         char *errhost = opal_get_proc_hostname(proc->proc_opal);
-        opal_show_help("help-mpi-btl-usnic.txt", "MTU mismatch",
-                       true,
-                       opal_process_info.nodename,
-                       module->linux_device_name,
-                       module->fabric_info->ep_attr->max_msg_size,
-                       errhost,
-                       proc->proc_modex[*index_out].max_msg_size);
+        opal_show_help("help-mpi-btl-usnic.txt", "MTU mismatch", true, opal_process_info.nodename,
+                       module->linux_device_name, module->fabric_info->ep_attr->max_msg_size,
+                       errhost, proc->proc_modex[*index_out].max_msg_size);
         free(errhost);
         *index_out = -1;
         return OPAL_ERR_UNREACH;
@@ -669,9 +612,8 @@ static int match_modex(opal_btl_usnic_module_t *module,
  * Initiate the process to create a USD dest.
  * It will be polled for completion later.
  */
-static int start_av_insert(opal_btl_usnic_module_t *module,
-                                  opal_btl_usnic_endpoint_t *endpoint,
-                                  int channel)
+static int start_av_insert(opal_btl_usnic_module_t *module, opal_btl_usnic_endpoint_t *endpoint,
+                           int channel)
 {
     int ret;
     opal_btl_usnic_modex_t *modex = &endpoint->endpoint_remote_modex;
@@ -683,11 +625,9 @@ static int start_av_insert(opal_btl_usnic_module_t *module,
     context->channel_id = channel;
 
     char str[IPV4STRADDRLEN];
-    opal_btl_usnic_snprintf_ipv4_addr(str, sizeof(str), modex->ipv4_addr,
-                                      modex->netmask);
-    opal_output_verbose(5, USNIC_OUT,
-                        "btl:usnic:start_av_insert: to channel %d at %s:%d",
-                        channel, str, modex->ports[channel]);
+    opal_btl_usnic_snprintf_ipv4_addr(str, sizeof(str), modex->ipv4_addr, modex->netmask);
+    opal_output_verbose(5, USNIC_OUT, "btl:usnic:start_av_insert: to channel %d at %s:%d", channel,
+                        str, modex->ports[channel]);
 
     /* build remote address */
     memset(&sin, 0, sizeof(sin));
@@ -695,17 +635,12 @@ static int start_av_insert(opal_btl_usnic_module_t *module,
     sin.sin_port = htons(modex->ports[channel]);
     sin.sin_addr.s_addr = modex->ipv4_addr;
 
-    ret = fi_av_insert(module->av, &sin, 1,
-            &endpoint->endpoint_remote_addrs[channel], 0, context);
+    ret = fi_av_insert(module->av, &sin, 1, &endpoint->endpoint_remote_addrs[channel], 0, context);
     /* Did an error occur? */
     if (0 != ret) {
-        opal_show_help("help-mpi-btl-usnic.txt", "libfabric API failed",
-                       true,
-                       opal_process_info.nodename,
-                       module->linux_device_name,
-                       "fi_av_insert()", __FILE__, __LINE__,
-                       ret,
-                       "Failed to initiate AV insert");
+        opal_show_help("help-mpi-btl-usnic.txt", "libfabric API failed", true,
+                       opal_process_info.nodename, module->linux_device_name, "fi_av_insert()",
+                       __FILE__, __LINE__, ret, "Failed to initiate AV insert");
         free(context);
         return OPAL_ERROR;
     }
@@ -716,10 +651,8 @@ static int start_av_insert(opal_btl_usnic_module_t *module,
 /*
  * Create an endpoint and claim the matched modex slot
  */
-int
-opal_btl_usnic_create_endpoint(opal_btl_usnic_module_t *module,
-                opal_btl_usnic_proc_t *proc,
-                opal_btl_usnic_endpoint_t **endpoint_o)
+int opal_btl_usnic_create_endpoint(opal_btl_usnic_module_t *module, opal_btl_usnic_proc_t *proc,
+                                   opal_btl_usnic_endpoint_t **endpoint_o)
 {
     int rc;
     int modex_index;
@@ -739,15 +672,15 @@ opal_btl_usnic_create_endpoint(opal_btl_usnic_module_t *module,
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
-    /* Initalize the endpoint */
+    /* Initialize the endpoint */
     endpoint->endpoint_module = module;
-    assert(modex_index >= 0 && modex_index < (int)proc->proc_modex_count);
+    assert(modex_index >= 0 && modex_index < (int) proc->proc_modex_count);
     endpoint->endpoint_remote_modex = proc->proc_modex[modex_index];
     endpoint->endpoint_send_credits = module->sd_num;
 
     /* Start creating destinations; one for each channel.  These
        progress in the background.a */
-    for (int i = 0; i < USNIC_NUM_CHANNELS; ++i)  {
+    for (int i = 0; i < USNIC_NUM_CHANNELS; ++i) {
         rc = start_av_insert(module, endpoint, i);
         if (OPAL_SUCCESS != rc) {
             OBJ_RELEASE(endpoint);
@@ -758,17 +691,15 @@ opal_btl_usnic_create_endpoint(opal_btl_usnic_module_t *module,
     /* Initialize endpoint sequence number info */
     endpoint->endpoint_next_seq_to_send = module->local_modex.isn;
     endpoint->endpoint_ack_seq_rcvd = endpoint->endpoint_next_seq_to_send - 1;
-    endpoint->endpoint_next_contig_seq_to_recv =
-        endpoint->endpoint_remote_modex.isn;
-    endpoint->endpoint_highest_seq_rcvd =
-        endpoint->endpoint_next_contig_seq_to_recv - 1;
+    endpoint->endpoint_next_contig_seq_to_recv = endpoint->endpoint_remote_modex.isn;
+    endpoint->endpoint_highest_seq_rcvd = endpoint->endpoint_next_contig_seq_to_recv - 1;
     endpoint->endpoint_rfstart = WINDOW_SIZE_MOD(endpoint->endpoint_next_contig_seq_to_recv);
 
     /* Now claim that modex slot */
     proc->proc_modex_claimed[modex_index] = true;
-    MSGDEBUG1_OUT("create_endpoint: module=%p claimed endpoint=%p on proc=%p (hash=0x%" PRIx64 ")\n",
-                  (void *)module, (void *)endpoint, (void *)proc,
-                  proc->proc_opal->proc_name);
+    MSGDEBUG1_OUT("create_endpoint: module=%p claimed endpoint=%p on proc=%p (hash=0x%" PRIx64
+                  ")\n",
+                  (void *) module, (void *) endpoint, (void *) proc, proc->proc_opal->proc_name);
 
     /* Save the endpoint on this proc's array of endpoints */
     proc->proc_endpoints[proc->proc_endpoint_count] = endpoint;
@@ -781,8 +712,7 @@ opal_btl_usnic_create_endpoint(opal_btl_usnic_module_t *module,
        not in the endpoint constructor because we aren't able to pass
        the module as a constructor argument -- doh!). */
     opal_mutex_lock(&module->all_endpoints_lock);
-    opal_list_append(&(module->all_endpoints),
-            &(endpoint->endpoint_endpoint_li));
+    opal_list_append(&(module->all_endpoints), &(endpoint->endpoint_endpoint_li));
     endpoint->endpoint_on_all_endpoints = true;
     opal_mutex_unlock(&module->all_endpoints_lock);
 
@@ -807,8 +737,7 @@ opal_btl_usnic_create_endpoint(opal_btl_usnic_module_t *module,
  * opal_btl_usnic_endpoint_t instances, and published addresses/modex
  * info.
  */
-int opal_btl_usnic_proc_match(opal_proc_t *opal_proc,
-                              opal_btl_usnic_module_t *module,
+int opal_btl_usnic_proc_match(opal_proc_t *opal_proc, opal_btl_usnic_module_t *module,
                               opal_btl_usnic_proc_t **proc)
 {
     /* Check if we have already created a proc structure for this peer
diff --git a/opal/mca/btl/usnic/btl_usnic_proc.h b/opal/mca/btl/usnic/btl_usnic_proc.h
index 2365c8b5d6a..7c8f2c4da06 100644
--- a/opal/mca/btl/usnic/btl_usnic_proc.h
+++ b/opal/mca/btl/usnic/btl_usnic_proc.h
@@ -45,7 +45,7 @@ typedef struct opal_btl_usnic_proc_t {
     opal_proc_t *proc_opal;
 
     /** Addresses received via modex for this remote proc */
-    opal_btl_usnic_modex_t* proc_modex;
+    opal_btl_usnic_modex_t *proc_modex;
     /** Number of entries in the proc_modex array */
     size_t proc_modex_count;
     /** Whether the modex entry is "claimed" by a module or not */
@@ -78,8 +78,7 @@ typedef struct opal_btl_usnic_proc_t {
 
 OBJ_CLASS_DECLARATION(opal_btl_usnic_proc_t);
 
-
-opal_btl_usnic_proc_t *opal_btl_usnic_proc_lookup_ompi(opal_proc_t* opal_proc);
+opal_btl_usnic_proc_t *opal_btl_usnic_proc_lookup_ompi(opal_proc_t *opal_proc);
 
 struct opal_btl_usnic_module_t;
 
@@ -87,13 +86,11 @@ opal_btl_usnic_endpoint_t *
 opal_btl_usnic_proc_lookup_endpoint(struct opal_btl_usnic_module_t *receiver,
                                     uint64_t sender_hashed_rte_name);
 
-int opal_btl_usnic_proc_match(opal_proc_t* opal_proc,
-                              struct opal_btl_usnic_module_t *module,
+int opal_btl_usnic_proc_match(opal_proc_t *opal_proc, struct opal_btl_usnic_module_t *module,
                               opal_btl_usnic_proc_t **proc);
-int
-opal_btl_usnic_create_endpoint(struct opal_btl_usnic_module_t *module,
-                opal_btl_usnic_proc_t *proc,
-                opal_btl_usnic_endpoint_t **endpoint_o);
+int opal_btl_usnic_create_endpoint(struct opal_btl_usnic_module_t *module,
+                                   opal_btl_usnic_proc_t *proc,
+                                   opal_btl_usnic_endpoint_t **endpoint_o);
 
 END_C_DECLS
 
diff --git a/opal/mca/btl/usnic/btl_usnic_recv.c b/opal/mca/btl/usnic/btl_usnic_recv.c
index 535ced87363..598dbd081db 100644
--- a/opal/mca/btl/usnic/btl_usnic_recv.c
+++ b/opal/mca/btl/usnic/btl_usnic_recv.c
@@ -26,33 +26,31 @@
 
 #include <unistd.h>
 
-#include "opal_stdint.h"
-#include "opal/mca/memchecker/base/base.h"
 #include "opal/constants.h"
+#include "opal/mca/memchecker/base/base.h"
+#include "opal_stdint.h"
 
-#include "opal/mca/btl/btl.h"
 #include "opal/mca/btl/base/base.h"
+#include "opal/mca/btl/btl.h"
 
 #include "btl_usnic.h"
-#include "btl_usnic_frag.h"
+#include "btl_usnic_ack.h"
 #include "btl_usnic_endpoint.h"
+#include "btl_usnic_frag.h"
 #include "btl_usnic_module.h"
 #include "btl_usnic_proc.h"
-#include "btl_usnic_ack.h"
 #include "btl_usnic_recv.h"
 #include "btl_usnic_util.h"
 
-
 /*
  * We have received a segment, take action based on the
  * packet type in the BTL header
  */
-void opal_btl_usnic_recv_call(opal_btl_usnic_module_t *module,
-                              opal_btl_usnic_recv_segment_t *seg,
+void opal_btl_usnic_recv_call(opal_btl_usnic_module_t *module, opal_btl_usnic_recv_segment_t *seg,
                               opal_btl_usnic_channel_t *channel)
 {
     opal_btl_usnic_segment_t *bseg;
-    mca_btl_active_message_callback_t* reg;
+    mca_btl_active_message_callback_t *reg;
     opal_btl_usnic_endpoint_t *endpoint;
     opal_btl_usnic_btl_chunk_header_t *chunk_hdr;
     opal_btl_usnic_btl_header_t *hdr;
@@ -68,16 +66,14 @@ void opal_btl_usnic_recv_call(opal_btl_usnic_module_t *module,
     ++module->stats.num_total_recvs;
 
     /* Valgrind help */
-    opal_memchecker_base_mem_defined((void*)(seg->rs_protocol_header),
-                                     seg->rs_len);
+    opal_memchecker_base_mem_defined((void *) (seg->rs_protocol_header), seg->rs_len);
 
     /* Find out who sent this segment */
     endpoint = seg->rs_endpoint;
     if (FAKE_RECV_DROP || OPAL_UNLIKELY(NULL == endpoint)) {
         /* No idea who this was from, so drop it */
 #if MSGDEBUG1
-        opal_output(0, "=== Unknown sender; dropped: seq %" UDSEQ,
-                    bseg->us_btl_header->pkt_seq);
+        opal_output(0, "=== Unknown sender; dropped: seq %" UDSEQ, bseg->us_btl_header->pkt_seq);
 #endif
         ++module->stats.num_unk_recvs;
         goto repost_no_endpoint;
@@ -87,12 +83,9 @@ void opal_btl_usnic_recv_call(opal_btl_usnic_module_t *module,
     struct opal_btl_usnic_modex_t *modex;
 
     modex = &module->local_modex;
-    opal_btl_usnic_snprintf_ipv4_addr(local_ip, sizeof(local_ip),
-                                      modex->ipv4_addr,
-                                      modex->netmask);
+    opal_btl_usnic_snprintf_ipv4_addr(local_ip, sizeof(local_ip), modex->ipv4_addr, modex->netmask);
     modex = &endpoint->endpoint_remote_modex;
-    opal_btl_usnic_snprintf_ipv4_addr(remote_ip, sizeof(remote_ip),
-                                      modex->ipv4_addr,
+    opal_btl_usnic_snprintf_ipv4_addr(remote_ip, sizeof(remote_ip), modex->ipv4_addr,
                                       modex->netmask);
 #endif
 
@@ -109,9 +102,9 @@ void opal_btl_usnic_recv_call(opal_btl_usnic_module_t *module,
         hdr = seg->rs_base.us_btl_header;
 
 #if MSGDEBUG1
-        opal_output(0, "<-- Received FRAG ep %p, seq %" UDSEQ ", len=%d\n",
-                    (void*) endpoint, hdr->pkt_seq, hdr->payload_len);
-#if 0
+        opal_output(0, "<-- Received FRAG ep %p, seq %" UDSEQ ", len=%d\n", (void *) endpoint,
+                    hdr->pkt_seq, hdr->payload_len);
+#    if 0
 
         opal_output(0, "<-- Received FRAG ep %p, seq %" UDSEQ " from %s to %s: GOOD! (rel seq %d, lowest seq %" UDSEQ ", highest seq: %" UDSEQ ", rwstart %d) seg %p, module %p\n",
                     (void*) endpoint,
@@ -126,7 +119,7 @@ void opal_btl_usnic_recv_call(opal_btl_usnic_module_t *module,
             opal_output(0, "  put_addr = %p\n",
                     seg->rs_base.us_btl_header->put_addr);
         }
-#endif
+#    endif
 #endif
 
         /* If this it not a PUT, Pass this segment up to the PML.
@@ -139,27 +132,25 @@ void opal_btl_usnic_recv_call(opal_btl_usnic_module_t *module,
             reg = mca_btl_base_active_message_trigger + hdr->tag;
             seg->rs_segment.seg_len = hdr->payload_len;
 #if MSGDEBUG2
-                opal_output(0, "small recv complete, pass up %u bytes, tag=%d\n",
-                        (unsigned)bseg->us_btl_header->payload_len,
-                        (int)bseg->us_btl_header->tag);
+            opal_output(0, "small recv complete, pass up %u bytes, tag=%d\n",
+                        (unsigned) bseg->us_btl_header->payload_len,
+                        (int) bseg->us_btl_header->tag);
 #endif
             seg->rs_desc.endpoint = endpoint;
             seg->rs_desc.tag = hdr->tag;
             seg->rs_desc.cbdata = reg->cbdata;
             reg->cbfunc(&module->super, &seg->rs_desc);
 
-        /*
-         * If this is a PUT, need to copy it to user buffer
-         */
+            /*
+             * If this is a PUT, need to copy it to user buffer
+             */
         } else {
 #if MSGDEBUG1
-            opal_output(0, "Copy %d PUT bytes to %p\n",
-                seg->rs_base.us_btl_header->payload_len,
-                (void*)seg->rs_base.us_btl_header->put_addr);
+            opal_output(0, "Copy %d PUT bytes to %p\n", seg->rs_base.us_btl_header->payload_len,
+                        (void *) seg->rs_base.us_btl_header->put_addr);
 #endif
-            memcpy(seg->rs_base.us_btl_header->put_addr,
-                    seg->rs_base.us_payload.raw,
-                    seg->rs_base.us_btl_header->payload_len);
+            memcpy(seg->rs_base.us_btl_header->put_addr, seg->rs_base.us_payload.raw,
+                   seg->rs_base.us_btl_header->payload_len);
         }
 
         /* do not jump to repost, already done by bookkeeping */
@@ -173,22 +164,20 @@ void opal_btl_usnic_recv_call(opal_btl_usnic_module_t *module,
         opal_btl_usnic_rx_frag_info_t *fip;
 
         /* Is incoming sequence # ok? */
-        if (OPAL_UNLIKELY(opal_btl_usnic_check_rx_seq(endpoint, seg,
-                        &window_index) != 0)) {
+        if (OPAL_UNLIKELY(opal_btl_usnic_check_rx_seq(endpoint, seg, &window_index) != 0)) {
             goto repost;
         }
 
 #if MSGDEBUG1
-        opal_output(0, "<-- Received CHUNK fid %d ep %p, seq %" UDSEQ " from %s to %s: GOOD! (rel seq %d, lowest seq %" UDSEQ ", highest seq: %" UDSEQ ", rwstart %d) seg %p, module %p\n",
-                    seg->rs_base.us_btl_chunk_header->ch_frag_id,
-                    (void*) endpoint,
-                    seg->rs_base.us_btl_chunk_header->ch_hdr.pkt_seq,
-                    remote_ip, local_ip,
-                    window_index,
-                    endpoint->endpoint_next_contig_seq_to_recv,
-                    endpoint->endpoint_highest_seq_rcvd,
-                    endpoint->endpoint_rfstart,
-                    (void*) seg, (void*) module);
+        opal_output(0,
+                    "<-- Received CHUNK fid %d ep %p, seq %" UDSEQ
+                    " from %s to %s: GOOD! (rel seq %d, lowest seq %" UDSEQ ", highest seq: %" UDSEQ
+                    ", rwstart %d) seg %p, module %p\n",
+                    seg->rs_base.us_btl_chunk_header->ch_frag_id, (void *) endpoint,
+                    seg->rs_base.us_btl_chunk_header->ch_hdr.pkt_seq, remote_ip, local_ip,
+                    window_index, endpoint->endpoint_next_contig_seq_to_recv,
+                    endpoint->endpoint_highest_seq_rcvd, endpoint->endpoint_rfstart, (void *) seg,
+                    (void *) module);
 #endif
 
         /* start a new fragment if not one in progress
@@ -211,13 +200,12 @@ void opal_btl_usnic_recv_call(opal_btl_usnic_module_t *module,
                 /* See which data pool this should come from,
                  * or if it should be malloc()ed
                  */
-                pool = usnic_fls(chunk_hdr->ch_frag_size-1);
-                if (pool >= module->first_pool &&
-                        pool <= module->last_pool) {
-                    opal_free_list_item_t* item;
+                pool = usnic_fls(chunk_hdr->ch_frag_size - 1);
+                if (pool >= module->first_pool && pool <= module->last_pool) {
+                    opal_free_list_item_t *item;
                     opal_btl_usnic_rx_buf_t *rx_buf;
                     USNIC_COMPAT_FREE_LIST_GET(&module->module_recv_buffers[pool], item);
-                    rx_buf = (opal_btl_usnic_rx_buf_t *)item;
+                    rx_buf = (opal_btl_usnic_rx_buf_t *) item;
                     if (OPAL_LIKELY(NULL != rx_buf)) {
                         fip->rfi_fl_elt = item;
                         fip->rfi_data = rx_buf->buf;
@@ -233,42 +221,38 @@ void opal_btl_usnic_recv_call(opal_btl_usnic_module_t *module,
                     opal_btl_usnic_util_abort("malloc failed", __FILE__, __LINE__);
                 }
 #if MSGDEBUG1
-                opal_output(0, "Start large recv to %p, size=%"PRIu32"\n",
-                    (void *)fip->rfi_data, chunk_hdr->ch_frag_size);
+                opal_output(0, "Start large recv to %p, size=%" PRIu32 "\n", (void *) fip->rfi_data,
+                            chunk_hdr->ch_frag_size);
 #endif
             } else {
 #if MSGDEBUG1
-                opal_output(0, "Start PUT to %p\n",
-                        (void *)chunk_hdr->ch_hdr.put_addr);
+                opal_output(0, "Start PUT to %p\n", (void *) chunk_hdr->ch_hdr.put_addr);
 #endif
                 fip->rfi_data = chunk_hdr->ch_hdr.put_addr;
             }
             fip->rfi_bytes_left = chunk_hdr->ch_frag_size;
             fip->rfi_frag_id = chunk_hdr->ch_frag_id;
 
-        /* frag_id is not 0 - it must match, drop if not */
+            /* frag_id is not 0 - it must match, drop if not */
         } else if (fip->rfi_frag_id != chunk_hdr->ch_frag_id) {
             ++module->stats.num_badfrag_recvs;
             goto repost;
         }
 #if MSGDEBUG1
-        opal_output(0, "put_addr=%p, copy_addr=%p, off=%d\n",
-                chunk_hdr->ch_hdr.put_addr,
-                fip->rfi_data+chunk_hdr->ch_frag_offset,
-                chunk_hdr->ch_frag_offset);
+        opal_output(0, "put_addr=%p, copy_addr=%p, off=%d\n", chunk_hdr->ch_hdr.put_addr,
+                    fip->rfi_data + chunk_hdr->ch_frag_offset, chunk_hdr->ch_frag_offset);
 #endif
 
         /* Stats */
         ++module->stats.num_chunk_recvs;
 
         /* validate offset and len to be within fragment */
-        assert(chunk_hdr->ch_frag_offset + chunk_hdr->ch_hdr.payload_len <=
-                fip->rfi_frag_size);
+        assert(chunk_hdr->ch_frag_offset + chunk_hdr->ch_hdr.payload_len <= fip->rfi_frag_size);
         assert(fip->rfi_frag_size == chunk_hdr->ch_frag_size);
 
         /* copy the data into place */
-        memcpy(fip->rfi_data + chunk_hdr->ch_frag_offset, (char *)(chunk_hdr+1),
-                chunk_hdr->ch_hdr.payload_len);
+        memcpy(fip->rfi_data + chunk_hdr->ch_frag_offset, (char *) (chunk_hdr + 1),
+               chunk_hdr->ch_hdr.payload_len);
 
         /* update sliding window */
         opal_btl_usnic_update_window(endpoint, window_index);
@@ -289,12 +273,10 @@ void opal_btl_usnic_recv_call(opal_btl_usnic_module_t *module,
                 /* Pass this segment up to the PML */
 #if MSGDEBUG2
                 opal_output(0, "large recv complete, pass up %p, %u bytes, tag=%d\n",
-                        desc.USNIC_RECV_LOCAL->seg_addr.pval,
-                        (unsigned)desc.USNIC_RECV_LOCAL->seg_len,
-                        (int)chunk_hdr->ch_hdr.tag);
+                            desc.USNIC_RECV_LOCAL->seg_addr.pval,
+                            (unsigned) desc.USNIC_RECV_LOCAL->seg_len, (int) chunk_hdr->ch_hdr.tag);
 #endif
-                reg = mca_btl_base_active_message_trigger +
-                    chunk_hdr->ch_hdr.tag;
+                reg = mca_btl_base_active_message_trigger + chunk_hdr->ch_hdr.tag;
 
                 /* mca_pml_ob1_recv_frag_callback_frag() */
                 desc.endpoint = endpoint;
@@ -327,8 +309,7 @@ void opal_btl_usnic_recv_call(opal_btl_usnic_module_t *module,
 
     /***********************************************************************/
     /* Frag is an incoming ACK */
-    else if (OPAL_LIKELY(OPAL_BTL_USNIC_PAYLOAD_TYPE_ACK ==
-                         bseg->us_btl_header->payload_type)) {
+    else if (OPAL_LIKELY(OPAL_BTL_USNIC_PAYLOAD_TYPE_ACK == bseg->us_btl_header->payload_type)) {
         opal_btl_usnic_seq_t ack_seq;
 
         /* sequence being ACKed */
@@ -352,21 +333,24 @@ void opal_btl_usnic_recv_call(opal_btl_usnic_module_t *module,
     else {
         ++module->stats.num_unk_recvs;
         if (module->stats.num_unk_recvs < 10) {
-            opal_output_verbose(15, USNIC_OUT, "unrecognized payload type %d", bseg->us_btl_header->payload_type);
-            opal_output_verbose(15, USNIC_OUT, "base = %p, proto = %p, hdr = %p", bseg->us_list.ptr, seg->rs_protocol_header, (void*) bseg->us_btl_header);
-            opal_btl_usnic_dump_hex(15, USNIC_OUT, bseg->us_list.ptr, 96+sizeof(*bseg->us_btl_header));
+            opal_output_verbose(15, USNIC_OUT, "unrecognized payload type %d",
+                                bseg->us_btl_header->payload_type);
+            opal_output_verbose(15, USNIC_OUT, "base = %p, proto = %p, hdr = %p", bseg->us_list.ptr,
+                                seg->rs_protocol_header, (void *) bseg->us_btl_header);
+            opal_btl_usnic_dump_hex(15, USNIC_OUT, bseg->us_list.ptr,
+                                    96 + sizeof(*bseg->us_btl_header));
         }
         goto repost;
     }
 
     /***********************************************************************/
- repost:
+repost:
 
     /* if endpoint exiting, and all ACKs received, release the endpoint */
     if (endpoint->endpoint_exiting && ENDPOINT_DRAINED(endpoint)) {
         OBJ_RELEASE(endpoint);
     }
- repost_no_endpoint:
+repost_no_endpoint:
     ++module->stats.num_recv_reposts;
 
     /* Add recv to linked list for reposting */
diff --git a/opal/mca/btl/usnic/btl_usnic_recv.h b/opal/mca/btl/usnic/btl_usnic_recv.h
index 0a663fe60cf..20ac70af718 100644
--- a/opal/mca/btl/usnic/btl_usnic_recv.h
+++ b/opal/mca/btl/usnic/btl_usnic_recv.h
@@ -12,17 +12,14 @@
 #define BTL_USNIC_RECV_H
 
 #include "btl_usnic.h"
-#include "btl_usnic_util.h"
 #include "btl_usnic_frag.h"
 #include "btl_usnic_proc.h"
+#include "btl_usnic_util.h"
 
-
-void opal_btl_usnic_recv_call(opal_btl_usnic_module_t *module,
-                              opal_btl_usnic_recv_segment_t *rseg,
+void opal_btl_usnic_recv_call(opal_btl_usnic_module_t *module, opal_btl_usnic_recv_segment_t *rseg,
                               opal_btl_usnic_channel_t *channel);
 
-static inline int
-opal_btl_usnic_post_recv_list(opal_btl_usnic_channel_t *channel)
+static inline int opal_btl_usnic_post_recv_list(opal_btl_usnic_channel_t *channel)
 {
     struct iovec iov;
     struct fi_msg msg;
@@ -57,35 +54,32 @@ opal_btl_usnic_post_recv_list(opal_btl_usnic_channel_t *channel)
 /*
  * Given an incoming segment, lookup the endpoint that sent it
  */
-static inline opal_btl_usnic_endpoint_t *
-lookup_sender(opal_btl_usnic_module_t *module, opal_btl_usnic_segment_t *seg)
+static inline opal_btl_usnic_endpoint_t *lookup_sender(opal_btl_usnic_module_t *module,
+                                                       opal_btl_usnic_segment_t *seg)
 {
     int ret;
     opal_btl_usnic_endpoint_t *sender;
 
-    /* Use the hashed ORTE process name in the BTL header to uniquely
+    /* Use the hashed PRTE process name in the BTL header to uniquely
        identify the sending process (using the MAC/hardware address
-       only identifies the sending server -- not the sending ORTE
+       only identifies the sending server -- not the sending PRTE
        process). */
     /* JMS We've experimented with using a handshake before sending
        any data so that instead of looking up a hash on the
        btl_header->sender, echo back the ptr to the sender's
        ompi_proc.  There was limited speedup with this scheme; more
        investigation is required. */
-    ret = opal_hash_table_get_value_uint64(&module->senders,
-                                           seg->us_btl_header->sender,
-                                           (void**) &sender);
+    ret = opal_hash_table_get_value_uint64(&module->senders, seg->us_btl_header->sender,
+                                           (void **) &sender);
     if (OPAL_LIKELY(OPAL_SUCCESS == ret)) {
         return sender;
     }
 
     /* The sender wasn't in the hash table, so do a slow lookup and
        put the result in the hash table */
-    sender = opal_btl_usnic_proc_lookup_endpoint(module,
-                                                 seg->us_btl_header->sender);
+    sender = opal_btl_usnic_proc_lookup_endpoint(module, seg->us_btl_header->sender);
     if (NULL != sender) {
-        opal_hash_table_set_value_uint64(&module->senders,
-                                         seg->us_btl_header->sender, sender);
+        opal_hash_table_set_value_uint64(&module->senders, seg->us_btl_header->sender, sender);
         return sender;
     }
 
@@ -98,27 +92,25 @@ lookup_sender(opal_btl_usnic_module_t *module, opal_btl_usnic_segment_t *seg)
  * to indicate that it and possible following contiguous sequence
  * numbers have been received.
  */
-static inline void
-opal_btl_usnic_update_window(
-    opal_btl_usnic_endpoint_t *endpoint,
-    uint32_t window_index)
+static inline void opal_btl_usnic_update_window(opal_btl_usnic_endpoint_t *endpoint,
+                                                uint32_t window_index)
 {
     uint32_t i;
 
     /* Enable ACK reply if not enabled */
 #if MSGDEBUG1
-    opal_output(0, "ep: %p, ack_needed = %s\n", (void*)endpoint, endpoint->endpoint_ack_needed?"true":"false");
+    opal_output(0, "ep: %p, ack_needed = %s\n", (void *) endpoint,
+                endpoint->endpoint_ack_needed ? "true" : "false");
 #endif
     if (!endpoint->endpoint_ack_needed) {
         opal_btl_usnic_add_to_endpoints_needing_ack(endpoint);
     }
 
-    /* A hueristic: set to send this ACK after we have checked our
+    /* A heuristic: set to send this ACK after we have checked our
        incoming DATA_CHANNEL component.act_iteration_delay times
        (i.e., so we can piggyback an ACK on an outgoing send) */
     if (0 == endpoint->endpoint_acktime) {
-        endpoint->endpoint_acktime =
-            get_ticks() + mca_btl_usnic_component.ack_iteration_delay;
+        endpoint->endpoint_acktime = get_ticks() + mca_btl_usnic_component.ack_iteration_delay;
     }
 
     /* Save this incoming segment in the received segmentss array on the
@@ -144,11 +136,9 @@ opal_btl_usnic_update_window(
     endpoint->endpoint_rfstart = i;
 }
 
-static inline int
-opal_btl_usnic_check_rx_seq(
-    opal_btl_usnic_endpoint_t *endpoint,
-    opal_btl_usnic_recv_segment_t *seg,
-    uint32_t *window_index)
+static inline int opal_btl_usnic_check_rx_seq(opal_btl_usnic_endpoint_t *endpoint,
+                                              opal_btl_usnic_recv_segment_t *seg,
+                                              uint32_t *window_index)
 {
     uint32_t i;
     opal_btl_usnic_seq_t seq;
@@ -159,11 +149,11 @@ opal_btl_usnic_check_rx_seq(
      */
     if (seg->rs_base.us_btl_header->ack_present) {
 #if MSGDEBUG1
-        opal_output(0, "Handle piggy-packed ACK seq %"UDSEQ"\n", seg->rs_base.us_btl_header->ack_seq);
+        opal_output(0, "Handle piggy-packed ACK seq %" UDSEQ "\n",
+                    seg->rs_base.us_btl_header->ack_seq);
 #endif
         OPAL_THREAD_LOCK(&btl_usnic_lock);
-        opal_btl_usnic_handle_ack(endpoint,
-                seg->rs_base.us_btl_header->ack_seq);
+        opal_btl_usnic_handle_ack(endpoint, seg->rs_base.us_btl_header->ack_seq);
         OPAL_THREAD_UNLOCK(&btl_usnic_lock);
     }
 
@@ -194,13 +184,13 @@ opal_btl_usnic_check_rx_seq(
     delta = SEQ_DIFF(seq, endpoint->endpoint_next_contig_seq_to_recv);
     if (delta < 0 || delta >= WINDOW_SIZE) {
 #if MSGDEBUG1
-            opal_output(0, "<-- Received FRAG/CHUNK ep %p, seq %" UDSEQ " outside of window (%" UDSEQ " - %" UDSEQ "), %p, module %p -- DROPPED\n",
-                        (void*)endpoint, seg->rs_base.us_btl_header->pkt_seq,
-                        endpoint->endpoint_next_contig_seq_to_recv,
-                        (endpoint->endpoint_next_contig_seq_to_recv +
-                         WINDOW_SIZE - 1),
-                        (void*) seg,
-                        (void*) endpoint->endpoint_module);
+        opal_output(0,
+                    "<-- Received FRAG/CHUNK ep %p, seq %" UDSEQ " outside of window (%" UDSEQ
+                    " - %" UDSEQ "), %p, module %p -- DROPPED\n",
+                    (void *) endpoint, seg->rs_base.us_btl_header->pkt_seq,
+                    endpoint->endpoint_next_contig_seq_to_recv,
+                    (endpoint->endpoint_next_contig_seq_to_recv + WINDOW_SIZE - 1), (void *) seg,
+                    (void *) endpoint->endpoint_module);
 #endif
 
         /* Stats */
@@ -218,7 +208,7 @@ opal_btl_usnic_check_rx_seq(
        ACKed).
 
        We have saved all un-ACKed segment in an array on the
-       endpoint that is the same legnth as the receiver's window
+       endpoint that is the same length as the receiver's window
        (i.e., WINDOW_SIZE).  We can use the incoming segment sequence
        number to find its position in the array.  It's a little
        tricky because the left edge of the receiver window keeps
@@ -239,15 +229,16 @@ opal_btl_usnic_check_rx_seq(
     i = WINDOW_SIZE_MOD(i + endpoint->endpoint_rfstart);
     if (endpoint->endpoint_rcvd_segs[i]) {
 #if MSGDEBUG1
-        opal_output(0, "<-- Received FRAG/CHUNK ep %p, seq %" UDSEQ ", seg %p: duplicate -- DROPPED\n",
-            (void*) endpoint, seg->rs_base.us_btl_header->pkt_seq, (void*) seg);
+        opal_output(0,
+                    "<-- Received FRAG/CHUNK ep %p, seq %" UDSEQ ", seg %p: duplicate -- DROPPED\n",
+                    (void *) endpoint, seg->rs_base.us_btl_header->pkt_seq, (void *) seg);
 #endif
         /* highest_seq_rcvd is for debug stats only; it's not used
            in any window calculations */
         assert(SEQ_LE(seq, endpoint->endpoint_highest_seq_rcvd));
         /* next_contig_seq_to_recv-1 is the ack number we'll
            send */
-        assert (SEQ_GT(seq, endpoint->endpoint_next_contig_seq_to_recv - 1));
+        assert(SEQ_GT(seq, endpoint->endpoint_next_contig_seq_to_recv - 1));
 
         /* Stats */
         ++endpoint->endpoint_module->stats.num_dup_recvs;
@@ -276,13 +267,12 @@ opal_btl_usnic_check_rx_seq(
  * possible.
  * See README.txt for a discussion of receive fastpath
  */
-static inline void
-opal_btl_usnic_recv_fast(opal_btl_usnic_module_t *module,
-                         opal_btl_usnic_recv_segment_t *seg,
-                         opal_btl_usnic_channel_t *channel)
+static inline void opal_btl_usnic_recv_fast(opal_btl_usnic_module_t *module,
+                                            opal_btl_usnic_recv_segment_t *seg,
+                                            opal_btl_usnic_channel_t *channel)
 {
     opal_btl_usnic_segment_t *bseg;
-    mca_btl_active_message_callback_t* reg;
+    mca_btl_active_message_callback_t *reg;
     opal_btl_usnic_seq_t seq;
     opal_btl_usnic_endpoint_t *endpoint;
     int delta;
@@ -305,10 +295,9 @@ opal_btl_usnic_dump_hex(15, USNIC_OUT, bseg->us_btl_header, bseg->us_btl_header-
        wholly contained in this one message -- it is not chunked
        across multiple messages), and it's not a PUT from the sender,
        then just handle it here. */
-    if (endpoint != NULL && !endpoint->endpoint_exiting &&
-            (OPAL_BTL_USNIC_PAYLOAD_TYPE_FRAG ==
-                bseg->us_btl_header->payload_type) &&
-            seg->rs_base.us_btl_header->put_addr == NULL) {
+    if (endpoint != NULL && !endpoint->endpoint_exiting
+        && (OPAL_BTL_USNIC_PAYLOAD_TYPE_FRAG == bseg->us_btl_header->payload_type)
+        && seg->rs_base.us_btl_header->put_addr == NULL) {
 
         seq = seg->rs_base.us_btl_header->pkt_seq;
         delta = SEQ_DIFF(seq, endpoint->endpoint_next_contig_seq_to_recv);
@@ -335,7 +324,7 @@ opal_btl_usnic_dump_hex(15, USNIC_OUT, bseg->us_btl_header, bseg->us_btl_header-
         seg->rs_desc.cbdata = reg->cbdata;
         reg->cbfunc(&module->super, &seg->rs_desc);
 
-drop:
+    drop:
         channel->chan_deferred_recv = seg;
     }
 
@@ -347,21 +336,18 @@ opal_btl_usnic_dump_hex(15, USNIC_OUT, bseg->us_btl_header, bseg->us_btl_header-
 
 /*
  */
-static inline int
-opal_btl_usnic_recv_frag_bookkeeping(
-    opal_btl_usnic_module_t* module,
-    opal_btl_usnic_recv_segment_t *seg,
-    opal_btl_usnic_channel_t *channel)
+static inline int opal_btl_usnic_recv_frag_bookkeeping(opal_btl_usnic_module_t *module,
+                                                       opal_btl_usnic_recv_segment_t *seg,
+                                                       opal_btl_usnic_channel_t *channel)
 {
-    opal_btl_usnic_endpoint_t* endpoint;
+    opal_btl_usnic_endpoint_t *endpoint;
     uint32_t window_index;
     int rc;
 
     endpoint = seg->rs_endpoint;
 
     /* Valgrind help */
-    opal_memchecker_base_mem_defined(
-                (void*)(seg->rs_protocol_header), seg->rs_len);
+    opal_memchecker_base_mem_defined((void *) (seg->rs_protocol_header), seg->rs_len);
 
     ++module->stats.num_total_recvs;
 
@@ -394,13 +380,12 @@ opal_btl_usnic_recv_frag_bookkeeping(
  * We have received a segment, take action based on the
  * packet type in the BTL header
  */
-static inline void
-opal_btl_usnic_recv(opal_btl_usnic_module_t *module,
-                    opal_btl_usnic_recv_segment_t *seg,
-                    opal_btl_usnic_channel_t *channel)
+static inline void opal_btl_usnic_recv(opal_btl_usnic_module_t *module,
+                                       opal_btl_usnic_recv_segment_t *seg,
+                                       opal_btl_usnic_channel_t *channel)
 {
     opal_btl_usnic_segment_t *bseg;
-    mca_btl_active_message_callback_t* reg;
+    mca_btl_active_message_callback_t *reg;
     opal_btl_usnic_endpoint_t *endpoint;
     int rc;
 
@@ -417,13 +402,12 @@ opal_btl_usnic_recv(opal_btl_usnic_module_t *module,
        wholly contained in this one message -- it is not chunked
        across multiple messages), and it's not a PUT from the sender,
        then just handle it here. */
-    if (endpoint != NULL && !endpoint->endpoint_exiting &&
-            (OPAL_BTL_USNIC_PAYLOAD_TYPE_FRAG ==
-                bseg->us_btl_header->payload_type) &&
-            seg->rs_base.us_btl_header->put_addr == NULL) {
+    if (endpoint != NULL && !endpoint->endpoint_exiting
+        && (OPAL_BTL_USNIC_PAYLOAD_TYPE_FRAG == bseg->us_btl_header->payload_type)
+        && seg->rs_base.us_btl_header->put_addr == NULL) {
 
         MSGDEBUG1_OUT("<-- Received FRAG (fastpath) ep %p, seq %" UDSEQ ", len=%" PRIu16 "\n",
-                      (void*) endpoint, bseg->us_btl_header->pkt_seq,
+                      (void *) endpoint, bseg->us_btl_header->pkt_seq,
                       bseg->us_btl_header->payload_len);
 
         /* do the receive bookkeeping */
diff --git a/opal/mca/btl/usnic/btl_usnic_send.c b/opal/mca/btl/usnic/btl_usnic_send.c
index 3c7b715fc70..c2e3ba74428 100644
--- a/opal/mca/btl/usnic/btl_usnic_send.c
+++ b/opal/mca/btl/usnic/btl_usnic_send.c
@@ -27,24 +27,22 @@
 
 #include "opal/constants.h"
 
-#include "opal/mca/btl/btl.h"
 #include "opal/mca/btl/base/base.h"
+#include "opal/mca/btl/btl.h"
 
 #include "btl_usnic.h"
+#include "btl_usnic_ack.h"
 #include "btl_usnic_frag.h"
-#include "btl_usnic_util.h"
 #include "btl_usnic_send.h"
-#include "btl_usnic_ack.h"
-
+#include "btl_usnic_util.h"
 
 /*
  * This function is called when a send of a segment completes that is
  * the one-and-only segment of an MPI message.  Return the WQE and
  * also return the segment if no ACK pending.
  */
-void
-opal_btl_usnic_frag_send_complete(opal_btl_usnic_module_t *module,
-                                    opal_btl_usnic_send_segment_t *sseg)
+void opal_btl_usnic_frag_send_complete(opal_btl_usnic_module_t *module,
+                                       opal_btl_usnic_send_segment_t *sseg)
 {
     opal_btl_usnic_send_frag_t *frag;
 
@@ -77,9 +75,8 @@ opal_btl_usnic_frag_send_complete(opal_btl_usnic_module_t *module,
  * segments that have not yet completed sending).  Return the WQE and
  * also return the segment if no ACK pending.
  */
-void
-opal_btl_usnic_chunk_send_complete(opal_btl_usnic_module_t *module,
-                                    opal_btl_usnic_send_segment_t *sseg)
+void opal_btl_usnic_chunk_send_complete(opal_btl_usnic_module_t *module,
+                                        opal_btl_usnic_send_segment_t *sseg)
 {
     opal_btl_usnic_send_frag_t *frag;
 
@@ -115,12 +112,9 @@ opal_btl_usnic_chunk_send_complete(opal_btl_usnic_module_t *module,
  *
  * The "tag" only applies to sends.
  */
-int
-opal_btl_usnic_finish_put_or_send(
-    opal_btl_usnic_module_t *module,
-    opal_btl_usnic_endpoint_t *endpoint,
-    opal_btl_usnic_send_frag_t *frag,
-    mca_btl_base_tag_t tag)
+int opal_btl_usnic_finish_put_or_send(opal_btl_usnic_module_t *module,
+                                      opal_btl_usnic_endpoint_t *endpoint,
+                                      opal_btl_usnic_send_frag_t *frag, mca_btl_base_tag_t tag)
 {
     int rc;
     opal_btl_usnic_small_send_frag_t *sfrag;
@@ -133,7 +127,7 @@ opal_btl_usnic_finish_put_or_send(
      */
     if (frag->sf_base.uf_type == OPAL_BTL_USNIC_FRAG_SMALL_SEND) {
 
-        sfrag = (opal_btl_usnic_small_send_frag_t *)frag;
+        sfrag = (opal_btl_usnic_small_send_frag_t *) frag;
         sseg = &sfrag->ssf_segment;
 
         /* Copy in user data if there is any, collapsing 2 segments into 1.
@@ -144,15 +138,14 @@ opal_btl_usnic_finish_put_or_send(
             /* no convertor */
             assert(NULL != frag->sf_base.uf_local_seg[1].seg_addr.pval);
 
-            memcpy(((char *)(intptr_t)frag->sf_base.uf_local_seg[0].seg_addr.lval +
-                        frag->sf_base.uf_local_seg[0].seg_len),
-                    frag->sf_base.uf_local_seg[1].seg_addr.pval,
-                    frag->sf_base.uf_local_seg[1].seg_len);
+            memcpy(((char *) (intptr_t) frag->sf_base.uf_local_seg[0].seg_addr.lval
+                    + frag->sf_base.uf_local_seg[0].seg_len),
+                   frag->sf_base.uf_local_seg[1].seg_addr.pval,
+                   frag->sf_base.uf_local_seg[1].seg_len);
 
             /* update 1st segment length */
             frag->sf_base.uf_base.USNIC_SEND_LOCAL_COUNT = 1;
-            frag->sf_base.uf_local_seg[0].seg_len +=
-                frag->sf_base.uf_local_seg[1].seg_len;
+            frag->sf_base.uf_local_seg[0].seg_len += frag->sf_base.uf_local_seg[1].seg_len;
         }
 
         sseg->ss_len = sizeof(opal_btl_usnic_btl_header_t) + frag->sf_size;
@@ -165,7 +158,7 @@ opal_btl_usnic_finish_put_or_send(
 
         /* Save info about the frag so that future invocations of
          * usnic_handle_large_send can generate segments to put on the wire. */
-        lfrag = (opal_btl_usnic_large_send_frag_t *)frag;
+        lfrag = (opal_btl_usnic_large_send_frag_t *) frag;
         lfrag->lsf_tag = tag;
         lfrag->lsf_cur_offset = 0;
         lfrag->lsf_cur_ptr = lfrag->lsf_des_src[0].seg_addr.pval;
diff --git a/opal/mca/btl/usnic/btl_usnic_send.h b/opal/mca/btl/usnic/btl_usnic_send.h
index 4d093b80156..a4d6fbf25f3 100644
--- a/opal/mca/btl/usnic/btl_usnic_send.h
+++ b/opal/mca/btl/usnic/btl_usnic_send.h
@@ -11,19 +11,17 @@
 #define BTL_USNIC_SEND_H
 
 #include "btl_usnic.h"
-#include "btl_usnic_frag.h"
 #include "btl_usnic_ack.h"
+#include "btl_usnic_frag.h"
 #if MSGDEBUG1
-#include "btl_usnic_util.h"
+#    include "btl_usnic_util.h"
 #endif
 
 /*
  * Check if conditions are right, and if so, put endpoint on
  * list of endpoints that have sends to be done
  */
-static inline void
-opal_btl_usnic_check_rts(
-    opal_btl_usnic_endpoint_t *endpoint)
+static inline void opal_btl_usnic_check_rts(opal_btl_usnic_endpoint_t *endpoint)
 {
     /*
      * If endpoint not already ready,
@@ -32,21 +30,17 @@ opal_btl_usnic_check_rts(
      * and its retransmission window is open,
      * make it ready
      */
-    if (!endpoint->endpoint_ready_to_send &&
-        !opal_list_is_empty(&endpoint->endpoint_frag_send_queue) &&
-         endpoint->endpoint_send_credits > 0 &&
-         WINDOW_OPEN(endpoint)) {
-        opal_list_append(&endpoint->endpoint_module->endpoints_with_sends,
-                &endpoint->super);
+    if (!endpoint->endpoint_ready_to_send
+        && !opal_list_is_empty(&endpoint->endpoint_frag_send_queue)
+        && endpoint->endpoint_send_credits > 0 && WINDOW_OPEN(endpoint)) {
+        opal_list_append(&endpoint->endpoint_module->endpoints_with_sends, &endpoint->super);
         endpoint->endpoint_ready_to_send = true;
 #if MSGDEBUG1
-        opal_output(0, "make endpoint %p RTS\n", (void*)endpoint);
+        opal_output(0, "make endpoint %p RTS\n", (void *) endpoint);
     } else {
-        opal_output(0, "rts:%d empty:%d cred:%d open%d\n",
-                endpoint->endpoint_ready_to_send,
-                opal_list_is_empty(&endpoint->endpoint_frag_send_queue),
-                endpoint->endpoint_send_credits,
-                WINDOW_OPEN(endpoint));
+        opal_output(0, "rts:%d empty:%d cred:%d open%d\n", endpoint->endpoint_ready_to_send,
+                    opal_list_is_empty(&endpoint->endpoint_frag_send_queue),
+                    endpoint->endpoint_send_credits, WINDOW_OPEN(endpoint));
 #endif
     }
 }
@@ -57,11 +51,9 @@ opal_btl_usnic_check_rts(
  * ASSUMES THAT THE CALLER HAS ALREADY CHECKED TO SEE IF WE HAVE
  * A SEND CREDIT!
  */
-static inline void
-opal_btl_usnic_post_segment(
-    opal_btl_usnic_module_t *module,
-    opal_btl_usnic_endpoint_t *endpoint,
-    opal_btl_usnic_send_segment_t *sseg)
+static inline void opal_btl_usnic_post_segment(opal_btl_usnic_module_t *module,
+                                               opal_btl_usnic_endpoint_t *endpoint,
+                                               opal_btl_usnic_send_segment_t *sseg)
 {
     int ret;
 
@@ -70,12 +62,9 @@ opal_btl_usnic_post_segment(
     opal_btl_usnic_channel_t *channel = &module->mod_channels[channel_id];
 
 #if MSGDEBUG1
-    opal_output(0, "post_send: type=%s, ep=%p, remote_addr=%p, addr=%p, len=%"
-                PRIsize_t,
-                usnic_seg_type_str(sseg->ss_base.us_type),
-                (void*) channel->ep,
-                (void*) endpoint->endpoint_remote_addrs[channel_id],
-                (void*) sseg->ss_ptr,
+    opal_output(0, "post_send: type=%s, ep=%p, remote_addr=%p, addr=%p, len=%" PRIsize_t,
+                usnic_seg_type_str(sseg->ss_base.us_type), (void *) channel->ep,
+                (void *) endpoint->endpoint_remote_addrs[channel_id], (void *) sseg->ss_ptr,
                 sseg->ss_len);
 #endif
 
@@ -83,15 +72,11 @@ opal_btl_usnic_post_segment(
     assert(channel->credits > 1);
 
     /* Send the segment */
-    ret = fi_send(channel->ep,
-            sseg->ss_ptr,
-            sseg->ss_len + mca_btl_usnic_component.prefix_send_offset,
-            NULL,
-            endpoint->endpoint_remote_addrs[channel_id],
-            sseg);
+    ret = fi_send(channel->ep, sseg->ss_ptr,
+                  sseg->ss_len + mca_btl_usnic_component.prefix_send_offset, NULL,
+                  endpoint->endpoint_remote_addrs[channel_id], sseg);
     if (OPAL_UNLIKELY(0 != ret)) {
-        opal_btl_usnic_util_abort("fi_send() failed",
-                                  __FILE__, __LINE__);
+        opal_btl_usnic_util_abort("fi_send() failed", __FILE__, __LINE__);
         /* Never returns */
     }
 
@@ -113,11 +98,9 @@ opal_btl_usnic_post_segment(
  * ASSUMES THAT THE CALLER HAS ALREADY CHECKED TO SEE IF WE HAVE
  * A SEND CREDIT!
  */
-static inline void
-opal_btl_usnic_post_ack(
-    opal_btl_usnic_module_t *module,
-    opal_btl_usnic_endpoint_t *endpoint,
-    opal_btl_usnic_send_segment_t *sseg)
+static inline void opal_btl_usnic_post_ack(opal_btl_usnic_module_t *module,
+                                           opal_btl_usnic_endpoint_t *endpoint,
+                                           opal_btl_usnic_send_segment_t *sseg)
 {
     int ret;
 
@@ -126,27 +109,20 @@ opal_btl_usnic_post_ack(
     opal_btl_usnic_channel_t *channel = &module->mod_channels[channel_id];
 
 #if MSGDEBUG1
-    opal_output(0, "post_send ACK: type=%s, ep=%p, remote_addr=%p, addr=%p, len=%"
-                PRIsize_t,
-                usnic_seg_type_str(sseg->ss_base.us_type),
-                (void*) channel->ep,
-                (void*) endpoint->endpoint_remote_addrs[channel_id],
-                (void*) sseg->ss_ptr,
+    opal_output(0, "post_send ACK: type=%s, ep=%p, remote_addr=%p, addr=%p, len=%" PRIsize_t,
+                usnic_seg_type_str(sseg->ss_base.us_type), (void *) channel->ep,
+                (void *) endpoint->endpoint_remote_addrs[channel_id], (void *) sseg->ss_ptr,
                 sseg->ss_len);
 #endif
 
     assert(channel_id == USNIC_PRIORITY_CHANNEL);
     assert(channel->credits > 1);
 
-    ret = fi_send(channel->ep,
-            sseg->ss_ptr,
-            sseg->ss_len + mca_btl_usnic_component.prefix_send_offset,
-            NULL,
-            endpoint->endpoint_remote_addrs[channel_id],
-            sseg);
+    ret = fi_send(channel->ep, sseg->ss_ptr,
+                  sseg->ss_len + mca_btl_usnic_component.prefix_send_offset, NULL,
+                  endpoint->endpoint_remote_addrs[channel_id], sseg);
     if (OPAL_UNLIKELY(0 != ret)) {
-        opal_btl_usnic_util_abort("fi_send() failed",
-                                  __FILE__, __LINE__);
+        opal_btl_usnic_util_abort("fi_send() failed", __FILE__, __LINE__);
         /* Never returns */
     }
 
@@ -159,10 +135,8 @@ opal_btl_usnic_post_ack(
 /*
  * Post a send to the work queue
  */
-static inline void
-opal_btl_usnic_endpoint_send_segment(
-    opal_btl_usnic_module_t *module,
-    opal_btl_usnic_send_segment_t *sseg)
+static inline void opal_btl_usnic_endpoint_send_segment(opal_btl_usnic_module_t *module,
+                                                        opal_btl_usnic_send_segment_t *sseg)
 {
     opal_btl_usnic_send_frag_t *frag;
     opal_btl_usnic_endpoint_t *endpoint;
@@ -192,17 +166,14 @@ opal_btl_usnic_endpoint_send_segment(
 
             next_seq_to_send > ack_seq_rcvd + WINDOW_SIZE
     */
-    assert(SEQ_GT(endpoint->endpoint_next_seq_to_send,
-                  endpoint->endpoint_ack_seq_rcvd));
+    assert(SEQ_GT(endpoint->endpoint_next_seq_to_send, endpoint->endpoint_ack_seq_rcvd));
     assert(WINDOW_OPEN(endpoint));
 
     /* Assign sequence number and increment */
-    sseg->ss_base.us_btl_header->pkt_seq =
-        endpoint->endpoint_next_seq_to_send++;
+    sseg->ss_base.us_btl_header->pkt_seq = endpoint->endpoint_next_seq_to_send++;
 
     /* Fill in remote address to indicate PUT or not */
-    sseg->ss_base.us_btl_header->put_addr =
-        frag->sf_base.uf_remote_seg[0].seg_addr.pval;
+    sseg->ss_base.us_btl_header->put_addr = frag->sf_base.uf_remote_seg[0].seg_addr.pval;
 
     /* piggy-back an ACK if needed */
     opal_btl_usnic_piggyback_ack(endpoint, sseg);
@@ -212,25 +183,23 @@ opal_btl_usnic_endpoint_send_segment(
         char local_ip[32];
         char remote_ip[32];
 
-        opal_btl_usnic_snprintf_ipv4_addr(local_ip, sizeof(local_ip),
-                                          module->local_modex.ipv4_addr,
+        opal_btl_usnic_snprintf_ipv4_addr(local_ip, sizeof(local_ip), module->local_modex.ipv4_addr,
                                           module->local_modex.netmask);
         opal_btl_usnic_snprintf_ipv4_addr(remote_ip, sizeof(remote_ip),
                                           endpoint->endpoint_remote_modex.ipv4_addr,
                                           endpoint->endpoint_remote_modex.netmask);
 
-        opal_output(0, "--> Sending %s: seq: %" UDSEQ ", sender: 0x%016lx from device %s, IP %s, port %u, seg %p, room %d, wc len %u, remote IP %s, port %u",
-            (sseg->ss_parent_frag->sf_base.uf_type == OPAL_BTL_USNIC_FRAG_LARGE_SEND)?
-                    "CHUNK" : "FRAG",
-                    sseg->ss_base.us_btl_header->pkt_seq,
-                    sseg->ss_base.us_btl_header->sender,
-                    endpoint->endpoint_module->linux_device_name,
-                    local_ip,
-                    module->local_modex.ports[sseg->ss_channel],
-                    (void*)sseg,
-                    sseg->ss_hotel_room,
-                    sseg->ss_ptr,
-                    remote_ip,
+        opal_output(0,
+                    "--> Sending %s: seq: %" UDSEQ
+                    ", sender: 0x%016lx from device %s, IP %s, port %u, seg %p, room %d, wc len "
+                    "%u, remote IP %s, port %u",
+                    (sseg->ss_parent_frag->sf_base.uf_type == OPAL_BTL_USNIC_FRAG_LARGE_SEND)
+                        ? "CHUNK"
+                        : "FRAG",
+                    sseg->ss_base.us_btl_header->pkt_seq, sseg->ss_base.us_btl_header->sender,
+                    endpoint->endpoint_module->linux_device_name, local_ip,
+                    module->local_modex.ports[sseg->ss_channel], (void *) sseg, sseg->ss_hotel_room,
+                    sseg->ss_ptr, remote_ip,
                     endpoint->endpoint_remote_modex.ports[sseg->ss_channel]);
     }
 #endif
@@ -251,8 +220,7 @@ opal_btl_usnic_endpoint_send_segment(
     --endpoint->endpoint_send_credits;
 
     /* Stats */
-    if (sseg->ss_parent_frag->sf_base.uf_type
-            == OPAL_BTL_USNIC_FRAG_LARGE_SEND) {
+    if (sseg->ss_parent_frag->sf_base.uf_type == OPAL_BTL_USNIC_FRAG_LARGE_SEND) {
         ++module->stats.num_chunk_sends;
     } else {
         ++module->stats.num_frag_sends;
@@ -260,34 +228,29 @@ opal_btl_usnic_endpoint_send_segment(
 
     /* If we have room in the sender's window, we also have room in
        endpoint hotel */
-    opal_hotel_checkin_with_res(&endpoint->endpoint_hotel, sseg,
-            &sseg->ss_hotel_room);
+    opal_hotel_checkin_with_res(&endpoint->endpoint_hotel, sseg, &sseg->ss_hotel_room);
 }
 
 /*
  * This enqueues a fragment send into the system.  A send of a fragment
  * may result in the sending of multiple segments
  */
-static inline int
-opal_btl_usnic_endpoint_enqueue_frag(
-    opal_btl_usnic_endpoint_t *endpoint,
-    opal_btl_usnic_send_frag_t *frag)
+static inline int opal_btl_usnic_endpoint_enqueue_frag(opal_btl_usnic_endpoint_t *endpoint,
+                                                       opal_btl_usnic_send_frag_t *frag)
 {
 #if MSGDEBUG1
-    opal_output(0, "enq_frag: frag=%p, endpoint=%p, %s, len=%lu\n",
-            (void*)frag, (void*)endpoint,
-            usnic_frag_type(frag->sf_base.uf_type),
-            (long unsigned)frag->sf_base.uf_base.des_src->seg_len);
+    opal_output(0, "enq_frag: frag=%p, endpoint=%p, %s, len=%lu\n", (void *) frag,
+                (void *) endpoint, usnic_frag_type(frag->sf_base.uf_type),
+                (long unsigned) frag->sf_base.uf_base.des_src->seg_len);
     if (frag->sf_base.uf_type == OPAL_BTL_USNIC_FRAG_LARGE_SEND) {
         opal_btl_usnic_large_send_frag_t *lfrag;
-        lfrag = (opal_btl_usnic_large_send_frag_t *)frag;
+        lfrag = (opal_btl_usnic_large_send_frag_t *) frag;
         opal_output(0, "   large size=%zd\n", lfrag->lsf_base.sf_size);
     }
 #endif
 
     /* add to tail of in-progress list */
-    opal_list_append(&endpoint->endpoint_frag_send_queue,
-            &frag->sf_base.uf_base.super.super);
+    opal_list_append(&endpoint->endpoint_frag_send_queue, &frag->sf_base.uf_base.super.super);
 
     /* possibly make this endpoint ready to send again */
     opal_btl_usnic_check_rts(endpoint);
@@ -295,11 +258,9 @@ opal_btl_usnic_endpoint_enqueue_frag(
     return OPAL_SUCCESS;
 }
 
-static inline void
-opal_btl_usnic_release_send_segment(
-    opal_btl_usnic_module_t *module,
-    opal_btl_usnic_send_frag_t *frag,
-    opal_btl_usnic_send_segment_t *sseg)
+static inline void opal_btl_usnic_release_send_segment(opal_btl_usnic_module_t *module,
+                                                       opal_btl_usnic_send_frag_t *frag,
+                                                       opal_btl_usnic_send_segment_t *sseg)
 {
     /* We only return CHUNK segments because they are the only send-style
      * segments that are dynamically allocated.
@@ -312,17 +273,14 @@ opal_btl_usnic_release_send_segment(
 void opal_btl_usnic_frag_complete(opal_btl_usnic_send_frag_t *frag);
 
 void opal_btl_usnic_frag_send_complete(opal_btl_usnic_module_t *module,
-                                    opal_btl_usnic_send_segment_t *sseg);
+                                       opal_btl_usnic_send_segment_t *sseg);
 
 void opal_btl_usnic_chunk_send_complete(opal_btl_usnic_module_t *module,
-                                    opal_btl_usnic_send_segment_t *sseg);
-
-int
-opal_btl_usnic_finish_put_or_send(
-    opal_btl_usnic_module_t *module,
-    opal_btl_usnic_endpoint_t *endpoint,
-    opal_btl_usnic_send_frag_t *frag,
-    mca_btl_base_tag_t tag)
-__opal_attribute_noinline__;
+                                        opal_btl_usnic_send_segment_t *sseg);
+
+int opal_btl_usnic_finish_put_or_send(opal_btl_usnic_module_t *module,
+                                      opal_btl_usnic_endpoint_t *endpoint,
+                                      opal_btl_usnic_send_frag_t *frag,
+                                      mca_btl_base_tag_t tag) __opal_attribute_noinline__;
 
 #endif /* BTL_USNIC_SEND_H */
diff --git a/opal/mca/btl/usnic/btl_usnic_stats.c b/opal/mca/btl/usnic/btl_usnic_stats.c
index ae2a816b5c4..01f8ea60c92 100644
--- a/opal/mca/btl/usnic/btl_usnic_stats.c
+++ b/opal/mca/btl/usnic/btl_usnic_stats.c
@@ -10,16 +10,16 @@
 
 #include "opal_config.h"
 
-#include <unistd.h>
 #include <stdlib.h>
+#include <unistd.h>
 
+#include "opal/mca/base/mca_base_pvar.h"
+#include "opal/mca/base/mca_base_var.h"
 #include "opal/util/output.h"
 #include "opal/util/printf.h"
-#include "opal/mca/base/mca_base_var.h"
-#include "opal/mca/base/mca_base_pvar.h"
 
-#include "btl_usnic_compat.h"
 #include "btl_usnic.h"
+#include "btl_usnic_compat.h"
 #include "btl_usnic_module.h"
 #include "btl_usnic_stats.h"
 #include "btl_usnic_util.h"
@@ -33,44 +33,30 @@ static inline void usnic_stats_reset(opal_btl_usnic_module_t *module)
 {
     int i;
 
-    module->stats.num_total_sends =
-        module->stats.num_resends =
-        module->stats.num_chunk_sends =
-        module->stats.num_frag_sends =
-        module->stats.num_ack_recvs =
-
-        module->stats.num_total_recvs =
-        module->stats.num_unk_recvs =
-        module->stats.num_dup_recvs =
-        module->stats.num_oow_low_recvs =
-        module->stats.num_oow_high_recvs =
-        module->stats.num_frag_recvs =
-        module->stats.num_chunk_recvs =
-        module->stats.num_badfrag_recvs =
-        module->stats.num_ack_sends =
-        module->stats.num_recv_reposts =
-        module->stats.num_crc_errors =
-
-        module->stats.num_old_dup_acks =
-        module->stats.num_dup_acks =
-        module->stats.num_fast_retrans =
-        module->stats.num_timeout_retrans =
-
-        module->stats.max_sent_window_size =
-        module->stats.max_rcvd_window_size =
-
-        module->stats.pml_module_sends =
-        module->stats.pml_send_callbacks =
-
-        module->stats.num_seg_total_completions =
-        module->stats.num_seg_ack_completions =
-        module->stats.num_seg_frag_completions =
-        module->stats.num_seg_chunk_completions =
-        module->stats.num_seg_recv_completions =
-
-        0;
-
-    for (i=0; i<USNIC_NUM_CHANNELS; ++i) {
+    module->stats.num_total_sends = module->stats.num_resends = module->stats.num_chunk_sends
+        = module->stats.num_frag_sends = module->stats.num_ack_recvs =
+
+            module->stats.num_total_recvs = module->stats.num_unk_recvs
+        = module->stats.num_dup_recvs = module->stats.num_oow_low_recvs
+        = module->stats.num_oow_high_recvs = module->stats.num_frag_recvs
+        = module->stats.num_chunk_recvs = module->stats.num_badfrag_recvs = module->stats
+                                                                                .num_ack_sends
+        = module->stats.num_recv_reposts = module->stats.num_crc_errors =
+
+            module->stats.num_old_dup_acks = module->stats.num_dup_acks
+        = module->stats.num_fast_retrans = module->stats.num_timeout_retrans =
+
+            module->stats.max_sent_window_size = module->stats.max_rcvd_window_size =
+
+                module->stats.pml_module_sends = module->stats.pml_send_callbacks =
+
+                    module->stats.num_seg_total_completions = module->stats.num_seg_ack_completions
+        = module->stats.num_seg_frag_completions = module->stats.num_seg_chunk_completions
+        = module->stats.num_seg_recv_completions =
+
+            0;
+
+    for (i = 0; i < USNIC_NUM_CHANNELS; ++i) {
         module->mod_channels[i].num_channel_sends = 0;
     }
 }
@@ -82,58 +68,47 @@ static inline void usnic_stats_reset(opal_btl_usnic_module_t *module)
  * NOTE: this routine ignores the setting of stats_enable, so it can be used
  * for debugging routines even when normal stats reporting is not enabled.
  */
-void opal_btl_usnic_print_stats(
-    opal_btl_usnic_module_t *module,
-    const char *prefix,
-    bool reset_stats)
+void opal_btl_usnic_print_stats(opal_btl_usnic_module_t *module, const char *prefix,
+                                bool reset_stats)
 {
     char tmp[128], str[2048];
 
     /* The usuals */
-    snprintf(str, sizeof(str), "%s:MCW:%3u, %s, ST(P+D)/F/C/R(T+F)/A:%8lu(%8u+%8u)/%8lu/%8lu/%4lu(%4lu+%4lu)/%8lu, RcvTot/Chk/F/C/L/H/D/BF/A:%8lu/%c%c/%8lu/%8lu/%4lu+%2lu/%4lu/%4lu/%6lu Comp:T(A/F/C/R) %8lu(%8lu/%8lu/%8lu/%8lu), OA/DA %4lu/%4lu CRC:%4lu ",
-             prefix,
-             opal_proc_local_get()->proc_name.vpid,
+    snprintf(str, sizeof(str),
+             "%s:MCW:%3u, %s, ST(P+D)/F/C/R(T+F)/A:%8lu(%8u+%8u)/%8lu/%8lu/%4lu(%4lu+%4lu)/%8lu, "
+             "RcvTot/Chk/F/C/L/H/D/BF/A:%8lu/%c%c/%8lu/%8lu/%4lu+%2lu/%4lu/%4lu/%6lu "
+             "Comp:T(A/F/C/R) %8lu(%8lu/%8lu/%8lu/%8lu), OA/DA %4lu/%4lu CRC:%4lu ",
+             prefix, opal_proc_local_get()->proc_name.vpid,
 
              module->linux_device_name,
 
              module->stats.num_total_sends,
              module->mod_channels[USNIC_PRIORITY_CHANNEL].num_channel_sends,
              module->mod_channels[USNIC_DATA_CHANNEL].num_channel_sends,
-             module->stats.num_frag_sends,
-             module->stats.num_chunk_sends,
-             module->stats.num_resends,
-             module->stats.num_timeout_retrans,
-             module->stats.num_fast_retrans,
+             module->stats.num_frag_sends, module->stats.num_chunk_sends, module->stats.num_resends,
+             module->stats.num_timeout_retrans, module->stats.num_fast_retrans,
              module->stats.num_ack_sends,
 
              module->stats.num_total_recvs,
-             (module->stats.num_total_recvs -
-              module->stats.num_recv_reposts) == 0 ? 'g' : 'B',
-             (module->stats.num_total_recvs -
-              module->stats.num_frag_recvs -
-              module->stats.num_chunk_recvs -
-              module->stats.num_badfrag_recvs -
-              module->stats.num_oow_low_recvs -
-              module->stats.num_oow_high_recvs -
-              module->stats.num_dup_recvs -
-              module->stats.num_ack_recvs -
-              module->stats.num_unk_recvs) == 0 ? 'g' : 'B',
-             module->stats.num_frag_recvs,
-             module->stats.num_chunk_recvs,
-             module->stats.num_oow_low_recvs,
-             module->stats.num_oow_high_recvs,
-             module->stats.num_dup_recvs,
-             module->stats.num_badfrag_recvs,
+             (module->stats.num_total_recvs - module->stats.num_recv_reposts) == 0 ? 'g' : 'B',
+             (module->stats.num_total_recvs - module->stats.num_frag_recvs
+              - module->stats.num_chunk_recvs - module->stats.num_badfrag_recvs
+              - module->stats.num_oow_low_recvs - module->stats.num_oow_high_recvs
+              - module->stats.num_dup_recvs - module->stats.num_ack_recvs
+              - module->stats.num_unk_recvs)
+                     == 0
+                 ? 'g'
+                 : 'B',
+             module->stats.num_frag_recvs, module->stats.num_chunk_recvs,
+             module->stats.num_oow_low_recvs, module->stats.num_oow_high_recvs,
+             module->stats.num_dup_recvs, module->stats.num_badfrag_recvs,
              module->stats.num_ack_recvs,
 
-	     module->stats.num_seg_total_completions,
-	     module->stats.num_seg_ack_completions,
-	     module->stats.num_seg_frag_completions,
-	     module->stats.num_seg_chunk_completions,
-	     module->stats.num_seg_recv_completions,
+             module->stats.num_seg_total_completions, module->stats.num_seg_ack_completions,
+             module->stats.num_seg_frag_completions, module->stats.num_seg_chunk_completions,
+             module->stats.num_seg_recv_completions,
 
-             module->stats.num_old_dup_acks,
-             module->stats.num_dup_acks,
+             module->stats.num_old_dup_acks, module->stats.num_dup_acks,
 
              module->stats.num_crc_errors);
 
@@ -144,8 +119,7 @@ void opal_btl_usnic_print_stats(
 
     /* If our PML calls were 0, then show send and receive window
        extents instead */
-    if (module->stats.pml_module_sends +
-        module->stats.pml_send_callbacks == 0) {
+    if (module->stats.pml_module_sends + module->stats.pml_send_callbacks == 0) {
         int64_t send_unacked, su_min = WINDOW_SIZE * 2, su_max = 0;
         int64_t recv_depth, rd_min = WINDOW_SIZE * 2, rd_max = 0;
         opal_btl_usnic_endpoint_t *endpoint;
@@ -157,39 +131,36 @@ void opal_btl_usnic_print_stats(
         opal_mutex_lock(&module->all_endpoints_lock);
         item = opal_list_get_first(&module->all_endpoints);
         while (item != opal_list_get_end(&(module->all_endpoints))) {
-            endpoint = container_of(item, mca_btl_base_endpoint_t,
-                    endpoint_endpoint_li);
+            endpoint = container_of(item, mca_btl_base_endpoint_t, endpoint_endpoint_li);
             item = opal_list_get_next(item);
 
             /* Number of un-acked sends (i.e., sends for which we're
                still waiting for ACK) */
-            send_unacked =
-                SEQ_DIFF(endpoint->endpoint_next_seq_to_send,
-                         SEQ_DIFF(endpoint->endpoint_ack_seq_rcvd, 1));
+            send_unacked = SEQ_DIFF(endpoint->endpoint_next_seq_to_send,
+                                    SEQ_DIFF(endpoint->endpoint_ack_seq_rcvd, 1));
 
-            if (send_unacked > su_max) su_max = send_unacked;
-            if (send_unacked < su_min) su_min = send_unacked;
+            if (send_unacked > su_max)
+                su_max = send_unacked;
+            if (send_unacked < su_min)
+                su_min = send_unacked;
 
             /* Receive window depth (i.e., difference between highest
                seq received and the next message we haven't ACKed
                yet) */
-            recv_depth =
-                endpoint->endpoint_highest_seq_rcvd -
-                endpoint->endpoint_next_contig_seq_to_recv;
-            if (recv_depth > rd_max) rd_max = recv_depth;
-            if (recv_depth < rd_min) rd_min = recv_depth;
+            recv_depth = endpoint->endpoint_highest_seq_rcvd
+                         - endpoint->endpoint_next_contig_seq_to_recv;
+            if (recv_depth > rd_max)
+                rd_max = recv_depth;
+            if (recv_depth < rd_min)
+                rd_min = recv_depth;
         }
         opal_mutex_unlock(&module->all_endpoints_lock);
         snprintf(tmp, sizeof(tmp), "PML S:%1ld, Win!A/R:%4ld/%4ld %4ld/%4ld",
-                 module->stats.pml_module_sends,
-                 su_min, su_max,
-                 rd_min, rd_max);
+                 module->stats.pml_module_sends, su_min, su_max, rd_min, rd_max);
     } else {
-        snprintf(tmp, sizeof(tmp), "PML S/CB/Diff:%4lu/%4lu=%4ld",
-                module->stats.pml_module_sends,
-                module->stats.pml_send_callbacks,
-                module->stats.pml_module_sends -
-                 module->stats.pml_send_callbacks);
+        snprintf(tmp, sizeof(tmp), "PML S/CB/Diff:%4lu/%4lu=%4ld", module->stats.pml_module_sends,
+                 module->stats.pml_send_callbacks,
+                 module->stats.pml_module_sends - module->stats.pml_send_callbacks);
     }
 
     strncat(str, tmp, sizeof(str) - strlen(str) - 1);
@@ -205,7 +176,7 @@ void opal_btl_usnic_print_stats(
  */
 static void usnic_stats_callback(int fd, short flags, void *arg)
 {
-    opal_btl_usnic_module_t *module = (opal_btl_usnic_module_t*) arg;
+    opal_btl_usnic_module_t *module = (opal_btl_usnic_module_t *) arg;
     char tmp[128];
 
     if (!mca_btl_usnic_component.stats_enabled) {
@@ -229,12 +200,9 @@ int opal_btl_usnic_stats_init(opal_btl_usnic_module_t *module)
         module->stats.timeout.tv_sec = mca_btl_usnic_component.stats_frequency;
         module->stats.timeout.tv_usec = 0;
 
-        opal_event_set(mca_btl_usnic_component.opal_evbase,
-                       &(module->stats.timer_event),
-                       -1, EV_TIMEOUT | EV_PERSIST,
-                       &usnic_stats_callback, module);
-        opal_event_add(&(module->stats.timer_event),
-                       &(module->stats.timeout));
+        opal_event_set(mca_btl_usnic_component.opal_evbase, &(module->stats.timer_event), -1,
+                       EV_TIMEOUT | EV_PERSIST, &usnic_stats_callback, module);
+        opal_event_add(&(module->stats.timer_event), &(module->stats.timeout));
     }
 
     return OPAL_SUCCESS;
@@ -261,9 +229,8 @@ int opal_btl_usnic_stats_finalize(opal_btl_usnic_module_t *module)
  * Function called by the pvar base upon MPI_T_pvar_handle_alloc,
  * handle_start, and handle_stop.
  */
-static int usnic_pvar_notify(struct mca_base_pvar_t *pvar,
-                             mca_base_pvar_event_t event,
-                             void *obj, int *count)
+static int usnic_pvar_notify(struct mca_base_pvar_t *pvar, mca_base_pvar_event_t event, void *obj,
+                             int *count)
 {
     if (MCA_BASE_PVAR_HANDLE_BIND == event) {
         *count = mca_btl_usnic_component.num_modules;
@@ -274,26 +241,23 @@ static int usnic_pvar_notify(struct mca_base_pvar_t *pvar,
     return OPAL_SUCCESS;
 }
 
-
 /*
  * Function called by the pvar base when a user wants to read the
  * value of an MPI_T performance variable.
  */
-static int usnic_pvar_read(const struct mca_base_pvar_t *pvar,
-                           void *value, void *bound_obj)
+static int usnic_pvar_read(const struct mca_base_pvar_t *pvar, void *value, void *bound_obj)
 {
     size_t offset = (size_t) pvar->ctx;
-    uint64_t *array = (uint64_t*) value;
+    uint64_t *array = (uint64_t *) value;
 
     for (int i = 0; i < mca_btl_usnic_component.num_modules; ++i) {
-        char *base = (char*) &(mca_btl_usnic_component.usnic_active_modules[i]->stats);
-        array[i] = *((uint64_t*) (base + offset));
+        char *base = (char *) &(mca_btl_usnic_component.usnic_active_modules[i]->stats);
+        array[i] = *((uint64_t *) (base + offset));
     }
 
     return OPAL_SUCCESS;
 }
 
-
 /*
  * Register an MPI_T performance variable of type CLASS_HIGHWATERMARK.
  */
@@ -301,31 +265,24 @@ static void register_pvar_highwater(char *name, char *desc, size_t offset)
 {
     int rc __opal_attribute_unused__;
 
-    rc = mca_base_component_pvar_register(&mca_btl_usnic_component.super.btl_version,
-                                          name, desc,
-                                          OPAL_INFO_LVL_5,
-                                          MCA_BASE_PVAR_CLASS_HIGHWATERMARK,
-                                          pvar_type,
-                                          NULL, /* enumeration */
+    rc = mca_base_component_pvar_register(&mca_btl_usnic_component.super.btl_version, name, desc,
+                                          OPAL_INFO_LVL_5, MCA_BASE_PVAR_CLASS_HIGHWATERMARK,
+                                          pvar_type, NULL, /* enumeration */
                                           MCA_BASE_VAR_BIND_NO_OBJECT,
-                                          (MCA_BASE_PVAR_FLAG_READONLY |
-                                           MCA_BASE_PVAR_FLAG_CONTINUOUS),
-                                          usnic_pvar_read,
-                                          NULL, /* write function */
-                                          usnic_pvar_notify,
-                                          (void *) offset);
+                                          (MCA_BASE_PVAR_FLAG_READONLY
+                                           | MCA_BASE_PVAR_FLAG_CONTINUOUS),
+                                          usnic_pvar_read, NULL, /* write function */
+                                          usnic_pvar_notify, (void *) offset);
     assert(rc >= 0);
 }
 
-
 /*
  * Function called by the pvar base when a user wants to read the
  * devices enum value.  The array is a simple list of 0..num_modules,
  * which will map to the strings in the devices_enum
  * setup_mpit_pvar_type().
  */
-static int usnic_pvar_enum_read(const struct mca_base_pvar_t *pvar,
-                                void *value, void *bound_obj)
+static int usnic_pvar_enum_read(const struct mca_base_pvar_t *pvar, void *value, void *bound_obj)
 {
     int *array = (int *) value;
 
@@ -336,7 +293,6 @@ static int usnic_pvar_enum_read(const struct mca_base_pvar_t *pvar,
     return OPAL_SUCCESS;
 }
 
-
 /*
  * Register an MPI_T performance variable of type CLASS_COUNTER.
  */
@@ -344,23 +300,17 @@ static void register_pvar_counter(char *name, char *desc, size_t offset)
 {
     int rc __opal_attribute_unused__;
 
-    rc = mca_base_component_pvar_register(&mca_btl_usnic_component.super.btl_version,
-                                          name, desc,
-                                          OPAL_INFO_LVL_5,
-                                          MCA_BASE_PVAR_CLASS_COUNTER,
-                                          pvar_type,
+    rc = mca_base_component_pvar_register(&mca_btl_usnic_component.super.btl_version, name, desc,
+                                          OPAL_INFO_LVL_5, MCA_BASE_PVAR_CLASS_COUNTER, pvar_type,
                                           NULL, /* enumeration */
                                           MCA_BASE_VAR_BIND_NO_OBJECT,
-                                          (MCA_BASE_PVAR_FLAG_READONLY |
-                                           MCA_BASE_PVAR_FLAG_CONTINUOUS),
-                                          usnic_pvar_read,
-                                          NULL, /* write function */
-                                          usnic_pvar_notify,
-                                          (void *) offset);
+                                          (MCA_BASE_PVAR_FLAG_READONLY
+                                           | MCA_BASE_PVAR_FLAG_CONTINUOUS),
+                                          usnic_pvar_read, NULL, /* write function */
+                                          usnic_pvar_notify, (void *) offset);
     assert(rc >= 0);
 }
 
-
 /*
  * Find the MPI_T type corresponding to our uint64_t counters and
  * highwatermarks.
@@ -384,7 +334,6 @@ static bool setup_mpit_pvar_type(void)
     return true;
 }
 
-
 /*
  * Setup the usnic_X device enumeration pvar
  */
@@ -398,8 +347,7 @@ static void setup_mpit_pvars_enum(void)
     unsigned char *c;
     struct sockaddr_in *sin;
 
-    devices = calloc(mca_btl_usnic_component.num_modules + 1,
-                     sizeof(*devices));
+    devices = calloc(mca_btl_usnic_component.num_modules + 1, sizeof(*devices));
     assert(devices != NULL);
 
     for (i = 0; i < mca_btl_usnic_component.num_modules; ++i) {
@@ -407,13 +355,11 @@ static void setup_mpit_pvars_enum(void)
 
         m = mca_btl_usnic_component.usnic_active_modules[i];
         sin = m->fabric_info->src_addr;
-        c = (unsigned char*) &sin->sin_addr.s_addr;
+        c = (unsigned char *) &sin->sin_addr.s_addr;
 
         devices[i].value = i;
-        rc = opal_asprintf(&str, "%s,%hhu.%hhu.%hhu.%hhu/%" PRIu32,
-                      m->linux_device_name,
-                      c[0], c[1], c[2], c[3],
-                      usnic_netmask_to_cidrlen(sin->sin_addr.s_addr));
+        rc = opal_asprintf(&str, "%s,%hhu.%hhu.%hhu.%hhu/%" PRIu32, m->linux_device_name, c[0],
+                           c[1], c[2], c[3], usnic_netmask_to_cidrlen(sin->sin_addr.s_addr));
         assert(rc > 0);
         devices[i].string = str;
     }
@@ -422,26 +368,20 @@ static void setup_mpit_pvars_enum(void)
     rc = mca_base_var_enum_create("btl_usnic", devices, &devices_enum);
     assert(OPAL_SUCCESS == rc);
 
-    rc = mca_base_component_pvar_register(&mca_btl_usnic_component.super.btl_version,
-                                          "devices",
-                                          "Enumeration representing which slot in btl_usnic_* MPI_T pvar value arrays correspond to which usnic_X Linux device",
-                                          OPAL_INFO_LVL_5,
-                                          MCA_BASE_PVAR_CLASS_STATE,
-                                          MCA_BASE_VAR_TYPE_INT,
-                                          devices_enum,
-                                          MCA_BASE_VAR_BIND_NO_OBJECT,
-                                          (MCA_BASE_PVAR_FLAG_READONLY |
-                                           MCA_BASE_PVAR_FLAG_CONTINUOUS),
-                                          usnic_pvar_enum_read,
-                                          NULL, /* write function */
-                                          usnic_pvar_notify,
-                                          NULL /* context */);
+    rc = mca_base_component_pvar_register(
+        &mca_btl_usnic_component.super.btl_version, "devices",
+        "Enumeration representing which slot in btl_usnic_* MPI_T pvar value arrays correspond to "
+        "which usnic_X Linux device",
+        OPAL_INFO_LVL_5, MCA_BASE_PVAR_CLASS_STATE, MCA_BASE_VAR_TYPE_INT, devices_enum,
+        MCA_BASE_VAR_BIND_NO_OBJECT, (MCA_BASE_PVAR_FLAG_READONLY | MCA_BASE_PVAR_FLAG_CONTINUOUS),
+        usnic_pvar_enum_read, NULL, /* write function */
+        usnic_pvar_notify, NULL /* context */);
     assert(rc >= 0);
 
     /* Free the strings (mca_base_var_enum_create() strdup()'ed them
        into private storage, so we don't need them any more) */
-    for (int i = 0; i < mca_btl_usnic_component.num_modules; ++i) {
-        free((char*) devices[i].string);
+    for (i = 0; i < mca_btl_usnic_component.num_modules; ++i) {
+        free((char *) devices[i].string);
     }
     free(devices);
 
@@ -451,7 +391,6 @@ static void setup_mpit_pvars_enum(void)
     OBJ_RELEASE(devices_enum);
 }
 
-
 /*
  * Setup high watermark MPI_T performance variables
  */
@@ -466,7 +405,6 @@ static void setup_mpit_pvars_highwatermark(void)
                "Maximum number of entries in all receive windows to this peer");
 }
 
-
 /*
  * Setup counter MPI_T performance variables
  */
@@ -475,57 +413,49 @@ static void setup_mpit_pvars_counters(void)
 #define REGISTERC(field, desc) \
     register_pvar_counter(#field, (desc), offsetof(opal_btl_usnic_module_stats_t, field))
 
-    REGISTERC(num_total_sends,
-              "Total number of sends (MPI data, ACKs, retransmissions, etc.)");
-    REGISTERC(num_resends,
-              "Total number of all retransmissions");
-    REGISTERC(num_timeout_retrans,
-              "Number of times chunk retransmissions have occured because an ACK was not received within the timeout");
+    REGISTERC(num_total_sends, "Total number of sends (MPI data, ACKs, retransmissions, etc.)");
+    REGISTERC(num_resends, "Total number of all retransmissions");
+    REGISTERC(num_timeout_retrans, "Number of times chunk retransmissions have occurred because an "
+                                   "ACK was not received within the timeout");
     REGISTERC(num_fast_retrans,
-              "Number of times chunk retransmissions have occured because due to a repeated ACK");
+              "Number of times chunk retransmissions have occurred because due to a repeated ACK");
     REGISTERC(num_chunk_sends,
-              "Number of sends that were part of a larger MPI message fragment (i.e., the MPI message was so long that it had to be split into multiple MTU/network sends)");
-    REGISTERC(num_frag_sends,
-              "Number of sends where the entire MPI message fragment fit into a single MTU/network send");
-    REGISTERC(num_ack_sends,
-              "Number of ACKs sent (i.e., usNIC-BTL-to-usNIC-BTL control messages)");
-
-    REGISTERC(num_total_recvs,
-              "Total number of receives completed");
-    REGISTERC(num_unk_recvs,
-              "Number of receives with an unknown source or type, and therefore ignored by the usNIC BTL (this should never be >0)");
-    REGISTERC(num_dup_recvs,
-              "Number of duplicate receives");
+              "Number of sends that were part of a larger MPI message fragment (i.e., the MPI "
+              "message was so long that it had to be split into multiple MTU/network sends)");
+    REGISTERC(
+        num_frag_sends,
+        "Number of sends where the entire MPI message fragment fit into a single MTU/network send");
+    REGISTERC(num_ack_sends, "Number of ACKs sent (i.e., usNIC-BTL-to-usNIC-BTL control messages)");
+
+    REGISTERC(num_total_recvs, "Total number of receives completed");
+    REGISTERC(num_unk_recvs, "Number of receives with an unknown source or type, and therefore "
+                             "ignored by the usNIC BTL (this should never be >0)");
+    REGISTERC(num_dup_recvs, "Number of duplicate receives");
     REGISTERC(num_oow_low_recvs,
               "Number of times a receive was out of the sliding window (on the low side)");
     REGISTERC(num_oow_high_recvs,
               "Number of times a receive was out of the sliding window (on the high side)");
-    REGISTERC(num_frag_recvs,
-              "Number of receives where the entire MPI message fragment fit into a single MTU/network send");
+    REGISTERC(num_frag_recvs, "Number of receives where the entire MPI message fragment fit into a "
+                              "single MTU/network send");
     REGISTERC(num_chunk_recvs,
-              "Number of receives that were part of a larger MPI message fragment (i.e., this receive was reassembled into a larger MPI message fragment)");
+              "Number of receives that were part of a larger MPI message fragment (i.e., this "
+              "receive was reassembled into a larger MPI message fragment)");
     REGISTERC(num_badfrag_recvs,
               "Number of chunks received that had a bad fragment ID (this should never be >0)");
 
-    REGISTERC(num_ack_recvs,
-              "Total number of ACKs received");
+    REGISTERC(num_ack_recvs, "Total number of ACKs received");
     REGISTERC(num_old_dup_acks,
               "Number of old duplicate ACKs received (i.e., before the current expected ACK)");
-    REGISTERC(num_dup_acks,
-              "Number of duplicate ACKs received (i.e., the current expected ACK)");
+    REGISTERC(num_dup_acks, "Number of duplicate ACKs received (i.e., the current expected ACK)");
 
-    REGISTERC(num_recv_reposts,
-              "Number of times buffers have been reposted for receives");
-    REGISTERC(num_crc_errors,
-              "Number of times receives were aborted because of a CRC error");
+    REGISTERC(num_recv_reposts, "Number of times buffers have been reposted for receives");
+    REGISTERC(num_crc_errors, "Number of times receives were aborted because of a CRC error");
 
-    REGISTERC(pml_module_sends,
-              "Number of times the PML has called down to send a message");
+    REGISTERC(pml_module_sends, "Number of times the PML has called down to send a message");
     REGISTERC(pml_send_callbacks,
               "Number of times the usNIC BTL has called up to the PML to complete a send");
 }
 
-
 /*
  * Initialize MPI_T performance variables
  */
diff --git a/opal/mca/btl/usnic/btl_usnic_stats.h b/opal/mca/btl/usnic/btl_usnic_stats.h
index ae18cb0fed6..efbdead634b 100644
--- a/opal/mca/btl/usnic/btl_usnic_stats.h
+++ b/opal/mca/btl/usnic/btl_usnic_stats.h
@@ -22,7 +22,6 @@
 
 #include "opal/util/event.h"
 
-
 /**
  * Struct containing all the statistics that are trackedx
  */
@@ -68,7 +67,6 @@ typedef struct opal_btl_usnic_module_stats_t {
     struct timeval timeout;
 } opal_btl_usnic_module_stats_t;
 
-
 /**
  * Initialize the stats on a module.  Must use "struct
  * opal_btl_usnic_module_t*" here to avoid an #include cycle.
diff --git a/opal/mca/btl/usnic/btl_usnic_test.c b/opal/mca/btl/usnic/btl_usnic_test.c
index 49fb7079b5f..8cb5baa5123 100644
--- a/opal/mca/btl/usnic/btl_usnic_test.c
+++ b/opal/mca/btl/usnic/btl_usnic_test.c
@@ -53,9 +53,7 @@ static void init_test_infra(void)
     }
 }
 
-void opal_btl_usnic_register_test(const char *name,
-                                  opal_btl_usnic_test_fn_t test_fn,
-                                  void *ctx)
+void opal_btl_usnic_register_test(const char *name, opal_btl_usnic_test_fn_t test_fn, void *ctx)
 {
     struct test_info *info = malloc(sizeof(*info));
     assert(info != NULL);
@@ -82,38 +80,34 @@ void opal_btl_usnic_run_tests(void)
     }
     test_out("STARTING TESTS\n");
 
-    OPAL_LIST_FOREACH(info, &all_tests, struct test_info) {
+    OPAL_LIST_FOREACH (info, &all_tests, struct test_info) {
         test_out("running test '%s'... ", info->name);
         result = info->test_fn(info->ctx);
 
         ++opal_btl_usnic_num_tests_run;
         switch (result) {
-            case TEST_PASSED:
-                ++opal_btl_usnic_num_tests_passed;
-                test_out("PASSED\n");
-                break;
-            case TEST_FAILED:
-                ++opal_btl_usnic_num_tests_failed;
-                test_out("FAILED\n");
-                break;
-            case TEST_SKIPPED:
-                ++opal_btl_usnic_num_tests_skipped;
-                test_out("SKIPPED\n");
-                break;
+        case TEST_PASSED:
+            ++opal_btl_usnic_num_tests_passed;
+            test_out("PASSED\n");
+            break;
+        case TEST_FAILED:
+            ++opal_btl_usnic_num_tests_failed;
+            test_out("FAILED\n");
+            break;
+        case TEST_SKIPPED:
+            ++opal_btl_usnic_num_tests_skipped;
+            test_out("SKIPPED\n");
+            break;
         }
     }
 
-    test_out("FINISHED TESTS (%d passed, %d failed, %d skipped)\n",
-             opal_btl_usnic_num_tests_passed,
-             opal_btl_usnic_num_tests_failed,
-             opal_btl_usnic_num_tests_skipped);
+    test_out("FINISHED TESTS (%d passed, %d failed, %d skipped)\n", opal_btl_usnic_num_tests_passed,
+             opal_btl_usnic_num_tests_failed, opal_btl_usnic_num_tests_skipped);
 }
 
 #else /* !OPAL_BTL_USNIC_UNIT_TESTS */
 
-void opal_btl_usnic_register_test(const char *name,
-                                  opal_btl_usnic_test_fn_t test_fn,
-                                  void *ctx)
+void opal_btl_usnic_register_test(const char *name, opal_btl_usnic_test_fn_t test_fn, void *ctx)
 {
     abort(); /* never should be called */
 }
diff --git a/opal/mca/btl/usnic/btl_usnic_test.h b/opal/mca/btl/usnic/btl_usnic_test.h
index e6f6b2d7d6c..7544d6d34dc 100644
--- a/opal/mca/btl/usnic/btl_usnic_test.h
+++ b/opal/mca/btl/usnic/btl_usnic_test.h
@@ -15,79 +15,71 @@
 typedef int (*opal_btl_usnic_test_fn_t)(void *ctx);
 
 #if OPAL_BTL_USNIC_UNIT_TESTS
-#  define test_out(...) fprintf(stderr, __VA_ARGS__)
-#  define check(a)                                                           \
-    do {                                                                     \
-        if (!(a)) {                                                          \
-            test_out("%s:%d: check failed, '%s'\n", __func__, __LINE__, #a); \
-            return TEST_FAILED;                                              \
-        }                                                                    \
-    } while (0)
-#  define check_str_eq(a,b)                                     \
-    do {                                                        \
-        const char *a_ = (a);                                   \
-        const char *b_ = (b);                                   \
-        if (0 != strcmp(a_,b_)) {                               \
-            test_out("%s:%d: check failed, \"%s\" != \"%s\"\n", \
-                     __func__, __LINE__, a_, b_);               \
-            return TEST_FAILED;                                 \
-        }                                                       \
-    } while (0)
-#  define check_int_eq(got, expected)                                   \
-    do {                                                                \
-        if ((got) != (expected)) {                                      \
-            test_out("%s:%d: check failed, \"%s\" != \"%s\", got %d\n", \
-                     __func__, __LINE__, #got, #expected, (got));       \
-            return TEST_FAILED;                                         \
-        }                                                               \
-    } while (0)
+#    define test_out(...) fprintf(stderr, __VA_ARGS__)
+#    define check(a)                                                             \
+        do {                                                                     \
+            if (!(a)) {                                                          \
+                test_out("%s:%d: check failed, '%s'\n", __func__, __LINE__, #a); \
+                return TEST_FAILED;                                              \
+            }                                                                    \
+        } while (0)
+#    define check_str_eq(a, b)                                                                   \
+        do {                                                                                     \
+            const char *a_ = (a);                                                                \
+            const char *b_ = (b);                                                                \
+            if (0 != strcmp(a_, b_)) {                                                           \
+                test_out("%s:%d: check failed, \"%s\" != \"%s\"\n", __func__, __LINE__, a_, b_); \
+                return TEST_FAILED;                                                              \
+            }                                                                                    \
+        } while (0)
+#    define check_int_eq(got, expected)                                                         \
+        do {                                                                                    \
+            if ((got) != (expected)) {                                                          \
+                test_out("%s:%d: check failed, \"%s\" != \"%s\", got %d\n", __func__, __LINE__, \
+                         #got, #expected, (got));                                               \
+                return TEST_FAILED;                                                             \
+            }                                                                                   \
+        } while (0)
 /* just use check_int_eq for now, no public error code to string routine
  * exists (opal_err2str is static) */
-#  define check_err_code(got, expected)                                 \
-    check_int_eq(got, expected)
-#  define check_msg(a, msg)                                \
-    do {                                                   \
-        if (!(a)) {                                        \
-            test_out("%s:%d: check failed, \"%s\" (%s)\n", \
-                     __func__, __LINE__, #a, (msg));       \
-            return TEST_FAILED;                            \
-        }                                                  \
-    } while (0)
+#    define check_err_code(got, expected) check_int_eq(got, expected)
+#    define check_msg(a, msg)                                                                  \
+        do {                                                                                   \
+            if (!(a)) {                                                                        \
+                test_out("%s:%d: check failed, \"%s\" (%s)\n", __func__, __LINE__, #a, (msg)); \
+                return TEST_FAILED;                                                            \
+            }                                                                                  \
+        } while (0)
 
 extern int opal_btl_usnic_num_tests_run;
 extern int opal_btl_usnic_num_tests_passed;
 extern int opal_btl_usnic_num_tests_failed;
 extern int opal_btl_usnic_num_tests_skipped;
 
-enum test_result {
-    TEST_PASSED = 0,
-    TEST_FAILED,
-    TEST_SKIPPED
-};
+enum test_result { TEST_PASSED = 0, TEST_FAILED, TEST_SKIPPED };
 
 /* let us actually paste __LINE__ with other tokens */
-#  define USNIC_PASTE(a,b) USNIC_PASTE2(a,b)
-#  define USNIC_PASTE2(a,b) a ## b
+#    define USNIC_PASTE(a, b)  USNIC_PASTE2(a, b)
+#    define USNIC_PASTE2(a, b) a##b
 /* A helper macro to de-clutter test registration. */
-#  define USNIC_REGISTER_TEST(name, test_fn, ctx)       \
-__attribute__((__constructor__))                        \
-static void USNIC_PASTE(usnic_reg_ctor_,__LINE__)(void) \
-{                                                       \
-    opal_btl_usnic_register_test(name, test_fn, ctx);   \
-}                                                       \
+#    define USNIC_REGISTER_TEST(name, test_fn, ctx)                                               \
+        __attribute__((__constructor__)) static void USNIC_PASTE(usnic_reg_ctor_, __LINE__)(void) \
+        {                                                                                         \
+            opal_btl_usnic_register_test(name, test_fn, ctx);                                     \
+        }
 
 #else /* !OPAL_BTL_USNIC_UNIT_TESTS */
-#  define test_out(...) do {} while(0)
-#  define USNIC_REGISTER_TEST(name, test_fn, ctx)
+#    define test_out(...) \
+        do {              \
+        } while (0)
+#    define USNIC_REGISTER_TEST(name, test_fn, ctx)
 #endif
 
 /* Run all registered tests.  Typically called by an external utility that
  * dlopens the usnic BTL shared object.  See run_usnic_tests.c. */
 void opal_btl_usnic_run_tests(void);
 
-void opal_btl_usnic_register_test(const char *name,
-                                  opal_btl_usnic_test_fn_t test_fn,
-                                  void *ctx);
+void opal_btl_usnic_register_test(const char *name, opal_btl_usnic_test_fn_t test_fn, void *ctx);
 
 /* should be called once, at component close time */
 void opal_btl_usnic_cleanup_tests(void);
diff --git a/opal/mca/btl/usnic/btl_usnic_util.c b/opal/mca/btl/usnic/btl_usnic_util.c
index 8b19f161f0a..3596ec311d3 100644
--- a/opal/mca/btl/usnic/btl_usnic_util.c
+++ b/opal/mca/btl/usnic/btl_usnic_util.c
@@ -12,14 +12,13 @@
 #include <stdio.h>
 #include <unistd.h>
 
-#include "opal/util/show_help.h"
 #include "opal/constants.h"
 #include "opal/util/if.h"
+#include "opal/util/show_help.h"
 
 #include "btl_usnic_module.h"
 #include "btl_usnic_util.h"
 
-
 // The following comment tells Coverity that this function does not return.
 // See https://scan.coverity.com/tune.
 
@@ -29,9 +28,9 @@ void opal_btl_usnic_exit(opal_btl_usnic_module_t *module)
     if (NULL == module) {
         /* Find the first module with an error callback */
         for (int i = 0; i < mca_btl_usnic_component.num_modules; ++i) {
-            if (NULL != mca_btl_usnic_component.usnic_active_modules &&
-                NULL != mca_btl_usnic_component.usnic_active_modules[i] &&
-                NULL != mca_btl_usnic_component.usnic_active_modules[i]->pml_error_callback) {
+            if (NULL != mca_btl_usnic_component.usnic_active_modules
+                && NULL != mca_btl_usnic_component.usnic_active_modules[i]
+                && NULL != mca_btl_usnic_component.usnic_active_modules[i]->pml_error_callback) {
                 module = mca_btl_usnic_component.usnic_active_modules[i];
                 break;
             }
@@ -50,10 +49,10 @@ void opal_btl_usnic_exit(opal_btl_usnic_module_t *module)
        if the passed proc is yourself (e.g., don't call del_procs() on
        yourself). */
     if (NULL != module->pml_error_callback) {
-        module->pml_error_callback(&module->super,
-                                   MCA_BTL_ERROR_FLAGS_FATAL,
-                                   (opal_proc_t*) opal_proc_local_get(),
-                                   "The usnic BTL is aborting the MPI job (via PML error callback).");
+        module
+            ->pml_error_callback(&module->super, MCA_BTL_ERROR_FLAGS_FATAL,
+                                 (opal_proc_t *) opal_proc_local_get(),
+                                 "The usnic BTL is aborting the MPI job (via PML error callback).");
     }
 
     /* If the PML error callback returns (or if there wasn't one),
@@ -61,7 +60,6 @@ void opal_btl_usnic_exit(opal_btl_usnic_module_t *module)
     exit(1);
 }
 
-
 /*
  * Simple utility in a .c file, mainly so that inline functions in .h
  * files don't need to include the show_help header file.
@@ -75,25 +73,20 @@ void opal_btl_usnic_exit(opal_btl_usnic_module_t *module)
 /* coverity[+kill] */
 void opal_btl_usnic_util_abort(const char *msg, const char *file, int line)
 {
-    opal_show_help("help-mpi-btl-usnic.txt", "internal error after init",
-                   true,
-                   opal_process_info.nodename,
-                   file, line, msg);
+    opal_show_help("help-mpi-btl-usnic.txt", "internal error after init", true,
+                   opal_process_info.nodename, file, line, msg);
 
     opal_btl_usnic_exit(NULL);
     /* Never returns */
 }
 
-
-void
-opal_btl_usnic_dump_hex(int verbose_level, int output_id,
-                        void *vaddr, int len)
+void opal_btl_usnic_dump_hex(int verbose_level, int output_id, void *vaddr, int len)
 {
     char buf[128];
     size_t bufspace;
     int i, ret;
     char *p;
-    uint32_t sum=0;
+    uint32_t sum = 0;
     uint8_t *addr;
 
     addr = vaddr;
@@ -101,65 +94,51 @@ opal_btl_usnic_dump_hex(int verbose_level, int output_id,
     memset(buf, 0, sizeof(buf));
     bufspace = sizeof(buf) - 1;
 
-    for (i=0; i<len; ++i) {
+    for (i = 0; i < len; ++i) {
         ret = snprintf(p, bufspace, "%02x ", addr[i]);
         p += ret;
         bufspace -= ret;
 
         sum += addr[i];
-        if ((i&15) == 15) {
-            opal_output_verbose(verbose_level, output_id,
-                                "%4x: %s\n", i&~15, buf);
+        if ((i & 15) == 15) {
+            opal_output_verbose(verbose_level, output_id, "%4x: %s\n", i & ~15, buf);
 
             p = buf;
             memset(buf, 0, sizeof(buf));
             bufspace = sizeof(buf) - 1;
         }
     }
-    if ((i&15) != 0) {
-        opal_output_verbose(verbose_level, output_id,
-                            "%4x: %s\n", i&~15, buf);
+    if ((i & 15) != 0) {
+        opal_output_verbose(verbose_level, output_id, "%4x: %s\n", i & ~15, buf);
     }
     /*opal_output_verbose(verbose_level, output_id, "buffer sum = %x\n", sum); */
 }
 
-
 /*
  * Trivial wrapper around snprintf'ing an IPv4 address, with or
  * without a CIDR mask (we don't usually carry around addresses in
  * struct sockaddr form, so this wrapper is marginally easier than
  * using inet_ntop()).
  */
-void opal_btl_usnic_snprintf_ipv4_addr(char *out, size_t maxlen,
-                                       uint32_t addr_be, uint32_t netmask_be)
+void opal_btl_usnic_snprintf_ipv4_addr(char *out, size_t maxlen, uint32_t addr_be,
+                                       uint32_t netmask_be)
 {
     int prefixlen;
     uint32_t netmask = ntohl(netmask_be);
     uint32_t addr = ntohl(addr_be);
-    uint8_t *p = (uint8_t*) &addr;
+    uint8_t *p = (uint8_t *) &addr;
 
     if (netmask != 0) {
         prefixlen = 33 - ffs(netmask);
-        snprintf(out, maxlen, "%u.%u.%u.%u/%u",
-                 p[3],
-                 p[2],
-                 p[1],
-                 p[0],
-                 prefixlen);
+        snprintf(out, maxlen, "%u.%u.%u.%u/%u", p[3], p[2], p[1], p[0], prefixlen);
     } else {
-        snprintf(out, maxlen, "%u.%u.%u.%u",
-                 p[3],
-                 p[2],
-                 p[1],
-                 p[0]);
+        snprintf(out, maxlen, "%u.%u.%u.%u", p[3], p[2], p[1], p[0]);
     }
 }
 
-
 /* Pretty-print the given boolean array as a hexadecimal string.  slen should
  * include space for any null terminator. */
-void opal_btl_usnic_snprintf_bool_array(char *s, size_t slen, bool a[],
-                                        size_t alen)
+void opal_btl_usnic_snprintf_bool_array(char *s, size_t slen, bool a[], size_t alen)
 {
     size_t i = 0;
     size_t j = 0;
@@ -172,10 +151,10 @@ void opal_btl_usnic_snprintf_bool_array(char *s, size_t slen, bool a[],
         unsigned char tmp = 0;
 
         /* first bool is the leftmost (most significant) bit of the nybble */
-        tmp |= !!a[i+0] << 3;
-        tmp |= !!a[i+1] << 2;
-        tmp |= !!a[i+2] << 1;
-        tmp |= !!a[i+3] << 0;
+        tmp |= !!a[i + 0] << 3;
+        tmp |= !!a[i + 1] << 2;
+        tmp |= !!a[i + 2] << 1;
+        tmp |= !!a[i + 3] << 0;
         tmp += '0';
         s[j] = tmp;
 
@@ -195,9 +174,7 @@ void opal_btl_usnic_snprintf_bool_array(char *s, size_t slen, bool a[],
  *
  * This routine internally clones the convertor and does not mutate it!
  */
-size_t opal_btl_usnic_convertor_pack_peek(
-    const opal_convertor_t *conv,
-    size_t max_len)
+size_t opal_btl_usnic_convertor_pack_peek(const opal_convertor_t *conv, size_t max_len)
 {
     int rc;
     size_t packable_len, position;
diff --git a/opal/mca/btl/usnic/btl_usnic_util.h b/opal/mca/btl/usnic/btl_usnic_util.h
index f3552e0b33b..3cd2f7584c5 100644
--- a/opal/mca/btl/usnic/btl_usnic_util.h
+++ b/opal/mca/btl/usnic/btl_usnic_util.h
@@ -15,7 +15,7 @@
 #include "btl_usnic.h"
 
 #ifndef MIN
-#  define MIN(a,b)                ((a) < (b) ? (a) : (b))
+#    define MIN(a, b) ((a) < (b) ? (a) : (b))
 #endif
 
 /*
@@ -39,20 +39,17 @@ void opal_btl_usnic_util_abort(const char *msg, const char *file, int line);
  * If netmask==0, it is not included in the output string.  addr is
  * expected to be in network byte order.
  */
-void opal_btl_usnic_snprintf_ipv4_addr(char *out, size_t maxlen,
-                                       uint32_t addr_be, uint32_t netmask_be);
+void opal_btl_usnic_snprintf_ipv4_addr(char *out, size_t maxlen, uint32_t addr_be,
+                                       uint32_t netmask_be);
 
 void opal_btl_usnic_snprintf_bool_array(char *s, size_t slen, bool a[], size_t alen);
 
-void opal_btl_usnic_dump_hex(int verbose_level, int output_id,
-                             void *vaddr, int len);
+void opal_btl_usnic_dump_hex(int verbose_level, int output_id, void *vaddr, int len);
 
-size_t opal_btl_usnic_convertor_pack_peek(const opal_convertor_t *conv,
-                                          size_t max_len);
+size_t opal_btl_usnic_convertor_pack_peek(const opal_convertor_t *conv, size_t max_len);
 
 /* avoid "defined but not used" warnings */
-static inline int __opal_attribute_always_inline__ usnic_fls(int x)
-    __opal_attribute_unused__;
+static inline int __opal_attribute_always_inline__ usnic_fls(int x) __opal_attribute_unused__;
 
 static inline int __opal_attribute_always_inline__ usnic_fls(int x)
 {
@@ -84,18 +81,14 @@ static inline int __opal_attribute_always_inline__ usnic_fls(int x)
 }
 
 /* a helper function that just declutters convertor packing */
-static inline void
-usnic_convertor_pack_simple(
-    opal_convertor_t *convertor,
-    void *dest,
-    size_t max_bytes_to_pack,
-    size_t *bytes_packed)
+static inline void usnic_convertor_pack_simple(opal_convertor_t *convertor, void *dest,
+                                               size_t max_bytes_to_pack, size_t *bytes_packed)
 {
     int rc;
     struct iovec iov;
     uint32_t iov_count;
 
-    iov.iov_base = (IOVBASE_TYPE*)dest;
+    iov.iov_base = (IOVBASE_TYPE *) dest;
     iov.iov_len = max_bytes_to_pack;
     iov_count = 1;
     *bytes_packed = max_bytes_to_pack;
@@ -105,16 +98,12 @@ usnic_convertor_pack_simple(
     }
 }
 
-static inline int
-usnic_netmask_to_cidrlen(
-    uint32_t netmask_be)
+static inline int usnic_netmask_to_cidrlen(uint32_t netmask_be)
 {
     return 33 - ffs(ntohl(netmask_be));
 }
 
-static inline uint32_t
-usnic_cidrlen_to_netmask(
-    int cidrlen)
+static inline uint32_t usnic_cidrlen_to_netmask(int cidrlen)
 {
     uint32_t mask;
 
diff --git a/opal/mca/btl/usnic/configure.m4 b/opal/mca/btl/usnic/configure.m4
index 0345baced6b..3f271f47691 100644
--- a/opal/mca/btl/usnic/configure.m4
+++ b/opal/mca/btl/usnic/configure.m4
@@ -15,6 +15,7 @@
 # Copyright (c) 2010-2020 Cisco Systems, Inc.  All rights reserved
 # Copyright (c) 2017      Los Alamos National Security, LLC.  All rights
 #                         reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -97,7 +98,7 @@ AC_DEFUN([_OPAL_BTL_USNIC_DO_CONFIG],[
 
     AS_IF([test "$opal_btl_usnic_happy" = "yes"],
           [ # The usnic BTL requires OFI libfabric support
-           OPAL_CHECK_OFI
+           OPAL_CHECK_OFI([btl_usnic])
            opal_btl_usnic_happy=$opal_ofi_happy])
 
     # The usnic BTL requires at least OFI libfabric v1.3.
@@ -109,13 +110,17 @@ AC_DEFUN([_OPAL_BTL_USNIC_DO_CONFIG],[
     # Make sure we can find the OFI libfabric usnic extensions header
     AS_IF([test "$opal_btl_usnic_happy" = "yes" ],
           [opal_btl_usnic_CPPFLAGS_save=$CPPFLAGS
-           CPPFLAGS="$opal_ofi_CPPFLAGS $CPPFLAGS"
+           CPPFLAGS="$btl_usnic_CPPFLAGS $CPPFLAGS"
            AC_CHECK_HEADER([rdma/fi_ext_usnic.h],
                             [],
                             [opal_btl_usnic_happy=no])
            CPPFLAGS=$opal_btl_usnic_CPPFLAGS_save
           ])
 
+    AC_SUBST([btl_usnic_CPPFLAGS])
+    AC_SUBST([btl_usnic_LDFLAGS])
+    AC_SUBST([btl_usnic_LIBS])
+
     # All done
     AS_IF([test "$opal_btl_usnic_happy" = "yes"],
           [$1],
@@ -125,6 +130,6 @@ AC_DEFUN([_OPAL_BTL_USNIC_DO_CONFIG],[
                  [$2])
           ])
 
-    OPAL_SUMMARY_ADD([[Transports]],[[Cisco usNIC]],[[btl_usnic]],[$opal_btl_usnic_happy])
+    OPAL_SUMMARY_ADD([Transports], [Cisco usNIC], [], [$opal_btl_usnic_happy])
     OPAL_VAR_SCOPE_POP
 ])dnl
diff --git a/opal/mca/btl/usnic/help-mpi-btl-usnic.txt b/opal/mca/btl/usnic/help-mpi-btl-usnic.txt
index ea3fc11a0e1..74c75f1ad0c 100644
--- a/opal/mca/btl/usnic/help-mpi-btl-usnic.txt
+++ b/opal/mca/btl/usnic/help-mpi-btl-usnic.txt
@@ -1,6 +1,6 @@
 # -*- text -*-
 #
-# Copyright (c) 2012-2020 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2012-2022 Cisco Systems, Inc.  All rights reserved.
 #
 # $COPYRIGHT$
 #
@@ -104,16 +104,6 @@ value will be ignored.
   Value:      %s
   Message:    %s
 #
-[device present but not up]
-Open MPI has found a usNIC device that is present / listed in Linux,
-but in a "down" state.  It will not be used by this MPI job.
-
-You may wish to check this device, especially if it is unexpectedly
-down.
-
-  Local server: %s
-  Device name:  %s
-#
 [MTU mismatch]
 The MTU does not match on local and remote hosts.  All interfaces on
 all hosts participating in an MPI job must be configured with the same
@@ -224,22 +214,6 @@ Check the resulting "mymap*" files to see the exact pairing of IP
 interfaces.  Inconsistent results may be indicative of underlying
 network misconfigurations.
 #
-[fi_av_insert timeout]
-The usnic BTL failed to create addresses for remote peers within the
-specified timeout.  This usually means that ARP requests failed to
-resolve in time.  You may be able to solve the problem by increasing
-the usnic BTL's ARP timeout.  If that doesn't work, you should
-diagnose why ARP replies are apparently not being delivered in a
-timely manner.
-
-The usNIC interface listed below will be ignored.  Your MPI
-application will likely either run with degraded performance and/or
-abort.
-
-  Server:              %s
-  usNIC interface:     %s
-  Current ARP timeout: %d (btl_usnic_arp_timeout MCA param)
-#
 [fi_av_eq too small]
 The usnic BTL was told to create an address resolution queue that was
 too small via the mca_btl_usnic_av_eq_num MCA parameter.  This
@@ -283,56 +257,6 @@ connectivity map file will not be written.
   Working directory: %s
   Error:             %s (%d)
 #
-[received too many short packets]
-WARNING: The usnic BTL received a significant number of abnormally
-short packets on a single network interface.  This may be due to
-corruption or congestion in the network fabric.  It may be useful to
-run a physical/layer 0 diagnostic.
-
-Your job will continue, but if this poor network behavior continues,
-you may experience lower-than-expected performance due to overheads
-caused by higher-than-usual retransmission rates (to compensate for
-the corrupted received packets).
-
-  Local server:       %s
-  usNIC interface:    %s
-  # of short packets
-     received so far: %d
-
-You will only receive this warning once per MPI process per job.
-
-If you know that your network environment is lossy/heavily congested
-such that short/corrupted packets are expected, you can disable this
-warning by setting the btl_usnic_max_short_packets MCA parameter to 0.
-#
-[non-receive completion error]
-WARNING: The usnic BTL has detected an error in the completion of a
-non-receive event.  This is highly unusual, and may indicate an error
-in the usNIC subsystem on this server.
-
-Your MPI job will continue, but you should monitor the job and ensure
-that it behaves correctly.
-
-  Local server:       %s
-  usNIC interface:    %s
-  Channel index:      %d
-  Completion status:  %s (%d)
-  Work request ID:    %p
-  Opcode:             %s (%d)
-
-If this error keeps happening, you should contact Cisco technical
-support.
-#
-[device present but not up]
-Open MPI has found a usNIC device that is present / listed in Linux,
-but in a "down" state.  It will not be used by this MPI job.
-
-You may wish to check this device, especially if it is unexpectedly
-down.
-
-  Local server: %s
-  Device name:  %s
-#
 [transport mismatch]
 Open MPI has found two servers with different underlying usNIC
 transports.  This is an unsupported configuration; all usNIC devices
diff --git a/opal/mca/btl/usnic/test/usnic_btl_run_tests.c b/opal/mca/btl/usnic/test/usnic_btl_run_tests.c
index 5b79e85fb78..73b9153c434 100644
--- a/opal/mca/btl/usnic/test/usnic_btl_run_tests.c
+++ b/opal/mca/btl/usnic/test/usnic_btl_run_tests.c
@@ -15,10 +15,10 @@
 #define _GNU_SOURCE
 
 #include <assert.h>
+#include <libgen.h> /* for dirname() */
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <libgen.h> /* for dirname() */
 
 #include <dlfcn.h>
 
@@ -39,22 +39,22 @@ int main(int argc, char **argv)
     char *to;
     int path_len;
 
-    mpi_handle = dlopen("lib" OMPI_LIBMPI_NAME ".so", RTLD_NOW|RTLD_GLOBAL);
+    mpi_handle = dlopen("lib" OMPI_LIBMPI_NAME ".so", RTLD_NOW | RTLD_GLOBAL);
     if (mpi_handle == NULL) {
         fprintf(stderr, "mpi_handle=NULL dlerror()=%s\n", dlerror());
         abort();
     }
 
     /* casting awfulness needed for GCC's "-pedantic" option :( */
-    *(void **)(&init) = dlsym(mpi_handle, "MPI_Init");
+    *(void **) (&init) = dlsym(mpi_handle, "MPI_Init");
     if (init == NULL) {
         fprintf(stderr, "init=NULL dlerror()=%s\n", dlerror());
         abort();
     }
     /* casting awfulness needed for GCC's "-pedantic" option :( */
-    *(void **)(&finalize) = dlsym(mpi_handle, "MPI_Finalize");
+    *(void **) (&finalize) = dlsym(mpi_handle, "MPI_Finalize");
     if (finalize == NULL) {
-        fprintf(stderr, "finalize=%p dlerror()=%s\n", *(void **)(&finalize), dlerror());
+        fprintf(stderr, "finalize=%p dlerror()=%s\n", *(void **) (&finalize), dlerror());
         abort();
     }
 
@@ -62,22 +62,22 @@ int main(int argc, char **argv)
     init(&argc, &argv);
 
     /* figure out where the usnic BTL shared object is relative to libmpi.so */
-    if (!dladdr(*(void **)(&init), &info)) {
+    if (!dladdr(*(void **) (&init), &info)) {
         fprintf(stderr, "ERROR: unable to dladdr(init,...)\n");
         abort();
     }
     libmpi_path = strdup(info.dli_fname);
     assert(libmpi_path != NULL);
     path_len = strlen(libmpi_path) + strlen("/openmpi/") + strlen(MCA_BTL_USNIC_SO);
-    path = calloc(path_len+1, 1);
+    path = calloc(path_len + 1, 1);
     to = path;
     to = stpcpy(to, dirname(libmpi_path));
     to = stpcpy(to, "/openmpi/");
     to = stpcpy(to, MCA_BTL_USNIC_SO);
 
-    usnic_handle = dlopen(path, RTLD_NOW|RTLD_LOCAL);
+    usnic_handle = dlopen(path, RTLD_NOW | RTLD_LOCAL);
     if (usnic_handle == NULL) {
-        fprintf(stderr, "usnic_handle=%p dlerror()=%s\n", (void *)usnic_handle, dlerror());
+        fprintf(stderr, "usnic_handle=%p dlerror()=%s\n", (void *) usnic_handle, dlerror());
         abort();
     }
 
@@ -85,10 +85,9 @@ int main(int argc, char **argv)
     free(path);
 
     /* casting awfulness needed for GCC's "-pedantic" option :( */
-    *(void **)(&run_tests) = dlsym(usnic_handle, BTL_USNIC_RUN_TESTS_SYMBOL);
+    *(void **) (&run_tests) = dlsym(usnic_handle, BTL_USNIC_RUN_TESTS_SYMBOL);
     if (run_tests == NULL) {
-        fprintf(stderr, "run_tests=%p dlerror()=%s\n",
-                *(void **)(&run_tests), dlerror());
+        fprintf(stderr, "run_tests=%p dlerror()=%s\n", *(void **) (&run_tests), dlerror());
         abort();
     }
     run_tests();
diff --git a/opal/mca/common/cuda/Makefile.am b/opal/mca/common/cuda/Makefile.am
deleted file mode 100644
index 38b0434c1d5..00000000000
--- a/opal/mca/common/cuda/Makefile.am
+++ /dev/null
@@ -1,117 +0,0 @@
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2013 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2011-2013 NVIDIA Corporation.  All rights reserved.
-# Copyright (c) 2014-2015 Cisco Systems, Inc.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-# A word of explanation...
-#
-# This library is linked against various MCA components because the
-# support for CUDA device pointers is needed in various places.
-# There's two cases:
-#
-# 1. libmca_common_cuda.la is a shared library.  By linking that shared
-# library to all components that need it, the OS linker will
-# automatically load it into the process as necessary, and there will
-# only be one copy (i.e., all the components will share *one* copy of
-# the code and data).
-#
-# 2. libmca_common_cuda.la is a static library.  In this case, it will
-# be rolled up into the top-level libmpi.la.  It will also be rolled
-# into each component, but then the component will also be rolled up
-# into the upper-level libmpi.la.  Linkers universally know how to
-# "figure this out" so that we end up with only one copy of the code
-# and data.
-#
-# Note that building this common component statically and linking
-# against other dynamic components is *not* supported!
-
-AM_CPPFLAGS = $(common_cuda_CPPFLAGS)
-
-# Header files
-
-headers = \
-        common_cuda.h
-
-# Source files
-
-sources = \
-        common_cuda.c
-
-dist_opaldata_DATA = help-mpi-common-cuda.txt
-
-# As per above, we'll either have an installable or noinst result.
-# The installable one should follow the same MCA prefix naming rules
-# (i.e., libmca_<type>_<name>.la).  The noinst one can be named
-# whatever it wants, although libmca_<type>_<name>_noinst.la is
-# recommended.
-
-# To simplify components that link to this library, we will *always*
-# have an output libtool library named libmca_<type>_<name>.la -- even
-# for case 2) described above (i.e., so there's no conditional logic
-# necessary in component Makefile.am's that link to this library).
-# Hence, if we're creating a noinst version of this library (i.e.,
-# case 2), we sym link it to the libmca_<type>_<name>.la name
-# (libtool will do the Right Things under the covers).  See the
-# all-local and clean-local rules, below, for how this is effected.
-
-lib_LTLIBRARIES =
-noinst_LTLIBRARIES =
-comp_inst = lib@OPAL_LIB_PREFIX@mca_common_cuda.la
-comp_noinst = lib@OPAL_LIB_PREFIX@mca_common_cuda_noinst.la
-
-if MCA_BUILD_opal_common_cuda_DSO
-lib_LTLIBRARIES += $(comp_inst)
-else
-noinst_LTLIBRARIES += $(comp_noinst)
-endif
-
-lib@OPAL_LIB_PREFIX@mca_common_cuda_la_SOURCES = $(headers) $(sources)
-lib@OPAL_LIB_PREFIX@mca_common_cuda_la_LDFLAGS = \
-        -version-info $(libmca_opal_common_cuda_so_version)
-lib@OPAL_LIB_PREFIX@mca_common_cuda_la_LIBADD = $(common_cuda_LIBS)
-
-lib@OPAL_LIB_PREFIX@mca_common_cuda_noinst_la_SOURCES = $(headers) $(sources)
-lib@OPAL_LIB_PREFIX@mca_common_cuda_noinst_la_LDFLAGS = $(common_cuda_LDFLAGS)
-lib@OPAL_LIB_PREFIX@mca_common_cuda_noinst_la_LIBADD = $(common_cuda_LIBS)
-
-# Conditionally install the header files
-
-if WANT_INSTALL_HEADERS
-opaldir = $(opalincludedir)/$(subdir)
-opal_HEADERS = $(headers)
-endif
-
-# These two rules will sym link the "noinst" libtool library filename
-# to the installable libtool library filename in the case where we are
-# compiling this component statically (case 2), described above).
-
-V=0
-OMPI_V_LN_SCOMP = $(ompi__v_LN_SCOMP_$V)
-ompi__v_LN_SCOMP_ = $(ompi__v_LN_SCOMP_$AM_DEFAULT_VERBOSITY)
-ompi__v_LN_SCOMP_0 = @echo "  LN_S    " `basename $(comp_inst)`;
-
-all-local:
-	$(OMPI_V_LN_SCOMP) if test -z "$(lib_LTLIBRARIES)"; then \
-	  rm -f "$(comp_inst)"; \
-	  $(LN_S) "$(comp_noinst)" "$(comp_inst)"; \
-	fi
-
-clean-local:
-	if test -z "$(lib_LTLIBRARIES)"; then \
-	  rm -f "$(comp_inst)"; \
-	fi
diff --git a/opal/mca/common/cuda/common_cuda.c b/opal/mca/common/cuda/common_cuda.c
deleted file mode 100644
index 81e6a134486..00000000000
--- a/opal/mca/common/cuda/common_cuda.c
+++ /dev/null
@@ -1,2356 +0,0 @@
-/*
- * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2014 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2006 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2011-2015 NVIDIA Corporation.  All rights reserved.
- * Copyright (c) 2015      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-/**
- * This file contains various support functions for doing CUDA
- * operations.
- */
-#include "opal_config.h"
-
-#include <errno.h>
-#include <unistd.h>
-#include <cuda.h>
-
-#include "opal/align.h"
-#include "opal/datatype/opal_convertor.h"
-#include "opal/util/output.h"
-#include "opal/util/show_help.h"
-#include "opal/util/proc.h"
-#include "opal/util/argv.h"
-#include "opal/util/printf.h"
-
-#include "opal/mca/rcache/base/base.h"
-#include "opal/runtime/opal_params.h"
-#include "opal/mca/timer/base/base.h"
-#include "opal/mca/dl/base/base.h"
-
-#include "common_cuda.h"
-
-/**
- * Since function names can get redefined in cuda.h file, we need to do this
- * stringifying to get the latest function name from the header file.  For
- * example, cuda.h may have something like this:
- * #define cuMemFree cuMemFree_v2
- * We want to make sure we find cuMemFree_v2, not cuMemFree.
- */
-#define STRINGIFY2(x) #x
-#define STRINGIFY(x) STRINGIFY2(x)
-
-#define OPAL_CUDA_DLSYM(libhandle, funcName)                                         \
-do {                                                                                 \
- char *err_msg;                                                                      \
- void *ptr;                                                                          \
- if (OPAL_SUCCESS !=                                                                 \
-     opal_dl_lookup(libhandle, STRINGIFY(funcName), &ptr, &err_msg)) {               \
-        opal_show_help("help-mpi-common-cuda.txt", "dlsym failed", true,             \
-                       STRINGIFY(funcName), err_msg);                                \
-        return 1;                                                                    \
-    } else {                                                                         \
-        *(void **)(&cuFunc.funcName) = ptr;                                          \
-        opal_output_verbose(15, mca_common_cuda_output,                              \
-                            "CUDA: successful dlsym of %s",                          \
-                            STRINGIFY(funcName));                                    \
-    }                                                                                \
-} while (0)
-
-/* Structure to hold CUDA function pointers that get dynamically loaded. */
-struct cudaFunctionTable {
-    int (*cuPointerGetAttribute)(void *, CUpointer_attribute, CUdeviceptr);
-    int (*cuMemcpyAsync)(CUdeviceptr, CUdeviceptr, size_t, CUstream);
-    int (*cuMemcpy)(CUdeviceptr, CUdeviceptr, size_t);
-    int (*cuMemAlloc)(CUdeviceptr *, size_t);
-    int (*cuMemFree)(CUdeviceptr buf);
-    int (*cuCtxGetCurrent)(void *cuContext);
-    int (*cuStreamCreate)(CUstream *, int);
-    int (*cuEventCreate)(CUevent *, int);
-    int (*cuEventRecord)(CUevent, CUstream);
-    int (*cuMemHostRegister)(void *, size_t, unsigned int);
-    int (*cuMemHostUnregister)(void *);
-    int (*cuEventQuery)(CUevent);
-    int (*cuEventDestroy)(CUevent);
-    int (*cuStreamWaitEvent)(CUstream, CUevent, unsigned int);
-    int (*cuMemGetAddressRange)(CUdeviceptr*, size_t*, CUdeviceptr);
-    int (*cuIpcGetEventHandle)(CUipcEventHandle*, CUevent);
-    int (*cuIpcOpenEventHandle)(CUevent*, CUipcEventHandle);
-    int (*cuIpcOpenMemHandle)(CUdeviceptr*, CUipcMemHandle, unsigned int);
-    int (*cuIpcCloseMemHandle)(CUdeviceptr);
-    int (*cuIpcGetMemHandle)(CUipcMemHandle*, CUdeviceptr);
-    int (*cuCtxGetDevice)(CUdevice *);
-    int (*cuDeviceCanAccessPeer)(int *, CUdevice, CUdevice);
-    int (*cuDeviceGet)(CUdevice *, int);
-#if OPAL_CUDA_GDR_SUPPORT
-    int (*cuPointerSetAttribute)(const void *, CUpointer_attribute, CUdeviceptr);
-#endif /* OPAL_CUDA_GDR_SUPPORT */
-    int (*cuCtxSetCurrent)(CUcontext);
-    int (*cuEventSynchronize)(CUevent);
-    int (*cuStreamSynchronize)(CUstream);
-    int (*cuStreamDestroy)(CUstream);
-#if OPAL_CUDA_GET_ATTRIBUTES
-    int (*cuPointerGetAttributes)(unsigned int, CUpointer_attribute *, void **, CUdeviceptr);
-#endif /* OPAL_CUDA_GET_ATTRIBUTES */
-};
-typedef struct cudaFunctionTable cudaFunctionTable_t;
-static cudaFunctionTable_t cuFunc;
-
-static int stage_one_init_ref_count = 0;
-static bool stage_three_init_complete = false;
-static bool common_cuda_initialized = false;
-static bool common_cuda_mca_parames_registered = false;
-static int mca_common_cuda_verbose;
-static int mca_common_cuda_output = 0;
-bool mca_common_cuda_enabled = false;
-static bool mca_common_cuda_register_memory = true;
-static bool mca_common_cuda_warning = false;
-static opal_list_t common_cuda_memory_registrations;
-static CUstream ipcStream = NULL;
-static CUstream dtohStream = NULL;
-static CUstream htodStream = NULL;
-static CUstream memcpyStream = NULL;
-static int mca_common_cuda_gpu_mem_check_workaround = (CUDA_VERSION > 7000) ? 0 : 1;
-static opal_mutex_t common_cuda_init_lock;
-static opal_mutex_t common_cuda_htod_lock;
-static opal_mutex_t common_cuda_dtoh_lock;
-static opal_mutex_t common_cuda_ipc_lock;
-
-/* Functions called by opal layer - plugged into opal function table */
-static int mca_common_cuda_is_gpu_buffer(const void*, opal_convertor_t*);
-static int mca_common_cuda_memmove(void*, void*, size_t);
-static int mca_common_cuda_cu_memcpy_async(void*, const void*, size_t, opal_convertor_t*);
-static int mca_common_cuda_cu_memcpy(void*, const void*, size_t);
-
-/* Function that gets plugged into opal layer */
-static int mca_common_cuda_stage_two_init(opal_common_cuda_function_table_t *);
-
-/* Structure to hold memory registrations that are delayed until first
- * call to send or receive a GPU pointer */
-struct common_cuda_mem_regs_t {
-    opal_list_item_t super;
-    void *ptr;
-    size_t amount;
-    char *msg;
-};
-typedef struct common_cuda_mem_regs_t common_cuda_mem_regs_t;
-OBJ_CLASS_DECLARATION(common_cuda_mem_regs_t);
-OBJ_CLASS_INSTANCE(common_cuda_mem_regs_t,
-                   opal_list_item_t,
-                   NULL,
-                   NULL);
-
-static int mca_common_cuda_async = 1;
-static int mca_common_cuda_cumemcpy_async;
-#if OPAL_ENABLE_DEBUG
-static int mca_common_cuda_cumemcpy_timing;
-#endif /* OPAL_ENABLE_DEBUG */
-
-/* Array of CUDA events to be queried for IPC stream, sending side and
- * receiving side. */
-CUevent *cuda_event_ipc_array = NULL;
-CUevent *cuda_event_dtoh_array = NULL;
-CUevent *cuda_event_htod_array = NULL;
-
-/* Array of fragments currently being moved by cuda async non-blocking
- * operations */
-struct mca_btl_base_descriptor_t **cuda_event_ipc_frag_array = NULL;
-struct mca_btl_base_descriptor_t **cuda_event_dtoh_frag_array = NULL;
-struct mca_btl_base_descriptor_t **cuda_event_htod_frag_array = NULL;
-
-/* First free/available location in cuda_event_status_array */
-static int cuda_event_ipc_first_avail, cuda_event_dtoh_first_avail, cuda_event_htod_first_avail;
-
-/* First currently-being used location in the cuda_event_status_array */
-static int cuda_event_ipc_first_used, cuda_event_dtoh_first_used, cuda_event_htod_first_used;
-
-/* Number of status items currently in use */
-static volatile int cuda_event_ipc_num_used, cuda_event_dtoh_num_used, cuda_event_htod_num_used;
-
-/* Size of array holding events */
-int cuda_event_max = 400;
-static int cuda_event_ipc_most = 0;
-static int cuda_event_dtoh_most = 0;
-static int cuda_event_htod_most = 0;
-
-/* Handle to libcuda.so */
-opal_dl_handle_t *libcuda_handle = NULL;
-
-/* Unused variable that we register at init time and unregister at fini time.
- * This is used to detect if user has done a device reset prior to MPI_Finalize.
- * This is a workaround to avoid SEGVs.
- */
-static int checkmem;
-static int ctx_ok = 1;
-
-#define CUDA_COMMON_TIMING 0
-#if OPAL_ENABLE_DEBUG
-/* Some timing support structures.  Enable this to help analyze
- * internal performance issues. */
-static opal_timer_t ts_start;
-static opal_timer_t ts_end;
-static double accum;
-#define THOUSAND  1000L
-#define MILLION   1000000L
-static float mydifftime(opal_timer_t ts_start, opal_timer_t ts_end);
-#endif /* OPAL_ENABLE_DEBUG */
-
-/* These functions are typically unused in the optimized builds. */
-static void cuda_dump_evthandle(int, void *, char *) __opal_attribute_unused__ ;
-static void cuda_dump_memhandle(int, void *, char *) __opal_attribute_unused__ ;
-#if OPAL_ENABLE_DEBUG
-#define CUDA_DUMP_MEMHANDLE(a) cuda_dump_memhandle a
-#define CUDA_DUMP_EVTHANDLE(a) cuda_dump_evthandle a
-#else
-#define CUDA_DUMP_MEMHANDLE(a)
-#define CUDA_DUMP_EVTHANDLE(a)
-#endif /* OPAL_ENABLE_DEBUG */
-
-/* This is a seperate function so we can see these variables with ompi_info and
- * also set them with the tools interface */
-void mca_common_cuda_register_mca_variables(void)
-{
-
-    if (false == common_cuda_mca_parames_registered) {
-        common_cuda_mca_parames_registered = true;
-    }
-    /* Set different levels of verbosity in the cuda related code. */
-    mca_common_cuda_verbose = 0;
-    (void) mca_base_var_register("ompi", "mpi", "common_cuda", "verbose",
-                                 "Set level of common cuda verbosity",
-                                 MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                 OPAL_INFO_LVL_9,
-                                 MCA_BASE_VAR_SCOPE_READONLY,
-                                 &mca_common_cuda_verbose);
-
-    /* Control whether system buffers get CUDA pinned or not.  Allows for
-     * performance analysis. */
-    mca_common_cuda_register_memory = true;
-    (void) mca_base_var_register("ompi", "mpi", "common_cuda", "register_memory",
-                                 "Whether to cuMemHostRegister preallocated BTL buffers",
-                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
-                                 OPAL_INFO_LVL_9,
-                                 MCA_BASE_VAR_SCOPE_READONLY,
-                                 &mca_common_cuda_register_memory);
-
-    /* Control whether we see warnings when CUDA memory registration fails.  This is
-     * useful when CUDA support is configured in, but we are running a regular MPI
-     * application without CUDA. */
-    mca_common_cuda_warning = true;
-    (void) mca_base_var_register("ompi", "mpi", "common_cuda", "warning",
-                                 "Whether to print warnings when CUDA registration fails",
-                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
-                                 OPAL_INFO_LVL_9,
-                                 MCA_BASE_VAR_SCOPE_READONLY,
-                                 &mca_common_cuda_warning);
-
-    /* Use this flag to test async vs sync copies */
-    mca_common_cuda_async = 1;
-    (void) mca_base_var_register("ompi", "mpi", "common_cuda", "memcpy_async",
-                                 "Set to 0 to force CUDA sync copy instead of async",
-                                 MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                 OPAL_INFO_LVL_9,
-                                 MCA_BASE_VAR_SCOPE_READONLY,
-                                 &mca_common_cuda_async);
-
-    /* Use this parameter to increase the number of outstanding events allows */
-    (void) mca_base_var_register("ompi", "mpi", "common_cuda", "event_max",
-                                 "Set number of oustanding CUDA events",
-                                 MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                 OPAL_INFO_LVL_9,
-                                 MCA_BASE_VAR_SCOPE_READONLY,
-                                 &cuda_event_max);
-
-    /* Use this flag to test cuMemcpyAsync vs cuMemcpy */
-    mca_common_cuda_cumemcpy_async = 1;
-    (void) mca_base_var_register("ompi", "mpi", "common_cuda", "cumemcpy_async",
-                                 "Set to 0 to force CUDA cuMemcpy instead of cuMemcpyAsync/cuStreamSynchronize",
-                                 MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                 OPAL_INFO_LVL_5,
-                                 MCA_BASE_VAR_SCOPE_READONLY,
-                                 &mca_common_cuda_cumemcpy_async);
-
-#if OPAL_ENABLE_DEBUG
-    /* Use this flag to dump out timing of cumempcy sync and async */
-    mca_common_cuda_cumemcpy_timing = 0;
-    (void) mca_base_var_register("ompi", "mpi", "common_cuda", "cumemcpy_timing",
-                                 "Set to 1 to dump timing of eager copies",
-                                 MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                 OPAL_INFO_LVL_5,
-                                 MCA_BASE_VAR_SCOPE_READONLY,
-                                 &mca_common_cuda_cumemcpy_timing);
-#endif /* OPAL_ENABLE_DEBUG */
-
-    (void) mca_base_var_register("ompi", "mpi", "common_cuda", "gpu_mem_check_workaround",
-                                 "Set to 0 to disable GPU memory check workaround. A user would rarely have to do this.",
-                                 MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                 OPAL_INFO_LVL_9,
-                                 MCA_BASE_VAR_SCOPE_READONLY,
-                                 &mca_common_cuda_gpu_mem_check_workaround);
-}
-
-/**
- * This is the first stage of initialization.  This function is called
- * explicitly by any BTLs that can support CUDA-aware. It is called during
- * the component open phase of initialization. This fuction will look for
- * the SONAME of the library which is libcuda.so.1. In most cases, this will
- * result in the library found.  However, there are some setups that require
- * the extra steps for searching. This function will then load the symbols
- * needed from the CUDA driver library. Any failure will result in this
- * initialization failing and status will be set showing that.
- */
-int mca_common_cuda_stage_one_init(void)
-{
-    int retval, i, j;
-    char *cudalibs[] = {"libcuda.so.1", "libcuda.dylib", NULL};
-    char *searchpaths[] = {"", "/usr/lib64", NULL};
-    char **errmsgs = NULL;
-    char *errmsg = NULL;
-    int errsize;
-    bool stage_one_init_passed = false;
-
-    stage_one_init_ref_count++;
-    if (stage_one_init_ref_count > 1) {
-        opal_output_verbose(10, mca_common_cuda_output,
-                            "CUDA: stage_one_init_ref_count is now %d, no need to init",
-                            stage_one_init_ref_count);
-        return OPAL_SUCCESS;
-    }
-
-    /* This is a no-op in most cases as the parameters were registered earlier */
-    mca_common_cuda_register_mca_variables();
-
-    OBJ_CONSTRUCT(&common_cuda_init_lock, opal_mutex_t);
-    OBJ_CONSTRUCT(&common_cuda_htod_lock, opal_mutex_t);
-    OBJ_CONSTRUCT(&common_cuda_dtoh_lock, opal_mutex_t);
-    OBJ_CONSTRUCT(&common_cuda_ipc_lock, opal_mutex_t);
-
-    mca_common_cuda_output = opal_output_open(NULL);
-    opal_output_set_verbosity(mca_common_cuda_output, mca_common_cuda_verbose);
-
-    opal_output_verbose(10, mca_common_cuda_output,
-                        "CUDA: stage_one_init_ref_count is now %d, initializing",
-                        stage_one_init_ref_count);
-
-    /* First check if the support is enabled.  In the case that the user has
-     * turned it off, we do not need to continue with any CUDA specific
-     * initialization.  Do this after MCA parameter registration. */
-    if (!opal_cuda_support) {
-        return 1;
-    }
-
-    if (!OPAL_HAVE_DL_SUPPORT) {
-        opal_show_help("help-mpi-common-cuda.txt", "dlopen disabled", true);
-        return 1;
-    }
-
-    /* Now walk through all the potential names libcuda and find one
-     * that works.  If it does, all is good.  If not, print out all
-     * the messages about why things failed.  This code was careful
-     * to try and save away all error messages if the loading ultimately
-     * failed to help with debugging.
-     *
-     * NOTE: On the first loop we just utilize the default loading
-     * paths from the system.  For the second loop, set /usr/lib64 to
-     * the search path and try again.  This is done to handle the case
-     * where we have both 32 and 64 bit libcuda.so libraries
-     * installed.  Even when running in 64-bit mode, the /usr/lib
-     * directory is searched first and we may find a 32-bit
-     * libcuda.so.1 library.  Loading of this library will fail as the
-     * OPAL DL framework does not handle having the wrong ABI in the
-     * search path (unlike ld or ld.so).  Note that we only set this
-     * search path after the original search.  This is so that
-     * LD_LIBRARY_PATH and run path settings are respected.  Setting
-     * this search path overrides them (rather then being
-     * appended). */
-    j = 0;
-    while (searchpaths[j] != NULL) {
-        i = 0;
-        while (cudalibs[i] != NULL) {
-            char *filename = NULL;
-            char *str = NULL;
-
-            /* If there's a non-empty search path, prepend it
-               to the library filename */
-            if (strlen(searchpaths[j]) > 0) {
-                opal_asprintf(&filename, "%s/%s", searchpaths[j], cudalibs[i]);
-            } else {
-                filename = strdup(cudalibs[i]);
-            }
-            if (NULL == filename) {
-                opal_show_help("help-mpi-common-cuda.txt", "No memory",
-                               true, OPAL_PROC_MY_HOSTNAME);
-                return 1;
-            }
-
-            retval = opal_dl_open(filename, false, false,
-                                  &libcuda_handle, &str);
-            if (OPAL_SUCCESS != retval || NULL == libcuda_handle) {
-                if (NULL != str) {
-                    opal_argv_append(&errsize, &errmsgs, str);
-                } else {
-                    opal_argv_append(&errsize, &errmsgs,
-                                     "opal_dl_open() returned NULL.");
-                }
-                opal_output_verbose(10, mca_common_cuda_output,
-                                    "CUDA: Library open error: %s",
-                                    errmsgs[errsize-1]);
-            } else {
-                opal_output_verbose(10, mca_common_cuda_output,
-                                    "CUDA: Library successfully opened %s",
-                                    cudalibs[i]);
-                stage_one_init_passed = true;
-                break;
-            }
-            i++;
-
-            free(filename);
-        }
-        if (true == stage_one_init_passed) {
-            break; /* Break out of outer loop */
-        }
-        j++;
-    }
-
-    if (true != stage_one_init_passed) {
-        errmsg = opal_argv_join(errmsgs, '\n');
-        if (opal_warn_on_missing_libcuda) {
-            opal_show_help("help-mpi-common-cuda.txt", "dlopen failed", true,
-                           errmsg);
-        }
-        opal_cuda_support = 0;
-    }
-    opal_argv_free(errmsgs);
-    free(errmsg);
-
-    if (true != stage_one_init_passed) {
-        return 1;
-    }
-    opal_cuda_add_initialization_function(&mca_common_cuda_stage_two_init);
-    OBJ_CONSTRUCT(&common_cuda_memory_registrations, opal_list_t);
-
-    /* Map in the functions that we need.  Note that if there is an error
-     * the macro OPAL_CUDA_DLSYM will print an error and call return.  */
-    OPAL_CUDA_DLSYM(libcuda_handle, cuStreamCreate);
-    OPAL_CUDA_DLSYM(libcuda_handle, cuCtxGetCurrent);
-    OPAL_CUDA_DLSYM(libcuda_handle, cuEventCreate);
-    OPAL_CUDA_DLSYM(libcuda_handle, cuEventRecord);
-    OPAL_CUDA_DLSYM(libcuda_handle, cuMemHostRegister);
-    OPAL_CUDA_DLSYM(libcuda_handle, cuMemHostUnregister);
-    OPAL_CUDA_DLSYM(libcuda_handle, cuPointerGetAttribute);
-    OPAL_CUDA_DLSYM(libcuda_handle, cuEventQuery);
-    OPAL_CUDA_DLSYM(libcuda_handle, cuEventDestroy);
-    OPAL_CUDA_DLSYM(libcuda_handle, cuStreamWaitEvent);
-    OPAL_CUDA_DLSYM(libcuda_handle, cuMemcpyAsync);
-    OPAL_CUDA_DLSYM(libcuda_handle, cuMemcpy);
-    OPAL_CUDA_DLSYM(libcuda_handle, cuMemFree);
-    OPAL_CUDA_DLSYM(libcuda_handle, cuMemAlloc);
-    OPAL_CUDA_DLSYM(libcuda_handle, cuMemGetAddressRange);
-    OPAL_CUDA_DLSYM(libcuda_handle, cuIpcGetEventHandle);
-    OPAL_CUDA_DLSYM(libcuda_handle, cuIpcOpenEventHandle);
-    OPAL_CUDA_DLSYM(libcuda_handle, cuIpcOpenMemHandle);
-    OPAL_CUDA_DLSYM(libcuda_handle, cuIpcCloseMemHandle);
-    OPAL_CUDA_DLSYM(libcuda_handle, cuIpcGetMemHandle);
-    OPAL_CUDA_DLSYM(libcuda_handle, cuCtxGetDevice);
-    OPAL_CUDA_DLSYM(libcuda_handle, cuDeviceCanAccessPeer);
-    OPAL_CUDA_DLSYM(libcuda_handle, cuDeviceGet);
-#if OPAL_CUDA_GDR_SUPPORT
-    OPAL_CUDA_DLSYM(libcuda_handle, cuPointerSetAttribute);
-#endif /* OPAL_CUDA_GDR_SUPPORT */
-    OPAL_CUDA_DLSYM(libcuda_handle, cuCtxSetCurrent);
-    OPAL_CUDA_DLSYM(libcuda_handle, cuEventSynchronize);
-    OPAL_CUDA_DLSYM(libcuda_handle, cuStreamSynchronize);
-    OPAL_CUDA_DLSYM(libcuda_handle, cuStreamDestroy);
-#if OPAL_CUDA_GET_ATTRIBUTES
-    OPAL_CUDA_DLSYM(libcuda_handle, cuPointerGetAttributes);
-#endif /* OPAL_CUDA_GET_ATTRIBUTES */
-    return 0;
-}
-
-/**
- * This function is registered with the OPAL CUDA support.  In that way,
- * these function pointers will be loaded into the OPAL CUDA code when
- * the first convertor is initialized.  This does not trigger any CUDA
- * specific initialization as this may just be a host buffer that is
- * triggering this call.
- */
-static int mca_common_cuda_stage_two_init(opal_common_cuda_function_table_t *ftable)
-{
-    if (OPAL_UNLIKELY(!opal_cuda_support)) {
-        return OPAL_ERROR;
-    }
-
-    ftable->gpu_is_gpu_buffer = &mca_common_cuda_is_gpu_buffer;
-    ftable->gpu_cu_memcpy_async = &mca_common_cuda_cu_memcpy_async;
-    ftable->gpu_cu_memcpy = &mca_common_cuda_cu_memcpy;
-    ftable->gpu_memmove = &mca_common_cuda_memmove;
-    ftable->gpu_malloc = &mca_common_cuda_malloc;
-    ftable->gpu_free = &mca_common_cuda_free;
-
-    opal_output_verbose(30, mca_common_cuda_output,
-                        "CUDA: support functions initialized");
-    return OPAL_SUCCESS;
-}
-
-/**
- * This is the last phase of initialization.  This is triggered when we examine
- * a buffer pointer and determine it is a GPU buffer.  We then assume the user
- * has selected their GPU and we can go ahead with all the CUDA related
- * initializations.  If we get an error, just return.  Cleanup of resources
- * will happen when fini is called.
- */
-static int mca_common_cuda_stage_three_init(void)
-{
-    int i, s, rc;
-    CUresult res;
-    CUcontext cuContext;
-    common_cuda_mem_regs_t *mem_reg;
-
-    OPAL_THREAD_LOCK(&common_cuda_init_lock);
-    opal_output_verbose(20, mca_common_cuda_output,
-                        "CUDA: entering stage three init");
-
-/* Compiled without support or user disabled support */
-    if (OPAL_UNLIKELY(!opal_cuda_support)) {
-        opal_output_verbose(20, mca_common_cuda_output,
-                            "CUDA: No mpi cuda support, exiting stage three init");
-        stage_three_init_complete = true;
-        OPAL_THREAD_UNLOCK(&common_cuda_init_lock);
-        return OPAL_ERROR;
-    }
-
-    /* In case another thread snuck in and completed the initialization */
-    if (true == stage_three_init_complete) {
-        if (common_cuda_initialized) {
-            opal_output_verbose(20, mca_common_cuda_output,
-                                "CUDA: Stage three already complete, exiting stage three init");
-            OPAL_THREAD_UNLOCK(&common_cuda_init_lock);
-            return OPAL_SUCCESS;
-        } else {
-            opal_output_verbose(20, mca_common_cuda_output,
-                                "CUDA: Stage three already complete, failed during the init");
-            OPAL_THREAD_UNLOCK(&common_cuda_init_lock);
-            return OPAL_ERROR;
-        }
-    }
-
-    /* Check to see if this process is running in a CUDA context.  If
-     * so, all is good.  If not, then disable registration of memory. */
-    res = cuFunc.cuCtxGetCurrent(&cuContext);
-    if (CUDA_SUCCESS != res) {
-        if (mca_common_cuda_warning) {
-            /* Check for the not initialized error since we can make suggestions to
-             * user for this error. */
-            if (CUDA_ERROR_NOT_INITIALIZED == res) {
-                opal_show_help("help-mpi-common-cuda.txt", "cuCtxGetCurrent failed not initialized",
-                               true);
-            } else {
-                opal_show_help("help-mpi-common-cuda.txt", "cuCtxGetCurrent failed",
-                               true, res);
-            }
-        }
-        mca_common_cuda_enabled = false;
-        mca_common_cuda_register_memory = false;
-    } else if ((CUDA_SUCCESS == res) && (NULL == cuContext)) {
-        if (mca_common_cuda_warning) {
-            opal_show_help("help-mpi-common-cuda.txt", "cuCtxGetCurrent returned NULL",
-                           true);
-        }
-        mca_common_cuda_enabled = false;
-        mca_common_cuda_register_memory = false;
-    } else {
-        /* All is good.  mca_common_cuda_register_memory will retain its original
-         * value.  Normally, that is 1, but the user can override it to disable
-         * registration of the internal buffers. */
-        mca_common_cuda_enabled = true;
-        opal_output_verbose(20, mca_common_cuda_output,
-                            "CUDA: cuCtxGetCurrent succeeded");
-    }
-
-    /* No need to go on at this point.  If we cannot create a context and we are at
-     * the point where we are making MPI calls, it is time to fully disable
-     * CUDA support.
-     */
-    if (false == mca_common_cuda_enabled) {
-        OPAL_THREAD_UNLOCK(&common_cuda_init_lock);
-        return OPAL_ERROR;
-    }
-
-    if (true == mca_common_cuda_enabled) {
-        /* Set up an array to store outstanding IPC async copy events */
-        cuda_event_ipc_num_used = 0;
-        cuda_event_ipc_first_avail = 0;
-        cuda_event_ipc_first_used = 0;
-
-        cuda_event_ipc_array = (CUevent *) calloc(cuda_event_max, sizeof(CUevent *));
-        if (NULL == cuda_event_ipc_array) {
-            opal_show_help("help-mpi-common-cuda.txt", "No memory",
-                           true, OPAL_PROC_MY_HOSTNAME);
-            rc = OPAL_ERROR;
-            goto cleanup_and_error;
-        }
-
-        /* Create the events since they can be reused. */
-        for (i = 0; i < cuda_event_max; i++) {
-            res = cuFunc.cuEventCreate(&cuda_event_ipc_array[i], CU_EVENT_DISABLE_TIMING);
-            if (CUDA_SUCCESS != res) {
-                opal_show_help("help-mpi-common-cuda.txt", "cuEventCreate failed",
-                               true, OPAL_PROC_MY_HOSTNAME, res);
-                rc = OPAL_ERROR;
-                goto cleanup_and_error;
-            }
-        }
-
-        /* The first available status index is 0.  Make an empty frag
-           array. */
-        cuda_event_ipc_frag_array = (struct mca_btl_base_descriptor_t **)
-            malloc(sizeof(struct mca_btl_base_descriptor_t *) * cuda_event_max);
-        if (NULL == cuda_event_ipc_frag_array) {
-            opal_show_help("help-mpi-common-cuda.txt", "No memory",
-                           true, OPAL_PROC_MY_HOSTNAME);
-            rc = OPAL_ERROR;
-            goto cleanup_and_error;
-        }
-    }
-
-    if (true == mca_common_cuda_enabled) {
-        /* Set up an array to store outstanding async dtoh events.  Used on the
-         * sending side for asynchronous copies. */
-        cuda_event_dtoh_num_used = 0;
-        cuda_event_dtoh_first_avail = 0;
-        cuda_event_dtoh_first_used = 0;
-
-        cuda_event_dtoh_array = (CUevent *) calloc(cuda_event_max, sizeof(CUevent *));
-        if (NULL == cuda_event_dtoh_array) {
-            opal_show_help("help-mpi-common-cuda.txt", "No memory",
-                           true, OPAL_PROC_MY_HOSTNAME);
-            rc = OPAL_ERROR;
-            goto cleanup_and_error;
-        }
-
-        /* Create the events since they can be reused. */
-        for (i = 0; i < cuda_event_max; i++) {
-            res = cuFunc.cuEventCreate(&cuda_event_dtoh_array[i], CU_EVENT_DISABLE_TIMING);
-            if (CUDA_SUCCESS != res) {
-                opal_show_help("help-mpi-common-cuda.txt", "cuEventCreate failed",
-                               true, OPAL_PROC_MY_HOSTNAME, res);
-                rc = OPAL_ERROR;
-                goto cleanup_and_error;
-            }
-        }
-
-        /* The first available status index is 0.  Make an empty frag
-           array. */
-        cuda_event_dtoh_frag_array = (struct mca_btl_base_descriptor_t **)
-            malloc(sizeof(struct mca_btl_base_descriptor_t *) * cuda_event_max);
-        if (NULL == cuda_event_dtoh_frag_array) {
-            opal_show_help("help-mpi-common-cuda.txt", "No memory",
-                           true, OPAL_PROC_MY_HOSTNAME);
-            rc = OPAL_ERROR;
-            goto cleanup_and_error;
-        }
-
-        /* Set up an array to store outstanding async htod events.  Used on the
-         * receiving side for asynchronous copies. */
-        cuda_event_htod_num_used = 0;
-        cuda_event_htod_first_avail = 0;
-        cuda_event_htod_first_used = 0;
-
-        cuda_event_htod_array = (CUevent *) calloc(cuda_event_max, sizeof(CUevent *));
-        if (NULL == cuda_event_htod_array) {
-            opal_show_help("help-mpi-common-cuda.txt", "No memory",
-                           true, OPAL_PROC_MY_HOSTNAME);
-           rc = OPAL_ERROR;
-           goto cleanup_and_error;
-        }
-
-        /* Create the events since they can be reused. */
-        for (i = 0; i < cuda_event_max; i++) {
-            res = cuFunc.cuEventCreate(&cuda_event_htod_array[i], CU_EVENT_DISABLE_TIMING);
-            if (CUDA_SUCCESS != res) {
-                opal_show_help("help-mpi-common-cuda.txt", "cuEventCreate failed",
-                               true, OPAL_PROC_MY_HOSTNAME, res);
-               rc = OPAL_ERROR;
-               goto cleanup_and_error;
-            }
-        }
-
-        /* The first available status index is 0.  Make an empty frag
-           array. */
-        cuda_event_htod_frag_array = (struct mca_btl_base_descriptor_t **)
-            malloc(sizeof(struct mca_btl_base_descriptor_t *) * cuda_event_max);
-        if (NULL == cuda_event_htod_frag_array) {
-            opal_show_help("help-mpi-common-cuda.txt", "No memory",
-                           true, OPAL_PROC_MY_HOSTNAME);
-           rc = OPAL_ERROR;
-           goto cleanup_and_error;
-        }
-    }
-
-    s = opal_list_get_size(&common_cuda_memory_registrations);
-    for(i = 0; i < s; i++) {
-        mem_reg = (common_cuda_mem_regs_t *)
-            opal_list_remove_first(&common_cuda_memory_registrations);
-        if (mca_common_cuda_enabled && mca_common_cuda_register_memory) {
-            res = cuFunc.cuMemHostRegister(mem_reg->ptr, mem_reg->amount, 0);
-            if (res != CUDA_SUCCESS) {
-                /* If registering the memory fails, print a message and continue.
-                 * This is not a fatal error. */
-                opal_show_help("help-mpi-common-cuda.txt", "cuMemHostRegister during init failed",
-                               true, mem_reg->ptr, mem_reg->amount,
-                               OPAL_PROC_MY_HOSTNAME, res, mem_reg->msg);
-            } else {
-                opal_output_verbose(20, mca_common_cuda_output,
-                                    "CUDA: cuMemHostRegister OK on rcache %s: "
-                                    "address=%p, bufsize=%d",
-                                    mem_reg->msg, mem_reg->ptr, (int)mem_reg->amount);
-            }
-        }
-        free(mem_reg->msg);
-        OBJ_RELEASE(mem_reg);
-    }
-
-    /* Create stream for use in ipc asynchronous copies */
-    res = cuFunc.cuStreamCreate(&ipcStream, 0);
-    if (OPAL_UNLIKELY(res != CUDA_SUCCESS)) {
-        opal_show_help("help-mpi-common-cuda.txt", "cuStreamCreate failed",
-                       true, OPAL_PROC_MY_HOSTNAME, res);
-        rc = OPAL_ERROR;
-        goto cleanup_and_error;
-    }
-
-    /* Create stream for use in dtoh asynchronous copies */
-    res = cuFunc.cuStreamCreate(&dtohStream, 0);
-    if (OPAL_UNLIKELY(res != CUDA_SUCCESS)) {
-        opal_show_help("help-mpi-common-cuda.txt", "cuStreamCreate failed",
-                       true, OPAL_PROC_MY_HOSTNAME, res);
-        rc = OPAL_ERROR;
-        goto cleanup_and_error;
-    }
-
-    /* Create stream for use in htod asynchronous copies */
-    res = cuFunc.cuStreamCreate(&htodStream, 0);
-    if (OPAL_UNLIKELY(res != CUDA_SUCCESS)) {
-        opal_show_help("help-mpi-common-cuda.txt", "cuStreamCreate failed",
-                       true, OPAL_PROC_MY_HOSTNAME, res);
-        rc = OPAL_ERROR;
-        goto cleanup_and_error;
-    }
-
-    if (mca_common_cuda_cumemcpy_async) {
-        /* Create stream for use in cuMemcpyAsync synchronous copies */
-        res = cuFunc.cuStreamCreate(&memcpyStream, 0);
-        if (OPAL_UNLIKELY(res != CUDA_SUCCESS)) {
-            opal_show_help("help-mpi-common-cuda.txt", "cuStreamCreate failed",
-                           true, OPAL_PROC_MY_HOSTNAME, res);
-            rc = OPAL_ERROR;
-            goto cleanup_and_error;
-        }
-    }
-
-    res = cuFunc.cuMemHostRegister(&checkmem, sizeof(int), 0);
-    if (res != CUDA_SUCCESS) {
-        /* If registering the memory fails, print a message and continue.
-         * This is not a fatal error. */
-        opal_show_help("help-mpi-common-cuda.txt", "cuMemHostRegister during init failed",
-                       true, &checkmem, sizeof(int),
-                       OPAL_PROC_MY_HOSTNAME, res, "checkmem");
-
-    } else {
-        opal_output_verbose(20, mca_common_cuda_output,
-                            "CUDA: cuMemHostRegister OK on test region");
-    }
-
-    opal_output_verbose(20, mca_common_cuda_output,
-                        "CUDA: the extra gpu memory check is %s", (mca_common_cuda_gpu_mem_check_workaround == 1) ? "on":"off");
-
-    opal_output_verbose(30, mca_common_cuda_output,
-                        "CUDA: initialized");
-    opal_atomic_mb();  /* Make sure next statement does not get reordered */
-    common_cuda_initialized = true;
-    stage_three_init_complete = true;
-    OPAL_THREAD_UNLOCK(&common_cuda_init_lock);
-    return OPAL_SUCCESS;
-
-    /* If we are here, something went wrong.  Cleanup and return an error. */
- cleanup_and_error:
-    opal_atomic_mb(); /* Make sure next statement does not get reordered */
-    stage_three_init_complete = true;
-    OPAL_THREAD_UNLOCK(&common_cuda_init_lock);
-    return rc;
-}
-
-/**
- * Cleanup all CUDA resources.
- *
- * Note: Still figuring out how to get cuMemHostUnregister called from the smcuda sm
- * rcache.  Looks like with the memory pool from openib (grdma), the unregistering is
- * called as the free list is destructed.  Not true for the sm mpool.  This means we
- * are currently still leaking some host memory we registered with CUDA.
- */
-void mca_common_cuda_fini(void)
-{
-    int i;
-    CUresult res;
-
-    if (false == common_cuda_initialized) {
-        stage_one_init_ref_count--;
-        opal_output_verbose(20, mca_common_cuda_output,
-                            "CUDA: mca_common_cuda_fini, never completed initialization so "
-                            "skipping fini, ref_count is now %d", stage_one_init_ref_count);
-        return;
-    }
-
-    if (0 == stage_one_init_ref_count) {
-        opal_output_verbose(20, mca_common_cuda_output,
-                            "CUDA: mca_common_cuda_fini, ref_count=%d, fini is already complete",
-                            stage_one_init_ref_count);
-        return;
-    }
-
-    if (1 == stage_one_init_ref_count) {
-        opal_output_verbose(20, mca_common_cuda_output,
-                            "CUDA: mca_common_cuda_fini, ref_count=%d, cleaning up started",
-                            stage_one_init_ref_count);
-
-        /* This call is in here to make sure the context is still valid.
-         * This was the one way of checking which did not cause problems
-         * while calling into the CUDA library.  This check will detect if
-         * a user has called cudaDeviceReset prior to MPI_Finalize. If so,
-         * then this call will fail and we skip cleaning up CUDA resources. */
-        res = cuFunc.cuMemHostUnregister(&checkmem);
-        if (CUDA_SUCCESS != res) {
-            ctx_ok = 0;
-        }
-        opal_output_verbose(20, mca_common_cuda_output,
-                            "CUDA: mca_common_cuda_fini, cuMemHostUnregister returned %d, ctx_ok=%d",
-                            res, ctx_ok);
-
-        if (NULL != cuda_event_ipc_array) {
-            if (ctx_ok) {
-                for (i = 0; i < cuda_event_max; i++) {
-                    if (NULL != cuda_event_ipc_array[i]) {
-                        cuFunc.cuEventDestroy(cuda_event_ipc_array[i]);
-                    }
-                }
-            }
-            free(cuda_event_ipc_array);
-        }
-        if (NULL != cuda_event_htod_array) {
-            if (ctx_ok) {
-                for (i = 0; i < cuda_event_max; i++) {
-                    if (NULL != cuda_event_htod_array[i]) {
-                        cuFunc.cuEventDestroy(cuda_event_htod_array[i]);
-                    }
-                }
-            }
-            free(cuda_event_htod_array);
-        }
-
-        if (NULL != cuda_event_dtoh_array) {
-            if (ctx_ok) {
-                for (i = 0; i < cuda_event_max; i++) {
-                    if (NULL != cuda_event_dtoh_array[i]) {
-                        cuFunc.cuEventDestroy(cuda_event_dtoh_array[i]);
-                    }
-                }
-            }
-            free(cuda_event_dtoh_array);
-        }
-
-        if (NULL != cuda_event_ipc_frag_array) {
-            free(cuda_event_ipc_frag_array);
-        }
-        if (NULL != cuda_event_htod_frag_array) {
-            free(cuda_event_htod_frag_array);
-        }
-        if (NULL != cuda_event_dtoh_frag_array) {
-            free(cuda_event_dtoh_frag_array);
-        }
-        if ((NULL != ipcStream) && ctx_ok) {
-            cuFunc.cuStreamDestroy(ipcStream);
-        }
-        if ((NULL != dtohStream) && ctx_ok) {
-            cuFunc.cuStreamDestroy(dtohStream);
-        }
-        if ((NULL != htodStream) && ctx_ok) {
-            cuFunc.cuStreamDestroy(htodStream);
-        }
-        if ((NULL != memcpyStream) && ctx_ok) {
-            cuFunc.cuStreamDestroy(memcpyStream);
-        }
-        OBJ_DESTRUCT(&common_cuda_init_lock);
-        OBJ_DESTRUCT(&common_cuda_htod_lock);
-        OBJ_DESTRUCT(&common_cuda_dtoh_lock);
-        OBJ_DESTRUCT(&common_cuda_ipc_lock);
-        if (NULL != libcuda_handle) {
-            opal_dl_close(libcuda_handle);
-        }
-
-        opal_output_verbose(20, mca_common_cuda_output,
-                            "CUDA: mca_common_cuda_fini, ref_count=%d, cleaning up all done",
-                            stage_one_init_ref_count);
-
-        opal_output_close(mca_common_cuda_output);
-
-    } else {
-        opal_output_verbose(20, mca_common_cuda_output,
-                            "CUDA: mca_common_cuda_fini, ref_count=%d, cuda still in use",
-                            stage_one_init_ref_count);
-    }
-    stage_one_init_ref_count--;
-}
-
-/**
- * Call the CUDA register function so we pin the memory in the CUDA
- * space.
- */
-void mca_common_cuda_register(void *ptr, size_t amount, char *msg) {
-    int res;
-
-    /* Always first check if the support is enabled.  If not, just return */
-    if (!opal_cuda_support)
-        return;
-
-    if (!common_cuda_initialized) {
-        OPAL_THREAD_LOCK(&common_cuda_init_lock);
-        if (!common_cuda_initialized) {
-            common_cuda_mem_regs_t *regptr;
-            regptr = OBJ_NEW(common_cuda_mem_regs_t);
-            regptr->ptr = ptr;
-            regptr->amount = amount;
-            regptr->msg = strdup(msg);
-            opal_list_append(&common_cuda_memory_registrations,
-                             (opal_list_item_t*)regptr);
-            OPAL_THREAD_UNLOCK(&common_cuda_init_lock);
-            return;
-        }
-        OPAL_THREAD_UNLOCK(&common_cuda_init_lock);
-    }
-
-    if (mca_common_cuda_enabled && mca_common_cuda_register_memory) {
-        res = cuFunc.cuMemHostRegister(ptr, amount, 0);
-        if (OPAL_UNLIKELY(res != CUDA_SUCCESS)) {
-            /* If registering the memory fails, print a message and continue.
-             * This is not a fatal error. */
-            opal_show_help("help-mpi-common-cuda.txt", "cuMemHostRegister failed",
-                           true, ptr, amount,
-                           OPAL_PROC_MY_HOSTNAME, res, msg);
-        } else {
-            opal_output_verbose(20, mca_common_cuda_output,
-                                "CUDA: cuMemHostRegister OK on rcache %s: "
-                                "address=%p, bufsize=%d",
-                                msg, ptr, (int)amount);
-        }
-    }
-}
-
-/**
- * Call the CUDA unregister function so we unpin the memory in the CUDA
- * space.
- */
-void mca_common_cuda_unregister(void *ptr, char *msg) {
-    int res, i, s;
-    common_cuda_mem_regs_t *mem_reg;
-
-    /* This can happen if memory was queued up to be registered, but
-     * no CUDA operations happened, so it never was registered.
-     * Therefore, just release any of the resources. */
-    if (!common_cuda_initialized) {
-        s = opal_list_get_size(&common_cuda_memory_registrations);
-        for(i = 0; i < s; i++) {
-            mem_reg = (common_cuda_mem_regs_t *)
-                opal_list_remove_first(&common_cuda_memory_registrations);
-            free(mem_reg->msg);
-            OBJ_RELEASE(mem_reg);
-        }
-        return;
-    }
-
-    if (mca_common_cuda_enabled && mca_common_cuda_register_memory) {
-        res = cuFunc.cuMemHostUnregister(ptr);
-        if (OPAL_UNLIKELY(res != CUDA_SUCCESS)) {
-            /* If unregistering the memory fails, just continue.  This is during
-             * shutdown.  Only print when running in verbose mode. */
-            opal_output_verbose(20, mca_common_cuda_output,
-                                "CUDA: cuMemHostUnregister failed: ptr=%p, res=%d, rcache=%s",
-                                ptr, res, msg);
-
-        } else {
-            opal_output_verbose(20, mca_common_cuda_output,
-                                "CUDA: cuMemHostUnregister OK on rcache %s: "
-                                "address=%p",
-                                msg, ptr);
-        }
-    }
-}
-
-/*
- * Get the memory handle of a local section of memory that can be sent
- * to the remote size so it can access the memory.  This is the
- * registration function for the sending side of a message transfer.
- */
-int cuda_getmemhandle(void *base, size_t size, mca_rcache_base_registration_t *newreg,
-                      mca_rcache_base_registration_t *hdrreg)
-
-{
-    CUmemorytype memType;
-    CUresult result;
-    CUipcMemHandle *memHandle;
-    CUdeviceptr pbase;
-    size_t psize;
-
-    mca_rcache_common_cuda_reg_t *cuda_reg = (mca_rcache_common_cuda_reg_t*)newreg;
-    memHandle = (CUipcMemHandle *)cuda_reg->data.memHandle;
-
-    /* We should only be there if this is a CUDA device pointer */
-    result = cuFunc.cuPointerGetAttribute(&memType,
-                                          CU_POINTER_ATTRIBUTE_MEMORY_TYPE, (CUdeviceptr)base);
-    assert(CUDA_SUCCESS == result);
-    assert(CU_MEMORYTYPE_DEVICE == memType);
-
-    /* Get the memory handle so we can send it to the remote process. */
-    result = cuFunc.cuIpcGetMemHandle(memHandle, (CUdeviceptr)base);
-    CUDA_DUMP_MEMHANDLE((100, memHandle, "GetMemHandle-After"));
-
-    if (CUDA_SUCCESS != result) {
-        opal_show_help("help-mpi-common-cuda.txt", "cuIpcGetMemHandle failed",
-                       true, result, base);
-        return OPAL_ERROR;
-    } else {
-        opal_output_verbose(20, mca_common_cuda_output,
-                            "CUDA: cuIpcGetMemHandle passed: base=%p size=%d",
-                            base, (int)size);
-    }
-
-    /* Need to get the real base and size of the memory handle.  This is
-     * how the remote side saves the handles in a cache. */
-    result = cuFunc.cuMemGetAddressRange(&pbase, &psize, (CUdeviceptr)base);
-    if (CUDA_SUCCESS != result) {
-        opal_show_help("help-mpi-common-cuda.txt", "cuMemGetAddressRange failed",
-                       true, result, base);
-        return OPAL_ERROR;
-    } else {
-        opal_output_verbose(10, mca_common_cuda_output,
-                            "CUDA: cuMemGetAddressRange passed: addr=%p, size=%d, pbase=%p, psize=%d ",
-                            base, (int)size, (void *)pbase, (int)psize);
-    }
-
-    /* Store all the information in the registration */
-    cuda_reg->base.base = (void *)pbase;
-    cuda_reg->base.bound = (unsigned char *)pbase + psize - 1;
-    cuda_reg->data.memh_seg_addr.pval = (void *) pbase;
-    cuda_reg->data.memh_seg_len = psize;
-
-#if OPAL_CUDA_SYNC_MEMOPS
-    /* With CUDA 6.0, we can set an attribute on the memory pointer that will
-     * ensure any synchronous copies are completed prior to any other access
-     * of the memory region.  This means we do not need to record an event
-     * and send to the remote side.
-     */
-    memType = 1; /* Just use this variable since we already have it */
-    result = cuFunc.cuPointerSetAttribute(&memType, CU_POINTER_ATTRIBUTE_SYNC_MEMOPS,
-                                          (CUdeviceptr)base);
-    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
-        opal_show_help("help-mpi-common-cuda.txt", "cuPointerSetAttribute failed",
-                       true, OPAL_PROC_MY_HOSTNAME, result, base);
-        return OPAL_ERROR;
-    }
-#else
-    /* Need to record the event to ensure that any memcopies into the
-     * device memory have completed.  The event handle associated with
-     * this event is sent to the remote process so that it will wait
-     * on this event prior to copying data out of the device memory.
-     * Note that this needs to be the NULL stream to make since it is
-     * unknown what stream any copies into the device memory were done
-     * with. */
-    result = cuFunc.cuEventRecord((CUevent)cuda_reg->data.event, 0);
-    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
-        opal_show_help("help-mpi-common-cuda.txt", "cuEventRecord failed",
-                       true, result, base);
-        return OPAL_ERROR;
-    }
-#endif /* OPAL_CUDA_SYNC_MEMOPS */
-
-    return OPAL_SUCCESS;
-}
-
-/*
- * This function is called by the local side that called the cuda_getmemhandle.
- * There is nothing to be done so just return.
- */
-int cuda_ungetmemhandle(void *reg_data, mca_rcache_base_registration_t *reg)
-{
-    opal_output_verbose(10, mca_common_cuda_output,
-                        "CUDA: cuda_ungetmemhandle (no-op): base=%p", reg->base);
-    CUDA_DUMP_MEMHANDLE((100, ((mca_rcache_common_cuda_reg_t *)reg)->data.memHandle, "cuda_ungetmemhandle"));
-
-    return OPAL_SUCCESS;
-}
-
-/*
- * Open a memory handle that refers to remote memory so we can get an address
- * that works on the local side.  This is the registration function for the
- * remote side of a transfer.  newreg contains the new handle.  hddrreg contains
- * the memory handle that was received from the remote side.
- */
-int cuda_openmemhandle(void *base, size_t size, mca_rcache_base_registration_t *newreg,
-                       mca_rcache_base_registration_t *hdrreg)
-{
-    CUresult result;
-    CUipcMemHandle *memHandle;
-    mca_rcache_common_cuda_reg_t *cuda_newreg = (mca_rcache_common_cuda_reg_t*)newreg;
-
-    /* Save in local variable to avoid ugly casting */
-    memHandle = (CUipcMemHandle *)cuda_newreg->data.memHandle;
-    CUDA_DUMP_MEMHANDLE((100, memHandle, "Before call to cuIpcOpenMemHandle"));
-
-    /* Open the memory handle and store it into the registration structure. */
-    result = cuFunc.cuIpcOpenMemHandle((CUdeviceptr *)&newreg->alloc_base, *memHandle,
-                                       CU_IPC_MEM_LAZY_ENABLE_PEER_ACCESS);
-
-    /* If there are some stale entries in the cache, they can cause other
-     * registrations to fail.  Let the caller know that so that can attempt
-     * to clear them out. */
-    if (CUDA_ERROR_ALREADY_MAPPED == result) {
-        opal_output_verbose(10, mca_common_cuda_output,
-                            "CUDA: cuIpcOpenMemHandle returned CUDA_ERROR_ALREADY_MAPPED for "
-                            "p=%p,size=%d: notify memory pool\n", base, (int)size);
-        return OPAL_ERR_WOULD_BLOCK;
-    }
-    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
-        opal_show_help("help-mpi-common-cuda.txt", "cuIpcOpenMemHandle failed",
-                       true, OPAL_PROC_MY_HOSTNAME, result, base);
-        /* Currently, this is a non-recoverable error */
-        return OPAL_ERROR;
-    } else {
-        opal_output_verbose(10, mca_common_cuda_output,
-                            "CUDA: cuIpcOpenMemHandle passed: base=%p (remote base=%p,size=%d)",
-                            newreg->alloc_base, base, (int)size);
-        CUDA_DUMP_MEMHANDLE((200, memHandle, "cuIpcOpenMemHandle"));
-    }
-
-    return OPAL_SUCCESS;
-}
-
-/*
- * Close a memory handle that refers to remote memory.
- */
-int cuda_closememhandle(void *reg_data, mca_rcache_base_registration_t *reg)
-{
-    CUresult result;
-    mca_rcache_common_cuda_reg_t *cuda_reg = (mca_rcache_common_cuda_reg_t*)reg;
-
-    /* Only attempt to close if we have valid context.  This can change if a call
-     * to the fini function is made and we discover context is gone. */
-    if (ctx_ok) {
-        result = cuFunc.cuIpcCloseMemHandle((CUdeviceptr)cuda_reg->base.alloc_base);
-        if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
-            if (CUDA_ERROR_DEINITIALIZED != result) {
-                opal_show_help("help-mpi-common-cuda.txt", "cuIpcCloseMemHandle failed",
-                true, result, cuda_reg->base.alloc_base);
-            }
-            /* We will just continue on and hope things continue to work. */
-        } else {
-            opal_output_verbose(10, mca_common_cuda_output,
-                                "CUDA: cuIpcCloseMemHandle passed: base=%p",
-                                cuda_reg->base.alloc_base);
-            CUDA_DUMP_MEMHANDLE((100, cuda_reg->data.memHandle, "cuIpcCloseMemHandle"));
-        }
-    }
-
-    return OPAL_SUCCESS;
-}
-
-void mca_common_cuda_construct_event_and_handle(uintptr_t *event, void *handle)
-{
-    CUresult result;
-
-    result = cuFunc.cuEventCreate((CUevent *)event, CU_EVENT_INTERPROCESS | CU_EVENT_DISABLE_TIMING);
-    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
-        opal_show_help("help-mpi-common-cuda.txt", "cuEventCreate failed",
-                       true, OPAL_PROC_MY_HOSTNAME, result);
-    }
-
-    result = cuFunc.cuIpcGetEventHandle((CUipcEventHandle *)handle, (CUevent)*event);
-    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
-        opal_show_help("help-mpi-common-cuda.txt", "cuIpcGetEventHandle failed",
-                       true, result);
-    }
-
-    CUDA_DUMP_EVTHANDLE((10, handle, "construct_event_and_handle"));
-
-}
-
-void mca_common_cuda_destruct_event(uintptr_t event)
-{
-    CUresult result;
-
-    /* Only attempt to destroy if we have valid context.  This can change if a call
-     * to the fini function is made and we discover context is gone. */
-    if (ctx_ok) {
-        result = cuFunc.cuEventDestroy((CUevent)event);
-        if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
-            opal_show_help("help-mpi-common-cuda.txt", "cuEventDestroy failed",
-                           true, result);
-        }
-    }
-}
-
-
-/*
- * Put remote event on stream to ensure that the the start of the
- * copy does not start until the completion of the event.
- */
-void mca_common_wait_stream_synchronize(mca_rcache_common_cuda_reg_t *rget_reg)
-{
-#if OPAL_CUDA_SYNC_MEMOPS
-    /* No need for any of this with SYNC_MEMOPS feature */
-    return;
-#else /* OPAL_CUDA_SYNC_MEMOPS */
-    CUipcEventHandle evtHandle;
-    CUevent event;
-    CUresult result;
-
-    memcpy(&evtHandle, rget_reg->data.evtHandle, sizeof(evtHandle));
-    CUDA_DUMP_EVTHANDLE((100, &evtHandle, "stream_synchronize"));
-
-    result = cuFunc.cuIpcOpenEventHandle(&event, evtHandle);
-    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
-        opal_show_help("help-mpi-common-cuda.txt", "cuIpcOpenEventHandle failed",
-                       true, result);
-    }
-
-    /* BEGIN of Workaround - There is a bug in CUDA 4.1 RC2 and earlier
-     * versions.  Need to record an event on the stream, even though
-     * it is not used, to make sure we do not short circuit our way
-     * out of the cuStreamWaitEvent test.
-     */
-    result = cuFunc.cuEventRecord(event, 0);
-    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
-        opal_show_help("help-mpi-common-cuda.txt", "cuEventRecord failed",
-                       true, OPAL_PROC_MY_HOSTNAME, result);
-    }
-    /* END of Workaround */
-
-    result = cuFunc.cuStreamWaitEvent(0, event, 0);
-    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
-        opal_show_help("help-mpi-common-cuda.txt", "cuStreamWaitEvent failed",
-                       true, result);
-    }
-
-    /* All done with this event. */
-    result = cuFunc.cuEventDestroy(event);
-    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
-        opal_show_help("help-mpi-common-cuda.txt", "cuEventDestroy failed",
-                       true, result);
-    }
-#endif /* OPAL_CUDA_SYNC_MEMOPS */
-}
-
-/*
- * Start the asynchronous copy.  Then record and save away an event that will
- * be queried to indicate the copy has completed.
- */
-int mca_common_cuda_memcpy(void *dst, void *src, size_t amount, char *msg,
-                           struct mca_btl_base_descriptor_t *frag, int *done)
-{
-    CUresult result;
-    int iter;
-
-    OPAL_THREAD_LOCK(&common_cuda_ipc_lock);
-    /* First make sure there is room to store the event.  If not, then
-     * return an error.  The error message will tell the user to try and
-     * run again, but with a larger array for storing events. */
-    if (cuda_event_ipc_num_used == cuda_event_max) {
-        opal_show_help("help-mpi-common-cuda.txt", "Out of cuEvent handles",
-                       true, cuda_event_max, cuda_event_max+100, cuda_event_max+100);
-        OPAL_THREAD_UNLOCK(&common_cuda_ipc_lock);
-        return OPAL_ERR_OUT_OF_RESOURCE;
-    }
-
-    if (cuda_event_ipc_num_used > cuda_event_ipc_most) {
-        cuda_event_ipc_most = cuda_event_ipc_num_used;
-        /* Just print multiples of 10 */
-        if (0 == (cuda_event_ipc_most % 10)) {
-            opal_output_verbose(20, mca_common_cuda_output,
-                                "Maximum ipc events used is now %d", cuda_event_ipc_most);
-        }
-    }
-
-    /* This is the standard way to run.  Running with synchronous copies is available
-     * to measure the advantages of asynchronous copies. */
-    if (OPAL_LIKELY(mca_common_cuda_async)) {
-        result = cuFunc.cuMemcpyAsync((CUdeviceptr)dst, (CUdeviceptr)src, amount, ipcStream);
-        if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
-            opal_show_help("help-mpi-common-cuda.txt", "cuMemcpyAsync failed",
-                           true, dst, src, amount, result);
-            OPAL_THREAD_UNLOCK(&common_cuda_ipc_lock);
-            return OPAL_ERROR;
-        } else {
-            opal_output_verbose(20, mca_common_cuda_output,
-                                "CUDA: cuMemcpyAsync passed: dst=%p, src=%p, size=%d",
-                                dst, src, (int)amount);
-        }
-        result = cuFunc.cuEventRecord(cuda_event_ipc_array[cuda_event_ipc_first_avail], ipcStream);
-        if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
-            opal_show_help("help-mpi-common-cuda.txt", "cuEventRecord failed",
-                           true, OPAL_PROC_MY_HOSTNAME, result);
-            OPAL_THREAD_UNLOCK(&common_cuda_ipc_lock);
-            return OPAL_ERROR;
-        }
-        cuda_event_ipc_frag_array[cuda_event_ipc_first_avail] = frag;
-
-        /* Bump up the first available slot and number used by 1 */
-        cuda_event_ipc_first_avail++;
-        if (cuda_event_ipc_first_avail >= cuda_event_max) {
-            cuda_event_ipc_first_avail = 0;
-        }
-        cuda_event_ipc_num_used++;
-
-        *done = 0;
-    } else {
-        /* Mimic the async function so they use the same memcpy call. */
-        result = cuFunc.cuMemcpyAsync((CUdeviceptr)dst, (CUdeviceptr)src, amount, ipcStream);
-        if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
-            opal_show_help("help-mpi-common-cuda.txt", "cuMemcpyAsync failed",
-                           true, dst, src, amount, result);
-            OPAL_THREAD_UNLOCK(&common_cuda_ipc_lock);
-            return OPAL_ERROR;
-        } else {
-            opal_output_verbose(20, mca_common_cuda_output,
-                                "CUDA: cuMemcpyAsync passed: dst=%p, src=%p, size=%d",
-                                dst, src, (int)amount);
-        }
-
-        /* Record an event, then wait for it to complete with calls to cuEventQuery */
-        result = cuFunc.cuEventRecord(cuda_event_ipc_array[cuda_event_ipc_first_avail], ipcStream);
-        if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
-            opal_show_help("help-mpi-common-cuda.txt", "cuEventRecord failed",
-                           true, OPAL_PROC_MY_HOSTNAME, result);
-            OPAL_THREAD_UNLOCK(&common_cuda_ipc_lock);
-            return OPAL_ERROR;
-        }
-
-        cuda_event_ipc_frag_array[cuda_event_ipc_first_avail] = frag;
-
-        /* Bump up the first available slot and number used by 1 */
-        cuda_event_ipc_first_avail++;
-        if (cuda_event_ipc_first_avail >= cuda_event_max) {
-            cuda_event_ipc_first_avail = 0;
-        }
-        cuda_event_ipc_num_used++;
-
-        result = cuFunc.cuEventQuery(cuda_event_ipc_array[cuda_event_ipc_first_used]);
-        if ((CUDA_SUCCESS != result) && (CUDA_ERROR_NOT_READY != result)) {
-            opal_show_help("help-mpi-common-cuda.txt", "cuEventQuery failed",
-                           true, result);
-            OPAL_THREAD_UNLOCK(&common_cuda_ipc_lock);
-            return OPAL_ERROR;
-        }
-
-        iter = 0;
-        while (CUDA_ERROR_NOT_READY == result) {
-            if (0 == (iter % 10)) {
-                opal_output(-1, "EVENT NOT DONE (iter=%d)", iter);
-            }
-            result = cuFunc.cuEventQuery(cuda_event_ipc_array[cuda_event_ipc_first_used]);
-            if ((CUDA_SUCCESS != result) && (CUDA_ERROR_NOT_READY != result)) {
-                opal_show_help("help-mpi-common-cuda.txt", "cuEventQuery failed",
-                               true, result);
-            OPAL_THREAD_UNLOCK(&common_cuda_ipc_lock);
-                return OPAL_ERROR;
-            }
-            iter++;
-        }
-
-        --cuda_event_ipc_num_used;
-        ++cuda_event_ipc_first_used;
-        if (cuda_event_ipc_first_used >= cuda_event_max) {
-            cuda_event_ipc_first_used = 0;
-        }
-        *done = 1;
-    }
-    OPAL_THREAD_UNLOCK(&common_cuda_ipc_lock);
-    return OPAL_SUCCESS;
-}
-
-/*
- * Record an event and save the frag.  This is called by the sending side and
- * is used to queue an event when a htod copy has been initiated.
- */
-int mca_common_cuda_record_dtoh_event(char *msg, struct mca_btl_base_descriptor_t *frag)
-{
-    CUresult result;
-
-    /* First make sure there is room to store the event.  If not, then
-     * return an error.  The error message will tell the user to try and
-     * run again, but with a larger array for storing events. */
-    OPAL_THREAD_LOCK(&common_cuda_dtoh_lock);
-    if (cuda_event_dtoh_num_used == cuda_event_max) {
-        opal_show_help("help-mpi-common-cuda.txt", "Out of cuEvent handles",
-                       true, cuda_event_max, cuda_event_max+100, cuda_event_max+100);
-        return OPAL_ERR_OUT_OF_RESOURCE;
-    }
-
-    if (cuda_event_dtoh_num_used > cuda_event_dtoh_most) {
-        cuda_event_dtoh_most = cuda_event_dtoh_num_used;
-        /* Just print multiples of 10 */
-        if (0 == (cuda_event_dtoh_most % 10)) {
-            opal_output_verbose(20, mca_common_cuda_output,
-                                "Maximum DtoH events used is now %d", cuda_event_dtoh_most);
-        }
-    }
-
-    result = cuFunc.cuEventRecord(cuda_event_dtoh_array[cuda_event_dtoh_first_avail], dtohStream);
-    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
-        opal_show_help("help-mpi-common-cuda.txt", "cuEventRecord failed",
-                       true, OPAL_PROC_MY_HOSTNAME, result);
-        OPAL_THREAD_UNLOCK(&common_cuda_dtoh_lock);
-        return OPAL_ERROR;
-    }
-    cuda_event_dtoh_frag_array[cuda_event_dtoh_first_avail] = frag;
-
-    /* Bump up the first available slot and number used by 1 */
-    cuda_event_dtoh_first_avail++;
-    if (cuda_event_dtoh_first_avail >= cuda_event_max) {
-        cuda_event_dtoh_first_avail = 0;
-    }
-    cuda_event_dtoh_num_used++;
-
-    OPAL_THREAD_UNLOCK(&common_cuda_dtoh_lock);
-    return OPAL_SUCCESS;
-}
-
-/*
- * Record an event and save the frag.  This is called by the receiving side and
- * is used to queue an event when a dtoh copy has been initiated.
- */
-int mca_common_cuda_record_htod_event(char *msg, struct mca_btl_base_descriptor_t *frag)
-{
-    CUresult result;
-
-    OPAL_THREAD_LOCK(&common_cuda_htod_lock);
-    /* First make sure there is room to store the event.  If not, then
-     * return an error.  The error message will tell the user to try and
-     * run again, but with a larger array for storing events. */
-    if (cuda_event_htod_num_used == cuda_event_max) {
-        opal_show_help("help-mpi-common-cuda.txt", "Out of cuEvent handles",
-                       true, cuda_event_max, cuda_event_max+100, cuda_event_max+100);
-        OPAL_THREAD_UNLOCK(&common_cuda_htod_lock);
-        return OPAL_ERR_OUT_OF_RESOURCE;
-    }
-
-    if (cuda_event_htod_num_used > cuda_event_htod_most) {
-        cuda_event_htod_most = cuda_event_htod_num_used;
-        /* Just print multiples of 10 */
-        if (0 == (cuda_event_htod_most % 10)) {
-            opal_output_verbose(20, mca_common_cuda_output,
-                                "Maximum HtoD events used is now %d", cuda_event_htod_most);
-        }
-    }
-
-    result = cuFunc.cuEventRecord(cuda_event_htod_array[cuda_event_htod_first_avail], htodStream);
-    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
-        opal_show_help("help-mpi-common-cuda.txt", "cuEventRecord failed",
-                       true, OPAL_PROC_MY_HOSTNAME, result);
-        OPAL_THREAD_UNLOCK(&common_cuda_htod_lock);
-        return OPAL_ERROR;
-    }
-    cuda_event_htod_frag_array[cuda_event_htod_first_avail] = frag;
-
-   /* Bump up the first available slot and number used by 1 */
-    cuda_event_htod_first_avail++;
-    if (cuda_event_htod_first_avail >= cuda_event_max) {
-        cuda_event_htod_first_avail = 0;
-    }
-    cuda_event_htod_num_used++;
-
-    OPAL_THREAD_UNLOCK(&common_cuda_htod_lock);
-    return OPAL_SUCCESS;
-}
-
-/**
- * Used to get the dtoh stream for initiating asynchronous copies.
- */
-void *mca_common_cuda_get_dtoh_stream(void) {
-    return (void *)dtohStream;
-}
-
-/**
- * Used to get the htod stream for initiating asynchronous copies.
- */
-void *mca_common_cuda_get_htod_stream(void) {
-    return (void *)htodStream;
-}
-
-/*
- * Function is called every time progress is called with the sm BTL.  If there
- * are outstanding events, check to see if one has completed.  If so, hand
- * back the fragment for further processing.
- */
-int progress_one_cuda_ipc_event(struct mca_btl_base_descriptor_t **frag) {
-    CUresult result;
-
-    if( OPAL_LIKELY(0 == cuda_event_ipc_num_used) )
-        return 0;
-
-    OPAL_THREAD_LOCK(&common_cuda_ipc_lock);
-    if (cuda_event_ipc_num_used > 0) {
-        opal_output_verbose(20, mca_common_cuda_output,
-                           "CUDA: progress_one_cuda_ipc_event, outstanding_events=%d",
-                            cuda_event_ipc_num_used);
-
-        result = cuFunc.cuEventQuery(cuda_event_ipc_array[cuda_event_ipc_first_used]);
-
-        /* We found an event that is not ready, so return. */
-        if (CUDA_ERROR_NOT_READY == result) {
-            opal_output_verbose(20, mca_common_cuda_output,
-                                "CUDA: cuEventQuery returned CUDA_ERROR_NOT_READY");
-            *frag = NULL;
-            OPAL_THREAD_UNLOCK(&common_cuda_ipc_lock);
-            return 0;
-        } else if (CUDA_SUCCESS != result) {
-            opal_show_help("help-mpi-common-cuda.txt", "cuEventQuery failed",
-                           true, result);
-            *frag = NULL;
-            OPAL_THREAD_UNLOCK(&common_cuda_ipc_lock);
-            return OPAL_ERROR;
-        }
-
-        *frag = cuda_event_ipc_frag_array[cuda_event_ipc_first_used];
-        opal_output_verbose(10, mca_common_cuda_output,
-                            "CUDA: cuEventQuery returned %d", result);
-
-        /* Bump counters, loop around the circular buffer if necessary */
-        --cuda_event_ipc_num_used;
-        ++cuda_event_ipc_first_used;
-        if (cuda_event_ipc_first_used >= cuda_event_max) {
-            cuda_event_ipc_first_used = 0;
-        }
-        /* A return value of 1 indicates an event completed and a frag was returned */
-        OPAL_THREAD_UNLOCK(&common_cuda_ipc_lock);
-        return 1;
-    }
-    OPAL_THREAD_UNLOCK(&common_cuda_ipc_lock);
-    return 0;
-}
-
-/**
- * Progress any dtoh event completions.
- */
-int progress_one_cuda_dtoh_event(struct mca_btl_base_descriptor_t **frag) {
-    CUresult result;
-
-    OPAL_THREAD_LOCK(&common_cuda_dtoh_lock);
-    if (cuda_event_dtoh_num_used > 0) {
-        opal_output_verbose(30, mca_common_cuda_output,
-                           "CUDA: progress_one_cuda_dtoh_event, outstanding_events=%d",
-                            cuda_event_dtoh_num_used);
-
-        result = cuFunc.cuEventQuery(cuda_event_dtoh_array[cuda_event_dtoh_first_used]);
-
-        /* We found an event that is not ready, so return. */
-        if (CUDA_ERROR_NOT_READY == result) {
-            opal_output_verbose(30, mca_common_cuda_output,
-                                "CUDA: cuEventQuery returned CUDA_ERROR_NOT_READY");
-            *frag = NULL;
-            OPAL_THREAD_UNLOCK(&common_cuda_dtoh_lock);
-            return 0;
-        } else if (CUDA_SUCCESS != result) {
-            opal_show_help("help-mpi-common-cuda.txt", "cuEventQuery failed",
-                           true, result);
-            *frag = NULL;
-            OPAL_THREAD_UNLOCK(&common_cuda_dtoh_lock);
-            return OPAL_ERROR;
-        }
-
-        *frag = cuda_event_dtoh_frag_array[cuda_event_dtoh_first_used];
-        opal_output_verbose(30, mca_common_cuda_output,
-                            "CUDA: cuEventQuery returned %d", result);
-
-        /* Bump counters, loop around the circular buffer if necessary */
-        --cuda_event_dtoh_num_used;
-        ++cuda_event_dtoh_first_used;
-        if (cuda_event_dtoh_first_used >= cuda_event_max) {
-            cuda_event_dtoh_first_used = 0;
-        }
-        /* A return value of 1 indicates an event completed and a frag was returned */
-        OPAL_THREAD_UNLOCK(&common_cuda_dtoh_lock);
-        return 1;
-    }
-    OPAL_THREAD_UNLOCK(&common_cuda_dtoh_lock);
-    return 0;
-}
-
-/**
- * Progress any dtoh event completions.
- */
-int progress_one_cuda_htod_event(struct mca_btl_base_descriptor_t **frag) {
-    CUresult result;
-
-    OPAL_THREAD_LOCK(&common_cuda_htod_lock);
-    if (cuda_event_htod_num_used > 0) {
-        opal_output_verbose(30, mca_common_cuda_output,
-                           "CUDA: progress_one_cuda_htod_event, outstanding_events=%d",
-                            cuda_event_htod_num_used);
-
-        result = cuFunc.cuEventQuery(cuda_event_htod_array[cuda_event_htod_first_used]);
-
-        /* We found an event that is not ready, so return. */
-        if (CUDA_ERROR_NOT_READY == result) {
-            opal_output_verbose(30, mca_common_cuda_output,
-                                "CUDA: cuEventQuery returned CUDA_ERROR_NOT_READY");
-            *frag = NULL;
-            OPAL_THREAD_UNLOCK(&common_cuda_htod_lock);
-            return 0;
-        } else if (CUDA_SUCCESS != result) {
-            opal_show_help("help-mpi-common-cuda.txt", "cuEventQuery failed",
-                           true, result);
-            *frag = NULL;
-            OPAL_THREAD_UNLOCK(&common_cuda_htod_lock);
-            return OPAL_ERROR;
-        }
-
-        *frag = cuda_event_htod_frag_array[cuda_event_htod_first_used];
-        opal_output_verbose(30, mca_common_cuda_output,
-                            "CUDA: cuEventQuery returned %d", result);
-
-        /* Bump counters, loop around the circular buffer if necessary */
-        --cuda_event_htod_num_used;
-        ++cuda_event_htod_first_used;
-        if (cuda_event_htod_first_used >= cuda_event_max) {
-            cuda_event_htod_first_used = 0;
-        }
-        /* A return value of 1 indicates an event completed and a frag was returned */
-        OPAL_THREAD_UNLOCK(&common_cuda_htod_lock);
-        return 1;
-    }
-    OPAL_THREAD_UNLOCK(&common_cuda_htod_lock);
-    return OPAL_ERR_RESOURCE_BUSY;
-}
-
-
-/**
- * Need to make sure the handle we are retrieving from the cache is still
- * valid.  Compare the cached handle to the one received.
- */
-int mca_common_cuda_memhandle_matches(mca_rcache_common_cuda_reg_t *new_reg,
-                                      mca_rcache_common_cuda_reg_t *old_reg)
-{
-
-    if (0 == memcmp(new_reg->data.memHandle, old_reg->data.memHandle, sizeof(new_reg->data.memHandle))) {
-        return 1;
-    } else {
-        return 0;
-    }
-
-}
-
-/*
- * Function to dump memory handle information.  This is based on
- * definitions from cuiinterprocess_private.h.
- */
-static void cuda_dump_memhandle(int verbose, void *memHandle, char *str) {
-
-    struct InterprocessMemHandleInternal
-    {
-        /* The first two entries are the CUinterprocessCtxHandle */
-        int64_t ctxId; /* unique (within a process) id of the sharing context */
-        int     pid;   /* pid of sharing context */
-
-        int64_t size;
-        int64_t blocksize;
-        int64_t offset;
-        int     gpuId;
-        int     subDeviceIndex;
-        int64_t serial;
-    } memH;
-
-    if (NULL == str) {
-        str = "CUDA";
-    }
-    memcpy(&memH, memHandle, sizeof(memH));
-    opal_output_verbose(verbose, mca_common_cuda_output,
-                        "%s:ctxId=0x%" PRIx64 ", pid=%d, size=%" PRIu64 ", blocksize=%" PRIu64 ", offset=%"
-                        PRIu64 ", gpuId=%d, subDeviceIndex=%d, serial=%" PRIu64,
-                        str, memH.ctxId, memH.pid, memH.size, memH.blocksize, memH.offset,
-                        memH.gpuId, memH.subDeviceIndex, memH.serial);
-}
-
-/*
- * Function to dump memory handle information.  This is based on
- * definitions from cuiinterprocess_private.h.
- */
-static void cuda_dump_evthandle(int verbose, void *evtHandle, char *str) {
-
-    struct InterprocessEventHandleInternal
-    {
-        unsigned long pid;
-        unsigned long serial;
-        int index;
-    } evtH;
-
-    if (NULL == str) {
-        str = "CUDA";
-    }
-    memcpy(&evtH, evtHandle, sizeof(evtH));
-    opal_output_verbose(verbose, mca_common_cuda_output,
-                        "CUDA: %s:pid=%lu, serial=%lu, index=%d",
-                        str, evtH.pid, evtH.serial, evtH.index);
-}
-
-
-/* Return microseconds of elapsed time. Microseconds are relevant when
- * trying to understand the fixed overhead of the communication. Used
- * when trying to time various functions.
- *
- * Cut and past the following to get timings where wanted.
- *
- *   clock_gettime(CLOCK_MONOTONIC, &ts_start);
- *   FUNCTION OF INTEREST
- *   clock_gettime(CLOCK_MONOTONIC, &ts_end);
- *   accum = mydifftime(ts_start, ts_end);
- *   opal_output(0, "Function took   %7.2f usecs\n", accum);
- *
- */
-#if OPAL_ENABLE_DEBUG
-static float mydifftime(opal_timer_t ts_start, opal_timer_t ts_end) {
-    return (ts_end - ts_start);
-}
-#endif /* OPAL_ENABLE_DEBUG */
-
-/* Routines that get plugged into the opal datatype code */
-static int mca_common_cuda_is_gpu_buffer(const void *pUserBuf, opal_convertor_t *convertor)
-{
-    int res;
-    CUmemorytype memType = 0;
-    CUdeviceptr dbuf = (CUdeviceptr)pUserBuf;
-    CUcontext ctx = NULL, memCtx = NULL;
-#if OPAL_CUDA_GET_ATTRIBUTES
-    uint32_t isManaged = 0;
-    /* With CUDA 7.0, we can get multiple attributes with a single call */
-    CUpointer_attribute attributes[3] = {CU_POINTER_ATTRIBUTE_MEMORY_TYPE,
-                                         CU_POINTER_ATTRIBUTE_CONTEXT,
-                                         CU_POINTER_ATTRIBUTE_IS_MANAGED};
-    void *attrdata[] = {(void *)&memType, (void *)&memCtx, (void *)&isManaged};
-
-    res = cuFunc.cuPointerGetAttributes(3, attributes, attrdata, dbuf);
-    OPAL_OUTPUT_VERBOSE((101, mca_common_cuda_output,
-                        "dbuf=%p, memType=%d, memCtx=%p, isManaged=%d, res=%d",
-                         (void *)dbuf, (int)memType, (void *)memCtx, isManaged, res));
-
-    /* Mark unified memory buffers with a flag.  This will allow all unified
-     * memory to be forced through host buffers.  Note that this memory can
-     * be either host or device so we need to set this flag prior to that check. */
-    if (1 == isManaged) {
-        if (NULL != convertor) {
-            convertor->flags |= CONVERTOR_CUDA_UNIFIED;
-        }
-    }
-    if (res != CUDA_SUCCESS) {
-        /* If we cannot determine it is device pointer,
-         * just assume it is not. */
-        return 0;
-    } else if (memType == CU_MEMORYTYPE_HOST) {
-        /* Host memory, nothing to do here */
-        return 0;
-    } else if (memType == 0) {
-        /* This can happen when CUDA is initialized but dbuf is not valid CUDA pointer */
-        return 0;
-    }
-    /* Must be a device pointer */
-    assert(memType == CU_MEMORYTYPE_DEVICE);
-#else /* OPAL_CUDA_GET_ATTRIBUTES */
-    res = cuFunc.cuPointerGetAttribute(&memType,
-                                       CU_POINTER_ATTRIBUTE_MEMORY_TYPE, dbuf);
-    if (res != CUDA_SUCCESS) {
-        /* If we cannot determine it is device pointer,
-         * just assume it is not. */
-        return 0;
-    } else if (memType == CU_MEMORYTYPE_HOST) {
-        /* Host memory, nothing to do here */
-        return 0;
-    }
-    /* Must be a device pointer */
-    assert(memType == CU_MEMORYTYPE_DEVICE);
-#endif /* OPAL_CUDA_GET_ATTRIBUTES */
-
-    /* This piece of code was added in to handle in a case involving
-     * OMP threads.  The user had initialized CUDA and then spawned
-     * two threads.  The first thread had the CUDA context, but the
-     * second thread did not.  We therefore had no context to act upon
-     * and future CUDA driver calls would fail.  Therefore, if we have
-     * GPU memory, but no context, get the context from the GPU memory
-     * and set the current context to that.  It is rare that we will not
-     * have a context. */
-    res = cuFunc.cuCtxGetCurrent(&ctx);
-    if (OPAL_UNLIKELY(NULL == ctx)) {
-        if (CUDA_SUCCESS == res) {
-#if !OPAL_CUDA_GET_ATTRIBUTES
-            res = cuFunc.cuPointerGetAttribute(&memCtx,
-                                               CU_POINTER_ATTRIBUTE_CONTEXT, dbuf);
-            if (OPAL_UNLIKELY(res != CUDA_SUCCESS)) {
-                opal_output(0, "CUDA: error calling cuPointerGetAttribute: "
-                            "res=%d, ptr=%p aborting...", res, pUserBuf);
-                return OPAL_ERROR;
-            }
-#endif /* OPAL_CUDA_GET_ATTRIBUTES */
-            res = cuFunc.cuCtxSetCurrent(memCtx);
-            if (OPAL_UNLIKELY(res != CUDA_SUCCESS)) {
-                opal_output(0, "CUDA: error calling cuCtxSetCurrent: "
-                            "res=%d, ptr=%p aborting...", res, pUserBuf);
-                return OPAL_ERROR;
-            } else {
-                OPAL_OUTPUT_VERBOSE((10, mca_common_cuda_output,
-                                     "CUDA: cuCtxSetCurrent passed: ptr=%p", pUserBuf));
-            }
-        } else {
-            /* Print error and proceed */
-            opal_output(0, "CUDA: error calling cuCtxGetCurrent: "
-                        "res=%d, ptr=%p aborting...", res, pUserBuf);
-            return OPAL_ERROR;
-        }
-    }
-
-    /* WORKAROUND - They are times when the above code determines a pice of memory
-     * is GPU memory, but it actually is not.  That has been seen on multi-GPU systems
-     * with 6 or 8 GPUs on them. Therefore, we will do this extra check.  Note if we
-     * made it this far, then the assumption at this point is we have GPU memory.
-     * Unfotunately, this extra call is costing us another 100 ns almost doubling
-     * the cost of this entire function. */
-    if (OPAL_LIKELY(mca_common_cuda_gpu_mem_check_workaround)) {
-        CUdeviceptr pbase;
-        size_t psize;
-        res = cuFunc.cuMemGetAddressRange(&pbase, &psize, dbuf);
-        if (CUDA_SUCCESS != res) {
-            opal_output_verbose(5, mca_common_cuda_output,
-                                "CUDA: cuMemGetAddressRange failed on this pointer: res=%d, buf=%p "
-                                "Overriding check and setting to host pointer. ",
-                              res, (void *)dbuf);
-            /* This cannot be GPU memory if the previous call failed */
-            return 0;
-        }
-    }
-
-    /* First access on a device pointer finalizes CUDA support initialization.
-     * If initialization fails, disable support. */
-    if (!stage_three_init_complete) {
-        if (0 != mca_common_cuda_stage_three_init()) {
-            opal_cuda_support = 0;
-        }
-    }
-
-    return 1;
-}
-
-static int mca_common_cuda_cu_memcpy_async(void *dest, const void *src, size_t size,
-                                         opal_convertor_t* convertor)
-{
-    return cuFunc.cuMemcpyAsync((CUdeviceptr)dest, (CUdeviceptr)src, size,
-                                (CUstream)convertor->stream);
-}
-
-/**
- * This function is plugged into various areas where a cuMemcpy would be called.
- * This is a synchronous operation that will not return until the copy is complete.
- */
-static int mca_common_cuda_cu_memcpy(void *dest, const void *src, size_t size)
-{
-    CUresult result;
-#if OPAL_ENABLE_DEBUG
-    CUmemorytype memTypeSrc, memTypeDst;
-    if (OPAL_UNLIKELY(mca_common_cuda_cumemcpy_timing)) {
-        /* Nice to know type of source and destination for timing output. Do
-         * not care about return code as memory type will just be set to 0 */
-        result = cuFunc.cuPointerGetAttribute(&memTypeDst,
-                                              CU_POINTER_ATTRIBUTE_MEMORY_TYPE, (CUdeviceptr)dest);
-        result = cuFunc.cuPointerGetAttribute(&memTypeSrc,
-                                              CU_POINTER_ATTRIBUTE_MEMORY_TYPE, (CUdeviceptr)src);
-        ts_start = opal_timer_base_get_usec();
-    }
-#endif
-    if (mca_common_cuda_cumemcpy_async) {
-        result = cuFunc.cuMemcpyAsync((CUdeviceptr)dest, (CUdeviceptr)src, size, memcpyStream);
-        if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
-            opal_show_help("help-mpi-common-cuda.txt", "cuMemcpyAsync failed",
-                           true, dest, src, size, result);
-            return OPAL_ERROR;
-        }
-        result = cuFunc.cuStreamSynchronize(memcpyStream);
-        if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
-            opal_show_help("help-mpi-common-cuda.txt", "cuStreamSynchronize failed",
-                           true, OPAL_PROC_MY_HOSTNAME, result);
-            return OPAL_ERROR;
-        }
-    } else {
-         result = cuFunc.cuMemcpy((CUdeviceptr)dest, (CUdeviceptr)src, size);
-         if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
-             opal_show_help("help-mpi-common-cuda.txt", "cuMemcpy failed",
-                            true, OPAL_PROC_MY_HOSTNAME, result);
-             return OPAL_ERROR;
-         }
-    }
-#if OPAL_ENABLE_DEBUG
-    if (OPAL_UNLIKELY(mca_common_cuda_cumemcpy_timing)) {
-        ts_end = opal_timer_base_get_usec();
-        accum = mydifftime(ts_start, ts_end);
-        if (mca_common_cuda_cumemcpy_async) {
-            opal_output(0, "cuMemcpyAsync took   %7.2f usecs, size=%d, (src=%p (%d), dst=%p (%d))\n",
-                        accum, (int)size, src, memTypeSrc, dest, memTypeDst);
-        } else {
-            opal_output(0, "cuMemcpy took   %7.2f usecs, size=%d,  (src=%p (%d), dst=%p (%d))\n",
-                        accum, (int)size, src, memTypeSrc, dest, memTypeDst);
-        }
-    }
-#endif
-    return OPAL_SUCCESS;
-}
-
-int mca_common_cuda_malloc(void **dptr, size_t size)
-{
-    int res, count = 0;
-    if (size > 0) {
-        res = cuFunc.cuMemAlloc((CUdeviceptr *)dptr, size);
-        if (OPAL_UNLIKELY(res != CUDA_SUCCESS)) {
-            opal_output(0, "CUDA: cuMemAlloc failed: res=%d",
-                        res);
-            return res;
-        }
-    }
-    return 0;
-}
-
-int mca_common_cuda_free(void *dptr)
-{
-    int res;
-    if (NULL != dptr) {
-        res = cuFunc.cuMemFree((CUdeviceptr)dptr);
-        if (OPAL_UNLIKELY(res != CUDA_SUCCESS)) {
-            opal_output(0, "CUDA: cuMemFree failed: res=%d",
-                        res);
-            return res;
-        }
-    }
-    return 0;
-}
-
-
-static int mca_common_cuda_memmove(void *dest, void *src, size_t size)
-{
-    CUdeviceptr tmp;
-    int result;
-
-    result = cuFunc.cuMemAlloc(&tmp,size);
-    if (mca_common_cuda_cumemcpy_async) {
-        result = cuFunc.cuMemcpyAsync(tmp, (CUdeviceptr)src, size, memcpyStream);
-        if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
-            opal_show_help("help-mpi-common-cuda.txt", "cuMemcpyAsync failed",
-                           true, tmp, src, size, result);
-            return OPAL_ERROR;
-        }
-        result = cuFunc.cuMemcpyAsync((CUdeviceptr)dest, tmp, size, memcpyStream);
-        if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
-            opal_show_help("help-mpi-common-cuda.txt", "cuMemcpyAsync failed",
-                           true, dest, tmp, size, result);
-            return OPAL_ERROR;
-        }
-        result = cuFunc.cuStreamSynchronize(memcpyStream);
-        if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
-            opal_show_help("help-mpi-common-cuda.txt", "cuStreamSynchronize failed",
-                           true, OPAL_PROC_MY_HOSTNAME, result);
-            return OPAL_ERROR;
-        }
-    } else {
-        result = cuFunc.cuMemcpy(tmp, (CUdeviceptr)src, size);
-        if (OPAL_UNLIKELY(result != CUDA_SUCCESS)) {
-            opal_output(0, "CUDA: memmove-Error in cuMemcpy: res=%d, dest=%p, src=%p, size=%d",
-                        result, (void *)tmp, src, (int)size);
-            return OPAL_ERROR;
-        }
-        result = cuFunc.cuMemcpy((CUdeviceptr)dest, tmp, size);
-        if (OPAL_UNLIKELY(result != CUDA_SUCCESS)) {
-            opal_output(0, "CUDA: memmove-Error in cuMemcpy: res=%d, dest=%p, src=%p, size=%d",
-                        result, dest, (void *)tmp, (int)size);
-            return OPAL_ERROR;
-        }
-    }
-    cuFunc.cuMemFree(tmp);
-    return OPAL_SUCCESS;
-}
-
-int mca_common_cuda_get_device(int *devicenum)
-{
-    CUdevice cuDev;
-    int res;
-
-    res = cuFunc.cuCtxGetDevice(&cuDev);
-    if (OPAL_UNLIKELY(res != CUDA_SUCCESS)) {
-        opal_output(0, "CUDA: cuCtxGetDevice failed: res=%d",
-                    res);
-        return res;
-    }
-    *devicenum = cuDev;
-    return 0;
-}
-
-int mca_common_cuda_device_can_access_peer(int *access, int dev1, int dev2)
-{
-    int res;
-    res = cuFunc.cuDeviceCanAccessPeer(access, (CUdevice)dev1, (CUdevice)dev2);
-    if (OPAL_UNLIKELY(res != CUDA_SUCCESS)) {
-        opal_output(0, "CUDA: cuDeviceCanAccessPeer failed: res=%d",
-                    res);
-        return res;
-    }
-    return 0;
-}
-
-int mca_common_cuda_get_address_range(void *pbase, size_t *psize, void *base)
-{
-    CUresult result;
-    result = cuFunc.cuMemGetAddressRange((CUdeviceptr *)pbase, psize, (CUdeviceptr)base);
-    if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
-        opal_show_help("help-mpi-common-cuda.txt", "cuMemGetAddressRange failed 2",
-                       true, OPAL_PROC_MY_HOSTNAME, result, base);
-        return OPAL_ERROR;
-    } else {
-        opal_output_verbose(50, mca_common_cuda_output,
-                            "CUDA: cuMemGetAddressRange passed: addr=%p, pbase=%p, psize=%lu ",
-                            base, *(char **)pbase, *psize);
-    }
-    return 0;
-}
-
-#if OPAL_CUDA_GDR_SUPPORT
-/* Check to see if the memory was freed between the time it was stored in
- * the registration cache and now.  Return true if the memory was previously
- * freed.  This is indicated by the BUFFER_ID value in the registration cache
- * not matching the BUFFER_ID of the buffer we are checking.  Return false
- * if the registration is still good.
- */
-bool mca_common_cuda_previously_freed_memory(mca_rcache_base_registration_t *reg)
-{
-    int res;
-    unsigned long long bufID;
-    unsigned char *dbuf = reg->base;
-
-    res = cuFunc.cuPointerGetAttribute(&bufID, CU_POINTER_ATTRIBUTE_BUFFER_ID,
-                                       (CUdeviceptr)dbuf);
-    /* If we cannot determine the BUFFER_ID, then print a message and default
-     * to forcing the registration to be kicked out. */
-    if (OPAL_UNLIKELY(res != CUDA_SUCCESS)) {
-        opal_show_help("help-mpi-common-cuda.txt", "bufferID failed",
-                       true, OPAL_PROC_MY_HOSTNAME, res);
-        return true;
-    }
-    opal_output_verbose(50, mca_common_cuda_output,
-                        "CUDA: base=%p, bufID=%llu, reg->gpu_bufID=%llu, %s", dbuf, bufID, reg->gpu_bufID,
-                        (reg->gpu_bufID == bufID ? "BUFFER_ID match":"BUFFER_ID do not match"));
-    if (bufID != reg->gpu_bufID) {
-        return true;
-    } else {
-        return false;
-    }
-}
-
-/*
- * Get the buffer ID from the memory and store it in the registration.
- * This is needed to ensure the cached registration is not stale.  If
- * we fail to get buffer ID, print an error and set buffer ID to 0.
- * Also set SYNC_MEMOPS on any GPU registration to ensure that
- * synchronous copies complete before the buffer is accessed.
- */
-void mca_common_cuda_get_buffer_id(mca_rcache_base_registration_t *reg)
-{
-    int res;
-    unsigned long long bufID = 0;
-    unsigned char *dbuf = reg->base;
-    int enable = 1;
-
-    res = cuFunc.cuPointerGetAttribute(&bufID, CU_POINTER_ATTRIBUTE_BUFFER_ID,
-                                       (CUdeviceptr)dbuf);
-    if (OPAL_UNLIKELY(res != CUDA_SUCCESS)) {
-        opal_show_help("help-mpi-common-cuda.txt", "bufferID failed",
-                       true, OPAL_PROC_MY_HOSTNAME, res);
-    }
-    reg->gpu_bufID = bufID;
-
-    res = cuFunc.cuPointerSetAttribute(&enable, CU_POINTER_ATTRIBUTE_SYNC_MEMOPS,
-                                       (CUdeviceptr)dbuf);
-    if (OPAL_UNLIKELY(CUDA_SUCCESS != res)) {
-        opal_show_help("help-mpi-common-cuda.txt", "cuPointerSetAttribute failed",
-                       true, OPAL_PROC_MY_HOSTNAME, res, dbuf);
-    }
-}
-
-static bool initialized = false;
-int opal_cuda_verbose = 0;
-static int opal_cuda_enabled = 0; /* Starts out disabled */
-static int opal_cuda_output = 0;
-static void opal_cuda_support_init(void);
-static int (*common_cuda_initialization_function)(opal_common_cuda_function_table_t *) = NULL;
-static opal_common_cuda_function_table_t ftable;
-
-/* This function allows the common cuda code to register an
- * initialization function that gets called the first time an attempt
- * is made to send or receive a GPU pointer.  This allows us to delay
- * some CUDA initialization until after MPI_Init().
- */
-void opal_cuda_add_initialization_function(int (*fptr)(opal_common_cuda_function_table_t *)) {
-    common_cuda_initialization_function = fptr;
-}
-
-/**
- * This function is called when a convertor is instantiated.  It has to call
- * the opal_cuda_support_init() function once to figure out if CUDA support
- * is enabled or not.  If CUDA is not enabled, then short circuit out
- * for all future calls.
- */
-void mca_cuda_convertor_init(opal_convertor_t* convertor, const void *pUserBuf)
-{
-    /* Only do the initialization on the first GPU access */
-    if (!initialized) {
-        opal_cuda_support_init();
-    }
-
-    /* This is needed to handle case where convertor is not fully initialized
-     * like when trying to do a sendi with convertor on the statck */
-    convertor->cbmemcpy = (memcpy_fct_t)&opal_cuda_memcpy;
-
-    /* If not enabled, then nothing else to do */
-    if (!opal_cuda_enabled) {
-        return;
-    }
-
-    if (ftable.gpu_is_gpu_buffer(pUserBuf, convertor)) {
-        convertor->flags |= CONVERTOR_CUDA;
-    }
-}
-
-/* Checks the type of pointer
- *
- * @param dest   One pointer to check
- * @param source Another pointer to check
- */
-bool opal_cuda_check_bufs(char *dest, char *src)
-{
-    /* Only do the initialization on the first GPU access */
-    if (!initialized) {
-        opal_cuda_support_init();
-    }
-
-    if (!opal_cuda_enabled) {
-        return false;
-    }
-
-    if (ftable.gpu_is_gpu_buffer(dest, NULL) || ftable.gpu_is_gpu_buffer(src, NULL)) {
-        return true;
-    } else {
-        return false;
-    }
-}
-
-/*
- * With CUDA enabled, all contiguous copies will pass through this function.
- * Therefore, the first check is to see if the convertor is a GPU buffer.
- * Note that if there is an error with any of the CUDA calls, the program
- * aborts as there is no recovering.
- */
-
-/* Checks the type of pointer
- *
- * @param buf   check one pointer providing a convertor.
- *  Provides aditional information, e.g. managed vs. unmanaged GPU buffer
- */
-bool  opal_cuda_check_one_buf(char *buf, opal_convertor_t *convertor )
-{
-    /* Only do the initialization on the first GPU access */
-    if (!initialized) {
-        opal_cuda_support_init();
-    }
-
-    if (!opal_cuda_enabled) {
-        return false;
-    }
-
-    return ( ftable.gpu_is_gpu_buffer(buf, convertor));
-}
-
-/*
- * This function allocates a buffer using either cuMemAlloc
- * or malloc, depending on if the convertor flag CONVERTOR_CUDA
- * is set.
- *
- * @param size       Size of buffer to be allocated
- * @param convertor  The convertor with flags describing if the buf
- *                   should be a Host or Cuda buffer.
- *
- * @returns void *   A pointer to the newly allocated buffer.
- */
-void *opal_cuda_malloc(size_t size, opal_convertor_t* convertor)
-{
-    int res;
-    void* buffer;
-    if (!(convertor->flags & CONVERTOR_CUDA)) {
-        return malloc(size);
-    }
-    res = ftable.gpu_malloc(buffer, size);
-    if (res != 0 ) {
-        opal_output(0, "CUDA: Error in cuMemAlloc: size=%d",
-                    (int)size);
-        abort();
-    } else {
-        return buffer;
-    }
-}
-
-/*
- * This function frees a buffer using either cuMemFree() or free(),
- * depending on if the convertor flag CONVERTOR_CUDA is set.
- *
- * @param buffer     Pointer to buffer to be freed
- * @param convertor  The convertor with flags describing if the buf
- *                   should be a Host or Cuda buffer.
- *
- */
-void opal_cuda_free(void *buffer, opal_convertor_t* convertor)
-{
-    int res;
-    if (!(convertor->flags & CONVERTOR_CUDA)) {
-        free(buffer);
-        return;
-    }
-    res = ftable.gpu_free(buffer);
-    if (res != 0 ) {
-        opal_output(0, "CUDA: Error in cuMemFree: ptr=%p",
-                    buffer);
-        abort();
-    }
-    return;
-}
-
-/*
- * With CUDA enabled, all contiguous copies will pass through this function.
- * Therefore, the first check is to see if the convertor is a GPU buffer.
- * Note that if there is an error with any of the CUDA calls, the program
- * aborts as there is no recovering.
- */
-
-void *opal_cuda_memcpy(void *dest, const void *src, size_t size, opal_convertor_t* convertor)
-{
-    int res;
-
-    if (!(convertor->flags & CONVERTOR_CUDA)) {
-        return memcpy(dest, src, size);
-    }
-
-    if (convertor->flags & CONVERTOR_CUDA_ASYNC) {
-        res = ftable.gpu_cu_memcpy_async(dest, (void *)src, size, convertor);
-    } else {
-        res = ftable.gpu_cu_memcpy(dest, (void *)src, size);
-    }
-
-    if (res != 0) {
-        opal_output(0, "CUDA: Error in cuMemcpy: res=%d, dest=%p, src=%p, size=%d",
-                    res, dest, src, (int)size);
-        abort();
-    } else {
-        return dest;
-    }
-}
-
-/*
- * This function is needed in cases where we do not have contiguous
- * datatypes.  The current code has macros that cannot handle a convertor
- * argument to the memcpy call.
- */
-void *opal_cuda_memcpy_sync(void *dest, const void *src, size_t size)
-{
-    int res;
-    res = ftable.gpu_cu_memcpy(dest, src, size);
-    if (res != 0) {
-        opal_output(0, "CUDA: Error in cuMemcpy: res=%d, dest=%p, src=%p, size=%d",
-                    res, dest, src, (int)size);
-        abort();
-    } else {
-        return dest;
-    }
-}
-
-/*
- * In some cases, need an implementation of memmove.  This is not fast, but
- * it is not often needed.
- */
-void *opal_cuda_memmove(void *dest, void *src, size_t size)
-{
-    int res;
-
-    res = ftable.gpu_memmove(dest, src, size);
-    if(res != 0){
-        opal_output(0, "CUDA: Error in gpu memmove: res=%d, dest=%p, src=%p, size=%d",
-                    res, dest, src, (int)size);
-        abort();
-    }
-    return dest;
-}
-
-/**
- * This function gets called once to check if the program is running in a cuda
- * environment.
- */
-static void opal_cuda_support_init(void)
-{
-    if (initialized) {
-        return;
-    }
-
-    /* Set different levels of verbosity in the cuda related code. */
-    opal_cuda_output = opal_output_open(NULL);
-    opal_output_set_verbosity(opal_cuda_output, opal_cuda_verbose);
-
-    /* Callback into the common cuda initialization routine. This is only
-     * set if some work had been done already in the common cuda code.*/
-    if (NULL != common_cuda_initialization_function) {
-        if (0 == common_cuda_initialization_function(&ftable)) {
-            opal_cuda_enabled = 1;
-        }
-    }
-
-    if (1 == opal_cuda_enabled) {
-        opal_output_verbose(10, opal_cuda_output,
-                            "CUDA: enabled successfully, CUDA device pointers will work");
-    } else {
-        opal_output_verbose(10, opal_cuda_output,
-                            "CUDA: not enabled, CUDA device pointers will not work");
-    }
-
-    initialized = true;
-}
-
-/**
- * Tell the convertor that copies will be asynchronous CUDA copies.  The
- * flags are cleared when the convertor is reinitialized.
- */
-void opal_cuda_set_copy_function_async(opal_convertor_t* convertor, void *stream)
-{
-    convertor->flags |= CONVERTOR_CUDA_ASYNC;
-    convertor->stream = stream;
-}
-#endif /* OPAL_CUDA_GDR_SUPPORT */
diff --git a/opal/mca/common/cuda/common_cuda.h b/opal/mca/common/cuda/common_cuda.h
deleted file mode 100644
index 695201bc134..00000000000
--- a/opal/mca/common/cuda/common_cuda.h
+++ /dev/null
@@ -1,138 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2013 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2006 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2011-2015 NVIDIA Corporation.  All rights reserved.
- * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef OPAL_MCA_COMMON_CUDA_H
-#define OPAL_MCA_COMMON_CUDA_H
-#include "opal/mca/btl/btl.h"
-#include "opal/datatype/opal_convertor.h"
-
-#define MEMHANDLE_SIZE 8
-#define EVTHANDLE_SIZE 8
-
-struct mca_rcache_common_cuda_reg_data_t {
-    uint64_t memHandle[MEMHANDLE_SIZE];
-    uint64_t evtHandle[EVTHANDLE_SIZE];
-    uint64_t event;
-    opal_ptr_t memh_seg_addr;
-    size_t memh_seg_len;
-};
-typedef struct mca_rcache_common_cuda_reg_data_t mca_rcache_common_cuda_reg_data_t;
-
-struct mca_rcache_common_cuda_reg_t {
-    mca_rcache_base_registration_t base;
-    mca_rcache_common_cuda_reg_data_t data;
-};
-typedef struct mca_rcache_common_cuda_reg_t mca_rcache_common_cuda_reg_t;
-extern bool mca_common_cuda_enabled;
-
-OPAL_DECLSPEC void mca_common_cuda_register_mca_variables(void);
-
-OPAL_DECLSPEC void mca_common_cuda_register(void *ptr, size_t amount, char *msg);
-
-OPAL_DECLSPEC void mca_common_cuda_unregister(void *ptr, char *msg);
-
-OPAL_DECLSPEC void mca_common_wait_stream_synchronize(mca_rcache_common_cuda_reg_t *rget_reg);
-
-OPAL_DECLSPEC int mca_common_cuda_malloc(void **buffer, size_t size);
-OPAL_DECLSPEC int mca_common_cuda_free(void *buffer);
-
-OPAL_DECLSPEC int mca_common_cuda_memcpy(void *dst, void *src, size_t amount, char *msg,
-                                         struct mca_btl_base_descriptor_t *, int *done);
-
-OPAL_DECLSPEC int mca_common_cuda_record_ipc_event(char *msg,
-                                               struct mca_btl_base_descriptor_t *frag);
-OPAL_DECLSPEC int mca_common_cuda_record_dtoh_event(char *msg,
-                                                    struct mca_btl_base_descriptor_t *frag);
-OPAL_DECLSPEC int mca_common_cuda_record_htod_event(char *msg,
-                                                    struct mca_btl_base_descriptor_t *frag);
-
-OPAL_DECLSPEC void *mca_common_cuda_get_dtoh_stream(void);
-OPAL_DECLSPEC void *mca_common_cuda_get_htod_stream(void);
-
-OPAL_DECLSPEC int progress_one_cuda_ipc_event(struct mca_btl_base_descriptor_t **);
-OPAL_DECLSPEC int progress_one_cuda_dtoh_event(struct mca_btl_base_descriptor_t **);
-OPAL_DECLSPEC int progress_one_cuda_htod_event(struct mca_btl_base_descriptor_t **);
-
-OPAL_DECLSPEC int mca_common_cuda_memhandle_matches(mca_rcache_common_cuda_reg_t *new_reg,
-                                                    mca_rcache_common_cuda_reg_t *old_reg);
-
-OPAL_DECLSPEC void mca_common_cuda_construct_event_and_handle(uintptr_t *event, void *handle);
-OPAL_DECLSPEC void mca_common_cuda_destruct_event(uintptr_t event);
-
-OPAL_DECLSPEC int cuda_getmemhandle(void *base, size_t, mca_rcache_base_registration_t *newreg,
-                                    mca_rcache_base_registration_t *hdrreg);
-OPAL_DECLSPEC int cuda_ungetmemhandle(void *reg_data, mca_rcache_base_registration_t *reg);
-OPAL_DECLSPEC int cuda_openmemhandle(void *base, size_t size, mca_rcache_base_registration_t *newreg,
-                                     mca_rcache_base_registration_t *hdrreg);
-OPAL_DECLSPEC int cuda_closememhandle(void *reg_data, mca_rcache_base_registration_t *reg);
-OPAL_DECLSPEC int mca_common_cuda_get_device(int *devicenum);
-OPAL_DECLSPEC int mca_common_cuda_device_can_access_peer(int *access, int dev1, int dev2);
-OPAL_DECLSPEC int mca_common_cuda_stage_one_init(void);
-OPAL_DECLSPEC int mca_common_cuda_get_address_range(void *pbase, size_t *psize, void *base);
-OPAL_DECLSPEC void mca_common_cuda_fini(void);
-#if OPAL_CUDA_GDR_SUPPORT
-OPAL_DECLSPEC bool mca_common_cuda_previously_freed_memory(mca_rcache_base_registration_t *reg);
-OPAL_DECLSPEC void mca_common_cuda_get_buffer_id(mca_rcache_base_registration_t *reg);
-#endif /* OPAL_CUDA_GDR_SUPPORT */
-/**
- * Return:   0 if no packing is required for sending (the upper layer
- *             can use directly the pointer to the contiguous user
- *             buffer).
- *           1 if data does need to be packed, i.e. heterogeneous peers
- *             (source arch != dest arch) or non contiguous memory
- *             layout.
- */
-static inline int32_t opal_convertor_cuda_need_buffers( opal_convertor_t* pConvertor )
-{
-    int32_t retval;
-    uint32_t cudaflag = pConvertor->flags & CONVERTOR_CUDA; /* Save CUDA flag */
-    pConvertor->flags &= ~CONVERTOR_CUDA;              /* Clear CUDA flag if it exists */
-    retval = opal_convertor_need_buffers(pConvertor);
-    pConvertor->flags |= cudaflag; /* Restore CUDA flag */
-    return retval;
-}
-
-/* Structure to hold CUDA support functions that gets filled in when the
- * common cuda code is initialized.  This removes any dependency on <cuda.h>
- * in the opal cuda datatype code. */
-struct opal_common_cuda_function_table {
-    int (*gpu_is_gpu_buffer)(const void*, opal_convertor_t*);
-    int (*gpu_cu_memcpy_async)(void*, const void*, size_t, opal_convertor_t*);
-    int (*gpu_cu_memcpy)(void*, const void*, size_t);
-    int (*gpu_memmove)(void*, void*, size_t);
-    int (*gpu_malloc)(void*, size_t);
-    int (*gpu_free)(void*);
-};
-typedef struct opal_common_cuda_function_table opal_common_cuda_function_table_t;
-
-void mca_cuda_convertor_init(opal_convertor_t* convertor, const void *pUserBuf);
-bool opal_cuda_check_bufs(char *dest, char *src);
-bool opal_cuda_check_one_buf(char *buf, opal_convertor_t *convertor );
-void* opal_cuda_malloc(size_t size, opal_convertor_t* convertor);
-void opal_cuda_free(void * buffer, opal_convertor_t* convertor);
-void* opal_cuda_memcpy(void * dest, const void * src, size_t size, opal_convertor_t* convertor);
-void* opal_cuda_memcpy_sync(void * dest, const void * src, size_t size);
-void* opal_cuda_memmove(void * dest, void * src, size_t size);
-void opal_cuda_add_initialization_function(int (*fptr)(opal_common_cuda_function_table_t *));
-void opal_cuda_set_copy_function_async(opal_convertor_t* convertor, void *stream);
-
-#endif /* OPAL_MCA_COMMON_CUDA_H */
diff --git a/opal/mca/common/cuda/configure.m4 b/opal/mca/common/cuda/configure.m4
deleted file mode 100644
index 5bda090e26f..00000000000
--- a/opal/mca/common/cuda/configure.m4
+++ /dev/null
@@ -1,30 +0,0 @@
-# -*- shell-script -*-
-#
-# Copyright (c) 2011-2013 NVIDIA Corporation.  All rights reserved.
-# Copyright (c) 2013      The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-#
-# If CUDA support was requested, then build the CUDA support library.
-# This code checks just makes sure the check was done earlier by the
-# opal_check_cuda.m4 code.
-#
-
-AC_DEFUN([MCA_opal_common_cuda_CONFIG],[
-    AC_CONFIG_FILES([opal/mca/common/cuda/Makefile])
-
-    # make sure that CUDA-aware checks have been done
-    AC_REQUIRE([OPAL_CHECK_CUDA])
-
-    AS_IF([test "x$CUDA_SUPPORT" = "x1"],
-          [$1],
-          [$2])
-
-])dnl
diff --git a/opal/mca/common/cuda/help-mpi-common-cuda.txt b/opal/mca/common/cuda/help-mpi-common-cuda.txt
deleted file mode 100644
index e6f7913316b..00000000000
--- a/opal/mca/common/cuda/help-mpi-common-cuda.txt
+++ /dev/null
@@ -1,212 +0,0 @@
-# -*- text -*-
-#
-# Copyright (c) 2011-2015 NVIDIA.  All rights reserved.
-# Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-[cuCtxGetCurrent failed not initialized]
-WARNING: The call to cuCtxGetCurrent() failed while attempting to register
-internal memory with the CUDA environment.  The program will continue to run,
-but the performance of GPU memory transfers may be reduced.  This failure
-indicates that the CUDA environment is not yet initialized.  To eliminate
-this warning, ensure that CUDA is initialized prior to calling MPI_Init.
-
-NOTE: You can turn off this warning by setting the MCA parameter
-      mpi_common_cuda_warning to 0.
-#
-[cuCtxGetCurrent failed]
-WARNING: The call to cuCtxGetCurrent() failed while attempting to register
-internal memory with the CUDA environment.  The program will continue to run,
-but the performance of GPU memory transfers may be reduced.
-  cuCtxGetCurrent return value:   %d
-
-NOTE: You can turn off this warning by setting the MCA parameter
-      mpi_common_cuda_warning to 0.
-#
-[cuCtxGetCurrent returned NULL]
-WARNING: The call to cuCtxGetCurrent() failed while attempting to register
-internal memory with the CUDA environment.  The program will continue to run,
-but the performance of GPU memory transfers may be reduced.  This failure
-indicates that there is no CUDA context yet.  To eliminate this warning,
-ensure that there is a CUDA context prior to calling MPI_Init.
-
-NOTE: You can turn off this warning by setting the MCA parameter
-      mpi_common_cuda_warning to 0.
-#
-[cuMemHostRegister during init failed]
-The call to cuMemHostRegister(%p, %d, 0) failed.
-  Host:  %s
-  cuMemHostRegister return value:  %d
-  Registration cache:  %s
-#
-[cuMemHostRegister failed]
-The call to cuMemHostRegister(%p, %d, 0) failed.
-  Host:  %s
-  cuMemHostRegister return value:  %d
-  Registration cache:  %s
-#
-[cuIpcGetMemHandle failed]
-The call to cuIpcGetMemHandle failed. This means the GPU RDMA protocol
-cannot be used.
-  cuIpcGetMemHandle return value:   %d
-  address: %p
-Check the cuda.h file for what the return value means. Perhaps a reboot
-of the node will clear the problem.
-#
-[cuMemGetAddressRange failed]
-The call to cuMemGetAddressRange failed. This means the GPU RDMA protocol
-cannot be used.
-  cuMemGetAddressRange return value:   %d
-  address: %p
-Check the cuda.h file for what the return value means. Perhaps a reboot
-of the node will clear the problem.
-#
-[cuMemGetAddressRange failed 2]
-The call to cuMemGetAddressRange failed during the GPU RDMA protocol.
-  Host:  %s
-  cuMemGetAddressRange return value:  %d
-  address:  %p
-Check the cuda.h file for what the return value means. This is highly
-unusual and should not happen. The program will probably abort.
-#
-[Out of cuEvent handles]
-The library has exceeded its number of outstanding event handles.
-For better performance, this number should be increased.
-  Current maximum handles:   %4d
-  Suggested new maximum:     %4d
-Rerun with --mca mpi_common_cuda_event_max %d
-#
-[cuIpcOpenMemHandle failed]
-The call to cuIpcOpenMemHandle failed. This is an unrecoverable error
-and will cause the program to abort.
-  Hostname:                         %s
-  cuIpcOpenMemHandle return value:  %d
-  address:                          %p
-Check the cuda.h file for what the return value means. A possible cause
-for this is not enough free device memory.  Try to reduce the device
-memory footprint of your application.
-#
-[cuIpcCloseMemHandle failed]
-The call to cuIpcCloseMemHandle failed. This is a warning and the program
-will continue to run.
-  cuIpcCloseMemHandle return value:   %d
-  address: %p
-Check the cuda.h file for what the return value means. Perhaps a reboot
-of the node will clear the problem.
-#
-[cuMemcpyAsync failed]
-The call to cuMemcpyAsync failed. This is a unrecoverable error and will
-cause the program to abort.
-  cuMemcpyAsync(%p, %p, %d) returned value %d
-Check the cuda.h file for what the return value means.
-#
-[cuEventCreate failed]
-The call to cuEventCreate failed. This is a unrecoverable error and will
-cause the program to abort.
-  Hostname:                     %s
-  cuEventCreate return value:   %d
-Check the cuda.h file for what the return value means.
-#
-[cuEventRecord failed]
-The call to cuEventRecord failed. This is a unrecoverable error and will
-cause the program to abort.
-  Hostname:                     %s
-  cuEventRecord return value:   %d
-Check the cuda.h file for what the return value means.
-#
-[cuEventQuery failed]
-The call to cuEventQuery failed. This is a unrecoverable error and will
-cause the program to abort.
-  cuEventQuery return value:   %d
-Check the cuda.h file for what the return value means.
-#
-[cuIpcGetEventHandle failed]
-The call to cuIpcGetEventHandle failed. This is a unrecoverable error and will
-cause the program to abort.
-  cuIpcGetEventHandle return value:   %d
-Check the cuda.h file for what the return value means.
-#
-[cuIpcOpenEventHandle failed]
-The call to cuIpcOpenEventHandle failed. This is a unrecoverable error and will
-cause the program to abort.
-  cuIpcOpenEventHandle return value:   %d
-Check the cuda.h file for what the return value means.
-#
-[cuStreamWaitEvent failed]
-The call to cuStreamWaitEvent failed. This is a unrecoverable error and will
-cause the program to abort.
-  cuStreamWaitEvent return value:   %d
-Check the cuda.h file for what the return value means.
-#
-[cuEventDestroy failed]
-The call to cuEventDestory failed. This is a unrecoverable error and will
-cause the program to abort.
-  cuEventDestory return value:   %d
-Check the cuda.h file for what the return value means.
-#
-[cuStreamCreate failed]
-The call to cuStreamCreate failed.  This is a unrecoverable error and will
-cause the program to abort.
-  Hostname:                      %s
-  cuStreamCreate return value:   %d
-Check the cuda.h file for what the return vale means.
-#
-[dlopen disabled]
-Open MPI was compiled without dynamic library support (e.g., with the
- --disable-dlopen flag), and therefore cannot utilize CUDA support.
-
-If you need CUDA support, reconfigure Open MPI with dynamic library support enabled.
-#
-[dlopen failed]
-The library attempted to open the following supporting CUDA libraries,
-but each of them failed.  CUDA-aware support is disabled.
-%s
-If you do not require CUDA-aware support, then run with
---mca opal_warn_on_missing_libcuda 0 to suppress this message.  If you do
-require CUDA-aware support, then try setting LD_LIBRARY_PATH to the location
-of libcuda.so.1 to resolve this issue.
-#
-[dlsym failed]
-An error occurred while trying to map in the address of a function.
-  Function Name: %s
-  Error string:  %s
-CUDA-aware support is disabled.
-#
-[bufferID failed]
-An error occurred while trying to get the BUFFER_ID of a GPU memory
-region.  This could cause incorrect results.  Turn of GPU Direct RDMA
-support by running with --mca btl_openib_cuda_want_gdr_support 0.
-  Hostname:                             %s
-  cuPointerGetAttribute return value:   %d
-Check the cuda.h file for what the return value means.
-[cuPointerSetAttribute failed]
-The call to cuPointerSetAttribute with CU_POINTER_ATTRIBUTE_SYNC_MEMOPS
-failed. This is highly unusual and should not happen.  The program will
-continue, but report this error to the Open MPI developers.
-  Hostname:                             %s
-  cuPointerSetAttribute return value:   %d
-  Address:                              %p
-Check the cuda.h file for what the return value means.
-#
-[cuStreamSynchronize failed]
-The call to cuStreamSynchronize failed. This is highly unusual and should
-not happen.  Please report this error to the Open MPI developers.
-  Hostname:                             %s
-  cuStreamSynchronize return value:     %d
-Check the cuda.h file for what the return value means.
-#
-[cuMemcpy failed]
-The call to cuMemcpy failed. This is highly unusual and should
-not happen.  Please report this error to the Open MPI developers.
-  Hostname:                  %s
-  cuMemcpy return value:     %d
-Check the cuda.h file for what the return value means.
-#
-[No memory]
-A call to allocate memory within the CUDA support failed.  This is
-an unrecoverable error and will cause the program to abort.
-  Hostname:  %s
diff --git a/opal/mca/common/cuda/owner.txt b/opal/mca/common/cuda/owner.txt
deleted file mode 100644
index 9a3b6b5a6d4..00000000000
--- a/opal/mca/common/cuda/owner.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-#
-# owner/status file
-# owner: institution that is responsible for this package
-# status: e.g. active, maintenance, unmaintained
-#
-owner: NVIDIA
-status:active
diff --git a/opal/mca/common/ofi/Makefile.am b/opal/mca/common/ofi/Makefile.am
index b2f22a7176c..81d2a88e40b 100644
--- a/opal/mca/common/ofi/Makefile.am
+++ b/opal/mca/common/ofi/Makefile.am
@@ -15,6 +15,8 @@
 # Copyright (c) 2017      Los Alamos National Security, LLC.  All rights
 #                         reserved.
 # Copyright (c) 2019      Hewlett Packard Enterprise. All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -29,9 +31,9 @@
 # Note that building this common component statically and linking
 # against other dynamic components is *not* supported!
 
-AM_CPPFLAGS = $(opal_ofi_CPPFLAGS)
+AM_CPPFLAGS = $(common_ofi_CPPFLAGS)
 
-dist_opaldata_DATA = help-common-ofi.txt
+EXTRA_DIST = help-common-ofi.txt
 
 # Header files
 
@@ -60,8 +62,8 @@ sources = \
 
 lib_LTLIBRARIES =
 noinst_LTLIBRARIES =
-comp_inst = lib@OPAL_LIB_PREFIX@mca_common_ofi.la
-comp_noinst = lib@OPAL_LIB_PREFIX@mca_common_ofi_noinst.la
+comp_inst = lib@OPAL_LIB_NAME@mca_common_ofi.la
+comp_noinst = lib@OPAL_LIB_NAME@mca_common_ofi_noinst.la
 
 
 if MCA_BUILD_opal_common_ofi_DSO
@@ -70,15 +72,15 @@ else
 noinst_LTLIBRARIES += $(comp_noinst)
 endif
 
-lib@OPAL_LIB_PREFIX@mca_common_ofi_la_SOURCES = $(headers) $(sources)
-lib@OPAL_LIB_PREFIX@mca_common_ofi_la_LDFLAGS = \
-        $(opal_ofi_LDFLAGS) \
+lib@OPAL_LIB_NAME@mca_common_ofi_la_SOURCES = $(headers) $(sources)
+lib@OPAL_LIB_NAME@mca_common_ofi_la_LDFLAGS = \
+        $(common_ofi_LDFLAGS) \
         -version-info $(libmca_opal_common_ofi_so_version)
-lib@OPAL_LIB_PREFIX@mca_common_ofi_la_LIBADD = $(opal_ofi_LIBS)
+lib@OPAL_LIB_NAME@mca_common_ofi_la_LIBADD = $(common_ofi_LIBS)
 
-lib@OPAL_LIB_PREFIX@mca_common_ofi_noinst_la_SOURCES = $(headers) $(sources)
-lib@OPAL_LIB_PREFIX@mca_common_ofi_noinst_la_LDFLAGS = $(opal_ofi_LDFLAGS)
-lib@OPAL_LIB_PREFIX@mca_common_ofi_noinst_la_LIBADD = $(opal_ofi_LIBS)
+lib@OPAL_LIB_NAME@mca_common_ofi_noinst_la_SOURCES = $(headers) $(sources)
+lib@OPAL_LIB_NAME@mca_common_ofi_noinst_la_LDFLAGS = $(common_ofi_LDFLAGS)
+lib@OPAL_LIB_NAME@mca_common_ofi_noinst_la_LIBADD = $(common_ofi_LIBS)
 
 # Conditionally install the header files
 
diff --git a/opal/mca/common/ofi/common_ofi.c b/opal/mca/common/ofi/common_ofi.c
index 3630aaec5ce..f6195b41af3 100644
--- a/opal/mca/common/ofi/common_ofi.c
+++ b/opal/mca/common/ofi/common_ofi.c
@@ -2,10 +2,13 @@
  * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2017      Los Alamos National Security, LLC.  All rights
  *                         reserved.
- * Copyright (c) 2020      Triad National Security, LLC. All rights
+ * Copyright (c) 2020-2022 Triad National Security, LLC. All rights
  *                         reserved.
- * Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved
- * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2020-2021 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2021-2023 Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates. All rights
+ *                         reserved.
+ * Copyright (c) 2023      UT-Battelle, LLC.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -14,30 +17,255 @@
  */
 
 
+#include "opal_config.h"
+
 #include <errno.h>
 #include <unistd.h>
+#include <rdma/fabric.h>
+#include <rdma/fi_errno.h>
+#include <rdma/fi_cm.h>
+#ifdef HAVE_RDMA_FI_EXT_H
+#include <rdma/fi_ext.h>
+#endif
 
-#include "opal_config.h"
 #include "common_ofi.h"
-#include "opal_config.h"
 #include "opal/constants.h"
-#include "opal/util/argv.h"
-#include "opal/mca/base/mca_base_var.h"
+#include "opal/mca/accelerator/accelerator.h"
 #include "opal/mca/base/mca_base_framework.h"
+#include "opal/mca/base/mca_base_var.h"
 #include "opal/mca/hwloc/base/base.h"
+#include "opal/mca/memory/base/base.h"
 #include "opal/mca/pmix/base/base.h"
+#include "opal/util/argv.h"
 #include "opal/util/show_help.h"
 
-OPAL_DECLSPEC opal_common_ofi_module_t opal_common_ofi = {
-    .prov_include = NULL,
-    .prov_exclude = NULL,
-    .registered = 0,
-    .verbose = 0
+extern opal_accelerator_base_module_t opal_accelerator;
+opal_common_ofi_module_t opal_common_ofi = {.prov_include = NULL,
+                                            .prov_exclude = NULL,
+                                            .output = -1};
+static const char default_prov_exclude_list[] = "shm,sockets,tcp,udp,rstream,usnic,net";
+static opal_mutex_t opal_common_ofi_mutex = OPAL_MUTEX_STATIC_INIT;
+static int opal_common_ofi_verbose_level = 0;
+static int opal_common_ofi_init_ref_cnt = 0;
+static int opal_common_ofi_accelerator_rank = -1;
+#ifdef HAVE_STRUCT_FI_OPS_MEM_MONITOR
+static bool opal_common_ofi_installed_memory_monitor = false;
+#endif
+
+/* Count providers returns the number of providers present in an fi_info list
+ *     @param (IN) provider_list    struct fi_info* list of providers available
+ *
+ *     @param (OUT)                 int number of providers present in the list
+ *
+ *     returns 0 if the list is NULL
+ */
+static int count_providers(struct fi_info *provider_list);
+
+#ifdef HAVE_STRUCT_FI_OPS_MEM_MONITOR
+
+/*
+ * Monitor object to export into Libfabric to provide memory release
+ * notifications using our own memory hooks framework.  Monitors may
+ * use the subscribe/unsubscribe notifications to reduce unnecessary
+ * notifications, but are not required to do so.  Because patcher
+ * notifies about all releases, it is cheaper for us to not filter and
+ * this monitor can safely ignore subscribe/unsubscribe notifications.
+ *
+ * Libfabric requires the object to be fully defined.  Unlike most of
+ * Open MPI, it does not have NULL function pointer checks in calling
+ * code.
+ */
+static int opal_common_ofi_monitor_start(struct fid_mem_monitor *monitor)
+{
+    return 0;
+}
+
+static void opal_common_ofi_monitor_stop(struct fid_mem_monitor *monitor)
+{
+    return;
+}
+
+static int opal_common_ofi_monitor_subscribe(struct fid_mem_monitor *monitor,
+                                             const void *addr, size_t len)
+{
+    return 0;
+}
+
+static void opal_common_ofi_monitor_unsubscribe(struct fid_mem_monitor *monitor,
+                                                const void *addr, size_t len)
+{
+    return;
+}
+
+static bool opal_common_ofi_monitor_valid(struct fid_mem_monitor *monitor,
+                                     const void *addr, size_t len)
+{
+    return true;
+}
+
+static struct fid_mem_monitor *opal_common_ofi_monitor = NULL;
+static struct fid *opal_common_ofi_cache_fid = NULL;
+static struct fi_ops_mem_monitor opal_common_ofi_export_ops = {
+    .size = sizeof(struct fi_ops_mem_monitor),
+    .start = opal_common_ofi_monitor_start,
+    .stop = opal_common_ofi_monitor_stop,
+    .subscribe = opal_common_ofi_monitor_subscribe,
+    .unsubscribe = opal_common_ofi_monitor_unsubscribe,
+    .valid = opal_common_ofi_monitor_valid,
 };
 
-static const char default_prov_exclude_list[] = "shm,sockets,tcp,udp,rstream";
+/**
+ * Callback function from Open MPI memory monitor
+ *
+ * Translation function between the callback function from Open MPI's
+ * memory notifier to the Libfabric memory monitor.
+ */
+static void opal_common_ofi_mem_release_cb(void *buf, size_t length,
+                                           void *cbdata, bool from_alloc)
+{
+    opal_common_ofi_monitor->import_ops->notify(opal_common_ofi_monitor,
+                                                buf, length);
+}
+
+#endif /* HAVE_STRUCT_FI_OPS_MEM_MONITOR */
+
+int opal_common_ofi_export_memory_monitor(void)
+{
+    int ret = -FI_ENOSYS;
+
+#ifdef HAVE_STRUCT_FI_OPS_MEM_MONITOR
+    bool memory_base_frame_open = false;
+    OPAL_THREAD_LOCK(&opal_common_ofi_mutex);
+
+    if (NULL != opal_common_ofi_cache_fid) {
+        OPAL_THREAD_UNLOCK(&opal_common_ofi_mutex);
+        return 0;
+    }
+
+    /*
+     * While the memory import functionality was introduced in 1.13,
+     * some deadlock bugs exist in the 1.13 series.  Require version
+     * 1.14 before this code is activated.  Not activating the code
+     * should not break any functionality directly, but may lead to
+     * sub-optimal memory monitors being used in Libfabric, as Open
+     * MPI will almost certainly install a patcher first.
+     */
+    if (FI_VERSION_LT(fi_version(), FI_VERSION(1, 14))) {
+        ret = -FI_ENOSYS;
+        goto err;
+    }
+
+    ret = mca_base_framework_open(&opal_memory_base_framework, 0);
+    if (OPAL_SUCCESS != ret) {
+        ret = -FI_ENOSYS;
+        goto err;
+    } else {
+       memory_base_frame_open = true;
+    }
+
+    if ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT)
+        != (((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT))
+        & opal_mem_hooks_support_level())) {
+        ret = -FI_ENOSYS;
+        goto err;
+    }
+
+    /*
+     * The monitor import object has the well known name "mr_cache"
+     * and was introduced in Libfabric 1.13
+     */
+    ret = fi_open(FI_VERSION(1,13), "mr_cache", NULL, 0, 0,
+                  &opal_common_ofi_cache_fid, NULL);
+    if (0 != ret) {
+        goto err;
+    }
+
+    opal_common_ofi_monitor = calloc(1, sizeof(*opal_common_ofi_monitor));
+    if (NULL == opal_common_ofi_monitor) {
+        ret = -FI_ENOMEM;
+        goto err;
+    }
+
+    opal_common_ofi_monitor->fid.fclass = FI_CLASS_MEM_MONITOR;
+    opal_common_ofi_monitor->export_ops = &opal_common_ofi_export_ops;
+    ret = fi_import_fid(opal_common_ofi_cache_fid,
+                        &opal_common_ofi_monitor->fid, 0);
+    if (0 != ret) {
+        goto err;
+    }
+    opal_mem_hooks_register_release(opal_common_ofi_mem_release_cb, NULL);
+    opal_common_ofi_installed_memory_monitor = true;
+
+    ret = 0;
+
+err:
+    if (0 != ret) {
+        if (NULL != opal_common_ofi_cache_fid) {
+            fi_close(opal_common_ofi_cache_fid);
+        }
+        if (NULL != opal_common_ofi_monitor) {
+            free(opal_common_ofi_monitor);
+        }
+
+        if (memory_base_frame_open) {
+            mca_base_framework_close(&opal_memory_base_framework);
+        }
+        opal_common_ofi_installed_memory_monitor = false;
+    }
+
+    OPAL_THREAD_UNLOCK(&opal_common_ofi_mutex);
+#endif
+
+    return ret;
+}
+
+static int opal_common_ofi_remove_memory_monitor(void)
+{
+#ifdef HAVE_STRUCT_FI_OPS_MEM_MONITOR
+    if (opal_common_ofi_installed_memory_monitor) {
+        opal_mem_hooks_unregister_release(opal_common_ofi_mem_release_cb);
+        fi_close(opal_common_ofi_cache_fid);
+        fi_close(&opal_common_ofi_monitor->fid);
+        free(opal_common_ofi_monitor);
+        mca_base_framework_close(&opal_memory_base_framework);
+        opal_common_ofi_installed_memory_monitor = false;
+    }
+#endif
+
+    return OPAL_SUCCESS;
+}
+
+int opal_common_ofi_open(void)
+{
+    if ((opal_common_ofi_init_ref_cnt++) > 0) {
+        return OPAL_SUCCESS;
+    }
+
+    return OPAL_SUCCESS;
+}
 
-OPAL_DECLSPEC int opal_common_ofi_is_in_list(char **list, char *item)
+int opal_common_ofi_close(void)
+{
+    int ret;
+
+    if ((--opal_common_ofi_init_ref_cnt) > 0) {
+        return OPAL_SUCCESS;
+    }
+
+    ret = opal_common_ofi_remove_memory_monitor();
+    if (OPAL_SUCCESS != ret) {
+        return ret;
+    }
+
+    if (-1 != opal_common_ofi.output) {
+        opal_output_close(opal_common_ofi.output);
+        opal_common_ofi.output = -1;
+    }
+
+    return OPAL_SUCCESS;
+}
+
+int opal_common_ofi_is_in_list(char **list, char *item)
 {
     int i = 0;
 
@@ -56,99 +284,207 @@ OPAL_DECLSPEC int opal_common_ofi_is_in_list(char **list, char *item)
     return 0;
 }
 
-OPAL_DECLSPEC int opal_common_ofi_register_mca_variables(const mca_base_component_t *component)
+int opal_common_ofi_count_providers_in_list(struct fi_info *provider_list, char **list)
+{
+    int count = 0, matched = 0;
+    struct fi_info *prov = provider_list, *prev_prov = NULL;
+    char *name;
+
+    while (prov) {
+        name = prov->fabric_attr->prov_name;
+        if (prev_prov && !strncasecmp(prev_prov->fabric_attr->prov_name, name, strlen(name))) {
+            /**
+             * Providers are usually sorted by name. We can reuse the previous matching result and
+             * avoid the potentially expensive list traversal.
+             */
+            count += matched;
+        } else if (opal_common_ofi_is_in_list(list, prov->fabric_attr->prov_name)) {
+            matched = 1;
+            ++count;
+        } else {
+            matched = 0;
+        }
+        prev_prov = prov;
+        prov = prov->next;
+    }
+
+    return count;
+}
+
+int opal_common_ofi_providers_subset_of_list(struct fi_info *provider_list, char **list)
+{
+    int num_prov = count_providers(provider_list);
+
+    if (!num_prov) {
+        return 1;
+    }
+
+    return num_prov == opal_common_ofi_count_providers_in_list(provider_list, list);
+}
+
+int opal_common_ofi_mca_register(const mca_base_component_t *component)
 {
-    static int registered = 0;
-    static int include_index;
-    static int exclude_index;
-    static int verbose_index;
+    int include_index;
+    int exclude_index;
+    int verbose_index;
+    int accelerator_rank_index;
+    int param;
+    int ret;
 
-    if (fi_version() < FI_VERSION(1,0)) {
+    if (fi_version() < FI_VERSION(1, 0)) {
         return OPAL_ERROR;
     }
 
-    if (!registered) {
+    OPAL_THREAD_LOCK(&opal_common_ofi_mutex);
+
+    param = mca_base_var_find("opal", "opal_common", "ofi", "provider_include");
+    if (0 > param) {
         /*
-         * this monkey business is needed because of the way the MCA VARs stuff tries to handle pointers to strings when
-         * when destructing the MCA var database.  If you don't do something like this,the MCA var framework will try
-         * to dereference a pointer which itself is no longer a valid address owing to having been previously dlclosed.
+         * this monkey business is needed because of the way the MCA VARs stuff tries to handle
+         * pointers to strings when when destructing the MCA var database.  If you don't do
+         * something like this,the MCA var framework will try to dereference a pointer which itself
+         * is no longer a valid address owing to having been previously dlclosed.
          */
-         opal_common_ofi.prov_include = (char **)malloc(sizeof(char *));
-         *opal_common_ofi.prov_include = NULL;
-         include_index = mca_base_var_register("opal", "opal_common", "ofi",
-                               "provider_include",
-                               "Comma-delimited list of OFI providers that are considered for use (e.g., \"psm,psm2\"; an empty value means that all providers will be considered). Mutually exclusive with mtl_ofi_provider_exclude.",
-                               MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
-                               OPAL_INFO_LVL_1,
-                               MCA_BASE_VAR_SCOPE_READONLY,
-                               opal_common_ofi.prov_include);
-        opal_common_ofi.prov_exclude = (char **)malloc(sizeof(char *));
+        if (NULL == opal_common_ofi.prov_include) {
+            opal_common_ofi.prov_include = (char **) malloc(sizeof(char *));
+            assert(NULL != opal_common_ofi.prov_include);
+        }
+        *opal_common_ofi.prov_include = NULL;
+        include_index = mca_base_var_register(
+            "opal", "opal_common", "ofi", "provider_include",
+            "Comma-delimited list of OFI providers that are considered for use (e.g., "
+            "\"psm,psm2\"; an empty value means that all providers will be considered). Mutually "
+            "exclusive with mtl_ofi_provider_exclude.",
+            MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_1, MCA_BASE_VAR_SCOPE_READONLY,
+            opal_common_ofi.prov_include);
+        if (0 > include_index) {
+            ret = include_index;
+            goto err;
+        }
+    } else {
+       include_index = param;
+    }
+
+    param = mca_base_var_find("opal", "opal_common", "ofi", "provider_exclude");
+    if (0 > param) {
+        if (NULL == opal_common_ofi.prov_exclude) {
+            opal_common_ofi.prov_exclude = (char **) malloc(sizeof(char *));
+            assert(NULL != opal_common_ofi.prov_exclude);
+        }
         *opal_common_ofi.prov_exclude = strdup(default_prov_exclude_list);
-        exclude_index = mca_base_var_register("opal", "opal_common", "ofi",
-                              "provider_exclude",
-                              "Comma-delimited list of OFI providers that are not considered for use (default: \"sockets,mxm\"; empty value means that all providers will be considered). Mutually exclusive with mtl_ofi_provider_include.",
-                              MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
-                              OPAL_INFO_LVL_1,
-                              MCA_BASE_VAR_SCOPE_READONLY,
-                              opal_common_ofi.prov_exclude);
+        exclude_index = mca_base_var_register(
+            "opal", "opal_common", "ofi", "provider_exclude",
+            "Comma-delimited list of OFI providers that are not considered for use (default: "
+            "\"sockets,mxm\"; empty value means that all providers will be considered). Mutually "
+            "exclusive with mtl_ofi_provider_include.",
+            MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_1, MCA_BASE_VAR_SCOPE_READONLY,
+            opal_common_ofi.prov_exclude);
+        if (0 > exclude_index) {
+            ret = exclude_index;
+            goto err;
+        }
+    } else {
+        exclude_index = param;
+    }
+
+    param = mca_base_var_find("opal", "opal_common", "ofi", "verbose");
+    if (0 > param) {
         verbose_index = mca_base_var_register("opal", "opal_common", "ofi", "verbose",
                                               "Verbose level of the OFI components",
                                               MCA_BASE_VAR_TYPE_INT, NULL, 0,
                                               MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
                                               MCA_BASE_VAR_SCOPE_LOCAL,
-                                              &opal_common_ofi.verbose);
-        registered = 1;
+                                              &opal_common_ofi_verbose_level);
+        if (0 > verbose_index) {
+            ret = verbose_index;
+            goto err;
+        }
+    } else {
+        verbose_index = param;
     }
 
-    if (component) {
-        mca_base_var_register_synonym(include_index, component->mca_project_name,
-                                      component->mca_type_name,
-                                      component->mca_component_name,
-                                      "provider_include", 0);
-        mca_base_var_register_synonym(exclude_index, component->mca_project_name,
-                                      component->mca_type_name,
-                                      component->mca_component_name,
-                                      "provider_exclude", 0);
-        mca_base_var_register_synonym(verbose_index, component->mca_project_name,
-                                      component->mca_type_name,
-                                      component->mca_component_name,
-                                      "verbose", 0);
+
+    param = mca_base_var_find("opal", "opal_common", "ofi", "accelerator_rank");
+    if (0 > param) {
+        accelerator_rank_index
+            = mca_base_var_register("opal", "opal_common", "ofi", "accelerator_rank",
+                                    "Process rank(non-negative) on the selected accelerator device",
+                                    MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
+                                    OPAL_INFO_LVL_1, MCA_BASE_VAR_SCOPE_LOCAL,
+                                    &opal_common_ofi_accelerator_rank);
+        if (0 > accelerator_rank_index) {
+            ret = accelerator_rank_index;
+            goto err;
+        }
+    } else {
+        accelerator_rank_index = param;
     }
 
-    return OPAL_SUCCESS;
-}
+    if (component) {
+        ret = mca_base_var_register_synonym(include_index,
+                                            component->mca_project_name,
+                                            component->mca_type_name,
+                                            component->mca_component_name,
+                                            "provider_include", 0);
+        if (0 > ret) {
+            goto err;
+        }
+        ret = mca_base_var_register_synonym(exclude_index,
+                                            component->mca_project_name,
+                                            component->mca_type_name,
+                                            component->mca_component_name,
+                                            "provider_exclude", 0);
+        if (0 > ret) {
+            goto err;
+        }
+        ret = mca_base_var_register_synonym(verbose_index,
+                                            component->mca_project_name,
+                                            component->mca_type_name,
+                                            component->mca_component_name,
+                                            "verbose", 0);
+        if (0 > ret) {
+            goto err;
+        }
 
-OPAL_DECLSPEC void opal_common_ofi_mca_register(void)
-{
-    opal_common_ofi.registered++;
-    if (opal_common_ofi.registered > 1) {
-         opal_output_set_verbosity(opal_common_ofi.output, opal_common_ofi.verbose);
-        return;
+        ret = mca_base_var_register_synonym(accelerator_rank_index,
+                                            component->mca_project_name,
+                                            component->mca_type_name,
+                                            component->mca_component_name,
+                                            "accelerator_rank_index", 0);
+        if (0 > ret) {
+            goto err;
+        }
     }
 
-    opal_common_ofi.output = opal_output_open(NULL);
-    opal_output_set_verbosity(opal_common_ofi.output, opal_common_ofi.verbose);
-}
-
-OPAL_DECLSPEC void opal_common_ofi_mca_deregister(void)
-{
-    /* unregister only on last deregister */
-    opal_common_ofi.registered--;
-    assert(opal_common_ofi.registered >= 0);
-    if (opal_common_ofi.registered) {
-        return;
+    /* The frameworks initialize their output streams during
+     * register(), so we similarly try to initialize the output stream
+     * as early as possible.  Because we may register synonyms for
+     * each dependent component, we don't necessarily have all the
+     * data to set verbosity during the first call to
+     * common_ofi_register().  The MCA infrastructure has rules on
+     * synonym value evaluation, so our rubric is to re-set verbosity
+     * after every call to register() (which has registered a new
+     * synonym).  This is not perfect, but it's not horrible, either.
+     */
+    if (opal_common_ofi.output == -1) {
+        opal_common_ofi.output = opal_output_open(NULL);
     }
-    opal_output_close(opal_common_ofi.output);
+    opal_output_set_verbosity(opal_common_ofi.output, opal_common_ofi_verbose_level);
+
+    ret = OPAL_SUCCESS;
+
+err:
+    OPAL_THREAD_UNLOCK(&opal_common_ofi_mutex);
+
+    return ret;
 }
 
 /* check that the tx attributes match */
-static int
-check_tx_attr(struct fi_tx_attr *provider_info,
-              struct fi_tx_attr *provider)
+static int check_tx_attr(struct fi_tx_attr *provider_info, struct fi_tx_attr *provider)
 {
-    if (!(provider->msg_order & ~(provider_info->msg_order)) &&
-        !(provider->op_flags & ~(provider_info->op_flags)) &&
-        (provider->inject_size == provider_info->inject_size)) {
+    if (!(provider->msg_order & ~(provider_info->msg_order))
+        && !(provider->op_flags & ~(provider_info->op_flags))
+        && (provider->inject_size == provider_info->inject_size)) {
         return 0;
     } else {
         return OPAL_ERROR;
@@ -156,12 +492,10 @@ check_tx_attr(struct fi_tx_attr *provider_info,
 }
 
 /* check that the rx attributes match */
-static int
-check_rx_attr(struct fi_rx_attr *provider_info,
-              struct fi_rx_attr *provider)
+static int check_rx_attr(struct fi_rx_attr *provider_info, struct fi_rx_attr *provider)
 {
-    if (!(provider->msg_order & ~(provider_info->msg_order)) &&
-        !(provider->op_flags & ~(provider_info->op_flags))) {
+    if (!(provider->msg_order & ~(provider_info->msg_order))
+        && !(provider->op_flags & ~(provider_info->op_flags))) {
         return 0;
     } else {
         return OPAL_ERROR;
@@ -169,15 +503,13 @@ check_rx_attr(struct fi_rx_attr *provider_info,
 }
 
 /* check that the ep attributes match */
-static int
-check_ep_attr(struct fi_ep_attr *provider_info,
-              struct fi_ep_attr *provider)
+static int check_ep_attr(struct fi_ep_attr *provider_info, struct fi_ep_attr *provider)
 {
-    if (!(provider->type & ~(provider_info->type)) &&
-        !(provider->mem_tag_format & ~(provider_info->mem_tag_format)) &&
-        (provider->max_msg_size == provider_info->max_msg_size) &&
-        (provider->tx_ctx_cnt == provider_info->tx_ctx_cnt) &&
-        (provider->rx_ctx_cnt == provider_info->rx_ctx_cnt)) {
+    if (!(provider->type & ~(provider_info->type))
+        && !(provider->mem_tag_format & ~(provider_info->mem_tag_format))
+        && (provider->max_msg_size == provider_info->max_msg_size)
+        && (provider->tx_ctx_cnt == provider_info->tx_ctx_cnt)
+        && (provider->rx_ctx_cnt == provider_info->rx_ctx_cnt)) {
         return 0;
     } else {
         return OPAL_ERROR;
@@ -185,96 +517,346 @@ check_ep_attr(struct fi_ep_attr *provider_info,
 }
 
 /* check that the provider attributes match */
-static int
-check_provider_attr(struct fi_info *provider_info,
-                    struct fi_info *provider)
+static int check_provider_attr(struct fi_info *provider_info, struct fi_info *provider)
 {
     /* make sure both info are the same provider and provide the same attributes */
-    if (0 == strcmp(provider_info->fabric_attr->prov_name, provider->fabric_attr->prov_name) &&
-        !check_tx_attr(provider_info->tx_attr, provider->tx_attr) &&
-        !check_rx_attr(provider_info->rx_attr, provider->rx_attr) &&
-        !check_ep_attr(provider_info->ep_attr, provider->ep_attr) &&
-        !(provider_info->caps & ~(provider->caps)) &&
-        !(provider_info->mode & ~(provider->mode)) &&
-        provider_info->addr_format == provider->addr_format) {
-        return 0;
+    if (0 == strcmp(provider_info->fabric_attr->prov_name, provider->fabric_attr->prov_name)
+        && 0 == strcmp(provider_info->fabric_attr->name, provider->fabric_attr->name)
+        && !check_tx_attr(provider_info->tx_attr, provider->tx_attr)
+        && !check_rx_attr(provider_info->rx_attr, provider->rx_attr)
+        && !check_ep_attr(provider_info->ep_attr, provider->ep_attr)
+        && !(provider_info->caps & ~(provider->caps)) && !(provider_info->mode & ~(provider->mode))
+        && provider_info->addr_format == provider->addr_format) {
+        return OPAL_SUCCESS;
     } else {
         return OPAL_ERROR;
     }
 }
 
 #if OPAL_OFI_PCI_DATA_AVAILABLE
-/* Check if a process and a pci device share the same cpuset
- *     @param (IN) pci              struct fi_pci_attr pci device attributes,
- *                                  used to find hwloc object for device.
+static int get_provider_nic_pci(struct fi_info *provider, struct fi_pci_attr *pci)
+{
+    if (NULL != provider->nic && NULL != provider->nic->bus_attr
+        && FI_BUS_PCI == provider->nic->bus_attr->bus_type) {
+        *pci = provider->nic->bus_attr->attr.pci;
+        return OPAL_SUCCESS;
+    }
+    return OPAL_ERR_NOT_AVAILABLE;
+}
+#endif /* OPAL_OFI_PCI_DATA_AVAILABLE */
+
+/**
+ * Calculate device distances
+ *
+ * Calculate the distances between the current thread and all devices of
+ * type OPENFABRICS or NETWORK.
+ *
+ * The shortest distances are the nearest and therefore most efficient
+ * devices to use.
+ *
+ * Return an array of all the distances. Each entry is of type
+ * pmix_device_distance_t
+ *
+ * This function is used if there is no PMIx server running.
+ *
+ * @param distances (OUT)     distances array
+ * @param ndist (OUT)    number of entries in the distances array
+ *
+ * @return   0 on success. Error otherwise.
+ *
+ */
+static int compute_dev_distances(pmix_device_distance_t **distances,
+                                  size_t *ndist)
+{
+    int ret = OPAL_SUCCESS;
+    size_t ninfo;
+    pmix_info_t *info;
+    pmix_cpuset_t cpuset;
+    pmix_topology_t pmix_topo = PMIX_TOPOLOGY_STATIC_INIT;
+    pmix_device_type_t type = PMIX_DEVTYPE_OPENFABRICS |
+      PMIX_DEVTYPE_NETWORK;
+
+    PMIX_CPUSET_CONSTRUCT(&cpuset);
+    ret = PMIx_Get_cpuset(&cpuset, PMIX_CPUBIND_THREAD);
+    if (PMIX_SUCCESS != ret) {
+        /* we are not bound */
+        ret = OPAL_ERR_NOT_BOUND;
+        goto out;
+    }
+    /* if we are not bound, then we cannot compute distances */
+    if (hwloc_bitmap_iszero(cpuset.bitmap) ||
+        hwloc_bitmap_isfull(cpuset.bitmap)) {
+        return OPAL_ERR_NOT_BOUND;
+    }
+
+    /* load the PMIX topology - this just loads a pointer to
+     * the local topology held in PMIx, so you must not
+     * free it */
+    ret = PMIx_Load_topology(&pmix_topo);
+    if (PMIX_SUCCESS != ret) {
+        goto out;
+    }
+
+    ninfo = 1;
+    info = PMIx_Info_create(ninfo);
+    PMIx_Info_load(&info[0], PMIX_DEVICE_TYPE, &type, PMIX_DEVTYPE);
+    ret = PMIx_Compute_distances(&pmix_topo, &cpuset, info, ninfo, distances,
+                                 ndist);
+    PMIx_Info_free(info, ninfo);
+
+out:
+    return ret;
+}
+
+/**
+ * @brief Get the provider distance from the provided distance metrics
+ *
+ * @param[in]    topology        hwloc topology
+ * @param[in]    provider        Provider object
+ * @param[in]    distances       List of known device distances
+ * @param[in]    num_distances   Length of distances
+ * @param[out]   distance        Pointer to store the provider distance
+ * @return  OPAL_SUCCESS if and only if the distance is found in the provided list
+ */
+#if OPAL_OFI_PCI_DATA_AVAILABLE
+static int get_provider_distance(hwloc_topology_t topology, struct fi_info *provider,
+                                 pmix_device_distance_t *distances, int num_distances,
+                                 uint16_t *distance)
+{
+    hwloc_obj_t pcidev, osdev;
+    struct fi_pci_attr pci = {0};
+
+    if (OPAL_SUCCESS != get_provider_nic_pci(provider, &pci)) {
+        opal_output_verbose(1, opal_common_ofi.output, "Cannot determine PCI attributes of provider %s",
+                            provider->domain_attr->name);
+        return OPAL_ERROR;
+    }
+
+    pcidev = hwloc_get_pcidev_by_busid(topology, pci.domain_id, pci.bus_id, pci.device_id,
+                                       pci.function_id);
+    if (!pcidev) {
+        opal_output_verbose(1, opal_common_ofi.output, "Cannot locate PCI device of provider %s",
+                            provider->domain_attr->name);
+        return OPAL_ERROR;
+    }
+
+#if HWLOC_API_VERSION < 0x00020000
+    osdev = pcidev->first_child;
+#else
+    osdev = pcidev->io_first_child;
+#endif /* HWLOC_API_VERSION */
+    for (; osdev != NULL; osdev = osdev->next_sibling) {
+        int i;
+
+        if (osdev->attr->osdev.type == HWLOC_OBJ_OSDEV_OPENFABRICS) {
+            const char *nguid = hwloc_obj_get_info_by_name(osdev, "NodeGUID");
+            const char *sguid = hwloc_obj_get_info_by_name(osdev, "SysImageGUID");
+
+            if (!nguid && !sguid)
+                continue;
+
+            for (i = 0; i < num_distances; i++) {
+                char lsguid[20], lnguid[20];
+                int ret;
+
+                if (PMIX_DEVTYPE_OPENFABRICS != distances[i].type) {
+                    continue;
+                }
+
+                if (!distances[i].osname || !osdev->name
+                    || strcmp(distances[i].osname, osdev->name))
+                    continue;
+
+                ret = sscanf(distances[i].uuid, "fab://%19s::%19s", lnguid, lsguid);
+                if (ret != 2)
+                    continue;
+
+                if ((nguid && (0 == strcasecmp(lnguid, nguid)))
+                    || (sguid && (0 == strcasecmp(lsguid, sguid)))) {
+                    *distance = distances[i].mindist;
+                    return OPAL_SUCCESS;
+                }
+            }
+        } else if (osdev->attr->osdev.type == HWLOC_OBJ_OSDEV_NETWORK) {
+            const char *address = hwloc_obj_get_info_by_name(osdev, "Address");
+            if (!address)
+                continue;
+            for (i = 0; i < num_distances; i++) {
+                if (PMIX_DEVTYPE_NETWORK != distances[i].type) {
+                    continue;
+                }
+                char *addr = strstr(distances[i].uuid, "://");
+                if (!addr || addr + 3 > distances[i].uuid + strlen(distances[i].uuid))
+                    continue;
+                if (!strcmp(addr + 3, address)) {
+                    *distance = distances[i].mindist;
+                    return OPAL_SUCCESS;
+                }
+            }
+        }
+    }
+
+    return OPAL_ERROR;
+}
+#else
+static int get_provider_distance(struct fi_info *provider, hwloc_topology_t topology,
+                                 pmix_device_distance_t *distances, size_t num_distances,
+                                 uint16_t *distance)
+{
+    return OPAL_ERROR;
+}
+#endif /* OPAL_OFI_PCI_DATA_AVAILABLE */
+
+/**
+ * @brief Get the nearest device to the current thread
  *
- *     @param (IN) topology         hwloc_topology_t topology to get the cpusets
- *                                  from
+ * Compute the distances from the current thread to each NIC in provider_list,
+ * and select the NIC with the shortest distance.
+ * If there are multiple equidistant devices, break the tie using local rank
+ * to balance NIC utilization.
  *
- *     @param (OUT)                 returns true if cpusets match and false if
- *                                  cpusets do not match or an error prevents comparison
+ * @param[in]   topoloy          hwloc topology
+ * @param[in]   provider_list    List of providers to select from
+ * @param[in]   num_providers    Number of providers in provider_list
+ * @param[in]   rank             local rank of the process
+ * @param[out]  provider         pointer to the selected provider
  *
- *     Uses a pci device to find an ancestor that contains a cpuset, and
- *     determines if it intersects with the cpuset that the process is bound to.
- *     if the process is not bound, or if a cpuset is unavailable for whatever
- *     reason, returns false. Otherwise, returns the result of
- *     hwloc_cpuset_intersects()
+ * @return OPAL_SUCCESS if and only if a nearest provider is found.
  */
-static bool
-compare_cpusets(hwloc_topology_t topology, struct fi_pci_attr pci)
+static int get_nearest_nic(hwloc_topology_t topology, struct fi_info *provider_list,
+                           size_t num_providers, uint32_t rank, struct fi_info **provider)
 {
-    bool result = false;
     int ret;
-    hwloc_bitmap_t proc_cpuset;
-    hwloc_obj_t obj = NULL;
+    pmix_data_array_t *dptr;
+    pmix_device_distance_t *distances;
+    pmix_info_t directive;
+    pmix_value_t *val = NULL;
+    size_t ndist, num_nearest = 0;
+    struct fi_info *current_provider = NULL;
+    uint16_t dists[num_providers], *dist = NULL, min_dist = USHRT_MAX;
+    uint32_t provider_rank = 0;
 
-    /* Cannot find topology info if no topology is found */
-    if (NULL == topology) {
-        return false;
+    PMIx_Info_load(&directive, PMIX_OPTIONAL, NULL, PMIX_BOOL);
+    ret = PMIx_Get(&opal_process_info.myprocid, PMIX_DEVICE_DISTANCES, &directive, 1, &val);
+    PMIx_Info_destruct(&directive);
+    if (ret != PMIX_SUCCESS || !val) {
+        ret = compute_dev_distances(&distances, &ndist);
+        if (ret) {
+            ret = OPAL_ERROR;
+            goto out;
+        }
+        goto find_nearest;
     }
 
-    /* Allocate memory for proc_cpuset */
-    proc_cpuset = hwloc_bitmap_alloc();
-    if (NULL == proc_cpuset) {
-        return false;
+    if (PMIX_DATA_ARRAY != val->type) {
+        ret = OPAL_ERROR;
+        goto out;
+    }
+    dptr = val->data.darray;
+    if (NULL == dptr) {
+        ret = OPAL_ERROR;
+        goto out;
+    }
+    if (PMIX_DEVICE_DIST != dptr->type) {
+        ret = OPAL_ERROR;
+        goto out;
     }
 
-    /* Fill cpuset with the collection of cpu cores that the process runs on */
-    ret = hwloc_get_cpubind(topology, proc_cpuset, HWLOC_CPUBIND_PROCESS);
-    if (0 > ret) {
-        goto error;
+    distances = (pmix_device_distance_t *) dptr->array;
+    ndist = dptr->size;
+
+find_nearest:
+    for (current_provider = provider_list, dist = dists; NULL != current_provider;
+         current_provider = current_provider->next, ++dist) {
+        if (OPAL_SUCCESS != check_provider_attr(provider_list, current_provider)) {
+            continue;
+        }
+        if (OPAL_SUCCESS != get_provider_distance(topology, current_provider, distances, ndist, dist)) {
+            *dist = USHRT_MAX;
+        }
+
+        if (*dist < min_dist) {
+            min_dist = *dist;
+            num_nearest = 1;
+        } else if (*dist == min_dist) {
+            ++num_nearest;
+        }
+
+        if (OPAL_SUCCESS == check_provider_attr(provider_list, current_provider)) {
+            opal_output_verbose(1, opal_common_ofi.output, "provider: %s dist: %d",
+                                current_provider->domain_attr->name, *dist);
+        }
     }
 
-    /* Get the pci device from bdf */
-    obj = hwloc_get_pcidev_by_busid(topology, pci.domain_id, pci.bus_id,
-                                    pci.device_id, pci.function_id);
-    if (NULL == obj) {
-        goto error;
+    ret = OPAL_ERROR;
+    if (0 >= num_nearest) {
+        return ret;
     }
 
-    /* pcidev objects don't have cpusets so find the first non-io object above */
-    obj = hwloc_get_non_io_ancestor_obj(topology, obj);
-    if (NULL != obj) {
-        result = hwloc_bitmap_intersects(proc_cpuset, obj->cpuset);
+    provider_rank = rank % num_nearest;
+    num_nearest = 0;
+    for (current_provider = provider_list, dist = dists; NULL != current_provider;
+         current_provider = current_provider->next) {
+        if (OPAL_SUCCESS == check_provider_attr(provider_list, current_provider)
+            && min_dist == *(dist++) && provider_rank == num_nearest++) {
+            *provider = current_provider;
+            ret = OPAL_SUCCESS;
+            goto out;
+        }
     }
+out:
+    if (val)
+        PMIx_Value_free(val, 1);
 
-error:
-    hwloc_bitmap_free(proc_cpuset);
-    return result;
+    return ret;
 }
-#endif
 
-/* Count providers returns the number of providers present in an fi_info list
- *     @param (IN) provider_list    struct fi_info* list of providers available
+/**
+ * @brief Selects a provider from the list in a round-robin fashion
  *
- *     @param (OUT)                 int number of providers present in the list
+ * This function implements a round-robin algorithm to select a provider from
+ * the provided list based on a rank. Only providers of the same type as the
+ * first provider are eligible for selection.
  *
- *     returns 0 if the list is NULL
+ * @param[in]   provider_list   A list of providers to select from.
+ * @param[out]  rank            A rank metric for the current process, such as
+ *                              the rank on the same node or CPU package.
+ * @return      Pointer to the selected provider
  */
-static int
-count_providers(struct fi_info* provider_list)
+static struct fi_info *select_provider_round_robin(struct fi_info *provider_list, uint32_t rank)
 {
-    struct fi_info* dev = provider_list;
+    uint32_t provider_rank = 0, current_rank = 0;
+    size_t num_providers = 0;
+    struct fi_info *current_provider = NULL;
+
+    for (current_provider = provider_list; NULL != current_provider;) {
+        if (OPAL_SUCCESS == check_provider_attr(provider_list, current_provider)) {
+            ++num_providers;
+        }
+        current_provider = current_provider->next;
+    }
+
+    current_provider = provider_list;
+    if (2 > num_providers) {
+        goto out;
+    }
+
+    provider_rank = rank % num_providers;
+
+    while (NULL != current_provider) {
+        if (OPAL_SUCCESS == check_provider_attr(provider_list, current_provider)
+            && provider_rank == current_rank++) {
+            break;
+        }
+        current_provider = current_provider->next;
+    }
+out:
+    return current_provider;
+}
+
+static int count_providers(struct fi_info *provider_list)
+{
+    struct fi_info *dev = provider_list;
     int num_provider = 0;
 
     while (NULL != dev) {
@@ -285,13 +867,14 @@ count_providers(struct fi_info* provider_list)
     return num_provider;
 }
 
-/* Calculate the currrent process package rank.
- *     @param (IN) process_info     struct opal_process_info_t information
- *                                  about the current process. used to get
- *                                  num_local_peers, myprocid.rank, and
- *                                  my_local_rank.
+/**
+ * @brief the current process package rank.
  *
- *     @param (OUT)                 uint32_t package rank or myprocid.rank
+ * @param[in]   process_info    struct opal_process_info_t information
+ *                              about the current process. used to get
+ *                              num_local_peers, myprocid.rank.
+ *
+ * @return package rank or myprocid.rank
  *
  * If successful, returns PMIX_PACKAGE_RANK, or an
  * equivalent calculated package rank.
@@ -300,10 +883,9 @@ count_providers(struct fi_info* provider_list)
  */
 static uint32_t get_package_rank(opal_process_info_t *process_info)
 {
-    int i;
+    int i, level = 10;
     uint16_t relative_locality, *package_rank_ptr;
-    uint16_t current_package_rank = 0;
-    uint16_t package_ranks[process_info->num_local_peers];
+    uint32_t ranks_on_package = 0;
     opal_process_name_t pname;
     pmix_status_t rc;
     char **peers = NULL;
@@ -313,197 +895,365 @@ static uint32_t get_package_rank(opal_process_info_t *process_info)
     pname.jobid = OPAL_PROC_MY_NAME.jobid;
     pname.vpid = OPAL_VPID_WILDCARD;
 
+    /*
+     * if we are a singleton just return myprocid.rank
+     * because we by definition don't know about any local peers
+     */
+    if (opal_process_info.is_singleton) {
+        return (uint32_t) process_info->myprocid.rank;
+    }
+
 #if HAVE_DECL_PMIX_PACKAGE_RANK
     // Try to get the PACKAGE_RANK from PMIx
-    OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_PACKAGE_RANK,
-                                   &pname, &package_rank_ptr, PMIX_UINT16);
+    OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_PACKAGE_RANK, &pname, &package_rank_ptr, PMIX_UINT16);
     if (PMIX_SUCCESS == rc) {
-        return (uint32_t)*package_rank_ptr;
+        return (uint32_t) *package_rank_ptr;
     }
 #endif
 
     // Get the local peers
-    OPAL_MODEX_RECV_VALUE(rc, PMIX_LOCAL_PEERS,
-                          &pname, &local_peers, PMIX_STRING);
+    OPAL_MODEX_RECV_VALUE(rc, PMIX_LOCAL_PEERS, &pname, &local_peers, PMIX_STRING);
     if (PMIX_SUCCESS != rc || NULL == local_peers) {
-        // We can't find package_rank, fall back to procid
-        opal_show_help("help-common-ofi.txt", "package_rank failed", true);
-        return (uint32_t)process_info->myprocid.rank;
+        goto err;
     }
     peers = opal_argv_split(local_peers, ',');
     free(local_peers);
 
     for (i = 0; NULL != peers[i]; i++) {
         pname.vpid = strtoul(peers[i], NULL, 10);
+
         locality_string = NULL;
         // Get the LOCALITY_STRING for process[i]
-        OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_LOCALITY_STRING,
-                                       &pname, &locality_string, PMIX_STRING);
+        OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_LOCALITY_STRING, &pname, &locality_string,
+                                       PMIX_STRING);
         if (PMIX_SUCCESS != rc || NULL == locality_string) {
-            // If we don't have information about locality, fall back to procid
-            int level = 10;
-            if (opal_output_get_verbosity(opal_common_ofi.output) >= level) {
-                opal_show_help("help-common-ofi.txt", "package_rank failed", true, level);
-            }
-            return (uint32_t)process_info->myprocid.rank;
+            goto err;
         }
 
         // compute relative locality
-        relative_locality = opal_hwloc_compute_relative_locality(process_info->locality, locality_string);
+        relative_locality = opal_hwloc_compute_relative_locality(process_info->locality,
+                                                                 locality_string);
         free(locality_string);
+        locality_string = NULL;
+
+        if ((uint16_t) pname.vpid == process_info->myprocid.rank) {
+            return ranks_on_package;
+        }
 
         if (relative_locality & OPAL_PROC_ON_SOCKET) {
-            package_ranks[i] = current_package_rank;
-            current_package_rank++;
+            ranks_on_package++;
         }
     }
+err:
+    if (opal_output_get_verbosity(opal_common_ofi.output) >= level) {
+        opal_show_help("help-common-ofi.txt", "package_rank failed", true, level);
+    }
+
+    if (locality_string)
+        free(locality_string);
 
-    return (uint32_t)package_ranks[process_info->my_local_rank];
+    return (uint32_t) process_info->myprocid.rank;
 }
 
-/* Selects a NIC based on hardware locality between process cpuset and device BDF.
- *
- * Initializes opal_hwloc_topology to access hardware topology if not previously
- * initialized
- *
- * There are 3 main cases that this covers:
- *
- *      1. If the first provider passed into this function is the only valid
- *      provider, this provider is returned.
- *
- *      2. If there is more than 1 provider that matches the type of the first
- *      provider in the list, and the BDF data
- *      is available then a provider is selected based on locality of device
- *      cpuset and process cpuset and tries to ensure that processes are distributed
- *      evenly across NICs. This has two separate cases:
- *
- *          i. There is one or more provider local to the process:
- *
- *              (local rank % number of providers of the same type that share the process cpuset)
- *              is used to select one of these providers.
- *
- *          ii. There is no provider that is local to the process:
- *
- *              (local rank % number of providers of the same type)
- *              is used to select one of these providers
- *
- *      3. If there is more than 1 providers of the same type in the list, and the BDF data
- *      is not available (the ofi version does not support fi_info.nic or the
- *      provider does not support BDF) then (local rank % number of providers of the same type)
- *      is used to select one of these providers
- *
- *      @param provider_list (IN)   struct fi_info* An initially selected
- *                                  provider NIC. The provider name and
- *                                  attributes are used to restrict NIC
- *                                  selection. This provider is returned if the
- *                                  NIC selection fails.
- *
- *      @param package_rank (IN)   uint32_t The rank of the process. Used to
- *                                  select one valid NIC if there is a case
- *                                  where more than one can be selected. This
- *                                  could occur when more than one provider
- *                                  shares the same cpuset as the process.
- *                                  This could either be a package_rank if one is
- *                                  successfully calculated, or the process id.
- *
- *      @param provider (OUT)       struct fi_info* object with the selected
- *                                  provider if the selection succeeds
- *                                  if the selection fails, returns the fi_info
- *                                  object that was initially provided.
- *
- * All errors should be recoverable and will return the initially provided
- * provider. However, if an error occurs we can no longer guarantee
- * that the provider returned is local to the process or that the processes will
- * balance across available NICs.
- */
-struct fi_info*
-opal_mca_common_ofi_select_provider(struct fi_info *provider_list, opal_process_info_t *process_info)
+static int get_parent_distance(hwloc_obj_t parent, hwloc_obj_t child, int *distance)
 {
-    struct fi_info *provider = provider_list, *current_provider = provider_list;
-    struct fi_info **provider_table;
+    int dist = 0;
+
+    while (child != parent) {
+        if (!child) {
+            return OPAL_ERROR;
+        }
+        child = child->parent;
+        ++dist;
+    }
+
+    *distance = dist;
+    return OPAL_SUCCESS;
+}
+
 #if OPAL_OFI_PCI_DATA_AVAILABLE
-    struct fi_pci_attr pci;
-#endif
-    int ret;
-    uint32_t package_rank;
-    unsigned int num_provider = 0, provider_limit = 0;
-    bool provider_found = false, cpusets_match = false;
+/**
+ * @brief Attempt to find a nearest provider from the accelerator.
+ * Check if opal_accelerator is initialized with a valid PCI device, and find a provider from the
+ * shortest distance.
+ * Special cases:
+ * 1. If not accelerator device is available, returns OPAL_ERR_NOT_AVAILABLE.
+ * 2. If the provider does not have PCI attributers, we do not attempt to make a selection, and
+ *    return OPAL_ERR_NOT_AVAILABLE.
+ * 3. If there are more than 1 providers with the same equal distance, break the tie using a modulo
+ *    i.e. (local rank on the same accelerator) % (number of nearest providers)
+ * @param[in]   provider_list   List of providers
+ * @param[in]   num_providers   Number of providers
+ * @param[in]   accl_id         Accelerator id
+ * @param[in]   device_rank     Local rank on the accelerator
+ * @param[out]  provider        Pointer to the selected provider
+ * @return      OPAL_SUCCESS if a provider is successfully selected
+ *              OPAL_ERR_NOT_AVAILABLE if a provider cannot be decided deterministically
+ *              OPAL_ERROR if a fatal error happened
+ */
+static int find_nearest_provider_from_accelerator(struct fi_info *provider_list,
+                                                  size_t num_providers,
+                                                  int accl_id,
+                                                  uint32_t device_rank,
+                                                  struct fi_info **provider)
+{
+    hwloc_obj_t accl_dev = NULL, prov_dev = NULL, common_ancestor = NULL;
+    int ret = -1, accl_distance = -1, prov_distance = -1, min_distance = INT_MAX;
+    opal_accelerator_pci_attr_t accl_pci_attr = {0};
+    struct fi_info *current_provider = NULL;
+    struct fi_pci_attr pci = {0};
+    uint32_t distances[num_providers], *distance = distances;
+    uint32_t near_provider_count = 0, provider_rank = 0;
 
-    /* Initialize opal_hwloc_topology if it is not already */
-    ret = opal_hwloc_base_get_topology();
-    if (0 > ret) {
-        /* Provider selection can continue but there is no guarantee of locality */
+    memset(distances, 0, sizeof(distances));
+
+    ret = opal_accelerator.get_device_pci_attr(accl_id, &accl_pci_attr);
+    if (OPAL_SUCCESS != ret) {
         opal_output_verbose(1, opal_common_ofi.output,
-                            "%s:%d:Failed to initialize topology\n",
-                            __FILE__, __LINE__);
+                            "%s:%d:Accelerator PCI info is not available", __FILE__, __LINE__);
+        return OPAL_ERROR;
     }
 
-    provider_limit = count_providers(provider_list);
-
-    /* Allocate memory for provider table */
-    provider_table = calloc(provider_limit, sizeof(struct fi_info*));
-    if (NULL == provider_table) {
+    accl_dev = hwloc_get_pcidev_by_busid(opal_hwloc_topology, accl_pci_attr.domain_id,
+                                         accl_pci_attr.bus_id, accl_pci_attr.device_id,
+                                         accl_pci_attr.function_id);
+    if (NULL == accl_dev) {
         opal_output_verbose(1, opal_common_ofi.output,
-                            "%s:%d:Failed to allocate memory for provider table\n",
-                            __FILE__, __LINE__);
-        return provider_list;
+                            "%s:%d:Failed to find accelerator PCI device", __FILE__, __LINE__);
+        return OPAL_ERROR;
     }
 
-    current_provider = provider;
+    opal_output_verbose(1, opal_common_ofi.output,
+                        "%s:%d:Found accelerator device %d: %04x:%02x:%02x.%x VID: %x DID: %x",
+                        __FILE__, __LINE__, accl_id, accl_pci_attr.domain_id, accl_pci_attr.bus_id,
+                        accl_pci_attr.device_id, accl_pci_attr.function_id,
+                        accl_dev->attr->pcidev.vendor_id, accl_dev->attr->pcidev.device_id);
 
-    /* Cycle through remaining fi_info objects, looking for alike providers */
+    current_provider = provider_list;
     while (NULL != current_provider) {
-        if (!check_provider_attr(provider, current_provider)) {
-            cpusets_match = false;
-#if OPAL_OFI_PCI_DATA_AVAILABLE
-            if (NULL != current_provider->nic) {
-                pci = current_provider->nic->bus_attr->attr.pci;
-                cpusets_match = compare_cpusets(opal_hwloc_topology, pci);
+        common_ancestor = NULL;
+        if (0 == check_provider_attr(provider_list, current_provider)
+            && OPAL_SUCCESS == get_provider_nic_pci(current_provider, &pci)) {
+            prov_dev = hwloc_get_pcidev_by_busid(opal_hwloc_topology, pci.domain_id, pci.bus_id,
+                                                 pci.device_id, pci.function_id);
+            if (NULL == prov_dev) {
+                opal_output_verbose(1, opal_common_ofi.output,
+                                    "%s:%d:Failed to find provider PCI device", __FILE__, __LINE__);
+                return OPAL_ERROR;
             }
-#endif
 
-            /* Reset the list if the cpusets match and no other provider was
-             * found on the same cpuset as the process.
-             */
-            if (cpusets_match && !provider_found) {
-                provider_found = true;
-                num_provider = 0;
+            common_ancestor = hwloc_get_common_ancestor_obj(opal_hwloc_topology, accl_dev,
+                                                            prov_dev);
+            if (!common_ancestor) {
+                opal_output_verbose(
+                    1, opal_common_ofi.output,
+                    "%s:%d:Failed to find common ancestor of accelerator and provider PCI device",
+                    __FILE__, __LINE__);
+                /**
+                 * Return error because any 2 PCI devices should share at least one common ancestor,
+                 * i.e. root
+                 */
+                return OPAL_ERROR;
             }
 
-            /* Add the provider to the provider list if the cpusets match or if
-             * no other provider was found on the same cpuset as the process.
-             */
-            if (cpusets_match || !provider_found) {
-                provider_table[num_provider] = current_provider;
-                num_provider++;
+            ret = get_parent_distance(common_ancestor, accl_dev, &accl_distance);
+            if (OPAL_SUCCESS != ret) {
+                opal_output_verbose(
+                    1, opal_common_ofi.output,
+                    "%s:%d:Failed to get distance between common ancestor and accelerator device",
+                    __FILE__, __LINE__);
+                return OPAL_ERROR;
+            }
+
+            ret = get_parent_distance(common_ancestor, prov_dev, &prov_distance);
+            if (OPAL_SUCCESS != ret) {
+                opal_output_verbose(
+                    1, opal_common_ofi.output,
+                    "%s:%d:Failed to get distance between common ancestor and provider device",
+                    __FILE__, __LINE__);
+                return OPAL_ERROR;
+            }
+
+            if (min_distance > accl_distance + prov_distance) {
+                min_distance = accl_distance + prov_distance;
+                near_provider_count = 1;
+            } else if (min_distance == accl_distance + prov_distance) {
+                ++near_provider_count;
             }
         }
+
+        *(distance++) = !common_ancestor ? 0 : accl_distance + prov_distance;
         current_provider = current_provider->next;
     }
 
-    /* Select provider from local rank % number of providers */
-    if (num_provider >= 2) {
-        // If there are multiple NICs "close" to the process, try to calculate package_rank
-        package_rank = get_package_rank(process_info);
-        provider = provider_table[package_rank % num_provider];
-    } else if (num_provider == 1) {
-        provider = provider_table[num_provider - 1];
+    if (0 == near_provider_count) {
+        opal_output_verbose(1, opal_common_ofi.output, "%s:%d:Provider does not have PCI device",
+                            __FILE__, __LINE__);
+        return OPAL_ERR_NOT_AVAILABLE;
     }
 
+    provider_rank = device_rank % near_provider_count;
+
+    distance = distances;
+    current_provider = provider_list;
+    near_provider_count = 0;
+    while (NULL != current_provider) {
+        if ((uint32_t) min_distance == *(distance++)
+            && provider_rank == near_provider_count++) {
+            *provider = current_provider;
+            return OPAL_SUCCESS;
+        }
+
+        current_provider = current_provider->next;
+    }
+
+    assert(0 == near_provider_count);
+
+    return OPAL_ERROR;
+}
+#endif /* OPAL_OFI_PCI_DATA_AVAILABLE */
+
+
+struct fi_info *opal_common_ofi_select_provider(struct fi_info *provider_list,
+                                                opal_process_info_t *process_info)
+{
+    int ret, num_providers = 0, accel_id = -1;
+    struct fi_info *provider = NULL;
+    uint32_t package_rank;
+
+    /* Current process' local rank on the same package(socket) */
+    package_rank = process_info->proc_is_bound ? get_package_rank(process_info)
+                                               : process_info->my_local_rank;
+    num_providers = count_providers(provider_list);
+
 #if OPAL_OFI_PCI_DATA_AVAILABLE
-    if (NULL != provider->nic) {
-        pci = provider->nic->bus_attr->attr.pci;
-        cpusets_match = compare_cpusets(opal_hwloc_topology, pci);
+    if (-1 < opal_common_ofi_accelerator_rank) {
+        ret = opal_accelerator.get_device(&accel_id);
+        if (OPAL_SUCCESS != ret) {
+            opal_output_verbose(1, opal_common_ofi.output, "%s:%d:Accelerator is not available",
+                                __FILE__, __LINE__);
+            accel_id = -1;
+        }
+    }
+#endif /* OPAL_OFI_PCI_DATA_AVAILABLE */
+
+    if ((!process_info->proc_is_bound && 0 > accel_id) || 2 > num_providers) {
+        goto round_robin;
     }
-#endif
 
+#if OPAL_OFI_PCI_DATA_AVAILABLE
+    /* Initialize opal_hwloc_topology if it is not already */
+    ret = opal_hwloc_base_get_topology();
+    if (0 > ret) {
+        /* Provider selection can continue but there is no guarantee of locality */
+        opal_output_verbose(1, opal_common_ofi.output, "%s:%d:Failed to initialize topology",
+                            __FILE__, __LINE__);
+    }
+
+    if (0 <= accel_id) {
+        ret = find_nearest_provider_from_accelerator(provider_list, num_providers, accel_id,
+                                                     opal_common_ofi_accelerator_rank, &provider);
+        if (OPAL_SUCCESS == ret) {
+            goto out;
+        }
+
+        opal_output_verbose(1, opal_common_ofi.output,
+                            "%s:%d:Failed to find a provider close to the accelerator. Error: %d",
+                            __FILE__, __LINE__, ret);
+
+        if (!process_info->proc_is_bound) {
+            goto round_robin;
+        }
+    }
+
+    /**
+     * If provider PCI BDF information is available, we calculate its physical distance
+     * to the current process, and select the provider with the shortest distance.
+     */
+    ret = get_nearest_nic(opal_hwloc_topology, provider_list, num_providers, package_rank,
+                          &provider);
+    if (OPAL_SUCCESS == ret) {
+        goto out;
+    }
+#endif /* OPAL_OFI_PCI_DATA_AVAILABLE */
+
+round_robin:
+    if (!process_info->proc_is_bound && 1 < num_providers
+        && opal_output_get_verbosity(opal_common_ofi.output) >= 1) {
+        opal_show_help("help-common-ofi.txt", "unbound_process", true, 1);
+    }
+
+    provider = select_provider_round_robin(provider_list, package_rank);
+out:
 #if OPAL_ENABLE_DEBUG
-    opal_output_verbose(1, opal_common_ofi.output,
-                        "package rank: %d device: %s cpusets match: %s\n",
-                         package_rank, provider->domain_attr->name,
-                         cpusets_match ? "true" : "false");
+    opal_output_verbose(1, opal_common_ofi.output, "package rank: %d device: %s", package_rank,
+                        provider->domain_attr->name);
 #endif
-
-    free(provider_table);
     return provider;
 }
+
+/**
+ * Obtain EP endpoint name
+ *
+ * Obtain the EP endpoint name and length for the supplied endpoint fid.
+ *
+ * @param fid (IN)     fid of (S)EP endpoint
+ * @param addr (OUT)   buffer containing endpoint name 
+ * @param addrlen (OUT) length of allocated buffer in bytes
+ *
+ * @return             OPAL_SUCCESS or OPAL error code
+ *
+ * The caller is responsible for freeing the buffer allocated to
+ * contain the endpoint name.
+ *
+ */
+OPAL_DECLSPEC int opal_common_ofi_fi_getname(fid_t fid, void **addr, size_t *addrlen)
+{
+    int ret=OPAL_SUCCESS;
+    size_t namelen = 0;
+    char *ep_name = NULL;
+
+    /**
+     * Get our address and publish it with modex.
+     * Use the two step process of first getting the required
+     * buffer size, then allocating the memory and calling
+     * fi_getname again.
+     */
+    namelen = 0;
+    ret = fi_getname(fid,
+                     NULL,
+                     &namelen);
+    if ((FI_SUCCESS != ret) && (-FI_ETOOSMALL != ret)) {
+        opal_output_verbose(1, opal_common_ofi.output, "%s:%d:fi_endpoint (namelen) returned %s\n",
+                            __FILE__, __LINE__, fi_strerror(-ret));
+        ret = OPAL_ERROR;
+        goto error;
+    }
+
+    ep_name = (char *)malloc(namelen);
+    if (NULL == ep_name) {
+        ret = OPAL_ERR_OUT_OF_RESOURCE;
+        goto error;
+    }
+
+    ret = fi_getname(fid,
+                     ep_name,
+                     &namelen);
+    if (ret) {
+        opal_output_verbose(1, opal_common_ofi.output, "%s:%d:fi_endpoint (ep_name) returned %s\n",
+                            __FILE__, __LINE__, fi_strerror(-ret));
+        ret = OPAL_ERROR;
+        goto error;
+    }
+
+    *addr = ep_name;
+    *addrlen = namelen;
+
+    return ret;
+
+error:
+    if (NULL != ep_name) {
+       free(ep_name); 
+    }
+    return ret;
+}
diff --git a/opal/mca/common/ofi/common_ofi.h b/opal/mca/common/ofi/common_ofi.h
index 5b7ebc13206..3deeb0c63ec 100644
--- a/opal/mca/common/ofi/common_ofi.h
+++ b/opal/mca/common/ofi/common_ofi.h
@@ -3,7 +3,9 @@
  * Copyright (c) 2015      Intel, Inc. All rights reserved.
  * Copyright (c) 2017      Los Alamos National Security, LLC.  All rights
  *                         reserved.
- * Copyright (c) 2020      Triad National Security, LLC. All rights
+ * Copyright (c) 2020-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2021      Amazon.com, Inc. or its affiliates. All rights
  *                         reserved.
  *
  * $COPYRIGHT$
@@ -16,29 +18,97 @@
 #ifndef OPAL_MCA_COMMON_OFI_H
 #define OPAL_MCA_COMMON_OFI_H
 
-#include "opal_config.h"
-#include "opal/mca/base/mca_base_var.h"
-#include "opal/mca/base/mca_base_framework.h"
 #include "opal/util/proc.h"
+#include "opal/memoryhooks/memory.h"
+
 #include <rdma/fabric.h>
+#include <rdma/fi_cm.h>
 
 BEGIN_C_DECLS
 
 typedef struct opal_common_ofi_module {
     char **prov_include;
     char **prov_exclude;
-    int verbose;
-    int registered;
     int output;
 } opal_common_ofi_module_t;
 
+/**
+ * When attempting to execute an OFI operation we need to handle
+ * resource overrun cases. When a call to an OFI OP fails with -FI_EAGAIN
+ * the OFI mtl/btl will attempt to progress any pending Completion Queue
+ * events that may prevent additional operations to be enqueued.
+ * If the call to ofi progress is successful, then the function call
+ * will be retried.
+ */
+#define OFI_RETRY_UNTIL_DONE(FUNC, RETURN)             \
+    do {                                               \
+        do {                                           \
+            RETURN = FUNC;                             \
+            if (OPAL_LIKELY(0 == RETURN)) {break;}     \
+            if (OPAL_LIKELY(RETURN == -FI_EAGAIN)) {   \
+                opal_progress();                       \
+            }                                          \
+        } while (OPAL_LIKELY(-FI_EAGAIN == RETURN));   \
+    } while (0);
+
+
 extern opal_common_ofi_module_t opal_common_ofi;
 
-OPAL_DECLSPEC int opal_common_ofi_register_mca_variables(const mca_base_component_t *component);
-OPAL_DECLSPEC void opal_common_ofi_mca_register(void);
-OPAL_DECLSPEC void opal_common_ofi_mca_deregister(void);
+/**
+ * Common MCA registration
+ *
+ * Common MCA registration handlinge.  After calling this function,
+ * \code opal_common_ofi.output will be properly initialized.
+ *
+ * @param component (IN) OFI component being initialized
+ *
+ * @returns OPAL_SUCCESS on success, OPAL error code on failure
+ */
+OPAL_DECLSPEC int opal_common_ofi_mca_register(const mca_base_component_t *component);
+
+/**
+ * Initializes common objects for libfabric
+ *
+ * Initialize common libfabric interface.  This should be called from
+ * any other OFI component's component_open() call.
+ *
+ * @note This function is not thread safe and must be called in a
+ * serial portion of the code.
+ */
+OPAL_DECLSPEC int opal_common_ofi_open(void);
 
-/*
+/**
+ * Cleans up common objects for libfabric
+ *
+ * Clean up common libfabric interface.  This should be called from
+ * any other OFI component's component_close() call.  Resource cleanup
+ * is reference counted, so any successful call to
+ * opal_common_ofi_init().
+ *
+ * @note This function is not thread safe and must be called in a
+ * serial portion of the code.
+ */
+OPAL_DECLSPEC int opal_common_ofi_close(void);
+
+/**
+ * Export our memory hooks into Libfabric monitor
+ *
+ * Use Open MPI's memory hooks to provide monitor notifications to
+ * Libfabric via the external mr_cache facility.  This must be called
+ * before any domain is initialized (ie, before any Libfabric memory
+ * monitor is configured).
+ *
+ * @returns A libfabric error code is returned on error
+ */
+OPAL_DECLSPEC int opal_common_ofi_export_memory_monitor(void);
+
+/**
+ * Search function for provider names
+ *
+ * This function will take a provider name string and a list of lower
+ * provider name strings as inputs. It will return true if the lower
+ * provider in the item string matches a lower provider in the list.
+ *
  * @param list (IN)    List of strings corresponding to lower providers.
  * @param item (IN)    Single string corresponding to a provider.
  *
@@ -47,15 +117,112 @@ OPAL_DECLSPEC void opal_common_ofi_mca_deregister(void);
  * @return 1           The lower provider of the item string matches
  *                     a string in the item list.
  *
- * This function will take a provider name string and a list of lower
- * provider name strings as inputs. It will return true if the lower
- * provider in the item string matches a lower provider in the list.
- *
  */
 OPAL_DECLSPEC int opal_common_ofi_is_in_list(char **list, char *item);
 
-END_C_DECLS
+/**
+ * Get the number of providers whose names are included in a list
+ *
+ * This function takes a list of providers and a list of name strings
+ * as inputs, and return the number of providers whose names are included
+ * in the name strings.
+ *
+ * @param provider_list (IN)    List of providers
+ * @param list          (IN)    List of name string
+ *
+ * @return                      Number of matched providers
+ *
+ */
+OPAL_DECLSPEC int opal_common_ofi_count_providers_in_list(struct fi_info *provider_list,
+                                                          char **list);
+
+/**
+ * Determine whether all providers are included in a list
+ *
+ * This function takes a list of providers and a list of name strings
+ * as inputs, and return whether all provider names are included in the name strings.
+ *
+ * @param provider_list (IN)    List of providers
+ * @param list          (IN)    List of name string
+ *
+ * @return  0                   At least one provider's name is not included in the name strings.
+ * @return  1                   All provider names are included in the name strings.
+ *
+ */
+OPAL_DECLSPEC int opal_common_ofi_providers_subset_of_list(struct fi_info *provider_list,
+                                                           char **list);
+
+/**
+ * Selects NIC (provider) based on hardware locality
+ *
+ * The selection is based on the following priority:
+ *
+ * Single-NIC:
+ * 
+ *      If only 1 provider is available, always return that provider.
+ * 
+ * Multi-NIC:
+ * 
+ *      1. If the process is NOT bound, pick a NIC using (local rank % number
+ *      of providers of the same type). This gives a fair chance to each
+ *      qualified NIC and balances overall utilization.
+ *
+ *      2. If the process is bound, we compare providers in the list that have
+ *      the same type as the first provider, and find the provider with the
+ *      shortest distance to the current process. 
+ * 
+ *          i. If the provider has PCI BDF data, we attempt to compute the
+ *          distance between the NIC and the current process cpuset. The NIC
+ *          with the shortest distance is returned.
+ * 
+ *              * For equidistant NICs, we select a NIC in round-robin fashion
+ *              using the package rank of the current process, i.e. (package
+ *              rank % number of providers with the same distance).
+ *
+ *          ii. If we cannot compute the distance between the NIC and the
+ *          current process, e.g. PCI BDF data is not available, a NIC will be
+ *          selected in a round-robin fashion using package rank, i.e. (package
+ *          rank % number of providers of the same type).
+ *
+ * @param[in]   provider_list   struct fi_info* An initially selected
+ *                              provider NIC. The provider name and
+ *                              attributes are used to restrict NIC
+ *                              selection. This provider is returned if the
+ *                              NIC selection fails.
+ * 
+ * @param[in]   process_info    opal_process_info_t* The current process info
+ *
+ * @param[out]  provider        struct fi_info* object with the selected
+ *                              provider if the selection succeeds
+ *                              if the selection fails, returns the fi_info
+ *                              object that was initially provided.
+ *
+ * All errors should be recoverable and will return the initially provided
+ * provider. However, if an error occurs we can no longer guarantee
+ * that the provider returned is local to the process or that the processes will
+ * balance across available NICs.
+ *
+ */
+OPAL_DECLSPEC struct fi_info *opal_common_ofi_select_provider(struct fi_info *provider_list,
+                                                              opal_process_info_t *process_info);
 
-struct fi_info* opal_mca_common_ofi_select_provider(struct fi_info *provider_list, opal_process_info_t *process_info);
+/**
+ * Obtain EP endpoint name
+ *
+ * Obtain the EP endpoint name and length for the supplied endpoint fid.
+ *
+ * @param fid (IN)     fid of (S)EP endpoint
+ * @param addr (OUT)   buffer containing endpoint name 
+ * @param addrlen (OUT) length of allocated buffer in bytes
+ *
+ * @return             OPAL_SUCCESS or OPAL error code
+ *
+ * The caller is responsible for freeing the buffer allocated to
+ * contain the endpoint name.
+ *
+ */
+OPAL_DECLSPEC int opal_common_ofi_fi_getname(fid_t fid, void **addr, size_t *addrlen);
+
+END_C_DECLS
 
 #endif /* OPAL_MCA_COMMON_OFI_H */
diff --git a/opal/mca/common/ofi/configure.m4 b/opal/mca/common/ofi/configure.m4
index 2877964244e..2b5592ebb24 100644
--- a/opal/mca/common/ofi/configure.m4
+++ b/opal/mca/common/ofi/configure.m4
@@ -1,4 +1,4 @@
-# -*- shell-script -*-
+# -*- autoconf -*-
 #
 # Copyright (c) 2011-2013 NVIDIA Corporation.  All rights reserved.
 # Copyright (c) 2013      The University of Tennessee and The University
@@ -9,6 +9,7 @@
 # Copyright (c) 2017       Los Alamos National Security, LLC.  All rights
 #                         reserved.
 # Copyright (c) 2019      Hewlett Packard Enterprise. All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -17,19 +18,19 @@
 #
 
 AC_DEFUN([MCA_opal_common_ofi_CONFIG],[
+    OPAL_VAR_SCOPE_PUSH([common_ofi_happy])
+
     AC_CONFIG_FILES([opal/mca/common/ofi/Makefile])
 
-    OPAL_CHECK_OFI
+    OPAL_CHECK_OFI([common_ofi], [common_ofi_happy=1], [common_ofi_happy=0])
+
+    AS_IF([test ${common_ofi_happy} -eq 1],
+          [$1],
+          [$2])
 
-    # Note that $opal_common_ofi_happy is
-    # used in other configure.m4's to know if ofi configured
-    # successfully.
-    AS_IF([test "$opal_ofi_happy" = "yes"],
-          [opal_common_ofi_happy=yes
-           common_ofi_WRAPPER_EXTRA_LDFLAGS=$opal_ofi_LDFLAGS
-           common_ofi_WRAPPER_EXTRA_LIBS=$opal_ofi_LIBS
-           $1],
-          [opal_common_ofi_happy=no
-           $2])
+    AC_SUBST([common_ofi_CPPFLAGS])
+    AC_SUBST([common_ofi_LDFLAGS])
+    AC_SUBST([common_ofi_LIBS])
 
+    OPAL_VAR_SCOPE_POP
 ])dnl
diff --git a/opal/mca/common/ofi/help-common-ofi.txt b/opal/mca/common/ofi/help-common-ofi.txt
index 44366a64c5f..de3630f7e7a 100644
--- a/opal/mca/common/ofi/help-common-ofi.txt
+++ b/opal/mca/common/ofi/help-common-ofi.txt
@@ -7,6 +7,12 @@
 #
 # $HEADER$
 #
+[unbound_process]
+Open MPI's OFI driver detected multiple NICs on the system but cannot select an
+optimal device because the current process is not bound. This may negatively
+impact performance. This can be resolved by specifying "--bind-to ..." on
+command line.
+
 [package_rank failed]
 Open MPI's OFI driver detected multiple equidistant NICs from the current process,
 but had insufficient information to ensure MPI processes fairly pick a NIC for use.
diff --git a/opal/mca/common/sm/Makefile.am b/opal/mca/common/sm/Makefile.am
index ba57c100da5..0c249088d37 100644
--- a/opal/mca/common/sm/Makefile.am
+++ b/opal/mca/common/sm/Makefile.am
@@ -12,6 +12,7 @@
 # Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2010-2015 Los Alamos National Security, LLC.
 #                         All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -55,7 +56,7 @@ sources = \
 
 # Help file
 
-dist_opaldata_DATA = help-mpi-common-sm.txt
+EXTRA_DIST = help-mpi-common-sm.txt
 
 # As per above, we'll either have an installable or noinst result.
 # The installable one should follow the same MCA prefix naming rules
@@ -74,8 +75,8 @@ dist_opaldata_DATA = help-mpi-common-sm.txt
 
 lib_LTLIBRARIES =
 noinst_LTLIBRARIES =
-comp_inst = lib@OPAL_LIB_PREFIX@mca_common_sm.la
-comp_noinst = lib@OPAL_LIB_PREFIX@mca_common_sm_noinst.la
+comp_inst = lib@OPAL_LIB_NAME@mca_common_sm.la
+comp_noinst = lib@OPAL_LIB_NAME@mca_common_sm_noinst.la
 
 if MCA_BUILD_opal_common_sm_DSO
 lib_LTLIBRARIES += $(comp_inst)
@@ -83,11 +84,11 @@ else
 noinst_LTLIBRARIES += $(comp_noinst)
 endif
 
-lib@OPAL_LIB_PREFIX@mca_common_sm_la_SOURCES = \
+lib@OPAL_LIB_NAME@mca_common_sm_la_SOURCES = \
         $(headers) $(sources)
-lib@OPAL_LIB_PREFIX@mca_common_sm_la_LDFLAGS = \
+lib@OPAL_LIB_NAME@mca_common_sm_la_LDFLAGS = \
         -version-info $(libmca_opal_common_sm_so_version)
-lib@OPAL_LIB_PREFIX@mca_common_sm_noinst_la_SOURCES = \
+lib@OPAL_LIB_NAME@mca_common_sm_noinst_la_SOURCES = \
         $(headers) $(sources)
 
 # Conditionally install the header files
diff --git a/opal/mca/common/sm/common_sm.c b/opal/mca/common/sm/common_sm.c
index 573254cb44e..0650cccea68 100644
--- a/opal/mca/common/sm/common_sm.c
+++ b/opal/mca/common/sm/common_sm.c
@@ -32,30 +32,24 @@
 
 #include "opal_config.h"
 
+#include "common_sm.h"
 #include "opal/align.h"
+#include "opal/constants.h"
+#include "opal/mca/shmem/base/base.h"
 #include "opal/util/argv.h"
-#include "opal/util/show_help.h"
 #include "opal/util/error.h"
-#include "opal/mca/shmem/base/base.h"
-#include "common_sm.h"
-#include "opal/constants.h"
-
-
-OBJ_CLASS_INSTANCE(mca_common_sm_module_t,opal_list_item_t,
-                   NULL, NULL);
+#include "opal/util/show_help.h"
 
+OBJ_CLASS_INSTANCE(mca_common_sm_module_t, opal_list_item_t, NULL, NULL);
 
 /* ////////////////////////////////////////////////////////////////////////// */
 /* static utility functions */
 /* ////////////////////////////////////////////////////////////////////////// */
 
 /* ////////////////////////////////////////////////////////////////////////// */
-static mca_common_sm_module_t *
-attach_and_init(opal_shmem_ds_t *shmem_bufp,
-                size_t size,
-                size_t size_ctl_structure,
-                size_t data_seg_alignment,
-                bool first_call)
+static mca_common_sm_module_t *attach_and_init(opal_shmem_ds_t *shmem_bufp, size_t size,
+                                               size_t size_ctl_structure, size_t data_seg_alignment,
+                                               bool first_call)
 {
     mca_common_sm_module_t *map = NULL;
     mca_common_sm_seg_header_t *seg = NULL;
@@ -64,22 +58,21 @@ attach_and_init(opal_shmem_ds_t *shmem_bufp,
     /* attach to the specified segment. note that at this point, the contents of
      * *shmem_bufp have already been initialized via opal_shmem_segment_create.
      */
-    if (NULL == (seg = (mca_common_sm_seg_header_t *)
-                       opal_shmem_segment_attach(shmem_bufp))) {
+    if (NULL == (seg = (mca_common_sm_seg_header_t *) opal_shmem_segment_attach(shmem_bufp))) {
         return NULL;
     }
     opal_atomic_rmb();
 
     if (NULL == (map = OBJ_NEW(mca_common_sm_module_t))) {
         OPAL_ERROR_LOG(OPAL_ERR_OUT_OF_RESOURCE);
-        (void)opal_shmem_segment_detach(shmem_bufp);
+        (void) opal_shmem_segment_detach(shmem_bufp);
         return NULL;
     }
 
     /* copy meta information into common sm module
      *                                     from ====> to                */
     if (OPAL_SUCCESS != opal_shmem_ds_copy(shmem_bufp, &map->shmem_ds)) {
-        (void)opal_shmem_segment_detach(shmem_bufp);
+        (void) opal_shmem_segment_detach(shmem_bufp);
         free(map);
         return NULL;
     }
@@ -90,7 +83,7 @@ attach_and_init(opal_shmem_ds_t *shmem_bufp,
      */
     map->module_seg = seg;
 
-    addr = ((unsigned char *)seg) + size_ctl_structure;
+    addr = ((unsigned char *) seg) + size_ctl_structure;
     /* if we have a data segment (i.e., if 0 != data_seg_alignment),
      * then make it the first aligned address after the control
      * structure.  IF THIS HAPPENS, THIS IS A PROGRAMMING ERROR IN
@@ -99,26 +92,23 @@ attach_and_init(opal_shmem_ds_t *shmem_bufp,
     if (0 != data_seg_alignment) {
         addr = OPAL_ALIGN_PTR(addr, data_seg_alignment, unsigned char *);
         /* is addr past end of the shared memory segment? */
-        if ((unsigned char *)seg + shmem_bufp->seg_size < addr) {
+        if ((unsigned char *) seg + shmem_bufp->seg_size < addr) {
             opal_show_help("help-mpi-common-sm.txt", "mmap too small", 1,
-                           opal_process_info.nodename,
-                           (unsigned long)shmem_bufp->seg_size,
-                           (unsigned long)size_ctl_structure,
-                           (unsigned long)data_seg_alignment);
-            (void)opal_shmem_segment_detach(shmem_bufp);
+                           opal_process_info.nodename, (unsigned long) shmem_bufp->seg_size,
+                           (unsigned long) size_ctl_structure, (unsigned long) data_seg_alignment);
+            (void) opal_shmem_segment_detach(shmem_bufp);
             free(map);
             return NULL;
         }
     }
 
     map->module_data_addr = addr;
-    map->module_seg_addr = (unsigned char *)seg;
+    map->module_seg_addr = (unsigned char *) seg;
 
     /* note that size is only used during the first call */
     if (first_call) {
         /* initialize some segment information */
-        size_t mem_offset = map->module_data_addr -
-                            (unsigned char *)map->module_seg;
+        size_t mem_offset = map->module_data_addr - (unsigned char *) map->module_seg;
         opal_atomic_lock_init(&map->module_seg->seg_lock, OPAL_ATOMIC_LOCK_UNLOCKED);
         map->module_seg->seg_inited = 0;
         map->module_seg->seg_num_procs_inited = 0;
@@ -128,7 +118,7 @@ attach_and_init(opal_shmem_ds_t *shmem_bufp,
     }
 
     /* increment the number of processes that are attached to the segment. */
-    (void)opal_atomic_add_fetch_size_t(&map->module_seg->seg_num_procs_inited, 1);
+    (void) opal_atomic_add_fetch_size_t(&map->module_seg->seg_num_procs_inited, 1);
 
     /* commit the changes before we return */
     opal_atomic_wmb();
@@ -141,11 +131,9 @@ attach_and_init(opal_shmem_ds_t *shmem_bufp,
 /* ////////////////////////////////////////////////////////////////////////// */
 
 /* ////////////////////////////////////////////////////////////////////////// */
-mca_common_sm_module_t *
-mca_common_sm_module_create_and_attach(size_t size,
-                                       char *file_name,
-                                       size_t size_ctl_structure,
-                                       size_t data_seg_alignment)
+mca_common_sm_module_t *mca_common_sm_module_create_and_attach(size_t size, char *file_name,
+                                                               size_t size_ctl_structure,
+                                                               size_t data_seg_alignment)
 {
     mca_common_sm_module_t *map = NULL;
     opal_shmem_ds_t *seg_meta = NULL;
@@ -155,8 +143,7 @@ mca_common_sm_module_create_and_attach(size_t size,
         return NULL;
     }
     if (OPAL_SUCCESS == opal_shmem_segment_create(seg_meta, file_name, size)) {
-        map = attach_and_init(seg_meta, size, size_ctl_structure,
-                              data_seg_alignment, true);
+        map = attach_and_init(seg_meta, size, size_ctl_structure, data_seg_alignment, true);
     }
     /* at this point, seg_meta has been copied to the newly created
      * shared memory segment, so we can free it */
@@ -172,20 +159,17 @@ mca_common_sm_module_create_and_attach(size_t size,
  * @return a pointer to the mca_common_sm_module_t associated with seg_meta if
  * everything was okay, otherwise returns NULL.
  */
-mca_common_sm_module_t *
-mca_common_sm_module_attach(opal_shmem_ds_t *seg_meta,
-                            size_t size_ctl_structure,
-                            size_t data_seg_alignment)
+mca_common_sm_module_t *mca_common_sm_module_attach(opal_shmem_ds_t *seg_meta,
+                                                    size_t size_ctl_structure,
+                                                    size_t data_seg_alignment)
 {
     /* notice that size is 0 here. it really doesn't matter because size WILL
      * NOT be used because this is an attach (first_call is false). */
-    return attach_and_init(seg_meta, 0, size_ctl_structure,
-                           data_seg_alignment, false);
+    return attach_and_init(seg_meta, 0, size_ctl_structure, data_seg_alignment, false);
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-int
-mca_common_sm_module_unlink(mca_common_sm_module_t *modp)
+int mca_common_sm_module_unlink(mca_common_sm_module_t *modp)
 {
     if (NULL == modp) {
         return OPAL_ERROR;
@@ -197,10 +181,8 @@ mca_common_sm_module_unlink(mca_common_sm_module_t *modp)
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-int
-mca_common_sm_local_proc_reorder(opal_proc_t **procs,
-                                 size_t num_procs,
-                                 size_t *out_num_local_procs)
+int mca_common_sm_local_proc_reorder(opal_proc_t **procs, size_t num_procs,
+                                     size_t *out_num_local_procs)
 {
     size_t num_local_procs = 0;
     bool found_lowest = false;
@@ -221,13 +203,12 @@ mca_common_sm_local_proc_reorder(opal_proc_t **procs,
             if (!found_lowest) {
                 procs[0] = procs[p];
                 found_lowest = true;
-            }
-            else {
+            } else {
                 /* save this proc */
                 procs[num_local_procs] = procs[p];
                 /* if we have a new lowest, swap it with position 0
                  * so that procs[0] is always the lowest named proc */
-                if( 0 > opal_compare_proc(procs[p]->proc_name, procs[0]->proc_name) ) {
+                if (0 > opal_compare_proc(procs[p]->proc_name, procs[0]->proc_name)) {
                     temp_proc = procs[0];
                     procs[0] = procs[p];
                     procs[num_local_procs] = temp_proc;
@@ -253,7 +234,7 @@ mca_common_sm_local_proc_reorder(opal_proc_t **procs,
  *
  *  @retval addr virtual address
  */
-void *mca_common_sm_seg_alloc (void *ctx, size_t *size)
+void *mca_common_sm_seg_alloc(void *ctx, size_t *size)
 {
     mca_common_sm_module_t *sm_module = (mca_common_sm_module_t *) ctx;
     mca_common_sm_seg_header_t *seg = sm_module->module_seg;
@@ -262,8 +243,7 @@ void *mca_common_sm_seg_alloc (void *ctx, size_t *size)
     opal_atomic_lock(&seg->seg_lock);
     if (seg->seg_offset + *size > seg->seg_size) {
         addr = NULL;
-    }
-    else {
+    } else {
         size_t fixup;
 
         /* add base address to segment offset */
@@ -271,7 +251,7 @@ void *mca_common_sm_seg_alloc (void *ctx, size_t *size)
         seg->seg_offset += *size;
 
         /* fix up seg_offset so next allocation is aligned on a
-         * sizeof(long) boundry.  Do it here so that we don't have to
+         * sizeof(long) boundary.  Do it here so that we don't have to
          * check before checking remaining size in buffer
          */
         if ((fixup = (seg->seg_offset & (sizeof(long) - 1))) > 0) {
@@ -284,14 +264,12 @@ void *mca_common_sm_seg_alloc (void *ctx, size_t *size)
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-int
-mca_common_sm_fini(mca_common_sm_module_t *mca_common_sm_module)
+int mca_common_sm_fini(mca_common_sm_module_t *module)
 {
     int rc = OPAL_SUCCESS;
 
-    if (NULL != mca_common_sm_module->module_seg) {
-        if (OPAL_SUCCESS !=
-            opal_shmem_segment_detach(&mca_common_sm_module->shmem_ds)) {
+    if (NULL != module->module_seg) {
+        if (OPAL_SUCCESS != opal_shmem_segment_detach(&module->shmem_ds)) {
             rc = OPAL_ERROR;
         }
     }
diff --git a/opal/mca/common/sm/common_sm.h b/opal/mca/common/sm/common_sm.h
index 7c4456f0457..b5a3704b312 100644
--- a/opal/mca/common/sm/common_sm.h
+++ b/opal/mca/common/sm/common_sm.h
@@ -25,15 +25,15 @@
 
 #include "opal_config.h"
 
-#include "opal/mca/mca.h"
-#include "opal/class/opal_object.h"
+#include "common_sm_mpool.h"
 #include "opal/class/opal_list.h"
-#include "opal/sys/atomic.h"
-#include "opal/mca/shmem/shmem.h"
+#include "opal/class/opal_object.h"
 #include "opal/mca/btl/base/base.h"
-#include "opal/util/proc.h"
 #include "opal/mca/btl/base/btl_base_error.h"
-#include "common_sm_mpool.h"
+#include "opal/mca/mca.h"
+#include "opal/mca/shmem/shmem.h"
+#include "opal/sys/atomic.h"
+#include "opal/util/proc.h"
 
 BEGIN_C_DECLS
 
@@ -81,10 +81,8 @@ OBJ_CLASS_DECLARATION(mca_common_sm_module_t);
  *
  * @returnvalue OPAL_SUCCESS on success, something else, otherwise.
  */
-OPAL_DECLSPEC extern int
-mca_common_sm_local_proc_reorder(opal_proc_t **procs,
-                                 size_t num_procs,
-                                 size_t *out_num_local_procs);
+OPAL_DECLSPEC extern int mca_common_sm_local_proc_reorder(opal_proc_t **procs, size_t num_procs,
+                                                          size_t *out_num_local_procs);
 
 /**
  *  This routine is used to create and attach to a shared memory segment
@@ -95,9 +93,7 @@ mca_common_sm_local_proc_reorder(opal_proc_t **procs,
  *  Returns NULL if an error occurred.
  */
 OPAL_DECLSPEC extern mca_common_sm_module_t *
-mca_common_sm_module_create_and_attach(size_t size,
-                                       char *file_name,
-                                       size_t size_ctl_structure,
+mca_common_sm_module_create_and_attach(size_t size, char *file_name, size_t size_ctl_structure,
                                        size_t data_seg_alignment);
 
 /**
@@ -110,10 +106,9 @@ mca_common_sm_module_create_and_attach(size_t size,
  *  @returnvalue pointer to control structure at head of shared memory segment.
  *  Returns NULL if an error occurred.
  */
-OPAL_DECLSPEC extern mca_common_sm_module_t *
-mca_common_sm_module_attach(opal_shmem_ds_t *seg_meta,
-                            size_t size_ctl_structure,
-                            size_t data_seg_alignment);
+OPAL_DECLSPEC extern mca_common_sm_module_t *mca_common_sm_module_attach(opal_shmem_ds_t *seg_meta,
+                                                                         size_t size_ctl_structure,
+                                                                         size_t data_seg_alignment);
 
 /**
  * A thin wrapper around opal_shmem_unlink.
@@ -123,13 +118,12 @@ mca_common_sm_module_attach(opal_shmem_ds_t *seg_meta,
  * @returnvalue OPAL_SUCCESS if the operation completed successfully,
  * OPAL_ERROR otherwise.
  */
-OPAL_DECLSPEC extern int
-mca_common_sm_module_unlink(mca_common_sm_module_t *modp);
+OPAL_DECLSPEC extern int mca_common_sm_module_unlink(mca_common_sm_module_t *modp);
 
 /**
  * callback from the sm mpool
  */
-OPAL_DECLSPEC extern void *mca_common_sm_seg_alloc (void *ctx, size_t *size);
+OPAL_DECLSPEC extern void *mca_common_sm_seg_alloc(void *ctx, size_t *size);
 
 /**
  * This function will release all local resources attached to the
@@ -142,15 +136,13 @@ OPAL_DECLSPEC extern void *mca_common_sm_seg_alloc (void *ctx, size_t *size);
  * @return OPAL_SUCCESS if everything was okay, otherwise return OPAL_ERROR.
  */
 
-OPAL_DECLSPEC extern int
-mca_common_sm_fini(mca_common_sm_module_t *mca_common_sm_module);
+OPAL_DECLSPEC extern int mca_common_sm_fini(mca_common_sm_module_t *mca_common_sm_module);
 
 /**
  * instance that is shared between components that use shared memory.
  */
 OPAL_DECLSPEC extern mca_common_sm_module_t *mca_common_sm_module;
 
-
 END_C_DECLS
 
 #endif /* _COMMON_SM_H_ */
diff --git a/opal/mca/common/sm/common_sm_mpool.c b/opal/mca/common/sm/common_sm_mpool.c
index c50d048eff9..dcdb181f4f2 100644
--- a/opal/mca/common/sm/common_sm_mpool.c
+++ b/opal/mca/common/sm/common_sm_mpool.c
@@ -14,7 +14,8 @@
  *                         All rights reserved.
  * Copyright (c) 2011-2014 NVIDIA Corporation.  All rights reserved.
  * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
- * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2018-2022 Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -23,41 +24,38 @@
  */
 
 #include "opal_config.h"
-#include <string.h>
-#include "opal/util/printf.h"
 #include "common_sm_mpool.h"
-#include "opal/mca/common/sm/common_sm.h"
-#include "opal/mca/common/cuda/common_cuda.h"
 #include "opal/mca/allocator/base/base.h"
+#include "opal/mca/common/sm/common_sm.h"
+#include "opal/util/printf.h"
+#include <string.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 #include "opal/mca/hwloc/base/base.h"
 
-static void sm_module_finalize(mca_mpool_base_module_t* module);
+static void sm_module_finalize(mca_mpool_base_module_t *module);
 
 /*
  *  Returns base address of shared memory mapping.
  */
-static void *mca_common_sm_mpool_base (mca_mpool_base_module_t *mpool);
+static void *mca_common_sm_mpool_base(mca_mpool_base_module_t *mpool);
 
 /**
-  *  Allocate block of shared memory.
-  */
-static void *mca_common_sm_mpool_alloc (mca_mpool_base_module_t *mpool,
-                                        size_t size, size_t align,
-                                        uint32_t flags);
+ *  Allocate block of shared memory.
+ */
+static void *mca_common_sm_mpool_alloc(mca_mpool_base_module_t *mpool, size_t size, size_t align,
+                                       uint32_t flags);
 
 /**
-  * free function typedef
-  */
-static void mca_common_sm_mpool_free(mca_mpool_base_module_t *mpool,
-                                     void *addr);
+ * free function typedef
+ */
+static void mca_common_sm_mpool_free(mca_mpool_base_module_t *mpool, void *addr);
 
 /*
  *  Initializes the mpool module.
  */
-static void mca_common_sm_mpool_module_init(mca_common_sm_mpool_module_t* mpool)
+static void mca_common_sm_mpool_module_init(mca_common_sm_mpool_module_t *mpool)
 {
     mpool->super.mpool_base = mca_common_sm_mpool_base;
     mpool->super.mpool_alloc = mca_common_sm_mpool_alloc;
@@ -72,13 +70,13 @@ static void mca_common_sm_mpool_module_init(mca_common_sm_mpool_module_t* mpool)
     mpool->mem_node = -1;
 }
 
-mca_mpool_base_module_t *common_sm_mpool_create (mca_common_sm_mpool_resources_t *resources)
+mca_mpool_base_module_t *opal_btl_smcuda_common_sm_mpool_create(mca_common_sm_mpool_resources_t *resources)
 {
     mca_common_sm_mpool_module_t *mpool_module;
-    mca_allocator_base_component_t* allocator_component;
+    mca_allocator_base_component_t *allocator_component;
 
     /* Make a new mpool module */
-    mpool_module = (mca_common_sm_mpool_module_t *) malloc (sizeof (*mpool_module));
+    mpool_module = (mca_common_sm_mpool_module_t *) malloc(sizeof(*mpool_module));
     mca_common_sm_mpool_module_init(mpool_module);
 
     /* set sm_size */
@@ -89,18 +87,17 @@ mca_mpool_base_module_t *common_sm_mpool_create (mca_common_sm_mpool_resources_t
     /* if specified allocator cannot be loaded - look for an alternative */
     if (NULL == allocator_component) {
         if (opal_list_get_size(&opal_allocator_base_framework.framework_components) == 0) {
-            mca_base_component_list_item_t *item =
-                (mca_base_component_list_item_t *)
+            mca_base_component_list_item_t *item = (mca_base_component_list_item_t *)
                 opal_list_get_first(&opal_allocator_base_framework.framework_components);
-            allocator_component =
-                (mca_allocator_base_component_t *)item->cli_component;
-            opal_output(
-                0, "mca_common_sm_mpool_init: "
-                "unable to locate allocator: %s - using %s\n",
-                resources->allocator,
-                allocator_component->allocator_version.mca_component_name);
+            allocator_component = (mca_allocator_base_component_t *) item->cli_component;
+            opal_output(0,
+                        "mca_common_sm_mpool_init: "
+                        "unable to locate allocator: %s - using %s\n",
+                        resources->allocator,
+                        allocator_component->allocator_version.mca_component_name);
         } else {
-            opal_output(0, "mca_common_sm_mpool_init: "
+            opal_output(0,
+                        "mca_common_sm_mpool_init: "
                         "unable to locate allocator: %s\n",
                         resources->allocator);
             free(mpool_module);
@@ -110,10 +107,11 @@ mca_mpool_base_module_t *common_sm_mpool_create (mca_common_sm_mpool_resources_t
 
     mpool_module->mem_node = resources->mem_node;
 
-    if (NULL == (mpool_module->sm_common_module =
-        mca_common_sm_module_attach(&resources->bs_meta_buf,
-                                    sizeof(mca_common_sm_module_t), 8))) {
-        opal_output(0, "mca_common_sm_mpool_init: "
+    if (NULL
+        == (mpool_module->sm_common_module = mca_common_sm_module_attach(
+                &resources->bs_meta_buf, sizeof(mca_common_sm_module_t), 8))) {
+        opal_output(0,
+                    "mca_common_sm_mpool_init: "
                     "unable to create shared memory mapping (%s)",
                     resources->bs_meta_buf.seg_name);
         free(mpool_module);
@@ -121,9 +119,9 @@ mca_mpool_base_module_t *common_sm_mpool_create (mca_common_sm_mpool_resources_t
     }
 
     /* setup allocator */
-    mpool_module->sm_allocator =
-      allocator_component->allocator_init (true, mca_common_sm_seg_alloc,
-                                           NULL, mpool_module->sm_common_module);
+    mpool_module->sm_allocator = allocator_component
+                                     ->allocator_init(true, mca_common_sm_seg_alloc, NULL,
+                                                      mpool_module->sm_common_module);
     if (NULL == mpool_module->sm_allocator) {
         opal_output(0, "mca_common_sm_mpool_init: unable to initialize allocator");
         free(mpool_module);
@@ -133,28 +131,26 @@ mca_mpool_base_module_t *common_sm_mpool_create (mca_common_sm_mpool_resources_t
     return &mpool_module->super;
 }
 
-
 /*
  * base address of shared memory mapping
  */
 static void *mca_common_sm_mpool_base(mca_mpool_base_module_t *mpool)
 {
     mca_common_sm_mpool_module_t *sm_mpool = (mca_common_sm_mpool_module_t *) mpool;
-    return (NULL != sm_mpool->sm_common_module) ?
-        sm_mpool->sm_common_module->module_seg_addr : NULL;
+    return (NULL != sm_mpool->sm_common_module) ? sm_mpool->sm_common_module->module_seg_addr
+                                                : NULL;
 }
 
 /**
-  * allocate function
-  */
-static void *mca_common_sm_mpool_alloc (mca_mpool_base_module_t* mpool,
-                                        size_t size, size_t align, uint32_t flags)
+ * allocate function
+ */
+static void *mca_common_sm_mpool_alloc(mca_mpool_base_module_t *mpool, size_t size, size_t align,
+                                       uint32_t flags)
 {
-    mca_common_sm_mpool_module_t* mpool_sm = (mca_common_sm_mpool_module_t*)mpool;
+    mca_common_sm_mpool_module_t *mpool_sm = (mca_common_sm_mpool_module_t *) mpool;
     opal_hwloc_base_memory_segment_t mseg;
 
-    mseg.mbs_start_addr =
-        mpool_sm->sm_allocator->alc_alloc(mpool_sm->sm_allocator, size, align);
+    mseg.mbs_start_addr = mpool_sm->sm_allocator->alc_alloc(mpool_sm->sm_allocator, size, align);
 
     if (mpool_sm->mem_node >= 0) {
         mseg.mbs_len = size;
@@ -165,21 +161,20 @@ static void *mca_common_sm_mpool_alloc (mca_mpool_base_module_t* mpool,
 }
 
 /**
-  * free function
-  */
+ * free function
+ */
 void mca_common_sm_mpool_free(mca_mpool_base_module_t *mpool, void *addr)
 {
-    mca_common_sm_mpool_module_t* mpool_sm = (mca_common_sm_mpool_module_t*)mpool;
+    mca_common_sm_mpool_module_t *mpool_sm = (mca_common_sm_mpool_module_t *) mpool;
     mpool_sm->sm_allocator->alc_free(mpool_sm->sm_allocator, addr);
 }
 
-static void sm_module_finalize(mca_mpool_base_module_t* module)
+static void sm_module_finalize(mca_mpool_base_module_t *module)
 {
-    mca_common_sm_mpool_module_t *sm_module = (mca_common_sm_mpool_module_t*) module;
+    mca_common_sm_mpool_module_t *sm_module = (mca_common_sm_mpool_module_t *) module;
 
     if (NULL != sm_module->sm_common_module) {
-        if (OPAL_SUCCESS ==
-            mca_common_sm_fini(sm_module->sm_common_module)) {
+        if (OPAL_SUCCESS == mca_common_sm_fini(sm_module->sm_common_module)) {
             unlink(sm_module->sm_common_module->shmem_ds.seg_name);
         }
         OBJ_RELEASE(sm_module->sm_common_module);
diff --git a/opal/mca/common/sm/common_sm_mpool.h b/opal/mca/common/sm/common_sm_mpool.h
index fdaa72e1830..99dc81fe7d3 100644
--- a/opal/mca/common/sm/common_sm_mpool.h
+++ b/opal/mca/common/sm/common_sm_mpool.h
@@ -16,6 +16,7 @@
  *                         All rights reserved.
  * Copyright (c) 2020      Amazon.com, Inc. or its affiliates.
  *                         All Rights reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -30,18 +31,18 @@
 
 #include "opal_config.h"
 
-#include "opal/util/event.h"
 #include "opal/mca/shmem/shmem.h"
+#include "opal/util/event.h"
 
-#include "opal/mca/mpool/mpool.h"
 #include "opal/mca/allocator/allocator.h"
+#include "opal/mca/mpool/mpool.h"
 
 BEGIN_C_DECLS
 
 struct mca_common_sm_module_t;
 
 typedef struct mca_common_sm_mpool_resources_t {
-    size_t  size;
+    size_t size;
     int32_t mem_node;
     const char *allocator;
     /* backing store metadata */
@@ -57,7 +58,7 @@ typedef struct mca_common_sm_mpool_module_t {
     int32_t mem_node;
 } mca_common_sm_mpool_module_t;
 
-OPAL_DECLSPEC mca_mpool_base_module_t *common_sm_mpool_create (mca_common_sm_mpool_resources_t *);
+OPAL_DECLSPEC mca_mpool_base_module_t *opal_btl_smcuda_common_sm_mpool_create(mca_common_sm_mpool_resources_t *);
 
 END_C_DECLS
 
diff --git a/opal/mca/common/sm/help-mpi-common-sm.txt b/opal/mca/common/sm/help-mpi-common-sm.txt
index f5630c27e83..d849cff12e5 100644
--- a/opal/mca/common/sm/help-mpi-common-sm.txt
+++ b/opal/mca/common/sm/help-mpi-common-sm.txt
@@ -1,6 +1,6 @@
 # -*- text -*-
 #
-# Copyright (c) 2009-2010 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2009-2022 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2010-2012 Los Alamos National Security, LLC.
 #                         All rights reserved.
 #
@@ -21,14 +21,4 @@ Open MPI, and if not, contact the Open MPI developers.
   Local host:        %s
   Requested size:    %ul
   Control seg size:  %ul
-  Data seg aligment: %ul
-#
-[unexpected message id]
-Open MPI received an unexpected message ID during common sm initialization.
-This is likely an error in Open MPI itself.  If you see this error, you should
-see if there is an update available for Open MPI that addresses this issue, and
-if not, contact the Open MPI developers.
-
-  Local Host:          %s
-  Expected Message ID: %s
-  Message ID Received: %s
+  Data seg alignment: %ul
diff --git a/opal/mca/common/ucx/Makefile.am b/opal/mca/common/ucx/Makefile.am
index c331773ff1f..1c796dc0cae 100644
--- a/opal/mca/common/ucx/Makefile.am
+++ b/opal/mca/common/ucx/Makefile.am
@@ -1,5 +1,6 @@
 #
 # Copyright (c) 2018      Mellanox Technologies.  All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -42,8 +43,8 @@ sources = \
 
 lib_LTLIBRARIES =
 noinst_LTLIBRARIES =
-comp_inst = lib@OPAL_LIB_PREFIX@mca_common_ucx.la
-comp_noinst = lib@OPAL_LIB_PREFIX@mca_common_ucx_noinst.la
+comp_inst = lib@OPAL_LIB_NAME@mca_common_ucx.la
+comp_noinst = lib@OPAL_LIB_NAME@mca_common_ucx_noinst.la
 
 if MCA_BUILD_opal_common_ucx_DSO
 lib_LTLIBRARIES += $(comp_inst)
@@ -51,28 +52,24 @@ else
 noinst_LTLIBRARIES += $(comp_noinst)
 endif
 
-lib@OPAL_LIB_PREFIX@mca_common_ucx_la_SOURCES = \
+lib@OPAL_LIB_NAME@mca_common_ucx_la_SOURCES = \
         $(headers) $(sources)
-lib@OPAL_LIB_PREFIX@mca_common_ucx_la_CFLAGS = \
-		$(common_ucx_CFLAGS)
-lib@OPAL_LIB_PREFIX@mca_common_ucx_la_CPPFLAGS = \
+lib@OPAL_LIB_NAME@mca_common_ucx_la_CPPFLAGS = \
 		$(common_ucx_CPPFLAGS)
-lib@OPAL_LIB_PREFIX@mca_common_ucx_la_LDFLAGS = \
+lib@OPAL_LIB_NAME@mca_common_ucx_la_LDFLAGS = \
         -version-info $(libmca_opal_common_ucx_so_version) \
 		$(common_ucx_LDFLAGS)
-lib@OPAL_LIB_PREFIX@mca_common_ucx_la_LIBADD = \
+lib@OPAL_LIB_NAME@mca_common_ucx_la_LIBADD = \
 		$(common_ucx_LIBS) \
-		$(OMPI_TOP_BUILDDIR)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
+		$(OMPI_TOP_BUILDDIR)/opal/lib@OPAL_LIB_NAME@.la \
 		$(OMPI_TOP_BUILDDIR)/opal/mca/memory/libmca_memory.la
-lib@OPAL_LIB_PREFIX@mca_common_ucx_noinst_la_SOURCES = \
+lib@OPAL_LIB_NAME@mca_common_ucx_noinst_la_SOURCES = \
         $(headers) $(sources)
-lib@OPAL_LIB_PREFIX@mca_common_ucx_noinst_la_CFLAGS = \
-		$(common_ucx_CFLAGS)
-lib@OPAL_LIB_PREFIX@mca_common_ucx_noinst_la_CPPFLAGS = \
+lib@OPAL_LIB_NAME@mca_common_ucx_noinst_la_CPPFLAGS = \
 		$(common_ucx_CPPFLAGS)
-lib@OPAL_LIB_PREFIX@mca_common_ucx_noinst_la_LDFLAGS = \
+lib@OPAL_LIB_NAME@mca_common_ucx_noinst_la_LDFLAGS = \
 		$(common_ucx_LDFLAGS)
-lib@OPAL_LIB_PREFIX@mca_common_ucx_noinst_la_LIBADD = \
+lib@OPAL_LIB_NAME@mca_common_ucx_noinst_la_LIBADD = \
 		$(common_ucx_LIBS)
 
 # Conditionally install the header files
diff --git a/opal/mca/common/ucx/common_ucx.c b/opal/mca/common/ucx/common_ucx.c
index 592044a7ccd..3b21213e134 100644
--- a/opal/mca/common/ucx/common_ucx.c
+++ b/opal/mca/common/ucx/common_ucx.c
@@ -1,8 +1,15 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (C) Mellanox Technologies Ltd. 2018. ALL RIGHTS RESERVED.
  * Copyright (c) 2019      Intel, Inc.  All rights reserved.
  * Copyright (c) 2019      Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2021-2024 Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2022      Google, LLC. All rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2023      NVIDIA Corporation.  All rights reserved.
+ *
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -13,112 +20,145 @@
 #include "opal_config.h"
 
 #include "common_ucx.h"
-#include "opal/mca/base/mca_base_var.h"
 #include "opal/mca/base/mca_base_framework.h"
+#include "opal/mca/base/mca_base_var.h"
 #include "opal/mca/pmix/pmix-internal.h"
 #include "opal/memoryhooks/memory.h"
 #include "opal/util/argv.h"
+#include "opal/util/printf.h"
+
+#include "mpi.h"
 
-#include <ucm/api/ucm.h>
 #include <fnmatch.h>
 #include <stdio.h>
+#include <ucm/api/ucm.h>
 
 /***********************************************************************/
 
 extern mca_base_framework_t opal_memory_base_framework;
 
-opal_common_ucx_module_t opal_common_ucx = {
-    .verbose             = 0,
+opal_common_ucx_module_t opal_common_ucx =
+{
     .progress_iterations = 100,
-    .registered          = 0,
-    .opal_mem_hooks      = 0,
-    .tls                 = NULL
+    .opal_mem_hooks = 1,
+    .tls = NULL,
+    .devices = NULL,
 };
 
-static void opal_common_ucx_mem_release_cb(void *buf, size_t length,
-                                           void *cbdata, bool from_alloc)
+static opal_mutex_t opal_common_ucx_mutex = OPAL_MUTEX_STATIC_INIT;
+
+static void opal_common_ucx_mem_release_cb(void *buf, size_t length, void *cbdata, bool from_alloc)
 {
     ucm_vm_munmap(buf, length);
 }
 
+ucs_thread_mode_t opal_common_ucx_thread_mode(int ompi_mode)
+{
+    switch (ompi_mode) {
+    case MPI_THREAD_MULTIPLE:
+        return UCS_THREAD_MODE_MULTI;
+    case MPI_THREAD_SERIALIZED:
+        return UCS_THREAD_MODE_SERIALIZED;
+    case MPI_THREAD_FUNNELED:
+    case MPI_THREAD_SINGLE:
+        return UCS_THREAD_MODE_SINGLE;
+    default:
+        MCA_COMMON_UCX_WARN("Unknown MPI thread mode %d, using multithread",
+                            ompi_mode);
+        return UCS_THREAD_MODE_MULTI;
+    }
+}
+
 OPAL_DECLSPEC void opal_common_ucx_mca_var_register(const mca_base_component_t *component)
 {
-    static const char *default_tls     = "rc_verbs,ud_verbs,rc_mlx5,dc_mlx5,cuda_ipc,rocm_ipc";
-    static const char *default_devices = "mlx*";
-    static int registered = 0;
-    static int hook_index;
-    static int verbose_index;
-    static int progress_index;
-    static int tls_index;
-    static int devices_index;
-
-    if (!registered) {
-        verbose_index = mca_base_var_register("opal", "opal_common", "ucx", "verbose",
-                                              "Verbose level of the UCX components",
-                                              MCA_BASE_VAR_TYPE_INT, NULL, 0,
-                                              MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
-                                              MCA_BASE_VAR_SCOPE_LOCAL,
-                                              &opal_common_ucx.verbose);
-        progress_index = mca_base_var_register("opal", "opal_common", "ucx", "progress_iterations",
-                                               "Set number of calls of internal UCX progress "
-                                               "calls per opal_progress call",
-                                               MCA_BASE_VAR_TYPE_INT, NULL, 0,
-                                               MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
-                                               MCA_BASE_VAR_SCOPE_LOCAL,
-                                               &opal_common_ucx.progress_iterations);
-        hook_index = mca_base_var_register("opal", "opal_common", "ucx", "opal_mem_hooks",
-                                           "Use OPAL memory hooks, instead of UCX internal "
-                                           "memory hooks", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
-                                           OPAL_INFO_LVL_3,
+    char *default_tls = "rc_verbs,ud_verbs,rc_mlx5,dc_mlx5,ud_mlx5,cuda_ipc,rocm_ipc";
+    char *default_devices = "mlx*";
+    int hook_index;
+    int verbose_index;
+    int progress_index;
+    int tls_index;
+    int devices_index;
+
+    OPAL_THREAD_LOCK(&opal_common_ucx_mutex);
+
+    /* It is harmless to re-register variables so go ahead an re-register. */
+    verbose_index = mca_base_var_register("opal", "opal_common", "ucx", "verbose",
+                                          "Verbose level of the UCX components",
+                                          MCA_BASE_VAR_TYPE_INT, NULL, 0,
+                                          MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
+                                          MCA_BASE_VAR_SCOPE_LOCAL, &opal_common_ucx.verbose);
+    progress_index = mca_base_var_register("opal", "opal_common", "ucx", "progress_iterations",
+                                           "Set number of calls of internal UCX progress "
+                                           "calls per opal_progress call",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0,
+                                           MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
                                            MCA_BASE_VAR_SCOPE_LOCAL,
-                                           &opal_common_ucx.opal_mem_hooks);
+                                           &opal_common_ucx.progress_iterations);
+    hook_index = mca_base_var_register("opal", "opal_common", "ucx", "opal_mem_hooks",
+                                       "Use OPAL memory hooks, instead of UCX internal "
+                                       "memory hooks",
+                                       MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_3,
+                                       MCA_BASE_VAR_SCOPE_LOCAL,
+                                       &opal_common_ucx.opal_mem_hooks);
+
+    if (NULL == opal_common_ucx.tls) {
+        // Extra level of string indirection needed to make ompi_info
+        // happy since it will unload this library before the MCA base
+        // cleans up the MCA vars. This will cause the string to go
+        // out of scope unless we place the pointer to it on the heap.
+        opal_common_ucx.tls = (char **) malloc(sizeof(char *));
+        *opal_common_ucx.tls = NULL;
+    }
 
-        opal_common_ucx.tls  = malloc(sizeof(*opal_common_ucx.tls));
+    if (NULL == *opal_common_ucx.tls) {
         *opal_common_ucx.tls = strdup(default_tls);
-        tls_index = mca_base_var_register("opal", "opal_common", "ucx", "tls",
-                                          "List of UCX transports which should be supported on the system, to enable "
-                                          "selecting the UCX component. Special values: any (any available). "
-                                          "A '^' prefix negates the list. "
-                                          "For example, in order to exclude on shared memory and TCP transports, "
-                                          "please set to '^posix,sysv,self,tcp,cma,knem,xpmem'.",
-                                          MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
-                                          OPAL_INFO_LVL_3,
-                                          MCA_BASE_VAR_SCOPE_LOCAL,
-                                          opal_common_ucx.tls);
-
-        opal_common_ucx.devices  = malloc(sizeof(*opal_common_ucx.devices));
+    }
+
+    tls_index = mca_base_var_register(
+        "opal", "opal_common", "ucx", "tls",
+        "List of UCX transports which should be supported on the system, to enable "
+        "selecting the UCX component. Special values: any (any available). "
+        "A '^' prefix negates the list. "
+        "For example, in order to exclude on shared memory and TCP transports, "
+        "please set to '^posix,sysv,self,tcp,cma,knem,xpmem'.",
+        MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE | MCA_BASE_VAR_FLAG_DWG,
+        OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_LOCAL, opal_common_ucx.tls);
+
+    if (NULL == opal_common_ucx.devices) {
+        opal_common_ucx.devices = (char**) malloc(sizeof(char*));
+        *opal_common_ucx.devices = NULL;
+    }
+
+    if (NULL == *opal_common_ucx.devices) {
         *opal_common_ucx.devices = strdup(default_devices);
-        devices_index = mca_base_var_register("opal", "opal_common", "ucx", "devices",
-                                              "List of device driver pattern names, which, if supported by UCX, will "
-                                              "bump its priority above ob1. Special values: any (any available)",
-                                              MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
-                                              OPAL_INFO_LVL_3,
-                                              MCA_BASE_VAR_SCOPE_LOCAL,
-                                              opal_common_ucx.devices);
-        registered = 1;
     }
+
+    devices_index = mca_base_var_register(
+        "opal", "opal_common", "ucx", "devices",
+        "List of device driver pattern names, which, if supported by UCX, will "
+        "bump its priority above ob1. Special values: any (any available)",
+        MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE | MCA_BASE_VAR_FLAG_DWG,
+        OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_LOCAL, opal_common_ucx.devices);
+
     if (component) {
         mca_base_var_register_synonym(verbose_index, component->mca_project_name,
-                                      component->mca_type_name,
-                                      component->mca_component_name,
+                                      component->mca_type_name, component->mca_component_name,
                                       "verbose", 0);
         mca_base_var_register_synonym(progress_index, component->mca_project_name,
-                                      component->mca_type_name,
-                                      component->mca_component_name,
+                                      component->mca_type_name, component->mca_component_name,
                                       "progress_iterations", 0);
         mca_base_var_register_synonym(hook_index, component->mca_project_name,
-                                      component->mca_type_name,
-                                      component->mca_component_name,
+                                      component->mca_type_name, component->mca_component_name,
                                       "opal_mem_hooks", 0);
         mca_base_var_register_synonym(tls_index, component->mca_project_name,
-                                      component->mca_type_name,
-                                      component->mca_component_name,
+                                      component->mca_type_name, component->mca_component_name,
                                       "tls", 0);
         mca_base_var_register_synonym(devices_index, component->mca_project_name,
-                                      component->mca_type_name,
-                                      component->mca_component_name,
+                                      component->mca_type_name, component->mca_component_name,
                                       "devices", 0);
     }
+
+    OPAL_THREAD_UNLOCK(&opal_common_ucx_mutex);
 }
 
 OPAL_DECLSPEC void opal_common_ucx_mca_register(void)
@@ -139,14 +179,16 @@ OPAL_DECLSPEC void opal_common_ucx_mca_register(void)
         ret = mca_base_framework_open(&opal_memory_base_framework, 0);
         if (OPAL_SUCCESS != ret) {
             /* failed to initialize memory framework - just exit */
-            MCA_COMMON_UCX_VERBOSE(1, "failed to initialize memory base framework: %d, "
-                                      "memory hooks will not be used", ret);
+            MCA_COMMON_UCX_VERBOSE(1,
+                                   "failed to initialize memory base framework: %d, "
+                                   "memory hooks will not be used",
+                                   ret);
             return;
         }
 
-        if ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) ==
-            ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) &
-             opal_mem_hooks_support_level())) {
+        if ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT)
+            == ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT)
+                & opal_mem_hooks_support_level())) {
             MCA_COMMON_UCX_VERBOSE(1, "%s", "using OPAL memory hooks as external events");
             ucm_set_external_event(UCM_EVENT_VM_UNMAPPED);
             opal_mem_hooks_register_release(opal_common_ucx_mem_release_cb, NULL);
@@ -169,15 +211,18 @@ OPAL_DECLSPEC void opal_common_ucx_mca_deregister(void)
 #if HAVE_DECL_OPEN_MEMSTREAM
 static bool opal_common_ucx_check_device(const char *device_name, char **device_list)
 {
-    char sysfs_driver_link[PATH_MAX];
-    char driver_path[PATH_MAX];
-    char *ib_device_name;
+    char sysfs_driver_link[OPAL_PATH_MAX];
+    char driver_path[OPAL_PATH_MAX];
+    char ib_device_name[NAME_MAX];
     char *driver_name;
     char **list_item;
     ssize_t ret;
+    char ib_device_name_fmt[NAME_MAX];
 
     /* mlx5_0:1 */
-    ret = sscanf(device_name, "%m[^:]%*d", &ib_device_name);
+    opal_snprintf(ib_device_name_fmt, sizeof(ib_device_name_fmt),
+                  "%%%u[^:]%%*d", NAME_MAX - 1);
+    ret = sscanf(device_name, ib_device_name_fmt, &ib_device_name);
     if (ret != 1) {
         return false;
     }
@@ -185,21 +230,18 @@ static bool opal_common_ucx_check_device(const char *device_name, char **device_
     sysfs_driver_link[sizeof(sysfs_driver_link) - 1] = '\0';
     snprintf(sysfs_driver_link, sizeof(sysfs_driver_link) - 1,
              "/sys/class/infiniband/%s/device/driver", ib_device_name);
-    free(ib_device_name);
 
-    driver_path[sizeof(driver_path) - 1] = '\0';
     ret = readlink(sysfs_driver_link, driver_path, sizeof(driver_path) - 1);
     if (ret < 0) {
-        MCA_COMMON_UCX_VERBOSE(2, "readlink(%s) failed: %s", sysfs_driver_link,
-                               strerror(errno));
+        MCA_COMMON_UCX_VERBOSE(2, "readlink(%s) failed: %s", sysfs_driver_link, strerror(errno));
         return false;
     }
+    driver_path[ret] = '\0'; /* readlink does not append \0 */
 
     driver_name = basename(driver_path);
     for (list_item = device_list; *list_item != NULL; ++list_item) {
         if (!fnmatch(*list_item, driver_name, 0)) {
-            MCA_COMMON_UCX_VERBOSE(2, "driver '%s' matched by '%s'",
-                                   driver_path, *list_item);
+            MCA_COMMON_UCX_VERBOSE(2, "driver '%s' matched by '%s'", driver_path, *list_item);
             return true;
         }
     }
@@ -208,17 +250,16 @@ static bool opal_common_ucx_check_device(const char *device_name, char **device_
 }
 #endif
 
-OPAL_DECLSPEC opal_common_ucx_support_level_t
-opal_common_ucx_support_level(ucp_context_h context)
+OPAL_DECLSPEC opal_common_ucx_support_level_t opal_common_ucx_support_level(ucp_context_h context)
 {
     opal_common_ucx_support_level_t support_level = OPAL_COMMON_UCX_SUPPORT_NONE;
-    static const char *support_level_names[] = {
-        [OPAL_COMMON_UCX_SUPPORT_NONE]      = "none",
-        [OPAL_COMMON_UCX_SUPPORT_TRANSPORT] = "transports only",
-        [OPAL_COMMON_UCX_SUPPORT_DEVICE]    = "transports and devices"
-    };
+    static const char *support_level_names[]
+        = {[OPAL_COMMON_UCX_SUPPORT_NONE] = "none",
+           [OPAL_COMMON_UCX_SUPPORT_TRANSPORT] = "transports only",
+           [OPAL_COMMON_UCX_SUPPORT_DEVICE] = "transports and devices"};
 #if HAVE_DECL_OPEN_MEMSTREAM
-    char *rsc_tl_name, *rsc_device_name;
+    char rsc_tl_name[NAME_MAX], rsc_device_name[NAME_MAX];
+    char rsc_name_fmt[NAME_MAX];
     char **tl_list, **device_list, **list_item;
     bool is_any_tl, is_any_device;
     bool found_tl, negate;
@@ -229,20 +270,23 @@ opal_common_ucx_support_level(ucp_context_h context)
     int ret;
 #endif
 
-    is_any_tl     = !strcmp(*opal_common_ucx.tls, "any");
+    if ((*opal_common_ucx.tls == NULL) || (*opal_common_ucx.devices == NULL)) {
+        opal_common_ucx_mca_var_register(NULL);
+    }
+
+    is_any_tl = !strcmp(*opal_common_ucx.tls, "any");
     is_any_device = !strcmp(*opal_common_ucx.devices, "any");
 
     /* Check for special value "any" */
     if (is_any_tl && is_any_device) {
-        MCA_COMMON_UCX_VERBOSE(1, "ucx is enabled on any transport or device",
-                               *opal_common_ucx.tls);
+        MCA_COMMON_UCX_VERBOSE(1, "ucx is enabled on any transport or device");
         support_level = OPAL_COMMON_UCX_SUPPORT_DEVICE;
         goto out;
     }
 
 #if HAVE_DECL_OPEN_MEMSTREAM
     /* Split transports list */
-    negate  = ('^' == (*opal_common_ucx.tls)[0]);
+    negate = ('^' == (*opal_common_ucx.tls)[0]);
     tl_list = opal_argv_split(*opal_common_ucx.tls + (negate ? 1 : 0), ',');
     if (tl_list == NULL) {
         MCA_COMMON_UCX_VERBOSE(1, "failed to split tl list '%s', ucx is disabled",
@@ -261,25 +305,27 @@ opal_common_ucx_support_level(ucp_context_h context)
     /* Open memory stream to dump UCX information to */
     stream = open_memstream(&buffer, &size);
     if (stream == NULL) {
-        MCA_COMMON_UCX_VERBOSE(1, "failed to open memory stream for ucx info (%s), "
-                               "ucx is disabled", strerror(errno));
+        MCA_COMMON_UCX_VERBOSE(1,
+                               "failed to open memory stream for ucx info (%s), "
+                               "ucx is disabled",
+                               strerror(errno));
         goto out_free_device_list;
     }
 
     /* Print ucx transports information to the memory stream */
     ucp_context_print_info(context, stream);
 
+    /* "# resource 6  :  md 5  dev 4  flags -- rc_verbs/mlx5_0:1" */
+    opal_snprintf(rsc_name_fmt, sizeof(rsc_name_fmt),
+        "# resource %%*d : md %%*d dev %%*d flags -- %%%u[^/ \n\r]/%%%u[^/ \n\r]",
+        NAME_MAX - 1, NAME_MAX - 1);
+
     /* Rewind and read transports/devices list from the stream */
     fseek(stream, 0, SEEK_SET);
-    while ((support_level != OPAL_COMMON_UCX_SUPPORT_DEVICE) &&
-           (fgets(line, sizeof(line), stream) != NULL)) {
-        rsc_tl_name = NULL;
-        ret = sscanf(line,
-                     /* "# resource 6  :  md 5  dev 4  flags -- rc_verbs/mlx5_0:1" */
-                     "# resource %*d : md %*d dev %*d flags -- %m[^/ \n\r]/%m[^/ \n\r]",
-                     &rsc_tl_name, &rsc_device_name);
+    while ((support_level != OPAL_COMMON_UCX_SUPPORT_DEVICE)
+           && (fgets(line, sizeof(line), stream) != NULL)) {
+        ret = sscanf(line, rsc_name_fmt, rsc_tl_name, rsc_device_name);
         if (ret != 2) {
-            free(rsc_tl_name);
             continue;
         }
 
@@ -292,23 +338,19 @@ opal_common_ucx_support_level(ucp_context_h context)
         /* Check if the transport has a match (either positive or negative) */
         assert(!(is_any_tl && negate));
         if (found_tl != negate) {
-            if (is_any_device ||
-                opal_common_ucx_check_device(rsc_device_name, device_list)) {
+            if (is_any_device || opal_common_ucx_check_device(rsc_device_name, device_list)) {
                 MCA_COMMON_UCX_VERBOSE(2, "%s/%s: matched both transport and device list",
-                                    rsc_tl_name, rsc_device_name);
+                                       rsc_tl_name, rsc_device_name);
                 support_level = OPAL_COMMON_UCX_SUPPORT_DEVICE;
             } else {
                 MCA_COMMON_UCX_VERBOSE(2, "%s/%s: matched transport list but not device list",
-                                    rsc_tl_name, rsc_device_name);
+                                       rsc_tl_name, rsc_device_name);
                 support_level = OPAL_COMMON_UCX_SUPPORT_TRANSPORT;
             }
         } else {
-            MCA_COMMON_UCX_VERBOSE(2, "%s/%s: did not match transport list",
-                                   rsc_tl_name, rsc_device_name);
+            MCA_COMMON_UCX_VERBOSE(2, "%s/%s: did not match transport list", rsc_tl_name,
+                                   rsc_device_name);
         }
-
-        free(rsc_device_name);
-        free(rsc_tl_name);
     }
 
     MCA_COMMON_UCX_VERBOSE(2, "support level is %s", support_level_names[support_level]);
@@ -332,17 +374,17 @@ void opal_common_ucx_empty_complete_cb(void *request, ucs_status_t status)
 
 static void opal_common_ucx_mca_fence_complete_cb(int status, void *fenced)
 {
-    *(int*)fenced = 1;
+    *(int *) fenced = 1;
 }
 
 #if HAVE_DECL_UCM_TEST_EVENTS
 static ucs_status_t opal_common_ucx_mca_test_external_events(int events)
 {
-#if HAVE_DECL_UCM_TEST_EXTERNAL_EVENTS
+#    if HAVE_DECL_UCM_TEST_EXTERNAL_EVENTS
     return ucm_test_external_events(UCM_EVENT_VM_UNMAPPED);
-#else
+#    else
     return ucm_test_events(UCM_EVENT_VM_UNMAPPED);
-#endif
+#    endif
 }
 
 static void opal_common_ucx_mca_test_events(void)
@@ -354,17 +396,18 @@ static void opal_common_ucx_mca_test_events(void)
     if (!warned) {
         if (opal_common_ucx.opal_mem_hooks) {
             suggestion = "Please check OPAL memory events infrastructure.";
-            status     = opal_common_ucx_mca_test_external_events(UCM_EVENT_VM_UNMAPPED);
+            status = opal_common_ucx_mca_test_external_events(UCM_EVENT_VM_UNMAPPED);
         } else {
             suggestion = "Pls try adding --mca opal_common_ucx_opal_mem_hooks 1 "
                          "to mpirun/oshrun command line to resolve this issue.";
-            status     = ucm_test_events(UCM_EVENT_VM_UNMAPPED);
+            status = ucm_test_events(UCM_EVENT_VM_UNMAPPED);
         }
 
         if (status != UCS_OK) {
             MCA_COMMON_UCX_WARN("UCX is unable to handle VM_UNMAP event. "
                                 "This may cause performance degradation or data "
-                                "corruption. %s", suggestion);
+                                "corruption. %s",
+                                suggestion);
             warned = 1;
         }
     }
@@ -380,7 +423,7 @@ void opal_common_ucx_mca_proc_added(void)
 
 OPAL_DECLSPEC int opal_common_ucx_mca_pmix_fence_nb(int *fenced)
 {
-    return PMIx_Fence_nb(NULL, 0, NULL, 0, opal_common_ucx_mca_fence_complete_cb, (void *)fenced);
+    return PMIx_Fence_nb(NULL, 0, NULL, 0, opal_common_ucx_mca_fence_complete_cb, (void *) fenced);
 }
 
 OPAL_DECLSPEC int opal_common_ucx_mca_pmix_fence(ucp_worker_h worker)
@@ -388,19 +431,21 @@ OPAL_DECLSPEC int opal_common_ucx_mca_pmix_fence(ucp_worker_h worker)
     volatile int fenced = 0;
     int ret = OPAL_SUCCESS;
 
-    if (OPAL_SUCCESS != (ret = PMIx_Fence_nb(NULL, 0, NULL, 0,
-                    opal_common_ucx_mca_fence_complete_cb, (void*)&fenced))){
+    if (OPAL_SUCCESS
+        != (ret = PMIx_Fence_nb(NULL, 0, NULL, 0, opal_common_ucx_mca_fence_complete_cb,
+                                (void *) &fenced))) {
         return ret;
     }
 
-    while (!fenced) {
-        ucp_worker_progress(worker);
+    MCA_COMMON_UCX_PROGRESS_LOOP(worker) {
+        if(fenced) {
+            break;
+        }
     }
 
     return ret;
 }
 
-
 static void opal_common_ucx_wait_all_requests(void **reqs, int count, ucp_worker_h worker)
 {
     int i;
@@ -412,9 +457,8 @@ static void opal_common_ucx_wait_all_requests(void **reqs, int count, ucp_worker
     }
 }
 
-OPAL_DECLSPEC int opal_common_ucx_del_procs_nofence(opal_common_ucx_del_proc_t *procs,
-                                                    size_t count, size_t my_rank,
-                                                    size_t max_disconnect,
+OPAL_DECLSPEC int opal_common_ucx_del_procs_nofence(opal_common_ucx_del_proc_t *procs, size_t count,
+                                                    size_t my_rank, size_t max_disconnect,
                                                     ucp_worker_h worker)
 {
     size_t num_reqs;
@@ -467,10 +511,15 @@ OPAL_DECLSPEC int opal_common_ucx_del_procs_nofence(opal_common_ucx_del_proc_t *
 }
 
 OPAL_DECLSPEC int opal_common_ucx_del_procs(opal_common_ucx_del_proc_t *procs, size_t count,
-                                            size_t my_rank, size_t max_disconnect, ucp_worker_h worker)
+                                            size_t my_rank, size_t max_disconnect,
+                                            ucp_worker_h worker)
 {
     opal_common_ucx_del_procs_nofence(procs, count, my_rank, max_disconnect, worker);
 
     return opal_common_ucx_mca_pmix_fence(worker);
 }
 
+static void safety_valve(void) __opal_attribute_destructor__;
+void safety_valve(void) {
+    opal_mem_hooks_unregister_release(opal_common_ucx_mem_release_cb);
+}
diff --git a/opal/mca/common/ucx/common_ucx.h b/opal/mca/common/ucx/common_ucx.h
index a96ba24f2f5..13e69891869 100644
--- a/opal/mca/common/ucx/common_ucx.h
+++ b/opal/mca/common/ucx/common_ucx.h
@@ -5,6 +5,7 @@
  *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2019-2020 High Performance Computing Center Stuttgart,
  *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2023      NVIDIA Corporation. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -21,80 +22,74 @@
 
 #include <ucp/api/ucp.h>
 
+#include "opal/class/opal_list.h"
+#include "opal/include/opal/constants.h"
 #include "opal/mca/mca.h"
-#include "opal/util/output.h"
 #include "opal/runtime/opal_progress.h"
-#include "opal/include/opal/constants.h"
-#include "opal/class/opal_list.h"
+#include "opal/util/output.h"
 
 BEGIN_C_DECLS
 
-#define MCA_COMMON_UCX_ENABLE_DEBUG   OPAL_ENABLE_DEBUG
+#define MCA_COMMON_UCX_ENABLE_DEBUG OPAL_ENABLE_DEBUG
 #if MCA_COMMON_UCX_ENABLE_DEBUG
-#  define MCA_COMMON_UCX_MAX_VERBOSE  100
-#  define MCA_COMMON_UCX_ASSERT(_x)   assert(_x)
+#    define MCA_COMMON_UCX_MAX_VERBOSE 100
+#    define MCA_COMMON_UCX_ASSERT(_x)  assert(_x)
 #else
-#  define MCA_COMMON_UCX_MAX_VERBOSE  2
-#  define MCA_COMMON_UCX_ASSERT(_x)
+#    define MCA_COMMON_UCX_MAX_VERBOSE 2
+#    define MCA_COMMON_UCX_ASSERT(_x)
 #endif
 
 #define MCA_COMMON_UCX_PER_TARGET_OPS_THRESHOLD 1000
-#define MCA_COMMON_UCX_GLOBAL_OPS_THRESHOLD 1000
+#define MCA_COMMON_UCX_GLOBAL_OPS_THRESHOLD     1000
 
-#define UCX_VERSION(_major, _minor, _build) (((_major) * 100) + (_minor))
+#define UCX_VERSION(_major, _minor, _build) (((_major) *100) + (_minor))
 
+#define _MCA_COMMON_UCX_QUOTE(_x) #_x
+#define MCA_COMMON_UCX_QUOTE(_x)  _MCA_COMMON_UCX_QUOTE(_x)
 
-#define _MCA_COMMON_UCX_QUOTE(_x) \
-    # _x
-#define MCA_COMMON_UCX_QUOTE(_x) \
-    _MCA_COMMON_UCX_QUOTE(_x)
+#define MCA_COMMON_UCX_ERROR(...) MCA_COMMON_UCX_VERBOSE(0, " Error: " __VA_ARGS__)
 
-#define MCA_COMMON_UCX_ERROR(...) \
-    MCA_COMMON_UCX_VERBOSE(0, " Error: " __VA_ARGS__)
+#define MCA_COMMON_UCX_WARN(...) MCA_COMMON_UCX_VERBOSE(0, " Warning: " __VA_ARGS__)
 
-#define MCA_COMMON_UCX_WARN(...) \
-    MCA_COMMON_UCX_VERBOSE(0, " Warning: " __VA_ARGS__)
-
-#define MCA_COMMON_UCX_VERBOSE(_level, ... )                                \
-    if (((_level) <= MCA_COMMON_UCX_MAX_VERBOSE) &&                         \
-        ((_level) <= opal_common_ucx.verbose)) {                            \
-        opal_output_verbose(_level, opal_common_ucx.output,                 \
-                            __FILE__ ":" MCA_COMMON_UCX_QUOTE(__LINE__) " " \
-                            __VA_ARGS__);                                   \
+#define MCA_COMMON_UCX_VERBOSE(_level, ...)                                                  \
+    if (((_level) <= MCA_COMMON_UCX_MAX_VERBOSE) && ((_level) <= opal_common_ucx.verbose)) { \
+        opal_output_verbose(_level, opal_common_ucx.output,                                  \
+                            __FILE__ ":" MCA_COMMON_UCX_QUOTE(__LINE__) " " __VA_ARGS__);    \
     }
 
 /* progress loop to allow call UCX/opal progress */
 /* used C99 for-statement variable initialization */
-#define MCA_COMMON_UCX_PROGRESS_LOOP(_worker)                                 \
-    for (unsigned iter = 0;; (++iter % opal_common_ucx.progress_iterations) ? \
-                        (void)ucp_worker_progress(_worker) : opal_progress())
-
-#define MCA_COMMON_UCX_WAIT_LOOP(_request, _worker, _msg, _completed)                    \
-    do {                                                                                 \
-        ucs_status_t status;                                                             \
-        /* call UCX progress */                                                          \
-        MCA_COMMON_UCX_PROGRESS_LOOP(_worker) {                                          \
-            status = opal_common_ucx_request_status(_request);                           \
-            if (UCS_INPROGRESS != status) {                                              \
-                _completed;                                                              \
-                if (OPAL_LIKELY(UCS_OK == status)) {                                     \
-                    return OPAL_SUCCESS;                                                 \
-                } else {                                                                 \
-                    MCA_COMMON_UCX_VERBOSE(1, "%s failed: %d, %s",                       \
-                                           (_msg) ? (_msg) : __func__,                   \
-                                           UCS_PTR_STATUS(_request),                     \
-                                           ucs_status_string(UCS_PTR_STATUS(_request))); \
-                    return OPAL_ERROR;                                                   \
-                }                                                                        \
-            }                                                                            \
-        }                                                                                \
+#define MCA_COMMON_UCX_PROGRESS_LOOP(_worker)                               \
+    for (unsigned iter = 0;; (++iter % opal_common_ucx.progress_iterations) \
+                                 ? (void) ucp_worker_progress(_worker)      \
+                                 : (void) opal_progress())
+
+#define MCA_COMMON_UCX_WAIT_LOOP(_request, _worker, _msg, _completed)                          \
+    do {                                                                                       \
+        ucs_status_t status;                                                                   \
+        /* call UCX progress */                                                                \
+        MCA_COMMON_UCX_PROGRESS_LOOP(_worker)                                                  \
+        {                                                                                      \
+            status = opal_common_ucx_request_status(_request);                                 \
+            if (UCS_INPROGRESS != status) {                                                    \
+                _completed;                                                                    \
+                if (OPAL_LIKELY(UCS_OK == status)) {                                           \
+                    return OPAL_SUCCESS;                                                       \
+                } else {                                                                       \
+                    MCA_COMMON_UCX_VERBOSE(1, "%s failed: %d, %s", (_msg) ? (_msg) : __func__, \
+                                           UCS_PTR_STATUS(_request),                           \
+                                           ucs_status_string(UCS_PTR_STATUS(_request)));       \
+                    return OPAL_ERROR;                                                         \
+                }                                                                              \
+            }                                                                                  \
+        }                                                                                      \
     } while (0)
 
 typedef struct opal_common_ucx_module {
-    int  output;
-    int  verbose;
-    int  progress_iterations;
-    int  registered;
+    int output;
+    int verbose;
+    int progress_iterations;
+    int registered;
     bool opal_mem_hooks;
     char **tls;
     char **devices;
@@ -102,7 +97,7 @@ typedef struct opal_common_ucx_module {
 
 typedef struct opal_common_ucx_del_proc {
     ucp_ep_h ep;
-    size_t   vpid;
+    size_t vpid;
 } opal_common_ucx_del_proc_t;
 
 typedef enum {
@@ -127,48 +122,47 @@ OPAL_DECLSPEC void opal_common_ucx_empty_complete_cb(void *request, ucs_status_t
 OPAL_DECLSPEC int opal_common_ucx_mca_pmix_fence(ucp_worker_h worker);
 OPAL_DECLSPEC int opal_common_ucx_mca_pmix_fence_nb(int *fenced);
 OPAL_DECLSPEC int opal_common_ucx_del_procs(opal_common_ucx_del_proc_t *procs, size_t count,
-                                            size_t my_rank, size_t max_disconnect, ucp_worker_h worker);
+                                            size_t my_rank, size_t max_disconnect,
+                                            ucp_worker_h worker);
 OPAL_DECLSPEC int opal_common_ucx_del_procs_nofence(opal_common_ucx_del_proc_t *procs, size_t count,
-                                               size_t my_rank, size_t max_disconnect, ucp_worker_h worker);
+                                                    size_t my_rank, size_t max_disconnect,
+                                                    ucp_worker_h worker);
 OPAL_DECLSPEC void opal_common_ucx_mca_var_register(const mca_base_component_t *component);
+OPAL_DECLSPEC ucs_thread_mode_t opal_common_ucx_thread_mode(int ompi_mode);
 
 /**
  * Load an integer value of \c size bytes from \c ptr and cast it to uint64_t.
  */
-static inline
-uint64_t opal_common_ucx_load_uint64(const void *ptr, size_t size)
+static inline uint64_t opal_common_ucx_load_uint64(const void *ptr, size_t size)
 {
     if (sizeof(uint8_t) == size) {
-      return *(uint8_t*)ptr;
+        return *(uint8_t *) ptr;
     } else if (sizeof(uint16_t) == size) {
-      return *(uint16_t*)ptr;
+        return *(uint16_t *) ptr;
     } else if (sizeof(uint32_t) == size) {
-      return *(uint32_t*)ptr;
+        return *(uint32_t *) ptr;
     } else {
-      return *(uint64_t*)ptr;
+        return *(uint64_t *) ptr;
     }
 }
 
 /**
  * Cast and store a uint64_t value to a value of \c size bytes pointed to by \c ptr.
  */
-static inline
-void opal_common_ucx_store_uint64(uint64_t value, void *ptr, size_t size)
+static inline void opal_common_ucx_store_uint64(uint64_t value, void *ptr, size_t size)
 {
     if (sizeof(uint8_t) == size) {
-      *(uint8_t*)ptr  = value;
+        *(uint8_t *) ptr = value;
     } else if (sizeof(uint16_t) == size) {
-      *(uint16_t*)ptr = value;
+        *(uint16_t *) ptr = value;
     } else if (sizeof(uint32_t) == size) {
-      *(uint32_t*)ptr = value;
+        *(uint32_t *) ptr = value;
     } else {
-      *(uint64_t*)ptr = value;
+        *(uint64_t *) ptr = value;
     }
 }
 
-
-static inline
-ucs_status_t opal_common_ucx_request_status(ucs_status_ptr_t request)
+static inline ucs_status_t opal_common_ucx_request_status(ucs_status_ptr_t request)
 {
 #if !HAVE_DECL_UCP_REQUEST_CHECK_STATUS
     ucp_tag_recv_info_t info;
@@ -179,25 +173,22 @@ ucs_status_t opal_common_ucx_request_status(ucs_status_ptr_t request)
 #endif
 }
 
-static inline
-int opal_common_ucx_wait_request(ucs_status_ptr_t request, ucp_worker_h worker,
-                                 const char *msg)
+static inline int opal_common_ucx_wait_request(ucs_status_ptr_t request, ucp_worker_h worker,
+                                               const char *msg)
 {
     /* check for request completed or failed */
     if (OPAL_LIKELY(UCS_OK == request)) {
         return OPAL_SUCCESS;
     } else if (OPAL_UNLIKELY(UCS_PTR_IS_ERR(request))) {
         MCA_COMMON_UCX_VERBOSE(1, "%s failed: %d, %s", msg ? msg : __func__,
-                               UCS_PTR_STATUS(request),
-                               ucs_status_string(UCS_PTR_STATUS(request)));
+                               UCS_PTR_STATUS(request), ucs_status_string(UCS_PTR_STATUS(request)));
         return OPAL_ERROR;
     }
 
     MCA_COMMON_UCX_WAIT_LOOP(request, worker, msg, ucp_request_free(request));
 }
 
-static inline
-int opal_common_ucx_ep_flush(ucp_ep_h ep, ucp_worker_h worker)
+static inline int opal_common_ucx_ep_flush(ucp_ep_h ep, ucp_worker_h worker)
 {
 #if HAVE_DECL_UCP_EP_FLUSH_NB
     ucs_status_ptr_t request;
@@ -212,8 +203,7 @@ int opal_common_ucx_ep_flush(ucp_ep_h ep, ucp_worker_h worker)
 #endif
 }
 
-static inline
-int opal_common_ucx_worker_flush(ucp_worker_h worker)
+static inline int opal_common_ucx_worker_flush(ucp_worker_h worker)
 {
 #if HAVE_DECL_UCP_WORKER_FLUSH_NB
     ucs_status_ptr_t request;
@@ -228,51 +218,43 @@ int opal_common_ucx_worker_flush(ucp_worker_h worker)
 #endif
 }
 
-static inline
-int opal_common_ucx_atomic_fetch(ucp_ep_h ep, ucp_atomic_fetch_op_t opcode,
-                                 uint64_t value, void *result, size_t op_size,
-                                 uint64_t remote_addr, ucp_rkey_h rkey,
-                                 ucp_worker_h worker)
+static inline int opal_common_ucx_atomic_fetch(ucp_ep_h ep, ucp_atomic_fetch_op_t opcode,
+                                               uint64_t value, void *result, size_t op_size,
+                                               uint64_t remote_addr, ucp_rkey_h rkey,
+                                               ucp_worker_h worker)
 {
     ucs_status_ptr_t request;
 
-    request = ucp_atomic_fetch_nb(ep, opcode, value, result, op_size,
-                                  remote_addr, rkey, opal_common_ucx_empty_complete_cb);
+    request = ucp_atomic_fetch_nb(ep, opcode, value, result, op_size, remote_addr, rkey,
+                                  opal_common_ucx_empty_complete_cb);
     return opal_common_ucx_wait_request(request, worker, "ucp_atomic_fetch_nb");
 }
 
-static inline
-ucs_status_ptr_t opal_common_ucx_atomic_fetch_nb(ucp_ep_h ep, ucp_atomic_fetch_op_t opcode,
-                                                 uint64_t value, void *result, size_t op_size,
-                                                 uint64_t remote_addr, ucp_rkey_h rkey,
-                                                 ucp_send_callback_t req_handler,
-                                                 ucp_worker_h worker)
+static inline ucs_status_ptr_t
+opal_common_ucx_atomic_fetch_nb(ucp_ep_h ep, ucp_atomic_fetch_op_t opcode, uint64_t value,
+                                void *result, size_t op_size, uint64_t remote_addr, ucp_rkey_h rkey,
+                                ucp_send_callback_t req_handler, ucp_worker_h worker)
 {
-    return ucp_atomic_fetch_nb(ep, opcode, value, result, op_size,
-                               remote_addr, rkey, req_handler);
+    return ucp_atomic_fetch_nb(ep, opcode, value, result, op_size, remote_addr, rkey, req_handler);
 }
 
-static inline
-int opal_common_ucx_atomic_cswap(ucp_ep_h ep, uint64_t compare,
-                                 uint64_t value, void *result, size_t op_size,
-                                 uint64_t remote_addr, ucp_rkey_h rkey,
-                                 ucp_worker_h worker)
+static inline int opal_common_ucx_atomic_cswap(ucp_ep_h ep, uint64_t compare, uint64_t value,
+                                               void *result, size_t op_size, uint64_t remote_addr,
+                                               ucp_rkey_h rkey, ucp_worker_h worker)
 {
     opal_common_ucx_store_uint64(value, result, op_size);
-    return opal_common_ucx_atomic_fetch(ep, UCP_ATOMIC_FETCH_OP_CSWAP, compare, result,
-                                        op_size, remote_addr, rkey, worker);
+    return opal_common_ucx_atomic_fetch(ep, UCP_ATOMIC_FETCH_OP_CSWAP, compare, result, op_size,
+                                        remote_addr, rkey, worker);
 }
 
-static inline
-ucs_status_ptr_t opal_common_ucx_atomic_cswap_nb(ucp_ep_h ep, uint64_t compare,
-                                                 uint64_t value, void *result, size_t op_size,
-                                                 uint64_t remote_addr, ucp_rkey_h rkey,
-                                                 ucp_send_callback_t req_handler,
-                                                 ucp_worker_h worker)
+static inline ucs_status_ptr_t
+opal_common_ucx_atomic_cswap_nb(ucp_ep_h ep, uint64_t compare, uint64_t value, void *result,
+                                size_t op_size, uint64_t remote_addr, ucp_rkey_h rkey,
+                                ucp_send_callback_t req_handler, ucp_worker_h worker)
 {
     opal_common_ucx_store_uint64(value, result, op_size);
-    return opal_common_ucx_atomic_fetch_nb(ep, UCP_ATOMIC_FETCH_OP_CSWAP, compare, result,
-                                           op_size, remote_addr, rkey, req_handler, worker);
+    return opal_common_ucx_atomic_fetch_nb(ep, UCP_ATOMIC_FETCH_OP_CSWAP, compare, result, op_size,
+                                           remote_addr, rkey, req_handler, worker);
 }
 
 END_C_DECLS
diff --git a/opal/mca/common/ucx/common_ucx_wpool.c b/opal/mca/common/ucx/common_ucx_wpool.c
index b93891cdd05..ae290201710 100644
--- a/opal/mca/common/ucx/common_ucx_wpool.c
+++ b/opal/mca/common/ucx/common_ucx_wpool.c
@@ -3,12 +3,13 @@
 #include "common_ucx.h"
 #include "common_ucx_wpool.h"
 #include "common_ucx_wpool_int.h"
-#include "opal/mca/base/mca_base_var.h"
 #include "opal/mca/base/mca_base_framework.h"
+#include "opal/mca/base/mca_base_var.h"
 #include "opal/mca/pmix/pmix-internal.h"
 #include "opal/memoryhooks/memory.h"
 #include "opal/util/proc.h"
-
+#include "opal/util/sys_limits.h"
+#include "opal/util/sys_limits.h"
 #include <ucm/api/ucm.h>
 
 /*******************************************************************************
@@ -29,11 +30,13 @@ OBJ_CLASS_INSTANCE(_mem_record_t, opal_list_item_t, NULL, NULL);
 __thread FILE *tls_pf = NULL;
 __thread int initialized = 0;
 #endif
-   
-static _ctx_record_t *
-_tlocal_add_ctx_rec(opal_common_ucx_ctx_t *ctx);
-static inline _ctx_record_t *
-_tlocal_get_ctx_rec(opal_tsd_tracked_key_t tls_key);
+
+bool opal_common_ucx_thread_enabled = false;
+opal_atomic_int64_t opal_common_ucx_ep_counts = 0;
+opal_atomic_int64_t opal_common_ucx_unpacked_rkey_counts = 0;
+
+static _ctx_record_t *_tlocal_add_ctx_rec(opal_common_ucx_ctx_t *ctx);
+static inline _ctx_record_t *_tlocal_get_ctx_rec(opal_tsd_tracked_key_t tls_key);
 static void _tlocal_ctx_rec_cleanup(_ctx_record_t *ctx_rec);
 static void _tlocal_mem_rec_cleanup(_mem_record_t *mem_rec);
 static void _ctx_rec_destructor(void *arg);
@@ -42,21 +45,25 @@ static void _mem_rec_destructor(void *arg);
 /* -----------------------------------------------------------------------------
  * Worker information (winfo) management functionality
  *----------------------------------------------------------------------------*/
-static opal_common_ucx_winfo_t *
-_winfo_create(opal_common_ucx_wpool_t *wpool)
+static opal_common_ucx_winfo_t *_winfo_create(opal_common_ucx_wpool_t *wpool)
 {
     ucp_worker_params_t worker_params;
     ucp_worker_h worker;
     ucs_status_t status;
     opal_common_ucx_winfo_t *winfo = NULL;
 
-    memset(&worker_params, 0, sizeof(worker_params));
-    worker_params.field_mask = UCP_WORKER_PARAM_FIELD_THREAD_MODE;
-    worker_params.thread_mode = UCS_THREAD_MODE_SINGLE;
-    status = ucp_worker_create(wpool->ucp_ctx, &worker_params, &worker);
-    if (UCS_OK != status) {
-        MCA_COMMON_UCX_ERROR("ucp_worker_create failed: %d", status);
-        goto exit;
+    if (opal_common_ucx_thread_enabled || wpool->dflt_winfo == NULL) {
+        memset(&worker_params, 0, sizeof(worker_params));
+        worker_params.field_mask = UCP_WORKER_PARAM_FIELD_THREAD_MODE;
+        worker_params.thread_mode = UCS_THREAD_MODE_SINGLE;
+        status = ucp_worker_create(wpool->ucp_ctx, &worker_params, &worker);
+        if (UCS_OK != status) {
+            MCA_COMMON_UCX_ERROR("ucp_worker_create failed: %d", status);
+            goto exit;
+        }
+    } else {
+        /* Single threaded application can reuse the default worker */
+        worker = wpool->dflt_winfo->worker;
     }
 
     winfo = OBJ_NEW(opal_common_ucx_winfo_t);
@@ -72,6 +79,7 @@ _winfo_create(opal_common_ucx_wpool_t *wpool)
     winfo->inflight_ops = NULL;
     winfo->global_inflight_ops = 0;
     winfo->inflight_req = UCS_OK;
+    winfo->is_dflt_winfo = false;
 
     return winfo;
 
@@ -81,24 +89,25 @@ _winfo_create(opal_common_ucx_wpool_t *wpool)
     return winfo;
 }
 
-static void
-_winfo_destructor(opal_common_ucx_winfo_t *winfo)
+static void _winfo_destructor(opal_common_ucx_winfo_t *winfo)
 {
     if (winfo->inflight_req != UCS_OK) {
-        opal_common_ucx_wait_request_mt(winfo->inflight_req,
-                                        "opal_common_ucx_flush");
+        opal_common_ucx_wait_request_mt(winfo->inflight_req, "opal_common_ucx_flush");
         winfo->inflight_req = UCS_OK;
     }
 
     assert(winfo->global_inflight_ops == 0);
 
-    if(winfo->comm_size != 0) {
+    if (winfo->comm_size != 0) {
         size_t i;
-        for (i = 0; i < winfo->comm_size; i++) {
-            if (NULL != winfo->endpoints[i]){
-                ucp_ep_destroy(winfo->endpoints[i]);
+        if (opal_common_ucx_thread_enabled) {
+            for (i = 0; i < winfo->comm_size; i++) {
+                if (NULL != winfo->endpoints[i]) {
+                    ucp_ep_destroy(winfo->endpoints[i]);
+                    OPAL_COMMON_UCX_DEBUG_ATOMIC_ADD(opal_common_ucx_ep_counts, -1);
+                }
+                assert(winfo->inflight_ops[i] == 0);
             }
-            assert(winfo->inflight_ops[i] == 0);
         }
         free(winfo->endpoints);
         free(winfo->inflight_ops);
@@ -107,15 +116,17 @@ _winfo_destructor(opal_common_ucx_winfo_t *winfo)
     winfo->comm_size = 0;
 
     OBJ_DESTRUCT(&winfo->mutex);
-    ucp_worker_destroy(winfo->worker);
+    if (opal_common_ucx_thread_enabled || winfo->is_dflt_winfo) {
+        ucp_worker_destroy(winfo->worker);
+    }
+
 }
 
 /* -----------------------------------------------------------------------------
  * Worker Pool management functionality
  *----------------------------------------------------------------------------*/
 
-OPAL_DECLSPEC opal_common_ucx_wpool_t *
-opal_common_ucx_wpool_allocate(void)
+OPAL_DECLSPEC opal_common_ucx_wpool_t *opal_common_ucx_wpool_allocate(void)
 {
     opal_common_ucx_wpool_t *ptr = calloc(1, sizeof(opal_common_ucx_wpool_t));
     ptr->refcnt = 0;
@@ -123,22 +134,17 @@ opal_common_ucx_wpool_allocate(void)
     return ptr;
 }
 
-OPAL_DECLSPEC void
-opal_common_ucx_wpool_free(opal_common_ucx_wpool_t *wpool)
+OPAL_DECLSPEC void opal_common_ucx_wpool_free(opal_common_ucx_wpool_t *wpool)
 {
     assert(wpool->refcnt == 0);
     free(wpool);
 }
 
 static int _wpool_list_put(opal_common_ucx_wpool_t *wpool, opal_list_t *list,
-                opal_common_ucx_winfo_t *winfo);
+                           opal_common_ucx_winfo_t *winfo);
 
-OPAL_DECLSPEC int
-opal_common_ucx_wpool_init(opal_common_ucx_wpool_t *wpool,
-                           int proc_world_size, bool enable_mt)
+OPAL_DECLSPEC int opal_common_ucx_wpool_init(opal_common_ucx_wpool_t *wpool)
 {
-    ucp_config_t *config = NULL;
-    ucp_params_t context_params;
     opal_common_ucx_winfo_t *winfo;
     ucs_status_t status;
     int rc = OPAL_SUCCESS;
@@ -151,54 +157,24 @@ opal_common_ucx_wpool_init(opal_common_ucx_wpool_t *wpool,
 
     OBJ_CONSTRUCT(&wpool->mutex, opal_mutex_t);
 
-    status = ucp_config_read("MPI", NULL, &config);
-    if (UCS_OK != status) {
-        MCA_COMMON_UCX_VERBOSE(1, "ucp_config_read failed: %d", status);
-        return OPAL_ERROR;
-    }
-
-    /* initialize UCP context */
-    memset(&context_params, 0, sizeof(context_params));
-    context_params.field_mask = UCP_PARAM_FIELD_FEATURES |
-                                UCP_PARAM_FIELD_MT_WORKERS_SHARED |
-                                UCP_PARAM_FIELD_ESTIMATED_NUM_EPS |
-                                UCP_PARAM_FIELD_REQUEST_INIT |
-                                UCP_PARAM_FIELD_REQUEST_SIZE;
-    context_params.features = UCP_FEATURE_RMA | UCP_FEATURE_AMO32 |
-                              UCP_FEATURE_AMO64;
-    context_params.mt_workers_shared = (enable_mt ? 1 : 0);
-    context_params.estimated_num_eps = proc_world_size;
-    context_params.request_init = opal_common_ucx_req_init;
-    context_params.request_size = sizeof(opal_common_ucx_request_t);
-
-#if HAVE_DECL_UCP_PARAM_FIELD_ESTIMATED_NUM_PPN
-    context_params.estimated_num_ppn = opal_process_info.num_local_peers + 1;
-    context_params.field_mask       |= UCP_PARAM_FIELD_ESTIMATED_NUM_PPN;
-#endif
-
-    status = ucp_init(&context_params, config, &wpool->ucp_ctx);
-    ucp_config_release(config);
-    if (UCS_OK != status) {
-        MCA_COMMON_UCX_VERBOSE(1, "ucp_init failed: %d", status);
-        rc = OPAL_ERROR;
-        goto err_ucp_init;
-    }
-
     /* create recv worker and add to idle pool */
     OBJ_CONSTRUCT(&wpool->idle_workers, opal_list_t);
     OBJ_CONSTRUCT(&wpool->active_workers, opal_list_t);
 
+    wpool->dflt_winfo = NULL;
+
     winfo = _winfo_create(wpool);
     if (NULL == winfo) {
         MCA_COMMON_UCX_ERROR("Failed to create receive worker");
         rc = OPAL_ERROR;
         goto err_worker_create;
     }
+    winfo->is_dflt_winfo = true;
     wpool->dflt_winfo = winfo;
     OBJ_RETAIN(wpool->dflt_winfo);
 
-    status = ucp_worker_get_address(wpool->dflt_winfo->worker,
-                                    &wpool->recv_waddr, &wpool->recv_waddr_len);
+    status = ucp_worker_get_address(wpool->dflt_winfo->worker, &wpool->recv_waddr,
+                                    &wpool->recv_waddr_len);
     if (status != UCS_OK) {
         MCA_COMMON_UCX_VERBOSE(1, "ucp_worker_get_address failed: %d", status);
         rc = OPAL_ERROR;
@@ -216,13 +192,11 @@ opal_common_ucx_wpool_init(opal_common_ucx_wpool_t *wpool,
     free(wpool->recv_waddr);
 err_get_addr:
     OBJ_RELEASE(winfo);
-    OBJ_RELEASE(wpool->dflt_winfo);
     wpool->dflt_winfo = NULL;
- err_worker_create:
+err_worker_create:
     OBJ_DESTRUCT(&wpool->idle_workers);
     OBJ_DESTRUCT(&wpool->active_workers);
     ucp_cleanup(wpool->ucp_ctx);
- err_ucp_init:
     return rc;
 }
 
@@ -241,8 +215,7 @@ void opal_common_ucx_wpool_finalize(opal_common_ucx_wpool_t *wpool)
     /* Go over the list, free idle list items */
     if (!opal_list_is_empty(&wpool->idle_workers)) {
         opal_common_ucx_winfo_t *winfo, *next;
-        OPAL_LIST_FOREACH_SAFE(winfo, next, &wpool->idle_workers,
-                               opal_common_ucx_winfo_t) {
+        OPAL_LIST_FOREACH_SAFE (winfo, next, &wpool->idle_workers, opal_common_ucx_winfo_t) {
             opal_list_remove_item(&wpool->idle_workers, &winfo->super);
             OBJ_RELEASE(winfo);
         }
@@ -253,8 +226,7 @@ void opal_common_ucx_wpool_finalize(opal_common_ucx_wpool_t *wpool)
      * because opal_common_ucx_wpool_finalize is being called. */
     if (!opal_list_is_empty(&wpool->active_workers)) {
         opal_common_ucx_winfo_t *winfo, *next;
-        OPAL_LIST_FOREACH_SAFE(winfo, next, &wpool->active_workers,
-                               opal_common_ucx_winfo_t) {
+        OPAL_LIST_FOREACH_SAFE (winfo, next, &wpool->active_workers, opal_common_ucx_winfo_t) {
             opal_list_remove_item(&wpool->active_workers, &winfo->super);
             OBJ_RELEASE(winfo);
         }
@@ -265,12 +237,15 @@ void opal_common_ucx_wpool_finalize(opal_common_ucx_wpool_t *wpool)
     wpool->dflt_winfo = NULL;
 
     OBJ_DESTRUCT(&wpool->mutex);
-    ucp_cleanup(wpool->ucp_ctx);
+    if (NULL != wpool->ucp_ctx) {
+        ucp_cleanup(wpool->ucp_ctx);
+        wpool->ucp_ctx = NULL;
+    }
+
     return;
 }
 
-OPAL_DECLSPEC int
-opal_common_ucx_wpool_progress(opal_common_ucx_wpool_t *wpool)
+OPAL_DECLSPEC int opal_common_ucx_wpool_progress(opal_common_ucx_wpool_t *wpool)
 {
     opal_common_ucx_winfo_t *winfo = NULL, *next = NULL;
     int completed = 0, progressed = 0;
@@ -283,8 +258,7 @@ opal_common_ucx_wpool_progress(opal_common_ucx_wpool_t *wpool)
     }
 
     bool progress_dflt_worker = true;
-    OPAL_LIST_FOREACH_SAFE(winfo, next, &wpool->active_workers,
-                           opal_common_ucx_winfo_t) {
+    OPAL_LIST_FOREACH_SAFE (winfo, next, &wpool->active_workers, opal_common_ucx_winfo_t) {
         if (0 != opal_mutex_trylock(&winfo->mutex)) {
             continue;
         }
@@ -308,29 +282,26 @@ opal_common_ucx_wpool_progress(opal_common_ucx_wpool_t *wpool)
     return completed;
 }
 
-static int
-_wpool_list_put(opal_common_ucx_wpool_t *wpool, opal_list_t *list,
-                opal_common_ucx_winfo_t *winfo)
+static int _wpool_list_put(opal_common_ucx_wpool_t *wpool, opal_list_t *list,
+                           opal_common_ucx_winfo_t *winfo)
 {
     opal_list_append(list, &winfo->super);
     return OPAL_SUCCESS;
 }
 
-static opal_common_ucx_winfo_t*
-_wpool_list_get(opal_common_ucx_wpool_t *wpool, opal_list_t *list)
+static opal_common_ucx_winfo_t *_wpool_list_get(opal_common_ucx_wpool_t *wpool, opal_list_t *list)
 {
     opal_common_ucx_winfo_t *winfo = NULL;
 
     if (!opal_list_is_empty(list)) {
-        winfo = (opal_common_ucx_winfo_t *)opal_list_get_first(list);
+        winfo = (opal_common_ucx_winfo_t *) opal_list_get_first(list);
         opal_list_remove_item(list, &winfo->super);
     }
 
     return winfo;
 }
 
-static opal_common_ucx_winfo_t *
-_wpool_get_winfo(opal_common_ucx_wpool_t *wpool, size_t comm_size)
+static opal_common_ucx_winfo_t *_wpool_get_winfo(opal_common_ucx_wpool_t *wpool, size_t comm_size)
 {
     opal_common_ucx_winfo_t *winfo;
     opal_mutex_lock(&wpool->mutex);
@@ -356,14 +327,30 @@ _wpool_get_winfo(opal_common_ucx_wpool_t *wpool, size_t comm_size)
     return winfo;
 }
 
-static void
-_wpool_put_winfo(opal_common_ucx_wpool_t *wpool, opal_common_ucx_winfo_t *winfo)
+/* Remove the winfo from active workers and add it to idle workers */
+static void _wpool_put_winfo(opal_common_ucx_wpool_t *wpool, opal_common_ucx_winfo_t *winfo)
 {
     opal_mutex_lock(&wpool->mutex);
+    if (winfo->comm_size != 0) {
+        size_t i;
+        if (opal_common_ucx_thread_enabled) {
+            for (i = 0; i < winfo->comm_size; i++) {
+                if (NULL != winfo->endpoints[i]) {
+                    ucp_ep_destroy(winfo->endpoints[i]);
+                    OPAL_COMMON_UCX_DEBUG_ATOMIC_ADD(opal_common_ucx_ep_counts, -1);
+                }
+                assert(winfo->inflight_ops[i] == 0);
+            }
+        }
+        free(winfo->endpoints);
+        free(winfo->inflight_ops);
+    }
+    winfo->endpoints = NULL;
+    winfo->comm_size = 0;
     opal_list_remove_item(&wpool->active_workers, &winfo->super);
     opal_list_prepend(&wpool->idle_workers, &winfo->super);
     opal_mutex_unlock(&wpool->mutex);
-    
+
     return;
 }
 
@@ -371,16 +358,15 @@ _wpool_put_winfo(opal_common_ucx_wpool_t *wpool, opal_common_ucx_winfo_t *winfo)
  * Worker Pool Communication context management functionality
  *----------------------------------------------------------------------------*/
 
-OPAL_DECLSPEC int
-opal_common_ucx_wpctx_create(opal_common_ucx_wpool_t *wpool, int comm_size,
-                             opal_common_ucx_exchange_func_t exchange_func,
-                             void *exchange_metadata,
-                             opal_common_ucx_ctx_t **ctx_ptr)
+OPAL_DECLSPEC int opal_common_ucx_wpctx_create(opal_common_ucx_wpool_t *wpool, int comm_size,
+                                               opal_common_ucx_exchange_func_t exchange_func,
+                                               void *exchange_metadata,
+                                               opal_common_ucx_ctx_t **ctx_ptr)
 {
     opal_common_ucx_ctx_t *ctx = calloc(1, sizeof(*ctx));
     int ret = OPAL_SUCCESS;
 
-    OBJ_CONSTRUCT(&ctx->mutex, opal_mutex_t);
+    OBJ_CONSTRUCT(&ctx->mutex, opal_recursive_mutex_t);
     OBJ_CONSTRUCT(&ctx->ctx_records, opal_list_t);
 
     ctx->wpool = wpool;
@@ -388,8 +374,8 @@ opal_common_ucx_wpctx_create(opal_common_ucx_wpool_t *wpool, int comm_size,
 
     ctx->recv_worker_addrs = NULL;
     ctx->recv_worker_displs = NULL;
-    ret = exchange_func(wpool->recv_waddr, wpool->recv_waddr_len,
-                        &ctx->recv_worker_addrs,
+    ctx->num_incomplete_req_ops = 0;
+    ret = exchange_func(wpool->recv_waddr, wpool->recv_waddr_len, &ctx->recv_worker_addrs,
                         &ctx->recv_worker_displs, exchange_metadata);
     if (ret != OPAL_SUCCESS) {
         goto error;
@@ -408,8 +394,7 @@ opal_common_ucx_wpctx_create(opal_common_ucx_wpool_t *wpool, int comm_size,
     return ret;
 }
 
-OPAL_DECLSPEC void
-opal_common_ucx_wpctx_release(opal_common_ucx_ctx_t *ctx)
+OPAL_DECLSPEC void opal_common_ucx_wpctx_release(opal_common_ucx_ctx_t *ctx)
 {
     _ctx_record_t *ctx_rec = NULL, *next;
 
@@ -420,8 +405,8 @@ opal_common_ucx_wpctx_release(opal_common_ucx_ctx_t *ctx)
     OBJ_DESTRUCT(&ctx->tls_key);
 
     /* loop through list of records */
-    OPAL_LIST_FOREACH_SAFE(ctx_rec, next, &ctx->ctx_records, _ctx_record_t) {
-       _tlocal_ctx_rec_cleanup(ctx_rec);
+    OPAL_LIST_FOREACH_SAFE (ctx_rec, next, &ctx->ctx_records, _ctx_record_t) {
+        _tlocal_ctx_rec_cleanup(ctx_rec);
     }
 
     free(ctx->recv_worker_addrs);
@@ -438,14 +423,12 @@ opal_common_ucx_wpctx_release(opal_common_ucx_ctx_t *ctx)
  *----------------------------------------------------------------------------*/
 
 OPAL_DECLSPEC
-int opal_common_ucx_wpmem_create(opal_common_ucx_ctx_t *ctx,
-                               void **mem_base, size_t mem_size,
-                               opal_common_ucx_mem_type_t mem_type,
-                               opal_common_ucx_exchange_func_t exchange_func,
-                               void *exchange_metadata,
-                                 char **my_mem_addr,
-                                 int *my_mem_addr_size,
-                               opal_common_ucx_wpmem_t **mem_ptr)
+int opal_common_ucx_wpmem_create(opal_common_ucx_ctx_t *ctx, void **mem_base, size_t mem_size,
+                                 opal_common_ucx_mem_type_t mem_type,
+                                 opal_common_ucx_exchange_func_t exchange_func,
+                                 opal_common_ucx_exchange_mode_t exchange_mode,
+                                 void *exchange_metadata, char **my_mem_addr, int *my_mem_addr_size,
+                                 opal_common_ucx_wpmem_t **mem_ptr)
 {
     opal_common_ucx_wpmem_t *mem = calloc(1, sizeof(*mem));
     void *rkey_addr = NULL;
@@ -456,31 +439,30 @@ int opal_common_ucx_wpmem_create(opal_common_ucx_ctx_t *ctx,
     mem->ctx = ctx;
     mem->mem_addrs = NULL;
     mem->mem_displs = NULL;
-    
+    mem->skip_periodic_flush = false;
+
     OBJ_CONSTRUCT(&mem->mutex, opal_mutex_t);
-    OBJ_CONSTRUCT(&mem->mem_records, opal_list_t);
 
-    ret = _comm_ucx_wpmem_map(ctx->wpool, mem_base, mem_size, &mem->memh,
-                            mem_type);
+    ret = _comm_ucx_wpmem_map(ctx->wpool, mem_base, mem_size, &mem->memh, mem_type);
     if (ret != OPAL_SUCCESS) {
         MCA_COMMON_UCX_VERBOSE(1, "_comm_ucx_mem_map failed: %d", ret);
         goto error_mem_map;
     }
 
-    status = ucp_rkey_pack(ctx->wpool->ucp_ctx, mem->memh,
-                           &rkey_addr, &rkey_addr_len);
+    status = ucp_rkey_pack(ctx->wpool->ucp_ctx, mem->memh, &rkey_addr, &rkey_addr_len);
     if (status != UCS_OK) {
         MCA_COMMON_UCX_VERBOSE(1, "ucp_rkey_pack failed: %d", status);
         ret = OPAL_ERROR;
         goto error_rkey_pack;
     }
 
-    ret = exchange_func(rkey_addr, rkey_addr_len,
-                        &mem->mem_addrs, &mem->mem_displs, exchange_metadata);
-    if (ret != OPAL_SUCCESS) {
-        goto error_rkey_pack;
+    if (exchange_mode == OPAL_COMMON_UCX_WPMEM_ADDR_EXCHANGE_FULL) {
+        ret = exchange_func(rkey_addr, rkey_addr_len, &mem->mem_addrs, &mem->mem_displs,
+                            exchange_metadata);
+        if (ret != OPAL_SUCCESS) {
+            goto error_rkey_pack;
+        }
     }
-
     OBJ_CONSTRUCT(&mem->tls_key, opal_tsd_tracked_key_t);
     opal_tsd_tracked_key_set_destructor(&mem->tls_key, _mem_rec_destructor);
 
@@ -490,17 +472,16 @@ int opal_common_ucx_wpmem_create(opal_common_ucx_ctx_t *ctx,
 
     return ret;
 
- error_rkey_pack:
+error_rkey_pack:
     ucp_mem_unmap(ctx->wpool->ucp_ctx, mem->memh);
- error_mem_map:
+error_mem_map:
     free(mem);
     (*mem_ptr) = NULL;
     return ret;
 }
 
-static int _comm_ucx_wpmem_map(opal_common_ucx_wpool_t *wpool,
-                             void **base, size_t size, ucp_mem_h *memh_ptr,
-                             opal_common_ucx_mem_type_t mem_type)
+static int _comm_ucx_wpmem_map(opal_common_ucx_wpool_t *wpool, void **base, size_t size,
+                               ucp_mem_h *memh_ptr, opal_common_ucx_mem_type_t mem_type)
 {
     ucp_mem_map_params_t mem_params;
     ucp_mem_attr_t mem_attrs;
@@ -508,9 +489,8 @@ static int _comm_ucx_wpmem_map(opal_common_ucx_wpool_t *wpool,
     int ret = OPAL_SUCCESS;
 
     memset(&mem_params, 0, sizeof(ucp_mem_map_params_t));
-    mem_params.field_mask = UCP_MEM_MAP_PARAM_FIELD_ADDRESS |
-                            UCP_MEM_MAP_PARAM_FIELD_LENGTH |
-                            UCP_MEM_MAP_PARAM_FIELD_FLAGS;
+    mem_params.field_mask = UCP_MEM_MAP_PARAM_FIELD_ADDRESS | UCP_MEM_MAP_PARAM_FIELD_LENGTH
+                            | UCP_MEM_MAP_PARAM_FIELD_FLAGS;
     mem_params.length = size;
     if (mem_type == OPAL_COMMON_UCX_MEM_ALLOCATE_MAP) {
         mem_params.address = NULL;
@@ -536,33 +516,27 @@ static int _comm_ucx_wpmem_map(opal_common_ucx_wpool_t *wpool,
 
     assert(mem_attrs.length >= size);
     if (mem_type != OPAL_COMMON_UCX_MEM_ALLOCATE_MAP) {
-        assert(mem_attrs.address == (*base));
+        /* Returned mapped address is aligned to ucs rcache->params.alignment.
+         * Alignment is less than page size */
+        assert(((mem_attrs.address <= (*base)) && ((*base) - opal_getpagesize()
+                < mem_attrs.address)) || (size == 0 && mem_attrs.address == NULL));
     } else {
         (*base) = mem_attrs.address;
     }
 
     return ret;
- error:
+error:
     ucp_mem_unmap(wpool->ucp_ctx, (*memh_ptr));
     return ret;
 }
 
 void opal_common_ucx_wpmem_free(opal_common_ucx_wpmem_t *mem)
 {
-    _mem_record_t *mem_rec = NULL, *next;
-
     if (NULL == mem) {
         return;
     }
-    
-    OBJ_DESTRUCT(&mem->tls_key);
 
-    /* Loop through list of records */
-    OPAL_LIST_FOREACH_SAFE(mem_rec, next, &mem->mem_records, _mem_record_t) {
-        _tlocal_mem_rec_cleanup(mem_rec);
-    }
-  
-    OBJ_DESTRUCT(&mem->mem_records);
+    OBJ_DESTRUCT(&mem->tls_key);
 
     free(mem->mem_addrs);
     free(mem->mem_displs);
@@ -571,10 +545,10 @@ void opal_common_ucx_wpmem_free(opal_common_ucx_wpmem_t *mem)
     free(mem);
 }
 
-static inline _ctx_record_t *
-_tlocal_get_ctx_rec(opal_tsd_tracked_key_t tls_key){
+static inline _ctx_record_t *_tlocal_get_ctx_rec(opal_tsd_tracked_key_t tls_key)
+{
     _ctx_record_t *ctx_rec = NULL;
-    int rc = opal_tsd_tracked_key_get(&tls_key, (void**)&ctx_rec);
+    int rc = opal_tsd_tracked_key_get(&tls_key, (void **) &ctx_rec);
 
     if (OPAL_SUCCESS != rc) {
         return NULL;
@@ -583,14 +557,14 @@ _tlocal_get_ctx_rec(opal_tsd_tracked_key_t tls_key){
     return ctx_rec;
 }
 
-static void _ctx_rec_destructor(void *arg) {
-    _tlocal_ctx_rec_cleanup ((_ctx_record_t *) arg);
+static void _ctx_rec_destructor(void *arg)
+{
+    _tlocal_ctx_rec_cleanup((_ctx_record_t *) arg);
     return;
 }
 
 /* Thread local storage destructor, also called from wpool_ctx release */
-static void
-_tlocal_ctx_rec_cleanup(_ctx_record_t *ctx_rec)
+static void _tlocal_ctx_rec_cleanup(_ctx_record_t *ctx_rec)
 {
     if (NULL == ctx_rec) {
         return;
@@ -599,6 +573,16 @@ _tlocal_ctx_rec_cleanup(_ctx_record_t *ctx_rec)
     opal_common_ucx_winfo_t *winfo = ctx_rec->winfo;
     opal_common_ucx_wpool_t *wpool = ctx_rec->gctx->wpool;
 
+    opal_mutex_lock(&winfo->mutex);
+    int rc = opal_common_ucx_winfo_flush(winfo, 0, OPAL_COMMON_UCX_FLUSH_B, OPAL_COMMON_UCX_SCOPE_WORKER, NULL);
+    winfo->global_inflight_ops = 0;
+    memset(winfo->inflight_ops, 0, winfo->comm_size * sizeof(short));
+    opal_mutex_unlock(&winfo->mutex);
+    if (rc != OPAL_SUCCESS) {
+        MCA_COMMON_UCX_ERROR("opal_common_ucx_flush failed: %d", rc);
+        return;
+    }
+
     /* Remove worker from active and return to idle list. */
     _wpool_put_winfo(wpool, winfo);
 
@@ -612,11 +596,10 @@ _tlocal_ctx_rec_cleanup(_ctx_record_t *ctx_rec)
     return;
 }
 
-static _ctx_record_t *
-_tlocal_add_ctx_rec(opal_common_ucx_ctx_t *ctx)
+static _ctx_record_t *_tlocal_add_ctx_rec(opal_common_ucx_ctx_t *ctx)
 {
     int rc;
-    
+
     _ctx_record_t *ctx_rec = OBJ_NEW(_ctx_record_t);
     if (!ctx_rec) {
         MCA_COMMON_UCX_ERROR("Failed to allocate new ctx_rec");
@@ -639,7 +622,7 @@ _tlocal_add_ctx_rec(opal_common_ucx_ctx_t *ctx)
     rc = opal_tsd_tracked_key_set(&ctx->tls_key, ctx_rec);
     if (OPAL_SUCCESS != rc) {
         MCA_COMMON_UCX_ERROR("Failed to set ctx_rec tls key");
-        goto error3; 
+        goto error3;
     }
 
     /* All good - return the record */
@@ -667,28 +650,29 @@ static int _tlocal_ctx_connect(_ctx_record_t *ctx_rec, int target)
     memset(&ep_params, 0, sizeof(ucp_ep_params_t));
     ep_params.field_mask = UCP_EP_PARAM_FIELD_REMOTE_ADDRESS;
 
+    assert(winfo->endpoints[target] == NULL);
     opal_mutex_lock(&winfo->mutex);
     displ = gctx->recv_worker_displs[target];
-    ep_params.address = (ucp_address_t *)&(gctx->recv_worker_addrs[displ]);
+    ep_params.address = (ucp_address_t *) &(gctx->recv_worker_addrs[displ]);
     status = ucp_ep_create(winfo->worker, &ep_params, &winfo->endpoints[target]);
     if (status != UCS_OK) {
-        opal_mutex_unlock(&winfo->mutex);
         MCA_COMMON_UCX_VERBOSE(1, "ucp_ep_create failed: %d", status);
         opal_mutex_unlock(&winfo->mutex);
         return OPAL_ERROR;
     }
+    OPAL_COMMON_UCX_DEBUG_ATOMIC_ADD(opal_common_ucx_ep_counts, 1);
     opal_mutex_unlock(&winfo->mutex);
+    assert(winfo->endpoints[target] != NULL);
     return OPAL_SUCCESS;
 }
 
-static void
-_mem_rec_destructor(void * arg) {
-    _tlocal_mem_rec_cleanup ((_mem_record_t *) arg);
+static void _mem_rec_destructor(void *arg)
+{
+    _tlocal_mem_rec_cleanup((_mem_record_t *) arg);
     return;
 }
 
-static void
-_tlocal_mem_rec_cleanup(_mem_record_t *mem_rec)
+static void _tlocal_mem_rec_cleanup(_mem_record_t *mem_rec)
 {
     size_t i;
     if (NULL == mem_rec) {
@@ -696,19 +680,15 @@ _tlocal_mem_rec_cleanup(_mem_record_t *mem_rec)
     }
 
     opal_mutex_lock(&mem_rec->winfo->mutex);
-    for(i = 0; i < mem_rec->gmem->ctx->comm_size; i++) {
+    for (i = 0; i < mem_rec->gmem->ctx->comm_size; i++) {
         if (mem_rec->rkeys[i]) {
             ucp_rkey_destroy(mem_rec->rkeys[i]);
+            OPAL_COMMON_UCX_DEBUG_ATOMIC_ADD(opal_common_ucx_unpacked_rkey_counts, -1);
         }
     }
     opal_mutex_unlock(&mem_rec->winfo->mutex);
     free(mem_rec->rkeys);
 
-    /* Remove item from the list */
-    opal_mutex_lock(&mem_rec->gmem->mutex);
-    opal_list_remove_item(&mem_rec->gmem->mem_records, &mem_rec->super);
-    opal_mutex_unlock(&mem_rec->gmem->mutex);
-
     OBJ_RELEASE(mem_rec);
 
     return;
@@ -726,29 +706,24 @@ static _mem_record_t *_tlocal_add_mem_rec(opal_common_ucx_wpmem_t *mem, _ctx_rec
     mem_rec->ctx_rec = ctx_rec;
     mem_rec->winfo = ctx_rec->winfo;
     mem_rec->rkeys = calloc(mem->ctx->comm_size, sizeof(*mem_rec->rkeys));
-    
+
     rc = opal_tsd_tracked_key_set(&mem->tls_key, mem_rec);
     if (OPAL_SUCCESS != rc) {
         return NULL;
     }
 
-    opal_mutex_lock(&mem->mutex);
-    opal_list_append(&mem->mem_records, &mem_rec->super);
-    opal_mutex_unlock(&mem->mutex);
-
     return mem_rec;
 }
 
-static int
-_tlocal_mem_create_rkey(_mem_record_t *mem_rec, ucp_ep_h ep, int target)
+static int _tlocal_mem_create_rkey(_mem_record_t *mem_rec, ucp_ep_h ep, int target)
 {
     opal_common_ucx_wpmem_t *gmem = mem_rec->gmem;
     int displ = gmem->mem_displs[target];
     ucs_status_t status;
 
     opal_mutex_lock(&mem_rec->winfo->mutex);
-    status = ucp_ep_rkey_unpack(ep, &gmem->mem_addrs[displ],
-                                &mem_rec->rkeys[target]);
+    status = ucp_ep_rkey_unpack(ep, &gmem->mem_addrs[displ], &mem_rec->rkeys[target]);
+    OPAL_COMMON_UCX_DEBUG_ATOMIC_ADD(opal_common_ucx_unpacked_rkey_counts, 1);
     opal_mutex_unlock(&mem_rec->winfo->mutex);
     if (status != UCS_OK) {
         MCA_COMMON_UCX_VERBOSE(1, "ucp_ep_rkey_unpack failed: %d", status);
@@ -759,8 +734,7 @@ _tlocal_mem_create_rkey(_mem_record_t *mem_rec, ucp_ep_h ep, int target)
 }
 
 /* Get the TLS in case of slow path (not everything has been yet initialized */
-OPAL_DECLSPEC int
-opal_common_ucx_tlocal_fetch_spath(opal_common_ucx_wpmem_t *mem, int target)
+OPAL_DECLSPEC int opal_common_ucx_tlocal_fetch_spath(opal_common_ucx_wpmem_t *mem, int target, ucp_ep_h *dflt_ep)
 {
     _ctx_record_t *ctx_rec = NULL;
     _mem_record_t *mem_rec = NULL;
@@ -779,18 +753,37 @@ opal_common_ucx_tlocal_fetch_spath(opal_common_ucx_wpmem_t *mem, int target)
 
     /* Obtain the endpoint */
     if (OPAL_UNLIKELY(NULL == winfo->endpoints[target])) {
-        rc = _tlocal_ctx_connect(ctx_rec, target);
-        if (rc != OPAL_SUCCESS) {
-            return rc;
+        if (opal_common_ucx_thread_enabled || (dflt_ep == NULL) ||
+                (*dflt_ep == NULL)) {
+            rc = _tlocal_ctx_connect(ctx_rec, target);
+            if (rc != OPAL_SUCCESS) {
+                return rc;
+            }
+            if (!opal_common_ucx_thread_enabled && (dflt_ep != NULL) &&
+                    (*dflt_ep == NULL)) {
+                /* set the proc ep */
+                *dflt_ep = winfo->endpoints[target];
+            }
+        } else {
+            /* reuse the previously created ep */
+            winfo->endpoints[target] = *dflt_ep;
         }
     }
     ep = winfo->endpoints[target];
 
-    /* Obtain the memory region info */
-    mem_rec = _tlocal_add_mem_rec(mem, ctx_rec);
+
+    rc = opal_tsd_tracked_key_get(&mem->tls_key, (void **) &mem_rec);
+    if (OPAL_SUCCESS != rc) {
+        return rc;
+    }
+
+    if (NULL == mem_rec) {
+        /* Allocate a memory region info */
+        mem_rec = _tlocal_add_mem_rec(mem, ctx_rec);
+    }
 
     /* Obtain the rkey */
-    if (OPAL_UNLIKELY(NULL == mem_rec->rkeys[target])) {
+    if (NULL == mem_rec->rkeys[target]) {
         /* Create the rkey */
         rc = _tlocal_mem_create_rkey(mem_rec, ep, target);
         if (rc) {
@@ -801,11 +794,10 @@ opal_common_ucx_tlocal_fetch_spath(opal_common_ucx_wpmem_t *mem, int target)
     return OPAL_SUCCESS;
 }
 
-OPAL_DECLSPEC int
-opal_common_ucx_winfo_flush(opal_common_ucx_winfo_t *winfo, int target,
-                            opal_common_ucx_flush_type_t type,
-                            opal_common_ucx_flush_scope_t scope,
-                            ucs_status_ptr_t *req_ptr)
+OPAL_DECLSPEC int opal_common_ucx_winfo_flush(opal_common_ucx_winfo_t *winfo, int target,
+                                              opal_common_ucx_flush_type_t type,
+                                              opal_common_ucx_flush_scope_t scope,
+                                              ucs_status_ptr_t *req_ptr)
 {
     ucs_status_ptr_t req;
     ucs_status_t status = UCS_OK;
@@ -818,10 +810,10 @@ opal_common_ucx_winfo_flush(opal_common_ucx_winfo_t *winfo, int target,
         req = ucp_worker_flush_nb(winfo->worker, 0, opal_common_ucx_empty_complete_cb);
     }
     if (UCS_PTR_IS_PTR(req)) {
-        ((opal_common_ucx_request_t *)req)->winfo = winfo;
+        ((opal_common_ucx_request_t *) req)->winfo = winfo;
     }
 
-    if(OPAL_COMMON_UCX_FLUSH_B) {
+    if (OPAL_COMMON_UCX_FLUSH_B == type) {
         rc = opal_common_ucx_wait_request_mt(req, "ucp_ep_flush_nb");
     } else {
         *req_ptr = req;
@@ -844,31 +836,25 @@ opal_common_ucx_winfo_flush(opal_common_ucx_winfo_t *winfo, int target,
     return rc;
 }
 
-OPAL_DECLSPEC int
-opal_common_ucx_wpmem_flush(opal_common_ucx_wpmem_t *mem,
-                          opal_common_ucx_flush_scope_t scope,
-                          int target)
+static inline int ctx_flush(opal_common_ucx_ctx_t *ctx,
+                                opal_common_ucx_flush_scope_t scope, int target)
 {
     _ctx_record_t *ctx_rec;
-    opal_common_ucx_ctx_t *ctx;
     int rc = OPAL_SUCCESS;
 
-    if (NULL == mem) {
+    if (NULL == ctx) {
         return OPAL_SUCCESS;
     }
 
-    ctx = mem->ctx;
     opal_mutex_lock(&ctx->mutex);
 
-    OPAL_LIST_FOREACH(ctx_rec, &ctx->ctx_records, _ctx_record_t) {
+    OPAL_LIST_FOREACH (ctx_rec, &ctx->ctx_records, _ctx_record_t) {
         opal_common_ucx_winfo_t *winfo = ctx_rec->winfo;
-        if ((scope == OPAL_COMMON_UCX_SCOPE_EP) &&
-                (NULL == winfo->endpoints[target])) {
+        if ((scope == OPAL_COMMON_UCX_SCOPE_EP) && (NULL == winfo->endpoints[target])) {
             continue;
         }
         opal_mutex_lock(&winfo->mutex);
-        rc = opal_common_ucx_winfo_flush(winfo, target, OPAL_COMMON_UCX_FLUSH_B,
-                                         scope, NULL);
+        rc = opal_common_ucx_winfo_flush(winfo, target, OPAL_COMMON_UCX_FLUSH_B, scope, NULL);
         switch (scope) {
         case OPAL_COMMON_UCX_SCOPE_WORKER:
             winfo->global_inflight_ops = 0;
@@ -882,33 +868,134 @@ opal_common_ucx_wpmem_flush(opal_common_ucx_wpmem_t *mem,
         opal_mutex_unlock(&winfo->mutex);
 
         if (rc != OPAL_SUCCESS) {
-            MCA_COMMON_UCX_ERROR("opal_common_ucx_flush failed: %d",
-                                 rc);
+            MCA_COMMON_UCX_ERROR("opal_common_ucx_flush failed: %d", rc);
             rc = OPAL_ERROR;
+            break;
         }
     }
+
     opal_mutex_unlock(&ctx->mutex);
 
     return rc;
 }
 
-OPAL_DECLSPEC int
-opal_common_ucx_wpmem_fence(opal_common_ucx_wpmem_t *mem) {
-    /* TODO */
-    return OPAL_SUCCESS;
+OPAL_DECLSPEC int opal_common_ucx_ctx_flush(opal_common_ucx_ctx_t *ctx,
+                                    opal_common_ucx_flush_scope_t scope, int target)
+{
+    int rc = OPAL_SUCCESS;
+    int spin = 0;
+
+    if (NULL == ctx) {
+        return OPAL_SUCCESS;
+    }
+
+    rc = ctx_flush(ctx, scope, target);
+    if (rc != OPAL_SUCCESS) {
+        return rc;
+    }
+
+    /* progress the nonblocking operations */
+    while (ctx->num_incomplete_req_ops > 0) {
+        spin++;
+        rc = ctx_flush(ctx, OPAL_COMMON_UCX_SCOPE_WORKER, 0);
+        if (rc != OPAL_SUCCESS) {
+            return rc;
+        }
+        if (spin == opal_common_ucx.progress_iterations) {
+            opal_progress();
+            spin = 0;
+        }
+    }
+
+    return rc;
 }
 
-OPAL_DECLSPEC void
-opal_common_ucx_req_init(void *request) {
-    opal_common_ucx_request_t *req = (opal_common_ucx_request_t *)request;
+
+OPAL_DECLSPEC int opal_common_ucx_wpmem_flush_ep_nb(opal_common_ucx_wpmem_t *mem,
+                                                    int target,
+                                                    opal_common_ucx_user_req_handler_t user_req_cb,
+                                                    void *user_req_ptr, ucp_ep_h *dflt_ep)
+{
+#if HAVE_DECL_UCP_EP_FLUSH_NB
+    int rc = OPAL_SUCCESS;
+    ucp_ep_h ep = NULL;
+    ucp_rkey_h rkey = NULL;
+    opal_common_ucx_winfo_t *winfo = NULL;
+
+    if (NULL == mem) {
+        return OPAL_SUCCESS;
+    }
+
+    rc = opal_common_ucx_tlocal_fetch(mem, target, &ep, &rkey, &winfo, dflt_ep);
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
+        MCA_COMMON_UCX_ERROR("tlocal_fetch failed: %d", rc);
+        return rc;
+    }
+
+    opal_mutex_lock(&winfo->mutex);
+    opal_common_ucx_request_t *req;
+    req = ucp_worker_flush_nb(winfo->worker, 0, opal_common_ucx_req_completion);
+    if (UCS_PTR_IS_PTR(req)) {
+        req->ext_req = user_req_ptr;
+        req->ext_cb = user_req_cb;
+        req->winfo = winfo;
+    } else {
+        if (user_req_cb != NULL) {
+            (*user_req_cb)(user_req_ptr);
+        }
+    }
+    opal_mutex_unlock(&winfo->mutex);
+    return rc;
+#else
+    return OPAL_ERR_NOT_SUPPORTED;
+#endif // HAVE_DECL_UCP_EP_FLUSH_NB
+
+}
+
+/* TODO Replace the input with opal_common_ucx_ctx_t */
+OPAL_DECLSPEC int opal_common_ucx_wpmem_fence(opal_common_ucx_wpmem_t *mem)
+{
+    ucs_status_t status = UCS_OK;
+    _ctx_record_t *ctx_rec;
+    opal_common_ucx_winfo_t *winfo;
+    opal_common_ucx_ctx_t *ctx;
+    int rc = OPAL_SUCCESS;
+
+    if (NULL == mem) {
+        return OPAL_SUCCESS;
+    }
+
+    ctx = mem->ctx;
+    opal_mutex_lock(&ctx->mutex);
+
+    OPAL_LIST_FOREACH (ctx_rec, &ctx->ctx_records, _ctx_record_t) {
+        winfo = ctx_rec->winfo;
+        opal_mutex_lock(&winfo->mutex);
+        status = ucp_worker_fence(winfo->worker);
+        opal_mutex_unlock(&winfo->mutex);
+        if (status != UCS_OK) {
+            MCA_COMMON_UCX_ERROR("opal_common_ucx_fence failed: %d", rc);
+            rc = OPAL_ERROR;
+            break;
+        }
+    }
+
+    opal_mutex_unlock(&ctx->mutex);
+
+    return rc;
+}
+
+OPAL_DECLSPEC void opal_common_ucx_req_init(void *request)
+{
+    opal_common_ucx_request_t *req = (opal_common_ucx_request_t *) request;
     req->ext_req = NULL;
     req->ext_cb = NULL;
     req->winfo = NULL;
 }
 
-OPAL_DECLSPEC void
-opal_common_ucx_req_completion(void *request, ucs_status_t status) {
-    opal_common_ucx_request_t *req = (opal_common_ucx_request_t *)request;
+OPAL_DECLSPEC void opal_common_ucx_req_completion(void *request, ucs_status_t status)
+{
+    opal_common_ucx_request_t *req = (opal_common_ucx_request_t *) request;
     if (req->ext_cb != NULL) {
         (*req->ext_cb)(req->ext_req);
     }
diff --git a/opal/mca/common/ucx/common_ucx_wpool.h b/opal/mca/common/ucx/common_ucx_wpool.h
index ce5ef4d3b4a..0d94e51cb64 100644
--- a/opal/mca/common/ucx/common_ucx_wpool.h
+++ b/opal/mca/common/ucx/common_ucx_wpool.h
@@ -12,7 +12,6 @@
 #ifndef COMMON_UCX_WPOOL_H
 #define COMMON_UCX_WPOOL_H
 
-
 #include "opal_config.h"
 
 #include "common_ucx.h"
@@ -21,18 +20,23 @@
 
 #include <ucp/api/ucp.h>
 
-#include "opal/mca/mca.h"
-#include "opal/util/output.h"
-#include "opal/runtime/opal_progress.h"
-#include "opal/include/opal/constants.h"
 #include "opal/class/opal_list.h"
+#include "opal/include/opal/constants.h"
+#include "opal/mca/mca.h"
 #include "opal/mca/threads/tsd.h"
+#include "opal/runtime/opal_progress.h"
+#include "opal/util/output.h"
 
 BEGIN_C_DECLS
 
 /* fordward declaration */
 typedef struct opal_common_ucx_winfo opal_common_ucx_winfo_t;
 
+typedef enum {
+    OPAL_COMMON_UCX_WPMEM_ADDR_EXCHANGE_FULL,
+    OPAL_COMMON_UCX_WPMEM_ADDR_EXCHANGE_DIRECT
+} opal_common_ucx_exchange_mode_t;
+
 /* Worker pool is a global object that that is allocated per component or can be
  * shared between multiple compatible components.
  * The lifetime of this object is normally equal to the lifetime of a component[s].
@@ -54,6 +58,19 @@ typedef struct {
     opal_list_t active_workers;
 } opal_common_ucx_wpool_t;
 
+extern bool opal_common_ucx_thread_enabled;
+extern opal_atomic_int64_t opal_common_ucx_ep_counts;
+extern opal_atomic_int64_t opal_common_ucx_unpacked_rkey_counts;
+
+#if OPAL_ENABLE_DEBUG
+#define OPAL_COMMON_UCX_DEBUG_ATOMIC_ADD(_var, _val)                                \
+    do {                                                                            \
+        opal_atomic_add_fetch_64(&(_var), (_val));                                  \
+    } while(0);
+#else
+#define OPAL_COMMON_UCX_DEBUG_ATOMIC_ADD(_var, _val);
+#endif
+
 /* Worker Pool Context (wpctx) is an object that is comprised of a set of UCP
  * workers that are considered as one logical communication entity.
  * One UCP worker per "active" thread is used.
@@ -64,7 +81,7 @@ typedef struct {
  * Context is bound to a particular Worker Pool object.
  */
 typedef struct {
-    opal_mutex_t mutex;
+    opal_recursive_mutex_t mutex;
 
     /* the reference to a Worker pool this context belongs to*/
     opal_common_ucx_wpool_t *wpool;
@@ -82,6 +99,7 @@ typedef struct {
     char *recv_worker_addrs;
     int *recv_worker_displs;
     size_t comm_size;
+    opal_atomic_int64_t num_incomplete_req_ops;
 } opal_common_ucx_ctx_t;
 
 /* Worker Pool memory (wpmem) is an object that represents a remotely accessible
@@ -101,11 +119,7 @@ typedef struct {
     ucp_mem_h memh;
     char *mem_addrs;
     int *mem_displs;
-
-    /* A list of mem records
-     * We need to kepp trakc o fallocated memory records so that we can free them at the end
-     * if a thread fails to release the memory record */
-    opal_list_t mem_records;
+    bool skip_periodic_flush;
 
     /* TLS item that allows each thread to
      * store endpoints and rkey arrays
@@ -128,6 +142,7 @@ struct opal_common_ucx_winfo {
     short *inflight_ops;
     short global_inflight_ops;
     ucs_status_ptr_t inflight_req;
+    bool is_dflt_winfo;
 };
 OBJ_CLASS_DECLARATION(opal_common_ucx_winfo_t);
 
@@ -143,10 +158,7 @@ typedef struct {
     opal_common_ucx_winfo_t *winfo;
 } opal_common_ucx_request_t;
 
-typedef enum {
-    OPAL_COMMON_UCX_PUT,
-    OPAL_COMMON_UCX_GET
-} opal_common_ucx_op_t;
+typedef enum { OPAL_COMMON_UCX_PUT, OPAL_COMMON_UCX_GET } opal_common_ucx_op_t;
 
 typedef enum {
     OPAL_COMMON_UCX_SCOPE_EP,
@@ -168,7 +180,7 @@ typedef struct {
     opal_list_item_t super;
     opal_common_ucx_ctx_t *gctx;
     opal_common_ucx_winfo_t *winfo;
-} _ctx_record_t; 
+} _ctx_record_t;
 OBJ_CLASS_DECLARATION(_ctx_record_t);
 
 typedef struct {
@@ -180,23 +192,20 @@ typedef struct {
 } _mem_record_t;
 OBJ_CLASS_DECLARATION(_mem_record_t);
 
-typedef int (*opal_common_ucx_exchange_func_t)(void *my_info, size_t my_info_len,
-                                               char **recv_info, int **disps,
-                                               void *metadata);
-
+typedef int (*opal_common_ucx_exchange_func_t)(void *my_info, size_t my_info_len, char **recv_info,
+                                               int **disps, void *metadata);
 /* Manage Worker Pool (wpool) */
-OPAL_DECLSPEC opal_common_ucx_wpool_t * opal_common_ucx_wpool_allocate(void);
+OPAL_DECLSPEC opal_common_ucx_wpool_t *opal_common_ucx_wpool_allocate(void);
 OPAL_DECLSPEC void opal_common_ucx_wpool_free(opal_common_ucx_wpool_t *wpool);
-OPAL_DECLSPEC int opal_common_ucx_wpool_init(opal_common_ucx_wpool_t *wpool,
-                                             int proc_world_size, bool enable_mt);
+OPAL_DECLSPEC int opal_common_ucx_wpool_init(opal_common_ucx_wpool_t *wpool);
 OPAL_DECLSPEC void opal_common_ucx_wpool_finalize(opal_common_ucx_wpool_t *wpool);
 OPAL_DECLSPEC int opal_common_ucx_wpool_progress(opal_common_ucx_wpool_t *wpool);
 
 /* Manage Communication context */
 OPAL_DECLSPEC int opal_common_ucx_wpctx_create(opal_common_ucx_wpool_t *wpool, int comm_size,
-                                             opal_common_ucx_exchange_func_t exchange_func,
-                                             void *exchange_metadata,
-                                             opal_common_ucx_ctx_t **ctx_ptr);
+                                               opal_common_ucx_exchange_func_t exchange_func,
+                                               void *exchange_metadata,
+                                               opal_common_ucx_ctx_t **ctx_ptr);
 OPAL_DECLSPEC void opal_common_ucx_wpctx_release(opal_common_ucx_ctx_t *ctx);
 
 /* request init / completion */
@@ -204,30 +213,29 @@ OPAL_DECLSPEC void opal_common_ucx_req_init(void *request);
 OPAL_DECLSPEC void opal_common_ucx_req_completion(void *request, ucs_status_t status);
 
 /* Managing thread local storage */
-OPAL_DECLSPEC int opal_common_ucx_tlocal_fetch_spath(opal_common_ucx_wpmem_t *mem, int target);
-static inline int
-opal_common_ucx_tlocal_fetch(opal_common_ucx_wpmem_t *mem, int target,
-                                ucp_ep_h *_ep, ucp_rkey_h *_rkey,
-                                opal_common_ucx_winfo_t **_winfo)
+OPAL_DECLSPEC int opal_common_ucx_tlocal_fetch_spath(opal_common_ucx_wpmem_t *mem, int target, ucp_ep_h *_dflt_ep);
+static inline int opal_common_ucx_tlocal_fetch(opal_common_ucx_wpmem_t *mem, int target,
+                                               ucp_ep_h *_ep, ucp_rkey_h *_rkey,
+                                               opal_common_ucx_winfo_t **_winfo,
+                                               ucp_ep_h *_dflt_ep)
 {
     _mem_record_t *mem_rec = NULL;
     int is_ready;
     int rc = OPAL_SUCCESS;
 
     /* First check the fast-path */
-    rc = opal_tsd_tracked_key_get(&mem->tls_key, (void**)&mem_rec);
+    rc = opal_tsd_tracked_key_get(&mem->tls_key, (void **) &mem_rec);
     if (OPAL_SUCCESS != rc) {
         return rc;
     }
-    is_ready = mem_rec && (mem_rec->winfo->endpoints[target]) &&
-            (NULL != mem_rec->rkeys[target]);
+    is_ready = mem_rec && (mem_rec->winfo->endpoints[target]) && (NULL != mem_rec->rkeys[target]);
     MCA_COMMON_UCX_ASSERT((NULL == mem_rec) || (NULL != mem_rec->winfo));
     if (OPAL_UNLIKELY(!is_ready)) {
-        rc = opal_common_ucx_tlocal_fetch_spath(mem, target);
+        rc = opal_common_ucx_tlocal_fetch_spath(mem, target, _dflt_ep);
         if (OPAL_SUCCESS != rc) {
             return rc;
         }
-        rc = opal_tsd_tracked_key_get(&mem->tls_key, (void**)&mem_rec);
+        rc = opal_tsd_tracked_key_get(&mem->tls_key, (void **) &mem_rec);
         if (OPAL_SUCCESS != rc) {
             return rc;
         }
@@ -244,19 +252,22 @@ opal_common_ucx_tlocal_fetch(opal_common_ucx_wpmem_t *mem, int target,
 }
 
 /* Manage & operations on the Memory registrations */
-OPAL_DECLSPEC int opal_common_ucx_wpmem_create(opal_common_ucx_ctx_t *ctx,
-                               void **mem_base, size_t mem_size,
-                               opal_common_ucx_mem_type_t mem_type,
-                               opal_common_ucx_exchange_func_t exchange_func,
-                               void *exchange_metadata,
-                                               char **my_mem_addr,
+OPAL_DECLSPEC int opal_common_ucx_wpmem_create(opal_common_ucx_ctx_t *ctx, void **mem_base,
+                                               size_t mem_size, opal_common_ucx_mem_type_t mem_type,
+                                               opal_common_ucx_exchange_func_t exchange_func,
+                                               opal_common_ucx_exchange_mode_t exchange_mode,
+                                               void *exchange_metadata, char **my_mem_addr,
                                                int *my_mem_addr_size,
-                               opal_common_ucx_wpmem_t **mem_ptr);
+                                               opal_common_ucx_wpmem_t **mem_ptr);
 OPAL_DECLSPEC void opal_common_ucx_wpmem_free(opal_common_ucx_wpmem_t *mem);
 
-OPAL_DECLSPEC int opal_common_ucx_wpmem_flush(opal_common_ucx_wpmem_t *mem,
-                                            opal_common_ucx_flush_scope_t scope,
-                                            int target);
+OPAL_DECLSPEC int opal_common_ucx_ctx_flush(opal_common_ucx_ctx_t *ctx,
+                                              opal_common_ucx_flush_scope_t scope,
+                                              int target);
+OPAL_DECLSPEC int opal_common_ucx_wpmem_flush_ep_nb(opal_common_ucx_wpmem_t *mem,
+                                                    int target,
+                                                    opal_common_ucx_user_req_handler_t user_req_cb,
+                                                    void *user_req_ptr, ucp_ep_h *_dflt_ep);
 OPAL_DECLSPEC int opal_common_ucx_wpmem_fence(opal_common_ucx_wpmem_t *mem);
 
 OPAL_DECLSPEC int opal_common_ucx_winfo_flush(opal_common_ucx_winfo_t *winfo, int target,
@@ -264,8 +275,7 @@ OPAL_DECLSPEC int opal_common_ucx_winfo_flush(opal_common_ucx_winfo_t *winfo, in
                                               opal_common_ucx_flush_scope_t scope,
                                               ucs_status_ptr_t *req_ptr);
 
-static inline
-int opal_common_ucx_wait_request_mt(ucs_status_ptr_t request, const char *msg)
+static inline int opal_common_ucx_wait_request_mt(ucs_status_ptr_t request, const char *msg)
 {
     ucs_status_t status;
     int ctr = 0, ret = 0;
@@ -276,12 +286,11 @@ int opal_common_ucx_wait_request_mt(ucs_status_ptr_t request, const char *msg)
         return OPAL_SUCCESS;
     } else if (OPAL_UNLIKELY(UCS_PTR_IS_ERR(request))) {
         MCA_COMMON_UCX_VERBOSE(1, "%s failed: %d, %s", msg ? msg : __func__,
-                               UCS_PTR_STATUS(request),
-                               ucs_status_string(UCS_PTR_STATUS(request)));
+                               UCS_PTR_STATUS(request), ucs_status_string(UCS_PTR_STATUS(request)));
         return OPAL_ERROR;
     }
 
-    winfo = ((opal_common_ucx_request_t *)request)->winfo;
+    winfo = ((opal_common_ucx_request_t *) request)->winfo;
     assert(winfo != NULL);
 
     do {
@@ -293,40 +302,39 @@ int opal_common_ucx_wait_request_mt(ucs_status_ptr_t request, const char *msg)
             if (status != UCS_INPROGRESS) {
                 ucp_request_free(request);
                 if (OPAL_UNLIKELY(UCS_OK != status)) {
-                    MCA_COMMON_UCX_VERBOSE(1, "%s failed: %d, %s",
-                                           msg ? msg : __func__,
+                    MCA_COMMON_UCX_VERBOSE(1, "%s failed: %d, %s", msg ? msg : __func__,
                                            UCS_PTR_STATUS(request),
                                            ucs_status_string(UCS_PTR_STATUS(request)));
-                   opal_mutex_unlock(&winfo->mutex);
-                   return OPAL_ERROR;
+                    opal_mutex_unlock(&winfo->mutex);
+                    return OPAL_ERROR;
                 }
                 break;
             }
             ctr--;
-        } while (ctr > 0 && ret > 0 && status == UCS_INPROGRESS);
+        } while (ctr > 0 && ret == 0 && status == UCS_INPROGRESS);
         opal_mutex_unlock(&winfo->mutex);
-        opal_progress();
+        if (!ctr) {
+            opal_progress();
+        }
     } while (status == UCS_INPROGRESS);
 
     return OPAL_SUCCESS;
 }
 
-static inline int _periodical_flush_nb(opal_common_ucx_wpmem_t *mem,
-                                       opal_common_ucx_winfo_t *winfo,
-                                       int target) {
+static inline int _periodical_flush_nb(opal_common_ucx_wpmem_t *mem, opal_common_ucx_winfo_t *winfo,
+                                       int target)
+{
     int rc = OPAL_SUCCESS;
 
-    winfo->inflight_ops[target]++;
-    winfo->global_inflight_ops++;
+    if (mem->skip_periodic_flush) return OPAL_SUCCESS;
 
-    if (OPAL_UNLIKELY(winfo->inflight_ops[target] >= MCA_COMMON_UCX_PER_TARGET_OPS_THRESHOLD) ||
-        OPAL_UNLIKELY(winfo->global_inflight_ops >= MCA_COMMON_UCX_GLOBAL_OPS_THRESHOLD)) {
+    if (OPAL_UNLIKELY(winfo->inflight_ops[target] >= MCA_COMMON_UCX_PER_TARGET_OPS_THRESHOLD)
+        || OPAL_UNLIKELY(winfo->global_inflight_ops >= MCA_COMMON_UCX_GLOBAL_OPS_THRESHOLD)) {
         opal_common_ucx_flush_scope_t scope;
 
         if (winfo->inflight_req != UCS_OK) {
-            rc = opal_common_ucx_wait_request_mt(winfo->inflight_req,
-                                                 "opal_common_ucx_flush_nb");
-            if(OPAL_UNLIKELY(OPAL_SUCCESS != rc)){
+            rc = opal_common_ucx_wait_request_mt(winfo->inflight_req, "opal_common_ucx_flush_nb");
+            if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
                 MCA_COMMON_UCX_VERBOSE(1, "opal_common_ucx_wait_request failed: %d", rc);
                 return rc;
             }
@@ -343,9 +351,9 @@ static inline int _periodical_flush_nb(opal_common_ucx_wpmem_t *mem,
             winfo->inflight_ops[target] = 0;
         }
 
-        rc = opal_common_ucx_winfo_flush(winfo, target, OPAL_COMMON_UCX_FLUSH_NB_PREFERRED,
-                                         scope, &winfo->inflight_req);
-        if(OPAL_UNLIKELY(OPAL_SUCCESS != rc)){
+        rc = opal_common_ucx_winfo_flush(winfo, target, OPAL_COMMON_UCX_FLUSH_NB_PREFERRED, scope,
+                                         &winfo->inflight_req);
+        if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
             MCA_COMMON_UCX_VERBOSE(1, "opal_common_ucx_flush failed: %d", rc);
             return rc;
         }
@@ -358,10 +366,9 @@ static inline int _periodical_flush_nb(opal_common_ucx_wpmem_t *mem,
     return rc;
 }
 
-static inline int
-opal_common_ucx_wpmem_putget(opal_common_ucx_wpmem_t *mem, opal_common_ucx_op_t op,
-                           int target, void *buffer, size_t len,
-                           uint64_t rem_addr)
+static inline int opal_common_ucx_wpmem_putget(opal_common_ucx_wpmem_t *mem,
+                                               opal_common_ucx_op_t op, int target, void *buffer,
+                                               size_t len, uint64_t rem_addr, ucp_ep_h *dflt_ep)
 {
     ucp_ep_h ep;
     ucp_rkey_h rkey;
@@ -370,21 +377,21 @@ opal_common_ucx_wpmem_putget(opal_common_ucx_wpmem_t *mem, opal_common_ucx_op_t
     int rc = OPAL_SUCCESS;
     char *called_func = "";
 
-    rc = opal_common_ucx_tlocal_fetch(mem, target, &ep, &rkey, &winfo);
-    if(OPAL_UNLIKELY(OPAL_SUCCESS != rc)){
+    rc = opal_common_ucx_tlocal_fetch(mem, target, &ep, &rkey, &winfo, dflt_ep);
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
         MCA_COMMON_UCX_VERBOSE(1, "tlocal_fetch failed: %d", rc);
         return rc;
     }
 
     /* Perform the operation */
     opal_mutex_lock(&winfo->mutex);
-    switch(op){
+    switch (op) {
     case OPAL_COMMON_UCX_PUT:
-        status = ucp_put_nbi(ep, buffer,len, rem_addr, rkey);
+        status = ucp_put_nbi(ep, buffer, len, rem_addr, rkey);
         called_func = "ucp_put_nbi";
         break;
     case OPAL_COMMON_UCX_GET:
-        status = ucp_get_nbi(ep, buffer,len, rem_addr, rkey);
+        status = ucp_get_nbi(ep, buffer, len, rem_addr, rkey);
         called_func = "ucp_get_nbi";
         break;
     }
@@ -395,8 +402,13 @@ opal_common_ucx_wpmem_putget(opal_common_ucx_wpmem_t *mem, opal_common_ucx_op_t
         goto out;
     }
 
+    if (status == UCS_INPROGRESS) {
+        winfo->inflight_ops[target]++;
+        winfo->global_inflight_ops++;
+    }
+
     rc = _periodical_flush_nb(mem, winfo, target);
-    if(OPAL_UNLIKELY(OPAL_SUCCESS != rc)){
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
         MCA_COMMON_UCX_VERBOSE(1, "_incr_and_check_inflight_ops failed: %d", rc);
     }
 
@@ -406,11 +418,9 @@ opal_common_ucx_wpmem_putget(opal_common_ucx_wpmem_t *mem, opal_common_ucx_op_t
     return rc;
 }
 
-
-static inline int
-opal_common_ucx_wpmem_cmpswp(opal_common_ucx_wpmem_t *mem, uint64_t compare,
-                           uint64_t value, int target, void *buffer, size_t len,
-                           uint64_t rem_addr)
+static inline int opal_common_ucx_wpmem_cmpswp(opal_common_ucx_wpmem_t *mem, uint64_t compare,
+                                               uint64_t value, int target, void *buffer, size_t len,
+                                               uint64_t rem_addr, ucp_ep_h *dflt_ep)
 {
     ucp_ep_h ep;
     ucp_rkey_h rkey;
@@ -418,7 +428,7 @@ opal_common_ucx_wpmem_cmpswp(opal_common_ucx_wpmem_t *mem, uint64_t compare,
     ucs_status_t status;
     int rc = OPAL_SUCCESS;
 
-    rc = opal_common_ucx_tlocal_fetch(mem, target, &ep, &rkey, &winfo);
+    rc = opal_common_ucx_tlocal_fetch(mem, target, &ep, &rkey, &winfo, dflt_ep);
     if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
         MCA_COMMON_UCX_ERROR("opal_common_ucx_tlocal_fetch failed: %d", rc);
         return rc;
@@ -426,9 +436,7 @@ opal_common_ucx_wpmem_cmpswp(opal_common_ucx_wpmem_t *mem, uint64_t compare,
 
     /* Perform the operation */
     opal_mutex_lock(&winfo->mutex);
-    status = opal_common_ucx_atomic_cswap(ep, compare, value,
-                                          buffer, len,
-                                          rem_addr, rkey,
+    status = opal_common_ucx_atomic_cswap(ep, compare, value, buffer, len, rem_addr, rkey,
                                           winfo->worker);
     if (OPAL_UNLIKELY(status != UCS_OK)) {
         MCA_COMMON_UCX_ERROR("opal_common_ucx_atomic_cswap failed: %d", status);
@@ -437,7 +445,7 @@ opal_common_ucx_wpmem_cmpswp(opal_common_ucx_wpmem_t *mem, uint64_t compare,
     }
 
     rc = _periodical_flush_nb(mem, winfo, target);
-    if(OPAL_UNLIKELY(OPAL_SUCCESS != rc)){
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
         MCA_COMMON_UCX_VERBOSE(1, "_incr_and_check_inflight_ops failed: %d", rc);
     }
 
@@ -447,13 +455,11 @@ opal_common_ucx_wpmem_cmpswp(opal_common_ucx_wpmem_t *mem, uint64_t compare,
     return rc;
 }
 
-
-static inline int
-opal_common_ucx_wpmem_cmpswp_nb(opal_common_ucx_wpmem_t *mem, uint64_t compare,
-                                uint64_t value, int target, void *buffer, size_t len,
-                                uint64_t rem_addr,
-                                opal_common_ucx_user_req_handler_t user_req_cb,
-                                void *user_req_ptr)
+static inline int opal_common_ucx_wpmem_cmpswp_nb(opal_common_ucx_wpmem_t *mem, uint64_t compare,
+                                                  uint64_t value, int target, void *buffer,
+                                                  size_t len, uint64_t rem_addr,
+                                                  opal_common_ucx_user_req_handler_t user_req_cb,
+                                                  void *user_req_ptr, ucp_ep_h *dflt_ep)
 {
     ucp_ep_h ep;
     ucp_rkey_h rkey;
@@ -461,7 +467,7 @@ opal_common_ucx_wpmem_cmpswp_nb(opal_common_ucx_wpmem_t *mem, uint64_t compare,
     opal_common_ucx_request_t *req;
     int rc = OPAL_SUCCESS;
 
-    rc = opal_common_ucx_tlocal_fetch(mem, target, &ep, &rkey, &winfo);
+    rc = opal_common_ucx_tlocal_fetch(mem, target, &ep, &rkey, &winfo, dflt_ep);
     if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
         MCA_COMMON_UCX_ERROR("opal_common_ucx_tlocal_fetch failed: %d", rc);
         return rc;
@@ -469,12 +475,11 @@ opal_common_ucx_wpmem_cmpswp_nb(opal_common_ucx_wpmem_t *mem, uint64_t compare,
 
     /* Perform the operation */
     opal_mutex_lock(&winfo->mutex);
-    req = opal_common_ucx_atomic_cswap_nb(ep, compare, value,
-                                          buffer, len,
-                                          rem_addr, rkey, opal_common_ucx_req_completion,
-                                          winfo->worker);
-
+    req = opal_common_ucx_atomic_cswap_nb(ep, compare, value, buffer, len, rem_addr, rkey,
+                                          opal_common_ucx_req_completion, winfo->worker);
     if (UCS_PTR_IS_PTR(req)) {
+        winfo->inflight_ops[target]++;
+        winfo->global_inflight_ops++;
         req->ext_req = user_req_ptr;
         req->ext_cb = user_req_cb;
         req->winfo = winfo;
@@ -484,9 +489,8 @@ opal_common_ucx_wpmem_cmpswp_nb(opal_common_ucx_wpmem_t *mem, uint64_t compare,
         }
     }
 
-
     rc = _periodical_flush_nb(mem, winfo, target);
-    if(OPAL_UNLIKELY(OPAL_SUCCESS != rc)){
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
         MCA_COMMON_UCX_VERBOSE(1, "_incr_and_check_inflight_ops failed: %d", rc);
     }
 
@@ -495,10 +499,9 @@ opal_common_ucx_wpmem_cmpswp_nb(opal_common_ucx_wpmem_t *mem, uint64_t compare,
     return rc;
 }
 
-
-static inline int
-opal_common_ucx_wpmem_post(opal_common_ucx_wpmem_t *mem, ucp_atomic_post_op_t opcode,
-                         uint64_t value, int target, size_t len, uint64_t rem_addr)
+static inline int opal_common_ucx_wpmem_post(opal_common_ucx_wpmem_t *mem,
+                                             ucp_atomic_post_op_t opcode, uint64_t value,
+                                             int target, size_t len, uint64_t rem_addr, ucp_ep_h *dflt_ep)
 {
     ucp_ep_h ep;
     ucp_rkey_h rkey;
@@ -506,17 +509,15 @@ opal_common_ucx_wpmem_post(opal_common_ucx_wpmem_t *mem, ucp_atomic_post_op_t op
     ucs_status_t status;
     int rc = OPAL_SUCCESS;
 
-
-    rc =opal_common_ucx_tlocal_fetch(mem, target, &ep, &rkey, &winfo);
-    if(OPAL_UNLIKELY(OPAL_SUCCESS != rc)){
+    rc = opal_common_ucx_tlocal_fetch(mem, target, &ep, &rkey, &winfo, dflt_ep);
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
         MCA_COMMON_UCX_ERROR("tlocal_fetch failed: %d", rc);
         return rc;
     }
 
     /* Perform the operation */
     opal_mutex_lock(&winfo->mutex);
-    status = ucp_atomic_post(ep, opcode, value,
-                             len, rem_addr, rkey);
+    status = ucp_atomic_post(ep, opcode, value, len, rem_addr, rkey);
     if (OPAL_UNLIKELY(status != UCS_OK)) {
         MCA_COMMON_UCX_ERROR("ucp_atomic_post failed: %d", status);
         rc = OPAL_ERROR;
@@ -524,7 +525,7 @@ opal_common_ucx_wpmem_post(opal_common_ucx_wpmem_t *mem, ucp_atomic_post_op_t op
     }
 
     rc = _periodical_flush_nb(mem, winfo, target);
-    if(OPAL_UNLIKELY(OPAL_SUCCESS != rc)){
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
         MCA_COMMON_UCX_VERBOSE(1, "_incr_and_check_inflight_ops failed: %d", rc);
     }
 
@@ -533,11 +534,10 @@ opal_common_ucx_wpmem_post(opal_common_ucx_wpmem_t *mem, ucp_atomic_post_op_t op
     return rc;
 }
 
-static inline int
-opal_common_ucx_wpmem_fetch(opal_common_ucx_wpmem_t *mem,
-                            ucp_atomic_fetch_op_t opcode, uint64_t value,
-                            int target, void *buffer, size_t len,
-                            uint64_t rem_addr)
+static inline int opal_common_ucx_wpmem_fetch(opal_common_ucx_wpmem_t *mem,
+                                              ucp_atomic_fetch_op_t opcode, uint64_t value,
+                                              int target, void *buffer, size_t len,
+                                              uint64_t rem_addr, ucp_ep_h *dflt_ep)
 {
     ucp_ep_h ep = NULL;
     ucp_rkey_h rkey = NULL;
@@ -545,17 +545,15 @@ opal_common_ucx_wpmem_fetch(opal_common_ucx_wpmem_t *mem,
     ucs_status_t status;
     int rc = OPAL_SUCCESS;
 
-    rc = opal_common_ucx_tlocal_fetch(mem, target, &ep, &rkey, &winfo);
-    if(OPAL_UNLIKELY(OPAL_SUCCESS != rc)){
+    rc = opal_common_ucx_tlocal_fetch(mem, target, &ep, &rkey, &winfo, dflt_ep);
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
         MCA_COMMON_UCX_ERROR("tlocal_fetch failed: %d", rc);
         return rc;
     }
 
     /* Perform the operation */
     opal_mutex_lock(&winfo->mutex);
-    status = opal_common_ucx_atomic_fetch(ep, opcode, value,
-                                          buffer, len,
-                                          rem_addr, rkey,
+    status = opal_common_ucx_atomic_fetch(ep, opcode, value, buffer, len, rem_addr, rkey,
                                           winfo->worker);
     if (OPAL_UNLIKELY(status != UCS_OK)) {
         MCA_COMMON_UCX_ERROR("ucp_atomic_cswap64 failed: %d", status);
@@ -564,7 +562,7 @@ opal_common_ucx_wpmem_fetch(opal_common_ucx_wpmem_t *mem,
     }
 
     rc = _periodical_flush_nb(mem, winfo, target);
-    if(OPAL_UNLIKELY(OPAL_SUCCESS != rc)){
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
         MCA_COMMON_UCX_VERBOSE(1, "_incr_and_check_inflight_ops failed: %d", rc);
     }
 
@@ -574,14 +572,12 @@ opal_common_ucx_wpmem_fetch(opal_common_ucx_wpmem_t *mem,
     return rc;
 }
 
-static inline int
-opal_common_ucx_wpmem_fetch_nb(opal_common_ucx_wpmem_t *mem,
-                               ucp_atomic_fetch_op_t opcode,
-                               uint64_t value,
-                               int target, void *buffer, size_t len,
-                               uint64_t rem_addr,
-                               opal_common_ucx_user_req_handler_t user_req_cb,
-                               void *user_req_ptr)
+static inline int opal_common_ucx_wpmem_fetch_nb(opal_common_ucx_wpmem_t *mem,
+                                                 ucp_atomic_fetch_op_t opcode, uint64_t value,
+                                                 int target, void *buffer, size_t len,
+                                                 uint64_t rem_addr,
+                                                 opal_common_ucx_user_req_handler_t user_req_cb,
+                                                 void *user_req_ptr, ucp_ep_h *dflt_ep)
 {
     ucp_ep_h ep = NULL;
     ucp_rkey_h rkey = NULL;
@@ -589,17 +585,18 @@ opal_common_ucx_wpmem_fetch_nb(opal_common_ucx_wpmem_t *mem,
     int rc = OPAL_SUCCESS;
     opal_common_ucx_request_t *req;
 
-    rc = opal_common_ucx_tlocal_fetch(mem, target, &ep, &rkey, &winfo);
-    if(OPAL_UNLIKELY(OPAL_SUCCESS != rc)){
+    rc = opal_common_ucx_tlocal_fetch(mem, target, &ep, &rkey, &winfo, dflt_ep);
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
         MCA_COMMON_UCX_ERROR("tlocal_fetch failed: %d", rc);
         return rc;
     }
     /* Perform the operation */
     opal_mutex_lock(&winfo->mutex);
-    req = opal_common_ucx_atomic_fetch_nb(ep, opcode, value, buffer, len,
-                                          rem_addr, rkey, opal_common_ucx_req_completion,
-                                          winfo->worker);
+    req = opal_common_ucx_atomic_fetch_nb(ep, opcode, value, buffer, len, rem_addr, rkey,
+                                          opal_common_ucx_req_completion, winfo->worker);
     if (UCS_PTR_IS_PTR(req)) {
+        winfo->inflight_ops[target]++;
+        winfo->global_inflight_ops++;
         req->ext_req = user_req_ptr;
         req->ext_cb = user_req_cb;
         req->winfo = winfo;
@@ -610,7 +607,7 @@ opal_common_ucx_wpmem_fetch_nb(opal_common_ucx_wpmem_t *mem,
     }
 
     rc = _periodical_flush_nb(mem, winfo, target);
-    if(OPAL_UNLIKELY(OPAL_SUCCESS != rc)){
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
         MCA_COMMON_UCX_VERBOSE(1, "_incr_and_check_inflight_ops failed: %d", rc);
     }
 
diff --git a/opal/mca/common/ucx/common_ucx_wpool_int.h b/opal/mca/common/ucx/common_ucx_wpool_int.h
index 2dcbf2c970c..2c4d0fe6134 100644
--- a/opal/mca/common/ucx/common_ucx_wpool_int.h
+++ b/opal/mca/common/ucx/common_ucx_wpool_int.h
@@ -15,8 +15,7 @@ static opal_common_ucx_winfo_t *_winfo_create(opal_common_ucx_wpool_t *wpool);
 static void _winfo_destructor(opal_common_ucx_winfo_t *winfo);
 
 /* Internal Worker Pool Memory management */
-static int _comm_ucx_wpmem_map(opal_common_ucx_wpool_t *wpool,
-                               void **base, size_t size, ucp_mem_h *memh_ptr,
-                               opal_common_ucx_mem_type_t mem_type);
+static int _comm_ucx_wpmem_map(opal_common_ucx_wpool_t *wpool, void **base, size_t size,
+                               ucp_mem_h *memh_ptr, opal_common_ucx_mem_type_t mem_type);
 
 #endif // COMMON_UCX_WPOOL_INT_H
diff --git a/opal/mca/common/ucx/configure.m4 b/opal/mca/common/ucx/configure.m4
index af8628a889c..1fe4f44ae2e 100644
--- a/opal/mca/common/ucx/configure.m4
+++ b/opal/mca/common/ucx/configure.m4
@@ -1,6 +1,7 @@
 # -*- shell-script -*-
 #
 # Copyright (c) 2018      Mellanox Technologies.  All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -8,12 +9,12 @@
 # $HEADER$
 #
 
-# MCA_opal_common_sm_CONFIG([action-if-can-compile],
+# MCA_opal_common_ucx_CONFIG([action-if-can-compile],
 #                           [action-if-cant-compile])
 # ------------------------------------------------
 AC_DEFUN([MCA_opal_common_ucx_CONFIG],[
     AC_CONFIG_FILES([opal/mca/common/ucx/Makefile])
-    common_ucx_happy="no"
+
     OMPI_CHECK_UCX([common_ucx],
                [common_ucx_happy="yes"],
                [common_ucx_happy="no"])
@@ -24,9 +25,7 @@ AC_DEFUN([MCA_opal_common_ucx_CONFIG],[
           [$1],
           [$2])
 
-
     # substitute in the things needed to build common_ucx
-    AC_SUBST([common_ucx_CFLAGS])
     AC_SUBST([common_ucx_CPPFLAGS])
     AC_SUBST([common_ucx_LDFLAGS])
     AC_SUBST([common_ucx_LIBS])
diff --git a/opal/mca/dl/base/base.h b/opal/mca/dl/base/base.h
index 50b07ec02b2..5bfaca4a3a5 100644
--- a/opal/mca/dl/base/base.h
+++ b/opal/mca/dl/base/base.h
@@ -19,18 +19,15 @@
 
 #include "opal/mca/base/base.h"
 
-
 BEGIN_C_DECLS
 
 /**
  * Globals
  */
 OPAL_DECLSPEC extern mca_base_framework_t opal_dl_base_framework;
-OPAL_DECLSPEC extern opal_dl_base_component_t
-*opal_dl_base_selected_component;
+OPAL_DECLSPEC extern opal_dl_base_component_t *opal_dl_base_selected_component;
 OPAL_DECLSPEC extern opal_dl_base_module_t *opal_dl;
 
-
 /**
  * Initialize the DL MCA framework
  *
@@ -67,8 +64,7 @@ OPAL_DECLSPEC int opal_dl_base_close(void);
  * (see opal_dl_base_module_open_ft_t in opal/mca/dl/dl.h for
  * documentation of this function)
  */
-OPAL_DECLSPEC int opal_dl_open(const char *fname,
-                               bool use_ext, bool private_namespace,
+OPAL_DECLSPEC int opal_dl_open(const char *fname, bool use_ext, bool private_namespace,
                                opal_dl_handle_t **handle, char **err_msg);
 
 /**
@@ -77,9 +73,8 @@ OPAL_DECLSPEC int opal_dl_open(const char *fname,
  * (see opal_dl_base_module_lookup_ft_t in opal/mca/dl/dl.h for
  * documentation of this function)
  */
-OPAL_DECLSPEC int opal_dl_lookup(opal_dl_handle_t *handle,
-                                 const char *symbol,
-                                 void **ptr, char **err_msg);
+OPAL_DECLSPEC int opal_dl_lookup(opal_dl_handle_t *handle, const char *symbol, void **ptr,
+                                 char **err_msg);
 
 /**
  * Close a DSO
@@ -96,8 +91,7 @@ OPAL_DECLSPEC int opal_dl_close(opal_dl_handle_t *handle);
  * documentation of this function)
  */
 OPAL_DECLSPEC int opal_dl_foreachfile(const char *search_path,
-                                      int (*cb_func)(const char *filename,
-                                                     void *context),
+                                      int (*cb_func)(const char *filename, void *context),
                                       void *context);
 
 END_C_DECLS
diff --git a/opal/mca/dl/base/dl_base_close.c b/opal/mca/dl/base/dl_base_close.c
index 3bf286ab443..1c2bdf1d0c9 100644
--- a/opal/mca/dl/base/dl_base_close.c
+++ b/opal/mca/dl/base/dl_base_close.c
@@ -11,12 +11,11 @@
 
 #include "opal_config.h"
 
-#include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
+#include "opal/mca/mca.h"
 
-#include "opal/mca/dl/dl.h"
 #include "opal/mca/dl/base/base.h"
-
+#include "opal/mca/dl/dl.h"
 
 int opal_dl_base_close(void)
 {
diff --git a/opal/mca/dl/base/dl_base_fns.c b/opal/mca/dl/base/dl_base_fns.c
index 32b70111236..4a9696cf134 100644
--- a/opal/mca/dl/base/dl_base_fns.c
+++ b/opal/mca/dl/base/dl_base_fns.c
@@ -21,24 +21,19 @@
 
 #include "opal/mca/dl/base/base.h"
 
-
-int opal_dl_open(const char *fname,
-                 bool use_ext, bool private_namespace,
-                 opal_dl_handle_t **handle, char **err_msg)
+int opal_dl_open(const char *fname, bool use_ext, bool private_namespace, opal_dl_handle_t **handle,
+                 char **err_msg)
 {
     *handle = NULL;
 
     if (NULL != opal_dl && NULL != opal_dl->open) {
-        return opal_dl->open(fname, use_ext, private_namespace,
-                             handle, err_msg);
+        return opal_dl->open(fname, use_ext, private_namespace, handle, err_msg);
     }
 
     return OPAL_ERR_NOT_SUPPORTED;
 }
 
-int opal_dl_lookup(opal_dl_handle_t *handle,
-                   const char *symbol,
-                   void **ptr, char **err_msg)
+int opal_dl_lookup(opal_dl_handle_t *handle, const char *symbol, void **ptr, char **err_msg)
 {
     if (NULL != opal_dl && NULL != opal_dl->lookup) {
         return opal_dl->lookup(handle, symbol, ptr, err_msg);
@@ -57,8 +52,7 @@ int opal_dl_close(opal_dl_handle_t *handle)
 }
 
 int opal_dl_foreachfile(const char *search_path,
-                        int (*cb_func)(const char *filename, void *context),
-                        void *context)
+                        int (*cb_func)(const char *filename, void *context), void *context)
 {
     if (NULL != opal_dl && NULL != opal_dl->foreachfile) {
         return opal_dl->foreachfile(search_path, cb_func, context);
diff --git a/opal/mca/dl/base/dl_base_open.c b/opal/mca/dl/base/dl_base_open.c
index 5e65cff9c0b..864fa8ea885 100644
--- a/opal/mca/dl/base/dl_base_open.c
+++ b/opal/mca/dl/base/dl_base_open.c
@@ -17,14 +17,12 @@
 
 #include "opal/mca/dl/base/static-components.h"
 
-
 /*
  * Globals
  */
 opal_dl_base_module_t *opal_dl = NULL;
 opal_dl_base_component_t *opal_dl_base_selected_component = NULL;
 
-
 /*
  * Function for finding and opening either all MCA components,
  * or the one that was specifically requested via a MCA parameter.
@@ -42,13 +40,10 @@ int opal_dl_base_open(mca_base_open_flag_t flags)
 }
 
 /* VERY IMPORTANT: This framework is static, and is opened before any
-   other dyanmic frameworks are opened (which makes sense, of course).
+   other dynamic frameworks are opened (which makes sense, of course).
    But we must mark this framework is NO_DSO so that the MCA framework
    base doesn't try to open any dynamic components in this
    framework. */
-MCA_BASE_FRAMEWORK_DECLARE(opal, dl, "Dynamic loader framework",
-                           NULL /* register */,
-                           opal_dl_base_open /* open */,
-                           NULL /* close */,
-                           mca_dl_base_static_components,
-                           MCA_BASE_FRAMEWORK_FLAG_NO_DSO);
+MCA_BASE_FRAMEWORK_DECLARE(opal, dl, "Dynamic loader framework", NULL /* register */,
+                           opal_dl_base_open /* open */, NULL /* close */,
+                           mca_dl_base_static_components, MCA_BASE_FRAMEWORK_FLAG_NO_DSO);
diff --git a/opal/mca/dl/base/dl_base_select.c b/opal/mca/dl/base/dl_base_select.c
index 8db0b5e99b7..beb55c8f2fc 100644
--- a/opal/mca/dl/base/dl_base_select.c
+++ b/opal/mca/dl/base/dl_base_select.c
@@ -16,16 +16,15 @@
 #include "opal_config.h"
 
 #ifdef HAVE_UNISTD_H
-#include "unistd.h"
+#    include "unistd.h"
 #endif
 
 #include "opal/include/opal/constants.h"
-#include "opal/util/output.h"
-#include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
-#include "opal/mca/dl/dl.h"
 #include "opal/mca/dl/base/base.h"
-
+#include "opal/mca/dl/dl.h"
+#include "opal/mca/mca.h"
+#include "opal/util/output.h"
 
 int opal_dl_base_select(void)
 {
@@ -36,11 +35,11 @@ int opal_dl_base_select(void)
     /*
      * Select the best component
      */
-    if (OPAL_SUCCESS != mca_base_select("dl",
-                                        opal_dl_base_framework.framework_output,
-                                        &opal_dl_base_framework.framework_components,
-                                        (mca_base_module_t **) &best_module,
-                                        (mca_base_component_t **) &best_component, NULL) ) {
+    if (OPAL_SUCCESS
+        != mca_base_select("dl", opal_dl_base_framework.framework_output,
+                           &opal_dl_base_framework.framework_components,
+                           (mca_base_module_t **) &best_module,
+                           (mca_base_component_t **) &best_component, NULL)) {
         /* This will only happen if no component was selected */
         exit_status = OPAL_ERROR;
         goto cleanup;
@@ -50,6 +49,6 @@ int opal_dl_base_select(void)
     opal_dl_base_selected_component = best_component;
     opal_dl = best_module;
 
- cleanup:
+cleanup:
     return exit_status;
 }
diff --git a/opal/mca/dl/configure.m4 b/opal/mca/dl/configure.m4
index a3e2d00595c..289b351a31d 100644
--- a/opal/mca/dl/configure.m4
+++ b/opal/mca/dl/configure.m4
@@ -1,6 +1,7 @@
 dnl -*- shell-script -*-
 dnl
 dnl Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
+dnl Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
@@ -8,6 +9,9 @@ dnl
 dnl $HEADER$
 dnl
 
+dnl need in the core library
+AC_DEFUN([MCA_opal_dl_CORE_LIB], [1])
+
 dnl There will only be one component used in this framework, and it will
 dnl be selected at configure time by priority.  Components must set
 dnl their priorities in their configure.m4 file.
@@ -42,35 +46,7 @@ AC_DEFUN([MCA_opal_dl_CONFIG],[
     # happen if --disable-dlopen was *not* specified), do some more
     # logic.
     AS_IF([test "$MCA_opal_dl_STATIC_COMPONENTS" != ""],
-       [ # We had a winner -- w00t!
-
-        OPAL_HAVE_DL_SUPPORT=1
-        # If we added any -L flags to ADD_LDFLAGS, then we (might)
-        # need to add those directories to LD_LIBRARY_PATH.
-        # Otherwise, if we try to AC RUN_IFELSE anything here in
-        # configure, it might die because it can't find the libraries
-        # we just linked against.
-        OPAL_VAR_SCOPE_PUSH([opal_dl_base_found_l opal_dl_base_token opal_dl_base_tmp opal_dl_base_dir])
-        opal_dl_base_found_l=0
-        eval "opal_dl_base_tmp=\$opal_dl_${opal_dl_winner}_ADD_LIBS"
-        for opal_dl_base_token in $opal_dl_base_tmp; do
-            case $opal_dl_base_token in
-            -l*) opal_dl_base_found_l=1 ;;
-            esac
-        done
-        AS_IF([test $opal_dl_base_found_l -eq 1],
-              [eval "opal_dl_base_tmp=\$opal_dl_${opal_dl_winner}_ADD_LDFLAGS"
-               for opal_dl_base_token in $opal_dl_base_tmp; do
-                   case $opal_dl_base_token in
-                   -L*)
-                       opal_dl_base_dir=`echo $opal_dl_base_token | cut -c3-`
-                       export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$opal_dl_base_dir
-                       AC_MSG_WARN([Adding to LD_LIBRARY_PATH: $opal_dl_base_dir])
-                       ;;
-                   esac
-               done])
-        OPAL_VAR_SCOPE_POP
-    ])
+       [OPAL_HAVE_DL_SUPPORT=1])
 
     AC_DEFINE_UNQUOTED([OPAL_HAVE_DL_SUPPORT], [$OPAL_HAVE_DL_SUPPORT],
                        [Whether the OPAL DL framework is functional or not])
diff --git a/opal/mca/dl/dl.h b/opal/mca/dl/dl.h
index b2541672264..fa14328f8e0 100644
--- a/opal/mca/dl/dl.h
+++ b/opal/mca/dl/dl.h
@@ -49,8 +49,8 @@
 
 #include "opal_config.h"
 
-#include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
+#include "opal/mca/mca.h"
 
 BEGIN_C_DECLS
 
@@ -85,9 +85,9 @@ typedef struct opal_dl_handle_t opal_dl_handle_t;
  * or freed by the caller.  The contents of the err_msg string may
  * change after successive calls to opal_dl API calls.
  */
-typedef int (*opal_dl_base_module_open_fn_t)
-    (const char *fname, bool use_ext, bool private_namespace,
-     opal_dl_handle_t **handle, char **err_msg);
+typedef int (*opal_dl_base_module_open_fn_t)(const char *fname, bool use_ext,
+                                             bool private_namespace, opal_dl_handle_t **handle,
+                                             char **err_msg);
 
 /**
  * Lookup a symbol in an opened file.
@@ -106,8 +106,8 @@ typedef int (*opal_dl_base_module_open_fn_t)
  * or freed by the caller.  The contents of the err_msg string may
  * change after successive calls to opal_dl API calls.
  */
-typedef int (*opal_dl_base_module_lookup_fn_t)
-    (opal_dl_handle_t *handle, const char *symbol, void **ptr, char **err_msg);
+typedef int (*opal_dl_base_module_lookup_fn_t)(opal_dl_handle_t *handle, const char *symbol,
+                                               void **ptr, char **err_msg);
 
 /**
  * Dynamically close a previously dynamically-opened file.
@@ -120,8 +120,7 @@ typedef int (*opal_dl_base_module_lookup_fn_t)
  * This function should close the file and free and resources
  * associated with it (e.g., whatever is cached on the handle).
  */
-typedef int (*opal_dl_base_module_close_fn_t)
-    (opal_dl_handle_t *handle);
+typedef int (*opal_dl_base_module_close_fn_t)(opal_dl_handle_t *handle);
 
 /**
  * Search through a path of directories, invoking a callback on each
@@ -136,10 +135,8 @@ typedef int (*opal_dl_base_module_close_fn_t)
  * Returns:
  *   OPAL_SUCESS on success, OPAL_ERR* otherwise
  */
-typedef int (*opal_dl_base_module_foreachfile_fn_t)
-    (const char *search_path,
-     int (*cb_func)(const char *filename, void *context),
-     void *context);
+typedef int (*opal_dl_base_module_foreachfile_fn_t)(
+    const char *search_path, int (*cb_func)(const char *filename, void *context), void *context);
 
 /**
  * Structure for DL components.
@@ -160,17 +157,17 @@ typedef struct opal_dl_base_component_1_0_0_t opal_dl_base_component_t;
  * Structure for DL modules
  */
 struct opal_dl_base_module_1_0_0_t {
-    mca_base_module_2_0_0_t                 super;
+    mca_base_module_2_0_0_t super;
 
     /** Open / close */
-    opal_dl_base_module_open_fn_t           open;
-    opal_dl_base_module_close_fn_t          close;
+    opal_dl_base_module_open_fn_t open;
+    opal_dl_base_module_close_fn_t close;
 
     /** Lookup a symbol */
-    opal_dl_base_module_lookup_fn_t         lookup;
+    opal_dl_base_module_lookup_fn_t lookup;
 
     /** Iterate looking for files */
-    opal_dl_base_module_foreachfile_fn_t    foreachfile;
+    opal_dl_base_module_foreachfile_fn_t foreachfile;
 };
 typedef struct opal_dl_base_module_1_0_0_t opal_dl_base_module_1_0_0_t;
 typedef struct opal_dl_base_module_1_0_0_t opal_dl_base_module_t;
@@ -178,8 +175,7 @@ typedef struct opal_dl_base_module_1_0_0_t opal_dl_base_module_t;
 /**
  * Macro for use in components that are of type DL
  */
-#define OPAL_DL_BASE_VERSION_1_0_0              \
-    OPAL_MCA_BASE_VERSION_2_1_0("dl", 1, 0, 0)
+#define OPAL_DL_BASE_VERSION_1_0_0 OPAL_MCA_BASE_VERSION_2_1_0("dl", 1, 0, 0)
 
 END_C_DECLS
 
diff --git a/opal/mca/dl/dlopen/configure.m4 b/opal/mca/dl/dlopen/configure.m4
index 714b880edc1..4ae625b1fb5 100644
--- a/opal/mca/dl/dlopen/configure.m4
+++ b/opal/mca/dl/dlopen/configure.m4
@@ -26,30 +26,16 @@ AC_DEFUN([MCA_opal_dl_dlopen_COMPILE_MODE], [
 AC_DEFUN([MCA_opal_dl_dlopen_CONFIG],[
     AC_CONFIG_FILES([opal/mca/dl/dlopen/Makefile])
 
-    dnl This is effectively a back-door for Open MPI developers to
-    dnl force the use of the libltdl dl component.
-    AC_ARG_ENABLE([dl-dlopen],
-        [AS_HELP_STRING([--disable-dl-dlopen],
-            [Disable the "dlopen" DL component (and probably force the use of the "libltdl" DL component).  This option should really only be used by Open MPI developers.  You are probably actually looking for the "--disable-dlopen" option, which disables all dlopen-like functionality from Open MPI.])
-        ])
-
-    opal_dl_dlopen_happy=no
-    AS_IF([test "$enable_dl_dlopen" != "no"],
-          [OPAL_CHECK_PACKAGE([opal_dl_dlopen],
+    OAC_CHECK_PACKAGE([dlopen],
+              [opal_dl_dlopen],
               [dlfcn.h],
               [dl],
               [dlopen],
-              [],
-              [],
-              [],
               [opal_dl_dlopen_happy=yes],
               [opal_dl_dlopen_happy=no])
-          ])
 
     AS_IF([test "$opal_dl_dlopen_happy" = "yes"],
-          [dl_dlopen_ADD_LIBS=$opal_dl_dlopen_LIBS
-           dl_dlopen_WRAPPER_EXTRA_LIBS=$opal_dl_dlopen_LIBS
-           $1],
+          [$1],
           [$2])
 
     AC_SUBST(opal_dl_dlopen_LIBS)
diff --git a/opal/mca/dl/dlopen/dl_dlopen_component.c b/opal/mca/dl/dlopen/dl_dlopen_component.c
index 97f8872f14e..046878bb9e9 100644
--- a/opal/mca/dl/dlopen/dl_dlopen_component.c
+++ b/opal/mca/dl/dlopen/dl_dlopen_component.c
@@ -18,13 +18,11 @@
 
 #include "dl_dlopen.h"
 
-
 /*
  * Public string showing the sysinfo ompi_linux component version number
  */
-const char *opal_dl_dlopen_component_version_string =
-    "OPAL dl dlopen MCA component version " OPAL_VERSION;
-
+const char *opal_dl_dlopen_component_version_string
+    = "OPAL dl dlopen MCA component version " OPAL_VERSION;
 
 /*
  * Local functions
@@ -42,58 +40,51 @@ static int dlopen_component_query(mca_base_module_t **module, int *priority);
 opal_dl_dlopen_component_t mca_dl_dlopen_component = {
 
     /* Fill in the mca_dl_base_component_t */
-    .base = {
-
-        /* First, the mca_component_t struct containing meta information
-           about the component itself */
-        .base_version = {
-            OPAL_DL_BASE_VERSION_1_0_0,
-
-            /* Component name and version */
-            .mca_component_name = "dlopen",
-            MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
-                                  OPAL_RELEASE_VERSION),
-
-            /* Component functions */
-            .mca_register_component_params = dlopen_component_register,
-            .mca_open_component = dlopen_component_open,
-            .mca_close_component = dlopen_component_close,
-            .mca_query_component = dlopen_component_query,
-        },
-
-        .base_data = {
-            /* The component is checkpoint ready */
-            MCA_BASE_METADATA_PARAM_CHECKPOINT
-        },
-
-        /* The dl framework members */
-        .priority = 80
-    },
+    .base =
+        {
+
+            /* First, the mca_component_t struct containing meta information
+               about the component itself */
+            .base_version =
+                {
+                    OPAL_DL_BASE_VERSION_1_0_0,
+
+                    /* Component name and version */
+                    .mca_component_name = "dlopen",
+                    MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
+                                          OPAL_RELEASE_VERSION),
+
+                    /* Component functions */
+                    .mca_register_component_params = dlopen_component_register,
+                    .mca_open_component = dlopen_component_open,
+                    .mca_close_component = dlopen_component_close,
+                    .mca_query_component = dlopen_component_query,
+                },
+
+            .base_data =
+                {/* The component is checkpoint ready */
+                 MCA_BASE_METADATA_PARAM_CHECKPOINT},
+
+            /* The dl framework members */
+            .priority = 80},
 };
 
-
 static int dlopen_component_register(void)
 {
     int ret;
 
     mca_dl_dlopen_component.filename_suffixes_mca_storage = ".so,.dylib,.dll,.sl";
-    ret =
-        mca_base_component_var_register(&mca_dl_dlopen_component.base.base_version,
-                                        "filename_suffixes",
-                                        "Comma-delimited list of filename suffixes that the dlopen component will try",
-                                        MCA_BASE_VAR_TYPE_STRING,
-                                        NULL,
-                                        0,
-                                        MCA_BASE_VAR_FLAG_SETTABLE,
-                                        OPAL_INFO_LVL_5,
-                                        MCA_BASE_VAR_SCOPE_LOCAL,
-                                        &mca_dl_dlopen_component.filename_suffixes_mca_storage);
+    ret = mca_base_component_var_register(
+        &mca_dl_dlopen_component.base.base_version, "filename_suffixes",
+        "Comma-delimited list of filename suffixes that the dlopen component will try",
+        MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5,
+        MCA_BASE_VAR_SCOPE_LOCAL, &mca_dl_dlopen_component.filename_suffixes_mca_storage);
     if (ret < 0) {
         return ret;
     }
-    mca_dl_dlopen_component.filename_suffixes =
-        opal_argv_split(mca_dl_dlopen_component.filename_suffixes_mca_storage,
-                        ',');
+    mca_dl_dlopen_component.filename_suffixes = opal_argv_split(mca_dl_dlopen_component
+                                                                    .filename_suffixes_mca_storage,
+                                                                ',');
 
     return OPAL_SUCCESS;
 }
@@ -103,7 +94,6 @@ static int dlopen_component_open(void)
     return OPAL_SUCCESS;
 }
 
-
 static int dlopen_component_close(void)
 {
     if (NULL != mca_dl_dlopen_component.filename_suffixes) {
@@ -114,7 +104,6 @@ static int dlopen_component_close(void)
     return OPAL_SUCCESS;
 }
 
-
 static int dlopen_component_query(mca_base_module_t **module, int *priority)
 {
     /* The priority value is somewhat meaningless here; by
diff --git a/opal/mca/dl/dlopen/dl_dlopen_module.c b/opal/mca/dl/dlopen/dl_dlopen_module.c
index 57f392fe046..930e9554b81 100644
--- a/opal/mca/dl/dlopen/dl_dlopen_module.c
+++ b/opal/mca/dl/dlopen/dl_dlopen_module.c
@@ -14,11 +14,11 @@
 
 #include "opal_config.h"
 
-#include <stdlib.h>
-#include <dlfcn.h>
-#include <sys/types.h>
 #include <dirent.h>
+#include <dlfcn.h>
+#include <stdlib.h>
 #include <sys/stat.h>
+#include <sys/types.h>
 #include <unistd.h>
 
 #include "opal/constants.h"
@@ -28,12 +28,10 @@
 
 #include "dl_dlopen.h"
 
-
 /*
  * Trivial helper function to avoid replicating code
  */
-static void do_dlopen(const char *fname, int flags,
-                      void **handle, char **err_msg)
+static void do_dlopen(const char *fname, int flags, void **handle, char **err_msg)
 {
     assert(handle);
 
@@ -48,7 +46,6 @@ static void do_dlopen(const char *fname, int flags,
     }
 }
 
-
 static int dlopen_open(const char *fname, bool use_ext, bool private_namespace,
                        opal_dl_handle_t **handle, char **err_msg)
 {
@@ -71,8 +68,7 @@ static int dlopen_open(const char *fname, bool use_ext, bool private_namespace,
         int i;
         char *ext;
 
-        for (i = 0, ext = mca_dl_dlopen_component.filename_suffixes[i];
-             NULL != ext;
+        for (i = 0, ext = mca_dl_dlopen_component.filename_suffixes[i]; NULL != ext;
              ext = mca_dl_dlopen_component.filename_suffixes[++i]) {
             char *name;
 
@@ -110,10 +106,9 @@ static int dlopen_open(const char *fname, bool use_ext, bool private_namespace,
         (*handle)->dlopen_handle = local_handle;
 
 #if OPAL_ENABLE_DEBUG
-        if( NULL != fname ) {
+        if (NULL != fname) {
             (*handle)->filename = strdup(fname);
-        }
-        else {
+        } else {
             (*handle)->filename = strdup("(null)");
         }
 #endif
@@ -121,9 +116,7 @@ static int dlopen_open(const char *fname, bool use_ext, bool private_namespace,
     return (NULL != local_handle) ? OPAL_SUCCESS : OPAL_ERROR;
 }
 
-
-static int dlopen_lookup(opal_dl_handle_t *handle, const char *symbol,
-                         void **ptr, char **err_msg)
+static int dlopen_lookup(opal_dl_handle_t *handle, const char *symbol, void **ptr, char **err_msg)
 {
     assert(handle);
     assert(handle->dlopen_handle);
@@ -141,7 +134,6 @@ static int dlopen_lookup(opal_dl_handle_t *handle, const char *symbol,
     return OPAL_ERROR;
 }
 
-
 static int dlopen_close(opal_dl_handle_t *handle)
 {
     assert(handle);
@@ -162,8 +154,7 @@ static int dlopen_close(opal_dl_handle_t *handle)
  * on each one.
  */
 static int dlopen_foreachfile(const char *search_path,
-                              int (*func)(const char *filename, void *data),
-                              void *data)
+                              int (*func)(const char *filename, void *data), void *data)
 {
     int ret;
     DIR *dp = NULL;
@@ -209,9 +200,8 @@ static int dlopen_foreachfile(const char *search_path,
             if (NULL != ptr) {
 
                 /* Skip libtool files */
-                if (strcmp(ptr, ".la") == 0 ||
-                    strcmp(ptr, ".lo") == 0) {
-                    free (abs_name);
+                if (strcmp(ptr, ".la") == 0 || strcmp(ptr, ".lo") == 0) {
+                    free(abs_name);
                     continue;
                 }
 
@@ -221,8 +211,7 @@ static int dlopen_foreachfile(const char *search_path,
             /* Have we already found this file?  Or already found a
                file with the same basename (but different suffix)? */
             bool found = false;
-            for (int j = 0; NULL != good_files &&
-                     NULL != good_files[j]; ++j) {
+            for (int j = 0; NULL != good_files && NULL != good_files[j]; ++j) {
                 if (strcmp(good_files[j], abs_name) == 0) {
                     found = true;
                     break;
@@ -250,7 +239,7 @@ static int dlopen_foreachfile(const char *search_path,
 
     ret = OPAL_SUCCESS;
 
- error:
+error:
     if (NULL != dp) {
         closedir(dp);
     }
@@ -264,13 +253,10 @@ static int dlopen_foreachfile(const char *search_path,
     return ret;
 }
 
-
 /*
  * Module definition
  */
-opal_dl_base_module_t opal_dl_dlopen_module = {
-    .open = dlopen_open,
-    .lookup = dlopen_lookup,
-    .close = dlopen_close,
-    .foreachfile = dlopen_foreachfile
-};
+opal_dl_base_module_t opal_dl_dlopen_module = {.open = dlopen_open,
+                                               .lookup = dlopen_lookup,
+                                               .close = dlopen_close,
+                                               .foreachfile = dlopen_foreachfile};
diff --git a/opal/mca/dl/libltdl/Makefile.am b/opal/mca/dl/libltdl/Makefile.am
index 4509829766e..a74f6bb0168 100644
--- a/opal/mca/dl/libltdl/Makefile.am
+++ b/opal/mca/dl/libltdl/Makefile.am
@@ -19,8 +19,8 @@ sources = \
 noinst_LTLIBRARIES = libmca_dl_libltdl.la
 
 libmca_dl_libltdl_la_SOURCES = $(sources)
-libmca_dl_libltdl_la_CPPFLAGS = $(opal_dl_libltdl_CPPFLAGS)
+libmca_dl_libltdl_la_CPPFLAGS = $(dl_libltdl_CPPFLAGS)
 libmca_dl_libltdl_la_LDFLAGS = \
-        $(opal_dl_libltdl_LDFLAGS) \
+        $(dl_libltdl_LDFLAGS) \
         -module -avoid-version
-libmca_dl_libltdl_la_LIBADD = $(opal_dl_libltdl_LIBS)
+libmca_dl_libltdl_la_LIBADD = $(dl_libltdl_LIBS)
diff --git a/opal/mca/dl/libltdl/configure.m4 b/opal/mca/dl/libltdl/configure.m4
index ef69edab90b..da5a813d6c3 100644
--- a/opal/mca/dl/libltdl/configure.m4
+++ b/opal/mca/dl/libltdl/configure.m4
@@ -21,19 +21,6 @@ AC_DEFUN([MCA_opal_dl_libltdl_COMPILE_MODE], [
     AC_MSG_RESULT([$$4])
 ])
 
-# MCA_opal_dl_libltdl_POST_CONFIG()
-# ---------------------------------
-AC_DEFUN([MCA_opal_dl_libltdl_POST_CONFIG],[
-    # If we won, then do all the rest of the setup
-    AS_IF([test "$1" = "1"],
-          [
-           # Add some stuff to CPPFLAGS so that the rest of the source
-           # tree can be built
-           LDFLAGS="$LDFLAGS $opal_dl_libltdl_ADD_LDFLAGS"
-           LIBS="$LIBS $opal_dl_libltdl_ADD_LIBS"
-          ])
-])dnl
-
 # MCA_dl_libltdl_CONFIG([action-if-can-compile],
 #                       [action-if-cant-compile])
 # ------------------------------------------------
@@ -49,41 +36,13 @@ AC_DEFUN([MCA_opal_dl_libltdl_CONFIG],[
        [AS_HELP_STRING([--with-libltdl-libdir=DIR],
              [Search for libltdl libraries in DIR])])
 
-    # Sanity check the --with values
-    OPAL_CHECK_WITHDIR([libltdl], [$with_libltdl],
-                       [include/ltdl.h])
-    OPAL_CHECK_WITHDIR([libltdl-libdir], [$with_libltdl_libdir],
-                       [libltdl.*])
-
-    # Defaults
-    opal_check_libltdl_dir_msg="compiler default"
-    opal_check_libltdl_libdir_msg="linker default"
-
-    # Save directory names if supplied
-    AS_IF([test ! -z "$with_libltdl" && test "$with_libltdl" != "yes"],
-          [opal_check_libltdl_dir=$with_libltdl
-           opal_check_libltdl_dir_msg="$opal_check_libltdl_dir (from --with-libltdl)"])
-    AS_IF([test ! -z "$with_libltdl_libdir" && test "$with_libltdl_libdir" != "yes"],
-          [opal_check_libltdl_libdir=$with_libltdl_libdir
-           opal_check_libltdl_libdir_msg="$opal_check_libltdl_libdir (from --with-libltdl-libdir)"])
-
-    opal_dl_libltdl_happy=no
-    AS_IF([test "$with_libltdl" != "no"],
-          [AC_MSG_CHECKING([for libltdl dir])
-           AC_MSG_RESULT([$opal_check_libltdl_dir_msg])
-           AC_MSG_CHECKING([for libltdl library dir])
-           AC_MSG_RESULT([$opal_check_libltdl_libdir_msg])
-
-           OPAL_CHECK_PACKAGE([opal_dl_libltdl],
-                  [ltdl.h],
-                  [ltdl],
-                  [lt_dlopen],
-                  [],
-                  [$opal_check_libltdl_dir],
-                  [$opal_check_libltdl_libdir],
-                  [opal_dl_libltdl_happy=yes],
-                  [opal_dl_libltdl_happy=no])
-              ])
+    OAC_CHECK_PACKAGE([libltdl],
+                      [dl_libltdl],
+                      [ltdl.h],
+                      [ltdl],
+                      [lt_dlopen],
+                      [opal_dl_libltdl_happy=yes],
+                      [opal_dl_libltdl_happy=no])
 
     # If we have libltdl, do we have lt_dladvise?
     opal_dl_libltdl_have_lt_dladvise=0
@@ -92,9 +51,9 @@ AC_DEFUN([MCA_opal_dl_libltdl_CONFIG],[
            LDFLAGS_save=$LDFLAGS
            LIBS_save=$LIBS
 
-           CPPFLAGS="$opal_dl_libltdl_CPPFLAGS $CPPFLAGS"
-           LDFLAGS="$opal_dl_libltdl_LDFLAGS $LDFLAGS"
-           LIBS="$opal_dl_libltdl_LIBS $LIBS"
+           CPPFLAGS="$dl_libltdl_CPPFLAGS $CPPFLAGS"
+           LDFLAGS="$dl_libltdl_LDFLAGS $LDFLAGS"
+           LIBS="$dl_libltdl_LIBS $LIBS"
            AC_CHECK_FUNC([lt_dladvise_init],
                          [opal_dl_libltdl_have_lt_dladvise=1])
            CPPFLAGS=$CPPFLAGS_save
@@ -106,19 +65,16 @@ AC_DEFUN([MCA_opal_dl_libltdl_CONFIG],[
         [Whether we have lt_dladvise or not])
 
     AS_IF([test "$opal_dl_libltdl_happy" = "yes"],
-          [opal_dl_libltdl_ADD_CPPFLAGS=$opal_dl_libltdl_CPPFLAGS
-           opal_dl_libltdl_ADD_LDFLAGS=$opal_dl_libltdl_LDFLAGS
-           opal_dl_libltdl_ADD_LIBS=$opal_dl_libltdl_LIBS
-           $1],
+          [$1],
           [AS_IF([test ! -z "$with_libltdl" && \
                   test "$with_libltdl" != "no"],
                  [AC_MSG_WARN([Libltdl support requested (via --with-libltdl) but not found.])
                   AC_MSG_ERROR([Cannot continue.])])
            $2])
 
-    AC_SUBST(opal_dl_libltdl_CPPFLAGS)
-    AC_SUBST(opal_dl_libltdl_LDFLAGS)
-    AC_SUBST(opal_dl_libltdl_LIBS)
+    AC_SUBST(dl_libltdl_CPPFLAGS)
+    AC_SUBST(dl_libltdl_LDFLAGS)
+    AC_SUBST(dl_libltdl_LIBS)
 
     OPAL_VAR_SCOPE_POP
 ])
diff --git a/opal/mca/dl/libltdl/dl_libltdl.h b/opal/mca/dl/libltdl/dl_libltdl.h
index c5b5b16788f..93a9c4a87cd 100644
--- a/opal/mca/dl/libltdl/dl_libltdl.h
+++ b/opal/mca/dl/libltdl/dl_libltdl.h
@@ -16,7 +16,6 @@
 
 #include <ltdl.h>
 
-
 OPAL_DECLSPEC extern opal_dl_base_module_t opal_dl_libltdl_module;
 
 /*
diff --git a/opal/mca/dl/libltdl/dl_libltdl_component.c b/opal/mca/dl/libltdl/dl_libltdl_component.c
index 5855bf0a312..a75ba852252 100644
--- a/opal/mca/dl/libltdl/dl_libltdl_component.c
+++ b/opal/mca/dl/libltdl/dl_libltdl_component.c
@@ -13,19 +13,17 @@
 #include "opal_config.h"
 
 #include "opal/constants.h"
-#include "opal/mca/dl/dl.h"
 #include "opal/mca/base/mca_base_var.h"
+#include "opal/mca/dl/dl.h"
 #include "opal/util/argv.h"
 
 #include "dl_libltdl.h"
 
-
 /*
  * Public string showing the sysinfo ompi_linux component version number
  */
-const char *opal_dl_libltdl_component_version_string =
-    "OPAL dl libltdl MCA component version " OPAL_VERSION;
-
+const char *opal_dl_libltdl_component_version_string
+    = "OPAL dl libltdl MCA component version " OPAL_VERSION;
 
 /*
  * Local functions
@@ -43,53 +41,47 @@ static int libltdl_component_query(mca_base_module_t **module, int *priority);
 opal_dl_libltdl_component_t mca_dl_libltdl_component = {
 
     /* Fill in the mca_dl_base_component_t */
-    .base = {
-
-        /* First, the mca_component_t struct containing meta information
-           about the component itself */
-        .base_version = {
-            OPAL_DL_BASE_VERSION_1_0_0,
-
-            /* Component name and version */
-            .mca_component_name = "libltdl",
-            MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
-                                  OPAL_RELEASE_VERSION),
-
-            /* Component functions */
-            .mca_register_component_params = libltdl_component_register,
-            .mca_open_component = libltdl_component_open,
-            .mca_close_component = libltdl_component_close,
-            .mca_query_component = libltdl_component_query,
-        },
-
-        .base_data = {
-            /* The component is checkpoint ready */
-            MCA_BASE_METADATA_PARAM_CHECKPOINT
-        },
-
-        /* The dl framework members */
-        .priority = 50
-    }
+    .base =
+        {
+
+            /* First, the mca_component_t struct containing meta information
+               about the component itself */
+            .base_version =
+                {
+                    OPAL_DL_BASE_VERSION_1_0_0,
+
+                    /* Component name and version */
+                    .mca_component_name = "libltdl",
+                    MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
+                                          OPAL_RELEASE_VERSION),
+
+                    /* Component functions */
+                    .mca_register_component_params = libltdl_component_register,
+                    .mca_open_component = libltdl_component_open,
+                    .mca_close_component = libltdl_component_close,
+                    .mca_query_component = libltdl_component_query,
+                },
+
+            .base_data =
+                {/* The component is checkpoint ready */
+                 MCA_BASE_METADATA_PARAM_CHECKPOINT},
+
+            /* The dl framework members */
+            .priority = 50}
 
     /* Now fill in the libltdl component-specific members */
 };
 
-
 static int libltdl_component_register(void)
 {
     /* Register an info param indicating whether we have lt_dladvise
        support or not */
     bool supported = OPAL_INT_TO_BOOL(OPAL_DL_LIBLTDL_HAVE_LT_DLADVISE);
-    mca_base_component_var_register(&mca_dl_libltdl_component.base.base_version,
-                                    "have_lt_dladvise",
-                                    "Whether the version of libltdl that this component is built against supports lt_dladvise functionality or not",
-                                    MCA_BASE_VAR_TYPE_BOOL,
-                                    NULL,
-                                    0,
-                                    MCA_BASE_VAR_FLAG_DEFAULT_ONLY,
-                                    OPAL_INFO_LVL_7,
-                                    MCA_BASE_VAR_SCOPE_CONSTANT,
-                                    &supported);
+    mca_base_component_var_register(&mca_dl_libltdl_component.base.base_version, "have_lt_dladvise",
+                                    "Whether the version of libltdl that this component is built "
+                                    "against supports lt_dladvise functionality or not",
+                                    MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_DEFAULT_ONLY,
+                                    OPAL_INFO_LVL_7, MCA_BASE_VAR_SCOPE_CONSTANT, &supported);
 
     return OPAL_SUCCESS;
 }
@@ -107,19 +99,16 @@ static int libltdl_component_open(void)
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
-    if (lt_dladvise_init(&c->advise_private_ext) ||
-        lt_dladvise_ext(&c->advise_private_ext)) {
+    if (lt_dladvise_init(&c->advise_private_ext) || lt_dladvise_ext(&c->advise_private_ext)) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
-    if (lt_dladvise_init(&c->advise_public_noext) ||
-        lt_dladvise_global(&c->advise_public_noext)) {
+    if (lt_dladvise_init(&c->advise_public_noext) || lt_dladvise_global(&c->advise_public_noext)) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
-    if (lt_dladvise_init(&c->advise_public_ext) ||
-        lt_dladvise_global(&c->advise_public_ext) ||
-        lt_dladvise_ext(&c->advise_public_ext)) {
+    if (lt_dladvise_init(&c->advise_public_ext) || lt_dladvise_global(&c->advise_public_ext)
+        || lt_dladvise_ext(&c->advise_public_ext)) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 #endif
@@ -127,7 +116,6 @@ static int libltdl_component_open(void)
     return OPAL_SUCCESS;
 }
 
-
 static int libltdl_component_close(void)
 {
 #if OPAL_DL_LIBLTDL_HAVE_LT_DLADVISE
@@ -144,7 +132,6 @@ static int libltdl_component_close(void)
     return OPAL_SUCCESS;
 }
 
-
 static int libltdl_component_query(mca_base_module_t **module, int *priority)
 {
     /* The priority value is somewhat meaningless here; by
diff --git a/opal/mca/dl/libltdl/dl_libltdl_module.c b/opal/mca/dl/libltdl/dl_libltdl_module.c
index 4704d000984..a76f6370884 100644
--- a/opal/mca/dl/libltdl/dl_libltdl_module.c
+++ b/opal/mca/dl/libltdl/dl_libltdl_module.c
@@ -15,9 +15,8 @@
 
 #include "dl_libltdl.h"
 
-
 static int libltdl_open(const char *fname, bool use_ext, bool private_namespace,
-                       opal_dl_handle_t **handle, char **err_msg)
+                        opal_dl_handle_t **handle, char **err_msg)
 {
     assert(handle);
 
@@ -53,10 +52,9 @@ static int libltdl_open(const char *fname, bool use_ext, bool private_namespace,
         (*handle)->ltdl_handle = local_handle;
 
 #if OPAL_ENABLE_DEBUG
-        if( NULL != fname ) {
+        if (NULL != fname) {
             (*handle)->filename = strdup(fname);
-        }
-        else {
+        } else {
             (*handle)->filename = strdup("(null)");
         }
 #endif
@@ -65,14 +63,12 @@ static int libltdl_open(const char *fname, bool use_ext, bool private_namespace,
     }
 
     if (NULL != err_msg) {
-        *err_msg = (char*) lt_dlerror();
+        *err_msg = (char *) lt_dlerror();
     }
     return OPAL_ERROR;
 }
 
-
-static int libltdl_lookup(opal_dl_handle_t *handle, const char *symbol,
-                         void **ptr, char **err_msg)
+static int libltdl_lookup(opal_dl_handle_t *handle, const char *symbol, void **ptr, char **err_msg)
 {
     assert(handle);
     assert(handle->ltdl_handle);
@@ -89,12 +85,11 @@ static int libltdl_lookup(opal_dl_handle_t *handle, const char *symbol,
     }
 
     if (NULL != err_msg) {
-        *err_msg = (char*) lt_dlerror();
+        *err_msg = (char *) lt_dlerror();
     }
     return OPAL_ERROR;
 }
 
-
 static int libltdl_close(opal_dl_handle_t *handle)
 {
     assert(handle);
@@ -111,8 +106,7 @@ static int libltdl_close(opal_dl_handle_t *handle)
 }
 
 static int libltdl_foreachfile(const char *search_path,
-                               int (*func)(const char *filename, void *data),
-                               void *data)
+                               int (*func)(const char *filename, void *data), void *data)
 {
     assert(search_path);
     assert(func);
@@ -121,13 +115,10 @@ static int libltdl_foreachfile(const char *search_path,
     return (0 == ret) ? OPAL_SUCCESS : OPAL_ERROR;
 }
 
-
 /*
  * Module definition
  */
-opal_dl_base_module_t opal_dl_libltdl_module = {
-    .open = libltdl_open,
-    .lookup = libltdl_lookup,
-    .close = libltdl_close,
-    .foreachfile = libltdl_foreachfile
-};
+opal_dl_base_module_t opal_dl_libltdl_module = {.open = libltdl_open,
+                                                .lookup = libltdl_lookup,
+                                                .close = libltdl_close,
+                                                .foreachfile = libltdl_foreachfile};
diff --git a/opal/mca/hwloc/base/Makefile.am b/opal/mca/hwloc/base/Makefile.am
index 29790aac434..73341d53c68 100644
--- a/opal/mca/hwloc/base/Makefile.am
+++ b/opal/mca/hwloc/base/Makefile.am
@@ -1,6 +1,7 @@
 #
 # Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -8,7 +9,7 @@
 # $HEADER$
 #
 
-dist_opaldata_DATA = base/help-opal-hwloc-base.txt
+EXTRA_DIST += base/help-opal-hwloc-base.txt
 
 headers += \
         base/base.h \
@@ -18,4 +19,3 @@ libmca_hwloc_la_SOURCES += \
         base/hwloc_base_frame.c \
         base/hwloc_base_util.c \
         base/hwloc_base_maffinity.c
-
diff --git a/opal/mca/hwloc/base/base.h b/opal/mca/hwloc/base/base.h
index 80fa9ab356a..299f2992599 100644
--- a/opal/mca/hwloc/base/base.h
+++ b/opal/mca/hwloc/base/base.h
@@ -19,9 +19,9 @@
 #include "opal/mca/hwloc/hwloc-internal.h"
 
 #if HWLOC_API_VERSION < 0x20000
-#define HWLOC_OBJ_L3CACHE HWLOC_OBJ_CACHE
-#define HWLOC_OBJ_L2CACHE HWLOC_OBJ_CACHE
-#define HWLOC_OBJ_L1CACHE HWLOC_OBJ_CACHE
+#    define HWLOC_OBJ_L3CACHE HWLOC_OBJ_CACHE
+#    define HWLOC_OBJ_L2CACHE HWLOC_OBJ_CACHE
+#    define HWLOC_OBJ_L1CACHE HWLOC_OBJ_CACHE
 #endif
 
 /*
@@ -37,12 +37,9 @@ BEGIN_C_DECLS
  * opal_hwloc_topology, nor does it set the process-wide memory
  * affinity policy.  Filling opal_hwloc_topology via
  * hwloc_topology_load() can be expensive (and/or serialized by the
- * OS); it may not be desireable to call this function in every MPI
+ * OS); it may not be desirable to call this function in every MPI
  * process on a machine.  Hence, it is the responsibility for an upper
- * layer to both fill opal_hwloc_topology in some scalable way, as
- * well as to invoke opal_hwloc_base_set_process_membind_policy()
- * (after opal_hwloc_topology has been loaded) to set the process-wide
- * memory affinity policy.
+ * layer to both fill opal_hwloc_topology in some scalable way.
  */
 
 /**
@@ -53,91 +50,6 @@ OPAL_DECLSPEC extern bool opal_hwloc_topology_inited;
 
 OPAL_DECLSPEC extern mca_base_framework_t opal_hwloc_base_framework;
 
-/* we always must have some minimal locality support */
-#define OPAL_HWLOC_PRINT_MAX_SIZE   50
-#define OPAL_HWLOC_PRINT_NUM_BUFS   16
-typedef struct {
-    char *buffers[OPAL_HWLOC_PRINT_NUM_BUFS];
-    int cntr;
-} opal_hwloc_print_buffers_t;
-opal_hwloc_print_buffers_t *opal_hwloc_get_print_buffer(void);
-extern char* opal_hwloc_print_null;
-OPAL_DECLSPEC char* opal_hwloc_base_print_locality(opal_hwloc_locality_t locality);
-
-OPAL_DECLSPEC extern char *opal_hwloc_base_cpu_list;
-OPAL_DECLSPEC extern hwloc_cpuset_t opal_hwloc_base_given_cpus;
-OPAL_DECLSPEC extern char *opal_hwloc_base_topo_file;
-
-/* convenience macro for debugging */
-#define OPAL_HWLOC_SHOW_BINDING(n, v, t)                                \
-    do {                                                                \
-        char tmp1[1024];                                                \
-        hwloc_cpuset_t bind;                                            \
-        bind = opal_hwloc_alloc();                                      \
-        if (hwloc_get_cpubind(t, bind,                                  \
-                              HWLOC_CPUBIND_PROCESS) < 0) {             \
-            opal_output_verbose(n, v,                                   \
-                                "CANNOT DETERMINE BINDING AT %s:%d",    \
-                                __FILE__, __LINE__);                    \
-        } else {                                                        \
-            opal_hwloc_base_cset2mapstr(tmp1, sizeof(tmp1), t, bind);   \
-            opal_output_verbose(n, v,                                   \
-                                "BINDINGS AT %s:%d: %s",                \
-                                __FILE__, __LINE__, tmp1);              \
-        }                                                               \
-        hwloc_bitmap_free(bind);                                        \
-    } while(0);
-
-#if HWLOC_API_VERSION < 0x20000
-#define OPAL_HWLOC_MAKE_OBJ_CACHE(level, obj, cache_level)              \
-    do {                                                                \
-        obj = HWLOC_OBJ_CACHE;                                          \
-        cache_level = level;                                            \
-    } while(0)
-#else
-#define OPAL_HWLOC_MAKE_OBJ_CACHE(level, obj, cache_level)              \
-    do {                                                                \
-        obj = HWLOC_OBJ_L##level##CACHE;                                \
-        cache_level = 0;                                                \
-    } while(0)
-#endif
-
-OPAL_DECLSPEC opal_hwloc_locality_t opal_hwloc_base_get_relative_locality(hwloc_topology_t topo,
-                                                                          char *cpuset1, char *cpuset2);
-
-OPAL_DECLSPEC int opal_hwloc_base_set_binding_policy(opal_binding_policy_t *policy, char *spec);
-
-/**
- * Loads opal_hwloc_my_cpuset (global variable in
- * opal/mca/hwloc/hwloc-internal.h) for this process.  opal_hwloc_my_cpuset
- * will be loaded with this process' binding, or, if the process is
- * not bound, use the hwloc root object's (available and online)
- * cpuset.
- */
-OPAL_DECLSPEC void opal_hwloc_base_get_local_cpuset(void);
-
-struct opal_rmaps_numa_node_t {
-    opal_list_item_t super;
-    int index;
-    float dist_from_closed;
-};
-typedef struct opal_rmaps_numa_node_t opal_rmaps_numa_node_t;
-OBJ_CLASS_DECLARATION(opal_rmaps_numa_node_t);
-
-/**
- * Enum for what memory allocation policy we want for user allocations.
- * MAP = memory allocation policy.
- */
-typedef enum {
-    OPAL_HWLOC_BASE_MAP_NONE,
-    OPAL_HWLOC_BASE_MAP_LOCAL_ONLY
-} opal_hwloc_base_map_t;
-
-/**
- * Global reflecting the MAP (set by MCA param).
- */
-OPAL_DECLSPEC extern opal_hwloc_base_map_t opal_hwloc_base_map;
-
 /**
  * Enum for what to do if the hwloc framework tries to bind memory
  * and fails.  BFA = bind failure action.
@@ -148,28 +60,12 @@ typedef enum {
     OPAL_HWLOC_BASE_MBFA_ERROR
 } opal_hwloc_base_mbfa_t;
 
-/**
- * Global reflecting the BFA (set by MCA param).
- */
-OPAL_DECLSPEC extern opal_hwloc_base_mbfa_t opal_hwloc_base_mbfa;
-
 /**
  * Discover / load the hwloc topology (i.e., call hwloc_topology_init() and
  * hwloc_topology_load()).
  */
 OPAL_DECLSPEC int opal_hwloc_base_get_topology(void);
 
-/**
- * Set the hwloc topology to that from the given topo file
- */
-OPAL_DECLSPEC int opal_hwloc_base_set_topology(char *topofile);
-
-OPAL_DECLSPEC int opal_hwloc_base_filter_cpus(hwloc_topology_t topo);
-
-/**
- * Free the hwloc topology.
- */
-OPAL_DECLSPEC void opal_hwloc_base_free_topology(hwloc_topology_t topo);
 OPAL_DECLSPEC unsigned int opal_hwloc_base_get_nbobjs_by_type(hwloc_topology_t topo,
                                                               hwloc_obj_type_t target,
                                                               unsigned cache_level,
@@ -181,77 +77,18 @@ OPAL_DECLSPEC hwloc_obj_t opal_hwloc_base_get_obj_by_type(hwloc_topology_t topo,
                                                           unsigned int instance,
                                                           opal_hwloc_resource_type_t rtype);
 
-OPAL_DECLSPEC char* opal_hwloc_base_print_binding(opal_binding_policy_t binding);
-
-/**
- * Determine if there is a single cpu in a bitmap.
- */
-OPAL_DECLSPEC bool opal_hwloc_base_single_cpu(hwloc_cpuset_t cpuset);
-
-/**
- * Report a bind failure using the normal mechanisms if a component
- * fails to bind memory -- according to the value of the
- * hwloc_base_bind_failure_action MCA parameter.
- */
-OPAL_DECLSPEC int opal_hwloc_base_report_bind_failure(const char *file,
-                                                      int line,
-                                                      const char *msg,
-                                                      int rc);
-
-/**
- * This function sets the process-wide memory affinity policy
- * according to opal_hwloc_base_map and opal_hwloc_base_mbfa.  It needs
- * to be a separate, standalone function (as opposed to being done
- * during opal_hwloc_base_open()) because opal_hwloc_topology is not
- * loaded by opal_hwloc_base_open().  Hence, an upper layer needs to
- * invoke this function after opal_hwloc_topology has been loaded.
- */
-OPAL_DECLSPEC int opal_hwloc_base_set_process_membind_policy(void);
-
-OPAL_DECLSPEC int opal_hwloc_base_membind(opal_hwloc_base_memory_segment_t *segs,
-                                          size_t count, int node_id);
-
-OPAL_DECLSPEC int opal_hwloc_base_node_name_to_id(char *node_name, int *id);
+OPAL_DECLSPEC int opal_hwloc_base_membind(opal_hwloc_base_memory_segment_t *segs, size_t count,
+                                          int node_id);
 
 OPAL_DECLSPEC int opal_hwloc_base_memory_set(opal_hwloc_base_memory_segment_t *segments,
                                              size_t num_segments);
 
-/**
- * Make a prettyprint string for a hwloc_cpuset_t (e.g., "socket
- * 2[core 3]").
- */
-OPAL_DECLSPEC int opal_hwloc_base_cset2str(char *str, int len,
-                                           hwloc_topology_t topo,
-                                           hwloc_cpuset_t cpuset);
-
-/**
- * Make a prettyprint string for a cset in a map format.
- * Example: [B./..]
- * Key:  [] - signifies socket
- *        / - divider between cores
- *        . - signifies PU a process not bound to
- *        B - signifies PU a process is bound to
- */
-OPAL_DECLSPEC int opal_hwloc_base_cset2mapstr(char *str, int len,
-                                              hwloc_topology_t topo,
-                                              hwloc_cpuset_t cpuset);
-
-/* get the hwloc object that corresponds to the given processor id  and type */
-OPAL_DECLSPEC hwloc_obj_t opal_hwloc_base_get_pu(hwloc_topology_t topo,
-                                                 int lid,
-                                                 opal_hwloc_resource_type_t rtype);
-
-/* get a string describing the locality of a given process */
-OPAL_DECLSPEC char* opal_hwloc_base_get_locality_string(hwloc_topology_t topo, char *bitmap);
-
 /* extract a location from the locality string */
-OPAL_DECLSPEC char* opal_hwloc_base_get_location(char *locality,
-                                                 hwloc_obj_type_t type,
+OPAL_DECLSPEC char *opal_hwloc_base_get_location(char *locality, hwloc_obj_type_t type,
                                                  unsigned index);
 
 OPAL_DECLSPEC opal_hwloc_locality_t opal_hwloc_compute_relative_locality(char *loc1, char *loc2);
 
-OPAL_DECLSPEC int opal_hwloc_base_topology_set_flags (hwloc_topology_t topology, unsigned long flags, bool io);
 END_C_DECLS
 
 #endif /* OPAL_HWLOC_BASE_H */
diff --git a/opal/mca/hwloc/base/help-opal-hwloc-base.txt b/opal/mca/hwloc/base/help-opal-hwloc-base.txt
index a2c6af0c444..f105de9964e 100644
--- a/opal/mca/hwloc/base/help-opal-hwloc-base.txt
+++ b/opal/mca/hwloc/base/help-opal-hwloc-base.txt
@@ -19,43 +19,4 @@ message will only be reported at most once per process.
   File:       %s:%d
   Message:    %s
   Severity:   %s
-#
-[invalid binding_policy]
-The specified %s policy is not recognized:
-
-  Policy: %s
-
-Please check for a typo or ensure that the option is a supported
-one.
-#
-[redefining-policy]
-Conflicting directives for binding policy are causing the policy
-to be redefined:
-
-  New policy:   %s
-  Prior policy:  %s
-
-Please check that only one policy is defined.
-#
-[deprecated]
-The following command line option and corresponding MCA parameter have
-been deprecated and replaced as follows:
-
-  Command line option:
-    Deprecated:  %s
-    Replacement: %s
-
-  Equivalent MCA parameter:
-    Deprecated:  %s
-    Replacement: %s
-
-The deprecated forms *will* disappear in a future version of Open MPI.
-Please update to the new syntax.
-#
-[obj-idx-failed]
-Open MPI failed to find a cache of a specified type.  This is a highly
-unusual error; it may indicate a system configuration error.  This
-additional information may be of help:
 
-  Message:     %s
-  Cache level: %d
diff --git a/opal/mca/hwloc/base/hwloc_base_frame.c b/opal/mca/hwloc/base/hwloc_base_frame.c
index ee69bc6306e..2ac049f3523 100644
--- a/opal/mca/hwloc/base/hwloc_base_frame.c
+++ b/opal/mca/hwloc/base/hwloc_base_frame.c
@@ -12,20 +12,17 @@
  * $HEADER$
  */
 
-
 #include "opal_config.h"
 
 #include "opal/constants.h"
-#include "opal/util/argv.h"
-#include "opal/util/output.h"
-#include "opal/util/show_help.h"
-#include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
+#include "opal/mca/mca.h"
 #include "opal/mca/threads/tsd.h"
+#include "opal/util/argv.h"
+#include "opal/util/output.h"
 
-#include "opal/mca/hwloc/hwloc-internal.h"
 #include "opal/mca/hwloc/base/base.h"
-
+#include "opal/mca/hwloc/hwloc-internal.h"
 
 /*
  * The following file was created by configure.  It contains extern
@@ -34,217 +31,114 @@
  */
 #include "opal/mca/hwloc/base/static-components.h"
 
-
 /*
  * Globals
  */
 bool opal_hwloc_base_inited = false;
-hwloc_topology_t opal_hwloc_topology=NULL;
-hwloc_cpuset_t opal_hwloc_my_cpuset=NULL;
-hwloc_cpuset_t opal_hwloc_base_given_cpus=NULL;
-opal_hwloc_base_map_t opal_hwloc_base_map = OPAL_HWLOC_BASE_MAP_NONE;
+hwloc_topology_t opal_hwloc_topology = NULL;
+hwloc_cpuset_t opal_hwloc_my_cpuset = NULL;
 opal_hwloc_base_mbfa_t opal_hwloc_base_mbfa = OPAL_HWLOC_BASE_MBFA_WARN;
-opal_binding_policy_t opal_hwloc_binding_policy=0;
-char *opal_hwloc_base_cpu_list=NULL;
-bool opal_hwloc_report_bindings=false;
-hwloc_obj_type_t opal_hwloc_levels[] = {
-    HWLOC_OBJ_MACHINE,
-    HWLOC_OBJ_NODE,
-    HWLOC_OBJ_SOCKET,
-    HWLOC_OBJ_L3CACHE,
-    HWLOC_OBJ_L2CACHE,
-    HWLOC_OBJ_L1CACHE,
-    HWLOC_OBJ_CORE,
-    HWLOC_OBJ_PU
-};
-bool opal_hwloc_use_hwthreads_as_cpus = false;
-char *opal_hwloc_base_topo_file = NULL;
 
-static mca_base_var_enum_value_t hwloc_base_map[] = {
-    {OPAL_HWLOC_BASE_MAP_NONE, "none"},
-    {OPAL_HWLOC_BASE_MAP_LOCAL_ONLY, "local_only"},
-    {0, NULL}
-};
+extern bool opal_hwloc_topo_in_shmem;
 
-static mca_base_var_enum_value_t hwloc_failure_action[] = {
-    {OPAL_HWLOC_BASE_MBFA_SILENT, "silent"},
-    {OPAL_HWLOC_BASE_MBFA_WARN, "warn"},
-    {OPAL_HWLOC_BASE_MBFA_ERROR, "error"},
-    {0, NULL}
-};
+static mca_base_var_enum_value_t hwloc_failure_action[] = {{OPAL_HWLOC_BASE_MBFA_SILENT, "silent"},
+                                                           {OPAL_HWLOC_BASE_MBFA_WARN, "warn"},
+                                                           {OPAL_HWLOC_BASE_MBFA_ERROR, "error"},
+                                                           {0, NULL}};
 
 static int opal_hwloc_base_register(mca_base_register_flag_t flags);
 static int opal_hwloc_base_open(mca_base_open_flag_t flags);
 static int opal_hwloc_base_close(void);
 
-MCA_BASE_FRAMEWORK_DECLARE(opal, hwloc, NULL, opal_hwloc_base_register, opal_hwloc_base_open, opal_hwloc_base_close,
-                           mca_hwloc_base_static_components, 0);
-
-static char *opal_hwloc_base_binding_policy = NULL;
-static bool opal_hwloc_base_bind_to_core = false;
-static bool opal_hwloc_base_bind_to_socket = false;
+MCA_BASE_FRAMEWORK_DECLARE(opal, hwloc, NULL, opal_hwloc_base_register, opal_hwloc_base_open,
+                           opal_hwloc_base_close, mca_hwloc_base_static_components, 0);
 
 static int opal_hwloc_base_register(mca_base_register_flag_t flags)
 {
     mca_base_var_enum_t *new_enum;
-    int ret, varid;
-
-    /* hwloc_base_mbind_policy */
-
-    opal_hwloc_base_map = OPAL_HWLOC_BASE_MAP_NONE;
-    mca_base_var_enum_create("hwloc memory allocation policy", hwloc_base_map, &new_enum);
-    ret = mca_base_var_register("opal", "hwloc", "base", "mem_alloc_policy",
-                                "General memory allocations placement policy (this is not memory binding). "
-                                "\"none\" means that no memory policy is applied. \"local_only\" means that a process' memory allocations will be restricted to its local NUMA node. "
-                                "If using direct launch, this policy will not be in effect until after MPI_INIT. "
-                                "Note that operating system paging policies are unaffected by this setting. For example, if \"local_only\" is used and local NUMA node memory is exhausted, a new memory allocation may cause paging.",
-                                MCA_BASE_VAR_TYPE_INT, new_enum, 0, 0, OPAL_INFO_LVL_9,
-                                MCA_BASE_VAR_SCOPE_READONLY, &opal_hwloc_base_map);
-    OBJ_RELEASE(new_enum);
-    if (0 > ret) {
-        return ret;
-    }
+    int ret;
 
     /* hwloc_base_bind_failure_action */
     opal_hwloc_base_mbfa = OPAL_HWLOC_BASE_MBFA_WARN;
     mca_base_var_enum_create("hwloc memory bind failure action", hwloc_failure_action, &new_enum);
-    ret = mca_base_var_register("opal", "hwloc", "base", "mem_bind_failure_action",
-                                "What Open MPI will do if it explicitly tries to bind memory to a specific NUMA location, and fails.  Note that this is a different case than the general allocation policy described by hwloc_base_alloc_policy.  A value of \"silent\" means that Open MPI will proceed without comment. A value of \"warn\" means that Open MPI will warn the first time this happens, but allow the job to continue (possibly with degraded performance).  A value of \"error\" means that Open MPI will abort the job if this happens.",
-                                MCA_BASE_VAR_TYPE_INT, new_enum, 0, 0, OPAL_INFO_LVL_9,
-                                MCA_BASE_VAR_SCOPE_READONLY, &opal_hwloc_base_mbfa);
+    ret = mca_base_var_register(
+        "opal", "hwloc", "base", "mem_bind_failure_action",
+        "What Open MPI will do if it explicitly tries to bind memory to a specific NUMA location, "
+        "and fails.  Note that this is a different case than the general allocation policy "
+        "described by hwloc_base_alloc_policy.  A value of \"silent\" means that Open MPI will "
+        "proceed without comment. A value of \"warn\" means that Open MPI will warn the first time "
+        "this happens, but allow the job to continue (possibly with degraded performance).  A "
+        "value of \"error\" means that Open MPI will abort the job if this happens.",
+        MCA_BASE_VAR_TYPE_INT, new_enum, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+        &opal_hwloc_base_mbfa);
     OBJ_RELEASE(new_enum);
     if (0 > ret) {
         return ret;
     }
 
-    opal_hwloc_base_binding_policy = NULL;
-    (void) mca_base_var_register("opal", "hwloc", "base", "binding_policy",
-                                 "Policy for binding processes. Allowed values: none, hwthread, core, l1cache, l2cache, "
-                                 "l3cache, socket, numa, board, cpu-list (\"none\" is the default when oversubscribed, \"core\" is "
-                                 "the default when np<=2, and \"numa\" is the default when np>2). Allowed qualifiers: "
-                                 "overload-allowed, if-supported, ordered",
-                                 MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_9,
-                                 MCA_BASE_VAR_SCOPE_READONLY, &opal_hwloc_base_binding_policy);
-
-    /* backward compatibility */
-    opal_hwloc_base_bind_to_core = false;
-    (void) mca_base_var_register("opal", "hwloc", "base", "bind_to_core", "Bind processes to cores",
-                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9,
-                                 MCA_BASE_VAR_SCOPE_READONLY, &opal_hwloc_base_bind_to_core);
 
-    opal_hwloc_base_bind_to_socket = false;
-    (void) mca_base_var_register("opal", "hwloc", "base", "bind_to_socket", "Bind processes to sockets",
-                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9,
-                                 MCA_BASE_VAR_SCOPE_READONLY, &opal_hwloc_base_bind_to_socket);
-
-    opal_hwloc_report_bindings = false;
-    (void) mca_base_var_register("opal", "hwloc", "base", "report_bindings", "Report bindings to stderr",
-                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9,
-                                 MCA_BASE_VAR_SCOPE_READONLY, &opal_hwloc_report_bindings);
-
-    opal_hwloc_base_cpu_list = NULL;
-    varid = mca_base_var_register("opal", "hwloc", "base", "cpu_list",
-                                  "Comma-separated list of ranges specifying logical cpus to be used by these processes [default: none]",
-                                  MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_9,
-                                  MCA_BASE_VAR_SCOPE_READONLY, &opal_hwloc_base_cpu_list);
-    mca_base_var_register_synonym (varid, "opal", "hwloc", "base", "slot_list", MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
-    mca_base_var_register_synonym (varid, "opal", "hwloc", "base", "cpu_set", MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
-
-    /* declare hwthreads as independent cpus */
-    opal_hwloc_use_hwthreads_as_cpus = false;
-    (void) mca_base_var_register("opal", "hwloc", "base", "use_hwthreads_as_cpus",
-                                 "Use hardware threads as independent cpus",
-                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9,
-                                 MCA_BASE_VAR_SCOPE_READONLY, &opal_hwloc_use_hwthreads_as_cpus);
-
-    opal_hwloc_base_topo_file = NULL;
-    (void) mca_base_var_register("opal", "hwloc", "base", "topo_file",
-                                 "Read local topology from file instead of directly sensing it",
-                                 MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_9,
-                                 MCA_BASE_VAR_SCOPE_READONLY, &opal_hwloc_base_topo_file);
-
-    /* register parameters */
     return OPAL_SUCCESS;
 }
 
 static int opal_hwloc_base_open(mca_base_open_flag_t flags)
 {
-    int rc;
-
     if (opal_hwloc_base_inited) {
         return OPAL_SUCCESS;
     }
     opal_hwloc_base_inited = true;
 
-    if (OPAL_SUCCESS != (rc = opal_hwloc_base_set_binding_policy(&opal_hwloc_binding_policy,
-                                                                 opal_hwloc_base_binding_policy))) {
-        return rc;
+   /* to support tools such as ompi_info, add the components
+     * to a list
+     */
+    if (OPAL_SUCCESS != mca_base_framework_components_open(&opal_hwloc_base_framework, flags)) {
+        return OPAL_ERROR;
     }
 
-    if (opal_hwloc_base_bind_to_core) {
-        opal_show_help("help-opal-hwloc-base.txt", "deprecated", true,
-                       "--bind-to-core", "--bind-to core",
-                       "hwloc_base_bind_to_core", "hwloc_base_binding_policy=core");
-        /* set binding policy to core - error if something else already set */
-        if (OPAL_BINDING_POLICY_IS_SET(opal_hwloc_binding_policy) &&
-            OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy) != OPAL_BIND_TO_CORE) {
-            /* error - cannot redefine the default ranking policy */
-            opal_show_help("help-opal-hwloc-base.txt", "redefining-policy", true,
-                           "core", opal_hwloc_base_print_binding(opal_hwloc_binding_policy));
-            return OPAL_ERR_BAD_PARAM;
-        }
-        OPAL_SET_BINDING_POLICY(opal_hwloc_binding_policy, OPAL_BIND_TO_CORE);
-    }
+    return OPAL_SUCCESS;
+}
 
-    if (opal_hwloc_base_bind_to_socket) {
-        opal_show_help("help-opal-hwloc-base.txt", "deprecated", true,
-                       "--bind-to-socket", "--bind-to socket",
-                       "hwloc_base_bind_to_socket", "hwloc_base_binding_policy=socket");
-        /* set binding policy to socket - error if something else already set */
-        if (OPAL_BINDING_POLICY_IS_SET(opal_hwloc_binding_policy) &&
-            OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy) != OPAL_BIND_TO_SOCKET) {
-            /* error - cannot redefine the default ranking policy */
-            opal_show_help("help-opal-hwloc-base.txt", "redefining-policy", true,
-                           "socket", opal_hwloc_base_print_binding(opal_hwloc_binding_policy));
-            return OPAL_ERR_SILENT;
-        }
-        OPAL_SET_BINDING_POLICY(opal_hwloc_binding_policy, OPAL_BIND_TO_SOCKET);
-    }
+static void free_object(hwloc_obj_t obj)
+{
+    opal_hwloc_obj_data_t *data;
+    unsigned k;
 
-    /* did the user provide a slot list? */
-    if (NULL != opal_hwloc_base_cpu_list) {
-        /* it is okay if a binding policy was already given - just ensure that
-         * we do bind to the given cpus if provided, otherwise this would be
-         * ignored if someone didn't also specify a binding policy
-         */
-// Restoring pre ef86707fbe3392c8ed15f79cc4892f0313b409af behavior.
-// Formerly -cpu-set #,#,# along with -use_hwthread-cpus resulted
-// in the binding policy staying OPAL_BIND_TO_HWTHREAD
-// I think that should be right because I thought -cpu-set was a contraint you put
-// on another binding policy, not a binding policy in itself.
-        if (!OPAL_BINDING_POLICY_IS_SET(opal_hwloc_binding_policy)) {
-            OPAL_SET_BINDING_POLICY(opal_hwloc_binding_policy, OPAL_BIND_TO_CPUSET);
-        }
+    /* free any data hanging on this object */
+    if (NULL != obj->userdata) {
+        data = (opal_hwloc_obj_data_t *) obj->userdata;
+        OBJ_RELEASE(data);
+        obj->userdata = NULL;
     }
 
-    /* if we are binding to hwthreads, then we must use hwthreads as cpus */
-    if (OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy) == OPAL_BIND_TO_HWTHREAD) {
-        opal_hwloc_use_hwthreads_as_cpus = true;
+    /* loop thru our children */
+    for (k = 0; k < obj->arity; k++) {
+        free_object(obj->children[k]);
     }
+}
 
-    /* to support tools such as ompi_info, add the components
-     * to a list
-     */
-    if (OPAL_SUCCESS !=
-        mca_base_framework_components_open(&opal_hwloc_base_framework, flags)) {
-        return OPAL_ERROR;
+static void free_topology(hwloc_topology_t topo)
+{
+    hwloc_obj_t obj;
+    opal_hwloc_topo_data_t *rdata;
+    unsigned k;
+
+    if (!opal_hwloc_topo_in_shmem) {
+        obj = hwloc_get_root_obj(topo);
+        /* release the root-level userdata */
+        if (NULL != obj->userdata) {
+            rdata = (opal_hwloc_topo_data_t *) obj->userdata;
+            OBJ_RELEASE(rdata);
+            obj->userdata = NULL;
+        }
+        /* now recursively descend and release userdata
+         * in the rest of the objects
+         */
+        for (k = 0; k < obj->arity; k++) {
+            free_object(obj->children[k]);
+        }
     }
-
-    return OPAL_SUCCESS;
+    hwloc_topology_destroy(topo);
 }
 
-static opal_tsd_tracked_key_t *print_tsd_key = NULL;
-
 static int opal_hwloc_base_close(void)
 {
     int ret;
@@ -252,15 +146,10 @@ static int opal_hwloc_base_close(void)
         return OPAL_SUCCESS;
     }
 
-    if (NULL != print_tsd_key) {
-        OBJ_RELEASE(print_tsd_key);
-        print_tsd_key = NULL;
-    }
-
     /* no need to close the component as it was statically opened */
 
     /* for support of tools such as ompi_info */
-    ret = mca_base_framework_components_close (&opal_hwloc_base_framework, NULL);
+    ret = mca_base_framework_components_close(&opal_hwloc_base_framework, NULL);
     if (OPAL_SUCCESS != ret) {
         return ret;
     }
@@ -273,146 +162,15 @@ static int opal_hwloc_base_close(void)
 
     /* destroy the topology */
     if (NULL != opal_hwloc_topology) {
-        opal_hwloc_base_free_topology(opal_hwloc_topology);
+        free_topology(opal_hwloc_topology);
         opal_hwloc_topology = NULL;
     }
 
-
     /* All done */
     opal_hwloc_base_inited = false;
     return OPAL_SUCCESS;
 }
 
-static bool fns_init=false;
-char* opal_hwloc_print_null = "NULL";
-
-static void buffer_cleanup(void *value)
-{
-    int i;
-    opal_hwloc_print_buffers_t *ptr;
-
-    if (NULL != value) {
-        ptr = (opal_hwloc_print_buffers_t*)value;
-        for (i=0; i < OPAL_HWLOC_PRINT_NUM_BUFS; i++) {
-            free(ptr->buffers[i]);
-        }
-        free(ptr);
-    }
-}
-
-opal_hwloc_print_buffers_t *opal_hwloc_get_print_buffer(void)
-{
-    opal_hwloc_print_buffers_t *ptr;
-    int ret, i;
-
-    if (!fns_init) {
-        /* setup the print_args function */
-        print_tsd_key = OBJ_NEW(opal_tsd_tracked_key_t);
-        opal_tsd_tracked_key_set_destructor(print_tsd_key, buffer_cleanup);
-        fns_init = true;
-    }
-
-    ret = opal_tsd_tracked_key_get(print_tsd_key, (void**)&ptr);
-    if (OPAL_SUCCESS != ret) return NULL;
-
-    if (NULL == ptr) {
-        ptr = (opal_hwloc_print_buffers_t*)malloc(sizeof(opal_hwloc_print_buffers_t));
-        for (i=0; i < OPAL_HWLOC_PRINT_NUM_BUFS; i++) {
-            ptr->buffers[i] = (char *) malloc((OPAL_HWLOC_PRINT_MAX_SIZE+1) * sizeof(char));
-        }
-        ptr->cntr = 0;
-        ret = opal_tsd_tracked_key_set(print_tsd_key, (void*)ptr);
-    }
-
-    return (opal_hwloc_print_buffers_t*) ptr;
-}
-
-char* opal_hwloc_base_print_locality(opal_hwloc_locality_t locality)
-{
-    opal_hwloc_print_buffers_t *ptr;
-    int idx;
-
-    ptr = opal_hwloc_get_print_buffer();
-    if (NULL == ptr) {
-        return opal_hwloc_print_null;
-    }
-    /* cycle around the ring */
-    if (OPAL_HWLOC_PRINT_NUM_BUFS == ptr->cntr) {
-        ptr->cntr = 0;
-    }
-
-    idx = 0;
-
-    if (OPAL_PROC_ON_LOCAL_CLUSTER(locality)) {
-        ptr->buffers[ptr->cntr][idx++] = 'C';
-        ptr->buffers[ptr->cntr][idx++] = 'L';
-        ptr->buffers[ptr->cntr][idx++] = ':';
-    }
-    if (OPAL_PROC_ON_LOCAL_CU(locality)) {
-        ptr->buffers[ptr->cntr][idx++] = 'C';
-        ptr->buffers[ptr->cntr][idx++] = 'U';
-        ptr->buffers[ptr->cntr][idx++] = ':';
-    }
-    if (OPAL_PROC_ON_LOCAL_NODE(locality)) {
-        ptr->buffers[ptr->cntr][idx++] = 'N';
-        ptr->buffers[ptr->cntr][idx++] = ':';
-    }
-    if (OPAL_PROC_ON_LOCAL_BOARD(locality)) {
-        ptr->buffers[ptr->cntr][idx++] = 'B';
-        ptr->buffers[ptr->cntr][idx++] = ':';
-    }
-    if (OPAL_PROC_ON_LOCAL_NUMA(locality)) {
-        ptr->buffers[ptr->cntr][idx++] = 'N';
-        ptr->buffers[ptr->cntr][idx++] = 'u';
-        ptr->buffers[ptr->cntr][idx++] = ':';
-    }
-    if (OPAL_PROC_ON_LOCAL_SOCKET(locality)) {
-        ptr->buffers[ptr->cntr][idx++] = 'S';
-        ptr->buffers[ptr->cntr][idx++] = ':';
-    }
-    if (OPAL_PROC_ON_LOCAL_L3CACHE(locality)) {
-        ptr->buffers[ptr->cntr][idx++] = 'L';
-        ptr->buffers[ptr->cntr][idx++] = '3';
-        ptr->buffers[ptr->cntr][idx++] = ':';
-    }
-    if (OPAL_PROC_ON_LOCAL_L2CACHE(locality)) {
-        ptr->buffers[ptr->cntr][idx++] = 'L';
-        ptr->buffers[ptr->cntr][idx++] = '2';
-        ptr->buffers[ptr->cntr][idx++] = ':';
-    }
-    if (OPAL_PROC_ON_LOCAL_L1CACHE(locality)) {
-        ptr->buffers[ptr->cntr][idx++] = 'L';
-        ptr->buffers[ptr->cntr][idx++] = '1';
-        ptr->buffers[ptr->cntr][idx++] = ':';
-    }
-    if (OPAL_PROC_ON_LOCAL_CORE(locality)) {
-        ptr->buffers[ptr->cntr][idx++] = 'C';
-        ptr->buffers[ptr->cntr][idx++] = ':';
-    }
-    if (OPAL_PROC_ON_LOCAL_HWTHREAD(locality)) {
-        ptr->buffers[ptr->cntr][idx++] = 'H';
-        ptr->buffers[ptr->cntr][idx++] = 'w';
-        ptr->buffers[ptr->cntr][idx++] = 't';
-        ptr->buffers[ptr->cntr][idx++] = ':';
-    }
-    if (0 < idx) {
-        ptr->buffers[ptr->cntr][idx-1] = '\0';
-    } else if (OPAL_PROC_NON_LOCAL & locality) {
-        ptr->buffers[ptr->cntr][idx++] = 'N';
-        ptr->buffers[ptr->cntr][idx++] = 'O';
-        ptr->buffers[ptr->cntr][idx++] = 'N';
-        ptr->buffers[ptr->cntr][idx++] = '\0';
-    } else {
-        /* must be an unknown locality */
-        ptr->buffers[ptr->cntr][idx++] = 'U';
-        ptr->buffers[ptr->cntr][idx++] = 'N';
-        ptr->buffers[ptr->cntr][idx++] = 'K';
-        ptr->buffers[ptr->cntr][idx++] = '\0';
-    }
-
-    return ptr->buffers[ptr->cntr];
-}
-
 static void obj_data_const(opal_hwloc_obj_data_t *ptr)
 {
     ptr->npus_calculated = false;
@@ -420,9 +178,7 @@ static void obj_data_const(opal_hwloc_obj_data_t *ptr)
     ptr->idx = UINT_MAX;
     ptr->num_bound = 0;
 }
-OBJ_CLASS_INSTANCE(opal_hwloc_obj_data_t,
-                   opal_object_t,
-                   obj_data_const, NULL);
+OBJ_CLASS_INSTANCE(opal_hwloc_obj_data_t, opal_object_t, obj_data_const, NULL);
 
 static void sum_const(opal_hwloc_summary_t *ptr)
 {
@@ -438,9 +194,7 @@ static void sum_dest(opal_hwloc_summary_t *ptr)
     }
     OBJ_DESTRUCT(&ptr->sorted_by_dist_list);
 }
-OBJ_CLASS_INSTANCE(opal_hwloc_summary_t,
-                   opal_list_item_t,
-                   sum_const, sum_dest);
+OBJ_CLASS_INSTANCE(opal_hwloc_summary_t, opal_list_item_t, sum_const, sum_dest);
 static void topo_data_const(opal_hwloc_topo_data_t *ptr)
 {
     ptr->available = NULL;
@@ -460,97 +214,4 @@ static void topo_data_dest(opal_hwloc_topo_data_t *ptr)
     OBJ_DESTRUCT(&ptr->summaries);
     ptr->userdata = NULL;
 }
-OBJ_CLASS_INSTANCE(opal_hwloc_topo_data_t,
-                   opal_object_t,
-                   topo_data_const,
-                   topo_data_dest);
-
-OBJ_CLASS_INSTANCE(opal_rmaps_numa_node_t,
-        opal_list_item_t,
-        NULL,
-        NULL);
-
-int opal_hwloc_base_set_binding_policy(opal_binding_policy_t *policy, char *spec)
-{
-    int i;
-    opal_binding_policy_t tmp;
-    char **tmpvals, **quals;
-
-    /* set default */
-    tmp = 0;
-
-    /* binding specification */
-    if (NULL == spec) {
-        if (opal_hwloc_use_hwthreads_as_cpus) {
-            /* default to bind-to hwthread */
-            OPAL_SET_DEFAULT_BINDING_POLICY(tmp, OPAL_BIND_TO_HWTHREAD);
-        } else {
-            /* default to bind-to core */
-            OPAL_SET_DEFAULT_BINDING_POLICY(tmp, OPAL_BIND_TO_CORE);
-        }
-    } else if (0 == strncasecmp(spec, "none", strlen("none"))) {
-        OPAL_SET_BINDING_POLICY(tmp, OPAL_BIND_TO_NONE);
-    } else {
-        tmpvals = opal_argv_split(spec, ':');
-        if (1 < opal_argv_count(tmpvals) || ':' == spec[0]) {
-            if (':' == spec[0]) {
-                quals = opal_argv_split(&spec[1], ',');
-            } else {
-                quals = opal_argv_split(tmpvals[1], ',');
-            }
-            for (i=0; NULL != quals[i]; i++) {
-                if (0 == strncasecmp(quals[i], "if-supported", strlen(quals[i]))) {
-                    tmp |= OPAL_BIND_IF_SUPPORTED;
-                } else if (0 == strncasecmp(quals[i], "overload-allowed", strlen(quals[i])) ||
-                           0 == strncasecmp(quals[i], "oversubscribe-allowed", strlen(quals[i]))) {
-                    tmp |= OPAL_BIND_ALLOW_OVERLOAD;
-                } else if (0 == strncasecmp(quals[i], "ordered", strlen(quals[i]))) {
-                    tmp |= OPAL_BIND_ORDERED;
-                } else {
-                    /* unknown option */
-                    opal_output(0, "Unknown qualifier to binding policy: %s", spec);
-                    opal_argv_free(quals);
-                    opal_argv_free(tmpvals);
-                    return OPAL_ERR_BAD_PARAM;
-                }
-            }
-            opal_argv_free(quals);
-        }
-        if (NULL == tmpvals[0] || ':' == spec[0]) {
-            OPAL_SET_BINDING_POLICY(tmp, OPAL_BIND_TO_CORE);
-            tmp &= ~OPAL_BIND_GIVEN;
-        } else {
-            if (0 == strcasecmp(tmpvals[0], "hwthread")) {
-                OPAL_SET_BINDING_POLICY(tmp, OPAL_BIND_TO_HWTHREAD);
-            } else if (0 == strcasecmp(tmpvals[0], "core")) {
-                OPAL_SET_BINDING_POLICY(tmp, OPAL_BIND_TO_CORE);
-            } else if (0 == strcasecmp(tmpvals[0], "l1cache")) {
-                OPAL_SET_BINDING_POLICY(tmp, OPAL_BIND_TO_L1CACHE);
-            } else if (0 == strcasecmp(tmpvals[0], "l2cache")) {
-                OPAL_SET_BINDING_POLICY(tmp, OPAL_BIND_TO_L2CACHE);
-            } else if (0 == strcasecmp(tmpvals[0], "l3cache")) {
-                OPAL_SET_BINDING_POLICY(tmp, OPAL_BIND_TO_L3CACHE);
-            } else if (0 == strcasecmp(tmpvals[0], "socket")) {
-                OPAL_SET_BINDING_POLICY(tmp, OPAL_BIND_TO_SOCKET);
-            } else if (0 == strcasecmp(tmpvals[0], "numa")) {
-                OPAL_SET_BINDING_POLICY(tmp, OPAL_BIND_TO_NUMA);
-            } else if (0 == strcasecmp(tmpvals[0], "board")) {
-                OPAL_SET_BINDING_POLICY(tmp, OPAL_BIND_TO_BOARD);
-            } else if (0 == strcasecmp(tmpvals[0], "cpu-list") ||
-                       0 == strcasecmp(tmpvals[0], "cpulist")) {
-                // Accept both "cpu-list" (which matches the
-                // "--cpu-list" CLI option) and "cpulist" (because
-                // people will be lazy)
-                OPAL_SET_BINDING_POLICY(tmp, OPAL_BIND_TO_CPUSET);
-            } else {
-                opal_show_help("help-opal-hwloc-base.txt", "invalid binding_policy", true, "binding", spec);
-                opal_argv_free(tmpvals);
-                return OPAL_ERR_BAD_PARAM;
-            }
-        }
-        opal_argv_free(tmpvals);
-    }
-
-    *policy = tmp;
-    return OPAL_SUCCESS;
-}
+OBJ_CLASS_INSTANCE(opal_hwloc_topo_data_t, opal_object_t, topo_data_const, topo_data_dest);
diff --git a/opal/mca/hwloc/base/hwloc_base_maffinity.c b/opal/mca/hwloc/base/hwloc_base_maffinity.c
index 4b442037e3d..1064aa8c653 100644
--- a/opal/mca/hwloc/base/hwloc_base_maffinity.c
+++ b/opal/mca/hwloc/base/hwloc_base_maffinity.c
@@ -8,74 +8,45 @@
  * $HEADER$
  */
 
-
 #include "opal_config.h"
 
 #include "opal/constants.h"
+#include "opal/util/show_help.h"
 
-#include "opal/mca/hwloc/hwloc-internal.h"
 #include "opal/mca/hwloc/base/base.h"
+#include "opal/mca/hwloc/hwloc-internal.h"
 
+/**
+ * Global reflecting the BFA (set by MCA param).
+ */
+extern opal_hwloc_base_mbfa_t opal_hwloc_base_mbfa;
 
-/*
- * Don't use show_help() here (or print any error message at all).
- * Let the upper layer output a relevant message, because doing so may
- * be complicated (e.g., this might be called from the ORTE ODLS,
- * which has to do some extra steps to get error messages to be
- * displayed).
+/**
+ * Report a bind failure using the normal mechanisms if a component
+ * fails to bind memory -- according to the value of the
+ * hwloc_base_bind_failure_action MCA parameter.
  */
-int opal_hwloc_base_set_process_membind_policy(void)
+static int opal_hwloc_base_report_bind_failure(const char *file, int line, const char *msg, int rc)
 {
-    int rc = 0, flags;
-    hwloc_membind_policy_t policy;
-    hwloc_cpuset_t cpuset;
-
-    /* Make sure opal_hwloc_topology has been set by the time we've
-       been called */
-    if (OPAL_SUCCESS != opal_hwloc_base_get_topology()) {
-        return OPAL_ERR_BAD_PARAM;
+    static int already_reported = 0;
+
+    if (!already_reported && OPAL_HWLOC_BASE_MBFA_SILENT != opal_hwloc_base_mbfa) {
+        char hostname[OPAL_MAXHOSTNAMELEN];
+        gethostname(hostname, sizeof(hostname));
+
+        opal_show_help(
+            "help-opal-hwloc-base.txt", "mbind failure", true, hostname, getpid(), file, line, msg,
+            (OPAL_HWLOC_BASE_MBFA_WARN == opal_hwloc_base_mbfa)
+                ? "Warning -- your job will continue, but possibly with degraded performance"
+                : "ERROR -- your job may abort or behave erraticly");
+        already_reported = 1;
+        return rc;
     }
 
-    /* Set the default memory allocation policy according to MCA
-       param */
-    switch (opal_hwloc_base_map) {
-    case OPAL_HWLOC_BASE_MAP_LOCAL_ONLY:
-        policy = HWLOC_MEMBIND_BIND;
-        flags = HWLOC_MEMBIND_STRICT;
-        break;
-
-    case OPAL_HWLOC_BASE_MAP_NONE:
-    default:
-        policy = HWLOC_MEMBIND_DEFAULT;
-        flags = 0;
-        break;
-    }
-
-    cpuset = hwloc_bitmap_alloc();
-    if (NULL == cpuset) {
-        rc = OPAL_ERR_OUT_OF_RESOURCE;
-    } else {
-        int e;
-        hwloc_get_cpubind(opal_hwloc_topology, cpuset, 0);
-        rc = hwloc_set_membind(opal_hwloc_topology,
-                               cpuset, policy, flags);
-        e = errno;
-        hwloc_bitmap_free(cpuset);
-
-        /* See if hwloc was able to do it.  If hwloc failed due to
-           ENOSYS, but the base_map == NONE, then it's not really an
-           error. */
-        if (0 != rc && ENOSYS == e &&
-            OPAL_HWLOC_BASE_MAP_NONE == opal_hwloc_base_map) {
-            rc = 0;
-        }
-    }
-
-    return (0 == rc) ? OPAL_SUCCESS : OPAL_ERROR;
+    return OPAL_SUCCESS;
 }
 
-int opal_hwloc_base_memory_set(opal_hwloc_base_memory_segment_t *segments,
-                               size_t num_segments)
+int opal_hwloc_base_memory_set(opal_hwloc_base_memory_segment_t *segments, size_t num_segments)
 {
     int rc = OPAL_SUCCESS;
     char *msg = NULL;
@@ -85,8 +56,7 @@ int opal_hwloc_base_memory_set(opal_hwloc_base_memory_segment_t *segments,
     /* bozo check */
     if (OPAL_SUCCESS != opal_hwloc_base_get_topology()) {
         msg = "hwloc_set_area_membind() failure - topology not available";
-        return opal_hwloc_base_report_bind_failure(__FILE__, __LINE__,
-                                                   msg, rc);
+        return opal_hwloc_base_report_bind_failure(__FILE__, __LINE__, msg, rc);
     }
 
     /* This module won't be used unless the process is already
@@ -100,18 +70,17 @@ int opal_hwloc_base_memory_set(opal_hwloc_base_memory_segment_t *segments,
     }
     hwloc_get_cpubind(opal_hwloc_topology, cpuset, 0);
     for (i = 0; i < num_segments; ++i) {
-        if (0 != hwloc_set_area_membind(opal_hwloc_topology,
-                                        segments[i].mbs_start_addr,
-                                        segments[i].mbs_len, cpuset,
-                                        HWLOC_MEMBIND_BIND,
-                                        HWLOC_MEMBIND_STRICT)) {
+        if (0
+            != hwloc_set_area_membind(opal_hwloc_topology, segments[i].mbs_start_addr,
+                                      segments[i].mbs_len, cpuset, HWLOC_MEMBIND_BIND,
+                                      HWLOC_MEMBIND_STRICT)) {
             rc = OPAL_ERROR;
             msg = "hwloc_set_area_membind() failure";
             goto out;
         }
     }
 
- out:
+out:
     if (NULL != cpuset) {
         hwloc_bitmap_free(cpuset);
     }
@@ -121,16 +90,7 @@ int opal_hwloc_base_memory_set(opal_hwloc_base_memory_segment_t *segments,
     return OPAL_SUCCESS;
 }
 
-int opal_hwloc_base_node_name_to_id(char *node_name, int *id)
-{
-    /* GLB: fix me */
-    *id = atoi(node_name + 3);
-
-    return OPAL_SUCCESS;
-}
-
-int opal_hwloc_base_membind(opal_hwloc_base_memory_segment_t *segs,
-                            size_t count, int node_id)
+int opal_hwloc_base_membind(opal_hwloc_base_memory_segment_t *segs, size_t count, int node_id)
 {
     size_t i;
     int rc = OPAL_SUCCESS;
@@ -140,8 +100,7 @@ int opal_hwloc_base_membind(opal_hwloc_base_memory_segment_t *segs,
     /* bozo check */
     if (OPAL_SUCCESS != opal_hwloc_base_get_topology()) {
         msg = "hwloc_set_area_membind() failure - topology not available";
-        return opal_hwloc_base_report_bind_failure(__FILE__, __LINE__,
-                                                   msg, rc);
+        return opal_hwloc_base_report_bind_failure(__FILE__, __LINE__, msg, rc);
     }
 
     cpuset = hwloc_bitmap_alloc();
@@ -151,19 +110,17 @@ int opal_hwloc_base_membind(opal_hwloc_base_memory_segment_t *segs,
         goto out;
     }
     hwloc_bitmap_set(cpuset, node_id);
-    for(i = 0; i < count; i++) {
-        if (0 != hwloc_set_area_membind(opal_hwloc_topology,
-                                        segs[i].mbs_start_addr,
-                                        segs[i].mbs_len, cpuset,
-                                        HWLOC_MEMBIND_BIND,
-                                        HWLOC_MEMBIND_STRICT)) {
+    for (i = 0; i < count; i++) {
+        if (0
+            != hwloc_set_area_membind(opal_hwloc_topology, segs[i].mbs_start_addr, segs[i].mbs_len,
+                                      cpuset, HWLOC_MEMBIND_BIND, HWLOC_MEMBIND_STRICT)) {
             rc = OPAL_ERROR;
             msg = "hwloc_set_area_membind() failure";
             goto out;
         }
     }
 
- out:
+out:
     if (NULL != cpuset) {
         hwloc_bitmap_free(cpuset);
     }
diff --git a/opal/mca/hwloc/base/hwloc_base_proc_mempolicy.c b/opal/mca/hwloc/base/hwloc_base_proc_mempolicy.c
deleted file mode 100644
index 54c55648a78..00000000000
--- a/opal/mca/hwloc/base/hwloc_base_proc_mempolicy.c
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Copyright (c) 2011-2017 Cisco Systems, Inc.  All rights reserved
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-
-#include "opal_config.h"
-
-#include "opal/constants.h"
-
-#include "opal/mca/hwloc/hwloc-internal.h"
-#include "opal/mca/hwloc/base/base.h"
-
-
-/*
- * Don't use show_help() here (or print any error message at all).
- * Let the upper layer output a relevant message, because doing so may
- * be complicated (e.g., this might be called from the ORTE ODLS,
- * which has to do some extra steps to get error messages to be
- * displayed).
- */
-int opal_hwloc_base_set_process_membind_policy(void)
-{
-    int rc = 0, flags;
-    hwloc_membind_policy_t policy;
-    hwloc_cpuset_t cpuset;
-
-    /* Make sure opal_hwloc_topology has been set by the time we've
-       been called */
-    if (NULL == opal_hwloc_topology) {
-        return OPAL_ERR_BAD_PARAM;
-    }
-
-    /* Set the default memory allocation policy according to MCA
-       param */
-    switch (opal_hwloc_base_map) {
-    case OPAL_HWLOC_BASE_MAP_LOCAL_ONLY:
-        policy = HWLOC_MEMBIND_BIND;
-        flags = HWLOC_MEMBIND_STRICT;
-        break;
-
-    case OPAL_HWLOC_BASE_MAP_NONE:
-    default:
-        policy = HWLOC_MEMBIND_DEFAULT;
-        flags = 0;
-        break;
-    }
-
-    cpuset = hwloc_bitmap_alloc();
-    if (NULL == cpuset) {
-        rc = OPAL_ERR_OUT_OF_RESOURCE;
-    } else {
-        int e;
-        hwloc_get_cpubind(opal_hwloc_topology, cpuset, 0);
-        rc = hwloc_set_membind(opal_hwloc_topology,
-                               cpuset, policy, flags);
-        e = errno;
-        hwloc_bitmap_free(cpuset);
-
-        /* See if hwloc was able to do it.  If hwloc failed due to
-           ENOSYS, but the base_map == NONE, then it's not really an
-           error. */
-        if (0 != rc && ENOSYS == e &&
-            OPAL_HWLOC_BASE_MAP_NONE == opal_hwloc_base_map) {
-            rc = 0;
-        }
-    }
-
-    return (0 == rc) ? OPAL_SUCCESS : OPAL_ERROR;
-}
diff --git a/opal/mca/hwloc/base/hwloc_base_util.c b/opal/mca/hwloc/base/hwloc_base_util.c
index d8ac76b0199..7e43b476c09 100644
--- a/opal/mca/hwloc/base/hwloc_base_util.c
+++ b/opal/mca/hwloc/base/hwloc_base_util.c
@@ -21,6 +21,7 @@
  * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * Copyright (c) 2019-2021 IBM Corporation. All rights reserved.
  * Copyright (c) 2019-2020 Inria.  All rights reserved.
+ * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -33,243 +34,157 @@
 #include "opal_config.h"
 
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 #ifdef HAVE_ENDIAN_H
-#include <endian.h>
+#    include <endian.h>
 #endif
 #ifdef HAVE_SYS_STAT_H
-#include <sys/stat.h>
+#    include <sys/stat.h>
 #endif
 #if HAVE_FCNTL_H
-#include <fcntl.h>
+#    include <fcntl.h>
 #endif
 
-#include "opal/runtime/opal.h"
 #include "opal/constants.h"
+#include "opal/mca/pmix/pmix-internal.h"
+#include "opal/mca/threads/tsd.h"
+#include "opal/runtime/opal.h"
 #include "opal/util/argv.h"
-#include "opal/util/output.h"
 #include "opal/util/os_dirpath.h"
-#include "opal/util/show_help.h"
+#include "opal/util/output.h"
 #include "opal/util/printf.h"
-#include "opal/mca/pmix/pmix-internal.h"
-#include "opal/mca/threads/tsd.h"
+#include "opal/util/show_help.h"
 
-#include "opal/mca/hwloc/hwloc-internal.h"
 #include "opal/mca/hwloc/base/base.h"
+#include "opal/mca/hwloc/hwloc-internal.h"
 
-static bool topo_in_shmem = false;
+bool opal_hwloc_topo_in_shmem = false;
 
-/*
- * Provide the hwloc object that corresponds to the given
- * processor id of the given type.  Remember: "processor" here [usually] means "core" --
- * except that on some platforms, hwloc won't find any cores; it'll
- * only find PUs (!).  On such platforms, then do the same calculation
- * but with PUs instead of COREs.
- */
-hwloc_obj_t opal_hwloc_base_get_pu(hwloc_topology_t topo,
-                                   int lid,
-                                   opal_hwloc_resource_type_t rtype)
+static void fill_cache_line_size(void)
 {
-    hwloc_obj_type_t obj_type = HWLOC_OBJ_CORE;
+    int i = 0, cache_level = 2;
+    unsigned size;
+    unsigned int cache_object = HWLOC_OBJ_L2CACHE;
     hwloc_obj_t obj;
+    bool found = false;
 
-    /* hwloc isn't able to find cores on all platforms.  Example:
-       PPC64 running RHEL 5.4 (linux kernel 2.6.18) only reports NUMA
-       nodes and PU's.  Fine.
-
-       However, note that hwloc_get_obj_by_type() will return NULL in
-       2 (effectively) different cases:
-
-       - no objects of the requested type were found
-       - the Nth object of the requested type was not found
+    /* Look for the smallest L2 cache size */
+    size = 4096;
+    while (cache_level > 0 && !found) {
+        i = 0;
+        while (1) {
+            obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology, cache_object, cache_level, i,
+                                                  OPAL_HWLOC_LOGICAL);
+            if (NULL == obj) {
+                --cache_level;
+                cache_object = HWLOC_OBJ_L1CACHE;
+                break;
+            } else {
+                if (NULL != obj->attr && obj->attr->cache.linesize > 0
+                    && size > obj->attr->cache.linesize) {
+                    size = obj->attr->cache.linesize;
+                    found = true;
+                }
+            }
+            ++i;
+        }
+    }
 
-       So first we have to see if we can find *any* cores by looking
-       for the 0th core.  If we find it, then try to find the Nth
-       core.  Otherwise, try to find the Nth PU. */
-    if (opal_hwloc_use_hwthreads_as_cpus || (NULL == hwloc_get_obj_by_type(topo, HWLOC_OBJ_CORE, 0))) {
-        obj_type = HWLOC_OBJ_PU;
+    /* If we found an L2 cache size in the hwloc data, save it in
+       opal_cache_line_size.  Otherwise, we'll leave whatever default
+       was set in opal_init.c */
+    if (found) {
+        opal_cache_line_size = (int) size;
     }
+}
 
-    if (OPAL_HWLOC_PHYSICAL == rtype) {
-        /* find the pu - note that we can only find physical PUs
-         * as cores do not have unique physical numbers (they are
-         * numbered within their sockets instead). So we find the
-         * specified PU, and then return the core object that contains it */
-        obj = hwloc_get_pu_obj_by_os_index(topo, lid);
-        OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
-                             "physical cpu %d %s found in cpuset %s",
-                             lid, (NULL == obj) ? "not" : "is",
-                             (NULL == opal_hwloc_base_cpu_list) ? "None" : opal_hwloc_base_cpu_list));
-        /* we now need to shift upward to the core including this PU */
-        if (NULL != obj && HWLOC_OBJ_CORE == obj_type) {
-            obj = obj->parent;
+static int opal_hwloc_base_topology_set_flags(hwloc_topology_t topology, unsigned long flags, bool io)
+{
+    if (io) {
+#if HWLOC_API_VERSION < 0x20000
+        flags |= HWLOC_TOPOLOGY_FLAG_IO_DEVICES;
+#else
+        int ret = hwloc_topology_set_io_types_filter(topology, HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
+        if (0 != ret) {
+            return ret;
         }
-        return obj;
+#endif
     }
-
-    opal_output_verbose(5, opal_hwloc_base_framework.framework_output,
-                        "Searching for %d LOGICAL PU", lid);
-
-    /* Now do the actual lookup. */
-    obj = hwloc_get_obj_by_type(topo, obj_type, lid);
-    OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
-                         "logical cpu %d %s found in cpuset %s",
-                         lid, (NULL == obj) ? "not" : "is",
-                         (NULL == opal_hwloc_base_cpu_list) ? "None" : opal_hwloc_base_cpu_list));
-
-    /* Found the right core (or PU). Return the object */
-    return obj;
+    return hwloc_topology_set_flags(topology, flags);
 }
 
 /* determine the node-level available cpuset based on
- * online vs allowed vs user-specified cpus
+ * online vs allowed vs user-specified cpus.
+ *
+ * Only used in 'self discovery' case, when the topology
+ * could not be retrieved from the RM.
  */
-int opal_hwloc_base_filter_cpus(hwloc_topology_t topo)
+static int opal_hwloc_base_filter_cpus(hwloc_topology_t topo)
 {
-    hwloc_obj_t root, pu;
-    hwloc_cpuset_t avail = NULL, pucpus, res;
+    hwloc_obj_t root;
+    hwloc_cpuset_t avail = NULL;
     opal_hwloc_topo_data_t *sum;
-    opal_hwloc_obj_data_t *data;
-    char **ranges=NULL, **range=NULL;
-    int idx, cpu, start, end;
 
     root = hwloc_get_root_obj(topo);
 
     if (NULL == root->userdata) {
-        root->userdata = (void*)OBJ_NEW(opal_hwloc_topo_data_t);
+        root->userdata = (void *) OBJ_NEW(opal_hwloc_topo_data_t);
     }
-    sum = (opal_hwloc_topo_data_t*)root->userdata;
+    sum = (opal_hwloc_topo_data_t *) root->userdata;
 
     /* should only ever enter here once, but check anyway */
     if (NULL != sum->available) {
         return OPAL_SUCCESS;
     }
 
-    /* process any specified default cpu set against this topology */
-    if (NULL == opal_hwloc_base_cpu_list) {
-        /* get the root available cpuset */
-        #if HWLOC_API_VERSION < 0x20000
-            avail = hwloc_bitmap_alloc();
-            hwloc_bitmap_and(avail, root->online_cpuset, root->allowed_cpuset);
-        #else
-            avail = hwloc_bitmap_dup(root->cpuset);
-        #endif
-        OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
-                             "hwloc:base: no cpus specified - using root available cpuset"));
+/* get the root available cpuset */
+#if HWLOC_API_VERSION < 0x20000
+    if (NULL == root->online_cpuset || NULL == root->allowed_cpuset) {
+        if (NULL == root->cpuset) {
+            /* we have a really bad topology */
+            return OPAL_ERR_NOT_SUPPORTED;
+        }
+        avail = hwloc_bitmap_dup(root->cpuset);
     } else {
-        OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
-                             "hwloc:base: filtering cpuset"));
-        /* find the specified logical cpus */
-        ranges = opal_argv_split(opal_hwloc_base_cpu_list, ',');
         avail = hwloc_bitmap_alloc();
-        hwloc_bitmap_zero(avail);
-        res = hwloc_bitmap_alloc();
-        pucpus = hwloc_bitmap_alloc();
-        for (idx=0; idx < opal_argv_count(ranges); idx++) {
-            range = opal_argv_split(ranges[idx], '-');
-            switch (opal_argv_count(range)) {
-            case 1:
-                /* only one cpu given - get that object */
-                cpu = strtoul(range[0], NULL, 10);
-                if (NULL != (pu = opal_hwloc_base_get_pu(topo, cpu, OPAL_HWLOC_LOGICAL))) {
-                    #if HWLOC_API_VERSION < 0x20000
-                        hwloc_bitmap_and(pucpus, pu->online_cpuset, pu->allowed_cpuset);
-                    #else
-                        hwloc_bitmap_free(pucpus);
-                        pucpus = hwloc_bitmap_dup(pu->cpuset);
-                    #endif
-                    hwloc_bitmap_or(res, avail, pucpus);
-                    hwloc_bitmap_copy(avail, res);
-                    data = (opal_hwloc_obj_data_t*)pu->userdata;
-                    if (NULL == data) {
-                        pu->userdata = (void*)OBJ_NEW(opal_hwloc_obj_data_t);
-                        data = (opal_hwloc_obj_data_t*)pu->userdata;
-                    }
-                    data->npus++;
-                }
-                break;
-            case 2:
-                /* range given */
-                start = strtoul(range[0], NULL, 10);
-                end = strtoul(range[1], NULL, 10);
-                for (cpu=start; cpu <= end; cpu++) {
-                    if (NULL != (pu = opal_hwloc_base_get_pu(topo, cpu, OPAL_HWLOC_LOGICAL))) {
-                        #if HWLOC_API_VERSION < 0x20000
-                            hwloc_bitmap_and(pucpus, pu->online_cpuset, pu->allowed_cpuset);
-                        #else
-                            hwloc_bitmap_free(pucpus);
-                            pucpus = hwloc_bitmap_dup(pu->cpuset);
-                        #endif
-                        hwloc_bitmap_or(res, avail, pucpus);
-                        hwloc_bitmap_copy(avail, res);
-                        data = (opal_hwloc_obj_data_t*)pu->userdata;
-                        if (NULL == data) {
-                            pu->userdata = (void*)OBJ_NEW(opal_hwloc_obj_data_t);
-                            data = (opal_hwloc_obj_data_t*)pu->userdata;
-                        }
-                        data->npus++;
-                    }
-                }
-                break;
-            default:
-                break;
-            }
-            opal_argv_free(range);
-        }
-        if (NULL != ranges) {
-            opal_argv_free(ranges);
-        }
-        hwloc_bitmap_free(res);
-        hwloc_bitmap_free(pucpus);
+        hwloc_bitmap_and(avail, root->online_cpuset, root->allowed_cpuset);
     }
-
+#else
+    avail = hwloc_bitmap_dup(root->cpuset);
+#endif
     /* cache this info */
     sum->available = avail;
 
     return OPAL_SUCCESS;
 }
 
-static void fill_cache_line_size(void)
+/**
+ * Initializes opal_hwloc_my_cpuset (global variable in
+ * opal/mca/hwloc/hwloc-internal.h) for this process.  opal_hwloc_my_cpuset
+ * will be loaded with this process' binding, or, if the process is
+ * not bound, use the hwloc root object's (available and online)
+ * cpuset.
+ */
+static void opal_hwloc_base_set_local_cpuset(void)
 {
-    int i = 0, cache_level = 2;
-    unsigned size;
-    unsigned int cache_object = HWLOC_OBJ_L2CACHE;
-    hwloc_obj_t obj;
-    bool found = false;
+    hwloc_obj_t root;
 
-    /* Look for the smallest L2 cache size */
-    size = 4096;
-    while (cache_level > 0 && !found) {
-        i=0;
-        while (1) {
-            obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology,
-                                                  cache_object, cache_level,
-                                                  i, OPAL_HWLOC_LOGICAL);
-            if (NULL == obj) {
-                --cache_level;
-                cache_object = HWLOC_OBJ_L1CACHE;
-                break;
-            } else {
-                if (NULL != obj->attr &&
-                    obj->attr->cache.linesize > 0 &&
-                    size > obj->attr->cache.linesize) {
-                    size = obj->attr->cache.linesize;
-                    found = true;
-                }
-            }
-            ++i;
+    if (NULL != opal_hwloc_topology) {
+        if (NULL == opal_hwloc_my_cpuset) {
+            opal_hwloc_my_cpuset = hwloc_bitmap_alloc();
         }
-    }
 
-    /* If we found an L2 cache size in the hwloc data, save it in
-       opal_cache_line_size.  Otherwise, we'll leave whatever default
-       was set in opal_init.c */
-    if (found) {
-        opal_cache_line_size = (int) size;
+        /* get the cpus we are bound to */
+        if (hwloc_get_cpubind(opal_hwloc_topology, opal_hwloc_my_cpuset, HWLOC_CPUBIND_PROCESS)
+            < 0) {
+            /* we are not bound - use the root's available cpuset */
+            root = hwloc_get_root_obj(opal_hwloc_topology);
+            hwloc_bitmap_copy(opal_hwloc_my_cpuset, root->cpuset);
+        }
     }
 }
 
@@ -284,8 +199,7 @@ int opal_hwloc_base_get_topology(void)
     char *shmemfile;
 #endif
 
-    opal_output_verbose(2, opal_hwloc_base_framework.framework_output,
-                         "hwloc:base:get_topology");
+    opal_output_verbose(2, opal_hwloc_base_framework.framework_output, "hwloc:base:get_topology");
 
     /* see if we already have it */
     if (NULL != opal_hwloc_topology) {
@@ -294,31 +208,19 @@ int opal_hwloc_base_get_topology(void)
     wildcard_rank.jobid = OPAL_PROC_MY_NAME.jobid;
     wildcard_rank.vpid = OPAL_VPID_WILDCARD;
 
-    // Did the user ask for a topo file at the mca line?
-    // Check this first, before main methods.
-    if(NULL != opal_hwloc_base_topo_file) {
-        opal_output_verbose(1, opal_hwloc_base_framework.framework_output,
-                            "hwloc:base loading topology from file %s",
-                            opal_hwloc_base_topo_file);
-        if(OPAL_SUCCESS != (rc = opal_hwloc_base_set_topology(opal_hwloc_base_topo_file))) {
-            return rc;
-        }
-        goto done;
-    }
-
 #if HWLOC_API_VERSION >= 0x20000
     opal_output_verbose(2, opal_hwloc_base_framework.framework_output,
-                         "hwloc:base: looking for topology in shared memory");
+                        "hwloc:base: looking for topology in shared memory");
 
     /* first try to get the shmem link, if available */
     aptr = &addr;
     sptr = &size;
-    OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_HWLOC_SHMEM_FILE,
-                                   &wildcard_rank, (void**)&shmemfile, PMIX_STRING);
-    OPAL_MODEX_RECV_VALUE_OPTIONAL(rc2, PMIX_HWLOC_SHMEM_ADDR,
-                                   &wildcard_rank, (void**)&aptr, PMIX_SIZE);
-    OPAL_MODEX_RECV_VALUE_OPTIONAL(rc3, PMIX_HWLOC_SHMEM_SIZE,
-                                   &wildcard_rank, (void**)&sptr, PMIX_SIZE);
+    OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_HWLOC_SHMEM_FILE, &wildcard_rank, (void **) &shmemfile,
+                                   PMIX_STRING);
+    OPAL_MODEX_RECV_VALUE_OPTIONAL(rc2, PMIX_HWLOC_SHMEM_ADDR, &wildcard_rank, (void **) &aptr,
+                                   PMIX_SIZE);
+    OPAL_MODEX_RECV_VALUE_OPTIONAL(rc3, PMIX_HWLOC_SHMEM_SIZE, &wildcard_rank, (void **) &sptr,
+                                   PMIX_SIZE);
     if (OPAL_SUCCESS == rc && OPAL_SUCCESS == rc2 && OPAL_SUCCESS == rc3) {
         if (0 > (fd = open(shmemfile, O_RDONLY))) {
             free(shmemfile);
@@ -326,8 +228,7 @@ int opal_hwloc_base_get_topology(void)
             return OPAL_ERR_FILE_OPEN_FAILURE;
         }
         free(shmemfile);
-        if (0 != hwloc_shmem_topology_adopt(&opal_hwloc_topology, fd,
-                                            0, (void*)addr, size, 0)) {
+        if (0 != hwloc_shmem_topology_adopt(&opal_hwloc_topology, fd, 0, (void *) addr, size, 0)) {
             if (4 < opal_output_get_verbosity(opal_hwloc_base_framework.framework_output)) {
                 FILE *file = fopen("/proc/self/maps", "r");
                 if (file) {
@@ -348,7 +249,7 @@ int opal_hwloc_base_get_topology(void)
         } else {
             opal_output_verbose(2, opal_hwloc_base_framework.framework_output,
                                 "hwloc:base: topology in shared memory");
-            topo_in_shmem = true;
+            opal_hwloc_topo_in_shmem = true;
             goto done;
         }
     }
@@ -356,19 +257,15 @@ int opal_hwloc_base_get_topology(void)
     /* if that isn't available, then try to retrieve
      * the xml representation from the PMIx data store */
     opal_output_verbose(1, opal_hwloc_base_framework.framework_output,
-                        "hwloc:base[%s:%d] getting topology XML string",
-                        __FILE__, __LINE__);
+                        "hwloc:base[%s:%d] getting topology XML string", __FILE__, __LINE__);
 #if HWLOC_API_VERSION >= 0x20000
-    OPAL_MODEX_RECV_VALUE_IMMEDIATE(rc, PMIX_HWLOC_XML_V2,
-                                    &wildcard_rank, &val, PMIX_STRING);
+    OPAL_MODEX_RECV_VALUE_IMMEDIATE(rc, PMIX_HWLOC_XML_V2, &wildcard_rank, &val, PMIX_STRING);
 #else
-    OPAL_MODEX_RECV_VALUE_IMMEDIATE(rc, PMIX_HWLOC_XML_V1,
-                                    &wildcard_rank, &val, PMIX_STRING);
+    OPAL_MODEX_RECV_VALUE_IMMEDIATE(rc, PMIX_HWLOC_XML_V1, &wildcard_rank, &val, PMIX_STRING);
 #endif
     if (rc != OPAL_SUCCESS) {
         /* check the old topo key to keep compatibility with older RMs */
-        OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_LOCAL_TOPO,
-                                       &wildcard_rank, &val, PMIX_STRING);
+        OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_LOCAL_TOPO, &wildcard_rank, &val, PMIX_STRING);
     }
 
     if (OPAL_SUCCESS == rc && NULL != val) {
@@ -379,39 +276,38 @@ int opal_hwloc_base_get_topology(void)
             free(val);
             return OPAL_ERROR;
         }
-        if (0 != hwloc_topology_set_xmlbuffer(opal_hwloc_topology, val, strlen(val)+1)) {
+        if (0 != hwloc_topology_set_xmlbuffer(opal_hwloc_topology, val, strlen(val) + 1)) {
+            /* default to discovery */
             free(val);
             hwloc_topology_destroy(opal_hwloc_topology);
-            return OPAL_ERROR;
+            goto discover;
         }
         /* since we are loading this from an external source, we have to
          * explicitly set a flag so hwloc sets things up correctly
          */
-        if (0 != opal_hwloc_base_topology_set_flags(opal_hwloc_topology,
-                                                    HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM,
-                                                    true)) {
+        if (0
+            != opal_hwloc_base_topology_set_flags(opal_hwloc_topology,
+                                                  HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM, true)) {
+            /* default to discovery */
             hwloc_topology_destroy(opal_hwloc_topology);
             free(val);
-            return OPAL_ERROR;
+            goto discover;
         }
         /* now load the topology */
         if (0 != hwloc_topology_load(opal_hwloc_topology)) {
+            /* default to discovery */
             hwloc_topology_destroy(opal_hwloc_topology);
             free(val);
-            return OPAL_ERROR;
+            goto discover;
         }
         free(val);
-        /* filter the cpus thru any default cpu set */
-        if (OPAL_SUCCESS != (rc = opal_hwloc_base_filter_cpus(opal_hwloc_topology))) {
-            hwloc_topology_destroy(opal_hwloc_topology);
-            return rc;
-        }
     } else {
+    discover:
         opal_output_verbose(1, opal_hwloc_base_framework.framework_output,
                             "hwloc:base discovering topology");
-        if (0 != hwloc_topology_init(&opal_hwloc_topology) ||
-            0 != opal_hwloc_base_topology_set_flags(opal_hwloc_topology, 0, true) ||
-            0 != hwloc_topology_load(opal_hwloc_topology)) {
+        if (0 != hwloc_topology_init(&opal_hwloc_topology)
+            || 0 != opal_hwloc_base_topology_set_flags(opal_hwloc_topology, 0, true)
+            || 0 != hwloc_topology_load(opal_hwloc_topology)) {
             OPAL_ERROR_LOG(OPAL_ERR_NOT_SUPPORTED);
             return OPAL_ERR_NOT_SUPPORTED;
         }
@@ -422,198 +318,29 @@ int opal_hwloc_base_get_topology(void)
         }
     }
 
-    done:
+done:
 
     /* fill opal_cache_line_size global with the smallest L1 cache
        line size */
     fill_cache_line_size();
 
-    /* get or update our local cpuset - it will get used multiple
-     * times, so it's more efficient to keep a global copy
+    /* Set or update our local cpuset - it could get used multiple
+     * times, so it's more efficient to keep a global copy.
      */
-    opal_hwloc_base_get_local_cpuset();
-
-    return OPAL_SUCCESS;
-}
-
-int opal_hwloc_base_set_topology(char *topofile)
-{
-    struct hwloc_topology_support *support;
-
-     OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
-                          "hwloc:base:set_topology %s", topofile));
-
-   if (NULL != opal_hwloc_topology) {
-        hwloc_topology_destroy(opal_hwloc_topology);
-    }
-    if (0 != hwloc_topology_init(&opal_hwloc_topology)) {
-        return OPAL_ERR_NOT_SUPPORTED;
-    }
-    if (0 != hwloc_topology_set_xml(opal_hwloc_topology, topofile)) {
-        hwloc_topology_destroy(opal_hwloc_topology);
-        OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
-                             "hwloc:base:set_topology bad topo file"));
-        return OPAL_ERR_NOT_SUPPORTED;
-    }
-    /* since we are loading this from an external source, we have to
-     * explicitly set a flag so hwloc sets things up correctly
-     */
-    if (0 != opal_hwloc_base_topology_set_flags(opal_hwloc_topology,
-                                                HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM,
-                                                true)) {
-        hwloc_topology_destroy(opal_hwloc_topology);
-        return OPAL_ERR_NOT_SUPPORTED;
-    }
-    if (0 != hwloc_topology_load(opal_hwloc_topology)) {
-        hwloc_topology_destroy(opal_hwloc_topology);
-        OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
-                             "hwloc:base:set_topology failed to load"));
-        return OPAL_ERR_NOT_SUPPORTED;
-    }
-
-    /* unfortunately, hwloc does not include support info in its
-     * xml output :-(( We default to assuming it is present as
-     * systems that use this option are likely to provide
-     * binding support
-     */
-    support = (struct hwloc_topology_support*)hwloc_topology_get_support(opal_hwloc_topology);
-    support->cpubind->set_thisproc_cpubind = true;
-    support->membind->set_thisproc_membind = true;
-
-    /* fill opal_cache_line_size global with the smallest L1 cache
-       line size */
-    fill_cache_line_size();
-
-    /* all done */
-    return OPAL_SUCCESS;
-}
-
-static void free_object(hwloc_obj_t obj)
-{
-    opal_hwloc_obj_data_t *data;
-    unsigned k;
-
-    /* free any data hanging on this object */
-    if (NULL != obj->userdata) {
-        data = (opal_hwloc_obj_data_t*)obj->userdata;
-        OBJ_RELEASE(data);
-        obj->userdata = NULL;
-    }
-
-    /* loop thru our children */
-    for (k=0; k < obj->arity; k++) {
-        free_object(obj->children[k]);
-    }
-}
-
-void opal_hwloc_base_free_topology(hwloc_topology_t topo)
-{
-    hwloc_obj_t obj;
-    opal_hwloc_topo_data_t *rdata;
-    unsigned k;
-
-    if (!topo_in_shmem) {
-        obj = hwloc_get_root_obj(topo);
-        /* release the root-level userdata */
-        if (NULL != obj->userdata) {
-            rdata = (opal_hwloc_topo_data_t*)obj->userdata;
-            OBJ_RELEASE(rdata);
-            obj->userdata = NULL;
-        }
-        /* now recursively descend and release userdata
-         * in the rest of the objects
-         */
-        for (k=0; k < obj->arity; k++) {
-            free_object(obj->children[k]);
-        }
-    }
-    hwloc_topology_destroy(topo);
-}
-
-void opal_hwloc_base_get_local_cpuset(void)
-{
-    hwloc_obj_t root;
-
-    if (NULL != opal_hwloc_topology) {
-        if (NULL == opal_hwloc_my_cpuset) {
-            opal_hwloc_my_cpuset = hwloc_bitmap_alloc();
-        }
-
-        /* get the cpus we are bound to */
-        if (hwloc_get_cpubind(opal_hwloc_topology,
-                              opal_hwloc_my_cpuset,
-                              HWLOC_CPUBIND_PROCESS) < 0) {
-            /* we are not bound - use the root's available cpuset */
-            root = hwloc_get_root_obj(opal_hwloc_topology);
-            hwloc_bitmap_copy(opal_hwloc_my_cpuset, root->cpuset);
-        }
-    }
-}
-
-int opal_hwloc_base_report_bind_failure(const char *file,
-                                        int line,
-                                        const char *msg, int rc)
-{
-    static int already_reported = 0;
-
-    if (!already_reported &&
-        OPAL_HWLOC_BASE_MBFA_SILENT != opal_hwloc_base_mbfa) {
-        char hostname[OPAL_MAXHOSTNAMELEN];
-        gethostname(hostname, sizeof(hostname));
-
-        opal_show_help("help-opal-hwloc-base.txt", "mbind failure", true,
-                       hostname, getpid(), file, line, msg,
-                       (OPAL_HWLOC_BASE_MBFA_WARN == opal_hwloc_base_mbfa) ?
-                       "Warning -- your job will continue, but possibly with degraded performance" :
-                       "ERROR -- your job may abort or behave erraticly");
-        already_reported = 1;
-        return rc;
-    }
+    opal_hwloc_base_set_local_cpuset();
 
     return OPAL_SUCCESS;
 }
 
-/* determine if there is a single cpu in a bitmap */
-bool opal_hwloc_base_single_cpu(hwloc_cpuset_t cpuset)
-{
-    int i;
-    bool one=false;
-
-    /* count the number of bits that are set - there is
-     * one bit for each available pu. We could just
-     * subtract the first and last indices, but there
-     * may be "holes" in the bitmap corresponding to
-     * offline or unallowed cpus - so we have to
-     * search for them. Return false if we anything
-     * other than one
-     */
-    for (i=hwloc_bitmap_first(cpuset);
-         i <= hwloc_bitmap_last(cpuset);
-         i++) {
-        if (hwloc_bitmap_isset(cpuset, i)) {
-            if (one) {
-                return false;
-            }
-            one = true;
-        }
-    }
-
-    return one;
-}
-
 /* hwloc treats cache objects as special
  * cases. Instead of having a unique type for each cache level,
  * there is a single cache object type, and the level is encoded
  * in an attribute union. So looking for cache objects involves
  * a multi-step test :-(
  */
-static hwloc_obj_t df_search(hwloc_topology_t topo,
-                             hwloc_obj_t start,
-                             hwloc_obj_type_t target,
-                             unsigned cache_level,
-                             unsigned int nobj,
-                             opal_hwloc_resource_type_t rtype,
-                             unsigned int *num_objs)
+static hwloc_obj_t df_search(hwloc_topology_t topo, hwloc_obj_t start, hwloc_obj_type_t target,
+                             unsigned cache_level, unsigned int nobj,
+                             opal_hwloc_resource_type_t rtype, unsigned int *num_objs)
 {
     hwloc_obj_t obj;
     int search_depth;
@@ -629,12 +356,14 @@ static hwloc_obj_t df_search(hwloc_topology_t topo,
         search_depth = hwloc_get_cache_type_depth(topo, cache_level, (hwloc_obj_cache_type_t) -1);
 #endif
     }
-    if (HWLOC_TYPE_DEPTH_UNKNOWN == search_depth)
+    if (HWLOC_TYPE_DEPTH_UNKNOWN == search_depth) {
         return NULL;
+    }
 
     if (OPAL_HWLOC_LOGICAL == rtype) {
-        if (num_objs)
+        if (num_objs) {
             *num_objs = hwloc_get_nbobjs_by_depth(topo, search_depth);
+        }
         return hwloc_get_obj_by_depth(topo, search_depth, nobj);
     }
     if (OPAL_HWLOC_PHYSICAL == rtype) {
@@ -651,25 +380,28 @@ static hwloc_obj_t df_search(hwloc_topology_t topo,
          */
         hwloc_obj_t found = NULL;
         obj = NULL;
-        if (num_objs)
+        if (num_objs) {
             *num_objs = 0;
+        }
         while ((obj = hwloc_get_next_obj_by_depth(topo, search_depth, obj)) != NULL) {
-            if (num_objs && obj->os_index > *num_objs)
+            if (num_objs && obj->os_index > *num_objs) {
                 *num_objs = obj->os_index;
-            if (obj->os_index == nobj)
+            }
+            if (obj->os_index == nobj) {
                 found = obj;
+            }
         }
         return found;
     }
     if (OPAL_HWLOC_AVAILABLE == rtype) {
-// The previous (3.x) code included a check for
-// available = opal_hwloc_base_get_available_cpus(topo, start)
-// and skipped objs that had hwloc_bitmap_iszero(available)
+        // The previous (3.x) code included a check for
+        // available = opal_hwloc_base_get_available_cpus(topo, start)
+        // and skipped objs that had hwloc_bitmap_iszero(available)
         hwloc_obj_t root;
         opal_hwloc_topo_data_t *rdata = NULL;
         root = hwloc_get_root_obj(topo);
-        if(false == topo_in_shmem) {
-            rdata = (opal_hwloc_topo_data_t*)root->userdata;
+        if (false == opal_hwloc_topo_in_shmem) {
+            rdata = (opal_hwloc_topo_data_t *) root->userdata;
         }
         hwloc_cpuset_t constrained_cpuset;
 
@@ -681,10 +413,14 @@ static hwloc_obj_t df_search(hwloc_topology_t topo,
         }
 
         unsigned idx = 0;
-        if (num_objs)
-            *num_objs = hwloc_get_nbobjs_inside_cpuset_by_depth(topo, constrained_cpuset, search_depth);
+        if (num_objs) {
+            *num_objs = hwloc_get_nbobjs_inside_cpuset_by_depth(topo, constrained_cpuset,
+                                                                search_depth);
+        }
         obj = NULL;
-        while ((obj = hwloc_get_next_obj_inside_cpuset_by_depth(topo, constrained_cpuset, search_depth, obj)) != NULL) {
+        while ((obj = hwloc_get_next_obj_inside_cpuset_by_depth(topo, constrained_cpuset,
+                                                                search_depth, obj))
+               != NULL) {
             if (idx == nobj) {
                 hwloc_bitmap_free(constrained_cpuset);
                 return obj;
@@ -697,8 +433,7 @@ static hwloc_obj_t df_search(hwloc_topology_t topo,
     return NULL;
 }
 
-unsigned int opal_hwloc_base_get_nbobjs_by_type(hwloc_topology_t topo,
-                                                hwloc_obj_type_t target,
+unsigned int opal_hwloc_base_get_nbobjs_by_type(hwloc_topology_t topo, hwloc_obj_type_t target,
                                                 unsigned cache_level,
                                                 opal_hwloc_resource_type_t rtype)
 {
@@ -710,8 +445,8 @@ unsigned int opal_hwloc_base_get_nbobjs_by_type(hwloc_topology_t topo,
 
     /* bozo check */
     if (NULL == topo) {
-        OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
-                             "hwloc:base:get_nbobjs NULL topology"));
+        OPAL_OUTPUT_VERBOSE(
+            (5, opal_hwloc_base_framework.framework_output, "hwloc:base:get_nbobjs NULL topology"));
         return 0;
     }
 
@@ -723,7 +458,7 @@ unsigned int opal_hwloc_base_get_nbobjs_by_type(hwloc_topology_t topo,
 #if HWLOC_API_VERSION < 0x20000
         && HWLOC_OBJ_CACHE != target
 #endif
-       ) {
+    ) {
         /* we should not get an error back, but just in case... */
         if (0 > (rc = hwloc_get_nbobjs_by_type(topo, target))) {
             opal_output(0, "UNKNOWN HWLOC ERROR");
@@ -737,22 +472,20 @@ unsigned int opal_hwloc_base_get_nbobjs_by_type(hwloc_topology_t topo,
     obj = hwloc_get_root_obj(topo);
 
     /* first see if the topology already has this summary */
-    if(false == topo_in_shmem) {
-        data = (opal_hwloc_topo_data_t*)obj->userdata;
+    if (false == opal_hwloc_topo_in_shmem) {
+        data = (opal_hwloc_topo_data_t *) obj->userdata;
     }
     if (NULL == data) {
         data = OBJ_NEW(opal_hwloc_topo_data_t);
-        if(false == topo_in_shmem) {
+        if (false == opal_hwloc_topo_in_shmem) {
             // Can't touch userdata if in read-only shmem!
             // We have to protect here for the case where obj->userdata
             // is in shmem and it is NULL.
-            obj->userdata = (void*) data;
+            obj->userdata = (void *) data;
         }
     } else {
-        OPAL_LIST_FOREACH(sum, &data->summaries, opal_hwloc_summary_t) {
-            if (target == sum->type &&
-                cache_level == sum->cache_level &&
-                rtype == sum->rtype) {
+        OPAL_LIST_FOREACH (sum, &data->summaries, opal_hwloc_summary_t) {
+            if (target == sum->type && cache_level == sum->cache_level && rtype == sum->rtype) {
                 /* yep - return the value */
                 OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
                                      "hwloc:base:get_nbojbs pre-existing data %u of %s:%u",
@@ -774,19 +507,17 @@ unsigned int opal_hwloc_base_get_nbobjs_by_type(hwloc_topology_t topo,
     opal_list_append(&data->summaries, &sum->super);
 
     OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
-                         "hwloc:base:get_nbojbs computed data %u of %s:%u",
-                         num_objs, hwloc_obj_type_string(target), cache_level));
+                         "hwloc:base:get_nbojbs computed data %u of %s:%u", num_objs,
+                         hwloc_obj_type_string(target), cache_level));
 
     return num_objs;
 }
 
-/* as above, only return the Nth instance of the specified object
+/* Return the Nth instance of the specified object
  * type from inside the topology
  */
-hwloc_obj_t opal_hwloc_base_get_obj_by_type(hwloc_topology_t topo,
-                                            hwloc_obj_type_t target,
-                                            unsigned cache_level,
-                                            unsigned int instance,
+hwloc_obj_t opal_hwloc_base_get_obj_by_type(hwloc_topology_t topo, hwloc_obj_type_t target,
+                                            unsigned cache_level, unsigned int instance,
                                             opal_hwloc_resource_type_t rtype)
 {
     hwloc_obj_t obj;
@@ -804,7 +535,7 @@ hwloc_obj_t opal_hwloc_base_get_obj_by_type(hwloc_topology_t topo,
 #if HWLOC_API_VERSION < 0x20000
         && HWLOC_OBJ_CACHE != target
 #endif
-       ) {
+    ) {
         return hwloc_get_obj_by_type(topo, target, instance);
     }
 
@@ -813,484 +544,7 @@ hwloc_obj_t opal_hwloc_base_get_obj_by_type(hwloc_topology_t topo,
     return df_search(topo, obj, target, cache_level, instance, rtype, NULL);
 }
 
-static void opal_hwloc_base_get_relative_locality_by_depth(hwloc_topology_t topo, unsigned d,
-                                                           hwloc_cpuset_t loc1, hwloc_cpuset_t loc2,
-                                                           opal_hwloc_locality_t *locality, bool *shared)
-{
-    unsigned width, w;
-    hwloc_obj_t obj;
-    int sect1, sect2;
-
-    /* get the width of the topology at this depth */
-    width = hwloc_get_nbobjs_by_depth(topo, d);
-
-    /* scan all objects at this depth to see if
-     * our locations overlap with them
-     */
-    for (w=0; w < width; w++) {
-        /* get the object at this depth/index */
-        obj = hwloc_get_obj_by_depth(topo, d, w);
-        /* see if our locations intersect with the cpuset for this obj */
-        sect1 = hwloc_bitmap_intersects(obj->cpuset, loc1);
-        sect2 = hwloc_bitmap_intersects(obj->cpuset, loc2);
-        /* if both intersect, then we share this level */
-        if (sect1 && sect2) {
-            *shared = true;
-            switch(obj->type) {
-            case HWLOC_OBJ_NODE:
-                *locality |= OPAL_PROC_ON_NUMA;
-                break;
-            case HWLOC_OBJ_SOCKET:
-                *locality |= OPAL_PROC_ON_SOCKET;
-                break;
-#if HWLOC_API_VERSION < 0x20000
-            case HWLOC_OBJ_CACHE:
-                if (3 == obj->attr->cache.depth) {
-                    *locality |= OPAL_PROC_ON_L3CACHE;
-                } else if (2 == obj->attr->cache.depth) {
-                    *locality |= OPAL_PROC_ON_L2CACHE;
-                } else {
-                    *locality |= OPAL_PROC_ON_L1CACHE;
-                }
-                break;
-#else
-            case HWLOC_OBJ_L3CACHE:
-                *locality |= OPAL_PROC_ON_L3CACHE;
-                break;
-            case HWLOC_OBJ_L2CACHE:
-                *locality |= OPAL_PROC_ON_L2CACHE;
-                break;
-            case HWLOC_OBJ_L1CACHE:
-                *locality |= OPAL_PROC_ON_L1CACHE;
-                break;
-#endif
-            case HWLOC_OBJ_CORE:
-                *locality |= OPAL_PROC_ON_CORE;
-                break;
-            case HWLOC_OBJ_PU:
-                *locality |= OPAL_PROC_ON_HWTHREAD;
-                break;
-            default:
-                /* just ignore it */
-                break;
-            }
-            break;
-        }
-        /* otherwise, we don't share this
-         * object - but we still might share another object
-         * on this level, so we have to keep searching
-         */
-    }
-}
-
-opal_hwloc_locality_t opal_hwloc_base_get_relative_locality(hwloc_topology_t topo,
-                                                            char *cpuset1, char *cpuset2)
-{
-    opal_hwloc_locality_t locality;
-    hwloc_cpuset_t loc1, loc2;
-    unsigned depth, d;
-    bool shared;
-    hwloc_obj_type_t type;
-
-    /* start with what we know - they share a node on a cluster
-     * NOTE: we may alter that latter part as hwloc's ability to
-     * sense multi-cu, multi-cluster systems grows
-     */
-    locality = OPAL_PROC_ON_NODE | OPAL_PROC_ON_HOST | OPAL_PROC_ON_CU | OPAL_PROC_ON_CLUSTER;
-
-    /* if either cpuset is NULL, then that isn't bound */
-    if (NULL == cpuset1 || NULL == cpuset2) {
-        return locality;
-    }
-
-    /* get the max depth of the topology */
-    depth = hwloc_topology_get_depth(topo);
-
-    /* convert the strings to cpusets */
-    loc1 = hwloc_bitmap_alloc();
-    hwloc_bitmap_list_sscanf(loc1, cpuset1);
-    loc2 = hwloc_bitmap_alloc();
-    hwloc_bitmap_list_sscanf(loc2, cpuset2);
-
-    /* start at the first depth below the top machine level */
-    for (d=1; d < depth; d++) {
-        shared = false;
-        /* get the object type at this depth */
-        type = hwloc_get_depth_type(topo, d);
-        /* if it isn't one of interest, then ignore it */
-        if (HWLOC_OBJ_NODE != type &&
-            HWLOC_OBJ_SOCKET != type &&
-#if HWLOC_API_VERSION < 0x20000
-            HWLOC_OBJ_CACHE != type &&
-#else
-            HWLOC_OBJ_L3CACHE != type &&
-            HWLOC_OBJ_L2CACHE != type &&
-            HWLOC_OBJ_L1CACHE != type &&
-#endif
-            HWLOC_OBJ_CORE != type &&
-            HWLOC_OBJ_PU != type) {
-            continue;
-        }
-        opal_hwloc_base_get_relative_locality_by_depth(topo, d, loc1, loc2, &locality, &shared);
-
-        /* if we spanned the entire width without finding
-         * a point of intersection, then no need to go
-         * deeper
-         */
-        if (!shared) {
-            break;
-        }
-    }
-#if HWLOC_API_VERSION >= 0x20000
-    opal_hwloc_base_get_relative_locality_by_depth(topo, HWLOC_TYPE_DEPTH_NUMANODE, loc1, loc2, &locality, &shared);
-#endif
-
-    opal_output_verbose(5, opal_hwloc_base_framework.framework_output,
-                        "locality: %s",
-                        opal_hwloc_base_print_locality(locality));
-    hwloc_bitmap_free(loc1);
-    hwloc_bitmap_free(loc2);
-
-    return locality;
-}
-
-
-char* opal_hwloc_base_print_binding(opal_binding_policy_t binding)
-{
-    char *ret, *bind;
-    opal_hwloc_print_buffers_t *ptr;
-
-    switch(OPAL_GET_BINDING_POLICY(binding)) {
-    case OPAL_BIND_TO_NONE:
-        bind = "NONE";
-        break;
-    case OPAL_BIND_TO_BOARD:
-        bind = "BOARD";
-        break;
-    case OPAL_BIND_TO_NUMA:
-        bind = "NUMA";
-        break;
-    case OPAL_BIND_TO_SOCKET:
-        bind = "SOCKET";
-        break;
-    case OPAL_BIND_TO_L3CACHE:
-        bind = "L3CACHE";
-        break;
-    case OPAL_BIND_TO_L2CACHE:
-        bind = "L2CACHE";
-        break;
-    case OPAL_BIND_TO_L1CACHE:
-        bind = "L1CACHE";
-        break;
-    case OPAL_BIND_TO_CORE:
-        bind = "CORE";
-        break;
-    case OPAL_BIND_TO_HWTHREAD:
-        bind = "HWTHREAD";
-        break;
-    case OPAL_BIND_TO_CPUSET:
-        bind = "CPUSET";
-        break;
-    default:
-        bind = "UNKNOWN";
-    }
-    ptr = opal_hwloc_get_print_buffer();
-    if (NULL == ptr) {
-        return opal_hwloc_print_null;
-    }
-    /* cycle around the ring */
-    if (OPAL_HWLOC_PRINT_NUM_BUFS == ptr->cntr) {
-        ptr->cntr = 0;
-    }
-    if (!OPAL_BINDING_REQUIRED(binding) &&
-        OPAL_BIND_OVERLOAD_ALLOWED(binding)) {
-        snprintf(ptr->buffers[ptr->cntr], OPAL_HWLOC_PRINT_MAX_SIZE,
-                 "%s:IF-SUPPORTED:OVERLOAD-ALLOWED", bind);
-    } else if (OPAL_BIND_OVERLOAD_ALLOWED(binding)) {
-        snprintf(ptr->buffers[ptr->cntr], OPAL_HWLOC_PRINT_MAX_SIZE,
-                 "%s:OVERLOAD-ALLOWED", bind);
-    } else if (!OPAL_BINDING_REQUIRED(binding)) {
-        snprintf(ptr->buffers[ptr->cntr], OPAL_HWLOC_PRINT_MAX_SIZE,
-                 "%s:IF-SUPPORTED", bind);
-    } else {
-        snprintf(ptr->buffers[ptr->cntr], OPAL_HWLOC_PRINT_MAX_SIZE, "%s", bind);
-    }
-    ret = ptr->buffers[ptr->cntr];
-    ptr->cntr++;
-
-    return ret;
-}
-
-/*
- * Turn an int bitmap to a "a-b,c" range kind of string
- */
-static char *bitmap2rangestr(int bitmap)
-{
-    size_t i;
-    int range_start, range_end;
-    bool first, isset;
-    char tmp[BUFSIZ];
-    const int stmp = sizeof(tmp) - 1;
-    static char ret[BUFSIZ];
-
-    memset(ret, 0, sizeof(ret));
-
-    first = true;
-    range_start = -999;
-    for (i = 0; i < sizeof(int) * 8; ++i) {
-        isset = (bitmap & (1 << i));
-
-        /* Do we have a running range? */
-        if (range_start >= 0) {
-            if (isset) {
-                continue;
-            } else {
-                /* A range just ended; output it */
-                if (!first) {
-                    strncat(ret, ",", sizeof(ret) - strlen(ret) - 1);
-                } else {
-                    first = false;
-                }
-
-                range_end = i - 1;
-                if (range_start == range_end) {
-                    snprintf(tmp, stmp, "%d", range_start);
-                } else {
-                    snprintf(tmp, stmp, "%d-%d", range_start, range_end);
-                }
-                strncat(ret, tmp, sizeof(ret) - strlen(ret) - 1);
-
-                range_start = -999;
-            }
-        }
-
-        /* No running range */
-        else {
-            if (isset) {
-                range_start = i;
-            }
-        }
-    }
-
-    /* If we ended the bitmap with a range open, output it */
-    if (range_start >= 0) {
-        if (!first) {
-            strncat(ret, ",", sizeof(ret) - strlen(ret) - 1);
-            first = false;
-        }
-
-        range_end = i - 1;
-        if (range_start == range_end) {
-            snprintf(tmp, stmp, "%d", range_start);
-        } else {
-            snprintf(tmp, stmp, "%d-%d", range_start, range_end);
-        }
-        strncat(ret, tmp, sizeof(ret) - strlen(ret) - 1);
-    }
-
-    return ret;
-}
-
-/*
- * Make a map of socket/core/hwthread tuples
- */
-static int build_map(int *num_sockets_arg, int *num_cores_arg,
-                     hwloc_cpuset_t cpuset, int ***map, hwloc_topology_t topo)
-{
-    int num_sockets, num_cores;
-    int socket_index, core_index, pu_index;
-    hwloc_obj_t socket, core, pu;
-    int **data;
-
-    /* Find out how many sockets we have */
-    num_sockets = hwloc_get_nbobjs_by_type(topo, HWLOC_OBJ_SOCKET);
-    /* some systems (like the iMac) only have one
-     * socket and so don't report a socket
-     */
-    if (0 == num_sockets) {
-        num_sockets = 1;
-    }
-    /* Lazy: take the total number of cores that we have in the
-       topology; that'll be more than the max number of cores
-       under any given socket */
-    num_cores = hwloc_get_nbobjs_by_type(topo, HWLOC_OBJ_CORE);
-    *num_sockets_arg = num_sockets;
-    *num_cores_arg = num_cores;
-
-    /* Alloc a 2D array: sockets x cores. */
-    data = malloc(num_sockets * sizeof(int *));
-    if (NULL == data) {
-        return OPAL_ERR_OUT_OF_RESOURCE;
-    }
-    data[0] = calloc(num_sockets * num_cores, sizeof(int));
-    if (NULL == data[0]) {
-        free(data);
-        return OPAL_ERR_OUT_OF_RESOURCE;
-    }
-    for (socket_index = 1; socket_index < num_sockets; ++socket_index) {
-        data[socket_index] = data[socket_index - 1] + num_cores;
-    }
-
-    /* Iterate the PUs in this cpuset; fill in the data[][] array with
-       the socket/core/pu triples */
-    for (pu_index = 0,
-             pu = hwloc_get_obj_inside_cpuset_by_type(topo,
-                                                      cpuset, HWLOC_OBJ_PU,
-                                                      pu_index);
-         NULL != pu;
-         pu = hwloc_get_obj_inside_cpuset_by_type(topo,
-                                                  cpuset, HWLOC_OBJ_PU,
-                                                  ++pu_index)) {
-        /* Go upward and find the core this PU belongs to */
-        core = pu;
-        while (NULL != core && core->type != HWLOC_OBJ_CORE) {
-            core = core->parent;
-        }
-        core_index = 0;
-        if (NULL != core) {
-            core_index = core->logical_index;
-        }
-
-        /* Go upward and find the socket this PU belongs to */
-        socket = pu;
-        while (NULL != socket && socket->type != HWLOC_OBJ_SOCKET) {
-            socket = socket->parent;
-        }
-        socket_index = 0;
-        if (NULL != socket) {
-            socket_index = socket->logical_index;
-        }
-
-        /* Save this socket/core/pu combo.  LAZY: Assuming that we
-           won't have more PU's per core than (sizeof(int)*8). */
-        data[socket_index][core_index] |= (1 << pu->sibling_rank);
-    }
-
-    *map = data;
-    return OPAL_SUCCESS;
-}
-
-/*
- * Make a prettyprint string for a hwloc_cpuset_t
- */
-int opal_hwloc_base_cset2str(char *str, int len,
-                             hwloc_topology_t topo,
-                             hwloc_cpuset_t cpuset)
-{
-    bool first;
-    int num_sockets, num_cores;
-    int ret, socket_index, core_index;
-    char tmp[BUFSIZ];
-    const int stmp = sizeof(tmp) - 1;
-    int **map=NULL;
-
-    str[0] = tmp[stmp] = '\0';
-
-    /* if the cpuset is all zero, then not bound */
-    if (hwloc_bitmap_iszero(cpuset)) {
-        return OPAL_ERR_NOT_BOUND;
-    }
-
-    if (OPAL_SUCCESS != (ret = build_map(&num_sockets, &num_cores, cpuset, &map, topo))) {
-        return ret;
-    }
-    /* Iterate over the data matrix and build up the string */
-    first = true;
-    for (socket_index = 0; socket_index < num_sockets; ++socket_index) {
-        for (core_index = 0; core_index < num_cores; ++core_index) {
-            if (map[socket_index][core_index] > 0) {
-                if (!first) {
-                    strncat(str, ", ", len - strlen(str) - 1);
-                }
-                first = false;
-
-                snprintf(tmp, stmp, "socket %d[core %d[hwt %s]]",
-                         socket_index, core_index,
-                         bitmap2rangestr(map[socket_index][core_index]));
-                strncat(str, tmp, len - strlen(str) - 1);
-            }
-        }
-    }
-    if (NULL != map) {
-        if (NULL != map[0]) {
-            free(map[0]);
-        }
-        free(map);
-    }
-
-    return OPAL_SUCCESS;
-}
-
-/*
- * Make a prettyprint string for a cset in a map format.
- * Example: [B./..]
- * Key:  [] - signifies socket
- *        / - divider between cores
- *        . - signifies PU a process not bound to
- *        B - signifies PU a process is bound to
- */
-int opal_hwloc_base_cset2mapstr(char *str, int len,
-                                hwloc_topology_t topo,
-                                hwloc_cpuset_t cpuset)
-{
-    char tmp[BUFSIZ];
-    int core_index, pu_index;
-    const int stmp = sizeof(tmp) - 1;
-    hwloc_obj_t socket, core, pu;
-
-    str[0] = tmp[stmp] = '\0';
-
-    /* if the cpuset is all zero, then not bound */
-    if (hwloc_bitmap_iszero(cpuset)) {
-        return OPAL_ERR_NOT_BOUND;
-    }
-
-    /* Iterate over all existing sockets */
-    for (socket = hwloc_get_obj_by_type(topo, HWLOC_OBJ_SOCKET, 0);
-         NULL != socket;
-         socket = socket->next_cousin) {
-        strncat(str, "[", len - strlen(str) - 1);
-
-        /* Iterate over all existing cores in this socket */
-        core_index = 0;
-        for (core = hwloc_get_obj_inside_cpuset_by_type(topo,
-                                                        socket->cpuset,
-                                                        HWLOC_OBJ_CORE, core_index);
-             NULL != core;
-             core = hwloc_get_obj_inside_cpuset_by_type(topo,
-                                                        socket->cpuset,
-                                                        HWLOC_OBJ_CORE, ++core_index)) {
-            if (core_index > 0) {
-                strncat(str, "/", len - strlen(str) - 1);
-            }
-
-            /* Iterate over all existing PUs in this core */
-            pu_index = 0;
-            for (pu = hwloc_get_obj_inside_cpuset_by_type(topo,
-                                                          core->cpuset,
-                                                          HWLOC_OBJ_PU, pu_index);
-                 NULL != pu;
-                 pu = hwloc_get_obj_inside_cpuset_by_type(topo,
-                                                          core->cpuset,
-                                                          HWLOC_OBJ_PU, ++pu_index)) {
-
-                /* Is this PU in the cpuset? */
-                if (hwloc_bitmap_isset(cpuset, pu->os_index)) {
-                    strncat(str, "B", len - strlen(str) - 1);
-                } else {
-                    strncat(str, ".", len - strlen(str) - 1);
-                }
-            }
-        }
-        strncat(str, "]", len - strlen(str) - 1);
-    }
-
-    return OPAL_SUCCESS;
-}
-
-char* opal_hwloc_base_get_location(char *locality,
-                                   hwloc_obj_type_t type,
-                                   unsigned index)
+char *opal_hwloc_base_get_location(char *locality, hwloc_obj_type_t type, unsigned index)
 {
     char **loc;
     char *srch, *ans = NULL;
@@ -1299,45 +553,45 @@ char* opal_hwloc_base_get_location(char *locality,
     if (NULL == locality) {
         return NULL;
     }
-    switch(type) {
-        case HWLOC_OBJ_NODE:
-            srch = "NM";
-            break;
-        case HWLOC_OBJ_SOCKET:
-            srch = "SK";
-            break;
+    switch (type) {
+    case HWLOC_OBJ_NODE:
+        srch = "NM";
+        break;
+    case HWLOC_OBJ_SOCKET:
+        srch = "SK";
+        break;
 #if HWLOC_API_VERSION < 0x20000
-        case HWLOC_OBJ_CACHE:
-            if (3 == index) {
-                srch = "L3";
-            } else if (2 == index) {
-                srch = "L2";
-            } else {
-                srch = "L1";
-            }
-            break;
-#else
-        case HWLOC_OBJ_L3CACHE:
+    case HWLOC_OBJ_CACHE:
+        if (3 == index) {
             srch = "L3";
-            break;
-        case HWLOC_OBJ_L2CACHE:
+        } else if (2 == index) {
             srch = "L2";
-            break;
-        case HWLOC_OBJ_L1CACHE:
+        } else {
             srch = "L1";
-            break;
+        }
+        break;
+#else
+    case HWLOC_OBJ_L3CACHE:
+        srch = "L3";
+        break;
+    case HWLOC_OBJ_L2CACHE:
+        srch = "L2";
+        break;
+    case HWLOC_OBJ_L1CACHE:
+        srch = "L1";
+        break;
 #endif
-        case HWLOC_OBJ_CORE:
-            srch = "CR";
-            break;
-        case HWLOC_OBJ_PU:
-            srch = "HT";
-            break;
-        default:
-            return NULL;
+    case HWLOC_OBJ_CORE:
+        srch = "CR";
+        break;
+    case HWLOC_OBJ_PU:
+        srch = "HT";
+        break;
+    default:
+        return NULL;
     }
     loc = opal_argv_split(locality, ':');
-    for (n=0; NULL != loc[n]; n++) {
+    for (n = 0; NULL != loc[n]; n++) {
         if (0 == strncmp(loc[n], srch, 2)) {
             ans = strdup(&loc[n][2]);
             break;
@@ -1372,11 +626,11 @@ opal_hwloc_locality_t opal_hwloc_compute_relative_locality(char *loc1, char *loc
     bit2 = hwloc_bitmap_alloc();
 
     /* check each matching type */
-    for (n1=0; NULL != set1[n1]; n1++) {
+    for (n1 = 0; NULL != set1[n1]; n1++) {
         /* convert the location into bitmap */
         hwloc_bitmap_list_sscanf(bit1, &set1[n1][2]);
         /* find the matching type in set2 */
-        for (n2=0; NULL != set2[n2]; n2++) {
+        for (n2 = 0; NULL != set2[n2]; n2++) {
             if (0 == strncmp(set1[n1], set2[n2], 2)) {
                 /* convert the location into bitmap */
                 hwloc_bitmap_list_sscanf(bit2, &set2[n2][2]);
@@ -1412,15 +666,3 @@ opal_hwloc_locality_t opal_hwloc_compute_relative_locality(char *loc1, char *loc
     hwloc_bitmap_free(bit2);
     return locality;
 }
-
-int opal_hwloc_base_topology_set_flags (hwloc_topology_t topology, unsigned long flags, bool io) {
-    if (io) {
-#if HWLOC_API_VERSION < 0x20000
-        flags |= HWLOC_TOPOLOGY_FLAG_IO_DEVICES;
-#else
-        int ret = hwloc_topology_set_io_types_filter(topology, HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
-        if (0 != ret) return ret;
-#endif
-    }
-    return hwloc_topology_set_flags(topology, flags);
-}
diff --git a/opal/mca/hwloc/base/static-components.h b/opal/mca/hwloc/base/static-components.h
index dd18f72da26..3e5b0f9b19d 100644
--- a/opal/mca/hwloc/base/static-components.h
+++ b/opal/mca/hwloc/base/static-components.h
@@ -5,14 +5,10 @@
 extern "C" {
 #endif
 
-
-
 const mca_base_component_t *mca_hwloc_base_static_components[] = {
 
-  NULL
-};
+    NULL};
 
 #if defined(c_plusplus) || defined(__cplusplus)
 }
 #endif
-
diff --git a/opal/mca/hwloc/hwloc-internal.h b/opal/mca/hwloc/hwloc-internal.h
index 57eae358afc..ea160dc3b45 100644
--- a/opal/mca/hwloc/hwloc-internal.h
+++ b/opal/mca/hwloc/hwloc-internal.h
@@ -26,26 +26,26 @@
 #include "opal_config.h"
 
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 #ifdef HAVE_SYS_TIME_H
-#include <sys/time.h>
+#    include <sys/time.h>
 #endif
-#include <stdint.h>
 #include <stdarg.h>
+#include <stdint.h>
 
 #include "opal/class/opal_list.h"
 #include "opal/class/opal_value_array.h"
 
-#include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
+#include "opal/mca/mca.h"
 
 BEGIN_C_DECLS
 
 #ifdef WIN32
-#define WIN32_LEAN_AND_MEAN
-#include <windows.h>
-#undef WIN32_LEAN_AND_MEAN
+#    define WIN32_LEAN_AND_MEAN
+#    include <windows.h>
+#    undef WIN32_LEAN_AND_MEAN
 typedef unsigned char u_char;
 typedef unsigned short u_short;
 #endif
@@ -69,9 +69,7 @@ typedef struct opal_hwloc_base_component_2_0_0_t opal_hwloc_component_t;
 /**
  * Macro for use in components that are of type hwloc
  */
-#define OPAL_HWLOC_BASE_VERSION_2_0_0 \
-    OPAL_MCA_BASE_VERSION_2_1_0("hwloc", 2, 0, 0)
-
+#define OPAL_HWLOC_BASE_VERSION_2_0_0 OPAL_MCA_BASE_VERSION_2_1_0("hwloc", 2, 0, 0)
 
 /* ******************************************************************** */
 /* Although we cannot bind if --without-hwloc is set,
@@ -85,36 +83,36 @@ typedef uint16_t opal_hwloc_locality_t;
 
 /** Process locality definitions */
 enum {
-    OPAL_PROC_LOCALITY_UNKNOWN  = 0x0000,
-    OPAL_PROC_NON_LOCAL         = 0x8000,
-    OPAL_PROC_ON_CLUSTER        = 0x0001,
-    OPAL_PROC_ON_CU             = 0x0002,
-    OPAL_PROC_ON_HOST           = 0x0004,
-    OPAL_PROC_ON_BOARD          = 0x0008,
-    OPAL_PROC_ON_NODE           = 0x000c,   // same host and board
-    OPAL_PROC_ON_NUMA           = 0x0010,
-    OPAL_PROC_ON_SOCKET         = 0x0020,
-    OPAL_PROC_ON_L3CACHE        = 0x0040,
-    OPAL_PROC_ON_L2CACHE        = 0x0080,
-    OPAL_PROC_ON_L1CACHE        = 0x0100,
-    OPAL_PROC_ON_CORE           = 0x0200,
-    OPAL_PROC_ON_HWTHREAD       = 0x0400,
-    OPAL_PROC_ALL_LOCAL         = 0x0fff,
+    OPAL_PROC_LOCALITY_UNKNOWN = 0x0000,
+    OPAL_PROC_NON_LOCAL = 0x8000,
+    OPAL_PROC_ON_CLUSTER = 0x0001,
+    OPAL_PROC_ON_CU = 0x0002,
+    OPAL_PROC_ON_HOST = 0x0004,
+    OPAL_PROC_ON_BOARD = 0x0008,
+    OPAL_PROC_ON_NODE = 0x000c, // same host and board
+    OPAL_PROC_ON_NUMA = 0x0010,
+    OPAL_PROC_ON_SOCKET = 0x0020,
+    OPAL_PROC_ON_L3CACHE = 0x0040,
+    OPAL_PROC_ON_L2CACHE = 0x0080,
+    OPAL_PROC_ON_L1CACHE = 0x0100,
+    OPAL_PROC_ON_CORE = 0x0200,
+    OPAL_PROC_ON_HWTHREAD = 0x0400,
+    OPAL_PROC_ALL_LOCAL = 0x0fff,
 };
 
 /** Process locality macros */
-#define OPAL_PROC_ON_LOCAL_CLUSTER(n)   (!!((n) & OPAL_PROC_ON_CLUSTER))
-#define OPAL_PROC_ON_LOCAL_CU(n)        (!!((n) & OPAL_PROC_ON_CU))
-#define OPAL_PROC_ON_LOCAL_HOST(n)      (!!((n) & OPAL_PROC_ON_HOST))
-#define OPAL_PROC_ON_LOCAL_BOARD(n)     (!!((n) & OPAL_PROC_ON_BOARD))
-#define OPAL_PROC_ON_LOCAL_NODE(n)      (OPAL_PROC_ON_LOCAL_HOST(n) && OPAL_PROC_ON_LOCAL_BOARD(n))
-#define OPAL_PROC_ON_LOCAL_NUMA(n)      (!!((n) & OPAL_PROC_ON_NUMA))
-#define OPAL_PROC_ON_LOCAL_SOCKET(n)    (!!((n) & OPAL_PROC_ON_SOCKET))
-#define OPAL_PROC_ON_LOCAL_L3CACHE(n)   (!!((n) & OPAL_PROC_ON_L3CACHE))
-#define OPAL_PROC_ON_LOCAL_L2CACHE(n)   (!!((n) & OPAL_PROC_ON_L2CACHE))
-#define OPAL_PROC_ON_LOCAL_L1CACHE(n)   (!!((n) & OPAL_PROC_ON_L1CACHE))
-#define OPAL_PROC_ON_LOCAL_CORE(n)      (!!((n) & OPAL_PROC_ON_CORE))
-#define OPAL_PROC_ON_LOCAL_HWTHREAD(n)  (!!((n) & OPAL_PROC_ON_HWTHREAD))
+#define OPAL_PROC_ON_LOCAL_CLUSTER(n)  (!!((n) &OPAL_PROC_ON_CLUSTER))
+#define OPAL_PROC_ON_LOCAL_CU(n)       (!!((n) &OPAL_PROC_ON_CU))
+#define OPAL_PROC_ON_LOCAL_HOST(n)     (!!((n) &OPAL_PROC_ON_HOST))
+#define OPAL_PROC_ON_LOCAL_BOARD(n)    (!!((n) &OPAL_PROC_ON_BOARD))
+#define OPAL_PROC_ON_LOCAL_NODE(n)     (OPAL_PROC_ON_LOCAL_HOST(n) && OPAL_PROC_ON_LOCAL_BOARD(n))
+#define OPAL_PROC_ON_LOCAL_NUMA(n)     (!!((n) &OPAL_PROC_ON_NUMA))
+#define OPAL_PROC_ON_LOCAL_SOCKET(n)   (!!((n) &OPAL_PROC_ON_SOCKET))
+#define OPAL_PROC_ON_LOCAL_L3CACHE(n)  (!!((n) &OPAL_PROC_ON_L3CACHE))
+#define OPAL_PROC_ON_LOCAL_L2CACHE(n)  (!!((n) &OPAL_PROC_ON_L2CACHE))
+#define OPAL_PROC_ON_LOCAL_L1CACHE(n)  (!!((n) &OPAL_PROC_ON_L1CACHE))
+#define OPAL_PROC_ON_LOCAL_CORE(n)     (!!((n) &OPAL_PROC_ON_CORE))
+#define OPAL_PROC_ON_LOCAL_HWTHREAD(n) (!!((n) &OPAL_PROC_ON_HWTHREAD))
 
 /* ******************************************************************** */
 
@@ -143,21 +141,22 @@ typedef struct {
 #    if HWLOC_API_VERSION >= 0x20000
 #        include <hwloc/shmem.h>
 #    endif
-/* Do nothing in the 1.x case because the caller doesn't know HWLOC_API_VERSION when it sets OPAL_HWLOC_WANT_SHMEM.
- * Calls to hwloc/shmem.h are protected by HWLOC_API_VERSION >= 0x20000 in the actual code.
+/* Do nothing in the 1.x case because the caller doesn't know HWLOC_API_VERSION when it sets
+ * OPAL_HWLOC_WANT_SHMEM. Calls to hwloc/shmem.h are protected by HWLOC_API_VERSION >= 0x20000 in
+ * the actual code.
  */
 #endif
 
 #if HWLOC_API_VERSION < 0x00010b00
-#define HWLOC_OBJ_NUMANODE HWLOC_OBJ_NODE
-#define HWLOC_OBJ_PACKAGE HWLOC_OBJ_SOCKET
+#    define HWLOC_OBJ_NUMANODE HWLOC_OBJ_NODE
+#    define HWLOC_OBJ_PACKAGE  HWLOC_OBJ_SOCKET
 #endif
 
 /* define type of processor info requested */
 typedef uint8_t opal_hwloc_resource_type_t;
-#define OPAL_HWLOC_PHYSICAL   1
-#define OPAL_HWLOC_LOGICAL    2
-#define OPAL_HWLOC_AVAILABLE  3
+#define OPAL_HWLOC_PHYSICAL  1
+#define OPAL_HWLOC_LOGICAL   2
+#define OPAL_HWLOC_AVAILABLE 3
 
 /* structs for storing info on objects */
 typedef struct {
@@ -189,65 +188,9 @@ typedef struct {
 } opal_hwloc_topo_data_t;
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_hwloc_topo_data_t);
 
-/* define binding policies */
-typedef uint16_t opal_binding_policy_t;
-#define OPAL_BINDING_POLICY OPAL_UINT16
-
-/* binding directives */
-#define OPAL_BIND_IF_SUPPORTED      0x1000
-/* allow assignment of multiple procs to
- * same cpu */
-#define OPAL_BIND_ALLOW_OVERLOAD    0x2000
-/* the binding policy was specified by the user */
-#define OPAL_BIND_GIVEN             0x4000
-/* bind each rank to the cpu in the given
- * cpu list based on its node-local-rank */
-#define OPAL_BIND_ORDERED           0x8000
-
-/* binding policies - any changes in these
- * values must be reflected in orte/mca/rmaps/rmaps.h
- */
-#define OPAL_BIND_TO_NONE           1
-#define OPAL_BIND_TO_BOARD          2
-#define OPAL_BIND_TO_NUMA           3
-#define OPAL_BIND_TO_SOCKET         4
-#define OPAL_BIND_TO_L3CACHE        5
-#define OPAL_BIND_TO_L2CACHE        6
-#define OPAL_BIND_TO_L1CACHE        7
-#define OPAL_BIND_TO_CORE           8
-#define OPAL_BIND_TO_HWTHREAD       9
-#define OPAL_BIND_TO_CPUSET         10
-#define OPAL_GET_BINDING_POLICY(pol) \
-    ((pol) & 0x0fff)
-#define OPAL_SET_BINDING_POLICY(target, pol) \
-    (target) = (pol) | (((target) & 0xf000) | OPAL_BIND_GIVEN)
-#define OPAL_SET_DEFAULT_BINDING_POLICY(target, pol)            \
-    do {                                                        \
-        if (!OPAL_BINDING_POLICY_IS_SET((target))) {            \
-            (target) = (pol) | (((target) & 0xf000) |           \
-                                OPAL_BIND_IF_SUPPORTED);        \
-        }                                                       \
-    } while(0);
-
-/* check if policy is set */
-#define OPAL_BINDING_POLICY_IS_SET(pol) \
-    ((pol) & 0x4000)
-/* macro to detect if binding was qualified */
-#define OPAL_BINDING_REQUIRED(n) \
-    (!(OPAL_BIND_IF_SUPPORTED & (n)))
-/* macro to detect if binding is forced */
-#define OPAL_BIND_OVERLOAD_ALLOWED(n) \
-    (OPAL_BIND_ALLOW_OVERLOAD & (n))
-#define OPAL_BIND_ORDERED_REQUESTED(n) \
-    (OPAL_BIND_ORDERED & (n))
-
 /* some global values */
 OPAL_DECLSPEC extern hwloc_topology_t opal_hwloc_topology;
-OPAL_DECLSPEC extern opal_binding_policy_t opal_hwloc_binding_policy;
 OPAL_DECLSPEC extern hwloc_cpuset_t opal_hwloc_my_cpuset;
-OPAL_DECLSPEC extern bool opal_hwloc_report_bindings;
-OPAL_DECLSPEC extern hwloc_obj_type_t opal_hwloc_levels[];
-OPAL_DECLSPEC extern bool opal_hwloc_use_hwthreads_as_cpus;
 
 END_C_DECLS
 
diff --git a/opal/mca/if/base/if_base_components.c b/opal/mca/if/base/if_base_components.c
index 7c1322a5a99..86d2aea4ee6 100644
--- a/opal/mca/if/base/if_base_components.c
+++ b/opal/mca/if/base/if_base_components.c
@@ -14,19 +14,19 @@
 #include "opal_config.h"
 
 #include "opal/constants.h"
-#include "opal/util/output.h"
-#include "opal/mca/mca.h"
-#include "opal/mca/if/if.h"
 #include "opal/mca/if/base/base.h"
 #include "opal/mca/if/base/static-components.h"
+#include "opal/mca/if/if.h"
+#include "opal/mca/mca.h"
+#include "opal/util/output.h"
 
 /* instantiate the global list of interfaces */
 opal_list_t opal_if_list = {{0}};
 bool opal_if_do_not_resolve = false;
 bool opal_if_retain_loopback = false;
 
-static int opal_if_base_register (mca_base_register_flag_t flags);
-static int opal_if_base_open (mca_base_open_flag_t flags);
+static int opal_if_base_register(mca_base_register_flag_t flags);
+static int opal_if_base_open(mca_base_open_flag_t flags);
 static int opal_if_base_close(void);
 static void opal_if_construct(opal_if_t *obj);
 
@@ -35,30 +35,29 @@ static bool frameopen = false;
 /* instance the opal_if_t object */
 OBJ_CLASS_INSTANCE(opal_if_t, opal_list_item_t, opal_if_construct, NULL);
 
-MCA_BASE_FRAMEWORK_DECLARE(opal, if, NULL, opal_if_base_register, opal_if_base_open, opal_if_base_close,
-                           mca_if_base_static_components, 0);
+MCA_BASE_FRAMEWORK_DECLARE(opal, if, NULL, opal_if_base_register, opal_if_base_open,
+                           opal_if_base_close, mca_if_base_static_components, 0);
 
-static int opal_if_base_register (mca_base_register_flag_t flags)
+static int opal_if_base_register(mca_base_register_flag_t flags)
 {
     opal_if_do_not_resolve = false;
-    (void) mca_base_framework_var_register (&opal_if_base_framework, "do_not_resolve",
-                                            "If nonzero, do not attempt to resolve interfaces",
-                                            MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-                                            OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_ALL_EQ,
-                                            &opal_if_do_not_resolve);
+    (void) mca_base_framework_var_register(&opal_if_base_framework, "do_not_resolve",
+                                           "If nonzero, do not attempt to resolve interfaces",
+                                           MCA_BASE_VAR_TYPE_BOOL, NULL, 0,
+                                           MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_ALL_EQ, &opal_if_do_not_resolve);
 
     opal_if_retain_loopback = false;
-    (void) mca_base_framework_var_register (&opal_if_base_framework, "retain_loopback",
-                                            "If nonzero, retain loopback interfaces",
-                                            MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-                                            OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_ALL_EQ,
-                                            &opal_if_retain_loopback);
+    (void) mca_base_framework_var_register(&opal_if_base_framework, "retain_loopback",
+                                           "If nonzero, retain loopback interfaces",
+                                           MCA_BASE_VAR_TYPE_BOOL, NULL, 0,
+                                           MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_ALL_EQ, &opal_if_retain_loopback);
 
     return OPAL_SUCCESS;
 }
 
-
-static int opal_if_base_open (mca_base_open_flag_t flags)
+static int opal_if_base_open(mca_base_open_flag_t flags)
 {
     if (frameopen) {
         return OPAL_SUCCESS;
@@ -71,7 +70,6 @@ static int opal_if_base_open (mca_base_open_flag_t flags)
     return mca_base_framework_components_open(&opal_if_base_framework, flags);
 }
 
-
 static int opal_if_base_close(void)
 {
     opal_list_item_t *item;
diff --git a/opal/mca/if/bsdx_ipv4/configure.m4 b/opal/mca/if/bsdx_ipv4/configure.m4
index d572cc44d70..a373b58f949 100644
--- a/opal/mca/if/bsdx_ipv4/configure.m4
+++ b/opal/mca/if/bsdx_ipv4/configure.m4
@@ -3,6 +3,8 @@
 # Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2015      Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2025      Nanook Consulting  All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -22,25 +24,26 @@ AC_DEFUN([MCA_opal_if_bsdx_ipv4_COMPILE_MODE], [
 AC_DEFUN([MCA_opal_if_bsdx_ipv4_CONFIG], [
     AC_CONFIG_FILES([opal/mca/if/bsdx_ipv4/Makefile])
 
-    AC_REQUIRE([OPAL_CHECK_OS_FLAVORS])
+    AC_REQUIRE([OAC_CHECK_OS_FLAVORS])
 
-    # If we found struct sockaddr and we're on any of the BSDs, we're
+    # If we previously found struct sockaddr_in (we don't repeat the
+    # AC CHECK_TYPES test here simply because it's cumbersome with all
+    # the required #includes) and we're on any of the BSDs, we're
     # happy.  I.e., this:
     #if defined(__NetBSD__) || defined(__FreeBSD__) || \
     #    defined(__OpenBSD__) || defined(__DragonFly__)
-    AC_MSG_CHECKING([struct sockaddr])
-    AS_IF([test "$opal_found_sockaddr" = "yes"],
-          [AC_MSG_RESULT([yes (cached)])
+    AC_MSG_CHECKING([struct sockaddr_in again])
+    AS_IF([test "$ac_cv_type_struct_sockaddr_in" = "yes"],
+          [AC_MSG_RESULT([yes (OPAL cached)])
            AC_MSG_CHECKING([NetBSD, FreeBSD, OpenBSD, or DragonFly])
-           AS_IF([test "$opal_found_netbsd" = "yes" || \
-                  test "$opal_found_freebsd" = "yes" || \
-                  test "$opal_found_openbsd" = "yes" || \
-                  test "$opal_found_dragonfly" = "yes"],
+           AS_IF([test "$oac_found_netbsd" = "yes" || \
+                  test "$oac_found_freebsd" = "yes" || \
+                  test "$oac_found_openbsd" = "yes" || \
+                  test "$oac_found_dragonfly" = "yes"],
                  [AC_MSG_RESULT([yes])
                   $1],
                  [AC_MSG_RESULT([no])
                   $2])],
-          [AC_MSG_RESULT([no (cached)])
+          [AC_MSG_RESULT([no (OPAL cached)])
            $2])
 ])
-
diff --git a/opal/mca/if/bsdx_ipv4/if_bsdx.c b/opal/mca/if/bsdx_ipv4/if_bsdx.c
index 1043ddca84b..86fa2b5e784 100644
--- a/opal/mca/if/bsdx_ipv4/if_bsdx.c
+++ b/opal/mca/if/bsdx_ipv4/if_bsdx.c
@@ -14,9 +14,9 @@
 #include <string.h>
 
 #include "opal/constants.h"
+#include "opal/mca/if/if.h"
 #include "opal/util/output.h"
 #include "opal/util/string_copy.h"
-#include "opal/mca/if/if.h"
 
 static int if_bsdx_open(void);
 
@@ -29,27 +29,19 @@ static int if_bsdx_open(void);
 opal_if_base_component_t mca_if_bsdx_ipv4_component = {
     /* First, the mca_component_t struct containing meta information
        about the component itself */
-    {
-        OPAL_IF_BASE_VERSION_2_0_0,
-
-        /* Component name and version */
-        "bsdx_ipv4",
-        OPAL_MAJOR_VERSION,
-        OPAL_MINOR_VERSION,
-        OPAL_RELEASE_VERSION,
-
-        /* Component open and close functions */
-        if_bsdx_open,
-        NULL
-    },
-    {
-        /* This component is checkpointable */
-        MCA_BASE_METADATA_PARAM_CHECKPOINT
-    },
+    {OPAL_IF_BASE_VERSION_2_0_0,
+
+     /* Component name and version */
+     "bsdx_ipv4", OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION, OPAL_RELEASE_VERSION,
+
+     /* Component open and close functions */
+     if_bsdx_open, NULL},
+    {/* This component is checkpointable */
+     MCA_BASE_METADATA_PARAM_CHECKPOINT},
 };
 
 /* convert a netmask (in network byte order) to CIDR notation */
-static int prefix (uint32_t netmask)
+static int prefix(uint32_t netmask)
 {
     uint32_t mask = ntohl(netmask);
     int plen = 0;
@@ -71,24 +63,22 @@ static int if_bsdx_open(void)
 {
     struct ifaddrs **ifadd_list;
     struct ifaddrs *cur_ifaddrs;
-    struct sockaddr_in* sin_addr;
+    struct sockaddr_in *sin_addr;
 
     /*
      * the manpage claims that getifaddrs() allocates the memory,
      * and freeifaddrs() is later used to release the allocated memory.
      * however, without this malloc the call to getifaddrs() segfaults
      */
-    ifadd_list = (struct ifaddrs **) malloc(sizeof(struct ifaddrs*));
+    ifadd_list = (struct ifaddrs **) malloc(sizeof(struct ifaddrs *));
 
     /* create the linked list of ifaddrs structs */
     if (getifaddrs(ifadd_list) < 0) {
-        opal_output(0, "opal_ifinit: getifaddrs() failed with error=%d\n",
-                    errno);
+        opal_output(0, "opal_ifinit: getifaddrs() failed with error=%d\n", errno);
         return OPAL_ERROR;
     }
 
-    for (cur_ifaddrs = *ifadd_list; NULL != cur_ifaddrs;
-         cur_ifaddrs = cur_ifaddrs->ifa_next) {
+    for (cur_ifaddrs = *ifadd_list; NULL != cur_ifaddrs; cur_ifaddrs = cur_ifaddrs->ifa_next) {
         opal_if_t *intf;
         struct in_addr a4;
 
@@ -117,8 +107,7 @@ static int if_bsdx_open(void)
 
         intf = OBJ_NEW(opal_if_t);
         if (NULL == intf) {
-            opal_output(0, "opal_ifinit: unable to allocate %d bytes\n",
-                        (int) sizeof(opal_if_t));
+            opal_output(0, "opal_ifinit: unable to allocate %d bytes\n", (int) sizeof(opal_if_t));
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
         intf->af_family = AF_INET;
@@ -128,20 +117,17 @@ static int if_bsdx_open(void)
 
         opal_string_copy(intf->if_name, cur_ifaddrs->ifa_name, OPAL_IF_NAMESIZE);
         intf->if_index = opal_list_get_size(&opal_if_list) + 1;
-        ((struct sockaddr_in*) &intf->if_addr)->sin_addr = a4;
-        ((struct sockaddr_in*) &intf->if_addr)->sin_family = AF_INET;
-        ((struct sockaddr_in*) &intf->if_addr)->sin_len =  cur_ifaddrs->ifa_addr->sa_len;
+        ((struct sockaddr_in *) &intf->if_addr)->sin_addr = a4;
+        ((struct sockaddr_in *) &intf->if_addr)->sin_family = AF_INET;
+        ((struct sockaddr_in *) &intf->if_addr)->sin_len = cur_ifaddrs->ifa_addr->sa_len;
 
-        intf->if_mask = prefix( sin_addr->sin_addr.s_addr);
+        intf->if_mask = prefix(sin_addr->sin_addr.s_addr);
         intf->if_flags = cur_ifaddrs->ifa_flags;
 
-        intf->if_kernel_index =
-            (uint16_t) if_nametoindex(cur_ifaddrs->ifa_name);
+        intf->if_kernel_index = (uint16_t) if_nametoindex(cur_ifaddrs->ifa_name);
 
         opal_list_append(&opal_if_list, &(intf->super));
-    }   /*  of for loop over ifaddrs list */
+    } /*  of for loop over ifaddrs list */
 
     return OPAL_SUCCESS;
 }
-
-
diff --git a/opal/mca/if/bsdx_ipv6/configure.m4 b/opal/mca/if/bsdx_ipv6/configure.m4
index 4b2122f7012..ed38208e65e 100644
--- a/opal/mca/if/bsdx_ipv6/configure.m4
+++ b/opal/mca/if/bsdx_ipv6/configure.m4
@@ -3,6 +3,8 @@
 # Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2015      Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2025      Nanook Consulting  All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -22,26 +24,28 @@ AC_DEFUN([MCA_opal_if_bsdx_ipv6_COMPILE_MODE], [
 AC_DEFUN([MCA_opal_if_bsdx_ipv6_CONFIG], [
     AC_CONFIG_FILES([opal/mca/if/bsdx_ipv6/Makefile])
 
-    AC_REQUIRE([OPAL_CHECK_OS_FLAVORS])
+    AC_REQUIRE([OAC_CHECK_OS_FLAVORS])
 
-    # If we found struct sockaddr and we're on any of the BSDs, we're
+    # If we previously found struct sockaddr_in6 (we don't repeat the
+    # AC CHECK_TYPES test here simply because it's cumbersome with all
+    # the required #includes) and we're on any of the BSDs, we're
     # happy.  I.e., this:
     #if defined( __NetBSD__) || defined(__OpenBSD__) || defined(__FreeBSD__) || \
     #             defined(__386BSD__) || defined(__bsdi__) || defined(__APPLE__)
-    AC_MSG_CHECKING([struct sockaddr])
-    AS_IF([test "$opal_found_sockaddr" = "yes"],
-          [AC_MSG_RESULT([yes (cached)])
+    AC_MSG_CHECKING([struct sockaddr_in6 again])
+    AS_IF([test "$ac_cv_type_struct_sockaddr_in6" = "yes"],
+          [AC_MSG_RESULT([yes (OPAL cached)])
            AC_MSG_CHECKING([some flavor of BSD])
-           AS_IF([test "$opal_found_netbsd" = "yes" || \
-                  test "$opal_found_freebsd" = "yes" || \
-                  test "$opal_found_openbsd" = "yes" || \
-                  test "$opal_found_386bsd" = "yes" || \
-                  test "$opal_found_bsdi" = "yes" ||
-                  test "$opal_found_apple" = "yes"],
+           AS_IF([test "$oac_found_netbsd" = "yes" || \
+                  test "$oac_found_freebsd" = "yes" || \
+                  test "$oac_found_openbsd" = "yes" || \
+                  test "$oac_found_386bsd" = "yes" || \
+                  test "$oac_found_bsdi" = "yes" ||
+                  test "$oac_found_apple" = "yes"],
                  [AC_MSG_RESULT([yes])
                   $1],
                  [AC_MSG_RESULT([no])
                   $2])],
-          [AC_MSG_RESULT([no (cached)])
+          [AC_MSG_RESULT([no (OPAL cached)])
            $2])
 ])dnl
diff --git a/opal/mca/if/bsdx_ipv6/if_bsdx_ipv6.c b/opal/mca/if/bsdx_ipv6/if_bsdx_ipv6.c
index 7716f7933b2..7b219ac9e97 100644
--- a/opal/mca/if/bsdx_ipv6/if_bsdx_ipv6.c
+++ b/opal/mca/if/bsdx_ipv6/if_bsdx_ipv6.c
@@ -17,39 +17,39 @@
 
 #include <string.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 #include <errno.h>
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 #ifdef HAVE_SYS_SOCKET_H
-#include <sys/socket.h>
+#    include <sys/socket.h>
 #endif
 #ifdef HAVE_SYS_SOCKIO_H
-#include <sys/sockio.h>
+#    include <sys/sockio.h>
 #endif
 #ifdef HAVE_SYS_IOCTL_H
-#include <sys/ioctl.h>
+#    include <sys/ioctl.h>
 #endif
 #ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
+#    include <netinet/in.h>
 #endif
 #ifdef HAVE_ARPA_INET_H
-#include <arpa/inet.h>
+#    include <arpa/inet.h>
 #endif
 #ifdef HAVE_NET_IF_H
-#include <net/if.h>
+#    include <net/if.h>
 #endif
 #ifdef HAVE_NETDB_H
-#include <netdb.h>
+#    include <netdb.h>
 #endif
 #ifdef HAVE_IFADDRS_H
-#include <ifaddrs.h>
+#    include <ifaddrs.h>
 #endif
 
-#include "opal/mca/if/if.h"
 #include "opal/mca/if/base/base.h"
+#include "opal/mca/if/if.h"
 
 static int if_bsdx_ipv6_open(void);
 
@@ -65,23 +65,15 @@ static int if_bsdx_ipv6_open(void);
 opal_if_base_component_t mca_if_bsdx_ipv6_component = {
     /* First, the mca_component_t struct containing meta information
        about the component itself */
-    {
-        OPAL_IF_BASE_VERSION_2_0_0,
-
-        /* Component name and version */
-        "bsdx_ipv6",
-        OPAL_MAJOR_VERSION,
-        OPAL_MINOR_VERSION,
-        OPAL_RELEASE_VERSION,
-
-        /* Component open and close functions */
-        if_bsdx_ipv6_open,
-        NULL
-    },
-    {
-        /* This component is checkpointable */
-        MCA_BASE_METADATA_PARAM_CHECKPOINT
-    },
+    {OPAL_IF_BASE_VERSION_2_0_0,
+
+     /* Component name and version */
+     "bsdx_ipv6", OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION, OPAL_RELEASE_VERSION,
+
+     /* Component open and close functions */
+     if_bsdx_ipv6_open, NULL},
+    {/* This component is checkpointable */
+     MCA_BASE_METADATA_PARAM_CHECKPOINT},
 };
 
 /* configure using getifaddrs(3) */
@@ -90,7 +82,7 @@ static int if_bsdx_ipv6_open(void)
 #if OPAL_ENABLE_IPV6
     struct ifaddrs **ifadd_list;
     struct ifaddrs *cur_ifaddrs;
-    struct sockaddr_in6* sin_addr;
+    struct sockaddr_in6 *sin_addr;
 
     opal_output_verbose(1, opal_if_base_framework.framework_output,
                         "searching for IPv6 interfaces");
@@ -100,26 +92,24 @@ static int if_bsdx_ipv6_open(void)
      * and freeifaddrs() is later used to release the allocated memory.
      * however, without this malloc the call to getifaddrs() segfaults
      */
-    ifadd_list = (struct ifaddrs **) malloc(sizeof(struct ifaddrs*));
+    ifadd_list = (struct ifaddrs **) malloc(sizeof(struct ifaddrs *));
 
     /* create the linked list of ifaddrs structs */
     if (getifaddrs(ifadd_list) < 0) {
-        opal_output(0, "opal_ifinit: getifaddrs() failed with error=%d\n",
-                    errno);
+        opal_output(0, "opal_ifinit: getifaddrs() failed with error=%d\n", errno);
         free(ifadd_list);
         return OPAL_ERROR;
     }
 
-    for (cur_ifaddrs = *ifadd_list; NULL != cur_ifaddrs;
-         cur_ifaddrs = cur_ifaddrs->ifa_next) {
+    for (cur_ifaddrs = *ifadd_list; NULL != cur_ifaddrs; cur_ifaddrs = cur_ifaddrs->ifa_next) {
         opal_if_t *intf;
         struct in6_addr a6;
 
         /* skip non-ipv6 interface addresses */
         if (AF_INET6 != cur_ifaddrs->ifa_addr->sa_family) {
             opal_output_verbose(1, opal_if_base_framework.framework_output,
-                                "skipping non-ipv6 interface %s[%d].\n",
-                                cur_ifaddrs->ifa_name, (int)cur_ifaddrs->ifa_addr->sa_family);
+                                "skipping non-ipv6 interface %s[%d].\n", cur_ifaddrs->ifa_name,
+                                (int) cur_ifaddrs->ifa_addr->sa_family);
             continue;
         }
 
@@ -139,7 +129,7 @@ static int if_bsdx_ipv6_open(void)
 
         /* or if it is a point-to-point interface */
         /* TODO: do we really skip p2p? */
-        if (0!= (cur_ifaddrs->ifa_flags & IFF_POINTOPOINT)) {
+        if (0 != (cur_ifaddrs->ifa_flags & IFF_POINTOPOINT)) {
             opal_output_verbose(1, opal_if_base_framework.framework_output,
                                 "skipping p2p interface %s.\n", cur_ifaddrs->ifa_name);
             continue;
@@ -158,7 +148,7 @@ static int if_bsdx_ipv6_open(void)
          * so the scope returned by getifaddrs() isn't working properly
          */
 
-        if ((IN6_IS_ADDR_LINKLOCAL (&sin_addr->sin6_addr))) {
+        if ((IN6_IS_ADDR_LINKLOCAL(&sin_addr->sin6_addr))) {
             opal_output_verbose(1, opal_if_base_framework.framework_output,
                                 "skipping link-local ipv6 address on interface "
                                 "%s with scope %d.\n",
@@ -167,8 +157,8 @@ static int if_bsdx_ipv6_open(void)
         }
 
         if (0 < opal_output_get_verbosity(opal_if_base_framework.framework_output)) {
-            char *addr_name = (char *) malloc(48*sizeof(char));
-            inet_ntop(AF_INET6, &sin_addr->sin6_addr, addr_name, 48*sizeof(char));
+            char *addr_name = (char *) malloc(48 * sizeof(char));
+            inet_ntop(AF_INET6, &sin_addr->sin6_addr, addr_name, 48 * sizeof(char));
             opal_output(0, "ipv6 capable interface %s discovered, address %s.\n",
                         cur_ifaddrs->ifa_name, addr_name);
             free(addr_name);
@@ -179,19 +169,18 @@ static int if_bsdx_ipv6_open(void)
 
         intf = OBJ_NEW(opal_if_t);
         if (NULL == intf) {
-            opal_output(0, "opal_ifinit: unable to allocate %lu bytes\n",
-                        sizeof(opal_if_t));
+            opal_output(0, "opal_ifinit: unable to allocate %lu bytes\n", sizeof(opal_if_t));
             free(ifadd_list);
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
         intf->af_family = AF_INET6;
         opal_string_copy(intf->if_name, cur_ifaddrs->ifa_name, OPAL_IF_NAMESIZE);
         intf->if_index = opal_list_get_size(&opal_if_list) + 1;
-        ((struct sockaddr_in6*) &intf->if_addr)->sin6_addr = a6;
-        ((struct sockaddr_in6*) &intf->if_addr)->sin6_family = AF_INET6;
+        ((struct sockaddr_in6 *) &intf->if_addr)->sin6_addr = a6;
+        ((struct sockaddr_in6 *) &intf->if_addr)->sin6_family = AF_INET6;
 
         /* since every scope != 0 is ignored, we just set the scope to 0 */
-        ((struct sockaddr_in6*) &intf->if_addr)->sin6_scope_id = 0;
+        ((struct sockaddr_in6 *) &intf->if_addr)->sin6_scope_id = 0;
 
         /*
          * hardcoded netmask, adrian says that's ok
@@ -204,15 +193,12 @@ static int if_bsdx_ipv6_open(void)
          * (or create our own), getifaddrs() does not contain such
          * data
          */
-        intf->if_kernel_index =
-            (uint16_t) if_nametoindex(cur_ifaddrs->ifa_name);
+        intf->if_kernel_index = (uint16_t) if_nametoindex(cur_ifaddrs->ifa_name);
         opal_list_append(&opal_if_list, &(intf->super));
-    }   /*  of for loop over ifaddrs list */
+    } /*  of for loop over ifaddrs list */
 
     free(ifadd_list);
-#endif  /* OPAL_ENABLE_IPV6 */
+#endif /* OPAL_ENABLE_IPV6 */
 
     return OPAL_SUCCESS;
 }
-
-
diff --git a/opal/mca/if/if.h b/opal/mca/if/if.h
index c2993aebb72..d2f8f68d9b6 100644
--- a/opal/mca/if/if.h
+++ b/opal/mca/if/if.h
@@ -19,40 +19,40 @@
 
 #include <string.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 #include <errno.h>
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 #ifdef HAVE_SYS_SOCKET_H
-#include <sys/socket.h>
+#    include <sys/socket.h>
 #endif
 #ifdef HAVE_SYS_SOCKIO_H
-#include <sys/sockio.h>
+#    include <sys/sockio.h>
 #endif
 #ifdef HAVE_SYS_IOCTL_H
-#include <sys/ioctl.h>
+#    include <sys/ioctl.h>
 #endif
 #ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
+#    include <netinet/in.h>
 #endif
 #ifdef HAVE_ARPA_INET_H
-#include <arpa/inet.h>
+#    include <arpa/inet.h>
 #endif
 #ifdef HAVE_NET_IF_H
-#include <net/if.h>
+#    include <net/if.h>
 #endif
 #ifdef HAVE_NETDB_H
-#include <netdb.h>
+#    include <netdb.h>
 #endif
 #ifdef HAVE_IFADDRS_H
-#include <ifaddrs.h>
+#    include <ifaddrs.h>
 #endif
 
-#include "opal/util/if.h"
-#include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
+#include "opal/mca/mca.h"
+#include "opal/util/if.h"
 
 BEGIN_C_DECLS
 
@@ -65,31 +65,29 @@ BEGIN_C_DECLS
  */
 
 #if !defined(INADDR_NONE)
-#define INADDR_NONE -1
+#    define INADDR_NONE -1
 #endif
 
 #define DEFAULT_NUMBER_INTERFACES 10
-#define MAX_IFCONF_SIZE 10 * 1024 * 1024
-
+#define MAX_IFCONF_SIZE           10 * 1024 * 1024
 
 typedef struct opal_if_t {
-    opal_list_item_t     super;
-    char                if_name[OPAL_IF_NAMESIZE];
-    int                 if_index;
-    uint16_t            if_kernel_index;
-    uint16_t            af_family;
-    int                 if_flags;
-    int                 if_speed;
-    struct sockaddr_storage  if_addr;
-    uint32_t            if_mask;
-    uint32_t            if_bandwidth;
-    uint8_t             if_mac[6];
-    int                 ifmtu; /* Can't use if_mtu because of a
-                                  #define collision on some BSDs */
+    opal_list_item_t super;
+    char if_name[OPAL_IF_NAMESIZE];
+    int if_index;
+    uint16_t if_kernel_index;
+    uint16_t af_family;
+    int if_flags;
+    int if_speed;
+    struct sockaddr_storage if_addr;
+    uint32_t if_mask;
+    uint32_t if_bandwidth;
+    uint8_t if_mac[6];
+    int ifmtu; /* Can't use if_mtu because of a
+                  #define collision on some BSDs */
 } opal_if_t;
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_if_t);
 
-
 /* "global" list of available interfaces */
 OPAL_DECLSPEC extern opal_list_t opal_if_list;
 
@@ -114,8 +112,7 @@ typedef struct opal_if_base_component_2_0_0_t opal_if_base_component_t;
 /*
  * Macro for use in components that are of type if
  */
-#define OPAL_IF_BASE_VERSION_2_0_0 \
-    OPAL_MCA_BASE_VERSION_2_1_0("if", 2, 0, 0)
+#define OPAL_IF_BASE_VERSION_2_0_0 OPAL_MCA_BASE_VERSION_2_1_0("if", 2, 0, 0)
 
 END_C_DECLS
 
diff --git a/opal/mca/if/linux_ipv6/Makefile.am b/opal/mca/if/linux_ipv6/Makefile.am
index cfa428ab5ff..3e9b8ef713a 100644
--- a/opal/mca/if/linux_ipv6/Makefile.am
+++ b/opal/mca/if/linux_ipv6/Makefile.am
@@ -1,5 +1,6 @@
 #
 # Copyright (c) 2019 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -7,7 +8,7 @@
 # $HEADER$
 #
 
-dist_opaldata_DATA = help-opal-if-linux-ipv6.txt
+EXTRA_DIST = help-opal-if-linux-ipv6.txt
 
 noinst_LTLIBRARIES = libmca_if_linux_ipv6.la
 
diff --git a/opal/mca/if/linux_ipv6/configure.m4 b/opal/mca/if/linux_ipv6/configure.m4
index 583d59e93fe..096fdbe5665 100644
--- a/opal/mca/if/linux_ipv6/configure.m4
+++ b/opal/mca/if/linux_ipv6/configure.m4
@@ -3,6 +3,8 @@
 # Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2015      Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2025      Nanook Consulting  All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -22,12 +24,13 @@ AC_DEFUN([MCA_opal_if_linux_ipv6_COMPILE_MODE], [
 AC_DEFUN([MCA_opal_if_linux_ipv6_CONFIG], [
     AC_CONFIG_FILES([opal/mca/if/linux_ipv6/Makefile])
 
-    AC_REQUIRE([OPAL_CHECK_OS_FLAVORS])
+    AC_REQUIRE([OAC_CHECK_OS_FLAVORS])
 
-    AC_MSG_CHECKING([if we are on Linux with TCP])
-    # If we have struct sockaddr and we're on Linux, then we're
-    # happy.
-    AS_IF([test "$opal_found_sockaddr" = "yes" && test "$opal_found_linux" = "yes"],
+    # If we previously found struct sockaddr_in6 (we don't repeat the
+    # AC CHECK_TYPES test here simply because it's cumbersome with all
+    # the required #includes) and we're on Linux, we're happy.
+    AC_MSG_CHECKING([for Linux with struct sockaddr_in6])
+    AS_IF([test "$ac_cv_type_struct_sockaddr_in6" = "yes" && test "$oac_found_linux" = "yes"],
           [AC_MSG_RESULT([yes])
            $1],
           [AC_MSG_RESULT([no])
diff --git a/opal/mca/if/linux_ipv6/if_linux_ipv6.c b/opal/mca/if/linux_ipv6/if_linux_ipv6.c
index e44b09de724..583c1ab115f 100644
--- a/opal/mca/if/linux_ipv6/if_linux_ipv6.c
+++ b/opal/mca/if/linux_ipv6/if_linux_ipv6.c
@@ -16,45 +16,45 @@
 
 #include <string.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 #include <errno.h>
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 #ifdef HAVE_SYS_SOCKET_H
-#include <sys/socket.h>
+#    include <sys/socket.h>
 #endif
 #ifdef HAVE_SYS_SOCKIO_H
-#include <sys/sockio.h>
+#    include <sys/sockio.h>
 #endif
 #ifdef HAVE_SYS_IOCTL_H
-#include <sys/ioctl.h>
+#    include <sys/ioctl.h>
 #endif
 #ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
+#    include <netinet/in.h>
 #endif
 #ifdef HAVE_ARPA_INET_H
-#include <arpa/inet.h>
+#    include <arpa/inet.h>
 #endif
 #ifdef HAVE_NET_IF_H
-#include <net/if.h>
+#    include <net/if.h>
 #endif
 #ifdef HAVE_NETDB_H
-#include <netdb.h>
+#    include <netdb.h>
 #endif
 #ifdef HAVE_IFADDRS_H
-#include <ifaddrs.h>
+#    include <ifaddrs.h>
 #endif
 
-#include "opal/runtime/opal.h"
 #include "opal/constants.h"
+#include "opal/mca/if/base/base.h"
+#include "opal/mca/if/if.h"
+#include "opal/runtime/opal.h"
 #include "opal/util/if.h"
 #include "opal/util/output.h"
 #include "opal/util/show_help.h"
 #include "opal/util/string_copy.h"
-#include "opal/mca/if/if.h"
-#include "opal/mca/if/base/base.h"
 
 #define LOG_PREFIX "mca: if: linux_ipv6: "
 
@@ -64,23 +64,15 @@ static int if_linux_ipv6_open(void);
 opal_if_base_component_t mca_if_linux_ipv6_component = {
     /* First, the mca_component_t struct containing meta information
        about the component itself */
-    {
-        OPAL_IF_BASE_VERSION_2_0_0,
-
-        /* Component name and version */
-        "linux_ipv6",
-        OPAL_MAJOR_VERSION,
-        OPAL_MINOR_VERSION,
-        OPAL_RELEASE_VERSION,
-
-        /* Component open and close functions */
-        if_linux_ipv6_open,
-        NULL
-    },
-    {
-        /* This component is checkpointable */
-        MCA_BASE_METADATA_PARAM_CHECKPOINT
-    },
+    {OPAL_IF_BASE_VERSION_2_0_0,
+
+     /* Component name and version */
+     "linux_ipv6", OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION, OPAL_RELEASE_VERSION,
+
+     /* Component open and close functions */
+     if_linux_ipv6_open, NULL},
+    {/* This component is checkpointable */
+     MCA_BASE_METADATA_PARAM_CHECKPOINT},
 };
 
 #if OPAL_ENABLE_IPV6
@@ -96,7 +88,6 @@ static bool hex2int(char hex, int *dst)
         return false;
     }
     return true;
-
 }
 
 static bool hexdecode(const char *src, uint8_t *dst, size_t dstsize)
@@ -125,36 +116,35 @@ static int if_linux_ipv6_open(void)
         char addrhex[sizeof a6.s6_addr * 2 + 1];
         char addrstr[INET6_ADDRSTRLEN];
 
-        while (fscanf(f, "%s %x %x %x %x %s\n", addrhex,
-                      &idx, &pfxlen, &scope, &dadstat, ifname) != EOF) {
+        while (fscanf(f, "%s %x %x %x %x %s\n", addrhex, &idx, &pfxlen, &scope, &dadstat, ifname)
+               != EOF) {
             opal_if_t *intf;
 
             if (!hexdecode(addrhex, a6.s6_addr, sizeof a6.s6_addr)) {
                 const char *hostname;
                 hostname = opal_gethostname();
-                opal_show_help("help-opal-if-linux-ipv6.txt",
-                               "fail to parse if_inet6", true,
+                opal_show_help("help-opal-if-linux-ipv6.txt", "fail to parse if_inet6", true,
                                hostname, ifname, addrhex);
                 continue;
             };
             inet_ntop(AF_INET6, a6.s6_addr, addrstr, sizeof addrstr);
 
             opal_output_verbose(1, opal_if_base_framework.framework_output,
-                                LOG_PREFIX "found interface %s inet6 %s scope %x\n",
-                                ifname, addrstr, scope);
+                                LOG_PREFIX "found interface %s inet6 %s scope %x\n", ifname,
+                                addrstr, scope);
 
             /* Only interested in global (0x00) scope */
-            if (scope != 0x00)  {
+            if (scope != 0x00) {
                 opal_output_verbose(1, opal_if_base_framework.framework_output,
-                                    LOG_PREFIX "skipped interface %s inet6 %s scope %x\n",
-                                    ifname, addrstr, scope);
+                                    LOG_PREFIX "skipped interface %s inet6 %s scope %x\n", ifname,
+                                    addrstr, scope);
                 continue;
             }
 
             intf = OBJ_NEW(opal_if_t);
             if (NULL == intf) {
                 opal_output(0, LOG_PREFIX "unable to allocate %lu bytes\n",
-                            (unsigned long)sizeof(opal_if_t));
+                            (unsigned long) sizeof(opal_if_t));
                 fclose(f);
                 return OPAL_ERR_OUT_OF_RESOURCE;
             }
@@ -162,13 +152,13 @@ static int if_linux_ipv6_open(void)
 
             /* now construct the opal_if_t */
             opal_string_copy(intf->if_name, ifname, OPAL_IF_NAMESIZE);
-            intf->if_index = opal_list_get_size(&opal_if_list)+1;
+            intf->if_index = opal_list_get_size(&opal_if_list) + 1;
             intf->if_kernel_index = (uint16_t) idx;
-            ((struct sockaddr_in6*) &intf->if_addr)->sin6_addr = a6;
-            ((struct sockaddr_in6*) &intf->if_addr)->sin6_family = AF_INET6;
-            ((struct sockaddr_in6*) &intf->if_addr)->sin6_scope_id = scope;
+            ((struct sockaddr_in6 *) &intf->if_addr)->sin6_addr = a6;
+            ((struct sockaddr_in6 *) &intf->if_addr)->sin6_family = AF_INET6;
+            ((struct sockaddr_in6 *) &intf->if_addr)->sin6_scope_id = scope;
             intf->if_mask = pfxlen;
-            if (OPAL_SUCCESS == opal_ifindextoflags(opal_ifnametoindex (ifname), &flag)) {
+            if (OPAL_SUCCESS == opal_ifindextoflags(opal_ifnametoindex(ifname), &flag)) {
                 intf->if_flags = flag;
             } else {
                 intf->if_flags = IFF_UP;
@@ -178,12 +168,12 @@ static int if_linux_ipv6_open(void)
                to list */
             opal_list_append(&opal_if_list, &(intf->super));
             opal_output_verbose(1, opal_if_base_framework.framework_output,
-                                LOG_PREFIX "added interface %s inet6 %s scope %x\n",
-                                ifname, addrstr, scope);
+                                LOG_PREFIX "added interface %s inet6 %s scope %x\n", ifname,
+                                addrstr, scope);
         } /* of while */
         fclose(f);
     }
-#endif  /* OPAL_ENABLE_IPV6 */
+#endif /* OPAL_ENABLE_IPV6 */
 
     return OPAL_SUCCESS;
 }
diff --git a/opal/mca/if/posix_ipv4/configure.m4 b/opal/mca/if/posix_ipv4/configure.m4
index 73548efb985..f5574434937 100644
--- a/opal/mca/if/posix_ipv4/configure.m4
+++ b/opal/mca/if/posix_ipv4/configure.m4
@@ -3,6 +3,8 @@
 # Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2015      Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2025      Nanook Consulting  All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -25,22 +27,24 @@ AC_DEFUN([MCA_opal_if_posix_ipv4_CONFIG], [
     OPAL_VAR_SCOPE_PUSH([opal_if_posix_ipv4_happy])
     opal_if_posix_ipv4_happy=no
 
-    AC_REQUIRE([OPAL_CHECK_OS_FLAVORS])
+    AC_REQUIRE([OAC_CHECK_OS_FLAVORS])
 
-    # If we found struct sockaddr and we're NOT on most of the BSDs,
-    # we're happy.  I.e., if posix but not:
+    # If we previously found struct sockaddr_in (we don't repeat the
+    # AC CHECK_TYPES test here simply because it's cumbersome with all
+    # the required #includes) and we're NOT on most of the BSDs, we're
+    # happy.  I.e., this:
     #if defined(__NetBSD__) || defined(__FreeBSD__) || \
     #    defined(__OpenBSD__) || defined(__DragonFly__)
-    AC_MSG_CHECKING([struct sockaddr])
-    AS_IF([test "$opal_found_sockaddr" = "yes"],
-          [AC_MSG_RESULT([yes (cached)])
+    AC_MSG_CHECKING([struct sockaddr_in again])
+    AS_IF([test "$ac_cv_type_struct_sockaddr_in" = "yes"],
+          [AC_MSG_RESULT([yes (OPAL cached)])
            AC_MSG_CHECKING([not NetBSD, FreeBSD, OpenBSD, or DragonFly])
-           AS_IF([test "$opal_found_netbsd" = "no" && test "$opal_found_freebsd" = "no" && test "$opal_found_openbsd" = "no" && test "$opal_found_dragonfly" = "no"],
+           AS_IF([test "$oac_found_netbsd" = "no" && test "$oac_found_freebsd" = "no" && test "$oac_found_openbsd" = "no" && test "$oac_found_dragonfly" = "no"],
                  [AC_MSG_RESULT([yes])
                   opal_if_posix_ipv4_happy=yes],
                  [AC_MSG_RESULT([no])]
                 )],
-          [AC_MSG_RESULT([no (cached)])]
+          [AC_MSG_RESULT([no (OPAL cached)])]
          )
 
     AS_IF([test "$opal_if_posix_ipv4_happy" = "yes"],
diff --git a/opal/mca/if/posix_ipv4/if_posix.c b/opal/mca/if/posix_ipv4/if_posix.c
index 0145ac8488b..48574b1fa7a 100644
--- a/opal/mca/if/posix_ipv4/if_posix.c
+++ b/opal/mca/if/posix_ipv4/if_posix.c
@@ -18,10 +18,10 @@
 #include <string.h>
 
 #include "opal/constants.h"
+#include "opal/mca/if/base/base.h"
+#include "opal/mca/if/if.h"
 #include "opal/util/output.h"
 #include "opal/util/string_copy.h"
-#include "opal/mca/if/if.h"
-#include "opal/mca/if/base/base.h"
 
 static int if_posix_open(void);
 
@@ -31,27 +31,19 @@ static int if_posix_open(void);
 opal_if_base_component_t mca_if_posix_ipv4_component = {
     /* First, the mca_component_t struct containing meta information
        about the component itself */
-    {
-        OPAL_IF_BASE_VERSION_2_0_0,
-
-        /* Component name and version */
-        "posix_ipv4",
-        OPAL_MAJOR_VERSION,
-        OPAL_MINOR_VERSION,
-        OPAL_RELEASE_VERSION,
-
-        /* Component open and close functions */
-        if_posix_open,
-        NULL
-    },
-    {
-        /* This component is checkpointable */
-        MCA_BASE_METADATA_PARAM_CHECKPOINT
-    },
+    {OPAL_IF_BASE_VERSION_2_0_0,
+
+     /* Component name and version */
+     "posix_ipv4", OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION, OPAL_RELEASE_VERSION,
+
+     /* Component open and close functions */
+     if_posix_open, NULL},
+    {/* This component is checkpointable */
+     MCA_BASE_METADATA_PARAM_CHECKPOINT},
 };
 
 /* convert a netmask (in network byte order) to CIDR notation */
-static int prefix (uint32_t netmask)
+static int prefix(uint32_t netmask)
 {
     uint32_t mask = ntohl(netmask);
     int plen = 0;
@@ -82,8 +74,7 @@ static int if_posix_open(void)
        using AF_UNSPEC or AF_INET6 will cause everything to
        fail. */
     if ((sd = socket(AF_INET, SOCK_DGRAM, 0)) < 0) {
-        opal_output(0, "opal_ifinit: socket() failed with errno=%d\n",
-                    errno);
+        opal_output(0, "opal_ifinit: socket() failed with errno=%d\n", errno);
         return OPAL_ERROR;
     }
 
@@ -158,12 +149,12 @@ static int if_posix_open(void)
     /*
      * Setup indexes
      */
-    ptr = (char*) ifconf.ifc_req;
+    ptr = (char *) ifconf.ifc_req;
     rem = ifconf.ifc_len;
 
     /* loop through all interfaces */
     while (rem > 0) {
-        struct ifreq* ifr = (struct ifreq*) ptr;
+        struct ifreq *ifr = (struct ifreq *) ptr;
         opal_if_t *intf;
         int length;
 
@@ -210,7 +201,8 @@ static int if_posix_open(void)
 
         intf = OBJ_NEW(opal_if_t);
         if (NULL == intf) {
-            opal_output(0, "opal_ifinit: unable to allocated %lu bytes\n", (unsigned long)sizeof(opal_if_t));
+            opal_output(0, "opal_ifinit: unable to allocated %lu bytes\n",
+                        (unsigned long) sizeof(opal_if_t));
             free(ifconf.ifc_req);
             close(sd);
             return OPAL_ERR_OUT_OF_RESOURCE;
@@ -223,27 +215,27 @@ static int if_posix_open(void)
         intf->if_flags = ifr->ifr_flags;
 
         /* every new address gets its own internal if_index */
-        intf->if_index = opal_list_get_size(&opal_if_list)+1;
+        intf->if_index = opal_list_get_size(&opal_if_list) + 1;
 
-        opal_output_verbose(1, opal_if_base_framework.framework_output,
-                            "found interface %s", intf->if_name);
+        opal_output_verbose(1, opal_if_base_framework.framework_output, "found interface %s",
+                            intf->if_name);
 
         /* assign the kernel index to distinguish different NICs */
 #ifndef SIOCGIFINDEX
         intf->if_kernel_index = intf->if_index;
 #else
         if (ioctl(sd, SIOCGIFINDEX, ifr) < 0) {
-            opal_output(0,"opal_ifinit: ioctl(SIOCGIFINDEX) failed with errno=%d", errno);
+            opal_output(0, "opal_ifinit: ioctl(SIOCGIFINDEX) failed with errno=%d", errno);
             OBJ_RELEASE(intf);
             continue;
         }
-#if defined(ifr_ifindex)
+#    if defined(ifr_ifindex)
         intf->if_kernel_index = ifr->ifr_ifindex;
-#elif defined(ifr_index)
+#    elif defined(ifr_index)
         intf->if_kernel_index = ifr->ifr_index;
-#else
+#    else
         intf->if_kernel_index = -1;
-#endif
+#    endif
 #endif /* SIOCGIFINDEX */
 
         /* This call returns IPv4 addresses only. Use SIOCGLIFADDR
@@ -268,7 +260,7 @@ static int if_posix_open(void)
         }
 
         /* generate CIDR and assign to netmask */
-        intf->if_mask = prefix(((struct sockaddr_in*) &ifr->ifr_addr)->sin_addr.s_addr);
+        intf->if_mask = prefix(((struct sockaddr_in *) &ifr->ifr_addr)->sin_addr.s_addr);
 
 #if defined(SIOCGIFHWADDR) && defined(HAVE_STRUCT_IFREQ_IFR_HWADDR)
         /* get the MAC address */
@@ -295,5 +287,3 @@ static int if_posix_open(void)
 
     return OPAL_SUCCESS;
 }
-
-
diff --git a/opal/mca/if/solaris_ipv6/Makefile.am b/opal/mca/if/solaris_ipv6/Makefile.am
deleted file mode 100644
index b8e9606c327..00000000000
--- a/opal/mca/if/solaris_ipv6/Makefile.am
+++ /dev/null
@@ -1,12 +0,0 @@
-#
-# Copyright (c) 2010     Cisco Systems, Inc.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-noinst_LTLIBRARIES = libmca_if_solaris_ipv6.la
-
-libmca_if_solaris_ipv6_la_SOURCES = if_solaris_ipv6.c
diff --git a/opal/mca/if/solaris_ipv6/configure.m4 b/opal/mca/if/solaris_ipv6/configure.m4
deleted file mode 100644
index eecf6f435dd..00000000000
--- a/opal/mca/if/solaris_ipv6/configure.m4
+++ /dev/null
@@ -1,32 +0,0 @@
-# -*- shell-script -*-
-#
-# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-AC_DEFUN([MCA_opal_if_solaris_ipv6_COMPILE_MODE], [
-    AC_MSG_CHECKING([for MCA component $2:$3 compile mode])
-    $4="static"
-    AC_MSG_RESULT([$$4])
-])
-
-# MCA_if_config_CONFIG(action-if-can-compile,
-#                        [action-if-cant-compile])
-# ------------------------------------------------
-AC_DEFUN([MCA_opal_if_solaris_ipv6_CONFIG], [
-    AC_CONFIG_FILES([opal/mca/if/solaris_ipv6/Makefile])
-
-    AC_REQUIRE([OPAL_CHECK_OS_FLAVORS])
-
-    # check to see if we are on a solaris machine
-    AS_IF([test "$opal_found_sun" = "yes"], [$1], [$2])
-])dnl
-
-#
-# ifdef __sun__
-#
-
diff --git a/opal/mca/if/solaris_ipv6/if_solaris_ipv6.c b/opal/mca/if/solaris_ipv6/if_solaris_ipv6.c
deleted file mode 100644
index 81c8ae1cda0..00000000000
--- a/opal/mca/if/solaris_ipv6/if_solaris_ipv6.c
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "opal_config.h"
-
-#include <stdlib.h>
-#include <string.h>
-
-#include "opal/constants.h"
-#include "opal/util/output.h"
-#include "opal/util/string_copy.h"
-#include "opal/mca/if/if.h"
-
-static int if_solaris_ipv6_open(void);
-
-/* Discovers Solaris IPv6 interfaces */
-opal_if_base_component_t mca_if_solaris_ipv6_component = {
-    /* First, the mca_component_t struct containing meta information
-       about the component itself */
-    {
-        OPAL_IF_BASE_VERSION_2_0_0,
-
-        /* Component name and version */
-        "solaris_ipv6",
-        OPAL_MAJOR_VERSION,
-        OPAL_MINOR_VERSION,
-        OPAL_RELEASE_VERSION,
-
-        /* Component open and close functions */
-        if_solaris_ipv6_open,
-        NULL
-    },
-    {
-        /* This component is checkpointable */
-        MCA_BASE_METADATA_PARAM_CHECKPOINT
-    },
-};
-
-/* configure using getifaddrs(3) */
-static int if_solaris_ipv6_open(void)
-{
-#if OPAL_ENABLE_IPV6
-    int i;
-    int sd;
-    int error;
-    uint16_t kindex;
-    struct lifnum lifnum;
-    struct lifconf lifconf;
-    struct lifreq *lifreq, lifquery;
-
-    sd = socket (AF_INET6, SOCK_DGRAM, 0);
-    if (sd < 0) {
-        opal_output (0, "opal_ifinit: unable to open IPv6 socket\n");
-        return OPAL_ERROR;
-    }
-
-    /* we only ask for IPv6; IPv4 discovery has already been done */
-    lifnum.lifn_family = AF_INET6;
-    lifnum.lifn_flags = 0;
-    lifnum.lifn_count = 0;
-
-    /* get the number of interfaces in the system */
-    error = ioctl (sd, SIOCGLIFNUM, &lifnum);
-    if (error < 0) {
-        opal_output (0,
-                     "opal_ifinit: ioctl SIOCGLIFNUM failed with errno=%d\n", errno);
-        return OPAL_ERROR;
-    }
-
-    memset (&lifconf, 0, sizeof (struct lifconf));
-    memset (&lifquery, 0, sizeof (struct lifreq));
-    lifconf.lifc_family = AF_INET6;
-    lifconf.lifc_flags = 0;
-    lifconf.lifc_len = lifnum.lifn_count * sizeof (struct lifreq) * 2;
-    lifconf.lifc_buf = malloc (lifconf.lifc_len);
-    if (NULL == lifconf.lifc_buf) {
-        opal_output (0, "opal_ifinit: IPv6 discovery: malloc() failed\n");
-        return OPAL_ERR_OUT_OF_RESOURCE;
-    }
-
-    memset (lifconf.lifc_buf, 0, lifconf.lifc_len);
-
-    error = ioctl (sd, SIOCGLIFCONF, &lifconf);
-    if (error < 0) {
-        opal_output (0,
-                     "opal_ifinit: IPv6 SIOCGLIFCONF failed with errno=%d\n", errno);
-    }
-
-    for (i = 0; i + sizeof (struct lifreq) <= lifconf.lifc_len;
-         i += sizeof (*lifreq)) {
-
-        lifreq = (struct lifreq *)((caddr_t)lifconf.lifc_buf + i);
-        opal_string_copy (lifquery.lifr_name, lifreq->lifr_name,
-                 sizeof (lifquery.lifr_name));
-
-        /* lookup kernel index */
-        error = ioctl (sd, SIOCGLIFINDEX, &lifquery);
-        if (error < 0) {
-            opal_output (0,
-                         "opal_ifinit: SIOCGLIFINDEX failed with errno=%d\n", errno);
-            return OPAL_ERROR;
-        }
-        kindex = lifquery.lifr_index;
-
-        /* lookup interface flags */
-        error = ioctl (sd, SIOCGLIFFLAGS, &lifquery);
-        if (error < 0) {
-            opal_output (0,
-                         "opal_ifinit: SIOCGLIFFLAGS failed with errno=%d\n", errno);
-            return OPAL_ERROR;
-        }
-
-        if (AF_INET6 == lifreq->lifr_addr.ss_family) {
-            struct sockaddr_in6* my_addr = (struct sockaddr_in6*) &lifreq->lifr_addr;
-            /* we surely want to check for sin6_scope_id, but Solaris
-               does not set it correctly, so we have to look for
-               global scope. For now, global is anything which is
-               neither loopback nor link local.
-
-               Bug, FIXME: site-local, multicast, ... missing
-               Check for 2000::/3?
-            */
-            if ( (!opal_if_retain_loopback && !IN6_IS_ADDR_LOOPBACK (&my_addr->sin6_addr)) &&
-                 (! IN6_IS_ADDR_LINKLOCAL (&my_addr->sin6_addr))) {
-                /* create interface for newly found address */
-                opal_if_t *intf;
-
-                intf = OBJ_NEW(opal_if_t);
-                if (NULL == intf) {
-                    opal_output (0,
-                                 "opal_ifinit: unable to allocate %d bytes\n",
-                                 sizeof (opal_if_t));
-                    return OPAL_ERR_OUT_OF_RESOURCE;
-                }
-                intf->af_family = AF_INET6;
-
-                opal_string_copy (intf->if_name, lifreq->lifr_name, OPAL_IF_NAMESIZE);
-                intf->if_index = opal_list_get_size(&opal_if_list)+1;
-                memcpy(&intf->if_addr, my_addr, sizeof (*my_addr));
-                intf->if_mask = 64;
-                /* lifrq flags are uint64_t */
-                intf->if_flags =
-                    (uint32_t)(0x00000000ffffffff) & lifquery.lifr_flags;
-
-                /* append to list */
-                opal_list_append (&opal_if_list, &(intf->super));
-            }
-        }
-    } /* for */
-
-    if (NULL != lifconf.lifc_buf) {
-        free (lifconf.lifc_buf);
-    }
-#endif  /* OPAL_ENABLE_IPV6 */
-
-    return OPAL_SUCCESS;
-}
-
-
diff --git a/opal/mca/if/solaris_ipv6/owner.txt b/opal/mca/if/solaris_ipv6/owner.txt
deleted file mode 100644
index 6b13fdc7a25..00000000000
--- a/opal/mca/if/solaris_ipv6/owner.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-#
-# owner/status file
-# owner: institution that is responsible for this package
-# status: e.g. active, maintenance, unmaintained
-#
-owner: nobody
-status: maintenance
diff --git a/opal/mca/installdirs/base/base.h b/opal/mca/installdirs/base/base.h
index 94ad6fbfed9..32b3ae099c5 100644
--- a/opal/mca/installdirs/base/base.h
+++ b/opal/mca/installdirs/base/base.h
@@ -32,7 +32,7 @@ OPAL_DECLSPEC extern mca_base_framework_t opal_installdirs_base_framework;
    also insert the value of the environment variable $OPAL_DESTDIR, if
    it exists/is set.  This function should *only* be used during the
    setup routines of installdirs. */
-char * opal_install_dirs_expand_setup(const char* input);
+char *opal_install_dirs_expand_setup(const char *input);
 
 END_C_DECLS
 
diff --git a/opal/mca/installdirs/base/installdirs_base_components.c b/opal/mca/installdirs/base/installdirs_base_components.c
index 0c10e12fb6f..1daae990df0 100644
--- a/opal/mca/installdirs/base/installdirs_base_components.c
+++ b/opal/mca/installdirs/base/installdirs_base_components.c
@@ -16,108 +16,76 @@
 #include "opal_config.h"
 
 #include "opal/constants.h"
-#include "opal/mca/mca.h"
-#include "opal/mca/installdirs/installdirs.h"
 #include "opal/mca/installdirs/base/base.h"
 #include "opal/mca/installdirs/base/static-components.h"
+#include "opal/mca/installdirs/installdirs.h"
+#include "opal/mca/mca.h"
 
 opal_install_dirs_t opal_install_dirs = {0};
 
-#define CONDITIONAL_COPY(target, origin, field)                 \
-    do {                                                        \
-        if (origin.field != NULL && target.field == NULL) {     \
-            target.field = origin.field;                        \
-        }                                                       \
+#define CONDITIONAL_COPY(target, origin, field)             \
+    do {                                                    \
+        if (origin.field != NULL && target.field == NULL) { \
+            target.field = origin.field;                    \
+        }                                                   \
     } while (0)
 
-static int
-opal_installdirs_base_open(mca_base_open_flag_t flags)
+static int opal_installdirs_base_open(mca_base_open_flag_t flags)
 {
     mca_base_component_list_item_t *component_item;
     int ret;
 
-    ret = mca_base_framework_components_open (&opal_installdirs_base_framework,
-                                              flags);
+    ret = mca_base_framework_components_open(&opal_installdirs_base_framework, flags);
     if (OPAL_SUCCESS != ret) {
         return ret;
     }
 
-    OPAL_LIST_FOREACH(component_item, &opal_installdirs_base_framework.framework_components, mca_base_component_list_item_t) {
-        const opal_installdirs_base_component_t *component =
-            (const opal_installdirs_base_component_t *) component_item->cli_component;
+    OPAL_LIST_FOREACH (component_item, &opal_installdirs_base_framework.framework_components,
+                       mca_base_component_list_item_t) {
+        const opal_installdirs_base_component_t *component
+            = (const opal_installdirs_base_component_t *) component_item->cli_component;
 
         /* copy over the data, if something isn't already there */
-        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data,
-                         prefix);
-        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data,
-                         exec_prefix);
-        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data,
-                         bindir);
-        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data,
-                         sbindir);
-        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data,
-                         libexecdir);
-        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data,
-                         datarootdir);
-        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data,
-                         datadir);
-        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data,
-                         sysconfdir);
-        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data,
-                         sharedstatedir);
-        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data,
-                         localstatedir);
-        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data,
-                         libdir);
-        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data,
-                         includedir);
-        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data,
-                         infodir);
-        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data,
-                         mandir);
-        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data,
-                         opaldatadir);
-        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data,
-                         opallibdir);
-        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data,
-                         opalincludedir);
+        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data, prefix);
+        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data, exec_prefix);
+        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data, bindir);
+        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data, sbindir);
+        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data, libexecdir);
+        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data, datarootdir);
+        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data, datadir);
+        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data, sysconfdir);
+        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data, sharedstatedir);
+        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data, localstatedir);
+        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data, libdir);
+        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data, includedir);
+        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data, infodir);
+        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data, mandir);
+        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data, opaldatadir);
+        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data, opallibdir);
+        CONDITIONAL_COPY(opal_install_dirs, component->install_dirs_data, opalincludedir);
     }
 
     /* expand out all the fields */
-    opal_install_dirs.prefix =
-        opal_install_dirs_expand_setup(opal_install_dirs.prefix);
-    opal_install_dirs.exec_prefix =
-        opal_install_dirs_expand_setup(opal_install_dirs.exec_prefix);
-    opal_install_dirs.bindir =
-        opal_install_dirs_expand_setup(opal_install_dirs.bindir);
-    opal_install_dirs.sbindir =
-        opal_install_dirs_expand_setup(opal_install_dirs.sbindir);
-    opal_install_dirs.libexecdir =
-        opal_install_dirs_expand_setup(opal_install_dirs.libexecdir);
-    opal_install_dirs.datarootdir =
-        opal_install_dirs_expand_setup(opal_install_dirs.datarootdir);
-    opal_install_dirs.datadir =
-        opal_install_dirs_expand_setup(opal_install_dirs.datadir);
-    opal_install_dirs.sysconfdir =
-        opal_install_dirs_expand_setup(opal_install_dirs.sysconfdir);
-    opal_install_dirs.sharedstatedir =
-        opal_install_dirs_expand_setup(opal_install_dirs.sharedstatedir);
-    opal_install_dirs.localstatedir =
-        opal_install_dirs_expand_setup(opal_install_dirs.localstatedir);
-    opal_install_dirs.libdir =
-        opal_install_dirs_expand_setup(opal_install_dirs.libdir);
-    opal_install_dirs.includedir =
-        opal_install_dirs_expand_setup(opal_install_dirs.includedir);
-    opal_install_dirs.infodir =
-        opal_install_dirs_expand_setup(opal_install_dirs.infodir);
-    opal_install_dirs.mandir =
-        opal_install_dirs_expand_setup(opal_install_dirs.mandir);
-    opal_install_dirs.opaldatadir =
-        opal_install_dirs_expand_setup(opal_install_dirs.opaldatadir);
-    opal_install_dirs.opallibdir =
-        opal_install_dirs_expand_setup(opal_install_dirs.opallibdir);
-    opal_install_dirs.opalincludedir =
-        opal_install_dirs_expand_setup(opal_install_dirs.opalincludedir);
+    opal_install_dirs.prefix = opal_install_dirs_expand_setup(opal_install_dirs.prefix);
+    opal_install_dirs.exec_prefix = opal_install_dirs_expand_setup(opal_install_dirs.exec_prefix);
+    opal_install_dirs.bindir = opal_install_dirs_expand_setup(opal_install_dirs.bindir);
+    opal_install_dirs.sbindir = opal_install_dirs_expand_setup(opal_install_dirs.sbindir);
+    opal_install_dirs.libexecdir = opal_install_dirs_expand_setup(opal_install_dirs.libexecdir);
+    opal_install_dirs.datarootdir = opal_install_dirs_expand_setup(opal_install_dirs.datarootdir);
+    opal_install_dirs.datadir = opal_install_dirs_expand_setup(opal_install_dirs.datadir);
+    opal_install_dirs.sysconfdir = opal_install_dirs_expand_setup(opal_install_dirs.sysconfdir);
+    opal_install_dirs.sharedstatedir = opal_install_dirs_expand_setup(
+        opal_install_dirs.sharedstatedir);
+    opal_install_dirs.localstatedir = opal_install_dirs_expand_setup(
+        opal_install_dirs.localstatedir);
+    opal_install_dirs.libdir = opal_install_dirs_expand_setup(opal_install_dirs.libdir);
+    opal_install_dirs.includedir = opal_install_dirs_expand_setup(opal_install_dirs.includedir);
+    opal_install_dirs.infodir = opal_install_dirs_expand_setup(opal_install_dirs.infodir);
+    opal_install_dirs.mandir = opal_install_dirs_expand_setup(opal_install_dirs.mandir);
+    opal_install_dirs.opaldatadir = opal_install_dirs_expand_setup(opal_install_dirs.opaldatadir);
+    opal_install_dirs.opallibdir = opal_install_dirs_expand_setup(opal_install_dirs.opallibdir);
+    opal_install_dirs.opalincludedir = opal_install_dirs_expand_setup(
+        opal_install_dirs.opalincludedir);
 
 #if 0
     fprintf(stderr, "prefix:         %s\n", opal_install_dirs.prefix);
@@ -145,9 +113,7 @@ opal_installdirs_base_open(mca_base_open_flag_t flags)
     return OPAL_SUCCESS;
 }
 
-
-static int
-opal_installdirs_base_close(void)
+static int opal_installdirs_base_close(void)
 {
     free(opal_install_dirs.prefix);
     free(opal_install_dirs.exec_prefix);
@@ -166,9 +132,9 @@ opal_installdirs_base_close(void)
     free(opal_install_dirs.opaldatadir);
     free(opal_install_dirs.opallibdir);
     free(opal_install_dirs.opalincludedir);
-    memset (&opal_install_dirs, 0, sizeof (opal_install_dirs));
+    memset(&opal_install_dirs, 0, sizeof(opal_install_dirs));
 
-    return mca_base_framework_components_close (&opal_installdirs_base_framework, NULL);
+    return mca_base_framework_components_close(&opal_installdirs_base_framework, NULL);
 }
 
 /* Declare the installdirs framework */
diff --git a/opal/mca/installdirs/base/installdirs_base_expand.c b/opal/mca/installdirs/base/installdirs_base_expand.c
index 8c3d1324798..c483c545da4 100644
--- a/opal/mca/installdirs/base/installdirs_base_expand.c
+++ b/opal/mca/installdirs/base/installdirs_base_expand.c
@@ -17,10 +17,10 @@
 
 #include <string.h>
 
-#include "opal/util/os_path.h"
-#include "opal/util/printf.h"
 #include "opal/mca/installdirs/base/base.h"
 #include "opal/mca/installdirs/installdirs.h"
+#include "opal/util/os_path.h"
+#include "opal/util/printf.h"
 
 /* Support both ${name} and @{name} forms.  The latter allows us to
    pass values through AC_SUBST without being munged by m4 (e.g., if
@@ -28,36 +28,32 @@
    whatever the actual value of the shell variable is. */
 #define EXPAND_STRING(name) EXPAND_STRING2(name, name)
 
-#define EXPAND_STRING2(ompiname, fieldname)                             \
-    do {                                                                \
-        if (NULL != (start_pos = strstr(retval, "${" #fieldname "}"))) { \
-            tmp = retval;                                               \
-            *start_pos = '\0';                                          \
-            end_pos = start_pos + strlen("${" #fieldname "}");          \
-            opal_asprintf(&retval, "%s%s%s", tmp,                            \
-                     opal_install_dirs.ompiname + destdir_offset,       \
-                     end_pos);                                          \
-            free(tmp);                                                  \
-            changed = true;                                             \
-        } else if (NULL != (start_pos = strstr(retval, "@{" #fieldname "}"))) { \
-            tmp = retval;                                               \
-            *start_pos = '\0';                                          \
-            end_pos = start_pos + strlen("@{" #fieldname "}");          \
-            opal_asprintf(&retval, "%s%s%s", tmp,                            \
-                     opal_install_dirs.ompiname + destdir_offset,       \
-                     end_pos);                                          \
-            free(tmp);                                                  \
-            changed = true;                                             \
-        }                                                               \
+#define EXPAND_STRING2(ompiname, fieldname)                                                    \
+    do {                                                                                       \
+        if (NULL != (start_pos = strstr(retval, "${" #fieldname "}"))) {                       \
+            tmp = retval;                                                                      \
+            *start_pos = '\0';                                                                 \
+            end_pos = start_pos + strlen("${" #fieldname "}");                                 \
+            opal_asprintf(&retval, "%s%s%s", tmp, opal_install_dirs.ompiname + destdir_offset, \
+                          end_pos);                                                            \
+            free(tmp);                                                                         \
+            changed = true;                                                                    \
+        } else if (NULL != (start_pos = strstr(retval, "@{" #fieldname "}"))) {                \
+            tmp = retval;                                                                      \
+            *start_pos = '\0';                                                                 \
+            end_pos = start_pos + strlen("@{" #fieldname "}");                                 \
+            opal_asprintf(&retval, "%s%s%s", tmp, opal_install_dirs.ompiname + destdir_offset, \
+                          end_pos);                                                            \
+            free(tmp);                                                                         \
+            changed = true;                                                                    \
+        }                                                                                      \
     } while (0)
 
-
 /*
  * Read the lengthy comment below to understand the value of the
  * is_setup parameter.
  */
-static char *
-opal_install_dirs_expand_internal(const char* input, bool is_setup)
+static char *opal_install_dirs_expand_internal(const char *input, bool is_setup)
 {
     size_t len, i;
     bool needs_expand = false;
@@ -113,7 +109,7 @@ opal_install_dirs_expand_internal(const char* input, bool is_setup)
     }
 
     len = strlen(input);
-    for (i = 0 ; i < len ; ++i) {
+    for (i = 0; i < len; ++i) {
         if ('$' == input[i] || '@' == input[i]) {
             needs_expand = true;
             break;
@@ -121,7 +117,9 @@ opal_install_dirs_expand_internal(const char* input, bool is_setup)
     }
 
     retval = strdup(input);
-    if (NULL == retval) return NULL;
+    if (NULL == retval) {
+        return NULL;
+    }
 
     if (needs_expand) {
         bool changed = false;
@@ -158,17 +156,13 @@ opal_install_dirs_expand_internal(const char* input, bool is_setup)
     return retval;
 }
 
-
-char *
-opal_install_dirs_expand(const char* input)
+char *opal_install_dirs_expand(const char *input)
 {
     /* We do NOT want OPAL_DESTDIR expansion in this case. */
     return opal_install_dirs_expand_internal(input, false);
 }
 
-
-char *
-opal_install_dirs_expand_setup(const char* input)
+char *opal_install_dirs_expand_setup(const char *input)
 {
     /* We DO want OPAL_DESTDIR expansion in this case. */
     return opal_install_dirs_expand_internal(input, true);
diff --git a/opal/mca/installdirs/config/opal_installdirs_config.c b/opal/mca/installdirs/config/opal_installdirs_config.c
index d2f8fa8d1de..201cc497f16 100644
--- a/opal/mca/installdirs/config/opal_installdirs_config.c
+++ b/opal/mca/installdirs/config/opal_installdirs_config.c
@@ -10,47 +10,23 @@
 
 #include "opal_config.h"
 
-#include "opal/mca/installdirs/installdirs.h"
 #include "opal/mca/installdirs/config/install_dirs.h"
+#include "opal/mca/installdirs/installdirs.h"
 
 const opal_installdirs_base_component_t mca_installdirs_config_component = {
     /* First, the mca_component_t struct containing meta information
        about the component itself */
-    {
-        OPAL_INSTALLDIRS_BASE_VERSION_2_0_0,
+    {OPAL_INSTALLDIRS_BASE_VERSION_2_0_0,
 
-        /* Component name and version */
-        "config",
-        OPAL_MAJOR_VERSION,
-        OPAL_MINOR_VERSION,
-        OPAL_RELEASE_VERSION,
+     /* Component name and version */
+     "config", OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION, OPAL_RELEASE_VERSION,
 
-        /* Component open and close functions */
-        NULL,
-        NULL
-    },
-    {
-        /* This component is Checkpointable */
-        MCA_BASE_METADATA_PARAM_CHECKPOINT
-    },
+     /* Component open and close functions */
+     NULL, NULL},
+    {/* This component is Checkpointable */
+     MCA_BASE_METADATA_PARAM_CHECKPOINT},
 
-    {
-        OPAL_PREFIX,
-        OPAL_EXEC_PREFIX,
-        OPAL_BINDIR,
-        OPAL_SBINDIR,
-        OPAL_LIBEXECDIR,
-        OPAL_DATAROOTDIR,
-        OPAL_DATADIR,
-        OPAL_SYSCONFDIR,
-        OPAL_SHAREDSTATEDIR,
-        OPAL_LOCALSTATEDIR,
-        OPAL_LIBDIR,
-        OPAL_INCLUDEDIR,
-        OPAL_INFODIR,
-        OPAL_MANDIR,
-        OPAL_PKGDATADIR,
-        OPAL_PKGLIBDIR,
-        OPAL_PKGINCLUDEDIR
-    }
-};
+    {OPAL_PREFIX, OPAL_EXEC_PREFIX, OPAL_BINDIR, OPAL_SBINDIR, OPAL_LIBEXECDIR, OPAL_DATAROOTDIR,
+     OPAL_DATADIR, OPAL_SYSCONFDIR, OPAL_SHAREDSTATEDIR, OPAL_LOCALSTATEDIR, OPAL_LIBDIR,
+     OPAL_INCLUDEDIR, OPAL_INFODIR, OPAL_MANDIR, OPAL_PKGDATADIR, OPAL_PKGLIBDIR,
+     OPAL_PKGINCLUDEDIR}};
diff --git a/opal/mca/installdirs/configure.m4 b/opal/mca/installdirs/configure.m4
index 081a3dcfc67..0d8934675ec 100644
--- a/opal/mca/installdirs/configure.m4
+++ b/opal/mca/installdirs/configure.m4
@@ -1,6 +1,7 @@
 dnl -*- shell-script -*-
 dnl
 dnl Copyright (c) 2006-2010 Sandia National Laboratories. All rights reserved.
+dnl Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
@@ -8,4 +9,7 @@ dnl
 dnl $HEADER$
 dnl
 
+dnl need in the core library
+AC_DEFUN([MCA_opal_installdirs_CORE_LIB], [1])
+
 AC_DEFUN([MCA_opal_installdirs_CONFIGURE_MODE], [PRIORITY])
diff --git a/opal/mca/installdirs/env/opal_installdirs_env.c b/opal/mca/installdirs/env/opal_installdirs_env.c
index 340008e91d4..63cada8ee93 100644
--- a/opal/mca/installdirs/env/opal_installdirs_env.c
+++ b/opal/mca/installdirs/env/opal_installdirs_env.c
@@ -19,27 +19,18 @@
 
 static int installdirs_env_open(void);
 
-
 opal_installdirs_base_component_t mca_installdirs_env_component = {
     /* First, the mca_component_t struct containing meta information
        about the component itself */
-    {
-        OPAL_INSTALLDIRS_BASE_VERSION_2_0_0,
+    {OPAL_INSTALLDIRS_BASE_VERSION_2_0_0,
 
-        /* Component name and version */
-        "env",
-        OPAL_MAJOR_VERSION,
-        OPAL_MINOR_VERSION,
-        OPAL_RELEASE_VERSION,
+     /* Component name and version */
+     "env", OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION, OPAL_RELEASE_VERSION,
 
-        /* Component open and close functions */
-        installdirs_env_open,
-        NULL
-    },
-    {
-        /* This component is checkpointable */
-        MCA_BASE_METADATA_PARAM_CHECKPOINT
-    },
+     /* Component open and close functions */
+     installdirs_env_open, NULL},
+    {/* This component is checkpointable */
+     MCA_BASE_METADATA_PARAM_CHECKPOINT},
 
     /* Next the opal_install_dirs_t install_dirs_data information */
     {
@@ -47,19 +38,16 @@ opal_installdirs_base_component_t mca_installdirs_env_component = {
     },
 };
 
-
-#define SET_FIELD(field, envname)                                         \
-    do {                                                                  \
-        char *tmp = getenv(envname);                                      \
-         if (NULL != tmp && 0 == strlen(tmp)) {                           \
-             tmp = NULL;                                                  \
-         }                                                                \
-         mca_installdirs_env_component.install_dirs_data.field = tmp;     \
+#define SET_FIELD(field, envname)                                    \
+    do {                                                             \
+        char *tmp = getenv(envname);                                 \
+        if (NULL != tmp && 0 == strlen(tmp)) {                       \
+            tmp = NULL;                                              \
+        }                                                            \
+        mca_installdirs_env_component.install_dirs_data.field = tmp; \
     } while (0)
 
-
-static int
-installdirs_env_open(void)
+static int installdirs_env_open(void)
 {
     SET_FIELD(prefix, "OPAL_PREFIX");
     SET_FIELD(exec_prefix, "OPAL_EXEC_PREFIX");
diff --git a/opal/mca/installdirs/installdirs.h b/opal/mca/installdirs/installdirs.h
index 7015adf31ea..bae70867ea6 100644
--- a/opal/mca/installdirs/installdirs.h
+++ b/opal/mca/installdirs/installdirs.h
@@ -14,8 +14,8 @@
 
 #include "opal_config.h"
 
-#include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
+#include "opal/mca/mca.h"
 
 BEGIN_C_DECLS
 
@@ -24,20 +24,20 @@ BEGIN_C_DECLS
  * once opal_init has been called is the opal_install_dirs structure
  * and the opal_install_dirs_expand() call */
 struct opal_install_dirs_t {
-    char* prefix;
-    char* exec_prefix;
-    char* bindir;
-    char* sbindir;
-    char* libexecdir;
-    char* datarootdir;
-    char* datadir;
-    char* sysconfdir;
-    char* sharedstatedir;
-    char* localstatedir;
-    char* libdir;
-    char* includedir;
-    char* infodir;
-    char* mandir;
+    char *prefix;
+    char *exec_prefix;
+    char *bindir;
+    char *sbindir;
+    char *libexecdir;
+    char *datarootdir;
+    char *datadir;
+    char *sysconfdir;
+    char *sharedstatedir;
+    char *localstatedir;
+    char *libdir;
+    char *includedir;
+    char *infodir;
+    char *mandir;
 
     /* Note that the following fields intentionally have an "ompi"
        prefix, even though they're down in the OPAL layer.  This is
@@ -55,9 +55,9 @@ struct opal_install_dirs_t {
        Note that these field names match macros set by configure that
        are used in Makefile.am files.  E.g., project help files are
        installed into $(opaldatadir). */
-    char* opaldatadir;
-    char* opallibdir;
-    char* opalincludedir;
+    char *opaldatadir;
+    char *opallibdir;
+    char *opalincludedir;
 };
 typedef struct opal_install_dirs_t opal_install_dirs_t;
 
@@ -67,8 +67,7 @@ OPAL_DECLSPEC extern opal_install_dirs_t opal_install_dirs;
 /**
  * Expand out path variables (such as ${prefix}) in the input string
  * using the current opal_install_dirs structure */
-OPAL_DECLSPEC char * opal_install_dirs_expand(const char* input);
-
+OPAL_DECLSPEC char *opal_install_dirs_expand(const char *input);
 
 /**
  * Structure for installdirs components.
@@ -89,8 +88,7 @@ typedef struct opal_installdirs_base_component_2_0_0_t opal_installdirs_base_com
 /*
  * Macro for use in components that are of type installdirs
  */
-#define OPAL_INSTALLDIRS_BASE_VERSION_2_0_0 \
-    OPAL_MCA_BASE_VERSION_2_1_0("installdirs", 2, 0, 0)
+#define OPAL_INSTALLDIRS_BASE_VERSION_2_0_0 OPAL_MCA_BASE_VERSION_2_1_0("installdirs", 2, 0, 0)
 
 END_C_DECLS
 
diff --git a/opal/mca/mca.h b/opal/mca/mca.h
index 61bb5c7b963..a501d68bdaa 100644
--- a/opal/mca/mca.h
+++ b/opal/mca/mca.h
@@ -92,7 +92,6 @@
 
 #include "opal_config.h"
 
-
 /**
  * Common type for all MCA modules.
  *
@@ -110,7 +109,6 @@ typedef struct mca_base_module_2_0_0_t mca_base_module_t;
 /** Versioned convenience typedef */
 typedef struct mca_base_module_2_0_0_t mca_base_module_2_0_0_t;
 
-
 /**
  * MCA component open function.
  *
@@ -146,7 +144,7 @@ typedef struct mca_base_module_2_0_0_t mca_base_module_2_0_0_t;
  *
  * If the component a) has no MCA parameters to register, b) no
  * resources to allocate, and c) can always be used in a process
- * (albiet perhaps not selected), it may provide NULL for this
+ * (albeit perhaps not selected), it may provide NULL for this
  * function.  In this cause, the MCA will act as if it called the open
  * function and it returned OPAL_SUCCESS.
  */
@@ -189,7 +187,7 @@ typedef int (*mca_base_close_component_1_0_0_fn_t)(void);
  *
  * This function is used by the mca_base_select function to find the
  * highest priority component to select. Frameworks are free to
- * implement their own query function, but must also implment their
+ * implement their own query function, but must also implement their
  * own select function as a result.
  */
 typedef int (*mca_base_query_component_2_0_0_fn_t)(mca_base_module_2_0_0_t **module, int *priority);
@@ -237,13 +235,12 @@ typedef int (*mca_base_query_component_2_0_0_fn_t)(mca_base_module_2_0_0_t **mod
  *
  * If the component a) has no MCA parameters to register, b) no
  * resources to allocate, and c) can always be used in a process
- * (albiet perhaps not selected), it may provide NULL for this
+ * (albeit perhaps not selected), it may provide NULL for this
  * function.  In this cause, the MCA will act as if it called the
  * registration function and it returned OPAL_SUCCESS.
  */
 typedef int (*mca_base_register_component_params_2_0_0_fn_t)(void);
 
-
 /**
  * Maximum length of MCA project string names.
  */
@@ -284,61 +281,61 @@ enum {
  */
 struct mca_base_component_2_1_0_t {
 
-  int mca_major_version;
-  /**< Major number of the MCA. */
-  int mca_minor_version;
-  /**< Minor number of the MCA. */
-  int mca_release_version;
-  /**< Release number of the MCA. */
+    int mca_major_version;
+    /**< Major number of the MCA. */
+    int mca_minor_version;
+    /**< Minor number of the MCA. */
+    int mca_release_version;
+    /**< Release number of the MCA. */
 
-  char mca_project_name[MCA_BASE_MAX_PROJECT_NAME_LEN + 1];
-  /**< String name of the project that this component belongs to. */
-  int mca_project_major_version;
-  /**< Major version number of the project that this component
-     belongs to. */
-  int mca_project_minor_version;
-  /**< Minor version number of the project that this component
-     belongs to. */
-  int mca_project_release_version;
-  /**< Release version number of the project that this component
-     belongs to. */
+    char mca_project_name[MCA_BASE_MAX_PROJECT_NAME_LEN + 1];
+    /**< String name of the project that this component belongs to. */
+    int mca_project_major_version;
+    /**< Major version number of the project that this component
+       belongs to. */
+    int mca_project_minor_version;
+    /**< Minor version number of the project that this component
+       belongs to. */
+    int mca_project_release_version;
+    /**< Release version number of the project that this component
+       belongs to. */
 
-  char mca_type_name[MCA_BASE_MAX_TYPE_NAME_LEN + 1];
-  /**< String name of the framework that this component belongs to. */
-  int mca_type_major_version;
-  /**< Major version number of the framework that this component
-     belongs to. */
-  int mca_type_minor_version;
-  /**< Minor version number of the framework that this component
-     belongs to. */
-  int mca_type_release_version;
-  /**< Release version number of the framework that this component
-     belongs to. */
+    char mca_type_name[MCA_BASE_MAX_TYPE_NAME_LEN + 1];
+    /**< String name of the framework that this component belongs to. */
+    int mca_type_major_version;
+    /**< Major version number of the framework that this component
+       belongs to. */
+    int mca_type_minor_version;
+    /**< Minor version number of the framework that this component
+       belongs to. */
+    int mca_type_release_version;
+    /**< Release version number of the framework that this component
+       belongs to. */
 
-  char mca_component_name[MCA_BASE_MAX_COMPONENT_NAME_LEN + 1];
-  /**< This comopnent's string name. */
-  int mca_component_major_version;
-  /**< This component's major version number. */
-  int mca_component_minor_version;
-  /**< This component's minor version number. */
-  int mca_component_release_version;
-  /**< This component's release version number. */
+    char mca_component_name[MCA_BASE_MAX_COMPONENT_NAME_LEN + 1];
+    /**< This comopnent's string name. */
+    int mca_component_major_version;
+    /**< This component's major version number. */
+    int mca_component_minor_version;
+    /**< This component's minor version number. */
+    int mca_component_release_version;
+    /**< This component's release version number. */
 
-  mca_base_open_component_1_0_0_fn_t mca_open_component;
-  /**< Method for opening this component. */
-  mca_base_close_component_1_0_0_fn_t mca_close_component;
-  /**< Method for closing this component. */
-  mca_base_query_component_2_0_0_fn_t mca_query_component;
-  /**< Method for querying this component. */
-  mca_base_register_component_params_2_0_0_fn_t mca_register_component_params;
-  /**< Method for registering the component's MCA parameters */
+    mca_base_open_component_1_0_0_fn_t mca_open_component;
+    /**< Method for opening this component. */
+    mca_base_close_component_1_0_0_fn_t mca_close_component;
+    /**< Method for closing this component. */
+    mca_base_query_component_2_0_0_fn_t mca_query_component;
+    /**< Method for querying this component. */
+    mca_base_register_component_params_2_0_0_fn_t mca_register_component_params;
+    /**< Method for registering the component's MCA parameters */
 
-  int32_t mca_component_flags;
-  /**< flags for this component */
+    int32_t mca_component_flags;
+    /**< flags for this component */
 
-  /** Extra space to allow for expansion in the future without
-      breaking older components. */
-  char reserved[28];
+    /** Extra space to allow for expansion in the future without
+        breaking older components. */
+    char reserved[28];
 };
 /** Unversioned convenience typedef; use this name in
     frameworks/components to stay forward source-compatible */
@@ -349,9 +346,9 @@ typedef struct mca_base_component_2_1_0_t mca_base_component_2_1_0_t;
 /*
  * Metadata Bit field parameters
  */
-#define MCA_BASE_METADATA_PARAM_NONE        (uint32_t)0x00 /**< No Metadata flags */
-#define MCA_BASE_METADATA_PARAM_CHECKPOINT  (uint32_t)0x02 /**< Checkpoint enabled Component */
-#define MCA_BASE_METADATA_PARAM_DEBUG       (uint32_t)0x04 /**< Debug enabled/only Component */
+#define MCA_BASE_METADATA_PARAM_NONE       (uint32_t) 0x00 /**< No Metadata flags */
+#define MCA_BASE_METADATA_PARAM_CHECKPOINT (uint32_t) 0x02 /**< Checkpoint enabled Component */
+#define MCA_BASE_METADATA_PARAM_DEBUG      (uint32_t) 0x04 /**< Debug enabled/only Component */
 
 /**
  * Meta data for MCA v2.0.0 components.
@@ -378,26 +375,23 @@ typedef struct mca_base_component_data_2_0_0_t mca_base_component_data_2_0_0_t;
  * indicating that they subscribe to the MCA version 2.0.0.  See
  * component header files (e.g., coll.h) for examples of its usage.
  */
-#define MCA_BASE_VERSION_MAJOR 2
-#define MCA_BASE_VERSION_MINOR 1
+#define MCA_BASE_VERSION_MAJOR   2
+#define MCA_BASE_VERSION_MINOR   1
 #define MCA_BASE_VERSION_RELEASE 0
 
-#define MCA_BASE_MAKE_VERSION(level, MAJOR, MINOR, RELEASE) \
-    .mca_## level ##_major_version = MAJOR,                 \
-    .mca_## level ##_minor_version = MINOR,                 \
-    .mca_## level ##_release_version = RELEASE
-
+#define MCA_BASE_MAKE_VERSION(level, MAJOR, MINOR, RELEASE)                     \
+    .mca_##level##_major_version = MAJOR, .mca_##level##_minor_version = MINOR, \
+    .mca_##level##_release_version = RELEASE
 
-#define MCA_BASE_VERSION_2_1_0(PROJECT, project_major, project_minor, project_release, TYPE, type_major, type_minor, type_release) \
-    .mca_major_version = MCA_BASE_VERSION_MAJOR,                        \
-    .mca_minor_version = MCA_BASE_VERSION_MINOR,                        \
-    .mca_release_version = MCA_BASE_VERSION_RELEASE,                    \
-    .mca_project_name = PROJECT,                                        \
-    MCA_BASE_MAKE_VERSION(project, project_major, project_minor, project_release), \
-    .mca_type_name = TYPE,                                              \
-    MCA_BASE_MAKE_VERSION(type, type_major, type_minor, type_release)
+#define MCA_BASE_VERSION_2_1_0(PROJECT, project_major, project_minor, project_release, TYPE,  \
+                               type_major, type_minor, type_release)                          \
+    .mca_major_version = MCA_BASE_VERSION_MAJOR, .mca_minor_version = MCA_BASE_VERSION_MINOR, \
+    .mca_release_version = MCA_BASE_VERSION_RELEASE, .mca_project_name = PROJECT,             \
+    MCA_BASE_MAKE_VERSION(project, project_major, project_minor, project_release),            \
+    .mca_type_name = TYPE, MCA_BASE_MAKE_VERSION(type, type_major, type_minor, type_release)
 
-#define OPAL_MCA_BASE_VERSION_2_1_0(type, type_major, type_minor, type_release) \
-    MCA_BASE_VERSION_2_1_0("opal", OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION, OPAL_RELEASE_VERSION, type, type_major, type_minor, type_release)
+#define OPAL_MCA_BASE_VERSION_2_1_0(type, type_major, type_minor, type_release)                  \
+    MCA_BASE_VERSION_2_1_0("opal", OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION, OPAL_RELEASE_VERSION, \
+                           type, type_major, type_minor, type_release)
 
 #endif /* OPAL_MCA_H */
diff --git a/opal/mca/memchecker/base/base.h b/opal/mca/memchecker/base/base.h
index a9796193413..6226124c3e4 100644
--- a/opal/mca/memchecker/base/base.h
+++ b/opal/mca/memchecker/base/base.h
@@ -45,14 +45,12 @@ OPAL_DECLSPEC extern bool opal_memchecker_base_selected;
 /**
  * Global component struct for the selected component
  */
-OPAL_DECLSPEC extern const opal_memchecker_base_component_2_0_0_t
-    *opal_memchecker_base_component;
+OPAL_DECLSPEC extern const opal_memchecker_base_component_2_0_0_t *opal_memchecker_base_component;
 
 /**
  * Global module struct for the selected module
  */
-OPAL_DECLSPEC extern const opal_memchecker_base_module_1_0_0_t
-    *opal_memchecker_base_module;
+OPAL_DECLSPEC extern const opal_memchecker_base_module_1_0_0_t *opal_memchecker_base_module;
 
 /**
  * Check if we are running under the memory debugger.
@@ -63,10 +61,9 @@ OPAL_DECLSPEC extern const opal_memchecker_base_module_1_0_0_t
  */
 OPAL_DECLSPEC int opal_memchecker_base_runindebugger(void);
 #if OPAL_WANT_MEMCHECKER == 0
-#define opal_memchecker_base_runindebugger() 0
+#    define opal_memchecker_base_runindebugger() 0
 #endif
 
-
 /**
  * Check if a memory region is valid to address
  *
@@ -78,12 +75,11 @@ OPAL_DECLSPEC int opal_memchecker_base_runindebugger(void);
  * This function calls the selected memchecker, whether
  *  every Byte of this memory region is addressable
  */
-OPAL_DECLSPEC int opal_memchecker_base_isaddressable(void * p, size_t len);
+OPAL_DECLSPEC int opal_memchecker_base_isaddressable(void *p, size_t len);
 #if OPAL_WANT_MEMCHECKER == 0
-#define opal_memchecker_base_isaddressable(p, len) 0
+#    define opal_memchecker_base_isaddressable(p, len) 0
 #endif
 
-
 /**
  * Check if a memory region is defined
  *
@@ -95,9 +91,9 @@ OPAL_DECLSPEC int opal_memchecker_base_isaddressable(void * p, size_t len);
  * This function calls the selected memchecker, whether
  * every Byte of this memory region is correctly initialized.
  */
-OPAL_DECLSPEC int opal_memchecker_base_isdefined(void * p, size_t len);
+OPAL_DECLSPEC int opal_memchecker_base_isdefined(void *p, size_t len);
 #if OPAL_WANT_MEMCHECKER == 0
-#define opal_memchecker_base_isdefined(p, len) 0
+#    define opal_memchecker_base_isdefined(p, len) 0
 #endif
 
 /**
@@ -111,9 +107,9 @@ OPAL_DECLSPEC int opal_memchecker_base_isdefined(void * p, size_t len);
  * This function calls the selected memchecker, to set
  * every Byte of this memory region to not accessible.
  */
-OPAL_DECLSPEC int opal_memchecker_base_mem_noaccess(void * p, size_t len);
+OPAL_DECLSPEC int opal_memchecker_base_mem_noaccess(void *p, size_t len);
 #if OPAL_WANT_MEMCHECKER == 0
-#define opal_memchecker_base_mem_noaccess(p, len)
+#    define opal_memchecker_base_mem_noaccess(p, len)
 #endif
 
 /**
@@ -127,9 +123,9 @@ OPAL_DECLSPEC int opal_memchecker_base_mem_noaccess(void * p, size_t len);
  * This function calls the selected memchecker, to set
  * every Byte of this memory region to not contain initialized data.
  */
-OPAL_DECLSPEC int opal_memchecker_base_mem_undefined(void * p, size_t len);
+OPAL_DECLSPEC int opal_memchecker_base_mem_undefined(void *p, size_t len);
 #if OPAL_WANT_MEMCHECKER == 0
-#define opal_memchecker_base_mem_undefined(p, len)
+#    define opal_memchecker_base_mem_undefined(p, len)
 #endif
 
 /**
@@ -143,9 +139,9 @@ OPAL_DECLSPEC int opal_memchecker_base_mem_undefined(void * p, size_t len);
  * This function calls the selected memchecker, to set
  * every Byte of this memory region to contain valid, initialized data.
  */
-OPAL_DECLSPEC int opal_memchecker_base_mem_defined(void * p, size_t len);
+OPAL_DECLSPEC int opal_memchecker_base_mem_defined(void *p, size_t len);
 #if OPAL_WANT_MEMCHECKER == 0
-#define opal_memchecker_base_mem_defined(p, len)
+#    define opal_memchecker_base_mem_defined(p, len)
 #endif
 
 /**
@@ -160,9 +156,9 @@ OPAL_DECLSPEC int opal_memchecker_base_mem_defined(void * p, size_t len);
  * every Byte of this memory region to contain valid, initialized data,
  *  but only, if the memory region is addressable.
  */
-OPAL_DECLSPEC int opal_memchecker_base_mem_defined_if_addressable(void * p, size_t len);
+OPAL_DECLSPEC int opal_memchecker_base_mem_defined_if_addressable(void *p, size_t len);
 #if OPAL_WANT_MEMCHECKER == 0
-#define opal_memchecker_base_mem_defined_if_addressable(p, len)
+#    define opal_memchecker_base_mem_defined_if_addressable(p, len)
 #endif
 
 /**
@@ -177,9 +173,9 @@ OPAL_DECLSPEC int opal_memchecker_base_mem_defined_if_addressable(void * p, size
  * This function calls the selected memchecker, to name
  * this memory region.
  */
-OPAL_DECLSPEC int opal_memchecker_base_create_block(void * p, size_t len, char * description);
+OPAL_DECLSPEC int opal_memchecker_base_create_block(void *p, size_t len, char *description);
 #if OPAL_WANT_MEMCHECKER == 0
-#define opal_memchecker_base_create_block(p, len, description)
+#    define opal_memchecker_base_create_block(p, len, description)
 #endif
 
 /**
@@ -192,9 +188,9 @@ OPAL_DECLSPEC int opal_memchecker_base_create_block(void * p, size_t len, char *
  * This function calls the selected memchecker, to discard
  * the name information of the memory region.
  */
-OPAL_DECLSPEC int opal_memchecker_base_discard_block(void * p);
+OPAL_DECLSPEC int opal_memchecker_base_discard_block(void *p);
 #if OPAL_WANT_MEMCHECKER == 0
-#define opal_memchecker_base_discard_block(p)
+#    define opal_memchecker_base_discard_block(p)
 #endif
 
 /**
@@ -209,7 +205,7 @@ OPAL_DECLSPEC int opal_memchecker_base_discard_block(void * p);
  */
 OPAL_DECLSPEC int opal_memchecker_base_leakcheck(void);
 #if OPAL_WANT_MEMCHECKER == 0
-#define opal_memchecker_base_leakcheck
+#    define opal_memchecker_base_leakcheck
 #endif
 
 /**
@@ -224,9 +220,9 @@ OPAL_DECLSPEC int opal_memchecker_base_leakcheck(void);
  * This function calls the selected memchecker, to get
  * every vbit of this memory region.
  */
-OPAL_DECLSPEC int opal_memchecker_base_get_vbits(void * p, char * vbits, size_t len);
+OPAL_DECLSPEC int opal_memchecker_base_get_vbits(void *p, char *vbits, size_t len);
 #if OPAL_WANT_MEMCHECKER == 0
-#define opal_memchecker_base_get_vbits(p, vbits, len)
+#    define opal_memchecker_base_get_vbits(p, vbits, len)
 #endif
 
 /**
@@ -241,9 +237,9 @@ OPAL_DECLSPEC int opal_memchecker_base_get_vbits(void * p, char * vbits, size_t
  * This function calls the selected memchecker, to get
  * every vbit of this memory region.
  */
-OPAL_DECLSPEC int opal_memchecker_base_set_vbits(void * p, char * vbits, size_t len);
+OPAL_DECLSPEC int opal_memchecker_base_set_vbits(void *p, char *vbits, size_t len);
 #if OPAL_WANT_MEMCHECKER == 0
-#define opal_memchecker_base_set_vbits(p, vbits, len)
+#    define opal_memchecker_base_set_vbits(p, vbits, len)
 #endif
 
 END_C_DECLS
diff --git a/opal/mca/memchecker/base/memchecker_base_open.c b/opal/mca/memchecker/base/memchecker_base_open.c
index 69eabb3370b..8f7bd11cf8b 100644
--- a/opal/mca/memchecker/base/memchecker_base_open.c
+++ b/opal/mca/memchecker/base/memchecker_base_open.c
@@ -8,15 +8,14 @@
  * $HEADER$
  */
 
-
 #include "opal_config.h"
 
 #include "opal/constants.h"
-#include "opal/util/output.h"
-#include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
-#include "opal/mca/memchecker/memchecker.h"
+#include "opal/mca/mca.h"
 #include "opal/mca/memchecker/base/base.h"
+#include "opal/mca/memchecker/memchecker.h"
+#include "opal/util/output.h"
 
 /*
  * The following file was created by configure.  It contains extern
diff --git a/opal/mca/memchecker/base/memchecker_base_select.c b/opal/mca/memchecker/base/memchecker_base_select.c
index 3e701ed2edf..2200776857f 100644
--- a/opal/mca/memchecker/base/memchecker_base_select.c
+++ b/opal/mca/memchecker/base/memchecker_base_select.c
@@ -13,14 +13,13 @@
  * $HEADER$
  */
 
-
 #include "opal_config.h"
 
 #include "opal/constants.h"
-#include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
-#include "opal/mca/memchecker/memchecker.h"
+#include "opal/mca/mca.h"
 #include "opal/mca/memchecker/base/base.h"
+#include "opal/mca/memchecker/memchecker.h"
 
 /*
  * Globals
@@ -29,7 +28,6 @@ bool opal_memchecker_base_selected = false;
 const opal_memchecker_base_component_2_0_0_t *opal_memchecker_base_component = NULL;
 const opal_memchecker_base_module_1_0_0_t *opal_memchecker_base_module = NULL;
 
-
 int opal_memchecker_base_select(void)
 {
 #if OPAL_WANT_MEMCHECKER
@@ -40,10 +38,11 @@ int opal_memchecker_base_select(void)
     /*
      * Select the best component
      */
-    if( OPAL_SUCCESS != mca_base_select("memchecker", opal_memchecker_base_framework.framework_output,
-                                        &opal_memchecker_base_framework.framework_components,
-                                        (mca_base_module_t **) &best_module,
-                                        (mca_base_component_t **) &best_component, NULL) ) {
+    if (OPAL_SUCCESS
+        != mca_base_select("memchecker", opal_memchecker_base_framework.framework_output,
+                           &opal_memchecker_base_framework.framework_components,
+                           (mca_base_module_t **) &best_module,
+                           (mca_base_component_t **) &best_component, NULL)) {
         /* This will only happen if no component was selected */
         exit_status = OPAL_ERR_NOT_FOUND;
         goto cleanup;
@@ -51,21 +50,20 @@ int opal_memchecker_base_select(void)
 
     /* Save the winner */
     opal_memchecker_base_component = best_component;
-    opal_memchecker_base_module    = best_module;
-    opal_memchecker_base_selected  = true;
+    opal_memchecker_base_module = best_module;
+    opal_memchecker_base_selected = true;
 
     /* Initialize the winner */
     if (NULL != opal_memchecker_base_module) {
-        if (OPAL_SUCCESS != (ret = opal_memchecker_base_module->init()) ) {
+        if (OPAL_SUCCESS != (ret = opal_memchecker_base_module->init())) {
             exit_status = ret;
             goto cleanup;
         }
     }
 
- cleanup:
+cleanup:
     return exit_status;
 #else
     return OPAL_SUCCESS;
 #endif /* OPAL_WANT_MEMCHECKER */
 }
-
diff --git a/opal/mca/memchecker/base/memchecker_base_wrappers.c b/opal/mca/memchecker/base/memchecker_base_wrappers.c
index babadb918fb..d08f1d54654 100644
--- a/opal/mca/memchecker/base/memchecker_base_wrappers.c
+++ b/opal/mca/memchecker/base/memchecker_base_wrappers.c
@@ -8,79 +8,69 @@
  * $HEADER$
  */
 
-
 #include "opal_config.h"
 
-
 #if OPAL_WANT_MEMCHECKER
 
-#include "opal/mca/memchecker/memchecker.h"
-#include "opal/mca/memchecker/base/base.h"
+#    include "opal/mca/memchecker/base/base.h"
+#    include "opal/mca/memchecker/memchecker.h"
 
 int opal_memchecker_base_runindebugger(void)
 {
     return opal_memchecker_base_module->runindebugger();
 }
 
-int opal_memchecker_base_isaddressable(void * p, size_t len)
+int opal_memchecker_base_isaddressable(void *p, size_t len)
 {
     return opal_memchecker_base_module->isaddressable(p, len);
 }
 
-
-int opal_memchecker_base_isdefined(void * p, size_t len)
+int opal_memchecker_base_isdefined(void *p, size_t len)
 {
     return opal_memchecker_base_module->isdefined(p, len);
 }
 
-
-int opal_memchecker_base_mem_noaccess(void * p, size_t len)
+int opal_memchecker_base_mem_noaccess(void *p, size_t len)
 {
     return opal_memchecker_base_module->mem_noaccess(p, len);
 }
 
-
-int opal_memchecker_base_mem_undefined(void * p, size_t len)
+int opal_memchecker_base_mem_undefined(void *p, size_t len)
 {
     return opal_memchecker_base_module->mem_undefined(p, len);
 }
 
-
-int opal_memchecker_base_mem_defined(void * p, size_t len)
+int opal_memchecker_base_mem_defined(void *p, size_t len)
 {
     return opal_memchecker_base_module->mem_defined(p, len);
 }
 
-
-int opal_memchecker_base_mem_defined_if_addressable(void * p, size_t len)
+int opal_memchecker_base_mem_defined_if_addressable(void *p, size_t len)
 {
     return opal_memchecker_base_module->mem_defined_if_addressable(p, len);
 }
 
-
-int opal_memchecker_base_create_block(void * p, size_t len, char * description)
+int opal_memchecker_base_create_block(void *p, size_t len, char *description)
 {
     return opal_memchecker_base_module->create_block(p, len, description);
 }
 
-
-int opal_memchecker_base_discard_block(void * p)
+int opal_memchecker_base_discard_block(void *p)
 {
     return opal_memchecker_base_module->discard_block(p);
 }
 
-
 int opal_memchecker_base_leakcheck(void)
 {
     return opal_memchecker_base_module->leakcheck();
 }
 
-int opal_memchecker_base_get_vbits(void * p, char * vbits, size_t len)
+int opal_memchecker_base_get_vbits(void *p, char *vbits, size_t len)
 {
     return opal_memchecker_base_module->get_vbits(p, vbits, len);
 }
 
-int opal_memchecker_base_set_vbits(void * p, char * vbits, size_t len)
+int opal_memchecker_base_set_vbits(void *p, char *vbits, size_t len)
 {
     return opal_memchecker_base_module->set_vbits(p, vbits, len);
 }
diff --git a/opal/mca/memchecker/memchecker.h b/opal/mca/memchecker/memchecker.h
index 120b0a417c7..eb9ce80f082 100644
--- a/opal/mca/memchecker/memchecker.h
+++ b/opal/mca/memchecker/memchecker.h
@@ -39,8 +39,8 @@
 
 #include "opal_config.h"
 
-#include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
+#include "opal/mca/mca.h"
 
 /**
  * Module initialization function.  Should return OPAL_SUCCESS.
@@ -56,42 +56,44 @@ typedef int (*opal_memchecker_base_module_runindebugger_fn_t)(void);
 /**
  * Module function to check, whether memory region is addressable
  */
-typedef int (*opal_memchecker_base_module_isaddressable_fn_t)(void * p, size_t len);
+typedef int (*opal_memchecker_base_module_isaddressable_fn_t)(void *p, size_t len);
 
 /**
  * Module function to check, whether memory region is defined
  */
-typedef int (*opal_memchecker_base_module_isdefined_fn_t)(void * p, size_t len);
+typedef int (*opal_memchecker_base_module_isdefined_fn_t)(void *p, size_t len);
 
 /**
  * Module function to set memory region to not accessible
  */
-typedef int (*opal_memchecker_base_module_mem_noaccess_fn_t)(void * p, size_t len);
+typedef int (*opal_memchecker_base_module_mem_noaccess_fn_t)(void *p, size_t len);
 
 /**
  * Module function to set memory region to undefined
  */
-typedef int (*opal_memchecker_base_module_mem_undefined_fn_t)(void * p, size_t len);
+typedef int (*opal_memchecker_base_module_mem_undefined_fn_t)(void *p, size_t len);
 
 /**
  * Module function to set memory region to defined
  */
-typedef int (*opal_memchecker_base_module_mem_defined_fn_t)(void * p, size_t len);
+typedef int (*opal_memchecker_base_module_mem_defined_fn_t)(void *p, size_t len);
 
 /**
  * Module function to set memory region to defined, but only if addressable
  */
-typedef int (*opal_memchecker_base_module_mem_defined_if_addressable_fn_t)(void * p, size_t len);
+typedef int (*opal_memchecker_base_module_mem_defined_if_addressable_fn_t)(void *p, size_t len);
 
 /**
  * Module function name a specific memory region
  */
-typedef int (*opal_memchecker_base_module_create_block_fn_t)(void * p, size_t len, char * description);
+typedef int (*opal_memchecker_base_module_create_block_fn_t)(void *p, size_t len,
+                                                             char *description);
 
 /**
  * Module function to discard a named memory region
  */
-typedef int (*opal_memchecker_base_module_discard_block_fn_t)(void * p); /* Here, we need to do some mapping for valgrind */
+typedef int (*opal_memchecker_base_module_discard_block_fn_t)(
+    void *p); /* Here, we need to do some mapping for valgrind */
 
 /**
  * Module function to check for any leaks
@@ -101,14 +103,12 @@ typedef int (*opal_memchecker_base_module_leakcheck_fn_t)(void);
 /**
  * Module function to get vbits
  */
-typedef int (*opal_memchecker_base_module_get_vbits_fn_t)(void * p, char * vbits, size_t len);
+typedef int (*opal_memchecker_base_module_get_vbits_fn_t)(void *p, char *vbits, size_t len);
 
 /**
  * Module function to set vbits
  */
-typedef int (*opal_memchecker_base_module_set_vbits_fn_t)(void * p, char * vbits, size_t len);
-
-
+typedef int (*opal_memchecker_base_module_set_vbits_fn_t)(void *p, char *vbits, size_t len);
 
 /**
  * Structure for memchecker components.
@@ -176,11 +176,9 @@ struct opal_memchecker_base_module_1_0_0_t {
 typedef struct opal_memchecker_base_module_1_0_0_t opal_memchecker_base_module_1_0_0_t;
 typedef struct opal_memchecker_base_module_1_0_0_t opal_memchecker_base_module_t;
 
-
 /**
  * Macro for use in components that are of type memchecker
  */
-#define OPAL_MEMCHECKER_BASE_VERSION_2_0_0 \
-    OPAL_MCA_BASE_VERSION_2_1_0("memchecker", 2, 0, 0)
+#define OPAL_MEMCHECKER_BASE_VERSION_2_0_0 OPAL_MCA_BASE_VERSION_2_1_0("memchecker", 2, 0, 0)
 
 #endif /* OPAL_MCA_MEMCHECKER_MEMCHECKER_H */
diff --git a/opal/mca/memchecker/valgrind/Makefile.am b/opal/mca/memchecker/valgrind/Makefile.am
index 7100d422f63..4b1c9becaaf 100644
--- a/opal/mca/memchecker/valgrind/Makefile.am
+++ b/opal/mca/memchecker/valgrind/Makefile.am
@@ -32,7 +32,7 @@ mcacomponentdir = $(libdir)/openmpi
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_memchecker_valgrind_la_SOURCES = $(sources)
 mca_memchecker_valgrind_la_LDFLAGS = -module -avoid-version
-mca_memchecker_valgrind_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
+mca_memchecker_valgrind_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
         $(OPAL_TOP_BUILDDIR)/opal/libopal.la
 
 noinst_LTLIBRARIES = $(component_noinst)
diff --git a/opal/mca/memchecker/valgrind/memchecker_valgrind.h b/opal/mca/memchecker/valgrind/memchecker_valgrind.h
index ed504f05d39..34c5406bc9c 100644
--- a/opal/mca/memchecker/valgrind/memchecker_valgrind.h
+++ b/opal/mca/memchecker/valgrind/memchecker_valgrind.h
@@ -29,8 +29,7 @@ BEGIN_C_DECLS
 /**
  * Globally exported variable
  */
-OPAL_DECLSPEC extern const opal_memchecker_base_component_2_0_0_t
-    mca_memchecker_valgrind_component;
+OPAL_DECLSPEC extern const opal_memchecker_base_component_2_0_0_t mca_memchecker_valgrind_component;
 
 /**
  * memchecker query API function
diff --git a/opal/mca/memchecker/valgrind/memchecker_valgrind_component.c b/opal/mca/memchecker/valgrind/memchecker_valgrind_component.c
index b06d882ec4a..ad7a1f322a9 100644
--- a/opal/mca/memchecker/valgrind/memchecker_valgrind_component.c
+++ b/opal/mca/memchecker/valgrind/memchecker_valgrind_component.c
@@ -24,17 +24,17 @@
 
 #include "opal_config.h"
 
+#include "memchecker_valgrind.h"
 #include "opal/constants.h"
 #include "opal/mca/memchecker/memchecker.h"
-#include "memchecker_valgrind.h"
 
 int opal_memchecker_component_priority = 0;
 
 /*
  * Public string showing the memchecker ompi_linux component version number
  */
-const char *opal_memchecker_valgrind_component_version_string =
-    "OPAL valgrind memchecker MCA component version " OPAL_VERSION;
+const char *opal_memchecker_valgrind_component_version_string
+    = "OPAL valgrind memchecker MCA component version " OPAL_VERSION;
 
 /*
  * Local function
@@ -52,35 +52,30 @@ const opal_memchecker_base_component_2_0_0_t mca_memchecker_valgrind_component =
 
     /* First, the mca_component_t struct containing meta information
        about the component itself */
-    .base_version = {
-        OPAL_MEMCHECKER_BASE_VERSION_2_0_0,
-
-        /* Component name and version */
-        .mca_component_name = "valgrind",
-        MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
-                              OPAL_RELEASE_VERSION),
+    .base_version = {OPAL_MEMCHECKER_BASE_VERSION_2_0_0,
 
-        /* Component open and close functions */
-        .mca_open_component = valgrind_open,
-        .mca_close_component = valgrind_close,
-        .mca_query_component = opal_memchecker_valgrind_component_query,
-        .mca_register_component_params = valgrind_register
-    },
-    .base_data = {
-        /* Valgrind does not offer functionality to save the state  */
-        MCA_BASE_METADATA_PARAM_CHECKPOINT
-    }
-};
+                     /* Component name and version */
+                     .mca_component_name = "valgrind",
+                     MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
+                                           OPAL_RELEASE_VERSION),
 
+                     /* Component open and close functions */
+                     .mca_open_component = valgrind_open, .mca_close_component = valgrind_close,
+                     .mca_query_component = opal_memchecker_valgrind_component_query,
+                     .mca_register_component_params = valgrind_register},
+    .base_data = {/* Valgrind does not offer functionality to save the state  */
+                  MCA_BASE_METADATA_PARAM_CHECKPOINT}};
 
 static int valgrind_register(void)
 {
     opal_memchecker_component_priority = 0;
     (void) mca_base_component_var_register(&mca_memchecker_valgrind_component.base_version,
-                                           "priority", "Priority for the memchecker valgrind "
-                                           "component (default: 0)", MCA_BASE_VAR_TYPE_INT,
-                                           NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-                                           OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_ALL_EQ,
+                                           "priority",
+                                           "Priority for the memchecker valgrind "
+                                           "component (default: 0)",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0,
+                                           MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
+                                           MCA_BASE_VAR_SCOPE_ALL_EQ,
                                            &opal_memchecker_component_priority);
 
     return OPAL_SUCCESS;
@@ -98,7 +93,6 @@ static int valgrind_open(void)
     return OPAL_SUCCESS;
 }
 
-
 static int valgrind_close(void)
 {
     /*
@@ -110,4 +104,3 @@ static int valgrind_close(void)
      */
     return OPAL_SUCCESS;
 }
-
diff --git a/opal/mca/memchecker/valgrind/memchecker_valgrind_module.c b/opal/mca/memchecker/valgrind/memchecker_valgrind_module.c
index a989736b477..2ece5a61780 100644
--- a/opal/mca/memchecker/valgrind/memchecker_valgrind_module.c
+++ b/opal/mca/memchecker/valgrind/memchecker_valgrind_module.c
@@ -20,28 +20,28 @@
 
 #include "opal_config.h"
 
+#include "memchecker_valgrind.h"
 #include "opal/constants.h"
 #include "opal/mca/base/mca_base_var.h"
-#include "opal/mca/memchecker/memchecker.h"
 #include "opal/mca/memchecker/base/base.h"
-#include "memchecker_valgrind.h"
-#include "valgrind/valgrind.h"
+#include "opal/mca/memchecker/memchecker.h"
 #include "valgrind/memcheck.h"
-
+#include "valgrind/valgrind.h"
 
 /*
  * Local functions
  */
 static int valgrind_module_init(void);
 static int valgrind_module_runindebugger(void);
-static int valgrind_module_isaddressable(void * p, size_t len);
-static int valgrind_module_isdefined(void * p, size_t len);
-static int valgrind_module_mem_noaccess(void * p, size_t len);
-static int valgrind_module_mem_undefined(void * p, size_t len);
-static int valgrind_module_mem_defined(void * p, size_t len);
-static int valgrind_module_mem_defined_if_addressable(void * p, size_t len);
-static int valgrind_module_create_block(void * p, size_t len, char * description);
-static int valgrind_module_discard_block(void * p); /* Here, we need to do some mapping for valgrind */
+static int valgrind_module_isaddressable(void *p, size_t len);
+static int valgrind_module_isdefined(void *p, size_t len);
+static int valgrind_module_mem_noaccess(void *p, size_t len);
+static int valgrind_module_mem_undefined(void *p, size_t len);
+static int valgrind_module_mem_defined(void *p, size_t len);
+static int valgrind_module_mem_defined_if_addressable(void *p, size_t len);
+static int valgrind_module_create_block(void *p, size_t len, char *description);
+static int
+valgrind_module_discard_block(void *p); /* Here, we need to do some mapping for valgrind */
 static int valgrind_module_leakcheck(void);
 #if 0
 static int valgrind_module_get_vbits(void * p, char * vbits, size_t len);
@@ -58,29 +58,20 @@ static const opal_memchecker_base_module_1_0_0_t loc_module = {
     valgrind_module_init,
 
     /* Module function pointers */
-    valgrind_module_runindebugger,
-    valgrind_module_isaddressable,
-    valgrind_module_isdefined,
-    valgrind_module_mem_noaccess,
-    valgrind_module_mem_undefined,
-    valgrind_module_mem_defined,
-    valgrind_module_mem_defined_if_addressable,
-    valgrind_module_create_block,
-    valgrind_module_discard_block,
-    valgrind_module_leakcheck
-};
-
+    valgrind_module_runindebugger, valgrind_module_isaddressable, valgrind_module_isdefined,
+    valgrind_module_mem_noaccess, valgrind_module_mem_undefined, valgrind_module_mem_defined,
+    valgrind_module_mem_defined_if_addressable, valgrind_module_create_block,
+    valgrind_module_discard_block, valgrind_module_leakcheck};
 
 int opal_memchecker_valgrind_component_query(mca_base_module_t **module, int *priority)
 {
     *priority = opal_memchecker_component_priority;
 
-    *module = (mca_base_module_t *)&loc_module;
+    *module = (mca_base_module_t *) &loc_module;
 
     return OPAL_SUCCESS;
 }
 
-
 static int valgrind_module_init(void)
 {
     /* Nothing to do yet, possibly update the amount of memory blocks. */
@@ -88,14 +79,12 @@ static int valgrind_module_init(void)
     return OPAL_SUCCESS;
 }
 
-
 static int valgrind_module_runindebugger(void)
 {
     return RUNNING_ON_VALGRIND;
 }
 
-
-static int valgrind_module_isaddressable(void * p, size_t len)
+static int valgrind_module_isaddressable(void *p, size_t len)
 {
     if (len > 0) {
         VALGRIND_CHECK_MEM_IS_ADDRESSABLE(p, len);
@@ -104,8 +93,7 @@ static int valgrind_module_isaddressable(void * p, size_t len)
     return OPAL_SUCCESS;
 }
 
-
-static int valgrind_module_isdefined(void * p, size_t len)
+static int valgrind_module_isdefined(void *p, size_t len)
 {
     if (len > 0) {
         VALGRIND_CHECK_MEM_IS_DEFINED(p, len);
@@ -114,8 +102,7 @@ static int valgrind_module_isdefined(void * p, size_t len)
     return OPAL_SUCCESS;
 }
 
-
-static int valgrind_module_mem_noaccess(void * p, size_t len)
+static int valgrind_module_mem_noaccess(void *p, size_t len)
 {
     if (len > 0) {
         VALGRIND_MAKE_MEM_NOACCESS(p, len);
@@ -124,8 +111,7 @@ static int valgrind_module_mem_noaccess(void * p, size_t len)
     return OPAL_SUCCESS;
 }
 
-
-static int valgrind_module_mem_undefined(void * p, size_t len)
+static int valgrind_module_mem_undefined(void *p, size_t len)
 {
     if (len > 0) {
         VALGRIND_MAKE_MEM_UNDEFINED(p, len);
@@ -134,8 +120,7 @@ static int valgrind_module_mem_undefined(void * p, size_t len)
     return OPAL_SUCCESS;
 }
 
-
-static int valgrind_module_mem_defined(void * p, size_t len)
+static int valgrind_module_mem_defined(void *p, size_t len)
 {
     if (len > 0) {
         VALGRIND_MAKE_MEM_DEFINED(p, len);
@@ -144,8 +129,7 @@ static int valgrind_module_mem_defined(void * p, size_t len)
     return OPAL_SUCCESS;
 }
 
-
-static int valgrind_module_mem_defined_if_addressable(void * p, size_t len)
+static int valgrind_module_mem_defined_if_addressable(void *p, size_t len)
 {
     if (len > 0) {
         VALGRIND_MAKE_MEM_DEFINED_IF_ADDRESSABLE(p, len);
@@ -154,11 +138,10 @@ static int valgrind_module_mem_defined_if_addressable(void * p, size_t len)
     return OPAL_SUCCESS;
 }
 
-
-static int valgrind_module_create_block(void * p, size_t len, char * description)
+static int valgrind_module_create_block(void *p, size_t len, char *description)
 {
     if (len > 0) {
-        VALGRIND_CREATE_BLOCK (p, len, description);
+        VALGRIND_CREATE_BLOCK(p, len, description);
         /*
          * Add p to some list atomically
          */
@@ -166,8 +149,7 @@ static int valgrind_module_create_block(void * p, size_t len, char * description
     return OPAL_SUCCESS;
 }
 
-
-static int valgrind_module_discard_block(void * p)
+static int valgrind_module_discard_block(void *p)
 {
     /* Here, we need to do some mapping for valgrind */
     /*
@@ -176,14 +158,12 @@ static int valgrind_module_discard_block(void * p)
     return OPAL_SUCCESS;
 }
 
-
 static int valgrind_module_leakcheck(void)
 {
     VALGRIND_DO_LEAK_CHECK;
     return OPAL_SUCCESS;
 }
 
-
 #if 0
 static int valgrind_module_get_vbits(void * p, char * vbits, size_t len)
 {
@@ -204,4 +184,3 @@ static int valgrind_module_set_vbits(void * p, char * vbits, size_t len)
     return OPAL_SUCCESS;
 }
 #endif
-
diff --git a/opal/mca/memcpy/base/base.h b/opal/mca/memcpy/base/base.h
index 7bd44254d14..7a7bc0a75bd 100644
--- a/opal/mca/memcpy/base/base.h
+++ b/opal/mca/memcpy/base/base.h
@@ -17,7 +17,6 @@
 #include "opal/mca/base/mca_base_framework.h"
 #include "opal/mca/memcpy/memcpy.h"
 
-
 /*
  * Global functions for MCA overall memcpy open and close
  */
diff --git a/opal/mca/memcpy/base/memcpy_base_default.h b/opal/mca/memcpy/base/memcpy_base_default.h
index c567602ca43..f3d058bff0e 100644
--- a/opal/mca/memcpy/base/memcpy_base_default.h
+++ b/opal/mca/memcpy/base/memcpy_base_default.h
@@ -12,31 +12,28 @@
 #ifndef OPAL_MCA_MEMCPY_BASE_MEMCPY_BASE_NULL_H
 #define OPAL_MCA_MEMCPY_BASE_MEMCPY_BASE_NULL_H
 
-#define opal_memcpy( dst, src, length ) \
-    memcpy( (dst), (src), (length) );
+#define opal_memcpy(dst, src, length) memcpy((dst), (src), (length));
 
-#define opal_memcpy_tov( dst_iov, src, count )        \
-    do {                                              \
-        int _i;                                       \
-        char* _src = (char*)src;                      \
-                                                      \
-        for( _i = 0; _i < count; _i++ ) {             \
-            opal_memcpy( dst_iov[_i].iov_base, _src,  \
-                         dst_iov[_i].iov_len );       \
-            _src += dst_iov[_i].iov_len;              \
-        }                                             \
+#define opal_memcpy_tov(dst_iov, src, count)                              \
+    do {                                                                  \
+        int _i;                                                           \
+        char *_src = (char *) src;                                        \
+                                                                          \
+        for (_i = 0; _i < count; _i++) {                                  \
+            opal_memcpy(dst_iov[_i].iov_base, _src, dst_iov[_i].iov_len); \
+            _src += dst_iov[_i].iov_len;                                  \
+        }                                                                 \
     } while (0)
 
-#define opal_memcpy_fromv( dst, src_iov, count )        \
-    do {                                                \
-        int _i;                                         \
-        char* _dst = (char*)dst;                        \
-                                                        \
-        for( _i = 0; _i < count; _i++ ) {               \
-            opal_memcpy( _dst, src_iov[_i].iov_base,    \
-                         src_iov[_i].iov_len );         \
-            _dst += src_iov[_i].iov_len;                \
-        }                                               \
+#define opal_memcpy_fromv(dst, src_iov, count)                            \
+    do {                                                                  \
+        int _i;                                                           \
+        char *_dst = (char *) dst;                                        \
+                                                                          \
+        for (_i = 0; _i < count; _i++) {                                  \
+            opal_memcpy(_dst, src_iov[_i].iov_base, src_iov[_i].iov_len); \
+            _dst += src_iov[_i].iov_len;                                  \
+        }                                                                 \
     } while (0)
 
 #endif
diff --git a/opal/mca/memcpy/base/memcpy_base_open.c b/opal/mca/memcpy/base/memcpy_base_open.c
index 9b737f71984..84e9bbc00ea 100644
--- a/opal/mca/memcpy/base/memcpy_base_open.c
+++ b/opal/mca/memcpy/base/memcpy_base_open.c
@@ -9,15 +9,13 @@
  * $HEADER$
  */
 
-
 #include "opal_config.h"
 
 #include "opal/constants.h"
-#include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
-#include "opal/mca/memcpy/memcpy.h"
+#include "opal/mca/mca.h"
 #include "opal/mca/memcpy/base/base.h"
-
+#include "opal/mca/memcpy/memcpy.h"
 
 /*
  * The following file was created by configure.  It contains extern
@@ -30,5 +28,5 @@
  * Globals
  */
 /* Use default register/open/close functions */
-MCA_BASE_FRAMEWORK_DECLARE(opal, memcpy, NULL, NULL, NULL, NULL,
-                           mca_memcpy_base_static_components, 0);
+MCA_BASE_FRAMEWORK_DECLARE(opal, memcpy, NULL, NULL, NULL, NULL, mca_memcpy_base_static_components,
+                           0);
diff --git a/opal/mca/memcpy/memcpy.h b/opal/mca/memcpy/memcpy.h
index d8744844c4d..885a3094dee 100644
--- a/opal/mca/memcpy/memcpy.h
+++ b/opal/mca/memcpy/memcpy.h
@@ -22,8 +22,8 @@
 
 #include "opal_config.h"
 
-#include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
+#include "opal/mca/mca.h"
 
 /**
  * Structure for memcpy components.
@@ -43,7 +43,6 @@ typedef struct opal_memcpy_base_component_2_0_0_t opal_memcpy_base_component_2_0
 /*
  * Macro for use in components that are of type memcpy
  */
-#define OPAL_MEMCPY_BASE_VERSION_2_0_0 \
-    OPAL_MCA_BASE_VERSION_2_1_0("memcpy", 2, 0, 0)
+#define OPAL_MEMCPY_BASE_VERSION_2_0_0 OPAL_MCA_BASE_VERSION_2_1_0("memcpy", 2, 0, 0)
 
 #endif /* OPAL_MCA_MEMCPY_MEMCPY_H */
diff --git a/opal/mca/memory/base/base.h b/opal/mca/memory/base/base.h
index 63e5b568031..2f1e227f89b 100644
--- a/opal/mca/memory/base/base.h
+++ b/opal/mca/memory/base/base.h
@@ -24,7 +24,6 @@
 #include "opal/mca/base/mca_base_framework.h"
 #include "opal/mca/memory/memory.h"
 
-
 BEGIN_C_DECLS
 
 /**
@@ -32,7 +31,7 @@ BEGIN_C_DECLS
  */
 OPAL_DECLSPEC extern mca_base_framework_t opal_memory_base_framework;
 
-OPAL_DECLSPEC void opal_memory_base_malloc_init_hook (void);
+OPAL_DECLSPEC void opal_memory_base_malloc_init_hook(void);
 
 END_C_DECLS
 #endif /* OPAL_BASE_MEMORY_H */
diff --git a/opal/mca/memory/base/empty.h b/opal/mca/memory/base/empty.h
index 46cacc56e34..66fc8abc769 100644
--- a/opal/mca/memory/base/empty.h
+++ b/opal/mca/memory/base/empty.h
@@ -40,8 +40,7 @@ BEGIN_C_DECLS
  *
  * See opal/mca/memory/memory.h for a description of the parameters.
  */
-OPAL_DECLSPEC int opal_memory_base_component_register_empty(void *start,
-                                                            size_t len,
+OPAL_DECLSPEC int opal_memory_base_component_register_empty(void *start, size_t len,
                                                             uint64_t cookie);
 
 /**
@@ -49,8 +48,7 @@ OPAL_DECLSPEC int opal_memory_base_component_register_empty(void *start,
  *
  * See opal/mca/memory/memory.h for a description of the parameters.
  */
-OPAL_DECLSPEC int opal_memory_base_component_deregister_empty(void *start,
-                                                              size_t len,
+OPAL_DECLSPEC int opal_memory_base_component_deregister_empty(void *start, size_t len,
                                                               uint64_t cookie);
 
 /**
@@ -61,7 +59,6 @@ OPAL_DECLSPEC int opal_memory_base_component_deregister_empty(void *start,
 OPAL_DECLSPEC void opal_memory_base_component_set_alignment_empty(int use_memalign,
                                                                   size_t memalign_threshold);
 
-
 END_C_DECLS
 
 #endif
diff --git a/opal/mca/memory/base/memory_base_empty.c b/opal/mca/memory/base/memory_base_empty.c
index c31db234873..5633c12aaa2 100644
--- a/opal/mca/memory/base/memory_base_empty.c
+++ b/opal/mca/memory/base/memory_base_empty.c
@@ -23,22 +23,16 @@
 #include "opal/constants.h"
 #include "opal/mca/memory/base/empty.h"
 
-
-int opal_memory_base_component_register_empty(void *base, size_t len,
-                                              uint64_t cookie)
+int opal_memory_base_component_register_empty(void *base, size_t len, uint64_t cookie)
 {
     return OPAL_SUCCESS;
 }
 
-
-int opal_memory_base_component_deregister_empty(void *base, size_t len,
-                                                uint64_t cookie)
+int opal_memory_base_component_deregister_empty(void *base, size_t len, uint64_t cookie)
 {
     return OPAL_SUCCESS;
 }
 
-void opal_memory_base_component_set_alignment_empty(int use_memalign,
-                                                    size_t memalign_threshold)
+void opal_memory_base_component_set_alignment_empty(int use_memalign, size_t memalign_threshold)
 {
 }
-
diff --git a/opal/mca/memory/base/memory_base_open.c b/opal/mca/memory/base/memory_base_open.c
index 0560b93ae08..22dc8c4f1d8 100644
--- a/opal/mca/memory/base/memory_base_open.c
+++ b/opal/mca/memory/base/memory_base_open.c
@@ -22,16 +22,14 @@
  * $HEADER$
  */
 
-
 #include "opal_config.h"
 
 #include "opal/constants.h"
-#include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
-#include "opal/mca/memory/memory.h"
+#include "opal/mca/mca.h"
 #include "opal/mca/memory/base/base.h"
 #include "opal/mca/memory/base/empty.h"
-
+#include "opal/mca/memory/memory.h"
 
 /*
  * The following file was created by configure.  It contains extern
@@ -45,7 +43,7 @@ static int empty_process(void)
     return OPAL_SUCCESS;
 }
 
-static int empty_query (int *priority)
+static int empty_query(int *priority)
 {
     *priority = 0;
     return OPAL_SUCCESS;
@@ -55,7 +53,7 @@ static int empty_query (int *priority)
  * Local variables
  */
 static opal_memory_base_component_2_0_0_t empty_component = {
-    /* Empty / safe functions to call if no memory componet is selected */
+    /* Empty / safe functions to call if no memory component is selected */
     .memoryc_query = empty_query,
     .memoryc_process = empty_process,
     .memoryc_register = opal_memory_base_component_register_empty,
@@ -63,17 +61,15 @@ static opal_memory_base_component_2_0_0_t empty_component = {
     .memoryc_set_alignment = opal_memory_base_component_set_alignment_empty,
 };
 
-
 /*
  * Globals
  */
 opal_memory_base_component_2_0_0_t *opal_memory = &empty_component;
 
-
-void opal_memory_base_malloc_init_hook (void)
+void opal_memory_base_malloc_init_hook(void)
 {
     if (opal_memory->memoryc_init_hook) {
-        opal_memory->memoryc_init_hook ();
+        opal_memory->memoryc_init_hook();
     }
 }
 
@@ -89,9 +85,10 @@ static int opal_memory_base_open(mca_base_open_flag_t flags)
     int ret;
 
     /* can only be zero or one */
-    OPAL_LIST_FOREACH(item, &opal_memory_base_framework.framework_components, mca_base_component_list_item_t) {
+    OPAL_LIST_FOREACH (item, &opal_memory_base_framework.framework_components,
+                       mca_base_component_list_item_t) {
         tmp = (opal_memory_base_component_2_0_0_t *) item->cli_component;
-        ret = tmp->memoryc_query (&priority);
+        ret = tmp->memoryc_query(&priority);
         if (OPAL_SUCCESS != ret || priority < highest_priority) {
             continue;
         }
@@ -100,15 +97,17 @@ static int opal_memory_base_open(mca_base_open_flag_t flags)
         opal_memory = tmp;
     }
 
-    OPAL_LIST_FOREACH_SAFE(item, next, &opal_memory_base_framework.framework_components, mca_base_component_list_item_t) {
+    OPAL_LIST_FOREACH_SAFE (item, next, &opal_memory_base_framework.framework_components,
+                            mca_base_component_list_item_t) {
         if ((void *) opal_memory != (void *) item->cli_component) {
-            mca_base_component_unload (item->cli_component, opal_memory_base_framework.framework_output);
-            opal_list_remove_item (&opal_memory_base_framework.framework_components, &item->super);
+            mca_base_component_unload(item->cli_component,
+                                      opal_memory_base_framework.framework_output);
+            opal_list_remove_item(&opal_memory_base_framework.framework_components, &item->super);
         }
     }
 
     /* open remaining component */
-    ret = mca_base_framework_components_open (&opal_memory_base_framework, flags);
+    ret = mca_base_framework_components_open(&opal_memory_base_framework, flags);
     if (ret != OPAL_SUCCESS) {
         return ret;
     }
diff --git a/opal/mca/memory/malloc_solaris/Makefile.am b/opal/mca/memory/malloc_solaris/Makefile.am
deleted file mode 100644
index e2d270efd36..00000000000
--- a/opal/mca/memory/malloc_solaris/Makefile.am
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2007-2008 Sun Microsystems, Inc.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-noinst_LTLIBRARIES = libmca_memory_malloc_solaris.la
-
-# Source code files
-libmca_memory_malloc_solaris_la_SOURCES = \
-	memory_malloc_solaris_component.c
-libmca_memory_malloc_solaris_la_LIBADD = \
-    $(memory_malloc_solaris_LIBS)
diff --git a/opal/mca/memory/malloc_solaris/configure.m4 b/opal/mca/memory/malloc_solaris/configure.m4
deleted file mode 100644
index 42d42333374..00000000000
--- a/opal/mca/memory/malloc_solaris/configure.m4
+++ /dev/null
@@ -1,118 +0,0 @@
-# -*- shell-script -*-
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2007-2011 Oracle and/or its affiliates.  All rights reserved.
-# Copyright (c) 2010-2011 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2015      Research Organization for Information Science
-#                         and Technology (RIST). All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-AC_DEFUN([MCA_opal_memory_malloc_solaris_PRIORITY], [0])
-
-AC_DEFUN([MCA_opal_memory_malloc_solaris_COMPILE_MODE], [
-    AC_MSG_CHECKING([for MCA component $2:$3 compile mode])
-    $4="static"
-    AC_MSG_RESULT([$$4])
-])
-
-# MCA_memory_malloc_solaris_CONFIG(action-if-can-compile,
-#                        [action-if-cant-compile])
-# ------------------------------------------------
-AC_DEFUN([MCA_opal_memory_malloc_solaris_CONFIG],[
-    AC_CONFIG_FILES([opal/mca/memory/malloc_solaris/Makefile])
-
-    OPAL_VAR_SCOPE_PUSH([memory_malloc_solaris_happy memory_malloc_solaris_should_use memory_malloc_solaris_munmap memory_alloc_solaris_legacy])
-
-    AC_MSG_CHECKING([for Solaris])
-
-    case "$host" in
-    *solaris*)
-
-        AC_MSG_RESULT([yes])
-
-        AS_IF([test "$with_memory_manager" = "malloc_solaris"],
-              [memory_malloc_solaris_happy="yes"
-                memory_malloc_solaris_should_use=1],
-              [memory_malloc_solaris_should_use=0
-                AS_IF([test "$with_memory_manager" = ""],
-                      [memory_malloc_solaris_happy="yes"],
-                      [memory_malloc_solaris_happy="no"])])
-
-        AS_IF([test "$memory_malloc_solaris_happy" = "yes"],
-              [AC_CHECK_HEADER([malloc.h], [],
-              [memory_malloc_solaris_happy="no"])])
-
-        AS_IF([test "$memory_malloc_solaris_happy" = "yes"],
-              [memory_malloc_solaris_munmap=0
-
-              AC_CHECK_HEADER([sys/syscall.h],
-                              [AC_CHECK_FUNCS([syscall],
-                                              [memory_malloc_solaris_munmap=1])])
-
-              AC_CHECK_FUNCS([__munmap], [memory_malloc_solaris_munmap=1])
-
-              # only allow dlsym (and therefore add -ldl) if we
-              # really need to
-              AS_IF([test "$memory_malloc_solaris_munmap" = "0"],
-                    [memory_malloc_solaris_LIBS_SAVE="$LIBS"
-                      AC_CHECK_LIB([dl],
-                                   [dlsym],
-                                   [memory_malloc_solaris_LIBS="-ldl"
-                                     memory_malloc_solaris_munmap=1])
-                      AC_CHECK_FUNCS([dlsym])
-                      LIBS="$memory_malloc_solaris_LIBS_SAVE"])
-
-              AS_IF([test "$memory_malloc_solaris_munmap" = "0"],
-                    [memory_malloc_solaris_happy="no"])])
-
-        # There is a difference in the munmap prototypes for different
-        # Solaris versions.  So determine whether we are to use Legacy
-        # S10 or later prototypes.
-        memory_alloc_solaris_legacy=0
-        AS_IF([test "$memory_malloc_solaris_happy" = "yes"],
-              [AC_MSG_CHECKING([for Solaris Legacy MUNMAP])
-               AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/mman.h>]],
-                                                  [[char *addr;
-                                                    extern int munmap(caddr_t addr, size_t len);]])],
-                                 [memory_alloc_solaris_legacy=1
-                                  AC_MSG_RESULT([yes])],
-                                 [AC_MSG_RESULT([no])])
-               AC_DEFINE_UNQUOTED([USE_SOLARIS_LEGACY_MUNMAP_PROTOTYPE],
-                                  [$memory_alloc_solaris_legacy],
-                                  [Whether to use the legacy Solaris munmap prototype or not])
-               ])
-
-        AS_IF([test "$memory_malloc_solaris_happy" = "no" && \
-               test "$memory_malloc_solaris_should_use" = "1"],
-              [AC_MSG_ERROR([malloc_solaris memory management requested but not available.  Aborting.])])
-
-        AC_SUBST(memory_malloc_solaris_LIBS)
-
-    ;;
-    *)
-        AC_MSG_RESULT([no])
-        AS_IF([test "$with_memory_manager" = "malloc_solaris"],
-              [memory_malloc_solaris_happy="no"
-                memory_malloc_solaris_should_use=0])
-    ;;
-    esac
-
-    AS_IF([test "$memory_malloc_solaris_happy" = "yes"],
-          [memory_base_found=1
-           $1], [$2])
-
-    OPAL_VAR_SCOPE_POP
-])
diff --git a/opal/mca/memory/malloc_solaris/memory_malloc_solaris_component.c b/opal/mca/memory/malloc_solaris/memory_malloc_solaris_component.c
deleted file mode 100644
index 035d14ade41..00000000000
--- a/opal/mca/memory/malloc_solaris/memory_malloc_solaris_component.c
+++ /dev/null
@@ -1,145 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007-2011 Oracle and/or its affiliates.  All rights reserved.
- * Copyright (c) 2009-2011 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015-2016 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2016      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "opal_config.h"
-#include "opal/mca/memory/memory.h"
-#include "opal/mca/memory/base/empty.h"
-#include "opal/memoryhooks/memory_internal.h"
-#include "opal/constants.h"
-
-#include <sys/types.h>
-#include <stdlib.h>
-#include <sys/mman.h>
-#include <dlfcn.h>
-#if defined(HAVE___MUNMAP)
-/* here so we only include others if we absolutely have to */
-#elif defined(HAVE_SYSCALL)
-#include <sys/syscall.h>
-#include <unistd.h>
-#endif
-
-
-#if defined(HAVE___MUNMAP)
-int  __munmap(caddr_t addr, size_t len);
-#endif
-
-static int opal_memory_malloc_open(void);
-static int opal_memory_malloc_query(int *);
-
-const opal_memory_base_component_2_0_0_t mca_memory_malloc_solaris_component = {
-    /* First, the mca_component_t struct containing meta information
-       about the component itself */
-    .memoryc_version = {
-        OPAL_MEMORY_BASE_VERSION_2_0_0,
-
-        /* Component name and version */
-        .mca_component_name = "malloc_solaris",
-        MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
-                              OPAL_RELEASE_VERSION),
-
-        /* Component open and close functions */
-        .mca_open_component = opal_memory_malloc_open,
-    },
-    .memoryc_data = {
-        /* The component is checkpoint ready */
-        MCA_BASE_METADATA_PARAM_CHECKPOINT
-    },
-
-    /* This component doesn't need these functions, but need to
-       provide safe/empty register/deregister functions to call */
-    .memoryc_query = opal_memory_malloc_query,
-    .memoryc_register = opal_memory_base_component_register_empty,
-    .memoryc_deregister = opal_memory_base_component_deregister_empty,
-    .memoryc_set_alignment = opal_memory_base_component_set_alignment_empty,
-};
-
-/*
- * This component exists to establish the memory hook support
- * level available on Solaris. By default Solaris does not
- * return memory to the system, i.e. does not unmap memory
- * from the process space, when a user calls free(). This allows
- * us to declare OPAL_MEMORY_FREE_SUPPORT. Additionally, by
- * intercepting munmap we can declare OPAL_MEMORY_MUNMAP_SUPPORT.
- *
- * NOTE: Not releasing memory back to the system when calling
- * free() may be unique to Solaris which is why this component
- * was created.
- */
-static int
-opal_memory_malloc_open(void)
-{
-    opal_mem_hooks_set_support(
-        (OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT));
-    return OPAL_SUCCESS;
-}
-
-static int opal_memory_malloc_query (int *priority)
-{
-    *priority = 79;
-    return OPAL_SUCCESS;
-}
-
-/*
- * Three ways to call munmap.  Prefered is to call __munmap, which
- * will exist if munmap is a weak symbol.  If not available next try
- * the syscall, and if that doesn't work, try looking in the dynamic
- * libc.
- */
-#if USE_SOLARIS_LEGACY_MUNMAP_PROTOTYPE
-/* We are compiling using S10 so use its munmap prototype */
-int
-munmap(caddr_t addr, size_t len)
-#else
-/* From S11 on forward munmap's addr is void * */
-int
-munmap(void *addr, size_t len)
-#endif
-{
-#if !defined(HAVE___MUNMAP) && \
-    !defined(HAVE_SYSCALL) && defined(HAVE_DLSYM)
-    static int (*realmunmap)(void*, size_t);
-#endif
-
-    opal_mem_hooks_release_hook(addr, len, 0);
-
-#if defined(HAVE___MUNMAP)
-    return __munmap(addr, len);
-#elif defined(HAVE_SYSCALL)
-    return syscall(SYS_munmap, addr, len);
-#elif defined(HAVE_DLSYM)
-    if (NULL == realmunmap) {
-        union {
-            int (*munmap_fp)(void*, size_t);
-            void *munmap_p;
-        } tmp;
-
-        tmp.munmap_p = dlsym(RTLD_NEXT, "munmap");
-        realmunmap = tmp.munmap_fp;
-    }
-    return realmunmap(addr, len);
-#else
-    #error "Can not determine how to call munmap"
-#endif
-}
diff --git a/opal/mca/memory/malloc_solaris/owner.txt b/opal/mca/memory/malloc_solaris/owner.txt
deleted file mode 100644
index 29b7476009c..00000000000
--- a/opal/mca/memory/malloc_solaris/owner.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-#
-# owner/status file
-# owner: institution that is responsible for this package
-# status: e.g. active, maintenance, unmaintained
-#
-owner: nobody
-status: unmaintained
diff --git a/opal/mca/memory/memory.h b/opal/mca/memory/memory.h
index b5968fd2640..56f9e643f68 100644
--- a/opal/mca/memory/memory.h
+++ b/opal/mca/memory/memory.h
@@ -22,7 +22,7 @@
  * $HEADER$
  */
 
-/* NOTE: This framework accomodates two kinds of memory hooks systems:
+/* NOTE: This framework accommodates two kinds of memory hooks systems:
 
    1. Those that only require being setup once and then will
       automatically call back to internal component/module functions
@@ -63,8 +63,8 @@
 
 #include "opal_config.h"
 
-#include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
+#include "opal/mca/mca.h"
 
 BEGIN_C_DECLS
 
@@ -80,7 +80,7 @@ typedef int (*opal_memory_base_component_process_fn_t)(void);
 
 /**
  * Prototype for a function that is invoked when the memory base is
- * trying to select a component. This funtionality is required.
+ * trying to select a component. This functionality is required.
  */
 typedef int (*opal_memory_base_component_query_fn_t)(int *priority);
 
@@ -94,10 +94,7 @@ typedef int (*opal_memory_base_component_query_fn_t)(int *priority);
  * can use the value opal_memory_base_register_empty (an empty
  * implementation of this function).
  */
-typedef int (*opal_memory_base_component_register_fn_t)(void *base,
-                                                        size_t len,
-                                                        uint64_t cookie);
-
+typedef int (*opal_memory_base_component_register_fn_t)(void *base, size_t len, uint64_t cookie);
 
 /**
  * Prototype for a function that is the opposite of
@@ -114,10 +111,7 @@ typedef int (*opal_memory_base_component_register_fn_t)(void *base,
  * can use the value opal_memory_base_deregister_empty (an empty
  * implementation of this function).
  */
-typedef int (*opal_memory_base_component_deregister_fn_t)(void *base,
-                                                          size_t len,
-                                                          uint64_t cookie);
-
+typedef int (*opal_memory_base_component_deregister_fn_t)(void *base, size_t len, uint64_t cookie);
 
 /**
  * Prototype for a function that set the memory alignment
@@ -168,7 +162,6 @@ END_C_DECLS
 /*
  * Macro for use in components that are of type memory
  */
-#define OPAL_MEMORY_BASE_VERSION_2_0_0 \
-    OPAL_MCA_BASE_VERSION_2_1_0("memory", 2, 0, 0)
+#define OPAL_MEMORY_BASE_VERSION_2_0_0 OPAL_MCA_BASE_VERSION_2_1_0("memory", 2, 0, 0)
 
 #endif /* OPAL_MCA_MEMORY_MEMORY_H */
diff --git a/opal/mca/memory/patcher/configure.m4 b/opal/mca/memory/patcher/configure.m4
index 579c480fb02..9dff16def65 100644
--- a/opal/mca/memory/patcher/configure.m4
+++ b/opal/mca/memory/patcher/configure.m4
@@ -51,7 +51,7 @@ AC_DEFUN([MCA_opal_memory_patcher_CONFIG],[
     # Per the above logic, memory patcher no longer supports MacOS/Darwin,
     # so we no longer support Darwin-specific logic for intercept_mmap.
     # See issue #6853: mmap infinite recurse in opal/mca/memory/patcher
-    AS_IF([test "$opal_memory_patcher_happy" == "yes"], [
+    AS_IF([test "$opal_memory_patcher_happy" = "yes"], [
         AC_CHECK_FUNCS([__curbrk])
         AC_CHECK_HEADERS([linux/mman.h sys/syscall.h])
         AC_CHECK_DECLS([__syscall], [], [], [#include <sys/syscall.h>])
diff --git a/opal/mca/memory/patcher/memory_patcher.h b/opal/mca/memory/patcher/memory_patcher.h
index 1909443c549..ef9b7501f4c 100644
--- a/opal/mca/memory/patcher/memory_patcher.h
+++ b/opal/mca/memory/patcher/memory_patcher.h
@@ -11,15 +11,15 @@
  */
 
 #if !defined(OPAL_MEMORY_PATCHER_H)
-#define OPAL_MEMORY_PATCHER_H
+#    define OPAL_MEMORY_PATCHER_H
 
-#include "opal_config.h"
+#    include "opal_config.h"
 
-#include "opal/mca/memory/memory.h"
-#include "opal/mca/patcher/patcher.h"
+#    include "opal/mca/memory/memory.h"
+#    include "opal/mca/patcher/patcher.h"
 
 typedef struct opal_memory_patcher_component_t {
-  opal_memory_base_component_2_0_0_t super;
+    opal_memory_base_component_2_0_0_t super;
 } opal_memory_patcher_component_t;
 
 extern opal_memory_patcher_component_t mca_memory_patcher_component;
diff --git a/opal/mca/memory/patcher/memory_patcher_component.c b/opal/mca/memory/patcher/memory_patcher_component.c
index 8a83458fa1f..02ddbc7a6e5 100644
--- a/opal/mca/memory/patcher/memory_patcher_component.c
+++ b/opal/mca/memory/patcher/memory_patcher_component.c
@@ -16,6 +16,7 @@
  * Copyright (c) 2016-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016-2020 IBM Corporation.  All rights reserved.
+ * Copyright (c) 2025      Triad National Security, LLC. All rights reserved.
  *
  * $COPYRIGHT$
  *
@@ -26,81 +27,83 @@
 
 #include "memory_patcher.h"
 
-#include "opal/util/output.h"
-#include "opal/util/show_help.h"
-#include "opal/mca/memory/base/empty.h"
 #include "opal/mca/memory/base/base.h"
-#include "opal/memoryhooks/memory.h"
+#include "opal/mca/memory/base/empty.h"
 #include "opal/mca/patcher/base/base.h"
+#include "opal/memoryhooks/memory.h"
+#include "opal/util/output.h"
+#include "opal/util/show_help.h"
 
-#include <stdlib.h>
-#include <stdint.h>
-#include <unistd.h>
+#include <assert.h>
+#include <dlfcn.h>
 #include <errno.h>
 #include <stdarg.h>
+#include <stdint.h>
+#include <stdlib.h>
 #include <sys/mman.h>
-#include <dlfcn.h>
-#include <assert.h>
 #include <sys/time.h>
+#include <unistd.h>
 #if defined(HAVE_SYS_SYSCALL_H)
-#include <sys/syscall.h>
+#    include <sys/syscall.h>
 #endif
 #if defined(HAVE_LINUX_MMAN_H)
-#include <linux/mman.h>
+#    include <linux/mman.h>
 #endif
 #if defined(HAVE_SYS_IPC_H)
-#include <sys/ipc.h>
+#    include <sys/ipc.h>
 #endif
 
 #include "memory_patcher.h"
 #undef opal_memory_changed
 
 #if defined(SYS_shmdt) || (defined(IPCOP_shmdt) && defined(SYS_ipc))
-#define HAS_SHMDT 1
+#    define HAS_SHMDT 1
 #else
-#define HAS_SHMDT 0
+#    define HAS_SHMDT 0
 #endif
 
-#if defined(SYS_shmat) ||(defined(IPCOP_shmat) && defined(SYS_ipc))
-#define HAS_SHMAT 1
+#if defined(SYS_shmat) || (defined(IPCOP_shmat) && defined(SYS_ipc))
+#    define HAS_SHMAT 1
 #else
-#define HAS_SHMAT 0
+#    define HAS_SHMAT 0
 #endif
 
 static int patcher_open(void);
 static int patcher_close(void);
 static int patcher_register(void);
-static int patcher_query (int *);
+static int patcher_query(int *);
 
 static int mca_memory_patcher_priority;
+static int was_executed_already = 0;
 
 opal_memory_patcher_component_t mca_memory_patcher_component = {
-    .super = {
-        .memoryc_version = {
-            OPAL_MEMORY_BASE_VERSION_2_0_0,
-
-            /* Component name and version */
-            .mca_component_name = "patcher",
-            MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
-                                  OPAL_RELEASE_VERSION),
-
-            /* Component open and close functions */
-            .mca_open_component = patcher_open,
-            .mca_close_component = patcher_close,
-            .mca_register_component_params = patcher_register,
-        },
-        .memoryc_data = {
-            /* The component is checkpoint ready */
-            MCA_BASE_METADATA_PARAM_CHECKPOINT
+    .super =
+        {
+            .memoryc_version =
+                {
+                    OPAL_MEMORY_BASE_VERSION_2_0_0,
+
+                    /* Component name and version */
+                    .mca_component_name = "patcher",
+                    MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
+                                          OPAL_RELEASE_VERSION),
+
+                    /* Component open and close functions */
+                    .mca_open_component = patcher_open,
+                    .mca_close_component = patcher_close,
+                    .mca_register_component_params = patcher_register,
+                },
+            .memoryc_data =
+                {/* The component is checkpoint ready */
+                 MCA_BASE_METADATA_PARAM_CHECKPOINT},
+
+            /* Memory framework functions. */
+            .memoryc_query = patcher_query,
+            .memoryc_register = opal_memory_base_component_register_empty,
+            .memoryc_deregister = opal_memory_base_component_deregister_empty,
+            .memoryc_set_alignment = opal_memory_base_component_set_alignment_empty,
         },
 
-        /* Memory framework functions. */
-        .memoryc_query = patcher_query,
-        .memoryc_register = opal_memory_base_component_register_empty,
-        .memoryc_deregister = opal_memory_base_component_deregister_empty,
-        .memoryc_set_alignment = opal_memory_base_component_set_alignment_empty,
-    },
-
     /* Component-specific data, filled in later (compiler will 0/NULL
        it out) */
 };
@@ -108,9 +111,9 @@ opal_memory_patcher_component_t mca_memory_patcher_component = {
 #if HAVE_DECL___SYSCALL && defined(HAVE___SYSCALL)
 /* calling __syscall is preferred on some systems when some arguments may be 64-bit. it also
  * has the benefit of having an off_t return type */
-#define memory_patcher_syscall __syscall
+#    define memory_patcher_syscall __syscall
 #else
-#define memory_patcher_syscall syscall
+#    define memory_patcher_syscall syscall
 #endif
 
 /* All the hooks in this file have two levels. The first level has the OPAL_PATCHER_* macros
@@ -123,34 +126,38 @@ opal_memory_patcher_component_t mca_memory_patcher_component = {
  * Nathan's original fix described above can have the same problem reappear if the
  * interception functions inline themselves.
  */
-static void *_intercept_mmap(void *start, size_t length, int prot, int flags, int fd, off_t offset) __opal_attribute_noinline__;
+static void *_intercept_mmap(void *start, size_t length, int prot, int flags, int fd,
+                             off_t offset) __opal_attribute_noinline__;
 static int _intercept_munmap(void *start, size_t length) __opal_attribute_noinline__;
-#if defined (SYS_mremap)
-#if defined(__linux__)
-static void *_intercept_mremap (void *start, size_t oldlen, size_t newlen, int flags, void *new_address) __opal_attribute_noinline__;
-#else
-static void *_intercept_mremap (void *start, size_t oldlen, void *new_address, size_t newlen, int flags) __opal_attribute_noinline__;
-#endif // defined(__linux__)
+#if defined(SYS_mremap)
+#    if defined(__linux__)
+static void *_intercept_mremap(void *start, size_t oldlen, size_t newlen, int flags,
+                               void *new_address) __opal_attribute_noinline__;
+#    else
+static void *_intercept_mremap(void *start, size_t oldlen, void *new_address, size_t newlen,
+                               int flags) __opal_attribute_noinline__;
+#    endif // defined(__linux__)
 #endif // defined(SYS_mremap)
-static int _intercept_madvise (void *start, size_t length, int advice) __opal_attribute_noinline__;
+static int _intercept_madvise(void *start, size_t length, int advice) __opal_attribute_noinline__;
 #if defined SYS_brk
-static int _intercept_brk (void *addr) __opal_attribute_noinline__;
+static int _intercept_brk(void *addr) __opal_attribute_noinline__;
 #endif
 #if defined(__linux__)
-#if HAS_SHMAT
-static void *_intercept_shmat(int shmid, const void *shmaddr, int shmflg) __opal_attribute_noinline__;
-#endif // HAS_SHMAT
-#if HAS_SHMDT
-static int _intercept_shmdt (const void *shmaddr) __opal_attribute_noinline__;
-#endif // HAS_SHMDT
+#    if HAS_SHMAT
+static void *_intercept_shmat(int shmid, const void *shmaddr,
+                              int shmflg) __opal_attribute_noinline__;
+#    endif // HAS_SHMAT
+#    if HAS_SHMDT
+static int _intercept_shmdt(const void *shmaddr) __opal_attribute_noinline__;
+#    endif // HAS_SHMDT
 #endif // defined(__linux__)
 
-#if defined (SYS_mmap)
+#if defined(SYS_mmap)
 
-#if defined(HAVE___MMAP) && !HAVE_DECL___MMAP
+#    if defined(HAVE___MMAP) && !HAVE_DECL___MMAP
 /* prototype for Apple's internal mmap function */
-void *__mmap (void *start, size_t length, int prot, int flags, int fd, off_t offset);
-#endif
+void *__mmap(void *start, size_t length, int prot, int flags, int fd, off_t offset);
+#    endif
 
 static void *(*original_mmap)(void *, size_t, int, int, int, off_t);
 
@@ -159,13 +166,14 @@ static void *_intercept_mmap(void *start, size_t length, int prot, int flags, in
     void *result = 0;
 
     if ((flags & MAP_FIXED) && (start != NULL)) {
-        opal_mem_hooks_release_hook (start, length, true);
+        opal_mem_hooks_release_hook(start, length, true);
     }
 
     if (!original_mmap) {
-        result = (void*)(intptr_t) memory_patcher_syscall(SYS_mmap, start, length, prot, flags, fd, offset);
+        result = (void *) (intptr_t) memory_patcher_syscall(SYS_mmap, start, length, prot, flags,
+                                                            fd, offset);
     } else {
-        result = original_mmap (start, length, prot, flags, fd, offset);
+        result = original_mmap(start, length, prot, flags, fd, offset);
     }
 
     return result;
@@ -174,26 +182,26 @@ static void *_intercept_mmap(void *start, size_t length, int prot, int flags, in
 static void *intercept_mmap(void *start, size_t length, int prot, int flags, int fd, off_t offset)
 {
     OPAL_PATCHER_BEGIN;
-    void *result = _intercept_mmap (start, length, prot, flags, fd, offset);
+    void *result = _intercept_mmap(start, length, prot, flags, fd, offset);
     OPAL_PATCHER_END;
     return result;
 }
 #endif
 
-#if defined (SYS_munmap)
-static int (*original_munmap) (void *, size_t);
+#if defined(SYS_munmap)
+static int (*original_munmap)(void *, size_t);
 
 static int _intercept_munmap(void *start, size_t length)
 {
     int result = 0;
 
     /* could be in a malloc implementation */
-    opal_mem_hooks_release_hook (start, length, true);
+    opal_mem_hooks_release_hook(start, length, true);
 
     if (!original_munmap) {
         result = memory_patcher_syscall(SYS_munmap, start, length);
     } else {
-        result = original_munmap (start, length);
+        result = original_munmap(start, length);
     }
 
     return result;
@@ -202,111 +210,116 @@ static int _intercept_munmap(void *start, size_t length)
 static int intercept_munmap(void *start, size_t length)
 {
     OPAL_PATCHER_BEGIN;
-    int result = _intercept_munmap (start, length);
+    int result = _intercept_munmap(start, length);
     OPAL_PATCHER_END;
     return result;
 }
 
 #endif
 
-#if defined (SYS_mremap)
+#if defined(SYS_mremap)
 
-#if defined(__linux__)
+#    if defined(__linux__)
 /* on linux this function has an optional extra argument but ... can not be used here because it
  * causes issues when intercepting a 4-argument mremap call */
-static void *(*original_mremap) (void *, size_t, size_t, int, void *);
-#else
+static void *(*original_mremap)(void *, size_t, size_t, int, void *);
+#    else
 /* mremap has a different signature on BSD systems */
-static void *(*original_mremap) (void *, size_t, void *, size_t, int);
-#endif
-
-#if defined(__linux__)
-static void *_intercept_mremap (void *start, size_t oldlen, size_t newlen, int flags, void *new_address)
-#else
-static void *_intercept_mremap (void *start, size_t oldlen, void *new_address, size_t newlen, int flags)
-#endif
+static void *(*original_mremap)(void *, size_t, void *, size_t, int);
+#    endif
+
+#    if defined(__linux__)
+static void *_intercept_mremap(void *start, size_t oldlen, size_t newlen, int flags,
+                               void *new_address)
+#    else
+static void *_intercept_mremap(void *start, size_t oldlen, void *new_address, size_t newlen,
+                               int flags)
+#    endif
 {
     void *result = MAP_FAILED;
 
     if (MAP_FAILED != start && oldlen > 0) {
-        opal_mem_hooks_release_hook (start, oldlen, true);
+        opal_mem_hooks_release_hook(start, oldlen, true);
     }
 
-#if defined(MREMAP_FIXED)
+#    if defined(MREMAP_FIXED)
     if (!(flags & MREMAP_FIXED)) {
         new_address = NULL;
     }
-#endif
+#    endif
 
-#if defined(__linux__)
+#    if defined(__linux__)
     if (!original_mremap) {
-        result = (void *)(intptr_t) memory_patcher_syscall (SYS_mremap, start, oldlen, newlen, flags, new_address);
+        result = (void *) (intptr_t) memory_patcher_syscall(SYS_mremap, start, oldlen, newlen,
+                                                            flags, new_address);
     } else {
-        result = original_mremap (start, oldlen, newlen, flags, new_address);
+        result = original_mremap(start, oldlen, newlen, flags, new_address);
     }
-#else
+#    else
     if (!original_mremap) {
-        result = (void *)(intptr_t) memory_patcher_syscall (SYS_mremap, start, oldlen, new_address, newlen, flags);
+        result = (void *) (intptr_t) memory_patcher_syscall(SYS_mremap, start, oldlen, new_address,
+                                                            newlen, flags);
     } else {
-        result = original_mremap (start, oldlen, new_address, newlen, flags);
+        result = original_mremap(start, oldlen, new_address, newlen, flags);
     }
-#endif
+#    endif
 
     return result;
 }
 
-#if defined(__linux__)
-static void *intercept_mremap (void *start, size_t oldlen, size_t newlen, int flags, void *new_address)
+#    if defined(__linux__)
+static void *intercept_mremap(void *start, size_t oldlen, size_t newlen, int flags,
+                              void *new_address)
 {
     OPAL_PATCHER_BEGIN;
-    void *result = _intercept_mremap (start, oldlen, newlen, flags, new_address);
+    void *result = _intercept_mremap(start, oldlen, newlen, flags, new_address);
     OPAL_PATCHER_END;
     return result;
 }
-#else
-static void *intercept_mremap (void *start, size_t oldlen, void *new_address, size_t newlen, int flags)
+#    else
+static void *intercept_mremap(void *start, size_t oldlen, void *new_address, size_t newlen,
+                              int flags)
 {
     OPAL_PATCHER_BEGIN;
-    void *result = _intercept_mremap (start, oldlen, new_address, newlen, flags);
+    void *result = _intercept_mremap(start, oldlen, new_address, newlen, flags);
     OPAL_PATCHER_END;
     return result;
 }
-#endif
+#    endif
 
 #endif
 
-#if defined (SYS_madvise)
+#if defined(SYS_madvise)
 
-static int (*original_madvise) (void *, size_t, int);
+static int (*original_madvise)(void *, size_t, int);
 
-static int _intercept_madvise (void *start, size_t length, int advice)
+static int _intercept_madvise(void *start, size_t length, int advice)
 {
     int result = 0;
 
     if (advice == MADV_DONTNEED ||
-#ifdef MADV_FREE
+#    ifdef MADV_FREE
         advice == MADV_FREE ||
-#endif
-#ifdef MADV_REMOVE
+#    endif
+#    ifdef MADV_REMOVE
         advice == MADV_REMOVE ||
-#endif
-        advice == POSIX_MADV_DONTNEED)
-    {
-        opal_mem_hooks_release_hook (start, length, false);
+#    endif
+        advice == POSIX_MADV_DONTNEED) {
+        opal_mem_hooks_release_hook(start, length, true);
     }
 
     if (!original_madvise) {
         result = memory_patcher_syscall(SYS_madvise, start, length, advice);
     } else {
-        result = original_madvise (start, length, advice);
+        result = original_madvise(start, length, advice);
     }
 
     return result;
 }
-static int intercept_madvise (void *start, size_t length, int advice)
+static int intercept_madvise(void *start, size_t length, int advice)
 {
     OPAL_PATCHER_BEGIN;
-    int result = _intercept_madvise (start, length, advice);
+    int result = _intercept_madvise(start, length, advice);
     OPAL_PATCHER_END;
     return result;
 }
@@ -315,57 +328,57 @@ static int intercept_madvise (void *start, size_t length, int advice)
 
 #if defined SYS_brk
 
-#ifdef HAVE___CURBRK
+#    ifdef HAVE___CURBRK
 extern void *__curbrk; /* in libc */
-#endif
+#    endif
 
-static int (*original_brk) (void *);
+static int (*original_brk)(void *);
 
-static int _intercept_brk (void *addr)
+static int _intercept_brk(void *addr)
 {
     int result = 0;
     void *old_addr, *new_addr;
 
-#ifdef HAVE___CURBRK
+#    ifdef HAVE___CURBRK
     old_addr = __curbrk;
-#else
-    old_addr = sbrk (0);
-#endif
+#    else
+    old_addr = sbrk(0);
+#    endif
 
     if (!original_brk) {
         /* get the current_addr */
         new_addr = (void *) (intptr_t) memory_patcher_syscall(SYS_brk, addr);
 
-#ifdef HAVE___CURBRK
+#    ifdef HAVE___CURBRK
         /*
          * Note: if we were using glibc brk/sbrk, their __curbrk would get
          * updated, but since we're going straight to the syscall, we have
          * to update __curbrk or else glibc won't see it.
          */
         __curbrk = new_addr;
-#endif
+#    endif
     } else {
-        result = original_brk (addr);
-#ifdef HAVE___CURBRK
+        result = original_brk(addr);
+#    ifdef HAVE___CURBRK
         new_addr = __curbrk;
-#else
-        new_addr = sbrk (0);
-#endif
+#    else
+        new_addr = sbrk(0);
+#    endif
     }
 
     if (new_addr < addr) {
         errno = ENOMEM;
         result = -1;
     } else if (new_addr < old_addr) {
-        opal_mem_hooks_release_hook (new_addr, (intptr_t) old_addr - (intptr_t) new_addr, true);
+        opal_mem_hooks_release_hook(new_addr, (intptr_t) old_addr - (intptr_t) new_addr, true);
     }
     return result;
 }
 
-static int intercept_brk (void *addr)
+static int intercept_brk(void *addr)
 {
     OPAL_PATCHER_BEGIN;
-    int result = _intercept_brk (addr);
+    int result = _intercept_brk(addr);
     OPAL_PATCHER_END;
     return result;
 }
@@ -378,20 +391,20 @@ static int intercept_brk (void *addr)
 // and when glibc uses that syscall it seems to do so from its own definitions:
 // https://github.com/bminor/glibc/search?q=IPCOP_shmat&unscoped_q=IPCOP_shmat
 #ifndef IPCOP_shmat
-#define IPCOP_shmat                21
+#    define IPCOP_shmat 21
 #endif
 #ifndef IPCOP_shmdt
-#define IPCOP_shmdt                22
+#    define IPCOP_shmdt 22
 #endif
 
 #if HAS_SHMDT || HAS_SHMAT
-#if defined(__linux__)
+#    if defined(__linux__)
 
-#include <stdio.h>
-#include <fcntl.h>
-#include <sys/shm.h>
+#        include <fcntl.h>
+#        include <stdio.h>
+#        include <sys/shm.h>
 
-static size_t memory_patcher_get_shm_seg_size (const void *shmaddr)
+static size_t memory_patcher_get_shm_seg_size(const void *shmaddr)
 {
     unsigned long start_addr, end_addr;
     char *ptr, *newline;
@@ -401,12 +414,12 @@ static size_t memory_patcher_get_shm_seg_size (const void *shmaddr)
 
     seg_size = 0;
 
-    fd = open ("/proc/self/maps", O_RDONLY);
+    fd = open("/proc/self/maps", O_RDONLY);
     if (fd < 0) {
         return 0;
     }
 
-    for (size_t read_offset = 0 ; ; ) {
+    for (size_t read_offset = 0;;) {
         ssize_t nread = read(fd, buffer + read_offset, sizeof(buffer) - 1 - read_offset);
         if (nread <= 0) {
             if (errno == EINTR) {
@@ -419,14 +432,14 @@ static size_t memory_patcher_get_shm_seg_size (const void *shmaddr)
         }
 
         ptr = buffer;
-        while ( (newline = strchr(ptr, '\n')) != NULL ) {
+        while ((newline = strchr(ptr, '\n')) != NULL) {
             /* 00400000-0040b000 r-xp ... \n */
             int ret = sscanf(ptr, "%lx-%lx ", &start_addr, &end_addr);
             if (ret != 2) {
                 continue;
             }
 
-            if (start_addr == (uintptr_t)shmaddr) {
+            if (start_addr == (uintptr_t) shmaddr) {
                 seg_size = end_addr - start_addr;
                 goto out_close;
             }
@@ -443,7 +456,7 @@ static size_t memory_patcher_get_shm_seg_size (const void *shmaddr)
         memmove(buffer, ptr, read_offset);
     }
 
- out_close:
+out_close:
     close(fd);
     return seg_size;
 }
@@ -460,11 +473,11 @@ static size_t get_shm_size(int shmid)
 
     return ds.shm_segsz;
 }
-#endif
+#    endif
 #endif
 
 #if HAS_SHMAT
-#if defined(__linux__)
+#    if defined(__linux__)
 static void *(*original_shmat)(int shmid, const void *shmaddr, int shmflg);
 
 static void *_intercept_shmat(int shmid, const void *shmaddr, int shmflg)
@@ -474,95 +487,94 @@ static void *_intercept_shmat(int shmid, const void *shmaddr, int shmflg)
     size_t size = get_shm_size(shmid);
 
     if ((shmflg & SHM_REMAP) && (shmaddr != NULL)) {
-// I don't really know what REMAP combined with SHM_RND does, so I'll just
-// guess it remaps all the way down to the lower attach_addr, and all the
-// way up to the original shmaddr+size
-        uintptr_t attach_addr = (uintptr_t)shmaddr;
+        // I don't really know what REMAP combined with SHM_RND does, so I'll just
+        // guess it remaps all the way down to the lower attach_addr, and all the
+        // way up to the original shmaddr+size
+        uintptr_t attach_addr = (uintptr_t) shmaddr;
 
         if (shmflg & SHM_RND) {
-            attach_addr -= ((uintptr_t)shmaddr) % SHMLBA;
-            size += ((uintptr_t)shmaddr) % SHMLBA;
+            attach_addr -= ((uintptr_t) shmaddr) % SHMLBA;
+            size += ((uintptr_t) shmaddr) % SHMLBA;
         }
-        opal_mem_hooks_release_hook ((void*)attach_addr, size, false);
+        opal_mem_hooks_release_hook((void *) attach_addr, size, false);
     }
 
     if (!original_shmat) {
-#if defined(SYS_shmat)
-        result = (void*) memory_patcher_syscall(SYS_shmat, shmid, shmaddr, shmflg);
-#else // IPCOP_shmat
+#        if defined(SYS_shmat)
+        result = (void *) memory_patcher_syscall(SYS_shmat, shmid, shmaddr, shmflg);
+#        else // IPCOP_shmat
         unsigned long ret;
-        ret = memory_patcher_syscall(SYS_ipc, IPCOP_shmat,
-                                     shmid, shmflg, &shmaddr, shmaddr);
-        result = (ret > -(unsigned long)SHMLBA) ? (void *)ret : (void *)shmaddr;
-#endif
+        ret = memory_patcher_syscall(SYS_ipc, IPCOP_shmat, shmid, shmflg, &shmaddr, shmaddr);
+        result = (ret > -(unsigned long) SHMLBA) ? (void *) ret : (void *) shmaddr;
+#        endif
     } else {
-        result = original_shmat (shmid, shmaddr, shmflg);
+        result = original_shmat(shmid, shmaddr, shmflg);
     }
 
     return result;
 }
 
-static void* intercept_shmat (int shmid, const void * shmaddr, int shmflg)
+static void *intercept_shmat(int shmid, const void *shmaddr, int shmflg)
 {
     OPAL_PATCHER_BEGIN;
-    void *result = _intercept_shmat (shmid, shmaddr, shmflg);
+    void *result = _intercept_shmat(shmid, shmaddr, shmflg);
     OPAL_PATCHER_END;
     return result;
 }
-#endif
+#    endif
 #endif
 
 #if HAS_SHMDT
-#if defined(__linux__)
-static int (*original_shmdt) (const void *);
+#    if defined(__linux__)
+static int (*original_shmdt)(const void *);
 
-static int _intercept_shmdt (const void *shmaddr)
+static int _intercept_shmdt(const void *shmaddr)
 {
     int result;
 
     /* opal_mem_hooks_release_hook should probably be updated to take a const void *.
      * for now just cast away the const */
-    opal_mem_hooks_release_hook ((void *) shmaddr, memory_patcher_get_shm_seg_size (shmaddr), false);
+    opal_mem_hooks_release_hook((void *) shmaddr, memory_patcher_get_shm_seg_size(shmaddr), false);
 
     if (original_shmdt) {
-        result = original_shmdt (shmaddr);
+        result = original_shmdt(shmaddr);
     } else {
-#if defined(SYS_shmdt)
-        result = memory_patcher_syscall (SYS_shmdt, shmaddr);
-#else // IPCOP_shmdt
+#        if defined(SYS_shmdt)
+        result = memory_patcher_syscall(SYS_shmdt, shmaddr);
+#        else // IPCOP_shmdt
         result = memory_patcher_syscall(SYS_ipc, IPCOP_shmdt, 0, 0, 0, shmaddr);
-#endif
+#        endif
     }
 
     return result;
 }
 
-static int intercept_shmdt (const void *shmaddr)
+static int intercept_shmdt(const void *shmaddr)
 {
     OPAL_PATCHER_BEGIN;
-    int result = _intercept_shmdt (shmaddr);
+    int result = _intercept_shmdt(shmaddr);
     OPAL_PATCHER_END;
     return result;
 }
-#endif
+#    endif
 #endif
 
-static int patcher_register (void)
+static int patcher_register(void)
 {
     mca_memory_patcher_priority = 80;
-    mca_base_component_var_register (&mca_memory_patcher_component.super.memoryc_version,
-                                     "priority", "Priority of the patcher memory hook component",
-                                     MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
-                                     MCA_BASE_VAR_SCOPE_CONSTANT, &mca_memory_patcher_priority);
+    mca_base_component_var_register(&mca_memory_patcher_component.super.memoryc_version, "priority",
+                                    "Priority of the patcher memory hook component",
+                                    MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                    MCA_BASE_VAR_SCOPE_CONSTANT, &mca_memory_patcher_priority);
 
     return OPAL_SUCCESS;
 }
 
-static int patcher_query (int *priority)
+static int patcher_query(int *priority)
 {
     int rc;
 
-    rc = mca_base_framework_open (&opal_patcher_base_framework, 0);
+    rc = mca_base_framework_open(&opal_patcher_base_framework, 0);
     if (OPAL_SUCCESS != rc) {
         *priority = -1;
         return OPAL_SUCCESS;
@@ -573,9 +585,8 @@ static int patcher_query (int *priority)
     return OPAL_SUCCESS;
 }
 
-static int patcher_open (void)
+static int patcher_open(void)
 {
-    static int was_executed_already = 0;
     int rc;
 
     if (was_executed_already) {
@@ -584,71 +595,91 @@ static int patcher_open (void)
 
     was_executed_already = 1;
 
-    rc = opal_patcher_base_select ();
+    rc = opal_patcher_base_select();
     if (OPAL_SUCCESS != rc) {
-        mca_base_framework_close (&opal_patcher_base_framework);
+        mca_base_framework_close(&opal_patcher_base_framework);
         return OPAL_ERR_NOT_AVAILABLE;
     }
 
     /* set memory hooks support level */
-    opal_mem_hooks_set_support (OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT);
+    opal_mem_hooks_set_support(OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT);
 
-#if defined (SYS_mmap)
-    rc = opal_patcher->patch_symbol ("mmap", (uintptr_t) intercept_mmap, (uintptr_t *) &original_mmap);
+#if defined(SYS_mmap)
+    rc = opal_patcher->patch_symbol("mmap", (uintptr_t) intercept_mmap,
+                                    (uintptr_t *) &original_mmap);
     if (OPAL_SUCCESS != rc) {
-        return rc;
+        goto err_patching;
     }
 #endif
 
-#if defined (SYS_munmap)
-    rc = opal_patcher->patch_symbol ("munmap", (uintptr_t)intercept_munmap, (uintptr_t *) &original_munmap);
+#if defined(SYS_munmap)
+    rc = opal_patcher->patch_symbol("munmap", (uintptr_t) intercept_munmap,
+                                    (uintptr_t *) &original_munmap);
     if (OPAL_SUCCESS != rc) {
-        return rc;
+        goto err_patching;
     }
 #endif
 
-#if defined (SYS_mremap)
-    rc = opal_patcher->patch_symbol ("mremap",(uintptr_t)intercept_mremap, (uintptr_t *) &original_mremap);
+#if defined(SYS_mremap)
+    rc = opal_patcher->patch_symbol("mremap", (uintptr_t) intercept_mremap,
+                                    (uintptr_t *) &original_mremap);
     if (OPAL_SUCCESS != rc) {
-        return rc;
+        goto err_patching;
     }
 #endif
 
-#if defined (SYS_madvise)
-    rc = opal_patcher->patch_symbol ("madvise", (uintptr_t)intercept_madvise, (uintptr_t *) &original_madvise);
+#if defined(SYS_madvise)
+    rc = opal_patcher->patch_symbol("madvise", (uintptr_t) intercept_madvise,
+                                    (uintptr_t *) &original_madvise);
     if (OPAL_SUCCESS != rc) {
-        return rc;
+        goto err_patching;
     }
 #endif
 
 #if HAS_SHMAT
-#if defined(__linux__)
-    rc = opal_patcher->patch_symbol ("shmat", (uintptr_t) intercept_shmat, (uintptr_t *) &original_shmat);
+#    if defined(__linux__)
+    rc = opal_patcher->patch_symbol("shmat", (uintptr_t) intercept_shmat,
+                                    (uintptr_t *) &original_shmat);
     if (OPAL_SUCCESS != rc) {
-        return rc;
+        goto err_patching;
     }
-#endif
+#    endif
 #endif
 
 #if defined(__linux__)
-#if HAS_SHMDT
-    rc = opal_patcher->patch_symbol ("shmdt", (uintptr_t) intercept_shmdt, (uintptr_t *) &original_shmdt);
+#    if HAS_SHMDT
+    rc = opal_patcher->patch_symbol("shmdt", (uintptr_t) intercept_shmdt,
+                                    (uintptr_t *) &original_shmdt);
     if (OPAL_SUCCESS != rc) {
-        return rc;
+        goto err_patching;
     }
-#endif
+#    endif
 #endif
 
-#if defined (SYS_brk)
-    rc = opal_patcher->patch_symbol ("brk", (uintptr_t)intercept_brk, (uintptr_t *) &original_brk);
+#if defined(SYS_brk)
+    rc = opal_patcher->patch_symbol("brk", (uintptr_t) intercept_brk, (uintptr_t *) &original_brk);
 #endif
 
+    if (OPAL_SUCCESS != rc) {
+        goto err_patching;
+    }
+
+    return OPAL_SUCCESS;
+
+err_patching:
+    /* In the case we had a problem patching, set this flag to 0 so we do not
+       directly return OPAL_SUCCESS if we call patcher_open() again. */
+    was_executed_already = 0;
+    opal_patcher_base_restore_all();
+
     return rc;
 }
 
 static int patcher_close(void)
 {
-    mca_base_framework_close (&opal_patcher_base_framework);
+    mca_base_framework_close(&opal_patcher_base_framework);
+
+    was_executed_already = 0;
 
     /* Note that we don't need to unpatch any symbols here; the
        patcher framework will take care of all of that for us. */
diff --git a/opal/mca/mpool/Makefile.am b/opal/mca/mpool/Makefile.am
index 27501f794fc..961383c51d0 100644
--- a/opal/mca/mpool/Makefile.am
+++ b/opal/mca/mpool/Makefile.am
@@ -10,6 +10,7 @@
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
 # Copyright (c) 2007-2010 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -31,7 +32,7 @@ opaldir = $(opalincludedir)/$(subdir)
 nobase_opal_HEADERS = $(headers)
 endif
 
-dist_opaldata_DATA =
+EXTRA_DIST =
 include base/Makefile.am
 
 distclean-local:
diff --git a/opal/mca/mpool/base/Makefile.am b/opal/mca/mpool/base/Makefile.am
index dd85a97fb6b..66b2bdbb668 100644
--- a/opal/mca/mpool/base/Makefile.am
+++ b/opal/mca/mpool/base/Makefile.am
@@ -12,6 +12,7 @@
 # Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2015      Los Alamos National Security, LLC. All rights
 #                         reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -31,5 +32,5 @@ libmca_mpool_la_SOURCES += \
 	base/mpool_base_default.c \
 	base/mpool_base_basic.c
 
-dist_opaldata_DATA += \
+EXTRA_DIST += \
         base/help-mpool-base.txt
diff --git a/opal/mca/mpool/base/base.h b/opal/mca/mpool/base/base.h
index 6d95665bff7..b56bad24fa0 100644
--- a/opal/mca/mpool/base/base.h
+++ b/opal/mca/mpool/base/base.h
@@ -50,10 +50,11 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(mca_mpool_base_selected_module_t);
  * Global functions for MCA: overall mpool open and close
  */
 
-OPAL_DECLSPEC mca_mpool_base_component_t* mca_mpool_base_component_lookup(const char* name);
-OPAL_DECLSPEC mca_mpool_base_module_t* mca_mpool_base_module_lookup(const char* name);
+OPAL_DECLSPEC mca_mpool_base_component_t *mca_mpool_base_component_lookup(const char *name);
+OPAL_DECLSPEC mca_mpool_base_module_t *mca_mpool_base_module_lookup(const char *name);
 
-OPAL_DECLSPEC mca_mpool_base_module_t *mca_mpool_basic_create (void *base, size_t size, unsigned min_align);
+OPAL_DECLSPEC mca_mpool_base_module_t *mca_mpool_basic_create(void *base, size_t size,
+                                                              unsigned min_align);
 
 /*
  * Globals
@@ -62,7 +63,6 @@ extern opal_list_t mca_mpool_base_modules;
 extern mca_mpool_base_module_t *mca_mpool_base_default_module;
 extern int mca_mpool_base_default_priority;
 
-
 OPAL_DECLSPEC extern mca_base_framework_t opal_mpool_base_framework;
 
 END_C_DECLS
diff --git a/opal/mca/mpool/base/mpool_base_alloc.c b/opal/mca/mpool/base/mpool_base_alloc.c
index b75d8b07ad6..09e5ae07edf 100644
--- a/opal/mca/mpool/base/mpool_base_alloc.c
+++ b/opal/mca/mpool/base/mpool_base_alloc.c
@@ -6,7 +6,7 @@
  * Copyright (c) 2004-2006 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ * Copyright (c) 2004-2020 High Performance Computing Center Stuttgart,
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
@@ -24,15 +24,14 @@
  */
 
 #include "opal_config.h"
-#include <stdint.h>
-#include <string.h>
-#include "opal/mca/mpool/mpool.h"
 #include "base.h"
 #include "mpool_base_tree.h"
+#include "opal/align.h"
+#include "opal/mca/mpool/mpool.h"
 #include "opal/mca/threads/mutex.h"
 #include "opal/util/info.h"
-#include "opal/align.h"
-
+#include <stdint.h>
+#include <string.h>
 
 static void unregister_tree_item(mca_mpool_base_tree_item_t *mpool_tree_item)
 {
@@ -63,32 +62,46 @@ void *mca_mpool_base_alloc(size_t size, opal_info_t *info, const char *hints)
     mca_mpool_base_tree_item_t *mpool_tree_item = NULL;
     mca_mpool_base_module_t *mpool;
     void *mem = NULL;
-#if defined(TODO_BTL_GB)
-    int flag = 0;
-#endif  /* defined(TODO_BTL_GB) */
+    opal_cstring_t *align_info_str;
+    long long memory_alignment = OPAL_ALIGN_MIN;
 
-    mpool_tree_item = mca_mpool_base_tree_item_get ();
+    mpool_tree_item = mca_mpool_base_tree_item_get();
     if (!mpool_tree_item) {
         return NULL;
     }
 
+    if (NULL != info) {
+        int flag;
+        opal_info_get(info, "mpi_minimum_memory_alignment",
+                      &align_info_str, &flag);
+
+        if (flag) {
+            long long tmp_align = atoll(align_info_str->string);
+            OBJ_RELEASE(align_info_str);
+            if (tmp_align > memory_alignment) {
+                memory_alignment = tmp_align;
+            }
+        }
+    }
+
     mpool_tree_item->num_bytes = size;
     mpool_tree_item->count = 0;
 
-    mpool = mca_mpool_base_module_lookup (hints);
+    mpool = mca_mpool_base_module_lookup(hints);
     if (NULL != mpool) {
-        mem = mpool->mpool_alloc (mpool, size, OPAL_ALIGN_MIN, 0);
+        mem = mpool->mpool_alloc (mpool, size, memory_alignment, 0);
     }
 
     if (NULL == mem) {
-        /* fall back on malloc */
-        mem = malloc(size);
+        /* fall back to default mpool */
+        mem = mca_mpool_base_default_module->mpool_alloc(mca_mpool_base_default_module,
+                                                         size, memory_alignment, 0);
 
-        mca_mpool_base_tree_item_put (mpool_tree_item);
+        mca_mpool_base_tree_item_put(mpool_tree_item);
     } else {
         mpool_tree_item->mpool = mpool;
         mpool_tree_item->key = mem;
-        mca_mpool_base_tree_insert (mpool_tree_item);
+        mca_mpool_base_tree_insert(mpool_tree_item);
     }
 
     return mem;
@@ -107,20 +120,20 @@ int mca_mpool_base_free(void *base)
     mca_mpool_base_tree_item_t *mpool_tree_item = NULL;
     int rc;
 
-    if(!base) {
+    if (!base) {
         return OPAL_ERROR;
     }
 
     mpool_tree_item = mca_mpool_base_tree_find(base);
 
-    if(!mpool_tree_item) {
+    if (!mpool_tree_item) {
         /* nothing in the tree this was just plain old malloc'd memory */
-        free(base);
+        mca_mpool_base_default_module->mpool_free(mca_mpool_base_default_module, base);
         return OPAL_SUCCESS;
     }
 
     rc = mca_mpool_base_tree_delete(mpool_tree_item);
-    if(OPAL_SUCCESS == rc) {
+    if (OPAL_SUCCESS == rc) {
         unregister_tree_item(mpool_tree_item);
         mca_mpool_base_tree_item_put(mpool_tree_item);
     }
diff --git a/opal/mca/mpool/base/mpool_base_basic.c b/opal/mca/mpool/base/mpool_base_basic.c
index e11f5273eae..61930c416ca 100644
--- a/opal/mca/mpool/base/mpool_base_basic.c
+++ b/opal/mca/mpool/base/mpool_base_basic.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2016      Intel, Inc. All rights reserved.
- * Copyrigth (c) 2018      Triad National Security, LLC. All rights
+ * Copyright (c) 2018      Triad National Security, LLC. All rights
  *                         reserved.
  * $COPYRIGHT$
  *
@@ -18,13 +18,13 @@
 #include <stdio.h>
 #include <stdlib.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif  /* HAVE_UNISTD_H */
+#    include <unistd.h>
+#endif /* HAVE_UNISTD_H */
 
-#include "opal/mca/mca.h"
+#include "opal/constants.h"
 #include "opal/mca/base/base.h"
+#include "opal/mca/mca.h"
 #include "opal/mca/mpool/base/base.h"
-#include "opal/constants.h"
 #include "opal/util/sys_limits.h"
 
 struct mca_mpool_base_basic_module_t {
@@ -37,14 +37,14 @@ struct mca_mpool_base_basic_module_t {
 };
 typedef struct mca_mpool_base_basic_module_t mca_mpool_base_basic_module_t;
 
-static void *mca_mpool_base_basic_alloc (mca_mpool_base_module_t *mpool, size_t size,
-                                         size_t align, uint32_t flags)
+static void *mca_mpool_base_basic_alloc(mca_mpool_base_module_t *mpool, size_t size, size_t align,
+                                        uint32_t flags)
 {
     mca_mpool_base_basic_module_t *basic_module = (mca_mpool_base_basic_module_t *) mpool;
     uintptr_t next_ptr;
     void *ptr;
 
-    opal_mutex_lock (&basic_module->lock);
+    opal_mutex_lock(&basic_module->lock);
 
     align = align > basic_module->min_align ? align : basic_module->min_align;
 
@@ -53,7 +53,7 @@ static void *mca_mpool_base_basic_alloc (mca_mpool_base_module_t *mpool, size_t
     size = OPAL_ALIGN(size, 8, size_t) + next_ptr - basic_module->ptr;
 
     if (size > basic_module->avail) {
-        opal_mutex_unlock (&basic_module->lock);
+        opal_mutex_unlock(&basic_module->lock);
         return NULL;
     }
 
@@ -61,31 +61,31 @@ static void *mca_mpool_base_basic_alloc (mca_mpool_base_module_t *mpool, size_t
     basic_module->avail -= size;
     basic_module->ptr += size;
 
-    opal_mutex_unlock (&basic_module->lock);
+    opal_mutex_unlock(&basic_module->lock);
     return ptr;
 }
 
 /**
-  * free function
-  */
-static void mca_mpool_base_basic_free (mca_mpool_base_module_t *mpool, void *addr)
+ * free function
+ */
+static void mca_mpool_base_basic_free(mca_mpool_base_module_t *mpool, void *addr)
 {
     /* nothing to do for now */
 }
 
-static void mca_mpool_base_basic_finalize (struct mca_mpool_base_module_t *mpool)
+static void mca_mpool_base_basic_finalize(struct mca_mpool_base_module_t *mpool)
 {
     mca_mpool_base_basic_module_t *basic_module = (mca_mpool_base_basic_module_t *) mpool;
 
     OBJ_DESTRUCT(&basic_module->lock);
-    free (mpool);
+    free(mpool);
 }
 
 static void *mca_mpool_base_basic_base(mca_mpool_base_module_t *mpool)
 {
     mca_mpool_base_basic_module_t *basic_module = (mca_mpool_base_basic_module_t *) mpool;
 
-    return (void*) basic_module->ptr;
+    return (void *) basic_module->ptr;
 }
 
 static mca_mpool_base_module_t mca_mpool_basic_template = {
@@ -96,15 +96,15 @@ static mca_mpool_base_module_t mca_mpool_basic_template = {
     .flags = MCA_MPOOL_FLAGS_MPI_ALLOC_MEM,
 };
 
-mca_mpool_base_module_t *mca_mpool_basic_create (void *base, size_t size, unsigned min_align)
+mca_mpool_base_module_t *mca_mpool_basic_create(void *base, size_t size, unsigned min_align)
 {
-    mca_mpool_base_basic_module_t *basic_module = calloc (1, sizeof (*basic_module));
+    mca_mpool_base_basic_module_t *basic_module = calloc(1, sizeof(*basic_module));
 
     if (OPAL_UNLIKELY(NULL == basic_module)) {
         return NULL;
     }
 
-    memcpy (&basic_module->super, &mca_mpool_basic_template, sizeof (mca_mpool_basic_template));
+    memcpy(&basic_module->super, &mca_mpool_basic_template, sizeof(mca_mpool_basic_template));
 
     OBJ_CONSTRUCT(&basic_module->lock, opal_mutex_t);
 
diff --git a/opal/mca/mpool/base/mpool_base_default.c b/opal/mca/mpool/base/mpool_base_default.c
index b176d3aaa75..e3816b1192f 100644
--- a/opal/mca/mpool/base/mpool_base_default.c
+++ b/opal/mca/mpool/base/mpool_base_default.c
@@ -16,67 +16,72 @@
 #include <stdio.h>
 #include <stdlib.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif  /* HAVE_UNISTD_H */
+#    include <unistd.h>
+#endif /* HAVE_UNISTD_H */
 
-#include "opal/mca/mca.h"
+#include "opal/constants.h"
 #include "opal/mca/base/base.h"
+#include "opal/mca/mca.h"
 #include "opal/mca/mpool/base/base.h"
-#include "opal/constants.h"
 #include "opal/util/sys_limits.h"
 
-static void *mca_mpool_default_alloc (mca_mpool_base_module_t *mpool, size_t size,
-                                      size_t align, uint32_t flags)
+static void *mca_mpool_default_alloc(mca_mpool_base_module_t *mpool, size_t size, size_t align,
+                                     uint32_t flags)
 {
 #if HAVE_POSIX_MEMALIGN
     void *addr = NULL;
 
     if (align <= sizeof(void *)) {
-        addr = malloc (size);
+        addr = malloc(size);
     } else {
-        (void) posix_memalign (&addr, align, size);
+        int ret = posix_memalign(&addr, align, size);
+        if (ret < 0) {
+            /* old systems may not guarantee that addr wasn't modified
+               on failure */
+            addr = NULL;
+        }
     }
     return addr;
 #else
     void *addr, *ret;
 
-    addr = malloc (size + align + sizeof (void *));
+    addr = malloc(size + align + sizeof(void *));
     ret = OPAL_ALIGN_PTR((intptr_t) addr + 8, align, void *);
     *((void **) ret - 1) = addr;
     return ret;
 #endif
 }
 
-static void *mca_mpool_default_realloc (mca_mpool_base_module_t *mpool, void *addr, size_t size)
+static void *mca_mpool_default_realloc(mca_mpool_base_module_t *mpool, void *addr, size_t size)
 {
 #if HAVE_POSIX_MEMALIGN
-    return realloc (addr, size);
+    return realloc(addr, size);
 #else
     if (NULL != addr) {
         void *base = *((void **) addr - 1);
-        void *ptr = realloc (base, size + (intptr_t) addr - (intptr_t) - size);
-        void *ret = (void *)((intptr_t) ptr + (intptr_t) addr - (intptr_t) - size);
+        void *ptr = realloc(base, size + (intptr_t) addr - (intptr_t) -size);
+        void *ret = (void *) ((intptr_t) ptr + (intptr_t) addr - (intptr_t) -size);
         *((void **) ret - 1) = ptr;
         return ret;
     } else {
-        return mca_mpool_default_alloc (mpool, size, 8, 0);
+        return mca_mpool_default_alloc(mpool, size, 8, 0);
     }
 #endif
 }
 
-static void mca_mpool_default_free (mca_mpool_base_module_t *mpool, void *addr)
+static void mca_mpool_default_free(mca_mpool_base_module_t *mpool, void *addr)
 {
 #if HAVE_POSIX_MEMALIGN
-    free (addr);
+    free(addr);
 #else
     if (NULL != addr) {
         void *base = *((void **) addr - 1);
-        free (base);
+        free(base);
     }
 #endif
 }
 
-static void mca_mpool_default_finalize (struct mca_mpool_base_module_t *mpool)
+static void mca_mpool_default_finalize(struct mca_mpool_base_module_t *mpool)
 {
 }
 
diff --git a/opal/mca/mpool/base/mpool_base_frame.c b/opal/mca/mpool/base/mpool_base_frame.c
index da0b146dac4..d003de0b31c 100644
--- a/opal/mca/mpool/base/mpool_base_frame.c
+++ b/opal/mca/mpool/base/mpool_base_frame.c
@@ -28,13 +28,13 @@
 #include <stdio.h>
 #include <stdlib.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif  /* HAVE_UNISTD_H */
+#    include <unistd.h>
+#endif /* HAVE_UNISTD_H */
 
-#include "opal/mca/mca.h"
+#include "opal/constants.h"
 #include "opal/mca/base/base.h"
+#include "opal/mca/mca.h"
 #include "opal/mca/mpool/base/base.h"
-#include "opal/constants.h"
 #include "opal/util/sys_limits.h"
 
 /*
@@ -56,23 +56,20 @@ int mca_mpool_base_default_priority = 50;
 
 OBJ_CLASS_INSTANCE(mca_mpool_base_selected_module_t, opal_list_item_t, NULL, NULL);
 
-static int mca_mpool_base_register (mca_base_register_flag_t flags)
+static int mca_mpool_base_register(mca_base_register_flag_t flags)
 {
     mca_mpool_base_default_hints = NULL;
-    (void) mca_base_var_register ("opal", "mpool", "base", "default_hints",
-                                  "Hints to use when selecting the default memory pool",
-                                  MCA_BASE_VAR_TYPE_STRING, NULL, 0,
-                                  MCA_BASE_VAR_FLAG_INTERNAL,
-                                  OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL,
-                                  &mca_mpool_base_default_hints);
+    (void) mca_base_var_register("opal", "mpool", "base", "default_hints",
+                                 "Hints to use when selecting the default memory pool",
+                                 MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_INTERNAL,
+                                 OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL,
+                                 &mca_mpool_base_default_hints);
 
     mca_mpool_base_default_priority = 50;
-    (void) mca_base_var_register ("opal", "mpool", "base", "default_priority",
-                                  "Priority of the default mpool module",
-                                  MCA_BASE_VAR_TYPE_INT, NULL, 0,
-                                  MCA_BASE_VAR_FLAG_INTERNAL,
-                                  OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL,
-                                  &mca_mpool_base_default_priority);
+    (void) mca_base_var_register("opal", "mpool", "base", "default_priority",
+                                 "Priority of the default mpool module", MCA_BASE_VAR_TYPE_INT,
+                                 NULL, 0, MCA_BASE_VAR_FLAG_INTERNAL, OPAL_INFO_LVL_9,
+                                 MCA_BASE_VAR_SCOPE_LOCAL, &mca_mpool_base_default_priority);
 
     return OPAL_SUCCESS;
 }
@@ -85,17 +82,16 @@ static int mca_mpool_base_open(mca_base_open_flag_t flags)
 {
     /* Open up all available components - and populate the
        opal_mpool_base_framework.framework_components list */
-    if (OPAL_SUCCESS !=
-        mca_base_framework_components_open(&opal_mpool_base_framework, flags)) {
+    if (OPAL_SUCCESS != mca_base_framework_components_open(&opal_mpool_base_framework, flags)) {
         return OPAL_ERROR;
     }
 
     if (mca_mpool_base_default_hints) {
-        mca_mpool_base_default_module = mca_mpool_base_module_lookup (mca_mpool_base_default_hints);
+        mca_mpool_base_default_module = mca_mpool_base_module_lookup(mca_mpool_base_default_hints);
     }
 
-     /* Initialize the list so that in mca_mpool_base_close(), we can
-        iterate over it (even if it's empty, as in the case of opal_info) */
+    /* Initialize the list so that in mca_mpool_base_close(), we can
+       iterate over it (even if it's empty, as in the case of opal_info) */
     OBJ_CONSTRUCT(&mca_mpool_base_modules, opal_list_t);
 
     /* setup tree for tracking MPI_Alloc_mem */
@@ -106,33 +102,34 @@ static int mca_mpool_base_open(mca_base_open_flag_t flags)
 
 static int mca_mpool_base_close(void)
 {
-  opal_list_item_t *item;
-  mca_mpool_base_selected_module_t *sm;
+    opal_list_item_t *item;
+    mca_mpool_base_selected_module_t *sm;
 
-  /* Finalize all the mpool components and free their list items */
+    /* Finalize all the mpool components and free their list items */
 
-  while(NULL != (item = opal_list_remove_first(&mca_mpool_base_modules))) {
-    sm = (mca_mpool_base_selected_module_t *) item;
+    while (NULL != (item = opal_list_remove_first(&mca_mpool_base_modules))) {
+        sm = (mca_mpool_base_selected_module_t *) item;
 
-    /* Blatently ignore the return code (what would we do to recover,
-       anyway?  This component is going away, so errors don't matter
-       anymore).  Note that it's legal for the module to have NULL for
-       the finalize function. */
+        /* Blatantly ignore the return code (what would we do to recover,
+           anyway?  This component is going away, so errors don't matter
+           anymore).  Note that it's legal for the module to have NULL for
+           the finalize function. */
 
-    if (NULL != sm->mpool_module->mpool_finalize) {
-        sm->mpool_module->mpool_finalize(sm->mpool_module);
+        if (NULL != sm->mpool_module->mpool_finalize) {
+            sm->mpool_module->mpool_finalize(sm->mpool_module);
+        }
+        OBJ_RELEASE(sm);
     }
-    OBJ_RELEASE(sm);
-  }
 
-  /* Close all remaining available components (may be one if this is a
-     OMPI RTE program, or [possibly] multiple if this is opal_info) */
-  (void) mca_base_framework_components_close(&opal_mpool_base_framework, NULL);
+    /* Close all remaining available components (may be one if this is a
+       OMPI RTE program, or [possibly] multiple if this is opal_info) */
+    (void) mca_base_framework_components_close(&opal_mpool_base_framework, NULL);
 
-  mca_mpool_base_tree_fini();
+    mca_mpool_base_tree_fini();
 
-  return OPAL_SUCCESS;
+    return OPAL_SUCCESS;
 }
 
-MCA_BASE_FRAMEWORK_DECLARE(opal, mpool, "Memory pools", mca_mpool_base_register, mca_mpool_base_open,
-                           mca_mpool_base_close, mca_mpool_base_static_components, 0);
+MCA_BASE_FRAMEWORK_DECLARE(opal, mpool, "Memory pools", mca_mpool_base_register,
+                           mca_mpool_base_open, mca_mpool_base_close,
+                           mca_mpool_base_static_components, 0);
diff --git a/opal/mca/mpool/base/mpool_base_lookup.c b/opal/mca/mpool/base/mpool_base_lookup.c
index fa0e0ce34af..d64fabcec3f 100644
--- a/opal/mca/mpool/base/mpool_base_lookup.c
+++ b/opal/mca/mpool/base/mpool_base_lookup.c
@@ -26,21 +26,22 @@
 #include <stdio.h>
 #include <string.h>
 
-#include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
-#include "opal/util/show_help.h"
+#include "opal/mca/mca.h"
 #include "opal/util/proc.h"
+#include "opal/util/show_help.h"
 
-#include "opal/mca/mpool/mpool.h"
 #include "opal/mca/mpool/base/base.h"
+#include "opal/mca/mpool/mpool.h"
 
-mca_mpool_base_component_t* mca_mpool_base_component_lookup(const char *name)
+mca_mpool_base_component_t *mca_mpool_base_component_lookup(const char *name)
 {
     mca_base_component_list_item_t *cli;
 
     /* Traverse the list of available modules; call their init functions. */
-    OPAL_LIST_FOREACH(cli, &opal_mpool_base_framework.framework_components, mca_base_component_list_item_t) {
-        mca_mpool_base_component_t* component = (mca_mpool_base_component_t *) cli->cli_component;
+    OPAL_LIST_FOREACH (cli, &opal_mpool_base_framework.framework_components,
+                       mca_base_component_list_item_t) {
+        mca_mpool_base_component_t *component = (mca_mpool_base_component_t *) cli->cli_component;
         if (strcmp(component->mpool_version.mca_component_name, name) == 0) {
             return component;
         }
@@ -49,27 +50,26 @@ mca_mpool_base_component_t* mca_mpool_base_component_lookup(const char *name)
     return NULL;
 }
 
-
-
-mca_mpool_base_module_t *mca_mpool_base_module_lookup (const char *hints)
+mca_mpool_base_module_t *mca_mpool_base_module_lookup(const char *hints)
 {
     mca_mpool_base_module_t *best_module = mca_mpool_base_default_module;
     mca_base_component_list_item_t *cli;
     int best_priority = mca_mpool_base_default_priority;
     int rc;
 
-    OPAL_LIST_FOREACH(cli, &opal_mpool_base_framework.framework_components, mca_base_component_list_item_t) {
+    OPAL_LIST_FOREACH (cli, &opal_mpool_base_framework.framework_components,
+                       mca_base_component_list_item_t) {
         mca_mpool_base_component_t *component = (mca_mpool_base_component_t *) cli->cli_component;
-	mca_mpool_base_module_t *module;
-	int priority;
+        mca_mpool_base_module_t *module;
+        int priority;
 
-	rc = component->mpool_query (hints, &priority, &module);
-	if (OPAL_SUCCESS == rc) {
-	    if (priority > best_priority) {
-		best_priority = priority;
-		best_module = module;
-	    }
-	}
+        rc = component->mpool_query(hints, &priority, &module);
+        if (OPAL_SUCCESS == rc) {
+            if (priority > best_priority) {
+                best_priority = priority;
+                best_module = module;
+            }
+        }
     }
 
     return best_module;
diff --git a/opal/mca/mpool/base/mpool_base_tree.c b/opal/mca/mpool/base/mpool_base_tree.c
index a5937f03820..29a5ba3e690 100644
--- a/opal/mca/mpool/base/mpool_base_tree.c
+++ b/opal/mca/mpool/base/mpool_base_tree.c
@@ -29,13 +29,12 @@
 #include "opal_config.h"
 
 #include "opal/mca/mca.h"
-#include "opal/util/show_help.h"
-#include "opal/util/proc.h"
 #include "opal/util/printf.h"
+#include "opal/util/proc.h"
+#include "opal/util/show_help.h"
 
-#include "opal/class/opal_rb_tree.h"
 #include "mpool_base_tree.h"
-
+#include "opal/class/opal_rb_tree.h"
 
 static int num_leaks = 0;
 static int max_mem_leaks = -1;
@@ -44,11 +43,13 @@ static char *leak_msg = NULL;
 static int condition(void *value);
 static void action(void *key, void *value);
 
-static void opal_mca_mpool_base_tree_constructor(mca_mpool_base_tree_item_t *item) {
+static void opal_mca_mpool_base_tree_constructor(mca_mpool_base_tree_item_t *item)
+{
     item->key = NULL;
 }
 
-OBJ_CLASS_INSTANCE(mca_mpool_base_tree_item_t, opal_free_list_item_t, opal_mca_mpool_base_tree_constructor, NULL);
+OBJ_CLASS_INSTANCE(mca_mpool_base_tree_item_t, opal_free_list_item_t,
+                   opal_mca_mpool_base_tree_constructor, NULL);
 
 /*
  * use globals for the tree and the tree_item free list..
@@ -60,18 +61,13 @@ static opal_mutex_t tree_lock;
 /*
  *  simple minded compare function...
  */
-int mca_mpool_base_tree_node_compare(void * key1, void * key2)
+int mca_mpool_base_tree_node_compare(void *key1, void *key2)
 {
-    if(key1 < key2)
-    {
+    if (key1 < key2) {
         return -1;
-    }
-    else if(key1 > key2)
-    {
+    } else if (key1 > key2) {
         return 1;
-    }
-    else
-    {
+    } else {
         return 0;
     }
 }
@@ -79,18 +75,17 @@ int mca_mpool_base_tree_node_compare(void * key1, void * key2)
 /*
  * initialize the rb tree
  */
-int mca_mpool_base_tree_init(void) {
+int mca_mpool_base_tree_init(void)
+{
     int rc;
     OBJ_CONSTRUCT(&mca_mpool_base_tree, opal_rb_tree_t);
     OBJ_CONSTRUCT(&mca_mpool_base_tree_item_free_list, opal_free_list_t);
     OBJ_CONSTRUCT(&tree_lock, opal_mutex_t);
-    rc = opal_free_list_init (&mca_mpool_base_tree_item_free_list,
-            sizeof(mca_mpool_base_tree_item_t),
-            opal_cache_line_size,
-            OBJ_CLASS(mca_mpool_base_tree_item_t),
-            0,opal_cache_line_size,
-            0, -1 , 4, NULL, 0, NULL, NULL, NULL);
-    if(OPAL_SUCCESS == rc) {
+    rc = opal_free_list_init(&mca_mpool_base_tree_item_free_list,
+                             sizeof(mca_mpool_base_tree_item_t), opal_cache_line_size,
+                             OBJ_CLASS(mca_mpool_base_tree_item_t), 0, opal_cache_line_size, 0, -1,
+                             4, NULL, 0, NULL, NULL, NULL);
+    if (OPAL_SUCCESS == rc) {
         rc = opal_rb_tree_init(&mca_mpool_base_tree, mca_mpool_base_tree_node_compare);
     }
     return rc;
@@ -110,7 +105,8 @@ int mca_mpool_base_tree_fini(void)
 /*
  * insert an item in the rb tree
  */
-int mca_mpool_base_tree_insert(mca_mpool_base_tree_item_t* item) {
+int mca_mpool_base_tree_insert(mca_mpool_base_tree_item_t *item)
+{
     int rc;
 
     OPAL_THREAD_LOCK(&tree_lock);
@@ -130,7 +126,8 @@ int mca_mpool_base_tree_insert(mca_mpool_base_tree_item_t* item) {
  * race conditions can occur
  *
  */
-int mca_mpool_base_tree_delete(mca_mpool_base_tree_item_t* item) {
+int mca_mpool_base_tree_delete(mca_mpool_base_tree_item_t *item)
+{
     int rc;
 
     OPAL_THREAD_LOCK(&tree_lock);
@@ -143,12 +140,12 @@ int mca_mpool_base_tree_delete(mca_mpool_base_tree_item_t* item) {
 /**
  *  find the item in the rb tree
  */
-mca_mpool_base_tree_item_t* mca_mpool_base_tree_find(void* base) {
-    mca_mpool_base_tree_item_t* item;
+mca_mpool_base_tree_item_t *mca_mpool_base_tree_find(void *base)
+{
+    mca_mpool_base_tree_item_t *item;
 
     OPAL_THREAD_LOCK(&tree_lock);
-    item = (mca_mpool_base_tree_item_t*)opal_rb_tree_find(&mca_mpool_base_tree,
-            base);
+    item = (mca_mpool_base_tree_item_t *) opal_rb_tree_find(&mca_mpool_base_tree, base);
     OPAL_THREAD_UNLOCK(&tree_lock);
 
     return item;
@@ -157,20 +154,19 @@ mca_mpool_base_tree_item_t* mca_mpool_base_tree_find(void* base) {
 /*
  * get a tree item from the free list
  */
-mca_mpool_base_tree_item_t* mca_mpool_base_tree_item_get(void) {
-    return (mca_mpool_base_tree_item_t *)
-        opal_free_list_get (&mca_mpool_base_tree_item_free_list);
+mca_mpool_base_tree_item_t *mca_mpool_base_tree_item_get(void)
+{
+    return (mca_mpool_base_tree_item_t *) opal_free_list_get(&mca_mpool_base_tree_item_free_list);
 }
 
 /*
  * put an item back into the free list
  */
-void mca_mpool_base_tree_item_put(mca_mpool_base_tree_item_t* item) {
-    opal_free_list_return (&mca_mpool_base_tree_item_free_list,
-                           &item->super);
+void mca_mpool_base_tree_item_put(mca_mpool_base_tree_item_t *item)
+{
+    opal_free_list_return(&mca_mpool_base_tree_item_free_list, &item->super);
 }
 
-
 /*
  * Print a show_help kind of message for an items still left in the
  * tree
@@ -189,52 +185,43 @@ void mca_mpool_base_tree_print(int show_up_to_mem_leaks)
         return;
     }
 
-    if (num_leaks <= show_up_to_mem_leaks ||
-        show_up_to_mem_leaks < 0) {
-        opal_show_help("help-mpool-base.txt", "all mem leaks",
-                       true, OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),
-                       opal_process_info.nodename,
-                       getpid(), leak_msg);
+    if (num_leaks <= show_up_to_mem_leaks || show_up_to_mem_leaks < 0) {
+        opal_show_help("help-mpool-base.txt", "all mem leaks", true,
+                       OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), opal_process_info.nodename, getpid(),
+                       leak_msg);
     } else {
         int i = num_leaks - show_up_to_mem_leaks;
-        opal_show_help("help-mpool-base.txt", "some mem leaks",
-                       true, OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),
-                       opal_process_info.nodename,
-                       getpid(), leak_msg, i,
-                       (i > 1) ? "s were" : " was",
-                       (i > 1) ? "are" : "is");
+        opal_show_help("help-mpool-base.txt", "some mem leaks", true,
+                       OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), opal_process_info.nodename, getpid(),
+                       leak_msg, i, (i > 1) ? "s were" : " was", (i > 1) ? "are" : "is");
     }
     free(leak_msg);
     leak_msg = NULL;
 }
 
-
 /* Condition function for rb traversal */
 static int condition(void *value)
 {
     return 1;
 }
 
-
 /* Action function for rb traversal */
 static void action(void *key, void *value)
 {
     char *tmp;
     mca_mpool_base_tree_item_t *item = (mca_mpool_base_tree_item_t *) value;
 
-    if( (++num_leaks <= max_mem_leaks) || (max_mem_leaks < 0) ) {
+    if ((++num_leaks <= max_mem_leaks) || (max_mem_leaks < 0)) {
 
         /* We know that we're supposed to make the first one; check on
            successive items if we're supposed to catenate more
            notices. */
         if (NULL == leak_msg) {
             opal_asprintf(&leak_msg, "    %lu bytes at address 0x%lx",
-                     (unsigned long) item->num_bytes,
-                     (unsigned long) key);
+                          (unsigned long) item->num_bytes, (unsigned long) key);
         } else {
-            opal_asprintf(&tmp, "%s\n    %lu bytes at address 0x%lx",
-                     leak_msg, (unsigned long) item->num_bytes,
-                     (unsigned long) key);
+            opal_asprintf(&tmp, "%s\n    %lu bytes at address 0x%lx", leak_msg,
+                          (unsigned long) item->num_bytes, (unsigned long) key);
             free(leak_msg);
             leak_msg = tmp;
         }
diff --git a/opal/mca/mpool/base/mpool_base_tree.h b/opal/mca/mpool/base/mpool_base_tree.h
index 2a31175a77f..6842eb98f9f 100644
--- a/opal/mca/mpool/base/mpool_base_tree.h
+++ b/opal/mca/mpool/base/mpool_base_tree.h
@@ -40,15 +40,14 @@ BEGIN_C_DECLS
 /**
  * The item in the tree itself
  */
-struct mca_mpool_base_tree_item_t
-{
-    opal_free_list_item_t super;   /**< the parent class */
-    void* key; /**< the address this was alloc'd on */
-    size_t num_bytes; /**< the number of bytes in this alloc, only for
-                           debugging reporting with
-                           mpi_show_mpi_alloc_mem_leaks */
+struct mca_mpool_base_tree_item_t {
+    opal_free_list_item_t super; /**< the parent class */
+    void *key;                   /**< the address this was alloc'd on */
+    size_t num_bytes;            /**< the number of bytes in this alloc, only for
+                                      debugging reporting with
+                                      mpi_show_mpi_alloc_mem_leaks */
     mca_mpool_base_module_t *mpool;
-    mca_rcache_base_module_t *rcaches[MCA_MPOOL_BASE_TREE_MAX]; /**< the registration caches */
+    mca_rcache_base_module_t *rcaches[MCA_MPOOL_BASE_TREE_MAX];    /**< the registration caches */
     mca_rcache_base_registration_t *regs[MCA_MPOOL_BASE_TREE_MAX]; /**< the registrations */
     uint8_t count; /**< length of the mpools/regs array */
 };
@@ -65,28 +64,27 @@ int mca_mpool_base_tree_fini(void);
 /*
  * insert an item in the rb tree
  */
-int mca_mpool_base_tree_insert(mca_mpool_base_tree_item_t* item);
+int mca_mpool_base_tree_insert(mca_mpool_base_tree_item_t *item);
 
 /*
  * remove an item from the rb tree
  */
-int mca_mpool_base_tree_delete(mca_mpool_base_tree_item_t* item);
-
+int mca_mpool_base_tree_delete(mca_mpool_base_tree_item_t *item);
 
 /**
  *  find the item in the rb tree
  */
-mca_mpool_base_tree_item_t* mca_mpool_base_tree_find(void* base);
+mca_mpool_base_tree_item_t *mca_mpool_base_tree_find(void *base);
 
 /*
  * get a tree item from the free list
  */
-mca_mpool_base_tree_item_t* mca_mpool_base_tree_item_get(void);
+mca_mpool_base_tree_item_t *mca_mpool_base_tree_item_get(void);
 
 /*
  * put tree item back into the free list
  */
-void mca_mpool_base_tree_item_put(mca_mpool_base_tree_item_t* item);
+void mca_mpool_base_tree_item_put(mca_mpool_base_tree_item_t *item);
 
 /*
  * For debugging, print a show_help kind of message if there are items
diff --git a/opal/mca/mpool/hugepage/Makefile.am b/opal/mca/mpool/hugepage/Makefile.am
index 1ecb9f7f9bc..3485166128b 100644
--- a/opal/mca/mpool/hugepage/Makefile.am
+++ b/opal/mca/mpool/hugepage/Makefile.am
@@ -45,7 +45,7 @@ mcacomponentdir = $(opallibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_mpool_hugepage_la_SOURCES = $(sources)
 mca_mpool_hugepage_la_LDFLAGS = -module -avoid-version
-mca_mpool_hugepage_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
+mca_mpool_hugepage_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
 	$(mpool_hugepage_LIBS)
 
 noinst_LTLIBRARIES = $(component_noinst)
diff --git a/opal/mca/mpool/hugepage/mpool_hugepage.h b/opal/mca/mpool/hugepage/mpool_hugepage.h
index 75da4a28c73..e78bac03591 100644
--- a/opal/mca/mpool/hugepage/mpool_hugepage.h
+++ b/opal/mca/mpool/hugepage/mpool_hugepage.h
@@ -29,13 +29,13 @@
 #define MCA_MPOOL_HUGEPAGE_H
 
 #include "opal_config.h"
-#include "opal/class/opal_list.h"
 #include "opal/class/opal_free_list.h"
+#include "opal/class/opal_list.h"
 #include "opal/class/opal_rb_tree.h"
-#include "opal/util/event.h"
+#include "opal/mca/allocator/allocator.h"
 #include "opal/mca/mpool/mpool.h"
+#include "opal/util/event.h"
 #include "opal/util/proc.h"
-#include "opal/mca/allocator/allocator.h"
 #include "opal/util/sys_limits.h"
 
 BEGIN_C_DECLS
@@ -60,13 +60,13 @@ struct mca_mpool_hugepage_hugepage_t {
     /** opal list item superclass */
     opal_list_item_t super;
     /** page size in bytes */
-    unsigned long    page_size;
+    unsigned long page_size;
     /** path for mmapped files */
-    char            *path;
+    char *path;
     /** counter to help ensure unique file names for mmaped files */
     opal_atomic_int32_t count;
     /** some platforms allow allocation of hugepages through mmap flags */
-    int              mmap_flags;
+    int mmap_flags;
 };
 typedef struct mca_mpool_hugepage_hugepage_t mca_mpool_hugepage_hugepage_t;
 
@@ -83,11 +83,11 @@ struct mca_mpool_hugepage_module_t {
 /*
  *  Initializes the mpool module.
  */
-int mca_mpool_hugepage_module_init (mca_mpool_hugepage_module_t *mpool,
-                                    mca_mpool_hugepage_hugepage_t *huge_page);
+int mca_mpool_hugepage_module_init(mca_mpool_hugepage_module_t *mpool,
+                                   mca_mpool_hugepage_hugepage_t *huge_page);
 
-void *mca_mpool_hugepage_seg_alloc (void *ctx, size_t *sizep);
-void mca_mpool_hugepage_seg_free (void *ctx, void *addr);
+void *mca_mpool_hugepage_seg_alloc(void *ctx, size_t *sizep);
+void mca_mpool_hugepage_seg_free(void *ctx, void *addr);
 
 END_C_DECLS
 #endif
diff --git a/opal/mca/mpool/hugepage/mpool_hugepage_component.c b/opal/mca/mpool/hugepage/mpool_hugepage_component.c
index f2af73345c4..191fc405e7b 100644
--- a/opal/mca/mpool/hugepage/mpool_hugepage_component.c
+++ b/opal/mca/mpool/hugepage/mpool_hugepage_component.c
@@ -11,7 +11,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2006      Voltaire. All rights reserved.
- * Copyright (c) 2007-2009 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2007-2022 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2012-2016 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2016      Intel, Inc. All rights reserved.
@@ -27,36 +27,36 @@
 
 #define OPAL_DISABLE_ENABLE_MEM_DEBUG 1
 #include "opal_config.h"
+#include "opal/mca/allocator/base/base.h"
 #include "opal/mca/base/base.h"
-#include "opal/runtime/opal_params.h"
 #include "opal/mca/base/mca_base_pvar.h"
 #include "opal/mca/mpool/base/base.h"
-#include "opal/mca/allocator/base/base.h"
+#include "opal/runtime/opal_params.h"
 
 #include "opal/util/argv.h"
 
 #include "mpool_hugepage.h"
 
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 #ifdef HAVE_MALLOC_H
-#include <malloc.h>
+#    include <malloc.h>
 #endif
 #ifdef HAVE_SYS_VFS_H
-#include <sys/vfs.h>
+#    include <sys/vfs.h>
 #endif
 #ifdef HAVE_SYS_MOUNT_H
-#include <sys/mount.h>
+#    include <sys/mount.h>
 #endif
 #ifdef HAVE_SYS_PARAM_H
-#include <sys/param.h>
+#    include <sys/param.h>
 #endif
 #ifdef HAVE_SYS_MMAN_H
-#include <sys/mman.h>
+#    include <sys/mman.h>
 #endif
 #ifdef HAVE_MNTENT_H
-#include <mntent.h>
+#    include <mntent.h>
 #endif
 
 #include <fcntl.h>
@@ -66,21 +66,20 @@
  * no struct statfs (!).  So check to make sure we have struct statfs
  * before allowing the use of statfs().
  */
-#if defined(HAVE_STATFS) && (defined(HAVE_STRUCT_STATFS_F_FSTYPENAME) || \
-                             defined(HAVE_STRUCT_STATFS_F_TYPE))
-#define USE_STATFS 1
+#if defined(HAVE_STATFS) \
+    && (defined(HAVE_STRUCT_STATFS_F_FSTYPENAME) || defined(HAVE_STRUCT_STATFS_F_TYPE))
+#    define USE_STATFS 1
 #endif
 
-
 /*
  * Local functions
  */
-static int mca_mpool_hugepage_open (void);
-static int mca_mpool_hugepage_close (void);
-static int mca_mpool_hugepage_register (void);
-static int mca_mpool_hugepage_query (const char *hints, int *priority,
-                                     mca_mpool_base_module_t **module);
-static void mca_mpool_hugepage_find_hugepages (void);
+static int mca_mpool_hugepage_open(void);
+static int mca_mpool_hugepage_close(void);
+static int mca_mpool_hugepage_register(void);
+static int mca_mpool_hugepage_query(const char *hints, int *priority,
+                                    mca_mpool_base_module_t **module);
+static void mca_mpool_hugepage_find_hugepages(void);
 
 static int mca_mpool_hugepage_priority;
 static unsigned long mca_mpool_hugepage_page_size;
@@ -90,80 +89,88 @@ mca_mpool_hugepage_component_t mca_mpool_hugepage_component = {
         /* First, the mca_base_component_t struct containing meta
            information about the component itself */
 
-        .mpool_version ={
-            MCA_MPOOL_BASE_VERSION_3_1_0,
-
-            .mca_component_name = "hugepage",
-            MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
-                                  OPAL_RELEASE_VERSION),
-            .mca_open_component = mca_mpool_hugepage_open,
-            .mca_close_component = mca_mpool_hugepage_close,
-            .mca_register_component_params = mca_mpool_hugepage_register,
-        },
-        .mpool_data = {
-            /* The component is checkpoint ready */
-            MCA_BASE_METADATA_PARAM_CHECKPOINT
-        },
+        .mpool_version =
+            {
+                MCA_MPOOL_BASE_VERSION_3_1_0,
+
+                .mca_component_name = "hugepage",
+                MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
+                                      OPAL_RELEASE_VERSION),
+                .mca_open_component = mca_mpool_hugepage_open,
+                .mca_close_component = mca_mpool_hugepage_close,
+                .mca_register_component_params = mca_mpool_hugepage_register,
+            },
+        .mpool_data =
+            {/* The component is checkpoint ready */
+             MCA_BASE_METADATA_PARAM_CHECKPOINT},
 
         .mpool_query = mca_mpool_hugepage_query,
     },
 };
 
 /**
-  * component open/close/init function
-  */
+ * component open/close/init function
+ */
 
 static int mca_mpool_hugepage_register(void)
 {
     mca_mpool_hugepage_priority = 50;
-    (void) mca_base_component_var_register (&mca_mpool_hugepage_component.super.mpool_version,
-                                            "priority", "Default priority of the hugepage mpool component "
-                                            "(default: 50)", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                            OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL,
-                                            &mca_mpool_hugepage_priority);
+    (void) mca_base_component_var_register(&mca_mpool_hugepage_component.super.mpool_version,
+                                           "priority",
+                                           "Default priority of the hugepage mpool component "
+                                           "(default: 50)",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_LOCAL, &mca_mpool_hugepage_priority);
 
     mca_mpool_hugepage_page_size = 1 << 21;
-    (void) mca_base_component_var_register (&mca_mpool_hugepage_component.super.mpool_version,
-                                            "page_size", "Default huge page size of the hugepage mpool component "
-                                            "(default: 2M)", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                            OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL,
-                                            &mca_mpool_hugepage_page_size);
+    (void) mca_base_component_var_register(&mca_mpool_hugepage_component.super.mpool_version,
+                                           "page_size",
+                                           "Default huge page size of the hugepage mpool component "
+                                           "(default: 2M)",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_LOCAL, &mca_mpool_hugepage_page_size);
 
     mca_mpool_hugepage_component.bytes_allocated = 0;
-    (void) mca_base_component_pvar_register (&mca_mpool_hugepage_component.super.mpool_version,
-                                             "bytes_allocated", "Number of bytes currently allocated in the mpool "
-                                             "hugepage component", OPAL_INFO_LVL_3, MCA_BASE_PVAR_CLASS_SIZE,
-                                             MCA_BASE_VAR_TYPE_UNSIGNED_LONG, NULL, MCA_BASE_VAR_BIND_NO_OBJECT,
-                                             MCA_BASE_PVAR_FLAG_READONLY | MCA_BASE_PVAR_FLAG_CONTINUOUS,
-                                             NULL, NULL, NULL, (void *) &mca_mpool_hugepage_component.bytes_allocated);
+    (void) mca_base_component_pvar_register(&mca_mpool_hugepage_component.super.mpool_version,
+                                            "bytes_allocated",
+                                            "Number of bytes currently allocated in the mpool "
+                                            "hugepage component",
+                                            OPAL_INFO_LVL_3, MCA_BASE_PVAR_CLASS_SIZE,
+                                            MCA_BASE_VAR_TYPE_UNSIGNED_LONG, NULL,
+                                            MCA_BASE_VAR_BIND_NO_OBJECT,
+                                            MCA_BASE_PVAR_FLAG_READONLY
+                                                | MCA_BASE_PVAR_FLAG_CONTINUOUS,
+                                            NULL, NULL, NULL,
+                                            (void *) &mca_mpool_hugepage_component.bytes_allocated);
 
     return OPAL_SUCCESS;
 }
 
-static int mca_mpool_hugepage_open (void)
+static int mca_mpool_hugepage_open(void)
 {
     mca_mpool_hugepage_module_t *hugepage_module;
     mca_mpool_hugepage_hugepage_t *hp;
     int module_index, rc;
 
     OBJ_CONSTRUCT(&mca_mpool_hugepage_component.huge_pages, opal_list_t);
-    mca_mpool_hugepage_find_hugepages ();
+    mca_mpool_hugepage_find_hugepages();
 
-    if (0 == opal_list_get_size (&mca_mpool_hugepage_component.huge_pages)) {
+    if (0 == opal_list_get_size(&mca_mpool_hugepage_component.huge_pages)) {
         return OPAL_SUCCESS;
     }
 
     mca_mpool_hugepage_component.modules = (mca_mpool_hugepage_module_t *)
-        calloc (opal_list_get_size (&mca_mpool_hugepage_component.huge_pages),
-                sizeof (mca_mpool_hugepage_module_t));
+        calloc(opal_list_get_size(&mca_mpool_hugepage_component.huge_pages),
+               sizeof(mca_mpool_hugepage_module_t));
     if (NULL == mca_mpool_hugepage_component.modules) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
     module_index = 0;
-    OPAL_LIST_FOREACH(hp, &mca_mpool_hugepage_component.huge_pages, mca_mpool_hugepage_hugepage_t) {
+    OPAL_LIST_FOREACH (hp, &mca_mpool_hugepage_component.huge_pages,
+                       mca_mpool_hugepage_hugepage_t) {
         hugepage_module = mca_mpool_hugepage_component.modules + module_index;
-        rc = mca_mpool_hugepage_module_init (hugepage_module, hp);
+        rc = mca_mpool_hugepage_module_init(hugepage_module, hp);
         if (OPAL_SUCCESS != rc) {
             continue;
         }
@@ -175,23 +182,24 @@ static int mca_mpool_hugepage_open (void)
     return OPAL_SUCCESS;
 }
 
-static int mca_mpool_hugepage_close (void)
+static int mca_mpool_hugepage_close(void)
 {
     OPAL_LIST_DESTRUCT(&mca_mpool_hugepage_component.huge_pages);
 
-    for (int i = 0 ; i < mca_mpool_hugepage_component.module_count ; ++i) {
-        mca_mpool_hugepage_module_t *module =  mca_mpool_hugepage_component.modules + i;
-        module->super.mpool_finalize (&module->super);
+    for (int i = 0; i < mca_mpool_hugepage_component.module_count; ++i) {
+        mca_mpool_hugepage_module_t *module = mca_mpool_hugepage_component.modules + i;
+        module->super.mpool_finalize(&module->super);
     }
 
-    free (mca_mpool_hugepage_component.modules);
+    free(mca_mpool_hugepage_component.modules);
     mca_mpool_hugepage_component.modules = NULL;
 
     return OPAL_SUCCESS;
 }
 
 #ifdef HAVE_MNTENT_H
-static int page_compare (opal_list_item_t **a, opal_list_item_t **b) {
+static int page_compare(opal_list_item_t **a, opal_list_item_t **b)
+{
     mca_mpool_hugepage_hugepage_t *pagea = (mca_mpool_hugepage_hugepage_t *) *a;
     mca_mpool_hugepage_hugepage_t *pageb = (mca_mpool_hugepage_hugepage_t *) *b;
     if (pagea->page_size > pageb->page_size) {
@@ -204,14 +212,14 @@ static int page_compare (opal_list_item_t **a, opal_list_item_t **b) {
 }
 #endif
 
-static void mca_mpool_hugepage_find_hugepages (void) {
+static void mca_mpool_hugepage_find_hugepages(void)
+{
 #ifdef HAVE_MNTENT_H
     mca_mpool_hugepage_hugepage_t *hp;
     FILE *fh;
     struct mntent *mntent;
-    char *opts, *tok, *ctx;
 
-    fh = setmntent ("/proc/mounts", "r");
+    fh = setmntent("/proc/mounts", "r");
     if (NULL == fh) {
         return;
     }
@@ -223,34 +231,39 @@ static void mca_mpool_hugepage_find_hugepages (void) {
             continue;
         }
 
+#if defined(USE_STATFS)
+        struct statfs info;
+        statfs(mntent->mnt_dir, &info);
+        page_size = info.f_bsize;
+#elif defined(HAVE_STATVFS)
+        struct statvfs info;
+        statvfs(mntent->mnt_dir, &info);
+        page_size = info.f_bsize;
+#else
+        // Fallback for extremely old systems that do not have
+        // statfs().
+        char *opts, *tok, *ctx;
         opts = strdup(mntent->mnt_opts);
         if (NULL == opts) {
             break;
         }
 
-        tok = strtok_r (opts, ",", &ctx);
-
+        tok = strtok_r(opts, ",", &ctx);
         do {
-            if (0 == strncmp (tok, "pagesize", 8)) {
-                break;
+            if (NULL != tok && 0 == strncmp(tok, "pagesize", 8)) {
+                // It is expected that pagesize=X will be an integer
+                // number with no units qualifier following it.
+                // Specifically: Linux circa 2025 has /proc/mounts
+                // output like "... rw,relatime,pagesize=2M".  But if
+                // your system is signifncantly older than that
+                // (statfs() was introduced around 1994), we're
+                // assuming that there is no units qualifier.
+                (void) sscanf(tok, "pagesize=%lu", &page_size);
             }
-            tok = strtok_r (NULL, ",", &ctx);
+            tok = strtok_r(NULL, ",", &ctx);
         } while (tok);
-
-        if (!tok) {
-#if defined(USE_STATFS)
-            struct statfs info;
-
-            statfs (mntent->mnt_dir, &info);
-#elif defined(HAVE_STATVFS)
-            struct statvfs info;
-            statvfs (mntent->mnt_dir, &info);
-#endif
-            page_size = info.f_bsize;
-        } else {
-            (void) sscanf (tok, "pagesize=%lu", &page_size);
-        }
         free(opts);
+#endif
 
         if (0 == page_size) {
             /* could not get page size */
@@ -262,30 +275,33 @@ static void mca_mpool_hugepage_find_hugepages (void) {
             break;
         }
 
-        hp->path = strdup (mntent->mnt_dir);
+        hp->path = strdup(mntent->mnt_dir);
         hp->page_size = page_size;
-        
-        if(0 == access (hp->path, R_OK | W_OK)){        
-            opal_output_verbose (MCA_BASE_VERBOSE_INFO, opal_mpool_base_framework.framework_output,
-                                 "found huge page with size = %lu, path = %s, mmap flags = 0x%x, adding to list",
-                                 hp->page_size, hp->path, hp->mmap_flags);
-            opal_list_append (&mca_mpool_hugepage_component.huge_pages, &hp->super);
+
+        if (0 == access(hp->path, R_OK | W_OK)) {
+            opal_output_verbose(
+                MCA_BASE_VERBOSE_INFO, opal_mpool_base_framework.framework_output,
+                "found huge page with size = %lu, path = %s, mmap flags = 0x%x, adding to list",
+                hp->page_size, hp->path, hp->mmap_flags);
+            opal_list_append(&mca_mpool_hugepage_component.huge_pages, &hp->super);
         } else {
-            opal_output_verbose (MCA_BASE_VERBOSE_INFO, opal_mpool_base_framework.framework_output,
-                                 "found huge page with size = %lu, path = %s, mmap flags = 0x%x, with invalid " 
-                                 "permissions, skipping", hp->page_size, hp->path, hp->mmap_flags);
+            opal_output_verbose(
+                MCA_BASE_VERBOSE_INFO, opal_mpool_base_framework.framework_output,
+                "found huge page with size = %lu, path = %s, mmap flags = 0x%x, with invalid "
+                "permissions, skipping",
+                hp->page_size, hp->path, hp->mmap_flags);
             OBJ_RELEASE(hp);
-        }        
+        }
     }
 
-    opal_list_sort (&mca_mpool_hugepage_component.huge_pages, page_compare);
+    opal_list_sort(&mca_mpool_hugepage_component.huge_pages, page_compare);
 
-    endmntent (fh);
+    endmntent(fh);
 #endif
 }
 
-static int mca_mpool_hugepage_query (const char *hints, int *priority_out,
-                                     mca_mpool_base_module_t **module)
+static int mca_mpool_hugepage_query(const char *hints, int *priority_out,
+                                    mca_mpool_base_module_t **module)
 {
     unsigned long page_size = 0;
     char **hints_array;
@@ -298,38 +314,40 @@ static int mca_mpool_hugepage_query (const char *hints, int *priority_out,
     }
 
     if (hints) {
-        hints_array = opal_argv_split (hints, ',');
+        hints_array = opal_argv_split(hints, ',');
         if (NULL == hints_array) {
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
 
-        for (int i = 0 ; hints_array[i] ; ++i) {
+        for (int i = 0; hints_array[i]; ++i) {
             char *key = hints_array[i];
             char *value = NULL;
 
-            if (NULL != (tmp = strchr (key, '='))) {
+            if (NULL != (tmp = strchr(key, '='))) {
                 value = tmp + 1;
                 *tmp = '\0';
             }
 
-            if (0 == strcasecmp ("mpool", key)) {
-                if (value && 0 == strcasecmp ("hugepage", value)) {
+            if (0 == strcasecmp("mpool", key)) {
+                if (value && 0 == strcasecmp("hugepage", value)) {
                     /* this mpool was requested by name */
                     my_priority = 100;
-                    opal_output_verbose (MCA_BASE_VERBOSE_INFO, opal_mpool_base_framework.framework_output,
-                                         "hugepage mpool matches hint: %s=%s", key, value);
+                    opal_output_verbose(MCA_BASE_VERBOSE_INFO,
+                                        opal_mpool_base_framework.framework_output,
+                                        "hugepage mpool matches hint: %s=%s", key, value);
                 } else {
                     /* different mpool requested */
                     my_priority = 0;
-                    opal_output_verbose (MCA_BASE_VERBOSE_INFO, opal_mpool_base_framework.framework_output,
-                                         "hugepage mpool does not match hint: %s=%s", key, value);
-                    opal_argv_free (hints_array);
+                    opal_output_verbose(MCA_BASE_VERBOSE_INFO,
+                                        opal_mpool_base_framework.framework_output,
+                                        "hugepage mpool does not match hint: %s=%s", key, value);
+                    opal_argv_free(hints_array);
                     return OPAL_ERR_NOT_FOUND;
                 }
             }
 
-            if (0 == strcasecmp ("page_size", key) && value) {
-                page_size = strtoul (value, &tmp, 0);
+            if (0 == strcasecmp("page_size", key) && value) {
+                page_size = strtoul(value, &tmp, 0);
                 if (*tmp) {
                     switch (*tmp) {
                     case 'g':
@@ -348,12 +366,13 @@ static int mca_mpool_hugepage_query (const char *hints, int *priority_out,
                         page_size = -1;
                     }
                 }
-                opal_output_verbose (MCA_BASE_VERBOSE_INFO, opal_mpool_base_framework.framework_output,
-                                     "hugepage mpool requested page size: %lu", page_size);
+                opal_output_verbose(MCA_BASE_VERBOSE_INFO,
+                                    opal_mpool_base_framework.framework_output,
+                                    "hugepage mpool requested page size: %lu", page_size);
             }
         }
 
-        opal_argv_free (hints_array);
+        opal_argv_free(hints_array);
     }
 
     if (0 == page_size) {
@@ -363,11 +382,11 @@ static int mca_mpool_hugepage_query (const char *hints, int *priority_out,
             /* take a priority hit if this mpool was not asked for by name */
             my_priority = 0;
         }
-        opal_output_verbose (MCA_BASE_VERBOSE_WARN, opal_mpool_base_framework.framework_output,
-                             "hugepage mpool did not match any hints: %s", hints);
+        opal_output_verbose(MCA_BASE_VERBOSE_WARN, opal_mpool_base_framework.framework_output,
+                            "hugepage mpool did not match any hints: %s", hints);
     }
 
-    for (int i = 0 ; i < mca_mpool_hugepage_component.module_count ; ++i) {
+    for (int i = 0; i < mca_mpool_hugepage_component.module_count; ++i) {
         mca_mpool_hugepage_module_t *hugepage_module = mca_mpool_hugepage_component.modules + i;
 
         if (hugepage_module->huge_page->page_size != page_size) {
@@ -380,17 +399,18 @@ static int mca_mpool_hugepage_query (const char *hints, int *priority_out,
             *module = &hugepage_module->super;
         }
 
-        opal_output_verbose (MCA_BASE_VERBOSE_INFO, opal_mpool_base_framework.framework_output,
-                             "matches page size hint. page size: %lu, path: %s, mmap flags: "
-                             "0x%x", page_size, hugepage_module->huge_page->path,
-                             hugepage_module->huge_page->mmap_flags);
+        opal_output_verbose(MCA_BASE_VERBOSE_INFO, opal_mpool_base_framework.framework_output,
+                            "matches page size hint. page size: %lu, path: %s, mmap flags: "
+                            "0x%x",
+                            page_size, hugepage_module->huge_page->path,
+                            hugepage_module->huge_page->mmap_flags);
         found = true;
         break;
     }
 
     if (!found) {
-        opal_output_verbose (MCA_BASE_VERBOSE_WARN, opal_mpool_base_framework.framework_output,
-                             "could not find page matching page request: %lu", page_size);
+        opal_output_verbose(MCA_BASE_VERBOSE_WARN, opal_mpool_base_framework.framework_output,
+                            "could not find page matching page request: %lu", page_size);
         return OPAL_ERR_NOT_FOUND;
     }
 
diff --git a/opal/mca/mpool/hugepage/mpool_hugepage_module.c b/opal/mca/mpool/hugepage/mpool_hugepage_module.c
index d2010e8a6e8..338dc92fb50 100644
--- a/opal/mca/mpool/hugepage/mpool_hugepage_module.c
+++ b/opal/mca/mpool/hugepage/mpool_hugepage_module.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2013 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -29,44 +29,43 @@
 
 #define OPAL_DISABLE_ENABLE_MEM_DEBUG 1
 #include "opal_config.h"
-#include "opal/align.h"
 #include "mpool_hugepage.h"
+#include "opal/align.h"
 #include <errno.h>
 #include <string.h>
 #ifdef HAVE_MALLOC_H
-#include <malloc.h>
+#    include <malloc.h>
 #endif
-#include "opal/mca/mpool/base/base.h"
-#include "opal/runtime/opal_params.h"
 #include "opal/include/opal_stdint.h"
 #include "opal/mca/allocator/base/base.h"
+#include "opal/mca/mpool/base/base.h"
+#include "opal/runtime/opal_params.h"
 #include "opal/util/printf.h"
 
 #include <fcntl.h>
 #include <sys/mman.h>
 
+static void *mca_mpool_hugepage_alloc(mca_mpool_base_module_t *mpool, size_t size, size_t align,
+                                      uint32_t flags);
+static void *mca_mpool_hugepage_realloc(mca_mpool_base_module_t *mpool, void *addr, size_t size);
+static void mca_mpool_hugepage_free(mca_mpool_base_module_t *mpool, void *addr);
+static void mca_mpool_hugepage_finalize(mca_mpool_base_module_t *mpool);
 
-static void *mca_mpool_hugepage_alloc (mca_mpool_base_module_t *mpool, size_t size, size_t align,
-                                   uint32_t flags);
-static void *mca_mpool_hugepage_realloc (mca_mpool_base_module_t *mpool, void *addr, size_t size);
-static void mca_mpool_hugepage_free (mca_mpool_base_module_t *mpool, void *addr);
-static void mca_mpool_hugepage_finalize (mca_mpool_base_module_t *mpool);
-
-static void mca_mpool_hugepage_hugepage_constructor (mca_mpool_hugepage_hugepage_t *huge_page)
+static void mca_mpool_hugepage_hugepage_constructor(mca_mpool_hugepage_hugepage_t *huge_page)
 {
-    memset ((char *)huge_page + sizeof(huge_page->super), 0, sizeof (*huge_page) - sizeof (huge_page->super));
+    memset((char *) huge_page + sizeof(huge_page->super), 0,
+           sizeof(*huge_page) - sizeof(huge_page->super));
 }
 
-static void mca_mpool_hugepage_hugepage_destructor (mca_mpool_hugepage_hugepage_t *huge_page)
+static void mca_mpool_hugepage_hugepage_destructor(mca_mpool_hugepage_hugepage_t *huge_page)
 {
-    free (huge_page->path);
+    free(huge_page->path);
 }
 
 OBJ_CLASS_INSTANCE(mca_mpool_hugepage_hugepage_t, opal_list_item_t,
-                   mca_mpool_hugepage_hugepage_constructor,
-                   mca_mpool_hugepage_hugepage_destructor);
+                   mca_mpool_hugepage_hugepage_constructor, mca_mpool_hugepage_hugepage_destructor);
 
-static int mca_mpool_rb_hugepage_compare (void *key1, void *key2)
+static int mca_mpool_rb_hugepage_compare(void *key1, void *key2)
 {
     if (key1 == key2) {
         return 0;
@@ -79,7 +78,7 @@ static int mca_mpool_rb_hugepage_compare (void *key1, void *key2)
  *  Initializes the mpool module.
  */
 int mca_mpool_hugepage_module_init(mca_mpool_hugepage_module_t *mpool,
-                                  mca_mpool_hugepage_hugepage_t *huge_page)
+                                   mca_mpool_hugepage_hugepage_t *huge_page)
 {
     mca_allocator_base_component_t *allocator_component;
     int rc;
@@ -97,16 +96,16 @@ int mca_mpool_hugepage_module_init(mca_mpool_hugepage_module_t *mpool,
     mpool->huge_page = huge_page;
 
     /* use an allocator component to reduce waste when making small allocations */
-    allocator_component = mca_allocator_component_lookup ("bucket");
+    allocator_component = mca_allocator_component_lookup("bucket");
     if (NULL == allocator_component) {
         return OPAL_ERR_NOT_AVAILABLE;
     }
 
-    mpool->allocator = allocator_component->allocator_init (true, mca_mpool_hugepage_seg_alloc,
-                                                            mca_mpool_hugepage_seg_free, mpool);
+    mpool->allocator = allocator_component->allocator_init(true, mca_mpool_hugepage_seg_alloc,
+                                                           mca_mpool_hugepage_seg_free, mpool);
 
     OBJ_CONSTRUCT(&mpool->allocation_tree, opal_rb_tree_t);
-    rc = opal_rb_tree_init (&mpool->allocation_tree, mca_mpool_rb_hugepage_compare);
+    rc = opal_rb_tree_init(&mpool->allocation_tree, mca_mpool_rb_hugepage_compare);
     if (OPAL_SUCCESS != rc) {
         OBJ_DESTRUCT(&mpool->allocation_tree);
         return OPAL_ERR_NOT_AVAILABLE;
@@ -115,7 +114,7 @@ int mca_mpool_hugepage_module_init(mca_mpool_hugepage_module_t *mpool,
     return OPAL_SUCCESS;
 }
 
-void *mca_mpool_hugepage_seg_alloc (void *ctx, size_t *sizep)
+void *mca_mpool_hugepage_seg_alloc(void *ctx, size_t *sizep)
 {
     mca_mpool_hugepage_module_t *hugepage_module = (mca_mpool_hugepage_module_t *) ctx;
     mca_mpool_hugepage_hugepage_t *huge_page = hugepage_module->huge_page;
@@ -131,24 +130,23 @@ void *mca_mpool_hugepage_seg_alloc (void *ctx, size_t *sizep)
     if (huge_page->path) {
         int32_t count;
 
-        count = opal_atomic_add_fetch_32 (&huge_page->count, 1);
+        count = opal_atomic_add_fetch_32(&huge_page->count, 1);
 
-        rc = opal_asprintf (&path, "%s/hugepage.openmpi.%d.%d", huge_page->path,
-                       getpid (), count);
+        rc = opal_asprintf(&path, "%s/hugepage.openmpi.%d.%d", huge_page->path, getpid(), count);
         if (0 > rc) {
             return NULL;
         }
 
-        fd = open (path, O_RDWR | O_CREAT, 0600);
+        fd = open(path, O_RDWR | O_CREAT, 0600);
         if (-1 == fd) {
-            free (path);
+            free(path);
             return NULL;
         }
 
-        if (0 != ftruncate (fd, size)) {
-            close (fd);
-            unlink (path);
-            free (path);
+        if (0 != ftruncate(fd, size)) {
+            close(fd);
+            unlink(path);
+            free(path);
             return NULL;
         }
     } else {
@@ -160,31 +158,31 @@ void *mca_mpool_hugepage_seg_alloc (void *ctx, size_t *sizep)
 #endif
     }
 
-    base = mmap (NULL, size, PROT_READ | PROT_WRITE, flags | huge_page->mmap_flags, fd, 0);
+    base = mmap(NULL, size, PROT_READ | PROT_WRITE, flags | huge_page->mmap_flags, fd, 0);
     if (path) {
-        unlink (path);
-        free (path);
+        unlink(path);
+        free(path);
     }
 
     if (fd >= 0) {
-        close (fd);
+        close(fd);
     }
 
     if (MAP_FAILED == base) {
-        opal_output_verbose (MCA_BASE_VERBOSE_WARN, opal_mpool_base_framework.framework_verbose,
-                             "could not allocate huge page(s). falling back on standard pages");
+        opal_output_verbose(MCA_BASE_VERBOSE_WARN, opal_mpool_base_framework.framework_verbose,
+                            "could not allocate huge page(s). falling back on standard pages");
         /* fall back on regular pages */
-        base = mmap (NULL, size, PROT_READ | PROT_WRITE, flags, -1, 0);
+        base = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, -1, 0);
     }
 
     if (MAP_FAILED == base) {
         return NULL;
     }
 
-    opal_mutex_lock (&hugepage_module->lock);
-    opal_rb_tree_insert (&hugepage_module->allocation_tree, base, (void *) (intptr_t) size);
-    (void) opal_atomic_fetch_add_size_t (&mca_mpool_hugepage_component.bytes_allocated, size);
-    opal_mutex_unlock (&hugepage_module->lock);
+    opal_mutex_lock(&hugepage_module->lock);
+    opal_rb_tree_insert(&hugepage_module->allocation_tree, base, (void *) (intptr_t) size);
+    (void) opal_atomic_fetch_add_size_t(&mca_mpool_hugepage_component.bytes_allocated, size);
+    opal_mutex_unlock(&hugepage_module->lock);
 
     OPAL_OUTPUT_VERBOSE((MCA_BASE_VERBOSE_TRACE, opal_mpool_base_framework.framework_verbose,
                          "allocated segment %p of size %lu bytes", base, size));
@@ -194,64 +192,64 @@ void *mca_mpool_hugepage_seg_alloc (void *ctx, size_t *sizep)
     return base;
 }
 
-void mca_mpool_hugepage_seg_free (void *ctx, void *addr)
+void mca_mpool_hugepage_seg_free(void *ctx, void *addr)
 {
     mca_mpool_hugepage_module_t *hugepage_module = (mca_mpool_hugepage_module_t *) ctx;
     size_t size;
 
-    opal_mutex_lock (&hugepage_module->lock);
+    opal_mutex_lock(&hugepage_module->lock);
 
-    size = (size_t) (intptr_t) opal_rb_tree_find (&hugepage_module->allocation_tree, addr);
+    size = (size_t)(intptr_t) opal_rb_tree_find(&hugepage_module->allocation_tree, addr);
     if (size > 0) {
-        opal_rb_tree_delete (&hugepage_module->allocation_tree, addr);
+        opal_rb_tree_delete(&hugepage_module->allocation_tree, addr);
         OPAL_OUTPUT_VERBOSE((MCA_BASE_VERBOSE_TRACE, opal_mpool_base_framework.framework_verbose,
                              "freeing segment %p of size %lu bytes", addr, size));
-        munmap (addr, size);
-        (void) opal_atomic_fetch_add_size_t (&mca_mpool_hugepage_component.bytes_allocated, -size);
+        munmap(addr, size);
+        (void) opal_atomic_fetch_add_size_t(&mca_mpool_hugepage_component.bytes_allocated, -size);
     }
 
-    opal_mutex_unlock (&hugepage_module->lock);
+    opal_mutex_unlock(&hugepage_module->lock);
 }
 
 /**
-  * allocate function
-  */
-static void *mca_mpool_hugepage_alloc (mca_mpool_base_module_t *mpool, size_t size,
-                                       size_t align, uint32_t flags)
+ * allocate function
+ */
+static void *mca_mpool_hugepage_alloc(mca_mpool_base_module_t *mpool, size_t size, size_t align,
+                                      uint32_t flags)
 {
     mca_mpool_hugepage_module_t *hugepage_module = (mca_mpool_hugepage_module_t *) mpool;
-    return hugepage_module->allocator->alc_alloc (hugepage_module->allocator, size, align);
+    return hugepage_module->allocator->alc_alloc(hugepage_module->allocator, size, align);
 }
 
 /**
-  * allocate function
-  */
-static void *mca_mpool_hugepage_realloc (mca_mpool_base_module_t *mpool, void *addr, size_t size)
+ * allocate function
+ */
+static void *mca_mpool_hugepage_realloc(mca_mpool_base_module_t *mpool, void *addr, size_t size)
 {
     mca_mpool_hugepage_module_t *hugepage_module = (mca_mpool_hugepage_module_t *) mpool;
 
-    return hugepage_module->allocator->alc_realloc (hugepage_module->allocator, addr, size);
+    return hugepage_module->allocator->alc_realloc(hugepage_module->allocator, addr, size);
 }
 
 /**
-  * free function
-  */
-static void mca_mpool_hugepage_free (mca_mpool_base_module_t *mpool, void *addr)
+ * free function
+ */
+static void mca_mpool_hugepage_free(mca_mpool_base_module_t *mpool, void *addr)
 {
     mca_mpool_hugepage_module_t *hugepage_module = (mca_mpool_hugepage_module_t *) mpool;
 
-    hugepage_module->allocator->alc_free (hugepage_module->allocator, addr);
+    hugepage_module->allocator->alc_free(hugepage_module->allocator, addr);
 }
 
-static void mca_mpool_hugepage_finalize (struct mca_mpool_base_module_t *mpool)
+static void mca_mpool_hugepage_finalize(struct mca_mpool_base_module_t *mpool)
 {
     mca_mpool_hugepage_module_t *hugepage_module = (mca_mpool_hugepage_module_t *) mpool;
 
-    OBJ_DESTRUCT(&hugepage_module->lock);
-    OBJ_DESTRUCT(&hugepage_module->allocation_tree);
-
     if (hugepage_module->allocator) {
-        (void) hugepage_module->allocator->alc_finalize (hugepage_module->allocator);
+        (void) hugepage_module->allocator->alc_finalize(hugepage_module->allocator);
         hugepage_module->allocator = NULL;
     }
+    OBJ_DESTRUCT(&hugepage_module->lock);
+    OBJ_DESTRUCT(&hugepage_module->allocation_tree);
+
 }
diff --git a/opal/mca/mpool/memkind/Makefile.am b/opal/mca/mpool/memkind/Makefile.am
index ac3e4552b12..cf4afb2ae60 100644
--- a/opal/mca/mpool/memkind/Makefile.am
+++ b/opal/mca/mpool/memkind/Makefile.am
@@ -31,7 +31,7 @@ mcacomponentdir = $(opallibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_mpool_memkind_la_SOURCES = $(memkind_SOURCES)
 nodist_mca_mpool_memkind_la_SOURCES = $(memkind_nodist_SOURCES)
-mca_mpool_memkind_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
+mca_mpool_memkind_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
 	$(mpool_memkind_LIBS)
 mca_mpool_memkind_la_LDFLAGS = -module -avoid-version $(mpool_memkind_LDFLAGS)
 
diff --git a/opal/mca/mpool/memkind/configure.m4 b/opal/mca/mpool/memkind/configure.m4
index 4bc49733496..7c891f63a7b 100644
--- a/opal/mca/mpool/memkind/configure.m4
+++ b/opal/mca/mpool/memkind/configure.m4
@@ -15,31 +15,28 @@ AC_DEFUN([MCA_opal_mpool_memkind_CONFIG],[
 
     AC_ARG_WITH([memkind], [AS_HELP_STRING([--with-memkind(=DIR)]),
 		[Build with MEMKIND, searching for headers in DIR])])
-    OPAL_CHECK_WITHDIR([memkind], [$with_memkind], [include/memkind.h])
 
-    opal_mpool_memkind_happy="no"
-
-    if test "$with_memkind" != "no" ; then
-	    if test -n "$with_memkind" -a "$with_memkind" != "yes" ; then
-	        opal_check_memkind_dir=$with_memkind
-	    fi
-
-            #
-            # memkind_create_kind was introduced with memkind v1.4.0 release
-            #
-            OPAL_CHECK_PACKAGE([mpool_memkind], [memkind.h], [memkind], [memkind_create_kind], [ -lnuma],
-	        [$opal_check_memkind_dir], [], [opal_mpool_memkind_happy="yes"], [])
-
-	    if test "$opal_mpool_memkind_happy" != "yes" -a -n "$with_memkind" ; then
-	        AC_MSG_ERROR([MEMKIND support requested but not found.  Aborting])
-	    fi
-    fi
-
-    AS_IF([test "$opal_mpool_memkind_happy" = "yes"], [$1], [$2])
+    #
+    # memkind_create_kind was introduced with memkind v1.4.0 release
+    #
+    OAC_CHECK_PACKAGE([memkind],
+                      [mpool_memkind],
+                      [memkind.h],
+                      [memkind -lnuma],
+                      [memkind_create_kind],
+                      [opal_mpool_memkind_happy="yes"],
+                      [opal_mpool_memkind_happy="no"])
+
+    AS_IF([test "$ompi_mpool_memkind_happy" = "yes"],
+          [$1],
+          [AS_IF([test -n "$with_memkind" && test "$with_memkind" != "no"],
+                 [AC_MSG_ERROR([Lustre support requested but not found.  Aborting])])
+           $2])
 
     # substitute in the things needed to build memkind
     AC_SUBST([mpool_memkind_CPPFLAGS])
     AC_SUBST([mpool_memkind_LDFLAGS])
     AC_SUBST([mpool_memkind_LIBS])
+
     OPAL_VAR_SCOPE_POP
 ])dnl
diff --git a/opal/mca/mpool/memkind/mpool_memkind.h b/opal/mca/mpool/memkind/mpool_memkind.h
index 75ca0384511..667f4abce83 100644
--- a/opal/mca/mpool/memkind/mpool_memkind.h
+++ b/opal/mca/mpool/memkind/mpool_memkind.h
@@ -30,8 +30,8 @@
 
 #include "opal_config.h"
 
-#include "opal/util/event.h"
 #include "opal/mca/mpool/mpool.h"
+#include "opal/util/event.h"
 
 #include "opal/mca/allocator/allocator.h"
 #include <memkind.h>
@@ -40,13 +40,12 @@ BEGIN_C_DECLS
 
 static const int mca_mpool_memkind_default_pagesize = 4096;
 
-
 struct mca_mpool_memkind_module_t {
     mca_mpool_base_module_t super;
     memkind_t kind;
     memkind_memtype_t type;
-    memkind_policy_t  policy;
-    memkind_bits_t    memkind_bits;
+    memkind_policy_t policy;
+    memkind_bits_t memkind_bits;
     int page_size;
 };
 
@@ -62,20 +61,20 @@ OBJ_CLASS_DECLARATION(mca_mpool_memkind_module_le_t);
 
 struct mca_mpool_memkind_component_t {
     mca_mpool_base_component_t super;
-    int  hbw;
-    int  pagesize;
-    int  bind;
+    int hbw;
+    int pagesize;
+    int bind;
     memkind_memtype_t default_type;
-    memkind_policy_t  default_policy;
-    memkind_bits_t    default_memkind_bits;
-    memkind_t         default_kind;
-    int  priority;
-    int  output;
+    memkind_policy_t default_policy;
+    memkind_bits_t default_memkind_bits;
+    memkind_t default_kind;
+    int priority;
+    int output;
     opal_list_t module_list;
 };
 
 typedef struct mca_mpool_memkind_component_t mca_mpool_memkind_component_t;
-OPAL_MODULE_DECLSPEC extern mca_mpool_memkind_component_t mca_mpool_memkind_component;
+OPAL_DECLSPEC extern mca_mpool_memkind_component_t mca_mpool_memkind_component;
 
 /**
  *  Initializes the mpool module.
@@ -83,30 +82,21 @@ OPAL_MODULE_DECLSPEC extern mca_mpool_memkind_component_t mca_mpool_memkind_comp
 
 void mca_mpool_memkind_module_init(mca_mpool_memkind_module_t *mpool);
 
-
 /**
-  *  Allocate block of high bandwidth memory.
-  */
-void* mca_mpool_memkind_alloc(
-    mca_mpool_base_module_t* mpool,
-    size_t size,
-    size_t align,
-    uint32_t flags);
+ *  Allocate block of high bandwidth memory.
+ */
+void *mca_mpool_memkind_alloc(mca_mpool_base_module_t *mpool, size_t size, size_t align,
+                              uint32_t flags);
 
 /**
-  * realloc function typedef
-  */
-void* mca_mpool_memkind_realloc(
-    mca_mpool_base_module_t* mpool,
-    void* addr,
-    size_t size);
+ * realloc function typedef
+ */
+void *mca_mpool_memkind_realloc(mca_mpool_base_module_t *mpool, void *addr, size_t size);
 
 /**
-  * free function typedef
-  */
-void mca_mpool_memkind_free(
-    mca_mpool_base_module_t* mpool,
-    void * addr);
+ * free function typedef
+ */
+void mca_mpool_memkind_free(mca_mpool_base_module_t *mpool, void *addr);
 
 END_C_DECLS
 
diff --git a/opal/mca/mpool/memkind/mpool_memkind_component.c b/opal/mca/mpool/memkind/mpool_memkind_component.c
index 02d53eb39d9..55be1a657e5 100644
--- a/opal/mca/mpool/memkind/mpool_memkind_component.c
+++ b/opal/mca/mpool/memkind/mpool_memkind_component.c
@@ -26,57 +26,47 @@
 
 #include "opal_config.h"
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif  /* HAVE_UNISTD_H*/
+#    include <unistd.h>
+#endif /* HAVE_UNISTD_H*/
 #ifdef HAVE_STDLIB_H
-#include <stdlib.h>
-#endif  /* HAVE_STDLIB_H */
-#include <errno.h>
-#include <memkind.h>
-#include "opal/mca/base/base.h"
+#    include <stdlib.h>
+#endif /* HAVE_STDLIB_H */
+#include "mpool_memkind.h"
 #include "opal/mca/allocator/base/base.h"
+#include "opal/mca/base/base.h"
 #include "opal/mca/mpool/base/base.h"
 #include "opal/util/argv.h"
-#include "mpool_memkind.h"
-
+#include <errno.h>
+#include <memkind.h>
 
 /*
  * Local functions
  */
 
-static int
-mca_mpool_memkind_register(void);
-
-static int
-mca_mpool_memkind_open(void);
-
-static int
-mca_mpool_memkind_close(void);
-
-static int mca_mpool_memkind_query (const char *hints, int *priority,
-                                    mca_mpool_base_module_t **module);
-
-mca_mpool_memkind_component_t mca_mpool_memkind_component = {
-    {
-        /* First, the mca_base_component_t struct containing meta
-         information about the component itself */
-        .mpool_version = {
-            MCA_MPOOL_BASE_VERSION_3_1_0,
-            "memkind", /* MCA component name */
-             MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
-                                   OPAL_RELEASE_VERSION),
-            .mca_open_component = mca_mpool_memkind_open,
-            .mca_close_component = mca_mpool_memkind_close,
-            .mca_register_component_params = mca_mpool_memkind_register
-        },
-        .mpool_data = {
-            /* The component is checkpoint ready */
-            MCA_BASE_METADATA_PARAM_CHECKPOINT
-        },
-
-        .mpool_query = mca_mpool_memkind_query,
-    }
-};
+static int mca_mpool_memkind_register(void);
+
+static int mca_mpool_memkind_open(void);
+
+static int mca_mpool_memkind_close(void);
+
+static int mca_mpool_memkind_query(const char *hints, int *priority,
+                                   mca_mpool_base_module_t **module);
+
+mca_mpool_memkind_component_t mca_mpool_memkind_component = {{
+    /* First, the mca_base_component_t struct containing meta
+     information about the component itself */
+    .mpool_version = {MCA_MPOOL_BASE_VERSION_3_1_0, "memkind", /* MCA component name */
+                      MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
+                                            OPAL_RELEASE_VERSION),
+                      .mca_open_component = mca_mpool_memkind_open,
+                      .mca_close_component = mca_mpool_memkind_close,
+                      .mca_register_component_params = mca_mpool_memkind_register},
+    .mpool_data =
+        {/* The component is checkpoint ready */
+         MCA_BASE_METADATA_PARAM_CHECKPOINT},
+
+    .mpool_query = mca_mpool_memkind_query,
+}};
 
 static void mca_mpool_memkind_module_le_destroy(mca_mpool_memkind_module_le_t *elem)
 {
@@ -85,30 +75,28 @@ static void mca_mpool_memkind_module_le_destroy(mca_mpool_memkind_module_le_t *e
     }
 }
 
-OBJ_CLASS_INSTANCE(mca_mpool_memkind_module_le_t,
-                   opal_list_item_t,
-                   NULL,
+OBJ_CLASS_INSTANCE(mca_mpool_memkind_module_le_t, opal_list_item_t, NULL,
                    mca_mpool_memkind_module_le_destroy);
 
 static mca_base_var_enum_value_t memory_types[] = {
-  {.value = MEMKIND_MEMTYPE_DEFAULT, .string = "memkind_default"},
-  {.value = MEMKIND_MEMTYPE_HIGH_BANDWIDTH, .string = "memkind_hbw"},
-  {.string = NULL},
+    {.value = MEMKIND_MEMTYPE_DEFAULT, .string = "memkind_default"},
+    {.value = MEMKIND_MEMTYPE_HIGH_BANDWIDTH, .string = "memkind_hbw"},
+    {.string = NULL},
 };
 
 static mca_base_var_enum_value_t memory_policy[] = {
-  {.value = MEMKIND_POLICY_BIND_LOCAL, .string = "mempolicy_bind_local"},
-  {.value = MEMKIND_POLICY_BIND_ALL, .string = "mempolicy_bind_all"},
-  {.value = MEMKIND_POLICY_PREFERRED_LOCAL, .string = "mempolicy_perferred_local"},
-  {.value = MEMKIND_POLICY_INTERLEAVE_LOCAL, .string = "mempolicy_interleave_local"},
-  {.value = MEMKIND_POLICY_INTERLEAVE_ALL, .string = "mempolicy_interleave_all"},
-  {.string = NULL},
+    {.value = MEMKIND_POLICY_BIND_LOCAL, .string = "mempolicy_bind_local"},
+    {.value = MEMKIND_POLICY_BIND_ALL, .string = "mempolicy_bind_all"},
+    {.value = MEMKIND_POLICY_PREFERRED_LOCAL, .string = "mempolicy_perferred_local"},
+    {.value = MEMKIND_POLICY_INTERLEAVE_LOCAL, .string = "mempolicy_interleave_local"},
+    {.value = MEMKIND_POLICY_INTERLEAVE_ALL, .string = "mempolicy_interleave_all"},
+    {.string = NULL},
 };
 
 static mca_base_var_enum_value_t memory_kind_bits[] = {
-  {.value = 0, .string = "memkind_mask_page_size_4KB"},
-  {.value = MEMKIND_MASK_PAGE_SIZE_2MB, .string = "memkind_mask_page_size_2MB"},
-  {.string = NULL},
+    {.value = 0, .string = "memkind_mask_page_size_4KB"},
+    {.value = MEMKIND_MASK_PAGE_SIZE_2MB, .string = "memkind_mask_page_size_2MB"},
+    {.string = NULL},
 };
 
 static mca_base_var_enum_t *mca_mpool_memkind_policy_enum = NULL;
@@ -124,8 +112,8 @@ static int mca_mpool_memkind_register(void)
 
     mca_mpool_memkind_component.default_type = memory_types[0].value;
 
-    rc = mca_base_var_enum_create ("memkind memory types", memory_types,
-                                   &mca_mpool_memkind_type_enum);
+    rc = mca_base_var_enum_create("memkind memory types", memory_types,
+                                  &mca_mpool_memkind_type_enum);
     if (OPAL_SUCCESS != rc) {
         return rc;
     }
@@ -137,41 +125,41 @@ static int mca_mpool_memkind_register(void)
                                            &mca_mpool_memkind_component.default_type);
 
     /*
-     * see memkind source to understand the 2 
+     * see memkind source to understand the 2
      */
     mca_mpool_memkind_component.default_policy = memory_policy[2].value;
 
-    rc = mca_base_var_enum_create ("memkind memory policy", memory_policy, 
-                                   &mca_mpool_memkind_policy_enum);
+    rc = mca_base_var_enum_create("memkind memory policy", memory_policy,
+                                  &mca_mpool_memkind_policy_enum);
     if (OPAL_SUCCESS != rc) {
         return rc;
     }
 
     (void) mca_base_component_var_register(&mca_mpool_memkind_component.super.mpool_version,
                                            "default_policy", "Default memkind policy to use",
-                                           MCA_BASE_VAR_TYPE_INT, mca_mpool_memkind_policy_enum, 0, 0,
-                                           OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL,
+                                           MCA_BASE_VAR_TYPE_INT, mca_mpool_memkind_policy_enum, 0,
+                                           0, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL,
                                            &mca_mpool_memkind_component.default_policy);
 
     mca_mpool_memkind_component.default_memkind_bits = memory_kind_bits[0].value;
 
-    rc = mca_base_var_enum_create ("memkind memory bits", memory_kind_bits, 
-                                   &mca_mpool_memkind_kind_bits_enum);
+    rc = mca_base_var_enum_create("memkind memory bits", memory_kind_bits,
+                                  &mca_mpool_memkind_kind_bits_enum);
     if (OPAL_SUCCESS != rc) {
         return rc;
     }
 
     (void) mca_base_component_var_register(&mca_mpool_memkind_component.super.mpool_version,
                                            "default_bits", "Default memkind bits to use",
-                                           MCA_BASE_VAR_TYPE_INT, mca_mpool_memkind_kind_bits_enum, 0, 0,
-                                           OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL,
+                                           MCA_BASE_VAR_TYPE_INT, mca_mpool_memkind_kind_bits_enum,
+                                           0, 0, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL,
                                            &mca_mpool_memkind_component.default_memkind_bits);
 
     mca_mpool_memkind_component.priority = 10;
     (void) mca_base_component_var_register(&mca_mpool_memkind_component.super.mpool_version,
                                            "priority", "Default priority of the memkind component",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL,
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                           MCA_BASE_VAR_SCOPE_LOCAL,
                                            &mca_mpool_memkind_component.priority);
 
     opal_mpool_memkind_verbose = 0;
@@ -185,9 +173,9 @@ static int mca_mpool_memkind_register(void)
 }
 
 /**
-  * component open/close/init function
-  */
-static int mca_mpool_memkind_open (void)
+ * component open/close/init function
+ */
+static int mca_mpool_memkind_open(void)
 {
     int rc;
     mca_mpool_memkind_module_le_t *item = NULL;
@@ -205,15 +193,15 @@ static int mca_mpool_memkind_open (void)
                              mca_mpool_memkind_component.default_memkind_bits,
                              &mca_mpool_memkind_component.default_kind);
     if (MEMKIND_SUCCESS != rc) {
-        opal_output_verbose (MCA_BASE_VERBOSE_WARN, opal_mpool_base_framework.framework_output,
-                                     "memkind_create_kind default returned %d", rc);
+        opal_output_verbose(MCA_BASE_VERBOSE_WARN, opal_mpool_base_framework.framework_output,
+                            "memkind_create_kind default returned %d", rc);
         return OPAL_ERR_NOT_AVAILABLE;
     }
 
     item = OBJ_NEW(mca_mpool_memkind_module_le_t);
 
-    item->module.type =  mca_mpool_memkind_component.default_type;
-    item->module.policy =  mca_mpool_memkind_component.default_policy;
+    item->module.type = mca_mpool_memkind_component.default_type;
+    item->module.policy = mca_mpool_memkind_component.default_policy;
     item->module.memkind_bits = mca_mpool_memkind_component.default_memkind_bits;
     item->module.kind = mca_mpool_memkind_component.default_kind;
     /*
@@ -225,16 +213,14 @@ static int mca_mpool_memkind_open (void)
         item->module.page_size = 4096;
     }
 
-    opal_list_append(&mca_mpool_memkind_component.module_list,
-                    (opal_list_item_t *)item);
- 
+    opal_list_append(&mca_mpool_memkind_component.module_list, (opal_list_item_t *) item);
 
     return OPAL_SUCCESS;
 }
 
 static int mca_mpool_memkind_close(void)
 {
-    opal_output_close (mca_mpool_memkind_component.output);
+    opal_output_close(mca_mpool_memkind_component.output);
     mca_mpool_memkind_component.output = -1;
 
     OPAL_LIST_DESTRUCT(&mca_mpool_memkind_component.module_list);
@@ -257,16 +243,16 @@ static int mca_mpool_memkind_close(void)
     return OPAL_SUCCESS;
 }
 
-static int mca_mpool_memkind_query (const char *hints, int *priority_out,
-                                    mca_mpool_base_module_t **module)
+static int mca_mpool_memkind_query(const char *hints, int *priority_out,
+                                   mca_mpool_base_module_t **module)
 {
     int my_priority = mca_mpool_memkind_component.priority;
     char **hint_array;
     char *tmp, *key, *value = NULL;
     int rc;
     memkind_memtype_t type = mca_mpool_memkind_component.default_type;
-    memkind_policy_t  policy = mca_mpool_memkind_component.default_policy;
-    memkind_bits_t    memkind_bits =  mca_mpool_memkind_component.default_memkind_bits;
+    memkind_policy_t policy = mca_mpool_memkind_component.default_policy;
+    memkind_bits_t memkind_bits = mca_mpool_memkind_component.default_memkind_bits;
     mca_mpool_memkind_module_le_t *item = NULL;
     mca_mpool_base_module_t *found_module = NULL;
     memkind_t kind;
@@ -278,7 +264,7 @@ static int mca_mpool_memkind_query (const char *hints, int *priority_out,
         return OPAL_SUCCESS;
     }
 
-    hint_array = opal_argv_split (hints, ',');
+    hint_array = opal_argv_split(hints, ',');
     if (NULL == hint_array) {
         if (priority_out) {
             *priority_out = my_priority;
@@ -286,10 +272,10 @@ static int mca_mpool_memkind_query (const char *hints, int *priority_out,
         return OPAL_SUCCESS;
     }
 
-    for (int i = 0 ; hint_array[i] ; ++i) {
+    for (int i = 0; hint_array[i]; ++i) {
 
         key = hint_array[i];
-        tmp = strchr (key, '=');
+        tmp = strchr(key, '=');
         if (tmp) {
             *tmp = '\0';
             value = tmp + 1;
@@ -298,10 +284,11 @@ static int mca_mpool_memkind_query (const char *hints, int *priority_out,
         /*
          * TODO: may want to emit a warning
          */
-        if (value == NULL) break;
+        if (value == NULL)
+            break;
 
-        if (0 == strcasecmp (key, "mpool")) {
-            if (0 == strcasecmp (value, "memkind")) {
+        if (0 == strcasecmp(key, "mpool")) {
+            if (0 == strcasecmp(value, "memkind")) {
                 /* specifically selected */
 
                 my_priority = 100;
@@ -311,31 +298,35 @@ static int mca_mpool_memkind_query (const char *hints, int *priority_out,
                 }
                 return OPAL_SUCCESS;
             }
-        } else if (0 == strcasecmp (key, "policy")) {
+        } else if (0 == strcasecmp(key, "policy")) {
 
-            rc = mca_mpool_memkind_policy_enum->value_from_string (mca_mpool_memkind_policy_enum,
-                                                                   value, (int *)&policy);
+            rc = mca_mpool_memkind_policy_enum->value_from_string(mca_mpool_memkind_policy_enum,
+                                                                  value, (int *) &policy);
             if (OPAL_SUCCESS != rc) {
-                opal_output_verbose (MCA_BASE_VERBOSE_WARN, opal_mpool_base_framework.framework_output,
-                                     "invalid memkind policy %s specified", value);
+                opal_output_verbose(MCA_BASE_VERBOSE_WARN,
+                                    opal_mpool_base_framework.framework_output,
+                                    "invalid memkind policy %s specified", value);
             }
 
-        } else if (0 == strcasecmp (key, "type")) {
+        } else if (0 == strcasecmp(key, "type")) {
 
-            rc = mca_mpool_memkind_type_enum->value_from_string (mca_mpool_memkind_type_enum,
-                                                                 value, (int *)&type);
+            rc = mca_mpool_memkind_type_enum->value_from_string(mca_mpool_memkind_type_enum, value,
+                                                                (int *) &type);
             if (OPAL_SUCCESS != rc) {
-                opal_output_verbose (MCA_BASE_VERBOSE_WARN, opal_mpool_base_framework.framework_output,
-                                     "invalid memkind type %s specified", value);
+                opal_output_verbose(MCA_BASE_VERBOSE_WARN,
+                                    opal_mpool_base_framework.framework_output,
+                                    "invalid memkind type %s specified", value);
             }
 
-        } else if (0 == strcasecmp (key, "kind_bits")) {
+        } else if (0 == strcasecmp(key, "kind_bits")) {
 
-            rc = mca_mpool_memkind_kind_bits_enum->value_from_string (mca_mpool_memkind_kind_bits_enum,
-                                                                      value, (int *)&memkind_bits);
+            rc = mca_mpool_memkind_kind_bits_enum
+                     ->value_from_string(mca_mpool_memkind_kind_bits_enum, value,
+                                         (int *) &memkind_bits);
             if (OPAL_SUCCESS != rc) {
-                opal_output_verbose (MCA_BASE_VERBOSE_WARN, opal_mpool_base_framework.framework_output,
-                                     "invalid memkind kind_bits %s specified", value);
+                opal_output_verbose(MCA_BASE_VERBOSE_WARN,
+                                    opal_mpool_base_framework.framework_output,
+                                    "invalid memkind kind_bits %s specified", value);
             }
         }
     }
@@ -344,18 +335,17 @@ static int mca_mpool_memkind_query (const char *hints, int *priority_out,
      * now look for an existing module with matching policy, type, memkind bits
      */
 
-    OPAL_LIST_FOREACH(item, &mca_mpool_memkind_component.module_list, 
-                      mca_mpool_memkind_module_le_t) {
-        if ((item->module.type == type) &&
-            (item->module.policy == policy) &&
-            (item->module.memkind_bits = memkind_bits)) {
-               found_module = &item->module.super;
-               break;
+    OPAL_LIST_FOREACH (item, &mca_mpool_memkind_component.module_list,
+                       mca_mpool_memkind_module_le_t) {
+        if ((item->module.type == type) && (item->module.policy == policy)
+            && (item->module.memkind_bits = memkind_bits)) {
+            found_module = &item->module.super;
+            break;
         }
-    } 
+    }
 
     /*
-     * didn't find a matching module, try to create one 
+     * didn't find a matching module, try to create one
      */
 
     if (NULL == found_module) {
@@ -364,8 +354,8 @@ static int mca_mpool_memkind_query (const char *hints, int *priority_out,
 
             item = OBJ_NEW(mca_mpool_memkind_module_le_t);
 
-            item->module.type =  type;
-            item->module.policy =  policy;
+            item->module.type = type;
+            item->module.policy = policy;
             item->module.memkind_bits = memkind_bits;
             item->module.kind = kind;
 
@@ -377,13 +367,12 @@ static int mca_mpool_memkind_query (const char *hints, int *priority_out,
 
             mca_mpool_memkind_module_init(&item->module);
 
-            opal_list_append(&mca_mpool_memkind_component.module_list,
-                            (opal_list_item_t *)item);
+            opal_list_append(&mca_mpool_memkind_component.module_list, (opal_list_item_t *) item);
             found_module = &item->module.super;
 
         } else {
-            opal_output_verbose (MCA_BASE_VERBOSE_WARN, opal_mpool_base_framework.framework_output,
-                                     "memkind_create_kind returned %d", rc);
+            opal_output_verbose(MCA_BASE_VERBOSE_WARN, opal_mpool_base_framework.framework_output,
+                                "memkind_create_kind returned %d", rc);
             if (priority_out) {
                 *priority_out = 0;
             }
@@ -395,7 +384,7 @@ static int mca_mpool_memkind_query (const char *hints, int *priority_out,
         *module = found_module;
     }
 
-    opal_argv_free (hint_array);
+    opal_argv_free(hint_array);
 
     if (priority_out) {
         *priority_out = my_priority;
diff --git a/opal/mca/mpool/memkind/mpool_memkind_module.c b/opal/mca/mpool/memkind/mpool_memkind_module.c
index d53f80a27d1..ecedaa77a4b 100644
--- a/opal/mca/mpool/memkind/mpool_memkind_module.c
+++ b/opal/mca/mpool/memkind/mpool_memkind_module.c
@@ -22,10 +22,10 @@
  */
 
 #include "opal_config.h"
-#include <string.h>
 #include "mpool_memkind.h"
+#include <string.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 #include "opal/mca/mpool/base/base.h"
 
@@ -42,11 +42,8 @@ void mca_mpool_memkind_module_init(mca_mpool_memkind_module_t *mpool)
     mpool->super.flags = MCA_MPOOL_FLAGS_MPI_ALLOC_MEM;
 }
 
-void* mca_mpool_memkind_alloc(
-    mca_mpool_base_module_t* mpool,
-    size_t size,
-    size_t align,
-    uint32_t flags)
+void *mca_mpool_memkind_alloc(mca_mpool_base_module_t *mpool, size_t size, size_t align,
+                              uint32_t flags)
 {
     mca_mpool_memkind_module_t *memkind_module = (mca_mpool_memkind_module_t *) mpool;
     void *addr;
@@ -55,18 +52,17 @@ void* mca_mpool_memkind_alloc(
         align = memkind_module->page_size;
     }
 
-    if ((errno = memkind_posix_memalign(memkind_module->kind, &addr, align, size))!= 0){
+    if ((errno = memkind_posix_memalign(memkind_module->kind, &addr, align, size)) != 0) {
         return NULL;
     }
 
     return addr;
 }
 
-void* mca_mpool_memkind_realloc(mca_mpool_base_module_t *mpool, void *addr,
-                                size_t size)
+void *mca_mpool_memkind_realloc(mca_mpool_base_module_t *mpool, void *addr, size_t size)
 {
     mca_mpool_memkind_module_t *memkind_module = (mca_mpool_memkind_module_t *) mpool;
-    return memkind_realloc (memkind_module->kind, addr, size);
+    return memkind_realloc(memkind_module->kind, addr, size);
 }
 
 void mca_mpool_memkind_free(mca_mpool_base_module_t *mpool, void *addr)
diff --git a/opal/mca/mpool/mpool.h b/opal/mca/mpool/mpool.h
index 3c9e6355398..59fce62608f 100644
--- a/opal/mca/mpool/mpool.h
+++ b/opal/mca/mpool/mpool.h
@@ -21,20 +21,20 @@
  * $HEADER$
  */
 /**
-  * @file
-  * Description of the Memory Pool framework
-  */
+ * @file
+ * Description of the Memory Pool framework
+ */
 #ifndef MCA_MPOOL_H
 #define MCA_MPOOL_H
 #include "opal_config.h"
-#include "opal/mca/mca.h"
 #include "opal/class/opal_free_list.h"
+#include "opal/mca/mca.h"
 #include "opal/mca/rcache/base/rcache_base_vma.h"
 
-#define MCA_MPOOL_ALLOC_FLAG_DEFAULT      0x00
-#define MCA_MPOOL_ALLOC_FLAG_USER         0x01
+#define MCA_MPOOL_ALLOC_FLAG_DEFAULT 0x00
+#define MCA_MPOOL_ALLOC_FLAG_USER    0x01
 
-#define MCA_MPOOL_FLAGS_MPI_ALLOC_MEM     0x80
+#define MCA_MPOOL_FLAGS_MPI_ALLOC_MEM 0x80
 
 struct opal_info_t;
 struct mca_mpool_base_module_t;
@@ -59,36 +59,34 @@ typedef struct mca_mpool_base_module_t mca_mpool_base_module_t;
  * Memory pools should try to support at a minimum name=value but can define
  * any additional keys.
  */
-typedef int (*mca_mpool_base_component_query_fn_t) (const char *hints, int *priority,
-                                                    mca_mpool_base_module_t **module);
+typedef int (*mca_mpool_base_component_query_fn_t)(const char *hints, int *priority,
+                                                   mca_mpool_base_module_t **module);
 
 /**
-  * allocate function typedef
-  */
-typedef void *(*mca_mpool_base_module_alloc_fn_t) (mca_mpool_base_module_t *mpool,
-                                                   size_t size, size_t align,
-                                                   uint32_t flags);
+ * allocate function typedef
+ */
+typedef void *(*mca_mpool_base_module_alloc_fn_t)(mca_mpool_base_module_t *mpool, size_t size,
+                                                  size_t align, uint32_t flags);
 
 /**
-  * allocate function typedef
-  */
-typedef void *(*mca_mpool_base_module_realloc_fn_t) (mca_mpool_base_module_t *mpool,
-                                                     void *addr, size_t size);
+ * allocate function typedef
+ */
+typedef void *(*mca_mpool_base_module_realloc_fn_t)(mca_mpool_base_module_t *mpool, void *addr,
+                                                    size_t size);
 
 /**
-  * free function typedef
-  */
-typedef void (*mca_mpool_base_module_free_fn_t) (mca_mpool_base_module_t *mpool,
-                                                 void *addr);
+ * free function typedef
+ */
+typedef void (*mca_mpool_base_module_free_fn_t)(mca_mpool_base_module_t *mpool, void *addr);
 
 /**
-  * if appropriate - returns base address of memory pool
-  */
-typedef void* (*mca_mpool_base_module_address_fn_t) (mca_mpool_base_module_t *mpool);
+ * if appropriate - returns base address of memory pool
+ */
+typedef void *(*mca_mpool_base_module_address_fn_t)(mca_mpool_base_module_t *mpool);
 
 /**
-  * finalize
-  */
+ * finalize
+ */
 typedef void (*mca_mpool_base_module_finalize_fn_t)(mca_mpool_base_module_t *mpool);
 
 /**
@@ -96,18 +94,18 @@ typedef void (*mca_mpool_base_module_finalize_fn_t)(mca_mpool_base_module_t *mpo
  * and open/close/init functions.
  */
 struct mca_mpool_base_component_3_1_0_t {
-    mca_base_component_t mpool_version;        /**< version */
-    mca_base_component_data_t mpool_data;/**< metadata */
+    mca_base_component_t mpool_version;   /**< version */
+    mca_base_component_data_t mpool_data; /**< metadata */
 
-    mca_mpool_base_component_query_fn_t mpool_query;  /**< query for matching pools */
+    mca_mpool_base_component_query_fn_t mpool_query; /**< query for matching pools */
 };
 /**
  * Convenience typedef.
  */
 typedef struct mca_mpool_base_component_3_1_0_t mca_mpool_base_component_3_1_0_t;
 /**
-  * Convenience typedef
-  */
+ * Convenience typedef
+ */
 typedef struct mca_mpool_base_component_3_1_0_t mca_mpool_base_component_t;
 
 /**
@@ -116,20 +114,19 @@ typedef struct mca_mpool_base_component_3_1_0_t mca_mpool_base_component_t;
  *  details.
  */
 struct mca_mpool_base_module_t {
-    mca_mpool_base_component_t *mpool_component;         /**< component stuct */
-    mca_mpool_base_module_address_fn_t mpool_base;       /**< returns the base address */
-    mca_mpool_base_module_alloc_fn_t mpool_alloc;        /**< allocate function */
-    mca_mpool_base_module_realloc_fn_t mpool_realloc;    /**< reallocate function */
-    mca_mpool_base_module_free_fn_t mpool_free;          /**< free function */
+    mca_mpool_base_component_t *mpool_component;      /**< component struct */
+    mca_mpool_base_module_address_fn_t mpool_base;    /**< returns the base address */
+    mca_mpool_base_module_alloc_fn_t mpool_alloc;     /**< allocate function */
+    mca_mpool_base_module_realloc_fn_t mpool_realloc; /**< reallocate function */
+    mca_mpool_base_module_free_fn_t mpool_free;       /**< free function */
 
-    mca_mpool_base_module_finalize_fn_t mpool_finalize;  /**< finalize */
-    uint32_t flags; /**< mpool flags */
+    mca_mpool_base_module_finalize_fn_t mpool_finalize; /**< finalize */
+    uint32_t flags;                                     /**< mpool flags */
 
-    size_t mpool_allocation_unit;                        /**< allocation unit used by this mpool */
-    char *mpool_name; /**< name of this pool module */
+    size_t mpool_allocation_unit; /**< allocation unit used by this mpool */
+    char *mpool_name;             /**< name of this pool module */
 };
 
-
 /**
  * Function to allocate special memory according to what the user requests in
  * the info object.
@@ -141,7 +138,7 @@ struct mca_mpool_base_module_t {
  * key in info that does not match any mpool, an error will be returned.
  *
  * If the info parameter is MPI_INFO_NULL, then this function will try to allocate
- * the memory and register it wih as many mpools as possible. However,
+ * the memory and register it with as many mpools as possible. However,
  * if any of the registratons fail the mpool will simply be ignored.
  *
  * @param size the size of the memory area to allocate
@@ -150,7 +147,7 @@ struct mca_mpool_base_module_t {
  * @retval pointer to the allocated memory
  * @retval NULL on failure
  */
-OPAL_DECLSPEC void * mca_mpool_base_alloc(size_t size, struct opal_info_t * info, const char *hints);
+OPAL_DECLSPEC void *mca_mpool_base_alloc(size_t size, struct opal_info_t *info, const char *hints);
 
 /**
  * Function to free memory previously allocated by mca_mpool_base_alloc
@@ -160,7 +157,7 @@ OPAL_DECLSPEC void * mca_mpool_base_alloc(size_t size, struct opal_info_t * info
  * @retval OPAL_SUCCESS
  * @retval OPAL_ERR_BAD_PARAM if the passed base pointer was invalid
  */
-OPAL_DECLSPEC int mca_mpool_base_free(void * base);
+OPAL_DECLSPEC int mca_mpool_base_free(void *base);
 
 /**
  * Function for the red black tree to compare 2 keys
@@ -172,13 +169,11 @@ OPAL_DECLSPEC int mca_mpool_base_free(void * base);
  * @retval 1 if key 1 is above key2
  * @retval 0 if the keys are the same
  */
-OPAL_DECLSPEC int mca_mpool_base_tree_node_compare(void * key1, void * key2);
+OPAL_DECLSPEC int mca_mpool_base_tree_node_compare(void *key1, void *key2);
 
 /**
  * Macro for use in components that are of type mpool
  */
-#define MCA_MPOOL_BASE_VERSION_3_1_0 \
-    OPAL_MCA_BASE_VERSION_2_1_0("mpool", 3, 1, 0)
+#define MCA_MPOOL_BASE_VERSION_3_1_0 OPAL_MCA_BASE_VERSION_2_1_0("mpool", 3, 1, 0)
 
 #endif /* MCA_MPOOL_H */
-
diff --git a/opal/mca/patcher/base/base.h b/opal/mca/patcher/base/base.h
index 0be3cf1dd7c..66f4ce8516a 100644
--- a/opal/mca/patcher/base/base.h
+++ b/opal/mca/patcher/base/base.h
@@ -12,6 +12,8 @@
  *                         All rights reserved.
  * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -24,33 +26,33 @@
 #define OPAL_PATCHER_BASE_H
 
 #include "opal_config.h"
+#include "opal/opal_portable_platform.h"
 #include "opal/mca/base/mca_base_framework.h"
 #include "opal/mca/patcher/patcher.h"
 
-
 BEGIN_C_DECLS
 
 #define MCA_BASE_PATCHER_MAX_PATCH 32
 
 struct mca_patcher_base_patch_t;
 
-typedef void (*mca_patcher_base_restore_fn_t) (struct mca_patcher_base_patch_t *);
+typedef void (*mca_patcher_base_restore_fn_t)(struct mca_patcher_base_patch_t *);
 
 struct mca_patcher_base_patch_t {
     /** patches are list items */
     opal_list_item_t super;
     /** name symbol to patch */
-    char            *patch_symbol;
+    char *patch_symbol;
     /** address of function to call instead */
-    uintptr_t        patch_value;
+    uintptr_t patch_value;
     /** original address of function */
-    uintptr_t        patch_orig;
+    uintptr_t patch_orig;
     /** patch data */
-    unsigned char    patch_data[MCA_BASE_PATCHER_MAX_PATCH];
+    unsigned char patch_data[MCA_BASE_PATCHER_MAX_PATCH];
     /** original data */
-    unsigned char    patch_orig_data[MCA_BASE_PATCHER_MAX_PATCH];
+    unsigned char patch_orig_data[MCA_BASE_PATCHER_MAX_PATCH];
     /** size of patch data */
-    unsigned         patch_data_size;
+    unsigned patch_data_size;
     /** function to undo the patch */
     mca_patcher_base_restore_fn_t patch_restore;
 };
@@ -63,17 +65,19 @@ OBJ_CLASS_DECLARATION(mca_patcher_base_patch_t);
  * Framework struct declaration for this framework
  */
 OPAL_DECLSPEC extern mca_base_framework_t opal_patcher_base_framework;
-OPAL_DECLSPEC int opal_patcher_base_select (void);
-OPAL_DECLSPEC int mca_patcher_base_patch_hook (mca_patcher_base_module_t *module, uintptr_t hook);
-OPAL_DECLSPEC void mca_base_patcher_patch_apply_binary (mca_patcher_base_patch_t *patch);
+OPAL_DECLSPEC int opal_patcher_base_select(void);
+OPAL_DECLSPEC int opal_patcher_base_restore_all(void);
+OPAL_DECLSPEC int mca_patcher_base_patch_hook(mca_patcher_base_module_t *module, uintptr_t hook);
+OPAL_DECLSPEC void mca_base_patcher_patch_apply_binary(mca_patcher_base_patch_t *patch);
 
-static inline uintptr_t mca_patcher_base_addr_text (uintptr_t addr) {
-#if (OPAL_ASSEMBLY_ARCH == OPAL_POWERPC64) && (!defined (_CALL_ELF) || (_CALL_ELF != 2))
+static inline uintptr_t mca_patcher_base_addr_text(uintptr_t addr)
+{
+#if defined(PLATFORM_ARCH_POWERPC) && defined(PLATFORM_ARCH_64)  && (!defined(_CALL_ELF) || (_CALL_ELF != 2))
     struct odp_t {
         uintptr_t text;
         uintptr_t toc;
     } *odp = (struct odp_t *) addr;
-    return (odp)?odp->text:0;
+    return (odp) ? odp->text : 0;
 #else
     return addr;
 #endif
diff --git a/opal/mca/patcher/base/patcher_base_frame.c b/opal/mca/patcher/base/patcher_base_frame.c
index 8d685d3fa43..6229f6aed80 100644
--- a/opal/mca/patcher/base/patcher_base_frame.c
+++ b/opal/mca/patcher/base/patcher_base_frame.c
@@ -11,9 +11,9 @@
 
 #include "opal_config.h"
 
-#include "opal/mca/patcher/patcher.h"
 #include "opal/mca/patcher/base/base.h"
 #include "opal/mca/patcher/base/static-components.h"
+#include "opal/mca/patcher/patcher.h"
 
 /*
  * Local variables
@@ -25,16 +25,16 @@ static mca_patcher_base_module_t empty_module;
  */
 mca_patcher_base_module_t *opal_patcher = &empty_module;
 
-int opal_patcher_base_select (void)
+int opal_patcher_base_select(void)
 {
     mca_patcher_base_module_t *best_module;
     mca_patcher_base_component_t *best_component;
     int rc, priority;
 
-    rc = mca_base_select ("patcher", opal_patcher_base_framework.framework_output,
-                          &opal_patcher_base_framework.framework_components,
-                          (mca_base_module_t **) &best_module, (mca_base_component_t **) &best_component,
-                          &priority);
+    rc = mca_base_select("patcher", opal_patcher_base_framework.framework_output,
+                         &opal_patcher_base_framework.framework_components,
+                         (mca_base_module_t **) &best_module,
+                         (mca_base_component_t **) &best_component, &priority);
     if (OPAL_SUCCESS != rc) {
         return rc;
     }
@@ -43,7 +43,7 @@ int opal_patcher_base_select (void)
     OBJ_CONSTRUCT(&best_module->patch_list_mutex, opal_mutex_t);
 
     if (best_module->patch_init) {
-        rc = best_module->patch_init ();
+        rc = best_module->patch_init();
         if (OPAL_SUCCESS != rc) {
             return rc;
         }
@@ -54,22 +54,40 @@ int opal_patcher_base_select (void)
     return OPAL_SUCCESS;
 }
 
-static int opal_patcher_base_close (void)
+int opal_patcher_base_restore_all(void)
 {
+    mca_patcher_base_patch_t *patch, *patch_next;
+
     if (opal_patcher == &empty_module) {
         return OPAL_SUCCESS;
     }
 
-    mca_patcher_base_patch_t *patch;
-    OPAL_LIST_FOREACH_REV(patch, &opal_patcher->patch_list, mca_patcher_base_patch_t) {
+    opal_mutex_lock(&opal_patcher->patch_list_mutex);
+
+    OPAL_LIST_FOREACH_SAFE_REV(patch, patch_next, &opal_patcher->patch_list, mca_patcher_base_patch_t) {
         patch->patch_restore (patch);
+        opal_list_remove_item(&opal_patcher->patch_list, &patch->super);
+        OBJ_RELEASE(patch);
+    }
+
+    opal_mutex_unlock(&opal_patcher->patch_list_mutex);
+
+    return OPAL_SUCCESS;
+}
+
+static int opal_patcher_base_close(void)
+{
+    if (opal_patcher == &empty_module) {
+        return OPAL_SUCCESS;
     }
 
+    opal_patcher_base_restore_all();
+
     OPAL_LIST_DESTRUCT(&opal_patcher->patch_list);
     OBJ_DESTRUCT(&opal_patcher->patch_list_mutex);
 
     if (opal_patcher->patch_fini) {
-        return opal_patcher->patch_fini ();
+        return opal_patcher->patch_fini();
     }
 
     return OPAL_SUCCESS;
@@ -77,5 +95,4 @@ static int opal_patcher_base_close (void)
 
 /* Use default register/open functions */
 MCA_BASE_FRAMEWORK_DECLARE(opal, patcher, "runtime code patching", NULL, NULL,
-                           opal_patcher_base_close, mca_patcher_base_static_components,
-                           0);
+                           opal_patcher_base_close, mca_patcher_base_static_components, 0);
diff --git a/opal/mca/patcher/base/patcher_base_patch.c b/opal/mca/patcher/base/patcher_base_patch.c
index c424e63997f..26f4c65c190 100644
--- a/opal/mca/patcher/base/patcher_base_patch.c
+++ b/opal/mca/patcher/base/patcher_base_patch.c
@@ -4,6 +4,9 @@
  *                         reserved.
  * Copyright (c) 2017      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -13,74 +16,79 @@
 
 #include "opal_config.h"
 
-#include "opal/mca/patcher/patcher.h"
 #include "opal/mca/patcher/base/base.h"
-#include "opal/util/sys_limits.h"
+#include "opal/mca/patcher/patcher.h"
+#include "opal/opal_portable_platform.h"
 #include "opal/prefetch.h"
+#include "opal/util/sys_limits.h"
 #include <sys/mman.h>
 
-static void mca_patcher_base_patch_construct (mca_patcher_base_patch_t *patch)
+static void mca_patcher_base_patch_construct(mca_patcher_base_patch_t *patch)
 {
     patch->patch_symbol = NULL;
     patch->patch_data_size = 0;
 }
 
-static void mca_patcher_base_patch_destruct (mca_patcher_base_patch_t *patch)
+static void mca_patcher_base_patch_destruct(mca_patcher_base_patch_t *patch)
 {
-    free (patch->patch_symbol);
+    free(patch->patch_symbol);
 }
 
-OBJ_CLASS_INSTANCE(mca_patcher_base_patch_t, opal_list_item_t,
-                   mca_patcher_base_patch_construct,
+OBJ_CLASS_INSTANCE(mca_patcher_base_patch_t, opal_list_item_t, mca_patcher_base_patch_construct,
                    mca_patcher_base_patch_destruct);
 
-#if defined(__PPC__)
+#if defined(PLATFORM_ARCH_POWERPC)
 
 // PowerPC instructions used in patching
 // Reference: "PowerPC User Instruction Set Architecture"
-static unsigned int addis(unsigned int RT, unsigned int RS, unsigned int UI) {
-    return (15<<26) + (RT<<21) + (RS<<16) + (UI&0xffff);
+static unsigned int addis(unsigned int RT, unsigned int RS, unsigned int UI)
+{
+    return (15 << 26) + (RT << 21) + (RS << 16) + (UI & 0xffff);
 }
-static unsigned int ori(unsigned int RT, unsigned int RS, unsigned int UI) {
-    return (24<<26) + (RS<<21) + (RT<<16) + (UI&0xffff);
+static unsigned int ori(unsigned int RT, unsigned int RS, unsigned int UI)
+{
+    return (24 << 26) + (RS << 21) + (RT << 16) + (UI & 0xffff);
 }
-static unsigned int oris(unsigned int RT, unsigned int RS, unsigned int UI) {
-    return (25<<26) + (RS<<21) + (RT<<16) + (UI&0xffff);
+static unsigned int oris(unsigned int RT, unsigned int RS, unsigned int UI)
+{
+    return (25 << 26) + (RS << 21) + (RT << 16) + (UI & 0xffff);
 }
-static unsigned int mtspr(unsigned int SPR, unsigned int RS) {
-    return (31<<26) + (RS<<21) + ((SPR&0x1f)<<16) + ((SPR>>5)<<11) + (467<<1);
+static unsigned int mtspr(unsigned int SPR, unsigned int RS)
+{
+    return (31 << 26) + (RS << 21) + ((SPR & 0x1f) << 16) + ((SPR >> 5) << 11) + (467 << 1);
 }
-static unsigned int bcctr(unsigned int BO, unsigned int BI, unsigned int BH) {
-    return (19<<26) + (BO<<21) + (BI<<16) + (BH<<11) + (528<<1);
+static unsigned int bcctr(unsigned int BO, unsigned int BI, unsigned int BH)
+{
+    return (19 << 26) + (BO << 21) + (BI << 16) + (BH << 11) + (528 << 1);
 }
 static unsigned int rldicr(unsigned int RT, unsigned int RS, unsigned int SH, unsigned int MB)
 {
-    return (30<<26) + (RS<<21) + (RT<<16) + ((SH&0x1f)<<11) + ((SH>>5)<<1)
-        + ((MB&0x1f)<<6) + ((MB>>5)<<5) + (1<<2);
+    return (30 << 26) + (RS << 21) + (RT << 16) + ((SH & 0x1f) << 11) + ((SH >> 5) << 1)
+           + ((MB & 0x1f) << 6) + ((MB >> 5) << 5) + (1 << 2);
 }
 
-static int PatchLoadImm (uintptr_t addr, unsigned int reg, size_t value)
+static int PatchLoadImm(uintptr_t addr, unsigned int reg, size_t value)
 {
-#if defined(__PPC64__)
-    *(unsigned int *) (addr + 0) = addis ( reg, 0,   (value >> 48));
-    *(unsigned int *) (addr + 4) = ori   ( reg, reg, (value >> 32));
-    *(unsigned int *) (addr + 8) = rldicr( reg, reg, 32, 31);
-    *(unsigned int *) (addr +12) = oris  ( reg, reg, (value >> 16));
-    *(unsigned int *) (addr +16) = ori   ( reg, reg, (value >>  0));
+#    if defined(PLATFORM_ARCH_64)
+    *(unsigned int *) (addr + 0) = addis(reg, 0, (value >> 48));
+    *(unsigned int *) (addr + 4) = ori(reg, reg, (value >> 32));
+    *(unsigned int *) (addr + 8) = rldicr(reg, reg, 32, 31);
+    *(unsigned int *) (addr + 12) = oris(reg, reg, (value >> 16));
+    *(unsigned int *) (addr + 16) = ori(reg, reg, (value >> 0));
     return 20;
-#else
-    *(unsigned int *) (addr + 0) = addis ( reg, 0,   (value >> 16));
-    *(unsigned int *) (addr + 4) = ori   ( reg, reg, (value >>  0));
+#    else
+    *(unsigned int *) (addr + 0) = addis(reg, 0, (value >> 16));
+    *(unsigned int *) (addr + 4) = ori(reg, reg, (value >> 0));
     return 8;
-#endif
+#    endif
 }
 
 #endif
 
-#if !HAVE___CLEAR_CACHE
-static void flush_and_invalidate_cache (unsigned long a)
+#if !defined(HAVE___CLEAR_CACHE)
+static void flush_and_invalidate_cache(unsigned long a)
 {
-#if OPAL_ASSEMBLY_ARCH == OPAL_IA32
+#    if defined(PLATFORM_ARCH_X86)
     static int have_clflush = -1;
 
     if (OPAL_UNLIKELY(-1 == have_clflush)) {
@@ -89,99 +97,98 @@ static void flush_and_invalidate_cache (unsigned long a)
 
         /* cpuid clobbers ebx but it must be restored for -fPIC so save
          * then restore ebx */
-        __asm__ volatile ("xchgl %%ebx, %2\n"
-                          "cpuid\n"
-                          "xchgl %%ebx, %2\n":
-                          "=a" (cpuid1), "=d" (cpuid2), "=r" (tmp) :
-                          "a" (level) :
-                          "ecx");
+        __asm__ volatile("xchgl %%ebx, %2\n"
+                         "cpuid\n"
+                         "xchgl %%ebx, %2\n"
+                         : "=a"(cpuid1), "=d"(cpuid2), "=r"(tmp)
+                         : "a"(level)
+                         : "ecx");
         /* clflush is in edx bit 19 */
         have_clflush = !!(cpuid2 & (1 << 19));
     }
 
     if (have_clflush) {
         /* does not work with AMD processors */
-        __asm__ volatile("mfence;clflush %0;mfence" : :"m" (*(char*)a));
+        __asm__ volatile("mfence;clflush %0;mfence" : : "m"(*(char *) a));
     }
-#elif OPAL_ASSEMBLY_ARCH == OPAL_X86_64
-    __asm__ volatile("mfence;clflush %0;mfence" : :"m" (*(char*)a));
-#elif OPAL_ASSEMBLY_ARCH == OPAL_ARM64
-    __asm__ volatile ("dc cvau, %0\n\t"
-                      "dsb ish\n\t"
-                      "ic ivau, %0\n\t"
-                      "dsb ish\n\t"
-                      "isb":: "r" (a));
-#endif
+#    elif defined(PLATFORM_ARCH_X86_64)
+    __asm__ volatile("mfence;clflush %0;mfence" : : "m"(*(char *) a));
+#    elif defined(PLATFORM_ARCH_AARCH64)
+    __asm__ volatile("dc cvau, %0\n\t"
+                     "dsb ish\n\t"
+                     "ic ivau, %0\n\t"
+                     "dsb ish\n\t"
+                     "isb" ::"r"(a));
+#    endif
 }
-#endif   // !HAVE___CLEAR_CACHE
+#endif // !HAVE___CLEAR_CACHE
 
 // modify protection of memory range
-static void ModifyMemoryProtection (uintptr_t addr, size_t length, int prot)
+static void ModifyMemoryProtection(uintptr_t addr, size_t length, int prot)
 {
-    long      page_size = opal_getpagesize ();
-    uintptr_t base = (addr & ~(page_size-1));
-    uintptr_t bound = ((addr + length + page_size-1) & ~(page_size-1));
+    long page_size = opal_getpagesize();
+    uintptr_t base = (addr & ~(page_size - 1));
+    uintptr_t bound = ((addr + length + page_size - 1) & ~(page_size - 1));
 
     length = bound - base;
 
-#if defined(__PPC__)
+#if defined(PLATFORM_ARCH_POWERPC)
     /* NTH: is a loop necessary here? */
     do {
-        if (mprotect((void *)base, page_size, prot))
+        if (mprotect((void *) base, page_size, prot))
             perror("MemHook: mprotect failed");
         base += page_size;
     } while (base < bound);
 #else
     if (mprotect((void *) base, length, prot)) {
-            perror("MemHook: mprotect failed");
+        perror("MemHook: mprotect failed");
     }
 #endif
 }
 
-static inline void apply_patch (unsigned char *patch_data, uintptr_t address, size_t data_size)
+static inline void apply_patch(unsigned char *patch_data, uintptr_t address, size_t data_size)
 {
-    ModifyMemoryProtection (address, data_size, PROT_EXEC|PROT_READ|PROT_WRITE);
-    memcpy ((void *) address, patch_data, data_size);
-#if HAVE___CLEAR_CACHE
+    ModifyMemoryProtection(address, data_size, PROT_EXEC | PROT_READ | PROT_WRITE);
+    memcpy((void *) address, patch_data, data_size);
+#if defined(HAVE___CLEAR_CACHE)
     /* do not allow global declaration of compiler intrinsic */
-    void __clear_cache(void* beg, void* end);
+    void __clear_cache(void *beg, void *end);
 
-    __clear_cache ((void *) address, (void *) (address + data_size));
+    __clear_cache((void *) address, (void *) (address + data_size));
 #else
     size_t offset_jump = 16;
 
-#if OPAL_ASSEMBLY_ARCH == OPAL_ARM64
+#    if defined(PLATFORM_ARCH_AARCH64)
     offset_jump = 32;
-#endif
+#    endif
 
     /* align the address */
     address &= ~(offset_jump - 1);
 
-    for (size_t i = 0 ; i < data_size ; i += offset_jump) {
-        flush_and_invalidate_cache (address + i);
+    for (size_t i = 0; i < data_size; i += offset_jump) {
+        flush_and_invalidate_cache(address + i);
     }
 
 #endif
 
-    ModifyMemoryProtection (address, data_size, PROT_EXEC|PROT_READ);
+    ModifyMemoryProtection(address, data_size, PROT_EXEC | PROT_READ);
 }
 
-static void mca_base_patcher_patch_unapply_binary (mca_patcher_base_patch_t *patch)
+static void mca_base_patcher_patch_unapply_binary(mca_patcher_base_patch_t *patch)
 {
-    apply_patch (patch->patch_orig_data, patch->patch_orig, patch->patch_data_size);
+    apply_patch(patch->patch_orig_data, patch->patch_orig, patch->patch_data_size);
 }
 
-void mca_base_patcher_patch_apply_binary (mca_patcher_base_patch_t *patch)
+void mca_base_patcher_patch_apply_binary(mca_patcher_base_patch_t *patch)
 {
-    memcpy (patch->patch_orig_data, (void *) patch->patch_orig, patch->patch_data_size);
-    apply_patch (patch->patch_data, patch->patch_orig, patch->patch_data_size);
+    memcpy(patch->patch_orig_data, (void *) patch->patch_orig, patch->patch_data_size);
+    apply_patch(patch->patch_data, patch->patch_orig, patch->patch_data_size);
     patch->patch_restore = mca_base_patcher_patch_unapply_binary;
 }
 
-
-int mca_patcher_base_patch_hook (mca_patcher_base_module_t *module, uintptr_t hook_addr)
+int mca_patcher_base_patch_hook(mca_patcher_base_module_t *module, uintptr_t hook_addr)
 {
-#if (OPAL_ASSEMBLY_ARCH == OPAL_POWERPC64)
+#if defined(PLATFORM_ARCH_POWERPC) && defined(PLATFORM_ARCH_64)
     mca_patcher_base_patch_t *hook_patch;
     const unsigned int nop = 0x60000000;
 
@@ -191,9 +198,9 @@ int mca_patcher_base_patch_hook (mca_patcher_base_module_t *module, uintptr_t ho
     }
 
     // locate reserved code space in hook function
-    for (unsigned int *nop_addr = (unsigned int *)hook_addr ; ; nop_addr++) {
-        if (nop_addr[0] == nop && nop_addr[1] == nop && nop_addr[2] == nop
-                && nop_addr[3] == nop && nop_addr[4] == nop) {
+    for (unsigned int *nop_addr = (unsigned int *) hook_addr;; nop_addr++) {
+        if (nop_addr[0] == nop && nop_addr[1] == nop && nop_addr[2] == nop && nop_addr[3] == nop
+            && nop_addr[4] == nop) {
             hook_patch->patch_orig = (uintptr_t) nop_addr;
             break;
         }
@@ -202,14 +209,14 @@ int mca_patcher_base_patch_hook (mca_patcher_base_module_t *module, uintptr_t ho
     // generate code to restore TOC
     unsigned long toc;
 
-    asm volatile ("std 2, %0" : "=m" (toc));
+    asm volatile("std 2, %0" : "=m"(toc));
 
-    hook_patch->patch_data_size = PatchLoadImm((uintptr_t)hook_patch->patch_data, 2, toc);
+    hook_patch->patch_data_size = PatchLoadImm((uintptr_t) hook_patch->patch_data, 2, toc);
 
     /* put the hook patch on the patch list so it will be undone on finalize */
-    opal_list_append (&module->patch_list, &hook_patch->super);
+    opal_list_append(&module->patch_list, &hook_patch->super);
 
-    mca_base_patcher_patch_apply_binary (hook_patch);
+    mca_base_patcher_patch_apply_binary(hook_patch);
 #endif
 
     return OPAL_SUCCESS;
diff --git a/opal/mca/patcher/overwrite/Makefile.am b/opal/mca/patcher/overwrite/Makefile.am
index e63e553ec35..18be14a2874 100644
--- a/opal/mca/patcher/overwrite/Makefile.am
+++ b/opal/mca/patcher/overwrite/Makefile.am
@@ -40,7 +40,7 @@ mcacomponent_LTLIBRARIES = $(component_install)
 mca_patcher_overwrite_la_SOURCES = $(overwrite_SOURCES)
 nodist_mca_patcher_overwrite_la_SOURCES = $(overwrite_nodist_SOURCES)
 mca_patcher_overwrite_la_LDFLAGS = -module -avoid-version
-mca_patcher_overwrite_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+mca_patcher_overwrite_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 
 noinst_LTLIBRARIES = $(component_noinst)
 libmca_patcher_overwrite_la_SOURCES = $(overwrite_SOURCES)
diff --git a/opal/mca/patcher/overwrite/patcher_overwrite.h b/opal/mca/patcher/overwrite/patcher_overwrite.h
index 9c2ad58dfd5..849e5ae6613 100644
--- a/opal/mca/patcher/overwrite/patcher_overwrite.h
+++ b/opal/mca/patcher/overwrite/patcher_overwrite.h
@@ -11,7 +11,7 @@
 /**
  * @file pather_overwrite.h
  *
- * This component works by overwritting the first couple instructions in
+ * This component works by overwriting the first couple instructions in
  * the target function with a jump instruction to the hook function. The
  * hook function will be expected to implement the functionality of the
  * hooked function when using this module.
@@ -20,11 +20,11 @@
  */
 
 #if !defined(OPAL_PATCHER_OVERWRITE_H)
-#define OPAL_PATCHER_OVERWRITE_H
+#    define OPAL_PATCHER_OVERWRITE_H
 
-#include "opal_config.h"
-#include "opal/mca/patcher/patcher.h"
-#include "opal/class/opal_list.h"
+#    include "opal_config.h"
+#    include "opal/class/opal_list.h"
+#    include "opal/mca/patcher/patcher.h"
 
 extern mca_patcher_base_module_t mca_patcher_overwrite_module;
 extern mca_patcher_base_component_t mca_patcher_overwrite_component;
diff --git a/opal/mca/patcher/overwrite/patcher_overwrite_component.c b/opal/mca/patcher/overwrite/patcher_overwrite_component.c
index 5211d4deaef..90abcf6f418 100644
--- a/opal/mca/patcher/overwrite/patcher_overwrite_component.c
+++ b/opal/mca/patcher/overwrite/patcher_overwrite_component.c
@@ -9,24 +9,24 @@
  * $HEADER$
  */
 
-#include "patcher_overwrite.h"
-#include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
+#include "opal/mca/mca.h"
+#include "patcher_overwrite.h"
 
 static int mca_patcher_overwrite_priority;
 
-static int mca_patcher_overwrite_register (void)
+static int mca_patcher_overwrite_register(void)
 {
     mca_patcher_overwrite_priority = 37;
-    mca_base_component_var_register (&mca_patcher_overwrite_component.patcherc_version,
-                                     "priority", "Priority of the overwrite binary patcher component",
-                                     MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
-                                     MCA_BASE_VAR_SCOPE_CONSTANT, &mca_patcher_overwrite_priority);
+    mca_base_component_var_register(&mca_patcher_overwrite_component.patcherc_version, "priority",
+                                    "Priority of the overwrite binary patcher component",
+                                    MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                    MCA_BASE_VAR_SCOPE_CONSTANT, &mca_patcher_overwrite_priority);
 
     return OPAL_SUCCESS;
 }
 
-static int mca_patcher_overwrite_query (mca_base_module_t **module, int *priority)
+static int mca_patcher_overwrite_query(mca_base_module_t **module, int *priority)
 {
     *module = &mca_patcher_overwrite_module.super;
     *priority = mca_patcher_overwrite_priority;
@@ -34,12 +34,13 @@ static int mca_patcher_overwrite_query (mca_base_module_t **module, int *priorit
 }
 
 mca_patcher_base_component_t mca_patcher_overwrite_component = {
-    .patcherc_version = {
-        OPAL_PATCHER_BASE_VERSION_1_0_0,
-        .mca_component_name = "overwrite",
-        MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
-                              OPAL_RELEASE_VERSION),
-        .mca_query_component = mca_patcher_overwrite_query,
-        .mca_register_component_params = mca_patcher_overwrite_register,
-    },
+    .patcherc_version =
+        {
+            OPAL_PATCHER_BASE_VERSION_1_0_0,
+            .mca_component_name = "overwrite",
+            MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
+                                  OPAL_RELEASE_VERSION),
+            .mca_query_component = mca_patcher_overwrite_query,
+            .mca_register_component_params = mca_patcher_overwrite_register,
+        },
 };
diff --git a/opal/mca/patcher/overwrite/patcher_overwrite_module.c b/opal/mca/patcher/overwrite/patcher_overwrite_module.c
index dae4cd1ae1d..c8f5d9975f2 100644
--- a/opal/mca/patcher/overwrite/patcher_overwrite_module.c
+++ b/opal/mca/patcher/overwrite/patcher_overwrite_module.c
@@ -5,6 +5,8 @@
  * Copyright (c) 2016      IBM Corporation.  All rights reserved.
  * Copyright (c) 2017      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -17,91 +19,96 @@
 #include "opal/mca/patcher/base/base.h"
 
 #include "opal/constants.h"
-#include "opal/util/sys_limits.h"
-#include "opal/util/output.h"
 #include "opal/prefetch.h"
+#include "opal/util/output.h"
+#include "opal/util/sys_limits.h"
+#include "opal/opal_portable_platform.h"
 
+#include <assert.h>
+#include <dlfcn.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <stdint.h>
-#include <unistd.h>
 #include <string.h>
 #include <sys/mman.h>
-#include <dlfcn.h>
-#include <assert.h>
+#include <unistd.h>
 
-#if (OPAL_ASSEMBLY_ARCH == OPAL_IA32) || (OPAL_ASSEMBLY_ARCH == OPAL_X86_64)
+#if defined(PLATFORM_ARCH_X86) || defined(PLATFORM_ARCH_X86_64)
 
-static int mca_patcher_overwrite_apply_patch (mca_patcher_base_patch_t *patch)
+static int mca_patcher_overwrite_apply_patch(mca_patcher_base_patch_t *patch)
 {
     uintptr_t func_new_addr = patch->patch_value;
 
     {
-#if (OPAL_ASSEMBLY_ARCH == OPAL_IA32)
+#    if defined(PLATFORM_ARCH_32)
         patch->patch_data_size = 5;
-        *(unsigned char *)(patch->patch_data+0) = 0xe9;
-        *(unsigned int *) (patch->patch_data+1) = (unsigned int)(func_new_addr - patch->patch_orig - 5);
-#elif (OPAL_ASSEMBLY_ARCH == OPAL_X86_64)
+        *(unsigned char *) (patch->patch_data + 0) = 0xe9;
+        *(unsigned int *) (patch->patch_data + 1) = (unsigned int) (func_new_addr
+                                                                    - patch->patch_orig - 5);
+#    elif defined(PLATFORM_ARCH_64)
         patch->patch_data_size = 13;
-        *(unsigned short*)(patch->patch_data + 0) = 0xbb49;
-        *(unsigned long* )(patch->patch_data + 2) = (unsigned long) func_new_addr;
-        *(unsigned char*) (patch->patch_data +10) = 0x41;
-        *(unsigned char*) (patch->patch_data +11) = 0xff;
-        *(unsigned char*) (patch->patch_data +12) = 0xe3;
-#endif
+        *(unsigned short *) (patch->patch_data + 0) = 0xbb49;
+        *(unsigned long *) (patch->patch_data + 2) = (unsigned long) func_new_addr;
+        *(unsigned char *) (patch->patch_data + 10) = 0x41;
+        *(unsigned char *) (patch->patch_data + 11) = 0xff;
+        *(unsigned char *) (patch->patch_data + 12) = 0xe3;
+#    endif
     }
 
-    mca_base_patcher_patch_apply_binary (patch);
+    mca_base_patcher_patch_apply_binary(patch);
 
     return OPAL_SUCCESS;
 }
 
-/* end of #if defined(__i386__) || defined(__x86_64__) || defined(__ia64__) */
+/* end of #if defined(__i386__) || defined(__x86_64__) */
 // ------------------------------------------------- PPC equivalent:
-#elif (OPAL_ASSEMBLY_ARCH == OPAL_POWERPC32) || (OPAL_ASSEMBLY_ARCH == OPAL_POWERPC64)
+#elif defined(PLATFORM_ARCH_POWERPC)
 
 // PowerPC instructions used in patching
 // Reference: "PowerPC User Instruction Set Architecture"
-static unsigned int addis(unsigned int RT, unsigned int RS, unsigned int UI) {
-    return (15<<26) + (RT<<21) + (RS<<16) + (UI&0xffff);
+static unsigned int addis(unsigned int RT, unsigned int RS, unsigned int UI)
+{
+    return (15 << 26) + (RT << 21) + (RS << 16) + (UI & 0xffff);
 }
-static unsigned int ori(unsigned int RT, unsigned int RS, unsigned int UI) {
-    return (24<<26) + (RS<<21) + (RT<<16) + (UI&0xffff);
+static unsigned int ori(unsigned int RT, unsigned int RS, unsigned int UI)
+{
+    return (24 << 26) + (RS << 21) + (RT << 16) + (UI & 0xffff);
 }
-static unsigned int oris(unsigned int RT, unsigned int RS, unsigned int UI) {
-    return (25<<26) + (RS<<21) + (RT<<16) + (UI&0xffff);
+static unsigned int oris(unsigned int RT, unsigned int RS, unsigned int UI)
+{
+    return (25 << 26) + (RS << 21) + (RT << 16) + (UI & 0xffff);
 }
-static unsigned int mtspr(unsigned int SPR, unsigned int RS) {
-    return (31<<26) + (RS<<21) + ((SPR&0x1f)<<16) + ((SPR>>5)<<11) + (467<<1);
+static unsigned int mtspr(unsigned int SPR, unsigned int RS)
+{
+    return (31 << 26) + (RS << 21) + ((SPR & 0x1f) << 16) + ((SPR >> 5) << 11) + (467 << 1);
 }
-static unsigned int bcctr(unsigned int BO, unsigned int BI, unsigned int BH) {
-    return (19<<26) + (BO<<21) + (BI<<16) + (BH<<11) + (528<<1);
+static unsigned int bcctr(unsigned int BO, unsigned int BI, unsigned int BH)
+{
+    return (19 << 26) + (BO << 21) + (BI << 16) + (BH << 11) + (528 << 1);
 }
 static unsigned int rldicr(unsigned int RT, unsigned int RS, unsigned int SH, unsigned int MB)
 {
-    return (30<<26) + (RS<<21) + (RT<<16) + ((SH&0x1f)<<11) + ((SH>>5)<<1)
-        + ((MB&0x1f)<<6) + ((MB>>5)<<5) + (1<<2);
+    return (30 << 26) + (RS << 21) + (RT << 16) + ((SH & 0x1f) << 11) + ((SH >> 5) << 1)
+           + ((MB & 0x1f) << 6) + ((MB >> 5) << 5) + (1 << 2);
 }
 
-static int
-PatchLoadImm(uintptr_t addr, unsigned int reg, size_t value)
+static int PatchLoadImm(uintptr_t addr, unsigned int reg, size_t value)
 {
-#if (OPAL_ASSEMBLY_ARCH == OPAL_POWERPC64)
-    *(unsigned int *) (addr + 0) = addis ( reg, 0,   (value >> 48));
-    *(unsigned int *) (addr + 4) = ori   ( reg, reg, (value >> 32));
-    *(unsigned int *) (addr + 8) = rldicr( reg, reg, 32, 31);
-    *(unsigned int *) (addr +12) = oris  ( reg, reg, (value >> 16));
-    *(unsigned int *) (addr +16) = ori   ( reg, reg, (value >>  0));
+#    if defined(PLATFORM_ARCH_64)
+    *(unsigned int *) (addr + 0) = addis(reg, 0, (value >> 48));
+    *(unsigned int *) (addr + 4) = ori(reg, reg, (value >> 32));
+    *(unsigned int *) (addr + 8) = rldicr(reg, reg, 32, 31);
+    *(unsigned int *) (addr + 12) = oris(reg, reg, (value >> 16));
+    *(unsigned int *) (addr + 16) = ori(reg, reg, (value >> 0));
     return 20;
-#else
-    *(unsigned int *) (addr + 0) = addis ( reg, 0,   (value >> 16));
-    *(unsigned int *) (addr + 4) = ori   ( reg, reg, (value >>  0));
+#    else
+    *(unsigned int *) (addr + 0) = addis(reg, 0, (value >> 16));
+    *(unsigned int *) (addr + 4) = ori(reg, reg, (value >> 0));
     return 8;
-#endif
+#    endif
 }
 
-
-static int mca_patcher_overwrite_apply_patch (mca_patcher_base_patch_t *patch)
+static int mca_patcher_overwrite_apply_patch(mca_patcher_base_patch_t *patch)
 {
     uintptr_t sys_addr, hook_addr;
     int offset, rc;
@@ -111,34 +118,34 @@ static int mca_patcher_overwrite_apply_patch (mca_patcher_base_patch_t *patch)
     hook_addr = mca_patcher_base_addr_text(patch->patch_value);
 
 // Patch for hook function:
-#if (OPAL_ASSEMBLY_ARCH == OPAL_POWERPC64)
-    rc = mca_patcher_base_patch_hook (&mca_patcher_overwrite_module, hook_addr);
+#    if defined(PLATFORM_ARCH_64)
+    rc = mca_patcher_base_patch_hook(&mca_patcher_overwrite_module, hook_addr);
     if (OPAL_SUCCESS != rc) {
         return rc;
     }
 
-#if defined(_CALL_ELF) && (_CALL_ELF == 2)
+#        if defined(_CALL_ELF) && (_CALL_ELF == 2)
     sys_addr += 8;
     hook_addr += 8;
-#endif /* _CALL_ELF == 2*/
-#endif
+#        endif /* _CALL_ELF == 2*/
+#    endif
 
     // Patch for system function:
     // generate patch code
     // r11 is a volatile register according to PowerPC EABI
     const unsigned int gr = 11;
-    offset = PatchLoadImm ((uintptr_t) patch->patch_data, gr, hook_addr);
-    *(unsigned int *) (patch->patch_data + offset + 0) = mtspr (9, gr);   // 9 = CTR
-    *(unsigned int *) (patch->patch_data + offset + 4) = bcctr (20, 0, 0);// 20 = always
+    offset = PatchLoadImm((uintptr_t) patch->patch_data, gr, hook_addr);
+    *(unsigned int *) (patch->patch_data + offset + 0) = mtspr(9, gr);    // 9 = CTR
+    *(unsigned int *) (patch->patch_data + offset + 4) = bcctr(20, 0, 0); // 20 = always
     patch->patch_data_size = offset + 8;
     patch->patch_orig = sys_addr;
 
-    mca_base_patcher_patch_apply_binary (patch);
+    mca_base_patcher_patch_apply_binary(patch);
 
     return OPAL_SUCCESS;
 }
 
-#elif defined(__aarch64__)
+#elif defined(PLATFORM_ARCH_AARCH64)
 
 /**
  * @brief Generate a mov immediate instruction
@@ -147,7 +154,7 @@ static int mca_patcher_overwrite_apply_patch (mca_patcher_base_patch_t *patch)
  * @param[in] shift shift amount (0-3) * 16-bits
  * @param[in] value immediate value
  */
-static uint32_t mov (unsigned int reg, uint16_t shift, uint16_t value)
+static uint32_t mov(unsigned int reg, uint16_t shift, uint16_t value)
 {
     return (0x1a5 << 23) + ((uint32_t) shift << 21) + ((uint32_t) value << 5) + reg;
 }
@@ -159,27 +166,26 @@ static uint32_t mov (unsigned int reg, uint16_t shift, uint16_t value)
  * @param[in] shift shift amount (0-3) * 16-bits
  * @param[in] value immediate value
  */
-static uint32_t movk (unsigned int reg, uint16_t shift, uint16_t value)
+static uint32_t movk(unsigned int reg, uint16_t shift, uint16_t value)
 {
     return (0x1e5 << 23) + ((uint32_t) shift << 21) + ((uint32_t) value << 5) + reg;
 }
 
-static uint32_t br (unsigned int reg)
+static uint32_t br(unsigned int reg)
 {
     return (0xd61f << 16) + (reg << 5);
 }
 
-static int
-PatchLoadImm(uintptr_t addr, unsigned int reg, uint64_t value)
+static int PatchLoadImm(uintptr_t addr, unsigned int reg, uint64_t value)
 {
-    *(uint32_t *) (addr +  0) = mov(reg, 3, value >> 48);
-    *(uint32_t *) (addr +  4) = movk(reg, 2, value >> 32);
-    *(uint32_t *) (addr +  8) = movk(reg, 1, value >> 16);
+    *(uint32_t *) (addr + 0) = mov(reg, 3, value >> 48);
+    *(uint32_t *) (addr + 4) = movk(reg, 2, value >> 32);
+    *(uint32_t *) (addr + 8) = movk(reg, 1, value >> 16);
     *(uint32_t *) (addr + 12) = movk(reg, 0, value);
     return 16;
 }
 
-static int mca_patcher_overwrite_apply_patch (mca_patcher_base_patch_t *patch)
+static int mca_patcher_overwrite_apply_patch(mca_patcher_base_patch_t *patch)
 {
     uintptr_t sys_addr, hook_addr;
     int offset, rc;
@@ -189,7 +195,7 @@ static int mca_patcher_overwrite_apply_patch (mca_patcher_base_patch_t *patch)
     hook_addr = mca_patcher_base_addr_text(patch->patch_value);
 
     /* Patch for hook function: */
-    rc = mca_patcher_base_patch_hook (&mca_patcher_overwrite_module, hook_addr);
+    rc = mca_patcher_base_patch_hook(&mca_patcher_overwrite_module, hook_addr);
     if (OPAL_SUCCESS != rc) {
         return rc;
     }
@@ -199,19 +205,91 @@ static int mca_patcher_overwrite_apply_patch (mca_patcher_base_patch_t *patch)
      * r15 is the highest numbered temporary register. I am assuming this one is safe
      * to use. */
     const unsigned int gr = 15;
-    offset = PatchLoadImm ((uintptr_t) patch->patch_data, gr, hook_addr);
+    offset = PatchLoadImm((uintptr_t) patch->patch_data, gr, hook_addr);
     *(uint32_t *) (patch->patch_data + offset) = br(gr);
     patch->patch_data_size = offset + 4;
     patch->patch_orig = sys_addr;
 
-    mca_base_patcher_patch_apply_binary (patch);
+    mca_base_patcher_patch_apply_binary(patch);
 
     return OPAL_SUCCESS;
 }
 
 #endif
 
-static int mca_patcher_overwrite_patch_address (uintptr_t sys_addr, uintptr_t hook_addr)
+/*
+ * The logic in this function for each platform is based on code from
+ * mca_patcher_overwrite_apply_patch(). There are 2 general approaches:
+ * 1: Directly check constant instructions (ignoring addresses as parameters)
+ * 2: Generate a bit mask by passing min and max values to underlying helper
+ *    functions and negate the XOR'ed results. These results can be used to
+ *    mask off transient values (like addresses) and non-instruction values
+ *    (like register contents). Once the masks are applied, the results are
+ *    compared against the min values directly to check for equality. If equal,
+ *    we consider the memory to be previously patched.
+ */
+static bool mca_patcher_is_function_patched(unsigned char *target)
+{
+
+#if defined(PLATFORM_ARCH_X86)
+    return (*(unsigned char *)target == 0xe9);
+#elif defined(PLATFORM_ARCH_X86_64)
+	return (
+		(*(unsigned short*)(target + 0) == 0xbb49) &&
+		(*(unsigned char* )(target +10) == 0x41  ) &&
+		(*(unsigned char* )(target +11) == 0xff  ) &&
+		(*(unsigned char* )(target +12) == 0xe3  )
+	);
+#elif defined(PLATFORM_ARCH_POWERPC)
+    const unsigned int gr_max = 0xF; //11 is used in our patching code, but is the max 4 or 5 bits?
+    const unsigned int addr_max = 0xFFFF;
+    unsigned int  addis_base = addis( 0, 0, 0);
+    unsigned int  addis_mask = ~(addis_base  ^  addis( gr_max,      0,  addr_max));
+    unsigned int    ori_base =   ori( 0, 0, 0);
+    unsigned int    ori_mask = ~(  ori_base  ^    ori( gr_max, gr_max,  addr_max));
+    unsigned int  mtspr_base = mtspr( 9, 0);    // 9 = CTR
+    unsigned int  mtspr_mask = ~(mtspr_base  ^  mtspr( 9,      gr_max));
+    unsigned int  bcctr_base = bcctr(20, 0, 0); // 20 = always
+    unsigned int  bcctr_mask = ~(bcctr_base  ^  bcctr(20,           0,         0));
+#if defined(PLATFORM_ARCH_32)
+
+    return (
+        ((*(unsigned int *) (target + 0 )) &  addis_mask) ==  addis_base &&
+        ((*(unsigned int *) (target + 4 )) &    ori_mask) ==    ori_base &&
+        ((*(unsigned int *) (target + 8 )) &  mtspr_mask) ==  mtspr_base &&
+        ((*(unsigned int *) (target + 12)) &  bcctr_mask) ==  bcctr_base
+    );
+#else
+    unsigned int rldicr_base = rldicr( 0, 0, 32, 31);
+    unsigned int rldicr_mask = ~(rldicr_base ^ rldicr( gr_max, gr_max, 32, 31));
+    unsigned int   oris_base =   oris( 0, 0, 0);
+    unsigned int   oris_mask = ~(oris_base ^     oris( gr_max, gr_max,  addr_max));
+
+    return (
+        ((*(unsigned int *) (target + 0 )) &  addis_mask) ==  addis_base &&
+        ((*(unsigned int *) (target + 4 )) &    ori_mask) ==    ori_base &&
+        ((*(unsigned int *) (target + 8 )) & rldicr_mask) == rldicr_base &&
+        ((*(unsigned int *) (target + 12)) &   oris_mask) ==   oris_base &&
+        ((*(unsigned int *) (target + 16)) &    ori_mask) ==    ori_base &&
+        ((*(unsigned int *) (target + 20)) &  mtspr_mask) ==  mtspr_base &&
+        ((*(unsigned int *) (target + 24)) &  bcctr_mask) ==  bcctr_base
+    );
+#endif
+#elif defined(PLATFORM_ARCH_AARCH64)
+    uint32_t mov_mask=~((0xFFFF << 5) | 0x1F);
+    uint32_t br_mask=~(0x1F << 5);
+
+    return (
+        ((*(uint32_t *) (target +  0)) & mov_mask) == mov(0, 3, 0) &&
+        ((*(uint32_t *) (target +  4)) & mov_mask) == movk(0, 2, 0) &&
+        ((*(uint32_t *) (target +  8)) & mov_mask) == movk(0, 1, 0) &&
+        ((*(uint32_t *) (target + 12)) & mov_mask) == movk(0, 0, 0) &&
+        ((*(uint32_t *) (target + 16)) & br_mask) == br(0)
+    );
+#endif
+}
+
+static int mca_patcher_overwrite_patch_address(uintptr_t sys_addr, uintptr_t hook_addr)
 {
     mca_patcher_base_patch_t *patch;
     int rc;
@@ -224,23 +302,23 @@ static int mca_patcher_overwrite_patch_address (uintptr_t sys_addr, uintptr_t ho
     patch->patch_orig = sys_addr;
     patch->patch_value = hook_addr;
 
-    opal_mutex_lock (&mca_patcher_overwrite_module.patch_list_mutex);
+    opal_mutex_lock(&mca_patcher_overwrite_module.patch_list_mutex);
     do {
-        rc = mca_patcher_overwrite_apply_patch (patch);
+        rc = mca_patcher_overwrite_apply_patch(patch);
         if (OPAL_SUCCESS != rc) {
             break;
         }
 
-        opal_list_append (&mca_patcher_overwrite_module.patch_list, &patch->super);
+        opal_list_append(&mca_patcher_overwrite_module.patch_list, &patch->super);
     } while (0);
 
-    opal_mutex_unlock (&mca_patcher_overwrite_module.patch_list_mutex);
+    opal_mutex_unlock(&mca_patcher_overwrite_module.patch_list_mutex);
 
     return OPAL_SUCCESS;
 }
 
-static int mca_patcher_overwrite_patch_symbol (const char *func_symbol_name, uintptr_t func_new_addr,
-                                               uintptr_t *func_old_addr)
+static int mca_patcher_overwrite_patch_symbol(const char *func_symbol_name, uintptr_t func_new_addr,
+                                              uintptr_t *func_old_addr)
 {
     void *sym_addr;
     char *error;
@@ -248,25 +326,30 @@ static int mca_patcher_overwrite_patch_symbol (const char *func_symbol_name, uin
 
     /* NTH: might want to update opal/mca/dl to handle lookups in the default
      * handle. */
-    sym_addr = dlsym (RTLD_NEXT, func_symbol_name);
+    sym_addr = dlsym(RTLD_NEXT, func_symbol_name);
     if (NULL == sym_addr) {
         sym_addr = dlsym(RTLD_DEFAULT, func_symbol_name);
-        if ( (sym_addr == NULL) && ((error = dlerror()) != NULL) )  {
-            opal_output(0, "error locating symbol %s to patch. %s", func_symbol_name,
-                        error);
+        if ((sym_addr == NULL) && ((error = dlerror()) != NULL)) {
+            opal_output(0, "error locating symbol %s to patch. %s", func_symbol_name, error);
             return OPAL_ERR_NOT_FOUND;
         }
     }
 
-    old_addr = (unsigned long)sym_addr;
+    old_addr = (unsigned long) sym_addr;
 
     if (func_old_addr) {
-        /* we will be overwritting part of the original function. do not return
+        /* we will be overwriting part of the original function. do not return
          * its address */
         *func_old_addr = 0;
     }
 
-    return mca_patcher_overwrite_patch_address (old_addr, func_new_addr);
+    if (mca_patcher_is_function_patched((unsigned char*)old_addr)) {
+        opal_output_verbose(10, 0, "function %s is already patched; stopping further patching\n",
+                func_symbol_name);
+        return OPAL_ERR_RESOURCE_BUSY;
+	} else {
+        return mca_patcher_overwrite_patch_address(old_addr, func_new_addr);
+	}
 }
 
 mca_patcher_base_module_t mca_patcher_overwrite_module = {
diff --git a/opal/mca/patcher/patcher.h b/opal/mca/patcher/patcher.h
index 8e8d13a1c93..1d371b0147f 100644
--- a/opal/mca/patcher/patcher.h
+++ b/opal/mca/patcher/patcher.h
@@ -2,6 +2,8 @@
 /*
  * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -14,28 +16,32 @@
 
 #include "opal_config.h"
 
-#include "opal/mca/mca.h"
-#include "opal/mca/base/base.h"
 #include "opal/class/opal_list.h"
+#include "opal/mca/base/base.h"
+#include "opal/mca/mca.h"
+#include "opal/opal_portable_platform.h"
 
 /* Any function being patched in as a hook must use SYMBOLPATCH_BEGIN at the top,
  * and SYMBOLPATCH_END before it returns (this is just for PPC). */
 
-#if (OPAL_ASSEMBLY_ARCH == OPAL_POWERPC64)
+#if defined(PLATFORM_ARCH_POWERPC) && defined(PLATFORM_ARCH_64)
 
 /* special processing for ppc64 to save and restore TOC (r2)
- * Reference: "64-bit PowerPC ELF Application Binary Interface Supplement 1.9" */
-#define OPAL_PATCHER_BEGIN \
-    unsigned long toc_save; \
-    asm volatile ("std 2, %0" : "=m" (toc_save)); \
-    asm volatile ("nop; nop; nop; nop; nop");
-#define OPAL_PATCHER_END \
-    asm volatile ("ld  2, %0" : : "m" (toc_save));
+ * Reference: "64-bit PowerPC ELF Application Binary Interface Supplement 1.9"
+ *
+ * As suggested here https://gcc.gnu.org/bugzilla/show_bug.cgi?id=95692
+ * we need to clobber the memory to guarantee ordering.
+ */
+#    define OPAL_PATCHER_BEGIN                      \
+        unsigned long toc_save;                     \
+        asm volatile("std 2, %0" : "=m"(toc_save) :: "memory"); \
+        asm volatile("nop; nop; nop; nop; nop"::: "memory");
+#    define OPAL_PATCHER_END asm volatile("ld  2, %0" : : "m"(toc_save) : "memory");
 
 #else /* !__PPC64__ */
 
-#define OPAL_PATCHER_BEGIN
-#define OPAL_PATCHER_END
+#    define OPAL_PATCHER_BEGIN
+#    define OPAL_PATCHER_END
 
 #endif
 
@@ -51,8 +57,9 @@
  * function to call the original function the patcher module will return
  * the old function's address in func_old_addr.
  */
-typedef int (*mca_patcher_base_patch_symbol_fn_t)(const char *func_symbol_name, uintptr_t func_new_addr,
-                                                   uintptr_t *func_old_addr);
+typedef int (*mca_patcher_base_patch_symbol_fn_t)(const char *func_symbol_name,
+                                                  uintptr_t func_new_addr,
+                                                  uintptr_t *func_old_addr);
 
 /**
  * Make any calls to a function redirect to a new function
@@ -69,12 +76,12 @@ typedef int (*mca_patcher_base_patch_address_fn_t)(uintptr_t func_addr, uintptr_
 /**
  * Set up the patcher module
  */
-typedef int (*mca_patcher_base_init_fn_t) (void);
+typedef int (*mca_patcher_base_init_fn_t)(void);
 
 /**
  * Finalize the patcher module
  */
-typedef int (*mca_patcher_base_fini_fn_t) (void);
+typedef int (*mca_patcher_base_fini_fn_t)(void);
 
 /**
  * Structure for patcher modules.
@@ -82,22 +89,21 @@ typedef int (*mca_patcher_base_fini_fn_t) (void);
 typedef struct mca_patcher_base_module_t {
     mca_base_module_t super;
     /** list of patches */
-    opal_list_t                         patch_list;
+    opal_list_t patch_list;
     /** lock for patch list */
-    opal_mutex_t                        patch_list_mutex;
+    opal_mutex_t patch_list_mutex;
     /** function to call if the patcher module is used. can
      * be NULL. */
-    mca_patcher_base_init_fn_t          patch_init;
+    mca_patcher_base_init_fn_t patch_init;
     /** function to call when patcher is unloaded. this function
      * MUST clean up all active patches. can be NULL. */
-    mca_patcher_base_fini_fn_t          patch_fini;
+    mca_patcher_base_fini_fn_t patch_fini;
     /** hook a symbol. may be NULL */
-    mca_patcher_base_patch_symbol_fn_t  patch_symbol;
+    mca_patcher_base_patch_symbol_fn_t patch_symbol;
     /** hook a function pointer. may be NULL */
     mca_patcher_base_patch_address_fn_t patch_address;
 } mca_patcher_base_module_t;
 
-
 OPAL_DECLSPEC extern mca_patcher_base_module_t *opal_patcher;
 
 /**
@@ -115,7 +121,6 @@ typedef mca_patcher_base_component_1_0_0_t mca_patcher_base_component_t;
 /*
  * Macro for use in components that are of type patcher
  */
-#define OPAL_PATCHER_BASE_VERSION_1_0_0 \
-    OPAL_MCA_BASE_VERSION_2_1_0("patcher", 1, 0, 0)
+#define OPAL_PATCHER_BASE_VERSION_1_0_0 OPAL_MCA_BASE_VERSION_2_1_0("patcher", 1, 0, 0)
 
 #endif /* OPAL_MCA_PATCHER_PATCHER_H */
diff --git a/opal/mca/pmix/base/Makefile.am b/opal/mca/pmix/base/Makefile.am
index a715518eed7..e2b5aece7c1 100644
--- a/opal/mca/pmix/base/Makefile.am
+++ b/opal/mca/pmix/base/Makefile.am
@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2010-2022 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2012-2013 Los Alamos National Security, Inc.  All rights reserved.
 # Copyright (c) 2014-2019 Intel, Inc.  All rights reserved.
 # $COPYRIGHT$
@@ -9,8 +9,6 @@
 # $HEADER$
 #
 
-dist_opaldata_DATA += base/help-pmix-base.txt
-
 headers += \
 	base/base.h \
 	base/static-components.h
diff --git a/opal/mca/pmix/base/base.h b/opal/mca/pmix/base/base.h
index 323a92f9800..380a98dbcec 100644
--- a/opal/mca/pmix/base/base.h
+++ b/opal/mca/pmix/base/base.h
@@ -15,10 +15,10 @@
 #define MCA_PMI_BASE_H
 
 #include "opal_config.h"
-#include "opal/types.h"
-#include "opal/mca/threads/threads.h"
-#include "opal/mca/mca.h"
 #include "opal/mca/base/mca_base_framework.h"
+#include "opal/mca/mca.h"
+#include "opal/mca/threads/threads.h"
+#include "opal/types.h"
 
 #include "opal/mca/pmix/pmix-internal.h"
 
@@ -33,9 +33,7 @@ OPAL_DECLSPEC int opal_pmix_base_select(void);
 
 OPAL_DECLSPEC extern bool opal_pmix_base_allow_delayed_server;
 
-OPAL_DECLSPEC int opal_pmix_base_exchange(pmix_info_t *info,
-                                          pmix_pdata_t *pdat,
-                                          int timeout);
+OPAL_DECLSPEC int opal_pmix_base_exchange(pmix_info_t *info, pmix_pdata_t *pdat, int timeout);
 
 typedef struct {
     opal_event_base_t *evbase;
diff --git a/opal/mca/pmix/base/help-pmix-base.txt b/opal/mca/pmix/base/help-pmix-base.txt
deleted file mode 100644
index 5a57507d4ec..00000000000
--- a/opal/mca/pmix/base/help-pmix-base.txt
+++ /dev/null
@@ -1,37 +0,0 @@
- -*- text -*-
-#
-# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2016      Los Alamos National Security, LLC. All rights
-#                         reserved.
-#
-# Copyright (c) 2016-2018 Intel, Inc. All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-# This is the US/English general help file for OPAL PMIx base.
-#
-[pmix2-init-failed]
-PMI2_Init failed to intialize.  Return code: %d
-#
-[pmix2-init-returned-bad-values]
-PMI2_Init was intialized but negative values for job size and/or
-rank was returned.
-#
-[old-pmix]
-A version of PMIx was detected that is too old:
-
-  Version:      %s
-  Min version:  %s
-
-Please reconfigure against an updated version of PMIx.
-#
-[incorrect-pmix]
-An unexpected version of PMIx was loaded:
-
-  Detected:  %s
-  Expected:  %s
-
-Please check the library path and reconfigure if required.
diff --git a/opal/mca/pmix/base/pmix_base_fns.c b/opal/mca/pmix/base/pmix_base_fns.c
index 13bfc092b60..60ce2df06cb 100644
--- a/opal/mca/pmix/base/pmix_base_fns.c
+++ b/opal/mca/pmix/base/pmix_base_fns.c
@@ -7,7 +7,7 @@
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016      Mellanox Technologies, Inc.
  *                         All rights reserved.
- * Copyright (c) 2016      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2016-2022 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
  * $COPYRIGHT$
  *
@@ -20,28 +20,29 @@
 #include "opal_config.h"
 #include "opal/constants.h"
 
-
 #include <regex.h>
 
-#include <time.h>
 #include <string.h>
+#include <time.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 
-#include "opal_stdint.h"
 #include "opal/class/opal_pointer_array.h"
 #include "opal/util/argv.h"
+#include "opal/util/opal_environ.h"
+#include "opal/util/os_path.h"
 #include "opal/util/output.h"
+#include "opal/util/printf.h"
 #include "opal/util/proc.h"
-#include "opal/util/show_help.h"
+#include "opal_stdint.h"
 
+#include "opal/mca/base/mca_base_vari.h"
 #include "opal/mca/pmix/base/base.h"
 
+#include "src/include/pmix_frameworks.h"
 
-int opal_pmix_base_exchange(pmix_info_t *indat,
-                            pmix_pdata_t *outdat,
-                            int timeout)
+int opal_pmix_base_exchange(pmix_info_t *indat, pmix_pdata_t *outdat, int timeout)
 {
     pmix_status_t rc;
     pmix_info_t info[2];
@@ -78,22 +79,171 @@ int opal_pmix_base_exchange(pmix_info_t *indat,
     return opal_pmix_convert_status(rc);
 }
 
+static bool check_pmix_param(char *param)
+{
+    char *p;
+    size_t n;
+    int len;
+
+    p = strchr(param, '_');
+    len = (int)(p - param);
+
+    if (0 == strncmp(param, "pmix", len)) {
+        return true;
+    }
+    for (n=0; NULL != pmix_framework_names[n]; n++) {
+        if (0 == strncmp(param, pmix_framework_names[n], len)) {
+            return true;
+        }
+    }
+    return false;
+}
+
+static bool check_pmix_overlap(char *var, char *value)
+{
+    char *tmp;
+
+    if (0 == strncmp(var, "dl_", 3)) {
+        opal_asprintf(&tmp, "PMIX_MCA_pdl_%s", &var[3]);
+        // set it, but don't overwrite if they already
+        // have a value in our environment
+        setenv(tmp, value, false);
+        free(tmp);
+        return true;
+    } else if (0 == strncmp(var, "oob_", 4)) {
+        opal_asprintf(&tmp, "PMIX_MCA_ptl_%s", &var[4]);
+        // set it, but don't overwrite if they already
+        // have a value in our environment
+        setenv(tmp, value, false);
+        free(tmp);
+        return true;
+    } else if (0 == strncmp(var, "hwloc_", 6)) {
+        opal_asprintf(&tmp, "PMIX_MCA_%s", var);
+        // set it, but don't overwrite if they already
+        // have a value in our environment
+        setenv(tmp, value, false);
+        free(tmp);
+        return true;
+    } else if (0 == strncmp(var, "if_", 3)) {
+        // need to convert if to pif
+        opal_asprintf(&tmp, "PMIX_MCA_pif_%s", &var[3]);
+        // set it, but don't overwrite if they already
+        // have a value in our environment
+        setenv(tmp, value, false);
+        free(tmp);
+        return true;
+    }
+    return false;
+}
+
+// NOTE: This code is fundamentally the same (module PMIX <-> OPAL)
+//      as the translate_params() routine in the PRRTE repo's
+//      src/mca/schizo/ompi/schizo_ompi.c file.  If there are
+//      changes here, there are likely to be changes there.
+static void translate_params(void)
+{
+    char *evar = NULL, *tmp, *e2 = NULL;
+    char *file;
+    const char *home;
+    opal_list_t params;
+    mca_base_var_file_value_t *fv;
+    bool pmix_overlap;
+    int n, len;
+
+    /* Since we are direct launched, we need to check the OMPI default
+     * MCA params to see if there is something relating to PRRTE
+     * in them - this would be "old" references to things from
+     * ORTE, as well as a few OPAL references that also impact us
+     *
+     * NOTE: we do this in the following precedence order. Note
+     * that we do not overwrite at any step - this is so that we
+     * don't overwrite something previously set by the user. So
+     * the order to execution is the opposite of the intended
+     * precedence order.
+     *
+     * 1. check the environmental paramaters for OMPI_MCA values
+     *    that need to be translated
+     *
+     * 2. the user's home directory file as it should
+     *    overwrite the system default file, but not the
+     *    envars
+     *
+     * 3. the system default parameter file
+     */
+    len = strlen("OMPI_MCA_");
+    for (n=0; NULL != environ[n]; n++) {
+        if (0 == strncmp(environ[n], "OMPI_MCA_", len)) {
+            e2 = strdup(environ[n]);
+            evar = strrchr(e2, '=');
+            *evar = '\0';
+            ++evar;
+            pmix_overlap = check_pmix_overlap(&e2[len], evar);
+            if (!pmix_overlap && check_pmix_param(&e2[len])) {
+                opal_asprintf(&tmp, "PMIX_MCA_%s", &e2[len]);
+                // set it, but don't overwrite if they already
+                // have a value in our environment
+                setenv(tmp, evar, false);
+                free(tmp);
+            }
+            free(e2);
+        }
+    }
+
+    /* try to get user's home directory */
+    home = opal_home_directory();
+    if (NULL != home) {
+        file = opal_os_path(false, home, ".openmpi", "mca-params.conf", NULL);
+        if (NULL != file) {
+            OBJ_CONSTRUCT(&params, opal_list_t);
+            mca_base_parse_paramfile(file, &params);
+            free(file);
+            OPAL_LIST_FOREACH (fv, &params, mca_base_var_file_value_t) {
+               pmix_overlap = check_pmix_overlap(fv->mbvfv_var, fv->mbvfv_value);
+               if (!pmix_overlap && check_pmix_param(fv->mbvfv_var)) {
+                   opal_asprintf(&tmp, "PMIX_MCA_%s", fv->mbvfv_var);
+                   // set it, but don't overwrite if they already
+                   // have a value in our environment
+                   setenv(tmp, fv->mbvfv_value, false);
+                   free(tmp);
+               }
+            }
+            OPAL_LIST_DESTRUCT(&params);
+        }
+    }
+
+    /* check if the user has set OMPIHOME in their environment */
+    if (NULL != (evar = getenv("OMPIHOME"))) {
+        /* look for the default MCA param file */
+        file = opal_os_path(false, evar, "etc", "openmpi-mca-params.conf", NULL);
+        OBJ_CONSTRUCT(&params, opal_list_t);
+        mca_base_parse_paramfile(file, &params);
+        free(file);
+        OPAL_LIST_FOREACH (fv, &params, mca_base_var_file_value_t) {
+            check_pmix_overlap(fv->mbvfv_var, fv->mbvfv_value);
+        }
+        OPAL_LIST_DESTRUCT(&params);
+    }
+}
+
 typedef struct {
     opal_list_item_t super;
     pmix_nspace_t nspace;
     opal_jobid_t jobid;
 } opal_nptr_t;
-static OBJ_CLASS_INSTANCE(opal_nptr_t,
-                          opal_list_item_t,
-                          NULL, NULL);
+static OBJ_CLASS_INSTANCE(opal_nptr_t, opal_list_item_t, NULL, NULL);
 
 static opal_list_t localnspaces;
 
 void opal_pmix_setup_nspace_tracker(void)
 {
     /* check if we were launched by PRRTE */
-    if (NULL != getenv("PRRTE_LAUNCHED")) {
+    if (NULL != getenv("PRTE_LAUNCHED")) {
         opal_process_info.nativelaunch = true;
+    } else {
+        // When direct launched, translate MCA parameters from older releases
+        // into newer versions here, since PRRTE isn't involved.  (When
+        // natively launched, PRRTE will already have translated the params)
+        translate_params();
     }
 
     OBJ_CONSTRUCT(&localnspaces, opal_list_t);
@@ -112,7 +262,7 @@ int opal_pmix_convert_jobid(pmix_nspace_t nspace, opal_jobid_t jobid)
     PMIX_LOAD_NSPACE(nspace, NULL);
 
     /* cycle across our list of known jobids */
-    OPAL_LIST_FOREACH(nptr, &localnspaces, opal_nptr_t) {
+    OPAL_LIST_FOREACH (nptr, &localnspaces, opal_nptr_t) {
         if (jobid == nptr->jobid) {
             PMIX_LOAD_NSPACE(nspace, nptr->nspace);
             return OPAL_SUCCESS;
@@ -141,7 +291,7 @@ int opal_pmix_convert_nspace(opal_jobid_t *jobid, pmix_nspace_t nspace)
     }
 
     /* cycle across our list of known nspace's */
-    OPAL_LIST_FOREACH(nptr, &localnspaces, opal_nptr_t) {
+    OPAL_LIST_FOREACH (nptr, &localnspaces, opal_nptr_t) {
         if (PMIX_CHECK_NSPACE(nspace, nptr->nspace)) {
             if (NULL != jobid) {
                 *jobid = nptr->jobid;
@@ -164,7 +314,7 @@ int opal_pmix_convert_nspace(opal_jobid_t *jobid, pmix_nspace_t nspace)
 
     /* now compress to 16-bits */
     jobfam = (uint16_t)(((0x0000ffff & (0xffff0000 & hash32) >> 16)) ^ (0x0000ffff & hash32));
-    jid = (0xffff0000 & ((uint32_t)jobfam << 16)) | (0x0000ffff & localjob);
+    jid = (0xffff0000 & ((uint32_t) jobfam << 16)) | (0x0000ffff & localjob);
     if (NULL != jobid) {
         *jobid = jid;
     }
@@ -356,102 +506,99 @@ int opal_pmix_convert_status(pmix_status_t status)
 
 pmix_proc_state_t opal_pmix_convert_state(int state)
 {
-    switch(state) {
-        case 0:
-            return PMIX_PROC_STATE_UNDEF;
-        case 1:
-            return PMIX_PROC_STATE_LAUNCH_UNDERWAY;
-        case 2:
-            return PMIX_PROC_STATE_RESTART;
-        case 3:
-            return PMIX_PROC_STATE_TERMINATE;
-        case 4:
-            return PMIX_PROC_STATE_RUNNING;
-        case 5:
-            return PMIX_PROC_STATE_CONNECTED;
-        case 51:
-            return PMIX_PROC_STATE_KILLED_BY_CMD;
-        case 52:
-            return PMIX_PROC_STATE_ABORTED;
-        case 53:
-            return PMIX_PROC_STATE_FAILED_TO_START;
-        case 54:
-            return PMIX_PROC_STATE_ABORTED_BY_SIG;
-        case 55:
-            return PMIX_PROC_STATE_TERM_WO_SYNC;
-        case 56:
-            return PMIX_PROC_STATE_COMM_FAILED;
-        case 58:
-            return PMIX_PROC_STATE_CALLED_ABORT;
-        case 59:
-            return PMIX_PROC_STATE_MIGRATING;
-        case 61:
-            return PMIX_PROC_STATE_CANNOT_RESTART;
-        case 62:
-            return PMIX_PROC_STATE_TERM_NON_ZERO;
-        case 63:
-            return PMIX_PROC_STATE_FAILED_TO_LAUNCH;
-        default:
-            return PMIX_PROC_STATE_UNDEF;
+    switch (state) {
+    case 0:
+        return PMIX_PROC_STATE_UNDEF;
+    case 1:
+        return PMIX_PROC_STATE_LAUNCH_UNDERWAY;
+    case 2:
+        return PMIX_PROC_STATE_RESTART;
+    case 3:
+        return PMIX_PROC_STATE_TERMINATE;
+    case 4:
+        return PMIX_PROC_STATE_RUNNING;
+    case 5:
+        return PMIX_PROC_STATE_CONNECTED;
+    case 51:
+        return PMIX_PROC_STATE_KILLED_BY_CMD;
+    case 52:
+        return PMIX_PROC_STATE_ABORTED;
+    case 53:
+        return PMIX_PROC_STATE_FAILED_TO_START;
+    case 54:
+        return PMIX_PROC_STATE_ABORTED_BY_SIG;
+    case 55:
+        return PMIX_PROC_STATE_TERM_WO_SYNC;
+    case 56:
+        return PMIX_PROC_STATE_COMM_FAILED;
+    case 58:
+        return PMIX_PROC_STATE_CALLED_ABORT;
+    case 59:
+        return PMIX_PROC_STATE_MIGRATING;
+    case 61:
+        return PMIX_PROC_STATE_CANNOT_RESTART;
+    case 62:
+        return PMIX_PROC_STATE_TERM_NON_ZERO;
+    case 63:
+        return PMIX_PROC_STATE_FAILED_TO_LAUNCH;
+    default:
+        return PMIX_PROC_STATE_UNDEF;
     }
 }
 
 int opal_pmix_convert_pstate(pmix_proc_state_t state)
 {
-    switch(state) {
-        case PMIX_PROC_STATE_UNDEF:
-            return 0;
-        case PMIX_PROC_STATE_PREPPED:
-        case PMIX_PROC_STATE_LAUNCH_UNDERWAY:
-            return 1;
-        case PMIX_PROC_STATE_RESTART:
-            return 2;
-        case PMIX_PROC_STATE_TERMINATE:
-            return 3;
-        case PMIX_PROC_STATE_RUNNING:
-            return 4;
-        case PMIX_PROC_STATE_CONNECTED:
-            return 5;
-        case PMIX_PROC_STATE_UNTERMINATED:
-            return 15;
-        case PMIX_PROC_STATE_TERMINATED:
-            return 20;
-        case PMIX_PROC_STATE_KILLED_BY_CMD:
-            return 51;
-        case PMIX_PROC_STATE_ABORTED:
-            return 52;
-        case PMIX_PROC_STATE_FAILED_TO_START:
-            return 53;
-        case PMIX_PROC_STATE_ABORTED_BY_SIG:
-            return 54;
-        case PMIX_PROC_STATE_TERM_WO_SYNC:
-            return 55;
-        case PMIX_PROC_STATE_COMM_FAILED:
-            return 56;
-        case PMIX_PROC_STATE_CALLED_ABORT:
-            return 58;
-        case PMIX_PROC_STATE_MIGRATING:
-            return 60;
-        case PMIX_PROC_STATE_CANNOT_RESTART:
-            return 61;
-        case PMIX_PROC_STATE_TERM_NON_ZERO:
-            return 62;
-        case PMIX_PROC_STATE_FAILED_TO_LAUNCH:
-            return 63;
-        default:
-            return 0;  // undef
+    switch (state) {
+    case PMIX_PROC_STATE_UNDEF:
+        return 0;
+    case PMIX_PROC_STATE_PREPPED:
+    case PMIX_PROC_STATE_LAUNCH_UNDERWAY:
+        return 1;
+    case PMIX_PROC_STATE_RESTART:
+        return 2;
+    case PMIX_PROC_STATE_TERMINATE:
+        return 3;
+    case PMIX_PROC_STATE_RUNNING:
+        return 4;
+    case PMIX_PROC_STATE_CONNECTED:
+        return 5;
+    case PMIX_PROC_STATE_UNTERMINATED:
+        return 15;
+    case PMIX_PROC_STATE_TERMINATED:
+        return 20;
+    case PMIX_PROC_STATE_KILLED_BY_CMD:
+        return 51;
+    case PMIX_PROC_STATE_ABORTED:
+        return 52;
+    case PMIX_PROC_STATE_FAILED_TO_START:
+        return 53;
+    case PMIX_PROC_STATE_ABORTED_BY_SIG:
+        return 54;
+    case PMIX_PROC_STATE_TERM_WO_SYNC:
+        return 55;
+    case PMIX_PROC_STATE_COMM_FAILED:
+        return 56;
+    case PMIX_PROC_STATE_CALLED_ABORT:
+        return 58;
+    case PMIX_PROC_STATE_MIGRATING:
+        return 60;
+    case PMIX_PROC_STATE_CANNOT_RESTART:
+        return 61;
+    case PMIX_PROC_STATE_TERM_NON_ZERO:
+        return 62;
+    case PMIX_PROC_STATE_FAILED_TO_LAUNCH:
+        return 63;
+    default:
+        return 0; // undef
     }
 }
 
 #if PMIX_NUMERIC_VERSION >= 0x00030000
 
-static void cleanup_cbfunc(pmix_status_t status,
-                           pmix_info_t *info, size_t ninfo,
-                           void *cbdata,
-                           pmix_release_cbfunc_t release_fn,
-                           void *release_cbdata)
+static void cleanup_cbfunc(pmix_status_t status, pmix_info_t *info, size_t ninfo, void *cbdata,
+                           pmix_release_cbfunc_t release_fn, void *release_cbdata)
 {
-    opal_pmix_lock_t *lk = (opal_pmix_lock_t*)cbdata;
+    opal_pmix_lock_t *lk = (opal_pmix_lock_t *) cbdata;
 
     OPAL_POST_OBJECT(lk);
 
@@ -472,7 +619,7 @@ int opal_pmix_register_cleanup(char *path, bool directory, bool ignore, bool job
 #if PMIX_NUMERIC_VERSION >= 0x00030000
     opal_pmix_lock_t lk;
     pmix_info_t pinfo[3];
-    size_t n, ninfo=0;
+    size_t n, ninfo = 0;
     pmix_status_t rc, ret;
     pmix_proc_t proc;
 
@@ -498,27 +645,32 @@ int opal_pmix_register_cleanup(char *path, bool directory, bool ignore, bool job
 
     /* if they want this applied to the job, then indicate so */
     if (jobscope) {
-        rc = PMIx_Job_control_nb(NULL, 0, pinfo, ninfo, cleanup_cbfunc, (void*)&lk);
+        rc = PMIx_Job_control_nb(NULL, 0, pinfo, ninfo, cleanup_cbfunc, (void *) &lk);
     } else {
         /* only applies to us */
-        (void)snprintf(proc.nspace, PMIX_MAX_NSLEN, "%s",
-                       OPAL_JOBID_PRINT(OPAL_PROC_MY_NAME.jobid));
+        pmix_nspace_t nspace;
+        if(OPAL_SUCCESS == opal_pmix_convert_jobid(nspace, OPAL_PROC_MY_NAME.jobid)) {
+          (void) snprintf(proc.nspace, PMIX_MAX_NSLEN, "%s", nspace);
+        }
+        else {
+          (void) snprintf(proc.nspace, PMIX_MAX_NSLEN, "%s", OPAL_JOBID_PRINT(OPAL_PROC_MY_NAME.jobid));
+        }
         proc.rank = OPAL_PROC_MY_NAME.vpid;
-        rc = PMIx_Job_control_nb(&proc, 1, pinfo, ninfo, cleanup_cbfunc, (void*)&lk);
+        rc = PMIx_Job_control_nb(&proc, 1, pinfo, ninfo, cleanup_cbfunc, (void *) &lk);
     }
     if (PMIX_SUCCESS != rc) {
         ret = rc;
     } else {
-#if PMIX_VERSION_MAJOR == 3 && PMIX_VERSION_MINOR == 0 && PMIX_VERSION_RELEASE < 3
+#    if PMIX_VERSION_MAJOR == 3 && PMIX_VERSION_MINOR == 0 && PMIX_VERSION_RELEASE < 3
         /* There is a bug in PMIx 3.0.0 up to 3.0.2 that causes the callback never
          * being called, so assumes the everything went well and avoid a deadlock. */
-        cleanup_cbfunc(PMIX_SUCCESS, NULL, 0, (void *)&lk, NULL, NULL);
-#endif
+        cleanup_cbfunc(PMIX_SUCCESS, NULL, 0, (void *) &lk, NULL, NULL);
+#    endif
         OPAL_PMIX_WAIT_THREAD(&lk);
         ret = lk.status;
     }
     OPAL_PMIX_DESTRUCT_LOCK(&lk);
-    for (n=0; n < ninfo; n++) {
+    for (n = 0; n < ninfo; n++) {
         PMIX_INFO_DESTRUCT(&pinfo[n]);
     }
     return ret;
@@ -527,7 +679,6 @@ int opal_pmix_register_cleanup(char *path, bool directory, bool ignore, bool job
 #endif
 }
 
-
 /* CLASS INSTANTIATIONS */
 static void dsicon(opal_ds_info_t *p)
 {
@@ -539,9 +690,7 @@ static void dsicon(opal_ds_info_t *p)
     p->persistence = PMIX_PERSIST_INDEF;
 #endif
 }
-OBJ_CLASS_INSTANCE(opal_ds_info_t,
-                   opal_list_item_t,
-                   dsicon, NULL);
+OBJ_CLASS_INSTANCE(opal_ds_info_t, opal_list_item_t, dsicon, NULL);
 
 static void infoitmcon(opal_info_item_t *p)
 {
@@ -551,10 +700,6 @@ static void infoitdecon(opal_info_item_t *p)
 {
     PMIX_INFO_DESTRUCT(&p->info);
 }
-OBJ_CLASS_INSTANCE(opal_info_item_t,
-                   opal_list_item_t,
-                   infoitmcon, infoitdecon);
+OBJ_CLASS_INSTANCE(opal_info_item_t, opal_list_item_t, infoitmcon, infoitdecon);
 
-OBJ_CLASS_INSTANCE(opal_proclist_t,
-                   opal_list_item_t,
-                   NULL, NULL);
+OBJ_CLASS_INSTANCE(opal_proclist_t, opal_list_item_t, NULL, NULL);
diff --git a/opal/mca/pmix/base/pmix_base_frame.c b/opal/mca/pmix/base/pmix_base_frame.c
index 0d509b5bfd6..cda84d77aba 100644
--- a/opal/mca/pmix/base/pmix_base_frame.c
+++ b/opal/mca/pmix/base/pmix_base_frame.c
@@ -1,6 +1,9 @@
 /*
  * Copyright (c) 2014-2019 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015-2016 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2022      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -8,19 +11,18 @@
  * $HEADER$
  */
 
-
 #include "opal_config.h"
 #include "opal/constants.h"
 
+#include "opal/mca/base/base.h"
 #include "opal/mca/mca.h"
 #include "opal/mca/threads/thread_usage.h"
 #include "opal/util/argv.h"
 #include "opal/util/output.h"
-#include "opal/mca/base/base.h"
+#include "opal/util/proc.h"
 
-#include "opal/mca/pmix/pmix-internal.h"
 #include "opal/mca/pmix/base/base.h"
-
+#include "opal/mca/pmix/pmix-internal.h"
 
 /*
  * The following file was created by configure.  It contains extern
@@ -33,27 +35,25 @@
 bool opal_pmix_collect_all_data = true;
 int opal_pmix_verbose_output = -1;
 bool opal_pmix_base_async_modex = false;
-opal_pmix_base_t opal_pmix_base = {
-    .evbase = NULL,
-    .timeout = 0,
-    .initialized = 0,
-    .lock = {
-        .mutex = OPAL_MUTEX_STATIC_INIT,
-        .cond = OPAL_PMIX_CONDITION_STATIC_INIT,
-        .active = false
-    }
-};
+opal_pmix_base_t opal_pmix_base = {.evbase = NULL,
+                                   .timeout = 0,
+                                   .initialized = 0,
+                                   .lock = {.mutex = OPAL_MUTEX_STATIC_INIT,
+                                            .cond = OPAL_PMIX_CONDITION_STATIC_INIT,
+                                            .active = false}};
 
 static int opal_pmix_base_frame_register(mca_base_register_flag_t flags)
 {
     opal_pmix_base_async_modex = false;
-    (void) mca_base_var_register("opal", "pmix", "base", "async_modex", "Use asynchronous modex mode",
-                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9,
-                                 MCA_BASE_VAR_SCOPE_READONLY, &opal_pmix_base_async_modex);
+    (void) mca_base_var_register("opal", "pmix", "base", "async_modex",
+                                 "Use asynchronous modex mode", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
+                                 OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+                                 &opal_pmix_base_async_modex);
     opal_pmix_collect_all_data = true;
-    (void) mca_base_var_register("opal", "pmix", "base", "collect_data", "Collect all data during modex",
-                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9,
-                                 MCA_BASE_VAR_SCOPE_READONLY, &opal_pmix_collect_all_data);
+    (void) mca_base_var_register("opal", "pmix", "base", "collect_data",
+                                 "Collect all data during modex", MCA_BASE_VAR_TYPE_BOOL, NULL, 0,
+                                 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+                                 &opal_pmix_collect_all_data);
 
     opal_pmix_base.timeout = -1;
     (void) mca_base_var_register("opal", "pmix", "base", "exchange_timeout",
@@ -63,6 +63,32 @@ static int opal_pmix_base_frame_register(mca_base_register_flag_t flags)
     return OPAL_SUCCESS;
 }
 
+static char*
+opal_get_proc_hostname_using_pmix(const opal_proc_t *proc)
+{
+    int ret;
+    char *hostname;
+
+    /* if the proc is NULL, then we can't know */
+    if (NULL == proc) {
+        return strdup("unknown");
+    }
+
+    /* if it is my own hostname we are after, then just hand back
+     * the value in opal_process_info */
+    if (proc == opal_proc_local_get()) {
+        return strdup(opal_process_info.nodename);
+    }
+    /* if we don't already have it, then try to get it */
+    OPAL_MODEX_RECV_VALUE_OPTIONAL(ret, PMIX_HOSTNAME, &proc->proc_name, (char **) &hostname,
+                                   PMIX_STRING);
+    if (OPAL_SUCCESS != ret) {
+        return strdup("unknown"); // return something so the caller doesn't segfault
+    }
+    /* user is not allowed to release the data */
+    return hostname;
+}
+
 static int opal_pmix_base_frame_close(void)
 {
     int rc;
@@ -81,11 +107,11 @@ static int opal_pmix_base_frame_open(mca_base_open_flag_t flags)
     opal_pmix_base.evbase = opal_sync_event_base;
     /* pass across the verbosity */
     opal_pmix_verbose_output = opal_pmix_base_framework.framework_output;
+    /* Set the distributed name service via PMIx */
+    opal_get_proc_hostname = opal_get_proc_hostname_using_pmix;
     return rc;
 }
 
-MCA_BASE_FRAMEWORK_DECLARE(opal, pmix, "OPAL PMI Client Framework",
-                           opal_pmix_base_frame_register,
-                           opal_pmix_base_frame_open,
-                           opal_pmix_base_frame_close,
+MCA_BASE_FRAMEWORK_DECLARE(opal, pmix, "OPAL PMI Client Framework", opal_pmix_base_frame_register,
+                           opal_pmix_base_frame_open, opal_pmix_base_frame_close,
                            mca_pmix_base_static_components, 0);
diff --git a/opal/mca/pmix/base/static-components.h b/opal/mca/pmix/base/static-components.h
index a4e4303659b..f2be97a0c2b 100644
--- a/opal/mca/pmix/base/static-components.h
+++ b/opal/mca/pmix/base/static-components.h
@@ -9,10 +9,8 @@ extern const mca_base_component_t mca_pmix_pmix4x_component;
 
 const mca_base_component_t *mca_pmix_base_static_components[] = {
 
-  NULL
-};
+    NULL};
 
 #if defined(c_plusplus) || defined(__cplusplus)
 }
 #endif
-
diff --git a/opal/mca/pmix/pmix-internal.h b/opal/mca/pmix/pmix-internal.h
index 23aa731ec78..3c00306f501 100644
--- a/opal/mca/pmix/pmix-internal.h
+++ b/opal/mca/pmix/pmix-internal.h
@@ -5,11 +5,13 @@
  *                         reserved.
  * Copyright (c) 2019      Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2020      Triad National Security, LLC. All rights
+ * Copyright (c) 2020-2021 Triad National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2020      Amazon.com, Inc. or its affiliates.
  *                         All Rights reserved.
- * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2021-2024 Nanook Consulting  All rights reserved.
+ * Copyright (c) 2021      Argonne National Laboratory.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -32,18 +34,18 @@
 #include "opal/types.h"
 
 #ifdef HAVE_SYS_UN_H
-#include <sys/un.h>
+#    include <sys/un.h>
 #endif
 
+#include "opal/hash_string.h"
 #include "opal/mca/mca.h"
-#include "opal/util/event.h"
+#include "opal/class/opal_list.h"
 #include "opal/mca/threads/threads.h"
 #include "opal/util/error.h"
-#include "opal/hash_string.h"
+#include "opal/util/event.h"
 
 #include <pmix.h>
 
-
 BEGIN_C_DECLS
 
 typedef uint32_t opal_jobid_t;
@@ -54,9 +56,9 @@ typedef struct {
 } opal_process_name_t;
 
 #define OPAL_SIZEOF_PROCESS_NAME_T 8
-#define OPAL_EQUAL 0
-#define OPAL_VALUE1_GREATER 1
-#define OPAL_VALUE2_GREATER -1
+#define OPAL_EQUAL                 0
+#define OPAL_VALUE1_GREATER        1
+#define OPAL_VALUE2_GREATER        -1
 
 /* provide access to the framework verbose output without
  * exposing the entire base */
@@ -89,7 +91,6 @@ typedef struct {
 } opal_proclist_t;
 OBJ_CLASS_DECLARATION(opal_proclist_t);
 
-
 typedef opal_cond_t opal_pmix_condition_t;
 
 typedef struct {
@@ -97,130 +98,124 @@ typedef struct {
     opal_pmix_condition_t cond;
     volatile bool active;
     int status;
+    size_t errhandler_ref;
     char *msg;
 } opal_pmix_lock_t;
 
-#define opal_pmix_condition_wait(a,b)    opal_cond_wait(a, b)
+#define opal_pmix_condition_wait(a, b)   opal_cond_wait(a, b)
 #define opal_pmix_condition_broadcast(a) opal_cond_broadcast(a)
 
-#define OPAL_PMIX_CONSTRUCT_LOCK(l)                     \
-    do {                                                \
-        OBJ_CONSTRUCT(&(l)->mutex, opal_mutex_t);       \
-        opal_cond_init(&(l)->cond);                     \
-        (l)->active = true;                             \
-        (l)->status = 0;                                \
-        (l)->msg = NULL;                                \
-        OPAL_POST_OBJECT((l));                          \
-    } while(0)
-
-#define OPAL_PMIX_DESTRUCT_LOCK(l)          \
-    do {                                    \
-        OPAL_ACQUIRE_OBJECT((l));           \
-        OBJ_DESTRUCT(&(l)->mutex);          \
-        opal_cond_destroy(&(l)->cond);      \
-        if (NULL != (l)->msg) {             \
-            free((l)->msg);                 \
-        }                                   \
-    } while(0)
-
+#define OPAL_PMIX_CONSTRUCT_LOCK(l)               \
+    do {                                          \
+        OBJ_CONSTRUCT(&(l)->mutex, opal_mutex_t); \
+        opal_cond_init(&(l)->cond);               \
+        (l)->active = true;                       \
+        (l)->status = 0;                          \
+        (l)->msg = NULL;                          \
+        OPAL_POST_OBJECT((l));                    \
+    } while (0)
+
+#define OPAL_PMIX_DESTRUCT_LOCK(l)     \
+    do {                               \
+        OPAL_ACQUIRE_OBJECT((l));      \
+        OBJ_DESTRUCT(&(l)->mutex);     \
+        opal_cond_destroy(&(l)->cond); \
+        if (NULL != (l)->msg) {        \
+            free((l)->msg);            \
+        }                              \
+    } while (0)
 
 #if OPAL_ENABLE_DEBUG
-#define OPAL_PMIX_ACQUIRE_THREAD(lck)                               \
-    do {                                                            \
-        opal_mutex_lock(&(lck)->mutex);                             \
-        if (opal_debug_threads) {                                   \
-            opal_output(0, "Waiting for thread %s:%d",              \
-                        __FILE__, __LINE__);                        \
-        }                                                           \
-        while ((lck)->active) {                                     \
-            opal_pmix_condition_wait(&(lck)->cond, &(lck)->mutex);  \
-        }                                                           \
-        if (opal_debug_threads) {                                   \
-            opal_output(0, "Thread obtained %s:%d",                 \
-                        __FILE__, __LINE__);                        \
-        }                                                           \
-        (lck)->active = true;                                       \
-    } while(0)
+#    define OPAL_PMIX_ACQUIRE_THREAD(lck)                                       \
+        do {                                                                    \
+            opal_mutex_lock(&(lck)->mutex);                                     \
+            if (opal_debug_threads) {                                           \
+                opal_output(0, "Waiting for thread %s:%d", __FILE__, __LINE__); \
+            }                                                                   \
+            while ((lck)->active) {                                             \
+                opal_pmix_condition_wait(&(lck)->cond, &(lck)->mutex);          \
+            }                                                                   \
+            if (opal_debug_threads) {                                           \
+                opal_output(0, "Thread obtained %s:%d", __FILE__, __LINE__);    \
+            }                                                                   \
+            (lck)->active = true;                                               \
+        } while (0)
 #else
-#define OPAL_PMIX_ACQUIRE_THREAD(lck)                               \
-    do {                                                            \
-        opal_mutex_lock(&(lck)->mutex);                             \
-        while ((lck)->active) {                                     \
-            opal_pmix_condition_wait(&(lck)->cond, &(lck)->mutex);  \
-        }                                                           \
-        (lck)->active = true;                                       \
-    } while(0)
+#    define OPAL_PMIX_ACQUIRE_THREAD(lck)                              \
+        do {                                                           \
+            opal_mutex_lock(&(lck)->mutex);                            \
+            while ((lck)->active) {                                    \
+                opal_pmix_condition_wait(&(lck)->cond, &(lck)->mutex); \
+            }                                                          \
+            (lck)->active = true;                                      \
+        } while (0)
 #endif
 
-
 #if OPAL_ENABLE_DEBUG
-#define OPAL_PMIX_WAIT_THREAD(lck)                                  \
-    do {                                                            \
-        opal_mutex_lock(&(lck)->mutex);                             \
-        if (opal_debug_threads) {                                   \
-            opal_output(0, "Waiting for thread %s:%d",              \
-                        __FILE__, __LINE__);                        \
-        }                                                           \
-        while ((lck)->active) {                                     \
-            opal_pmix_condition_wait(&(lck)->cond, &(lck)->mutex);  \
-        }                                                           \
-        if (opal_debug_threads) {                                   \
-            opal_output(0, "Thread obtained %s:%d",                 \
-                        __FILE__, __LINE__);                        \
-        }                                                           \
-        OPAL_ACQUIRE_OBJECT(&lck);                                  \
-        opal_mutex_unlock(&(lck)->mutex);                           \
-    } while(0)
+#    define OPAL_PMIX_WAIT_THREAD(lck)                                          \
+        do {                                                                    \
+            opal_mutex_lock(&(lck)->mutex);                                     \
+            if (opal_debug_threads) {                                           \
+                opal_output(0, "Waiting for thread %s:%d", __FILE__, __LINE__); \
+            }                                                                   \
+            while ((lck)->active) {                                             \
+                opal_pmix_condition_wait(&(lck)->cond, &(lck)->mutex);          \
+            }                                                                   \
+            if (opal_debug_threads) {                                           \
+                opal_output(0, "Thread obtained %s:%d", __FILE__, __LINE__);    \
+            }                                                                   \
+            OPAL_ACQUIRE_OBJECT(&lck);                                          \
+            opal_mutex_unlock(&(lck)->mutex);                                   \
+        } while (0)
 #else
-#define OPAL_PMIX_WAIT_THREAD(lck)                                  \
-    do {                                                            \
-        opal_mutex_lock(&(lck)->mutex);                             \
-        while ((lck)->active) {                                     \
-            opal_pmix_condition_wait(&(lck)->cond, &(lck)->mutex);  \
-        }                                                           \
-        OPAL_ACQUIRE_OBJECT(lck);                                   \
-        opal_mutex_unlock(&(lck)->mutex);                           \
-    } while(0)
+#    define OPAL_PMIX_WAIT_THREAD(lck)                                 \
+        do {                                                           \
+            opal_mutex_lock(&(lck)->mutex);                            \
+            while ((lck)->active) {                                    \
+                opal_pmix_condition_wait(&(lck)->cond, &(lck)->mutex); \
+            }                                                          \
+            OPAL_ACQUIRE_OBJECT(lck);                                  \
+            opal_mutex_unlock(&(lck)->mutex);                          \
+        } while (0)
 #endif
 
-
 #if OPAL_ENABLE_DEBUG
-#define OPAL_PMIX_RELEASE_THREAD(lck)                   \
-    do {                                                \
-        if (opal_debug_threads) {                       \
-            opal_output(0, "Releasing thread %s:%d",    \
-                        __FILE__, __LINE__);            \
-        }                                               \
-        (lck)->active = false;                          \
-        opal_pmix_condition_broadcast(&(lck)->cond);    \
-        opal_mutex_unlock(&(lck)->mutex);               \
-    } while(0)
+#    define OPAL_PMIX_RELEASE_THREAD(lck)                                     \
+        do {                                                                  \
+            if (opal_debug_threads) {                                         \
+                opal_output(0, "Releasing thread %s:%d", __FILE__, __LINE__); \
+            }                                                                 \
+            (lck)->active = false;                                            \
+            opal_pmix_condition_broadcast(&(lck)->cond);                      \
+            opal_mutex_unlock(&(lck)->mutex);                                 \
+        } while (0)
 #else
-#define OPAL_PMIX_RELEASE_THREAD(lck)                   \
-    do {                                                \
-        assert(0 != opal_mutex_trylock(&(lck)->mutex)); \
-        (lck)->active = false;                          \
-        opal_pmix_condition_broadcast(&(lck)->cond);    \
-        opal_mutex_unlock(&(lck)->mutex);               \
-    } while(0)
+#    define OPAL_PMIX_RELEASE_THREAD(lck)                   \
+        do {                                                \
+            assert(0 != opal_mutex_trylock(&(lck)->mutex)); \
+            (lck)->active = false;                          \
+            opal_pmix_condition_broadcast(&(lck)->cond);    \
+            opal_mutex_unlock(&(lck)->mutex);               \
+        } while (0)
 #endif
 
-#define OPAL_PMIX_WAKEUP_THREAD(lck)                    \
-    do {                                                \
-        opal_mutex_lock(&(lck)->mutex);                 \
-        (lck)->active = false;                          \
-        OPAL_POST_OBJECT(lck);                          \
-        opal_pmix_condition_broadcast(&(lck)->cond);    \
-        opal_mutex_unlock(&(lck)->mutex);               \
-    } while(0)
+#define OPAL_PMIX_WAKEUP_THREAD(lck)                 \
+    do {                                             \
+        opal_mutex_lock(&(lck)->mutex);              \
+        (lck)->active = false;                       \
+        OPAL_POST_OBJECT(lck);                       \
+        opal_pmix_condition_broadcast(&(lck)->cond); \
+        opal_mutex_unlock(&(lck)->mutex);            \
+    } while (0)
 
 /*
  * Count the fash for the the external RM
  */
-#define OPAL_HASH_JOBID( str, hash ){               \
-    OPAL_HASH_STR( str, hash );                     \
-    hash &= ~(0x8000);                              \
-}
+#define OPAL_HASH_JOBID(str, hash) \
+    {                              \
+        OPAL_HASH_STR(str, hash);  \
+        hash &= ~(0x8000);         \
+    }
 
 /**
  * Provide a simplified macro for sending data via modex
@@ -232,12 +227,12 @@ typedef struct {
  * d - pointer to the data object being posted
  * t - the type of the data
  */
-#define OPAL_MODEX_SEND_VALUE(r, sc, s, d, t)   \
-    do {                                        \
-        pmix_value_t _kv;                       \
-        PMIX_VALUE_LOAD(&_kv, (d), (t));        \
-        (r) = PMIx_Put((sc), (s), &(_kv));      \
-    } while(0);
+#define OPAL_MODEX_SEND_VALUE(r, sc, s, d, t) \
+    do {                                      \
+        pmix_value_t _kv;                     \
+        PMIX_VALUE_LOAD(&_kv, (d), (t));      \
+        (r) = PMIx_Put((sc), (s), &(_kv));    \
+    } while (0);
 
 /**
  * Provide a simplified macro for sending data via modex
@@ -249,14 +244,14 @@ typedef struct {
  * d - the data object being posted
  * sz - the number of bytes in the data object
  */
-#define OPAL_MODEX_SEND_STRING(r, sc, s, d, sz)     \
-    do {                                            \
-        pmix_value_t _kv;                           \
-        _kv.type = PMIX_BYTE_OBJECT;                \
-        _kv.data.bo.bytes = (char*)(d);             \
-        _kv.data.bo.size = (sz);                    \
-        (r) = PMIx_Put(sc, (s), &(_kv));            \
-    } while(0);
+#define OPAL_MODEX_SEND_STRING(r, sc, s, d, sz) \
+    do {                                        \
+        pmix_value_t _kv;                       \
+        _kv.type = PMIX_BYTE_OBJECT;            \
+        _kv.data.bo.bytes = (char *) (d);       \
+        _kv.data.bo.size = (sz);                \
+        (r) = PMIx_Put(sc, (s), &(_kv));        \
+    } while (0);
 
 /**
  * Provide a simplified macro for sending data via modex
@@ -268,13 +263,13 @@ typedef struct {
  * d - the data object being posted
  * sz - the number of bytes in the data object
  */
-#define OPAL_MODEX_SEND(r, sc, s, d, sz)                        \
-    do {                                                        \
-        char *_key;                                             \
-        _key = mca_base_component_to_string((s));               \
-        OPAL_MODEX_SEND_STRING((r), (sc), _key, (d), (sz));     \
-        free(_key);                                             \
-    } while(0);
+#define OPAL_MODEX_SEND(r, sc, s, d, sz)                    \
+    do {                                                    \
+        char *_key;                                         \
+        _key = mca_base_component_to_string((s));           \
+        OPAL_MODEX_SEND_STRING((r), (sc), _key, (d), (sz)); \
+        free(_key);                                         \
+    } while (0);
 
 /**
  * Provide a simplified macro for retrieving modex data
@@ -289,32 +284,31 @@ typedef struct {
  *     is to be returned
  * t - the expected data type
  */
-#define OPAL_MODEX_RECV_VALUE_OPTIONAL(r, s, p, d, t)                                   \
-    do {                                                                                \
-        pmix_proc_t _proc;                                                              \
-        pmix_value_t *_kv = NULL;                                                       \
-        pmix_info_t _info;                                                              \
-        size_t _sz;                                                                     \
-        OPAL_OUTPUT_VERBOSE((1, opal_pmix_verbose_output,                               \
-                            "%s[%s:%d] MODEX RECV VALUE OPTIONAL FOR PROC %s KEY %s",   \
-                            OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),                         \
-                            __FILE__, __LINE__,                                         \
-                            OPAL_NAME_PRINT(*(p)), (s)));                               \
-        OPAL_PMIX_CONVERT_NAME(&_proc, (p));                                            \
-        PMIX_INFO_LOAD(&_info, PMIX_OPTIONAL, NULL, PMIX_BOOL);                         \
-        (r) = PMIx_Get(&(_proc), (s), &(_info), 1, &(_kv));                             \
-        PMIX_INFO_DESTRUCT(&_info);                                                     \
-        if (NULL == _kv) {                                                              \
-            (r) = PMIX_ERR_NOT_FOUND;                                                   \
-        } else if (_kv->type != (t)) {                                                  \
-            (r) = PMIX_ERR_TYPE_MISMATCH;                                               \
-        } else if (PMIX_SUCCESS == (r)) {                                               \
-            PMIX_VALUE_UNLOAD((r), _kv, (void**)(d), &_sz);                             \
-        }                                                                               \
-        if (NULL != _kv) {                                                              \
-            PMIX_VALUE_RELEASE(_kv);                                                    \
-        }                                                                               \
-    } while(0);
+#define OPAL_MODEX_RECV_VALUE_OPTIONAL(r, s, p, d, t)                                  \
+    do {                                                                               \
+        pmix_proc_t _proc;                                                             \
+        pmix_value_t *_kv = NULL;                                                      \
+        pmix_info_t _info;                                                             \
+        size_t _sz;                                                                    \
+        OPAL_OUTPUT_VERBOSE((1, opal_pmix_verbose_output,                              \
+                             "%s[%s:%d] MODEX RECV VALUE OPTIONAL FOR PROC %s KEY %s", \
+                             OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), __FILE__, __LINE__,   \
+                             OPAL_NAME_PRINT(*(p)), PMIx_Get_attribute_name(s)));      \
+        OPAL_PMIX_CONVERT_NAME(&_proc, (p));                                           \
+        PMIX_INFO_LOAD(&_info, PMIX_OPTIONAL, NULL, PMIX_BOOL);                        \
+        (r) = PMIx_Get(&(_proc), (s), &(_info), 1, &(_kv));                            \
+        PMIX_INFO_DESTRUCT(&_info);                                                    \
+        if (NULL == _kv) {                                                             \
+            (r) = PMIX_ERR_NOT_FOUND;                                                  \
+        } else if (_kv->type != (t)) {                                                 \
+            (r) = PMIX_ERR_TYPE_MISMATCH;                                              \
+        } else if (PMIX_SUCCESS == (r) && NULL != (d)) {                               \
+            PMIX_VALUE_UNLOAD((r), _kv, (void **) (d), &_sz);                          \
+        }                                                                              \
+        if (NULL != _kv) {                                                             \
+            PMIX_VALUE_RELEASE(_kv);                                                   \
+        }                                                                              \
+    } while (0);
 
 /**
  * Provide a simplified macro for retrieving modex data
@@ -338,10 +332,9 @@ typedef struct {
         pmix_info_t _info;                                                              \
         size_t _sz;                                                                     \
         OPAL_OUTPUT_VERBOSE((1, opal_pmix_verbose_output,                               \
-                            "%s[%s:%d] MODEX RECV VALUE IMMEDIATE FOR PROC %s KEY %s",  \
-                            OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),                         \
-                            __FILE__, __LINE__,                                         \
-                            OPAL_NAME_PRINT(*(p)), (s)));                               \
+                             "%s[%s:%d] MODEX RECV VALUE IMMEDIATE FOR PROC %s KEY %s", \
+                             OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), __FILE__, __LINE__,    \
+                             OPAL_NAME_PRINT(*(p)), PMIx_Get_attribute_name(s)));       \
         OPAL_PMIX_CONVERT_NAME(&_proc, (p));                                            \
         PMIX_INFO_LOAD(&_info, PMIX_IMMEDIATE, NULL, PMIX_BOOL);                        \
         (r) = PMIx_Get(&(_proc), (s), &(_info), 1, &(_kv));                             \
@@ -351,12 +344,12 @@ typedef struct {
         } else if (_kv->type != (t)) {                                                  \
             (r) = PMIX_ERR_TYPE_MISMATCH;                                               \
         } else if (PMIX_SUCCESS == (r)) {                                               \
-            PMIX_VALUE_UNLOAD((r), _kv, (void**)(d), &_sz);                             \
+            PMIX_VALUE_UNLOAD((r), _kv, (void **) (d), &_sz);                           \
         }                                                                               \
         if (NULL != _kv) {                                                              \
             PMIX_VALUE_RELEASE(_kv);                                                    \
         }                                                                               \
-    } while(0);
+    } while (0);
 
 /**
  * Provide a simplified macro for retrieving modex data
@@ -370,29 +363,28 @@ typedef struct {
  *     is to be returned
  * t - the expected data type
  */
-#define OPAL_MODEX_RECV_VALUE(r, s, p, d, t)                                    \
-    do {                                                                        \
-        pmix_proc_t _proc;                                                      \
-        pmix_value_t *_kv = NULL;                                               \
-        size_t _sz;                                                             \
-        OPAL_OUTPUT_VERBOSE((1, opal_pmix_verbose_output,                       \
-                            "%s[%s:%d] MODEX RECV VALUE FOR PROC %s KEY %s",    \
-                            OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),                 \
-                            __FILE__, __LINE__,                                 \
-                            OPAL_NAME_PRINT(*(p)), (s)));                       \
-        OPAL_PMIX_CONVERT_NAME(&_proc, (p));                                    \
-        (r) = PMIx_Get(&(_proc), (s), NULL, 0, &(_kv));                         \
-        if (NULL == _kv) {                                                      \
-            (r) = PMIX_ERR_NOT_FOUND;                                           \
-        } else if (_kv->type != (t)) {                                          \
-            (r) = PMIX_ERR_TYPE_MISMATCH;                                       \
-        } else if (PMIX_SUCCESS == (r)) {                                       \
-            PMIX_VALUE_UNLOAD((r), _kv, (void**)(d), &_sz);                     \
-        }                                                                       \
-        if (NULL != _kv) {                                                      \
-            PMIX_VALUE_RELEASE(_kv);                                            \
-        }                                                                       \
-    } while(0);
+#define OPAL_MODEX_RECV_VALUE(r, s, p, d, t)                                                       \
+    do {                                                                                           \
+        pmix_proc_t _proc;                                                                         \
+        pmix_value_t *_kv = NULL;                                                                  \
+        size_t _sz;                                                                                \
+        OPAL_OUTPUT_VERBOSE(                                                                       \
+            (1, opal_pmix_verbose_output, "%s[%s:%d] MODEX RECV VALUE FOR PROC %s KEY %s",         \
+             OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), __FILE__, __LINE__, OPAL_NAME_PRINT(*(p)),        \
+             PMIx_Get_attribute_name(s))); \
+        OPAL_PMIX_CONVERT_NAME(&_proc, (p));                                                       \
+        (r) = PMIx_Get(&(_proc), (s), NULL, 0, &(_kv));                                            \
+        if (NULL == _kv) {                                                                         \
+            (r) = PMIX_ERR_NOT_FOUND;                                                              \
+        } else if (_kv->type != (t)) {                                                             \
+            (r) = PMIX_ERR_TYPE_MISMATCH;                                                          \
+        } else if (PMIX_SUCCESS == (r)) {                                                          \
+            PMIX_VALUE_UNLOAD((r), _kv, (void **) (d), &_sz);                                      \
+        }                                                                                          \
+        if (NULL != _kv) {                                                                         \
+            PMIX_VALUE_RELEASE(_kv);                                                               \
+        }                                                                                          \
+    } while (0);
 
 /**
  * Provide a simplified macro for retrieving modex data
@@ -407,32 +399,31 @@ typedef struct {
  * sz - pointer to a location wherein the number of bytes
  *     in the data object can be returned (size_t)
  */
-#define OPAL_MODEX_RECV_STRING_OPTIONAL(r, s, p, d, sz)                                \
-    do {                                                                               \
-        pmix_proc_t _proc;                                                             \
-        pmix_value_t *_kv = NULL;                                                      \
-        pmix_info_t _info;                                                             \
-        OPAL_OUTPUT_VERBOSE((1, opal_pmix_verbose_output,                              \
-                            "%s[%s:%d] MODEX RECV STRING OPTIONAL FOR PROC %s KEY %s", \
-                            OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),                        \
-                            __FILE__, __LINE__,                                        \
-                            OPAL_NAME_PRINT(*(p)), (s)));                              \
-        *(d) = NULL;                                                                   \
-        *(sz) = 0;                                                                     \
-        OPAL_PMIX_CONVERT_NAME(&_proc, (p));                                           \
-        PMIX_INFO_LOAD(&_info, PMIX_OPTIONAL, NULL, PMIX_BOOL);                        \
-        (r) = PMIx_Get(&(_proc), (s), &(_info), 1, &(_kv));                            \
-        if (NULL == _kv) {                                                             \
-            (r) = PMIX_ERR_NOT_FOUND;                                                  \
-        } else if (PMIX_SUCCESS == (r)) {                                              \
-            *(d) = (uint8_t*)_kv->data.bo.bytes;                                       \
-            *(sz) = _kv->data.bo.size;                                                 \
-            _kv->data.bo.bytes = NULL; /* protect the data */                          \
-        }                                                                              \
-        if (NULL != _kv) {                                                             \
-            PMIX_VALUE_RELEASE(_kv);                                                   \
-        }                                                                              \
-    } while(0);
+#define OPAL_MODEX_RECV_STRING_OPTIONAL(r, s, p, d, sz)                                 \
+    do {                                                                                \
+        pmix_proc_t _proc;                                                              \
+        pmix_value_t *_kv = NULL;                                                       \
+        pmix_info_t _info;                                                              \
+        OPAL_OUTPUT_VERBOSE((1, opal_pmix_verbose_output,                               \
+                             "%s[%s:%d] MODEX RECV STRING OPTIONAL FOR PROC %s KEY %s", \
+                             OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), __FILE__, __LINE__,    \
+                             OPAL_NAME_PRINT(*(p)), PMIx_Get_attribute_name(s)));       \
+        *(d) = NULL;                                                                    \
+        *(sz) = 0;                                                                      \
+        OPAL_PMIX_CONVERT_NAME(&_proc, (p));                                            \
+        PMIX_INFO_LOAD(&_info, PMIX_OPTIONAL, NULL, PMIX_BOOL);                         \
+        (r) = PMIx_Get(&(_proc), (s), &(_info), 1, &(_kv));                             \
+        if (NULL == _kv) {                                                              \
+            (r) = PMIX_ERR_NOT_FOUND;                                                   \
+        } else if (PMIX_SUCCESS == (r)) {                                               \
+            *(d) = (uint8_t *) _kv->data.bo.bytes;                                      \
+            *(sz) = _kv->data.bo.size;                                                  \
+            _kv->data.bo.bytes = NULL; /* protect the data */                           \
+        }                                                                               \
+        if (NULL != _kv) {                                                              \
+            PMIX_VALUE_RELEASE(_kv);                                                    \
+        }                                                                               \
+    } while (0);
 
 /**
  * Provide a simplified macro for retrieving modex data
@@ -447,33 +438,32 @@ typedef struct {
  * sz - pointer to a location wherein the number of bytes
  *     in the data object can be returned (size_t)
  */
-#define OPAL_MODEX_RECV_STRING_IMMEDIATE(r, s, p, d, sz)                        \
-    do {                                                                        \
-        pmix_proc_t _proc;                                                      \
-        pmix_value_t *_kv = NULL;                                               \
-        pmix_info_t _info;                                                      \
-        OPAL_OUTPUT_VERBOSE((1, opal_pmix_verbose_output,                       \
-                            "%s[%s:%d] MODEX RECV STRING FOR PROC %s KEY %s",   \
-                            OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),                 \
-                            __FILE__, __LINE__,                                 \
-                            OPAL_NAME_PRINT(*(p)), (s)));                       \
-        *(d) = NULL;                                                            \
-        *(sz) = 0;                                                              \
-        OPAL_PMIX_CONVERT_NAME(&_proc, (p));                                    \
-        PMIX_INFO_LOAD(&_info, PMIX_IMMEDIATE, NULL, PMIX_BOOL);                \
-        (r) = PMIx_Get(&(_proc), (s), &_info, 1, &(_kv));                       \
-        PMIX_INFO_DESTRUCT(&_info);                                             \
-        if (NULL == _kv) {                                                      \
-            (r) = PMIX_ERR_NOT_FOUND;                                           \
-        } else if (PMIX_SUCCESS == (r)) {                                       \
-            *(d) = (uint8_t*)_kv->data.bo.bytes;                                \
-            *(sz) = _kv->data.bo.size;                                          \
-            _kv->data.bo.bytes = NULL; /* protect the data */                   \
-        }                                                                       \
-        if (NULL != _kv) {                                                      \
-            PMIX_VALUE_RELEASE(_kv);                                            \
-        }                                                                       \
-    } while(0);
+#define OPAL_MODEX_RECV_STRING_IMMEDIATE(r, s, p, d, sz)                                           \
+    do {                                                                                           \
+        pmix_proc_t _proc;                                                                         \
+        pmix_value_t *_kv = NULL;                                                                  \
+        pmix_info_t _info;                                                                         \
+        OPAL_OUTPUT_VERBOSE(                                                                       \
+            (1, opal_pmix_verbose_output, "%s[%s:%d] MODEX RECV STRING FOR PROC %s KEY %s",        \
+             OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), __FILE__, __LINE__, OPAL_NAME_PRINT(*(p)),        \
+             PMIx_Get_attribute_name(s)));                                                         \
+        *(d) = NULL;                                                                               \
+        *(sz) = 0;                                                                                 \
+        OPAL_PMIX_CONVERT_NAME(&_proc, (p));                                                       \
+        PMIX_INFO_LOAD(&_info, PMIX_IMMEDIATE, NULL, PMIX_BOOL);                                   \
+        (r) = PMIx_Get(&(_proc), (s), &_info, 1, &(_kv));                                          \
+        PMIX_INFO_DESTRUCT(&_info);                                                                \
+        if (NULL == _kv) {                                                                         \
+            (r) = PMIX_ERR_NOT_FOUND;                                                              \
+        } else if (PMIX_SUCCESS == (r)) {                                                          \
+            *(d) = (uint8_t *) _kv->data.bo.bytes;                                                 \
+            *(sz) = _kv->data.bo.size;                                                             \
+            _kv->data.bo.bytes = NULL; /* protect the data */                                      \
+        }                                                                                          \
+        if (NULL != _kv) {                                                                         \
+            PMIX_VALUE_RELEASE(_kv);                                                               \
+        }                                                                                          \
+    } while (0);
 
 /**
  * Provide a simplified macro for retrieving modex data
@@ -488,30 +478,28 @@ typedef struct {
  * sz - pointer to a location wherein the number of bytes
  *     in the data object can be returned (size_t)
  */
-#define OPAL_MODEX_RECV_STRING(r, s, p, d, sz)                                  \
-    do {                                                                        \
-        pmix_proc_t _proc;                                                      \
-        pmix_value_t *_kv = NULL;                                               \
-        OPAL_OUTPUT_VERBOSE((1, opal_pmix_verbose_output,                       \
-                            "%s[%s:%d] MODEX RECV STRING FOR PROC %s KEY %s",   \
-                            OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),                 \
-                            __FILE__, __LINE__,                                 \
-                            OPAL_NAME_PRINT(*(p)), (s)));                       \
-        *(d) = NULL;                                                            \
-        *(sz) = 0;                                                              \
-        OPAL_PMIX_CONVERT_NAME(&_proc, (p));                                    \
-        (r) = PMIx_Get(&(_proc), (s), NULL, 0, &(_kv));                         \
-        if (NULL == _kv) {                                                      \
-            (r) = PMIX_ERR_NOT_FOUND;                                           \
-        } else if (PMIX_SUCCESS == (r)) {                                       \
-            *(d) = (uint8_t*)_kv->data.bo.bytes;                                \
-            *(sz) = _kv->data.bo.size;                                          \
-            _kv->data.bo.bytes = NULL; /* protect the data */                   \
-        }                                                                       \
-        if (NULL != _kv) {                                                      \
-            PMIX_VALUE_RELEASE(_kv);                                            \
-        }                                                                       \
-    } while(0);
+#define OPAL_MODEX_RECV_STRING(r, s, p, d, sz)                                                     \
+    do {                                                                                           \
+        pmix_proc_t _proc;                                                                         \
+        pmix_value_t *_kv = NULL;                                                                  \
+        OPAL_OUTPUT_VERBOSE(                                                                       \
+            (1, opal_pmix_verbose_output, "%s[%s:%d] MODEX RECV STRING FOR PROC %s KEY %s",        \
+             OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), __FILE__, __LINE__, OPAL_NAME_PRINT(*(p)), (s))); \
+        *(d) = NULL;                                                                               \
+        *(sz) = 0;                                                                                 \
+        OPAL_PMIX_CONVERT_NAME(&_proc, (p));                                                       \
+        (r) = PMIx_Get(&(_proc), (s), NULL, 0, &(_kv));                                            \
+        if (NULL == _kv) {                                                                         \
+            (r) = PMIX_ERR_NOT_FOUND;                                                              \
+        } else if (PMIX_SUCCESS == (r)) {                                                          \
+            *(d) = (uint8_t *) _kv->data.bo.bytes;                                                 \
+            *(sz) = _kv->data.bo.size;                                                             \
+            _kv->data.bo.bytes = NULL; /* protect the data */                                      \
+        }                                                                                          \
+        if (NULL != _kv) {                                                                         \
+            PMIX_VALUE_RELEASE(_kv);                                                               \
+        }                                                                                          \
+    } while (0);
 
 /**
  * Provide a simplified macro for retrieving modex data
@@ -526,24 +514,22 @@ typedef struct {
  * sz - pointer to a location wherein the number of bytes
  *     in the data object can be returned (size_t)
  */
-#define OPAL_MODEX_RECV_OPTIONAL(r, s, p, d, sz)                            \
-    do {                                                                    \
-        char *_key;                                                         \
-        _key = mca_base_component_to_string((s));                           \
-        OPAL_OUTPUT_VERBOSE((1, opal_pmix_verbose_output,                   \
-                            "%s[%s:%d] MODEX RECV FOR PROC %s KEY %s",      \
-                            OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),             \
-                            __FILE__, __LINE__,                             \
-                            OPAL_NAME_PRINT(*(p)), _key));                  \
-        if (NULL == _key) {                                                 \
-            OPAL_ERROR_LOG(OPAL_ERR_OUT_OF_RESOURCE);                       \
-            (r) = OPAL_ERR_OUT_OF_RESOURCE;                                 \
-        } else {                                                            \
-            OPAL_MODEX_RECV_STRING_OPTIONAL((r), _key, (p), (d), (sz));     \
-            free(_key);                                                     \
-        }                                                                   \
-    } while(0);
-
+#define OPAL_MODEX_RECV_OPTIONAL(r, s, p, d, sz)                                     \
+    do {                                                                             \
+        char *_key;                                                                  \
+        _key = mca_base_component_to_string((s));                                    \
+        OPAL_OUTPUT_VERBOSE((1, opal_pmix_verbose_output,                            \
+                             "%s[%s:%d] MODEX RECV FOR PROC %s KEY %s",              \
+                             OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), __FILE__, __LINE__, \
+                             OPAL_NAME_PRINT(*(p)), _key));                          \
+        if (NULL == _key) {                                                          \
+            OPAL_ERROR_LOG(OPAL_ERR_OUT_OF_RESOURCE);                                \
+            (r) = OPAL_ERR_OUT_OF_RESOURCE;                                          \
+        } else {                                                                     \
+            OPAL_MODEX_RECV_STRING_OPTIONAL((r), _key, (p), (d), (sz));              \
+            free(_key);                                                              \
+        }                                                                            \
+    } while (0);
 
 /**
  * Provide a simplified macro for retrieving modex data
@@ -558,24 +544,22 @@ typedef struct {
  * sz - pointer to a location wherein the number of bytes
  *     in the data object can be returned (size_t)
  */
-#define OPAL_MODEX_RECV_IMMEDIATE(r, s, p, d, sz)                           \
-    do {                                                                    \
-        char *_key;                                                         \
-        _key = mca_base_component_to_string((s));                           \
-        OPAL_OUTPUT_VERBOSE((1, opal_pmix_verbose_output,                   \
-                            "%s[%s:%d] MODEX RECV FOR PROC %s KEY %s",      \
-                            OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),             \
-                            __FILE__, __LINE__,                             \
-                            OPAL_NAME_PRINT(*(p)), _key));                  \
-        if (NULL == _key) {                                                 \
-            OPAL_ERROR_LOG(OPAL_ERR_OUT_OF_RESOURCE);                       \
-            (r) = OPAL_ERR_OUT_OF_RESOURCE;                                 \
-        } else {                                                            \
-            OPAL_MODEX_RECV_STRING_IMMEDIATE((r), _key, (p), (d), (sz));    \
-            free(_key);                                                     \
-        }                                                                   \
-    } while(0);
-
+#define OPAL_MODEX_RECV_IMMEDIATE(r, s, p, d, sz)                                    \
+    do {                                                                             \
+        char *_key;                                                                  \
+        _key = mca_base_component_to_string((s));                                    \
+        OPAL_OUTPUT_VERBOSE((1, opal_pmix_verbose_output,                            \
+                             "%s[%s:%d] MODEX RECV FOR PROC %s KEY %s",              \
+                             OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), __FILE__, __LINE__, \
+                             OPAL_NAME_PRINT(*(p)), _key));                          \
+        if (NULL == _key) {                                                          \
+            OPAL_ERROR_LOG(OPAL_ERR_OUT_OF_RESOURCE);                                \
+            (r) = OPAL_ERR_OUT_OF_RESOURCE;                                          \
+        } else {                                                                     \
+            OPAL_MODEX_RECV_STRING_IMMEDIATE((r), _key, (p), (d), (sz));             \
+            free(_key);                                                              \
+        }                                                                            \
+    } while (0);
 
 /**
  * Provide a simplified macro for retrieving modex data
@@ -590,29 +574,27 @@ typedef struct {
  * sz - pointer to a location wherein the number of bytes
  *     in the data object can be returned (size_t)
  */
-#define OPAL_MODEX_RECV(r, s, p, d, sz)                                 \
-    do {                                                                \
-        char *_key;                                                     \
-        _key = mca_base_component_to_string((s));                       \
-        OPAL_OUTPUT_VERBOSE((1, opal_pmix_verbose_output,               \
-                            "%s[%s:%d] MODEX RECV FOR PROC %s KEY %s",  \
-                            OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),         \
-                            __FILE__, __LINE__,                         \
-                            OPAL_NAME_PRINT(*(p)), _key));              \
-        if (NULL == _key) {                                             \
-            OPAL_ERROR_LOG(OPAL_ERR_OUT_OF_RESOURCE);                   \
-            (r) = OPAL_ERR_OUT_OF_RESOURCE;                             \
-        } else {                                                        \
-            OPAL_MODEX_RECV_STRING((r), _key, (p), (d), (sz));          \
-            free(_key);                                                 \
-        }                                                               \
-    } while(0);
-
-
-#define PMIX_ERROR_LOG(r)          \
+#define OPAL_MODEX_RECV(r, s, p, d, sz)                                              \
+    do {                                                                             \
+        char *_key;                                                                  \
+        _key = mca_base_component_to_string((s));                                    \
+        OPAL_OUTPUT_VERBOSE((1, opal_pmix_verbose_output,                            \
+                             "%s[%s:%d] MODEX RECV FOR PROC %s KEY %s",              \
+                             OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), __FILE__, __LINE__, \
+                             OPAL_NAME_PRINT(*(p)), _key));                          \
+        if (NULL == _key) {                                                          \
+            OPAL_ERROR_LOG(OPAL_ERR_OUT_OF_RESOURCE);                                \
+            (r) = OPAL_ERR_OUT_OF_RESOURCE;                                          \
+        } else {                                                                     \
+            OPAL_MODEX_RECV_STRING((r), _key, (p), (d), (sz));                       \
+            free(_key);                                                              \
+        }                                                                            \
+    } while (0);
+
+#define PMIX_ERROR_LOG(r) \
     opal_output(0, "[%s:%d] PMIx Error: %s", __FILE__, __LINE__, PMIx_Error_string((r)))
 
-#define OPAL_PMIX_SHOW_HELP    "opal.show.help"
+#define OPAL_PMIX_SHOW_HELP "opal.show.help"
 
 /* some helper functions */
 OPAL_DECLSPEC pmix_proc_state_t opal_pmix_convert_state(int state);
@@ -624,121 +606,115 @@ OPAL_DECLSPEC int opal_pmix_convert_nspace(opal_jobid_t *jobid, pmix_nspace_t ns
 OPAL_DECLSPEC void opal_pmix_setup_nspace_tracker(void);
 OPAL_DECLSPEC void opal_pmix_finalize_nspace_tracker(void);
 
-#define OPAL_SCHEMA_DELIMITER_CHAR      '.'
-#define OPAL_SCHEMA_WILDCARD_CHAR       '*'
-#define OPAL_SCHEMA_WILDCARD_STRING     "*"
-#define OPAL_SCHEMA_INVALID_CHAR        '$'
-#define OPAL_SCHEMA_INVALID_STRING      "$"
+#define OPAL_SCHEMA_DELIMITER_CHAR  '.'
+#define OPAL_SCHEMA_WILDCARD_CHAR   '*'
+#define OPAL_SCHEMA_WILDCARD_STRING "*"
+#define OPAL_SCHEMA_INVALID_CHAR    '$'
+#define OPAL_SCHEMA_INVALID_STRING  "$"
 
 /* convert jobid to nspace */
-#define OPAL_PMIX_CONVERT_JOBID(n, j) \
-    opal_pmix_convert_jobid((n), (j))
+#define OPAL_PMIX_CONVERT_JOBID(n, j) opal_pmix_convert_jobid((n), (j))
 
 /* convert vpid to rank */
-#define OPAL_PMIX_CONVERT_VPID(r, v)                \
-    do {                                            \
-        if (OPAL_VPID_WILDCARD == (v)) {            \
-            (r) = PMIX_RANK_WILDCARD;               \
-        } else if (OPAL_VPID_INVALID == (v)) {      \
-            (r) = PMIX_RANK_INVALID;                \
-        } else {                                    \
-            (r) = (v);                              \
-        }                                           \
-    } while(0)
+#define OPAL_PMIX_CONVERT_VPID(r, v)           \
+    do {                                       \
+        if (OPAL_VPID_WILDCARD == (v)) {       \
+            (r) = PMIX_RANK_WILDCARD;          \
+        } else if (OPAL_VPID_INVALID == (v)) { \
+            (r) = PMIX_RANK_INVALID;           \
+        } else {                               \
+            (r) = (v);                         \
+        }                                      \
+    } while (0)
 
 /* convert opal_process_name_t to pmix_proc_t */
-#define OPAL_PMIX_CONVERT_NAME(p, n)                        \
-    do {                                                    \
-        OPAL_PMIX_CONVERT_JOBID((p)->nspace, (n)->jobid);   \
-        OPAL_PMIX_CONVERT_VPID((p)->rank, (n)->vpid);       \
-    } while(0)
-
+#define OPAL_PMIX_CONVERT_NAME(p, n)                      \
+    do {                                                  \
+        OPAL_PMIX_CONVERT_JOBID((p)->nspace, (n)->jobid); \
+        OPAL_PMIX_CONVERT_VPID((p)->rank, (n)->vpid);     \
+    } while (0)
 
 /* convert nspace to jobid */
-#define OPAL_PMIX_CONVERT_NSPACE(r, j, n)       \
-    (r) = opal_pmix_convert_nspace((j), (n))
+#define OPAL_PMIX_CONVERT_NSPACE(r, j, n) (r) = opal_pmix_convert_nspace((j), (n))
 
 /* convert pmix rank to opal vpid */
-#define OPAL_PMIX_CONVERT_RANK(v, r)            \
-    do {                                        \
-        if (PMIX_RANK_WILDCARD == (r)) {        \
-            (v) = OPAL_VPID_WILDCARD;           \
-        } else if (PMIX_RANK_INVALID == (r)) {  \
-            (v) = OPAL_VPID_INVALID;            \
-        } else {                                \
-            (v) = (r);                          \
-        }                                       \
-    } while(0)
+#define OPAL_PMIX_CONVERT_RANK(v, r)           \
+    do {                                       \
+        if (PMIX_RANK_WILDCARD == (r)) {       \
+            (v) = OPAL_VPID_WILDCARD;          \
+        } else if (PMIX_RANK_INVALID == (r)) { \
+            (v) = OPAL_VPID_INVALID;           \
+        } else {                               \
+            (v) = (r);                         \
+        }                                      \
+    } while (0)
 
 /* convert pmix_proc_t to opal_process_name_t */
-#define OPAL_PMIX_CONVERT_PROCT(r, n, p)                            \
-    do {                                                            \
-        OPAL_PMIX_CONVERT_NSPACE((r), &(n)->jobid, (p)->nspace);    \
-        if (OPAL_SUCCESS == (r)) {                                  \
-            OPAL_PMIX_CONVERT_RANK((n)->vpid, (p)->rank);           \
-        }                                                           \
-    } while(0)
+#define OPAL_PMIX_CONVERT_PROCT(r, n, p)                         \
+    do {                                                         \
+        OPAL_PMIX_CONVERT_NSPACE((r), &(n)->jobid, (p)->nspace); \
+        if (OPAL_SUCCESS == (r)) {                               \
+            OPAL_PMIX_CONVERT_RANK((n)->vpid, (p)->rank);        \
+        }                                                        \
+    } while (0)
 
 #define OPAL_PMIX_CONVERT_PROCT_TO_STRING(s, p)                         \
     do {                                                                \
         if (PMIX_RANK_WILDCARD == (p)->rank) {                          \
-            (void)opal_asprintf((s), "%s.*", (p)->nspace);              \
+            (void) opal_asprintf((s), "%s.*", (p)->nspace);             \
         } else if (PMIX_RANK_INVALID == (p)->rank) {                    \
-            (void)opal_asprintf((s), "%s.$", (p)->nspace);              \
+            (void) opal_asprintf((s), "%s.$", (p)->nspace);             \
         } else {                                                        \
-            (void)opal_asprintf((s), "%s.%u", (p)->nspace, (p)->rank);  \
+            (void) opal_asprintf((s), "%s.%u", (p)->nspace, (p)->rank); \
         }                                                               \
-    } while(0)
-
-#define OPAL_PMIX_CONVERT_STRING_TO_PROCT(p, s)     \
-    do {                                            \
-        char *_ptr;                                 \
-        _ptr = strrchr((s), '.');                   \
-        *_ptr = '\0';                               \
-        _ptr++;                                     \
-        PMIX_LOAD_NSPACE((p)->nspace, (s));         \
-        if ('*' == *_ptr) {                         \
-            (p)->rank = PMIX_RANK_WILDCARD;         \
-        } else if ('$' == *_ptr) {                  \
-            (p)->rank = PMIX_RANK_INVALID;          \
-        } else {                                    \
-            (p)->rank = strtoul(_ptr, NULL, 10);    \
-        }                                           \
-    } while(0)
-
-OPAL_DECLSPEC int opal_pmix_register_cleanup(char *path,
-                                             bool directory,
-                                             bool ignore,
+    } while (0)
+
+#define OPAL_PMIX_CONVERT_STRING_TO_PROCT(p, s)  \
+    do {                                         \
+        char *_ptr;                              \
+        _ptr = strrchr((s), '.');                \
+        *_ptr = '\0';                            \
+        _ptr++;                                  \
+        PMIX_LOAD_NSPACE((p)->nspace, (s));      \
+        if ('*' == *_ptr) {                      \
+            (p)->rank = PMIX_RANK_WILDCARD;      \
+        } else if ('$' == *_ptr) {               \
+            (p)->rank = PMIX_RANK_INVALID;       \
+        } else {                                 \
+            (p)->rank = strtoul(_ptr, NULL, 10); \
+        }                                        \
+    } while (0)
+
+OPAL_DECLSPEC int opal_pmix_register_cleanup(char *path, bool directory, bool ignore,
                                              bool jobscope);
 
 /* protect against early versions of PMIx */
 #if PMIX_VERSION_MAJOR == 3
-#if   PMIX_VERSION_MINOR == 0
-#if     PMIX_VERSION_RELEASE == 0
-#define   PMIX_NUMERIC_VERSION 0x00030000
-#define   PMIX_OPERATION_SUCCEEDED (PMIX_ERR_OP_BASE - 27)
-#elif   PMIX_VERSION_RELEASE == 2
-#undef    PMIX_NUMERIC_VERSION
-#define   PMIX_NUMERIC_VERSION 0x00030002
-#endif
-#define PMIX_LOAD_KEY(a, b) \
-    do {                                            \
-        memset((a), 0, PMIX_MAX_KEYLEN+1);          \
-        pmix_strncpy((a), (b), PMIX_MAX_KEYLEN);    \
-    }while(0)
-#if     PMIX_VERSION_RELEASE < 2
-#define PMIX_CHECK_KEY(a, b) \
-    (0 == strncmp((a)->key, (b), PMIX_MAX_KEYLEN))
-#endif
-#elif PMIX_VERSION_MINOR == 1
-#if     PMIX_VERSION_RELEASE == 1
-#undef    PMIX_NUMERIC_VERSION
-#define   PMIX_NUMERIC_VERSION 0x00030101
-#elif   PMIX_VERSION_RELEASE == 2
-#undef    PMIX_NUMERIC_VERSION
-#define   PMIX_NUMERIC_VERSION 0x00030102
-#endif
-#endif
+#    if PMIX_VERSION_MINOR == 0
+#        if PMIX_VERSION_RELEASE == 0
+#            define PMIX_NUMERIC_VERSION     0x00030000
+#            define PMIX_OPERATION_SUCCEEDED (PMIX_ERR_OP_BASE - 27)
+#        elif PMIX_VERSION_RELEASE == 2
+#            undef PMIX_NUMERIC_VERSION
+#            define PMIX_NUMERIC_VERSION 0x00030002
+#        endif
+#        define PMIX_LOAD_KEY(a, b)                      \
+            do {                                         \
+                memset((a), 0, PMIX_MAX_KEYLEN + 1);     \
+                pmix_strncpy((a), (b), PMIX_MAX_KEYLEN); \
+            } while (0)
+#        if PMIX_VERSION_RELEASE < 2
+#            define PMIX_CHECK_KEY(a, b) (0 == strncmp((a)->key, (b), PMIX_MAX_KEYLEN))
+#        endif
+#    elif PMIX_VERSION_MINOR == 1
+#        if PMIX_VERSION_RELEASE == 1
+#            undef PMIX_NUMERIC_VERSION
+#            define PMIX_NUMERIC_VERSION 0x00030101
+#        elif PMIX_VERSION_RELEASE == 2
+#            undef PMIX_NUMERIC_VERSION
+#            define PMIX_NUMERIC_VERSION 0x00030102
+#        endif
+#    endif
 #endif
 
 /**
@@ -760,9 +736,7 @@ typedef struct opal_pmix_base_component_2_0_0_t opal_pmix_component_t;
 /**
  * Macro for use in components that are of type hwloc
  */
-#define OPAL_PMIX_BASE_VERSION_2_0_0 \
-    OPAL_MCA_BASE_VERSION_2_1_0("pmix", 2, 0, 0)
-
+#define OPAL_PMIX_BASE_VERSION_2_0_0 OPAL_MCA_BASE_VERSION_2_1_0("pmix", 2, 0, 0)
 
 END_C_DECLS
 
diff --git a/opal/mca/rcache/base/Makefile.am b/opal/mca/rcache/base/Makefile.am
index bd27121c93c..61df57f3f21 100644
--- a/opal/mca/rcache/base/Makefile.am
+++ b/opal/mca/rcache/base/Makefile.am
@@ -11,6 +11,7 @@
 #                         All rights reserved.
 # Copyright (c) 2013-2016 Los Alamos National Security, LLC. All rights
 #                         reserved
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -32,5 +33,5 @@ libmca_rcache_la_SOURCES += \
 	base/rcache_base_vma_tree.c \
 	base/rcache_base_mem_cb.c
 
-dist_opaldata_DATA = \
+EXTRA_DIST = \
         base/help-rcache-base.txt
diff --git a/opal/mca/rcache/base/base.h b/opal/mca/rcache/base/base.h
index 37526814a46..9288f454070 100644
--- a/opal/mca/rcache/base/base.h
+++ b/opal/mca/rcache/base/base.h
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2006 The University of Tennessee and The University
+ * Copyright (c) 2004-2024 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -28,16 +28,17 @@
 
 #include "opal/class/opal_list.h"
 #include "opal/mca/mca.h"
-#include "opal/mca/rcache/rcache.h"
 #include "opal/mca/memory/base/base.h"
+#include "opal/mca/rcache/rcache.h"
 
 BEGIN_C_DECLS
 
 /*
  * create a module by name
  */
-OPAL_DECLSPEC mca_rcache_base_module_t *mca_rcache_base_module_create (const char *name, void *user_data,
-                                                                       mca_rcache_base_resources_t *rcache_resources);
+OPAL_DECLSPEC mca_rcache_base_module_t *
+mca_rcache_base_module_create(const char *name, void *user_data,
+                              mca_rcache_base_resources_t *rcache_resources);
 
 /*
  * MCA framework
@@ -55,7 +56,7 @@ typedef struct mca_rcache_base_selected_module_t mca_rcache_base_selected_module
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(mca_rcache_base_selected_module_t);
 
 OPAL_DECLSPEC mca_rcache_base_component_t *mca_rcache_base_component_lookup(const char *name);
-OPAL_DECLSPEC mca_rcache_base_module_t *mca_rcache_base_module_lookup (const char *name);
+OPAL_DECLSPEC mca_rcache_base_module_t *mca_rcache_base_module_lookup(const char *name);
 OPAL_DECLSPEC int mca_rcache_base_module_destroy(mca_rcache_base_module_t *module);
 
 extern opal_free_list_t mca_rcache_base_vma_tree_items;
@@ -72,6 +73,10 @@ extern int mca_rcache_base_used_mem_hooks;
  */
 OPAL_DECLSPEC extern opal_list_t mca_rcache_base_modules;
 
+OPAL_DECLSPEC void mca_rcache_base_module_init(mca_rcache_base_module_t *rcache);
+
+OPAL_DECLSPEC void mca_rcache_base_module_fini(mca_rcache_base_module_t *rcache);
+
 END_C_DECLS
 
 #endif /* MCA_RCACHE_BASE_H */
diff --git a/opal/mca/rcache/base/rcache_base_create.c b/opal/mca/rcache/base/rcache_base_create.c
index df74b7784c9..61d824be311 100644
--- a/opal/mca/rcache/base/rcache_base_create.c
+++ b/opal/mca/rcache/base/rcache_base_create.c
@@ -13,6 +13,7 @@
  * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2020      Intel, Inc.  All rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -25,23 +26,24 @@
 #include <stdio.h>
 #include <string.h>
 
-#include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
-#include "opal/mca/rcache/rcache.h"
+#include "opal/mca/mca.h"
 #include "opal/mca/rcache/base/base.h"
 #include "opal/mca/rcache/base/rcache_base_mem_cb.h"
-#include "rcache_base_vma_tree.h"
-#include "opal/util/show_help.h"
+#include "opal/mca/rcache/rcache.h"
 #include "opal/util/proc.h"
+#include "opal/util/show_help.h"
+#include "rcache_base_vma_tree.h"
 
-#include "opal/runtime/opal_params.h"
 #include "opal/memoryhooks/memory.h"
+#include "opal/runtime/opal_params.h"
 
-mca_rcache_base_module_t* mca_rcache_base_module_create (const char* name, void *user_data,
-                                                         struct mca_rcache_base_resources_t* resources)
+mca_rcache_base_module_t *
+mca_rcache_base_module_create(const char *name, void *user_data,
+                              struct mca_rcache_base_resources_t *resources)
 {
-    mca_rcache_base_component_t* component = NULL;
-    mca_rcache_base_module_t* module = NULL;
+    mca_rcache_base_component_t *component = NULL;
+    mca_rcache_base_module_t *module = NULL;
     mca_base_component_list_item_t *cli;
     mca_rcache_base_selected_module_t *sm;
 
@@ -60,19 +62,18 @@ mca_rcache_base_module_t* mca_rcache_base_module_create (const char* name, void
                triggered before this (any time after mem_free_init(),
                actually).  This is a hook available for memory manager hooks
                without good initialization routine support */
-            (void) mca_base_framework_open (&opal_memory_base_framework, 0);
+            (void) mca_base_framework_open(&opal_memory_base_framework, 0);
 
-            if ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) ==
-                ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) &
-                 opal_mem_hooks_support_level())) {
+            if ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT)
+                == ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT)
+                    & opal_mem_hooks_support_level())) {
                 if (-1 == opal_leave_pinned) {
                     opal_leave_pinned = !opal_leave_pinned_pipeline;
                 }
                 opal_mem_hooks_register_release(mca_rcache_base_mem_cb, NULL);
             } else if (1 == opal_leave_pinned || opal_leave_pinned_pipeline) {
-                opal_show_help("help-rcache-base.txt", "leave pinned failed",
-                               true, name, OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),
-                               opal_process_info.nodename);
+                opal_show_help("help-rcache-base.txt", "leave pinned failed", true, name,
+                               OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), opal_process_info.nodename);
                 return NULL;
             }
 
@@ -82,15 +83,16 @@ mca_rcache_base_module_t* mca_rcache_base_module_create (const char* name, void
         }
     }
 
-    OPAL_LIST_FOREACH(cli, &opal_rcache_base_framework.framework_components, mca_base_component_list_item_t) {
-         component = (mca_rcache_base_component_t *) cli->cli_component;
-         if(0 == strcmp(component->rcache_version.mca_component_name, name)) {
-             module = component->rcache_init (resources);
-             break;
-         }
+    OPAL_LIST_FOREACH (cli, &opal_rcache_base_framework.framework_components,
+                       mca_base_component_list_item_t) {
+        component = (mca_rcache_base_component_t *) cli->cli_component;
+        if (0 == strcmp(component->rcache_version.mca_component_name, name)) {
+            module = component->rcache_init(resources);
+            break;
+        }
     }
 
-    if ( NULL == module ) {
+    if (NULL == module) {
         return NULL;
     }
 
@@ -98,7 +100,7 @@ mca_rcache_base_module_t* mca_rcache_base_module_create (const char* name, void
     sm->rcache_component = component;
     sm->rcache_module = module;
     sm->user_data = user_data;
-    opal_list_append(&mca_rcache_base_modules, (opal_list_item_t*) sm);
+    opal_list_append(&mca_rcache_base_modules, (opal_list_item_t *) sm);
 
     return module;
 }
@@ -107,9 +109,9 @@ int mca_rcache_base_module_destroy(mca_rcache_base_module_t *module)
 {
     mca_rcache_base_selected_module_t *sm, *next;
 
-    OPAL_LIST_FOREACH_SAFE(sm, next, &mca_rcache_base_modules, mca_rcache_base_selected_module_t) {
+    OPAL_LIST_FOREACH_SAFE (sm, next, &mca_rcache_base_modules, mca_rcache_base_selected_module_t) {
         if (module == sm->rcache_module) {
-            opal_list_remove_item(&mca_rcache_base_modules, (opal_list_item_t*)sm);
+            opal_list_remove_item(&mca_rcache_base_modules, (opal_list_item_t *) sm);
             if (NULL != sm->rcache_module->rcache_finalize) {
                 sm->rcache_module->rcache_finalize(sm->rcache_module);
             }
@@ -120,3 +122,8 @@ int mca_rcache_base_module_destroy(mca_rcache_base_module_t *module)
 
     return OPAL_ERR_NOT_FOUND;
 }
+
+static void safety_valve(void) __opal_attribute_destructor__;
+void safety_valve(void) {
+    opal_mem_hooks_unregister_release(mca_rcache_base_mem_cb);
+}
diff --git a/opal/mca/rcache/base/rcache_base_frame.c b/opal/mca/rcache/base/rcache_base_frame.c
index fd0c2832172..4bd035ead85 100644
--- a/opal/mca/rcache/base/rcache_base_frame.c
+++ b/opal/mca/rcache/base/rcache_base_frame.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2024 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -22,22 +22,21 @@
  * $HEADER$
  */
 
-
 #include "opal_config.h"
 #include <stdio.h>
 
-#include "opal/mca/mca.h"
+#include "opal/constants.h"
 #include "opal/mca/base/base.h"
 #include "opal/mca/base/mca_base_pvar.h"
-#include "opal/mca/rcache/rcache.h"
+#include "opal/mca/mca.h"
 #include "opal/mca/rcache/base/base.h"
+#include "opal/mca/rcache/rcache.h"
 #include "opal/memoryhooks/memory.h"
-#include "opal/constants.h"
 #include "rcache_base_mem_cb.h"
 
 /* two-level macro for stringifying a number */
 #define STRINGIFYX(x) #x
-#define STRINGIFY(x) STRINGIFYX(x)
+#define STRINGIFY(x)  STRINGIFYX(x)
 
 /*
  * The following file was created by configure.  It contains extern
@@ -53,7 +52,7 @@ int mca_rcache_base_used_mem_hooks = 0;
  * Memory Pool Registration
  */
 
-static void mca_rcache_base_registration_constructor( mca_rcache_base_registration_t * reg )
+static void mca_rcache_base_registration_constructor(mca_rcache_base_registration_t *reg)
 {
     reg->rcache = NULL;
     reg->base = NULL;
@@ -86,7 +85,7 @@ static int mca_rcache_base_close(void)
     while (NULL != (item = opal_list_remove_first(&mca_rcache_base_modules))) {
         sm = (mca_rcache_base_selected_module_t *) item;
 
-        /* Blatently ignore the return code (what would we do to recover,
+        /* Blatantly ignore the return code (what would we do to recover,
            anyway?  This component is going away, so errors don't matter
            anymore).  Note that it's legal for the module to have NULL for
            the finalize function. */
@@ -106,7 +105,7 @@ static int mca_rcache_base_close(void)
            opal_mem_free_finalize(), and callbacks from the memory manager
            hooks to the bowels of the mem_free code can still occur any
            time between now and end of application (even post main()!) */
-        (void) mca_base_framework_close (&opal_memory_base_framework);
+        (void) mca_base_framework_close(&opal_memory_base_framework);
     }
 
     /* All done */
@@ -125,17 +124,24 @@ static int mca_rcache_base_open(mca_base_open_flag_t flags)
 
     OBJ_CONSTRUCT(&mca_rcache_base_modules, opal_list_t);
 
-     /* Open up all available components */
+    /* Open up all available components */
     return mca_base_framework_components_open(&opal_rcache_base_framework, flags);
 }
 
-static int mca_rcache_base_register_mca_variables (mca_base_register_flag_t flags)
+static int mca_rcache_base_register_mca_variables(mca_base_register_flag_t flags)
 {
     return OPAL_SUCCESS;
 }
 
 MCA_BASE_FRAMEWORK_DECLARE(opal, rcache, "OPAL Registration Cache",
-                           mca_rcache_base_register_mca_variables,
-                           mca_rcache_base_open, mca_rcache_base_close,
-                           mca_rcache_base_static_components, 0);
+                           mca_rcache_base_register_mca_variables, mca_rcache_base_open,
+                           mca_rcache_base_close, mca_rcache_base_static_components, 0);
+
 
+void mca_rcache_base_module_init(mca_rcache_base_module_t *rcache) {
+    OBJ_CONSTRUCT(&rcache->lock, opal_mutex_t);
+}
+
+void mca_rcache_base_module_fini(mca_rcache_base_module_t *rcache) {
+    OBJ_DESTRUCT(&rcache->lock);
+}
diff --git a/opal/mca/rcache/base/rcache_base_mem_cb.c b/opal/mca/rcache/base/rcache_base_mem_cb.c
index 8c164d00e6f..f77037fc43e 100644
--- a/opal/mca/rcache/base/rcache_base_mem_cb.c
+++ b/opal/mca/rcache/base/rcache_base_mem_cb.c
@@ -26,31 +26,31 @@
 #include "opal_config.h"
 
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 
-#include "opal/util/show_help.h"
-#include "opal/util/proc.h"
 #include "opal/runtime/opal_params.h"
+#include "opal/util/output.h"
+#include "opal/util/proc.h"
+#include "opal/util/show_help.h"
 
-#include "opal/mca/rcache/base/rcache_base_mem_cb.h"
 #include "opal/mca/rcache/base/base.h"
+#include "opal/mca/rcache/base/rcache_base_mem_cb.h"
 
 #include "opal/mca/mca.h"
 #include "opal/memoryhooks/memory.h"
 
 static char msg[512];
 
-
 /*
  *  memory hook callback, called when memory is free'd out from under
  *  us.  Be wary of the from_alloc flag -- if you're called with
  *  from_alloc==true, then you cannot call malloc (or any of its
  *  friends)!
  */
-void mca_rcache_base_mem_cb (void* base, size_t size, void* cbdata, bool from_alloc)
+void mca_rcache_base_mem_cb(void *base, size_t size, void *cbdata, bool from_alloc)
 {
-    mca_rcache_base_selected_module_t* current;
+    mca_rcache_base_selected_module_t *current;
     int rc;
 
     /* Only do anything meaningful if the OPAL layer is up and running
@@ -59,25 +59,25 @@ void mca_rcache_base_mem_cb (void* base, size_t size, void* cbdata, bool from_al
         return;
     }
 
-    OPAL_LIST_FOREACH(current, &mca_rcache_base_modules, mca_rcache_base_selected_module_t) {
+    OPAL_LIST_FOREACH (current, &mca_rcache_base_modules, mca_rcache_base_selected_module_t) {
         if (current->rcache_module->rcache_invalidate_range != NULL) {
-            rc = current->rcache_module->rcache_invalidate_range (current->rcache_module,
-                                                                  base, size);
+            rc = current->rcache_module->rcache_invalidate_range(current->rcache_module, base,
+                                                                 size);
             if (rc != OPAL_SUCCESS) {
                 if (from_alloc) {
                     int len;
-                    len = snprintf(msg, sizeof(msg), "[%s:%05d] Attempt to free memory that is still in "
-                                   "use by an ongoing MPI communication (buffer %p, size %lu).  MPI job "
-                                   "will now abort.\n", opal_process_info.nodename,
-                                   getpid(), base, (unsigned long) size);
+                    len = snprintf(
+                        msg, sizeof(msg),
+                        "[%s:%05d] Attempt to free memory that is still in "
+                        "use by an ongoing MPI communication (buffer %p, size %lu).  MPI job "
+                        "will now abort.\n",
+                        opal_process_info.nodename, getpid(), base, (unsigned long) size);
                     msg[sizeof(msg) - 1] = '\0';
-                    write(2, msg, len);
+                    opal_best_effort_write(2, msg, len);
                 } else {
-                    opal_show_help("help-rcache-base.txt",
-                                   "cannot deregister in-use memory", true,
+                    opal_show_help("help-rcache-base.txt", "cannot deregister in-use memory", true,
                                    current->rcache_component->rcache_version.mca_component_name,
-                                   opal_process_info.nodename,
-                                   base, (unsigned long) size);
+                                   opal_process_info.nodename, base, (unsigned long) size);
                 }
 
                 /* We're in a callback from somewhere; we can't do
diff --git a/opal/mca/rcache/base/rcache_base_mem_cb.h b/opal/mca/rcache/base/rcache_base_mem_cb.h
index 3192aeecb4b..db8dfeaf13f 100644
--- a/opal/mca/rcache/base/rcache_base_mem_cb.h
+++ b/opal/mca/rcache/base/rcache_base_mem_cb.h
@@ -31,7 +31,7 @@ BEGIN_C_DECLS
 /*
  *  memory hook callback, called when memory is free'd out from under us
  */
-void mca_rcache_base_mem_cb (void* base, size_t size, void* cbdata, bool from_alloc);
+void mca_rcache_base_mem_cb(void *base, size_t size, void *cbdata, bool from_alloc);
 
 END_C_DECLS
 
diff --git a/opal/mca/rcache/base/rcache_base_vma.c b/opal/mca/rcache/base/rcache_base_vma.c
index c31e5fd3274..d45e6241ec2 100644
--- a/opal/mca/rcache/base/rcache_base_vma.c
+++ b/opal/mca/rcache/base/rcache_base_vma.c
@@ -28,36 +28,37 @@
 
 #include MCA_memory_IMPLEMENTATION_HEADER
 #include "opal/mca/memory/memory.h"
+#include "opal/mca/rcache/base/base.h"
 #include "opal/mca/rcache/rcache.h"
 #include "rcache_base_vma.h"
 #include "rcache_base_vma_tree.h"
-#include "opal/mca/rcache/base/base.h"
 
 /**
  * Initialize the rcache
  */
 
-static void mca_rcache_base_vma_module_construct (mca_rcache_base_vma_module_t *vma_module) {
+static void mca_rcache_base_vma_module_construct(mca_rcache_base_vma_module_t *vma_module)
+{
     OBJ_CONSTRUCT(&vma_module->vma_lock, opal_recursive_mutex_t);
-    (void) mca_rcache_base_vma_tree_init (vma_module);
+    (void) mca_rcache_base_vma_tree_init(vma_module);
 }
 
-static void mca_rcache_base_vma_module_destruct (mca_rcache_base_vma_module_t *vma_module) {
+static void mca_rcache_base_vma_module_destruct(mca_rcache_base_vma_module_t *vma_module)
+{
     OBJ_DESTRUCT(&vma_module->vma_lock);
-    mca_rcache_base_vma_tree_finalize (vma_module);
+    mca_rcache_base_vma_tree_finalize(vma_module);
 }
 
 OBJ_CLASS_INSTANCE(mca_rcache_base_vma_module_t, opal_object_t,
-                   mca_rcache_base_vma_module_construct,
-                   mca_rcache_base_vma_module_destruct);
+                   mca_rcache_base_vma_module_construct, mca_rcache_base_vma_module_destruct);
 
-mca_rcache_base_vma_module_t *mca_rcache_base_vma_module_alloc (void)
+mca_rcache_base_vma_module_t *mca_rcache_base_vma_module_alloc(void)
 {
     return OBJ_NEW(mca_rcache_base_vma_module_t);
 }
 
-int mca_rcache_base_vma_find (mca_rcache_base_vma_module_t *vma_module, void *addr,
-                              size_t size, mca_rcache_base_registration_t **reg)
+int mca_rcache_base_vma_find(mca_rcache_base_vma_module_t *vma_module, void *addr, size_t size,
+                             mca_rcache_base_registration_t **reg)
 {
     int rc;
     unsigned char *bound_addr;
@@ -69,43 +70,40 @@ int mca_rcache_base_vma_find (mca_rcache_base_vma_module_t *vma_module, void *ad
     bound_addr = (unsigned char *) ((intptr_t) addr + size - 1);
 
     /* Check to ensure that the cache is valid */
-    if (OPAL_UNLIKELY(opal_memory_changed() &&
-                      NULL != opal_memory->memoryc_process &&
-                      OPAL_SUCCESS != (rc = opal_memory->memoryc_process()))) {
+    if (OPAL_UNLIKELY(opal_memory_changed() && NULL != opal_memory->memoryc_process
+                      && OPAL_SUCCESS != (rc = opal_memory->memoryc_process()))) {
         return rc;
     }
 
-    *reg = mca_rcache_base_vma_tree_find (vma_module, (unsigned char *) addr, bound_addr);
+    *reg = mca_rcache_base_vma_tree_find(vma_module, (unsigned char *) addr, bound_addr);
 
     return OPAL_SUCCESS;
 }
 
-int mca_rcache_base_vma_find_all (mca_rcache_base_vma_module_t *vma_module, void *addr,
-                                  size_t size, mca_rcache_base_registration_t **regs,
-                                  int reg_cnt)
+int mca_rcache_base_vma_find_all(mca_rcache_base_vma_module_t *vma_module, void *addr, size_t size,
+                                 mca_rcache_base_registration_t **regs, int reg_cnt)
 {
     int rc;
     unsigned char *bound_addr;
 
-    if(size == 0) {
+    if (size == 0) {
         return OPAL_ERROR;
     }
 
     bound_addr = (unsigned char *) ((intptr_t) addr + size - 1);
 
     /* Check to ensure that the cache is valid */
-    if (OPAL_UNLIKELY(opal_memory_changed() &&
-                      NULL != opal_memory->memoryc_process &&
-                      OPAL_SUCCESS != (rc = opal_memory->memoryc_process()))) {
+    if (OPAL_UNLIKELY(opal_memory_changed() && NULL != opal_memory->memoryc_process
+                      && OPAL_SUCCESS != (rc = opal_memory->memoryc_process()))) {
         return rc;
     }
 
-    return mca_rcache_base_vma_tree_find_all (vma_module, (unsigned char *) addr,
-                                              bound_addr, regs, reg_cnt);
+    return mca_rcache_base_vma_tree_find_all(vma_module, (unsigned char *) addr, bound_addr, regs,
+                                             reg_cnt);
 }
 
-int mca_rcache_base_vma_insert (mca_rcache_base_vma_module_t *vma_module,
-                                mca_rcache_base_registration_t *reg, size_t limit)
+int mca_rcache_base_vma_insert(mca_rcache_base_vma_module_t *vma_module,
+                               mca_rcache_base_registration_t *reg, size_t limit)
 {
     size_t reg_size = reg->bound - reg->base + 1;
     int rc;
@@ -117,49 +115,46 @@ int mca_rcache_base_vma_insert (mca_rcache_base_vma_module_t *vma_module,
     }
 
     /* Check to ensure that the cache is valid */
-    if (OPAL_UNLIKELY(opal_memory_changed() &&
-                      NULL != opal_memory->memoryc_process &&
-                      OPAL_SUCCESS != (rc = opal_memory->memoryc_process()))) {
+    if (OPAL_UNLIKELY(opal_memory_changed() && NULL != opal_memory->memoryc_process
+                      && OPAL_SUCCESS != (rc = opal_memory->memoryc_process()))) {
         return rc;
     }
 
-    rc = mca_rcache_base_vma_tree_insert (vma_module, reg, limit);
+    rc = mca_rcache_base_vma_tree_insert(vma_module, reg, limit);
     if (OPAL_LIKELY(OPAL_SUCCESS == rc)) {
         /* If we successfully registered, then tell the memory manager
            to start monitoring this region */
-        opal_memory->memoryc_register (reg->base, (uint64_t) reg_size,
-                                       (uint64_t) (uintptr_t) reg);
+        opal_memory->memoryc_register(reg->base, (uint64_t) reg_size, (uint64_t)(uintptr_t) reg);
     }
 
     return rc;
 }
 
-int mca_rcache_base_vma_delete (mca_rcache_base_vma_module_t *vma_module,
-                                mca_rcache_base_registration_t *reg)
+int mca_rcache_base_vma_delete(mca_rcache_base_vma_module_t *vma_module,
+                               mca_rcache_base_registration_t *reg)
 {
     /* Tell the memory manager that we no longer care about this
        region */
-    opal_memory->memoryc_deregister (reg->base,
-                                     (uint64_t) (reg->bound - reg->base),
-                                     (uint64_t) (uintptr_t) reg);
-    return mca_rcache_base_vma_tree_delete (vma_module, reg);
+    opal_memory->memoryc_deregister(reg->base, (uint64_t)(reg->bound - reg->base),
+                                    (uint64_t)(uintptr_t) reg);
+    return mca_rcache_base_vma_tree_delete(vma_module, reg);
 }
 
-int mca_rcache_base_vma_iterate (mca_rcache_base_vma_module_t *vma_module,
-                                 unsigned char *base, size_t size, bool partial_ok,
-                                 int (*callback_fn) (struct mca_rcache_base_registration_t *, void *),
-                                 void *ctx)
+int mca_rcache_base_vma_iterate(mca_rcache_base_vma_module_t *vma_module, unsigned char *base,
+                                size_t size, bool partial_ok,
+                                int (*callback_fn)(struct mca_rcache_base_registration_t *, void *),
+                                void *ctx)
 {
-    return mca_rcache_base_vma_tree_iterate (vma_module, base, size, partial_ok, callback_fn, ctx);
+    return mca_rcache_base_vma_tree_iterate(vma_module, base, size, partial_ok, callback_fn, ctx);
 }
 
-void mca_rcache_base_vma_dump_range (mca_rcache_base_vma_module_t *vma_module,
-                                     unsigned char *base, size_t size, char *msg)
+void mca_rcache_base_vma_dump_range(mca_rcache_base_vma_module_t *vma_module, unsigned char *base,
+                                    size_t size, char *msg)
 {
-    mca_rcache_base_vma_tree_dump_range (vma_module, base, size, msg);
+    mca_rcache_base_vma_tree_dump_range(vma_module, base, size, msg);
 }
 
-size_t mca_rcache_base_vma_size (mca_rcache_base_vma_module_t *vma_module)
+size_t mca_rcache_base_vma_size(mca_rcache_base_vma_module_t *vma_module)
 {
-    return mca_rcache_base_vma_tree_size (vma_module);
+    return mca_rcache_base_vma_tree_size(vma_module);
 }
diff --git a/opal/mca/rcache/base/rcache_base_vma.h b/opal/mca/rcache/base/rcache_base_vma.h
index 261cad67196..a549d670ebc 100644
--- a/opal/mca/rcache/base/rcache_base_vma.h
+++ b/opal/mca/rcache/base/rcache_base_vma.h
@@ -15,6 +15,7 @@
  * Copyright (c) 2009      IBM Corporation.  All rights reserved.
  * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2025      NVIDIA Corporation.  All rights reserved.
  *
  * $COPYRIGHT$
  *
@@ -32,9 +33,9 @@
 #define MCA_RCACHE_BASE_VMA_H
 
 #include "opal_config.h"
-#include "opal/class/opal_list.h"
 #include "opal/class/opal_interval_tree.h"
 #include "opal/class/opal_lifo.h"
+#include "opal/class/opal_list.h"
 
 BEGIN_C_DECLS
 
@@ -43,33 +44,28 @@ struct mca_rcache_base_registration_t;
 struct mca_rcache_base_vma_module_t {
     opal_object_t super;
     opal_interval_tree_t tree;
-    opal_list_t vma_list;
-    opal_lifo_t vma_gc_lifo;
-    size_t reg_cur_cache_size;
     opal_mutex_t vma_lock;
 };
 typedef struct mca_rcache_base_vma_module_t mca_rcache_base_vma_module_t;
 
 OBJ_CLASS_DECLARATION(mca_rcache_base_vma_module_t);
 
-mca_rcache_base_vma_module_t *mca_rcache_base_vma_module_alloc (void);
+mca_rcache_base_vma_module_t *mca_rcache_base_vma_module_alloc(void);
 
-int mca_rcache_base_vma_find (mca_rcache_base_vma_module_t *vma_module, void *addr,
-                              size_t size, struct mca_rcache_base_registration_t **reg);
+int mca_rcache_base_vma_find(mca_rcache_base_vma_module_t *vma_module, void *addr, size_t size,
+                             struct mca_rcache_base_registration_t **reg);
 
-int mca_rcache_base_vma_find_all (mca_rcache_base_vma_module_t *vma_module, void *addr,
-                                  size_t size, struct mca_rcache_base_registration_t **regs,
-                                  int reg_cnt);
+int mca_rcache_base_vma_find_all(mca_rcache_base_vma_module_t *vma_module, void *addr, size_t size,
+                                 struct mca_rcache_base_registration_t **regs, int reg_cnt);
 
-int mca_rcache_base_vma_insert (mca_rcache_base_vma_module_t *vma_module,
-                                struct mca_rcache_base_registration_t *registration,
-                                size_t limit);
+int mca_rcache_base_vma_insert(mca_rcache_base_vma_module_t *vma_module,
+                               struct mca_rcache_base_registration_t *registration, size_t limit);
 
-int mca_rcache_base_vma_delete (mca_rcache_base_vma_module_t *vma_module,
-                                struct mca_rcache_base_registration_t *registration);
+int mca_rcache_base_vma_delete(mca_rcache_base_vma_module_t *vma_module,
+                               struct mca_rcache_base_registration_t *registration);
 
-void mca_rcache_base_vma_dump_range (mca_rcache_base_vma_module_t *vma_module,
-                                     unsigned char *base, size_t size, char *msg);
+void mca_rcache_base_vma_dump_range(mca_rcache_base_vma_module_t *vma_module, unsigned char *base,
+                                    size_t size, char *msg);
 
 /**
  * Iterate over registrations in the specified range.
@@ -87,12 +83,12 @@ void mca_rcache_base_vma_dump_range (mca_rcache_base_vma_module_t *vma_module,
  * from the callback. The iteration will terminate if the callback returns anything
  * other than OPAL_SUCCESS.
  */
-int mca_rcache_base_vma_iterate (mca_rcache_base_vma_module_t *vma_module,
-                                 unsigned char *base, size_t size, bool partial_ok,
-                                 int (*callback_fn) (struct mca_rcache_base_registration_t *, void *),
-                                 void *ctx);
+int mca_rcache_base_vma_iterate(mca_rcache_base_vma_module_t *vma_module, unsigned char *base,
+                                size_t size, bool partial_ok,
+                                int (*callback_fn)(struct mca_rcache_base_registration_t *, void *),
+                                void *ctx);
 
-size_t mca_rcache_base_vma_size (mca_rcache_base_vma_module_t *vma_module);
+size_t mca_rcache_base_vma_size(mca_rcache_base_vma_module_t *vma_module);
 
 END_C_DECLS
 
diff --git a/opal/mca/rcache/base/rcache_base_vma_tree.c b/opal/mca/rcache/base/rcache_base_vma_tree.c
index 09362f4f2b5..2dc5e0d70e8 100644
--- a/opal/mca/rcache/base/rcache_base_vma_tree.c
+++ b/opal/mca/rcache/base/rcache_base_vma_tree.c
@@ -14,7 +14,7 @@
  * Copyright (c) 2006      Voltaire. All rights reserved.
  * Copyright (c) 2007      Mellanox Technologies. All rights reserved.
  * Copyright (c) 2009      IBM Corporation.  All rights reserved.
- * Copyright (c) 2013      NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2013-2025 NVIDIA Corporation.  All rights reserved.
  * Copyright (c) 2013-2018 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights
  *                         reserved.
@@ -27,27 +27,28 @@
  * $HEADER$
  */
 
-#include "opal/util/output.h"
 #include "rcache_base_vma_tree.h"
 #include "opal/mca/rcache/base/base.h"
+#include "opal/util/output.h"
 
-int mca_rcache_base_vma_tree_init (mca_rcache_base_vma_module_t *vma_module)
+int mca_rcache_base_vma_tree_init(mca_rcache_base_vma_module_t *vma_module)
 {
     OBJ_CONSTRUCT(&vma_module->tree, opal_interval_tree_t);
-    vma_module->reg_cur_cache_size = 0;
-    return opal_interval_tree_init (&vma_module->tree);
+    return opal_interval_tree_init(&vma_module->tree);
 }
 
-void mca_rcache_base_vma_tree_finalize (mca_rcache_base_vma_module_t *vma_module)
+void mca_rcache_base_vma_tree_finalize(mca_rcache_base_vma_module_t *vma_module)
 {
     OBJ_DESTRUCT(&vma_module->tree);
 }
 
-mca_rcache_base_registration_t *mca_rcache_base_vma_tree_find (mca_rcache_base_vma_module_t *vma_module,
-                                                               unsigned char *base, unsigned char *bound)
+mca_rcache_base_registration_t *
+mca_rcache_base_vma_tree_find(mca_rcache_base_vma_module_t *vma_module, unsigned char *base,
+                              unsigned char *bound)
 {
-    return (mca_rcache_base_registration_t *) opal_interval_tree_find_overlapping (&vma_module->tree, (uintptr_t) base,
-                                                                                   ((uintptr_t) bound) + 1);
+    return (mca_rcache_base_registration_t *)
+        opal_interval_tree_find_overlapping(&vma_module->tree, (uintptr_t) base,
+                                            ((uintptr_t) bound) + 1);
 }
 
 struct mca_rcache_base_vma_tree_find_all_helper_args_t {
@@ -56,11 +57,14 @@ struct mca_rcache_base_vma_tree_find_all_helper_args_t {
     int reg_max;
 };
 
-typedef struct mca_rcache_base_vma_tree_find_all_helper_args_t mca_rcache_base_vma_tree_find_all_helper_args_t;
+typedef struct mca_rcache_base_vma_tree_find_all_helper_args_t
+    mca_rcache_base_vma_tree_find_all_helper_args_t;
 
-static int mca_rcache_base_vma_tree_find_all_helper (uint64_t low, uint64_t high, void *data, void *ctx)
+static int mca_rcache_base_vma_tree_find_all_helper(uint64_t low, uint64_t high, void *data,
+                                                    void *ctx)
 {
-    mca_rcache_base_vma_tree_find_all_helper_args_t *args = (mca_rcache_base_vma_tree_find_all_helper_args_t *) ctx;
+    mca_rcache_base_vma_tree_find_all_helper_args_t *args
+        = (mca_rcache_base_vma_tree_find_all_helper_args_t *) ctx;
     mca_rcache_base_registration_t *reg = (mca_rcache_base_registration_t *) data;
 
     if (args->reg_cnt == args->reg_max) {
@@ -72,44 +76,51 @@ static int mca_rcache_base_vma_tree_find_all_helper (uint64_t low, uint64_t high
     return OPAL_SUCCESS;
 }
 
-int mca_rcache_base_vma_tree_find_all (mca_rcache_base_vma_module_t *vma_module, unsigned char *base,
-                                       unsigned char *bound, mca_rcache_base_registration_t **regs,
-                                       int reg_cnt)
+int mca_rcache_base_vma_tree_find_all(mca_rcache_base_vma_module_t *vma_module, unsigned char *base,
+                                      unsigned char *bound, mca_rcache_base_registration_t **regs,
+                                      int reg_cnt)
 {
-    mca_rcache_base_vma_tree_find_all_helper_args_t args = {.regs = regs, .reg_max = reg_cnt, .reg_cnt = 0};
+    mca_rcache_base_vma_tree_find_all_helper_args_t args = {.regs = regs,
+                                                            .reg_max = reg_cnt,
+                                                            .reg_cnt = 0};
 
-    (void) opal_interval_tree_traverse (&vma_module->tree, (uint64_t) (uintptr_t) base, ((uint64_t) (uintptr_t) bound) + 1,
-                                        true, mca_rcache_base_vma_tree_find_all_helper, &args);
+    (void) opal_interval_tree_traverse(&vma_module->tree, (uint64_t)(uintptr_t) base,
+                                       ((uint64_t)(uintptr_t) bound) + 1, true,
+                                       mca_rcache_base_vma_tree_find_all_helper, &args);
     return args.reg_cnt;
 }
 
 struct mca_rcache_base_vma_tree_iterate_helper_args_t {
-    int (*callback_fn) (struct mca_rcache_base_registration_t *, void *);
+    int (*callback_fn)(struct mca_rcache_base_registration_t *, void *);
     void *ctx;
 };
-typedef struct mca_rcache_base_vma_tree_iterate_helper_args_t mca_rcache_base_vma_tree_iterate_helper_args_t;
+typedef struct mca_rcache_base_vma_tree_iterate_helper_args_t
+    mca_rcache_base_vma_tree_iterate_helper_args_t;
 
-static int mca_rcache_base_vma_tree_iterate_helper (uint64_t low, uint64_t high, void *data, void *ctx)
+static int mca_rcache_base_vma_tree_iterate_helper(uint64_t low, uint64_t high, void *data,
+                                                   void *ctx)
 {
-    mca_rcache_base_vma_tree_iterate_helper_args_t *args = (mca_rcache_base_vma_tree_iterate_helper_args_t *) ctx;
-    return args->callback_fn ((mca_rcache_base_registration_t *) data, args->ctx);
+    mca_rcache_base_vma_tree_iterate_helper_args_t *args
+        = (mca_rcache_base_vma_tree_iterate_helper_args_t *) ctx;
+    return args->callback_fn((mca_rcache_base_registration_t *) data, args->ctx);
 }
 
-int mca_rcache_base_vma_tree_iterate (mca_rcache_base_vma_module_t *vma_module, unsigned char *base, size_t size,
-                                      bool partial_ok, int (*callback_fn) (struct mca_rcache_base_registration_t *, void *),
-                                      void *ctx)
+int mca_rcache_base_vma_tree_iterate(
+    mca_rcache_base_vma_module_t *vma_module, unsigned char *base, size_t size, bool partial_ok,
+    int (*callback_fn)(struct mca_rcache_base_registration_t *, void *), void *ctx)
 {
     mca_rcache_base_vma_tree_iterate_helper_args_t args = {.callback_fn = callback_fn, .ctx = ctx};
     uintptr_t bound = (uintptr_t) base + size;
 
-    return opal_interval_tree_traverse (&vma_module->tree, (uint64_t) (intptr_t) base, bound, partial_ok,
-                                        mca_rcache_base_vma_tree_iterate_helper, &args);
+    return opal_interval_tree_traverse(&vma_module->tree, (uint64_t)(intptr_t) base, bound,
+                                       partial_ok, mca_rcache_base_vma_tree_iterate_helper, &args);
 }
 
-int mca_rcache_base_vma_tree_insert (mca_rcache_base_vma_module_t *vma_module,
-                                     mca_rcache_base_registration_t *reg, size_t limit)
+int mca_rcache_base_vma_tree_insert(mca_rcache_base_vma_module_t *vma_module,
+                                    mca_rcache_base_registration_t *reg, size_t limit)
 {
-    return opal_interval_tree_insert (&vma_module->tree, reg, (uintptr_t) reg->base, (uintptr_t) reg->bound + 1);
+    return opal_interval_tree_insert(&vma_module->tree, reg, (uintptr_t) reg->base,
+                                     (uintptr_t) reg->bound + 1);
 }
 
 /**
@@ -120,39 +131,41 @@ int mca_rcache_base_vma_tree_insert (mca_rcache_base_vma_module_t *vma_module,
  * @retval OPAL_SUCCESS
  * @retval OPAL_ERR_BAD_PARAM if the passed base pointer was invalid
  */
-int mca_rcache_base_vma_tree_delete (mca_rcache_base_vma_module_t *vma_module,
-                                     mca_rcache_base_registration_t *reg)
+int mca_rcache_base_vma_tree_delete(mca_rcache_base_vma_module_t *vma_module,
+                                    mca_rcache_base_registration_t *reg)
 {
-    return opal_interval_tree_delete (&vma_module->tree, (uintptr_t) reg->base, (uintptr_t) reg->bound + 1, reg);
+    return opal_interval_tree_delete(&vma_module->tree, (uintptr_t) reg->base,
+                                     (uintptr_t) reg->bound + 1, reg);
 }
 
-static int mca_rcache_base_tree_dump_range_helper (uint64_t low, uint64_t high, void *data, void *ctx)
+static int mca_rcache_base_tree_dump_range_helper(uint64_t low, uint64_t high, void *data,
+                                                  void *ctx)
 {
-    mca_rcache_base_registration_t *reg = ( mca_rcache_base_registration_t *) data;
+    mca_rcache_base_registration_t *reg = (mca_rcache_base_registration_t *) data;
 
-    opal_output(0, "    reg: base=%p, bound=%p, ref_count=%d, flags=0x%x",
-                (void *) reg->base, (void *) reg->bound, reg->ref_count, reg->flags);
+    opal_output(0, "    reg: base=%p, bound=%p, ref_count=%d, flags=0x%x", (void *) reg->base,
+                (void *) reg->bound, reg->ref_count, reg->flags);
 
     return OPAL_SUCCESS;
 }
 
 /* Dump out rcache entries within a range of memory.  Useful for debugging. */
-void mca_rcache_base_vma_tree_dump_range (mca_rcache_base_vma_module_t *vma_module,
-                                          unsigned char *base, size_t size, char *msg)
+void mca_rcache_base_vma_tree_dump_range(mca_rcache_base_vma_module_t *vma_module,
+                                         unsigned char *base, size_t size, char *msg)
 {
     uintptr_t bound = (uintptr_t) base + size;
 
     opal_output(0, "Dumping rcache entries: %s", msg ? msg : "");
 
-    if (opal_interval_tree_size (&vma_module->tree)) {
-        (void) opal_interval_tree_traverse (&vma_module->tree, (uintptr_t) base, bound, false,
-                                            mca_rcache_base_tree_dump_range_helper, NULL);
+    if (opal_interval_tree_size(&vma_module->tree)) {
+        (void) opal_interval_tree_traverse(&vma_module->tree, (uintptr_t) base, bound, false,
+                                           mca_rcache_base_tree_dump_range_helper, NULL);
     } else {
         opal_output(0, "  rcache is empty");
     }
 }
 
-size_t mca_rcache_base_vma_tree_size (mca_rcache_base_vma_module_t *vma_module)
+size_t mca_rcache_base_vma_tree_size(mca_rcache_base_vma_module_t *vma_module)
 {
-    return opal_interval_tree_size (&vma_module->tree);
+    return opal_interval_tree_size(&vma_module->tree);
 }
diff --git a/opal/mca/rcache/base/rcache_base_vma_tree.h b/opal/mca/rcache/base/rcache_base_vma_tree.h
index 87f8de2a02e..6ba251d1b51 100644
--- a/opal/mca/rcache/base/rcache_base_vma_tree.h
+++ b/opal/mca/rcache/base/rcache_base_vma_tree.h
@@ -24,9 +24,9 @@
  * $HEADER$
  */
 /**
-  * @file
-  * Registation cache VMA tree implementation
-  */
+ * @file
+ * Registration cache VMA tree implementation
+ */
 #ifndef MCA_RCACHE_BASE_VMA_TREE_H
 #define MCA_RCACHE_BASE_VMA_TREE_H
 
@@ -38,7 +38,7 @@
 /*
  * initialize the vma tree
  */
-int mca_rcache_base_vma_tree_init (mca_rcache_base_vma_module_t *vma_module);
+int mca_rcache_base_vma_tree_init(mca_rcache_base_vma_module_t *vma_module);
 
 /*
  * clean up the vma tree
@@ -48,43 +48,40 @@ void mca_rcache_base_vma_tree_finalize(mca_rcache_base_vma_module_t *vma_module)
 /**
  *  Returns the item in the vma tree
  */
-mca_rcache_base_registration_t *mca_rcache_base_vma_tree_find (mca_rcache_base_vma_module_t *vma_module,
-                                                               unsigned char *base,
-                                                               unsigned char *bound);
+mca_rcache_base_registration_t *
+mca_rcache_base_vma_tree_find(mca_rcache_base_vma_module_t *vma_module, unsigned char *base,
+                              unsigned char *bound);
 /**
  * Returns all registration that overlaps given memory region
  */
-int mca_rcache_base_vma_tree_find_all (
-        mca_rcache_base_vma_module_t *vma_module, unsigned char *base,
-        unsigned char *bound, mca_rcache_base_registration_t **regs,
-        int reg_cnt);
+int mca_rcache_base_vma_tree_find_all(mca_rcache_base_vma_module_t *vma_module, unsigned char *base,
+                                      unsigned char *bound, mca_rcache_base_registration_t **regs,
+                                      int reg_cnt);
 
 /*
  * insert an item in the vma tree
  */
-int mca_rcache_base_vma_tree_insert (mca_rcache_base_vma_module_t *vma_module,
-                                     mca_rcache_base_registration_t* reg, size_t limit);
+int mca_rcache_base_vma_tree_insert(mca_rcache_base_vma_module_t *vma_module,
+                                    mca_rcache_base_registration_t *reg, size_t limit);
 
 /*
  * remove an item from the vma tree
  */
-int mca_rcache_base_vma_tree_delete (mca_rcache_base_vma_module_t *vma_module,
-                                     mca_rcache_base_registration_t *reg);
+int mca_rcache_base_vma_tree_delete(mca_rcache_base_vma_module_t *vma_module,
+                                    mca_rcache_base_registration_t *reg);
 
 /*
  * Dump out the contents of the rcache for debugging.
  */
-void mca_rcache_base_vma_tree_dump_range (mca_rcache_base_vma_module_t *vma_module,
-                                          unsigned char *base, size_t size, char *msg);
-
+void mca_rcache_base_vma_tree_dump_range(mca_rcache_base_vma_module_t *vma_module,
+                                         unsigned char *base, size_t size, char *msg);
 
 /*
  * Iterate over matching registration handles in the tree.
  */
-int mca_rcache_base_vma_tree_iterate (mca_rcache_base_vma_module_t *vma_module,
-                                      unsigned char *base, size_t size, bool partial_ok,
-                                      int (*callback_fn) (struct mca_rcache_base_registration_t *, void *),
-                                      void *ctx);
+int mca_rcache_base_vma_tree_iterate(
+    mca_rcache_base_vma_module_t *vma_module, unsigned char *base, size_t size, bool partial_ok,
+    int (*callback_fn)(struct mca_rcache_base_registration_t *, void *), void *ctx);
 
 /**
  * @brief Get the current size of the vma tree
@@ -93,6 +90,6 @@ int mca_rcache_base_vma_tree_iterate (mca_rcache_base_vma_module_t *vma_module,
  *
  * @returns the current number of vma regions in the tree
  */
-size_t mca_rcache_base_vma_tree_size (mca_rcache_base_vma_module_t *vma_module);
+size_t mca_rcache_base_vma_tree_size(mca_rcache_base_vma_module_t *vma_module);
 
 #endif /* MCA_RCACHE_BASE_VMA_TREE_H */
diff --git a/opal/mca/rcache/gpusm/Makefile.am b/opal/mca/rcache/gpusm/Makefile.am
index 6b1f7191ebe..1ff63b35d58 100644
--- a/opal/mca/rcache/gpusm/Makefile.am
+++ b/opal/mca/rcache/gpusm/Makefile.am
@@ -14,6 +14,7 @@
 # Copyright (c) 2015      Los Alamos National Security, LLC. All rights
 #                         reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -47,15 +48,11 @@ endif
 mcacomponentdir = $(opallibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_rcache_gpusm_la_SOURCES = $(sources)
-mca_rcache_gpusm_la_LDFLAGS = -module -avoid-version
-mca_rcache_gpusm_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
+mca_rcache_gpusm_la_LDFLAGS = -module -avoid-version $(rcache_gpusm_LDFLAGS)
+mca_rcache_gpusm_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
 	$(rcache_gpusm_LIBS)
-if OPAL_cuda_support
-mca_rcache_gpusm_la_LIBADD += \
-    $(OPAL_TOP_BUILDDIR)/opal/mca/common/cuda/lib@OPAL_LIB_PREFIX@mca_common_cuda.la
-endif
 
 noinst_LTLIBRARIES = $(component_noinst)
 libmca_rcache_gpusm_la_SOURCES = $(sources)
-libmca_rcache_gpusm_la_LDFLAGS = -module -avoid-version
+libmca_rcache_gpusm_la_LDFLAGS = -module -avoid-version $(rcache_gpusm_LDFLAGS)
 libmca_rcache_gpusm_la_LIBADD = $(rcache_gpusm_LIBS)
diff --git a/opal/mca/rcache/gpusm/configure.m4 b/opal/mca/rcache/gpusm/configure.m4
deleted file mode 100644
index 2b792d7cc8c..00000000000
--- a/opal/mca/rcache/gpusm/configure.m4
+++ /dev/null
@@ -1,27 +0,0 @@
-# -*- shell-script -*-
-#
-# Copyright (c) 2012-2015 NVIDIA Corporation.  All rights reserved.
-# Copyright (c) 2015      Los Alamos National Security, LLC. All rights
-#                         reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-#
-# If CUDA support was requested, then build the CUDA memory pools.
-# This code checks the variable CUDA_SUPPORT which was set earlier in
-# the configure sequence by the opal_configure_options.m4 code.
-#
-
-AC_DEFUN([MCA_opal_rcache_gpusm_CONFIG],[
-    AC_CONFIG_FILES([opal/mca/rcache/gpusm/Makefile])
-
-    # Use CUDA_SUPPORT which was filled in by the opal configure code.
-    AS_IF([test "x$CUDA_SUPPORT" = "x1"],
-          [$1],
-          [$2])
-
-])dnl
diff --git a/opal/mca/rcache/gpusm/rcache_gpusm.h b/opal/mca/rcache/gpusm/rcache_gpusm.h
index 00733f3c00f..1d1de5a5625 100644
--- a/opal/mca/rcache/gpusm/rcache_gpusm.h
+++ b/opal/mca/rcache/gpusm/rcache_gpusm.h
@@ -30,16 +30,16 @@
 #include "opal_config.h"
 #include "opal/class/opal_list.h"
 #include "opal/mca/rcache/rcache.h"
+#include "opal/mca/accelerator/accelerator.h"
 
 BEGIN_C_DECLS
+extern int mca_rcache_gpusm_use_sync_memops;
 
-#define MEMHANDLE_SIZE 8
-#define EVTHANDLE_SIZE 8
 struct mca_rcache_gpusm_registration_t {
-    mca_rcache_base_registration_t base;
-    uint64_t memHandle[MEMHANDLE_SIZE]; /* CUipcMemHandle */
-    uint64_t evtHandle[EVTHANDLE_SIZE]; /* CUipcEventHandle */
-    uintptr_t event;                    /* CUevent */
+    mca_rcache_base_registration_t      base;
+    opal_accelerator_ipc_handle_t       memHandle;
+    opal_accelerator_ipc_event_handle_t evtHandle;
+    opal_accelerator_event_t            *event;
 };
 typedef struct mca_rcache_gpusm_registration_t mca_rcache_gpusm_registration_t;
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(mca_rcache_gpusm_registration_t);
@@ -54,7 +54,8 @@ OPAL_DECLSPEC extern mca_rcache_gpusm_component_t mca_rcache_gpusm_component;
 struct mca_rcache_gpusm_module_t {
     mca_rcache_base_module_t super;
     opal_free_list_t reg_list;
-}; typedef struct mca_rcache_gpusm_module_t mca_rcache_gpusm_module_t;
+};
+typedef struct mca_rcache_gpusm_module_t mca_rcache_gpusm_module_t;
 
 /*
  *  Initializes the rcache module.
@@ -62,22 +63,23 @@ struct mca_rcache_gpusm_module_t {
 void mca_rcache_gpusm_module_init(mca_rcache_gpusm_module_t *rcache);
 
 /**
-  * register block of memory
-  */
-int mca_rcache_gpusm_register(mca_rcache_base_module_t* rcache, void *addr,
-        size_t size, uint32_t flags, int32_t access_flags, mca_rcache_base_registration_t **reg);
+ * register block of memory
+ */
+int mca_rcache_gpusm_register(mca_rcache_base_module_t *rcache, void *addr, size_t size,
+                              uint32_t flags, int32_t access_flags,
+                              mca_rcache_base_registration_t **reg);
 
 /**
  * deregister memory
  */
 int mca_rcache_gpusm_deregister(mca_rcache_base_module_t *rcache,
-        mca_rcache_base_registration_t *reg);
+                                mca_rcache_base_registration_t *reg);
 
 /**
  * find registration for a given block of memory
  */
-int mca_rcache_gpusm_find(struct mca_rcache_base_module_t* rcache, void* addr,
-        size_t size, mca_rcache_base_registration_t **reg);
+int mca_rcache_gpusm_find(struct mca_rcache_base_module_t *rcache, void *addr, size_t size,
+                          mca_rcache_base_registration_t **reg);
 
 /**
  * finalize rcache
diff --git a/opal/mca/rcache/gpusm/rcache_gpusm_component.c b/opal/mca/rcache/gpusm/rcache_gpusm_component.c
index fed00b62cf5..7c398a484d0 100644
--- a/opal/mca/rcache/gpusm/rcache_gpusm_component.c
+++ b/opal/mca/rcache/gpusm/rcache_gpusm_component.c
@@ -28,10 +28,10 @@
 #include "opal/mca/base/base.h"
 #include "rcache_gpusm.h"
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 #ifdef HAVE_MALLOC_H
-#include <malloc.h>
+#    include <malloc.h>
 #endif
 
 /*
@@ -40,14 +40,14 @@
 static int gpusm_open(void);
 static int gpusm_close(void);
 static int gpusm_register(void);
-static mca_rcache_base_module_t* gpusm_init(struct mca_rcache_base_resources_t* resources);
+static mca_rcache_base_module_t *gpusm_init(struct mca_rcache_base_resources_t *resources);
 
-mca_rcache_gpusm_component_t mca_rcache_gpusm_component = {
-    {
-      /* First, the mca_base_component_t struct containing meta
-         information about the component itself */
+mca_rcache_gpusm_component_t mca_rcache_gpusm_component = {{
+    /* First, the mca_base_component_t struct containing meta
+       information about the component itself */
 
-        .rcache_version = {
+    .rcache_version =
+        {
             MCA_RCACHE_BASE_VERSION_3_0_0,
 
             .mca_component_name = "gpusm",
@@ -57,44 +57,39 @@ mca_rcache_gpusm_component_t mca_rcache_gpusm_component = {
             .mca_close_component = gpusm_close,
             .mca_register_component_params = gpusm_register,
         },
-        .rcache_data = {
-            /* The component is checkpoint ready */
-            MCA_BASE_METADATA_PARAM_CHECKPOINT
-        },
+    .rcache_data =
+        {/* The component is checkpoint ready */
+         MCA_BASE_METADATA_PARAM_CHECKPOINT},
 
-        .rcache_init = gpusm_init,
-    }
-};
+    .rcache_init = gpusm_init,
+}};
 
 /**
-  * Component open/close/init/register functions.  Most do not do anything,
-  * but keep around for placeholders.
-  */
+ * Component open/close/init/register functions.  Most do not do anything,
+ * but keep around for placeholders.
+ */
 static int gpusm_open(void)
 {
     return OPAL_SUCCESS;
 }
 
-
 static int gpusm_register(void)
 {
-	return OPAL_SUCCESS;
+    return OPAL_SUCCESS;
 }
 
-
 static int gpusm_close(void)
 {
     return OPAL_SUCCESS;
 }
 
-
-static mca_rcache_base_module_t* gpusm_init(struct mca_rcache_base_resources_t *resources)
+static mca_rcache_base_module_t *gpusm_init(struct mca_rcache_base_resources_t *resources)
 {
-    mca_rcache_gpusm_module_t* rcache_module;
+    mca_rcache_gpusm_module_t *rcache_module;
 
     (void) resources;
 
-    rcache_module = (mca_rcache_gpusm_module_t *) calloc (1, sizeof (*rcache_module));
+    rcache_module = (mca_rcache_gpusm_module_t *) calloc(1, sizeof(*rcache_module));
     if (NULL == rcache_module) {
         return NULL;
     }
diff --git a/opal/mca/rcache/gpusm/rcache_gpusm_module.c b/opal/mca/rcache/gpusm/rcache_gpusm_module.c
index caf8913a938..91f153ef957 100644
--- a/opal/mca/rcache/gpusm/rcache_gpusm_module.c
+++ b/opal/mca/rcache/gpusm/rcache_gpusm_module.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2013 The University of Tennessee and The University
+ * Copyright (c) 2004-2024 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -18,6 +18,8 @@
  * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
  *                         reserved.
  *
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2023      Advanced Micro Devices, Inc. All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -40,24 +42,34 @@
 #include "opal_config.h"
 #include "opal/mca/rcache/base/base.h"
 #include "opal/mca/rcache/gpusm/rcache_gpusm.h"
-#include "opal/mca/common/cuda/common_cuda.h"
+#include "opal/include/opal/opal_gpu.h"
+#include "opal/mca/accelerator/base/base.h"
 
 /**
  * Called when the registration free list is created.  An event is created
  * for each entry.
  */
-static void mca_rcache_gpusm_registration_constructor( mca_rcache_gpusm_registration_t *item )
+static void mca_rcache_gpusm_registration_constructor(mca_rcache_gpusm_registration_t *item)
 {
-    mca_common_cuda_construct_event_and_handle(&item->event,
-                                               (void *)&item->evtHandle);
+    int result;
+
+    result = opal_accelerator.create_event(MCA_ACCELERATOR_NO_DEVICE_ID, &item->event, true);
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != result)) {
+        opal_output(0, "create_ipc_event failed\n");
+    }
+
+    result = opal_accelerator.get_ipc_event_handle(item->event, &item->evtHandle);
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != result)) {
+        opal_output(0, "get_ipc_event_handle failed\n");
+    }
 }
 
 /**
  * Called when the program is exiting.  This destroys the events.
  */
-static void mca_rcache_gpusm_registration_destructor( mca_rcache_gpusm_registration_t *item )
+static void mca_rcache_gpusm_registration_destructor(mca_rcache_gpusm_registration_t *item)
 {
-    mca_common_cuda_destruct_event(item->event);
+    OBJ_RELEASE(item->event);
 }
 
 OBJ_CLASS_INSTANCE(mca_rcache_gpusm_registration_t, mca_rcache_base_registration_t,
@@ -67,8 +79,10 @@ OBJ_CLASS_INSTANCE(mca_rcache_gpusm_registration_t, mca_rcache_base_registration
 /*
  *  Initializes the rcache module.
  */
-void mca_rcache_gpusm_module_init(mca_rcache_gpusm_module_t* rcache)
+void mca_rcache_gpusm_module_init(mca_rcache_gpusm_module_t *rcache)
 {
+    mca_rcache_base_module_init(&rcache->super);
+
     rcache->super.rcache_component = &mca_rcache_gpusm_component.super;
     rcache->super.rcache_register = mca_rcache_gpusm_register;
     rcache->super.rcache_find = mca_rcache_gpusm_find;
@@ -80,36 +94,87 @@ void mca_rcache_gpusm_module_init(mca_rcache_gpusm_module_t* rcache)
     /* Start with 0 entries in the free list since CUDA may not have
      * been initialized when this free list is created and there is
      * some CUDA specific activities that need to be done. */
-    opal_free_list_init (&rcache->reg_list, sizeof(struct mca_rcache_common_cuda_reg_t),
-            opal_cache_line_size,
-            OBJ_CLASS(mca_rcache_gpusm_registration_t),
-            0,opal_cache_line_size,
-            0, -1, 64, NULL, 0, NULL, NULL, NULL);
-
+    opal_free_list_init(&rcache->reg_list, sizeof(struct mca_opal_gpu_reg_t),
+                        opal_cache_line_size, OBJ_CLASS(mca_rcache_gpusm_registration_t), 0,
+                        opal_cache_line_size, 0, -1, 64, NULL, 0, NULL, NULL, NULL);
 }
 
 /**
  * Just go ahead and get a new registration.  The find and register
  * functions are the same thing for this memory pool.
  */
-int mca_rcache_gpusm_find(mca_rcache_base_module_t *rcache, void *addr,
-                         size_t size,
-                         mca_rcache_base_registration_t **reg)
+int mca_rcache_gpusm_find(mca_rcache_base_module_t *rcache, void *addr, size_t size,
+                          mca_rcache_base_registration_t **reg)
 {
     return mca_rcache_gpusm_register(rcache, addr, size, 0, 0, reg);
 }
 
+/*
+ * Get the memory handle of a local section of memory that can be sent
+ * to the remote size so it can access the memory.  This is the
+ * registration function for the sending side of a message transfer.
+ */
+static int mca_rcache_gpusm_get_mem_handle(void *base, size_t size, mca_rcache_base_registration_t *newreg)
+{
+    mca_opal_gpu_reg_t *gpu_reg = (mca_opal_gpu_reg_t *) newreg;
+    opal_accelerator_buffer_id_t buffer_id;
+    uint64_t flags;
+    int      dev_id;
+    int      result;
+    void*    pbase;
+    size_t   psize;
+
+    result = opal_accelerator.check_addr(base, &dev_id, &flags);
+    if (0 >= result) {
+        // This is either an error or host memory. In either case we do not continue
+        return OPAL_ERROR;
+    }
+
+    result = opal_accelerator.get_ipc_handle(dev_id, base, &gpu_reg->data.ipcHandle);
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != result)) {
+        return OPAL_ERROR;
+    }
+    result = opal_accelerator.get_address_range (dev_id, base, &pbase, &psize);
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != result)) {
+        return OPAL_ERROR;
+    }
+
+    /* Store all the information in the registration */
+    gpu_reg->base.base  = pbase;
+    gpu_reg->base.bound = (unsigned char *) pbase + psize - 1;
+    gpu_reg->data.memh_seg_addr.pval = (void *) pbase;
+    gpu_reg->data.memh_seg_len = psize;
+
+    // converting the ifdef into a mca runtime parameter
+    if (opal_accelerator_use_sync_memops) {
+        // need to revisit. This function also sets sync_memops
+        // we might want to separate that out into a separate function
+        result = opal_accelerator.get_buffer_id(dev_id, base, &buffer_id);
+        if (OPAL_UNLIKELY(OPAL_SUCCESS != result)) {
+            return OPAL_ERROR;
+        }
+    }
+    else {
+        result = opal_accelerator.record_event(dev_id, gpu_reg->data.event, 0);
+        if (OPAL_UNLIKELY(OPAL_SUCCESS != result)) {
+            return OPAL_ERROR;
+        }
+    }
+
+    return OPAL_SUCCESS;
+}
+
 /*
  * This is the one function that does all the work.  It will call into
  * the register function to get the memory handle for the sending
  * buffer.  There is no need to deregister the memory handle so the
  * deregister function is a no-op.
  */
-int mca_rcache_gpusm_register(mca_rcache_base_module_t *rcache, void *addr,
-                             size_t size, uint32_t flags, int32_t access_flags,
-                             mca_rcache_base_registration_t **reg)
+int mca_rcache_gpusm_register(mca_rcache_base_module_t *rcache, void *addr, size_t size,
+                              uint32_t flags, int32_t access_flags,
+                              mca_rcache_base_registration_t **reg)
 {
-    mca_rcache_gpusm_module_t *rcache_gpusm = (mca_rcache_gpusm_module_t*)rcache;
+    mca_rcache_gpusm_module_t *rcache_gpusm = (mca_rcache_gpusm_module_t *) rcache;
     mca_rcache_base_registration_t *gpusm_reg;
     opal_free_list_item_t *item;
     unsigned char *base, *bound;
@@ -122,13 +187,13 @@ int mca_rcache_gpusm_register(mca_rcache_base_module_t *rcache, void *addr,
     *reg = NULL;
 
     base = addr;
-    bound = (unsigned char *)addr + size - 1;
+    bound = (unsigned char *) addr + size - 1;
 
-    item = opal_free_list_get (&rcache_gpusm->reg_list);
-    if(NULL == item) {
+    item = opal_free_list_get(&rcache_gpusm->reg_list);
+    if (NULL == item) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
-    gpusm_reg = (mca_rcache_base_registration_t*)item;
+    gpusm_reg = (mca_rcache_base_registration_t *) item;
 
     gpusm_reg->rcache = rcache;
     gpusm_reg->base = base;
@@ -136,30 +201,27 @@ int mca_rcache_gpusm_register(mca_rcache_base_module_t *rcache, void *addr,
     gpusm_reg->flags = flags;
     gpusm_reg->access_flags = access_flags;
 
-    rc = cuda_getmemhandle (base, size, gpusm_reg, NULL);
+    rc = mca_rcache_gpusm_get_mem_handle(base, size, gpusm_reg);
 
-    if(rc != OPAL_SUCCESS) {
-        opal_free_list_return (&rcache_gpusm->reg_list, item);
+    if (OPAL_SUCCESS != rc) {
+        opal_free_list_return(&rcache_gpusm->reg_list, item);
         return rc;
     }
 
     *reg = gpusm_reg;
     (*reg)->ref_count++;
     return OPAL_SUCCESS;
-
 }
 
 /*
  * Return the registration to the free list.
  */
 int mca_rcache_gpusm_deregister(struct mca_rcache_base_module_t *rcache,
-                               mca_rcache_base_registration_t *reg)
+                                mca_rcache_base_registration_t *reg)
 {
-    int rc;
-    mca_rcache_gpusm_module_t *rcache_gpusm = (mca_rcache_gpusm_module_t *)rcache;
+    mca_rcache_gpusm_module_t *rcache_gpusm = (mca_rcache_gpusm_module_t *) rcache;
 
-    rc = cuda_ungetmemhandle (NULL, reg);
-    opal_free_list_return (&rcache_gpusm->reg_list, (opal_free_list_item_t *) reg);
+    opal_free_list_return(&rcache_gpusm->reg_list, (opal_free_list_item_t *) reg);
     return OPAL_SUCCESS;
 }
 
@@ -169,15 +231,19 @@ int mca_rcache_gpusm_deregister(struct mca_rcache_base_module_t *rcache,
 void mca_rcache_gpusm_finalize(struct mca_rcache_base_module_t *rcache)
 {
     opal_free_list_item_t *item;
-    mca_rcache_gpusm_module_t *rcache_gpusm = (mca_rcache_gpusm_module_t *)rcache;
+    mca_rcache_gpusm_module_t *rcache_gpusm = (mca_rcache_gpusm_module_t *) rcache;
 
     /* Need to run the destructor on each item in the free list explicitly.
      * The destruction of the free list only runs the destructor on the
      * main free list, not each item. */
-    while (NULL != (item = (opal_free_list_item_t *)opal_lifo_pop(&(rcache_gpusm->reg_list.super)))) {
+    while (NULL
+           != (item = (opal_free_list_item_t *) opal_lifo_pop(&(rcache_gpusm->reg_list.super)))) {
         OBJ_DESTRUCT(item);
     }
 
     OBJ_DESTRUCT(&rcache_gpusm->reg_list);
+
+    mca_rcache_base_module_fini(rcache);
+
     return;
 }
diff --git a/opal/mca/rcache/grdma/Makefile.am b/opal/mca/rcache/grdma/Makefile.am
index 64826f6cb66..5af5064ee95 100644
--- a/opal/mca/rcache/grdma/Makefile.am
+++ b/opal/mca/rcache/grdma/Makefile.am
@@ -13,6 +13,7 @@
 # Copyright (c) 2012-2015 Los Alamos National Security, LLC. All rights
 #                         reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -47,12 +48,8 @@ mcacomponentdir = $(opallibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_rcache_grdma_la_SOURCES = $(sources)
 mca_rcache_grdma_la_LDFLAGS = -module -avoid-version
-mca_rcache_grdma_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
+mca_rcache_grdma_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
 	$(rcache_grdma_LIBS)
-if OPAL_cuda_support
-mca_rcache_grdma_la_LIBADD += \
-    $(OPAL_TOP_BUILDDIR)/opal/mca/common/cuda/lib@OPAL_LIB_PREFIX@mca_common_cuda.la
-endif
 
 noinst_LTLIBRARIES = $(component_noinst)
 libmca_rcache_grdma_la_SOURCES = $(sources)
diff --git a/opal/mca/rcache/grdma/rcache_grdma.h b/opal/mca/rcache/grdma/rcache_grdma.h
index 7d912dd6aeb..00e8d30f08a 100644
--- a/opal/mca/rcache/grdma/rcache_grdma.h
+++ b/opal/mca/rcache/grdma/rcache_grdma.h
@@ -30,10 +30,10 @@
 
 #include "opal_config.h"
 #include "opal/class/opal_list.h"
-#include "opal/util/event.h"
 #include "opal/mca/rcache/rcache.h"
+#include "opal/util/event.h"
 #if HAVE_SYS_MMAN_H
-#include <sys/mman.h>
+#    include <sys/mman.h>
 #endif
 
 #define MCA_RCACHE_GRDMA_REG_FLAG_IN_LRU MCA_RCACHE_FLAGS_MOD_RESV0
@@ -80,7 +80,8 @@ typedef struct mca_rcache_grdma_module_t mca_rcache_grdma_module_t;
 /*
  *  Initializes the rcache module.
  */
-void mca_rcache_grdma_module_init(mca_rcache_grdma_module_t *rcache, mca_rcache_grdma_cache_t *cache);
+void mca_rcache_grdma_module_init(mca_rcache_grdma_module_t *rcache,
+                                  mca_rcache_grdma_cache_t *cache);
 
 END_C_DECLS
 #endif
diff --git a/opal/mca/rcache/grdma/rcache_grdma_component.c b/opal/mca/rcache/grdma/rcache_grdma_component.c
index 3660eb8c055..f55d752378b 100644
--- a/opal/mca/rcache/grdma/rcache_grdma_component.c
+++ b/opal/mca/rcache/grdma/rcache_grdma_component.c
@@ -28,10 +28,10 @@
 #include "opal/runtime/opal_params.h"
 #include "rcache_grdma.h"
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
-#include <stdlib.h>
 #include <fcntl.h>
+#include <stdlib.h>
 
 /*
  * Local functions
@@ -39,15 +39,14 @@
 static int grdma_open(void);
 static int grdma_close(void);
 static int grdma_register(void);
-static mca_rcache_base_module_t* grdma_init(
-        struct mca_rcache_base_resources_t* resources);
+static mca_rcache_base_module_t *grdma_init(struct mca_rcache_base_resources_t *resources);
 
-mca_rcache_grdma_component_t mca_rcache_grdma_component = {
-    {
-      /* First, the mca_base_component_t struct containing meta
-         information about the component itself */
+mca_rcache_grdma_component_t mca_rcache_grdma_component = {{
+    /* First, the mca_base_component_t struct containing meta
+       information about the component itself */
 
-        .rcache_version = {
+    .rcache_version =
+        {
             MCA_RCACHE_BASE_VERSION_3_0_0,
 
             .mca_component_name = "grdma",
@@ -57,18 +56,16 @@ mca_rcache_grdma_component_t mca_rcache_grdma_component = {
             .mca_close_component = grdma_close,
             .mca_register_component_params = grdma_register,
         },
-        .rcache_data = {
-            /* The component is checkpoint ready */
-            MCA_BASE_METADATA_PARAM_CHECKPOINT
-        },
+    .rcache_data =
+        {/* The component is checkpoint ready */
+         MCA_BASE_METADATA_PARAM_CHECKPOINT},
 
-        .rcache_init = grdma_init,
-    }
-};
+    .rcache_init = grdma_init,
+}};
 
 /**
-  * component open/close/init function
-  */
+ * component open/close/init function
+ */
 static int grdma_open(void)
 {
     OBJ_CONSTRUCT(&mca_rcache_grdma_component.caches, opal_list_t);
@@ -76,30 +73,25 @@ static int grdma_open(void)
     return OPAL_SUCCESS;
 }
 
-
 static int grdma_register(void)
 {
     mca_rcache_grdma_component.print_stats = false;
-    (void) mca_base_component_var_register(&mca_rcache_grdma_component.super.rcache_version,
-                                           "print_stats", "print registration cache usage statistics at the end of the run",
-                                           MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
-                                           &mca_rcache_grdma_component.print_stats);
+    (void) mca_base_component_var_register(
+        &mca_rcache_grdma_component.super.rcache_version, "print_stats",
+        "print registration cache usage statistics at the end of the run", MCA_BASE_VAR_TYPE_BOOL,
+        NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+        &mca_rcache_grdma_component.print_stats);
 
     return OPAL_SUCCESS;
 }
 
-
 static int grdma_close(void)
 {
     OPAL_LIST_DESTRUCT(&mca_rcache_grdma_component.caches);
     return OPAL_SUCCESS;
 }
 
-
-static mca_rcache_base_module_t *
-grdma_init(struct mca_rcache_base_resources_t *resources)
+static mca_rcache_base_module_t *grdma_init(struct mca_rcache_base_resources_t *resources)
 {
     mca_rcache_grdma_module_t *rcache_module;
     mca_rcache_grdma_cache_t *cache = NULL, *item;
@@ -107,12 +99,12 @@ grdma_init(struct mca_rcache_base_resources_t *resources)
     /* Set this here (vs in component.c) because
        opal_leave_pinned* may have been set after MCA params were
        read (e.g., by the openib btl) */
-    mca_rcache_grdma_component.leave_pinned = (int)
-        (1 == opal_leave_pinned || opal_leave_pinned_pipeline);
+    mca_rcache_grdma_component.leave_pinned = (int) (1 == opal_leave_pinned
+                                                     || opal_leave_pinned_pipeline);
 
     /* find the specified pool */
-    OPAL_LIST_FOREACH(item, &mca_rcache_grdma_component.caches, mca_rcache_grdma_cache_t) {
-        if (0 == strcmp (item->cache_name, resources->cache_name)) {
+    OPAL_LIST_FOREACH (item, &mca_rcache_grdma_component.caches, mca_rcache_grdma_cache_t) {
+        if (0 == strcmp(item->cache_name, resources->cache_name)) {
             cache = item;
             break;
         }
@@ -125,16 +117,16 @@ grdma_init(struct mca_rcache_base_resources_t *resources)
             return NULL;
         }
 
-        cache->cache_name = strdup (resources->cache_name);
+        cache->cache_name = strdup(resources->cache_name);
 
-        opal_list_append (&mca_rcache_grdma_component.caches, &cache->super);
+        opal_list_append(&mca_rcache_grdma_component.caches, &cache->super);
     }
 
-    rcache_module = (mca_rcache_grdma_module_t *) malloc (sizeof (*rcache_module));
+    rcache_module = (mca_rcache_grdma_module_t *) malloc(sizeof(*rcache_module));
 
     rcache_module->resources = *resources;
 
-    mca_rcache_grdma_module_init (rcache_module, cache);
+    mca_rcache_grdma_module_init(rcache_module, cache);
 
     return &rcache_module->super;
 }
diff --git a/opal/mca/rcache/grdma/rcache_grdma_module.c b/opal/mca/rcache/grdma/rcache_grdma_module.c
index 3f163c2d7f1..e8177f21b1e 100644
--- a/opal/mca/rcache/grdma/rcache_grdma_module.c
+++ b/opal/mca/rcache/grdma/rcache_grdma_module.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2013 The University of Tennessee and The University
+ * Copyright (c) 2004-2024 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -16,11 +16,12 @@
  * Copyright (c) 2010      IBM Corporation.  All rights reserved.
  * Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights
  *                         reserved.
- * Copyright (c) 2013      NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2013-2025 NVIDIA Corporation.  All rights reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2020      Google, LLC. All rights reserved.
  *
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -32,67 +33,60 @@
 #include "opal_config.h"
 
 #include <errno.h>
-#include <string.h>
 #include <stdlib.h>
+#include <string.h>
 
 #include "opal/align.h"
 
 #include "opal/util/proc.h"
-#if OPAL_CUDA_GDR_SUPPORT
-#include "opal/mca/common/cuda/common_cuda.h"
-#endif /* OPAL_CUDA_GDR_SUPPORT */
-#include "opal/mca/rcache/rcache.h"
 #include "opal/mca/rcache/base/base.h"
-
-#include "opal/util/sys_limits.h"
+#include "opal/mca/rcache/rcache.h"
+#include "opal/mca/accelerator/accelerator.h"
 #include "opal/align.h"
+#include "opal/util/sys_limits.h"
 #include "rcache_grdma.h"
 
-
-static int mca_rcache_grdma_register (mca_rcache_base_module_t *rcache, void *addr,
-                                      size_t size, uint32_t flags, int32_t access_flags,
-                                      mca_rcache_base_registration_t **reg);
-static int mca_rcache_grdma_deregister (mca_rcache_base_module_t *rcache,
-                                        mca_rcache_base_registration_t *reg);
-static int mca_rcache_grdma_find (mca_rcache_base_module_t *rcache, void *addr,
-                                  size_t size, mca_rcache_base_registration_t **reg);
-static int mca_rcache_grdma_invalidate_range (mca_rcache_base_module_t *rcache, void *base,
-                                              size_t size);
-static void mca_rcache_grdma_finalize (mca_rcache_base_module_t *rcache);
-static bool mca_rcache_grdma_evict (mca_rcache_base_module_t *rcache);
-static int mca_rcache_grdma_add_to_gc (mca_rcache_base_registration_t *grdma_reg);
-
-static inline bool registration_flags_cacheable (uint32_t flags)
+static int mca_rcache_grdma_register(mca_rcache_base_module_t *rcache, void *addr, size_t size,
+                                     uint32_t flags, int32_t access_flags,
+                                     mca_rcache_base_registration_t **reg);
+static int mca_rcache_grdma_deregister(mca_rcache_base_module_t *rcache,
+                                       mca_rcache_base_registration_t *reg);
+static int mca_rcache_grdma_find(mca_rcache_base_module_t *rcache, void *addr, size_t size,
+                                 mca_rcache_base_registration_t **reg);
+static int mca_rcache_grdma_invalidate_range(mca_rcache_base_module_t *rcache, void *base,
+                                             size_t size);
+static void mca_rcache_grdma_finalize(mca_rcache_base_module_t *rcache);
+static bool mca_rcache_grdma_evict(mca_rcache_base_module_t *rcache);
+static int mca_rcache_grdma_add_to_gc(mca_rcache_base_registration_t *grdma_reg);
+static int check_for_accelerator_freed_memory(mca_rcache_base_module_t *rcache, void *addr, size_t size);
+
+static inline bool registration_flags_cacheable(uint32_t flags)
 {
-    return (mca_rcache_grdma_component.leave_pinned &&
-            !(flags &
-              (MCA_RCACHE_FLAGS_CACHE_BYPASS |
-               MCA_RCACHE_FLAGS_PERSIST |
-               MCA_RCACHE_FLAGS_INVALID)));
+    return (mca_rcache_grdma_component.leave_pinned
+            && !(flags
+                 & (MCA_RCACHE_FLAGS_CACHE_BYPASS | MCA_RCACHE_FLAGS_PERSIST
+                    | MCA_RCACHE_FLAGS_INVALID)));
 }
 
 static inline bool registration_is_cacheable(mca_rcache_base_registration_t *reg)
 {
-    return registration_flags_cacheable (reg->flags);
+    return registration_flags_cacheable(reg->flags);
 }
 
-#if OPAL_CUDA_GDR_SUPPORT
-static int check_for_cuda_freed_memory(mca_rcache_base_module_t *rcache, void *addr, size_t size);
-#endif /* OPAL_CUDA_GDR_SUPPORT */
-static void mca_rcache_grdma_cache_contructor (mca_rcache_grdma_cache_t *cache)
+static void mca_rcache_grdma_cache_contructor(mca_rcache_grdma_cache_t *cache)
 {
-    memset ((void *)((uintptr_t)cache + sizeof (cache->super)), 0, sizeof (*cache) - sizeof (cache->super));
-
     OBJ_CONSTRUCT(&cache->lru_list, opal_list_t);
     OBJ_CONSTRUCT(&cache->gc_lifo, opal_lifo_t);
 
-    cache->vma_module = mca_rcache_base_vma_module_alloc ();
+    cache->cache_name = NULL;
+    cache->vma_module = mca_rcache_base_vma_module_alloc();
 }
 
-static void mca_rcache_grdma_cache_destructor (mca_rcache_grdma_cache_t *cache)
+static void mca_rcache_grdma_cache_destructor(mca_rcache_grdma_cache_t *cache)
 {
     /* clear the lru before releasing the list */
-    while (NULL != opal_list_remove_first (&cache->lru_list));
+    while (NULL != opal_list_remove_first(&cache->lru_list)) {
+    }
 
     OBJ_DESTRUCT(&cache->lru_list);
     OBJ_DESTRUCT(&cache->gc_lifo);
@@ -100,21 +94,23 @@ static void mca_rcache_grdma_cache_destructor (mca_rcache_grdma_cache_t *cache)
         OBJ_RELEASE(cache->vma_module);
     }
 
-    free (cache->cache_name);
+    free(cache->cache_name);
 }
 
-OBJ_CLASS_INSTANCE(mca_rcache_grdma_cache_t, opal_list_item_t,
-                   mca_rcache_grdma_cache_contructor,
+OBJ_CLASS_INSTANCE(mca_rcache_grdma_cache_t, opal_list_item_t, mca_rcache_grdma_cache_contructor,
                    mca_rcache_grdma_cache_destructor);
 
 /*
  *  Initializes the rcache module.
  */
-void mca_rcache_grdma_module_init(mca_rcache_grdma_module_t* rcache, mca_rcache_grdma_cache_t *cache)
+void mca_rcache_grdma_module_init(mca_rcache_grdma_module_t *rcache,
+                                  mca_rcache_grdma_cache_t *cache)
 {
     OBJ_RETAIN(cache);
     rcache->cache = cache;
 
+    mca_rcache_base_module_init(&rcache->super);
+
     rcache->super.rcache_component = &mca_rcache_grdma_component.super;
     rcache->super.rcache_register = mca_rcache_grdma_register;
     rcache->super.rcache_find = mca_rcache_grdma_find;
@@ -127,11 +123,9 @@ void mca_rcache_grdma_module_init(mca_rcache_grdma_module_t* rcache, mca_rcache_
     rcache->stat_cache_found = rcache->stat_cache_notfound = 0;
 
     OBJ_CONSTRUCT(&rcache->reg_list, opal_free_list_t);
-    opal_free_list_init (&rcache->reg_list, rcache->resources.sizeof_reg,
-                         opal_cache_line_size,
-                         OBJ_CLASS(mca_rcache_base_registration_t),
-                         0, opal_cache_line_size, 0, -1, 32, NULL, 0,
-                         NULL, NULL, NULL);
+    opal_free_list_init(&rcache->reg_list, rcache->resources.sizeof_reg, opal_cache_line_size,
+                        OBJ_CLASS(mca_rcache_base_registration_t), 0, opal_cache_line_size, 0, -1,
+                        32, NULL, 0, NULL, NULL, NULL);
 }
 
 static inline int dereg_mem(mca_rcache_base_registration_t *reg)
@@ -142,13 +136,12 @@ static inline int dereg_mem(mca_rcache_base_registration_t *reg)
     reg->ref_count = 0;
 
     if (!(reg->flags & MCA_RCACHE_FLAGS_CACHE_BYPASS)) {
-        mca_rcache_base_vma_delete (rcache_grdma->cache->vma_module, reg);
+        mca_rcache_base_vma_delete(rcache_grdma->cache->vma_module, reg);
     }
 
-    rc = rcache_grdma->resources.deregister_mem (rcache_grdma->resources.reg_data, reg);
+    rc = rcache_grdma->resources.deregister_mem(rcache_grdma->resources.reg_data, reg);
     if (OPAL_LIKELY(OPAL_SUCCESS == rc)) {
-        opal_free_list_return_mt (&rcache_grdma->reg_list,
-                                  (opal_free_list_item_t *) reg);
+        opal_free_list_return_mt(&rcache_grdma->reg_list, (opal_free_list_item_t *) reg);
     }
 
     OPAL_OUTPUT_VERBOSE((MCA_BASE_VERBOSE_TRACE, opal_rcache_base_framework.framework_output,
@@ -156,28 +149,30 @@ static inline int dereg_mem(mca_rcache_base_registration_t *reg)
     return rc;
 }
 
-static inline void do_unregistration_gc (mca_rcache_base_module_t *rcache)
+static inline void do_unregistration_gc(mca_rcache_base_module_t *rcache)
 {
     mca_rcache_grdma_module_t *rcache_grdma = (mca_rcache_grdma_module_t *) rcache;
     opal_list_item_t *item;
 
     /* Remove registration from garbage collection list before deregistering it */
-    while (NULL != (item = opal_lifo_pop_atomic (&rcache_grdma->cache->gc_lifo))) {
+    while (NULL != (item = opal_lifo_pop_atomic(&rcache_grdma->cache->gc_lifo))) {
         OPAL_OUTPUT_VERBOSE((MCA_BASE_VERBOSE_TRACE, opal_rcache_base_framework.framework_output,
                              "deleting stale registration %p", (void *) item));
-        dereg_mem ((mca_rcache_base_registration_t *) item);
+        dereg_mem((mca_rcache_base_registration_t *) item);
     }
 }
 
-static inline mca_rcache_base_registration_t *mca_rcache_grdma_remove_lru_head(mca_rcache_grdma_cache_t *cache) {
+static inline mca_rcache_base_registration_t *
+mca_rcache_grdma_remove_lru_head(mca_rcache_grdma_cache_t *cache)
+{
     mca_rcache_base_registration_t *old_reg;
     int32_t old_flags;
 
     do {
-        opal_mutex_lock (&cache->vma_module->vma_lock);
-        old_reg = (mca_rcache_base_registration_t *) opal_list_remove_first (&cache->lru_list);
+        opal_mutex_lock(&cache->vma_module->vma_lock);
+        old_reg = (mca_rcache_base_registration_t *) opal_list_remove_first(&cache->lru_list);
         if (NULL == old_reg) {
-            opal_mutex_unlock (&cache->vma_module->vma_lock);
+            opal_mutex_unlock(&cache->vma_module->vma_lock);
             break;
         }
 
@@ -187,11 +182,12 @@ static inline mca_rcache_base_registration_t *mca_rcache_grdma_remove_lru_head(m
             /* registration has been selected for removal and is no longer in the LRU. mark it
              * as such. */
             new_flags = (old_flags & ~MCA_RCACHE_GRDMA_REG_FLAG_IN_LRU) | MCA_RCACHE_FLAGS_INVALID;
-            if (opal_atomic_compare_exchange_strong_32((opal_atomic_int32_t*)&old_reg->flags, &old_flags, new_flags)) {
+            if (opal_atomic_compare_exchange_strong_32((opal_atomic_int32_t *) &old_reg->flags,
+                                                       &old_flags, new_flags)) {
                 break;
             }
         } while (1);
-        opal_mutex_unlock (&cache->vma_module->vma_lock);
+        opal_mutex_unlock(&cache->vma_module->vma_lock);
 
         if (old_flags & MCA_RCACHE_FLAGS_INVALID) {
             /* registration was already invalidated. in this case its fate is being determined
@@ -205,7 +201,7 @@ static inline mca_rcache_base_registration_t *mca_rcache_grdma_remove_lru_head(m
     return NULL;
 }
 
-static inline bool mca_rcache_grdma_evict_lru_local (mca_rcache_grdma_cache_t *cache)
+static inline bool mca_rcache_grdma_evict_lru_local(mca_rcache_grdma_cache_t *cache)
 {
     mca_rcache_grdma_module_t *rcache_grdma;
     mca_rcache_base_registration_t *old_reg;
@@ -217,15 +213,15 @@ static inline bool mca_rcache_grdma_evict_lru_local (mca_rcache_grdma_cache_t *c
 
     rcache_grdma = (mca_rcache_grdma_module_t *) old_reg->rcache;
 
-    (void) dereg_mem (old_reg);
+    (void) dereg_mem(old_reg);
     rcache_grdma->stat_evicted++;
 
     return true;
 }
 
-static bool mca_rcache_grdma_evict (mca_rcache_base_module_t *rcache)
+static bool mca_rcache_grdma_evict(mca_rcache_base_module_t *rcache)
 {
-    return mca_rcache_grdma_evict_lru_local (((mca_rcache_grdma_module_t *) rcache)->cache);
+    return mca_rcache_grdma_evict_lru_local(((mca_rcache_grdma_module_t *) rcache)->cache);
 }
 
 struct mca_rcache_base_find_args_t {
@@ -238,44 +234,48 @@ struct mca_rcache_base_find_args_t {
 
 typedef struct mca_rcache_base_find_args_t mca_rcache_base_find_args_t;
 
-static inline void mca_rcache_grdma_add_to_lru (mca_rcache_grdma_module_t *rcache_grdma, mca_rcache_base_registration_t *grdma_reg)
+static inline void mca_rcache_grdma_add_to_lru(mca_rcache_grdma_module_t *rcache_grdma,
+                                               mca_rcache_base_registration_t *grdma_reg)
 {
-    opal_mutex_lock (&rcache_grdma->cache->vma_module->vma_lock);
+    opal_mutex_lock(&rcache_grdma->cache->vma_module->vma_lock);
 
     opal_list_append(&rcache_grdma->cache->lru_list, (opal_list_item_t *) grdma_reg);
 
     /* ensure the append is complete before setting the flag */
-    opal_atomic_wmb ();
+    opal_atomic_wmb();
 
     /* mark this registration as being in the LRU */
-    opal_atomic_fetch_or_32 ((opal_atomic_int32_t *) &grdma_reg->flags, MCA_RCACHE_GRDMA_REG_FLAG_IN_LRU);
+    opal_atomic_fetch_or_32((opal_atomic_int32_t *) &grdma_reg->flags,
+                            MCA_RCACHE_GRDMA_REG_FLAG_IN_LRU);
 
-    opal_mutex_unlock (&rcache_grdma->cache->vma_module->vma_lock);
+    opal_mutex_unlock(&rcache_grdma->cache->vma_module->vma_lock);
 }
 
-static inline void mca_rcache_grdma_remove_from_lru (mca_rcache_grdma_module_t *rcache_grdma, mca_rcache_base_registration_t *grdma_reg)
+static inline void mca_rcache_grdma_remove_from_lru(mca_rcache_grdma_module_t *rcache_grdma,
+                                                    mca_rcache_base_registration_t *grdma_reg)
 {
     /* if the reference count was observed to be 0 (which must be the case for this
      * function to be called then some thread deregistered the region. it may be the
      * case that the deregistration is still ongoing so wait until the deregistration
      * thread has marked this registration as being in the lru before continuing */
-    while (!(grdma_reg->flags & MCA_RCACHE_GRDMA_REG_FLAG_IN_LRU));
+    while (!(grdma_reg->flags & MCA_RCACHE_GRDMA_REG_FLAG_IN_LRU)) {
+    }
 
     /* opal lists are not thread safe at this time so we must lock :'( */
-    opal_mutex_lock (&rcache_grdma->cache->vma_module->vma_lock);
-    opal_list_remove_item (&rcache_grdma->cache->lru_list, (opal_list_item_t *) grdma_reg);
+    opal_mutex_lock(&rcache_grdma->cache->vma_module->vma_lock);
+    opal_list_remove_item(&rcache_grdma->cache->lru_list, (opal_list_item_t *) grdma_reg);
     /* clear the LRU flag */
     grdma_reg->flags &= ~MCA_RCACHE_GRDMA_REG_FLAG_IN_LRU;
-    opal_mutex_unlock (&rcache_grdma->cache->vma_module->vma_lock);
+    opal_mutex_unlock(&rcache_grdma->cache->vma_module->vma_lock);
 }
 
-static int mca_rcache_grdma_check_cached (mca_rcache_base_registration_t *grdma_reg, void *ctx)
+static int mca_rcache_grdma_check_cached(mca_rcache_base_registration_t *grdma_reg, void *ctx)
 {
     mca_rcache_base_find_args_t *args = (mca_rcache_base_find_args_t *) ctx;
     mca_rcache_grdma_module_t *rcache_grdma = args->rcache_grdma;
 
-    if ((grdma_reg->flags & MCA_RCACHE_FLAGS_INVALID) || &rcache_grdma->super != grdma_reg->rcache ||
-        grdma_reg->base > args->base || grdma_reg->bound < args->bound) {
+    if ((grdma_reg->flags & MCA_RCACHE_FLAGS_INVALID) || &rcache_grdma->super != grdma_reg->rcache
+        || grdma_reg->base > args->base || grdma_reg->bound < args->bound) {
         return 0;
     }
 
@@ -283,37 +283,38 @@ static int mca_rcache_grdma_check_cached (mca_rcache_base_registration_t *grdma_
         args->access_flags |= grdma_reg->access_flags;
 
         /* can't use this registration */
-        return mca_rcache_grdma_add_to_gc (grdma_reg);
+        return mca_rcache_grdma_add_to_gc(grdma_reg);
     }
 
-    int32_t ref_cnt = opal_atomic_fetch_add_32 (&grdma_reg->ref_count, 1);
+    int32_t ref_cnt = opal_atomic_fetch_add_32(&grdma_reg->ref_count, 1);
     args->reg = grdma_reg;
 
     if (0 == ref_cnt) {
-        mca_rcache_grdma_remove_from_lru (rcache_grdma, grdma_reg);
+        mca_rcache_grdma_remove_from_lru(rcache_grdma, grdma_reg);
     }
 
     /* This segment fits fully within an existing segment. */
-    (void) opal_atomic_fetch_add_32 ((opal_atomic_int32_t *) &rcache_grdma->stat_cache_hit, 1);
+    (void) opal_atomic_fetch_add_32((opal_atomic_int32_t *) &rcache_grdma->stat_cache_hit, 1);
     OPAL_OUTPUT_VERBOSE((MCA_BASE_VERBOSE_TRACE, opal_rcache_base_framework.framework_output,
-                         "returning existing registration %p. references %d", (void *) grdma_reg, ref_cnt));
+                         "returning existing registration %p. references %d", (void *) grdma_reg,
+                         ref_cnt));
     return 1;
 }
 
 /*
  * register memory
  */
-static int mca_rcache_grdma_register (mca_rcache_base_module_t *rcache, void *addr,
-                                      size_t size, uint32_t flags, int32_t access_flags,
-                                      mca_rcache_base_registration_t **reg)
+static int mca_rcache_grdma_register(mca_rcache_base_module_t *rcache, void *addr, size_t size,
+                                     uint32_t flags, int32_t access_flags,
+                                     mca_rcache_base_registration_t **reg)
 {
-    mca_rcache_grdma_module_t *rcache_grdma = (mca_rcache_grdma_module_t*)rcache;
+    mca_rcache_grdma_module_t *rcache_grdma = (mca_rcache_grdma_module_t *) rcache;
     const bool bypass_cache = !!(flags & MCA_RCACHE_FLAGS_CACHE_BYPASS);
     const bool persist = !!(flags & MCA_RCACHE_FLAGS_PERSIST);
     mca_rcache_base_registration_t *grdma_reg;
     opal_free_list_item_t *item;
     unsigned char *base, *bound;
-    unsigned int page_size = opal_getpagesize ();
+    unsigned int page_size = opal_getpagesize();
     int rc;
 
     *reg = NULL;
@@ -322,28 +323,31 @@ static int mca_rcache_grdma_register (mca_rcache_base_module_t *rcache, void *ad
     base = OPAL_DOWN_ALIGN_PTR(addr, page_size, unsigned char *);
     bound = OPAL_ALIGN_PTR((intptr_t) addr + size, page_size, unsigned char *) - 1;
 
-#if OPAL_CUDA_GDR_SUPPORT
-    if (flags & MCA_RCACHE_FLAGS_CUDA_GPU_MEM) {
+    if (flags & MCA_RCACHE_FLAGS_ACCELERATOR_MEM && !bypass_cache) {
         size_t psize;
-        mca_common_cuda_get_address_range(&base, &psize, addr);
+        int res = opal_accelerator.get_address_range(MCA_ACCELERATOR_NO_DEVICE_ID, addr, (void **)&base, &psize);
+        if (OPAL_SUCCESS != res) {
+            abort();
+        }
         bound = base + psize - 1;
         /* Check to see if this memory is in the cache and if it has been freed. If so,
          * this call will boot it out of the cache. */
-        check_for_cuda_freed_memory(rcache, base, psize);
+        check_for_accelerator_freed_memory(rcache, base, psize);
     }
-#endif /* OPAL_CUDA_GDR_SUPPORT */
 
-    do_unregistration_gc (rcache);
+    do_unregistration_gc(rcache);
 
     /* look through existing regs if not persistent registration requested.
      * Persistent registration are always registered and placed in the cache */
     if (!(bypass_cache || persist)) {
-        mca_rcache_base_find_args_t find_args = {.reg = NULL, .rcache_grdma = rcache_grdma,
-                                                 .base = base, .bound = bound,
+        mca_rcache_base_find_args_t find_args = {.reg = NULL,
+                                                 .rcache_grdma = rcache_grdma,
+                                                 .base = base,
+                                                 .bound = bound,
                                                  .access_flags = access_flags};
         /* check to see if memory is registered */
-        rc = mca_rcache_base_vma_iterate (rcache_grdma->cache->vma_module, base, size, false,
-                                          mca_rcache_grdma_check_cached, (void *) &find_args);
+        rc = mca_rcache_base_vma_iterate(rcache_grdma->cache->vma_module, base, size, false,
+                                         mca_rcache_grdma_check_cached, (void *) &find_args);
         if (1 == rc) {
             *reg = find_args.reg;
             return OPAL_SUCCESS;
@@ -355,11 +359,11 @@ static int mca_rcache_grdma_register (mca_rcache_base_module_t *rcache, void *ad
         OPAL_THREAD_ADD_FETCH32((opal_atomic_int32_t *) &rcache_grdma->stat_cache_miss, 1);
     }
 
-    item = opal_free_list_get_mt (&rcache_grdma->reg_list);
-    if(NULL == item) {
+    item = opal_free_list_get_mt(&rcache_grdma->reg_list);
+    if (NULL == item) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
-    grdma_reg = (mca_rcache_base_registration_t*)item;
+    grdma_reg = (mca_rcache_base_registration_t *) item;
 
     grdma_reg->rcache = rcache;
     grdma_reg->base = base;
@@ -367,23 +371,21 @@ static int mca_rcache_grdma_register (mca_rcache_base_module_t *rcache, void *ad
     grdma_reg->flags = flags;
     grdma_reg->access_flags = access_flags;
     grdma_reg->ref_count = 1;
-#if OPAL_CUDA_GDR_SUPPORT
-    if (flags & MCA_RCACHE_FLAGS_CUDA_GPU_MEM) {
-        mca_common_cuda_get_buffer_id(grdma_reg);
+    if (flags & MCA_RCACHE_FLAGS_ACCELERATOR_MEM && !bypass_cache) {
+        opal_accelerator.get_buffer_id(MCA_ACCELERATOR_NO_DEVICE_ID, grdma_reg->base, &grdma_reg->gpu_bufID);
     }
-#endif /* OPAL_CUDA_GDR_SUPPORT */
 
-    while (OPAL_ERR_OUT_OF_RESOURCE ==
-           (rc = rcache_grdma->resources.register_mem(rcache_grdma->resources.reg_data,
-                                                     base, bound - base + 1, grdma_reg))) {
+    while (OPAL_ERR_OUT_OF_RESOURCE
+           == (rc = rcache_grdma->resources.register_mem(rcache_grdma->resources.reg_data, base,
+                                                         bound - base + 1, grdma_reg))) {
         /* try to remove one unused reg and retry */
-        if (!mca_rcache_grdma_evict (rcache)) {
+        if (!mca_rcache_grdma_evict(rcache)) {
             break;
         }
     }
 
     if (OPAL_UNLIKELY(rc != OPAL_SUCCESS)) {
-        opal_free_list_return_mt (&rcache_grdma->reg_list, item);
+        opal_free_list_return_mt(&rcache_grdma->reg_list, item);
         return rc;
     }
 
@@ -393,80 +395,78 @@ static int mca_rcache_grdma_register (mca_rcache_base_module_t *rcache, void *ad
          * no leave pinned protocol is in use but the same segment is in
          * use in multiple simultaneous transactions. We used to set bypass_cache
          * here is !mca_rcache_grdma_component.leave_pinned. */
-        rc = mca_rcache_base_vma_insert (rcache_grdma->cache->vma_module, grdma_reg, 0);
+        rc = mca_rcache_base_vma_insert(rcache_grdma->cache->vma_module, grdma_reg, 0);
         if (OPAL_UNLIKELY(rc != OPAL_SUCCESS)) {
-            rcache_grdma->resources.deregister_mem (rcache_grdma->resources.reg_data, grdma_reg);
-            opal_free_list_return_mt (&rcache_grdma->reg_list, item);
+            rcache_grdma->resources.deregister_mem(rcache_grdma->resources.reg_data, grdma_reg);
+            opal_free_list_return_mt(&rcache_grdma->reg_list, item);
             return rc;
         }
     }
 
     OPAL_OUTPUT_VERBOSE((MCA_BASE_VERBOSE_TRACE, opal_rcache_base_framework.framework_output,
                          "created new registration %p for region {%p, %p} with flags 0x%x",
-                         (void *)grdma_reg, (void*)base, (void*)bound, grdma_reg->flags));
+                         (void *) grdma_reg, (void *) base, (void *) bound, grdma_reg->flags));
 
     *reg = grdma_reg;
 
     return OPAL_SUCCESS;
 }
 
-static int mca_rcache_grdma_find (mca_rcache_base_module_t *rcache, void *addr,
-                                  size_t size, mca_rcache_base_registration_t **reg)
+static int mca_rcache_grdma_find(mca_rcache_base_module_t *rcache, void *addr, size_t size,
+                                 mca_rcache_base_registration_t **reg)
 {
-    mca_rcache_grdma_module_t *rcache_grdma = (mca_rcache_grdma_module_t*)rcache;
-    unsigned long page_size = opal_getpagesize ();
+    mca_rcache_grdma_module_t *rcache_grdma = (mca_rcache_grdma_module_t *) rcache;
+    unsigned long page_size = opal_getpagesize();
     unsigned char *base, *bound;
     int rc;
 
     base = OPAL_DOWN_ALIGN_PTR(addr, page_size, unsigned char *);
     bound = OPAL_ALIGN_PTR((intptr_t) addr + size - 1, page_size, unsigned char *);
 
-    opal_mutex_lock (&rcache_grdma->cache->vma_module->vma_lock);
-
-    rc = mca_rcache_base_vma_find (rcache_grdma->cache->vma_module, base, bound - base + 1, reg);
-    if(NULL != *reg &&
-            (mca_rcache_grdma_component.leave_pinned ||
-             ((*reg)->flags & MCA_RCACHE_FLAGS_PERSIST) ||
-             ((*reg)->base == base && (*reg)->bound == bound))) {
-        assert(((void*)(*reg)->bound) >= addr);
-        if(0 == (*reg)->ref_count &&
-                mca_rcache_grdma_component.leave_pinned) {
-            opal_list_remove_item(&rcache_grdma->cache->lru_list,
-                                  (opal_list_item_t*)(*reg));
+    opal_mutex_lock(&rcache_grdma->cache->vma_module->vma_lock);
+
+    rc = mca_rcache_base_vma_find(rcache_grdma->cache->vma_module, base, bound - base + 1, reg);
+    if (NULL != *reg
+        && (mca_rcache_grdma_component.leave_pinned || ((*reg)->flags & MCA_RCACHE_FLAGS_PERSIST)
+            || ((*reg)->base == base && (*reg)->bound == bound))) {
+        assert(((void *) (*reg)->bound) >= addr);
+        if (0 == (*reg)->ref_count && mca_rcache_grdma_component.leave_pinned) {
+            opal_list_remove_item(&rcache_grdma->cache->lru_list, (opal_list_item_t *) (*reg));
         }
         rcache_grdma->stat_cache_found++;
-        opal_atomic_add_fetch_32 (&(*reg)->ref_count, 1);
+        opal_atomic_add_fetch_32(&(*reg)->ref_count, 1);
     } else {
         rcache_grdma->stat_cache_notfound++;
     }
 
-    opal_mutex_unlock (&rcache_grdma->cache->vma_module->vma_lock);
+    opal_mutex_unlock(&rcache_grdma->cache->vma_module->vma_lock);
 
     return rc;
 }
 
-static int mca_rcache_grdma_deregister (mca_rcache_base_module_t *rcache,
-                                        mca_rcache_base_registration_t *reg)
+static int mca_rcache_grdma_deregister(mca_rcache_base_module_t *rcache,
+                                       mca_rcache_base_registration_t *reg)
 {
     mca_rcache_grdma_module_t *rcache_grdma = (mca_rcache_grdma_module_t *) rcache;
     int32_t ref_count;
 
-    ref_count = opal_atomic_add_fetch_32 (&reg->ref_count, -1);
+    ref_count = opal_atomic_add_fetch_32(&reg->ref_count, -1);
 
     OPAL_OUTPUT_VERBOSE((MCA_BASE_VERBOSE_TRACE, opal_rcache_base_framework.framework_output,
-                         "returning registration %p, remaining references %d", (void *) reg, ref_count));
+                         "returning registration %p, remaining references %d", (void *) reg,
+                         ref_count));
 
-    assert (ref_count >= 0);
+    assert(ref_count >= 0);
     if (ref_count > 0) {
         return OPAL_SUCCESS;
     }
 
     if (registration_is_cacheable(reg)) {
-        mca_rcache_grdma_add_to_lru (rcache_grdma, reg);
+        mca_rcache_grdma_add_to_lru(rcache_grdma, reg);
         return OPAL_SUCCESS;
     }
 
-    return dereg_mem (reg);
+    return dereg_mem(reg);
 }
 
 struct gc_add_args_t {
@@ -475,10 +475,11 @@ struct gc_add_args_t {
 };
 typedef struct gc_add_args_t gc_add_args_t;
 
-static int mca_rcache_grdma_add_to_gc (mca_rcache_base_registration_t *grdma_reg)
+static int mca_rcache_grdma_add_to_gc(mca_rcache_base_registration_t *grdma_reg)
 {
     mca_rcache_grdma_module_t *rcache_grdma = (mca_rcache_grdma_module_t *) grdma_reg->rcache;
-    uint32_t flags = opal_atomic_fetch_or_32 ((opal_atomic_int32_t *) &grdma_reg->flags, MCA_RCACHE_FLAGS_INVALID);
+    uint32_t flags = opal_atomic_fetch_or_32((opal_atomic_int32_t *) &grdma_reg->flags,
+                                             MCA_RCACHE_FLAGS_INVALID);
 
     if ((flags & MCA_RCACHE_FLAGS_INVALID) || 0 != grdma_reg->ref_count) {
         /* nothing to do */
@@ -488,16 +489,16 @@ static int mca_rcache_grdma_add_to_gc (mca_rcache_base_registration_t *grdma_reg
     /* This may be called from free() so avoid recursively calling into free by just
      * shifting this registration into the garbage collection list. The cleanup will
      * be done on the next registration attempt. */
-    if (registration_flags_cacheable (flags)) {
-        mca_rcache_grdma_remove_from_lru (rcache_grdma, grdma_reg);
+    if (registration_flags_cacheable(flags)) {
+        mca_rcache_grdma_remove_from_lru(rcache_grdma, grdma_reg);
     }
 
-    opal_lifo_push_atomic (&rcache_grdma->cache->gc_lifo, (opal_list_item_t *) grdma_reg);
+    opal_lifo_push_atomic(&rcache_grdma->cache->gc_lifo, (opal_list_item_t *) grdma_reg);
 
     return OPAL_SUCCESS;
 }
 
-static int gc_add (mca_rcache_base_registration_t *grdma_reg, void *ctx)
+static int gc_add(mca_rcache_base_registration_t *grdma_reg, void *ctx)
 {
     gc_add_args_t *args = (gc_add_args_t *) ctx;
 
@@ -516,69 +517,93 @@ static int gc_add (mca_rcache_base_registration_t *grdma_reg, void *ctx)
         return OPAL_ERROR;
     }
 
-    return mca_rcache_grdma_add_to_gc (grdma_reg);
+    return mca_rcache_grdma_add_to_gc(grdma_reg);
 }
 
-static int mca_rcache_grdma_invalidate_range (mca_rcache_base_module_t *rcache,
-                                              void *base, size_t size)
+static int mca_rcache_grdma_invalidate_range(mca_rcache_base_module_t *rcache, void *base,
+                                             size_t size)
 {
     mca_rcache_grdma_module_t *rcache_grdma = (mca_rcache_grdma_module_t *) rcache;
     gc_add_args_t args = {.base = base, .size = size};
-    return mca_rcache_base_vma_iterate (rcache_grdma->cache->vma_module, base, size, true, gc_add, &args);
+    return mca_rcache_base_vma_iterate(rcache_grdma->cache->vma_module, base, size, true, gc_add,
+                                       &args);
+}
+
+/* Check to see if the memory was freed between the time it was stored in
+ * the registration cache and now.  Return true if the memory was previously
+ * freed.  This is indicated by the BUFFER_ID value in the registration cache
+ * not matching the BUFFER_ID of the buffer we are checking.  Return false
+ * if the registration is still good.
+ */
+static bool mca_rcache_accelerator_previously_freed_memory(mca_rcache_base_registration_t *reg)
+{
+    int res;
+    opal_accelerator_buffer_id_t buf_id;
+    unsigned char *dbuf = reg->base;
+    res = opal_accelerator.get_buffer_id(MCA_ACCELERATOR_NO_DEVICE_ID, dbuf, &buf_id);
+    if (OPAL_UNLIKELY(res != OPAL_SUCCESS)) {
+        return true;
+    }
+    if (buf_id != reg->gpu_bufID) {
+        return true;
+    } else {
+        return false;
+    }
 }
 
 /* Make sure this registration request is not stale.  In other words, ensure
  * that we do not have a cuMemAlloc, cuMemFree, cuMemAlloc state.  If we do
  * kick out the regisrations and deregister.  This function needs to be called
  * with the rcache->vma_module->vma_lock held. */
-#if OPAL_CUDA_GDR_SUPPORT
-
-static int check_for_cuda_freed_memory (mca_rcache_base_module_t *rcache, void *addr, size_t size)
+static int check_for_accelerator_freed_memory(mca_rcache_base_module_t *rcache, void *addr, size_t size)
 {
     mca_rcache_grdma_module_t *rcache_grdma = (mca_rcache_grdma_module_t *) rcache;
     mca_rcache_base_registration_t *reg;
 
-    mca_rcache_base_vma_find (rcache_grdma->cache->vma_module, addr, size, &reg);
+    mca_rcache_base_vma_find(rcache_grdma->cache->vma_module, addr, size, &reg);
     if (NULL == reg) {
         return OPAL_SUCCESS;
     }
 
     /* If not previously freed memory, just return 0 */
-    if (!(mca_common_cuda_previously_freed_memory(reg))) {
+    if (!(mca_rcache_accelerator_previously_freed_memory(reg))) {
         return OPAL_SUCCESS;
     }
 
     /* This memory has been freed.  Find all registrations and delete. Ensure they are deregistered
      * now by passing dereg_mem as the delete function. This is safe because the vma lock is
      * recursive and this is only called from register. */
-    return mca_rcache_base_vma_iterate (rcache_grdma->cache->vma_module, addr, size, true, gc_add, NULL);
+    return mca_rcache_base_vma_iterate(rcache_grdma->cache->vma_module, addr, size, true, gc_add,
+                                       NULL);
 }
-#endif /* OPAL_CUDA_GDR_SUPPORT */
 
-static void mca_rcache_grdma_finalize (mca_rcache_base_module_t *rcache)
+static void mca_rcache_grdma_finalize(mca_rcache_base_module_t *rcache)
 {
-    mca_rcache_grdma_module_t *rcache_grdma = (mca_rcache_grdma_module_t*)rcache;
+    mca_rcache_grdma_module_t *rcache_grdma = (mca_rcache_grdma_module_t *) rcache;
 
     /* Statistic */
     if (true == mca_rcache_grdma_component.print_stats) {
-        opal_output(0, "%s grdma: stats "
-                "(hit/miss/found/not found/evicted/tree size): %d/%d/%d/%d/%d/%ld\n",
-                OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),
-                rcache_grdma->stat_cache_hit, rcache_grdma->stat_cache_miss,
-                rcache_grdma->stat_cache_found, rcache_grdma->stat_cache_notfound,
-                    rcache_grdma->stat_evicted, (long) mca_rcache_base_vma_size (rcache_grdma->cache->vma_module));
+        opal_output(0,
+                    "%s grdma: stats "
+                    "(hit/miss/found/not found/evicted/tree size): %d/%d/%d/%d/%d/%ld\n",
+                    OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), rcache_grdma->stat_cache_hit,
+                    rcache_grdma->stat_cache_miss, rcache_grdma->stat_cache_found,
+                    rcache_grdma->stat_cache_notfound, rcache_grdma->stat_evicted,
+                    (long) mca_rcache_base_vma_size(rcache_grdma->cache->vma_module));
     }
 
-   do_unregistration_gc (&rcache_grdma->super);
+    do_unregistration_gc(&rcache_grdma->super);
 
-    (void) mca_rcache_base_vma_iterate (rcache_grdma->cache->vma_module, NULL, (size_t) -1, true,
-                                        gc_add, (void *) rcache);
-    do_unregistration_gc (rcache);
+    (void) mca_rcache_base_vma_iterate(rcache_grdma->cache->vma_module, NULL, (size_t) -1, true,
+                                       gc_add, (void *) rcache);
+    do_unregistration_gc(rcache);
 
     OBJ_RELEASE(rcache_grdma->cache);
 
     OBJ_DESTRUCT(&rcache_grdma->reg_list);
 
+    mca_rcache_base_module_fini(rcache);
+
     /* this rcache was allocated by grdma_init in rcache_grdma_component.c */
     free(rcache);
 }
diff --git a/opal/mca/rcache/rcache.h b/opal/mca/rcache/rcache.h
index ed68deca7ac..fbff737b44d 100644
--- a/opal/mca/rcache/rcache.h
+++ b/opal/mca/rcache/rcache.h
@@ -20,15 +20,15 @@
  * $HEADER$
  */
 /**
-  * @file
-  * Description of the Registration Cache framework
-  */
+ * @file
+ * Description of the Registration Cache framework
+ */
 #ifndef MCA_RCACHE_H
 #define MCA_RCACHE_H
 #include "opal/mca/mca.h"
 #include "opal/mca/mpool/mpool.h"
 #include "opal/mca/threads/mutex.h"
-
+#include "opal/mca/accelerator/accelerator.h"
 
 /* forward-declaration of rcache module structure */
 struct mca_rcache_base_module_t;
@@ -36,47 +36,47 @@ typedef struct mca_rcache_base_module_t mca_rcache_base_module_t;
 
 enum {
     /** bypass the cache when registering */
-    MCA_RCACHE_FLAGS_CACHE_BYPASS      = 0x0001,
+    MCA_RCACHE_FLAGS_CACHE_BYPASS = 0x0001,
     /** persistent registration */
-    MCA_RCACHE_FLAGS_PERSIST           = 0x0002,
-    /** registation requires strong ordering (disables relaxed ordering) */
-    MCA_RCACHE_FLAGS_SO_MEM            = 0x0004,
-    /** address range is cuda buffer */
-    MCA_RCACHE_FLAGS_CUDA_GPU_MEM      = 0x0008,
-    /** register with common cuda */
-    MCA_RCACHE_FLAGS_CUDA_REGISTER_MEM = 0x0010,
+    MCA_RCACHE_FLAGS_PERSIST = 0x0002,
+    /** registration requires strong ordering (disables relaxed ordering) */
+    MCA_RCACHE_FLAGS_SO_MEM = 0x0004,
+    /** address range is accelerator buffer */
+    MCA_RCACHE_FLAGS_ACCELERATOR_MEM = 0x0008,
+    /** register with accelerator framework */
+    MCA_RCACHE_FLAGS_ACCELERATOR_REGISTER_MEM = 0x0010,
     /** invalid registration (no valid for passing to rcache register) */
-    MCA_RCACHE_FLAGS_INVALID           = 0x0080,
+    MCA_RCACHE_FLAGS_INVALID = 0x0080,
     /** reserved for rcache module */
-    MCA_RCACHE_FLAGS_MOD_RESV0         = 0x0100,
+    MCA_RCACHE_FLAGS_MOD_RESV0 = 0x0100,
     /** reserved for rcache module */
-    MCA_RCACHE_FLAGS_MOD_RESV1         = 0x0200,
+    MCA_RCACHE_FLAGS_MOD_RESV1 = 0x0200,
     /** reserved for rcache module */
-    MCA_RCACHE_FLAGS_MOD_RESV2         = 0x0400,
+    MCA_RCACHE_FLAGS_MOD_RESV2 = 0x0400,
     /** reserved for rcache module */
-    MCA_RCACHE_FLAGS_MOD_RESV3         = 0x0800,
+    MCA_RCACHE_FLAGS_MOD_RESV3 = 0x0800,
     /** reserved for register function */
-    MCA_RCACHE_FLAGS_RESV0             = 0x1000,
+    MCA_RCACHE_FLAGS_RESV0 = 0x1000,
     /** reserved for register function */
-    MCA_RCACHE_FLAGS_RESV1             = 0x2000,
+    MCA_RCACHE_FLAGS_RESV1 = 0x2000,
     /** reserved for register function */
-    MCA_RCACHE_FLAGS_RESV2             = 0x4000,
+    MCA_RCACHE_FLAGS_RESV2 = 0x4000,
     /** reserved for register function */
-    MCA_RCACHE_FLAGS_RESV3             = 0x8000,
+    MCA_RCACHE_FLAGS_RESV3 = 0x8000,
 };
 
 /** access flags */
 enum {
     /** register for local write */
-    MCA_RCACHE_ACCESS_LOCAL_WRITE   = 0x01,
+    MCA_RCACHE_ACCESS_LOCAL_WRITE = 0x01,
     /** register for remote read */
-    MCA_RCACHE_ACCESS_REMOTE_READ   = 0x02,
+    MCA_RCACHE_ACCESS_REMOTE_READ = 0x02,
     /** register for remote write */
-    MCA_RCACHE_ACCESS_REMOTE_WRITE  = 0x04,
+    MCA_RCACHE_ACCESS_REMOTE_WRITE = 0x04,
     /** register for local/remote atomic operations */
     MCA_RCACHE_ACCESS_REMOTE_ATOMIC = 0x08,
     /** register for any access */
-    MCA_RCACHE_ACCESS_ANY           = 0x0f,
+    MCA_RCACHE_ACCESS_ANY = 0x0f,
 };
 
 /** base class for all rcache registrations */
@@ -89,7 +89,7 @@ struct mca_rcache_base_registration_t {
     unsigned char *base;
     /** bound of registered region */
     unsigned char *bound;
-    /** artifact of old mpool/rcache architecture. used by cuda code */
+    /** artifact of old mpool/rcache architecture. */
     unsigned char *alloc_base;
     /** number of outstanding references */
     opal_atomic_int32_t ref_count;
@@ -97,10 +97,8 @@ struct mca_rcache_base_registration_t {
     opal_atomic_uint32_t flags;
     /** internal rcache context */
     void *rcache_context;
-#if OPAL_CUDA_GDR_SUPPORT
-    /** CUDA gpu buffer identifier */
-    unsigned long long gpu_bufID;
-#endif /* OPAL_CUDA_GDR_SUPPORT */
+    /** Accelerator buffer identifier */
+    opal_accelerator_buffer_id_t gpu_bufID;
     /** registration access flags */
     int32_t access_flags;
     unsigned char padding[64];
@@ -111,47 +109,47 @@ typedef struct mca_rcache_base_registration_t mca_rcache_base_registration_t;
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(mca_rcache_base_registration_t);
 
 struct mca_rcache_base_resources_t {
-    char  *cache_name;
-    void  *reg_data;
+    char *cache_name;
+    void *reg_data;
     size_t sizeof_reg;
-    int (*register_mem) (void *reg_data, void *base, size_t size,
-                         mca_rcache_base_registration_t *reg);
-    int (*deregister_mem) (void *reg_data, mca_rcache_base_registration_t *reg);
+    int (*register_mem)(void *reg_data, void *base, size_t size,
+                        mca_rcache_base_registration_t *reg);
+    int (*deregister_mem)(void *reg_data, mca_rcache_base_registration_t *reg);
 };
 typedef struct mca_rcache_base_resources_t mca_rcache_base_resources_t;
 
-
 /**
  * component initialize
  */
-typedef struct mca_rcache_base_module_t *(*mca_rcache_base_component_init_fn_t)(mca_rcache_base_resources_t *);
+typedef struct mca_rcache_base_module_t *(*mca_rcache_base_component_init_fn_t)(
+    mca_rcache_base_resources_t *);
 
 /**
-  * register memory
-  */
-typedef int (*mca_rcache_base_module_register_fn_t) (mca_rcache_base_module_t *rcache,
-                                                     void *addr, size_t size, uint32_t flags,
-                                                     int32_t access_flags,
-                                                     mca_rcache_base_registration_t **reg);
+ * register memory
+ */
+typedef int (*mca_rcache_base_module_register_fn_t)(mca_rcache_base_module_t *rcache, void *addr,
+                                                    size_t size, uint32_t flags,
+                                                    int32_t access_flags,
+                                                    mca_rcache_base_registration_t **reg);
 
 /**
-  * deregister memory
-  */
-typedef int (*mca_rcache_base_module_deregister_fn_t) (mca_rcache_base_module_t *rcache,
-                                                       mca_rcache_base_registration_t *reg);
+ * deregister memory
+ */
+typedef int (*mca_rcache_base_module_deregister_fn_t)(mca_rcache_base_module_t *rcache,
+                                                      mca_rcache_base_registration_t *reg);
 
 /**
  * find registration in this memory pool
  */
 
-typedef int (*mca_rcache_base_module_find_fn_t) (mca_rcache_base_module_t *rcache, void *addr,
-                                                 size_t size, mca_rcache_base_registration_t **reg);
+typedef int (*mca_rcache_base_module_find_fn_t)(mca_rcache_base_module_t *rcache, void *addr,
+                                                size_t size, mca_rcache_base_registration_t **reg);
 
 /**
  * release memory region
  */
-typedef int (*mca_rcache_base_module_invalidate_range_fn_t) (mca_rcache_base_module_t *rcache,
-                                                             void *addr, size_t size);
+typedef int (*mca_rcache_base_module_invalidate_range_fn_t)(mca_rcache_base_module_t *rcache,
+                                                            void *addr, size_t size);
 
 /**
  * evict one stale registration
@@ -159,11 +157,11 @@ typedef int (*mca_rcache_base_module_invalidate_range_fn_t) (mca_rcache_base_mod
  * @returns true if successful
  * @returns false if no registration could be evicted
  */
-typedef bool (*mca_rcache_base_module_evict_fn_t) (mca_rcache_base_module_t *rcache);
+typedef bool (*mca_rcache_base_module_evict_fn_t)(mca_rcache_base_module_t *rcache);
 
 /**
-  * finalize
-  */
+ * finalize
+ */
 typedef void (*mca_rcache_base_module_finalize_fn_t)(mca_rcache_base_module_t *rcache);
 
 /**
@@ -171,9 +169,9 @@ typedef void (*mca_rcache_base_module_finalize_fn_t)(mca_rcache_base_module_t *r
  * open/close/init functions
  */
 
-struct mca_rcache_base_component_2_0_0_t{
-    mca_base_component_t rcache_version;      /**< version */
-    mca_base_component_data_t rcache_data; /**<metadata */
+struct mca_rcache_base_component_2_0_0_t {
+    mca_base_component_t rcache_version;             /**< version */
+    mca_base_component_data_t rcache_data;           /**<metadata */
     mca_rcache_base_component_init_fn_t rcache_init; /**<init function */
 };
 
@@ -181,7 +179,6 @@ typedef struct mca_rcache_base_component_2_0_0_t mca_rcache_base_component_2_0_0
 
 typedef struct mca_rcache_base_component_2_0_0_t mca_rcache_base_component_t;
 
-
 /**
  * rcache module descriptor
  */
@@ -197,15 +194,15 @@ struct mca_rcache_base_module_t {
     opal_mutex_t lock;
 };
 
-#define RCACHE_MAJOR_VERSION 3
-#define RCACHE_MINOR_VERSION 0
+#define RCACHE_MAJOR_VERSION   3
+#define RCACHE_MINOR_VERSION   0
 #define RCACHE_RELEASE_VERSION 0
 
 /**
  * Macro for use in components that are of type rcache
  */
-#define MCA_RCACHE_BASE_VERSION_3_0_0 \
-    OPAL_MCA_BASE_VERSION_2_1_0("rcache", RCACHE_MAJOR_VERSION, RCACHE_MAJOR_VERSION, RCACHE_RELEASE_VERSION)
+#define MCA_RCACHE_BASE_VERSION_3_0_0                                                 \
+    OPAL_MCA_BASE_VERSION_2_1_0("rcache", RCACHE_MAJOR_VERSION, RCACHE_MAJOR_VERSION, \
+                                RCACHE_RELEASE_VERSION)
 
 #endif /* MCA_RCACHE_H */
-
diff --git a/opal/mca/rcache/rgpusm/Makefile.am b/opal/mca/rcache/rgpusm/Makefile.am
index 8afc4235c3f..dde814113be 100644
--- a/opal/mca/rcache/rgpusm/Makefile.am
+++ b/opal/mca/rcache/rgpusm/Makefile.am
@@ -12,6 +12,7 @@
 # Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2012      NVIDIA Corporation.  All rights reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -45,15 +46,11 @@ endif
 mcacomponentdir = $(opallibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_rcache_rgpusm_la_SOURCES = $(sources)
-mca_rcache_rgpusm_la_LDFLAGS = -module -avoid-version
-mca_rcache_rgpusm_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
+mca_rcache_rgpusm_la_LDFLAGS = -module -avoid-version $(rcache_rgpusm_LDFLAGS)
+mca_rcache_rgpusm_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
 	$(rcache_rgpusm_LIBS)
-if OPAL_cuda_support
-mca_rcache_rgpusm_la_LIBADD += \
-    $(OPAL_TOP_BUILDDIR)/opal/mca/common/cuda/lib@OPAL_LIB_PREFIX@mca_common_cuda.la
-endif
 
 noinst_LTLIBRARIES = $(component_noinst)
 libmca_rcache_rgpusm_la_SOURCES = $(sources)
-libmca_rcache_rgpusm_la_LDFLAGS = -module -avoid-version
+libmca_rcache_rgpusm_la_LDFLAGS = -module -avoid-version $(rcache_rgpusm_LDFLAGS)
 libmca_rcache_rgpusm_la_LIBADD = $(rcache_rgpusm_LIBS)
diff --git a/opal/mca/rcache/rgpusm/configure.m4 b/opal/mca/rcache/rgpusm/configure.m4
deleted file mode 100644
index a9bce3c39dd..00000000000
--- a/opal/mca/rcache/rgpusm/configure.m4
+++ /dev/null
@@ -1,27 +0,0 @@
-# -*- shell-script -*-
-#
-# Copyright (c) 2012-2015 NVIDIA Corporation.  All rights reserved.
-# Copyright (c) 2015      Los Alamos National Security, LLC. All rights
-#                         reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-#
-# If CUDA support was requested, then build the CUDA memory pools.
-# This code checks the variable CUDA_SUPPORT which was set earlier in
-# the configure sequence by the opal_configure_options.m4 code.
-#
-
-AC_DEFUN([MCA_opal_rcache_rgpusm_CONFIG],[
-    AC_CONFIG_FILES([opal/mca/rcache/rgpusm/Makefile])
-
-    # Use CUDA_SUPPORT which was filled in by the opal configure code.
-    AS_IF([test "x$CUDA_SUPPORT" = "x1"],
-          [$1],
-          [$2])
-
-])dnl
diff --git a/opal/mca/rcache/rgpusm/rcache_rgpusm.h b/opal/mca/rcache/rgpusm/rcache_rgpusm.h
index 268d3d1548c..18e645f871d 100644
--- a/opal/mca/rcache/rgpusm/rcache_rgpusm.h
+++ b/opal/mca/rcache/rgpusm/rcache_rgpusm.h
@@ -28,15 +28,15 @@
 #define MCA_RCACHE_RGPUSM_H
 
 #include "opal_config.h"
-#include "opal/class/opal_list.h"
 #include "opal/class/opal_free_list.h"
+#include "opal/class/opal_list.h"
 #include "opal/mca/rcache/rcache.h"
 
 BEGIN_C_DECLS
 
 struct mca_rcache_rgpusm_component_t {
     mca_rcache_base_component_t super;
-    char* rcache_name;
+    char *rcache_name;
     unsigned long long rcache_size_limit;
     bool print_stats;
     int leave_pinned;
@@ -59,7 +59,8 @@ struct mca_rcache_rgpusm_module_t {
     uint32_t stat_cache_found;
     uint32_t stat_cache_notfound;
     mca_rcache_base_vma_module_t *vma_module;
-}; typedef struct mca_rcache_rgpusm_module_t mca_rcache_rgpusm_module_t;
+};
+typedef struct mca_rcache_rgpusm_module_t mca_rcache_rgpusm_module_t;
 
 /*
  *  Initializes the rcache module.
@@ -67,34 +68,34 @@ struct mca_rcache_rgpusm_module_t {
 void mca_rcache_rgpusm_module_init(mca_rcache_rgpusm_module_t *rcache);
 
 /**
-  * register block of memory
-  */
-int mca_rcache_rgpusm_register(mca_rcache_base_module_t* rcache, void *addr,
-        size_t size, uint32_t flags, int32_t access_flags, mca_rcache_base_registration_t **reg);
+ * register block of memory
+ */
+int mca_rcache_rgpusm_register(mca_rcache_base_module_t *rcache, void *addr, size_t size,
+                               uint32_t flags, int32_t access_flags,
+                               mca_rcache_base_registration_t **reg);
 
 /**
  * deregister memory
  */
 int mca_rcache_rgpusm_deregister(mca_rcache_base_module_t *rcache,
-        mca_rcache_base_registration_t *reg);
+                                 mca_rcache_base_registration_t *reg);
 
 /**
-  * free memory allocated by alloc function
-  */
-void mca_rcache_rgpusm_free(mca_rcache_base_module_t *rcache, void * addr,
-        mca_rcache_base_registration_t *reg);
+ * free memory allocated by alloc function
+ */
+void mca_rcache_rgpusm_free(mca_rcache_base_module_t *rcache, void *addr,
+                            mca_rcache_base_registration_t *reg);
 
 /**
  * find registration for a given block of memory
  */
-int mca_rcache_rgpusm_find(struct mca_rcache_base_module_t* rcache, void* addr,
-        size_t size, mca_rcache_base_registration_t **reg);
+int mca_rcache_rgpusm_find(struct mca_rcache_base_module_t *rcache, void *addr, size_t size,
+                           mca_rcache_base_registration_t **reg);
 
 /**
  * unregister all registration covering the block of memory
  */
-int mca_rcache_rgpusm_release_memory(mca_rcache_base_module_t* rcache, void *base,
-        size_t size);
+int mca_rcache_rgpusm_release_memory(mca_rcache_base_module_t *rcache, void *base, size_t size);
 
 /**
  * finalize rcache
diff --git a/opal/mca/rcache/rgpusm/rcache_rgpusm_component.c b/opal/mca/rcache/rgpusm/rcache_rgpusm_component.c
index 8794d4f047d..7ab84873763 100644
--- a/opal/mca/rcache/rgpusm/rcache_rgpusm_component.c
+++ b/opal/mca/rcache/rgpusm/rcache_rgpusm_component.c
@@ -28,10 +28,10 @@
 #include "opal/mca/base/base.h"
 #include "rcache_rgpusm.h"
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 #ifdef HAVE_MALLOC_H
-#include <malloc.h>
+#    include <malloc.h>
 #endif
 
 /*
@@ -40,37 +40,34 @@
 static int rgpusm_open(void);
 static int rgpusm_close(void);
 static int rgpusm_register(void);
-static mca_rcache_base_module_t* rgpusm_init(struct mca_rcache_base_resources_t* resources);
+static mca_rcache_base_module_t *rgpusm_init(struct mca_rcache_base_resources_t *resources);
 
 static int opal_rcache_rgpusm_verbose = 0;
 
 mca_rcache_rgpusm_component_t mca_rcache_rgpusm_component = {
-    {
-        /* First, the mca_base_component_t struct containing meta
-           information about the component itself */
-
-        .rcache_version = {
-            MCA_RCACHE_BASE_VERSION_3_0_0,
-
-            .mca_component_name = "rgpusm",
-            MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
-                                  OPAL_RELEASE_VERSION),
-            .mca_open_component = rgpusm_open,
-            .mca_close_component = rgpusm_close,
-            .mca_register_component_params = rgpusm_register,
-        },
-        .rcache_data = {
-            /* The component is checkpoint ready */
-            MCA_BASE_METADATA_PARAM_CHECKPOINT
-        },
-
-        .rcache_init = rgpusm_init
-    }
-};
+    {/* First, the mca_base_component_t struct containing meta
+        information about the component itself */
+
+     .rcache_version =
+         {
+             MCA_RCACHE_BASE_VERSION_3_0_0,
+
+             .mca_component_name = "rgpusm",
+             MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
+                                   OPAL_RELEASE_VERSION),
+             .mca_open_component = rgpusm_open,
+             .mca_close_component = rgpusm_close,
+             .mca_register_component_params = rgpusm_register,
+         },
+     .rcache_data =
+         {/* The component is checkpoint ready */
+          MCA_BASE_METADATA_PARAM_CHECKPOINT},
+
+     .rcache_init = rgpusm_init}};
 
 /**
-  * component open/close/init function
-  */
+ * component open/close/init function
+ */
 static int rgpusm_open(void)
 {
     mca_rcache_rgpusm_component.output = opal_output_open(NULL);
@@ -79,42 +76,37 @@ static int rgpusm_open(void)
     return OPAL_SUCCESS;
 }
 
-
 static int rgpusm_register(void)
 {
     mca_rcache_rgpusm_component.rcache_name = "vma";
-    (void) mca_base_component_var_register(&mca_rcache_rgpusm_component.super.rcache_version,
-                                           "rcache_name",
-                                           "The name of the registration cache the rcache should use",
-                                           MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
-                                           &mca_rcache_rgpusm_component.rcache_name);
+    (void)
+        mca_base_component_var_register(&mca_rcache_rgpusm_component.super.rcache_version,
+                                        "rcache_name",
+                                        "The name of the registration cache the rcache should use",
+                                        MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_9,
+                                        MCA_BASE_VAR_SCOPE_READONLY,
+                                        &mca_rcache_rgpusm_component.rcache_name);
     mca_rcache_rgpusm_component.rcache_size_limit = 0;
     (void) mca_base_component_var_register(&mca_rcache_rgpusm_component.super.rcache_version,
                                            "rcache_size_limit",
                                            "the maximum size of registration cache in bytes. "
                                            "0 is unlimited (default 0)",
                                            MCA_BASE_VAR_TYPE_UNSIGNED_LONG_LONG, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
                                            &mca_rcache_rgpusm_component.rcache_size_limit);
 
     mca_rcache_rgpusm_component.leave_pinned = 1;
-    (void) mca_base_component_var_register(&mca_rcache_rgpusm_component.super.rcache_version,
-                                           "leave_pinned",
-                                           "Whether to keep memory handles around or release them when done. ",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
-                                           &mca_rcache_rgpusm_component.leave_pinned);
+    (void) mca_base_component_var_register(
+        &mca_rcache_rgpusm_component.super.rcache_version, "leave_pinned",
+        "Whether to keep memory handles around or release them when done. ", MCA_BASE_VAR_TYPE_INT,
+        NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+        &mca_rcache_rgpusm_component.leave_pinned);
 
     mca_rcache_rgpusm_component.print_stats = false;
     (void) mca_base_component_var_register(&mca_rcache_rgpusm_component.super.rcache_version,
                                            "print_stats",
                                            "print pool usage statistics at the end of the run",
-                                           MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9,
                                            MCA_BASE_VAR_SCOPE_READONLY,
                                            &mca_rcache_rgpusm_component.print_stats);
 
@@ -122,39 +114,36 @@ static int rgpusm_register(void)
     opal_rcache_rgpusm_verbose = 0;
     (void) mca_base_component_var_register(&mca_rcache_rgpusm_component.super.rcache_version,
                                            "verbose", "Set level of rcache rgpusm verbosity",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9,
                                            MCA_BASE_VAR_SCOPE_READONLY,
                                            &opal_rcache_rgpusm_verbose);
 
     /* Force emptying of entire registration cache when it gets full */
     mca_rcache_rgpusm_component.empty_cache = false;
-    (void) mca_base_component_var_register(&mca_rcache_rgpusm_component.super.rcache_version,
-                                           "empty_cache", "When set, empty entire registration cache when it is full",
-                                           MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
-                                           OPAL_INFO_LVL_5,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
-                                           &mca_rcache_rgpusm_component.empty_cache);
+    (void)
+        mca_base_component_var_register(&mca_rcache_rgpusm_component.super.rcache_version,
+                                        "empty_cache",
+                                        "When set, empty entire registration cache when it is full",
+                                        MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                        MCA_BASE_VAR_SCOPE_READONLY,
+                                        &mca_rcache_rgpusm_component.empty_cache);
 
     return OPAL_SUCCESS;
 }
 
-
 static int rgpusm_close(void)
 {
     return OPAL_SUCCESS;
 }
 
-
-static mca_rcache_base_module_t* rgpusm_init(
-     struct mca_rcache_base_resources_t *resources)
+static mca_rcache_base_module_t *rgpusm_init(struct mca_rcache_base_resources_t *resources)
 {
-    mca_rcache_rgpusm_module_t* rcache_module;
+    mca_rcache_rgpusm_module_t *rcache_module;
 
     /* ignore passed in resource structure */
     (void) resources;
 
-    rcache_module = (mca_rcache_rgpusm_module_t *) calloc (1, sizeof (*rcache_module));
+    rcache_module = (mca_rcache_rgpusm_module_t *) calloc(1, sizeof(*rcache_module));
     if (NULL == rcache_module) {
         return NULL;
     }
diff --git a/opal/mca/rcache/rgpusm/rcache_rgpusm_module.c b/opal/mca/rcache/rgpusm/rcache_rgpusm_module.c
index 38645cb2768..d0f55771e6b 100644
--- a/opal/mca/rcache/rgpusm/rcache_rgpusm_module.c
+++ b/opal/mca/rcache/rgpusm/rcache_rgpusm_module.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2013 The University of Tennessee and The University
+ * Copyright (c) 2004-2024 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -17,7 +17,8 @@
  * Copyright (c) 2012-2015 NVIDIA Corporation.  All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
  *                         reserved.
- *
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2023      Advanced Micro Devices, Inc. All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -84,38 +85,85 @@
 #include <errno.h>
 #include <string.h>
 #ifdef HAVE_MALLOC_H
-#include <malloc.h>
+#    include <malloc.h>
 #endif
-#include "opal/util/proc.h"
-#include "opal/mca/rcache/rcache.h"
+#include "opal/include/opal/opal_gpu.h"
 #include "opal/mca/rcache/base/base.h"
-#include "opal/mca/rcache/base/base.h"
-#include "opal/mca/common/cuda/common_cuda.h"
+#include "opal/mca/rcache/rcache.h"
+#include "opal/util/proc.h"
+
+/*
+ * Open a memory handle that refers to remote memory so we can get an address
+ * that works on the local side.  This is the registration function for the
+ * remote side of a transfer.  newreg contains the new handle.  hddrreg contains
+ * the memory handle that was received from the remote side.
+ */
+static int mca_rcache_rgpusm_open_mem_handle(void *base, size_t size, mca_rcache_base_registration_t *newreg)
+{
+    int result;
+    mca_opal_gpu_reg_t *gpu_newreg = (mca_opal_gpu_reg_t *) newreg;
+
+    // Note: It is expected that the ipc_handle object was created previously in the smcuda component
+    result = opal_accelerator.open_ipc_handle(MCA_ACCELERATOR_NO_DEVICE_ID, &gpu_newreg->data.ipcHandle,
+                                              (void**)&newreg->alloc_base);
+    if (OPAL_ERR_WOULD_BLOCK == result) {
+        // ERROR_ALREADY_MAPPED
+        opal_output_verbose(10, mca_rcache_rgpusm_component.output,
+                            "open_ipc_mem_handle returned OPAL_ERR_WOULD_BLOCK for "
+                            "p=%p,size=%d: notify memory pool\n",
+                            base, (int) size);
+        return result;
+    }
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != result)) {
+        opal_output_verbose(10, mca_rcache_rgpusm_component.output,
+                            "open_ipc_handle failed: base=%p (remote base=%p,size=%d)",
+                            (void*)newreg->alloc_base, base, (int) size);
+        /* Currently, this is a non-recoverable error */
+        return OPAL_ERROR;
+    }
+
+    opal_output_verbose(10, mca_rcache_rgpusm_component.output,
+                        "open_ipc_handle passed: base=%p (remote base=%p,size=%d)",
+                        (void*)newreg->alloc_base, base, (int) size);
+
+    return OPAL_SUCCESS;
+}
+
+/*
+ * Close a memory handle that refers to remote memory.
+ */
+static int mca_rcache_rgpusm_close_mem_handle(void *reg_data, mca_rcache_base_registration_t *reg)
+{
+    mca_opal_gpu_reg_t *gpu_reg = (mca_opal_gpu_reg_t *) reg;
 
+    OBJ_DESTRUCT(&gpu_reg->data.ipcHandle);
+    
+    return OPAL_SUCCESS;
+}
 
 static int mca_rcache_rgpusm_deregister_no_lock(struct mca_rcache_base_module_t *,
-                                               mca_rcache_base_registration_t *);
-static inline bool mca_rcache_rgpusm_deregister_lru (mca_rcache_base_module_t *rcache) {
+                                                mca_rcache_base_registration_t *);
+static inline bool mca_rcache_rgpusm_deregister_lru(mca_rcache_base_module_t *rcache)
+{
     mca_rcache_rgpusm_module_t *rcache_rgpusm = (mca_rcache_rgpusm_module_t *) rcache;
     mca_rcache_base_registration_t *old_reg;
     int rc;
 
     /* Remove the registration from the cache and list before
        deregistering the memory */
-    old_reg = (mca_rcache_base_registration_t*)
-        opal_list_remove_first (&rcache_rgpusm->lru_list);
+    old_reg = (mca_rcache_base_registration_t *) opal_list_remove_first(&rcache_rgpusm->lru_list);
     if (NULL == old_reg) {
         opal_output_verbose(10, mca_rcache_rgpusm_component.output,
                             "RGPUSM: The LRU list is empty. There is nothing to deregister");
         return false;
     }
 
-    mca_rcache_base_vma_delete (rcache_rgpusm->vma_module, old_reg);
+    mca_rcache_base_vma_delete(rcache_rgpusm->vma_module, old_reg);
 
     /* Drop the rcache lock while we deregister the memory */
     OPAL_THREAD_UNLOCK(&rcache->lock);
     assert(old_reg->ref_count == 0);
-    rc = cuda_closememhandle (NULL, old_reg);
+    rc = mca_rcache_rgpusm_close_mem_handle(NULL, old_reg);
     OPAL_THREAD_LOCK(&rcache->lock);
 
     /* This introduces a potential leak of registrations if
@@ -124,41 +172,37 @@ static inline bool mca_rcache_rgpusm_deregister_lru (mca_rcache_base_module_t *r
     if (OPAL_SUCCESS != rc) {
         opal_output_verbose(10, mca_rcache_rgpusm_component.output,
                             "RGPUSM: Failed to deregister the memory addr=%p, size=%d",
-                            old_reg->base, (int)(old_reg->bound - old_reg->base + 1));
+                            (void*)old_reg->base, (int) (old_reg->bound - old_reg->base + 1));
         return false;
     }
 
-    opal_free_list_return (&rcache_rgpusm->reg_list,
-                           (opal_free_list_item_t*)old_reg);
+    opal_free_list_return(&rcache_rgpusm->reg_list, (opal_free_list_item_t *) old_reg);
     rcache_rgpusm->stat_evicted++;
 
     return true;
 }
 
-
 /*
  *  Initializes the rcache module.
  */
-void mca_rcache_rgpusm_module_init(mca_rcache_rgpusm_module_t* rcache)
+void mca_rcache_rgpusm_module_init(mca_rcache_rgpusm_module_t *rcache)
 {
+    mca_rcache_base_module_init(&rcache->super);
     rcache->super.rcache_component = &mca_rcache_rgpusm_component.super;
     rcache->super.rcache_register = mca_rcache_rgpusm_register;
     rcache->super.rcache_find = mca_rcache_rgpusm_find;
     rcache->super.rcache_deregister = mca_rcache_rgpusm_deregister;
     rcache->super.rcache_finalize = mca_rcache_rgpusm_finalize;
-    rcache->vma_module = mca_rcache_base_vma_module_alloc ();
+    rcache->vma_module = mca_rcache_base_vma_module_alloc();
 
     OBJ_CONSTRUCT(&rcache->reg_list, opal_free_list_t);
-    opal_free_list_init (&rcache->reg_list, sizeof(struct mca_rcache_common_cuda_reg_t),
-            opal_cache_line_size,
-            OBJ_CLASS(mca_rcache_base_registration_t),
-            0,opal_cache_line_size,
-            0, -1, 32, NULL, 0, NULL, NULL, NULL);
+    opal_free_list_init(&rcache->reg_list, sizeof(struct mca_opal_gpu_reg_t),
+                        opal_cache_line_size, OBJ_CLASS(mca_rcache_base_registration_t), 0,
+                        opal_cache_line_size, 0, -1, 32, NULL, 0, NULL, NULL, NULL);
     OBJ_CONSTRUCT(&rcache->lru_list, opal_list_t);
     rcache->stat_cache_hit = rcache->stat_cache_miss = rcache->stat_evicted = 0;
     rcache->stat_cache_found = rcache->stat_cache_notfound = 0;
     rcache->stat_cache_valid = rcache->stat_cache_invalid = 0;
-
 }
 
 /*
@@ -166,22 +210,22 @@ void mca_rcache_rgpusm_module_init(mca_rcache_rgpusm_module_t* rcache)
  * from the remote memory.  It uses the addr and size of the remote
  * memory for caching the registration.
  */
-int mca_rcache_rgpusm_register (mca_rcache_base_module_t *rcache, void *addr,
-                               size_t size, uint32_t flags, int32_t access_flags,
+int mca_rcache_rgpusm_register(mca_rcache_base_module_t *rcache, void *addr, size_t size,
+                               uint32_t flags, int32_t access_flags,
                                mca_rcache_base_registration_t **reg)
 {
-    mca_rcache_rgpusm_module_t *rcache_rgpusm = (mca_rcache_rgpusm_module_t*)rcache;
-    mca_rcache_common_cuda_reg_t *rgpusm_reg;
-    mca_rcache_common_cuda_reg_t *rget_reg;
+    mca_rcache_rgpusm_module_t *rcache_rgpusm = (mca_rcache_rgpusm_module_t *) rcache;
+    mca_opal_gpu_reg_t *rgpusm_reg;
+    mca_opal_gpu_reg_t *rget_reg;
     opal_free_list_item_t *item;
     int rc;
-    int mypeer;  /* just for debugging */
+    int mypeer; /* just for debugging */
 
     /* In order to preserve the signature of the mca_rcache_rgpusm_register
      * function, we are using the **reg variable to not only get back the
      * registration information, but to hand in the memory handle received
      * from the remote side. */
-    rget_reg = (mca_rcache_common_cuda_reg_t *)*reg;
+    rget_reg = (mca_opal_gpu_reg_t *) *reg;
 
     mypeer = flags;
     flags = 0;
@@ -191,44 +235,50 @@ int mca_rcache_rgpusm_register (mca_rcache_base_module_t *rcache, void *addr,
     /* This chunk of code handles the case where leave pinned is not
      * set and we do not use the cache.  This is not typically how we
      * will be running.  This means that one can have an unlimited
-     * number of registrations occuring at the same time.  Since we
+     * number of registrations occurring at the same time.  Since we
      * are not leaving the registrations pinned, the number of
      * registrations is unlimited and there is no need for a cache. */
-    if(!mca_rcache_rgpusm_component.leave_pinned && 0 == mca_rcache_rgpusm_component.rcache_size_limit) {
-        item = opal_free_list_get (&rcache_rgpusm->reg_list);
-        if(NULL == item) {
+    if (!mca_rcache_rgpusm_component.leave_pinned
+        && 0 == mca_rcache_rgpusm_component.rcache_size_limit) {
+        item = opal_free_list_get(&rcache_rgpusm->reg_list);
+        if (NULL == item) {
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
-        rgpusm_reg = (mca_rcache_common_cuda_reg_t*)item;
+        rgpusm_reg = (mca_opal_gpu_reg_t *) item;
         rgpusm_reg->base.rcache = rcache;
         rgpusm_reg->base.base = addr;
-        rgpusm_reg->base.bound = (unsigned char *)addr + size - 1;;
+        rgpusm_reg->base.bound = (unsigned char *) addr + size - 1;
         rgpusm_reg->base.flags = flags;
 
-        /* Copy the memory handle received into the registration */
-        memcpy(rgpusm_reg->data.memHandle, rget_reg->data.memHandle, sizeof(rget_reg->data.memHandle));
-
         /* The rget_reg registration is holding the memory handle needed
          * to register the remote memory.  This was received from the remote
          * process.  A pointer to the memory is returned in the alloc_base field. */
-        rc = cuda_openmemhandle (addr, size, (mca_rcache_base_registration_t *)rgpusm_reg,
-                                 (mca_rcache_base_registration_t *)rget_reg);
+        rc = opal_accelerator.import_ipc_handle(MCA_ACCELERATOR_NO_DEVICE_ID, rget_reg->data.ipcHandle.handle,
+                                                &rgpusm_reg->data.ipcHandle);
+        if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
+            opal_output_verbose(10, mca_rcache_rgpusm_component.output,
+                                "import_ipc_handle failed: addr=%p size=%lu",
+                                addr, size);
+            /* Currently, this is a non-recoverable error */
+            return OPAL_ERROR;
+        }
+        rc = mca_rcache_rgpusm_open_mem_handle(addr, size, (mca_rcache_base_registration_t *) rgpusm_reg);
 
         /* This error should not happen with no cache in use. */
         assert(OPAL_ERR_WOULD_BLOCK != rc);
 
-        if(rc != OPAL_SUCCESS) {
-            opal_free_list_return (&rcache_rgpusm->reg_list, item);
+        if (rc != OPAL_SUCCESS) {
+            opal_free_list_return(&rcache_rgpusm->reg_list, item);
             return rc;
         }
         rgpusm_reg->base.ref_count++;
-        *reg = (mca_rcache_base_registration_t *)rgpusm_reg;
+        *reg = (mca_rcache_base_registration_t *) rgpusm_reg;
         return OPAL_SUCCESS;
     }
 
     /* Check to see if memory is registered and stored in the cache. */
     OPAL_THREAD_LOCK(&rcache->lock);
-    mca_rcache_base_vma_find (rcache_rgpusm->vma_module, addr, size, reg);
+    mca_rcache_base_vma_find(rcache_rgpusm->vma_module, addr, size, reg);
 
     /* If *reg is not NULL, we have a registration.  Let us see if the
      * memory handle matches the one we were looking for.  If not, the
@@ -240,11 +290,11 @@ int mca_rcache_rgpusm_register (mca_rcache_base_module_t *rcache, void *addr,
     if (*reg != NULL) {
         rcache_rgpusm->stat_cache_hit++;
         opal_output_verbose(10, mca_rcache_rgpusm_component.output,
-                            "RGPUSM: Found addr=%p,size=%d (base=%p,size=%d) in cache",
-                            addr, (int)size, (*reg)->base,
-                            (int)((*reg)->bound - (*reg)->base));
+                            "RGPUSM: Found addr=%p,size=%d (base=%p,size=%d) in cache", addr,
+                            (int) size, (void*)(*reg)->base, (int) ((*reg)->bound - (*reg)->base));
 
-        if (mca_common_cuda_memhandle_matches((mca_rcache_common_cuda_reg_t *)*reg, rget_reg)) {
+        if (0 == opal_accelerator.compare_ipc_handles(((mca_opal_gpu_reg_t *)*reg)->data.ipcHandle.handle,
+                                                      rget_reg->data.ipcHandle.handle)) {
             /* Registration matches what was requested.  All is good. */
             rcache_rgpusm->stat_cache_valid++;
         } else {
@@ -252,15 +302,14 @@ int mca_rcache_rgpusm_register (mca_rcache_base_module_t *rcache, void *addr,
             opal_output_verbose(10, mca_rcache_rgpusm_component.output,
                                 "RGPUSM: Mismatched Handle: Evicting/unregistering "
                                 "addr=%p,size=%d (base=%p,size=%d) from cache",
-                                addr, (int)size, (*reg)->base,
-                                (int)((*reg)->bound - (*reg)->base));
+                                addr, (int) size, (void*)(*reg)->base,
+                                (int) ((*reg)->bound - (*reg)->base));
 
             /* The ref_count has to be zero as this memory cannot possibly
              * be in use.  Assert on that just to make sure. */
             assert(0 == (*reg)->ref_count);
             if (mca_rcache_rgpusm_component.leave_pinned) {
-                opal_list_remove_item(&rcache_rgpusm->lru_list,
-                                      (opal_list_item_t*)(*reg));
+                opal_list_remove_item(&rcache_rgpusm->lru_list, (opal_list_item_t *) (*reg));
             }
 
             /* Bump the reference count to keep things copacetic in deregister */
@@ -279,54 +328,61 @@ int mca_rcache_rgpusm_register (mca_rcache_base_module_t *rcache, void *addr,
     /* If we have a registration here, then we know it is valid. */
     if (*reg != NULL) {
         opal_output_verbose(10, mca_rcache_rgpusm_component.output,
-                            "RGPUSM: CACHE HIT is good: ep=%d, addr=%p, size=%d in cache",
-                            mypeer, addr, (int)size);
+                            "RGPUSM: CACHE HIT is good: ep=%d, addr=%p, size=%d in cache", mypeer,
+                            addr, (int) size);
 
         /* When using leave pinned, we keep an LRU list. */
         if ((0 == (*reg)->ref_count) && mca_rcache_rgpusm_component.leave_pinned) {
             opal_output_verbose(20, mca_rcache_rgpusm_component.output,
-                                "RGPUSM: POP OFF LRU: ep=%d, addr=%p, size=%d in cache",
-                                mypeer, addr, (int)size);
-            opal_list_remove_item(&rcache_rgpusm->lru_list,
-                                  (opal_list_item_t*)(*reg));
+                                "RGPUSM: POP OFF LRU: ep=%d, addr=%p, size=%d in cache", mypeer,
+                                addr, (int) size);
+            opal_list_remove_item(&rcache_rgpusm->lru_list, (opal_list_item_t *) (*reg));
         }
         (*reg)->ref_count++;
         OPAL_THREAD_UNLOCK(&rcache->lock);
-        opal_output(-1, "reg->ref_count=%d", (int)(*reg)->ref_count);
+        opal_output(-1, "reg->ref_count=%d", (int) (*reg)->ref_count);
         opal_output_verbose(80, mca_rcache_rgpusm_component.output,
-                           "RGPUSM: Found entry in cache addr=%p, size=%d", addr, (int)size);
+                            "RGPUSM: Found entry in cache addr=%p, size=%d", addr, (int) size);
         return OPAL_SUCCESS;
     }
 
     /* If we are here, then we did not find a registration, or it was invalid,
      * so this is a new one, and we are going to use the cache. */
     assert(NULL == *reg);
-    opal_output_verbose(10, mca_rcache_rgpusm_component.output,
-                        "RGPUSM: New registration ep=%d, addr=%p, size=%d. Need to register and insert in cache",
-                         mypeer, addr, (int)size);
+    opal_output_verbose(
+        10, mca_rcache_rgpusm_component.output,
+        "RGPUSM: New registration ep=%d, addr=%p, size=%d. Need to register and insert in cache",
+        mypeer, addr, (int) size);
 
-    item = opal_free_list_get (&rcache_rgpusm->reg_list);
-    if(NULL == item) {
+    item = opal_free_list_get(&rcache_rgpusm->reg_list);
+    if (NULL == item) {
         OPAL_THREAD_UNLOCK(&rcache->lock);
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
-    rgpusm_reg = (mca_rcache_common_cuda_reg_t*)item;
+    rgpusm_reg = (mca_opal_gpu_reg_t *) item;
 
     rgpusm_reg->base.rcache = rcache;
     rgpusm_reg->base.base = addr;
-    rgpusm_reg->base.bound = (unsigned char *)addr + size - 1;
+    rgpusm_reg->base.bound = (unsigned char *) addr + size - 1;
     rgpusm_reg->base.flags = flags;
 
     /* Need the memory handle saved in the registration */
-    memcpy(rgpusm_reg->data.memHandle, rget_reg->data.memHandle, sizeof(rget_reg->data.memHandle));
+    rc = opal_accelerator.import_ipc_handle(MCA_ACCELERATOR_NO_DEVICE_ID, rget_reg->data.ipcHandle.handle,
+                                            &rgpusm_reg->data.ipcHandle);
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
+        opal_output_verbose(10, mca_rcache_rgpusm_component.output,
+                            "import_ipc_handle failed: addr=%p size=%lu",
+                            addr, size);
+        /* Currently, this is a non-recoverable error */
+        return OPAL_ERROR;
+    }
 
     /* Actually register the memory, which opens the memory handle.
      * Need to do this prior to putting in the cache as the base and
      * bound values may be changed by the registration.  The memory
      * associated with the handle comes back in the alloc_base
      * value. */
-    rc = cuda_openmemhandle (addr, size, (mca_rcache_base_registration_t *)rgpusm_reg,
-                             (mca_rcache_base_registration_t *)rget_reg);
+    rc = mca_rcache_rgpusm_open_mem_handle(addr, size, (mca_rcache_base_registration_t *) rgpusm_reg);
     /* There is a chance we can get the OPAL_ERR_WOULD_BLOCK from the
      * CUDA codes attempt to register the memory.  The case that this
      * can happen is as follows.  A block of memory is registered.
@@ -343,7 +399,7 @@ int mca_rcache_rgpusm_register (mca_rcache_base_module_t *rcache, void *addr,
 
         /* Need to make sure it is at least 4 bytes in size  This will
          * ensure we get the hit in the cache. */
-        mca_rcache_base_vma_find (rcache_rgpusm->vma_module, addr, 4, &oldreg);
+        mca_rcache_base_vma_find(rcache_rgpusm->vma_module, addr, 4, &oldreg);
 
         /* For most cases, we will find a registration that overlaps.
          * Removal of it should allow the registration we are
@@ -353,8 +409,7 @@ int mca_rcache_rgpusm_register (mca_rcache_base_module_t *rcache, void *addr,
              * possibly be in use.  Assert on that just to make sure. */
             assert(0 == oldreg->ref_count);
             if (mca_rcache_rgpusm_component.leave_pinned) {
-                opal_list_remove_item(&rcache_rgpusm->lru_list,
-                                      (opal_list_item_t*)oldreg);
+                opal_list_remove_item(&rcache_rgpusm->lru_list, (opal_list_item_t *) oldreg);
             }
 
             /* Bump the reference count to keep things copacetic in deregister */
@@ -365,8 +420,7 @@ int mca_rcache_rgpusm_register (mca_rcache_base_module_t *rcache, void *addr,
             rcache_rgpusm->stat_evicted++;
 
             /* And try again.  This one usually works. */
-            rc = cuda_openmemhandle (addr, size, (mca_rcache_base_registration_t *)rgpusm_reg,
-                                     (mca_rcache_base_registration_t *)rget_reg);
+            rc = mca_rcache_rgpusm_open_mem_handle(addr, size, (mca_rcache_base_registration_t *) rgpusm_reg);
         }
 
         /* There is a chance that another registration is blocking our
@@ -378,25 +432,26 @@ int mca_rcache_rgpusm_register (mca_rcache_base_module_t *rcache, void *addr,
                 break;
             }
             /* Clear out one registration. */
-            rc = cuda_openmemhandle (addr, size, (mca_rcache_base_registration_t *)rgpusm_reg,
-                                     (mca_rcache_base_registration_t *)rget_reg);
+            rc = mca_rcache_rgpusm_open_mem_handle(addr, size, (mca_rcache_base_registration_t *) rgpusm_reg);
         }
     }
 
-    if(rc != OPAL_SUCCESS) {
+    if (rc != OPAL_SUCCESS) {
         OPAL_THREAD_UNLOCK(&rcache->lock);
-        opal_free_list_return (&rcache_rgpusm->reg_list, item);
+        opal_free_list_return(&rcache_rgpusm->reg_list, item);
         return rc;
     }
 
     opal_output_verbose(80, mca_rcache_rgpusm_component.output,
-                        "RGPUSM: About to insert in rgpusm cache addr=%p, size=%d", addr, (int)size);
-    rc = mca_rcache_base_vma_insert (rcache_rgpusm->vma_module, (mca_rcache_base_registration_t *)rgpusm_reg,
-                                      mca_rcache_rgpusm_component.rcache_size_limit);
+                        "RGPUSM: About to insert in rgpusm cache addr=%p, size=%d", addr,
+                        (int) size);
+    rc = mca_rcache_base_vma_insert(rcache_rgpusm->vma_module,
+                                    (mca_rcache_base_registration_t *) rgpusm_reg,
+                                    mca_rcache_rgpusm_component.rcache_size_limit);
     if (OPAL_ERR_TEMP_OUT_OF_RESOURCE == rc) {
         opal_output_verbose(40, mca_rcache_rgpusm_component.output,
                             "RGPUSM: No room in the cache - boot the first one out");
-        (void)mca_rcache_rgpusm_deregister_lru(rcache);
+        (void) mca_rcache_rgpusm_deregister_lru(rcache);
         if (mca_rcache_rgpusm_component.empty_cache) {
             int remNum = 1;
             /* Empty out every registration from LRU until it is empty */
@@ -407,13 +462,15 @@ int mca_rcache_rgpusm_register (mca_rcache_base_module_t *rcache, void *addr,
             }
             opal_output_verbose(40, mca_rcache_rgpusm_component.output,
                                 "RGPUSM: Deleted and deregistered %d entries", remNum);
-            rc = mca_rcache_base_vma_insert (rcache_rgpusm->vma_module, (mca_rcache_base_registration_t *)rgpusm_reg,
-                                             mca_rcache_rgpusm_component.rcache_size_limit);
+            rc = mca_rcache_base_vma_insert(rcache_rgpusm->vma_module,
+                                            (mca_rcache_base_registration_t *) rgpusm_reg,
+                                            mca_rcache_rgpusm_component.rcache_size_limit);
         } else {
             /* Check for room after one removal. If not, remove another one until there is space */
-            while((rc = mca_rcache_base_vma_insert (rcache_rgpusm->vma_module, (mca_rcache_base_registration_t *)rgpusm_reg,
-                                                    mca_rcache_rgpusm_component.rcache_size_limit)) ==
-                  OPAL_ERR_TEMP_OUT_OF_RESOURCE) {
+            while ((rc = mca_rcache_base_vma_insert(rcache_rgpusm->vma_module,
+                                                    (mca_rcache_base_registration_t *) rgpusm_reg,
+                                                    mca_rcache_rgpusm_component.rcache_size_limit))
+                   == OPAL_ERR_TEMP_OUT_OF_RESOURCE) {
                 opal_output_verbose(40, mca_rcache_rgpusm_component.output,
                                     "RGPUSM: No room in the cache - boot one out");
                 if (!mca_rcache_rgpusm_deregister_lru(rcache)) {
@@ -423,9 +480,9 @@ int mca_rcache_rgpusm_register (mca_rcache_base_module_t *rcache, void *addr,
         }
     }
 
-    if(rc != OPAL_SUCCESS) {
+    if (rc != OPAL_SUCCESS) {
         OPAL_THREAD_UNLOCK(&rcache->lock);
-        opal_free_list_return (&rcache_rgpusm->reg_list, item);
+        opal_free_list_return(&rcache_rgpusm->reg_list, item);
         /* We cannot recover from this.  We can be here if the size of
          * the cache is smaller than the amount of memory we are
          * trying to register in a single transfer.  In that case, rc
@@ -433,33 +490,29 @@ int mca_rcache_rgpusm_register (mca_rcache_base_module_t *rcache, void *addr,
          * that point.  Therefore, just error out completely.
          */
         opal_output_verbose(10, mca_rcache_rgpusm_component.output,
-                            "RGPUSM: Failed to register addr=%p, size=%d", addr, (int)size);
+                            "RGPUSM: Failed to register addr=%p, size=%d", addr, (int) size);
         return OPAL_ERROR;
     }
 
     rgpusm_reg->base.ref_count++;
-    *reg = (mca_rcache_base_registration_t *)rgpusm_reg;
+    *reg = (mca_rcache_base_registration_t *) rgpusm_reg;
     OPAL_THREAD_UNLOCK(&rcache->lock);
 
     return OPAL_SUCCESS;
 }
 
-int mca_rcache_rgpusm_find(struct mca_rcache_base_module_t *rcache, void *addr,
-        size_t size, mca_rcache_base_registration_t **reg)
+int mca_rcache_rgpusm_find(struct mca_rcache_base_module_t *rcache, void *addr, size_t size,
+                           mca_rcache_base_registration_t **reg)
 {
-    mca_rcache_rgpusm_module_t *rcache_rgpusm = (mca_rcache_rgpusm_module_t*)rcache;
+    mca_rcache_rgpusm_module_t *rcache_rgpusm = (mca_rcache_rgpusm_module_t *) rcache;
     int rc;
-    unsigned char *base, *bound;
-
-    base = addr;
-    bound = base + size - 1; /* To keep cache hits working correctly */
 
     OPAL_THREAD_LOCK(&rcache->lock);
-    opal_output(-1, "Looking for addr=%p, size=%d", addr, (int)size);
-    rc = mca_rcache_base_vma_find (rcache_rgpusm->vma_module, addr, size, reg);
-    if(*reg != NULL && mca_rcache_rgpusm_component.leave_pinned) {
-        if(0 == (*reg)->ref_count && mca_rcache_rgpusm_component.leave_pinned) {
-            opal_list_remove_item(&rcache_rgpusm->lru_list, (opal_list_item_t*)(*reg));
+    opal_output(-1, "Looking for addr=%p, size=%d", addr, (int) size);
+    rc = mca_rcache_base_vma_find(rcache_rgpusm->vma_module, addr, size, reg);
+    if (*reg != NULL && mca_rcache_rgpusm_component.leave_pinned) {
+        if (0 == (*reg)->ref_count && mca_rcache_rgpusm_component.leave_pinned) {
+            opal_list_remove_item(&rcache_rgpusm->lru_list, (opal_list_item_t *) (*reg));
         }
         rcache_rgpusm->stat_cache_found++;
         (*reg)->ref_count++;
@@ -473,51 +526,48 @@ int mca_rcache_rgpusm_find(struct mca_rcache_base_module_t *rcache, void *addr,
 
 static inline bool registration_is_cachebale(mca_rcache_base_registration_t *reg)
 {
-     return !(reg->flags &
-             (MCA_RCACHE_FLAGS_CACHE_BYPASS |
-              MCA_RCACHE_FLAGS_INVALID));
+    return !(reg->flags & (MCA_RCACHE_FLAGS_CACHE_BYPASS | MCA_RCACHE_FLAGS_INVALID));
 }
 
 int mca_rcache_rgpusm_deregister(struct mca_rcache_base_module_t *rcache,
-                            mca_rcache_base_registration_t *reg)
+                                 mca_rcache_base_registration_t *reg)
 {
-    mca_rcache_rgpusm_module_t *rcache_rgpusm = (mca_rcache_rgpusm_module_t*)rcache;
+    mca_rcache_rgpusm_module_t *rcache_rgpusm = (mca_rcache_rgpusm_module_t *) rcache;
     int rc = OPAL_SUCCESS;
     assert(reg->ref_count > 0);
 
     OPAL_THREAD_LOCK(&rcache->lock);
     reg->ref_count--;
-    opal_output(-1, "Deregister: reg->ref_count=%d", (int)reg->ref_count);
-    if(reg->ref_count > 0) {
+    opal_output(-1, "Deregister: reg->ref_count=%d", (int) reg->ref_count);
+    if (reg->ref_count > 0) {
         OPAL_THREAD_UNLOCK(&rcache->lock);
         return OPAL_SUCCESS;
     }
-    if(mca_rcache_rgpusm_component.leave_pinned && registration_is_cachebale(reg))
-    {
+    if (mca_rcache_rgpusm_component.leave_pinned && registration_is_cachebale(reg)) {
         /* if leave_pinned is set don't deregister memory, but put it
          * on LRU list for future use */
         opal_output_verbose(20, mca_rcache_rgpusm_component.output,
-                            "RGPUSM: Deregister: addr=%p, size=%d: cacheable and pinned, leave in cache, PUSH IN LRU",
-                            reg->base, (int)(reg->bound - reg->base + 1));
-        opal_list_prepend(&rcache_rgpusm->lru_list, (opal_list_item_t*)reg);
+                            "RGPUSM: Deregister: addr=%p, size=%d: cacheable and pinned, leave in "
+                            "cache, PUSH IN LRU",
+                            (void*)reg->base, (int) (reg->bound - reg->base + 1));
+        opal_list_prepend(&rcache_rgpusm->lru_list, (opal_list_item_t *) reg);
     } else {
         /* Remove from rcache first */
-        if(!(reg->flags & MCA_RCACHE_FLAGS_CACHE_BYPASS))
-            mca_rcache_base_vma_delete (rcache_rgpusm->vma_module, reg);
+        if (!(reg->flags & MCA_RCACHE_FLAGS_CACHE_BYPASS))
+            mca_rcache_base_vma_delete(rcache_rgpusm->vma_module, reg);
 
         /* Drop the rcache lock before deregistring the memory */
         OPAL_THREAD_UNLOCK(&rcache->lock);
 
         {
-             assert(reg->ref_count == 0);
-             rc = cuda_closememhandle (NULL, reg);
-         }
+            assert(reg->ref_count == 0);
+            rc = mca_rcache_rgpusm_close_mem_handle(NULL, reg);
+        }
 
         OPAL_THREAD_LOCK(&rcache->lock);
 
-        if(OPAL_SUCCESS == rc) {
-            opal_free_list_return (&rcache_rgpusm->reg_list,
-                                   (opal_free_list_item_t*)reg);
+        if (OPAL_SUCCESS == rc) {
+            opal_free_list_return(&rcache_rgpusm->reg_list, (opal_free_list_item_t *) reg);
         }
     }
     OPAL_THREAD_UNLOCK(&rcache->lock);
@@ -526,33 +576,31 @@ int mca_rcache_rgpusm_deregister(struct mca_rcache_base_module_t *rcache,
 }
 
 int mca_rcache_rgpusm_deregister_no_lock(struct mca_rcache_base_module_t *rcache,
-                            mca_rcache_base_registration_t *reg)
+                                         mca_rcache_base_registration_t *reg)
 {
-    mca_rcache_rgpusm_module_t *rcache_rgpusm = (mca_rcache_rgpusm_module_t*)rcache;
+    mca_rcache_rgpusm_module_t *rcache_rgpusm = (mca_rcache_rgpusm_module_t *) rcache;
     int rc = OPAL_SUCCESS;
     assert(reg->ref_count > 0);
 
     reg->ref_count--;
-    opal_output(-1, "Deregister: reg->ref_count=%d", (int)reg->ref_count);
-    if(reg->ref_count > 0) {
+    opal_output(-1, "Deregister: reg->ref_count=%d", (int) reg->ref_count);
+    if (reg->ref_count > 0) {
         return OPAL_SUCCESS;
     }
-    if(mca_rcache_rgpusm_component.leave_pinned && registration_is_cachebale(reg))
-    {
+    if (mca_rcache_rgpusm_component.leave_pinned && registration_is_cachebale(reg)) {
         /* if leave_pinned is set don't deregister memory, but put it
          * on LRU list for future use */
-        opal_list_prepend(&rcache_rgpusm->lru_list, (opal_list_item_t*)reg);
+        opal_list_prepend(&rcache_rgpusm->lru_list, (opal_list_item_t *) reg);
     } else {
         /* Remove from rcache first */
-        if(!(reg->flags & MCA_RCACHE_FLAGS_CACHE_BYPASS))
-            mca_rcache_base_vma_delete (rcache_rgpusm->vma_module, reg);
+        if (!(reg->flags & MCA_RCACHE_FLAGS_CACHE_BYPASS))
+            mca_rcache_base_vma_delete(rcache_rgpusm->vma_module, reg);
 
         assert(reg->ref_count == 0);
-        rc = cuda_closememhandle (NULL, reg);
+        rc = mca_rcache_rgpusm_close_mem_handle(NULL, reg);
 
-        if(OPAL_SUCCESS == rc) {
-            opal_free_list_return (&rcache_rgpusm->reg_list,
-                                   (opal_free_list_item_t*)reg);
+        if (OPAL_SUCCESS == rc) {
+            opal_free_list_return(&rcache_rgpusm->reg_list, (opal_free_list_item_t *) reg);
         }
     }
 
@@ -563,59 +611,58 @@ int mca_rcache_rgpusm_deregister_no_lock(struct mca_rcache_base_module_t *rcache
 
 void mca_rcache_rgpusm_finalize(struct mca_rcache_base_module_t *rcache)
 {
-    mca_rcache_rgpusm_module_t *rcache_rgpusm = (mca_rcache_rgpusm_module_t*)rcache;
+    mca_rcache_rgpusm_module_t *rcache_rgpusm = (mca_rcache_rgpusm_module_t *) rcache;
     mca_rcache_base_registration_t *reg;
     mca_rcache_base_registration_t *regs[RGPUSM_RCACHE_NREGS];
     int reg_cnt, i;
     int rc;
 
     /* Statistic */
-    if(true == mca_rcache_rgpusm_component.print_stats) {
-        opal_output(0, "%s rgpusm: stats "
-                "(hit/valid/invalid/miss/evicted): %d/%d/%d/%d/%d\n",
-                OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),
-                rcache_rgpusm->stat_cache_hit, rcache_rgpusm->stat_cache_valid,
-                rcache_rgpusm->stat_cache_invalid, rcache_rgpusm->stat_cache_miss,
-                rcache_rgpusm->stat_evicted);
+    if (true == mca_rcache_rgpusm_component.print_stats) {
+        opal_output(0,
+                    "%s rgpusm: stats "
+                    "(hit/valid/invalid/miss/evicted): %d/%d/%d/%d/%d\n",
+                    OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), rcache_rgpusm->stat_cache_hit,
+                    rcache_rgpusm->stat_cache_valid, rcache_rgpusm->stat_cache_invalid,
+                    rcache_rgpusm->stat_cache_miss, rcache_rgpusm->stat_evicted);
     }
 
     OPAL_THREAD_LOCK(&rcache->lock);
     do {
-        reg_cnt = mca_rcache_base_vma_find_all (rcache_rgpusm->vma_module, 0, (size_t)-1,
-                regs, RGPUSM_RCACHE_NREGS);
+        reg_cnt = mca_rcache_base_vma_find_all(rcache_rgpusm->vma_module, 0, (size_t) -1, regs,
+                                               RGPUSM_RCACHE_NREGS);
         opal_output(-1, "Registration size at finalize = %d", reg_cnt);
 
-        for(i = 0; i < reg_cnt; i++) {
+        for (i = 0; i < reg_cnt; i++) {
             reg = regs[i];
 
-            if(reg->ref_count) {
+            if (reg->ref_count) {
                 reg->ref_count = 0; /* otherway dereg will fail on assert */
             } else if (mca_rcache_rgpusm_component.leave_pinned) {
-                opal_list_remove_item(&rcache_rgpusm->lru_list,
-                        (opal_list_item_t*)reg);
+                opal_list_remove_item(&rcache_rgpusm->lru_list, (opal_list_item_t *) reg);
             }
 
             /* Remove from rcache first */
-            mca_rcache_base_vma_delete (rcache_rgpusm->vma_module, reg);
+            mca_rcache_base_vma_delete(rcache_rgpusm->vma_module, reg);
 
             /* Drop lock before deregistering memory */
             OPAL_THREAD_UNLOCK(&rcache->lock);
             assert(reg->ref_count == 0);
-            rc = cuda_closememhandle (NULL, reg);
+            rc = mca_rcache_rgpusm_close_mem_handle(NULL, reg);
             OPAL_THREAD_LOCK(&rcache->lock);
 
-            if(rc != OPAL_SUCCESS) {
+            if (rc != OPAL_SUCCESS) {
                 /* Potentially lose track of registrations
                    do we have to put it back? */
                 continue;
             }
 
-            opal_free_list_return (&rcache_rgpusm->reg_list,
-                                   (opal_free_list_item_t *) reg);
+            opal_free_list_return(&rcache_rgpusm->reg_list, (opal_free_list_item_t *) reg);
         }
-    } while(reg_cnt == RGPUSM_RCACHE_NREGS);
+    } while (reg_cnt == RGPUSM_RCACHE_NREGS);
 
     OBJ_DESTRUCT(&rcache_rgpusm->lru_list);
     OBJ_DESTRUCT(&rcache_rgpusm->reg_list);
     OPAL_THREAD_UNLOCK(&rcache->lock);
+    mca_rcache_base_module_fini(rcache);
 }
diff --git a/opal/mca/rcache/udreg/Makefile.am b/opal/mca/rcache/udreg/Makefile.am
deleted file mode 100644
index 63ce17b2658..00000000000
--- a/opal/mca/rcache/udreg/Makefile.am
+++ /dev/null
@@ -1,54 +0,0 @@
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2013 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2012-2015 Los Alamos National Security, LLC. All rights
-#                         reserved.
-# Copyright (c) 2017      IBM Corporation.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-AM_CPPFLAGS = $(rcache_udreg_CPPFLAGS)
-
-sources = rcache_udreg_module.c rcache_udreg_component.c
-
-if WANT_INSTALL_HEADERS
-opaldir = $(opalincludedir)/$(subdir)
-opal_HEADERS = rcache_udreg.h
-endif
-
-# Make the output library in this directory, and name it either
-# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
-# (for static builds).
-
-if MCA_BUILD_opal_rcache_udreg_DSO
-component_noinst =
-component_install = mca_rcache_udreg.la
-else
-component_noinst = libmca_rcache_udreg.la
-component_install =
-endif
-
-mcacomponentdir = $(opallibdir)
-mcacomponent_LTLIBRARIES = $(component_install)
-mca_rcache_udreg_la_SOURCES = $(sources)
-mca_rcache_udreg_la_LDFLAGS = -module -avoid-version $(rcache_udreg_LDFLAGS)
-mca_rcache_udreg_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
-	$(rcache_udreg_LIBS)
-
-noinst_LTLIBRARIES = $(component_noinst)
-libmca_rcache_udreg_la_SOURCES = $(sources)
-libmca_rcache_udreg_la_LIBADD = $(rcache_udreg_LIBS)
-libmca_rcache_udreg_la_LDFLAGS = -module -avoid-version $(rcache_udreg_LDFLAGS)
diff --git a/opal/mca/rcache/udreg/configure.m4 b/opal/mca/rcache/udreg/configure.m4
deleted file mode 100644
index 513518d0281..00000000000
--- a/opal/mca/rcache/udreg/configure.m4
+++ /dev/null
@@ -1,48 +0,0 @@
-# -*- shell-script -*-
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2013 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2006 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2006      QLogic Corp. All rights reserved.
-# Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2011-2015 Los Alamos National Security, LLC.
-#                         All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-AC_DEFUN([MCA_opal_rcache_udreg_CONFIG],[
-    AC_CONFIG_FILES([opal/mca/rcache/udreg/Makefile])
-
-    AC_ARG_WITH([udreg], [AS_HELP_STRING([--with-udreg],
-		[Build support for Cray udreg support. Set PKG_CONFIG_PATH env. variable to specify alternate path.])])
-
-    rcache_udreg_happy="no"
-
-    AS_IF([test "$with_udreg" = "no"],
-          [rcache_udreg_happy="no"],
-          [PKG_CHECK_MODULES([CRAY_UDREG], [cray-udreg],
-                      [rcache_udreg_LDFLAGS="$CRAY_UDREG_LIBS"
-                       rcache_udreg_CPPFLAGS="$CRAY_UDREG_CFLAGS"
-                       rcache_udreg_happy="yes"],
-                      [AC_MSG_RESULT([no])
-                       rcache_udreg_happ="no"])])
-
-    AS_IF([test "$rcache_udreg_happy" = "yes"], [$1], [$2])
-
-
-    # substitute in the things needed to build udreg/rcache
-    AC_SUBST([rcache_udreg_CPPFLAGS])
-    AC_SUBST([rcache_udreg_LDFLAGS])
-    AC_SUBST([rcache_udreg_LIBS])
-])dnl
diff --git a/opal/mca/rcache/udreg/owner.txt b/opal/mca/rcache/udreg/owner.txt
deleted file mode 100644
index 52961b5d12f..00000000000
--- a/opal/mca/rcache/udreg/owner.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-#
-# owner/status file
-# owner: institution that is responsible for this package
-# status: e.g. active, maintenance, unmaintained
-#
-owner: LANL
-status: maintenance
diff --git a/opal/mca/rcache/udreg/rcache_udreg.h b/opal/mca/rcache/udreg/rcache_udreg.h
deleted file mode 100644
index d78ef9883ae..00000000000
--- a/opal/mca/rcache/udreg/rcache_udreg.h
+++ /dev/null
@@ -1,85 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2013 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Voltaire. All rights reserved.
- * Copyright (c) 2011-2016 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2020      Amazon.com, Inc. or its affiliates.
- *                         All Rights reserved.
- *
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-/**
- * @file
- */
-#ifndef MCA_RCACHE_UDREG_H
-#define MCA_RCACHE_UDREG_H
-
-#include "opal_config.h"
-#include "opal/class/opal_list.h"
-#include "opal/class/opal_free_list.h"
-#include "opal/util/event.h"
-#include "opal/mca/rcache/rcache.h"
-#include "opal/util/proc.h"
-#if HAVE_SYS_MMAN_H
-#include <sys/mman.h>
-#endif
-
-BEGIN_C_DECLS
-
-struct mca_rcache_udreg_component_t {
-    mca_rcache_base_component_t super;
-    bool print_stats;
-    int leave_pinned;
-};
-typedef struct mca_rcache_udreg_component_t mca_rcache_udreg_component_t;
-
-OPAL_DECLSPEC extern mca_rcache_udreg_component_t mca_rcache_udreg_component;
-
-struct mca_rcache_udreg_resources_t {
-    mca_rcache_base_resources_t base;
-
-    /* udreg specific resources */
-    bool   use_kernel_cache;
-    bool   use_evict_w_unreg;
-    int    max_entries;
-    size_t page_size;
-};
-typedef struct mca_rcache_udreg_resources_t mca_rcache_udreg_resources_t;
-
-struct mca_rcache_udreg_module_t;
-
-struct mca_rcache_udreg_module_t {
-    mca_rcache_base_module_t super;
-    mca_rcache_udreg_resources_t resources;
-    opal_free_list_t reg_list;
-    opal_mutex_t lock;
-    void *udreg_handle;
-    /** used to communicate the access flags to the underlying registration
-     * function */
-    int requested_access_flags;
-    int requested_flags;
-};
-typedef struct mca_rcache_udreg_module_t mca_rcache_udreg_module_t;
-
-
-/*
- *  Initializes the rcache module.
- */
-int mca_rcache_udreg_module_init(mca_rcache_udreg_module_t *rcache);
-
-END_C_DECLS
-#endif
diff --git a/opal/mca/rcache/udreg/rcache_udreg_component.c b/opal/mca/rcache/udreg/rcache_udreg_component.c
deleted file mode 100644
index 8a4b44b9381..00000000000
--- a/opal/mca/rcache/udreg/rcache_udreg_component.c
+++ /dev/null
@@ -1,130 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2013 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Voltaire. All rights reserved.
- * Copyright (c) 2007-2009 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012-2015 Los Alamos National Security, LLC. All rights
- *                         reserved.
- *
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#define OPAL_DISABLE_ENABLE_MEM_DEBUG 1
-#include "opal_config.h"
-#include "opal/mca/base/base.h"
-#include "opal/runtime/opal_params.h"
-#include "rcache_udreg.h"
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_MALLOC_H
-#include <malloc.h>
-#endif
-
-#include <fcntl.h>
-
-/*
- * Local functions
- */
-static int udreg_open(void);
-static int udreg_close(void);
-static int udreg_register(void);
-static mca_rcache_base_module_t* udreg_init(
-        struct mca_rcache_base_resources_t* resources);
-
-mca_rcache_udreg_component_t mca_rcache_udreg_component = {
-    {
-        /* First, the mca_base_component_t struct containing meta
-           information about the component itself */
-
-        .rcache_version ={
-            MCA_RCACHE_BASE_VERSION_3_0_0,
-
-            .mca_component_name = "udreg",
-            MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
-                                  OPAL_RELEASE_VERSION),
-            .mca_open_component = udreg_open,
-            .mca_close_component = udreg_close,
-            .mca_register_component_params = udreg_register,
-        },
-        .rcache_data = {
-            /* The component is checkpoint ready */
-            MCA_BASE_METADATA_PARAM_CHECKPOINT
-        },
-
-        .rcache_init = udreg_init
-    }
-};
-
-/**
-  * component open/close/init function
-  */
-static int udreg_open(void)
-{
-    return OPAL_SUCCESS;
-}
-
-
-static int udreg_register(void)
-{
-    mca_rcache_udreg_component.print_stats = false;
-    (void) mca_base_component_var_register(&mca_rcache_udreg_component.super.rcache_version,
-                                           "print_stats", "print pool usage statistics at the end of the run",
-                                           MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
-                                           &mca_rcache_udreg_component.print_stats);
-
-    return OPAL_SUCCESS;
-}
-
-
-static int udreg_close(void)
-{
-    return OPAL_SUCCESS;
-}
-
-static mca_rcache_base_module_t *
-udreg_init(struct mca_rcache_base_resources_t *resources)
-{
-    mca_rcache_udreg_resources_t *udreg_resources = (mca_rcache_udreg_resources_t *) resources;
-    mca_rcache_udreg_module_t* rcache_module;
-    static int inited = false;
-    int rc;
-
-    /* Set this here (vs in component.c) because
-       opal_leave_pinned* may have been set after MCA params were
-       read (e.g., by the openib btl) */
-    mca_rcache_udreg_component.leave_pinned = (int)
-        (1 == opal_leave_pinned || opal_leave_pinned_pipeline);
-
-    if (!inited) {
-        inited = true;
-    }
-
-    rcache_module =
-        (mca_rcache_udreg_module_t *) malloc (sizeof (mca_rcache_udreg_module_t));
-
-    memmove (&rcache_module->resources, udreg_resources, sizeof (*udreg_resources));
-
-    rc = mca_rcache_udreg_module_init(rcache_module);
-    if (OPAL_SUCCESS != rc) {
-        free (rcache_module);
-        return NULL;
-    }
-
-    return &rcache_module->super;
-}
diff --git a/opal/mca/rcache/udreg/rcache_udreg_module.c b/opal/mca/rcache/udreg/rcache_udreg_module.c
deleted file mode 100644
index f6fc44e8c30..00000000000
--- a/opal/mca/rcache/udreg/rcache_udreg_module.c
+++ /dev/null
@@ -1,361 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2013 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2009 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2006      Voltaire. All rights reserved.
- * Copyright (c) 2007      Mellanox Technologies. All rights reserved.
- * Copyright (c) 2010      IBM Corporation.  All rights reserved.
- * Copyright (c) 2011-2016 Los Alamos National Security, LLC. All rights
- *                         reserved.
- *
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#define OPAL_DISABLE_ENABLE_MEM_DEBUG 1
-#include "opal_config.h"
-#include "opal/align.h"
-#include "rcache_udreg.h"
-#include <errno.h>
-#include <string.h>
-#ifdef HAVE_MALLOC_H
-#include <malloc.h>
-#endif
-#include "opal/mca/rcache/base/base.h"
-#include "opal/runtime/opal_params.h"
-#include "opal/include/opal_stdint.h"
-#include "opal/util/sys_limits.h"
-#include "opal/util/string_copy.h"
-
-#include <fcntl.h>
-
-#include <udreg_pub.h>
-
-#include <sys/mman.h>
-
-
-static int mca_rcache_udreg_register (mca_rcache_base_module_t* rcache, void *addr,
-                                      size_t size, uint32_t flags, int32_t access_flags,
-                                      mca_rcache_base_registration_t **reg);
-static int mca_rcache_udreg_deregister (mca_rcache_base_module_t *rcache,
-                                        mca_rcache_base_registration_t *reg);
-static int mca_rcache_udreg_find (mca_rcache_base_module_t* rcache, void* addr,
-                                  size_t size, mca_rcache_base_registration_t **reg);
-static void mca_rcache_udreg_finalize (mca_rcache_base_module_t *rcache);
-static bool mca_rcache_udreg_evict (mca_rcache_base_module_t *rcache);
-
-static void *mca_rcache_udreg_reg_func (void *addr, uint64_t len, void *reg_context);
-static uint32_t mca_rcache_udreg_dereg_func (void *device_data, void *dreg_context);
-
-
-/*
- *  Initializes the rcache module.
- */
-int mca_rcache_udreg_module_init (mca_rcache_udreg_module_t *rcache)
-{
-    struct udreg_cache_attr cache_attr;
-    int urc;
-
-    rcache->super.rcache_component = &mca_rcache_udreg_component.super;
-    rcache->super.rcache_register = mca_rcache_udreg_register;
-    rcache->super.rcache_find = mca_rcache_udreg_find;
-    rcache->super.rcache_deregister = mca_rcache_udreg_deregister;
-    /* This module relies on udreg for notification of memory release */
-    rcache->super.rcache_invalidate_range = NULL;
-    rcache->super.rcache_finalize = mca_rcache_udreg_finalize;
-
-    cache_attr.modes = 0;
-
-    /* Create udreg cache */
-    if (rcache->resources.use_kernel_cache) {
-        cache_attr.modes |= UDREG_CC_MODE_USE_KERNEL_CACHE;
-    }
-
-    if (rcache->resources.use_evict_w_unreg) {
-        cache_attr.modes |= UDREG_CC_MODE_USE_EVICT_W_UNREG;
-    }
-
-    if (mca_rcache_udreg_component.leave_pinned) {
-        cache_attr.modes |= UDREG_CC_MODE_USE_LAZY_DEREG;
-    }
-
-    OBJ_CONSTRUCT(&rcache->lock, opal_mutex_t);
-
-    opal_string_copy (cache_attr.cache_name, rcache->resources.base.cache_name, UDREG_MAX_CACHENAME_LEN);
-    cache_attr.max_entries         = rcache->resources.max_entries;
-    cache_attr.debug_mode          = 0;
-    cache_attr.debug_rank          = 0;
-    cache_attr.reg_context         = rcache;
-    cache_attr.dreg_context        = rcache;
-    cache_attr.destructor_context  = rcache;
-    cache_attr.device_reg_func     = mca_rcache_udreg_reg_func;
-    cache_attr.device_dereg_func   = mca_rcache_udreg_dereg_func;
-    cache_attr.destructor_callback = NULL;
-
-    opal_output_verbose (MCA_BASE_VERBOSE_INFO, opal_rcache_base_framework.framework_output,
-                         "rcache/udreg: creating udreg cache with name %s", cache_attr.cache_name);
-
-    /* attempt to create the udreg cache. this will fail if one already exists */
-    (void) UDREG_CacheCreate (&cache_attr);
-
-    urc = UDREG_CacheAccess (rcache->resources.base.cache_name, (udreg_cache_handle_t *) &rcache->udreg_handle);
-    if (UDREG_RC_SUCCESS != urc) {
-        opal_output_verbose (MCA_BASE_VERBOSE_WARN, opal_rcache_base_framework.framework_output,
-                             "rcache/udreg: call to UDREG_CacheAccess failed with rc: %d", urc);
-        return OPAL_ERROR;
-    }
-
-    OBJ_CONSTRUCT(&rcache->reg_list, opal_free_list_t);
-    opal_free_list_init (&rcache->reg_list, rcache->resources.base.sizeof_reg,
-                         opal_cache_line_size, OBJ_CLASS(mca_rcache_base_registration_t),
-                         0, opal_cache_line_size, 0, -1, 32, NULL, 0,
-                         NULL, NULL, NULL);
-
-    return OPAL_SUCCESS;
-}
-
-/* udreg callback functions */
-static void *mca_rcache_udreg_reg_func (void *addr, uint64_t size, void *reg_context)
-{
-    mca_rcache_udreg_module_t *rcache_udreg = (mca_rcache_udreg_module_t *) reg_context;
-    unsigned int page_size = opal_getpagesize ();
-    mca_rcache_base_registration_t *udreg_reg;
-    opal_free_list_item_t *item;
-    int rc;
-
-    item = opal_free_list_get (&rcache_udreg->reg_list);
-    if (NULL == item) {
-        return NULL;
-    }
-
-    udreg_reg = (mca_rcache_base_registration_t *) item;
-
-    udreg_reg->rcache = reg_context;
-    udreg_reg->base  = OPAL_DOWN_ALIGN_PTR(addr, page_size, unsigned char *);
-    udreg_reg->bound = OPAL_ALIGN_PTR((intptr_t) addr + size, page_size, unsigned char *) - 1;
-    udreg_reg->ref_count = 0;
-
-    addr = (void *) udreg_reg->base;
-    size = (uint64_t) (udreg_reg->bound - udreg_reg->base + 1);
-
-    /* pull the flags and access flags out of the rcache module */
-    udreg_reg->access_flags = rcache_udreg->requested_access_flags;
-    udreg_reg->flags = rcache_udreg->requested_flags;
-
-    opal_output_verbose (MCA_BASE_VERBOSE_INFO, opal_rcache_base_framework.framework_output,
-                         "rcache/udreg: calling underlying register function for address range {%p, %p}",
-                         addr, (void *)((intptr_t) addr + size));
-    rc = rcache_udreg->resources.base.register_mem (rcache_udreg->resources.base.reg_data, udreg_reg->base, size,
-                                                    udreg_reg);
-    if (OPAL_SUCCESS != rc) {
-        opal_output_verbose (MCA_BASE_VERBOSE_WARN, opal_rcache_base_framework.framework_output,
-                             "rcache/udreg: could not register memory. rc: %d", rc);
-        opal_free_list_return (&rcache_udreg->reg_list, item);
-        /* NTH: this is the only way to get UDReg_Register to recognize a failure */
-        udreg_reg = UDREG_DEVICE_REG_FAILED;
-    }
-
-    return udreg_reg;
-}
-
-static uint32_t mca_rcache_udreg_dereg_func (void *device_data, void *dreg_context)
-{
-    mca_rcache_udreg_module_t *rcache_udreg = (mca_rcache_udreg_module_t *) dreg_context;
-    mca_rcache_base_registration_t *udreg_reg = (mca_rcache_base_registration_t *) device_data;
-    int rc;
-
-    assert (udreg_reg->ref_count == 0);
-
-    rc = rcache_udreg->resources.base.deregister_mem (rcache_udreg->resources.base.reg_data, udreg_reg);
-    if (OPAL_LIKELY(OPAL_SUCCESS == rc)) {
-        opal_free_list_return (&rcache_udreg->reg_list,
-                               (opal_free_list_item_t *) udreg_reg);
-    }
-    /* might be worth printing out a warning if an error occurs here */
-
-    return 0;
-}
-
-static bool mca_rcache_udreg_evict (mca_rcache_base_module_t *rcache)
-{
-    mca_rcache_udreg_module_t *rcache_udreg = (mca_rcache_udreg_module_t *) rcache;
-    udreg_return_t urc;
-
-    urc = UDREG_Evict (rcache_udreg->udreg_handle);
-    return (UDREG_RC_SUCCESS == urc);
-}
-
-/*
- * register memory
- */
-static int mca_rcache_udreg_register(mca_rcache_base_module_t *rcache, void *addr,
-                                    size_t size, uint32_t flags, int32_t access_flags,
-                                    mca_rcache_base_registration_t **reg)
-{
-    mca_rcache_udreg_module_t *rcache_udreg = (mca_rcache_udreg_module_t *) rcache;
-    mca_rcache_base_registration_t *udreg_reg, *old_reg;
-    bool bypass_cache = !!(flags & MCA_RCACHE_FLAGS_CACHE_BYPASS);
-    const unsigned int page_size = opal_getpagesize ();
-    unsigned char *base, *bound;
-    udreg_entry_t *udreg_entry = NULL;
-
-    *reg = NULL;
-
-    OPAL_THREAD_LOCK(&rcache_udreg->lock);
-
-    /* we hold the lock so no other thread can modify these flags until the registration is complete */
-    rcache_udreg->requested_access_flags = access_flags;
-    rcache_udreg->requested_flags = flags;
-
-    base = OPAL_DOWN_ALIGN_PTR(addr, page_size, unsigned char *);
-    bound = OPAL_ALIGN_PTR((intptr_t) addr + size, page_size, unsigned char *) - 1;
-
-    addr = base;
-    size = (size_t) (uintptr_t) (bound - base) + 1;
-
-    if (false == bypass_cache) {
-        /* Get a udreg entry for this region */
-        do {
-            opal_output_verbose (MCA_BASE_VERBOSE_INFO, opal_rcache_base_framework.framework_output,
-                                 "rcache/udreg: XXX registering region {%p, %p} with udreg", addr, (void *)((intptr_t) addr + size));
-            while (UDREG_RC_SUCCESS != UDREG_Register (rcache_udreg->udreg_handle, addr, size, &udreg_entry)) {
-                /* try to remove one unused reg and retry */
-                opal_output_verbose (MCA_BASE_VERBOSE_INFO, opal_rcache_base_framework.framework_output,
-                                     "calling evict!");
-                if (!mca_rcache_udreg_evict (rcache)) {
-                    opal_output_verbose (MCA_BASE_VERBOSE_INFO, opal_rcache_base_framework.framework_output,
-                                         "rcache/udreg: could not register memory with udreg");
-                    OPAL_THREAD_UNLOCK(&rcache_udreg->lock);
-                    return OPAL_ERR_OUT_OF_RESOURCE;
-                }
-            }
-
-            udreg_reg = (mca_rcache_base_registration_t *) udreg_entry->device_data;
-            if (NULL != udreg_reg && (udreg_reg->access_flags & access_flags) == access_flags) {
-                /* sufficient access */
-                break;
-            }
-
-            old_reg = udreg_reg;
-
-            if (old_reg) {
-                /* to not confuse udreg make sure the new registration covers the same address
-                 * range as the old one. */
-                addr = old_reg->base;
-                size = (size_t)((intptr_t) old_reg->bound - (intptr_t) old_reg->base);
-
-                /* make the new access flags more permissive */
-                access_flags |= old_reg->access_flags;
-
-                if (!old_reg->ref_count) {
-                    /* deregister the region before attempting to re-register */
-                    mca_rcache_udreg_dereg_func (old_reg, rcache);
-                    udreg_entry->device_data = NULL;
-                    old_reg = NULL;
-                } else {
-                    /* ensure that mca_rcache_udreg_deregister does not call into udreg since
-                     * we are forcefully evicting the registration here */
-                    old_reg->flags |= MCA_RCACHE_FLAGS_CACHE_BYPASS | MCA_RCACHE_FLAGS_INVALID;
-                }
-            }
-
-            rcache_udreg->requested_access_flags = access_flags;
-
-            /* get a new registration */
-            while (UDREG_DEVICE_REG_FAILED == (udreg_reg = mca_rcache_udreg_reg_func (addr, size, rcache))) {
-                if (!mca_rcache_udreg_evict (rcache)) {
-                    opal_output_verbose (MCA_BASE_VERBOSE_INFO, opal_rcache_base_framework.framework_output,
-                                         "rcache/udreg: could not register memory with udreg");
-                    OPAL_THREAD_UNLOCK(&rcache_udreg->lock);
-                    return OPAL_ERR_OUT_OF_RESOURCE;
-                }
-            }
-
-            /* update the device data with the new registration */
-            udreg_entry->device_data = udreg_reg;
-        } while (0);
-    } else {
-        /* if cache bypass is requested don't use the udreg cache */
-        while (UDREG_DEVICE_REG_FAILED == (udreg_reg = mca_rcache_udreg_reg_func (addr, size, rcache))) {
-            /* try to remove one unused reg and retry */
-            if (!mca_rcache_udreg_evict (rcache)) {
-                opal_output_verbose (MCA_BASE_VERBOSE_INFO, opal_rcache_base_framework.framework_output,
-                                     "rcache/udreg: could not register memory");
-                OPAL_THREAD_UNLOCK(&rcache_udreg->lock);
-                return OPAL_ERR_OUT_OF_RESOURCE;
-            }
-        }
-    }
-
-    OPAL_THREAD_UNLOCK(&rcache_udreg->lock);
-
-    *reg = udreg_reg;
-    (void) OPAL_THREAD_ADD_FETCH32(&udreg_reg->ref_count, 1);
-    udreg_reg->rcache_context = udreg_entry;
-
-    return OPAL_SUCCESS;
-}
-
-static int mca_rcache_udreg_find (mca_rcache_base_module_t *rcache, void *addr,
-                                 size_t size, mca_rcache_base_registration_t **reg)
-{
-    *reg = NULL;
-    return OPAL_ERR_NOT_FOUND;
-}
-
-static int mca_rcache_udreg_deregister(mca_rcache_base_module_t *rcache,
-                                      mca_rcache_base_registration_t *reg)
-{
-    mca_rcache_udreg_module_t *rcache_udreg = (mca_rcache_udreg_module_t *) rcache;
-    int32_t ref_count = OPAL_THREAD_ADD_FETCH32 (&reg->ref_count, -1);
-
-    assert(ref_count >= 0);
-
-    if (!(reg->flags & MCA_RCACHE_FLAGS_CACHE_BYPASS)) {
-        OPAL_THREAD_LOCK(&rcache_udreg->lock);
-        UDREG_DecrRefcount (rcache_udreg->udreg_handle, reg->rcache_context);
-        OPAL_THREAD_UNLOCK(&rcache_udreg->lock);
-    } else if (!ref_count) {
-        mca_rcache_udreg_dereg_func (reg, rcache);
-    }
-
-    return OPAL_SUCCESS;
-}
-
-static void mca_rcache_udreg_finalize (mca_rcache_base_module_t *rcache)
-{
-    mca_rcache_udreg_module_t *rcache_udreg = (mca_rcache_udreg_module_t*)rcache;
-
-    /* Statistic */
-    if (true == mca_rcache_udreg_component.print_stats) {
-        uint64_t hit = 0, miss = 0, evicted = 0;
-
-        (void) UDREG_GetStat (rcache_udreg->udreg_handle,
-                              UDREG_STAT_CACHE_HIT, &hit);
-
-        (void) UDREG_GetStat (rcache_udreg->udreg_handle,
-                              UDREG_STAT_CACHE_MISS, &miss);
-
-        (void) UDREG_GetStat (rcache_udreg->udreg_handle,
-                              UDREG_STAT_CACHE_EVICTED, &evicted);
-
-        opal_output(0, "%s udreg: stats (hit/miss/evicted): %" PRIu64 "/%" PRIu64 "/%" PRIu64 "\n",
-                    OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), hit, miss, evicted);
-    }
-
-    UDREG_CacheRelease (rcache_udreg->udreg_handle);
-    OBJ_DESTRUCT(&rcache_udreg->reg_list);
-    OBJ_DESTRUCT(&rcache_udreg->lock);
-}
diff --git a/opal/mca/reachable/base/base.h b/opal/mca/reachable/base/base.h
index 6ab36d5b62f..4e92cb048e4 100644
--- a/opal/mca/reachable/base/base.h
+++ b/opal/mca/reachable/base/base.h
@@ -15,8 +15,8 @@
 #include "opal_config.h"
 #include "opal/types.h"
 
-#include "opal/mca/mca.h"
 #include "opal/mca/base/mca_base_framework.h"
+#include "opal/mca/mca.h"
 
 #include "opal/mca/reachable/reachable.h"
 
@@ -29,9 +29,8 @@ OPAL_DECLSPEC extern mca_base_framework_t opal_reachable_base_framework;
  */
 OPAL_DECLSPEC int opal_reachable_base_select(void);
 
-OPAL_DECLSPEC opal_reachable_t * opal_reachable_allocate(unsigned int num_local,
-							 unsigned int num_remote);
-
+OPAL_DECLSPEC opal_reachable_t *opal_reachable_allocate(unsigned int num_local,
+                                                        unsigned int num_remote);
 
 END_C_DECLS
 
diff --git a/opal/mca/reachable/base/reachable_base_alloc.c b/opal/mca/reachable/base/reachable_base_alloc.c
index 5f3a6834883..78f3d88d701 100644
--- a/opal/mca/reachable/base/reachable_base_alloc.c
+++ b/opal/mca/reachable/base/reachable_base_alloc.c
@@ -12,26 +12,22 @@
 
 #include "opal/class/opal_object.h"
 
-#include "opal/mca/reachable/reachable.h"
 #include "opal/mca/reachable/base/base.h"
-
+#include "opal/mca/reachable/reachable.h"
 
 static void opal_reachable_construct(opal_reachable_t *reachable)
 {
     reachable->weights = NULL;
 }
 
-
-static void opal_reachable_destruct(opal_reachable_t * reachable)
+static void opal_reachable_destruct(opal_reachable_t *reachable)
 {
     if (NULL != reachable->memory) {
         free(reachable->memory);
     }
 }
 
-
-opal_reachable_t * opal_reachable_allocate(unsigned int num_local,
-                                           unsigned int num_remote)
+opal_reachable_t *opal_reachable_allocate(unsigned int num_local, unsigned int num_remote)
 {
     char *memory;
     unsigned int i;
@@ -42,28 +38,23 @@ opal_reachable_t * opal_reachable_allocate(unsigned int num_local,
 
     /* allocate all the pieces of the two dimensional array in one
        malloc, rather than a bunch of little allocations */
-    memory = malloc(sizeof(int*) * num_local +
-                    num_local * (sizeof(int) * num_remote));
+    memory = malloc(sizeof(int *) * num_local + num_local * (sizeof(int) * num_remote));
     if (memory == NULL) {
         OBJ_RELEASE(reachable);
         return NULL;
     }
 
-    reachable->memory = (void*)memory;
-    reachable->weights = (int**)reachable->memory;
-    memory += (sizeof(int*) * num_local);
+    reachable->memory = (void *) memory;
+    reachable->weights = (int **) reachable->memory;
+    memory += (sizeof(int *) * num_local);
 
     for (i = 0; i < num_local; i++) {
-        reachable->weights[i] = (int*)memory;
+        reachable->weights[i] = (int *) memory;
         memory += (sizeof(int) * num_remote);
     }
 
     return reachable;
 }
 
-OBJ_CLASS_INSTANCE(
-    opal_reachable_t,
-    opal_object_t,
-    opal_reachable_construct,
-    opal_reachable_destruct
-);
+OBJ_CLASS_INSTANCE(opal_reachable_t, opal_object_t, opal_reachable_construct,
+                   opal_reachable_destruct);
diff --git a/opal/mca/reachable/base/reachable_base_frame.c b/opal/mca/reachable/base/reachable_base_frame.c
index d9781086ab9..3caa3cc9214 100644
--- a/opal/mca/reachable/base/reachable_base_frame.c
+++ b/opal/mca/reachable/base/reachable_base_frame.c
@@ -9,17 +9,15 @@
  * $HEADER$
  */
 
-
 #include "opal_config.h"
 #include "opal/constants.h"
 
+#include "opal/mca/base/base.h"
 #include "opal/mca/mca.h"
 #include "opal/util/output.h"
-#include "opal/mca/base/base.h"
 
 #include "opal/mca/reachable/base/base.h"
 
-
 /*
  * The following file was created by configure.  It contains extern
  * components and the definition of an array of pointers to each
@@ -47,7 +45,6 @@ static int opal_reachable_base_frame_open(mca_base_open_flag_t flags)
 }
 
 MCA_BASE_FRAMEWORK_DECLARE(opal, reachable, "OPAL Reachability Framework",
-                           opal_reachable_base_frame_register,
-                           opal_reachable_base_frame_open,
-                           opal_reachable_base_frame_close,
-                           mca_reachable_base_static_components, 0);
+                           opal_reachable_base_frame_register, opal_reachable_base_frame_open,
+                           opal_reachable_base_frame_close, mca_reachable_base_static_components,
+                           0);
diff --git a/opal/mca/reachable/base/reachable_base_select.c b/opal/mca/reachable/base/reachable_base_select.c
index 24cdfc592ed..613d5a06688 100644
--- a/opal/mca/reachable/base/reachable_base_select.c
+++ b/opal/mca/reachable/base/reachable_base_select.c
@@ -10,14 +10,13 @@
  * $HEADER$
  */
 
-
 #include "opal_config.h"
 
 #include "opal/constants.h"
-#include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
-#include "opal/mca/reachable/reachable.h"
+#include "opal/mca/mca.h"
 #include "opal/mca/reachable/base/base.h"
+#include "opal/mca/reachable/reachable.h"
 
 /*
  * Globals
@@ -32,10 +31,11 @@ int opal_reachable_base_select(void)
     /*
      * Select the best component
      */
-    if( OPAL_SUCCESS != mca_base_select("reachable", opal_reachable_base_framework.framework_output,
-                                        &opal_reachable_base_framework.framework_components,
-                                        (mca_base_module_t **) &best_module,
-                                        (mca_base_component_t **) &best_component, NULL) ) {
+    if (OPAL_SUCCESS
+        != mca_base_select("reachable", opal_reachable_base_framework.framework_output,
+                           &opal_reachable_base_framework.framework_components,
+                           (mca_base_module_t **) &best_module,
+                           (mca_base_component_t **) &best_component, NULL)) {
         /* notify caller that no available component found */
         return OPAL_ERR_NOT_FOUND;
     }
diff --git a/opal/mca/reachable/netlink/Makefile.am b/opal/mca/reachable/netlink/Makefile.am
index 77b96f729fe..7126b119777 100644
--- a/opal/mca/reachable/netlink/Makefile.am
+++ b/opal/mca/reachable/netlink/Makefile.am
@@ -3,6 +3,7 @@
 # Copyright (c) 2016      Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -31,17 +32,17 @@ component_install =
 endif
 
 AM_CPPFLAGS = \
-        $(opal_reachable_netlink_CPPFLAGS) \
+        $(reachable_netlink_CPPFLAGS) \
         -DOPAL_HAVE_LIBNL3=$(OPAL_HAVE_LIBNL3)
 
 mcacomponentdir = $(opallibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_reachable_netlink_la_SOURCES = $(sources)
-mca_reachable_netlink_la_LDFLAGS = -module -avoid-version
-mca_reachable_netlink_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
-	$(opal_reachable_netlink_LIBS)
+mca_reachable_netlink_la_LDFLAGS = -module -avoid-version $(reachable_netlink_LDFLAGS)
+mca_reachable_netlink_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
+	$(reachable_netlink_LIBS)
 
 noinst_LTLIBRARIES = $(component_noinst)
 libmca_reachable_netlink_la_SOURCES =$(sources)
-libmca_reachable_netlink_la_LDFLAGS = -module -avoid-version
-libmca_reachable_netlink_la_LIBADD = $(opal_reachable_netlink_LIBS)
+libmca_reachable_netlink_la_LDFLAGS = -module -avoid-version $(reachable_netlink_LDFLAGS)
+libmca_reachable_netlink_la_LIBADD = $(reachable_netlink_LIBS)
diff --git a/opal/mca/reachable/netlink/configure.m4 b/opal/mca/reachable/netlink/configure.m4
index 15a996a4e97..7d35c344b2a 100644
--- a/opal/mca/reachable/netlink/configure.m4
+++ b/opal/mca/reachable/netlink/configure.m4
@@ -20,33 +20,49 @@ AC_DEFUN([MCA_opal_reachable_netlink_CONFIG],[
 
     OPAL_VAR_SCOPE_PUSH([opal_reachable_netlink_happy])
 
-    opal_reachable_netlink_happy=1
-    AC_CHECK_HEADER([linux/netlink.h], [],
-                    [opal_reachable_netlink_happy=0], [
-#include <sys/types.h>
-#include <net/if.h>
-])
-
     # this is terrible, but libnl-1 and libnl-3 are incompatible in
     # weird ways, and once there are libraries in LIBS for one, the
     # other is hard to get right.  So if someone has already decided
     # we have libnl version 1, get out.  Otherwise, see if we have
     # libnl-3, which is the only version supported by the netlink
     # component.
-    AS_IF([test $opal_libnl_version -eq 1],
-	  [opal_reachable_netlink_happy=0],
-          [OPAL_CHECK_LIBNL_V3([$opal_libnl_location],
-			       [opal_reachable_netlink])
-	   AS_IF([test "$OPAL_HAVE_LIBNL3" != "1"],
-		 [opal_reachable_netlink_happy=0])])
+    AS_IF([test $opal_libnl_version -eq 3],
+        [# Use the libnl base options for the route search
+         with_libnl_route=${with_libnl}
+         with_libnl_route_libdir=${with_libnl_libdir}
+         with_libnl_route_incdir=${with_libnl_incdir}
+
+         # The first argument of OAC_CHECK_PACKAGE must be a valid
+         # shell variable name, which means no dashes.  Deal with that
+         # by being explicit with our module.
+         m4_define([libnl_route_pkgconfig_module], [libnl-route-3.0])
+         OAC_CHECK_PACKAGE([libnl_route],
+                           [reachable_netlink],
+                           [netlink/route/route.h],
+                           [nl-route-3 nl-3],
+                           [rtnl_route_get],
+                           [opal_reachable_netlink_happy=1],
+                           [opal_reachable_netlink_happy=0])
+
+         # See if we have linux/netlink.h, which is part of the kernel
+         # ABI headers and not the libnl3 package (so we assume are in
+         # default includes, as opposed to the libnl directory.
+         AS_IF([test ${opal_reachable_netlink_happy} -eq 1],
+                [AC_CHECK_HEADER([linux/netlink.h], [],
+                   [opal_reachable_netlink_happy=0], [
+#include <sys/types.h>
+#include <net/if.h>
+                 ])])],
+        [AC_MSG_NOTICE([Disabling component because libnl v1 already linked])
+         opal_reachable_netlink_happy=0])
 
     AS_IF([test $opal_reachable_netlink_happy -eq 1],
           [$1],
           [$2])
 
-    AC_SUBST([opal_reachable_netlink_CPPFLAGS])
-    AC_SUBST([opal_reachable_netlink_LDFLAGS])
-    AC_SUBST([opal_reachable_netlink_LIBS])
+    AC_SUBST([reachable_netlink_CPPFLAGS])
+    AC_SUBST([reachable_netlink_LDFLAGS])
+    AC_SUBST([reachable_netlink_LIBS])
 
     OPAL_VAR_SCOPE_POP()
 ])
diff --git a/opal/mca/reachable/netlink/libnl3_utils.h b/opal/mca/reachable/netlink/libnl3_utils.h
index 3668685824f..583ccc15892 100644
--- a/opal/mca/reachable/netlink/libnl3_utils.h
+++ b/opal/mca/reachable/netlink/libnl3_utils.h
@@ -45,38 +45,38 @@
 #include <linux/netlink.h>
 #include <linux/rtnetlink.h>
 #include <netlink/netlink.h>
-#include <netlink/route/rtnl.h>
 #include <netlink/route/route.h>
+#include <netlink/route/rtnl.h>
 
 typedef struct nl_sock NL_HANDLE;
 
-#define NLMSG_SIZE(size) nlmsg_size(size)
-#define NL_GETERROR(err) nl_geterror(err)
-#define NL_HANDLE_ALLOC nl_socket_alloc
-#define NL_HANDLE_FREE nl_socket_free
+#define NLMSG_SIZE(size)     nlmsg_size(size)
+#define NL_GETERROR(err)     nl_geterror(err)
+#define NL_HANDLE_ALLOC      nl_socket_alloc
+#define NL_HANDLE_FREE       nl_socket_free
 #define NL_DISABLE_SEQ_CHECK nl_socket_disable_seq_check
 #define INC_CB_MSGCNT(arg)
 
 /* err will be returned as -NLE_AGAIN */
 /* if the socket times out */
-#define NL_RECVMSGS(nlh, cb_arg, rc, err, out) \
-	do { \
-		err = nl_recvmsgs_default(nlh); \
-		if (err < 0) { \
-                    opal_output(0, "Failed to receive netlink reply message, error %s\n", \
-				NL_GETERROR(err)); \
-			if (err == -NLE_AGAIN) \
-				err = rc; \
-			goto out; \
-		} \
-	} while (0)
+#define NL_RECVMSGS(nlh, cb_arg, rc, err, out)                                    \
+    do {                                                                          \
+        err = nl_recvmsgs_default(nlh);                                           \
+        if (err < 0) {                                                            \
+            opal_output(0, "Failed to receive netlink reply message, error %s\n", \
+                        NL_GETERROR(err));                                        \
+            if (err == -NLE_AGAIN)                                                \
+                err = rc;                                                         \
+            goto out;                                                             \
+        }                                                                         \
+    } while (0)
 
 struct opal_reachable_netlink_rt_cb_arg {
-	int			oif;
-	int			found;
-	int			has_gateway;
-	int			replied;
-	struct opal_reachable_netlink_sk	*unlsk;
+    int oif;
+    int found;
+    int has_gateway;
+    int replied;
+    struct opal_reachable_netlink_sk *unlsk;
 };
 
 #endif /* LIBNL3_UTILS_H */
diff --git a/opal/mca/reachable/netlink/libnl_utils.h b/opal/mca/reachable/netlink/libnl_utils.h
index 6a7c7cc5538..6b55aa78d66 100644
--- a/opal/mca/reachable/netlink/libnl_utils.h
+++ b/opal/mca/reachable/netlink/libnl_utils.h
@@ -49,8 +49,8 @@
 #include "libnl3_utils.h"
 
 struct opal_reachable_netlink_sk {
-    NL_HANDLE	*nlh;
-    uint32_t	seq;
+    NL_HANDLE *nlh;
+    uint32_t seq;
 };
 
 /* returns 0 if host is reachable, EHOSTUNREACH if the host
@@ -59,9 +59,8 @@ struct opal_reachable_netlink_sk {
  * If the route to the destination is through a gateway, *has_gateway
  * is set to 1.  Otherwise, it is set to 0.
  */
-int opal_reachable_netlink_rt_lookup(uint32_t src_addr,
-				     uint32_t dst_addr, int oif,
-				     int *has_gateway);
+int opal_reachable_netlink_rt_lookup(uint32_t src_addr, uint32_t dst_addr, int oif,
+                                     int *has_gateway);
 
 #if OPAL_ENABLE_IPV6
 /* returns 0 if host is reachable, EHOSTUNREACH if the host
@@ -70,9 +69,8 @@ int opal_reachable_netlink_rt_lookup(uint32_t src_addr,
  * If the route to the destination is through a gateway, *has_gateway
  * is set to 1.  Otherwise, it is set to 0.
  */
-int opal_reachable_netlink_rt_lookup6(struct in6_addr *src_addr,
-				      struct in6_addr *dst_addr, int oif,
-				      int *has_gateway);
+int opal_reachable_netlink_rt_lookup6(struct in6_addr *src_addr, struct in6_addr *dst_addr, int oif,
+                                      int *has_gateway);
 #endif
 
 #endif /* LIBNL_UTILS_H */
diff --git a/opal/mca/reachable/netlink/reachable_netlink.h b/opal/mca/reachable/netlink/reachable_netlink.h
index 3581d89a315..537cd38987b 100644
--- a/opal/mca/reachable/netlink/reachable_netlink.h
+++ b/opal/mca/reachable/netlink/reachable_netlink.h
@@ -16,11 +16,9 @@
 
 BEGIN_C_DECLS
 
-OPAL_DECLSPEC extern opal_reachable_base_component_t
-    mca_reachable_netlink_component;
+OPAL_DECLSPEC extern opal_reachable_base_component_t mca_reachable_netlink_component;
 
-OPAL_DECLSPEC extern const opal_reachable_base_module_t
-    opal_reachable_netlink_module;
+OPAL_DECLSPEC extern const opal_reachable_base_module_t opal_reachable_netlink_module;
 
 END_C_DECLS
 
diff --git a/opal/mca/reachable/netlink/reachable_netlink_component.c b/opal/mca/reachable/netlink/reachable_netlink_component.c
index f958eb75c49..f015e068faa 100644
--- a/opal/mca/reachable/netlink/reachable_netlink_component.c
+++ b/opal/mca/reachable/netlink/reachable_netlink_component.c
@@ -14,15 +14,15 @@
 #include "opal_config.h"
 
 #include "opal/constants.h"
-#include "opal/util/proc.h"
 #include "opal/mca/reachable/reachable.h"
+#include "opal/util/proc.h"
 #include "reachable_netlink.h"
 
 /*
  * Public string showing the reachable netlink component version number
  */
-const char *opal_reachable_netlink_component_version_string =
-    "OPAL netlink reachable MCA component version " OPAL_VERSION;
+const char *opal_reachable_netlink_component_version_string
+    = "OPAL netlink reachable MCA component version " OPAL_VERSION;
 
 /*
  * Local function
@@ -32,7 +32,6 @@ static int reachable_netlink_close(void);
 static int reachable_netlink_component_query(mca_base_module_t **module, int *priority);
 static int component_register(void);
 
-
 /*
  * Instantiate the public struct with all of our public information
  * and pointers to our public functions in it
@@ -43,31 +42,29 @@ opal_reachable_base_component_t mca_reachable_netlink_component = {
     /* First, the mca_component_t struct containing meta information
        about the component itself */
 
-    .base_version = {
-        /* Indicate that we are a reachable v1.1.0 component (which also
-           implies a specific MCA version) */
+    .base_version =
+        {
+            /* Indicate that we are a reachable v1.1.0 component (which also
+               implies a specific MCA version) */
 
-        OPAL_REACHABLE_BASE_VERSION_2_0_0,
+            OPAL_REACHABLE_BASE_VERSION_2_0_0,
 
-        /* Component name and version */
+            /* Component name and version */
 
-        .mca_component_name = "netlink",
-        MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
-                              OPAL_RELEASE_VERSION),
+            .mca_component_name = "netlink",
+            MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
+                                  OPAL_RELEASE_VERSION),
 
-        /* Component open and close functions */
+            /* Component open and close functions */
 
-        .mca_open_component = reachable_netlink_open,
-        .mca_close_component = reachable_netlink_close,
-        .mca_query_component = reachable_netlink_component_query,
-        .mca_register_component_params = component_register,
-    },
+            .mca_open_component = reachable_netlink_open,
+            .mca_close_component = reachable_netlink_close,
+            .mca_query_component = reachable_netlink_component_query,
+            .mca_register_component_params = component_register,
+        },
     /* Next the MCA v1.0.0 component meta data */
-    .base_data = {
-        /* The component is checkpoint ready */
-        MCA_BASE_METADATA_PARAM_CHECKPOINT
-    }
-};
+    .base_data = {/* The component is checkpoint ready */
+                  MCA_BASE_METADATA_PARAM_CHECKPOINT}};
 
 static int reachable_netlink_open(void)
 {
@@ -86,8 +83,7 @@ static int component_register(void)
     return OPAL_SUCCESS;
 }
 
-static int
-reachable_netlink_component_query(mca_base_module_t **module, int *priority)
+static int reachable_netlink_component_query(mca_base_module_t **module, int *priority)
 {
     *priority = 50;
     *module = (mca_base_module_t *) &opal_reachable_netlink_module;
diff --git a/opal/mca/reachable/netlink/reachable_netlink_module.c b/opal/mca/reachable/netlink/reachable_netlink_module.c
index cf6609242d5..9f772b53811 100644
--- a/opal/mca/reachable/netlink/reachable_netlink_module.c
+++ b/opal/mca/reachable/netlink/reachable_netlink_module.c
@@ -16,27 +16,22 @@
 #include "opal/types.h"
 
 #ifdef HAVE_MATH_H
-#include <math.h>
+#    include <math.h>
 #endif
 
+#include "libnl_utils.h"
+#include "opal/mca/reachable/base/base.h"
 #include "opal/util/net.h"
 #include "opal/util/string_copy.h"
-#include "opal/mca/reachable/base/base.h"
 #include "reachable_netlink.h"
-#include "libnl_utils.h"
 
-enum connection_quality {
-    CQ_NO_CONNECTION = 0,
-    CQ_DIFFERENT_NETWORK = 50,
-    CQ_SAME_NETWORK = 100
-};
+enum connection_quality { CQ_NO_CONNECTION = 0, CQ_DIFFERENT_NETWORK = 50, CQ_SAME_NETWORK = 100 };
 
 /* Local variables */
 static int init_counter = 0;
 
 static int get_weights(opal_if_t *local_if, opal_if_t *remote_if);
-static int calculate_weight(int bandwidth_local, int bandwidth_remote,
-                            int connection_quality);
+static int calculate_weight(int bandwidth_local, int bandwidth_remote, int connection_quality);
 
 static int netlink_init(void)
 {
@@ -59,8 +54,7 @@ static int netlink_fini(void)
  * Higher weightings are given to connections on the same
  * network.
  */
-static opal_reachable_t* netlink_reachable(opal_list_t *local_ifs,
-                                           opal_list_t *remote_ifs)
+static opal_reachable_t *netlink_reachable(opal_list_t *local_ifs, opal_list_t *remote_ifs)
 {
     opal_reachable_t *reachable_results = NULL;
     int i, j;
@@ -73,9 +67,9 @@ static opal_reachable_t* netlink_reachable(opal_list_t *local_ifs,
     }
 
     i = 0;
-    OPAL_LIST_FOREACH(local_iter, local_ifs, opal_if_t) {
+    OPAL_LIST_FOREACH (local_iter, local_ifs, opal_if_t) {
         j = 0;
-        OPAL_LIST_FOREACH(remote_iter, remote_ifs, opal_if_t) {
+        OPAL_LIST_FOREACH (remote_iter, remote_ifs, opal_if_t) {
             reachable_results->weights[i][j] = get_weights(local_iter, remote_iter);
             j++;
         }
@@ -85,7 +79,6 @@ static opal_reachable_t* netlink_reachable(opal_list_t *local_ifs,
     return reachable_results;
 }
 
-
 static int get_weights(opal_if_t *local_if, opal_if_t *remote_if)
 {
     char str_local[128], str_remote[128], *conn_type;
@@ -93,13 +86,11 @@ static int get_weights(opal_if_t *local_if, opal_if_t *remote_if)
 
     /* opal_net_get_hostname returns a static buffer.  Great for
        single address printfs, need to copy in this case */
-    opal_string_copy(str_local,
-            opal_net_get_hostname((struct sockaddr *)&local_if->if_addr),
-            sizeof(str_local));
+    opal_string_copy(str_local, opal_net_get_hostname((struct sockaddr *) &local_if->if_addr),
+                     sizeof(str_local));
     str_local[sizeof(str_local) - 1] = '\0';
-    opal_string_copy(str_remote,
-            opal_net_get_hostname((struct sockaddr *)&remote_if->if_addr),
-            sizeof(str_remote));
+    opal_string_copy(str_remote, opal_net_get_hostname((struct sockaddr *) &remote_if->if_addr),
+                     sizeof(str_remote));
     str_remote[sizeof(str_remote) - 1] = '\0';
 
     /*  initially, assume no connection is possible */
@@ -108,8 +99,8 @@ static int get_weights(opal_if_t *local_if, opal_if_t *remote_if)
     if (AF_INET == local_if->af_family && AF_INET == remote_if->af_family) {
         uint32_t local_ip, remote_ip;
 
-        local_ip = (uint32_t)((struct sockaddr_in *)&(local_if->if_addr))->sin_addr.s_addr;
-        remote_ip = (uint32_t)((struct sockaddr_in *)&(remote_if->if_addr))->sin_addr.s_addr;
+        local_ip = (uint32_t)((struct sockaddr_in *) &(local_if->if_addr))->sin_addr.s_addr;
+        remote_ip = (uint32_t)((struct sockaddr_in *) &(remote_if->if_addr))->sin_addr.s_addr;
         outgoing_interface = local_if->if_kernel_index;
 
         /* If the ips are identical, assume reachable through loopback. This
@@ -117,26 +108,21 @@ static int get_weights(opal_if_t *local_if, opal_if_t *remote_if)
            maintain similar behavior to previous implementations. */
         if (local_ip == remote_ip) {
             conn_type = "IPv4 SAME NETWORK";
-            weight = calculate_weight(local_if->if_bandwidth,
-                                      remote_if->if_bandwidth,
+            weight = calculate_weight(local_if->if_bandwidth, remote_if->if_bandwidth,
                                       CQ_SAME_NETWORK);
             goto out;
         }
 
-        ret = opal_reachable_netlink_rt_lookup(local_ip,
-                                               remote_ip,
-                                               outgoing_interface,
+        ret = opal_reachable_netlink_rt_lookup(local_ip, remote_ip, outgoing_interface,
                                                &has_gateway);
         if (0 == ret) {
             if (0 == has_gateway) {
                 conn_type = "IPv4 SAME NETWORK";
-                weight = calculate_weight(local_if->if_bandwidth,
-                                          remote_if->if_bandwidth,
+                weight = calculate_weight(local_if->if_bandwidth, remote_if->if_bandwidth,
                                           CQ_SAME_NETWORK);
             } else {
                 conn_type = "IPv4 DIFFERENT NETWORK";
-                weight = calculate_weight(local_if->if_bandwidth,
-                                          remote_if->if_bandwidth,
+                weight = calculate_weight(local_if->if_bandwidth, remote_if->if_bandwidth,
                                           CQ_DIFFERENT_NETWORK);
             }
         } else {
@@ -148,8 +134,8 @@ static int get_weights(opal_if_t *local_if, opal_if_t *remote_if)
     } else if (AF_INET6 == local_if->af_family && AF_INET6 == remote_if->af_family) {
         struct in6_addr *local_ip, *remote_ip;
 
-        local_ip = &((struct sockaddr_in6 *)&(local_if->if_addr))->sin6_addr;
-        remote_ip = &((struct sockaddr_in6 *)&(remote_if->if_addr))->sin6_addr;
+        local_ip = &((struct sockaddr_in6 *) &(local_if->if_addr))->sin6_addr;
+        remote_ip = &((struct sockaddr_in6 *) &(remote_if->if_addr))->sin6_addr;
         outgoing_interface = local_if->if_kernel_index;
 
         /* If the ips are identical, assume reachable through loopback. This
@@ -157,28 +143,23 @@ static int get_weights(opal_if_t *local_if, opal_if_t *remote_if)
            maintain similar behavior to previous implementations. */
         if (local_ip == remote_ip) {
             conn_type = "IPv6 SAME NETWORK";
-            weight = calculate_weight(local_if->if_bandwidth,
-                                      remote_if->if_bandwidth,
+            weight = calculate_weight(local_if->if_bandwidth, remote_if->if_bandwidth,
                                       CQ_SAME_NETWORK);
 
             goto out;
         }
 
-        ret = opal_reachable_netlink_rt_lookup6(local_ip,
-                                                remote_ip,
-                                                outgoing_interface,
+        ret = opal_reachable_netlink_rt_lookup6(local_ip, remote_ip, outgoing_interface,
                                                 &has_gateway);
 
         if (0 == ret) {
             if (0 == has_gateway) {
                 conn_type = "IPv6 SAME NETWORK";
-                weight = calculate_weight(local_if->if_bandwidth,
-                                          remote_if->if_bandwidth,
+                weight = calculate_weight(local_if->if_bandwidth, remote_if->if_bandwidth,
                                           CQ_SAME_NETWORK);
             } else {
                 conn_type = "IPv6 DIFFERENT NETWORK";
-                weight = calculate_weight(local_if->if_bandwidth,
-                                          remote_if->if_bandwidth,
+                weight = calculate_weight(local_if->if_bandwidth, remote_if->if_bandwidth,
                                           CQ_DIFFERENT_NETWORK);
             }
         } else {
@@ -196,19 +177,14 @@ static int get_weights(opal_if_t *local_if, opal_if_t *remote_if)
 
 out:
     opal_output_verbose(20, opal_reachable_base_framework.framework_output,
-                        "reachable:netlink: path from %s to %s: %s",
-                        str_local, str_remote, conn_type);
+                        "reachable:netlink: path from %s to %s: %s", str_local, str_remote,
+                        conn_type);
 
     return weight;
 }
 
-
-const opal_reachable_base_module_t opal_reachable_netlink_module = {
-    netlink_init,
-    netlink_fini,
-    netlink_reachable
-};
-
+const opal_reachable_base_module_t opal_reachable_netlink_module = {netlink_init, netlink_fini,
+                                                                    netlink_reachable};
 
 /*
  * Weights determined by bandwidth between
@@ -236,10 +212,10 @@ const opal_reachable_base_module_t opal_reachable_netlink_module = {
  * connection_quality to be large enough
  * to capture decimals
  */
-static int calculate_weight(int bandwidth_local, int bandwidth_remote,
-                            int connection_quality)
+static int calculate_weight(int bandwidth_local, int bandwidth_remote, int connection_quality)
 {
-    int weight = connection_quality * (MIN(bandwidth_local, bandwidth_remote) +
-                                       1.0/(1.0 + (double)abs(bandwidth_local - bandwidth_remote)));
+    int weight = connection_quality
+                 * (MIN(bandwidth_local, bandwidth_remote)
+                    + 1.0 / (1.0 + (double) abs(bandwidth_local - bandwidth_remote)));
     return weight;
 }
diff --git a/opal/mca/reachable/netlink/reachable_netlink_utils_common.c b/opal/mca/reachable/netlink/reachable_netlink_utils_common.c
index 9422c22d180..163f7f831c9 100644
--- a/opal/mca/reachable/netlink/reachable_netlink_utils_common.c
+++ b/opal/mca/reachable/netlink/reachable_netlink_utils_common.c
@@ -42,11 +42,11 @@
 
 #include "opal_config.h"
 
-#include <errno.h>
 #include <arpa/inet.h>
+#include <errno.h>
 #include <time.h>
 #ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
+#    include <netinet/in.h>
 #endif
 
 #include "libnl_utils.h"
@@ -54,29 +54,31 @@
 /* Adapt this copied code for Open MPI */
 #include "opal/util/output.h"
 
-
-static struct nla_policy route_policy[RTA_MAX+1] = {
-	[RTA_IIF]	= { .type = NLA_STRING,
-			    .maxlen = IFNAMSIZ, },
-	[RTA_OIF]	= { .type = NLA_U32 },
-	[RTA_PRIORITY]	= { .type = NLA_U32 },
-	[RTA_FLOW]	= { .type = NLA_U32 },
-	[RTA_MP_ALGO]	= { .type = NLA_U32 },
-	[RTA_CACHEINFO]	= { .minlen = sizeof(struct rta_cacheinfo) },
-	[RTA_METRICS]	= { .type = NLA_NESTED },
-	[RTA_MULTIPATH]	= { .type = NLA_NESTED },
+static struct nla_policy route_policy[RTA_MAX + 1] = {
+    [RTA_IIF] =
+        {
+            .type = NLA_STRING,
+            .maxlen = IFNAMSIZ,
+        },
+    [RTA_OIF] = {.type = NLA_U32},
+    [RTA_PRIORITY] = {.type = NLA_U32},
+    [RTA_FLOW] = {.type = NLA_U32},
+    [RTA_MP_ALGO] = {.type = NLA_U32},
+    [RTA_CACHEINFO] = {.minlen = sizeof(struct rta_cacheinfo)},
+    [RTA_METRICS] = {.type = NLA_NESTED},
+    [RTA_MULTIPATH] = {.type = NLA_NESTED},
 };
 
 static int opal_reachable_netlink_is_nlreply_expected(struct opal_reachable_netlink_sk *unlsk,
-						      struct nlmsghdr *nlm_hdr)
+                                                      struct nlmsghdr *nlm_hdr)
 {
 #if OPAL_ENABLE_DEBUG
     if (nlm_hdr->nlmsg_pid != nl_socket_get_local_port(unlsk->nlh)
         || nlm_hdr->nlmsg_seq != unlsk->seq) {
-        opal_output(0, "Not an expected reply msg pid: %u local pid: %u msg seq: %u expected seq: %u\n",
-                    nlm_hdr->nlmsg_pid,
-                    nl_socket_get_local_port(unlsk->nlh),
-                    nlm_hdr->nlmsg_seq, unlsk->seq);
+        opal_output(
+            0, "Not an expected reply msg pid: %u local pid: %u msg seq: %u expected seq: %u\n",
+            nlm_hdr->nlmsg_pid, nl_socket_get_local_port(unlsk->nlh), nlm_hdr->nlmsg_seq,
+            unlsk->seq);
         return 0;
     }
 #endif
@@ -87,13 +89,11 @@ static int opal_reachable_netlink_is_nlreply_expected(struct opal_reachable_netl
 static int opal_reachable_netlink_is_nlreply_err(struct nlmsghdr *nlm_hdr)
 {
     if (nlm_hdr->nlmsg_type == NLMSG_ERROR) {
-        struct nlmsgerr *e = (struct nlmsgerr *)nlmsg_data(nlm_hdr);
-        if (nlm_hdr->nlmsg_len >= (__u32)NLMSG_SIZE(sizeof(*e)))
-            opal_output_verbose(20, 0,
-                                "Received a netlink error message");
+        struct nlmsgerr *e = (struct nlmsgerr *) nlmsg_data(nlm_hdr);
+        if (nlm_hdr->nlmsg_len >= (__u32) NLMSG_SIZE(sizeof(*e)))
+            opal_output_verbose(20, 0, "Received a netlink error message");
         else
-            opal_output_verbose(20, 0,
-                                "Received a truncated netlink error message\n");
+            opal_output_verbose(20, 0, "Received a truncated netlink error message\n");
         return 1;
     }
 
@@ -101,8 +101,7 @@ static int opal_reachable_netlink_is_nlreply_err(struct nlmsghdr *nlm_hdr)
 }
 
 static int opal_reachable_netlink_send_query(struct opal_reachable_netlink_sk *unlsk,
-					     struct nl_msg *msg,
-					     int protocol, int flag)
+                                             struct nl_msg *msg, int protocol, int flag)
 {
     struct nlmsghdr *nlhdr;
 
@@ -123,8 +122,8 @@ static int opal_reachable_netlink_set_rcvsk_timer(NL_HANDLE *nlh)
     timeout.tv_sec = 1;
     timeout.tv_usec = 0;
 
-    err = setsockopt(nl_socket_get_fd(nlh), SOL_SOCKET, SO_RCVTIMEO,
-                     (char *)&timeout, sizeof(timeout));
+    err = setsockopt(nl_socket_get_fd(nlh), SOL_SOCKET, SO_RCVTIMEO, (char *) &timeout,
+                     sizeof(timeout));
 #if OPAL_ENABLE_DEBUG
     if (err < 0)
         opal_output(0, "Failed to set SO_RCVTIMEO for nl socket");
@@ -154,8 +153,7 @@ static int opal_reachable_netlink_sk_alloc(struct opal_reachable_netlink_sk **p_
 
     err = nl_connect(nlh, protocol);
     if (err < 0) {
-        opal_output(0, "Failed to connnect netlink route socket error: %s\n",
-                    NL_GETERROR(err));
+        opal_output(0, "Failed to connect netlink route socket error: %s\n", NL_GETERROR(err));
         err = EINVAL;
         goto err_free_nlh;
     }
@@ -170,11 +168,11 @@ static int opal_reachable_netlink_sk_alloc(struct opal_reachable_netlink_sk **p_
     *p_sk = unlsk;
     return 0;
 
- err_close_nlh:
+err_close_nlh:
     nl_close(nlh);
- err_free_nlh:
+err_free_nlh:
     NL_HANDLE_FREE(nlh);
- err_free_unlsk:
+err_free_unlsk:
     free(unlsk);
     return err;
 }
@@ -188,7 +186,8 @@ static void opal_reachable_netlink_sk_free(struct opal_reachable_netlink_sk *unl
 
 static int opal_reachable_netlink_rt_raw_parse_cb(struct nl_msg *msg, void *arg)
 {
-    struct opal_reachable_netlink_rt_cb_arg *lookup_arg = (struct opal_reachable_netlink_rt_cb_arg *)arg;
+    struct opal_reachable_netlink_rt_cb_arg *lookup_arg = (struct opal_reachable_netlink_rt_cb_arg
+                                                               *) arg;
     struct opal_reachable_netlink_sk *unlsk = lookup_arg->unlsk;
     struct nlmsghdr *nlm_hdr = nlmsg_hdr(msg);
     struct rtmsg *rtm;
@@ -216,8 +215,7 @@ static int opal_reachable_netlink_rt_raw_parse_cb(struct nl_msg *msg, void *arg)
 #if OPAL_ENABLE_DEBUG
         char buf[128];
         nl_nlmsgtype2str(nlm_hdr->nlmsg_type, buf, sizeof(buf));
-        opal_output(0, "Received an invalid route request reply message type: %s\n",
-                    buf);
+        opal_output(0, "Received an invalid route request reply message type: %s\n", buf);
         nl_msg_dump(msg, stderr);
 #endif
         return NL_SKIP;
@@ -226,19 +224,17 @@ static int opal_reachable_netlink_rt_raw_parse_cb(struct nl_msg *msg, void *arg)
     rtm = nlmsg_data(nlm_hdr);
     if (rtm->rtm_family != AF_INET
 #if OPAL_ENABLE_IPV6
-	&& rtm->rtm_family != AF_INET6
+        && rtm->rtm_family != AF_INET6
 #endif
-	) {
+    ) {
 #if OPAL_ENABLE_DEBUG
-        opal_output(0, "RTM message contains invalid AF family: %u\n",
-                    rtm->rtm_family);
+        opal_output(0, "RTM message contains invalid AF family: %u\n", rtm->rtm_family);
         nl_msg_dump(msg, stderr);
 #endif
         return NL_SKIP;
     }
 
-    err = nlmsg_parse(nlm_hdr, sizeof(struct rtmsg), tb, RTA_MAX,
-                      route_policy);
+    err = nlmsg_parse(nlm_hdr, sizeof(struct rtmsg), tb, RTA_MAX, route_policy);
     if (err < 0) {
 #if OPAL_ENABLE_DEBUG
         opal_output(0, "nlmsg parse error %s\n", NL_GETERROR(err));
@@ -248,15 +244,15 @@ static int opal_reachable_netlink_rt_raw_parse_cb(struct nl_msg *msg, void *arg)
     }
 
     if (tb[RTA_OIF]) {
-        if (nla_get_u32(tb[RTA_OIF]) == (uint32_t)lookup_arg->oif)
+        if (nla_get_u32(tb[RTA_OIF]) == (uint32_t) lookup_arg->oif)
             found = 1;
         else
             /* usually, this means that there is a route to the remote
                host, but that it's not through the given interface.  For
                our purposes, that means it's not reachable. */
-            opal_output_verbose(20, 0, "Retrieved route has a different outgoing interface %d (expected %d)\n",
-				nla_get_u32(tb[RTA_OIF]),
-				lookup_arg->oif);
+            opal_output_verbose(
+                20, 0, "Retrieved route has a different outgoing interface %d (expected %d)\n",
+                nla_get_u32(tb[RTA_OIF]), lookup_arg->oif);
     }
 
     if (found && tb[RTA_GATEWAY]) {
@@ -266,14 +262,12 @@ static int opal_reachable_netlink_rt_raw_parse_cb(struct nl_msg *msg, void *arg)
     return NL_STOP;
 }
 
-int opal_reachable_netlink_rt_lookup(uint32_t src_addr,
-				     uint32_t dst_addr,
-				     int outgoing_interface,
+int opal_reachable_netlink_rt_lookup(uint32_t src_addr, uint32_t dst_addr, int outgoing_interface,
                                      int *has_gateway)
 {
-    struct opal_reachable_netlink_sk *unlsk; /* netlink socket */
-    struct nl_msg *nlm; /* netlink message */
-    struct rtmsg rmsg; /* route message */
+    struct opal_reachable_netlink_sk *unlsk;     /* netlink socket */
+    struct nl_msg *nlm;                          /* netlink message */
+    struct rtmsg rmsg;                           /* route message */
     struct opal_reachable_netlink_rt_cb_arg arg; /* callback argument */
     int err;
 
@@ -292,8 +286,7 @@ int opal_reachable_netlink_rt_lookup(uint32_t src_addr,
     /* allocate netlink message of type RTM_GETROUTE */
     nlm = nlmsg_alloc_simple(RTM_GETROUTE, 0);
     if (!nlm) {
-        opal_output(0, "Failed to alloc nl message, %s\n",
-                    NL_GETERROR(err));
+        opal_output(0, "Failed to alloc nl message, %s\n", NL_GETERROR(err));
         err = ENOMEM;
         goto out;
     }
@@ -307,8 +300,7 @@ int opal_reachable_netlink_rt_lookup(uint32_t src_addr,
     err = opal_reachable_netlink_send_query(unlsk, nlm, NETLINK_ROUTE, NLM_F_REQUEST);
     nlmsg_free(nlm);
     if (err < 0) {
-        opal_output(0, "Failed to send RTM_GETROUTE query message, error %s\n",
-                    NL_GETERROR(err));
+        opal_output(0, "Failed to send RTM_GETROUTE query message, error %s\n", NL_GETERROR(err));
         err = EINVAL;
         goto out;
     }
@@ -320,13 +312,12 @@ int opal_reachable_netlink_rt_lookup(uint32_t src_addr,
     err = nl_socket_modify_cb(unlsk->nlh, NL_CB_MSG_IN, NL_CB_CUSTOM,
                               opal_reachable_netlink_rt_raw_parse_cb, &arg);
     if (err != 0) {
-        opal_output(0, "Failed to setup callback function, error %s\n",
-                    NL_GETERROR(err));
+        opal_output(0, "Failed to setup callback function, error %s\n", NL_GETERROR(err));
         err = EINVAL;
         goto out;
     }
 
-    /* recieve results */
+    /* receive results */
     NL_RECVMSGS(unlsk->nlh, arg, EHOSTUNREACH, err, out);
 
     /* check whether a route was found */
@@ -338,22 +329,19 @@ int opal_reachable_netlink_rt_lookup(uint32_t src_addr,
         err = EHOSTUNREACH;
     }
 
- out:
+out:
     opal_reachable_netlink_sk_free(unlsk);
     return err;
 }
 
-
 #if OPAL_ENABLE_IPV6
-int opal_reachable_netlink_rt_lookup6(struct in6_addr *src_addr,
-				      struct in6_addr *dst_addr,
-				      int outgoing_interface,
-				      int *has_gateway)
+int opal_reachable_netlink_rt_lookup6(struct in6_addr *src_addr, struct in6_addr *dst_addr,
+                                      int outgoing_interface, int *has_gateway)
 {
 
-    struct opal_reachable_netlink_sk *unlsk; /* netlink socket */
-    struct nl_msg *nlm; /* netlink message */
-    struct rtmsg rmsg; /* route message */
+    struct opal_reachable_netlink_sk *unlsk;     /* netlink socket */
+    struct nl_msg *nlm;                          /* netlink message */
+    struct rtmsg rmsg;                           /* route message */
     struct opal_reachable_netlink_rt_cb_arg arg; /* callback argument */
     int err;
 
@@ -361,7 +349,7 @@ int opal_reachable_netlink_rt_lookup6(struct in6_addr *src_addr,
     unlsk = NULL;
     err = opal_reachable_netlink_sk_alloc(&unlsk, NETLINK_ROUTE);
     if (err)
-	return err;
+        return err;
 
     /* allocate route message */
     memset(&rmsg, 0, sizeof(rmsg));
@@ -372,10 +360,9 @@ int opal_reachable_netlink_rt_lookup6(struct in6_addr *src_addr,
     /* allocate netlink message of type RTM_GETROUTE */
     nlm = nlmsg_alloc_simple(RTM_GETROUTE, 0);
     if (!nlm) {
-	opal_output(0, "Failed to alloc nl message, %s\n",
-		    NL_GETERROR(err));
-	err = ENOMEM;
-	goto out;
+        opal_output(0, "Failed to alloc nl message, %s\n", NL_GETERROR(err));
+        err = ENOMEM;
+        goto out;
     }
 
     /* append route message and addresses to netlink message.   */
@@ -387,10 +374,9 @@ int opal_reachable_netlink_rt_lookup6(struct in6_addr *src_addr,
     err = opal_reachable_netlink_send_query(unlsk, nlm, NETLINK_ROUTE, NLM_F_REQUEST);
     nlmsg_free(nlm);
     if (err < 0) {
-	opal_output(0, "Failed to send RTM_GETROUTE query message, error %s\n",
-		    NL_GETERROR(err));
-	err = EINVAL;
-	goto out;
+        opal_output(0, "Failed to send RTM_GETROUTE query message, error %s\n", NL_GETERROR(err));
+        err = EINVAL;
+        goto out;
     }
 
     /* Setup callback function */
@@ -398,12 +384,11 @@ int opal_reachable_netlink_rt_lookup6(struct in6_addr *src_addr,
     arg.oif = outgoing_interface;
     arg.unlsk = unlsk;
     err = nl_socket_modify_cb(unlsk->nlh, NL_CB_MSG_IN, NL_CB_CUSTOM,
-			      opal_reachable_netlink_rt_raw_parse_cb, &arg);
+                              opal_reachable_netlink_rt_raw_parse_cb, &arg);
     if (err != 0) {
-	opal_output(0, "Failed to setup callback function, error %s\n",
-		    NL_GETERROR(err));
-	err = EINVAL;
-	goto out;
+        opal_output(0, "Failed to setup callback function, error %s\n", NL_GETERROR(err));
+        err = EINVAL;
+        goto out;
     }
 
     /* receive results */
@@ -412,13 +397,13 @@ int opal_reachable_netlink_rt_lookup6(struct in6_addr *src_addr,
     /* check whether a route was found */
     if (arg.found) {
         *has_gateway = arg.has_gateway;
-	err = 0;
+        err = 0;
     } else {
         *has_gateway = 0;
-	err = EHOSTUNREACH;
+        err = EHOSTUNREACH;
     }
 
- out:
+out:
     opal_reachable_netlink_sk_free(unlsk);
     return err;
 }
diff --git a/opal/mca/reachable/reachable.h b/opal/mca/reachable/reachable.h
index 9d2f31f706d..0eb9159609d 100644
--- a/opal/mca/reachable/reachable.h
+++ b/opal/mca/reachable/reachable.h
@@ -16,12 +16,11 @@
 #define OPAL_REACHABLE_H
 
 #include "opal_config.h"
-#include "opal/types.h"
 #include "opal/class/opal_object.h"
+#include "opal/types.h"
 
-#include "opal/mca/mca.h"
 #include "opal/mca/if/if.h"
-
+#include "opal/mca/mca.h"
 
 BEGIN_C_DECLS
 
@@ -79,37 +78,33 @@ typedef int (*opal_reachable_base_module_fini_fn_t)(void);
  * reachable.
  *
  */
-typedef opal_reachable_t*
-(*opal_reachable_base_module_reachable_fn_t)(opal_list_t *local_ifs,
-                                             opal_list_t *remote_ifs);
-
+typedef opal_reachable_t *(*opal_reachable_base_module_reachable_fn_t)(opal_list_t *local_ifs,
+                                                                       opal_list_t *remote_ifs);
 
 /*
  * the standard public API data structure
  */
 typedef struct {
     /* currently used APIs */
-    opal_reachable_base_module_init_fn_t                   init;
-    opal_reachable_base_module_fini_fn_t                   finalize;
-    opal_reachable_base_module_reachable_fn_t              reachable;
+    opal_reachable_base_module_init_fn_t init;
+    opal_reachable_base_module_fini_fn_t finalize;
+    opal_reachable_base_module_reachable_fn_t reachable;
 } opal_reachable_base_module_t;
 
 typedef struct {
-    mca_base_component_t                      base_version;
-    mca_base_component_data_t                 base_data;
+    mca_base_component_t base_version;
+    mca_base_component_data_t base_data;
     int priority;
 } opal_reachable_base_component_t;
 
 /*
  * Macro for use in components that are of type reachable
  */
-#define OPAL_REACHABLE_BASE_VERSION_2_0_0             \
-    OPAL_MCA_BASE_VERSION_2_1_0("reachable", 2, 0, 0)
+#define OPAL_REACHABLE_BASE_VERSION_2_0_0 OPAL_MCA_BASE_VERSION_2_1_0("reachable", 2, 0, 0)
 
 /* Global structure for accessing reachability functions */
 OPAL_DECLSPEC extern opal_reachable_base_module_t opal_reachable;
 
-
 END_C_DECLS
 
 #endif
diff --git a/opal/mca/reachable/weighted/Makefile.am b/opal/mca/reachable/weighted/Makefile.am
index a0ef2867476..87b472f109f 100644
--- a/opal/mca/reachable/weighted/Makefile.am
+++ b/opal/mca/reachable/weighted/Makefile.am
@@ -29,7 +29,7 @@ mcacomponentdir = $(opallibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_reachable_weighted_la_SOURCES = $(sources)
 mca_reachable_weighted_la_LDFLAGS = -module -avoid-version
-mca_reachable_weighted_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+mca_reachable_weighted_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 
 noinst_LTLIBRARIES = $(component_noinst)
 libmca_reachable_weighted_la_SOURCES =$(sources)
diff --git a/opal/mca/reachable/weighted/reachable_weighted.c b/opal/mca/reachable/weighted/reachable_weighted.c
index ff5c60548ac..69c0c1300a3 100644
--- a/opal/mca/reachable/weighted/reachable_weighted.c
+++ b/opal/mca/reachable/weighted/reachable_weighted.c
@@ -20,27 +20,25 @@
 
 #include <string.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 #ifdef HAVE_MATH_H
-#include <math.h>
+#    include <math.h>
 #endif
 
 #include "opal/mca/if/if.h"
 
 #include "opal/mca/reachable/base/base.h"
-#include "reachable_weighted.h"
 #include "opal/util/net.h"
 #include "opal/util/string_copy.h"
+#include "reachable_weighted.h"
 
 static int weighted_init(void);
 static int weighted_fini(void);
-static opal_reachable_t* weighted_reachable(opal_list_t *local_ifs,
-                                            opal_list_t *remote_ifs);
+static opal_reachable_t *weighted_reachable(opal_list_t *local_ifs, opal_list_t *remote_ifs);
 
 static int get_weights(opal_if_t *local_if, opal_if_t *remote_if);
-static int calculate_weight(int bandwidth_local, int bandwidth_remote,
-                            int connection_quality);
+static int calculate_weight(int bandwidth_local, int bandwidth_remote, int connection_quality);
 
 /*
  * Describes the quality of a possible connection between a local and
@@ -58,16 +56,12 @@ enum connection_quality {
     CQ_PUBLIC_SAME_NETWORK = 100
 };
 
-const opal_reachable_base_module_t opal_reachable_weighted_module = {
-    weighted_init,
-    weighted_fini,
-    weighted_reachable
-};
+const opal_reachable_base_module_t opal_reachable_weighted_module = {weighted_init, weighted_fini,
+                                                                     weighted_reachable};
 
 // local variables
 static int init_cntr = 0;
 
-
 static int weighted_init(void)
 {
     ++init_cntr;
@@ -82,9 +76,7 @@ static int weighted_fini(void)
     return OPAL_SUCCESS;
 }
 
-
-static opal_reachable_t* weighted_reachable(opal_list_t *local_ifs,
-                                            opal_list_t *remote_ifs)
+static opal_reachable_t *weighted_reachable(opal_list_t *local_ifs, opal_list_t *remote_ifs)
 {
     opal_reachable_t *reachable_results = NULL;
     int i, j;
@@ -97,9 +89,9 @@ static opal_reachable_t* weighted_reachable(opal_list_t *local_ifs,
     }
 
     i = 0;
-    OPAL_LIST_FOREACH(local_iter, local_ifs, opal_if_t) {
+    OPAL_LIST_FOREACH (local_iter, local_ifs, opal_if_t) {
         j = 0;
-        OPAL_LIST_FOREACH(remote_iter, remote_ifs, opal_if_t) {
+        OPAL_LIST_FOREACH (remote_iter, remote_ifs, opal_if_t) {
             reachable_results->weights[i][j] = get_weights(local_iter, remote_iter);
             j++;
         }
@@ -109,15 +101,14 @@ static opal_reachable_t* weighted_reachable(opal_list_t *local_ifs,
     return reachable_results;
 }
 
-
 static int get_weights(opal_if_t *local_if, opal_if_t *remote_if)
 {
     char str_local[128], str_remote[128], *conn_type;
     struct sockaddr *local_sockaddr, *remote_sockaddr;
     int weight;
 
-    local_sockaddr = (struct sockaddr *)&local_if->if_addr;
-    remote_sockaddr = (struct sockaddr *)&remote_if->if_addr;
+    local_sockaddr = (struct sockaddr *) &local_if->if_addr;
+    remote_sockaddr = (struct sockaddr *) &remote_if->if_addr;
 
     /* opal_net_get_hostname returns a static buffer.  Great for
        single address printfs, need to copy in this case */
@@ -129,52 +120,41 @@ static int get_weights(opal_if_t *local_if, opal_if_t *remote_if)
     /*  initially, assume no connection is possible */
     weight = calculate_weight(0, 0, CQ_NO_CONNECTION);
 
-    if (AF_INET == local_sockaddr->sa_family &&
-        AF_INET == remote_sockaddr->sa_family) {
+    if (AF_INET == local_sockaddr->sa_family && AF_INET == remote_sockaddr->sa_family) {
 
-        if (opal_net_addr_isipv4public(local_sockaddr) &&
-            opal_net_addr_isipv4public(remote_sockaddr)) {
-            if (opal_net_samenetwork(local_sockaddr,
-                                     remote_sockaddr,
-                                     local_if->if_mask)) {
+        if (opal_net_addr_isipv4public(local_sockaddr)
+            && opal_net_addr_isipv4public(remote_sockaddr)) {
+            if (opal_net_samenetwork(local_sockaddr, remote_sockaddr, local_if->if_mask)) {
                 conn_type = "IPv4 PUBLIC SAME NETWORK";
-                weight = calculate_weight(local_if->if_bandwidth,
-                                          remote_if->if_bandwidth,
+                weight = calculate_weight(local_if->if_bandwidth, remote_if->if_bandwidth,
                                           CQ_PUBLIC_SAME_NETWORK);
             } else {
                 conn_type = "IPv4 PUBLIC DIFFERENT NETWORK";
-                weight = calculate_weight(local_if->if_bandwidth,
-                                          remote_if->if_bandwidth,
+                weight = calculate_weight(local_if->if_bandwidth, remote_if->if_bandwidth,
                                           CQ_PUBLIC_DIFFERENT_NETWORK);
             }
-        } else if (!opal_net_addr_isipv4public(local_sockaddr) &&
-                   !opal_net_addr_isipv4public(remote_sockaddr)) {
-            if (opal_net_samenetwork(local_sockaddr,
-                                     remote_sockaddr,
-                                     local_if->if_mask)) {
+        } else if (!opal_net_addr_isipv4public(local_sockaddr)
+                   && !opal_net_addr_isipv4public(remote_sockaddr)) {
+            if (opal_net_samenetwork(local_sockaddr, remote_sockaddr, local_if->if_mask)) {
                 conn_type = "IPv4 PRIVATE SAME NETWORK";
-                weight = calculate_weight(local_if->if_bandwidth,
-                                          remote_if->if_bandwidth,
+                weight = calculate_weight(local_if->if_bandwidth, remote_if->if_bandwidth,
                                           CQ_PRIVATE_SAME_NETWORK);
             } else {
                 conn_type = "IPv4 PRIVATE DIFFERENT NETWORK";
-                weight = calculate_weight(local_if->if_bandwidth,
-                                          remote_if->if_bandwidth,
+                weight = calculate_weight(local_if->if_bandwidth, remote_if->if_bandwidth,
                                           CQ_PRIVATE_DIFFERENT_NETWORK);
             }
         } else {
             /* one private, one public address.  likely not a match. */
             conn_type = "IPv4 NO CONNECTION";
-            weight = calculate_weight(local_if->if_bandwidth,
-                                      remote_if->if_bandwidth,
+            weight = calculate_weight(local_if->if_bandwidth, remote_if->if_bandwidth,
                                       CQ_NO_CONNECTION);
         }
 
 #if OPAL_ENABLE_IPV6
-    } else if (AF_INET6 == local_sockaddr->sa_family &&
-               AF_INET6 == remote_sockaddr->sa_family) {
-        if (opal_net_addr_isipv6linklocal(local_sockaddr) &&
-            opal_net_addr_isipv6linklocal(remote_sockaddr)) {
+    } else if (AF_INET6 == local_sockaddr->sa_family && AF_INET6 == remote_sockaddr->sa_family) {
+        if (opal_net_addr_isipv6linklocal(local_sockaddr)
+            && opal_net_addr_isipv6linklocal(remote_sockaddr)) {
             /* we can't actually tell if link local addresses are on
              * the same network or not with the weighted component.
              * Assume they are on the same network, so that they'll be
@@ -189,29 +169,23 @@ static int get_weights(opal_if_t *local_if, opal_if_t *remote_if)
              * either case, do so.
              */
             conn_type = "IPv6 LINK-LOCAL SAME NETWORK";
-            weight = calculate_weight(local_if->if_bandwidth,
-                                      remote_if->if_bandwidth,
+            weight = calculate_weight(local_if->if_bandwidth, remote_if->if_bandwidth,
                                       CQ_PRIVATE_SAME_NETWORK);
-        } else if (!opal_net_addr_isipv6linklocal(local_sockaddr) &&
-                   !opal_net_addr_isipv6linklocal(remote_sockaddr)) {
-            if (opal_net_samenetwork(local_sockaddr,
-                                     remote_sockaddr,
-                                     local_if->if_mask)) {
+        } else if (!opal_net_addr_isipv6linklocal(local_sockaddr)
+                   && !opal_net_addr_isipv6linklocal(remote_sockaddr)) {
+            if (opal_net_samenetwork(local_sockaddr, remote_sockaddr, local_if->if_mask)) {
                 conn_type = "IPv6 PUBLIC SAME NETWORK";
-                weight = calculate_weight(local_if->if_bandwidth,
-                                          remote_if->if_bandwidth,
+                weight = calculate_weight(local_if->if_bandwidth, remote_if->if_bandwidth,
                                           CQ_PUBLIC_SAME_NETWORK);
             } else {
                 conn_type = "IPv6 PUBLIC DIFFERENT NETWORK";
-                weight = calculate_weight(local_if->if_bandwidth,
-                                          remote_if->if_bandwidth,
+                weight = calculate_weight(local_if->if_bandwidth, remote_if->if_bandwidth,
                                           CQ_PUBLIC_DIFFERENT_NETWORK);
             }
         } else {
             /* one link-local, one public address.  likely not a match. */
             conn_type = "IPv6 NO CONNECTION";
-            weight = calculate_weight(local_if->if_bandwidth,
-                                      remote_if->if_bandwidth,
+            weight = calculate_weight(local_if->if_bandwidth, remote_if->if_bandwidth,
                                       CQ_NO_CONNECTION);
         }
 #endif /* #if OPAL_ENABLE_IPV6 */
@@ -224,13 +198,12 @@ static int get_weights(opal_if_t *local_if, opal_if_t *remote_if)
     }
 
     opal_output_verbose(20, opal_reachable_base_framework.framework_output,
-                        "reachable:weighted: path from %s to %s: %s",
-                        str_local, str_remote, conn_type);
+                        "reachable:weighted: path from %s to %s: %s", str_local, str_remote,
+                        conn_type);
 
     return weight;
 }
 
-
 /*
  * Weights determined by bandwidth between
  * interfaces (limited by lower bandwidth
@@ -257,10 +230,10 @@ static int get_weights(opal_if_t *local_if, opal_if_t *remote_if)
  * connection_quality to be large enough
  * to capture decimals
  */
-static int calculate_weight(int bandwidth_local, int bandwidth_remote,
-                            int connection_quality)
+static int calculate_weight(int bandwidth_local, int bandwidth_remote, int connection_quality)
 {
-    int weight = connection_quality * (MIN(bandwidth_local, bandwidth_remote) +
-                                       1.0 / (1.0 + (double)abs(bandwidth_local - bandwidth_remote)));
+    int weight = connection_quality
+                 * (MIN(bandwidth_local, bandwidth_remote)
+                    + 1.0 / (1.0 + (double) abs(bandwidth_local - bandwidth_remote)));
     return weight;
 }
diff --git a/opal/mca/reachable/weighted/reachable_weighted.h b/opal/mca/reachable/weighted/reachable_weighted.h
index effdcaf3087..85bc50232c3 100644
--- a/opal/mca/reachable/weighted/reachable_weighted.h
+++ b/opal/mca/reachable/weighted/reachable_weighted.h
@@ -15,14 +15,14 @@
 #include "opal_config.h"
 
 #ifdef HAVE_SYS_SOCKET_H
-#include <sys/socket.h>
+#    include <sys/socket.h>
 #endif
 #ifdef HAVE_SYS_UN_H
-#include <sys/un.h>
+#    include <sys/un.h>
 #endif
 
-#include "opal/mca/reachable/reachable.h"
 #include "opal/mca/mca.h"
+#include "opal/mca/reachable/reachable.h"
 #include "opal/util/event.h"
 #include "opal/util/proc.h"
 
@@ -38,7 +38,6 @@ OPAL_DECLSPEC extern opal_reachable_weighted_component_t mca_reachable_weighted_
 
 OPAL_DECLSPEC extern const opal_reachable_base_module_t opal_reachable_weighted_module;
 
-
 END_C_DECLS
 
 #endif /* MCA_REACHABLE_WEIGHTED_H */
diff --git a/opal/mca/reachable/weighted/reachable_weighted_component.c b/opal/mca/reachable/weighted/reachable_weighted_component.c
index 6e8098b7698..13222d0fb7b 100644
--- a/opal/mca/reachable/weighted/reachable_weighted_component.c
+++ b/opal/mca/reachable/weighted/reachable_weighted_component.c
@@ -23,15 +23,15 @@
 #include "opal_config.h"
 
 #include "opal/constants.h"
-#include "opal/util/proc.h"
 #include "opal/mca/reachable/reachable.h"
+#include "opal/util/proc.h"
 #include "reachable_weighted.h"
 
 /*
  * Public string showing the reachable weighted component version number
  */
-const char *opal_reachable_weighted_component_version_string =
-    "OPAL weighted reachable MCA component version " OPAL_VERSION;
+const char *opal_reachable_weighted_component_version_string
+    = "OPAL weighted reachable MCA component version " OPAL_VERSION;
 
 /*
  * Local function
@@ -41,19 +41,18 @@ static int reachable_weighted_close(void);
 static int reachable_weighted_component_query(mca_base_module_t **module, int *priority);
 static int component_register(void);
 
-
 /*
  * Instantiate the public struct with all of our public information
  * and pointers to our public functions in it
  */
 
-opal_reachable_weighted_component_t mca_reachable_weighted_component = {
-    {
+opal_reachable_weighted_component_t mca_reachable_weighted_component = {{
 
-        /* First, the mca_component_t struct containing meta information
-           about the component itself */
+    /* First, the mca_component_t struct containing meta information
+       about the component itself */
 
-        .base_version = {
+    .base_version =
+        {
             /* Indicate that we are a reachable v1.1.0 component (which also
                implies a specific MCA version) */
 
@@ -72,13 +71,11 @@ opal_reachable_weighted_component_t mca_reachable_weighted_component = {
             .mca_query_component = reachable_weighted_component_query,
             .mca_register_component_params = component_register,
         },
-        /* Next the MCA v1.0.0 component meta data */
-        .base_data = {
-            /* The component is checkpoint ready */
-            MCA_BASE_METADATA_PARAM_CHECKPOINT
-        },
-    }
-};
+    /* Next the MCA v1.0.0 component meta data */
+    .base_data =
+        {/* The component is checkpoint ready */
+         MCA_BASE_METADATA_PARAM_CHECKPOINT},
+}};
 
 static int reachable_weighted_open(void)
 {
@@ -100,6 +97,6 @@ static int component_register(void)
 static int reachable_weighted_component_query(mca_base_module_t **module, int *priority)
 {
     *priority = 1;
-    *module = (mca_base_module_t *)&opal_reachable_weighted_module;
+    *module = (mca_base_module_t *) &opal_reachable_weighted_module;
     return OPAL_SUCCESS;
 }
diff --git a/opal/mca/shmem/base/base.h b/opal/mca/shmem/base/base.h
index 57ee7785603..ecc38cd5962 100644
--- a/opal/mca/shmem/base/base.h
+++ b/opal/mca/shmem/base/base.h
@@ -31,23 +31,16 @@ BEGIN_C_DECLS
 /* ////////////////////////////////////////////////////////////////////////// */
 /* Public API for the shmem framework */
 /* ////////////////////////////////////////////////////////////////////////// */
-OPAL_DECLSPEC int
-opal_shmem_segment_create(opal_shmem_ds_t *ds_buf,
-                          const char *file_name,
-                          size_t size);
+OPAL_DECLSPEC int opal_shmem_segment_create(opal_shmem_ds_t *ds_buf, const char *file_name,
+                                            size_t size);
 
-OPAL_DECLSPEC int
-opal_shmem_ds_copy(const opal_shmem_ds_t *from,
-                   opal_shmem_ds_t *to);
+OPAL_DECLSPEC int opal_shmem_ds_copy(const opal_shmem_ds_t *from, opal_shmem_ds_t *to);
 
-OPAL_DECLSPEC void *
-opal_shmem_segment_attach(opal_shmem_ds_t *ds_buf);
+OPAL_DECLSPEC void *opal_shmem_segment_attach(opal_shmem_ds_t *ds_buf);
 
-OPAL_DECLSPEC int
-opal_shmem_segment_detach(opal_shmem_ds_t *ds_buf);
+OPAL_DECLSPEC int opal_shmem_segment_detach(opal_shmem_ds_t *ds_buf);
 
-OPAL_DECLSPEC int
-opal_shmem_unlink(opal_shmem_ds_t *ds_buf);
+OPAL_DECLSPEC int opal_shmem_unlink(opal_shmem_ds_t *ds_buf);
 /* ////////////////////////////////////////////////////////////////////////// */
 /* End Public API for the shmem framework */
 /* ////////////////////////////////////////////////////////////////////////// */
@@ -62,10 +55,9 @@ opal_shmem_unlink(opal_shmem_ds_t *ds_buf);
  *
  * @retval name of best component.  NULL if no component is found.
  *
- * see: orte/mca/odls/base/odls_base_default_fns.c
+ * see: 3rd-party/prrte/src/mca/odls/base/odls_base_default_fns.c
  */
-OPAL_DECLSPEC char *
-opal_shmem_base_best_runnable_component_name(void);
+OPAL_DECLSPEC char *opal_shmem_base_best_runnable_component_name(void);
 
 /**
  * Select an available component.
@@ -98,8 +90,7 @@ opal_shmem_base_best_runnable_component_name(void);
  * selected.  If no component was selected, subsequent invocation
  * of the shmem wrapper functions will return an error.
  */
-OPAL_DECLSPEC int
-opal_shmem_base_select(void);
+OPAL_DECLSPEC int opal_shmem_base_select(void);
 
 /**
  * Shut down the shmem MCA framework.
@@ -112,8 +103,7 @@ opal_shmem_base_select(void);
  * It must be the last function invoked on the shmem MCA
  * framework.
  */
-OPAL_DECLSPEC int
-opal_shmem_base_close(void);
+OPAL_DECLSPEC int opal_shmem_base_close(void);
 
 /**
  * Indication of whether a component was successfully selected or
diff --git a/opal/mca/shmem/base/shmem_base_close.c b/opal/mca/shmem/base/shmem_base_close.c
index 4df2ff2e25a..415ea6c22e1 100644
--- a/opal/mca/shmem/base/shmem_base_close.c
+++ b/opal/mca/shmem/base/shmem_base_close.c
@@ -22,19 +22,17 @@
 #include "opal_config.h"
 
 #include "opal/constants.h"
-#include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
-#include "opal/mca/shmem/shmem.h"
+#include "opal/mca/mca.h"
 #include "opal/mca/shmem/base/base.h"
+#include "opal/mca/shmem/shmem.h"
 #include "opal/util/output.h"
 
 /* ////////////////////////////////////////////////////////////////////////// */
-int
-opal_shmem_base_close(void)
+int opal_shmem_base_close(void)
 {
     /* if there is a selected shmem module, finalize it */
-    if (NULL != opal_shmem_base_module &&
-        NULL != opal_shmem_base_module->module_finalize) {
+    if (NULL != opal_shmem_base_module && NULL != opal_shmem_base_module->module_finalize) {
         opal_shmem_base_module->module_finalize();
     }
 
@@ -42,7 +40,5 @@ opal_shmem_base_close(void)
     opal_shmem_base_component = NULL;
     opal_shmem_base_module = NULL;
 
-    return mca_base_framework_components_close (&opal_shmem_base_framework,
-                                                NULL);
+    return mca_base_framework_components_close(&opal_shmem_base_framework, NULL);
 }
-
diff --git a/opal/mca/shmem/base/shmem_base_open.c b/opal/mca/shmem/base/shmem_base_open.c
index 0332a5aac58..a9055432d84 100644
--- a/opal/mca/shmem/base/shmem_base_open.c
+++ b/opal/mca/shmem/base/shmem_base_open.c
@@ -22,12 +22,12 @@
 #include "opal_config.h"
 
 #include "opal/constants.h"
-#include "opal/util/output.h"
-#include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
 #include "opal/mca/base/mca_base_var.h"
-#include "opal/mca/shmem/shmem.h"
+#include "opal/mca/mca.h"
 #include "opal/mca/shmem/base/base.h"
+#include "opal/mca/shmem/shmem.h"
+#include "opal/util/output.h"
 
 /*
  * The following file was created by configure.  It contains extern
@@ -45,8 +45,7 @@ char *opal_shmem_base_RUNTIME_QUERY_hint = NULL;
 /**
  * Register some shmem-wide MCA params
  */
-static int
-opal_shmem_base_register (mca_base_register_flag_t flags)
+static int opal_shmem_base_register(mca_base_register_flag_t flags)
 {
     int ret;
 
@@ -54,17 +53,17 @@ opal_shmem_base_register (mca_base_register_flag_t flags)
      * hint to the shmem framework.
      */
     opal_shmem_base_RUNTIME_QUERY_hint = NULL;
-    ret = mca_base_framework_var_register (&opal_shmem_base_framework, "RUNTIME_QUERY_hint",
-                                           "Internal OMPI parameter used to provide a "
-                                           "component selection hint to the shmem "
-                                           "framework.  The value of this parameter "
-                                           "is the name of the component that is "
-                                           "available, selectable, and meets our "
-                                           "run-time behavior requirements.",
-                                           MCA_BASE_VAR_TYPE_STRING, NULL, 0,
-                                           MCA_BASE_VAR_FLAG_INTERNAL,
-                                           OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_ALL,
-                                           &opal_shmem_base_RUNTIME_QUERY_hint);
+    ret = mca_base_framework_var_register(&opal_shmem_base_framework, "RUNTIME_QUERY_hint",
+                                          "Internal OMPI parameter used to provide a "
+                                          "component selection hint to the shmem "
+                                          "framework.  The value of this parameter "
+                                          "is the name of the component that is "
+                                          "available, selectable, and meets our "
+                                          "run-time behavior requirements.",
+                                          MCA_BASE_VAR_TYPE_STRING, NULL, 0,
+                                          MCA_BASE_VAR_FLAG_INTERNAL, OPAL_INFO_LVL_9,
+                                          MCA_BASE_VAR_SCOPE_ALL,
+                                          &opal_shmem_base_RUNTIME_QUERY_hint);
 
     return (0 > ret) ? ret : OPAL_SUCCESS;
 }
diff --git a/opal/mca/shmem/base/shmem_base_select.c b/opal/mca/shmem/base/shmem_base_select.c
index 766f0321968..2cd0805b1c7 100644
--- a/opal/mca/shmem/base/shmem_base_select.c
+++ b/opal/mca/shmem/base/shmem_base_select.c
@@ -24,11 +24,11 @@
 #include <string.h>
 
 #include "opal/constants.h"
-#include "opal/util/output.h"
-#include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
-#include "opal/mca/shmem/shmem.h"
+#include "opal/mca/mca.h"
 #include "opal/mca/shmem/base/base.h"
+#include "opal/mca/shmem/shmem.h"
+#include "opal/util/output.h"
 
 /*
  * globals
@@ -42,11 +42,10 @@ opal_shmem_base_module_t *opal_shmem_base_module = NULL;
 /*
  * Performs a run-time query across all available shmem components.
  * Similar to mca_base_select, but take into consideration environment
- * hints provided by orte.
+ * hints provided by prte.
  */
-static int
-opal_shmem_base_runtime_query(mca_base_module_t **best_module,
-                              mca_base_component_t **best_component)
+static int opal_shmem_base_runtime_query(mca_base_module_t **best_module,
+                                         mca_base_component_t **best_component)
 {
     mca_base_component_list_item_t *cli = NULL;
     mca_base_component_t *component = NULL;
@@ -73,14 +72,14 @@ opal_shmem_base_runtime_query(mca_base_module_t **best_module,
      * for each call their 'run-time query' functions to determine relative
      * priority.
      */
-    OPAL_LIST_FOREACH(cli, &opal_shmem_base_framework.framework_components, mca_base_component_list_item_t) {
-        component = (mca_base_component_t *)cli->cli_component;
+    OPAL_LIST_FOREACH (cli, &opal_shmem_base_framework.framework_components,
+                       mca_base_component_list_item_t) {
+        component = (mca_base_component_t *) cli->cli_component;
 
         /* if there is a run-time query function then use it. otherwise, skip
          * the component.
          */
-        if (NULL == ((opal_shmem_base_component_2_0_0_t *)
-                     component)->runtime_query) {
+        if (NULL == ((opal_shmem_base_component_2_0_0_t *) component)->runtime_query) {
             opal_output_verbose(5, opal_shmem_base_framework.framework_output,
                                 "shmem: base: runtime_query: "
                                 "(shmem) Skipping component [%s]. It does not "
@@ -95,8 +94,8 @@ opal_shmem_base_runtime_query(mca_base_module_t **best_module,
                             "(shmem) Querying component (run-time) [%s]",
                             component->mca_component_name);
 
-        ((opal_shmem_base_component_2_0_0_t *)
-         component)->runtime_query(&module, &priority, opal_shmem_base_RUNTIME_QUERY_hint);
+        ((opal_shmem_base_component_2_0_0_t *) component)
+            ->runtime_query(&module, &priority, opal_shmem_base_RUNTIME_QUERY_hint);
 
         /* if no module was returned, then skip component.
          * this probably means that the run-time test deemed the shared memory
@@ -131,30 +130,30 @@ opal_shmem_base_runtime_query(mca_base_module_t **best_module,
     if (NULL == *best_component) {
         opal_output_verbose(5, opal_shmem_base_framework.framework_output,
                             "shmem: base: runtime_query: "
-                            "(%5s) No component selected!", "shmem");
+                            "(%5s) No component selected!",
+                            "shmem");
         return OPAL_ERR_NOT_FOUND;
     }
 
     opal_output_verbose(5, opal_shmem_base_framework.framework_output,
                         "shmem: base: runtime_query: "
-                        "(%5s) Selected component [%s]", "shmem",
-                        (*best_component)->mca_component_name);
+                        "(%5s) Selected component [%s]",
+                        "shmem", (*best_component)->mca_component_name);
 
     /* close the non-selected components */
-    (void) mca_base_framework_components_close (&opal_shmem_base_framework,
-                                                (mca_base_component_t *)(*best_component));
+    (void) mca_base_framework_components_close(&opal_shmem_base_framework,
+                                               (mca_base_component_t *) (*best_component));
 
     /* save the winner */
-    opal_shmem_base_component = (opal_shmem_base_component_t*) *best_component;
-    opal_shmem_base_module = (opal_shmem_base_module_t*) *best_module;
+    opal_shmem_base_component = (opal_shmem_base_component_t *) *best_component;
+    opal_shmem_base_module = (opal_shmem_base_module_t *) *best_module;
     opal_shmem_base_selected = true;
 
     return OPAL_SUCCESS;
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-char *
-opal_shmem_base_best_runnable_component_name(void)
+char *opal_shmem_base_best_runnable_component_name(void)
 {
     mca_base_component_t *best_component = NULL;
     mca_base_module_t *best_module = NULL;
@@ -163,20 +162,17 @@ opal_shmem_base_best_runnable_component_name(void)
                         "shmem: base: best_runnable_component_name: "
                         "Searching for best runnable component.");
     /* select the best component so we can get its name. */
-    if (OPAL_SUCCESS != opal_shmem_base_runtime_query(&best_module,
-                                                      &best_component)) {
+    if (OPAL_SUCCESS != opal_shmem_base_runtime_query(&best_module, &best_component)) {
         /* fail! */
         return NULL;
-    }
-    else {
+    } else {
         if (NULL != best_component) {
             opal_output_verbose(10, opal_shmem_base_framework.framework_output,
                                 "shmem: base: best_runnable_component_name: "
                                 "Found best runnable component: (%s).",
                                 best_component->mca_component_name);
             return strdup(best_component->mca_component_name);
-        }
-        else {
+        } else {
             opal_output_verbose(10, opal_shmem_base_framework.framework_output,
                                 "shmem: base: best_runnable_component_name: "
                                 "Could not find runnable component.");
@@ -187,15 +183,14 @@ opal_shmem_base_best_runnable_component_name(void)
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-int
-opal_shmem_base_select(void)
+int opal_shmem_base_select(void)
 {
     opal_shmem_base_component_2_0_0_t *best_component = NULL;
     opal_shmem_base_module_2_0_0_t *best_module = NULL;
     /* select the best component */
-    if (OPAL_SUCCESS != opal_shmem_base_runtime_query(
-                                (mca_base_module_t **)&best_module,
-                                (mca_base_component_t **)&best_component)) {
+    if (OPAL_SUCCESS
+        != opal_shmem_base_runtime_query((mca_base_module_t **) &best_module,
+                                         (mca_base_component_t **) &best_component)) {
         /* it is NOT okay if we don't find a module because we need at
          * least one shared memory backing facility component instance.
          */
@@ -205,9 +200,7 @@ opal_shmem_base_select(void)
     /* initialize the winner */
     if (NULL != opal_shmem_base_module) {
         return opal_shmem_base_module->module_init();
-    }
-    else {
+    } else {
         return OPAL_ERROR;
     }
 }
-
diff --git a/opal/mca/shmem/base/shmem_base_wrappers.c b/opal/mca/shmem/base/shmem_base_wrappers.c
index c323c3b9acd..b1b0c02f6ec 100644
--- a/opal/mca/shmem/base/shmem_base_wrappers.c
+++ b/opal/mca/shmem/base/shmem_base_wrappers.c
@@ -22,14 +22,11 @@
 #include "opal_config.h"
 
 #include "opal/constants.h"
-#include "opal/mca/shmem/shmem.h"
 #include "opal/mca/shmem/base/base.h"
+#include "opal/mca/shmem/shmem.h"
 
 /* ////////////////////////////////////////////////////////////////////////// */
-int
-opal_shmem_segment_create(opal_shmem_ds_t *ds_buf,
-                          const char *file_name,
-                          size_t size)
+int opal_shmem_segment_create(opal_shmem_ds_t *ds_buf, const char *file_name, size_t size)
 {
     if (!opal_shmem_base_selected) {
         return OPAL_ERROR;
@@ -39,9 +36,7 @@ opal_shmem_segment_create(opal_shmem_ds_t *ds_buf,
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-int
-opal_shmem_ds_copy(const opal_shmem_ds_t *from,
-                   opal_shmem_ds_t *to)
+int opal_shmem_ds_copy(const opal_shmem_ds_t *from, opal_shmem_ds_t *to)
 {
     if (!opal_shmem_base_selected) {
         return OPAL_ERROR;
@@ -51,8 +46,7 @@ opal_shmem_ds_copy(const opal_shmem_ds_t *from,
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-void *
-opal_shmem_segment_attach(opal_shmem_ds_t *ds_buf)
+void *opal_shmem_segment_attach(opal_shmem_ds_t *ds_buf)
 {
     if (!opal_shmem_base_selected) {
         return NULL;
@@ -62,8 +56,7 @@ opal_shmem_segment_attach(opal_shmem_ds_t *ds_buf)
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-int
-opal_shmem_segment_detach(opal_shmem_ds_t *ds_buf)
+int opal_shmem_segment_detach(opal_shmem_ds_t *ds_buf)
 {
     if (!opal_shmem_base_selected) {
         return OPAL_ERROR;
@@ -73,8 +66,7 @@ opal_shmem_segment_detach(opal_shmem_ds_t *ds_buf)
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-int
-opal_shmem_unlink(opal_shmem_ds_t *ds_buf)
+int opal_shmem_unlink(opal_shmem_ds_t *ds_buf)
 {
     if (!opal_shmem_base_selected) {
         return OPAL_ERROR;
@@ -82,4 +74,3 @@ opal_shmem_unlink(opal_shmem_ds_t *ds_buf)
 
     return opal_shmem_base_module->unlink(ds_buf);
 }
-
diff --git a/opal/mca/shmem/mmap/Makefile.am b/opal/mca/shmem/mmap/Makefile.am
index 0c18919763e..e8c7a14b8fe 100644
--- a/opal/mca/shmem/mmap/Makefile.am
+++ b/opal/mca/shmem/mmap/Makefile.am
@@ -13,6 +13,7 @@
 # Copyright (c) 2010-2011 Los Alamos National Security, LLC.
 #                         All rights reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -38,13 +39,13 @@ component_install =
 endif
 
 # help file
-dist_opaldata_DATA = help-opal-shmem-mmap.txt
+EXTRA_DIST = help-opal-shmem-mmap.txt
 
 mcacomponentdir = $(opallibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_shmem_mmap_la_SOURCES = $(sources)
 mca_shmem_mmap_la_LDFLAGS = -module -avoid-version
-mca_shmem_mmap_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+mca_shmem_mmap_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 
 noinst_LTLIBRARIES = $(component_noinst)
 libmca_shmem_mmap_la_SOURCES =$(sources)
diff --git a/opal/mca/shmem/mmap/help-opal-shmem-mmap.txt b/opal/mca/shmem/mmap/help-opal-shmem-mmap.txt
index 67fa8f94e33..b5157c5c098 100644
--- a/opal/mca/shmem/mmap/help-opal-shmem-mmap.txt
+++ b/opal/mca/shmem/mmap/help-opal-shmem-mmap.txt
@@ -40,7 +40,7 @@ the MCA parameter "orte_no_session_dir".
   Local host: %s
   Filename:   %s
 
-You can set the MCA paramter shmem_mmap_enable_nfs_warning to 0 to
+You can set the MCA parameter shmem_mmap_enable_nfs_warning to 0 to
 disable this message.
 #
 [target full]
diff --git a/opal/mca/shmem/mmap/shmem_mmap.h b/opal/mca/shmem/mmap/shmem_mmap.h
index 5cea8a5bcf9..8efcafb8f25 100644
--- a/opal/mca/shmem/mmap/shmem_mmap.h
+++ b/opal/mca/shmem/mmap/shmem_mmap.h
@@ -43,8 +43,7 @@ typedef struct opal_shmem_mmap_component_t {
     int priority;
 } opal_shmem_mmap_component_t;
 
-OPAL_MODULE_DECLSPEC extern opal_shmem_mmap_component_t
-mca_shmem_mmap_component;
+OPAL_DECLSPEC extern opal_shmem_mmap_component_t mca_shmem_mmap_component;
 
 typedef struct opal_shmem_mmap_module_t {
     opal_shmem_base_module_t super;
diff --git a/opal/mca/shmem/mmap/shmem_mmap_component.c b/opal/mca/shmem/mmap/shmem_mmap_component.c
index 324f1493a94..ae006b2773c 100644
--- a/opal/mca/shmem/mmap/shmem_mmap_component.c
+++ b/opal/mca/shmem/mmap/shmem_mmap_component.c
@@ -37,8 +37,8 @@
 /**
  * public string showing the shmem ompi_mmap component version number
  */
-const char *opal_shmem_mmap_component_version_string =
-    "OPAL mmap shmem MCA component version " OPAL_VERSION;
+const char *opal_shmem_mmap_component_version_string
+    = "OPAL mmap shmem MCA component version " OPAL_VERSION;
 
 int opal_shmem_mmap_relocate_backing_file = 0;
 char *opal_shmem_mmap_backing_file_base_dir = NULL;
@@ -51,66 +51,60 @@ static int mmap_register(void);
 static int mmap_open(void);
 static int mmap_close(void);
 static int mmap_query(mca_base_module_t **module, int *priority);
-static int mmap_runtime_query(mca_base_module_t **module,
-                              int *priority,
-                              const char *hint);
+static int mmap_runtime_query(mca_base_module_t **module, int *priority, const char *hint);
 
 /**
  * instantiate the public struct with all of our public information
  * and pointers to our public functions in it
  */
 opal_shmem_mmap_component_t mca_shmem_mmap_component = {
-    .super = {
-        .base_version = {
-            OPAL_SHMEM_BASE_VERSION_2_0_0,
-
-            /* component name and version */
-            .mca_component_name = "mmap",
-            MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
-                                  OPAL_RELEASE_VERSION),
-
-            .mca_open_component = mmap_open,
-            .mca_close_component = mmap_close,
-            .mca_query_component = mmap_query,
-            .mca_register_component_params = mmap_register,
+    .super =
+        {
+            .base_version =
+                {
+                    OPAL_SHMEM_BASE_VERSION_2_0_0,
+
+                    /* component name and version */
+                    .mca_component_name = "mmap",
+                    MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
+                                          OPAL_RELEASE_VERSION),
+
+                    .mca_open_component = mmap_open,
+                    .mca_close_component = mmap_close,
+                    .mca_query_component = mmap_query,
+                    .mca_register_component_params = mmap_register,
+                },
+            /* MCA v2.0.0 component meta data */
+            .base_data =
+                {/* the component is checkpoint ready */
+                 MCA_BASE_METADATA_PARAM_CHECKPOINT},
+            .runtime_query = mmap_runtime_query,
         },
-        /* MCA v2.0.0 component meta data */
-        .base_data = {
-            /* the component is checkpoint ready */
-            MCA_BASE_METADATA_PARAM_CHECKPOINT
-        },
-        .runtime_query = mmap_runtime_query,
-    },
 };
 
 /* ////////////////////////////////////////////////////////////////////////// */
-static int
-mmap_runtime_query(mca_base_module_t **module,
-                   int *priority,
-                   const char *hint)
+static int mmap_runtime_query(mca_base_module_t **module, int *priority, const char *hint)
 {
     /* no run-time query needed for mmap, so this is easy */
     *priority = mca_shmem_mmap_component.priority;
-    *module = (mca_base_module_t *)&opal_shmem_mmap_module.super;
+    *module = (mca_base_module_t *) &opal_shmem_mmap_module.super;
     return OPAL_SUCCESS;
 }
 
-static int
-mmap_register(void)
+static int mmap_register(void)
 {
     int ret;
 
-
     /* ////////////////////////////////////////////////////////////////////// */
     /* (default) priority - set high to make mmap the default */
     mca_shmem_mmap_component.priority = 50;
-    ret = mca_base_component_var_register (&mca_shmem_mmap_component.super.base_version,
-                                           "priority", "Priority for shmem mmap "
-                                           "component (default: 50)", MCA_BASE_VAR_TYPE_INT,
-                                           NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-                                           OPAL_INFO_LVL_3,
-                                           MCA_BASE_VAR_SCOPE_ALL_EQ,
-                                           &mca_shmem_mmap_component.priority);
+    ret = mca_base_component_var_register(&mca_shmem_mmap_component.super.base_version, "priority",
+                                          "Priority for shmem mmap "
+                                          "component (default: 50)",
+                                          MCA_BASE_VAR_TYPE_INT, NULL, 0,
+                                          MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
+                                          MCA_BASE_VAR_SCOPE_ALL_EQ,
+                                          &mca_shmem_mmap_component.priority);
     if (0 > ret) {
         return ret;
     }
@@ -126,42 +120,43 @@ mmap_register(void)
      * effect doesn't matter.
      */
     opal_shmem_mmap_nfs_warning = true;
-    ret = mca_base_component_var_register (&mca_shmem_mmap_component.super.base_version,
-                                           "enable_nfs_warning",
-                                           "Enable the warning emitted when Open MPI detects "
-                                           "that its shared memory backing file is located on "
-                                           "a network filesystem (1 = enabled, 0 = disabled).",
-                                           MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-                                           OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL,
-                                           &opal_shmem_mmap_nfs_warning);
+    ret = mca_base_component_var_register(&mca_shmem_mmap_component.super.base_version,
+                                          "enable_nfs_warning",
+                                          "Enable the warning emitted when Open MPI detects "
+                                          "that its shared memory backing file is located on "
+                                          "a network filesystem (1 = enabled, 0 = disabled).",
+                                          MCA_BASE_VAR_TYPE_BOOL, NULL, 0,
+                                          MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9,
+                                          MCA_BASE_VAR_SCOPE_LOCAL, &opal_shmem_mmap_nfs_warning);
     if (0 > ret) {
         return ret;
     }
 
     opal_shmem_mmap_relocate_backing_file = 0;
-    ret = mca_base_component_var_register (&mca_shmem_mmap_component.super.base_version,
-                                           "relocate_backing_file",
-                                           "Whether to change the default placement of backing files or not "
-                                           "(Negative = try to relocate backing files to an area rooted at "
-                                           "the path specified by "
-                                           "shmem_mmap_backing_file_base_dir, but continue "
-                                           "with the default path if the relocation fails, 0 = do not relocate, "
-                                           "Positive = same as the negative option, but will fail if the "
-                                           "relocation fails.", MCA_BASE_VAR_TYPE_INT, NULL, 0,
-                                           MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_ALL_EQ, &opal_shmem_mmap_relocate_backing_file);
+    ret = mca_base_component_var_register(
+        &mca_shmem_mmap_component.super.base_version, "relocate_backing_file",
+        "Whether to change the default placement of backing files or not "
+        "(Negative = try to relocate backing files to an area rooted at "
+        "the path specified by "
+        "shmem_mmap_backing_file_base_dir, but continue "
+        "with the default path if the relocation fails, 0 = do not relocate, "
+        "Positive = same as the negative option, but will fail if the "
+        "relocation fails.",
+        MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9,
+        MCA_BASE_VAR_SCOPE_ALL_EQ, &opal_shmem_mmap_relocate_backing_file);
     if (0 > ret) {
         return ret;
     }
 
     opal_shmem_mmap_backing_file_base_dir = "/dev/shm";
-    ret = mca_base_component_var_register (&mca_shmem_mmap_component.super.base_version,
-                                           "backing_file_base_dir",
-                                           "Specifies where backing files will be created when "
-                                           "shmem_mmap_relocate_backing_file is in use.", MCA_BASE_VAR_TYPE_STRING,
-                                           NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_ALL_EQ,
-                                           &opal_shmem_mmap_backing_file_base_dir);
+    ret = mca_base_component_var_register(&mca_shmem_mmap_component.super.base_version,
+                                          "backing_file_base_dir",
+                                          "Specifies where backing files will be created when "
+                                          "shmem_mmap_relocate_backing_file is in use.",
+                                          MCA_BASE_VAR_TYPE_STRING, NULL, 0,
+                                          MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9,
+                                          MCA_BASE_VAR_SCOPE_ALL_EQ,
+                                          &opal_shmem_mmap_backing_file_base_dir);
     if (0 > ret) {
         return ret;
     }
@@ -170,25 +165,21 @@ mmap_register(void)
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-static int
-mmap_open(void)
+static int mmap_open(void)
 {
     return OPAL_SUCCESS;
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-static int
-mmap_query(mca_base_module_t **module, int *priority)
+static int mmap_query(mca_base_module_t **module, int *priority)
 {
     *priority = mca_shmem_mmap_component.priority;
-    *module = (mca_base_module_t *)&opal_shmem_mmap_module.super;
+    *module = (mca_base_module_t *) &opal_shmem_mmap_module.super;
     return OPAL_SUCCESS;
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-static int
-mmap_close(void)
+static int mmap_close(void)
 {
     return OPAL_SUCCESS;
 }
-
diff --git a/opal/mca/shmem/mmap/shmem_mmap_module.c b/opal/mca/shmem/mmap/shmem_mmap_module.c
index 0cfea545d8e..e36efb738e3 100644
--- a/opal/mca/shmem/mmap/shmem_mmap_module.c
+++ b/opal/mca/shmem/mmap/shmem_mmap_module.c
@@ -29,36 +29,36 @@
 
 #include <errno.h>
 #ifdef HAVE_FCNTL_H
-#include <fcntl.h>
-#endif  /* HAVE_FCNTL_H */
+#    include <fcntl.h>
+#endif /* HAVE_FCNTL_H */
 #ifdef HAVE_SYS_MMAN_H
-#include <sys/mman.h>
+#    include <sys/mman.h>
 #endif /* HAVE_SYS_MMAN_H */
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif /* HAVE_UNISTD_H */
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif /* HAVE_SYS_TYPES_H */
 #include <string.h>
 #ifdef HAVE_NETDB_H
-#include <netdb.h>
+#    include <netdb.h>
 #endif /* HAVE_NETDB_H */
 #include <time.h>
 #ifdef HAVE_SYS_STAT_H
-#include <sys/stat.h>
+#    include <sys/stat.h>
 #endif /* HAVE_SYS_STAT_H */
 
-#include "opal/runtime/opal.h"
-#include "opal_stdint.h"
 #include "opal/constants.h"
+#include "opal/mca/shmem/base/base.h"
+#include "opal/mca/shmem/shmem.h"
+#include "opal/runtime/opal.h"
 #include "opal/util/alfg.h"
 #include "opal/util/output.h"
 #include "opal/util/path.h"
 #include "opal/util/show_help.h"
 #include "opal/util/string_copy.h"
-#include "opal/mca/shmem/shmem.h"
-#include "opal/mca/shmem/base/base.h"
+#include "opal_stdint.h"
 
 #include "shmem_mmap.h"
 
@@ -67,50 +67,33 @@
 /* ////////////////////////////////////////////////////////////////////////// */
 /*local functions */
 /* local functions */
-static int
-module_init(void);
+static int module_init(void);
 
-static int
-segment_create(opal_shmem_ds_t *ds_buf,
-               const char *file_name,
-               size_t size);
+static int segment_create(opal_shmem_ds_t *ds_buf, const char *file_name, size_t size);
 
-static int
-ds_copy(const opal_shmem_ds_t *from,
-        opal_shmem_ds_t *to);
+static int ds_copy(const opal_shmem_ds_t *from, opal_shmem_ds_t *to);
 
-static void *
-segment_attach(opal_shmem_ds_t *ds_buf);
+static void *segment_attach(opal_shmem_ds_t *ds_buf);
 
-static int
-segment_detach(opal_shmem_ds_t *ds_buf);
+static int segment_detach(opal_shmem_ds_t *ds_buf);
 
-static int
-segment_unlink(opal_shmem_ds_t *ds_buf);
+static int segment_unlink(opal_shmem_ds_t *ds_buf);
 
-static int
-module_finalize(void);
+static int module_finalize(void);
 
-static int
-enough_space(const char *filename,
-             size_t space_req,
-             uint64_t *space_avail,
-             bool *result);
+static int enough_space(const char *filename, size_t space_req, uint64_t *space_avail,
+                        bool *result);
 
 /*
  * mmap shmem module
  */
-opal_shmem_mmap_module_t opal_shmem_mmap_module = {
-    .super = {
-        .module_init = module_init,
-        .segment_create = segment_create,
-        .ds_copy = ds_copy,
-        .segment_attach = segment_attach,
-        .segment_detach = segment_detach,
-        .unlink = segment_unlink,
-        .module_finalize = module_finalize
-    }
-};
+opal_shmem_mmap_module_t opal_shmem_mmap_module = {.super = {.module_init = module_init,
+                                                             .segment_create = segment_create,
+                                                             .ds_copy = ds_copy,
+                                                             .segment_attach = segment_attach,
+                                                             .segment_detach = segment_detach,
+                                                             .unlink = segment_unlink,
+                                                             .module_finalize = module_finalize}};
 
 /* ////////////////////////////////////////////////////////////////////////// */
 /* private utility functions */
@@ -120,16 +103,13 @@ opal_shmem_mmap_module_t opal_shmem_mmap_module = {
 /**
  * completely resets the contents of *ds_buf
  */
-static inline void
-shmem_ds_reset(opal_shmem_ds_t *ds_buf)
+static inline void shmem_ds_reset(opal_shmem_ds_t *ds_buf)
 {
     /* don't print ds_buf info here, as we may be printing garbage. */
-    OPAL_OUTPUT_VERBOSE(
-        (70, opal_shmem_base_framework.framework_output,
-         "%s: %s: shmem_ds_resetting\n",
-         mca_shmem_mmap_component.super.base_version.mca_type_name,
-         mca_shmem_mmap_component.super.base_version.mca_component_name)
-    );
+    OPAL_OUTPUT_VERBOSE((70, opal_shmem_base_framework.framework_output,
+                         "%s: %s: shmem_ds_resetting\n",
+                         mca_shmem_mmap_component.super.base_version.mca_type_name,
+                         mca_shmem_mmap_component.super.base_version.mca_component_name));
 
     ds_buf->seg_cpid = 0;
     OPAL_SHMEM_DS_RESET_FLAGS(ds_buf);
@@ -140,11 +120,7 @@ shmem_ds_reset(opal_shmem_ds_t *ds_buf)
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-static int
-enough_space(const char *filename,
-             size_t space_req,
-             uint64_t *space_avail,
-             bool *result)
+static int enough_space(const char *filename, size_t space_req, uint64_t *space_avail, bool *result)
 {
     uint64_t avail = 0;
     size_t fluff = (size_t)(.05 * space_req);
@@ -165,22 +141,17 @@ enough_space(const char *filename,
     /* now check space availability */
     if (OPAL_SUCCESS != (rc = opal_path_df(target_dir, &avail))) {
         OPAL_OUTPUT_VERBOSE(
-            (70, opal_shmem_base_framework.framework_output,
-             "WARNING: opal_path_df failure!")
-        );
+            (70, opal_shmem_base_framework.framework_output, "WARNING: opal_path_df failure!"));
         goto out;
     }
     /* do we have enough space? */
     if (avail >= space_req + fluff) {
         enough = true;
-    }
-    else {
-        OPAL_OUTPUT_VERBOSE(
-            (70, opal_shmem_base_framework.framework_output,
-             "WARNING: not enough space on %s to meet request!"
-             "available: %"PRIu64 "requested: %lu", target_dir,
-             avail, (unsigned long)space_req + fluff)
-        );
+    } else {
+        OPAL_OUTPUT_VERBOSE((70, opal_shmem_base_framework.framework_output,
+                             "WARNING: not enough space on %s to meet request!"
+                             "available: %" PRIu64 "requested: %lu",
+                             target_dir, avail, (unsigned long) space_req + fluff));
     }
 
 out:
@@ -193,48 +164,40 @@ enough_space(const char *filename,
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-static int
-module_init(void)
+static int module_init(void)
 {
     /* nothing to do */
     return OPAL_SUCCESS;
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-static int
-module_finalize(void)
+static int module_finalize(void)
 {
     /* nothing to do */
     return OPAL_SUCCESS;
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-static int
-ds_copy(const opal_shmem_ds_t *from,
-        opal_shmem_ds_t *to)
+static int ds_copy(const opal_shmem_ds_t *from, opal_shmem_ds_t *to)
 {
     memcpy(to, from, sizeof(opal_shmem_ds_t));
 
-    OPAL_OUTPUT_VERBOSE(
-        (70, opal_shmem_base_framework.framework_output,
-         "%s: %s: ds_copy complete "
-         "from: (id: %d, size: %lu, "
-         "name: %s flags: 0x%02x) "
-         "to: (id: %d, size: %lu, "
-         "name: %s flags: 0x%02x)\n",
-         mca_shmem_mmap_component.super.base_version.mca_type_name,
-         mca_shmem_mmap_component.super.base_version.mca_component_name,
-         from->seg_id, (unsigned long)from->seg_size, from->seg_name,
-         from->flags, to->seg_id, (unsigned long)to->seg_size, to->seg_name,
-         to->flags)
-    );
+    OPAL_OUTPUT_VERBOSE((70, opal_shmem_base_framework.framework_output,
+                         "%s: %s: ds_copy complete "
+                         "from: (id: %d, size: %lu, "
+                         "name: %s flags: 0x%02x) "
+                         "to: (id: %d, size: %lu, "
+                         "name: %s flags: 0x%02x)\n",
+                         mca_shmem_mmap_component.super.base_version.mca_type_name,
+                         mca_shmem_mmap_component.super.base_version.mca_component_name,
+                         from->seg_id, (unsigned long) from->seg_size, from->seg_name, from->flags,
+                         to->seg_id, (unsigned long) to->seg_size, to->seg_name, to->flags));
 
     return OPAL_SUCCESS;
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-static unsigned long
-sdbm_hash(const unsigned char *hash_key)
+static unsigned long sdbm_hash(const unsigned char *hash_key)
 {
     unsigned long str_hash = 0;
     int c;
@@ -247,8 +210,7 @@ sdbm_hash(const unsigned char *hash_key)
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-static bool
-path_usable(const char *path, int *stat_errno)
+static bool path_usable(const char *path, int *stat_errno)
 {
     struct stat buf;
     int rc;
@@ -266,8 +228,7 @@ path_usable(const char *path, int *stat_errno)
  * caller is responsible for freeing returned resources. the returned string
  * will be OPAL_PATH_MAX long.
  */
-static char *
-get_uniq_file_name(const char *base_path, const char *hash_key)
+static char *get_uniq_file_name(const char *base_path, const char *hash_key)
 {
     char *uniq_name_buf = NULL;
     unsigned long str_hash = 0;
@@ -285,21 +246,18 @@ get_uniq_file_name(const char *base_path, const char *hash_key)
     }
 
     my_pid = getpid();
-    opal_srand(&rand_buff,((uint32_t)(time(NULL) + my_pid)));
+    opal_srand(&rand_buff, ((uint32_t)(time(NULL) + my_pid)));
     rand_num = opal_rand(&rand_buff) % 1024;
-    str_hash = sdbm_hash((unsigned char *)hash_key);
+    str_hash = sdbm_hash((unsigned char *) hash_key);
     /* build the name */
-    snprintf(uniq_name_buf, OPAL_PATH_MAX, "%s/open_mpi_shmem_mmap.%d_%lu_%d",
-             base_path, (int)my_pid, str_hash, rand_num);
+    snprintf(uniq_name_buf, OPAL_PATH_MAX, "%s/open_mpi_shmem_mmap.%d_%lu_%d", base_path,
+             (int) my_pid, str_hash, rand_num);
 
     return uniq_name_buf;
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-static int
-segment_create(opal_shmem_ds_t *ds_buf,
-               const char *file_name,
-               size_t size)
+static int segment_create(opal_shmem_ds_t *ds_buf, const char *file_name, size_t size)
 {
     int rc = OPAL_SUCCESS;
     char *real_file_name = NULL;
@@ -315,10 +273,9 @@ segment_create(opal_shmem_ds_t *ds_buf,
     if (0 != opal_shmem_mmap_relocate_backing_file) {
         int err;
         if (path_usable(opal_shmem_mmap_backing_file_base_dir, &err)) {
-            if (NULL ==
-                (real_file_name =
-                     get_uniq_file_name(opal_shmem_mmap_backing_file_base_dir,
-                                        file_name))) {
+            if (NULL
+                == (real_file_name = get_uniq_file_name(opal_shmem_mmap_backing_file_base_dir,
+                                                        file_name))) {
                 /* out of resources */
                 return OPAL_ERROR;
             }
@@ -328,17 +285,19 @@ segment_create(opal_shmem_ds_t *ds_buf,
          * with the default path.  otherwise, fail.
          */
         else if (opal_shmem_mmap_relocate_backing_file < 0) {
-            opal_output(0, "shmem: mmap: WARNING: could not relocate "
+            opal_output(0,
+                        "shmem: mmap: WARNING: could not relocate "
                         "backing store to \"%s\" (%s).  Continuing with "
                         "default path.\n",
                         opal_shmem_mmap_backing_file_base_dir, strerror(err));
         }
         /* must be positive, so fail */
         else {
-            opal_output(0, "shmem: mmap: WARNING: could not relocate "
+            opal_output(0,
+                        "shmem: mmap: WARNING: could not relocate "
                         "backing store to \"%s\" (%s).  Cannot continue with "
-                        "shmem mmap.\n", opal_shmem_mmap_backing_file_base_dir,
-                        strerror(err));
+                        "shmem mmap.\n",
+                        opal_shmem_mmap_backing_file_base_dir, strerror(err));
             return OPAL_ERROR;
         }
     }
@@ -351,13 +310,11 @@ segment_create(opal_shmem_ds_t *ds_buf,
         }
     }
 
-    OPAL_OUTPUT_VERBOSE(
-        (70, opal_shmem_base_framework.framework_output,
-         "%s: %s: backing store base directory: %s\n",
-         mca_shmem_mmap_component.super.base_version.mca_type_name,
-         mca_shmem_mmap_component.super.base_version.mca_component_name,
-         real_file_name)
-    );
+    OPAL_OUTPUT_VERBOSE((70, opal_shmem_base_framework.framework_output,
+                         "%s: %s: backing store base directory: %s\n",
+                         mca_shmem_mmap_component.super.base_version.mca_type_name,
+                         mca_shmem_mmap_component.super.base_version.mca_component_name,
+                         real_file_name));
 
     /* determine whether the specified filename is on a network file system.
      * this is an important check because if the backing store is located on
@@ -366,15 +323,15 @@ segment_create(opal_shmem_ds_t *ds_buf,
     if (opal_shmem_mmap_nfs_warning && opal_path_nfs(real_file_name, NULL)) {
         const char *hn;
         hn = opal_gethostname();
-        opal_show_help("help-opal-shmem-mmap.txt", "mmap on nfs", 1, hn,
-                       real_file_name);
+        opal_show_help("help-opal-shmem-mmap.txt", "mmap on nfs", 1, hn, real_file_name);
     }
     /* let's make sure we have enough space for the backing file */
-    if (OPAL_SUCCESS != (rc = enough_space(real_file_name, size,
-                                           &amount_space_avail,
-                                           &space_available))) {
-        opal_output(0, "shmem: mmap: an error occurred while determining "
-                    "whether or not %s could be created.", real_file_name);
+    if (OPAL_SUCCESS
+        != (rc = enough_space(real_file_name, size, &amount_space_avail, &space_available))) {
+        opal_output(0,
+                    "shmem: mmap: an error occurred while determining "
+                    "whether or not %s could be created.",
+                    real_file_name);
         /* rc is set */
         goto out;
     }
@@ -382,9 +339,8 @@ segment_create(opal_shmem_ds_t *ds_buf,
         const char *hn;
         hn = opal_gethostname();
         rc = OPAL_ERR_OUT_OF_RESOURCE;
-        opal_show_help("help-opal-shmem-mmap.txt", "target full", 1,
-                       real_file_name, hn, (unsigned long)size,
-                       (unsigned long long)amount_space_avail);
+        opal_show_help("help-opal-shmem-mmap.txt", "target full", 1, real_file_name, hn,
+                       (unsigned long) size, (unsigned long long) amount_space_avail);
         goto out;
     }
     /* enough space is available, so create the segment */
@@ -392,8 +348,8 @@ segment_create(opal_shmem_ds_t *ds_buf,
         int err = errno;
         const char *hn;
         hn = opal_gethostname();
-        opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn,
-                       "open(2)", "", strerror(err), err);
+        opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn, "open(2)", "",
+                       strerror(err), err);
         rc = OPAL_ERROR;
         goto out;
     }
@@ -402,19 +358,18 @@ segment_create(opal_shmem_ds_t *ds_buf,
         int err = errno;
         const char *hn;
         hn = opal_gethostname();
-        opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn,
-                       "ftruncate(2)", "", strerror(err), err);
+        opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn, "ftruncate(2)", "",
+                       strerror(err), err);
         rc = OPAL_ERROR;
         goto out;
     }
-    if (MAP_FAILED == (segment = mmap(NULL, size,
-                                       PROT_READ | PROT_WRITE, MAP_SHARED,
-                                       ds_buf->seg_id, 0))) {
+    if (MAP_FAILED
+        == (segment = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, ds_buf->seg_id, 0))) {
         int err = errno;
         const char *hn;
         hn = opal_gethostname();
-        opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn,
-                       "mmap(2)", "", strerror(err), err);
+        opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn, "mmap(2)", "",
+                       strerror(err), err);
         rc = OPAL_ERROR;
         goto out;
     }
@@ -425,19 +380,17 @@ segment_create(opal_shmem_ds_t *ds_buf,
         ds_buf->seg_cpid = my_pid;
         ds_buf->seg_size = size;
         ds_buf->seg_base_addr = segment;
-        (void)opal_string_copy(ds_buf->seg_name, real_file_name, OPAL_PATH_MAX);
+        (void) opal_string_copy(ds_buf->seg_name, real_file_name, OPAL_PATH_MAX);
 
         /* set "valid" bit because setment creation was successful */
         OPAL_SHMEM_DS_SET_VALID(ds_buf);
 
-        OPAL_OUTPUT_VERBOSE(
-            (70, opal_shmem_base_framework.framework_output,
-             "%s: %s: create successful "
-             "(id: %d, size: %lu, name: %s)\n",
-             mca_shmem_mmap_component.super.base_version.mca_type_name,
-             mca_shmem_mmap_component.super.base_version.mca_component_name,
-             ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
-        );
+        OPAL_OUTPUT_VERBOSE((70, opal_shmem_base_framework.framework_output,
+                             "%s: %s: create successful "
+                             "(id: %d, size: %lu, name: %s)\n",
+                             mca_shmem_mmap_component.super.base_version.mca_type_name,
+                             mca_shmem_mmap_component.super.base_version.mca_component_name,
+                             ds_buf->seg_id, (unsigned long) ds_buf->seg_size, ds_buf->seg_name));
     }
 
 out:
@@ -451,12 +404,12 @@ segment_create(opal_shmem_ds_t *ds_buf,
             int err = errno;
             const char *hn;
             hn = opal_gethostname();
-            opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn,
-                           "close(2)", "", strerror(err), err);
+            opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn, "close(2)", "",
+                           strerror(err), err);
             rc = OPAL_ERROR;
-         }
-     }
-    /* an error occured, so invalidate the shmem object and munmap if needed */
+        }
+    }
+    /* an error occurred, so invalidate the shmem object and munmap if needed */
     if (OPAL_SUCCESS != rc) {
         if (MAP_FAILED != segment) {
             munmap(segment, size);
@@ -474,8 +427,7 @@ segment_create(opal_shmem_ds_t *ds_buf,
 /**
  * segment_attach can only be called after a successful call to segment_create
  */
-static void *
-segment_attach(opal_shmem_ds_t *ds_buf)
+static void *segment_attach(opal_shmem_ds_t *ds_buf)
 {
     pid_t my_pid = getpid();
 
@@ -484,18 +436,18 @@ segment_attach(opal_shmem_ds_t *ds_buf)
             int err = errno;
             const char *hn;
             hn = opal_gethostname();
-            opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn,
-                           "open(2)", "", strerror(err), err);
+            opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn, "open(2)", "",
+                           strerror(err), err);
             return NULL;
         }
-        if (MAP_FAILED == (ds_buf->seg_base_addr = mmap(NULL, ds_buf->seg_size,
-                                                        PROT_READ | PROT_WRITE, MAP_SHARED,
-                                                        ds_buf->seg_id, 0))) {
+        if (MAP_FAILED
+            == (ds_buf->seg_base_addr = mmap(NULL, ds_buf->seg_size, PROT_READ | PROT_WRITE,
+                                             MAP_SHARED, ds_buf->seg_id, 0))) {
             int err = errno;
             const char *hn;
             hn = opal_gethostname();
-            opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn,
-                           "mmap(2)", "", strerror(err), err);
+            opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn, "mmap(2)", "",
+                           strerror(err), err);
             /* mmap failed, so close the file and return NULL - no error check
              * here because we are already in an error path...
              */
@@ -510,48 +462,43 @@ segment_attach(opal_shmem_ds_t *ds_buf)
             int err = errno;
             const char *hn;
             hn = opal_gethostname();
-            opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1,
-                           hn, "close(2)", "", strerror(err), err);
+            opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn, "close(2)", "",
+                           strerror(err), err);
         }
     }
     /* else i was the segment creator.  nothing to do here because all the hard
      * work was done in segment_create :-).
      */
 
-    OPAL_OUTPUT_VERBOSE(
-        (70, opal_shmem_base_framework.framework_output,
-         "%s: %s: attach successful "
-         "(id: %d, size: %lu, name: %s)\n",
-         mca_shmem_mmap_component.super.base_version.mca_type_name,
-         mca_shmem_mmap_component.super.base_version.mca_component_name,
-         ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
-    );
+    OPAL_OUTPUT_VERBOSE((70, opal_shmem_base_framework.framework_output,
+                         "%s: %s: attach successful "
+                         "(id: %d, size: %lu, name: %s)\n",
+                         mca_shmem_mmap_component.super.base_version.mca_type_name,
+                         mca_shmem_mmap_component.super.base_version.mca_component_name,
+                         ds_buf->seg_id, (unsigned long) ds_buf->seg_size, ds_buf->seg_name));
 
     /* update returned base pointer with an offset that hides our stuff */
     return ds_buf->seg_base_addr;
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-static int
-segment_detach(opal_shmem_ds_t *ds_buf)
+static int segment_detach(opal_shmem_ds_t *ds_buf)
 {
     int rc = OPAL_SUCCESS;
 
-    OPAL_OUTPUT_VERBOSE(
-        (70, opal_shmem_base_framework.framework_output,
-         "%s: %s: detaching "
-         "(id: %d, size: %lu, name: %s)\n",
-         mca_shmem_mmap_component.super.base_version.mca_type_name,
-         mca_shmem_mmap_component.super.base_version.mca_component_name,
-         ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
-    );
+    OPAL_OUTPUT_VERBOSE((70, opal_shmem_base_framework.framework_output,
+                         "%s: %s: detaching "
+                         "(id: %d, size: %lu, name: %s)\n",
+                         mca_shmem_mmap_component.super.base_version.mca_type_name,
+                         mca_shmem_mmap_component.super.base_version.mca_component_name,
+                         ds_buf->seg_id, (unsigned long) ds_buf->seg_size, ds_buf->seg_name));
 
     if (0 != munmap(ds_buf->seg_base_addr, ds_buf->seg_size)) {
         int err = errno;
         const char *hn;
         hn = opal_gethostname();
-        opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn,
-                       "munmap(2)", "", strerror(err), err);
+        opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn, "munmap(2)", "",
+                       strerror(err), err);
         rc = OPAL_ERROR;
     }
     /* reset the contents of the opal_shmem_ds_t associated with this
@@ -562,24 +509,21 @@ segment_detach(opal_shmem_ds_t *ds_buf)
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-static int
-segment_unlink(opal_shmem_ds_t *ds_buf)
+static int segment_unlink(opal_shmem_ds_t *ds_buf)
 {
-    OPAL_OUTPUT_VERBOSE(
-        (70, opal_shmem_base_framework.framework_output,
-         "%s: %s: unlinking"
-         "(id: %d, size: %lu, name: %s)\n",
-         mca_shmem_mmap_component.super.base_version.mca_type_name,
-         mca_shmem_mmap_component.super.base_version.mca_component_name,
-         ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
-    );
+    OPAL_OUTPUT_VERBOSE((70, opal_shmem_base_framework.framework_output,
+                         "%s: %s: unlinking"
+                         "(id: %d, size: %lu, name: %s)\n",
+                         mca_shmem_mmap_component.super.base_version.mca_type_name,
+                         mca_shmem_mmap_component.super.base_version.mca_component_name,
+                         ds_buf->seg_id, (unsigned long) ds_buf->seg_size, ds_buf->seg_name));
 
     if (-1 == unlink(ds_buf->seg_name)) {
         int err = errno;
         const char *hn;
         hn = opal_gethostname();
-        opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn,
-                       "unlink(2)", ds_buf->seg_name, strerror(err), err);
+        opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn, "unlink(2)",
+                       ds_buf->seg_name, strerror(err), err);
         return OPAL_ERROR;
     }
 
@@ -588,8 +532,7 @@ segment_unlink(opal_shmem_ds_t *ds_buf)
      * across unlinks. other information stored in flags will remain untouched.
      */
     ds_buf->seg_id = OPAL_SHMEM_DS_ID_INVALID;
-    /* note: this is only chaning the valid bit to 0. */
+    /* note: this is only changing the valid bit to 0. */
     OPAL_SHMEM_DS_INVALIDATE(ds_buf);
     return OPAL_SUCCESS;
 }
-
diff --git a/opal/mca/shmem/posix/Makefile.am b/opal/mca/shmem/posix/Makefile.am
index 6ac26f0b926..a317e9236c1 100644
--- a/opal/mca/shmem/posix/Makefile.am
+++ b/opal/mca/shmem/posix/Makefile.am
@@ -13,6 +13,7 @@
 # Copyright (c) 2010-2011 Los Alamos National Security, LLC.
 #                         All rights reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -42,11 +43,11 @@ mcacomponentdir = $(opallibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_shmem_posix_la_SOURCES = $(sources)
 mca_shmem_posix_la_LDFLAGS = -module -avoid-version
-mca_shmem_posix_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+mca_shmem_posix_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 
 noinst_LTLIBRARIES = $(component_noinst)
 libmca_shmem_posix_la_SOURCES = $(sources)
 libmca_shmem_posix_la_LDFLAGS = -module -avoid-version
 
 # help file
-dist_opaldata_DATA = help-opal-shmem-posix.txt
+EXTRA_DIST = help-opal-shmem-posix.txt
diff --git a/opal/mca/shmem/posix/shmem_posix.h b/opal/mca/shmem/posix/shmem_posix.h
index 9794bb22d8f..16cfff5c6b6 100644
--- a/opal/mca/shmem/posix/shmem_posix.h
+++ b/opal/mca/shmem/posix/shmem_posix.h
@@ -54,8 +54,7 @@ typedef struct opal_shmem_posix_component_t {
     int priority;
 } opal_shmem_posix_component_t;
 
-OPAL_MODULE_DECLSPEC extern opal_shmem_posix_component_t
-mca_shmem_posix_component;
+OPAL_DECLSPEC extern opal_shmem_posix_component_t mca_shmem_posix_component;
 
 typedef struct opal_shmem_posix_module_t {
     opal_shmem_base_module_t super;
diff --git a/opal/mca/shmem/posix/shmem_posix_common_utils.c b/opal/mca/shmem/posix/shmem_posix_common_utils.c
index d5fbddbbe26..4b6d164e145 100644
--- a/opal/mca/shmem/posix/shmem_posix_common_utils.c
+++ b/opal/mca/shmem/posix/shmem_posix_common_utils.c
@@ -17,7 +17,9 @@
  * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2019      Triad National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
  *
+ * Copyright (c) 2025      Nanook Consulting  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -29,40 +31,33 @@
 
 #include <errno.h>
 #ifdef HAVE_FCNTL_H
-#include <fcntl.h>
-#endif  /* HAVE_FCNTL_H */
+#    include <fcntl.h>
+#endif /* HAVE_FCNTL_H */
 #include <string.h>
-#if OPAL_HAVE_SOLARIS && !defined(_POSIX_C_SOURCE)
-  #define _POSIX_C_SOURCE 200112L /* Required for shm_{open,unlink} decls */
-  #include <sys/mman.h>
-  #undef _POSIX_C_SOURCE
-#else
 #ifdef HAVE_SYS_MMAN_H
-#include <sys/mman.h>
+#    include <sys/mman.h>
 #endif /* HAVE_SYS_MMAN_H */
-#endif
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif /* HAVE_UNISTD_H */
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif /* HAVE_SYS_TYPES_H */
 #ifdef HAVE_NETDB_H
-#include <netdb.h>
+#    include <netdb.h>
 #endif /* HAVE_NETDB_H */
 
+#include "opal/mca/shmem/base/base.h"
+#include "opal/mca/shmem/shmem.h"
 #include "opal/runtime/opal.h"
 #include "opal/util/output.h"
 #include "opal/util/show_help.h"
-#include "opal/mca/shmem/base/base.h"
-#include "opal/mca/shmem/shmem.h"
 
 #include "shmem_posix.h"
 #include "shmem_posix_common_utils.h"
 
 /* ////////////////////////////////////////////////////////////////////////// */
-int
-shmem_posix_shm_open(char *posix_file_name_buff, size_t size)
+int opal_shmem_posix_shm_open(char *posix_file_name_buff, size_t size)
 {
     int attempt = 0, fd = -1;
 
@@ -75,13 +70,12 @@ shmem_posix_shm_open(char *posix_file_name_buff, size_t size)
          * see comment in shmem_posix.h that explains why we chose to do things
          * this way.
          */
-        snprintf(posix_file_name_buff, size, "%s%04d",
-                 OPAL_SHMEM_POSIX_FILE_NAME_PREFIX, attempt++);
+        snprintf(posix_file_name_buff, size, "%s%04d", OPAL_SHMEM_POSIX_FILE_NAME_PREFIX,
+                 attempt++);
         /* the check for the existence of the object and its creation if it
          * does not exist are performed atomically.
          */
-        if (-1 == (fd = shm_open(posix_file_name_buff,
-                                 O_CREAT | O_EXCL | O_RDWR, 0600))) {
+        if (-1 == (fd = shm_open(posix_file_name_buff, O_CREAT | O_EXCL | O_RDWR, 0600))) {
             int err = errno;
             /* the object already exists, so try again with a new name */
             if (EEXIST == err) {
@@ -92,9 +86,9 @@ shmem_posix_shm_open(char *posix_file_name_buff, size_t size)
              */
             else {
                 opal_output_verbose(10, opal_shmem_base_framework.framework_output,
-                     "shmem_posix_shm_open: disqualifying posix because "
-                     "shm_open(2) failed with error: %s (errno %d)\n",
-                     strerror(err), err);
+                                    "shmem_posix_shm_open: disqualifying posix because "
+                                    "shm_open(2) failed with error: %s (errno %d)\n",
+                                    strerror(err), err);
                 break;
             }
         }
@@ -107,8 +101,7 @@ shmem_posix_shm_open(char *posix_file_name_buff, size_t size)
     /* if we didn't find a name, let the user know that we tried and failed */
     if (attempt >= OPAL_SHMEM_POSIX_MAX_ATTEMPTS) {
         opal_output(0, "shmem: posix: file name search - max attempts exceeded."
-                    "cannot continue with posix.\n");
+                       "cannot continue with posix.\n");
     }
     return fd;
 }
-
diff --git a/opal/mca/shmem/posix/shmem_posix_common_utils.h b/opal/mca/shmem/posix/shmem_posix_common_utils.h
index e83a2bc736a..58b947f9fad 100644
--- a/opal/mca/shmem/posix/shmem_posix_common_utils.h
+++ b/opal/mca/shmem/posix/shmem_posix_common_utils.h
@@ -14,6 +14,7 @@
  * Copyright (c) 2010      IBM Corporation.  All rights reserved.
  * Copyright (c) 2010-2011 Los Alamos National Security, LLC.
  *                         All rights reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -42,8 +43,7 @@ BEGIN_C_DECLS
  * successful shm_open.  otherwise, -1 is returned and the contents of
  * posix_file_name_buff are undefined.
  */
-OPAL_DECLSPEC extern int shmem_posix_shm_open(char *posix_file_name_buff,
-                                              size_t size);
+OPAL_DECLSPEC extern int opal_shmem_posix_shm_open(char *posix_file_name_buff, size_t size);
 
 END_C_DECLS
 
diff --git a/opal/mca/shmem/posix/shmem_posix_component.c b/opal/mca/shmem/posix/shmem_posix_component.c
index fea44efe34c..0993b69246a 100644
--- a/opal/mca/shmem/posix/shmem_posix_component.c
+++ b/opal/mca/shmem/posix/shmem_posix_component.c
@@ -17,6 +17,7 @@
  * Copyright (c) 2016      Intel, Inc. All rights reserved.
  * Copyright (c) 2019      Triad National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -32,39 +33,36 @@
 
 #include "opal_config.h"
 
-
 #include <errno.h>
 #include <string.h>
 #ifdef HAVE_SYS_MMAN_H
-#include <sys/mman.h>
+#    include <sys/mman.h>
 #endif /* HAVE_SYS_MMAN_H */
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif /* HAVE_UNISTD_H */
 #ifdef HAVE_NETDB_H
-#include <netdb.h>
+#    include <netdb.h>
 #endif /* HAVE_NETDB_H */
 
-#include "opal/runtime/opal.h"
 #include "opal/constants.h"
-#include "opal/util/show_help.h"
-#include "opal/util/output.h"
 #include "opal/mca/shmem/base/base.h"
 #include "opal/mca/shmem/shmem.h"
+#include "opal/runtime/opal.h"
+#include "opal/util/output.h"
+#include "opal/util/show_help.h"
 #include "shmem_posix.h"
 #include "shmem_posix_common_utils.h"
 
 /* public string showing the shmem ompi_posix component version number */
-const char *opal_shmem_posix_component_version_string =
-    "OPAL posix shmem MCA component version " OPAL_VERSION;
+const char *opal_shmem_posix_component_version_string
+    = "OPAL posix shmem MCA component version " OPAL_VERSION;
 
 /* local functions */
-static int posix_register (void);
+static int posix_register(void);
 static int posix_open(void);
 static int posix_query(mca_base_module_t **module, int *priority);
-static int posix_runtime_query(mca_base_module_t **module,
-                               int *priority,
-                               const char *hint);
+static int posix_runtime_query(mca_base_module_t **module, int *priority, const char *hint);
 
 /* local variables */
 static bool rt_successful = false;
@@ -73,41 +71,37 @@ static bool rt_successful = false;
  * and pointers to our public functions in it
  */
 opal_shmem_posix_component_t mca_shmem_posix_component = {
-    .super = {
-        .base_version = {
-            OPAL_SHMEM_BASE_VERSION_2_0_0,
-
-            /* component name and version */
-            .mca_component_name = "posix",
-            MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
-                                  OPAL_RELEASE_VERSION),
-
-            .mca_open_component = posix_open,
-            .mca_query_component = posix_query,
-            .mca_register_component_params = posix_register
-        },
-        /* MCA v2.0.0 component meta data */
-        .base_data = {
-            /* the component is checkpoint ready */
-            MCA_BASE_METADATA_PARAM_CHECKPOINT
+    .super =
+        {
+            .base_version = {OPAL_SHMEM_BASE_VERSION_2_0_0,
+
+                             /* component name and version */
+                             .mca_component_name = "posix",
+                             MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION,
+                                                   OPAL_MINOR_VERSION, OPAL_RELEASE_VERSION),
+
+                             .mca_open_component = posix_open, .mca_query_component = posix_query,
+                             .mca_register_component_params = posix_register},
+            /* MCA v2.0.0 component meta data */
+            .base_data =
+                {/* the component is checkpoint ready */
+                 MCA_BASE_METADATA_PARAM_CHECKPOINT},
+            .runtime_query = posix_runtime_query,
         },
-        .runtime_query = posix_runtime_query,
-    },
 };
 
-
 /* ////////////////////////////////////////////////////////////////////////// */
-static int
-posix_register(void)
+static int posix_register(void)
 {
     /* ////////////////////////////////////////////////////////////////////// */
     /* (default) priority - set lower than mmap's priority */
     mca_shmem_posix_component.priority = 40;
     (void) mca_base_component_var_register(&mca_shmem_posix_component.super.base_version,
-                                           "priority", "Priority for the shmem posix "
-                                           "component (default: 40)", MCA_BASE_VAR_TYPE_INT,
-                                           NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-                                           OPAL_INFO_LVL_3,
+                                           "priority",
+                                           "Priority for the shmem posix "
+                                           "component (default: 40)",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0,
+                                           MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
                                            MCA_BASE_VAR_SCOPE_ALL_EQ,
                                            &mca_shmem_posix_component.priority);
 
@@ -116,8 +110,7 @@ posix_register(void)
 
 /* ////////////////////////////////////////////////////////////////////////// */
 
-static int
-posix_open(void)
+static int posix_open(void)
 {
     return OPAL_SUCCESS;
 }
@@ -130,10 +123,7 @@ posix_open(void)
  *
  * @return OPAL_SUCCESS when posix can safely be used.
  */
-static int
-posix_runtime_query(mca_base_module_t **module,
-                    int *priority,
-                    const char *hint)
+static int posix_runtime_query(mca_base_module_t **module, int *priority, const char *hint)
 {
     char tmp_buff[OPAL_SHMEM_POSIX_FILE_LEN_MAX];
     int fd = -1;
@@ -146,21 +136,19 @@ posix_runtime_query(mca_base_module_t **module,
      * don't have to perform a run-time query.
      */
     if (NULL != hint) {
-        OPAL_OUTPUT_VERBOSE(
-            (70, opal_shmem_base_framework.framework_output,
-             "shmem: posix: runtime_query: "
-             "attempting to use runtime hint (%s)\n", hint)
-        );
+        OPAL_OUTPUT_VERBOSE((70, opal_shmem_base_framework.framework_output,
+                             "shmem: posix: runtime_query: "
+                             "attempting to use runtime hint (%s)\n",
+                             hint));
         /* was i selected? if so, then we are done.
          * otherwise, disqualify myself.
          */
-        if (0 == strcasecmp(hint,
-            mca_shmem_posix_component.super.base_version.mca_component_name)) {
+        if (0
+            == strcasecmp(hint, mca_shmem_posix_component.super.base_version.mca_component_name)) {
             *priority = mca_shmem_posix_component.priority;
-            *module = (mca_base_module_t *)&opal_shmem_posix_module.super;
+            *module = (mca_base_module_t *) &opal_shmem_posix_module.super;
             return OPAL_SUCCESS;
-        }
-        else {
+        } else {
             *priority = 0;
             *module = NULL;
             return OPAL_SUCCESS;
@@ -170,28 +158,25 @@ posix_runtime_query(mca_base_module_t **module,
      * hint.  it's either up to us to figure it out, or the caller wants us to
      * re-run the runtime query.
      */
-    OPAL_OUTPUT_VERBOSE(
-        (70, opal_shmem_base_framework.framework_output,
-         "shmem: posix: runtime_query: NO HINT PROVIDED:"
-         "starting run-time test...\n")
-    );
+    OPAL_OUTPUT_VERBOSE((70, opal_shmem_base_framework.framework_output,
+                         "shmem: posix: runtime_query: NO HINT PROVIDED:"
+                         "starting run-time test...\n"));
     /* shmem_posix_shm_open successfully shm_opened - we can use posix sm! */
-    if (-1 != (fd = shmem_posix_shm_open(tmp_buff,
-                                         OPAL_SHMEM_POSIX_FILE_LEN_MAX -1))) {
+    if (-1 != (fd = opal_shmem_posix_shm_open(tmp_buff, OPAL_SHMEM_POSIX_FILE_LEN_MAX - 1))) {
         /* free up allocated resources before we return */
         if (0 != shm_unlink(tmp_buff)) {
             int err = errno;
             const char *hn;
             hn = opal_gethostname();
-            opal_show_help("help-opal-shmem-posix.txt", "sys call fail", 1,
-                           hn, "shm_unlink(2)", "", strerror(err), err);
+            opal_show_help("help-opal-shmem-posix.txt", "sys call fail", 1, hn, "shm_unlink(2)", "",
+                           strerror(err), err);
             /* something strange happened, so consider this a run-time test
              * failure even though shmem_posix_shm_open was successful */
         }
         /* all is well */
         else {
             *priority = mca_shmem_posix_component.priority;
-            *module = (mca_base_module_t *)&opal_shmem_posix_module.super;
+            *module = (mca_base_module_t *) &opal_shmem_posix_module.super;
             rt_successful = true;
         }
     }
@@ -200,11 +185,9 @@ posix_runtime_query(mca_base_module_t **module,
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-static int
-posix_query(mca_base_module_t **module, int *priority)
+static int posix_query(mca_base_module_t **module, int *priority)
 {
     *priority = mca_shmem_posix_component.priority;
-    *module = (mca_base_module_t *)&opal_shmem_posix_module.super;
+    *module = (mca_base_module_t *) &opal_shmem_posix_module.super;
     return OPAL_SUCCESS;
 }
-
diff --git a/opal/mca/shmem/posix/shmem_posix_module.c b/opal/mca/shmem/posix/shmem_posix_module.c
index 95bdfb8d3d5..ef1843535f7 100644
--- a/opal/mca/shmem/posix/shmem_posix_module.c
+++ b/opal/mca/shmem/posix/shmem_posix_module.c
@@ -16,6 +16,8 @@
  * Copyright (c) 2014      Intel, Inc. All rights reserved.
  * Copyright (c) 2019      Triad National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      IBM Corporation. All rights reserved
+ * Copyright (c) 2025      Nanook Consulting  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -27,36 +29,30 @@
 
 #include <errno.h>
 #ifdef HAVE_FCNTL_H
-#include <fcntl.h>
-#endif  /* HAVE_FCNTL_H */
-#if OPAL_HAVE_SOLARIS && !defined(_POSIX_C_SOURCE)
-  #define _POSIX_C_SOURCE 200112L /* Required for shm_{open,unlink} decls */
-  #include <sys/mman.h>
-  #undef _POSIX_C_SOURCE
-#else
+#    include <fcntl.h>
+#endif /* HAVE_FCNTL_H */
 #ifdef HAVE_SYS_MMAN_H
-#include <sys/mman.h>
+#    include <sys/mman.h>
 #endif /* HAVE_SYS_MMAN_H */
-#endif
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif /* HAVE_UNISTD_H */
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif /* HAVE_SYS_TYPES_H */
 #include <string.h>
 #ifdef HAVE_NETDB_H
-#include <netdb.h>
+#    include <netdb.h>
 #endif /* HAVE_NETDB_H */
 
-#include "opal/runtime/opal.h"
 #include "opal/constants.h"
-#include "opal_stdint.h"
+#include "opal/mca/shmem/base/base.h"
+#include "opal/mca/shmem/shmem.h"
+#include "opal/runtime/opal.h"
 #include "opal/util/output.h"
 #include "opal/util/path.h"
 #include "opal/util/show_help.h"
-#include "opal/mca/shmem/shmem.h"
-#include "opal/mca/shmem/base/base.h"
+#include "opal_stdint.h"
 
 #include "shmem_posix.h"
 #include "shmem_posix_common_utils.h"
@@ -65,42 +61,28 @@
 
 /* ////////////////////////////////////////////////////////////////////////// */
 /* local functions */
-static int
-module_init(void);
+static int module_init(void);
 
-static int
-segment_create(opal_shmem_ds_t *ds_buf,
-               const char *file_name,
-               size_t size);
+static int segment_create(opal_shmem_ds_t *ds_buf, const char *file_name, size_t size);
 
-static int
-ds_copy(const opal_shmem_ds_t *from,
-        opal_shmem_ds_t *to);
+static int ds_copy(const opal_shmem_ds_t *from, opal_shmem_ds_t *to);
 
-static void *
-segment_attach(opal_shmem_ds_t *ds_buf);
+static void *segment_attach(opal_shmem_ds_t *ds_buf);
 
-static int
-segment_detach(opal_shmem_ds_t *ds_buf);
+static int segment_detach(opal_shmem_ds_t *ds_buf);
 
-static int
-segment_unlink(opal_shmem_ds_t *ds_buf);
+static int segment_unlink(opal_shmem_ds_t *ds_buf);
 
-static int
-module_finalize(void);
+static int module_finalize(void);
 
 /* posix shmem module */
-opal_shmem_posix_module_t opal_shmem_posix_module = {
-    .super = {
-        .module_init = module_init,
-        .segment_create = segment_create,
-        .ds_copy = ds_copy,
-        .segment_attach = segment_attach,
-        .segment_detach = segment_detach,
-        .unlink = segment_unlink,
-        .module_finalize = module_finalize
-    }
-};
+opal_shmem_posix_module_t opal_shmem_posix_module = {.super = {.module_init = module_init,
+                                                               .segment_create = segment_create,
+                                                               .ds_copy = ds_copy,
+                                                               .segment_attach = segment_attach,
+                                                               .segment_detach = segment_detach,
+                                                               .unlink = segment_unlink,
+                                                               .module_finalize = module_finalize}};
 
 /* ////////////////////////////////////////////////////////////////////////// */
 /* private utility functions */
@@ -110,16 +92,13 @@ opal_shmem_posix_module_t opal_shmem_posix_module = {
 /**
  * completely resets the contents of *ds_buf
  */
-static inline void
-shmem_ds_reset(opal_shmem_ds_t *ds_buf)
+static inline void shmem_ds_reset(opal_shmem_ds_t *ds_buf)
 {
     /* don't print ds_buf info here, as we may be printing garbage. */
-    OPAL_OUTPUT_VERBOSE(
-        (70, opal_shmem_base_framework.framework_output,
-         "%s: %s: shmem_ds_resetting\n",
-         mca_shmem_posix_component.super.base_version.mca_type_name,
-         mca_shmem_posix_component.super.base_version.mca_component_name)
-    );
+    OPAL_OUTPUT_VERBOSE((70, opal_shmem_base_framework.framework_output,
+                         "%s: %s: shmem_ds_resetting\n",
+                         mca_shmem_posix_component.super.base_version.mca_type_name,
+                         mca_shmem_posix_component.super.base_version.mca_component_name));
 
     ds_buf->seg_cpid = 0;
     OPAL_SHMEM_DS_RESET_FLAGS(ds_buf);
@@ -130,50 +109,40 @@ shmem_ds_reset(opal_shmem_ds_t *ds_buf)
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-static int
-module_init(void)
+static int module_init(void)
 {
     /* nothing to do */
     return OPAL_SUCCESS;
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-static int
-module_finalize(void)
+static int module_finalize(void)
 {
     /* nothing to do */
     return OPAL_SUCCESS;
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-static int
-ds_copy(const opal_shmem_ds_t *from,
-        opal_shmem_ds_t *to)
+static int ds_copy(const opal_shmem_ds_t *from, opal_shmem_ds_t *to)
 {
     memcpy(to, from, sizeof(opal_shmem_ds_t));
 
-    OPAL_OUTPUT_VERBOSE(
-        (70, opal_shmem_base_framework.framework_output,
-         "%s: %s: ds_copy complete "
-         "from: (id: %d, size: %lu, "
-         "name: %s flags: 0x%02x) "
-         "to: (id: %d, size: %lu, "
-         "name: %s flags: 0x%02x)\n",
-         mca_shmem_posix_component.super.base_version.mca_type_name,
-         mca_shmem_posix_component.super.base_version.mca_component_name,
-         from->seg_id, (unsigned long)from->seg_size, from->seg_name,
-         from->flags, to->seg_id, (unsigned long)to->seg_size, to->seg_name,
-         to->flags)
-    );
+    OPAL_OUTPUT_VERBOSE((70, opal_shmem_base_framework.framework_output,
+                         "%s: %s: ds_copy complete "
+                         "from: (id: %d, size: %lu, "
+                         "name: %s flags: 0x%02x) "
+                         "to: (id: %d, size: %lu, "
+                         "name: %s flags: 0x%02x)\n",
+                         mca_shmem_posix_component.super.base_version.mca_type_name,
+                         mca_shmem_posix_component.super.base_version.mca_component_name,
+                         from->seg_id, (unsigned long) from->seg_size, from->seg_name, from->flags,
+                         to->seg_id, (unsigned long) to->seg_size, to->seg_name, to->flags));
 
     return OPAL_SUCCESS;
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-static int
-segment_create(opal_shmem_ds_t *ds_buf,
-               const char *file_name,
-               size_t size)
+static int segment_create(opal_shmem_ds_t *ds_buf, const char *file_name, size_t size)
 {
     int rc = OPAL_SUCCESS;
     pid_t my_pid = getpid();
@@ -190,9 +159,9 @@ segment_create(opal_shmem_ds_t *ds_buf,
      * memory object name and upon successful completion populates the name
      * buffer
      */
-    if (-1 == (ds_buf->seg_id = shmem_posix_shm_open(
-                                    ds_buf->seg_name,
-                                    OPAL_SHMEM_POSIX_FILE_LEN_MAX - 1))) {
+    if (-1
+        == (ds_buf->seg_id = opal_shmem_posix_shm_open(ds_buf->seg_name,
+                                                  OPAL_SHMEM_POSIX_FILE_LEN_MAX - 1))) {
         /* snaps!  something happened in posix_shm_open.  don't report anything
          * here because posix_shm_open will display all the necessary info.
          */
@@ -204,19 +173,18 @@ segment_create(opal_shmem_ds_t *ds_buf,
         int err = errno;
         const char *hn;
         hn = opal_gethostname();
-        opal_show_help("help-opal-shmem-posix.txt", "sys call fail", 1, hn,
-                       "ftruncate(2)", "", strerror(err), err);
+        opal_show_help("help-opal-shmem-posix.txt", "sys call fail", 1, hn, "ftruncate(2)", "",
+                       strerror(err), err);
         rc = OPAL_ERROR;
         goto out;
-    }
-    else if (MAP_FAILED == (segment = mmap(NULL, size,
-                                           PROT_READ | PROT_WRITE, MAP_SHARED,
-                                           ds_buf->seg_id, 0))) {
+    } else if (MAP_FAILED
+               == (segment = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, ds_buf->seg_id,
+                                  0))) {
         int err = errno;
         const char *hn;
         hn = opal_gethostname();
-        opal_show_help("help-opal-shmem-posix.txt", "sys call fail", 1, hn,
-                       "mmap(2)", "", strerror(err), err);
+        opal_show_help("help-opal-shmem-posix.txt", "sys call fail", 1, hn, "mmap(2)", "",
+                       strerror(err), err);
         rc = OPAL_ERROR;
         goto out;
     }
@@ -236,14 +204,12 @@ segment_create(opal_shmem_ds_t *ds_buf,
         /* set "valid" bit because setment creation was successful */
         OPAL_SHMEM_DS_SET_VALID(ds_buf);
 
-        OPAL_OUTPUT_VERBOSE(
-            (70, opal_shmem_base_framework.framework_output,
-             "%s: %s: create successful "
-             "(id: %d, size: %lu, name: %s)\n",
-             mca_shmem_posix_component.super.base_version.mca_type_name,
-             mca_shmem_posix_component.super.base_version.mca_component_name,
-             ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
-        );
+        OPAL_OUTPUT_VERBOSE((70, opal_shmem_base_framework.framework_output,
+                             "%s: %s: create successful "
+                             "(id: %d, size: %lu, name: %s)\n",
+                             mca_shmem_posix_component.super.base_version.mca_type_name,
+                             mca_shmem_posix_component.super.base_version.mca_component_name,
+                             ds_buf->seg_id, (unsigned long) ds_buf->seg_size, ds_buf->seg_name));
     }
 
 out:
@@ -257,12 +223,12 @@ segment_create(opal_shmem_ds_t *ds_buf,
             int err = errno;
             const char *hn;
             hn = opal_gethostname();
-            opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn,
-                           "close(2)", "", strerror(err), err);
+            opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn, "close(2)", "",
+                           strerror(err), err);
             rc = OPAL_ERROR;
-         }
-     }
-    /* an error occured, so invalidate the shmem object and release any
+        }
+    }
+    /* an error occurred, so invalidate the shmem object and release any
      * allocated resources.
      */
     if (OPAL_SUCCESS != rc) {
@@ -274,7 +240,7 @@ segment_create(opal_shmem_ds_t *ds_buf,
             shm_unlink(ds_buf->seg_name);
         }
         if (MAP_FAILED != segment) {
-            munmap((void*)segment, size);
+            munmap((void *) segment, size);
         }
         /* always invalidate in this error path */
         shmem_ds_reset(ds_buf);
@@ -286,8 +252,7 @@ segment_create(opal_shmem_ds_t *ds_buf,
 /**
  * segment_attach can only be called after a successful call to segment_create
  */
-static void *
-segment_attach(opal_shmem_ds_t *ds_buf)
+static void *segment_attach(opal_shmem_ds_t *ds_buf)
 {
     pid_t my_pid = getpid();
 
@@ -296,18 +261,17 @@ segment_attach(opal_shmem_ds_t *ds_buf)
             int err = errno;
             const char *hn;
             hn = opal_gethostname();
-            opal_show_help("help-opal-shmem-posix.txt", "sys call fail", 1, hn,
-                           "open(2)", "", strerror(err), err);
+            opal_show_help("help-opal-shmem-posix.txt", "sys call fail", 1, hn, "open(2)", "",
+                           strerror(err), err);
             return NULL;
-        }
-        else if (MAP_FAILED == (ds_buf->seg_base_addr = mmap(NULL, ds_buf->seg_size,
-                                                             PROT_READ | PROT_WRITE, MAP_SHARED,
-                                                             ds_buf->seg_id, 0))) {
+        } else if (MAP_FAILED
+                   == (ds_buf->seg_base_addr = mmap(NULL, ds_buf->seg_size, PROT_READ | PROT_WRITE,
+                                                    MAP_SHARED, ds_buf->seg_id, 0))) {
             int err = errno;
             const char *hn;
             hn = opal_gethostname();
-            opal_show_help("help-opal-shmem-posix.txt", "sys call fail", 1, hn,
-                           "mmap(2)", "", strerror(err), err);
+            opal_show_help("help-opal-shmem-posix.txt", "sys call fail", 1, hn, "mmap(2)", "",
+                           strerror(err), err);
             /* mmap failed, so shm_unlink and return NULL - no error check here
              * because we are already in an error path...
              */
@@ -323,49 +287,44 @@ segment_attach(opal_shmem_ds_t *ds_buf)
                 int err = errno;
                 const char *hn;
                 hn = opal_gethostname();
-                opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1,
-                               hn, "close(2)", "", strerror(err), err);
-             }
+                opal_show_help("help-opal-shmem-mmap.txt", "sys call fail", 1, hn, "close(2)", "",
+                               strerror(err), err);
+            }
         }
     }
     /* else i was the segment creator.  nothing to do here because all the hard
      * work was done in segment_create :-).
      */
 
-    OPAL_OUTPUT_VERBOSE(
-        (70, opal_shmem_base_framework.framework_output,
-         "%s: %s: attach successful "
-         "(id: %d, size: %lu, name: %s)\n",
-         mca_shmem_posix_component.super.base_version.mca_type_name,
-         mca_shmem_posix_component.super.base_version.mca_component_name,
-         ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
-    );
+    OPAL_OUTPUT_VERBOSE((70, opal_shmem_base_framework.framework_output,
+                         "%s: %s: attach successful "
+                         "(id: %d, size: %lu, name: %s)\n",
+                         mca_shmem_posix_component.super.base_version.mca_type_name,
+                         mca_shmem_posix_component.super.base_version.mca_component_name,
+                         ds_buf->seg_id, (unsigned long) ds_buf->seg_size, ds_buf->seg_name));
 
     /* update returned base pointer with an offset that hides our stuff */
     return ds_buf->seg_base_addr;
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-static int
-segment_detach(opal_shmem_ds_t *ds_buf)
+static int segment_detach(opal_shmem_ds_t *ds_buf)
 {
     int rc = OPAL_SUCCESS;
 
-    OPAL_OUTPUT_VERBOSE(
-        (70, opal_shmem_base_framework.framework_output,
-         "%s: %s: detaching "
-         "(id: %d, size: %lu, name: %s)\n",
-         mca_shmem_posix_component.super.base_version.mca_type_name,
-         mca_shmem_posix_component.super.base_version.mca_component_name,
-         ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
-    );
+    OPAL_OUTPUT_VERBOSE((70, opal_shmem_base_framework.framework_output,
+                         "%s: %s: detaching "
+                         "(id: %d, size: %lu, name: %s)\n",
+                         mca_shmem_posix_component.super.base_version.mca_type_name,
+                         mca_shmem_posix_component.super.base_version.mca_component_name,
+                         ds_buf->seg_id, (unsigned long) ds_buf->seg_size, ds_buf->seg_name));
 
     if (0 != munmap(ds_buf->seg_base_addr, ds_buf->seg_size)) {
         int err = errno;
         const char *hn;
         hn = opal_gethostname();
-        opal_show_help("help-opal-shmem-posix.txt", "sys call fail", 1, hn,
-                       "munmap(2)", "", strerror(err), err);
+        opal_show_help("help-opal-shmem-posix.txt", "sys call fail", 1, hn, "munmap(2)", "",
+                       strerror(err), err);
         rc = OPAL_ERROR;
     }
     /* reset the contents of the opal_shmem_ds_t associated with this
@@ -376,24 +335,21 @@ segment_detach(opal_shmem_ds_t *ds_buf)
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-static int
-segment_unlink(opal_shmem_ds_t *ds_buf)
+static int segment_unlink(opal_shmem_ds_t *ds_buf)
 {
-    OPAL_OUTPUT_VERBOSE(
-        (70, opal_shmem_base_framework.framework_output,
-         "%s: %s: unlinking "
-         "(id: %d, size: %lu, name: %s)\n",
-         mca_shmem_posix_component.super.base_version.mca_type_name,
-         mca_shmem_posix_component.super.base_version.mca_component_name,
-         ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
-    );
+    OPAL_OUTPUT_VERBOSE((70, opal_shmem_base_framework.framework_output,
+                         "%s: %s: unlinking "
+                         "(id: %d, size: %lu, name: %s)\n",
+                         mca_shmem_posix_component.super.base_version.mca_type_name,
+                         mca_shmem_posix_component.super.base_version.mca_component_name,
+                         ds_buf->seg_id, (unsigned long) ds_buf->seg_size, ds_buf->seg_name));
 
     if (-1 == shm_unlink(ds_buf->seg_name)) {
         int err = errno;
         const char *hn;
         hn = opal_gethostname();
-        opal_show_help("help-opal-shmem-posix.txt", "sys call fail", 1, hn,
-                       "shm_unlink(2)", ds_buf->seg_name, strerror(err), err);
+        opal_show_help("help-opal-shmem-posix.txt", "sys call fail", 1, hn, "shm_unlink(2)",
+                       ds_buf->seg_name, strerror(err), err);
         return OPAL_ERROR;
     }
 
@@ -402,8 +358,7 @@ segment_unlink(opal_shmem_ds_t *ds_buf)
      * across unlinks. other information stored in flags will remain untouched.
      */
     ds_buf->seg_id = OPAL_SHMEM_DS_ID_INVALID;
-    /* note: this is only chaning the valid bit to 0. */
+    /* note: this is only changing the valid bit to 0. */
     OPAL_SHMEM_DS_INVALIDATE(ds_buf);
     return OPAL_SUCCESS;
 }
-
diff --git a/opal/mca/shmem/shmem.h b/opal/mca/shmem/shmem.h
index c10f47ed4e7..1ee669e0577 100644
--- a/opal/mca/shmem/shmem.h
+++ b/opal/mca/shmem/shmem.h
@@ -46,18 +46,16 @@
 
 #include "opal_config.h"
 
-#include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
+#include "opal/mca/mca.h"
 
 #include "opal/mca/shmem/shmem_types.h"
 
 BEGIN_C_DECLS
 
 /* ////////////////////////////////////////////////////////////////////////// */
-typedef int
-(*mca_shmem_base_component_runtime_query_fn_t)(mca_base_module_t **module,
-                                               int *priority,
-                                               const char *hint);
+typedef int (*mca_shmem_base_component_runtime_query_fn_t)(mca_base_module_t **module,
+                                                           int *priority, const char *hint);
 
 /* structure for shmem components. */
 struct opal_shmem_base_component_2_0_0_t {
@@ -70,8 +68,7 @@ struct opal_shmem_base_component_2_0_0_t {
 };
 
 /* convenience typedefs */
-typedef struct opal_shmem_base_component_2_0_0_t
-opal_shmem_base_component_2_0_0_t;
+typedef struct opal_shmem_base_component_2_0_0_t opal_shmem_base_component_2_0_0_t;
 
 typedef struct opal_shmem_base_component_2_0_0_t opal_shmem_base_component_t;
 
@@ -82,8 +79,7 @@ typedef struct opal_shmem_base_component_2_0_0_t opal_shmem_base_component_t;
  * module initialization function.
  * @return OPAL_SUCCESS on success.
  */
-typedef int
-(*opal_shmem_base_module_init_fn_t)(void);
+typedef int (*opal_shmem_base_module_init_fn_t)(void);
 
 /**
  * copy shmem data structure information pointed to by from to the structure
@@ -95,9 +91,7 @@ typedef int
  *
  * @return OPAL_SUCCESS on success.
  */
-typedef int
-(*opal_shmem_base_ds_copy_fn_t)(const opal_shmem_ds_t *from,
-                                opal_shmem_ds_t *to);
+typedef int (*opal_shmem_base_ds_copy_fn_t)(const opal_shmem_ds_t *from, opal_shmem_ds_t *to);
 
 /**
  * create a new shared memory segment and initialize members in structure
@@ -113,10 +107,8 @@ typedef int
  *
  * @return OPAL_SUCCESS on success.
  */
-typedef int
-(*opal_shmem_base_module_segment_create_fn_t)(opal_shmem_ds_t *ds_buf,
-                                              const char *file_name,
-                                              size_t size);
+typedef int (*opal_shmem_base_module_segment_create_fn_t)(opal_shmem_ds_t *ds_buf,
+                                                          const char *file_name, size_t size);
 
 /**
  * attach to an existing shared memory segment initialized by segment_create.
@@ -127,8 +119,7 @@ typedef int
  * @return        base address of shared memory segment on success. returns
  *                NULL otherwise.
  */
-typedef void *
-(*opal_shmem_base_module_segment_attach_fn_t)(opal_shmem_ds_t *ds_buf);
+typedef void *(*opal_shmem_base_module_segment_attach_fn_t)(opal_shmem_ds_t *ds_buf);
 
 /**
  * detach from an existing shared memory segment.
@@ -138,8 +129,7 @@ typedef void *
  *
  * @return OPAL_SUCCESS on success.
  */
-typedef int
-(*opal_shmem_base_module_segment_detach_fn_t)(opal_shmem_ds_t *ds_buf);
+typedef int (*opal_shmem_base_module_segment_detach_fn_t)(opal_shmem_ds_t *ds_buf);
 
 /**
  * unlink an existing shared memory segment.
@@ -149,8 +139,7 @@ typedef int
  *
  * @return OPAL_SUCCESS on success.
  */
-typedef int
-(*opal_shmem_base_module_unlink_fn_t)(opal_shmem_ds_t *ds_buf);
+typedef int (*opal_shmem_base_module_unlink_fn_t)(opal_shmem_ds_t *ds_buf);
 
 /**
  * module finalize function.  invoked by the base on the selected
@@ -162,14 +151,14 @@ typedef int (*opal_shmem_base_module_finalize_fn_t)(void);
  * structure for shmem modules
  */
 struct opal_shmem_base_module_2_0_0_t {
-    mca_base_module_t                           base;
-    opal_shmem_base_module_init_fn_t            module_init;
-    opal_shmem_base_module_segment_create_fn_t  segment_create;
-    opal_shmem_base_ds_copy_fn_t                ds_copy;
-    opal_shmem_base_module_segment_attach_fn_t  segment_attach;
-    opal_shmem_base_module_segment_detach_fn_t  segment_detach;
-    opal_shmem_base_module_unlink_fn_t          unlink;
-    opal_shmem_base_module_finalize_fn_t        module_finalize;
+    mca_base_module_t base;
+    opal_shmem_base_module_init_fn_t module_init;
+    opal_shmem_base_module_segment_create_fn_t segment_create;
+    opal_shmem_base_ds_copy_fn_t ds_copy;
+    opal_shmem_base_module_segment_attach_fn_t segment_attach;
+    opal_shmem_base_module_segment_detach_fn_t segment_detach;
+    opal_shmem_base_module_unlink_fn_t unlink;
+    opal_shmem_base_module_finalize_fn_t module_finalize;
 };
 
 /**
@@ -182,8 +171,7 @@ typedef struct opal_shmem_base_module_2_0_0_t opal_shmem_base_module_t;
  * macro for use in components that are of type shmem
  * see: opal/mca/mca.h for more information
  */
-#define OPAL_SHMEM_BASE_VERSION_2_0_0                                   \
-    OPAL_MCA_BASE_VERSION_2_1_0("shmem", 2, 0, 0)
+#define OPAL_SHMEM_BASE_VERSION_2_0_0 OPAL_MCA_BASE_VERSION_2_1_0("shmem", 2, 0, 0)
 
 END_C_DECLS
 
diff --git a/opal/mca/shmem/shmem_types.h b/opal/mca/shmem/shmem_types.h
index 0fc7e10ee56..78583c84286 100644
--- a/opal/mca/shmem/shmem_types.h
+++ b/opal/mca/shmem/shmem_types.h
@@ -67,33 +67,32 @@ BEGIN_C_DECLS
 /**
  * macro that sets all bits in flags to 0
  */
-#define OPAL_SHMEM_DS_RESET_FLAGS(ds_buf)                                      \
-do {                                                                           \
-    (ds_buf)->flags = 0x00;                                                    \
-} while (0)
+#define OPAL_SHMEM_DS_RESET_FLAGS(ds_buf) \
+    do {                                  \
+        (ds_buf)->flags = 0x00;           \
+    } while (0)
 
 /**
  * sets valid bit in flags to 1
  */
-#define OPAL_SHMEM_DS_SET_VALID(ds_buf)                                        \
-do {                                                                           \
-    (ds_buf)->flags |= OPAL_SHMEM_DS_FLAGS_VALID;                              \
-} while (0)
+#define OPAL_SHMEM_DS_SET_VALID(ds_buf)               \
+    do {                                              \
+        (ds_buf)->flags |= OPAL_SHMEM_DS_FLAGS_VALID; \
+    } while (0)
 
 /**
  * sets valid bit in flags to 0
  */
-#define OPAL_SHMEM_DS_INVALIDATE(ds_buf)                                       \
-do {                                                                           \
-    (ds_buf)->flags &= ~OPAL_SHMEM_DS_FLAGS_VALID;                             \
-} while (0)
+#define OPAL_SHMEM_DS_INVALIDATE(ds_buf)               \
+    do {                                               \
+        (ds_buf)->flags &= ~OPAL_SHMEM_DS_FLAGS_VALID; \
+    } while (0)
 
 /**
  * evaluates to 1 if the valid bit in flags is set to 1.  evaluates to 0
  * otherwise.
  */
-#define OPAL_SHMEM_DS_IS_VALID(ds_buf)                                         \
-    ( (ds_buf)->flags & OPAL_SHMEM_DS_FLAGS_VALID )
+#define OPAL_SHMEM_DS_IS_VALID(ds_buf) ((ds_buf)->flags & OPAL_SHMEM_DS_FLAGS_VALID)
 
 typedef uint8_t opal_shmem_ds_flag_t;
 
@@ -124,13 +123,12 @@ typedef struct opal_shmem_ds_t opal_shmem_ds_t;
  * opal_shmem_ds_t payload isn't viable -- due to the potential disparity
  * between the reserved buffer space and what is actually in use.
  */
-static inline size_t
-opal_shmem_sizeof_shmem_ds(const opal_shmem_ds_t *ds_bufp)
+static inline size_t opal_shmem_sizeof_shmem_ds(const opal_shmem_ds_t *ds_bufp)
 {
     char *name_base = NULL;
     size_t name_buf_offset = offsetof(opal_shmem_ds_t, seg_name);
 
-    name_base = (char *)ds_bufp + name_buf_offset;
+    name_base = (char *) ds_bufp + name_buf_offset;
 
     return name_buf_offset + strlen(name_base) + 1;
 }
diff --git a/opal/mca/shmem/sysv/Makefile.am b/opal/mca/shmem/sysv/Makefile.am
index fccadf85ca2..206573a39e4 100644
--- a/opal/mca/shmem/sysv/Makefile.am
+++ b/opal/mca/shmem/sysv/Makefile.am
@@ -13,6 +13,7 @@
 # Copyright (c) 2010-2011 Los Alamos National Security, LLC.
 #                         All rights reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -40,11 +41,11 @@ mcacomponentdir = $(opallibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_shmem_sysv_la_SOURCES = $(sources)
 mca_shmem_sysv_la_LDFLAGS = -module -avoid-version
-mca_shmem_sysv_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+mca_shmem_sysv_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 
 noinst_LTLIBRARIES = $(component_noinst)
 libmca_shmem_sysv_la_SOURCES = $(sources)
 libmca_shmem_sysv_la_LDFLAGS = -module -avoid-version
 
 # help file
-dist_opaldata_DATA = help-opal-shmem-sysv.txt
+EXTRA_DIST = help-opal-shmem-sysv.txt
diff --git a/opal/mca/shmem/sysv/shmem_sysv.h b/opal/mca/shmem/sysv/shmem_sysv.h
index b2bb3719d5f..262ec9c761c 100644
--- a/opal/mca/shmem/sysv/shmem_sysv.h
+++ b/opal/mca/shmem/sysv/shmem_sysv.h
@@ -39,8 +39,7 @@ typedef struct opal_shmem_sysv_component_t {
     int priority;
 } opal_shmem_sysv_component_t;
 
-OPAL_MODULE_DECLSPEC extern opal_shmem_sysv_component_t
-mca_shmem_sysv_component;
+OPAL_DECLSPEC extern opal_shmem_sysv_component_t mca_shmem_sysv_component;
 
 typedef struct opal_shmem_sysv_module_t {
     opal_shmem_base_module_t super;
diff --git a/opal/mca/shmem/sysv/shmem_sysv_component.c b/opal/mca/shmem/sysv/shmem_sysv_component.c
index b53fd3bed1f..33aa8f142bd 100644
--- a/opal/mca/shmem/sysv/shmem_sysv_component.c
+++ b/opal/mca/shmem/sysv/shmem_sysv_component.c
@@ -33,82 +33,76 @@
 #include "opal_config.h"
 
 #ifdef HAVE_SYS_MMAN_H
-#include <sys/mman.h>
+#    include <sys/mman.h>
 #endif /* HAVE_SYS_MMAN_H */
 #include <string.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif /* HAVE_UNISTD_H */
 
 #ifdef HAVE_SYS_IPC_H
-#include <sys/ipc.h>
+#    include <sys/ipc.h>
 #endif /* HAVE_SYS_IPC_H */
 #if HAVE_SYS_SHM_H
-#include <sys/shm.h>
+#    include <sys/shm.h>
 #endif /* HAVE_SYS_SHM_H */
 #if HAVE_SYS_STAT_H
-#include <sys/stat.h>
+#    include <sys/stat.h>
 #endif /* HAVE_SYS_STAT_H */
 
 #include "opal/constants.h"
-#include "opal/util/show_help.h"
-#include "opal/util/output.h"
-#include "opal/util/sys_limits.h"
 #include "opal/mca/shmem/base/base.h"
 #include "opal/mca/shmem/shmem.h"
+#include "opal/util/output.h"
+#include "opal/util/show_help.h"
+#include "opal/util/sys_limits.h"
 #include "shmem_sysv.h"
 
 /* public string showing the shmem ompi_sysv component version number */
-const char *opal_shmem_sysv_component_version_string =
-    "OPAL sysv shmem MCA component version " OPAL_VERSION;
+const char *opal_shmem_sysv_component_version_string
+    = "OPAL sysv shmem MCA component version " OPAL_VERSION;
 
 /* local functions */
-static int sysv_register (void);
+static int sysv_register(void);
 static int sysv_open(void);
 static int sysv_query(mca_base_module_t **module, int *priority);
-static int sysv_runtime_query(mca_base_module_t **module,
-                              int *priority,
-                              const char *hint);
+static int sysv_runtime_query(mca_base_module_t **module, int *priority, const char *hint);
 
 /* instantiate the public struct with all of our public information
  * and pointers to our public functions in it
  */
 opal_shmem_sysv_component_t mca_shmem_sysv_component = {
-    .super = {
-        /* common MCA component data */
+    .super =
         {
-            OPAL_SHMEM_BASE_VERSION_2_0_0,
-
-            /* component name and version */
-            .mca_component_name = "sysv",
-            MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
-                                  OPAL_RELEASE_VERSION),
-
-            .mca_open_component = sysv_open,
-            .mca_query_component = sysv_query,
-            .mca_register_component_params = sysv_register
-        },
-        /* MCA v2.0.0 component meta data */
-        .base_data = {
-            /* the component is checkpoint ready */
-            MCA_BASE_METADATA_PARAM_CHECKPOINT
+            /* common MCA component data */
+            {OPAL_SHMEM_BASE_VERSION_2_0_0,
+
+             /* component name and version */
+             .mca_component_name = "sysv",
+             MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
+                                   OPAL_RELEASE_VERSION),
+
+             .mca_open_component = sysv_open, .mca_query_component = sysv_query,
+             .mca_register_component_params = sysv_register},
+            /* MCA v2.0.0 component meta data */
+            .base_data =
+                {/* the component is checkpoint ready */
+                 MCA_BASE_METADATA_PARAM_CHECKPOINT},
+            .runtime_query = sysv_runtime_query,
         },
-        .runtime_query = sysv_runtime_query,
-    },
 };
 
 /* ////////////////////////////////////////////////////////////////////////// */
-static int
-sysv_register(void)
+static int sysv_register(void)
 {
     /* ////////////////////////////////////////////////////////////////////// */
     /* (default) priority - set lower than mmap's priority */
     mca_shmem_sysv_component.priority = 30;
-    (void) mca_base_component_var_register(&mca_shmem_sysv_component.super.base_version,
-                                           "priority", "Priority for the shmem sysv "
-                                           "component (default: 30)", MCA_BASE_VAR_TYPE_INT,
-                                           NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-                                           OPAL_INFO_LVL_3,
+    (void) mca_base_component_var_register(&mca_shmem_sysv_component.super.base_version, "priority",
+                                           "Priority for the shmem sysv "
+                                           "component (default: 30)",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0,
+                                           MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
                                            MCA_BASE_VAR_SCOPE_ALL_EQ,
                                            &mca_shmem_sysv_component.priority);
 
@@ -116,8 +110,7 @@ sysv_register(void)
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-static int
-sysv_open(void)
+static int sysv_open(void)
 {
     return OPAL_SUCCESS;
 }
@@ -130,12 +123,11 @@ sysv_open(void)
  *
  * @return OPAL_SUCCESS when sysv can safely be used.
  */
-static int
-sysv_runtime_query(mca_base_module_t **module, int *priority, const char *hint)
+static int sysv_runtime_query(mca_base_module_t **module, int *priority, const char *hint)
 {
-    char c     = 'j';
-    int shmid  = -1;
-    char *a    = NULL;
+    char c = 'j';
+    int shmid = -1;
+    char *a = NULL;
     char *addr = NULL;
     struct shmid_ds tmp_buff;
 
@@ -147,21 +139,18 @@ sysv_runtime_query(mca_base_module_t **module, int *priority, const char *hint)
      * don't have to perform a run-time query.
      */
     if (NULL != hint) {
-        OPAL_OUTPUT_VERBOSE(
-            (70, opal_shmem_base_framework.framework_output,
-             "shmem: sysv: runtime_query: "
-             "attempting to use runtime hint (%s)\n", hint)
-        );
+        OPAL_OUTPUT_VERBOSE((70, opal_shmem_base_framework.framework_output,
+                             "shmem: sysv: runtime_query: "
+                             "attempting to use runtime hint (%s)\n",
+                             hint));
         /* was i selected? if so, then we are done.
          * otherwise, disqualify myself.
          */
-        if (0 == strcasecmp(hint,
-            mca_shmem_sysv_component.super.base_version.mca_component_name)) {
+        if (0 == strcasecmp(hint, mca_shmem_sysv_component.super.base_version.mca_component_name)) {
             *priority = mca_shmem_sysv_component.priority;
-            *module = (mca_base_module_t *)&opal_shmem_sysv_module.super;
+            *module = (mca_base_module_t *) &opal_shmem_sysv_module.super;
             return OPAL_SUCCESS;
-        }
-        else {
+        } else {
             *priority = 0;
             *module = NULL;
             return OPAL_SUCCESS;
@@ -169,17 +158,15 @@ sysv_runtime_query(mca_base_module_t **module, int *priority, const char *hint)
     }
 
     /* if we are here, then let the run-time test games begin */
-    OPAL_OUTPUT_VERBOSE(
-        (70, opal_shmem_base_framework.framework_output,
-         "shmem: sysv: runtime_query: NO HINT PROVIDED:"
-         "starting run-time test...\n")
-    );
-
-    if (-1 == (shmid = shmget(IPC_PRIVATE, (size_t)(opal_getpagesize()),
-                              IPC_CREAT | IPC_EXCL | S_IRWXU ))) {
+    OPAL_OUTPUT_VERBOSE((70, opal_shmem_base_framework.framework_output,
+                         "shmem: sysv: runtime_query: NO HINT PROVIDED:"
+                         "starting run-time test...\n"));
+
+    if (-1
+        == (shmid = shmget(IPC_PRIVATE, (size_t)(opal_getpagesize()),
+                           IPC_CREAT | IPC_EXCL | S_IRWXU))) {
         goto out;
-    }
-    else if ((void *)-1 == (addr = shmat(shmid, NULL, 0))) {
+    } else if ((void *) -1 == (addr = shmat(shmid, NULL, 0))) {
         goto out;
     }
 
@@ -189,29 +176,26 @@ sysv_runtime_query(mca_base_module_t **module, int *priority, const char *hint)
 
     if (-1 == shmctl(shmid, IPC_RMID, NULL)) {
         goto out;
-    }
-    else if (-1 == shmctl(shmid, IPC_STAT, &tmp_buff)) {
+    } else if (-1 == shmctl(shmid, IPC_STAT, &tmp_buff)) {
         goto out;
     }
     /* all is well - rainbows and butterflies */
     else {
         *priority = mca_shmem_sysv_component.priority;
-        *module = (mca_base_module_t *)&opal_shmem_sysv_module.super;
+        *module = (mca_base_module_t *) &opal_shmem_sysv_module.super;
     }
 
 out:
-    if (NULL != addr && (char *)-1 != addr) {
+    if (NULL != addr && (char *) -1 != addr) {
         shmdt(addr);
     }
     return OPAL_SUCCESS;
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-static int
-sysv_query(mca_base_module_t **module, int *priority)
+static int sysv_query(mca_base_module_t **module, int *priority)
 {
     *priority = mca_shmem_sysv_component.priority;
-    *module = (mca_base_module_t *)&opal_shmem_sysv_module.super;
+    *module = (mca_base_module_t *) &opal_shmem_sysv_module.super;
     return OPAL_SUCCESS;
 }
-
diff --git a/opal/mca/shmem/sysv/shmem_sysv_module.c b/opal/mca/shmem/sysv/shmem_sysv_module.c
index 64b56aa56f7..314cf6ddbaa 100644
--- a/opal/mca/shmem/sysv/shmem_sysv_module.c
+++ b/opal/mca/shmem/sysv/shmem_sysv_module.c
@@ -27,39 +27,39 @@
 
 #include <errno.h>
 #ifdef HAVE_FCNTL_H
-#include <fcntl.h>
-#endif  /* HAVE_FCNTL_H */
+#    include <fcntl.h>
+#endif /* HAVE_FCNTL_H */
 #ifdef HAVE_SYS_MMAN_H
-#include <sys/mman.h>
+#    include <sys/mman.h>
 #endif /* HAVE_SYS_MMAN_H */
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif /* HAVE_UNISTD_H */
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif /* HAVE_SYS_TYPES_H */
 #ifdef HAVE_SYS_IPC_H
-#include <sys/ipc.h>
+#    include <sys/ipc.h>
 #endif /* HAVE_SYS_IPC_H */
 #if HAVE_SYS_SHM_H
-#include <sys/shm.h>
+#    include <sys/shm.h>
 #endif /* HAVE_SYS_SHM_H */
 #if HAVE_SYS_STAT_H
-#include <sys/stat.h>
+#    include <sys/stat.h>
 #endif /* HAVE_SYS_STAT_H */
 #include <string.h>
 #ifdef HAVE_NETDB_H
-#include <netdb.h>
+#    include <netdb.h>
 #endif /* HAVE_NETDB_H */
 
-#include "opal/runtime/opal.h"
 #include "opal/constants.h"
-#include "opal_stdint.h"
+#include "opal/mca/shmem/base/base.h"
+#include "opal/mca/shmem/shmem.h"
+#include "opal/runtime/opal.h"
 #include "opal/util/output.h"
 #include "opal/util/path.h"
 #include "opal/util/show_help.h"
-#include "opal/mca/shmem/shmem.h"
-#include "opal/mca/shmem/base/base.h"
+#include "opal_stdint.h"
 
 #include "shmem_sysv.h"
 
@@ -67,42 +67,28 @@
 
 /* ////////////////////////////////////////////////////////////////////////// */
 /* local functions */
-static int
-module_init(void);
+static int module_init(void);
 
-static int
-segment_create(opal_shmem_ds_t *ds_buf,
-               const char *file_name,
-               size_t size);
+static int segment_create(opal_shmem_ds_t *ds_buf, const char *file_name, size_t size);
 
-static int
-ds_copy(const opal_shmem_ds_t *from,
-        opal_shmem_ds_t *to);
+static int ds_copy(const opal_shmem_ds_t *from, opal_shmem_ds_t *to);
 
-static void *
-segment_attach(opal_shmem_ds_t *ds_buf);
+static void *segment_attach(opal_shmem_ds_t *ds_buf);
 
-static int
-segment_detach(opal_shmem_ds_t *ds_buf);
+static int segment_detach(opal_shmem_ds_t *ds_buf);
 
-static int
-segment_unlink(opal_shmem_ds_t *ds_buf);
+static int segment_unlink(opal_shmem_ds_t *ds_buf);
 
-static int
-module_finalize(void);
+static int module_finalize(void);
 
 /* sysv shmem module */
-opal_shmem_sysv_module_t opal_shmem_sysv_module = {
-    .super = {
-        .module_init = module_init,
-        .segment_create = segment_create,
-        .ds_copy = ds_copy,
-        .segment_attach = segment_attach,
-        .segment_detach = segment_detach,
-        .unlink = segment_unlink,
-        .module_finalize = module_finalize
-    }
-};
+opal_shmem_sysv_module_t opal_shmem_sysv_module = {.super = {.module_init = module_init,
+                                                             .segment_create = segment_create,
+                                                             .ds_copy = ds_copy,
+                                                             .segment_attach = segment_attach,
+                                                             .segment_detach = segment_detach,
+                                                             .unlink = segment_unlink,
+                                                             .module_finalize = module_finalize}};
 
 /* ////////////////////////////////////////////////////////////////////////// */
 /* private utility functions */
@@ -112,70 +98,57 @@ opal_shmem_sysv_module_t opal_shmem_sysv_module = {
 /**
  * completely resets the contents of *ds_buf
  */
-static inline void
-shmem_ds_reset(opal_shmem_ds_t *ds_buf)
+static inline void shmem_ds_reset(opal_shmem_ds_t *ds_buf)
 {
     /* don't print ds_buf info here, as we may be printing garbage. */
-    OPAL_OUTPUT_VERBOSE(
-        (70, opal_shmem_base_framework.framework_output,
-         "%s: %s: shmem_ds_resetting\n",
-         mca_shmem_sysv_component.super.base_version.mca_type_name,
-         mca_shmem_sysv_component.super.base_version.mca_component_name)
-    );
+    OPAL_OUTPUT_VERBOSE((70, opal_shmem_base_framework.framework_output,
+                         "%s: %s: shmem_ds_resetting\n",
+                         mca_shmem_sysv_component.super.base_version.mca_type_name,
+                         mca_shmem_sysv_component.super.base_version.mca_component_name));
 
     ds_buf->seg_cpid = 0;
     OPAL_SHMEM_DS_RESET_FLAGS(ds_buf);
     ds_buf->seg_id = OPAL_SHMEM_DS_ID_INVALID;
     ds_buf->seg_size = 0;
     memset(ds_buf->seg_name, '\0', OPAL_PATH_MAX);
-    ds_buf->seg_base_addr = (unsigned char *)-1;
+    ds_buf->seg_base_addr = (unsigned char *) -1;
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-static int
-module_init(void)
+static int module_init(void)
 {
     /* nothing to do */
     return OPAL_SUCCESS;
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-static int
-module_finalize(void)
+static int module_finalize(void)
 {
     /* nothing to do */
     return OPAL_SUCCESS;
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-static int
-ds_copy(const opal_shmem_ds_t *from,
-        opal_shmem_ds_t *to)
+static int ds_copy(const opal_shmem_ds_t *from, opal_shmem_ds_t *to)
 {
     memcpy(to, from, sizeof(opal_shmem_ds_t));
 
-    OPAL_OUTPUT_VERBOSE(
-        (70, opal_shmem_base_framework.framework_output,
-         "%s: %s: ds_copy complete "
-         "from: (id: %d, size: %lu, "
-         "name: %s flags: 0x%02x) "
-         "to: (id: %d, size: %lu, "
-         "name: %s flags: 0x%02x)\n",
-         mca_shmem_sysv_component.super.base_version.mca_type_name,
-         mca_shmem_sysv_component.super.base_version.mca_component_name,
-         from->seg_id, (unsigned long)from->seg_size, from->seg_name,
-         from->flags, to->seg_id, (unsigned long)to->seg_size, to->seg_name,
-         to->flags)
-    );
+    OPAL_OUTPUT_VERBOSE((70, opal_shmem_base_framework.framework_output,
+                         "%s: %s: ds_copy complete "
+                         "from: (id: %d, size: %lu, "
+                         "name: %s flags: 0x%02x) "
+                         "to: (id: %d, size: %lu, "
+                         "name: %s flags: 0x%02x)\n",
+                         mca_shmem_sysv_component.super.base_version.mca_type_name,
+                         mca_shmem_sysv_component.super.base_version.mca_component_name,
+                         from->seg_id, (unsigned long) from->seg_size, from->seg_name, from->flags,
+                         to->seg_id, (unsigned long) to->seg_size, to->seg_name, to->flags));
 
     return OPAL_SUCCESS;
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-static int
-segment_create(opal_shmem_ds_t *ds_buf,
-               const char *file_name,
-               size_t size)
+static int segment_create(opal_shmem_ds_t *ds_buf, const char *file_name, size_t size)
 {
     int rc = OPAL_SUCCESS;
     pid_t my_pid = getpid();
@@ -189,23 +162,22 @@ segment_create(opal_shmem_ds_t *ds_buf,
      */
 
     /* create a new shared memory segment and save the shmid. */
-    if (-1 == (ds_buf->seg_id = shmget(IPC_PRIVATE, size,
-                                       IPC_CREAT | IPC_EXCL | S_IRWXU))) {
+    if (-1 == (ds_buf->seg_id = shmget(IPC_PRIVATE, size, IPC_CREAT | IPC_EXCL | S_IRWXU))) {
         int err = errno;
         const char *hn;
         hn = opal_gethostname();
-        opal_show_help("help-opal-shmem-sysv.txt", "sys call fail", 1, hn,
-                       "shmget(2)", "", strerror(err), err);
+        opal_show_help("help-opal-shmem-sysv.txt", "sys call fail", 1, hn, "shmget(2)", "",
+                       strerror(err), err);
         rc = OPAL_ERROR;
         goto out;
     }
-    /* attach to the sement */
-    else if ((void *)-1 == (segment = shmat(ds_buf->seg_id, NULL, 0))) {
+    /* attach to the segment */
+    else if ((void *) -1 == (segment = shmat(ds_buf->seg_id, NULL, 0))) {
         int err = errno;
         const char *hn;
         hn = opal_gethostname();
-        opal_show_help("help-opal-shmem-sysv.txt", "sys call fail", 1, hn,
-                       "shmat(2)", "", strerror(err), err);
+        opal_show_help("help-opal-shmem-sysv.txt", "sys call fail", 1, hn, "shmat(2)", "",
+                       strerror(err), err);
         shmctl(ds_buf->seg_id, IPC_RMID, NULL);
         rc = OPAL_ERROR;
         goto out;
@@ -218,8 +190,8 @@ segment_create(opal_shmem_ds_t *ds_buf,
         int err = errno;
         const char *hn;
         hn = opal_gethostname();
-        opal_show_help("help-opal-shmem-sysv.txt", "sys call fail", 1, hn,
-                       "shmctl(2)", "", strerror(err), err);
+        opal_show_help("help-opal-shmem-sysv.txt", "sys call fail", 1, hn, "shmctl(2)", "",
+                       strerror(err), err);
         rc = OPAL_ERROR;
         goto out;
     }
@@ -229,7 +201,7 @@ segment_create(opal_shmem_ds_t *ds_buf,
         /* -- initialize the contents of opal_shmem_ds_t -- */
         ds_buf->seg_cpid = my_pid;
         ds_buf->seg_size = size;
-        ds_buf->seg_base_addr = (unsigned char *)segment;
+        ds_buf->seg_base_addr = (unsigned char *) segment;
 
         /* notice that we are not setting ds_buf->name here. sysv doesn't use
          * it, so don't worry about it - shmem_ds_reset took care of
@@ -239,24 +211,22 @@ segment_create(opal_shmem_ds_t *ds_buf,
         /* set "valid" bit because setment creation was successful */
         OPAL_SHMEM_DS_SET_VALID(ds_buf);
 
-        OPAL_OUTPUT_VERBOSE(
-            (70, opal_shmem_base_framework.framework_output,
-             "%s: %s: create successful "
-             "(id: %d, size: %lu, name: %s)\n",
-             mca_shmem_sysv_component.super.base_version.mca_type_name,
-             mca_shmem_sysv_component.super.base_version.mca_component_name,
-             ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
-        );
+        OPAL_OUTPUT_VERBOSE((70, opal_shmem_base_framework.framework_output,
+                             "%s: %s: create successful "
+                             "(id: %d, size: %lu, name: %s)\n",
+                             mca_shmem_sysv_component.super.base_version.mca_type_name,
+                             mca_shmem_sysv_component.super.base_version.mca_component_name,
+                             ds_buf->seg_id, (unsigned long) ds_buf->seg_size, ds_buf->seg_name));
     }
 
 out:
-    /* an error occured, so invalidate the shmem object and release any
+    /* an error occurred, so invalidate the shmem object and release any
      * allocated resources.
      */
     if (OPAL_SUCCESS != rc) {
         /* best effort to delete the segment. */
-        if ((void *)-1 != segment) {
-            shmdt((char*)segment);
+        if ((void *) -1 != segment) {
+            shmdt((char *) segment);
         }
         shmctl(ds_buf->seg_id, IPC_RMID, NULL);
 
@@ -270,19 +240,17 @@ segment_create(opal_shmem_ds_t *ds_buf,
 /**
  * segment_attach can only be called after a successful call to segment_create
  */
-static void *
-segment_attach(opal_shmem_ds_t *ds_buf)
+static void *segment_attach(opal_shmem_ds_t *ds_buf)
 {
     pid_t my_pid = getpid();
 
     if (my_pid != ds_buf->seg_cpid) {
-        if ((void *)-1 == (ds_buf->seg_base_addr = shmat(ds_buf->seg_id, NULL,
-                                                         0))) {
+        if ((void *) -1 == (ds_buf->seg_base_addr = shmat(ds_buf->seg_id, NULL, 0))) {
             int err = errno;
             const char *hn;
             hn = opal_gethostname();
-            opal_show_help("help-opal-shmem-sysv.txt", "sys call fail", 1, hn,
-                           "shmat(2)", "", strerror(err), err);
+            opal_show_help("help-opal-shmem-sysv.txt", "sys call fail", 1, hn, "shmat(2)", "",
+                           strerror(err), err);
             shmctl(ds_buf->seg_id, IPC_RMID, NULL);
             return NULL;
         }
@@ -291,40 +259,35 @@ segment_attach(opal_shmem_ds_t *ds_buf)
      * work was done in segment_create :-).
      */
 
-    OPAL_OUTPUT_VERBOSE(
-        (70, opal_shmem_base_framework.framework_output,
-         "%s: %s: attach successful "
-         "(id: %d, size: %lu, name: %s)\n",
-         mca_shmem_sysv_component.super.base_version.mca_type_name,
-         mca_shmem_sysv_component.super.base_version.mca_component_name,
-         ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
-    );
+    OPAL_OUTPUT_VERBOSE((70, opal_shmem_base_framework.framework_output,
+                         "%s: %s: attach successful "
+                         "(id: %d, size: %lu, name: %s)\n",
+                         mca_shmem_sysv_component.super.base_version.mca_type_name,
+                         mca_shmem_sysv_component.super.base_version.mca_component_name,
+                         ds_buf->seg_id, (unsigned long) ds_buf->seg_size, ds_buf->seg_name));
 
     /* update returned base pointer with an offset that hides our stuff */
     return ds_buf->seg_base_addr;
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-static int
-segment_detach(opal_shmem_ds_t *ds_buf)
+static int segment_detach(opal_shmem_ds_t *ds_buf)
 {
     int rc = OPAL_SUCCESS;
 
-    OPAL_OUTPUT_VERBOSE(
-        (70, opal_shmem_base_framework.framework_output,
-         "%s: %s: detaching "
-         "(id: %d, size: %lu, name: %s)\n",
-         mca_shmem_sysv_component.super.base_version.mca_type_name,
-         mca_shmem_sysv_component.super.base_version.mca_component_name,
-         ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
-    );
+    OPAL_OUTPUT_VERBOSE((70, opal_shmem_base_framework.framework_output,
+                         "%s: %s: detaching "
+                         "(id: %d, size: %lu, name: %s)\n",
+                         mca_shmem_sysv_component.super.base_version.mca_type_name,
+                         mca_shmem_sysv_component.super.base_version.mca_component_name,
+                         ds_buf->seg_id, (unsigned long) ds_buf->seg_size, ds_buf->seg_name));
 
-    if (0 != shmdt((char*)ds_buf->seg_base_addr)) {
+    if (0 != shmdt((char *) ds_buf->seg_base_addr)) {
         int err = errno;
         const char *hn;
         hn = opal_gethostname();
-        opal_show_help("help-opal-shmem-sysv.txt", "sys call fail", 1, hn,
-                       "shmdt(2)", "", strerror(err), err);
+        opal_show_help("help-opal-shmem-sysv.txt", "sys call fail", 1, hn, "shmdt(2)", "",
+                       strerror(err), err);
         rc = OPAL_ERROR;
     }
 
@@ -336,27 +299,23 @@ segment_detach(opal_shmem_ds_t *ds_buf)
 }
 
 /* ////////////////////////////////////////////////////////////////////////// */
-static int
-segment_unlink(opal_shmem_ds_t *ds_buf)
+static int segment_unlink(opal_shmem_ds_t *ds_buf)
 {
     /* not much unlink work needed for sysv */
 
-    OPAL_OUTPUT_VERBOSE(
-        (70, opal_shmem_base_framework.framework_output,
-         "%s: %s: unlinking "
-         "(id: %d, size: %lu, name: %s)\n",
-         mca_shmem_sysv_component.super.base_version.mca_type_name,
-         mca_shmem_sysv_component.super.base_version.mca_component_name,
-         ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
-    );
+    OPAL_OUTPUT_VERBOSE((70, opal_shmem_base_framework.framework_output,
+                         "%s: %s: unlinking "
+                         "(id: %d, size: %lu, name: %s)\n",
+                         mca_shmem_sysv_component.super.base_version.mca_type_name,
+                         mca_shmem_sysv_component.super.base_version.mca_component_name,
+                         ds_buf->seg_id, (unsigned long) ds_buf->seg_size, ds_buf->seg_name));
 
     /* don't completely reset the opal_shmem_ds_t.  in particular, only reset
      * the id and flip the invalid bit.  size and name values will remain valid
      * across unlinks. other information stored in flags will remain untouched.
      */
     ds_buf->seg_id = OPAL_SHMEM_DS_ID_INVALID;
-    /* note: this is only chaning the valid bit to 0. */
+    /* note: this is only changing the valid bit to 0. */
     OPAL_SHMEM_DS_INVALIDATE(ds_buf);
     return OPAL_SUCCESS;
 }
-
diff --git a/opal/mca/smsc/Makefile.am b/opal/mca/smsc/Makefile.am
new file mode 100644
index 00000000000..975eee7705f
--- /dev/null
+++ b/opal/mca/smsc/Makefile.am
@@ -0,0 +1,38 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2021      Google, LLC. All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# main library setup
+noinst_LTLIBRARIES = libmca_smsc.la
+libmca_smsc_la_SOURCES =
+
+# local files
+headers = smsc.h
+libmca_smsc_la_SOURCES += $(headers)
+
+# Conditionally install the header files
+if WANT_INSTALL_HEADERS
+opaldir = $(opalincludedir)/$(subdir)
+nobase_opal_HEADERS = $(headers)
+endif
+
+include base/Makefile.am
+
+distclean-local:
+	rm -f base/static-components.h
diff --git a/opal/mca/smsc/accelerator/Makefile.am b/opal/mca/smsc/accelerator/Makefile.am
new file mode 100644
index 00000000000..145fea1f53c
--- /dev/null
+++ b/opal/mca/smsc/accelerator/Makefile.am
@@ -0,0 +1,58 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2009 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2009-2014 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights
+#                         reserved.
+# Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2020-2021 Google, LLC. All rights reserved.
+# Copyright (c) 2024      Advanced Micro Devices, Inc. All Rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+EXTRA_DIST = post_configure.sh
+
+AM_CPPFLAGS = $(smsc_accelerator_CPPFLAGS)
+
+
+libmca_smsc_accelerator_la_sources = \
+    smsc_accelerator_component.c \
+    smsc_accelerator_module.c \
+    smsc_accelerator_internal.h \
+    smsc_accelerator.h
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_opal_smsc_accelerator_DSO
+component_noinst =
+component_install = mca_smsc_accelerator.la
+else
+component_noinst = libmca_smsc_accelerator.la
+component_install =
+endif
+
+mcacomponentdir = $(opallibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_smsc_accelerator_la_SOURCES = $(libmca_smsc_accelerator_la_sources)
+mca_smsc_accelerator_la_LDFLAGS = -module -avoid-version $(smsc_accelerator_LDFLAGS)
+mca_smsc_accelerator_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
+	$(smsc_accelerator_LIBS)
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_smsc_accelerator_la_SOURCES = $(libmca_smsc_accelerator_la_sources)
+libmca_smsc_accelerator_la_LIBADD = $(smsc_accelerator_LIBS)
+libmca_smsc_accelerator_la_LDFLAGS = -module -avoid-version $(smsc_accelerator_LDFLAGS)
diff --git a/opal/mca/smsc/accelerator/owner.txt b/opal/mca/smsc/accelerator/owner.txt
new file mode 100644
index 00000000000..d44b1deaf99
--- /dev/null
+++ b/opal/mca/smsc/accelerator/owner.txt
@@ -0,0 +1,7 @@
+#
+# owner/status file
+# owner: institution that is responsible for this package
+# status: e.g. active, maintenance, unmaintained
+#
+owner: AMD
+status: active
diff --git a/opal/mca/smsc/accelerator/post_configure.sh b/opal/mca/smsc/accelerator/post_configure.sh
new file mode 100644
index 00000000000..eaa00df0d92
--- /dev/null
+++ b/opal/mca/smsc/accelerator/post_configure.sh
@@ -0,0 +1 @@
+DIRECT_CALL_HEADER="opal/mca/smsc/accelerator/smsc_accelerator.h"
diff --git a/opal/mca/smsc/accelerator/smsc_accelerator.h b/opal/mca/smsc/accelerator/smsc_accelerator.h
new file mode 100644
index 00000000000..c7ed72b029e
--- /dev/null
+++ b/opal/mca/smsc/accelerator/smsc_accelerator.h
@@ -0,0 +1,34 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2024      Advanced Micro Devices, Inc. All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef OPAL_MCA_SMSC_ACCELERATOR_H
+#define OPAL_MCA_SMSC_ACCELERATOR_H
+
+#include "opal_config.h"
+
+#include "opal/mca/smsc/smsc.h"
+
+mca_smsc_endpoint_t *mca_smsc_accelerator_get_endpoint(opal_proc_t *peer_proc);
+void mca_smsc_accelerator_return_endpoint(mca_smsc_endpoint_t *endpoint);
+
+int mca_smsc_accelerator_copy_to(mca_smsc_endpoint_t *endpoint, void *local_address, void *remote_address,
+                           size_t size, void *reg_handle);
+
+int mca_smsc_accelerator_copy_from(mca_smsc_endpoint_t *endpoint, void *local_address,
+                             void *remote_address, size_t size, void *reg_handle);
+
+void *mca_smsc_accelerator_map_peer_region(mca_smsc_endpoint_t *endpoint, uint64_t flags,
+                                     void *remote_ptr, size_t size, void **local_ptr);
+void mca_smsc_accelerator_unmap_peer_region(void *ctx);
+
+void *mca_smsc_accelerator_register_region(void *local_address, size_t size);
+void mca_smsc_accelerator_deregister_region(void *reg_data);
+
+#endif /* OPAL_MCA_SMSC_ACCELERATOR__H */
diff --git a/opal/mca/smsc/accelerator/smsc_accelerator_component.c b/opal/mca/smsc/accelerator/smsc_accelerator_component.c
new file mode 100644
index 00000000000..2bb276a98ba
--- /dev/null
+++ b/opal/mca/smsc/accelerator/smsc_accelerator_component.c
@@ -0,0 +1,106 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2024      Advanced Micro Devices, Inc. All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "opal_config.h"
+
+#include "opal/mca/smsc/base/base.h"
+#include "opal/mca/smsc/accelerator/smsc_accelerator_internal.h"
+#include "opal/mca/accelerator/accelerator.h"
+#include "opal/mca/accelerator/base/base.h"
+
+static int mca_smsc_accelerator_component_register(void);
+static int mca_smsc_accelerator_component_open(void);
+static int mca_smsc_accelerator_component_close(void);
+static int mca_smsc_accelerator_component_query(void);
+static mca_smsc_module_t *mca_smsc_accelerator_component_enable(void);
+
+#define MCA_SMSC_ACCELERATOR_DEFAULT_PRIORITY 0
+static const int mca_smsc_accelerator_default_priority = MCA_SMSC_ACCELERATOR_DEFAULT_PRIORITY;
+
+mca_smsc_accelerator_component_t mca_smsc_accelerator_component = {
+    .super = {
+        .smsc_version = {
+            MCA_SMSC_DEFAULT_VERSION("accelerator"),
+            .mca_open_component = mca_smsc_accelerator_component_open,
+            .mca_close_component = mca_smsc_accelerator_component_close,
+            .mca_register_component_params = mca_smsc_accelerator_component_register,
+        },
+        .priority = MCA_SMSC_ACCELERATOR_DEFAULT_PRIORITY,
+        .query = mca_smsc_accelerator_component_query,
+        .enable = mca_smsc_accelerator_component_enable,
+    },
+};
+
+static int mca_smsc_accelerator_component_register(void)
+{
+    mca_smsc_base_register_default_params(&mca_smsc_accelerator_component.super,
+                                          mca_smsc_accelerator_default_priority);
+    return OPAL_SUCCESS;
+}
+
+static int mca_smsc_accelerator_component_open(void)
+{
+    return OPAL_SUCCESS;
+}
+
+static int mca_smsc_accelerator_component_close(void)
+{
+    if (mca_smsc_accelerator_module.rcache) {
+        (void) mca_rcache_base_module_destroy(mca_smsc_accelerator_module.rcache);
+        mca_smsc_accelerator_module.rcache = NULL;
+    }
+
+    return OPAL_SUCCESS;
+}
+
+static int mca_smsc_accelerator_component_query(void)
+{
+    if (0 == strcmp(opal_accelerator_base_selected_component.base_version.mca_component_name,
+                    "null")) {
+        opal_output_verbose(10, opal_smsc_base_framework.framework_output,
+                          "smsc:accelerator:component_query: accelerator component is null: disqualifying myself");
+        return OPAL_ERROR;
+    }
+
+    if (!opal_accelerator.is_ipc_enabled()) {
+        opal_output_verbose(10, opal_smsc_base_framework.framework_output,
+                          "smsc:accelerator:component_query: accelerator component does not have support for IPC operations: disqualifying myself");
+        return OPAL_ERROR;
+    }
+
+    if (1 > mca_smsc_accelerator_component.super.priority) {
+        return OPAL_ERROR;
+    }
+
+    return OPAL_SUCCESS;
+}
+
+static mca_smsc_module_t *mca_smsc_accelerator_component_enable(void)
+{
+    if (0 > mca_smsc_accelerator_component.super.priority) {
+        return NULL;
+    }
+
+    mca_rcache_base_resources_t rcache_resources;
+    mca_smsc_accelerator_module.rcache = mca_rcache_base_module_create("gpusm", NULL, &rcache_resources);
+    if (NULL == mca_smsc_accelerator_module.rcache) {
+        return NULL;
+    }
+
+    /* Not set. Will initialize later */
+    mca_smsc_accelerator_module.device_id = MCA_ACCELERATOR_NO_DEVICE_ID;
+
+    mca_smsc_accelerator_module.prev_smsc = mca_smsc;
+    if ((NULL != mca_smsc_accelerator_module.prev_smsc) &&
+        (mca_smsc_accelerator_module.prev_smsc->features & MCA_SMSC_FEATURE_CAN_MAP)) {
+            mca_smsc_accelerator_module.super.features |= MCA_SMSC_FEATURE_CAN_MAP;
+    }
+
+    return &mca_smsc_accelerator_module.super;
+}
diff --git a/opal/mca/smsc/accelerator/smsc_accelerator_internal.h b/opal/mca/smsc/accelerator/smsc_accelerator_internal.h
new file mode 100644
index 00000000000..1c2ee329438
--- /dev/null
+++ b/opal/mca/smsc/accelerator/smsc_accelerator_internal.h
@@ -0,0 +1,69 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2021      Google, Inc. All rights reserved.
+ * Copyright (c) 2024      Advanced Micro Devices, Inc. All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef OPAL_MCA_SMSC_ACCELERATOR_INTERNAL_H
+#define OPAL_MCA_SMSC_ACCELERATOR_INTERNAL_H
+
+#include "opal_config.h"
+
+#include "opal/mca/rcache/base/base.h"
+#include "opal/mca/rcache/rcache.h"
+#include "opal/mca/smsc/accelerator/smsc_accelerator.h"
+#include "opal/mca/accelerator/accelerator.h"
+#include "opal/include/opal/opal_gpu.h"
+
+#define SMSC_ACCELERATOR_HANDLE_SIZE IPC_MAX_HANDLE_SIZE
+struct mca_smsc_accelerator_registration_data_t {
+    uint64_t base_addr;
+    union {
+        uint8_t accelerator[SMSC_ACCELERATOR_HANDLE_SIZE];
+        void* host;
+    } handle;
+};
+typedef struct mca_smsc_accelerator_registration_data_t mca_smsc_accelerator_registration_data_t;
+
+struct mca_smsc_accelerator_registration_handle_t {
+    mca_opal_gpu_reg_t *gpu_reg;                   // contains mca_rcache_base_registration_t base
+    mca_smsc_accelerator_registration_data_t data;
+};
+typedef struct mca_smsc_accelerator_registration_handle_t mca_smsc_accelerator_registration_handle_t;
+OBJ_CLASS_DECLARATION(mca_smsc_accelerator_registration_handle_t);
+
+#define MCA_SMSC_ACCELERATOR_REG_DATA_TO_HANDLE(data_ptr)                                            \
+    ((mca_smsc_accelerator_registration_handle_t *) ((uintptr_t) data_ptr                            \
+                                              - offsetof(mca_smsc_accelerator_registration_handle_t, \
+                                                         data)))
+
+struct mca_smsc_accelerator_endpoint_t {
+    mca_smsc_endpoint_t super;
+    mca_smsc_endpoint_t *prev_endpoint;
+    mca_rcache_base_module_t *rcache;
+};
+typedef struct mca_smsc_accelerator_endpoint_t mca_smsc_accelerator_endpoint_t;
+OBJ_CLASS_DECLARATION(mca_smsc_accelerator_endpoint_t);
+
+struct mca_smsc_accelerator_component_t {
+    mca_smsc_component_t super;
+};
+typedef struct mca_smsc_accelerator_component_t mca_smsc_accelerator_component_t;
+
+struct mca_smsc_accelerator_module_t {
+    mca_smsc_module_t super;
+    mca_smsc_module_t *prev_smsc;
+    mca_rcache_base_module_t *rcache;
+    int device_id;
+};
+typedef struct mca_smsc_accelerator_module_t mca_smsc_accelerator_module_t;
+
+extern mca_smsc_accelerator_module_t mca_smsc_accelerator_module;
+extern mca_smsc_accelerator_component_t mca_smsc_accelerator_component;
+
+#endif /* OPAL_MCA_SMSC_ACCELERATOR_INTERNAL_H */
diff --git a/opal/mca/smsc/accelerator/smsc_accelerator_module.c b/opal/mca/smsc/accelerator/smsc_accelerator_module.c
new file mode 100644
index 00000000000..68148a48c4a
--- /dev/null
+++ b/opal/mca/smsc/accelerator/smsc_accelerator_module.c
@@ -0,0 +1,231 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2024      Advanced Micro Devices, Inc. All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "opal_config.h"
+
+#include "opal/include/opal/align.h"
+#include "opal/mca/rcache/rcache.h"
+#include "opal/mca/rcache/base/base.h"
+#include "opal/mca/accelerator/accelerator.h"
+#include "opal/mca/accelerator/base/base.h"
+#include "opal/mca/smsc/base/base.h"
+#include "opal/mca/smsc/accelerator/smsc_accelerator.h"
+#include "opal/mca/smsc/accelerator/smsc_accelerator_internal.h"
+#include "opal/include/opal/opal_gpu.h"
+
+OBJ_CLASS_INSTANCE(mca_smsc_accelerator_endpoint_t, opal_object_t, NULL, NULL);
+
+mca_smsc_endpoint_t *mca_smsc_accelerator_get_endpoint(opal_proc_t *peer_proc)
+{
+    mca_smsc_accelerator_endpoint_t *endpoint = OBJ_NEW(mca_smsc_accelerator_endpoint_t);
+    if (OPAL_UNLIKELY(NULL == endpoint)) {
+        return NULL;
+    }
+
+    if (NULL != mca_smsc_accelerator_module.prev_smsc) {
+        endpoint->prev_endpoint = mca_smsc_accelerator_module.prev_smsc->get_endpoint(peer_proc);
+    }
+
+    endpoint->rcache = mca_rcache_base_module_create("rgpusm", NULL, NULL);
+
+    return &endpoint->super;
+}
+
+void mca_smsc_accelerator_return_endpoint(mca_smsc_endpoint_t *endpoint)
+{
+    mca_smsc_accelerator_endpoint_t *ep = (mca_smsc_accelerator_endpoint_t *)endpoint;
+
+    if ((NULL != mca_smsc_accelerator_module.prev_smsc) &&
+        (NULL != ep->prev_endpoint)) {
+        mca_smsc_accelerator_module.prev_smsc->return_endpoint(ep->prev_endpoint);
+    }
+
+    mca_rcache_base_module_destroy(ep->rcache);
+    OBJ_RELEASE(endpoint);
+}
+
+void *mca_smsc_accelerator_map_peer_region(mca_smsc_endpoint_t *endpoint, uint64_t flags,
+                                     void *remote_ptr, size_t size, void **local_ptr)
+{
+    void* result = NULL;
+    mca_smsc_accelerator_endpoint_t *ep = (mca_smsc_accelerator_endpoint_t *)endpoint;
+
+    if (NULL != mca_smsc_accelerator_module.prev_smsc &&
+        (mca_smsc_accelerator_module.prev_smsc->features & MCA_SMSC_FEATURE_CAN_MAP)) {
+        result = mca_smsc_accelerator_module.prev_smsc->map_peer_region(ep->prev_endpoint,
+                                                                        flags, remote_ptr, size, local_ptr);
+    }
+    return result;
+}
+
+void mca_smsc_accelerator_unmap_peer_region(void *ctx)
+{
+    if (NULL != mca_smsc_accelerator_module.prev_smsc &&
+        (mca_smsc_accelerator_module.prev_smsc->features & MCA_SMSC_FEATURE_CAN_MAP)) {
+        mca_smsc_accelerator_module.prev_smsc->unmap_peer_region(ctx);
+    }
+}
+
+static int mca_smsc_accelerator_copy (mca_smsc_endpoint_t *endpoint, void *local_address, void *remote_address,
+                                      size_t size, void *reg_handle, bool is_remote_write)
+{
+    mca_smsc_accelerator_endpoint_t *ep = (mca_smsc_accelerator_endpoint_t *)endpoint;
+    mca_smsc_accelerator_registration_data_t *reg = (mca_smsc_accelerator_registration_data_t *)reg_handle;
+    int ret = OPAL_SUCCESS;
+    void *remote_memory_address;
+    mca_opal_gpu_reg_t rget_reg;
+    mca_opal_gpu_reg_t *reg_ptr = &rget_reg;
+    size_t offset;
+
+    memset(&rget_reg, 0, sizeof(rget_reg));
+    memcpy(&rget_reg.data.ipcHandle.handle, reg->handle.accelerator,
+           SMSC_ACCELERATOR_HANDLE_SIZE);
+    ret = ep->rcache->rcache_register(ep->rcache, remote_address, size,
+                                      /*ep->peer_smp_rank for debugging only */ 0,
+                                      MCA_RCACHE_ACCESS_LOCAL_WRITE,
+                                      (mca_rcache_base_registration_t **) &reg_ptr);
+    if (ret != OPAL_SUCCESS) {
+        opal_output(0, "mca_smsc_accelerator_copy: Failed to register remote memory, ret=%d", ret);
+        return ret;
+    }
+
+    offset = (size_t)((intptr_t) remote_address - (intptr_t) reg->base_addr);
+    remote_memory_address = (unsigned char *) reg_ptr->base.alloc_base + offset;
+
+    if (is_remote_write) {
+        ret = opal_accelerator.mem_copy(MCA_ACCELERATOR_NO_DEVICE_ID, mca_smsc_accelerator_module.device_id,
+                                        remote_memory_address, local_address, size,
+                                        MCA_ACCELERATOR_TRANSFER_DTOD);
+        if (ret != OPAL_SUCCESS) {
+            opal_output(0, "mca_smsc_accelerator_copy: accelerator mem_copy write failed, ret %d remote_address %p "
+                        "local_address %p size %lu", ret, remote_memory_address, local_address, size);
+        }
+    } else {
+        ret = opal_accelerator.mem_copy(mca_smsc_accelerator_module.device_id, MCA_ACCELERATOR_NO_DEVICE_ID,
+                                        local_address, remote_memory_address, size,
+                                        MCA_ACCELERATOR_TRANSFER_DTOD);
+        if (ret != OPAL_SUCCESS) {
+            opal_output(0, "mca_smsc_accelerator_copy: accelerator mem_copy read failed, ret %d remote_address %p "
+                        "local_address %p size %lu", ret, remote_memory_address, local_address, size);
+        }
+    }
+
+    return ret;
+}
+
+int mca_smsc_accelerator_copy_to(mca_smsc_endpoint_t *endpoint, void *local_address, void *remote_address,
+                                 size_t size, void *reg_handle)
+{
+    mca_smsc_accelerator_endpoint_t *ep = (mca_smsc_accelerator_endpoint_t *)endpoint;
+    mca_smsc_accelerator_registration_data_t *reg = (mca_smsc_accelerator_registration_data_t *)reg_handle;
+    int ret = OPAL_SUCCESS;
+
+    if ((NULL != reg) && (0 != reg->base_addr)) {
+        ret = mca_smsc_accelerator_copy (endpoint, local_address, remote_address, size, reg_handle, true);
+    }
+    else if (NULL != mca_smsc_accelerator_module.prev_smsc) {
+        ret = mca_smsc_accelerator_module.prev_smsc->copy_to(ep->prev_endpoint, local_address,
+                                                             remote_address, size, reg_handle);
+    }
+
+    return ret;
+}
+
+int mca_smsc_accelerator_copy_from(mca_smsc_endpoint_t *endpoint, void *local_address,
+                                   void *remote_address, size_t size, void *reg_handle)
+{
+    mca_smsc_accelerator_endpoint_t *ep = (mca_smsc_accelerator_endpoint_t *)endpoint;
+    mca_smsc_accelerator_registration_data_t *reg = (mca_smsc_accelerator_registration_data_t *)reg_handle;
+    int ret = OPAL_SUCCESS;
+
+    if ((NULL != reg) && (0 != reg->base_addr)) {
+        ret = mca_smsc_accelerator_copy (endpoint, local_address, remote_address, size, reg_handle, false);
+    }
+    else if (NULL != mca_smsc_accelerator_module.prev_smsc) {
+        ret = mca_smsc_accelerator_module.prev_smsc->copy_from(ep->prev_endpoint, local_address,
+                                                               remote_address, size, reg_handle);
+    }
+
+    return ret;
+}
+
+void *mca_smsc_accelerator_register_region(void *local_address, size_t size)
+{
+    int dev_id, ret;
+    mca_smsc_accelerator_registration_handle_t *reg;
+    uint64_t flags;
+
+    reg = (mca_smsc_accelerator_registration_handle_t *)malloc (sizeof(mca_smsc_accelerator_registration_handle_t));
+    if (NULL == reg) {
+        opal_output(0, "mca_smsc_accelerator_register_mem: failed to allocate memory");
+        return NULL;
+    }
+
+    if (opal_accelerator.check_addr(local_address, &dev_id, &flags) > 0) {
+        ret = mca_smsc_accelerator_module.rcache->rcache_register(mca_smsc_accelerator_module.rcache,
+                                                                  local_address, size,
+                                                                  MCA_RCACHE_FLAGS_ACCELERATOR_MEM,  MCA_RCACHE_ACCESS_ANY,
+                                                                  (mca_rcache_base_registration_t **) &reg->gpu_reg);
+        if (OPAL_UNLIKELY(OPAL_SUCCESS != ret)) {
+            opal_output_verbose(MCA_BASE_VERBOSE_WARN, opal_smsc_base_framework.framework_output,
+                                "mca_smsc_accelerator_register_mem: failed to register memory for single-copy");
+            return NULL;
+        }
+        memcpy(reg->data.handle.accelerator, reg->gpu_reg->data.ipcHandle.handle, SMSC_ACCELERATOR_HANDLE_SIZE);
+        reg->data.base_addr = (uint64_t) reg->gpu_reg->base.base;
+    }
+    else {
+        reg->data.handle.host = NULL;
+        if ( (NULL != mca_smsc_accelerator_module.prev_smsc) &&
+             (mca_smsc_accelerator_module.prev_smsc->features & MCA_SMSC_FEATURE_REQUIRE_REGISTRATION)) {
+            reg->data.handle.host = mca_smsc_accelerator_module.prev_smsc->register_region(local_address, size);
+        }
+        reg->data.base_addr = 0;
+    }
+
+    return &(reg->data);
+}
+
+void mca_smsc_accelerator_deregister_region(void *reg_data)
+{
+    if (NULL == reg_data) {
+        return;
+    }
+
+    mca_smsc_accelerator_registration_data_t *reg = (mca_smsc_accelerator_registration_data_t *)reg_data;
+    mca_smsc_accelerator_registration_handle_t *reg_handle = MCA_SMSC_ACCELERATOR_REG_DATA_TO_HANDLE(reg);
+    if (0 != reg->base_addr) {
+        mca_smsc_accelerator_module.rcache->rcache_deregister(mca_smsc_accelerator_module.rcache,
+                                                              &reg_handle->gpu_reg->base);
+    }
+    else {
+        if ((NULL != mca_smsc_accelerator_module.prev_smsc) &&
+            (mca_smsc_accelerator_module.prev_smsc->features & MCA_SMSC_FEATURE_REQUIRE_REGISTRATION) &&
+            (NULL != reg->handle.host)) {
+            mca_smsc_accelerator_module.prev_smsc->deregister_region(reg->handle.host);
+        }
+    }
+    free(reg_handle);
+}
+
+mca_smsc_accelerator_module_t mca_smsc_accelerator_module = {
+    .super = {
+        .registration_data_size = sizeof(mca_smsc_accelerator_registration_data_t),
+        .features = MCA_SMSC_FEATURE_REQUIRE_REGISTRATION | MCA_SMSC_FEATURE_ACCELERATOR, // also modified in component_open
+        .get_endpoint = mca_smsc_accelerator_get_endpoint,
+        .return_endpoint = mca_smsc_accelerator_return_endpoint,
+        .copy_to = mca_smsc_accelerator_copy_to,
+        .copy_from = mca_smsc_accelerator_copy_from,
+        .map_peer_region = mca_smsc_accelerator_map_peer_region,
+        .unmap_peer_region = mca_smsc_accelerator_unmap_peer_region,
+        .register_region = mca_smsc_accelerator_register_region,
+        .deregister_region = mca_smsc_accelerator_deregister_region,
+    },
+};
diff --git a/opal/mca/smsc/base/Makefile.am b/opal/mca/smsc/base/Makefile.am
new file mode 100644
index 00000000000..28a0678cd17
--- /dev/null
+++ b/opal/mca/smsc/base/Makefile.am
@@ -0,0 +1,26 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2021      Google, LLC. All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+AM_CPPFLAGS = ${smsc_@DIRECT_smsc@_CPPFLAGS}
+
+headers += \
+        base/base.h
+
+libmca_smsc_la_SOURCES += \
+        base/smsc_base_frame.c
diff --git a/opal/mca/smsc/base/base.h b/opal/mca/smsc/base/base.h
new file mode 100644
index 00000000000..6869e0a1580
--- /dev/null
+++ b/opal/mca/smsc/base/base.h
@@ -0,0 +1,31 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2021      Google, LLC. All rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef OPAL_MCA_SMSC_BASE_BASE_H
+#define OPAL_MCA_SMSC_BASE_BASE_H
+
+#include "ompi_config.h"
+
+#include "opal/mca/smsc/smsc.h"
+
+extern mca_base_framework_t opal_smsc_base_framework;
+
+struct mca_smsc_base_component_t {
+    opal_list_item_t super;
+    mca_smsc_component_t *smsc_component;
+};
+typedef struct mca_smsc_base_component_t mca_smsc_base_component_t;
+OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_smsc_base_component_t);
+
+int mca_smsc_base_select(void);
+void mca_smsc_base_register_default_params(mca_smsc_component_t *component, int default_priority);
+
+#endif /* OPAL_MCA_SMSC_BASE_BASE_H */
diff --git a/opal/mca/smsc/base/smsc_base_frame.c b/opal/mca/smsc/base/smsc_base_frame.c
new file mode 100644
index 00000000000..405d3545998
--- /dev/null
+++ b/opal/mca/smsc/base/smsc_base_frame.c
@@ -0,0 +1,153 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2021      Google, LLC. All rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2024      Advanced Micro Devices, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "opal_config.h"
+#include <stdio.h>
+#include <string.h>
+
+#include "opal/class/opal_list.h"
+#include "opal/mca/base/base.h"
+#include "opal/mca/mca.h"
+#include "opal/mca/smsc/base/base.h"
+#include "opal/mca/smsc/smsc.h"
+#include "opal/util/printf.h"
+
+/*
+ * The following file was created by configure.  It contains extern
+ * statements and the definition of an array of pointers to each
+ * component's public mca_base_component_t struct.
+ */
+#include "opal/mca/smsc/base/static-components.h"
+
+static mca_smsc_component_t *selected_component = NULL;
+mca_smsc_module_t *mca_smsc = NULL;
+
+OBJ_CLASS_INSTANCE(mca_smsc_base_component_t, opal_list_item_t, NULL, NULL);
+
+/*
+ * Global variables
+ */
+MCA_BASE_FRAMEWORK_DECLARE(opal, smsc, NULL, NULL, NULL, NULL, mca_smsc_base_static_components, 0);
+
+static int mca_smsc_compare_components(opal_list_item_t **a, opal_list_item_t **b)
+{
+    mca_smsc_component_t *componenta
+        = (mca_smsc_component_t *) ((mca_smsc_base_component_t *) *a)->smsc_component;
+    mca_smsc_component_t *componentb
+        = (mca_smsc_component_t *) ((mca_smsc_base_component_t *) *b)->smsc_component;
+
+    return (componenta->priority > componentb->priority)
+               ? -1
+               : ((componenta->priority < componentb->priority) ? 1 : 0);
+}
+
+int mca_smsc_base_select(void)
+{
+    mca_base_component_list_item_t *cli, *next;
+    mca_smsc_base_component_t *first=NULL, *second=NULL;
+    opal_list_t *selectable;
+    mca_smsc_component_t *second_component=NULL;
+    mca_smsc_base_component_t *avail;
+
+    /* Make a list of the components that query successfully */
+    selectable = OBJ_NEW(opal_list_t);
+
+    OPAL_LIST_FOREACH_SAFE (cli, next, &opal_smsc_base_framework.framework_components,
+                            mca_base_component_list_item_t) {
+        mca_smsc_component_t *component = (mca_smsc_component_t *) cli->cli_component;
+
+        opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT, opal_smsc_base_framework.framework_output,
+                            "mca_smsc_base_select: checking component %s",
+                            component->smsc_version.mca_component_name);
+
+        int ret = component->query();
+        if (OPAL_SUCCESS != ret) {
+            opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT,
+                                opal_smsc_base_framework.framework_output,
+                                "mca_smsc_base_select: could not select component %s. query "
+                                "returned error code %d",
+                                component->smsc_version.mca_component_name, ret);
+            opal_list_remove_item(&opal_smsc_base_framework.framework_components, &cli->super);
+            OBJ_RELEASE(cli);
+            mca_base_component_close(&component->smsc_version,
+                                     opal_smsc_base_framework.framework_output);
+            continue;
+        }
+        avail = OBJ_NEW(mca_smsc_base_component_t);
+        avail->smsc_component = component;
+
+        opal_list_append(selectable, &avail->super);
+
+        opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT, opal_smsc_base_framework.framework_output,
+                            "mca_smsc_base_select: component %s priority=%d",
+                            component->smsc_version.mca_component_name, component->priority);
+    }
+
+    opal_list_sort(selectable, mca_smsc_compare_components);
+
+    if (opal_list_get_size(selectable) > 0) {
+        first = (mca_smsc_base_component_t *) opal_list_remove_first(selectable);
+        if (opal_list_get_size(selectable) > 0) {
+            /* enable second component first. Allows for the component
+             * with the highest priority to store it as a pass-through
+             * component */
+            second = (mca_smsc_base_component_t *) opal_list_remove_first(selectable);
+            second_component = (mca_smsc_component_t *) second->smsc_component;
+            mca_smsc = second_component->enable();
+        }
+
+        selected_component = (mca_smsc_component_t *) first->smsc_component;
+        mca_smsc = selected_component->enable();
+
+        opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT, opal_smsc_base_framework.framework_output,
+                            "mca_smsc_base_select: selected shared-memory single-copy component: %s",
+                            selected_component->smsc_version.mca_component_name);
+        if (NULL != second_component) {
+            opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT,
+                                opal_smsc_base_framework.framework_output,
+                                "mca_smsc_base_select: selected %s as pass-through component for host memory",
+                                second_component->smsc_version.mca_component_name);
+        }
+    } else {
+        opal_output_verbose(
+                            MCA_BASE_VERBOSE_COMPONENT, opal_smsc_base_framework.framework_output,
+                            "mca_smsc_base_select: no shared-memory single-copy component available for selection");
+    }
+
+    OBJ_RELEASE(selectable);
+
+    return OPAL_SUCCESS;
+}
+
+void mca_smsc_base_register_default_params(mca_smsc_component_t *component, int default_priority)
+{
+
+    char *tmp;
+    (void) opal_asprintf(&tmp, "Priority of the %s component (default: %d)",
+                         component->smsc_version.mca_component_name, default_priority);
+    component->priority = default_priority;
+    (void) mca_base_component_var_register(&component->smsc_version, "priority", /*help_msg=*/tmp,
+                                           MCA_BASE_VAR_TYPE_INT, /*enumerator=*/NULL, /*bind=*/0,
+                                           MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
+                                           MCA_BASE_VAR_SCOPE_ALL_EQ, &component->priority);
+    free(tmp);
+}
diff --git a/opal/mca/smsc/cma/Makefile.am b/opal/mca/smsc/cma/Makefile.am
new file mode 100644
index 00000000000..02b539f0dcd
--- /dev/null
+++ b/opal/mca/smsc/cma/Makefile.am
@@ -0,0 +1,56 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2009 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2009-2014 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights
+#                         reserved.
+# Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2020-2021 Google, LLC. All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+EXTRA_DIST = post_configure.sh
+
+AM_CPPFLAGS = $(smsc_cma_CPPFLAGS)
+
+libmca_smsc_cma_la_sources = \
+    smsc_cma_component.c \
+    smsc_cma_module.c \
+    smsc_cma_internal.h \
+    smsc_cma.h
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_opal_smsc_cma_DSO
+component_noinst =
+component_install = mca_smsc_cma.la
+else
+component_noinst = libmca_smsc_cma.la
+component_install =
+endif
+
+mcacomponentdir = $(opallibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_smsc_cma_la_SOURCES = $(libmca_smsc_cma_la_sources)
+mca_smsc_cma_la_LDFLAGS = -module -avoid-version $(smsc_cma_LDFLAGS)
+mca_smsc_cma_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
+	$(smsc_cma_LIBS)
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_smsc_cma_la_SOURCES = $(libmca_smsc_cma_la_sources)
+libmca_smsc_cma_la_LIBADD = $(smsc_cma_LIBS)
+libmca_smsc_cma_la_LDFLAGS = -module -avoid-version $(smsc_cma_LDFLAGS)
diff --git a/opal/mca/smsc/cma/configure.m4 b/opal/mca/smsc/cma/configure.m4
new file mode 100644
index 00000000000..6c933406dba
--- /dev/null
+++ b/opal/mca/smsc/cma/configure.m4
@@ -0,0 +1,31 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2009      The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2009-2016 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights
+#                         reserved.
+# Copyright (c) 2015      Research Organization for Information Science
+#                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2021      Google, LLC. All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# MCA_smsc_cma_CONFIG([action-if-can-compile],
+#                      [action-if-cant-compile])
+# ------------------------------------------------
+AC_DEFUN([MCA_opal_smsc_cma_CONFIG],[
+    AC_CONFIG_FILES([opal/mca/smsc/cma/Makefile])
+
+    OPAL_CHECK_CMA([smsc_cma], [AC_CHECK_HEADER([sys/prctl.h]) $1], [$2])
+
+    AC_SUBST([smsc_cma_CPPFLAGS])
+    AC_SUBST([smsc_cma_LDFLAGS])
+    AC_SUBST([smsc_cma_LIBS])
+])dnl
diff --git a/opal/mca/smsc/cma/post_configure.sh b/opal/mca/smsc/cma/post_configure.sh
new file mode 100644
index 00000000000..3059fa936d4
--- /dev/null
+++ b/opal/mca/smsc/cma/post_configure.sh
@@ -0,0 +1 @@
+DIRECT_CALL_HEADER="opal/mca/smsc/cma/smsc_cma.h"
diff --git a/opal/mca/smsc/cma/smsc_cma.h b/opal/mca/smsc/cma/smsc_cma.h
new file mode 100644
index 00000000000..64aa9dd9497
--- /dev/null
+++ b/opal/mca/smsc/cma/smsc_cma.h
@@ -0,0 +1,33 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2021      Google, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef OPAL_MCA_SMSC_CMA_SMSC_CMA_H
+#define OPAL_MCA_SMSC_CMA_SMSC_CMA_H
+
+#include "opal_config.h"
+
+#include "opal/mca/smsc/smsc.h"
+
+mca_smsc_endpoint_t *mca_smsc_cma_get_endpoint(opal_proc_t *peer_proc);
+void mca_smsc_cma_return_endpoint(mca_smsc_endpoint_t *endpoint);
+
+int mca_smsc_cma_copy_to(mca_smsc_endpoint_t *endpoint, void *local_address, void *remote_address,
+                         size_t size, void *reg_handle);
+int mca_smsc_cma_copy_from(mca_smsc_endpoint_t *endpoint, void *local_address, void *remote_address,
+                           size_t size, void *reg_handle);
+
+/* unsupported interfaces defined to support MCA direct */
+void *mca_smsc_cma_map_peer_region(mca_smsc_endpoint_t *endpoint, uint64_t flags,
+                                   void *remote_address, size_t size, void **local_mapping);
+void mca_smsc_cma_unmap_peer_region(void *ctx);
+void *mca_smsc_cma_register_region(void *local_address, size_t size);
+void mca_smsc_cma_deregister_region(void *reg_data);
+
+#endif /* OPAL_MCA_SMSC_CMA_SMSC_CMA_H */
diff --git a/opal/mca/smsc/cma/smsc_cma_component.c b/opal/mca/smsc/cma/smsc_cma_component.c
new file mode 100644
index 00000000000..4ccd731ba24
--- /dev/null
+++ b/opal/mca/smsc/cma/smsc_cma_component.c
@@ -0,0 +1,147 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2021      Google, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "opal_config.h"
+
+#include "opal/mca/smsc/base/base.h"
+#include "opal/mca/smsc/cma/smsc_cma_internal.h"
+
+#include <fcntl.h>
+#include <stdio.h>
+#include <sys/prctl.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+static int mca_smsc_cma_component_register(void);
+static int mca_smsc_cma_component_open(void);
+static int mca_smsc_cma_component_close(void);
+static int mca_smsc_cma_component_query(void);
+static mca_smsc_module_t *mca_smsc_cma_component_enable(void);
+
+#define MCA_SMSC_CMA_DEFAULT_PRIORITY 37
+static const int mca_smsc_cma_default_priority = MCA_SMSC_CMA_DEFAULT_PRIORITY;
+
+mca_smsc_component_t mca_smsc_cma_component = {
+  .smsc_version = {
+    MCA_SMSC_DEFAULT_VERSION("cma"),
+    .mca_open_component = mca_smsc_cma_component_open,
+    .mca_close_component = mca_smsc_cma_component_close,
+    .mca_register_component_params = mca_smsc_cma_component_register,
+  },
+  .priority = MCA_SMSC_CMA_DEFAULT_PRIORITY,
+  .query = mca_smsc_cma_component_query,
+  .enable = mca_smsc_cma_component_enable,
+};
+
+static int mca_smsc_cma_component_register(void)
+{
+    mca_smsc_base_register_default_params(&mca_smsc_cma_component, mca_smsc_cma_default_priority);
+    return OPAL_SUCCESS;
+}
+
+static int mca_smsc_cma_component_open(void)
+{
+    /* nothing to do */
+    return OPAL_SUCCESS;
+}
+
+static int mca_smsc_cma_component_close(void)
+{
+    /* nothing to do */
+    return OPAL_SUCCESS;
+}
+
+/*
+ * mca_btl_sm_parse_proc_ns_user() tries to get the user namespace ID
+ * of the current process.
+ * Returns the ID of the user namespace. In the case of an error '0' is returned.
+ */
+ino_t mca_smsc_cma_get_user_ns_id(void)
+{
+    struct stat buf;
+
+    if (0 > stat("/proc/self/ns/user", &buf)) {
+        /*
+         * Something went wrong, probably an old kernel that does not support namespaces
+         * simply assume all processes are in the same user namespace and return 0
+         */
+        return 0;
+    }
+
+    return buf.st_ino;
+}
+
+static int mca_smsc_cma_send_modex(void)
+{
+    mca_smsc_cma_modex_t modex;
+
+    modex.pid = getpid();
+    modex.user_ns_id = mca_smsc_cma_get_user_ns_id();
+
+    int rc;
+    OPAL_MODEX_SEND(rc, PMIX_LOCAL, &mca_smsc_cma_component.smsc_version, &modex, sizeof(modex));
+    return rc;
+}
+
+static int mca_smsc_cma_component_query(void)
+{
+    /* Check if we have the proper permissions for CMA */
+    char buffer = '0';
+    bool cma_happy = false;
+
+    /* check system setting for current ptrace scope */
+    int fd = open("/proc/sys/kernel/yama/ptrace_scope", O_RDONLY);
+    if (0 <= fd) {
+        int ret = read(fd, &buffer, 1);
+        if (ret < 0) {
+            opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT,
+                                opal_smsc_base_framework.framework_output,
+                                "mca_smsc_cma_component_query: could not read ptrace_scope. "
+                                "assuming ptrace scope is 0");
+        }
+        close(fd);
+    }
+
+    /* ptrace scope 0 will allow an attach from any of the process owner's
+     * processes. ptrace scope 1 limits attachers to the process tree
+     * starting at the parent of this process. */
+    if ('0' != buffer) {
+#if defined PR_SET_PTRACER
+        /* try setting the ptrace scope to allow attach */
+        int ret = prctl(PR_SET_PTRACER, PR_SET_PTRACER_ANY, 0, 0, 0);
+        if (0 == ret) {
+            cma_happy = true;
+        }
+#endif
+    } else {
+        cma_happy = true;
+    }
+
+    if (!cma_happy) {
+        opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT, opal_smsc_base_framework.framework_output,
+                            "mca_smsc_cma_component_query: could not select for use. insufficient "
+                            "ptrace permissions.");
+        mca_smsc_cma_component.priority = -1;
+        return OPAL_ERR_NOT_AVAILABLE;
+    }
+
+    mca_smsc_cma_send_modex();
+
+    return OPAL_SUCCESS;
+}
+
+static mca_smsc_module_t *mca_smsc_cma_component_enable(void)
+{
+    if (0 > mca_smsc_cma_component.priority) {
+        return NULL;
+    }
+
+    return &mca_smsc_cma_module;
+}
diff --git a/opal/mca/smsc/cma/smsc_cma_internal.h b/opal/mca/smsc/cma/smsc_cma_internal.h
new file mode 100644
index 00000000000..7de42b64a0c
--- /dev/null
+++ b/opal/mca/smsc/cma/smsc_cma_internal.h
@@ -0,0 +1,37 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2021      Google, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef OPAL_MCA_SMSC_CMA_SMSC_CMA_INTERNAL_H
+#define OPAL_MCA_SMSC_CMA_SMSC_CMA_INTERNAL_H
+
+#include "opal/mca/smsc/cma/smsc_cma.h"
+
+struct mca_smsc_cma_modex_t {
+    pid_t pid;
+    ino_t user_ns_id;
+};
+
+typedef struct mca_smsc_cma_modex_t mca_smsc_cma_modex_t;
+
+struct mca_smsc_cma_endpoint_t {
+    mca_smsc_endpoint_t super;
+    pid_t pid;
+};
+
+typedef struct mca_smsc_cma_endpoint_t mca_smsc_cma_endpoint_t;
+
+OBJ_CLASS_DECLARATION(mca_smsc_cma_endpoint_t);
+
+extern mca_smsc_module_t mca_smsc_cma_module;
+extern mca_smsc_component_t mca_smsc_cma_component;
+
+ino_t mca_smsc_cma_get_user_ns_id(void);
+
+#endif /* OPAL_MCA_SMSC_CMA_SMSC_CMA_INTERNAL_H */
diff --git a/opal/mca/smsc/cma/smsc_cma_module.c b/opal/mca/smsc/cma/smsc_cma_module.c
new file mode 100644
index 00000000000..0aeec05c084
--- /dev/null
+++ b/opal/mca/smsc/cma/smsc_cma_module.c
@@ -0,0 +1,241 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2010-2014 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2014-2018 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2020      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2021      Google, Inc. All rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "opal_config.h"
+
+#include "opal/mca/pmix/pmix-internal.h"
+#include "opal/mca/smsc/base/base.h"
+#include "opal/mca/smsc/cma/smsc_cma_internal.h"
+
+#if HAVE_LINUX_KCMP_H
+#    include <linux/kcmp.h>       /* kcmp: Definition of KCMP_* constants */
+#endif /* HAVE_LINUX_KCMP_H */
+#if HAVE_SYS_SYSCALL_H
+#    include <sys/syscall.h>      /* kcmp: Definition of SYS_* constants */
+#endif /* HAVE_SYS_SYSCALL_H */
+
+#if OPAL_CMA_NEED_SYSCALL_DEFS
+#    include "opal/sys/cma.h"
+#else
+#    include <sys/uio.h>
+#endif /* OPAL_CMA_NEED_SYSCALL_DEFS */
+
+OBJ_CLASS_INSTANCE(mca_smsc_cma_endpoint_t, opal_object_t, NULL, NULL);
+
+mca_smsc_endpoint_t *mca_smsc_cma_get_endpoint(opal_proc_t *peer_proc)
+{
+    mca_smsc_cma_endpoint_t *endpoint = OBJ_NEW(mca_smsc_cma_endpoint_t);
+    if (OPAL_UNLIKELY(NULL == endpoint)) {
+        return NULL;
+    }
+
+    endpoint->super.proc = peer_proc;
+
+    int rc;
+    size_t modex_size;
+    mca_smsc_cma_modex_t *modex;
+    OPAL_MODEX_RECV_IMMEDIATE(rc, &mca_smsc_cma_component.smsc_version, &peer_proc->proc_name,
+                              (void **) &modex, &modex_size);
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
+        OBJ_RELEASE(endpoint);
+        return NULL;
+    }
+
+    ino_t my_ns_id = mca_smsc_cma_get_user_ns_id();
+    if (modex->user_ns_id != my_ns_id) {
+        opal_output_verbose(MCA_BASE_VERBOSE_ERROR, opal_smsc_base_framework.framework_output,
+                            "mca_smsc_cma_module_get_endpoint: can not proceed. processes are in "
+                            "difference namespaces");
+        /* can't use CMA with this peer */
+        OBJ_RELEASE(endpoint);
+        free(modex);
+        return NULL;
+    }
+
+#if OPAL_CMA_KCMP_AVAIL
+    /* Check if CAP_SYS_PTRACE capability is allowed between these two processes
+     * Calling process_vm_readv/writev requires CAP_SYS_PTRACE. We can use kcmp
+     * to check if these two processes share a kernel resource. Since kcmp
+     * also requires CAP_SYS_PTRACE it is a good proxy for process_vm_readv/writev.
+     */
+    rc = syscall(SYS_kcmp, getpid(), modex->pid, KCMP_VM, 0, 0);
+    if(rc < 0) {
+        opal_output_verbose(MCA_BASE_VERBOSE_ERROR, opal_smsc_base_framework.framework_output,
+                            "mca_smsc_cma_module_get_endpoint: can not proceed. processes do not have "
+                            "the necessary permissions (i.e., CAP_SYS_PTRACE). "
+                            "PID %d <-> %d (rc = %d) (errno: %d: %s)",
+                            getpid(), modex->pid, rc, errno, strerror(errno));
+        /* can't use CMA with this peer */
+        OBJ_RELEASE(endpoint);
+        free(modex);
+        return NULL;
+    }
+#endif /* OPAL_CMA_KCMP_AVAIL */
+
+    endpoint->pid = modex->pid;
+    return &endpoint->super;
+}
+
+void mca_smsc_cma_return_endpoint(mca_smsc_endpoint_t *endpoint)
+{
+    OBJ_RELEASE(endpoint);
+}
+
+static inline void mca_smsc_cma_iov_advance(struct iovec *iov, ssize_t length)
+{
+    iov->iov_base = (void *) ((uintptr_t) iov->iov_base + length);
+    iov->iov_len -= length;
+}
+
+int mca_smsc_cma_copy_to(mca_smsc_endpoint_t *endpoint, void *local_address, void *remote_address,
+                         size_t size, void *reg_handle)
+{
+    /* ignore the registration handle as it is not used for CMA */
+    (void) reg_handle;
+
+    mca_smsc_cma_endpoint_t *cma_endpoint = (mca_smsc_cma_endpoint_t *) endpoint;
+
+    /*
+     * According to the man page :
+     * "On success, process_vm_readv() returns the number of bytes read and
+     * process_vm_writev() returns the number of bytes written.  This return
+     * value may be less than the total number of requested bytes, if a
+     * partial read/write occurred.  (Partial transfers apply at the
+     * granularity of iovec elements.  These system calls won't perform a
+     * partial transfer that splits a single iovec element.)".
+     * So since we use a single iovec element, the returned size should either
+     * be 0 or size, and the do loop should not be needed here.
+     * We tried on various Linux kernels with size > 2 GB, and surprisingly,
+     * the returned value is always 0x7ffff000 (fwiw, it happens to be the size
+     * of the larger number of pages that fits a signed 32 bits integer).
+     * We do not know whether this is a bug from the kernel, the libc or even
+     * the man page, but for the time being, we do as is process_vm_readv() could
+     * return any value.
+     */
+    struct iovec src_iov = {
+        .iov_base = local_address,
+        .iov_len = size,
+    };
+    struct iovec dst_iov = {
+        .iov_base = remote_address,
+        .iov_len = size,
+    };
+    ssize_t ret;
+    do {
+        ret = process_vm_writev(cma_endpoint->pid, &src_iov, 1, &dst_iov, 1, 0);
+        if (0 > ret) {
+            if (ESRCH == errno) {
+                OPAL_OUTPUT_VERBOSE((MCA_BASE_VERBOSE_ERROR,
+                                     opal_smsc_base_framework.framework_output,
+                                     "CMA wrote %ld, expected %lu, errno = %d", (long) ret,
+                                     (unsigned long) size, errno));
+                return OPAL_ERROR;
+            }
+            OPAL_OUTPUT_VERBOSE((MCA_BASE_VERBOSE_ERROR, opal_smsc_base_framework.framework_output,
+                                 "CMA wrote %ld, expected %lu, errno = %d", (long) ret,
+                                 (unsigned long) size, errno));
+            return OPAL_ERROR;
+        }
+        mca_smsc_cma_iov_advance(&src_iov, ret);
+        mca_smsc_cma_iov_advance(&dst_iov, ret);
+    } while (0 < src_iov.iov_len);
+
+    return OPAL_SUCCESS;
+}
+
+int mca_smsc_cma_copy_from(mca_smsc_endpoint_t *endpoint, void *local_address, void *remote_address,
+                           size_t size, void *reg_handle)
+{
+    /* ignore the registration handle as it is not used for CMA */
+    (void) reg_handle;
+
+    mca_smsc_cma_endpoint_t *cma_endpoint = (mca_smsc_cma_endpoint_t *) endpoint;
+
+    /*
+     * According to the man page :
+     * "On success, process_vm_readv() returns the number of bytes read and
+     * process_vm_writev() returns the number of bytes written.  This return
+     * value may be less than the total number of requested bytes, if a
+     * partial read/write occurred.  (Partial transfers apply at the
+     * granularity of iovec elements.  These system calls won't perform a
+     * partial transfer that splits a single iovec element.)".
+     * So since we use a single iovec element, the returned size should either
+     * be 0 or size, and the do loop should not be needed here.
+     * We tried on various Linux kernels with size > 2 GB, and surprisingly,
+     * the returned value is always 0x7ffff000 (fwiw, it happens to be the size
+     * of the larger number of pages that fits a signed 32 bits integer).
+     * We do not know whether this is a bug from the kernel, the libc or even
+     * the man page, but for the time being, we do as is process_vm_readv() could
+     * return any value.
+     */
+    struct iovec src_iov = {
+        .iov_base = remote_address,
+        .iov_len = size,
+    };
+    struct iovec dst_iov = {
+        .iov_base = local_address,
+        .iov_len = size,
+    };
+    ssize_t ret;
+    do {
+        ret = process_vm_readv(cma_endpoint->pid, &dst_iov, 1, &src_iov, 1, 0);
+        if (0 > ret) {
+            if (ESRCH == errno) {
+                OPAL_OUTPUT_VERBOSE((MCA_BASE_VERBOSE_ERROR,
+                                     opal_smsc_base_framework.framework_output,
+                                     "CMA read %ld, expected %lu, errno = %d", (long) ret,
+                                     (unsigned long) size, errno));
+                return OPAL_ERROR;
+            }
+            OPAL_OUTPUT_VERBOSE((MCA_BASE_VERBOSE_ERROR, opal_smsc_base_framework.framework_output,
+                                 "CMA read %ld, expected %lu, errno = %d\n", (long) ret,
+                                 (unsigned long) size, errno));
+            return OPAL_ERROR;
+        }
+        mca_smsc_cma_iov_advance(&src_iov, ret);
+        mca_smsc_cma_iov_advance(&dst_iov, ret);
+    } while (0 < src_iov.iov_len);
+
+    return OPAL_SUCCESS;
+}
+
+/* unsupported interfaces defined to support MCA direct */
+void *mca_smsc_cma_map_peer_region(mca_smsc_endpoint_t *endpoint, uint64_t flags,
+                                   void *remote_address, size_t size, void **local_mapping)
+{
+    return NULL;
+}
+
+void mca_smsc_cma_unmap_peer_region(void *ctx)
+{
+}
+
+void *mca_smsc_cma_register_region(void *local_address, size_t size)
+{
+    return NULL;
+}
+
+void mca_smsc_cma_deregister_region(void *reg_data)
+{
+}
+
+mca_smsc_module_t mca_smsc_cma_module = {
+    .get_endpoint = mca_smsc_cma_get_endpoint,
+    .return_endpoint = mca_smsc_cma_return_endpoint,
+    .copy_to = mca_smsc_cma_copy_to,
+    .copy_from = mca_smsc_cma_copy_from,
+};
diff --git a/opal/mca/smsc/configure.m4 b/opal/mca/smsc/configure.m4
new file mode 100644
index 00000000000..51544fedc11
--- /dev/null
+++ b/opal/mca/smsc/configure.m4
@@ -0,0 +1,19 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2013      Sandia National Laboratories.  All rights reserved.
+# Copyright (c) 2021      Google, LLC. All rights reserved.
+#
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+AC_DEFUN([MCA_opal_smsc_CONFIG],[
+    # configure all the components
+    MCA_CONFIGURE_FRAMEWORK($1, $2, 1)
+
+    # this is a direct callable component, so set that up.
+    MCA_SETUP_DIRECT_CALL($1, $2)
+])
diff --git a/opal/mca/smsc/knem/Makefile.am b/opal/mca/smsc/knem/Makefile.am
new file mode 100644
index 00000000000..4694428f711
--- /dev/null
+++ b/opal/mca/smsc/knem/Makefile.am
@@ -0,0 +1,60 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2009 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2009-2014 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights
+#                         reserved.
+# Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2020-2021 Google, LLC. All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+EXTRA_DIST = \
+    post_configure.sh \
+    help-smsc-knem.txt
+
+AM_CPPFLAGS = $(smsc_knem_CPPFLAGS)
+
+
+libmca_smsc_knem_la_sources = \
+    smsc_knem_component.c \
+    smsc_knem_module.c \
+    smsc_knem_internal.h \
+    smsc_knem.h
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_opal_smsc_knem_DSO
+component_noinst =
+component_install = mca_smsc_knem.la
+else
+component_noinst = libmca_smsc_knem.la
+component_install =
+endif
+
+mcacomponentdir = $(opallibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_smsc_knem_la_SOURCES = $(libmca_smsc_knem_la_sources)
+mca_smsc_knem_la_LDFLAGS = -module -avoid-version $(smsc_knem_LDFLAGS)
+mca_smsc_knem_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
+	$(smsc_knem_LIBS)
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_smsc_knem_la_SOURCES = $(libmca_smsc_knem_la_sources)
+libmca_smsc_knem_la_LIBADD = $(smsc_knem_LIBS)
+libmca_smsc_knem_la_LDFLAGS = -module -avoid-version $(smsc_knem_LDFLAGS)
diff --git a/opal/mca/smsc/knem/configure.m4 b/opal/mca/smsc/knem/configure.m4
new file mode 100644
index 00000000000..091dd7e262a
--- /dev/null
+++ b/opal/mca/smsc/knem/configure.m4
@@ -0,0 +1,31 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2009      The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2009-2016 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights
+#                         reserved.
+# Copyright (c) 2015      Research Organization for Information Science
+#                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2021      Google, LLC. All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# MCA_smsc_knem_CONFIG([action-if-can-compile],
+#                      [action-if-cant-compile])
+# ------------------------------------------------
+AC_DEFUN([MCA_opal_smsc_knem_CONFIG],[
+    AC_CONFIG_FILES([opal/mca/smsc/knem/Makefile])
+
+    OPAL_CHECK_KNEM([smsc_knem], [$1], [$2])
+
+    AC_SUBST([smsc_knem_CPPFLAGS])
+    AC_SUBST([smsc_knem_LDFLAGS])
+    AC_SUBST([smsc_knem_LIBS])
+])dnl
diff --git a/opal/mca/smsc/knem/help-smsc-knem.txt b/opal/mca/smsc/knem/help-smsc-knem.txt
new file mode 100644
index 00000000000..acea1ca05d0
--- /dev/null
+++ b/opal/mca/smsc/knem/help-smsc-knem.txt
@@ -0,0 +1,66 @@
+# -*- text -*-
+#
+# Copyright (c) 2004-2009 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2006-2022 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2012-2014 Los Alamos National Security, LLC.
+#                         All rights reserved.
+# Copyright (c) 2014      Research Organization for Information Science
+#                         and Technology (RIST). All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+# This is the US/English help file for Open MPI's KNEM smsc support.
+#
+[knem permission denied]
+WARNING: Open MPI failed to open the /dev/knem device due to a
+permissions problem.  Please check with your system administrator to
+get the permissions fixed, or set the smsc MCA variable to "^knem" to
+silence this warning and run without knem support.
+
+  Local host:            %s
+  /dev/knem permissions: 0%o
+#
+[knem fail open]
+WARNING: Open MPI failed to open the /dev/knem device due to a local
+error. Please check with your system administrator to get the problem
+fixed, or set the smsc MCA variable to "^knem" to silence this warning
+and run without knem support.
+
+Open MPI will try to fall back on another single-copy mechanism if one
+is available.  This may result in lower performance.
+
+  Local host: %s
+  Errno:      %d (%s)
+#
+[knem get ABI fail]
+WARNING: Open MPI failed to retrieve the ABI version from the
+/dev/knem device due to a local error.  This usually indicates an
+error in your knem installation; please check with your system
+administrator, or set the smsc MCA variable to "^knem" to silence this
+warning and run without knem support.
+
+Open MPI will try to fall back on another single-copy mechanism if one
+is available.  This may result in lower performance.
+
+  Local host: %s
+  Errno:      %d (%s)
+#
+[knem ABI mismatch]
+WARNING: Open MPI was compiled with support for one version of the
+knem kernel module, but it discovered a different version running in
+/dev/knem. Open MPI needs to be installed with support for the same
+version of knem as is in the running Linux kernel. Please check with
+your system administrator, or set the smsc MCA variable to "^knem" to
+silence this warning and run without knem support.
+
+Open MPI will try to fall back on another single-copy mechanism if one
+is available.  This may result in lower performance.
+
+  Local host:              %s
+  Open MPI's knem version: 0x%x
+  /dev/knem's version:     0x%x
diff --git a/opal/mca/smsc/knem/post_configure.sh b/opal/mca/smsc/knem/post_configure.sh
new file mode 100644
index 00000000000..fd7a0eb1abe
--- /dev/null
+++ b/opal/mca/smsc/knem/post_configure.sh
@@ -0,0 +1 @@
+DIRECT_CALL_HEADER="opal/mca/smsc/knem/smsc_knem.h"
diff --git a/opal/mca/smsc/knem/smsc_knem.h b/opal/mca/smsc/knem/smsc_knem.h
new file mode 100644
index 00000000000..824b7ad7b39
--- /dev/null
+++ b/opal/mca/smsc/knem/smsc_knem.h
@@ -0,0 +1,34 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2021      Google, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef OPAL_MCA_SMSC_KNEM_SMSC_KNEM_H
+#define OPAL_MCA_SMSC_KNEM_SMSC_KNEM_H
+
+#include "opal_config.h"
+
+#include "opal/mca/smsc/smsc.h"
+
+mca_smsc_endpoint_t *mca_smsc_knem_get_endpoint(opal_proc_t *peer_proc);
+void mca_smsc_knem_return_endpoint(mca_smsc_endpoint_t *endpoint);
+
+int mca_smsc_knem_copy_to(mca_smsc_endpoint_t *endpoint, void *local_address, void *remote_address,
+                          size_t size, void *reg_data);
+int mca_smsc_knem_copy_from(mca_smsc_endpoint_t *endpoint, void *local_address,
+                            void *remote_address, size_t size, void *reg_data);
+
+void *mca_smsc_knem_register_region(void *local_address, size_t size);
+void mca_smsc_knem_deregister_region(void *reg_data);
+
+/* unsupported interfaces defined to support MCA direct */
+void *mca_smsc_knem_map_peer_region(mca_smsc_endpoint_t *endpoint, uint64_t flags,
+                                    void *remote_address, size_t size, void **local_mapping);
+void mca_smsc_knem_unmap_peer_region(void *ctx);
+
+#endif /* OPAL_MCA_SMSC_KNEM_SMSC_KNEM_H */
diff --git a/opal/mca/smsc/knem/smsc_knem_component.c b/opal/mca/smsc/knem/smsc_knem_component.c
new file mode 100644
index 00000000000..bd5c0f5a645
--- /dev/null
+++ b/opal/mca/smsc/knem/smsc_knem_component.c
@@ -0,0 +1,224 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2021      Google, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "opal_config.h"
+
+#include "opal/mca/smsc/base/base.h"
+#include "opal/mca/smsc/knem/smsc_knem_internal.h"
+#include "opal/util/show_help.h"
+
+#include <fcntl.h>
+#include <stdio.h>
+#include <sys/prctl.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+static int mca_smsc_knem_component_register(void);
+static int mca_smsc_knem_component_open(void);
+static int mca_smsc_knem_component_close(void);
+static int mca_smsc_knem_component_query(void);
+static mca_smsc_module_t *mca_smsc_knem_component_enable(void);
+
+static int mca_smsc_knem_fini(void);
+
+#define MCA_SMSC_KNEM_DEFAULT_PRIORITY 23
+static const int mca_smsc_knem_default_priority = MCA_SMSC_KNEM_DEFAULT_PRIORITY;
+
+mca_smsc_knem_component_t mca_smsc_knem_component = {
+    .super = {
+        .smsc_version = {
+            MCA_SMSC_DEFAULT_VERSION("knem"),
+            .mca_open_component = mca_smsc_knem_component_open,
+            .mca_close_component = mca_smsc_knem_component_close,
+            .mca_register_component_params = mca_smsc_knem_component_register,
+        },
+        .priority = MCA_SMSC_KNEM_DEFAULT_PRIORITY,
+        .query = mca_smsc_knem_component_query,
+        .enable = mca_smsc_knem_component_enable,
+    },
+};
+
+static int mca_smsc_knem_component_register(void)
+{
+    /* Currently disabling DMA mode by default; it's not clear that this is useful in all
+     * applications and architectures. */
+    mca_smsc_knem_component.dma_min = 0;
+    (void) mca_base_component_var_register(
+        &mca_smsc_knem_component.super.smsc_version, "dma_min",
+        "Minimum message size (in bytes) to use the knem DMA mode; "
+        "ignored if knem does not support DMA mode (0 = do not use the "
+        "knem DMA mode, default: 0)",
+        MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+        &mca_smsc_knem_component.dma_min);
+
+    mca_smsc_base_register_default_params(&mca_smsc_knem_component.super,
+                                          mca_smsc_knem_default_priority);
+    return OPAL_SUCCESS;
+}
+
+static int mca_smsc_knem_component_open(void)
+{
+    /* nothing to do */
+    return OPAL_SUCCESS;
+}
+
+static int mca_smsc_knem_component_close(void)
+{
+    return mca_smsc_knem_fini();
+}
+
+static int mca_smsc_knem_get_info(struct knem_cmd_info *knem_info)
+{
+    /* Only show the help message if this is the only component. */
+    bool show_help = (1 == opal_list_get_size(&opal_smsc_base_framework.framework_components));
+
+    /* Check that the ABI if kernel module running is the same
+     * as what we were compiled against. */
+    memset(knem_info, 0, sizeof(*knem_info));
+    int rc = ioctl(mca_smsc_knem_component.knem_fd, KNEM_CMD_GET_INFO, knem_info);
+    if (rc < 0) {
+        if (show_help) {
+            opal_show_help("help-smsc-knem.txt", "knem get ABI fail", true,
+                           opal_process_info.nodename, errno, strerror(errno));
+        }
+        return OPAL_ERR_NOT_AVAILABLE;
+    }
+
+    if (KNEM_ABI_VERSION != knem_info->abi) {
+        if (show_help) {
+            opal_show_help("help-smsc-knem.txt", "knem ABI mismatch", true,
+                           opal_process_info.nodename, KNEM_ABI_VERSION, knem_info->abi);
+        }
+        return OPAL_ERR_NOT_AVAILABLE;
+    }
+
+    return OPAL_SUCCESS;
+}
+static int mca_smsc_knem_reg(void *reg_data, void *base, size_t size,
+                             mca_rcache_base_registration_t *reg)
+{
+    mca_smsc_knem_registration_handle_t *knem_reg = (mca_smsc_knem_registration_handle_t *) reg;
+    struct knem_cmd_create_region knem_cr;
+    struct knem_cmd_param_iovec knem_iov;
+
+    knem_iov.base = (uintptr_t) base;
+    knem_iov.len = size;
+
+    knem_cr.iovec_array = (uintptr_t) &knem_iov;
+    knem_cr.iovec_nr = 1;
+    knem_cr.protection = 0;
+
+    if (reg->access_flags & (MCA_RCACHE_ACCESS_LOCAL_WRITE | MCA_RCACHE_ACCESS_REMOTE_WRITE)) {
+        knem_cr.protection |= PROT_WRITE;
+    }
+
+    if (reg->access_flags & MCA_RCACHE_ACCESS_REMOTE_READ) {
+        knem_cr.protection |= PROT_READ;
+    }
+
+    /* We will explicitly destroy this cookie. Do not use the single-use flag here. */
+    knem_cr.flags = 0;
+    if (OPAL_UNLIKELY(ioctl(mca_smsc_knem_component.knem_fd, KNEM_CMD_CREATE_REGION, &knem_cr)
+                      < 0)) {
+        return OPAL_ERROR;
+    }
+
+    knem_reg->data.cookie = knem_cr.cookie;
+    knem_reg->data.base_addr = (intptr_t) base;
+
+    return OPAL_SUCCESS;
+}
+
+static int mca_smsc_knem_dereg(void *reg_data, mca_rcache_base_registration_t *reg)
+{
+    mca_smsc_knem_registration_handle_t *knem_reg = (mca_smsc_knem_registration_handle_t *) reg;
+
+    /* NTH: explicitly ignore the return code. Don't care about this cookie anymore anyway. */
+    (void) ioctl(mca_smsc_knem_component.knem_fd, KNEM_CMD_DESTROY_REGION, &knem_reg->data.cookie);
+
+    return OPAL_SUCCESS;
+}
+
+static int mca_smsc_knem_fini(void)
+{
+    if (-1 != mca_smsc_knem_component.knem_fd) {
+        close(mca_smsc_knem_component.knem_fd);
+        mca_smsc_knem_component.knem_fd = -1;
+    }
+
+    if (mca_smsc_knem_module.rcache) {
+        (void) mca_rcache_base_module_destroy(mca_smsc_knem_module.rcache);
+        mca_smsc_knem_module.rcache = NULL;
+    }
+
+    return OPAL_SUCCESS;
+}
+
+static int mca_smsc_knem_component_query(void)
+{
+    struct knem_cmd_info knem_info;
+    int rc;
+
+    /* Open the knem device.  Try to print a helpful message if we
+       fail to open it. */
+    mca_smsc_knem_component.knem_fd = open("/dev/knem", O_RDWR);
+    if (mca_smsc_knem_component.knem_fd < 0) {
+        if (EACCES == errno) {
+            struct stat sbuf;
+            if (0 != stat("/dev/knem", &sbuf)) {
+                sbuf.st_mode = 0;
+            }
+            opal_show_help("help-smsc-knem.txt", "knem permission denied", true,
+                           opal_process_info.nodename, sbuf.st_mode);
+        } else {
+            opal_show_help("help-smsc-knem.txt", "knem fail open", true, opal_process_info.nodename,
+                           errno, strerror(errno));
+        }
+
+        return OPAL_ERR_NOT_AVAILABLE;
+    }
+
+    rc = mca_smsc_knem_get_info(&knem_info);
+    if (OPAL_SUCCESS != rc) {
+        mca_smsc_knem_fini();
+        return rc;
+    }
+
+    if (!(mca_smsc_knem_component.dma_min && (knem_info.features & KNEM_FEATURE_DMA))) {
+        /* disable DMA */
+        mca_smsc_knem_component.dma_min = UINT_MAX;
+    }
+
+    return OPAL_SUCCESS;
+}
+
+static mca_smsc_module_t *mca_smsc_knem_component_enable(void)
+{
+    if (0 > mca_smsc_knem_component.super.priority) {
+        return NULL;
+    }
+
+    mca_rcache_base_resources_t rcache_resources = {.cache_name = "smsc_knem",
+                                                    .reg_data = NULL,
+                                                    .sizeof_reg = sizeof(
+                                                        mca_smsc_knem_registration_handle_t),
+                                                    .register_mem = mca_smsc_knem_reg,
+                                                    .deregister_mem = mca_smsc_knem_dereg};
+
+    mca_smsc_knem_module.rcache = mca_rcache_base_module_create("grdma", NULL, &rcache_resources);
+    if (NULL == mca_smsc_knem_module.rcache) {
+        return NULL;
+    }
+
+    return &mca_smsc_knem_module.super;
+}
diff --git a/opal/mca/smsc/knem/smsc_knem_internal.h b/opal/mca/smsc/knem/smsc_knem_internal.h
new file mode 100644
index 00000000000..60a8ddbf676
--- /dev/null
+++ b/opal/mca/smsc/knem/smsc_knem_internal.h
@@ -0,0 +1,74 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2021      Google, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef OPAL_MCA_SMSC_KNEM_SMSC_KNEM_INTERNAL_H
+#define OPAL_MCA_SMSC_KNEM_SMSC_KNEM_INTERNAL_H
+
+#include "opal_config.h"
+
+#include "opal/mca/rcache/base/base.h"
+#include "opal/mca/rcache/rcache.h"
+#include "opal/mca/smsc/knem/smsc_knem.h"
+
+#include <sys/mman.h>
+
+#include <knem_io.h>
+
+struct mca_smsc_knem_registration_data_t {
+    uint64_t cookie;
+    intptr_t base_addr;
+};
+
+typedef struct mca_smsc_knem_registration_data_t mca_smsc_knem_registration_data_t;
+
+struct mca_smsc_knem_registration_handle_t {
+    mca_rcache_base_registration_t base;
+    mca_smsc_knem_registration_data_t data;
+};
+
+typedef struct mca_smsc_knem_registration_handle_t mca_smsc_knem_registration_handle_t;
+
+#define MCA_SMSC_KNEM_REG_HANDLE_TO_DATA(handle) (&(handle)->data)
+#define MCA_SMSC_KNEM_REG_DATA_TO_HANDLE(data_ptr)                                            \
+    ((mca_smsc_knem_registration_handle_t *) ((uintptr_t) data_ptr                            \
+                                              - offsetof(mca_smsc_knem_registration_handle_t, \
+                                                         data)))
+
+struct mca_smsc_knem_endpoint_t {
+    mca_smsc_endpoint_t super;
+};
+
+typedef struct mca_smsc_knem_endpoint_t mca_smsc_knem_endpoint_t;
+
+OBJ_CLASS_DECLARATION(mca_smsc_knem_endpoint_t);
+
+struct mca_smsc_knem_component_t {
+    mca_smsc_component_t super;
+
+    int knem_fd;
+    unsigned int dma_min;
+};
+
+typedef struct mca_smsc_knem_component_t mca_smsc_knem_component_t;
+
+struct mca_smsc_knem_module_t {
+    mca_smsc_module_t super;
+
+    /** cache of knem attachments. this cache holds attachments for all peers. the registrations
+     * are differentiated by the alloc_base which is set to the endpoint. */
+    mca_rcache_base_module_t *rcache;
+};
+
+typedef struct mca_smsc_knem_module_t mca_smsc_knem_module_t;
+
+extern mca_smsc_knem_module_t mca_smsc_knem_module;
+extern mca_smsc_knem_component_t mca_smsc_knem_component;
+
+#endif /* OPAL_MCA_SMSC_KNEM_SMSC_KNEM_INTERNAL_H */
diff --git a/opal/mca/smsc/knem/smsc_knem_module.c b/opal/mca/smsc/knem/smsc_knem_module.c
new file mode 100644
index 00000000000..6b38ddd3b7b
--- /dev/null
+++ b/opal/mca/smsc/knem/smsc_knem_module.c
@@ -0,0 +1,158 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2014      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2020-2021 Google, LLC. All rights reserved.
+ * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "opal_config.h"
+
+#include "opal/include/opal/align.h"
+#include "opal/mca/memchecker/base/base.h"
+#include "opal/mca/pmix/pmix-internal.h"
+#include "opal/mca/rcache/rcache.h"
+#include "opal/mca/smsc/base/base.h"
+#include "opal/mca/smsc/knem/smsc_knem_internal.h"
+#include "opal/util/minmax.h"
+
+OBJ_CLASS_INSTANCE(mca_smsc_knem_endpoint_t, opal_object_t, NULL, NULL);
+
+mca_smsc_endpoint_t *mca_smsc_knem_get_endpoint(opal_proc_t *peer_proc)
+{
+    mca_smsc_knem_endpoint_t *endpoint = OBJ_NEW(mca_smsc_knem_endpoint_t);
+    if (OPAL_UNLIKELY(NULL == endpoint)) {
+        return NULL;
+    }
+
+    endpoint->super.proc = peer_proc;
+    return &endpoint->super;
+}
+
+void mca_smsc_knem_return_endpoint(mca_smsc_endpoint_t *endpoint)
+{
+    OBJ_RELEASE(endpoint);
+}
+
+void *mca_smsc_knem_register_region(void *local_address, size_t size)
+{
+    mca_smsc_knem_module_t *knem_module = &mca_smsc_knem_module;
+    mca_smsc_knem_registration_handle_t *reg = NULL;
+    int rc;
+
+    rc = knem_module->rcache->rcache_register(knem_module->rcache, local_address, size,
+                                              /*flags=*/0, MCA_RCACHE_ACCESS_ANY,
+                                              (mca_rcache_base_registration_t **) &reg);
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
+        opal_output_verbose(
+            MCA_BASE_VERBOSE_WARN, opal_smsc_base_framework.framework_output,
+            "mca_smsc_knem_register_mem: failed to register memory for single-copy");
+        return NULL;
+    }
+
+    return MCA_SMSC_KNEM_REG_HANDLE_TO_DATA(reg);
+}
+
+void mca_smsc_knem_deregister_region(void *reg_data)
+{
+    mca_smsc_knem_module_t *knem_module = &mca_smsc_knem_module;
+    mca_smsc_knem_registration_handle_t *reg = MCA_SMSC_KNEM_REG_DATA_TO_HANDLE(reg_data);
+
+    knem_module->rcache->rcache_deregister(knem_module->rcache, &reg->base);
+}
+
+static int mca_smsc_knem_module_copy(mca_smsc_endpoint_t *endpoint, void *local_address,
+                                     void *remote_address, size_t size, void *reg_data,
+                                     bool is_write)
+{
+    if (OPAL_UNLIKELY(NULL == reg_data)) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
+    struct knem_cmd_param_iovec send_iovec = {
+        .base = (uintptr_t) local_address,
+        .len = size,
+    };
+    mca_smsc_knem_registration_data_t *reg = (mca_smsc_knem_registration_data_t *) reg_data;
+    /* Fill in the ioctl data fields.  There's no async completion, so
+       we don't need to worry about getting a slot, etc. */
+    struct knem_cmd_inline_copy icopy = {
+        .local_iovec_array = (uintptr_t) &send_iovec,
+        .local_iovec_nr = 1,
+        .remote_cookie = reg->cookie,
+        .remote_offset = (uintptr_t) remote_address - reg->base_addr,
+        .write = is_write,
+        .flags = 0,
+    };
+
+    /* Use the DMA flag if knem supports it *and* the segment length
+     * is greater than the cutoff. Not that if DMA is not supported
+     * or the user specified 0 for knem_dma_min the knem_dma_min was
+     * set to UINT_MAX in mca_smsc_knem_query. */
+    if (mca_smsc_knem_component.dma_min <= size) {
+        icopy.flags = KNEM_FLAG_DMA;
+    }
+    /* synchronous flags only, no need to specify icopy.async_status_index */
+
+    /* When the ioctl returns, the transfer is done and we can invoke
+       the btl callback and return the frag */
+    if (OPAL_UNLIKELY(0 != ioctl(mca_smsc_knem_component.knem_fd, KNEM_CMD_INLINE_COPY, &icopy))) {
+        opal_output_verbose(MCA_BASE_VERBOSE_WARN, opal_smsc_base_framework.framework_output,
+                            "mca_smsc_knem_module_copy: failed to intiate transfer");
+        return OPAL_ERROR;
+    }
+
+    if (KNEM_STATUS_FAILED == icopy.current_status) {
+        opal_output_verbose(MCA_BASE_VERBOSE_WARN, opal_smsc_base_framework.framework_output,
+                            "mca_smsc_knem_module_copy: transfter failed");
+        return OPAL_ERROR;
+    }
+
+    return OPAL_SUCCESS;
+}
+
+int mca_smsc_knem_copy_to(mca_smsc_endpoint_t *endpoint, void *local_address, void *remote_address,
+                          size_t size, void *reg_data)
+{
+    return mca_smsc_knem_module_copy(endpoint, local_address, remote_address, size, reg_data,
+                                     /*is_write=*/true);
+}
+
+int mca_smsc_knem_copy_from(mca_smsc_endpoint_t *endpoint, void *local_address,
+                            void *remote_address, size_t size, void *reg_data)
+{
+    return mca_smsc_knem_module_copy(endpoint, local_address, remote_address, size, reg_data,
+                                     /*is_write=*/false);
+}
+
+/* unsupported interfaces (for MCA direct) */
+void *mca_smsc_knem_map_peer_region(mca_smsc_endpoint_t *endpoint, uint64_t flags,
+                                    void *remote_address, size_t size, void **local_mapping)
+{
+    return NULL;
+}
+
+void mca_smsc_knem_unmap_peer_region(void *ctx)
+{
+}
+
+mca_smsc_knem_module_t mca_smsc_knem_module = {
+    .super = {
+        .features = MCA_SMSC_FEATURE_REQUIRE_REGISTRATION,
+        .registration_data_size = sizeof(mca_smsc_knem_registration_data_t),
+        .get_endpoint = mca_smsc_knem_get_endpoint,
+        .return_endpoint = mca_smsc_knem_return_endpoint,
+        .copy_to = mca_smsc_knem_copy_to,
+        .copy_from = mca_smsc_knem_copy_from,
+        .register_region = mca_smsc_knem_register_region,
+        .deregister_region = mca_smsc_knem_deregister_region,
+    }, 
+};
diff --git a/opal/mca/smsc/smsc.h b/opal/mca/smsc/smsc.h
new file mode 100644
index 00000000000..ddbf5402672
--- /dev/null
+++ b/opal/mca/smsc/smsc.h
@@ -0,0 +1,260 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2021      Google, LLC. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+/**
+ * @file
+ *
+ * Shared Memory Single Copy
+ *
+ * This framework provides support for copying memory from one process to another on the same host
+ * system. The components expose process read, write, and may provide a way to map peer memory into
+ * this processes memory space.
+ */
+
+#ifndef OPAL_MCA_SMSC_H
+#define OPAL_MCA_SMSC_H
+
+#include "opal_config.h"
+#include "opal/class/opal_object.h"
+#include "opal/util/proc.h"
+
+#define MCA_SMSC_BASE_MAJOR_VERSION 1
+#define MCA_SMSC_BASE_MINOR_VERSION 0
+#define MCA_SMSC_BASE_PATCH_VERSION 0
+
+struct mca_smsc_module_t;
+
+struct mca_smsc_endpoint_t {
+    opal_object_t super;
+    /** Opal proc object for this peer. */
+    opal_proc_t *proc;
+};
+
+typedef struct mca_smsc_endpoint_t mca_smsc_endpoint_t;
+
+OBJ_CLASS_DECLARATION(mca_smsc_endpoint_t);
+
+/**
+ * @brief Get an endpoint for a peer proc.
+ *
+ * @param(in) module    shared-memory single-copy module
+ * @param(in) peer_proc proc to get an endpoint for
+ */
+typedef mca_smsc_endpoint_t *(*mca_smsc_module_get_endpoint_fn_t)(opal_proc_t *peer_proc);
+
+/**
+ * @brief Return a shared-memory single-copy endpoint.
+ *
+ * @param(in) module   shared-memory single-copy module
+ * @param(in) endpoint shared-memory single-copy endpoint
+ *
+ * This method returns an endpoint created by get_endpoint. The endpoint should be considered
+ * invalid and may be freed after this call completes.
+ */
+typedef void (*mca_smsc_module_return_endpoint_fn_t)(mca_smsc_endpoint_t *endpoint);
+
+/**
+ * @brief Copy to/from a peer process.
+ *
+ * @param(in) module         shared-memory single-copy module
+ * @param(in) endpoint       shared-memory single-copy endpoint
+ * @param(in) local_address  local address to use
+ * @param(in) remote_address remote address to use
+ * @param(in) size           amount to copy
+ * @param(in) reg_data       pointer to memory containing registration data (if required)
+ *
+ * A module must provide both copy_from and copy_to function.
+ */
+typedef int (*mca_smsc_module_copy_fn_t)(mca_smsc_endpoint_t *endpoint, void *local_address,
+                                         void *remote_address, size_t size, void *reg_data);
+
+/**
+ * @brief Map a peer's memory onto local memory.
+ *
+ * @param(in)  module         shared-memory single-copy module
+ * @param(in)  endpoint       shared-memory single-copy endpoint
+ * @param(in)  flags          flags for this map operation (set to 0)
+ * @param(in)  remote_address pointer valid in peer's address space
+ * @param(in)  size           size of region to map
+ * @param(out) local_mapping local address for peer region
+ *
+ * @returns a reference to the mapping
+ *
+ * This method, if provided, provides support for mapping a local peer's memory into this address
+ * space. The caller is responsible for verifying that the address is valid or access to the region
+ * may result in an access violation (SEGV). The function returns a reference (if needed) that can
+ * be used to clear the mapping. It is the caller's responsibility to unmap the region using the
+ * returned context.
+ */
+typedef void *(*mca_smsc_module_map_peer_region_fn_t)(mca_smsc_endpoint_t *endpoint, uint64_t flags,
+                                                      void *remote_address, size_t size,
+                                                      void **local_mapping);
+
+/**
+ * @brief Clear a memory mapping.
+ *
+ * @param(in)  module         shared-memory single-copy module
+ * @param(in) ctx   memory mapping context
+ */
+typedef void (*mca_smsc_module_unmap_peer_region_fn_t)(void *ctx);
+
+/**
+ * @brief Register a memory region for remote access.
+ *
+ * @param(in) module          shared-memory single-copy module
+ * @param(in) local_address   local address to register (ideally page-aligned)
+ * @param(in) size            size of the memory region (ideally page-aligned)
+ *
+ * @returns a pointer to registration data that can be used for copy by a peer process
+ *
+ * This method registers a region for access by a local peer. The returned data can be passed to a
+ * local peer and used by that peer for either copy_to or copy_from.
+ */
+typedef void *(*mca_smsc_module_register_region_fn_t)(void *local_address, size_t size);
+
+/**
+ * @brief Deregister a registered region.
+ *
+ * @param(in) module     shared-memory single-copy module
+ * @param(in) reg_data   registration data returned by the registration function
+ *
+ * This function deregisters a region from use by a peers copy_from and copy_to function. Once a
+ * region has been deregistered the data is immediately not usable by any local peer.
+ */
+typedef void (*mca_smsc_module_deregister_region_fn_t)(void *reg_data);
+
+enum {
+    /** Module requires the local registration of any region that will be used for single-copy
+     * operations. It is theresponsibility of the caller to pass this data with the pointer to the
+     * peer. */
+    MCA_SMSC_FEATURE_REQUIRE_REGISTRATION = 1,
+    /** Module can map peer memory into the local processes' address space. */
+    MCA_SMSC_FEATURE_CAN_MAP = 2,
+    /** Module can handle accelerator memory types */
+    MCA_SMSC_FEATURE_ACCELERATOR = 4,
+};
+
+struct mca_smsc_module_t {
+    /** Module features. */
+    uint64_t features;
+
+    /** Ignore if MCA_SMSC_FEATURE_REQUIRES_REGISTRATION is not set. */
+    size_t registration_data_size;
+
+    /** Get an endpoint for a peer. This function should always return a newly-allocated endpoint.
+     * The base will be responsible for caching that endpoint. */
+    mca_smsc_module_get_endpoint_fn_t get_endpoint;
+    /** Delete an endpoint and clean up all resources associated with it. */
+    mca_smsc_module_return_endpoint_fn_t return_endpoint;
+
+    /* All components must provide an implementation of the copy functions. */
+    /** Copy data into a peer's memory space. */
+    mca_smsc_module_copy_fn_t copy_to;
+    /** Copy data from a peer's memory space. */
+    mca_smsc_module_copy_fn_t copy_from;
+
+    /* Defined if MCA_SMSC_FEATURE_CAN_MAP is set. */
+    /** Map a peer memory region into this processes address space. The module is allowed to cache
+     * the mapping and return it in subsequent calls. */
+    mca_smsc_module_map_peer_region_fn_t map_peer_region;
+    /** Delete a mapping. This is allowed to leave the mapping in place. */
+    mca_smsc_module_unmap_peer_region_fn_t unmap_peer_region;
+
+    /* Defined if MCA_SMSC_FEATURE_REQUIRES_REGISTRATION is set. */
+    /** Register a memory region for use with single-copy by a remote peer. The module may cache
+     * this registration for future use. */
+    mca_smsc_module_register_region_fn_t register_region;
+    /** Deregister a memory region for use with single-copy. */
+    mca_smsc_module_deregister_region_fn_t deregister_region;
+};
+
+typedef struct mca_smsc_module_t mca_smsc_module_t;
+
+/**
+ * @brief Query if this component can run.
+ *
+ * @returns OPAL_SUCCESS if the component can run or an opal error code otherwise
+ *
+ * This function is responsible for verifying the component can run. It should do the minimum amount
+ * of work to run at any time during execution. This includes sending any modex message if needed.
+ * It should refrain from allocating resources if possible.
+ */
+typedef int (*mca_smsc_component_query_fn_t)(void);
+
+/**
+ * @brief Enable the use of this component and return a module.
+ *
+ * @returns A module on success or NULL otherwise.
+ *
+ * This function should do any remaining work (not already done in query) to prepare the component
+ * for use. It should return a fully initialized module.
+ */
+typedef mca_smsc_module_t *(*mca_smsc_component_enable_fn_t)(void);
+
+struct mca_smsc_component_1_0_0_t {
+    mca_base_component_t smsc_version;
+    mca_base_component_data_t smsc_data;
+
+    /** Priority of this component. Only the winning component will be used. */
+    int priority;
+
+    /** Check if this component can be used. */
+    mca_smsc_component_query_fn_t query;
+    /** Enable the use of this component. */
+    mca_smsc_component_enable_fn_t enable;
+};
+
+typedef struct mca_smsc_component_1_0_0_t mca_smsc_component_1_0_0_t;
+typedef mca_smsc_component_1_0_0_t mca_smsc_component_t;
+
+OPAL_DECLSPEC extern mca_smsc_module_t *mca_smsc;
+
+#if MCA_opal_smsc_DIRECT_CALL
+#    include MCA_opal_smsc_DIRECT_CALL_HEADER
+
+#    define MCA_SMSC_CALL_STAMP(a, b, ...)    mca_smsc_##a##_##b(__VA_ARGS__)
+#    define MCA_SMSC_CALL_EXPANDER(a, b, ...) MCA_SMSC_CALL_STAMP(a, b, __VA_ARGS__)
+#    define MCA_SMSC_CALL(a, ...) \
+        MCA_SMSC_CALL_EXPANDER(MCA_opal_smsc_DIRECT_CALL_COMPONENT, a, __VA_ARGS__)
+
+#else
+
+#    define MCA_SMSC_CALL(a, ...) mca_smsc->a(__VA_ARGS__)
+
+#endif /* MCA_opal_smsc_DIRECT_CALL */
+
+/**
+ * @brief Check if the selected component has a feature.
+ *
+ * @param(in) feature  feature to check for (see smsc.h for list of features)
+ */
+static inline bool mca_smsc_base_has_feature(uint64_t feature)
+{
+    return (NULL != mca_smsc) && !!(mca_smsc->features & feature);
+}
+
+static inline ssize_t mca_smsc_base_registration_data_size(void)
+{
+    if (NULL == mca_smsc || !mca_smsc_base_has_feature(MCA_SMSC_FEATURE_REQUIRE_REGISTRATION)) {
+        return OPAL_ERR_NOT_AVAILABLE;
+    }
+
+    return mca_smsc->registration_data_size;
+}
+
+#define MCA_SMSC_BASE_VERSION_1_0_0                                                               \
+    OPAL_MCA_BASE_VERSION_2_1_0("smsc", MCA_SMSC_BASE_MAJOR_VERSION, MCA_SMSC_BASE_MINOR_VERSION, \
+                                MCA_SMSC_BASE_PATCH_VERSION)
+
+#define MCA_SMSC_DEFAULT_VERSION(name)                                                \
+    MCA_SMSC_BASE_VERSION_1_0_0, .mca_component_name = name,                          \
+                                 MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, \
+                                                       OPAL_MINOR_VERSION, OPAL_RELEASE_VERSION)
+
+#endif /* OPAL_MCA_SMSC_H */
diff --git a/opal/mca/smsc/xpmem/Makefile.am b/opal/mca/smsc/xpmem/Makefile.am
new file mode 100644
index 00000000000..1944d3ef24e
--- /dev/null
+++ b/opal/mca/smsc/xpmem/Makefile.am
@@ -0,0 +1,56 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2009 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2009-2014 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights
+#                         reserved.
+# Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2020-2021 Google, LLC. All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+EXTRA_DIST = post_configure.sh
+
+AM_CPPFLAGS = $(smsc_xpmem_CPPFLAGS)
+
+libmca_smsc_xpmem_la_sources = \
+    smsc_xpmem_component.c \
+    smsc_xpmem_module.c \
+    smsc_xpmem_internal.h \
+    smsc_xpmem.h
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_opal_smsc_xpmem_DSO
+component_noinst =
+component_install = mca_smsc_xpmem.la
+else
+component_noinst = libmca_smsc_xpmem.la
+component_install =
+endif
+
+mcacomponentdir = $(opallibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_smsc_xpmem_la_SOURCES = $(libmca_smsc_xpmem_la_sources)
+mca_smsc_xpmem_la_LDFLAGS = -module -avoid-version $(smsc_xpmem_LDFLAGS)
+mca_smsc_xpmem_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
+	$(smsc_xpmem_LIBS)
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_smsc_xpmem_la_SOURCES = $(libmca_smsc_xpmem_la_sources)
+libmca_smsc_xpmem_la_LIBADD = $(smsc_xpmem_LIBS)
+libmca_smsc_xpmem_la_LDFLAGS = -module -avoid-version $(smsc_xpmem_LDFLAGS)
diff --git a/opal/mca/smsc/xpmem/configure.m4 b/opal/mca/smsc/xpmem/configure.m4
new file mode 100644
index 00000000000..d666015b099
--- /dev/null
+++ b/opal/mca/smsc/xpmem/configure.m4
@@ -0,0 +1,31 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2009      The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2009-2016 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights
+#                         reserved.
+# Copyright (c) 2015      Research Organization for Information Science
+#                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2021      Google, LLC. All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# MCA_smsc_xpmem_CONFIG([action-if-can-compile],
+#                      [action-if-cant-compile])
+# ------------------------------------------------
+AC_DEFUN([MCA_opal_smsc_xpmem_CONFIG],[
+    AC_CONFIG_FILES([opal/mca/smsc/xpmem/Makefile])
+
+    OPAL_CHECK_XPMEM([smsc_xpmem], [$1], [$2])
+
+    AC_SUBST([smsc_xpmem_CPPFLAGS])
+    AC_SUBST([smsc_xpmem_LDFLAGS])
+    AC_SUBST([smsc_xpmem_LIBS])
+])dnl
diff --git a/opal/mca/smsc/xpmem/post_configure.sh b/opal/mca/smsc/xpmem/post_configure.sh
new file mode 100644
index 00000000000..a4d999a1161
--- /dev/null
+++ b/opal/mca/smsc/xpmem/post_configure.sh
@@ -0,0 +1 @@
+DIRECT_CALL_HEADER="opal/mca/smsc/xpmem/smsc_xpmem.h"
diff --git a/opal/mca/smsc/xpmem/smsc_xpmem.h b/opal/mca/smsc/xpmem/smsc_xpmem.h
new file mode 100644
index 00000000000..ae8d33efbb3
--- /dev/null
+++ b/opal/mca/smsc/xpmem/smsc_xpmem.h
@@ -0,0 +1,43 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2021      Google, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef OPAL_MCA_SMSC_XPMEM_SMSC_XPMEM_H
+#define OPAL_MCA_SMSC_XPMEM_SMSC_XPMEM_H
+
+#include "opal_config.h"
+
+#include "opal/mca/smsc/smsc.h"
+
+mca_smsc_endpoint_t *mca_smsc_xpmem_get_endpoint(opal_proc_t *peer_proc);
+void mca_smsc_xpmem_return_endpoint(mca_smsc_endpoint_t *endpoint);
+
+int mca_smsc_xpmem_copy_to(mca_smsc_endpoint_t *endpoint, void *local_address, void *remote_address,
+                           size_t size, void *reg_handle);
+
+int mca_smsc_xpmem_copy_from(mca_smsc_endpoint_t *endpoint, void *local_address,
+                             void *remote_address, size_t size, void *reg_handle);
+
+/**
+ * @brief Map a peer memory region into this processes address space.
+ *
+ * See the description in smsc.h.
+ *
+ * Caveats: XPMEM does not support futex operations within the region. Attempts to wake the
+ * process owning the mutex will result in an EFAULT error code.
+ */
+void *mca_smsc_xpmem_map_peer_region(mca_smsc_endpoint_t *endpoint, uint64_t flags,
+                                     void *remote_ptr, size_t size, void **local_ptr);
+void mca_smsc_xpmem_unmap_peer_region(void *ctx);
+
+/* unsupported interfaces defined to support MCA direct */
+void *mca_smsc_xpmem_register_region(void *local_address, size_t size);
+void mca_smsc_xpmem_deregister_region(void *reg_data);
+
+#endif /* OPAL_MCA_SMSC_XPMEM_SMSC_XPMEM_H */
diff --git a/opal/mca/smsc/xpmem/smsc_xpmem_component.c b/opal/mca/smsc/xpmem/smsc_xpmem_component.c
new file mode 100644
index 00000000000..0b48e95d712
--- /dev/null
+++ b/opal/mca/smsc/xpmem/smsc_xpmem_component.c
@@ -0,0 +1,164 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2021      Google, Inc. All rights reserved.
+ * Copyright (c) 2022      Computer Architecture and VLSI Systems (CARV)
+ *                         Laboratory, ICS Forth. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "opal_config.h"
+
+#include "opal/mca/smsc/base/base.h"
+#include "opal/mca/smsc/xpmem/smsc_xpmem_internal.h"
+#include "opal/util/minmax.h"
+
+#include <fcntl.h>
+#include <stdio.h>
+#include <sys/prctl.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+static int mca_smsc_xpmem_component_register(void);
+static int mca_smsc_xpmem_component_open(void);
+static int mca_smsc_xpmem_component_close(void);
+static int mca_smsc_xpmem_component_query(void);
+static mca_smsc_module_t *mca_smsc_xpmem_component_enable(void);
+
+#define MCA_SMSC_XPMEM_DEFAULT_PRIORITY 42
+static const int mca_smsc_xpmem_default_priority = MCA_SMSC_XPMEM_DEFAULT_PRIORITY;
+
+mca_smsc_xpmem_component_t mca_smsc_xpmem_component = {
+    .super = {
+        .smsc_version = {
+            MCA_SMSC_DEFAULT_VERSION("xpmem"),
+            .mca_open_component = mca_smsc_xpmem_component_open,
+            .mca_close_component = mca_smsc_xpmem_component_close,
+            .mca_register_component_params = mca_smsc_xpmem_component_register,
+        },
+        .priority = MCA_SMSC_XPMEM_DEFAULT_PRIORITY,
+        .query = mca_smsc_xpmem_component_query,
+        .enable = mca_smsc_xpmem_component_enable,
+    },
+};
+
+static int mca_smsc_xpmem_component_register(void)
+{
+    mca_smsc_xpmem_component.log_attach_align = 23;
+    (void) mca_base_component_var_register(&mca_smsc_xpmem_component.super.smsc_version,
+                                           "log_align",
+                                           "Log base 2 of the alignment to use for xpmem "
+                                           "segments (default: 23, minimum: 12, maximum: 25)",
+                                           MCA_BASE_VAR_TYPE_INT, /*enumerator=*/NULL, /*bind=*/0,
+                                           MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5,
+                                           MCA_BASE_VAR_SCOPE_LOCAL,
+                                           &mca_smsc_xpmem_component.log_attach_align);
+
+    mca_smsc_xpmem_component.memcpy_chunk_size = 262144;
+    (void) mca_base_component_var_register(
+        &mca_smsc_xpmem_component.super.smsc_version, "memcpy_chunk_size",
+        "Maximum size to copy with a single call to memcpy. On some systems a smaller or larger "
+        "number may provide better performance (default: 256k)",
+        MCA_BASE_VAR_TYPE_UINT64_T, /*enumerator=*/NULL, /*bind=*/0, MCA_BASE_VAR_FLAG_SETTABLE,
+        OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_smsc_xpmem_component.memcpy_chunk_size);
+
+    mca_smsc_base_register_default_params(&mca_smsc_xpmem_component.super,
+                                          mca_smsc_xpmem_default_priority);
+    return OPAL_SUCCESS;
+}
+
+static int mca_smsc_xpmem_component_open(void)
+{
+    /* nothing to do */
+    return OPAL_SUCCESS;
+}
+
+static int mca_smsc_xpmem_component_close(void)
+{
+    /* nothing to do */
+    return OPAL_SUCCESS;
+}
+
+static int mca_smsc_xpmem_send_modex(void)
+{
+    mca_smsc_xpmem_modex_t modex;
+
+    modex.seg_id = mca_smsc_xpmem_component.my_seg_id;
+    modex.address_max = mca_smsc_xpmem_component.my_address_max;
+
+    int rc;
+    OPAL_MODEX_SEND(rc, PMIX_LOCAL, &mca_smsc_xpmem_component.super.smsc_version, &modex,
+                    sizeof(modex));
+    return rc;
+}
+
+static int mca_smsc_xpmem_component_query(void)
+{
+    /* Any attachment that goes past the Linux TASK_SIZE will always fail. To prevent this we need
+     * to determine the value of TASK_SIZE. On x86_64 the value was hard-coded in sm to be
+     * 0x7ffffffffffful but this approach does not work with AARCH64 (and possibly other
+     * architectures). Since there is really no way to directly determine the value we can (in all
+     * cases?) look through the mapping for this process to determine what the largest address is.
+     * This should be the top of the stack. No heap allocations should be larger than this value.
+     * Since the largest address may differ between processes the value must be shared as part of
+     * the modex and stored in the endpoint. */
+    FILE *fh = fopen("/proc/self/maps", "r");
+    if (NULL == fh) {
+        opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT, opal_smsc_base_framework.framework_output,
+                            "mca_smsc_xpmem_component_query: could not open /proc/self/maps for "
+                            "reading. disabling XPMEM");
+        return OPAL_ERR_NOT_AVAILABLE;
+    }
+
+    char buffer[1024];
+    uintptr_t address_max = 0;
+    while (fgets(buffer, sizeof(buffer), fh)) {
+        uintptr_t high;
+        char *tmp;
+        /* each line of /proc/self/maps starts with low-high in hexadecimal (without a 0x) */
+        strtoul(buffer, &tmp, 16);
+        high = strtoul(tmp + 1, NULL, 16);
+        if (address_max < high) {
+            address_max = high;
+        }
+    }
+
+    fclose(fh);
+
+    if (0 == address_max) {
+        opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT, opal_smsc_base_framework.framework_output,
+                            "mca_smsc_xpmem_component_query: could not determine the address max");
+        return OPAL_ERR_NOT_AVAILABLE;
+    }
+
+    /* save the calculated maximum */
+    mca_smsc_xpmem_component.my_address_max = address_max - 1;
+
+    /* it is safe to use XPMEM_MAXADDR_SIZE here (which is always (size_t)-1 even though
+     * it is not safe for attach */
+    mca_smsc_xpmem_component.my_seg_id = xpmem_make(0, XPMEM_MAXADDR_SIZE, XPMEM_PERMIT_MODE,
+                                                    (void *) 0666);
+    if (-1 == mca_smsc_xpmem_component.my_seg_id) {
+        return OPAL_ERR_NOT_AVAILABLE;
+    }
+
+    mca_smsc_xpmem_send_modex();
+
+    return OPAL_SUCCESS;
+}
+
+static mca_smsc_module_t *mca_smsc_xpmem_component_enable(void)
+{
+    if (0 > mca_smsc_xpmem_component.super.priority) {
+        return NULL;
+    }
+
+    /* limit segment alignment to be between 4k and 16M */
+    mca_smsc_xpmem_component.log_attach_align
+        = opal_min(opal_max(mca_smsc_xpmem_component.log_attach_align, 12), 25);
+
+    return &mca_smsc_xpmem_module.super;
+}
diff --git a/opal/mca/smsc/xpmem/smsc_xpmem_internal.h b/opal/mca/smsc/xpmem/smsc_xpmem_internal.h
new file mode 100644
index 00000000000..ce2daef8300
--- /dev/null
+++ b/opal/mca/smsc/xpmem/smsc_xpmem_internal.h
@@ -0,0 +1,81 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2021      Google, Inc. All rights reserved.
+ * Copyright (c) 2022      Computer Architecture and VLSI Systems (CARV)
+ *                         Laboratory, ICS Forth. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef OPAL_MCA_SMSC_XPMEM_SMSC_XPMEM_INTERNAL_H
+#define OPAL_MCA_SMSC_XPMEM_SMSC_XPMEM_INTERNAL_H
+
+#include "opal/mca/smsc/xpmem/smsc_xpmem.h"
+
+#include "opal/mca/rcache/base/rcache_base_vma.h"
+#if defined(HAVE_XPMEM_H)
+#    include <xpmem.h>
+
+typedef struct xpmem_addr xpmem_addr_t;
+#elif defined(HAVE_SN_XPMEM_H)
+#    include <sn/xpmem.h>
+
+typedef int64_t xpmem_segid_t;
+typedef int64_t xpmem_apid_t;
+#endif
+
+typedef struct xpmem_addr xpmem_addr_t;
+
+struct mca_smsc_xpmem_modex_t {
+    /** XPMEM segment id for this peer */
+    xpmem_segid_t seg_id;
+    /** maximum address we can attach to on this peer */
+    uintptr_t address_max;
+};
+
+typedef struct mca_smsc_xpmem_modex_t mca_smsc_xpmem_modex_t;
+
+struct mca_smsc_xpmem_endpoint_t {
+    mca_smsc_endpoint_t super;
+    /** XPMEM apid for this peer */
+    xpmem_apid_t apid;
+    /** maximum address we can attach to on this peer */
+    uintptr_t address_max;
+    /** cache of xpmem attachments created using this endpoint */
+    mca_rcache_base_vma_module_t *vma_module;
+};
+
+typedef struct mca_smsc_xpmem_endpoint_t mca_smsc_xpmem_endpoint_t;
+
+OBJ_CLASS_DECLARATION(mca_smsc_xpmem_endpoint_t);
+
+struct mca_smsc_xpmem_component_t {
+    mca_smsc_component_t super;
+
+    /** maximum attachment address for this process. attempts to attach past this value may fail. */
+    uintptr_t my_address_max;
+    /** XPMEM segment id for this process */
+    xpmem_segid_t my_seg_id;
+    /** log base 2 of the attachment alignment. this controls how big the smallest attachment is. a
+     * larger value will produce fewer entries in the cache but will increase attachment time. */
+    unsigned int log_attach_align;
+    /** maximum size that will be used with a single memcpy call. on some systems we see better
+     * performance if we chunk the copy into multiple memcpy calls. */
+    uint64_t memcpy_chunk_size;
+};
+
+typedef struct mca_smsc_xpmem_component_t mca_smsc_xpmem_component_t;
+
+struct mca_smsc_xpmem_module_t {
+    mca_smsc_module_t super;
+};
+
+typedef struct mca_smsc_xpmem_module_t mca_smsc_xpmem_module_t;
+
+extern mca_smsc_xpmem_module_t mca_smsc_xpmem_module;
+extern mca_smsc_xpmem_component_t mca_smsc_xpmem_component;
+
+#endif /* OPAL_MCA_SMSC_XPMEM_SMSC_XPMEM_INTERNAL_H */
diff --git a/opal/mca/smsc/xpmem/smsc_xpmem_module.c b/opal/mca/smsc/xpmem/smsc_xpmem_module.c
new file mode 100644
index 00000000000..037d9d31fe7
--- /dev/null
+++ b/opal/mca/smsc/xpmem/smsc_xpmem_module.c
@@ -0,0 +1,339 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2014      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2020-2021 Google, LLC. All rights reserved.
+ * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2022-2023 Computer Architecture and VLSI Systems (CARV)
+ *                         Laboratory, ICS Forth. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "opal_config.h"
+
+#include "opal/include/opal/align.h"
+#include "opal/mca/memchecker/base/base.h"
+#include "opal/mca/pmix/pmix-internal.h"
+#include "opal/mca/rcache/rcache.h"
+#include "opal/mca/smsc/base/base.h"
+#include "opal/mca/smsc/xpmem/smsc_xpmem_internal.h"
+#include "opal/util/minmax.h"
+#include "opal/util/sys_limits.h"
+
+OBJ_CLASS_INSTANCE(mca_smsc_xpmem_endpoint_t, opal_object_t, NULL, NULL);
+
+mca_smsc_endpoint_t *mca_smsc_xpmem_get_endpoint(opal_proc_t *peer_proc)
+{
+    int rc;
+    size_t modex_size;
+    mca_smsc_xpmem_modex_t *modex;
+
+    OPAL_MODEX_RECV_IMMEDIATE(rc, &mca_smsc_xpmem_component.super.smsc_version,
+                              &peer_proc->proc_name, (void **) &modex, &modex_size);
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
+        return NULL;
+    }
+
+    mca_smsc_xpmem_endpoint_t *endpoint = OBJ_NEW(mca_smsc_xpmem_endpoint_t);
+    if (OPAL_UNLIKELY(NULL == endpoint)) {
+        return NULL;
+    }
+
+    endpoint->super.proc = peer_proc;
+    endpoint->address_max = modex->address_max;
+
+    endpoint->vma_module = mca_rcache_base_vma_module_alloc();
+    if (OPAL_UNLIKELY(NULL == endpoint->vma_module)) {
+        OBJ_RELEASE(endpoint);
+        return NULL;
+    }
+
+    endpoint->apid = xpmem_get(modex->seg_id, XPMEM_RDWR, XPMEM_PERMIT_MODE, (void *) 0666);
+
+    if(OPAL_UNLIKELY(-1 == endpoint->apid)) {
+        OBJ_RELEASE(endpoint->vma_module);
+        OBJ_RELEASE(endpoint);
+        return NULL;
+    }
+
+    return &endpoint->super;
+}
+
+/* look up the remote pointer in the peer rcache and attach if necessary */
+void *mca_smsc_xpmem_map_peer_region(mca_smsc_endpoint_t *endpoint, uint64_t flags,
+                                     void *remote_ptr, size_t size, void **local_ptr)
+{
+    mca_smsc_xpmem_endpoint_t *xpmem_endpoint = (mca_smsc_xpmem_endpoint_t *) endpoint;
+    mca_rcache_base_vma_module_t *vma_module = xpmem_endpoint->vma_module;
+    uintptr_t attach_align = 1 << mca_smsc_xpmem_component.log_attach_align;
+    mca_rcache_base_registration_t *reg = NULL;
+    xpmem_addr_t xpmem_addr;
+    uintptr_t base, bound;
+    int rc;
+
+    // base is the first byte of the region, bound is the last (inclusive)
+    base = OPAL_DOWN_ALIGN((uintptr_t) remote_ptr, attach_align, uintptr_t);
+    bound = OPAL_ALIGN((uintptr_t) remote_ptr + size, attach_align, uintptr_t) - 1;
+    if (OPAL_UNLIKELY(bound > xpmem_endpoint->address_max)) {
+        bound = xpmem_endpoint->address_max;
+    }
+
+    rc = mca_rcache_base_vma_find(vma_module, (void *) base, bound - base + 1, &reg);
+    assert(OPAL_SUCCESS == rc);
+
+    if (reg) {
+        int32_t old_ref_count = opal_atomic_fetch_add_32(&reg->ref_count, 1);
+
+        if (0 == old_ref_count) {
+            /* Registration is being deleted by another thread
+             * in mca_smsc_xpmem_unmap_peer_region, ignore it. */
+            reg = NULL;
+        }
+    } else {
+        /* If there is a registration that overlaps with the requested range, but
+         * does not fully cover it, we destroy it and make in its place a new one
+         * that covers both the existing and the new range. */
+
+        /* The search settings below will also match areas that would be right next to
+         * the new one (technically not overlapping, but uniteable under a single area).
+         * Whether we want this is debatable (re-establishing an XPMEM attachment can
+         * incur significant overhead). The current choice matches legacy behaviour. */
+
+        // Ideally, we would want a find() method capable of partial matching
+        uintptr_t search_base[] = {base, bound, base - 1, bound + 1};
+        for (size_t i = 0; i < sizeof(search_base)/sizeof(search_base[0]); i++) {
+            mca_rcache_base_registration_t *ov_reg = NULL;
+
+            rc = mca_rcache_base_vma_find(vma_module, (void *) search_base[i], 1, &ov_reg);
+            assert(OPAL_SUCCESS == rc);
+
+            if (ov_reg) {
+                /* Found an overlapping area. Set the invalid flag, to mark the deletion
+                 * of this old registration (will eventually take place in unmap_peer_region).
+                 * If another thread has already marked deletion, do nothing. */
+
+                uint32_t old_flags = opal_atomic_fetch_or_32(
+                    (volatile opal_atomic_int32_t *) &ov_reg->flags, MCA_RCACHE_FLAGS_INVALID);
+
+                if (!(old_flags & MCA_RCACHE_FLAGS_INVALID)) {
+                    base = opal_min(base, (uintptr_t) ov_reg->base);
+                    bound = opal_max(bound, (uintptr_t) ov_reg->bound);
+
+                    /* unmap_peer_region will decrement the ref count and dealloc the attachment
+                     * if it drops to 0. But we didn't increment the ref count when we found the
+                     * reg as is customary. If PERSIST was set, there is superfluous ref present
+                     * from when we initialized ref_count to 2 instead of 1, so we good. If not,
+                     * manually add the missing reference here; otherwise the count would drop to
+                     * -1, or the reg might be deleted while still in use elsewhere. */
+                    if (!(MCA_RCACHE_FLAGS_PERSIST & ov_reg->flags))
+                        opal_atomic_add(&ov_reg->ref_count, 1);
+
+                    mca_smsc_xpmem_unmap_peer_region(ov_reg);
+                }
+            }
+        }
+    }
+
+    if (NULL == reg) {
+        reg = OBJ_NEW(mca_rcache_base_registration_t);
+        if (OPAL_LIKELY(NULL == reg)) {
+            return NULL;
+        }
+
+        // PERSIST is implemented by keeping an extra reference around
+        reg->ref_count = ((flags & MCA_RCACHE_FLAGS_PERSIST)
+            && !(flags & MCA_RCACHE_FLAGS_CACHE_BYPASS) ? 2 : 1);
+        reg->flags = flags;
+        reg->base = (unsigned char *) base;
+        reg->bound = (unsigned char *) bound;
+        reg->alloc_base = (void *) endpoint;
+
+#if defined(HAVE_SN_XPMEM_H)
+        xpmem_addr.id = xpmem_endpoint->apid;
+#else
+        xpmem_addr.apid = xpmem_endpoint->apid;
+#endif
+        xpmem_addr.offset = base;
+
+        opal_output_verbose(MCA_BASE_VERBOSE_INFO, opal_smsc_base_framework.framework_output,
+                            "mca_smsc_xpmem_map_peer_region: creating region mapping "
+                            "for endpoint %p address range %p-%p",
+                            (void *) endpoint, reg->base, reg->bound);
+
+        reg->rcache_context = xpmem_attach(xpmem_addr, bound - base + 1, NULL);
+        if (OPAL_UNLIKELY((void *) -1 == reg->rcache_context)) {
+            uintptr_t old_bound = bound;
+
+            /* retry with the page as upper bound */
+            bound = OPAL_ALIGN((uintptr_t) remote_ptr + size, opal_getpagesize(), uintptr_t) - 1;
+            reg->bound = (unsigned char *) bound;
+
+            opal_output_verbose(MCA_BASE_VERBOSE_INFO, opal_smsc_base_framework.framework_output,
+                                "mca_smsc_xpmem_map_peer_region: region mapping "
+                                "for endpoint %p address range %p-%p failed. "
+                                "retrying with range %p-%p",
+                                (void *) endpoint, reg->base, (void *) old_bound,
+                                reg->base, reg->bound);
+
+            reg->rcache_context = xpmem_attach(xpmem_addr, bound - base + 1, NULL);
+            if (OPAL_UNLIKELY((void *) -1 == reg->rcache_context)) {
+                OBJ_RELEASE(reg);
+                return NULL;
+            }
+        }
+
+        opal_memchecker_base_mem_defined(reg->rcache_context, bound - base + 1);
+
+        if (!(reg->flags & MCA_RCACHE_FLAGS_CACHE_BYPASS)) {
+            rc = mca_rcache_base_vma_insert(vma_module, reg, 0);
+            assert(OPAL_SUCCESS == rc);
+
+            if (OPAL_SUCCESS != rc) {
+                reg->flags |= MCA_RCACHE_FLAGS_CACHE_BYPASS;
+            }
+        }
+    }
+
+    opal_atomic_wmb();
+    *local_ptr = (void *) ((uintptr_t) reg->rcache_context
+                           + (ptrdiff_t)((uintptr_t) remote_ptr - (uintptr_t) reg->base));
+
+    return (void *) reg;
+}
+
+void mca_smsc_xpmem_unmap_peer_region(void *ctx)
+{
+    mca_rcache_base_registration_t *reg = (mca_rcache_base_registration_t *) ctx;
+    mca_smsc_xpmem_endpoint_t *endpoint = (mca_smsc_xpmem_endpoint_t *) reg->alloc_base;
+    int32_t ref_count;
+
+    ref_count = opal_atomic_add_fetch_32(&reg->ref_count, -1);
+    if (OPAL_UNLIKELY(0 == ref_count)) {
+        opal_output_verbose(MCA_BASE_VERBOSE_INFO, opal_smsc_base_framework.framework_output,
+                            "mca_smsc_xpmem_unmap_peer_region: deleting region mapping for "
+                            "endpoint %p address range %p-%p",
+                            (void *) endpoint, reg->base, reg->bound);
+
+        if (!(reg->flags & MCA_RCACHE_FLAGS_CACHE_BYPASS)) {
+            int ret = mca_rcache_base_vma_delete(endpoint->vma_module, reg);
+            assert(OPAL_SUCCESS == ret);
+            (void) ret;
+        }
+
+        opal_memchecker_base_mem_noaccess(reg->rcache_context, (uintptr_t)(reg->bound - reg->base + 1));
+        (void) xpmem_detach(reg->rcache_context);
+
+        OBJ_RELEASE(reg);
+    }
+}
+
+static int mca_smsc_xpmem_endpoint_rcache_entry_cleanup(mca_rcache_base_registration_t *reg, void *ctx)
+{
+    // See respective comment in mca_smsc_xpmem_map_peer_region
+    if (!(MCA_RCACHE_FLAGS_PERSIST & reg->flags))
+        opal_atomic_add(&reg->ref_count, 1);
+
+    mca_smsc_xpmem_unmap_peer_region(reg);
+    return OPAL_SUCCESS;
+}
+
+static void mca_smsc_xpmem_cleanup_endpoint(mca_smsc_xpmem_endpoint_t *endpoint)
+{
+    opal_output_verbose(MCA_BASE_VERBOSE_INFO, opal_smsc_base_framework.framework_output,
+                        "mca_smsc_xpmem_cleanup_endpoint: cleaning up endpoint %p", (void *) endpoint);
+
+    opal_output_verbose(MCA_BASE_VERBOSE_INFO, opal_smsc_base_framework.framework_output,
+                        "mca_smsc_xpmem_cleanup_endpoint: deleting %" PRIsize_t " region mappings",
+                        endpoint->vma_module->tree.tree_size);
+
+    /* clean out the registration cache */
+    (void) mca_rcache_base_vma_iterate(endpoint->vma_module, NULL, (size_t) -1, true,
+                                       mca_smsc_xpmem_endpoint_rcache_entry_cleanup, NULL);
+
+    OBJ_RELEASE(endpoint->vma_module);
+    xpmem_release(endpoint->apid);
+
+    endpoint->vma_module = NULL;
+    endpoint->apid = 0;
+}
+
+void mca_smsc_xpmem_return_endpoint(mca_smsc_endpoint_t *endpoint)
+{
+    mca_smsc_xpmem_cleanup_endpoint((mca_smsc_xpmem_endpoint_t *) endpoint);
+    OBJ_RELEASE(endpoint);
+}
+
+/* memcpy is faster at larger sizes but is undefined if the
+   pointers are aliased (TODO -- read alias check) */
+static inline void mca_smsc_xpmem_memmove(void *dst, void *src, size_t size)
+{
+    while (size > 0) {
+        size_t copy_size = opal_min(size, mca_smsc_xpmem_component.memcpy_chunk_size);
+        memcpy(dst, src, copy_size);
+        dst = (void *) ((uintptr_t) dst + copy_size);
+        src = (void *) ((uintptr_t) src + copy_size);
+        size -= copy_size;
+    }
+}
+
+int mca_smsc_xpmem_copy_to(mca_smsc_endpoint_t *endpoint, void *local_address, void *remote_address,
+                           size_t size, void *reg_handle)
+{
+    /* ignore the registration handle as it is not used for XPMEM */
+    (void) reg_handle;
+
+    void *remote_ptr, *ctx;
+    ctx = mca_smsc_xpmem_map_peer_region(endpoint,
+        MCA_RCACHE_FLAGS_PERSIST, remote_address, size, &remote_ptr);
+    mca_smsc_xpmem_memmove(remote_ptr, local_address, size);
+
+    mca_smsc_xpmem_unmap_peer_region(ctx);
+
+    return OPAL_SUCCESS;
+}
+
+int mca_smsc_xpmem_copy_from(mca_smsc_endpoint_t *endpoint, void *local_address,
+                             void *remote_address, size_t size, void *reg_handle)
+{
+    /* ignore the registration handle as it is not used for XPMEM */
+    (void) reg_handle;
+
+    void *remote_ptr, *ctx;
+
+    ctx = mca_smsc_xpmem_map_peer_region(endpoint,
+        MCA_RCACHE_FLAGS_PERSIST, remote_address, size, &remote_ptr);
+    mca_smsc_xpmem_memmove(local_address, remote_ptr, size);
+
+    mca_smsc_xpmem_unmap_peer_region(ctx);
+
+    return OPAL_SUCCESS;
+}
+
+/* unsupported interfaces defined to support MCA direct */
+void *mca_smsc_xpmem_register_region(void *local_address, size_t size)
+{
+    return NULL;
+}
+
+void mca_smsc_xpmem_deregister_region(void *reg_data)
+{
+}
+
+mca_smsc_xpmem_module_t mca_smsc_xpmem_module = {
+    .super = {
+        .features = MCA_SMSC_FEATURE_CAN_MAP,
+        .get_endpoint = mca_smsc_xpmem_get_endpoint,
+        .return_endpoint = mca_smsc_xpmem_return_endpoint,
+        .copy_to = mca_smsc_xpmem_copy_to,
+        .copy_from = mca_smsc_xpmem_copy_from,
+        .map_peer_region = mca_smsc_xpmem_map_peer_region,
+        .unmap_peer_region = mca_smsc_xpmem_unmap_peer_region,
+    },
+};
diff --git a/opal/mca/threads/README.md b/opal/mca/threads/README.md
index 8c600ccb019..7c0fbdf0c8e 100644
--- a/opal/mca/threads/README.md
+++ b/opal/mca/threads/README.md
@@ -18,7 +18,7 @@ The MCA for threading libraries is implemented in two places, once as a set of `
 
 For performance reasons, in particular synchronization overhead, it is not possible to implement a threading model as a traditional MCA. This means --at least in the short term-- that threading models are chosen at compile time rather than runtime options, using mechanisms similar to Open MPI's libevent integration.
 
-The .h files are meant to be run on the fast path containing inline synchonization functions (threads_<threading_model>_mutex.h, thread local storage (threads_<threading_model>_tsd.h) and the opal_thread structure (threads_<threading_model>_thread.h).
+The .h files are meant to be run on the fast path containing inline synchronization functions (threads_<threading_model>_mutex.h, thread local storage (threads_<threading_model>_tsd.h) and the opal_thread structure (threads_<threading_model>_thread.h).
 
 The rest of the threading implementation follows the normal MCA model:
 
diff --git a/opal/mca/threads/argobots/Makefile.am b/opal/mca/threads/argobots/Makefile.am
index b0c90fd78de..546c38e6b73 100644
--- a/opal/mca/threads/argobots/Makefile.am
+++ b/opal/mca/threads/argobots/Makefile.am
@@ -11,6 +11,8 @@
 #                         All rights reserved.
 # Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2019      Sandia National Laboratories.  All rights reserved.
+# Copyright (c) 2021      Argonne National Laboratory.  All rights reserved.
+#
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -25,12 +27,9 @@ sources = \
     threads_argobots_component.c \
     threads_argobots_condition.c \
     threads_argobots_module.c \
-    threads_argobots_mutex.c \
     threads_argobots_mutex.h \
     threads_argobots_threads.h \
-    threads_argobots_tsd.h \
-    threads_argobots_wait_sync.c \
-    threads_argobots_wait_sync.h
+    threads_argobots_tsd.h
 
 #lib = libmca_threads_argobots.la
 lib_sources = $(sources)
@@ -47,7 +46,7 @@ mcacomponentdir = $(opallibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_threads_argobots_la_SOURCES = $(sources)
 mca_threads_argobots_la_LDFLAGS = -module -avoid-version
-mca_threads_argobots_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
+mca_threads_argobots_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
         $(opal_argo_LIBS)
 
 noinst_LTLIBRARIES = $(component_noinst)
diff --git a/opal/mca/threads/argobots/configure.m4 b/opal/mca/threads/argobots/configure.m4
index aeeba2b517d..c5ad0dc44e5 100644
--- a/opal/mca/threads/argobots/configure.m4
+++ b/opal/mca/threads/argobots/configure.m4
@@ -14,8 +14,9 @@
 # Copyright (c) 2015      Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
 # Copyright (c) 2019      Sandia National Laboratories.  All rights reserved.
-# Copyright (c) 2019      Triad National Security, LLC. All rights
+# Copyright (c) 2019-2022 Triad National Security, LLC. All rights
 #                         Reserved.
+# Copyright (c) 2021      Argonne National Laboratory.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -38,39 +39,26 @@ AC_DEFUN([OPAL_CONFIG_ARGOBOTS_THREADS],[
     opal_check_argo_save_LIBS=$LIBS
 
     opal_argo_happy=yes
-    AS_IF([test "$with_argo" = "no"],
-          [opal_argo_happy=no])
-
+    opal_argo11_happy=yes
+
+    OAC_CHECK_PACKAGE([argobots],
+                      [opal_argo],
+                      [abt.h],
+                      [abt],
+                      [ABT_init],
+                      [opal_argo_happy=yes],
+                      [opal_argo_happy=no])
+
+    # ABT_unit_get_thread() is a new Argobots 1.1 API.
+    # It was introduced after static mutex/cond initializers.
     AS_IF([test $opal_argo_happy = yes],
-          [AC_MSG_CHECKING([looking for argobots in])
-           AS_IF([test "$with_argobots" != "yes"],
-                 [opal_argo_dir=$with_argobots
-                  AC_MSG_RESULT([($opal_argo_dir)])],
-                 [AC_MSG_RESULT([(default search paths)])])
-           AS_IF([test ! -z "$with_argobots_libdir" && \
-                         test "$with_argobots_libdir" != "yes"],
-                 [opal_argo_libdir=$with_argobots_libdir])
-          ])
+          [AC_CHECK_FUNCS([ABT_unit_get_thread], [], [opal_argo11_happy="yes"])])
 
-    AS_IF([test $opal_argo_happy = yes],
-          [OPAL_CHECK_PACKAGE([opal_argo],
-                              [abt.h],
-                              [abt],
-                              [ABT_init],
-                              [],
-                              [$opal_argo_dir],
-                              [$opal_argo_libdir],
-                              [],
-                              [opal_argo_happy=no])])
-
-    AS_IF([test $opal_argo_happy = yes && test -n "$opal_argo_dir"],
-          [OPAL_ARGO_INCLUDE_PATH="$opal_argo_dir/include/"],
-          [OPAL_ARGO_INCLUDE_PATH=""])
+    AS_IF([test $opal_argo_happy = yes && test $opal_argo11_happy = no],
+          [AC_MSG_ERROR([Open MPI requires Argobots 1.1 or newer.])])
 
     AS_IF([test $opal_argo_happy = yes],
-          [AC_CONFIG_FILES([opal/mca/threads/argobots/threads_argobots.h])
-           AC_SUBST([OPAL_ARGO_INCLUDE_PATH])
-           AC_SUBST([opal_argo_CPPFLAGS])
+          [ AC_SUBST([opal_argo_CPPFLAGS])
            AC_SUBST([opal_argo_LDFLAGS])
            AC_SUBST([opal_argo_LIBS])
            TPKG_CFLAGS="$opal_argo_CPPFLAGS"
@@ -81,7 +69,7 @@ AC_DEFUN([OPAL_CONFIG_ARGOBOTS_THREADS],[
            TPKG_LDFLAGS="$opal_argo_LDFLAGS"
            TPKG_LIBS="$opal_argo_LIBS"])
 
-    CPPFLAGS=$opal_check_argo_save_CPPFLAGS
+    CPPFLAGS="${opal_check_argo_save_CPPFLAGS} ${opal_argo_CPPFLAGS}"
     LDFLAGS=$opal_check_argo_save_LDFLAGS
     LIBS=$opal_check_argo_save_LIBS
 
@@ -113,9 +101,6 @@ AC_DEFUN([MCA_opal_threads_argobots_POST_CONFIG],[
            AC_DEFINE_UNQUOTED([MCA_threads_tsd_base_include_HEADER],
                               ["opal/mca/threads/argobots/threads_argobots_tsd.h"],
                               [Header to include for tsd implementation])
-           AC_DEFINE_UNQUOTED([MCA_threads_wait_sync_base_include_HEADER],
-                              ["opal/mca/threads/argobots/threads_argobots_wait_sync.h"],
-                              [Header to include for wait_sync implementation])
            THREAD_CFLAGS="$TPKG_CFLAGS"
            THREAD_FCFLAGS="$TPKG_FCFLAGS"
            THREAD_CXXFLAGS="$TPKG_CXXFLAGS"
diff --git a/opal/mca/threads/argobots/threads_argobots.h b/opal/mca/threads/argobots/threads_argobots.h
new file mode 100644
index 00000000000..d0553759c2a
--- /dev/null
+++ b/opal/mca/threads/argobots/threads_argobots.h
@@ -0,0 +1,38 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007-2016 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2019      Sandia National Laboratories.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef OPAL_MCA_THREADS_ARGOBOTS_THREADS_ARGOBOTS_H
+#define OPAL_MCA_THREADS_ARGOBOTS_THREADS_ARGOBOTS_H
+
+#include "abt.h"
+
+static inline void opal_threads_argobots_ensure_init(void)
+{
+    if (ABT_SUCCESS != ABT_initialized()) {
+        ABT_init(0, 0);
+    }
+}
+
+#endif /* OPAL_MCA_THREADS_ARGOBOTS_THREADS_ARGOBOTS_H */
diff --git a/opal/mca/threads/argobots/threads_argobots.h.in b/opal/mca/threads/argobots/threads_argobots.h.in
deleted file mode 100644
index 324933c3e26..00000000000
--- a/opal/mca/threads/argobots/threads_argobots.h.in
+++ /dev/null
@@ -1,38 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007-2016 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2019      Sandia National Laboratories.  All rights reserved.
- *
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef OPAL_MCA_THREADS_ARGOBOTS_THREADS_ARGOBOTS_H
-#define OPAL_MCA_THREADS_ARGOBOTS_THREADS_ARGOBOTS_H
-
-#include "@OPAL_ARGO_INCLUDE_PATH@abt.h"
-
-static inline void opal_threads_argobots_ensure_init(void)
-{
-    if (ABT_SUCCESS != ABT_initialized()) {
-        ABT_init(0, 0);
-    }
-}
-
-#endif /* OPAL_MCA_THREADS_ARGOBOTS_THREADS_ARGOBOTS_H */
diff --git a/opal/mca/threads/argobots/threads_argobots_component.c b/opal/mca/threads/argobots/threads_argobots_component.c
index 6c4e6f815fa..bce182b1086 100644
--- a/opal/mca/threads/argobots/threads_argobots_component.c
+++ b/opal/mca/threads/argobots/threads_argobots_component.c
@@ -23,30 +23,30 @@
 
 #include "opal_config.h"
 
+#include "opal/constants.h"
 #include "opal/mca/threads/argobots/threads_argobots.h"
 #include "opal/mca/threads/thread.h"
 #include "opal/mca/threads/threads.h"
-#include "opal/constants.h"
 
 static int opal_threads_argobots_open(void);
 
 const opal_threads_base_component_1_0_0_t mca_threads_argobots_component = {
     /* First, the mca_component_t struct containing meta information
      * about the component itself */
-    .threadsc_version = {
-        OPAL_THREADS_BASE_VERSION_1_0_0,
+    .threadsc_version =
+        {
+            OPAL_THREADS_BASE_VERSION_1_0_0,
 
-        /* Component name and version */
-        .mca_component_name = "argobots",
-        MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
-                              OPAL_RELEASE_VERSION),
+            /* Component name and version */
+            .mca_component_name = "argobots",
+            MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
+                                  OPAL_RELEASE_VERSION),
 
-        .mca_open_component = opal_threads_argobots_open,
-    },
-    .threadsc_data = {
-        /* The component is checkpoint ready */
-        MCA_BASE_METADATA_PARAM_CHECKPOINT
-    },
+            .mca_open_component = opal_threads_argobots_open,
+        },
+    .threadsc_data =
+        {/* The component is checkpoint ready */
+         MCA_BASE_METADATA_PARAM_CHECKPOINT},
 };
 
 int opal_threads_argobots_open(void)
diff --git a/opal/mca/threads/argobots/threads_argobots_condition.c b/opal/mca/threads/argobots/threads_argobots_condition.c
index 56b11cf9526..ca48c9bdf1b 100644
--- a/opal/mca/threads/argobots/threads_argobots_condition.c
+++ b/opal/mca/threads/argobots/threads_argobots_condition.c
@@ -33,7 +33,5 @@ static void opal_condition_destruct(opal_condition_t *c)
 {
 }
 
-OBJ_CLASS_INSTANCE(opal_condition_t,
-                   opal_object_t,
-                   opal_condition_construct,
+OBJ_CLASS_INSTANCE(opal_condition_t, opal_object_t, opal_condition_construct,
                    opal_condition_destruct);
diff --git a/opal/mca/threads/argobots/threads_argobots_module.c b/opal/mca/threads/argobots/threads_argobots_module.c
index e1a0ba840e2..50acb60d8ce 100644
--- a/opal/mca/threads/argobots/threads_argobots_module.c
+++ b/opal/mca/threads/argobots/threads_argobots_module.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -11,8 +11,9 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2019      Sandia National Laboratories.  All rights reserved.
- *
  * Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2021      Argonne National Laboratory.  All rights reserved.
+ *
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -24,98 +25,13 @@
 
 #include <unistd.h>
 
-#include "opal/mca/threads/argobots/threads_argobots.h"
 #include "opal/constants.h"
-#include "opal/util/sys_limits.h"
-#include "opal/util/output.h"
-#include "opal/prefetch.h"
+#include "opal/mca/threads/argobots/threads_argobots.h"
 #include "opal/mca/threads/threads.h"
 #include "opal/mca/threads/tsd.h"
-
-struct opal_tsd_key_value {
-    opal_tsd_key_t key;
-    opal_tsd_destructor_t destructor;
-};
-
-static opal_mutex_t opal_tsd_lock = OPAL_MUTEX_STATIC_INIT;
-static struct opal_tsd_key_value *opal_tsd_key_values = NULL;
-static int opal_tsd_key_values_count = 0;
-static int opal_tsd_key_values_size = 0;
-
-/*
- * Constructor
- */
-static void opal_thread_construct(opal_thread_t *t)
-{
-    t->t_run = 0;
-    t->t_handle = ABT_THREAD_NULL;
-}
-
-OBJ_CLASS_INSTANCE(opal_thread_t,
-                   opal_object_t,
-                   opal_thread_construct, NULL);
-
-static inline ABT_thread opal_thread_get_argobots_self(void)
-{
-    ABT_thread self;
-    ABT_thread_self(&self);
-    return self;
-}
-
-static void opal_thread_argobots_wrapper(void *arg)
-{
-    opal_thread_t *t = (opal_thread_t *)arg;
-    t->t_ret = ((void *(*)(void *))t->t_run)(t);
-}
-
-opal_thread_t *opal_thread_get_self(void)
-{
-    opal_threads_argobots_ensure_init();
-    opal_thread_t *t = OBJ_NEW(opal_thread_t);
-    t->t_handle = opal_thread_get_argobots_self();
-    return t;
-}
-
-bool opal_thread_self_compare(opal_thread_t *t)
-{
-    opal_threads_argobots_ensure_init();
-    return opal_thread_get_argobots_self() == t->t_handle;
-}
-
-int opal_thread_join(opal_thread_t *t, void **thr_return)
-{
-    int rc = ABT_thread_free(&t->t_handle);
-    if (thr_return) {
-        *thr_return = t->t_ret;
-    }
-    t->t_handle = ABT_THREAD_NULL;
-    return (ABT_SUCCESS == rc) ? OPAL_SUCCESS : OPAL_ERROR;
-}
-
-void opal_thread_set_main()
-{
-}
-
-int opal_thread_start(opal_thread_t *t)
-{
-    opal_threads_argobots_ensure_init();
-    int rc;
-    if (OPAL_ENABLE_DEBUG) {
-        if (NULL == t->t_run || ABT_THREAD_NULL != t->t_handle) {
-            return OPAL_ERR_BAD_PARAM;
-        }
-    }
-
-    ABT_xstream self_xstream;
-    ABT_xstream_self(&self_xstream);
-    rc = ABT_thread_create_on_xstream(self_xstream,
-                                      opal_thread_argobots_wrapper, t,
-                                      ABT_THREAD_ATTR_NULL, &t->t_handle);
-
-    return (ABT_SUCCESS == rc) ? OPAL_SUCCESS : OPAL_ERROR;
-}
-
-OBJ_CLASS_DECLARATION(opal_thread_t);
+#include "opal/prefetch.h"
+#include "opal/util/output.h"
+#include "opal/util/sys_limits.h"
 
 int opal_tsd_key_create(opal_tsd_key_t *key, opal_tsd_destructor_t destructor)
 {
diff --git a/opal/mca/threads/argobots/threads_argobots_mutex.c b/opal/mca/threads/argobots/threads_argobots_mutex.c
deleted file mode 100644
index ce0df909629..00000000000
--- a/opal/mca/threads/argobots/threads_argobots_mutex.c
+++ /dev/null
@@ -1,174 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007-2016 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2019      Sandia National Laboratories.  All rights reserved.
- *
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "opal/mca/threads/argobots/threads_argobots.h"
-#include "opal_config.h"
-
-#include <errno.h>
-
-#include "opal/mca/threads/mutex.h"
-#include "opal/mca/threads/argobots/threads_argobots_mutex.h"
-#include "opal/constants.h"
-
-/*
- * Wait and see if some upper layer wants to use threads, if support
- * exists.
- */
-bool opal_uses_threads = false;
-
-static void mca_threads_argobots_mutex_constructor(opal_mutex_t *p_mutex)
-{
-    opal_threads_argobots_ensure_init();
-    p_mutex->m_lock_argobots = OPAL_ABT_MUTEX_NULL;
-    p_mutex->m_recursive = 0;
-#if OPAL_ENABLE_DEBUG
-    p_mutex->m_lock_debug = 0;
-    p_mutex->m_lock_file = NULL;
-    p_mutex->m_lock_line = 0;
-#endif
-    opal_atomic_lock_init(&p_mutex->m_lock_atomic, 0);
-}
-
-static void mca_threads_argobots_mutex_destructor(opal_mutex_t *p_mutex)
-{
-    if (OPAL_ABT_MUTEX_NULL != p_mutex->m_lock_argobots) {
-        ABT_mutex_free(&p_mutex->m_lock_argobots);
-    }
-}
-
-static void mca_threads_argobots_recursive_mutex_constructor
-        (opal_recursive_mutex_t *p_mutex)
-{
-    opal_threads_argobots_ensure_init();
-    p_mutex->m_lock_argobots = OPAL_ABT_MUTEX_NULL;
-    p_mutex->m_recursive = 1;
-#if OPAL_ENABLE_DEBUG
-    p_mutex->m_lock_debug = 0;
-    p_mutex->m_lock_file = NULL;
-    p_mutex->m_lock_line = 0;
-#endif
-    opal_atomic_lock_init(&p_mutex->m_lock_atomic, 0);
-}
-
-static void mca_threads_argobots_recursive_mutex_destructor
-        (opal_recursive_mutex_t *p_mutex)
-{
-    if (OPAL_ABT_MUTEX_NULL != p_mutex->m_lock_argobots) {
-        ABT_mutex_free(&p_mutex->m_lock_argobots);
-    }
-}
-
-OBJ_CLASS_INSTANCE(opal_mutex_t,
-                   opal_object_t,
-                   mca_threads_argobots_mutex_constructor,
-                   mca_threads_argobots_mutex_destructor);
-OBJ_CLASS_INSTANCE(opal_recursive_mutex_t,
-                   opal_object_t,
-                   mca_threads_argobots_recursive_mutex_constructor,
-                   mca_threads_argobots_recursive_mutex_destructor);
-
-void opal_mutex_create(struct opal_mutex_t *m)
-{
-    opal_threads_argobots_ensure_init();
-    while (OPAL_ABT_MUTEX_NULL == m->m_lock_argobots) {
-        ABT_mutex abt_mutex;
-        if (m->m_recursive) {
-            ABT_mutex_attr abt_mutex_attr;
-            ABT_mutex_attr_create(&abt_mutex_attr);
-            ABT_mutex_attr_set_recursive(abt_mutex_attr, ABT_TRUE);
-            ABT_mutex_create_with_attr(abt_mutex_attr, &abt_mutex);
-            ABT_mutex_attr_free(&abt_mutex_attr);
-        } else {
-            ABT_mutex_create(&abt_mutex);
-        }
-        void *null_ptr = OPAL_ABT_MUTEX_NULL;
-        if (opal_atomic_compare_exchange_strong_ptr(
-             (opal_atomic_intptr_t *)&m->m_lock_argobots, (intptr_t *)&null_ptr,
-             (intptr_t)abt_mutex)) {
-            /* mutex is successfully created and substituted. */
-            return;
-        }
-        ABT_mutex_free(&abt_mutex);
-    }
-}
-
-static void opal_cond_create(opal_cond_t *cond)
-{
-    opal_threads_argobots_ensure_init();
-    while (OPAL_ABT_COND_NULL == *cond) {
-        ABT_cond new_cond;
-        ABT_cond_create(&new_cond);
-        void *null_ptr = OPAL_ABT_COND_NULL;
-        if (opal_atomic_compare_exchange_strong_ptr((opal_atomic_intptr_t *)cond,
-                                                    (intptr_t *)&null_ptr,
-                                                    (intptr_t)new_cond)) {
-            /* cond is successfully created and substituted. */
-            return;
-        }
-        ABT_cond_free(&new_cond);
-    }
-}
-
-int opal_cond_init(opal_cond_t *cond)
-{
-    *cond = OPAL_ABT_COND_NULL;
-    return OPAL_SUCCESS;
-}
-
-int opal_cond_wait(opal_cond_t *cond, opal_mutex_t *lock)
-{
-    if (OPAL_ABT_COND_NULL == *cond) {
-        opal_cond_create(cond);
-    }
-    int ret = ABT_cond_wait(*cond, lock->m_lock_argobots);
-    return ABT_SUCCESS == ret ? OPAL_SUCCESS : OPAL_ERROR;
-}
-
-int opal_cond_broadcast(opal_cond_t *cond)
-{
-    if (OPAL_ABT_COND_NULL == *cond) {
-        opal_cond_create(cond);
-    }
-    int ret = ABT_cond_broadcast(*cond);
-    return ABT_SUCCESS == ret ? OPAL_SUCCESS : OPAL_ERROR;
-}
-
-int opal_cond_signal(opal_cond_t *cond)
-{
-    if (OPAL_ABT_COND_NULL == *cond) {
-        opal_cond_create(cond);
-    }
-    int ret = ABT_cond_signal(*cond);
-    return ABT_SUCCESS == ret ? OPAL_SUCCESS : OPAL_ERROR;
-}
-
-int opal_cond_destroy(opal_cond_t *cond)
-{
-    int ret = ABT_SUCCESS;
-    if (OPAL_ABT_COND_NULL != *cond) {
-        ret = ABT_cond_free(cond);
-    }
-    return ABT_SUCCESS == ret ? OPAL_SUCCESS : OPAL_ERROR;
-}
diff --git a/opal/mca/threads/argobots/threads_argobots_mutex.h b/opal/mca/threads/argobots/threads_argobots_mutex.h
index 7dff30f010d..58eaa3bc620 100644
--- a/opal/mca/threads/argobots/threads_argobots_mutex.h
+++ b/opal/mca/threads/argobots/threads_argobots_mutex.h
@@ -17,8 +17,9 @@
  * Copyright (c) 2019      Sandia National Laboratories.  All rights reserved.
  * Copyright (c) 2020      Triad National Security, LLC. All rights
  *                         reserved.
- *
  * Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2021      Argonne National Laboratory.  All rights reserved.
+ * Copyright (c) 2022      Sandia National Laboratories.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -31,198 +32,135 @@
 
 #include "opal_config.h"
 
-#include <errno.h>
 #include <stdio.h>
-
-#include "opal/mca/threads/argobots/threads_argobots.h"
+#include <string.h>
 
 #include "opal/class/opal_object.h"
-#include "opal/sys/atomic.h"
+#include "opal/constants.h"
+#include "opal/mca/threads/argobots/threads_argobots.h"
+#include "opal/mca/threads/mutex.h"
 #include "opal/util/output.h"
-
+#include "opal/util/show_help.h"
 
 BEGIN_C_DECLS
 
-/* Don't use ABT_MUTEX_NULL, since it might be not NULL. */
-#define OPAL_ABT_MUTEX_NULL 0
-
-struct opal_mutex_t {
-    opal_object_t super;
+typedef ABT_mutex_memory opal_thread_internal_mutex_t;
 
-    ABT_mutex m_lock_argobots;
-    int m_recursive;
+#define OPAL_THREAD_INTERNAL_MUTEX_INITIALIZER           ABT_MUTEX_INITIALIZER
+#define OPAL_THREAD_INTERNAL_RECURSIVE_MUTEX_INITIALIZER ABT_RECURSIVE_MUTEX_INITIALIZER
 
-#if OPAL_ENABLE_DEBUG
-    int m_lock_debug;
-    const char *m_lock_file;
-    int m_lock_line;
-#endif
-
-    opal_atomic_lock_t m_lock_atomic;
-};
-
-OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_mutex_t);
-OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_recursive_mutex_t);
-
-#if OPAL_ENABLE_DEBUG
-#define OPAL_MUTEX_STATIC_INIT                                          \
-    {                                                                   \
-        .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t),                    \
-        .m_lock_argobots = OPAL_ABT_MUTEX_NULL,                         \
-        .m_recursive = 0,                                               \
-        .m_lock_debug = 0,                                              \
-        .m_lock_file = NULL,                                            \
-        .m_lock_line = 0,                                               \
-        .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT,                         \
-    }
-#else
-#define OPAL_MUTEX_STATIC_INIT                                          \
-    {                                                                   \
-        .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t),                    \
-        .m_lock_argobots = OPAL_ABT_MUTEX_NULL,                         \
-        .m_recursive = 0,                                               \
-        .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT,                         \
+static inline int opal_thread_internal_mutex_init(opal_thread_internal_mutex_t *p_mutex,
+                                                  bool recursive)
+{
+    if (recursive) {
+        const ABT_mutex_memory init_mutex = ABT_RECURSIVE_MUTEX_INITIALIZER;
+        memcpy(p_mutex, &init_mutex, sizeof(ABT_mutex_memory));
+    } else {
+        const ABT_mutex_memory init_mutex = ABT_MUTEX_INITIALIZER;
+        memcpy(p_mutex, &init_mutex, sizeof(ABT_mutex_memory));
     }
-#endif
+    return OPAL_SUCCESS;
+}
 
+static inline void opal_thread_internal_mutex_lock(opal_thread_internal_mutex_t *p_mutex)
+{
+    ABT_mutex mutex = ABT_MUTEX_MEMORY_GET_HANDLE(p_mutex);
 #if OPAL_ENABLE_DEBUG
-#define OPAL_RECURSIVE_MUTEX_STATIC_INIT                                \
-    {                                                                   \
-        .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t),                    \
-        .m_lock_argobots = OPAL_ABT_MUTEX_NULL,                         \
-        .m_recursive = 1,                                               \
-        .m_lock_debug = 0,                                              \
-        .m_lock_file = NULL,                                            \
-        .m_lock_line = 0,                                               \
-        .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT,                         \
+    int ret = ABT_mutex_lock(mutex);
+    if (ABT_SUCCESS != ret) {
+        opal_show_help("help-opal-threads.txt", "mutex lock failed", true);
     }
 #else
-#define OPAL_RECURSIVE_MUTEX_STATIC_INIT                                \
-    {                                                                   \
-        .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t),                    \
-        .m_lock_argobots = OPAL_ABT_MUTEX_NULL,                         \
-        .m_recursive = 1,                                               \
-        .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT,                         \
-    }
+    ABT_mutex_lock(mutex);
 #endif
+}
 
-/************************************************************************
- *
- * mutex operations (non-atomic versions)
- *
- ************************************************************************/
-
-void opal_mutex_create(struct opal_mutex_t *m);
-
-static inline int opal_mutex_trylock(opal_mutex_t *m)
+static inline int opal_thread_internal_mutex_trylock(opal_thread_internal_mutex_t *p_mutex)
 {
-    if (OPAL_ABT_MUTEX_NULL == m->m_lock_argobots) {
-        opal_mutex_create(m);
-    }
-    int ret = ABT_mutex_trylock(m->m_lock_argobots);
+    ABT_mutex mutex = ABT_MUTEX_MEMORY_GET_HANDLE(p_mutex);
+    int ret = ABT_mutex_trylock(mutex);
     if (ABT_ERR_MUTEX_LOCKED == ret) {
         return 1;
     } else if (ABT_SUCCESS != ret) {
 #if OPAL_ENABLE_DEBUG
-        opal_output(0, "opal_mutex_trylock()");
+        opal_show_help("help-opal-threads.txt", "mutex trylock failed", true);
 #endif
         return 1;
     }
     return 0;
 }
 
-static inline void opal_mutex_lock(opal_mutex_t *m)
+static inline void opal_thread_internal_mutex_unlock(opal_thread_internal_mutex_t *p_mutex)
 {
-    if (OPAL_ABT_MUTEX_NULL == m->m_lock_argobots) {
-        opal_mutex_create(m);
-    }
+    ABT_mutex mutex = ABT_MUTEX_MEMORY_GET_HANDLE(p_mutex);
 #if OPAL_ENABLE_DEBUG
-    int ret = ABT_mutex_lock(m->m_lock_argobots);
+    int ret = ABT_mutex_unlock(mutex);
     if (ABT_SUCCESS != ret) {
-        opal_output(0, "opal_mutex_lock()");
+        opal_show_help("help-opal-threads.txt", "mutex unlock failed", true);
     }
 #else
-    ABT_mutex_lock(m->m_lock_argobots);
-#endif
-}
-
-static inline void opal_mutex_unlock(opal_mutex_t *m)
-{
-    if (OPAL_ABT_MUTEX_NULL == m->m_lock_argobots) {
-        opal_mutex_create(m);
-    }
-#if OPAL_ENABLE_DEBUG
-    int ret = ABT_mutex_unlock(m->m_lock_argobots);
-    if (ABT_SUCCESS != ret) {
-        opal_output(0, "opal_mutex_unlock()");
-    }
-#else
-    ABT_mutex_unlock(m->m_lock_argobots);
+    ABT_mutex_unlock(mutex);
 #endif
     /* For fairness of locking. */
     ABT_thread_yield();
 }
 
-/************************************************************************
- *
- * mutex operations (atomic versions)
- *
- ************************************************************************/
+static inline void opal_thread_internal_mutex_destroy(opal_thread_internal_mutex_t *p_mutex)
+{
+    /* No specific operation is needed to destroy opal_thread_internal_mutex_t. */
+}
 
-#if OPAL_HAVE_ATOMIC_SPINLOCKS
+typedef ABT_cond_memory opal_thread_internal_cond_t;
 
-/************************************************************************
- * Spin Locks
- ************************************************************************/
+#define OPAL_THREAD_INTERNAL_COND_INITIALIZER ABT_COND_INITIALIZER
 
-static inline int opal_mutex_atomic_trylock(opal_mutex_t *m)
+static inline int opal_thread_internal_cond_init(opal_thread_internal_cond_t *p_cond)
 {
-    return opal_atomic_trylock(&m->m_lock_atomic);
+    const ABT_cond_memory init_cond = ABT_COND_INITIALIZER;
+    memcpy(p_cond, &init_cond, sizeof(ABT_cond_memory));
+    return OPAL_SUCCESS;
 }
 
-static inline void opal_mutex_atomic_lock(opal_mutex_t *m)
+static inline void opal_thread_internal_cond_wait(opal_thread_internal_cond_t *p_cond,
+                                                  opal_thread_internal_mutex_t *p_mutex)
 {
-    opal_atomic_lock(&m->m_lock_atomic);
+    ABT_mutex mutex = ABT_MUTEX_MEMORY_GET_HANDLE(p_mutex);
+    ABT_cond cond = ABT_COND_MEMORY_GET_HANDLE(p_cond);
+#if OPAL_ENABLE_DEBUG
+    int ret = ABT_cond_wait(cond, mutex);
+    assert(ABT_SUCCESS == ret);
+#else
+    ABT_cond_wait(cond, mutex);
+#endif
 }
 
-static inline void opal_mutex_atomic_unlock(opal_mutex_t *m)
+static inline void opal_thread_internal_cond_broadcast(opal_thread_internal_cond_t *p_cond)
 {
-    opal_atomic_unlock(&m->m_lock_atomic);
-}
-
+    ABT_cond cond = ABT_COND_MEMORY_GET_HANDLE(p_cond);
+#if OPAL_ENABLE_DEBUG
+    int ret = ABT_cond_broadcast(cond);
+    assert(ABT_SUCCESS == ret);
 #else
-
-/************************************************************************
- * Standard locking
- ************************************************************************/
-
-static inline int opal_mutex_atomic_trylock(opal_mutex_t *m)
-{
-    return opal_mutex_trylock(m);
+    ABT_cond_broadcast(cond);
+#endif
 }
 
-static inline void opal_mutex_atomic_lock(opal_mutex_t *m)
+static inline void opal_thread_internal_cond_signal(opal_thread_internal_cond_t *p_cond)
 {
-    opal_mutex_lock(m);
+    ABT_cond cond = ABT_COND_MEMORY_GET_HANDLE(p_cond);
+#if OPAL_ENABLE_DEBUG
+    int ret = ABT_cond_signal(cond);
+    assert(ABT_SUCCESS == ret);
+#else
+    ABT_cond_signal(cond);
+#endif
 }
 
-static inline void opal_mutex_atomic_unlock(opal_mutex_t *m)
+static inline void opal_thread_internal_cond_destroy(opal_thread_internal_cond_t *p_cond)
 {
-    opal_mutex_unlock(m);
+    /* No destructor is needed. */
 }
 
-#endif
-
-#define OPAL_ABT_COND_NULL NULL
-typedef ABT_cond opal_cond_t;
-#define OPAL_CONDITION_STATIC_INIT OPAL_ABT_COND_NULL
-
-int opal_cond_init(opal_cond_t *cond);
-int opal_cond_wait(opal_cond_t *cond, opal_mutex_t *lock);
-int opal_cond_broadcast(opal_cond_t *cond);
-int opal_cond_signal(opal_cond_t *cond);
-int opal_cond_destroy(opal_cond_t *cond);
-
 END_C_DECLS
 
 #endif /* OPAL_MCA_THREADS_ARGOBOTS_THREADS_ARGOBOTS_MUTEX_H */
diff --git a/opal/mca/threads/argobots/threads_argobots_threads.h b/opal/mca/threads/argobots/threads_argobots_threads.h
index 3992b32f274..417ad3d5884 100644
--- a/opal/mca/threads/argobots/threads_argobots_threads.h
+++ b/opal/mca/threads/argobots/threads_argobots_threads.h
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2020 High Performance Computing Center Stuttgart,
@@ -26,23 +26,12 @@
 #ifndef OPAL_MCA_THREADS_ARGOBOTS_THREADS_ARGOBOTS_THREADS_H
 #define OPAL_MCA_THREADS_ARGOBOTS_THREADS_ARGOBOTS_THREADS_H
 
-#include <signal.h>
 #include "opal/mca/threads/argobots/threads_argobots.h"
 
-struct opal_thread_t {
-    opal_object_t super;
-    opal_thread_fn_t t_run;
-    void *t_arg;
-    ABT_thread t_handle;
-    void *t_ret;
-};
-
-
 /* Argobots are cooperatively scheduled so yield when idle */
 #define OPAL_THREAD_YIELD_WHEN_IDLE_DEFAULT true
 
-static inline
-void opal_thread_yield(void)
+static inline void opal_thread_yield(void)
 {
     ABT_thread_yield();
 }
diff --git a/opal/mca/threads/argobots/threads_argobots_tsd.h b/opal/mca/threads/argobots/threads_argobots_tsd.h
index e441d56992d..c1c2468438d 100644
--- a/opal/mca/threads/argobots/threads_argobots_tsd.h
+++ b/opal/mca/threads/argobots/threads_argobots_tsd.h
@@ -23,7 +23,6 @@
  * $HEADER$
  */
 
-
 #ifndef OPAL_MCA_THREADS_ARGOBOTS_THREADS_ARGOBOTS_TSD_H
 #define OPAL_MCA_THREADS_ARGOBOTS_THREADS_ARGOBOTS_TSD_H
 
diff --git a/opal/mca/threads/argobots/threads_argobots_wait_sync.c b/opal/mca/threads/argobots/threads_argobots_wait_sync.c
deleted file mode 100644
index 34ec10d5a74..00000000000
--- a/opal/mca/threads/argobots/threads_argobots_wait_sync.c
+++ /dev/null
@@ -1,116 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2014-2016 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
- * Copyright (c) 2019      Sandia National Laboratories.  All rights reserved.
- *
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "opal/mca/threads/argobots/threads_argobots.h"
-#include "opal/mca/threads/wait_sync.h"
-
-static opal_mutex_t wait_sync_lock = OPAL_MUTEX_STATIC_INIT;
-static ompi_wait_sync_t *wait_sync_list = NULL;
-
-static opal_atomic_int32_t num_thread_in_progress = 0;
-
-#define WAIT_SYNC_PASS_OWNERSHIP(who)                  \
-    do {                                               \
-        ABT_mutex_lock((who)->lock);                   \
-        ABT_cond_signal((who)->condition );            \
-        ABT_mutex_unlock((who)->lock);                 \
-    } while (0)
-
-int ompi_sync_wait_mt(ompi_wait_sync_t *sync)
-{
-    /* Don't stop if the waiting synchronization is completed. We avoid the
-     * race condition around the release of the synchronization using the
-     * signaling field.
-     */
-    if (sync->count <= 0) {
-        return (0 == sync->status) ? OPAL_SUCCESS : OPAL_ERROR;
-    }
-
-    /* lock so nobody can signal us during the list updating */
-    ABT_mutex_lock(sync->lock);
-
-    /* Now that we hold the lock make sure another thread has not already
-     * call cond_signal.
-     */
-    if (sync->count <= 0) {
-        ABT_mutex_unlock(sync->lock);
-        return (0 == sync->status) ? OPAL_SUCCESS : OPAL_ERROR;
-    }
-
-    /* Insert sync on the list of pending synchronization constructs */
-    OPAL_THREAD_LOCK(&wait_sync_lock);
-    if (NULL == wait_sync_list) {
-        sync->next = sync->prev = sync;
-        wait_sync_list = sync;
-    } else {
-        sync->prev = wait_sync_list->prev;
-        sync->prev->next = sync;
-        sync->next = wait_sync_list;
-        wait_sync_list->prev = sync;
-    }
-    OPAL_THREAD_UNLOCK(&wait_sync_lock);
-
-    /**
-     * If we are not responsible for progressing, go silent until something
-     * worth noticing happen:
-     *  - this thread has been promoted to take care of the progress
-     *  - our sync has been triggered.
-     */
-  check_status:
-    if (sync != wait_sync_list &&
-        num_thread_in_progress >= opal_max_thread_in_progress) {
-        ABT_cond_wait(sync->condition, sync->lock);
-
-        /**
-         * At this point either the sync was completed in which case
-         * we should remove it from the wait list, or/and I was
-         * promoted as the progress manager.
-         */
-
-        if (sync->count <= 0) { /* Completed? */
-            ABT_mutex_unlock(sync->lock);
-            goto i_am_done;
-        }
-        /* either promoted, or spurious wakeup ! */
-        goto check_status;
-    }
-    ABT_mutex_unlock(sync->lock);
-
-    OPAL_THREAD_ADD_FETCH32(&num_thread_in_progress, 1);
-    while (sync->count > 0) { /* progress till completion */
-        /* don't progress with the sync lock locked or you'll deadlock */
-        opal_progress();
-        ABT_thread_yield();
-    }
-    OPAL_THREAD_ADD_FETCH32(&num_thread_in_progress, -1);
-
-  i_am_done:
-    /* My sync is now complete. Trim the list: remove self, wake next */
-    OPAL_THREAD_LOCK(&wait_sync_lock);
-    sync->prev->next = sync->next;
-    sync->next->prev = sync->prev;
-    /* In case I am the progress manager, pass the duties on */
-    if (sync == wait_sync_list) {
-        wait_sync_list = (sync == sync->next) ? NULL : sync->next;
-        if (NULL != wait_sync_list) {
-            WAIT_SYNC_PASS_OWNERSHIP(wait_sync_list);
-        }
-    }
-    OPAL_THREAD_UNLOCK(&wait_sync_lock);
-
-    return (0 == sync->status) ? OPAL_SUCCESS : OPAL_ERROR;
-}
diff --git a/opal/mca/threads/argobots/threads_argobots_wait_sync.h b/opal/mca/threads/argobots/threads_argobots_wait_sync.h
deleted file mode 100644
index 0c46c7b872d..00000000000
--- a/opal/mca/threads/argobots/threads_argobots_wait_sync.h
+++ /dev/null
@@ -1,106 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007-2016 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2019      Sandia National Laboratories.  All rights reserved.
- *
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef OPAL_MCA_THREADS_ARGOBOTS_THREADS_ARGOBOTS_WAIT_SYNC_H
-#define OPAL_MCA_THREADS_ARGOBOTS_THREADS_ARGOBOTS_WAIT_SYNC_H
-
-#include "opal/mca/threads/argobots/threads_argobots.h"
-
-typedef struct ompi_wait_sync_t {
-    opal_atomic_int32_t count;
-    int32_t status;
-    ABT_cond condition;
-    ABT_mutex lock;
-    struct ompi_wait_sync_t *next;
-    struct ompi_wait_sync_t *prev;
-    volatile bool signaling;
-} ompi_wait_sync_t;
-
-#define SYNC_WAIT(sync) \
-    (opal_using_threads() ? ompi_sync_wait_mt (sync) : sync_wait_st (sync))
-
-/* The loop in release handles a race condition between the signaling
- * thread and the destruction of the condition variable. The signaling
- * member will be set to false after the final signaling thread has
- * finished operating on the sync object. This is done to avoid
- * extra atomics in the signalling function and keep it as fast
- * as possible. Note that the race window is small so spinning here
- * is more optimal than sleeping since this macro is called in
- * the critical path. */
-#define WAIT_SYNC_RELEASE(sync)                       \
-    if (opal_using_threads()) {                       \
-        while ((sync)->signaling) {                   \
-            ABT_thread_yield();                       \
-            continue;                                 \
-        }                                             \
-        ABT_cond_free(&(sync)->condition);            \
-        ABT_mutex_free(&(sync)->lock);                \
-    }
-
-#define WAIT_SYNC_RELEASE_NOWAIT(sync)                \
-    if (opal_using_threads()) {                       \
-        ABT_cond_free(&(sync)->condition);            \
-        ABT_mutex_free(&(sync)->lock);                \
-    }
-
-
-#define WAIT_SYNC_SIGNAL(sync)                        \
-    if (opal_using_threads()) {                       \
-        ABT_mutex_lock(sync->lock);                   \
-        ABT_cond_signal(sync->condition);             \
-        ABT_mutex_unlock(sync->lock);                 \
-        sync->signaling = false;                      \
-    }
-
-#define WAIT_SYNC_SIGNALLED(sync)                     \
-    {                                                 \
-        (sync)->signaling = false;                    \
-    }
-
-OPAL_DECLSPEC int ompi_sync_wait_mt(ompi_wait_sync_t *sync);
-static inline int sync_wait_st(ompi_wait_sync_t *sync)
-{
-    while (sync->count > 0) {
-        opal_progress();
-        ABT_thread_yield();
-    }
-    return sync->status;
-}
-
-
-#define WAIT_SYNC_INIT(sync,c)                                  \
-    do {                                                        \
-        (sync)->count = (c);                                    \
-        (sync)->next = NULL;                                    \
-        (sync)->prev = NULL;                                    \
-        (sync)->status = 0;                                     \
-        (sync)->signaling = (0 != (c));                         \
-        if (opal_using_threads()) {                             \
-            ABT_cond_create(&(sync)->condition);                \
-            ABT_mutex_create(&(sync)->lock);                    \
-        }                                                       \
-    } while (0)
-
-#endif /* OPAL_MCA_THREADS_ARGOBOTS_THREADS_ARGOBOTS_WAIT_SYNC_H */
diff --git a/opal/mca/threads/base/Makefile.am b/opal/mca/threads/base/Makefile.am
index fcf7c0b3a67..91e24f11198 100644
--- a/opal/mca/threads/base/Makefile.am
+++ b/opal/mca/threads/base/Makefile.am
@@ -10,6 +10,7 @@
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
 # Copyright (c) 2019      Sandia National Laboratories.  All rights reserved.
+# Copyright (c) 2021      Argonne National Laboratory.  All rights reserved.
 #
 # $COPYRIGHT$
 #
@@ -22,5 +23,8 @@ headers += \
         base/base.h
 
 libmca_threads_la_SOURCES += \
+        base/mutex.c \
+        base/create_join.c \
         base/threads_base.c \
-        base/tsd.c
+        base/tsd.c \
+        base/wait_sync.c
diff --git a/opal/mca/threads/base/base.h b/opal/mca/threads/base/base.h
index dd774005068..f6a0a6ab70f 100644
--- a/opal/mca/threads/base/base.h
+++ b/opal/mca/threads/base/base.h
@@ -28,7 +28,6 @@
 #include "opal/mca/base/mca_base_framework.h"
 #include "opal/mca/threads/threads.h"
 
-
 /*
  * Global functions for MCA overall threads open and close
  */
diff --git a/opal/mca/threads/base/create_join.c b/opal/mca/threads/base/create_join.c
new file mode 100644
index 00000000000..e134f085930
--- /dev/null
+++ b/opal/mca/threads/base/create_join.c
@@ -0,0 +1,83 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007-2018 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2016 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2019      Sandia National Laboratories.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include <unistd.h>
+#include <pthread.h>
+
+#include "opal/constants.h"
+#include "opal/mca/threads/threads.h"
+#include "opal/mca/threads/tsd.h"
+#include "opal/prefetch.h"
+#include "opal/util/output.h"
+#include "opal/util/sys_limits.h"
+
+/*
+ * Constructor
+ */
+static void opal_thread_construct(opal_thread_t *t)
+{
+    t->t_run = 0;
+    t->t_handle = (pthread_t) -1;
+}
+
+OBJ_CLASS_INSTANCE(opal_thread_t, opal_object_t, opal_thread_construct, NULL);
+
+int opal_thread_start(opal_thread_t *t)
+{
+    int rc;
+
+    if (OPAL_ENABLE_DEBUG) {
+        if (NULL == t->t_run || (pthread_t) -1 != t->t_handle) {
+            return OPAL_ERR_BAD_PARAM;
+        }
+    }
+
+    rc = pthread_create(&t->t_handle, NULL, (void *(*) (void *) ) t->t_run, t);
+
+    return 0 == rc ? OPAL_SUCCESS : OPAL_ERR_IN_ERRNO;
+}
+
+int opal_thread_join(opal_thread_t *t, void **thr_return)
+{
+    int rc = pthread_join(t->t_handle, thr_return);
+    t->t_handle = (pthread_t) -1;
+    return 0 == rc ? OPAL_SUCCESS : OPAL_ERR_IN_ERRNO;
+}
+
+bool opal_thread_self_compare(opal_thread_t *t)
+{
+    return pthread_self() == t->t_handle;
+}
+
+opal_thread_t *opal_thread_get_self(void)
+{
+    opal_thread_t *t = OBJ_NEW(opal_thread_t);
+    t->t_handle = pthread_self();
+    return t;
+}
+
+void opal_thread_set_main(void)
+{
+}
diff --git a/opal/mca/threads/base/help-opal-threads.txt b/opal/mca/threads/base/help-opal-threads.txt
new file mode 100644
index 00000000000..9f60eb67e0b
--- /dev/null
+++ b/opal/mca/threads/base/help-opal-threads.txt
@@ -0,0 +1,22 @@
+# -*- text -*-
+#
+# Copyright (c) 2022      Sandia National Laboratories.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+# This is the US/English help file for Open MPI's OPAL thread MCA.
+#
+[mutex init failed]
+A mutex initialization has failed.
+
+[mutex lock failed]
+A mutex acquire (mutex lock) has failed.
+
+[mutex unlock failed]
+A mutex release (mutex unlock) has failed.
+
+[mutex trylock failed]
+A try to acquire a mutex (mutex trylock) has failed.
\ No newline at end of file
diff --git a/opal/mca/threads/base/mutex.c b/opal/mca/threads/base/mutex.c
new file mode 100644
index 00000000000..fec6cee1d98
--- /dev/null
+++ b/opal/mca/threads/base/mutex.c
@@ -0,0 +1,109 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007-2016 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2019      Sandia National Laboratories.  All rights reserved.
+ * Copyright (c) 2021      Argonne National Laboratory.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "opal_config.h"
+
+#include "opal/constants.h"
+#include "opal/mca/threads/mutex.h"
+
+/*
+ * Wait and see if some upper layer wants to use threads, if support
+ * exists.
+ */
+bool opal_uses_threads = false;
+
+static void mca_threads_mutex_constructor(opal_mutex_t *p_mutex)
+{
+#if OPAL_ENABLE_DEBUG
+    int ret = opal_thread_internal_mutex_init(&p_mutex->m_lock, false);
+    assert(0 == ret);
+    p_mutex->m_lock_debug = 0;
+    p_mutex->m_lock_file = NULL;
+    p_mutex->m_lock_line = 0;
+#else
+    opal_thread_internal_mutex_init(&p_mutex->m_lock, false);
+#endif
+    opal_atomic_lock_init(&p_mutex->m_lock_atomic, 0);
+}
+
+static void mca_threads_mutex_destructor(opal_mutex_t *p_mutex)
+{
+    opal_thread_internal_mutex_destroy(&p_mutex->m_lock);
+}
+
+static void mca_threads_recursive_mutex_constructor(opal_recursive_mutex_t *p_mutex)
+{
+#if OPAL_ENABLE_DEBUG
+    int ret = opal_thread_internal_mutex_init(&p_mutex->m_lock, true);
+    assert(0 == ret);
+    p_mutex->m_lock_debug = 0;
+    p_mutex->m_lock_file = NULL;
+    p_mutex->m_lock_line = 0;
+#else
+    opal_thread_internal_mutex_init(&p_mutex->m_lock, true);
+#endif
+    opal_atomic_lock_init(&p_mutex->m_lock_atomic, 0);
+}
+
+static void mca_threads_recursive_mutex_destructor(opal_recursive_mutex_t *p_mutex)
+{
+    opal_thread_internal_mutex_destroy(&p_mutex->m_lock);
+}
+
+OBJ_CLASS_INSTANCE(opal_mutex_t, opal_object_t, mca_threads_mutex_constructor,
+                   mca_threads_mutex_destructor);
+
+OBJ_CLASS_INSTANCE(opal_recursive_mutex_t, opal_object_t, mca_threads_recursive_mutex_constructor,
+                   mca_threads_recursive_mutex_destructor);
+
+int opal_cond_init(opal_cond_t *cond)
+{
+    return opal_thread_internal_cond_init(cond);
+}
+
+int opal_cond_wait(opal_cond_t *cond, opal_mutex_t *lock)
+{
+    opal_thread_internal_cond_wait(cond, &lock->m_lock);
+    return OPAL_SUCCESS;
+}
+
+int opal_cond_broadcast(opal_cond_t *cond)
+{
+    opal_thread_internal_cond_broadcast(cond);
+    return OPAL_SUCCESS;
+}
+
+int opal_cond_signal(opal_cond_t *cond)
+{
+    opal_thread_internal_cond_signal(cond);
+    return OPAL_SUCCESS;
+}
+
+int opal_cond_destroy(opal_cond_t *cond)
+{
+    opal_thread_internal_cond_destroy(cond);
+    return OPAL_SUCCESS;
+}
diff --git a/opal/mca/threads/base/threads_base.c b/opal/mca/threads/base/threads_base.c
index a0611da1900..227aaeb64d3 100644
--- a/opal/mca/threads/base/threads_base.c
+++ b/opal/mca/threads/base/threads_base.c
@@ -45,6 +45,5 @@ static int mca_threads_base_register(mca_base_register_flag_t flags)
  * Globals
  */
 /* Use default register/open/close functions */
-MCA_BASE_FRAMEWORK_DECLARE(opal, threads, "OPAL threads",
-                           mca_threads_base_register, NULL, NULL,
+MCA_BASE_FRAMEWORK_DECLARE(opal, threads, "OPAL threads", mca_threads_base_register, NULL, NULL,
                            mca_threads_base_static_components, 0);
diff --git a/opal/mca/threads/base/tsd.c b/opal/mca/threads/base/tsd.c
index bb5c92afb99..4eec185c4c0 100644
--- a/opal/mca/threads/base/tsd.c
+++ b/opal/mca/threads/base/tsd.c
@@ -1,15 +1,16 @@
 #include "opal/mca/threads/tsd.h"
 
-static void _tracked_destructor(void *arg) {
+static void _tracked_destructor(void *arg)
+{
     opal_tsd_list_item_t *tsd = NULL;
     opal_tsd_tracked_key_t *key = NULL;
     if (NULL == arg) {
         return;
     }
 
-    tsd = (opal_tsd_list_item_t *)arg;
+    tsd = (opal_tsd_list_item_t *) arg;
     key = tsd->tracked_key;
-  
+
     opal_mutex_lock(&key->mutex);
     opal_list_remove_item(&key->tsd_list, &tsd->super);
     opal_mutex_unlock(&key->mutex);
@@ -20,7 +21,8 @@ static void _tracked_destructor(void *arg) {
     OBJ_RELEASE(tsd);
 }
 
-void opal_tsd_tracked_key_constructor(opal_tsd_tracked_key_t *key) {
+void opal_tsd_tracked_key_constructor(opal_tsd_tracked_key_t *key)
+{
     OBJ_CONSTRUCT(&key->mutex, opal_mutex_t);
     OBJ_CONSTRUCT(&key->tsd_list, opal_list_t);
     key->user_destructor = NULL;
@@ -32,7 +34,7 @@ void opal_tsd_tracked_key_destructor(opal_tsd_tracked_key_t *key)
     opal_tsd_list_item_t *tsd, *next;
 
     opal_tsd_key_delete(key->key);
-    OPAL_LIST_FOREACH_SAFE(tsd, next, &key->tsd_list, opal_tsd_list_item_t) {
+    OPAL_LIST_FOREACH_SAFE (tsd, next, &key->tsd_list, opal_tsd_list_item_t) {
         opal_list_remove_item(&key->tsd_list, &tsd->super);
         if (NULL != key->user_destructor) {
             key->user_destructor(tsd->data);
@@ -45,10 +47,10 @@ void opal_tsd_tracked_key_destructor(opal_tsd_tracked_key_t *key)
 
 int opal_tsd_tracked_key_set(opal_tsd_tracked_key_t *key, void *p)
 {
-    assert( NULL != key);
-   
+    assert(NULL != key);
+
     opal_tsd_list_item_t *tsd = NULL;
-    opal_tsd_get(key->key, (void **)&tsd); 
+    opal_tsd_get(key->key, (void **) &tsd);
 
     if (NULL == tsd) {
         tsd = OBJ_NEW(opal_tsd_list_item_t);
@@ -64,15 +66,16 @@ int opal_tsd_tracked_key_set(opal_tsd_tracked_key_t *key, void *p)
     tsd->data = p;
     tsd->tracked_key = key;
 
-    return opal_tsd_set(key->key, (void *)tsd);
+    return opal_tsd_set(key->key, (void *) tsd);
 }
 
-void opal_tsd_tracked_key_set_destructor(opal_tsd_tracked_key_t *key, opal_tsd_destructor_t destructor)
+void opal_tsd_tracked_key_set_destructor(opal_tsd_tracked_key_t *key,
+                                         opal_tsd_destructor_t destructor)
 {
-    assert (NULL != key);
+    assert(NULL != key);
     key->user_destructor = destructor;
 }
 
 OBJ_CLASS_INSTANCE(opal_tsd_list_item_t, opal_list_item_t, NULL, NULL);
-OBJ_CLASS_INSTANCE(opal_tsd_tracked_key_t, opal_object_t,
-        opal_tsd_tracked_key_constructor, opal_tsd_tracked_key_destructor);
+OBJ_CLASS_INSTANCE(opal_tsd_tracked_key_t, opal_object_t, opal_tsd_tracked_key_constructor,
+                   opal_tsd_tracked_key_destructor);
diff --git a/opal/mca/threads/base/wait_sync.c b/opal/mca/threads/base/wait_sync.c
new file mode 100644
index 00000000000..2458a9614be
--- /dev/null
+++ b/opal/mca/threads/base/wait_sync.c
@@ -0,0 +1,140 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2014-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2017-2022 IBM Corporation. All rights reserved.
+ * Copyright (c) 2019      Sandia National Laboratories.  All rights reserved.
+ * Copyright (c) 2021      Argonne National Laboratory.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "opal/mca/threads/wait_sync.h"
+
+static opal_mutex_t wait_sync_lock = OPAL_MUTEX_STATIC_INIT;
+ompi_wait_sync_t *opal_threads_base_wait_sync_list = NULL; /* not static for inline "wait_sync_st" */
+
+void opal_threads_base_wait_sync_global_wakeup_st(int status)
+{
+    ompi_wait_sync_t *sync;
+    for (sync = opal_threads_base_wait_sync_list; sync != NULL; sync = sync->next) {
+        wait_sync_update(sync, 0, status);
+    }
+}
+
+void opal_threads_base_wait_sync_global_wakeup_mt(int status)
+{
+    ompi_wait_sync_t *sync;
+    opal_mutex_lock(&wait_sync_lock);
+    for (sync = opal_threads_base_wait_sync_list; sync != NULL; sync = sync->next) {
+        /* sync_update is going to  take the sync->lock from within
+         * the wait_sync_lock. Thread lightly here: Idealy we should
+         * find a way to not take a lock in a lock as this is deadlock prone,
+         * but as of today we are the only place doing this so it is safe.
+         */
+        wait_sync_update(sync, 0, status);
+        if (sync->next == opal_threads_base_wait_sync_list) {
+            break; /* special case for rings */
+        }
+    }
+    opal_mutex_unlock(&wait_sync_lock);
+}
+
+static opal_atomic_int32_t num_thread_in_progress = 0;
+
+#define WAIT_SYNC_PASS_OWNERSHIP(who)                        \
+    do {                                                     \
+        opal_thread_internal_mutex_lock(&(who)->lock);       \
+        opal_thread_internal_cond_signal(&(who)->condition); \
+        opal_thread_internal_mutex_unlock(&(who)->lock);     \
+    } while (0)
+
+int ompi_sync_wait_mt(ompi_wait_sync_t *sync)
+{
+    /* Don't stop if the waiting synchronization is completed. We avoid the
+     * race condition around the release of the synchronization using the
+     * signaling field.
+     */
+    if (sync->count <= 0) {
+        return (0 == sync->status) ? OPAL_SUCCESS : OPAL_ERROR;
+    }
+
+    /* lock so nobody can signal us during the list updating */
+    opal_thread_internal_mutex_lock(&sync->lock);
+
+    /* Now that we hold the lock make sure another thread has not already
+     * call cond_signal.
+     */
+    if (sync->count <= 0) {
+        opal_thread_internal_mutex_unlock(&sync->lock);
+        return (0 == sync->status) ? OPAL_SUCCESS : OPAL_ERROR;
+    }
+
+    /* Insert sync on the list of pending synchronization constructs */
+    OPAL_THREAD_LOCK(&wait_sync_lock);
+    if (NULL == opal_threads_base_wait_sync_list) {
+        sync->next = sync->prev = sync;
+        opal_threads_base_wait_sync_list = sync;
+    } else {
+        sync->prev = opal_threads_base_wait_sync_list->prev;
+        sync->prev->next = sync;
+        sync->next = opal_threads_base_wait_sync_list;
+        opal_threads_base_wait_sync_list->prev = sync;
+    }
+    OPAL_THREAD_UNLOCK(&wait_sync_lock);
+
+    /**
+     * If we are not responsible for progressing, go silent until something
+     * worth noticing happen:
+     *  - this thread has been promoted to take care of the progress
+     *  - our sync has been triggered.
+     */
+check_status:
+    if (sync != opal_threads_base_wait_sync_list && num_thread_in_progress >= opal_max_thread_in_progress) {
+        opal_thread_internal_cond_wait(&sync->condition, &sync->lock);
+
+        /**
+         * At this point either the sync was completed in which case
+         * we should remove it from the wait list, or/and I was
+         * promoted as the progress manager.
+         */
+
+        if (sync->count <= 0) { /* Completed? */
+            opal_thread_internal_mutex_unlock(&sync->lock);
+            goto i_am_done;
+        }
+        /* either promoted, or spurious wakeup ! */
+        goto check_status;
+    }
+    opal_thread_internal_mutex_unlock(&sync->lock);
+
+    OPAL_THREAD_ADD_FETCH32(&num_thread_in_progress, 1);
+    while (sync->count > 0) { /* progress till completion */
+        /* don't progress with the sync lock locked or you'll deadlock */
+        opal_progress();
+    }
+    OPAL_THREAD_ADD_FETCH32(&num_thread_in_progress, -1);
+
+i_am_done:
+    /* My sync is now complete. Trim the list: remove self, wake next */
+    OPAL_THREAD_LOCK(&wait_sync_lock);
+    sync->prev->next = sync->next;
+    sync->next->prev = sync->prev;
+    /* In case I am the progress manager, pass the duties on */
+    if (sync == opal_threads_base_wait_sync_list) {
+        opal_threads_base_wait_sync_list = (sync == sync->next) ? NULL : sync->next;
+        if (NULL != opal_threads_base_wait_sync_list) {
+            WAIT_SYNC_PASS_OWNERSHIP(opal_threads_base_wait_sync_list);
+        }
+    }
+    OPAL_THREAD_UNLOCK(&wait_sync_lock);
+
+    return (0 == sync->status) ? OPAL_SUCCESS : OPAL_ERROR;
+}
diff --git a/opal/mca/threads/condition.h b/opal/mca/threads/condition.h
index 031c8e9b3ec..799b8bae0b3 100644
--- a/opal/mca/threads/condition.h
+++ b/opal/mca/threads/condition.h
@@ -30,10 +30,10 @@
 
 #include "opal_config.h"
 #ifdef HAVE_SYS_TIME_H
-#include <sys/time.h>
+#    include <sys/time.h>
 #endif
-#include <time.h>
 #include <pthread.h>
+#include <time.h>
 
 #include "opal/constants.h"
 #include "opal/mca/threads/mutex.h"
@@ -55,7 +55,6 @@ typedef struct opal_condition_t opal_condition_t;
 
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_condition_t);
 
-
 static inline int opal_condition_wait(opal_condition_t *c, opal_mutex_t *m)
 {
     int rc = OPAL_SUCCESS;
@@ -103,9 +102,9 @@ static inline int opal_condition_timedwait(opal_condition_t *c, opal_mutex_t *m,
                 opal_progress();
                 gettimeofday(&tv, NULL);
                 opal_mutex_lock(m);
-            } while (0 == c->c_signaled && (tv.tv_sec <= absolute.tv_sec ||
-                                            (tv.tv_sec == absolute.tv_sec &&
-                                             tv.tv_usec < absolute.tv_usec)));
+            } while (0 == c->c_signaled
+                     && (tv.tv_sec <= absolute.tv_sec
+                         || (tv.tv_sec == absolute.tv_sec && tv.tv_usec < absolute.tv_usec)));
         }
     } else {
         absolute.tv_sec = abstime->tv_sec;
@@ -115,9 +114,9 @@ static inline int opal_condition_timedwait(opal_condition_t *c, opal_mutex_t *m,
             do {
                 opal_progress();
                 gettimeofday(&tv, NULL);
-            } while (0 == c->c_signaled && (tv.tv_sec <= absolute.tv_sec ||
-                                            (tv.tv_sec == absolute.tv_sec &&
-                                             tv.tv_usec < absolute.tv_usec)));
+            } while (0 == c->c_signaled
+                     && (tv.tv_sec <= absolute.tv_sec
+                         || (tv.tv_sec == absolute.tv_sec && tv.tv_usec < absolute.tv_usec)));
         }
     }
 
diff --git a/opal/mca/threads/configure.m4 b/opal/mca/threads/configure.m4
index 10690b95873..6325401ad31 100644
--- a/opal/mca/threads/configure.m4
+++ b/opal/mca/threads/configure.m4
@@ -14,7 +14,7 @@ dnl Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
 dnl Copyright (c) 2019      Sandia National Laboratories.  All rights reserved.
 dnl Copyright (c) 2019      Triad National Security, LLC. All rights
 dnl                         reserved.
-dnl
+dnl Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
@@ -22,6 +22,9 @@ dnl
 dnl $HEADER$
 dnl
 
+dnl need in the core library
+AC_DEFUN([MCA_opal_threads_CORE_LIB], [1])
+
 dnl we only want one :)
 m4_define(MCA_opal_threads_CONFIGURE_MODE, STOP_AT_FIRST)
 
@@ -63,5 +66,5 @@ AC_SUBST(THREAD_CPPFLAGS)
 AC_SUBST(THREAD_LDFLAGS)
 AC_SUBST(THREAD_LIBS)
 
-OPAL_SUMMARY_ADD([[Miscellaneous]],[[Threading Package]],[], [$opal_thread_type_found])
+OPAL_SUMMARY_ADD([Miscellaneous], [Threading Package], [], [$opal_thread_type_found])
 ])dnl
diff --git a/opal/mca/threads/mutex.h b/opal/mca/threads/mutex.h
index a81e94cb6e8..94846e4d3c7 100644
--- a/opal/mca/threads/mutex.h
+++ b/opal/mca/threads/mutex.h
@@ -11,11 +11,15 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2007-2016 Los Alamos National Security, LLC.  All rights
+ * Copyright (c) 2007-2018 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2007      Voltaire. All rights reserved.
  * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved.
  * Copyright (c) 2019      Sandia National Laboratories.  All rights reserved.
+ * Copyright (c) 2020      Triad National Security, LLC.  All rights reserved.
+ * Copyright (c) 2021      Argonne National Laboratory.  All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  *
  * $COPYRIGHT$
  *
@@ -28,6 +32,7 @@
 #define OPAL_MCA_THREADS_MUTEX_H
 
 #include "opal_config.h"
+#include "opal/sys/atomic.h"
 
 BEGIN_C_DECLS
 
@@ -48,9 +53,53 @@ typedef struct opal_mutex_t opal_recursive_mutex_t;
 
 #include MCA_threads_mutex_base_include_HEADER
 
-OBJ_CLASS_DECLARATION(opal_mutex_t);
-OBJ_CLASS_DECLARATION(opal_recursive_mutex_t);
+struct opal_mutex_t {
+    opal_object_t super;
+    opal_thread_internal_mutex_t m_lock;
+#if OPAL_ENABLE_DEBUG
+    int m_lock_debug;
+    const char *m_lock_file;
+    int m_lock_line;
+#endif
+    opal_atomic_lock_t m_lock_atomic;
+};
 
+OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_mutex_t);
+OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_recursive_mutex_t);
+
+#if OPAL_ENABLE_DEBUG
+#    define OPAL_MUTEX_STATIC_INIT                                                         \
+        {                                                                                  \
+            .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t),                                   \
+            .m_lock = OPAL_THREAD_INTERNAL_MUTEX_INITIALIZER, .m_lock_debug = 0,           \
+            .m_lock_file = NULL, .m_lock_line = 0, .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \
+        }
+#else
+#    define OPAL_MUTEX_STATIC_INIT                            \
+        {                                                     \
+            .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t),      \
+            .m_lock = OPAL_THREAD_INTERNAL_MUTEX_INITIALIZER, \
+            .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT,           \
+        }
+#endif
+
+#if defined(OPAL_THREAD_INTERNAL_RECURSIVE_MUTEX_INITIALIZER)
+#    if OPAL_ENABLE_DEBUG
+#        define OPAL_RECURSIVE_MUTEX_STATIC_INIT                                               \
+            {                                                                                  \
+                .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t),                                   \
+                .m_lock = OPAL_THREAD_INTERNAL_RECURSIVE_MUTEX_INITIALIZER, .m_lock_debug = 0, \
+                .m_lock_file = NULL, .m_lock_line = 0, .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \
+            }
+#    else
+#        define OPAL_RECURSIVE_MUTEX_STATIC_INIT                            \
+            {                                                               \
+                .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t),                \
+                .m_lock = OPAL_THREAD_INTERNAL_RECURSIVE_MUTEX_INITIALIZER, \
+                .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT,                     \
+            }
+#    endif
+#endif /* OPAL_THREAD_INTERNAL_RECURSIVE_MUTEX_INITIALIZER */
 
 /**
  * Try to acquire a mutex.
@@ -58,24 +107,30 @@ OBJ_CLASS_DECLARATION(opal_recursive_mutex_t);
  * @param mutex         Address of the mutex.
  * @return              0 if the mutex was acquired, 1 otherwise.
  */
-static inline int opal_mutex_trylock(opal_mutex_t *mutex);
-
+static inline int opal_mutex_trylock(opal_mutex_t *mutex)
+{
+    return opal_thread_internal_mutex_trylock(&mutex->m_lock);
+}
 
 /**
  * Acquire a mutex.
  *
  * @param mutex         Address of the mutex.
  */
-static inline void opal_mutex_lock(opal_mutex_t *mutex);
-
+static inline void opal_mutex_lock(opal_mutex_t *mutex)
+{
+    opal_thread_internal_mutex_lock(&mutex->m_lock);
+}
 
 /**
  * Release a mutex.
  *
  * @param mutex         Address of the mutex.
  */
-static inline void opal_mutex_unlock(opal_mutex_t *mutex);
-
+static inline void opal_mutex_unlock(opal_mutex_t *mutex)
+{
+    opal_thread_internal_mutex_unlock(&mutex->m_lock);
+}
 
 /**
  * Try to acquire a mutex using atomic operations.
@@ -83,24 +138,30 @@ static inline void opal_mutex_unlock(opal_mutex_t *mutex);
  * @param mutex         Address of the mutex.
  * @return              0 if the mutex was acquired, 1 otherwise.
  */
-static inline int opal_mutex_atomic_trylock(opal_mutex_t *mutex);
-
+static inline int opal_mutex_atomic_trylock(opal_mutex_t *mutex)
+{
+    return opal_atomic_trylock(&mutex->m_lock_atomic);
+}
 
 /**
  * Acquire a mutex using atomic operations.
  *
  * @param mutex         Address of the mutex.
  */
-static inline void opal_mutex_atomic_lock(opal_mutex_t *mutex);
-
+static inline void opal_mutex_atomic_lock(opal_mutex_t *mutex)
+{
+    opal_atomic_lock(&mutex->m_lock_atomic);
+}
 
 /**
  * Release a mutex using atomic operations.
  *
  * @param mutex         Address of the mutex.
  */
-static inline void opal_mutex_atomic_unlock(opal_mutex_t *mutex);
-
+static inline void opal_mutex_atomic_unlock(opal_mutex_t *mutex)
+{
+    opal_atomic_unlock(&mutex->m_lock_atomic);
+}
 
 /**
  * Lock a mutex if opal_using_threads() says that multiple threads may
@@ -115,14 +176,13 @@ static inline void opal_mutex_atomic_unlock(opal_mutex_t *mutex);
  * If there is no possibility that multiple threads are running in the
  * process, return immediately.
  */
-#define OPAL_THREAD_LOCK(mutex)                     \
-    do {                                            \
-        if (OPAL_UNLIKELY(opal_using_threads())) {  \
-            opal_mutex_lock(mutex);                 \
-        }                                           \
+#define OPAL_THREAD_LOCK(mutex)                    \
+    do {                                           \
+        if (OPAL_UNLIKELY(opal_using_threads())) { \
+            opal_mutex_lock(mutex);                \
+        }                                          \
     } while (0)
 
-
 /**
  * Try to lock a mutex if opal_using_threads() says that multiple
  * threads may be active in the process.
@@ -161,7 +221,6 @@ static inline void opal_mutex_atomic_unlock(opal_mutex_t *mutex);
         }                                          \
     } while (0)
 
-
 /**
  * Lock a mutex if opal_using_threads() says that multiple threads may
  * be active in the process for the duration of the specified action.
@@ -188,6 +247,14 @@ static inline void opal_mutex_atomic_unlock(opal_mutex_t *mutex);
         }                                          \
     } while (0)
 
+typedef opal_thread_internal_cond_t opal_cond_t;
+#define OPAL_CONDITION_STATIC_INIT OPAL_THREAD_INTERNAL_COND_INITIALIZER
+int opal_cond_init(opal_cond_t *cond);
+int opal_cond_wait(opal_cond_t *cond, opal_mutex_t *lock);
+int opal_cond_broadcast(opal_cond_t *cond);
+int opal_cond_signal(opal_cond_t *cond);
+int opal_cond_destroy(opal_cond_t *cond);
+
 END_C_DECLS
 
 #endif /* OPAL_MCA_THREADS_MUTEX_H */
diff --git a/opal/mca/threads/pthreads/Makefile.am b/opal/mca/threads/pthreads/Makefile.am
index 0f5aa1d45de..2930a292cce 100644
--- a/opal/mca/threads/pthreads/Makefile.am
+++ b/opal/mca/threads/pthreads/Makefile.am
@@ -11,6 +11,8 @@
 #                         All rights reserved.
 # Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2019      Sandia National Laboratories.  All rights reserved.
+# Copyright (c) 2021      Argonne National Laboratory.  All rights reserved.
+#
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -24,11 +26,8 @@ libmca_threads_pthreads_la_SOURCES = \
     threads_pthreads_component.c \
     threads_pthreads_condition.c \
     threads_pthreads_module.c \
-    threads_pthreads_mutex.c \
     threads_pthreads_mutex.h \
     threads_pthreads_threads.h \
     threads_pthreads_tsd.h \
-    threads_pthreads_wait_sync.c \
-    threads_pthreads_wait_sync.h \
     threads_pthreads_yield.c \
     threads_pthreads.h
diff --git a/opal/mca/threads/pthreads/configure.m4 b/opal/mca/threads/pthreads/configure.m4
index 35b8766c8e7..126653dc008 100644
--- a/opal/mca/threads/pthreads/configure.m4
+++ b/opal/mca/threads/pthreads/configure.m4
@@ -16,6 +16,7 @@
 # Copyright (c) 2019      Sandia National Laboratories.  All rights reserved.
 # Copyright (c) 2019      Triad National Security, LLC. All rights
 #                         reserved.
+# Copyright (c) 2025      Nanook Consulting  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -30,101 +31,6 @@
 # ********************************************************************
 
 
-AC_DEFUN([OPAL_CHECK_PTHREAD_PIDS],[
-#
-# Arguments: none
-#
-# Dependencies: None
-#
-# Sets:
-#  OPAL_THREADS_HAVE_DIFFERENT_PIDS (variable)
-#
-# Test for Linux-like threads in the system.  OPAL no longer supports
-# systems with different PIDs for threads in the same process, so error
-# out if we detect that case.
-#
-
-AC_MSG_CHECKING([if threads have different pids (pthreads on linux)])
-
-OPAL_VAR_SCOPE_PUSH([CFLAGS_save CPPFLAGS_save LDFLAGS_save LIBS_save MSG])
-CFLAGS_save="$CFLAGS"
-CFLAGS="$CFLAGS $THREAD_CFLAGS"
-CPPFLAGS_save="$CPPFLAGS"
-CPPFLAGS="$CPPFLAGS $THREAD_CPPFLAGS"
-LDFLAGS_save="$LDFLAGS"
-LDFLAGS="$THREAD_LDFLAGS"
-LIBS_save="$LIBS"
-LIBS="$THREAD_LIBS"
-AC_RUN_IFELSE([AC_LANG_SOURCE([#include <pthread.h>
-#include <sys/types.h>
-#include <unistd.h>
-#include <stdlib.h>
-
-void *checkpid(void *arg);
-int main() {
-  pthread_t thr;
-  int pid, *retval;
-  pid = getpid();
-  pthread_create(&thr, NULL, checkpid, &pid);
-  pthread_join(thr, (void **) &retval);
-  exit(*retval);
-}
-
-static int ret;
-void *checkpid(void *arg) {
-   int ppid = *((int *) arg);
-   if (ppid == getpid())
-     ret = 0;
-   else
-     ret = 1;
-   pthread_exit((void *) &ret);
-}])],
-[MSG=no OPAL_THREADS_HAVE_DIFFERENT_PIDS=0],
-[MSG=yes OPAL_THREADS_HAVE_DIFFERENT_PIDS=1],
-[
- # If we're cross compiling, we can't do another AC_* function here because
- # it we haven't displayed the result from the last one yet.  So defer
- # another test until below.
- OPAL_THREADS_HAVE_DIFFERENT_PIDS=
- MSG="cross compiling (need another test)"])
-
-CFLAGS="$CFLAGS_save"
-CPPFLAGS="$CPPFLAGS_save"
-LDFLAGS="$LDFLAGS_save"
-LIBS="$LIBS_save"
-
-AC_MSG_RESULT([$MSG])
-
-AS_IF([test "x$OPAL_THREADS_HAVE_DIFFERENT_PIDS" = "x"],
-      [ # If we are cross-compiling, look for the symbol
-       # __linuxthreads_create_event, which seems to only exist in the
-       # Linux Threads-based pthreads implementation (i.e., the one
-       # that has different PIDs for each thread).  We *could* switch
-       # on $host here and only test *linux* hosts, but this test is
-       # pretty unique, so why bother?  Note that AC_CHECK_FUNC works
-       # properly in cross-compiling environments in recent-enough
-       # versions of Autoconf (which is one of the reasons we mandate
-       # recent versions in autogen!).
-       AC_CHECK_FUNC([__linuxthreads_create_event],
-                     [OPAL_THREADS_HAVE_DIFFERENT_PIDS=1])])
-
-AS_IF([test "$OPAL_THREADS_HAVE_DIFFERENT_PIDS" = "1"],
-      [AC_MSG_WARN([This version of Open MPI only supports environments where])
-       AC_MSG_WARN([threads have the same PID.  Please use an older version of])
-       AC_MSG_WARN([Open MPI if you need support on systems with different])
-       AC_MSG_WARN([PIDs for threads in the same process.  Open MPI 1.4.x])
-       AC_MSG_WARN([supports such systems, as does at least some versions the])
-       AC_MSG_WARN([Open MPI 1.5.x series.])
-       AC_MSG_ERROR([Cannot continue])
-      ])
-
-#
-# if pthreads is not available, then the system does not have an insane threads
-# model
-#
-OPAL_VAR_SCOPE_POP
-])dnl
-
 AC_DEFUN([OPAL_INTL_PTHREAD_TRY_LINK], [
 # BEGIN: OPAL_INTL_PTHREAD_TRY_LINK
 #
@@ -348,18 +254,17 @@ AC_DEFUN([OPAL_INTL_POSIX_THREADS_PLAIN], [
 # why take chances?
 #
 
-# Only run C++ and Fortran if those compilers already configured
-AC_PROVIDE_IFELSE([AC_PROG_CC],
-                  [OPAL_INTL_POSIX_THREADS_PLAIN_C],
-                  [opal_pthread_c_success=1])
+# Only run C++ and Fortran if those compilers were found
+
+OPAL_INTL_POSIX_THREADS_PLAIN_C
 
-AC_PROVIDE_IFELSE([AC_PROG_CXX],
-                  [OPAL_INTL_POSIX_THREADS_PLAIN_CXX],
-                  [opal_pthread_cxx_success=1])
+AS_IF([test "$CXX" != "no"],
+      [OPAL_INTL_POSIX_THREADS_PLAIN_CXX],
+      [opal_pthread_cxx_success=0])
 
-AC_PROVIDE_IFELSE([AC_PROG_FC],
-                  [OPAL_INTL_POSIX_THREADS_PLAIN_FC],
-                  [opal_pthread_fortran_success=1])
+AS_IF([test "$FC" != "no"],
+      [OPAL_INTL_POSIX_THREADS_PLAIN_FC],
+      [opal_pthread_fortran_success=0])
 
 # End: OPAL_INTL_POSIX_THREADS_PLAIN
 ])dnl
@@ -459,36 +364,27 @@ AC_DEFUN([OPAL_INTL_POSIX_THREADS_SPECIAL_FLAGS],[
 # -Kthread:
 # -kthread:  FreeBSD kernel threads
 # -pthread:  Modern GCC (most all platforms)
-# -pthreads: GCC on solaris
 # -mthreads:
-# -mt:       Solaris native compilers / HP-UX aCC
+# -mt:       HP-UX aCC
 #
 # Put -mt before -mthreads because HP-UX aCC will properly compile
 # with -mthreads (reading as -mt), but emit a warning about unknown
-# flags hreads.  Stupid compilers.
+# flags "hreads".  Stupid compilers.
 OPAL_VAR_SCOPE_PUSH([pflags])
 
-case "${host_cpu}-${host_os}" in
-  *solaris*)
-    pflags="-pthread -pthreads -mt"
-  ;;
-  *)
-    pflags="-Kthread -kthread -pthread -pthreads -mt -mthreads"
-  ;;
-esac
+pflags="-Kthread -kthread -pthread -pthreads -mt -mthreads"
 
-# Only run C++ and Fortran if those compilers already configured
-AC_PROVIDE_IFELSE([AC_PROG_CC],
-                  [OPAL_INTL_POSIX_THREADS_SPECIAL_FLAGS_C],
-                  [opal_pthread_c_success=1])
+# Only run C++ and Fortran if those compilers were found
 
-AC_PROVIDE_IFELSE([AC_PROG_CXX],
-                  [OPAL_INTL_POSIX_THREADS_SPECIAL_FLAGS_CXX],
-                  [opal_pthread_cxx_success=1])
+OPAL_INTL_POSIX_THREADS_SPECIAL_FLAGS_C
 
-AC_PROVIDE_IFELSE([AC_PROG_FC],
-                  [OPAL_INTL_POSIX_THREADS_SPECIAL_FLAGS_FC],
-                  [opal_pthread_fortran_success=1])
+AS_IF([test "$CXX" != "no"],
+      [OPAL_INTL_POSIX_THREADS_SPECIAL_FLAGS_CXX],
+      [opal_pthread_cxx_success=0])
+
+AS_IF([test "$FC" != "no"],
+      [OPAL_INTL_POSIX_THREADS_SPECIAL_FLAGS_FC],
+      [opal_pthread_fortran_success=0])
 
 OPAL_VAR_SCOPE_POP
 # End: OPAL_INTL_POSIX_THREADS_SPECIAL_FLAGS
@@ -668,18 +564,17 @@ AC_DEFUN([OPAL_INTL_POSIX_THREADS_LIBS],[
 OPAL_VAR_SCOPE_PUSH([plibs])
 plibs="-lpthreads -llthread -lpthread"
 
-# Only run C++ and Fortran if those compilers already configured
-AC_PROVIDE_IFELSE([AC_PROG_CC],
-                  [OPAL_INTL_POSIX_THREADS_LIBS_C],
-                  [opal_pthread_c_success=1])
+# Only run C++ and Fortran if those compilers were found
+
+OPAL_INTL_POSIX_THREADS_LIBS_C
 
-AC_PROVIDE_IFELSE([AC_PROG_CXX],
-                  [OPAL_INTL_POSIX_THREADS_LIBS_CXX],
-                  [opal_pthread_cxx_success=1])
+AS_IF([test "$CXX" != "no"],
+      [OPAL_INTL_POSIX_THREADS_LIBS_CXX],
+      [opal_pthread_cxx_success=0])
 
-AC_PROVIDE_IFELSE([AC_PROG_FC],
-                  [OPAL_INTL_POSIX_THREADS_LIBS_FC],
-                  [opal_pthread_fortran_success=1])
+AS_IF([test "$FC" != "no"],
+      [OPAL_INTL_POSIX_THREADS_LIBS_FC],
+      [opal_pthread_fortran_success=0])
 
 OPAL_VAR_SCOPE_POP
 # End: OPAL_INTL_POSIX_THREADS_LIBS]
@@ -771,7 +666,7 @@ THREAD_CXXCPPFLAGS="$TPKG_CXXCPPFLAGS"
 THREAD_LDFLAGS="$TPKG_LDFLAGS"
 THREAD_LIBS="$TPKG_LIBS"
 
-OPAL_CHECK_PTHREAD_PIDS
+AS_IF([test "$CXX" = "no"], [opal_pthread_cxx_success=1])
 
 if test "$OMPI_TRY_FORTRAN_BINDINGS" = "$OMPI_FORTRAN_NO_BINDINGS" || \
    test $ompi_fortran_happy -ne 1; then
@@ -803,7 +698,7 @@ AC_DEFUN([MCA_opal_threads_pthreads_COMPILE_MODE], [
 
 # If component was selected, $1 will be 1 and we should set the base header
 AC_DEFUN([MCA_opal_threads_pthreads_POST_CONFIG],[
-    AS_IF([test "$1" = "1"], 
+    AS_IF([test "$1" = "1"],
           [opal_thread_type_found="pthreads"
            AC_DEFINE_UNQUOTED([MCA_threads_base_include_HEADER],
                               ["opal/mca/threads/pthreads/threads_pthreads_threads.h"],
@@ -814,9 +709,6 @@ AC_DEFUN([MCA_opal_threads_pthreads_POST_CONFIG],[
            AC_DEFINE_UNQUOTED([MCA_threads_tsd_base_include_HEADER],
                               ["opal/mca/threads/pthreads/threads_pthreads_tsd.h"],
                               [Header to include for tsd implementation])
-           AC_DEFINE_UNQUOTED([MCA_threads_wait_sync_base_include_HEADER],
-                              ["opal/mca/threads/pthreads/threads_pthreads_wait_sync.h"],
-                              [Header to include for wait_sync implementation])
            THREAD_CFLAGS="$TPKG_CFLAGS"
            THREAD_FCFLAGS="$TPKG_FCFLAGS"
            THREAD_CXXFLAGS="$TPKG_CXXFLAGS"
diff --git a/opal/mca/threads/pthreads/threads_pthreads.h b/opal/mca/threads/pthreads/threads_pthreads.h
index 0801e56a09a..3d786516fc7 100644
--- a/opal/mca/threads/pthreads/threads_pthreads.h
+++ b/opal/mca/threads/pthreads/threads_pthreads.h
@@ -10,7 +10,6 @@
  * $HEADER$
  */
 
-
 #ifndef OPAL_MCA_THREADS_PTHREADS_THREADS_PTHREADS_H
 #define OPAL_MCA_THREADS_PTHREADS_THREADS_PTHREADS_H
 
@@ -18,7 +17,7 @@
 #include <stdint.h>
 #include <time.h>
 
-typedef void (opal_threads_pthreads_yield_fn_t)(void);
+typedef void(opal_threads_pthreads_yield_fn_t)(void);
 
 OPAL_DECLSPEC int opal_threads_pthreads_yield_init(const mca_base_component_t *component);
 
diff --git a/opal/mca/threads/pthreads/threads_pthreads_component.c b/opal/mca/threads/pthreads/threads_pthreads_component.c
index 30c6ca70336..d2d51adc23a 100644
--- a/opal/mca/threads/pthreads/threads_pthreads_component.c
+++ b/opal/mca/threads/pthreads/threads_pthreads_component.c
@@ -23,11 +23,10 @@
 
 #include "opal_config.h"
 
-#include "opal/mca/threads/thread.h"
-#include "opal/mca/threads/threads.h"
 #include "opal/constants.h"
 #include "opal/mca/threads/pthreads/threads_pthreads.h"
-
+#include "opal/mca/threads/thread.h"
+#include "opal/mca/threads/threads.h"
 
 static int opal_threads_pthreads_open(void);
 static int opal_threads_pthreads_register(void);
@@ -35,21 +34,18 @@ static int opal_threads_pthreads_register(void);
 const opal_threads_base_component_1_0_0_t mca_threads_pthreads_component = {
     /* First, the mca_component_t struct containing meta information
      * about the component itself */
-    .threadsc_version = {
-        OPAL_THREADS_BASE_VERSION_1_0_0,
-
-        /* Component name and version */
-        .mca_component_name = "pthreads",
-        MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
-                              OPAL_RELEASE_VERSION),
-
-        .mca_open_component = opal_threads_pthreads_open,
-        .mca_register_component_params = opal_threads_pthreads_register
-    },
-    .threadsc_data = {
-        /* The component is checkpoint ready */
-        MCA_BASE_METADATA_PARAM_CHECKPOINT
-    },
+    .threadsc_version = {OPAL_THREADS_BASE_VERSION_1_0_0,
+
+                         /* Component name and version */
+                         .mca_component_name = "pthreads",
+                         MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
+                                               OPAL_RELEASE_VERSION),
+
+                         .mca_open_component = opal_threads_pthreads_open,
+                         .mca_register_component_params = opal_threads_pthreads_register},
+    .threadsc_data =
+        {/* The component is checkpoint ready */
+         MCA_BASE_METADATA_PARAM_CHECKPOINT},
 };
 
 int opal_threads_pthreads_register(void)
diff --git a/opal/mca/threads/pthreads/threads_pthreads_condition.c b/opal/mca/threads/pthreads/threads_pthreads_condition.c
index f685a24cef5..ca48c9bdf1b 100644
--- a/opal/mca/threads/pthreads/threads_pthreads_condition.c
+++ b/opal/mca/threads/pthreads/threads_pthreads_condition.c
@@ -23,7 +23,6 @@
 
 #include "opal/mca/threads/condition.h"
 
-
 static void opal_condition_construct(opal_condition_t *c)
 {
     c->c_waiting = 0;
@@ -34,7 +33,5 @@ static void opal_condition_destruct(opal_condition_t *c)
 {
 }
 
-OBJ_CLASS_INSTANCE(opal_condition_t,
-                   opal_object_t,
-                   opal_condition_construct,
+OBJ_CLASS_INSTANCE(opal_condition_t, opal_object_t, opal_condition_construct,
                    opal_condition_destruct);
diff --git a/opal/mca/threads/pthreads/threads_pthreads_module.c b/opal/mca/threads/pthreads/threads_pthreads_module.c
index 2f1e918a4c6..bcbd1fded13 100644
--- a/opal/mca/threads/pthreads/threads_pthreads_module.c
+++ b/opal/mca/threads/pthreads/threads_pthreads_module.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2006 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -23,61 +23,18 @@
  * $HEADER$
  */
 
+/* needed for pthread_mutexattr_settype */
+#define _GNU_SOURCE
+
 #include <unistd.h>
+#include <pthread.h>
 
 #include "opal/constants.h"
-#include "opal/util/sys_limits.h"
-#include "opal/util/output.h"
-#include "opal/prefetch.h"
 #include "opal/mca/threads/threads.h"
 #include "opal/mca/threads/tsd.h"
-
-/*
- * Constructor
- */
-static void opal_thread_construct(opal_thread_t *t)
-{
-    t->t_run = 0;
-    t->t_handle = (pthread_t)-1;
-}
-
-OBJ_CLASS_INSTANCE(opal_thread_t,
-                   opal_object_t,
-                   opal_thread_construct, NULL);
-
-int opal_thread_start(opal_thread_t *t)
-{
-    int rc;
-
-    if (OPAL_ENABLE_DEBUG) {
-        if (NULL == t->t_run || (pthread_t)-1 != t->t_handle) {
-            return OPAL_ERR_BAD_PARAM;
-        }
-    }
-
-    rc = pthread_create(&t->t_handle, NULL, (void *(*)(void *))t->t_run, t);
-
-    return 0 == rc ? OPAL_SUCCESS : OPAL_ERR_IN_ERRNO;
-}
-
-int opal_thread_join(opal_thread_t *t, void **thr_return)
-{
-    int rc = pthread_join(t->t_handle, thr_return);
-    t->t_handle = (pthread_t)-1;
-    return 0 == rc ? OPAL_SUCCESS : OPAL_ERR_IN_ERRNO;
-}
-
-bool opal_thread_self_compare(opal_thread_t *t)
-{
-    return pthread_self() == t->t_handle;
-}
-
-opal_thread_t *opal_thread_get_self(void)
-{
-    opal_thread_t *t = OBJ_NEW(opal_thread_t);
-    t->t_handle = pthread_self();
-    return t;
-}
+#include "opal/prefetch.h"
+#include "opal/util/output.h"
+#include "opal/util/sys_limits.h"
 
 int opal_tsd_key_create(opal_tsd_key_t *key, opal_tsd_destructor_t destructor)
 {
@@ -86,6 +43,38 @@ int opal_tsd_key_create(opal_tsd_key_t *key, opal_tsd_destructor_t destructor)
     return 0 == rc ? OPAL_SUCCESS : OPAL_ERR_IN_ERRNO;
 }
 
-void opal_thread_set_main(void)
+
+int opal_thread_internal_mutex_init_recursive(opal_thread_internal_mutex_t *p_mutex)
 {
+    int ret;
+#if OPAL_ENABLE_DEBUG
+    pthread_mutexattr_t mutex_attr;
+    ret = pthread_mutexattr_init(&mutex_attr);
+    if (0 != ret)
+        return OPAL_ERR_IN_ERRNO;
+    ret = pthread_mutexattr_settype(&mutex_attr, PTHREAD_MUTEX_RECURSIVE);
+    if (0 != ret) {
+        ret = pthread_mutexattr_destroy(&mutex_attr);
+        assert(0 == ret);
+        return OPAL_ERR_IN_ERRNO;
+    }
+    ret = pthread_mutex_init(p_mutex, &mutex_attr);
+    if (0 != ret) {
+        ret = pthread_mutexattr_destroy(&mutex_attr);
+        assert(0 == ret);
+        return OPAL_ERR_IN_ERRNO;
+    }
+    ret = pthread_mutexattr_destroy(&mutex_attr);
+    assert(0 == ret);
+#else
+    pthread_mutexattr_t mutex_attr;
+    ret = pthread_mutexattr_init(&mutex_attr);
+    if (0 != ret) {
+        return OPAL_ERR_IN_ERRNO;
+    }
+    pthread_mutexattr_settype(&mutex_attr, PTHREAD_MUTEX_RECURSIVE);
+    ret = pthread_mutex_init(p_mutex, &mutex_attr);
+    pthread_mutexattr_destroy(&mutex_attr);
+#endif
+    return 0 == ret ? OPAL_SUCCESS : OPAL_ERR_IN_ERRNO;
 }
diff --git a/opal/mca/threads/pthreads/threads_pthreads_mutex.c b/opal/mca/threads/pthreads/threads_pthreads_mutex.c
deleted file mode 100644
index 8aa4671f46f..00000000000
--- a/opal/mca/threads/pthreads/threads_pthreads_mutex.c
+++ /dev/null
@@ -1,133 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007-2016 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2019      Sandia National Laboratories.  All rights reserved.
- *
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "opal_config.h"
-
-#include <errno.h>
-#include <pthread.h>
-
-#include "opal/mca/threads/mutex.h"
-#include "opal/mca/threads/pthreads/threads_pthreads_mutex.h"
-#include "opal/constants.h"
-
-/*
- * Wait and see if some upper layer wants to use threads, if support
- * exists.
- */
-bool opal_uses_threads = false;
-
-struct opal_pthread_mutex_t {
-    opal_object_t super;
-
-    pthread_mutex_t m_lock_pthread;
-    opal_atomic_lock_t m_lock_atomic;
-
-#if OPAL_ENABLE_DEBUG
-    int m_lock_debug;
-    const char *m_lock_file;
-    int m_lock_line;
-#endif
-};
-
-typedef struct opal_pthread_mutex_t opal_pthread_mutex_t;
-typedef struct opal_pthread_mutex_t opal_pthread_recursive_mutex_t;
-
-static void mca_threads_pthreads_mutex_constructor(opal_mutex_t *p_mutex)
-{
-    pthread_mutex_init(&p_mutex->m_lock_pthread, NULL);
-#if OPAL_ENABLE_DEBUG
-    p_mutex->m_lock_debug = 0;
-    p_mutex->m_lock_file = NULL;
-    p_mutex->m_lock_line = 0;
-#endif
-    opal_atomic_lock_init(&p_mutex->m_lock_atomic, 0);
-}
-
-static void mca_threads_pthreads_mutex_destructor(opal_mutex_t *p_mutex)
-{
-    pthread_mutex_destroy(&p_mutex->m_lock_pthread);
-}
-
-static void mca_threads_pthreads_recursive_mutex_constructor
-        (opal_recursive_mutex_t *p_mutex)
-{
-    pthread_mutexattr_t mutex_attr;
-    pthread_mutexattr_init(&mutex_attr);
-    pthread_mutexattr_settype(&mutex_attr, PTHREAD_MUTEX_RECURSIVE);
-    pthread_mutex_init(&p_mutex->m_lock_pthread, &mutex_attr);
-    pthread_mutexattr_destroy(&mutex_attr);
-#if OPAL_ENABLE_DEBUG
-    p_mutex->m_lock_debug = 0;
-    p_mutex->m_lock_file = NULL;
-    p_mutex->m_lock_line = 0;
-#endif
-    opal_atomic_lock_init(&p_mutex->m_lock_atomic, 0);
-}
-
-static void mca_threads_pthreads_recursive_mutex_destructor
-        (opal_recursive_mutex_t *p_mutex)
-{
-    pthread_mutex_destroy(&p_mutex->m_lock_pthread);
-}
-
-OBJ_CLASS_INSTANCE(opal_mutex_t,
-                   opal_object_t,
-                   mca_threads_pthreads_mutex_constructor,
-                   mca_threads_pthreads_mutex_destructor);
-
-OBJ_CLASS_INSTANCE(opal_recursive_mutex_t,
-                   opal_object_t,
-                   mca_threads_pthreads_recursive_mutex_constructor,
-                   mca_threads_pthreads_recursive_mutex_destructor);
-
-int opal_cond_init(opal_cond_t *cond)
-{
-    int ret = pthread_cond_init(cond, NULL);
-    return 0 == ret ? OPAL_SUCCESS : OPAL_ERR_IN_ERRNO;
-}
-
-int opal_cond_wait(opal_cond_t *cond, opal_mutex_t *lock)
-{
-    int ret = pthread_cond_wait(cond, &lock->m_lock_pthread);
-    return 0 == ret ? OPAL_SUCCESS : OPAL_ERR_IN_ERRNO;
-}
-
-int opal_cond_broadcast(opal_cond_t *cond)
-{
-    int ret = pthread_cond_broadcast(cond);
-    return 0 == ret ? OPAL_SUCCESS : OPAL_ERR_IN_ERRNO;
-}
-
-int opal_cond_signal(opal_cond_t *cond)
-{
-    int ret = pthread_cond_signal(cond);
-    return 0 == ret ? OPAL_SUCCESS : OPAL_ERR_IN_ERRNO;
-}
-
-int opal_cond_destroy(opal_cond_t *cond)
-{
-    int ret = pthread_cond_destroy(cond);
-    return 0 == ret ? OPAL_SUCCESS : OPAL_ERR_IN_ERRNO;
-}
diff --git a/opal/mca/threads/pthreads/threads_pthreads_mutex.h b/opal/mca/threads/pthreads/threads_pthreads_mutex.h
index 1547426d209..ee9dda81f18 100644
--- a/opal/mca/threads/pthreads/threads_pthreads_mutex.h
+++ b/opal/mca/threads/pthreads/threads_pthreads_mutex.h
@@ -17,6 +17,7 @@
  * Copyright (c) 2019      Sandia National Laboratories.  All rights reserved.
  * Copyright (c) 2020      Triad National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2021      Argonne National Laboratory.  All rights reserved.
  *
  * $COPYRIGHT$
  *
@@ -41,184 +42,129 @@
 
 #include "opal_config.h"
 
-#include <pthread.h>
 #include <errno.h>
+#include <pthread.h>
 #include <stdio.h>
 
 #include "opal/class/opal_object.h"
-#include "opal/sys/atomic.h"
-#include "opal/util/output.h"
+#include "opal/constants.h"
+#include "opal/util/show_help.h"
 
 BEGIN_C_DECLS
 
-struct opal_mutex_t {
-    opal_object_t super;
-
-    pthread_mutex_t m_lock_pthread;
-
-#if OPAL_ENABLE_DEBUG
-    int m_lock_debug;
-    const char *m_lock_file;
-    int m_lock_line;
-#endif
-
-    opal_atomic_lock_t m_lock_atomic;
-};
-OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_mutex_t);
-OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_recursive_mutex_t);
+typedef pthread_mutex_t opal_thread_internal_mutex_t;
 
+#define OPAL_THREAD_INTERNAL_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER
 #if defined(PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP)
-#define OPAL_PTHREAD_RECURSIVE_MUTEX_INITIALIZER \
-            PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP
+#    define OPAL_THREAD_INTERNAL_RECURSIVE_MUTEX_INITIALIZER PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP
 #elif defined(PTHREAD_RECURSIVE_MUTEX_INITIALIZER)
-#define OPAL_PTHREAD_RECURSIVE_MUTEX_INITIALIZER \
-            PTHREAD_RECURSIVE_MUTEX_INITIALIZER
+#    define OPAL_THREAD_INTERNAL_RECURSIVE_MUTEX_INITIALIZER PTHREAD_RECURSIVE_MUTEX_INITIALIZER
 #endif
 
-#if OPAL_ENABLE_DEBUG
-#define OPAL_MUTEX_STATIC_INIT                                          \
-    {                                                                   \
-        .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t),                    \
-        .m_lock_pthread = PTHREAD_MUTEX_INITIALIZER,                    \
-        .m_lock_debug = 0,                                              \
-        .m_lock_file = NULL,                                            \
-        .m_lock_line = 0,                                               \
-        .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT,                         \
-    }
-#else
-#define OPAL_MUTEX_STATIC_INIT                                          \
-    {                                                                   \
-        .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t),                    \
-        .m_lock_pthread = PTHREAD_MUTEX_INITIALIZER,                    \
-        .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT,                         \
-    }
-#endif
 
-#if defined(OPAL_PTHREAD_RECURSIVE_MUTEX_INITIALIZER)
+int opal_thread_internal_mutex_init_recursive(opal_thread_internal_mutex_t *p_mutex);
+
+static inline int opal_thread_internal_mutex_init(opal_thread_internal_mutex_t *p_mutex,
+                                                  bool recursive)
+{
+    int ret;
+    if (recursive) {
+        return opal_thread_internal_mutex_init_recursive(p_mutex);
+    } else {
+        ret = pthread_mutex_init(p_mutex, NULL);
+        return 0 == ret ? OPAL_SUCCESS : OPAL_ERR_IN_ERRNO;
+    }
+}
 
+static inline void opal_thread_internal_mutex_lock(opal_thread_internal_mutex_t *p_mutex)
+{
 #if OPAL_ENABLE_DEBUG
-#define OPAL_RECURSIVE_MUTEX_STATIC_INIT                                \
-    {                                                                   \
-        .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t),                    \
-        .m_lock_pthread = OPAL_PTHREAD_RECURSIVE_MUTEX_INITIALIZER,     \
-        .m_lock_debug = 0,                                              \
-        .m_lock_file = NULL,                                            \
-        .m_lock_line = 0,                                               \
-        .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT,                         \
+    int ret = pthread_mutex_lock(p_mutex);
+    if (EDEADLK == ret) {
+        opal_show_help("help-opal-threads.txt", "mutex lock failed", true);
     }
+    assert(0 == ret);
 #else
-#define OPAL_RECURSIVE_MUTEX_STATIC_INIT                                \
-    {                                                                   \
-        .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t),                    \
-        .m_lock_pthread = OPAL_PTHREAD_RECURSIVE_MUTEX_INITIALIZER,     \
-        .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT,                         \
-    }
-#endif
-
+    pthread_mutex_lock(p_mutex);
 #endif
+}
 
-/************************************************************************
- *
- * mutex operations (non-atomic versions)
- *
- ************************************************************************/
-
-static inline int opal_mutex_trylock(opal_mutex_t *m)
+static inline int opal_thread_internal_mutex_trylock(opal_thread_internal_mutex_t *p_mutex)
 {
-    int ret = pthread_mutex_trylock(&m->m_lock_pthread);
-    if (EDEADLK == ret) {
-#if OPAL_ENABLE_DEBUG
-        opal_output(0, "opal_mutex_trylock() %d",ret);
-#endif
-        return 1;
-    }
+    int ret = pthread_mutex_trylock(p_mutex);
     return 0 == ret ? 0 : 1;
 }
 
-static inline void opal_mutex_lock(opal_mutex_t *m)
+static inline void opal_thread_internal_mutex_unlock(opal_thread_internal_mutex_t *p_mutex)
 {
 #if OPAL_ENABLE_DEBUG
-    int ret = pthread_mutex_lock(&m->m_lock_pthread);
-    if (EDEADLK == ret) {
-        errno = ret;
-        opal_output(0, "opal_mutex_lock() %d", ret);
-    }
+    int ret = pthread_mutex_unlock(p_mutex);
+    assert(0 == ret);
 #else
-    pthread_mutex_lock(&m->m_lock_pthread);
+    pthread_mutex_unlock(p_mutex);
 #endif
 }
 
-static inline void opal_mutex_unlock(opal_mutex_t *m)
+static inline void opal_thread_internal_mutex_destroy(opal_thread_internal_mutex_t *p_mutex)
 {
 #if OPAL_ENABLE_DEBUG
-    int ret = pthread_mutex_unlock(&m->m_lock_pthread);
-    if (EPERM == ret) {
-        errno = ret;
-        opal_output(0, "opal_mutex_unlock() %d", ret);
-    }
+    int ret = pthread_mutex_destroy(p_mutex);
+    assert(0 == ret);
 #else
-    pthread_mutex_unlock(&m->m_lock_pthread);
+    pthread_mutex_destroy(p_mutex);
 #endif
 }
 
-/************************************************************************
- *
- * mutex operations (atomic versions)
- *
- ************************************************************************/
-
-#if OPAL_HAVE_ATOMIC_SPINLOCKS
+typedef pthread_cond_t opal_thread_internal_cond_t;
 
-/************************************************************************
- * Spin Locks
- ************************************************************************/
+#define OPAL_THREAD_INTERNAL_COND_INITIALIZER PTHREAD_COND_INITIALIZER
 
-static inline int opal_mutex_atomic_trylock(opal_mutex_t *m)
+static inline int opal_thread_internal_cond_init(opal_thread_internal_cond_t *p_cond)
 {
-    return opal_atomic_trylock(&m->m_lock_atomic);
+    int ret = pthread_cond_init(p_cond, NULL);
+    return 0 == ret ? OPAL_SUCCESS : OPAL_ERR_IN_ERRNO;
 }
 
-static inline void opal_mutex_atomic_lock(opal_mutex_t *m)
+static inline void opal_thread_internal_cond_wait(opal_thread_internal_cond_t *p_cond,
+                                                  opal_thread_internal_mutex_t *p_mutex)
 {
-    opal_atomic_lock(&m->m_lock_atomic);
+#if OPAL_ENABLE_DEBUG
+    int ret = pthread_cond_wait(p_cond, p_mutex);
+    assert(0 == ret);
+#else
+    pthread_cond_wait(p_cond, p_mutex);
+#endif
 }
 
-static inline void opal_mutex_atomic_unlock(opal_mutex_t *m)
+static inline void opal_thread_internal_cond_broadcast(opal_thread_internal_cond_t *p_cond)
 {
-    opal_atomic_unlock(&m->m_lock_atomic);
-}
-
+#if OPAL_ENABLE_DEBUG
+    int ret = pthread_cond_broadcast(p_cond);
+    assert(0 == ret);
 #else
-
-/************************************************************************
- * Standard locking
- ************************************************************************/
-
-static inline int opal_mutex_atomic_trylock(opal_mutex_t *m)
-{
-    return opal_mutex_trylock(m);
+    pthread_cond_broadcast(p_cond);
+#endif
 }
 
-static inline void opal_mutex_atomic_lock(opal_mutex_t *m)
+static inline void opal_thread_internal_cond_signal(opal_thread_internal_cond_t *p_cond)
 {
-    opal_mutex_lock(m);
+#if OPAL_ENABLE_DEBUG
+    int ret = pthread_cond_signal(p_cond);
+    assert(0 == ret);
+#else
+    pthread_cond_signal(p_cond);
+#endif
 }
 
-static inline void opal_mutex_atomic_unlock(opal_mutex_t *m)
+static inline void opal_thread_internal_cond_destroy(opal_thread_internal_cond_t *p_cond)
 {
-    opal_mutex_unlock(m);
-}
-
+#if OPAL_ENABLE_DEBUG
+    int ret = pthread_cond_destroy(p_cond);
+    assert(0 == ret);
+#else
+    pthread_cond_destroy(p_cond);
 #endif
-
-typedef pthread_cond_t opal_cond_t;
-#define OPAL_CONDITION_STATIC_INIT PTHREAD_COND_INITIALIZER
-
-int opal_cond_init(opal_cond_t *cond);
-int opal_cond_wait(opal_cond_t *cond, opal_mutex_t *lock);
-int opal_cond_broadcast(opal_cond_t *cond);
-int opal_cond_signal(opal_cond_t *cond);
-int opal_cond_destroy(opal_cond_t *cond);
+}
 
 END_C_DECLS
 
diff --git a/opal/mca/threads/pthreads/threads_pthreads_threads.h b/opal/mca/threads/pthreads/threads_pthreads_threads.h
index 2d5d062f1d0..4bdb3710715 100644
--- a/opal/mca/threads/pthreads/threads_pthreads_threads.h
+++ b/opal/mca/threads/pthreads/threads_pthreads_threads.h
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2006 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2020 High Performance Computing Center Stuttgart,
@@ -23,28 +23,19 @@
  * $HEADER$
  */
 
-
 #ifndef OPAL_MCA_THREADS_PTHREADS_THREADS_PTHREADS_THREADS_H
 #define OPAL_MCA_THREADS_PTHREADS_THREADS_PTHREADS_THREADS_H
 
 #include <pthread.h>
 #include <signal.h>
 
-#include "opal/mca/threads/threads.h"
 #include "opal/mca/threads/pthreads/threads_pthreads.h"
-
-struct opal_thread_t {
-    opal_object_t super;
-    opal_thread_fn_t t_run;
-    void *t_arg;
-    pthread_t t_handle;
-};
+#include "opal/mca/threads/threads.h"
 
 /* Pthreads do not need to yield when idle */
 #define OPAL_THREAD_YIELD_WHEN_IDLE_DEFAULT false
 
-static inline
-void opal_thread_yield(void)
+static inline void opal_thread_yield(void)
 {
     opal_threads_pthreads_yield_fn();
 }
diff --git a/opal/mca/threads/pthreads/threads_pthreads_tsd.h b/opal/mca/threads/pthreads/threads_pthreads_tsd.h
index 4db2ed4c3bc..f8e8e4e290f 100644
--- a/opal/mca/threads/pthreads/threads_pthreads_tsd.h
+++ b/opal/mca/threads/pthreads/threads_pthreads_tsd.h
@@ -23,7 +23,6 @@
  * $HEADER$
  */
 
-
 #ifndef OPAL_MCA_THREADS_PTHREADS_THREADS_PTHREADS_TSD_H
 #define OPAL_MCA_THREADS_PTHREADS_THREADS_PTHREADS_TSD_H
 
diff --git a/opal/mca/threads/pthreads/threads_pthreads_wait_sync.c b/opal/mca/threads/pthreads/threads_pthreads_wait_sync.c
deleted file mode 100644
index 3cb63127539..00000000000
--- a/opal/mca/threads/pthreads/threads_pthreads_wait_sync.c
+++ /dev/null
@@ -1,141 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2014-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
- * Copyright (c) 2019      Sandia National Laboratories.  All rights reserved.
- *
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "opal/mca/threads/wait_sync.h"
-
-static opal_mutex_t wait_sync_lock = OPAL_MUTEX_STATIC_INIT;
-ompi_wait_sync_t *wait_sync_list = NULL; /* not static for inline "wait_sync_st" */
-
-
-void wait_sync_global_wakeup_st(int status)
-{
-    ompi_wait_sync_t* sync;
-    for( sync = wait_sync_list; sync != NULL; sync = sync->next ) {
-        wait_sync_update(sync, 0, status);
-    }
-}
-
-void wait_sync_global_wakeup_mt(int status)
-{
-    ompi_wait_sync_t* sync;
-    opal_mutex_lock(&wait_sync_lock);
-    for( sync = wait_sync_list; sync != NULL; sync = sync->next ) {
-        /* sync_update is going to  take the sync->lock from within 
-         * the wait_sync_lock. Thread lightly here: Idealy we should 
-         * find a way to not take a lock in a lock as this is deadlock prone,
-         * but as of today we are the only place doing this so it is safe. 
-         */
-        wait_sync_update(sync, 0, status);
-        if( sync->next == wait_sync_list ) break; /* special case for rings */
-    }
-    opal_mutex_unlock(&wait_sync_lock);
-}
-
-
-
-static opal_atomic_int32_t num_thread_in_progress = 0;
-
-#define WAIT_SYNC_PASS_OWNERSHIP(who)           \
-    do {                                        \
-        pthread_mutex_lock(&(who)->lock);       \
-        pthread_cond_signal(&(who)->condition); \
-        pthread_mutex_unlock(&(who)->lock);     \
-    } while (0)
-
-int ompi_sync_wait_mt(ompi_wait_sync_t *sync)
-{
-    /* Don't stop if the waiting synchronization is completed. We avoid the
-     * race condition around the release of the synchronization using the
-     * signaling field.
-     */
-    if (sync->count <= 0) {
-        return (0 == sync->status) ? OPAL_SUCCESS : OPAL_ERROR;
-    }
-
-    /* lock so nobody can signal us during the list updating */
-    pthread_mutex_lock(&sync->lock);
-
-    /* Now that we hold the lock make sure another thread has not already
-     * call cond_signal.
-     */
-    if (sync->count <= 0) {
-        pthread_mutex_unlock(&sync->lock);
-        return (0 == sync->status) ? OPAL_SUCCESS : OPAL_ERROR;
-    }
-
-    /* Insert sync on the list of pending synchronization constructs */
-    OPAL_THREAD_LOCK(&wait_sync_lock);
-    if (NULL == wait_sync_list) {
-        sync->next = sync->prev = sync;
-        wait_sync_list = sync;
-    } else {
-        sync->prev = wait_sync_list->prev;
-        sync->prev->next = sync;
-        sync->next = wait_sync_list;
-        wait_sync_list->prev = sync;
-    }
-    OPAL_THREAD_UNLOCK(&wait_sync_lock);
-
-    /**
-     * If we are not responsible for progressing, go silent until something
-     * worth noticing happen:
-     *  - this thread has been promoted to take care of the progress
-     *  - our sync has been triggered.
-     */
-  check_status:
-    if (sync != wait_sync_list &&
-        num_thread_in_progress >= opal_max_thread_in_progress) {
-        pthread_cond_wait(&sync->condition, &sync->lock);
-
-        /**
-         * At this point either the sync was completed in which case
-         * we should remove it from the wait list, or/and I was
-         * promoted as the progress manager.
-         */
-
-        if (sync->count <= 0) { /* Completed? */
-            pthread_mutex_unlock(&sync->lock);
-            goto i_am_done;
-        }
-        /* either promoted, or spurious wakeup ! */
-        goto check_status;
-    }
-    pthread_mutex_unlock(&sync->lock);
-
-    OPAL_THREAD_ADD_FETCH32(&num_thread_in_progress, 1);
-    while (sync->count > 0) { /* progress till completion */
-        /* don't progress with the sync lock locked or you'll deadlock */
-        opal_progress();
-    }
-    OPAL_THREAD_ADD_FETCH32(&num_thread_in_progress, -1);
-
-  i_am_done:
-    /* My sync is now complete. Trim the list: remove self, wake next */
-    OPAL_THREAD_LOCK(&wait_sync_lock);
-    sync->prev->next = sync->next;
-    sync->next->prev = sync->prev;
-    /* In case I am the progress manager, pass the duties on */
-    if (sync == wait_sync_list) {
-        wait_sync_list = (sync == sync->next) ? NULL : sync->next;
-        if (NULL != wait_sync_list) {
-            WAIT_SYNC_PASS_OWNERSHIP(wait_sync_list);
-        }
-    }
-    OPAL_THREAD_UNLOCK(&wait_sync_lock);
-
-    return (0 == sync->status) ? OPAL_SUCCESS : OPAL_ERROR;
-}
diff --git a/opal/mca/threads/pthreads/threads_pthreads_wait_sync.h b/opal/mca/threads/pthreads/threads_pthreads_wait_sync.h
deleted file mode 100644
index e8f5bfe0b17..00000000000
--- a/opal/mca/threads/pthreads/threads_pthreads_wait_sync.h
+++ /dev/null
@@ -1,121 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007-2018 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015-2016 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2019      Sandia National Laboratories.  All rights reserved.
- *
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-
-#ifndef OPAL_MCA_THREADS_PTHREADS_THREADS_PTHREADS_WAIT_SYNC_H
-#define OPAL_MCA_THREADS_PTHREADS_THREADS_PTHREADS_WAIT_SYNC_H
-
-typedef struct ompi_wait_sync_t {
-    opal_atomic_int32_t count;
-    int32_t status;
-    pthread_cond_t condition;
-    pthread_mutex_t lock;
-    struct ompi_wait_sync_t *next;
-    struct ompi_wait_sync_t *prev;
-    volatile bool signaling;
-} ompi_wait_sync_t;
-
-#define SYNC_WAIT(sync) \
-    (opal_using_threads() ? ompi_sync_wait_mt(sync) : sync_wait_st(sync))
-
-/* The loop in release handles a race condition between the signaling
- * thread and the destruction of the condition variable. The signaling
- * member will be set to false after the final signaling thread has
- * finished operating on the sync object. This is done to avoid
- * extra atomics in the signalling function and keep it as fast
- * as possible. Note that the race window is small so spinning here
- * is more optimal than sleeping since this macro is called in
- * the critical path. */
-#define WAIT_SYNC_RELEASE(sync)                       \
-    if (opal_using_threads()) {                       \
-        while ((sync)->signaling) {                   \
-            continue;                                 \
-        }                                             \
-        pthread_cond_destroy(&(sync)->condition);     \
-        pthread_mutex_destroy(&(sync)->lock);         \
-    }
-
-#define WAIT_SYNC_RELEASE_NOWAIT(sync)                \
-    if (opal_using_threads()) {                       \
-        pthread_cond_destroy(&(sync)->condition);     \
-        pthread_mutex_destroy(&(sync)->lock);         \
-    }
-
-
-#define WAIT_SYNC_SIGNAL(sync)                        \
-    if (opal_using_threads()) {                       \
-        pthread_mutex_lock(&(sync->lock));            \
-        pthread_cond_signal(&sync->condition);        \
-        pthread_mutex_unlock(&(sync->lock));          \
-        sync->signaling = false;                      \
-    }
-
-#define WAIT_SYNC_SIGNALLED(sync)                     \
-    {                                                 \
-        (sync)->signaling = false;                    \
-    }
-
-
-/* not static for inline "wait_sync_st" */
-OPAL_DECLSPEC extern ompi_wait_sync_t *wait_sync_list;
-
-OPAL_DECLSPEC int ompi_sync_wait_mt(ompi_wait_sync_t *sync);
-static inline int sync_wait_st(ompi_wait_sync_t *sync)
-{
-    assert( NULL == wait_sync_list );
-    assert( NULL == sync->next );
-    wait_sync_list = sync;
-
-    while (sync->count > 0) {
-        opal_progress();
-    }
-    wait_sync_list = NULL;
-
-    return sync->status;
-}
-
-#define WAIT_SYNC_INIT(sync,c)                                  \
-    do {                                                        \
-        (sync)->count = (c);                                    \
-        (sync)->next = NULL;                                    \
-        (sync)->prev = NULL;                                    \
-        (sync)->status = 0;                                     \
-        (sync)->signaling = (0 != (c));                         \
-        if (opal_using_threads()) {                             \
-            pthread_cond_init(&(sync)->condition, NULL);        \
-            pthread_mutex_init(&(sync)->lock, NULL);            \
-        }                                                       \
-    } while (0)
-
-/**
- * Wake up all syncs with a particular status. If status is OMPI_SUCCESS this
- * operation is a NO-OP. Otherwise it will trigger the "error condition" from
- * all registered sync.
- */
-OPAL_DECLSPEC void wait_sync_global_wakeup_st(int status);
-OPAL_DECLSPEC void wait_sync_global_wakeup_mt(int status);
-#define wait_sync_global_wakeup(st) (opal_using_threads()? wait_sync_global_wakeup_mt(st): wait_sync_global_wakeup_st(st))
-
-#endif /* OPAL_MCA_THREADS_PTHREADS_THREADS_PTHREADS_WAIT_SYNC_H */
diff --git a/opal/mca/threads/pthreads/threads_pthreads_yield.c b/opal/mca/threads/pthreads/threads_pthreads_yield.c
index d68126fea4a..4ab5caa85ec 100644
--- a/opal/mca/threads/pthreads/threads_pthreads_yield.c
+++ b/opal/mca/threads/pthreads/threads_pthreads_yield.c
@@ -10,15 +10,18 @@
  * $HEADER$
  */
 
+/* needed for nanosleep() */
+#define _POSIX_C_SOURCE 200809L
+
 #include "opal_config.h"
 #include <time.h>
 #ifdef HAVE_SCHED_H
-#include <sched.h>
+#    include <sched.h>
 #endif
 
 #include "opal/constants.h"
-#include "opal/mca/threads/thread.h"
 #include "opal/mca/threads/pthreads/threads_pthreads.h"
+#include "opal/mca/threads/thread.h"
 
 static void opal_thread_pthreads_yield_sched_yield(void);
 static void opal_thread_pthreads_yield_nanosleep(void);
@@ -28,12 +31,11 @@ typedef enum {
     OPAL_PTHREADS_YIELD_NANOSLEEP
 } opal_threads_pthreads_yield_strategy_t;
 
-static mca_base_var_enum_value_t yield_strategy_values[] = {
-    {OPAL_PTHREADS_YIELD_SCHED_YIELD, "sched_yield"},
-    {OPAL_PTHREADS_YIELD_NANOSLEEP,   "nanosleep"},
-    {0, NULL}};
-
-
+static mca_base_var_enum_value_t yield_strategy_values[] = {{OPAL_PTHREADS_YIELD_SCHED_YIELD,
+                                                             "sched_yield"},
+                                                            {OPAL_PTHREADS_YIELD_NANOSLEEP,
+                                                             "nanosleep"},
+                                                            {0, NULL}};
 
 /* Number of nanoseconds to nanosleep, if enabled */
 static uint64_t yield_nsleep_nanosecs;
@@ -41,38 +43,40 @@ static uint64_t yield_nsleep_nanosecs;
 static struct timespec yield_nsleep_time = {.tv_sec = 0, .tv_nsec = 1};
 static opal_threads_pthreads_yield_strategy_t yield_strategy = OPAL_PTHREADS_YIELD_SCHED_YIELD;
 
-opal_threads_pthreads_yield_fn_t *opal_threads_pthreads_yield_fn = &opal_thread_pthreads_yield_sched_yield;
+opal_threads_pthreads_yield_fn_t *opal_threads_pthreads_yield_fn
+    = &opal_thread_pthreads_yield_sched_yield;
 
 int opal_threads_pthreads_yield_init(const mca_base_component_t *component)
 {
     mca_base_var_enum_t *yield_strategy_enumerator;
-    mca_base_var_enum_create("pthread_yield_strategies", yield_strategy_values, &yield_strategy_enumerator);
+    mca_base_var_enum_create("pthread_yield_strategies", yield_strategy_values,
+                             &yield_strategy_enumerator);
 
     (void) mca_base_component_var_register(component, "yield_strategy",
-                                           "Pthread yield strategy to use",
-                                           MCA_BASE_VAR_TYPE_INT, yield_strategy_enumerator, 0, 0, OPAL_INFO_LVL_3,
+                                           "Pthread yield strategy to use", MCA_BASE_VAR_TYPE_INT,
+                                           yield_strategy_enumerator, 0, 0, OPAL_INFO_LVL_3,
                                            MCA_BASE_VAR_SCOPE_LOCAL, &yield_strategy);
-    switch(yield_strategy) {
-        case OPAL_PTHREADS_YIELD_NANOSLEEP:
-            opal_threads_pthreads_yield_fn = &opal_thread_pthreads_yield_nanosleep;
-            break;
-        default:
-            /* use initial value */
-            break;
+    switch (yield_strategy) {
+    case OPAL_PTHREADS_YIELD_NANOSLEEP:
+        opal_threads_pthreads_yield_fn = &opal_thread_pthreads_yield_nanosleep;
+        break;
+    default:
+        /* use initial value */
+        break;
     }
 
     OBJ_RELEASE(yield_strategy_enumerator);
 
     yield_nsleep_nanosecs = (yield_nsleep_time.tv_sec * 1E9) + yield_nsleep_time.tv_nsec;
-    (void) mca_base_component_var_register(component, "nanosleep_time",
-                                           "Number of nanoseconds to sleep when using nanosleep as the pthread yield strategy",
-                                           MCA_BASE_VAR_TYPE_UINT64_T, NULL, 0, 0, OPAL_INFO_LVL_3,
-                                           MCA_BASE_VAR_SCOPE_LOCAL, &yield_nsleep_nanosecs);
-    yield_nsleep_time.tv_sec  = yield_nsleep_nanosecs / 1E9;
+    (void) mca_base_component_var_register(
+        component, "nanosleep_time",
+        "Number of nanoseconds to sleep when using nanosleep as the pthread yield strategy",
+        MCA_BASE_VAR_TYPE_UINT64_T, NULL, 0, 0, OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_LOCAL,
+        &yield_nsleep_nanosecs);
+    yield_nsleep_time.tv_sec = yield_nsleep_nanosecs / 1E9;
     yield_nsleep_time.tv_nsec = yield_nsleep_nanosecs - (uint64_t)(yield_nsleep_time.tv_sec * 1E9);
 
     return OPAL_SUCCESS;
-
 }
 
 void opal_thread_pthreads_yield_sched_yield(void)
@@ -86,4 +90,3 @@ void opal_thread_pthreads_yield_nanosleep(void)
 {
     nanosleep(&yield_nsleep_time, NULL);
 }
-
diff --git a/opal/mca/threads/qthreads/Makefile.am b/opal/mca/threads/qthreads/Makefile.am
index 683b9cc435e..e1f891dcc5a 100644
--- a/opal/mca/threads/qthreads/Makefile.am
+++ b/opal/mca/threads/qthreads/Makefile.am
@@ -11,6 +11,7 @@
 #                         All rights reserved.
 # Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2019      Sandia National Laboratories.  All rights reserved.
+# Copyright (c) 2021      Argonne National Laboratory.  All rights reserved.
 #
 # $COPYRIGHT$
 #
@@ -26,12 +27,9 @@ sources = \
     threads_qthreads_component.c \
     threads_qthreads_condition.c \
     threads_qthreads_module.c \
-    threads_qthreads_mutex.c \
-    threads_qthreads_wait_sync.c \
     threads_qthreads_mutex.h \
     threads_qthreads_threads.h \
-    threads_qthreads_tsd.h \
-    threads_qthreads_wait_sync.h
+    threads_qthreads_tsd.h
 
 lib_sources = $(sources)
 
@@ -47,7 +45,7 @@ mcacomponentdir = $(opallibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_threads_qthreads_la_SOURCES = $(sources)
 mca_threads_qthreads_la_LDFLAGS = -module -avoid-version
-mca_threads_qthreads_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
+mca_threads_qthreads_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
         $(opal_qthreads_LIBS)
 
 noinst_LTLIBRARIES = $(component_noinst)
diff --git a/opal/mca/threads/qthreads/configure.m4 b/opal/mca/threads/qthreads/configure.m4
index 4121711685a..5fa325784f0 100644
--- a/opal/mca/threads/qthreads/configure.m4
+++ b/opal/mca/threads/qthreads/configure.m4
@@ -13,7 +13,7 @@
 # Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2015      Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
-# Copyright (c) 2019      Triad National Security, LLC. All rights
+# Copyright (c) 2019-2022 Triad National Security, LLC. All rights
 #                         reserved.
 #
 # $COPYRIGHT$
@@ -37,35 +37,13 @@ AC_DEFUN([OPAL_CONFIG_QTHREADS],[
     opal_check_qthreads_save_LDFLAGS=$LDFLAGS
     opal_check_qthreads_save_LIBS=$LIBS
 
-    opal_qthreads_happy=yes
-    AS_IF([test "$with_qthreads" = "no"],
-          [opal_qthreads_happy=no])
-
-    AS_IF([test $opal_qthreads_happy = yes],
-          [AC_MSG_CHECKING([looking for qthreads in])
-           AS_IF([test "$with_qthreads" != "yes"],
-                 [opal_qthreads_dir=$with_qthreads
-                  AC_MSG_RESULT([($opal_qthreads_dir)])],
-                 [AC_MSG_RESULT([(default search paths)])])
-           AS_IF([test ! -z "$with_qthreads_libdir" && \
-                         test "$with_qthreads_libdir" != "yes"],
-                 [opal_qthreads_libdir=$with_qthreads_libdir])
-          ])
-
-    AS_IF([test $opal_qthreads_happy = yes],
-          [OPAL_CHECK_PACKAGE([opal_qthreads],
-                              [qthread.h],
-                              [qthread],
-                              [qthread_initialize],
-                              [],
-                              [$opal_qthreads_dir],
-                              [$opal_qthreads_libdir],
-                              [],
-                              [opal_qthreads_happy=no])])
-
-    AS_IF([test $opal_qthreads_happy = yes && test -n "$opal_qthreads_dir"],
-          [OPAL_QTHREADS_INCLUDE_PATH="$opal_qthreads_dir/include/"],
-          [OPAL_QTHREADS_INCLUDE_PATH=""])
+    OAC_CHECK_PACKAGE([qthreads],
+                      [opal_qthreads],
+                      [qthread.h],
+                      [qthread],
+                      [qthread_initialize],
+                      [opal_qthreads_happy=yes],
+                      [opal_qthreads_happy=no])
 
     AS_IF([test $opal_qthreads_happy = yes],
           [TPKG_CFLAGS="$opal_qthreads_CPPFLAGS"
@@ -76,13 +54,11 @@ AC_DEFUN([OPAL_CONFIG_QTHREADS],[
            TPKG_LDFLAGS="$opal_qthreads_LDFLAGS"
            TPKG_LIBS="$opal_qthreads_LIBS"])
 
-    AC_CONFIG_FILES([opal/mca/threads/qthreads/threads_qthreads.h])
-    AC_SUBST([OPAL_QTHREADS_INCLUDE_PATH])
     AC_SUBST([opal_qthreads_CPPFLAGS])
     AC_SUBST([opal_qthreads_LDFLAGS])
     AC_SUBST([opal_qthreads_LIBS])
 
-    CPPFLAGS=$opal_check_qthreads_save_CPPFLAGS
+    CPPFLAGS="${opal_check_argo_save_CPPFLAGS} ${opal_qthreads_CPPFLAGS}"
     LDFLAGS=$opal_check_qthreads_save_LDFLAGS
     LIBS=$opal_check_qthreads_save_LIBS
 
@@ -112,9 +88,6 @@ AC_DEFUN([MCA_opal_threads_qthreads_POST_CONFIG],[
            AC_DEFINE_UNQUOTED([MCA_threads_tsd_base_include_HEADER],
                               ["opal/mca/threads/qthreads/threads_qthreads_tsd.h"],
                               [Header to include for tsd implementation])
-           AC_DEFINE_UNQUOTED([MCA_threads_wait_sync_base_include_HEADER],
-                              ["opal/mca/threads/qthreads/threads_qthreads_wait_sync.h"],
-                              [Header to include for wait_sync implementation])
            THREAD_CFLAGS="$TPKG_CFLAGS"
            THREAD_FCFLAGS="$TPKG_FCFLAGS"
            THREAD_CXXFLAGS="$TPKG_CXXFLAGS"
diff --git a/opal/mca/threads/qthreads/threads_qthreads.h b/opal/mca/threads/qthreads/threads_qthreads.h
new file mode 100644
index 00000000000..dcb13c3201d
--- /dev/null
+++ b/opal/mca/threads/qthreads/threads_qthreads.h
@@ -0,0 +1,37 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007-2016 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2019      Sandia National Laboratories.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef OPAL_MCA_THREADS_QTHREADS_THREADS_QTHREADS_H
+#define OPAL_MCA_THREADS_QTHREADS_THREADS_QTHREADS_H 1
+
+#include "qthread.h"
+#include "qthread/tls.h"
+
+static inline void opal_threads_ensure_init_qthreads(void)
+{
+    qthread_initialize();
+}
+
+#endif /* OPAL_MCA_THREADS_QTHREADS_THREADS_QTHREADS_H */
diff --git a/opal/mca/threads/qthreads/threads_qthreads.h.in b/opal/mca/threads/qthreads/threads_qthreads.h.in
deleted file mode 100644
index 378eaa6e737..00000000000
--- a/opal/mca/threads/qthreads/threads_qthreads.h.in
+++ /dev/null
@@ -1,37 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007-2016 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2019      Sandia National Laboratories.  All rights reserved.
- *
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef OPAL_MCA_THREADS_QTHREADS_THREADS_QTHREADS_H
-#define OPAL_MCA_THREADS_QTHREADS_THREADS_QTHREADS_H 1
-
-#include "@OPAL_QTHREADS_INCLUDE_PATH@qthread.h"
-#include "@OPAL_QTHREADS_INCLUDE_PATH@qthread/tls.h"
-
-static inline void opal_threads_ensure_init_qthreads(void)
-{
-    qthread_initialize();
-}
-
-#endif /* OPAL_MCA_THREADS_QTHREADS_THREADS_QTHREADS_H */
diff --git a/opal/mca/threads/qthreads/threads_qthreads_component.c b/opal/mca/threads/qthreads/threads_qthreads_component.c
index cf8f5944620..9ce9ac6f630 100644
--- a/opal/mca/threads/qthreads/threads_qthreads_component.c
+++ b/opal/mca/threads/qthreads/threads_qthreads_component.c
@@ -23,30 +23,30 @@
 
 #include "opal_config.h"
 
+#include "opal/constants.h"
 #include "opal/mca/threads/qthreads/threads_qthreads.h"
 #include "opal/mca/threads/thread.h"
 #include "opal/mca/threads/threads.h"
-#include "opal/constants.h"
 
 static int opal_threads_qthreads_open(void);
 
 const opal_threads_base_component_1_0_0_t mca_threads_qthreads_component = {
     /* First, the mca_component_t struct containing meta information
      * about the component itself */
-    .threadsc_version = {
-        OPAL_THREADS_BASE_VERSION_1_0_0,
+    .threadsc_version =
+        {
+            OPAL_THREADS_BASE_VERSION_1_0_0,
 
-        /* Component name and version */
-        .mca_component_name = "qthreads",
-        MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
-                              OPAL_RELEASE_VERSION),
+            /* Component name and version */
+            .mca_component_name = "qthreads",
+            MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
+                                  OPAL_RELEASE_VERSION),
 
-        .mca_open_component = opal_threads_qthreads_open,
-    },
-    .threadsc_data = {
-        /* The component is checkpoint ready */
-        MCA_BASE_METADATA_PARAM_CHECKPOINT
-    },
+            .mca_open_component = opal_threads_qthreads_open,
+        },
+    .threadsc_data =
+        {/* The component is checkpoint ready */
+         MCA_BASE_METADATA_PARAM_CHECKPOINT},
 };
 
 int opal_threads_qthreads_open(void)
diff --git a/opal/mca/threads/qthreads/threads_qthreads_condition.c b/opal/mca/threads/qthreads/threads_qthreads_condition.c
index 7c569b3898a..f9d9b195336 100644
--- a/opal/mca/threads/qthreads/threads_qthreads_condition.c
+++ b/opal/mca/threads/qthreads/threads_qthreads_condition.c
@@ -27,16 +27,10 @@
 
 #include "opal/mca/threads/condition.h"
 
-
 static void opal_condition_construct(opal_condition_t *c)
 {
     c->c_waiting = 0;
     c->c_signaled = 0;
 }
 
-
-
-OBJ_CLASS_INSTANCE(opal_condition_t,
-                   opal_object_t,
-                   opal_condition_construct,
-                   NULL);
+OBJ_CLASS_INSTANCE(opal_condition_t, opal_object_t, opal_condition_construct, NULL);
diff --git a/opal/mca/threads/qthreads/threads_qthreads_module.c b/opal/mca/threads/qthreads/threads_qthreads_module.c
index d53c0681772..7dca13f5f4f 100644
--- a/opal/mca/threads/qthreads/threads_qthreads_module.c
+++ b/opal/mca/threads/qthreads/threads_qthreads_module.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -19,8 +19,8 @@
  * $HEADER$
  */
 
-#include "opal/mca/threads/qthreads/threads_qthreads.h"
 #include "opal/constants.h"
+#include "opal/mca/threads/qthreads/threads_qthreads.h"
 #include "opal/mca/threads/threads.h"
 #include "opal/mca/threads/tsd.h"
 
@@ -51,76 +51,6 @@ static inline void self_key_ensure_init(void)
     /* opal_thread_self_key has been already initialized. */
 }
 
-/*
- * Constructor
- */
-static void opal_thread_construct(opal_thread_t *t)
-{
-    t->t_run = 0;
-    t->t_thread_ret = 0;
-}
-
-OBJ_CLASS_INSTANCE(opal_thread_t,
-                   opal_object_t,
-                   opal_thread_construct, NULL);
-
-static inline aligned_t *opal_thread_get_qthreads_self(void)
-{
-    self_key_ensure_init();
-    void *ptr = qthread_getspecific(opal_thread_self_key);
-    return (aligned_t *)ptr;
-}
-
-static aligned_t opal_thread_qthreads_wrapper(void *arg)
-{
-    opal_thread_t *t = (opal_thread_t *)arg;
-
-    /* Register itself. */
-    self_key_ensure_init();
-    qthread_setspecific(opal_thread_self_key, t->t_thread_ret_ptr);
-
-    t->t_ret = ((void *(*)(void *))t->t_run)(t);
-    return 0;
-}
-
-opal_thread_t *opal_thread_get_self(void)
-{
-    opal_threads_ensure_init_qthreads();
-    opal_thread_t *t = OBJ_NEW(opal_thread_t);
-    t->t_thread_ret_ptr = opal_thread_get_qthreads_self();
-    return t;
-}
-
-bool opal_thread_self_compare(opal_thread_t *t)
-{
-    opal_threads_ensure_init_qthreads();
-    return opal_thread_get_qthreads_self() == &t->t_thread_ret;
-}
-
-int opal_thread_join(opal_thread_t *t, void **thr_return)
-{
-    qthread_readFF(NULL, t->t_thread_ret_ptr);
-    if (thr_return) {
-        *thr_return = t->t_ret;
-    }
-    t->t_thread_ret = 0;
-    return OPAL_SUCCESS;
-}
-
-void opal_thread_set_main(void)
-{
-}
-
-int opal_thread_start(opal_thread_t *t)
-{
-    opal_threads_ensure_init_qthreads();
-    t->t_thread_ret_ptr = &t->t_thread_ret;
-    qthread_fork(opal_thread_qthreads_wrapper, t, &t->t_thread_ret);
-    return OPAL_SUCCESS;
-}
-
-OBJ_CLASS_DECLARATION(opal_thread_t);
-
 int opal_tsd_key_create(opal_tsd_key_t *key, opal_tsd_destructor_t destructor)
 {
     opal_threads_ensure_init_qthreads();
diff --git a/opal/mca/threads/qthreads/threads_qthreads_mutex.c b/opal/mca/threads/qthreads/threads_qthreads_mutex.c
deleted file mode 100644
index 5fce8ec3239..00000000000
--- a/opal/mca/threads/qthreads/threads_qthreads_mutex.c
+++ /dev/null
@@ -1,139 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007-2016 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2019      Sandia National Laboratories.  All rights reserved.
- *
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "opal_config.h"
-
-#include "opal/mca/threads/mutex.h"
-#include "opal/constants.h"
-
-/*
- * Wait and see if some upper layer wants to use threads, if support
- * exists.
- */
-bool opal_uses_threads = false;
-
-static void opal_mutex_construct(opal_mutex_t *m)
-{
-    opal_threads_ensure_init_qthreads();
-    opal_mutex_create(m);
-}
-
-static void opal_mutex_destruct(opal_mutex_t *m)
-{
-}
-
-OBJ_CLASS_INSTANCE(opal_mutex_t,
-                   opal_object_t,
-                   opal_mutex_construct,
-                   opal_mutex_destruct);
-
-static void opal_recursive_mutex_construct(opal_recursive_mutex_t *m)
-{
-}
-
-OBJ_CLASS_INSTANCE(opal_recursive_mutex_t,
-                   opal_object_t,
-                   opal_recursive_mutex_construct,
-                   opal_mutex_destruct);
-
-int opal_cond_init(opal_cond_t *cond)
-{
-    opal_atomic_lock_init(&cond->m_lock, 0);
-    cond->m_waiter_head = NULL;
-    cond->m_waiter_tail = NULL;
-    return OPAL_SUCCESS;
-}
-
-typedef struct {
-    int m_signaled;
-    void *m_prev;
-} cond_waiter_t;
-
-int opal_cond_wait(opal_cond_t *cond, opal_mutex_t *lock)
-{
-    opal_threads_ensure_init_qthreads();
-    /* This thread is taking "lock", so only this thread can access this
-     * condition variable.  */
-    opal_atomic_lock(&cond->m_lock);
-    cond_waiter_t waiter = { 0, NULL };
-    if (NULL == cond->m_waiter_head) {
-        cond->m_waiter_tail = (void *)&waiter;
-    } else {
-        ((cond_waiter_t *)cond->m_waiter_head)->m_prev = (void *)&waiter;
-    }
-    cond->m_waiter_head = (void *)&waiter;
-    opal_atomic_unlock(&cond->m_lock);
-
-    while (1) {
-        opal_mutex_unlock(lock);
-        qthread_yield();
-        opal_mutex_lock(lock);
-        /* Check if someone woke me up. */
-        opal_atomic_lock(&cond->m_lock);
-        int signaled = waiter.m_signaled;
-        opal_atomic_unlock(&cond->m_lock);
-        if (1 == signaled) {
-            break;
-        }
-        /* Unlock the lock again. */
-    }
-    return OPAL_SUCCESS;
-}
-
-int opal_cond_broadcast(opal_cond_t *cond)
-{
-    opal_atomic_lock(&cond->m_lock);
-    while (NULL != cond->m_waiter_tail) {
-        cond_waiter_t *p_cur_tail = (cond_waiter_t *)cond->m_waiter_tail;
-        cond->m_waiter_tail = p_cur_tail->m_prev;
-        /* Awaken one of threads in a FIFO manner. */
-        p_cur_tail->m_signaled = 1;
-    }
-    /* No waiters. */
-    cond->m_waiter_head = NULL;
-    opal_atomic_unlock(&cond->m_lock);
-    return OPAL_SUCCESS;
-}
-
-int opal_cond_signal(opal_cond_t *cond)
-{
-    opal_atomic_lock(&cond->m_lock);
-    if (NULL != cond->m_waiter_tail) {
-        cond_waiter_t *p_cur_tail = (cond_waiter_t *)cond->m_waiter_tail;
-        cond->m_waiter_tail = p_cur_tail->m_prev;
-        /* Awaken one of threads. */
-        p_cur_tail->m_signaled = 1;
-        if (NULL == cond->m_waiter_tail) {
-            cond->m_waiter_head = NULL;
-        }
-    }
-    opal_atomic_unlock(&cond->m_lock);
-    return OPAL_SUCCESS;
-}
-
-int opal_cond_destroy(opal_cond_t *cond)
-{
-    return OPAL_SUCCESS;
-}
diff --git a/opal/mca/threads/qthreads/threads_qthreads_mutex.h b/opal/mca/threads/qthreads/threads_qthreads_mutex.h
index 7571d616362..bf14a7dabb2 100644
--- a/opal/mca/threads/qthreads/threads_qthreads_mutex.h
+++ b/opal/mca/threads/qthreads/threads_qthreads_mutex.h
@@ -19,6 +19,8 @@
  *                         reserved.
  *
  * Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2021      Argonne National Laboratory.  All rights reserved.
+ * Copyright (c) 2022      Sandia National Laboratories.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -31,194 +33,173 @@
 
 #include "opal_config.h"
 
-#include <errno.h>
 #include <stdio.h>
 
-#include "opal/mca/threads/qthreads/threads_qthreads.h"
-#include "opal/constants.h"
 #include "opal/class/opal_object.h"
+#include "opal/constants.h"
+#include "opal/mca/threads/qthreads/threads_qthreads.h"
 #include "opal/sys/atomic.h"
-#include "opal/util/output.h"
+#include "opal/util/show_help.h"
 
 BEGIN_C_DECLS
 
-struct opal_mutex_t {
-    opal_object_t super;
-
-    opal_atomic_lock_t m_lock;
-    int m_recursive;
-
-#if OPAL_ENABLE_DEBUG
-    int m_lock_debug;
-    const char *m_lock_file;
-    int m_lock_line;
-#endif
-
-    opal_atomic_lock_t m_lock_atomic;
-};
+typedef qthread_spinlock_t opal_thread_internal_mutex_t;
 
-OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_mutex_t);
-OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_recursive_mutex_t);
-
-#if OPAL_ENABLE_DEBUG
-#define OPAL_MUTEX_STATIC_INIT                                          \
-    {                                                                   \
-        .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t),                    \
-        .m_lock = OPAL_ATOMIC_LOCK_INIT,                                \
-        .m_recursive = 0,                                               \
-        .m_lock_debug = 0,                                              \
-        .m_lock_file = NULL,                                            \
-        .m_lock_line = 0,                                               \
-        .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT,                         \
-    }
-#else
-#define OPAL_MUTEX_STATIC_INIT                                          \
-    {                                                                   \
-        .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t),                    \
-        .m_lock = OPAL_ATOMIC_LOCK_INIT,                                \
-        .m_recursive = 0,                                               \
-        .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT,                         \
-    }
-#endif
+#define OPAL_THREAD_INTERNAL_MUTEX_INITIALIZER           QTHREAD_MUTEX_INITIALIZER
+#define OPAL_THREAD_INTERNAL_RECURSIVE_MUTEX_INITIALIZER QTHREAD_RECURSIVE_MUTEX_INITIALIZER
 
-#if OPAL_ENABLE_DEBUG
-#define OPAL_RECURSIVE_MUTEX_STATIC_INIT                                \
-    {                                                                   \
-        .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t),                    \
-        .m_lock = OPAL_ATOMIC_LOCK_INIT,                                \
-        .m_recursive = 1,                                               \
-        .m_lock_debug = 0,                                              \
-        .m_lock_file = NULL,                                            \
-        .m_lock_line = 0,                                               \
-        .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT,                         \
-    }
-#else
-#define OPAL_RECURSIVE_MUTEX_STATIC_INIT                                \
-    {                                                                   \
-        .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t),                    \
-        .m_lock = OPAL_ATOMIC_LOCK_INIT,                                \
-        .m_recursive = 1,                                               \
-        .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT,                         \
-    }
-#endif
-
-/************************************************************************
- *
- * mutex operations (non-atomic versions)
- *
- ************************************************************************/
-
-static inline void opal_mutex_create(struct opal_mutex_t *m)
+static inline int opal_thread_internal_mutex_init(opal_thread_internal_mutex_t *p_mutex,
+                                                  bool recursive)
 {
-    opal_atomic_lock_init(&m->m_lock, 0);
-    opal_atomic_lock_init(&m->m_lock_atomic, 0);
-    m->m_recursive = 0;
-#if OPAL_ENABLE_DEBUG
-    m->m_lock_debug = 0;
-    m->m_lock_file = NULL;
-    m->m_lock_line = 0;
-#endif
+    opal_threads_ensure_init_qthreads();
+    #if OPAL_ENABLE_DEBUG
+    int ret = qthread_spinlock_init(p_mutex,recursive);
+    if (QTHREAD_SUCCESS != ret) {
+        opal_show_help("help-opal-threads.txt", "mutex init failed", true);
+    }
+    #else
+    qthread_spinlock_init(p_mutex,recursive);
+    #endif
+    return OPAL_SUCCESS;
 }
 
-static inline int opal_mutex_trylock(opal_mutex_t *m)
+static inline void opal_thread_internal_mutex_lock(opal_thread_internal_mutex_t *p_mutex)
 {
     opal_threads_ensure_init_qthreads();
-    int ret = opal_atomic_trylock(&m->m_lock);
-    if (0 != ret) {
-        /* Yield to avoid a deadlock. */
-        qthread_yield();
+    #if OPAL_ENABLE_DEBUG
+    int ret = qthread_spinlock_lock(p_mutex);
+    if (QTHREAD_SUCCESS != ret) {
+        opal_show_help("help-opal-threads.txt", "mutex lock failed", true);
     }
-    return ret;
+    #else
+    qthread_spinlock_lock(p_mutex);
+    #endif
 }
 
-static inline void opal_mutex_lock(opal_mutex_t *m)
+static inline int opal_thread_internal_mutex_trylock(opal_thread_internal_mutex_t *p_mutex)
 {
     opal_threads_ensure_init_qthreads();
-    int ret = opal_atomic_trylock(&m->m_lock);
-    while (0 != ret) {
-        qthread_yield();
-        ret = opal_atomic_trylock(&m->m_lock);
-    }
+    int ret = qthread_spinlock_trylock(p_mutex);
+    if (QTHREAD_OPFAIL == ret) {
+        return 1;
+    } else if (QTHREAD_SUCCESS != ret) {
+#if OPAL_ENABLE_DEBUG
+        opal_show_help("help-opal-threads.txt", "mutex trylock failed", true);
+#endif
+        return 1;
+    } 
+    return 0;
 }
 
-static inline void opal_mutex_unlock(opal_mutex_t *m)
+static inline void opal_thread_internal_mutex_unlock(opal_thread_internal_mutex_t *p_mutex)
 {
     opal_threads_ensure_init_qthreads();
-
-    opal_atomic_unlock(&m->m_lock);
+    int ret; 
+    #if OPAL_ENABLE_DEBUG
+    ret = qthread_spinlock_unlock(p_mutex);
+    if (QTHREAD_SUCCESS != ret) {
+        opal_show_help("help-opal-threads.txt", "mutex unlock failed", true);
+    }
+    #else
+    qthread_spinlock_unlock(p_mutex);
+    #endif
     /* For fairness of locking. */
     qthread_yield();
 }
 
-/************************************************************************
- *
- * mutex operations (atomic versions)
- *
- ************************************************************************/
+static inline void opal_thread_internal_mutex_destroy(opal_thread_internal_mutex_t *p_mutex)
+{
+    /* No specific operation is needed to destroy opal_thread_internal_mutex_t. */
+}
 
-#if OPAL_HAVE_ATOMIC_SPINLOCKS
+typedef struct opal_thread_cond_waiter_t {
+    int m_signaled;
+    struct opal_thread_cond_waiter_t *m_prev;
+} opal_thread_cond_waiter_t;
 
-/************************************************************************
- * Spin Locks
- ************************************************************************/
+typedef struct {
+    opal_thread_internal_mutex_t m_lock;
+    opal_thread_cond_waiter_t *m_waiter_head;
+    opal_thread_cond_waiter_t *m_waiter_tail;
+} opal_thread_internal_cond_t;
 
-static inline int opal_mutex_atomic_trylock(opal_mutex_t *m)
-{
-    return opal_atomic_trylock(&m->m_lock_atomic);
-}
+#define OPAL_THREAD_INTERNAL_COND_INITIALIZER                                          \
+    {                                                                                  \
+        .m_lock = QTHREAD_MUTEX_INITIALIZER, .m_waiter_head = NULL, .m_waiter_tail = NULL, \
+    }
 
-static inline void opal_mutex_atomic_lock(opal_mutex_t *m)
+static inline int opal_thread_internal_cond_init(opal_thread_internal_cond_t *p_cond)
 {
-    opal_atomic_lock(&m->m_lock_atomic);
+    qthread_spinlock_init(&p_cond->m_lock, false /* is_recursive */);
+    p_cond->m_waiter_head = NULL;
+    p_cond->m_waiter_tail = NULL;
+    return OPAL_SUCCESS;
 }
 
-static inline void opal_mutex_atomic_unlock(opal_mutex_t *m)
+static inline void opal_thread_internal_cond_wait(opal_thread_internal_cond_t *p_cond,
+                                                  opal_thread_internal_mutex_t *p_mutex)
 {
-    opal_atomic_unlock(&m->m_lock_atomic);
+    opal_threads_ensure_init_qthreads();
+    /* This thread is taking "lock", so only this thread can access this
+     * condition variable.  */
+    qthread_spinlock_lock(&p_cond->m_lock);
+    opal_thread_cond_waiter_t waiter = {0, NULL};
+    if (NULL == p_cond->m_waiter_head) {
+        p_cond->m_waiter_tail = &waiter;
+    } else {
+        p_cond->m_waiter_head->m_prev = &waiter;
+    }
+    p_cond->m_waiter_head = &waiter;
+    qthread_spinlock_unlock(&p_cond->m_lock);
+    while (1) {       
+        opal_thread_internal_mutex_unlock(p_mutex);
+        qthread_yield();
+        opal_thread_internal_mutex_lock(p_mutex);
+        /* Check if someone woke me up. */
+        qthread_spinlock_lock(&p_cond->m_lock);
+        int signaled = waiter.m_signaled;
+        qthread_spinlock_unlock(&p_cond->m_lock);
+        if (1 == signaled) {
+            break;
+        }
+        /* Unlock the lock again. */
+    }
 }
 
-#else
-
-/************************************************************************
- * Standard locking
- ************************************************************************/
-
-static inline int opal_mutex_atomic_trylock(opal_mutex_t *m)
+static inline void opal_thread_internal_cond_broadcast(opal_thread_internal_cond_t *p_cond)
 {
-    return opal_mutex_trylock(m);
+    qthread_spinlock_lock(&p_cond->m_lock);
+    while (NULL != p_cond->m_waiter_tail) {
+        opal_thread_cond_waiter_t *p_cur_tail = p_cond->m_waiter_tail;
+        p_cond->m_waiter_tail = p_cur_tail->m_prev;
+        /* Awaken one of threads in a FIFO manner. */
+        p_cur_tail->m_signaled = 1;
+    }
+    /* No waiters. */
+    p_cond->m_waiter_head = NULL;
+    qthread_spinlock_unlock(&p_cond->m_lock);
 }
 
-static inline void opal_mutex_atomic_lock(opal_mutex_t *m)
+static inline void opal_thread_internal_cond_signal(opal_thread_internal_cond_t *p_cond)
 {
-    opal_mutex_lock(m);
+    qthread_spinlock_lock(&p_cond->m_lock);
+    if (NULL != p_cond->m_waiter_tail) {
+        opal_thread_cond_waiter_t *p_cur_tail = p_cond->m_waiter_tail;
+        p_cond->m_waiter_tail = p_cur_tail->m_prev;
+        /* Awaken one of threads. */
+        p_cur_tail->m_signaled = 1;
+        if (NULL == p_cond->m_waiter_tail) {
+            p_cond->m_waiter_head = NULL;
+        }
+    }
+    qthread_spinlock_unlock(&p_cond->m_lock);
 }
 
-static inline void opal_mutex_atomic_unlock(opal_mutex_t *m)
+static inline void opal_thread_internal_cond_destroy(opal_thread_internal_cond_t *p_cond)
 {
-    opal_mutex_unlock(m);
+    /* No destructor is needed. */
 }
 
-#endif
-
-typedef struct opal_cond_t {
-    opal_atomic_lock_t m_lock;
-    void *m_waiter_head;
-    void *m_waiter_tail;
-} opal_cond_t;
-
-#define OPAL_CONDITION_STATIC_INIT                                      \
-    {                                                                   \
-        .m_lock = OPAL_ATOMIC_LOCK_INIT,                                \
-        .m_waiter_head = NULL,                                          \
-        .m_waiter_tail = NULL,                                          \
-    }
-
-int opal_cond_init(opal_cond_t *cond);
-int opal_cond_wait(opal_cond_t *cond, opal_mutex_t *lock);
-int opal_cond_broadcast(opal_cond_t *cond);
-int opal_cond_signal(opal_cond_t *cond);
-int opal_cond_destroy(opal_cond_t *cond);
-
 END_C_DECLS
 
 #endif /* OPAL_MCA_THREADS_QTHREADS_THREADS_QTHREADS_MUTEX_H */
diff --git a/opal/mca/threads/qthreads/threads_qthreads_threads.h b/opal/mca/threads/qthreads/threads_qthreads_threads.h
index c5eebe10fc8..58630f4023a 100644
--- a/opal/mca/threads/qthreads/threads_qthreads_threads.h
+++ b/opal/mca/threads/qthreads/threads_qthreads_threads.h
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2020 High Performance Computing Center Stuttgart,
@@ -26,23 +26,12 @@
 #ifndef OPAL_MCA_THREADS_QTHREADS_THREADS_QTHREADS_THREADS_H
 #define OPAL_MCA_THREADS_QTHREADS_THREADS_QTHREADS_THREADS_H 1
 
-#include <signal.h>
 #include "opal/mca/threads/qthreads/threads_qthreads.h"
 
-struct opal_thread_t {
-    opal_object_t super;
-    opal_thread_fn_t t_run;
-    void *t_arg;
-    void *t_ret;
-    aligned_t t_thread_ret;
-    aligned_t *t_thread_ret_ptr;
-};
-
 /* Qthreads are cooperatively scheduled so yield when idle */
 #define OPAL_THREAD_YIELD_WHEN_IDLE_DEFAULT true
 
-static inline
-void opal_thread_yield(void)
+static inline void opal_thread_yield(void)
 {
     qthread_yield();
 }
diff --git a/opal/mca/threads/qthreads/threads_qthreads_tsd.h b/opal/mca/threads/qthreads/threads_qthreads_tsd.h
index 0482ed366b5..b1e0d06f3bc 100644
--- a/opal/mca/threads/qthreads/threads_qthreads_tsd.h
+++ b/opal/mca/threads/qthreads/threads_qthreads_tsd.h
@@ -23,7 +23,6 @@
  * $HEADER$
  */
 
-
 #ifndef OPAL_MCA_THREADS_QTHREADS_THREADS_QTHREADS_TSD_H
 #define OPAL_MCA_THREADS_QTHREADS_THREADS_QTHREADS_TSD_H 1
 
diff --git a/opal/mca/threads/qthreads/threads_qthreads_wait_sync.c b/opal/mca/threads/qthreads/threads_qthreads_wait_sync.c
deleted file mode 100644
index fc350263661..00000000000
--- a/opal/mca/threads/qthreads/threads_qthreads_wait_sync.c
+++ /dev/null
@@ -1,116 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2014-2016 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
- * Copyright (c) 2019      Sandia National Laboratories.  All rights reserved.
- *
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "opal/mca/threads/qthreads/threads_qthreads.h"
-#include "opal/mca/threads/wait_sync.h"
-
-static opal_mutex_t wait_sync_lock = OPAL_MUTEX_STATIC_INIT;
-static ompi_wait_sync_t *wait_sync_list = NULL;
-
-static opal_atomic_int32_t num_thread_in_progress = 0;
-
-#define WAIT_SYNC_PASS_OWNERSHIP(who)                  \
-    do {                                               \
-        opal_mutex_lock(&(who)->lock);                 \
-        opal_cond_signal(&(who)->condition);           \
-        opal_mutex_unlock(&(who)->lock);               \
-    } while (0)
-
-int ompi_sync_wait_mt(ompi_wait_sync_t *sync)
-{
-    /* Don't stop if the waiting synchronization is completed. We avoid the
-     * race condition around the release of the synchronization using the
-     * signaling field.
-     */
-    if (sync->count <= 0) {
-        return (0 == sync->status) ? OPAL_SUCCESS : OPAL_ERROR;
-    }
-
-    /* lock so nobody can signal us during the list updating */
-    opal_mutex_lock(&sync->lock);
-
-    /* Now that we hold the lock make sure another thread has not already
-     * call cond_signal.
-     */
-    if (sync->count <= 0) {
-        opal_mutex_unlock(&sync->lock);
-        return (0 == sync->status) ? OPAL_SUCCESS : OPAL_ERROR;
-    }
-
-    /* Insert sync on the list of pending synchronization constructs */
-    OPAL_THREAD_LOCK(&wait_sync_lock);
-    if (NULL == wait_sync_list) {
-        sync->next = sync->prev = sync;
-        wait_sync_list = sync;
-    } else {
-        sync->prev = wait_sync_list->prev;
-        sync->prev->next = sync;
-        sync->next = wait_sync_list;
-        wait_sync_list->prev = sync;
-    }
-    OPAL_THREAD_UNLOCK(&wait_sync_lock);
-
-    /**
-     * If we are not responsible for progressing, go silent until something
-     * worth noticing happen:
-     *  - this thread has been promoted to take care of the progress
-     *  - our sync has been triggered.
-     */
-  check_status:
-    if (sync != wait_sync_list &&
-        num_thread_in_progress >= opal_max_thread_in_progress) {
-        opal_cond_wait(&sync->condition, &sync->lock);
-
-        /**
-         * At this point either the sync was completed in which case
-         * we should remove it from the wait list, or/and I was
-         * promoted as the progress manager.
-         */
-
-        if (sync->count <= 0) { /* Completed? */
-            opal_mutex_unlock(&sync->lock);
-            goto i_am_done;
-        }
-        /* either promoted, or spurious wakeup ! */
-        goto check_status;
-    }
-    opal_mutex_unlock(&sync->lock);
-
-    OPAL_THREAD_ADD_FETCH32(&num_thread_in_progress, 1);
-    while (sync->count > 0) { /* progress till completion */
-        /* don't progress with the sync lock locked or you'll deadlock */
-        opal_progress();
-        qthread_yield();
-    }
-    OPAL_THREAD_ADD_FETCH32(&num_thread_in_progress, -1);
-
-  i_am_done:
-    /* My sync is now complete. Trim the list: remove self, wake next */
-    OPAL_THREAD_LOCK(&wait_sync_lock);
-    sync->prev->next = sync->next;
-    sync->next->prev = sync->prev;
-    /* In case I am the progress manager, pass the duties on */
-    if (sync == wait_sync_list) {
-        wait_sync_list = (sync == sync->next) ? NULL : sync->next;
-        if (NULL != wait_sync_list) {
-            WAIT_SYNC_PASS_OWNERSHIP(wait_sync_list);
-        }
-    }
-    OPAL_THREAD_UNLOCK(&wait_sync_lock);
-
-    return (0 == sync->status) ? OPAL_SUCCESS : OPAL_ERROR;
-}
diff --git a/opal/mca/threads/qthreads/threads_qthreads_wait_sync.h b/opal/mca/threads/qthreads/threads_qthreads_wait_sync.h
deleted file mode 100644
index 3e4b9d12874..00000000000
--- a/opal/mca/threads/qthreads/threads_qthreads_wait_sync.h
+++ /dev/null
@@ -1,106 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2007-2016 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2019      Sandia National Laboratories.  All rights reserved.
- *
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-
-#ifndef OPAL_MCA_THREADS_QTHREADS_THREADS_QTHREADS_WAIT_SYNC_H
-#define OPAL_MCA_THREADS_QTHREADS_THREADS_QTHREADS_WAIT_SYNC_H 1
-
-#include "opal/mca/threads/qthreads/threads_qthreads.h"
-#include "opal/mca/threads/mutex.h"
-
-typedef struct ompi_wait_sync_t {
-    opal_atomic_int32_t count;
-    int32_t status;
-    opal_cond_t condition;
-    opal_mutex_t lock;
-    struct ompi_wait_sync_t *next;
-    struct ompi_wait_sync_t *prev;
-    volatile bool signaling;
-} ompi_wait_sync_t;
-
-#define SYNC_WAIT(sync) \
-    (opal_using_threads() ? ompi_sync_wait_mt (sync) : sync_wait_st (sync))
-
-/* The loop in release handles a race condition between the signaling
- * thread and the destruction of the condition variable. The signaling
- * member will be set to false after the final signaling thread has
- * finished operating on the sync object. This is done to avoid
- * extra atomics in the signalling function and keep it as fast
- * as possible. Note that the race window is small so spinning here
- * is more optimal than sleeping since this macro is called in
- * the critical path. */
-#define WAIT_SYNC_RELEASE(sync)                       \
-    if (opal_using_threads()) {                       \
-        while ((sync)->signaling) {                   \
-            qthread_yield();                          \
-            continue;                                 \
-        }                                             \
-        opal_cond_destroy(&(sync)->condition);        \
-    }
-
-#define WAIT_SYNC_RELEASE_NOWAIT(sync)                \
-    if (opal_using_threads()) {                       \
-        opal_cond_destroy(&(sync)->condition);        \
-    }
-
-#define WAIT_SYNC_SIGNAL(sync)                        \
-    if (opal_using_threads()) {                       \
-        opal_mutex_lock(&(sync)->lock);               \
-        opal_cond_signal(&(sync)->condition);         \
-        opal_mutex_unlock(&(sync)->lock);             \
-        (sync)->signaling = false;                    \
-    }
-
-#define WAIT_SYNC_SIGNALLED(sync)                     \
-    {                                                 \
-        (sync)->signaling = false;                    \
-    }
-
-OPAL_DECLSPEC int ompi_sync_wait_mt(ompi_wait_sync_t *sync);
-static inline int sync_wait_st(ompi_wait_sync_t *sync)
-{
-    while (sync->count > 0) {
-        opal_progress();
-        qthread_yield();
-    }
-    return sync->status;
-}
-
-
-#define WAIT_SYNC_INIT(sync,c)                                  \
-    do {                                                        \
-        (sync)->count = (c);                                    \
-        (sync)->next = NULL;                                    \
-        (sync)->prev = NULL;                                    \
-        (sync)->status = 0;                                     \
-        (sync)->signaling = (0 != (c));                         \
-        if (opal_using_threads()) {                             \
-            opal_cond_init(&(sync)->condition);                 \
-            opal_mutex_create(&(sync)->lock);                   \
-        }                                                       \
-    } while (0)
-
-
-#endif /* OPAL_MCA_THREADS_QTHREADS_THREADS_QTHREADS_WAIT_SYNC_H */
diff --git a/opal/mca/threads/thread.h b/opal/mca/threads/thread.h
index 9be62eddb96..1f2de84d128 100644
--- a/opal/mca/threads/thread.h
+++ b/opal/mca/threads/thread.h
@@ -21,15 +21,13 @@
  * $HEADER$
  */
 
-
 #ifndef OPAL_MCA_THREADS_THREAD_H
 #define OPAL_MCA_THREADS_THREAD_H
 
 #include "opal_config.h"
 
-#include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
-
+#include "opal/mca/mca.h"
 
 /**
  * Structure for threads components.
@@ -43,13 +41,11 @@ struct opal_threads_base_component_1_0_0_t {
 /**
  * Convenience typedef
  */
-typedef struct opal_threads_base_component_1_0_0_t
-               opal_threads_base_component_1_0_0_t;
+typedef struct opal_threads_base_component_1_0_0_t opal_threads_base_component_1_0_0_t;
 
 /*
  * Macro for use in components that are of type threads
  */
-#define OPAL_THREADS_BASE_VERSION_1_0_0 \
-    OPAL_MCA_BASE_VERSION_2_1_0("threads", 1, 0, 0)
+#define OPAL_THREADS_BASE_VERSION_1_0_0 OPAL_MCA_BASE_VERSION_2_1_0("threads", 1, 0, 0)
 
 #endif /* OPAL_MCA_THREADS_THREAD_H */
diff --git a/opal/mca/threads/thread_usage.h b/opal/mca/threads/thread_usage.h
index 34a8c9f717f..4e2fd75a7e1 100644
--- a/opal/mca/threads/thread_usage.h
+++ b/opal/mca/threads/thread_usage.h
@@ -29,8 +29,8 @@
 
 #include "opal_config.h"
 
-#include "opal/sys/atomic.h"
 #include "opal/prefetch.h"
+#include "opal/sys/atomic.h"
 
 OPAL_DECLSPEC extern bool opal_uses_threads;
 
@@ -89,69 +89,64 @@ static inline bool opal_set_using_threads(bool have)
     return opal_using_threads();
 }
 
-
 /**
  * Use an atomic operation for increment/decrement if opal_using_threads()
  * indicates that threads are in use by the application or library.
  */
 
-#define OPAL_THREAD_DEFINE_ATOMIC_OP(type, name, operator, suffix)          \
-static inline type opal_thread_ ## name ## _fetch_ ## suffix                \
-        (opal_atomic_ ## type *addr, type delta)                            \
-{                                                                           \
-    if (OPAL_UNLIKELY(opal_using_threads())) {                              \
-        return opal_atomic_ ## name ## _fetch_ ## suffix (addr, delta);     \
-    }                                                                       \
-                                                                            \
-    *addr = *addr operator delta;                                           \
-    return *addr;                                                           \
-}                                                                           \
-                                                                            \
-static inline type opal_thread_fetch_ ## name ## _ ## suffix                \
-        (opal_atomic_ ## type *addr, type delta)                            \
-{                                                                           \
-    if (OPAL_UNLIKELY(opal_using_threads())) {                              \
-        return opal_atomic_fetch_ ## name ## _ ## suffix (addr, delta);     \
-    }                                                                       \
-                                                                            \
-    type old = *addr;                                                       \
-    *addr = old operator delta;                                             \
-    return old;                                                             \
-}
-
-#define OPAL_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(type, addr_type, suffix) \
-static inline bool opal_thread_compare_exchange_strong_ ## suffix           \
-        (opal_atomic_ ## addr_type *addr, type *compare, type value)        \
-{                                                                           \
-    if (OPAL_UNLIKELY(opal_using_threads())) {                              \
-        return opal_atomic_compare_exchange_strong_ ## suffix               \
-                (addr, (addr_type *)compare, (addr_type)value);             \
-    }                                                                       \
-                                                                            \
-    if ((type) *addr == *compare) {                                         \
-        ((type *)addr)[0] = value;                                          \
-        return true;                                                        \
-    }                                                                       \
-                                                                            \
-    *compare = ((type *)addr)[0];                                           \
-                                                                            \
-    return false;                                                           \
-}
-
-#define OPAL_THREAD_DEFINE_ATOMIC_SWAP(type, addr_type, suffix)             \
-static inline type opal_thread_swap_ ## suffix                              \
-        (opal_atomic_ ## addr_type *ptr, type newvalue)                     \
-{                                                                           \
-    if (opal_using_threads ()) {                                            \
-        return (type) opal_atomic_swap_ ## suffix                           \
-                (ptr, (addr_type) newvalue);                                \
-    }                                                                       \
-                                                                            \
-    type old = ((type *)ptr)[0];                                            \
-    ((type *)ptr)[0] = newvalue;                                            \
-                                                                            \
-    return old;                                                             \
-}
+#define OPAL_THREAD_DEFINE_ATOMIC_OP(type, name, operator, suffix)                               \
+    static inline type opal_thread_##name##_fetch_##suffix(opal_atomic_##type *addr, type delta) \
+    {                                                                                            \
+        if (OPAL_UNLIKELY(opal_using_threads())) {                                               \
+            return opal_atomic_##name##_fetch_##suffix(addr, delta);                             \
+        }                                                                                        \
+                                                                                                 \
+        *addr = *addr operator delta;                                                            \
+        return *addr;                                                                            \
+    }                                                                                            \
+                                                                                                 \
+    static inline type opal_thread_fetch_##name##_##suffix(opal_atomic_##type *addr, type delta) \
+    {                                                                                            \
+        if (OPAL_UNLIKELY(opal_using_threads())) {                                               \
+            return opal_atomic_fetch_##name##_##suffix(addr, delta);                             \
+        }                                                                                        \
+                                                                                                 \
+        type old = *addr;                                                                        \
+        *addr = old operator delta;                                                              \
+        return old;                                                                              \
+    }
+
+#define OPAL_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(type, addr_type, suffix)                        \
+    static inline bool opal_thread_compare_exchange_strong_##suffix(opal_atomic_##addr_type *addr, \
+                                                                    type *compare, type value)     \
+    {                                                                                              \
+        if (OPAL_UNLIKELY(opal_using_threads())) {                                                 \
+            return opal_atomic_compare_exchange_strong_##suffix(addr, (addr_type *) compare,       \
+                                                                (addr_type) value);                \
+        }                                                                                          \
+                                                                                                   \
+        if ((type) *addr == *compare) {                                                            \
+            ((type *) addr)[0] = value;                                                            \
+            return true;                                                                           \
+        }                                                                                          \
+                                                                                                   \
+        *compare = ((type *) addr)[0];                                                             \
+                                                                                                   \
+        return false;                                                                              \
+    }
+
+#define OPAL_THREAD_DEFINE_ATOMIC_SWAP(type, addr_type, suffix)                               \
+    static inline type opal_thread_swap_##suffix(opal_atomic_##addr_type *ptr, type newvalue) \
+    {                                                                                         \
+        if (opal_using_threads()) {                                                           \
+            return (type) opal_atomic_swap_##suffix(ptr, (addr_type) newvalue);               \
+        }                                                                                     \
+                                                                                              \
+        type old = ((type *) ptr)[0];                                                         \
+        ((type *) ptr)[0] = newvalue;                                                         \
+                                                                                              \
+        return old;                                                                           \
+    }
 
 OPAL_THREAD_DEFINE_ATOMIC_OP(int32_t, add, +, 32)
 OPAL_THREAD_DEFINE_ATOMIC_OP(size_t, add, +, size_t)
@@ -202,26 +197,19 @@ OPAL_THREAD_DEFINE_ATOMIC_SWAP(intptr_t, intptr_t, ptr)
 #define OPAL_THREAD_FETCH_SUB_SIZE_T opal_thread_fetch_sub_size_t
 #define OPAL_ATOMIC_FETCH_SUB_SIZE_T opal_thread_fetch_sub_size_t
 
-#define OPAL_THREAD_COMPARE_EXCHANGE_STRONG_32 \
-    opal_thread_compare_exchange_strong_32
-#define OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_32 \
-    opal_thread_compare_exchange_strong_32
+#define OPAL_THREAD_COMPARE_EXCHANGE_STRONG_32 opal_thread_compare_exchange_strong_32
+#define OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_32 opal_thread_compare_exchange_strong_32
 
-#define OPAL_THREAD_COMPARE_EXCHANGE_STRONG_PTR(x, y, z) \
-    opal_thread_compare_exchange_strong_ptr ((opal_atomic_intptr_t *) x, \
-                                             (intptr_t *) y, (intptr_t) z)
-#define OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR \
-        OPAL_THREAD_COMPARE_EXCHANGE_STRONG_PTR
+#define OPAL_THREAD_COMPARE_EXCHANGE_STRONG_PTR(x, y, z)                                \
+    opal_thread_compare_exchange_strong_ptr((opal_atomic_intptr_t *) x, (intptr_t *) y, \
+                                            (intptr_t) z)
+#define OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR OPAL_THREAD_COMPARE_EXCHANGE_STRONG_PTR
 
 #define OPAL_THREAD_SWAP_32 opal_thread_swap_32
 #define OPAL_ATOMIC_SWAP_32 opal_thread_swap_32
 
-#define OPAL_THREAD_SWAP_PTR(x, y) \
-    opal_thread_swap_ptr ((opal_atomic_intptr_t *) x, (intptr_t) y)
-#define OPAL_ATOMIC_SWAP_PTR OPAL_THREAD_SWAP_PTR
-
-/* define 64-bit macros is 64-bit atomic math is available */
-#if OPAL_HAVE_ATOMIC_MATH_64
+#define OPAL_THREAD_SWAP_PTR(x, y) opal_thread_swap_ptr((opal_atomic_intptr_t *) x, (intptr_t) y)
+#define OPAL_ATOMIC_SWAP_PTR       OPAL_THREAD_SWAP_PTR
 
 OPAL_THREAD_DEFINE_ATOMIC_OP(int64_t, add, +, 64)
 OPAL_THREAD_DEFINE_ATOMIC_OP(int64_t, and, &, 64)
@@ -231,52 +219,38 @@ OPAL_THREAD_DEFINE_ATOMIC_OP(int64_t, sub, -, 64)
 OPAL_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(int64_t, int64_t, 64)
 OPAL_THREAD_DEFINE_ATOMIC_SWAP(int64_t, int64_t, 64)
 
-#define OPAL_THREAD_ADD_FETCH64 opal_thread_add_fetch_64
-#define OPAL_ATOMIC_ADD_FETCH64 opal_thread_add_fetch_64
-
-#define OPAL_THREAD_AND_FETCH64 opal_thread_and_fetch_64
-#define OPAL_ATOMIC_AND_FETCH64 opal_thread_and_fetch_64
+#    define OPAL_THREAD_ADD_FETCH64 opal_thread_add_fetch_64
+#    define OPAL_ATOMIC_ADD_FETCH64 opal_thread_add_fetch_64
 
-#define OPAL_THREAD_OR_FETCH64 opal_thread_or_fetch_64
-#define OPAL_ATOMIC_OR_FETCH64 opal_thread_or_fetch_64
+#    define OPAL_THREAD_AND_FETCH64 opal_thread_and_fetch_64
+#    define OPAL_ATOMIC_AND_FETCH64 opal_thread_and_fetch_64
 
-#define OPAL_THREAD_XOR_FETCH64 opal_thread_xor_fetch_64
-#define OPAL_ATOMIC_XOR_FETCH64 opal_thread_xor_fetch_64
+#    define OPAL_THREAD_OR_FETCH64 opal_thread_or_fetch_64
+#    define OPAL_ATOMIC_OR_FETCH64 opal_thread_or_fetch_64
 
-#define OPAL_THREAD_FETCH_ADD64 opal_thread_fetch_add_64
-#define OPAL_ATOMIC_FETCH_ADD64 opal_thread_fetch_add_64
+#    define OPAL_THREAD_XOR_FETCH64 opal_thread_xor_fetch_64
+#    define OPAL_ATOMIC_XOR_FETCH64 opal_thread_xor_fetch_64
 
-#define OPAL_THREAD_FETCH_AND64 opal_thread_fetch_and_64
-#define OPAL_ATOMIC_FETCH_AND64 opal_thread_fetch_and_64
+#    define OPAL_THREAD_FETCH_ADD64 opal_thread_fetch_add_64
+#    define OPAL_ATOMIC_FETCH_ADD64 opal_thread_fetch_add_64
 
-#define OPAL_THREAD_FETCH_OR64 opal_thread_fetch_or_64
-#define OPAL_ATOMIC_FETCH_OR64 opal_thread_fetch_or_64
+#    define OPAL_THREAD_FETCH_AND64 opal_thread_fetch_and_64
+#    define OPAL_ATOMIC_FETCH_AND64 opal_thread_fetch_and_64
 
-#define OPAL_THREAD_FETCH_XOR64 opal_thread_fetch_xor_64
-#define OPAL_ATOMIC_FETCH_XOR64 opal_thread_fetch_xor_64
+#    define OPAL_THREAD_FETCH_OR64 opal_thread_fetch_or_64
+#    define OPAL_ATOMIC_FETCH_OR64 opal_thread_fetch_or_64
 
-#define OPAL_THREAD_COMPARE_EXCHANGE_STRONG_64 \
-    opal_thread_compare_exchange_strong_64
-#define OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_64 \
-    opal_thread_compare_exchange_strong_64
+#    define OPAL_THREAD_FETCH_XOR64 opal_thread_fetch_xor_64
+#    define OPAL_ATOMIC_FETCH_XOR64 opal_thread_fetch_xor_64
 
-#define OPAL_THREAD_SWAP_64 opal_thread_swap_64
-#define OPAL_ATOMIC_SWAP_64 opal_thread_swap_64
+#    define OPAL_THREAD_COMPARE_EXCHANGE_STRONG_64 opal_thread_compare_exchange_strong_64
+#    define OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_64 opal_thread_compare_exchange_strong_64
 
-#endif
+#    define OPAL_THREAD_SWAP_64 opal_thread_swap_64
+#    define OPAL_ATOMIC_SWAP_64 opal_thread_swap_64
 
 /* thread local storage */
-#if OPAL_C_HAVE__THREAD_LOCAL
-#define opal_thread_local _Thread_local
-#define OPAL_HAVE_THREAD_LOCAL 1
-
-#elif OPAL_C_HAVE___THREAD /* OPAL_C_HAVE__THREAD_LOCAL */
-#define opal_thread_local __thread
-#define OPAL_HAVE_THREAD_LOCAL 1
-#endif /* OPAL_C_HAVE___THREAD */
-
-#if !defined(OPAL_HAVE_THREAD_LOCAL)
-#define OPAL_HAVE_THREAD_LOCAL 0
-#endif /* !defined(OPAL_HAVE_THREAD_LOCAL) */
+#    define opal_thread_local      _Thread_local
+#    define OPAL_HAVE_THREAD_LOCAL 1
 
 #endif /* OPAL_MCA_THREADS_THREAD_USAGE_H */
diff --git a/opal/mca/threads/threads.h b/opal/mca/threads/threads.h
index 7d51a9c6a12..0f78830cdad 100644
--- a/opal/mca/threads/threads.h
+++ b/opal/mca/threads/threads.h
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2006 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -31,20 +31,27 @@
 
 #include "opal/class/opal_object.h"
 #if OPAL_ENABLE_DEBUG
-#include "opal/util/output.h"
+#    include "opal/util/output.h"
 #endif
 
-#include "mutex.h"
 #include "condition.h"
+#include "mutex.h"
 
 BEGIN_C_DECLS
 
 typedef void *(*opal_thread_fn_t)(opal_object_t *);
 
-#define OPAL_THREAD_CANCELLED ((void *)1);
+#define OPAL_THREAD_CANCELLED ((void *) 1);
 
 #include MCA_threads_base_include_HEADER
 
+struct opal_thread_t {
+    opal_object_t super;
+    opal_thread_fn_t t_run;
+    void *t_arg;
+    pthread_t t_handle;
+};
+
 typedef struct opal_thread_t opal_thread_t;
 
 OBJ_CLASS_DECLARATION(opal_thread_t);
@@ -53,63 +60,57 @@ OBJ_CLASS_DECLARATION(opal_thread_t);
 OPAL_DECLSPEC extern bool opal_debug_threads;
 #endif
 
-
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_thread_t);
 
 #if OPAL_ENABLE_DEBUG
-#define OPAL_ACQUIRE_THREAD(lck, cnd, act)               \
-    do {                                                 \
-        OPAL_THREAD_LOCK((lck));                         \
-        if (opal_debug_threads) {                        \
-            opal_output(0, "Waiting for thread %s:%d",   \
-                        __FILE__, __LINE__);             \
-        }                                                \
-        while (*(act)) {                                 \
-            opal_condition_wait((cnd), (lck));           \
-        }                                                \
-        if (opal_debug_threads) {                        \
-            opal_output(0, "Thread obtained %s:%d",      \
-                        __FILE__, __LINE__);             \
-        }                                                \
-        *(act) = true;                                   \
-    } while (0);
+#    define OPAL_ACQUIRE_THREAD(lck, cnd, act)                                  \
+        do {                                                                    \
+            OPAL_THREAD_LOCK((lck));                                            \
+            if (opal_debug_threads) {                                           \
+                opal_output(0, "Waiting for thread %s:%d", __FILE__, __LINE__); \
+            }                                                                   \
+            while (*(act)) {                                                    \
+                opal_condition_wait((cnd), (lck));                              \
+            }                                                                   \
+            if (opal_debug_threads) {                                           \
+                opal_output(0, "Thread obtained %s:%d", __FILE__, __LINE__);    \
+            }                                                                   \
+            *(act) = true;                                                      \
+        } while (0);
 #else
-#define OPAL_ACQUIRE_THREAD(lck, cnd, act)               \
-    do {                                                 \
-        OPAL_THREAD_LOCK((lck));                         \
-        while (*(act)) {                                 \
-            opal_condition_wait((cnd), (lck));           \
-        }                                                \
-        *(act) = true;                                   \
-    } while (0);
+#    define OPAL_ACQUIRE_THREAD(lck, cnd, act)     \
+        do {                                       \
+            OPAL_THREAD_LOCK((lck));               \
+            while (*(act)) {                       \
+                opal_condition_wait((cnd), (lck)); \
+            }                                      \
+            *(act) = true;                         \
+        } while (0);
 #endif
 
-
 #if OPAL_ENABLE_DEBUG
-#define OPAL_RELEASE_THREAD(lck, cnd, act)              \
-    do {                                                \
-        if (opal_debug_threads) {                       \
-            opal_output(0, "Releasing thread %s:%d",    \
-                        __FILE__, __LINE__);            \
-        }                                               \
-        *(act) = false;                                 \
-        opal_condition_broadcast((cnd));                \
-        OPAL_THREAD_UNLOCK((lck));                      \
-    } while (0);
+#    define OPAL_RELEASE_THREAD(lck, cnd, act)                                \
+        do {                                                                  \
+            if (opal_debug_threads) {                                         \
+                opal_output(0, "Releasing thread %s:%d", __FILE__, __LINE__); \
+            }                                                                 \
+            *(act) = false;                                                   \
+            opal_condition_broadcast((cnd));                                  \
+            OPAL_THREAD_UNLOCK((lck));                                        \
+        } while (0);
 #else
-#define OPAL_RELEASE_THREAD(lck, cnd, act)              \
-    do {                                                \
-        *(act) = false;                                 \
-        opal_condition_broadcast((cnd));                \
-        OPAL_THREAD_UNLOCK((lck));                      \
-    } while (0);
+#    define OPAL_RELEASE_THREAD(lck, cnd, act) \
+        do {                                   \
+            *(act) = false;                    \
+            opal_condition_broadcast((cnd));   \
+            OPAL_THREAD_UNLOCK((lck));         \
+        } while (0);
 #endif
 
-
-#define OPAL_WAKEUP_THREAD(cnd, act)        \
-    do {                                    \
-        *(act) = false;                     \
-        opal_condition_broadcast((cnd));    \
+#define OPAL_WAKEUP_THREAD(cnd, act)     \
+    do {                                 \
+        *(act) = false;                  \
+        opal_condition_broadcast((cnd)); \
     } while (0);
 
 /* provide a macro for forward-proofing the shifting
@@ -118,16 +119,14 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_thread_t);
 
 /* post an object to another thread - for now, we
  * only have a memory barrier */
-#define OPAL_POST_OBJECT(o)     opal_atomic_wmb()
+#define OPAL_POST_OBJECT(o) opal_atomic_wmb()
 
 /* acquire an object from another thread - for now,
  * we only have a memory barrier */
-#define OPAL_ACQUIRE_OBJECT(o)  opal_atomic_rmb()
-
-
+#define OPAL_ACQUIRE_OBJECT(o) opal_atomic_rmb()
 
-OPAL_DECLSPEC int  opal_thread_start(opal_thread_t *);
-OPAL_DECLSPEC int  opal_thread_join(opal_thread_t *, void **thread_return);
+OPAL_DECLSPEC int opal_thread_start(opal_thread_t *);
+OPAL_DECLSPEC int opal_thread_join(opal_thread_t *, void **thread_return);
 OPAL_DECLSPEC bool opal_thread_self_compare(opal_thread_t *);
 OPAL_DECLSPEC opal_thread_t *opal_thread_get_self(void);
 OPAL_DECLSPEC void opal_thread_kill(opal_thread_t *, int sig);
diff --git a/opal/mca/threads/tsd.h b/opal/mca/threads/tsd.h
index 658bfe9424c..837665a0ee2 100644
--- a/opal/mca/threads/tsd.h
+++ b/opal/mca/threads/tsd.h
@@ -14,7 +14,6 @@
  * $HEADER$
  */
 
-
 #ifndef OPAL_MCA_THREADS_TSD_H
 #define OPAL_MCA_THREADS_TSD_H
 
@@ -22,9 +21,9 @@
 
 #include <pthread.h>
 
-#include "opal/constants.h"
-#include "opal/class/opal_list.h"
 #include "mutex.h"
+#include "opal/class/opal_list.h"
+#include "opal/constants.h"
 
 BEGIN_C_DECLS
 
@@ -36,7 +35,6 @@ BEGIN_C_DECLS
  * Functions for providing thread-specific datastore capabilities.
  */
 
-
 /**
  * Prototype for callback when tsd data is being destroyed
  */
@@ -69,7 +67,6 @@ typedef void *opal_tsd_key_t;
  */
 OPAL_DECLSPEC int opal_tsd_key_delete(opal_tsd_key_t key);
 
-
 /**
  * Set a thread-specific data value
  *
@@ -89,7 +86,6 @@ OPAL_DECLSPEC int opal_tsd_key_delete(opal_tsd_key_t key);
  */
 OPAL_DECLSPEC int opal_tsd_setspecific(opal_tsd_key_t key, void *value);
 
-
 /**
  * Get a thread-specific data value
  *
@@ -109,7 +105,7 @@ OPAL_DECLSPEC int opal_tsd_getspecific(opal_tsd_key_t key, void **valuep);
 
 #else
 
-#include MCA_threads_tsd_base_include_HEADER
+#    include MCA_threads_tsd_base_include_HEADER
 
 #endif
 
@@ -136,11 +132,11 @@ void opal_tsd_tracked_key_destructor(opal_tsd_tracked_key_t *key);
 
 static inline int opal_tsd_tracked_key_get(opal_tsd_tracked_key_t *key, void **p)
 {
-    assert( NULL != key);
+    assert(NULL != key);
     *p = NULL;
 
     opal_tsd_list_item_t *tsd = NULL;
-    opal_tsd_get(key->key, (void **)&tsd); 
+    opal_tsd_get(key->key, (void **) &tsd);
     if (NULL != tsd) {
         *p = tsd->data;
     }
@@ -149,8 +145,8 @@ static inline int opal_tsd_tracked_key_get(opal_tsd_tracked_key_t *key, void **p
 }
 
 OPAL_DECLSPEC int opal_tsd_tracked_key_set(opal_tsd_tracked_key_t *key, void *p);
-OPAL_DECLSPEC void opal_tsd_tracked_key_set_destructor(
-        opal_tsd_tracked_key_t *key, opal_tsd_destructor_t destructor);
+OPAL_DECLSPEC void opal_tsd_tracked_key_set_destructor(opal_tsd_tracked_key_t *key,
+                                                       opal_tsd_destructor_t destructor);
 
 /**
  * Create thread-specific data key
@@ -177,8 +173,7 @@ OPAL_DECLSPEC void opal_tsd_tracked_key_set_destructor(
  * @retval OPAL_ERR          Error
  * @retval OPAL_ERR_IN_ERRNO Error
  */
-OPAL_DECLSPEC int opal_tsd_key_create(opal_tsd_key_t *key,
-                                      opal_tsd_destructor_t destructor);
+OPAL_DECLSPEC int opal_tsd_key_create(opal_tsd_key_t *key, opal_tsd_destructor_t destructor);
 
 END_C_DECLS
 
diff --git a/opal/mca/threads/wait_sync.h b/opal/mca/threads/wait_sync.h
index 9291a4b2e77..c77724bacbb 100644
--- a/opal/mca/threads/wait_sync.h
+++ b/opal/mca/threads/wait_sync.h
@@ -1,15 +1,23 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2020 The University of Tennessee and The University
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
- * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007-2018 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2016      Mellanox Technologies. All rights reserved.
- * Copyright (c) 2016      Research Organization for Information Science
+ * Copyright (c) 2015-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2017-2022 IBM Corporation. All rights reserved.
  * Copyright (c) 2019      Sandia National Laboratories.  All rights reserved.
+ * Copyright (c) 2021      Argonne National Laboratory.  All rights reserved.
  *
  * $COPYRIGHT$
  *
@@ -21,14 +29,108 @@
 #ifndef OPAL_MCA_THREADS_WAIT_SYNC_H
 #define OPAL_MCA_THREADS_WAIT_SYNC_H
 
-#include "opal/sys/atomic.h"
 #include "opal/mca/threads/condition.h"
+#include "opal/mca/threads/mutex.h"
+#include "opal/mca/threads/threads.h"
+#include "opal/runtime/opal_progress.h"
+#include "opal/sys/atomic.h"
 
 BEGIN_C_DECLS
 
 extern int opal_max_thread_in_progress;
 
-#include MCA_threads_wait_sync_base_include_HEADER
+typedef struct ompi_wait_sync_t {
+    opal_atomic_int32_t count;
+    int32_t status;
+    opal_thread_internal_cond_t condition;
+    opal_thread_internal_mutex_t lock;
+    struct ompi_wait_sync_t *next;
+    struct ompi_wait_sync_t *prev;
+    volatile bool signaling;
+} ompi_wait_sync_t;
+
+#define SYNC_WAIT(sync) (opal_using_threads() ? ompi_sync_wait_mt(sync) : sync_wait_st(sync))
+
+/* The loop in release handles a race condition between the signaling
+ * thread and the destruction of the condition variable. The signaling
+ * member will be set to false after the final signaling thread has
+ * finished operating on the sync object. This is done to avoid
+ * extra atomics in the signalling function and keep it as fast
+ * as possible. Note that the race window is small so spinning here
+ * is more optimal than sleeping since this macro is called in
+ * the critical path. */
+#define WAIT_SYNC_RELEASE(sync)                                \
+    if (opal_using_threads()) {                                \
+        while ((sync)->signaling) {                            \
+            if (opal_progress_yield_when_idle) {               \
+                opal_thread_yield();                           \
+            }                                                  \
+            continue;                                          \
+        }                                                      \
+        opal_thread_internal_cond_destroy(&(sync)->condition); \
+        opal_thread_internal_mutex_destroy(&(sync)->lock);     \
+    }
+
+#define WAIT_SYNC_RELEASE_NOWAIT(sync)                         \
+    if (opal_using_threads()) {                                \
+        opal_thread_internal_cond_destroy(&(sync)->condition); \
+        opal_thread_internal_mutex_destroy(&(sync)->lock);     \
+    }
+
+#define WAIT_SYNC_SIGNAL(sync)                                \
+    if (opal_using_threads()) {                               \
+        opal_thread_internal_mutex_lock(&(sync)->lock);       \
+        opal_thread_internal_cond_signal(&(sync)->condition); \
+        opal_thread_internal_mutex_unlock(&(sync)->lock);     \
+        (sync)->signaling = false;                            \
+    }
+
+#define WAIT_SYNC_SIGNALLED(sync)  \
+    {                              \
+        (sync)->signaling = false; \
+    }
+
+/* not static for inline "wait_sync_st" */
+OPAL_DECLSPEC extern ompi_wait_sync_t *opal_threads_base_wait_sync_list;
+
+OPAL_DECLSPEC int ompi_sync_wait_mt(ompi_wait_sync_t *sync);
+static inline int sync_wait_st(ompi_wait_sync_t *sync)
+{
+    assert(NULL == opal_threads_base_wait_sync_list);
+    assert(NULL == sync->next);
+    opal_threads_base_wait_sync_list = sync;
+
+    while (sync->count > 0) {
+        opal_progress();
+    }
+    opal_threads_base_wait_sync_list = NULL;
+
+    return sync->status;
+}
+
+#define WAIT_SYNC_INIT(sync, c)                                    \
+    do {                                                           \
+        (sync)->count = (c);                                       \
+        (sync)->next = NULL;                                       \
+        (sync)->prev = NULL;                                       \
+        (sync)->status = 0;                                        \
+        (sync)->signaling = (0 != (c));                            \
+        if (opal_using_threads()) {                                \
+            opal_thread_internal_cond_init(&(sync)->condition);    \
+            opal_thread_internal_mutex_init(&(sync)->lock, false); \
+        }                                                          \
+    } while (0)
+
+/**
+ * Wake up all syncs with a particular status. If status is OMPI_SUCCESS this
+ * operation is a NO-OP. Otherwise it will trigger the "error condition" from
+ * all registered sync.
+ */
+OPAL_DECLSPEC void opal_threads_base_wait_sync_global_wakeup_st(int status);
+OPAL_DECLSPEC void opal_threads_base_wait_sync_global_wakeup_mt(int status);
+#define wait_sync_global_wakeup(st) \
+    (opal_using_threads() ? opal_threads_base_wait_sync_global_wakeup_mt(st) : \
+    		                opal_threads_base_wait_sync_global_wakeup_st(st))
 
 /**
  * Update the status of the synchronization primitive. If an error is
@@ -36,8 +138,7 @@ extern int opal_max_thread_in_progress;
  * triggered. The status of the synchronization will be reported to
  * the waiting threads.
  */
-static inline void wait_sync_update(ompi_wait_sync_t *sync, int updates,
-                                    int status)
+static inline void wait_sync_update(ompi_wait_sync_t *sync, int updates, int status)
 {
     if (OPAL_LIKELY(OPAL_SUCCESS == status)) {
         if (0 != (OPAL_THREAD_ADD_FETCH32(&sync->count, -updates))) {
diff --git a/opal/mca/timer/altix/Makefile.am b/opal/mca/timer/altix/Makefile.am
deleted file mode 100644
index 8e75c15a339..00000000000
--- a/opal/mca/timer/altix/Makefile.am
+++ /dev/null
@@ -1,24 +0,0 @@
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-noinst_LTLIBRARIES = libmca_timer_altix.la
-
-libmca_timer_altix_la_SOURCES = \
-    timer_altix.h \
-    timer_altix_component.c
-
diff --git a/opal/mca/timer/altix/configure.m4 b/opal/mca/timer/altix/configure.m4
deleted file mode 100644
index 42a98afe44a..00000000000
--- a/opal/mca/timer/altix/configure.m4
+++ /dev/null
@@ -1,83 +0,0 @@
-# -*- shell-script -*-
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2015      Research Organization for Information Science
-#                         and Technology (RIST). All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-AC_DEFUN([MCA_opal_timer_altix_PRIORITY], [100])
-
-AC_DEFUN([MCA_opal_timer_altix_COMPILE_MODE], [
-    AC_MSG_CHECKING([for MCA component $2:$3 compile mode])
-    $4="static"
-    AC_MSG_RESULT([$$4])
-])
-
-AC_DEFUN([MCA_opal_timer_altix_POST_CONFIG],[
-    AS_IF([test "$1" = "1"], [timer_base_include="altix/timer_altix.h"])
-])dnl
-
-# MCA_timer_altix_CONFIG(action-if-can-compile,
-#                        [action-if-cant-compile])
-# ------------------------------------------------
-AC_DEFUN([MCA_opal_timer_altix_CONFIG],[
-    AC_CONFIG_FILES([opal/mca/timer/altix/Makefile])
-
-    AS_IF([test "$with_timer" = "altix"],
-          [timer_altix_happy="yes"
-           timer_altix_should_use=1],
-          [timer_altix_should_use=0
-           AS_IF([test "$with_timer" = ""],
-                 [timer_altix_happy="yes"],
-                 [timer_altix_happy="no"])])
-
-    AS_IF([test "$timer_altix_happy" = "yes"],
-          [AC_CHECK_HEADER([sn/mmtimer.h],
-			   [timer_altix_happy="yes"],
-			   [timer_altix_happy="no"])])
-
-    AS_IF([test "$timer_altix_happy" = "yes"],
-          [AC_CHECK_HEADERS([sys/ioctl.h sys/mman.h])])
-
-    AS_IF([test "$timer_altix_happy" = "yes"],
-          [AC_CACHE_CHECK([if MM timer can be opened],
-           	          [ompi_cv_mm_timer_mmap],
-                          [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <sys/ioctl.h>
-#include <sys/mman.h>
-#include <sn/mmtimer.h>
-], [
-    int fd;
-    fd = open(MMTIMER_FULLNAME, O_RDONLY);
-    if (-1 ==fd) return 1;])],
-                                             [ompi_cv_mm_timer_mmap="yes"],
-			                     [ompi_cv_mm_timer_mmap="no"])])
-           AS_IF([test "$ompi_cv_mm_timer_mmap" = "yes"],
-	         [timer_altix_happy="yes"],
-                 [timer_altix_happy="no"])])
-
-    AS_IF([test "$timer_altix_happy" = "no" && \
-           test "$timer_altix_should_use" = "1"],
-          [AC_MSG_ERROR([Altix timer requested but not available.  Aborting.])])
-
-    AS_IF([test "$timer_altix_happy" = "yes"],
-          [$1],
-          [$2])
-])
diff --git a/opal/mca/timer/altix/owner.txt b/opal/mca/timer/altix/owner.txt
deleted file mode 100644
index 1f73442ea02..00000000000
--- a/opal/mca/timer/altix/owner.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-#
-# owner/status file
-# owner: institution that is responsible for this package
-# status: e.g. active, maintenance, unmaintained
-#
-owner: SNL?
-status: unmaintained
diff --git a/opal/mca/timer/altix/timer_altix.h b/opal/mca/timer/altix/timer_altix.h
deleted file mode 100644
index b0161bd5c98..00000000000
--- a/opal/mca/timer/altix/timer_altix.h
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2014 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef OPAL_MCA_TIMER_ALTIX_TIMER_ALTIX_H
-#define OPAL_MCA_TIMER_ALTIX_TIMER_ALTIX_H
-
-#include "opal/sys/timer.h"
-
-extern opal_timer_t opal_timer_altix_freq;
-extern opal_timer_t opal_timer_altix_usec_conv;
-extern volatile unsigned long *opal_timer_altix_mmdev_timer_addr;
-
-static inline opal_timer_t
-opal_timer_base_get_cycles(void)
-{
-    return (*opal_timer_altix_mmdev_timer_addr);
-}
-
-
-static inline opal_timer_t
-opal_timer_base_get_usec(void)
-{
-    return opal_timer_base_get_cycles() / opal_timer_altix_usec_conv;
-}
-
-
-static inline opal_timer_t
-opal_timer_base_get_freq(void)
-{
-    return opal_timer_altix_freq;
-}
-
-
-#define OPAL_TIMER_CYCLE_NATIVE 1
-#define OPAL_TIMER_CYCLE_SUPPORTED 1
-#define OPAL_TIMER_USEC_NATIVE 0
-#define OPAL_TIMER_USEC_SUPPORTED 1
-
-#endif
diff --git a/opal/mca/timer/altix/timer_altix_component.c b/opal/mca/timer/altix/timer_altix_component.c
deleted file mode 100644
index 3f9c66a38ff..00000000000
--- a/opal/mca/timer/altix/timer_altix_component.c
+++ /dev/null
@@ -1,112 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2014      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "opal_config.h"
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <sys/ioctl.h>
-#include <sys/mman.h>
-#include <sn/mmtimer.h>
-#include <unistd.h>
-#include <errno.h>
-
-#include "opal/mca/timer/timer.h"
-#include "opal/mca/timer/altix/timer_altix.h"
-#include "opal/constants.h"
-#include "opal/util/sys_limits.h"
-
-opal_timer_t opal_timer_altix_freq;
-opal_timer_t opal_timer_altix_usec_conv;
-volatile unsigned long *opal_timer_altix_mmdev_timer_addr;
-static unsigned long *mmdev_map;
-
-static int opal_timer_altix_open(void);
-static int opal_timer_altix_close(void);
-
-const opal_timer_base_component_2_0_0_t mca_timer_altix_component = {
-    /* First, the mca_component_t struct containing meta information
-       about the component itself */
-    .timerc_version = {
-        OPAL_TIMER_BASE_VERSION_2_0_0,
-
-        /* Component name and version */
-        .mca_component_name = "altix",
-        MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
-                              OPAL_RELEASE_VERSION),
-
-        /* Component open and close functions */
-        .mca_open_component = opal_timer_altix_open,
-        .mca_close_component = opal_timer_altix_close,
-    },
-    .timerc_data = {
-        /* The component is checkpoint ready */
-        MCA_BASE_METADATA_PARAM_CHECKPOINT
-    },
-};
-
-
-static int
-opal_timer_altix_open(void)
-{
-    int fd, ret;
-    unsigned long val;
-    long offset;
-
-    fd = open(MMTIMER_FULLNAME, O_RDONLY);
-    if (fd < 0) return OPAL_ERR_NOT_FOUND;
-
-    /* make sure we can map the timer */
-    ret = ioctl(fd, MMTIMER_MMAPAVAIL, 0);
-    if (1 != ret) return OPAL_ERR_NOT_SUPPORTED;
-
-    /* find the frequency */
-    ret = ioctl(fd, MMTIMER_GETFREQ, &val);
-    if (ret == -ENOSYS) return OPAL_ERR_NOT_SUPPORTED;
-    opal_timer_altix_freq = val;
-    opal_timer_altix_usec_conv = opal_timer_altix_freq / 1000000;
-
-    /* find the address of the counter */
-    ret = ioctl(fd, MMTIMER_GETOFFSET, 0);
-    if (ret == -ENOSYS) return OPAL_ERR_NOT_SUPPORTED;
-    offset = ret;
-
-    mmdev_map = mmap(0, (size_t)opal_getpagesize(), PROT_READ, MAP_SHARED, fd, 0);
-    if (NULL == mmdev_map) return OPAL_ERR_NOT_SUPPORTED;
-    opal_timer_altix_mmdev_timer_addr = mmdev_map + offset;
-    close(fd);
-
-    return OPAL_SUCCESS;
-}
-
-
-static int
-opal_timer_altix_close(void)
-{
-    if (NULL != mmdev_map) {
-        munmap(mmdev_map, (size_t)opal_getpagesize());
-    }
-
-    return OPAL_SUCCESS;
-}
diff --git a/opal/mca/timer/base/base.h b/opal/mca/timer/base/base.h
index df1a0c3a131..189e9b0b2d2 100644
--- a/opal/mca/timer/base/base.h
+++ b/opal/mca/timer/base/base.h
@@ -25,7 +25,6 @@
 #include "opal/mca/base/mca_base_framework.h"
 #include "opal/mca/timer/timer.h"
 
-
 /*
  * Global functions for MCA overall timer open and close
  */
diff --git a/opal/mca/timer/base/timer_base_null.h b/opal/mca/timer/base/timer_base_null.h
index 89be9ad0134..f90291c2863 100644
--- a/opal/mca/timer/base/timer_base_null.h
+++ b/opal/mca/timer/base/timer_base_null.h
@@ -21,30 +21,24 @@
 
 typedef int opal_timer_t;
 
-static inline opal_timer_t
-opal_timer_base_get_cycles(void)
+static inline opal_timer_t opal_timer_base_get_cycles(void)
 {
     return 0;
 }
 
-
-static inline opal_timer_t
-opal_timer_base_get_usec(void)
+static inline opal_timer_t opal_timer_base_get_usec(void)
 {
     return 0;
 }
 
-
-static inline opal_timer_t
-opal_timer_base_get_freq(void)
+static inline opal_timer_t opal_timer_base_get_freq(void)
 {
     return 0;
 }
 
-
-#define OPAL_TIMER_CYCLE_NATIVE 0
+#define OPAL_TIMER_CYCLE_NATIVE    0
 #define OPAL_TIMER_CYCLE_SUPPORTED 0
-#define OPAL_TIMER_USEC_NATIVE 0
-#define OPAL_TIMER_USEC_SUPPORTED 0
+#define OPAL_TIMER_USEC_NATIVE     0
+#define OPAL_TIMER_USEC_SUPPORTED  0
 
 #endif
diff --git a/opal/mca/timer/base/timer_base_open.c b/opal/mca/timer/base/timer_base_open.c
index 1777ff9d356..491c23db206 100644
--- a/opal/mca/timer/base/timer_base_open.c
+++ b/opal/mca/timer/base/timer_base_open.c
@@ -10,6 +10,9 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2023      Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -17,11 +20,11 @@
  * $HEADER$
  */
 
-
 #include "opal_config.h"
 
 #include "opal/constants.h"
 #include "opal/mca/timer/base/base.h"
+#include "opal/util/timings.h"
 
 bool mca_timer_base_monotonic = true;
 
@@ -36,10 +39,26 @@ static int mca_timer_base_register(mca_base_register_flag_t flags)
 {
     (void) mca_base_var_register("opal", "timer", "require", "monotonic",
                                  "Node-level monotonic timer required (default yes)",
-                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
-                                 OPAL_INFO_LVL_5,
-                                 MCA_BASE_VAR_SCOPE_LOCAL,
-                                 &mca_timer_base_monotonic);
+                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                 MCA_BASE_VAR_SCOPE_LOCAL, &mca_timer_base_monotonic);
+
+    return OPAL_SUCCESS;
+}
+
+static int opal_timer_base_open(mca_base_open_flag_t flags)
+{
+    if (OPAL_SUCCESS != mca_base_framework_components_open(&opal_timer_base_framework, flags)) {
+        return OPAL_ERROR;
+    }
+
+    OPAL_TIMING_ENABLE_NATIVE_TIMERS;
+
+    return OPAL_SUCCESS;
+}
+
+static int opal_timer_base_close(void)
+{
+    OPAL_TIMING_DISABLE_NATIVE_TIMERS;
 
     return OPAL_SUCCESS;
 }
@@ -48,5 +67,7 @@ static int mca_timer_base_register(mca_base_register_flag_t flags)
  * Globals
  */
 /* Use default register/open/close functions */
-MCA_BASE_FRAMEWORK_DECLARE(opal, timer, "OPAL OS timer", mca_timer_base_register, NULL, NULL,
+MCA_BASE_FRAMEWORK_DECLARE(opal, timer, "OPAL OS timer", mca_timer_base_register,
+                           opal_timer_base_open,
+                           opal_timer_base_close,
                            mca_timer_base_static_components, 0);
diff --git a/opal/mca/timer/configure.m4 b/opal/mca/timer/configure.m4
index 14ec0ec9694..9bc41ea20c9 100644
--- a/opal/mca/timer/configure.m4
+++ b/opal/mca/timer/configure.m4
@@ -11,6 +11,7 @@ dnl                         University of Stuttgart.  All rights reserved.
 dnl Copyright (c) 2004-2005 The Regents of the University of California.
 dnl                         All rights reserved.
 dnl Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+dnl Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
@@ -18,6 +19,9 @@ dnl
 dnl $HEADER$
 dnl
 
+dnl need in the core library
+AC_DEFUN([MCA_opal_timer_CORE_LIB], [1])
+
 dnl we only want one :)
 m4_define(MCA_opal_timer_CONFIGURE_MODE, STOP_AT_FIRST)
 
diff --git a/opal/mca/timer/darwin/timer_darwin.h b/opal/mca/timer/darwin/timer_darwin.h
index fe35f99d5e4..cd3f219d820 100644
--- a/opal/mca/timer/darwin/timer_darwin.h
+++ b/opal/mca/timer/darwin/timer_darwin.h
@@ -33,15 +33,14 @@ OPAL_DECLSPEC extern opal_timer_t opal_timer_darwin_bias;
  * Use the pragmatic solution proposed at
  * http://stackoverflow.com/questions/23378063/how-can-i-use-mach-absolute-time-without-overflowing/23378064#23378064
  */
-static inline opal_timer_t
-opal_timer_base_get_cycles(void)
+static inline opal_timer_t opal_timer_base_get_cycles(void)
 {
     uint64_t now = mach_absolute_time();
 
-    if( opal_timer_darwin_info.denom == 0 ) {
-        (void)mach_timebase_info(&opal_timer_darwin_info);
-        if( opal_timer_darwin_info.denom > 1024 ) {
-            double frac = (double)opal_timer_darwin_info.numer/opal_timer_darwin_info.denom;
+    if (opal_timer_darwin_info.denom == 0) {
+        (void) mach_timebase_info(&opal_timer_darwin_info);
+        if (opal_timer_darwin_info.denom > 1024) {
+            double frac = (double) opal_timer_darwin_info.numer / opal_timer_darwin_info.denom;
             opal_timer_darwin_info.denom = 1024;
             opal_timer_darwin_info.numer = opal_timer_darwin_info.denom * frac + 0.5;
         }
@@ -49,28 +48,24 @@ opal_timer_base_get_cycles(void)
     }
     /* this is basically a wrapper around the "right" assembly to convert
        the tick counter off the PowerPC Time Base into nanos. */
-    return (now - opal_timer_darwin_bias) * opal_timer_darwin_info.numer / opal_timer_darwin_info.denom;
+    return (now - opal_timer_darwin_bias) * opal_timer_darwin_info.numer
+           / opal_timer_darwin_info.denom;
 }
 
-
-static inline opal_timer_t
-opal_timer_base_get_usec(void)
+static inline opal_timer_t opal_timer_base_get_usec(void)
 {
     /* freq is in Hz, so this gives usec */
     return opal_timer_base_get_cycles() / 1000;
 }
 
-
-static inline opal_timer_t
-opal_timer_base_get_freq(void)
+static inline opal_timer_t opal_timer_base_get_freq(void)
 {
     return opal_timer_darwin_freq;
 }
 
-
-#define OPAL_TIMER_CYCLE_NATIVE 0
+#define OPAL_TIMER_CYCLE_NATIVE    0
 #define OPAL_TIMER_CYCLE_SUPPORTED 1
-#define OPAL_TIMER_USEC_NATIVE 1
-#define OPAL_TIMER_USEC_SUPPORTED 1
+#define OPAL_TIMER_USEC_NATIVE     1
+#define OPAL_TIMER_USEC_SUPPORTED  1
 
 #endif
diff --git a/opal/mca/timer/darwin/timer_darwin_component.c b/opal/mca/timer/darwin/timer_darwin_component.c
index da134d2dcd6..0071c7e4b28 100644
--- a/opal/mca/timer/darwin/timer_darwin_component.c
+++ b/opal/mca/timer/darwin/timer_darwin_component.c
@@ -21,9 +21,9 @@
 
 #include "opal_config.h"
 
-#include "opal/mca/timer/timer.h"
-#include "opal/mca/timer/darwin/timer_darwin.h"
 #include "opal/constants.h"
+#include "opal/mca/timer/darwin/timer_darwin.h"
+#include "opal/mca/timer/timer.h"
 
 opal_timer_t opal_timer_darwin_freq = {0};
 mach_timebase_info_data_t opal_timer_darwin_info = {.denom = 0};
@@ -34,20 +34,20 @@ static int opal_timer_darwin_open(void);
 const opal_timer_base_component_2_0_0_t mca_timer_darwin_component = {
     /* First, the mca_component_t struct containing meta information
        about the component itself */
-    .timerc_version = {
-        OPAL_TIMER_BASE_VERSION_2_0_0,
+    .timerc_version =
+        {
+            OPAL_TIMER_BASE_VERSION_2_0_0,
 
-        /* Component name and version */
-        .mca_component_name = "darwin",
-        MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
-                              OPAL_RELEASE_VERSION),
+            /* Component name and version */
+            .mca_component_name = "darwin",
+            MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
+                                  OPAL_RELEASE_VERSION),
 
-        .mca_open_component = opal_timer_darwin_open,
-    },
-    .timerc_data = {
-        /* The component is checkpoint ready */
-        MCA_BASE_METADATA_PARAM_CHECKPOINT
-    },
+            .mca_open_component = opal_timer_darwin_open,
+        },
+    .timerc_data =
+        {/* The component is checkpoint ready */
+         MCA_BASE_METADATA_PARAM_CHECKPOINT},
 };
 
 /* mach_timebase_info() returns a fraction that can be multiplied
@@ -65,8 +65,8 @@ const opal_timer_base_component_2_0_0_t mca_timer_darwin_component = {
    arithmetic errors due to integer math.  But since we want the
    least amount of math in the critical path as possible and
    mach_absolute_time is already a cycle counter, we claim we have
-   native cycle count support and set the frequencey to be the
-   frequencey of the global clock, which is sTBI.denom *
+   native cycle count support and set the frequency to be the
+   frequency of the global clock, which is sTBI.denom *
    (1000000000 / sTBI.numer), which is sTBI.denom * (1 / 1), or
    sTBI.denom.
 
@@ -82,16 +82,17 @@ const opal_timer_base_component_2_0_0_t mca_timer_darwin_component = {
 
    More generally, since mach_timebase_info() gives the "keys" to
    transition the return from mach_absolute_time() into
-   nanoseconds, taking the reverse of that and multipling by
+   nanoseconds, taking the reverse of that and multiplying by
    1000000000 will give you a frequency in cycles / second if you
    think of mach_absolute_time() always returning a cycle count.
 */
 int opal_timer_darwin_open(void)
 {
     /* Call the opal_timer_base_get_cycles once to start the enging */
-    (void)opal_timer_base_get_cycles();
+    (void) opal_timer_base_get_cycles();
 
-    opal_timer_darwin_freq = opal_timer_darwin_info.denom * (1000000000 / opal_timer_darwin_info.numer);
+    opal_timer_darwin_freq = opal_timer_darwin_info.denom
+                             * (1000000000 / opal_timer_darwin_info.numer);
 
     return OPAL_SUCCESS;
 }
diff --git a/opal/mca/timer/linux/Makefile.am b/opal/mca/timer/linux/Makefile.am
index 98e4e6358e2..df9074b8427 100644
--- a/opal/mca/timer/linux/Makefile.am
+++ b/opal/mca/timer/linux/Makefile.am
@@ -10,6 +10,7 @@
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
 # Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -17,7 +18,7 @@
 # $HEADER$
 #
 
-dist_opaldata_DATA = help-opal-timer-linux.txt
+EXTRA_DIST = help-opal-timer-linux.txt
 
 noinst_LTLIBRARIES = libmca_timer_linux.la
 
diff --git a/opal/mca/timer/linux/timer_linux.h b/opal/mca/timer/linux/timer_linux.h
index 96fea46ce16..4d0c12ec3cc 100644
--- a/opal/mca/timer/linux/timer_linux.h
+++ b/opal/mca/timer/linux/timer_linux.h
@@ -30,9 +30,9 @@ OPAL_DECLSPEC extern opal_timer_t (*opal_timer_base_get_usec)(void);
 
 OPAL_DECLSPEC extern opal_timer_t opal_timer_base_get_freq(void);
 
-#define OPAL_TIMER_CYCLE_NATIVE OPAL_HAVE_SYS_TIMER_GET_CYCLES
+#define OPAL_TIMER_CYCLE_NATIVE    OPAL_HAVE_SYS_TIMER_GET_CYCLES
 #define OPAL_TIMER_CYCLE_SUPPORTED OPAL_HAVE_SYS_TIMER_GET_CYCLES
-#define OPAL_TIMER_USEC_NATIVE OPAL_HAVE_SYS_TIMER_GET_CYCLES
-#define OPAL_TIMER_USEC_SUPPORTED OPAL_HAVE_SYS_TIMER_GET_CYCLES
+#define OPAL_TIMER_USEC_NATIVE     OPAL_HAVE_SYS_TIMER_GET_CYCLES
+#define OPAL_TIMER_USEC_SUPPORTED  OPAL_HAVE_SYS_TIMER_GET_CYCLES
 
 #endif
diff --git a/opal/mca/timer/linux/timer_linux_component.c b/opal/mca/timer/linux/timer_linux_component.c
index a969dabc632..fabb3f4ea09 100644
--- a/opal/mca/timer/linux/timer_linux_component.c
+++ b/opal/mca/timer/linux/timer_linux_component.c
@@ -16,6 +16,8 @@
  *                         reserved.
  * Copyright (c) 2015-2017 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2016 Broadcom Limited. All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -28,10 +30,10 @@
 #include <string.h>
 #include <time.h>
 
-#include "opal/mca/timer/timer.h"
+#include "opal/constants.h"
 #include "opal/mca/timer/base/base.h"
 #include "opal/mca/timer/linux/timer_linux.h"
-#include "opal/constants.h"
+#include "opal/mca/timer/timer.h"
 #include "opal/util/show_help.h"
 
 static opal_timer_t opal_timer_linux_get_cycles_sys_timer(void);
@@ -44,16 +46,12 @@ static opal_timer_t opal_timer_linux_get_usec_sys_timer(void);
 static opal_timer_t opal_timer_linux_get_cycles_clock_gettime(void);
 static opal_timer_t opal_timer_linux_get_usec_clock_gettime(void);
 
-opal_timer_t (*opal_timer_base_get_cycles)(void) =
-    opal_timer_linux_get_cycles_clock_gettime;
-opal_timer_t (*opal_timer_base_get_usec)(void) =
-    opal_timer_linux_get_usec_clock_gettime;
+opal_timer_t (*opal_timer_base_get_cycles)(void) = opal_timer_linux_get_cycles_clock_gettime;
+opal_timer_t (*opal_timer_base_get_usec)(void) = opal_timer_linux_get_usec_clock_gettime;
 #else
-opal_timer_t (*opal_timer_base_get_cycles)(void) =
-    opal_timer_linux_get_cycles_sys_timer;
-opal_timer_t (*opal_timer_base_get_usec)(void) =
-    opal_timer_linux_get_usec_sys_timer;
-#endif  /* OPAL_HAVE_CLOCK_GETTIME */
+opal_timer_t (*opal_timer_base_get_cycles)(void) = opal_timer_linux_get_cycles_sys_timer;
+opal_timer_t (*opal_timer_base_get_usec)(void) = opal_timer_linux_get_usec_sys_timer;
+#endif /* OPAL_HAVE_CLOCK_GETTIME */
 
 static opal_timer_t opal_timer_linux_freq = {0};
 
@@ -62,25 +60,24 @@ static int opal_timer_linux_open(void);
 const opal_timer_base_component_2_0_0_t mca_timer_linux_component = {
     /* First, the mca_component_t struct containing meta information
        about the component itself */
-    .timerc_version = {
-        OPAL_TIMER_BASE_VERSION_2_0_0,
-
-        /* Component name and version */
-        .mca_component_name = "linux",
-        MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
-                              OPAL_RELEASE_VERSION),
-
-        /* Component open and close functions */
-        .mca_open_component = opal_timer_linux_open,
-    },
-    .timerc_data = {
-        /* The component is checkpoint ready */
-        MCA_BASE_METADATA_PARAM_CHECKPOINT
-    },
+    .timerc_version =
+        {
+            OPAL_TIMER_BASE_VERSION_2_0_0,
+
+            /* Component name and version */
+            .mca_component_name = "linux",
+            MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
+                                  OPAL_RELEASE_VERSION),
+
+            /* Component open and close functions */
+            .mca_open_component = opal_timer_linux_open,
+        },
+    .timerc_data =
+        {/* The component is checkpoint ready */
+         MCA_BASE_METADATA_PARAM_CHECKPOINT},
 };
 
-static char *
-find_info(FILE* fp, char *str, char *buf, size_t buflen)
+static char *find_info(FILE *fp, char *str, char *buf, size_t buflen)
 {
     char *tmp;
 
@@ -89,11 +86,13 @@ find_info(FILE* fp, char *str, char *buf, size_t buflen)
         if (strncmp(buf, str, strlen(str)) == 0) {
             /* we found the line.  Now eat everything up to,
                including, and one past the : */
-            for (tmp = buf ; (*tmp != '\0') && (*tmp != ':') ; ++tmp) ;
+            for (tmp = buf; (*tmp != '\0') && (*tmp != ':'); ++tmp)
+                ;
             if (*tmp == '\0') {
                 continue;
             }
-            for ( ++tmp ; *tmp == ' ' ; ++tmp);
+            for (++tmp; *tmp == ' '; ++tmp)
+                ;
             if ('\0' != *tmp) {
                 return tmp;
             }
@@ -135,7 +134,7 @@ static int opal_timer_linux_find_freq(void)
         }
     }
 
-#if ((OPAL_ASSEMBLY_ARCH == OPAL_IA32) || (OPAL_ASSEMBLY_ARCH == OPAL_X86_64))
+#if defined(PLATFORM_ARCH_X86) || defined(PLATFORM_ARCH_X86_64)
     if (0 == opal_timer_linux_freq && opal_sys_timer_is_monotonic()) {
         /* tsc is exposed through bogomips ~> loops_per_jiffy ~> tsc_khz */
         loc = find_info(fp, "bogomips", buf, 1024);
@@ -144,7 +143,7 @@ static int opal_timer_linux_find_freq(void)
             if (1 == ret) {
                 /* number is in MHz * 2 and has 2 decimal digits
                    convert to Hz and make an integer */
-                opal_timer_linux_freq = (opal_timer_t) (cpu_f * 100.0f) * 5000;
+                opal_timer_linux_freq = (opal_timer_t)(cpu_f * 100.0f) * 5000;
             }
         }
     }
@@ -157,7 +156,7 @@ static int opal_timer_linux_find_freq(void)
             ret = sscanf(loc, "%f", &cpu_f);
             if (1 == ret) {
                 /* numer is in MHz - convert to Hz and make an integer */
-                opal_timer_linux_freq = (opal_timer_t) (cpu_f * 1000000);
+                opal_timer_linux_freq = (opal_timer_t)(cpu_f * 1000000);
             }
         }
     }
@@ -187,10 +186,10 @@ int opal_timer_linux_open(void)
 {
     int ret = OPAL_SUCCESS;
 
-    if (mca_timer_base_monotonic && !opal_sys_timer_is_monotonic ()) {
+    if (mca_timer_base_monotonic && !opal_sys_timer_is_monotonic()) {
 #if OPAL_HAVE_CLOCK_GETTIME && (0 == OPAL_TIMER_MONOTONIC)
         struct timespec res;
-        if( 0 == clock_getres(CLOCK_MONOTONIC, &res)) {
+        if (0 == clock_getres(CLOCK_MONOTONIC, &res)) {
             opal_timer_linux_freq = 1.e3;
             opal_timer_base_get_cycles = opal_timer_linux_get_cycles_clock_gettime;
             opal_timer_base_get_usec = opal_timer_linux_get_usec_clock_gettime;
@@ -199,7 +198,7 @@ int opal_timer_linux_open(void)
 #else
         /* Monotonic time requested but cannot be found. Complain! */
         opal_show_help("help-opal-timer-linux.txt", "monotonic not supported", true);
-#endif  /* OPAL_HAVE_CLOCK_GETTIME && (0 == OPAL_TIMER_MONOTONIC) */
+#endif /* OPAL_HAVE_CLOCK_GETTIME && (0 == OPAL_TIMER_MONOTONIC) */
     }
     ret = opal_timer_linux_find_freq();
     opal_timer_base_get_cycles = opal_timer_linux_get_cycles_sys_timer;
@@ -212,9 +211,9 @@ opal_timer_t opal_timer_linux_get_usec_clock_gettime(void)
 {
     struct timespec tp = {.tv_sec = 0, .tv_nsec = 0};
 
-    (void) clock_gettime (CLOCK_MONOTONIC, &tp);
+    (void) clock_gettime(CLOCK_MONOTONIC, &tp);
 
-    return (tp.tv_sec * 1e6 + tp.tv_nsec/1000);
+    return (tp.tv_sec * 1e6 + tp.tv_nsec / 1000);
 }
 
 opal_timer_t opal_timer_linux_get_cycles_clock_gettime(void)
@@ -225,7 +224,7 @@ opal_timer_t opal_timer_linux_get_cycles_clock_gettime(void)
 
     return (tp.tv_sec * 1e9 + tp.tv_nsec);
 }
-#endif  /* OPAL_HAVE_CLOCK_GETTIME */
+#endif /* OPAL_HAVE_CLOCK_GETTIME */
 
 opal_timer_t opal_timer_linux_get_cycles_sys_timer(void)
 {
@@ -236,12 +235,11 @@ opal_timer_t opal_timer_linux_get_cycles_sys_timer(void)
 #endif
 }
 
-
 opal_timer_t opal_timer_linux_get_usec_sys_timer(void)
 {
 #if OPAL_HAVE_SYS_TIMER_GET_CYCLES
     /* freq is in MHz, so this gives usec */
-    return opal_sys_timer_get_cycles()  / opal_timer_linux_freq;
+    return opal_sys_timer_get_cycles() / opal_timer_linux_freq;
 #else
     return 0;
 #endif
diff --git a/opal/mca/timer/solaris/Makefile.am b/opal/mca/timer/solaris/Makefile.am
deleted file mode 100644
index dbe7e29396f..00000000000
--- a/opal/mca/timer/solaris/Makefile.am
+++ /dev/null
@@ -1,24 +0,0 @@
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-noinst_LTLIBRARIES = libmca_timer_solaris.la
-
-libmca_timer_solaris_la_SOURCES = \
-    timer_solaris.h \
-    timer_solaris_component.c
diff --git a/opal/mca/timer/solaris/configure.m4 b/opal/mca/timer/solaris/configure.m4
deleted file mode 100644
index 32b9893aa4d..00000000000
--- a/opal/mca/timer/solaris/configure.m4
+++ /dev/null
@@ -1,60 +0,0 @@
-# -*- shell-script -*-
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2015      Research Organization for Information Science
-#                         and Technology (RIST). All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-AC_DEFUN([MCA_opal_timer_solaris_PRIORITY], [30])
-
-AC_DEFUN([MCA_opal_timer_solaris_COMPILE_MODE], [
-    AC_MSG_CHECKING([for MCA component $2:$3 compile mode])
-    $4="static"
-    AC_MSG_RESULT([$$4])
-])
-
-AC_DEFUN([MCA_opal_timer_solaris_POST_CONFIG],[
-    AS_IF([test "$1" = "1"], [timer_base_include="solaris/timer_solaris.h"])
-])dnl
-
-# MCA_timer_solaris_CONFIG(action-if-can-compile,
-#                        [action-if-cant-compile])
-# ------------------------------------------------
-AC_DEFUN([MCA_opal_timer_solaris_CONFIG],[
-    AC_CONFIG_FILES([opal/mca/timer/solaris/Makefile])
-
-    AS_IF([test "$with_timer" = "solaris"],
-          [timer_solaris_happy="yes"
-           timer_solaris_should_use=1],
-          [timer_solaris_should_use=0
-           AS_IF([test "$with_timer" = ""],
-                 [timer_solaris_happy="yes"],
-                 [timer_solaris_happy="no"])])
-
-    AS_IF([test "$timer_solaris_happy" = "yes"],
-          [AC_CHECK_FUNC([gethrtime],
-                         [timer_solaris_happy="yes"],
-                         [timer_solaris_happy="no"])])
-
-   AS_IF([test "$timer_solaris_happy" = "no" && \
-          test "$timer_solaris_should_use" = "1"],
-         [AC_MSG_ERROR([Solaris timer requested but not available.  Aborting.])])
-
-    AS_IF([test "$timer_solaris_happy" = "yes"],
-          [$1],
-          [$2])
-])
diff --git a/opal/mca/timer/solaris/owner.txt b/opal/mca/timer/solaris/owner.txt
deleted file mode 100644
index 29b7476009c..00000000000
--- a/opal/mca/timer/solaris/owner.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-#
-# owner/status file
-# owner: institution that is responsible for this package
-# status: e.g. active, maintenance, unmaintained
-#
-owner: nobody
-status: unmaintained
diff --git a/opal/mca/timer/solaris/timer_solaris.h b/opal/mca/timer/solaris/timer_solaris.h
deleted file mode 100644
index e71cba92f73..00000000000
--- a/opal/mca/timer/solaris/timer_solaris.h
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2014 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef OPAL_MCA_TIMER_SOLARIS_TIMER_SOLARIS_H
-#define OPAL_MCA_TIMER_SOLARIS_TIMER_SOLARIS_H
-
-#include <sys/time.h>
-
-typedef hrtime_t opal_timer_t;
-
-
-static inline opal_timer_t
-opal_timer_base_get_cycles(void)
-{
-    return 0;
-}
-
-static inline opal_timer_t
-opal_timer_base_get_usec(void)
-{
-    /* gethrtime returns nanoseconds */
-    return gethrtime() / 1000;
-}
-
-static inline opal_timer_t
-opal_timer_base_get_freq(void)
-{
-    return 0;
-}
-
-
-#define OPAL_TIMER_CYCLE_NATIVE 0
-#define OPAL_TIMER_CYCLE_SUPPORTED 0
-#define OPAL_TIMER_USEC_NATIVE 1
-#define OPAL_TIMER_USEC_SUPPORTED 1
-
-#endif
diff --git a/opal/mca/timer/solaris/timer_solaris_component.c b/opal/mca/timer/solaris/timer_solaris_component.c
deleted file mode 100644
index b29eec29f90..00000000000
--- a/opal/mca/timer/solaris/timer_solaris_component.c
+++ /dev/null
@@ -1,43 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "opal_config.h"
-
-#include "opal/mca/timer/timer.h"
-#include "opal/mca/timer/solaris/timer_solaris.h"
-
-
-const opal_timer_base_component_2_0_0_t mca_timer_solaris_component = {
-    /* First, the mca_component_t struct containing meta information
-       about the component itself */
-    .timerc_version = {
-        OPAL_TIMER_BASE_VERSION_2_0_0,
-
-        /* Component name and version */
-        .mca_component_name = "solaris",
-        MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
-                              OPAL_RELEASE_VERSION),
-    },
-    .timerc_data = {
-        /* The component is checkpoint ready */
-        MCA_BASE_METADATA_PARAM_CHECKPOINT
-    },
-};
diff --git a/opal/mca/timer/timer.h b/opal/mca/timer/timer.h
index cb645f61d8a..c284019f34d 100644
--- a/opal/mca/timer/timer.h
+++ b/opal/mca/timer/timer.h
@@ -69,9 +69,8 @@
 
 #include "opal_config.h"
 
-#include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
-
+#include "opal/mca/mca.h"
 
 /**
  * Structure for timer components.
@@ -90,7 +89,6 @@ typedef struct opal_timer_base_component_2_0_0_t opal_timer_base_component_2_0_0
 /*
  * Macro for use in components that are of type timer
  */
-#define OPAL_TIMER_BASE_VERSION_2_0_0 \
-    OPAL_MCA_BASE_VERSION_2_1_0("timer", 2, 0, 0)
+#define OPAL_TIMER_BASE_VERSION_2_0_0 OPAL_MCA_BASE_VERSION_2_1_0("timer", 2, 0, 0)
 
 #endif /* OPAL_MCA_TIMER_TIMER_H */
diff --git a/opal/memoryhooks/Makefile.am b/opal/memoryhooks/Makefile.am
index 8b11687375e..8409695adf5 100644
--- a/opal/memoryhooks/Makefile.am
+++ b/opal/memoryhooks/Makefile.am
@@ -24,5 +24,5 @@ headers += \
 	memoryhooks/memory.h \
 	memoryhooks/memory_internal.h
 
-lib@OPAL_LIB_PREFIX@open_pal_la_SOURCES += \
+lib@OPAL_LIB_NAME@_la_SOURCES += \
 	memoryhooks/memory.c
diff --git a/opal/memoryhooks/memory.c b/opal/memoryhooks/memory.c
index 5504192d9b2..a74a3818041 100644
--- a/opal/memoryhooks/memory.c
+++ b/opal/memoryhooks/memory.c
@@ -12,6 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2017      Los Alamos National Security, LLC.  All rights
  *                         reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -22,19 +23,19 @@
 #include "opal_config.h"
 
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif  /* HAVE_SYS_TYPES_H */
+#    include <sys/types.h>
+#endif /* HAVE_SYS_TYPES_H */
 #ifdef HAVE_SYS_MMAN_H
-#include <sys/mman.h>
-#endif  /* HAVE_SYS_MMAN_H */
+#    include <sys/mman.h>
+#endif /* HAVE_SYS_MMAN_H */
 
+#include "opal/class/opal_list.h"
+#include "opal/class/opal_object.h"
 #include "opal/constants.h"
 #include "opal/memoryhooks/memory.h"
 #include "opal/memoryhooks/memory_internal.h"
-#include "opal/class/opal_list.h"
-#include "opal/class/opal_object.h"
-#include "opal/sys/atomic.h"
 #include "opal/runtime/opal.h"
+#include "opal/sys/atomic.h"
 
 /*
  * local types
@@ -55,6 +56,7 @@ static int hooks_support = 0;
 static opal_list_t release_cb_list;
 static opal_atomic_lock_t release_lock;
 static int release_run_callbacks;
+static int is_initialized = false;
 
 /**
  * Finalize the memory hooks subsystem
@@ -77,7 +79,7 @@ static void opal_mem_hooks_finalize(void)
     release_run_callbacks = false;
     opal_atomic_mb();
 
-    /* aquire the lock, just to make sure no one is currently
+    /* acquire the lock, just to make sure no one is currently
        twiddling with the list.  We know this won't last long, since
        no new calls will come in after we set run_callbacks to false */
     opal_atomic_lock(&release_lock);
@@ -88,39 +90,37 @@ static void opal_mem_hooks_finalize(void)
     opal_atomic_unlock(&release_lock);
 }
 
-int opal_mem_hooks_init (void)
+int opal_mem_hooks_init(void)
 {
     OBJ_CONSTRUCT(&release_cb_list, opal_list_t);
 
     opal_atomic_lock_init(&release_lock, OPAL_ATOMIC_LOCK_UNLOCKED);
+    is_initialized = true;
 
     /* delay running callbacks until there is something in the
        registration */
     release_run_callbacks = false;
     opal_atomic_mb();
 
-    opal_finalize_register_cleanup (opal_mem_hooks_finalize);
+    opal_finalize_register_cleanup(opal_mem_hooks_finalize);
 
     return OPAL_SUCCESS;
 }
 
-
-
 /* called from memory manager / memory-manager specific hooks */
-void
-opal_mem_hooks_set_support(int support)
+void opal_mem_hooks_set_support(int support)
 {
     hooks_support = support;
 }
 
-
 /* called from the memory manager / memory-manager specific hooks */
-void
-opal_mem_hooks_release_hook(void *buf, size_t length, bool from_alloc)
+void opal_mem_hooks_release_hook(void *buf, size_t length, bool from_alloc)
 {
     callback_list_item_t *cbitem, *next;
 
-    if (!release_run_callbacks) return;
+    if (!release_run_callbacks) {
+        return;
+    }
 
     /*
      * This is not really thread safe - but we can't hold the lock
@@ -133,7 +133,7 @@ opal_mem_hooks_release_hook(void *buf, size_t length, bool from_alloc)
      */
 
     opal_atomic_lock(&release_lock);
-    OPAL_LIST_FOREACH_SAFE(cbitem, next, &release_cb_list, callback_list_item_t) {
+    OPAL_LIST_FOREACH_SAFE (cbitem, next, &release_cb_list, callback_list_item_t) {
         opal_atomic_unlock(&release_lock);
         cbitem->cbfunc(buf, length, cbitem->cbdata, (bool) from_alloc);
         opal_atomic_lock(&release_lock);
@@ -141,21 +141,17 @@ opal_mem_hooks_release_hook(void *buf, size_t length, bool from_alloc)
     opal_atomic_unlock(&release_lock);
 }
 
-
-int
-opal_mem_hooks_support_level(void)
+int opal_mem_hooks_support_level(void)
 {
     return hooks_support;
 }
 
-
-int
-opal_mem_hooks_register_release(opal_mem_hooks_callback_fn_t *func, void *cbdata)
+int opal_mem_hooks_register_release(opal_mem_hooks_callback_fn_t *func, void *cbdata)
 {
     callback_list_item_t *cbitem, *new_cbitem;
     int ret = OPAL_SUCCESS;
 
-    if (0 == ((OPAL_MEMORY_FREE_SUPPORT|OPAL_MEMORY_MUNMAP_SUPPORT) & hooks_support)) {
+    if (0 == ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) & hooks_support)) {
         return OPAL_ERR_NOT_SUPPORTED;
     }
 
@@ -176,7 +172,7 @@ opal_mem_hooks_register_release(opal_mem_hooks_callback_fn_t *func, void *cbdata
     opal_atomic_mb();
 
     /* make sure the callback isn't already in the list */
-    OPAL_LIST_FOREACH(cbitem, &release_cb_list, callback_list_item_t) {
+    OPAL_LIST_FOREACH (cbitem, &release_cb_list, callback_list_item_t) {
         if (cbitem->cbfunc == func) {
             ret = OPAL_EXISTS;
             goto done;
@@ -186,9 +182,9 @@ opal_mem_hooks_register_release(opal_mem_hooks_callback_fn_t *func, void *cbdata
     new_cbitem->cbfunc = func;
     new_cbitem->cbdata = cbdata;
 
-    opal_list_append(&release_cb_list, (opal_list_item_t*) new_cbitem);
+    opal_list_append(&release_cb_list, (opal_list_item_t *) new_cbitem);
 
- done:
+done:
     opal_atomic_unlock(&release_lock);
 
     if (OPAL_EXISTS == ret && NULL != new_cbitem) {
@@ -198,18 +194,45 @@ opal_mem_hooks_register_release(opal_mem_hooks_callback_fn_t *func, void *cbdata
     return ret;
 }
 
-
-int
-opal_mem_hooks_unregister_release(opal_mem_hooks_callback_fn_t* func)
+int opal_mem_hooks_unregister_release(opal_mem_hooks_callback_fn_t *func)
 {
     callback_list_item_t *cbitem, *found_item = NULL;
     int ret = OPAL_ERR_NOT_FOUND;
 
+// I've added "is_initialized" to allow this call to be safe even if
+// a memory hooks .so was merely loaded but never used so this file's
+// init function was never called.  I'll give more context, first
+// describing a bug hit in open-shmem:
+//
+// Ordinarily the expected behavior of memhook users is they'd register a
+// callback and then deregister it in a nice matched pair.  And I think
+// most OMPI code does, but the open-shmem code isn't as clear and it
+// was loading a callback and just leaving it loaded after open-shmem was
+// unloaded.  This was a problem because upon every malloc/free/etc we're
+// going to keep trying to call their callback, and the function pointer
+// itself became illegal as soon as the open-shmem shared lib was unloaded.
+// So I figured the best solution was to add a "safety valve" to the callback
+// users where they would have a library-level destructor that un-conditionally
+// adds an extra unregister call regardless of whether the code is already
+// matched or not.
+//
+// With that happening, it's necessary to make sure
+// opal_mem_hooks_unregister_release() is safe when the system isn't
+// initialized and/or if it was initialized but the specified callback
+// is already removed.
+//
+// Note also, the reason for checking "cbitem" before looking at cbitem->cbfunc
+// is when the list is empty the OPAL_LIST_FOREACH() empirically still iterates
+// once and gives a null cbitem.  I'm not sure I like that, but that's what
+// it did so I needed to make that case safe too.
+    if (!is_initialized) {
+        return 0;
+    }
     opal_atomic_lock(&release_lock);
 
     /* make sure the callback isn't already in the list */
-    OPAL_LIST_FOREACH(cbitem, &release_cb_list, callback_list_item_t) {
-        if (cbitem->cbfunc == func) {
+    OPAL_LIST_FOREACH (cbitem, &release_cb_list, callback_list_item_t) {
+        if (cbitem && cbitem->cbfunc == func) {
             opal_list_remove_item(&release_cb_list, (opal_list_item_t *) cbitem);
             found_item = cbitem;
             ret = OPAL_SUCCESS;
diff --git a/opal/memoryhooks/memory.h b/opal/memoryhooks/memory.h
index f5dda4f5322..934bb14247a 100644
--- a/opal/memoryhooks/memory.h
+++ b/opal/memoryhooks/memory.h
@@ -46,7 +46,6 @@
 
 BEGIN_C_DECLS
 
-
 /**
  * Initialize the memory hooks subsystem
  *
@@ -62,7 +61,6 @@ BEGIN_C_DECLS
  */
 OPAL_DECLSPEC int opal_mem_hooks_init(void);
 
-
 /**
  * Query level of support provided by memory hooks
  *
@@ -71,7 +69,7 @@ OPAL_DECLSPEC int opal_mem_hooks_init(void);
  * support is provided or a bit-wise OR of the available return values
  * if support is provided.
  *
- * @retval OPAL_MEMORY_FREE_SUPPORT   Memory hooks subsytem can trigger
+ * @retval OPAL_MEMORY_FREE_SUPPORT   Memory hooks subsystem can trigger
  *                                    callback events when memory is going
  *                                    to be released by the process, either
  *                                    by the user calling an allocator
@@ -89,7 +87,6 @@ OPAL_DECLSPEC int opal_mem_hooks_init(void);
  */
 OPAL_DECLSPEC int opal_mem_hooks_support_level(void);
 
-
 /**
  * Memory status change callback
  *
@@ -99,16 +96,14 @@ OPAL_DECLSPEC int opal_mem_hooks_support_level(void);
  * opal_mem_hooks_register_release().
  *
  * @param buf     Pointer to the start of the allocation
- * @param lentgh  Length of the allocation
+ * @param length  Length of the allocation
  * @param cbdata  Data passed to memory hooks when callback
  *                was registered
  * @param from_alloc True if the callback is caused by a call to the
  *                general allocation routines (malloc, calloc, free,
  *                etc.) or directly from the user (mmap, munmap, etc.)
  */
-typedef void (opal_mem_hooks_callback_fn_t)(void *buf, size_t length,
-                                            void *cbdata, bool from_alloc);
-
+typedef void(opal_mem_hooks_callback_fn_t)(void *buf, size_t length, void *cbdata, bool from_alloc);
 
 /**
  * Register callback for when memory is to be released
@@ -126,8 +121,7 @@ typedef void (opal_mem_hooks_callback_fn_t)(void *buf, size_t length,
  * @retval OPAL_ERR_NOT_SUPPORTED There are no hooks available for
  *                      receiving callbacks when memory is to be released
  */
-OPAL_DECLSPEC int opal_mem_hooks_register_release(opal_mem_hooks_callback_fn_t *func,
-                                                  void *cbdata);
+OPAL_DECLSPEC int opal_mem_hooks_register_release(opal_mem_hooks_callback_fn_t *func, void *cbdata);
 
 /**
  * Unregister previously registered release callback
@@ -141,7 +135,6 @@ OPAL_DECLSPEC int opal_mem_hooks_register_release(opal_mem_hooks_callback_fn_t *
  */
 OPAL_DECLSPEC int opal_mem_hooks_unregister_release(opal_mem_hooks_callback_fn_t *func);
 
-
 END_C_DECLS
 
 #endif /* OPAL_MEMORY_MEMORY_H */
diff --git a/opal/memoryhooks/memory_internal.h b/opal/memoryhooks/memory_internal.h
index b812275cea7..56588303f1b 100644
--- a/opal/memoryhooks/memory_internal.h
+++ b/opal/memoryhooks/memory_internal.h
@@ -22,7 +22,7 @@
 
 #include "opal_config.h"
 #ifdef HAVE_SYS_TYPES_H
-# include <sys/types.h>
+#    include <sys/types.h>
 #endif
 
 /* see memory.h for meaning */
@@ -38,5 +38,4 @@ OPAL_DECLSPEC void opal_mem_hooks_release_hook(void *buf, size_t length, bool fr
 
 END_C_DECLS
 
-
 #endif /* OPAL_MEMORY_MEMORY_INTERNAL_H */
diff --git a/opal/runtime/Makefile.am b/opal/runtime/Makefile.am
index a0a49c082c2..a475666b37c 100644
--- a/opal/runtime/Makefile.am
+++ b/opal/runtime/Makefile.am
@@ -3,7 +3,7 @@
 # Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
 #                         University Research and Technology
 #                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2020 The University of Tennessee and The University
+# Copyright (c) 2004-2022 The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
 # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -14,6 +14,8 @@
 #                         All rights reserved.
 # Copyright (c) 2014      Intel, Inc. All rights reserved
 # Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -27,20 +29,26 @@ AM_CFLAGS = \
             -DOPAL_CONFIGURE_HOST="\"@OPAL_CONFIGURE_HOST@\""
 
 # Source code files
-dist_opaldata_DATA += runtime/help-opal-runtime.txt \
-                     runtime/help-opal_info.txt
+EXTRA_DIST += runtime/help-opal-runtime.txt \
+        runtime/help-opal_info.txt
 
 headers += \
         runtime/opal_progress.h \
         runtime/opal.h \
         runtime/opal_info_support.h \
         runtime/opal_params.h \
+        runtime/opal_params_core.h \
         runtime/opal_progress_threads.h
 
-lib@OPAL_LIB_PREFIX@open_pal_la_SOURCES += \
+libopen_pal_core_la_SOURCES += \
+        runtime/opal_params_core.c \
+        runtime/opal_finalize_core.c \
+        runtime/opal_init_core.c \
+        runtime/opal_info_support.c
+
+lib@OPAL_LIB_NAME@_la_SOURCES += \
+        runtime/opal_params.c \
         runtime/opal_progress.c \
         runtime/opal_finalize.c \
         runtime/opal_init.c \
-        runtime/opal_params.c \
-        runtime/opal_info_support.c \
         runtime/opal_progress_threads.c
diff --git a/opal/runtime/help-opal-runtime.txt b/opal/runtime/help-opal-runtime.txt
index b11b5b2370f..2adb479fec1 100644
--- a/opal/runtime/help-opal-runtime.txt
+++ b/opal/runtime/help-opal-runtime.txt
@@ -12,6 +12,8 @@
 #                         All rights reserved.
 # Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
 # Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2022      Computer Architecture and VLSI Systems (CARV)
+#                         Laboratory, ICS Forth. All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -57,3 +59,13 @@ WARNING: Cannot set both the MCA parameters opal_leave_pinned (a.k.a.,
 mpi_leave_pinned) and opal_leave_pinned_pipeline (a.k.a.,
 mpi_leave_pinned_pipeline) to "true".  Defaulting to mpi_leave_pinned
 ONLY.
+#
+[mpi-params:var_dump_color:format-error]
+WARNING: Token "%s" in opal_var_dump_color MCA parameter
+is not in the expected "key=value" format, ignoring.
+#
+[mpi-params:var_dump_color:unknown-key]
+WARNING: Key "%s" in opal_var_dump_color MCA parameter
+token "%s" is unknown, ignoring.
+
+  Valid keys are: %s
diff --git a/opal/runtime/opal.h b/opal/runtime/opal.h
index 437ea87e89a..5ccc2f6f1a0 100644
--- a/opal/runtime/opal.h
+++ b/opal/runtime/opal.h
@@ -28,8 +28,8 @@
 #define OPAL_H
 
 #include "opal_config.h"
-#include "opal/types.h"
 #include "opal/class/opal_list.h"
+#include "opal/types.h"
 #include "opal/util/proc.h"
 
 #include <assert.h>
@@ -65,7 +65,7 @@ extern opal_list_t opal_finalize_cleanup_fns;
  * \note If this function is called, opal_init_util() should *not* be
  * called.
  */
-OPAL_DECLSPEC int opal_init(int* pargc, char*** pargv);
+OPAL_DECLSPEC int opal_init(int *pargc, char ***pargv);
 
 /**
  * Finalize the OPAL layer, including the MCA system.
@@ -87,7 +87,7 @@ OPAL_DECLSPEC int opal_finalize(void);
  * \note If this function is called, opal_init() should *not*
  * be called.
  */
-OPAL_DECLSPEC int opal_init_util(int* pargc, char*** pargv);
+OPAL_DECLSPEC int opal_init_util(int *pargc, char ***pargv);
 
 /**
  * Disable PSM/PSM2 signal hijacking.
@@ -177,7 +177,7 @@ OBJ_CLASS_DECLARATION(opal_finalize_domain_t);
  * This function sets the name of a finalize domain. The domain must
  * have already been initialized by OBJ_CONSTRUCT() or OBJ_NEW().
  */
-void opal_finalize_domain_init (opal_finalize_domain_t *domain, const char *domain_name);
+void opal_finalize_domain_init(opal_finalize_domain_t *domain, const char *domain_name);
 
 /**
  * @brief Set the current finalize domain for opal_finalize_append_cleanup()
@@ -187,7 +187,7 @@ void opal_finalize_domain_init (opal_finalize_domain_t *domain, const char *doma
  * This function sets the current finalize domain. This API is not thread safe
  * and is must be protected from multi-threaded invocation.
  */
-void opal_finalize_set_domain (opal_finalize_domain_t *domain);
+void opal_finalize_set_domain(opal_finalize_domain_t *domain);
 
 /**
  * @brief Finalize a domain
@@ -199,7 +199,7 @@ void opal_finalize_set_domain (opal_finalize_domain_t *domain);
  * any memory allocated by the relevant calls to opal_finalize_append_cleanup()
  * and effectively empties the cleanup domain.
  */
-void opal_finalize_cleanup_domain (opal_finalize_domain_t *domain);
+void opal_finalize_cleanup_domain(opal_finalize_domain_t *domain);
 
 /**
  * @brief Cleanup domain function
@@ -207,7 +207,7 @@ void opal_finalize_cleanup_domain (opal_finalize_domain_t *domain);
  * The argument is optional. It is valid to use the opal_finalize_register_cleanup()
  * macro to register a function that is of type void (*) (void).
  */
-typedef void (*opal_cleanup_fn_t) (void *);
+typedef void (*opal_cleanup_fn_t)(void *);
 
 /**
  * @brief Append a cleanup function to the current domain
@@ -216,17 +216,20 @@ typedef void (*opal_cleanup_fn_t) (void *);
  * @param[in] fn_name        Name of the cleanup function (for debugging)
  * @param[in] user_data      User data to pass to the cleanup function
  */
-void opal_finalize_append_cleanup (opal_cleanup_fn_t cleanup_fn, const char *fn_name, void *user_data);
+void opal_finalize_append_cleanup(opal_cleanup_fn_t cleanup_fn, const char *fn_name,
+                                  void *user_data);
 
-#define opal_finalize_register_cleanup_3(x, y, z) opal_finalize_append_cleanup((opal_cleanup_fn_t) x, y, z)
-#define opal_finalize_register_cleanup_arg(x, y) opal_finalize_append_cleanup((opal_cleanup_fn_t) x, # x "(" #y ")", y)
-#define opal_finalize_register_cleanup(x) opal_finalize_register_cleanup_3((opal_cleanup_fn_t) (x), # x, NULL)
+#define opal_finalize_register_cleanup_3(x, y, z) \
+    opal_finalize_append_cleanup((opal_cleanup_fn_t) x, y, z)
+#define opal_finalize_register_cleanup_arg(x, y) \
+    opal_finalize_append_cleanup((opal_cleanup_fn_t) x, #x "(" #y ")", y)
+#define opal_finalize_register_cleanup(x) \
+    opal_finalize_register_cleanup_3((opal_cleanup_fn_t)(x), #x, NULL)
 
 /* opal cleanup domains */
 extern opal_finalize_domain_t opal_init_util_domain;
 extern opal_finalize_domain_t opal_init_domain;
 
-
 END_C_DECLS
 
 #endif
diff --git a/opal/runtime/opal_finalize.c b/opal/runtime/opal_finalize.c
index 665c2b43aeb..dd51b840616 100644
--- a/opal/runtime/opal_finalize.c
+++ b/opal/runtime/opal_finalize.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2022 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -32,133 +32,17 @@
 #include "opal_config.h"
 
 #include "opal/class/opal_object.h"
-#include "opal/util/output.h"
-#include "opal/util/malloc.h"
-#include "opal/util/proc.h"
-#include "opal/util/show_help.h"
-#include "opal/memoryhooks/memory.h"
-#include "opal/runtime/opal.h"
 #include "opal/constants.h"
 #include "opal/mca/threads/tsd.h"
+#include "opal/memoryhooks/memory.h"
+#include "opal/runtime/opal.h"
 #include "opal/runtime/opal_progress.h"
+#include "opal/util/malloc.h"
+#include "opal/util/output.h"
+#include "opal/util/proc.h"
+#include "opal/util/show_help.h"
 
 extern int opal_initialized;
-extern int opal_util_initialized;
-extern bool opal_init_called;
-
-static opal_mutex_t opal_finalize_cleanup_fns_lock = OPAL_MUTEX_STATIC_INIT;
-opal_list_t opal_finalize_cleanup_fns = {{0}};
-
-struct opal_cleanup_fn_item_t {
-    opal_list_item_t  super;
-    opal_cleanup_fn_t cleanup_fn;
-    void             *user_data;
-#if OPAL_ENABLE_DEBUG
-    char *cleanup_fn_name;
-#endif
-};
-
-typedef struct opal_cleanup_fn_item_t opal_cleanup_fn_item_t;
-OBJ_CLASS_DECLARATION(opal_cleanup_fn_item_t);
-
-static void opal_cleanup_fn_item_construct (opal_cleanup_fn_item_t *item)
-{
-#if OPAL_ENABLE_DEBUG
-    item->cleanup_fn_name = NULL;
-#endif
-}
-
-static void opal_cleanup_fn_item_destruct (opal_cleanup_fn_item_t *item)
-{
-#if OPAL_ENABLE_DEBUG
-    free (item->cleanup_fn_name);
-    item->cleanup_fn_name = NULL;
-#endif
-}
-
-
-OBJ_CLASS_INSTANCE(opal_cleanup_fn_item_t, opal_list_item_t,
-                   opal_cleanup_fn_item_construct, opal_cleanup_fn_item_destruct);
-
-static void opal_finalize_domain_construct (opal_finalize_domain_t *domain)
-{
-    domain->domain_name = NULL;
-}
-
-static void opal_finalize_domain_destruct (opal_finalize_domain_t *domain)
-{
-    free (domain->domain_name);
-    domain->domain_name = NULL;
-}
-
-OBJ_CLASS_INSTANCE(opal_finalize_domain_t, opal_list_t, opal_finalize_domain_construct,
-                   opal_finalize_domain_destruct);
-
-static opal_finalize_domain_t *current_finalize_domain;
-opal_finalize_domain_t opal_init_util_domain = {{{0}}};
-opal_finalize_domain_t opal_init_domain = {{{0}}};
-
-void opal_finalize_append_cleanup (opal_cleanup_fn_t cleanup_fn, const char *fn_name, void *user_data)
-{
-    opal_cleanup_fn_item_t *cleanup_item = OBJ_NEW(opal_cleanup_fn_item_t);
-    assert (NULL != cleanup_item);
-    cleanup_item->cleanup_fn = cleanup_fn;
-    cleanup_item->user_data = user_data;
-#if OPAL_ENABLE_DEBUG
-    cleanup_item->cleanup_fn_name = strdup (fn_name);
-    assert (NULL != cleanup_item->cleanup_fn_name);
-#else
-    (void) fn_name;
-#endif
-
-    opal_mutex_lock (&opal_finalize_cleanup_fns_lock);
-    opal_list_append (&current_finalize_domain->super, &cleanup_item->super);
-    opal_mutex_unlock (&opal_finalize_cleanup_fns_lock);
-}
-
-void opal_finalize_domain_init (opal_finalize_domain_t *domain, const char *domain_name)
-{
-    free (domain->domain_name);
-    domain->domain_name = domain_name ? strdup (domain_name) : NULL;
-}
-
-void opal_finalize_set_domain (opal_finalize_domain_t *domain)
-{
-    current_finalize_domain = domain;
-}
-
-void opal_finalize_cleanup_domain (opal_finalize_domain_t *domain)
-{
-    opal_cleanup_fn_item_t *cleanup_item, *next;
-    /* call any registered cleanup functions before tearing down OPAL */
-    OPAL_LIST_FOREACH_SAFE_REV(cleanup_item, next, &domain->super, opal_cleanup_fn_item_t) {
-        cleanup_item->cleanup_fn (cleanup_item->user_data);
-        opal_list_remove_item (&domain->super, &cleanup_item->super);
-        OBJ_RELEASE(cleanup_item);
-    }
-}
-
-int opal_finalize_util (void)
-{
-    if (--opal_util_initialized != 0) {
-        if (opal_util_initialized < 0) {
-            return OPAL_ERROR;
-        }
-        return OPAL_SUCCESS;
-    }
-
-    opal_finalize_cleanup_domain (&opal_init_util_domain);
-    OBJ_DESTRUCT(&opal_init_util_domain);
-
-    /* finalize the class/object system */
-    opal_class_finalize();
-
-    free (opal_process_info.nodename);
-    opal_process_info.nodename = NULL;
-
-    return OPAL_SUCCESS;
-}
-
 
 int opal_finalize(void)
 {
@@ -169,7 +53,7 @@ int opal_finalize(void)
         return OPAL_SUCCESS;
     }
 
-    opal_finalize_cleanup_domain (&opal_init_domain);
+    opal_finalize_cleanup_domain(&opal_init_domain);
     OBJ_DESTRUCT(&opal_init_domain);
 
     /* finalize libevent code */
diff --git a/opal/runtime/opal_finalize_core.c b/opal/runtime/opal_finalize_core.c
new file mode 100644
index 00000000000..c7516477d44
--- /dev/null
+++ b/opal/runtime/opal_finalize_core.c
@@ -0,0 +1,154 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2022 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008-2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2015 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2013-2017 Intel, Inc. All rights reserved.
+ * Copyright (c) 2016-2017 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * Copyright (c) 2018      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/** @file **/
+
+#include "opal_config.h"
+
+#include "opal/class/opal_object.h"
+#include "opal/constants.h"
+#include "opal/runtime/opal.h"
+#include "opal/util/output.h"
+#include "opal/util/proc.h"
+
+extern int opal_util_initialized;
+
+static opal_mutex_t opal_finalize_cleanup_fns_lock = OPAL_MUTEX_STATIC_INIT;
+opal_list_t opal_finalize_cleanup_fns = {{0}};
+
+struct opal_cleanup_fn_item_t {
+    opal_list_item_t super;
+    opal_cleanup_fn_t cleanup_fn;
+    void *user_data;
+#if OPAL_ENABLE_DEBUG
+    char *cleanup_fn_name;
+#endif
+};
+
+typedef struct opal_cleanup_fn_item_t opal_cleanup_fn_item_t;
+OBJ_CLASS_DECLARATION(opal_cleanup_fn_item_t);
+
+static void opal_cleanup_fn_item_construct(opal_cleanup_fn_item_t *item)
+{
+#if OPAL_ENABLE_DEBUG
+    item->cleanup_fn_name = NULL;
+#endif
+}
+
+static void opal_cleanup_fn_item_destruct(opal_cleanup_fn_item_t *item)
+{
+#if OPAL_ENABLE_DEBUG
+    free(item->cleanup_fn_name);
+    item->cleanup_fn_name = NULL;
+#endif
+}
+
+OBJ_CLASS_INSTANCE(opal_cleanup_fn_item_t, opal_list_item_t, opal_cleanup_fn_item_construct,
+                   opal_cleanup_fn_item_destruct);
+
+static void opal_finalize_domain_construct(opal_finalize_domain_t *domain)
+{
+    domain->domain_name = NULL;
+}
+
+static void opal_finalize_domain_destruct(opal_finalize_domain_t *domain)
+{
+    free(domain->domain_name);
+    domain->domain_name = NULL;
+}
+
+OBJ_CLASS_INSTANCE(opal_finalize_domain_t, opal_list_t, opal_finalize_domain_construct,
+                   opal_finalize_domain_destruct);
+
+static opal_finalize_domain_t *current_finalize_domain;
+opal_finalize_domain_t opal_init_util_domain = {{{0}}};
+opal_finalize_domain_t opal_init_domain = {{{0}}};
+
+void opal_finalize_append_cleanup(opal_cleanup_fn_t cleanup_fn, const char *fn_name,
+                                  void *user_data)
+{
+    opal_cleanup_fn_item_t *cleanup_item = OBJ_NEW(opal_cleanup_fn_item_t);
+    assert(NULL != cleanup_item);
+    cleanup_item->cleanup_fn = cleanup_fn;
+    cleanup_item->user_data = user_data;
+#if OPAL_ENABLE_DEBUG
+    cleanup_item->cleanup_fn_name = strdup(fn_name);
+    assert(NULL != cleanup_item->cleanup_fn_name);
+#else
+    (void) fn_name;
+#endif
+
+    opal_mutex_lock(&opal_finalize_cleanup_fns_lock);
+    opal_list_append(&current_finalize_domain->super, &cleanup_item->super);
+    opal_mutex_unlock(&opal_finalize_cleanup_fns_lock);
+}
+
+void opal_finalize_domain_init(opal_finalize_domain_t *domain, const char *domain_name)
+{
+    free(domain->domain_name);
+    domain->domain_name = domain_name ? strdup(domain_name) : NULL;
+}
+
+void opal_finalize_set_domain(opal_finalize_domain_t *domain)
+{
+    current_finalize_domain = domain;
+}
+
+void opal_finalize_cleanup_domain(opal_finalize_domain_t *domain)
+{
+    opal_cleanup_fn_item_t *cleanup_item, *next;
+    /* call any registered cleanup functions before tearing down OPAL */
+    OPAL_LIST_FOREACH_SAFE_REV (cleanup_item, next, &domain->super, opal_cleanup_fn_item_t) {
+        cleanup_item->cleanup_fn(cleanup_item->user_data);
+        opal_list_remove_item(&domain->super, &cleanup_item->super);
+        OBJ_RELEASE(cleanup_item);
+    }
+}
+
+int opal_finalize_util(void)
+{
+    if (--opal_util_initialized != 0) {
+        if (opal_util_initialized < 0) {
+            return OPAL_ERROR;
+        }
+        return OPAL_SUCCESS;
+    }
+
+    opal_finalize_cleanup_domain(&opal_init_util_domain);
+    OBJ_DESTRUCT(&opal_init_util_domain);
+
+    /* finalize the class/object system */
+    opal_class_finalize();
+
+    free(opal_process_info.nodename);
+    opal_process_info.nodename = NULL;
+
+    return OPAL_SUCCESS;
+}
+
diff --git a/opal/runtime/opal_info_support.c b/opal/runtime/opal_info_support.c
index c4bd5eee30e..e167451d67f 100644
--- a/opal/runtime/opal_info_support.c
+++ b/opal/runtime/opal_info_support.c
@@ -18,6 +18,9 @@
  * Copyright (c) 2017 IBM Corporation.  All rights reserved.
  * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2022      Computer Architecture and VLSI Systems (CARV)
+ *                         Laboratory, ICS Forth. All rights reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -27,11 +30,11 @@
 
 #include "opal_config.h"
 
-#include <string.h>
 #include <ctype.h>
+#include <string.h>
 
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 
 #include <errno.h>
@@ -39,21 +42,21 @@
 #include "opal/class/opal_list.h"
 #include "opal/class/opal_pointer_array.h"
 
-#include "opal/util/output.h"
+#include "opal/mca/base/mca_base_pvar.h"
+#include "opal/runtime/opal.h"
+#include "opal/util/argv.h"
 #include "opal/util/cmd_line.h"
 #include "opal/util/error.h"
-#include "opal/util/argv.h"
-#include "opal/util/show_help.h"
+#include "opal/util/output.h"
 #include "opal/util/printf.h"
-#include "opal/runtime/opal.h"
-#include "opal/mca/base/mca_base_pvar.h"
+#include "opal/util/show_help.h"
 
 #include "opal/include/opal/frameworks.h"
 
 #include "opal/mca/installdirs/installdirs.h"
 
-#include "opal/runtime/opal_info_support.h"
 #include "opal/mca/base/mca_base_component_repository.h"
+#include "opal/runtime/opal_info_support.h"
 
 const char *opal_info_path_prefix = "prefix";
 const char *opal_info_path_bindir = "bindir";
@@ -74,6 +77,7 @@ const char *opal_info_path_pkgdatadir = "pkgdatadir";
 const char *opal_info_path_pkgincludedir = "pkgincludedir";
 
 bool opal_info_pretty = true;
+bool opal_info_color = false;
 mca_base_register_flag_t opal_info_register_flags = MCA_BASE_REGISTER_ALL;
 
 const char *opal_info_type_all = "all";
@@ -108,16 +112,13 @@ static void component_map_destruct(opal_info_component_map_t *map)
      * list of components
      */
 }
-OBJ_CLASS_INSTANCE(opal_info_component_map_t,
-                   opal_list_item_t,
-                   component_map_construct,
+OBJ_CLASS_INSTANCE(opal_info_component_map_t, opal_list_item_t, component_map_construct,
                    component_map_destruct);
 
-static void opal_info_show_failed_component(const mca_base_component_repository_item_t* ri,
+static void opal_info_show_failed_component(const mca_base_component_repository_item_t *ri,
                                             const char *error_msg);
 
-int opal_info_init(int argc, char **argv,
-                   opal_cmd_line_t *opal_info_cmd_line)
+int opal_info_init(int argc, char **argv, opal_cmd_line_t *opal_info_cmd_line)
 {
     int ret;
     bool want_help = false;
@@ -126,34 +127,40 @@ int opal_info_init(int argc, char **argv,
 
     /* Initialize the argv parsing handle */
     if (OPAL_SUCCESS != (ret = opal_init_util(&argc, &argv))) {
-        opal_show_help("help-opal_info.txt", "lib-call-fail", true,
-                       "opal_init_util", __FILE__, __LINE__, NULL);
+        opal_show_help("help-opal_info.txt", "lib-call-fail", true, "opal_init_util", __FILE__,
+                       __LINE__, NULL);
         exit(ret);
     }
 
     /* add the cmd line options */
     opal_cmd_line_make_opt3(opal_info_cmd_line, 'V', NULL, "version", 0,
                             "Show version of Open MPI");
-    opal_cmd_line_make_opt3(opal_info_cmd_line, '\0', NULL, "param", 2,
-                            "Show MCA parameters.  The first parameter is the framework (or the keyword \"all\"); the second parameter is the specific component name (or the keyword \"all\").");
-    opal_cmd_line_make_opt3(opal_info_cmd_line, '\0', NULL, "params", 2,
-                            "Synonym for --param");
+    opal_cmd_line_make_opt3(
+        opal_info_cmd_line, '\0', NULL, "param", 2,
+        "Show MCA parameters.  The first parameter is the framework (or the keyword \"all\"); the "
+        "second parameter is the specific component name (or the keyword \"all\").");
+    opal_cmd_line_make_opt3(opal_info_cmd_line, '\0', NULL, "params", 2, "Synonym for --param");
     opal_cmd_line_make_opt3(opal_info_cmd_line, '\0', NULL, "internal", 0,
                             "Show internal MCA parameters (not meant to be modified by users)");
-    opal_cmd_line_make_opt3(opal_info_cmd_line, '\0', NULL, "path", 1,
-                            "Show paths that Open MPI was configured with.  Accepts the following parameters: prefix, bindir, libdir, incdir, mandir, pkglibdir, sysconfdir, all");
+    opal_cmd_line_make_opt3(
+        opal_info_cmd_line, '\0', NULL, "path", 1,
+        "Show paths that Open MPI was configured with.  Accepts the following parameters: prefix, "
+        "bindir, libdir, incdir, mandir, pkglibdir, sysconfdir, all");
     opal_cmd_line_make_opt3(opal_info_cmd_line, '\0', NULL, "arch", 0,
                             "Show architecture Open MPI was compiled on");
     opal_cmd_line_make_opt3(opal_info_cmd_line, 'c', NULL, "config", 0,
                             "Show configuration options");
     opal_cmd_line_make_opt3(opal_info_cmd_line, 't', NULL, "type", 1,
                             "Show internal MCA parameters with the type specified in parameter.");
-    opal_cmd_line_make_opt3(opal_info_cmd_line, 'h', NULL, "help", 0,
-                            "Show this help message");
+    opal_cmd_line_make_opt3(opal_info_cmd_line, 'h', NULL, "help", 0, "Show this help message");
     opal_cmd_line_make_opt3(opal_info_cmd_line, '\0', NULL, "pretty-print", 0,
-                            "When used in conjunction with other parameters, the output is displayed in 'pretty-print' format (default)");
+                            "When used in conjunction with other parameters, the output is "
+                            "displayed in 'pretty-print' format (default)");
+    opal_cmd_line_make_opt3(opal_info_cmd_line, '\0', NULL, "color", 1,
+                            "Control color coding: auto (default), never, always");
     opal_cmd_line_make_opt3(opal_info_cmd_line, '\0', NULL, "parsable", 0,
-                            "When used in conjunction with other parameters, the output is displayed in a machine-parsable format");
+                            "When used in conjunction with other parameters, the output is "
+                            "displayed in a machine-parsable format");
     opal_cmd_line_make_opt3(opal_info_cmd_line, '\0', NULL, "parseable", 0,
                             "Synonym for --parsable");
     opal_cmd_line_make_opt3(opal_info_cmd_line, '\0', NULL, "hostname", 0,
@@ -164,15 +171,17 @@ int opal_info_init(int argc, char **argv,
                             "Show only variables with at most this level (1-9)");
     opal_cmd_line_make_opt3(opal_info_cmd_line, 's', NULL, "selected-only", 0,
                             "Show only variables from selected components");
-    opal_cmd_line_make_opt3(opal_info_cmd_line, '\0', NULL, "show-failed", 0,
-                            "Show the components that failed to load along with the reason why they failed.");
+    opal_cmd_line_make_opt3(
+        opal_info_cmd_line, '\0', NULL, "show-failed", 0,
+        "Show the components that failed to load along with the reason why they failed.");
 
     /* set our threading level */
     opal_set_using_threads(false);
 
     /* Get MCA parameters, if any */
-    if( OPAL_SUCCESS != mca_base_open() ) {
-        opal_show_help("help-opal_info.txt", "lib-call-fail", true, "mca_base_open", __FILE__, __LINE__ );
+    if (OPAL_SUCCESS != mca_base_open()) {
+        opal_show_help("help-opal_info.txt", "lib-call-fail", true, "mca_base_open", __FILE__,
+                       __LINE__);
         opal_finalize_util();
         return OPAL_ERROR;
     }
@@ -188,19 +197,42 @@ int opal_info_init(int argc, char **argv,
     if (OPAL_SUCCESS != ret) {
         cmd_error = true;
         if (OPAL_ERR_SILENT != ret) {
-            fprintf(stderr, "%s: command line error (%s)\n", argv[0],
-                    opal_strerror(ret));
+            fprintf(stderr, "%s: command line error (%s)\n", argv[0], opal_strerror(ret));
         }
     }
-    if (!cmd_error &&
-        (opal_cmd_line_is_taken(opal_info_cmd_line, "help") ||
-         opal_cmd_line_is_taken(opal_info_cmd_line, "h"))) {
+    if (!cmd_error
+        && (opal_cmd_line_is_taken(opal_info_cmd_line, "help")
+            || opal_cmd_line_is_taken(opal_info_cmd_line, "h"))) {
+        want_help = true;
+    }
+
+    /* Determine color support */
+    if (!cmd_error && !want_help) {
+        char *color = (opal_cmd_line_is_taken(opal_info_cmd_line, "color") ?
+            opal_cmd_line_get_param(opal_info_cmd_line, "color", 0, 0) : "auto");
+
+        if (0 == strcasecmp(color, "auto")) {
+            #if HAVE_ISATTY
+                opal_info_color = isatty(STDOUT_FILENO);
+            #else
+                opal_info_color = false;
+            #endif
+        } else if (0 == strcasecmp(color, "always")) {
+            opal_info_color = true;
+        } else if (0 == strcasecmp(color, "never")) {
+            opal_info_color = false;
+        } else {
+            fprintf(stderr, "%s: Unrecognized value '%s' to color parameter\n", argv[0], color);
+            cmd_error = true;
+            want_help = true;
+        }
+    }
+
+    if(want_help) {
         char *str, *usage;
 
-        want_help = true;
         usage = opal_cmd_line_get_usage_msg(opal_info_cmd_line);
-        str = opal_show_help_string("help-opal_info.txt", "usage",
-                                    true, usage);
+        str = opal_show_help_string("help-opal_info.txt", "usage", true, usage);
         if (NULL != str) {
             printf("%s", str);
             free(str);
@@ -219,11 +251,11 @@ int opal_info_init(int argc, char **argv,
 
     mca_base_cmd_line_process_args(opal_info_cmd_line, &app_env, &global_env);
 
-
     /* set the flags */
     if (opal_cmd_line_is_taken(opal_info_cmd_line, "pretty-print")) {
         opal_info_pretty = true;
-    } else if (opal_cmd_line_is_taken(opal_info_cmd_line, "parsable") || opal_cmd_line_is_taken(opal_info_cmd_line, "parseable")) {
+    } else if (opal_cmd_line_is_taken(opal_info_cmd_line, "parsable")
+               || opal_cmd_line_is_taken(opal_info_cmd_line, "parseable")) {
         opal_info_pretty = false;
     }
 
@@ -232,7 +264,7 @@ int opal_info_init(int argc, char **argv,
         opal_info_register_flags = MCA_BASE_REGISTER_DEFAULT;
     }
 
-    if( opal_cmd_line_is_taken(opal_info_cmd_line, "show-failed") ) {
+    if (opal_cmd_line_is_taken(opal_info_cmd_line, "show-failed")) {
         mca_base_component_track_load_errors = true;
     }
 
@@ -244,7 +276,8 @@ void opal_info_finalize(void)
     opal_finalize_util();
 }
 
-static int info_register_framework (mca_base_framework_t *framework, opal_pointer_array_t *component_map)
+static int info_register_framework(mca_base_framework_t *framework,
+                                   opal_pointer_array_t *component_map)
 {
     opal_info_component_map_t *map;
     int rc;
@@ -265,19 +298,24 @@ static int info_register_framework (mca_base_framework_t *framework, opal_pointe
     return rc;
 }
 
-int opal_info_register_project_frameworks (const char *project_name, mca_base_framework_t **frameworks,
-                                           opal_pointer_array_t *component_map)
+int opal_info_register_project_frameworks(const char *project_name,
+                                          mca_base_framework_t **frameworks,
+                                          opal_pointer_array_t *component_map)
 {
-    int i, rc=OPAL_SUCCESS;
+    int i, rc = OPAL_SUCCESS;
 
-    for (i=0; NULL != frameworks[i]; i++) {
+    for (i = 0; NULL != frameworks[i]; i++) {
         if (OPAL_SUCCESS != (rc = info_register_framework(frameworks[i], component_map))) {
             if (OPAL_ERR_BAD_PARAM == rc) {
-                fprintf(stderr, "\nA \"bad parameter\" error was encountered when opening the %s %s framework\n",
+                fprintf(stderr,
+                        "\nA \"bad parameter\" error was encountered when opening the %s %s "
+                        "framework\n",
                         project_name, frameworks[i]->framework_name);
-                fprintf(stderr, "The output received from that framework includes the following parameters:\n\n");
+                fprintf(stderr, "The output received from that framework includes the following "
+                                "parameters:\n\n");
             } else if (OPAL_ERR_NOT_AVAILABLE != rc) {
-                fprintf(stderr, "%s_info_register: %s failed\n", project_name, frameworks[i]->framework_name);
+                fprintf(stderr, "%s_info_register: %s failed\n", project_name,
+                        frameworks[i]->framework_name);
                 rc = OPAL_ERROR;
             } else {
                 continue;
@@ -299,7 +337,7 @@ void opal_info_register_types(opal_pointer_array_t *mca_types)
     opal_pointer_array_add(mca_types, "opal");
 
     /* push all the types found by autogen */
-    for (i=0; NULL != opal_frameworks[i]; i++) {
+    for (i = 0; NULL != opal_frameworks[i]; i++) {
         opal_pointer_array_add(mca_types, opal_frameworks[i]->framework_name);
     }
 }
@@ -313,8 +351,9 @@ int opal_info_register_framework_params(opal_pointer_array_t *component_map)
     }
 
     /* Register mca/base parameters */
-    if( OPAL_SUCCESS != mca_base_open() ) {
-        opal_show_help("help-opal_info.txt", "lib-call-fail", true, "mca_base_open", __FILE__, __LINE__ );
+    if (OPAL_SUCCESS != mca_base_open()) {
+        opal_show_help("help-opal_info.txt", "lib-call-fail", true, "mca_base_open", __FILE__,
+                       __LINE__);
         return OPAL_ERROR;
     }
 
@@ -327,7 +366,6 @@ int opal_info_register_framework_params(opal_pointer_array_t *component_map)
     return opal_info_register_project_frameworks("opal", opal_frameworks, component_map);
 }
 
-
 void opal_info_close_components(void)
 {
     int i;
@@ -337,15 +375,14 @@ void opal_info_close_components(void)
         return;
     }
 
-    for (i=0; NULL != opal_frameworks[i]; i++) {
+    for (i = 0; NULL != opal_frameworks[i]; i++) {
         (void) mca_base_framework_close(opal_frameworks[i]);
     }
 
     /* release our reference to MCA */
-    mca_base_close ();
+    mca_base_close();
 }
 
-
 void opal_info_show_path(const char *type, const char *value)
 {
     char *pretty, *path;
@@ -423,7 +460,8 @@ void opal_info_do_path(bool want_all, opal_cmd_line_t *cmd_line)
             } else if (0 == strcmp(opal_info_path_datadir, scope)) {
                 opal_info_show_path(opal_info_path_datadir, opal_install_dirs.datadir);
             } else if (0 == strcmp(opal_info_path_sharedstatedir, scope)) {
-                opal_info_show_path(opal_info_path_sharedstatedir, opal_install_dirs.sharedstatedir);
+                opal_info_show_path(opal_info_path_sharedstatedir,
+                                    opal_install_dirs.sharedstatedir);
             } else if (0 == strcmp(opal_info_path_localstatedir, scope)) {
                 opal_info_show_path(opal_info_path_localstatedir, opal_install_dirs.localstatedir);
             } else if (0 == strcmp(opal_info_path_infodir, scope)) {
@@ -442,10 +480,8 @@ void opal_info_do_path(bool want_all, opal_cmd_line_t *cmd_line)
     }
 }
 
-void opal_info_do_params(bool want_all_in, bool want_internal,
-                         opal_pointer_array_t *mca_types,
-                         opal_pointer_array_t *component_map,
-                         opal_cmd_line_t *opal_info_cmd_line)
+void opal_info_do_params(bool want_all_in, bool want_internal, opal_pointer_array_t *mca_types,
+                         opal_pointer_array_t *component_map, opal_cmd_line_t *opal_info_cmd_line)
 {
     mca_base_var_info_lvl_t max_level = OPAL_INFO_LVL_1;
     int count;
@@ -460,15 +496,16 @@ void opal_info_do_params(bool want_all_in, bool want_internal,
     } else if (opal_cmd_line_is_taken(opal_info_cmd_line, "params")) {
         p = "params";
     } else {
-        p = "foo";  /* should never happen, but protect against segfault */
+        p = "foo"; /* should never happen, but protect against segfault */
     }
 
-    if (NULL != (str = opal_cmd_line_get_param (opal_info_cmd_line, "level", 0, 0))) {
+    if (NULL != (str = opal_cmd_line_get_param(opal_info_cmd_line, "level", 0, 0))) {
         char *tmp;
 
         errno = 0;
-        max_level = strtol (str, &tmp, 10) + OPAL_INFO_LVL_1 - 1;
-        if (0 != errno || '\0' != tmp[0] || max_level < OPAL_INFO_LVL_1 || max_level > OPAL_INFO_LVL_9) {
+        max_level = strtol(str, &tmp, 10) + OPAL_INFO_LVL_1 - 1;
+        if (0 != errno || '\0' != tmp[0] || max_level < OPAL_INFO_LVL_1
+            || max_level > OPAL_INFO_LVL_9) {
             char *usage = opal_cmd_line_get_usage_msg(opal_info_cmd_line);
             opal_show_help("help-opal_info.txt", "invalid-level", true, str);
             free(usage);
@@ -487,7 +524,7 @@ void opal_info_do_params(bool want_all_in, bool want_internal,
          */
         count = opal_cmd_line_get_ninsts(opal_info_cmd_line, p);
         for (i = 0; i < count; ++i) {
-            type = opal_cmd_line_get_param(opal_info_cmd_line, p, (int)i, 0);
+            type = opal_cmd_line_get_param(opal_info_cmd_line, p, (int) i, 0);
             if (0 == strcmp(opal_info_type_all, type)) {
                 want_all = true;
                 break;
@@ -502,18 +539,18 @@ void opal_info_do_params(bool want_all_in, bool want_internal,
                                          opal_info_component_all, opal_info_ver_full,
                                          opal_info_ver_all);
         for (i = 0; i < mca_types->size; ++i) {
-            if (NULL == (type = (char *)opal_pointer_array_get_item(mca_types, i))) {
+            if (NULL == (type = (char *) opal_pointer_array_get_item(mca_types, i))) {
                 continue;
             }
             opal_info_show_mca_params(type, opal_info_component_all, max_level, want_internal);
         }
     } else {
         for (i = 0; i < count; ++i) {
-            type = opal_cmd_line_get_param(opal_info_cmd_line, p, (int)i, 0);
-            component = opal_cmd_line_get_param(opal_info_cmd_line, p, (int)i, 1);
+            type = opal_cmd_line_get_param(opal_info_cmd_line, p, (int) i, 0);
+            component = opal_cmd_line_get_param(opal_info_cmd_line, p, (int) i, 1);
 
             for (found = false, i = 0; i < mca_types->size; ++i) {
-                if (NULL == (str = (char *)opal_pointer_array_get_item(mca_types, i))) {
+                if (NULL == (str = (char *) opal_pointer_array_get_item(mca_types, i))) {
                     continue;
                 }
                 if (0 == strcmp(str, type)) {
@@ -529,9 +566,8 @@ void opal_info_do_params(bool want_all_in, bool want_internal,
                 exit(1);
             }
 
-            opal_info_show_component_version(mca_types, component_map, type,
-                                             component, opal_info_ver_full,
-                                             opal_info_ver_all);
+            opal_info_show_component_version(mca_types, component_map, type, component,
+                                             opal_info_ver_full, opal_info_ver_all);
             opal_info_show_mca_params(type, component, max_level, want_internal);
         }
     }
@@ -539,14 +575,16 @@ void opal_info_do_params(bool want_all_in, bool want_internal,
 
 void opal_info_err_params(opal_pointer_array_t *component_map)
 {
-    opal_info_component_map_t *map=NULL, *mptr;
+    opal_info_component_map_t *map = NULL, *mptr;
     int i;
 
     /* all we want to do is display the LAST entry in the
      * component_map array as this is the one that generated the error
      */
-    for (i=0; i < component_map->size; i++) {
-        if (NULL == (mptr = (opal_info_component_map_t*)opal_pointer_array_get_item(component_map, i))) {
+    for (i = 0; i < component_map->size; i++) {
+        if (NULL
+            == (mptr = (opal_info_component_map_t *) opal_pointer_array_get_item(component_map,
+                                                                                 i))) {
             continue;
         }
         map = mptr;
@@ -568,16 +606,17 @@ void opal_info_do_type(opal_cmd_line_t *opal_info_cmd_line)
     int i, j, k, len, ret;
     char *p;
     const mca_base_var_t *var;
-    char** strings, *message;
+    char **strings, *message;
     const mca_base_var_group_t *group;
     p = "type";
 
-    if (NULL != (str = opal_cmd_line_get_param (opal_info_cmd_line, "level", 0, 0))) {
+    if (NULL != (str = opal_cmd_line_get_param(opal_info_cmd_line, "level", 0, 0))) {
         char *tmp;
 
         errno = 0;
-        max_level = strtol (str, &tmp, 10) + OPAL_INFO_LVL_1 - 1;
-        if (0 != errno || '\0' != tmp[0] || max_level < OPAL_INFO_LVL_1 || max_level > OPAL_INFO_LVL_9) {
+        max_level = strtol(str, &tmp, 10) + OPAL_INFO_LVL_1 - 1;
+        if (0 != errno || '\0' != tmp[0] || max_level < OPAL_INFO_LVL_1
+            || max_level > OPAL_INFO_LVL_9) {
             char *usage = opal_cmd_line_get_usage_msg(opal_info_cmd_line);
             opal_show_help("help-opal_info.txt", "invalid-level", true, str);
             free(usage);
@@ -586,24 +625,30 @@ void opal_info_do_type(opal_cmd_line_t *opal_info_cmd_line)
     }
 
     count = opal_cmd_line_get_ninsts(opal_info_cmd_line, p);
-    len = mca_base_var_get_count ();
+    len = mca_base_var_get_count();
 
     for (k = 0; k < count; ++k) {
         type = opal_cmd_line_get_param(opal_info_cmd_line, p, k, 0);
         for (i = 0; i < len; ++i) {
-            ret = mca_base_var_get (i, &var);
+            ret = mca_base_var_get(i, &var);
             if (OPAL_SUCCESS != ret) {
                 continue;
             }
-            if (0 == strcmp(type, ompi_var_type_names[var->mbv_type]) && (var->mbv_info_lvl <= max_level)) {
-                ret = mca_base_var_dump(var->mbv_index, &strings, !opal_info_pretty ? MCA_BASE_VAR_DUMP_PARSABLE : MCA_BASE_VAR_DUMP_READABLE);
+            if (0 == strcmp(type, ompi_var_type_names[var->mbv_type])
+                && (var->mbv_info_lvl <= max_level)) {
+
+                mca_base_var_dump_type_t dump_type = (!opal_info_pretty ? MCA_BASE_VAR_DUMP_PARSABLE
+                   : (opal_info_color ? MCA_BASE_VAR_DUMP_READABLE_COLOR : MCA_BASE_VAR_DUMP_READABLE));
+
+                ret = mca_base_var_dump(var->mbv_index, &strings, dump_type);
+
                 if (OPAL_SUCCESS != ret) {
                     continue;
                 }
                 (void) mca_base_var_group_get(var->mbv_group_index, &group);
-                for (j = 0 ; strings[j] ; ++j) {
+                for (j = 0; strings[j]; ++j) {
                     if (0 == j && opal_info_pretty) {
-                        opal_asprintf (&message, "MCA %s", group->group_framework);
+                        opal_asprintf(&message, "MCA %s", group->group_framework);
                         opal_info_out(message, message, strings[j]);
                         free(message);
                     } else {
@@ -617,7 +662,8 @@ void opal_info_do_type(opal_cmd_line_t *opal_info_cmd_line)
     }
 }
 
-static void opal_info_show_mca_group_params(const mca_base_var_group_t *group, mca_base_var_info_lvl_t max_level, bool want_internal)
+static void opal_info_show_mca_group_params(const mca_base_var_group_t *group,
+                                            mca_base_var_info_lvl_t max_level, bool want_internal)
 {
     const int *variables, *groups;
     const mca_base_pvar_t *pvar;
@@ -628,35 +674,36 @@ static void opal_info_show_mca_group_params(const mca_base_var_group_t *group, m
     int ret, i, j, count;
 
     variables = OPAL_VALUE_ARRAY_GET_BASE(&group->group_vars, const int);
-    count = opal_value_array_get_size((opal_value_array_t *)&group->group_vars);
+    count = opal_value_array_get_size((opal_value_array_t *) &group->group_vars);
 
     /* the default component name is "base". depending on how the
      * group was registered the group may or not have this set.  */
     group_component = group->group_component ? group->group_component : "base";
 
     /* check if this group may be disabled due to a selection variable */
-    if (0 != strcmp (group_component, "base")) {
+    if (0 != strcmp(group_component, "base")) {
         int var_id;
 
         /* read the selection parameter */
-        var_id = mca_base_var_find (group->group_project, group->group_framework, NULL, NULL);
+        var_id = mca_base_var_find(group->group_project, group->group_framework, NULL, NULL);
         if (0 <= var_id) {
-            const mca_base_var_storage_t *value=NULL;
+            const mca_base_var_storage_t *value = NULL;
             char **requested_components;
             bool include_mode;
 
-            mca_base_var_get_value (var_id, &value, NULL, NULL);
+            mca_base_var_get_value(var_id, &value, NULL, NULL);
             if (NULL != value && NULL != value->stringval && '\0' != value->stringval[0]) {
-                mca_base_component_parse_requested (value->stringval, &include_mode, &requested_components);
+                mca_base_component_parse_requested(value->stringval, &include_mode,
+                                                   &requested_components);
 
-                for (i = 0, requested = !include_mode ; requested_components[i] ; ++i) {
-                    if (0 == strcmp (requested_components[i], group_component)) {
+                for (i = 0, requested = !include_mode; requested_components[i]; ++i) {
+                    if (0 == strcmp(requested_components[i], group_component)) {
                         requested = include_mode;
                         break;
                     }
                 }
 
-                opal_argv_free (requested_components);
+                opal_argv_free(requested_components);
             }
         }
     }
@@ -665,33 +712,34 @@ static void opal_info_show_mca_group_params(const mca_base_var_group_t *group, m
     char *component_msg = NULL;
     opal_asprintf(&component_msg, " %s", group_component);
 
-    for (i = 0 ; i < count ; ++i) {
+    for (i = 0; i < count; ++i) {
         ret = mca_base_var_get(variables[i], &var);
-        if (OPAL_SUCCESS != ret || ((var->mbv_flags & MCA_BASE_VAR_FLAG_INTERNAL) &&
-                                    !want_internal) ||
-            max_level < var->mbv_info_lvl) {
+        if (OPAL_SUCCESS != ret || ((var->mbv_flags & MCA_BASE_VAR_FLAG_INTERNAL) && !want_internal)
+            || max_level < var->mbv_info_lvl) {
             continue;
         }
 
         if (opal_info_pretty && curr_group != group) {
-            opal_asprintf(&message, "MCA%s %s%s", requested ? "" : " (-)",
-                     group->group_framework,
-                     component_msg ? component_msg : "");
+            opal_asprintf(&message, "MCA%s %s%s", requested ? "" : " (-)", group->group_framework,
+                          component_msg ? component_msg : "");
             opal_info_out(message, message, "---------------------------------------------------");
             free(message);
             curr_group = group;
         }
 
-        ret = mca_base_var_dump(variables[i], &strings, !opal_info_pretty ? MCA_BASE_VAR_DUMP_PARSABLE : MCA_BASE_VAR_DUMP_READABLE);
+        mca_base_var_dump_type_t dump_type = (!opal_info_pretty ? MCA_BASE_VAR_DUMP_PARSABLE
+            : (opal_info_color ? MCA_BASE_VAR_DUMP_READABLE_COLOR : MCA_BASE_VAR_DUMP_READABLE));
+
+        ret = mca_base_var_dump(variables[i], &strings, dump_type);
+
         if (OPAL_SUCCESS != ret) {
             continue;
         }
 
-        for (j = 0 ; strings[j] ; ++j) {
+        for (j = 0; strings[j]; ++j) {
             if (0 == j && opal_info_pretty) {
-                opal_asprintf (&message, "MCA%s %s%s", requested ? "" : " (-)",
-                          group->group_framework,
-                          component_msg ? component_msg : "");
+                opal_asprintf(&message, "MCA%s %s%s", requested ? "" : " (-)",
+                              group->group_framework, component_msg ? component_msg : "");
                 opal_info_out(message, message, strings[j]);
                 free(message);
             } else {
@@ -702,42 +750,42 @@ static void opal_info_show_mca_group_params(const mca_base_var_group_t *group, m
         if (!opal_info_pretty) {
             /* generate an entry indicating whether this variable is disabled or not. if the
              * format in mca_base_var/pvar.c changes this needs to be changed as well */
-            opal_asprintf (&message, "mca:%s:%s:param:%s:disabled:%s", group->group_framework,
-                      group_component, var->mbv_full_name, requested ? "false" : "true");
+            opal_asprintf(&message, "mca:%s:%s:param:%s:disabled:%s", group->group_framework,
+                          group_component, var->mbv_full_name, requested ? "false" : "true");
             opal_info_out("", "", message);
-            free (message);
+            free(message);
         }
         free(strings);
     }
 
     variables = OPAL_VALUE_ARRAY_GET_BASE(&group->group_pvars, const int);
-    count = opal_value_array_get_size((opal_value_array_t *)&group->group_pvars);
+    count = opal_value_array_get_size((opal_value_array_t *) &group->group_pvars);
 
-    for (i = 0 ; i < count ; ++i) {
+    for (i = 0; i < count; ++i) {
         ret = mca_base_pvar_get(variables[i], &pvar);
         if (OPAL_SUCCESS != ret || max_level < pvar->verbosity) {
             continue;
         }
 
         if (opal_info_pretty && curr_group != group) {
-            opal_asprintf(&message, "MCA%s %s%s", requested ? "" : " (-)",
-                     group->group_framework,
-                     component_msg ? component_msg : "");
+            opal_asprintf(&message, "MCA%s %s%s", requested ? "" : " (-)", group->group_framework,
+                          component_msg ? component_msg : "");
             opal_info_out(message, message, "---------------------------------------------------");
             free(message);
             curr_group = group;
         }
 
-        ret = mca_base_pvar_dump (variables[i], &strings, !opal_info_pretty ? MCA_BASE_VAR_DUMP_PARSABLE : MCA_BASE_VAR_DUMP_READABLE);
+        ret = mca_base_pvar_dump(variables[i], &strings,
+                                 !opal_info_pretty ? MCA_BASE_VAR_DUMP_PARSABLE
+                                                   : MCA_BASE_VAR_DUMP_READABLE);
         if (OPAL_SUCCESS != ret) {
             continue;
         }
 
-        for (j = 0 ; strings[j] ; ++j) {
+        for (j = 0; strings[j]; ++j) {
             if (0 == j && opal_info_pretty) {
-                opal_asprintf (&message, "MCA%s %s%s", requested ? "" : " (-)",
-                          group->group_framework,
-                          component_msg ? component_msg : "");
+                opal_asprintf(&message, "MCA%s %s%s", requested ? "" : " (-)",
+                              group->group_framework, component_msg ? component_msg : "");
                 opal_info_out(message, message, strings[j]);
                 free(message);
             } else {
@@ -748,18 +796,18 @@ static void opal_info_show_mca_group_params(const mca_base_var_group_t *group, m
         if (!opal_info_pretty) {
             /* generate an entry indicating whether this variable is disabled or not. if the
              * format in mca_base_var/pvar.c changes this needs to be changed as well */
-            opal_asprintf (&message, "mca:%s:%s:pvar:%s:disabled:%s", group->group_framework,
-                      group_component, pvar->name, requested ? "false" : "true");
+            opal_asprintf(&message, "mca:%s:%s:pvar:%s:disabled:%s", group->group_framework,
+                          group_component, pvar->name, requested ? "false" : "true");
             opal_info_out("", "", message);
-            free (message);
+            free(message);
         }
         free(strings);
     }
 
     groups = OPAL_VALUE_ARRAY_GET_BASE(&group->group_subgroups, const int);
-    count = opal_value_array_get_size((opal_value_array_t *)&group->group_subgroups);
+    count = opal_value_array_get_size((opal_value_array_t *) &group->group_subgroups);
 
-    for (i = 0 ; i < count ; ++i) {
+    for (i = 0; i < count; ++i) {
         ret = mca_base_var_group_get(groups[i], &group);
         if (OPAL_SUCCESS != ret) {
             continue;
@@ -775,7 +823,7 @@ void opal_info_show_mca_params(const char *type, const char *component,
     const mca_base_var_group_t *group;
     int ret;
 
-    if (0 == strcmp (component, "all")) {
+    if (0 == strcmp(component, "all")) {
         ret = mca_base_var_group_find("*", type, NULL);
         if (0 > ret) {
             return;
@@ -795,20 +843,16 @@ void opal_info_show_mca_params(const char *type, const char *component,
     }
 }
 
-
-
-void opal_info_do_arch()
+void opal_info_do_arch(void)
 {
     opal_info_out("Configured architecture", "config:arch", OPAL_ARCH);
 }
 
-
-void opal_info_do_hostname()
+void opal_info_do_hostname(void)
 {
     opal_info_out("Configure host", "config:host", OPAL_CONFIGURE_HOST);
 }
 
-
 static char *escape_quotes(const char *value)
 {
     const char *src;
@@ -845,7 +889,6 @@ static char *escape_quotes(const char *value)
     return quoted_value;
 }
 
-
 /*
  * Private variables - set some reasonable screen size defaults
  */
@@ -858,7 +901,7 @@ static int screen_width = 78;
  */
 void opal_info_out(const char *pretty_message, const char *plain_message, const char *value)
 {
-    size_t len, max_value_width, value_offset;
+    size_t len, max_line_width, value_offset;
     char *spaces = NULL;
     char *filler = NULL;
     char *pos, *v, savev, *v_to_free;
@@ -875,7 +918,7 @@ void opal_info_out(const char *pretty_message, const char *plain_message, const
 #ifdef TIOCGWINSZ
     if (screen_width < INT_MAX) {
         struct winsize size;
-        if (ioctl(STDOUT_FILENO, TIOCGWINSZ, (char*) &size) >= 0) {
+        if (ioctl(STDOUT_FILENO, TIOCGWINSZ, (char *) &size) >= 0) {
             screen_width = size.ws_col;
         }
     }
@@ -893,25 +936,25 @@ void opal_info_out(const char *pretty_message, const char *plain_message, const
     len = strlen(v);
 
     if (len > 0) {
-        while (len > 0 && isspace(v[len-1])) len--;
+        while (len > 0 && isspace(v[len - 1])) {
+            len--;
+        }
         v[len] = '\0';
     }
 
     if (opal_info_pretty && NULL != pretty_message) {
-        if (centerpoint > (int)strlen(pretty_message)) {
-            opal_asprintf(&spaces, "%*s", centerpoint -
-                     (int)strlen(pretty_message), " ");
+        if (centerpoint > (int) strlen(pretty_message)) {
+            opal_asprintf(&spaces, "%*s", centerpoint - (int) strlen(pretty_message), " ");
         } else {
             spaces = strdup("");
 #if OPAL_ENABLE_DEBUG
-            if (centerpoint < (int)strlen(pretty_message)) {
-                opal_show_help("help-opal_info.txt",
-                               "developer warning: field too long", false,
+            if (centerpoint < (int) strlen(pretty_message)) {
+                opal_show_help("help-opal_info.txt", "developer warning: field too long", false,
                                pretty_message, centerpoint);
             }
 #endif
         }
-        max_value_width = screen_width - strlen(spaces) - strlen(pretty_message) - 2;
+        max_line_width = screen_width - strlen(spaces) - strlen(pretty_message) - 2;
         if (0 < strlen(pretty_message)) {
             opal_asprintf(&filler, "%s%s: ", spaces, pretty_message);
         } else {
@@ -921,24 +964,54 @@ void opal_info_out(const char *pretty_message, const char *plain_message, const
         spaces = NULL;
 
         while (true) {
-            if (strlen(v) < max_value_width) {
+            size_t max_value_chars = max_line_width;
+
+            if (opal_info_color) {
+                /* We want to print at most max_line_width characters to the
+                * output. Because `value` might contain bytes that do not
+                * appear in the output (ie. escape codes), simply printing
+                * `max_line_width` bytes will not maximize the number of
+                * visible characters that we can print. */
+                bool in_escape = false;
+                size_t invis = 0;
+
+                /* The loop counts how many invisible characters we encounter
+                * until the visible ones (= i - invis) reach `max_line_width` */
+                for (size_t i = 0; v[i] && (i - invis) < max_line_width; i++) {
+                    if ('\033' == v[i]) {
+                        in_escape = true;
+                    }
+                    if (in_escape) {
+                        invis++;
+                    }
+                    if ('m' == v[i]) {
+                        in_escape = false;
+                    }
+                }
+
+                /* Increase the line limit, so that the invisible
+                 * characters are effectively disregarded */
+                max_value_chars += invis;
+            }
+
+            if (strlen(v) < max_value_chars) {
                 printf("%s%s\n", filler, v);
                 break;
             } else {
                 opal_asprintf(&spaces, "%*s", centerpoint + 2, " ");
 
                 /* Work backwards to find the first space before
-                 * max_value_width
+                 * max_value_chars
                  */
-                savev = v[max_value_width];
-                v[max_value_width] = '\0';
-                pos = (char*)strrchr(v, (int)' ');
-                v[max_value_width] = savev;
+                savev = v[max_value_chars];
+                v[max_value_chars] = '\0';
+                pos = (char *) strrchr(v, (int) ' ');
+                v[max_value_chars] = savev;
                 if (NULL == pos) {
-                    /* No space found < max_value_width.  Look for the first
-                     * space after max_value_width.
+                    /* No space found < max_value_chars.  Look for the first
+                     * space after max_value_chars.
                      */
-                    pos = strchr(&v[max_value_width], ' ');
+                    pos = strchr(&v[max_value_chars], ' ');
 
                     if (NULL == pos) {
 
@@ -1001,13 +1074,11 @@ void opal_info_out(const char *pretty_message, const char *plain_message, const
 /*
  * Prints the passed integer in a pretty or parsable format.
  */
-void opal_info_out_int(const char *pretty_message,
-                       const char *plain_message,
-                       int value)
+void opal_info_out_int(const char *pretty_message, const char *plain_message, int value)
 {
     char *valstr;
 
-    opal_asprintf(&valstr, "%d", (int)value);
+    opal_asprintf(&valstr, "%d", (int) value);
     opal_info_out(pretty_message, plain_message, valstr);
     free(valstr);
 }
@@ -1017,10 +1088,9 @@ void opal_info_out_int(const char *pretty_message,
  * a wildcard)
  */
 void opal_info_show_component_version(opal_pointer_array_t *mca_types,
-                                      opal_pointer_array_t *component_map,
-                                      const char *type_name,
-                                      const char *component_name,
-                                      const char *scope, const char *ver_type)
+                                      opal_pointer_array_t *component_map, const char *type_name,
+                                      const char *component_name, const char *scope,
+                                      const char *ver_type)
 {
     bool want_all_components = false;
     bool want_all_types = false;
@@ -1041,7 +1111,7 @@ void opal_info_show_component_version(opal_pointer_array_t *mca_types,
         /* Check to see if the type is valid */
 
         for (found = false, j = 0; j < mca_types->size; ++j) {
-            if (NULL == (pos = (char*)opal_pointer_array_get_item(mca_types, j))) {
+            if (NULL == (pos = (char *) opal_pointer_array_get_item(mca_types, j))) {
                 continue;
             }
             if (0 == strcmp(pos, type_name)) {
@@ -1058,25 +1128,26 @@ void opal_info_show_component_version(opal_pointer_array_t *mca_types,
     }
 
     /* Now that we have a valid type, find the right components */
-    for (j=0; j < component_map->size; j++) {
-        if (NULL == (map = (opal_info_component_map_t*)opal_pointer_array_get_item(component_map, j))) {
+    for (j = 0; j < component_map->size; j++) {
+        if (NULL
+            == (map = (opal_info_component_map_t *) opal_pointer_array_get_item(component_map,
+                                                                                j))) {
             continue;
         }
         if ((want_all_types || 0 == strcmp(type_name, map->type)) && map->components) {
             /* found it! */
-            OPAL_LIST_FOREACH(cli, map->components, mca_base_component_list_item_t) {
+            OPAL_LIST_FOREACH (cli, map->components, mca_base_component_list_item_t) {
                 const mca_base_component_t *component = cli->cli_component;
-                if (want_all_components ||
-                    0 == strcmp(component->mca_component_name, component_name)) {
+                if (want_all_components
+                    || 0 == strcmp(component->mca_component_name, component_name)) {
                     opal_info_show_mca_version(component, scope, ver_type);
                 }
             }
 
             /* found it! */
-            OPAL_LIST_FOREACH(cli_failed, map->failed_components, mca_base_failed_component_t) {
+            OPAL_LIST_FOREACH (cli_failed, map->failed_components, mca_base_failed_component_t) {
                 mca_base_component_repository_item_t *ri = cli_failed->comp;
-                if (want_all_components ||
-                    0 == strcmp(component_name, ri->ri_name) ) {
+                if (want_all_components || 0 == strcmp(component_name, ri->ri_name)) {
                     opal_info_show_failed_component(ri, cli_failed->error_msg);
                 }
             }
@@ -1088,8 +1159,7 @@ void opal_info_show_component_version(opal_pointer_array_t *mca_types,
     }
 }
 
-
-static void opal_info_show_failed_component(const mca_base_component_repository_item_t* ri,
+static void opal_info_show_failed_component(const mca_base_component_repository_item_t *ri,
                                             const char *error_msg)
 {
     char *message, *content;
@@ -1116,8 +1186,8 @@ static void opal_info_show_failed_component(const mca_base_component_repository_
 /*
  * Given a component, display its relevant version(s)
  */
-void opal_info_show_mca_version(const mca_base_component_t* component,
-                                const char *scope, const char *ver_type)
+void opal_info_show_mca_version(const mca_base_component_t *component, const char *scope,
+                                const char *ver_type)
 {
     bool printed;
     bool want_mca = false;
@@ -1129,33 +1199,29 @@ void opal_info_show_mca_version(const mca_base_component_t* component,
     char *component_version;
     char *tmp;
 
-    if (0 == strcmp(ver_type, opal_info_ver_all) ||
-        0 == strcmp(ver_type, opal_info_ver_mca)) {
+    if (0 == strcmp(ver_type, opal_info_ver_all) || 0 == strcmp(ver_type, opal_info_ver_mca)) {
         want_mca = true;
     }
 
-    if (0 == strcmp(ver_type, opal_info_ver_all) ||
-        0 == strcmp(ver_type, opal_info_ver_type)) {
+    if (0 == strcmp(ver_type, opal_info_ver_all) || 0 == strcmp(ver_type, opal_info_ver_type)) {
         want_type = true;
     }
 
-    if (0 == strcmp(ver_type, opal_info_ver_all) ||
-        0 == strcmp(ver_type, opal_info_ver_component)) {
+    if (0 == strcmp(ver_type, opal_info_ver_all)
+        || 0 == strcmp(ver_type, opal_info_ver_component)) {
         want_component = true;
     }
 
     mca_version = opal_info_make_version_str(scope, component->mca_major_version,
                                              component->mca_minor_version,
-                                             component->mca_release_version, "",
-                                             "");
+                                             component->mca_release_version, "", "");
     api_version = opal_info_make_version_str(scope, component->mca_type_major_version,
                                              component->mca_type_minor_version,
-                                             component->mca_type_release_version, "",
-                                             "");
+                                             component->mca_type_release_version, "", "");
     component_version = opal_info_make_version_str(scope, component->mca_component_major_version,
                                                    component->mca_component_minor_version,
-                                                   component->mca_component_release_version,
-                                                   "", "");
+                                                   component->mca_component_release_version, "",
+                                                   "");
     if (opal_info_pretty) {
         opal_asprintf(&message, "MCA %s", component->mca_type_name);
         printed = false;
@@ -1205,7 +1271,8 @@ void opal_info_show_mca_version(const mca_base_component_t* component,
         }
 
     } else {
-        opal_asprintf(&message, "mca:%s:%s:version", component->mca_type_name, component->mca_component_name);
+        opal_asprintf(&message, "mca:%s:%s:version", component->mca_type_name,
+                      component->mca_component_name);
         if (want_mca) {
             opal_asprintf(&tmp, "mca:%s", mca_version);
             opal_info_out(NULL, message, tmp);
@@ -1235,18 +1302,14 @@ void opal_info_show_mca_version(const mca_base_component_t* component,
     }
 }
 
-
-char *opal_info_make_version_str(const char *scope,
-                                 int major, int minor, int release,
-                                 const char *greek,
-                                 const char *repo)
+char *opal_info_make_version_str(const char *scope, int major, int minor, int release,
+                                 const char *greek, const char *repo)
 {
     char *str = NULL, *tmp;
     char temp[BUFSIZ];
 
     temp[BUFSIZ - 1] = '\0';
-    if (0 == strcmp(scope, opal_info_ver_full) ||
-        0 == strcmp(scope, opal_info_ver_all)) {
+    if (0 == strcmp(scope, opal_info_ver_full) || 0 == strcmp(scope, opal_info_ver_all)) {
         snprintf(temp, BUFSIZ - 1, "%d.%d.%d", major, minor, release);
         str = strdup(temp);
         if (NULL != greek) {
@@ -1278,11 +1341,8 @@ void opal_info_show_opal_version(const char *scope)
     char *tmp, *tmp2;
 
     opal_asprintf(&tmp, "%s:version:full", opal_info_type_opal);
-    tmp2 = opal_info_make_version_str(scope,
-                                      OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
-                                      OPAL_RELEASE_VERSION,
-                                      OPAL_GREEK_VERSION,
-                                      OPAL_REPO_REV);
+    tmp2 = opal_info_make_version_str(scope, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
+                                      OPAL_RELEASE_VERSION, OPAL_GREEK_VERSION, OPAL_REPO_REV);
     opal_info_out("OPAL", tmp, tmp2);
     free(tmp);
     free(tmp2);
diff --git a/opal/runtime/opal_info_support.h b/opal/runtime/opal_info_support.h
index db68e6c497d..b9739b30277 100644
--- a/opal/runtime/opal_info_support.h
+++ b/opal/runtime/opal_info_support.h
@@ -1,9 +1,9 @@
 /*
  * Copyright (c) 2012-2013 Los Alamos National Security, LLC.
  *                         All rights reserved.
-* Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
-* Copyright (c) 2017 IBM Corporation.  All rights reserved.
-* $COPYRIGHT$
+ * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2017 IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
  *
  * Additional copyrights may follow
  *
@@ -19,8 +19,8 @@
 
 #include "opal/class/opal_list.h"
 #include "opal/class/opal_pointer_array.h"
-#include "opal/util/cmd_line.h"
 #include "opal/mca/base/base.h"
+#include "opal/util/cmd_line.h"
 
 BEGIN_C_DECLS
 
@@ -43,7 +43,6 @@ extern const char *opal_info_ver_mca;
 extern const char *opal_info_ver_type;
 extern const char *opal_info_ver_component;
 
-
 /*
  * Component-related functions
  */
@@ -55,9 +54,7 @@ typedef struct {
 } opal_info_component_map_t;
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_info_component_map_t);
 
-
-OPAL_DECLSPEC int opal_info_init(int argc, char **argv,
-                                 opal_cmd_line_t *opal_info_cmd_line);
+OPAL_DECLSPEC int opal_info_init(int argc, char **argv, opal_cmd_line_t *opal_info_cmd_line);
 
 OPAL_DECLSPEC void opal_info_finalize(void);
 
@@ -77,10 +74,8 @@ OPAL_DECLSPEC void opal_info_show_path(const char *type, const char *value);
 
 OPAL_DECLSPEC void opal_info_do_path(bool want_all, opal_cmd_line_t *cmd_line);
 
-OPAL_DECLSPEC void opal_info_show_mca_params(const char *type,
-                                             const char *component,
-                                             mca_base_var_info_lvl_t max_level,
-                                             bool want_internal);
+OPAL_DECLSPEC void opal_info_show_mca_params(const char *type, const char *component,
+                                             mca_base_var_info_lvl_t max_level, bool want_internal);
 
 OPAL_DECLSPEC void opal_info_show_mca_version(const mca_base_component_t *component,
                                               const char *scope, const char *ver_type);
@@ -88,13 +83,11 @@ OPAL_DECLSPEC void opal_info_show_mca_version(const mca_base_component_t *compon
 OPAL_DECLSPEC void opal_info_show_component_version(opal_pointer_array_t *mca_types,
                                                     opal_pointer_array_t *component_map,
                                                     const char *type_name,
-                                                    const char *component_name,
-                                                    const char *scope, const char *ver_type);
+                                                    const char *component_name, const char *scope,
+                                                    const char *ver_type);
 
-OPAL_DECLSPEC char *opal_info_make_version_str(const char *scope,
-                                               int major, int minor, int release,
-                                               const char *greek,
-                                               const char *repo);
+OPAL_DECLSPEC char *opal_info_make_version_str(const char *scope, int major, int minor, int release,
+                                               const char *greek, const char *repo);
 
 OPAL_DECLSPEC void opal_info_show_opal_version(const char *scope);
 
@@ -104,15 +97,15 @@ OPAL_DECLSPEC void opal_info_do_hostname(void);
 
 OPAL_DECLSPEC void opal_info_do_type(opal_cmd_line_t *opal_info_cmd_line);
 
-OPAL_DECLSPEC void opal_info_out(const char *pretty_message, const char *plain_message, const char *value);
+OPAL_DECLSPEC void opal_info_out(const char *pretty_message, const char *plain_message,
+                                 const char *value);
 
-OPAL_DECLSPEC void opal_info_out_int(const char *pretty_message,
-                                     const char *plain_message,
+OPAL_DECLSPEC void opal_info_out_int(const char *pretty_message, const char *plain_message,
                                      int value);
 
-OPAL_DECLSPEC int opal_info_register_project_frameworks (const char *project_name,
-                                                         mca_base_framework_t **frameworks,
-                                                         opal_pointer_array_t *component_map);
+OPAL_DECLSPEC int opal_info_register_project_frameworks(const char *project_name,
+                                                        mca_base_framework_t **frameworks,
+                                                        opal_pointer_array_t *component_map);
 
 END_C_DECLS
 
diff --git a/opal/runtime/opal_init.c b/opal/runtime/opal_init.c
index 274ed6df71f..5ad0792e34f 100644
--- a/opal/runtime/opal_init.c
+++ b/opal/runtime/opal_init.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2020 The University of Tennessee and The University
+ * Copyright (c) 2004-2022 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -18,7 +18,7 @@
  * Copyright (c) 2013-2019 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017-2020 Amazon.com, Inc. or its affiliates.
+ * Copyright (c) 2017-2022 Amazon.com, Inc. or its affiliates.  All Rights reserved.
  *                         All Rights reserved.
  * Copyright (c) 2018      Mellanox Technologies, Inc.
  *                         All rights reserved.
@@ -26,6 +26,7 @@
  *                         reserved.
  * Copyright (c) 2020      FUJITSU LIMITED.  All rights reserved.
  * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -36,392 +37,61 @@
 /** @file **/
 
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 
 #include "opal/include/opal_config.h"
 
-#include "opal/util/malloc.h"
-#include "opal/util/arch.h"
-#include "opal/util/output.h"
-#include "opal/util/show_help.h"
-#include "opal/util/proc.h"
-#include "opal/memoryhooks/memory.h"
+#include "opal/datatype/opal_datatype.h"
+#include "opal/mca/accelerator/base/base.h"
 #include "opal/mca/base/base.h"
 #include "opal/mca/base/mca_base_var.h"
-#include "opal/runtime/opal.h"
-#include "opal/util/net.h"
-#include "opal/datatype/opal_datatype.h"
+#include "opal/mca/hwloc/base/base.h"
+#include "opal/mca/if/base/base.h"
 #include "opal/mca/installdirs/base/base.h"
+#include "opal/mca/memchecker/base/base.h"
+#include "opal/mca/memcpy/base/base.h"
 #include "opal/mca/memory/base/base.h"
 #include "opal/mca/patcher/base/base.h"
 #include "opal/mca/pmix/base/base.h"
-#include "opal/mca/memcpy/base/base.h"
-#include "opal/mca/hwloc/base/base.h"
 #include "opal/mca/reachable/base/base.h"
-#include "opal/mca/timer/base/base.h"
-#include "opal/mca/memchecker/base/base.h"
-#include "opal/mca/if/base/base.h"
 #include "opal/mca/shmem/base/base.h"
+#include "opal/mca/smsc/base/base.h"
 #include "opal/mca/threads/threads.h"
 #include "opal/mca/threads/tsd.h"
+#include "opal/mca/timer/base/base.h"
+#include "opal/memoryhooks/memory.h"
+#include "opal/runtime/opal.h"
+#include "opal/util/arch.h"
+#include "opal/util/malloc.h"
+#include "opal/util/net.h"
+#include "opal/util/output.h"
+#include "opal/util/proc.h"
+#include "opal/util/show_help.h"
 
-#include "opal/runtime/opal_progress.h"
-#include "opal/mca/threads/base/base.h"
 #include "opal/mca/backtrace/base/base.h"
+#include "opal/mca/threads/base/base.h"
+#include "opal/runtime/opal_progress.h"
 #include "opal/util/opal_environ.h"
 
 #include "opal/constants.h"
 #include "opal/util/error.h"
 #include "opal/util/event.h"
-#include "opal/util/stacktrace.h"
 #include "opal/util/keyval_parse.h"
+#include "opal/util/stacktrace.h"
 #include "opal/util/sys_limits.h"
 #include "opal/util/timings.h"
 
-#if OPAL_CC_USE_PRAGMA_IDENT
-#pragma ident OPAL_IDENT_STRING
-#elif OPAL_CC_USE_IDENT
-#ident OPAL_IDENT_STRING
-#endif
 const char opal_version_string[] = OPAL_IDENT_STRING;
 
 int opal_initialized = 0;
-bool opal_init_called = false;
-int opal_util_initialized = 0;
 /* We have to put a guess in here in case hwloc is not available.  If
    hwloc is available, this value will be overwritten when the
    hwloc data is loaded. */
 int opal_cache_line_size = 128;
-bool opal_warn_on_fork = true;
-
-/* If there is a preprocessor macro that redefined the call to
- * gethostname, we undefine that here */
-#ifdef gethostname
-#undef gethostname
-#endif
-
-#define NUM_TRIES_FOR_NULL_HOSTNAME 8
-
-/*
- * This gethostname wrapper does not return the full-length hostname in
- * those rare cases where it is too long for the buffer. It does, however,
- * guarantee a null-terminated hostname is returned, even if it's
- * truncated. It also tries again in the case where gethostname returns an
- * error because the buffer is initially too short.
- */
-int opal_init_gethostname(void)
-{
-    size_t count, length = OPAL_LOCAL_MAXHOSTNAMELEN;
-    int ret_val, num_tries = 0;
-
-    char *buf = calloc( 1, length );
-    if( NULL == buf ) {
-        return OPAL_ERR_OUT_OF_RESOURCE;
-    }
-
-    while( num_tries < NUM_TRIES_FOR_NULL_HOSTNAME) {
-        ++num_tries;
-
-        /*
-         * Offer all but the last byte of the buffer to gethostname.
-         */
-        ret_val = gethostname( buf, length - 1 );
-        /*
-         * Terminate the buffer in the last position.
-         */
-        buf[length - 1] = '\0';
-        if( 0 == ret_val ) {
-            count = strlen( buf );
-            /* The result was not truncated */
-            if( count > 0 && count < length - 1 ) {
-                /*
-                 * If we got a good result, save it.  This value may
-                 * be longer than what callers to opal_gethostname()
-                 * are expecting, so that should be checked by the
-                 * caller.
-                 */
-                opal_process_info.nodename = buf;
-                return OPAL_SUCCESS;
-            }
-            /*
-             * "Good" cases:
-             *
-             * 0 == count: The buffer is empty. In some gethostname
-             *             implementations, this can be because the
-             *             buffer was too small.
-             * (length-1) == count: The result *may* be truncated.
-             *
-             * If it's one of these cases, we'll fall through to
-             * increase the length of the buffer and try again.
-             *
-             * If it's not one of these good cases, it's an error:
-             * return.
-             */
-            else if( !(0 == count || count == length - 1) ) {
-                free(buf);
-                return OPAL_ERR_IN_ERRNO;
-            }
-        }
-        /*
-         * "Good" cases:
-         *
-         * errno == EINVAL or ENAMETOOLONG: hostname was truncated and
-         *              there was an error. Perhaps there is something
-         *              in the buffer and perhaps not.
-         *
-         * If it's one of these cases, we'll fall through to
-         * increase the length of the buffer and try again.
-         *
-         * If it's not one of these good cases, it's an error: return.
-         */
-        else if( !(EINVAL == errno || ENAMETOOLONG == errno) ) {
-            free(buf);
-            return OPAL_ERR_IN_ERRNO;
-        }
-
-        /*
-         * If we get here, it means we want to double the length of
-         * the buffer and try again.
-         */
-        length *= 2;
-        buf = realloc( buf, length );
-        if( NULL == buf ) {
-            return OPAL_ERR_OUT_OF_RESOURCE;
-        }
-    } /* end while */
-
-    /* If we got here, it means that we tried too many times and are
-     * giving up. */
-    free(buf);
-    return OPAL_ERR_NOT_FOUND;
-}
-
-static int
-opal_err2str(int errnum, const char **errmsg)
-{
-    const char *retval;
-
-    switch (errnum) {
-    case OPAL_SUCCESS:
-        retval = "Success";
-        break;
-    case OPAL_ERROR:
-        retval = "Error";
-        break;
-    case OPAL_ERR_OUT_OF_RESOURCE:
-        retval = "Out of resource";
-        break;
-    case OPAL_ERR_TEMP_OUT_OF_RESOURCE:
-        retval = "Temporarily out of resource";
-        break;
-    case OPAL_ERR_RESOURCE_BUSY:
-        retval = "Resource busy";
-        break;
-    case OPAL_ERR_BAD_PARAM:
-        retval = "Bad parameter";
-        break;
-    case OPAL_ERR_FATAL:
-        retval = "Fatal";
-        break;
-    case OPAL_ERR_NOT_IMPLEMENTED:
-        retval = "Not implemented";
-        break;
-    case OPAL_ERR_NOT_SUPPORTED:
-        retval = "Not supported";
-        break;
-    case OPAL_ERR_INTERRUPTED:
-        retval = "Interrupted";
-        break;
-    case OPAL_ERR_WOULD_BLOCK:
-        retval = "Would block";
-        break;
-    case OPAL_ERR_IN_ERRNO:
-        retval = "In errno";
-        break;
-    case OPAL_ERR_UNREACH:
-        retval = "Unreachable";
-        break;
-    case OPAL_ERR_NOT_FOUND:
-        retval = "Not found";
-        break;
-    case OPAL_EXISTS:
-        retval = "Exists";
-        break;
-    case OPAL_ERR_TIMEOUT:
-        retval = "Timeout";
-        break;
-    case OPAL_ERR_NOT_AVAILABLE:
-        retval = "Not available";
-        break;
-    case OPAL_ERR_PERM:
-        retval = "No permission";
-        break;
-    case OPAL_ERR_VALUE_OUT_OF_BOUNDS:
-        retval = "Value out of bounds";
-        break;
-    case OPAL_ERR_FILE_READ_FAILURE:
-        retval = "File read failure";
-        break;
-    case OPAL_ERR_FILE_WRITE_FAILURE:
-        retval = "File write failure";
-        break;
-    case OPAL_ERR_FILE_OPEN_FAILURE:
-        retval = "File open failure";
-        break;
-    case OPAL_ERR_PACK_MISMATCH:
-        retval = "Pack data mismatch";
-        break;
-    case OPAL_ERR_PACK_FAILURE:
-        retval = "Data pack failed";
-        break;
-    case OPAL_ERR_UNPACK_FAILURE:
-        retval = "Data unpack failed";
-        break;
-    case OPAL_ERR_UNPACK_INADEQUATE_SPACE:
-        retval = "Data unpack had inadequate space";
-        break;
-    case OPAL_ERR_UNPACK_READ_PAST_END_OF_BUFFER:
-        retval = "Data unpack would read past end of buffer";
-        break;
-    case OPAL_ERR_OPERATION_UNSUPPORTED:
-        retval = "Requested operation is not supported on referenced data type";
-        break;
-    case OPAL_ERR_UNKNOWN_DATA_TYPE:
-        retval = "Unknown data type";
-        break;
-    case OPAL_ERR_BUFFER:
-        retval = "Buffer type (described vs non-described) mismatch - operation not allowed";
-        break;
-    case OPAL_ERR_DATA_TYPE_REDEF:
-        retval = "Attempt to redefine an existing data type";
-        break;
-    case OPAL_ERR_DATA_OVERWRITE_ATTEMPT:
-        retval = "Attempt to overwrite a data value";
-        break;
-    case OPAL_ERR_MODULE_NOT_FOUND:
-        retval = "Framework requires at least one active module, but none found";
-        break;
-    case OPAL_ERR_TOPO_SLOT_LIST_NOT_SUPPORTED:
-        retval = "OS topology does not support slot_list process affinity";
-        break;
-    case OPAL_ERR_TOPO_SOCKET_NOT_SUPPORTED:
-        retval = "Could not obtain socket topology information";
-        break;
-    case OPAL_ERR_TOPO_CORE_NOT_SUPPORTED:
-        retval = "Could not obtain core topology information";
-        break;
-    case OPAL_ERR_NOT_ENOUGH_SOCKETS:
-        retval = "Not enough sockets to meet request";
-        break;
-    case OPAL_ERR_NOT_ENOUGH_CORES:
-        retval = "Not enough cores to meet request";
-        break;
-    case OPAL_ERR_INVALID_PHYS_CPU:
-        retval = "Invalid physical cpu number returned";
-        break;
-    case OPAL_ERR_MULTIPLE_AFFINITIES:
-        retval = "Multiple methods for assigning process affinity were specified";
-        break;
-    case OPAL_ERR_SLOT_LIST_RANGE:
-        retval = "Provided slot_list range is invalid";
-        break;
-    case OPAL_ERR_NETWORK_NOT_PARSEABLE:
-        retval = "Provided network specification is not parseable";
-        break;
-    case OPAL_ERR_SILENT:
-        retval = NULL;
-        break;
-    case OPAL_ERR_NOT_INITIALIZED:
-        retval = "Not initialized";
-        break;
-    case OPAL_ERR_NOT_BOUND:
-        retval = "Not bound";
-        break;
-    case OPAL_ERR_TAKE_NEXT_OPTION:
-        retval = "Take next option";
-        break;
-    case OPAL_ERR_PROC_ENTRY_NOT_FOUND:
-        retval = "Database entry not found";
-        break;
-    case OPAL_ERR_DATA_VALUE_NOT_FOUND:
-        retval = "Data for specified key not found";
-        break;
-    case OPAL_ERR_CONNECTION_FAILED:
-        retval = "Connection failed";
-        break;
-    case OPAL_ERR_AUTHENTICATION_FAILED:
-        retval = "Authentication failed";
-        break;
-    case OPAL_ERR_COMM_FAILURE:
-        retval = "Comm failure";
-        break;
-    case OPAL_ERR_SERVER_NOT_AVAIL:
-        retval = "Server not available";
-        break;
-    case OPAL_ERR_IN_PROCESS:
-        retval = "Operation in process";
-        break;
-    case OPAL_ERR_DEBUGGER_RELEASE:
-        retval = "Release debugger";
-        break;
-    case OPAL_ERR_HANDLERS_COMPLETE:
-        retval = "Event handlers complete";
-        break;
-    case OPAL_ERR_PARTIAL_SUCCESS:
-        retval = "Partial success";
-        break;
-    case OPAL_ERR_PROC_ABORTED:
-        retval = "Process abnormally terminated";
-        break;
-    case OPAL_ERR_PROC_REQUESTED_ABORT:
-        retval = "Process requested abort";
-        break;
-    case OPAL_ERR_PROC_ABORTING:
-        retval = "Process is aborting";
-        break;
-    case OPAL_ERR_NODE_DOWN:
-        retval = "Node has gone down";
-        break;
-    case OPAL_ERR_NODE_OFFLINE:
-        retval = "Node has gone offline";
-        break;
-    case OPAL_ERR_JOB_TERMINATED:
-        retval = "Job terminated";
-        break;
-    case OPAL_ERR_PROC_RESTART:
-        retval = "Process restarted";
-        break;
-    case OPAL_ERR_PROC_CHECKPOINT:
-        retval = "Process checkpoint";
-        break;
-    case OPAL_ERR_PROC_MIGRATE:
-        retval = "Process migrate";
-        break;
-    case OPAL_ERR_EVENT_REGISTRATION:
-        retval = "Event registration";
-        break;
-    case OPAL_ERR_HEARTBEAT_ALERT:
-        retval = "Heartbeat not received";
-        break;
-    case OPAL_ERR_FILE_ALERT:
-        retval = "File alert - proc may have stalled";
-        break;
-    case OPAL_ERR_MODEL_DECLARED:
-        retval = "Model declared";
-        break;
-    case OPAL_PMIX_LAUNCH_DIRECTIVE:
-        retval = "Launch directive";
-        break;
-
-    default:
-        retval = "UNRECOGNIZED";
-    }
-
-    *errmsg = retval;
-    return OPAL_SUCCESS;
-}
 
+/* Defined in opal_util_init.c and part of the opal_util.la library */
+int opal_init_error(const char *error, int ret);
 
 int opal_init_psm(void)
 {
@@ -450,85 +120,56 @@ int opal_init_psm(void)
     return OPAL_SUCCESS;
 }
 
-static int opal_init_error (const char *error, int ret)
-{
-    if (OPAL_ERR_SILENT != ret) {
-        opal_show_help( "help-opal-runtime.txt",
-                        "opal_init:startup:internal-failure", true,
-                        error, ret );
-    }
-    return ret;
-}
-
-static mca_base_framework_t *opal_init_util_frameworks[] = {
-    &opal_installdirs_base_framework, &opal_if_base_framework, NULL,
+/* the memcpy component should be one of the first who get
+ * loaded in order to make sure we have all the available
+ * versions of memcpy correctly configured.
+ */
+static mca_base_framework_t *opal_init_frameworks[] = {
+    &opal_if_base_framework,
+    &opal_threads_base_framework, &opal_hwloc_base_framework,
+    &opal_memcpy_base_framework, &opal_memchecker_base_framework,
+    &opal_backtrace_base_framework, &opal_timer_base_framework,
+    &opal_shmem_base_framework, &opal_reachable_base_framework,
+    &opal_pmix_base_framework,
+    NULL,
 };
 
-int
-opal_init_util(int* pargc, char*** pargv)
+int opal_init(int *pargc, char ***pargv)
 {
     int ret;
-    char *error = NULL;
+
     OPAL_TIMING_ENV_INIT(otmng);
 
-    if( opal_util_initialized != 0 ) {
-        if( opal_util_initialized < 0 ) {
+    if (opal_initialized != 0) {
+        if (opal_initialized < 0) {
             return OPAL_ERROR;
         }
-        ++opal_util_initialized;
+        ++opal_initialized;
         return OPAL_SUCCESS;
     }
 
-
-    OBJ_CONSTRUCT(&opal_init_util_domain, opal_finalize_domain_t);
-    (void) opal_finalize_domain_init (&opal_init_util_domain, "opal_init_util");
-    opal_finalize_set_domain (&opal_init_util_domain);
-
-    opal_thread_set_main();
-
-    opal_init_called = true;
-
-    /* set the nodename right away so anyone who needs it has it. Note
-     * that we don't bother with fqdn and prefix issues here - we let
-     * the RTE later replace this with a modified name if the user
-     * requests it */
-    ret = opal_init_gethostname();
-    if (OPAL_SUCCESS != ret) {
-        fprintf(stderr, "opal_init_gethostname() failed -- process will likely abort (%s:%d, returned %d instead of OPAL_SUCCESS)\n",
-                __FILE__, __LINE__, ret);
+    /* initialize util code */
+    if (OPAL_SUCCESS != (ret = opal_init_util(pargc, pargv))) {
         return ret;
     }
 
-    /* initialize the memory allocator */
-    opal_malloc_init();
-
-    OPAL_TIMING_ENV_NEXT(otmng, "opal_malloc_init");
-
-    /* initialize the output system */
-    opal_output_init();
-
-    /* initialize install dirs code */
-    if (OPAL_SUCCESS != (ret = mca_base_framework_open(&opal_installdirs_base_framework, 0))) {
-        fprintf(stderr, "opal_installdirs_base_open() failed -- process will likely abort (%s:%d, returned %d instead of OPAL_SUCCESS)\n",
+    /* initialize if framework */
+    if (OPAL_SUCCESS != (ret = mca_base_framework_open(&opal_if_base_framework, 0))) {
+        fprintf(stderr,
+                "opal_if_base_open() failed -- process will likely abort (%s:%d, returned %d "
+                "instead of OPAL_SUCCESS)\n",
                 __FILE__, __LINE__, ret);
         return ret;
     }
 
-    /* initialize the help system */
-    opal_show_help_init();
-
-    OPAL_TIMING_ENV_NEXT(otmng, "opal_show_help_init");
+    OPAL_TIMING_ENV_NEXT(otmng, "opal_if_init");
 
-    /* register handler for errnum -> string converstion */
-    if (OPAL_SUCCESS !=
-        (ret = opal_error_register("OPAL",
-                                   OPAL_ERR_BASE, OPAL_ERR_MAX, opal_err2str))) {
-        return opal_init_error ("opal_error_register", ret);
-    }
+    /* register PMIx cleanup function for output streams */
+    opal_output_register_pmix_cleanup_fn(&opal_pmix_register_cleanup);
 
-    /* keyval lex-based parser */
-    if (OPAL_SUCCESS != (ret = opal_util_keyval_parse_init())) {
-        return opal_init_error ("opal_util_keyval_parse_init", ret);
+    /* register non-core MCA parameters for opal */
+    if (OPAL_SUCCESS != (ret = opal_register_params())) {
+        return opal_init_error("opal_register_params", ret);
     }
 
     // Disable PSM signal hijacking (see comment in function for more
@@ -537,124 +178,49 @@ opal_init_util(int* pargc, char*** pargv)
 
     OPAL_TIMING_ENV_NEXT(otmng, "opal_init_psm");
 
-    /* Setup the parameter system */
-    if (OPAL_SUCCESS != (ret = mca_base_var_init())) {
-        return opal_init_error ("mca_base_var_init", ret);
-    }
-    OPAL_TIMING_ENV_NEXT(otmng, "opal_var_init");
-
-    /* read any param files that were provided */
-    if (OPAL_SUCCESS != (ret = mca_base_var_cache_files(false))) {
-        return opal_init_error ("failed to cache files", ret);
-    }
-
-    OPAL_TIMING_ENV_NEXT(otmng, "opal_var_cache");
-
-
-    /* register params for opal */
-    if (OPAL_SUCCESS != (ret = opal_register_params())) {
-        return opal_init_error ("opal_register_params", ret);
-    }
-
     if (OPAL_SUCCESS != (ret = opal_net_init())) {
-        return opal_init_error ("opal_net_init", ret);
+        return opal_init_error("opal_net_init", ret);
     }
 
     OPAL_TIMING_ENV_NEXT(otmng, "opal_net_init");
 
-    /* pretty-print stack handlers */
-    if (OPAL_SUCCESS != (ret = opal_util_register_stackhandlers())) {
-        return opal_init_error ("opal_util_register_stackhandlers", ret);
-    }
+    OBJ_CONSTRUCT(&opal_init_domain, opal_finalize_domain_t);
+    (void) opal_finalize_domain_init(&opal_init_domain, "opal_init");
+    opal_finalize_set_domain(&opal_init_domain);
 
-    /* set system resource limits - internally protected against
-     * doing so twice in cases where the launch agent did it for us
-     */
-    if (OPAL_SUCCESS != (ret = opal_util_init_sys_limits(&error))) {
-        opal_show_help("help-opal-runtime.txt",
-                        "opal_init:syslimit", false,
-                        error);
-        return OPAL_ERR_SILENT;
-    }
+    /* register for cleanup */
+    opal_finalize_register_cleanup_arg(mca_base_framework_close_list, opal_init_frameworks);
 
-    /* initialize the arch string */
-    if (OPAL_SUCCESS != (ret = opal_arch_init ())) {
-        return opal_init_error ("opal_arch_init", ret);
+    ret = mca_base_framework_open_list(opal_init_frameworks, 0);
+    if (OPAL_UNLIKELY(OPAL_SUCCESS != ret)) {
+        return opal_init_error("opal_init framework open", ret);
     }
 
-    OPAL_TIMING_ENV_NEXT(otmng, "opal_arch_init");
-
-    /* initialize the datatype engine */
-    if (OPAL_SUCCESS != (ret = opal_datatype_init ())) {
-        return opal_init_error ("opal_datatype_init", ret);
+    /* Intitialize Accelerator framework
+     * The datatype convertor code has a dependency on the accelerator framework
+     * being initialized. */
+    ret = mca_base_framework_open(&opal_accelerator_base_framework, 0);
+    if (OPAL_SUCCESS == ret && OPAL_SUCCESS != (ret = opal_accelerator_base_select())) {
+        return opal_init_error("opal_accelerator_base_select", ret);
     }
+    opal_finalize_register_cleanup(opal_accelerator_base_selected_component.accelerator_finalize);
 
-    OPAL_TIMING_ENV_NEXT(otmng, "opal_datatype_init");
-
-    /* initialize the mca */
-    if (OPAL_SUCCESS != (ret = mca_base_open())) {
-        return opal_init_error ("mca_base_open", ret);
+    /* initialize the datatype engine */
+    if (OPAL_SUCCESS != (ret = opal_datatype_init())) {
+        return opal_init_error("opal_datatype_init", ret);
     }
-
-    OPAL_TIMING_ENV_NEXT(otmng, "mca_base_open");
-
-    /* initialize if framework */
-    if (OPAL_SUCCESS != (ret = mca_base_framework_open(&opal_if_base_framework, 0))) {
-        fprintf(stderr, "opal_if_base_open() failed -- process will likely abort (%s:%d, returned %d instead of OPAL_SUCCESS)\n",
-                __FILE__, __LINE__, ret);
+    /* The ddt engine has a few parameters */
+    ret = opal_datatype_register_params();
+    if (OPAL_SUCCESS != ret) {
         return ret;
     }
 
-    /* register for */
-    opal_finalize_register_cleanup_arg (mca_base_framework_close_list, opal_init_util_frameworks);
-
-    OPAL_TIMING_ENV_NEXT(otmng, "opal_if_init");
-
-    ++opal_util_initialized;
-
-    return OPAL_SUCCESS;
-}
-
-
-/* the memcpy component should be one of the first who get
- * loaded in order to make sure we have all the available
- * versions of memcpy correctly configured.
- */
-static mca_base_framework_t *opal_init_frameworks[] = {
-    &opal_threads_base_framework, &opal_hwloc_base_framework, &opal_memcpy_base_framework, &opal_memchecker_base_framework,
-    &opal_backtrace_base_framework, &opal_timer_base_framework,
-    &opal_shmem_base_framework, &opal_reachable_base_framework, &opal_pmix_base_framework,
-    NULL,
-};
-
-int
-opal_init(int* pargc, char*** pargv)
-{
-    int ret;
-
-    if( opal_initialized != 0 ) {
-        if( opal_initialized < 0 ) {
-            return OPAL_ERROR;
-        }
-        ++opal_initialized;
-        return OPAL_SUCCESS;
-    }
-
-    /* initialize util code */
-    if (OPAL_SUCCESS != (ret = opal_init_util(pargc, pargv))) {
+    ret = opal_event_register_params();
+    if (OPAL_SUCCESS != ret) {
         return ret;
     }
 
-    OBJ_CONSTRUCT(&opal_init_domain, opal_finalize_domain_t);
-    (void) opal_finalize_domain_init (&opal_init_domain, "opal_init");
-    opal_finalize_set_domain (&opal_init_domain);
-
-    opal_finalize_register_cleanup_arg (mca_base_framework_close_list, opal_init_frameworks);
-
-    ret = mca_base_framework_open_list (opal_init_frameworks, 0);
-    if (OPAL_UNLIKELY(OPAL_SUCCESS != ret)) {
-        return opal_init_error ("opal_init framework open", ret);
-    }
+    OPAL_TIMING_ENV_NEXT(otmng, "opal_datatype_init");
 
     /* initialize libevent */
     if (OPAL_SUCCESS != (ret = opal_event_init())) {
@@ -663,31 +229,31 @@ opal_init(int* pargc, char*** pargv)
 
     /* initialize the memory manager / tracker */
     if (OPAL_SUCCESS != (ret = opal_mem_hooks_init())) {
-        return opal_init_error ("opal_mem_hooks_init", ret);
+        return opal_init_error("opal_mem_hooks_init", ret);
     }
 
     /* select the memory checker */
     if (OPAL_SUCCESS != (ret = opal_memchecker_base_select())) {
-        return opal_init_error ("opal_memchecker_base_select", ret);
+        return opal_init_error("opal_memchecker_base_select", ret);
     }
 
     /*
      * Initialize the general progress engine
      */
     if (OPAL_SUCCESS != (ret = opal_progress_init())) {
-        return opal_init_error ("opal_progress_init", ret);
+        return opal_init_error("opal_progress_init", ret);
     }
     /* we want to tick the event library whenever possible */
     opal_progress_event_users_increment();
 
     /* setup the shmem framework */
     if (OPAL_SUCCESS != (ret = opal_shmem_base_select())) {
-        return opal_init_error ("opal_shmem_base_select", ret);
+        return opal_init_error("opal_shmem_base_select", ret);
     }
 
-    /* Intitialize reachable framework */
+    /* Initialize reachable framework */
     if (OPAL_SUCCESS != (ret = opal_reachable_base_select())) {
-        return opal_init_error ("opal_reachable_base_select", ret);
+        return opal_init_error("opal_reachable_base_select", ret);
     }
 
     ++opal_initialized;
diff --git a/opal/runtime/opal_init_core.c b/opal/runtime/opal_init_core.c
new file mode 100644
index 00000000000..d9279376455
--- /dev/null
+++ b/opal/runtime/opal_init_core.c
@@ -0,0 +1,547 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2022 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007-2020 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2007      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2009      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2010-2015 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2013-2019 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015-2017 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017-2022 Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ *                         All Rights reserved.
+ * Copyright (c) 2018      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2018-2019 Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2020      FUJITSU LIMITED.  All rights reserved.
+ * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/** @file **/
+
+#ifdef HAVE_UNISTD_H
+#    include <unistd.h>
+#endif
+
+#include "opal/include/opal_config.h"
+
+#include "opal/mca/base/base.h"
+#include "opal/mca/base/mca_base_var.h"
+#include "opal/mca/hwloc/base/base.h"
+#include "opal/mca/installdirs/base/base.h"
+#include "opal/mca/memchecker/base/base.h"
+#include "opal/mca/memcpy/base/base.h"
+#include "opal/mca/memory/base/base.h"
+#include "opal/mca/patcher/base/base.h"
+#include "opal/mca/pmix/base/base.h"
+#include "opal/mca/reachable/base/base.h"
+#include "opal/mca/shmem/base/base.h"
+#include "opal/mca/smsc/base/base.h"
+#include "opal/mca/threads/threads.h"
+#include "opal/mca/threads/tsd.h"
+#include "opal/mca/timer/base/base.h"
+#include "opal/memoryhooks/memory.h"
+#include "opal/runtime/opal.h"
+#include "opal/util/arch.h"
+#include "opal/util/malloc.h"
+#include "opal/util/net.h"
+#include "opal/util/output.h"
+#include "opal/util/proc.h"
+#include "opal/util/show_help.h"
+
+#include "opal/mca/backtrace/base/base.h"
+#include "opal/mca/threads/base/base.h"
+#include "opal/runtime/opal_progress.h"
+#include "opal/runtime/opal_params_core.h"
+#include "opal/util/opal_environ.h"
+
+#include "opal/constants.h"
+#include "opal/util/error.h"
+#include "opal/util/event.h"
+#include "opal/util/keyval_parse.h"
+#include "opal/util/stacktrace.h"
+#include "opal/util/sys_limits.h"
+#include "opal/util/timings.h"
+
+int opal_util_initialized = 0;
+bool opal_warn_on_fork = true;
+
+int opal_init_error(const char *error, int ret);
+
+static int opal_err2str(int errnum, const char **errmsg)
+{
+    const char *retval;
+
+    switch (errnum) {
+    case OPAL_SUCCESS:
+        retval = "Success";
+        break;
+    case OPAL_ERROR:
+        retval = "Error";
+        break;
+    case OPAL_ERR_OUT_OF_RESOURCE:
+        retval = "Out of resource";
+        break;
+    case OPAL_ERR_TEMP_OUT_OF_RESOURCE:
+        retval = "Temporarily out of resource";
+        break;
+    case OPAL_ERR_RESOURCE_BUSY:
+        retval = "Resource busy";
+        break;
+    case OPAL_ERR_BAD_PARAM:
+        retval = "Bad parameter";
+        break;
+    case OPAL_ERR_FATAL:
+        retval = "Fatal";
+        break;
+    case OPAL_ERR_NOT_IMPLEMENTED:
+        retval = "Not implemented";
+        break;
+    case OPAL_ERR_NOT_SUPPORTED:
+        retval = "Not supported";
+        break;
+    case OPAL_ERR_INTERRUPTED:
+        retval = "Interrupted";
+        break;
+    case OPAL_ERR_WOULD_BLOCK:
+        retval = "Would block";
+        break;
+    case OPAL_ERR_IN_ERRNO:
+        retval = "In errno";
+        break;
+    case OPAL_ERR_UNREACH:
+        retval = "Unreachable";
+        break;
+    case OPAL_ERR_NOT_FOUND:
+        retval = "Not found";
+        break;
+    case OPAL_EXISTS:
+        retval = "Exists";
+        break;
+    case OPAL_ERR_TIMEOUT:
+        retval = "Timeout";
+        break;
+    case OPAL_ERR_NOT_AVAILABLE:
+        retval = "Not available";
+        break;
+    case OPAL_ERR_PERM:
+        retval = "No permission";
+        break;
+    case OPAL_ERR_VALUE_OUT_OF_BOUNDS:
+        retval = "Value out of bounds";
+        break;
+    case OPAL_ERR_FILE_READ_FAILURE:
+        retval = "File read failure";
+        break;
+    case OPAL_ERR_FILE_WRITE_FAILURE:
+        retval = "File write failure";
+        break;
+    case OPAL_ERR_FILE_OPEN_FAILURE:
+        retval = "File open failure";
+        break;
+    case OPAL_ERR_PACK_MISMATCH:
+        retval = "Pack data mismatch";
+        break;
+    case OPAL_ERR_PACK_FAILURE:
+        retval = "Data pack failed";
+        break;
+    case OPAL_ERR_UNPACK_FAILURE:
+        retval = "Data unpack failed";
+        break;
+    case OPAL_ERR_UNPACK_INADEQUATE_SPACE:
+        retval = "Data unpack had inadequate space";
+        break;
+    case OPAL_ERR_UNPACK_READ_PAST_END_OF_BUFFER:
+        retval = "Data unpack would read past end of buffer";
+        break;
+    case OPAL_ERR_OPERATION_UNSUPPORTED:
+        retval = "Requested operation is not supported on referenced data type";
+        break;
+    case OPAL_ERR_UNKNOWN_DATA_TYPE:
+        retval = "Unknown data type";
+        break;
+    case OPAL_ERR_BUFFER:
+        retval = "Buffer type (described vs non-described) mismatch - operation not allowed";
+        break;
+    case OPAL_ERR_DATA_TYPE_REDEF:
+        retval = "Attempt to redefine an existing data type";
+        break;
+    case OPAL_ERR_DATA_OVERWRITE_ATTEMPT:
+        retval = "Attempt to overwrite a data value";
+        break;
+    case OPAL_ERR_MODULE_NOT_FOUND:
+        retval = "Framework requires at least one active module, but none found";
+        break;
+    case OPAL_ERR_TOPO_SLOT_LIST_NOT_SUPPORTED:
+        retval = "OS topology does not support slot_list process affinity";
+        break;
+    case OPAL_ERR_TOPO_SOCKET_NOT_SUPPORTED:
+        retval = "Could not obtain socket topology information";
+        break;
+    case OPAL_ERR_TOPO_CORE_NOT_SUPPORTED:
+        retval = "Could not obtain core topology information";
+        break;
+    case OPAL_ERR_NOT_ENOUGH_SOCKETS:
+        retval = "Not enough sockets to meet request";
+        break;
+    case OPAL_ERR_NOT_ENOUGH_CORES:
+        retval = "Not enough cores to meet request";
+        break;
+    case OPAL_ERR_INVALID_PHYS_CPU:
+        retval = "Invalid physical cpu number returned";
+        break;
+    case OPAL_ERR_MULTIPLE_AFFINITIES:
+        retval = "Multiple methods for assigning process affinity were specified";
+        break;
+    case OPAL_ERR_SLOT_LIST_RANGE:
+        retval = "Provided slot_list range is invalid";
+        break;
+    case OPAL_ERR_NETWORK_NOT_PARSEABLE:
+        retval = "Provided network specification is not parseable";
+        break;
+    case OPAL_ERR_SILENT:
+        retval = NULL;
+        break;
+    case OPAL_ERR_NOT_INITIALIZED:
+        retval = "Not initialized";
+        break;
+    case OPAL_ERR_NOT_BOUND:
+        retval = "Not bound";
+        break;
+    case OPAL_ERR_TAKE_NEXT_OPTION:
+        retval = "Take next option";
+        break;
+    case OPAL_ERR_PROC_ENTRY_NOT_FOUND:
+        retval = "Database entry not found";
+        break;
+    case OPAL_ERR_DATA_VALUE_NOT_FOUND:
+        retval = "Data for specified key not found";
+        break;
+    case OPAL_ERR_CONNECTION_FAILED:
+        retval = "Connection failed";
+        break;
+    case OPAL_ERR_AUTHENTICATION_FAILED:
+        retval = "Authentication failed";
+        break;
+    case OPAL_ERR_COMM_FAILURE:
+        retval = "Comm failure";
+        break;
+    case OPAL_ERR_SERVER_NOT_AVAIL:
+        retval = "Server not available";
+        break;
+    case OPAL_ERR_IN_PROCESS:
+        retval = "Operation in process";
+        break;
+    case OPAL_ERR_DEBUGGER_RELEASE:
+        retval = "Release debugger";
+        break;
+    case OPAL_ERR_HANDLERS_COMPLETE:
+        retval = "Event handlers complete";
+        break;
+    case OPAL_ERR_PARTIAL_SUCCESS:
+        retval = "Partial success";
+        break;
+    case OPAL_ERR_PROC_ABORTED:
+        retval = "Process abnormally terminated";
+        break;
+    case OPAL_ERR_PROC_REQUESTED_ABORT:
+        retval = "Process requested abort";
+        break;
+    case OPAL_ERR_PROC_ABORTING:
+        retval = "Process is aborting";
+        break;
+    case OPAL_ERR_NODE_DOWN:
+        retval = "Node has gone down";
+        break;
+    case OPAL_ERR_NODE_OFFLINE:
+        retval = "Node has gone offline";
+        break;
+    case OPAL_ERR_JOB_TERMINATED:
+        retval = "Job terminated";
+        break;
+    case OPAL_ERR_PROC_RESTART:
+        retval = "Process restarted";
+        break;
+    case OPAL_ERR_PROC_CHECKPOINT:
+        retval = "Process checkpoint";
+        break;
+    case OPAL_ERR_PROC_MIGRATE:
+        retval = "Process migrate";
+        break;
+    case OPAL_ERR_EVENT_REGISTRATION:
+        retval = "Event registration";
+        break;
+    case OPAL_ERR_HEARTBEAT_ALERT:
+        retval = "Heartbeat not received";
+        break;
+    case OPAL_ERR_FILE_ALERT:
+        retval = "File alert - proc may have stalled";
+        break;
+    case OPAL_ERR_MODEL_DECLARED:
+        retval = "Model declared";
+        break;
+    case OPAL_PMIX_LAUNCH_DIRECTIVE:
+        retval = "Launch directive";
+        break;
+
+    default:
+        retval = "UNRECOGNIZED";
+    }
+
+    *errmsg = retval;
+    return OPAL_SUCCESS;
+}
+
+int opal_init_error(const char *error, int ret)
+{
+    if (OPAL_ERR_SILENT != ret) {
+        opal_show_help("help-opal-runtime.txt", "opal_init:startup:internal-failure", true, error,
+                       ret);
+    }
+    return ret;
+}
+
+/* If there is a preprocessor macro that redefined the call to
+ * gethostname, we undefine that here */
+#ifdef gethostname
+#    undef gethostname
+#endif
+
+#define NUM_TRIES_FOR_NULL_HOSTNAME 8
+
+/*
+ * This gethostname wrapper does not return the full-length hostname in
+ * those rare cases where it is too long for the buffer. It does, however,
+ * guarantee a null-terminated hostname is returned, even if it's
+ * truncated. It also tries again in the case where gethostname returns an
+ * error because the buffer is initially too short.
+ */
+int opal_init_gethostname(void)
+{
+    size_t count, length = OPAL_LOCAL_MAXHOSTNAMELEN;
+    int ret_val, num_tries = 0;
+
+    char *newbuf;
+    char *buf = calloc(1, length);
+    if (NULL == buf) {
+        return OPAL_ERR_OUT_OF_RESOURCE;
+    }
+
+    while (num_tries < NUM_TRIES_FOR_NULL_HOSTNAME) {
+        ++num_tries;
+
+        /*
+         * Offer all but the last byte of the buffer to gethostname.
+         */
+        ret_val = gethostname(buf, length - 1);
+        /*
+         * Terminate the buffer in the last position.
+         */
+        buf[length - 1] = '\0';
+        if (0 == ret_val) {
+            count = strlen(buf);
+            /* The result was not truncated */
+            if (count > 0 && count < length - 1) {
+                /*
+                 * If we got a good result, save it.  This value may
+                 * be longer than what callers to opal_gethostname()
+                 * are expecting, so that should be checked by the
+                 * caller.
+                 */
+                opal_process_info.nodename = buf;
+                return OPAL_SUCCESS;
+            }
+            /*
+             * "Good" cases:
+             *
+             * 0 == count: The buffer is empty. In some gethostname
+             *             implementations, this can be because the
+             *             buffer was too small.
+             * (length-1) == count: The result *may* be truncated.
+             *
+             * If it's one of these cases, we'll fall through to
+             * increase the length of the buffer and try again.
+             *
+             * If it's not one of these good cases, it's an error:
+             * return.
+             */
+            else if (!(0 == count || count == length - 1)) {
+                free(buf);
+                return OPAL_ERR_IN_ERRNO;
+            }
+        }
+        /*
+         * "Good" cases:
+         *
+         * errno == EINVAL or ENAMETOOLONG: hostname was truncated and
+         *              there was an error. Perhaps there is something
+         *              in the buffer and perhaps not.
+         *
+         * If it's one of these cases, we'll fall through to
+         * increase the length of the buffer and try again.
+         *
+         * If it's not one of these good cases, it's an error: return.
+         */
+        else if (!(EINVAL == errno || ENAMETOOLONG == errno)) {
+            free(buf);
+            return OPAL_ERR_IN_ERRNO;
+        }
+
+        /*
+         * If we get here, it means we want to double the length of
+         * the buffer and try again.
+         */
+        length *= 2;
+        newbuf = realloc(buf, length);
+        if (NULL == newbuf) {
+            free(buf);
+            return OPAL_ERR_OUT_OF_RESOURCE;
+        }
+        buf = newbuf;
+    } /* end while */
+
+    /* If we got here, it means that we tried too many times and are
+     * giving up. */
+    free(buf);
+    return OPAL_ERR_NOT_FOUND;
+}
+
+static mca_base_framework_t *opal_init_util_frameworks[] = {
+    &opal_installdirs_base_framework,
+    NULL,
+};
+
+int opal_init_util(int *pargc, char ***pargv)
+{
+    int ret;
+    char *error = NULL;
+    OPAL_TIMING_ENV_INIT(otmng);
+
+    if (opal_util_initialized != 0) {
+        if (opal_util_initialized < 0) {
+            return OPAL_ERROR;
+        }
+        ++opal_util_initialized;
+        return OPAL_SUCCESS;
+    }
+
+    OBJ_CONSTRUCT(&opal_init_util_domain, opal_finalize_domain_t);
+    (void) opal_finalize_domain_init(&opal_init_util_domain, "opal_init_util");
+    opal_finalize_set_domain(&opal_init_util_domain);
+
+    opal_thread_set_main();
+
+    /* set the nodename right away so anyone who needs it has it. Note
+     * that we don't bother with fqdn and prefix issues here - we let
+     * the RTE later replace this with a modified name if the user
+     * requests it */
+    ret = opal_init_gethostname();
+    if (OPAL_SUCCESS != ret) {
+        fprintf(stderr,
+                "opal_init_gethostname() failed -- process will likely abort (%s:%d, returned %d "
+                "instead of OPAL_SUCCESS)\n",
+                __FILE__, __LINE__, ret);
+        return ret;
+    }
+
+    /* initialize the memory allocator */
+    opal_malloc_init();
+
+    OPAL_TIMING_ENV_NEXT(otmng, "opal_malloc_init");
+
+    /* initialize the output system */
+    opal_output_init();
+
+    /* initialize install dirs code */
+    if (OPAL_SUCCESS != (ret = mca_base_framework_open(&opal_installdirs_base_framework, 0))) {
+        fprintf(stderr,
+                "opal_installdirs_base_open() failed -- process will likely abort (%s:%d, returned "
+                "%d instead of OPAL_SUCCESS)\n",
+                __FILE__, __LINE__, ret);
+        return ret;
+    }
+
+    /* initialize the help system */
+    opal_show_help_init();
+
+    OPAL_TIMING_ENV_NEXT(otmng, "opal_show_help_init");
+
+    /* register handler for errnum -> string converstion */
+    if (OPAL_SUCCESS
+        != (ret = opal_error_register("OPAL", OPAL_ERR_BASE, OPAL_ERR_MAX, opal_err2str))) {
+        return opal_init_error("opal_error_register", ret);
+    }
+
+    /* keyval lex-based parser */
+    if (OPAL_SUCCESS != (ret = opal_util_keyval_parse_init())) {
+        return opal_init_error("opal_util_keyval_parse_init", ret);
+    }
+
+    /* Setup the parameter system */
+    if (OPAL_SUCCESS != (ret = mca_base_var_init())) {
+        return opal_init_error("mca_base_var_init", ret);
+    }
+    OPAL_TIMING_ENV_NEXT(otmng, "opal_var_init");
+
+    /* read any param files that were provided */
+    if (OPAL_SUCCESS != (ret = mca_base_var_cache_files(false))) {
+        return opal_init_error("failed to cache files", ret);
+    }
+
+    OPAL_TIMING_ENV_NEXT(otmng, "opal_var_cache");
+
+    /* register util params for opal */
+    if (OPAL_SUCCESS != (ret = opal_register_util_params())) {
+        return opal_init_error("opal_register_util_params", ret);
+    }
+
+    /* pretty-print stack handlers */
+    if (OPAL_SUCCESS != (ret = opal_util_register_stackhandlers())) {
+        return opal_init_error("opal_util_register_stackhandlers", ret);
+    }
+
+    /* set system resource limits - internally protected against
+     * doing so twice in cases where the launch agent did it for us
+     */
+    if (OPAL_SUCCESS != (ret = opal_util_init_sys_limits(&error))) {
+        opal_show_help("help-opal-runtime.txt", "opal_init:syslimit", false, error);
+        return OPAL_ERR_SILENT;
+    }
+
+    /* initialize the arch string */
+    if (OPAL_SUCCESS != (ret = opal_arch_init())) {
+        return opal_init_error("opal_arch_init", ret);
+    }
+
+    OPAL_TIMING_ENV_NEXT(otmng, "opal_arch_init");
+
+    /* initialize the mca */
+    if (OPAL_SUCCESS != (ret = mca_base_open())) {
+        return opal_init_error("mca_base_open", ret);
+    }
+
+    OPAL_TIMING_ENV_NEXT(otmng, "mca_base_open");
+
+    /* register for cleanup */
+    opal_finalize_register_cleanup_arg(mca_base_framework_close_list, opal_init_util_frameworks);
+
+    ++opal_util_initialized;
+
+    return OPAL_SUCCESS;
+}
diff --git a/opal/runtime/opal_params.c b/opal/runtime/opal_params.c
index 84c9c4a4b1f..effc6d3ce0f 100644
--- a/opal/runtime/opal_params.c
+++ b/opal/runtime/opal_params.c
@@ -22,8 +22,10 @@
  * Copyright (c) 2015      Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2017      IBM Corporation.  All rights reserved.
- * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2018-2022 Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2022      Advanced Micro Devices, Inc. All rights reserved.
+ * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -33,344 +35,64 @@
 
 #include "opal_config.h"
 
-#include <time.h>
 #include <signal.h>
+#include <time.h>
 
 #include "opal/constants.h"
-#include "opal/runtime/opal.h"
 #include "opal/datatype/opal_datatype.h"
 #include "opal/mca/base/mca_base_var.h"
+#include "opal/mca/shmem/base/base.h"
 #include "opal/mca/threads/mutex.h"
 #include "opal/mca/threads/threads.h"
-#include "opal/mca/shmem/base/base.h"
-#include "opal/mca/base/mca_base_var.h"
+#include "opal/runtime/opal.h"
 #include "opal/runtime/opal_params.h"
 #include "opal/util/opal_environ.h"
+#include "opal/util/printf.h"
 #include "opal/util/show_help.h"
 #include "opal/util/timings.h"
-#include "opal/util/printf.h"
 
-char *opal_signal_string = NULL;
-char *opal_stacktrace_output_filename = NULL;
-char *opal_net_private_ipv4 = NULL;
-char *opal_set_max_sys_limits = NULL;
-
-#if OPAL_ENABLE_TIMING
-char *opal_timing_sync_file = NULL;
-char *opal_timing_output = NULL;
-bool opal_timing_overhead = true;
-#endif
-
-bool opal_built_with_cuda_support = OPAL_INT_TO_BOOL(OPAL_CUDA_SUPPORT);
-bool opal_cuda_support = false;
-bool opal_warn_on_missing_libcuda = true;
-
-/**
- * Globals imported from the OMPI layer.
- */
-int opal_leave_pinned = -1;
-bool opal_leave_pinned_pipeline = false;
-bool opal_abort_print_stack = false;
-int opal_abort_delay = 0;
-
-int opal_max_thread_in_progress = 1;
 
 static bool opal_register_done = false;
 
-static void opal_deregister_params (void)
+static void opal_deregister_params(void)
 {
-    /* The MCA variable system will be torn down shortly so reset the registered
-     * flag. */
     opal_register_done = false;
 }
 
 int opal_register_params(void)
 {
-    int ret;
-    char *string = NULL;
-
     if (opal_register_done) {
         return OPAL_SUCCESS;
     }
 
     opal_register_done = true;
 
-    /*
-     * This string is going to be used in opal/util/stacktrace.c
-     */
-    {
-        int j;
-        int signals[] = {
-#ifdef SIGABRT
-            SIGABRT,
-#endif
-#ifdef SIGBUS
-            SIGBUS,
-#endif
-#ifdef SIGFPE
-            SIGFPE,
-#endif
-#ifdef SIGSEGV
-            SIGSEGV,
-#endif
-            -1
-        };
-        for (j = 0 ; signals[j] != -1 ; ++j) {
-            if (j == 0) {
-                opal_asprintf(&string, "%d", signals[j]);
-            } else {
-                char *tmp;
-                opal_asprintf(&tmp, "%s,%d", string, signals[j]);
-                free(string);
-                string = tmp;
-            }
-        }
-
-        opal_signal_string = string;
-        ret = mca_base_var_register ("opal", "opal", NULL, "signal",
-				     "Comma-delimited list of integer signal numbers to Open MPI to attempt to intercept.  Upon receipt of the intercepted signal, Open MPI will display a stack trace and abort.  Open MPI will *not* replace signals if handlers are already installed by the time MPI_INIT is invoked.  Optionally append \":complain\" to any signal number in the comma-delimited list to make Open MPI complain if it detects another signal handler (and therefore does not insert its own).",
-				     MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-				     OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_LOCAL,
-				     &opal_signal_string);
-        free (string);
-        if (0 > ret) {
-            return ret;
-        }
-    }
-
-    /*
-     * Where should the stack trace output be directed
-     * This string is going to be used in opal/util/stacktrace.c
-     */
-    string = strdup("stderr");
-    opal_stacktrace_output_filename = string;
-    ret = mca_base_var_register ("opal", "opal", NULL, "stacktrace_output",
-                                 "Specifies where the stack trace output stream goes.  "
-                                 "Accepts one of the following: none (disabled), stderr (default), stdout, file[:filename].   "
-                                 "If 'filename' is not specified, a default filename of 'stacktrace' is used.  "
-                                 "The 'filename' is appended with either '.PID' or '.RANK.PID', if RANK is available.  "
-                                 "The 'filename' can be an absolute path or a relative path to the current working directory.",
-                                 MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-                                 OPAL_INFO_LVL_3,
-                                 MCA_BASE_VAR_SCOPE_LOCAL,
-                                 &opal_stacktrace_output_filename);
-    free (string);
-    if (0 > ret) {
-        return ret;
-    }
-
-
 #if defined(HAVE_SCHED_YIELD)
     opal_progress_yield_when_idle = false;
-    ret = mca_base_var_register ("opal", "opal", "progress", "yield_when_idle",
-                                 "Yield the processor when waiting on progress",
-                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-                                 OPAL_INFO_LVL_8, MCA_BASE_VAR_SCOPE_LOCAL,
-                                 &opal_progress_yield_when_idle);
+    int ret1;
+    ret1 = mca_base_var_register("opal", "opal", "progress", "yield_when_idle",
+                                "Yield the processor when waiting on progress",
+                                MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
+                                OPAL_INFO_LVL_8, MCA_BASE_VAR_SCOPE_LOCAL,
+                                &opal_progress_yield_when_idle);
+    if (ret1 < 0) {
+        return ret1;
+    }
 #endif
 
 #if OPAL_ENABLE_DEBUG
     opal_progress_debug = false;
-    ret = mca_base_var_register ("opal", "opal", "progress", "debug",
-				 "Set to non-zero to debug progress engine features",
-				 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-				 OPAL_INFO_LVL_8, MCA_BASE_VAR_SCOPE_LOCAL,
-				 &opal_progress_debug);
-    if (0 > ret) {
-        return ret;
-    }
-
-    opal_debug_threads = false;
-    ret = mca_base_var_register ("opal", "opal", "debug", "threads",
-				 "Debug thread usage within OPAL. Reports out "
-				 "when threads are acquired and released.",
-				 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-				 OPAL_INFO_LVL_8, MCA_BASE_VAR_SCOPE_LOCAL,
-				 &opal_debug_threads);
-    if (0 > ret) {
-        return ret;
-    }
-#endif
-
-    /* RFC1918 defines
-       - 10.0.0./8
-       - 172.16.0.0/12
-       - 192.168.0.0/16
-
-       RFC3330 also mentions
-       - 169.254.0.0/16 for DHCP onlink iff there's no DHCP server
-    */
-    opal_net_private_ipv4 = "10.0.0.0/8;172.16.0.0/12;192.168.0.0/16;169.254.0.0/16";
-    ret = mca_base_var_register ("opal", "opal", "net", "private_ipv4",
-				 "Semicolon-delimited list of CIDR notation entries specifying what networks are considered \"private\" (default value based on RFC1918 and RFC3330)",
-				 MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-				 OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_ALL_EQ,
-				 &opal_net_private_ipv4);
-    if (0 > ret) {
-        return ret;
-    }
-
-    opal_set_max_sys_limits = NULL;
-    ret = mca_base_var_register ("opal", "opal", NULL, "set_max_sys_limits",
-				 "Set the specified system-imposed limits to the specified value, including \"unlimited\"."
-                                 "Supported params: core, filesize, maxmem, openfiles, stacksize, maxchildren",
-				 MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-				 OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_ALL_EQ,
-				 &opal_set_max_sys_limits);
-    if (0 > ret) {
-        return ret;
-    }
-
-    ret = mca_base_var_register("opal", "opal", NULL, "built_with_cuda_support",
-                                "Whether CUDA GPU buffer support is built into library or not",
-                                MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_DEFAULT_ONLY,
-                                OPAL_INFO_LVL_4, MCA_BASE_VAR_SCOPE_CONSTANT,
-                                &opal_built_with_cuda_support);
-    if (0 > ret) {
-        return ret;
-    }
-
-    /* Current default is to enable CUDA support if it is built into library */
-    opal_cuda_support = opal_built_with_cuda_support;
-    ret = mca_base_var_register ("opal", "opal", NULL, "cuda_support",
-                                 "Whether CUDA GPU buffer support is enabled or not",
-                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-                                 OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_ALL_EQ,
-                                 &opal_cuda_support);
-    if (0 > ret) {
-        return ret;
-    }
-
-    opal_warn_on_missing_libcuda = true;
-    ret = mca_base_var_register ("opal", "opal", NULL, "warn_on_missing_libcuda",
-                                 "Whether to print a message when CUDA support is enabled but libcuda is not found",
-                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
-                                 OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_ALL_EQ,
-                                 &opal_warn_on_missing_libcuda);
-    if (0 > ret) {
-        return ret;
-    }
-
-    /* Leave pinned parameter */
-    opal_leave_pinned = -1;
-    ret = mca_base_var_register("ompi", "mpi", NULL, "leave_pinned",
-                                "Whether to use the \"leave pinned\" protocol or not.  Enabling this setting can help bandwidth performance when repeatedly sending and receiving large messages with the same buffers over RDMA-based networks (false = do not use \"leave pinned\" protocol, true = use \"leave pinned\" protocol, auto = allow network to choose at runtime).",
-                                MCA_BASE_VAR_TYPE_INT, &mca_base_var_enum_auto_bool, 0, 0,
-                                OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
-                                &opal_leave_pinned);
-    mca_base_var_register_synonym(ret, "opal", "opal", NULL, "leave_pinned",
-                                  MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
-
-    opal_leave_pinned_pipeline = false;
-    ret = mca_base_var_register("ompi", "mpi", NULL, "leave_pinned_pipeline",
-                                "Whether to use the \"leave pinned pipeline\" protocol or not.",
-                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
-                                 OPAL_INFO_LVL_9,
-                                 MCA_BASE_VAR_SCOPE_READONLY,
-                                 &opal_leave_pinned_pipeline);
-    mca_base_var_register_synonym(ret, "opal", "opal", NULL, "leave_pinned_pipeline",
-                                  MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
-
-    if (opal_leave_pinned > 0 && opal_leave_pinned_pipeline) {
-        opal_leave_pinned_pipeline = 0;
-        opal_show_help("help-opal-runtime.txt",
-                       "mpi-params:leave-pinned-and-pipeline-selected",
-                       true);
-    }
-
-    opal_warn_on_fork = true;
-    (void) mca_base_var_register("ompi", "mpi", NULL, "warn_on_fork",
-                                 "If nonzero, issue a warning if program forks under conditions that could cause system errors",
-                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
-                                 OPAL_INFO_LVL_9,
-                                 MCA_BASE_VAR_SCOPE_READONLY,
-                                 &opal_warn_on_fork);
-
-    opal_abort_delay = 0;
-    ret = mca_base_var_register("opal", "opal", NULL, "abort_delay",
-                                "If nonzero, print out an identifying message when abort operation is invoked (hostname, PID of the process that called abort) and delay for that many seconds before exiting (a negative delay value means to never abort).  This allows attaching of a debugger before quitting the job.",
-                                 MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                 OPAL_INFO_LVL_5,
-                                 MCA_BASE_VAR_SCOPE_READONLY,
-                                 &opal_abort_delay);
+    int ret;
+    ret = mca_base_var_register("opal", "opal", "progress", "debug",
+                                "Set to non-zero to debug progress engine features",
+                                MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
+                                OPAL_INFO_LVL_8, MCA_BASE_VAR_SCOPE_LOCAL, &opal_progress_debug);
     if (0 > ret) {
         return ret;
     }
-
-    opal_abort_print_stack = false;
-    ret = mca_base_var_register("opal", "opal", NULL, "abort_print_stack",
-                                 "If nonzero, print out a stack trace when abort is invoked",
-                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0,
-                                /* If we do not have stack trace
-                                   capability, make this a constant
-                                   MCA variable */
-#if OPAL_WANT_PRETTY_PRINT_STACKTRACE
-                                 0,
-                                 OPAL_INFO_LVL_5,
-                                 MCA_BASE_VAR_SCOPE_READONLY,
-#else
-                                 MCA_BASE_VAR_FLAG_DEFAULT_ONLY,
-                                 OPAL_INFO_LVL_5,
-                                 MCA_BASE_VAR_SCOPE_CONSTANT,
 #endif
-                                 &opal_abort_print_stack);
-    if (0 > ret) {
-        return ret;
-    }
-
-    /* register the envar-forwarding params */
-    (void)mca_base_var_register ("opal", "mca", "base", "env_list",
-                                 "Set SHELL env variables",
-                                 MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_3,
-                                 MCA_BASE_VAR_SCOPE_READONLY, &mca_base_env_list);
-
-    mca_base_env_list_sep = MCA_BASE_ENV_LIST_SEP_DEFAULT;
-    (void)mca_base_var_register ("opal", "mca", "base", "env_list_delimiter",
-                                 "Set SHELL env variables delimiter. Default: semicolon ';'",
-                                 MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_3,
-                                 MCA_BASE_VAR_SCOPE_READONLY, &mca_base_env_list_sep);
-
-    /* Set OMPI_MCA_mca_base_env_list variable, it might not be set before
-     * if mca variable was taken from amca conf file. Need to set it
-     * here because mca_base_var_process_env_list is called from schizo_ompi.c
-     * only when this env variable was set.
-     */
-    if (NULL != mca_base_env_list) {
-        char *name = NULL;
-        (void) mca_base_var_env_name ("mca_base_env_list", &name);
-        if (NULL != name) {
-            opal_setenv(name, mca_base_env_list, false, &environ);
-            free(name);
-        }
-    }
-
-    /* Register internal MCA variable mca_base_env_list_internal. It can be set only during
-     * parsing of amca conf file and contains SHELL env variables specified via -x there.
-     * Its format is the same as for mca_base_env_list.
-     */
-    (void)mca_base_var_register ("opal", "mca", "base", "env_list_internal",
-            "Store SHELL env variables from amca conf file",
-            MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_INTERNAL, OPAL_INFO_LVL_3,
-            MCA_BASE_VAR_SCOPE_READONLY, &mca_base_env_list_internal);
-
-    /* Number of threads allowed in opal_progress. This might increase multithreaded performance. */
-    (void)mca_base_var_register ("opal", "opal", NULL, "max_thread_in_progress",
-            "Number of thread allowed in opal_progress. Default: 1",
-            MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_8,
-            MCA_BASE_VAR_SCOPE_READONLY, &opal_max_thread_in_progress);
-
-    /* The ddt engine has a few parameters */
-    ret = opal_datatype_register_params();
-    if (OPAL_SUCCESS != ret) {
-        return ret;
-    }
-
-    ret = opal_event_register_params();
-    if (OPAL_SUCCESS != ret) {
-        return ret;
-    }
 
-    opal_finalize_register_cleanup (opal_deregister_params);
+    opal_finalize_register_cleanup(opal_deregister_params);
 
     return OPAL_SUCCESS;
 }
diff --git a/opal/runtime/opal_params.h b/opal/runtime/opal_params.h
index 24b15973487..67fc2fb9e15 100644
--- a/opal/runtime/opal_params.h
+++ b/opal/runtime/opal_params.h
@@ -19,6 +19,7 @@
  * Copyright (c) 2015      Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2022      Advanced Micro Devices, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -27,61 +28,8 @@
  */
 
 #if !defined(OPAL_PARAMS_H)
-#define OPAL_PARAMS_H
+#    define OPAL_PARAMS_H
 
-extern char *opal_signal_string;
-extern char *opal_stacktrace_output_filename;
-extern char *opal_net_private_ipv4;
-extern char *opal_set_max_sys_limits;
-
-#if OPAL_ENABLE_TIMING
-extern char *opal_timing_sync_file;
-extern char *opal_timing_output;
-extern bool opal_timing_overhead;
-#endif
-
-OPAL_DECLSPEC extern int opal_initialized;
-OPAL_DECLSPEC extern bool opal_built_with_cuda_support;
-
-/**
- *  * Whether we want to enable CUDA GPU buffer send and receive support.
- *   */
-OPAL_DECLSPEC extern bool opal_cuda_support;
-
-/**
- *  * Whether we want to warn the user when libcuda is missing.
- *   */
-OPAL_DECLSPEC extern bool opal_warn_on_missing_libcuda;
-
-/**
- * Whether to use the "leave pinned" protocol or not (0 = no, 1 = yes,
- * -1 = determine at runtime).
- */
-OPAL_DECLSPEC extern int opal_leave_pinned;
-
-/**
- * Whether to use the "leave pinned pipeline" protocol or not.
- */
-OPAL_DECLSPEC extern bool opal_leave_pinned_pipeline;
-
-/**
- * Whether an abort operation should print out a stack trace or not.
- */
-OPAL_DECLSPEC extern bool opal_abort_print_stack;
-
-/**
- * Whether  abort operation  should  print  out an  identifying  message
- * (e.g., hostname  and PID)  and loop waiting  for a  debugger to
- * attach.  The value of the integer is how many seconds to wait:
- *
- * 0 = do not print the message and do not loop
- * negative value = print the message and loop forever
- * positive value = print the message and delay for that many seconds
- */
-OPAL_DECLSPEC extern int opal_abort_delay;
-
-#if OPAL_ENABLE_DEBUG
-extern bool opal_progress_debug;
-#endif
+#include "opal/runtime/opal_params_core.h"
 
 #endif
diff --git a/opal/runtime/opal_params_core.c b/opal/runtime/opal_params_core.c
new file mode 100644
index 00000000000..803c06fb9f4
--- /dev/null
+++ b/opal/runtime/opal_params_core.c
@@ -0,0 +1,534 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2014 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2008-2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2009      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2010-2014 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2014      Hochschule Esslingen.  All rights reserved.
+ * Copyright (c) 2015-2023 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2015      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2018-2022 Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2022      Advanced Micro Devices, Inc. All rights reserved.
+ * Copyright (c) 2022      Computer Architecture and VLSI Systems (CARV)
+ *                         Laboratory, ICS Forth. All rights reserved.
+ * Copyright (c) 2023      Triad National Security, LLC. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "opal_config.h"
+
+#include <signal.h>
+#include <time.h>
+
+#include "opal/constants.h"
+#include "opal/datatype/opal_datatype.h"
+#include "opal/mca/base/mca_base_var.h"
+#include "opal/mca/shmem/base/base.h"
+#include "opal/mca/threads/mutex.h"
+#include "opal/mca/threads/threads.h"
+#include "opal/runtime/opal.h"
+#include "opal/runtime/opal_params_core.h"
+#include "opal/util/opal_environ.h"
+#include "opal/util/printf.h"
+#include "opal/util/show_help.h"
+#include "opal/util/timings.h"
+#include "opal/util/argv.h"
+
+char *opal_signal_string = NULL;
+char *opal_stacktrace_output_filename = NULL;
+char *opal_net_private_ipv4 = NULL;
+char *opal_set_max_sys_limits = NULL;
+
+char *opal_var_dump_color[OPAL_VAR_DUMP_COLOR_KEY_COUNT] = {NULL};
+
+#if OPAL_ENABLE_TIMING
+char *opal_timing_sync_file = NULL;
+char *opal_timing_output = NULL;
+bool opal_timing_overhead = true;
+#endif
+
+bool opal_built_with_cuda_support = OPAL_INT_TO_BOOL(OPAL_CUDA_SUPPORT);
+bool opal_cuda_runtime_initialized = false;
+bool opal_cuda_support = false;
+bool opal_warn_on_missing_libcuda = true;
+
+bool opal_built_with_rocm_support = OPAL_INT_TO_BOOL(OPAL_ROCM_SUPPORT);
+bool opal_rocm_runtime_initialized = false;
+
+bool opal_built_with_ze_support = OPAL_INT_TO_BOOL(OPAL_ZE_SUPPORT);
+bool opal_ze_runtime_initialized = false;
+
+bool opal_accelerator_use_sync_memops = true;
+
+/**
+ * Globals imported from the OMPI layer.
+ */
+int opal_leave_pinned = -1;
+bool opal_leave_pinned_pipeline = false;
+bool opal_abort_print_stack = false;
+int opal_abort_delay = 0;
+
+int opal_max_thread_in_progress = 1;
+
+static bool opal_register_util_done = false;
+
+static char *opal_var_dump_color_string = NULL;
+
+static char *opal_var_dump_color_keys[OPAL_VAR_DUMP_COLOR_KEY_COUNT+1] = {
+    [OPAL_VAR_DUMP_COLOR_VAR_NAME] = "name",
+    [OPAL_VAR_DUMP_COLOR_VAR_VALUE] = "value",
+    [OPAL_VAR_DUMP_COLOR_VALID_VALUES] = "valid_values",
+    [OPAL_VAR_DUMP_COLOR_KEY_COUNT] = NULL
+};
+
+/**
+ * Local functions
+ */
+static int parse_color_string(char *color_string, char **key_names, int key_count,
+    char **values_out);
+
+static void opal_deregister_util_params(void)
+{
+    for (int i = 0; i < OPAL_VAR_DUMP_COLOR_KEY_COUNT; i++) {
+        free(opal_var_dump_color[i]);
+        opal_var_dump_color[i] = NULL;
+    }
+
+    /* The MCA variable system will be torn down shortly so reset the registered
+     * flag. */
+    opal_register_util_done = false;
+}
+
+int opal_register_util_params(void)
+{
+    int ret;
+    char *string = NULL, *tmp = NULL;
+
+    if (opal_register_util_done) {
+        return OPAL_SUCCESS;
+    }
+
+    opal_register_util_done = true;
+
+    /*
+     * This string is going to be used in opal/util/stacktrace.c
+     */
+    {
+        int j;
+        int signals[] = {
+#ifdef SIGABRT
+            SIGABRT,
+#endif
+#ifdef SIGBUS
+            SIGBUS,
+#endif
+#ifdef SIGFPE
+            SIGFPE,
+#endif
+#ifdef SIGSEGV
+            SIGSEGV,
+#endif
+            -1};
+        for (j = 0; signals[j] != -1; ++j) {
+            if (j == 0) {
+                ret = opal_asprintf(&string, "%d", signals[j]);
+            } else {
+                ret = opal_asprintf(&tmp, "%s,%d", string, signals[j]);
+                free(string);
+                string = tmp;
+            }
+
+            if (0 > ret) {
+                return OPAL_ERR_OUT_OF_RESOURCE;
+            }
+        }
+
+        opal_signal_string = string;
+        ret = mca_base_var_register(
+            "opal", "opal", NULL, "signal",
+            "Comma-delimited list of integer signal numbers to Open MPI to attempt to intercept.  "
+            "Upon receipt of the intercepted signal, Open MPI will display a stack trace and "
+            "abort.  Open MPI will *not* replace signals if handlers are already installed by the "
+            "time MPI_INIT is invoked.  Optionally append \":complain\" to any signal number in "
+            "the comma-delimited list to make Open MPI complain if it detects another signal "
+            "handler (and therefore does not insert its own).",
+            MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
+            MCA_BASE_VAR_SCOPE_LOCAL, &opal_signal_string);
+        free(string);
+        if (0 > ret) {
+            return ret;
+        }
+    }
+
+    /*
+     * Where should the stack trace output be directed
+     * This string is going to be used in opal/util/stacktrace.c
+     */
+    string = strdup("stderr");
+    opal_stacktrace_output_filename = string;
+    ret = mca_base_var_register(
+        "opal", "opal", NULL, "stacktrace_output",
+        "Specifies where the stack trace output stream goes.  "
+        "Accepts one of the following: none (disabled), stderr (default), stdout, file[:filename]. "
+        "  "
+        "If 'filename' is not specified, a default filename of 'stacktrace' is used.  "
+        "The 'filename' is appended with either '.PID' or '.RANK.PID', if RANK is available.  "
+        "The 'filename' can be an absolute path or a relative path to the current working "
+        "directory.",
+        MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
+        MCA_BASE_VAR_SCOPE_LOCAL, &opal_stacktrace_output_filename);
+    free(string);
+    if (0 > ret) {
+        return ret;
+    }
+
+#if OPAL_ENABLE_DEBUG
+    opal_debug_threads = false;
+    ret = mca_base_var_register("opal", "opal", "debug", "threads",
+                                "Debug thread usage within OPAL. Reports out "
+                                "when threads are acquired and released.",
+                                MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
+                                OPAL_INFO_LVL_8, MCA_BASE_VAR_SCOPE_LOCAL, &opal_debug_threads);
+    if (0 > ret) {
+        return ret;
+    }
+#endif
+
+    /* RFC1918 defines
+       - 10.0.0./8
+       - 172.16.0.0/12
+       - 192.168.0.0/16
+
+       RFC3330 also mentions
+       - 169.254.0.0/16 for DHCP onlink iff there's no DHCP server
+    */
+    opal_net_private_ipv4 = "10.0.0.0/8;172.16.0.0/12;192.168.0.0/16;169.254.0.0/16";
+    ret = mca_base_var_register(
+        "opal", "opal", "net", "private_ipv4",
+        "Semicolon-delimited list of CIDR notation entries specifying what networks are considered "
+        "\"private\" (default value based on RFC1918 and RFC3330)",
+        MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
+        MCA_BASE_VAR_SCOPE_ALL_EQ, &opal_net_private_ipv4);
+    if (0 > ret) {
+        return ret;
+    }
+
+    opal_set_max_sys_limits = NULL;
+    ret = mca_base_var_register(
+        "opal", "opal", NULL, "set_max_sys_limits",
+        "Set the specified system-imposed limits to the specified value, including \"unlimited\"."
+        "Supported params: core, filesize, maxmem, openfiles, stacksize, maxchildren",
+        MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
+        MCA_BASE_VAR_SCOPE_ALL_EQ, &opal_set_max_sys_limits);
+    if (0 > ret) {
+        return ret;
+    }
+
+    /* Var-dump color */
+
+    string = opal_argv_join_range(opal_var_dump_color_keys, 0, OPAL_VAR_DUMP_COLOR_KEY_COUNT, ',');
+    if (NULL == string) {
+        return OPAL_ERR_OUT_OF_RESOURCE;
+    }
+
+    ret = opal_asprintf(&tmp, "The colors to use when dumping MCA vars with color "
+        "(e.g. ompi_info). The format is a comma-delimited key=value list, where the "
+        "key is the attribute whose color to adjust, and the value is the ANSI color "
+        "code (see the ANSI X3.64 CSI SGR sequence). Available keys: %s.", string);
+    free(string);
+    string = tmp;
+    if (0 > ret) {
+        return OPAL_ERR_OUT_OF_RESOURCE;
+    }
+
+    /* Basic color options: 30=black, 31=red, 32=green,
+     * 33=yellow, 34=blue, 35=magenta, 36=cyan, 37=white
+     * https://en.wikipedia.org/wiki/ANSI_escape_code#Colors */
+    opal_var_dump_color_string = "name=34,value=32,valid_values=36";
+    ret = mca_base_var_register("opal", "opal", NULL, "var_dump_color", string,
+        MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_2, MCA_BASE_VAR_SCOPE_READONLY,
+        &opal_var_dump_color_string);
+    free(string);
+    if (0 > ret) {
+        return ret;
+    }
+
+    ret = parse_color_string(opal_var_dump_color_string, opal_var_dump_color_keys,
+        OPAL_VAR_DUMP_COLOR_KEY_COUNT, opal_var_dump_color);
+    if (OPAL_SUCCESS != ret) {
+        return ret;
+    }
+
+    /* CUDA support */
+
+    ret = mca_base_var_register("opal", "opal", NULL, "built_with_cuda_support",
+                                "Whether CUDA GPU buffer support is built into library or not",
+                                MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_DEFAULT_ONLY,
+                                OPAL_INFO_LVL_4, MCA_BASE_VAR_SCOPE_CONSTANT,
+                                &opal_built_with_cuda_support);
+    if (0 > ret) {
+        return ret;
+    }
+
+    /* Current default is to enable CUDA support if it is built into library */
+    opal_cuda_support = opal_built_with_cuda_support;
+    ret = mca_base_var_register("opal", "opal", NULL, "cuda_support",
+                                "Whether CUDA GPU buffer support is enabled or not",
+                                MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
+                                OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_ALL_EQ, &opal_cuda_support);
+    if (0 > ret) {
+        return ret;
+    }
+
+    opal_warn_on_missing_libcuda = true;
+    ret = mca_base_var_register(
+        "opal", "opal", NULL, "warn_on_missing_libcuda",
+        "Whether to print a message when CUDA support is enabled but libcuda is not found",
+        MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
+        MCA_BASE_VAR_SCOPE_ALL_EQ, &opal_warn_on_missing_libcuda);
+    if (0 > ret) {
+        return ret;
+    }
+
+    /* Leave pinned parameter */
+    opal_leave_pinned = -1;
+    ret = mca_base_var_register(
+        "ompi", "mpi", NULL, "leave_pinned",
+        "Whether to use the \"leave pinned\" protocol or not.  Enabling this setting can help "
+        "bandwidth performance when repeatedly sending and receiving large messages with the same "
+        "buffers over RDMA-based networks (false = do not use \"leave pinned\" protocol, true = "
+        "use \"leave pinned\" protocol, auto = allow network to choose at runtime).",
+        MCA_BASE_VAR_TYPE_INT, &mca_base_var_enum_auto_bool, 0, 0, OPAL_INFO_LVL_9,
+        MCA_BASE_VAR_SCOPE_READONLY, &opal_leave_pinned);
+    mca_base_var_register_synonym(ret, "opal", "opal", NULL, "leave_pinned",
+                                  MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
+
+    opal_leave_pinned_pipeline = false;
+    ret = mca_base_var_register("ompi", "mpi", NULL, "leave_pinned_pipeline",
+                                "Whether to use the \"leave pinned pipeline\" protocol or not.",
+                                MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9,
+                                MCA_BASE_VAR_SCOPE_READONLY, &opal_leave_pinned_pipeline);
+    mca_base_var_register_synonym(ret, "opal", "opal", NULL, "leave_pinned_pipeline",
+                                  MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
+
+    if (opal_leave_pinned > 0 && opal_leave_pinned_pipeline) {
+        opal_leave_pinned_pipeline = 0;
+        opal_show_help("help-opal-runtime.txt", "mpi-params:leave-pinned-and-pipeline-selected",
+                       true);
+    }
+
+    opal_warn_on_fork = true;
+    (void) mca_base_var_register("ompi", "mpi", NULL, "warn_on_fork",
+                                 "If nonzero, issue a warning if program forks under conditions "
+                                 "that could cause system errors",
+                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9,
+                                 MCA_BASE_VAR_SCOPE_READONLY, &opal_warn_on_fork);
+
+    opal_abort_delay = 0;
+    ret = mca_base_var_register(
+        "opal", "opal", NULL, "abort_delay",
+        "If nonzero, print out an identifying message when abort operation is invoked (hostname, "
+        "PID of the process that called abort) and delay for that many seconds before exiting (a "
+        "negative delay value means to never abort).  This allows attaching of a debugger before "
+        "quitting the job.",
+        MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_READONLY,
+        &opal_abort_delay);
+    if (0 > ret) {
+        return ret;
+    }
+
+    opal_abort_print_stack = false;
+    ret = mca_base_var_register("opal", "opal", NULL, "abort_print_stack",
+                                "If nonzero, print out a stack trace when abort is invoked",
+                                MCA_BASE_VAR_TYPE_BOOL, NULL, 0,
+    /* If we do not have stack trace
+       capability, make this a constant
+       MCA variable */
+#if OPAL_WANT_PRETTY_PRINT_STACKTRACE
+                                0, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_READONLY,
+#else
+                                MCA_BASE_VAR_FLAG_DEFAULT_ONLY, OPAL_INFO_LVL_5,
+                                MCA_BASE_VAR_SCOPE_CONSTANT,
+#endif
+                                &opal_abort_print_stack);
+    if (0 > ret) {
+        return ret;
+    }
+
+    /* register the envar-forwarding params */
+    (void) mca_base_var_register("opal", "mca", "base", "env_list", "Set SHELL env variables",
+                                 MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_3,
+                                 MCA_BASE_VAR_SCOPE_READONLY, &mca_base_env_list);
+
+    mca_base_env_list_sep = MCA_BASE_ENV_LIST_SEP_DEFAULT;
+    (void) mca_base_var_register("opal", "mca", "base", "env_list_delimiter",
+                                 "Set SHELL env variables delimiter. Default: semicolon ';'",
+                                 MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_3,
+                                 MCA_BASE_VAR_SCOPE_READONLY, &mca_base_env_list_sep);
+
+    /* Set OMPI_MCA_mca_base_env_list variable, it might not be set before
+     * if mca variable was taken from amca conf file. Need to set it
+     * here because mca_base_var_process_env_list is called from schizo_ompi.c
+     * only when this env variable was set.
+     */
+    if (NULL != mca_base_env_list) {
+        char *name = NULL;
+        (void) mca_base_var_env_name("mca_base_env_list", &name);
+        if (NULL != name) {
+            opal_setenv(name, mca_base_env_list, false, &environ);
+            free(name);
+        }
+    }
+
+    /* Register internal MCA variable mca_base_env_list_internal. It can be set only during
+     * parsing of amca conf file and contains SHELL env variables specified via -x there.
+     * Its format is the same as for mca_base_env_list.
+     */
+    (void) mca_base_var_register("opal", "mca", "base", "env_list_internal",
+                                 "Store SHELL env variables from amca conf file",
+                                 MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_INTERNAL,
+                                 OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_READONLY,
+                                 &mca_base_env_list_internal);
+
+    /* Number of threads allowed in opal_progress. This might increase multithreaded performance. */
+    (void) mca_base_var_register("opal", "opal", NULL, "max_thread_in_progress",
+                                 "Number of thread allowed in opal_progress. Default: 1",
+                                 MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_8,
+                                 MCA_BASE_VAR_SCOPE_READONLY, &opal_max_thread_in_progress);
+
+    /* Use sync_memops functionality with accelerator codes or deploy
+       alternative path using IPC events to ensure consistency */
+    opal_accelerator_use_sync_memops = true;
+    (void) mca_base_var_register("ompi", "mpi", NULL, "accelerator_use_sync_memops",
+                                 "whether to use synchronized accelerator memory based logical path for"
+                                 " GPU memory or use alternative path using IPC events",
+                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9,
+                                 MCA_BASE_VAR_SCOPE_READONLY, &opal_accelerator_use_sync_memops);
+
+    opal_finalize_register_cleanup(opal_deregister_util_params);
+
+    return OPAL_SUCCESS;
+}
+
+/* Parses a color 'string' and extracts the 'code' for each 'key'
+ * The string is in the format of 'key1=code1:key2=code2:...'
+ * Example: opal_var_dump_color MCA parameter
+ *
+ * color_string: The string containing the key=code sequences to parse
+ * key_names: The keys to extract (e.g. key1, key2)
+ * key_count: The length of key_names
+ * values_out: The extracted values for the keys, 1-1 with key_names
+ *   ATTENTION: Make sure it's at least as large as key_names...
+ */
+static int parse_color_string(char *color_string, char **key_names,
+        int key_count, char **values_out) {
+
+    char **tokens = NULL, **kv = NULL;
+    int return_code = OPAL_SUCCESS;
+
+    for (int k = 0; k < key_count; k++) {
+        values_out[k] = NULL;
+    }
+
+    if (NULL != color_string) {
+        tokens = opal_argv_split(color_string, ',');
+        if (NULL == tokens) {
+            return_code = OPAL_ERR_OUT_OF_RESOURCE;
+            goto end;
+        }
+    }
+
+    for (int i = 0; tokens && tokens[i] != NULL; i++) {
+        kv = opal_argv_split(tokens[i], '=');
+        if (NULL == kv) {
+            return_code = OPAL_ERR_OUT_OF_RESOURCE;
+            goto end;
+        }
+
+        // Expected format of token: key=code
+        if (opal_argv_count(kv) != 2) {
+            opal_show_help("help-opal-runtime.txt", "mpi-params:var_dump_color:format-error",
+                true, tokens[i]);
+            goto skip_token;
+        }
+
+        bool key_found = false;
+
+        // Look for key name and store value in respective position
+        for (int k = 0; k < key_count; ++k) {
+            if (strcasecmp(key_names[k], kv[0]) == 0) {
+                int ret = opal_asprintf(&values_out[k], "\033[%sm", kv[1]);
+                if (ret < 0) {
+                    return_code = OPAL_ERR_OUT_OF_RESOURCE;
+                    goto end;
+                }
+
+                key_found = true;
+                break;
+            }
+        }
+
+        if (!key_found) {
+            char *valid_keys = opal_argv_join_range(key_names, 0, key_count, ',');
+            if (NULL == valid_keys) {
+                return_code = OPAL_ERR_OUT_OF_RESOURCE;
+                goto end;
+            }
+
+            opal_show_help("help-opal-runtime.txt", "mpi-params:var_dump_color:unknown-key",
+                true, kv[0], tokens[i], valid_keys);
+            free(valid_keys);
+        }
+
+        skip_token:
+
+        opal_argv_free(kv);
+        kv = NULL;
+    }
+
+    // Set values for keys not in the MCA parameter to ""
+    for (int k = 0; k < key_count; k++) {
+        if (NULL == values_out[k]) {
+            values_out[k] = strdup(""); // needs to be free-able
+
+            if (NULL == values_out[k]) {
+                return_code = OPAL_ERR_OUT_OF_RESOURCE;
+                goto end;
+            }
+        }
+    }
+
+    end:
+
+    opal_argv_free(tokens);
+    opal_argv_free(kv);
+
+    if (return_code != OPAL_SUCCESS) {
+        for (int k = 0; k < key_count; k++) {
+            free(values_out[k]);
+            values_out[k] = NULL;
+        }
+    }
+
+    return return_code;
+}
diff --git a/opal/runtime/opal_params_core.h b/opal/runtime/opal_params_core.h
new file mode 100644
index 00000000000..b90d3fc2961
--- /dev/null
+++ b/opal/runtime/opal_params_core.h
@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2008-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2009      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2010-2013 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2014      Hochschule Esslingen.  All rights reserved.
+ * Copyright (c) 2015      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2022      Advanced Micro Devices, Inc. All rights reserved.
+ * Copyright (c) 2022      Computer Architecture and VLSI Systems (CARV)
+ *                         Laboratory, ICS Forth. All rights reserved.
+ * Copyright (c) 2023      Triad National Security, LLC. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#if !defined(OPAL_PARAMS_UTIL_H)
+#    define OPAL_PARAMS_UTIL_H
+
+extern char *opal_signal_string;
+extern char *opal_stacktrace_output_filename;
+extern char *opal_net_private_ipv4;
+extern char *opal_set_max_sys_limits;
+
+/* Supported color configuration keys
+ * for dumping MCA variables with colors */
+typedef enum {
+    OPAL_VAR_DUMP_COLOR_VAR_NAME = 0,
+    OPAL_VAR_DUMP_COLOR_VAR_VALUE = 1,
+    OPAL_VAR_DUMP_COLOR_VALID_VALUES = 2,
+    OPAL_VAR_DUMP_COLOR_KEY_COUNT
+} opal_var_dump_color_key_t;
+
+extern char *opal_var_dump_color[OPAL_VAR_DUMP_COLOR_KEY_COUNT];
+
+#    if OPAL_ENABLE_TIMING
+extern char *opal_timing_sync_file;
+extern char *opal_timing_output;
+extern bool opal_timing_overhead;
+#    endif
+
+OPAL_DECLSPEC extern int opal_initialized;
+OPAL_DECLSPEC extern bool opal_built_with_cuda_support;
+OPAL_DECLSPEC extern bool opal_built_with_rocm_support;
+OPAL_DECLSPEC extern bool opal_built_with_ze_support;
+
+/**
+ *  * Whether we want to enable CUDA GPU buffer send and receive support.
+ *   */
+OPAL_DECLSPEC extern bool opal_cuda_support;
+
+/**
+ * Whether cuda runtime support is initialized or not.
+ */
+OPAL_DECLSPEC extern bool opal_cuda_runtime_initialized;
+
+/**
+ * Whether rocm runtime support is initialized or not.
+ */
+OPAL_DECLSPEC extern bool opal_rocm_runtime_initialized;
+
+/**
+ * Whether ze runtime support is initialized or not.
+ */
+OPAL_DECLSPEC extern bool opal_ze_runtime_initialized;
+
+/**
+ *  * Whether we want to warn the user when libcuda is missing.
+ *   */
+OPAL_DECLSPEC extern bool opal_warn_on_missing_libcuda;
+
+
+/**
+ *  *whether we want to use sync_memops feature with accelerators
+ *   or deploy alternative logic with IPC events 
+ * */
+OPAL_DECLSPEC extern bool opal_accelerator_use_sync_memops;
+
+/**
+ * Whether to use the "leave pinned" protocol or not (0 = no, 1 = yes,
+ * -1 = determine at runtime).
+ */
+OPAL_DECLSPEC extern int opal_leave_pinned;
+
+/**
+ * Whether to use the "leave pinned pipeline" protocol or not.
+ */
+OPAL_DECLSPEC extern bool opal_leave_pinned_pipeline;
+
+/**
+ * Whether an abort operation should print out a stack trace or not.
+ */
+OPAL_DECLSPEC extern bool opal_abort_print_stack;
+
+/**
+ * Whether  abort operation  should  print  out an  identifying  message
+ * (e.g., hostname  and PID)  and loop waiting  for a  debugger to
+ * attach.  The value of the integer is how many seconds to wait:
+ *
+ * 0 = do not print the message and do not loop
+ * negative value = print the message and loop forever
+ * positive value = print the message and delay for that many seconds
+ */
+OPAL_DECLSPEC extern int opal_abort_delay;
+
+
+/**
+ * Register OPAL MCA parameters from the core
+ */
+OPAL_DECLSPEC int opal_register_util_params(void);
+
+#endif
diff --git a/opal/runtime/opal_progress.c b/opal/runtime/opal_progress.c
index 0ba1e1ff007..7bca660b9d6 100644
--- a/opal/runtime/opal_progress.c
+++ b/opal/runtime/opal_progress.c
@@ -27,17 +27,17 @@
 
 #include "opal_config.h"
 
-#include "opal/runtime/opal_progress.h"
-#include "opal/util/event.h"
-#include "opal/mca/base/mca_base_var.h"
 #include "opal/constants.h"
+#include "opal/mca/base/mca_base_var.h"
+#include "opal/mca/threads/threads.h"
 #include "opal/mca/timer/base/base.h"
-#include "opal/util/output.h"
-#include "opal/runtime/opal_params.h"
 #include "opal/runtime/opal.h"
-#include "opal/mca/threads/threads.h"
+#include "opal/runtime/opal_params.h"
+#include "opal/runtime/opal_progress.h"
+#include "opal/util/event.h"
+#include "opal/util/output.h"
 
-#define OPAL_PROGRESS_USE_TIMERS (OPAL_TIMER_CYCLE_SUPPORTED || OPAL_TIMER_USEC_SUPPORTED)
+#define OPAL_PROGRESS_USE_TIMERS       (OPAL_TIMER_CYCLE_SUPPORTED || OPAL_TIMER_USEC_SUPPORTED)
 #define OPAL_PROGRESS_ONLY_USEC_NATIVE (OPAL_TIMER_USEC_NATIVE && !OPAL_TIMER_CYCLE_NATIVE)
 
 #if OPAL_ENABLE_DEBUG
@@ -50,7 +50,6 @@ bool opal_progress_debug = false;
 static int opal_progress_event_flag = OPAL_EVLOOP_ONCE | OPAL_EVLOOP_NONBLOCK;
 int opal_progress_spin_count = 10000;
 
-
 /*
  * Local variables
  */
@@ -86,40 +85,42 @@ static int debug_output = -1;
 #endif
 
 /**
- * Fake callback used for threading purpose when one thread
- * progesses callbacks while another unregister somes. The root
+ * Fake callback used for threading purposes when one thread
+ * progresses callbacks while another unregisters some. The root
  * of the problem is that we allow modifications of the callback
  * array directly from the callbacks themselves. Now if
  * writing a pointer is atomic, we should not have any more
  * problems.
  */
-static int fake_cb(void) { return 0; }
+static int fake_cb(void)
+{
+    return 0;
+}
 
-static int _opal_progress_unregister (opal_progress_callback_t cb, volatile opal_progress_callback_t *callback_array,
-                                      size_t *callback_array_len);
+static int _opal_progress_unregister(opal_progress_callback_t cb,
+                                     volatile opal_progress_callback_t *callback_array,
+                                     size_t *callback_array_len);
 
-static void opal_progress_finalize (void)
+static void opal_progress_finalize(void)
 {
     /* free memory associated with the callbacks */
     opal_atomic_lock(&progress_lock);
 
     callbacks_len = 0;
     callbacks_size = 0;
-    free ((void *) callbacks);
+    free((void *) callbacks);
     callbacks = NULL;
 
     callbacks_lp_len = 0;
     callbacks_lp_size = 0;
-    free ((void *) callbacks_lp);
+    free((void *) callbacks_lp);
     callbacks_lp = NULL;
 
     opal_atomic_unlock(&progress_lock);
 }
 
-
 /* init the progress engine - called from orte_init */
-int
-opal_progress_init(void)
+int opal_progress_init(void)
 {
     /* reentrant issues */
     opal_atomic_lock_init(&progress_lock, OPAL_ATOMIC_LOCK_UNLOCKED);
@@ -129,41 +130,40 @@ opal_progress_init(void)
 
 #if OPAL_ENABLE_DEBUG
     if (opal_progress_debug) {
-       debug_output = opal_output_open(NULL);
+        debug_output = opal_output_open(NULL);
     }
 #endif
 
     callbacks_size = callbacks_lp_size = 8;
 
-    callbacks = malloc (callbacks_size * sizeof (callbacks[0]));
-    callbacks_lp = malloc (callbacks_lp_size * sizeof (callbacks_lp[0]));
+    callbacks = malloc(callbacks_size * sizeof(callbacks[0]));
+    callbacks_lp = malloc(callbacks_lp_size * sizeof(callbacks_lp[0]));
 
     if (NULL == callbacks || NULL == callbacks_lp) {
-        free ((void *) callbacks);
-        free ((void *) callbacks_lp);
+        free((void *) callbacks);
+        free((void *) callbacks_lp);
         callbacks_size = callbacks_lp_size = 0;
         callbacks = callbacks_lp = NULL;
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
-    for (size_t i = 0 ; i < callbacks_size ; ++i) {
+    for (size_t i = 0; i < callbacks_size; ++i) {
         callbacks[i] = fake_cb;
     }
 
-    for (size_t i = 0 ; i < callbacks_lp_size ; ++i) {
+    for (size_t i = 0; i < callbacks_lp_size; ++i) {
         callbacks_lp[i] = fake_cb;
     }
 
-    OPAL_OUTPUT((debug_output, "progress: initialized event flag to: %x",
-                 opal_progress_event_flag));
+    OPAL_OUTPUT(
+        (debug_output, "progress: initialized event flag to: %x", opal_progress_event_flag));
     OPAL_OUTPUT((debug_output, "progress: initialized yield_when_idle to: %s",
                  opal_progress_yield_when_idle ? "true" : "false"));
-    OPAL_OUTPUT((debug_output, "progress: initialized num users to: %d",
-                 num_event_users));
-    OPAL_OUTPUT((debug_output, "progress: initialized poll rate to: %ld",
-                 (long) event_progress_delta));
+    OPAL_OUTPUT((debug_output, "progress: initialized num users to: %d", num_event_users));
+    OPAL_OUTPUT(
+        (debug_output, "progress: initialized poll rate to: %ld", (long) event_progress_delta));
 
-    opal_finalize_register_cleanup (opal_progress_finalize);
+    opal_finalize_register_cleanup(opal_progress_finalize);
 
     return OPAL_SUCCESS;
 }
@@ -173,29 +173,27 @@ static int opal_progress_events(void)
     static opal_atomic_int32_t lock = 0;
     int events = 0;
 
-    if( opal_progress_event_flag != 0 && !OPAL_THREAD_SWAP_32(&lock, 1) ) {
+    if (opal_progress_event_flag != 0 && !OPAL_THREAD_SWAP_32(&lock, 1)) {
 #if OPAL_PROGRESS_USE_TIMERS
-#if OPAL_PROGRESS_ONLY_USEC_NATIVE
+#    if OPAL_PROGRESS_ONLY_USEC_NATIVE
         opal_timer_t now = opal_timer_base_get_usec();
-#else
+#    else
         opal_timer_t now = opal_timer_base_get_cycles();
-#endif  /* OPAL_PROGRESS_ONLY_USEC_NATIVE */
-    /* trip the event library if we've reached our tick rate and we are
-       enabled */
-        if (now - event_progress_last_time > event_progress_delta ) {
-                event_progress_last_time = (num_event_users > 0) ?
-                    now - event_progress_delta : now;
-
-                events += opal_event_loop(opal_sync_event_base, opal_progress_event_flag);
+#    endif /* OPAL_PROGRESS_ONLY_USEC_NATIVE */
+        /* trip the event library if we've reached our tick rate and we are
+           enabled */
+        if (now - event_progress_last_time > event_progress_delta) {
+            event_progress_last_time = (num_event_users > 0) ? now - event_progress_delta : now;
+
+            events += opal_event_loop(opal_sync_event_base, opal_progress_event_flag);
         }
 
 #else /* OPAL_PROGRESS_USE_TIMERS */
-    /* trip the event library if we've reached our tick rate and we are
-       enabled */
-        if (OPAL_THREAD_ADD_FETCH32(&event_progress_counter, -1) <= 0 ) {
-                event_progress_counter =
-                    (num_event_users > 0) ? 0 : event_progress_delta;
-                events += opal_event_loop(opal_sync_event_base, opal_progress_event_flag);
+        /* trip the event library if we've reached our tick rate and we are
+           enabled */
+        if (OPAL_THREAD_ADD_FETCH32(&event_progress_counter, -1) <= 0) {
+            event_progress_counter = (num_event_users > 0) ? 0 : event_progress_delta;
+            events += opal_event_loop(opal_sync_event_base, opal_progress_event_flag);
         }
 #endif /* OPAL_PROGRESS_USE_TIMERS */
         lock = 0;
@@ -206,24 +204,23 @@ static int opal_progress_events(void)
 
 /*
  * Progress the event library and any functions that have registered to
- * be called.  We don't propogate errors from the progress functions,
+ * be called.  We don't propagate errors from the progress functions,
  * so no action is taken if they return failures.  The functions are
  * expected to return the number of events progressed, to determine
  * whether or not we should yield the CPU during MPI progress.
- * This is only losely tracked, as an error return can cause the number
+ * This is only loosely tracked, as an error return can cause the number
  * of progressed events to appear lower than it actually is.  We don't
  * care, as the cost of that happening is far outweighed by the cost
  * of the if checks (they were resulting in bad pipe stalling behavior)
  */
-void
-opal_progress(void)
+int opal_progress(void)
 {
     static uint32_t num_calls = 0;
     size_t i;
     int events = 0;
 
     /* progress all registered callbacks */
-    for (i = 0 ; i < callbacks_len ; ++i) {
+    for (i = 0; i < callbacks_len; ++i) {
         events += (callbacks[i])();
     }
 
@@ -234,7 +231,7 @@ opal_progress(void)
      * it's not a problem.
      */
     if (((num_calls++) & 0x7) == 0) {
-        for (i = 0 ; i < callbacks_lp_len ; ++i) {
+        for (i = 0; i < callbacks_lp_len; ++i) {
             events += (callbacks_lp[i])();
         }
 
@@ -253,11 +250,11 @@ opal_progress(void)
          */
         opal_thread_yield();
     }
-}
 
+    return events;
+}
 
-int
-opal_progress_set_event_flag(int flag)
+int opal_progress_set_event_flag(int flag)
 {
     int tmp = opal_progress_event_flag;
     opal_progress_event_flag = flag;
@@ -267,9 +264,7 @@ opal_progress_set_event_flag(int flag)
     return tmp;
 }
 
-
-void
-opal_progress_event_users_increment(void)
+void opal_progress_event_users_increment(void)
 {
 #if OPAL_ENABLE_DEBUG
     int32_t val;
@@ -277,7 +272,7 @@ opal_progress_event_users_increment(void)
 
     OPAL_OUTPUT((debug_output, "progress: event_users_increment setting count to %d", val));
 #else
-    (void)opal_atomic_add_fetch_32(&num_event_users, 1);
+    (void) opal_atomic_add_fetch_32(&num_event_users, 1);
 #endif
 
 #if OPAL_PROGRESS_USE_TIMERS
@@ -289,53 +284,47 @@ opal_progress_event_users_increment(void)
 #endif
 }
 
-
-void
-opal_progress_event_users_decrement(void)
+void opal_progress_event_users_decrement(void)
 {
-#if OPAL_ENABLE_DEBUG || ! OPAL_PROGRESS_USE_TIMERS
+#if OPAL_ENABLE_DEBUG || !OPAL_PROGRESS_USE_TIMERS
     int32_t val;
     val = opal_atomic_sub_fetch_32(&num_event_users, 1);
 
     OPAL_OUTPUT((debug_output, "progress: event_users_decrement setting count to %d", val));
 #else
-    (void)opal_atomic_sub_fetch_32(&num_event_users, 1);
+    (void) opal_atomic_sub_fetch_32(&num_event_users, 1);
 #endif
 
 #if !OPAL_PROGRESS_USE_TIMERS
-   /* start now in delaying if it's easy */
-   if (val >= 0) {
-       event_progress_counter = event_progress_delta;
-   }
+    /* start now in delaying if it's easy */
+    if (val >= 0) {
+        event_progress_counter = event_progress_delta;
+    }
 #endif
 }
 
-
-bool
-opal_progress_set_yield_when_idle(bool yieldopt)
+bool opal_progress_set_yield_when_idle(bool yieldopt)
 {
     bool tmp = opal_progress_yield_when_idle;
     opal_progress_yield_when_idle = (yieldopt) ? 1 : 0;
 
     OPAL_OUTPUT((debug_output, "progress: progress_set_yield_when_idle to %s",
-                                    opal_progress_yield_when_idle ? "true" : "false"));
+                 opal_progress_yield_when_idle ? "true" : "false"));
 
     return tmp;
 }
 
-
-void
-opal_progress_set_event_poll_rate(int polltime)
+void opal_progress_set_event_poll_rate(int polltime)
 {
     OPAL_OUTPUT((debug_output, "progress: progress_set_event_poll_rate(%d)", polltime));
 
 #if OPAL_PROGRESS_USE_TIMERS
     event_progress_delta = 0;
-#  if OPAL_PROGRESS_ONLY_USEC_NATIVE
+#    if OPAL_PROGRESS_ONLY_USEC_NATIVE
     event_progress_last_time = opal_timer_base_get_usec();
-#  else
+#    else
     event_progress_last_time = opal_timer_base_get_cycles();
-#  endif
+#    endif
 #else
     event_progress_counter = event_progress_delta = 0;
 #endif
@@ -364,10 +353,10 @@ opal_progress_set_event_poll_rate(int polltime)
 #endif
 }
 
-static int opal_progress_find_cb (opal_progress_callback_t cb, volatile opal_progress_callback_t *cbs,
-                                     size_t cbs_len)
+static int opal_progress_find_cb(opal_progress_callback_t cb,
+                                 volatile opal_progress_callback_t *cbs, size_t cbs_len)
 {
-    for (size_t i = 0 ; i < cbs_len ; ++i) {
+    for (size_t i = 0; i < cbs_len; ++i) {
         if (cbs[i] == cb) {
             return (int) i;
         }
@@ -376,12 +365,13 @@ static int opal_progress_find_cb (opal_progress_callback_t cb, volatile opal_pro
     return OPAL_ERR_NOT_FOUND;
 }
 
-static int _opal_progress_register (opal_progress_callback_t cb, volatile opal_progress_callback_t **cbs,
-                                    size_t *cbs_size, size_t *cbs_len)
+static int _opal_progress_register(opal_progress_callback_t cb,
+                                   volatile opal_progress_callback_t **cbs, size_t *cbs_size,
+                                   size_t *cbs_len)
 {
     int ret = OPAL_SUCCESS;
 
-    if (OPAL_ERR_NOT_FOUND != opal_progress_find_cb (cb, *cbs, *cbs_len)) {
+    if (OPAL_ERR_NOT_FOUND != opal_progress_find_cb(cb, *cbs, *cbs_len)) {
         return OPAL_SUCCESS;
     }
 
@@ -389,73 +379,75 @@ static int _opal_progress_register (opal_progress_callback_t cb, volatile opal_p
     if (*cbs_len + 1 > *cbs_size) {
         opal_progress_callback_t *tmp, *old;
 
-        tmp = (opal_progress_callback_t *) malloc (sizeof (tmp[0]) * 2 * *cbs_size);
+        tmp = (opal_progress_callback_t *) malloc(sizeof(tmp[0]) * 2 * *cbs_size);
         if (tmp == NULL) {
             return OPAL_ERR_TEMP_OUT_OF_RESOURCE;
         }
 
         if (*cbs) {
             /* copy old callbacks */
-            memcpy (tmp, (void *) *cbs, sizeof(tmp[0]) * *cbs_size);
+            memcpy(tmp, (void *) *cbs, sizeof(tmp[0]) * *cbs_size);
         }
 
-        for (size_t i = *cbs_len ; i < 2 * *cbs_size ; ++i) {
+        for (size_t i = *cbs_len; i < 2 * *cbs_size; ++i) {
             tmp[i] = fake_cb;
         }
 
-        opal_atomic_wmb ();
+        opal_atomic_wmb();
 
         /* swap out callback array */
-        old = (opal_progress_callback_t *) opal_atomic_swap_ptr ((opal_atomic_intptr_t *) cbs, (intptr_t) tmp);
+        old = (opal_progress_callback_t *) opal_atomic_swap_ptr((opal_atomic_intptr_t *) cbs,
+                                                                (intptr_t) tmp);
 
-        opal_atomic_wmb ();
+        opal_atomic_wmb();
 
-        free (old);
+        free(old);
         *cbs_size *= 2;
     }
 
     cbs[0][*cbs_len] = cb;
     ++*cbs_len;
 
-    opal_atomic_wmb ();
+    opal_atomic_wmb();
 
     return ret;
 }
 
-int opal_progress_register (opal_progress_callback_t cb)
+int opal_progress_register(opal_progress_callback_t cb)
 {
     int ret;
 
     opal_atomic_lock(&progress_lock);
 
-    (void) _opal_progress_unregister (cb, callbacks_lp, &callbacks_lp_len);
+    (void) _opal_progress_unregister(cb, callbacks_lp, &callbacks_lp_len);
 
-    ret = _opal_progress_register (cb, &callbacks, &callbacks_size, &callbacks_len);
+    ret = _opal_progress_register(cb, &callbacks, &callbacks_size, &callbacks_len);
 
     opal_atomic_unlock(&progress_lock);
 
     return ret;
 }
 
-int opal_progress_register_lp (opal_progress_callback_t cb)
+int opal_progress_register_lp(opal_progress_callback_t cb)
 {
     int ret;
 
     opal_atomic_lock(&progress_lock);
 
-    (void) _opal_progress_unregister (cb, callbacks, &callbacks_len);
+    (void) _opal_progress_unregister(cb, callbacks, &callbacks_len);
 
-    ret = _opal_progress_register (cb, &callbacks_lp, &callbacks_lp_size, &callbacks_lp_len);
+    ret = _opal_progress_register(cb, &callbacks_lp, &callbacks_lp_size, &callbacks_lp_len);
 
     opal_atomic_unlock(&progress_lock);
 
     return ret;
 }
 
-static int _opal_progress_unregister (opal_progress_callback_t cb, volatile opal_progress_callback_t *callback_array,
-                                      size_t *callback_array_len)
+static int _opal_progress_unregister(opal_progress_callback_t cb,
+                                     volatile opal_progress_callback_t *callback_array,
+                                     size_t *callback_array_len)
 {
-    int ret = opal_progress_find_cb (cb, callback_array, *callback_array_len);
+    int ret = opal_progress_find_cb(cb, callback_array, *callback_array_len);
     if (OPAL_ERR_NOT_FOUND == ret) {
         return ret;
     }
@@ -464,10 +456,11 @@ static int _opal_progress_unregister (opal_progress_callback_t cb, volatile opal
        is 0, we're not goig to do anything interesting anyway, so
        skip.  If callbacks_len is 1, it will soon be 0, so no need to
        do any repacking. */
-    for (size_t i = (size_t) ret ; i < *callback_array_len - 1 ; ++i) {
+    for (size_t i = (size_t) ret; i < *callback_array_len - 1; ++i) {
         /* copy callbacks atomically since another thread may be in
          * opal_progress(). */
-        (void) opal_atomic_swap_ptr ((opal_atomic_intptr_t *) (callback_array + i), (intptr_t) callback_array[i+1]);
+        (void) opal_atomic_swap_ptr((opal_atomic_intptr_t *) (callback_array + i),
+                                    (intptr_t) callback_array[i + 1]);
     }
 
     --*callback_array_len;
@@ -476,18 +469,18 @@ static int _opal_progress_unregister (opal_progress_callback_t cb, volatile opal
     return OPAL_SUCCESS;
 }
 
-int opal_progress_unregister (opal_progress_callback_t cb)
+int opal_progress_unregister(opal_progress_callback_t cb)
 {
     int ret;
 
     opal_atomic_lock(&progress_lock);
 
-    ret = _opal_progress_unregister (cb, callbacks, &callbacks_len);
+    ret = _opal_progress_unregister(cb, callbacks, &callbacks_len);
 
     if (OPAL_SUCCESS != ret) {
         /* if not in the high-priority array try to remove from the lp array.
          * a callback will never be in both. */
-        ret = _opal_progress_unregister (cb, callbacks_lp, &callbacks_lp_len);
+        ret = _opal_progress_unregister(cb, callbacks_lp, &callbacks_lp_len);
     }
 
     opal_atomic_unlock(&progress_lock);
diff --git a/opal/runtime/opal_progress.h b/opal/runtime/opal_progress.h
index 576532c64c0..86a0f8add50 100644
--- a/opal/runtime/opal_progress.h
+++ b/opal/runtime/opal_progress.h
@@ -1,4 +1,5 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- *//*
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
@@ -56,26 +57,34 @@ OPAL_DECLSPEC int opal_progress_init(void);
  * opal_progress_event_users_delete()) or the time since the last call
  * into the event library is greater than the progress tick rate (by
  * default, 10ms).
+ *
+ * Returns 0 if no progress has been observed, non-zero otherwise.
  */
-OPAL_DECLSPEC void opal_progress(void);
-
+OPAL_DECLSPEC int opal_progress(void);
 
 /**
  * Control how the event library is called
  *
  * Adjust the flags argument used to call opal_event_loop() from
- * opal_progress().  The default argument is OPAL_EVLOOP_ONELOOP,
- * meaning that the call to opal_event_loop() will block pending
- * events, but may block for a period of time.
+ * opal_progress().  The default argument is
+ *
+ *               OPAL_EVLOOP_NONBLOCK | OPAL_EVLOOP_ONCE
+ *
+ * OPAL_EVLOOP_NONBLOCK means that if there is no active events,
+ * opal_event_loop() should return immediately (instead of
+ * waiting for active events).
  *
- * @param flags     One of the valid vlags argument to
+ * OPAL_EVLOOP_ONCE means opal_event_loop() should onlyrun one
+ * iteration, which includes poll the event queue and process
+ * the active events.
+ *
+ * @param flags     One of the valid flags argument to
  *                  opal_event_loop().
  * @return          Previous value of flags used to call
  *                  opal_event_loop().
  */
 OPAL_DECLSPEC int opal_progress_set_event_flag(int flags);
 
-
 /**
  * Increase the number of users of the event library
  *
@@ -90,7 +99,6 @@ OPAL_DECLSPEC int opal_progress_set_event_flag(int flags);
  */
 OPAL_DECLSPEC void opal_progress_event_users_increment(void);
 
-
 /**
  * Decrease the number of users of the event library
  *
@@ -101,7 +109,6 @@ OPAL_DECLSPEC void opal_progress_event_users_increment(void);
  */
 OPAL_DECLSPEC void opal_progress_event_users_decrement(void);
 
-
 /**
  * Set whether opal_progress() should yield when idle
  *
@@ -117,7 +124,6 @@ OPAL_DECLSPEC void opal_progress_event_users_decrement(void);
  */
 OPAL_DECLSPEC bool opal_progress_set_yield_when_idle(bool yieldopt);
 
-
 /**
  * Set time between calls into the event library
  *
@@ -131,7 +137,6 @@ OPAL_DECLSPEC bool opal_progress_set_yield_when_idle(bool yieldopt);
  */
 OPAL_DECLSPEC void opal_progress_set_event_poll_rate(int microseconds);
 
-
 /**
  * Progress callback function typedef
  *
@@ -146,7 +151,6 @@ OPAL_DECLSPEC void opal_progress_set_event_poll_rate(int microseconds);
  */
 typedef int (*opal_progress_callback_t)(void);
 
-
 /**
  * Register an event to be progressed
  *
@@ -155,8 +159,7 @@ typedef int (*opal_progress_callback_t)(void);
  */
 OPAL_DECLSPEC int opal_progress_register(opal_progress_callback_t cb);
 
-OPAL_DECLSPEC int opal_progress_register_lp (opal_progress_callback_t cb);
-
+OPAL_DECLSPEC int opal_progress_register_lp(opal_progress_callback_t cb);
 
 /**
  * Deregister previously registered event
@@ -166,6 +169,9 @@ OPAL_DECLSPEC int opal_progress_register_lp (opal_progress_callback_t cb);
  */
 OPAL_DECLSPEC int opal_progress_unregister(opal_progress_callback_t cb);
 
+#if OPAL_ENABLE_DEBUG
+OPAL_DECLSPEC extern bool opal_progress_debug;
+#endif
 
 OPAL_DECLSPEC extern int opal_progress_spin_count;
 
@@ -175,13 +181,13 @@ OPAL_DECLSPEC extern bool opal_progress_yield_when_idle;
 /**
  * Progress until flag is true or poll iterations completed
  */
-static inline bool opal_progress_spin(volatile bool* complete)
+static inline bool opal_progress_spin(volatile bool *complete)
 {
     int32_t c;
 
     for (c = 0; c < opal_progress_spin_count; c++) {
         if (true == *complete) {
-             return true;
+            return true;
         }
         opal_progress();
     }
@@ -189,8 +195,6 @@ static inline bool opal_progress_spin(volatile bool* complete)
     return false;
 }
 
-
 END_C_DECLS
 
 #endif
-
diff --git a/opal/runtime/opal_progress_threads.c b/opal/runtime/opal_progress_threads.c
index a8a312a883b..f927cc961ca 100644
--- a/opal/runtime/opal_progress_threads.c
+++ b/opal/runtime/opal_progress_threads.c
@@ -14,21 +14,20 @@
 #include "opal/constants.h"
 
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 #ifdef HAVE_STRING_H
-#include <string.h>
+#    include <string.h>
 #endif
 
 #include "opal/class/opal_list.h"
-#include "opal/util/event.h"
 #include "opal/mca/threads/threads.h"
 #include "opal/util/error.h"
+#include "opal/util/event.h"
 #include "opal/util/fd.h"
 
 #include "opal/runtime/opal_progress_threads.h"
 
-
 /* create a tracking object for progress threads */
 typedef struct {
     opal_list_item_t super;
@@ -52,7 +51,7 @@ typedef struct {
 
 static void tracker_constructor(opal_progress_tracker_t *p)
 {
-    p->refcount = 1;  // start at one since someone created it
+    p->refcount = 1; // start at one since someone created it
     p->name = NULL;
     p->ev_base = NULL;
     p->ev_active = false;
@@ -74,17 +73,12 @@ static void tracker_destructor(opal_progress_tracker_t *p)
     }
 }
 
-static OBJ_CLASS_INSTANCE(opal_progress_tracker_t,
-                          opal_list_item_t,
-                          tracker_constructor,
+static OBJ_CLASS_INSTANCE(opal_progress_tracker_t, opal_list_item_t, tracker_constructor,
                           tracker_destructor);
 
 static bool inited = false;
 static opal_list_t tracking;
-static struct timeval long_timeout = {
-    .tv_sec = 3600,
-    .tv_usec = 0
-};
+static struct timeval long_timeout = {.tv_sec = 3600, .tv_usec = 0};
 static const char *shared_thread_name = "OPAL-wide async progress thread";
 
 /*
@@ -93,7 +87,7 @@ static const char *shared_thread_name = "OPAL-wide async progress thread";
  */
 static void dummy_timeout_cb(int fd, short args, void *cbdata)
 {
-    opal_progress_tracker_t *trk = (opal_progress_tracker_t*)cbdata;
+    opal_progress_tracker_t *trk = (opal_progress_tracker_t *) cbdata;
 
     opal_event_add(&trk->block, &long_timeout);
 }
@@ -101,10 +95,10 @@ static void dummy_timeout_cb(int fd, short args, void *cbdata)
 /*
  * Main for the progress thread
  */
-static void* progress_engine(opal_object_t *obj)
+static void *progress_engine(opal_object_t *obj)
 {
-    opal_thread_t *t = (opal_thread_t*)obj;
-    opal_progress_tracker_t *trk = (opal_progress_tracker_t*)t->t_arg;
+    opal_thread_t *t = (opal_thread_t *) obj;
+    opal_progress_tracker_t *trk = (opal_progress_tracker_t *) t->t_arg;
 
     while (trk->ev_active) {
         opal_event_loop(trk->ev_base, OPAL_EVLOOP_ONCE);
@@ -157,7 +151,7 @@ opal_event_base_t *opal_progress_thread_init(const char *name)
     }
 
     /* check if we already have this thread */
-    OPAL_LIST_FOREACH(trk, &tracking, opal_progress_tracker_t) {
+    OPAL_LIST_FOREACH (trk, &tracking, opal_progress_tracker_t) {
         if (0 == strcmp(name, trk->name)) {
             /* we do, so up the refcount on it */
             ++trk->refcount;
@@ -187,8 +181,7 @@ opal_event_base_t *opal_progress_thread_init(const char *name)
 
     /* add an event to the new event base (if there are no events,
        opal_event_loop() will return immediately) */
-    opal_event_set(trk->ev_base, &trk->block, -1, OPAL_EV_PERSIST,
-                   dummy_timeout_cb, trk);
+    opal_event_set(trk->ev_base, &trk->block, -1, OPAL_EV_PERSIST, dummy_timeout_cb, trk);
     opal_event_add(&trk->block, &long_timeout);
 
     /* construct the thread object */
@@ -218,7 +211,7 @@ int opal_progress_thread_finalize(const char *name)
     }
 
     /* find the specified engine */
-    OPAL_LIST_FOREACH(trk, &tracking, opal_progress_tracker_t) {
+    OPAL_LIST_FOREACH (trk, &tracking, opal_progress_tracker_t) {
         if (0 == strcmp(name, trk->name)) {
             /* decrement the refcount */
             --trk->refcount;
@@ -259,7 +252,7 @@ int opal_progress_thread_pause(const char *name)
     }
 
     /* find the specified engine */
-    OPAL_LIST_FOREACH(trk, &tracking, opal_progress_tracker_t) {
+    OPAL_LIST_FOREACH (trk, &tracking, opal_progress_tracker_t) {
         if (0 == strcmp(name, trk->name)) {
             if (trk->ev_active) {
                 stop_progress_engine(trk);
@@ -286,7 +279,7 @@ int opal_progress_thread_resume(const char *name)
     }
 
     /* find the specified engine */
-    OPAL_LIST_FOREACH(trk, &tracking, opal_progress_tracker_t) {
+    OPAL_LIST_FOREACH (trk, &tracking, opal_progress_tracker_t) {
         if (0 == strcmp(name, trk->name)) {
             if (trk->ev_active) {
                 return OPAL_ERR_RESOURCE_BUSY;
diff --git a/opal/runtime/opal_progress_threads.h b/opal/runtime/opal_progress_threads.h
index 2795bc474ee..6003b72ade1 100644
--- a/opal/runtime/opal_progress_threads.h
+++ b/opal/runtime/opal_progress_threads.h
@@ -17,7 +17,6 @@
 
 #include "opal/util/event.h"
 
-
 /**
  * Initialize a progress thread name; if a progress thread is not
  * already associated with that name, start a progress thread.
diff --git a/opal/test/reachable/reachable_netlink.c b/opal/test/reachable/reachable_netlink.c
index ea270f339c8..b040fadb13f 100644
--- a/opal/test/reachable/reachable_netlink.c
+++ b/opal/test/reachable/reachable_netlink.c
@@ -14,17 +14,16 @@
 
 #include "reachable_shared.h"
 
-#include "opal/runtime/opal.h"
-#include "opal/mca/reachable/reachable.h"
-#include "opal/util/if.h"
 #include "opal/class/opal_list.h"
+#include "opal/mca/reachable/reachable.h"
+#include "opal/runtime/opal.h"
 #include "opal/util/if.h"
 
 /*
  * Creates list of remote interfaces for testing reachability.
  * Only minimum information is filled out.
  */
-opal_list_t* build_if_list(void)
+opal_list_t *build_if_list(void)
 {
     /* Allocate memory for and create interface list */
     opal_list_t *if_list = OBJ_NEW(opal_list_t);
@@ -93,7 +92,6 @@ opal_list_t* build_if_list(void)
     return if_list;
 }
 
-
 int main(int argc, char **argv)
 {
     opal_list_t *local_list, *remote_list;
@@ -121,69 +119,67 @@ int main(int argc, char **argv)
 
     printf("Local interfaces:\n");
     i = 0;
-    OPAL_LIST_FOREACH(local_if, local_list, opal_if_t) {
+    OPAL_LIST_FOREACH (local_if, local_list, opal_if_t) {
         char addr[128];
         char *family;
 
         switch (local_if->af_family) {
-            case AF_INET:
-                family = "IPv4";
-                inet_ntop(AF_INET, &(((struct sockaddr_in*) &local_if->if_addr))->sin_addr,
-                          addr, sizeof(addr));
-                break;
-            case AF_INET6:
-                family = "IPv6";
-                inet_ntop(AF_INET6, &(((struct sockaddr_in6*) &local_if->if_addr))->sin6_addr,
-                          addr, sizeof(addr));
-                break;
-            default:
-                family = "Unknown";
-                opal_string_copy(addr, "Unknown", sizeof(addr));
-                break;
+        case AF_INET:
+            family = "IPv4";
+            inet_ntop(AF_INET, &(((struct sockaddr_in *) &local_if->if_addr))->sin_addr, addr,
+                      sizeof(addr));
+            break;
+        case AF_INET6:
+            family = "IPv6";
+            inet_ntop(AF_INET6, &(((struct sockaddr_in6 *) &local_if->if_addr))->sin6_addr, addr,
+                      sizeof(addr));
+            break;
+        default:
+            family = "Unknown";
+            opal_string_copy(addr, "Unknown", sizeof(addr));
+            break;
         }
 
-        printf("  %3d: %s\t%s\t%s/%d\n", i, local_if->if_name,
-	       family, addr, local_if->if_mask);
+        printf("  %3d: %s\t%s\t%s/%d\n", i, local_if->if_name, family, addr, local_if->if_mask);
         i++;
     }
 
     printf("\nRemote interfaces:\n");
     i = 0;
-    OPAL_LIST_FOREACH(local_if, remote_list, opal_if_t) {
+    OPAL_LIST_FOREACH (local_if, remote_list, opal_if_t) {
         char addr[128];
         char *family;
 
         switch (local_if->af_family) {
-            case AF_INET:
-                family = "IPv4";
-                inet_ntop(AF_INET, &(((struct sockaddr_in*) &local_if->if_addr))->sin_addr,
-                          addr, sizeof(addr));
-                break;
-            case AF_INET6:
-                family = "IPv6";
-                inet_ntop(AF_INET6, &(((struct sockaddr_in6*) &local_if->if_addr))->sin6_addr,
-                          addr, sizeof(addr));
-                break;
-            default:
-                family = "Unknown";
-                opal_string_copy(addr, "Unknown", sizeof(addr));
-                break;
+        case AF_INET:
+            family = "IPv4";
+            inet_ntop(AF_INET, &(((struct sockaddr_in *) &local_if->if_addr))->sin_addr, addr,
+                      sizeof(addr));
+            break;
+        case AF_INET6:
+            family = "IPv6";
+            inet_ntop(AF_INET6, &(((struct sockaddr_in6 *) &local_if->if_addr))->sin6_addr, addr,
+                      sizeof(addr));
+            break;
+        default:
+            family = "Unknown";
+            opal_string_copy(addr, "Unknown", sizeof(addr));
+            break;
         }
 
-        printf("  %3d: %s\t%s\t%s/%d\n", i, local_if->if_name,
-	       family, addr, local_if->if_mask);
+        printf("  %3d: %s\t%s\t%s/%d\n", i, local_if->if_name, family, addr, local_if->if_mask);
         i++;
     }
 
     printf("\nConnectivity Table:\n       ");
-    for (j = 0 ; j < remote_ifs ; j++) {
+    for (j = 0; j < remote_ifs; j++) {
         printf("%3d ", j);
     }
     printf("\n");
 
-    for (i = 0; i < local_ifs ; i++) {
+    for (i = 0; i < local_ifs; i++) {
         printf("  %3d: ", i);
-        for (j = 0 ; j < remote_ifs ; j++) {
+        for (j = 0; j < remote_ifs; j++) {
             printf("%3d ", results->weights[i][j]);
         }
         printf("\n");
diff --git a/opal/test/reachable/reachable_shared.h b/opal/test/reachable/reachable_shared.h
index 6ac72d9a357..6523f7402f0 100644
--- a/opal/test/reachable/reachable_shared.h
+++ b/opal/test/reachable/reachable_shared.h
@@ -13,25 +13,29 @@
 
 #include <assert.h>
 
-#include "opal/runtime/opal.h"
 #include "opal/mca/reachable/reachable.h"
+#include "opal/runtime/opal.h"
 #include "opal/util/if.h"
 #include "opal/util/string_copy.h"
 
 BEGIN_C_DECLS
 
 /* Create and populate opal_if_t with information required by opal_reachable */
-opal_if_t* create_if(int af_family, char *address, int mask, int bandwidth)
+opal_if_t *create_if(int af_family, char *address, int mask, int bandwidth)
 {
     opal_if_t *interface = OBJ_NEW(opal_if_t);
     opal_string_copy(interface->if_name, "interface0", OPAL_IF_NAMESIZE);
     interface->af_family = af_family;
-    ((struct sockaddr *)&(interface->if_addr))->sa_family = af_family; 
-
-    if (AF_INET == af_family){
-	assert(1 == inet_pton(af_family, address, &((struct sockaddr_in *)&(interface->if_addr))->sin_addr));
-    } else if (AF_INET6 == af_family){
-	assert(1 == inet_pton(af_family, address, &((struct sockaddr_in6 *)&(interface->if_addr))->sin6_addr));
+    ((struct sockaddr *) &(interface->if_addr))->sa_family = af_family;
+
+    if (AF_INET == af_family) {
+        assert(1
+               == inet_pton(af_family, address,
+                            &((struct sockaddr_in *) &(interface->if_addr))->sin_addr));
+    } else if (AF_INET6 == af_family) {
+        assert(1
+               == inet_pton(af_family, address,
+                            &((struct sockaddr_in6 *) &(interface->if_addr))->sin6_addr));
     }
 
     interface->if_mask = mask;
@@ -40,7 +44,6 @@ opal_if_t* create_if(int af_family, char *address, int mask, int bandwidth)
     return interface;
 }
 
-
 /* Run a test between a pair of interfaces
  * and clean up the memory afterwards.
  * Return the weight between the pair of
diff --git a/opal/test/reachable/reachable_weighted.c b/opal/test/reachable/reachable_weighted.c
index 5a6db2fc091..9cd4400cf32 100644
--- a/opal/test/reachable/reachable_weighted.c
+++ b/opal/test/reachable/reachable_weighted.c
@@ -23,7 +23,6 @@ enum connection_quality {
     CQ_PUBLIC_SAME_NETWORK = 100
 };
 
-
 /* SUITE 1:
  * Tests IPv4 connections by
  * modifying ip addresses and
@@ -52,8 +51,8 @@ int ipv4_test()
     int2 = create_if(AF_INET, "127.0.0.2", 0, 0);
     result = run_single_test(int1, int2);
     if (result != expected_result) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
     OBJ_RELEASE(int1);
     OBJ_RELEASE(int2);
@@ -69,8 +68,8 @@ int ipv4_test()
     result = run_single_test(int1, int2);
 
     if (result != expected_result) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
     OBJ_RELEASE(int1);
     OBJ_RELEASE(int2);
@@ -86,8 +85,8 @@ int ipv4_test()
     result = run_single_test(int1, int2);
 
     if (result != expected_result) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
     OBJ_RELEASE(int1);
     OBJ_RELEASE(int2);
@@ -103,8 +102,8 @@ int ipv4_test()
     result = run_single_test(int1, int2);
 
     if (result != expected_result) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
     OBJ_RELEASE(int1);
     OBJ_RELEASE(int2);
@@ -120,8 +119,8 @@ int ipv4_test()
     result = run_single_test(int1, int2);
 
     if (result != expected_result) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
     OBJ_RELEASE(int1);
     OBJ_RELEASE(int2);
@@ -137,8 +136,8 @@ int ipv4_test()
     result = run_single_test(int1, int2);
 
     if (result != expected_result) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
     OBJ_RELEASE(int1);
     OBJ_RELEASE(int2);
@@ -154,8 +153,8 @@ int ipv4_test()
     result = run_single_test(int1, int2);
 
     if (result != expected_result) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
     OBJ_RELEASE(int1);
     OBJ_RELEASE(int2);
@@ -171,8 +170,8 @@ int ipv4_test()
     result = run_single_test(int1, int2);
 
     if (result != expected_result) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
     OBJ_RELEASE(int1);
     OBJ_RELEASE(int2);
@@ -188,8 +187,8 @@ int ipv4_test()
     result = run_single_test(int1, int2);
 
     if (result != expected_result) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
     OBJ_RELEASE(int1);
     OBJ_RELEASE(int2);
@@ -205,8 +204,8 @@ int ipv4_test()
     result = run_single_test(int1, int2);
 
     if (result != expected_result) {
-	++failed_no;
-	opal_output(0, "Failed this test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed this test #%d", test_no);
     }
     OBJ_RELEASE(int1);
     OBJ_RELEASE(int2);
@@ -222,8 +221,8 @@ int ipv4_test()
     result = run_single_test(int1, int2);
 
     if (result != expected_result) {
-	++failed_no;
-	opal_output(0, "Failed this test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed this test #%d", test_no);
     }
     OBJ_RELEASE(int1);
     OBJ_RELEASE(int2);
@@ -239,8 +238,8 @@ int ipv4_test()
     result = run_single_test(int1, int2);
 
     if (result != expected_result) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
     OBJ_RELEASE(int1);
     OBJ_RELEASE(int2);
@@ -256,8 +255,8 @@ int ipv4_test()
     result = run_single_test(int1, int2);
 
     if (result != expected_result) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
     OBJ_RELEASE(int1);
     OBJ_RELEASE(int2);
@@ -273,8 +272,8 @@ int ipv4_test()
     result = run_single_test(int1, int2);
 
     if (result != expected_result) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
     OBJ_RELEASE(int1);
     OBJ_RELEASE(int2);
@@ -290,8 +289,8 @@ int ipv4_test()
     result = run_single_test(int1, int2);
 
     if (result != expected_result) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
     OBJ_RELEASE(int1);
     OBJ_RELEASE(int2);
@@ -302,13 +301,13 @@ int ipv4_test()
      */
     test_no++;
     expected_result = CQ_NO_CONNECTION;
-    int1 = create_if(AF_INET, "27.27.27.27", 24 , 0);
+    int1 = create_if(AF_INET, "27.27.27.27", 24, 0);
     int2 = create_if(AF_INET, "192.168.0.1", 16, 0);
     result = run_single_test(int1, int2);
 
     if (result != expected_result) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
     OBJ_RELEASE(int1);
     OBJ_RELEASE(int2);
@@ -324,8 +323,8 @@ int ipv4_test()
     result = run_single_test(int1, int2);
 
     if (result != expected_result) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
     OBJ_RELEASE(int1);
     OBJ_RELEASE(int2);
@@ -339,8 +338,8 @@ int ipv4_test()
     result = run_single_test(int1, int2);
     test_no++;
     if (result != expected_result) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
     OBJ_RELEASE(int1);
     OBJ_RELEASE(int2);
@@ -354,22 +353,22 @@ int ipv4_test()
     result = run_single_test(int1, int2);
     test_no++;
     if (result != expected_result) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
     OBJ_RELEASE(int1);
     OBJ_RELEASE(int2);
 
-    opal_output(0, "Finished Reachable IPv4 Tests.  %d/%d successful", test_no-failed_no, test_no);
+    opal_output(0, "Finished Reachable IPv4 Tests.  %d/%d successful", test_no - failed_no,
+                test_no);
 
     if (0 == failed_no) {
-	return 0;
+        return 0;
     } else {
-	return 1;
+        return 1;
     }
 }
 
-
 /* SUITE 2:
  * Compares connections with different
  * bandwidths to see ensure the
@@ -411,8 +410,8 @@ int ranking_test()
     OBJ_RELEASE(int2);
 
     if (!(result1 > result2)) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
 
     /* TEST2
@@ -434,8 +433,8 @@ int ranking_test()
     OBJ_RELEASE(int2);
 
     if (!(result1 < result2)) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
 
     /* TEST3
@@ -458,8 +457,8 @@ int ranking_test()
     OBJ_RELEASE(int2);
 
     if (!(result1 < result2)) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
 
     /* TEST4
@@ -482,8 +481,8 @@ int ranking_test()
     OBJ_RELEASE(int2);
 
     if (!(result1 < result2)) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
 
     /* TEST5
@@ -506,8 +505,8 @@ int ranking_test()
     OBJ_RELEASE(int2);
 
     if (!(result1 < result2)) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
 
     /* TEST6
@@ -529,8 +528,8 @@ int ranking_test()
     OBJ_RELEASE(int2);
 
     if (!(result1 < result2)) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
 
     /* TEST7
@@ -553,13 +552,13 @@ int ranking_test()
     OBJ_RELEASE(int2);
 
     if (!(result1 < result2)) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
 
     /* TEST8
      * Compares interface pairs with bandwidth 10->11 and 11->10.
-     * These connections should be equivilant, as they have the same
+     * These connections should be equivalent, as they have the same
      * bandwidth and same discrepancy.
      */
     test_no++;
@@ -577,8 +576,8 @@ int ranking_test()
     OBJ_RELEASE(int2);
 
     if (!(result1 == result2)) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
 
     /* TEST9
@@ -600,19 +599,19 @@ int ranking_test()
     OBJ_RELEASE(int2);
 
     if (!(result1 < result2)) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
 
-    opal_output(0, "Finished Reachable Weighted Ranking Tests.  %d/%d successful", test_no-failed_no, test_no);
+    opal_output(0, "Finished Reachable Weighted Ranking Tests.  %d/%d successful",
+                test_no - failed_no, test_no);
     if (0 == failed_no) {
-	return 0;
+        return 0;
     } else {
-	return 1;
+        return 1;
     }
 }
 
-
 /* SUITE 3:
  * Tests interfaces lists of various sizes
  * to ensure no crashes occur and results
@@ -644,8 +643,8 @@ int loop_test()
     OBJ_RELEASE(if_list2);
 
     if (!(0 == results->num_local && 0 == results->num_remote)) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
     OBJ_RELEASE(results);
 
@@ -666,8 +665,8 @@ int loop_test()
     OBJ_RELEASE(if_list2);
 
     if (!(0 == results->num_local && 1 == results->num_remote)) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
     OBJ_RELEASE(results);
 
@@ -687,8 +686,8 @@ int loop_test()
     OBJ_RELEASE(if_list2);
 
     if (!(1 == results->num_local && 0 == results->num_remote)) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
     OBJ_RELEASE(results);
 
@@ -702,12 +701,12 @@ int loop_test()
     if_list1 = OBJ_NEW(opal_list_t);
     if_list2 = OBJ_NEW(opal_list_t);
     for (i = 0; i < 3; i++) {
-	intf = create_if(AF_INET, "31.14.19.92", 24, 0);
-	opal_list_append(if_list1, &(intf->super));
+        intf = create_if(AF_INET, "31.14.19.92", 24, 0);
+        opal_list_append(if_list1, &(intf->super));
     }
     for (i = 0; i < 14; i++) {
-	intf = create_if(AF_INET, "31.14.19.92", 24, 0);
-	opal_list_append(if_list2, &(intf->super));
+        intf = create_if(AF_INET, "31.14.19.92", 24, 0);
+        opal_list_append(if_list2, &(intf->super));
     }
     results = opal_reachable.reachable(if_list1, if_list2);
 
@@ -715,8 +714,8 @@ int loop_test()
     OBJ_RELEASE(if_list2);
 
     if (!(3 == results->num_local && 14 == results->num_remote)) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
     OBJ_RELEASE(results);
 
@@ -730,12 +729,12 @@ int loop_test()
     if_list1 = OBJ_NEW(opal_list_t);
     if_list2 = OBJ_NEW(opal_list_t);
     for (i = 0; i < 14; i++) {
-	intf = create_if(AF_INET, "31.14.19.92", 24, 0);
-	opal_list_append(if_list1, &(intf->super));
+        intf = create_if(AF_INET, "31.14.19.92", 24, 0);
+        opal_list_append(if_list1, &(intf->super));
     }
     for (i = 0; i < 3; i++) {
-	intf = create_if(AF_INET, "31.14.19.92", 24, 0);
-	opal_list_append(if_list2, &(intf->super));
+        intf = create_if(AF_INET, "31.14.19.92", 24, 0);
+        opal_list_append(if_list2, &(intf->super));
     }
     results = opal_reachable.reachable(if_list1, if_list2);
 
@@ -743,8 +742,8 @@ int loop_test()
     OBJ_RELEASE(if_list2);
 
     if (!(14 == results->num_local && 3 == results->num_remote)) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
     OBJ_RELEASE(results);
 
@@ -758,12 +757,12 @@ int loop_test()
     if_list1 = OBJ_NEW(opal_list_t);
     if_list2 = OBJ_NEW(opal_list_t);
     for (i = 0; i < 27; i++) {
-	intf = create_if(AF_INET, "31.14.19.92", 24, 0);
-	opal_list_append(if_list1, &(intf->super));
+        intf = create_if(AF_INET, "31.14.19.92", 24, 0);
+        opal_list_append(if_list1, &(intf->super));
     }
     for (i = 0; i < 27; i++) {
-	intf = create_if(AF_INET, "31.14.19.92", 24, 0);
-	opal_list_append(if_list2, &(intf->super));
+        intf = create_if(AF_INET, "31.14.19.92", 24, 0);
+        opal_list_append(if_list2, &(intf->super));
     }
     results = opal_reachable.reachable(if_list1, if_list2);
 
@@ -771,8 +770,8 @@ int loop_test()
     OBJ_RELEASE(if_list2);
 
     if (!(27 == results->num_local && 27 == results->num_remote)) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
     OBJ_RELEASE(results);
 
@@ -800,12 +799,12 @@ int loop_test()
     OBJ_RELEASE(if_list1);
     OBJ_RELEASE(if_list2);
 
-    if (!(CQ_PUBLIC_SAME_NETWORK == results->weights[0][0] &&
-	  CQ_PUBLIC_DIFFERENT_NETWORK == results->weights[0][1] &&
-	  CQ_PUBLIC_DIFFERENT_NETWORK == results->weights[1][0] &&
-	  CQ_PUBLIC_SAME_NETWORK == results->weights[1][1])) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+    if (!(CQ_PUBLIC_SAME_NETWORK == results->weights[0][0]
+          && CQ_PUBLIC_DIFFERENT_NETWORK == results->weights[0][1]
+          && CQ_PUBLIC_DIFFERENT_NETWORK == results->weights[1][0]
+          && CQ_PUBLIC_SAME_NETWORK == results->weights[1][1])) {
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
     OBJ_RELEASE(results);
 
@@ -835,14 +834,14 @@ int loop_test()
     OBJ_RELEASE(if_list1);
     OBJ_RELEASE(if_list2);
 
-    if (!(CQ_PUBLIC_SAME_NETWORK == results->weights[0][0] &&
-	  CQ_PUBLIC_DIFFERENT_NETWORK == results->weights[0][1] &&
-	  CQ_PUBLIC_DIFFERENT_NETWORK == results->weights[0][2] &&
-	  CQ_PUBLIC_DIFFERENT_NETWORK == results->weights[1][0] &&
-	  CQ_PUBLIC_SAME_NETWORK == results->weights[1][1]) &&
-	CQ_PUBLIC_DIFFERENT_NETWORK == results->weights[1][2]) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+    if (!(CQ_PUBLIC_SAME_NETWORK == results->weights[0][0]
+          && CQ_PUBLIC_DIFFERENT_NETWORK == results->weights[0][1]
+          && CQ_PUBLIC_DIFFERENT_NETWORK == results->weights[0][2]
+          && CQ_PUBLIC_DIFFERENT_NETWORK == results->weights[1][0]
+          && CQ_PUBLIC_SAME_NETWORK == results->weights[1][1])
+        && CQ_PUBLIC_DIFFERENT_NETWORK == results->weights[1][2]) {
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
     OBJ_RELEASE(results);
 
@@ -872,28 +871,27 @@ int loop_test()
     OBJ_RELEASE(if_list1);
     OBJ_RELEASE(if_list2);
 
-    if (!(CQ_PUBLIC_SAME_NETWORK == results->weights[0][0] &&
-	  CQ_PUBLIC_DIFFERENT_NETWORK == results->weights[0][1] &&
-	  CQ_PUBLIC_DIFFERENT_NETWORK == results->weights[1][0] &&
-	  CQ_PUBLIC_SAME_NETWORK == results->weights[1][1] &&
-	  CQ_PUBLIC_DIFFERENT_NETWORK == results->weights[2][0]) &&
-	CQ_PUBLIC_DIFFERENT_NETWORK == results->weights[2][1]) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+    if (!(CQ_PUBLIC_SAME_NETWORK == results->weights[0][0]
+          && CQ_PUBLIC_DIFFERENT_NETWORK == results->weights[0][1]
+          && CQ_PUBLIC_DIFFERENT_NETWORK == results->weights[1][0]
+          && CQ_PUBLIC_SAME_NETWORK == results->weights[1][1]
+          && CQ_PUBLIC_DIFFERENT_NETWORK == results->weights[2][0])
+        && CQ_PUBLIC_DIFFERENT_NETWORK == results->weights[2][1]) {
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
     OBJ_RELEASE(results);
 
-    opal_output(0, "Finished Reachable Weighted Loop Tests.  %d/%d successful", test_no-failed_no, test_no);
+    opal_output(0, "Finished Reachable Weighted Loop Tests.  %d/%d successful", test_no - failed_no,
+                test_no);
 
     if (0 == failed_no) {
-	return 0;
+        return 0;
     } else {
-	return 1;
+        return 1;
     }
-
 }
 
-
 /* SUITE 4:
  * Test IPv6
  */
@@ -918,8 +916,8 @@ int test_ipv6()
     result = run_single_test(int1, int2);
     test_no++;
     if (result != expected_result) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
     OBJ_RELEASE(int1);
     OBJ_RELEASE(int2);
@@ -934,8 +932,8 @@ int test_ipv6()
     result = run_single_test(int1, int2);
     test_no++;
     if (result != expected_result) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
     OBJ_RELEASE(int1);
     OBJ_RELEASE(int2);
@@ -946,8 +944,8 @@ int test_ipv6()
     result = run_single_test(int1, int2);
     test_no++;
     if (result != expected_result) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
     OBJ_RELEASE(int1);
     OBJ_RELEASE(int2);
@@ -958,18 +956,19 @@ int test_ipv6()
     result = run_single_test(int1, int2);
     test_no++;
     if (result != expected_result) {
-	++failed_no;
-	opal_output(0, "Failed test #%d", test_no);
+        ++failed_no;
+        opal_output(0, "Failed test #%d", test_no);
     }
     OBJ_RELEASE(int1);
     OBJ_RELEASE(int2);
 
-    opal_output(0, "Finished Reachable Weighted IPv6 Tests.  %d/%d successful", test_no-failed_no, test_no);
+    opal_output(0, "Finished Reachable Weighted IPv6 Tests.  %d/%d successful", test_no - failed_no,
+                test_no);
 
     if (0 == failed_no) {
-	return 0;
+        return 0;
     } else {
-	return 1;
+        return 1;
     }
 #else
     opal_output(0, "No IPv6 support; skipped tests");
@@ -987,28 +986,28 @@ int main(int argc, char **argv)
 
     total++;
     if (ipv4_test()) {
-	failed++;
+        failed++;
     }
 
     total++;
     if (ranking_test()) {
-	failed++;
+        failed++;
     }
 
     total++;
     if (loop_test()) {
-	failed++;
+        failed++;
     }
 
     total++;
     if (test_ipv6()) {
-	failed++;
+        failed++;
     }
 
     if (0 != failed) {
-	opal_output(0, "\n\nFailed %d/%d Reachable Weighted Test Suites :(\n\n", failed, total);	
+        opal_output(0, "\n\nFailed %d/%d Reachable Weighted Test Suites :(\n\n", failed, total);
     } else {
-	opal_output(0, "\n\nPassed %d/%d Reachable Weighted Test Suites :)\n\n", total, total);	
+        opal_output(0, "\n\nPassed %d/%d Reachable Weighted Test Suites :)\n\n", total, total);
     }
 
     return failed;
diff --git a/opal/test/reachable/tests b/opal/test/reachable/tests
index 344bdf24a16..dc2d01697d7 100755
--- a/opal/test/reachable/tests
+++ b/opal/test/reachable/tests
@@ -8,5 +8,5 @@
 # $HEADER$
 #
 
-mpirun -np 1 --mca reachable netlink reachable_netlink 
-mpirun -np 1 --mca reachable weighted reachable_weighted 
+mpirun -n 1 --mca reachable netlink reachable_netlink
+mpirun -n 1 --mca reachable weighted reachable_weighted
diff --git a/opal/tools/wrappers/Makefile.am b/opal/tools/wrappers/Makefile.am
index 53850b28bb7..1164560e853 100644
--- a/opal/tools/wrappers/Makefile.am
+++ b/opal/tools/wrappers/Makefile.am
@@ -2,17 +2,19 @@
 # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
 #                         University Research and Technology
 #                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2006 The University of Tennessee and The University
+# Copyright (c) 2004-2022 The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
 # Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2006-2014 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2006-2022 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
 # Copyright (c) 2014      Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -22,60 +24,26 @@
 
 DEFS="-DEXEEXT=\"$(EXEEXT)\""
 
-real_man_pages = generic_wrapper.1 opal_wrapper.1
-EXTRA_DIST = $(real_man_pages:.1=.1in)
-
-include $(top_srcdir)/Makefile.ompi-rules
-
 if !OPAL_WANT_SCRIPT_WRAPPER_COMPILERS
 if OPAL_INSTALL_BINARIES
 
 bin_PROGRAMS = opal_wrapper
 
-nodist_man_MANS = opal_wrapper.1
-
-dist_opaldata_DATA = help-opal-wrapper.txt
+EXTRA_DIST = help-opal-wrapper.txt
 
 # Only install the following for developer-level installs
 if WANT_INSTALL_HEADERS
 
 nodist_opaldata_DATA = \
-	opalcc-wrapper-data.txt \
-	opalc++-wrapper-data.txt
-
-nodist_man_MANS += opalcc.1 opalc++.1
+	opalcc-wrapper-data.txt
 
 pkgconfigdir = $(libdir)/pkgconfig
 pkgconfig_DATA = opal.pc
 
-install-exec-hook:
-	(cd $(DESTDIR)$(bindir); rm -f opalcc$(EXEEXT); $(LN_S) opal_wrapper$(EXEECT) opalcc$(EXEEXT))
-	(cd $(DESTDIR)$(bindir); rm -f opalc++$(EXEEXT); $(LN_S) opal_wrapper$(EXEECT) opalc++$(EXEEXT))
-
-uninstall-local:
-	rm -f $(DESTDIR)$(bindir)/opalcc$(EXEEXT) \
-	$(DESTDIR)$(bindir)/opalc++$(EXEEXT)
-
 endif # WANT_INSTALL_HEADERS
 
 endif # OPAL_INSTALL_BINARIES
 endif # OPAL_WANT_SCRIPT_WRAPPER_COMPILERS
 
 opal_wrapper_SOURCES = opal_wrapper.c
-opal_wrapper_LDADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
-
-# Ensure that the man pages are rebuilt if the opal_config.h file
-# changes; a "good enough" way to know if configure was run again (and
-# therefore the release date or version may have changed)
-$(real_man_pages): $(top_builddir)/opal/include/opal_config.h
-
-opalcc.1: generic_wrapper.1
-	rm -f opalcc.1
-	sed -e 's/#COMMAND#/opalcc/g' -e 's/#PROJECT#/Open PAL/g' -e 's/#PROJECT_SHORT#/OPAL/g' -e 's/#LANGUAGE#/C/g' < $(top_builddir)/opal/tools/wrappers/generic_wrapper.1 > opalcc.1
-
-opalc++.1: generic_wrapper.1
-	rm -f opalc++.1
-	sed -e 's/#COMMAND#/opalc++/g' -e 's/#PROJECT#/Open PAL/g' -e 's/#PROJECT_SHORT#/OPAL/g' -e 's/#LANGUAGE#/C++/g' < $(top_builddir)/opal/tools/wrappers/generic_wrapper.1 > opalc++.1
-
-distclean-local:
-	rm -f $(real_man_pages) opalcc.1 opalc++.1
+opal_wrapper_LDADD = $(top_builddir)/opal/libopen-pal_core.la
diff --git a/opal/tools/wrappers/generic_wrapper.1in b/opal/tools/wrappers/generic_wrapper.1in
deleted file mode 100644
index 27adaa4a10f..00000000000
--- a/opal/tools/wrappers/generic_wrapper.1in
+++ /dev/null
@@ -1,265 +0,0 @@
-.\" Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
-.\" Copyright (c) 2011-2012 Cisco Systems, Inc.  All rights reserved.
-.TH #COMMAND# 1 "#OPAL_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.
-.SH NAME
-#COMMAND# -- #PROJECT# #LANGUAGE# wrapper compiler
-.
-.SH SYNTAX
-#COMMAND# [-showme|-showme:compile|-showme:link] ...
-.
-.SH OPTIONS
-.TP
---showme
-This option comes in several different variants (see below).  None of
-the variants invokes the underlying compiler; they all provide
-information on how the underlying compiler would have been invoked had
-.I --showme
-not been used.
-The basic
-.I --showme
-option outputs the command line that would be executed to compile the
-program.  \fBNOTE:\fR If a non-filename argument is passed on the
-command line, the \fI-showme\fR option will \fInot\fR display any
-additional flags.  For example, both "#COMMAND# --showme" and
-"#COMMAND# --showme my_source.c" will show all the wrapper-supplied
-flags.  But "#COMMAND# --showme -v" will only show the underlying
-compiler name and "-v".
-.TP
---showme:compile
-Output the compiler flags that would have been supplied to the
-#LANGUAGE# compiler.
-.TP
---showme:link
-Output the linker flags that would have been supplied to the
-#LANGUAGE# compiler.
-.TP
---showme:command
-Outputs the underlying #LANGUAGE# compiler command (which may be one
-or more tokens).
-.TP
---showme:incdirs
-Outputs a space-delimited (but otherwise undecorated) list of
-directories that the wrapper compiler would have provided to the
-underlying #LANGUAGE# compiler to indicate where relevant header files
-are located.
-.TP
---showme:libdirs
-Outputs a space-delimited (but otherwise undecorated) list of
-directories that the wrapper compiler would have provided to the
-underlying linker to indicate where relevant libraries are located.
-.TP
---showme:libs
-Outputs a space-delimited (but otherwise undecorated) list of library
-names that the wrapper compiler would have used to link an
-application.  For example: "mpi open-rte open-pal util".
-.TP
---showme:version
-Outputs the version number of Open MPI.
-.TP
---showme:help
-Output a brief usage help message.
-.PP
-See the man page for your underlying #LANGUAGE# compiler for other
-options that can be passed through #COMMAND#.
-.
-.
-.SH DESCRIPTION
-.PP
-Conceptually, the role of these commands is quite simple:
-transparently add relevant compiler and linker flags to the user's
-command line that are necessary to compile / link #PROJECT#
-programs, and then invoke the underlying compiler to actually perform
-the command.
-.
-.PP
-As such, these commands are frequently referred to as "wrapper"
-compilers because they do not actually compile or link applications
-themselves; they only add in command line flags and invoke the
-back-end compiler.
-.
-.
-.SS Background
-Open MPI is comprised of three software layers: OPAL (Open Portable
-Access Layer), ORTE (Open Run-Time Environment), and OMPI (Open MPI).
-There are wrapper compilers for each layer; each layer's wrapper only
-links in the libraries relevant for that layer.  Specifically, each
-layer provides the following wrapper compilers:
-.
-.TP 4
-OPAL
-\fIopalcc\fR and \fIopalc++\fR
-.
-.TP
-ORTE
-\fIortecc\fR and \fIortec++\fR
-.
-.TP
-OMPI
-\fImpicc\fR, \fImpic++\fR, \fImpicxx\fR, \fImpiCC\fR (only on systems with
-case-senstive file systems), and \fImpifort\fR (and its legacy/deprecated
-names \fImpif77\fR and \fImpif90\fR).  Note
-that \fImpic++\fR, \fImpicxx\fR, and \fImpiCC\fR all invoke the same
-underlying C++ compiler with the same options.  All are provided as
-compatibility with other MPI implementations.
-.
-.
-.SS Fortran Notes
-.PP
-The Fortran wrapper compiler for MPI (\fImpifort\fR, and its
-legacy/deprecated names \fImpif77\fR and \fImpif90\fR) can compile and
-link MPI applications that use any/all of the MPI Fortran bindings:
-.IR mpif.h ,
-the
-.I mpi
-module, and the
-.I mpi_f08
-module (assuming Open MPI was installed with support for each of these
-Fortran bindings).  Specifically: it is no longer necessary to use
-different wrapper compilers for applications that use
-.I mpif.h
-vs. applications that use the
-.I mpi
-module -- just use
-.I mpifort
-for all Fortran MPI applications.
-.
-.PP
-Note, however, that the Fortran compiler may require additional
-command-line options to enforce a specific Fortran dialect.  For
-example, in some versions of the IBM XLF compiler, if xlf90 is the
-underlying Fortran compiler,
-.IR -qfixed
-may be necessary to compile fixed-format Fortran source files.
-.
-.PP
-Finally, note that
-.I mpifort
-will be inoperative and will return an error on use if Fortran support
-was not built into the MP Ilayer.
-.
-.
-.SS Overview
-\fI#COMMAND#\fR is a convenience wrappers for the underlying
-#LANGUAGE# compiler.  Translation of an #PROJECT# program requires the
-linkage of the #PROJECT#-specific libraries which may not reside in
-one of the standard search directories of ld(1).  It also often
-requires the inclusion of header files what may also not be found in a
-standard location.
-.
-.PP
-\fI#COMMAND#\fR passes its arguments to the underlying #LANGUAGE#
-compiler along with the -I, -L and -l options required by #PROJECT#
-programs.
-.
-.PP
-The #PROJECT# Team \fIstrongly\fR encourages using the wrapper
-compilers instead of attempting to link to the #PROJECT# libraries
-manually.  This allows the specific implementation of #PROJECT# to
-change without forcing changes to linker directives in users'
-Makefiles.  Indeed, the specific set of flags and libraries used by
-the wrapper compilers depends on how #PROJECT# was configured and
-built; the values can change between different installations of the
-same version of #PROJECT#.
-.
-.PP
-Indeed, since the wrappers are simply thin shells on top of an
-underlying compiler, there are very, very few compelling reasons
-\fInot\fR to use \fI#COMMAND#\fR.  When it is not possible to use the
-wrappers directly, the \fI-showme:compile\fR and \fI-showme:link\fR
-options should be used to determine what flags the wrappers would have
-used.  For example:
-.
-.PP
-shell$ cc -c file1.c `mpicc -showme:compile`
-.
-.PP
-shell$ cc -c file2.c `mpicc -showme:compile`
-.
-.PP
-shell$ cc file1.o file2.o `mpicc -showme:link` -o my_mpi_program
-.
-.
-.SH NOTES
-.PP
-It is possible to make the wrapper compilers multi-lib aware.  That
-is, the libraries and includes specified may differ based on the
-compiler flags specified (for example, with the GNU compilers on
-Linux, a different library path may be used if -m32 is seen versus
--m64 being seen).  This is not the default behavior in a standard
-build, but can be activated (for example, in a binary package
-providing both 32 and 64 bit support).  More information can be found
-at:
-.PP
-  https://github.com/open-mpi/ompi/wiki/compilerwrapper3264
-.
-.
-.SH FILES
-.PP
-The string that the wrapper compilers insert into the command line
-before invoking the underlying compiler are stored in a text file
-created by #PROJECT# and installed to
-\fI$pkgdata/#COMMAND#-wrapper-data.txt\fR, where \fI$pkgdata\fR
-is typically \fI$prefix/share/openmpi\fR, and \fI$prefix\fR is the top
-installation directory of #PROJECT#.
-.
-.PP
-It is rarely necessary to edit this file, but it can be examined to
-gain insight into what flags the wrappers are placing on the command
-line.
-.
-.
-.SH ENVIRONMENT VARIABLES
-.PP
-By default, the wrappers use the compilers that were selected when
-#PROJECT# was configured.  These compilers were either found
-automatically by Open MPI's "configure" script, or were selected by
-the user in the CC, CXX, F77, and/or FC environment variables
-before "configure" was invoked.  Additionally, other arguments
-specific to the compiler may have been selected by configure.
-.
-.PP
-These values can be selectively overridden by either editing the text
-files containing this configuration information (see the \fBFILES\fR
-section), or by setting selected environment variables of the
-form "#PROJECT_SHORT#_value".
-.
-.PP
-Valid value names are:
-.
-.TP
-CPPFLAGS
-Flags added when invoking the preprocessor (C or C++)
-.
-.TP
-LDFLAGS
-Flags added when invoking the linker (C, C++, or Fortran)
-.
-.TP
-LIBS
-Libraries added when invoking the linker (C, C++, or Fortran)
-.
-.TP
-CC
-C compiler
-.
-.TP
-CFLAGS
-C compiler flags
-.
-.TP
-CXX
-C++ compiler
-.
-.TP
-CXXFLAGS
-C++ compiler flags
-.
-.
-.TP
-FC
-Fortran compiler
-.
-.TP
-FCFLAGS
-Fortran compiler flags
diff --git a/opal/tools/wrappers/help-opal-wrapper.txt b/opal/tools/wrappers/help-opal-wrapper.txt
index 3302f14c099..21c67ab5d8d 100644
--- a/opal/tools/wrappers/help-opal-wrapper.txt
+++ b/opal/tools/wrappers/help-opal-wrapper.txt
@@ -11,7 +11,7 @@
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
 # Copyright (c) 2006-2010 Oracle and/or its affiliates.  All rights reserved.
-# Copyright (c) 2012-2014 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2012-2022 Cisco Systems, Inc.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -25,12 +25,6 @@
 Unfortunately, this installation of Open MPI was not compiled with
 %s support.  As such, the %s compiler is non-functional.
 #
-[no-profiling-support]
-Warning: the "-lpmpi" flag was included on the command line,
-indicating use of MPI profiling layer.  However, Open MPI was not
-compiled with support for the MPI profiling layer.  This flag has
-therefore been ignored.
-#
 [no-compiler-specified]
 No underlying compiler was specified in the wrapper compiler data file
 (e.g., mpicc-wrapper-data.txt)
@@ -67,10 +61,6 @@ Unable to call the compiler (%s). The failure return the error
 %s.
 The failed command was [%s].
 #
-[compiler-failed]
-The child process (%s) exit with error %d.
-The failed command was [%s].
-#
 [no-options-support]
 The %s wrapper compiler was unable to find a block of options in its
 configuration to support your compiler mode.  Your application was not
diff --git a/opal/tools/wrappers/opal.pc.in b/opal/tools/wrappers/opal.pc.in
index 9b71f338ffe..f2604ba2e44 100644
--- a/opal/tools/wrappers/opal.pc.in
+++ b/opal/tools/wrappers/opal.pc.in
@@ -15,7 +15,7 @@ pkgincludedir=@opalincludedir@
 # (they're pulled in via libopen-pal.so's implicit dependencies), so
 # list them in Libs.private.
 #
-Libs: -L${libdir} @OPAL_PKG_CONFIG_LDFLAGS@ -lopen-pal
+Libs: -L${libdir} @OPAL_PKG_CONFIG_LDFLAGS@ -l@OPAL_LIB_NAME@
 Libs.private: @OPAL_WRAPPER_EXTRA_LIBS@
 #
 # It is safe to hard-wire the -I before the EXTRA_INCLUDES because we
diff --git a/opal/tools/wrappers/opal_wrapper.1in b/opal/tools/wrappers/opal_wrapper.1in
deleted file mode 100644
index c4132bb18d8..00000000000
--- a/opal/tools/wrappers/opal_wrapper.1in
+++ /dev/null
@@ -1,78 +0,0 @@
-.\" Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
-.\" Man page contributed by Dirk Eddelbuettel <edd@debian.org>
-.\" and released under the BSD license
-.TH OPAL_WRAPPER 1 "#OPAL_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-opal_wrapper - Back-end Open MPI wrapper command
-.SH SYNOPSIS
-.B opal_wrapper [options]
-.SH DESCRIPTION
-.PP
-.B opal_wrapper
-is not meant to be called directly by end users.  It is automatically
-invoked as the back-end by the Open MPI wrapper commands such as:
-.BR mpicc ,
-.BR mpiCC ,
-.BR mpic++ ,
-and
-.BR mpifort
-(and its legacy/deprecated names
-.BR mpif77
-and
-.BR mpif90 ).
-.PP
-Some Open MPI installations may have additional wrapper commands,
-and/or have renamed the wrapper compilers listed above to avoid
-executable name conflicts with other MPI implementations.  Hence, you
-may also have wrapper compilers installed including the following
-names:
-.BR mpifort.openmpi
-(and the legacy/deprecated names
-.BR mpif90.openmpi
-and
-.BR mpif77.openmpi ),
-.BR mpicxx.openmpi ,
-.BR mpiCC.openmpi ,
-.BR mpicc.openmpi ,
-.BR mpic++.openmpi ,
-.BR opalcc ,
-.BR opalc++ ,
-.BR ortecc ,
-and
-.BR ortec++ ,
-.
-.
-.\" **************************
-.\"    See Also Section
-.\" **************************
-.SH SEE ALSO
-The following may exist depending on your particular Open MPI
-installation:
-.BR mpicc (1),
-.BR mpiCC (1),
-.BR mpic++ (1),
-.BR mpifort (1),
-.BR mpifort.openmpi (1),
-.BR mpicxx.openmpi (1),
-.BR mpiCC.openmpi (1),
-.BR mpicc.openmpi (1),
-.BR mpic++.openmpi (1),
-.BR ortecc (1),
-.BR ortec++ (1),
-.BR opalccc (1),
-and the website at
-.IR https://www.open-mpi.org/ .
-.
-.
-.\" **************************
-.\"    Authors Section
-.\" **************************
-.SH AUTHORS
-The Open MPI maintainers -- see
-.I https://www.open-mpi.org/
-or the file
-.IR AUTHORS .
-.PP
-This manual page was originally contributed by Dirk Eddelbuettel
-<edd@debian.org>, one of the Debian GNU/Linux maintainers for Open
-MPI, and may be used by others.
diff --git a/opal/tools/wrappers/opal_wrapper.c b/opal/tools/wrappers/opal_wrapper.c
index f880d261498..893bbcd59d9 100644
--- a/opal/tools/wrappers/opal_wrapper.c
+++ b/opal/tools/wrappers/opal_wrapper.c
@@ -9,13 +9,13 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2007-2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2007-2021 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2007-2013 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2018-2022 Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -25,62 +25,136 @@
 
 #include "opal_config.h"
 
-#include <stdio.h>
 #include <errno.h>
+#include <stdio.h>
 #include <stdlib.h>
 #ifdef HAVE_SYS_STAT_H
-#include <sys/stat.h>
-#endif  /* HAVE_SYS_STAT_H */
+#    include <sys/stat.h>
+#endif /* HAVE_SYS_STAT_H */
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif  /* HAVE_SYS_TYPES_H */
+#    include <sys/types.h>
+#endif /* HAVE_SYS_TYPES_H */
 #ifdef HAVE_REGEX_H
-#include <regex.h>
+#    include <regex.h>
 #endif
 #ifdef HAVE_SYS_WAIT_H
-#include <sys/wait.h>
-#endif  /* HAVE_SYS_WAIT_H */
+#    include <sys/wait.h>
+#endif /* HAVE_SYS_WAIT_H */
 #include <string.h>
 
+#include "opal/constants.h"
 #include "opal/mca/installdirs/installdirs.h"
 #include "opal/runtime/opal.h"
-#include "opal/constants.h"
 #include "opal/util/argv.h"
+#include "opal/util/basename.h"
 #include "opal/util/error.h"
+#include "opal/util/few.h"
 #include "opal/util/keyval_parse.h"
 #include "opal/util/opal_environ.h"
-#include "opal/util/show_help.h"
-#include "opal/util/path.h"
-#include "opal/util/few.h"
-#include "opal/util/basename.h"
 #include "opal/util/os_path.h"
+#include "opal/util/path.h"
 #include "opal/util/printf.h"
+#include "opal/util/show_help.h"
 
-#define OPAL_INCLUDE_FLAG  "-I"
-#define OPAL_LIBDIR_FLAG   "-L"
+#define OPAL_INCLUDE_FLAG "-I"
+#define OPAL_LIBDIR_FLAG  "-L"
 
+static const char * filtered_args[] = { "-I/usr/include",
+                                        "-L/usr/lib",
+                                        "-L/usr/lib64",
+                                        NULL };
+
+/* This structure is used to represent the config file read in during
+ * initialization.  A single file may have multiple options_data
+ * structures, with the compiler_args keying on which section to
+ * use in returning results. */
 struct options_data_t {
+    /* compiler_args is a key to chose the right block in the case
+     * that there are multiple blocks in a single file.  All
+     * arguments in compiler_args in must appear in the compiler's argv
+     * in order for this block to be selected. */
     char **compiler_args;
+    /* language for this compiler.  This field is only used for
+     * pretty-printing messages.  It does not have any impact on the
+     * behavior of the wrapper compiler. */
     char *language;
+    /* project name.  This field is only used for pretty-printing
+     * messages.  It does not have any impact on the behavior of the
+     * wrapper compiler. */
     char *project;
+    /* project short name.  The environment variables to influence
+     * setting of compiler flags are named <project_short>_{CPPFLAGS,
+     * LDFLAGS, LIBS}. */
     char *project_short;
+    /* project release version.  Used to pretty-print help strings. */
     char *version;
+    /* Second part of the variable name to change the underlying
+     * compiler the wrapper compiler invokes.  For example, for C,
+     * compiler_env should be CC, resulting in <project_short>_CC
+     * being the environment variable used to change the underlying
+     * compiler that is invoked. */
     char *compiler_env;
+    /* Second part of the variable name to change the compiler flags
+     * passed to the underlying compiler.  For example, for C,
+     * compiler_flags_env should be CFLAGS, resulting in
+     * <project_short>_CFLAGS being the environment variable used to
+     * change the CFLAGS passed to the underlying compiler. */
     char *compiler_flags_env;
+    /* Default underlying compiler to invoke */
     char *compiler;
+    /* Default preprocessor (ie, CPPFLAGS) to pass to underlying
+     * compiler.  These are only passed to the underlying compiler
+     * when it looks like we are compiling (ie, not in a link-only
+     * situation). */
     char **preproc_flags;
+    /* Default compiler flags (ie, CFLAGS) to pass to the underlying
+     * compiler. */
     char **comp_flags;
+    /* Compiler flags to pass to the underlying compiler before
+     * comp_flags.  These args are generally passed directly from
+     * configure options provided from the user. */
     char **comp_flags_prefix;
+    /* Linker flags to pass to to the underlying compiler when it
+     * appears that we are linking.  These flags will be passed to the
+     * underlying compiler whether or not we see -static (or friends) in
+     * the arguments. */
     char **link_flags;
+    /* Linker flags to pass to to the underlying compiler when it
+     * appears that we are linking.  These flags ONLY will be passed
+     * to the underlying compiler when we see -static (or friends) in
+     * the arguments. */
+    char **link_flags_static;
+    /* Libs flags to pass to to the underlying compiler when it
+     * appears that we are linking.  These flags will be passed to the
+     * underlying compiler whether or not we see -static (or friends)
+     * in the arguments. */
     char **libs;
+    /* Libs flags to pass to to the underlying compiler when it
+     * appears that we are linking.  These flags ONLY will be passed
+     * to the underlying compiler when we see -static (or friends) in
+     * the arguments. */
     char **libs_static;
+    /* Name of a file that should exist in ${libdir} if shared
+     * libraries were properly installed.  Historically, this was
+     * used to determine what flags should be added to the compiler.
+     * Now it is not used. */
     char *dyn_lib_file;
+    /* Name of a file that should exist in ${libdir} if static
+     * libraries were properly installed.  Historically, this was
+     * used to determine what flags should be added to the compiler.
+     * Now it is not used. */
     char *static_lib_file;
+    /* A (full path) of a file that must exist for the wrapper
+     * compiler to succeed.  A common use case is to specify a
+     * language-binding specific file (for example, a C++ header file)
+     * when invoking an optional language binding. */
     char *req_file;
+    /* Default includedir, before variable expansion.  Almost always
+     * set as ${includedir} outside of multilib situations. */
     char *path_includedir;
+    /* Default libdir, before variable expansion.  Almost always set
+    as ${libdir} outside of multilib situations */
     char *path_libdir;
-    char *path_opalincludedir;
-    char *path_opallibdir;
 };
 
 static struct options_data_t *options_data = NULL;
@@ -91,20 +165,19 @@ static int user_data_idx = -1;
 /* index of options to use by default */
 static int default_data_idx = -1;
 
-#define COMP_DRY_RUN       0x001
-#define COMP_SHOW_ERROR    0x002
-#define COMP_WANT_COMMAND  0x004
-#define COMP_WANT_PREPROC  0x008
-#define COMP_WANT_COMPILE  0x010
-#define COMP_WANT_LINK     0x020
-#define COMP_WANT_PMPI     0x040
-#define COMP_WANT_STATIC   0x080
-#define COMP_WANT_LINKALL  0x100
-
-static void
-options_data_init(struct options_data_t *data)
+#define COMP_DRY_RUN      0x001
+#define COMP_SHOW_ERROR   0x002
+#define COMP_WANT_COMMAND 0x004
+#define COMP_WANT_PREPROC 0x008
+#define COMP_WANT_COMPILE 0x010
+#define COMP_WANT_LINK    0x020
+#define COMP_WANT_PMPI    0x040
+#define COMP_WANT_STATIC  0x080
+#define COMP_WANT_LINKALL 0x100
+
+static void options_data_init(struct options_data_t *data)
 {
-    data->compiler_args = (char **) malloc(sizeof(char*));
+    data->compiler_args = (char **) malloc(sizeof(char *));
     data->compiler_args[0] = NULL;
     data->language = NULL;
     data->compiler = NULL;
@@ -113,61 +186,83 @@ options_data_init(struct options_data_t *data)
     data->version = NULL;
     data->compiler_env = NULL;
     data->compiler_flags_env = NULL;
-    data->preproc_flags = (char **) malloc(sizeof(char*));
+    data->preproc_flags = (char **) malloc(sizeof(char *));
     data->preproc_flags[0] = NULL;
-    data->comp_flags = (char **) malloc(sizeof(char*));
+    data->comp_flags = (char **) malloc(sizeof(char *));
     data->comp_flags[0] = NULL;
-    data->comp_flags_prefix = (char **) malloc(sizeof(char*));
+    data->comp_flags_prefix = (char **) malloc(sizeof(char *));
     data->comp_flags_prefix[0] = NULL;
-    data->link_flags = (char **) malloc(sizeof(char*));
+    data->link_flags = (char **) malloc(sizeof(char *));
     data->link_flags[0] = NULL;
-    data->libs = (char **) malloc(sizeof(char*));
+    data->link_flags_static = (char **) malloc(sizeof(char *));
+    data->link_flags_static[0] = NULL;
+    data->libs = (char **) malloc(sizeof(char *));
     data->libs[0] = NULL;
-    data->libs_static = (char **) malloc(sizeof(char*));
+    data->libs_static = (char **) malloc(sizeof(char *));
     data->libs_static[0] = NULL;
     data->dyn_lib_file = NULL;
     data->static_lib_file = NULL;
     data->req_file = NULL;
     data->path_includedir = NULL;
     data->path_libdir = NULL;
-    data->path_opalincludedir = NULL;
-    data->path_opallibdir = NULL;
 }
 
-static void
-options_data_free(struct options_data_t *data)
+static void options_data_free(struct options_data_t *data)
 {
     if (NULL != data->compiler_args) {
         opal_argv_free(data->compiler_args);
     }
-    if (NULL != data->language) free(data->language);
-    if (NULL != data->compiler) free(data->compiler);
-    if (NULL != data->project) free(data->project);
-    if (NULL != data->project_short) free(data->project_short);
-    if (NULL != data->version) free(data->version);
-    if (NULL != data->compiler_env) free(data->compiler_env);
-    if (NULL != data->compiler_flags_env) free(data->compiler_flags_env);
+    if (NULL != data->language) {
+        free(data->language);
+    }
+    if (NULL != data->compiler) {
+        free(data->compiler);
+    }
+    if (NULL != data->project) {
+        free(data->project);
+    }
+    if (NULL != data->project_short) {
+        free(data->project_short);
+    }
+    if (NULL != data->version) {
+        free(data->version);
+    }
+    if (NULL != data->compiler_env) {
+        free(data->compiler_env);
+    }
+    if (NULL != data->compiler_flags_env) {
+        free(data->compiler_flags_env);
+    }
     opal_argv_free(data->preproc_flags);
     opal_argv_free(data->comp_flags);
     opal_argv_free(data->comp_flags_prefix);
     opal_argv_free(data->link_flags);
+    opal_argv_free(data->link_flags_static);
     opal_argv_free(data->libs);
     opal_argv_free(data->libs_static);
-    if (NULL != data->dyn_lib_file) free(data->dyn_lib_file);
-    if (NULL != data->static_lib_file) free(data->static_lib_file);
-    if (NULL != data->req_file) free(data->req_file);
-    if (NULL != data->path_includedir) free(data->path_includedir);
-    if (NULL != data->path_libdir) free(data->path_libdir);
-    if (NULL != data->path_opalincludedir) free(data->path_opalincludedir);
-    if (NULL != data->path_opallibdir) free(data->path_opallibdir);
+    if (NULL != data->dyn_lib_file) {
+        free(data->dyn_lib_file);
+    }
+    if (NULL != data->static_lib_file) {
+        free(data->static_lib_file);
+    }
+    if (NULL != data->req_file) {
+        free(data->req_file);
+    }
+    if (NULL != data->path_includedir) {
+        free(data->path_includedir);
+    }
+    if (NULL != data->path_libdir) {
+        free(data->path_libdir);
+    }
 }
 
-static void
-options_data_expand(const char *value)
+static void options_data_expand(const char *value)
 {
     /* make space for the new set of args */
     parse_options_idx++;
-    options_data = (struct options_data_t *) realloc(options_data, sizeof(struct options_data_t) * (parse_options_idx + 1));
+    options_data = (struct options_data_t *) realloc(options_data, sizeof(struct options_data_t)
+                                                                       * (parse_options_idx + 1));
     options_data_init(&(options_data[parse_options_idx]));
 
     /* if there are values, this is not the default case.
@@ -175,8 +270,7 @@ options_data_expand(const char *value)
     if (NULL != value && 0 != strcmp(value, "")) {
         char **values = opal_argv_split(value, ';');
         opal_argv_insert(&(options_data[parse_options_idx].compiler_args),
-                         opal_argv_count(options_data[parse_options_idx].compiler_args),
-                         values);
+                         opal_argv_count(options_data[parse_options_idx].compiler_args), values);
         opal_argv_free(values);
     } else {
         free(options_data[parse_options_idx].compiler_args);
@@ -186,9 +280,7 @@ options_data_expand(const char *value)
     }
 }
 
-
-static int
-find_options_index(const char *arg)
+static int find_options_index(const char *arg)
 {
     int i, j;
 #ifdef HAVE_REGEXEC
@@ -196,14 +288,14 @@ find_options_index(const char *arg)
     regex_t res;
 #endif
 
-    for (i = 0 ; i <= parse_options_idx ; ++i) {
+    for (i = 0; i <= parse_options_idx; ++i) {
         if (NULL == options_data[i].compiler_args) {
             continue;
         }
 
 #ifdef HAVE_REGEXEC
         args_count = opal_argv_count(options_data[i].compiler_args);
-        for (j = 0 ; j < args_count ; ++j) {
+        for (j = 0; j < args_count; ++j) {
             if (0 != regcomp(&res, options_data[i].compiler_args[j], REG_NOSUB)) {
                 return -1;
             }
@@ -216,7 +308,7 @@ find_options_index(const char *arg)
             regfree(&res);
         }
 #else
-        for (j = 0 ; j < opal_argv_count(options_data[i].compiler_args) ; ++j) {
+        for (j = 0; j < opal_argv_count(options_data[i].compiler_args); ++j) {
             if (0 == strcmp(arg, options_data[i].compiler_args[j])) {
                 return i;
             }
@@ -227,14 +319,12 @@ find_options_index(const char *arg)
     return -1;
 }
 
-
-static void
-expand_flags(char **argv)
+static void expand_flags(char **argv)
 {
     int i;
     char *tmp;
 
-    for (i = 0 ; argv[i] != NULL ; ++i) {
+    for (i = 0; argv[i] != NULL; ++i) {
         tmp = opal_install_dirs_expand(argv[i]);
         if (tmp != argv[i]) {
             free(argv[i]);
@@ -243,9 +333,38 @@ expand_flags(char **argv)
     }
 }
 
+static void filter_flags(char*** argvp)
+{
+    int argc,  idx;
+
+    argc = opal_argv_count(*argvp);
+    idx = 0;
+
+    while (idx < argc) {
+        char *arg = (*argvp)[idx];
+        size_t j = 0;
+        bool found = false;
 
-static void
-data_callback(const char *key, const char *value)
+        while (filtered_args[j] != NULL) {
+            if (0 == strcmp(arg, filtered_args[j])) {
+                opal_argv_delete(&argc, argvp, idx, 1);
+                found = true;
+                break;
+            }
+            j++;
+        }
+
+        if (!found) {
+            /* skip moving the idx pointer if we found an entry (since
+             * the previous idx + 1 entry is the new idx pointer).  Note
+             * that argc might have moved, so the outer while loop make still
+             * give a different answer to its conditional. */
+            idx++;
+        }
+    }
+}
+
+static void data_callback(const char *key, const char *value)
 {
     /* handle case where text file does not contain any special
        compiler options field */
@@ -256,26 +375,34 @@ data_callback(const char *key, const char *value)
     if (0 == strcmp(key, "compiler_args")) {
         options_data_expand(value);
     } else if (0 == strcmp(key, "language")) {
-        if (NULL != value) options_data[parse_options_idx].language = strdup(value);
+        if (NULL != value) {
+            options_data[parse_options_idx].language = strdup(value);
+        }
     } else if (0 == strcmp(key, "compiler")) {
-        if (NULL != value) options_data[parse_options_idx].compiler = strdup(value);
+        if (NULL != value) {
+            options_data[parse_options_idx].compiler = strdup(value);
+        }
     } else if (0 == strcmp(key, "project")) {
-        if (NULL != value) options_data[parse_options_idx].project = strdup(value);
+        if (NULL != value) {
+            options_data[parse_options_idx].project = strdup(value);
+        }
     } else if (0 == strcmp(key, "version")) {
-        if (NULL != value) options_data[parse_options_idx].version = strdup(value);
+        if (NULL != value) {
+            options_data[parse_options_idx].version = strdup(value);
+        }
     } else if (0 == strcmp(key, "preprocessor_flags")) {
         char **values = opal_argv_split(value, ' ');
         opal_argv_insert(&options_data[parse_options_idx].preproc_flags,
-                         opal_argv_count(options_data[parse_options_idx].preproc_flags),
-                         values);
+                         opal_argv_count(options_data[parse_options_idx].preproc_flags), values);
         expand_flags(options_data[parse_options_idx].preproc_flags);
+        filter_flags(&options_data[parse_options_idx].preproc_flags);
         opal_argv_free(values);
     } else if (0 == strcmp(key, "compiler_flags")) {
         char **values = opal_argv_split(value, ' ');
         opal_argv_insert(&options_data[parse_options_idx].comp_flags,
-                         opal_argv_count(options_data[parse_options_idx].comp_flags),
-                         values);
+                         opal_argv_count(options_data[parse_options_idx].comp_flags), values);
         expand_flags(options_data[parse_options_idx].comp_flags);
+        filter_flags(&options_data[parse_options_idx].comp_flags);
         opal_argv_free(values);
     } else if (0 == strcmp(key, "compiler_flags_prefix")) {
         char **values = opal_argv_split(value, ' ');
@@ -287,112 +414,88 @@ data_callback(const char *key, const char *value)
     } else if (0 == strcmp(key, "linker_flags")) {
         char **values = opal_argv_split(value, ' ');
         opal_argv_insert(&options_data[parse_options_idx].link_flags,
-                         opal_argv_count(options_data[parse_options_idx].link_flags),
-                         values);
+                         opal_argv_count(options_data[parse_options_idx].link_flags), values);
         expand_flags(options_data[parse_options_idx].link_flags);
+        filter_flags(&options_data[parse_options_idx].link_flags);
+        opal_argv_free(values);
+    } else if (0 == strcmp(key, "linker_flags_static")) {
+        char **values = opal_argv_split(value, ' ');
+        opal_argv_insert(&options_data[parse_options_idx].link_flags_static,
+                         opal_argv_count(options_data[parse_options_idx].link_flags_static), values);
+        expand_flags(options_data[parse_options_idx].link_flags_static);
+        filter_flags(&options_data[parse_options_idx].link_flags_static);
         opal_argv_free(values);
     } else if (0 == strcmp(key, "libs")) {
         char **values = opal_argv_split(value, ' ');
         opal_argv_insert(&options_data[parse_options_idx].libs,
-                         opal_argv_count(options_data[parse_options_idx].libs),
-                         values);
+                         opal_argv_count(options_data[parse_options_idx].libs), values);
         opal_argv_free(values);
     } else if (0 == strcmp(key, "libs_static")) {
         char **values = opal_argv_split(value, ' ');
         opal_argv_insert(&options_data[parse_options_idx].libs_static,
-                         opal_argv_count(options_data[parse_options_idx].libs_static),
-                         values);
+                         opal_argv_count(options_data[parse_options_idx].libs_static), values);
         opal_argv_free(values);
     } else if (0 == strcmp(key, "dyn_lib_file")) {
-        if (NULL != value) options_data[parse_options_idx].dyn_lib_file = strdup(value);
+        if (NULL != value) {
+            options_data[parse_options_idx].dyn_lib_file = strdup(value);
+        }
     } else if (0 == strcmp(key, "static_lib_file")) {
-        if (NULL != value) options_data[parse_options_idx].static_lib_file = strdup(value);
+        if (NULL != value) {
+            options_data[parse_options_idx].static_lib_file = strdup(value);
+        }
     } else if (0 == strcmp(key, "required_file")) {
-        if (NULL != value) options_data[parse_options_idx].req_file = strdup(value);
+        if (NULL != value) {
+            options_data[parse_options_idx].req_file = strdup(value);
+        }
     } else if (0 == strcmp(key, "project_short")) {
-        if (NULL != value) options_data[parse_options_idx].project_short = strdup(value);
+        if (NULL != value) {
+            options_data[parse_options_idx].project_short = strdup(value);
+        }
     } else if (0 == strcmp(key, "compiler_env")) {
-        if (NULL != value) options_data[parse_options_idx].compiler_env = strdup(value);
+        if (NULL != value) {
+            options_data[parse_options_idx].compiler_env = strdup(value);
+        }
     } else if (0 == strcmp(key, "compiler_flags_env")) {
-        if (NULL != value) options_data[parse_options_idx].compiler_flags_env = strdup(value);
+        if (NULL != value) {
+            options_data[parse_options_idx].compiler_flags_env = strdup(value);
+        }
     } else if (0 == strcmp(key, "includedir")) {
         if (NULL != value) {
-            options_data[parse_options_idx].path_includedir =
-                opal_install_dirs_expand(value);
-            if (0 != strcmp(options_data[parse_options_idx].path_includedir, "/usr/include") ||
-                0 == strncmp(options_data[parse_options_idx].language, "Fortran", strlen("Fortran"))) {
-                char *line;
-                opal_asprintf(&line, OPAL_INCLUDE_FLAG"%s",
-                         options_data[parse_options_idx].path_includedir);
-                opal_argv_append_nosize(&options_data[parse_options_idx].preproc_flags, line);
-                free(line);
-            }
+            options_data[parse_options_idx].path_includedir = opal_install_dirs_expand(value);
         }
     } else if (0 == strcmp(key, "libdir")) {
-        if (NULL != value) options_data[parse_options_idx].path_libdir =
-                               opal_install_dirs_expand(value);
-        if (0 != strcmp(options_data[parse_options_idx].path_libdir, "/usr/lib")) {
-            char *line;
-            opal_asprintf(&line, OPAL_LIBDIR_FLAG"%s",
-                     options_data[parse_options_idx].path_libdir);
-            opal_argv_append_nosize(&options_data[parse_options_idx].link_flags, line);
-            free(line);
-        }
-    } else if (0 == strcmp(key, "opalincludedir")) {
-        printf("EXPANDING!\n");
         if (NULL != value) {
-            options_data[parse_options_idx].path_opalincludedir =
-                opal_install_dirs_expand(value);
-            if (0 != strcmp(options_data[parse_options_idx].path_opalincludedir, "/usr/include") ||
-                0 == strncmp(options_data[parse_options_idx].language, "Fortran", strlen("Fortran"))) {
-                char *line;
-                opal_asprintf(&line, OPAL_INCLUDE_FLAG"%s",
-                         options_data[parse_options_idx].path_opalincludedir);
-                opal_argv_append_nosize(&options_data[parse_options_idx].preproc_flags, line);
-                free(line);
-            }
-        }
-    } else if (0 == strcmp(key, "opallibdir")) {
-        if (NULL != value) options_data[parse_options_idx].path_opallibdir =
-                               opal_install_dirs_expand(value);
-        if (0 != strcmp(options_data[parse_options_idx].path_opallibdir, "/usr/lib")) {
-            char *line;
-            opal_asprintf(&line, OPAL_LIBDIR_FLAG"%s",
-                     options_data[parse_options_idx].path_opallibdir);
-            opal_argv_append_nosize(&options_data[parse_options_idx].link_flags, line);
-            free(line);
+            options_data[parse_options_idx].path_libdir = opal_install_dirs_expand(value);
         }
     }
 }
 
-
-static int
-data_init(const char *appname)
+static int data_init(const char *appname)
 {
     int ret;
     char *datafile;
 
     /* now load the data */
-    opal_asprintf(&datafile, "%s%s%s-wrapper-data.txt",
-             opal_install_dirs.opaldatadir, OPAL_PATH_SEP, appname);
-    if (NULL == datafile) return OPAL_ERR_TEMP_OUT_OF_RESOURCE;
+    opal_asprintf(&datafile, "%s%s%s-wrapper-data.txt", opal_install_dirs.opaldatadir,
+                  OPAL_PATH_SEP, appname);
+    if (NULL == datafile) {
+        return OPAL_ERR_TEMP_OUT_OF_RESOURCE;
+    }
 
     ret = opal_util_keyval_parse(datafile, data_callback);
-    if( OPAL_SUCCESS != ret ) {
-        fprintf(stderr, "Cannot open configuration file %s\n", datafile );
+    if (OPAL_SUCCESS != ret) {
+        fprintf(stderr, "Cannot open configuration file %s\n", datafile);
     }
     free(datafile);
 
     return ret;
 }
 
-
-static int
-data_finalize(void)
+static int data_finalize(void)
 {
     int i;
 
-    for (i = 0 ; i <= parse_options_idx ; ++i) {
+    for (i = 0; i <= parse_options_idx; ++i) {
         options_data_free(&(options_data[i]));
     }
     free(options_data);
@@ -400,32 +503,39 @@ data_finalize(void)
     return OPAL_SUCCESS;
 }
 
-
-static void
-print_flags(char **args, char *pattern)
+/*
+ * Print the flags in args, stripping off pattern (which must be a
+ * simple exact match string, no regex) from the front of every
+ * argument in args.
+ */
+static void print_flags(char **args, char *pattern)
 {
     int i;
     bool found = false;
 
-    for (i = 0 ; args[i] != NULL ; ++i) {
+    for (i = 0; args[i] != NULL; ++i) {
         if (0 == strncmp(args[i], pattern, strlen(pattern))) {
-            if (found)  printf(" ");
+            if (found) {
+                printf(" ");
+            }
             printf("%s", args[i] + strlen(pattern));
             found = true;
         }
     }
 
-    if (found) printf("\n");
+    if (found) {
+        printf("\n");
+    }
 }
 
-
-static void
-load_env_data(const char *project, const char *flag, char **data)
+static void load_env_data(const char *project, const char *flag, char **data)
 {
     char *envname;
     char *envvalue;
 
-    if (NULL == project || NULL == flag) return;
+    if (NULL == project || NULL == flag) {
+        return;
+    }
 
     opal_asprintf(&envname, "%s_MPI%s", project, flag);
     if (NULL == (envvalue = getenv(envname))) {
@@ -438,18 +548,20 @@ load_env_data(const char *project, const char *flag, char **data)
     }
     free(envname);
 
-    if (NULL != *data) free(*data);
+    if (NULL != *data) {
+        free(*data);
+    }
     *data = strdup(envvalue);
 }
 
-
-static void
-load_env_data_argv(const char *project, const char *flag, char ***data)
+static void load_env_data_argv(const char *project, const char *flag, char ***data)
 {
     char *envname;
     char *envvalue;
 
-    if (NULL == project || NULL == flag) return;
+    if (NULL == project || NULL == flag) {
+        return;
+    }
 
     opal_asprintf(&envname, "%s_MPI%s", project, flag);
     if (NULL == (envvalue = getenv(envname))) {
@@ -462,14 +574,14 @@ load_env_data_argv(const char *project, const char *flag, char ***data)
     }
     free(envname);
 
-    if (NULL != *data) opal_argv_free(*data);
+    if (NULL != *data) {
+        opal_argv_free(*data);
+    }
 
     *data = opal_argv_split(envvalue, ' ');
 }
 
-
-int
-main(int argc, char *argv[])
+int main(int argc, char *argv[])
 {
     int exit_status = 0, ret, flags = 0, i;
     int exec_argc = 0, user_argc = 0;
@@ -490,27 +602,27 @@ main(int argc, char *argv[])
 
     base_argv0 = opal_basename(argv[0]);
 #if defined(EXEEXT)
-    if( 0 != strlen(EXEEXT) ) {
+    if (0 != strlen(EXEEXT)) {
         char extension[] = EXEEXT;
-        char* temp = strstr( base_argv0, extension );
-        char* old_match = temp;
-        while( NULL != temp ) {
+        char *temp = strstr(base_argv0, extension);
+        char *old_match = temp;
+        while (NULL != temp) {
             old_match = temp;
-            temp = strstr( temp + 1, extension );
+            temp = strstr(temp + 1, extension);
         }
-        /* Only if there was a match of .exe, erase the last occurence of .exe */
-        if ( NULL != old_match ) {
+        /* Only if there was a match of .exe, erase the last occurrence of .exe */
+        if (NULL != old_match) {
             *old_match = '\0';
         }
     }
-#endif  /* defined(EXEEXT) */
+#endif /* defined(EXEEXT) */
 
     if (OPAL_SUCCESS != (ret = data_init(base_argv0))) {
         fprintf(stderr, "Error parsing data file %s: %s\n", base_argv0, opal_strerror(ret));
         return ret;
     }
 
-    for (i = 1 ; i < argc && user_data_idx < 0 ; ++i) {
+    for (i = 1; i < argc && user_data_idx < 0; ++i) {
         user_data_idx = find_options_index(argv[i]);
     }
     /* if we didn't find a match, look for the NULL (base case) options */
@@ -520,28 +632,31 @@ main(int argc, char *argv[])
     /* if we still didn't find a match, abort */
     if (user_data_idx < 0) {
         char *flat = opal_argv_join(argv, ' ');
-        opal_show_help("help-opal-wrapper.txt", "no-options-support", true,
-                       base_argv0, flat, NULL);
+        opal_show_help("help-opal-wrapper.txt", "no-options-support", true, base_argv0, flat, NULL);
         free(flat);
         exit(1);
     }
 
     /* compiler */
-    load_env_data(options_data[user_data_idx].project_short, options_data[user_data_idx].compiler_env, &options_data[user_data_idx].compiler);
+    load_env_data(options_data[user_data_idx].project_short,
+                  options_data[user_data_idx].compiler_env, &options_data[user_data_idx].compiler);
 
     /* preprocessor flags */
-    load_env_data_argv(options_data[user_data_idx].project_short, "CPPFLAGS", &options_data[user_data_idx].preproc_flags);
+    load_env_data_argv(options_data[user_data_idx].project_short, "CPPFLAGS",
+                       &options_data[user_data_idx].preproc_flags);
 
     /* compiler flags */
-    load_env_data_argv(options_data[user_data_idx].project_short, options_data[user_data_idx].compiler_flags_env,
+    load_env_data_argv(options_data[user_data_idx].project_short,
+                       options_data[user_data_idx].compiler_flags_env,
                        &options_data[user_data_idx].comp_flags);
 
     /* linker flags */
-    load_env_data_argv(options_data[user_data_idx].project_short, "LDFLAGS", &options_data[user_data_idx].link_flags);
+    load_env_data_argv(options_data[user_data_idx].project_short, "LDFLAGS",
+                       &options_data[user_data_idx].link_flags);
 
     /* libs */
-    load_env_data_argv(options_data[user_data_idx].project_short, "LIBS", &options_data[user_data_idx].libs);
-
+    load_env_data_argv(options_data[user_data_idx].project_short, "LIBS",
+                       &options_data[user_data_idx].libs);
 
     /****************************************************
      *
@@ -561,10 +676,12 @@ main(int argc, char *argv[])
         if (options_data[user_data_idx].req_file[0] != '\0') {
             char *filename;
             struct stat buf;
-            filename = opal_os_path( false, options_data[user_data_idx].path_libdir, options_data[user_data_idx].req_file, NULL );
+            filename = opal_os_path(false, options_data[user_data_idx].path_libdir,
+                                    options_data[user_data_idx].req_file, NULL);
             if (0 != stat(filename, &buf)) {
-                opal_show_help("help-opal-wrapper.txt", "file-not-found", true,
-                               base_argv0, options_data[user_data_idx].req_file, options_data[user_data_idx].language, NULL);
+                opal_show_help("help-opal-wrapper.txt", "file-not-found", true, base_argv0,
+                               options_data[user_data_idx].req_file,
+                               options_data[user_data_idx].language, NULL);
             }
         }
     }
@@ -574,68 +691,98 @@ main(int argc, char *argv[])
      * Parse user flags
      *
      ****************************************************/
-    flags = COMP_WANT_COMMAND|COMP_WANT_PREPROC|
-        COMP_WANT_COMPILE|COMP_WANT_LINK;
+    flags = COMP_WANT_COMMAND | COMP_WANT_PREPROC | COMP_WANT_COMPILE | COMP_WANT_LINK;
 
     user_argv = opal_argv_copy(argv + 1);
     user_argc = opal_argv_count(user_argv);
 
-    for (i = 0 ; i < user_argc ; ++i) {
-        if (0 == strncmp(user_argv[i], "-showme", strlen("-showme")) ||
-            0 == strncmp(user_argv[i], "--showme", strlen("--showme")) ||
-            0 == strncmp(user_argv[i], "-show", strlen("-show")) ||
-            0 == strncmp(user_argv[i], "--show", strlen("--show"))) {
+    for (i = 0; i < user_argc; ++i) {
+        if (0 == strncmp(user_argv[i], "-showme", strlen("-showme"))
+            || 0 == strncmp(user_argv[i], "--showme", strlen("--showme"))
+            || 0 == strncmp(user_argv[i], "-show", strlen("-show"))
+            || 0 == strncmp(user_argv[i], "--show", strlen("--show"))) {
             bool done_now = false;
 
             /* check for specific things we want to see.  First three
                still invoke all the building routines.  Last set want
                to parse out certain flags, so we don't go through the
                normal build routine - skip to cleanup. */
-            if (0 == strncmp(user_argv[i], "-showme:command", strlen("-showme:command")) ||
-                0 == strncmp(user_argv[i], "--showme:command", strlen("--showme:command"))) {
+            if (0 == strncmp(user_argv[i], "-showme:command", strlen("-showme:command"))
+                || 0 == strncmp(user_argv[i], "--showme:command", strlen("--showme:command"))) {
                 flags = COMP_WANT_COMMAND;
                 /* we know what we want, so don't process any more args */
                 done_now = true;
-            } else if (0 == strncmp(user_argv[i], "-showme:compile", strlen("-showme:compile")) ||
-                0 == strncmp(user_argv[i], "--showme:compile", strlen("--showme:compile"))) {
-                flags = COMP_WANT_PREPROC|COMP_WANT_COMPILE;
+            } else if (0 == strncmp(user_argv[i], "-showme:compile", strlen("-showme:compile"))
+                       || 0
+                              == strncmp(user_argv[i], "--showme:compile",
+                                         strlen("--showme:compile"))) {
+                flags = COMP_WANT_PREPROC | COMP_WANT_COMPILE;
                 /* we know what we want, so don't process any more args */
                 done_now = true;
-            } else if (0 == strncmp(user_argv[i], "-showme:link", strlen("-showme:link")) ||
-                       0 == strncmp(user_argv[i], "--showme:link", strlen("--showme:link"))) {
-                flags = COMP_WANT_COMPILE|COMP_WANT_LINK;
+            } else if (0 == strncmp(user_argv[i], "-showme:link", strlen("-showme:link"))
+                       || 0 == strncmp(user_argv[i], "--showme:link", strlen("--showme:link"))) {
+                flags = COMP_WANT_COMPILE | COMP_WANT_LINK;
                 /* we know what we want, so don't process any more args */
                 done_now = true;
-            } else if (0 == strncmp(user_argv[i], "-showme:incdirs", strlen("-showme:incdirs")) ||
-                       0 == strncmp(user_argv[i], "--showme:incdirs", strlen("--showme:incdirs"))) {
+            } else if (0 == strncmp(user_argv[i], "-showme:incdirs", strlen("-showme:incdirs"))
+                       || 0
+                              == strncmp(user_argv[i], "--showme:incdirs",
+                                         strlen("--showme:incdirs"))) {
                 print_flags(options_data[user_data_idx].preproc_flags, OPAL_INCLUDE_FLAG);
                 goto cleanup;
-            } else if (0 == strncmp(user_argv[i], "-showme:libdirs", strlen("-showme:libdirs")) ||
-                       0 == strncmp(user_argv[i], "--showme:libdirs", strlen("--showme:libdirs"))) {
+            } else if (0 == strncmp(user_argv[i], "-showme:libdirs_static", strlen("-showme:libdirs_static"))
+                       || 0
+                              == strncmp(user_argv[i], "--showme:libdirs_static",
+                                         strlen("--showme:libdirs_static"))) {
+                char **all_args = NULL;
+                int args_count;
+
+                all_args = opal_argv_copy(options_data[user_data_idx].link_flags);
+                args_count = opal_argv_count(all_args);
+                opal_argv_insert(&all_args, args_count, options_data[user_data_idx].link_flags_static);
+                print_flags(all_args, OPAL_LIBDIR_FLAG);
+                opal_argv_free(all_args);
+                goto cleanup;
+            } else if (0 == strncmp(user_argv[i], "-showme:libdirs", strlen("-showme:libdirs"))
+                       || 0
+                              == strncmp(user_argv[i], "--showme:libdirs",
+                                         strlen("--showme:libdirs"))) {
                 print_flags(options_data[user_data_idx].link_flags, OPAL_LIBDIR_FLAG);
                 goto cleanup;
-            } else if (0 == strncmp(user_argv[i], "-showme:libs", strlen("-showme:libs")) ||
-                       0 == strncmp(user_argv[i], "--showme:libs", strlen("--showme:libs"))) {
+            } else if (0 == strncmp(user_argv[i], "-showme:libs_static", strlen("-showme:libs_static"))
+                       || 0 == strncmp(user_argv[i], "--showme:libs_static", strlen("--showme:libs_static"))) {
+                char **all_args = NULL;
+                int args_count;
+
+                all_args = opal_argv_copy(options_data[user_data_idx].libs);
+                args_count = opal_argv_count(all_args);
+                opal_argv_insert(&all_args, args_count, options_data[user_data_idx].libs_static);
+                print_flags(all_args, "-l");
+                opal_argv_free(all_args);
+                goto cleanup;
+            } else if (0 == strncmp(user_argv[i], "-showme:libs", strlen("-showme:libs"))
+                       || 0 == strncmp(user_argv[i], "--showme:libs", strlen("--showme:libs"))) {
                 print_flags(options_data[user_data_idx].libs, "-l");
                 goto cleanup;
-            } else if (0 == strncmp(user_argv[i], "-showme:version", strlen("-showme:version")) ||
-                       0 == strncmp(user_argv[i], "--showme:version", strlen("--showme:version"))) {
-                char * str;
-                str = opal_show_help_string("help-opal-wrapper.txt",
-                                            "version", false,
-                                            argv[0], options_data[user_data_idx].project, options_data[user_data_idx].version, options_data[user_data_idx].language, NULL);
+            } else if (0 == strncmp(user_argv[i], "-showme:version", strlen("-showme:version"))
+                       || 0
+                              == strncmp(user_argv[i], "--showme:version",
+                                         strlen("--showme:version"))) {
+                char *str;
+                str = opal_show_help_string("help-opal-wrapper.txt", "version", false, base_argv0,
+                                            options_data[user_data_idx].project,
+                                            options_data[user_data_idx].version,
+                                            options_data[user_data_idx].language, NULL);
                 if (NULL != str) {
                     printf("%s", str);
                     free(str);
                 }
                 goto cleanup;
-            } else if (0 == strncmp(user_argv[i], "-showme:help", strlen("-showme:help")) ||
-                       0 == strncmp(user_argv[i], "--showme:help", strlen("--showme:help"))) {
+            } else if (0 == strncmp(user_argv[i], "-showme:help", strlen("-showme:help"))
+                       || 0 == strncmp(user_argv[i], "--showme:help", strlen("--showme:help"))) {
                 char *str;
-                str = opal_show_help_string("help-opal-wrapper.txt", "usage",
-                                            false, argv[0],
-                                            options_data[user_data_idx].project,
-                                            NULL);
+                str = opal_show_help_string("help-opal-wrapper.txt", "usage", false, base_argv0,
+                                            options_data[user_data_idx].project, NULL);
                 if (NULL != str) {
                     printf("%s", str);
                     free(str);
@@ -643,17 +790,15 @@ main(int argc, char *argv[])
 
                 exit_status = 0;
                 goto cleanup;
-            } else if (0 == strncmp(user_argv[i], "-showme:", strlen("-showme:")) ||
-                       0 == strncmp(user_argv[i], "--showme:", strlen("--showme:"))) {
-                fprintf(stderr, "%s: unrecognized option: %s\n", argv[0],
-                        user_argv[i]);
-                fprintf(stderr, "Type '%s --showme:help' for usage.\n",
-                        argv[0]);
+            } else if (0 == strncmp(user_argv[i], "-showme:", strlen("-showme:"))
+                       || 0 == strncmp(user_argv[i], "--showme:", strlen("--showme:"))) {
+                fprintf(stderr, "%s: unrecognized option: %s\n", base_argv0, user_argv[i]);
+                fprintf(stderr, "Type '%s --showme:help' for usage.\n", argv[0]);
                 exit_status = 1;
                 goto cleanup;
             }
 
-            flags |= (COMP_DRY_RUN|COMP_SHOW_ERROR);
+            flags |= (COMP_DRY_RUN | COMP_SHOW_ERROR);
             /* remove element from user_argv */
             opal_argv_delete(&user_argc, &user_argv, i, 1);
             --i;
@@ -666,8 +811,7 @@ main(int argc, char *argv[])
         } else if (0 == strcmp(user_argv[i], "-c")) {
             flags &= ~COMP_WANT_LINK;
             real_flag = true;
-        } else if (0 == strcmp(user_argv[i], "-E") ||
-                   0 == strcmp(user_argv[i], "-M")) {
+        } else if (0 == strcmp(user_argv[i], "-E") || 0 == strcmp(user_argv[i], "-M")) {
             flags &= ~(COMP_WANT_COMPILE | COMP_WANT_LINK);
             real_flag = true;
         } else if (0 == strcmp(user_argv[i], "-S")) {
@@ -679,40 +823,38 @@ main(int argc, char *argv[])
             /* remove element from user_argv */
             opal_argv_delete(&user_argc, &user_argv, i, 1);
             --i;
-        } else if (0 == strcmp(user_argv[i], "-static") ||
-                   0 == strcmp(user_argv[i], "--static") ||
-                   0 == strcmp(user_argv[i], "-Bstatic") ||
-                   0 == strcmp(user_argv[i], "-Wl,-static") ||
-                   0 == strcmp(user_argv[i], "-Wl,--static") ||
-                   0 == strcmp(user_argv[i], "-Wl,-Bstatic")) {
+        } else if (0 == strcmp(user_argv[i], "-static") || 0 == strcmp(user_argv[i], "--static")
+                   || 0 == strcmp(user_argv[i], "-Bstatic")
+                   || 0 == strcmp(user_argv[i], "-Wl,-static")
+                   || 0 == strcmp(user_argv[i], "-Wl,--static")
+                   || 0 == strcmp(user_argv[i], "-Wl,-Bstatic")) {
             flags |= COMP_WANT_STATIC;
-        } else if (0 == strcmp(user_argv[i], "-dynamic") ||
-                   0 == strcmp(user_argv[i], "--dynamic") ||
-                   0 == strcmp(user_argv[i], "-Bdynamic") ||
-                   0 == strcmp(user_argv[i], "-Wl,-dynamic") ||
-                   0 == strcmp(user_argv[i], "-Wl,--dynamic") ||
-                   0 == strcmp(user_argv[i], "-Wl,-Bdynamic")) {
+        } else if (0 == strcmp(user_argv[i], "-dynamic") || 0 == strcmp(user_argv[i], "--dynamic")
+                   || 0 == strcmp(user_argv[i], "-Bdynamic")
+                   || 0 == strcmp(user_argv[i], "-Wl,-dynamic")
+                   || 0 == strcmp(user_argv[i], "-Wl,--dynamic")
+                   || 0 == strcmp(user_argv[i], "-Wl,-Bdynamic")) {
             flags &= ~COMP_WANT_STATIC;
         } else if (0 == strcmp(user_argv[i], "--openmpi:linkall")) {
-            /* This is an intentionally undocummented wrapper compiler
+            /* This is an intentionally undocumented wrapper compiler
                switch.  It should only be used by Open MPI developers
                -- not end users.  It will cause mpicc to use the
                static library list, even if we're compiling
-               dynamically (i.e., it'll specifically -lopen-rte and
-               -lopen-pal (and all their dependent libs)).  We provide
+               dynamically (i.e., it'll specifically -l@OPAL_LIB_NAME@
+               (and all their dependent libs)).  We provide
                this flag for test MPI applications that also invoke
-               ORTE and/or OPAL function calls.
+               OPAL function calls.
 
                On some systems (e.g., OS X), if the top-level
-               application calls ORTE/OPAL functions and you don't -l
-               ORTE and OPAL, then the functions won't be resolved at
+               application calls OPAL functions and you don't -l
+               OPAL, then the functions won't be resolved at
                link time (i.e., the implicit library dependencies of
                libmpi won't be pulled in at link time), and therefore
                the link will fail.  This flag will cause the wrapper
-               to explicitly list the ORTE and OPAL libs on the
+               to explicitly list the OPAL libs on the
                underlying compiler command line, so the application
                will therefore link properly. */
-            flags |= COMP_WANT_LINKALL;
+            flags |= COMP_WANT_STATIC;
 
             /* remove element from user_argv */
             opal_argv_delete(&user_argc, &user_argv, i, 1);
@@ -745,7 +887,7 @@ main(int argc, char *argv[])
        opal_wrapper -showme a.c    : don't clear our flags
     */
     if (disable_flags && !((flags & COMP_DRY_RUN) && !real_flag)) {
-        flags &= ~(COMP_WANT_PREPROC|COMP_WANT_COMPILE|COMP_WANT_LINK);
+        flags &= ~(COMP_WANT_PREPROC | COMP_WANT_COMPILE | COMP_WANT_LINK);
     }
 
     /****************************************************
@@ -759,7 +901,7 @@ main(int argc, char *argv[])
         exec_argv = opal_argv_split(options_data[user_data_idx].compiler, ' ');
         exec_argc = opal_argv_count(exec_argv);
     } else {
-        exec_argv = (char **) malloc(sizeof(char*));
+        exec_argv = (char **) malloc(sizeof(char *));
         exec_argv[0] = NULL;
         exec_argc = 0;
     }
@@ -772,8 +914,7 @@ main(int argc, char *argv[])
     }
 
     if (flags & COMP_WANT_COMPILE) {
-        opal_argv_insert(&exec_argv, exec_argc,
-                         options_data[user_data_idx].comp_flags_prefix);
+        opal_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].comp_flags_prefix);
         exec_argc = opal_argv_count(exec_argv);
     }
 
@@ -796,93 +937,26 @@ main(int argc, char *argv[])
 
     /* link flags and libs */
     if (flags & COMP_WANT_LINK) {
-        bool have_static_lib;
-        bool have_dyn_lib;
-        bool use_static_libs;
-        char *filename1, *filename2;
-        struct stat buf;
-
+        /* Configure will set the libs, libs_static, link_flags, and
+           link_flags_static based on whether or not shared and static
+           libraries are enabled (see the large comment in
+           opal_setup_wrappers.m4).  The wrapper will always add libs
+           and link_flags in a link situation, and should add the
+           _static variants if -static was seen. */
         opal_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].link_flags);
         exec_argc = opal_argv_count(exec_argv);
-
-        /* Are we linking statically?  If so, decide what libraries to
-           list.  It depends on two factors:
-
-           1. Was --static (etc.) specified?
-           2. Does OMPI have static, dynamic, or both libraries installed?
-
-           Here's a matrix showing what we'll do in all 6 cases:
-
-           What's installed    --static    no --static
-           ----------------    ----------  -----------
-           ompi .so libs       -lmpi       -lmpi
-           ompi .a libs        all         all
-           ompi both libs      all         -lmpi
-
-        */
-
-        filename1 = opal_os_path( false, options_data[user_data_idx].path_libdir, options_data[user_data_idx].static_lib_file, NULL );
-        if (0 == stat(filename1, &buf)) {
-            have_static_lib = true;
-        } else {
-            have_static_lib = false;
-        }
-
-        filename2 = opal_os_path( false, options_data[user_data_idx].path_libdir, options_data[user_data_idx].dyn_lib_file, NULL );
-        if (0 == stat(filename2, &buf)) {
-            have_dyn_lib = true;
-        } else {
-            have_dyn_lib = false;
-        }
-
-        /* Determine which set of libs to use: dynamic or static.  Be
-           pedantic to make the code easy to read. */
-        if (flags & COMP_WANT_LINKALL) {
-            /* If --openmpi:linkall was specified, list all the libs
-               (i.e., the static libs) if they're available, either in
-               static or dynamic form. */
-            if (have_static_lib || have_dyn_lib) {
-                use_static_libs = true;
-            } else {
-                fprintf(stderr, "The linkall option has failed as we were unable to find either static or dynamic libs\n"
-                        "Files looked for:\n  Static: %s\n  Dynamic: %s\n",
-                        filename1, filename2);
-                free(filename1);
-                free(filename2);
-                exit(1);
-            }
-        } else if (flags & COMP_WANT_STATIC) {
-            /* If --static (or something like it) was specified, if we
-               have the static libs, then use them.  Otherwise, use
-               the dynamic libs. */
-            if (have_static_lib) {
-                use_static_libs = true;
-            } else {
-                use_static_libs = false;
-            }
-        } else {
-            /* If --static (or something like it) was NOT specified
-               (or if --dyanic, or something like it, was specified),
-               if we have the dynamic libs, then use them.  Otherwise,
-               use the static libs. */
-            if (have_dyn_lib) {
-                use_static_libs = false;
-            } else {
-                use_static_libs = true;
-            }
+        if (flags & COMP_WANT_STATIC) {
+            opal_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].link_flags_static);
+            exec_argc = opal_argv_count(exec_argv);
         }
-        free(filename1);
-        free(filename2);
-
-        if (use_static_libs) {
+        opal_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].libs);
+        exec_argc = opal_argv_count(exec_argv);
+        if (flags & COMP_WANT_STATIC) {
             opal_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].libs_static);
-        } else {
-            opal_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].libs);
+            exec_argc = opal_argv_count(exec_argv);
         }
-        exec_argc = opal_argv_count(exec_argv);
     }
 
-
     /****************************************************
      *
      * Execute the command
@@ -892,41 +966,31 @@ main(int argc, char *argv[])
     if (flags & COMP_DRY_RUN) {
         exec_command = opal_argv_join(exec_argv, ' ');
         printf("%s\n", exec_command);
+        free(exec_command);
     } else {
         char *tmp;
-
-#if 0
-        exec_command = opal_argv_join(exec_argv, ' ');
-        printf("command: %s\n", exec_command);
-#endif
-
         tmp = opal_path_findv(exec_argv[0], 0, environ, NULL);
         if (NULL == tmp) {
-            opal_show_help("help-opal-wrapper.txt", "no-compiler-found", true,
-                           exec_argv[0], NULL);
+            opal_show_help("help-opal-wrapper.txt", "no-compiler-found", true, exec_argv[0], NULL);
             errno = 0;
             exit_status = 1;
-        }  else {
+        } else {
             int status;
 
             free(exec_argv[0]);
             exec_argv[0] = tmp;
             ret = opal_few(exec_argv, &status);
-            exit_status = WIFEXITED(status) ? WEXITSTATUS(status) :
-                              (WIFSIGNALED(status) ? WTERMSIG(status) :
-                                  (WIFSTOPPED(status) ? WSTOPSIG(status) : 255));
-            if( (OPAL_SUCCESS != ret) || ((0 != exit_status) && (flags & COMP_SHOW_ERROR)) ) {
-                char* exec_command = opal_argv_join(exec_argv, ' ');
-                if( OPAL_SUCCESS != ret ) {
-                    opal_show_help("help-opal-wrapper.txt", "spawn-failed", true,
-                                   exec_argv[0], strerror(status), exec_command, NULL);
-                } else {
-#if 0
-                    opal_show_help("help-opal-wrapper.txt", "compiler-failed", true,
-                                   exec_argv[0], exit_status, exec_command, NULL);
-#endif
+            exit_status = WIFEXITED(status) ? WEXITSTATUS(status)
+                                            : (WIFSIGNALED(status)
+                                                   ? WTERMSIG(status)
+                                                   : (WIFSTOPPED(status) ? WSTOPSIG(status) : 255));
+            if ((OPAL_SUCCESS != ret) || ((0 != exit_status) && (flags & COMP_SHOW_ERROR))) {
+                if (OPAL_SUCCESS != ret) {
+                    exec_command = opal_argv_join(exec_argv, ' ');
+                    opal_show_help("help-opal-wrapper.txt", "spawn-failed", true, exec_argv[0],
+                                   strerror(status), exec_command, NULL);
+                    free(exec_command);
                 }
-                free(exec_command);
             }
         }
     }
@@ -936,11 +1000,13 @@ main(int argc, char *argv[])
      * Cleanup
      *
      ****************************************************/
- cleanup:
+cleanup:
 
     opal_argv_free(exec_argv);
     opal_argv_free(user_argv);
-    if (NULL != base_argv0) free(base_argv0);
+    if (NULL != base_argv0) {
+        free(base_argv0);
+    }
 
     if (OPAL_SUCCESS != (ret = data_finalize())) {
         return ret;
diff --git a/opal/tools/wrappers/opalc++-wrapper-data.txt.in b/opal/tools/wrappers/opalc++-wrapper-data.txt.in
index 9b3d599593d..60f6ba0b46e 100644
--- a/opal/tools/wrappers/opalc++-wrapper-data.txt.in
+++ b/opal/tools/wrappers/opalc++-wrapper-data.txt.in
@@ -16,10 +16,10 @@ preprocessor_flags=@OPAL_WRAPPER_EXTRA_CPPFLAGS@
 compiler_flags_prefix=@OPAL_WRAPPER_EXTRA_CXXFLAGS_PREFIX@
 compiler_flags=@OPAL_WRAPPER_EXTRA_CXXFLAGS@
 linker_flags=@OPAL_WRAPPER_EXTRA_LDFLAGS@
-libs=-l@OPAL_LIB_PREFIX@open-pal
-libs_static=-l@OPAL_LIB_PREFIX@open-pal @OPAL_WRAPPER_EXTRA_LIBS@
-dyn_lib_file=lib@OPAL_LIB_PREFIX@open-pal.@OPAL_DYN_LIB_SUFFIX@
-static_lib_file=lib@OPAL_LIB_PREFIX@open-pal.a
+libs=-l@OPAL_LIB_NAME@
+libs_static=-l@OPAL_LIB_NAME@ @OPAL_WRAPPER_EXTRA_LIBS@
+dyn_lib_file=lib@OPAL_LIB_NAME.@OPAL_DYN_LIB_SUFFIX@
+static_lib_file=lib@OPAL_LIB_NAME@.a
 required_file=
 includedir=${includedir}
 libdir=${libdir}
diff --git a/opal/tools/wrappers/opalcc-wrapper-data.txt.in b/opal/tools/wrappers/opalcc-wrapper-data.txt.in
index e9f71a9525b..b2617bdf5aa 100644
--- a/opal/tools/wrappers/opalcc-wrapper-data.txt.in
+++ b/opal/tools/wrappers/opalcc-wrapper-data.txt.in
@@ -16,10 +16,10 @@ preprocessor_flags=@OPAL_WRAPPER_EXTRA_CPPFLAGS@
 compiler_flags_prefix=@OPAL_WRAPPER_EXTRA_CFLAGS_PREFIX@
 compiler_flags=@OPAL_WRAPPER_EXTRA_CFLAGS@
 linker_flags=@OPAL_WRAPPER_EXTRA_LDFLAGS@
-libs=-l@OPAL_LIB_PREFIX@open-pal
-libs_static=-l@OPAL_LIB_PREFIX@open-pal @OPAL_WRAPPER_EXTRA_LIBS@
-dyn_lib_file=lib@OPAL_LIB_PREFIX@open-pal.@OPAL_DYN_LIB_SUFFIX@
-static_lib_file=lib@OPAL_LIB_PREFIX@open-pal.a
+libs=-l@OPAL_LIB_NAME@
+libs_static=-l@OPAL_LIB_NAME@ @OPAL_WRAPPER_EXTRA_LIBS@
+dyn_lib_file=lib@OPAL_LIB_NAME@.@OPAL_DYN_LIB_SUFFIX@
+static_lib_file=lib@OPAL_LIB_NAME@.a
 required_file=
 includedir=${includedir}
 libdir=${libdir}
diff --git a/opal/util/Makefile.am b/opal/util/Makefile.am
index 9018a596dd1..03020fb31e9 100644
--- a/opal/util/Makefile.am
+++ b/opal/util/Makefile.am
@@ -9,7 +9,7 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2007-2015 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2007-2022 Cisco Systems, Inc.  All rights reserved
 # Copyright (c) 2013      NVIDIA Corporation.  All rights reserved.
 # Copyright (c) 2013      Intel, Inc. All rights reserved
 # Copyright (c) 2016      Los Alamos National Security, LLC. All rights
@@ -17,9 +17,10 @@
 # Copyright (c) 2016      Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
 # Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
-# Copyright (c) 2020      Amazon.com, Inc. or its affiliates.
+# Copyright (c) 2020-2024 Amazon.com, Inc. or its affiliates.
 #                         All Rights reserved.
 # Copyright (c) 2021      Google, LLC. All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -27,14 +28,18 @@
 # $HEADER$
 #
 
-SUBDIRS = keyval
+include $(top_srcdir)/Makefile.ompi-rules
 
-dist_opaldata_DATA = help-opal-util.txt
+SUBDIRS = \
+        json \
+        keyval
 
-AM_LFLAGS = -Popal_show_help_yy
-LEX_OUTPUT_ROOT = lex.opal_show_help_yy
+EXTRA_DIST = \
+        help-opal-util.txt \
+        json/help-json.txt \
+        convert-help-files-to-c-code.py
 
-noinst_LTLIBRARIES = libopalutil.la
+noinst_LTLIBRARIES = libopalutil.la libopalutil_core.la
 
 AM_CPPFLAGS = $(LTDLINCL)
 
@@ -48,9 +53,9 @@ headers = \
 	bipartite_graph.h \
 	bipartite_graph_internal.h \
         bit_ops.h \
+        clock_gettime.h \
         cmd_line.h \
         crc.h \
-        daemon_init.h \
 	ethtool.h \
         error.h \
 	event.h \
@@ -59,11 +64,11 @@ headers = \
         if.h \
         keyval_parse.h \
         malloc.h \
+        misc.h \
         net.h \
         numtostr.h \
         opal_environ.h \
         opal_getcwd.h \
-        opal_pty.h \
         os_dirpath.h \
         os_path.h \
         output.h \
@@ -72,7 +77,6 @@ headers = \
         proc.h \
         qsort.h \
         show_help.h \
-        show_help_lex.h \
         stacktrace.h \
         string_copy.h \
         sys_limits.h \
@@ -80,10 +84,19 @@ headers = \
         uri.h \
         info_subscriber.h \
 	info.h \
-	minmax.h
+	minmax.h \
+	sha256.h
 
 libopalutil_la_SOURCES = \
         $(headers) \
+	ethtool.c \
+	event.c \
+        if.c \
+        net.c \
+        info_subscriber.c \
+        info.c
+
+libopalutil_core_la_SOURCES = \
         alfg.c \
         arch.c \
         argv.c \
@@ -91,20 +104,14 @@ libopalutil_la_SOURCES = \
 	bipartite_graph.c \
         cmd_line.c \
         crc.c \
-        daemon_init.c \
-	ethtool.c \
         error.c \
-	event.c \
         fd.c \
         few.c \
-        if.c \
         keyval_parse.c \
         malloc.c \
-        net.c \
         numtostr.c \
         opal_environ.c \
         opal_getcwd.c \
-        opal_pty.c \
         os_dirpath.c \
         os_path.c \
         output.c \
@@ -112,22 +119,28 @@ libopalutil_la_SOURCES = \
         printf.c \
         proc.c \
         qsort.c \
+        sha256.c \
         show_help.c \
-        show_help_lex.l \
+        show_help_content.c \
         stacktrace.c \
         string_copy.c \
         sys_limits.c \
-        uri.c \
-        info_subscriber.c \
-        info.c
+        uri.c
+
+show_help_content.c: convert-help-files-to-c-code.py
+	$(OMPI_V_GEN) $(PYTHON) $(abs_srcdir)/convert-help-files-to-c-code.py \
+	    --root $(abs_top_srcdir) \
+	    --out show_help_content.c
 
 if OPAL_COMPILE_TIMING
-libopalutil_la_SOURCES += timings.c
+libopalutil_core_la_SOURCES += timings.c
 endif
 
-libopalutil_la_LIBADD = \
+libopalutil_core_la_LIBADD = \
+        json/libopalutil_json.la \
         keyval/libopalutilkeyval.la
-libopalutil_la_DEPENDENCIES = \
+libopalutil_core_la_DEPENDENCIES = \
+        json/libopalutil_json.la \
         keyval/libopalutilkeyval.la
 
 # Conditionally install the header files
@@ -138,4 +151,4 @@ opal_HEADERS = $(headers)
 endif
 
 maintainer-clean-local:
-	rm -f show_help_lex.c
+	rm -f show_help_content.c
diff --git a/opal/util/alfg.c b/opal/util/alfg.c
index 2affae2cecf..40c5f7b7833 100644
--- a/opal/util/alfg.c
+++ b/opal/util/alfg.c
@@ -34,8 +34,7 @@
  */
 #define TAP1 127
 #define TAP2 97
-#define CBIT 21  /* Canonical bit */
-
+#define CBIT 21 /* Canonical bit */
 
 /**
  * @brief      Galois shift register: Used to seed the ALFG's
@@ -45,13 +44,14 @@
  * @param[out] uint32_t lsb: least significant bit of the Galois
  *             register after shift
  */
-static uint32_t galois(unsigned int *seed){
+static uint32_t galois(unsigned int *seed)
+{
 
     uint32_t lsb;
     lsb = (*seed & 1) ? 1 : 0;
     *seed >>= 1;
     /* tap it with the mask */
-    *seed = *seed ^ (lsb*MASK);
+    *seed = *seed ^ (lsb * MASK);
 
     return lsb;
 }
@@ -65,7 +65,8 @@ static opal_rng_buff_t alfg_buffer;
  * @param[in]   uint32_t seed
  * @param[out]  opal_rng_buff_t *buff: handle to ALFG buffer state
  */
-int opal_srand(opal_rng_buff_t *buff, uint32_t seed) {
+int opal_srand(opal_rng_buff_t *buff, uint32_t seed)
+{
 
     int i, j;
     uint32_t seed_cpy = seed;
@@ -73,7 +74,7 @@ int opal_srand(opal_rng_buff_t *buff, uint32_t seed) {
     buff->tap2 = TAP2 - 1;
 
     /* zero out the register */
-    for( i = 0; i < TAP1; i++){
+    for (i = 0; i < TAP1; i++) {
         buff->alfg[i] = 0;
     }
     /* set the canonical bit */
@@ -81,17 +82,16 @@ int opal_srand(opal_rng_buff_t *buff, uint32_t seed) {
 
     /* seed the ALFG register by blasting
      * the canonical rectangle with bits
-    */
-    for ( j = 1; j < TAP1; j++){
-        for( i = 1; i < 32; i++){
-            buff->alfg[j] = buff->alfg[j] ^ ((galois(&seed_cpy))<<i);
+     */
+    for (j = 1; j < TAP1; j++) {
+        for (i = 1; i < 32; i++) {
+            buff->alfg[j] = buff->alfg[j] ^ ((galois(&seed_cpy)) << i);
         }
     }
     /* copy the ALFG to the global buffer */
     memcpy(&alfg_buffer, buff, sizeof(alfg_buffer));
 
     return 1;
-
 }
 
 /**
@@ -101,7 +101,8 @@ int opal_srand(opal_rng_buff_t *buff, uint32_t seed) {
  * @param[out]  32-bit unsigned random integer
  */
 
-uint32_t opal_rand(opal_rng_buff_t *buff){
+uint32_t opal_rand(opal_rng_buff_t *buff)
+{
 
     int *tap1 = &(buff->tap1);
     int *tap2 = &(buff->tap2);
@@ -111,16 +112,15 @@ uint32_t opal_rand(opal_rng_buff_t *buff){
     /* prevent overflow */
     overflow = (uint64_t) buff->alfg[*tap1] + (uint64_t) buff->alfg[*tap2];
     /* circular buffer arithmetic */
-    temp = (*tap1 + 1) == TAP1 ?  0 :  (*tap1 + 1);
+    temp = (*tap1 + 1) == TAP1 ? 0 : (*tap1 + 1);
     /* Division modulo 2^32 */
-    buff->alfg[temp] = (uint32_t) ( overflow & ((1ULL<<32) -1));
+    buff->alfg[temp] = (uint32_t)(overflow & ((1ULL << 32) - 1));
 
     /* increment tap points */
-    *tap1 = (*tap1 + 1)%TAP1;
-    *tap2 = (*tap2 + 1)%TAP1;
+    *tap1 = (*tap1 + 1) % TAP1;
+    *tap2 = (*tap2 + 1) % TAP1;
 
     return buff->alfg[temp];
-
 }
 
 /**
@@ -129,7 +129,8 @@ uint32_t opal_rand(opal_rng_buff_t *buff){
  * @param[in]  none
  * @param[out] int, the same as normal rand(3)
  */
-int opal_random(void){
+int opal_random(void)
+{
     /* always return a positive int */
-    return (int)(opal_rand(&alfg_buffer) & 0x7FFFFFFF);
+    return (int) (opal_rand(&alfg_buffer) & 0x7FFFFFFF);
 }
diff --git a/opal/util/alfg.h b/opal/util/alfg.h
index fe8b48c0fe1..7103ee0e7a3 100644
--- a/opal/util/alfg.h
+++ b/opal/util/alfg.h
@@ -17,7 +17,6 @@
 
 #include "opal_stdint.h"
 
-
 struct opal_rng_buff_t {
     uint32_t alfg[127];
     int tap1;
@@ -25,7 +24,6 @@ struct opal_rng_buff_t {
 };
 typedef struct opal_rng_buff_t opal_rng_buff_t;
 
-
 /* NOTE: UNLIKE OTHER OPAL FUNCTIONS, THIS FUNCTION RETURNS A 1 IF
  * SUCCESSFUL INSTEAD OF OPAL_SUCCESS */
 OPAL_DECLSPEC int opal_srand(opal_rng_buff_t *buff, uint32_t seed);
diff --git a/opal/util/arch.c b/opal/util/arch.c
index 8c18ed5305f..207d1d11d7e 100644
--- a/opal/util/arch.c
+++ b/opal/util/arch.c
@@ -23,54 +23,58 @@
 
 uint32_t opal_local_arch = 0xFFFFFFFF;
 
-static inline int32_t opal_arch_isbigendian ( void )
+static inline int32_t opal_arch_isbigendian(void)
 {
     const uint32_t value = 0x12345678;
-    const char *ptr = (char*)&value;
+    const char *ptr = (char *) &value;
     int x = 0;
 
     /* if( sizeof(int) == 8 ) x = 4; */
-    if( ptr[x] == 0x12)  return 1; /* big endian, true */
-    if( ptr[x] == 0x78 ) return 0; /* little endian, false */
-    assert( 0 );  /* unknown architecture not little nor big endian */
+    if (ptr[x] == 0x12) {
+        return 1; /* big endian, true */
+    }
+    if (ptr[x] == 0x78) {
+        return 0; /* little endian, false */
+    }
+    assert(0); /* unknown architecture not little nor big endian */
     return -1;
 }
 
 /* we must find which representation of long double is used
  * intel or sparc. Both of them represent the long doubles using a close to
  * IEEE representation (seeeeeee..emmm...m) where the mantissa look like
- * 1.????. For the intel representaion the 1 is explicit, and for the sparc
+ * 1.????. For the intel representation the 1 is explicit, and for the sparc
  * the first one is implicit. If we take the number 2.0 the exponent is 1
  * and the mantissa is 1.0 (the sign of course should be 0). So if we check
  * for the first one in the binary representation of the number, we will
- * find the bit from the exponent, so the next one should be the begining
- * of the mantissa. If it's 1 then we have an intel representaion, if not
+ * find the bit from the exponent, so the next one should be the beginning
+ * of the mantissa. If it's 1 then we have an intel representation, if not
  * we have a sparc one. QED
  */
-static inline int32_t opal_arch_ldisintel( void )
+static inline int32_t opal_arch_ldisintel(void)
 {
     long double ld = 2.0;
     int i, j;
-    uint32_t* pui = (uint32_t*)(void*)&ld;
+    uint32_t *pui = (uint32_t *) (void *) &ld;
 
     j = LDBL_MANT_DIG / 32;
     i = (LDBL_MANT_DIG % 32) - 1;
-    if( opal_arch_isbigendian() ) { /* big endian */
+    if (opal_arch_isbigendian()) { /* big endian */
         j = (sizeof(long double) / sizeof(unsigned int)) - j;
-        if( i < 0 ) {
+        if (i < 0) {
             i = 31;
-            j = j+1;
+            j = j + 1;
         }
     } else {
-        if( i < 0 ) {
+        if (i < 0) {
             i = 31;
-            j = j-1;
+            j = j - 1;
         }
     }
     return ((pui[j] & (1u << i)) ? 1 : 0);
 }
 
-static inline void opal_arch_setmask ( uint32_t *var, uint32_t mask)
+static inline void opal_arch_setmask(uint32_t *var, uint32_t mask)
 {
     *var |= mask;
 }
@@ -80,63 +84,69 @@ int opal_arch_init(void)
     opal_local_arch = (OPAL_ARCH_HEADERMASK | OPAL_ARCH_UNUSEDMASK);
 
     /* Handle the size of long (can hold a pointer) */
-    if( 8 == sizeof(long) )
-        opal_arch_setmask( &opal_local_arch, OPAL_ARCH_LONGIS64 );
+    if (8 == sizeof(long)) {
+        opal_arch_setmask(&opal_local_arch, OPAL_ARCH_LONGIS64);
+    }
 
     /* sizeof bool */
-    if (1 == sizeof(bool) ) {
-        opal_arch_setmask( &opal_local_arch, OPAL_ARCH_BOOLIS8);
+    if (1 == sizeof(bool)) {
+        opal_arch_setmask(&opal_local_arch, OPAL_ARCH_BOOLIS8);
     } else if (2 == sizeof(bool)) {
-        opal_arch_setmask( &opal_local_arch, OPAL_ARCH_BOOLIS16);
+        opal_arch_setmask(&opal_local_arch, OPAL_ARCH_BOOLIS16);
     } else if (4 == sizeof(bool)) {
-        opal_arch_setmask( &opal_local_arch, OPAL_ARCH_BOOLIS32);
+        opal_arch_setmask(&opal_local_arch, OPAL_ARCH_BOOLIS32);
     }
 
     /* Note that fortran logical size is set later, to make
        abstractions a little less painful... */
 
     /* Initialize the information regarding the long double */
-    if( 12 == sizeof(long double) )
-        opal_arch_setmask( &opal_local_arch, OPAL_ARCH_LONGDOUBLEIS96 );
-    else if( 16 == sizeof(long double) )
-        opal_arch_setmask( &opal_local_arch, OPAL_ARCH_LONGDOUBLEIS128 );
+    if (12 == sizeof(long double)) {
+        opal_arch_setmask(&opal_local_arch, OPAL_ARCH_LONGDOUBLEIS96);
+    } else if (16 == sizeof(long double)) {
+        opal_arch_setmask(&opal_local_arch, OPAL_ARCH_LONGDOUBLEIS128);
+    }
 
     /* Big endian or little endian ? That's the question */
-    if( opal_arch_isbigendian() )
-        opal_arch_setmask( &opal_local_arch, OPAL_ARCH_ISBIGENDIAN );
+    if (opal_arch_isbigendian()) {
+        opal_arch_setmask(&opal_local_arch, OPAL_ARCH_ISBIGENDIAN);
+    }
 
     /* What's the maximum exponent ? */
-    if ( LDBL_MAX_EXP == 16384 )
-        opal_arch_setmask( &opal_local_arch, OPAL_ARCH_LDEXPSIZEIS15 );
+    if (LDBL_MAX_EXP == 16384) {
+        opal_arch_setmask(&opal_local_arch, OPAL_ARCH_LDEXPSIZEIS15);
+    }
 
     /* How about the length in bits of the mantissa */
-    if ( LDBL_MANT_DIG == 64 )
-        opal_arch_setmask( &opal_local_arch, OPAL_ARCH_LDMANTDIGIS64 );
-    else if ( LDBL_MANT_DIG == 105 )
-        opal_arch_setmask( &opal_local_arch, OPAL_ARCH_LDMANTDIGIS105 );
-    else if ( LDBL_MANT_DIG == 106 )
-        opal_arch_setmask( &opal_local_arch, OPAL_ARCH_LDMANTDIGIS106 );
-    else if ( LDBL_MANT_DIG == 107 )
-        opal_arch_setmask( &opal_local_arch, OPAL_ARCH_LDMANTDIGIS107 );
-    else if ( LDBL_MANT_DIG == 113 )
-        opal_arch_setmask( &opal_local_arch, OPAL_ARCH_LDMANTDIGIS113 );
+    if (LDBL_MANT_DIG == 64) {
+        opal_arch_setmask(&opal_local_arch, OPAL_ARCH_LDMANTDIGIS64);
+    } else if (LDBL_MANT_DIG == 105) {
+        opal_arch_setmask(&opal_local_arch, OPAL_ARCH_LDMANTDIGIS105);
+    } else if (LDBL_MANT_DIG == 106) {
+        opal_arch_setmask(&opal_local_arch, OPAL_ARCH_LDMANTDIGIS106);
+    } else if (LDBL_MANT_DIG == 107) {
+        opal_arch_setmask(&opal_local_arch, OPAL_ARCH_LDMANTDIGIS107);
+    } else if (LDBL_MANT_DIG == 113) {
+        opal_arch_setmask(&opal_local_arch, OPAL_ARCH_LDMANTDIGIS113);
+    }
 
     /* Intel data representation or Sparc ? */
-    if( opal_arch_ldisintel() )
-        opal_arch_setmask( &opal_local_arch, OPAL_ARCH_LDISINTEL );
+    if (opal_arch_ldisintel()) {
+        opal_arch_setmask(&opal_local_arch, OPAL_ARCH_LDISINTEL);
+    }
 
     return OPAL_SUCCESS;
 }
 
-int32_t opal_arch_checkmask ( uint32_t *var, uint32_t mask )
+int32_t opal_arch_checkmask(uint32_t *var, uint32_t mask)
 {
     unsigned int tmpvar = *var;
 
     /* Check whether the headers are set correctly,
        or whether this is an erroneous integer */
-    if( !((*var) & OPAL_ARCH_HEADERMASK) ) {
-        if( (*var) & OPAL_ARCH_HEADERMASK2 ) {
-            char* pcDest, *pcSrc;
+    if (!((*var) & OPAL_ARCH_HEADERMASK)) {
+        if ((*var) & OPAL_ARCH_HEADERMASK2) {
+            char *pcDest, *pcSrc;
             /* Both ends of this integer have the wrong settings,
                maybe its just the wrong endian-representation. Try
                to swap it and check again. If it looks now correct,
@@ -144,33 +154,34 @@ int32_t opal_arch_checkmask ( uint32_t *var, uint32_t mask )
             */
 
             pcDest = (char *) &tmpvar;
-            pcSrc  = (char *) var + 3;
+            pcSrc = (char *) var + 3;
             *pcDest++ = *pcSrc--;
             *pcDest++ = *pcSrc--;
             *pcDest++ = *pcSrc--;
             *pcDest++ = *pcSrc--;
 
-            if( (tmpvar & OPAL_ARCH_HEADERMASK) && (!(tmpvar & OPAL_ARCH_HEADERMASK2)) ) {
+            if ((tmpvar & OPAL_ARCH_HEADERMASK) && (!(tmpvar & OPAL_ARCH_HEADERMASK2))) {
                 *var = tmpvar;
-            } else
+            } else {
                 return -1;
-        } else
+            }
+        } else {
             return -1;
+        }
     }
 
     /* Here is the real evaluation of the bitmask */
-    return ( ((*var) & mask) == mask );
+    return (((*var) & mask) == mask);
 }
 
-int
-opal_arch_set_fortran_logical_size(uint32_t size)
+int opal_arch_set_fortran_logical_size(uint32_t size)
 {
     if (1 == size) {
-        opal_arch_setmask( &opal_local_arch, OPAL_ARCH_LOGICALIS8);
+        opal_arch_setmask(&opal_local_arch, OPAL_ARCH_LOGICALIS8);
     } else if (2 == size) {
-        opal_arch_setmask( &opal_local_arch, OPAL_ARCH_LOGICALIS16);
+        opal_arch_setmask(&opal_local_arch, OPAL_ARCH_LOGICALIS16);
     } else if (4 == size) {
-        opal_arch_setmask( &opal_local_arch, OPAL_ARCH_LOGICALIS32);
+        opal_arch_setmask(&opal_local_arch, OPAL_ARCH_LOGICALIS32);
     }
 
     return OPAL_SUCCESS;
diff --git a/opal/util/arch.h b/opal/util/arch.h
index a1a0446e1f3..b5d9050f6ef 100644
--- a/opal/util/arch.h
+++ b/opal/util/arch.h
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2006 The University of Tennessee and The University
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2006 High Performance Computing Center Stuttgart,
@@ -22,9 +22,8 @@
 
 #include "opal_config.h"
 
-#include <float.h>
 #include <assert.h>
-
+#include <float.h>
 
 /***************************************************
 ** This file tries to classify the most relevant
@@ -38,7 +37,6 @@
 **
 *****************************************************/
 
-
 /*****************************************************************
 ** Part 1: Integer representation.
 **
@@ -53,7 +51,7 @@
 ** The fortran integer is dismissed here, since there is no
 ** platform known to me, were fortran and C-integer do not match
 **
-** The following abbriviations are introduced:
+** The following abbreviations are introduced:
 **
 ** a) il32 (int long are 32 bits) (e.g. IA32 LINUX, SGI n32, SUN)
 **
@@ -88,7 +86,7 @@
 **   real
 **   double precision
 **
-**   Unfortunatly, here we have to take care, whether float and real,
+**   Unfortunately, here we have to take care, whether float and real,
 **   respectively double and double precision do match...
 **
 **  a) fr32 (float and real are 32 bits) (e.g. SGI n32 and 64, SUN, NEC SX5,...)
@@ -128,7 +126,7 @@
 **                (128                 106       10) (special flags required).
 **  SX5:           128                 105       22
 **
-** We will not implement all of these routiens, but we consider them
+** We will not implement all of these routines, but we consider them
 ** now when defining the header-settings
 **
 ***********************************************************************/
@@ -153,7 +151,6 @@
 ** ? alpha supports both, big and little endian
 ***********************************************************************/
 
-
 /* Current conclusions:
 ** we need at the moment three settings:
 ** - big/little endian ?
@@ -165,7 +162,7 @@
 ** To store this in a 32 bit integer, we use the following definition:
 **
 **     1        2        3        4
-** 12345678 12345678 12345678 12345678
+** 87654321 87654321 87654321 87654321
 **
 ** 1. Byte:
 **   bits 1 & 2: 00 (header) (to recognize the correct end)
@@ -174,14 +171,14 @@
 **   bits 7 & 8: reserved for later use. currently set to 00
 ** 2. Byte:
 **   bits 1 & 2: length of long: 00 = 32, 01 = 64
-**   bits 3 & 4: lenght of long long (not used currently, set to 00).
+**   bits 3 & 4: length of long long (not used currently, set to 00).
 **   bits 5 & 6: length of C/C++ bool (00 = 8, 01 = 16, 10 = 32)
 **   bits 7 & 8: length of Fortran Logical (00 = 8, 01 = 16, 10 = 32)
 ** 3. Byte:
 **   bits 1 & 2: length of long double: 00=64, 01=96,10 = 128
 **   bits 3 & 4: no. of rel. bits in the exponent: 00 = 10, 01 = 14)
-**   bits 5 - 7: no. of bits of mantisse ( 000 = 53,  001 = 64, 010 = 105,
-**                                         011 = 106, 100 = 107,101 = 113 )
+**   bits 5 - 7: no. of bits of mantisse ( 000 = 53,  001 = 64,  010 = 105,
+**                                         011 = 106, 100 = 107, 101 = 113 )
 **   bit      8: intel or sparc representation of mantisse (0 = sparc,
 **                                         1 = intel )
 ** 4. Byte:
@@ -193,41 +190,49 @@
 
 /* These masks implement the specification above above */
 
-#define OPAL_ARCH_HEADERMASK      0x03000000 /* set the fields for the header */
-#define OPAL_ARCH_HEADERMASK2     0x00000003 /* other end, needed for checks */
-#define OPAL_ARCH_UNUSEDMASK      0xfc000000 /* mark the unused fields */
+#define OPAL_ARCH_HEADERMASK  0x03000000 /* set the fields for the header */
+#define OPAL_ARCH_HEADERMASK2 0x00000003 /* other end, needed for checks */
+#define OPAL_ARCH_UNUSEDMASK  0xfc000000 /* mark the unused fields */
 
 /* BYTE 1 */
-#define OPAL_ARCH_ISBIGENDIAN     0x00000008
+#define OPAL_ARCH_ISBIGENDIAN 0x00000008
 
 /* BYTE 2 */
-#define OPAL_ARCH_LONGISxx        0x0000c000  /* mask for sizeof long */
-#define OPAL_ARCH_LONGIS64        0x00001000
-#define OPAL_ARCH_LONGLONGISxx    0x00003000  /* mask for sizeof long long */
+#define OPAL_ARCH_LONGISxx     0x00004000 /* mask for sizeof long */
+#define OPAL_ARCH_LONGIS32     0x00000000
+#define OPAL_ARCH_LONGIS64     0x00004000
+#define OPAL_ARCH_LONGLONGISxx 0x00003000 /* mask for sizeof long long */
 
-#define OPAL_ARCH_BOOLISxx        0x00000c00  /* mask for sizeof bool */
-#define OPAL_ARCH_BOOLIS8         0x00000000  /* bool is 8 bits */
-#define OPAL_ARCH_BOOLIS16        0x00000400  /* bool is 16 bits */
-#define OPAL_ARCH_BOOLIS32        0x00000800  /* bool is 32 bits */
+#define OPAL_ARCH_BOOLISxx 0x00000c00 /* mask for sizeof bool */
+#define OPAL_ARCH_BOOLIS8  0x00000000 /* bool is 8 bits */
+#define OPAL_ARCH_BOOLIS16 0x00000400 /* bool is 16 bits */
+#define OPAL_ARCH_BOOLIS32 0x00000800 /* bool is 32 bits */
 
-#define OPAL_ARCH_LOGICALISxx     0x00000300  /* mask for sizeof Fortran logical */
-#define OPAL_ARCH_LOGICALIS8      0x00000000  /* logical is 8 bits */
-#define OPAL_ARCH_LOGICALIS16     0x00000100  /* logical is 16 bits */
-#define OPAL_ARCH_LOGICALIS32     0x00000200  /* logical is 32 bits */
+#define OPAL_ARCH_LOGICALISxx 0x00000300 /* mask for sizeof Fortran logical */
+#define OPAL_ARCH_LOGICALIS8  0x00000000 /* logical is 8 bits */
+#define OPAL_ARCH_LOGICALIS16 0x00000100 /* logical is 16 bits */
+#define OPAL_ARCH_LOGICALIS32 0x00000200 /* logical is 32 bits */
 
 /* BYTE 3 */
-#define OPAL_ARCH_LONGDOUBLEIS96  0x00020000
-#define OPAL_ARCH_LONGDOUBLEIS128 0x00010000
-
-#define OPAL_ARCH_LDEXPSIZEIS15   0x00080000
-
-#define OPAL_ARCH_LDMANTDIGIS64   0x00400000
-#define OPAL_ARCH_LDMANTDIGIS105  0x00200000
-#define OPAL_ARCH_LDMANTDIGIS106  0x00600000
-#define OPAL_ARCH_LDMANTDIGIS107  0x00100000
-#define OPAL_ARCH_LDMANTDIGIS113  0x00500000
-
-#define OPAL_ARCH_LDISINTEL       0x00800000
+#define OPAL_ARCH_LONGDOUBLEISxx  0x00030000
+#define OPAL_ARCH_LONGDOUBLEIS64  0x00000000
+#define OPAL_ARCH_LONGDOUBLEIS96  0x00010000
+#define OPAL_ARCH_LONGDOUBLEIS128 0x00020000
+
+#define OPAL_ARCH_LDEXPSIZEISxx 0x000c0000
+#define OPAL_ARCH_LDEXPSIZEIS11 0x00000000
+#define OPAL_ARCH_LDEXPSIZEIS15 0x00080000
+
+#define OPAL_ARCH_LDMANTDIGISxx  0x00700000 /* b 0111 0000 */
+/* 53:b000 64:b001 105:b010 106:b011 107:b100 113:b101 */
+#define OPAL_ARCH_LDMANTDIGIS53  0x00000000
+#define OPAL_ARCH_LDMANTDIGIS64  0x00100000
+#define OPAL_ARCH_LDMANTDIGIS105 0x00200000
+#define OPAL_ARCH_LDMANTDIGIS106 0x00300000
+#define OPAL_ARCH_LDMANTDIGIS107 0x00400000
+#define OPAL_ARCH_LDMANTDIGIS113 0x00500000
+
+#define OPAL_ARCH_LDISINTEL 0x00800000
 
 BEGIN_C_DECLS
 
@@ -237,12 +242,11 @@ OPAL_DECLSPEC extern uint32_t opal_local_arch;
 /* Initialize architecture and determine all but fortran logical fields */
 OPAL_DECLSPEC int opal_arch_init(void);
 
-OPAL_DECLSPEC int32_t opal_arch_checkmask ( uint32_t *var, uint32_t mask );
+OPAL_DECLSPEC int32_t opal_arch_checkmask(uint32_t *var, uint32_t mask);
 
 /* Set fortran logical fields after init, to keep fortran out of opal... */
 OPAL_DECLSPEC int opal_arch_set_fortran_logical_size(uint32_t size);
 
 END_C_DECLS
 
-#endif  /* OPAL_ARCH_H_HAS_BEEN_INCLUDED */
-
+#endif /* OPAL_ARCH_H_HAS_BEEN_INCLUDED */
diff --git a/opal/util/argv.c b/opal/util/argv.c
index 2464013e46c..cc725db824b 100644
--- a/opal/util/argv.c
+++ b/opal/util/argv.c
@@ -25,13 +25,12 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include "opal/constants.h"
 #include "opal/util/argv.h"
 #include "opal/util/string_copy.h"
-#include "opal/constants.h"
 
 #define ARGSIZE 128
 
-
 /*
  * Append a string to the end of a new or existing argv array.
  */
@@ -53,24 +52,24 @@ int opal_argv_append_nosize(char ***argv, const char *arg)
 {
     int argc;
 
-  /* Create new argv. */
+    /* Create new argv. */
 
-  if (NULL == *argv) {
-    *argv = (char**) malloc(2 * sizeof(char *));
     if (NULL == *argv) {
-        return OPAL_ERR_OUT_OF_RESOURCE;
+        *argv = (char **) malloc(2 * sizeof(char *));
+        if (NULL == *argv) {
+            return OPAL_ERR_OUT_OF_RESOURCE;
+        }
+        argc = 0;
+        (*argv)[0] = NULL;
+        (*argv)[1] = NULL;
     }
-    argc = 0;
-    (*argv)[0] = NULL;
-    (*argv)[1] = NULL;
-  }
 
-  /* Extend existing argv. */
-  else {
+    /* Extend existing argv. */
+    else {
         /* count how many entries currently exist */
         argc = opal_argv_count(*argv);
 
-        *argv = (char**) realloc(*argv, (argc + 2) * sizeof(char *));
+        *argv = (char **) realloc(*argv, (argc + 2) * sizeof(char *));
         if (NULL == *argv) {
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
@@ -97,7 +96,7 @@ int opal_argv_prepend_nosize(char ***argv, const char *arg)
     /* Create new argv. */
 
     if (NULL == *argv) {
-        *argv = (char**) malloc(2 * sizeof(char *));
+        *argv = (char **) malloc(2 * sizeof(char *));
         if (NULL == *argv) {
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
@@ -107,15 +106,15 @@ int opal_argv_prepend_nosize(char ***argv, const char *arg)
         /* count how many entries currently exist */
         argc = opal_argv_count(*argv);
 
-        *argv = (char**) realloc(*argv, (argc + 2) * sizeof(char *));
+        *argv = (char **) realloc(*argv, (argc + 2) * sizeof(char *));
         if (NULL == *argv) {
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
-        (*argv)[argc+1] = NULL;
+        (*argv)[argc + 1] = NULL;
 
         /* shift all existing elements down 1 */
-        for (i=argc; 0 < i; i--) {
-            (*argv)[i] = (*argv)[i-1];
+        for (i = argc; 0 < i; i--) {
+            (*argv)[i] = (*argv)[i - 1];
         }
         (*argv)[0] = strdup(arg);
     }
@@ -135,7 +134,7 @@ int opal_argv_append_unique_nosize(char ***argv, const char *arg, bool overwrite
     }
 
     /* see if this arg is already present in the array */
-    for (i=0; NULL != (*argv)[i]; i++) {
+    for (i = 0; NULL != (*argv)[i]; i++) {
         if (0 == strcmp(arg, (*argv)[i])) {
             /* already exists - are we authorized to overwrite? */
             if (overwrite) {
@@ -155,94 +154,97 @@ int opal_argv_append_unique_nosize(char ***argv, const char *arg, bool overwrite
  */
 void opal_argv_free(char **argv)
 {
-  char **p;
+    char **p;
 
-  if (NULL == argv)
-    return;
+    if (NULL == argv) {
+        return;
+    }
 
-  for (p = argv; NULL != *p; ++p) {
-    free(*p);
-  }
+    for (p = argv; NULL != *p; ++p) {
+        free(*p);
+    }
 
-  free(argv);
+    free(argv);
 }
 
-
 /*
  * Split a string into a NULL-terminated argv array.
  */
-static char **opal_argv_split_inter(const char *src_string, int delimiter,
-        int include_empty)
+static char **opal_argv_split_inter(const char *src_string, int delimiter, int include_empty)
 {
-  char arg[ARGSIZE];
-  char **argv = NULL;
-  const char *p;
-  char *argtemp;
-  int argc = 0;
-  size_t arglen;
-
-  while (src_string && *src_string) {
-    p = src_string;
-    arglen = 0;
+    char arg[ARGSIZE];
+    char **argv = NULL;
+    const char *p;
+    char *argtemp;
+    int argc = 0;
+    size_t arglen;
+
+    while (src_string && *src_string) {
+        p = src_string;
+        arglen = 0;
+
+        while (('\0' != *p) && (*p != delimiter)) {
+            ++p;
+            ++arglen;
+        }
 
-    while (('\0' != *p) && (*p != delimiter)) {
-      ++p;
-      ++arglen;
-    }
+        /* zero length argument, skip */
 
-    /* zero length argument, skip */
+        if (src_string == p) {
+            if (include_empty) {
+                arg[0] = '\0';
+                if (OPAL_SUCCESS != opal_argv_append(&argc, &argv, arg)) {
+                    return NULL;
+                }
+            }
+        }
 
-    if (src_string == p) {
-      if (include_empty) {
-        arg[0] = '\0';
-        if (OPAL_SUCCESS != opal_argv_append(&argc, &argv, arg))
-          return NULL;
-      }
-    }
+        /* tail argument, add straight from the original string */
 
-    /* tail argument, add straight from the original string */
+        else if ('\0' == *p) {
+            if (OPAL_SUCCESS != opal_argv_append(&argc, &argv, src_string)) {
+                return NULL;
+            }
+            src_string = p;
+            continue;
+        }
 
-    else if ('\0' == *p) {
-      if (OPAL_SUCCESS != opal_argv_append(&argc, &argv, src_string))
-	return NULL;
-      src_string = p;
-      continue;
-    }
+        /* long argument, malloc buffer, copy and add */
 
-    /* long argument, malloc buffer, copy and add */
+        else if (arglen > (ARGSIZE - 1)) {
+            argtemp = (char *) malloc(arglen + 1);
+            if (NULL == argtemp) {
+                return NULL;
+            }
 
-    else if (arglen > (ARGSIZE - 1)) {
-        argtemp = (char*) malloc(arglen + 1);
-      if (NULL == argtemp)
-	return NULL;
+            opal_string_copy(argtemp, src_string, arglen + 1);
+            argtemp[arglen] = '\0';
 
-      opal_string_copy(argtemp, src_string, arglen + 1);
-      argtemp[arglen] = '\0';
+            if (OPAL_SUCCESS != opal_argv_append(&argc, &argv, argtemp)) {
+                free(argtemp);
+                return NULL;
+            }
 
-      if (OPAL_SUCCESS != opal_argv_append(&argc, &argv, argtemp)) {
-	free(argtemp);
-	return NULL;
-      }
+            free(argtemp);
+        }
 
-      free(argtemp);
-    }
+        /* short argument, copy to buffer and add */
 
-    /* short argument, copy to buffer and add */
+        else {
+            opal_string_copy(arg, src_string, arglen + 1);
+            arg[arglen] = '\0';
 
-    else {
-      opal_string_copy(arg, src_string, arglen + 1);
-      arg[arglen] = '\0';
+            if (OPAL_SUCCESS != opal_argv_append(&argc, &argv, arg)) {
+                return NULL;
+            }
+        }
 
-      if (OPAL_SUCCESS != opal_argv_append(&argc, &argv, arg))
-	return NULL;
+        src_string = p + 1;
     }
 
-    src_string = p + 1;
-  }
-
-  /* All done */
+    /* All done */
 
-  return argv;
+    return argv;
 }
 
 char **opal_argv_split(const char *src_string, int delimiter)
@@ -260,75 +262,76 @@ char **opal_argv_split_with_empty(const char *src_string, int delimiter)
  */
 int opal_argv_count(char **argv)
 {
-  char **p;
-  int i;
+    char **p;
+    int i;
 
-  if (NULL == argv)
-    return 0;
+    if (NULL == argv) {
+        return 0;
+    }
 
-  for (i = 0, p = argv; *p; i++, p++)
-    continue;
+    for (i = 0, p = argv; *p; i++, p++) {
+        continue;
+    }
 
-  return i;
+    return i;
 }
 
-
 /*
  * Join all the elements of an argv array into a single
  * newly-allocated string.
  */
 char *opal_argv_join(char **argv, int delimiter)
 {
-  char **p;
-  char *pp;
-  char *str;
-  size_t str_len = 0;
-  size_t i;
+    char **p;
+    char *pp;
+    char *str;
+    size_t str_len = 0;
+    size_t i;
 
-  /* Bozo case */
+    /* Bozo case */
 
-  if (NULL == argv || NULL == argv[0]) {
-      return strdup("");
-  }
+    if (NULL == argv || NULL == argv[0]) {
+        return strdup("");
+    }
 
-  /* Find the total string length in argv including delimiters.  The
-     last delimiter is replaced by the NULL character. */
+    /* Find the total string length in argv including delimiters.  The
+       last delimiter is replaced by the NULL character. */
 
-  for (p = argv; *p; ++p) {
-    str_len += strlen(*p) + 1;
-  }
+    for (p = argv; *p; ++p) {
+        str_len += strlen(*p) + 1;
+    }
 
-  /* Allocate the string. */
+    /* Allocate the string. */
 
-  if (NULL == (str = (char*) malloc(str_len)))
-    return NULL;
+    if (NULL == (str = (char *) malloc(str_len))) {
+        return NULL;
+    }
 
-  /* Loop filling in the string. */
+    /* Loop filling in the string. */
 
-  str[--str_len] = '\0';
-  p = argv;
-  pp = *p;
+    str[--str_len] = '\0';
+    p = argv;
+    pp = *p;
 
-  for (i = 0; i < str_len; ++i) {
-    if ('\0' == *pp) {
+    for (i = 0; i < str_len; ++i) {
+        if ('\0' == *pp) {
 
-      /* End of a string, fill in a delimiter and go to the next
-         string. */
+            /* End of a string, fill in a delimiter and go to the next
+               string. */
 
-      str[i] = (char) delimiter;
-      ++p;
-      pp = *p;
-    } else {
-      str[i] = *pp++;
+            str[i] = (char) delimiter;
+            ++p;
+            pp = *p;
+        } else {
+            str[i] = *pp++;
+        }
     }
-  }
 
-  /* All done */
+    /* All done */
 
-  return str;
+    return str;
 }
 
-
 /*
  * Join all the elements of an argv array from within a
  * specified range into a single newly-allocated string.
@@ -343,14 +346,14 @@ char *opal_argv_join_range(char **argv, size_t start, size_t end, int delimiter)
 
     /* Bozo case */
 
-    if (NULL == argv || NULL == argv[0] || (int)start >= opal_argv_count(argv)) {
+    if (NULL == argv || NULL == argv[0] || (int) start >= opal_argv_count(argv)) {
         return strdup("");
     }
 
     /* Find the total string length in argv including delimiters.  The
      last delimiter is replaced by the NULL character. */
 
-    for (p = &argv[start], i=start; *p && i < end; ++p, ++i) {
+    for (p = &argv[start], i = start; *p && i < end; ++p, ++i) {
         str_len += strlen(*p) + 1;
     }
 
@@ -360,7 +363,7 @@ char *opal_argv_join_range(char **argv, size_t start, size_t end, int delimiter)
 
     /* Allocate the string. */
 
-    if (NULL == (str = (char*) malloc(str_len))) {
+    if (NULL == (str = (char *) malloc(str_len))) {
         return NULL;
     }
 
@@ -389,59 +392,58 @@ char *opal_argv_join_range(char **argv, size_t start, size_t end, int delimiter)
     return str;
 }
 
-
 /*
  * Return the number of bytes consumed by an argv array.
  */
 size_t opal_argv_len(char **argv)
 {
-  char **p;
-  size_t length;
+    char **p;
+    size_t length;
 
-  if (NULL == argv)
-    return (size_t) 0;
+    if (NULL == argv) {
+        return (size_t) 0;
+    }
 
-  length = sizeof(char *);
+    length = sizeof(char *);
 
-  for (p = argv; *p; ++p) {
-    length += strlen(*p) + 1 + sizeof(char *);
-  }
+    for (p = argv; *p; ++p) {
+        length += strlen(*p) + 1 + sizeof(char *);
+    }
 
-  return length;
+    return length;
 }
 
-
 /*
  * Copy a NULL-terminated argv array.
  */
 char **opal_argv_copy(char **argv)
 {
-  char **dupv = NULL;
-  int dupc = 0;
+    char **dupv = NULL;
+    int dupc = 0;
 
-  if (NULL == argv)
-    return NULL;
+    if (NULL == argv) {
+        return NULL;
+    }
 
-  /* create an "empty" list, so that we return something valid if we
-     were passed a valid list with no contained elements */
-  dupv = (char**) malloc(sizeof(char*));
-  dupv[0] = NULL;
+    /* create an "empty" list, so that we return something valid if we
+       were passed a valid list with no contained elements */
+    dupv = (char **) malloc(sizeof(char *));
+    dupv[0] = NULL;
 
-  while (NULL != *argv) {
-    if (OPAL_SUCCESS != opal_argv_append(&dupc, &dupv, *argv)) {
-      opal_argv_free(dupv);
-      return NULL;
-    }
+    while (NULL != *argv) {
+        if (OPAL_SUCCESS != opal_argv_append(&dupc, &dupv, *argv)) {
+            opal_argv_free(dupv);
+            return NULL;
+        }
 
-    ++argv;
-  }
+        ++argv;
+    }
 
-  /* All done */
+    /* All done */
 
-  return dupv;
+    return dupv;
 }
 
-
 int opal_argv_delete(int *argc, char ***argv, int start, int num_to_delete)
 {
     int i;
@@ -485,8 +487,10 @@ int opal_argv_delete(int *argc, char ***argv, int start, int num_to_delete)
     (*argv)[i] = NULL;
 
     /* adjust the argv array */
-    tmp = (char**)realloc(*argv, sizeof(char*) * (i + 1));
-    if (NULL != tmp) *argv = tmp;
+    tmp = (char **) realloc(*argv, sizeof(char *) * (i + 1));
+    if (NULL != tmp) {
+        *argv = tmp;
+    }
 
     /* adjust the argc */
     (*argc) -= num_to_delete;
@@ -494,7 +498,6 @@ int opal_argv_delete(int *argc, char ***argv, int start, int num_to_delete)
     return OPAL_SUCCESS;
 }
 
-
 int opal_argv_insert(char ***target, int start, char **source)
 {
     int i, source_count, target_count;
@@ -524,15 +527,13 @@ int opal_argv_insert(char ***target, int start, char **source)
 
         /* Alloc out new space */
 
-        *target = (char**) realloc(*target,
-                                   sizeof(char *) * (target_count + source_count + 1));
+        *target = (char **) realloc(*target, sizeof(char *) * (target_count + source_count + 1));
 
         /* Move suffix items down to the end */
 
         suffix_count = target_count - start;
         for (i = suffix_count - 1; i >= 0; --i) {
-            (*target)[start + source_count + i] =
-                (*target)[start + i];
+            (*target)[start + source_count + i] = (*target)[start + i];
         }
         (*target)[start + suffix_count + source_count] = NULL;
 
@@ -569,14 +570,12 @@ int opal_argv_insert_element(char ***target, int location, char *source)
     }
 
     /* Alloc out new space */
-    *target = (char**) realloc(*target,
-                               sizeof(char*) * (target_count + 2));
+    *target = (char **) realloc(*target, sizeof(char *) * (target_count + 2));
 
     /* Move suffix items down to the end */
     suffix_count = target_count - location;
     for (i = suffix_count - 1; i >= 0; --i) {
-        (*target)[location + 1 + i] =
-        (*target)[location + i];
+        (*target)[location + 1 + i] = (*target)[location + i];
     }
     (*target)[location + suffix_count + 1] = NULL;
 
diff --git a/opal/util/argv.h b/opal/util/argv.h
index c119d1b0bd8..48816899c0a 100644
--- a/opal/util/argv.h
+++ b/opal/util/argv.h
@@ -33,58 +33,59 @@
 #include "opal_config.h"
 
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 
 BEGIN_C_DECLS
 
-  /**
-   * Append a string (by value) to an new or existing NULL-terminated
-   * argv array.
-   *
-   * @param argc Pointer to the length of the argv array.  Must not be
-   * NULL.
-   * @param argv Pointer to an argv array.
-   * @param str Pointer to the string to append.
-   *
-   * @retval OPAL_SUCCESS On success
-   * @retval OPAL_ERROR On failure
-   *
-   * This function adds a string to an argv array of strings by value;
-   * it is permissable to pass a string on the stack as the str
-   * argument to this function.
-   *
-   * To add the first entry to an argv array, call this function with
-   * (*argv == NULL).  This function will allocate an array of length
-   * 2; the first entry will point to a copy of the string passed in
-   * arg, the second entry will be set to NULL.
-   *
-   * If (*argv != NULL), it will be realloc'ed to be 1 (char*) larger,
-   * and the next-to-last entry will point to a copy of the string
-   * passed in arg.  The last entry will be set to NULL.
-   *
-   * Just to reinforce what was stated above: the string is copied by
-   * value into the argv array; there is no need to keep the original
-   * string (i.e., the arg parameter) after invoking this function.
-   */
-OPAL_DECLSPEC  int opal_argv_append(int *argc, char ***argv, const char *arg) __opal_attribute_nonnull__(1) __opal_attribute_nonnull__(3);
+/**
+ * Append a string (by value) to an new or existing NULL-terminated
+ * argv array.
+ *
+ * @param argc Pointer to the length of the argv array.  Must not be
+ * NULL.
+ * @param argv Pointer to an argv array.
+ * @param str Pointer to the string to append.
+ *
+ * @retval OPAL_SUCCESS On success
+ * @retval OPAL_ERROR On failure
+ *
+ * This function adds a string to an argv array of strings by value;
+ * it is permissible to pass a string on the stack as the str
+ * argument to this function.
+ *
+ * To add the first entry to an argv array, call this function with
+ * (*argv == NULL).  This function will allocate an array of length
+ * 2; the first entry will point to a copy of the string passed in
+ * arg, the second entry will be set to NULL.
+ *
+ * If (*argv != NULL), it will be realloc'ed to be 1 (char*) larger,
+ * and the next-to-last entry will point to a copy of the string
+ * passed in arg.  The last entry will be set to NULL.
+ *
+ * Just to reinforce what was stated above: the string is copied by
+ * value into the argv array; there is no need to keep the original
+ * string (i.e., the arg parameter) after invoking this function.
+ */
+OPAL_DECLSPEC int opal_argv_append(int *argc, char ***argv, const char *arg)
+    __opal_attribute_nonnull__(1) __opal_attribute_nonnull__(3);
 
-  /**
-   * Append to an argv-style array, but ignore the size of the array.
-   *
-   * @param argv Pointer to an argv array.
-   * @param str Pointer to the string to append.
-   *
-   * @retval OPAL_SUCCESS On success
-   * @retval OPAL_ERROR On failure
-   *
-   * This function is identical to the opal_argv_append() function
-   * except that it does not take a pointer to an argc (integer
-   * representing the size of the array).  This is handy for
-   * argv-style arrays that do not have integers that are actively
-   * maintaing their sizes.
-   */
-OPAL_DECLSPEC  int opal_argv_append_nosize(char ***argv, const char *arg);
+/**
+ * Append to an argv-style array, but ignore the size of the array.
+ *
+ * @param argv Pointer to an argv array.
+ * @param str Pointer to the string to append.
+ *
+ * @retval OPAL_SUCCESS On success
+ * @retval OPAL_ERROR On failure
+ *
+ * This function is identical to the opal_argv_append() function
+ * except that it does not take a pointer to an argc (integer
+ * representing the size of the array).  This is handy for
+ * argv-style arrays that do not have integers that are actively
+ * maintaining their sizes.
+ */
+OPAL_DECLSPEC int opal_argv_append_nosize(char ***argv, const char *arg);
 
 /**
  * Insert the provided arg at the beginning of the array
@@ -112,164 +113,168 @@ OPAL_DECLSPEC int opal_argv_prepend_nosize(char ***argv, const char *arg);
  * except that it only appends the provided argument if it does not already
  * exist in the provided array, or overwrites it if it is.
  */
-OPAL_DECLSPEC  int opal_argv_append_unique_nosize(char ***argv, const char *arg, bool overwrite);
+OPAL_DECLSPEC int opal_argv_append_unique_nosize(char ***argv, const char *arg, bool overwrite);
 
 /**
-   * Free a NULL-terminated argv array.
-   *
-   * @param argv Argv array to free.
-   *
-   * This function frees an argv array and all of the strings that it
-   * contains.  Since the argv parameter is passed by value, it is not
-   * set to NULL in the caller's scope upon return.
-   *
-   * It is safe to invoke this function with a NULL pointer.  It is
-   * not safe to invoke this function with a non-NULL-terminated argv
-   * array.
-   */
-OPAL_DECLSPEC  void opal_argv_free(char **argv);
+ * Free a NULL-terminated argv array.
+ *
+ * @param argv Argv array to free.
+ *
+ * This function frees an argv array and all of the strings that it
+ * contains.  Since the argv parameter is passed by value, it is not
+ * set to NULL in the caller's scope upon return.
+ *
+ * It is safe to invoke this function with a NULL pointer.  It is
+ * not safe to invoke this function with a non-NULL-terminated argv
+ * array.
+ */
+OPAL_DECLSPEC void opal_argv_free(char **argv);
 
-  /**
-   * Split a string into a NULL-terminated argv array. Do not include empty
-   * strings in result array.
-   *
-   * @param src_string Input string.
-   * @param delimiter Delimiter character.
-   *
-   * @retval argv pointer to new argv array on success
-   * @retval NULL on error
-   *
-   * All strings are inserted into the argv array by value; the
-   * newly-allocated array makes no references to the src_string
-   * argument (i.e., it can be freed after calling this function
-   * without invalidating the output argv).
-   */
-OPAL_DECLSPEC  char **opal_argv_split(const char *src_string, int delimiter) __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
+/**
+ * Split a string into a NULL-terminated argv array. Do not include empty
+ * strings in result array.
+ *
+ * @param src_string Input string.
+ * @param delimiter Delimiter character.
+ *
+ * @retval argv pointer to new argv array on success
+ * @retval NULL on error
+ *
+ * All strings are inserted into the argv array by value; the
+ * newly-allocated array makes no references to the src_string
+ * argument (i.e., it can be freed after calling this function
+ * without invalidating the output argv).
+ */
+OPAL_DECLSPEC char **opal_argv_split(const char *src_string, int delimiter)
+    __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
 
-  /**
-   * Split a string into a NULL-terminated argv array. Include empty
-   * strings in result array.
-   *
-   * @param src_string Input string.
-   * @param delimiter Delimiter character.
-   *
-   * @retval argv pointer to new argv array on success
-   * @retval NULL on error
-   *
-   * All strings are inserted into the argv array by value; the
-   * newly-allocated array makes no references to the src_string
-   * argument (i.e., it can be freed after calling this function
-   * without invalidating the output argv).
-   */
-OPAL_DECLSPEC  char **opal_argv_split_with_empty(const char *src_string, int delimiter) __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
+/**
+ * Split a string into a NULL-terminated argv array. Include empty
+ * strings in result array.
+ *
+ * @param src_string Input string.
+ * @param delimiter Delimiter character.
+ *
+ * @retval argv pointer to new argv array on success
+ * @retval NULL on error
+ *
+ * All strings are inserted into the argv array by value; the
+ * newly-allocated array makes no references to the src_string
+ * argument (i.e., it can be freed after calling this function
+ * without invalidating the output argv).
+ */
+OPAL_DECLSPEC char **opal_argv_split_with_empty(const char *src_string, int delimiter)
+    __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
 
-  /**
-   * Return the length of a NULL-terminated argv array.
-   *
-   * @param argv The input argv array.
-   *
-   * @retval 0 If NULL is passed as argv.
-   * @retval count Number of entries in the argv array.
-   *
-   * The argv array must be NULL-terminated.
-   */
-OPAL_DECLSPEC  int opal_argv_count(char **argv);
+/**
+ * Return the length of a NULL-terminated argv array.
+ *
+ * @param argv The input argv array.
+ *
+ * @retval 0 If NULL is passed as argv.
+ * @retval count Number of entries in the argv array.
+ *
+ * The argv array must be NULL-terminated.
+ */
+OPAL_DECLSPEC int opal_argv_count(char **argv);
 
-  /**
-   * Join all the elements of an argv array into a single
-   * newly-allocated string.
-   *
-   * @param argv The input argv array.
-   * @param delimiter Delimiter character placed between each argv string.
-   *
-   * @retval new_string Output string on success.
-   * @retval NULL On failure.
-   *
-   * Similar to the Perl join function, this function takes an input
-   * argv and joins them into into a single string separated by the
-   * delimiter character.
-   *
-   * It is the callers responsibility to free the returned string.
-   */
-OPAL_DECLSPEC  char *opal_argv_join(char **argv, int delimiter) __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
+/**
+ * Join all the elements of an argv array into a single
+ * newly-allocated string.
+ *
+ * @param argv The input argv array.
+ * @param delimiter Delimiter character placed between each argv string.
+ *
+ * @retval new_string Output string on success.
+ * @retval NULL On failure.
+ *
+ * Similar to the Perl join function, this function takes an input
+ * argv and joins them into into a single string separated by the
+ * delimiter character.
+ *
+ * It is the callers responsibility to free the returned string.
+ */
+OPAL_DECLSPEC char *opal_argv_join(char **argv, int delimiter)
+    __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
 
-OPAL_DECLSPEC  char *opal_argv_join_range(char **argv, size_t start, size_t end, int delimiter) __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
+OPAL_DECLSPEC char *opal_argv_join_range(char **argv, size_t start, size_t end, int delimiter)
+    __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
 
-  /**
-   * Return the number of bytes consumed by an argv array.
-   *
-   * @param argv The input argv array.
-   *
-   * Count the number of bytes consumed by a NULL-terminated argv
-   * array.  This includes the number of bytes used by each of the
-   * strings as well as the pointers used in the argv array.
-   */
-OPAL_DECLSPEC  size_t opal_argv_len(char **argv);
+/**
+ * Return the number of bytes consumed by an argv array.
+ *
+ * @param argv The input argv array.
+ *
+ * Count the number of bytes consumed by a NULL-terminated argv
+ * array.  This includes the number of bytes used by each of the
+ * strings as well as the pointers used in the argv array.
+ */
+OPAL_DECLSPEC size_t opal_argv_len(char **argv);
 
-  /**
-   * Copy a NULL-terminated argv array.
-   *
-   * @param argv The input argv array.
-   *
-   * @retval argv Copied argv array on success.
-   * @retval NULL On failure.
-   *
-   * Copy an argv array, including copying all off its strings.
-   * Specifically, the output argv will be an array of the same length
-   * as the input argv, and strcmp(argv_in[i], argv_out[i]) will be 0.
-   */
-OPAL_DECLSPEC  char **opal_argv_copy(char **argv) __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
+/**
+ * Copy a NULL-terminated argv array.
+ *
+ * @param argv The input argv array.
+ *
+ * @retval argv Copied argv array on success.
+ * @retval NULL On failure.
+ *
+ * Copy an argv array, including copying all off its strings.
+ * Specifically, the output argv will be an array of the same length
+ * as the input argv, and strcmp(argv_in[i], argv_out[i]) will be 0.
+ */
+OPAL_DECLSPEC char **
+opal_argv_copy(char **argv) __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
 
-    /**
-     * Delete one or more tokens from the middle of an argv.
-     *
-     * @param argv The argv to delete from
-     * @param start The index of the first token to delete
-     * @param num_to_delete How many tokens to delete
-     *
-     * @retval OPAL_SUCCESS Always
-     *
-     * Delete some tokens from within an existing argv.  The start
-     * parameter specifies the first token to delete, and will delete
-     * (num_to_delete-1) tokens following it.  argv will be realloc()ed
-     * to *argc - num_deleted size.
-     *
-     * If start is beyond the end of the argv array, this function is
-     * a no-op.
-     *
-     * If num_to_delete runs beyond the end of the argv array, this
-     * function will delete all tokens starting with start to the end
-     * of the array.
-     *
-     * All deleted items in the argv array will have their contents
-     * free()ed (it is assumed that the argv "owns" the memory that
-     * the pointer points to).
-     */
-OPAL_DECLSPEC  int opal_argv_delete(int *argc, char ***argv,
-                                    int start, int num_to_delete);
+/**
+ * Delete one or more tokens from the middle of an argv.
+ *
+ * @param argv The argv to delete from
+ * @param start The index of the first token to delete
+ * @param num_to_delete How many tokens to delete
+ *
+ * @retval OPAL_SUCCESS Always
+ *
+ * Delete some tokens from within an existing argv.  The start
+ * parameter specifies the first token to delete, and will delete
+ * (num_to_delete-1) tokens following it.  argv will be realloc()ed
+ * to *argc - num_deleted size.
+ *
+ * If start is beyond the end of the argv array, this function is
+ * a no-op.
+ *
+ * If num_to_delete runs beyond the end of the argv array, this
+ * function will delete all tokens starting with start to the end
+ * of the array.
+ *
+ * All deleted items in the argv array will have their contents
+ * free()ed (it is assumed that the argv "owns" the memory that
+ * the pointer points to).
+ */
+OPAL_DECLSPEC int opal_argv_delete(int *argc, char ***argv, int start, int num_to_delete);
 
-    /**
-     * Insert one argv array into the middle of another
-     *
-     * @param target The argv to insert tokens into
-     * @param start Index where the first token will be placed in target
-     * @param source The argv to copy tokens from
-     *
-     * @retval OPAL_SUCCESS upon success
-     * @retval OPAL_BAD_PARAM if any parameters are non-sensical
-     *
-     * This function takes one arg and inserts it in the middle of
-     * another.  The first token in source will be inserted at index
-     * start in the target argv; all other tokens will follow it.
-     * Similar to opal_argv_append(), the target may be realloc()'ed
-     * to accomodate the new storage requirements.
-     *
-     * The source array is left unaffected -- its contents are copied
-     * by value over to the target array (i.e., the strings that
-     * source points to are strdup'ed into the new locations in
-     * target).
-     */
-OPAL_DECLSPEC  int opal_argv_insert(char ***target, int start, char **source);
+/**
+ * Insert one argv array into the middle of another
+ *
+ * @param target The argv to insert tokens into
+ * @param start Index where the first token will be placed in target
+ * @param source The argv to copy tokens from
+ *
+ * @retval OPAL_SUCCESS upon success
+ * @retval OPAL_BAD_PARAM if any parameters are non-sensical
+ *
+ * This function takes one arg and inserts it in the middle of
+ * another.  The first token in source will be inserted at index
+ * start in the target argv; all other tokens will follow it.
+ * Similar to opal_argv_append(), the target may be realloc()'ed
+ * to accommodate the new storage requirements.
+ *
+ * The source array is left unaffected -- its contents are copied
+ * by value over to the target array (i.e., the strings that
+ * source points to are strdup'ed into the new locations in
+ * target).
+ */
+OPAL_DECLSPEC int opal_argv_insert(char ***target, int start, char **source);
 
 /**
  * Insert one argv element in front of a specific position in an array
@@ -285,14 +290,14 @@ OPAL_DECLSPEC  int opal_argv_insert(char ***target, int start, char **source);
  * another.  The token will be inserted at the specified index
  * in the target argv; all other tokens will be shifted down.
  * Similar to opal_argv_append(), the target may be realloc()'ed
- * to accomodate the new storage requirements.
+ * to accommodate the new storage requirements.
  *
  * The source token is left unaffected -- its contents are copied
  * by value over to the target array (i.e., the string that
  * source points to is strdup'ed into the new location in
  * target).
  */
-OPAL_DECLSPEC  int opal_argv_insert_element(char ***target, int location, char *source);
+OPAL_DECLSPEC int opal_argv_insert_element(char ***target, int location, char *source);
 
 END_C_DECLS
 
diff --git a/opal/util/basename.c b/opal/util/basename.c
index ab63e48e427..f5121156498 100644
--- a/opal/util/basename.c
+++ b/opal/util/basename.c
@@ -10,8 +10,8 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2009-2014 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2014-2015 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2014-2024 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2014      Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
@@ -25,8 +25,8 @@
 #include <stdlib.h>
 #include <string.h>
 #ifdef HAVE_LIBGEN_H
-#include <libgen.h>
-#endif  /* HAVE_LIBGEN_H */
+#    include <libgen.h>
+#endif /* HAVE_LIBGEN_H */
 
 #include "opal/util/basename.h"
 #include "opal/util/os_path.h"
@@ -38,22 +38,24 @@
  * of characters.
  * If the last character on the string is a path separator, it will be skipped.
  */
-static inline char* opal_find_last_path_separator( const char* filename, size_t n )
+static inline char *opal_find_last_path_separator(const char *filename, size_t n)
 {
-    char* p = (char*)filename + n;
+    char *p = (char *) filename + n;
 
     /* First skip the latest separators */
-    for ( ; p >= filename; p-- ) {
-        if( *p != OPAL_PATH_SEP[0] )
+    for (; p >= filename; p--) {
+        if (*p != OPAL_PATH_SEP[0]) {
             break;
+        }
     }
 
-    for ( ; p >= filename; p-- ) {
-        if( *p == OPAL_PATH_SEP[0] )
+    for (; p >= filename; p--) {
+        if (*p == OPAL_PATH_SEP[0]) {
             return p;
+        }
     }
 
-    return NULL;  /* nothing found inside the filename */
+    return NULL; /* nothing found inside the filename */
 }
 
 char *opal_basename(const char *filename)
@@ -75,20 +77,22 @@ char *opal_basename(const char *filename)
 
     /* Remove trailing sep's (note that we already know that strlen > 0) */
     tmp = strdup(filename);
-    for (i = strlen(tmp) - 1; i > 0; --i) {
-        if (sep == tmp[i]) {
-            tmp[i] = '\0';
-        } else {
-            break;
+    if (1 < strlen(tmp)) {
+        for (i = strlen(tmp) - 1; i > 0; --i) {
+            if (sep == tmp[i]) {
+                tmp[i] = '\0';
+            } else {
+                break;
+            }
+        }
+        if (0 == i) {
+            tmp[0] = sep;
+            return tmp;
         }
-    }
-    if (0 == i) {
-        tmp[0] = sep;
-        return tmp;
     }
 
     /* Look for the final sep */
-    ret = opal_find_last_path_separator( tmp, strlen(tmp) );
+    ret = opal_find_last_path_separator(tmp, strlen(tmp));
     if (NULL == ret) {
         return tmp;
     }
@@ -97,10 +101,10 @@ char *opal_basename(const char *filename)
     return ret;
 }
 
-char* opal_dirname(const char* filename)
+char *opal_dirname(const char *filename)
 {
 #if defined(HAVE_DIRNAME) || OPAL_HAVE_DIRNAME
-    char* safe_tmp = strdup(filename), *result;
+    char *safe_tmp = strdup(filename), *result;
     if (NULL == safe_tmp) {
         return NULL;
     }
@@ -108,21 +112,21 @@ char* opal_dirname(const char* filename)
     free(safe_tmp);
     return result;
 #else
-    const char* p = opal_find_last_path_separator(filename, strlen(filename));
+    const char *p = opal_find_last_path_separator(filename, strlen(filename));
     /* NOTE: p will be NULL if no path separator was in the filename - i.e.,
      * if filename is just a local file */
 
-    for( ; NULL != p && p != filename; p-- ) {
-        if( (*p == '\\') || (*p == '/') ) {
+    for (; NULL != p && p != filename; p--) {
+        if ((*p == '\\') || (*p == '/')) {
             /* If there are several delimiters remove them all */
-            for( --p; p != filename; p-- ) {
-                if( (*p != '\\') && (*p != '/') ) {
+            for (--p; p != filename; p--) {
+                if ((*p != '\\') && (*p != '/')) {
                     p++;
                     break;
                 }
             }
-            if( p != filename ) {
-                char* ret = (char*)malloc( p - filename + 1 );
+            if (p != filename) {
+                char *ret = (char *) malloc(p - filename + 1);
                 if (NULL == ret) {
                     return NULL;
                 }
@@ -130,9 +134,9 @@ char* opal_dirname(const char* filename)
                 ret[p - filename] = '\0';
                 return opal_make_filename_os_friendly(ret);
             }
-            break;  /* return the duplicate of "." */
+            break; /* return the duplicate of "." */
         }
     }
     return strdup(".");
-#endif  /* defined(HAVE_DIRNAME) || OPAL_HAVE_DIRNAME */
+#endif /* defined(HAVE_DIRNAME) || OPAL_HAVE_DIRNAME */
 }
diff --git a/opal/util/basename.h b/opal/util/basename.h
index 5d0cf04b6e5..b1cba4337f8 100644
--- a/opal/util/basename.h
+++ b/opal/util/basename.h
@@ -19,7 +19,7 @@
 /**
  * @file
  *
- * Returns an OS-independant basename() of a given filename.
+ * Returns an OS-independent basename() of a given filename.
  */
 
 #ifndef OPAL_BASENAME_H
@@ -66,7 +66,8 @@ BEGIN_C_DECLS
  *
  * The caller is responsible for freeing the returned string.
  */
-OPAL_DECLSPEC char *opal_basename(const char* filename) __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
+OPAL_DECLSPEC char *
+opal_basename(const char *filename) __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
 
 /**
  * Return the dirname of a filename.
@@ -105,7 +106,8 @@ OPAL_DECLSPEC char *opal_basename(const char* filename) __opal_attribute_malloc_
  *
  * The caller is responsible for freeing the returned string.
  */
-OPAL_DECLSPEC char *opal_dirname(const char* filename) __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
+OPAL_DECLSPEC char *
+opal_dirname(const char *filename) __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
 
 END_C_DECLS
 
diff --git a/opal/util/bipartite_graph.c b/opal/util/bipartite_graph.c
index c372d90f1ab..0ec510619ca 100644
--- a/opal/util/bipartite_graph.c
+++ b/opal/util/bipartite_graph.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2014-2022 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2017-2019 Amazon.com, Inc. or its affiliates.  All Rights
  *                         reserved.
  * $COPYRIGHT$
@@ -11,48 +11,45 @@
 
 #include "opal_config.h"
 
-#include <stdlib.h>
 #include <stddef.h>
+#include <stdlib.h>
 
-#include "opal_stdint.h"
-#include "opal/constants.h"
 #include "opal/class/opal_list.h"
 #include "opal/class/opal_pointer_array.h"
-#include "opal/util/output.h"
+#include "opal/constants.h"
 #include "opal/util/error.h"
+#include "opal/util/misc.h"
+#include "opal/util/output.h"
+#include "opal_stdint.h"
 
 #include "opal/util/bipartite_graph.h"
 #include "opal/util/bipartite_graph_internal.h"
 
-#ifndef container_of
-#define container_of(ptr, type, member) ( \
-        (type *)( ((char *)(ptr)) - offsetof(type,member) ))
-#endif
-
 #define GRAPH_DEBUG 0
 #if GRAPH_DEBUG
-#  define GRAPH_DEBUG_OUT(args) printf(args)
+#    define GRAPH_DEBUG_OUT(args) printf(args)
 #else
-#  define GRAPH_DEBUG_OUT(args) do {} while(0)
+#    define GRAPH_DEBUG_OUT(args) \
+        do {                      \
+        } while (0)
 #endif
 
 #define MAX_COST INT64_MAX
 
 #ifndef MAX
-#  define MAX(a,b) ((a) > (b) ? (a) : (b))
+#    define MAX(a, b) ((a) > (b) ? (a) : (b))
 #endif
 
 #ifndef MIN
-#  define MIN(a,b) ((a) < (b) ? (a) : (b))
+#    define MIN(a, b) ((a) < (b) ? (a) : (b))
 #endif
 
-#define f(i,j) flow[n*i + j]
+#define f(i, j) flow[n * i + j]
 
 /* ensure that (a+b<=max) */
 static inline void check_add64_overflow(int64_t a, int64_t b)
 {
-    assert(!((b > 0) && (a > (INT64_MAX - b))) &&
-           !((b < 0) && (a < (INT64_MIN - b))));
+    assert(!((b > 0) && (a > (INT64_MAX - b))) && !((b < 0) && (a < (INT64_MIN - b))));
 }
 
 static void edge_constructor(opal_bp_graph_edge_t *e)
@@ -68,11 +65,9 @@ static void edge_destructor(opal_bp_graph_edge_t *e)
 }
 
 OBJ_CLASS_DECLARATION(opal_bp_graph_edge_t);
-OBJ_CLASS_INSTANCE(opal_bp_graph_edge_t, opal_object_t,
-                   edge_constructor, edge_destructor);
+OBJ_CLASS_INSTANCE(opal_bp_graph_edge_t, opal_object_t, edge_constructor, edge_destructor);
 
-static void dump_vec(const char *name, int *vec, int n)
-    __opal_attribute_unused__;
+static void dump_vec(const char *name, int *vec, int n) __opal_attribute_unused__;
 
 static void dump_vec(const char *name, int *vec, int n)
 {
@@ -84,8 +79,7 @@ static void dump_vec(const char *name, int *vec, int n)
     fprintf(stderr, "}\n");
 }
 
-static void dump_vec64(const char *name, int64_t *vec, int n)
-    __opal_attribute_unused__;
+static void dump_vec64(const char *name, int64_t *vec, int n) __opal_attribute_unused__;
 
 static void dump_vec64(const char *name, int64_t *vec, int n)
 {
@@ -97,9 +91,7 @@ static void dump_vec64(const char *name, int64_t *vec, int n)
     fprintf(stderr, "}\n");
 }
 
-
-static void dump_flow(int *flow, int n)
-    __opal_attribute_unused__;
+static void dump_flow(int *flow, int n) __opal_attribute_unused__;
 
 static void dump_flow(int *flow, int n)
 {
@@ -109,14 +101,13 @@ static void dump_flow(int *flow, int n)
     for (u = 0; u < n; ++u) {
         fprintf(stderr, "u=%d| ", u);
         for (v = 0; v < n; ++v) {
-            fprintf(stderr, "%2d,", f(u,v));
+            fprintf(stderr, "%2d,", f(u, v));
         }
         fprintf(stderr, "\n");
     }
     fprintf(stderr, "}\n");
 }
 
-
 static int get_capacity(opal_bp_graph_t *g, int source, int target)
 {
     opal_bp_graph_edge_t *e;
@@ -124,7 +115,8 @@ static int get_capacity(opal_bp_graph_t *g, int source, int target)
     CHECK_VERTEX_RANGE(g, source);
     CHECK_VERTEX_RANGE(g, target);
 
-    FOREACH_OUT_EDGE(g, source, e) {
+    FOREACH_OUT_EDGE(g, source, e)
+    {
         assert(e->source == source);
         if (e->target == target) {
             return e->capacity;
@@ -134,15 +126,15 @@ static int get_capacity(opal_bp_graph_t *g, int source, int target)
     return 0;
 }
 
-static int
-set_capacity(opal_bp_graph_t *g, int source, int target, int cap)
+static int set_capacity(opal_bp_graph_t *g, int source, int target, int cap)
 {
     opal_bp_graph_edge_t *e;
 
     CHECK_VERTEX_RANGE(g, source);
     CHECK_VERTEX_RANGE(g, target);
 
-    FOREACH_OUT_EDGE(g, source, e) {
+    FOREACH_OUT_EDGE(g, source, e)
+    {
         assert(e->source == source);
         if (e->target == target) {
             e->capacity = cap;
@@ -153,8 +145,7 @@ set_capacity(opal_bp_graph_t *g, int source, int target, int cap)
     return OPAL_ERR_NOT_FOUND;
 }
 
-static void free_vertex(opal_bp_graph_t *g,
-                        opal_bp_graph_vertex_t *v)
+static void free_vertex(opal_bp_graph_t *g, opal_bp_graph_vertex_t *v)
 {
     if (NULL != v) {
         if (NULL != g->v_data_cleanup_fn && NULL != v->v_data) {
@@ -165,8 +156,7 @@ static void free_vertex(opal_bp_graph_t *g,
 }
 
 int opal_bp_graph_create(opal_bp_graph_cleanup_fn_t v_data_cleanup_fn,
-			 opal_bp_graph_cleanup_fn_t e_data_cleanup_fn,
-			 opal_bp_graph_t **g_out)
+                         opal_bp_graph_cleanup_fn_t e_data_cleanup_fn, opal_bp_graph_t **g_out)
 {
     int err;
     opal_bp_graph_t *g = NULL;
@@ -199,7 +189,7 @@ int opal_bp_graph_create(opal_bp_graph_cleanup_fn_t v_data_cleanup_fn,
     *g_out = g;
     return OPAL_SUCCESS;
 
- out_free_g:
+out_free_g:
     free(g);
     return err;
 }
@@ -213,8 +203,8 @@ int opal_bp_graph_free(opal_bp_graph_t *g)
     /* remove all edges from all out_edges lists */
     for (i = 0; i < NUM_VERTICES(g); ++i) {
         v = V_ID_TO_PTR(g, i);
-        LIST_FOREACH_SAFE_CONTAINED(e, next, &v->out_edges,
-                                    opal_bp_graph_edge_t, outbound_li) {
+        LIST_FOREACH_SAFE_CONTAINED(e, next, &v->out_edges, opal_bp_graph_edge_t, outbound_li)
+        {
             opal_list_remove_item(&v->out_edges, &e->outbound_li);
             OBJ_RELEASE(e);
         }
@@ -222,8 +212,8 @@ int opal_bp_graph_free(opal_bp_graph_t *g)
     /* now remove from all in_edges lists and free the edge */
     for (i = 0; i < NUM_VERTICES(g); ++i) {
         v = V_ID_TO_PTR(g, i);
-        LIST_FOREACH_SAFE_CONTAINED(e, next, &v->in_edges,
-                                    opal_bp_graph_edge_t, inbound_li) {
+        LIST_FOREACH_SAFE_CONTAINED(e, next, &v->in_edges, opal_bp_graph_edge_t, inbound_li)
+        {
             opal_list_remove_item(&v->in_edges, &e->inbound_li);
 
             if (NULL != g->e_data_cleanup_fn && NULL != e->e_data) {
@@ -243,9 +233,8 @@ int opal_bp_graph_free(opal_bp_graph_t *g)
     return OPAL_SUCCESS;
 }
 
-int opal_bp_graph_clone(const opal_bp_graph_t *g,
-			bool copy_user_data,
-			opal_bp_graph_t **g_clone_out)
+int opal_bp_graph_clone(const opal_bp_graph_t *g, bool copy_user_data,
+                        opal_bp_graph_t **g_clone_out)
 {
     int err;
     int i;
@@ -259,8 +248,8 @@ int opal_bp_graph_clone(const opal_bp_graph_t *g,
     *g_clone_out = NULL;
 
     if (copy_user_data) {
-	opal_output(0, "[%s:%d:%s] user data copy requested but not yet supported", 
-		    __FILE__, __LINE__, __func__);
+        opal_output(0, "[%s:%d:%s] user data copy requested but not yet supported", __FILE__,
+                    __LINE__, __func__);
         abort();
         return OPAL_ERR_FATAL;
     }
@@ -284,10 +273,10 @@ int opal_bp_graph_clone(const opal_bp_graph_t *g,
     /* now reconstruct all the edges (iterate by source vertex only to avoid
      * double-adding) */
     for (i = 0; i < NUM_VERTICES(g); ++i) {
-        FOREACH_OUT_EDGE(g, i, e) {
+        FOREACH_OUT_EDGE(g, i, e)
+        {
             assert(i == e->source);
-            err = opal_bp_graph_add_edge(gx, e->source, e->target,
-					    e->cost, e->capacity, NULL);
+            err = opal_bp_graph_add_edge(gx, e->source, e->target, e->cost, e->capacity, NULL);
             if (OPAL_SUCCESS != err) {
                 goto out_free_gx;
             }
@@ -297,15 +286,14 @@ int opal_bp_graph_clone(const opal_bp_graph_t *g,
     *g_clone_out = gx;
     return OPAL_SUCCESS;
 
- out_free_gx:
+out_free_gx:
     /* we don't reach in and manipulate gx's state directly, so it should be
      * safe to use the standard free function */
     opal_bp_graph_free(gx);
     return err;
 }
 
-int opal_bp_graph_indegree(const opal_bp_graph_t *g,
-			   int vertex)
+int opal_bp_graph_indegree(const opal_bp_graph_t *g, int vertex)
 {
     opal_bp_graph_vertex_t *v;
 
@@ -313,8 +301,7 @@ int opal_bp_graph_indegree(const opal_bp_graph_t *g,
     return opal_list_get_size(&v->in_edges);
 }
 
-int opal_bp_graph_outdegree(const opal_bp_graph_t *g,
-			       int vertex)
+int opal_bp_graph_outdegree(const opal_bp_graph_t *g, int vertex)
 {
     opal_bp_graph_vertex_t *v;
 
@@ -322,12 +309,8 @@ int opal_bp_graph_outdegree(const opal_bp_graph_t *g,
     return opal_list_get_size(&v->out_edges);
 }
 
-int opal_bp_graph_add_edge(opal_bp_graph_t *g,
-			   int from,
-			   int to,
-			   int64_t cost,
-			   int capacity,
-			   void *e_data)
+int opal_bp_graph_add_edge(opal_bp_graph_t *g, int from, int to, int64_t cost, int capacity,
+                           void *e_data)
 {
     opal_bp_graph_edge_t *e;
     opal_bp_graph_vertex_t *v_from, *v_to;
@@ -346,7 +329,8 @@ int opal_bp_graph_add_edge(opal_bp_graph_t *g,
          * handled appropriately */
         return OPAL_ERR_BAD_PARAM;
     }
-    FOREACH_OUT_EDGE(g, from, e) {
+    FOREACH_OUT_EDGE(g, from, e)
+    {
         assert(e->source == from);
         if (e->target == to) {
             return OPAL_EXISTS;
@@ -360,11 +344,11 @@ int opal_bp_graph_add_edge(opal_bp_graph_t *g,
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
-    e->source   = from;
-    e->target   = to;
-    e->cost     = cost;
+    e->source = from;
+    e->target = to;
+    e->cost = cost;
     e->capacity = capacity;
-    e->e_data   = e_data;
+    e->e_data = e_data;
 
     v_from = V_ID_TO_PTR(g, from);
     opal_list_append(&v_from->out_edges, &e->outbound_li);
@@ -376,9 +360,7 @@ int opal_bp_graph_add_edge(opal_bp_graph_t *g,
     return OPAL_SUCCESS;
 }
 
-int opal_bp_graph_add_vertex(opal_bp_graph_t *g,
-			     void *v_data,
-			     int *index_out)
+int opal_bp_graph_add_vertex(opal_bp_graph_t *g, void *v_data, int *index_out)
 {
     opal_bp_graph_vertex_t *v;
 
@@ -411,11 +393,9 @@ int opal_bp_graph_add_vertex(opal_bp_graph_t *g,
     return OPAL_SUCCESS;
 }
 
-int opal_bp_graph_get_vertex_data(opal_bp_graph_t *g,
-                                  int v_index,
-                                  void** v_data_out)
+int opal_bp_graph_get_vertex_data(opal_bp_graph_t *g, int v_index, void **v_data_out)
 {
-    opal_bp_graph_vertex_t* v;
+    opal_bp_graph_vertex_t *v;
 
     v = V_ID_TO_PTR(g, v_index);
     if (NULL == v) {
@@ -459,7 +439,7 @@ static void shrink_flow_matrix(int *flow, int old_n, int new_n)
 
     for (u = 0; u < new_n; ++u) {
         for (v = 0; v < new_n; ++v) {
-            flow[new_n*u + v] = flow[old_n*u + v];
+            flow[new_n * u + v] = flow[old_n * u + v];
         }
     }
 }
@@ -468,17 +448,14 @@ static void shrink_flow_matrix(int *flow, int old_n, int new_n)
  * Compute the so-called "bottleneck" capacity value for a path "pred" through
  * graph "gx".
  */
-static int
-bottleneck_path(
-		opal_bp_graph_t *gx,
-		int n,
-		int *pred)
+static int bottleneck_path(opal_bp_graph_t *gx, int n, int *pred)
 {
     int u, v;
     int min;
 
     min = INT_MAX;
-    FOREACH_UV_ON_PATH(pred, gx->source_idx, gx->sink_idx, u, v) {
+    FOREACH_UV_ON_PATH(pred, gx->source_idx, gx->sink_idx, u, v)
+    {
         int cap_f_uv = get_capacity(gx, u, v);
         min = MIN(min, cap_f_uv);
     }
@@ -486,11 +463,10 @@ bottleneck_path(
     return min;
 }
 
-
 /**
  * This routine implements the Bellman-Ford shortest paths algorithm, slightly
  * specialized for our forumlation of flow networks:
- * http://en.wikipedia.org/wiki/Bellman%E2%80%93Ford_algorithm
+ * https://en.wikipedia.org/wiki/Bellman%E2%80%93Ford_algorithm
  *
  * Specifically, it attempts to find the shortest path from "source" to
  * "target".  It returns true if such a path was found, false otherwise.  Any
@@ -499,10 +475,7 @@ bottleneck_path(
  *
  * The contents of "pred" are only valid if this routine returns true.
  */
-bool opal_bp_graph_bellman_ford(opal_bp_graph_t *gx,
-				int source,
-				int target,
-				int *pred)
+bool opal_bp_graph_bellman_ford(opal_bp_graph_t *gx, int source, int target, int *pred)
 {
     int64_t *dist;
     int i;
@@ -549,13 +522,14 @@ bool opal_bp_graph_bellman_ford(opal_bp_graph_t *gx,
         for (u = 0; u < NUM_VERTICES(gx); ++u) {
             opal_bp_graph_edge_t *e_ptr;
 
-            FOREACH_OUT_EDGE(gx, u, e_ptr) {
+            FOREACH_OUT_EDGE(gx, u, e_ptr)
+            {
                 v = e_ptr->target;
 
                 /* make sure to only construct paths from edges that actually have
                  * non-zero capacity */
-                if (e_ptr->capacity > 0 &&
-                    dist[u] != MAX_COST) { /* avoid signed overflow for "infinity" */
+                if (e_ptr->capacity > 0
+                    && dist[u] != MAX_COST) { /* avoid signed overflow for "infinity" */
                     check_add64_overflow(dist[u], e_ptr->cost);
                     if ((dist[u] + e_ptr->cost) < dist[v]) {
                         dist[v] = dist[u] + e_ptr->cost;
@@ -575,15 +549,15 @@ bool opal_bp_graph_bellman_ford(opal_bp_graph_t *gx,
 
     /* check for negative-cost cycles */
     for (u = 0; u < NUM_VERTICES(gx); ++u) {
-        opal_bp_graph_edge_t * e_ptr;
+        opal_bp_graph_edge_t *e_ptr;
 
-        FOREACH_OUT_EDGE(gx, u, e_ptr) {
+        FOREACH_OUT_EDGE(gx, u, e_ptr)
+        {
             v = e_ptr->target;
-            if (e_ptr->capacity > 0 &&
-                dist[u] != MAX_COST && /* avoid signed overflow */
+            if (e_ptr->capacity > 0 && dist[u] != MAX_COST && /* avoid signed overflow */
                 (dist[u] + e_ptr->cost) < dist[v]) {
-                opal_output(0, "[%s:%d:%s] negative-weight cycle detected",
-			    __FILE__, __LINE__, __func__);
+                opal_output(0, "[%s:%d:%s] negative-weight cycle detected", __FILE__, __LINE__,
+                            __func__);
                 abort();
                 goto out;
             }
@@ -594,7 +568,7 @@ bool opal_bp_graph_bellman_ford(opal_bp_graph_t *gx,
         found_target = true;
     }
 
- out:
+out:
 #if GRAPH_DEBUG
     dump_vec("pred", pred, NUM_VERTICES(gx));
 #endif
@@ -655,29 +629,25 @@ int opal_bp_graph_bipartite_to_flow(opal_bp_graph_t *g)
         int outbound = opal_bp_graph_outdegree(g, u);
 
         if (inbound > 0 && outbound > 0) {
-            opal_output(0, "[%s:%d:%s] graph is not (unidirectionally) bipartite",
-			__FILE__, __LINE__, __func__);
+            opal_output(0, "[%s:%d:%s] graph is not (unidirectionally) bipartite", __FILE__,
+                        __LINE__, __func__);
             abort();
-        }
-        else if (inbound > 0) {
+        } else if (inbound > 0) {
             /* "right" side of the graph, create edges to the sink */
             ++num_right;
-            err = opal_bp_graph_add_edge(g, u, g->sink_idx,
-					    0, /* no cost */
-					    /*capacity=*/1,
-					    /*e_data=*/NULL);
+            err = opal_bp_graph_add_edge(g, u, g->sink_idx, 0, /* no cost */
+                                         /*capacity=*/1,
+                                         /*e_data=*/NULL);
             if (OPAL_SUCCESS != err) {
                 GRAPH_DEBUG_OUT(("add_edge failed"));
                 return err;
             }
-        }
-        else if (outbound > 0) {
+        } else if (outbound > 0) {
             /* "left" side of the graph, create edges to the source */
             ++num_left;
-            err = opal_bp_graph_add_edge(g, g->source_idx, u,
-					    0, /* no cost */
-					    /*capacity=*/1,
-					    /*e_data=*/NULL);
+            err = opal_bp_graph_add_edge(g, g->source_idx, u, 0, /* no cost */
+                                         /*capacity=*/1,
+                                         /*e_data=*/NULL);
             if (OPAL_SUCCESS != err) {
                 GRAPH_DEBUG_OUT(("add_edge failed"));
                 return err;
@@ -697,19 +667,19 @@ int opal_bp_graph_bipartite_to_flow(opal_bp_graph_t *g)
      * exist in the original graph.
      */
     order = opal_bp_graph_order(g); /* need residuals for newly created
-					  source/sink edges too */
+                                          source/sink edges too */
     for (u = 0; u < order; ++u) {
-        opal_bp_graph_edge_t * e_ptr;
-        FOREACH_OUT_EDGE(g, u, e_ptr) {
+        opal_bp_graph_edge_t *e_ptr;
+        FOREACH_OUT_EDGE(g, u, e_ptr)
+        {
             v = e_ptr->target;
 
             /* (u,v) exists, add (v,u) if not already present.  Cost is
              * negative for these edges because "giving back" flow pays us
              * back any cost already incurred. */
-            err = opal_bp_graph_add_edge(g, v, u,
-					    -e_ptr->cost,
-					    /*capacity=*/0,
-					    /*e_data=*/NULL);
+            err = opal_bp_graph_add_edge(g, v, u, -e_ptr->cost,
+                                         /*capacity=*/0,
+                                         /*e_data=*/NULL);
             if (OPAL_SUCCESS != err && OPAL_EXISTS != err) {
                 return err;
             }
@@ -760,8 +730,7 @@ int opal_bp_graph_bipartite_to_flow(opal_bp_graph_t *g)
  * the faster running time will be worth the additional implementation
  * complexity.
  */
-static int min_cost_flow_ssp(opal_bp_graph_t *gx,
-                             int **flow_out)
+static int min_cost_flow_ssp(opal_bp_graph_t *gx, int **flow_out)
 {
     int err = OPAL_SUCCESS;
     int n;
@@ -779,7 +748,7 @@ static int min_cost_flow_ssp(opal_bp_graph_t *gx,
 
     n = opal_bp_graph_order(gx);
 
-    pred = malloc(n*sizeof(*pred));
+    pred = malloc(n * sizeof(*pred));
     if (NULL == pred) {
         OPAL_ERROR_LOG(OPAL_ERR_OUT_OF_RESOURCE);
         err = OPAL_ERR_OUT_OF_RESOURCE;
@@ -787,7 +756,7 @@ static int min_cost_flow_ssp(opal_bp_graph_t *gx,
     }
 
     /* "flow" is a 2d matrix of current flow values, all initialized to zero */
-    flow = calloc(n*n, sizeof(*flow));
+    flow = calloc(n * n, sizeof(*flow));
     if (NULL == flow) {
         OPAL_ERROR_LOG(OPAL_ERR_OUT_OF_RESOURCE);
         err = OPAL_ERR_OUT_OF_RESOURCE;
@@ -808,13 +777,14 @@ static int min_cost_flow_ssp(opal_bp_graph_t *gx,
         cap_f_path = bottleneck_path(gx, n, pred);
 
         /* augment current flow along P */
-        FOREACH_UV_ON_PATH(pred, gx->source_idx, gx->sink_idx, u, v) {
+        FOREACH_UV_ON_PATH(pred, gx->source_idx, gx->sink_idx, u, v)
+        {
             assert(u == pred[v]);
 
-            f(u,v) = f(u,v) + cap_f_path; /* "forward" edge */
-            f(v,u) = f(v,u) - cap_f_path; /* residual network edge */
+            f(u, v) = f(u, v) + cap_f_path; /* "forward" edge */
+            f(v, u) = f(v, u) - cap_f_path; /* residual network edge */
 
-            assert(f(u,v) == -f(v,u)); /* skew symmetry invariant */
+            assert(f(u, v) == -f(v, u)); /* skew symmetry invariant */
 
             /* update Gx as we go along: decrease capacity by this new
              * augmenting flow */
@@ -822,8 +792,8 @@ static int min_cost_flow_ssp(opal_bp_graph_t *gx,
             assert(c >= 0);
             err = set_capacity(gx, u, v, c);
             if (OPAL_SUCCESS != err) {
-                opal_output(0, "[%s:%d:%s] unable to set capacity, missing edge?",
-			    __FILE__, __LINE__, __func__);
+                opal_output(0, "[%s:%d:%s] unable to set capacity, missing edge?", __FILE__,
+                            __LINE__, __func__);
                 abort();
             }
 
@@ -831,27 +801,26 @@ static int min_cost_flow_ssp(opal_bp_graph_t *gx,
             assert(c >= 0);
             err = set_capacity(gx, v, u, c);
             if (OPAL_SUCCESS != err) {
-                opal_output(0, "[%s:%d:%s] unable to set capacity, missing edge?",
-			    __FILE__, __LINE__, __func__);
+                opal_output(0, "[%s:%d:%s] unable to set capacity, missing edge?", __FILE__,
+                            __LINE__, __func__);
                 abort();
             }
         }
     }
 
- out:
+out:
     *flow_out = flow;
     free(pred);
     return err;
 
- out_error:
+out_error:
     free(*flow_out);
     GRAPH_DEBUG_OUT(("returning error %d", err));
     goto out;
 }
 
-int opal_bp_graph_solve_bipartite_assignment(const opal_bp_graph_t *g,
-					     int *num_match_edges_out,
-					     int **match_edges_out)
+int opal_bp_graph_solve_bipartite_assignment(const opal_bp_graph_t *g, int *num_match_edges_out,
+                                             int **match_edges_out)
 {
     int err;
     int i;
@@ -918,7 +887,7 @@ int opal_bp_graph_solve_bipartite_assignment(const opal_bp_graph_t *g,
 
     for (u = 0; u < n; ++u) {
         for (v = 0; v < n; ++v) {
-            if (f(u,v) > 0) {
+            if (f(u, v) > 0) {
                 ++(*num_match_edges_out);
             }
         }
@@ -941,14 +910,14 @@ int opal_bp_graph_solve_bipartite_assignment(const opal_bp_graph_t *g,
     for (u = 0; u < n; ++u) {
         for (v = 0; v < n; ++v) {
             /* flow exists on this edge so include this edge in the matching */
-            if (f(u,v) > 0) {
+            if (f(u, v) > 0) {
                 (*match_edges_out)[i++] = u;
                 (*match_edges_out)[i++] = v;
             }
         }
     }
 
- out:
+out:
     free(flow);
     opal_bp_graph_free(gx);
     return err;
diff --git a/opal/util/bipartite_graph.h b/opal/util/bipartite_graph.h
index 1df1b937c57..2249384cb07 100644
--- a/opal/util/bipartite_graph.h
+++ b/opal/util/bipartite_graph.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2014-2022 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2017-2019 Amazon.com, Inc. or its affiliates.  All Rights
  *                         reserved.
  * $COPYRIGHT$
@@ -45,8 +45,7 @@ typedef void (*opal_bp_graph_cleanup_fn_t)(void *user_data);
  * @returns OPAL_SUCCESS or an OMPI error code
  */
 int opal_bp_graph_create(opal_bp_graph_cleanup_fn_t v_data_cleanup_fn,
-			 opal_bp_graph_cleanup_fn_t e_data_cleanup_fn,
-			 opal_bp_graph_t **g_out);
+                         opal_bp_graph_cleanup_fn_t e_data_cleanup_fn, opal_bp_graph_t **g_out);
 
 /**
  * free the given graph
@@ -70,9 +69,8 @@ int opal_bp_graph_free(opal_bp_graph_t *g);
  * @param[in] g_clone_out     the resulting cloned graph
  * @returns OPAL_SUCCESS or an OMPI error code
  */
-int opal_bp_graph_clone(const opal_bp_graph_t *g,
-			bool copy_user_data,
-			opal_bp_graph_t **g_clone_out);
+int opal_bp_graph_clone(const opal_bp_graph_t *g, bool copy_user_data,
+                        opal_bp_graph_t **g_clone_out);
 
 /**
  * return the number of edges for which this vertex is a destination
@@ -81,8 +79,7 @@ int opal_bp_graph_clone(const opal_bp_graph_t *g,
  * @param[in] vertex  the vertex id to query
  * @returns the number of edges for which this vertex is a destination
  */
-int opal_bp_graph_indegree(const opal_bp_graph_t *g,
-			   int vertex);
+int opal_bp_graph_indegree(const opal_bp_graph_t *g, int vertex);
 
 /**
  * return the number of edges for which this vertex is a source
@@ -91,8 +88,7 @@ int opal_bp_graph_indegree(const opal_bp_graph_t *g,
  * @param[in] vertex  the vertex id to query
  * @returns the number of edges for which this vertex is a source
  */
-int opal_bp_graph_outdegree(const opal_bp_graph_t *g,
-			    int vertex);
+int opal_bp_graph_outdegree(const opal_bp_graph_t *g, int vertex);
 
 /**
  * add an edge to the given graph
@@ -106,12 +102,8 @@ int opal_bp_graph_outdegree(const opal_bp_graph_t *g,
  *
  * @returns OPAL_SUCCESS or an OMPI error code
  */
-int opal_bp_graph_add_edge(opal_bp_graph_t *g,
-			   int from,
-			   int to,
-			   int64_t cost,
-			   int capacity,
-			   void *e_data);
+int opal_bp_graph_add_edge(opal_bp_graph_t *g, int from, int to, int64_t cost, int capacity,
+                           void *e_data);
 
 /**
  * add a vertex to the given graph
@@ -122,9 +114,7 @@ int opal_bp_graph_add_edge(opal_bp_graph_t *g,
  *
  * @returns OPAL_SUCCESS or an OMPI error code
  */
-int opal_bp_graph_add_vertex(opal_bp_graph_t *g,
-			     void *v_data,
-			     int *index_out);
+int opal_bp_graph_add_vertex(opal_bp_graph_t *g, void *v_data, int *index_out);
 
 /**
  * Get a pointer to the vertex data given the graph and vertex index
@@ -136,9 +126,7 @@ int opal_bp_graph_add_vertex(opal_bp_graph_t *g,
  *
  * @returns OPAL_SUCCESS or an OMPI error code
  */
-int opal_bp_graph_get_vertex_data(opal_bp_graph_t *g,
-                                  int v_index,
-                                  void** v_data_out);
+int opal_bp_graph_get_vertex_data(opal_bp_graph_t *g, int v_index, void **v_data_out);
 
 /**
  * compute the order of a graph (number of vertices)
@@ -149,7 +137,7 @@ int opal_bp_graph_order(const opal_bp_graph_t *g);
 
 /**
  * This function solves the "assignment problem":
- * http://en.wikipedia.org/wiki/Assignment_problem
+ * https://en.wikipedia.org/wiki/Assignment_problem
  *
  * The goal is to find a maximum cardinality, minimum cost matching in a
  * weighted bipartite graph.  Maximum cardinality takes priority over minimum
@@ -171,8 +159,7 @@ int opal_bp_graph_order(const opal_bp_graph_t *g);
  *
  * @returns OPAL_SUCCESS or an OMPI error code
  */
-int opal_bp_graph_solve_bipartite_assignment(const opal_bp_graph_t *g,
-					     int *num_match_edges_out,
-					     int **match_edges_out);
+int opal_bp_graph_solve_bipartite_assignment(const opal_bp_graph_t *g, int *num_match_edges_out,
+                                             int **match_edges_out);
 
 #endif /* OPAL_BP_GRAPH_H */
diff --git a/opal/util/bipartite_graph_internal.h b/opal/util/bipartite_graph_internal.h
index 64582016311..1243c139973 100644
--- a/opal/util/bipartite_graph_internal.h
+++ b/opal/util/bipartite_graph_internal.h
@@ -77,63 +77,49 @@ struct opal_bp_graph_t {
     opal_bp_graph_cleanup_fn_t e_data_cleanup_fn;
 };
 
-
-#define LIST_FOREACH_CONTAINED(item, list, type, member)		\
-    for (item = container_of( (list)->opal_list_sentinel.opal_list_next, type, member ); \
-	 &item->member != &(list)->opal_list_sentinel;			\
-	 item = container_of(						\
-			     ((opal_list_item_t *) (&item->member))->opal_list_next, type, member ))
-
-#define LIST_FOREACH_SAFE_CONTAINED(item, next, list, type, member)	\
-    for (item = container_of( (list)->opal_list_sentinel.opal_list_next, type, member ), \
-	     next = container_of(					\
-				 ((opal_list_item_t *) (&item->member))->opal_list_next, type, member ); \
-	 &item->member != &(list)->opal_list_sentinel;			\
-	 item = next,							\
-	     next = container_of(					\
-				 ((opal_list_item_t *) (&item->member))->opal_list_next, type, member ))
+#define LIST_FOREACH_CONTAINED(item, list, type, member)                                   \
+    for (item = container_of((list)->opal_list_sentinel.opal_list_next, type, member);     \
+         &item->member != &(list)->opal_list_sentinel;                                     \
+         item = container_of(((opal_list_item_t *) (&item->member))->opal_list_next, type, \
+                             member))
+
+#define LIST_FOREACH_SAFE_CONTAINED(item, next, list, type, member)                                \
+    for (item = container_of((list)->opal_list_sentinel.opal_list_next, type, member),             \
+        next = container_of(((opal_list_item_t *) (&item->member))->opal_list_next, type, member); \
+         &item->member != &(list)->opal_list_sentinel; item = next,                                \
+        next = container_of(((opal_list_item_t *) (&item->member))->opal_list_next, type, member))
 
 #define NUM_VERTICES(g) (g->num_vertices)
 
-#define CHECK_VERTEX_RANGE(g,v)			\
-    do {					\
-        if ((v) < 0 ||				\
-            (v) >= NUM_VERTICES(g)) {		\
-            return OPAL_ERR_BAD_PARAM;		\
-        }					\
+#define CHECK_VERTEX_RANGE(g, v)                 \
+    do {                                         \
+        if ((v) < 0 || (v) >= NUM_VERTICES(g)) { \
+            return OPAL_ERR_BAD_PARAM;           \
+        }                                        \
     } while (0)
 
 /* cast away any constness of &g->vertices b/c the opal_pointer_array API is
  * not const-correct */
-#define V_ID_TO_PTR(g, v_id)						\
-    ((opal_bp_graph_vertex_t *)						\
-     opal_pointer_array_get_item((opal_pointer_array_t *)&g->vertices, v_id))
-
-#define FOREACH_OUT_EDGE(g,v_id,e_ptr)				\
-    LIST_FOREACH_CONTAINED(e_ptr,				\
-                           &(V_ID_TO_PTR(g, v_id)->out_edges),	\
-                           opal_bp_graph_edge_t,		\
+#define V_ID_TO_PTR(g, v_id)                                                                       \
+    ((opal_bp_graph_vertex_t *) opal_pointer_array_get_item((opal_pointer_array_t *) &g->vertices, \
+                                                            v_id))
+
+#define FOREACH_OUT_EDGE(g, v_id, e_ptr)                                                    \
+    LIST_FOREACH_CONTAINED(e_ptr, &(V_ID_TO_PTR(g, v_id)->out_edges), opal_bp_graph_edge_t, \
                            outbound_li)
 
-#define FOREACH_IN_EDGE(g,v_id,e_ptr)				\
-    LIST_FOREACH_CONTAINED(e_ptr,				\
-                           &(V_ID_TO_PTR(g, v_id)->in_edges),	\
-                           opal_bp_graph_edge_t,		\
+#define FOREACH_IN_EDGE(g, v_id, e_ptr)                                                    \
+    LIST_FOREACH_CONTAINED(e_ptr, &(V_ID_TO_PTR(g, v_id)->in_edges), opal_bp_graph_edge_t, \
                            inbound_li)
 
-
 /* Iterate over (u,v) edge pairs along the given path, where path is defined
  * by the predecessor array "pred".  Stops when a -1 predecessor is
  * encountered.  Note: because it is a *predecessor* array, the traversal
  * starts at the sink and progresses towards the source. */
-#define FOREACH_UV_ON_PATH(pred, source, sink, u, v)		\
+#define FOREACH_UV_ON_PATH(pred, source, sink, u, v) \
     for (u = pred[sink], v = sink; u != -1; v = u, u = pred[u])
 
-
-bool opal_bp_graph_bellman_ford(opal_bp_graph_t *gx,
-				int source,
-				int target,
-				int *pred);
+bool opal_bp_graph_bellman_ford(opal_bp_graph_t *gx, int source, int target, int *pred);
 
 int opal_bp_graph_bipartite_to_flow(opal_bp_graph_t *g);
 
diff --git a/opal/util/bit_ops.h b/opal/util/bit_ops.h
index 34ea4418abf..e20fbb1c654 100644
--- a/opal/util/bit_ops.h
+++ b/opal/util/bit_ops.h
@@ -47,11 +47,11 @@ static inline int opal_hibit(int value, int start)
     /* Only look at the part that the caller wanted looking at */
     mask = value & ((1 << start) - 1);
 
-    if (OPAL_UNLIKELY (0 == mask)) {
+    if (OPAL_UNLIKELY(0 == mask)) {
         return -1;
     }
 
-    start = (8*sizeof(int)-1) - __builtin_clz(mask);
+    start = (8 * sizeof(int) - 1) - __builtin_clz(mask);
 #else
     --start;
     mask = 1 << start;
@@ -66,7 +66,6 @@ static inline int opal_hibit(int value, int start)
     return start;
 }
 
-
 /**
  * Returns the cube dimension of a given value.
  *
@@ -87,19 +86,19 @@ static inline int opal_cube_dim(int value)
     int dim, size;
 
 #if OPAL_C_HAVE_BUILTIN_CLZ
-    if (OPAL_UNLIKELY (1 >= value)) {
+    if (OPAL_UNLIKELY(1 >= value)) {
         return 0;
     }
     size = 8 * sizeof(int);
-    dim = size - __builtin_clz(value-1);
+    dim = size - __builtin_clz(value - 1);
 #else
-    for (dim = 0, size = 1; size < value; ++dim, size <<= 1) /* empty */;
+    for (dim = 0, size = 1; size < value; ++dim, size <<= 1) /* empty */
+        ;
 #endif
 
     return dim;
 }
 
-
 /**
  * @brief Returns next power-of-two of the given value.
  *
@@ -117,20 +116,21 @@ static inline int opal_next_poweroftwo(int value)
     int power2;
 
 #if OPAL_C_HAVE_BUILTIN_CLZ
-    if (OPAL_UNLIKELY (0 == value)) {
+    if (OPAL_UNLIKELY(0 == value)) {
         return 1;
     }
-    power2 = 1 << (8 * sizeof (int) - __builtin_clz(value));
+    power2 = 1 << (8 * sizeof(int) - __builtin_clz(value));
 #else
-    for (power2 = 1; value > 0; value >>= 1, power2 <<= 1) /* empty */;
+    for (power2 = 1; value > 0; value >>= 1, power2 <<= 1) /* empty */
+        ;
 #endif
 
     return power2;
 }
 
-
 /**
- * @brief Returns next power-of-two of the given value (and the value itselve if already power-of-two).
+ * @brief Returns next power-of-two of the given value (and the value itselve if already
+ * power-of-two).
  *
  * @param value The integer value to return power of 2
  *
@@ -146,17 +146,16 @@ static inline int opal_next_poweroftwo_inclusive(int value)
     int power2;
 
 #if OPAL_C_HAVE_BUILTIN_CLZ
-    if (OPAL_UNLIKELY (1 >= value)) {
+    if (OPAL_UNLIKELY(1 >= value)) {
         return 1;
     }
-    power2 = 1 << (8 * sizeof (int) - __builtin_clz(value - 1));
+    power2 = 1 << (8 * sizeof(int) - __builtin_clz(value - 1));
 #else
-    for (power2 = 1 ; power2 < value; power2 <<= 1) /* empty */;
+    for (power2 = 1; power2 < value; power2 <<= 1) /* empty */
+        ;
 #endif
 
     return power2;
 }
 
-
 #endif /* OPAL_BIT_OPS_H */
-
diff --git a/opal/util/clock_gettime.h b/opal/util/clock_gettime.h
new file mode 100644
index 00000000000..73eb2f3c2ad
--- /dev/null
+++ b/opal/util/clock_gettime.h
@@ -0,0 +1,116 @@
+/*
+ * Copyright (c) 2022 Cisco Systems, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/** @file clock_gettime.h
+ *
+ * Simple, portable wrappers around clock_gettime(3) and
+ * clock_getres(3) to always get monotonically-increasing time.
+ *
+ * If the underlying OS does not have clock_gettime(3), use
+ * gettimeofday(3) instead.
+ *
+ * We intentionally do not use the OPAL timer framework for
+ * high-prevision time here; see
+ * https://github.com/open-mpi/ompi/issues/3003 for more details.
+ *
+ * As of Dec 2021, it turns out that CLOCK_MONOTONIC can actually go
+ * backwards on macOS (!).  CLOCK_MONOTONIC does *not* go backwards on
+ * Linux (or anywhere else we can find), though, even in the presence
+ * of small NTP time adjustments -- e.g., adjtime(3) simply slightly
+ * speeds up or slows down the system clock to make it eventually get
+ * to the desired time.  On macOS, we can use CLOCK_MONOTONIC_RAW,
+ * which never goes backwards.
+ *
+ * Hence, for these wrappers, use CLOCK_MONOTONIC_RAW on Darwin, and
+ * use CLOCK_MONOTONIC everywhere else.
+ *
+ * See
+ * https://github.com/open-mpi/ompi/pull/8057#discussion_r762612710
+ * and
+ * https://github.com/open-mpi/ompi/pull/8057#discussion_r762618783
+ * for more details.
+ */
+
+#ifndef OPAL_UTIL_CLOCK_GETTIME_H_
+#define OPAL_UTIL_CLOCK_GETTIME_H_
+
+#include "opal_config.h"
+
+#if HAVE_TIME_H
+#include <time.h>
+#endif
+#if HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
+
+#if OPAL_HAVE_CLOCK_GETTIME
+#if defined(__darwin__)
+#define OPAL_CLOCK_TYPE CLOCK_MONOTONIC_RAW
+#else
+#define OPAL_CLOCK_TYPE CLOCK_MONOTONIC
+#endif
+#endif // OPAL_HAVE_CLOCK_GETTIME
+
+#if !defined(HAVE_STRUCT_TIMESPEC_TV_NSEC)
+// Make sure that we have struct timespec; if not, define it.
+struct timespec {
+    time_t tv_sec;
+    long tv_nsec;
+};
+#endif
+
+/**
+ * Simple, portable wrapper around clock_gettime(3) for high-precision time.
+ *
+ * If the underlying system does not have clock_gettime(3), use
+ * gettimeofday(3) instead.
+ *
+ * @param spec (OUT) Struct to return the time
+ * @return Return value from underlying clock_gettime()
+ */
+static inline int opal_clock_gettime(struct timespec *spec)
+{
+#if OPAL_HAVE_CLOCK_GETTIME
+    return clock_gettime(OPAL_CLOCK_TYPE, spec);
+#else
+    // If we do not have clock_gettime(), fall back to gettimeofday()
+    struct timeval tv;
+    int ret = gettimeofday(&tv, NULL);
+
+    spec->tv_sec = tv.tv_sec;
+    // Elevate the micrseconds to nanoseconds
+    spec->tv_nsec = tv.tv_usec * 1000;
+
+    return ret;
+#endif
+}
+
+/**
+ * Simple, portable wrapper around clock_getres(3) for high-precision time.
+ *
+ * If the underlying system does not have clock_gettime(3), return usec
+ * precision (because opal_clock_gettime() will be using gettimeofday(3)).
+ *
+ * @param spec (OUT) Struct to return the resolution
+ * @return Return value from underlying clock_getres()
+ */
+static inline int opal_clock_getres(struct timespec *spec)
+{
+#if OPAL_HAVE_CLOCK_GETTIME
+    return clock_getres(OPAL_CLOCK_TYPE, spec);
+#else
+    // If we don't have clock_gettime(), just return usec precision.
+    spec->tv_sec = 0;
+    spec->tv_nsec = 1000;
+
+    return 0;
+#endif
+}
+
+#endif // OPAL_UTIL_CLOCK_GETTIME_H_
diff --git a/opal/util/cmd_line.c b/opal/util/cmd_line.c
index 39c5bf2c624..0669b27ecf7 100644
--- a/opal/util/cmd_line.c
+++ b/opal/util/cmd_line.c
@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights
  *                         reserved.
- * Copyright (c) 2012-2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2022 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2015-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016-2017 Intel, Inc. All rights reserved.
@@ -26,21 +26,20 @@
 
 #include "opal_config.h"
 
+#include <ctype.h>
 #include <stdio.h>
 #include <string.h>
-#include <ctype.h>
 
-#include "opal/class/opal_object.h"
 #include "opal/class/opal_list.h"
+#include "opal/class/opal_object.h"
 #include "opal/mca/threads/mutex.h"
 #include "opal/util/argv.h"
 #include "opal/util/cmd_line.h"
-#include "opal/util/output.h"
 #include "opal/util/opal_environ.h"
+#include "opal/util/output.h"
 
-#include "opal/mca/base/mca_base_var.h"
 #include "opal/constants.h"
-
+#include "opal/mca/base/mca_base_var.h"
 
 /*
  * Some usage message constants
@@ -77,9 +76,7 @@ typedef struct ompi_cmd_line_option_t ompi_cmd_line_option_t;
 static void option_constructor(ompi_cmd_line_option_t *cmd);
 static void option_destructor(ompi_cmd_line_option_t *cmd);
 
-OBJ_CLASS_INSTANCE(ompi_cmd_line_option_t,
-                   opal_list_item_t,
-                   option_constructor, option_destructor);
+OBJ_CLASS_INSTANCE(ompi_cmd_line_option_t, opal_list_item_t, option_constructor, option_destructor);
 
 /*
  * An option that was used in the argv that was parsed
@@ -88,7 +85,7 @@ struct ompi_cmd_line_param_t {
     opal_list_item_t super;
 
     /* Note that clp_arg points to storage "owned" by someone else; it
-       has the original option string by referene, not by value.
+       has the original option string by reference, not by value.
        Hence, it should not be free()'ed. */
 
     char *clp_arg;
@@ -108,50 +105,39 @@ struct ompi_cmd_line_param_t {
 typedef struct ompi_cmd_line_param_t ompi_cmd_line_param_t;
 static void param_constructor(ompi_cmd_line_param_t *cmd);
 static void param_destructor(ompi_cmd_line_param_t *cmd);
-OBJ_CLASS_INSTANCE(ompi_cmd_line_param_t,
-                   opal_list_item_t,
-                   param_constructor, param_destructor);
+OBJ_CLASS_INSTANCE(ompi_cmd_line_param_t, opal_list_item_t, param_constructor, param_destructor);
 
 /*
  * Instantiate the opal_cmd_line_t class
  */
 static void cmd_line_constructor(opal_cmd_line_t *cmd);
 static void cmd_line_destructor(opal_cmd_line_t *cmd);
-OBJ_CLASS_INSTANCE(opal_cmd_line_t,
-                   opal_object_t,
-                   cmd_line_constructor,
-                   cmd_line_destructor);
+OBJ_CLASS_INSTANCE(opal_cmd_line_t, opal_object_t, cmd_line_constructor, cmd_line_destructor);
 
 /*
  * Private variables
  */
-static char special_empty_token[] = {
-    1, 2, 3, 4, 5, 6, 7, 8, 9, 10, '\0'
-};
+static char special_empty_token[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, '\0'};
 
 /*
  * Private functions
  */
 static int make_opt(opal_cmd_line_t *cmd, opal_cmd_line_init_t *e);
 static void free_parse_results(opal_cmd_line_t *cmd);
-static int split_shorts(opal_cmd_line_t *cmd,
-                        char *token, char **args,
-                        int *output_argc, char ***output_argv,
-                        int *num_args_used, bool ignore_unknown);
-static ompi_cmd_line_option_t *find_option(opal_cmd_line_t *cmd,
-                                      const char *option_name) __opal_attribute_nonnull__(1) __opal_attribute_nonnull__(2);
+static int split_shorts(opal_cmd_line_t *cmd, char *token, char **args, int *output_argc,
+                        char ***output_argv, int *num_args_used, bool ignore_unknown);
+static ompi_cmd_line_option_t *find_option(opal_cmd_line_t *cmd, const char *option_name)
+    __opal_attribute_nonnull__(1) __opal_attribute_nonnull__(2);
 static int set_dest(ompi_cmd_line_option_t *option, char *sval);
 static void fill(const ompi_cmd_line_option_t *a, char result[3][BUFSIZ]);
 static int qsort_callback(const void *a, const void *b);
 static opal_cmd_line_otype_t get_help_otype(opal_cmd_line_t *cmd);
 static char *build_parsable(ompi_cmd_line_option_t *option);
 
-
 /*
  * Create an entire command line handle from a table
  */
-int opal_cmd_line_create(opal_cmd_line_t *cmd,
-                         opal_cmd_line_init_t *table)
+int opal_cmd_line_create(opal_cmd_line_t *cmd, opal_cmd_line_init_t *table)
 {
     int ret = OPAL_SUCCESS;
 
@@ -169,8 +155,7 @@ int opal_cmd_line_create(opal_cmd_line_t *cmd,
 }
 
 /* Add a table to an existing cmd line object */
-int opal_cmd_line_add(opal_cmd_line_t *cmd,
-                      opal_cmd_line_init_t *table)
+int opal_cmd_line_add(opal_cmd_line_t *cmd, opal_cmd_line_init_t *table)
 {
     int i, ret;
 
@@ -181,11 +166,10 @@ int opal_cmd_line_add(opal_cmd_line_t *cmd,
 
     /* Loop through the table */
 
-    for (i = 0; ; ++i) {
+    for (i = 0;; ++i) {
         /* Is this the end? */
-        if ('\0' == table[i].ocl_cmd_short_name &&
-            NULL == table[i].ocl_cmd_single_dash_name &&
-            NULL == table[i].ocl_cmd_long_name) {
+        if ('\0' == table[i].ocl_cmd_short_name && NULL == table[i].ocl_cmd_single_dash_name
+            && NULL == table[i].ocl_cmd_long_name) {
             break;
         }
 
@@ -201,26 +185,22 @@ int opal_cmd_line_add(opal_cmd_line_t *cmd,
 /*
  * Append a command line entry to the previously constructed command line
  */
-int opal_cmd_line_make_opt_mca(opal_cmd_line_t *cmd,
-                               opal_cmd_line_init_t entry)
+int opal_cmd_line_make_opt_mca(opal_cmd_line_t *cmd, opal_cmd_line_init_t entry)
 {
     /* Ensure we got an entry */
-    if ('\0' == entry.ocl_cmd_short_name &&
-        NULL == entry.ocl_cmd_single_dash_name &&
-        NULL == entry.ocl_cmd_long_name) {
+    if ('\0' == entry.ocl_cmd_short_name && NULL == entry.ocl_cmd_single_dash_name
+        && NULL == entry.ocl_cmd_long_name) {
         return OPAL_SUCCESS;
     }
 
     return make_opt(cmd, &entry);
 }
 
-
 /*
  * Create a command line option, --long-name and/or -s (short name).
  */
-int opal_cmd_line_make_opt3(opal_cmd_line_t *cmd, char short_name,
-                            const char *sd_name, const char *long_name,
-                            int num_params, const char *desc)
+int opal_cmd_line_make_opt3(opal_cmd_line_t *cmd, char short_name, const char *sd_name,
+                            const char *long_name, int num_params, const char *desc)
 {
     opal_cmd_line_init_t e;
 
@@ -237,12 +217,11 @@ int opal_cmd_line_make_opt3(opal_cmd_line_t *cmd, char short_name,
 
     e.ocl_description = desc;
 
-    e.ocl_otype = OPAL_CMD_LINE_OTYPE_NULL;
+    e.ocl_otype = OPAL_CMD_LINE_OTYPE_GENERAL;
 
     return make_opt(cmd, &e);
 }
 
-
 /*
  * Parse a command line according to a pre-built OPAL command line
  * handle.
@@ -290,12 +269,12 @@ int opal_cmd_line_parse(opal_cmd_line_t *cmd, bool ignore_unknown, bool ignore_u
     }
 
     /* Now traverse the easy-to-parse sequence of tokens.  Note that
-       incrementing i must happen elsehwere; it can't be the third
+       incrementing i must happen elsewhere; it can't be the third
        clause in the "if" statement. */
 
     param = NULL;
     option = NULL;
-    for (i = 1; i < cmd->lcl_argc; ) {
+    for (i = 1; i < cmd->lcl_argc;) {
         is_unknown_option = false;
         is_unknown_token = false;
         is_option = false;
@@ -307,8 +286,7 @@ int opal_cmd_line_parse(opal_cmd_line_t *cmd, bool ignore_unknown, bool ignore_u
         if (0 == strcmp(cmd->lcl_argv[i], "--")) {
             ++i;
             while (i < cmd->lcl_argc) {
-                opal_argv_append(&cmd->lcl_tail_argc, &cmd->lcl_tail_argv,
-                                 cmd->lcl_argv[i]);
+                opal_argv_append(&cmd->lcl_tail_argc, &cmd->lcl_tail_argv, cmd->lcl_argv[i]);
                 ++i;
             }
 
@@ -345,17 +323,13 @@ int opal_cmd_line_parse(opal_cmd_line_t *cmd, bool ignore_unknown, bool ignore_u
             if (NULL == option) {
                 shortsv = NULL;
                 shortsc = 0;
-                ret = split_shorts(cmd, cmd->lcl_argv[i] + 1,
-                                   &(cmd->lcl_argv[i + 1]),
-                                   &shortsc, &shortsv,
-                                   &num_args_used, ignore_unknown);
+                ret = split_shorts(cmd, cmd->lcl_argv[i] + 1, &(cmd->lcl_argv[i + 1]), &shortsc,
+                                   &shortsv, &num_args_used, ignore_unknown);
                 if (OPAL_SUCCESS == ret) {
                     option = find_option(cmd, shortsv[0] + 1);
 
                     if (NULL != option) {
-                        opal_argv_delete(&cmd->lcl_argc,
-                                         &cmd->lcl_argv, i,
-                                         1 + num_args_used);
+                        opal_argv_delete(&cmd->lcl_argc, &cmd->lcl_argv, i, 1 + num_args_used);
                         opal_argv_insert(&cmd->lcl_argv, i, shortsv);
                         cmd->lcl_argc = opal_argv_count(cmd->lcl_argv);
                     } else {
@@ -404,37 +378,32 @@ int opal_cmd_line_parse(opal_cmd_line_t *cmd, bool ignore_unknown, bool ignore_u
                     /* If we run out of parameters, error, unless its a help request
                        which can have 0 or 1 arguments */
                     if (i >= cmd->lcl_argc) {
-                    /* If this is a help request, can have no arguments */
-                        if((NULL != option->clo_single_dash_name &&
-                           0 == strcmp(option->clo_single_dash_name, "h")) ||
-                           (NULL != option->clo_long_name &&
-                           0 == strcmp(option->clo_long_name, "help"))) {
+                        /* If this is a help request, can have no arguments */
+                        if ((NULL != option->clo_single_dash_name
+                             && 0 == strcmp(option->clo_single_dash_name, "h"))
+                            || (NULL != option->clo_long_name
+                                && 0 == strcmp(option->clo_long_name, "help"))) {
                             help_without_arg = true;
                             continue;
                         }
-                        fprintf(stderr, "%s: Error: option \"%s\" did not "
+                        fprintf(stderr,
+                                "%s: Error: option \"%s\" did not "
                                 "have enough parameters (%d)\n",
-                                cmd->lcl_argv[0],
-                                cmd->lcl_argv[orig],
-                                option->clo_num_params);
+                                cmd->lcl_argv[0], cmd->lcl_argv[orig], option->clo_num_params);
                         if (have_help_option) {
-                            fprintf(stderr, "Type '%s --help' for usage.\n",
-                                    cmd->lcl_argv[0]);
+                            fprintf(stderr, "Type '%s --help' for usage.\n", cmd->lcl_argv[0]);
                         }
                         OBJ_RELEASE(param);
                         printed_error = true;
                         goto error;
                     } else {
-                        if (0 == strcmp(cmd->lcl_argv[i],
-                                        special_empty_token)) {
-                            fprintf(stderr, "%s: Error: option \"%s\" did not "
+                        if (0 == strcmp(cmd->lcl_argv[i], special_empty_token)) {
+                            fprintf(stderr,
+                                    "%s: Error: option \"%s\" did not "
                                     "have enough parameters (%d)\n",
-                                    cmd->lcl_argv[0],
-                                    cmd->lcl_argv[orig],
-                                    option->clo_num_params);
+                                    cmd->lcl_argv[0], cmd->lcl_argv[orig], option->clo_num_params);
                             if (have_help_option) {
-                                fprintf(stderr, "Type '%s --help' for usage.\n",
-                                        cmd->lcl_argv[0]);
+                                fprintf(stderr, "Type '%s --help' for usage.\n", cmd->lcl_argv[0]);
                             }
                             if (NULL != param->clp_argv) {
                                 opal_argv_free(param->clp_argv);
@@ -449,16 +418,14 @@ int opal_cmd_line_parse(opal_cmd_line_t *cmd, bool ignore_unknown, bool ignore_u
                         else {
                             /* Save in the argv on the param entry */
 
-                            opal_argv_append(&param->clp_argc,
-                                             &param->clp_argv,
-                                             cmd->lcl_argv[i]);
+                            opal_argv_append(&param->clp_argc, &param->clp_argv, cmd->lcl_argv[i]);
 
                             /* If it's the first, save it in the
                                variable dest and/or MCA parameter */
 
-                            if (0 == j &&
-                                (NULL != option->clo_mca_param_env_var ||
-                                 NULL != option->clo_variable_dest)) {
+                            if (0 == j
+                                && (NULL != option->clo_mca_param_env_var
+                                    || NULL != option->clo_variable_dest)) {
                                 if (OPAL_SUCCESS != (ret = set_dest(option, cmd->lcl_argv[i]))) {
                                     opal_mutex_unlock(&cmd->lcl_mutex);
                                     return ret;
@@ -494,18 +461,16 @@ int opal_cmd_line_parse(opal_cmd_line_t *cmd, bool ignore_unknown, bool ignore_u
            an error and return. */
         if (is_unknown_option || is_unknown_token) {
             if (!ignore_unknown || (is_unknown_option && !ignore_unknown_option)) {
-                fprintf(stderr, "%s: Error: unknown option \"%s\"\n",
-                        cmd->lcl_argv[0], cmd->lcl_argv[i]);
+                fprintf(stderr, "%s: Error: unknown option \"%s\"\n", cmd->lcl_argv[0],
+                        cmd->lcl_argv[i]);
                 printed_error = true;
                 if (have_help_option) {
-                    fprintf(stderr, "Type '%s --help' for usage.\n",
-                            cmd->lcl_argv[0]);
+                    fprintf(stderr, "Type '%s --help' for usage.\n", cmd->lcl_argv[0]);
                 }
             }
         error:
             while (i < cmd->lcl_argc) {
-                opal_argv_append(&cmd->lcl_tail_argc, &cmd->lcl_tail_argv,
-                                 cmd->lcl_argv[i]);
+                opal_argv_append(&cmd->lcl_tail_argc, &cmd->lcl_tail_argv, cmd->lcl_argv[i]);
                 ++i;
             }
 
@@ -525,7 +490,6 @@ int opal_cmd_line_parse(opal_cmd_line_t *cmd, bool ignore_unknown, bool ignore_u
     return OPAL_SUCCESS;
 }
 
-
 /*
  * Return a consolidated "usage" message for a OPAL command line handle.
  */
@@ -535,7 +499,7 @@ char *opal_cmd_line_get_usage_msg(opal_cmd_line_t *cmd)
     int argc;
     size_t j;
     char **argv;
-    char *ret, temp[MAX_WIDTH * 2], line[MAX_WIDTH * 2];
+    char *ret, line[(MAX_WIDTH * 2) + 1];
     char *start, *desc, *ptr;
     opal_list_item_t *item;
     ompi_cmd_line_option_t *option, **sorted;
@@ -553,17 +517,17 @@ char *opal_cmd_line_get_usage_msg(opal_cmd_line_t *cmd)
 
     /* First, take the original list and sort it */
 
-    sorted = (ompi_cmd_line_option_t**)malloc(sizeof(ompi_cmd_line_option_t *) *
-                                         opal_list_get_size(&cmd->lcl_options));
+    sorted = (ompi_cmd_line_option_t **) malloc(sizeof(ompi_cmd_line_option_t *)
+                                                * opal_list_get_size(&cmd->lcl_options));
     if (NULL == sorted) {
         opal_mutex_unlock(&cmd->lcl_mutex);
         return NULL;
     }
     i = 0;
-    OPAL_LIST_FOREACH(item, &cmd->lcl_options, opal_list_item_t) {
+    OPAL_LIST_FOREACH (item, &cmd->lcl_options, opal_list_item_t) {
         sorted[i++] = (ompi_cmd_line_option_t *) item;
     }
-    qsort(sorted, i, sizeof(ompi_cmd_line_option_t*), qsort_callback);
+    qsort(sorted, i, sizeof(ompi_cmd_line_option_t *), qsort_callback);
 
     /* Find if a help argument was passed, and return its type if it was. */
 
@@ -573,12 +537,12 @@ char *opal_cmd_line_get_usage_msg(opal_cmd_line_t *cmd)
 
     for (j = 0; j < opal_list_get_size(&cmd->lcl_options); ++j) {
         option = sorted[j];
-        if(otype == OPAL_CMD_LINE_OTYPE_PARSABLE) {
+        if (otype == OPAL_CMD_LINE_OTYPE_PARSABLE) {
             ret = build_parsable(option);
             opal_argv_append(&argc, &argv, ret);
             free(ret);
             ret = NULL;
-        } else if(otype == OPAL_CMD_LINE_OTYPE_NULL || option->clo_otype == otype) {
+        } else if (otype == OPAL_CMD_LINE_OTYPE_NULL || option->clo_otype == otype) {
             if (NULL != option->clo_description) {
                 bool filled = false;
 
@@ -595,27 +559,27 @@ char *opal_cmd_line_get_usage_msg(opal_cmd_line_t *cmd)
                 }
                 if (NULL != option->clo_single_dash_name) {
                     line[2] = (filled) ? '|' : ' ';
-                    strncat(line, "-", sizeof(line) - 1);
-                    strncat(line, option->clo_single_dash_name, sizeof(line) - 1);
+                    strncat(line, "-", sizeof(line) - strlen(line) - 1);
+                    strncat(line, option->clo_single_dash_name, sizeof(line) - strlen(line) - 1);
                     filled = true;
                 }
                 if (NULL != option->clo_long_name) {
                     if (filled) {
-                        strncat(line, "|", sizeof(line) - 1);
+                        strncat(line, "|", sizeof(line) - strlen(line) - 1);
                     } else {
-                        strncat(line, " ", sizeof(line) - 1);
+                        strncat(line, " ", sizeof(line) - strlen(line) - 1);
                     }
-                    strncat(line, "--", sizeof(line) - 1);
-                    strncat(line, option->clo_long_name, sizeof(line) - 1);
+                    strncat(line, "--", sizeof(line) - strlen(line) - 1);
+                    strncat(line, option->clo_long_name, sizeof(line) - strlen(line) - 1);
                 }
-                strncat(line, " ", sizeof(line) - 1);
-                for (i = 0; (int)i < option->clo_num_params; ++i) {
-                    len = sizeof(temp);
-                    snprintf(temp, len, "<arg%d> ", (int)i);
-                    strncat(line, temp, sizeof(line) - 1);
+                strncat(line, " ", sizeof(line) - strlen(line) - 1);
+                for (i = 0; (int) i < option->clo_num_params; ++i) {
+                    char temp[MAX_WIDTH * 2];
+                    snprintf(temp, MAX_WIDTH * 2, "<arg%d> ", (int) i);
+                    strncat(line, temp, sizeof(line) - strlen(line) - 1);
                 }
                 if (option->clo_num_params > 0) {
-                    strncat(line, " ", sizeof(line) - 1);
+                    strncat(line, " ", sizeof(line) - strlen(line) - 1);
                 }
 
                 /* If we're less than param width, then start adding the
@@ -644,7 +608,7 @@ char *opal_cmd_line_get_usage_msg(opal_cmd_line_t *cmd)
 
                 /* Loop over adding the description to the array, breaking
                    the string at most at MAX_WIDTH characters.  We need a
-                   modifyable description (for simplicity), so strdup the
+                   modifiable description (for simplicity), so strdup the
                    clo_description (because it's likely a compiler
                    constant, and may barf if we write temporary \0's in
                    the middle). */
@@ -671,7 +635,7 @@ char *opal_cmd_line_get_usage_msg(opal_cmd_line_t *cmd)
                     /* Last line */
 
                     if (strlen(start) < (MAX_WIDTH - PARAM_WIDTH)) {
-                        strncat(line, start, sizeof(line) - 1);
+                        strncat(line, start, sizeof(line) - strlen(line) - 1);
                         opal_argv_append(&argc, &argv, line);
                         break;
                     }
@@ -680,11 +644,10 @@ char *opal_cmd_line_get_usage_msg(opal_cmd_line_t *cmd)
                        line's worth and add it to the array.  Then reset
                        and loop around to get the next line's worth. */
 
-                    for (ptr = start + (MAX_WIDTH - PARAM_WIDTH);
-                         ptr > start; --ptr) {
+                    for (ptr = start + (MAX_WIDTH - PARAM_WIDTH); ptr > start; --ptr) {
                         if (isspace(*ptr)) {
                             *ptr = '\0';
-                            strncat(line, start, sizeof(line) - 1);
+                            strncat(line, start, sizeof(line) - strlen(line) - 1);
                             opal_argv_append(&argc, &argv, line);
 
                             start = ptr + 1;
@@ -699,12 +662,11 @@ char *opal_cmd_line_get_usage_msg(opal_cmd_line_t *cmd)
                        and break there. */
 
                     if (ptr == start) {
-                        for (ptr = start + (MAX_WIDTH - PARAM_WIDTH);
-                             ptr < start + len; ++ptr) {
+                        for (ptr = start + (MAX_WIDTH - PARAM_WIDTH); ptr < start + len; ++ptr) {
                             if (isspace(*ptr)) {
                                 *ptr = '\0';
 
-                                strncat(line, start, sizeof(line) - 1);
+                                strncat(line, start, sizeof(line) - strlen(line) - 1);
                                 opal_argv_append(&argc, &argv, line);
 
                                 start = ptr + 1;
@@ -718,7 +680,7 @@ char *opal_cmd_line_get_usage_msg(opal_cmd_line_t *cmd)
                            whitespace, then just add it on and be done */
 
                         if (ptr >= start + len) {
-                            strncat(line, start, sizeof(line) - 1);
+                            strncat(line, start, sizeof(line) - strlen(line) - 1);
                             opal_argv_append(&argc, &argv, line);
                             start = desc + len + 1;
                         }
@@ -728,11 +690,6 @@ char *opal_cmd_line_get_usage_msg(opal_cmd_line_t *cmd)
             }
         }
     }
-    if(otype == OPAL_CMD_LINE_OTYPE_NULL || otype == OPAL_CMD_LINE_OTYPE_GENERAL) {
-        char *argument_line = "\nFor additional mpirun arguments, run 'mpirun --help <category>'\n\nThe following categories exist: general (Defaults to this option), debug,\n    output, input, mapping, ranking, binding, devel (arguments useful to OMPI\n    Developers), compatibility (arguments supported for backwards compatibility),\n    launch (arguments to modify launch options), and dvm (Distributed Virtual\n    Machine arguments).";
-
-        opal_argv_append(&argc, &argv, argument_line);
-    }
     if (NULL != argv) {
         ret = opal_argv_join(argv, '\n');
         opal_argv_free(argv);
@@ -748,7 +705,6 @@ char *opal_cmd_line_get_usage_msg(opal_cmd_line_t *cmd)
     return ret;
 }
 
-
 /*
  * Test if a given option was taken on the parsed command line.
  */
@@ -757,7 +713,6 @@ bool opal_cmd_line_is_taken(opal_cmd_line_t *cmd, const char *opt)
     return (opal_cmd_line_get_ninsts(cmd, opt) > 0);
 }
 
-
 /*
  * Return the number of instances of an option found during parsing.
  */
@@ -777,7 +732,7 @@ int opal_cmd_line_get_ninsts(opal_cmd_line_t *cmd, const char *opt)
     ret = 0;
     option = find_option(cmd, opt);
     if (NULL != option) {
-        OPAL_LIST_FOREACH(param, &cmd->lcl_params, ompi_cmd_line_param_t) {
+        OPAL_LIST_FOREACH (param, &cmd->lcl_params, ompi_cmd_line_param_t) {
             if (param->clp_option == option) {
                 ++ret;
             }
@@ -793,13 +748,11 @@ int opal_cmd_line_get_ninsts(opal_cmd_line_t *cmd, const char *opt)
     return ret;
 }
 
-
 /*
  * Return a specific parameter for a specific instance of a option
  * from the parsed command line.
  */
-char *opal_cmd_line_get_param(opal_cmd_line_t *cmd, const char *opt, int inst,
-                              int idx)
+char *opal_cmd_line_get_param(opal_cmd_line_t *cmd, const char *opt, int inst, int idx)
 {
     int num_found;
     ompi_cmd_line_param_t *param;
@@ -820,7 +773,7 @@ char *opal_cmd_line_get_param(opal_cmd_line_t *cmd, const char *opt, int inst,
            parameter index greater than we will have */
 
         if (idx < option->clo_num_params) {
-            OPAL_LIST_FOREACH(param, &cmd->lcl_params, ompi_cmd_line_param_t) {
+            OPAL_LIST_FOREACH (param, &cmd->lcl_params, ompi_cmd_line_param_t) {
                 if (param->clp_argc > 0 && param->clp_option == option) {
                     if (num_found == inst) {
                         opal_mutex_unlock(&cmd->lcl_mutex);
@@ -841,7 +794,6 @@ char *opal_cmd_line_get_param(opal_cmd_line_t *cmd, const char *opt, int inst,
     return NULL;
 }
 
-
 /*
  * Return the number of arguments parsed on a OPAL command line handle.
  */
@@ -850,17 +802,16 @@ int opal_cmd_line_get_argc(opal_cmd_line_t *cmd)
     return (NULL != cmd) ? cmd->lcl_argc : OPAL_ERROR;
 }
 
-
 /*
  * Return a string argument parsed on a OPAL command line handle.
  */
 char *opal_cmd_line_get_argv(opal_cmd_line_t *cmd, int index)
 {
-    return (NULL == cmd) ? NULL :
-        (index >= cmd->lcl_argc || index < 0) ? NULL : cmd->lcl_argv[index];
+    return (NULL == cmd)                           ? NULL
+           : (index >= cmd->lcl_argc || index < 0) ? NULL
+                                                   : cmd->lcl_argv[index];
 }
 
-
 /*
  * Return the entire "tail" of unprocessed argv from a OPAL command
  * line handle.
@@ -878,7 +829,6 @@ int opal_cmd_line_get_tail(opal_cmd_line_t *cmd, int *tailc, char ***tailv)
     }
 }
 
-
 /**************************************************************************
  * Static functions
  **************************************************************************/
@@ -898,7 +848,6 @@ static void option_constructor(ompi_cmd_line_option_t *o)
     o->clo_otype = OPAL_CMD_LINE_OTYPE_NULL;
 }
 
-
 static void option_destructor(ompi_cmd_line_option_t *o)
 {
     if (NULL != o->clo_single_dash_name) {
@@ -915,7 +864,6 @@ static void option_destructor(ompi_cmd_line_option_t *o)
     }
 }
 
-
 static void param_constructor(ompi_cmd_line_param_t *p)
 {
     p->clp_arg = NULL;
@@ -924,7 +872,6 @@ static void param_constructor(ompi_cmd_line_param_t *p)
     p->clp_argv = NULL;
 }
 
-
 static void param_destructor(ompi_cmd_line_param_t *p)
 {
     if (NULL != p->clp_argv) {
@@ -932,7 +879,6 @@ static void param_destructor(ompi_cmd_line_param_t *p)
     }
 }
 
-
 static void cmd_line_constructor(opal_cmd_line_t *cmd)
 {
     /* Initialize the mutex.  Since we're creating (and therefore the
@@ -954,7 +900,6 @@ static void cmd_line_constructor(opal_cmd_line_t *cmd)
     cmd->lcl_tail_argv = NULL;
 }
 
-
 static void cmd_line_destructor(opal_cmd_line_t *cmd)
 {
     opal_list_item_t *item;
@@ -962,8 +907,7 @@ static void cmd_line_destructor(opal_cmd_line_t *cmd)
     /* Free the contents of the options list (do not free the list
        itself; it was not allocated from the heap) */
 
-    for (item = opal_list_remove_first(&cmd->lcl_options);
-         NULL != item;
+    for (item = opal_list_remove_first(&cmd->lcl_options); NULL != item;
          item = opal_list_remove_first(&cmd->lcl_options)) {
         OBJ_RELEASE(item);
     }
@@ -982,7 +926,6 @@ static void cmd_line_destructor(opal_cmd_line_t *cmd)
     OBJ_DESTRUCT(&cmd->lcl_mutex);
 }
 
-
 static int make_opt(opal_cmd_line_t *cmd, opal_cmd_line_init_t *e)
 {
     ompi_cmd_line_option_t *option;
@@ -991,22 +934,20 @@ static int make_opt(opal_cmd_line_t *cmd, opal_cmd_line_init_t *e)
 
     if (NULL == cmd) {
         return OPAL_ERR_BAD_PARAM;
-    } else if ('\0' == e->ocl_cmd_short_name &&
-               NULL == e->ocl_cmd_single_dash_name &&
-               NULL == e->ocl_cmd_long_name) {
+    } else if ('\0' == e->ocl_cmd_short_name && NULL == e->ocl_cmd_single_dash_name
+               && NULL == e->ocl_cmd_long_name) {
         return OPAL_ERR_BAD_PARAM;
     } else if (e->ocl_num_params < 0) {
         return OPAL_ERR_BAD_PARAM;
     }
 
     /* see if the option already exists */
-    if (NULL != e->ocl_cmd_single_dash_name &&
-        NULL != find_option(cmd, e->ocl_cmd_single_dash_name)) {
+    if (NULL != e->ocl_cmd_single_dash_name
+        && NULL != find_option(cmd, e->ocl_cmd_single_dash_name)) {
         opal_output(0, "Duplicate cmd line entry %s", e->ocl_cmd_single_dash_name);
         return OPAL_ERR_BAD_PARAM;
     }
-    if (NULL != e->ocl_cmd_long_name &&
-        NULL != find_option(cmd, e->ocl_cmd_long_name)) {
+    if (NULL != e->ocl_cmd_long_name && NULL != find_option(cmd, e->ocl_cmd_long_name)) {
         opal_output(0, "Duplicate cmd line entry %s", e->ocl_cmd_long_name);
         return OPAL_ERR_BAD_PARAM;
     }
@@ -1032,8 +973,7 @@ static int make_opt(opal_cmd_line_t *cmd, opal_cmd_line_init_t *e)
     option->clo_type = e->ocl_variable_type;
     option->clo_variable_dest = e->ocl_variable_dest;
     if (NULL != e->ocl_mca_param_name) {
-        (void) mca_base_var_env_name (e->ocl_mca_param_name,
-                                     &option->clo_mca_param_env_var);
+        (void) mca_base_var_env_name(e->ocl_mca_param_name, &option->clo_mca_param_env_var);
     }
 
     option->clo_otype = e->ocl_otype;
@@ -1049,7 +989,6 @@ static int make_opt(opal_cmd_line_t *cmd, opal_cmd_line_init_t *e)
     return OPAL_SUCCESS;
 }
 
-
 static void free_parse_results(opal_cmd_line_t *cmd)
 {
     opal_list_item_t *item;
@@ -1057,8 +996,7 @@ static void free_parse_results(opal_cmd_line_t *cmd)
     /* Free the contents of the params list (do not free the list
        itself; it was not allocated from the heap) */
 
-    for (item = opal_list_remove_first(&cmd->lcl_params);
-         NULL != item;
+    for (item = opal_list_remove_first(&cmd->lcl_params); NULL != item;
          item = opal_list_remove_first(&cmd->lcl_params)) {
         OBJ_RELEASE(item);
     }
@@ -1078,16 +1016,14 @@ static void free_parse_results(opal_cmd_line_t *cmd)
     cmd->lcl_tail_argc = 0;
 }
 
-
 /*
  * Traverse a token and split it into individual letter options (the
  * token has already been certified to not be a long name and not be a
  * short name).  Ensure to differentiate the resulting options from
  * "single dash" names.
  */
-static int split_shorts(opal_cmd_line_t *cmd, char *token, char **args,
-                        int *output_argc, char ***output_argv,
-                        int *num_args_used, bool ignore_unknown)
+static int split_shorts(opal_cmd_line_t *cmd, char *token, char **args, int *output_argc,
+                        char ***output_argv, int *num_args_used, bool ignore_unknown)
 {
     int i, j, len;
     ompi_cmd_line_option_t *option;
@@ -1104,7 +1040,7 @@ static int split_shorts(opal_cmd_line_t *cmd, char *token, char **args,
        (argv[i] + 1), will be empty by the time it gets down here),
        just return that we didn't find a short option. */
 
-    len = (int)strlen(token);
+    len = (int) strlen(token);
     if (0 == len) {
         return OPAL_ERR_BAD_PARAM;
     }
@@ -1126,7 +1062,7 @@ static int split_shorts(opal_cmd_line_t *cmd, char *token, char **args,
         }
 
         /* If we do find the option, copy it and all of its parameters
-           to the output args.  If we run out of paramters (i.e., no
+           to the output args.  If we run out of parameters (i.e., no
            more tokens in the original argv), that error will be
            handled at a higher level) */
 
@@ -1134,12 +1070,10 @@ static int split_shorts(opal_cmd_line_t *cmd, char *token, char **args,
             opal_argv_append(output_argc, output_argv, fake_token);
             for (j = 0; j < option->clo_num_params; ++j) {
                 if (*num_args_used < num_args) {
-                    opal_argv_append(output_argc, output_argv,
-                                     args[*num_args_used]);
+                    opal_argv_append(output_argc, output_argv, args[*num_args_used]);
                     ++(*num_args_used);
                 } else {
-                    opal_argv_append(output_argc, output_argv,
-                                     special_empty_token);
+                    opal_argv_append(output_argc, output_argv, special_empty_token);
                 }
             }
         }
@@ -1150,9 +1084,7 @@ static int split_shorts(opal_cmd_line_t *cmd, char *token, char **args,
     return OPAL_SUCCESS;
 }
 
-
-static ompi_cmd_line_option_t *find_option(opal_cmd_line_t *cmd,
-                                      const char *option_name)
+static ompi_cmd_line_option_t *find_option(opal_cmd_line_t *cmd, const char *option_name)
 {
     ompi_cmd_line_option_t *option;
 
@@ -1160,13 +1092,11 @@ static ompi_cmd_line_option_t *find_option(opal_cmd_line_t *cmd,
        opal_cmd_line_t and see if we find a match in either the short
        or long names */
 
-    OPAL_LIST_FOREACH(option, &cmd->lcl_options, ompi_cmd_line_option_t) {
-        if ((NULL != option->clo_long_name &&
-             0 == strcmp(option_name, option->clo_long_name)) ||
-            (NULL != option->clo_single_dash_name &&
-             0 == strcmp(option_name, option->clo_single_dash_name)) ||
-            (strlen(option_name) == 1 &&
-             option_name[0] == option->clo_short_name)) {
+    OPAL_LIST_FOREACH (option, &cmd->lcl_options, ompi_cmd_line_option_t) {
+        if ((NULL != option->clo_long_name && 0 == strcmp(option_name, option->clo_long_name))
+            || (NULL != option->clo_single_dash_name
+                && 0 == strcmp(option_name, option->clo_single_dash_name))
+            || (strlen(option_name) == 1 && option_name[0] == option->clo_short_name)) {
             return option;
         }
     }
@@ -1176,7 +1106,6 @@ static ompi_cmd_line_option_t *find_option(opal_cmd_line_t *cmd,
     return NULL;
 }
 
-
 static int set_dest(ompi_cmd_line_option_t *option, char *sval)
 {
     int ival = atol(sval);
@@ -1194,7 +1123,7 @@ static int set_dest(ompi_cmd_line_option_t *option, char *sval)
        during a nromal parameter lookup, and all will be well. */
 
     if (NULL != option->clo_mca_param_env_var) {
-        switch(option->clo_type) {
+        switch (option->clo_type) {
         case OPAL_CMD_LINE_TYPE_STRING:
         case OPAL_CMD_LINE_TYPE_INT:
         case OPAL_CMD_LINE_TYPE_SIZE_T:
@@ -1211,19 +1140,21 @@ static int set_dest(ompi_cmd_line_option_t *option, char *sval)
     /* Set variable */
 
     if (NULL != option->clo_variable_dest) {
-        switch(option->clo_type) {
+        switch (option->clo_type) {
         case OPAL_CMD_LINE_TYPE_STRING:
-            *((char**) option->clo_variable_dest) = strdup(sval);
+            *((char **) option->clo_variable_dest) = strdup(sval);
             break;
         case OPAL_CMD_LINE_TYPE_INT:
             /* check to see that the value given to us truly is an int */
-            for (i=0; i < strlen(sval); i++) {
+            for (i = 0; i < strlen(sval); i++) {
                 if (!isdigit(sval[i]) && '-' != sval[i]) {
                     /* show help isn't going to be available yet, so just
                      * print the msg
                      */
-                    fprintf(stderr, "----------------------------------------------------------------------------\n");
-                    fprintf(stderr, "Open MPI has detected that a parameter given to a command line\n");
+                    fprintf(stderr, "--------------------------------------------------------------"
+                                    "--------------\n");
+                    fprintf(stderr,
+                            "Open MPI has detected that a parameter given to a command line\n");
                     fprintf(stderr, "option does not match the expected format:\n\n");
                     if (NULL != option->clo_long_name) {
                         fprintf(stderr, "  Option: %s\n", option->clo_long_name);
@@ -1233,23 +1164,28 @@ static int set_dest(ompi_cmd_line_option_t *option, char *sval)
                         fprintf(stderr, "  Option: <unknown>\n");
                     }
                     fprintf(stderr, "  Param:  %s\n\n", sval);
-                    fprintf(stderr, "This is frequently caused by omitting to provide the parameter\n");
-                    fprintf(stderr, "to an option that requires one. Please check the command line and try again.\n");
-                    fprintf(stderr, "----------------------------------------------------------------------------\n");
+                    fprintf(stderr,
+                            "This is frequently caused by omitting to provide the parameter\n");
+                    fprintf(stderr, "to an option that requires one. Please check the command line "
+                                    "and try again.\n");
+                    fprintf(stderr, "--------------------------------------------------------------"
+                                    "--------------\n");
                     return OPAL_ERR_SILENT;
                 }
             }
-            *((int*) option->clo_variable_dest) = ival;
+            *((int *) option->clo_variable_dest) = ival;
             break;
         case OPAL_CMD_LINE_TYPE_SIZE_T:
             /* check to see that the value given to us truly is a size_t */
-            for (i=0; i < strlen(sval); i++) {
+            for (i = 0; i < strlen(sval); i++) {
                 if (!isdigit(sval[i]) && '-' != sval[i]) {
                     /* show help isn't going to be available yet, so just
                      * print the msg
                      */
-                    fprintf(stderr, "----------------------------------------------------------------------------\n");
-                    fprintf(stderr, "Open MPI has detected that a parameter given to a command line\n");
+                    fprintf(stderr, "--------------------------------------------------------------"
+                                    "--------------\n");
+                    fprintf(stderr,
+                            "Open MPI has detected that a parameter given to a command line\n");
                     fprintf(stderr, "option does not match the expected format:\n\n");
                     if (NULL != option->clo_long_name) {
                         fprintf(stderr, "  Option: %s\n", option->clo_long_name);
@@ -1259,16 +1195,19 @@ static int set_dest(ompi_cmd_line_option_t *option, char *sval)
                         fprintf(stderr, "  Option: <unknown>\n");
                     }
                     fprintf(stderr, "  Param:  %s\n\n", sval);
-                    fprintf(stderr, "This is frequently caused by omitting to provide the parameter\n");
-                    fprintf(stderr, "to an option that requires one. Please check the command line and try again.\n");
-                    fprintf(stderr, "----------------------------------------------------------------------------\n");
+                    fprintf(stderr,
+                            "This is frequently caused by omitting to provide the parameter\n");
+                    fprintf(stderr, "to an option that requires one. Please check the command line "
+                                    "and try again.\n");
+                    fprintf(stderr, "--------------------------------------------------------------"
+                                    "--------------\n");
                     return OPAL_ERR_SILENT;
                 }
             }
-            *((size_t*) option->clo_variable_dest) = lval;
+            *((size_t *) option->clo_variable_dest) = lval;
             break;
         case OPAL_CMD_LINE_TYPE_BOOL:
-            *((bool*) option->clo_variable_dest) = 1;
+            *((bool *) option->clo_variable_dest) = 1;
             break;
         default:
             break;
@@ -1277,7 +1216,6 @@ static int set_dest(ompi_cmd_line_option_t *option, char *sval)
     return OPAL_SUCCESS;
 }
 
-
 /*
  * Helper function to qsort_callback
  */
@@ -1303,13 +1241,12 @@ static void fill(const ompi_cmd_line_option_t *a, char result[3][BUFSIZ])
     }
 }
 
-
 static int qsort_callback(const void *aa, const void *bb)
 {
     int ret, i;
     char str1[3][BUFSIZ], str2[3][BUFSIZ];
-    const ompi_cmd_line_option_t *a = *((const ompi_cmd_line_option_t**) aa);
-    const ompi_cmd_line_option_t *b = *((const ompi_cmd_line_option_t**) bb);
+    const ompi_cmd_line_option_t *a = *((const ompi_cmd_line_option_t **) aa);
+    const ompi_cmd_line_option_t *b = *((const ompi_cmd_line_option_t **) bb);
 
     /* Icky comparison of command line options.  There are multiple
        forms of each command line option, so we first have to check
@@ -1330,7 +1267,6 @@ static int qsort_callback(const void *aa, const void *bb)
     return 0;
 }
 
-
 /*
  * Helper function to find the option type specified in the help
  * command.
@@ -1345,12 +1281,12 @@ static opal_cmd_line_otype_t get_help_otype(opal_cmd_line_t *cmd)
     arg = opal_cmd_line_get_param(cmd, "help", 0, 0);
 
     /* If not "help", check for "h" */
-    if(NULL == arg) {
+    if (NULL == arg) {
         arg = opal_cmd_line_get_param(cmd, "h", 0, 0);
     }
 
     /* If arg is still NULL, give them the General info by default */
-    if(NULL == arg) {
+    if (NULL == arg) {
         arg = "general";
     }
 
@@ -1387,21 +1323,24 @@ static opal_cmd_line_otype_t get_help_otype(opal_cmd_line_t *cmd)
  * Helper function to build a parsable string for the help
  * output.
  */
-static char *build_parsable(ompi_cmd_line_option_t *option) {
+static char *build_parsable(ompi_cmd_line_option_t *option)
+{
     char *line;
     int length;
 
-    length = snprintf(NULL, 0, "%c:%s:%s:%d:%s\n", option->clo_short_name, option->clo_single_dash_name,
-                      option->clo_long_name, option->clo_num_params, option->clo_description);
+    length = snprintf(NULL, 0, "%c:%s:%s:%d:%s\n", option->clo_short_name,
+                      option->clo_single_dash_name, option->clo_long_name, option->clo_num_params,
+                      option->clo_description);
 
-    line = (char *)malloc(length * sizeof(char));
+    line = (char *) malloc(length * sizeof(char));
 
-    if('\0' == option->clo_short_name) {
-        snprintf(line, length, "0:%s:%s:%d:%s\n", option->clo_single_dash_name, option->clo_long_name,
-                 option->clo_num_params, option->clo_description);
-    } else {
-        snprintf(line, length, "%c:%s:%s:%d:%s\n", option->clo_short_name, option->clo_single_dash_name,
+    if ('\0' == option->clo_short_name) {
+        snprintf(line, length, "0:%s:%s:%d:%s\n", option->clo_single_dash_name,
                  option->clo_long_name, option->clo_num_params, option->clo_description);
+    } else {
+        snprintf(line, length, "%c:%s:%s:%d:%s\n", option->clo_short_name,
+                 option->clo_single_dash_name, option->clo_long_name, option->clo_num_params,
+                 option->clo_description);
     }
 
     return line;
diff --git a/opal/util/cmd_line.h b/opal/util/cmd_line.h
index ce92259cec3..c883d2e6f88 100644
--- a/opal/util/cmd_line.h
+++ b/opal/util/cmd_line.h
@@ -10,7 +10,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2012      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2022 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2015-2016 Intel, Inc. All rights reserved.
  * Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights
  *                         reserved.
@@ -118,506 +118,497 @@
 
 #include "opal_config.h"
 
-#include "opal/class/opal_object.h"
 #include "opal/class/opal_list.h"
+#include "opal/class/opal_object.h"
 #include "opal/mca/threads/mutex.h"
 
 BEGIN_C_DECLS
-    /**
-     * \internal
-     *
-     * Main top-level handle.  This interface should not be used by users!
-     */
-    struct opal_cmd_line_t {
-        /** Make this an OBJ handle */
-        opal_object_t super;
-
-        /** Thread safety */
-        opal_recursive_mutex_t lcl_mutex;
-
-        /** List of ompi_cmd_line_option_t's (defined internally) */
-        opal_list_t lcl_options;
-
-        /** Duplicate of argc from opal_cmd_line_parse() */
-        int lcl_argc;
-        /** Duplicate of argv from opal_cmd_line_parse() */
-        char **lcl_argv;
-
-        /** Parsed output; list of ompi_cmd_line_param_t's (defined internally) */
-        opal_list_t lcl_params;
-
-        /** List of tail (unprocessed) arguments */
-        int lcl_tail_argc;
-        /** List of tail (unprocessed) arguments */
-        char **lcl_tail_argv;
-    };
-    /**
-     * \internal
-     *
-     * Convenience typedef
-     */
-    typedef struct opal_cmd_line_t opal_cmd_line_t;
-
-    /**
-     * Data types supported by the parser
-     */
-    enum opal_cmd_line_type_t {
-        OPAL_CMD_LINE_TYPE_NULL,
-        OPAL_CMD_LINE_TYPE_STRING,
-        OPAL_CMD_LINE_TYPE_INT,
-        OPAL_CMD_LINE_TYPE_SIZE_T,
-        OPAL_CMD_LINE_TYPE_BOOL,
-
-        OPAL_CMD_LINE_TYPE_MAX
-    };
-    /**
-     * \internal
-     *
-     * Convenience typedef
-     */
-    typedef enum opal_cmd_line_type_t opal_cmd_line_type_t;
-
-    /**
-     * Command line option type, for use in
-     * mpirun --help output.
-     */
-    enum opal_cmd_line_otype_t {
-        OPAL_CMD_LINE_OTYPE_GENERAL,
-        OPAL_CMD_LINE_OTYPE_DEBUG,
-        OPAL_CMD_LINE_OTYPE_OUTPUT,
-        OPAL_CMD_LINE_OTYPE_INPUT,
-        OPAL_CMD_LINE_OTYPE_MAPPING,
-        OPAL_CMD_LINE_OTYPE_RANKING,
-        OPAL_CMD_LINE_OTYPE_BINDING,
-        OPAL_CMD_LINE_OTYPE_DEVEL,
-        OPAL_CMD_LINE_OTYPE_COMPAT, /* Backwards compatibility */
-        OPAL_CMD_LINE_OTYPE_LAUNCH,
-        OPAL_CMD_LINE_OTYPE_DVM,
-        OPAL_CMD_LINE_OTYPE_UNSUPPORTED,
-        OPAL_CMD_LINE_OTYPE_PARSABLE,
-        OPAL_CMD_LINE_OTYPE_NULL /* include in full help only */
-    };
-    /**
-     * \internal
-     *
-     * Convenience typedef
-     */
-    typedef enum opal_cmd_line_otype_t opal_cmd_line_otype_t;
-
-    /**
-     * Datatype used to construct a command line handle; see
-     * opal_cmd_line_create().
-     */
-    struct opal_cmd_line_init_t {
-        /** If want to set an MCA parameter, set its parameter name
-            here. */
-        const char *ocl_mca_param_name;
-
-        /** "Short" name (i.e., "-X", where "X" is a single letter) */
-        char ocl_cmd_short_name;
-        /** "Single dash" name (i.e., "-foo").  The use of these are
-            discouraged. */
-        const char *ocl_cmd_single_dash_name;
-        /** Long name (i.e., "--foo"). */
-        const char *ocl_cmd_long_name;
-
-        /** Number of parameters that this option takes */
-        int ocl_num_params;
-
-        /** If this parameter is encountered, its *first* parameter it
-            saved here.  If the parameter is encountered again, the
-            value is overwritten. */
-        void *ocl_variable_dest;
-        /** If an ocl_variable_dest is given, its datatype must be
-            supplied as well. */
-        opal_cmd_line_type_t ocl_variable_type;
-
-        /** Description of the command line option, to be used with
-            opal_cmd_line_get_usage_msg(). */
-        const char *ocl_description;
-
-        /** Category for mpirun --help output */
-        opal_cmd_line_otype_t ocl_otype;
-    };
-    /**
-     * \internal
-     *
-     * Convenience typedef
-     */
-    typedef struct opal_cmd_line_init_t opal_cmd_line_init_t;
-
-    /**
-     * Top-level command line handle.
-     *
-     * This handle is used for accessing all command line functionality
-     * (i.e., all opal_cmd_line*() functions).  Multiple handles can be
-     * created and simultaneously processed; each handle is independant
-     * from others.
-     *
-     * The opal_cmd_line_t handles are [simplisticly] thread safe;
-     * processing is guaranteed to be mutually exclusive if multiple
-     * threads invoke functions on the same handle at the same time --
-     * access will be serialized in an unspecified order.
-     *
-     * Once finished, handles should be released with OBJ_RELEASE().  The
-     * destructor for opal_cmd_line_t handles will free all memory
-     * associated with the handle.
-     */
-    OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_cmd_line_t);
-
-    /**
-     * Make a command line handle from a table of initializers.
-     *
-     * @param cmd OPAL command line handle.
-     * @param table Table of opal_cmd_line_init_t instances for all
-     * the options to be included in the resulting command line
-     * handler.
-     *
-     * @retval OPAL_SUCCESS Upon success.
-     *
-     * This function takes a table of opal_cmd_line_init_t instances
-     * to pre-seed an OPAL command line handle.  The last instance in
-     * the table must have '\0' for the short name and NULL for the
-     * single-dash and long names.  The handle is expected to have
-     * been OBJ_NEW'ed or OBJ_CONSTRUCT'ed already.
-     *
-     * Upon return, the command line handle is just like any other.  A
-     * sample using this syntax:
-     *
-     * \code
-     * opal_cmd_line_init_t cmd_line_init[] = {
-     *    { NULL, 'h', NULL, "help", 0,
-     *      &orterun_globals.help, OPAL_CMD_LINE_TYPE_BOOL,
-     *      "This help message", OPAL_CMD_LINE_OTYPE_GENERAL },
-     *
-     *    { NULL, '\0', NULL, "wd", 1,
-     *      &orterun_globals.wd, OPAL_CMD_LINE_TYPE_STRING,
-     *      "Set the working directory of the started processes",
-     *      OPAL_CMD_LINE_OTYPE_LAUNCH },
-     *
-     *    { NULL, '\0', NULL, NULL, 0,
-     *      NULL, OPAL_CMD_LINE_TYPE_NULL, NULL, OPAL_CMD_LINE_OTYPE_NULL }
-     * };
-     * \endcode
-     */
-    OPAL_DECLSPEC int opal_cmd_line_create(opal_cmd_line_t *cmd,
-                                           opal_cmd_line_init_t *table);
-
-    /* Add a table of opal_cmd_line_init_t instances
-     * to an existing OPAL command line handle.
-     *
-     * Multiple calls to opal_cmd_line_add are permitted - each
-     * subsequent call will simply append new options to the existing
-     * handle. Note that any duplicates will return an error.
-     */
-     OPAL_DECLSPEC int opal_cmd_line_add(opal_cmd_line_t *cmd,
-                                         opal_cmd_line_init_t *table);
-
-    /**
-     * Create a command line option.
-     *
-     * @param cmd OPAL command line handle.
-     * @param entry Command line entry to add to the command line.
-     *
-     * @retval OPAL_SUCCESS Upon success.
-     *
-     */
-    OPAL_DECLSPEC int opal_cmd_line_make_opt_mca(opal_cmd_line_t *cmd,
-                                                 opal_cmd_line_init_t entry);
-
-    /**
-     * Create a command line option.
-     *
-     * @param cmd OPAL command line handle.
-     * @param short_name "Short" name of the command line option.
-     * @param sd_name "Single dash" name of the command line option.
-     * @param long_name "Long" name of the command line option.
-     * @param num_params How many parameters this option takes.
-     * @param dest Short string description of this option.
-     *
-     * @retval OPAL_ERR_OUT_OF_RESOURCE If out of memory.
-     * @retval OPAL_ERR_BAD_PARAM If bad parameters passed.
-     * @retval OPAL_SUCCESS Upon success.
-     *
-     * Adds a command line option to the list of options that an OPAL
-     * command line handle will accept.  The short_name may take the
-     * special value '\0' to not have a short name.  Likewise, the
-     * sd_name and long_name may take the special value NULL to not have
-     * a single dash or long name, respectively.  However, one of the
-     * three must have a name.
-     *
-     * num_params indicates how many parameters this option takes.  It
-     * must be greater than or equal to 0.
-     *
-     * Finally, desc is a short string description of this option.  It is
-     * used to generate the output from opal_cmd_line_get_usage_msg().
-     *
-     */
-    OPAL_DECLSPEC int opal_cmd_line_make_opt3(opal_cmd_line_t *cmd,
-                                              char short_name,
-                                              const char *sd_name,
-                                              const char *long_name,
-                                              int num_params,
-                                              const char *desc);
-
-    /**
-     * Parse a command line according to a pre-built OPAL command line
-     * handle.
-     *
-     * @param cmd OPAL command line handle.
-     * @param ignore_unknown Whether to print an error message upon
-     * finding an unknown token or not
-     * @param ignore_unknown_option Whether to print an error message upon
-     * finding an unknown option or not
-     * @param argc Length of the argv array.
-     * @param argv Array of strings from the command line.
-     *
-     * @retval OPAL_SUCCESS Upon success.
-     * @retval OPAL_ERR_SILENT If an error message was printed.  This
-     * value will only be returned if the command line was not
-     * successfully parsed.
-     *
-     * Parse a series of command line tokens according to the option
-     * descriptions from a OPAL command line handle.  The OPAL command line
-     * handle can then be queried to see what options were used, what
-     * their parameters were, etc.
-     *
-     * If an unknown token is found in the command line (i.e., a token
-     * that is not a parameter or a registered option), the parsing will
-     * stop (see below).  If ignore_unknown is false, an error message
-     * is displayed.  If ignore_unknown is true, the error message is
-     * not displayed.
-     *
-     * Error messages are always displayed regardless of the value
-     * of ignore_unknown (to stderr, and OPAL_ERR_SILENT is
-     * returned) if:
-     *
-     * 1. A token was encountered that required N parameters, but <N
-     * parameters were found (e.g., "cmd --param foo", but --param was
-     * registered to require 2 option tokens).
-     *
-     * 2. An unknown token beginning with "-" is encountered.  For
-     * example, if "--fo" is specified, and no "fo" option is
-     * registered (e.g., perhaps the user meant to type "--foo"), an
-     * error message is always printed, UNLESS this unknown token
-     * happens after a "--" token (see below).
-     *
-     * The contents of argc and argv are not changed during parsing.
-     * argv[0] is assumed to be the executable name, and is ignored during
-     * parsing, except when printing error messages.
-     *
-     * Parsing will stop in the following conditions:
-     *
-     * - all argv tokens are processed
-     * - the token "--" is found
-     * - an unrecognized token is found
-     * - a parameter registered with an integer type option finds a
-     *   non-integer option token
-     * - a parameted registered N option tokens, but finds less then
-     *   <N tokens available
-     *
-     * Upon any of these conditions, any remaining tokens will be placed
-     * in the "tail" (and therefore not examined by the parser),
-     * regardless of the value of ignore_unknown.  The set of tail
-     * tokens is available from the opal_cmd_line_get_tail() function.
-     *
-     * Note that "--" is ignored if it is found in the middle an expected
-     * number of arguments.  For example, if "--foo" is expected to have 3
-     * arguments, and the command line is:
-     *
-     * executable --foo a b -- other arguments
-     *
-     * This will result in an error, because "--" will be parsed as the
-     * third parameter to the first instance of "foo", and "other" will be
-     * an unrecognized option.
-     *
-     * Note that -- can be used to allow unknown tokens that begin
-     * with "-".  For example, if a user wants to mpirun an executable
-     * named "-my-mpi-program", the "usual" way:
-     *
-     *   mpirun -my-mpi-program
-     *
-     * will cause an error, because mpirun won't find single-letter
-     * options registered for some/all of those letters.  But two
-     * workarounds are possible:
-     *
-     *   mpirun -- -my-mpi-program
-     * or
-     *   mpirun ./-my-mpi-program
-     *
-     * Finally, note that invoking this function multiple times on
-     * different sets of argv tokens is safe, but will erase any
-     * previous parsing results.
-     */
-    OPAL_DECLSPEC int opal_cmd_line_parse(opal_cmd_line_t *cmd,
-                                          bool ignore_unknown,
-                                          bool ignore_unknown_option,
-                                          int argc, char **argv);
-
-    /**
-     * Return a consolidated "usage" message for a OPAL command line handle.
-     *
-     * @param cmd OPAL command line handle.
-     *
-     * @retval str Usage message.
-     *
-     * Returns a formatted string suitable for printing that lists the
-     * expected usage message and a short description of each option on
-     * the OPAL command line handle.  Options that passed a NULL
-     * description to opal_cmd_line_make_opt3() will not be included in the
-     * display (to allow for undocumented options).
-     *
-     * This function is typically only invoked internally by the
-     * opal_show_help() function.
-     *
-     * This function should probably be fixed up to produce prettier
-     * output.
-     *
-     * The returned string must be freed by the caller.
-     */
-    OPAL_DECLSPEC char *opal_cmd_line_get_usage_msg(opal_cmd_line_t *cmd) __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
-
-    /**
-     * Test if a given option was taken on the parsed command line.
-     *
-     * @param cmd OPAL command line handle.
-     * @param opt Short or long name of the option to check for.
-     *
-     * @retval true If the command line option was found during
-     * opal_cmd_line_parse().
-     *
-     * @retval false If the command line option was not found during
-     * opal_cmd_line_parse(), or opal_cmd_line_parse() was not invoked on
-     * this handle.
-     *
-     * This function should only be called after opal_cmd_line_parse().
-     *
-     * The function will return true if the option matching opt was found
-     * (either by its short or long name) during token parsing.
-     * Otherwise, it will return false.
-     */
-    OPAL_DECLSPEC bool opal_cmd_line_is_taken(opal_cmd_line_t *cmd,
-                                              const char *opt) __opal_attribute_nonnull__(1) __opal_attribute_nonnull__(2);
-
-    /**
-     * Return the number of arguments parsed on a OPAL command line handle.
-     *
-     * @param cmd A pointer to the OPAL command line handle.
-     *
-     * @retval OPAL_ERROR If cmd is NULL.
-     * @retval argc Number of arguments previously added to the handle.
-     *
-     * Arguments are added to the handle via the opal_cmd_line_parse()
-     * function.
-     */
-    OPAL_DECLSPEC int opal_cmd_line_get_argc(opal_cmd_line_t *cmd) __opal_attribute_unused__;
-
-    /**
-     * Return a string argument parsed on a OPAL command line handle.
-     *
-     * @param cmd A pointer to the OPAL command line handle.
-     * @param index The nth argument from the command line (0 is
-     * argv[0], etc.).
-     *
-     * @retval NULL If cmd is NULL or index is invalid
-     * @retval argument String of original argv[index]
-     *
-     * This function returns a single token from the arguments parsed
-     * on this handle.  Arguments are added bia the
-     * opal_cmd_line_parse() function.
-     *
-     * What is returned is a pointer to the actual string that is on
-     * the handle; it should not be modified or freed.
-     */
-    OPAL_DECLSPEC char *opal_cmd_line_get_argv(opal_cmd_line_t *cmd,
-                                               int index);
-
-    /**
-     * Return the number of instances of an option found during parsing.
-     *
-     * @param cmd OPAL command line handle.
-     * @param opt Short or long name of the option to check for.
-     *
-     * @retval num Number of instances (to include 0) of a given potion
-     * found during opal_cmd_line_parse().
-     *
-     * @retval OPAL_ERR If the command line option was not found during
-     * opal_cmd_line_parse(), or opal_cmd_line_parse() was not invoked on
-     * this handle.
-     *
-     * This function should only be called after opal_cmd_line_parse().
-     *
-     * The function will return the number of instances of a given option
-     * (either by its short or long name) -- to include 0 -- or OPAL_ERR if
-     * either the option was not specified as part of the OPAL command line
-     * handle, or opal_cmd_line_parse() was not invoked on this handle.
-     */
-    OPAL_DECLSPEC int opal_cmd_line_get_ninsts(opal_cmd_line_t *cmd,
-                                               const char *opt) __opal_attribute_nonnull__(1) __opal_attribute_nonnull__(2);
-
-    /**
-     * Return a specific parameter for a specific instance of a option
-     * from the parsed command line.
-     *
-     * @param cmd OPAL command line handle.
-     * @param opt Short or long name of the option to check for.
-     * @param instance_num Instance number of the option to query.
-     * @param param_num Which parameter to return.
-     *
-     * @retval param String of the parameter.
-     * @retval NULL If any of the input values are invalid.
-     *
-     * This function should only be called after opal_cmd_line_parse().
-     *
-     * This function returns the Nth parameter for the Ith instance of a
-     * given option on the parsed command line (both N and I are
-     * zero-indexed).  For example, on the command line:
-     *
-     * executable --foo bar1 bar2 --foo bar3 bar4
-     *
-     * The call to opal_cmd_line_get_param(cmd, "foo", 1, 1) would return
-     * "bar4".  opal_cmd_line_get_param(cmd, "bar", 0, 0) would return
-     * NULL, as would opal_cmd_line_get_param(cmd, "foo", 2, 2);
-     *
-     * The returned string should \em not be modified or freed by the
-     * caller.
-     */
-    OPAL_DECLSPEC char *opal_cmd_line_get_param(opal_cmd_line_t *cmd,
-                                                const char *opt,
-                                                int instance_num,
-                                                int param_num);
-
-    /**
-     * Return the entire "tail" of unprocessed argv from a OPAL
-     * command line handle.
-     *
-     * @param cmd A pointer to the OPAL command line handle.
-     * @param tailc Pointer to the output length of the null-terminated
-     * tail argv array.
-     * @param tailv Pointer to the output null-terminated argv of all
-     * unprocessed arguments from the command line.
-     *
-     * @retval OPAL_ERROR If cmd is NULL or otherwise invalid.
-     * @retval OPAL_SUCCESS Upon success.
-     *
-     * The "tail" is all the arguments on the command line that were
-     * not processed for some reason.  Reasons for not processing
-     * arguments include:
-     *
-     * \sa The argument was not recognized
-     * \sa The argument "--" was seen, and therefore all arguments
-     * following it were not processed
-     *
-     * The output tailc parameter will be filled in with the integer
-     * length of the null-terminated tailv array (length including the
-     * final NULL entry).  The output tailv parameter will be a copy
-     * of the tail parameters, and must be freed (likely with a call
-     * to opal_argv_free()) by the caller.
-     */
-    OPAL_DECLSPEC int opal_cmd_line_get_tail(opal_cmd_line_t *cmd, int *tailc,
-                                             char ***tailv);
+/**
+ * \internal
+ *
+ * Main top-level handle.  This interface should not be used by users!
+ */
+struct opal_cmd_line_t {
+    /** Make this an OBJ handle */
+    opal_object_t super;
 
-END_C_DECLS
+    /** Thread safety */
+    opal_recursive_mutex_t lcl_mutex;
+
+    /** List of ompi_cmd_line_option_t's (defined internally) */
+    opal_list_t lcl_options;
+
+    /** Duplicate of argc from opal_cmd_line_parse() */
+    int lcl_argc;
+    /** Duplicate of argv from opal_cmd_line_parse() */
+    char **lcl_argv;
+
+    /** Parsed output; list of ompi_cmd_line_param_t's (defined internally) */
+    opal_list_t lcl_params;
+
+    /** List of tail (unprocessed) arguments */
+    int lcl_tail_argc;
+    /** List of tail (unprocessed) arguments */
+    char **lcl_tail_argv;
+};
+/**
+ * \internal
+ *
+ * Convenience typedef
+ */
+typedef struct opal_cmd_line_t opal_cmd_line_t;
+
+/**
+ * Data types supported by the parser
+ */
+enum opal_cmd_line_type_t {
+    OPAL_CMD_LINE_TYPE_NULL,
+    OPAL_CMD_LINE_TYPE_STRING,
+    OPAL_CMD_LINE_TYPE_INT,
+    OPAL_CMD_LINE_TYPE_SIZE_T,
+    OPAL_CMD_LINE_TYPE_BOOL,
+
+    OPAL_CMD_LINE_TYPE_MAX
+};
+/**
+ * \internal
+ *
+ * Convenience typedef
+ */
+typedef enum opal_cmd_line_type_t opal_cmd_line_type_t;
+
+/**
+ * Command line option type, for use in
+ * mpirun --help output.
+ */
+enum opal_cmd_line_otype_t {
+    OPAL_CMD_LINE_OTYPE_GENERAL, /* This type is shown via --help by
+                                    default (i.e., if no arg is
+                                    specified to narrow the help
+                                    scope) */
+    OPAL_CMD_LINE_OTYPE_DEBUG,
+    OPAL_CMD_LINE_OTYPE_OUTPUT,
+    OPAL_CMD_LINE_OTYPE_INPUT,
+    OPAL_CMD_LINE_OTYPE_MAPPING,
+    OPAL_CMD_LINE_OTYPE_RANKING,
+    OPAL_CMD_LINE_OTYPE_BINDING,
+    OPAL_CMD_LINE_OTYPE_DEVEL,
+    OPAL_CMD_LINE_OTYPE_COMPAT, /* Backwards compatibility */
+    OPAL_CMD_LINE_OTYPE_LAUNCH,
+    OPAL_CMD_LINE_OTYPE_DVM,
+    OPAL_CMD_LINE_OTYPE_UNSUPPORTED,
+    OPAL_CMD_LINE_OTYPE_PARSABLE,
+    OPAL_CMD_LINE_OTYPE_NULL /* include in full help only */
+};
+/**
+ * \internal
+ *
+ * Convenience typedef
+ */
+typedef enum opal_cmd_line_otype_t opal_cmd_line_otype_t;
+
+/**
+ * Datatype used to construct a command line handle; see
+ * opal_cmd_line_create().
+ */
+struct opal_cmd_line_init_t {
+    /** If want to set an MCA parameter, set its parameter name
+        here. */
+    const char *ocl_mca_param_name;
+
+    /** "Short" name (i.e., "-X", where "X" is a single letter) */
+    char ocl_cmd_short_name;
+    /** "Single dash" name (i.e., "-foo").  The use of these are
+        discouraged. */
+    const char *ocl_cmd_single_dash_name;
+    /** Long name (i.e., "--foo"). */
+    const char *ocl_cmd_long_name;
+
+    /** Number of parameters that this option takes */
+    int ocl_num_params;
+
+    /** If this parameter is encountered, its *first* parameter it
+        saved here.  If the parameter is encountered again, the
+        value is overwritten. */
+    void *ocl_variable_dest;
+    /** If an ocl_variable_dest is given, its datatype must be
+        supplied as well. */
+    opal_cmd_line_type_t ocl_variable_type;
+
+    /** Description of the command line option, to be used with
+        opal_cmd_line_get_usage_msg(). */
+    const char *ocl_description;
+
+    /** Category for mpirun --help output */
+    opal_cmd_line_otype_t ocl_otype;
+};
+/**
+ * \internal
+ *
+ * Convenience typedef
+ */
+typedef struct opal_cmd_line_init_t opal_cmd_line_init_t;
+
+/**
+ * Top-level command line handle.
+ *
+ * This handle is used for accessing all command line functionality
+ * (i.e., all opal_cmd_line*() functions).  Multiple handles can be
+ * created and simultaneously processed; each handle is independent
+ * from others.
+ *
+ * The opal_cmd_line_t handles are [simplisticly] thread safe;
+ * processing is guaranteed to be mutually exclusive if multiple
+ * threads invoke functions on the same handle at the same time --
+ * access will be serialized in an unspecified order.
+ *
+ * Once finished, handles should be released with OBJ_RELEASE().  The
+ * destructor for opal_cmd_line_t handles will free all memory
+ * associated with the handle.
+ */
+OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_cmd_line_t);
+
+/**
+ * Make a command line handle from a table of initializers.
+ *
+ * @param cmd OPAL command line handle.
+ * @param table Table of opal_cmd_line_init_t instances for all
+ * the options to be included in the resulting command line
+ * handler.
+ *
+ * @retval OPAL_SUCCESS Upon success.
+ *
+ * This function takes a table of opal_cmd_line_init_t instances
+ * to pre-seed an OPAL command line handle.  The last instance in
+ * the table must have '\0' for the short name and NULL for the
+ * single-dash and long names.  The handle is expected to have
+ * been OBJ_NEW'ed or OBJ_CONSTRUCT'ed already.
+ *
+ * Upon return, the command line handle is just like any other.  A
+ * sample using this syntax:
+ *
+ * \code
+ * opal_cmd_line_init_t cmd_line_init[] = {
+ *    { NULL, 'h', NULL, "help", 0,
+ *      &orterun_globals.help, OPAL_CMD_LINE_TYPE_BOOL,
+ *      "This help message", OPAL_CMD_LINE_OTYPE_GENERAL },
+ *
+ *    { NULL, '\0', NULL, "wd", 1,
+ *      &orterun_globals.wd, OPAL_CMD_LINE_TYPE_STRING,
+ *      "Set the working directory of the started processes",
+ *      OPAL_CMD_LINE_OTYPE_LAUNCH },
+ *
+ *    { NULL, '\0', NULL, NULL, 0,
+ *      NULL, OPAL_CMD_LINE_TYPE_NULL, NULL, OPAL_CMD_LINE_OTYPE_NULL }
+ * };
+ * \endcode
+ */
+OPAL_DECLSPEC int opal_cmd_line_create(opal_cmd_line_t *cmd, opal_cmd_line_init_t *table);
+
+/* Add a table of opal_cmd_line_init_t instances
+ * to an existing OPAL command line handle.
+ *
+ * Multiple calls to opal_cmd_line_add are permitted - each
+ * subsequent call will simply append new options to the existing
+ * handle. Note that any duplicates will return an error.
+ */
+OPAL_DECLSPEC int opal_cmd_line_add(opal_cmd_line_t *cmd, opal_cmd_line_init_t *table);
+
+/**
+ * Create a command line option.
+ *
+ * @param cmd OPAL command line handle.
+ * @param entry Command line entry to add to the command line.
+ *
+ * @retval OPAL_SUCCESS Upon success.
+ *
+ */
+OPAL_DECLSPEC int opal_cmd_line_make_opt_mca(opal_cmd_line_t *cmd, opal_cmd_line_init_t entry);
+
+/**
+ * Create a command line option.
+ *
+ * @param cmd OPAL command line handle.
+ * @param short_name "Short" name of the command line option.
+ * @param sd_name "Single dash" name of the command line option.
+ * @param long_name "Long" name of the command line option.
+ * @param num_params How many parameters this option takes.
+ * @param dest Short string description of this option.
+ *
+ * @retval OPAL_ERR_OUT_OF_RESOURCE If out of memory.
+ * @retval OPAL_ERR_BAD_PARAM If bad parameters passed.
+ * @retval OPAL_SUCCESS Upon success.
+ *
+ * Adds a command line option to the list of options that an OPAL
+ * command line handle will accept.  The short_name may take the
+ * special value '\0' to not have a short name.  Likewise, the
+ * sd_name and long_name may take the special value NULL to not have
+ * a single dash or long name, respectively.  However, one of the
+ * three must have a name.
+ *
+ * num_params indicates how many parameters this option takes.  It
+ * must be greater than or equal to 0.
+ *
+ * Finally, desc is a short string description of this option.  It is
+ * used to generate the output from opal_cmd_line_get_usage_msg().
+ *
+ */
+OPAL_DECLSPEC int opal_cmd_line_make_opt3(opal_cmd_line_t *cmd, char short_name,
+                                          const char *sd_name, const char *long_name,
+                                          int num_params, const char *desc);
+
+/**
+ * Parse a command line according to a pre-built OPAL command line
+ * handle.
+ *
+ * @param cmd OPAL command line handle.
+ * @param ignore_unknown Whether to print an error message upon
+ * finding an unknown token or not
+ * @param ignore_unknown_option Whether to print an error message upon
+ * finding an unknown option or not
+ * @param argc Length of the argv array.
+ * @param argv Array of strings from the command line.
+ *
+ * @retval OPAL_SUCCESS Upon success.
+ * @retval OPAL_ERR_SILENT If an error message was printed.  This
+ * value will only be returned if the command line was not
+ * successfully parsed.
+ *
+ * Parse a series of command line tokens according to the option
+ * descriptions from a OPAL command line handle.  The OPAL command line
+ * handle can then be queried to see what options were used, what
+ * their parameters were, etc.
+ *
+ * If an unknown token is found in the command line (i.e., a token
+ * that is not a parameter or a registered option), the parsing will
+ * stop (see below).  If ignore_unknown is false, an error message
+ * is displayed.  If ignore_unknown is true, the error message is
+ * not displayed.
+ *
+ * Error messages are always displayed regardless of the value
+ * of ignore_unknown (to stderr, and OPAL_ERR_SILENT is
+ * returned) if:
+ *
+ * 1. A token was encountered that required N parameters, but <N
+ * parameters were found (e.g., "cmd --param foo", but --param was
+ * registered to require 2 option tokens).
+ *
+ * 2. An unknown token beginning with "-" is encountered.  For
+ * example, if "--fo" is specified, and no "fo" option is
+ * registered (e.g., perhaps the user meant to type "--foo"), an
+ * error message is always printed, UNLESS this unknown token
+ * happens after a "--" token (see below).
+ *
+ * The contents of argc and argv are not changed during parsing.
+ * argv[0] is assumed to be the executable name, and is ignored during
+ * parsing, except when printing error messages.
+ *
+ * Parsing will stop in the following conditions:
+ *
+ * - all argv tokens are processed
+ * - the token "--" is found
+ * - an unrecognized token is found
+ * - a parameter registered with an integer type option finds a
+ *   non-integer option token
+ * - a parameted registered N option tokens, but finds less then
+ *   <N tokens available
+ *
+ * Upon any of these conditions, any remaining tokens will be placed
+ * in the "tail" (and therefore not examined by the parser),
+ * regardless of the value of ignore_unknown.  The set of tail
+ * tokens is available from the opal_cmd_line_get_tail() function.
+ *
+ * Note that "--" is ignored if it is found in the middle an expected
+ * number of arguments.  For example, if "--foo" is expected to have 3
+ * arguments, and the command line is:
+ *
+ * executable --foo a b -- other arguments
+ *
+ * This will result in an error, because "--" will be parsed as the
+ * third parameter to the first instance of "foo", and "other" will be
+ * an unrecognized option.
+ *
+ * Note that -- can be used to allow unknown tokens that begin
+ * with "-".  For example, if a user wants to mpirun an executable
+ * named "-my-mpi-program", the "usual" way:
+ *
+ *   mpirun -my-mpi-program
+ *
+ * will cause an error, because mpirun won't find single-letter
+ * options registered for some/all of those letters.  But two
+ * workarounds are possible:
+ *
+ *   mpirun -- -my-mpi-program
+ * or
+ *   mpirun ./-my-mpi-program
+ *
+ * Finally, note that invoking this function multiple times on
+ * different sets of argv tokens is safe, but will erase any
+ * previous parsing results.
+ */
+OPAL_DECLSPEC int opal_cmd_line_parse(opal_cmd_line_t *cmd, bool ignore_unknown,
+                                      bool ignore_unknown_option, int argc, char **argv);
 
+/**
+ * Return a consolidated "usage" message for a OPAL command line handle.
+ *
+ * @param cmd OPAL command line handle.
+ *
+ * @retval str Usage message.
+ *
+ * Returns a formatted string suitable for printing that lists the
+ * expected usage message and a short description of each option on
+ * the OPAL command line handle.  Options that passed a NULL
+ * description to opal_cmd_line_make_opt3() will not be included in the
+ * display (to allow for undocumented options).
+ *
+ * This function is typically only invoked internally by the
+ * opal_show_help() function.
+ *
+ * This function should probably be fixed up to produce prettier
+ * output.
+ *
+ * The returned string must be freed by the caller.
+ */
+OPAL_DECLSPEC char *opal_cmd_line_get_usage_msg(opal_cmd_line_t *cmd)
+    __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
+
+/**
+ * Test if a given option was taken on the parsed command line.
+ *
+ * @param cmd OPAL command line handle.
+ * @param opt Short or long name of the option to check for.
+ *
+ * @retval true If the command line option was found during
+ * opal_cmd_line_parse().
+ *
+ * @retval false If the command line option was not found during
+ * opal_cmd_line_parse(), or opal_cmd_line_parse() was not invoked on
+ * this handle.
+ *
+ * This function should only be called after opal_cmd_line_parse().
+ *
+ * The function will return true if the option matching opt was found
+ * (either by its short or long name) during token parsing.
+ * Otherwise, it will return false.
+ */
+OPAL_DECLSPEC bool opal_cmd_line_is_taken(opal_cmd_line_t *cmd, const char *opt)
+    __opal_attribute_nonnull__(1) __opal_attribute_nonnull__(2);
+
+/**
+ * Return the number of arguments parsed on a OPAL command line handle.
+ *
+ * @param cmd A pointer to the OPAL command line handle.
+ *
+ * @retval OPAL_ERROR If cmd is NULL.
+ * @retval argc Number of arguments previously added to the handle.
+ *
+ * Arguments are added to the handle via the opal_cmd_line_parse()
+ * function.
+ */
+OPAL_DECLSPEC int opal_cmd_line_get_argc(opal_cmd_line_t *cmd) __opal_attribute_unused__;
+
+/**
+ * Return a string argument parsed on a OPAL command line handle.
+ *
+ * @param cmd A pointer to the OPAL command line handle.
+ * @param index The nth argument from the command line (0 is
+ * argv[0], etc.).
+ *
+ * @retval NULL If cmd is NULL or index is invalid
+ * @retval argument String of original argv[index]
+ *
+ * This function returns a single token from the arguments parsed
+ * on this handle.  Arguments are added bia the
+ * opal_cmd_line_parse() function.
+ *
+ * What is returned is a pointer to the actual string that is on
+ * the handle; it should not be modified or freed.
+ */
+OPAL_DECLSPEC char *opal_cmd_line_get_argv(opal_cmd_line_t *cmd, int index);
+
+/**
+ * Return the number of instances of an option found during parsing.
+ *
+ * @param cmd OPAL command line handle.
+ * @param opt Short or long name of the option to check for.
+ *
+ * @retval num Number of instances (to include 0) of a given potion
+ * found during opal_cmd_line_parse().
+ *
+ * @retval OPAL_ERR If the command line option was not found during
+ * opal_cmd_line_parse(), or opal_cmd_line_parse() was not invoked on
+ * this handle.
+ *
+ * This function should only be called after opal_cmd_line_parse().
+ *
+ * The function will return the number of instances of a given option
+ * (either by its short or long name) -- to include 0 -- or OPAL_ERR if
+ * either the option was not specified as part of the OPAL command line
+ * handle, or opal_cmd_line_parse() was not invoked on this handle.
+ */
+OPAL_DECLSPEC int opal_cmd_line_get_ninsts(opal_cmd_line_t *cmd, const char *opt)
+    __opal_attribute_nonnull__(1) __opal_attribute_nonnull__(2);
+
+/**
+ * Return a specific parameter for a specific instance of a option
+ * from the parsed command line.
+ *
+ * @param cmd OPAL command line handle.
+ * @param opt Short or long name of the option to check for.
+ * @param instance_num Instance number of the option to query.
+ * @param param_num Which parameter to return.
+ *
+ * @retval param String of the parameter.
+ * @retval NULL If any of the input values are invalid.
+ *
+ * This function should only be called after opal_cmd_line_parse().
+ *
+ * This function returns the Nth parameter for the Ith instance of a
+ * given option on the parsed command line (both N and I are
+ * zero-indexed).  For example, on the command line:
+ *
+ * executable --foo bar1 bar2 --foo bar3 bar4
+ *
+ * The call to opal_cmd_line_get_param(cmd, "foo", 1, 1) would return
+ * "bar4".  opal_cmd_line_get_param(cmd, "bar", 0, 0) would return
+ * NULL, as would opal_cmd_line_get_param(cmd, "foo", 2, 2);
+ *
+ * The returned string should \em not be modified or freed by the
+ * caller.
+ */
+OPAL_DECLSPEC char *opal_cmd_line_get_param(opal_cmd_line_t *cmd, const char *opt, int instance_num,
+                                            int param_num);
+
+/**
+ * Return the entire "tail" of unprocessed argv from a OPAL
+ * command line handle.
+ *
+ * @param cmd A pointer to the OPAL command line handle.
+ * @param tailc Pointer to the output length of the null-terminated
+ * tail argv array.
+ * @param tailv Pointer to the output null-terminated argv of all
+ * unprocessed arguments from the command line.
+ *
+ * @retval OPAL_ERROR If cmd is NULL or otherwise invalid.
+ * @retval OPAL_SUCCESS Upon success.
+ *
+ * The "tail" is all the arguments on the command line that were
+ * not processed for some reason.  Reasons for not processing
+ * arguments include:
+ *
+ * \sa The argument was not recognized
+ * \sa The argument "--" was seen, and therefore all arguments
+ * following it were not processed
+ *
+ * The output tailc parameter will be filled in with the integer
+ * length of the null-terminated tailv array (length including the
+ * final NULL entry).  The output tailv parameter will be a copy
+ * of the tail parameters, and must be freed (likely with a call
+ * to opal_argv_free()) by the caller.
+ */
+OPAL_DECLSPEC int opal_cmd_line_get_tail(opal_cmd_line_t *cmd, int *tailc, char ***tailv);
+
+END_C_DECLS
 
 #endif /* OPAL_CMD_LINE_H */
diff --git a/opal/util/convert-help-files-to-c-code.py b/opal/util/convert-help-files-to-c-code.py
new file mode 100755
index 00000000000..d8814d9fb1b
--- /dev/null
+++ b/opal/util/convert-help-files-to-c-code.py
@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+import os
+import sys
+import argparse
+
+def find_help_files(root, verbose=False):
+    # Search for help-*.txt files across the source tree, skipping
+    # some directories (e.g., 3rd-party)
+    help_files = []
+    skip_dirs = ['.git', '3rd-party']
+    for root_dir, dirs, files in os.walk(root):
+        for sd in skip_dirs:
+            if sd in dirs:
+                dirs.remove(sd)
+
+        for file in files:
+            if file.startswith("help-") and file.endswith(".txt"):
+                full_path = os.path.join(root_dir, file)
+                help_files.append(full_path)
+                if verbose:
+                    print(f"Found: {full_path}")
+    return help_files
+
+def parse_ini_files(file_paths, verbose=False):
+    # Parse INI-style files, returning a dictionary with filenames as
+    # keys.  Don't use the Python configparse module in order to
+    # reduce dependencies (i.e., so that we don't have to pip install
+    # anything to run this script).
+    data = {}
+    for file_path in file_paths:
+        sections = {}
+        current_section = None
+        with open(file_path) as file:
+            for line in file:
+                line = line.rstrip()
+                if line.startswith('#'):
+                    continue
+                if line.startswith('[') and line.endswith(']'):
+                    current_section = line[1:-1]
+                    sections[current_section] = list()
+                elif current_section is not None:
+                    sections[current_section].append(line)
+
+        data[os.path.basename(file_path)] = sections
+
+        if verbose:
+            print(f"Parsed: {file_path} ({len(sections)} sections found)")
+
+    return data
+
+def generate_c_code(parsed_data):
+    # Generate C code with an array of filenames and their
+    # corresponding INI sections.
+    c_code = []
+    c_code.append(f"""// THIS FILE IS GENERATED AUTOMATICALLY! EDITS WILL BE LOST!
+// This file generated by {sys.argv[0]}
+
+""")
+    # Rather than escaping the C code {} in f strings, make this a
+    # separate (non-f-string) addition to c_code.
+    c_code.append("""#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+#include "opal/util/argv.h"
+#include "opal/util/show_help.h"
+
+typedef struct {
+    const char *section;
+    const char **content;
+    char *joined;
+} ini_entry;
+
+typedef struct {
+    const char *filename;
+    ini_entry *entries;
+} file_entry;
+
+static file_entry help_files[] = {""")
+
+    ini_arrays = []
+    file_entries = []
+
+    sp4 = f'{" " * 4}'
+    sp8 = f'{" " * 8}'
+    sp12 = f'{" " * 12}'
+    sp16 = f'{" " * 16}'
+    sp20 = f'{" " * 20}'
+
+    for filename, sections in parsed_data.items():
+        c_code.append(f'{sp4}{{')
+        c_code.append(f'{sp8}.filename = "{filename}",')
+        c_code.append(f'{sp8}.entries = (ini_entry[]) {{')
+
+        for section_name, lines in sections.items():
+            c_code.append(f'{sp12}{{')
+            c_code.append(f'{sp16}.section = "{section_name}",')
+            c_code.append(f'{sp16}.content = (const char*[]) {{')
+            for line in lines:
+                c_string = line.replace('"','\\"').replace("\n", '\\n"\n"')
+                c_code.append(f'{sp20}"{c_string}",')
+            c_code.append(f'{sp20}NULL')
+            c_code.append(f'{sp16}}},')
+            c_code.append(f'{sp12}}}, // End of section {section_name}')
+
+        c_code.append(f'{sp12}{{0}}')
+        c_code.append(f'{sp8}}}, // End of file {filename}')
+        c_code.append(f'{sp4}}},')
+
+    c_code.append(f'{sp4}{{0}}')
+    c_code.append(f'}};')
+
+    c_code.append("""
+const char *opal_show_help_get_content(const char *filename, const char* topic)
+{
+    const file_entry *fe;
+    ini_entry *ie;
+
+    for (int i = 0; help_files[i].filename != NULL; ++i) {
+        fe = &(help_files[i]);
+        if (strcmp(fe->filename, filename) == 0) {
+            for (int j = 0; fe->entries[j].section != NULL; ++j) {
+                ie = &(fe->entries[j]);
+                if (strcmp(ie->section, topic) == 0) {
+                    if (NULL == ie->joined) {
+                        ie->joined = opal_argv_join((char**)ie->content, '\\n');
+                    }
+                    return ie->joined;
+                }
+            }
+        }
+    }
+
+    return NULL;
+}
+
+void opal_show_help_content_free(void)
+{
+    for (int i = 0; help_files[i].filename != NULL; ++i) {
+        for (int j = 0; help_files[i].entries[j].section != NULL; j++) {
+            if (NULL != help_files[i].entries[j].joined) {
+                free(help_files[i].entries[j].joined);
+                help_files[i].entries[j].joined = NULL;
+            }
+        }
+    }
+}
+""")
+
+    return '\n'.join(c_code)
+
+#-------------------------------
+
+def main():
+    parser = argparse.ArgumentParser(description="Generate C code from help text INI files.")
+    parser.add_argument("--root",
+                        required=True,
+                        help="Root directory to search for help-*.txt files")
+    parser.add_argument("--out",
+                        required=True,
+                        help="Output C file")
+    parser.add_argument("--verbose",
+                        action="store_true",
+                        help="Enable verbose output")
+    args = parser.parse_args()
+
+    if args.verbose:
+        print(f"Searching in: {args.root}")
+
+    file_paths = find_help_files(args.root, args.verbose)
+    parsed_data = parse_ini_files(file_paths, args.verbose)
+    c_code = generate_c_code(parsed_data)
+
+    if os.path.exists(args.out):
+        with open(args.out) as f:
+            existing_content = f.read()
+
+            if existing_content == c_code:
+                if args.verbose:
+                    print(f"Help string content has not changed; not re-writing {args.out}")
+                exit(0)
+
+    with open(args.out, "w") as f:
+        f.write(c_code)
+
+    if args.verbose:
+        print(f"Generated C code written to {args.out}")
+
+if __name__ == "__main__":
+    main()
diff --git a/opal/util/crc.c b/opal/util/crc.c
index d74d434fe89..f762c6c9e7f 100644
--- a/opal/util/crc.c
+++ b/opal/util/crc.c
@@ -17,37 +17,31 @@
  * $HEADER$
  */
 
-
 #include "opal_config.h"
 #ifdef HAVE_STDIO_H
-#include <stdio.h>
+#    include <stdio.h>
 #endif /* HAVE_STDIO_H */
 #include <stdlib.h>
 #ifdef HAVE_STRINGS_H
-#include <strings.h>
-#endif  /* HAVE_STRINGS_H */
+#    include <strings.h>
+#endif /* HAVE_STRINGS_H */
 #include <string.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif  /* HAVE_UNISTD_H */
+#    include <unistd.h>
+#endif /* HAVE_UNISTD_H */
 #include "opal/util/crc.h"
 
-
 #if (OPAL_ALIGNMENT_LONG == 8)
-#define OPAL_CRC_WORD_MASK_ 0x7
+#    define OPAL_CRC_WORD_MASK_ 0x7
 #elif (OPAL_ALIGNMENT_LONG == 4)
-#define OPAL_CRC_WORD_MASK_ 0x3
+#    define OPAL_CRC_WORD_MASK_ 0x3
 #else
-#define OPAL_CRC_WORD_MASK_ 0xFFFF
+#    define OPAL_CRC_WORD_MASK_ 0xFFFF
 #endif
 
+#define WORDALIGNED(v) (((intptr_t) v & OPAL_CRC_WORD_MASK_) ? false : true)
 
-#define WORDALIGNED(v) \
-    (((intptr_t)v & OPAL_CRC_WORD_MASK_) ? false : true)
-
-
-#define INTALIGNED(v) \
-    (((intptr_t)v & 3) ? false : true)
+#define INTALIGNED(v) (((intptr_t) v & 3) ? false : true)
 
 /*
  * this version of bcopy_csum() looks a little too long, but it
@@ -59,18 +53,12 @@
  * of bcopy_csum() - Mitch
  */
 
-unsigned long
-opal_bcopy_csum_partial (
-    const void *  source,
-    void *  destination,
-    size_t copylen,
-    size_t csumlen,
-    unsigned long *  lastPartialLong,
-    size_t*  lastPartialLength
-    )
+unsigned long opal_bcopy_csum_partial(const void *source, void *destination, size_t copylen,
+                                      size_t csumlen, unsigned long *lastPartialLong,
+                                      size_t *lastPartialLength)
 {
-    unsigned long *  src = (unsigned long *) source;
-    unsigned long *  dest = (unsigned long *) destination;
+    unsigned long *src = (unsigned long *) source;
+    unsigned long *dest = (unsigned long *) destination;
     unsigned long csum = 0;
     size_t csumlenresidue;
     unsigned long i, temp;
@@ -79,345 +67,325 @@ opal_bcopy_csum_partial (
     temp = *lastPartialLong;
 
     if (WORDALIGNED(source) && WORDALIGNED(dest)) {
-	if (*lastPartialLength) {
-	    /* do we have enough data to fill out the partial word? */
-	    if (copylen >= (sizeof(unsigned long) - *lastPartialLength)) { /* YES, we do... */
-		memcpy(((char *)&temp + *lastPartialLength), src,
-		       (sizeof(unsigned long) - *lastPartialLength));
-		memcpy(dest, ((char *)&temp + *lastPartialLength),
-		       (sizeof(unsigned long) - *lastPartialLength));
-		src = (unsigned long *)((char *)src + sizeof(unsigned long) - *lastPartialLength);
-		dest = (unsigned long *)((char *)dest + sizeof(unsigned long) - *lastPartialLength);
-		csum += (temp - *lastPartialLong);
-		copylen -= sizeof(unsigned long) - *lastPartialLength;
-		/* now we have an unaligned source and an unaligned destination */
-		for( ;copylen >= sizeof(*src); copylen -= sizeof(*src)) {
-		    memcpy(&temp, src, sizeof(temp));
-		    src++;
-		    csum += temp;
-		    memcpy(dest, &temp, sizeof(temp));
-		    dest++;
-		}
-		*lastPartialLength = 0;
-		*lastPartialLong = 0;
-	    }
-	    else { /* NO, we don't... */
-		memcpy(((char *)&temp + *lastPartialLength), src, copylen);
-		memcpy(dest, ((char *)&temp + *lastPartialLength), copylen);
-		src = (unsigned long *)((char *)src + copylen);
-		dest = (unsigned long *)((char *)dest + copylen);
-		csum += (temp - *lastPartialLong);
-		*lastPartialLong = temp;
-		*lastPartialLength += copylen;
-		copylen = 0;
-	    }
-	}
-	else { /* fast path... */
-	    size_t numLongs = copylen/sizeof(unsigned long);
-	    for(i = 0; i < numLongs; i++) {
-		    csum += *src;
-		    *dest++ = *src++;
-	    }
-	    *lastPartialLong = 0;
-	    *lastPartialLength = 0;
-	    if (WORDALIGNED(copylen) && (csumlenresidue == 0)) {
-		    return(csum);
-	    }
-	    else {
-		    copylen -= i * sizeof(unsigned long);
-	    }
-	}
+        if (*lastPartialLength) {
+            /* do we have enough data to fill out the partial word? */
+            if (copylen >= (sizeof(unsigned long) - *lastPartialLength)) { /* YES, we do... */
+                memcpy(((char *) &temp + *lastPartialLength), src,
+                       (sizeof(unsigned long) - *lastPartialLength));
+                memcpy(dest, ((char *) &temp + *lastPartialLength),
+                       (sizeof(unsigned long) - *lastPartialLength));
+                src = (unsigned long *) ((char *) src + sizeof(unsigned long) - *lastPartialLength);
+                dest = (unsigned long *) ((char *) dest + sizeof(unsigned long)
+                                          - *lastPartialLength);
+                csum += (temp - *lastPartialLong);
+                copylen -= sizeof(unsigned long) - *lastPartialLength;
+                /* now we have an unaligned source and an unaligned destination */
+                for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) {
+                    memcpy(&temp, src, sizeof(temp));
+                    src++;
+                    csum += temp;
+                    memcpy(dest, &temp, sizeof(temp));
+                    dest++;
+                }
+                *lastPartialLength = 0;
+                *lastPartialLong = 0;
+            } else { /* NO, we don't... */
+                memcpy(((char *) &temp + *lastPartialLength), src, copylen);
+                memcpy(dest, ((char *) &temp + *lastPartialLength), copylen);
+                src = (unsigned long *) ((char *) src + copylen);
+                dest = (unsigned long *) ((char *) dest + copylen);
+                csum += (temp - *lastPartialLong);
+                *lastPartialLong = temp;
+                *lastPartialLength += copylen;
+                copylen = 0;
+            }
+        } else { /* fast path... */
+            size_t numLongs = copylen / sizeof(unsigned long);
+            for (i = 0; i < numLongs; i++) {
+                csum += *src;
+                *dest++ = *src++;
+            }
+            *lastPartialLong = 0;
+            *lastPartialLength = 0;
+            if (WORDALIGNED(copylen) && (csumlenresidue == 0)) {
+                return (csum);
+            } else {
+                copylen -= i * sizeof(unsigned long);
+            }
+        }
     } else if (WORDALIGNED(source)) {
-	if (*lastPartialLength) {
-	    /* do we have enough data to fill out the partial word? */
-	    if (copylen >= (sizeof(unsigned long) - *lastPartialLength)) { /* YES, we do... */
-		memcpy(((char *)&temp + *lastPartialLength), src,
-		       (sizeof(unsigned long) - *lastPartialLength));
-		memcpy(dest, ((char *)&temp + *lastPartialLength),
-		       (sizeof(unsigned long) - *lastPartialLength));
-		src = (unsigned long *)((char *)src + sizeof(unsigned long) - *lastPartialLength);
-		dest = (unsigned long *)((char *)dest + sizeof(unsigned long) - *lastPartialLength);
-		csum += (temp - *lastPartialLong);
-		copylen -= sizeof(unsigned long) - *lastPartialLength;
-		/* now we have an unaligned source and an unknown alignment for our destination */
-		if (WORDALIGNED(dest)) {
-		    size_t numLongs = copylen/sizeof(unsigned long);
-		    for(i = 0; i < numLongs; i++) {
-			    memcpy(&temp, src, sizeof(temp));
-			    src++;
-			    csum += temp;
-			    *dest++ = temp;
-		    }
-		    copylen -= i * sizeof(unsigned long);
-		}
-		else {
-		    for( ;copylen >= sizeof(*src); copylen -= sizeof(*src)) {
-			    memcpy(&temp, src, sizeof(temp));
-			    src++;
-			    csum += temp;
-			    memcpy(dest, &temp, sizeof(temp));
-			    dest++;
-		    }
-		}
-		*lastPartialLong = 0;
-		*lastPartialLength = 0;
-	    }
-	    else { /* NO, we don't... */
-		    memcpy(((char *)&temp + *lastPartialLength), src, copylen);
-		    memcpy(dest, ((char *)&temp + *lastPartialLength), copylen);
-		    src = (unsigned long *)((char *)src + copylen);
-		    dest = (unsigned long *)((char *)dest + copylen);
-		    csum += (temp - *lastPartialLong);
-		    *lastPartialLong = temp;
-		    *lastPartialLength += copylen;
-		    copylen = 0;
-	    }
-	}
-	else {
-	    for( ;copylen >= sizeof(*src); copylen -= sizeof(*src)) {
-		temp = *src++;
-		csum += temp;
-		memcpy(dest, &temp, sizeof(temp));
-		dest++;
-	    }
-	    *lastPartialLong = 0;
-	    *lastPartialLength = 0;
-	}
+        if (*lastPartialLength) {
+            /* do we have enough data to fill out the partial word? */
+            if (copylen >= (sizeof(unsigned long) - *lastPartialLength)) { /* YES, we do... */
+                memcpy(((char *) &temp + *lastPartialLength), src,
+                       (sizeof(unsigned long) - *lastPartialLength));
+                memcpy(dest, ((char *) &temp + *lastPartialLength),
+                       (sizeof(unsigned long) - *lastPartialLength));
+                src = (unsigned long *) ((char *) src + sizeof(unsigned long) - *lastPartialLength);
+                dest = (unsigned long *) ((char *) dest + sizeof(unsigned long)
+                                          - *lastPartialLength);
+                csum += (temp - *lastPartialLong);
+                copylen -= sizeof(unsigned long) - *lastPartialLength;
+                /* now we have an unaligned source and an unknown alignment for our destination */
+                if (WORDALIGNED(dest)) {
+                    size_t numLongs = copylen / sizeof(unsigned long);
+                    for (i = 0; i < numLongs; i++) {
+                        memcpy(&temp, src, sizeof(temp));
+                        src++;
+                        csum += temp;
+                        *dest++ = temp;
+                    }
+                    copylen -= i * sizeof(unsigned long);
+                } else {
+                    for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) {
+                        memcpy(&temp, src, sizeof(temp));
+                        src++;
+                        csum += temp;
+                        memcpy(dest, &temp, sizeof(temp));
+                        dest++;
+                    }
+                }
+                *lastPartialLong = 0;
+                *lastPartialLength = 0;
+            } else { /* NO, we don't... */
+                memcpy(((char *) &temp + *lastPartialLength), src, copylen);
+                memcpy(dest, ((char *) &temp + *lastPartialLength), copylen);
+                src = (unsigned long *) ((char *) src + copylen);
+                dest = (unsigned long *) ((char *) dest + copylen);
+                csum += (temp - *lastPartialLong);
+                *lastPartialLong = temp;
+                *lastPartialLength += copylen;
+                copylen = 0;
+            }
+        } else {
+            for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) {
+                temp = *src++;
+                csum += temp;
+                memcpy(dest, &temp, sizeof(temp));
+                dest++;
+            }
+            *lastPartialLong = 0;
+            *lastPartialLength = 0;
+        }
     } else if (WORDALIGNED(dest)) {
-	if (*lastPartialLength) {
-	    /* do we have enough data to fill out the partial word? */
-	    if (copylen >= (sizeof(unsigned long) - *lastPartialLength)) { /* YES, we do... */
-		memcpy(((char *)&temp + *lastPartialLength), src,
-		       (sizeof(unsigned long) - *lastPartialLength));
-		memcpy(dest, ((char *)&temp + *lastPartialLength),
-		       (sizeof(unsigned long) - *lastPartialLength));
-		src = (unsigned long *)((char *)src + sizeof(unsigned long) - *lastPartialLength);
-		dest = (unsigned long *)((char *)dest + sizeof(unsigned long) - *lastPartialLength);
-		csum += (temp - *lastPartialLong);
-		copylen -= sizeof(unsigned long) - *lastPartialLength;
-		/* now we have a source of unknown alignment and a unaligned destination */
-		if (WORDALIGNED(src)) {
-		    for( ;copylen >= sizeof(*src); copylen -= sizeof(*src)) {
-			temp = *src++;
-			csum += temp;
-			memcpy(dest, &temp, sizeof(temp));
-			dest++;
-		    }
-		    *lastPartialLong = 0;
-		    *lastPartialLength = 0;
-		}
-		else {
-		    for( ;copylen >= sizeof(*src); copylen -= sizeof(*src)) {
-			memcpy(&temp, src, sizeof(temp));
-			src++;
-			csum += temp;
-			memcpy(dest, &temp, sizeof(temp));
-			dest++;
-		    }
-		    *lastPartialLength = 0;
-		    *lastPartialLong = 0;
-		}
-	    }
-	    else { /* NO, we don't... */
-		memcpy(((char *)&temp + *lastPartialLength), src, copylen);
-		memcpy(dest, ((char *)&temp + *lastPartialLength), copylen);
-		src = (unsigned long *)((char *)src + copylen);
-		dest = (unsigned long *)((char *)dest + copylen);
-		csum += (temp - *lastPartialLong);
-		*lastPartialLong = temp;
-		*lastPartialLength += copylen;
-		copylen = 0;
-	    }
-	}
-	else {
-	    for( ;copylen >= sizeof(*src); copylen -= sizeof(*src)) {
-		memcpy(&temp, src, sizeof(temp));
-		src++;
-		csum += temp;
-		*dest++ = temp;
-	    }
-	    *lastPartialLength = 0;
-	    *lastPartialLong = 0;
-	}
+        if (*lastPartialLength) {
+            /* do we have enough data to fill out the partial word? */
+            if (copylen >= (sizeof(unsigned long) - *lastPartialLength)) { /* YES, we do... */
+                memcpy(((char *) &temp + *lastPartialLength), src,
+                       (sizeof(unsigned long) - *lastPartialLength));
+                memcpy(dest, ((char *) &temp + *lastPartialLength),
+                       (sizeof(unsigned long) - *lastPartialLength));
+                src = (unsigned long *) ((char *) src + sizeof(unsigned long) - *lastPartialLength);
+                dest = (unsigned long *) ((char *) dest + sizeof(unsigned long)
+                                          - *lastPartialLength);
+                csum += (temp - *lastPartialLong);
+                copylen -= sizeof(unsigned long) - *lastPartialLength;
+                /* now we have a source of unknown alignment and a unaligned destination */
+                if (WORDALIGNED(src)) {
+                    for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) {
+                        temp = *src++;
+                        csum += temp;
+                        memcpy(dest, &temp, sizeof(temp));
+                        dest++;
+                    }
+                    *lastPartialLong = 0;
+                    *lastPartialLength = 0;
+                } else {
+                    for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) {
+                        memcpy(&temp, src, sizeof(temp));
+                        src++;
+                        csum += temp;
+                        memcpy(dest, &temp, sizeof(temp));
+                        dest++;
+                    }
+                    *lastPartialLength = 0;
+                    *lastPartialLong = 0;
+                }
+            } else { /* NO, we don't... */
+                memcpy(((char *) &temp + *lastPartialLength), src, copylen);
+                memcpy(dest, ((char *) &temp + *lastPartialLength), copylen);
+                src = (unsigned long *) ((char *) src + copylen);
+                dest = (unsigned long *) ((char *) dest + copylen);
+                csum += (temp - *lastPartialLong);
+                *lastPartialLong = temp;
+                *lastPartialLength += copylen;
+                copylen = 0;
+            }
+        } else {
+            for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) {
+                memcpy(&temp, src, sizeof(temp));
+                src++;
+                csum += temp;
+                *dest++ = temp;
+            }
+            *lastPartialLength = 0;
+            *lastPartialLong = 0;
+        }
     } else {
-	if (*lastPartialLength) {
-	    /* do we have enough data to fill out the partial word? */
-	    if (copylen >= (sizeof(unsigned long) - *lastPartialLength)) { /* YES, we do... */
-		memcpy(((char *)&temp + *lastPartialLength), src,
-		       (sizeof(unsigned long) - *lastPartialLength));
-		memcpy(dest, ((char *)&temp + *lastPartialLength),
-		       (sizeof(unsigned long) - *lastPartialLength));
-		src = (unsigned long *)((char *)src + sizeof(unsigned long) - *lastPartialLength);
-		dest = (unsigned long *)((char *)dest + sizeof(unsigned long) - *lastPartialLength);
-		csum += (temp - *lastPartialLong);
-		copylen -= sizeof(unsigned long) - *lastPartialLength;
-		/* now we have an unknown alignment for our source and destination */
-		if (WORDALIGNED(src) && WORDALIGNED(dest)) {
-		    size_t numLongs = copylen/sizeof(unsigned long);
-		    for(i = 0; i < numLongs; i++) {
-			    csum += *src;
-			    *dest++ = *src++;
-		    }
-		    copylen -= i * sizeof(unsigned long);
-		}
-		else { /* safe but slower for all other alignments */
-		    for( ;copylen >= sizeof(*src); copylen -= sizeof(*src)) {
-			    memcpy(&temp, src, sizeof(temp));
-			    src++;
-			    csum += temp;
-			    memcpy(dest, &temp, sizeof(temp));
-			    dest++;
-		    }
-		}
-		*lastPartialLong = 0;
-		*lastPartialLength = 0;
-	    }
-	    else { /* NO, we don't... */
-		memcpy(((char *)&temp + *lastPartialLength), src, copylen);
-		memcpy(dest, ((char *)&temp + *lastPartialLength), copylen);
-		src = (unsigned long *)((char *)src + copylen);
-		dest = (unsigned long *)((char *)dest + copylen);
-		csum += (temp - *lastPartialLong);
-		*lastPartialLong = temp;
-		*lastPartialLength += copylen;
-		copylen = 0;
-	    }
-	}
-	else {
-	    for( ;copylen >= sizeof(*src); copylen -= sizeof(*src)) {
-		memcpy(&temp, src, sizeof(temp));
-		src++;
-		csum += temp;
-		memcpy(dest, &temp, sizeof(temp));
-		dest++;
-	    }
-	    *lastPartialLength = 0;
-	    *lastPartialLong = 0;
-	}
+        if (*lastPartialLength) {
+            /* do we have enough data to fill out the partial word? */
+            if (copylen >= (sizeof(unsigned long) - *lastPartialLength)) { /* YES, we do... */
+                memcpy(((char *) &temp + *lastPartialLength), src,
+                       (sizeof(unsigned long) - *lastPartialLength));
+                memcpy(dest, ((char *) &temp + *lastPartialLength),
+                       (sizeof(unsigned long) - *lastPartialLength));
+                src = (unsigned long *) ((char *) src + sizeof(unsigned long) - *lastPartialLength);
+                dest = (unsigned long *) ((char *) dest + sizeof(unsigned long)
+                                          - *lastPartialLength);
+                csum += (temp - *lastPartialLong);
+                copylen -= sizeof(unsigned long) - *lastPartialLength;
+                /* now we have an unknown alignment for our source and destination */
+                if (WORDALIGNED(src) && WORDALIGNED(dest)) {
+                    size_t numLongs = copylen / sizeof(unsigned long);
+                    for (i = 0; i < numLongs; i++) {
+                        csum += *src;
+                        *dest++ = *src++;
+                    }
+                    copylen -= i * sizeof(unsigned long);
+                } else { /* safe but slower for all other alignments */
+                    for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) {
+                        memcpy(&temp, src, sizeof(temp));
+                        src++;
+                        csum += temp;
+                        memcpy(dest, &temp, sizeof(temp));
+                        dest++;
+                    }
+                }
+                *lastPartialLong = 0;
+                *lastPartialLength = 0;
+            } else { /* NO, we don't... */
+                memcpy(((char *) &temp + *lastPartialLength), src, copylen);
+                memcpy(dest, ((char *) &temp + *lastPartialLength), copylen);
+                src = (unsigned long *) ((char *) src + copylen);
+                dest = (unsigned long *) ((char *) dest + copylen);
+                csum += (temp - *lastPartialLong);
+                *lastPartialLong = temp;
+                *lastPartialLength += copylen;
+                copylen = 0;
+            }
+        } else {
+            for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) {
+                memcpy(&temp, src, sizeof(temp));
+                src++;
+                csum += temp;
+                memcpy(dest, &temp, sizeof(temp));
+                dest++;
+            }
+            *lastPartialLength = 0;
+            *lastPartialLong = 0;
+        }
     }
 
     /* if copylen is non-zero there was a bit left, less than an unsigned long's worth */
     if ((copylen != 0) && (csumlenresidue == 0)) {
-	temp = *lastPartialLong;
-	if (*lastPartialLength) {
-	    if (copylen >= (sizeof(unsigned long) - *lastPartialLength)) {
-		/* copy all remaining bytes from src to dest */
-		unsigned long copytemp = 0;
-		memcpy(&copytemp, src, copylen);
-		memcpy(dest, &copytemp, copylen);
-		/* fill out rest of partial word and add to checksum */
-		memcpy(((char *)&temp + *lastPartialLength), src,
-		       (sizeof(unsigned long) - *lastPartialLength));
-		/* avoid unsigned arithmetic overflow by subtracting the old partial
-		 * word from the new one before adding to the checksum...
-        */
-		csum += (temp - *lastPartialLong);
-		copylen -= sizeof(unsigned long) - *lastPartialLength;
-		src = (unsigned long *)((char *)src + sizeof(unsigned long) - *lastPartialLength);
-		*lastPartialLength = copylen;
-		/* reset temp, and calculate next partial word */
-		temp = 0;
-		if (copylen) {
-		    memcpy(&temp, src, copylen);
-		}
-		/* add it to the the checksum */
-		csum += temp;
-		*lastPartialLong = temp;
-	    }
-	    else {
-		/* copy all remaining bytes from src to dest */
-		unsigned long copytemp = 0;
-		memcpy(&copytemp, src, copylen);
-		memcpy(dest, &copytemp, copylen);
-		/* fill out rest of partial word and add to checksum */
-		memcpy(((char *)&temp + *lastPartialLength), src,
-		       copylen);
-		/* avoid unsigned arithmetic overflow by subtracting the old partial
-		 * word from the new one before adding to the checksum...
-        */
-		csum += temp - *lastPartialLong;
-		*lastPartialLong = temp;
-		*lastPartialLength += copylen;
-	    }
-	}
-	else { /* fast path... */
-	    /* temp and *lastPartialLong are 0 if *lastPartialLength is 0... */
-	    memcpy(&temp, src, copylen);
-	    csum += temp;
-	    memcpy(dest, &temp, copylen);
-	    *lastPartialLong = temp;
-	    *lastPartialLength = copylen;
-	    /* done...return the checksum */
-	}
-    }
-    else if (csumlenresidue != 0) {
-	if (copylen != 0) {
-	    temp = 0;
-	    memcpy(&temp, src, copylen);
-	    memcpy(dest, &temp, copylen);
-	}
-	if (csumlenresidue < (sizeof(unsigned long) - copylen - *lastPartialLength)) {
-	    temp = *lastPartialLong;
-	    memcpy(((char *)&temp + *lastPartialLength), src, (copylen + csumlenresidue));
-	    /* avoid unsigned arithmetic overflow by subtracting the old partial */
-	    /* word from the new one before adding to the checksum... */
-	    csum += temp - *lastPartialLong;
-	    src++;
-	    *lastPartialLong = temp;
-	    *lastPartialLength += copylen + csumlenresidue;
-	    csumlenresidue = 0;
-	}
-	else {
-	    /* we have enough chksum data to fill out our last partial */
-	    /* word */
-	    temp = *lastPartialLong;
-	    memcpy(((char *)&temp + *lastPartialLength), src,
-		   (sizeof(unsigned long) - *lastPartialLength));
-	    /* avoid unsigned arithmetic overflow by subtracting the old partial */
-	    /* word from the new one before adding to the checksum... */
-	    csum += temp - *lastPartialLong;
-	    src = (unsigned long *)((char *)src + sizeof(unsigned long) - *lastPartialLength);
-	    csumlenresidue -= sizeof(unsigned long) - *lastPartialLength - copylen;
-	    *lastPartialLength = 0;
-	    *lastPartialLong = 0;
-	}
-	if (WORDALIGNED(src)) {
-	    for (i = 0; i < csumlenresidue/sizeof(unsigned long); i++) {
-		csum += *src++;
-	    }
-	}
-	else {
-	    for (i = 0; i < csumlenresidue/sizeof(unsigned long); i++) {
-		memcpy(&temp, src, sizeof(temp));
-		csum += temp;
-		src++;
-	    }
-	}
-	csumlenresidue -= i * sizeof(unsigned long);
-	if (csumlenresidue) {
-	    temp = 0;
-	    memcpy(&temp, src, csumlenresidue);
-	    csum += temp;
-	    *lastPartialLong = temp;
-	    *lastPartialLength = csumlenresidue;
-	}
+        temp = *lastPartialLong;
+        if (*lastPartialLength) {
+            if (copylen >= (sizeof(unsigned long) - *lastPartialLength)) {
+                /* copy all remaining bytes from src to dest */
+                unsigned long copytemp = 0;
+                memcpy(&copytemp, src, copylen);
+                memcpy(dest, &copytemp, copylen);
+                /* fill out rest of partial word and add to checksum */
+                memcpy(((char *) &temp + *lastPartialLength), src,
+                       (sizeof(unsigned long) - *lastPartialLength));
+                /* avoid unsigned arithmetic overflow by subtracting the old partial
+                 * word from the new one before adding to the checksum...
+                 */
+                csum += (temp - *lastPartialLong);
+                copylen -= sizeof(unsigned long) - *lastPartialLength;
+                src = (unsigned long *) ((char *) src + sizeof(unsigned long) - *lastPartialLength);
+                *lastPartialLength = copylen;
+                /* reset temp, and calculate next partial word */
+                temp = 0;
+                if (copylen) {
+                    memcpy(&temp, src, copylen);
+                }
+                /* add it to the the checksum */
+                csum += temp;
+                *lastPartialLong = temp;
+            } else {
+                /* copy all remaining bytes from src to dest */
+                unsigned long copytemp = 0;
+                memcpy(&copytemp, src, copylen);
+                memcpy(dest, &copytemp, copylen);
+                /* fill out rest of partial word and add to checksum */
+                memcpy(((char *) &temp + *lastPartialLength), src, copylen);
+                /* avoid unsigned arithmetic overflow by subtracting the old partial
+                 * word from the new one before adding to the checksum...
+                 */
+                csum += temp - *lastPartialLong;
+                *lastPartialLong = temp;
+                *lastPartialLength += copylen;
+            }
+        } else { /* fast path... */
+            /* temp and *lastPartialLong are 0 if *lastPartialLength is 0... */
+            memcpy(&temp, src, copylen);
+            csum += temp;
+            memcpy(dest, &temp, copylen);
+            *lastPartialLong = temp;
+            *lastPartialLength = copylen;
+            /* done...return the checksum */
+        }
+    } else if (csumlenresidue != 0) {
+        if (copylen != 0) {
+            temp = 0;
+            memcpy(&temp, src, copylen);
+            memcpy(dest, &temp, copylen);
+        }
+        if (csumlenresidue < (sizeof(unsigned long) - copylen - *lastPartialLength)) {
+            temp = *lastPartialLong;
+            memcpy(((char *) &temp + *lastPartialLength), src, (copylen + csumlenresidue));
+            /* avoid unsigned arithmetic overflow by subtracting the old partial */
+            /* word from the new one before adding to the checksum... */
+            csum += temp - *lastPartialLong;
+            src++;
+            *lastPartialLong = temp;
+            *lastPartialLength += copylen + csumlenresidue;
+            csumlenresidue = 0;
+        } else {
+            /* we have enough chksum data to fill out our last partial */
+            /* word */
+            temp = *lastPartialLong;
+            memcpy(((char *) &temp + *lastPartialLength), src,
+                   (sizeof(unsigned long) - *lastPartialLength));
+            /* avoid unsigned arithmetic overflow by subtracting the old partial */
+            /* word from the new one before adding to the checksum... */
+            csum += temp - *lastPartialLong;
+            src = (unsigned long *) ((char *) src + sizeof(unsigned long) - *lastPartialLength);
+            csumlenresidue -= sizeof(unsigned long) - *lastPartialLength - copylen;
+            *lastPartialLength = 0;
+            *lastPartialLong = 0;
+        }
+        if (WORDALIGNED(src)) {
+            for (i = 0; i < csumlenresidue / sizeof(unsigned long); i++) {
+                csum += *src++;
+            }
+        } else {
+            for (i = 0; i < csumlenresidue / sizeof(unsigned long); i++) {
+                memcpy(&temp, src, sizeof(temp));
+                csum += temp;
+                src++;
+            }
+        }
+        csumlenresidue -= i * sizeof(unsigned long);
+        if (csumlenresidue) {
+            temp = 0;
+            memcpy(&temp, src, csumlenresidue);
+            csum += temp;
+            *lastPartialLong = temp;
+            *lastPartialLength = csumlenresidue;
+        }
     } /* end else if (csumlenresidue != 0) */
 
     return csum;
 }
 
-unsigned int
-opal_bcopy_uicsum_partial (
-    const void *  source,
-    void *  destination,
-    size_t copylen,
-    size_t csumlen,
-    unsigned int*  lastPartialInt,
-    size_t*  lastPartialLength
-    )
+unsigned int opal_bcopy_uicsum_partial(const void *source, void *destination, size_t copylen,
+                                       size_t csumlen, unsigned int *lastPartialInt,
+                                       size_t *lastPartialLength)
 {
-    unsigned int *  src = (unsigned int *) source;
-    unsigned int *  dest = (unsigned int *) destination;
+    unsigned int *src = (unsigned int *) source;
+    unsigned int *dest = (unsigned int *) destination;
     unsigned int csum = 0;
     size_t csumlenresidue;
     unsigned long i;
@@ -427,637 +395,587 @@ opal_bcopy_uicsum_partial (
     temp = *lastPartialInt;
 
     if (INTALIGNED(source) && INTALIGNED(dest)) {
-	if (*lastPartialLength) {
-	    /* do we have enough data to fill out the partial word? */
-	    if (copylen >= (sizeof(unsigned int) - *lastPartialLength)) { /* YES, we do... */
-		memcpy(((char *)&temp + *lastPartialLength), src,
-		       (sizeof(unsigned int) - *lastPartialLength));
-		memcpy(dest, ((char *)&temp + *lastPartialLength),
-		       (sizeof(unsigned int) - *lastPartialLength));
-		src = (unsigned int *)((char *)src + sizeof(unsigned int) - *lastPartialLength);
-		dest = (unsigned int *)((char *)dest + sizeof(unsigned int) - *lastPartialLength);
-		csum += (temp - *lastPartialInt);
-		copylen -= sizeof(unsigned int) - *lastPartialLength;
-		/* now we have an unaligned source and an unaligned destination */
-		for( ;copylen >= sizeof(*src); copylen -= sizeof(*src)) {
-		    memcpy(&temp, src, sizeof(temp));
-		    src++;
-		    csum += temp;
-		    memcpy(dest, &temp, sizeof(temp));
-		    dest++;
-		}
-		*lastPartialLength = 0;
-		*lastPartialInt = 0;
-	    }
-	    else { /* NO, we don't... */
-		memcpy(((char *)&temp + *lastPartialLength), src, copylen);
-		memcpy(dest, ((char *)&temp + *lastPartialLength), copylen);
-		src = (unsigned int *)((char *)src + copylen);
-		dest = (unsigned int *)((char *)dest + copylen);
-		csum += (temp - *lastPartialInt);
-		*lastPartialInt = temp;
-		*lastPartialLength += copylen;
-		copylen = 0;
-	    }
-	}
-	else { /* fast path... */
-	    size_t numLongs = copylen/sizeof(unsigned int);
-	    for(i = 0; i < numLongs; i++) {
-		    csum += *src;
-		    *dest++ = *src++;
-	    }
-	    *lastPartialInt = 0;
-	    *lastPartialLength = 0;
-	    if (INTALIGNED(copylen) && (csumlenresidue == 0)) {
-		    return(csum);
-	    }
-	    else {
-		    copylen -= i * sizeof(unsigned int);
-	    }
-	}
+        if (*lastPartialLength) {
+            /* do we have enough data to fill out the partial word? */
+            if (copylen >= (sizeof(unsigned int) - *lastPartialLength)) { /* YES, we do... */
+                memcpy(((char *) &temp + *lastPartialLength), src,
+                       (sizeof(unsigned int) - *lastPartialLength));
+                memcpy(dest, ((char *) &temp + *lastPartialLength),
+                       (sizeof(unsigned int) - *lastPartialLength));
+                src = (unsigned int *) ((char *) src + sizeof(unsigned int) - *lastPartialLength);
+                dest = (unsigned int *) ((char *) dest + sizeof(unsigned int) - *lastPartialLength);
+                csum += (temp - *lastPartialInt);
+                copylen -= sizeof(unsigned int) - *lastPartialLength;
+                /* now we have an unaligned source and an unaligned destination */
+                for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) {
+                    memcpy(&temp, src, sizeof(temp));
+                    src++;
+                    csum += temp;
+                    memcpy(dest, &temp, sizeof(temp));
+                    dest++;
+                }
+                *lastPartialLength = 0;
+                *lastPartialInt = 0;
+            } else { /* NO, we don't... */
+                memcpy(((char *) &temp + *lastPartialLength), src, copylen);
+                memcpy(dest, ((char *) &temp + *lastPartialLength), copylen);
+                src = (unsigned int *) ((char *) src + copylen);
+                dest = (unsigned int *) ((char *) dest + copylen);
+                csum += (temp - *lastPartialInt);
+                *lastPartialInt = temp;
+                *lastPartialLength += copylen;
+                copylen = 0;
+            }
+        } else { /* fast path... */
+            size_t numLongs = copylen / sizeof(unsigned int);
+            for (i = 0; i < numLongs; i++) {
+                csum += *src;
+                *dest++ = *src++;
+            }
+            *lastPartialInt = 0;
+            *lastPartialLength = 0;
+            if (INTALIGNED(copylen) && (csumlenresidue == 0)) {
+                return (csum);
+            } else {
+                copylen -= i * sizeof(unsigned int);
+            }
+        }
     } else if (INTALIGNED(source)) {
-	if (*lastPartialLength) {
-	    /* do we have enough data to fill out the partial word? */
-	    if (copylen >= (sizeof(unsigned int) - *lastPartialLength)) { /* YES, we do... */
-		memcpy(((char *)&temp + *lastPartialLength), src,
-		       (sizeof(unsigned int) - *lastPartialLength));
-		memcpy(dest, ((char *)&temp + *lastPartialLength),
-		       (sizeof(unsigned int) - *lastPartialLength));
-		src = (unsigned int *)((char *)src + sizeof(unsigned int) - *lastPartialLength);
-		dest = (unsigned int *)((char *)dest + sizeof(unsigned int) - *lastPartialLength);
-		csum += (temp - *lastPartialInt);
-		copylen -= sizeof(unsigned int) - *lastPartialLength;
-		/* now we have an unaligned source and an unknown alignment for our destination */
-		if (INTALIGNED(dest)) {
-		    size_t numLongs = copylen/sizeof(unsigned int);
-		    for(i = 0; i < numLongs; i++) {
-			memcpy(&temp, src, sizeof(temp));
-			src++;
-			csum += temp;
-			*dest++ = temp;
-		    }
-		    copylen -= i * sizeof(unsigned int);
-		}
-		else {
-		    for( ;copylen >= sizeof(*src); copylen -= sizeof(*src)) {
-			memcpy(&temp, src, sizeof(temp));
-			src++;
-			csum += temp;
-			memcpy(dest, &temp, sizeof(temp));
-			dest++;
-		    }
-		}
-		*lastPartialInt = 0;
-		*lastPartialLength = 0;
-	    }
-	    else { /* NO, we don't... */
-		memcpy(((char *)&temp + *lastPartialLength), src, copylen);
-		memcpy(dest, ((char *)&temp + *lastPartialLength), copylen);
-		src = (unsigned int *)((char *)src + copylen);
-		dest = (unsigned int *)((char *)dest + copylen);
-		csum += (temp - *lastPartialInt);
-		*lastPartialInt = temp;
-		*lastPartialLength += copylen;
-		copylen = 0;
-	    }
-	}
-	else {
-	    for( ;copylen >= sizeof(*src); copylen -= sizeof(*src)) {
-		temp = *src++;
-		csum += temp;
-		memcpy(dest, &temp, sizeof(temp));
-		dest++;
-	    }
-	    *lastPartialInt = 0;
-	    *lastPartialLength = 0;
-	}
+        if (*lastPartialLength) {
+            /* do we have enough data to fill out the partial word? */
+            if (copylen >= (sizeof(unsigned int) - *lastPartialLength)) { /* YES, we do... */
+                memcpy(((char *) &temp + *lastPartialLength), src,
+                       (sizeof(unsigned int) - *lastPartialLength));
+                memcpy(dest, ((char *) &temp + *lastPartialLength),
+                       (sizeof(unsigned int) - *lastPartialLength));
+                src = (unsigned int *) ((char *) src + sizeof(unsigned int) - *lastPartialLength);
+                dest = (unsigned int *) ((char *) dest + sizeof(unsigned int) - *lastPartialLength);
+                csum += (temp - *lastPartialInt);
+                copylen -= sizeof(unsigned int) - *lastPartialLength;
+                /* now we have an unaligned source and an unknown alignment for our destination */
+                if (INTALIGNED(dest)) {
+                    size_t numLongs = copylen / sizeof(unsigned int);
+                    for (i = 0; i < numLongs; i++) {
+                        memcpy(&temp, src, sizeof(temp));
+                        src++;
+                        csum += temp;
+                        *dest++ = temp;
+                    }
+                    copylen -= i * sizeof(unsigned int);
+                } else {
+                    for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) {
+                        memcpy(&temp, src, sizeof(temp));
+                        src++;
+                        csum += temp;
+                        memcpy(dest, &temp, sizeof(temp));
+                        dest++;
+                    }
+                }
+                *lastPartialInt = 0;
+                *lastPartialLength = 0;
+            } else { /* NO, we don't... */
+                memcpy(((char *) &temp + *lastPartialLength), src, copylen);
+                memcpy(dest, ((char *) &temp + *lastPartialLength), copylen);
+                src = (unsigned int *) ((char *) src + copylen);
+                dest = (unsigned int *) ((char *) dest + copylen);
+                csum += (temp - *lastPartialInt);
+                *lastPartialInt = temp;
+                *lastPartialLength += copylen;
+                copylen = 0;
+            }
+        } else {
+            for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) {
+                temp = *src++;
+                csum += temp;
+                memcpy(dest, &temp, sizeof(temp));
+                dest++;
+            }
+            *lastPartialInt = 0;
+            *lastPartialLength = 0;
+        }
     } else if (INTALIGNED(dest)) {
-	if (*lastPartialLength) {
-	    /* do we have enough data to fill out the partial word? */
-	    if (copylen >= (sizeof(unsigned int) - *lastPartialLength)) { /* YES, we do... */
-		memcpy(((char *)&temp + *lastPartialLength), src,
-		       (sizeof(unsigned int) - *lastPartialLength));
-		memcpy(dest, ((char *)&temp + *lastPartialLength),
-		       (sizeof(unsigned int) - *lastPartialLength));
-		src = (unsigned int *)((char *)src + sizeof(unsigned int) - *lastPartialLength);
-		dest = (unsigned int *)((char *)dest + sizeof(unsigned int) - *lastPartialLength);
-		csum += (temp - *lastPartialInt);
-		copylen -= sizeof(unsigned int) - *lastPartialLength;
-		/* now we have a source of unknown alignment and a unaligned destination */
-		if (INTALIGNED(src)) {
-		    for( ;copylen >= sizeof(*src); copylen -= sizeof(*src)) {
-			temp = *src++;
-			csum += temp;
-			memcpy(dest, &temp, sizeof(temp));
-			dest++;
-		    }
-		    *lastPartialInt = 0;
-		    *lastPartialLength = 0;
-		}
-		else {
-		    for( ;copylen >= sizeof(*src); copylen -= sizeof(*src)) {
-			memcpy(&temp, src, sizeof(temp));
-			src++;
-			csum += temp;
-			memcpy(dest, &temp, sizeof(temp));
-			dest++;
-		    }
-		    *lastPartialLength = 0;
-		    *lastPartialInt = 0;
-		}
-	    }
-	    else { /* NO, we don't... */
-		memcpy(((char *)&temp + *lastPartialLength), src, copylen);
-		memcpy(dest, ((char *)&temp + *lastPartialLength), copylen);
-		src = (unsigned int *)((char *)src + copylen);
-		dest = (unsigned int *)((char *)dest + copylen);
-		csum += (temp - *lastPartialInt);
-		*lastPartialInt = temp;
-		*lastPartialLength += copylen;
-		copylen = 0;
-	    }
-	}
-	else {
-	    for( ;copylen >= sizeof(*src); copylen -= sizeof(*src)) {
-		memcpy(&temp, src, sizeof(temp));
-		src++;
-		csum += temp;
-		*dest++ = temp;
-	    }
-	    *lastPartialLength = 0;
-	    *lastPartialInt = 0;
-	}
+        if (*lastPartialLength) {
+            /* do we have enough data to fill out the partial word? */
+            if (copylen >= (sizeof(unsigned int) - *lastPartialLength)) { /* YES, we do... */
+                memcpy(((char *) &temp + *lastPartialLength), src,
+                       (sizeof(unsigned int) - *lastPartialLength));
+                memcpy(dest, ((char *) &temp + *lastPartialLength),
+                       (sizeof(unsigned int) - *lastPartialLength));
+                src = (unsigned int *) ((char *) src + sizeof(unsigned int) - *lastPartialLength);
+                dest = (unsigned int *) ((char *) dest + sizeof(unsigned int) - *lastPartialLength);
+                csum += (temp - *lastPartialInt);
+                copylen -= sizeof(unsigned int) - *lastPartialLength;
+                /* now we have a source of unknown alignment and a unaligned destination */
+                if (INTALIGNED(src)) {
+                    for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) {
+                        temp = *src++;
+                        csum += temp;
+                        memcpy(dest, &temp, sizeof(temp));
+                        dest++;
+                    }
+                    *lastPartialInt = 0;
+                    *lastPartialLength = 0;
+                } else {
+                    for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) {
+                        memcpy(&temp, src, sizeof(temp));
+                        src++;
+                        csum += temp;
+                        memcpy(dest, &temp, sizeof(temp));
+                        dest++;
+                    }
+                    *lastPartialLength = 0;
+                    *lastPartialInt = 0;
+                }
+            } else { /* NO, we don't... */
+                memcpy(((char *) &temp + *lastPartialLength), src, copylen);
+                memcpy(dest, ((char *) &temp + *lastPartialLength), copylen);
+                src = (unsigned int *) ((char *) src + copylen);
+                dest = (unsigned int *) ((char *) dest + copylen);
+                csum += (temp - *lastPartialInt);
+                *lastPartialInt = temp;
+                *lastPartialLength += copylen;
+                copylen = 0;
+            }
+        } else {
+            for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) {
+                memcpy(&temp, src, sizeof(temp));
+                src++;
+                csum += temp;
+                *dest++ = temp;
+            }
+            *lastPartialLength = 0;
+            *lastPartialInt = 0;
+        }
     } else {
-	if (*lastPartialLength) {
-	    /* do we have enough data to fill out the partial word? */
-	    if (copylen >= (sizeof(unsigned int) - *lastPartialLength)) { /* YES, we do... */
-		memcpy(((char *)&temp + *lastPartialLength), src,
-		       (sizeof(unsigned int) - *lastPartialLength));
-		memcpy(dest, ((char *)&temp + *lastPartialLength),
-		       (sizeof(unsigned int) - *lastPartialLength));
-		src = (unsigned int *)((char *)src + sizeof(unsigned int) - *lastPartialLength);
-		dest = (unsigned int *)((char *)dest + sizeof(unsigned int) - *lastPartialLength);
-		csum += (temp - *lastPartialInt);
-		copylen -= sizeof(unsigned int) - *lastPartialLength;
-		/* now we have an unknown alignment for our source and destination */
-		if (INTALIGNED(src) && INTALIGNED(dest)) {
-		    size_t numLongs = copylen/sizeof(unsigned int);
-		    for(i = 0; i < numLongs; i++) {
-			csum += *src;
-			*dest++ = *src++;
-		    }
-		    copylen -= i * sizeof(unsigned int);
-		}
-		else { /* safe but slower for all other alignments */
-		    for( ;copylen >= sizeof(*src); copylen -= sizeof(*src)) {
-			memcpy(&temp, src, sizeof(temp));
-			src++;
-			csum += temp;
-			memcpy(dest, &temp, sizeof(temp));
-			dest++;
-		    }
-		}
-		*lastPartialInt = 0;
-		*lastPartialLength = 0;
-	    }
-	    else { /* NO, we don't... */
-		memcpy(((char *)&temp + *lastPartialLength), src, copylen);
-		memcpy(dest, ((char *)&temp + *lastPartialLength), copylen);
-		src = (unsigned int *)((char *)src + copylen);
-		dest = (unsigned int *)((char *)dest + copylen);
-		csum += (temp - *lastPartialInt);
-		*lastPartialInt = temp;
-		*lastPartialLength += copylen;
-		copylen = 0;
-	    }
-	}
-	else {
-	    for( ;copylen >= sizeof(*src); copylen -= sizeof(*src)) {
-		memcpy(&temp, src, sizeof(temp));
-		src++;
-		csum += temp;
-		memcpy(dest, &temp, sizeof(temp));
-		dest++;
-	    }
-	    *lastPartialLength = 0;
-	    *lastPartialInt = 0;
-	}
+        if (*lastPartialLength) {
+            /* do we have enough data to fill out the partial word? */
+            if (copylen >= (sizeof(unsigned int) - *lastPartialLength)) { /* YES, we do... */
+                memcpy(((char *) &temp + *lastPartialLength), src,
+                       (sizeof(unsigned int) - *lastPartialLength));
+                memcpy(dest, ((char *) &temp + *lastPartialLength),
+                       (sizeof(unsigned int) - *lastPartialLength));
+                src = (unsigned int *) ((char *) src + sizeof(unsigned int) - *lastPartialLength);
+                dest = (unsigned int *) ((char *) dest + sizeof(unsigned int) - *lastPartialLength);
+                csum += (temp - *lastPartialInt);
+                copylen -= sizeof(unsigned int) - *lastPartialLength;
+                /* now we have an unknown alignment for our source and destination */
+                if (INTALIGNED(src) && INTALIGNED(dest)) {
+                    size_t numLongs = copylen / sizeof(unsigned int);
+                    for (i = 0; i < numLongs; i++) {
+                        csum += *src;
+                        *dest++ = *src++;
+                    }
+                    copylen -= i * sizeof(unsigned int);
+                } else { /* safe but slower for all other alignments */
+                    for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) {
+                        memcpy(&temp, src, sizeof(temp));
+                        src++;
+                        csum += temp;
+                        memcpy(dest, &temp, sizeof(temp));
+                        dest++;
+                    }
+                }
+                *lastPartialInt = 0;
+                *lastPartialLength = 0;
+            } else { /* NO, we don't... */
+                memcpy(((char *) &temp + *lastPartialLength), src, copylen);
+                memcpy(dest, ((char *) &temp + *lastPartialLength), copylen);
+                src = (unsigned int *) ((char *) src + copylen);
+                dest = (unsigned int *) ((char *) dest + copylen);
+                csum += (temp - *lastPartialInt);
+                *lastPartialInt = temp;
+                *lastPartialLength += copylen;
+                copylen = 0;
+            }
+        } else {
+            for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) {
+                memcpy(&temp, src, sizeof(temp));
+                src++;
+                csum += temp;
+                memcpy(dest, &temp, sizeof(temp));
+                dest++;
+            }
+            *lastPartialLength = 0;
+            *lastPartialInt = 0;
+        }
     }
 
     /* if copylen is non-zero there was a bit left, less than an unsigned int's worth */
     if ((copylen != 0) && (csumlenresidue == 0)) {
-	temp = *lastPartialInt;
-	if (*lastPartialLength) {
-	    if (copylen >= (sizeof(unsigned int) - *lastPartialLength)) {
-		/* copy all remaining bytes from src to dest */
-		unsigned int copytemp = 0;
-		memcpy(&copytemp, src, copylen);
-		memcpy(dest, &copytemp, copylen);
-		/* fill out rest of partial word and add to checksum */
-		memcpy(((char *)&temp + *lastPartialLength), src,
-		       (sizeof(unsigned int) - *lastPartialLength));
-		/* avoid unsigned arithmetic overflow by subtracting the old partial
-		 * word from the new one before adding to the checksum...
-        */
-		csum += (temp - *lastPartialInt);
-		copylen -= sizeof(unsigned int) - *lastPartialLength;
-		src = (unsigned int *)((char *)src + sizeof(unsigned int) - *lastPartialLength);
-		*lastPartialLength = copylen;
-		/* reset temp, and calculate next partial word */
-		temp = 0;
-		if (copylen) {
-		    memcpy(&temp, src, copylen);
-		}
-		/* add it to the the checksum */
-		csum += temp;
-		*lastPartialInt = temp;
-	    }
-	    else {
-		/* copy all remaining bytes from src to dest */
-		unsigned int copytemp = 0;
-		memcpy(&copytemp, src, copylen);
-		memcpy(dest, &copytemp, copylen);
-		/* fill out rest of partial word and add to checksum */
-		memcpy(((char *)&temp + *lastPartialLength), src,
-		       copylen);
-		/* avoid unsigned arithmetic overflow by subtracting the old partial
-		 * word from the new one before adding to the checksum...
-        */
-		csum += temp - *lastPartialInt;
-		*lastPartialInt = temp;
-		*lastPartialLength += copylen;
-	    }
-	}
-	else { /* fast path... */
-	    /* temp and *lastPartialInt are 0 if *lastPartialLength is 0... */
-	    memcpy(&temp, src, copylen);
-	    csum += temp;
-	    memcpy(dest, &temp, copylen);
-	    *lastPartialInt = temp;
-	    *lastPartialLength = copylen;
-	    /* done...return the checksum */
-	}
-    }
-    else if (csumlenresidue != 0) {
-	if (copylen != 0) {
-	    temp = 0;
-	    memcpy(&temp, src, copylen);
-	    memcpy(dest, &temp, copylen);
-	}
-	if (csumlenresidue < (sizeof(unsigned int) - copylen - *lastPartialLength)) {
-	    temp = *lastPartialInt;
-	    memcpy(((char *)&temp + *lastPartialLength), src, (copylen + csumlenresidue));
-	    /* avoid unsigned arithmetic overflow by subtracting the old partial
-	     * word from the new one before adding to the checksum...
-        */
-	    csum += temp - *lastPartialInt;
-	    src++;
-	    *lastPartialInt = temp;
-	    *lastPartialLength += copylen + csumlenresidue;
-	    csumlenresidue = 0;
-	}
-	else {
-	    /* we have enough chksum data to fill out our last partial
-	     * word
-        */
-	    temp = *lastPartialInt;
-	    memcpy(((char *)&temp + *lastPartialLength), src,
-		   (sizeof(unsigned int) - *lastPartialLength));
-	    /* avoid unsigned arithmetic overflow by subtracting the old partial
-	     * word from the new one before adding to the checksum...
-        */
-	    csum += temp - *lastPartialInt;
-	    src = (unsigned int *)((char *)src + sizeof(unsigned int) - *lastPartialLength);
-	    csumlenresidue -= sizeof(unsigned int) - *lastPartialLength - copylen;
-	    *lastPartialLength = 0;
-	    *lastPartialInt = 0;
-	}
-	if (INTALIGNED(src)) {
-	    for (i = 0; i < csumlenresidue/sizeof(unsigned int); i++) {
-		csum += *src++;
-	    }
-	}
-	else {
-	    for (i = 0; i < csumlenresidue/sizeof(unsigned int); i++) {
-		memcpy(&temp, src, sizeof(temp));
-		csum += temp;
-		src++;
-	    }
-	}
-	csumlenresidue -= i * sizeof(unsigned int);
-	if (csumlenresidue) {
-	    temp = 0;
-	    memcpy(&temp, src, csumlenresidue);
-	    csum += temp;
-	    *lastPartialInt = temp;
-	    *lastPartialLength = csumlenresidue;
-	}
+        temp = *lastPartialInt;
+        if (*lastPartialLength) {
+            if (copylen >= (sizeof(unsigned int) - *lastPartialLength)) {
+                /* copy all remaining bytes from src to dest */
+                unsigned int copytemp = 0;
+                memcpy(&copytemp, src, copylen);
+                memcpy(dest, &copytemp, copylen);
+                /* fill out rest of partial word and add to checksum */
+                memcpy(((char *) &temp + *lastPartialLength), src,
+                       (sizeof(unsigned int) - *lastPartialLength));
+                /* avoid unsigned arithmetic overflow by subtracting the old partial
+                 * word from the new one before adding to the checksum...
+                 */
+                csum += (temp - *lastPartialInt);
+                copylen -= sizeof(unsigned int) - *lastPartialLength;
+                src = (unsigned int *) ((char *) src + sizeof(unsigned int) - *lastPartialLength);
+                *lastPartialLength = copylen;
+                /* reset temp, and calculate next partial word */
+                temp = 0;
+                if (copylen) {
+                    memcpy(&temp, src, copylen);
+                }
+                /* add it to the the checksum */
+                csum += temp;
+                *lastPartialInt = temp;
+            } else {
+                /* copy all remaining bytes from src to dest */
+                unsigned int copytemp = 0;
+                memcpy(&copytemp, src, copylen);
+                memcpy(dest, &copytemp, copylen);
+                /* fill out rest of partial word and add to checksum */
+                memcpy(((char *) &temp + *lastPartialLength), src, copylen);
+                /* avoid unsigned arithmetic overflow by subtracting the old partial
+                 * word from the new one before adding to the checksum...
+                 */
+                csum += temp - *lastPartialInt;
+                *lastPartialInt = temp;
+                *lastPartialLength += copylen;
+            }
+        } else { /* fast path... */
+            /* temp and *lastPartialInt are 0 if *lastPartialLength is 0... */
+            memcpy(&temp, src, copylen);
+            csum += temp;
+            memcpy(dest, &temp, copylen);
+            *lastPartialInt = temp;
+            *lastPartialLength = copylen;
+            /* done...return the checksum */
+        }
+    } else if (csumlenresidue != 0) {
+        if (copylen != 0) {
+            temp = 0;
+            memcpy(&temp, src, copylen);
+            memcpy(dest, &temp, copylen);
+        }
+        if (csumlenresidue < (sizeof(unsigned int) - copylen - *lastPartialLength)) {
+            temp = *lastPartialInt;
+            memcpy(((char *) &temp + *lastPartialLength), src, (copylen + csumlenresidue));
+            /* avoid unsigned arithmetic overflow by subtracting the old partial
+             * word from the new one before adding to the checksum...
+             */
+            csum += temp - *lastPartialInt;
+            src++;
+            *lastPartialInt = temp;
+            *lastPartialLength += copylen + csumlenresidue;
+            csumlenresidue = 0;
+        } else {
+            /* we have enough chksum data to fill out our last partial
+             * word
+             */
+            temp = *lastPartialInt;
+            memcpy(((char *) &temp + *lastPartialLength), src,
+                   (sizeof(unsigned int) - *lastPartialLength));
+            /* avoid unsigned arithmetic overflow by subtracting the old partial
+             * word from the new one before adding to the checksum...
+             */
+            csum += temp - *lastPartialInt;
+            src = (unsigned int *) ((char *) src + sizeof(unsigned int) - *lastPartialLength);
+            csumlenresidue -= sizeof(unsigned int) - *lastPartialLength - copylen;
+            *lastPartialLength = 0;
+            *lastPartialInt = 0;
+        }
+        if (INTALIGNED(src)) {
+            for (i = 0; i < csumlenresidue / sizeof(unsigned int); i++) {
+                csum += *src++;
+            }
+        } else {
+            for (i = 0; i < csumlenresidue / sizeof(unsigned int); i++) {
+                memcpy(&temp, src, sizeof(temp));
+                csum += temp;
+                src++;
+            }
+        }
+        csumlenresidue -= i * sizeof(unsigned int);
+        if (csumlenresidue) {
+            temp = 0;
+            memcpy(&temp, src, csumlenresidue);
+            csum += temp;
+            *lastPartialInt = temp;
+            *lastPartialLength = csumlenresidue;
+        }
     } /* end else if (csumlenresidue != 0) */
 
     return csum;
 }
 
-
 /*
  * csum() generates a bcopy_csum() - compatible checksum that can be
  * called multiple times
  */
 
-unsigned long
-opal_csum_partial (
-    const void *  source,
-    size_t csumlen,
-    unsigned long*  lastPartialLong,
-    size_t* lastPartialLength
-    )
+unsigned long opal_csum_partial(const void *source, size_t csumlen, unsigned long *lastPartialLong,
+                                size_t *lastPartialLength)
 {
-    unsigned long *  src = (unsigned long *) source;
+    unsigned long *src = (unsigned long *) source;
     unsigned long csum = 0;
     unsigned long i, temp;
 
-
-
     temp = *lastPartialLong;
 
-    if (WORDALIGNED(source))  {
-	if (*lastPartialLength) {
-	    /* do we have enough data to fill out the partial word? */
-	    if (csumlen >= (sizeof(unsigned long) - *lastPartialLength)) { /* YES, we do... */
-		memcpy(((char *)&temp + *lastPartialLength), src,
-		       (sizeof(unsigned long) - *lastPartialLength));
-		src = (unsigned long *)((char *)src + sizeof(unsigned long) - *lastPartialLength);
-		csum += (temp - *lastPartialLong);
-		csumlen -= sizeof(unsigned long) - *lastPartialLength;
-		/* now we have an unaligned source */
-		for(i = 0; i < csumlen/sizeof(unsigned long); i++) {
-		    memcpy(&temp, src, sizeof(temp));
-		    csum += temp;
-		    src++;
-		}
-		csumlen -= i * sizeof(unsigned long);
-		*lastPartialLong = 0;
-		*lastPartialLength = 0;
-	    }
-	    else { /* NO, we don't... */
-		memcpy(((char *)&temp + *lastPartialLength), src, csumlen);
-		src = (unsigned long *)((char *)src + csumlen);
-		csum += (temp - *lastPartialLong);
-		*lastPartialLong = temp;
-		*lastPartialLength += csumlen;
-		csumlen = 0;
-	    }
-	}
-	else { /* fast path... */
-	    size_t numLongs = csumlen/sizeof(unsigned long);
-	    for(i = 0; i < numLongs; i++) {
-		    csum += *src++;
-	    }
-	    *lastPartialLong = 0;
-	    *lastPartialLength = 0;
-	    if (WORDALIGNED(csumlen)) {
-		    return(csum);
-	    }
-	    else {
-		    csumlen -= i * sizeof(unsigned long);
-	    }
-	}
+    if (WORDALIGNED(source)) {
+        if (*lastPartialLength) {
+            /* do we have enough data to fill out the partial word? */
+            if (csumlen >= (sizeof(unsigned long) - *lastPartialLength)) { /* YES, we do... */
+                memcpy(((char *) &temp + *lastPartialLength), src,
+                       (sizeof(unsigned long) - *lastPartialLength));
+                src = (unsigned long *) ((char *) src + sizeof(unsigned long) - *lastPartialLength);
+                csum += (temp - *lastPartialLong);
+                csumlen -= sizeof(unsigned long) - *lastPartialLength;
+                /* now we have an unaligned source */
+                for (i = 0; i < csumlen / sizeof(unsigned long); i++) {
+                    memcpy(&temp, src, sizeof(temp));
+                    csum += temp;
+                    src++;
+                }
+                csumlen -= i * sizeof(unsigned long);
+                *lastPartialLong = 0;
+                *lastPartialLength = 0;
+            } else { /* NO, we don't... */
+                memcpy(((char *) &temp + *lastPartialLength), src, csumlen);
+                src = (unsigned long *) ((char *) src + csumlen);
+                csum += (temp - *lastPartialLong);
+                *lastPartialLong = temp;
+                *lastPartialLength += csumlen;
+                csumlen = 0;
+            }
+        } else { /* fast path... */
+            size_t numLongs = csumlen / sizeof(unsigned long);
+            for (i = 0; i < numLongs; i++) {
+                csum += *src++;
+            }
+            *lastPartialLong = 0;
+            *lastPartialLength = 0;
+            if (WORDALIGNED(csumlen)) {
+                return (csum);
+            } else {
+                csumlen -= i * sizeof(unsigned long);
+            }
+        }
     } else {
-	if (*lastPartialLength) {
-	    /* do we have enough data to fill out the partial word? */
-	    if (csumlen >= (sizeof(unsigned long) - *lastPartialLength)) { /* YES, we do... */
-		memcpy(((char *)&temp + *lastPartialLength), src,
-		       (sizeof(unsigned long) - *lastPartialLength));
-		src = (unsigned long *)((char *)src + sizeof(unsigned long) - *lastPartialLength);
-		csum += (temp - *lastPartialLong);
-		csumlen -= sizeof(unsigned long) - *lastPartialLength;
-		/* now we have a source of unknown alignment */
-		if (WORDALIGNED(src)) {
-		    for(i = 0; i < csumlen/sizeof(unsigned long); i++) {
-			csum += *src++;
-		    }
-		    csumlen -= i * sizeof(unsigned long);
-		    *lastPartialLong = 0;
-		    *lastPartialLength = 0;
-		}
-		else {
-		    for(i = 0; i < csumlen/sizeof(unsigned long); i++) {
-			memcpy(&temp, src, sizeof(temp));
-			csum += temp;
-			src++;
-		    }
-		    csumlen -= i * sizeof(unsigned long);
-		    *lastPartialLong = 0;
-		    *lastPartialLength = 0;
-		}
-	    }
-	    else { /* NO, we don't... */
-		memcpy(((char *)&temp + *lastPartialLength), src, csumlen);
-		src = (unsigned long *)((char *)src + csumlen);
-		csum += (temp - *lastPartialLong);
-		*lastPartialLong = temp;
-		*lastPartialLength += csumlen;
-		csumlen = 0;
-	    }
-	}
-	else {
-	    for( ;csumlen >= sizeof(*src); csumlen -= sizeof(*src)) {
-		memcpy(&temp, src, sizeof(temp));
-		src++;
-		csum += temp;
-	    }
-	    *lastPartialLength = 0;
-	    *lastPartialLong = 0;
-	}
+        if (*lastPartialLength) {
+            /* do we have enough data to fill out the partial word? */
+            if (csumlen >= (sizeof(unsigned long) - *lastPartialLength)) { /* YES, we do... */
+                memcpy(((char *) &temp + *lastPartialLength), src,
+                       (sizeof(unsigned long) - *lastPartialLength));
+                src = (unsigned long *) ((char *) src + sizeof(unsigned long) - *lastPartialLength);
+                csum += (temp - *lastPartialLong);
+                csumlen -= sizeof(unsigned long) - *lastPartialLength;
+                /* now we have a source of unknown alignment */
+                if (WORDALIGNED(src)) {
+                    for (i = 0; i < csumlen / sizeof(unsigned long); i++) {
+                        csum += *src++;
+                    }
+                    csumlen -= i * sizeof(unsigned long);
+                    *lastPartialLong = 0;
+                    *lastPartialLength = 0;
+                } else {
+                    for (i = 0; i < csumlen / sizeof(unsigned long); i++) {
+                        memcpy(&temp, src, sizeof(temp));
+                        csum += temp;
+                        src++;
+                    }
+                    csumlen -= i * sizeof(unsigned long);
+                    *lastPartialLong = 0;
+                    *lastPartialLength = 0;
+                }
+            } else { /* NO, we don't... */
+                memcpy(((char *) &temp + *lastPartialLength), src, csumlen);
+                src = (unsigned long *) ((char *) src + csumlen);
+                csum += (temp - *lastPartialLong);
+                *lastPartialLong = temp;
+                *lastPartialLength += csumlen;
+                csumlen = 0;
+            }
+        } else {
+            for (; csumlen >= sizeof(*src); csumlen -= sizeof(*src)) {
+                memcpy(&temp, src, sizeof(temp));
+                src++;
+                csum += temp;
+            }
+            *lastPartialLength = 0;
+            *lastPartialLong = 0;
+        }
     }
 
     /* if csumlen is non-zero there was a bit left, less than an unsigned long's worth */
     if (csumlen != 0) {
-	temp = *lastPartialLong;
-	if (*lastPartialLength) {
-	    if (csumlen >= (sizeof(unsigned long) - *lastPartialLength)) {
-		/* fill out rest of partial word and add to checksum */
-		memcpy(((char *)&temp + *lastPartialLength), src,
-		       (sizeof(unsigned long) - *lastPartialLength));
-		csum += (temp - *lastPartialLong);
-		csumlen -= sizeof(unsigned long) - *lastPartialLength;
-		src = (unsigned long *)((char *)src + sizeof(unsigned long) - *lastPartialLength);
-		*lastPartialLength = csumlen;
-		/* reset temp, and calculate next partial word */
-		temp = 0;
-		if (csumlen) {
-		    memcpy(&temp, src, csumlen);
-		}
-		/* add it to the the checksum */
-		csum += temp;
-		*lastPartialLong = temp;
-	    }
-	    else {
-		/* fill out rest of partial word and add to checksum */
-		memcpy(((char *)&temp + *lastPartialLength), src,
-		       csumlen);
-		csum += (temp - *lastPartialLong);
-		*lastPartialLong = temp;
-		*lastPartialLength += csumlen;
-	    }
-	}
-	else { /* fast path... */
-	    /* temp and *lastPartialLong are 0 if *lastPartialLength is 0... */
-	    memcpy(&temp, src, csumlen);
-	    csum += temp;
-	    *lastPartialLong = temp;
-	    *lastPartialLength = csumlen;
-	    /* done...return the checksum */
-	}
+        temp = *lastPartialLong;
+        if (*lastPartialLength) {
+            if (csumlen >= (sizeof(unsigned long) - *lastPartialLength)) {
+                /* fill out rest of partial word and add to checksum */
+                memcpy(((char *) &temp + *lastPartialLength), src,
+                       (sizeof(unsigned long) - *lastPartialLength));
+                csum += (temp - *lastPartialLong);
+                csumlen -= sizeof(unsigned long) - *lastPartialLength;
+                src = (unsigned long *) ((char *) src + sizeof(unsigned long) - *lastPartialLength);
+                *lastPartialLength = csumlen;
+                /* reset temp, and calculate next partial word */
+                temp = 0;
+                if (csumlen) {
+                    memcpy(&temp, src, csumlen);
+                }
+                /* add it to the the checksum */
+                csum += temp;
+                *lastPartialLong = temp;
+            } else {
+                /* fill out rest of partial word and add to checksum */
+                memcpy(((char *) &temp + *lastPartialLength), src, csumlen);
+                csum += (temp - *lastPartialLong);
+                *lastPartialLong = temp;
+                *lastPartialLength += csumlen;
+            }
+        } else { /* fast path... */
+            /* temp and *lastPartialLong are 0 if *lastPartialLength is 0... */
+            memcpy(&temp, src, csumlen);
+            csum += temp;
+            *lastPartialLong = temp;
+            *lastPartialLength = csumlen;
+            /* done...return the checksum */
+        }
     }
 
     return csum;
 }
 
-unsigned int
-opal_uicsum_partial (
-    const void *  source,
-    size_t csumlen,
-    unsigned int*  lastPartialInt,
-    size_t*  lastPartialLength
-    )
+unsigned int opal_uicsum_partial(const void *source, size_t csumlen, unsigned int *lastPartialInt,
+                                 size_t *lastPartialLength)
 {
-    unsigned int *  src = (unsigned int *) source;
+    unsigned int *src = (unsigned int *) source;
     unsigned int csum = 0;
     unsigned int temp;
     unsigned long i;
 
-
     temp = *lastPartialInt;
 
-    if (INTALIGNED(source))  {
-	if (*lastPartialLength) {
-	    /* do we have enough data to fill out the partial word? */
-	    if (csumlen >= (sizeof(unsigned int) - *lastPartialLength)) { /* YES, we do... */
-		memcpy(((char *)&temp + *lastPartialLength), src,
-		       (sizeof(unsigned int) - *lastPartialLength));
-		src = (unsigned int *)((char *)src + sizeof(unsigned int) - *lastPartialLength);
-		csum += (temp - *lastPartialInt);
-		csumlen -= sizeof(unsigned int) - *lastPartialLength;
-		/* now we have an unaligned source */
-		for(i = 0; i < csumlen/sizeof(unsigned int); i++) {
-		    memcpy(&temp, src, sizeof(temp));
-		    csum += temp;
-		    src++;
-		}
-		csumlen -= i * sizeof(unsigned int);
-		*lastPartialInt = 0;
-		*lastPartialLength = 0;
-	    }
-	    else { /* NO, we don't... */
-		memcpy(((char *)&temp + *lastPartialLength), src, csumlen);
-		src = (unsigned int *)((char *)src + csumlen);
-		csum += (temp - *lastPartialInt);
-		*lastPartialInt = temp;
-		*lastPartialLength += csumlen;
-		csumlen = 0;
-	    }
-	}
-	else { /* fast path... */
-	    size_t numLongs = csumlen/sizeof(unsigned int);
-	    for(i = 0; i < numLongs; i++) {
-		    csum += *src++;
-	    }
-	    *lastPartialInt = 0;
-	    *lastPartialLength = 0;
-	    if (INTALIGNED(csumlen)) {
-		    return(csum);
-	    }
-	    else {
-		    csumlen -= i * sizeof(unsigned int);
-	    }
-	}
+    if (INTALIGNED(source)) {
+        if (*lastPartialLength) {
+            /* do we have enough data to fill out the partial word? */
+            if (csumlen >= (sizeof(unsigned int) - *lastPartialLength)) { /* YES, we do... */
+                memcpy(((char *) &temp + *lastPartialLength), src,
+                       (sizeof(unsigned int) - *lastPartialLength));
+                src = (unsigned int *) ((char *) src + sizeof(unsigned int) - *lastPartialLength);
+                csum += (temp - *lastPartialInt);
+                csumlen -= sizeof(unsigned int) - *lastPartialLength;
+                /* now we have an unaligned source */
+                for (i = 0; i < csumlen / sizeof(unsigned int); i++) {
+                    memcpy(&temp, src, sizeof(temp));
+                    csum += temp;
+                    src++;
+                }
+                csumlen -= i * sizeof(unsigned int);
+                *lastPartialInt = 0;
+                *lastPartialLength = 0;
+            } else { /* NO, we don't... */
+                memcpy(((char *) &temp + *lastPartialLength), src, csumlen);
+                src = (unsigned int *) ((char *) src + csumlen);
+                csum += (temp - *lastPartialInt);
+                *lastPartialInt = temp;
+                *lastPartialLength += csumlen;
+                csumlen = 0;
+            }
+        } else { /* fast path... */
+            size_t numLongs = csumlen / sizeof(unsigned int);
+            for (i = 0; i < numLongs; i++) {
+                csum += *src++;
+            }
+            *lastPartialInt = 0;
+            *lastPartialLength = 0;
+            if (INTALIGNED(csumlen)) {
+                return (csum);
+            } else {
+                csumlen -= i * sizeof(unsigned int);
+            }
+        }
     } else {
-	if (*lastPartialLength) {
-	    /* do we have enough data to fill out the partial word? */
-	    if (csumlen >= (sizeof(unsigned int) - *lastPartialLength)) { /* YES, we do... */
-		memcpy(((char *)&temp + *lastPartialLength), src,
-		       (sizeof(unsigned int) - *lastPartialLength));
-		src = (unsigned int *)((char *)src + sizeof(unsigned int) - *lastPartialLength);
-		csum += (temp - *lastPartialInt);
-		csumlen -= sizeof(unsigned int) - *lastPartialLength;
-		/* now we have a source of unknown alignment */
-		if (INTALIGNED(src)) {
-		    for(i = 0; i < csumlen/sizeof(unsigned int); i++) {
-			csum += *src++;
-		    }
-		    csumlen -= i * sizeof(unsigned int);
-		    *lastPartialInt = 0;
-		    *lastPartialLength = 0;
-		}
-		else {
-		    for(i = 0; i < csumlen/sizeof(unsigned int); i++) {
-			memcpy(&temp, src, sizeof(temp));
-			csum += temp;
-			src++;
-		    }
-		    csumlen -= i * sizeof(unsigned int);
-		    *lastPartialInt = 0;
-		    *lastPartialLength = 0;
-		}
-	    }
-	    else { /* NO, we don't... */
-		memcpy(((char *)&temp + *lastPartialLength), src, csumlen);
-		src = (unsigned int *)((char *)src + csumlen);
-		csum += (temp - *lastPartialInt);
-		*lastPartialInt = temp;
-		*lastPartialLength += csumlen;
-		csumlen = 0;
-	    }
-	}
-	else {
-	    for( ;csumlen >= sizeof(*src); csumlen -= sizeof(*src)) {
-		memcpy(&temp, src, sizeof(temp));
-		src++;
-		csum += temp;
-	    }
-	    *lastPartialLength = 0;
-	    *lastPartialInt = 0;
-	}
+        if (*lastPartialLength) {
+            /* do we have enough data to fill out the partial word? */
+            if (csumlen >= (sizeof(unsigned int) - *lastPartialLength)) { /* YES, we do... */
+                memcpy(((char *) &temp + *lastPartialLength), src,
+                       (sizeof(unsigned int) - *lastPartialLength));
+                src = (unsigned int *) ((char *) src + sizeof(unsigned int) - *lastPartialLength);
+                csum += (temp - *lastPartialInt);
+                csumlen -= sizeof(unsigned int) - *lastPartialLength;
+                /* now we have a source of unknown alignment */
+                if (INTALIGNED(src)) {
+                    for (i = 0; i < csumlen / sizeof(unsigned int); i++) {
+                        csum += *src++;
+                    }
+                    csumlen -= i * sizeof(unsigned int);
+                    *lastPartialInt = 0;
+                    *lastPartialLength = 0;
+                } else {
+                    for (i = 0; i < csumlen / sizeof(unsigned int); i++) {
+                        memcpy(&temp, src, sizeof(temp));
+                        csum += temp;
+                        src++;
+                    }
+                    csumlen -= i * sizeof(unsigned int);
+                    *lastPartialInt = 0;
+                    *lastPartialLength = 0;
+                }
+            } else { /* NO, we don't... */
+                memcpy(((char *) &temp + *lastPartialLength), src, csumlen);
+                src = (unsigned int *) ((char *) src + csumlen);
+                csum += (temp - *lastPartialInt);
+                *lastPartialInt = temp;
+                *lastPartialLength += csumlen;
+                csumlen = 0;
+            }
+        } else {
+            for (; csumlen >= sizeof(*src); csumlen -= sizeof(*src)) {
+                memcpy(&temp, src, sizeof(temp));
+                src++;
+                csum += temp;
+            }
+            *lastPartialLength = 0;
+            *lastPartialInt = 0;
+        }
     }
 
     /* if csumlen is non-zero there was a bit left, less than an unsigned int's worth */
     if (csumlen != 0) {
-	temp = *lastPartialInt;
-	if (*lastPartialLength) {
-	    if (csumlen >= (sizeof(unsigned int) - *lastPartialLength)) {
-		/* fill out rest of partial word and add to checksum */
-		memcpy(((char *)&temp + *lastPartialLength), src,
-		       (sizeof(unsigned int) - *lastPartialLength));
-		csum += (temp - *lastPartialInt);
-		csumlen -= sizeof(unsigned int) - *lastPartialLength;
-		src = (unsigned int *)((char *)src + sizeof(unsigned int) - *lastPartialLength);
-		*lastPartialLength = csumlen;
-		/* reset temp, and calculate next partial word */
-		temp = 0;
-		if (csumlen) {
-		    memcpy(&temp, src, csumlen);
-		}
-		/* add it to the the checksum */
-		csum += temp;
-		*lastPartialInt = temp;
-	    }
-	    else {
-		/* fill out rest of partial word and add to checksum */
-		memcpy(((char *)&temp + *lastPartialLength), src,
-		       csumlen);
-		csum += (temp - *lastPartialInt);
-		*lastPartialInt = temp;
-		*lastPartialLength += csumlen;
-	    }
-	}
-	else { /* fast path... */
-	    /* temp and *lastPartialInt are 0 if *lastPartialLength is 0... */
-	    memcpy(&temp, src, csumlen);
-	    csum += temp;
-	    *lastPartialInt = temp;
-	    *lastPartialLength = csumlen;
-	    /* done...return the checksum */
-	}
+        temp = *lastPartialInt;
+        if (*lastPartialLength) {
+            if (csumlen >= (sizeof(unsigned int) - *lastPartialLength)) {
+                /* fill out rest of partial word and add to checksum */
+                memcpy(((char *) &temp + *lastPartialLength), src,
+                       (sizeof(unsigned int) - *lastPartialLength));
+                csum += (temp - *lastPartialInt);
+                csumlen -= sizeof(unsigned int) - *lastPartialLength;
+                src = (unsigned int *) ((char *) src + sizeof(unsigned int) - *lastPartialLength);
+                *lastPartialLength = csumlen;
+                /* reset temp, and calculate next partial word */
+                temp = 0;
+                if (csumlen) {
+                    memcpy(&temp, src, csumlen);
+                }
+                /* add it to the the checksum */
+                csum += temp;
+                *lastPartialInt = temp;
+            } else {
+                /* fill out rest of partial word and add to checksum */
+                memcpy(((char *) &temp + *lastPartialLength), src, csumlen);
+                csum += (temp - *lastPartialInt);
+                *lastPartialInt = temp;
+                *lastPartialLength += csumlen;
+            }
+        } else { /* fast path... */
+            /* temp and *lastPartialInt are 0 if *lastPartialLength is 0... */
+            memcpy(&temp, src, csumlen);
+            csum += temp;
+            *lastPartialInt = temp;
+            *lastPartialLength = csumlen;
+            /* done...return the checksum */
+        }
     }
 
     return csum;
@@ -1074,16 +992,17 @@ static unsigned int _opal_crc_table[256];
 
 void opal_initialize_crc_table(void)
 {
-    register int i,j;
+    register int i, j;
     register unsigned int crc_accum;
 
     for (i = 0; i < 256; i++) {
         crc_accum = (i << 24);
         for (j = 0; j < 8; j++) {
-            if (crc_accum & 0x80000000)
+            if (crc_accum & 0x80000000) {
                 crc_accum = (crc_accum << 1) ^ CRC_POLYNOMIAL;
-            else
+            } else {
                 crc_accum = (crc_accum << 1);
+            }
         }
         _opal_crc_table[i] = crc_accum;
     }
@@ -1093,12 +1012,8 @@ void opal_initialize_crc_table(void)
     return;
 }
 
-unsigned int opal_bcopy_uicrc_partial(
-    const void *  source,
-    void *  destination,
-    size_t copylen,
-    size_t crclen,
-    unsigned int partial_crc)
+unsigned int opal_bcopy_uicrc_partial(const void *source, void *destination, size_t copylen,
+                                      size_t crclen, unsigned int partial_crc)
 {
     size_t crclenresidue = (crclen > copylen) ? (crclen - copylen) : 0;
     register int i, j;
@@ -1110,22 +1025,22 @@ unsigned int opal_bcopy_uicrc_partial(
     }
 
     if (INTALIGNED(source) && INTALIGNED(destination)) {
-        register unsigned int *  src = (unsigned int *)source;
-        register unsigned int *  dst = (unsigned int *)destination;
+        register unsigned int *src = (unsigned int *) source;
+        register unsigned int *dst = (unsigned int *) destination;
         register unsigned char *ts, *td;
         /* copy whole integers */
         while (copylen >= sizeof(unsigned int)) {
             tmp = *src++;
             *dst++ = tmp;
-            ts = (unsigned char *)&tmp;
-            for (j = 0; j < (int)sizeof(unsigned int); j++) {
+            ts = (unsigned char *) &tmp;
+            for (j = 0; j < (int) sizeof(unsigned int); j++) {
                 i = ((partial_crc >> 24) ^ *ts++) & 0xff;
                 partial_crc = (partial_crc << 8) ^ _opal_crc_table[i];
             }
             copylen -= sizeof(unsigned int);
         }
-        ts = (unsigned char *)src;
-        td = (unsigned char *)dst;
+        ts = (unsigned char *) src;
+        td = (unsigned char *) dst;
         /* copy partial integer */
         while (copylen--) {
             t = *ts++;
@@ -1138,10 +1053,9 @@ unsigned int opal_bcopy_uicrc_partial(
             i = ((partial_crc >> 24) ^ *ts++) & 0xff;
             partial_crc = (partial_crc << 8) ^ _opal_crc_table[i];
         }
-    }
-    else {
-        register unsigned char *  src = (unsigned char *)source;
-        register unsigned char *  dst = (unsigned char *)destination;
+    } else {
+        register unsigned char *src = (unsigned char *) source;
+        register unsigned char *dst = (unsigned char *) destination;
         while (copylen--) {
             t = *src++;
             *dst++ = t;
@@ -1157,12 +1071,10 @@ unsigned int opal_bcopy_uicrc_partial(
     return partial_crc;
 }
 
-
-unsigned int opal_uicrc_partial(
-    const void *  source, size_t crclen, unsigned int partial_crc)
+unsigned int opal_uicrc_partial(const void *source, size_t crclen, unsigned int partial_crc)
 {
     register int i, j;
-    register unsigned char * t;
+    register unsigned char *t;
     unsigned int tmp;
 
     if (!_opal_crc_table_initialized) {
@@ -1170,24 +1082,23 @@ unsigned int opal_uicrc_partial(
     }
 
     if (INTALIGNED(source)) {
-        register unsigned int *  src = (unsigned int *)source;
+        register unsigned int *src = (unsigned int *) source;
         while (crclen >= sizeof(unsigned int)) {
             tmp = *src++;
-            t = (unsigned char *)&tmp;
-            for (j = 0; j < (int)sizeof(unsigned int); j++) {
+            t = (unsigned char *) &tmp;
+            for (j = 0; j < (int) sizeof(unsigned int); j++) {
                 i = ((partial_crc >> 24) ^ *t++) & 0xff;
                 partial_crc = (partial_crc << 8) ^ _opal_crc_table[i];
             }
             crclen -= sizeof(unsigned int);
         }
-        t = (unsigned char *)src;
+        t = (unsigned char *) src;
         while (crclen--) {
             i = ((partial_crc >> 24) ^ *t++) & 0xff;
             partial_crc = (partial_crc << 8) ^ _opal_crc_table[i];
         }
-    }
-    else {
-        register unsigned char *  src = (unsigned char *)source;
+    } else {
+        register unsigned char *src = (unsigned char *) source;
         while (crclen--) {
             i = ((partial_crc >> 24) ^ *src++) & 0xff;
             partial_crc = (partial_crc << 8) ^ _opal_crc_table[i];
@@ -1196,4 +1107,3 @@ unsigned int opal_uicrc_partial(
 
     return partial_crc;
 }
-
diff --git a/opal/util/crc.h b/opal/util/crc.h
index 22228d1a789..dd4235f4b79 100644
--- a/opal/util/crc.h
+++ b/opal/util/crc.h
@@ -28,75 +28,46 @@
 
 BEGIN_C_DECLS
 
-#define CRC_POLYNOMIAL ((unsigned int)0x04c11db7)
-#define CRC_INITIAL_REGISTER ((unsigned int)0xffffffff)
-
-
-#define OPAL_CSUM( SRC, LEN )  opal_uicsum( SRC, LEN )
-#define OPAL_CSUM_PARTIAL( SRC, LEN, UI1, UI2 ) \
-    opal_uicsum_partial( SRC, LEN, UI1, UI2 )
-#define OPAL_CSUM_BCOPY_PARTIAL( SRC, DST, LEN1, LEN2, UI1, UI2 ) \
-    opal_bcopy_uicsum_partial( SRC, DST, LEN1, LEN2, UI1, UI2 )
-#define OPAL_CSUM_ZERO  0
-
-
-OPAL_DECLSPEC unsigned long
-opal_bcopy_csum_partial(
-    const void *  source,
-    void *  destination,
-    size_t copylen,
-    size_t csumlen,
-    unsigned long*  lastPartialLong,
-    size_t*  lastPartialLength
-    );
-
-static inline unsigned long
-opal_bcopy_csum (
-    const void *  source,
-    void *  destination,
-    size_t copylen,
-    size_t csumlen
-    )
+#define CRC_POLYNOMIAL       ((unsigned int) 0x04c11db7)
+#define CRC_INITIAL_REGISTER ((unsigned int) 0xffffffff)
+
+#define OPAL_CSUM(SRC, LEN)                   opal_uicsum(SRC, LEN)
+#define OPAL_CSUM_PARTIAL(SRC, LEN, UI1, UI2) opal_uicsum_partial(SRC, LEN, UI1, UI2)
+#define OPAL_CSUM_BCOPY_PARTIAL(SRC, DST, LEN1, LEN2, UI1, UI2) \
+    opal_bcopy_uicsum_partial(SRC, DST, LEN1, LEN2, UI1, UI2)
+#define OPAL_CSUM_ZERO 0
+
+OPAL_DECLSPEC unsigned long opal_bcopy_csum_partial(const void *source, void *destination,
+                                                    size_t copylen, size_t csumlen,
+                                                    unsigned long *lastPartialLong,
+                                                    size_t *lastPartialLength);
+
+static inline unsigned long opal_bcopy_csum(const void *source, void *destination, size_t copylen,
+                                            size_t csumlen)
 {
     unsigned long plong = 0;
     size_t plength = 0;
     return opal_bcopy_csum_partial(source, destination, copylen, csumlen, &plong, &plength);
 }
 
-OPAL_DECLSPEC unsigned int
-opal_bcopy_uicsum_partial (
-    const void *  source,
-    void *  destination,
-    size_t copylen,
-    size_t csumlen,
-    unsigned int*  lastPartialInt,
-    size_t*  lastPartialLength
-    );
-
-static inline unsigned int
-opal_bcopy_uicsum (
-    const void *  source,
-    void *  destination,
-    size_t copylen,
-    size_t csumlen
-    )
+OPAL_DECLSPEC unsigned int opal_bcopy_uicsum_partial(const void *source, void *destination,
+                                                     size_t copylen, size_t csumlen,
+                                                     unsigned int *lastPartialInt,
+                                                     size_t *lastPartialLength);
+
+static inline unsigned int opal_bcopy_uicsum(const void *source, void *destination, size_t copylen,
+                                             size_t csumlen)
 {
     unsigned int pint = 0;
     size_t plength = 0;
     return opal_bcopy_uicsum_partial(source, destination, copylen, csumlen, &pint, &plength);
 }
 
-OPAL_DECLSPEC unsigned long
-opal_csum_partial (
-    const void *  source,
-    size_t csumlen,
-    unsigned long*  lastPartialLong,
-    size_t*  lastPartialLength
-    );
-
+OPAL_DECLSPEC unsigned long opal_csum_partial(const void *source, size_t csumlen,
+                                              unsigned long *lastPartialLong,
+                                              size_t *lastPartialLength);
 
-static inline unsigned long
-opal_csum(const void *  source, size_t csumlen)
+static inline unsigned long opal_csum(const void *source, size_t csumlen)
 {
     unsigned long lastPartialLong = 0;
     size_t lastPartialLength = 0;
@@ -105,36 +76,30 @@ opal_csum(const void *  source, size_t csumlen)
 /*
  * The buffer passed to this function is assumed to be 16-bit aligned
  */
-static inline uint16_t
-opal_csum16 (const void *  source, size_t csumlen)
+static inline uint16_t opal_csum16(const void *source, size_t csumlen)
 {
     uint16_t *src = (uint16_t *) source;
     register uint32_t csum = 0;
 
     while (csumlen > 1) {
-	    csum += *src++;
+        csum += *src++;
         csumlen -= 2;
     }
     /* Add leftover byte, if any */
-    if(csumlen > 0)
-        csum += *((unsigned char*)src);
+    if (csumlen > 0)
+        csum += *((unsigned char *) src);
     /* Fold 32-bit checksum to 16 bits */
-    while(csum >> 16) {
+    while (csum >> 16) {
         csum = (csum & 0xFFFF) + (csum >> 16);
     }
     return csum;
 }
 
-OPAL_DECLSPEC unsigned int
-opal_uicsum_partial (
-    const void *  source,
-    size_t csumlen,
-    unsigned int *  lastPartialInt,
-    size_t*  lastPartialLength
-    );
+OPAL_DECLSPEC unsigned int opal_uicsum_partial(const void *source, size_t csumlen,
+                                               unsigned int *lastPartialInt,
+                                               size_t *lastPartialLength);
 
-static inline unsigned int
-opal_uicsum(const void *  source, size_t csumlen)
+static inline unsigned int opal_uicsum(const void *source, size_t csumlen)
 {
     unsigned int lastPartialInt = 0;
     size_t lastPartialLength = 0;
@@ -147,33 +112,20 @@ opal_uicsum(const void *  source, size_t csumlen)
 
 void opal_initialize_crc_table(void);
 
-OPAL_DECLSPEC unsigned int
-opal_bcopy_uicrc_partial(
-    const void *  source,
-    void *  destination,
-    size_t copylen,
-    size_t crclen,
-    unsigned int partial_crc);
-
-static inline unsigned int
-opal_bcopy_uicrc(
-    const void *  source,
-    void *  destination,
-    size_t copylen,
-    size_t crclen)
+OPAL_DECLSPEC unsigned int opal_bcopy_uicrc_partial(const void *source, void *destination,
+                                                    size_t copylen, size_t crclen,
+                                                    unsigned int partial_crc);
+
+static inline unsigned int opal_bcopy_uicrc(const void *source, void *destination, size_t copylen,
+                                            size_t crclen)
 {
     return opal_bcopy_uicrc_partial(source, destination, copylen, crclen, CRC_INITIAL_REGISTER);
 }
 
-OPAL_DECLSPEC unsigned int
-opal_uicrc_partial(
-    const void *  source,
-    size_t crclen,
-    unsigned int partial_crc);
+OPAL_DECLSPEC unsigned int opal_uicrc_partial(const void *source, size_t crclen,
+                                              unsigned int partial_crc);
 
-
-static inline unsigned int
-opal_uicrc(const void *  source, size_t crclen)
+static inline unsigned int opal_uicrc(const void *source, size_t crclen)
 {
     return opal_uicrc_partial(source, crclen, CRC_INITIAL_REGISTER);
 }
@@ -181,4 +133,3 @@ opal_uicrc(const void *  source, size_t crclen)
 END_C_DECLS
 
 #endif
-
diff --git a/opal/util/daemon_init.c b/opal/util/daemon_init.c
deleted file mode 100644
index 6caae124224..00000000000
--- a/opal/util/daemon_init.c
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-
-#include "opal_config.h"
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-#include <sys/stat.h>
-#include <fcntl.h>
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#include <stdlib.h>
-
-#include "opal/util/daemon_init.h"
-#include "opal/constants.h"
-
-
-int opal_daemon_init(char *working_dir)
-{
-#if defined(HAVE_FORK)
-    pid_t pid;
-    int fd;
-
-    if ((pid = fork()) < 0) {
-        return OPAL_ERROR;
-    } else if (pid != 0) {
-        exit(0);   /* parent goes bye-bye */
-    }
-
-    /* child continues */
-#if defined(HAVE_SETSID)
-    setsid();  /* become session leader */
-#endif
-
-    if (NULL != working_dir) {
-        chdir(working_dir);  /* change working directory */
-    }
-
-    /* connect input to /dev/null */
-    fd = open("/dev/null", O_RDONLY);
-    if (0 > fd) {
-        return OPAL_ERR_FATAL;
-    }
-    dup2(fd, STDIN_FILENO);
-    if(fd != STDIN_FILENO) {
-        close(fd);
-    }
-
-    /* connect outputs to /dev/null */
-    fd = open("/dev/null", O_RDWR|O_CREAT|O_TRUNC, 0666);
-    if (fd >= 0) {
-        dup2(fd, STDOUT_FILENO);
-        dup2(fd, STDERR_FILENO);
-        /* just to be safe, make sure we aren't trying
-         * to close stdout or stderr! since we dup'd both
-         * of them to the same fd, we can't just close it
-         * since one of the two would still be open and
-         * someone could attempt to use it.
-         */
-        if(fd != STDOUT_FILENO && fd != STDERR_FILENO) {
-           close(fd);
-        }
-    } else {
-        return OPAL_ERR_FATAL;
-    }
-
-    return OPAL_SUCCESS;
-
-#else /* HAVE_FORK */
-    return OPAL_ERR_NOT_SUPPORTED;
-#endif
-}
diff --git a/opal/util/daemon_init.h b/opal/util/daemon_init.h
deleted file mode 100644
index ebcf46074d8..00000000000
--- a/opal/util/daemon_init.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2006 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-/** @file **/
-
-#ifndef OPAL_DAEMON_INIT_H
-#define OPAL_DAEMON_INIT_H
-
-#include "opal_config.h"
-
-BEGIN_C_DECLS
-
-/*
- * Turn a process into a daemon.
- *
- * This function converts a process into a daemon in an orderly manner. It first forks a child process,
- * then the parent exits. The child continues on to become a session leader, reset the file mode creation
- * mask, and changes working directories to the one specified.
- *
- * @param working_dir Pointer to a character string containing the desired working directory. Providing
- * a value of NULL will cause the function to leave the program in the current working directory.
- *
- * @retval OPAL_SUCCESS Indicates that the conversion was successful
- * @retval OPAL_ERROR Indicates that the conversion was not successful - a fork could not be completed.
- */
-OPAL_DECLSPEC int opal_daemon_init(char *working_dir);
-
-END_C_DECLS
-
-#endif /* OPAL_DAEMON_INIT_H */
diff --git a/opal/util/error.c b/opal/util/error.c
index 25a49faa561..7f8be1e5817 100644
--- a/opal/util/error.c
+++ b/opal/util/error.c
@@ -26,20 +26,21 @@
 
 #include "opal_config.h"
 
-#include <string.h>
 #include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 #include <unistd.h>
 
-#include "opal/util/error.h"
-#include "opal/util/string_copy.h"
 #include "opal/constants.h"
-#include "opal/util/proc.h"
-#include "opal/util/printf.h"
 #include "opal/runtime/opal_params.h"
+#include "opal/util/error.h"
+#include "opal/util/output.h"
+#include "opal/util/printf.h"
+#include "opal/util/proc.h"
+#include "opal/util/string_copy.h"
 
-#define MAX_CONVERTERS 5
+#define MAX_CONVERTERS            5
 #define MAX_CONVERTER_PROJECT_LEN 10
 
 struct converter_info_t {
@@ -54,16 +55,14 @@ typedef struct converter_info_t converter_info_t;
 /* all default to NULL */
 static converter_info_t converters[MAX_CONVERTERS] = {{0}};
 
-static int
-opal_strerror_int(int errnum, const char **str)
+static int opal_strerror_int(int errnum, const char **str)
 {
     int i, ret = OPAL_SUCCESS;
     *str = NULL;
 
-    for (i = 0 ; i < MAX_CONVERTERS ; ++i) {
-        if (0 != converters[i].init &&
-            errnum < converters[i].err_base &&
-            converters[i].err_max < errnum) {
+    for (i = 0; i < MAX_CONVERTERS; ++i) {
+        if (0 != converters[i].init && errnum < converters[i].err_base
+            && converters[i].err_max < errnum) {
             ret = converters[i].converter(errnum, str);
             break;
         }
@@ -72,21 +71,17 @@ opal_strerror_int(int errnum, const char **str)
     return ret;
 }
 
-
 /* caller must free string */
-static int
-opal_strerror_unknown(int errnum, char **str)
+static int opal_strerror_unknown(int errnum, char **str)
 {
     int i;
     *str = NULL;
 
-    for (i = 0 ; i < MAX_CONVERTERS ; ++i) {
+    for (i = 0; i < MAX_CONVERTERS; ++i) {
         if (0 != converters[i].init) {
-            if (errnum < converters[i].err_base &&
-                errnum > converters[i].err_max) {
-                opal_asprintf(str, "Unknown error: %d (%s error %d)",
-                         errnum, converters[i].project,
-                         errnum - converters[i].err_base);
+            if (errnum < converters[i].err_base && errnum > converters[i].err_max) {
+                opal_asprintf(str, "Unknown error: %d (%s error %d)", errnum, converters[i].project,
+                              errnum - converters[i].err_base);
                 return OPAL_SUCCESS;
             }
         }
@@ -97,12 +92,10 @@ opal_strerror_unknown(int errnum, char **str)
     return OPAL_SUCCESS;
 }
 
-
-void
-opal_perror(int errnum, const char *msg)
+void opal_perror(int errnum, const char *msg)
 {
     int ret;
-    const char* errmsg;
+    const char *errmsg;
     ret = opal_strerror_int(errnum, &errmsg);
 
     if (NULL != msg && errnum != OPAL_ERR_IN_ERRNO) {
@@ -129,11 +122,10 @@ opal_perror(int errnum, const char *msg)
 #define UNKNOWN_RETBUF_LEN 50
 static char unknown_retbuf[UNKNOWN_RETBUF_LEN];
 
-const char *
-opal_strerror(int errnum)
+const char *opal_strerror(int errnum)
 {
     int ret;
-    const char* errmsg;
+    const char *errmsg;
 
     if (errnum == OPAL_ERR_IN_ERRNO) {
         return strerror(errno);
@@ -147,17 +139,15 @@ opal_strerror(int errnum)
         snprintf(unknown_retbuf, UNKNOWN_RETBUF_LEN, "%s", ue_msg);
         free(ue_msg);
         errno = EINVAL;
-        return (const char*) unknown_retbuf;
+        return (const char *) unknown_retbuf;
     } else {
         return errmsg;
     }
 }
 
-
-int
-opal_strerror_r(int errnum, char *strerrbuf, size_t buflen)
+int opal_strerror_r(int errnum, char *strerrbuf, size_t buflen)
 {
-    const char* errmsg;
+    const char *errmsg;
     int ret, len;
 
     ret = opal_strerror_int(errnum, &errmsg);
@@ -169,7 +159,7 @@ opal_strerror_r(int errnum, char *strerrbuf, size_t buflen)
         } else {
             char *ue_msg;
             ret = opal_strerror_unknown(errnum, &ue_msg);
-            len =  snprintf(strerrbuf, buflen, "%s", ue_msg);
+            len = snprintf(strerrbuf, buflen, "%s", ue_msg);
             free(ue_msg);
             if (len > (int) buflen) {
                 errno = ERANGE;
@@ -180,7 +170,7 @@ opal_strerror_r(int errnum, char *strerrbuf, size_t buflen)
             }
         }
     } else {
-        len =  snprintf(strerrbuf, buflen, "%s", errmsg);
+        len = snprintf(strerrbuf, buflen, "%s", errmsg);
         if (len > (int) buflen) {
             errno = ERANGE;
             return OPAL_ERR_OUT_OF_RESOURCE;
@@ -190,25 +180,21 @@ opal_strerror_r(int errnum, char *strerrbuf, size_t buflen)
     }
 }
 
-
-int
-opal_error_register(const char *project, int err_base, int err_max,
-                    opal_err2str_fn_t converter)
+int opal_error_register(const char *project, int err_base, int err_max, opal_err2str_fn_t converter)
 {
     int i;
 
-    for (i = 0 ; i < MAX_CONVERTERS ; ++i) {
+    for (i = 0; i < MAX_CONVERTERS; ++i) {
         if (0 == converters[i].init) {
             converters[i].init = 1;
             opal_string_copy(converters[i].project, project, MAX_CONVERTER_PROJECT_LEN);
-            converters[i].project[MAX_CONVERTER_PROJECT_LEN-1] = '\0';
+            converters[i].project[MAX_CONVERTER_PROJECT_LEN - 1] = '\0';
             converters[i].err_base = err_base;
             converters[i].err_max = err_max;
             converters[i].converter = converter;
             return OPAL_SUCCESS;
-        } else if (converters[i].err_base == err_base &&
-                   converters[i].err_max == err_max &&
-                   !strcmp (project, converters[i].project)) {
+        } else if (converters[i].err_base == err_base && converters[i].err_max == err_max
+                   && !strcmp(project, converters[i].project)) {
             converters[i].converter = converter;
             return OPAL_SUCCESS;
         }
@@ -217,9 +203,7 @@ opal_error_register(const char *project, int err_base, int err_max,
     return OPAL_ERR_OUT_OF_RESOURCE;
 }
 
-
-void
-opal_delay_abort(void)
+void opal_delay_abort(void)
 {
     // Though snprintf and strlen are not guaranteed to be async-signal-safe
     // in POSIX, it is async-signal-safe on many implementations probably.
@@ -234,15 +218,18 @@ opal_delay_abort(void)
                      "[%s:%05d] Looping forever "
                      "(MCA parameter opal_abort_delay is < 0)\n",
                      opal_process_info.nodename, (int) pid);
-            write(STDERR_FILENO, msg, strlen(msg));
+        } else {
+            snprintf(msg, sizeof(msg), "[%s:%05d] Delaying for %d seconds before aborting\n",
+                     opal_process_info.nodename, (int) pid, delay);
+        }
+
+        opal_best_effort_write(STDERR_FILENO, msg, strlen(msg));
+
+        if (delay < 0) {
             while (1) {
                 sleep(5);
             }
         } else {
-            snprintf(msg, sizeof(msg),
-                     "[%s:%05d] Delaying for %d seconds before aborting\n",
-                     opal_process_info.nodename, (int) pid, delay);
-            write(STDERR_FILENO, msg, strlen(msg));
             do {
                 sleep(1);
             } while (--delay > 0);
diff --git a/opal/util/error.h b/opal/util/error.h
index d90fe9f0807..2ffa92a48a6 100644
--- a/opal/util/error.h
+++ b/opal/util/error.h
@@ -27,9 +27,7 @@
 BEGIN_C_DECLS
 
 #define OPAL_ERROR_LOG(r) \
-    opal_output(0, "OPAL ERROR: %s in file %s at line %d", \
-                opal_strerror((r)), __FILE__, __LINE__);
-
+    opal_output(0, "OPAL ERROR: %s in file %s at line %d", opal_strerror((r)), __FILE__, __LINE__);
 
 /**
  * Prints error message for errnum on stderr
@@ -72,7 +70,6 @@ OPAL_DECLSPEC const char *opal_strerror(int errnum);
  */
 OPAL_DECLSPEC int opal_strerror_r(int errnum, char *strerrbuf, size_t buflen);
 
-
 typedef int (*opal_err2str_fn_t)(int errnum, const char **str);
 
 /**
@@ -86,8 +83,7 @@ typedef int (*opal_err2str_fn_t)(int errnum, const char **str);
  * \note A maximum of 5 converters can be registered.  The 6th
  * converter registration attempt will return OPAL_ERR_OUT_OF_RESOURCE
  */
-OPAL_DECLSPEC int opal_error_register(const char *project,
-                                      int err_base, int err_max,
+OPAL_DECLSPEC int opal_error_register(const char *project, int err_base, int err_max,
                                       opal_err2str_fn_t converter);
 
 /**
diff --git a/opal/util/ethtool.c b/opal/util/ethtool.c
index 84aa5c09c64..9cfa7ad5416 100644
--- a/opal/util/ethtool.c
+++ b/opal/util/ethtool.c
@@ -2,7 +2,7 @@
  * Copyright (c) 2016      Karol Mroz.  All rights reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2016      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2016-2022 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -12,29 +12,29 @@
 
 #include "opal_config.h"
 
-#include <string.h>
 #include <limits.h>
+#include <string.h>
 
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 #ifdef HAVE_SYS_SOCKET_H
-#include <sys/socket.h>
+#    include <sys/socket.h>
 #endif
 #ifdef HAVE_NET_IF_H
-#include <net/if.h>
+#    include <net/if.h>
 #endif
 #ifdef HAVE_LINUX_ETHTOOL_H
-#include <linux/ethtool.h>
+#    include <linux/ethtool.h>
 #endif
 #ifdef HAVE_SYS_IOCTL_H
-#include <sys/ioctl.h>
+#    include <sys/ioctl.h>
 #endif
 #ifdef HAVE_LINUX_SOCKIOS_H
-#include <linux/sockios.h>
+#    include <linux/sockios.h>
 #endif
 
 #include "opal/util/ethtool.h"
@@ -46,8 +46,7 @@
  * get this via an ioctl(). Elsewhere or in the error case, we return the
  * speed as 0.
  */
-unsigned int
-opal_ethtool_get_speed (const char *if_name)
+unsigned int opal_ethtool_get_speed(const char *if_name)
 {
     unsigned int speed = 0;
 
@@ -64,20 +63,20 @@ opal_ethtool_get_speed (const char *if_name)
     }
 
     memset(&ifr, 0, sizeof(struct ifreq));
-    opal_string_copy(ifr.ifr_name, if_name, OPAL_IF_NAMESIZE);
-    ifr.ifr_data = (char *)&edata;
+    opal_string_copy(ifr.ifr_name, if_name, IF_NAMESIZE);
+    ifr.ifr_data = (char *) &edata;
 
     if (ioctl(sockfd, SIOCETHTOOL, &ifr) < 0) {
         goto out;
     }
 
-#if HAVE_DECL_ETHTOOL_CMD_SPEED
+#    if HAVE_DECL_ETHTOOL_CMD_SPEED
     speed = ethtool_cmd_speed(&edata);
-#elif defined(HAVE_STRUCT_ETHTOOL_CMD_SPEED_HI)
+#    elif defined(HAVE_STRUCT_ETHTOOL_CMD_SPEED_HI)
     speed = (edata.speed_hi << 16) | edata.speed;
-#else
+#    else
     speed = edata.speed;
-#endif
+#    endif
     if (UINT_MAX == speed) {
         speed = 0;
     }
diff --git a/opal/util/event.c b/opal/util/event.c
index 6330ac2838d..263adecd97d 100644
--- a/opal/util/event.c
+++ b/opal/util/event.c
@@ -12,6 +12,7 @@
  * Copyright (c) 2017      IBM Corporation.  All rights reserved.
  * Copyright (c) 2018-2020 Amazon.com, Inc. or its affiliates.  All
  *                         Rights reserved.
+ * Copyright (c) 2022      Google, LLC. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -24,12 +25,12 @@
 #include "opal/util/event.h"
 
 #include "opal/constants.h"
+#include "opal/mca/base/mca_base_var.h"
 #include "opal/util/argv.h"
 #include "opal/util/output.h"
 #include "opal/util/printf.h"
-#include "opal/mca/base/mca_base_var.h"
 
-opal_event_base_t *opal_sync_event_base=NULL;
+opal_event_base_t *opal_sync_event_base = NULL;
 
 static char *opal_event_module_include = NULL;
 static struct event_config *opal_event_config = NULL;
@@ -45,26 +46,25 @@ int opal_event_register_params(void)
     // Get supported methods
     opal_event_all_available_eventops = event_get_supported_methods();
 
-#ifdef __APPLE__
-    opal_event_module_include ="select";
+#if defined(PLATFORM_OS_DARWIN)
+    opal_event_module_include = "select";
+#elif defined(PLATFORM_OS_LINUX)
+    opal_event_module_include = "epoll";
 #else
     opal_event_module_include = "poll";
 #endif
 
-    avail = opal_argv_join((char**)opal_event_all_available_eventops, ',');
-    opal_asprintf( &help_msg,
-              "Comma-delimited list of libevent subsystems "
-              "to use (%s -- available on your platform)",
-              avail );
-
-    ret = mca_base_var_register("opal", "opal", "event", "include",
-				help_msg,
-				MCA_BASE_VAR_TYPE_STRING, NULL, 0,
-				MCA_BASE_VAR_FLAG_SETTABLE,
-				OPAL_INFO_LVL_3,
-				MCA_BASE_VAR_SCOPE_LOCAL,
-				&opal_event_module_include);
-    free(help_msg);  /* release the help message */
+    avail = opal_argv_join((char **) opal_event_all_available_eventops, ',');
+    opal_asprintf(&help_msg,
+                  "Comma-delimited list of libevent subsystems "
+                  "to use (%s -- available on your platform)",
+                  avail);
+
+    ret = mca_base_var_register("opal", "opal", "event", "include", help_msg,
+                                MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
+                                OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_LOCAL,
+                                &opal_event_module_include);
+    free(help_msg); /* release the help message */
     free(avail);
     avail = NULL;
 
@@ -72,19 +72,16 @@ int opal_event_register_params(void)
         return ret;
     }
 
-    ret = mca_base_var_register_synonym (ret, "opal", "event", "external", "include", 0);
+    ret = mca_base_var_register_synonym(ret, "opal", "event", "external", "include", 0);
     if (0 > ret) {
         return ret;
     }
 
     ret = mca_base_var_register("opal", "opal", "event", "verbose",
                                 "Verbosity level for the event framework (default: 0)",
-                                MCA_BASE_VAR_TYPE_INT,
-                                &mca_base_var_enum_verbose, 0,
-                                MCA_BASE_VAR_FLAG_SETTABLE,
-                                OPAL_INFO_LVL_3,
-                                MCA_BASE_VAR_SCOPE_LOCAL,
-                                &opal_event_verbose);
+                                MCA_BASE_VAR_TYPE_INT, &mca_base_var_enum_verbose, 0,
+                                MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
+                                MCA_BASE_VAR_SCOPE_LOCAL, &opal_event_verbose);
     if (0 > ret) {
         return ret;
     }
@@ -92,7 +89,7 @@ int opal_event_register_params(void)
     /* The event wrapper used to be a framework.  Help the user out by
      * providing a backwards compatible verbose flag
      */
-    ret = mca_base_var_register_synonym (ret, "opal", "event", "base", "verbose", 0);
+    ret = mca_base_var_register_synonym(ret, "opal", "event", "base", "verbose", 0);
     if (0 > ret) {
         return ret;
     }
@@ -102,8 +99,8 @@ int opal_event_register_params(void)
 
 int opal_event_init(void)
 {
-    char **includes=NULL;
-    bool dumpit=false;
+    char **includes = NULL;
+    bool dumpit = false;
     int i, j;
 
     if (opal_event_verbose > 4) {
@@ -114,19 +111,19 @@ int opal_event_init(void)
         /* Shouldn't happen, but... */
         opal_event_module_include = strdup("select");
     }
-    includes = opal_argv_split(opal_event_module_include,',');
+    includes = opal_argv_split(opal_event_module_include, ',');
 
     /* get a configuration object */
     opal_event_config = event_config_new();
     /* cycle thru the available subsystems */
-    for (i = 0 ; NULL != opal_event_all_available_eventops[i] ; ++i) {
+    for (i = 0; NULL != opal_event_all_available_eventops[i]; ++i) {
         /* if this module isn't included in the given ones,
          * then exclude it
          */
         dumpit = true;
-        for (j=0; NULL != includes[j]; j++) {
-            if (0 == strcmp("all", includes[j]) ||
-                0 == strcmp(opal_event_all_available_eventops[i], includes[j])) {
+        for (j = 0; NULL != includes[j]; j++) {
+            if (0 == strcmp("all", includes[j])
+                || 0 == strcmp(opal_event_all_available_eventops[i], includes[j])) {
                 dumpit = false;
                 break;
             }
@@ -158,7 +155,7 @@ int opal_event_finalize(void)
     return OPAL_SUCCESS;
 }
 
-opal_event_base_t* opal_event_base_create(void)
+opal_event_base_t *opal_event_base_create(void)
 {
     opal_event_base_t *base;
 
@@ -170,10 +167,10 @@ opal_event_base_t* opal_event_base_create(void)
     return base;
 }
 
-opal_event_t* opal_event_alloc(void)
+opal_event_t *opal_event_alloc(void)
 {
     opal_event_t *ev;
 
-    ev = (opal_event_t*)malloc(sizeof(opal_event_t));
+    ev = (opal_event_t *) malloc(sizeof(opal_event_t));
     return ev;
 }
diff --git a/opal/util/event.h b/opal/util/event.h
index b66eb0344fd..975f6cb10e8 100644
--- a/opal/util/event.h
+++ b/opal/util/event.h
@@ -18,33 +18,33 @@
 
 #include "opal_config.h"
 
-#include <event2/event_struct.h>
 #include <event2/event.h>
+#include <event2/event_struct.h>
 #include <event2/thread.h>
 
 BEGIN_C_DECLS
 
 /* set the number of event priority levels */
-#define OPAL_EVENT_NUM_PRI   8
-
-#define OPAL_EV_ERROR_PRI         0
-#define OPAL_EV_MSG_HI_PRI        1
-#define OPAL_EV_SYS_HI_PRI        2
-#define OPAL_EV_MSG_LO_PRI        3
-#define OPAL_EV_SYS_LO_PRI        4
-#define OPAL_EV_INFO_HI_PRI       5
-#define OPAL_EV_INFO_LO_PRI       6
-#define OPAL_EV_LOWEST_PRI        7
+#define OPAL_EVENT_NUM_PRI 8
 
-#define OPAL_EVENT_SIGNAL(ev)	opal_event_get_signal(ev)
+#define OPAL_EV_ERROR_PRI   0
+#define OPAL_EV_MSG_HI_PRI  1
+#define OPAL_EV_SYS_HI_PRI  2
+#define OPAL_EV_MSG_LO_PRI  3
+#define OPAL_EV_SYS_LO_PRI  4
+#define OPAL_EV_INFO_HI_PRI 5
+#define OPAL_EV_INFO_LO_PRI 6
+#define OPAL_EV_LOWEST_PRI  7
 
-#define OPAL_TIMEOUT_DEFAULT	{1, 0}
+#define OPAL_EVENT_SIGNAL(ev) opal_event_get_signal(ev)
 
+#define OPAL_TIMEOUT_DEFAULT \
+    {                        \
+        1, 0                 \
+    }
 
 typedef event_callback_fn opal_event_cbfunc_t;
 
-BEGIN_C_DECLS
-
 typedef struct event_base opal_event_base_t;
 typedef struct event opal_event_t;
 
@@ -57,11 +57,11 @@ OPAL_DECLSPEC extern opal_event_base_t *opal_sync_event_base;
 /* Persistent event: won't get removed automatically when activated. */
 #define OPAL_EV_PERSIST EV_PERSIST
 
-#define OPAL_EVLOOP_ONCE     EVLOOP_ONCE        /**< Block at most once. */
-#define OPAL_EVLOOP_NONBLOCK EVLOOP_NONBLOCK    /**< Do not block. */
+#define OPAL_EVLOOP_ONCE     EVLOOP_ONCE /**< Block at most once. */
+#define OPAL_EVLOOP_NONBLOCK EVLOOP_NONBLOCK /**< Do not block. */
 
 /* Global function to create and release an event base */
-OPAL_DECLSPEC opal_event_base_t* opal_event_base_create(void);
+OPAL_DECLSPEC opal_event_base_t *opal_event_base_create(void);
 
 #define opal_event_base_free(x) event_base_free(x)
 
@@ -90,9 +90,11 @@ OPAL_DECLSPEC int opal_event_finalize(void);
 /* Basic event APIs */
 #define opal_event_enable_debug_mode() event_enable_debug_mode()
 
-#define opal_event_set(b, x, fd, fg, cb, arg) event_assign((x), (b), (fd), (fg), (event_callback_fn) (cb), (arg))
+#define opal_event_set(b, x, fd, fg, cb, arg) \
+    event_assign((x), (b), (fd), (fg), (event_callback_fn)(cb), (arg))
 
-#define opal_event_assign(x, b, fd, fg, cb, arg) event_assign((x), (b), (fd), (fg), (event_callback_fn) (cb), (arg))
+#define opal_event_assign(x, b, fd, fg, cb, arg) \
+    event_assign((x), (b), (fd), (fg), (event_callback_fn)(cb), (arg))
 
 #define opal_event_add(ev, tv) event_add((ev), (tv))
 
@@ -100,9 +102,10 @@ OPAL_DECLSPEC int opal_event_finalize(void);
 
 #define opal_event_active(x, y, z) event_active((x), (y), (z))
 
-#define opal_event_new(b, fd, fg, cb, arg) event_new((b), (fd), (fg), (event_callback_fn) (cb), (arg))
+#define opal_event_new(b, fd, fg, cb, arg) \
+    event_new((b), (fd), (fg), (event_callback_fn)(cb), (arg))
 
-OPAL_DECLSPEC opal_event_t* opal_event_alloc(void);
+OPAL_DECLSPEC opal_event_t *opal_event_alloc(void);
 
 #define opal_event_free(x) event_free((x))
 
@@ -111,7 +114,8 @@ OPAL_DECLSPEC opal_event_t* opal_event_alloc(void);
 
 #define opal_event_evtimer_add(x, tv) opal_event_add((x), (tv))
 
-#define opal_event_evtimer_set(b, x, cb, arg) event_assign((x), (b), -1, 0, (event_callback_fn) (cb), (arg))
+#define opal_event_evtimer_set(b, x, cb, arg) \
+    event_assign((x), (b), -1, 0, (event_callback_fn)(cb), (arg))
 
 #define opal_event_evtimer_del(x) opal_event_del((x))
 
@@ -122,7 +126,8 @@ OPAL_DECLSPEC opal_event_t* opal_event_alloc(void);
 /* Signal APIs */
 #define opal_event_signal_add(x, tv) event_add((x), (tv))
 
-#define opal_event_signal_set(b, x, fd, cb, arg) event_assign((x), (b), (fd), EV_SIGNAL|EV_PERSIST, (event_callback_fn) (cb), (arg))
+#define opal_event_signal_set(b, x, fd, cb, arg) \
+    event_assign((x), (b), (fd), EV_SIGNAL | EV_PERSIST, (event_callback_fn)(cb), (arg))
 
 #define opal_event_signal_del(x) event_del((x))
 
diff --git a/opal/util/fd.c b/opal/util/fd.c
index 7891beb8e71..3ab95fa2391 100644
--- a/opal/util/fd.c
+++ b/opal/util/fd.c
@@ -13,32 +13,31 @@
 #include "opal_config.h"
 
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 #ifdef HAVE_SYS_STAT_H
-#include <sys/stat.h>
+#    include <sys/stat.h>
 #endif
 #ifdef HAVE_SYS_SOCKET_H
-#include <sys/socket.h>
+#    include <sys/socket.h>
 #endif
 #ifdef HAVE_ARPA_INET_H
-#include <arpa/inet.h>
+#    include <arpa/inet.h>
 #endif
 #ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
+#    include <netinet/in.h>
 #endif
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 #include <errno.h>
 #include <fcntl.h>
 #include <stdlib.h>
 #include <string.h>
 
+#include "opal/constants.h"
 #include "opal/util/fd.h"
 #include "opal/util/string_copy.h"
-#include "opal/constants.h"
-
 
 /*
  * Simple loop over reading from a fd
@@ -64,7 +63,6 @@ int opal_fd_read(int fd, int len, void *buffer)
     return OPAL_SUCCESS;
 }
 
-
 /*
  * Simple loop over writing to an fd
  */
@@ -88,7 +86,6 @@ int opal_fd_write(int fd, int len, const void *buffer)
     return OPAL_SUCCESS;
 }
 
-
 int opal_fd_set_cloexec(int fd)
 {
 #ifdef FD_CLOEXEC
@@ -160,7 +157,7 @@ const char *opal_fd_get_peer_name(int fd)
 
     if (sa.sa_family == AF_INET) {
         struct sockaddr_in *si;
-        si = (struct sockaddr_in*) &sa;
+        si = (struct sockaddr_in *) &sa;
         ret = inet_ntop(AF_INET, &(si->sin_addr), str, INET_ADDRSTRLEN);
         if (NULL == ret) {
             free(str);
@@ -169,7 +166,7 @@ const char *opal_fd_get_peer_name(int fd)
 #if OPAL_ENABLE_IPV6
     else if (sa.sa_family == AF_INET6) {
         struct sockaddr_in6 *si6;
-        si6 = (struct sockaddr_in6*) &sa;
+        si6 = (struct sockaddr_in6 *) &sa;
         ret = inet_ntop(AF_INET6, &(si6->sin6_addr), str, INET6_ADDRSTRLEN);
         if (NULL == ret) {
             free(str);
diff --git a/opal/util/few.c b/opal/util/few.c
index 45e827e8f50..d029e0e63cc 100644
--- a/opal/util/few.c
+++ b/opal/util/few.c
@@ -16,23 +16,20 @@
  * $HEADER$
  */
 
-
 #include "opal_config.h"
 
-#include <stdio.h>
 #include <errno.h>
+#include <stdio.h>
 #ifdef HAVE_SYS_WAIT_H
-#include <sys/wait.h>
+#    include <sys/wait.h>
 #endif
 #include <stdlib.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 
-#include "opal/util/few.h"
-#include "opal/util/basename.h"
-#include "opal/util/argv.h"
 #include "opal/constants.h"
+#include "opal/util/few.h"
 
 int opal_few(char *argv[], int *status)
 {
@@ -40,38 +37,38 @@ int opal_few(char *argv[], int *status)
     pid_t pid, ret;
 
     if ((pid = fork()) < 0) {
-      return OPAL_ERR_IN_ERRNO;
+        return OPAL_ERR_IN_ERRNO;
     }
 
     /* Child execs.  If it fails to exec, exit. */
 
     else if (0 == pid) {
-      execvp(argv[0], argv);
-      exit(errno);
+        execvp(argv[0], argv);
+        exit(errno);
     }
 
     /* Parent loops waiting for the child to die. */
 
     else {
-      do {
-        /* If the child exited, return */
+        do {
+            /* If the child exited, return */
 
-        if (pid == (ret = waitpid(pid, status, 0))) {
-          break;
-        }
+            if (pid == (ret = waitpid(pid, status, 0))) {
+                break;
+            }
 
-        /* If waitpid was interrupted, loop around again */
+            /* If waitpid was interrupted, loop around again */
 
-        else if (ret < 0) {
-          if (EINTR == errno) {
-            continue;
-          }
+            else if (ret < 0) {
+                if (EINTR == errno) {
+                    continue;
+                }
 
-          /* Otherwise, some bad juju happened -- need to quit */
+                /* Otherwise, some bad juju happened -- need to quit */
 
-          return OPAL_ERR_IN_ERRNO;
-        }
-      } while (true);
+                return OPAL_ERR_IN_ERRNO;
+            }
+        } while (true);
     }
 
     /* Return the status to the caller */
diff --git a/opal/util/if.c b/opal/util/if.c
index bd73e987690..c16d3e69344 100644
--- a/opal/util/if.c
+++ b/opal/util/if.c
@@ -11,7 +11,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2022 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2015-2016 Research Organization for Information Science
@@ -29,65 +29,65 @@
 
 #include <string.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 #include <errno.h>
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 #ifdef HAVE_SYS_SOCKET_H
-#include <sys/socket.h>
+#    include <sys/socket.h>
 #endif
 #ifdef HAVE_SYS_SOCKIO_H
-#include <sys/sockio.h>
+#    include <sys/sockio.h>
 #endif
 #ifdef HAVE_SYS_IOCTL_H
-#include <sys/ioctl.h>
+#    include <sys/ioctl.h>
 #endif
 #ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
+#    include <netinet/in.h>
 #endif
 #ifdef HAVE_ARPA_INET_H
-#include <arpa/inet.h>
+#    include <arpa/inet.h>
 #endif
 #ifdef HAVE_NET_IF_H
-#include <net/if.h>
+#    include <net/if.h>
 #endif
 #ifdef HAVE_NETDB_H
-#include <netdb.h>
+#    include <netdb.h>
 #endif
 #ifdef HAVE_IFADDRS_H
-#include <ifaddrs.h>
+#    include <ifaddrs.h>
 #endif
 #include <ctype.h>
 
 #include "opal/class/opal_list.h"
+#include "opal/constants.h"
+#include "opal/util/argv.h"
 #include "opal/util/if.h"
 #include "opal/util/net.h"
 #include "opal/util/output.h"
-#include "opal/util/argv.h"
 #include "opal/util/show_help.h"
 #include "opal/util/string_copy.h"
-#include "opal/constants.h"
 
 #include "opal/mca/if/base/base.h"
 
 #ifdef HAVE_STRUCT_SOCKADDR_IN
 
-#ifndef MIN
-#  define MIN(a,b)                ((a) < (b) ? (a) : (b))
-#endif
+#    ifndef MIN
+#        define MIN(a, b) ((a) < (b) ? (a) : (b))
+#    endif
 
 /*
  *  Look for interface by name and returns its address
  *  as a dotted decimal formatted string.
  */
 
-int opal_ifnametoaddr(const char* if_name, struct sockaddr* addr, int length)
+int opal_ifnametoaddr(const char *if_name, struct sockaddr *addr, int length)
 {
-    opal_if_t* intf;
+    opal_if_t *intf;
 
-    OPAL_LIST_FOREACH(intf, &opal_if_list, opal_if_t) {
+    OPAL_LIST_FOREACH (intf, &opal_if_list, opal_if_t) {
         if (strcmp(intf->if_name, if_name) == 0) {
             memcpy(addr, &intf->if_addr, length);
             return OPAL_SUCCESS;
@@ -96,17 +96,16 @@ int opal_ifnametoaddr(const char* if_name, struct sockaddr* addr, int length)
     return OPAL_ERROR;
 }
 
-
 /*
  *  Look for interface by name and returns its
  *  corresponding opal_list index.
  */
 
-int opal_ifnametoindex(const char* if_name)
+int opal_ifnametoindex(const char *if_name)
 {
-    opal_if_t* intf;
+    opal_if_t *intf;
 
-    OPAL_LIST_FOREACH(intf, &opal_if_list, opal_if_t) {
+    OPAL_LIST_FOREACH (intf, &opal_if_list, opal_if_t) {
         if (strcmp(intf->if_name, if_name) == 0) {
             return intf->if_index;
         }
@@ -114,17 +113,16 @@ int opal_ifnametoindex(const char* if_name)
     return -1;
 }
 
-
 /*
  *  Look for interface by name and returns its
  *  corresponding kernel index.
  */
 
-int opal_ifnametokindex(const char* if_name)
+int opal_ifnametokindex(const char *if_name)
 {
-    opal_if_t* intf;
+    opal_if_t *intf;
 
-    OPAL_LIST_FOREACH(intf, &opal_if_list, opal_if_t) {
+    OPAL_LIST_FOREACH (intf, &opal_if_list, opal_if_t) {
         if (strcmp(intf->if_name, if_name) == 0) {
             return intf->if_kernel_index;
         }
@@ -132,7 +130,6 @@ int opal_ifnametokindex(const char* if_name)
     return -1;
 }
 
-
 /*
  *  Look for interface by opal_list index and returns its
  *  corresponding kernel index.
@@ -140,9 +137,9 @@ int opal_ifnametokindex(const char* if_name)
 
 int opal_ifindextokindex(int if_index)
 {
-    opal_if_t* intf;
+    opal_if_t *intf;
 
-    OPAL_LIST_FOREACH(intf, &opal_if_list, opal_if_t) {
+    OPAL_LIST_FOREACH (intf, &opal_if_list, opal_if_t) {
         if (if_index == intf->if_index) {
             return intf->if_kernel_index;
         }
@@ -150,15 +147,14 @@ int opal_ifindextokindex(int if_index)
     return -1;
 }
 
-
 /*
- *  Attempt to resolve the adddress (given as either IPv4/IPv6 string
+ *  Attempt to resolve the address (given as either IPv4/IPv6 string
  *  or hostname) and lookup corresponding interface.
  */
 
-int opal_ifaddrtoname(const char* if_addr, char* if_name, int length)
+int opal_ifaddrtoname(const char *if_addr, char *if_name, int length)
 {
-    opal_if_t* intf;
+    opal_if_t *intf;
     int error;
     struct addrinfo hints, *res = NULL, *r;
 
@@ -177,40 +173,40 @@ int opal_ifaddrtoname(const char* if_addr, char* if_name, int length)
 
     if (error) {
         if (NULL != res) {
-            freeaddrinfo (res);
+            freeaddrinfo(res);
         }
         return OPAL_ERR_NOT_FOUND;
     }
 
     for (r = res; r != NULL; r = r->ai_next) {
-        OPAL_LIST_FOREACH(intf, &opal_if_list, opal_if_t) {
+        OPAL_LIST_FOREACH (intf, &opal_if_list, opal_if_t) {
             if (AF_INET == r->ai_family) {
                 struct sockaddr_in ipv4;
                 struct sockaddr_in *inaddr;
 
-                inaddr = (struct sockaddr_in*) &intf->if_addr;
-                memcpy (&ipv4, r->ai_addr, r->ai_addrlen);
+                inaddr = (struct sockaddr_in *) &intf->if_addr;
+                memcpy(&ipv4, r->ai_addr, r->ai_addrlen);
 
                 if (inaddr->sin_addr.s_addr == ipv4.sin_addr.s_addr) {
                     opal_string_copy(if_name, intf->if_name, length);
-                    freeaddrinfo (res);
+                    freeaddrinfo(res);
                     return OPAL_SUCCESS;
                 }
             }
-#if OPAL_ENABLE_IPV6
+#    if OPAL_ENABLE_IPV6
             else {
-                if (IN6_ARE_ADDR_EQUAL(&((struct sockaddr_in6*) &intf->if_addr)->sin6_addr,
-                    &((struct sockaddr_in6*) r->ai_addr)->sin6_addr)) {
+                if (IN6_ARE_ADDR_EQUAL(&((struct sockaddr_in6 *) &intf->if_addr)->sin6_addr,
+                                       &((struct sockaddr_in6 *) r->ai_addr)->sin6_addr)) {
                     opal_string_copy(if_name, intf->if_name, length);
-                    freeaddrinfo (res);
+                    freeaddrinfo(res);
                     return OPAL_SUCCESS;
                 }
             }
-#endif
+#    endif
         }
     }
     if (NULL != res) {
-        freeaddrinfo (res);
+        freeaddrinfo(res);
     }
 
     /* if we get here, it wasn't found */
@@ -222,9 +218,9 @@ int opal_ifaddrtoname(const char* if_addr, char* if_name, int length)
  *  or hostname) and return the kernel index of the interface
  *  on the same network as the specified address
  */
-int opal_ifaddrtokindex(const char* if_addr)
+int opal_ifaddrtokindex(const char *if_addr)
 {
-    opal_if_t* intf;
+    opal_if_t *intf;
     int error;
     struct addrinfo hints, *res = NULL, *r;
     int if_kernel_index;
@@ -237,40 +233,44 @@ int opal_ifaddrtokindex(const char* if_addr)
 
     if (error) {
         if (NULL != res) {
-            freeaddrinfo (res);
+            freeaddrinfo(res);
         }
         return OPAL_ERR_NOT_FOUND;
     }
 
     for (r = res; r != NULL; r = r->ai_next) {
-        OPAL_LIST_FOREACH(intf, &opal_if_list, opal_if_t) {
+        OPAL_LIST_FOREACH (intf, &opal_if_list, opal_if_t) {
             if (AF_INET == r->ai_family && AF_INET == intf->af_family) {
                 struct sockaddr_in ipv4;
-                len = (r->ai_addrlen < sizeof(struct sockaddr_in)) ? r->ai_addrlen : sizeof(struct sockaddr_in);
+                len = (r->ai_addrlen < sizeof(struct sockaddr_in)) ? r->ai_addrlen
+                                                                   : sizeof(struct sockaddr_in);
                 memcpy(&ipv4, r->ai_addr, len);
-                if (opal_net_samenetwork((struct sockaddr*)&ipv4, (struct sockaddr*)&intf->if_addr, intf->if_mask)) {
+                if (opal_net_samenetwork((struct sockaddr *) &ipv4,
+                                         (struct sockaddr *) &intf->if_addr, intf->if_mask)) {
                     if_kernel_index = intf->if_kernel_index;
-                    freeaddrinfo (res);
+                    freeaddrinfo(res);
                     return if_kernel_index;
                 }
             }
-#if OPAL_ENABLE_IPV6
+#    if OPAL_ENABLE_IPV6
             else if (AF_INET6 == r->ai_family && AF_INET6 == intf->af_family) {
                 struct sockaddr_in6 ipv6;
-                len = (r->ai_addrlen < sizeof(struct sockaddr_in6)) ? r->ai_addrlen : sizeof(struct sockaddr_in6);
+                len = (r->ai_addrlen < sizeof(struct sockaddr_in6)) ? r->ai_addrlen
+                                                                    : sizeof(struct sockaddr_in6);
                 memcpy(&ipv6, r->ai_addr, len);
-                if (opal_net_samenetwork((struct sockaddr*)((struct sockaddr_in6*)&intf->if_addr),
-                                         (struct sockaddr*)&ipv6, intf->if_mask)) {
+                if (opal_net_samenetwork((struct sockaddr *) ((struct sockaddr_in6 *) &intf
+                                                                  ->if_addr),
+                                         (struct sockaddr *) &ipv6, intf->if_mask)) {
                     if_kernel_index = intf->if_kernel_index;
-                    freeaddrinfo (res);
+                    freeaddrinfo(res);
                     return if_kernel_index;
                 }
             }
-#endif
+#    endif
         }
     }
     if (NULL != res) {
-        freeaddrinfo (res);
+        freeaddrinfo(res);
     }
     return OPAL_ERR_NOT_FOUND;
 }
@@ -284,7 +284,6 @@ int opal_ifcount(void)
     return opal_list_get_size(&opal_if_list);
 }
 
-
 /*
  *  Return the opal_list interface index for the first
  *  interface in our list.
@@ -294,13 +293,13 @@ int opal_ifbegin(void)
 {
     opal_if_t *intf;
 
-    intf = (opal_if_t*)opal_list_get_first(&opal_if_list);
-    if (NULL != intf)
+    intf = (opal_if_t *) opal_list_get_first(&opal_if_list);
+    if (NULL != intf) {
         return intf->if_index;
+    }
     return (-1);
 }
 
-
 /*
  *  Located the current position in the list by if_index and
  *  return the interface index of the next element in our list
@@ -311,70 +310,67 @@ int opal_ifnext(int if_index)
 {
     opal_if_t *intf;
 
-    OPAL_LIST_FOREACH(intf, &opal_if_list, opal_if_t) {
+    OPAL_LIST_FOREACH (intf, &opal_if_list, opal_if_t) {
         if (intf->if_index == if_index) {
             do {
-                opal_if_t* if_next = (opal_if_t*)opal_list_get_next(intf);
-                opal_if_t* if_end =  (opal_if_t*)opal_list_get_end(&opal_if_list);
+                opal_if_t *if_next = (opal_if_t *) opal_list_get_next(intf);
+                opal_if_t *if_end = (opal_if_t *) opal_list_get_end(&opal_if_list);
                 if (if_next == if_end) {
                     return -1;
                 }
                 intf = if_next;
-            } while(intf->if_index == if_index);
+            } while (intf->if_index == if_index);
             return intf->if_index;
         }
     }
     return (-1);
 }
 
-
 /*
  *  Lookup the interface by opal_list index and return the
  *  primary address assigned to the interface.
  */
 
-int opal_ifindextoaddr(int if_index, struct sockaddr* if_addr, unsigned int length)
+int opal_ifindextoaddr(int if_index, struct sockaddr *if_addr, unsigned int length)
 {
-    opal_if_t* intf;
+    opal_if_t *intf;
 
-    OPAL_LIST_FOREACH(intf, &opal_if_list, opal_if_t) {
+    OPAL_LIST_FOREACH (intf, &opal_if_list, opal_if_t) {
         if (intf->if_index == if_index) {
-            memcpy(if_addr, &intf->if_addr, MIN(length, sizeof (intf->if_addr)));
+            memcpy(if_addr, &intf->if_addr, MIN(length, sizeof(intf->if_addr)));
             return OPAL_SUCCESS;
         }
     }
     return OPAL_ERROR;
 }
 
-
 /*
  *  Lookup the interface by opal_list kindex and return the
  *  primary address assigned to the interface.
  */
-int opal_ifkindextoaddr(int if_kindex, struct sockaddr* if_addr, unsigned int length)
+int opal_ifkindextoaddr(int if_kindex, struct sockaddr *if_addr, unsigned int length)
 {
-    opal_if_t* intf;
+    opal_if_t *intf;
 
-    OPAL_LIST_FOREACH(intf, &opal_if_list, opal_if_t) {
+    OPAL_LIST_FOREACH (intf, &opal_if_list, opal_if_t) {
         if (intf->if_kernel_index == if_kindex) {
-            memcpy(if_addr, &intf->if_addr, MIN(length, sizeof (intf->if_addr)));
+            memcpy(if_addr, &intf->if_addr, MIN(length, sizeof(intf->if_addr)));
             return OPAL_SUCCESS;
         }
     }
     return OPAL_ERROR;
 }
 
-
 /*
  *  Lookup the interface by opal_list index and return the
  *  network mask assigned to the interface.
  */
 
-int opal_ifindextomask(int if_index, uint32_t* if_mask, int length)
+int opal_ifindextomask(int if_index, uint32_t *if_mask, int length)
 {
-    opal_if_t* intf;
+    opal_if_t *intf;
 
-    OPAL_LIST_FOREACH(intf, &opal_if_list, opal_if_t) {
+    OPAL_LIST_FOREACH (intf, &opal_if_list, opal_if_t) {
         if (intf->if_index == if_index) {
             memcpy(if_mask, &intf->if_mask, length);
             return OPAL_SUCCESS;
@@ -390,9 +386,9 @@ int opal_ifindextomask(int if_index, uint32_t* if_mask, int length)
 
 int opal_ifindextomac(int if_index, uint8_t mac[6])
 {
-    opal_if_t* intf;
+    opal_if_t *intf;
 
-    OPAL_LIST_FOREACH(intf, &opal_if_list, opal_if_t) {
+    OPAL_LIST_FOREACH (intf, &opal_if_list, opal_if_t) {
         if (intf->if_index == if_index) {
             memcpy(mac, &intf->if_mac, 6);
             return OPAL_SUCCESS;
@@ -408,9 +404,9 @@ int opal_ifindextomac(int if_index, uint8_t mac[6])
 
 int opal_ifindextomtu(int if_index, int *mtu)
 {
-    opal_if_t* intf;
+    opal_if_t *intf;
 
-    OPAL_LIST_FOREACH(intf, &opal_if_list, opal_if_t) {
+    OPAL_LIST_FOREACH (intf, &opal_if_list, opal_if_t) {
         if (intf->if_index == if_index) {
             *mtu = intf->ifmtu;
             return OPAL_SUCCESS;
@@ -424,11 +420,11 @@ int opal_ifindextomtu(int if_index, int *mtu)
  *  flags assigned to the interface.
  */
 
-int opal_ifindextoflags(int if_index, uint32_t* if_flags)
+int opal_ifindextoflags(int if_index, uint32_t *if_flags)
 {
-    opal_if_t* intf;
+    opal_if_t *intf;
 
-    OPAL_LIST_FOREACH(intf, &opal_if_list, opal_if_t) {
+    OPAL_LIST_FOREACH (intf, &opal_if_list, opal_if_t) {
         if (intf->if_index == if_index) {
             memcpy(if_flags, &intf->if_flags, sizeof(uint32_t));
             return OPAL_SUCCESS;
@@ -437,18 +433,16 @@ int opal_ifindextoflags(int if_index, uint32_t* if_flags)
     return OPAL_ERROR;
 }
 
-
-
 /*
  *  Lookup the interface by opal_list index and return
  *  the associated name.
  */
 
-int opal_ifindextoname(int if_index, char* if_name, int length)
+int opal_ifindextoname(int if_index, char *if_name, int length)
 {
     opal_if_t *intf;
 
-    OPAL_LIST_FOREACH(intf, &opal_if_list, opal_if_t) {
+    OPAL_LIST_FOREACH (intf, &opal_if_list, opal_if_t) {
         if (intf->if_index == if_index) {
             opal_string_copy(if_name, intf->if_name, length);
             return OPAL_SUCCESS;
@@ -457,17 +451,16 @@ int opal_ifindextoname(int if_index, char* if_name, int length)
     return OPAL_ERROR;
 }
 
-
 /*
  *  Lookup the interface by kernel index and return
  *  the associated name.
  */
 
-int opal_ifkindextoname(int if_kindex, char* if_name, int length)
+int opal_ifkindextoname(int if_kindex, char *if_name, int length)
 {
     opal_if_t *intf;
 
-    OPAL_LIST_FOREACH(intf, &opal_if_list, opal_if_t) {
+    OPAL_LIST_FOREACH (intf, &opal_if_list, opal_if_t) {
         if (intf->if_kernel_index == if_kindex) {
             opal_string_copy(if_name, intf->if_name, length);
             return OPAL_SUCCESS;
@@ -476,17 +469,15 @@ int opal_ifkindextoname(int if_kindex, char* if_name, int length)
     return OPAL_ERROR;
 }
 
-
-#define ADDRLEN 100
-bool
-opal_ifislocal(const char *hostname)
+#    define ADDRLEN 100
+bool opal_ifislocal(const char *hostname)
 {
-#if OPAL_ENABLE_IPV6
+#    if OPAL_ENABLE_IPV6
     char addrname[NI_MAXHOST]; /* should be larger than ADDRLEN, but I think
                                   they really mean IFNAMESIZE */
-#else
+#    else
     char addrname[ADDRLEN + 1];
-#endif
+#    endif
 
     if (OPAL_SUCCESS == opal_ifaddrtoname(hostname, addrname, ADDRLEN)) {
         return true;
@@ -495,16 +486,16 @@ opal_ifislocal(const char *hostname)
     return false;
 }
 
-static int parse_ipv4_dots(const char *addr, uint32_t* net, int* dots)
+static int parse_ipv4_dots(const char *addr, uint32_t *net, int *dots)
 {
     const char *start = addr, *end;
-    uint32_t n[]={0,0,0,0};
+    uint32_t n[] = {0, 0, 0, 0};
     int i;
 
     /* now assemble the address */
-    for( i = 0; i < 4; i++ ) {
-        n[i] = strtoul(start, (char**)&end, 10);
-        if( end == start ) {
+    for (i = 0; i < 4; i++) {
+        n[i] = strtoul(start, (char **) &end, 10);
+        if (end == start) {
             /* this is not an error, but indicates that
              * we were given a partial address - e.g.,
              * 192.168 - usually indicating an IP range
@@ -513,20 +504,22 @@ static int parse_ipv4_dots(const char *addr, uint32_t* net, int* dots)
             break;
         }
         /* did we read something sensible? */
-        if( n[i] > 255 ) {
+        if (n[i] > 255) {
             return OPAL_ERR_NETWORK_NOT_PARSEABLE;
         }
         /* skip all the . */
-        for( start = end; '\0' != *start; start++ )
-            if( '.' != *start ) break;
+        for (start = end; '\0' != *start; start++) {
+            if ('.' != *start) {
+                break;
+            }
+        }
     }
     *dots = i;
     *net = OPAL_IF_ASSEMBLE_NETWORK(n[0], n[1], n[2], n[3]);
     return OPAL_SUCCESS;
 }
 
-int
-opal_iftupletoaddr(const char *inaddr, uint32_t *net, uint32_t *mask)
+int opal_iftupletoaddr(const char *inaddr, uint32_t *net, uint32_t *mask)
 {
     int pval, dots, rc = OPAL_SUCCESS;
     const char *ptr;
@@ -538,14 +531,14 @@ opal_iftupletoaddr(const char *inaddr, uint32_t *net, uint32_t *mask)
 
         /* if entry includes mask, split that off */
         if (NULL != (ptr = strchr(inaddr, '/'))) {
-            ptr = ptr + 1;  /* skip the / */
+            ptr = ptr + 1; /* skip the / */
             /* is the mask a tuple? */
             if (NULL != strchr(ptr, '.')) {
                 /* yes - extract mask from it */
                 rc = parse_ipv4_dots(ptr, mask, &dots);
             } else {
                 /* no - must be an int telling us how much of the addr to use: e.g., /16
-                 * For more information please read http://en.wikipedia.org/wiki/Subnetwork.
+                 * For more information please read https://en.wikipedia.org/wiki/Subnetwork.
                  */
                 pval = strtol(ptr, NULL, 10);
                 if ((pval > 31) || (pval < 1)) {
@@ -556,7 +549,7 @@ opal_iftupletoaddr(const char *inaddr, uint32_t *net, uint32_t *mask)
             }
         } else {
             /* use the number of dots to determine it */
-            for (ptr = inaddr, pval = 0; '\0'!= *ptr; ptr++) {
+            for (ptr = inaddr, pval = 0; '\0' != *ptr; ptr++) {
                 if ('.' == *ptr) {
                     pval++;
                 }
@@ -567,11 +560,11 @@ opal_iftupletoaddr(const char *inaddr, uint32_t *net, uint32_t *mask)
              */
             if (3 == pval) {
                 *mask = 0xFFFFFFFF;
-            } else if (2 == pval) {         /* 2 dots */
+            } else if (2 == pval) { /* 2 dots */
                 *mask = 0xFFFFFF00;
-            } else if (1 == pval) {  /* 1 dot */
+            } else if (1 == pval) { /* 1 dot */
                 *mask = 0xFFFF0000;
-            } else if (0 == pval) {  /* no dots */
+            } else if (0 == pval) { /* no dots */
                 *mask = 0xFF000000;
             } else {
                 opal_output(0, "opal_iftupletoaddr: unknown mask");
@@ -595,9 +588,9 @@ opal_iftupletoaddr(const char *inaddr, uint32_t *net, uint32_t *mask)
 
 bool opal_ifisloopback(int if_index)
 {
-    opal_if_t* intf;
+    opal_if_t *intf;
 
-    OPAL_LIST_FOREACH(intf, &opal_if_list, opal_if_t) {
+    OPAL_LIST_FOREACH (intf, &opal_if_list, opal_if_t) {
         if (intf->if_index == if_index) {
             if ((intf->if_flags & IFF_LOOPBACK) != 0) {
                 return true;
@@ -621,17 +614,18 @@ int opal_ifmatches(int kidx, char **nets)
     uint32_t addr, netaddr, netmask;
 
     /* get the address info for the given network in case we need it */
-    if (OPAL_SUCCESS != (rc = opal_ifkindextoaddr(kidx, (struct sockaddr*)&inaddr, sizeof(inaddr)))) {
+    if (OPAL_SUCCESS
+        != (rc = opal_ifkindextoaddr(kidx, (struct sockaddr *) &inaddr, sizeof(inaddr)))) {
         return rc;
     }
     addr = ntohl(inaddr.sin_addr.s_addr);
 
-    for (i=0; NULL != nets[i]; i++) {
+    for (i = 0; NULL != nets[i]; i++) {
         /* if the specified interface contains letters in it, then it
          * was given as an interface name and not an IP tuple
          */
         named_if = false;
-        for (j=0; j < strlen(nets[i]); j++) {
+        for (j = 0; j < strlen(nets[i]); j++) {
             if (isalpha(nets[i][j]) && '.' != nets[i][j]) {
                 named_if = true;
                 break;
@@ -660,19 +654,19 @@ int opal_ifmatches(int kidx, char **nets)
 
 void opal_ifgetaliases(char ***aliases)
 {
-    opal_if_t* intf;
+    opal_if_t *intf;
     char ipv4[INET_ADDRSTRLEN];
     struct sockaddr_in *addr;
-#if OPAL_ENABLE_IPV6
+#    if OPAL_ENABLE_IPV6
     char ipv6[INET6_ADDRSTRLEN];
     struct sockaddr_in6 *addr6;
-#endif
+#    endif
 
     /* set default answer */
     *aliases = NULL;
 
-    OPAL_LIST_FOREACH(intf, &opal_if_list, opal_if_t) {
-        addr = (struct sockaddr_in*) &intf->if_addr;
+    OPAL_LIST_FOREACH (intf, &opal_if_list, opal_if_t) {
+        addr = (struct sockaddr_in *) &intf->if_addr;
         /* ignore purely loopback interfaces */
         if ((intf->if_flags & IFF_LOOPBACK) != 0) {
             continue;
@@ -681,13 +675,13 @@ void opal_ifgetaliases(char ***aliases)
             inet_ntop(AF_INET, &(addr->sin_addr.s_addr), ipv4, INET_ADDRSTRLEN);
             opal_argv_append_nosize(aliases, ipv4);
         }
-#if OPAL_ENABLE_IPV6
+#    if OPAL_ENABLE_IPV6
         else {
-            addr6 = (struct sockaddr_in6*) &intf->if_addr;
+            addr6 = (struct sockaddr_in6 *) &intf->if_addr;
             inet_ntop(AF_INET6, &(addr6->sin6_addr), ipv6, INET6_ADDRSTRLEN);
             opal_argv_append_nosize(aliases, ipv6);
         }
-#endif
+#    endif
     }
 }
 
@@ -696,88 +690,72 @@ void opal_ifgetaliases(char ***aliases)
 /* if we don't have struct sockaddr_in, we don't have traditional
    ethernet devices.  Just make everything a no-op error call */
 
-int
-opal_ifnametoaddr(const char* if_name,
-                  struct sockaddr* if_addr, int size)
+int opal_ifnametoaddr(const char *if_name, struct sockaddr *if_addr, int size)
 {
     return OPAL_ERR_NOT_SUPPORTED;
 }
 
-int
-opal_ifaddrtoname(const char* if_addr,
-                  char* if_name, int size)
+int opal_ifaddrtoname(const char *if_addr, char *if_name, int size)
 {
     return OPAL_ERR_NOT_SUPPORTED;
 }
 
-int
-opal_ifnametoindex(const char* if_name)
+int opal_ifnametoindex(const char *if_name)
 {
     return OPAL_ERR_NOT_SUPPORTED;
 }
 
-int
-opal_ifnametokindex(const char* if_name)
+int opal_ifnametokindex(const char *if_name)
 {
     return OPAL_ERR_NOT_SUPPORTED;
 }
 
-int
-opal_ifindextokindex(int if_index)
+int opal_ifindextokindex(int if_index)
 {
     return OPAL_ERR_NOT_SUPPORTED;
 }
 
-int
-opal_ifcount(void)
+int opal_ifcount(void)
 {
     return OPAL_ERR_NOT_SUPPORTED;
 }
 
-int
-opal_ifbegin(void)
+int opal_ifbegin(void)
 {
     return OPAL_ERR_NOT_SUPPORTED;
 }
 
-int
-opal_ifnext(int if_index)
+int opal_ifnext(int if_index)
 {
     return OPAL_ERR_NOT_SUPPORTED;
 }
 
-int
-opal_ifindextoname(int if_index, char* if_name, int length)
+int opal_ifindextoname(int if_index, char *if_name, int length)
 {
     return OPAL_ERR_NOT_SUPPORTED;
 }
 
-int
-opal_ifkindextoname(int kif_index, char* if_name, int length)
+int opal_ifkindextoname(int kif_index, char *if_name, int length)
 {
     return OPAL_ERR_NOT_SUPPORTED;
 }
 
-int
-opal_ifindextoaddr(int if_index, struct sockaddr* if_addr, unsigned int length)
+int opal_ifindextoaddr(int if_index, struct sockaddr *if_addr, unsigned int length)
 {
     return OPAL_ERR_NOT_SUPPORTED;
 }
 
-int
-opal_ifindextomask(int if_index, uint32_t* if_addr, int length)
+int opal_ifindextomask(int if_index, uint32_t *if_addr, int length)
 {
     return OPAL_ERR_NOT_SUPPORTED;
 }
 
-bool
-opal_ifislocal(const char *hostname)
+bool opal_ifislocal(const char *hostname)
 {
     return false;
 }
 
-int
-opal_iftupletoaddr(const char *inaddr, uint32_t *net, uint32_t *mask)
+int opal_iftupletoaddr(const char *inaddr, uint32_t *net, uint32_t *mask)
 {
     return 0;
 }
@@ -794,4 +772,3 @@ void opal_ifgetaliases(char ***aliases)
 }
 
 #endif /* HAVE_STRUCT_SOCKADDR_IN */
-
diff --git a/opal/util/if.h b/opal/util/if.h
index 5165abd2e6d..2fc83ecf77a 100644
--- a/opal/util/if.h
+++ b/opal/util/if.h
@@ -28,13 +28,13 @@
 #include "opal_config.h"
 
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 #ifdef HAVE_SYS_SOCKET_H
-#include <sys/socket.h>
+#    include <sys/socket.h>
 #endif
 #ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
+#    include <netinet/in.h>
 #endif
 
 /*
@@ -49,15 +49,13 @@
 
 BEGIN_C_DECLS
 
-#define OPAL_IF_FORMAT_ADDR(n)                              \
-    (((n) >> 24) & 0x000000FF), (((n) >> 16) & 0x000000FF), \
-    (((n) >> 8) & 0x000000FF), ((n) & 0x000000FF)
+#define OPAL_IF_FORMAT_ADDR(n)                                                         \
+    (((n) >> 24) & 0x000000FF), (((n) >> 16) & 0x000000FF), (((n) >> 8) & 0x000000FF), \
+        ((n) &0x000000FF)
 
-#define OPAL_IF_ASSEMBLE_NETWORK(n1, n2, n3, n4)    \
-    (((n1) << 24) & 0xFF000000) |                   \
-    (((n2) << 16) & 0x00FF0000) |                   \
-    (((n3) <<  8) & 0x0000FF00) |                   \
-    ( (n4)        & 0x000000FF)
+#define OPAL_IF_ASSEMBLE_NETWORK(n1, n2, n3, n4)                                           \
+    (((n1) << 24) & 0xFF000000) | (((n2) << 16) & 0x00FF0000) | (((n3) << 8) & 0x0000FF00) \
+        | ((n4) &0x000000FF)
 
 /**
  *  Lookup an interface by name and return its primary address.
@@ -66,9 +64,7 @@ BEGIN_C_DECLS
  *  @param if_addr (OUT)  Interface address buffer
  *  @param size    (IN)   Interface address buffer size
  */
-OPAL_DECLSPEC int opal_ifnametoaddr(const char* if_name,
-                                    struct sockaddr* if_addr,
-                                    int size);
+OPAL_DECLSPEC int opal_ifnametoaddr(const char *if_name, struct sockaddr *if_addr, int size);
 
 /**
  *  Lookup an interface by address and return its name.
@@ -77,8 +73,7 @@ OPAL_DECLSPEC int opal_ifnametoaddr(const char* if_name,
  *  @param if_name (OUT)  Interface name buffer
  *  @param size    (IN)   Interface name buffer size
  */
-OPAL_DECLSPEC int opal_ifaddrtoname(const char* if_addr,
-                                    char* if_name, int size);
+OPAL_DECLSPEC int opal_ifaddrtoname(const char *if_addr, char *if_name, int size);
 
 /**
  *  Lookup an interface by name and return its opal_list index.
@@ -86,7 +81,7 @@ OPAL_DECLSPEC int opal_ifaddrtoname(const char* if_addr,
  *  @param if_name (IN)  Interface name
  *  @return              Interface opal_list index
  */
-OPAL_DECLSPEC int opal_ifnametoindex(const char* if_name);
+OPAL_DECLSPEC int opal_ifnametoindex(const char *if_name);
 
 /**
  *  Lookup an interface by name and return its kernel index.
@@ -94,14 +89,14 @@ OPAL_DECLSPEC int opal_ifnametoindex(const char* if_name);
  *  @param if_name (IN)  Interface name
  *  @return              Interface kernel index
  */
-OPAL_DECLSPEC int opal_ifnametokindex(const char* if_name);
+OPAL_DECLSPEC int opal_ifnametokindex(const char *if_name);
 
 /*
  *  Attempt to resolve an address (given as either IPv4/IPv6 string
  *  or hostname) and return the kernel index of the interface
  *  that is on the same network as the specified address
  */
-OPAL_DECLSPEC int opal_ifaddrtokindex(const char* if_addr);
+OPAL_DECLSPEC int opal_ifaddrtokindex(const char *if_addr);
 
 /**
  *  Lookup an interface by opal_list index and return its kernel index.
@@ -137,7 +132,7 @@ OPAL_DECLSPEC int opal_ifnext(int if_index);
  *  @param if_name (OUT)  Interface name buffer
  *  @param size (IN)      Interface name buffer size
  */
-OPAL_DECLSPEC int opal_ifindextoname(int if_index, char* if_name, int);
+OPAL_DECLSPEC int opal_ifindextoname(int if_index, char *if_name, int);
 
 /**
  *  Lookup an interface by kernel index and return its name.
@@ -146,7 +141,7 @@ OPAL_DECLSPEC int opal_ifindextoname(int if_index, char* if_name, int);
  *  @param if_name (OUT)  Interface name buffer
  *  @param size (IN)      Interface name buffer size
  */
-OPAL_DECLSPEC int opal_ifkindextoname(int if_kindex, char* if_name, int);
+OPAL_DECLSPEC int opal_ifkindextoname(int if_kindex, char *if_name, int);
 
 /**
  *  Lookup an interface by index and return its primary address.
@@ -155,11 +150,8 @@ OPAL_DECLSPEC int opal_ifkindextoname(int if_kindex, char* if_name, int);
  *  @param if_name (OUT)  Interface address buffer
  *  @param size (IN)      Interface address buffer size
  */
-OPAL_DECLSPEC int opal_ifindextoaddr(int if_index, struct sockaddr*,
-                                     unsigned int);
-OPAL_DECLSPEC int opal_ifkindextoaddr(int if_kindex,
-                                      struct sockaddr* if_addr,
-                                      unsigned int length);
+OPAL_DECLSPEC int opal_ifindextoaddr(int if_index, struct sockaddr *, unsigned int);
+OPAL_DECLSPEC int opal_ifkindextoaddr(int if_kindex, struct sockaddr *if_addr, unsigned int length);
 
 /**
  *  Lookup an interface by index and return its network mask (in CIDR
@@ -169,7 +161,7 @@ OPAL_DECLSPEC int opal_ifkindextoaddr(int if_kindex,
  *  @param if_name (OUT)  Interface address buffer
  *  @param size (IN)      Interface address buffer size
  */
-OPAL_DECLSPEC int opal_ifindextomask(int if_index, uint32_t*, int);
+OPAL_DECLSPEC int opal_ifindextomask(int if_index, uint32_t *, int);
 
 /**
  *  Lookup an interface by index and return its MAC address.
@@ -193,7 +185,7 @@ OPAL_DECLSPEC int opal_ifindextomtu(int if_index, int *mtu);
  *  @param if_index (IN)  Interface index
  *  @param if_flags (OUT) Interface flags
  */
-OPAL_DECLSPEC int opal_ifindextoflags(int if_index, uint32_t*);
+OPAL_DECLSPEC int opal_ifindextoflags(int if_index, uint32_t *);
 
 /**
  * Determine if given hostname / IP address is a local address
@@ -234,4 +226,3 @@ OPAL_DECLSPEC void opal_ifgetaliases(char ***aliases);
 END_C_DECLS
 
 #endif
-
diff --git a/opal/util/info.c b/opal/util/info.c
index e77384ea985..d65fd2349a2 100644
--- a/opal/util/info.c
+++ b/opal/util/info.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2007 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -25,24 +25,24 @@
  * $HEADER$
  */
 
-#include <string.h>
 #include <errno.h>
 #include <stdlib.h>
+#include <string.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
-#include <limits.h>
 #include <ctype.h>
+#include <limits.h>
 #ifdef HAVE_SYS_UTSNAME_H
-#include <sys/utsname.h>
+#    include <sys/utsname.h>
 #endif
 #include <assert.h>
 
 #include "opal/util/argv.h"
+#include "opal/util/info.h"
 #include "opal/util/opal_getcwd.h"
 #include "opal/util/output.h"
 #include "opal/util/string_copy.h"
-#include "opal/util/info.h"
 
 /*
  * Local functions
@@ -51,336 +51,247 @@ static void info_constructor(opal_info_t *info);
 static void info_destructor(opal_info_t *info);
 static void info_entry_constructor(opal_info_entry_t *entry);
 static void info_entry_destructor(opal_info_entry_t *entry);
-static opal_info_entry_t *info_find_key (opal_info_t *info, const char *key);
-
+static opal_info_entry_t *info_find_key(opal_info_t *info, const char *key);
 
 /*
  * opal_info_t classes
  */
-OBJ_CLASS_INSTANCE(opal_info_t,
-                   opal_list_t,
-                   info_constructor,
-                   info_destructor);
+OBJ_CLASS_INSTANCE(opal_info_t, opal_list_t, info_constructor, info_destructor);
 
 /*
  * opal_info_entry_t classes
  */
-OBJ_CLASS_INSTANCE(opal_info_entry_t,
-                   opal_list_item_t,
-                   info_entry_constructor,
+OBJ_CLASS_INSTANCE(opal_info_entry_t, opal_list_item_t, info_entry_constructor,
                    info_entry_destructor);
 
-
-
 /*
- * Duplicate an info
+ * Duplicate an info into newinfo. If public_info is true we only duplicate
+ * key-value pairs that are not internal and that had been referenced,
+ * either through opal_info_get or opal_info_set.
  */
-int opal_info_dup (opal_info_t *info, opal_info_t **newinfo)
+static int opal_info_dup_impl(opal_info_t *info, opal_info_t **newinfo, bool public_only)
 {
-    int err;
     opal_info_entry_t *iterator;
 
     OPAL_THREAD_LOCK(info->i_lock);
-    OPAL_LIST_FOREACH(iterator, &info->super, opal_info_entry_t) {
-        err = opal_info_set(*newinfo, iterator->ie_key, iterator->ie_value);
-        if (OPAL_SUCCESS != err) {
-            OPAL_THREAD_UNLOCK(info->i_lock);
-            return err;
+    OPAL_LIST_FOREACH (iterator, &info->super, opal_info_entry_t) {
+        /* skip keys that are internal if we didn't ask for them */
+        if (public_only && (iterator->ie_internal || iterator->ie_referenced == 0)) continue;
+        /* create a new info entry and retain the string objects */
+        opal_info_entry_t *newentry = OBJ_NEW(opal_info_entry_t);
+        newentry->ie_key = iterator->ie_key;
+        OBJ_RETAIN(iterator->ie_key);
+        newentry->ie_value = iterator->ie_value;
+        if (public_only) {
+            /* for public info-dup we also duplicate the references so that subsequent
+             * duplications see the same info keys */
+            newentry->ie_referenced = iterator->ie_referenced;
         }
-     }
+        OBJ_RETAIN(iterator->ie_value);
+        opal_list_append (&((*newinfo)->super), (opal_list_item_t *) newentry);
+    }
     OPAL_THREAD_UNLOCK(info->i_lock);
     return OPAL_SUCCESS;
 }
 
-static void opal_info_get_nolock (opal_info_t *info, const char *key, int valuelen,
-                                 char *value, int *flag)
+int opal_info_dup_public(opal_info_t *info, opal_info_t **newinfo)
+{
+    return opal_info_dup_impl(info, newinfo, true);
+}
+
+int opal_info_dup(opal_info_t *info, opal_info_t **newinfo)
+{
+    return opal_info_dup_impl(info, newinfo, false);
+}
+
+static void opal_info_get_nolock(opal_info_t *info, const char *key, opal_cstring_t **value,
+                                 int *flag)
 {
     opal_info_entry_t *search;
 
-    search = info_find_key (info, key);
-    if (NULL == search){
+    search = info_find_key(info, key);
+    if (NULL == search) {
         *flag = 0;
-    } else if (value && valuelen) {
+    } else {
         /*
          * We have found the element, so we can return the value
          * Set the flag and value
          */
         *flag = 1;
-        // Note: we copy exactly (valuelen) characters, because that's
-        // what the caller asked for.  Don't use opal_string_copy()
-        // here, because that will guarantee to \0-terminate what is
-        // copied (i.e., potentially copy (valuelen-1) chars and then
-        // an additional \0).  Instead: copy over exactly (valuelen)
-        // characters, and if that's not \0-terminated, then so be it.
-        memcpy(value, search->ie_value, valuelen);
+        if (NULL != value) {
+            OBJ_RETAIN(search->ie_value);
+            *value = search->ie_value;
+            search->ie_referenced++;
+        }
     }
 }
 
-static int opal_info_set_nolock (opal_info_t *info, const char *key, const char *value)
+static int opal_info_set_cstring_nolock(opal_info_t *info, const char *key, opal_cstring_t *value)
 {
-    char *new_value;
-    opal_info_entry_t *new_info;
     opal_info_entry_t *old_info;
 
-    new_value = strdup(value);
-    if (NULL == new_value) {
-      return OPAL_ERR_OUT_OF_RESOURCE;
-    }
-
-    old_info = info_find_key (info, key);
+    old_info = info_find_key(info, key);
     if (NULL != old_info) {
         /*
          * key already exists. remove the value associated with it
          */
-        free(old_info->ie_value);
-        old_info->ie_value = new_value;
+        OBJ_RELEASE(old_info->ie_value);
+        OBJ_RETAIN(value);
+        old_info->ie_value = value;
+        old_info->ie_referenced++;
     } else {
+        opal_info_entry_t *new_info;
         new_info = OBJ_NEW(opal_info_entry_t);
         if (NULL == new_info) {
-            free(new_value);
-            OPAL_THREAD_UNLOCK(info->i_lock);
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
-        opal_string_copy (new_info->ie_key, key, OPAL_MAX_INFO_KEY);
-        new_info->ie_value = new_value;
-        opal_list_append (&(info->super), (opal_list_item_t *) new_info);
+        opal_cstring_t *key_str = opal_cstring_create(key);
+        new_info->ie_key = key_str;
+        OBJ_RETAIN(value);
+        new_info->ie_value = value;
+        new_info->ie_referenced++;
+        opal_list_append(&(info->super), (opal_list_item_t *) new_info);
     }
     return OPAL_SUCCESS;
 }
-/*
- * An object's info can be set, but those settings can be modified by
- * system callbacks. When those callbacks happen, we save a "__IN_<key>"/"val"
- * copy of changed or erased values.
- *
- * extra options for how to dup:
- *   include_system_extras (default 1)
- *   omit_ignored          (default 1)
- *   show_modifications    (default 0)
- */
-static
-int opal_info_dup_mode (opal_info_t *info, opal_info_t **newinfo,
-    int include_system_extras,  // (k/v with no corresponding __IN_k)
-    int omit_ignored,           // (__IN_k with no k/v)
-    int show_modifications)     // (pick v from k/v or __IN_k/v)
+
+static int opal_info_set_nolock(opal_info_t *info, const char *key, const char *value, bool internal)
 {
-    int err, flag;
-    opal_info_entry_t *iterator;
-    char savedkey[OPAL_MAX_INFO_KEY + 1]; // iterator->ie_key has this as its size
-    char savedval[OPAL_MAX_INFO_VAL];
-    char *valptr, *pkey;
-    int is_IN_key;
-    int exists_IN_key, exists_reg_key;
+    opal_info_entry_t *old_info;
 
-    OPAL_THREAD_LOCK(info->i_lock);
-    OPAL_LIST_FOREACH(iterator, &info->super, opal_info_entry_t) {
-// If we see an __IN_<key> key but no <key>, decide what to do based on mode.
-// If we see an __IN_<key> and a <key>, skip since it'll be handled when
-// we process <key>.
-         is_IN_key = 0;
-         exists_IN_key = 0;
-         exists_reg_key = 0;
-         pkey = iterator->ie_key;
-         if (0 == strncmp(iterator->ie_key, OPAL_INFO_SAVE_PREFIX,
-             strlen(OPAL_INFO_SAVE_PREFIX)))
-        {
-             pkey += strlen(OPAL_INFO_SAVE_PREFIX);
-
-             is_IN_key = 1;
-             exists_IN_key = 1;
-             opal_info_get_nolock (info, pkey, 0, NULL, &flag);
-             if (flag) {
-                 exists_reg_key = 1;
-             }
-         } else {
-             is_IN_key = 0;
-             exists_reg_key = 1;
-
-// see if there is an __IN_<key> for the current <key>
-             if (strlen(OPAL_INFO_SAVE_PREFIX) + strlen(pkey) < OPAL_MAX_INFO_KEY) {
-                 snprintf(savedkey, OPAL_MAX_INFO_KEY+1,
-                     OPAL_INFO_SAVE_PREFIX "%s", pkey);
-// (the prefix macro is a string, so the unreadable part above is a string concatenation)
-                 opal_info_get_nolock (info, savedkey, OPAL_MAX_INFO_VAL,
-                                       savedval, &flag);
-             } else {
-                 flag = 0;
-             }
-             if (flag) {
-                 exists_IN_key = 1;
-             }
-         }
-
-         if (is_IN_key) {
-             if (exists_reg_key) {
-// we're processing __IN_<key> and there exists a <key> so we'll handle it then
-                 continue;
-             } else {
-// we're processing __IN_<key> and no <key> exists
-// this would mean <key> was set by the user but ignored by the system
-// so base our behavior on the omit_ignored
-                 if (!omit_ignored) {
-                     err = opal_info_set_nolock(*newinfo, pkey, iterator->ie_value);
-                     if (OPAL_SUCCESS != err) {
-                         OPAL_THREAD_UNLOCK(info->i_lock);
-                         return err;
-                     }
-                 }
-             }
-         } else {
-             valptr = 0;
-             if (!exists_IN_key) {
-// we're processing <key> and no __IN_<key> <key> exists
-// this would mean it's a system setting, not something that came from the user
-                 if (include_system_extras) {
-                     valptr = iterator->ie_value;
-                 }
-             } else {
-// we're processing <key> and __IN_<key> also exists
-// pick which value to use
-                 if (!show_modifications) {
-                     valptr = savedval;
-                 } else {
-                     valptr = iterator->ie_value;
-                 }
-             }
-             if (valptr) {
-                 err = opal_info_set_nolock(*newinfo, pkey, valptr);
-                 if (OPAL_SUCCESS != err) {
-                     OPAL_THREAD_UNLOCK(info->i_lock);
-                     return err;
-                 }
-             }
-         }
-     }
-     OPAL_THREAD_UNLOCK(info->i_lock);
-     return OPAL_SUCCESS;
+    old_info = info_find_key(info, key);
+    if (NULL != old_info) {
+        /*
+         * key already exists, check whether it is the same
+         */
+        size_t value_len = strlen(value);
+        old_info->ie_referenced++;
+        old_info->ie_internal = internal;
+        if (old_info->ie_value->length == value_len
+            && 0 == strcmp(old_info->ie_value->string, value)) {
+            return OPAL_SUCCESS;
+        }
+        /* value is different so replace it */
+        OBJ_RELEASE(old_info->ie_value);
+        old_info->ie_value = opal_cstring_create_l(value, value_len);
+        if (NULL == old_info->ie_value) {
+            OBJ_RELEASE(old_info);
+            return OPAL_ERR_OUT_OF_RESOURCE;
+        }
+    } else {
+        opal_info_entry_t *new_info;
+        new_info = OBJ_NEW(opal_info_entry_t);
+        if (NULL == new_info) {
+            return OPAL_ERR_OUT_OF_RESOURCE;
+        }
+        new_info->ie_key = opal_cstring_create(key);
+        new_info->ie_value = opal_cstring_create(value);
+        if (NULL == new_info->ie_key || NULL == new_info->ie_value) {
+            OBJ_RELEASE(new_info);
+            return OPAL_ERR_OUT_OF_RESOURCE;
+        }
+        new_info->ie_referenced++;
+        new_info->ie_internal = internal;
+        opal_list_append(&(info->super), (opal_list_item_t *) new_info);
+    }
+    return OPAL_SUCCESS;
 }
 
 /*
- * Implement opal_info_dup_mpistandard by using whatever mode
- * settings represent our interpretation of the standard
+ * Set a value on the info
  */
-int opal_info_dup_mpistandard (opal_info_t *info, opal_info_t **newinfo)
+int opal_info_set(opal_info_t *info, const char *key, const char *value)
 {
-    return opal_info_dup_mode (info, newinfo, 1, 1, 0);
+    int ret;
+
+    OPAL_THREAD_LOCK(info->i_lock);
+    ret = opal_info_set_nolock(info, key, value, false);
+    OPAL_THREAD_UNLOCK(info->i_lock);
+    return ret;
 }
 
 /*
  * Set a value on the info
  */
-int opal_info_set (opal_info_t *info, const char *key, const char *value)
+int opal_info_set_internal(opal_info_t *info, const char *key, const char *value)
 {
     int ret;
 
     OPAL_THREAD_LOCK(info->i_lock);
-    ret = opal_info_set_nolock(info, key, value);
+    ret = opal_info_set_nolock(info, key, value, true);
     OPAL_THREAD_UNLOCK(info->i_lock);
     return ret;
 }
 
+int opal_info_set_cstring(opal_info_t *info, const char *key, opal_cstring_t *value)
+{
+    int ret;
+
+    OPAL_THREAD_LOCK(info->i_lock);
+    ret = opal_info_set_cstring_nolock(info, key, value);
+    OPAL_THREAD_UNLOCK(info->i_lock);
+    return ret;
+}
 
-int opal_info_set_value_enum (opal_info_t *info, const char *key, int value,
-                              mca_base_var_enum_t *var_enum)
+int opal_info_set_value_enum(opal_info_t *info, const char *key, int value,
+                             mca_base_var_enum_t *var_enum)
 {
     char *string_value;
     int ret;
 
-    ret = var_enum->string_from_value (var_enum, value, &string_value);
+    ret = var_enum->string_from_value(var_enum, value, &string_value);
     if (OPAL_SUCCESS != ret) {
         return ret;
     }
 
-    return opal_info_set (info, key, string_value);
+    return opal_info_set(info, key, string_value);
 }
 
-
 /*
  * Get a value from an info
  */
-int opal_info_get (opal_info_t *info, const char *key, int valuelen,
-                   char *value, int *flag)
+int opal_info_get(opal_info_t *info, const char *key, opal_cstring_t **value, int *flag)
 {
     OPAL_THREAD_LOCK(info->i_lock);
-    opal_info_get_nolock(info, key, valuelen, value, flag);
+    opal_info_get_nolock(info, key, value, flag);
     OPAL_THREAD_UNLOCK(info->i_lock);
     return OPAL_SUCCESS;
 }
 
-int opal_info_get_value_enum (opal_info_t *info, const char *key, int *value,
-                              int default_value, mca_base_var_enum_t *var_enum,
-                              int *flag)
+int opal_info_get_value_enum(opal_info_t *info, const char *key, int *value, int default_value,
+                             mca_base_var_enum_t *var_enum, int *flag)
 {
-    opal_info_entry_t *search;
     int ret;
+    opal_cstring_t *str;
 
     *value = default_value;
 
-    OPAL_THREAD_LOCK(info->i_lock);
-    search = info_find_key (info, key);
-    if (NULL == search){
-        OPAL_THREAD_UNLOCK(info->i_lock);
-        *flag = 0;
-        return OPAL_SUCCESS;
-    }
-
-    /* we found a mathing key. pass the string value to the enumerator and
-     * return */
-    *flag = 1;
+    ret = opal_info_get(info, key, &str, flag);
 
-    ret = var_enum->value_from_string (var_enum, search->ie_value, value);
-    OPAL_THREAD_UNLOCK(info->i_lock);
+    if (*flag) {
+        ret = var_enum->value_from_string(var_enum, str->string, value);
+        OBJ_RELEASE(str);
+    }
 
     return ret;
 }
 
-
 /*
  * Similar to opal_info_get(), but cast the result into a boolean
  * using some well-defined rules.
  */
 int opal_info_get_bool(opal_info_t *info, const char *key, bool *value, int *flag)
 {
-    char str[256];
+    int ret;
+    opal_cstring_t *str;
 
-    str[sizeof(str) - 1] = '\0';
-    opal_info_get(info, key, sizeof(str) - 1, str, flag);
+    ret = opal_info_get(info, key, &str, flag);
     if (*flag) {
-        *value = opal_str_to_bool(str);
+        ret = opal_cstring_to_bool(str, value);
+        OBJ_RELEASE(str);
     }
 
-    return OPAL_SUCCESS;
-}
-
-
-bool
-opal_str_to_bool(char *str)
-{
-    bool result = false;
-    char *ptr;
-
-    /* Trim whitespace */
-    ptr = str + strlen(str) - 1;
-    while (ptr >= str && isspace(*ptr)) {
-        *ptr = '\0';
-        --ptr;
-    }
-    ptr = str;
-    while (ptr < str + strlen(str) - 1 && *ptr != '\0' &&
-           isspace(*ptr)) {
-        ++ptr;
-    }
-    if ('\0' != *ptr) {
-        if (isdigit(*ptr)) {
-            result = (bool) atoi(ptr);
-        } else if (0 == strcasecmp(ptr, "yes") ||
-                   0 == strcasecmp(ptr, "true")) {
-            result = true;
-        } else if (0 != strcasecmp(ptr, "no") &&
-                   0 != strcasecmp(ptr, "false")) {
-            /* RHC unrecognized value -- print a warning? */
-        }
-    }
-    return result;
+    return ret;
 }
 
 /*
@@ -391,54 +302,50 @@ int opal_info_delete(opal_info_t *info, const char *key)
     opal_info_entry_t *search;
 
     OPAL_THREAD_LOCK(info->i_lock);
-    search = info_find_key (info, key);
-    if (NULL == search){
-         OPAL_THREAD_UNLOCK(info->i_lock);
-         return OPAL_ERR_NOT_FOUND;
+    search = info_find_key(info, key);
+    if (NULL == search) {
+        OPAL_THREAD_UNLOCK(info->i_lock);
+        return OPAL_ERR_NOT_FOUND;
     } else {
-         /*
-          * An entry with this key value was found. Remove the item
-          * and free the memory allocated to it.
-          * As this key *must* be available, we do not check for errors.
-          */
-          opal_list_remove_item (&(info->super),
-                                 (opal_list_item_t *)search);
-          OBJ_RELEASE(search);
+        /*
+         * An entry with this key value was found. Remove the item
+         * and free the memory allocated to it.
+         * As this key *must* be available, we do not check for errors.
+         */
+        opal_list_remove_item(&(info->super), (opal_list_item_t *) search);
+        OBJ_RELEASE(search);
     }
     OPAL_THREAD_UNLOCK(info->i_lock);
     return OPAL_SUCCESS;
 }
 
-
 /*
  * Return the length of a value
  */
-int opal_info_get_valuelen (opal_info_t *info, const char *key, int *valuelen,
-                            int *flag)
+int opal_info_get_valuelen(opal_info_t *info, const char *key, int *valuelen, int *flag)
 {
     opal_info_entry_t *search;
 
     OPAL_THREAD_LOCK(info->i_lock);
-    search = info_find_key (info, key);
-    if (NULL == search){
+    search = info_find_key(info, key);
+    if (NULL == search) {
         *flag = 0;
     } else {
         /*
          * We have found the element, so we can return the value
          * Set the flag, value_length and value
          */
-         *flag = 1;
-         *valuelen = strlen(search->ie_value);
+        *flag = 1;
+        *valuelen = search->ie_value->length;
     }
     OPAL_THREAD_UNLOCK(info->i_lock);
     return OPAL_SUCCESS;
 }
 
-
 /*
  * Get the nth key
  */
-int opal_info_get_nthkey (opal_info_t *info, int n, char *key)
+int opal_info_get_nthkey(opal_info_t *info, int n, opal_cstring_t **key)
 {
     opal_info_entry_t *iterator;
 
@@ -446,28 +353,19 @@ int opal_info_get_nthkey (opal_info_t *info, int n, char *key)
      * Iterate over and over till we get to the nth key
      */
     OPAL_THREAD_LOCK(info->i_lock);
-    for (iterator = (opal_info_entry_t *)opal_list_get_first(&(info->super));
-         n > 0;
-         --n) {
-         iterator = (opal_info_entry_t *)opal_list_get_next(iterator);
-         if (opal_list_get_end(&(info->super)) ==
-             (opal_list_item_t *) iterator) {
-             OPAL_THREAD_UNLOCK(info->i_lock);
-             return OPAL_ERR_BAD_PARAM;
-         }
+    for (iterator = (opal_info_entry_t *) opal_list_get_first(&(info->super)); n > 0; --n) {
+        iterator = (opal_info_entry_t *) opal_list_get_next(iterator);
+        if (opal_list_get_end(&(info->super)) == (opal_list_item_t *) iterator) {
+            OPAL_THREAD_UNLOCK(info->i_lock);
+            return OPAL_ERR_BAD_PARAM;
+        }
     }
-    /*
-     * iterator is of the type opal_list_item_t. We have to
-     * cast it to opal_info_entry_t before we can use it to
-     * access the value
-     */
-    opal_string_copy(key, iterator->ie_key, OPAL_MAX_INFO_KEY);
+    OBJ_RETAIN(iterator->ie_key);
+    *key = iterator->ie_key;
     OPAL_THREAD_UNLOCK(info->i_lock);
     return OPAL_SUCCESS;
 }
 
-
-
 /*
  * This function is invoked when OBJ_NEW() is called. Here, we add this
  * info pointer to the table and then store its index as the handle
@@ -489,8 +387,7 @@ static void info_destructor(opal_info_t *info)
 
     /* Remove every key in the list */
 
-    for (item = opal_list_remove_first(&(info->super));
-         NULL != item;
+    for (item = opal_list_remove_first(&(info->super)); NULL != item;
          item = opal_list_remove_first(&(info->super))) {
         iterator = (opal_info_entry_t *) item;
         OBJ_RELEASE(iterator);
@@ -501,32 +398,37 @@ static void info_destructor(opal_info_t *info)
     OBJ_RELEASE(info->i_lock);
 }
 
-
 /*
  * opal_info_entry_t interface functions
  */
 static void info_entry_constructor(opal_info_entry_t *entry)
 {
-    memset(entry->ie_key, 0, sizeof(entry->ie_key));
-    entry->ie_key[OPAL_MAX_INFO_KEY] = 0;
+    entry->ie_key = NULL;
+    entry->ie_value = NULL;
+    entry->ie_referenced = 0;
+    entry->ie_internal = false;
 }
 
-
 static void info_entry_destructor(opal_info_entry_t *entry)
 {
+    if (NULL != entry->ie_key) {
+        OBJ_RELEASE(entry->ie_key);
+        entry->ie_key = NULL;
+    }
+
     if (NULL != entry->ie_value) {
-        free(entry->ie_value);
+        OBJ_RELEASE(entry->ie_value);
+        entry->ie_value = NULL;
     }
 }
 
-
 /*
  * Find a key
  *
  * Do NOT thread lock in here -- the calling function is responsible
  * for that.
  */
-static opal_info_entry_t *info_find_key (opal_info_t *info, const char *key)
+static opal_info_entry_t *info_find_key(opal_info_t *info, const char *key)
 {
     opal_info_entry_t *iterator;
 
@@ -536,61 +438,10 @@ static opal_info_entry_t *info_find_key (opal_info_t *info, const char *key)
      * return immediately. Else, the loop will fall of the edge
      * and NULL is returned
      */
-    OPAL_LIST_FOREACH(iterator, &info->super, opal_info_entry_t) {
-        if (0 == strcmp(key, iterator->ie_key)) {
+    OPAL_LIST_FOREACH (iterator, &info->super, opal_info_entry_t) {
+        if (0 == strcmp(key, iterator->ie_key->string)) {
             return iterator;
         }
     }
     return NULL;
 }
-
-
-int
-opal_info_value_to_int(char *value, int *interp)
-{
-    long tmp;
-    char *endp;
-
-    if (NULL == value || '\0' == value[0]) return OPAL_ERR_BAD_PARAM;
-
-    errno = 0;
-    tmp = strtol(value, &endp, 10);
-    /* we found something not a number */
-    if (*endp != '\0') return OPAL_ERR_BAD_PARAM;
-    /* underflow */
-    if (tmp == 0 && errno == EINVAL) return OPAL_ERR_BAD_PARAM;
-
-    *interp = (int) tmp;
-
-    return OPAL_SUCCESS;
-}
-
-
-int
-opal_info_value_to_bool(char *value, bool *interp)
-{
-    int tmp;
-
-    /* idiot case */
-    if (NULL == value || NULL == interp) return OPAL_ERR_BAD_PARAM;
-
-    /* is it true / false? */
-    if (0 == strcmp(value, "true")) {
-        *interp = true;
-        return OPAL_SUCCESS;
-    } else if (0 == strcmp(value, "false")) {
-        *interp = false;
-        return OPAL_SUCCESS;
-
-    /* is it a number? */
-    } else if (OPAL_SUCCESS == opal_info_value_to_int(value, &tmp)) {
-        if (tmp == 0) {
-            *interp = false;
-        } else {
-            *interp = true;
-        }
-        return OPAL_SUCCESS;
-    }
-
-    return OPAL_ERR_BAD_PARAM;
-}
diff --git a/opal/util/info.h b/opal/util/info.h
index 8521d765f34..5539b09d584 100644
--- a/opal/util/info.h
+++ b/opal/util/info.h
@@ -28,10 +28,11 @@
 
 #include <string.h>
 
+#include "opal/class/opal_cstring.h"
 #include "opal/class/opal_list.h"
 #include "opal/class/opal_pointer_array.h"
-#include "opal/mca/threads/mutex.h"
 #include "opal/mca/base/mca_base_var_enum.h"
+#include "opal/mca/threads/mutex.h"
 
 /**
  * \internal
@@ -39,8 +40,8 @@
  */
 
 struct opal_info_t {
-  opal_list_t   super;
-  opal_mutex_t	*i_lock;
+    opal_list_t super;
+    opal_mutex_t *i_lock;
 };
 
 /**
@@ -49,13 +50,11 @@ struct opal_info_t {
  */
 typedef struct opal_info_t opal_info_t;
 
-
 /**
  * Table for Fortran <-> C translation table
  */
 extern opal_pointer_array_t ompi_info_f_to_c_table;
 
-
 /**
  * \internal
  *
@@ -63,12 +62,14 @@ extern opal_pointer_array_t ompi_info_f_to_c_table;
  * type. It contains (key,value) pairs
  */
 struct opal_info_entry_t {
-    opal_list_item_t super; /**< required for opal_list_t type */
-    char *ie_value; /**< value part of the (key, value) pair.
-                  * Maximum length is MPI_MAX_INFO_VAL */
-    char ie_key[OPAL_MAX_INFO_KEY + 1]; /**< "key" part of the (key, value)
-                                     * pair */
+    opal_list_item_t super;   /**< required for opal_list_t type */
+    opal_cstring_t *ie_value; /**< value part of the (key, value) pair. */
+    opal_cstring_t *ie_key;   /**< "key" part of the (key, value) pair */
+    uint32_t ie_referenced;   /**< number of times this entry was internally
+                                   referenced */
+    bool ie_internal;         /**< internal keys are not handed back to the user */
 };
+
 /**
  * \internal
  * Convenience typedef
@@ -89,11 +90,8 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_info_t);
  */
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_info_entry_t);
 
-
-int opal_mpiinfo_init(void*);
-
 /**
- *   opal_info_dup - Duplicate an 'MPI_Info' object
+ *   opal_info_dup - Duplicate public keys of an 'MPI_Info' object
  *
  *   @param info source info object (handle)
  *   @param newinfo pointer to the new info object (handle)
@@ -104,16 +102,12 @@ int opal_mpiinfo_init(void*);
  *   Not only will the (key, value) pairs be duplicated, the order
  *   of keys will be the same in 'newinfo' as it is in 'info'.  When
  *   an info object is no longer being used, it should be freed with
- *   'MPI_Info_free'.
+ *   \c opal_info_free.
  */
-int opal_info_dup (opal_info_t *info, opal_info_t **newinfo);
-
-// Comments might still say __IN_<key>, but the code should be using the
-// below macro instead.
-#define OPAL_INFO_SAVE_PREFIX "_OMPI_IN_"
+int opal_info_dup_public(opal_info_t *info, opal_info_t **newinfo);
 
 /**
- *   opal_info_dup_mpistandard - Duplicate an 'MPI_Info' object
+ *   opal_info_dup - Duplicate all entries of an 'MPI_Info' object
  *
  *   @param info source info object (handle)
  *   @param newinfo pointer to the new info object (handle)
@@ -121,27 +115,27 @@ int opal_info_dup (opal_info_t *info, opal_info_t **newinfo);
  *   @retval OPAL_SUCCESS upon success
  *   @retval OPAL_ERR_OUT_OF_RESOURCE if out of memory
  *
- *   The user sets an info object with key/value pairs and once processed,
- *   we keep key/val pairs that might have been modified vs what the user
- *   provided, and some user inputs might have been ignored too.  The original
- *   user inpust are kept as __IN_<key>/<val>.
- *
- *   This routine then outputs key/value pairs as:
- *
- *   if <key> and __IN_<key> both exist:
- *       This means the user set a k/v pair and it was used.
- *       output: <key> / value(__IN_<key>), the original user input
- *   if <key> exists but __IN_<key> doesn't:
- *       This is a system-provided setting.
- *       output: <key>/value(<key>)
- *   if __IN_<key> exists but <key> doesn't:
- *       The user provided a setting that was rejected (ignored) by the system
- *       output: nothing for this key
+ *   Not only will the (key, value) pairs be duplicated, the order
+ *   of keys will be the same in 'newinfo' as it is in 'info'.  When
+ *   an info object is no longer being used, it should be freed with
+ *   \c opal_info_free.
  */
-int opal_info_dup_mpistandard (opal_info_t *info, opal_info_t **newinfo);
+int opal_info_dup(opal_info_t *info, opal_info_t **newinfo);
 
 /**
- * Set a new key,value pair on info.
+ * Set a new key,value pair on info and mark it as referenced.
+ *
+ * @param info pointer to opal_info_t object
+ * @param key pointer to the new key object
+ * @param value pointer to the new value object
+ *
+ * @retval OPAL_SUCCESS upon success
+ * @retval OPAL_ERR_OUT_OF_RESOURCE if out of memory
+ */
+OPAL_DECLSPEC int opal_info_set(opal_info_t *info, const char *key, const char *value);
+
+/**
+ * Set a new key,value pair on info and mark it as internal.
  *
  * @param info pointer to opal_info_t object
  * @param key pointer to the new key object
@@ -150,7 +144,22 @@ int opal_info_dup_mpistandard (opal_info_t *info, opal_info_t **newinfo);
  * @retval OPAL_SUCCESS upon success
  * @retval OPAL_ERR_OUT_OF_RESOURCE if out of memory
  */
-OPAL_DECLSPEC int opal_info_set (opal_info_t *info, const char *key, const char *value);
+OPAL_DECLSPEC int opal_info_set_internal(opal_info_t *info, const char *key, const char *value);
+
+/**
+ * Set a new key,value pair on info.
+ *
+ * @param info pointer to opal_info_t object
+ * @param key pointer to the new key string
+ * @param value pointer to the new value string object
+ *
+ * @retval OPAL_SUCCESS upon success
+ * @retval OPAL_ERR_OUT_OF_RESOURCE if out of memory
+ *
+ * The \c value string object will be retained and can be safely released if necessary
+ * by the caller.
+ */
+OPAL_DECLSPEC int opal_info_set_cstring(opal_info_t *info, const char *key, opal_cstring_t *value);
 
 /**
  * Set a new key,value pair from a variable enumerator.
@@ -164,8 +173,8 @@ OPAL_DECLSPEC int opal_info_set (opal_info_t *info, const char *key, const char
  * @retval OPAL_ERR_OUT_OF_RESOURCE if out of memory
  * @retval OPAL_ERR_VALUE_OUT_OF_BOUNDS if the value is not valid in the enumerator
  */
-OPAL_DECLSPEC int opal_info_set_value_enum (opal_info_t *info, const char *key, int value,
-                                            mca_base_var_enum_t *var_enum);
+OPAL_DECLSPEC int opal_info_set_value_enum(opal_info_t *info, const char *key, int value,
+                                           mca_base_var_enum_t *var_enum);
 
 /**
  * opal_info_free - Free an 'MPI_Info' object.
@@ -179,33 +188,34 @@ OPAL_DECLSPEC int opal_info_set_value_enum (opal_info_t *info, const char *key,
  *   'MPI_INFO_NULL'.  Free the info handle and all of its keys and
  *   values.
  */
-int opal_info_free (opal_info_t **info);
+int opal_info_free(opal_info_t **info);
 
-  /**
-   *   Get a (key, value) pair from an 'MPI_Info' object and assign it
-   *   into a boolen output.
-   *
-   *   @param info Pointer to opal_info_t object
-   *   @param key null-terminated character string of the index key
-   *   @param value Boolean output value
-   *   @param flag true (1) if 'key' defined on 'info', false (0) if not
-   *               (logical)
-   *
-   *   @retval OPAL_SUCCESS
-   *
-   *   If found, the string value will be cast to the boolen output in
-   *   the following manner:
-   *
-   *   - If the string value is digits, the return value is "(bool)
-   *     atoi(value)"
-   *   - If the string value is (case-insensitive) "yes" or "true", the
-   *     result is true
-   *   - If the string value is (case-insensitive) "no" or "false", the
-   *     result is false
-   *   - All other values are false
-   */
-OPAL_DECLSPEC int opal_info_get_bool (opal_info_t *info, const char *key, bool *value,
-                                      int *flag);
+/**
+ *   Get a (key, value) pair from an 'MPI_Info' object and assign it
+ *   into a boolean output.
+ *
+ *   This call marks the entry referenced.
+ *
+ *   @param info Pointer to opal_info_t object
+ *   @param key null-terminated character string of the index key
+ *   @param value Boolean output value
+ *   @param flag true (1) if 'key' defined on 'info', false (0) if not
+ *               (logical)
+ *
+ *   @retval OPAL_SUCCESS
+ *
+ *   If found, the string value will be cast to the boolean output in
+ *   the following manner:
+ *
+ *   - If the string value is digits, the return value is "(bool)
+ *     atoi(value)"
+ *   - If the string value is (case-insensitive) "yes" or "true", the
+ *     result is true
+ *   - If the string value is (case-insensitive) "no" or "false", the
+ *     result is false
+ *   - All other values are false
+ */
+OPAL_DECLSPEC int opal_info_get_bool(opal_info_t *info, const char *key, bool *value, int *flag);
 
 /**
  *   Get a (key, value) pair from an 'MPI_Info' object and assign it
@@ -223,27 +233,29 @@ OPAL_DECLSPEC int opal_info_get_bool (opal_info_t *info, const char *key, bool *
  *   @retval OPAL_SUCCESS
  */
 
-OPAL_DECLSPEC int opal_info_get_value_enum (opal_info_t *info, const char *key,
-                                            int *value, int default_value,
-                                            mca_base_var_enum_t *var_enum, int *flag);
+OPAL_DECLSPEC int opal_info_get_value_enum(opal_info_t *info, const char *key, int *value,
+                                           int default_value, mca_base_var_enum_t *var_enum,
+                                           int *flag);
 
 /**
- *   Get a (key, value) pair from an 'MPI_Info' object
+ *   Get a (key, value) pair from an 'MPI_Info' object and mark the entry
+ *   as referenced.
  *
  *   @param info Pointer to opal_info_t object
  *   @param key null-terminated character string of the index key
- *   @param valuelen maximum length of 'value' (integer)
- *   @param value null-terminated character string of the value
+ *   @param string null-terminated character string of the value
  *   @param flag true (1) if 'key' defined on 'info', false (0) if not
  *               (logical)
  *
  *   @retval OPAL_SUCCESS
  *
- *   In C and C++, 'valuelen' should be one less than the allocated
- *   space to allow for for the null terminator.
+ *   The \c string pointer will only be set if the key is found, i.e., if \c flag
+ *   is set to \c true. It is the caller's responsibility to decrement the
+ *   reference count of the \c string object by calling \c OBJ_RELEASE on it
+ *   once the object is not needed any more.
  */
-OPAL_DECLSPEC int opal_info_get (opal_info_t *info, const char *key, int valuelen,
-                                 char *value, int *flag);
+OPAL_DECLSPEC int opal_info_get(opal_info_t *info, const char *key, opal_cstring_t **string,
+                                int *flag);
 
 /**
  * Delete a (key,value) pair from "info"
@@ -272,53 +284,24 @@ int opal_info_delete(opal_info_t *info, const char *key);
  *   character.  If the 'key' is not found on 'info', 'valuelen' is left
  *   alone.
  */
-OPAL_DECLSPEC int opal_info_get_valuelen (opal_info_t *info, const char *key, int *valuelen,
-                                          int *flag);
+OPAL_DECLSPEC int opal_info_get_valuelen(opal_info_t *info, const char *key, int *valuelen,
+                                         int *flag);
 
 /**
- *   opal_info_get_nthkey - Get a key indexed by integer from an 'MPI_Info' o
+ *   opal_info_get_nthkey - Get a key indexed by integer from an info object
  *
  *   @param info Pointer to opal_info_t object
  *   @param n index of key to retrieve (integer)
- *   @param key character string of at least 'MPI_MAX_INFO_KEY' characters
+ *   @param key output opal_cstring_t object, set if the n'th key exists
  *
  *   @retval OPAL_SUCCESS
  *   @retval OPAL_ERR_BAD_PARAM
- */
-int opal_info_get_nthkey (opal_info_t *info, int n, char *key);
-
-/**
- * Convert value string to boolean
- *
- * Convert value string \c value into a boolean, using the
- * interpretation rules specified in MPI-2 Section 4.10.  The
- * strings "true", "false", and integer numbers can be converted
- * into booleans.  All others will return \c OMPI_ERR_BAD_PARAM
- *
- * @param value Value string for info key to interpret
- * @param interp returned interpretation of the value key
- *
- * @retval OPAL_SUCCESS string was successfully interpreted
- * @retval OPAL_ERR_BAD_PARAM string was not able to be interpreted
- */
-OPAL_DECLSPEC int opal_info_value_to_bool(char *value, bool *interp);
-
-/**
- * Convert value string to integer
- *
- * Convert value string \c value into a integer, using the
- * interpretation rules specified in MPI-2 Section 4.10.
- * All others will return \c OPAL_ERR_BAD_PARAM
- *
- * @param value Value string for info key to interpret
- * @param interp returned interpretation of the value key
  *
- * @retval OPAL_SUCCESS string was successfully interpreted
- * @retval OPAL_ERR_BAD_PARAM string was not able to be interpreted
+ *   It is the caller's responsibility to decrement the reference count of the
+ *   \c key string by calling \c OBJ_RELEASE on it once the object is not needed
+ *   any more.
  */
-int opal_info_value_to_int(char *value, int *interp);
-
-END_C_DECLS
+int opal_info_get_nthkey(opal_info_t *info, int n, opal_cstring_t **key);
 
 /**
  * Get the number of keys defined on on an MPI_Info object
@@ -327,13 +310,12 @@ END_C_DECLS
  *
  * @retval The number of keys defined on info
  */
-static inline int
-opal_info_get_nkeys(opal_info_t *info, int *nkeys)
+static inline int opal_info_get_nkeys(opal_info_t *info, int *nkeys)
 {
     *nkeys = (int) opal_list_get_size(&(info->super));
     return OPAL_SUCCESS;
 }
 
-bool opal_str_to_bool(char*);
+END_C_DECLS
 
 #endif /* OPAL_INFO_H */
diff --git a/opal/util/info_subscriber.c b/opal/util/info_subscriber.c
index 7c5491434cd..3382612ac17 100644
--- a/opal/util/info_subscriber.c
+++ b/opal/util/info_subscriber.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2007 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -25,25 +25,28 @@
  * $HEADER$
  */
 
-#include <string.h>
 #include <errno.h>
 #include <stdlib.h>
+#include <string.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
-#include <limits.h>
 #include <ctype.h>
+#include <limits.h>
 #ifdef HAVE_SYS_UTSNAME_H
-#include <sys/utsname.h>
+#    include <sys/utsname.h>
 #endif
 #include <assert.h>
+#include <string.h>
 
 #include "opal/util/argv.h"
+#include "opal/util/info_subscriber.h"
 #include "opal/util/opal_getcwd.h"
 #include "opal/util/output.h"
-#include "opal/util/info_subscriber.h"
 
-static char* opal_infosubscribe_inform_subscribers(opal_infosubscriber_t * object, char *key, char *new_value, int *found_callback);
+static const char *opal_infosubscribe_inform_subscribers(opal_infosubscriber_t *object,
+                                                         const char *key, const char *new_value,
+                                                         int *found_callback);
 static void infosubscriber_construct(opal_infosubscriber_t *obj);
 static void infosubscriber_destruct(opal_infosubscriber_t *obj);
 
@@ -55,30 +58,28 @@ typedef struct opal_callback_list_t opal_callback_list_t;
 
 struct opal_callback_list_item_t {
     opal_list_item_t super;
-    char *default_value;
+    opal_cstring_t *default_value;
     opal_key_interest_callback_t *callback;
 };
 typedef struct opal_callback_list_item_t opal_callback_list_item_t;
 
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_infosubscriber_t);
-OBJ_CLASS_INSTANCE(opal_infosubscriber_t,
-                   opal_object_t,
-                   infosubscriber_construct,
+OBJ_CLASS_INSTANCE(opal_infosubscriber_t, opal_object_t, infosubscriber_construct,
                    infosubscriber_destruct);
 
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_callback_list_item_t);
 static void opal_callback_list_item_destruct(opal_callback_list_item_t *obj);
-OBJ_CLASS_INSTANCE(opal_callback_list_item_t,
-                   opal_list_item_t,
-                   NULL,
+OBJ_CLASS_INSTANCE(opal_callback_list_item_t, opal_list_item_t, NULL,
                    opal_callback_list_item_destruct);
 
-static void infosubscriber_construct(opal_infosubscriber_t *obj) {
+static void infosubscriber_construct(opal_infosubscriber_t *obj)
+{
     OBJ_CONSTRUCT(&obj->s_subscriber_table, opal_hash_table_t);
     opal_hash_table_init(&obj->s_subscriber_table, 10);
 }
 
-static void infosubscriber_destruct(opal_infosubscriber_t *obj) {
+static void infosubscriber_destruct(opal_infosubscriber_t *obj)
+{
     opal_hash_table_t *table = &obj->s_subscriber_table;
     void *node = NULL;
     int err;
@@ -86,13 +87,13 @@ static void infosubscriber_destruct(opal_infosubscriber_t *obj) {
     size_t key_size;
     opal_list_t *list = NULL;
 
-    err = opal_hash_table_get_first_key_ptr(table,
-        (void**) &next_key, &key_size, (void**) &list, &node);
+    err = opal_hash_table_get_first_key_ptr(table, (void **) &next_key, &key_size, (void **) &list,
+                                            &node);
     while (list && err == OPAL_SUCCESS) {
         OPAL_LIST_RELEASE(list);
 
-        err = opal_hash_table_get_next_key_ptr(table,
-          (void**) &next_key, &key_size, (void**) &list, node, &node);
+        err = opal_hash_table_get_next_key_ptr(table, (void **) &next_key, &key_size,
+                                               (void **) &list, node, &node);
     }
 
     OBJ_DESTRUCT(&obj->s_subscriber_table);
@@ -102,35 +103,43 @@ static void infosubscriber_destruct(opal_infosubscriber_t *obj) {
     }
 }
 
-static void opal_callback_list_item_destruct(opal_callback_list_item_t *obj) {
+static void opal_callback_list_item_destruct(opal_callback_list_item_t *obj)
+{
     if (obj->default_value) {
-        free(obj->default_value); // came from a strdup()
+        OBJ_RELEASE(obj->default_value);
     }
 }
 
-static char* opal_infosubscribe_inform_subscribers(opal_infosubscriber_t *object, char *key, char *new_value, int *found_callback)
+static const char *opal_infosubscribe_inform_subscribers(opal_infosubscriber_t *object,
+                                                         const char *key, const char *new_value,
+                                                         int *found_callback)
 {
     opal_hash_table_t *table = &object->s_subscriber_table;
     opal_list_t *list = NULL;
     opal_callback_list_item_t *item;
-    char *updated_value = NULL;
+    const char *updated_value = NULL;
 
-    if (found_callback) { *found_callback = 0; }
-/*
- * Present the new value to each subscriber.  They can decide to accept it, ignore it, or
- * over-ride it with their own value (like ignore, but they specify what value they want it to have).
- *
- * Since multiple subscribers could set values, only the last setting is kept as the
- * returned value.
- */
+    if (found_callback) {
+        *found_callback = 0;
+    }
+    /*
+     * Present the new value to each subscriber.  They can decide to accept it, ignore it, or
+     * over-ride it with their own value (like ignore, but they specify what value they want it to
+     * have).
+     *
+     * Since multiple subscribers could set values, only the last setting is kept as the
+     * returned value.
+     */
     if (table) {
-        opal_hash_table_get_value_ptr(table, key, strlen(key), (void**) &list);
+        opal_hash_table_get_value_ptr(table, key, strlen(key), (void **) &list);
 
         if (list) {
             updated_value = new_value;
-            OPAL_LIST_FOREACH(item, list, opal_callback_list_item_t) {
+            OPAL_LIST_FOREACH (item, list, opal_callback_list_item_t) {
                 updated_value = item->callback(object, key, updated_value);
-                if (found_callback) { *found_callback = 1; }
+                if (found_callback) {
+                    *found_callback = 1;
+                }
             }
         }
     }
@@ -138,9 +147,6 @@ static char* opal_infosubscribe_inform_subscribers(opal_infosubscriber_t *object
     return updated_value;
 }
 
-
-
-
 /*
  * Testing-only static data, all paths using this code should be
  * inactive in a normal run.  In particular ntesting_callbacks is 0
@@ -150,17 +156,17 @@ static int ntesting_callbacks = 0;
 static opal_key_interest_callback_t *testing_callbacks[5];
 static char *testing_keys[5];
 static char *testing_initialvals[5];
+
 // User-level call, user adds their own callback function to be subscribed
 // to every object:
-int opal_infosubscribe_testcallback(opal_key_interest_callback_t *callback,
-  char *key, char *val);
+int opal_infosubscribe_testcallback(opal_key_interest_callback_t *callback, char *key, char *val);
 
-int
-opal_infosubscribe_testcallback(opal_key_interest_callback_t *callback,
-  char *key, char *val)
+int opal_infosubscribe_testcallback(opal_key_interest_callback_t *callback, char *key, char *val)
 {
     int i = ntesting_callbacks;
-    if (ntesting_callbacks >= 5) { return -1; }
+    if (ntesting_callbacks >= 5) {
+        return -1;
+    }
 
     testing_callbacks[i] = callback;
     testing_keys[i] = key;
@@ -170,31 +176,27 @@ opal_infosubscribe_testcallback(opal_key_interest_callback_t *callback,
 }
 
 int opal_infosubscribe_testregister(opal_infosubscriber_t *object);
-int
-opal_infosubscribe_testregister(opal_infosubscriber_t *object)
+int opal_infosubscribe_testregister(opal_infosubscriber_t *object)
 {
     opal_hash_table_t *table = &object->s_subscriber_table;
     opal_callback_list_item_t *item;
     opal_list_t *list = NULL;
 
-// The testing section should only ever be activated if the testing callback
-// above is used.
+    // The testing section should only ever be activated if the testing callback
+    // above is used.
     if (ntesting_callbacks != 0) {
         int i;
-        for (i=0; i<ntesting_callbacks; ++i) {
-// The testing-code only wants to add test-subscriptions
-// once for an obj.  So before adding a test-subscription, see
-// if it's already there.
+        for (i = 0; i < ntesting_callbacks; ++i) {
+            // The testing-code only wants to add test-subscriptions
+            // once for an obj.  So before adding a test-subscription, see
+            // if it's already there.
             int found = 0;
-            opal_hash_table_get_value_ptr(table, testing_keys[i],
-                strlen(testing_keys[i]), (void**) &list);
+            opal_hash_table_get_value_ptr(table, testing_keys[i], strlen(testing_keys[i]),
+                                          (void **) &list);
             if (list) {
-                OPAL_LIST_FOREACH(item, list, opal_callback_list_item_t) {
-                    if (0 ==
-                        strcmp(item->default_value, testing_initialvals[i])
-                        &&
-                        item->callback == testing_callbacks[i])
-                    {
+                OPAL_LIST_FOREACH (item, list, opal_callback_list_item_t) {
+                    if (0 == strcmp(item->default_value->string, testing_initialvals[i])
+                        && item->callback == testing_callbacks[i]) {
                         found = 1;
                     }
                 }
@@ -202,15 +204,14 @@ opal_infosubscribe_testregister(opal_infosubscriber_t *object)
             list = NULL;
 
             if (!found) {
-                opal_infosubscribe_subscribe(object,
-                    testing_keys[i],
-                    testing_initialvals[i], testing_callbacks[i]);
+                opal_infosubscribe_subscribe(object, testing_keys[i], testing_initialvals[i],
+                                             testing_callbacks[i]);
             }
         }
     }
 
-// For testing-mode only, while we're here, lets walk the whole list
-// to see if there are any duplicates.
+    // For testing-mode only, while we're here, lets walk the whole list
+    // to see if there are any duplicates.
     if (ntesting_callbacks != 0) {
         int err;
         void *node = NULL;
@@ -218,80 +219,36 @@ opal_infosubscribe_testregister(opal_infosubscriber_t *object)
         char *next_key;
         opal_callback_list_item_t *item1, *item2;
 
-        err = opal_hash_table_get_first_key_ptr(table, (void**) &next_key,
-            &key_size, (void**) &list, &node);
+        err = opal_hash_table_get_first_key_ptr(table, (void **) &next_key, &key_size,
+                                                (void **) &list, &node);
         while (list && err == OPAL_SUCCESS) {
             int counter = 0;
-            OPAL_LIST_FOREACH(item1, list, opal_callback_list_item_t) {
-                OPAL_LIST_FOREACH(item2, list, opal_callback_list_item_t) {
-                    if (0 ==
-                        strcmp(item1->default_value, item2->default_value)
-                        &&
-                        item1->callback == item2->callback)
-                    {
+            OPAL_LIST_FOREACH (item1, list, opal_callback_list_item_t) {
+                OPAL_LIST_FOREACH (item2, list, opal_callback_list_item_t) {
+                    if (0 == strcmp(item1->default_value->string, item2->default_value->string)
+                        && item1->callback == item2->callback) {
                         ++counter;
                     }
                 }
             }
             if (counter > 1) {
                 printf("ERROR: duplicate info key/val subscription found "
-                    "in hash table\n");
+                       "in hash table\n");
                 exit(-1);
             }
 
-            err = opal_hash_table_get_next_key_ptr(table,
-                (void**) &next_key, &key_size, (void**) &list, node, &node);
+            err = opal_hash_table_get_next_key_ptr(table, (void **) &next_key, &key_size,
+                                                   (void **) &list, node, &node);
         }
     }
 
     return OPAL_SUCCESS;
 }
 
-// This routine is to be used after making a callback for a
-// key/val pair. The callback would have ggiven a new value to associate
-// with <key>, and this function saves the previous value under
-// __IN_<key>.
-//
-// The last argument indicates whether to overwrite a previous
-// __IN_<key> or not.
-static int
-save_original_key_val(opal_info_t *info, char *key, char *val, int overwrite)
-{
-    char modkey[OPAL_MAX_INFO_KEY];
-    int flag, err;
-
-    // Checking strlen, even though it should be unnecessary.
-    // This should only happen on predefined keys with short lengths.
-    if (strlen(key) + strlen(OPAL_INFO_SAVE_PREFIX) < OPAL_MAX_INFO_KEY) {
-        snprintf(modkey, OPAL_MAX_INFO_KEY,
-            OPAL_INFO_SAVE_PREFIX "%s", key);
-// (the prefix macro is a string, so the unreadable part above is a string concatenation)
-        flag = 0;
-        opal_info_get(info, modkey, 0, NULL, &flag);
-        if (!flag || overwrite) {
-            err = opal_info_set(info, modkey, val);
-            if (OPAL_SUCCESS != err) {
-                return err;
-            }
-        }
-// FIXME: use whatever the Open MPI convention is for DEBUG options like this
-// Even though I don't expect this codepath to happen, if it somehow DID happen
-// in a real run with user-keys, I'd rather it be silent at that point rather
-// being noisy and/or aborting.
-#ifdef OMPI_DEBUG
-    } else {
-        printf("WARNING: Unexpected key length [%s]\n", key);
-#endif
-    }
-    return OPAL_SUCCESS;
-}
-
-int
-opal_infosubscribe_change_info(opal_infosubscriber_t *object, opal_info_t *new_info)
+int opal_infosubscribe_change_info(opal_infosubscriber_t *object, opal_info_t *new_info)
 {
-    int err;
     opal_info_entry_t *iterator;
-    char *updated_value;
+    const char *updated_value;
 
     /* for each key/value in new info, let subscribers know of new value */
     int found_callback;
@@ -301,29 +258,29 @@ opal_infosubscribe_change_info(opal_infosubscriber_t *object, opal_info_t *new_i
     }
 
     if (NULL != new_info) {
-    OPAL_LIST_FOREACH(iterator, &new_info->super, opal_info_entry_t) {
-
-        updated_value = opal_infosubscribe_inform_subscribers(object, iterator->ie_key, iterator->ie_value, &found_callback);
-        if (updated_value) {
-            err = opal_info_set(object->s_info, iterator->ie_key, updated_value);
-        } else {
-// This path would happen if there was no callback for this key,
-// or if there was a callback and it returned null. One way the
-// setting was unrecognized the other way it was recognized and ignored,
-// either way it shouldn't be set, which we'll ensure with an unset
-// in case a previous value exists.
-            err = opal_info_delete(object->s_info, iterator->ie_key);
-            err = OPAL_SUCCESS; // we don't care if the key was found or not
-        }
-        if (OPAL_SUCCESS != err) {
-            return err;
+        OPAL_LIST_FOREACH (iterator, &new_info->super, opal_info_entry_t) {
+            int err = OPAL_SUCCESS;
+            opal_cstring_t *value_str, *key_str;
+            value_str = iterator->ie_value;
+            OBJ_RETAIN(value_str);
+            key_str = iterator->ie_key;
+            OBJ_RETAIN(key_str);
+
+            updated_value = opal_infosubscribe_inform_subscribers(object, iterator->ie_key->string,
+                                                                  iterator->ie_value->string,
+                                                                  &found_callback);
+            if (NULL != updated_value) {
+                err = opal_info_set(object->s_info, key_str->string, updated_value);
+            } else {
+                err = opal_info_set_internal(object->s_info, key_str->string, value_str->string);
+            }
+            OBJ_RELEASE(value_str);
+            OBJ_RELEASE(key_str);
+            if (OPAL_SUCCESS != err) {
+                return err;
+            }
         }
-// Save the original at "__IN_<key>":"original"
-// And if multiple set-info calls happen, the last would be the most relevant
-// to save, so overwrite a previously saved value if there is one.
-        save_original_key_val(object->s_info,
-            iterator->ie_key, iterator->ie_value, 1);
-    }}
+    }
 
     return OPAL_SUCCESS;
 }
@@ -346,100 +303,67 @@ opal_infosubscribe_change_info(opal_infosubscriber_t *object, opal_info_t *new_i
 // to me this might be required if the strings become more dynamic than the
 // simple true/false values seen in the current code. It'll be an easy change,
 // callback() is only used two places.
-int opal_infosubscribe_subscribe(opal_infosubscriber_t *object, char *key, char *value, opal_key_interest_callback_t *callback)
+int opal_infosubscribe_subscribe(opal_infosubscriber_t *object, const char *key, const char *value,
+                                 opal_key_interest_callback_t *callback)
 {
     opal_list_t *list = NULL;
     opal_hash_table_t *table = &object->s_subscriber_table;
     opal_callback_list_item_t *callback_list_item;
-    size_t max_len = OPAL_MAX_INFO_KEY - strlen(OPAL_INFO_SAVE_PREFIX);
-
-    if (strlen(key) > max_len) {
-        opal_output(0, "DEVELOPER WARNING: Unexpected MPI info key length [%s]: "
-                    "OMPI internal callback keys are limited to %" PRIsize_t " chars.",
-                    key, max_len);
-#if OPAL_ENABLE_DEBUG
-        opal_output(0, "Aborting because this is a developer / debugging build.  Go fix this error.");
-        // Do not assert() / dump core.  Just exit un-gracefully.
-        exit(1);
-#else
-        opal_output(0, "The \"%s\" MPI info key almost certainly will not work properly.  You should inform an Open MPI developer about this.", key);
-        key[max_len] = '\0';
-#endif
-    }
 
     if (table) {
-        opal_hash_table_get_value_ptr(table, key, strlen(key), (void**) &list);
+        opal_hash_table_get_value_ptr(table, key, strlen(key), (void **) &list);
 
         if (!list) {
             list = OBJ_NEW(opal_list_t);
             opal_hash_table_set_value_ptr(table, key, strlen(key), list);
         }
-
+        opal_cstring_t *value_str = opal_cstring_create(value);
         callback_list_item = OBJ_NEW(opal_callback_list_item_t);
         callback_list_item->callback = callback;
-        if (value) {
-            callback_list_item->default_value = strdup(value);
-        } else {
-            callback_list_item->default_value = NULL;
-        }
+        callback_list_item->default_value = value_str;
 
-        opal_list_append(list, (opal_list_item_t*) callback_list_item);
+        opal_list_append(list, (opal_list_item_t *) callback_list_item);
 
-// Trigger callback() on either the default value or the info that's in the
-// object if there is one. Unfortunately there's some code duplication as
-// this is similar to the job of opal_infosubscribe_change_info().
-//
-// The value we store for key is whatever the callback() returns.
-// We also leave a backup __IN_* key with the previous value.
+        // Trigger callback() on either the default value or the info that's in the
+        // object if there is one. Unfortunately there's some code duplication as
+        // this is similar to the job of opal_infosubscribe_change_info().
+        //
+        // The value we store for key is whatever the callback() returns.
+        // We also leave a backup __IN_* key with the previous value.
 
-//  - is there an info object yet attached to this object
+        //  - is there an info object yet attached to this object
         if (NULL == object->s_info) {
             object->s_info = OBJ_NEW(opal_info_t);
         }
-// - is there a value already associated with key in this obj's info:
-//   to use in the callback()
-        char *buffer = malloc(OPAL_MAX_INFO_VAL+1); // (+1 shouldn't be needed)
-        char *val = value; // start as default value
+        // - is there a value already associated with key in this obj's info:
+        //   to use in the callback()
+        opal_cstring_t *val;
         int flag = 0;
-        char *updated_value;
+        const char *updated_value;
         int err;
-        opal_info_get(object->s_info, key, OPAL_MAX_INFO_VAL, buffer, &flag);
-        if (flag) {
-            val = buffer; // become info value if this key was in info
+        opal_info_get(object->s_info, key, &val, &flag);
+        if (!flag) {
+            val = value_str; // fall back to default string
+            OBJ_RETAIN(val);
         }
-// - callback() and modify the val in info
-        updated_value = callback(object, key, val);
+        // - callback() and modify the val in info
+        updated_value = callback(object, key, val->string);
         if (updated_value) {
             err = opal_info_set(object->s_info, key, updated_value);
         } else {
             err = opal_info_delete(object->s_info, key);
             err = OPAL_SUCCESS; // we don't care if the key was found or not
         }
+        OBJ_RELEASE(val);
+
         if (OPAL_SUCCESS != err) {
-            free(buffer);
             return err;
         }
-// - save the previous val under key __IN_*
-//   This function might be called separately for the same key multiple
-//   times (multiple modules might register an interest in the same key),
-//   so we only save __IN_<key> for the first.
-//   Note we're saving the first k/v regardless of whether it was the default
-//   or whether it came from info. This means system settings will show
-//   up if the user queries later with get_info.
-        save_original_key_val(object->s_info, key, val, 0);
-
-        free(buffer);
     } else {
-/*
- * TODO: This should not happen
- */
+        /*
+         * TODO: This should not happen
+         */
     }
 
     return OPAL_SUCCESS;
 }
-
-/*
-    OBJ_DESTRUCT(&opal_comm_info_hashtable);
-    OBJ_DESTRUCT(&opal_win_info_hashtable);
-    OBJ_DESTRUCT(&opal_file_info_hashtable);
-*/
diff --git a/opal/util/info_subscriber.h b/opal/util/info_subscriber.h
index c7ce58f95d7..a5cfd6e1591 100644
--- a/opal/util/info_subscriber.h
+++ b/opal/util/info_subscriber.h
@@ -27,27 +27,27 @@
 
 #include <string.h>
 
+#include "opal/class/opal_hash_table.h"
 #include "opal/class/opal_list.h"
 #include "opal/class/opal_pointer_array.h"
-#include "opal/class/opal_hash_table.h"
 #include "opal/mca/threads/mutex.h"
 #include "opal/util/info.h"
 
 #include "opal/mca/base/mca_base_var_enum.h"
 
-
 #define INFO_SUBSCRIBER_SIZE 5
 
 struct opal_infosubscriber_t {
-    opal_object_t	s_base;
-    opal_hash_table_t 	s_subscriber_table;
-    opal_info_t 	*s_info;
+    opal_object_t s_base;
+    opal_hash_table_t s_subscriber_table;
+    opal_info_t *s_info;
 };
 typedef struct opal_infosubscriber_t opal_infosubscriber_t;
 
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_infosubscriber_t);
 
-typedef char*(opal_key_interest_callback_t)(opal_infosubscriber_t*, char*, char*);
+typedef const char *(opal_key_interest_callback_t)(opal_infosubscriber_t *, const char *,
+                                                   const char *);
 
 /**
  *   opal_infosubscribe_change_info - Make changes to a Comm/Win/File Info
@@ -61,8 +61,7 @@ typedef char*(opal_key_interest_callback_t)(opal_infosubscriber_t*, char*, char*
  *
  *   Notifies subscribers of info's that have gone away and new info settings
  */
-int opal_infosubscribe_change_info(opal_infosubscriber_t*, opal_info_t *);
-
+int opal_infosubscribe_change_info(opal_infosubscriber_t *, opal_info_t *);
 
 /**
  *   opal_infosubscribe_subscribe - Request to be updated about info changes to a Comm/Win/File Info
@@ -79,6 +78,7 @@ int opal_infosubscribe_change_info(opal_infosubscriber_t*, opal_info_t *);
  *   Does not try to optimize settings that are the same between old and new
  *   info's.
  */
-int opal_infosubscribe_subscribe(opal_infosubscriber_t*, char *, char *, opal_key_interest_callback_t);
+int opal_infosubscribe_subscribe(opal_infosubscriber_t *, const char *, const char *,
+                                 opal_key_interest_callback_t);
 
 #endif /* OMPI_INFO_H */
diff --git a/opal/util/json/3rd-party/json.c b/opal/util/json/3rd-party/json.c
new file mode 100644
index 00000000000..83247de7fd8
--- /dev/null
+++ b/opal/util/json/3rd-party/json.c
@@ -0,0 +1,1062 @@
+/* vim: set et ts=3 sw=3 sts=3 ft=c:
+ *
+ * Copyright (C) 2012-2021 the json-parser authors  All rights reserved.
+ * https://github.com/json-parser/json-parser
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "json.h"
+
+#ifdef _MSC_VER
+   #ifndef _CRT_SECURE_NO_WARNINGS
+      #define _CRT_SECURE_NO_WARNINGS
+   #endif
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include <limits.h>
+#include <math.h>
+
+#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
+   /* C99 might give us uintptr_t and UINTPTR_MAX but they also might not be provided */
+   #include <stdint.h>
+#endif
+
+#ifndef JSON_INT_T_OVERRIDDEN
+   #if defined(_MSC_VER)
+      /* https://docs.microsoft.com/en-us/cpp/cpp/data-type-ranges */
+      #define JSON_INT_MAX 9223372036854775807LL
+   #elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
+      /* C99 */
+      #define JSON_INT_MAX INT_FAST64_MAX
+   #else
+      /* C89 */
+      #include <limits.h>
+      #define JSON_INT_MAX LONG_MAX
+   #endif
+#endif
+
+#ifndef JSON_INT_MAX
+#define JSON_INT_MAX (json_int_t)(((unsigned json_int_t)(-1)) / (unsigned json_int_t)2);
+#endif
+
+typedef unsigned int json_uchar;
+
+__attribute__((visibility ("hidden")))
+const struct _json_value json_value_none;
+
+static unsigned char hex_value (json_char c)
+{
+   if (isdigit((unsigned char)c))
+      return c - '0';
+
+   switch (c) {
+      case 'a': case 'A': return 0x0A;
+      case 'b': case 'B': return 0x0B;
+      case 'c': case 'C': return 0x0C;
+      case 'd': case 'D': return 0x0D;
+      case 'e': case 'E': return 0x0E;
+      case 'f': case 'F': return 0x0F;
+      default: return 0xFF;
+   }
+}
+
+static int would_overflow (json_int_t value, json_char b)
+{
+   return ((JSON_INT_MAX - (b - '0')) / 10 ) < value;
+}
+
+typedef struct
+{
+   size_t used_memory;
+
+   json_settings settings;
+   int first_pass;
+
+   const json_char * ptr;
+   unsigned int cur_line, cur_col;
+
+} json_state;
+
+static void * default_alloc (size_t size, int zero, void * user_data)
+{
+   (void)user_data; /* ignore unused-parameter warn */
+   return zero ? calloc (1, size) : malloc (size);
+}
+
+static void default_free (void * ptr, void * user_data)
+{
+   (void)user_data; /* ignore unused-parameter warn */
+   free (ptr);
+}
+
+static void * json_alloc (json_state * state, size_t size, int zero)
+{
+   if ((ULONG_MAX - 8 - state->used_memory) < size)
+      return 0;
+
+   if (state->settings.max_memory
+         && (state->used_memory += size) > state->settings.max_memory)
+   {
+      return 0;
+   }
+
+   return state->settings.mem_alloc (size, zero, state->settings.user_data);
+}
+
+static int new_value (json_state * state,
+                      json_value ** top, json_value ** root, json_value ** alloc,
+                      json_type type)
+{
+   json_value * value;
+   size_t values_size;
+
+   if (!state->first_pass)
+   {
+      value = *top = *alloc;
+      *alloc = (*alloc)->_reserved.next_alloc;
+
+      if (!*root)
+         *root = value;
+
+      switch (value->type)
+      {
+         case json_array:
+
+            if (value->u.array.length == 0)
+               break;
+
+            if (! (value->u.array.values = (json_value **) json_alloc
+               (state, value->u.array.length * sizeof (json_value *), 0)) )
+            {
+               return 0;
+            }
+
+            value->u.array.length = 0;
+            break;
+
+         case json_object:
+
+            if (value->u.object.length == 0)
+               break;
+
+            values_size = sizeof (*value->u.object.values) * value->u.object.length;
+
+            if (! (value->u.object.values = (json_object_entry *) json_alloc
+               #ifdef UINTPTR_MAX
+                  (state, values_size + ((uintptr_t) value->u.object.values), 0)) )
+               #else
+                  (state, values_size + ((size_t) value->u.object.values), 0)) )
+               #endif
+            {
+               return 0;
+            }
+
+            value->_reserved.object_mem = (void *) (((char *) value->u.object.values) + values_size);
+
+            value->u.object.length = 0;
+            break;
+
+         case json_string:
+
+            if (! (value->u.string.ptr = (json_char *) json_alloc
+               (state, (value->u.string.length + 1) * sizeof (json_char), 0)) )
+            {
+               return 0;
+            }
+
+            value->u.string.length = 0;
+            break;
+
+         default:
+            break;
+      };
+
+      return 1;
+   }
+
+   if (! (value = (json_value *) json_alloc
+         (state, sizeof (json_value) + state->settings.value_extra, 1)))
+   {
+      return 0;
+   }
+
+   if (!*root)
+      *root = value;
+
+   value->type = type;
+   value->parent = *top;
+
+   #ifdef JSON_TRACK_SOURCE
+      value->line = state->cur_line;
+      value->col = state->cur_col;
+   #endif
+
+   if (*alloc)
+      (*alloc)->_reserved.next_alloc = value;
+
+   *alloc = *top = value;
+
+   return 1;
+}
+
+#define whitespace \
+   case '\n': ++ state.cur_line;  state.cur_col = 0; /* FALLTHRU */ \
+   case ' ': /* FALLTHRU */ case '\t': /* FALLTHRU */ case '\r'
+
+#define string_add(b)  \
+   do { if (!state.first_pass) string [string_length] = b;  ++ string_length; } while (0);
+
+#define line_and_col \
+   state.cur_line, state.cur_col
+
+static const long
+   flag_next             = 1 << 0,
+   flag_reproc           = 1 << 1,
+   flag_need_comma       = 1 << 2,
+   flag_seek_value       = 1 << 3,
+   flag_escaped          = 1 << 4,
+   flag_string           = 1 << 5,
+   flag_need_colon       = 1 << 6,
+   flag_done             = 1 << 7,
+   flag_num_negative     = 1 << 8,
+   flag_num_zero         = 1 << 9,
+   flag_num_e            = 1 << 10,
+   flag_num_e_got_sign   = 1 << 11,
+   flag_num_e_negative   = 1 << 12,
+   flag_line_comment     = 1 << 13,
+   flag_block_comment    = 1 << 14,
+   flag_num_got_decimal  = 1 << 15;
+
+__attribute__((visibility ("hidden")))
+json_value * json_parse_ex (json_settings * settings,
+                            const json_char * json,
+                            size_t length,
+                            char * error_buf)
+{
+   char error [json_error_max];
+   const json_char * end;
+   json_value * top, * root, * alloc = 0;
+   json_state state = { 0 };
+   long flags = 0;
+   int num_digits = 0;
+   double num_e = 0, num_fraction = 0;
+
+   /* Skip UTF-8 BOM
+    */
+   if (length >= 3 && ((unsigned char) json [0]) == 0xEF
+                   && ((unsigned char) json [1]) == 0xBB
+                   && ((unsigned char) json [2]) == 0xBF)
+   {
+      json += 3;
+      length -= 3;
+   }
+
+   error[0] = '\0';
+   end = (json + length);
+
+   memcpy (&state.settings, settings, sizeof (json_settings));
+
+   if (!state.settings.mem_alloc)
+      state.settings.mem_alloc = default_alloc;
+
+   if (!state.settings.mem_free)
+      state.settings.mem_free = default_free;
+
+   for (state.first_pass = 1; state.first_pass >= 0; -- state.first_pass)
+   {
+      json_uchar uchar;
+      unsigned char uc_b1, uc_b2, uc_b3, uc_b4;
+      json_char * string = 0;
+      unsigned int string_length = 0;
+
+      top = root = 0;
+      flags = flag_seek_value;
+
+      state.cur_line = 1;
+
+      for (state.ptr = json ;; ++ state.ptr)
+      {
+         json_char b = (state.ptr == end ? 0 : *state.ptr);
+
+         if (flags & flag_string)
+         {
+            if (!b)
+            {  sprintf (error, "%u:%u: Unexpected EOF in string", line_and_col);
+               goto e_failed;
+            }
+
+            if (string_length > UINT_MAX - 8)
+               goto e_overflow;
+
+            if (flags & flag_escaped)
+            {
+               flags &= ~ flag_escaped;
+
+               switch (b)
+               {
+                  case 'b':  string_add ('\b');  break;
+                  case 'f':  string_add ('\f');  break;
+                  case 'n':  string_add ('\n');  break;
+                  case 'r':  string_add ('\r');  break;
+                  case 't':  string_add ('\t');  break;
+                  case 'u':
+
+                    if (end - state.ptr <= 4 ||
+                        (uc_b1 = hex_value (*++ state.ptr)) == 0xFF ||
+                        (uc_b2 = hex_value (*++ state.ptr)) == 0xFF ||
+                        (uc_b3 = hex_value (*++ state.ptr)) == 0xFF ||
+                        (uc_b4 = hex_value (*++ state.ptr)) == 0xFF)
+                    {
+                        sprintf (error, "%u:%u: Invalid character value `%c`", line_and_col, b);
+                        goto e_failed;
+                    }
+
+                    uc_b1 = (uc_b1 << 4) | uc_b2;
+                    uc_b2 = (uc_b3 << 4) | uc_b4;
+                    uchar = (uc_b1 << 8) | uc_b2;
+
+                    if ((uchar & 0xF800) == 0xD800) {
+                        json_uchar uchar2;
+
+                        if (end - state.ptr <= 6 || (*++ state.ptr) != '\\' || (*++ state.ptr) != 'u' ||
+                            (uc_b1 = hex_value (*++ state.ptr)) == 0xFF ||
+                            (uc_b2 = hex_value (*++ state.ptr)) == 0xFF ||
+                            (uc_b3 = hex_value (*++ state.ptr)) == 0xFF ||
+                            (uc_b4 = hex_value (*++ state.ptr)) == 0xFF)
+                        {
+                            sprintf (error, "%u:%u: Invalid character value `%c`", line_and_col, b);
+                            goto e_failed;
+                        }
+
+                        uc_b1 = (uc_b1 << 4) | uc_b2;
+                        uc_b2 = (uc_b3 << 4) | uc_b4;
+                        uchar2 = (uc_b1 << 8) | uc_b2;
+
+                        uchar = 0x010000 | ((uchar & 0x3FF) << 10) | (uchar2 & 0x3FF);
+                    }
+
+                    if (sizeof (json_char) >= sizeof (json_uchar) || (uchar <= 0x7F))
+                    {
+                       string_add ((json_char) uchar);
+                       break;
+                    }
+
+                    if (uchar <= 0x7FF)
+                    {
+                        if (state.first_pass)
+                           string_length += 2;
+                        else
+                        {  string [string_length ++] = 0xC0 | (uchar >> 6);
+                           string [string_length ++] = 0x80 | (uchar & 0x3F);
+                        }
+
+                        break;
+                    }
+
+                    if (uchar <= 0xFFFF) {
+                        if (state.first_pass)
+                           string_length += 3;
+                        else
+                        {  string [string_length ++] = 0xE0 | (uchar >> 12);
+                           string [string_length ++] = 0x80 | ((uchar >> 6) & 0x3F);
+                           string [string_length ++] = 0x80 | (uchar & 0x3F);
+                        }
+
+                        break;
+                    }
+
+                    if (state.first_pass)
+                       string_length += 4;
+                    else
+                    {  string [string_length ++] = 0xF0 | (uchar >> 18);
+                       string [string_length ++] = 0x80 | ((uchar >> 12) & 0x3F);
+                       string [string_length ++] = 0x80 | ((uchar >> 6) & 0x3F);
+                       string [string_length ++] = 0x80 | (uchar & 0x3F);
+                    }
+
+                    break;
+
+                  default:
+                     string_add (b);
+               };
+
+               continue;
+            }
+
+            if (b == '\\')
+            {
+               flags |= flag_escaped;
+               continue;
+            }
+
+            if (b == '"')
+            {
+               if (!state.first_pass)
+                  string [string_length] = 0;
+
+               flags &= ~ flag_string;
+               string = 0;
+
+               switch (top->type)
+               {
+                  case json_string:
+
+                     top->u.string.length = string_length;
+                     flags |= flag_next;
+
+                     break;
+
+                  case json_object:
+
+                     if (state.first_pass) {
+                        json_char **chars = (json_char **) &top->u.object.values;
+                        chars[0] += string_length + 1;
+                     }
+                     else
+                     {
+                        top->u.object.values [top->u.object.length].name
+                           = (json_char *) top->_reserved.object_mem;
+
+                        top->u.object.values [top->u.object.length].name_length
+                           = string_length;
+
+                        (*(json_char **) &top->_reserved.object_mem) += string_length + 1;
+                     }
+
+                     flags |= flag_seek_value | flag_need_colon;
+                     continue;
+
+                  default:
+                     break;
+               };
+            }
+            else
+            {
+               string_add (b);
+               continue;
+            }
+         }
+
+         if (state.settings.settings & json_enable_comments)
+         {
+            if (flags & (flag_line_comment | flag_block_comment))
+            {
+               if (flags & flag_line_comment)
+               {
+                  if (b == '\r' || b == '\n' || !b)
+                  {
+                     flags &= ~ flag_line_comment;
+                     -- state.ptr;  /* so null can be reproc'd */
+                  }
+
+                  continue;
+               }
+
+               if (flags & flag_block_comment)
+               {
+                  if (!b)
+                  {  sprintf (error, "%u:%u: Unexpected EOF in block comment", line_and_col);
+                     goto e_failed;
+                  }
+
+                  if (b == '*' && state.ptr < (end - 1) && state.ptr [1] == '/')
+                  {
+                     flags &= ~ flag_block_comment;
+                     ++ state.ptr;  /* skip closing sequence */
+                  }
+
+                  continue;
+               }
+            }
+            else if (b == '/')
+            {
+               if (! (flags & (flag_seek_value | flag_done)) && top->type != json_object)
+               {  sprintf (error, "%u:%u: Comment not allowed here", line_and_col);
+                  goto e_failed;
+               }
+
+               if (++ state.ptr == end)
+               {  sprintf (error, "%u:%u: EOF unexpected", line_and_col);
+                  goto e_failed;
+               }
+
+               switch (b = *state.ptr)
+               {
+                  case '/':
+                     flags |= flag_line_comment;
+                     continue;
+
+                  case '*':
+                     flags |= flag_block_comment;
+                     continue;
+
+                  default:
+                     sprintf (error, "%u:%u: Unexpected `%c` in comment opening sequence", line_and_col, b);
+                     goto e_failed;
+               };
+            }
+         }
+
+         if (flags & flag_done)
+         {
+            if (!b)
+               break;
+
+            switch (b)
+            {
+               whitespace:
+                  continue;
+
+               default:
+
+                  sprintf (error, "%u:%u: Trailing garbage: `%c`",
+                           line_and_col, b);
+
+                  goto e_failed;
+            };
+         }
+
+         if (flags & flag_seek_value)
+         {
+            switch (b)
+            {
+               whitespace:
+                  continue;
+
+               case ']':
+
+                  if (top && top->type == json_array)
+                     flags = (flags & ~ (flag_need_comma | flag_seek_value)) | flag_next;
+                  else
+                  {  sprintf (error, "%u:%u: Unexpected `]`", line_and_col);
+                     goto e_failed;
+                  }
+
+                  break;
+
+               default:
+
+                  if (flags & flag_need_comma)
+                  {
+                     if (b == ',')
+                     {  flags &= ~ flag_need_comma;
+                        continue;
+                     }
+                     else
+                     {
+                        sprintf (error, "%u:%u: Expected `,` before `%c`",
+                                 line_and_col, b);
+
+                        goto e_failed;
+                     }
+                  }
+
+                  if (flags & flag_need_colon)
+                  {
+                     if (b == ':')
+                     {  flags &= ~ flag_need_colon;
+                        continue;
+                     }
+                     else
+                     {
+                        sprintf (error, "%u:%u: Expected `:` before `%c`",
+                                 line_and_col, b);
+
+                        goto e_failed;
+                     }
+                  }
+
+                  flags &= ~ flag_seek_value;
+
+                  switch (b)
+                  {
+                     case '{':
+
+                        if (!new_value (&state, &top, &root, &alloc, json_object))
+                           goto e_alloc_failure;
+
+                        continue;
+
+                     case '[':
+
+                        if (!new_value (&state, &top, &root, &alloc, json_array))
+                           goto e_alloc_failure;
+
+                        flags |= flag_seek_value;
+                        continue;
+
+                     case '"':
+
+                        if (!new_value (&state, &top, &root, &alloc, json_string))
+                           goto e_alloc_failure;
+
+                        flags |= flag_string;
+
+                        string = top->u.string.ptr;
+                        string_length = 0;
+
+                        continue;
+
+                     case 't':
+
+                        if ((end - state.ptr) <= 3 || *(++ state.ptr) != 'r' ||
+                            *(++ state.ptr) != 'u' || *(++ state.ptr) != 'e')
+                        {
+                           goto e_unknown_value;
+                        }
+
+                        if (!new_value (&state, &top, &root, &alloc, json_boolean))
+                           goto e_alloc_failure;
+
+                        top->u.boolean = 1;
+
+                        flags |= flag_next;
+                        break;
+
+                     case 'f':
+
+                        if ((end - state.ptr) <= 4 || *(++ state.ptr) != 'a' ||
+                            *(++ state.ptr) != 'l' || *(++ state.ptr) != 's' ||
+                            *(++ state.ptr) != 'e')
+                        {
+                           goto e_unknown_value;
+                        }
+
+                        if (!new_value (&state, &top, &root, &alloc, json_boolean))
+                           goto e_alloc_failure;
+
+                        flags |= flag_next;
+                        break;
+
+                     case 'n':
+
+                        if ((end - state.ptr) <= 3 || *(++ state.ptr) != 'u' ||
+                            *(++ state.ptr) != 'l' || *(++ state.ptr) != 'l')
+                        {
+                           goto e_unknown_value;
+                        }
+
+                        if (!new_value (&state, &top, &root, &alloc, json_null))
+                           goto e_alloc_failure;
+
+                        flags |= flag_next;
+                        break;
+
+                     default:
+
+                        if (isdigit ((unsigned char) b) || b == '-')
+                        {
+                           if (!new_value (&state, &top, &root, &alloc, json_integer))
+                              goto e_alloc_failure;
+
+                           if (!state.first_pass)
+                           {
+                              while (isdigit ((unsigned char) b) || b == '+' || b == '-'
+                                        || b == 'e' || b == 'E' || b == '.')
+                              {
+                                 if ( (++ state.ptr) == end)
+                                 {
+                                    b = 0;
+                                    break;
+                                 }
+
+                                 b = *state.ptr;
+                              }
+
+                              flags |= flag_next | flag_reproc;
+                              break;
+                           }
+
+                           flags &= ~ (flag_num_negative | flag_num_e |
+                                        flag_num_e_got_sign | flag_num_e_negative |
+                                           flag_num_zero);
+
+                           num_digits = 0;
+                           num_fraction = 0;
+                           num_e = 0;
+
+                           if (b != '-')
+                           {
+                              flags |= flag_reproc;
+                              break;
+                           }
+
+                           flags |= flag_num_negative;
+                           continue;
+                        }
+                        else
+                        {  sprintf (error, "%u:%u: Unexpected `%c` when seeking value", line_and_col, b);
+                           goto e_failed;
+                        }
+                  };
+            };
+         }
+         else
+         {
+            switch (top->type)
+            {
+            case json_object:
+
+               switch (b)
+               {
+                  whitespace:
+                     continue;
+
+                  case '"':
+
+                     if (flags & flag_need_comma)
+                     {  sprintf (error, "%u:%u: Expected `,` before `\"`", line_and_col);
+                        goto e_failed;
+                     }
+
+                     flags |= flag_string;
+
+                     string = (json_char *) top->_reserved.object_mem;
+                     string_length = 0;
+
+                     break;
+
+                  case '}':
+
+                     flags = (flags & ~ flag_need_comma) | flag_next;
+                     break;
+
+                  case ',':
+
+                     if (flags & flag_need_comma)
+                     {
+                        flags &= ~ flag_need_comma;
+                        break;
+                     } /* FALLTHRU */
+
+                  default:
+                     sprintf (error, "%u:%u: Unexpected `%c` in object", line_and_col, b);
+                     goto e_failed;
+               };
+
+               break;
+
+            case json_integer:
+            case json_double:
+
+               if (isdigit ((unsigned char)b))
+               {
+                  ++ num_digits;
+
+                  if (top->type == json_integer || flags & flag_num_e)
+                  {
+                     if (! (flags & flag_num_e))
+                     {
+                        if (flags & flag_num_zero)
+                        {  sprintf (error, "%u:%u: Unexpected `0` before `%c`", line_and_col, b);
+                           goto e_failed;
+                        }
+
+                        if (num_digits == 1 && b == '0')
+                           flags |= flag_num_zero;
+                     }
+                     else
+                     {
+                        flags |= flag_num_e_got_sign;
+                        num_e = (num_e * 10) + (b - '0');
+                        continue;
+                     }
+
+                     if (would_overflow(top->u.integer, b))
+                     {
+                        json_int_t integer = top->u.integer;
+                        -- num_digits;
+                        -- state.ptr;
+                        top->type = json_double;
+                        top->u.dbl = (double)integer;
+                        continue;
+                     }
+
+                     top->u.integer = (top->u.integer * 10) + (b - '0');
+                     continue;
+                  }
+
+                  if (flags & flag_num_got_decimal)
+                     num_fraction = (num_fraction * 10) + (b - '0');
+                  else
+                     top->u.dbl = (top->u.dbl * 10) + (b - '0');
+
+                  continue;
+               }
+
+               if (b == '+' || b == '-')
+               {
+                  if ( (flags & flag_num_e) && !(flags & flag_num_e_got_sign))
+                  {
+                     flags |= flag_num_e_got_sign;
+
+                     if (b == '-')
+                        flags |= flag_num_e_negative;
+
+                     continue;
+                  }
+               }
+               else if (b == '.' && top->type == json_integer)
+               {
+                  json_int_t integer = top->u.integer;
+
+                  if (!num_digits)
+                  {  sprintf (error, "%u:%u: Expected digit before `.`", line_and_col);
+                     goto e_failed;
+                  }
+
+                  top->type = json_double;
+                  top->u.dbl = (double) integer;
+
+                  flags |= flag_num_got_decimal;
+                  num_digits = 0;
+                  continue;
+               }
+
+               if (! (flags & flag_num_e))
+               {
+                  if (top->type == json_double)
+                  {
+                     if (!num_digits)
+                     {  sprintf (error, "%u:%u: Expected digit after `.`", line_and_col);
+                        goto e_failed;
+                     }
+
+                     top->u.dbl += num_fraction / pow (10.0, num_digits);
+                  }
+
+                  if (b == 'e' || b == 'E')
+                  {
+                     flags |= flag_num_e;
+
+                     if (top->type == json_integer)
+                     {
+                        json_int_t integer = top->u.integer;
+                        top->type = json_double;
+                        top->u.dbl = (double) integer;
+                     }
+
+                     num_digits = 0;
+                     flags &= ~ flag_num_zero;
+
+                     continue;
+                  }
+               }
+               else
+               {
+                  if (!num_digits)
+                  {  sprintf (error, "%u:%u: Expected digit after `e`", line_and_col);
+                     goto e_failed;
+                  }
+
+                  top->u.dbl *= pow (10.0, (flags & flag_num_e_negative ? - num_e : num_e));
+               }
+
+               if (flags & flag_num_negative)
+               {
+                  if (top->type == json_integer)
+                     top->u.integer = - top->u.integer;
+                  else
+                     top->u.dbl = - top->u.dbl;
+               }
+
+               flags |= flag_next | flag_reproc;
+               break;
+
+            default:
+               break;
+            };
+         }
+
+         if (flags & flag_reproc)
+         {
+            flags &= ~ flag_reproc;
+            -- state.ptr;
+         }
+
+         if (flags & flag_next)
+         {
+            flags = (flags & ~ flag_next) | flag_need_comma;
+
+            if (!top->parent)
+            {
+               /* root value done */
+
+               flags |= flag_done;
+               continue;
+            }
+
+            if (top->parent->type == json_array)
+               flags |= flag_seek_value;
+
+            if (!state.first_pass)
+            {
+               json_value * parent = top->parent;
+
+               switch (parent->type)
+               {
+                  case json_object:
+
+                     parent->u.object.values
+                        [parent->u.object.length].value = top;
+
+                     break;
+
+                  case json_array:
+
+                     parent->u.array.values
+                           [parent->u.array.length] = top;
+
+                     break;
+
+                  default:
+                     break;
+               };
+            }
+
+            if ( (++ top->parent->u.array.length) > UINT_MAX - 8)
+               goto e_overflow;
+
+            top = top->parent;
+
+            continue;
+         }
+      }
+
+      alloc = root;
+   }
+
+   return root;
+
+e_unknown_value:
+
+   sprintf (error, "%u:%u: Unknown value", line_and_col);
+   goto e_failed;
+
+e_alloc_failure:
+
+   strcpy (error, "Memory allocation failure");
+   goto e_failed;
+
+e_overflow:
+
+   sprintf (error, "%u:%u: Too long (caught overflow)", line_and_col);
+   goto e_failed;
+
+e_failed:
+
+   if (error_buf)
+   {
+      if (*error)
+         strcpy (error_buf, error);
+      else
+         strcpy (error_buf, "Unknown error");
+   }
+
+   if (state.first_pass)
+      alloc = root;
+
+   while (alloc)
+   {
+      top = alloc->_reserved.next_alloc;
+      state.settings.mem_free (alloc, state.settings.user_data);
+      alloc = top;
+   }
+
+   if (!state.first_pass)
+      json_value_free_ex (&state.settings, root);
+
+   return 0;
+}
+
+__attribute__((visibility ("hidden")))
+json_value * json_parse (const json_char * json, size_t length)
+{
+   json_settings settings = { 0 };
+   return json_parse_ex (&settings, json, length, 0);
+}
+
+__attribute__((visibility ("hidden")))
+void json_value_free_ex (json_settings * settings, json_value * value)
+{
+   json_value * cur_value;
+
+   if (!value)
+      return;
+
+   value->parent = 0;
+
+   while (value)
+   {
+      switch (value->type)
+      {
+         case json_array:
+
+            if (!value->u.array.length)
+            {
+               settings->mem_free (value->u.array.values, settings->user_data);
+               break;
+            }
+
+            value = value->u.array.values [-- value->u.array.length];
+            continue;
+
+         case json_object:
+
+            if (!value->u.object.length)
+            {
+               settings->mem_free (value->u.object.values, settings->user_data);
+               break;
+            }
+
+            value = value->u.object.values [-- value->u.object.length].value;
+            continue;
+
+         case json_string:
+
+            settings->mem_free (value->u.string.ptr, settings->user_data);
+            break;
+
+         default:
+            break;
+      };
+
+      cur_value = value;
+      value = value->parent;
+      settings->mem_free (cur_value, settings->user_data);
+   }
+}
+
+__attribute__((visibility ("hidden")))
+void json_value_free (json_value * value)
+{
+   json_settings settings = { 0 };
+   settings.mem_free = default_free;
+   json_value_free_ex (&settings, value);
+}
diff --git a/opal/util/json/3rd-party/json.h b/opal/util/json/3rd-party/json.h
new file mode 100644
index 00000000000..1f6431614c1
--- /dev/null
+++ b/opal/util/json/3rd-party/json.h
@@ -0,0 +1,288 @@
+
+/* vim: set et ts=3 sw=3 sts=3 ft=c:
+ *
+ * Copyright (C) 2012-2021 the json-parser authors  All rights reserved.
+ * https://github.com/json-parser/json-parser
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _JSON_H
+#define _JSON_H
+
+#ifndef json_char
+   #define json_char char
+#endif
+
+#ifndef json_int_t
+   #undef JSON_INT_T_OVERRIDDEN
+   #if defined(_MSC_VER)
+      #define json_int_t __int64
+   #elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || (defined(__cplusplus) && __cplusplus >= 201103L)
+      /* C99 and C++11 */
+      #include <stdint.h>
+      #define json_int_t int_fast64_t
+   #else
+      /* C89 */
+      #define json_int_t long
+   #endif
+#else
+   #define JSON_INT_T_OVERRIDDEN 1
+#endif
+
+#include <stddef.h>
+
+#ifdef __cplusplus
+
+   #include <string.h>
+
+   extern "C"
+   {
+
+#endif
+
+typedef struct
+{
+   unsigned long max_memory;  /* should be size_t, but would modify the API */
+   int settings;
+
+   /* Custom allocator support (leave null to use malloc/free)
+    */
+
+   void * (* mem_alloc) (size_t, int zero, void * user_data);
+   void (* mem_free) (void *, void * user_data);
+
+   void * user_data;  /* will be passed to mem_alloc and mem_free */
+
+   size_t value_extra;  /* how much extra space to allocate for values? */
+
+} json_settings;
+
+#define json_enable_comments  0x01
+
+typedef enum
+{
+   json_none,
+   json_object,
+   json_array,
+   json_integer,
+   json_double,
+   json_string,
+   json_boolean,
+   json_null
+
+} json_type;
+
+extern const struct _json_value json_value_none;
+
+typedef struct _json_object_entry
+{
+    json_char * name;
+    unsigned int name_length;
+
+    struct _json_value * value;
+
+} json_object_entry;
+
+typedef struct _json_value
+{
+   struct _json_value * parent;
+
+   json_type type;
+
+   union
+   {
+      int boolean;
+      json_int_t integer;
+      double dbl;
+
+      struct
+      {
+         unsigned int length;
+         json_char * ptr; /* null terminated */
+
+      } string;
+
+      struct
+      {
+         unsigned int length;
+
+         json_object_entry * values;
+
+         #if defined(__cplusplus)
+         json_object_entry * begin () const
+         {  return values;
+         }
+         json_object_entry * end () const
+         {  return values + length;
+         }
+         #endif
+
+      } object;
+
+      struct
+      {
+         unsigned int length;
+         struct _json_value ** values;
+
+         #if defined(__cplusplus)
+         _json_value ** begin () const
+         {  return values;
+         }
+         _json_value ** end () const
+         {  return values + length;
+         }
+         #endif
+
+      } array;
+
+   } u;
+
+   union
+   {
+      struct _json_value * next_alloc;
+      void * object_mem;
+
+   } _reserved;
+
+   #ifdef JSON_TRACK_SOURCE
+
+      /* Location of the value in the source JSON
+       */
+      unsigned int line, col;
+
+   #endif
+
+
+   /* Some C++ operator sugar */
+
+   #ifdef __cplusplus
+
+      public:
+
+         inline _json_value ()
+         {  memset (this, 0, sizeof (_json_value));
+         }
+
+         inline const struct _json_value &operator [] (int index) const
+         {
+            if (type != json_array || index < 0
+                     || ((unsigned int) index) >= u.array.length)
+            {
+               return json_value_none;
+            }
+
+            return *u.array.values [index];
+         }
+
+         inline const struct _json_value &operator [] (const char * index) const
+         {
+            if (type != json_object)
+               return json_value_none;
+
+            for (unsigned int i = 0; i < u.object.length; ++ i)
+               if (!strcmp (u.object.values [i].name, index))
+                  return *u.object.values [i].value;
+
+            return json_value_none;
+         }
+
+         inline operator const char * () const
+         {
+            switch (type)
+            {
+               case json_string:
+                  return u.string.ptr;
+
+               default:
+                  return "";
+            };
+         }
+
+         inline operator json_int_t () const
+         {
+            switch (type)
+            {
+               case json_integer:
+                  return u.integer;
+
+               case json_double:
+                  return (json_int_t) u.dbl;
+
+               default:
+                  return 0;
+            };
+         }
+
+         inline operator bool () const
+         {
+            if (type != json_boolean)
+               return false;
+
+            return u.boolean != 0;
+         }
+
+         inline operator double () const
+         {
+            switch (type)
+            {
+               case json_integer:
+                  return (double) u.integer;
+
+               case json_double:
+                  return u.dbl;
+
+               default:
+                  return 0;
+            };
+         }
+
+   #endif
+
+} json_value;
+
+json_value * json_parse (const json_char * json,
+                         size_t length);
+
+#define json_error_max 128
+json_value * json_parse_ex (json_settings * settings,
+                            const json_char * json,
+                            size_t length,
+                            char * error);
+
+void json_value_free (json_value *);
+
+
+/* Not usually necessary, unless you used a custom mem_alloc and now want to
+ * use a custom mem_free.
+ */
+void json_value_free_ex (json_settings * settings,
+                         json_value *);
+
+
+#ifdef __cplusplus
+   } /* extern "C" */
+#endif
+
+#endif
diff --git a/opal/util/json/Makefile.am b/opal/util/json/Makefile.am
new file mode 100644
index 00000000000..75cc978eea5
--- /dev/null
+++ b/opal/util/json/Makefile.am
@@ -0,0 +1,17 @@
+#
+# Copyright (c) 2024      Amazon.com, Inc. or its affiliates.
+#                         All Rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+noinst_LTLIBRARIES = libopalutil_json.la
+
+libopalutil_json_la_SOURCES = \
+        3rd-party/json.h \
+        3rd-party/json.c \
+	opal_json.h \
+	opal_json.c
diff --git a/opal/util/json/help-json.txt b/opal/util/json/help-json.txt
new file mode 100644
index 00000000000..e6adb5e8bc5
--- /dev/null
+++ b/opal/util/json/help-json.txt
@@ -0,0 +1,28 @@
+# -*- text -*-
+#
+# Copyright (c) 2024    Amazon.com, Inc. or its affiliates. All Rights reserved.
+#
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+[Unknown JSON value type]
+The JSON value has an unknown type %s.
+
+[Invalid argument type]
+Function %s expects input type %s(%d) but got %d.
+
+[Invalid JSON string]
+The input is not a valid JSON string.
+
+[Unable to open file]
+Failed to open file %s. Please check the file path and permissions.
+
+[Memory allocation failure]
+Cannot allocate memory.
+
+[Unable to read file]
+Encountered error reading file %s.
+
+[Index out of bound]
+Index(%zu) is out of bound(length %ud).
diff --git a/opal/util/json/opal_json.c b/opal/util/json/opal_json.c
new file mode 100644
index 00000000000..d20c48fe263
--- /dev/null
+++ b/opal/util/json/opal_json.c
@@ -0,0 +1,296 @@
+/*
+ * Copyright (c) 2024-2025 Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "opal/util/json/opal_json.h"
+#include "opal/constants.h"
+#include "opal/util/json/3rd-party/json.h"
+#include "opal/util/show_help.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define CHECK_OBJ_TYPE(obj, expected)                                                           \
+    do {                                                                                        \
+        if ((obj)->type != (expected)) {                                                        \
+            return OPAL_ERROR;                                                                  \
+        }                                                                                       \
+    } while (0)
+
+struct opal_json_internal_t {
+    struct opal_json_t parent;
+    json_value *value;
+};
+typedef struct opal_json_internal_t opal_json_internal_t;
+
+static void opal_json_internal_free(opal_json_internal_t *json)
+{
+    if (NULL == json) {
+        return;
+    }
+
+    /* The root JSON object will release the memory of its children */
+    if (json->value && NULL == json->value->parent) {
+        json_value_free(json->value);
+    }
+
+    free(json);
+}
+
+static inline int opal_json_internal_translate_type(json_type type, opal_json_type *out)
+{
+    int ret = OPAL_SUCCESS;
+    switch (type) {
+    case json_null:
+        *out = OPAL_JSON_NULL;
+        break;
+    case json_array:
+        *out = OPAL_JSON_ARRAY;
+        break;
+    case json_boolean:
+        *out = OPAL_JSON_BOOL;
+        break;
+    case json_integer:
+        *out = OPAL_JSON_INTEGER;
+        break;
+    case json_double:
+        *out = OPAL_JSON_DOUBLE;
+        break;
+    case json_string:
+        *out = OPAL_JSON_STRING;
+        break;
+    case json_object:
+        *out = OPAL_JSON_OBJECT;
+        break;
+    default:
+        opal_show_help("help-json.txt", "Unknown JSON value type", true, type);
+        *out = OPAL_JSON_TYPE_COUNT;
+        ret = OPAL_ERROR;
+    }
+
+    return ret;
+}
+
+static int opal_json_internal_new(const json_value *in, opal_json_internal_t **out)
+{
+    *out = malloc(sizeof(opal_json_internal_t));
+    if (NULL == *out) {
+        return OPAL_ERROR;
+    }
+
+    (*out)->value = (json_value *) in;
+    return opal_json_internal_translate_type(in->type, &(*out)->parent.type);
+}
+
+int opal_json_load(const char *str, const size_t len, const opal_json_t **json)
+{
+    opal_json_internal_t *out = NULL;
+    int ret = OPAL_SUCCESS;
+
+    json_value *value = json_parse(str, len);
+    if (!value) {
+        ret = OPAL_ERROR;
+        goto out;
+    }
+
+    ret = opal_json_internal_new(value, &out);
+
+out:
+    if (OPAL_SUCCESS == ret) {
+        *json = (opal_json_t *) out;
+    } else if (out) {
+        opal_json_internal_free(out);
+    }
+
+    return ret;
+}
+
+int opal_json_load_file(const char *filename, const opal_json_t **json, int show_errors)
+{
+    FILE *fp = NULL;
+    size_t file_size;
+    char *file_contents = NULL;
+    int ret = OPAL_SUCCESS;
+
+    fp = fopen(filename, "r");
+    if (fp == NULL) {
+        if (show_errors) {
+            opal_show_help("help-json.txt", "Unable to open file", true, filename);
+        }
+        ret = OPAL_ERROR;
+        goto out;
+    }
+
+    (void) fseek(fp, 0L, SEEK_END);
+    file_size = ftell(fp);
+    rewind(fp);
+
+    file_contents = (char *) malloc(file_size);
+    if (!file_contents) {
+        if (show_errors) {
+            opal_show_help("help-json.txt", "Memory allocation failure", true);
+        }
+        ret = OPAL_ERROR;
+        goto out;
+    }
+
+    if (file_size > fread(file_contents, 1, file_size, fp)) {
+        if (show_errors) {
+            opal_show_help("help-json.txt", "Unable to read file", true, filename);
+        }
+        ret = OPAL_ERROR;
+        goto out;
+    }
+
+    ret = opal_json_load(file_contents, file_size, json);
+    if (ret != OPAL_SUCCESS && show_errors) {
+        opal_show_help("help-json.txt", "Invalid JSON string", true);
+    }
+
+out:
+    if (fp) {
+        fclose(fp);
+    }
+    if (file_contents) {
+        free(file_contents);
+    }
+
+    return ret;
+}
+
+int opal_json_get_key(const opal_json_t *json, const char *key, const opal_json_t **out)
+{
+    int ret = OPAL_ERROR;
+
+    CHECK_OBJ_TYPE(json, OPAL_JSON_OBJECT);
+
+    opal_json_internal_t *in = (opal_json_internal_t *) json;
+    opal_json_internal_t *result = NULL;
+
+    json_object_entry entry = {0};
+
+    for (unsigned int i = 0; i < in->value->u.object.length; ++i) {
+        entry = in->value->u.object.values[i];
+        if (0 == strcmp(entry.name, key)) {
+            ret = opal_json_internal_new(entry.value, &result);
+            break;
+        }
+    }
+
+    if (OPAL_SUCCESS == ret) {
+        *out = (opal_json_t *) result;
+    } else if (result) {
+        opal_json_internal_free(result);
+    }
+
+    return ret;
+}
+
+int opal_json_get_key_by_index(const opal_json_t *json, const size_t index, const char **key, const opal_json_t **out)
+{
+    int ret = OPAL_ERROR;
+
+    CHECK_OBJ_TYPE(json, OPAL_JSON_OBJECT);
+
+    opal_json_internal_t *in = (opal_json_internal_t *) json;
+    opal_json_internal_t *result = NULL;
+
+    json_object_entry entry = {0};
+
+    if (index >= in->value->u.object.length) {
+        opal_show_help("help-json.txt", "Index out of bound", true, index,
+                       in->value->u.array.length);
+        return ret;
+    }
+    entry = in->value->u.object.values[index];
+    *key = entry.name;
+    ret = opal_json_internal_new(entry.value, &result);
+    if (OPAL_SUCCESS == ret) {
+        *out = (opal_json_t *) result;
+    } else if (result) {
+        opal_json_internal_free(result);
+    }
+    return ret;
+}
+
+void opal_json_free(const opal_json_t **json)
+{
+    opal_json_internal_free((struct opal_json_internal_t *) *json);
+}
+
+int opal_json_get_index(const opal_json_t *json, const size_t index, const opal_json_t **out)
+{
+    int ret = OPAL_ERROR;
+
+    CHECK_OBJ_TYPE(json, OPAL_JSON_ARRAY);
+
+    opal_json_internal_t *in = (opal_json_internal_t *) json;
+    opal_json_internal_t *result = NULL;
+
+    if ((size_t) in->value->u.array.length > index) {
+        ret = opal_json_internal_new(in->value->u.array.values[index], &result);
+    } else {
+        opal_show_help("help-json.txt", "Index out of bound", true, index,
+                       in->value->u.array.length);
+    }
+
+    if (OPAL_SUCCESS == ret) {
+        *out = (opal_json_t *) result;
+    } else if (result) {
+        opal_json_internal_free(result);
+    }
+
+    return ret;
+}
+
+int opal_json_get_container_size(const opal_json_t *json, size_t *len)
+{
+    int ret = OPAL_ERROR;
+    if (OPAL_JSON_OBJECT == json->type) {
+        *len = (size_t) ((opal_json_internal_t *) json)->value->u.object.length;
+        ret = OPAL_SUCCESS;
+    } else if (OPAL_JSON_ARRAY == json->type) {
+        *len = (size_t) ((opal_json_internal_t *) json)->value->u.array.length;
+        ret = OPAL_SUCCESS;
+    }
+    return ret;
+}
+
+int opal_json_read_integer(const opal_json_t *json, int64_t *out)
+{
+    CHECK_OBJ_TYPE(json, OPAL_JSON_INTEGER);
+    opal_json_internal_t *in = (opal_json_internal_t *) json;
+    *out = in->value->u.integer;
+    return OPAL_SUCCESS;
+}
+
+int opal_json_read_double(const opal_json_t *json, double *out)
+{
+    CHECK_OBJ_TYPE(json, OPAL_JSON_DOUBLE);
+    opal_json_internal_t *in = (opal_json_internal_t *) json;
+    *out = in->value->u.dbl;
+    return OPAL_SUCCESS;
+}
+
+int opal_json_read_bool(const opal_json_t *json, bool *out)
+{
+    CHECK_OBJ_TYPE(json, OPAL_JSON_BOOL);
+    opal_json_internal_t *in = (opal_json_internal_t *) json;
+    *out = in->value->u.boolean;
+    return OPAL_SUCCESS;
+}
+
+int opal_json_read_string(const opal_json_t *json, const char **out, size_t *len)
+{
+    CHECK_OBJ_TYPE(json, OPAL_JSON_STRING);
+    opal_json_internal_t *in = (opal_json_internal_t *) json;
+    *out = in->value->u.string.ptr;
+    *len = in->value->u.string.length;
+    return OPAL_SUCCESS;
+}
diff --git a/opal/util/json/opal_json.h b/opal/util/json/opal_json.h
new file mode 100644
index 00000000000..d990029e661
--- /dev/null
+++ b/opal/util/json/opal_json.h
@@ -0,0 +1,170 @@
+/*
+ * Copyright (c) 2024-2025 Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/** @file:
+ *
+ * Utility data structures and functions to parse JSON from a string or file.
+ * Note: The internal 3rd-party code should NOT be directly used. Implement the
+ * functionality here if necessary.
+ *
+ * Below is a trivial example to parse a JSON file with the content {"key": "val"}.
+
+int ret = OPAL_SUCCESS;
+const opal_json_t *json = NULL, *item = NULL;
+char *val;
+size_t len;
+
+ret = opal_json_load_file("my_file.json", &json, 1); // Parse the file content into json
+if (OPAL_SUCCESS != ret) {
+     goto out;
+}
+
+ret = opal_json_get_key(json, "key", &item); // Get the item pointed to by "key"
+if (OPAL_SUCCESS != ret) {
+     goto out;
+}
+
+ret = opal_json_read_string(item, (const char **) &val, &len); // Read the string value and length
+if (OPAL_SUCCESS != ret) {
+    goto out;
+}
+
+out:
+
+// Cleanup
+opal_json_free(&json);
+opal_json_free(&item);
+
+ */
+
+#ifndef OPAL_JSON_H
+#define OPAL_JSON_H
+
+#include "opal/mca/mca.h"
+
+BEGIN_C_DECLS
+
+typedef enum {
+    OPAL_JSON_NULL,
+    OPAL_JSON_OBJECT,
+    OPAL_JSON_ARRAY,
+    OPAL_JSON_INTEGER,
+    OPAL_JSON_DOUBLE,
+    OPAL_JSON_BOOL,
+    OPAL_JSON_STRING,
+    OPAL_JSON_TYPE_COUNT,
+} opal_json_type;
+
+struct opal_json_t {
+    opal_json_type type;
+};
+typedef struct opal_json_t opal_json_t;
+
+/**
+ * Load JSON from a string.
+ *
+ * @param[in]   str     A JSON string
+ * @param[in]   len     First bytes to parse
+ * @param[out]  json    Output JSON object if the string is valid
+ *
+ * @returns     OPAL_SUCCESS if the string is parsed successfully
+ *              OPAL_ERROR otherwise
+ */
+OPAL_DECLSPEC int opal_json_load(const char *str, const size_t len, const opal_json_t **json);
+
+/**
+ * Load JSON from a file.
+ *
+ * @param[in]   filename    File name
+ * @param[out]  json        Output JSON object if the file valid
+ *
+ * @returns     OPAL_SUCCESS if the file is read and parsed successfully
+ *              OPAL_ERROR otherwise
+ */
+OPAL_DECLSPEC int opal_json_load_file(const char *filename, const opal_json_t **json, int show_errors);
+
+/**
+ * Free JSON resources
+ *
+ * @param[in]  json     Pointer to JSON object to free
+ */
+OPAL_DECLSPEC void opal_json_free(const opal_json_t **json);
+
+/**
+ * Get the JSON object with a key value from a parent object
+ *
+ * @param[in]   json    Parent JSON object
+ * @param[in]   key     Key value
+ * @param[out]  out     Output JSON object with the specified key value
+ *
+ * @returns     OPAL_SUCCESS if an object is found with the specified key value
+ *              OPAL_ERROR otherwise
+ */
+OPAL_DECLSPEC int opal_json_get_key(const opal_json_t *json, const char *key,
+                                    const opal_json_t **out);
+
+/**
+ * Get the JSON object at index from a JSON array
+ *
+ * @param[in]   json    Parent JSON array
+ * @param[in]   index   Index value
+ * @param[out]  out     Output JSON object at the specified index
+ *
+ * @returns     OPAL_SUCCESS if an object is found at the specified index
+ *              OPAL_ERROR otherwise
+ */
+OPAL_DECLSPEC int opal_json_get_index(const opal_json_t *json, const size_t index,
+                                      const opal_json_t **out);
+
+/**
+ * Get the JSON object and key at index from a JSON object
+ *
+ * @param[in]   json    Parent JSON array
+ * @param[in]   index   Index value
+ * @param[out]  key     The name of the key at the given index. This pointer
+ *                      remains valid for the lifetime of the parent JSON
+ *                      object.  The caller should not free it.
+ * @param[out]  out     Output JSON object at the specified index. This object
+ *                      should be freed by caller with opal_json_free.
+ */
+OPAL_DECLSPEC int
+opal_json_get_key_by_index( const opal_json_t *json, const size_t index,
+                            const char **key, const opal_json_t **out);
+
+
+/**
+ * Get the number of objects in a container-type value, i.e. object, array.
+ *
+ * @param[in]   json    A container JSON object
+ * @param[out]  len     Number of elements in the container
+ *
+ * @returns     OPAL_SUCCESS if successful
+ *              OPAL_ERROR otherwise
+ */
+OPAL_DECLSPEC int opal_json_get_container_size(const opal_json_t *json, size_t *len);
+
+/**
+ * Value reader functions
+ *
+ * The caller is responsible for ensuring the function be called for the correct the object type,
+ * i.e. *read_integer should only be called for integer objects.
+ *
+ * @returns     OPAL_SUCCESS if successful
+ *              OPAL_ERROR otherwise
+ */
+
+OPAL_DECLSPEC int opal_json_read_bool(const opal_json_t *json, bool *out);
+OPAL_DECLSPEC int opal_json_read_integer(const opal_json_t *json, int64_t *out);
+OPAL_DECLSPEC int opal_json_read_double(const opal_json_t *json, double *out);
+OPAL_DECLSPEC int opal_json_read_string(const opal_json_t *json, const char **out, size_t *len);
+
+END_C_DECLS
+
+#endif
diff --git a/opal/util/keyval/Makefile.am b/opal/util/keyval/Makefile.am
index b00083d5d80..438c39121d4 100644
--- a/opal/util/keyval/Makefile.am
+++ b/opal/util/keyval/Makefile.am
@@ -30,5 +30,11 @@ libopalutilkeyval_la_SOURCES = \
 	keyval_lex.h \
         keyval_lex.l
 
+# flex prior to version 2.6.6 uses the POSIX extension fileno()
+# without providing the proper feature test macro, so
+# we add the _POSIX_C_SOURCE macro here.
+# See https://github.com/westes/flex/issues/263
+libopalutilkeyval_la_CFLAGS = -D_POSIX_C_SOURCE=200809L
+
 maintainer-clean-local:
 	rm -f keyval_lex.c
diff --git a/opal/util/keyval/keyval_lex.h b/opal/util/keyval/keyval_lex.h
index 95174ad9b39..004f6b82a8f 100644
--- a/opal/util/keyval/keyval_lex.h
+++ b/opal/util/keyval/keyval_lex.h
@@ -23,13 +23,13 @@
 #include "opal_config.h"
 
 #ifdef malloc
-#undef malloc
+#    undef malloc
 #endif
 #ifdef realloc
-#undef realloc
+#    undef realloc
 #endif
 #ifdef free
-#undef free
+#    undef free
 #endif
 
 #include <stdio.h>
@@ -47,12 +47,12 @@ extern int opal_util_keyval_yylineno;
 /*
  * Make lex-generated files not issue compiler warnings
  */
-#define YY_STACK_USED 0
+#define YY_STACK_USED         0
 #define YY_ALWAYS_INTERACTIVE 0
-#define YY_NEVER_INTERACTIVE 0
-#define YY_MAIN 0
-#define YY_NO_UNPUT 1
-#define YY_SKIP_YYWRAP 1
+#define YY_NEVER_INTERACTIVE  0
+#define YY_MAIN               0
+#define YY_NO_UNPUT           1
+#define YY_SKIP_YYWRAP        1
 
 enum opal_keyval_parse_state_t {
     OPAL_UTIL_KEYVAL_PARSE_DONE,
diff --git a/opal/util/keyval_parse.c b/opal/util/keyval_parse.c
index ea7e2d72f24..4a9fd53ca11 100644
--- a/opal/util/keyval_parse.c
+++ b/opal/util/keyval_parse.c
@@ -25,14 +25,14 @@
 #include "opal_config.h"
 
 #include "opal/constants.h"
+#include "opal/mca/threads/mutex.h"
 #include "opal/runtime/opal.h"
-#include "opal/util/keyval_parse.h"
 #include "opal/util/keyval/keyval_lex.h"
+#include "opal/util/keyval_parse.h"
 #include "opal/util/output.h"
 #include "opal/util/string_copy.h"
-#include "opal/mca/threads/mutex.h"
-#include <string.h>
 #include <ctype.h>
+#include <string.h>
 
 int opal_util_keyval_parse_lineno = 0;
 
@@ -49,7 +49,7 @@ static void parse_error(int num);
 static char *env_str = NULL;
 static int envsize = 1024;
 
-static void opal_util_keyval_parse_finalize (void)
+static void opal_util_keyval_parse_finalize(void)
 {
     free(key_buffer);
     key_buffer = NULL;
@@ -62,17 +62,16 @@ int opal_util_keyval_parse_init(void)
 {
     OBJ_CONSTRUCT(&keyval_mutex, opal_mutex_t);
 
-    opal_finalize_register_cleanup (opal_util_keyval_parse_finalize);
+    opal_finalize_register_cleanup(opal_util_keyval_parse_finalize);
 
     return OPAL_SUCCESS;
 }
 
-int
-opal_util_keyval_parse(const char *filename,
-                       opal_keyval_parse_fn_t callback)
+int opal_util_keyval_parse(const char *filename, opal_keyval_parse_fn_t callback)
 {
     int val;
-    int ret = OPAL_SUCCESS;;
+    int ret = OPAL_SUCCESS;
+    ;
 
     OPAL_THREAD_LOCK(&keyval_mutex);
 
@@ -118,15 +117,13 @@ opal_util_keyval_parse(const char *filename,
         }
     }
     fclose(opal_util_keyval_yyin);
-    opal_util_keyval_yylex_destroy ();
+    opal_util_keyval_yylex_destroy();
 
 cleanup:
     OPAL_THREAD_UNLOCK(&keyval_mutex);
     return ret;
 }
 
-
-
 static int parse_line(void)
 {
     int val;
@@ -137,7 +134,7 @@ static int parse_line(void)
     if (key_buffer_len < strlen(opal_util_keyval_yytext) + 1) {
         char *tmp;
         key_buffer_len = strlen(opal_util_keyval_yytext) + 1;
-        tmp = (char*)realloc(key_buffer, key_buffer_len);
+        tmp = (char *) realloc(key_buffer, key_buffer_len);
         if (NULL == tmp) {
             free(key_buffer);
             key_buffer_len = 0;
@@ -160,23 +157,20 @@ static int parse_line(void)
     /* Next we get the value */
 
     val = opal_util_keyval_yylex();
-    if (OPAL_UTIL_KEYVAL_PARSE_SINGLE_WORD == val ||
-        OPAL_UTIL_KEYVAL_PARSE_VALUE == val) {
+    if (OPAL_UTIL_KEYVAL_PARSE_SINGLE_WORD == val || OPAL_UTIL_KEYVAL_PARSE_VALUE == val) {
         keyval_callback(key_buffer, opal_util_keyval_yytext);
 
         /* Now we need to see the newline */
 
         val = opal_util_keyval_yylex();
-        if (OPAL_UTIL_KEYVAL_PARSE_NEWLINE == val ||
-            OPAL_UTIL_KEYVAL_PARSE_DONE == val) {
+        if (OPAL_UTIL_KEYVAL_PARSE_NEWLINE == val || OPAL_UTIL_KEYVAL_PARSE_DONE == val) {
             return OPAL_SUCCESS;
         }
     }
 
     /* Did we get an EOL or EOF? */
 
-    else if (OPAL_UTIL_KEYVAL_PARSE_DONE == val ||
-             OPAL_UTIL_KEYVAL_PARSE_NEWLINE == val) {
+    else if (OPAL_UTIL_KEYVAL_PARSE_DONE == val || OPAL_UTIL_KEYVAL_PARSE_NEWLINE == val) {
         keyval_callback(key_buffer, NULL);
         return OPAL_SUCCESS;
     }
@@ -186,12 +180,11 @@ static int parse_line(void)
     return OPAL_ERROR;
 }
 
-
 static void parse_error(int num)
 {
     /* JMS need better error/warning message here */
-    opal_output(0, "keyval parser: error %d reading file %s at line %d:\n  %s\n",
-                num, keyval_filename, opal_util_keyval_yynewlines, opal_util_keyval_yytext);
+    opal_output(0, "keyval parser: error %d reading file %s at line %d:\n  %s\n", num,
+                keyval_filename, opal_util_keyval_yynewlines, opal_util_keyval_yytext);
 }
 
 int opal_util_keyval_save_internal_envars(opal_keyval_parse_fn_t callback)
@@ -204,7 +197,7 @@ int opal_util_keyval_save_internal_envars(opal_keyval_parse_fn_t callback)
     return OPAL_SUCCESS;
 }
 
-static void trim_name(char *buffer, const char* prefix, const char* suffix)
+static void trim_name(char *buffer, const char *prefix, const char *suffix)
 {
     char *pchr, *echr;
     size_t buffer_len;
@@ -213,54 +206,54 @@ static void trim_name(char *buffer, const char* prefix, const char* suffix)
         return;
     }
 
-    buffer_len = strlen (buffer);
+    buffer_len = strlen(buffer);
 
     pchr = buffer;
     if (NULL != prefix) {
-        size_t prefix_len = strlen (prefix);
+        size_t prefix_len = strlen(prefix);
 
-        if (0 == strncmp (buffer, prefix, prefix_len)) {
+        if (0 == strncmp(buffer, prefix, prefix_len)) {
             pchr += prefix_len;
         }
     }
 
     /* trim spaces at the beginning */
-    while (isspace (*pchr)) {
+    while (isspace(*pchr)) {
         pchr++;
     }
 
     /* trim spaces at the end */
     echr = buffer + buffer_len;
-    while (echr > buffer && isspace (*(echr - 1))) {
+    while (echr > buffer && isspace(*(echr - 1))) {
         echr--;
     }
     echr[0] = '\0';
 
-    if (NULL != suffix && (uintptr_t) (echr - buffer) > strlen (suffix)) {
-        size_t suffix_len = strlen (suffix);
+    if (NULL != suffix && (uintptr_t)(echr - buffer) > strlen(suffix)) {
+        size_t suffix_len = strlen(suffix);
 
         echr -= suffix_len;
 
-        if (0 == strncmp (echr, suffix, strlen(suffix))) {
+        if (0 == strncmp(echr, suffix, strlen(suffix))) {
             do {
                 echr--;
-            } while (isspace (*echr));
+            } while (isspace(*echr));
             echr[1] = '\0';
         }
     }
 
     if (buffer != pchr) {
         /* move the trimmed string to the beginning of the buffer */
-        memmove (buffer, pchr, strlen (pchr) + 1);
+        memmove(buffer, pchr, strlen(pchr) + 1);
     }
 }
 
-static int save_param_name (void)
+static int save_param_name(void)
 {
     if (key_buffer_len < strlen(opal_util_keyval_yytext) + 1) {
         char *tmp;
         key_buffer_len = strlen(opal_util_keyval_yytext) + 1;
-        tmp = (char*)realloc(key_buffer, key_buffer_len);
+        tmp = (char *) realloc(key_buffer, key_buffer_len);
         if (NULL == tmp) {
             free(key_buffer);
             key_buffer_len = 0;
@@ -270,7 +263,7 @@ static int save_param_name (void)
         key_buffer = tmp;
     }
 
-    opal_string_copy (key_buffer, opal_util_keyval_yytext, key_buffer_len);
+    opal_string_copy(key_buffer, opal_util_keyval_yytext, key_buffer_len);
 
     return OPAL_SUCCESS;
 }
@@ -287,7 +280,7 @@ static int add_to_env_str(char *var, char *val)
     varsz = strlen(var);
     if (NULL != val) {
         valsz = strlen(val);
-        /* If we have a value, it will be preceeded by a '=', so be
+        /* If we have a value, it will be preceded by a '=', so be
            sure to account for that */
         valsz += 1;
     }
@@ -342,22 +335,22 @@ static int parse_line_new(opal_keyval_parse_state_t first_val)
 
     val = first_val;
     while (OPAL_UTIL_KEYVAL_PARSE_NEWLINE != val && OPAL_UTIL_KEYVAL_PARSE_DONE != val) {
-        rc = save_param_name ();
+        rc = save_param_name();
         if (OPAL_SUCCESS != rc) {
             return rc;
         }
 
         if (OPAL_UTIL_KEYVAL_PARSE_MCAVAR == val) {
-            trim_name (key_buffer, "-mca", NULL);
-            trim_name (key_buffer, "--mca", NULL);
+            trim_name(key_buffer, "-mca", NULL);
+            trim_name(key_buffer, "--mca", NULL);
 
             val = opal_util_keyval_yylex();
             if (OPAL_UTIL_KEYVAL_PARSE_VALUE == val) {
                 if (NULL != opal_util_keyval_yytext) {
                     tmp = strdup(opal_util_keyval_yytext);
                     if ('\'' == tmp[0] || '\"' == tmp[0]) {
-                        trim_name (tmp, "\'", "\'");
-                        trim_name (tmp, "\"", "\"");
+                        trim_name(tmp, "\'", "\'");
+                        trim_name(tmp, "\"", "\"");
                     }
                     keyval_callback(key_buffer, tmp);
                     free(tmp);
@@ -367,8 +360,8 @@ static int parse_line_new(opal_keyval_parse_state_t first_val)
                 return OPAL_ERROR;
             }
         } else if (OPAL_UTIL_KEYVAL_PARSE_ENVEQL == val) {
-            trim_name (key_buffer, "-x", "=");
-            trim_name (key_buffer, "--x", NULL);
+            trim_name(key_buffer, "-x", "=");
+            trim_name(key_buffer, "--x", NULL);
 
             val = opal_util_keyval_yylex();
             if (OPAL_UTIL_KEYVAL_PARSE_VALUE == val) {
@@ -378,8 +371,8 @@ static int parse_line_new(opal_keyval_parse_state_t first_val)
                 return OPAL_ERROR;
             }
         } else if (OPAL_UTIL_KEYVAL_PARSE_ENVVAR == val) {
-            trim_name (key_buffer, "-x", "=");
-            trim_name (key_buffer, "--x", NULL);
+            trim_name(key_buffer, "-x", "=");
+            trim_name(key_buffer, "--x", NULL);
             add_to_env_str(key_buffer, NULL);
         } else {
             /* we got something unexpected.  Bonk! */
diff --git a/opal/util/keyval_parse.h b/opal/util/keyval_parse.h
index dd4d8ec82b9..d2034bf7911 100644
--- a/opal/util/keyval_parse.h
+++ b/opal/util/keyval_parse.h
@@ -48,8 +48,7 @@ typedef void (*opal_keyval_parse_fn_t)(const char *key, const char *value);
  * called exactly once.  In a multithreaded context, calls to
  * opal_util_keyval_parse() will serialize multiple calls.
  */
-OPAL_DECLSPEC int opal_util_keyval_parse(const char *filename,
-                                         opal_keyval_parse_fn_t callback);
+OPAL_DECLSPEC int opal_util_keyval_parse(const char *filename, opal_keyval_parse_fn_t callback);
 
 OPAL_DECLSPEC int opal_util_keyval_parse_init(void);
 
diff --git a/opal/util/malloc.c b/opal/util/malloc.c
index be917a379d1..b10167a97a4 100644
--- a/opal/util/malloc.c
+++ b/opal/util/malloc.c
@@ -23,26 +23,25 @@
 
 #include <stdlib.h>
 
+#include "opal/runtime/opal.h"
 #include "opal/util/malloc.h"
 #include "opal/util/output.h"
-#include "opal/runtime/opal.h"
-
 
 /*
  * Undefine "malloc" and "free"
  */
 
 #if defined(malloc)
-#undef malloc
+#    undef malloc
 #endif
 #if defined(calloc)
-#undef calloc
+#    undef calloc
 #endif
 #if defined(free)
-#undef free
+#    undef free
 #endif
 #if defined(realloc)
-#undef realloc
+#    undef realloc
 #endif
 
 /*
@@ -51,8 +50,6 @@
 int opal_malloc_debug_level = OPAL_MALLOC_DEBUG_LEVEL;
 int opal_malloc_output = -1;
 
-
-
 #if OPAL_ENABLE_DEBUG
 
 /*
@@ -83,13 +80,13 @@ void opal_malloc_init(void)
     malloc_stream.lds_prefix = "malloc debug: ";
     malloc_stream.lds_want_stderr = true;
     opal_malloc_output = opal_output_open(&malloc_stream);
-    opal_finalize_register_cleanup (opal_malloc_finalize);
+    opal_finalize_register_cleanup(opal_malloc_finalize);
 }
 #else
-void opal_malloc_init (void)
+void opal_malloc_init(void)
 {
 }
-#endif  /* OPAL_ENABLE_DEBUG */
+#endif /* OPAL_ENABLE_DEBUG */
 
 /*
  * Debug version of malloc
@@ -100,8 +97,8 @@ void *opal_malloc(size_t size, const char *file, int line)
 #if OPAL_ENABLE_DEBUG
     if (opal_malloc_debug_level > 1) {
         if (size <= 0) {
-            opal_output(opal_malloc_output, "Request for %ld bytes (%s, %d)",
-                        (long) size, file, line);
+            opal_output(opal_malloc_output, "Request for %ld bytes (%s, %d)", (long) size, file,
+                        line);
         }
     }
 #endif /* OPAL_ENABLE_DEBUG */
@@ -111,16 +108,14 @@ void *opal_malloc(size_t size, const char *file, int line)
 #if OPAL_ENABLE_DEBUG
     if (opal_malloc_debug_level > 0) {
         if (NULL == addr) {
-            opal_output(opal_malloc_output,
-                        "Request for %ld bytes failed (%s, %d)",
-                        (long) size, file, line);
+            opal_output(opal_malloc_output, "Request for %ld bytes failed (%s, %d)", (long) size,
+                        file, line);
         }
     }
-#endif  /* OPAL_ENABLE_DEBUG */
+#endif /* OPAL_ENABLE_DEBUG */
     return addr;
 }
 
-
 /*
  * Debug version of calloc
  */
@@ -130,12 +125,11 @@ void *opal_calloc(size_t nmembers, size_t size, const char *file, int line)
 #if OPAL_ENABLE_DEBUG
     if (opal_malloc_debug_level > 1) {
         if (size <= 0) {
-            opal_output(opal_malloc_output,
-                        "Request for %ld zeroed elements of size %ld (%s, %d)",
+            opal_output(opal_malloc_output, "Request for %ld zeroed elements of size %ld (%s, %d)",
                         (long) nmembers, (long) size, file, line);
         }
     }
-#endif  /* OPAL_ENABLE_DEBUG */
+#endif /* OPAL_ENABLE_DEBUG */
     addr = calloc(nmembers, size);
 #if OPAL_ENABLE_DEBUG
     if (opal_malloc_debug_level > 0) {
@@ -145,11 +139,10 @@ void *opal_calloc(size_t nmembers, size_t size, const char *file, int line)
                         (long) nmembers, (long) size, file, line);
         }
     }
-#endif  /* OPAL_ENABLE_DEBUG */
+#endif /* OPAL_ENABLE_DEBUG */
     return addr;
 }
 
-
 /*
  * Debug version of realloc
  */
@@ -160,30 +153,27 @@ void *opal_realloc(void *ptr, size_t size, const char *file, int line)
     if (opal_malloc_debug_level > 1) {
         if (size <= 0) {
             if (NULL == ptr) {
-                opal_output(opal_malloc_output,
-                            "Realloc NULL for %ld bytes (%s, %d)",
-                            (long) size, file, line);
+                opal_output(opal_malloc_output, "Realloc NULL for %ld bytes (%s, %d)", (long) size,
+                            file, line);
             } else {
-                opal_output(opal_malloc_output, "Realloc %p for %ld bytes (%s, %d)",
-                            ptr, (long) size, file, line);
+                opal_output(opal_malloc_output, "Realloc %p for %ld bytes (%s, %d)", ptr,
+                            (long) size, file, line);
             }
         }
     }
-#endif  /* OPAL_ENABLE_DEBUG */
+#endif /* OPAL_ENABLE_DEBUG */
     addr = realloc(ptr, size);
 #if OPAL_ENABLE_DEBUG
     if (opal_malloc_debug_level > 0) {
         if (NULL == addr) {
-            opal_output(opal_malloc_output,
-                        "Realloc %p for %ld bytes failed (%s, %d)",
-                        ptr, (long) size, file, line);
+            opal_output(opal_malloc_output, "Realloc %p for %ld bytes failed (%s, %d)", ptr,
+                        (long) size, file, line);
         }
     }
-#endif  /* OPAL_ENABLE_DEBUG */
+#endif /* OPAL_ENABLE_DEBUG */
     return addr;
 }
 
-
 /*
  * Debug version of free
  */
@@ -196,5 +186,5 @@ void opal_malloc_debug(int level)
 {
 #if OPAL_ENABLE_DEBUG
     opal_malloc_debug_level = level;
-#endif  /* OPAL_ENABLE_DEBUG */
+#endif /* OPAL_ENABLE_DEBUG */
 }
diff --git a/opal/util/malloc.h b/opal/util/malloc.h
index 26f9cf5e2be..1c777037a1a 100644
--- a/opal/util/malloc.h
+++ b/opal/util/malloc.h
@@ -12,6 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2018      Triad National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -43,88 +44,91 @@
  */
 
 #ifndef OPAL_MALLOC_DEBUG_LEVEL
-#define OPAL_MALLOC_DEBUG_LEVEL 2
+#    define OPAL_MALLOC_DEBUG_LEVEL 2
 #endif
 
 BEGIN_C_DECLS
-  /**
-   * Initialize malloc debug output.
-   *
-   * This function is invoked to setup a dedicated output stream for
-   * malloc debug functions.  It does \em not (currently) do anything
-   * other than that (i.e., no internal accounting for tracking
-   * malloc/free statements, etc.).
-   *
-   * It is invoked as part of opal_init().  Although this function is
-   * not \em necessary for OPAL_MALLOC() and OPAL_FREE(), it is strong
-   * recommended because no output messages -- regardless of the
-   * malloc debug level set by opal_malloc_debug() -- will be displayed
-   * unless this function is invoked first.
-   */
+/**
+ * Initialize malloc debug output.
+ *
+ * This function is invoked to setup a dedicated output stream for
+ * malloc debug functions.  It does \em not (currently) do anything
+ * other than that (i.e., no internal accounting for tracking
+ * malloc/free statements, etc.).
+ *
+ * It is invoked as part of opal_init().  Although this function is
+ * not \em necessary for OPAL_MALLOC() and OPAL_FREE(), it is strong
+ * recommended because no output messages -- regardless of the
+ * malloc debug level set by opal_malloc_debug() -- will be displayed
+ * unless this function is invoked first.
+ */
 void opal_malloc_init(void);
 
-  /**
-   * \internal
-   *
-   * Back-end error-checking malloc function for OPAL (you should use
-   * the normal malloc() instead of this function).
-   *
-   * @param size The number of bytes to allocate
-   * @param file Typically the __FILE__ macro
-   * @param line Typically the __LINE__ macro
-   *
-   * This function is only used when --enable-mem-debug was specified to
-   * configure (or by default if you're building in a SVN checkout).
-   */
-OPAL_DECLSPEC void *opal_malloc(size_t size, const char *file, int line) __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
+/**
+ * \internal
+ *
+ * Back-end error-checking malloc function for OPAL (you should use
+ * the normal malloc() instead of this function).
+ *
+ * @param size The number of bytes to allocate
+ * @param file Typically the __FILE__ macro
+ * @param line Typically the __LINE__ macro
+ *
+ * This function is only used when --enable-mem-debug was specified to
+ * configure (or by default if you're building in a SVN checkout).
+ */
+OPAL_DECLSPEC void *opal_malloc(size_t size, const char *file, int line)
+    __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
 
-  /**
-   * \internal
-   *
-   * Back-end error-checking calloc function for OPAL (you should use
-   * the normal calloc() instead of this function).
-   *
-   * @param nmembers Number of elements to malloc
-   * @param size Size of each elements
-   * @param file Typically the __FILE__ macro
-   * @param line Typically the __LINE__ macro
-   *
-   * This function is only used when --enable-mem-debug was specified to
-   * configure (or by default if you're building in a SVN checkout).
-   */
-OPAL_DECLSPEC void *opal_calloc(size_t nmembers, size_t size, const char *file, int line) __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
+/**
+ * \internal
+ *
+ * Back-end error-checking calloc function for OPAL (you should use
+ * the normal calloc() instead of this function).
+ *
+ * @param nmembers Number of elements to malloc
+ * @param size Size of each elements
+ * @param file Typically the __FILE__ macro
+ * @param line Typically the __LINE__ macro
+ *
+ * This function is only used when --enable-mem-debug was specified to
+ * configure (or by default if you're building in a SVN checkout).
+ */
+OPAL_DECLSPEC void *opal_calloc(size_t nmembers, size_t size, const char *file, int line)
+    __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
 
-  /**
-   * \internal
-   *
-   * Back-end error-checking realloc function for OPAL (you should use
-   * the normal realloc() instead of this function).
-   *
-   * @param ptr Pointer to reallocate
-   * @param size The number of bytes to allocate
-   * @param file Typically the __FILE__ macro
-   * @param line Typically the __LINE__ macro
-   *
-   * This function is only used when --enable-mem-debug was specified to
-   * configure (or by default if you're building in a SVN checkout).
-   */
-OPAL_DECLSPEC void *opal_realloc(void *ptr, size_t size, const char *file, int line) __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
+/**
+ * \internal
+ *
+ * Back-end error-checking realloc function for OPAL (you should use
+ * the normal realloc() instead of this function).
+ *
+ * @param ptr Pointer to reallocate
+ * @param size The number of bytes to allocate
+ * @param file Typically the __FILE__ macro
+ * @param line Typically the __LINE__ macro
+ *
+ * This function is only used when --enable-mem-debug was specified to
+ * configure (or by default if you're building in a SVN checkout).
+ */
+OPAL_DECLSPEC void *opal_realloc(void *ptr, size_t size, const char *file, int line)
+    __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
 
-  /**
-   * \internal
-   *
-   * Back-end error-checking free function for OPAL (you should use
-   * free() instead of this function).
-   *
-   * @param addr Address on the heap to free()
-   * @param file Typically the __FILE__ macro
-   * @param line Typically the __LINE__ macro
-   *
-   * This function is only used when --enable-mem-debug was specified
-   * to configure (or by default if you're building in a SVN
-   * checkout).
-   */
-OPAL_DECLSPEC void opal_free(void *addr, const char *file, int line) __opal_attribute_nonnull__(1);
+/**
+ * \internal
+ *
+ * Back-end error-checking free function for OPAL (you should use
+ * free() instead of this function).
+ *
+ * @param addr Address on the heap to free()
+ * @param file Typically the __FILE__ macro
+ * @param line Typically the __LINE__ macro
+ *
+ * This function is only used when --enable-mem-debug was specified
+ * to configure (or by default if you're building in a SVN
+ * checkout).
+ */
+OPAL_DECLSPEC void opal_free(void *addr, const char *file, int line);
 
 /**
  * Used to set the debug level for malloc debug.
diff --git a/opal/util/minmax.h b/opal/util/minmax.h
index d56a7c39311..5b7f3c768b8 100644
--- a/opal/util/minmax.h
+++ b/opal/util/minmax.h
@@ -16,14 +16,16 @@
 #ifndef OPAL_MINMAX_H
 #define OPAL_MINMAX_H
 
-#define OPAL_DEFINE_MINMAX(type, suffix)                                                           \
-    static inline const type opal_min_##suffix(const type a, const type b)                         \
-    {                                                                                              \
-        return (a < b) ? a : b;                                                                    \
-    }                                                                                              \
-    static inline const type opal_max_##suffix(const type a, const type b)                         \
-    {                                                                                              \
-        return (a > b) ? a : b;                                                                    \
+#include "opal_stdint.h"
+
+#define OPAL_DEFINE_MINMAX(type, suffix)                                   \
+    static inline const type opal_min_##suffix(const type a, const type b) \
+    {                                                                      \
+        return (a < b) ? a : b;                                            \
+    }                                                                      \
+    static inline const type opal_max_##suffix(const type a, const type b) \
+    {                                                                      \
+        return (a > b) ? a : b;                                            \
     }
 
 OPAL_DEFINE_MINMAX(int8_t, 8)
@@ -41,8 +43,8 @@ OPAL_DEFINE_MINMAX(double, double)
 OPAL_DEFINE_MINMAX(void *, ptr)
 
 #if OPAL_C_HAVE__GENERIC
-#define opal_min(a, b)                                                  \
-    (_Generic((a) + (b),                                                \
+#    define opal_min(a, b) \
+        (_Generic((a) + (b),                                                \
                                  int8_t: opal_min_8,                    \
                                  uint8_t: opal_min_u8,                  \
                                  int16_t: opal_min_16,                  \
@@ -56,8 +58,8 @@ OPAL_DEFINE_MINMAX(void *, ptr)
                                  void *: opal_min_ptr,                  \
                                  default: opal_min_64)((a), (b)))
 
-#define opal_max(a, b)                                                  \
-    (_Generic((a) + (b),                                                \
+#    define opal_max(a, b) \
+        (_Generic((a) + (b),                                                \
                                  int8_t: opal_max_8,                    \
                                  uint8_t: opal_max_u8,                  \
                                  int16_t: opal_max_16,                  \
@@ -74,8 +76,8 @@ OPAL_DEFINE_MINMAX(void *, ptr)
 
 /* these versions suffer from double-evaluation. please upgrade to a modern compiler */
 
-#define opal_min(a, b) (((a) < (b)) ? (a) : (b))
-#define opal_max(a, b) (((a) > (b)) ? (a) : (b))
+#    define opal_min(a, b) (((a) < (b)) ? (a) : (b))
+#    define opal_max(a, b) (((a) > (b)) ? (a) : (b))
 
 #endif
 
diff --git a/opal/util/misc.h b/opal/util/misc.h
new file mode 100644
index 00000000000..7f0d5b3137a
--- /dev/null
+++ b/opal/util/misc.h
@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2024      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/** @file:
+ * Miscellaneous utilities
+ */
+
+#ifndef OPAL_UTIL_MISC_H
+#define OPAL_UTIL_MISC_H
+
+#include "opal/include/opal_config.h"
+
+BEGIN_C_DECLS
+
+#ifndef container_of
+#    define container_of(ptr, type, member) ((type *) (((char *) (ptr)) - offsetof(type, member)))
+#endif
+
+END_C_DECLS
+#endif
diff --git a/opal/util/net.c b/opal/util/net.c
index 7c16d03d5d7..ec0c372dd99 100644
--- a/opal/util/net.c
+++ b/opal/util/net.c
@@ -32,45 +32,45 @@
 #include <stdio.h>
 #include <string.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 #include <errno.h>
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 #ifdef HAVE_SYS_SOCKET_H
-#include <sys/socket.h>
+#    include <sys/socket.h>
 #endif
 #ifdef HAVE_SYS_SOCKIO_H
-#include <sys/sockio.h>
+#    include <sys/sockio.h>
 #endif
 #ifdef HAVE_SYS_IOCTL_H
-#include <sys/ioctl.h>
+#    include <sys/ioctl.h>
 #endif
 #ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
+#    include <netinet/in.h>
 #endif
 #ifdef HAVE_ARPA_INET_H
-#include <arpa/inet.h>
+#    include <arpa/inet.h>
 #endif
 #ifdef HAVE_NET_IF_H
-#include <net/if.h>
+#    include <net/if.h>
 #endif
 #ifdef HAVE_NETDB_H
-#include <netdb.h>
+#    include <netdb.h>
 #endif
 #ifdef HAVE_IFADDRS_H
-#include <ifaddrs.h>
+#    include <ifaddrs.h>
 #endif
 
+#include "opal/constants.h"
+#include "opal/mca/threads/tsd.h"
 #include "opal/runtime/opal.h"
+#include "opal/runtime/opal_params.h"
+#include "opal/util/argv.h"
 #include "opal/util/net.h"
 #include "opal/util/output.h"
-#include "opal/util/argv.h"
 #include "opal/util/show_help.h"
-#include "opal/constants.h"
-#include "opal/mca/threads/tsd.h"
-#include "opal/runtime/opal_params.h"
 
 /* this function doesn't depend on sockaddr_h */
 bool opal_net_isaddr(const char *name)
@@ -102,36 +102,34 @@ typedef struct private_ipv4_t {
     uint32_t netmask_bits;
 } private_ipv4_t;
 
-static private_ipv4_t* private_ipv4 = NULL;
+static private_ipv4_t *private_ipv4 = NULL;
 
-#if OPAL_ENABLE_IPV6
+#    if OPAL_ENABLE_IPV6
 static opal_tsd_tracked_key_t *hostname_tsd_key = NULL;
 
-
-static void
-hostname_cleanup(void *value)
+static void hostname_cleanup(void *value)
 {
-    if (NULL != value) free(value);
+    if (NULL != value)
+        free(value);
 }
 
-
-static char*
-get_hostname_buffer(void)
+static char *get_hostname_buffer(void)
 {
     void *buffer;
     int ret;
 
     ret = opal_tsd_tracked_key_get(hostname_tsd_key, &buffer);
-    if (OPAL_SUCCESS != ret) return NULL;
+    if (OPAL_SUCCESS != ret)
+        return NULL;
 
     if (NULL == buffer) {
-        buffer = (void*) malloc((NI_MAXHOST + 1) * sizeof(char));
+        buffer = (void *) malloc((NI_MAXHOST + 1) * sizeof(char));
         ret = opal_tsd_tracked_key_set(hostname_tsd_key, buffer);
     }
 
-    return (char*) buffer;
+    return (char *) buffer;
 }
-#endif
+#    endif
 
 /**
  * Finalize the network helper subsystem
@@ -142,45 +140,42 @@ get_hostname_buffer(void)
  *
  * @retval OPAL_SUCCESS   Success
  */
-static void opal_net_finalize (void)
+static void opal_net_finalize(void)
 {
-#if OPAL_ENABLE_IPV6
+#    if OPAL_ENABLE_IPV6
     if (NULL != hostname_tsd_key) {
         OBJ_RELEASE(hostname_tsd_key);
         hostname_tsd_key = NULL;
     }
-#endif
+#    endif
     free(private_ipv4);
     private_ipv4 = NULL;
 }
 
-int
-opal_net_init(void)
+int opal_net_init(void)
 {
     char **args, *arg;
     uint32_t a, b, c, d, bits, addr;
     int i, count, found_bad = 0;
 
-    args = opal_argv_split( opal_net_private_ipv4, ';' );
-    if( NULL != args ) {
+    args = opal_argv_split(opal_net_private_ipv4, ';');
+    if (NULL != args) {
         count = opal_argv_count(args);
-        private_ipv4 = (private_ipv4_t*)malloc( (count + 1) * sizeof(private_ipv4_t));
-        if( NULL == private_ipv4 ) {
-            opal_output(0, "Unable to allocate memory for the private addresses array" );
+        private_ipv4 = (private_ipv4_t *) malloc((count + 1) * sizeof(private_ipv4_t));
+        if (NULL == private_ipv4) {
+            opal_output(0, "Unable to allocate memory for the private addresses array");
             opal_argv_free(args);
             goto do_local_init;
         }
-        for( i = 0; i < count; i++ ) {
+        for (i = 0; i < count; i++) {
             arg = args[i];
 
-            (void)sscanf( arg, "%u.%u.%u.%u/%u", &a, &b, &c, &d, &bits );
+            (void) sscanf(arg, "%u.%u.%u.%u/%u", &a, &b, &c, &d, &bits);
 
-            if( (a > 255) || (b > 255) || (c > 255) ||
-                (d > 255) || (bits > 32) ) {
+            if ((a > 255) || (b > 255) || (c > 255) || (d > 255) || (bits > 32)) {
                 if (0 == found_bad) {
-                    opal_show_help("help-opal-util.txt",
-                                   "malformed net_private_ipv4",
-                                   true, args[i]);
+                    opal_show_help("help-opal-util.txt", "malformed net_private_ipv4", true,
+                                   args[i]);
                     found_bad = 1;
                 }
                 continue;
@@ -189,208 +184,182 @@ opal_net_init(void)
             private_ipv4[i].addr = htonl(addr);
             private_ipv4[i].netmask_bits = bits;
         }
-        private_ipv4[i].addr         = 0;
+        private_ipv4[i].addr = 0;
         private_ipv4[i].netmask_bits = 0;
         opal_argv_free(args);
     }
 
-    opal_finalize_register_cleanup (opal_net_finalize);
+    opal_finalize_register_cleanup(opal_net_finalize);
 
- do_local_init:
-#if OPAL_ENABLE_IPV6
+do_local_init:
+#    if OPAL_ENABLE_IPV6
     hostname_tsd_key = OBJ_NEW(opal_tsd_tracked_key_t);
     opal_tsd_tracked_key_set_destructor(hostname_tsd_key, hostname_cleanup);
-#endif
+#    endif
     return OPAL_SUCCESS;
 }
 
 /* convert a CIDR prefixlen to netmask (in network byte order) */
-uint32_t
-opal_net_prefix2netmask(uint32_t prefixlen)
+uint32_t opal_net_prefix2netmask(uint32_t prefixlen)
 {
-    return htonl (((1u << prefixlen) - 1u) << (32 - prefixlen));
+    return htonl(((1u << prefixlen) - 1u) << (32 - prefixlen));
 }
 
-
-bool
-opal_net_islocalhost(const struct sockaddr *addr)
+bool opal_net_islocalhost(const struct sockaddr *addr)
 {
     switch (addr->sa_family) {
-    case AF_INET:
-        {
-            const struct sockaddr_in *inaddr = (struct sockaddr_in*) addr;
-            /* if it's in the 127. domain, it shouldn't be routed
-               (0x7f == 127) */
-            if (0x7F000000 == (0x7F000000 & ntohl(inaddr->sin_addr.s_addr))) {
-                return true;
-            }
-            return false;
+    case AF_INET: {
+        const struct sockaddr_in *inaddr = (struct sockaddr_in *) addr;
+        /* if it's in the 127. domain, it shouldn't be routed
+           (0x7f == 127) */
+        if (0x7F000000 == (0x7F000000 & ntohl(inaddr->sin_addr.s_addr))) {
+            return true;
         }
-        break;
-#if OPAL_ENABLE_IPV6
-    case AF_INET6:
-        {
-            const struct sockaddr_in6 *inaddr = (struct sockaddr_in6*) addr;
-            if (IN6_IS_ADDR_LOOPBACK (&inaddr->sin6_addr)) {
-               return true; /* Bug, FIXME: check for 127.0.0.1/8 */
-            }
-            return false;
+        return false;
+    } break;
+#    if OPAL_ENABLE_IPV6
+    case AF_INET6: {
+        const struct sockaddr_in6 *inaddr = (struct sockaddr_in6 *) addr;
+        if (IN6_IS_ADDR_LOOPBACK(&inaddr->sin6_addr)) {
+            return true; /* Bug, FIXME: check for 127.0.0.1/8 */
         }
-        break;
-#endif
+        return false;
+    } break;
+#    endif
 
     default:
-        opal_output(0, "unhandled sa_family %d passed to opal_net_islocalhost",
-                    addr->sa_family);
+        opal_output(0, "unhandled sa_family %d passed to opal_net_islocalhost", addr->sa_family);
         return false;
         break;
     }
 }
 
-
-bool
-opal_net_samenetwork(const struct sockaddr *addr1,
-                     const struct sockaddr *addr2,
-                     uint32_t plen)
+bool opal_net_samenetwork(const struct sockaddr *addr1, const struct sockaddr *addr2, uint32_t plen)
 {
     uint32_t prefixlen;
 
-    if(addr1->sa_family != addr2->sa_family) {
+    if (addr1->sa_family != addr2->sa_family) {
         return false; /* address families must be equal */
     }
 
     switch (addr1->sa_family) {
-    case AF_INET:
-        {
-            if (0 == plen) {
-                prefixlen = 32;
-            } else {
-                prefixlen = plen;
-            }
-            struct sockaddr_in inaddr1, inaddr2;
-            /* Use temporary variables and memcpy's so that we don't
-               run into bus errors on Solaris/SPARC */
-            memcpy(&inaddr1, addr1, sizeof(inaddr1));
-            memcpy(&inaddr2, addr2, sizeof(inaddr2));
-            uint32_t netmask = opal_net_prefix2netmask (prefixlen);
-
-            if((inaddr1.sin_addr.s_addr & netmask) ==
-               (inaddr2.sin_addr.s_addr & netmask)) {
+    case AF_INET: {
+        if (0 == plen) {
+            prefixlen = 32;
+        } else {
+            prefixlen = plen;
+        }
+        struct sockaddr_in inaddr1, inaddr2;
+        /* Use temporary variables and memcpy's so that we don't
+           run into bus errors on Solaris/SPARC */
+        memcpy(&inaddr1, addr1, sizeof(inaddr1));
+        memcpy(&inaddr2, addr2, sizeof(inaddr2));
+        uint32_t netmask = opal_net_prefix2netmask(prefixlen);
+
+        if ((inaddr1.sin_addr.s_addr & netmask) == (inaddr2.sin_addr.s_addr & netmask)) {
+            return true;
+        }
+        return false;
+    } break;
+#    if OPAL_ENABLE_IPV6
+    case AF_INET6: {
+        struct sockaddr_in6 inaddr1, inaddr2;
+        /* Use temporary variables and memcpy's so that we don't
+           run into bus errors on Solaris/SPARC */
+        memcpy(&inaddr1, addr1, sizeof(inaddr1));
+        memcpy(&inaddr2, addr2, sizeof(inaddr2));
+        struct in6_addr *a6_1 = (struct in6_addr *) &inaddr1.sin6_addr;
+        struct in6_addr *a6_2 = (struct in6_addr *) &inaddr2.sin6_addr;
+
+        if (0 == plen) {
+            prefixlen = 64;
+        } else {
+            prefixlen = plen;
+        }
+        if (64 == prefixlen) {
+            /* prefixlen is always /64, any other case would be routing.
+               Compare the first eight bytes (64 bits) and hope that
+               endianness is not an issue on any system as long as
+               addresses are always stored in network byte order.
+            */
+            if (((const uint32_t *) (a6_1))[0] == ((const uint32_t *) (a6_2))[0]
+                && ((const uint32_t *) (a6_1))[1] == ((const uint32_t *) (a6_2))[1]) {
                 return true;
             }
-            return false;
         }
-        break;
-#if OPAL_ENABLE_IPV6
-    case AF_INET6:
-        {
-            struct sockaddr_in6 inaddr1, inaddr2;
-            /* Use temporary variables and memcpy's so that we don't
-               run into bus errors on Solaris/SPARC */
-            memcpy(&inaddr1, addr1, sizeof(inaddr1));
-            memcpy(&inaddr2, addr2, sizeof(inaddr2));
-            struct in6_addr *a6_1 = (struct in6_addr*) &inaddr1.sin6_addr;
-            struct in6_addr *a6_2 = (struct in6_addr*) &inaddr2.sin6_addr;
-
-            if (0 == plen) {
-                prefixlen = 64;
-            } else {
-                prefixlen = plen;
-            }
-            if (64 == prefixlen) {
-                /* prefixlen is always /64, any other case would be routing.
-                   Compare the first eight bytes (64 bits) and hope that
-                   endianess is not an issue on any system as long as
-                   addresses are always stored in network byte order.
-                */
-                if (((const uint32_t *) (a6_1))[0] ==
-                    ((const uint32_t *) (a6_2))[0] &&
-                    ((const uint32_t *) (a6_1))[1] ==
-                    ((const uint32_t *) (a6_2))[1]) {
-                    return true;
-                }
-            }
-            return false;
-        }
-        break;
-#endif
+        return false;
+    } break;
+#    endif
     default:
-        opal_output(0, "unhandled sa_family %d passed to opal_samenetwork",
-                    addr1->sa_family);
+        opal_output(0, "unhandled sa_family %d passed to opal_samenetwork", addr1->sa_family);
     }
 
     return false;
 }
 
-
 /**
  * Returns true if the given address is a public IPv4 address.
  */
-bool
-opal_net_addr_isipv4public(const struct sockaddr *addr)
+bool opal_net_addr_isipv4public(const struct sockaddr *addr)
 {
     switch (addr->sa_family) {
-#if OPAL_ENABLE_IPV6
-        case AF_INET6:
-            return false;
-#endif
-        case AF_INET:
-            {
-                const struct sockaddr_in *inaddr = (struct sockaddr_in*) addr;
-                int i;
-
-                if( NULL == private_ipv4 ) {
-                    return true;
-                }
+#    if OPAL_ENABLE_IPV6
+    case AF_INET6:
+        return false;
+#    endif
+    case AF_INET: {
+        const struct sockaddr_in *inaddr = (struct sockaddr_in *) addr;
+        int i;
 
-                for( i = 0; private_ipv4[i].addr != 0; i++ ) {
-                    if( private_ipv4[i].addr == (inaddr->sin_addr.s_addr &
-                                                 opal_net_prefix2netmask(private_ipv4[i].netmask_bits)) )
-                        return false;
-                }
+        if (NULL == private_ipv4) {
+            return true;
+        }
 
+        for (i = 0; private_ipv4[i].addr != 0; i++) {
+            if (private_ipv4[i].addr
+                == (inaddr->sin_addr.s_addr
+                    & opal_net_prefix2netmask(private_ipv4[i].netmask_bits))) {
+                return false;
             }
-            return true;
-        default:
-            opal_output (0,
-                         "unhandled sa_family %d passed to opal_net_addr_isipv4public\n",
-                         addr->sa_family);
+        }
+    }
+        return true;
+    default:
+        opal_output(0, "unhandled sa_family %d passed to opal_net_addr_isipv4public\n",
+                    addr->sa_family);
     }
 
     return false;
 }
 
-bool
-opal_net_addr_isipv6linklocal(const struct sockaddr *addr)
+bool opal_net_addr_isipv6linklocal(const struct sockaddr *addr)
 {
-#if OPAL_ENABLE_IPV6
+#    if OPAL_ENABLE_IPV6
     struct sockaddr_in6 if_addr;
-#endif
+#    endif
 
     switch (addr->sa_family) {
-#if OPAL_ENABLE_IPV6
-        case AF_INET6:
-            if_addr.sin6_family = AF_INET6;
-            if (1 != inet_pton(AF_INET6, "fe80::0000", &if_addr.sin6_addr)) {
-                return false;
-            }
-            return opal_net_samenetwork(addr, (struct sockaddr*)&if_addr, 64);
-#endif
-        case AF_INET:
+#    if OPAL_ENABLE_IPV6
+    case AF_INET6:
+        if_addr.sin6_family = AF_INET6;
+        if (1 != inet_pton(AF_INET6, "fe80::0000", &if_addr.sin6_addr)) {
             return false;
-        default:
-            opal_output (0,
-                         "unhandled sa_family %d passed to opal_net_addr_isipv6linklocal\n",
-                         addr->sa_family);
+        }
+        return opal_net_samenetwork(addr, (struct sockaddr *) &if_addr, 64);
+#    endif
+    case AF_INET:
+        return false;
+    default:
+        opal_output(0, "unhandled sa_family %d passed to opal_net_addr_isipv6linklocal\n",
+                    addr->sa_family);
     }
 
     return false;
 }
 
-char*
-opal_net_get_hostname(const struct sockaddr *addr)
+char *opal_net_get_hostname(const struct sockaddr *addr)
 {
-#if OPAL_ENABLE_IPV6
+#    if OPAL_ENABLE_IPV6
     char *name = get_hostname_buffer();
     int error;
     socklen_t addrlen;
@@ -404,119 +373,98 @@ opal_net_get_hostname(const struct sockaddr *addr)
 
     switch (addr->sa_family) {
     case AF_INET:
-        addrlen = sizeof (struct sockaddr_in);
+        addrlen = sizeof(struct sockaddr_in);
         break;
     case AF_INET6:
-#if defined( __NetBSD__)
+#        if defined(__NetBSD__)
         /* hotfix for netbsd: on my netbsd machine, getnameinfo
-           returns an unkown error code. */
-        if(NULL == inet_ntop(AF_INET6, &((struct sockaddr_in6*) addr)->sin6_addr,
-                             name, NI_MAXHOST)) {
+           returns an unknown error code. */
+        if (NULL
+            == inet_ntop(AF_INET6, &((struct sockaddr_in6 *) addr)->sin6_addr, name, NI_MAXHOST)) {
             opal_output(0, "opal_sockaddr2str failed with error code %d", errno);
             return NULL;
         }
         return name;
-#else
-        addrlen = sizeof (struct sockaddr_in6);
-#endif
+#        else
+        addrlen = sizeof(struct sockaddr_in6);
+#        endif
         break;
     default:
         return NULL;
     }
 
-    error = getnameinfo(addr, addrlen,
-                        name, NI_MAXHOST, NULL, 0, NI_NUMERICHOST);
+    error = getnameinfo(addr, addrlen, name, NI_MAXHOST, NULL, 0, NI_NUMERICHOST);
 
     if (error) {
-       int err = errno;
-       opal_output (0, "opal_sockaddr2str failed:%s (return code %i)\n",
-                    gai_strerror(err), error);
-       return NULL;
+        int err = errno;
+        opal_output(0, "opal_sockaddr2str failed:%s (return code %i)\n", gai_strerror(err), error);
+        return NULL;
     }
     /* strip any trailing % data as it isn't pertinent */
     if (NULL != (p = strrchr(name, '%'))) {
         *p = '\0';
     }
     return name;
-#else
-    return inet_ntoa(((struct sockaddr_in*) addr)->sin_addr);
-#endif
+#    else
+    return inet_ntoa(((struct sockaddr_in *) addr)->sin_addr);
+#    endif
 }
 
-
-int
-opal_net_get_port(const struct sockaddr *addr)
+int opal_net_get_port(const struct sockaddr *addr)
 {
     switch (addr->sa_family) {
     case AF_INET:
-        return ntohs(((struct sockaddr_in*) addr)->sin_port);
+        return ntohs(((struct sockaddr_in *) addr)->sin_port);
         break;
-#if OPAL_ENABLE_IPV6
+#    if OPAL_ENABLE_IPV6
     case AF_INET6:
-        return ntohs(((struct sockaddr_in6*) addr)->sin6_port);
+        return ntohs(((struct sockaddr_in6 *) addr)->sin6_port);
         break;
-#endif
+#    endif
     }
 
     return -1;
 }
 
-
 #else /* HAVE_STRUCT_SOCKADDR_IN */
 
-int
-opal_net_init()
+int opal_net_init()
 {
     return OPAL_SUCCESS;
 }
 
-
-int
-opal_net_finalize()
+int opal_net_finalize()
 {
     return OPAL_SUCCESS;
 }
 
-
-uint32_t
-opal_net_prefix2netmask(uint32_t prefixlen)
+uint32_t opal_net_prefix2netmask(uint32_t prefixlen)
 {
     return 0;
 }
 
-
-bool
-opal_net_islocalhost(const struct sockaddr *addr)
+bool opal_net_islocalhost(const struct sockaddr *addr)
 {
     return false;
 }
 
-
-bool
-opal_net_samenetwork(const struct sockaddr *addr1,
-                     const struct sockaddr *addr2,
-                     uint32_t prefixlen)
+bool opal_net_samenetwork(const struct sockaddr *addr1, const struct sockaddr *addr2,
+                          uint32_t prefixlen)
 {
     return false;
 }
 
-
-bool
-opal_net_addr_isipv4public(const struct sockaddr *addr)
+bool opal_net_addr_isipv4public(const struct sockaddr *addr)
 {
     return false;
 }
 
-
-char*
-opal_net_get_hostname(const struct sockaddr *addr)
+char *opal_net_get_hostname(const struct sockaddr *addr)
 {
     return NULL;
 }
 
-
-int
-opal_net_get_port(const struct sockaddr *addr)
+int opal_net_get_port(const struct sockaddr *addr)
 {
     return -1;
 }
diff --git a/opal/util/net.h b/opal/util/net.h
index f880b9e4da9..ebf992fbf6a 100644
--- a/opal/util/net.h
+++ b/opal/util/net.h
@@ -29,19 +29,19 @@
 #include "opal_config.h"
 
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 #ifdef HAVE_SYS_SOCKET_H
-#include <sys/socket.h>
+#    include <sys/socket.h>
 #endif
 #ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
+#    include <netinet/in.h>
 #endif
 
 BEGIN_C_DECLS
 
 /**
- * Intiailize the network helper subsystem
+ * Initialize the network helper subsystem
  *
  * Initialize the network helper subsystem.  Should be called exactly
  * once for any process that will use any function in the network
@@ -61,7 +61,6 @@ OPAL_DECLSPEC int opal_net_init(void);
  */
 OPAL_DECLSPEC uint32_t opal_net_prefix2netmask(uint32_t prefixlen);
 
-
 /**
  * Determine if given IP address is in the localhost range
  *
@@ -75,7 +74,6 @@ OPAL_DECLSPEC uint32_t opal_net_prefix2netmask(uint32_t prefixlen);
  */
 OPAL_DECLSPEC bool opal_net_islocalhost(const struct sockaddr *addr);
 
-
 /**
  * Are we on the same network?
  *
@@ -88,11 +86,9 @@ OPAL_DECLSPEC bool opal_net_islocalhost(const struct sockaddr *addr);
  * @return                  true if \c addr1 and \c addr2 are on the
  *                          same net, false otherwise.
  */
-OPAL_DECLSPEC bool opal_net_samenetwork(const struct sockaddr *addr1,
-                                        const struct sockaddr *addr2,
+OPAL_DECLSPEC bool opal_net_samenetwork(const struct sockaddr *addr1, const struct sockaddr *addr2,
                                         uint32_t prefixlen);
 
-
 /**
  * Is the given address a public IPv4 address?  Returns false for IPv6
  * address.
@@ -121,8 +117,7 @@ OPAL_DECLSPEC bool opal_net_addr_isipv6linklocal(const struct sockaddr *addr);
  * @param addr              struct sockaddr of address
  * @return                  literal representation of \c addr
  */
-OPAL_DECLSPEC char* opal_net_get_hostname(const struct sockaddr *addr);
-
+OPAL_DECLSPEC char *opal_net_get_hostname(const struct sockaddr *addr);
 
 /**
  * Get port number from struct sockaddr
diff --git a/opal/util/numtostr.c b/opal/util/numtostr.c
index 349378b286a..c937b5b1fdb 100644
--- a/opal/util/numtostr.c
+++ b/opal/util/numtostr.c
@@ -22,17 +22,17 @@
 #include <stdio.h>
 #include <stdlib.h>
 
-
-char*
-opal_ltostr(long num)
+char *opal_ltostr(long num)
 {
     /* waste a little bit of space, but always have a big enough buffer */
     int buflen = sizeof(long) * 8;
     char *buf = NULL;
     int ret = 0;
 
-    buf = (char*) malloc(sizeof(char) * buflen);
-    if (NULL == buf) return NULL;
+    buf = (char *) malloc(sizeof(char) * buflen);
+    if (NULL == buf) {
+        return NULL;
+    }
 
     ret = snprintf(buf, buflen, "%ld", num);
     if (ret < 0) {
@@ -43,17 +43,17 @@ opal_ltostr(long num)
     return buf;
 }
 
-
-char*
-opal_dtostr(double num)
+char *opal_dtostr(double num)
 {
     /* waste a little bit of space, but always have a big enough buffer */
     int buflen = sizeof(long) * 8;
     char *buf = NULL;
     int ret = 0;
 
-    buf = (char*) malloc(sizeof(char) * buflen);
-    if (NULL == buf) return NULL;
+    buf = (char *) malloc(sizeof(char) * buflen);
+    if (NULL == buf) {
+        return NULL;
+    }
 
     ret = snprintf(buf, buflen, "%f", num);
     if (ret < 0) {
diff --git a/opal/util/numtostr.h b/opal/util/numtostr.h
index c2e517f9fd0..5e754eaf0fb 100644
--- a/opal/util/numtostr.h
+++ b/opal/util/numtostr.h
@@ -30,8 +30,7 @@
  *  @param num (IN)      Input number
  *  @return              String containing number (NULL on failure)
  */
-OPAL_DECLSPEC char* opal_ltostr(long num);
-
+OPAL_DECLSPEC char *opal_ltostr(long num);
 
 /**
  * Convert a double to a char* string.  The returned buffer is allocated
@@ -40,6 +39,6 @@ OPAL_DECLSPEC char* opal_ltostr(long num);
  * @param num (IN)       Input number
  * @return               String containing number (NULL on failure)
  */
-OPAL_DECLSPEC char* opal_dtostr(double num);
+OPAL_DECLSPEC char *opal_dtostr(double num);
 
 #endif /* OPAL_NUMTOSTR_UTIL */
diff --git a/opal/util/opal_environ.c b/opal/util/opal_environ.c
index 5f01756bcfb..338312ab5c5 100644
--- a/opal/util/opal_environ.c
+++ b/opal/util/opal_environ.c
@@ -27,16 +27,16 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "opal/util/printf.h"
+#include "opal/constants.h"
 #include "opal/util/argv.h"
 #include "opal/util/opal_environ.h"
-#include "opal/constants.h"
+#include "opal/util/printf.h"
 
 #define OPAL_DEFAULT_TMPDIR "/tmp"
 
 /*
  * Merge two environ-like char arrays, ensuring that there are no
- * duplicate entires
+ * duplicate entries
  */
 char **opal_environ_merge(char **minor, char **major)
 {
@@ -92,8 +92,7 @@ char **opal_environ_merge(char **minor, char **major)
  * Portable version of setenv(), allowing editing of any environ-like
  * array
  */
-int opal_setenv(const char *name, const char *value, bool overwrite,
-                char ***env)
+int opal_setenv(const char *name, const char *value, bool overwrite, char ***env)
 {
     int i;
     char *newvalue, *compare;
@@ -113,7 +112,7 @@ int opal_setenv(const char *name, const char *value, bool overwrite,
 
     /* Check the bozo case */
 
-    if( NULL == env ) {
+    if (NULL == env) {
         return OPAL_ERR_BAD_PARAM;
     } else if (NULL == *env) {
         i = 0;
@@ -123,13 +122,13 @@ int opal_setenv(const char *name, const char *value, bool overwrite,
     }
 
     /* If this is the "environ" array, use putenv */
-    if( *env == environ ) {
+    if (*env == environ) {
         /* THIS IS POTENTIALLY A MEMORY LEAK!  But I am doing it
            so that we don't violate the law of least
            astonishment for OPAL developers (i.e., those that don't
            check the return code of opal_setenv() and notice that we
            returned an error if you passed in the real environ) */
-#if defined (HAVE_SETENV)
+#if defined(HAVE_SETENV)
         setenv(name, value, overwrite);
         /* setenv copies the value, so we can free it here */
         free(newvalue);
@@ -199,7 +198,6 @@ int opal_setenv(const char *name, const char *value, bool overwrite,
     return OPAL_SUCCESS;
 }
 
-
 /*
  * Portable version of unsetenv(), allowing editing of any
  * environ-like array
@@ -231,13 +229,15 @@ int opal_unsetenv(const char *name, char ***env)
 
     found = false;
     for (i = 0; (*env)[i] != NULL; ++i) {
-        if (0 != strncmp((*env)[i], compare, len))
+        if (0 != strncmp((*env)[i], compare, len)) {
             continue;
+        }
         if (environ != *env) {
             free((*env)[i]);
         }
-        for (; (*env)[i] != NULL; ++i)
+        for (; (*env)[i] != NULL; ++i) {
             (*env)[i] = (*env)[i + 1];
+        }
         found = true;
         break;
     }
@@ -248,21 +248,23 @@ int opal_unsetenv(const char *name, char ***env)
     return (found) ? OPAL_SUCCESS : OPAL_ERR_NOT_FOUND;
 }
 
-const char* opal_tmp_directory( void )
+const char *opal_tmp_directory(void)
 {
-    const char* str;
+    const char *str;
 
-    if( NULL == (str = getenv("TMPDIR")) )
-        if( NULL == (str = getenv("TEMP")) )
-            if( NULL == (str = getenv("TMP")) )
+    if (NULL == (str = getenv("TMPDIR"))) {
+        if (NULL == (str = getenv("TEMP"))) {
+            if (NULL == (str = getenv("TMP"))) {
                 str = OPAL_DEFAULT_TMPDIR;
+            }
+        }
+    }
     return str;
 }
 
-const char* opal_home_directory( void )
+const char *opal_home_directory(void)
 {
-    char* home = getenv("HOME");
+    char *home = getenv("HOME");
 
     return home;
 }
-
diff --git a/opal/util/opal_environ.h b/opal/util/opal_environ.h
index f7dac6adfe4..2ba1821ed96 100644
--- a/opal/util/opal_environ.h
+++ b/opal/util/opal_environ.h
@@ -30,7 +30,7 @@
 #include "opal_config.h"
 
 #ifdef HAVE_CRT_EXTERNS_H
-#include <crt_externs.h>
+#    include <crt_externs.h>
 #endif
 
 BEGIN_C_DECLS
@@ -44,7 +44,7 @@ BEGIN_C_DECLS
  * @retval New array of environ
  *
  * Merge two environ-like arrays into a single, new array,
- * ensuring that there are no duplicate entires.  If there are
+ * ensuring that there are no duplicate entries.  If there are
  * duplicates, entries in the \em major array are favored over
  * those in the \em minor array.
  *
@@ -58,7 +58,8 @@ BEGIN_C_DECLS
  * one of the two is NULL, the other list is simply copied to the
  * output.  If both are NULL, NULL is returned.
  */
-OPAL_DECLSPEC char **opal_environ_merge(char **minor, char **major) __opal_attribute_warn_unused_result__;
+OPAL_DECLSPEC char **opal_environ_merge(char **minor,
+                                        char **major) __opal_attribute_warn_unused_result__;
 
 /**
  * Portable version of setenv(3), allowing editing of any
@@ -85,7 +86,7 @@ OPAL_DECLSPEC char **opal_environ_merge(char **minor, char **major) __opal_attri
  *
  * The \em env array will be grown if necessary.
  *
- * It is permissable to invoke this function with the
+ * It is permissible to invoke this function with the
  * system-defined \em environ variable.  For example:
  *
  * \code
@@ -98,7 +99,7 @@ OPAL_DECLSPEC char **opal_environ_merge(char **minor, char **major) __opal_attri
  * environment.  This may very well lead to a memory leak, so its
  * use is strongly discouraged.
  *
- * It is also permissable to call this function with an empty \em
+ * It is also permissible to call this function with an empty \em
  * env, as long as it is pre-initialized with NULL:
  *
  * \code
@@ -106,8 +107,8 @@ OPAL_DECLSPEC char **opal_environ_merge(char **minor, char **major) __opal_attri
  *   opal_setenv("foo", "bar", true, &my_env);
  * \endcode
  */
-OPAL_DECLSPEC int opal_setenv(const char *name, const char *value,
-                              bool overwrite, char ***env) __opal_attribute_nonnull__(1);
+OPAL_DECLSPEC int opal_setenv(const char *name, const char *value, bool overwrite, char ***env)
+    __opal_attribute_nonnull__(1);
 
 /**
  * Portable version of unsetenv(3), allowing editing of any
@@ -128,13 +129,13 @@ OPAL_DECLSPEC int opal_unsetenv(const char *name, char ***env) __opal_attribute_
 /* A consistent way to retrieve the home and tmp directory on all supported
  * platforms.
  */
-OPAL_DECLSPEC const char* opal_home_directory( void );
-OPAL_DECLSPEC const char* opal_tmp_directory( void );
+OPAL_DECLSPEC const char *opal_home_directory(void);
+OPAL_DECLSPEC const char *opal_tmp_directory(void);
 
 /* Some care is needed with environ on OS X when dealing with shared
    libraries.  Handle that care here... */
 #ifdef HAVE__NSGETENVIRON
-#define environ (*_NSGetEnviron())
+#    define environ (*_NSGetEnviron())
 #else
 OPAL_DECLSPEC extern char **environ;
 #endif
diff --git a/opal/util/opal_getcwd.c b/opal/util/opal_getcwd.c
index 4ac08300e19..da484e3eec0 100644
--- a/opal/util/opal_getcwd.c
+++ b/opal/util/opal_getcwd.c
@@ -12,21 +12,20 @@
 #include <stdio.h>
 #include <stdlib.h>
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 #ifdef HAVE_SYS_STAT_H
-#include <sys/stat.h>
+#    include <sys/stat.h>
 #endif
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 #include <string.h>
 
+#include "opal/constants.h"
 #include "opal/util/basename.h"
 #include "opal/util/opal_getcwd.h"
 #include "opal/util/string_copy.h"
-#include "opal/constants.h"
-
 
 /*
  * Use $PWD instead of getcwd() a) if $PWD exists and b) is a valid
diff --git a/opal/util/opal_getcwd.h b/opal/util/opal_getcwd.h
index 1d5533190a5..64e4c8f01ed 100644
--- a/opal/util/opal_getcwd.h
+++ b/opal/util/opal_getcwd.h
@@ -44,7 +44,6 @@ BEGIN_C_DECLS
  */
 OPAL_DECLSPEC int opal_getcwd(char *buf, size_t size);
 
-
 END_C_DECLS
 
 #endif /* OPAL_GETCWD_H */
diff --git a/opal/util/opal_pty.c b/opal/util/opal_pty.c
deleted file mode 100644
index 9c70ecc6683..00000000000
--- a/opal/util/opal_pty.c
+++ /dev/null
@@ -1,261 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2018      Cisco Systems, Inc.  All rights reserved
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-/*-
- * Copyright (c) 1990, 1993
- *      The Regents of the University of California.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include "opal_config.h"
-
-#ifdef HAVE_SYS_CDEFS_H
-# include <sys/cdefs.h>
-#endif
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-#include <sys/stat.h>
-#ifdef HAVE_SYS_IOCTL_H
-#include <sys/ioctl.h>
-#endif
-#ifdef HAVE_FCNTL_H
-#include <fcntl.h>
-#endif
-#ifdef HAVE_TERMIOS_H
-# include <termios.h>
-#else
-# ifdef HAVE_TERMIO_H
-#  include <termio.h>
-# endif
-#endif
-#include <errno.h>
-#ifdef HAVE_UNISTD_H
-# include <unistd.h>
-#endif
-#include <stdio.h>
-# include <string.h>
-#ifdef HAVE_GRP_H
-#include <grp.h>
-#endif
-#ifdef HAVE_PTY_H
-#include <pty.h>
-#endif
-#ifdef HAVE_UTMP_H
-#include <utmp.h>
-#endif
-
-#ifdef HAVE_PTSNAME
-# include <stdlib.h>
-# ifdef HAVE_STROPTS_H
-#  include <stropts.h>
-# endif
-#endif
-
-#ifdef HAVE_UTIL_H
-#include <util.h>
-#endif
-
-#include "opal/util/opal_pty.h"
-
-/* The only public interface is openpty - all others are to support
-   openpty() */
-
-#if OPAL_ENABLE_PTY_SUPPORT == 0
-
-int opal_openpty(int *amaster, int *aslave, char *name,
-                 void *termp, void *winpp)
-{
-    return -1;
-}
-
-#elif defined(HAVE_OPENPTY)
-
-int opal_openpty(int *amaster, int *aslave, char *name,
-                 struct termios *termp, struct winsize *winp)
-{
-    return openpty(amaster, aslave, name, termp, winp);
-}
-
-#else
-
-/* implement openpty in terms of ptym_open and ptys_open */
-
-static int ptym_open(char *pts_name);
-static int ptys_open(int fdm, char *pts_name);
-
-int opal_openpty(int *amaster, int *aslave, char *name,
-                 struct termios *termp, struct winsize *winp)
-{
-    char line[20];
-    *amaster = ptym_open(line);
-    if (*amaster < 0) {
-        return -1;
-    }
-    *aslave = ptys_open(*amaster, line);
-    if (*aslave < 0) {
-        close(*amaster);
-        return -1;
-    }
-    if (name) {
-        // We don't know the max length of name, but we do know the
-        // max length of the source, so at least use that.
-        opal_string_copy(name, line, sizeof(line));
-    }
-#ifndef TCSAFLUSH
-#define TCSAFLUSH TCSETAF
-#endif
-    if (termp) {
-        (void) tcsetattr(*aslave, TCSAFLUSH, termp);
-    }
-#ifdef TIOCSWINSZ
-    if (winp) {
-        (void) ioctl(*aslave, TIOCSWINSZ, (char *) winp);
-    }
-#endif
-    return 0;
-}
-
-
-static int ptym_open(char *pts_name)
-{
-    int fdm;
-#ifdef HAVE_PTSNAME
-    char *ptr;
-
-#ifdef _AIX
-    strcpy(pts_name, "/dev/ptc");
-#else
-    strcpy(pts_name, "/dev/ptmx");
-#endif
-    fdm = open(pts_name, O_RDWR);
-    if (fdm < 0) {
-        return -1;
-    }
-    if (grantpt(fdm) < 0) {     /* grant access to slave */
-        close(fdm);
-        return -2;
-    }
-    if (unlockpt(fdm) < 0) {    /* clear slave's lock flag */
-        close(fdm);
-        return -3;
-    }
-    ptr = ptsname(fdm);
-    if (ptr == NULL) {          /* get slave's name */
-        close(fdm);
-        return -4;
-    }
-    strcpy(pts_name, ptr);      /* return name of slave */
-    return fdm;                 /* return fd of master */
-#else
-    char *ptr1, *ptr2;
-
-    strcpy(pts_name, "/dev/ptyXY");
-    /* array index: 012345689 (for references in following code) */
-    for (ptr1 = "pqrstuvwxyzPQRST"; *ptr1 != 0; ptr1++) {
-        pts_name[8] = *ptr1;
-        for (ptr2 = "0123456789abcdef"; *ptr2 != 0; ptr2++) {
-            pts_name[9] = *ptr2;
-            /* try to open master */
-            fdm = open(pts_name, O_RDWR);
-            if (fdm < 0) {
-                if (errno == ENOENT) {  /* different from EIO */
-                    return -1;  /* out of pty devices */
-                } else {
-                    continue;   /* try next pty device */
-                }
-            }
-            pts_name[5] = 't';  /* chage "pty" to "tty" */
-            return fdm;         /* got it, return fd of master */
-        }
-    }
-    return -1;                  /* out of pty devices */
-#endif
-}
-
-
-static int ptys_open(int fdm, char *pts_name)
-{
-    int fds;
-#ifdef HAVE_PTSNAME
-    /* following should allocate controlling terminal */
-    fds = open(pts_name, O_RDWR);
-    if (fds < 0) {
-        close(fdm);
-        return -5;
-    }
-#if defined(__SVR4) && defined(__sun)
-    if (ioctl(fds, I_PUSH, "ptem") < 0) {
-        close(fdm);
-        close(fds);
-        return -6;
-    }
-    if (ioctl(fds, I_PUSH, "ldterm") < 0) {
-        close(fdm);
-        close(fds);
-        return -7;
-    }
-#endif
-
-    return fds;
-#else
-    int gid;
-    struct group *grptr;
-
-    grptr = getgrnam("tty");
-    if (grptr != NULL) {
-        gid = grptr->gr_gid;
-    } else {
-        gid = -1;               /* group tty is not in the group file */
-    }
-    /* following two functions don't work unless we're root */
-    chown(pts_name, getuid(), gid);
-    chmod(pts_name, S_IRUSR | S_IWUSR | S_IWGRP);
-    fds = open(pts_name, O_RDWR);
-    if (fds < 0) {
-        close(fdm);
-        return -1;
-    }
-    return fds;
-#endif
-}
-
-#endif /* #ifdef HAVE_OPENPTY */
diff --git a/opal/util/opal_pty.h b/opal/util/opal_pty.h
deleted file mode 100644
index 4bcb7a97f9d..00000000000
--- a/opal/util/opal_pty.h
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2006 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef OPAL_UTIL_PTY_H
-#define OPAL_UTIL_PTY_H
-
-#include "opal_config.h"
-
-#ifdef HAVE_UTIL_H
-#include <util.h>
-#endif
-#ifdef HAVE_LIBUTIL_H
-#include <libutil.h>
-#endif
-#ifdef HAVE_TERMIOS_H
-# include <termios.h>
-#else
-# ifdef HAVE_TERMIO_H
-#  include <termio.h>
-# endif
-#endif
-
-BEGIN_C_DECLS
-
-#if OPAL_ENABLE_PTY_SUPPORT
-
-OPAL_DECLSPEC int opal_openpty(int *amaster, int *aslave, char *name,
-                               struct termios *termp, struct winsize *winp);
-
-#else
-
-OPAL_DECLSPEC int opal_openpty(int *amaster, int *aslave, char *name,
-                               void *termp, void *winpp);
-
-#endif
-
-END_C_DECLS
-
-#endif /* OPAL_UTIL_PTY_H */
diff --git a/opal/util/os_dirpath.c b/opal/util/os_dirpath.c
index 2328179d7f6..bac062e896c 100644
--- a/opal/util/os_dirpath.c
+++ b/opal/util/os_dirpath.c
@@ -19,31 +19,30 @@
  * $HEADER$
  */
 
-
 #include "opal_config.h"
 
 #include <errno.h>
 #include <string.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif  /* HAVE_UNISTD_H */
+#    include <unistd.h>
+#endif /* HAVE_UNISTD_H */
 #include <stdlib.h>
 #if HAVE_SYS_STAT_H
-#include <sys/stat.h>
+#    include <sys/stat.h>
 #endif /* HAVE_SYS_STAT_H */
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif  /* HAVE_SYS_TYPES_H */
+#    include <sys/types.h>
+#endif /* HAVE_SYS_TYPES_H */
 #ifdef HAVE_DIRENT_H
-#include <dirent.h>
-#endif  /* HAVE_DIRENT_H */
+#    include <dirent.h>
+#endif /* HAVE_DIRENT_H */
 
-#include "opal/util/output.h"
-#include "opal/util/os_dirpath.h"
-#include "opal/util/show_help.h"
+#include "opal/constants.h"
 #include "opal/util/argv.h"
+#include "opal/util/os_dirpath.h"
 #include "opal/util/os_path.h"
-#include "opal/constants.h"
+#include "opal/util/output.h"
+#include "opal/util/show_help.h"
 
 static const char path_sep[] = OPAL_PATH_SEP;
 
@@ -55,27 +54,26 @@ int opal_os_dirpath_create(const char *path, const mode_t mode)
     int ret;
 
     if (NULL == path) { /* protect ourselves from errors */
-        return(OPAL_ERR_BAD_PARAM);
+        return (OPAL_ERR_BAD_PARAM);
     }
 
-    if (0 == (ret = stat(path, &buf))) { /* already exists */
+    if (0 == (ret = stat(path, &buf))) {    /* already exists */
         if (mode == (mode & buf.st_mode)) { /* has correct mode */
-            return(OPAL_SUCCESS);
+            return (OPAL_SUCCESS);
         }
         if (0 == (ret = chmod(path, (buf.st_mode | mode)))) { /* successfully change mode */
-            return(OPAL_SUCCESS);
+            return (OPAL_SUCCESS);
         }
-        opal_show_help("help-opal-util.txt", "dir-mode", true,
-                    path, mode, strerror(errno));
-        return(OPAL_ERR_PERM); /* can't set correct mode */
+        opal_show_help("help-opal-util.txt", "dir-mode", true, path, mode, strerror(errno));
+        return (OPAL_ERR_PERM); /* can't set correct mode */
     }
 
     /* quick -- try to make directory */
     if (0 == mkdir(path, mode)) {
-        return(OPAL_SUCCESS);
+        return (OPAL_SUCCESS);
     }
 
-    /* didnt work, so now have to build our way down the tree */
+    /* didn't work, so now have to build our way down the tree */
     /* Split the requested path up into its individual parts */
 
     parts = opal_argv_split(path, path_sep[0]);
@@ -83,7 +81,7 @@ int opal_os_dirpath_create(const char *path, const mode_t mode)
     /* Ensure to allocate enough space for tmp: the strlen of the
        incoming path + 1 (for \0) */
 
-    tmp = (char*)malloc(strlen(path) + 1);
+    tmp = (char *) malloc(strlen(path) + 1);
     tmp[0] = '\0';
 
     /* Iterate through all the subdirectory names in the path,
@@ -103,7 +101,7 @@ int opal_os_dirpath_create(const char *path, const mode_t mode)
         }
 
         /* If it's not the first part, ensure that there's a
-           preceeding path_sep and then append this part */
+           preceding path_sep and then append this part */
 
         else {
             if (path_sep[0] != tmp[strlen(tmp) - 1]) {
@@ -114,19 +112,18 @@ int opal_os_dirpath_create(const char *path, const mode_t mode)
 
         /* Now that we have the name, try to create it */
         mkdir(tmp, mode);
-        ret = errno;  // save the errno for an error msg, if needed
+        ret = errno; // save the errno for an error msg, if needed
         if (0 != stat(tmp, &buf)) {
-            opal_show_help("help-opal-util.txt", "mkdir-failed", true,
-                        tmp, strerror(ret));
+            opal_show_help("help-opal-util.txt", "mkdir-failed", true, tmp, strerror(ret));
             opal_argv_free(parts);
             free(tmp);
             return OPAL_ERROR;
-        } else if (i == (len-1) && (mode != (mode & buf.st_mode)) && (0 > chmod(tmp, (buf.st_mode | mode)))) {
-            opal_show_help("help-opal-util.txt", "dir-mode", true,
-                           tmp, mode, strerror(errno));
+        } else if (i == (len - 1) && (mode != (mode & buf.st_mode))
+                   && (0 > chmod(tmp, (buf.st_mode | mode)))) {
+            opal_show_help("help-opal-util.txt", "dir-mode", true, tmp, mode, strerror(errno));
             opal_argv_free(parts);
             free(tmp);
-            return(OPAL_ERR_PERM); /* can't set correct mode */
+            return (OPAL_ERR_PERM); /* can't set correct mode */
         }
     }
 
@@ -145,8 +142,7 @@ int opal_os_dirpath_create(const char *path, const mode_t mode)
  * removed.  If the callback returns non-zero, then no removal is
  * done.
  */
-int opal_os_dirpath_destroy(const char *path,
-                            bool recursive,
+int opal_os_dirpath_destroy(const char *path, bool recursive,
                             opal_os_dirpath_destroy_callback_fn_t cbfunc)
 {
     int rc, exit_status = OPAL_SUCCESS;
@@ -156,7 +152,7 @@ int opal_os_dirpath_destroy(const char *path,
     char *filenm;
     struct stat buf;
 
-    if (NULL == path) {  /* protect against error */
+    if (NULL == path) { /* protect against error */
         return OPAL_ERROR;
     }
 
@@ -178,8 +174,7 @@ int opal_os_dirpath_destroy(const char *path,
         /* skip:
          *  - . and ..
          */
-        if ((0 == strcmp(ep->d_name, ".")) ||
-            (0 == strcmp(ep->d_name, ".."))) {
+        if ((0 == strcmp(ep->d_name, ".")) || (0 == strcmp(ep->d_name, ".."))) {
             continue;
         }
 
@@ -207,7 +202,7 @@ int opal_os_dirpath_destroy(const char *path,
         }
 
         /*
-         * If not recursively decending, then if we find a directory then fail
+         * If not recursively descending, then if we find a directory then fail
          * since we were not told to remove it.
          */
         if (is_dir && !recursive) {
@@ -251,31 +246,31 @@ int opal_os_dirpath_destroy(const char *path,
     /* Done with this directory */
     closedir(dp);
 
- cleanup:
+cleanup:
 
     /*
      * If the directory is empty, them remove it
      */
-    if(opal_os_dirpath_is_empty(path)) {
+    if (opal_os_dirpath_is_empty(path)) {
         rmdir(path);
     }
 
     return exit_status;
 }
 
-bool opal_os_dirpath_is_empty(const char *path ) {
+bool opal_os_dirpath_is_empty(const char *path)
+{
     DIR *dp;
     struct dirent *ep;
 
-    if (NULL != path) {  /* protect against error */
+    if (NULL != path) { /* protect against error */
         dp = opendir(path);
         if (NULL != dp) {
             while ((ep = readdir(dp))) {
-                        if ((0 != strcmp(ep->d_name, ".")) &&
-                            (0 != strcmp(ep->d_name, ".."))) {
-                            closedir(dp);
-                            return false;
-                        }
+                if ((0 != strcmp(ep->d_name, ".")) && (0 != strcmp(ep->d_name, ".."))) {
+                    closedir(dp);
+                    return false;
+                }
             }
             closedir(dp);
             return true;
@@ -286,9 +281,10 @@ bool opal_os_dirpath_is_empty(const char *path ) {
     return true;
 }
 
-int opal_os_dirpath_access(const char *path, const mode_t in_mode ) {
+int opal_os_dirpath_access(const char *path, const mode_t in_mode)
+{
     struct stat buf;
-    mode_t loc_mode = S_IRWXU;  /* looking for full rights */
+    mode_t loc_mode = S_IRWXU; /* looking for full rights */
 
     /*
      * If there was no mode specified, use the default mode
@@ -297,15 +293,15 @@ int opal_os_dirpath_access(const char *path, const mode_t in_mode ) {
         loc_mode = in_mode;
     }
 
-    if (0 == stat(path, &buf)) { /* exists - check access */
+    if (0 == stat(path, &buf)) {                    /* exists - check access */
         if ((buf.st_mode & loc_mode) == loc_mode) { /* okay, I can work here */
-            return(OPAL_SUCCESS);
+            return (OPAL_SUCCESS);
         } else {
             /* Don't have access rights to the existing path */
-            return(OPAL_ERROR);
+            return (OPAL_ERROR);
         }
     } else {
         /* We could not find the path */
-        return( OPAL_ERR_NOT_FOUND );
+        return (OPAL_ERR_NOT_FOUND);
     }
 }
diff --git a/opal/util/os_dirpath.h b/opal/util/os_dirpath.h
index 5564a77370a..c9a769e4a41 100644
--- a/opal/util/os_dirpath.h
+++ b/opal/util/os_dirpath.h
@@ -42,7 +42,7 @@
 
 #include "opal_config.h"
 #ifdef HAVE_SYS_STAT_H
-#include <sys/stat.h>
+#    include <sys/stat.h>
 #endif
 
 BEGIN_C_DECLS
@@ -80,14 +80,16 @@ OPAL_DECLSPEC bool opal_os_dirpath_is_empty(const char *path);
  * @retval OPAL_ERR_NOT_FOUND If directory does not exist
  * @retval OPAL_ERROR   If directory exists, and permissions do not match
  */
-OPAL_DECLSPEC int opal_os_dirpath_access(const char *path, const mode_t mode );
+OPAL_DECLSPEC int opal_os_dirpath_access(const char *path, const mode_t mode);
 
 /**
  * Callback for opal_os_dirpath_destroy(). Call for every file/directory before
  * taking action to remove/unlink it.
  *
- * @param root A pointer to a string that contains the base path name (e.g., /tmp/foo from /tmp/foo/bar)
- * @param path A pointer to a string that contains the file or directory (e.g., bar from /tmp/foo/bar)
+ * @param root A pointer to a string that contains the base path name (e.g., /tmp/foo from
+ * /tmp/foo/bar)
+ * @param path A pointer to a string that contains the file or directory (e.g., bar from
+ * /tmp/foo/bar)
  *
  * @retval true  Allow the program to remove the file/directory
  * @retval false Do not allow the program to remove the file/directory
@@ -106,11 +108,10 @@ typedef bool (*opal_os_dirpath_destroy_callback_fn_t)(const char *root, const ch
  * @retval OPAL_SUCCESS If the directory was successfully removed or removed to the
  *                      specification of the user (i.e., obeyed the callback function).
  * @retval OPAL_ERR_NOT_FOUND If directory does not exist.
- * @retval OPAL_ERROR If the directory cannnot be removed, accessed properly, or contains
+ * @retval OPAL_ERROR If the directory cannot be removed, accessed properly, or contains
  *                    directories that could not be removed..
  */
-OPAL_DECLSPEC int opal_os_dirpath_destroy(const char *path,
-                                          bool recursive,
+OPAL_DECLSPEC int opal_os_dirpath_destroy(const char *path, bool recursive,
                                           opal_os_dirpath_destroy_callback_fn_t cbfunc);
 
 END_C_DECLS
diff --git a/opal/util/os_path.c b/opal/util/os_path.c
index 8eb54f43f74..6b67737dacb 100644
--- a/opal/util/os_path.c
+++ b/opal/util/os_path.c
@@ -21,13 +21,13 @@
 
 #include <string.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif  /* HAVE_UNISTD_H */
+#    include <unistd.h>
+#endif /* HAVE_UNISTD_H */
 #ifdef HAVE_SYS_PARAM_H
-#include <sys/param.h>
-#endif  /* HAVE_SYS_PARAM_H */
-#include <stdlib.h>
+#    include <sys/param.h>
+#endif /* HAVE_SYS_PARAM_H */
 #include <stdarg.h>
+#include <stdlib.h>
 
 #include "opal/util/os_path.h"
 
@@ -46,37 +46,39 @@ char *opal_os_path(int relative, ...)
 
     num_elements = 0;
     total_length = 0;
-    while (NULL != (element = va_arg(ap, char*))) {
+    while (NULL != (element = va_arg(ap, char *))) {
         num_elements++;
         total_length = total_length + strlen(element);
-        if( path_sep[0] != element[0] ) total_length++;
+        if (path_sep[0] != element[0]) {
+            total_length++;
+        }
     }
     va_end(ap);
 
     if (0 == num_elements) { /* must be looking for a simple answer */
         size_t len = 3;
-        path = (char *)calloc(len, sizeof(char));
-    	if (relative) {
+        path = (char *) calloc(len, sizeof(char));
+        if (relative) {
             path[0] = '.';
         }
         strncat(path, path_sep, len - 1);
-    	return(path);
+        return (path);
     }
 
     /* setup path with enough room for the string terminator, the elements, and
        the separator between each of the elements */
     total_length = total_length + num_elements * strlen(path_sep) + 1;
-    if(relative) {
+    if (relative) {
         total_length++;
     }
 
-    if (total_length > OPAL_PATH_MAX) {  /* path length is too long - reject it */
-    	return(NULL);
+    if (total_length > OPAL_PATH_MAX) { /* path length is too long - reject it */
+        return (NULL);
     }
 
-    path = (char *)calloc(total_length, sizeof(char));
+    path = (char *) calloc(total_length, sizeof(char));
     if (NULL == path) {
-        return(NULL);
+        return (NULL);
     }
 
     if (relative) {
@@ -84,17 +86,17 @@ char *opal_os_path(int relative, ...)
     }
 
     va_start(ap, relative);
-    if( NULL != (element = va_arg(ap, char*)) ) {
-    	if (path_sep[0] != element[0]) {
-            strncat(path, path_sep, total_length);
+    if (NULL != (element = va_arg(ap, char *))) {
+        if (path_sep[0] != element[0]) {
+            strncat(path, path_sep, total_length - strlen(path) - 1);
         }
-        strcat(path, element);
+        strncat(path, element, total_length - strlen(path) - 1);
     }
-    while (NULL != (element=va_arg(ap, char*))) {
-    	if (path_sep[0] != element[0]) {
-            strncat(path, path_sep, total_length);
+    while (NULL != (element = va_arg(ap, char *))) {
+        if (path_sep[0] != element[0]) {
+            strncat(path, path_sep, total_length - strlen(path) - 1);
         }
-        strncat(path, element, total_length);
+        strncat(path, element, total_length - strlen(path) - 1);
     }
 
     va_end(ap);
diff --git a/opal/util/os_path.h b/opal/util/os_path.h
index db3d9dcbe72..6cb5d782072 100644
--- a/opal/util/os_path.h
+++ b/opal/util/os_path.h
@@ -44,8 +44,8 @@
 
 #include "opal_config.h"
 
-#include <stdio.h>
 #include <stdarg.h>
+#include <stdio.h>
 
 BEGIN_C_DECLS
 
@@ -69,13 +69,14 @@ BEGIN_C_DECLS
  * va_start() has undefined behavior (according to clang warnings on
  * MacOS High Sierra).
  */
-OPAL_DECLSPEC char *opal_os_path(int relative, ...) __opal_attribute_malloc__ __opal_attribute_sentinel__ __opal_attribute_warn_unused_result__;
+OPAL_DECLSPEC char *opal_os_path(int relative, ...)
+    __opal_attribute_malloc__ __opal_attribute_sentinel__ __opal_attribute_warn_unused_result__;
 
 /**
  * Convert the path to be OS friendly. On UNIX this function will
  * be empty.
  */
-#define opal_make_filename_os_friendly(PATH)   (PATH)
+#define opal_make_filename_os_friendly(PATH) (PATH)
 
 END_C_DECLS
 
diff --git a/opal/util/output.c b/opal/util/output.c
index b09c21c87cd..a1d1259a71c 100644
--- a/opal/util/output.c
+++ b/opal/util/output.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2006 High Performance Computing Center Stuttgart,
@@ -22,6 +22,7 @@
  *                         reserved.
  * Copyright (c) 2019      Triad National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2024      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -31,29 +32,28 @@
 
 #include "opal_config.h"
 
-#include <stdio.h>
 #include <stdarg.h>
+#include <stdio.h>
 #include <stdlib.h>
 #ifdef HAVE_SYSLOG_H
-#include <syslog.h>
+#    include <syslog.h>
 #endif
-#include <string.h>
 #include <fcntl.h>
+#include <string.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 #ifdef HAVE_SYS_PARAM_H
-#include <sys/param.h>
+#    include <sys/param.h>
 #endif
 
+#include "opal/constants.h"
+#include "opal/mca/threads/mutex.h"
 #include "opal/runtime/opal.h"
 #include "opal/util/opal_environ.h"
 #include "opal/util/output.h"
-#include "opal/util/string_copy.h"
 #include "opal/util/printf.h"
-#include "opal/mca/threads/mutex.h"
-#include "opal/constants.h"
-#include "opal/mca/pmix/pmix-internal.h"
+#include "opal/util/string_copy.h"
 
 /*
  * Private data
@@ -63,6 +63,7 @@ static opal_output_stream_t verbose;
 static char *output_dir = NULL;
 static char *output_prefix = NULL;
 
+static opal_output_pmix_cleanup_fn_t output_pmix_cleanup_fn = NULL;
 
 /*
  * Internal data structures and helpers for the generalized output
@@ -91,6 +92,7 @@ typedef struct {
     char *ldi_file_suffix;
     int ldi_fd;
     int ldi_file_num_lines_lost;
+    char *ldi_filename;
 } output_desc_t;
 
 /*
@@ -98,20 +100,19 @@ typedef struct {
  */
 static void construct(opal_object_t *stream);
 static void destruct(opal_object_t *stream);
-static int do_open(int output_id, opal_output_stream_t * lds);
+static int do_open(int output_id, opal_output_stream_t *lds);
 static int open_file(int i);
 static void free_descriptor(int output_id);
-static int make_string(char **no_newline_string, output_desc_t *ldi,
-                       const char *format, va_list arglist);
+static int make_string(char **no_newline_string, output_desc_t *ldi, const char *format,
+                       va_list arglist);
 static int output(int output_id, const char *format, va_list arglist);
-static void opal_output_finalize (void);
-
+static void opal_output_finalize(void);
 
 #define OPAL_OUTPUT_MAX_STREAMS 64
 #if defined(HAVE_SYSLOG)
-#define USE_SYSLOG 1
+#    define USE_SYSLOG 1
 #else
-#define USE_SYSLOG 0
+#    define USE_SYSLOG 0
 #endif
 
 /* global state */
@@ -172,7 +173,7 @@ bool opal_output_init(void)
     } else {
         opal_output_redirected_syslog_pri = LOG_ERR;
     }
-#endif  /* HAVE_SYSLOG_H */
+#endif /* HAVE_SYSLOG_H */
     str = getenv("OPAL_OUTPUT_SYSLOG_IDENT");
     if (NULL != str) {
         redirect_syslog_ident = strdup(str);
@@ -191,8 +192,7 @@ bool opal_output_init(void)
         str = getenv("OPAL_OUTPUT_INTERNAL_TO_STDOUT");
         if (NULL != str && str[0] == '1') {
             verbose.lds_want_stdout = true;
-        }
-        else {
+        } else {
             verbose.lds_want_stderr = true;
         }
     }
@@ -208,6 +208,7 @@ bool opal_output_init(void)
         info[i].ldi_file_suffix = NULL;
         info[i].ldi_file_want_append = false;
         info[i].ldi_fd = -1;
+        info[i].ldi_filename = NULL;
         info[i].ldi_file_num_lines_lost = 0;
     }
 
@@ -225,30 +226,27 @@ bool opal_output_init(void)
     verbose_stream = opal_output_open(&verbose);
 
     /* make sure opal output is cleaned up on finalize */
-    opal_finalize_register_cleanup (opal_output_finalize);
+    opal_finalize_register_cleanup(opal_output_finalize);
 
     return true;
 }
 
-
 /*
  * Open a stream
  */
-int opal_output_open(opal_output_stream_t * lds)
+int opal_output_open(opal_output_stream_t *lds)
 {
     return do_open(-1, lds);
 }
 
-
 /*
  * Reset the parameters on a stream
  */
-int opal_output_reopen(int output_id, opal_output_stream_t * lds)
+int opal_output_reopen(int output_id, opal_output_stream_t *lds)
 {
     return do_open(output_id, lds);
 }
 
-
 /*
  * Enable and disable output streams
  */
@@ -270,7 +268,6 @@ bool opal_output_switch(int output_id, bool enable)
     return ret;
 }
 
-
 /*
  * Reopen all the streams; used during checkpoint/restart.
  */
@@ -287,7 +284,7 @@ void opal_output_reopen_all(void)
     }
 
     hostname = opal_gethostname();
-    if( NULL != verbose.lds_prefix ) {
+    if (NULL != verbose.lds_prefix) {
         free(verbose.lds_prefix);
         verbose.lds_prefix = NULL;
     }
@@ -310,13 +307,13 @@ void opal_output_reopen_all(void)
          */
         info[i].ldi_used = false;
 
-#if USE_SYSLOG
+#    if USE_SYSLOG
         lds.lds_want_syslog = info[i].ldi_syslog;
         lds.lds_syslog_priority = info[i].ldi_syslog_priority;
         lds.lds_syslog_ident = info[i].ldi_syslog_ident;
-#else
+#    else
         lds.lds_want_syslog = false;
-#endif
+#    endif
         lds.lds_prefix = info[i].ldi_prefix;
         lds.lds_suffix = info[i].ldi_suffix;
         lds.lds_want_stdout = info[i].ldi_stdout;
@@ -335,7 +332,6 @@ void opal_output_reopen_all(void)
 #endif
 }
 
-
 /*
  * Close a stream
  */
@@ -353,8 +349,8 @@ void opal_output_close(int output_id)
      * free the resources associated with the descriptor */
 
     OPAL_THREAD_LOCK(&mutex);
-    if (output_id < OPAL_OUTPUT_MAX_STREAMS &&
-        info[output_id].ldi_used && info[output_id].ldi_enabled) {
+    if (output_id < OPAL_OUTPUT_MAX_STREAMS && info[output_id].ldi_used
+        && info[output_id].ldi_enabled) {
         free_descriptor(output_id);
 
         /* If no one has the syslog open, we should close it */
@@ -375,7 +371,6 @@ void opal_output_close(int output_id)
     OPAL_THREAD_UNLOCK(&mutex);
 }
 
-
 /*
  * Main function to send output to a stream
  */
@@ -389,30 +384,26 @@ void opal_output(int output_id, const char *format, ...)
     }
 }
 
-
 /*
  * Check whether the verbose level is high enough for the given stream
  */
 bool opal_output_check_verbosity(int level, int output_id)
 {
-    return (output_id >= 0 && output_id < OPAL_OUTPUT_MAX_STREAMS &&
-        info[output_id].ldi_verbose_level >= level);
+    return (output_id >= 0 && output_id < OPAL_OUTPUT_MAX_STREAMS
+            && info[output_id].ldi_verbose_level >= level);
 }
 
-
 /*
  * Send a message to a stream if the verbose level is high enough
  */
-void opal_output_vverbose(int level, int output_id, const char *format,
-                          va_list arglist)
+void opal_output_vverbose(int level, int output_id, const char *format, va_list arglist)
 {
-    if (output_id >= 0 && output_id < OPAL_OUTPUT_MAX_STREAMS &&
-        info[output_id].ldi_verbose_level >= level) {
+    if (output_id >= 0 && output_id < OPAL_OUTPUT_MAX_STREAMS
+        && info[output_id].ldi_verbose_level >= level) {
         output(output_id, format, arglist);
     }
 }
 
-
 /*
  * Send a message to a string if the verbose level is high enough
  */
@@ -421,8 +412,8 @@ char *opal_output_string(int level, int output_id, const char *format, ...)
     int rc;
     char *ret = NULL;
 
-    if (output_id >= 0 && output_id < OPAL_OUTPUT_MAX_STREAMS &&
-        info[output_id].ldi_verbose_level >= level) {
+    if (output_id >= 0 && output_id < OPAL_OUTPUT_MAX_STREAMS
+        && info[output_id].ldi_verbose_level >= level) {
         va_list arglist;
         va_start(arglist, format);
         rc = make_string(&ret, &info[output_id], format, arglist);
@@ -435,18 +426,16 @@ char *opal_output_string(int level, int output_id, const char *format, ...)
     return ret;
 }
 
-
 /*
  * Send a message to a string if the verbose level is high enough
  */
-char *opal_output_vstring(int level, int output_id, const char *format,
-                          va_list arglist)
+char *opal_output_vstring(int level, int output_id, const char *format, va_list arglist)
 {
     int rc;
     char *ret = NULL;
 
-    if (output_id >= 0 && output_id < OPAL_OUTPUT_MAX_STREAMS &&
-        info[output_id].ldi_verbose_level >= level) {
+    if (output_id >= 0 && output_id < OPAL_OUTPUT_MAX_STREAMS
+        && info[output_id].ldi_verbose_level >= level) {
         rc = make_string(&ret, &info[output_id], format, arglist);
         if (OPAL_SUCCESS != rc) {
             ret = NULL;
@@ -456,7 +445,6 @@ char *opal_output_vstring(int level, int output_id, const char *format,
     return ret;
 }
 
-
 /*
  * Set the verbosity level of a stream
  */
@@ -467,13 +455,10 @@ void opal_output_set_verbosity(int output_id, int level)
     }
 }
 
-
 /*
  * Control where output flies will go
  */
-void opal_output_set_output_file_info(const char *dir,
-                                      const char *prefix,
-                                      char **olddir,
+void opal_output_set_output_file_info(const char *dir, const char *prefix, char **olddir,
                                       char **oldprefix)
 {
     if (NULL != olddir) {
@@ -493,6 +478,21 @@ void opal_output_set_output_file_info(const char *dir,
     }
 }
 
+void opal_output_register_pmix_cleanup_fn(opal_output_pmix_cleanup_fn_t cleanup_fn)
+{
+    int n;
+
+    OPAL_THREAD_LOCK(&mutex);
+    /* Register any filenames setup prior to this call */
+    for (n = 0; n < OPAL_OUTPUT_MAX_STREAMS; n++) {
+        if (NULL != info[n].ldi_filename) {
+            output_pmix_cleanup_fn(info[n].ldi_filename, false, true, false);
+        }
+    }
+    /* Save the function pointer for later calls */
+    output_pmix_cleanup_fn = cleanup_fn;
+    OPAL_THREAD_UNLOCK(&mutex);
+}
 
 /*
  * Shut down the output stream system
@@ -508,13 +508,13 @@ static void opal_output_finalize(void)
 
         verbose_stream = -1;
 
-        free (output_prefix);
+        free(output_prefix);
         output_prefix = NULL;
 
-        free (output_dir);
+        free(output_dir);
         output_dir = NULL;
 
-        if(NULL != temp_str) {
+        if (NULL != temp_str) {
             free(temp_str);
             temp_str = NULL;
             temp_str_len = 0;
@@ -533,7 +533,7 @@ static void opal_output_finalize(void)
  */
 static void construct(opal_object_t *obj)
 {
-    opal_output_stream_t *stream = (opal_output_stream_t*) obj;
+    opal_output_stream_t *stream = (opal_output_stream_t *) obj;
 
     stream->lds_verbose_level = 0;
     stream->lds_syslog_priority = 0;
@@ -550,9 +550,9 @@ static void construct(opal_object_t *obj)
 }
 static void destruct(opal_object_t *obj)
 {
-    opal_output_stream_t *stream = (opal_output_stream_t*) obj;
+    opal_output_stream_t *stream = (opal_output_stream_t *) obj;
 
-    if( NULL != stream->lds_file_suffix ) {
+    if (NULL != stream->lds_file_suffix) {
         free(stream->lds_file_suffix);
         stream->lds_file_suffix = NULL;
     }
@@ -563,7 +563,7 @@ static void destruct(opal_object_t *obj)
  * back-end function so that we can do the thread locking properly
  * (especially upon reopen).
  */
-static int do_open(int output_id, opal_output_stream_t * lds)
+static int do_open(int output_id, opal_output_stream_t *lds)
 {
     int i;
     bool redirect_to_file = false;
@@ -575,6 +575,13 @@ static int do_open(int output_id, opal_output_stream_t * lds)
         opal_output_init();
     }
 
+    /* Bozo check */
+
+    if (output_id >= OPAL_OUTPUT_MAX_STREAMS ||
+        output_id < -1) {
+        return OPAL_ERR_BAD_PARAM;
+    }
+
     str = getenv("OPAL_OUTPUT_REDIRECT");
     if (NULL != str && 0 == strcasecmp(str, "file")) {
         redirect_to_file = true;
@@ -618,12 +625,11 @@ static int do_open(int output_id, opal_output_stream_t * lds)
     if (-1 == output_id) {
         OPAL_THREAD_UNLOCK(&mutex);
     }
-    info[i].ldi_enabled = lds->lds_is_debugging ?
-        (bool) OPAL_ENABLE_DEBUG : true;
+    info[i].ldi_enabled = lds->lds_is_debugging ? (bool) OPAL_ENABLE_DEBUG : true;
     info[i].ldi_verbose_level = lds->lds_verbose_level;
 
 #if USE_SYSLOG
-#if defined(HAVE_SYSLOG) && defined(HAVE_SYSLOG_H)
+#    if defined(HAVE_SYSLOG) && defined(HAVE_SYSLOG_H)
     if (opal_output_redirected_to_syslog) {
         info[i].ldi_syslog = true;
         info[i].ldi_syslog_priority = opal_output_redirected_syslog_pri;
@@ -636,11 +642,11 @@ static int do_open(int output_id, opal_output_stream_t * lds)
         }
         syslog_opened = true;
     } else {
-#endif
+#    endif
         info[i].ldi_syslog = lds->lds_want_syslog;
         if (lds->lds_want_syslog) {
 
-#if defined(HAVE_SYSLOG) && defined(HAVE_SYSLOG_H)
+#    if defined(HAVE_SYSLOG) && defined(HAVE_SYSLOG_H)
             if (NULL != lds->lds_syslog_ident) {
                 info[i].ldi_syslog_ident = strdup(lds->lds_syslog_ident);
                 openlog(lds->lds_syslog_ident, LOG_PID, LOG_USER);
@@ -648,14 +654,14 @@ static int do_open(int output_id, opal_output_stream_t * lds)
                 info[i].ldi_syslog_ident = NULL;
                 openlog("opal", LOG_PID, LOG_USER);
             }
-#endif
+#    endif
             syslog_opened = true;
             info[i].ldi_syslog_priority = lds->lds_syslog_priority;
         }
 
-#if defined(HAVE_SYSLOG) && defined(HAVE_SYSLOG_H)
+#    if defined(HAVE_SYSLOG) && defined(HAVE_SYSLOG_H)
     }
-#endif
+#    endif
 
 #else
     info[i].ldi_syslog = false;
@@ -663,7 +669,7 @@ static int do_open(int output_id, opal_output_stream_t * lds)
 
     if (NULL != lds->lds_prefix) {
         info[i].ldi_prefix = strdup(lds->lds_prefix);
-        info[i].ldi_prefix_len = (int)strlen(lds->lds_prefix);
+        info[i].ldi_prefix_len = (int) strlen(lds->lds_prefix);
     } else {
         info[i].ldi_prefix = NULL;
         info[i].ldi_prefix_len = 0;
@@ -671,7 +677,7 @@ static int do_open(int output_id, opal_output_stream_t * lds)
 
     if (NULL != lds->lds_suffix) {
         info[i].ldi_suffix = strdup(lds->lds_suffix);
-        info[i].ldi_suffix_len = (int)strlen(lds->lds_suffix);
+        info[i].ldi_suffix_len = (int) strlen(lds->lds_suffix);
     } else {
         info[i].ldi_suffix = NULL;
         info[i].ldi_suffix_len = 0;
@@ -703,8 +709,8 @@ static int do_open(int output_id, opal_output_stream_t * lds)
         if (NULL != sfx) {
             info[i].ldi_file_suffix = strdup(sfx);
         } else {
-            info[i].ldi_file_suffix = (NULL == lds->lds_file_suffix) ? NULL :
-                strdup(lds->lds_file_suffix);
+            info[i].ldi_file_suffix = (NULL == lds->lds_file_suffix) ? NULL
+                                                                     : strdup(lds->lds_file_suffix);
         }
         info[i].ldi_file_want_append = lds->lds_want_file_append;
         info[i].ldi_file_num_lines_lost = 0;
@@ -714,12 +720,12 @@ static int do_open(int output_id, opal_output_stream_t * lds)
      * This is the verbose stream, so update the internal 'verbose_stream'
      * to match the parameters set in the info[i]
      */
-    if( verbose_stream == i ) {
-        verbose.lds_want_syslog     = info[i].ldi_syslog;
+    if (verbose_stream == i) {
+        verbose.lds_want_syslog = info[i].ldi_syslog;
         verbose.lds_syslog_priority = info[i].ldi_syslog_priority;
-        verbose.lds_syslog_ident    = info[i].ldi_syslog_ident;
-        verbose.lds_want_stdout     = info[i].ldi_stdout;
-        verbose.lds_want_stderr     = info[i].ldi_stderr;
+        verbose.lds_syslog_ident = info[i].ldi_syslog_ident;
+        verbose.lds_want_stdout = info[i].ldi_stdout;
+        verbose.lds_want_stderr = info[i].ldi_stderr;
     }
 
     /* Don't open a file in the session directory now -- do that lazily
@@ -728,18 +734,16 @@ static int do_open(int output_id, opal_output_stream_t * lds)
     return i;
 }
 
-
 static int open_file(int i)
 {
     int flags;
-    char *filename;
     int n;
 
     /* first check to see if this file is already open
      * on someone else's stream - if so, we don't want
      * to open it twice
      */
-    for (n=0; n < OPAL_OUTPUT_MAX_STREAMS; n++) {
+    for (n = 0; n < OPAL_OUTPUT_MAX_STREAMS; n++) {
         if (i == n) {
             continue;
         }
@@ -749,18 +753,15 @@ static int open_file(int i)
         if (!info[n].ldi_file) {
             continue;
         }
-        if (NULL != info[i].ldi_file_suffix &&
-            NULL != info[n].ldi_file_suffix) {
+        if (NULL != info[i].ldi_file_suffix && NULL != info[n].ldi_file_suffix) {
             if (0 != strcmp(info[i].ldi_file_suffix, info[n].ldi_file_suffix)) {
                 break;
             }
         }
-        if (NULL == info[i].ldi_file_suffix &&
-            NULL != info[n].ldi_file_suffix) {
+        if (NULL == info[i].ldi_file_suffix && NULL != info[n].ldi_file_suffix) {
             break;
         }
-        if (NULL != info[i].ldi_file_suffix &&
-            NULL == info[n].ldi_file_suffix) {
+        if (NULL != info[i].ldi_file_suffix && NULL == info[n].ldi_file_suffix) {
             break;
         }
         if (info[n].ldi_fd < 0) {
@@ -773,20 +774,23 @@ static int open_file(int i)
     /* Setup the filename and open flags */
 
     if (NULL != output_dir) {
-        filename = (char *) malloc(OPAL_PATH_MAX);
-        if (NULL == filename) {
+        if (NULL != info[i].ldi_filename) {
+            free(info[i].ldi_filename);
+        }
+        info[i].ldi_filename = (char *) malloc(OPAL_PATH_MAX);
+        if (NULL == info[i].ldi_filename) {
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
-        opal_string_copy(filename, output_dir, OPAL_PATH_MAX);
-        strcat(filename, "/");
+        opal_string_copy(info[i].ldi_filename, output_dir, OPAL_PATH_MAX);
+        strcat(info[i].ldi_filename, "/");
         if (NULL != output_prefix) {
-            strcat(filename, output_prefix);
+            strcat(info[i].ldi_filename, output_prefix);
         }
         if (info[i].ldi_file_suffix != NULL) {
-            strcat(filename, info[i].ldi_file_suffix);
+            strcat(info[i].ldi_filename, info[i].ldi_file_suffix);
         } else {
             info[i].ldi_file_suffix = NULL;
-            strcat(filename, "output.txt");
+            strcat(info[i].ldi_filename, "output.txt");
         }
         flags = O_CREAT | O_RDWR;
         if (!info[i].ldi_file_want_append) {
@@ -794,23 +798,28 @@ static int open_file(int i)
         }
 
         /* Actually open the file */
-        info[i].ldi_fd = open(filename, flags, 0644);
+        info[i].ldi_fd = open(info[i].ldi_filename, flags, 0644);
         if (-1 == info[i].ldi_fd) {
             info[i].ldi_used = false;
-            free(filename);  /* release the filename in all cases */
+            /* release the filename in all error cases */
+            free(info[i].ldi_filename);
+            info[i].ldi_filename = NULL;
             return OPAL_ERR_IN_ERRNO;
         }
 
         /* Make the file be close-on-exec to prevent child inheritance
          * problems */
         if (-1 == fcntl(info[i].ldi_fd, F_SETFD, 1)) {
-            free(filename);  /* release the filename in all cases */
+            /* release the filename in all error cases */
+            free(info[i].ldi_filename);
+            info[i].ldi_filename = NULL;
             return OPAL_ERR_IN_ERRNO;
         }
 
         /* register it to be ignored */
-        opal_pmix_register_cleanup(filename, false, true, false);
-        free(filename);  /* release the filename in all cases */
+        if (NULL != output_pmix_cleanup_fn) {
+            output_pmix_cleanup_fn(info[i].ldi_filename, false, true, false);
+        }
     }
 
     /* Return successfully even if the session dir did not exist yet;
@@ -819,7 +828,6 @@ static int open_file(int i)
     return OPAL_SUCCESS;
 }
 
-
 /*
  * Free all the resources associated with a descriptor.
  */
@@ -827,13 +835,17 @@ static void free_descriptor(int output_id)
 {
     output_desc_t *ldi;
 
-    if (output_id >= 0 && output_id < OPAL_OUTPUT_MAX_STREAMS &&
-        info[output_id].ldi_used && info[output_id].ldi_enabled) {
+    if (output_id >= 0 && output_id < OPAL_OUTPUT_MAX_STREAMS && info[output_id].ldi_used
+        && info[output_id].ldi_enabled) {
         ldi = &info[output_id];
 
         if (-1 != ldi->ldi_fd) {
             close(ldi->ldi_fd);
         }
+        if (NULL != ldi->ldi_filename) {
+            free(ldi->ldi_filename);
+            ldi->ldi_filename = NULL;
+        }
         ldi->ldi_used = false;
 
         /* If we strduped a prefix, suffix, or syslog ident, free it */
@@ -843,12 +855,12 @@ static void free_descriptor(int output_id)
         }
         ldi->ldi_prefix = NULL;
 
-    if (NULL != ldi->ldi_suffix) {
-        free(ldi->ldi_suffix);
-    }
-    ldi->ldi_suffix = NULL;
+        if (NULL != ldi->ldi_suffix) {
+            free(ldi->ldi_suffix);
+        }
+        ldi->ldi_suffix = NULL;
 
-    if (NULL != ldi->ldi_file_suffix) {
+        if (NULL != ldi->ldi_file_suffix) {
             free(ldi->ldi_file_suffix);
         }
         ldi->ldi_file_suffix = NULL;
@@ -860,9 +872,8 @@ static void free_descriptor(int output_id)
     }
 }
 
-
-static int make_string(char **no_newline_string, output_desc_t *ldi,
-                       const char *format, va_list arglist)
+static int make_string(char **no_newline_string, output_desc_t *ldi, const char *format,
+                       va_list arglist)
 {
     size_t len, total_len;
     bool want_newline = false;
@@ -902,27 +913,23 @@ static int make_string(char **no_newline_string, output_desc_t *ldi,
     }
     if (NULL != ldi->ldi_prefix && NULL != ldi->ldi_suffix) {
         if (want_newline) {
-            snprintf(temp_str, temp_str_len, "%s%s%s\n",
-                     ldi->ldi_prefix, *no_newline_string, ldi->ldi_suffix);
+            snprintf(temp_str, temp_str_len, "%s%s%s\n", ldi->ldi_prefix, *no_newline_string,
+                     ldi->ldi_suffix);
         } else {
-            snprintf(temp_str, temp_str_len, "%s%s%s", ldi->ldi_prefix,
-                     *no_newline_string, ldi->ldi_suffix);
+            snprintf(temp_str, temp_str_len, "%s%s%s", ldi->ldi_prefix, *no_newline_string,
+                     ldi->ldi_suffix);
         }
     } else if (NULL != ldi->ldi_prefix) {
         if (want_newline) {
-            snprintf(temp_str, temp_str_len, "%s%s\n",
-                     ldi->ldi_prefix, *no_newline_string);
+            snprintf(temp_str, temp_str_len, "%s%s\n", ldi->ldi_prefix, *no_newline_string);
         } else {
-            snprintf(temp_str, temp_str_len, "%s%s", ldi->ldi_prefix,
-                     *no_newline_string);
+            snprintf(temp_str, temp_str_len, "%s%s", ldi->ldi_prefix, *no_newline_string);
         }
     } else if (NULL != ldi->ldi_suffix) {
         if (want_newline) {
-            snprintf(temp_str, temp_str_len, "%s%s\n",
-                     *no_newline_string, ldi->ldi_suffix);
+            snprintf(temp_str, temp_str_len, "%s%s\n", *no_newline_string, ldi->ldi_suffix);
         } else {
-            snprintf(temp_str, temp_str_len, "%s%s",
-                     *no_newline_string, ldi->ldi_suffix);
+            snprintf(temp_str, temp_str_len, "%s%s", *no_newline_string, ldi->ldi_suffix);
         }
     } else {
         if (want_newline) {
@@ -954,8 +961,8 @@ static int output(int output_id, const char *format, va_list arglist)
 
     /* If it's valid, used, and enabled, output */
 
-    if (output_id >= 0 && output_id < OPAL_OUTPUT_MAX_STREAMS &&
-        info[output_id].ldi_used && info[output_id].ldi_enabled) {
+    if (output_id >= 0 && output_id < OPAL_OUTPUT_MAX_STREAMS && info[output_id].ldi_used
+        && info[output_id].ldi_enabled) {
         OPAL_THREAD_LOCK(&mutex);
         ldi = &info[output_id];
 
@@ -979,15 +986,20 @@ static int output(int output_id, const char *format, va_list arglist)
 
         /* stdout output */
         if (ldi->ldi_stdout) {
-            write(fileno(stdout), out, (int)strlen(out));
+            int tmp = opal_best_effort_write(fileno(stdout), out, strlen(out));
+            if (OPAL_SUCCESS != tmp) {
+                rc = tmp;
+            }
             fflush(stdout);
         }
 
         /* stderr output */
         if (ldi->ldi_stderr) {
-            write((-1 == default_stderr_fd) ?
-                  fileno(stderr) : default_stderr_fd,
-                  out, (int)strlen(out));
+            int tmp = opal_best_effort_write((-1 == default_stderr_fd) ? fileno(stderr) : default_stderr_fd,
+                                             out, strlen(out));
+            if (OPAL_SUCCESS != tmp) {
+                rc = tmp;
+            }
             fflush(stderr);
         }
 
@@ -1002,20 +1014,24 @@ static int output(int output_id, const char *format, va_list arglist)
                     ++ldi->ldi_file_num_lines_lost;
                 } else if (ldi->ldi_file_num_lines_lost > 0) {
                     char buffer[BUFSIZ];
-                    char *out = buffer;
+                    int tmp;
                     memset(buffer, 0, BUFSIZ);
                     snprintf(buffer, BUFSIZ - 1,
-                             "[WARNING: %d lines lost because the Open MPI process session directory did\n not exist when opal_output() was invoked]\n",
+                             "[WARNING: %d lines lost because the Open MPI process session "
+                             "directory did\n not exist when opal_output() was invoked]\n",
                              ldi->ldi_file_num_lines_lost);
-                   write(ldi->ldi_fd, buffer, (int)strlen(buffer));
-                    ldi->ldi_file_num_lines_lost = 0;
-                    if (out != buffer) {
-                        free(out);
+                    tmp = opal_best_effort_write(ldi->ldi_fd, buffer, strlen(buffer));
+                    if (OPAL_SUCCESS != tmp) {
+                        rc = tmp;
                     }
+                    ldi->ldi_file_num_lines_lost = 0;
                 }
             }
             if (ldi->ldi_fd != -1) {
-                write(ldi->ldi_fd, out, (int)strlen(out));
+                int tmp = opal_best_effort_write(ldi->ldi_fd, out, strlen(out));
+                if (OPAL_SUCCESS != tmp) {
+                    rc = tmp;
+                }
             }
         }
         OPAL_THREAD_UNLOCK(&mutex);
diff --git a/opal/util/output.h b/opal/util/output.h
index 76424c60015..ff14622f2e5 100644
--- a/opal/util/output.h
+++ b/opal/util/output.h
@@ -71,7 +71,10 @@
 #include "opal_config.h"
 
 #include <stdarg.h>
+#include <unistd.h>
+#include <errno.h>
 
+#include "opal/constants.h"
 #include "opal/class/opal_object.h"
 
 BEGIN_C_DECLS
@@ -152,7 +155,7 @@ struct opal_output_stream_t {
     char *lds_syslog_ident;
 #else
     HANDLE lds_syslog_ident;
-#endif  /* !defined(__WINDOWS__) */
+#endif /* !defined(__WINDOWS__) */
 
     /**
      * String prefix added to all output on the stream.
@@ -244,304 +247,302 @@ struct opal_output_stream_t {
      * for details on what happens in this situation.
      */
     char *lds_file_suffix;
-
 };
 
-    /**
-     * Convenience typedef
-     */
-    typedef struct opal_output_stream_t opal_output_stream_t;
+/**
+ * Convenience typedef
+ */
+typedef struct opal_output_stream_t opal_output_stream_t;
 
-    /**
-     * Initializes the output stream system and opens a default
-     * "verbose" stream.
-     *
-     * @retval true Upon success.
-     * @retval false Upon failure.
-     *
-     * This should be the first function invoked in the output
-     * subsystem.  After this call, the default "verbose" stream is open
-     * and can be written to via calls to opal_output_verbose() and
-     * opal_output_error().
-     *
-     * By definition, the default verbose stream has a handle ID of 0,
-     * and has a verbose level of 0.
-     */
-    OPAL_DECLSPEC bool opal_output_init(void);
+/**
+ * Initializes the output stream system and opens a default
+ * "verbose" stream.
+ *
+ * @retval true Upon success.
+ * @retval false Upon failure.
+ *
+ * This should be the first function invoked in the output
+ * subsystem.  After this call, the default "verbose" stream is open
+ * and can be written to via calls to opal_output_verbose() and
+ * opal_output_error().
+ *
+ * By definition, the default verbose stream has a handle ID of 0,
+ * and has a verbose level of 0.
+ */
+OPAL_DECLSPEC bool opal_output_init(void);
 
-    /**
-     * Opens an output stream.
-     *
-     * @param lds A pointer to opal_output_stream_t describing what the
-     * characteristics of the output stream should be.
-     *
-     * This function opens an output stream and returns an integer
-     * handle.  The caller is responsible for maintaining the handle and
-     * using it in successive calls to OPAL_OUTPUT(), opal_output(),
-     * opal_output_switch(), and opal_output_close().
-     *
-     * If lds is NULL, the default descriptions will be used, meaning
-     * that output will only be sent to stderr.
-     *
-     * It is safe to have multiple threads invoke this function
-     * simultaneously; their execution will be serialized in an
-     * unspecified manner.
-     *
-     * Be sure to see opal_output() for a description of what happens
-     * when open_open() / opal_output() is directed to send output to a
-     * file but the process session directory does not yet exist.
-     */
-    OPAL_DECLSPEC int opal_output_open(opal_output_stream_t *lds);
+/**
+ * Opens an output stream.
+ *
+ * @param lds A pointer to opal_output_stream_t describing what the
+ * characteristics of the output stream should be.
+ *
+ * This function opens an output stream and returns an integer
+ * handle.  The caller is responsible for maintaining the handle and
+ * using it in successive calls to OPAL_OUTPUT(), opal_output(),
+ * opal_output_switch(), and opal_output_close().
+ *
+ * If lds is NULL, the default descriptions will be used, meaning
+ * that output will only be sent to stderr.
+ *
+ * It is safe to have multiple threads invoke this function
+ * simultaneously; their execution will be serialized in an
+ * unspecified manner.
+ *
+ * Be sure to see opal_output() for a description of what happens
+ * when open_open() / opal_output() is directed to send output to a
+ * file but the process session directory does not yet exist.
+ */
+OPAL_DECLSPEC int opal_output_open(opal_output_stream_t *lds);
 
-    /**
-     * Re-opens / redirects an output stream.
-     *
-     * @param output_id Stream handle to reopen
-     * @param lds A pointer to opal_output_stream_t describing what the
-     * characteristics of the reopened output stream should be.
-     *
-     * This function redirects an existing stream into a new [set of]
-     * location[s], as specified by the lds parameter.  If the output_id
-     * passed is invalid, this call is effectively the same as opening a
-     * new stream with a specific stream handle.
-     */
-    OPAL_DECLSPEC int opal_output_reopen(int output_id, opal_output_stream_t *lds);
+/**
+ * Re-opens / redirects an output stream.
+ *
+ * @param output_id Stream handle to reopen
+ * @param lds A pointer to opal_output_stream_t describing what the
+ * characteristics of the reopened output stream should be.
+ *
+ * This function redirects an existing stream into a new [set of]
+ * location[s], as specified by the lds parameter.  If the output_id
+ * passed is invalid, this call is effectively the same as opening a
+ * new stream with a specific stream handle.
+ */
+OPAL_DECLSPEC int opal_output_reopen(int output_id, opal_output_stream_t *lds);
 
-    /**
-     * Enables and disables output streams.
-     *
-     * @param output_id Stream handle to switch
-     * @param enable Boolean indicating whether to enable the stream
-     * output or not.
-     *
-     * @returns The previous enable state of the stream (true == enabled,
-     * false == disabled).
-     *
-     * The output of a stream can be temporarily disabled by passing an
-     * enable value to false, and later resumed by passing an enable
-     * value of true.  This does not close the stream -- it simply tells
-     * the opal_output subsystem to intercept and discard any output sent
-     * to the stream via OPAL_OUTPUT() or opal_output() until the output
-     * is re-enabled.
-     */
-    OPAL_DECLSPEC bool opal_output_switch(int output_id, bool enable);
+/**
+ * Enables and disables output streams.
+ *
+ * @param output_id Stream handle to switch
+ * @param enable Boolean indicating whether to enable the stream
+ * output or not.
+ *
+ * @returns The previous enable state of the stream (true == enabled,
+ * false == disabled).
+ *
+ * The output of a stream can be temporarily disabled by passing an
+ * enable value to false, and later resumed by passing an enable
+ * value of true.  This does not close the stream -- it simply tells
+ * the opal_output subsystem to intercept and discard any output sent
+ * to the stream via OPAL_OUTPUT() or opal_output() until the output
+ * is re-enabled.
+ */
+OPAL_DECLSPEC bool opal_output_switch(int output_id, bool enable);
 
-    /**
-     * \internal
-     *
-     * Reopens all existing output streams.
-     *
-     * This function should never be called by user applications; it is
-     * typically only invoked after a restart (i.e., in a new process)
-     * where output streams need to be re-initialized.
-     */
-    OPAL_DECLSPEC void opal_output_reopen_all(void);
+/**
+ * \internal
+ *
+ * Reopens all existing output streams.
+ *
+ * This function should never be called by user applications; it is
+ * typically only invoked after a restart (i.e., in a new process)
+ * where output streams need to be re-initialized.
+ */
+OPAL_DECLSPEC void opal_output_reopen_all(void);
 
-    /**
-     * Close an output stream.
-     *
-     * @param output_id Handle of the stream to close.
-     *
-     * Close an output stream.  No output will be sent to the stream
-     * after it is closed.  Be aware that output handles tend to be
-     * re-used; it is possible that after a stream is closed, if another
-     * stream is opened, it will get the same handle value.
-     */
-    OPAL_DECLSPEC void opal_output_close(int output_id);
+/**
+ * Close an output stream.
+ *
+ * @param output_id Handle of the stream to close.
+ *
+ * Close an output stream.  No output will be sent to the stream
+ * after it is closed.  Be aware that output handles tend to be
+ * re-used; it is possible that after a stream is closed, if another
+ * stream is opened, it will get the same handle value.
+ */
+OPAL_DECLSPEC void opal_output_close(int output_id);
 
-    /**
-     * Main function to send output to a stream.
-     *
-     * @param output_id Stream id returned from opal_output_open().
-     * @param format printf-style format string.
-     * @param varargs printf-style varargs list to fill the string
-     * specified by the format parameter.
-     *
-     * This is the main function to send output to custom streams (note
-     * that output to the default "verbose" stream is handled through
-     * opal_output_verbose() and opal_output_error()).
-     *
-     * It is never necessary to send a trailing "\n" in the strings to
-     * this function; some streams requires newlines, others do not --
-     * this function will append newlines as necessary.
-     *
-     * Verbosity levels are ignored in this function.
-     *
-     * Note that for output streams that are directed to files, the
-     * files are stored under the process' session directory.  If the
-     * session directory does not exist when opal_output() is invoked,
-     * the output will be discarded!  Once the session directory is
-     * created, opal_output() will automatically create the file and
-     * writing to it.
-     */
-    OPAL_DECLSPEC void opal_output(int output_id, const char *format, ...) __opal_attribute_format__(__printf__, 2, 3);
+/**
+ * Main function to send output to a stream.
+ *
+ * @param output_id Stream id returned from opal_output_open().
+ * @param format printf-style format string.
+ * @param varargs printf-style varargs list to fill the string
+ * specified by the format parameter.
+ *
+ * This is the main function to send output to custom streams (note
+ * that output to the default "verbose" stream is handled through
+ * opal_output_verbose() and opal_output_error()).
+ *
+ * It is never necessary to send a trailing "\n" in the strings to
+ * this function; some streams requires newlines, others do not --
+ * this function will append newlines as necessary.
+ *
+ * Verbosity levels are ignored in this function.
+ *
+ * Note that for output streams that are directed to files, the
+ * files are stored under the process' session directory.  If the
+ * session directory does not exist when opal_output() is invoked,
+ * the output will be discarded!  Once the session directory is
+ * created, opal_output() will automatically create the file and
+ * writing to it.
+ */
+OPAL_DECLSPEC void opal_output(int output_id, const char *format, ...)
+    __opal_attribute_format__(__printf__, 2, 3);
 
-    /**
-     * Send output to a stream only if the passed verbosity level is
-     * high enough.
-     *
-     * @param output_id Stream id returned from opal_output_open().
-     * @param level Target verbosity level.
-     * @param format printf-style format string.
-     * @param varargs printf-style varargs list to fill the string
-     * specified by the format parameter.
-     *
-     * Output is only sent to the stream if the current verbosity level
-     * is greater than or equal to the level parameter.  This mechanism
-     * can be used to send "information" kinds of output to user
-     * applications, but only when the user has asked for a high enough
-     * verbosity level.
-     *
-     * It is never necessary to send a trailing "\n" in the strings to
-     * this function; some streams requires newlines, others do not --
-     * this function will append newlines as necessary.
-     *
-     * This function is really a convenience wrapper around checking the
-     * current verbosity level set on the stream, and if the passed
-     * level is less than or equal to the stream's verbosity level, this
-     * function will effectively invoke opal_output to send the output to
-     * the stream.
-     *
-     * @see opal_output_set_verbosity()
-     */
+/**
+ * Send output to a stream only if the passed verbosity level is
+ * high enough.
+ *
+ * @param output_id Stream id returned from opal_output_open().
+ * @param level Target verbosity level.
+ * @param format printf-style format string.
+ * @param varargs printf-style varargs list to fill the string
+ * specified by the format parameter.
+ *
+ * Output is only sent to the stream if the current verbosity level
+ * is greater than or equal to the level parameter.  This mechanism
+ * can be used to send "information" kinds of output to user
+ * applications, but only when the user has asked for a high enough
+ * verbosity level.
+ *
+ * It is never necessary to send a trailing "\n" in the strings to
+ * this function; some streams requires newlines, others do not --
+ * this function will append newlines as necessary.
+ *
+ * This function is really a convenience wrapper around checking the
+ * current verbosity level set on the stream, and if the passed
+ * level is less than or equal to the stream's verbosity level, this
+ * function will effectively invoke opal_output to send the output to
+ * the stream.
+ *
+ * @see opal_output_set_verbosity()
+ */
 #define opal_output_verbose(verbose_level, output_id, ...)           \
     do {                                                             \
         if (opal_output_check_verbosity(verbose_level, output_id)) { \
             opal_output(output_id, __VA_ARGS__);                     \
         }                                                            \
-    } while(0)
+    } while (0)
 
-    OPAL_DECLSPEC bool opal_output_check_verbosity(int verbose_level, int output_id);
+OPAL_DECLSPEC bool opal_output_check_verbosity(int verbose_level, int output_id);
 
-   /**
-    * Same as opal_output_verbose(), but takes a va_list form of varargs.
-    */
-    OPAL_DECLSPEC void opal_output_vverbose(int verbose_level, int output_id,
-                                            const char *format, va_list ap) __opal_attribute_format__(__printf__, 3, 0);
+/**
+ * Same as opal_output_verbose(), but takes a va_list form of varargs.
+ */
+OPAL_DECLSPEC void opal_output_vverbose(int verbose_level, int output_id, const char *format,
+                                        va_list ap) __opal_attribute_format__(__printf__, 3, 0);
 
-    /**
-     * Send output to a string if the verbosity level is high enough.
-     *
-     * @param output_id Stream id returned from opal_output_open().
-     * @param level Target verbosity level.
-     * @param format printf-style format string.
-     * @param varargs printf-style varargs list to fill the string
-     * specified by the format parameter.
-     *
-     * Exactly the same as opal_output_verbose(), except the output it
-     * sent to a string instead of to the stream.  If the verbose
-     * level is not high enough, NULL is returned.  The caller is
-     * responsible for free()'ing the returned string.
-     */
-    OPAL_DECLSPEC char *opal_output_string(int verbose_level, int output_id,
-                                           const char *format, ...) __opal_attribute_format__(__printf__, 3, 4);
+/**
+ * Send output to a string if the verbosity level is high enough.
+ *
+ * @param output_id Stream id returned from opal_output_open().
+ * @param level Target verbosity level.
+ * @param format printf-style format string.
+ * @param varargs printf-style varargs list to fill the string
+ * specified by the format parameter.
+ *
+ * Exactly the same as opal_output_verbose(), except the output it
+ * sent to a string instead of to the stream.  If the verbose
+ * level is not high enough, NULL is returned.  The caller is
+ * responsible for free()'ing the returned string.
+ */
+OPAL_DECLSPEC char *opal_output_string(int verbose_level, int output_id, const char *format, ...)
+    __opal_attribute_format__(__printf__, 3, 4);
 
-   /**
-    * Same as opal_output_string, but accepts a va_list form of varargs.
-    */
-    OPAL_DECLSPEC char *opal_output_vstring(int verbose_level, int output_id,
-                                            const char *format, va_list ap) __opal_attribute_format__(__printf__, 3, 0);
+/**
+ * Same as opal_output_string, but accepts a va_list form of varargs.
+ */
+OPAL_DECLSPEC char *opal_output_vstring(int verbose_level, int output_id, const char *format,
+                                        va_list ap) __opal_attribute_format__(__printf__, 3, 0);
 
-    /**
-     * Set the verbosity level for a stream.
-     *
-     * @param output_id Stream id returned from opal_output_open().
-     * @param level New verbosity level
-     *
-     * This function sets the verbosity level on a given stream.  It
-     * will be used for all future invocations of opal_output_verbose().
-     */
-    OPAL_DECLSPEC void opal_output_set_verbosity(int output_id, int level);
+/**
+ * Set the verbosity level for a stream.
+ *
+ * @param output_id Stream id returned from opal_output_open().
+ * @param level New verbosity level
+ *
+ * This function sets the verbosity level on a given stream.  It
+ * will be used for all future invocations of opal_output_verbose().
+ */
+OPAL_DECLSPEC void opal_output_set_verbosity(int output_id, int level);
 
-    /**
-     * Get the verbosity level for a stream
-     *
-     * @param output_id Stream id returned from opal_output_open()
-     * @returns Verbosity of stream
-     */
-    OPAL_DECLSPEC int opal_output_get_verbosity(int output_id);
+/**
+ * Get the verbosity level for a stream
+ *
+ * @param output_id Stream id returned from opal_output_open()
+ * @returns Verbosity of stream
+ */
+OPAL_DECLSPEC int opal_output_get_verbosity(int output_id);
 
-    /**
-     * Set characteristics for output files.
-     *
-     * @param dir Directory where output files will go
-     * @param olddir If non-NULL, the directory where output files
-     * were previously opened
-     * @param prefix Prefix of files in the output directory
-     * @param oldprefix If non-NULL, the old prefix
-     *
-     * This function controls the final filename used for all new
-     * output streams that request output files.  Specifically, when
-     * opal_output_stream_t::lds_want_file is true, the output
-     * filename will be of the form $dir/$prefix$suffix.
-     *
-     * The default value for the output directory is whatever is
-     * specified in the TMPDIR environment variable if it exists, or
-     * $HOME if it does not.  The default value for the prefix is
-     * "output-pid<pid>-" (where "<pid>" is replaced by the PID of the
-     * current process).
-     *
-     * If dir or prefix are NULL, new values are not set.  The strings
-     * represented by dir and prefix are copied into internal storage;
-     * it is safe to pass string constants or free() these values
-     * after opal_output_set_output_file_info() returns.
-     *
-     * If olddir or oldprefix are not NULL, copies of the old
-     * directory and prefix (respectively) are returned in these
-     * parameters.  The caller is responsible for calling (free) on
-     * these values.  This allows one to get the old values, output an
-     * output file in a specific directory and/or with a specific
-     * prefix, and then restore the old values.
-     *
-     * Note that this function only affects the creation of \em new
-     * streams -- streams that have already started writing to output
-     * files are not affected (i.e., their output files are not moved
-     * to the new directory).  More specifically, the opal_output
-     * system only opens/creates output files lazily -- so calling
-     * this function affects both new streams \em and any stream that
-     * was previously opened but had not yet output anything.
-     */
-    OPAL_DECLSPEC void opal_output_set_output_file_info(const char *dir,
-                                                        const char *prefix,
-                                                        char **olddir,
-                                                        char **oldprefix);
+/**
+ * Set characteristics for output files.
+ *
+ * @param dir Directory where output files will go
+ * @param olddir If non-NULL, the directory where output files
+ * were previously opened
+ * @param prefix Prefix of files in the output directory
+ * @param oldprefix If non-NULL, the old prefix
+ *
+ * This function controls the final filename used for all new
+ * output streams that request output files.  Specifically, when
+ * opal_output_stream_t::lds_want_file is true, the output
+ * filename will be of the form $dir/$prefix$suffix.
+ *
+ * The default value for the output directory is whatever is
+ * specified in the TMPDIR environment variable if it exists, or
+ * $HOME if it does not.  The default value for the prefix is
+ * "output-pid<pid>-" (where "<pid>" is replaced by the PID of the
+ * current process).
+ *
+ * If dir or prefix are NULL, new values are not set.  The strings
+ * represented by dir and prefix are copied into internal storage;
+ * it is safe to pass string constants or free() these values
+ * after opal_output_set_output_file_info() returns.
+ *
+ * If olddir or oldprefix are not NULL, copies of the old
+ * directory and prefix (respectively) are returned in these
+ * parameters.  The caller is responsible for calling (free) on
+ * these values.  This allows one to get the old values, output an
+ * output file in a specific directory and/or with a specific
+ * prefix, and then restore the old values.
+ *
+ * Note that this function only affects the creation of \em new
+ * streams -- streams that have already started writing to output
+ * files are not affected (i.e., their output files are not moved
+ * to the new directory).  More specifically, the opal_output
+ * system only opens/creates output files lazily -- so calling
+ * this function affects both new streams \em and any stream that
+ * was previously opened but had not yet output anything.
+ */
+OPAL_DECLSPEC void opal_output_set_output_file_info(const char *dir, const char *prefix,
+                                                    char **olddir, char **oldprefix);
 
 #if OPAL_ENABLE_DEBUG
-    /**
-     * Main macro for use in sending debugging output to output streams;
-     * will be "compiled out" when OPAL is configured without
-     * --enable-debug.
-     *
-     * @see opal_output()
-     */
-#define OPAL_OUTPUT(a) opal_output a
+/**
+ * Main macro for use in sending debugging output to output streams;
+ * will be "compiled out" when OPAL is configured without
+ * --enable-debug.
+ *
+ * @see opal_output()
+ */
+#    define OPAL_OUTPUT(a) opal_output a
 
-    /**
-     * Macro for use in sending debugging output to the output
-     * streams.  Will be "compiled out" when OPAL is configured
-     * without --enable-debug.
-     *
-     * @see opal_output_verbose()
-     */
-#define OPAL_OUTPUT_VERBOSE(a) opal_output_verbose a
+/**
+ * Macro for use in sending debugging output to the output
+ * streams.  Will be "compiled out" when OPAL is configured
+ * without --enable-debug.
+ *
+ * @see opal_output_verbose()
+ */
+#    define OPAL_OUTPUT_VERBOSE(a) opal_output_verbose a
 #else
-    /**
-     * Main macro for use in sending debugging output to output streams;
-     * will be "compiled out" when OPAL is configured without
-     * --enable-debug.
-     *
-     * @see opal_output()
-     */
-#define OPAL_OUTPUT(a)
+/**
+ * Main macro for use in sending debugging output to output streams;
+ * will be "compiled out" when OPAL is configured without
+ * --enable-debug.
+ *
+ * @see opal_output()
+ */
+#    define OPAL_OUTPUT(a)
 
-    /**
-     * Macro for use in sending debugging output to the output
-     * streams.  Will be "compiled out" when OPAL is configured
-     * without --enable-debug.
-     *
-     * @see opal_output_verbose()
-     */
-#define OPAL_OUTPUT_VERBOSE(a)
+/**
+ * Macro for use in sending debugging output to the output
+ * streams.  Will be "compiled out" when OPAL is configured
+ * without --enable-debug.
+ *
+ * @see opal_output_verbose()
+ */
+#    define OPAL_OUTPUT_VERBOSE(a)
 #endif
 
 /**
@@ -554,7 +555,38 @@ struct opal_output_stream_t {
  */
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_output_stream_t);
 
+/**
+ * Best effort write for blocking file descriptors
+ *
+ * A wrapper around write() that will spin trying to write the entire
+ * buffer until either an error occurs or a write is not able to write
+ * any data.  Useful for situations where opal_output cannot be used
+ * for signal reasons.  Do not use with non-blocking file descriptors.
+ */
+static inline int opal_best_effort_write(int fd, const void *buf, size_t count)
+{
+    size_t written = 0;
+
+    while (written != count) {
+        ssize_t tmp = write(fd, (char *)buf + written, count - written);
+        /* we explicitly do not recover from errors in this wrapper */
+        if ((tmp < 0 && errno != EINTR) || (tmp == 0)) {
+            return OPAL_ERROR;
+        }
+        written += tmp;
+    }
+
+    return OPAL_SUCCESS;
+}
+
+/**
+ * Cleanup output files function:
+ * Signature: opal_pmix_register_cleanup
+ */
+typedef int (*opal_output_pmix_cleanup_fn_t)(char *path, bool directory, bool ignore,
+                                             bool jobscope);
+void opal_output_register_pmix_cleanup_fn(opal_output_pmix_cleanup_fn_t cleanup_fn);
+
 END_C_DECLS
 
 #endif /* OPAL_OUTPUT_H_ */
-
diff --git a/opal/util/path.c b/opal/util/path.c
index 8128e5dc2c1..9e4200a9384 100644
--- a/opal/util/path.c
+++ b/opal/util/path.c
@@ -17,6 +17,7 @@
  * Copyright (c) 2016      University of Houston. All rights reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2025      Nanook Consulting  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -25,83 +26,82 @@
  */
 
 #include "opal_config.h"
+#include <errno.h>
 #include <stdlib.h>
 #include <string.h>
-#include <errno.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 #ifdef HAVE_SHLWAPI_H
-#include <shlwapi.h>
+#    include <shlwapi.h>
 #endif
 #ifdef HAVE_SYS_PARAM_H
-#include <sys/param.h>
+#    include <sys/param.h>
 #endif
 #ifdef HAVE_SYS_MOUNT_H
-#include <sys/mount.h>
+#    include <sys/mount.h>
 #endif
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 #ifdef HAVE_SYS_STAT_H
-#include <sys/stat.h>
+#    include <sys/stat.h>
 #endif
 #ifdef HAVE_SYS_VFS_H
-#include <sys/vfs.h>
+#    include <sys/vfs.h>
 #endif
 #ifdef HAVE_SYS_STATFS_H
-#include <sys/statfs.h>
+#    include <sys/statfs.h>
 #endif
 #ifdef HAVE_SYS_STATVFS_H
-#include <sys/statvfs.h>
+#    include <sys/statvfs.h>
 #endif
 #ifdef HAVE_SYS_MOUNT_H
-#include <sys/mount.h>
+#    include <sys/mount.h>
 #endif
 #ifdef HAVE_MNTENT_H
-#include <mntent.h>
+#    include <mntent.h>
 #endif
 #ifdef HAVE_PATHS_H
-#include <paths.h>
+#    include <paths.h>
 #endif
 
 #ifdef _PATH_MOUNTED
-#define MOUNTED_FILE _PATH_MOUNTED
+#    define MOUNTED_FILE _PATH_MOUNTED
 #else
-#define MOUNTED_FILE "/etc/mtab"
+#    define MOUNTED_FILE "/etc/mtab"
 #endif
 
-
-#include "opal_stdint.h"
+#include "opal/util/argv.h"
+#include "opal/util/os_path.h"
 #include "opal/util/output.h"
 #include "opal/util/path.h"
-#include "opal/util/os_path.h"
-#include "opal/util/argv.h"
 #include "opal/util/printf.h"
+#include "opal_stdint.h"
 
 /*
  * Sanity check to ensure we have either statfs or statvfs
  */
 #if !defined(HAVE_STATFS) && !defined(HAVE_STATVFS)
-#error Must have either statfs() or statvfs()
+#    error Must have either statfs() or statvfs()
 #endif
 
 /*
- * Note that some OS's (e.g., NetBSD and Solaris) have statfs(), but
+ * Note that some OS's (e.g., NetBSD has statfs(), but
  * no struct statfs (!).  So check to make sure we have struct statfs
  * before allowing the use of statfs().
  */
-#if defined(HAVE_STATFS) && (defined(HAVE_STRUCT_STATFS_F_FSTYPENAME) || \
-                             defined(HAVE_STRUCT_STATFS_F_TYPE))
-#define USE_STATFS 1
+#if defined(HAVE_STATFS) \
+    && (defined(HAVE_STRUCT_STATFS_F_FSTYPENAME) || defined(HAVE_STRUCT_STATFS_F_TYPE))
+#    define USE_STATFS 1
 #endif
 
 static void path_env_load(char *path, int *pargc, char ***pargv);
 static char *list_env_get(char *var, char **list);
 
-bool opal_path_is_absolute( const char *path )
+bool opal_path_is_absolute(const char *path)
 {
-    if( OPAL_PATH_SEP[0] == *path ) {
+    if (OPAL_PATH_SEP[0] == *path) {
         return true;
     }
     return false;
@@ -119,7 +119,7 @@ char *opal_path_find(char *fname, char **pathv, int mode, char **envv)
     int i;
 
     /* If absolute path is given, return it without searching. */
-    if( opal_path_is_absolute(fname) ) {
+    if (opal_path_is_absolute(fname)) {
         return opal_path_access(fname, NULL, mode);
     }
 
@@ -139,7 +139,7 @@ char *opal_path_find(char *fname, char **pathv, int mode, char **envv)
             if (delimit) {
                 *delimit = '\0';
             }
-            env = list_env_get(pathv[i]+1, envv);
+            env = list_env_get(pathv[i] + 1, envv);
             if (delimit) {
                 *delimit = OPAL_PATH_SEP[0];
             }
@@ -152,8 +152,7 @@ char *opal_path_find(char *fname, char **pathv, int mode, char **envv)
                     free(pfix);
                 }
             }
-        }
-        else {
+        } else {
             fullpath = opal_path_access(fname, pathv[i], mode);
         }
         i++;
@@ -190,7 +189,7 @@ char *opal_path_findv(char *fname, int mode, char **envv, char *wrkdir)
                 found_dot = true;
                 free(dirv[i]);
                 dirv[i] = strdup(wrkdir);
-                if (NULL == dirv[i]){
+                if (NULL == dirv[i]) {
                     return NULL;
                 }
             }
@@ -204,16 +203,16 @@ char *opal_path_findv(char *fname, int mode, char **envv, char *wrkdir)
         opal_argv_append(&dirc, &dirv, wrkdir);
     }
 
-    if(NULL == dirv)
+    if (NULL == dirv) {
         return NULL;
+    }
     fullpath = opal_path_find(fname, dirv, mode, envv);
     opal_argv_free(dirv);
     return fullpath;
 }
 
-
 /**
- *  Forms a complete pathname and checks it for existance and
+ *  Forms a complete pathname and checks it for existence and
  *  permissions
  *
  *  Accepts:
@@ -252,8 +251,7 @@ char *opal_path_access(char *fname, char *path, int mode)
         return NULL;
     }
 
-    if (!(S_IFREG & buf.st_mode) &&
-        !(S_IFLNK & buf.st_mode)) {
+    if (!(S_IFREG & buf.st_mode) && !(S_IFLNK & buf.st_mode)) {
         /* this isn't a regular file or a symbolic link, so
          * ignore it
          */
@@ -288,7 +286,6 @@ char *opal_path_access(char *fname, char *path, int mode)
     return fullpath;
 }
 
-
 /**
  *
  *  Loads argument array with $PATH env var.
@@ -337,7 +334,6 @@ static void path_env_load(char *path, int *pargc, char ***pargv)
     }
 }
 
-
 /**
  *  Gets value of variable in list or environment. Looks in the list first
  *
@@ -376,31 +372,37 @@ static char *list_env_get(char *var, char **list)
  * function will return NULL. Otherwise, an newly allocated string
  * will be returned.
  */
-char* opal_find_absolute_path( char* app_name )
+char *opal_find_absolute_path(char *app_name)
 {
-    char* abs_app_name;
+    char *abs_app_name;
     char cwd[OPAL_PATH_MAX], *pcwd;
 
-    if( opal_path_is_absolute(app_name) ) { /* already absolute path */
+    if (opal_path_is_absolute(app_name)) { /* already absolute path */
         abs_app_name = app_name;
-    } else if ( '.' == app_name[0] ||
-               NULL != strchr(app_name, OPAL_PATH_SEP[0])) {
+    } else if ('.' == app_name[0] || NULL != strchr(app_name, OPAL_PATH_SEP[0])) {
         /* the app is in the current directory or below it */
-        pcwd = getcwd( cwd, OPAL_PATH_MAX );
-        if( NULL == pcwd ) {
+        pcwd = getcwd(cwd, OPAL_PATH_MAX);
+        if (NULL == pcwd) {
             /* too bad there is no way we can get the app absolute name */
             return NULL;
         }
-        abs_app_name = opal_os_path( false, pcwd, app_name, NULL );
+        abs_app_name = opal_os_path(false, pcwd, app_name, NULL);
     } else {
         /* Otherwise try to search for the application in the PATH ... */
-        abs_app_name = opal_path_findv( app_name, X_OK, NULL, NULL );
+        abs_app_name = opal_path_findv(app_name, X_OK, NULL, NULL);
     }
 
-    if( NULL != abs_app_name ) {
-        char* resolved_path = (char*)malloc(OPAL_PATH_MAX);
-        realpath( abs_app_name, resolved_path );
-        if( abs_app_name != app_name ) free(abs_app_name);
+    if (NULL != abs_app_name) {
+        char *resolved_path = (char *) malloc(OPAL_PATH_MAX);
+        char *ret;
+        ret = realpath(abs_app_name, resolved_path);
+        if (abs_app_name != app_name) {
+            free(abs_app_name);
+        }
+        if (NULL == ret) {
+            free(resolved_path);
+            resolved_path = NULL;
+        }
         return resolved_path;
     }
     return NULL;
@@ -408,23 +410,21 @@ char* opal_find_absolute_path( char* app_name )
 
 /**
  * Read real FS type from /etc/mtab, needed to translate autofs fs type into real fs type
- * TODO: solaris? OSX?
- * Limitations: autofs on solaris/osx will be assumed as "nfs" type
+ * TODO: OSX?
+ * Limitations: autofs on osx will be assumed as "nfs" type
  */
 
 static char *opal_check_mtab(char *dev_path)
 {
 
 #ifdef HAVE_MNTENT_H
-    FILE * mtab = NULL;
-    struct mntent * part = NULL;
+    FILE *mtab = NULL;
+    struct mntent *part = NULL;
 
     if ((mtab = setmntent(MOUNTED_FILE, "r")) != NULL) {
         while (NULL != (part = getmntent(mtab))) {
-            if ((NULL != part->mnt_dir) &&
-                (NULL != part->mnt_type) &&
-                (0 == strcmp(part->mnt_dir, dev_path)))
-            {
+            if ((NULL != part->mnt_dir) && (NULL != part->mnt_type)
+                && (0 == strcmp(part->mnt_dir, dev_path))) {
                 endmntent(mtab);
                 return strdup(part->mnt_type);
             }
@@ -435,7 +435,6 @@ static char *opal_check_mtab(char *dev_path)
     return NULL;
 }
 
-
 /**
  * @brief Figure out, whether fname is on network file system
  *
@@ -459,10 +458,6 @@ static char *opal_check_mtab(char *dev_path)
  *   statvfs (const char *path, struct statvfs *buf);
  *          with unsigned long  f_fsid;   -- returns wrong info
  *          return 0 success, -1 on failure with errno set.
- * Solaris:
- *   statvfs (const char *path, struct statvfs *buf);
- *          with f_basetype, contains a string of length FSTYPSZ
- *          return 0 success, -1 on failure with errno set.
  * FreeBSD:
  *   statfs(const char *path, struct statfs *buf);
  *          with f_fstypename, contains a string of length MFSNAMELEN
@@ -481,26 +476,26 @@ static char *opal_check_mtab(char *dev_path)
  *          return 0 success, -1 on failure with errno set.
  */
 #ifndef LL_SUPER_MAGIC
-#define LL_SUPER_MAGIC                    0x0BD00BD0     /* Lustre magic number */
+#    define LL_SUPER_MAGIC 0x0BD00BD0 /* Lustre magic number */
 #endif
 #ifndef NFS_SUPER_MAGIC
-#define NFS_SUPER_MAGIC                   0x6969
+#    define NFS_SUPER_MAGIC 0x6969
 #endif
 #ifndef PAN_KERNEL_FS_CLIENT_SUPER_MAGIC
-#define PAN_KERNEL_FS_CLIENT_SUPER_MAGIC  0xAAD7AAEA     /* Panasas FS */
+#    define PAN_KERNEL_FS_CLIENT_SUPER_MAGIC 0xAAD7AAEA /* Panasas FS */
 #endif
 #ifndef GPFS_SUPER_MAGIC
-#define GPFS_SUPER_MAGIC  0x47504653    /* Thats GPFS in ASCII */
+#    define GPFS_SUPER_MAGIC 0x47504653 /* That's GPFS in ASCII */
 #endif
 #ifndef AUTOFS_SUPER_MAGIC
-#define AUTOFS_SUPER_MAGIC 0x0187
+#    define AUTOFS_SUPER_MAGIC 0x0187
 #endif
 #ifndef PVFS2_SUPER_MAGIC
-#define PVFS2_SUPER_MAGIC 0x20030528
+#    define PVFS2_SUPER_MAGIC 0x20030528
 #endif
 
-#define MASK2        0xffff
-#define MASK4    0xffffffff
+#define MASK2 0xffff
+#define MASK4 0xffffffff
 
 bool opal_path_nfs(char *fname, char **ret_fstype)
 {
@@ -508,7 +503,7 @@ bool opal_path_nfs(char *fname, char **ret_fstype)
     int fsrc = -1;
     int vfsrc = -1;
     int trials;
-    char * file = strdup (fname);
+    char *file = strdup(fname);
 #if defined(USE_STATFS)
     struct statfs fsbuf;
 #endif
@@ -522,16 +517,14 @@ bool opal_path_nfs(char *fname, char **ret_fstype)
     static struct fs_types_t {
         unsigned long long f_fsid;
         unsigned long long f_mask;
-        const char * f_fsname;
-    } fs_types[] = {
-        {LL_SUPER_MAGIC,                   MASK4, "lustre"},
-        {NFS_SUPER_MAGIC,                  MASK2, "nfs"},
-        {AUTOFS_SUPER_MAGIC,               MASK2, "autofs"},
-        {PAN_KERNEL_FS_CLIENT_SUPER_MAGIC, MASK4, "panfs"},
-        {GPFS_SUPER_MAGIC,                 MASK4, "gpfs"},
-        {PVFS2_SUPER_MAGIC,                MASK4, "pvfs2"}
-    };
-#define FS_TYPES_NUM (int)(sizeof (fs_types)/sizeof (fs_types[0]))
+        const char *f_fsname;
+    } fs_types[] = {{LL_SUPER_MAGIC, MASK4, "lustre"},
+                    {NFS_SUPER_MAGIC, MASK2, "nfs"},
+                    {AUTOFS_SUPER_MAGIC, MASK2, "autofs"},
+                    {PAN_KERNEL_FS_CLIENT_SUPER_MAGIC, MASK4, "panfs"},
+                    {GPFS_SUPER_MAGIC, MASK4, "gpfs"},
+                    {PVFS2_SUPER_MAGIC, MASK4, "pvfs2"}};
+#define FS_TYPES_NUM (int) (sizeof(fs_types) / sizeof(fs_types[0]))
 
     /*
      * First, get the OS-dependent struct stat(v)fs buf.  This may
@@ -555,13 +548,14 @@ bool opal_path_nfs(char *fname, char **ret_fstype)
     /* In case some error with the current filename, try the parent
        directory */
     if (-1 == fsrc && -1 == vfsrc) {
-        char * last_sep;
+        char *last_sep;
 
-        OPAL_OUTPUT_VERBOSE((10, 0, "opal_path_nfs: stat(v)fs on file:%s failed errno:%d directory:%s\n",
+        OPAL_OUTPUT_VERBOSE((10, 0,
+                             "opal_path_nfs: stat(v)fs on file:%s failed errno:%d directory:%s\n",
                              fname, errno, file));
         if (EPERM == errno) {
             free(file);
-            if ( NULL != ret_fstype ) {
+            if (NULL != ret_fstype) {
                 *ret_fstype = NULL;
             }
             return false;
@@ -569,11 +563,10 @@ bool opal_path_nfs(char *fname, char **ret_fstype)
 
         last_sep = strrchr(file, OPAL_PATH_SEP[0]);
         /* Stop the search, when we have searched past root '/' */
-        if (NULL == last_sep || (1 == strlen(last_sep) &&
-            OPAL_PATH_SEP[0] == *last_sep)) {
-            free (file);
-            if ( NULL != ret_fstype ) {
-                *ret_fstype=NULL;
+        if (NULL == last_sep || (1 == strlen(last_sep) && OPAL_PATH_SEP[0] == *last_sep)) {
+            free(file);
+            if (NULL != ret_fstype) {
+                *ret_fstype = NULL;
             }
             return false;
         }
@@ -587,15 +580,15 @@ bool opal_path_nfs(char *fname, char **ret_fstype)
 #if defined(USE_STATFS)
         /* These are uses of struct statfs */
 #    if defined(HAVE_STRUCT_STATFS_F_FSTYPENAME)
-        if (0 == fsrc &&
-            0 == strncasecmp(fs_types[i].f_fsname, fsbuf.f_fstypename,
-                             sizeof(fsbuf.f_fstypename))) {
+        if (0 == fsrc
+            && 0
+                   == strncasecmp(fs_types[i].f_fsname, fsbuf.f_fstypename,
+                                  sizeof(fsbuf.f_fstypename))) {
             goto found;
         }
 #    endif
 #    if defined(HAVE_STRUCT_STATFS_F_TYPE)
-        if (0 == fsrc &&
-            fs_types[i].f_fsid == (fsbuf.f_type & fs_types[i].f_mask)) {
+        if (0 == fsrc && fs_types[i].f_fsid == (fsbuf.f_type & fs_types[i].f_mask)) {
             goto found;
         }
 #    endif
@@ -604,31 +597,33 @@ bool opal_path_nfs(char *fname, char **ret_fstype)
 #if defined(HAVE_STATVFS)
         /* These are uses of struct statvfs */
 #    if defined(HAVE_STRUCT_STATVFS_F_BASETYPE)
-        if (0 == vfsrc &&
-            0 == strncasecmp(fs_types[i].f_fsname, vfsbuf.f_basetype,
-                             sizeof(vfsbuf.f_basetype))) {
+        if (0 == vfsrc
+            && 0
+                   == strncasecmp(fs_types[i].f_fsname, vfsbuf.f_basetype,
+                                  sizeof(vfsbuf.f_basetype))) {
             goto found;
         }
 #    endif
 #    if defined(HAVE_STRUCT_STATVFS_F_FSTYPENAME)
-        if (0 == vfsrc &&
-            0 == strncasecmp(fs_types[i].f_fsname, vfsbuf.f_fstypename,
-                             sizeof(vfsbuf.f_fstypename))) {
+        if (0 == vfsrc
+            && 0
+                   == strncasecmp(fs_types[i].f_fsname, vfsbuf.f_fstypename,
+                                  sizeof(vfsbuf.f_fstypename))) {
             goto found;
         }
 #    endif
 #endif
     }
 
-    free (file);
-    if ( NULL != ret_fstype ) {
-        *ret_fstype=NULL;
+    free(file);
+    if (NULL != ret_fstype) {
+        *ret_fstype = NULL;
     }
     return false;
 
 found:
 
-    free (file);
+    free(file);
     if (AUTOFS_SUPER_MAGIC == fs_types[i].f_fsid) {
         char *fs_type = opal_check_mtab(fname);
         int x;
@@ -638,35 +633,33 @@ bool opal_path_nfs(char *fname, char **ret_fstype)
                     continue;
                 }
                 if (0 == strcasecmp(fs_types[x].f_fsname, fs_type)) {
-                    OPAL_OUTPUT_VERBOSE((10, 0, "opal_path_nfs: file:%s on fs:%s\n", fname, fs_type));
+                    OPAL_OUTPUT_VERBOSE(
+                        (10, 0, "opal_path_nfs: file:%s on fs:%s\n", fname, fs_type));
                     free(fs_type);
-                    if ( NULL != ret_fstype ) {
+                    if (NULL != ret_fstype) {
                         *ret_fstype = strdup(fs_types[x].f_fsname);
                     }
                     return true;
                 }
             }
             free(fs_type);
-            if ( NULL != ret_fstype ) {
-                *ret_fstype=NULL;
+            if (NULL != ret_fstype) {
+                *ret_fstype = NULL;
             }
             return false;
         }
     }
 
-    OPAL_OUTPUT_VERBOSE((10, 0, "opal_path_nfs: file:%s on fs:%s\n",
-                fname, fs_types[i].f_fsname));
-    if ( NULL != ret_fstype ) {
-        *ret_fstype = strdup (fs_types[i].f_fsname);
+    OPAL_OUTPUT_VERBOSE((10, 0, "opal_path_nfs: file:%s on fs:%s\n", fname, fs_types[i].f_fsname));
+    if (NULL != ret_fstype) {
+        *ret_fstype = strdup(fs_types[i].f_fsname);
     }
     return true;
 
 #undef FS_TYPES_NUM
 }
 
-int
-opal_path_df(const char *path,
-             uint64_t *out_avail)
+int opal_path_df(const char *path, uint64_t *out_avail)
 {
     int rc = -1;
     int trials = 5;
@@ -692,18 +685,20 @@ opal_path_df(const char *path,
     } while (-1 == rc && ESTALE == err && (--trials > 0));
 
     if (-1 == rc) {
-        OPAL_OUTPUT_VERBOSE((10, 2, "opal_path_df: stat(v)fs on "
+        OPAL_OUTPUT_VERBOSE((10, 2,
+                             "opal_path_df: stat(v)fs on "
                              "path: %s failed with errno: %d (%s)\n",
                              path, err, strerror(err)));
         return OPAL_ERROR;
     }
 
     /* now set the amount of free space available on path */
-                               /* sometimes buf.f_bavail is negative */
-    *out_avail = (uint64_t)buf.f_bsize * (uint64_t)((long)buf.f_bavail < 0 ? 0 : buf.f_bavail);
+    /* sometimes buf.f_bavail is negative */
+    *out_avail = (uint64_t) buf.f_bsize * (uint64_t)((long) buf.f_bavail < 0 ? 0 : buf.f_bavail);
 
-    OPAL_OUTPUT_VERBOSE((10, 2, "opal_path_df: stat(v)fs states "
-                         "path: %s has %"PRIu64 " B of free space.",
+    OPAL_OUTPUT_VERBOSE((10, 2,
+                         "opal_path_df: stat(v)fs states "
+                         "path: %s has %" PRIu64 " B of free space.",
                          path, *out_avail));
 
     return OPAL_SUCCESS;
diff --git a/opal/util/path.h b/opal/util/path.h
index d723545159b..32e5a6edf57 100644
--- a/opal/util/path.h
+++ b/opal/util/path.h
@@ -29,7 +29,7 @@
 #include "opal/constants.h"
 
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 
 BEGIN_C_DECLS
@@ -53,8 +53,8 @@ BEGIN_C_DECLS
  *
  * The caller is responsible for freeing the returned string.
  */
-OPAL_DECLSPEC char *opal_path_find(char *fname, char **pathv, int mode,
-                                   char **envv) __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
+OPAL_DECLSPEC char *opal_path_find(char *fname, char **pathv, int mode, char **envv)
+    __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
 
 /**
  *  Locates a file with certain permissions from a list of search
@@ -74,8 +74,8 @@ OPAL_DECLSPEC char *opal_path_find(char *fname, char **pathv, int mode,
  *
  * The caller is responsible for freeing the returned string.
  */
-OPAL_DECLSPEC char *opal_path_findv(char *fname, int mode,
-                                    char **envv, char *wrkdir) __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
+OPAL_DECLSPEC char *opal_path_findv(char *fname, int mode, char **envv, char *wrkdir)
+    __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
 /**
  *  Detect if the requested path is absolute or relative.
  *
@@ -88,7 +88,7 @@ OPAL_DECLSPEC char *opal_path_findv(char *fname, int mode,
  *  with special care as an absolute path on Windows starts
  *  with [A-Za-z]: or \\ instead of the usual / on UNIX.
  */
-OPAL_DECLSPEC bool opal_path_is_absolute( const char *path );
+OPAL_DECLSPEC bool opal_path_is_absolute(const char *path);
 
 /**
  * Find the absolute path for an executable and return it.
@@ -107,10 +107,10 @@ OPAL_DECLSPEC bool opal_path_is_absolute( const char *path );
  * function will return NULL. Otherwise, an newly allocated string
  * will be returned.
  */
-OPAL_DECLSPEC char* opal_find_absolute_path( char* app_name ) __opal_attribute_warn_unused_result__;
+OPAL_DECLSPEC char *opal_find_absolute_path(char *app_name) __opal_attribute_warn_unused_result__;
 
 /**
- * Forms a complete pathname and checks it for existance and
+ * Forms a complete pathname and checks it for existence and
  * permissions
  *
  * @param fname File name
@@ -122,8 +122,8 @@ OPAL_DECLSPEC char* opal_find_absolute_path( char* app_name ) __opal_attribute_w
  *
  * The caller is responsible for freeing the returned string.
  */
-OPAL_DECLSPEC char *opal_path_access(char *fname, char *path, int mode) __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
-
+OPAL_DECLSPEC char *opal_path_access(char *fname, char *path, int mode)
+    __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
 
 /**
  * @brief Figure out, whether fname is on network file system
@@ -153,9 +153,8 @@ OPAL_DECLSPEC bool opal_path_nfs(char *fname, char **fstype) __opal_attribute_wa
  * @retval OPAL_SUCCESS  If the operation was successful
  * @retval OPAL_ERROR    otherwise
  */
-OPAL_DECLSPEC int
-opal_path_df(const char *path,
-             uint64_t *out_avail)__opal_attribute_warn_unused_result__;
+OPAL_DECLSPEC int opal_path_df(const char *path,
+                               uint64_t *out_avail) __opal_attribute_warn_unused_result__;
 
 END_C_DECLS
 #endif /* OPAL_PATH_H */
diff --git a/opal/util/printf.c b/opal/util/printf.c
index fad3d949157..da88c2b28d2 100644
--- a/opal/util/printf.c
+++ b/opal/util/printf.c
@@ -24,15 +24,14 @@
 
 #include "opal_config.h"
 
-#include "opal/util/printf.h"
 #include "opal/util/output.h"
+#include "opal/util/printf.h"
 
 #include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
-
 #ifndef HAVE_VASPRINTF
 /*
  * Make a good guess about how long a printf-style varargs formatted
@@ -42,13 +41,13 @@
  */
 static int guess_strlen(const char *fmt, va_list ap)
 {
-#if HAVE_VSNPRINTF
+#    if HAVE_VSNPRINTF
     char dummy[1];
 
     /* vsnprintf() returns the number of bytes that would have been
        copied if the provided buffer were infinite. */
     return 1 + vsnprintf(dummy, sizeof(dummy), fmt, ap);
-#else
+#    else
     char *sarg, carg;
     double darg;
     float farg;
@@ -60,16 +59,16 @@ static int guess_strlen(const char *fmt, va_list ap)
     /* Start off with a fudge factor of 128 to handle the % escapes that
        we aren't calculating here */
 
-    len = (int)strlen(fmt) + 128;
+    len = (int) strlen(fmt) + 128;
     for (i = 0; i < strlen(fmt); ++i) {
-        if ('%' == fmt[i] && i + 1 < strlen(fmt)
-            && '%' != fmt[i + 1]) {
+        if ('%' == fmt[i] && i + 1 < strlen(fmt) && '%' != fmt[i + 1]) {
             ++i;
             switch (fmt[i]) {
             case 'c':
                 carg = va_arg(ap, int);
-                len += 1;  /* let's suppose it's a printable char */
-                (void)carg;  /* prevent compiler from complaining about set but not used variables */
+                len += 1; /* let's suppose it's a printable char */
+                (void)
+                    carg; /* prevent compiler from complaining about set but not used variables */
                 break;
             case 's':
                 sarg = va_arg(ap, char *);
@@ -78,11 +77,11 @@ static int guess_strlen(const char *fmt, va_list ap)
                  * use (null) */
 
                 if (NULL != sarg) {
-                    len += (int)strlen(sarg);
+                    len += (int) strlen(sarg);
                 } else {
-#if OPAL_ENABLE_DEBUG
+#        if OPAL_ENABLE_DEBUG
                     opal_output(0, "OPAL DEBUG WARNING: Got a NULL argument to opal_vasprintf!\n");
-#endif
+#        endif
                     len += 5;
                 }
                 break;
@@ -111,7 +110,7 @@ static int guess_strlen(const char *fmt, va_list ap)
                 break;
 
             case 'f':
-                farg = (float)va_arg(ap, int);
+                farg = (float) va_arg(ap, int);
                 /* Alloc for minus sign */
                 if (farg < 0) {
                     ++len;
@@ -192,7 +191,7 @@ static int guess_strlen(const char *fmt, va_list ap)
     }
 
     return len;
-#endif
+#    endif
 }
 #endif /* #ifndef HAVE_VASPRINTF */
 
@@ -209,7 +208,6 @@ int opal_asprintf(char **ptr, const char *fmt, ...)
     return length;
 }
 
-
 int opal_vasprintf(char **ptr, const char *fmt, va_list ap)
 {
 #ifdef HAVE_VASPRINTF
@@ -228,13 +226,13 @@ int opal_vasprintf(char **ptr, const char *fmt, va_list ap)
     /* va_list might have pointer to internal state and using
        it twice is a bad idea.  So make a copy for the second
        use.  Copy order taken from Autoconf docs. */
-#if OPAL_HAVE_VA_COPY
+#    if OPAL_HAVE_VA_COPY
     va_copy(ap2, ap);
-#elif OPAL_HAVE_UNDERSCORE_VA_COPY
+#    elif OPAL_HAVE_UNDERSCORE_VA_COPY
     __va_copy(ap2, ap);
-#else
-    memcpy (&ap2, &ap, sizeof(va_list));
-#endif
+#    else
+    memcpy(&ap2, &ap, sizeof(va_list));
+#    endif
 
     /* guess the size */
     length = guess_strlen(fmt, ap);
@@ -249,12 +247,12 @@ int opal_vasprintf(char **ptr, const char *fmt, va_list ap)
 
     /* fill the buffer */
     length = vsprintf(*ptr, fmt, ap2);
-#if OPAL_HAVE_VA_COPY || OPAL_HAVE_UNDERSCORE_VA_COPY
+#    if OPAL_HAVE_VA_COPY || OPAL_HAVE_UNDERSCORE_VA_COPY
     va_end(ap2);
-#endif  /* OPAL_HAVE_VA_COPY || OPAL_HAVE_UNDERSCORE_VA_COPY */
+#    endif /* OPAL_HAVE_VA_COPY || OPAL_HAVE_UNDERSCORE_VA_COPY */
 
     /* realloc */
-    *ptr = (char*) realloc(*ptr, (size_t) length + 1);
+    *ptr = (char *) realloc(*ptr, (size_t) length + 1);
     if (NULL == *ptr) {
         errno = ENOMEM;
         return -1;
@@ -264,7 +262,6 @@ int opal_vasprintf(char **ptr, const char *fmt, va_list ap)
 #endif
 }
 
-
 int opal_snprintf(char *str, size_t size, const char *fmt, ...)
 {
     int length;
@@ -277,7 +274,6 @@ int opal_snprintf(char *str, size_t size, const char *fmt, ...)
     return length;
 }
 
-
 int opal_vsnprintf(char *str, size_t size, const char *fmt, va_list ap)
 {
     int length;
@@ -304,7 +300,6 @@ int opal_vsnprintf(char *str, size_t size, const char *fmt, va_list ap)
     return length;
 }
 
-
 #ifdef TEST
 
 int main(int argc, char *argv[])
diff --git a/opal/util/printf.h b/opal/util/printf.h
index 1b18d036762..3f7ba0054ff 100644
--- a/opal/util/printf.h
+++ b/opal/util/printf.h
@@ -51,8 +51,8 @@ BEGIN_C_DECLS
  *
  * THIS IS A PORTABILITY FEATURE: USE snprintf() in CODE.
  */
-OPAL_DECLSPEC int  opal_snprintf(char *str, size_t size, const char *fmt, ...) __opal_attribute_format__(__printf__, 3, 4);
-
+OPAL_DECLSPEC int opal_snprintf(char *str, size_t size, const char *fmt, ...)
+    __opal_attribute_format__(__printf__, 3, 4);
 
 /**
  * Writes to a string under the control of a format string that
@@ -76,7 +76,8 @@ OPAL_DECLSPEC int  opal_snprintf(char *str, size_t size, const char *fmt, ...) _
  *
  * THIS IS A PORTABILITY FEATURE: USE vsnprintf() in CODE.
  */
-OPAL_DECLSPEC int  opal_vsnprintf(char *str, size_t size, const char *fmt, va_list ap) __opal_attribute_format__(__printf__, 3, 0);
+OPAL_DECLSPEC int opal_vsnprintf(char *str, size_t size, const char *fmt, va_list ap)
+    __opal_attribute_format__(__printf__, 3, 0);
 
 /**
  * Allocates and writes to a string under the control of a format
@@ -102,8 +103,8 @@ OPAL_DECLSPEC int  opal_vsnprintf(char *str, size_t size, const char *fmt, va_li
  * such behavior.
  *
  */
-OPAL_DECLSPEC int  opal_asprintf(char **ptr, const char *fmt, ...) __opal_attribute_format__(__printf__, 2, 3);
-
+OPAL_DECLSPEC int opal_asprintf(char **ptr, const char *fmt, ...)
+    __opal_attribute_format__(__printf__, 2, 3);
 
 /**
  * Allocates and writes to a string under the control of a format
@@ -131,10 +132,9 @@ OPAL_DECLSPEC int  opal_asprintf(char **ptr, const char *fmt, ...) __opal_attrib
  * such behavior.
  *
  */
-OPAL_DECLSPEC int  opal_vasprintf(char **ptr, const char *fmt, va_list ap) __opal_attribute_format__(__printf__, 2, 0);
-
+OPAL_DECLSPEC int opal_vasprintf(char **ptr, const char *fmt, va_list ap)
+    __opal_attribute_format__(__printf__, 2, 0);
 
 END_C_DECLS
 
 #endif /* OPAL_PRINTF_H */
-
diff --git a/opal/util/proc.c b/opal/util/proc.c
index 06f35b8ef54..25c0603ac33 100644
--- a/opal/util/proc.c
+++ b/opal/util/proc.c
@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2013      The University of Tennessee and The University
+ * Copyright (c) 2013-2022 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2013      Inria.  All rights reserved.
@@ -11,6 +11,8 @@
  *                         reserved.
  * Copyright (c) 2020      Amazon.com, Inc. or its affiliates.  All Rights
  *                         reserved.
+ * Copyright (c) 2022      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -20,11 +22,11 @@
 
 #include "opal_config.h"
 
-#include "proc.h"
-#include "opal/util/proc.h"
+#include "opal/mca/pmix/pmix-internal.h"
 #include "opal/util/arch.h"
+#include "opal/util/proc.h"
 #include "opal/util/string_copy.h"
-#include "opal/mca/pmix/pmix-internal.h"
+#include "proc.h"
 
 opal_process_name_t opal_name_wildcard = {OPAL_JOBID_WILDCARD, OPAL_VPID_WILDCARD};
 opal_process_name_t opal_name_invalid = {OPAL_JOBID_INVALID, OPAL_VPID_INVALID};
@@ -37,8 +39,8 @@ opal_process_info_t opal_process_info = {
     .top_session_dir = NULL,
     .job_session_dir = NULL,
     .proc_session_dir = NULL,
-    .num_local_peers = 0,  /* there is nobody else but me */
-    .my_local_rank = 0,    /* I'm the only process around here */
+    .num_local_peers = 0, /* there is nobody else but me */
+    .my_local_rank = 0,   /* I'm the only process around here */
     .my_node_rank = 0,
     .cpuset = NULL,
     .locality = NULL,
@@ -54,19 +56,17 @@ opal_process_info_t opal_process_info = {
     .reincarnation = 0,
     .proc_is_bound = false,
     .initial_errhandler = NULL,
+    .is_singleton = false,
 };
 
-static opal_proc_t opal_local_proc = {
-    { .opal_list_next = NULL,
-      .opal_list_prev = NULL},
-    .proc_name = {OPAL_JOBID_INVALID, OPAL_VPID_INVALID},
-    .proc_arch = 0,
-    .proc_flags = 0,
-    .proc_convertor = NULL
-};
-static opal_proc_t* opal_proc_my_name = &opal_local_proc;
+static opal_proc_t opal_local_proc = {{.opal_list_next = NULL, .opal_list_prev = NULL},
+                                      .proc_name = {OPAL_JOBID_INVALID, OPAL_VPID_INVALID},
+                                      .proc_arch = 0,
+                                      .proc_flags = 0,
+                                      .proc_convertor = NULL};
+static opal_proc_t *opal_proc_my_name = &opal_local_proc;
 
-static void opal_proc_construct(opal_proc_t* proc)
+static void opal_proc_construct(opal_proc_t *proc)
 {
     proc->proc_arch = opal_local_arch;
     proc->proc_convertor = NULL;
@@ -74,33 +74,29 @@ static void opal_proc_construct(opal_proc_t* proc)
     proc->proc_name = *OPAL_NAME_INVALID;
 }
 
-static void opal_proc_destruct(opal_proc_t* proc)
+static void opal_proc_destruct(opal_proc_t *proc)
 {
-    proc->proc_flags     = 0;
-    proc->proc_name      = *OPAL_NAME_INVALID;
+    proc->proc_flags = 0;
+    proc->proc_name = *OPAL_NAME_INVALID;
     proc->proc_convertor = NULL;
 }
 
-OBJ_CLASS_INSTANCE(opal_proc_t, opal_list_item_t,
-                   opal_proc_construct, opal_proc_destruct);
+OBJ_CLASS_INSTANCE(opal_proc_t, opal_list_item_t, opal_proc_construct, opal_proc_destruct);
 
-OBJ_CLASS_INSTANCE(opal_namelist_t, opal_list_item_t,
-                   NULL, NULL);
+OBJ_CLASS_INSTANCE(opal_namelist_t, opal_list_item_t, NULL, NULL);
 
-static int
-opal_compare_opal_procs(const opal_process_name_t p1,
-                        const opal_process_name_t p2)
+static int opal_compare_opal_procs(const opal_process_name_t p1, const opal_process_name_t p2)
 {
-    if( p1.jobid < p2.jobid ) {
-        return  -1;
+    if (p1.jobid < p2.jobid) {
+        return -1;
     }
-    if( p1.jobid > p2.jobid ) {
-        return  1;
+    if (p1.jobid > p2.jobid) {
+        return 1;
     }
-    if( p1.vpid <  p2.vpid ) {
+    if (p1.vpid < p2.vpid) {
         return -1;
     }
-    if( p1.vpid >  p2.vpid ) {
+    if (p1.vpid > p2.vpid) {
         return 1;
     }
     return 0;
@@ -108,19 +104,20 @@ opal_compare_opal_procs(const opal_process_name_t p1,
 
 opal_compare_proc_fct_t opal_compare_proc = opal_compare_opal_procs;
 
-opal_proc_t* opal_proc_local_get(void)
+opal_proc_t *opal_proc_local_get(void)
 {
     return opal_proc_my_name;
 }
 
-int opal_proc_local_set(opal_proc_t* proc)
+int opal_proc_local_set(opal_proc_t *proc)
 {
-    if( proc != opal_proc_my_name ) {
-        if( NULL != proc )
+    if (proc != opal_proc_my_name) {
+        if (NULL != proc)
             OBJ_RETAIN(proc);
-        if( &opal_local_proc != opal_proc_my_name )
+        if (&opal_local_proc != opal_proc_my_name) {
             OBJ_RELEASE(opal_proc_my_name);
-        if( NULL != proc ) {
+        }
+        if (NULL != proc) {
             opal_proc_my_name = proc;
         } else {
             opal_proc_my_name = &opal_local_proc;
@@ -145,66 +142,61 @@ void opal_proc_set_name(opal_process_name_t *name)
  * before the call to opal_init, to make them point to the correct accessors based on the
  * underlying RTE.
  */
-static char*
-opal_process_name_print_should_never_be_called(const opal_process_name_t procname)
+static char *opal_process_name_print_should_never_be_called(const opal_process_name_t procname)
 {
     return "My Name is Nobody";
 }
 
-static char*
-opal_vpid_print_should_never_be_called(const opal_vpid_t unused)
+static char *opal_vpid_print_should_never_be_called(const opal_vpid_t unused)
 {
     return "My VPID";
 }
 
-static char*
-opal_jobid_print_should_never_be_called(const opal_jobid_t unused)
+static char *opal_jobid_print_should_never_be_called(const opal_jobid_t unused)
 {
     return "My JOBID";
 }
 
 static int opal_convert_string_to_process_name_should_never_be_called(opal_process_name_t *name,
-                                                                      const char* name_string)
+                                                                      const char *name_string)
 {
     return OPAL_ERR_NOT_SUPPORTED;
 }
 
-static int opal_convert_process_name_to_string_should_never_be_called(char** name_string,
-                                                                      const opal_process_name_t *name)
+static int
+opal_convert_process_name_to_string_should_never_be_called(char **name_string,
+                                                           const opal_process_name_t *name)
 {
     return OPAL_ERR_NOT_SUPPORTED;
 }
 
-static int opal_snprintf_jobid_should_never_be_called(char* name_string, size_t size, opal_jobid_t jobid)
+static int opal_snprintf_jobid_should_never_be_called(char *name_string, size_t size,
+                                                      opal_jobid_t jobid)
 {
-    (void)opal_string_copy(name_string, "My JOBID", size);
+    (void) opal_string_copy(name_string, "My JOBID", size);
     return OPAL_SUCCESS;
 }
 
-static int opal_convert_string_to_jobid_should_never_be_called(opal_jobid_t *jobid, const char *jobid_string)
+static int opal_convert_string_to_jobid_should_never_be_called(opal_jobid_t *jobid,
+                                                               const char *jobid_string)
 {
     return OPAL_ERR_NOT_SUPPORTED;
 }
 
-static struct opal_proc_t *opal_proc_for_name_should_never_be_called (opal_process_name_t name)
+static struct opal_proc_t *opal_proc_for_name_should_never_be_called(opal_process_name_t name)
 {
     return NULL;
 }
 
-char* (*opal_process_name_print)(const opal_process_name_t) = opal_process_name_print_should_never_be_called;
-char* (*opal_vpid_print)(const opal_vpid_t) = opal_vpid_print_should_never_be_called;
-char* (*opal_jobid_print)(const opal_jobid_t) = opal_jobid_print_should_never_be_called;
-int (*opal_convert_string_to_process_name)(opal_process_name_t *name, const char* name_string) = opal_convert_string_to_process_name_should_never_be_called;
-int (*opal_convert_process_name_to_string)(char** name_string, const opal_process_name_t *name) = opal_convert_process_name_to_string_should_never_be_called;
-int (*opal_snprintf_jobid)(char* name_string, size_t size, opal_jobid_t jobid) = opal_snprintf_jobid_should_never_be_called;
-int (*opal_convert_string_to_jobid)(opal_jobid_t *jobid, const char *jobid_string) = opal_convert_string_to_jobid_should_never_be_called;
-struct opal_proc_t *(*opal_proc_for_name) (const opal_process_name_t name) = opal_proc_for_name_should_never_be_called;
-
-char* opal_get_proc_hostname(const opal_proc_t *proc)
+/**
+ * Return the hostname of the target proc. The default implementation
+ * only support the current proc. Once a RTE is initialized it must replace
+ * the default function with one that can handle multiple, potentially
+ * distributed, processes.
+ */
+static char*
+opal_get_proc_hostname_local_only(const opal_proc_t *proc)
 {
-    int ret;
-    char *hostname;
-
     /* if the proc is NULL, then we can't know */
     if (NULL == proc) {
         return strdup("unknown");
@@ -215,14 +207,23 @@ char* opal_get_proc_hostname(const opal_proc_t *proc)
     if (proc == opal_proc_my_name) {
         return strdup(opal_process_info.nodename);
     }
+    return strdup("unknown");
+}
 
-    /* if we don't already have it, then try to get it */
-    OPAL_MODEX_RECV_VALUE_OPTIONAL(ret, PMIX_HOSTNAME, &proc->proc_name,
-                                   (char**)&hostname, PMIX_STRING);
-    if (OPAL_SUCCESS != ret) {
-        return strdup("unknown");  // return something so the caller doesn't segfault
-    }
+char *(*opal_process_name_print)(const opal_process_name_t)
+    = opal_process_name_print_should_never_be_called;
+char *(*opal_vpid_print)(const opal_vpid_t) = opal_vpid_print_should_never_be_called;
+char *(*opal_jobid_print)(const opal_jobid_t) = opal_jobid_print_should_never_be_called;
+int (*opal_convert_string_to_process_name)(opal_process_name_t *name, const char *name_string)
+    = opal_convert_string_to_process_name_should_never_be_called;
+int (*opal_convert_process_name_to_string)(char **name_string, const opal_process_name_t *name)
+    = opal_convert_process_name_to_string_should_never_be_called;
+int (*opal_snprintf_jobid)(char *name_string, size_t size, opal_jobid_t jobid)
+    = opal_snprintf_jobid_should_never_be_called;
+int (*opal_convert_string_to_jobid)(opal_jobid_t *jobid, const char *jobid_string)
+    = opal_convert_string_to_jobid_should_never_be_called;
+struct opal_proc_t *(*opal_proc_for_name)(const opal_process_name_t name)
+    = opal_proc_for_name_should_never_be_called;
+char *(*opal_get_proc_hostname)(const opal_proc_t *proc)
+    = opal_get_proc_hostname_local_only;
 
-    /* user is not allowed to release the data */
-    return hostname;
-}
diff --git a/opal/util/proc.h b/opal/util/proc.h
index 9d871edded1..df8c60dddcb 100644
--- a/opal/util/proc.h
+++ b/opal/util/proc.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013      The University of Tennessee and The University
+ * Copyright (c) 2013-2022 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2013      Inria.  All rights reserved.
@@ -10,6 +10,8 @@
  * Copyright (c) 2020      Amazon.com, Inc. or its affiliates.  All Rights
  *                         reserved.
  * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2022      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -27,7 +29,7 @@
 #include "opal/types.h"
 
 #if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
-#include <arpa/inet.h>
+#    include <arpa/inet.h>
 #endif
 
 /**
@@ -39,39 +41,40 @@
  * only be used via the accessors defined below.
  */
 #define OPAL_JOBID_T        OPAL_UINT32
-#define OPAL_JOBID_MAX      UINT32_MAX-2
+#define OPAL_JOBID_MAX      UINT32_MAX - 2
 #define OPAL_JOBID_MIN      0
 #define OPAL_JOBID_INVALID  (OPAL_JOBID_MAX + 2)
 #define OPAL_JOBID_WILDCARD (OPAL_JOBID_MAX + 1)
 
-#define OPAL_VPID_T         OPAL_UINT32
-#define OPAL_VPID_MAX       UINT32_MAX-2
-#define OPAL_VPID_MIN       0
-#define OPAL_VPID_INVALID   (OPAL_VPID_MAX + 2)
-#define OPAL_VPID_WILDCARD  (OPAL_VPID_MAX + 1)
+#define OPAL_VPID_T        OPAL_UINT32
+#define OPAL_VPID_MAX      UINT32_MAX - 2
+#define OPAL_VPID_MIN      0
+#define OPAL_VPID_INVALID  (OPAL_VPID_MAX + 2)
+#define OPAL_VPID_WILDCARD (OPAL_VPID_MAX + 1)
 
-#define OPAL_PROC_MY_NAME           (opal_proc_local_get()->proc_name)
-#define OPAL_PROC_MY_HOSTNAME       (opal_process_info.nodename)
+#define OPAL_PROC_MY_NAME     (opal_proc_local_get()->proc_name)
+#define OPAL_PROC_MY_HOSTNAME (opal_process_info.nodename)
 
-#define OPAL_NAME_WILDCARD      (&opal_name_wildcard)
+#define OPAL_NAME_WILDCARD (&opal_name_wildcard)
 OPAL_DECLSPEC extern opal_process_name_t opal_name_wildcard;
-#define OPAL_NAME_INVALID       (&opal_name_invalid)
+#define OPAL_NAME_INVALID (&opal_name_invalid)
 OPAL_DECLSPEC extern opal_process_name_t opal_name_invalid;
 
-
-#define OPAL_NAME_ARGS(n) \
-    (unsigned long) ((NULL == n) ? (unsigned long)OPAL_JOBID_INVALID : (unsigned long)(n)->jobid), \
-    (unsigned long) ((NULL == n) ? (unsigned long)OPAL_VPID_INVALID : (unsigned long)(n)->vpid) \
+#define OPAL_NAME_ARGS(n)                                                \
+    (unsigned long) ((NULL == n) ? (unsigned long) OPAL_JOBID_INVALID    \
+                                 : (unsigned long) (n)->jobid),          \
+        (unsigned long) ((NULL == n) ? (unsigned long) OPAL_VPID_INVALID \
+                                     : (unsigned long) (n)->vpid)
 
 #if OPAL_ENABLE_HETEROGENEOUS_SUPPORT && !defined(WORDS_BIGENDIAN)
-#define OPAL_PROCESS_NAME_NTOH(guid) opal_process_name_ntoh_intr(&(guid))
+#    define OPAL_PROCESS_NAME_NTOH(guid) opal_process_name_ntoh_intr(&(guid))
 static inline __opal_attribute_always_inline__ void
 opal_process_name_ntoh_intr(opal_process_name_t *name)
 {
     name->jobid = ntohl(name->jobid);
     name->vpid = ntohl(name->vpid);
 }
-#define OPAL_PROCESS_NAME_HTON(guid) opal_process_name_hton_intr(&(guid))
+#    define OPAL_PROCESS_NAME_HTON(guid) opal_process_name_hton_intr(&(guid))
 static inline __opal_attribute_always_inline__ void
 opal_process_name_hton_intr(opal_process_name_t *name)
 {
@@ -79,21 +82,21 @@ opal_process_name_hton_intr(opal_process_name_t *name)
     name->vpid = htonl(name->vpid);
 }
 #else
-#define OPAL_PROCESS_NAME_NTOH(guid)
-#define OPAL_PROCESS_NAME_HTON(guid)
+#    define OPAL_PROCESS_NAME_NTOH(guid)
+#    define OPAL_PROCESS_NAME_HTON(guid)
 #endif
 
 typedef struct opal_proc_t {
     /** allow proc to be placed on a list */
-    opal_list_item_t                super;
+    opal_list_item_t super;
     /** this process' name */
-    opal_process_name_t             proc_name;
+    opal_process_name_t proc_name;
     /** architecture of this process */
-    uint32_t                        proc_arch;
+    uint32_t proc_arch;
     /** flags for this proc */
-    opal_hwloc_locality_t           proc_flags;
+    opal_hwloc_locality_t proc_flags;
     /** Base convertor for the proc described by this process */
-    struct opal_convertor_t*        proc_convertor;
+    struct opal_convertor_t *proc_convertor;
 } opal_proc_t;
 OBJ_CLASS_DECLARATION(opal_proc_t);
 
@@ -106,14 +109,15 @@ OBJ_CLASS_DECLARATION(opal_namelist_t);
 typedef struct opal_process_info_t {
     opal_process_name_t my_name;
     pmix_proc_t myprocid;
-    bool nativelaunch;                  /**< launched by mpirun */
-    char *nodename;                     /**< string name for this node */
-    char *top_session_dir;              /**< Top-level session directory */
-    char *job_session_dir;              /**< Session directory for job */
-    char *proc_session_dir;             /**< Session directory for the process */
-    uint32_t num_local_peers;           /**< number of procs from my job that share my node with me */
-    uint16_t my_local_rank;             /**< local rank on this node within my job */
+    bool nativelaunch;        /**< launched by mpirun */
+    char *nodename;           /**< string name for this node */
+    char *top_session_dir;    /**< Top-level session directory */
+    char *job_session_dir;    /**< Session directory for job */
+    char *proc_session_dir;   /**< Session directory for the process */
+    uint32_t num_local_peers; /**< number of procs from my job that share my node with me */
+    uint16_t my_local_rank;   /**< local rank on this node within my job */
     uint16_t my_node_rank;
+    uint16_t my_numa_rank;              /**< rank on this processes NUMA node. A value of UINT16_MAX indicates unavailable numa_rank */
     char *cpuset;                       /**< String-representation of bitmap where we are bound */
     char *locality;                     /**< String-representation of process locality */
     pid_t pid;
@@ -128,11 +132,12 @@ typedef struct opal_process_info_t {
     uint32_t reincarnation;
     bool proc_is_bound;
     char *initial_errhandler;
+    bool is_singleton;         /**<note this value can transition from false to true in some cases */
 } opal_process_info_t;
 OPAL_DECLSPEC extern opal_process_info_t opal_process_info;
 
-OPAL_DECLSPEC extern opal_proc_t* opal_proc_local_get(void);
-OPAL_DECLSPEC extern int opal_proc_local_set(opal_proc_t* proc);
+OPAL_DECLSPEC extern opal_proc_t *opal_proc_local_get(void);
+OPAL_DECLSPEC extern int opal_proc_local_set(opal_proc_t *proc);
 OPAL_DECLSPEC extern void opal_proc_set_name(opal_process_name_t *name);
 
 /**
@@ -144,31 +149,32 @@ typedef int (*opal_compare_proc_fct_t)(const opal_process_name_t, const opal_pro
 OPAL_DECLSPEC extern opal_compare_proc_fct_t opal_compare_proc;
 
 /* Provide print functions that will be overwritten by the RTE layer */
-OPAL_DECLSPEC extern char* (*opal_process_name_print)(const opal_process_name_t);
+OPAL_DECLSPEC extern char *(*opal_process_name_print)(const opal_process_name_t);
 OPAL_DECLSPEC extern int (*opal_convert_string_to_process_name)(opal_process_name_t *name,
-                                                                const char* name_string);
-OPAL_DECLSPEC extern int (*opal_convert_process_name_to_string)(char** name_string,
+                                                                const char *name_string);
+OPAL_DECLSPEC extern int (*opal_convert_process_name_to_string)(char **name_string,
                                                                 const opal_process_name_t *name);
-OPAL_DECLSPEC extern char* (*opal_vpid_print)(const opal_vpid_t);
-OPAL_DECLSPEC extern char* (*opal_jobid_print)(const opal_jobid_t);
-OPAL_DECLSPEC extern int (*opal_snprintf_jobid)(char* name_string, size_t size, opal_jobid_t jobid);
-OPAL_DECLSPEC extern int (*opal_convert_string_to_jobid)(opal_jobid_t *jobid, const char *jobid_string);
+OPAL_DECLSPEC extern char *(*opal_vpid_print)(const opal_vpid_t);
+OPAL_DECLSPEC extern char *(*opal_jobid_print)(const opal_jobid_t);
+OPAL_DECLSPEC extern int (*opal_snprintf_jobid)(char *name_string, size_t size, opal_jobid_t jobid);
+OPAL_DECLSPEC extern int (*opal_convert_string_to_jobid)(opal_jobid_t *jobid,
+                                                         const char *jobid_string);
 
 /**
  * Lookup an opal_proc_t by name
  *
  * @param name (IN) name to lookup
  */
-OPAL_DECLSPEC extern struct opal_proc_t *(*opal_proc_for_name) (const opal_process_name_t name);
+OPAL_DECLSPEC extern struct opal_proc_t *(*opal_proc_for_name)(const opal_process_name_t name);
 
-#define OPAL_NAME_PRINT(OPAL_PN)    opal_process_name_print(OPAL_PN)
-#define OPAL_JOBID_PRINT(OPAL_PN)   opal_jobid_print(OPAL_PN)
-#define OPAL_VPID_PRINT(OPAL_PN)    opal_vpid_print(OPAL_PN)
+#define OPAL_NAME_PRINT(OPAL_PN)  opal_process_name_print(OPAL_PN)
+#define OPAL_JOBID_PRINT(OPAL_PN) opal_jobid_print(OPAL_PN)
+#define OPAL_VPID_PRINT(OPAL_PN)  opal_vpid_print(OPAL_PN)
 
 /* provide a safe way to retrieve the hostname of a proc, including
  * our own. This is to be used by all BTLs so we don't retrieve hostnames
  * unless needed. The returned value MUST NOT be free'd as it is
  * owned by the proc_t */
-OPAL_DECLSPEC char* opal_get_proc_hostname(const opal_proc_t *proc);
+OPAL_DECLSPEC extern char *(*opal_get_proc_hostname)(const opal_proc_t *proc);
 
-#endif  /* OPAL_PROC_H */
+#endif /* OPAL_PROC_H */
diff --git a/opal/util/qsort.c b/opal/util/qsort.c
index 42141f44ba2..f018f6905b9 100644
--- a/opal/util/qsort.c
+++ b/opal/util/qsort.c
@@ -35,143 +35,138 @@
 
 #if OPAL_HAVE_BROKEN_QSORT
 
-#include <stdlib.h>
+#    include <stdlib.h>
 
-#include "opal/util/minmix.h"
-#include "opal/util/qsort.h"
+#    include "opal/util/minmix.h"
+#    include "opal/util/qsort.h"
 
-typedef int		 cmp_t(const void *, const void *);
-static inline char	*med3(char *, char *, char *, cmp_t *, void *);
-static inline void	 swapfunc(char *, char *, int, int);
+typedef int cmp_t(const void *, const void *);
+static inline char *med3(char *, char *, char *, cmp_t *, void *);
+static inline void swapfunc(char *, char *, int, int);
 
 /*
  * Qsort routine from Bentley & McIlroy's "Engineering a Sort Function".
  */
-#define swapcode(TYPE, parmi, parmj, n) { 		\
-	long i = (n) / sizeof (TYPE); 			\
-	TYPE *pi = (TYPE *) (parmi); 		\
-	TYPE *pj = (TYPE *) (parmj); 		\
-	do { 						\
-		TYPE	t = *pi;		\
-		*pi++ = *pj;				\
-		*pj++ = t;				\
-        } while (--i > 0);				\
-}
-
-#define SWAPINIT(a, es) swaptype = ((char *)a - (char *)0) % sizeof(long) || \
-	es % sizeof(long) ? 2 : es == sizeof(long)? 0 : 1;
-
-static inline void
-swapfunc(a, b, n, swaptype)
-	char *a, *b;
-	int n, swaptype;
+#    define swapcode(TYPE, parmi, parmj, n) \
+        {                                   \
+            long i = (n) / sizeof(TYPE);    \
+            TYPE *pi = (TYPE *) (parmi);    \
+            TYPE *pj = (TYPE *) (parmj);    \
+            do {                            \
+                TYPE t = *pi;               \
+                *pi++ = *pj;                \
+                *pj++ = t;                  \
+            } while (--i > 0);              \
+        }
+
+#    define SWAPINIT(a, es)                                                          \
+        swaptype = ((char *) a - (char *) 0) % sizeof(long) || es % sizeof(long) ? 2 \
+                   : es == sizeof(long)                                          ? 0 \
+                                                                                 : 1;
+
+static inline void swapfunc(a, b, n, swaptype) char *a, *b;
+int n, swaptype;
 {
-	if(swaptype <= 1)
-		swapcode(long, a, b, n)
-	else
-		swapcode(char, a, b, n)
+    if (swaptype <= 1)
+        swapcode(long, a, b, n) else swapcode(char, a, b, n)
 }
 
-#define swap(a, b)					\
-	if (swaptype == 0) {				\
-		long t = *(long *)(a);			\
-		*(long *)(a) = *(long *)(b);		\
-		*(long *)(b) = t;			\
-	} else						\
-		swapfunc(a, b, es, swaptype)
+#    define swap(a, b)                     \
+        if (swaptype == 0) {               \
+            long t = *(long *) (a);        \
+            *(long *) (a) = *(long *) (b); \
+            *(long *) (b) = t;             \
+        } else                             \
+            swapfunc(a, b, es, swaptype)
 
-#define vecswap(a, b, n) 	if ((n) > 0) swapfunc(a, b, n, swaptype)
+#    define vecswap(a, b, n) \
+        if ((n) > 0)         \
+        swapfunc(a, b, n, swaptype)
 
-#define	CMP(t, x, y) (cmp((x), (y)))
+#    define CMP(t, x, y) (cmp((x), (y)))
 
-static inline char *
-med3(char *a, char *b, char *c, cmp_t *cmp, void *thunk)
+static inline char *med3(char *a, char *b, char *c, cmp_t *cmp, void *thunk)
 {
-	return CMP(thunk, a, b) < 0 ?
-	       (CMP(thunk, b, c) < 0 ? b : (CMP(thunk, a, c) < 0 ? c : a ))
-              :(CMP(thunk, b, c) > 0 ? b : (CMP(thunk, a, c) < 0 ? a : c ));
+    return CMP(thunk, a, b) < 0 ? (CMP(thunk, b, c) < 0 ? b : (CMP(thunk, a, c) < 0 ? c : a))
+                                : (CMP(thunk, b, c) > 0 ? b : (CMP(thunk, a, c) < 0 ? a : c));
 }
 
-#define thunk NULL
-void
-opal_qsort(void *a, size_t n, size_t es, cmp_t *cmp)
+#    define thunk NULL
+void opal_qsort(void *a, size_t n, size_t es, cmp_t *cmp)
 {
-	char *pa, *pb, *pc, *pd, *pl, *pm, *pn;
-	int d, r, swaptype, swap_cnt;
-
-loop:	SWAPINIT(a, es);
-	swap_cnt = 0;
-	if (n < 7) {
-		for (pm = (char *)a + es; pm < (char *)a + n * es; pm += es)
-			for (pl = pm;
-			     pl > (char *)a && CMP(thunk, pl - es, pl) > 0;
-			     pl -= es)
-				swap(pl, pl - es);
-		return;
-	}
-	pm = (char *)a + (n / 2) * es;
-	if (n > 7) {
-		pl = a;
-		pn = (char *)a + (n - 1) * es;
-		if (n > 40) {
-			d = (n / 8) * es;
-			pl = med3(pl, pl + d, pl + 2 * d, cmp, thunk);
-			pm = med3(pm - d, pm, pm + d, cmp, thunk);
-			pn = med3(pn - 2 * d, pn - d, pn, cmp, thunk);
-		}
-		pm = med3(pl, pm, pn, cmp, thunk);
-	}
-	swap(a, pm);
-	pa = pb = (char *)a + es;
-
-	pc = pd = (char *)a + (n - 1) * es;
-	for (;;) {
-		while (pb <= pc && (r = CMP(thunk, pb, a)) <= 0) {
-			if (r == 0) {
-				swap_cnt = 1;
-				swap(pa, pb);
-				pa += es;
-			}
-			pb += es;
-		}
-		while (pb <= pc && (r = CMP(thunk, pc, a)) >= 0) {
-			if (r == 0) {
-				swap_cnt = 1;
-				swap(pc, pd);
-				pd -= es;
-			}
-			pc -= es;
-		}
-		if (pb > pc)
-			break;
-		swap(pb, pc);
-		swap_cnt = 1;
-		pb += es;
-		pc -= es;
-	}
-	if (swap_cnt == 0) {  /* Switch to insertion sort */
-		for (pm = (char *)a + es; pm < (char *)a + n * es; pm += es)
-			for (pl = pm;
-			     pl > (char *)a && CMP(thunk, pl - es, pl) > 0;
-			     pl -= es)
-				swap(pl, pl - es);
-		return;
-	}
-
-	pn = (char *)a + n * es;
-	r = (int) opal_min(pa - (char *)a, pb - pa);
-	vecswap(a, pb - r, r);
-	r = (int) (opal_min((char*) (pd - pc), (char*) (pn - pd - es)));
-	vecswap(pb, pn - r, r);
-	if ((size_t) (r = pb - pa) > es)
-		opal_qsort(a, r / es, es, cmp);
-	if ((size_t) (r = pd - pc) > es) {
-		/* Iterate rather than recurse to save stack space */
-		a = pn - r;
-		n = r / es;
-		goto loop;
-	}
-/*		qsort(pn - r, r / es, es, cmp);*/
+    char *pa, *pb, *pc, *pd, *pl, *pm, *pn;
+    int d, r, swaptype, swap_cnt;
+
+loop:
+    SWAPINIT(a, es);
+    swap_cnt = 0;
+    if (n < 7) {
+        for (pm = (char *) a + es; pm < (char *) a + n * es; pm += es)
+            for (pl = pm; pl > (char *) a && CMP(thunk, pl - es, pl) > 0; pl -= es)
+                swap(pl, pl - es);
+        return;
+    }
+    pm = (char *) a + (n / 2) * es;
+    if (n > 7) {
+        pl = a;
+        pn = (char *) a + (n - 1) * es;
+        if (n > 40) {
+            d = (n / 8) * es;
+            pl = med3(pl, pl + d, pl + 2 * d, cmp, thunk);
+            pm = med3(pm - d, pm, pm + d, cmp, thunk);
+            pn = med3(pn - 2 * d, pn - d, pn, cmp, thunk);
+        }
+        pm = med3(pl, pm, pn, cmp, thunk);
+    }
+    swap(a, pm);
+    pa = pb = (char *) a + es;
+
+    pc = pd = (char *) a + (n - 1) * es;
+    for (;;) {
+        while (pb <= pc && (r = CMP(thunk, pb, a)) <= 0) {
+            if (r == 0) {
+                swap_cnt = 1;
+                swap(pa, pb);
+                pa += es;
+            }
+            pb += es;
+        }
+        while (pb <= pc && (r = CMP(thunk, pc, a)) >= 0) {
+            if (r == 0) {
+                swap_cnt = 1;
+                swap(pc, pd);
+                pd -= es;
+            }
+            pc -= es;
+        }
+        if (pb > pc)
+            break;
+        swap(pb, pc);
+        swap_cnt = 1;
+        pb += es;
+        pc -= es;
+    }
+    if (swap_cnt == 0) { /* Switch to insertion sort */
+        for (pm = (char *) a + es; pm < (char *) a + n * es; pm += es)
+            for (pl = pm; pl > (char *) a && CMP(thunk, pl - es, pl) > 0; pl -= es)
+                swap(pl, pl - es);
+        return;
+    }
+
+    pn = (char *) a + n * es;
+    r = (int) opal_min(pa - (char *) a, pb - pa);
+    vecswap(a, pb - r, r);
+    r = (int) (opal_min((char *) (pd - pc), (char *) (pn - pd - es)));
+    vecswap(pb, pn - r, r);
+    if ((size_t)(r = pb - pa) > es)
+        opal_qsort(a, r / es, es, cmp);
+    if ((size_t)(r = pd - pc) > es) {
+        /* Iterate rather than recurse to save stack space */
+        a = pn - r;
+        n = r / es;
+        goto loop;
+    }
+    /*		qsort(pn - r, r / es, es, cmp);*/
 }
 
 #endif /* OPAL_HAVE_BROKEN_QSORT */
diff --git a/opal/util/qsort.h b/opal/util/qsort.h
index cd6b77c0813..013707e5c94 100644
--- a/opal/util/qsort.h
+++ b/opal/util/qsort.h
@@ -21,18 +21,18 @@
 
 #if OPAL_HAVE_BROKEN_QSORT
 
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h> /* for size_t */
-#endif
+#    ifdef HAVE_SYS_TYPES_H
+#        include <sys/types.h> /* for size_t */
+#    endif
 
 BEGIN_C_DECLS
 
-void opal_qsort(void *a, size_t n, size_t es, int (*cmp)(const void *, const void*));
+void opal_qsort(void *a, size_t n, size_t es, int (*cmp)(const void *, const void *));
 
 END_C_DECLS
 
 #else
-#error "Don't include opal/qsort/qsort.h directly"
+#    error "Don't include opal/qsort/qsort.h directly"
 #endif /* OPAL_HAVE_BROKEN_QSORT */
 
 #endif
diff --git a/opal/util/sha256.c b/opal/util/sha256.c
new file mode 100644
index 00000000000..fb9bbd87b57
--- /dev/null
+++ b/opal/util/sha256.c
@@ -0,0 +1,170 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2022      Triad National Security, LLC. All rights
+ *                         reserved.
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+
+/*********************************************************************
+* Filename:   sha256.c
+* Author:     Brad Conte (brad AT bradconte.com)
+* Copyright:
+* Disclaimer: This code is presented "as is" without any guarantees.
+* Details:    Implementation of the SHA-256 hashing algorithm.
+              SHA-256 is one of the three algorithms in the SHA2
+              specification. The others, SHA-384 and SHA-512, are not
+              offered in this implementation.
+              Algorithm specification can be found here:
+               * http://csrc.nist.gov/publications/fips/fips180-2/fips180-2withchangenotice.pdf
+              This implementation uses little endian byte order.
+          See https://github.com/B-Con/crypto-algorithms
+*********************************************************************/
+
+/*************************** HEADER FILES ***************************/
+#include <stdlib.h>
+#include <memory.h>
+#include "sha256.h"
+
+/****************************** MACROS ******************************/
+#define ROTLEFT(a,b) (((a) << (b)) | ((a) >> (32-(b))))
+#define ROTRIGHT(a,b) (((a) >> (b)) | ((a) << (32-(b))))
+
+#define CH(x,y,z) (((x) & (y)) ^ (~(x) & (z)))
+#define MAJ(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z)))
+#define EP0(x) (ROTRIGHT(x,2) ^ ROTRIGHT(x,13) ^ ROTRIGHT(x,22))
+#define EP1(x) (ROTRIGHT(x,6) ^ ROTRIGHT(x,11) ^ ROTRIGHT(x,25))
+#define SIG0(x) (ROTRIGHT(x,7) ^ ROTRIGHT(x,18) ^ ((x) >> 3))
+#define SIG1(x) (ROTRIGHT(x,17) ^ ROTRIGHT(x,19) ^ ((x) >> 10))
+
+/**************************** VARIABLES *****************************/
+static const WORD k[64] = {
+    0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5,
+    0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174,
+    0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da,
+    0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967,
+    0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85,
+    0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,0xd192e819,0xd6990624,0xf40e3585,0x106aa070,
+    0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3,
+    0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
+};
+
+/*********************** FUNCTION DEFINITIONS ***********************/
+static void sha256_transform(opal_sha256_ctx *ctx, const BYTE data[])
+{
+    WORD a, b, c, d, e, f, g, h, i, j, t1, t2, m[64];
+
+    for (i = 0, j = 0; i < 16; ++i, j += 4)
+        m[i] = (data[j] << 24) | (data[j + 1] << 16) | (data[j + 2] << 8) | (data[j + 3]);
+    for ( ; i < 64; ++i)
+        m[i] = SIG1(m[i - 2]) + m[i - 7] + SIG0(m[i - 15]) + m[i - 16];
+
+    a = ctx->state[0];
+    b = ctx->state[1];
+    c = ctx->state[2];
+    d = ctx->state[3];
+    e = ctx->state[4];
+    f = ctx->state[5];
+    g = ctx->state[6];
+    h = ctx->state[7];
+
+    for (i = 0; i < 64; ++i) {
+        t1 = h + EP1(e) + CH(e,f,g) + k[i] + m[i];
+        t2 = EP0(a) + MAJ(a,b,c);
+        h = g;
+        g = f;
+        f = e;
+        e = d + t1;
+        d = c;
+        c = b;
+        b = a;
+        a = t1 + t2;
+    }
+
+    ctx->state[0] += a;
+    ctx->state[1] += b;
+    ctx->state[2] += c;
+    ctx->state[3] += d;
+    ctx->state[4] += e;
+    ctx->state[5] += f;
+    ctx->state[6] += g;
+    ctx->state[7] += h;
+}
+
+void opal_sha256_init(opal_sha256_ctx *ctx)
+{
+    ctx->datalen = 0;
+    ctx->bitlen = 0;
+    ctx->state[0] = 0x6a09e667;
+    ctx->state[1] = 0xbb67ae85;
+    ctx->state[2] = 0x3c6ef372;
+    ctx->state[3] = 0xa54ff53a;
+    ctx->state[4] = 0x510e527f;
+    ctx->state[5] = 0x9b05688c;
+    ctx->state[6] = 0x1f83d9ab;
+    ctx->state[7] = 0x5be0cd19;
+}
+
+void opal_sha256_update(opal_sha256_ctx *ctx, const BYTE data[], size_t len)
+{
+    WORD i;
+
+    for (i = 0; i < len; ++i) {
+        ctx->data[ctx->datalen] = data[i];
+        ctx->datalen++;
+        if (ctx->datalen == 64) {
+            sha256_transform(ctx, ctx->data);
+            ctx->bitlen += 512;
+            ctx->datalen = 0;
+        }
+    }
+}
+
+void opal_sha256_final(opal_sha256_ctx *ctx, BYTE hash[])
+{
+    WORD i;
+
+    i = ctx->datalen;
+
+    // Pad whatever data is left in the buffer.
+    if (ctx->datalen < 56) {
+        ctx->data[i++] = 0x80;
+        while (i < 56)
+            ctx->data[i++] = 0x00;
+    }
+    else {
+        ctx->data[i++] = 0x80;
+        while (i < 64)
+            ctx->data[i++] = 0x00;
+        sha256_transform(ctx, ctx->data);
+        memset(ctx->data, 0, 56);
+    }
+
+    // Append to the padding the total message's length in bits and transform.
+    ctx->bitlen += ctx->datalen * 8;
+    ctx->data[63] = ctx->bitlen;
+    ctx->data[62] = ctx->bitlen >> 8;
+    ctx->data[61] = ctx->bitlen >> 16;
+    ctx->data[60] = ctx->bitlen >> 24;
+    ctx->data[59] = ctx->bitlen >> 32;
+    ctx->data[58] = ctx->bitlen >> 40;
+    ctx->data[57] = ctx->bitlen >> 48;
+    ctx->data[56] = ctx->bitlen >> 56;
+    sha256_transform(ctx, ctx->data);
+
+    // Since this implementation uses little endian byte ordering and SHA uses big endian,
+    // reverse all the bytes when copying the final state to the output hash.
+    for (i = 0; i < 4; ++i) {
+        hash[i]      = (ctx->state[0] >> (24 - i * 8)) & 0x000000ff;
+        hash[i + 4]  = (ctx->state[1] >> (24 - i * 8)) & 0x000000ff;
+        hash[i + 8]  = (ctx->state[2] >> (24 - i * 8)) & 0x000000ff;
+        hash[i + 12] = (ctx->state[3] >> (24 - i * 8)) & 0x000000ff;
+        hash[i + 16] = (ctx->state[4] >> (24 - i * 8)) & 0x000000ff;
+        hash[i + 20] = (ctx->state[5] >> (24 - i * 8)) & 0x000000ff;
+        hash[i + 24] = (ctx->state[6] >> (24 - i * 8)) & 0x000000ff;
+        hash[i + 28] = (ctx->state[7] >> (24 - i * 8)) & 0x000000ff;
+    }
+}
diff --git a/opal/util/sha256.h b/opal/util/sha256.h
new file mode 100644
index 00000000000..58a6e4a18e4
--- /dev/null
+++ b/opal/util/sha256.h
@@ -0,0 +1,45 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2022      Triad National Security, LLC. All rights
+ *                         reserved.
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/*********************************************************************
+* Filename:   sha256.h
+* Author:     Brad Conte (brad AT bradconte.com)
+* Copyright:
+* Disclaimer: This code is presented "as is" without any guarantees.
+* Details:    Defines the API for the corresponding SHA1 implementation.
+* Notes:      see https://github.com/B-Con/crypto-algorithms
+*********************************************************************/
+
+#ifndef OPAL_SHA256_H
+#define OPAL_SHA256_H
+
+/*************************** HEADER FILES ***************************/
+#include <stddef.h>
+
+/****************************** MACROS ******************************/
+#define OPAL_SHA256_BLOCK_SIZE 32            // SHA256 outputs a 32 byte digest
+
+/**************************** DATA TYPES ****************************/
+typedef unsigned char BYTE;             // 8-bit byte
+typedef unsigned int  WORD;             // 32-bit word, change to "long" for 16-bit machines
+
+typedef struct {
+    BYTE data[64];
+    WORD datalen;
+    unsigned long long bitlen;
+    WORD state[8];
+} opal_sha256_ctx;
+
+/*********************** FUNCTION DECLARATIONS **********************/
+void opal_sha256_init(opal_sha256_ctx *ctx);
+void opal_sha256_update(opal_sha256_ctx *ctx, const BYTE data[], size_t len);
+void opal_sha256_final(opal_sha256_ctx *ctx, BYTE hash[]);
+
+#endif   // OPAL_SHA256_H
diff --git a/opal/util/show_help.c b/opal/util/show_help.c
index d84f42d16b8..a4ab63ca614 100644
--- a/opal/util/show_help.c
+++ b/opal/util/show_help.c
@@ -16,6 +16,7 @@
  * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * Copyright (c) 2018      Triad National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2025      Jeffrey M. Squyres.  All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -25,59 +26,65 @@
 
 #include "opal_config.h"
 
+#include <errno.h>
+#include <locale.h>
 #include <stdio.h>
 #include <string.h>
-#include <locale.h>
-#include <errno.h>
 
-#include "opal/runtime/opal.h"
+#include "opal/constants.h"
 #include "opal/mca/installdirs/installdirs.h"
-#include "opal/util/show_help.h"
-#include "opal/util/show_help_lex.h"
-#include "opal/util/printf.h"
+#include "opal/runtime/opal.h"
 #include "opal/util/argv.h"
 #include "opal/util/os_path.h"
 #include "opal/util/output.h"
-#include "opal/constants.h"
-
+#include "opal/util/printf.h"
+#include "opal/util/show_help.h"
 
 /*
  * Private variables
  */
-static const char *default_filename = "help-messages";
-static const char *dash_line = "--------------------------------------------------------------------------\n";
+static const char *dash_line
+    = "--------------------------------------------------------------------------\n";
 static int output_stream = -1;
 static char **search_dirs = NULL;
+static bool opal_help_want_aggregate = true;
 
 /*
  * Local functions
  */
-static int opal_show_vhelp_internal(const char *filename, const char *topic,
-                                    int want_error_header, va_list arglist);
-static int opal_show_help_internal(const char *filename, const char *topic,
-                                   int want_error_header, ...);
-static void opal_show_help_finalize (void);
-
-opal_show_help_fn_t opal_show_help = opal_show_help_internal;
-opal_show_vhelp_fn_t opal_show_vhelp = opal_show_vhelp_internal;
+static void opal_show_help_finalize(void);
 
+typedef struct {
+    pmix_info_t *info;
+    pmix_info_t *dirs;
+    char *msg;
+} opal_log_info_t;
 
 int opal_show_help_init(void)
 {
     opal_output_stream_t lds;
 
+    opal_help_want_aggregate = true;
+    mca_base_var_register("opal", NULL, "base", "help_aggregate",
+                            "If opal_base_help_aggregate is true, duplicate help messages will be aggregated rather "
+                            "than displayed individually.  This can be helpful for parallel jobs that experience "
+                            "multiple identical failures; rather than print out the same help/failure message N times, "
+                            "display it once with a count of how many processes sent the same message. Default: true.",
+                            MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9,
+                            MCA_BASE_VAR_SCOPE_LOCAL, &opal_help_want_aggregate);
+
     OBJ_CONSTRUCT(&lds, opal_output_stream_t);
     lds.lds_want_stderr = true;
     output_stream = opal_output_open(&lds);
 
     opal_argv_append_nosize(&search_dirs, opal_install_dirs.opaldatadir);
 
-    opal_finalize_register_cleanup (opal_show_help_finalize);
+    opal_finalize_register_cleanup(opal_show_help_finalize);
 
     return OPAL_SUCCESS;
 }
 
-static void opal_show_help_finalize (void)
+static void opal_show_help_finalize(void)
 {
     opal_output_close(output_stream);
     output_stream = -1;
@@ -87,6 +94,89 @@ static void opal_show_help_finalize (void)
         opal_argv_free(search_dirs);
         search_dirs = NULL;
     }
+
+    /* free the rendered help strings */
+    opal_show_help_content_free();
+}
+
+static void opal_show_help_output(const char *msg) {
+
+    if(-1 < output_stream) {
+        opal_output(output_stream, "%s", msg);
+    }
+    else {
+        // opal_show_help() was not initialized yet, but we should still be
+        // able to print to stderr.
+        fprintf(stderr, "%s", msg);
+    }
+}
+
+static void opal_show_help_cbfunc(pmix_status_t status, void *cbdata)
+{
+    opal_log_info_t *info = (opal_log_info_t *) cbdata;
+    if(PMIX_SUCCESS != status && PMIX_OPERATION_SUCCEEDED != status) {
+        // Aggregation/de-duplication functionality is *probably* lost,
+        // but let's print the error anyway since duplicate error messages
+        // is better than hiding it.
+        opal_show_help_output(info->msg);
+    }
+    PMIX_INFO_DESTRUCT(info->info);
+    if(info->dirs) {
+        PMIX_INFO_DESTRUCT(info->dirs);
+    }
+    free(info->msg);
+    free(info);
+}
+
+static void local_delivery(const char *file, const char *topic, char *msg) {
+    pmix_info_t *info, *dirs = NULL;
+    int ninfo = 0, ndirs = 0;
+    PMIX_INFO_CREATE(info, 1);
+    PMIX_INFO_LOAD(&info[ninfo++], PMIX_LOG_STDERR, msg, PMIX_STRING);
+
+    opal_log_info_t *cbdata = calloc(1, sizeof(opal_log_info_t));
+    if(opal_help_want_aggregate) {
+// Not available in PMIx releases of v4.1.2 and earlier.
+// This should be available in future v4.1 and v4.2 releases,
+// as well as future major releases.
+// Disabling the aggregate behavior here, but still log it, as
+// seeing duplicate messages is better than not seeing anything at all.
+#ifdef PMIX_LOG_AGG
+        PMIX_INFO_CREATE(dirs, 3);
+        PMIX_INFO_LOAD(&dirs[ndirs++], PMIX_LOG_AGG, &opal_help_want_aggregate, PMIX_BOOL);
+        PMIX_INFO_LOAD(&dirs[ndirs++], PMIX_LOG_KEY, file, PMIX_STRING);
+        PMIX_INFO_LOAD(&dirs[ndirs++], PMIX_LOG_VAL, topic, PMIX_STRING);
+        cbdata->dirs = dirs;
+#endif
+    }
+
+    cbdata->info = info;
+    cbdata->msg  = msg;
+
+    // PMIx and the runtime will aggregate, de-duplicate, and print this
+    // message to stderr.
+    pmix_status_t rc = PMIx_Log_nb(info, ninfo, dirs, ndirs, opal_show_help_cbfunc, cbdata);
+    if(PMIX_SUCCESS != rc) {
+        // Aggregation/de-duplication functionality is *definitely* lost,
+        // but let's print the error anyway since duplicate error messages
+        // is better than hiding it.
+        opal_show_help_output(msg);
+        PMIX_INFO_DESTRUCT(info);
+        if(opal_help_want_aggregate) {
+            PMIX_INFO_DESTRUCT(dirs);
+        }
+        free(msg);
+        free(cbdata);
+    }
+}
+
+static void opal_show_help_error(const char *file, const char *topic) {
+    char *msg;
+    opal_asprintf(&msg,
+                   "%sSorry!  You were supposed to get help about:\n    %s\nfrom topic:\n  "
+                   "  %s\nBut I couldn't find that help topic.  Sorry!\n%s",
+                    dash_line, topic ? topic : "(Not specified)", file ? file : "(Not specified)", dash_line);
+    local_delivery(topic, file, msg);
 }
 
 /*
@@ -94,8 +184,7 @@ static void opal_show_help_finalize (void)
  * efficient method in the world, but we're going for clarity here --
  * not optimization.  :-)
  */
-static int array2string(char **outstring,
-                        int want_error_header, char **lines)
+static int array2string(char **outstring, int want_error_header, char **lines)
 {
     int i, count;
     size_t len;
@@ -113,7 +202,7 @@ static int array2string(char **outstring,
 
     /* Malloc it out */
 
-    (*outstring) = (char*) malloc(len + 1);
+    (*outstring) = (char *) malloc(len + 1);
     if (NULL == *outstring) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
@@ -138,176 +227,18 @@ static int array2string(char **outstring,
     return OPAL_SUCCESS;
 }
 
-
-/*
- * Find the right file to open
- */
-static int open_file(const char *base, const char *topic)
-{
-    char *filename;
-    char *err_msg = NULL;
-    size_t base_len;
-    int i;
-
-    /* If no filename was supplied, use the default */
-
-    if (NULL == base) {
-        base = default_filename;
-    }
-
-    /* if this is called prior to someone initializing the system,
-     * then don't try to look
-     */
-    if (NULL != search_dirs) {
-        /* Try to open the file.  If we can't find it, try it with a .txt
-         * extension.
-         */
-        for (i=0; NULL != search_dirs[i]; i++) {
-            filename = opal_os_path( false, search_dirs[i], base, NULL );
-            opal_show_help_yyin = fopen(filename, "r");
-            if (NULL == opal_show_help_yyin) {
-                opal_asprintf(&err_msg, "%s: %s", filename, strerror(errno));
-                base_len = strlen(base);
-                if (4 > base_len || 0 != strcmp(base + base_len - 4, ".txt")) {
-                    free(filename);
-                    opal_asprintf(&filename, "%s%s%s.txt", search_dirs[i], OPAL_PATH_SEP, base);
-                    opal_show_help_yyin = fopen(filename, "r");
-                }
-            }
-            free(filename);
-            if (NULL != opal_show_help_yyin) {
-                break;
-            }
-        }
-    }
-
-    /* If we still couldn't open it, then something is wrong */
-    if (NULL == opal_show_help_yyin) {
-        opal_output(output_stream, "%sSorry!  You were supposed to get help about:\n    %s\nBut I couldn't open the help file:\n    %s.  Sorry!\n%s", dash_line, topic, err_msg, dash_line);
-        free(err_msg);
-        return OPAL_ERR_NOT_FOUND;
-    }
-
-    if (NULL != err_msg) {
-        free(err_msg);
-    }
-
-    /* Set the buffer */
-
-    opal_show_help_init_buffer(opal_show_help_yyin);
-
-    /* Happiness */
-
-    return OPAL_SUCCESS;
-}
-
-
-/*
- * In the file that has already been opened, find the topic that we're
- * supposed to output
- */
-static int find_topic(const char *base, const char *topic)
+char *opal_show_help_vstring(const char *filename, const char *topic, int want_error_header,
+                             va_list arglist)
 {
-    int token, ret;
-    char *tmp;
-
-    /* Examine every topic */
-
-    while (1) {
-        token = opal_show_help_yylex();
-        switch (token) {
-        case OPAL_SHOW_HELP_PARSE_TOPIC:
-            tmp = strdup(opal_show_help_yytext);
-            if (NULL == tmp) {
-                return OPAL_ERR_OUT_OF_RESOURCE;
-            }
-            tmp[strlen(tmp) - 1] = '\0';
-            ret = strcmp(tmp + 1, topic);
-            free(tmp);
-            if (0 == ret) {
-                return OPAL_SUCCESS;
-            }
-            break;
-
-        case OPAL_SHOW_HELP_PARSE_MESSAGE:
-            break;
-
-        case OPAL_SHOW_HELP_PARSE_DONE:
-            opal_output(output_stream, "%sSorry!  You were supposed to get help about:\n    %s\nfrom the file:\n    %s\nBut I couldn't find that topic in the file.  Sorry!\n%s", dash_line, topic, base, dash_line);
-            return OPAL_ERR_NOT_FOUND;
-            break;
-
-        default:
-            break;
-        }
-    }
-
-    /* Never get here */
-}
-
-
-/*
- * We have an open file, and we're pointed at the right topic.  So
- * read in all the lines in the topic and make a list of them.
- */
-static int read_topic(char ***array)
-{
-    int token, rc;
-
-    while (1) {
-        token = opal_show_help_yylex();
-        switch (token) {
-        case OPAL_SHOW_HELP_PARSE_MESSAGE:
-            /* opal_argv_append_nosize does strdup(opal_show_help_yytext) */
-            rc = opal_argv_append_nosize(array, opal_show_help_yytext);
-            if (rc != OPAL_SUCCESS) {
-                return rc;
-            }
-            break;
-
-        default:
-            return OPAL_SUCCESS;
-            break;
-        }
-    }
-
-    /* Never get here */
-}
-
-
-static int load_array(char ***array, const char *filename, const char *topic)
-{
-    int ret;
-
-    if (OPAL_SUCCESS != (ret = open_file(filename, topic))) {
-        return ret;
-    }
-
-    ret = find_topic(filename, topic);
-    if (OPAL_SUCCESS == ret) {
-        ret = read_topic(array);
-    }
-
-    fclose(opal_show_help_yyin);
-    opal_show_help_yylex_destroy ();
-
-    if (OPAL_SUCCESS != ret) {
-        opal_argv_free(*array);
+    const char *content = opal_show_help_get_content(filename, topic);
+    if (NULL == content) {
+        opal_show_help_error(filename, topic);
+        return NULL;
     }
 
-    return ret;
-}
-
-char *opal_show_help_vstring(const char *filename, const char *topic,
-                             int want_error_header, va_list arglist)
-{
     int rc;
-    char *single_string, *output, **array = NULL;
-
-    /* Load the message */
-    if (OPAL_SUCCESS != (rc = load_array(&array, filename, topic))) {
-        return NULL;
-    }
+    char *single_string, *output;
+    char *array[] = { (char*) content, NULL };
 
     /* Convert it to a single raw string */
     rc = array2string(&single_string, want_error_header, array);
@@ -318,44 +249,38 @@ char *opal_show_help_vstring(const char *filename, const char *topic,
         free(single_string);
     }
 
-    opal_argv_free(array);
     return (OPAL_SUCCESS == rc) ? output : NULL;
 }
 
-char *opal_show_help_string(const char *filename, const char *topic,
-                            int want_error_handler, ...)
+char *opal_show_help_string(const char *filename, const char *topic, int want_error_handler, ...)
 {
     char *output;
     va_list arglist;
 
     va_start(arglist, want_error_handler);
-    output = opal_show_help_vstring(filename, topic, want_error_handler,
-                                    arglist);
+    output = opal_show_help_vstring(filename, topic, want_error_handler, arglist);
     va_end(arglist);
 
     return output;
 }
 
-static int opal_show_vhelp_internal(const char *filename, const char *topic,
-                                    int want_error_header, va_list arglist)
+int opal_show_vhelp(const char *filename, const char *topic, int want_error_header,
+                    va_list arglist)
 {
     char *output;
 
     /* Convert it to a single string */
-    output = opal_show_help_vstring(filename, topic, want_error_header,
-                                    arglist);
+    output = opal_show_help_vstring(filename, topic, want_error_header, arglist);
 
     /* If we got a single string, output it with formatting */
     if (NULL != output) {
-        opal_output(output_stream, "%s", output);
-        free(output);
+        local_delivery(filename, topic, output);
     }
 
     return (NULL == output) ? OPAL_ERROR : OPAL_SUCCESS;
 }
 
-static int opal_show_help_internal(const char *filename, const char *topic,
-                                   int want_error_header, ...)
+int opal_show_help(const char *filename, const char *topic, int want_error_header, ...)
 {
     va_list arglist;
     int rc;
diff --git a/opal/util/show_help.h b/opal/util/show_help.h
index 27335fd10c6..1a24b42c79c 100644
--- a/opal/util/show_help.h
+++ b/opal/util/show_help.h
@@ -13,6 +13,7 @@
  * Copyright (c) 2008-2018 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2018      Triad National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2025      Jeffrey M. Squyres.  All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -49,7 +50,7 @@
  * appropriate help message to display.  It looks for the message name
  * in the file, reads in the message, and displays it.  printf()-like
  * substitutions are performed (e.g., %d, %s, etc.) --
- * opal_show_help() takes a variable legnth argument list that are
+ * opal_show_help() takes a variable length argument list that are
  * used for these substitutions.
  *
  * The format of the help file is simplistic:
@@ -127,33 +128,29 @@ OPAL_DECLSPEC int opal_show_help_init(void);
  * promotion to va_start() has undefined behavior (according to clang
  * warnings on MacOS High Sierra).
  */
-typedef int (*opal_show_help_fn_t)(const char *filename, const char *topic,
-                                   int want_error_header, ...);
-OPAL_DECLSPEC extern opal_show_help_fn_t opal_show_help;
+OPAL_DECLSPEC int opal_show_help(const char *filename, const char *topic, int want_error_header,
+                                 ...);
 
 /**
  * This function does the same thing as opal_show_help(), but accepts
  * a va_list form of varargs.
  */
-typedef int (*opal_show_vhelp_fn_t)(const char *filename, const char *topic,
-                                    int want_error_header, va_list ap);
-OPAL_DECLSPEC extern opal_show_vhelp_fn_t opal_show_vhelp;
+OPAL_DECLSPEC int opal_show_vhelp(const char *filename, const char *topic, int want_error_header,
+                                  va_list ap);
 
 /**
  * This function does the same thing as opal_show_help(), but returns
  * its output in a string (that must be freed by the caller).
  */
-OPAL_DECLSPEC char* opal_show_help_string(const char *filename,
-                                          const char *topic,
+OPAL_DECLSPEC char *opal_show_help_string(const char *filename, const char *topic,
                                           int want_error_header, ...);
 
 /**
  * This function does the same thing as opal_show_help_string(), but
  * accepts a va_list form of varargs.
  */
-OPAL_DECLSPEC char* opal_show_help_vstring(const char *filename,
-                                          const char *topic,
-                                          int want_error_header, va_list ap);
+OPAL_DECLSPEC char *opal_show_help_vstring(const char *filename, const char *topic,
+                                           int want_error_header, va_list ap);
 
 /**
  * This function adds another search location for the files that
@@ -165,11 +162,31 @@ OPAL_DECLSPEC char* opal_show_help_vstring(const char *filename,
  * of the show_help functionality. OMPI defines the show_help directory
  * based on where OMPI was installed. However, if the library wants to
  * use show_help to provide error output specific to itself, then it
- * nees to tell show_help how to find its own show_help files - without
- * interfering with the linked ORTE libs when they need to do show_help.
+ * needs to tell show_help how to find its own show_help files.
  */
 OPAL_DECLSPEC int opal_show_help_add_dir(const char *directory);
 
+/**
+ * \internal
+ *
+ * This is an internal function that returns a string representing the
+ * content of the section "topic" from the help file "filename".  This
+ * content is a hard-coded C string and should not be freed by the
+ * caller.
+ *
+ * NOTE: This hard-coded C string is generated during "make" in the
+ * opal/util directory -- it is not read from a text file at run time.
+ */
+const char *opal_show_help_get_content(const char *filename, const char* topic);
+
+/**
+ * \internal
+ *
+ * Free up any strings that may have been allocated in when rendering
+ * show_help strings.
+ */
+void opal_show_help_content_free(void);
+
 END_C_DECLS
 
 #endif
diff --git a/opal/util/show_help_lex.h b/opal/util/show_help_lex.h
deleted file mode 100644
index 864888039b9..00000000000
--- a/opal/util/show_help_lex.h
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef OPAL_SHOW_HELP_LEX_H
-#define OPAL_SHOW_HELP_LEX_H
-
-#include "opal_config.h"
-
-#ifdef malloc
-#undef malloc
-#endif
-#ifdef realloc
-#undef realloc
-#endif
-#ifdef free
-#undef free
-#endif
-
-#include <stdio.h>
-BEGIN_C_DECLS
-int opal_show_help_yylex(void);
-int opal_show_help_init_buffer(FILE *file);
-int opal_show_help_yylex_destroy(void);
-
-extern FILE *opal_show_help_yyin;
-extern bool opal_show_help_parse_done;
-extern char *opal_show_help_yytext;
-extern int opal_show_help_yynewlines;
-
-/*
- * Make lex-generated files not issue compiler warnings
- */
-#define YY_STACK_USED 0
-#define YY_ALWAYS_INTERACTIVE 0
-#define YY_NEVER_INTERACTIVE 0
-#define YY_MAIN 0
-#define YY_NO_UNPUT 1
-#define YY_SKIP_YYWRAP 1
-
-enum {
-    OPAL_SHOW_HELP_PARSE_DONE,
-    OPAL_SHOW_HELP_PARSE_ERROR,
-
-    OPAL_SHOW_HELP_PARSE_TOPIC,
-    OPAL_SHOW_HELP_PARSE_MESSAGE,
-
-    OPAL_SHOW_HELP_PARSE_MAX
-};
-END_C_DECLS
-#endif
diff --git a/opal/util/show_help_lex.l b/opal/util/show_help_lex.l
deleted file mode 100644
index 2de61e830ec..00000000000
--- a/opal/util/show_help_lex.l
+++ /dev/null
@@ -1,113 +0,0 @@
-%option nounput
-%option noinput
-
-%{ /* -*- C -*- */
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2006 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "opal_config.h"
-
-#include <stdio.h>
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-
-#include "opal/util/show_help_lex.h"
-
-BEGIN_C_DECLS
-
-/*
- * public functions
- */
-extern int opal_show_help_finish_parsing(void);
-
-/*
- * local functions
- */
-static int opal_show_help_yywrap(void);
-
-END_C_DECLS
-
-/*
- * global variables
- */
-int opal_show_help_yynewlines = 1;
-bool opal_show_help_parse_done = false;
-
-%}
-
-WHITE       [\f\t\v ]
-CHAR        [A-Za-z0-9_\-\.]
-
-%x CHOMP
-
-%%
-
-#.*\n               ; /* comment line */
-
-^\[.+\]/[^\]\n]*\n { BEGIN(CHOMP); return OPAL_SHOW_HELP_PARSE_TOPIC; }
-
-<CHOMP>.*\n { BEGIN(INITIAL); }
-
-.*/\n { BEGIN(CHOMP); return OPAL_SHOW_HELP_PARSE_MESSAGE; }
-
-%%
-
-/* Old flex (2.5.4a? and older) does not define a destroy function */
-#if !defined(YY_FLEX_SUBMINOR_VERSION)
-#define YY_FLEX_SUBMINOR_VERSION 0
-#endif
-
-#if (YY_FLEX_MAJOR_VERSION < 2) || (YY_FLEX_MAJOR_VERSION == 2 && (YY_FLEX_MINOR_VERSION < 5 || (YY_FLEX_MINOR_VERSION == 5 && YY_FLEX_SUBMINOR_VERSION < 5)))
-int opal_show_help_yylex_destroy(void)
-{
-    if (NULL != YY_CURRENT_BUFFER) {
-        yy_delete_buffer(YY_CURRENT_BUFFER);
-#if defined(YY_CURRENT_BUFFER_LVALUE)
-        YY_CURRENT_BUFFER_LVALUE = NULL;
-#else
-        YY_CURRENT_BUFFER = NULL;
-#endif  /* YY_CURRENT_BUFFER_LVALUE */
-    }
-    return YY_NULL;
-}
-#endif
-
-static int opal_show_help_yywrap(void)
-{
-    opal_show_help_parse_done = true;
-    return 1;
-}
-
-
-/*
- * Ensure that we have a valid yybuffer to use.  Specifically, if this
- * scanner is invoked a second time, finish_parsing() (above) will
- * have been executed, and the current buffer will have been freed.
- * Flex doesn't recognize this fact because as far as it's concerned,
- * its internal state was already initialized, so it thinks it should
- * have a valid buffer.  Hence, here we ensure to give it a valid
- * buffer.
- */
-int opal_show_help_init_buffer(FILE *file)
-{
-    YY_BUFFER_STATE buf = yy_create_buffer(file, YY_BUF_SIZE);
-    yy_switch_to_buffer(buf);
-
-    return 0;
-}
diff --git a/opal/util/stacktrace.c b/opal/util/stacktrace.c
index 907461e0f6f..9a71de16bc2 100644
--- a/opal/util/stacktrace.c
+++ b/opal/util/stacktrace.c
@@ -10,7 +10,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2006      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2008-2009 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2008-2022 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2017      IBM Corporation.  All rights reserved.
  * Copyright (c) 2017      FUJITSU LIMITED.  All rights reserved.
  * Copyright (c) 2019      Triad National Security, LLC. All rights
@@ -27,44 +27,44 @@
 
 #include <stdio.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 #ifdef HAVE_SYS_STAT_H
-#include <sys/stat.h>
+#    include <sys/stat.h>
 #endif
 #ifdef HAVE_FCNTL_H
-#include <fcntl.h>
+#    include <fcntl.h>
 #else
-#ifdef HAVE_SYS_FCNTL_H
-#include <sys/fcntl.h>
-#endif
+#    ifdef HAVE_SYS_FCNTL_H
+#        include <sys/fcntl.h>
+#    endif
 #endif
 
-#include <string.h>
 #include <signal.h>
+#include <string.h>
 
-#include "opal/runtime/opal.h"
-#include "opal/util/stacktrace.h"
-#include "opal/mca/backtrace/backtrace.h"
 #include "opal/constants.h"
-#include "opal/util/output.h"
-#include "opal/util/show_help.h"
+#include "opal/mca/backtrace/backtrace.h"
+#include "opal/runtime/opal.h"
+#include "opal/runtime/opal_params.h"
 #include "opal/util/argv.h"
-#include "opal/util/proc.h"
 #include "opal/util/error.h"
-#include "opal/runtime/opal_params.h"
+#include "opal/util/output.h"
+#include "opal/util/proc.h"
+#include "opal/util/show_help.h"
+#include "opal/util/stacktrace.h"
 
 #ifndef _NSIG
-#define _NSIG 32
+#    define _NSIG 32
 #endif
 
 #define HOSTFORMAT "[%s:%05d] "
 
-int    opal_stacktrace_output_fileno = -1;
-static char  *opal_stacktrace_output_filename_base = NULL;
+int opal_stacktrace_output_fileno = -1;
+static char *opal_stacktrace_output_filename_base = NULL;
 static size_t opal_stacktrace_output_filename_max_len = 0;
 static char stacktrace_hostname[OPAL_MAXHOSTNAMELEN];
 static const char *stacktrace_hostname_full;
@@ -76,18 +76,17 @@ static char *unable_to_print_msg = "Unable to print stack trace!\n";
  * -or, if VPID is available-
  * stacktrace.VPID.PID
  */
-static void set_stacktrace_filename(void) {
+static void set_stacktrace_filename(void)
+{
     opal_proc_t *my_proc = opal_proc_local_get();
 
-    if( NULL == my_proc ) {
-        snprintf(opal_stacktrace_output_filename, opal_stacktrace_output_filename_max_len,
-                 "%s.%lu",
-                 opal_stacktrace_output_filename_base, (unsigned long)getpid());
-    }
-    else {
+    if (NULL == my_proc) {
+        snprintf(opal_stacktrace_output_filename, opal_stacktrace_output_filename_max_len, "%s.%lu",
+                 opal_stacktrace_output_filename_base, (unsigned long) getpid());
+    } else {
         snprintf(opal_stacktrace_output_filename, opal_stacktrace_output_filename_max_len,
-                 "%s.%lu.%lu",
-                 opal_stacktrace_output_filename_base, (unsigned long)my_proc->proc_name.vpid, (unsigned long)getpid());
+                 "%s.%lu.%lu", opal_stacktrace_output_filename_base,
+                 (unsigned long) my_proc->proc_name.vpid, (unsigned long) getpid());
     }
 
     return;
@@ -108,16 +107,16 @@ static void set_stacktrace_filename(void) {
  * FIXME: Should distinguish for systems, which don't have siginfo...
  */
 #if OPAL_WANT_PRETTY_PRINT_STACKTRACE
-static void show_stackframe (int signo, siginfo_t * info, void * p)
+static void show_stackframe(int signo, siginfo_t *info, void *p)
 {
     char print_buffer[1024];
-    char * tmp = print_buffer;
-    int size = sizeof (print_buffer);
+    char *tmp = print_buffer;
+    int size = sizeof(print_buffer);
     int ret;
     char *si_code_str = "";
 
     /* Do not print the stack trace */
-    if( 0 > opal_stacktrace_output_fileno && 0 == opal_stacktrace_output_filename_max_len ) {
+    if (0 > opal_stacktrace_output_fileno && 0 == opal_stacktrace_output_filename_max_len) {
         /* Raise the signal again, so we don't accidentally mask critical signals.
          * For critical signals, it is preferred that we call 'raise' instead of
          * 'exit' or 'abort' so that the return status is set properly for this
@@ -130,262 +129,334 @@ static void show_stackframe (int signo, siginfo_t * info, void * p)
     }
 
     /* Update the file name with the RANK, if available */
-    if( 0 < opal_stacktrace_output_filename_max_len ) {
+    if (0 < opal_stacktrace_output_filename_max_len) {
         set_stacktrace_filename();
         opal_stacktrace_output_fileno = open(opal_stacktrace_output_filename,
-                                             O_CREAT|O_WRONLY|O_TRUNC, S_IRUSR|S_IWUSR);
-        if( 0 > opal_stacktrace_output_fileno ) {
-            opal_output(0, "Error: Failed to open the stacktrace output file. Default: stderr\n\tFilename: %s\n\tErrno: %s",
+                                             O_CREAT | O_WRONLY | O_TRUNC, S_IRUSR | S_IWUSR);
+        if (0 > opal_stacktrace_output_fileno) {
+            opal_output(0,
+                        "Error: Failed to open the stacktrace output file. Default: "
+                        "stderr\n\tFilename: %s\n\tErrno: %s",
                         opal_stacktrace_output_filename, strerror(errno));
             opal_stacktrace_output_fileno = fileno(stderr);
         }
     }
 
     /* write out the footer information */
-    memset (print_buffer, 0, sizeof (print_buffer));
+    memset(print_buffer, 0, sizeof(print_buffer));
     ret = snprintf(print_buffer, sizeof(print_buffer),
-                   HOSTFORMAT "*** Process received signal ***\n",
-                   stacktrace_hostname, getpid());
-    write(opal_stacktrace_output_fileno, print_buffer, ret);
-
+                   HOSTFORMAT "*** Process received signal ***\n", stacktrace_hostname, getpid());
+    opal_best_effort_write(opal_stacktrace_output_fileno, print_buffer, ret);
 
-    memset (print_buffer, 0, sizeof (print_buffer));
+    memset(print_buffer, 0, sizeof(print_buffer));
 
-#ifdef HAVE_STRSIGNAL
-    ret = snprintf (tmp, size, HOSTFORMAT "Signal: %s (%d)\n",
-                    stacktrace_hostname, getpid(), strsignal(signo), signo);
-#else
-    ret = snprintf (tmp, size, HOSTFORMAT "Signal: %d\n",
-                    stacktrace_hostname, getpid(), signo);
-#endif
+#    ifdef HAVE_STRSIGNAL
+    ret = snprintf(tmp, size, HOSTFORMAT "Signal: %s (%d)\n", stacktrace_hostname, getpid(),
+                   strsignal(signo), signo);
+#    else
+    ret = snprintf(tmp, size, HOSTFORMAT "Signal: %d\n", stacktrace_hostname, getpid(), signo);
+#    endif
     size -= ret;
     tmp += ret;
 
     if (NULL != info) {
-        switch (signo)
-        {
+        switch (signo) {
         case SIGILL:
-            switch (info->si_code)
-            {
-#ifdef ILL_ILLOPC
-            case ILL_ILLOPC: si_code_str = "Illegal opcode"; break;
-#endif
-#ifdef ILL_ILLOPN
-            case ILL_ILLOPN: si_code_str = "Illegal operand"; break;
-#endif
-#ifdef ILL_ILLADR
-            case ILL_ILLADR: si_code_str = "Illegal addressing mode"; break;
-#endif
-#ifdef ILL_ILLTRP
-            case ILL_ILLTRP: si_code_str = "Illegal trap"; break;
-#endif
-#ifdef ILL_PRVOPC
-            case ILL_PRVOPC: si_code_str = "Privileged opcode"; break;
-#endif
-#ifdef ILL_PRVREG
-            case ILL_PRVREG: si_code_str = "Privileged register"; break;
-#endif
-#ifdef ILL_COPROC
-            case ILL_COPROC: si_code_str = "Coprocessor error"; break;
-#endif
-#ifdef ILL_BADSTK
-            case ILL_BADSTK: si_code_str = "Internal stack error"; break;
-#endif
+            switch (info->si_code) {
+#    ifdef ILL_ILLOPC
+            case ILL_ILLOPC:
+                si_code_str = "Illegal opcode";
+                break;
+#    endif
+#    ifdef ILL_ILLOPN
+            case ILL_ILLOPN:
+                si_code_str = "Illegal operand";
+                break;
+#    endif
+#    ifdef ILL_ILLADR
+            case ILL_ILLADR:
+                si_code_str = "Illegal addressing mode";
+                break;
+#    endif
+#    ifdef ILL_ILLTRP
+            case ILL_ILLTRP:
+                si_code_str = "Illegal trap";
+                break;
+#    endif
+#    ifdef ILL_PRVOPC
+            case ILL_PRVOPC:
+                si_code_str = "Privileged opcode";
+                break;
+#    endif
+#    ifdef ILL_PRVREG
+            case ILL_PRVREG:
+                si_code_str = "Privileged register";
+                break;
+#    endif
+#    ifdef ILL_COPROC
+            case ILL_COPROC:
+                si_code_str = "Coprocessor error";
+                break;
+#    endif
+#    ifdef ILL_BADSTK
+            case ILL_BADSTK:
+                si_code_str = "Internal stack error";
+                break;
+#    endif
             }
             break;
         case SIGFPE:
-            switch (info->si_code)
-            {
-#ifdef FPE_INTDIV
-            case FPE_INTDIV: si_code_str = "Integer divide-by-zero"; break;
-#endif
-#ifdef FPE_INTOVF
-            case FPE_INTOVF: si_code_str = "Integer overflow"; break;
-#endif
-            case FPE_FLTDIV: si_code_str = "Floating point divide-by-zero"; break;
-            case FPE_FLTOVF: si_code_str = "Floating point overflow"; break;
-            case FPE_FLTUND: si_code_str = "Floating point underflow"; break;
-#ifdef FPE_FLTRES
-            case FPE_FLTRES: si_code_str = "Floating point inexact result"; break;
-#endif
-#ifdef FPE_FLTINV
-            case FPE_FLTINV: si_code_str = "Invalid floating point operation"; break;
-#endif
-#ifdef FPE_FLTSUB
-            case FPE_FLTSUB: si_code_str = "Subscript out of range"; break;
-#endif
+            switch (info->si_code) {
+#    ifdef FPE_INTDIV
+            case FPE_INTDIV:
+                si_code_str = "Integer divide-by-zero";
+                break;
+#    endif
+#    ifdef FPE_INTOVF
+            case FPE_INTOVF:
+                si_code_str = "Integer overflow";
+                break;
+#    endif
+            case FPE_FLTDIV:
+                si_code_str = "Floating point divide-by-zero";
+                break;
+            case FPE_FLTOVF:
+                si_code_str = "Floating point overflow";
+                break;
+            case FPE_FLTUND:
+                si_code_str = "Floating point underflow";
+                break;
+#    ifdef FPE_FLTRES
+            case FPE_FLTRES:
+                si_code_str = "Floating point inexact result";
+                break;
+#    endif
+#    ifdef FPE_FLTINV
+            case FPE_FLTINV:
+                si_code_str = "Invalid floating point operation";
+                break;
+#    endif
+#    ifdef FPE_FLTSUB
+            case FPE_FLTSUB:
+                si_code_str = "Subscript out of range";
+                break;
+#    endif
             }
             break;
         case SIGSEGV:
-            switch (info->si_code)
-            {
-#ifdef SEGV_MAPERR
-            case SEGV_MAPERR: si_code_str = "Address not mapped"; break;
-#endif
-#ifdef SEGV_ACCERR
-            case SEGV_ACCERR: si_code_str = "Invalid permissions"; break;
-#endif
+            switch (info->si_code) {
+#    ifdef SEGV_MAPERR
+            case SEGV_MAPERR:
+                si_code_str = "Address not mapped";
+                break;
+#    endif
+#    ifdef SEGV_ACCERR
+            case SEGV_ACCERR:
+                si_code_str = "Invalid permissions";
+                break;
+#    endif
             }
             break;
         case SIGBUS:
-            switch (info->si_code)
-            {
-#ifdef BUS_ADRALN
-            case BUS_ADRALN: si_code_str = "Invalid address alignment"; break;
-#endif
-#ifdef BUS_ADRERR
-            case BUS_ADRERR: si_code_str = "Non-existant physical address"; break;
-#endif
-#ifdef BUS_OBJERR
-            case BUS_OBJERR: si_code_str = "Object-specific hardware error"; break;
-#endif
+            switch (info->si_code) {
+#    ifdef BUS_ADRALN
+            case BUS_ADRALN:
+                si_code_str = "Invalid address alignment";
+                break;
+#    endif
+#    ifdef BUS_ADRERR
+            case BUS_ADRERR:
+                si_code_str = "Non-existent physical address";
+                break;
+#    endif
+#    ifdef BUS_OBJERR
+            case BUS_OBJERR:
+                si_code_str = "Object-specific hardware error";
+                break;
+#    endif
             }
             break;
         case SIGTRAP:
-            switch (info->si_code)
-            {
-#ifdef TRAP_BRKPT
-            case TRAP_BRKPT: si_code_str = "Process breakpoint"; break;
-#endif
-#ifdef TRAP_TRACE
-            case TRAP_TRACE: si_code_str = "Process trace trap"; break;
-#endif
+            switch (info->si_code) {
+#    ifdef TRAP_BRKPT
+            case TRAP_BRKPT:
+                si_code_str = "Process breakpoint";
+                break;
+#    endif
+#    ifdef TRAP_TRACE
+            case TRAP_TRACE:
+                si_code_str = "Process trace trap";
+                break;
+#    endif
             }
             break;
         case SIGCHLD:
-            switch (info->si_code)
-            {
-#ifdef CLD_EXITED
-            case CLD_EXITED: si_code_str = "Child has exited"; break;
-#endif
-#ifdef CLD_KILLED
-            case CLD_KILLED: si_code_str = "Child has terminated abnormally and did not create a core file"; break;
-#endif
-#ifdef CLD_DUMPED
-            case CLD_DUMPED: si_code_str = "Child has terminated abnormally and created a core file"; break;
-#endif
-#ifdef CLD_WTRAPPED
-            case CLD_TRAPPED: si_code_str = "Traced child has trapped"; break;
-#endif
-#ifdef CLD_STOPPED
-            case CLD_STOPPED: si_code_str = "Child has stopped"; break;
-#endif
-#ifdef CLD_CONTINUED
-            case CLD_CONTINUED: si_code_str = "Stopped child has continued"; break;
-#endif
+            switch (info->si_code) {
+#    ifdef CLD_EXITED
+            case CLD_EXITED:
+                si_code_str = "Child has exited";
+                break;
+#    endif
+#    ifdef CLD_KILLED
+            case CLD_KILLED:
+                si_code_str = "Child has terminated abnormally and did not create a core file";
+                break;
+#    endif
+#    ifdef CLD_DUMPED
+            case CLD_DUMPED:
+                si_code_str = "Child has terminated abnormally and created a core file";
+                break;
+#    endif
+#    ifdef CLD_WTRAPPED
+            case CLD_TRAPPED:
+                si_code_str = "Traced child has trapped";
+                break;
+#    endif
+#    ifdef CLD_STOPPED
+            case CLD_STOPPED:
+                si_code_str = "Child has stopped";
+                break;
+#    endif
+#    ifdef CLD_CONTINUED
+            case CLD_CONTINUED:
+                si_code_str = "Stopped child has continued";
+                break;
+#    endif
             }
             break;
-#ifdef SIGPOLL
+#    ifdef SIGPOLL
         case SIGPOLL:
-            switch (info->si_code)
-            {
-#ifdef POLL_IN
-            case POLL_IN: si_code_str = "Data input available"; break;
-#endif
-#ifdef POLL_OUT
-            case POLL_OUT: si_code_str = "Output buffers available"; break;
-#endif
-#ifdef POLL_MSG
-            case POLL_MSG: si_code_str = "Input message available"; break;
-#endif
-#ifdef POLL_ERR
-            case POLL_ERR: si_code_str = "I/O error"; break;
-#endif
-#ifdef POLL_PRI
-            case POLL_PRI: si_code_str = "High priority input available"; break;
-#endif
-#ifdef POLL_HUP
-            case POLL_HUP: si_code_str = "Device disconnected"; break;
-#endif
+            switch (info->si_code) {
+#        ifdef POLL_IN
+            case POLL_IN:
+                si_code_str = "Data input available";
+                break;
+#        endif
+#        ifdef POLL_OUT
+            case POLL_OUT:
+                si_code_str = "Output buffers available";
+                break;
+#        endif
+#        ifdef POLL_MSG
+            case POLL_MSG:
+                si_code_str = "Input message available";
+                break;
+#        endif
+#        ifdef POLL_ERR
+            case POLL_ERR:
+                si_code_str = "I/O error";
+                break;
+#        endif
+#        ifdef POLL_PRI
+            case POLL_PRI:
+                si_code_str = "High priority input available";
+                break;
+#        endif
+#        ifdef POLL_HUP
+            case POLL_HUP:
+                si_code_str = "Device disconnected";
+                break;
+#        endif
             }
             break;
-#endif /* SIGPOLL */
+#    endif /* SIGPOLL */
         default:
-            switch (info->si_code)
-            {
-#ifdef SI_ASYNCNL
-            case SI_ASYNCNL: si_code_str = "SI_ASYNCNL"; break;
-#endif
-#ifdef SI_SIGIO
-            case SI_SIGIO: si_code_str = "Queued SIGIO"; break;
-#endif
-#ifdef SI_ASYNCIO
-            case SI_ASYNCIO: si_code_str = "Asynchronous I/O request completed"; break;
-#endif
-#ifdef SI_MESGQ
-            case SI_MESGQ: si_code_str = "Message queue state changed"; break;
-#endif
-            case SI_TIMER: si_code_str = "Timer expiration"; break;
-            case SI_QUEUE: si_code_str = "Sigqueue() signal"; break;
-            case SI_USER: si_code_str = "User function (kill, sigsend, abort, etc.)"; break;
-#ifdef SI_KERNEL
-            case SI_KERNEL: si_code_str = "Kernel signal"; break;
-#endif
+            switch (info->si_code) {
+#    ifdef SI_ASYNCNL
+            case SI_ASYNCNL:
+                si_code_str = "SI_ASYNCNL";
+                break;
+#    endif
+#    ifdef SI_SIGIO
+            case SI_SIGIO:
+                si_code_str = "Queued SIGIO";
+                break;
+#    endif
+#    ifdef SI_ASYNCIO
+            case SI_ASYNCIO:
+                si_code_str = "Asynchronous I/O request completed";
+                break;
+#    endif
+#    ifdef SI_MESGQ
+            case SI_MESGQ:
+                si_code_str = "Message queue state changed";
+                break;
+#    endif
+            case SI_TIMER:
+                si_code_str = "Timer expiration";
+                break;
+            case SI_QUEUE:
+                si_code_str = "Sigqueue() signal";
+                break;
+            case SI_USER:
+                si_code_str = "User function (kill, sigsend, abort, etc.)";
+                break;
+#    ifdef SI_KERNEL
+            case SI_KERNEL:
+                si_code_str = "Kernel signal";
+                break;
+#    endif
 /* Dragonfly defines SI_USER and SI_UNDEFINED both as zero: */
 /* For some reason, the PGI compiler will not let us combine these two
    #if tests into a single statement.  Sigh. */
-#if defined(SI_UNDEFINED)
-#if SI_UNDEFINED != SI_USER
-            case SI_UNDEFINED: si_code_str = "Undefined code"; break;
-#endif
-#endif
+#    if defined(SI_UNDEFINED)
+#        if SI_UNDEFINED != SI_USER
+            case SI_UNDEFINED:
+                si_code_str = "Undefined code";
+                break;
+#        endif
+#    endif
             }
         }
 
         /* print signal errno information */
         if (0 != info->si_errno) {
-            ret = snprintf(tmp, size, HOSTFORMAT "Associated errno: %s (%d)\n",
-                           stacktrace_hostname, getpid(),
-                           strerror (info->si_errno), info->si_errno);
+            ret = snprintf(tmp, size, HOSTFORMAT "Associated errno: %s (%d)\n", stacktrace_hostname,
+                           getpid(), strerror(info->si_errno), info->si_errno);
             size -= ret;
             tmp += ret;
         }
 
-        ret = snprintf(tmp, size, HOSTFORMAT "Signal code: %s (%d)\n",
-                       stacktrace_hostname, getpid(),
-                       si_code_str, info->si_code);
+        ret = snprintf(tmp, size, HOSTFORMAT "Signal code: %s (%d)\n", stacktrace_hostname,
+                       getpid(), si_code_str, info->si_code);
         size -= ret;
         tmp += ret;
 
-        switch (signo)
-        {
+        switch (signo) {
         case SIGILL:
         case SIGFPE:
         case SIGSEGV:
-        case SIGBUS:
-        {
-            ret = snprintf(tmp, size, HOSTFORMAT "Failing at address: %p\n",
-                           stacktrace_hostname, getpid(), info->si_addr);
+        case SIGBUS: {
+            ret = snprintf(tmp, size, HOSTFORMAT "Failing at address: %p\n", stacktrace_hostname,
+                           getpid(), info->si_addr);
             size -= ret;
             tmp += ret;
             break;
         }
-        case SIGCHLD:
-        {
+        case SIGCHLD: {
             ret = snprintf(tmp, size, HOSTFORMAT "Sending PID: %d, Sending UID: %d, Status: %d\n",
-                           stacktrace_hostname, getpid(),
-                           info->si_pid, info->si_uid, info->si_status);
+                           stacktrace_hostname, getpid(), info->si_pid, info->si_uid,
+                           info->si_status);
             size -= ret;
             tmp += ret;
             break;
         }
-#ifdef SIGPOLL
-        case SIGPOLL:
-        {
-#ifdef HAVE_SIGINFO_T_SI_FD
+#    ifdef SIGPOLL
+        case SIGPOLL: {
+#        ifdef HAVE_SIGINFO_T_SI_FD
             ret = snprintf(tmp, size, HOSTFORMAT "Band event: %ld, File Descriptor : %d\n",
-                           stacktrace_hostname, getpid(), (long)info->si_band, info->si_fd);
-#elif HAVE_SIGINFO_T_SI_BAND
-            ret = snprintf(tmp, size, HOSTFORMAT "Band event: %ld\n",
-                           stacktrace_hostname, getpid(), (long)info->si_band);
-#else
+                           stacktrace_hostname, getpid(), (long) info->si_band, info->si_fd);
+#        elif HAVE_SIGINFO_T_SI_BAND
+            ret = snprintf(tmp, size, HOSTFORMAT "Band event: %ld\n", stacktrace_hostname, getpid(),
+                           (long) info->si_band);
+#        else
             ret = 0;
-#endif
+#        endif
             size -= ret;
             tmp += ret;
             break;
         }
-#endif
+#    endif
         }
     } else {
         ret = snprintf(tmp, size,
@@ -396,29 +467,29 @@ static void show_stackframe (int signo, siginfo_t * info, void * p)
     }
 
     /* write out the signal information generated above */
-    write(opal_stacktrace_output_fileno, print_buffer, sizeof(print_buffer)-size);
+    opal_best_effort_write(opal_stacktrace_output_fileno, print_buffer, sizeof(print_buffer) - size);
 
     /* print out the stack trace */
-    snprintf(print_buffer, sizeof(print_buffer), HOSTFORMAT,
-             stacktrace_hostname, getpid());
+    snprintf(print_buffer, sizeof(print_buffer), HOSTFORMAT, stacktrace_hostname, getpid());
     ret = opal_backtrace_print(NULL, print_buffer, 2);
     if (OPAL_SUCCESS != ret) {
-        write(opal_stacktrace_output_fileno, unable_to_print_msg, strlen(unable_to_print_msg));
+        opal_best_effort_write(opal_stacktrace_output_fileno, unable_to_print_msg,
+                               strlen(unable_to_print_msg));
     }
 
     /* write out the footer information */
-    memset (print_buffer, 0, sizeof (print_buffer));
-    ret = snprintf(print_buffer, sizeof(print_buffer),
-                   HOSTFORMAT "*** End of error message ***\n",
+    memset(print_buffer, 0, sizeof(print_buffer));
+    ret = snprintf(print_buffer, sizeof(print_buffer), HOSTFORMAT "*** End of error message ***\n",
                    stacktrace_hostname, getpid());
     if (ret > 0) {
-        write(opal_stacktrace_output_fileno, print_buffer, ret);
+        opal_best_effort_write(opal_stacktrace_output_fileno, print_buffer, ret);
     } else {
-        write(opal_stacktrace_output_fileno, unable_to_print_msg, strlen(unable_to_print_msg));
+        opal_best_effort_write(opal_stacktrace_output_fileno, unable_to_print_msg,
+                               strlen(unable_to_print_msg));
     }
 
-    if( fileno(stdout) != opal_stacktrace_output_fileno &&
-        fileno(stderr) != opal_stacktrace_output_fileno ) {
+    if (fileno(stdout) != opal_stacktrace_output_fileno
+        && fileno(stderr) != opal_stacktrace_output_fileno) {
         close(opal_stacktrace_output_fileno);
         opal_stacktrace_output_fileno = -1;
     }
@@ -437,7 +508,6 @@ static void show_stackframe (int signo, siginfo_t * info, void * p)
 
 #endif /* OPAL_WANT_PRETTY_PRINT_STACKTRACE */
 
-
 #if OPAL_WANT_PRETTY_PRINT_STACKTRACE
 void opal_stackframe_output(int stream)
 {
@@ -455,17 +525,19 @@ void opal_stackframe_output(int stream)
         }
     } else {
         /* Do not print the stack trace */
-        if( 0 > opal_stacktrace_output_fileno && 0 == opal_stacktrace_output_filename_max_len ) {
+        if (0 > opal_stacktrace_output_fileno && 0 == opal_stacktrace_output_filename_max_len) {
             return;
         }
 
         /* Update the file name with the RANK, if available */
-        if( 0 < opal_stacktrace_output_filename_max_len ) {
+        if (0 < opal_stacktrace_output_filename_max_len) {
             set_stacktrace_filename();
             opal_stacktrace_output_fileno = open(opal_stacktrace_output_filename,
-                                                 O_CREAT|O_WRONLY|O_TRUNC, S_IRUSR|S_IWUSR);
-            if( 0 > opal_stacktrace_output_fileno ) {
-                opal_output(0, "Error: Failed to open the stacktrace output file. Default: stderr\n\tFilename: %s\n\tErrno: %s",
+                                                 O_CREAT | O_WRONLY | O_TRUNC, S_IRUSR | S_IWUSR);
+            if (0 > opal_stacktrace_output_fileno) {
+                opal_output(0,
+                            "Error: Failed to open the stacktrace output file. Default: "
+                            "stderr\n\tFilename: %s\n\tErrno: %s",
                             opal_stacktrace_output_filename, strerror(errno));
                 opal_stacktrace_output_fileno = fileno(stderr);
             }
@@ -473,8 +545,8 @@ void opal_stackframe_output(int stream)
 
         opal_backtrace_print(NULL, NULL, 2);
 
-        if( fileno(stdout) != opal_stacktrace_output_fileno &&
-            fileno(stderr) != opal_stacktrace_output_fileno ) {
+        if (fileno(stdout) != opal_stacktrace_output_fileno
+            && fileno(stderr) != opal_stacktrace_output_fileno) {
             close(opal_stacktrace_output_fileno);
             opal_stacktrace_output_fileno = -1;
         }
@@ -494,7 +566,7 @@ char *opal_stackframe_output_string(void)
 
     /* Calculate the space needed for the string */
     for (i = 3; i < traces_size; i++) {
-	    if (NULL == traces[i]) {
+        if (NULL == traces[i]) {
             break;
         }
         len += strlen(traces[i]) + 1;
@@ -530,12 +602,12 @@ char *opal_stackframe_output_string(void)
  *    is not a valid signal-number
  *
  */
-int opal_util_register_stackhandlers (void)
+int opal_util_register_stackhandlers(void)
 {
 #if OPAL_WANT_PRETTY_PRINT_STACKTRACE
     struct sigaction act, old;
-    char * tmp;
-    char * next;
+    char *tmp;
+    char *next;
     int i;
     bool complain, showed_help = false;
 
@@ -543,7 +615,7 @@ int opal_util_register_stackhandlers (void)
     strncpy(stacktrace_hostname, stacktrace_hostname_full, OPAL_MAXHOSTNAMELEN - 1);
     stacktrace_hostname[OPAL_MAXHOSTNAMELEN - 1] = '\0';
     /* to keep these somewhat readable, only print the machine name */
-    for (i = 0 ; i < (int)strlen(stacktrace_hostname) ; ++i) {
+    for (i = 0; i < (int) strlen(stacktrace_hostname); ++i) {
         if (stacktrace_hostname[i] == '.') {
             stacktrace_hostname[i] = '\0';
             break;
@@ -551,29 +623,25 @@ int opal_util_register_stackhandlers (void)
     }
 
     /* Setup the output stream to use */
-    if( NULL == opal_stacktrace_output_filename ||
-        0 == strcasecmp(opal_stacktrace_output_filename, "none") ) {
+    if (NULL == opal_stacktrace_output_filename
+        || 0 == strcasecmp(opal_stacktrace_output_filename, "none")) {
         opal_stacktrace_output_fileno = -1;
-    }
-    else if( 0 == strcasecmp(opal_stacktrace_output_filename, "stdout") ) {
+    } else if (0 == strcasecmp(opal_stacktrace_output_filename, "stdout")) {
         opal_stacktrace_output_fileno = fileno(stdout);
-    }
-    else if( 0 == strcasecmp(opal_stacktrace_output_filename, "stderr") ) {
+    } else if (0 == strcasecmp(opal_stacktrace_output_filename, "stderr")) {
         opal_stacktrace_output_fileno = fileno(stderr);
-    }
-    else if( 0 == strcasecmp(opal_stacktrace_output_filename, "file" ) ||
-             0 == strcasecmp(opal_stacktrace_output_filename, "file:") ) {
+    } else if (0 == strcasecmp(opal_stacktrace_output_filename, "file")
+               || 0 == strcasecmp(opal_stacktrace_output_filename, "file:")) {
         opal_stacktrace_output_filename_base = strdup("stacktrace");
 
         free(opal_stacktrace_output_filename);
         // Magic number: 8 = space for .PID and .RANK (allow 7 digits each)
         opal_stacktrace_output_filename_max_len = strlen("stacktrace") + 8 + 8;
-        opal_stacktrace_output_filename = (char*)malloc(sizeof(char) * opal_stacktrace_output_filename_max_len);
+        opal_stacktrace_output_filename = (char *) malloc(
+            sizeof(char) * opal_stacktrace_output_filename_max_len);
         set_stacktrace_filename();
         opal_stacktrace_output_fileno = -1;
-    }
-    else if( 0 == strncasecmp(opal_stacktrace_output_filename, "file:", 5) ) {
-        char *filename_cpy = NULL;
+    } else if (0 == strncasecmp(opal_stacktrace_output_filename, "file:", 5)) {
         next = strchr(opal_stacktrace_output_filename, ':');
         next++; // move past the ':' to the filename specified
 
@@ -581,85 +649,76 @@ int opal_util_register_stackhandlers (void)
 
         free(opal_stacktrace_output_filename);
         // Magic number: 8 = space for .PID and .RANK (allow 7 digits each)
-        opal_stacktrace_output_filename_max_len = strlen(opal_stacktrace_output_filename_base) + 8 + 8;
-        opal_stacktrace_output_filename = (char*)malloc(sizeof(char) * opal_stacktrace_output_filename_max_len);
+        opal_stacktrace_output_filename_max_len = strlen(opal_stacktrace_output_filename_base) + 8
+                                                  + 8;
+        opal_stacktrace_output_filename = (char *) malloc(
+            sizeof(char) * opal_stacktrace_output_filename_max_len);
         set_stacktrace_filename();
         opal_stacktrace_output_fileno = -1;
-
-        free(filename_cpy);
-    }
-    else {
+    } else {
         opal_stacktrace_output_fileno = fileno(stderr);
     }
 
-
     /* Setup the signals to catch */
     memset(&act, 0, sizeof(act));
     act.sa_sigaction = show_stackframe;
     act.sa_flags = SA_SIGINFO;
-#ifdef SA_ONESHOT
+#    ifdef SA_ONESHOT
     act.sa_flags |= SA_ONESHOT;
-#else
+#    else
     act.sa_flags |= SA_RESETHAND;
-#endif
+#    endif
+
+    for (tmp = next = opal_signal_string; next != NULL && *next != '\0'; tmp = next + 1) {
+        int sig;
+        int ret;
 
-    for (tmp = next = opal_signal_string ;
-	 next != NULL && *next != '\0';
-	 tmp = next + 1)
-    {
-      int sig;
-      int ret;
-
-      complain = false;
-      sig = strtol (tmp, &next, 10);
-
-      /*
-       *  If there is no sensible number in the string, exit.
-       *  Similarly for any number which is not in the signal-number range
-       */
-      if (((0 == sig) && (tmp == next)) || (0 > sig) || (_NSIG <= sig)) {
-          opal_show_help("help-opal-util.txt",
-                         "stacktrace bad signal", true,
-                         opal_signal_string, tmp);
-          return OPAL_ERR_SILENT;
-      } else if (next == NULL) {
-	 return OPAL_ERR_BAD_PARAM;
-      } else if (':' == *next &&
-                 0 == strncasecmp(next, ":complain", 9)) {
-          complain = true;
-          next += 9;
-      } else if (',' != *next && '\0' != *next) {
-          return OPAL_ERR_BAD_PARAM;
-      }
-
-      /* Just query first */
-      ret = sigaction (sig, NULL, &old);
-      if (0 != ret) {
-          return OPAL_ERR_IN_ERRNO;
-      }
-      /* Was there something already there? */
-      if (SIG_IGN != old.sa_handler && SIG_DFL != old.sa_handler) {
-          if (!showed_help && complain) {
-              /* JMS This is icky; there is no error message
-                 aggregation here so this message may be repeated for
-                 every single MPI process... */
-              opal_show_help("help-opal-util.txt",
-                             "stacktrace signal override",
-                             true, sig, sig, sig, opal_signal_string);
-              showed_help = true;
-          }
-      }
-
-      /* Nope, nothing was there, so put in ours */
-      else {
-          if (0 != sigaction(sig, &act, NULL)) {
-              return OPAL_ERR_IN_ERRNO;
-          }
-      }
+        complain = false;
+        sig = strtol(tmp, &next, 10);
+
+        /*
+         *  If there is no sensible number in the string, exit.
+         *  Similarly for any number which is not in the signal-number range
+         */
+        if (((0 == sig) && (tmp == next)) || (0 > sig) || (_NSIG <= sig)) {
+            opal_show_help("help-opal-util.txt", "stacktrace bad signal", true, opal_signal_string,
+                           tmp);
+            return OPAL_ERR_SILENT;
+        } else if (next == NULL) {
+            return OPAL_ERR_BAD_PARAM;
+        } else if (':' == *next && 0 == strncasecmp(next, ":complain", 9)) {
+            complain = true;
+            next += 9;
+        } else if (',' != *next && '\0' != *next) {
+            return OPAL_ERR_BAD_PARAM;
+        }
+
+        /* Just query first */
+        ret = sigaction(sig, NULL, &old);
+        if (0 != ret) {
+            return OPAL_ERR_IN_ERRNO;
+        }
+        /* Was there something already there? */
+        if (SIG_IGN != old.sa_handler && SIG_DFL != old.sa_handler) {
+            if (!showed_help && complain) {
+                /* JMS This is icky; there is no error message
+                   aggregation here so this message may be repeated for
+                   every single MPI process... */
+                opal_show_help("help-opal-util.txt", "stacktrace signal override", true, sig, sig,
+                               sig, opal_signal_string);
+                showed_help = true;
+            }
+        }
+
+        /* Nope, nothing was there, so put in ours */
+        else {
+            if (0 != sigaction(sig, &act, NULL)) {
+                return OPAL_ERR_IN_ERRNO;
+            }
+        }
     }
 
 #endif /* OPAL_WANT_PRETTY_PRINT_STACKTRACE */
 
     return OPAL_SUCCESS;
 }
-
diff --git a/opal/util/stacktrace.h b/opal/util/stacktrace.h
index c4484ae91aa..c30476eb094 100644
--- a/opal/util/stacktrace.h
+++ b/opal/util/stacktrace.h
@@ -53,6 +53,6 @@ OPAL_DECLSPEC char *opal_stackframe_output_string(void);
  *    is not a valid signal-number
  *
  */
-OPAL_DECLSPEC int opal_util_register_stackhandlers (void);
+OPAL_DECLSPEC int opal_util_register_stackhandlers(void);
 
 #endif /* OPAL_STACKTRACE_H */
diff --git a/opal/util/string_copy.c b/opal/util/string_copy.c
index 90e8112d7a0..531abb1f098 100644
--- a/opal/util/string_copy.c
+++ b/opal/util/string_copy.c
@@ -13,14 +13,13 @@
 
 #include "opal/util/string_copy.h"
 
-
 void opal_string_copy(char *dest, const char *src, size_t dest_len)
 {
     size_t i;
     char *new_dest = dest;
 
     // Open MPI does not do *giant* string copies.  Hence, we use the
-    // hueristic: if "dest_len" is too large, this is a programmer
+    // heuristic: if "dest_len" is too large, this is a programmer
     // error.  We pseudo-arbitrarily pick a large value to be the max
     // allowable dest_len: 128K.  If we ever need to increase this
     // value someday (because something has a legit reason to
diff --git a/opal/util/string_copy.h b/opal/util/string_copy.h
index 9d0827664c8..2945c7ffa60 100644
--- a/opal/util/string_copy.h
+++ b/opal/util/string_copy.h
@@ -21,7 +21,7 @@
 
 #include "opal_config.h"
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 
 BEGIN_C_DECLS
@@ -59,8 +59,7 @@ BEGIN_C_DECLS
  *   (len-1) characters of the source string will be copied to the
  *   destination, and dest[len-1] will be set to '\0'.
  */
-OPAL_DECLSPEC void opal_string_copy(char *dest, const char *src,
-                                    size_t dest_len)
+OPAL_DECLSPEC void opal_string_copy(char *dest, const char *src, size_t dest_len)
     __opal_attribute_nonnull__(1) __opal_attribute_nonnull__(2);
 
 /**
diff --git a/opal/util/sys_limits.c b/opal/util/sys_limits.c
index c6b95a35334..c8464be0290 100644
--- a/opal/util/sys_limits.c
+++ b/opal/util/sys_limits.c
@@ -31,35 +31,34 @@
 
 #include <errno.h>
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 #ifdef HAVE_SYS_TIME_H
-#include <sys/time.h>
+#    include <sys/time.h>
 #endif
 #ifdef HAVE_SYS_RESOURCE_H
-#include <sys/resource.h>
+#    include <sys/resource.h>
 #endif
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 
 #include "opal/constants.h"
 #include "opal/runtime/opal_params.h"
 
-#include "opal/util/sys_limits.h"
-#include "opal/util/show_help.h"
-#include "opal/util/output.h"
 #include "opal/util/argv.h"
+#include "opal/util/output.h"
+#include "opal/util/show_help.h"
+#include "opal/util/sys_limits.h"
 
 /*
  * Create and initialize storage for the system limits
  */
 OPAL_DECLSPEC opal_sys_limits_t opal_sys_limits = {
-    /* initialized = */     false,
-    /* num_files   = */     -1,
-    /* num_procs   = */     -1,
-    /* file_size   = */      0
-};
+    /* initialized = */ false,
+    /* num_files   = */ -1,
+    /* num_procs   = */ -1,
+    /* file_size   = */ 0};
 
 static int opal_setlimit(int resource, char *value, rlim_t *out)
 {
@@ -69,9 +68,9 @@ static int opal_setlimit(int resource, char *value, rlim_t *out)
     rlim.rlim_cur = 0;
 
     if (0 == strcmp(value, "max")) {
-            maxlim = -1;
+        maxlim = -1;
     } else if (0 == strncmp(value, "unlimited", strlen(value))) {
-            maxlim = RLIM_INFINITY;
+        maxlim = RLIM_INFINITY;
     } else {
         maxlim = strtol(value, NULL, 10);
     }
@@ -109,7 +108,7 @@ static int opal_setlimit(int resource, char *value, rlim_t *out)
 
 int opal_util_init_sys_limits(char **errmsg)
 {
-    char **lims, **lim=NULL, *setlim;
+    char **lims, **lim = NULL, *setlim;
     int i, rc = OPAL_ERROR;
     rlim_t value;
 
@@ -125,7 +124,7 @@ int opal_util_init_sys_limits(char **errmsg)
     }
 
     /* each limit is expressed as a "param:value" pair */
-    for (i=0; NULL != lims[i]; i++) {
+    for (i = 0; NULL != lims[i]; i++) {
         lim = opal_argv_split(lims[i], ':');
         if (1 == opal_argv_count(lim)) {
             setlim = "max";
@@ -139,24 +138,25 @@ int opal_util_init_sys_limits(char **errmsg)
          */
         if (0 == strcmp(lim[0], "1")) {
 #if HAVE_DECL_RLIMIT_NOFILE
-            if (OPAL_SUCCESS !=
-                opal_setlimit(RLIMIT_NOFILE, "max", &value)) {
-                *errmsg = opal_show_help_string("help-opal-util.txt", "sys-limit-failed", true, "openfiles", "max");
+            if (OPAL_SUCCESS != opal_setlimit(RLIMIT_NOFILE, "max", &value)) {
+                *errmsg = opal_show_help_string("help-opal-util.txt", "sys-limit-failed", true,
+                                                "openfiles", "max");
                 goto out;
             }
             opal_sys_limits.num_files = value;
 #endif
 #if HAVE_DECL_RLIMIT_NPROC
             if (OPAL_SUCCESS != opal_setlimit(RLIMIT_NPROC, "max", &value)) {
-                *errmsg = opal_show_help_string("help-opal-util.txt", "sys-limit-failed", true, "maxchildren", "max");
+                *errmsg = opal_show_help_string("help-opal-util.txt", "sys-limit-failed", true,
+                                                "maxchildren", "max");
                 goto out;
             }
             opal_sys_limits.num_procs = value;
 #endif
 #if HAVE_DECL_RLIMIT_FSIZE
-            if (OPAL_SUCCESS !=
-                opal_setlimit(RLIMIT_FSIZE, "max", &value)) {
-                *errmsg = opal_show_help_string("help-opal-util.txt", "sys-limit-failed", true, "filesize", "max");
+            if (OPAL_SUCCESS != opal_setlimit(RLIMIT_FSIZE, "max", &value)) {
+                *errmsg = opal_show_help_string("help-opal-util.txt", "sys-limit-failed", true,
+                                                "filesize", "max");
                 goto out;
             }
             opal_sys_limits.file_size = value;
@@ -171,14 +171,16 @@ int opal_util_init_sys_limits(char **errmsg)
         if (0 == strcmp(lim[0], "core")) {
 #if HAVE_DECL_RLIMIT_CORE
             if (OPAL_SUCCESS != opal_setlimit(RLIMIT_CORE, setlim, &value)) {
-                *errmsg = opal_show_help_string("help-opal-util.txt", "sys-limit-failed", true, "openfiles", setlim);
+                *errmsg = opal_show_help_string("help-opal-util.txt", "sys-limit-failed", true,
+                                                "openfiles", setlim);
                 goto out;
             }
 #endif
         } else if (0 == strcmp(lim[0], "filesize")) {
 #if HAVE_DECL_RLIMIT_FSIZE
             if (OPAL_SUCCESS != opal_setlimit(RLIMIT_FSIZE, setlim, &value)) {
-                *errmsg = opal_show_help_string("help-opal-util.txt", "sys-limit-failed", true, "filesize", setlim);
+                *errmsg = opal_show_help_string("help-opal-util.txt", "sys-limit-failed", true,
+                                                "filesize", setlim);
                 goto out;
             }
             opal_sys_limits.file_size = value;
@@ -186,14 +188,16 @@ int opal_util_init_sys_limits(char **errmsg)
         } else if (0 == strcmp(lim[0], "maxmem")) {
 #if HAVE_DECL_RLIMIT_AS
             if (OPAL_SUCCESS != opal_setlimit(RLIMIT_AS, setlim, &value)) {
-                *errmsg = opal_show_help_string("help-opal-util.txt", "sys-limit-failed", true, "maxmem", setlim);
+                *errmsg = opal_show_help_string("help-opal-util.txt", "sys-limit-failed", true,
+                                                "maxmem", setlim);
                 goto out;
             }
 #endif
         } else if (0 == strcmp(lim[0], "openfiles")) {
 #if HAVE_DECL_RLIMIT_NOFILE
             if (OPAL_SUCCESS != opal_setlimit(RLIMIT_NOFILE, setlim, &value)) {
-                *errmsg = opal_show_help_string("help-opal-util.txt", "sys-limit-failed", true, "openfiles", setlim);
+                *errmsg = opal_show_help_string("help-opal-util.txt", "sys-limit-failed", true,
+                                                "openfiles", setlim);
                 goto out;
             }
             opal_sys_limits.num_files = value;
@@ -201,20 +205,23 @@ int opal_util_init_sys_limits(char **errmsg)
         } else if (0 == strcmp(lim[0], "stacksize")) {
 #if HAVE_DECL_RLIMIT_STACK
             if (OPAL_SUCCESS != opal_setlimit(RLIMIT_STACK, setlim, &value)) {
-                *errmsg = opal_show_help_string("help-opal-util.txt", "sys-limit-failed", true, "stacksize", setlim);
+                *errmsg = opal_show_help_string("help-opal-util.txt", "sys-limit-failed", true,
+                                                "stacksize", setlim);
                 goto out;
             }
 #endif
         } else if (0 == strcmp(lim[0], "maxchildren")) {
 #if HAVE_DECL_RLIMIT_NPROC
             if (OPAL_SUCCESS != opal_setlimit(RLIMIT_NPROC, setlim, &value)) {
-                *errmsg = opal_show_help_string("help-opal-util.txt", "sys-limit-failed", true, "maxchildren", setlim);
+                *errmsg = opal_show_help_string("help-opal-util.txt", "sys-limit-failed", true,
+                                                "maxchildren", setlim);
                 goto out;
             }
             opal_sys_limits.num_procs = value;
 #endif
         } else {
-            *errmsg = opal_show_help_string("help-opal-util.txt", "sys-limit-unrecognized", true, lim[0], setlim);
+            *errmsg = opal_show_help_string("help-opal-util.txt", "sys-limit-unrecognized", true,
+                                            lim[0], setlim);
             goto out;
         }
         opal_argv_free(lim);
@@ -240,13 +247,13 @@ int opal_getpagesize(void)
     static int page_size = -1;
 
     if (page_size != -1) {
-// testing in a loop showed sysconf() took ~5 usec vs ~0.3 usec with it cached
+        // testing in a loop showed sysconf() took ~5 usec vs ~0.3 usec with it cached
         return page_size;
     }
 
 #ifdef HAVE_GETPAGESIZE
     return page_size = getpagesize();
-#elif defined(_SC_PAGESIZE )
+#elif defined(_SC_PAGESIZE)
     return page_size = sysconf(_SC_PAGESIZE);
 #elif defined(_SC_PAGE_SIZE)
     return page_size = sysconf(_SC_PAGE_SIZE);
diff --git a/opal/util/sys_limits.h b/opal/util/sys_limits.h
index 728f2d4b72d..3d1e5bb00a0 100644
--- a/opal/util/sys_limits.h
+++ b/opal/util/sys_limits.h
@@ -25,7 +25,7 @@
 #include "opal_config.h"
 
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 
 BEGIN_C_DECLS
@@ -55,7 +55,6 @@ OPAL_DECLSPEC int opal_util_init_sys_limits(char **errmsg);
  */
 OPAL_DECLSPEC int opal_getpagesize(void);
 
-
 END_C_DECLS
 
 #endif /* OPAL_STRNCPY_H */
diff --git a/opal/util/timings.c b/opal/util/timings.c
index 7e9981fcb3c..2a765d77272 100644
--- a/opal/util/timings.c
+++ b/opal/util/timings.c
@@ -2,6 +2,8 @@
  * Copyright (C) 2014      Artem Polyakov <artpol84@gmail.com>
  * Copyright (c) 2014      Intel, Inc. All rights reserved.
  * Copyright (c) 2017      Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2023      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -11,37 +13,38 @@
 
 #include "opal_config.h"
 
-#include <stdlib.h>
 #include <stdarg.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <unistd.h>
 
 #include <string.h>
 
 #include <errno.h>
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 #ifdef HAVE_SYS_TIME_H
-#include <sys/time.h>
+#    include <sys/time.h>
 #endif
 #ifdef HAVE_SYS_RESOURCE_H
-#include <sys/resource.h>
+#    include <sys/resource.h>
 #endif
 
 #include "opal/constants.h"
 #include "opal/runtime/opal_params.h"
 
-
-#include "opal/class/opal_pointer_array.h"
 #include "opal/class/opal_list.h"
-#include "opal/util/timings.h"
-#include "opal/util/output.h"
-#include "opal/util/basename.h"
+#include "opal/class/opal_pointer_array.h"
 #include "opal/mca/timer/timer.h"
+#include "opal/util/basename.h"
+#include "opal/util/output.h"
+#include "opal/util/timings.h"
 
 #include MCA_timer_IMPLEMENTATION_HEADER
 
+static bool opal_timer_native_timers_avail = false;
+
 static double get_ts_gettimeofday(void)
 {
     double ret;
@@ -49,52 +52,62 @@ static double get_ts_gettimeofday(void)
     struct timeval tv;
     gettimeofday(&tv, NULL);
     ret = tv.tv_sec;
-    ret += (double)tv.tv_usec / 1000000.0;
+    ret += (double) tv.tv_usec / 1000000.0;
     return ret;
 }
 
 #if OPAL_TIMER_CYCLE_NATIVE
 static double get_ts_cycle(void)
 {
-        return ((double) opal_timer_base_get_cycles()) / opal_timer_base_get_freq();
+    return ((double) opal_timer_base_get_cycles()) / opal_timer_base_get_freq();
 }
 #endif
 
 #if OPAL_TIMER_USEC_NATIVE
 static double get_ts_usec(void)
 {
-        return ((double) opal_timer_base_get_usec()) / 1000000.0;
+    return ((double) opal_timer_base_get_usec()) / 1000000.0;
 }
 #endif
 
+void opal_timing_enable_native_timers(void)
+{
+    opal_timer_native_timers_avail = true;
+}
+
+void opal_timing_disable_native_timers(void)
+{
+    opal_timer_native_timers_avail = false;
+}
+
+
 opal_timing_ts_func_t opal_timing_ts_func(opal_timer_type_t type)
 {
     switch (type) {
-        case OPAL_TIMING_GET_TIME_OF_DAY:
-            return get_ts_gettimeofday;
+    case OPAL_TIMING_GET_TIME_OF_DAY:
+        return get_ts_gettimeofday;
 
-        case OPAL_TIMING_CYCLE_NATIVE:
+    case OPAL_TIMING_CYCLE_NATIVE:
 #if OPAL_TIMER_CYCLE_NATIVE
-           return get_ts_cycle;
+        return get_ts_cycle;
 #else
-           return NULL;
+        return NULL;
 #endif // OPAL_TIMER_CYCLE_NATIVE
-        case OPAL_TIMING_USEC_NATIVE:
+    case OPAL_TIMING_USEC_NATIVE:
 #if OPAL_TIMER_USEC_NATIVE
-            return get_ts_usec;
+        return get_ts_usec;
 #else
-            return NULL;
+        return NULL;
 #endif // OPAL_TIMER_USEC_NATIVE
-        default:
-            if( !opal_initialized ){
-                return get_ts_gettimeofday;
-            }
+    default:
+        if( false == opal_timer_native_timers_avail ){
+            return get_ts_gettimeofday;
+        }
 #if OPAL_TIMER_CYCLE_NATIVE
-            return get_ts_cycle;
+        return get_ts_cycle;
 #elif OPAL_TIMER_USEC_NATIVE
-            return get_ts_usec;
+        return get_ts_usec;
 #endif
-            return get_ts_gettimeofday;
+        return get_ts_gettimeofday;
     }
 }
-
diff --git a/opal/util/timings.h b/opal/util/timings.h
index 6d561472d77..6dfc2ef6440 100644
--- a/opal/util/timings.h
+++ b/opal/util/timings.h
@@ -2,6 +2,8 @@
  * Copyright (C) 2014      Artem Polyakov <artpol84@gmail.com>
  * Copyright (c) 2014-2017 Intel, Inc. All rights reserved.
  * Copyright (c) 2017-2018 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2023      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -28,7 +30,7 @@ typedef enum {
 
 typedef double (*opal_timing_ts_func_t)(void);
 
-#define OPAL_TIMING_STR_LEN 256
+#    define OPAL_TIMING_STR_LEN 256
 
 typedef struct {
     char id[OPAL_TIMING_STR_LEN], cntr_env[OPAL_TIMING_STR_LEN];
@@ -39,180 +41,192 @@ typedef struct {
 } opal_timing_env_t;
 
 opal_timing_ts_func_t opal_timing_ts_func(opal_timer_type_t type);
-
-#define OPAL_TIMING_ENV_START_TYPE(func, _nm, type, prefix)                       \
-    do {                                                                          \
-        char *ptr = NULL;                                                         \
-        char *_prefix = prefix;                                                   \
-        int n;                                                                    \
-        if( NULL == prefix ){                                                     \
-            _prefix = "";                                                         \
-        }                                                                         \
-        (_nm)->error = 0;                                                         \
-        n = snprintf((_nm)->id, OPAL_TIMING_STR_LEN, "%s_%s", _prefix, func);     \
-        if( n > OPAL_TIMING_STR_LEN ){                                            \
-             (_nm)->error = 1;                                                    \
-        }                                                                         \
-        n = sprintf((_nm)->cntr_env,"OMPI_TIMING_%s_CNT", (_nm)->id);             \
-        if( n > OPAL_TIMING_STR_LEN ){                                            \
-            (_nm)->error = 1;                                                     \
-        }                                                                         \
-        ptr = getenv((_nm)->id);                                                  \
-        if( NULL == ptr || strcmp(ptr, "1")){                                     \
-            (_nm)->enabled = 0;                                                   \
-        }                                                                         \
-        (_nm)->get_ts = opal_timing_ts_func(type);                                \
-        ptr = getenv("OPAL_TIMING_ENABLE");                                       \
-        if (NULL != ptr) {                                                        \
-            (_nm)->enabled = atoi(ptr);                                           \
-        }                                                                         \
-        (_nm)->cntr = 0;                                                          \
-        ptr = getenv((_nm)->id);                                                  \
-        if( NULL != ptr ){                                                        \
-            (_nm)->cntr = atoi(ptr);                                              \
-        }                                                                         \
-        (_nm)->ts = (_nm)->get_ts();                                              \
-        if ( 0 != (_nm)->error ){                                                 \
-            (_nm)->enabled = 0;                                                   \
-        }                                                                         \
-    } while(0)
-
-/* We use function names for identification
- * however this might be a problem for the private
- * functions declared as static as their names may
- * conflict.
- * Use prefix to do a finer-grained identification if needed
- */
-#define OPAL_TIMING_ENV_INIT_PREFIX(prefix, name)                                 \
-    opal_timing_env_t name ## _val, *name = &(name ## _val);                      \
-    OPAL_TIMING_ENV_START_TYPE(__func__, name, OPAL_TIMING_AUTOMATIC_TIMER, prefix);
-
-#define OPAL_TIMING_ENV_INIT(name) OPAL_TIMING_ENV_INIT_PREFIX("", name)
-
-#define OPAL_TIMING_ENV_NEXT(h, ...)                                              \
-    do {                                                                          \
-        int n;                                                                    \
-        char buf1[OPAL_TIMING_STR_LEN], buf2[OPAL_TIMING_STR_LEN];                \
-        double time;                                                              \
-        char *filename;                                                           \
-        if( h->enabled ){                                                         \
-            /* enabled codepath */                                                \
-            time = h->get_ts() - h->ts;                                           \
-            n = snprintf(buf1, OPAL_TIMING_STR_LEN, "OMPI_TIMING_%s_DESC_%d", h->id, h->cntr); \
-            if ( n > OPAL_TIMING_STR_LEN ){                                       \
-                h->error = 1;                                                     \
-            }                                                                     \
-            n = snprintf(buf2, OPAL_TIMING_STR_LEN, __VA_ARGS__ );                \
-            if ( n > OPAL_TIMING_STR_LEN ){                                       \
-                h->error = 1;                                                     \
+void opal_timing_enable_native_timers(void);
+void opal_timing_disable_native_timers(void);
+
+#    define OPAL_TIMING_ENABLE_NATIVE_TIMERS  opal_timing_enable_native_timers()
+#    define OPAL_TIMING_DISABLE_NATIVE_TIMERS  opal_timing_disable_native_timers()
+
+#    define OPAL_TIMING_ENV_START_TYPE(func, _nm, type, prefix)                   \
+        do {                                                                      \
+            char *ptr = NULL;                                                     \
+            char *_prefix = prefix;                                               \
+            int n;                                                                \
+            if (NULL == prefix) {                                                 \
+                _prefix = "";                                                     \
             }                                                                     \
-            setenv(buf1, buf2, 1);                                                \
-            n = snprintf(buf1, OPAL_TIMING_STR_LEN, "OMPI_TIMING_%s_VAL_%d", h->id, h->cntr);  \
-            if ( n > OPAL_TIMING_STR_LEN ){                                       \
-                h->error = 1;                                                     \
+            (_nm)->error = 0;                                                     \
+            n = snprintf((_nm)->id, OPAL_TIMING_STR_LEN, "%s_%s", _prefix, func); \
+            if (n > OPAL_TIMING_STR_LEN) {                                        \
+                (_nm)->error = 1;                                                 \
             }                                                                     \
-            n = snprintf(buf2, OPAL_TIMING_STR_LEN, "%lf", time);                 \
-            if ( n > OPAL_TIMING_STR_LEN ){                                       \
-                h->error = 1;                                                     \
+            n = sprintf((_nm)->cntr_env, "OMPI_TIMING_%s_CNT", (_nm)->id);        \
+            if (n > OPAL_TIMING_STR_LEN) {                                        \
+                (_nm)->error = 1;                                                 \
             }                                                                     \
-            setenv(buf1, buf2, 1);                                                \
-            filename = strrchr(__FILE__, '/');                                    \
-            filename = (filename == NULL) ? strdup(__FILE__) : filename+1;        \
-            n = snprintf(buf1, OPAL_TIMING_STR_LEN, "OMPI_TIMING_%s_FILE_%d", h->id, h->cntr); \
-            if ( n > OPAL_TIMING_STR_LEN ){                                       \
-                h->error = 1;                                                     \
+            ptr = getenv((_nm)->id);                                              \
+            if (NULL == ptr || strcmp(ptr, "1")) {                                \
+                (_nm)->enabled = 0;                                               \
             }                                                                     \
-            n = snprintf(buf2, OPAL_TIMING_STR_LEN, "%s", filename);              \
-            if ( n > OPAL_TIMING_STR_LEN ){                                       \
-                h->error = 1;                                                     \
+            (_nm)->get_ts = opal_timing_ts_func(type);                            \
+            ptr = getenv("OPAL_TIMING_ENABLE");                                   \
+            if (NULL != ptr) {                                                    \
+                (_nm)->enabled = atoi(ptr);                                       \
             }                                                                     \
-            setenv(buf1, buf2, 1);                                                \
-            h->cntr++;                                                            \
-            sprintf(buf1, "%d", h->cntr);                                         \
-            setenv(h->cntr_env, buf1, 1);                                         \
-            /* We don't include env operations into the consideration.
-             * Hopefully this will help to make measurements more accurate.
-             */                                                                   \
-            h->ts = h->get_ts();                                                  \
-        }                                                                         \
-        if (h->error) {                                                           \
-            n = snprintf(buf1, OPAL_TIMING_STR_LEN, "OMPI_TIMING_%s_ERROR", h->id);\
-            if ( n > OPAL_TIMING_STR_LEN ){                                       \
-                h->error = 1;                                                     \
+            (_nm)->cntr = 0;                                                      \
+            ptr = getenv((_nm)->id);                                              \
+            if (NULL != ptr) {                                                    \
+                (_nm)->cntr = atoi(ptr);                                          \
             }                                                                     \
-            n = snprintf(buf2, OPAL_TIMING_STR_LEN, "%d", h->error);              \
-            if ( n > OPAL_TIMING_STR_LEN ){                                       \
-                h->error = 1;                                                     \
+            (_nm)->ts = (_nm)->get_ts();                                          \
+            if (0 != (_nm)->error) {                                              \
+                (_nm)->enabled = 0;                                               \
             }                                                                     \
-            setenv(buf1, buf2, 1);                                                \
-        }                                                                         \
-    } while(0)
+        } while (0)
+
+/* We use function names for identification
+ * however this might be a problem for the private
+ * functions declared as static as their names may
+ * conflict.
+ * Use prefix to do a finer-grained identification if needed
+ */
+#    define OPAL_TIMING_ENV_INIT_PREFIX(prefix, name)        \
+        opal_timing_env_t name##_val, *name = &(name##_val); \
+        OPAL_TIMING_ENV_START_TYPE(__func__, name, OPAL_TIMING_AUTOMATIC_TIMER, prefix);
+
+#    define OPAL_TIMING_ENV_INIT(name) OPAL_TIMING_ENV_INIT_PREFIX("", name)
+
+#    define OPAL_TIMING_ENV_NEXT(h, ...)                                                           \
+        do {                                                                                       \
+            int n;                                                                                 \
+            char buf1[OPAL_TIMING_STR_LEN], buf2[OPAL_TIMING_STR_LEN];                             \
+            double time;                                                                           \
+            char *filename;                                                                        \
+            if (h->enabled) {                                                                      \
+                /* enabled codepath */                                                             \
+                time = h->get_ts() - h->ts;                                                        \
+                n = snprintf(buf1, OPAL_TIMING_STR_LEN, "OMPI_TIMING_%s_DESC_%d", h->id, h->cntr); \
+                if (n > OPAL_TIMING_STR_LEN) {                                                     \
+                    h->error = 1;                                                                  \
+                }                                                                                  \
+                n = snprintf(buf2, OPAL_TIMING_STR_LEN, __VA_ARGS__);                              \
+                if (n > OPAL_TIMING_STR_LEN) {                                                     \
+                    h->error = 1;                                                                  \
+                }                                                                                  \
+                setenv(buf1, buf2, 1);                                                             \
+                n = snprintf(buf1, OPAL_TIMING_STR_LEN, "OMPI_TIMING_%s_VAL_%d", h->id, h->cntr);  \
+                if (n > OPAL_TIMING_STR_LEN) {                                                     \
+                    h->error = 1;                                                                  \
+                }                                                                                  \
+                n = snprintf(buf2, OPAL_TIMING_STR_LEN, "%lf", time);                              \
+                if (n > OPAL_TIMING_STR_LEN) {                                                     \
+                    h->error = 1;                                                                  \
+                }                                                                                  \
+                setenv(buf1, buf2, 1);                                                             \
+                filename = strrchr(__FILE__, '/');                                                 \
+                filename = (filename == NULL) ? strdup(__FILE__) : filename + 1;                   \
+                n = snprintf(buf1, OPAL_TIMING_STR_LEN, "OMPI_TIMING_%s_FILE_%d", h->id, h->cntr); \
+                if (n > OPAL_TIMING_STR_LEN) {                                                     \
+                    h->error = 1;                                                                  \
+                }                                                                                  \
+                n = snprintf(buf2, OPAL_TIMING_STR_LEN, "%s", filename);                           \
+                if (n > OPAL_TIMING_STR_LEN) {                                                     \
+                    h->error = 1;                                                                  \
+                }                                                                                  \
+                setenv(buf1, buf2, 1);                                                             \
+                h->cntr++;                                                                         \
+                sprintf(buf1, "%d", h->cntr);                                                      \
+                setenv(h->cntr_env, buf1, 1);                                                      \
+                /* We don't include env operations into the consideration.                         \
+                 * Hopefully this will help to make measurements more accurate.                    \
+                 */                                                                                \
+                h->ts = h->get_ts();                                                               \
+            }                                                                                      \
+            if (h->error) {                                                                        \
+                n = snprintf(buf1, OPAL_TIMING_STR_LEN, "OMPI_TIMING_%s_ERROR", h->id);            \
+                if (n > OPAL_TIMING_STR_LEN) {                                                     \
+                    h->error = 1;                                                                  \
+                }                                                                                  \
+                n = snprintf(buf2, OPAL_TIMING_STR_LEN, "%d", h->error);                           \
+                if (n > OPAL_TIMING_STR_LEN) {                                                     \
+                    h->error = 1;                                                                  \
+                }                                                                                  \
+                setenv(buf1, buf2, 1);                                                             \
+            }                                                                                      \
+        } while (0)
 
 /* This function supposed to be called from the code that will
  * do the postprocessing, i.e. OMPI timing portion that will
  * do the reduction of accumulated values
  */
-#define OPAL_TIMING_ENV_CNT_PREFIX(prefix, func, _cnt)                            \
-    do {                                                                          \
-        char ename[OPAL_TIMING_STR_LEN];                                          \
-        char *ptr = NULL;                                                         \
-        int n = snprintf(ename, OPAL_TIMING_STR_LEN, "OMPI_TIMING_%s_%s_CNT", prefix, func);    \
-        (_cnt) = 0;                                                               \
-        if ( n <= OPAL_TIMING_STR_LEN ){                                          \
-            ptr = getenv(ename);                                                  \
-            if( NULL != ptr ){ (_cnt) = atoi(ptr); };                             \
-        }                                                                         \
-    } while(0)
-
-#define OPAL_TIMING_ENV_ERROR_PREFIX(prefix, func, _err)                          \
-    do {                                                                          \
-        char ename[OPAL_TIMING_STR_LEN];                                          \
-        (_err) = 0;                                                               \
-        char *ptr = NULL;                                                         \
-        int n = snprintf(ename, OPAL_TIMING_STR_LEN, "OMPI_TIMING_%s%s_ERROR", prefix, func);    \
-        if ( n <= OPAL_TIMING_STR_LEN ){                                          \
-            ptr = getenv(ename);                                                  \
-            if( NULL != ptr ){ (_err) = atoi(ptr); };                             \
-        }                                                                         \
-    } while(0)
-
-#define OPAL_TIMING_ENV_GETDESC_PREFIX(prefix, filename, func, i, desc, _t)       \
-    do {                                                                          \
-        char vname[OPAL_TIMING_STR_LEN];                                          \
-        (_t) = 0.0;                                                               \
-        sprintf(vname, "OMPI_TIMING_%s_%s_FILE_%d", prefix, func, i);              \
-        *filename = getenv(vname);                                                \
-        sprintf(vname, "OMPI_TIMING_%s_%s_DESC_%d", prefix, func, i);              \
-        *desc = getenv(vname);                                                    \
-        sprintf(vname, "OMPI_TIMING_%s_%s_VAL_%d", prefix, func, i);               \
-        char *ptr = getenv(vname);                                                \
-        if ( NULL != ptr ) {                                                      \
-            sscanf(ptr,"%lf", &(_t));                                             \
-        }                                                                         \
-    } while(0)
-
-#define OPAL_TIMING_ENV_GETDESC(file, func, index, desc)                          \
-    OPAL_TIMING_ENV_GETDESC_PREFIX("", file, func, index, desc)
+#    define OPAL_TIMING_ENV_CNT_PREFIX(prefix, func, _cnt)                                       \
+        do {                                                                                     \
+            char ename[OPAL_TIMING_STR_LEN];                                                     \
+            char *ptr = NULL;                                                                    \
+            int n = snprintf(ename, OPAL_TIMING_STR_LEN, "OMPI_TIMING_%s_%s_CNT", prefix, func); \
+            (_cnt) = 0;                                                                          \
+            if (n <= OPAL_TIMING_STR_LEN) {                                                      \
+                ptr = getenv(ename);                                                             \
+                if (NULL != ptr) {                                                               \
+                    (_cnt) = atoi(ptr);                                                          \
+                };                                                                               \
+            }                                                                                    \
+        } while (0)
+
+#    define OPAL_TIMING_ENV_ERROR_PREFIX(prefix, func, _err)                                      \
+        do {                                                                                      \
+            char ename[OPAL_TIMING_STR_LEN];                                                      \
+            (_err) = 0;                                                                           \
+            char *ptr = NULL;                                                                     \
+            int n = snprintf(ename, OPAL_TIMING_STR_LEN, "OMPI_TIMING_%s%s_ERROR", prefix, func); \
+            if (n <= OPAL_TIMING_STR_LEN) {                                                       \
+                ptr = getenv(ename);                                                              \
+                if (NULL != ptr) {                                                                \
+                    (_err) = atoi(ptr);                                                           \
+                };                                                                                \
+            }                                                                                     \
+        } while (0)
+
+#    define OPAL_TIMING_ENV_GETDESC_PREFIX(prefix, filename, func, i, desc, _t) \
+        do {                                                                    \
+            char vname[OPAL_TIMING_STR_LEN];                                    \
+            (_t) = 0.0;                                                         \
+            sprintf(vname, "OMPI_TIMING_%s_%s_FILE_%d", prefix, func, i);       \
+            *filename = getenv(vname);                                          \
+            sprintf(vname, "OMPI_TIMING_%s_%s_DESC_%d", prefix, func, i);       \
+            *desc = getenv(vname);                                              \
+            sprintf(vname, "OMPI_TIMING_%s_%s_VAL_%d", prefix, func, i);        \
+            char *ptr = getenv(vname);                                          \
+            if (NULL != ptr) {                                                  \
+                sscanf(ptr, "%lf", &(_t));                                      \
+            }                                                                   \
+        } while (0)
+
+#    define OPAL_TIMING_ENV_GETDESC(file, func, index, desc) \
+        OPAL_TIMING_ENV_GETDESC_PREFIX("", file, func, index, desc)
 
 #else
 
-#define OPAL_TIMING_ENV_START_TYPE(func, type, prefix)
+#    define OPAL_TIMING_ENABLE_NATIVE_TIMERS
+#    define OPAL_TIMING_DISABLE_NATIVE_TIMERS
+
+#    define OPAL_TIMING_ENV_START_TYPE(func, type, prefix)
 
-#define OPAL_TIMING_ENV_INIT(name)
+#    define OPAL_TIMING_ENV_INIT(name)
 
-#define OPAL_TIMING_ENV_INIT_PREFIX(prefix, name)
+#    define OPAL_TIMING_ENV_INIT_PREFIX(prefix, name)
 
-#define OPAL_TIMING_ENV_NEXT(h, ... )
+#    define OPAL_TIMING_ENV_NEXT(h, ...)
 
-#define OPAL_TIMING_ENV_CNT_PREFIX(prefix, func)
+#    define OPAL_TIMING_ENV_CNT_PREFIX(prefix, func)
 
-#define OPAL_TIMING_ENV_CNT(func)
+#    define OPAL_TIMING_ENV_CNT(func)
 
-#define OPAL_TIMING_ENV_GETDESC_PREFIX(prefix, func, i, desc)
+#    define OPAL_TIMING_ENV_GETDESC_PREFIX(prefix, func, i, desc)
 
-#define OPAL_TIMING_ENV_GETDESC(func, index, desc)
+#    define OPAL_TIMING_ENV_GETDESC(func, index, desc)
 
-#define OPAL_TIMING_ENV_ERROR_PREFIX(prefix, func)
+#    define OPAL_TIMING_ENV_ERROR_PREFIX(prefix, func)
 
 #endif
 
diff --git a/opal/util/uri.c b/opal/util/uri.c
index 1e1e154b972..233037ee246 100644
--- a/opal/util/uri.c
+++ b/opal/util/uri.c
@@ -17,17 +17,17 @@
 #include <stdlib.h>
 #include <string.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 
 #include "opal/util/output.h"
 #include "opal/util/path.h"
-#include "opal/util/show_help.h"
 #include "opal/util/printf.h"
+#include "opal/util/show_help.h"
 
 #include "opal/util/uri.h"
 
-const static char *uri_reserved_path_chars="!$&'()*+,;=:@ ";
+const static char *uri_reserved_path_chars = "!$&'()*+,;=:@ ";
 
 char *opal_uri_get_scheme(const char *uri)
 {
@@ -35,8 +35,7 @@ char *opal_uri_get_scheme(const char *uri)
     char *ptr;
 
     if (NULL == (ptr = strchr(turi, ':'))) {
-        opal_show_help("help-opal-util.txt", "malformed-uri",
-                       true, uri);
+        opal_show_help("help-opal-util.txt", "malformed-uri", true, uri);
         free(turi);
         return NULL;
     }
@@ -44,16 +43,14 @@ char *opal_uri_get_scheme(const char *uri)
     return turi;
 }
 
-char *opal_filename_to_uri(const char *filename,
-                           const char *hostname)
+char *opal_filename_to_uri(const char *filename, const char *hostname)
 {
     char *uri, *fn;
     size_t i, j, k, n;
 
     /* filename must be an absolute path */
     if (!opal_path_is_absolute(filename)) {
-        opal_show_help("help-opal-util.txt", "relative-path",
-                       true, filename);
+        opal_show_help("help-opal-util.txt", "relative-path", true, filename);
         return NULL;
     }
 
@@ -68,18 +65,18 @@ char *opal_filename_to_uri(const char *filename,
     /* count the number of characters that require escaping
      * in the filename
      */
-    n=0;
-    for (j=0; j < strlen(uri_reserved_path_chars)-1; j++) {
+    n = 0;
+    for (j = 0; j < strlen(uri_reserved_path_chars) - 1; j++) {
         if (NULL != strchr(filename, uri_reserved_path_chars[j])) {
             n++;
         }
     }
     /* escape them if necessary */
     if (0 < n) {
-        fn = (char*)malloc(strlen(filename) + n + 1);
-        i=0;
-        for (k=0; k < strlen(filename)-1; k++) {
-            for (j=0; j < strlen(uri_reserved_path_chars)-1; j++) {
+        fn = (char *) malloc(strlen(filename) + n + 1);
+        i = 0;
+        for (k = 0; k < strlen(filename) - 1; k++) {
+            for (j = 0; j < strlen(uri_reserved_path_chars) - 1; j++) {
                 if (filename[k] == uri_reserved_path_chars[j]) {
                     fn[i] = '\\';
                     i++;
@@ -103,8 +100,7 @@ char *opal_filename_to_uri(const char *filename,
     return uri;
 }
 
-char *opal_filename_from_uri(const char *uri,
-                             char **hostname)
+char *opal_filename_from_uri(const char *uri, char **hostname)
 {
     char *turi;
     char *ptr, *fn, *sp;
@@ -120,13 +116,12 @@ char *opal_filename_from_uri(const char *uri,
 
     /* extract the scheme */
     if (NULL == (ptr = strchr(turi, ':'))) {
-        opal_show_help("help-opal-util.txt", "malformed-uri",
-                       true, uri);
+        opal_show_help("help-opal-util.txt", "malformed-uri", true, uri);
         free(turi);
         return NULL;
     }
     *ptr = '\0';
-    ptr++;  /* step over the new NULL */
+    ptr++; /* step over the new NULL */
 
     /* if there are three '/', then there is no
      * hostname and the file is local
@@ -140,14 +135,12 @@ char *opal_filename_from_uri(const char *uri,
         fn = strdup(ptr);
     } else if (0 != strncmp(ptr, "//", 2)) {
         /* error */
-        opal_show_help("help-opal-util.txt", "malformed-uri",
-                       true, uri);
+        opal_show_help("help-opal-util.txt", "malformed-uri", true, uri);
     } else {
-        ptr += 2;  /* step to the hostname */
+        ptr += 2; /* step to the hostname */
         /* find the separator to the filename */
         if (NULL == (sp = strchr(ptr, '/'))) {
-            opal_show_help("help-opal-util.txt", "malformed-uri",
-                           true, uri);
+            opal_show_help("help-opal-util.txt", "malformed-uri", true, uri);
         } else {
             *sp = '\0';
             if (NULL != hostname) {
diff --git a/opal/util/uri.h b/opal/util/uri.h
index 87cdec8849c..5747abbf401 100644
--- a/opal/util/uri.h
+++ b/opal/util/uri.h
@@ -53,7 +53,7 @@
 #include "opal_config.h"
 
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 
 BEGIN_C_DECLS
@@ -63,15 +63,16 @@ BEGIN_C_DECLS
  *
  * The caller is responsible for freeing the returned string.
  */
-OPAL_DECLSPEC char *opal_uri_get_scheme(const char *uri) __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
+OPAL_DECLSPEC char *opal_uri_get_scheme(const char *uri)
+    __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
 
 /**
  *  Create a uri from a hostname and filename
  *
  * The caller is responsible for freeing the returned string.
  */
-OPAL_DECLSPEC char *opal_filename_to_uri(const char *filename,
-                                         const char *hostname) __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
+OPAL_DECLSPEC char *opal_filename_to_uri(const char *filename, const char *hostname)
+    __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
 /**
  * Extract the filename (and hostname) from a uri
  *
@@ -83,8 +84,8 @@ OPAL_DECLSPEC char *opal_filename_to_uri(const char *filename,
  *
  * The caller is responsible for freeing the returned string.
  */
-OPAL_DECLSPEC char *opal_filename_from_uri(const char *uri,
-                                           char **hostname) __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
+OPAL_DECLSPEC char *opal_filename_from_uri(const char *uri, char **hostname)
+    __opal_attribute_malloc__ __opal_attribute_warn_unused_result__;
 
 END_C_DECLS
 #endif /* OPAL_URI_H */
diff --git a/opal/win32/opal_inet.c b/opal/win32/opal_inet.c
deleted file mode 100644
index 9e21a8f668f..00000000000
--- a/opal/win32/opal_inet.c
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2014 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-
-#include "opal_config.h"
-
-#include "opal/win32/opal_inet.h"
-#include "opal/util/output.h"
-
-/*
- * convert from presentation format (which usually means ASCII printable)
- *    to network format (which is usually some kind of binary format).
- *
- * return:
- *    1 if the address was valid for the specified address family
- *    0 if the address wasn't valid (`dst' is untouched in this case)
- *    -1 if some other error occurred (`dst' is untouched in this case, too)
- */
-int opal_inet_pton(int af, const char *src, void *dst)
-{
-    int addr_len;
-    struct sockaddr sa;
-    struct sockaddr_in *sin = (struct sockaddr_in *)&sa;
-    struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&sa;
-
-    memset (&sa, 0, sizeof(struct sockaddr));
-
-    switch (af) {
-    case AF_INET:
-        addr_len = sizeof(struct sockaddr_in);
-        break;
-
-    case AF_INET6:
-        addr_len = sizeof(struct sockaddr_in6);
-        break;
-
-    default:
-        return -1;
-    }
-
-    if ( 0 == WSAStringToAddress ((LPTSTR) src, af, NULL, (LPSOCKADDR) &sa, &addr_len )) {
-        switch (af) {
-        case AF_INET:
-            memcpy (dst, &sin->sin_addr, sizeof(struct in_addr));
-            break;
-
-        case AF_INET6:
-            memcpy (dst, &sin6->sin6_addr, sizeof(struct in6_addr));
-            break;
-        }
-        return 1;
-    } else {
-        opal_output(0, "WSAStringToAddress failed %s:%d. Error code: %d", __FILE__, __LINE__, GetLastError());
-        return 0;
-    }
-}
-
-
-/*
- * convert a network format address to presentation format.
- *
- * return:
- *    pointer to presentation format address (`dst'), or NULL.
- */
-const char *opal_inet_ntop(int af, const void *src, char *dst, size_t size)
-{
-    int addr_len;
-    struct sockaddr sa;
-    DWORD str_len = size;
-    struct sockaddr_in *sin = (struct sockaddr_in *)&sa;
-    struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&sa;
-
-    memset (&sa, 0, sizeof (struct sockaddr));
-
-    switch (af) {
-    case AF_INET:
-        addr_len = sizeof(struct sockaddr_in);
-        sin->sin_family = af;
-        memcpy (&sin->sin_addr, src, sizeof (struct in_addr));
-        break;
-
-    case AF_INET6:
-        addr_len = sizeof(struct sockaddr_in6);
-        sin6->sin6_family = af;
-        memcpy (&sin6->sin6_addr, src, sizeof (struct in6_addr));
-        break;
-
-    default:
-        return NULL;
-    }
-
-    if ( 0 == WSAAddressToString ((LPSOCKADDR) &sa, addr_len, NULL, dst, &str_len )) {
-        return dst;
-    } else {
-        opal_output(0, "WSAAddressToString failed %s:%d. Error code: %d", __FILE__, __LINE__, GetLastError());
-        return NULL;
-    }
-}
diff --git a/opal/win32/opal_inet.h b/opal/win32/opal_inet.h
deleted file mode 100644
index a3e6a71e88d..00000000000
--- a/opal/win32/opal_inet.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2014 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef OPAL_INET_H
-#define OPAL_INET_H
-
-#include "opal_config.h"
-
-#ifndef OPAL_WIN_COMPAT_H
-#error This file is supposed to be included only from win_compat.h
-#endif  /* OPAL_WIN_COMPAT_H */
-
-BEGIN_C_DECLS
-
-OPAL_DECLSPEC int opal_inet_pton(int af, const char *src, void *dst);
-
-OPAL_DECLSPEC const char *opal_inet_ntop(int af, const void *src, char *dst, size_t size);
-
-END_C_DECLS
-
-#endif /* OPAL_INET_H */
diff --git a/opal/win32/opal_misc.h b/opal/win32/opal_misc.h
deleted file mode 100644
index 545f20ed523..00000000000
--- a/opal/win32/opal_misc.h
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2014 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef OPAL_MISC_H
-#define OPAL_MISC_H
-
-#include <stdio.h>
-#include <stdlib.h>
-
-#define _SC_PAGESIZE 0
-#define _SC_OPEN_MAX 1
-
-#if 0
-/* currently, this is a memory leak */
-static __inline char* getenv (const char *name)
-{
-    int ret;
-    char *buffer;
-    DWORD length = GetEnvironmentVariable( (LPCSTR)name, NULL, 0 );
-
-    if( 0 == length ) return NULL;
-    buffer = (char *)malloc(sizeof(char) * length);
-    ret = GetEnvironmentVariable((LPCSTR)name, (LPSTR)buffer, length);
-    return (ret > 0) ? buffer: NULL;
-}
-
-
-static __inline int setenv (const char *name, const char *value, int rewrite)
-{
-    int ret;
-    if( 0 == rewrite  ) {
-        DWORD length = 0;
-        if( 0 == (length = GetEnvironmentVariable( (LPCSTR)name, NULL, length )) ) {
-            if( ERROR_ENVVAR_NOT_FOUND == GetLastError() ) {  /* do not exist */
-                return 0;
-            }
-        }
-    }
-    /* just push it back to the windows thingy */
-    ret = SetEnvironmentVariable ((LPCSTR)name, (LPCSTR)value);
-    return (0 != ret)? 1: 0;
-}
-#endif
-
-static __inline unsigned int sleep(unsigned int seconds) {
-
-    /* Allow interruptions */
-    SleepEx(seconds * 1000, TRUE);
-    return 0;
-}
-
-/* this function can currently ONLY return the page size. for it to
-   do the entire sysconf range it needs to be extended */
-static __inline size_t sysconf(int option) {
-
-    SYSTEM_INFO sys_info;
-
-    if( _SC_OPEN_MAX == option ) {
-        return _getmaxstdio();
-    }
-
-    GetSystemInfo(&sys_info);
-    if (_SC_PAGESIZE == option){
-        return (size_t)sys_info.dwPageSize;
-    }
-    printf( "This functionality is not supported: line: %d\tfile: %s\n",
-            __LINE__, __FILE__ );
-    abort();
-    return 0;
-}
-
-#define F_GETFL 0
-#define F_SETFL 1
-#define O_NONBLOCK 0
-/*
- * this function is currently defined only for setting the socket to be
- * in the non-blocking mode. Else this function returns error not implemented.
- * This calls ioctlsocket in the winsock library
- */
-static __inline int fcntl (int fildes, int cmd, ...) {
-    int ret;
-    int mode;
-
-    switch (cmd) {
-        case F_SETFL: mode = 1; ret = ioctlsocket ((SOCKET)fildes, FIONBIO, (u_long FAR*) &mode);
-                      break;
-        case F_GETFL: ret = 0;
-                      break;
-        default: printf("Option not supported: %d %s\n", __LINE__, __FILE__);
-                      abort();
-    };
-
-    return ret;
-}
-
-#endif /* OPAL_MISC_H */
diff --git a/opal/win32/opal_process.c b/opal/win32/opal_process.c
deleted file mode 100644
index 92d0b32a4a2..00000000000
--- a/opal/win32/opal_process.c
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-                         University Research and Technology
-                         Corporation.  All rights reserved.
- Copyright (c) 2004-2014 The University of Tennessee and The University
-                         of Tennessee Research Foundation.  All rights
-                         reserved.
- Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-                         University of Stuttgart.  All rights reserved.
- Copyright (c) 2004-2005 The Regents of the University of California.
-                         All rights reserved.
- $COPYRIGHT$
-
- Additional copyrights may follow
-
- $HEADER$
- */
-
-#include "opal_config.h"
-#include "win32/opal_process.h"
-
-pid_t waitpid(pid_t pid, int *status, int options) {
-   return _cwait(status, pid, _WAIT_CHILD);
-}
-
-int kill(pid_t pid, int sig) {
-   /* XXX fill this in */
-   /* Need to connect to the child process
-      Then raise the signal since Windows doesn;t
-      have the ability to 'send a signal' to a
-      process, a la the kill command in UNIX
-
-      MSVC functions to look at:
-       - OpenProcess
-       - TerminateProcess
-       - raise
-   */
-
-   return 0;
-}
diff --git a/opal/win32/opal_process.h b/opal/win32/opal_process.h
deleted file mode 100644
index 34ffe4dc06f..00000000000
--- a/opal/win32/opal_process.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- *Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                        University Research and Technology
- *                        Corporation.  All rights reserved.
- *Copyright (c) 2004-2014 The University of Tennessee and The University
- *                        of Tennessee Research Foundation.  All rights
- *                        reserved.
- *Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                        University of Stuttgart.  All rights reserved.
- *Copyright (c) 2004-2005 The Regents of the University of California.
- *                        All rights reserved.
- *$COPYRIGHT$
- *
- *Additional copyrights may follow
- *
- *$HEADER$
- */
-
-#ifndef OPAL_PROCESS_H
-#define OPAL_PROCESS_H
-
-#include "opal_config.h"
-
-#ifndef OPAL_WIN_COMPAT_H
-#error This file is supposed to be included only from win_compat.h
-#endif  /* OPAL_WIN_COMPAT_H */
-
-BEGIN_C_DECLS
-
-OPAL_DECLSPEC pid_t waitpid (pid_t pid, int *status, int options) ;
-
-OPAL_DECLSPEC int kill(pid_t pid, int sig) ;
-
-END_C_DECLS
-
-#endif  /* OPAL_PROCESS_H */
diff --git a/opal/win32/opal_socket.c b/opal/win32/opal_socket.c
deleted file mode 100644
index f293b463842..00000000000
--- a/opal/win32/opal_socket.c
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Copyright (c) 2010      High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "opal_config.h"
-
-#include "opal/win32/opal_socket.h"
-#include "opal/util/output.h"
-
-int
-create_socketpair(int family, int type, int protocol, int fd[2])
-{
-    /* This code is originally from Tor.  Used with permission. */
-
-    /* This socketpair does not work when localhost is down. So
-     * it's really not the same thing at all. But it's close enough
-     * for now, and really, when localhost is down sometimes, we
-     * have other problems too.
-     */
-#ifdef WIN32
-#define ERR(e) WSA##e
-#else
-#define ERR(e) e
-#endif
-    int listener = -1;
-    int connector = -1;
-    int acceptor = -1;
-    struct sockaddr_in listen_addr;
-    struct sockaddr_in connect_addr;
-    int size;
-    int saved_errno = -1;
-
-    if (protocol
-        || (family != AF_INET
-#ifdef AF_UNIX
-            && family != AF_UNIX
-#endif
-        )) {
-        opal_output(0, "Protocol not support: %d", (ERR(EAFNOSUPPORT)));
-        return -1;
-    }
-    if (!fd) {
-        opal_output(0, "Invalid socked: %d", (ERR(EINVAL)));
-        return -1;
-    }
-
-    listener = socket(AF_INET, type, 0);
-    if (listener < 0)
-        return -1;
-    memset(&listen_addr, 0, sizeof(listen_addr));
-    listen_addr.sin_family = AF_INET;
-    listen_addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
-    listen_addr.sin_port = 0;    /* kernel chooses port.     */
-    if (bind(listener, (struct sockaddr *) &listen_addr, sizeof (listen_addr))
-        == -1)
-        goto tidy_up_and_fail;
-    if (listen(listener, 1) == -1)
-        goto tidy_up_and_fail;
-
-    connector = socket(AF_INET, type, 0);
-    if (connector < 0)
-        goto tidy_up_and_fail;
-    /* We want to find out the port number to connect to.  */
-    size = sizeof(connect_addr);
-    if (getsockname(listener, (struct sockaddr *) &connect_addr, &size) == -1)
-        goto tidy_up_and_fail;
-    if (size != sizeof (connect_addr))
-        goto abort_tidy_up_and_fail;
-    if (connect(connector, (struct sockaddr *) &connect_addr,
-                sizeof(connect_addr)) == -1)
-        goto tidy_up_and_fail;
-
-    size = sizeof(listen_addr);
-    acceptor = accept(listener, (struct sockaddr *) &listen_addr, &size);
-    if (acceptor < 0)
-        goto tidy_up_and_fail;
-    if (size != sizeof(listen_addr))
-        goto abort_tidy_up_and_fail;
-    closesocket(listener);
-    /* Now check we are talking to ourself by matching port and host on the
-       two sockets.     */
-    if (getsockname(connector, (struct sockaddr *) &connect_addr, &size) == -1)
-        goto tidy_up_and_fail;
-    if (size != sizeof (connect_addr)
-        || listen_addr.sin_family != connect_addr.sin_family
-        || listen_addr.sin_addr.s_addr != connect_addr.sin_addr.s_addr
-        || listen_addr.sin_port != connect_addr.sin_port)
-        goto abort_tidy_up_and_fail;
-    fd[0] = connector;
-    fd[1] = acceptor;
-
-    return 0;
-
- abort_tidy_up_and_fail:
-    saved_errno = ERR(ECONNABORTED);
- tidy_up_and_fail:
-    if (saved_errno < 0)
-        saved_errno = WSAGetLastError();
-    if (listener != -1)
-        closesocket(listener);
-    if (connector != -1)
-        closesocket(connector);
-    if (acceptor != -1)
-        closesocket(acceptor);
-
-    WSASetLastError(saved_errno);
-    return -1;
-#undef ERR
-}
diff --git a/opal/win32/opal_socket.h b/opal/win32/opal_socket.h
deleted file mode 100644
index dfd0f28e997..00000000000
--- a/opal/win32/opal_socket.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Copyright (c) 2010      High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef OPAL_SOCKET_H
-#define OPAL_SOCKET_H
-
-#include "opal_config.h"
-
-#ifndef OPAL_WIN_COMPAT_H
-#error This file is supposed to be included only from win_compat.h
-#endif  /* OPAL_WIN_COMPAT_H */
-
-
-BEGIN_C_DECLS
-
-OPAL_DECLSPEC int create_socketpair(int d, int type, int protocol, int sv[2]);
-
-END_C_DECLS
-
-#endif /* OPAL_SOCKET_H */
diff --git a/opal/win32/opal_time.c b/opal/win32/opal_time.c
deleted file mode 100644
index 6eb4f8b943d..00000000000
--- a/opal/win32/opal_time.c
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2014 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "opal_config.h"
-#include "opal/win32/opal_time.h"
-
-#include<time.h>
-
-#define EPOCHFILETIME (116444736000000000LL)
-
-int gettimeofday(struct timeval *tv, struct timezone *tz)
-{
-    FILETIME file_time;
-    LARGE_INTEGER place_holder;
-    __int64 time;
-
-
-    /* returns 64 bit value which is the number of 100 nanosecond
-       intervals since 1601(UTC) */
-    GetSystemTimeAsFileTime (&file_time);
-
-    /* Windows recommends that we should copy the FILETIME returned
-       into a ULARGE_INTEGER and then perform the arithmetic on that */
-    place_holder.LowPart = file_time.dwLowDateTime;
-    place_holder.HighPart = file_time.dwHighDateTime;
-    time = place_holder.QuadPart;
-    time -= EPOCHFILETIME;
-
-    /* Now we can use arithmetic operations on time which is nothing but
-       a 64 bit integer holding time in 100 nanosec intervals */
-
-    /* convert 100 nanoseconds intervals into microseconds .. divide by 10 */
-    time /= 10;
-
-    tv->tv_sec = (long)(time / 1000000);
-    tv->tv_usec = (long)(time % 1000000);
-
-    return 0;
-}
diff --git a/opal/win32/opal_time.h b/opal/win32/opal_time.h
deleted file mode 100644
index daa2aff5806..00000000000
--- a/opal/win32/opal_time.h
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- *Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                        University Research and Technology
- *                        Corporation.  All rights reserved.
- *Copyright (c) 2004-2014 The University of Tennessee and The University
- *                        of Tennessee Research Foundation.  All rights
- *                        reserved.
- *Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                        University of Stuttgart.  All rights reserved.
- *Copyright (c) 2004-2005 The Regents of the University of California.
- *                        All rights reserved.
- *$COPYRIGHT$
- *
- *Additional copyrights may follow
- *
- *$HEADER$
- */
-
-#ifndef OPAL_TIME_H
-#define OPAL_TIME_H
-
-#include "opal_config.h"
-
-#ifndef OPAL_WIN_COMPAT_H
-#error This file is supposed to be included only from win_compat.h
-#endif  /* OPAL_WIN_COMPAT_H */
-
-#define DST_NONE    0   /* not on dst */
-#define DST_USA     1   /* USA style dst */
-#define DST_AUST    2   /* Australian style dst */
-#define DST_WET     3   /* Western European dst */
-#define DST_MET     4   /* Middle European dst */
-#define DST_EET     5   /* Eastern European dst */
-#define DST_CAN     6   /* Canada */
-
-#define TIMEVAL_TO_TIMESPEC(tv, ts) \
-(ts)->tv_sec = (tv)->tv_sec;    \
-(ts)->tv_nsec = (tv)->tv_usec * 1000;
-
-#define TIMESPEC_TO_TIMEVAL(tv, ts) \
-(tv)->tv_sec = (ts)->tv_sec;  \
-(tv)->tv_usec = (ts)->tv_nsec / 1000;
-
-
-/* some more utility functions */
-/* Operations on timevals. */
-#ifndef timerclear
-#define timerclear(tvp)     (tvp)->tv_sec = (tvp)->tv_usec = 0
-#endif
-
-#ifndef timerisset
-#define timerisset(tvp)     ((tvp)->tv_sec || (tvp)->tv_usec)
-#endif
-
-#ifndef timercmp
-#define timercmp(tvp, uvp, cmp)                     \
-    (((tvp)->tv_sec == (uvp)->tv_sec) ?             \
-    ((tvp)->tv_usec cmp (uvp)->tv_usec) :           \
-    ((tvp)->tv_sec cmp (uvp)->tv_sec))
-#endif
-
-#ifndef timeradd
-#define timeradd(tvp, uvp, vvp)                     \
-    do {                                \
-    (vvp)->tv_sec = (tvp)->tv_sec + (uvp)->tv_sec;      \
-    (vvp)->tv_usec = (tvp)->tv_usec + (uvp)->tv_usec;   \
-    if ((vvp)->tv_usec >= 1000000) {            \
-        (vvp)->tv_sec++;                \
-        (vvp)->tv_usec -= 1000000;          \
-        }                           \
-    } while (0)
-#endif
-
-#ifndef timersub
-#define timersub(tvp, uvp, vvp)                     \
-    do {                                \
-    (vvp)->tv_sec = (tvp)->tv_sec - (uvp)->tv_sec;      \
-    (vvp)->tv_usec = (tvp)->tv_usec - (uvp)->tv_usec;   \
-    if ((vvp)->tv_usec < 0) {               \
-    (vvp)->tv_sec--;                \
-    (vvp)->tv_usec += 1000000;          \
-    }                           \
-    } while (0)
-#endif
-
-/* Operations on timespecs. */
-
-#ifndef timespecclear
-#define timespecclear(tsp)      (tsp)->tv_sec = (tsp)->tv_nsec = 0
-#endif
-
-#ifndef timespecisset
-#define timespecisset(tsp)      ((tsp)->tv_sec || (tsp)->tv_nsec)
-#endif
-
-#ifndef timespeccmp
-#define timespeccmp(tsp, usp, cmp)                  \
-    (((tsp)->tv_sec == (usp)->tv_sec) ?             \
-    ((tsp)->tv_nsec cmp (usp)->tv_nsec) :           \
-    ((tsp)->tv_sec cmp (usp)->tv_sec))
-#endif
-
-#ifndef timespecadd
-#define timespecadd(tsp, usp, vsp)                  \
-    do {                                \
-    (vsp)->tv_sec = (tsp)->tv_sec + (usp)->tv_sec;      \
-    (vsp)->tv_nsec = (tsp)->tv_nsec + (usp)->tv_nsec;   \
-    if ((vsp)->tv_nsec >= 1000000000L) {            \
-    (vsp)->tv_sec++;                \
-    (vsp)->tv_nsec -= 1000000000L;          \
-    }                           \
-    } while (0)
-#endif
-
-#ifndef timespecsub
-#define timespecsub(tsp, usp, vsp)                  \
-    do {                                \
-    (vsp)->tv_sec = (tsp)->tv_sec - (usp)->tv_sec;      \
-    (vsp)->tv_nsec = (tsp)->tv_nsec - (usp)->tv_nsec;   \
-    if ((vsp)->tv_nsec < 0) {               \
-    (vsp)->tv_sec--;                \
-    (vsp)->tv_nsec += 1000000000L;          \
-    }                           \
-    } while (0)
-#endif
-
-/*
- * Names of the interval timers, and structure
-  * defining a timer setting.
-   */
-#define ITIMER_REAL 0
-#define ITIMER_VIRTUAL  1
-#define ITIMER_PROF 2
-
-struct itimerval {
-  struct  timeval it_interval;    /* timer interval */
-  struct  timeval it_value;   /* current value */
-};
-
-/*
- * Getkerninfo clock information structure
- */
-struct clockinfo {
-   int hz;     /* clock frequency */
-   int tick;       /* micro-seconds per hz tick */
-   int tickadj;    /* clock skew rate for adjtime() */
-   int stathz;     /* statistics clock frequency */
-   int profhz;     /* profiling clock frequency */
-};
-
-#define CLOCK_REALTIME  0
-#define CLOCK_VIRTUAL   1
-#define CLOCK_PROF  2
-
-#define TIMER_RELTIME   0x0 /* relative timer */
-#define TIMER_ABSTIME   0x1 /* absolute timer */
-
-#ifndef OPAL_TIMESPEC
-#define OPAL_TIMESPEC
-struct timespec
-{
-  long tv_sec;
-  long tv_nsec;
-};
-#endif
-
-
-/*
-NOTE: The use of timezone is obsolete even in linux and my gettimeofday
-function is not going to support it either. So, please be aware of the
-fact that if you expect to pass anything here, then you are DEAD :-D */
-struct timezone
-{
-  int tz_minuteswest;
-  int tz_dsttime;
-};
-
-BEGIN_C_DECLS
-
-OPAL_DECLSPEC int gettimeofday (struct timeval *tv, struct timezone *tz);
-
-END_C_DECLS
-
-#endif  /* OPAL_TIME_H */
diff --git a/opal/win32/opal_uio.c b/opal/win32/opal_uio.c
deleted file mode 100644
index 221a2297417..00000000000
--- a/opal/win32/opal_uio.c
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-                         University Research and Technology
-                         Corporation.  All rights reserved.
- Copyright (c) 2004-2005 The University of Tennessee and The University
-                         of Tennessee Research Foundation.  All rights
-                         reserved.
- Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-                         University of Stuttgart.  All rights reserved.
- Copyright (c) 2004-2005 The Regents of the University of California.
-                         All rights reserved.
- $COPYRIGHT$
-
- Additional copyrights may follow
-
- $HEADER$
- */
-
-#include "opal_config.h"
-#include "opal/win32/opal_uio.h"
-#include <errno.h>
-
-/*
- Highly doubt if the windows sockets ever set errno to EAGAIN. There might
- be some weird conversion to map this or I might have to rewrite this piece
- of code to handle the windows error flags
- */
-
-int writev( int fd, struct iovec * iov, int cnt )
-{
-   int err;
-   DWORD sendlen;
-
-   err = WSASend((SOCKET) fd, &(iov->data), cnt, &sendlen, 0, NULL, NULL);
-
-   if (err < 0) {
-      return err;
-   }
-   return (int) sendlen;
-}
-
-
-int readv( int fd, struct iovec * iov, int cnt )
-{
-   int err;
-   DWORD recvlen = 0;
-   DWORD flags = 0;
-
-   err = WSARecv((SOCKET) fd, &(iov->data), cnt, &recvlen, &flags, NULL, NULL);
-
-   if( err < 0 ) {
-	   return err;
-   }
-   return (int) recvlen;
-}
-
diff --git a/opal/win32/opal_uio.h b/opal/win32/opal_uio.h
deleted file mode 100644
index b97be357a7d..00000000000
--- a/opal/win32/opal_uio.h
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2014 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef OPAL_UIO_H
-#define OPAL_UIO_H
-
-#include "opal_config.h"
-
-#ifndef OPAL_WIN_COMPAT_H
-#error This file is supposed to be included only from win_compat.h
-#endif  /* OPAL_WIN_COMPAT_H */
-
-/* define the iovec structure */
-struct iovec {
-  WSABUF data;
-};
-#define iov_base data.buf
-#define iov_len data.len
-
-BEGIN_C_DECLS
-/*
- * writev:
-   writev  writes  data  to  file  descriptor  fd,  and  from  the buffers
-   described by iov. The number of buffers is specified by  cnt.  The
-   buffers  are  used  in  the  order specified.  Operates just like write
-   except that data is taken from iov instead of a contiguous buffer.
- */
-OPAL_DECLSPEC int writev (int fd, struct iovec *iov, int cnt);
-
-/*
-   readv  reads  data  from file descriptor fd, and puts the result in the
-   buffers described by iov. The number  of  buffers  is  specified  by
-   cnt.  The  buffers  are filled in the order specified.  Operates just
-   like read except that data is put in iov  instead  of  a  contiguous
-   buffer.
- */
-OPAL_DECLSPEC int readv (int fd, struct iovec *iov, int cnt);
-
-END_C_DECLS
-
-#endif /* OPAL_UIO_H */
diff --git a/opal/win32/opal_util.h b/opal/win32/opal_util.h
deleted file mode 100644
index 20b78dc7c79..00000000000
--- a/opal/win32/opal_util.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2014 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef OPAL_UTIL_H
-#define OPAL_UTIL_H
-
-static __inline int getpagesize(void)
-{
-    SYSTEM_INFO sys_info;
-
-    GetSystemInfo(&sys_info);
-    return (int)sys_info.dwPageSize;
-}
-
-#endif
diff --git a/opal/win32/opal_utsname.c b/opal/win32/opal_utsname.c
deleted file mode 100644
index b6d06876fba..00000000000
--- a/opal/win32/opal_utsname.c
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-                         University Research and Technology
-                         Corporation.  All rights reserved.
- Copyright (c) 2004-2014 The University of Tennessee and The University
-                         of Tennessee Research Foundation.  All rights
-                         reserved.
- Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-                         University of Stuttgart.  All rights reserved.
- Copyright (c) 2004-2005 The Regents of the University of California.
-                         All rights reserved.
- $COPYRIGHT$
-
- Additional copyrights may follow
-
- $HEADER$
- */
-
-#include "opal_config.h"
-#include "opal/win32/opal_utsname.h"
-#include "opal/util/string_copy.h"
-
-/*
-    This has to fill in the following information
-
-    1. sysname: name of the operating system --
-    2. nodename: GetComputerName
-    3. release: GetVersionEx
-    4. version: GetVersionEx
-    5. machine: GetSystemInfo
-*/
-
-int uname( struct utsname *un )
-{
-    TCHAR env_variable[] = "OS=%OS%";
-    DWORD info_buf_count;
-    OSVERSIONINFO version_info;
-    SYSTEM_INFO sys_info;
-    TCHAR info_buf[OPAL_UTSNAME_LEN];
-
-    info_buf_count = ExpandEnvironmentStrings( env_variable, info_buf, OPAL_UTSNAME_LEN);
-    if (0 == info_buf_count) {
-        snprintf( un->sysname, OPAL_UTSNAME_LEN, "Unknown" );
-    } else {
-        /* remove the "OS=" from the beginning of the string */
-        opal_string_copy( un->sysname, info_buf + 3, OPAL_UTSNAME_LEN );
-    }
-    info_buf_count = OPAL_UTSNAME_LEN;
-    if (!GetComputerName( un->nodename, &info_buf_count)) {
-        snprintf(un->nodename, OPAL_UTSNAME_LEN, "undefined");
-    }
-
-    version_info.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
-    if (!GetVersionEx(&version_info)) {
-        snprintf(un->release, OPAL_UTSNAME_LEN, "undefined");
-        snprintf(un->version, OPAL_UTSNAME_LEN, "undefined");
-    } else {
-        /* fill in both release and version information */
-        snprintf( un->release, OPAL_UTSNAME_LEN, "%d.%d.%d",
-                  version_info.dwMajorVersion,
-                  version_info.dwMinorVersion,
-                  version_info.dwBuildNumber);
-        snprintf( un->version, OPAL_UTSNAME_LEN, "%s", version_info.szCSDVersion );
-    }
-
-    /* get machine information */
-    GetSystemInfo(&sys_info);
-    switch( sys_info.wProcessorArchitecture ) {
-        case PROCESSOR_ARCHITECTURE_UNKNOWN:
-            snprintf( un->machine, OPAL_UTSNAME_LEN, "Unknown %d", sys_info.wProcessorLevel );
-            break;
-        case PROCESSOR_ARCHITECTURE_INTEL:
-            snprintf( un->machine, OPAL_UTSNAME_LEN, "Intel %d", sys_info.wProcessorLevel );
-            break;
-        case PROCESSOR_ARCHITECTURE_IA64:
-            snprintf( un->machine, OPAL_UTSNAME_LEN, "IA64 %d", sys_info.wProcessorLevel );
-            break;
-        case PROCESSOR_ARCHITECTURE_AMD64:
-            snprintf( un->machine, OPAL_UTSNAME_LEN, "AMD %d", sys_info.wProcessorLevel );
-            break;
-        default:
-            snprintf( un->machine, OPAL_UTSNAME_LEN, "UFO hardware %d", sys_info.wProcessorLevel );
-            break;
-    }
-
-    return 0;
-}
diff --git a/opal/win32/opal_utsname.h b/opal/win32/opal_utsname.h
deleted file mode 100644
index 11bd45c52fd..00000000000
--- a/opal/win32/opal_utsname.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2014 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef OPAL_UTSNAME_H
-#define OPAL_UTSNAME_H
-
-#include "opal_config.h"
-
-#define OPAL_UTSNAME_LEN 64
-
-struct utsname {
-    char sysname[OPAL_UTSNAME_LEN];
-    char nodename[OPAL_UTSNAME_LEN];
-    char release[OPAL_UTSNAME_LEN];
-    char version[OPAL_UTSNAME_LEN];
-    char machine[OPAL_UTSNAME_LEN];
-};
-
-BEGIN_C_DECLS
-    OPAL_DECLSPEC int uname(struct utsname *un);
-END_C_DECLS
-
-#endif /* OPAL_UTSNAME_H */
diff --git a/opal/win32/win_compat.h b/opal/win32/win_compat.h
deleted file mode 100644
index 99072b9f9c1..00000000000
--- a/opal/win32/win_compat.h
+++ /dev/null
@@ -1,336 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2006 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef OPAL_WIN_COMPAT_H
-#define OPAL_WIN_COMPAT_H
-
-/**
- * don't complain about all the deprecated functions.
- */
-#define _CRT_SECURE_NO_DEPRECATE
-
-/**
- * Allow usage of some recent functions (such as SwitchToThread)
- *  0x0400 - for SwitchToThread
- *  0x0500 - for using Event Objects
- */
-#define _WIN32_WINNT 0x0502
-
-/**
- * Windows does not define the exact same names in stat.h
- * Redefine them to our names.
- * Supposedly, links are not available
- */
-#define S_IFLNK        0xFFFF          /* identifies the file as a symbolic link */
-#define S_IXUSR        _S_IEXEC        /* execute/search permission, owner */
-#define S_IRUSR        _S_IREAD        /* read permission, owner */
-#define S_IWUSR        _S_IWRITE       /* write permission, owner */
-
-/**
- * Define it in order to get access to the "secure" version of rand.
- */
-#define _CRT_RAND_S
-
-/* It is always better to include windows.h with the lean and mean option.
-   So, include it with that option and then include some which are required.
-   Note: this file is included only on windows */
-
-#ifndef WIN32_LEAN_AND_MEAN
-#define WIN32_LEAN_AND_MEAN
-#endif  /* WIN32_LEAN_AND_MEAN */
-#ifndef VC_EXTRALEAN
-#define VC_EXTRALEAN
-#endif  /* VC_EXTRALEAN */
-#include <windows.h>
-
-/* FD_SETSIZE determines how many sockets windows can select() on. If not defined
-   before including winsock2.h, it is defined to be 64. We are going to go ahead and
-   make it 1024 for now. PLEASE CHECK IF THIS IS RIGHT */
-#define FD_SETSIZE 1024
-
-/* other utility header files */
-#include <shellapi.h>
-#include <winsock2.h>
-#include <ws2tcpip.h>
-#include <process.h>
-#include <signal.h>
-#include <conio.h>
-#include <fcntl.h>
-#include <stdint.h>
-
-/**
- * For all file io operations
- */
-#include <direct.h>
-#include <io.h>
-
-#include <stdlib.h>
-/* for alloca */
-#include <malloc.h>
-
-
-#define STDIN_FILENO  0
-#define STDOUT_FILENO 1
-#define STDERR_FILENO 2
-typedef unsigned short mode_t;
-typedef long ssize_t;
-typedef DWORD in_port_t;
-typedef char* caddr_t;
-typedef unsigned int uint;
-
-#ifndef WIN32_LEAN_AND_MEAN
-#define WIN32_LEAN_AND_MEAN
-#endif
-
-#ifdef _MSC_VER
-#if defined(OMPI_BUILDING) && OMPI_BUILDING
-#include "opal/win32/opal_uio.h"
-#include "opal/win32/opal_time.h"
-#include "opal/win32/opal_utsname.h"
-#include "opal/win32/opal_util.h"
-#include "opal/win32/opal_misc.h"
-#include "opal/win32/opal_inet.h"
-#include "opal/win32/opal_socket.h"
-#endif
-
-/* Defines for the access functions */
-#define F_OK  0x00
-#define R_OK  0x04
-#define W_OK  0x02
-#define X_OK  R_OK  /* no execution right on Windows */
-#define S_IRWXU (_S_IREAD | _S_IWRITE | _S_IEXEC)
-
-
-/**
- * Microsoft compiler complain about non conformance of the default UNIX function.
- * Non conformance to the POSIX standard, and they suggest to use the version
- * starting with an _ instead. So, in order to keep cl.exe happy (and quiet) we can
- * use the followings defines.
- */
-#define getpid                    _getpid
-#define strdup                    _strdup
-#define putenv                    _putenv
-#define getcwd                    _getcwd
-#define rmdir                     _rmdir
-#define chdir                     _chdir
-#define chmod                     _chmod
-#define access                    _access
-#define open                      _open
-#define close                     _close
-#define unlink                    _unlink
-#define dup2                      _dup2
-#define dup                       _dup
-#define write                     _write
-#define read                      _read
-#define fileno                    _fileno
-#define isatty                    _isatty
-#define execvp                    _execvp
-#define S_ISDIR(STAT_MODE)        ((STAT_MODE) & _S_IFDIR)
-#define S_ISREG(STAT_MODE)        ((STAT_MODE) & _S_IFREG)
-#define strncasecmp               _strnicmp
-#define strcasecmp                _stricmp
-#define umask                     _umask
-#define getch                     _getch
-#define random                    rand
-#define strtok_r                  strtok_s
-#define srand48                   srand
-#define lrand48                   rand
-#define usleep(t)                 Sleep(t/1000)
-#define posix_memalign(p, a, s)   *p=_aligned_malloc(s,a)
-
-#else
-
-#undef WSABASEERR
-/* in MinGW, PACKED is defined to __attribute__((packed)), will have problem for our basic types */
-#undef PACKED
-#define pthread_atfork
-#include <winsock.h>
-#if defined(OMPI_BUILDING) && OMPI_BUILDING
-#include "opal/win32/opal_uio.h"
-#include "opal/win32/opal_utsname.h"
-#include "opal/win32/opal_util.h"
-#include "opal/win32/opal_inet.h"
-#include "opal/win32/opal_misc.h"
-#include "opal/win32/opal_socket.h"
-#endif
-
-#define strtok_r(s,d,p)           *p = strtok(s,d)
-
-#endif
-
-#define MAXPATHLEN _MAX_PATH
-#define MAXHOSTNAMELEN _MAX_PATH
-#define OPAL_MAXHOSTNAMELEN (MAXHOSTNAMELEN + 1)
-#define PATH_MAX _MAX_PATH
-#define WTERMSIG(EXIT_CODE)    (1)
-#define WIFEXITED(EXIT_CODE)   (1)
-#define WEXITSTATUS(EXIT_CODE) (EXIT_CODE)
-#define WIFSIGNALED(EXIT_CODE) (0)
-#define WIFSTOPPED(EXIT_CODE)  (0)
-#define WSTOPSIG(EXIT_CODE)    (11)
-
-#define mkdir(PATH, MODE)         _mkdir((PATH))
-#define nanosleep(tp, rem)        Sleep(*tp.tv_sec*1000+*tp.tv_nsec/1000000)
-#define pipe(array_fd)            _pipe(array_fd, 1024, O_BINARY )
-#define inet_ntop                 opal_inet_ntop
-#define inet_pton                 opal_inet_pton
-#define lstat                     stat
-
-#ifndef UINT64_MAX
-#define UINT64_MAX            0xffffffffffffffffULL  /* 18446744073709551615ULL */
-#endif
-#ifndef UINT64_MIN
-#define UINT64_MIN            0
-#endif
-#ifndef INT64_MAX
-#define INT64_MAX             0x7fffffffffffffffLL  /*9223372036854775807LL*/
-#endif
-#ifndef INT64_MIN
-#define INT64_MIN             (-0x7fffffffffffffffLL - 1)  /* (-9223372036854775807 - 1) */
-#endif
-#ifndef UINT32_MAX
-#define UINT32_MAX            0xffffffff  /* 4294967295U */
-#endif
-#ifndef UINT32_MIN
-#define UINT32_MIN            0
-#endif
-#ifndef INT32_MAX
-#define INT32_MAX             0x7fffffff  /* 2147483647 */
-#endif
-#ifndef INT32_MIN
-#define INT32_MIN             (-0x7fffffff - 1)  /* (-2147483647 - 1) */
-#endif
-#ifndef UINT16_MAX
-#define UINT16_MAX            0xffff  /* 65535U */
-#endif
-#ifndef UINT16_MIN
-#define UINT16_MIN            0
-#endif
-#ifndef INT16_MAX
-#define INT16_MAX             0x7fff  /* 32767 */
-#endif
-#ifndef INT16_MIN
-#define INT16_MIN             (-0x7fff - 1)  /* (-32768) */
-#endif
-#ifndef UINT8_MAX
-#define UINT8_MAX             0xff  /* 255U */
-#endif
-#ifndef UINT8_MIN
-#define UINT8_MIN             0
-#endif
-#ifndef INT8_MAX
-#define INT8_MAX             0x7f  /* 127 */
-#endif
-#ifndef INT8_MIN
-#define INT8_MIN             (-0x7f - 1)  /* (-128) */
-#endif
-
-/* Make sure we let the compiler know that we support __func__ */
-#if !defined(HAVE_DECL___FUNC__)
-#define HAVE_DECL___FUNC__ 1
-#endif
-
-/*#define sprintf sprintf_s*/
-
-/* Ugly signal mapping since windows doesn't support the full spectrum
- * just a very small subset... :/
- * http://msdn.microsoft.com/library/default.asp?url=/library/en-us/vclib/html/_crt_raise.asp
- * http://msdn.microsoft.com/library/default.asp?url=/library/en-us/dnucmg/html/UCMGch09.asp
- */
-#define SIGHUP    1
-/* 2 is used for SIGINT on windows */
-#define SIGQUIT   3
-/* 4 is used for SIGILL on windows */
-#define SIGTRAP   5
-#define SIGIOT    6
-#define SIGBUS    7
-/* 8 is used for SIGFPE on windows */
-#define SIGKILL   9
-#define SIGUSR1   10
-/* 11 is used for SIGSEGV on windows */
-#define SIGUSR2   12
-#define SIGPIPE   13
-#define SIGALRM   14
-/* 15 is used for SIGTERM on windows */
-#define SIGSTKFLT 16
-#define SIGCHLD   17
-#define SIGCONT   18
-#define SIGSTOP   19
-#define SIGTSTP   20
-/* 21 is used for SIGBREAK on windows */
-/* 22 is used for SIGABRT on windows */
-#define SIGTTIN   23
-#define SIGTTOU   24
-#define SIGURG    25
-#define SIGXCPU   26
-#define SIGXFSZ   27
-#define SIGVTALRM 28
-#define SIGPROF   29
-#define SIGWINCH  30
-#define SIGIO     31
-
-/* Note:
- *   The two defines below are likely to break the orte_wait
- *   functionality. The proper method of replacing these bits
- *   of functionality is left for further investigated.
- */
-#define WUNTRACED 0
-#define WNOHANG   0
-
-#define sigset_t int
-#define in_addr_t uint32_t
-
-/* Need to define _Bool here for different version of VS.
-   The definition in opal_config_bottom.h won't help,
-   as long as we have a mixed C and C++ projects in one solution. */
-#if defined(_MSC_VER) && _MSC_VER < 1600
-#define _Bool BOOL
-#else
-#define _Bool bool
-#endif
-
-/*
- * No syslog.h on Windows, but these have to be defined somehow.
- * There could also be a notifier component later for Windows.
- */
-#define LOG_EMERG   0
-#define LOG_ALERT   1
-#define LOG_CRIT    2
-#define LOG_ERR     3
-#define LOG_WARNING 4
-#define LOG_NOTICE  5
-#define LOG_INFO    6
-#define LOG_DEBUG   7
-
-
-/*
- * Mask these to Windows equivalents
- */
-#define bzero(p, l) memset(p, 0, l)
-#define bcopy(s, t, l) memmove(t, s, l)
-#define isblank(c) (c == ' ' || c == '\t')?1:0
-
-/*
- * OMPI functions that need to be redefined.
- */
-#define ompi_debugger_notify_abort(x)
-#define ompi_wait_for_debugger(x)
-
-
-#endif /* compat */
diff --git a/opal/win_makefile b/opal/win_makefile
deleted file mode 100644
index c86ff8e7555..00000000000
--- a/opal/win_makefile
+++ /dev/null
@@ -1,252 +0,0 @@
-#!/bin/sh
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-# This is a simple makefile for windows which makes all the components
-# which are required for super computing. Too lazy to open up visual
-# studio each and every time to compile a component, so just adding
-# this to the svn repository
-#
-
-topdir := $(shell cygpath -m $(CURDIR))/..
-prefix = ${topdir}/src/Debug
-installdir = ${prefix}
-pkgdatadir = ${prefix}/share
-sysconfdir = ${prefix}/etc
-pkglibdir = ${prefix}/lib
-
-# list of components to build with the c compiler
-C_SUBDIRS =  \
-        attribute \
-        class \
-        communicator \
-        datatype \
-        errhandler \
-        event \
-        event/WIN32-Code \
-        dps \
-        file \
-        group \
-        info \
-        util \
-        mpi/c \
-        mpi/runtime \
-        threads \
-        op \
-        proc \
-        request \
-        runtime \
-        mca/base \
-        mca/allocator/base \
-        mca/coll/base \
-        mca/gpr/base \
-        mca/gpr/base/data_type_support \
-        mca/gpr/base/pack_api_cmd \
-        mca/gpr/base/unpack_api_response \
-        mca/io/base \
-        mca/iof/base \
-		  mca/errmgr/base \
-		  mca/rml/base \
-		  mca/rmgr/base \
-        mca/rmgr/base/data_type_support \
-        mca/rmaps/base \
-		  mca/soh/base \
-        mca/soh/base/data_type_support \
-        mca/mpool/base \
-        mca/ns/base \
-        mca/ns/base/data_type_support \
-        mca/oob/base \
-        mca/pml/base \
-        mca/ptl/base \
-        mca/pls/base \
-        mca/pls/fork \
-        mca/pls/proxy \
-        mca/pls/rsh \
-        mca/ras/base \
-        mca/ras/host \
-        mca/rds/base \
-        mca/rds/hostfile \
-        mca/rds/resfile \
-        mca/topo/base \
-        mca/schema/base \
-        win32/generated_source \
-        win \
-        win32
-
-EXTRA_DIRS =
-
-#
-# Files included by the dist that we don't want to compile
-#
-EXCLUDE_FILES = dt_arch.c \
-                epoll.c \
-                epoll_sub.c \
-                kqueue.c \
-                llm_base_parse_hostfile_lex.c \
-                misc.c \
-                poll.c \
-                rtsig.c \
-                stacktrace.c \
-                condition_spinlock.c
-
-#
-# A small function to extract the list of all files in these directories
-#
-SOURCE_FILES = $(foreach dir, ${C_SUBDIRS}, $(wildcard ${dir}/*.[c]))
-OBJECT_FILES = $(SOURCE_FILES:.c=.obj)
-DEBUG = Debug
-LINKED_SOURCE = $(foreach dir, ${DEBUG}, $(wildcard ${dir}/*.[c]))
-LINKED_OBJS = $(LINKED_SOURCE:.c=.obj)
-
-.SUFFIXES: .c .obj
-CC = cl
-CXX = $(CC)
-
-INCL = \
-        /I"${topdir}/src/win32/generated_include" \
-        /I"${topdir}/src/win32" \
-        /I"${topdir}/include" \
-        /I"${topdir}/src" \
-        /I"${topdir}/src/event/compat/" \
-        /I"${topdir}/src/event" \
-        /I"${topdir}/src/event/WIN32-Code/"
-
-CFLAGS = \
-        /DWIN32 \
-        /DOPAL_SYSCONFDIR="\"${sysconfdir}\"" \
-        /DOPAL_PKGLIBDIR="\"${pkglibdir}\"" \
-        /DOPAL_PKGDATADIR="\"${pkgdatadir}\"" \
-        /DHAVE_CONFIG_H \
-        /D_WINDLL \
-        /EHsc \
-        /ML \
-        /W0 \
-        /Wp64 \
-        /TC \
-        /D_MBCS \
-        /Fo"${topdir}/src/Debug/" \
-        /nologo \
-        /c
-
-STATIC_LIBS = \
-        allocator \
-        coll\
-        ns\
-        gpr \
-        topo\
-		  errmgr \
-		  rml \
-		  rmgr \
-		  rmaps \
-		  soh \
-        ptl\
-        oob\
-        pml\
-        pls\
-        ras \
-        rds \
-        schema \
-        iof
-
-CXXFLAGS = $(CFLAGS) $(INCL)
-
-LINK = link
-
-LINKFLAGS = \
-        /DLL \
-        /OPT:NOICF \
-        /OPT:NOREF \
-        /IMPLIB:libmpi.lib\
-        /nologo \
-        /OUT:libmpi.dll
-
-ADDLIBS = \
-        ws2_32.lib \
-        kernel32.lib \
-        user32.lib \
-        gdi32.lib \
-        winspool.lib \
-        comdlg32.lib \
-        advapi32.lib \
-        shell32.lib \
-        ole32.lib \
-        oleaut32.lib \
-        uuid.lib \
-        odbc32.lib \
-        odbccp32.lib
-
-.c.obj:
-	$(CXX) $(CXXFLAGS) $*.c
-
-all: libmpi
-
-prebuild:
-	@echo "Creating mca_base_parse_paramfile_lex.c"
-	@/usr/bin/flex -t -Pmca_base_yy "${topdir}/src/mca/base/mca_base_parse_paramfile_lex.l" \
-                >"${topdir}/src/mca/base/mca_base_parse_paramfile_lex.c" 2>/dev/null
-	@echo "Creating mca_rds_hostfile_lex.c"
-	@/usr/bin/flex -t -Porte_rds_hostfile_ "${topdir}/src/mca/rds/hostfile/rds_hostfile_lex.l" \
-                > "${topdir}/src/mca/rds/hostfile/rds_hostfile_lex.c" 2>/dev/null
-	@echo "Creating show_help_lex.c"
-	@/usr/bin/flex -t -Popal_show_help_yy "${topdir}/src/opal/util/show_help_lex.l" \
-                > "${topdir}/src/opal/util/show_help_lex.c" 2>/dev/null
-	@for dirs in ${STATIC_LIBS}; do \
-        (dir="mca/$${dirs}/base"; echo "Making Static components in $${dirs}"; comp_name="$${dirs}_static-components.h"; cp "${topdir}/src/win32/generated_include/$${comp_name}" "$${dir}/static-components.h";); \
-     done
-	@echo "Creating Version Headers"
-	@cp "${topdir}/src/win32/generated_include/pls-fork-version.h"  "${topdir}/src/mca/pls/fork/pls-fork-version.h";
-	@cp "${topdir}/src/win32/generated_include/pls-proxy-version.h" "${topdir}/src/mca/pls/proxy/pls-proxy-version.h";
-	@cp "${topdir}/src/win32/generated_include/pls-rsh-version.h"   "${topdir}/src/mca/pls/rsh/pls-rsh-version.h";
-	@for dirs in ${C_SUBDIRS}; do \
-	    (echo "Entering $${dirs}"; cd $$dirs; if test -f .compile_files; then (more .compile_files| xargs -i'{}' ln -f '{}' ${topdir}/src/Debug/'{}'); else (ls *.c 2>/dev/null | xargs -i'{}' ln -f '{}' ${topdir}/src/Debug/'{}'); fi); \
-	done
-	@for file in ${EXCLUDE_FILES}; do \
-		(echo "Removing $${file}"; cd ${topdir}/src/Debug/; rm -f $${file}); \
-	done
-	@echo "done"
-
-
-libmpi: $(LINKED_OBJS)
-	${LINK} ${LINKFLAGS} ${ADDLIBS} $(LINKED_OBJS)
-
-install:
-	@echo -n "Installing libmpi and include file ................"
-	@install -d ${topdir}/src ${installdir}/lib
-	@install -p ${topdir}/src/libmpi.lib ${installdir}/lib/libmpi.lib
-	@install -p ${topdir}/src/libmpi.exp ${installdir}/lib/libmpi.exp
-	@install -p ${topdir}/src/libmpi.dll ${installdir}/lib/libmpi.dll
-	@install -d ${topdir}/include ${installdir}/include/
-	@install -p ${topdir}/include/ompi_config_bottom.h \
-                                  ${installdir}/include/ompi_config_bottom.h
-	@install -p ${topdir}/src/win32/generated_include/opal_config.h \
-                                  ${installdir}/include/opal_config.h
-	@install -p ${topdir}/include/mpi.h ${installdir}/include/mpi.h
-	@install -p ${topdir}/include/ompi_stdint.h ${installdir}/include/ompi_stdint.h
-	@install -d ${topdir}/src/win32 ${installdir}/include/win32
-	@install -p ${topdir}/src/win32/win_compat.h ${installdir}/include/win32/win_compat.h
-	@echo "done"
-
-.PHONY: clean
-
-clean:
-	@for dirs in ${C_SUBDIRS}; do \
-	    (echo "Entering $$dirs"; cd $$dirs; rm -rf *.lib *.obj *.exp;); \
-	done
-	@for dirs in ${C_PPSUBDIRS}; do \
-	    (echo "Entering $$dirs"; cd $$dirs; rm -rf *.lib *.obj *.exp;); \
-	done
-	@echo "Entering ${topdir}/src/Debug ... "; cd ${topdir}/src/Debug; rm -rf *.lib *.obj *.exp *.c;
diff --git a/oshmem/Makefile.am b/oshmem/Makefile.am
index 69a2a747182..73b31ad9397 100644
--- a/oshmem/Makefile.am
+++ b/oshmem/Makefile.am
@@ -1,11 +1,14 @@
 #
 # Copyright (c) 2013-2015 Mellanox Technologies, Inc.
 #                         All rights reserved.
-# Copyright (c) 2013-2014 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2013-2022 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2014      Intel, Inc.  All rights reserved.
 # Copyright (c) 2015      Los Alamos National Security, LLC. All rights
 #                         reserved.
 # Copyright (c) 2016      IBM Corporation.  All rights reserved.
+# Copyright (c) 2021      Amazon.com, Inc. or its affiliates.
+#                         All Rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -13,24 +16,11 @@
 # $HEADER$
 #
 
-# Do we have profiling?
-if OSHMEM_PROFILING
-c_pshmem_lib = shmem/c/profile/liboshmem_c_pshmem.la
-else
-c_pshmem_lib =
-endif
-
 # Do we have the Fortran bindings?
 if OSHMEM_BUILD_FORTRAN_BINDINGS
 fortran_oshmem_lib = shmem/fortran/liboshmem_fortran.la
-
-if OSHMEM_PROFILING
-fortran_pshmem_lib = shmem/fortran/profile/liboshmem_fortran_pshmem.la
-endif
-
 else
 fortran_oshmem_lib =
-fortran_pshmem_lib =
 endif
 
 SUBDIRS = \
@@ -63,9 +53,7 @@ endif
 liboshmem_la_SOURCES =
 liboshmem_la_LIBADD = \
 	shmem/c/liboshmem_c.la \
-	$(c_pshmem_lib) \
 	$(fortran_oshmem_lib) \
-	$(fortran_pshmem_lib) \
 	$(MCA_oshmem_FRAMEWORK_LIBS) \
 	$(OSHMEM_TOP_BUILDDIR)/ompi/lib@OMPI_LIBMPI_NAME@.la
 liboshmem_la_DEPENDENCIES = $(liboshmem_la_LIBADD)
@@ -77,9 +65,8 @@ liboshmem_la_LDFLAGS = \
 headers =
 noinst_LTLIBRARIES =
 include_HEADERS =
-dist_oshmemdata_DATA =
+EXTRA_DIST =
 liboshmem_la_SOURCES += $(headers)
-nodist_man_MANS =
 
 # Conditionally install the header files
 
@@ -94,25 +81,6 @@ include request/Makefile.am
 include info/Makefile.am
 include runtime/Makefile.am
 include shmem/Makefile.am
-include shmem/man/man3/Makefile.extra
 include mca/Makefile.am
 include tools/Makefile.am
 include util/Makefile.am
-
-# Ensure that the man page directory exists before we try to make man
-# page files (because ompi/mpi/man/man3 has no config.status-generated
-# Makefile)
-dir_stamp = $(top_builddir)/$(subdir)/shmem/man/man3/.dir-stamp
-
-# Also ensure that the man pages are rebuilt if the opal_config.h file
-# changes (e.g., configure was run again, meaning that the release
-# date or version may have changed)
-$(nodist_man_MANS): $(dir_stamp) $(top_builddir)/opal/include/opal_config.h
-
-$(dir_stamp):
-	$(MKDIR_P) `dirname $@`
-	touch "$@"
-
-# Remove the generated man pages
-distclean-local:
-	rm -f $(nodist_man_MANS) $(dir_stamp)
diff --git a/oshmem/include/oshmem/constants.h b/oshmem/include/oshmem/constants.h
index a77849a6b55..cecb5c3a302 100644
--- a/oshmem/include/oshmem/constants.h
+++ b/oshmem/include/oshmem/constants.h
@@ -80,6 +80,7 @@ enum {
   SHMEM_NULL	= 0,
   SHMEM_CHAR,
   SHMEM_UCHAR,
+  SHMEM_SCHAR,
   SHMEM_SHORT,
   SHMEM_USHORT,
   SHMEM_INT,
@@ -87,12 +88,23 @@ enum {
   SHMEM_LONG,
   SHMEM_ULONG,
   SHMEM_LLONG,
+  SHMEM_BYTE,
+  SHMEM_INT8_T,
+  SHMEM_INT16_T,
   SHMEM_INT32_T,
   SHMEM_INT64_T,
+  SHMEM_UINT8_T,
+  SHMEM_UINT16_T,
+  SHMEM_UINT32_T,
+  SHMEM_UINT64_T,
+  SHMEM_SIZE_T,
+  SHMEM_PTRDIFF_T,
   SHMEM_ULLONG,
   SHMEM_FLOAT,
   SHMEM_DOUBLE,
   SHMEM_LDOUBLE,
+  SHMEM_COMPLEXD,
+  SHMEM_COMPLEXF,
 
   SHMEM_FINT,
   SHMEM_FINT4,
diff --git a/oshmem/include/oshmem_config.h b/oshmem/include/oshmem_config.h
index d5d44ebf920..484166e149c 100644
--- a/oshmem/include/oshmem_config.h
+++ b/oshmem/include/oshmem_config.h
@@ -2,7 +2,7 @@
  *
  * Copyright (c) 2013      Mellanox Technologies, Inc.
  *                         All rights reserved.
- * Copyright (c) 2013      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013-2022 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -22,46 +22,14 @@
 
 #define OSHMEM_IDENT_STRING OPAL_IDENT_STRING
 
-#if defined(__WINDOWS__)
-
-#  if defined(_USRDLL)    /* building shared libraries (.DLL) */
-#    if defined(OSHMEM_EXPORTS)
-#      define OSHMEM_DECLSPEC        __declspec(dllexport)
-#      define OSHMEM_MODULE_DECLSPEC
-#    else
-#      define OSHMEM_DECLSPEC        __declspec(dllimport)
-#      if defined(OSHMEM_MODULE_EXPORTS)
-#        define OSHMEM_MODULE_DECLSPEC __declspec(dllexport)
-#      else
-#        define OSHMEM_MODULE_DECLSPEC __declspec(dllimport)
-#      endif  /* defined(OSHMEM_MODULE_EXPORTS) */
-#    endif  /* defined(OSHMEM_EXPORTS) */
-#  else          /* building static library */
-#    if defined(OSHMEM_IMPORTS)
-#      define OSHMEM_DECLSPEC        __declspec(dllimport)
-#    else
-#      define OSHMEM_DECLSPEC
-#    endif  /* defined(OSHMEM_IMPORTS) */
-#    define OSHMEM_MODULE_DECLSPEC
-#  endif  /* defined(_USRDLL) */
-
-#else
-
 #  if OPAL_C_HAVE_VISIBILITY
 #    ifndef OSHMEM_DECLSPEC
 #      define OSHMEM_DECLSPEC            __opal_attribute_visibility__("default")
 #    endif
-#    ifndef OSHMEM_MODULE_DECLSPEC
-#      define OSHMEM_MODULE_DECLSPEC     __opal_attribute_visibility__("default")
-#    endif
 #  else
 #    ifndef OSHMEM_DECLSPEC
 #      define OSHMEM_DECLSPEC
 #    endif
-#    ifndef OSHMEM_MODULE_DECLSPEC
-#      define OSHMEM_MODULE_DECLSPEC
-#    endif
 #  endif
-#endif  /* defined(__WINDOWS__) */
 
 #endif
diff --git a/oshmem/include/pshmem.h b/oshmem/include/pshmem.h
index cc51b453477..2057336e5b8 100644
--- a/oshmem/include/pshmem.h
+++ b/oshmem/include/pshmem.h
@@ -45,7 +45,7 @@ OSHMEM_DECLSPEC  int pshmem_my_pe(void);
 OSHMEM_DECLSPEC  void pshmem_query_thread(int *provided);
 
 /*
- * Accessability routines
+ * Accessibility routines
  */
 OSHMEM_DECLSPEC int pshmem_pe_accessible(int pe);
 OSHMEM_DECLSPEC int pshmem_addr_accessible(const void *addr, int pe);
@@ -57,8 +57,10 @@ OSHMEM_DECLSPEC  void* pshmem_malloc(size_t size);
 OSHMEM_DECLSPEC  void* pshmem_calloc(size_t count, size_t size);
 OSHMEM_DECLSPEC  void* pshmem_align(size_t align, size_t size);
 OSHMEM_DECLSPEC  void* pshmem_realloc(void *ptr, size_t size);
+OSHMEM_DECLSPEC  void* pshmem_malloc_with_hints(size_t size, long hints);
 OSHMEM_DECLSPEC  void pshmem_free(void* ptr);
 
+
 /*
  * Remote pointer operations
  */
@@ -70,6 +72,560 @@ OSHMEM_DECLSPEC  void *pshmem_ptr(const void *ptr, int pe);
 OSHMEM_DECLSPEC int pshmem_ctx_create(long options, shmem_ctx_t *ctx);
 OSHMEM_DECLSPEC void pshmem_ctx_destroy(shmem_ctx_t ctx);
 
+/*
+ * Team management operations
+ */
+OSHMEM_DECLSPEC int pshmem_team_my_pe(shmem_team_t team);
+OSHMEM_DECLSPEC int pshmem_team_n_pes(shmem_team_t team);
+OSHMEM_DECLSPEC int pshmem_team_get_config(shmem_team_t team, long config_mask, shmem_team_config_t *config);
+OSHMEM_DECLSPEC int pshmem_team_translate_pe(shmem_team_t src_team, int src_pe, shmem_team_t dest_team);
+OSHMEM_DECLSPEC int pshmem_team_split_strided(shmem_team_t parent_team, int start, int stride, int size, const shmem_team_config_t *config, long config_mask, shmem_team_t *new_team);
+OSHMEM_DECLSPEC int pshmem_team_split_2d(shmem_team_t parent_team, int xrange, const shmem_team_config_t *xaxis_config, long xaxis_mask, shmem_team_t *xaxis_team, const shmem_team_config_t *yaxis_config, long yaxis_mask, shmem_team_t *yaxis_team);
+OSHMEM_DECLSPEC void pshmem_team_destroy(shmem_team_t team);
+OSHMEM_DECLSPEC int pshmem_ctx_get_team(shmem_ctx_t ctx, shmem_team_t *team);
+OSHMEM_DECLSPEC int pshmem_team_create_ctx(shmem_team_t team, long options, shmem_ctx_t *ctx);
+
+/*
+ * Teams-based Collectives
+ */
+
+/* Teams sync */
+OSHMEM_DECLSPEC  void pshmem_team_sync(shmem_team_t team);
+
+
+/* Teams alltoall */
+OSHMEM_DECLSPEC  int pshmem_char_alltoall(shmem_team_t team, char *target, const char *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_short_alltoall(shmem_team_t team, short *target, const short *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_int_alltoall(shmem_team_t team, int *target, const int *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_long_alltoall(shmem_team_t team, long *target, const long *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_float_alltoall(shmem_team_t team, float *target, const float *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_double_alltoall(shmem_team_t team, double *target, const double *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_longlong_alltoall(shmem_team_t team, long long *target, const long long *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_schar_alltoall(shmem_team_t team, signed char *target, const signed char *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_uchar_alltoall(shmem_team_t team, unsigned char *target, const unsigned char *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_ushort_alltoall(shmem_team_t team, unsigned short *target, const unsigned short *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_uint_alltoall(shmem_team_t team, unsigned int *target, const unsigned int *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_ulong_alltoall(shmem_team_t team, unsigned long *target, const unsigned long *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_ulonglong_alltoall(shmem_team_t team, unsigned long long *target, const unsigned long long *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_longdouble_alltoall(shmem_team_t team, long double *target, const long double *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_int8_alltoall(shmem_team_t team, int8_t *target, const int8_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_int16_alltoall(shmem_team_t team, int16_t *target, const int16_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_int32_alltoall(shmem_team_t team, int32_t *target, const int32_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_int64_alltoall(shmem_team_t team, int64_t *target, const int64_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_uint8_alltoall(shmem_team_t team, uint8_t *target, const uint8_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_uint16_alltoall(shmem_team_t team, uint16_t *target, const uint16_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_uint32_alltoall(shmem_team_t team, uint32_t *target, const uint32_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_uint64_alltoall(shmem_team_t team, uint64_t *target, const uint64_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_size_alltoall(shmem_team_t team, size_t *target, const size_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_ptrdiff_alltoall(shmem_team_t team, ptrdiff_t *target, const ptrdiff_t *source, size_t nelems);
+#if OSHMEM_HAVE_C11
+#define pshmem_alltoall(...)                                            \
+    _Generic(&*(__OSHMEM_VAR_ARG2(__VA_ARGS__)),                        \
+            char*:        pshmem_char_alltoall,               	        \
+            short*:       pshmem_short_alltoall,              	        \
+            int*:         pshmem_int_alltoall,                	        \
+            long*:        pshmem_long_alltoall,               	        \
+            long long*:   pshmem_longlong_alltoall,           	        \
+            signed char*:        pshmem_schar_alltoall,       	        \
+            unsigned char*:      pshmem_uchar_alltoall,       	        \
+            unsigned short*:     pshmem_ushort_alltoall,      	        \
+            unsigned int*:       pshmem_uint_alltoall,        	        \
+            unsigned long*:      pshmem_ulong_alltoall,       	        \
+            unsigned long long*: pshmem_ulonglong_alltoall,   	        \
+            float*:       pshmem_float_alltoall,              	        \
+            double*:      pshmem_double_alltoall,             	        \
+            long double*: pshmem_longdouble_alltoall,         	        \
+            default:      __opshmem_datatype_ignore)(__VA_ARGS__)
+#endif
+
+OSHMEM_DECLSPEC  int pshmem_alltoallmem(shmem_team_t team, void *target, const void *source, size_t nelems);
+
+/* Teams alltoalls */
+OSHMEM_DECLSPEC  int pshmem_char_alltoalls(shmem_team_t team, char *target, const char *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_short_alltoalls(shmem_team_t team, short *target, const short *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_int_alltoalls(shmem_team_t team, int *target, const int *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_long_alltoalls(shmem_team_t team, long *target, const long *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_float_alltoalls(shmem_team_t team, float *target, const float *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_double_alltoalls(shmem_team_t team, double *target, const double *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_longlong_alltoalls(shmem_team_t team, long long *target, const long long *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_schar_alltoalls(shmem_team_t team, signed char *target, const signed char *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_uchar_alltoalls(shmem_team_t team, unsigned char *target, const unsigned char *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_ushort_alltoalls(shmem_team_t team, unsigned short *target, const unsigned short *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_uint_alltoalls(shmem_team_t team, unsigned int *target, const unsigned int *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_ulong_alltoalls(shmem_team_t team, unsigned long *target, const unsigned long *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_ulonglong_alltoalls(shmem_team_t team, unsigned long long *target, const unsigned long long *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_longdouble_alltoalls(shmem_team_t team, long double *target, const long double *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_int8_alltoalls(shmem_team_t team, int8_t *target, const int8_t *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_int16_alltoalls(shmem_team_t team, int16_t *target, const int16_t *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_int32_alltoalls(shmem_team_t team, int32_t *target, const int32_t *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_int64_alltoalls(shmem_team_t team, int64_t *target, const int64_t *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_uint8_alltoalls(shmem_team_t team, uint8_t *target, const uint8_t *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_uint16_alltoalls(shmem_team_t team, uint16_t *target, const uint16_t *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_uint32_alltoalls(shmem_team_t team, uint32_t *target, const uint32_t *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_uint64_alltoalls(shmem_team_t team, uint64_t *target, const uint64_t *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_size_alltoalls(shmem_team_t team, size_t *target, const size_t *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_ptrdiff_alltoalls(shmem_team_t team, ptrdiff_t *target, const ptrdiff_t *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+#if OSHMEM_HAVE_C11
+#define pshmem_alltoalls(...)                                   \
+    _Generic(&*(__OSHMEM_VAR_ARG2(__VA_ARGS__)),                \
+            char*:        pshmem_char_alltoalls,               	\
+            short*:       pshmem_short_alltoalls,              	\
+            int*:         pshmem_int_alltoalls,                	\
+            long*:        pshmem_long_alltoalls,               	\
+            long long*:   pshmem_longlong_alltoalls,           	\
+            signed char*:        pshmem_schar_alltoalls,       	\
+            unsigned char*:      pshmem_uchar_alltoalls,       	\
+            unsigned short*:     pshmem_ushort_alltoalls,      	\
+            unsigned int*:       pshmem_uint_alltoalls,        	\
+            unsigned long*:      pshmem_ulong_alltoalls,       	\
+            unsigned long long*: pshmem_ulonglong_alltoalls,   	\
+            float*:       pshmem_float_alltoalls,              	\
+            double*:      pshmem_double_alltoalls,             	\
+            long double*: pshmem_longdouble_alltoalls,         	\
+            default:      __opshmem_datatype_ignore)(__VA_ARGS__)
+#endif
+
+OSHMEM_DECLSPEC  int pshmem_alltoallsmem(shmem_team_t team, void *target, const void *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+
+
+/* Teams broadcast */
+OSHMEM_DECLSPEC  int pshmem_char_broadcast(shmem_team_t team, char *target, const char *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int pshmem_short_broadcast(shmem_team_t team, short *target, const short *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int pshmem_int_broadcast(shmem_team_t team, int *target, const int *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int pshmem_long_broadcast(shmem_team_t team, long *target, const long *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int pshmem_float_broadcast(shmem_team_t team, float *target, const float *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int pshmem_double_broadcast(shmem_team_t team, double *target, const double *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int pshmem_longlong_broadcast(shmem_team_t team, long long *target, const long long *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int pshmem_schar_broadcast(shmem_team_t team, signed char *target, const signed char *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int pshmem_uchar_broadcast(shmem_team_t team, unsigned char *target, const unsigned char *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int pshmem_ushort_broadcast(shmem_team_t team, unsigned short *target, const unsigned short *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int pshmem_uint_broadcast(shmem_team_t team, unsigned int *target, const unsigned int *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int pshmem_ulong_broadcast(shmem_team_t team, unsigned long *target, const unsigned long *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int pshmem_ulonglong_broadcast(shmem_team_t team, unsigned long long *target, const unsigned long long *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int pshmem_longdouble_broadcast(shmem_team_t team, long double *target, const long double *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int pshmem_int8_broadcast(shmem_team_t team, int8_t *target, const int8_t *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int pshmem_int16_broadcast(shmem_team_t team, int16_t *target, const int16_t *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int pshmem_int32_broadcast(shmem_team_t team, int32_t *target, const int32_t *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int pshmem_int64_broadcast(shmem_team_t team, int64_t *target, const int64_t *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int pshmem_uint8_broadcast(shmem_team_t team, uint8_t *target, const uint8_t *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int pshmem_uint16_broadcast(shmem_team_t team, uint16_t *target, const uint16_t *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int pshmem_uint32_broadcast(shmem_team_t team, uint32_t *target, const uint32_t *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int pshmem_uint64_broadcast(shmem_team_t team, uint64_t *target, const uint64_t *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int pshmem_size_broadcast(shmem_team_t team, size_t *target, const size_t *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int pshmem_ptrdiff_broadcast(shmem_team_t team, ptrdiff_t *target, const ptrdiff_t *source, size_t nelems, int PE_root);
+#if OSHMEM_HAVE_C11
+#define pshmem_broadcast(...)                                   \
+    _Generic(&*(__OSHMEM_VAR_ARG2(__VA_ARGS__)),                \
+            char*:        pshmem_char_broadcast,               	\
+            short*:       pshmem_short_broadcast,              	\
+            int*:         pshmem_int_broadcast,                	\
+            long*:        pshmem_long_broadcast,               	\
+            long long*:   pshmem_longlong_broadcast,           	\
+            signed char*:        pshmem_schar_broadcast,       	\
+            unsigned char*:      pshmem_uchar_broadcast,       	\
+            unsigned short*:     pshmem_ushort_broadcast,      	\
+            unsigned int*:       pshmem_uint_broadcast,        	\
+            unsigned long*:      pshmem_ulong_broadcast,       	\
+            unsigned long long*: pshmem_ulonglong_broadcast,   	\
+            float*:       pshmem_float_broadcast,              	\
+            double*:      pshmem_double_broadcast,             	\
+            long double*: pshmem_longdouble_broadcast,         	\
+            default:      __opshmem_datatype_ignore)(__VA_ARGS__)
+#endif
+
+OSHMEM_DECLSPEC  int pshmem_broadcastmem(shmem_team_t team, void *target, const void *source, size_t nelems, int PE_root);
+
+/* Teams collect */
+OSHMEM_DECLSPEC  int pshmem_char_collect(shmem_team_t team, char *target, const char *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_short_collect(shmem_team_t team, short *target, const short *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_int_collect(shmem_team_t team, int *target, const int *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_long_collect(shmem_team_t team, long *target, const long *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_float_collect(shmem_team_t team, float *target, const float *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_double_collect(shmem_team_t team, double *target, const double *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_longlong_collect(shmem_team_t team, long long *target, const long long *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_schar_collect(shmem_team_t team, signed char *target, const signed char *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_uchar_collect(shmem_team_t team, unsigned char *target, const unsigned char *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_ushort_collect(shmem_team_t team, unsigned short *target, const unsigned short *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_uint_collect(shmem_team_t team, unsigned int *target, const unsigned int *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_ulong_collect(shmem_team_t team, unsigned long *target, const unsigned long *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_ulonglong_collect(shmem_team_t team, unsigned long long *target, const unsigned long long *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_longdouble_collect(shmem_team_t team, long double *target, const long double *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_int8_collect(shmem_team_t team, int8_t *target, const int8_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_int16_collect(shmem_team_t team, int16_t *target, const int16_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_int32_collect(shmem_team_t team, int32_t *target, const int32_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_int64_collect(shmem_team_t team, int64_t *target, const int64_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_uint8_collect(shmem_team_t team, uint8_t *target, const uint8_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_uint16_collect(shmem_team_t team, uint16_t *target, const uint16_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_uint32_collect(shmem_team_t team, uint32_t *target, const uint32_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_uint64_collect(shmem_team_t team, uint64_t *target, const uint64_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_size_collect(shmem_team_t team, size_t *target, const size_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_ptrdiff_collect(shmem_team_t team, ptrdiff_t *target, const ptrdiff_t *source, size_t nelems);
+#if OSHMEM_HAVE_C11
+#define pshmem_collect(...)                                     \
+    _Generic(&*(__OSHMEM_VAR_ARG2(__VA_ARGS__)),                \
+            char*:        pshmem_char_collect,               	\
+            short*:       pshmem_short_collect,              	\
+            int*:         pshmem_int_collect,                	\
+            long*:        pshmem_long_collect,               	\
+            long long*:   pshmem_longlong_collect,           	\
+            signed char*:        pshmem_schar_collect,       	\
+            unsigned char*:      pshmem_uchar_collect,       	\
+            unsigned short*:     pshmem_ushort_collect,      	\
+            unsigned int*:       pshmem_uint_collect,        	\
+            unsigned long*:      pshmem_ulong_collect,       	\
+            unsigned long long*: pshmem_ulonglong_collect,   	\
+            float*:       pshmem_float_collect,              	\
+            double*:      pshmem_double_collect,             	\
+            long double*: pshmem_longdouble_collect,         	\
+            default:      __opshmem_datatype_ignore)(__VA_ARGS__)
+#endif
+
+
+OSHMEM_DECLSPEC  int pshmem_collectmem(shmem_team_t team, void *target, const void *source, size_t nelems);
+
+/* Teams fcollect */
+OSHMEM_DECLSPEC  int pshmem_char_fcollect(shmem_team_t team, char *target, const char *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_short_fcollect(shmem_team_t team, short *target, const short *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_int_fcollect(shmem_team_t team, int *target, const int *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_long_fcollect(shmem_team_t team, long *target, const long *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_float_fcollect(shmem_team_t team, float *target, const float *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_double_fcollect(shmem_team_t team, double *target, const double *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_longlong_fcollect(shmem_team_t team, long long *target, const long long *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_schar_fcollect(shmem_team_t team, signed char *target, const signed char *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_uchar_fcollect(shmem_team_t team, unsigned char *target, const unsigned char *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_ushort_fcollect(shmem_team_t team, unsigned short *target, const unsigned short *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_uint_fcollect(shmem_team_t team, unsigned int *target, const unsigned int *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_ulong_fcollect(shmem_team_t team, unsigned long *target, const unsigned long *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_ulonglong_fcollect(shmem_team_t team, unsigned long long *target, const unsigned long long *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_longdouble_fcollect(shmem_team_t team, long double *target, const long double *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_int8_fcollect(shmem_team_t team, int8_t *target, const int8_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_int16_fcollect(shmem_team_t team, int16_t *target, const int16_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_int32_fcollect(shmem_team_t team, int32_t *target, const int32_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_int64_fcollect(shmem_team_t team, int64_t *target, const int64_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_uint8_fcollect(shmem_team_t team, uint8_t *target, const uint8_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_uint16_fcollect(shmem_team_t team, uint16_t *target, const uint16_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_uint32_fcollect(shmem_team_t team, uint32_t *target, const uint32_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_uint64_fcollect(shmem_team_t team, uint64_t *target, const uint64_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_size_fcollect(shmem_team_t team, size_t *target, const size_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int pshmem_ptrdiff_fcollect(shmem_team_t team, ptrdiff_t *target, const ptrdiff_t *source, size_t nelems);
+#if OSHMEM_HAVE_C11
+#define pshmem_fcollect(...)                                    \
+    _Generic(&*(__OSHMEM_VAR_ARG2(__VA_ARGS__)),                \
+            char*:        pshmem_char_fcollect,               	\
+            short*:       pshmem_short_fcollect,              	\
+            int*:         pshmem_int_fcollect,                	\
+            long*:        pshmem_long_fcollect,               	\
+            long long*:   pshmem_longlong_fcollect,           	\
+            signed char*:        pshmem_schar_fcollect,       	\
+            unsigned char*:      pshmem_uchar_fcollect,       	\
+            unsigned short*:     pshmem_ushort_fcollect,      	\
+            unsigned int*:       pshmem_uint_fcollect,        	\
+            unsigned long*:      pshmem_ulong_fcollect,       	\
+            unsigned long long*: pshmem_ulonglong_fcollect,   	\
+            float*:       pshmem_float_fcollect,              	\
+            double*:      pshmem_double_fcollect,             	\
+            long double*: pshmem_longdouble_fcollect,         	\
+            default:      __opshmem_datatype_ignore)(__VA_ARGS__)
+#endif
+
+OSHMEM_DECLSPEC  int pshmem_fcollectmem(shmem_team_t team, void *target, const void *source, size_t nelems);
+
+
+
+/* Teams reduction: AND */
+OSHMEM_DECLSPEC  int pshmem_uchar_and_reduce(shmem_team_t team, unsigned char *target, const unsigned char *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_ushort_and_reduce(shmem_team_t team, unsigned short *target, const unsigned short *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint_and_reduce(shmem_team_t team, unsigned int *target, const unsigned int *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_ulong_and_reduce(shmem_team_t team, unsigned long *target, const unsigned long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_ulonglong_and_reduce(shmem_team_t team, unsigned long long *target, const unsigned long long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int_and_reduce(shmem_team_t team, int *target, const int *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_longlong_and_reduce(shmem_team_t team, long long *target, const long long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int8_and_reduce(shmem_team_t team, int8_t *target, const int8_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int16_and_reduce(shmem_team_t team, int16_t *target, const int16_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int32_and_reduce(shmem_team_t team, int32_t *target, const int32_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int64_and_reduce(shmem_team_t team, int64_t *target, const int64_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint8_and_reduce(shmem_team_t team, uint8_t *target, const uint8_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint16_and_reduce(shmem_team_t team, uint16_t *target, const uint16_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint32_and_reduce(shmem_team_t team, uint32_t *target, const uint32_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint64_and_reduce(shmem_team_t team, uint64_t *target, const uint64_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_size_and_reduce(shmem_team_t team, size_t *target, const size_t *source, size_t nreduce);
+#if OSHMEM_HAVE_C11
+#define pshmem_and_reduce(...)                                          \
+    _Generic(&*(__OSHMEM_VAR_ARG2(__VA_ARGS__)),                        \
+            unsigned char*:      pshmem_uchar_and_reduce,       	\
+            unsigned short*:     pshmem_ushort_and_reduce,      	\
+            unsigned int*:       pshmem_uint_and_reduce,        	\
+            unsigned long*:      pshmem_ulong_and_reduce,       	\
+            unsigned long long*: pshmem_ulonglong_and_reduce,   	\
+            int8_t*: pshmem_int8_and_reduce,                    	\
+            int16_t*: pshmem_int16_and_reduce,                  	\
+            int32_t*: pshmem_int32_and_reduce,                  	\
+            int64_t*: pshmem_int64_and_reduce,                  	\
+            long long*: pshmem_longlong_and_reduce,                     \
+            default:      __opshmem_datatype_ignore)(__VA_ARGS__)
+#endif
+
+
+/* Teams reduction: OR */
+OSHMEM_DECLSPEC  int pshmem_uchar_or_reduce(shmem_team_t team, unsigned char *target, const unsigned char *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_ushort_or_reduce(shmem_team_t team, unsigned short *target, const unsigned short *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint_or_reduce(shmem_team_t team, unsigned int *target, const unsigned int *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_ulong_or_reduce(shmem_team_t team, unsigned long *target, const unsigned long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_ulonglong_or_reduce(shmem_team_t team, unsigned long long *target, const unsigned long long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int_or_reduce(shmem_team_t team, int *target, const int *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_longlong_or_reduce(shmem_team_t team, long long *target, const long long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int8_or_reduce(shmem_team_t team, int8_t *target, const int8_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int16_or_reduce(shmem_team_t team, int16_t *target, const int16_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int32_or_reduce(shmem_team_t team, int32_t *target, const int32_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int64_or_reduce(shmem_team_t team, int64_t *target, const int64_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint8_or_reduce(shmem_team_t team, uint8_t *target, const uint8_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint16_or_reduce(shmem_team_t team, uint16_t *target, const uint16_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint32_or_reduce(shmem_team_t team, uint32_t *target, const uint32_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint64_or_reduce(shmem_team_t team, uint64_t *target, const uint64_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_size_or_reduce(shmem_team_t team, size_t *target, const size_t *source, size_t nreduce);
+#if OSHMEM_HAVE_C11
+#define pshmem_or_reduce(...)                                   \
+    _Generic(&*(__OSHMEM_VAR_ARG2(__VA_ARGS__)),                \
+            unsigned char*:      pshmem_uchar_or_reduce,       	\
+            unsigned short*:     pshmem_ushort_or_reduce,      	\
+            unsigned int*:       pshmem_uint_or_reduce,        	\
+            unsigned long*:      pshmem_ulong_or_reduce,       	\
+            unsigned long long*: pshmem_ulonglong_or_reduce,   	\
+            int8_t*: pshmem_int8_or_reduce,                    	\
+            int16_t*: pshmem_int16_or_reduce,                  	\
+            int32_t*: pshmem_int32_or_reduce,                  	\
+            int64_t*: pshmem_int64_or_reduce,                  	\
+            long long*: pshmem_longlong_or_reduce,             	\
+            default:      __opshmem_datatype_ignore)(__VA_ARGS__)
+#endif
+
+
+
+/* Teams reduction: XOR */
+OSHMEM_DECLSPEC  int pshmem_uchar_xor_reduce(shmem_team_t team, unsigned char *target, const unsigned char *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_ushort_xor_reduce(shmem_team_t team, unsigned short *target, const unsigned short *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint_xor_reduce(shmem_team_t team, unsigned int *target, const unsigned int *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_ulong_xor_reduce(shmem_team_t team, unsigned long *target, const unsigned long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_ulonglong_xor_reduce(shmem_team_t team, unsigned long long *target, const unsigned long long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int_xor_reduce(shmem_team_t team, int *target, const int *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_longlong_xor_reduce(shmem_team_t team, long long *target, const long long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int8_xor_reduce(shmem_team_t team, int8_t *target, const int8_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int16_xor_reduce(shmem_team_t team, int16_t *target, const int16_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int32_xor_reduce(shmem_team_t team, int32_t *target, const int32_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int64_xor_reduce(shmem_team_t team, int64_t *target, const int64_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint8_xor_reduce(shmem_team_t team, uint8_t *target, const uint8_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint16_xor_reduce(shmem_team_t team, uint16_t *target, const uint16_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint32_xor_reduce(shmem_team_t team, uint32_t *target, const uint32_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint64_xor_reduce(shmem_team_t team, uint64_t *target, const uint64_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_size_xor_reduce(shmem_team_t team, size_t *target, const size_t *source, size_t nreduce);
+#if OSHMEM_HAVE_C11
+#define pshmem_xor_reduce(...)                                          \
+    _Generic(&*(__OSHMEM_VAR_ARG2(__VA_ARGS__)),                        \
+            unsigned char*:      pshmem_uchar_xor_reduce,       	\
+            unsigned short*:     pshmem_ushort_xor_reduce,      	\
+            unsigned int*:       pshmem_uint_xor_reduce,        	\
+            unsigned long*:      pshmem_ulong_xor_reduce,       	\
+            unsigned long long*: pshmem_ulonglong_xor_reduce,   	\
+            int8_t*: pshmem_int8_xor_reduce,                    	\
+            int16_t*: pshmem_int16_xor_reduce,                  	\
+            int32_t*: pshmem_int32_xor_reduce,                  	\
+            int64_t*: pshmem_int64_xor_reduce,                  	\
+            long long*: pshmem_longlong_xor_reduce,             	\
+            default:      __opshmem_datatype_ignore)(__VA_ARGS__)
+#endif
+
+/* Teams reduction: MAX */
+OSHMEM_DECLSPEC  int pshmem_char_max_reduce(shmem_team_t team, char *target, const char *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_short_max_reduce(shmem_team_t team, short *target, const short *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int_max_reduce(shmem_team_t team, int *target, const int *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_long_max_reduce(shmem_team_t team, long *target, const long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_float_max_reduce(shmem_team_t team, float *target, const float *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_double_max_reduce(shmem_team_t team, double *target, const double *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_longlong_max_reduce(shmem_team_t team, long long *target, const long long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_schar_max_reduce(shmem_team_t team, signed char *target, const signed char *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uchar_max_reduce(shmem_team_t team, unsigned char *target, const unsigned char *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_ushort_max_reduce(shmem_team_t team, unsigned short *target, const unsigned short *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint_max_reduce(shmem_team_t team, unsigned int *target, const unsigned int *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_ulong_max_reduce(shmem_team_t team, unsigned long *target, const unsigned long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_ulonglong_max_reduce(shmem_team_t team, unsigned long long *target, const unsigned long long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_longdouble_max_reduce(shmem_team_t team, long double *target, const long double *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int8_max_reduce(shmem_team_t team, int8_t *target, const int8_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int16_max_reduce(shmem_team_t team, int16_t *target, const int16_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int32_max_reduce(shmem_team_t team, int32_t *target, const int32_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int64_max_reduce(shmem_team_t team, int64_t *target, const int64_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint8_max_reduce(shmem_team_t team, uint8_t *target, const uint8_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint16_max_reduce(shmem_team_t team, uint16_t *target, const uint16_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint32_max_reduce(shmem_team_t team, uint32_t *target, const uint32_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint64_max_reduce(shmem_team_t team, uint64_t *target, const uint64_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_size_max_reduce(shmem_team_t team, size_t *target, const size_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_ptrdiff_max_reduce(shmem_team_t team, ptrdiff_t *target, const ptrdiff_t *source, size_t nreduce);
+#if OSHMEM_HAVE_C11
+#define pshmem_max_reduce(...)                                          \
+    _Generic(&*(__OSHMEM_VAR_ARG2(__VA_ARGS__)),                        \
+            char*:        pshmem_char_max_reduce,               	\
+            short*:       pshmem_short_max_reduce,              	\
+            int*:         pshmem_int_max_reduce,                	\
+            long*:        pshmem_long_max_reduce,               	\
+            long long*:   pshmem_longlong_max_reduce,           	\
+            signed char*:        pshmem_schar_max_reduce,       	\
+            unsigned char*:      pshmem_uchar_max_reduce,       	\
+            unsigned short*:     pshmem_ushort_max_reduce,      	\
+            unsigned int*:       pshmem_uint_max_reduce,        	\
+            unsigned long*:      pshmem_ulong_max_reduce,       	\
+            unsigned long long*: pshmem_ulonglong_max_reduce,   	\
+            float*:       pshmem_float_max_reduce,              	\
+            double*:      pshmem_double_max_reduce,             	\
+            long double*: pshmem_longdouble_max_reduce,         	\
+            default:      __opshmem_datatype_ignore)(__VA_ARGS__)
+#endif
+
+
+/* Teams reduction: MIN */
+OSHMEM_DECLSPEC  int pshmem_char_min_reduce(shmem_team_t team, char *target, const char *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_short_min_reduce(shmem_team_t team, short *target, const short *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int_min_reduce(shmem_team_t team, int *target, const int *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_long_min_reduce(shmem_team_t team, long *target, const long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_float_min_reduce(shmem_team_t team, float *target, const float *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_double_min_reduce(shmem_team_t team, double *target, const double *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_longlong_min_reduce(shmem_team_t team, long long *target, const long long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_schar_min_reduce(shmem_team_t team, signed char *target, const signed char *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uchar_min_reduce(shmem_team_t team, unsigned char *target, const unsigned char *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_ushort_min_reduce(shmem_team_t team, unsigned short *target, const unsigned short *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint_min_reduce(shmem_team_t team, unsigned int *target, const unsigned int *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_ulong_min_reduce(shmem_team_t team, unsigned long *target, const unsigned long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_ulonglong_min_reduce(shmem_team_t team, unsigned long long *target, const unsigned long long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_longdouble_min_reduce(shmem_team_t team, long double *target, const long double *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int8_min_reduce(shmem_team_t team, int8_t *target, const int8_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int16_min_reduce(shmem_team_t team, int16_t *target, const int16_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int32_min_reduce(shmem_team_t team, int32_t *target, const int32_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int64_min_reduce(shmem_team_t team, int64_t *target, const int64_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint8_min_reduce(shmem_team_t team, uint8_t *target, const uint8_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint16_min_reduce(shmem_team_t team, uint16_t *target, const uint16_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint32_min_reduce(shmem_team_t team, uint32_t *target, const uint32_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint64_min_reduce(shmem_team_t team, uint64_t *target, const uint64_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_size_min_reduce(shmem_team_t team, size_t *target, const size_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_ptrdiff_min_reduce(shmem_team_t team, ptrdiff_t *target, const ptrdiff_t *source, size_t nreduce);
+#if OSHMEM_HAVE_C11
+#define pshmem_min_reduce(...)                                          \
+    _Generic(&*(__OSHMEM_VAR_ARG2(__VA_ARGS__)),                        \
+            char*:        pshmem_char_min_reduce,               	\
+            short*:       pshmem_short_min_reduce,              	\
+            int*:         pshmem_int_min_reduce,                	\
+            long*:        pshmem_long_min_reduce,               	\
+            long long*:   pshmem_longlong_min_reduce,           	\
+            signed char*:        pshmem_schar_min_reduce,       	\
+            unsigned char*:      pshmem_uchar_min_reduce,       	\
+            unsigned short*:     pshmem_ushort_min_reduce,      	\
+            unsigned int*:       pshmem_uint_min_reduce,        	\
+            unsigned long*:      pshmem_ulong_min_reduce,       	\
+            unsigned long long*: pshmem_ulonglong_min_reduce,   	\
+            float*:       pshmem_float_min_reduce,              	\
+            double*:      pshmem_double_min_reduce,             	\
+            long double*: pshmem_longdouble_min_reduce,         	\
+            default:      __opshmem_datatype_ignore)(__VA_ARGS__)
+#endif
+
+
+
+/* Teams reduction: SUM */
+OSHMEM_DECLSPEC  int pshmem_char_sum_reduce(shmem_team_t team, char *target, const char *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_short_sum_reduce(shmem_team_t team, short *target, const short *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int_sum_reduce(shmem_team_t team, int *target, const int *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_long_sum_reduce(shmem_team_t team, long *target, const long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_float_sum_reduce(shmem_team_t team, float *target, const float *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_double_sum_reduce(shmem_team_t team, double *target, const double *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_longlong_sum_reduce(shmem_team_t team, long long *target, const long long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_schar_sum_reduce(shmem_team_t team, signed char *target, const signed char *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uchar_sum_reduce(shmem_team_t team, unsigned char *target, const unsigned char *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_ushort_sum_reduce(shmem_team_t team, unsigned short *target, const unsigned short *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint_sum_reduce(shmem_team_t team, unsigned int *target, const unsigned int *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_ulong_sum_reduce(shmem_team_t team, unsigned long *target, const unsigned long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_ulonglong_sum_reduce(shmem_team_t team, unsigned long long *target, const unsigned long long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_longdouble_sum_reduce(shmem_team_t team, long double *target, const long double *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int8_sum_reduce(shmem_team_t team, int8_t *target, const int8_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int16_sum_reduce(shmem_team_t team, int16_t *target, const int16_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int32_sum_reduce(shmem_team_t team, int32_t *target, const int32_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int64_sum_reduce(shmem_team_t team, int64_t *target, const int64_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint8_sum_reduce(shmem_team_t team, uint8_t *target, const uint8_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint16_sum_reduce(shmem_team_t team, uint16_t *target, const uint16_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint32_sum_reduce(shmem_team_t team, uint32_t *target, const uint32_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint64_sum_reduce(shmem_team_t team, uint64_t *target, const uint64_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_size_sum_reduce(shmem_team_t team, size_t *target, const size_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_ptrdiff_sum_reduce(shmem_team_t team, ptrdiff_t *target, const ptrdiff_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_complexd_sum_reduce(shmem_team_t team, OSHMEM_COMPLEX_TYPE(double) *target, const OSHMEM_COMPLEX_TYPE(double) *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_complexf_sum_reduce(shmem_team_t team, OSHMEM_COMPLEX_TYPE(float) *target, const OSHMEM_COMPLEX_TYPE(float) *source, size_t nreduce);
+#if OSHMEM_HAVE_C11
+#define pshmem_sum_reduce(...)                                                  \
+    _Generic(&*(__OSHMEM_VAR_ARG2(__VA_ARGS__)),                                \
+            char*:        pshmem_char_sum_reduce,               		\
+            short*:       pshmem_short_sum_reduce,              		\
+            int*:         pshmem_int_sum_reduce,                		\
+            long*:        pshmem_long_sum_reduce,               		\
+            long long*:   pshmem_longlong_sum_reduce,           		\
+            signed char*:        pshmem_schar_sum_reduce,       		\
+            unsigned char*:      pshmem_uchar_sum_reduce,       		\
+            unsigned short*:     pshmem_ushort_sum_reduce,      		\
+            unsigned int*:       pshmem_uint_sum_reduce,        		\
+            unsigned long*:      pshmem_ulong_sum_reduce,       		\
+            unsigned long long*: pshmem_ulonglong_sum_reduce,   		\
+            float*:       pshmem_float_sum_reduce,              		\
+            double*:      pshmem_double_sum_reduce,             		\
+            long double*: pshmem_longdouble_sum_reduce,         		\
+            OSHMEM_COMPLEX_TYPE(double)*: pshmem_complexd_sum_reduce,           \
+            OSHMEM_COMPLEX_TYPE(float)*: pshmem_complexf_sum_reduce,            \
+            default:      __opshmem_datatype_ignore)(__VA_ARGS__)
+#endif
+
+
+/* Teams reduction: PROD */
+OSHMEM_DECLSPEC  int pshmem_char_prod_reduce(shmem_team_t team, char *target, const char *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_short_prod_reduce(shmem_team_t team, short *target, const short *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int_prod_reduce(shmem_team_t team, int *target, const int *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_long_prod_reduce(shmem_team_t team, long *target, const long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_float_prod_reduce(shmem_team_t team, float *target, const float *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_double_prod_reduce(shmem_team_t team, double *target, const double *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_longlong_prod_reduce(shmem_team_t team, long long *target, const long long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_schar_prod_reduce(shmem_team_t team, signed char *target, const signed char *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uchar_prod_reduce(shmem_team_t team, unsigned char *target, const unsigned char *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_ushort_prod_reduce(shmem_team_t team, unsigned short *target, const unsigned short *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint_prod_reduce(shmem_team_t team, unsigned int *target, const unsigned int *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_ulong_prod_reduce(shmem_team_t team, unsigned long *target, const unsigned long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_ulonglong_prod_reduce(shmem_team_t team, unsigned long long *target, const unsigned long long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_longdouble_prod_reduce(shmem_team_t team, long double *target, const long double *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int8_prod_reduce(shmem_team_t team, int8_t *target, const int8_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int16_prod_reduce(shmem_team_t team, int16_t *target, const int16_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int32_prod_reduce(shmem_team_t team, int32_t *target, const int32_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_int64_prod_reduce(shmem_team_t team, int64_t *target, const int64_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint8_prod_reduce(shmem_team_t team, uint8_t *target, const uint8_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint16_prod_reduce(shmem_team_t team, uint16_t *target, const uint16_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint32_prod_reduce(shmem_team_t team, uint32_t *target, const uint32_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_uint64_prod_reduce(shmem_team_t team, uint64_t *target, const uint64_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_size_prod_reduce(shmem_team_t team, size_t *target, const size_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_ptrdiff_prod_reduce(shmem_team_t team, ptrdiff_t *target, const ptrdiff_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_complexd_prod_reduce(shmem_team_t team, OSHMEM_COMPLEX_TYPE(double) *target, const OSHMEM_COMPLEX_TYPE(double) *source, size_t nreduce);
+OSHMEM_DECLSPEC  int pshmem_complexf_prod_reduce(shmem_team_t team, OSHMEM_COMPLEX_TYPE(float) *target, const OSHMEM_COMPLEX_TYPE(float) *source, size_t nreduce);
+#if OSHMEM_HAVE_C11
+#define pshmem_prod_reduce(...)                                                 \
+    _Generic(&*(__OSHMEM_VAR_ARG2(__VA_ARGS__)),                                \
+            char*:        pshmem_char_prod_reduce,               		\
+            short*:       pshmem_short_prod_reduce,              		\
+            int*:         pshmem_int_prod_reduce,                		\
+            long*:        pshmem_long_prod_reduce,               		\
+            long long*:   pshmem_longlong_prod_reduce,           		\
+            signed char*:        pshmem_schar_prod_reduce,       		\
+            unsigned char*:      pshmem_uchar_prod_reduce,       		\
+            unsigned short*:     pshmem_ushort_prod_reduce,      		\
+            unsigned int*:       pshmem_uint_prod_reduce,        		\
+            unsigned long*:      pshmem_ulong_prod_reduce,       		\
+            unsigned long long*: pshmem_ulonglong_prod_reduce,   		\
+            float*:       pshmem_float_prod_reduce,              		\
+            double*:      pshmem_double_prod_reduce,             		\
+            long double*: pshmem_longdouble_prod_reduce,         		\
+            OSHMEM_COMPLEX_TYPE(double)*: pshmem_complexd_prod_reduce,          \
+            OSHMEM_COMPLEX_TYPE(float)*: pshmem_complexf_prod_reduce,           \
+            default:      __opshmem_datatype_ignore)(__VA_ARGS__)
+#endif
+
 /*
  * Elemental put routines
  */
@@ -458,6 +1014,217 @@ OSHMEM_DECLSPEC  void pshmem_put64_nbi(void *target, const void *source, size_t
 OSHMEM_DECLSPEC  void pshmem_put128_nbi(void *target, const void *source, size_t len, int pe);
 OSHMEM_DECLSPEC  void pshmem_putmem_nbi(void *target, const void *source, size_t len, int pe);
 
+
+/*
+ * Signaled put routines
+ */
+OSHMEM_DECLSPEC  void pshmem_ctx_char_put_signal(shmem_ctx_t ctx, char *dest, const char *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_short_put_signal(shmem_ctx_t ctx, short *dest, const short *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_int_put_signal(shmem_ctx_t ctx, int *dest, const int *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_long_put_signal(shmem_ctx_t ctx, long *dest, const long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_float_put_signal(shmem_ctx_t ctx, float *dest, const float *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_double_put_signal(shmem_ctx_t ctx, double *dest, const double *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_longlong_put_signal(shmem_ctx_t ctx, long long *dest, const long long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_schar_put_signal(shmem_ctx_t ctx, signed char *dest, const signed char *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_uchar_put_signal(shmem_ctx_t ctx, unsigned char *dest, const unsigned char *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_ushort_put_signal(shmem_ctx_t ctx, unsigned short *dest, const unsigned short *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_uint_put_signal(shmem_ctx_t ctx, unsigned int *dest, const unsigned int *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_ulong_put_signal(shmem_ctx_t ctx, unsigned long *dest, const unsigned long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_ulonglong_put_signal(shmem_ctx_t ctx, unsigned long long *dest, const unsigned long long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_longdouble_put_signal(shmem_ctx_t ctx, long double *dest, const long double *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_int8_put_signal(shmem_ctx_t ctx, int8_t *dest, const int8_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_int16_put_signal(shmem_ctx_t ctx, int16_t *dest, const int16_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_int32_put_signal(shmem_ctx_t ctx, int32_t *dest, const int32_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_int64_put_signal(shmem_ctx_t ctx, int64_t *dest, const int64_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_uint8_put_signal(shmem_ctx_t ctx, uint8_t *dest, const uint8_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_uint16_put_signal(shmem_ctx_t ctx, uint16_t *dest, const uint16_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_uint32_put_signal(shmem_ctx_t ctx, uint32_t *dest, const uint32_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_uint64_put_signal(shmem_ctx_t ctx, uint64_t *dest, const uint64_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_size_put_signal(shmem_ctx_t ctx, size_t *dest, const size_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_ptrdiff_put_signal(shmem_ctx_t ctx, ptrdiff_t *dest, const ptrdiff_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+
+OSHMEM_DECLSPEC  void pshmem_char_put_signal(char *dest, const char *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_short_put_signal(short *dest, const short *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_int_put_signal(int *dest, const int *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_long_put_signal(long *dest, const long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_float_put_signal(float *dest, const float *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_double_put_signal(double *dest, const double *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_longlong_put_signal(long long *dest, const long long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_schar_put_signal(signed char *dest, const signed char *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_uchar_put_signal(unsigned char *dest, const unsigned char *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ushort_put_signal(unsigned short *dest, const unsigned short *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_uint_put_signal(unsigned int *dest, const unsigned int *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ulong_put_signal(unsigned long *dest, const unsigned long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ulonglong_put_signal(unsigned long long *dest, const unsigned long long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_longdouble_put_signal(long double *dest, const long double *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_int8_put_signal(int8_t *dest, const int8_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_int16_put_signal(int16_t *dest, const int16_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_int32_put_signal(int32_t *dest, const int32_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_int64_put_signal(int64_t *dest, const int64_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_uint8_put_signal(uint8_t *dest, const uint8_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_uint16_put_signal(uint16_t *dest, const uint16_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_uint32_put_signal(uint32_t *dest, const uint32_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_uint64_put_signal(uint64_t *dest, const uint64_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_size_put_signal(size_t *dest, const size_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ptrdiff_put_signal(ptrdiff_t *dest, const ptrdiff_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+
+#if OSHMEM_HAVE_C11
+#define pshmem_put_signal(...)                                                \
+    _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                              \
+            shmem_ctx_t:  _Generic((__OSHMEM_VAR_ARG2(__VA_ARGS__)),          \
+                char*:       pshmem_ctx_char_put_signal,                      \
+                short*:      pshmem_ctx_short_put_signal,                     \
+                int*:        pshmem_ctx_int_put_signal,                       \
+                long*:       pshmem_ctx_long_put_signal,                      \
+                long long*:  pshmem_ctx_longlong_put_signal,                  \
+                signed char*:        pshmem_ctx_schar_put_signal,             \
+                unsigned char*:      pshmem_ctx_uchar_put_signal,             \
+                unsigned short*:     pshmem_ctx_ushort_put_signal,            \
+                unsigned int*:       pshmem_ctx_uint_put_signal,              \
+                unsigned long*:      pshmem_ctx_ulong_put_signal,             \
+                unsigned long long*: pshmem_ctx_ulonglong_put_signal,         \
+                float*:       pshmem_ctx_float_put_signal,                    \
+                double*:      pshmem_ctx_double_put_signal,                   \
+                long double*: pshmem_ctx_longdouble_put_signal,               \
+                default:      __opshmem_datatype_ignore),                     \
+            char*:       pshmem_char_put_signal,                              \
+            short*:      pshmem_short_put_signal,                             \
+            int*:        pshmem_int_put_signal,                               \
+            long*:       pshmem_long_put_signal,                              \
+            long long*:  pshmem_longlong_put_signal,                          \
+            signed char*:        pshmem_schar_put_signal,                     \
+            unsigned char*:      pshmem_uchar_put_signal,                     \
+            unsigned short*:     pshmem_ushort_put_signal,                    \
+            unsigned int*:       pshmem_uint_put_signal,                      \
+            unsigned long*:      pshmem_ulong_put_signal,                     \
+            unsigned long long*: pshmem_ulonglong_put_signal,                 \
+            float*:       pshmem_float_put_signal,                            \
+            double*:      pshmem_double_put_signal,                           \
+            long double*: pshmem_longdouble_put_signal)(__VA_ARGS__)
+#endif
+
+OSHMEM_DECLSPEC void pshmem_put8_signal(void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void pshmem_put16_signal(void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void pshmem_put32_signal(void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void pshmem_put64_signal(void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void pshmem_put128_signal(void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+
+OSHMEM_DECLSPEC void pshmem_ctx_put8_signal(shmem_ctx_t ctx, void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_put16_signal(shmem_ctx_t ctx, void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_put32_signal(shmem_ctx_t ctx, void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_put64_signal(shmem_ctx_t ctx, void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_put128_signal(shmem_ctx_t ctx, void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+
+OSHMEM_DECLSPEC void pshmem_putmem_signal(void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_putmem_signal(shmem_ctx_t ctx, void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+
+/*
+ * Nonblocking signaled put routines
+ */
+OSHMEM_DECLSPEC  void pshmem_ctx_char_put_signal_nbi(shmem_ctx_t ctx, char *dest, const char *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_short_put_signal_nbi(shmem_ctx_t ctx, short *dest, const short *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_int_put_signal_nbi(shmem_ctx_t ctx, int *dest, const int *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_long_put_signal_nbi(shmem_ctx_t ctx, long *dest, const long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_float_put_signal_nbi(shmem_ctx_t ctx, float *dest, const float *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_double_put_signal_nbi(shmem_ctx_t ctx, double *dest, const double *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_longlong_put_signal_nbi(shmem_ctx_t ctx, long long *dest, const long long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_schar_put_signal_nbi(shmem_ctx_t ctx, signed char *dest, const signed char *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_uchar_put_signal_nbi(shmem_ctx_t ctx, unsigned char *dest, const unsigned char *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_ushort_put_signal_nbi(shmem_ctx_t ctx, unsigned short *dest, const unsigned short *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_uint_put_signal_nbi(shmem_ctx_t ctx, unsigned int *dest, const unsigned int *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_ulong_put_signal_nbi(shmem_ctx_t ctx, unsigned long *dest, const unsigned long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_ulonglong_put_signal_nbi(shmem_ctx_t ctx, unsigned long long *dest, const unsigned long long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_longdouble_put_signal_nbi(shmem_ctx_t ctx, long double *dest, const long double *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_int8_put_signal_nbi(shmem_ctx_t ctx, int8_t *dest, const int8_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_int16_put_signal_nbi(shmem_ctx_t ctx, int16_t *dest, const int16_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_int32_put_signal_nbi(shmem_ctx_t ctx, int32_t *dest, const int32_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_int64_put_signal_nbi(shmem_ctx_t ctx, int64_t *dest, const int64_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_uint8_put_signal_nbi(shmem_ctx_t ctx, uint8_t *dest, const uint8_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_uint16_put_signal_nbi(shmem_ctx_t ctx, uint16_t *dest, const uint16_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_uint32_put_signal_nbi(shmem_ctx_t ctx, uint32_t *dest, const uint32_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_uint64_put_signal_nbi(shmem_ctx_t ctx, uint64_t *dest, const uint64_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_size_put_signal_nbi(shmem_ctx_t ctx, size_t *dest, const size_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ctx_ptrdiff_put_signal_nbi(shmem_ctx_t ctx, ptrdiff_t *dest, const ptrdiff_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+
+OSHMEM_DECLSPEC  void pshmem_char_put_signal_nbi(char *dest, const char *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_short_put_signal_nbi(short *dest, const short *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_int_put_signal_nbi(int *dest, const int *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_long_put_signal_nbi(long *dest, const long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_float_put_signal_nbi(float *dest, const float *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_double_put_signal_nbi(double *dest, const double *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_longlong_put_signal_nbi(long long *dest, const long long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_schar_put_signal_nbi(signed char *dest, const signed char *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_uchar_put_signal_nbi(unsigned char *dest, const unsigned char *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ushort_put_signal_nbi(unsigned short *dest, const unsigned short *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_uint_put_signal_nbi(unsigned int *dest, const unsigned int *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ulong_put_signal_nbi(unsigned long *dest, const unsigned long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ulonglong_put_signal_nbi(unsigned long long *dest, const unsigned long long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_longdouble_put_signal_nbi(long double *dest, const long double *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_int8_put_signal_nbi(int8_t *dest, const int8_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_int16_put_signal_nbi(int16_t *dest, const int16_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_int32_put_signal_nbi(int32_t *dest, const int32_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_int64_put_signal_nbi(int64_t *dest, const int64_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_uint8_put_signal_nbi(uint8_t *dest, const uint8_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_uint16_put_signal_nbi(uint16_t *dest, const uint16_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_uint32_put_signal_nbi(uint32_t *dest, const uint32_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_uint64_put_signal_nbi(uint64_t *dest, const uint64_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_size_put_signal_nbi(size_t *dest, const size_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void pshmem_ptrdiff_put_signal_nbi(ptrdiff_t *dest, const ptrdiff_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+
+#if OSHMEM_HAVE_C11
+#define pshmem_put_signal_nbi(...)                                                \
+    _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                                  \
+            shmem_ctx_t:  _Generic((__OSHMEM_VAR_ARG2(__VA_ARGS__)),              \
+                char*:       pshmem_ctx_char_put_signal_nbi,                      \
+                short*:      pshmem_ctx_short_put_signal_nbi,                     \
+                int*:        pshmem_ctx_int_put_signal_nbi,                       \
+                long*:       pshmem_ctx_long_put_signal_nbi,                      \
+                long long*:  pshmem_ctx_longlong_put_signal_nbi,                  \
+                signed char*:        pshmem_ctx_schar_put_signal_nbi,             \
+                unsigned char*:      pshmem_ctx_uchar_put_signal_nbi,             \
+                unsigned short*:     pshmem_ctx_ushort_put_signal_nbi,            \
+                unsigned int*:       pshmem_ctx_uint_put_signal_nbi,              \
+                unsigned long*:      pshmem_ctx_ulong_put_signal_nbi,             \
+                unsigned long long*: pshmem_ctx_ulonglong_put_signal_nbi,         \
+                float*:       pshmem_ctx_float_put_signal_nbi,                    \
+                double*:      pshmem_ctx_double_put_signal_nbi,                   \
+                long double*: pshmem_ctx_longdouble_put_signal_nbi,               \
+                default:      __opshmem_datatype_ignore),                         \
+            char*:       pshmem_char_put_signal_nbi,                              \
+            short*:      pshmem_short_put_signal_nbi,                             \
+            int*:        pshmem_int_put_signal_nbi,                               \
+            long*:       pshmem_long_put_signal_nbi,                              \
+            long long*:  pshmem_longlong_put_signal_nbi,                          \
+            signed char*:        pshmem_schar_put_signal_nbi,                     \
+            unsigned char*:      pshmem_uchar_put_signal_nbi,                     \
+            unsigned short*:     pshmem_ushort_put_signal_nbi,                    \
+            unsigned int*:       pshmem_uint_put_signal_nbi,                      \
+            unsigned long*:      pshmem_ulong_put_signal_nbi,                     \
+            unsigned long long*: pshmem_ulonglong_put_signal_nbi,                 \
+            float*:       pshmem_float_put_signal_nbi,                            \
+            double*:      pshmem_double_put_signal_nbi,                           \
+            long double*: pshmem_longdouble_put_signal_nbi)(__VA_ARGS__)
+#endif
+
+OSHMEM_DECLSPEC void pshmem_put8_signal_nbi(void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void pshmem_put16_signal_nbi(void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void pshmem_put32_signal_nbi(void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void pshmem_put64_signal_nbi(void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void pshmem_put128_signal_nbi(void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+
+OSHMEM_DECLSPEC void pshmem_ctx_put8_signal_nbi(shmem_ctx_t ctx, void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_put16_signal_nbi(shmem_ctx_t ctx, void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_put32_signal_nbi(shmem_ctx_t ctx, void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_put64_signal_nbi(shmem_ctx_t ctx, void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_put128_signal_nbi(shmem_ctx_t ctx, void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+
+OSHMEM_DECLSPEC void pshmem_putmem_signal_nbi(void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_putmem_signal_nbi(shmem_ctx_t ctx, void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+
+
+OSHMEM_DECLSPEC uint64_t pshmem_signal_fetch(const uint64_t *sig_addr);
+
+
 /*
  * Elemental get routines
  */
@@ -858,6 +1625,12 @@ OSHMEM_DECLSPEC unsigned long pshmem_ctx_ulong_atomic_swap(shmem_ctx_t ctx, unsi
 OSHMEM_DECLSPEC unsigned long long pshmem_ctx_ulonglong_atomic_swap(shmem_ctx_t ctx, unsigned long long *target, unsigned long long value, int pe);
 OSHMEM_DECLSPEC float pshmem_ctx_float_atomic_swap(shmem_ctx_t ctx, float *target, float value, int pe);
 OSHMEM_DECLSPEC double pshmem_ctx_double_atomic_swap(shmem_ctx_t ctx, double *target, double value, int pe);
+OSHMEM_DECLSPEC int32_t pshmem_ctx_int32_atomic_swap(shmem_ctx_t ctx, int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC int64_t pshmem_ctx_int64_atomic_swap(shmem_ctx_t ctx, int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC uint32_t pshmem_ctx_uint32_atomic_swap(shmem_ctx_t ctx, uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC uint64_t pshmem_ctx_uint64_atomic_swap(shmem_ctx_t ctx, uint64_t *target, uint64_t value, int pe);
+OSHMEM_DECLSPEC size_t pshmem_ctx_size_atomic_swap(shmem_ctx_t ctx, size_t *target, size_t value, int pe);
+OSHMEM_DECLSPEC ptrdiff_t pshmem_ctx_ptrdiff_atomic_swap(shmem_ctx_t ctx, ptrdiff_t *target, ptrdiff_t value, int pe);
 
 OSHMEM_DECLSPEC int pshmem_int_atomic_swap(int *target, int value, int pe);
 OSHMEM_DECLSPEC long pshmem_long_atomic_swap(long *target, long value, int pe);
@@ -867,6 +1640,13 @@ OSHMEM_DECLSPEC unsigned long pshmem_ulong_atomic_swap(unsigned long *target, un
 OSHMEM_DECLSPEC unsigned long long pshmem_ulonglong_atomic_swap(unsigned long long *target, unsigned long long value, int pe);
 OSHMEM_DECLSPEC float pshmem_float_atomic_swap(float *target, float value, int pe);
 OSHMEM_DECLSPEC double pshmem_double_atomic_swap(double *target, double value, int pe);
+OSHMEM_DECLSPEC int32_t pshmem_int32_atomic_swap( int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC int64_t pshmem_int64_atomic_swap( int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC uint32_t pshmem_uint32_atomic_swap( uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC uint64_t pshmem_uint64_atomic_swap( uint64_t *target, uint64_t value, int pe);
+OSHMEM_DECLSPEC size_t pshmem_size_atomic_swap( size_t *target, size_t value, int pe);
+OSHMEM_DECLSPEC ptrdiff_t pshmem_ptrdiff_atomic_swap( ptrdiff_t *target, ptrdiff_t value, int pe);
+
 #if OSHMEM_HAVE_C11
 #define pshmem_atomic_swap(...)                                       \
     _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                      \
@@ -914,6 +1694,12 @@ OSHMEM_DECLSPEC void pshmem_ctx_ulong_atomic_set(shmem_ctx_t ctx, unsigned long
 OSHMEM_DECLSPEC void pshmem_ctx_ulonglong_atomic_set(shmem_ctx_t ctx, unsigned long long *target, unsigned long long value, int pe);
 OSHMEM_DECLSPEC void pshmem_ctx_float_atomic_set(shmem_ctx_t ctx, float *target, float value, int pe);
 OSHMEM_DECLSPEC void pshmem_ctx_double_atomic_set(shmem_ctx_t ctx, double *target, double value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_int32_atomic_set(shmem_ctx_t ctx, int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_int64_atomic_set(shmem_ctx_t ctx, int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_uint32_atomic_set(shmem_ctx_t ctx, uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_uint64_atomic_set(shmem_ctx_t ctx, uint64_t *target, uint64_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_size_atomic_set(shmem_ctx_t ctx, size_t *target, size_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_ptrdiff_atomic_set(shmem_ctx_t ctx, ptrdiff_t *target, ptrdiff_t value, int pe);
 
 OSHMEM_DECLSPEC void pshmem_int_atomic_set(int *target, int value, int pe);
 OSHMEM_DECLSPEC void pshmem_long_atomic_set(long *target, long value, int pe);
@@ -923,6 +1709,13 @@ OSHMEM_DECLSPEC void pshmem_ulong_atomic_set(unsigned long *target, unsigned lon
 OSHMEM_DECLSPEC void pshmem_ulonglong_atomic_set(unsigned long long *target, unsigned long long value, int pe);
 OSHMEM_DECLSPEC void pshmem_float_atomic_set(float *target, float value, int pe);
 OSHMEM_DECLSPEC void pshmem_double_atomic_set(double *target, double value, int pe);
+OSHMEM_DECLSPEC void pshmem_int32_atomic_set(int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_int64_atomic_set(int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_uint32_atomic_set(uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_uint64_atomic_set(uint64_t *target, uint64_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_size_atomic_set(size_t *target, size_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ptrdiff_atomic_set(ptrdiff_t *target, ptrdiff_t value, int pe);
+
 #if OSHMEM_HAVE_C11
 #define pshmem_atomic_set(...)                                       \
     _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                     \
@@ -968,6 +1761,12 @@ OSHMEM_DECLSPEC long long pshmem_ctx_longlong_atomic_compare_swap(shmem_ctx_t ct
 OSHMEM_DECLSPEC unsigned int pshmem_ctx_uint_atomic_compare_swap(shmem_ctx_t ctx, unsigned int *target, unsigned int cond, unsigned int value, int pe);
 OSHMEM_DECLSPEC unsigned long pshmem_ctx_ulong_atomic_compare_swap(shmem_ctx_t ctx, unsigned long *target, unsigned long cond, unsigned long value, int pe);
 OSHMEM_DECLSPEC unsigned long long pshmem_ctx_ulonglong_atomic_compare_swap(shmem_ctx_t ctx, unsigned long long *target, unsigned long long cond, unsigned long long value, int pe);
+OSHMEM_DECLSPEC int32_t pshmem_ctx_int32_atomic_compare_swap(shmem_ctx_t ctx, int32_t *target, int32_t cond, int32_t value, int pe);
+OSHMEM_DECLSPEC int64_t pshmem_ctx_int64_atomic_compare_swap(shmem_ctx_t ctx, int64_t *target, int64_t cond, int64_t value, int pe);
+OSHMEM_DECLSPEC uint32_t pshmem_ctx_uint32_atomic_compare_swap(shmem_ctx_t ctx, uint32_t *target, uint32_t cond, uint32_t value, int pe);
+OSHMEM_DECLSPEC uint64_t pshmem_ctx_uint64_atomic_compare_swap(shmem_ctx_t ctx, uint64_t *target, uint64_t cond, uint64_t value, int pe);
+OSHMEM_DECLSPEC size_t pshmem_ctx_size_atomic_compare_swap(shmem_ctx_t ctx, size_t *target, size_t cond, size_t value, int pe);
+OSHMEM_DECLSPEC ptrdiff_t pshmem_ctx_ptrdiff_atomic_compare_swap(shmem_ctx_t ctx, ptrdiff_t *target, ptrdiff_t cond, ptrdiff_t value, int pe);
 
 OSHMEM_DECLSPEC int pshmem_int_atomic_compare_swap(int *target, int cond, int value, int pe);
 OSHMEM_DECLSPEC long pshmem_long_atomic_compare_swap(long *target, long cond, long value, int pe);
@@ -975,6 +1774,12 @@ OSHMEM_DECLSPEC long long pshmem_longlong_atomic_compare_swap(long long *target,
 OSHMEM_DECLSPEC unsigned int pshmem_uint_atomic_compare_swap(unsigned int *target, unsigned int cond, unsigned int value, int pe);
 OSHMEM_DECLSPEC unsigned long pshmem_ulong_atomic_compare_swap(unsigned long *target, unsigned long cond, unsigned long value, int pe);
 OSHMEM_DECLSPEC unsigned long long pshmem_ulonglong_atomic_compare_swap(unsigned long long *target, unsigned long long cond, unsigned long long value, int pe);
+OSHMEM_DECLSPEC int32_t pshmem_int32_atomic_compare_swap( int32_t *target, int32_t cond, int32_t value, int pe);
+OSHMEM_DECLSPEC int64_t pshmem_int64_atomic_compare_swap( int64_t *target, int64_t cond, int64_t value, int pe);
+OSHMEM_DECLSPEC uint32_t pshmem_uint32_atomic_compare_swap( uint32_t *target, uint32_t cond, uint32_t value, int pe);
+OSHMEM_DECLSPEC uint64_t pshmem_uint64_atomic_compare_swap( uint64_t *target, uint64_t cond, uint64_t value, int pe);
+OSHMEM_DECLSPEC size_t pshmem_size_atomic_compare_swap( size_t *target, size_t cond, size_t value, int pe);
+OSHMEM_DECLSPEC ptrdiff_t pshmem_ptrdiff_atomic_compare_swap( ptrdiff_t *target, ptrdiff_t cond, ptrdiff_t value, int pe);
 
 #if OSHMEM_HAVE_C11
 #define pshmem_atomic_compare_swap(...)                                \
@@ -1013,6 +1818,12 @@ OSHMEM_DECLSPEC long long pshmem_ctx_longlong_atomic_fetch_add(shmem_ctx_t ctx,
 OSHMEM_DECLSPEC unsigned int pshmem_ctx_uint_atomic_fetch_add(shmem_ctx_t ctx, unsigned int *target, unsigned int value, int pe);
 OSHMEM_DECLSPEC unsigned long pshmem_ctx_ulong_atomic_fetch_add(shmem_ctx_t ctx, unsigned long *target, unsigned long value, int pe);
 OSHMEM_DECLSPEC unsigned long long pshmem_ctx_ulonglong_atomic_fetch_add(shmem_ctx_t ctx, unsigned long long *target, unsigned long long value, int pe);
+OSHMEM_DECLSPEC int32_t pshmem_ctx_int32_atomic_fetch_add(shmem_ctx_t ctx, int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC int64_t pshmem_ctx_int64_atomic_fetch_add(shmem_ctx_t ctx, int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC uint32_t pshmem_ctx_uint32_atomic_fetch_add(shmem_ctx_t ctx, uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC uint64_t pshmem_ctx_uint64_atomic_fetch_add(shmem_ctx_t ctx, uint64_t *target, uint64_t value, int pe);
+OSHMEM_DECLSPEC size_t pshmem_ctx_size_atomic_fetch_add(shmem_ctx_t ctx, size_t *target, size_t value, int pe);
+OSHMEM_DECLSPEC ptrdiff_t pshmem_ctx_ptrdiff_atomic_fetch_add(shmem_ctx_t ctx, ptrdiff_t *target, ptrdiff_t value, int pe);
 
 OSHMEM_DECLSPEC int pshmem_int_atomic_fetch_add(int *target, int value, int pe);
 OSHMEM_DECLSPEC long pshmem_long_atomic_fetch_add(long *target, long value, int pe);
@@ -1020,6 +1831,13 @@ OSHMEM_DECLSPEC long long pshmem_longlong_atomic_fetch_add(long long *target, lo
 OSHMEM_DECLSPEC unsigned int pshmem_uint_atomic_fetch_add(unsigned int *target, unsigned int value, int pe);
 OSHMEM_DECLSPEC unsigned long pshmem_ulong_atomic_fetch_add(unsigned long *target, unsigned long value, int pe);
 OSHMEM_DECLSPEC unsigned long long pshmem_ulonglong_atomic_fetch_add(unsigned long long *target, unsigned long long value, int pe);
+OSHMEM_DECLSPEC int32_t pshmem_int32_atomic_fetch_add(int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC int64_t pshmem_int64_atomic_fetch_add(int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC uint32_t pshmem_uint32_atomic_fetch_add(uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC uint64_t pshmem_uint64_atomic_fetch_add(uint64_t *target, uint64_t value, int pe);
+OSHMEM_DECLSPEC size_t pshmem_size_atomic_fetch_add(size_t *target, size_t value, int pe);
+OSHMEM_DECLSPEC ptrdiff_t pshmem_ptrdiff_atomic_fetch_add(ptrdiff_t *target, ptrdiff_t value, int pe);
+
 #if OSHMEM_HAVE_C11
 #define pshmem_atomic_fetch_add(...)                                        \
     _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                            \
@@ -1206,6 +2024,12 @@ OSHMEM_DECLSPEC unsigned long pshmem_ctx_ulong_atomic_fetch(shmem_ctx_t ctx, con
 OSHMEM_DECLSPEC unsigned long long pshmem_ctx_ulonglong_atomic_fetch(shmem_ctx_t ctx, const unsigned long long *target, int pe);
 OSHMEM_DECLSPEC float pshmem_ctx_float_atomic_fetch(shmem_ctx_t ctx, const float *target, int pe);
 OSHMEM_DECLSPEC double pshmem_ctx_double_atomic_fetch(shmem_ctx_t ctx, const double *target, int pe);
+OSHMEM_DECLSPEC int32_t pshmem_ctx_int32_atomic_fetch(shmem_ctx_t ctx, const int32_t *target, int pe);
+OSHMEM_DECLSPEC int64_t pshmem_ctx_int64_atomic_fetch(shmem_ctx_t ctx, const int64_t *target, int pe);
+OSHMEM_DECLSPEC uint32_t pshmem_ctx_uint32_atomic_fetch(shmem_ctx_t ctx, const uint32_t *target, int pe);
+OSHMEM_DECLSPEC uint64_t pshmem_ctx_uint64_atomic_fetch(shmem_ctx_t ctx, const uint64_t *target, int pe);
+OSHMEM_DECLSPEC size_t pshmem_ctx_size_atomic_fetch(shmem_ctx_t ctx, const size_t *target, int pe);
+OSHMEM_DECLSPEC ptrdiff_t pshmem_ctx_ptrdiff_atomic_fetch(shmem_ctx_t ctx, const ptrdiff_t *target, int pe);
 
 OSHMEM_DECLSPEC int pshmem_int_atomic_fetch(const int *target, int pe);
 OSHMEM_DECLSPEC long pshmem_long_atomic_fetch(const long *target, int pe);
@@ -1215,6 +2039,12 @@ OSHMEM_DECLSPEC unsigned long pshmem_ulong_atomic_fetch(const unsigned long *tar
 OSHMEM_DECLSPEC unsigned long long pshmem_ulonglong_atomic_fetch(const unsigned long long *target, int pe);
 OSHMEM_DECLSPEC float pshmem_float_atomic_fetch(const float *target, int pe);
 OSHMEM_DECLSPEC double pshmem_double_atomic_fetch(const double *target, int pe);
+OSHMEM_DECLSPEC int32_t pshmem_int32_atomic_fetch(const int32_t *target, int pe);
+OSHMEM_DECLSPEC int64_t pshmem_int64_atomic_fetch(const int64_t *target, int pe);
+OSHMEM_DECLSPEC uint32_t pshmem_uint32_atomic_fetch(const uint32_t *target, int pe);
+OSHMEM_DECLSPEC uint64_t pshmem_uint64_atomic_fetch(const uint64_t *target, int pe);
+OSHMEM_DECLSPEC size_t pshmem_size_atomic_fetch(const size_t *target, int pe);
+OSHMEM_DECLSPEC ptrdiff_t pshmem_ptrdiff_atomic_fetch(const ptrdiff_t *target, int pe);
 #if OSHMEM_HAVE_C11
 #define pshmem_atomic_fetch(...)                                        \
     _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                        \
@@ -1260,6 +2090,12 @@ OSHMEM_DECLSPEC long long pshmem_ctx_longlong_atomic_fetch_inc(shmem_ctx_t ctx,
 OSHMEM_DECLSPEC unsigned int pshmem_ctx_uint_atomic_fetch_inc(shmem_ctx_t ctx, unsigned int *target, int pe);
 OSHMEM_DECLSPEC unsigned long pshmem_ctx_ulong_atomic_fetch_inc(shmem_ctx_t ctx, unsigned long *target, int pe);
 OSHMEM_DECLSPEC unsigned long long pshmem_ctx_ulonglong_atomic_fetch_inc(shmem_ctx_t ctx, unsigned long long *target, int pe);
+OSHMEM_DECLSPEC int32_t pshmem_ctx_int32_atomic_fetch_inc(shmem_ctx_t ctx, int32_t *target, int pe);
+OSHMEM_DECLSPEC int64_t pshmem_ctx_int64_atomic_fetch_inc(shmem_ctx_t ctx, int64_t *target, int pe);
+OSHMEM_DECLSPEC uint32_t pshmem_ctx_uint32_atomic_fetch_inc(shmem_ctx_t ctx, uint32_t *target, int pe);
+OSHMEM_DECLSPEC uint64_t pshmem_ctx_uint64_atomic_fetch_inc(shmem_ctx_t ctx, uint64_t *target, int pe);
+OSHMEM_DECLSPEC size_t pshmem_ctx_size_atomic_fetch_inc(shmem_ctx_t ctx, size_t *target, int pe);
+OSHMEM_DECLSPEC ptrdiff_t pshmem_ctx_ptrdiff_atomic_fetch_inc(shmem_ctx_t ctx, ptrdiff_t *target, int pe);
 
 OSHMEM_DECLSPEC int pshmem_int_atomic_fetch_inc(int *target, int pe);
 OSHMEM_DECLSPEC long pshmem_long_atomic_fetch_inc(long *target, int pe);
@@ -1267,6 +2103,13 @@ OSHMEM_DECLSPEC long long pshmem_longlong_atomic_fetch_inc(long long *target, in
 OSHMEM_DECLSPEC unsigned int pshmem_uint_atomic_fetch_inc(unsigned int *target, int pe);
 OSHMEM_DECLSPEC unsigned long pshmem_ulong_atomic_fetch_inc(unsigned long *target, int pe);
 OSHMEM_DECLSPEC unsigned long long pshmem_ulonglong_atomic_fetch_inc(unsigned long long *target, int pe);
+OSHMEM_DECLSPEC int32_t pshmem_int32_atomic_fetch_inc(int32_t *target, int pe);
+OSHMEM_DECLSPEC int64_t pshmem_int64_atomic_fetch_inc(int64_t *target, int pe);
+OSHMEM_DECLSPEC uint32_t pshmem_uint32_atomic_fetch_inc(uint32_t *target, int pe);
+OSHMEM_DECLSPEC uint64_t pshmem_uint64_atomic_fetch_inc(uint64_t *target, int pe);
+OSHMEM_DECLSPEC size_t pshmem_size_atomic_fetch_inc(size_t *target, int pe);
+OSHMEM_DECLSPEC ptrdiff_t pshmem_ptrdiff_atomic_fetch_inc(ptrdiff_t *target, int pe);
+
 #if OSHMEM_HAVE_C11
 #define pshmem_atomic_fetch_inc(...)                                 \
     _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                     \
@@ -1304,6 +2147,12 @@ OSHMEM_DECLSPEC void pshmem_ctx_longlong_atomic_add(shmem_ctx_t ctx, long long *
 OSHMEM_DECLSPEC void pshmem_ctx_uint_atomic_add(shmem_ctx_t ctx, unsigned int *target, unsigned int value, int pe);
 OSHMEM_DECLSPEC void pshmem_ctx_ulong_atomic_add(shmem_ctx_t ctx, unsigned long *target, unsigned long value, int pe);
 OSHMEM_DECLSPEC void pshmem_ctx_ulonglong_atomic_add(shmem_ctx_t ctx, unsigned long long *target, unsigned long long value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_int32_atomic_add(shmem_ctx_t ctx, int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_int64_atomic_add(shmem_ctx_t ctx, int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_uint32_atomic_add(shmem_ctx_t ctx, uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_uint64_atomic_add(shmem_ctx_t ctx, uint64_t *target, uint64_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_size_atomic_add(shmem_ctx_t ctx, size_t *target, size_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_ptrdiff_atomic_add(shmem_ctx_t ctx, ptrdiff_t *target, ptrdiff_t value, int pe);
 
 OSHMEM_DECLSPEC void pshmem_int_atomic_add(int *target, int value, int pe);
 OSHMEM_DECLSPEC void pshmem_long_atomic_add(long *target, long value, int pe);
@@ -1311,6 +2160,13 @@ OSHMEM_DECLSPEC void pshmem_longlong_atomic_add(long long *target, long long val
 OSHMEM_DECLSPEC void pshmem_uint_atomic_add(unsigned int *target, unsigned int value, int pe);
 OSHMEM_DECLSPEC void pshmem_ulong_atomic_add(unsigned long *target, unsigned long value, int pe);
 OSHMEM_DECLSPEC void pshmem_ulonglong_atomic_add(unsigned long long *target, unsigned long long value, int pe);
+OSHMEM_DECLSPEC void pshmem_int32_atomic_add(int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_int64_atomic_add(int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_uint32_atomic_add(uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_uint64_atomic_add(uint64_t *target, uint64_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_size_atomic_add(size_t *target, size_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ptrdiff_atomic_add(ptrdiff_t *target, ptrdiff_t value, int pe);
+
 #if OSHMEM_HAVE_C11
 #define pshmem_atomic_add(...)                                        \
     _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                      \
@@ -1471,6 +2327,12 @@ OSHMEM_DECLSPEC void pshmem_ctx_longlong_atomic_inc(shmem_ctx_t ctx, long long *
 OSHMEM_DECLSPEC void pshmem_ctx_uint_atomic_inc(shmem_ctx_t ctx, unsigned int *target, int pe);
 OSHMEM_DECLSPEC void pshmem_ctx_ulong_atomic_inc(shmem_ctx_t ctx, unsigned long *target, int pe);
 OSHMEM_DECLSPEC void pshmem_ctx_ulonglong_atomic_inc(shmem_ctx_t ctx, unsigned long long *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_int32_atomic_inc(shmem_ctx_t ctx, int32_t *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_int64_atomic_inc(shmem_ctx_t ctx, int64_t *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_uint32_atomic_inc(shmem_ctx_t ctx, uint32_t *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_uint64_atomic_inc(shmem_ctx_t ctx, uint64_t *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_size_atomic_inc(shmem_ctx_t ctx, size_t *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_ptrdiff_atomic_inc(shmem_ctx_t ctx, ptrdiff_t *target, int pe);
 
 OSHMEM_DECLSPEC void pshmem_int_atomic_inc(int *target, int pe);
 OSHMEM_DECLSPEC void pshmem_long_atomic_inc(long *target, int pe);
@@ -1478,6 +2340,13 @@ OSHMEM_DECLSPEC void pshmem_longlong_atomic_inc(long long *target, int pe);
 OSHMEM_DECLSPEC void pshmem_uint_atomic_inc(unsigned int *target, int pe);
 OSHMEM_DECLSPEC void pshmem_ulong_atomic_inc(unsigned long *target, int pe);
 OSHMEM_DECLSPEC void pshmem_ulonglong_atomic_inc(unsigned long long *target, int pe);
+OSHMEM_DECLSPEC void pshmem_int32_atomic_inc(int32_t *target, int pe);
+OSHMEM_DECLSPEC void pshmem_int64_atomic_inc(int64_t *target, int pe);
+OSHMEM_DECLSPEC void pshmem_uint32_atomic_inc(uint32_t *target, int pe);
+OSHMEM_DECLSPEC void pshmem_uint64_atomic_inc(uint64_t *target, int pe);
+OSHMEM_DECLSPEC void pshmem_size_atomic_inc(size_t *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ptrdiff_atomic_inc(ptrdiff_t *target, int pe);
+
 #if OSHMEM_HAVE_C11
 #define pshmem_atomic_inc(...)                                       \
     _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                     \
@@ -1508,6 +2377,377 @@ OSHMEM_DECLSPEC void pshmem_longlong_inc(long long *target, int pe);
             long long*:   pshmem_longlong_inc)(dst, pe)
 #endif
 
+/*
+ * Nonblocking atomic memory operations
+ */
+
+
+/* Atomic Nonblocking Fetch */
+OSHMEM_DECLSPEC void pshmem_ctx_int_atomic_fetch_nbi(shmem_ctx_t ctx, int *fetch, const int *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_long_atomic_fetch_nbi(shmem_ctx_t ctx, long *fetch, const long *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_longlong_atomic_fetch_nbi(shmem_ctx_t ctx, long long *fetch, const long long *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_uint_atomic_fetch_nbi(shmem_ctx_t ctx, unsigned int *fetch, const unsigned int *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_ulong_atomic_fetch_nbi(shmem_ctx_t ctx, unsigned long *fetch,  const unsigned long *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_ulonglong_atomic_fetch_nbi(shmem_ctx_t ctx, unsigned long long *fetch, const unsigned long long *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_float_atomic_fetch_nbi(shmem_ctx_t ctx, float *fetch, const float *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_double_atomic_fetch_nbi(shmem_ctx_t ctx, double *fetch, const double *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_int32_atomic_fetch_nbi(shmem_ctx_t ctx, int32_t *fetch, const int32_t *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_int64_atomic_fetch_nbi(shmem_ctx_t ctx, int64_t *fetch, const int64_t *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_uint32_atomic_fetch_nbi(shmem_ctx_t ctx, uint32_t *fetch, const uint32_t *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_uint64_atomic_fetch_nbi(shmem_ctx_t ctx, uint64_t *fetch, const uint64_t *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_size_atomic_fetch_nbi(shmem_ctx_t ctx, size_t *fetch, const size_t *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_ptrdiff_atomic_fetch_nbi(shmem_ctx_t ctx, ptrdiff_t *fetch, const ptrdiff_t *target, int pe);
+
+OSHMEM_DECLSPEC void pshmem_int_atomic_fetch_nbi(int *fetch, const int *target, int pe);
+OSHMEM_DECLSPEC void pshmem_long_atomic_fetch_nbi(long *fetch, const long *target, int pe);
+OSHMEM_DECLSPEC void pshmem_longlong_atomic_fetch_nbi(long long *fetch, const long long *target, int pe);
+OSHMEM_DECLSPEC void pshmem_uint_atomic_fetch_nbi(unsigned int *fetch, const unsigned int *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ulong_atomic_fetch_nbi(unsigned long *fetch, const unsigned long *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ulonglong_atomic_fetch_nbi(unsigned long long *fetch, const unsigned long long *target, int pe);
+OSHMEM_DECLSPEC void pshmem_float_atomic_fetch_nbi(float *fetch, const float *target, int pe);
+OSHMEM_DECLSPEC void pshmem_double_atomic_fetch_nbi(double *fetch, const double *target, int pe);
+OSHMEM_DECLSPEC void pshmem_int32_atomic_fetch_nbi(int32_t *fetch, const int32_t *target, int pe);
+OSHMEM_DECLSPEC void pshmem_int64_atomic_fetch_nbi(int64_t *fetch, const int64_t *target, int pe);
+OSHMEM_DECLSPEC void pshmem_uint32_atomic_fetch_nbi(uint32_t *fetch, const uint32_t *target, int pe);
+OSHMEM_DECLSPEC void pshmem_uint64_atomic_fetch_nbi(uint64_t *fetch, const uint64_t *target, int pe);
+OSHMEM_DECLSPEC void pshmem_size_atomic_fetch_nbi(size_t *fetch, const size_t *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ptrdiff_atomic_fetch_nbi(ptrdiff_t *fetch, const ptrdiff_t *target, int pe);
+
+#if OSHMEM_HAVE_C11
+#define pshmem_atomic_fetch_nbi(...)                                        \
+    _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                            \
+            shmem_ctx_t:  _Generic((__OSHMEM_VAR_ARG2(__VA_ARGS__)),        \
+                int*:         pshmem_ctx_int_atomic_fetch_nbi,              \
+                long*:        pshmem_ctx_long_atomic_fetch_nbi,             \
+                long long*:   pshmem_ctx_longlong_atomic_fetch_nbi,         \
+                unsigned int*:       pshmem_ctx_uint_atomic_fetch_nbi,      \
+                unsigned long*:      pshmem_ctx_ulong_atomic_fetch_nbi,     \
+                unsigned long long*: pshmem_ctx_ulonglong_atomic_fetch_nbi, \
+                float*:       pshmem_ctx_float_atomic_fetch_nbi,            \
+                double*:      pshmem_ctx_double_atomic_fetch_nbi,           \
+                size_t*:      pshmem_ctx_size_atomic_fetch_nbi,             \
+                ptrdiff_t*:      pshmem_ctx_ptrdiff_atomic_fetch_nbi,       \
+                default:      __opshmem_datatype_ignore),                   \
+            int*:        pshmem_int_atomic_fetch_nbi,                       \
+            long*:       pshmem_long_atomic_fetch_nbi,                      \
+            long long*:  pshmem_longlong_atomic_fetch_nbi,                  \
+            unsigned int*:       pshmem_uint_atomic_fetch_nbi,              \
+            unsigned long*:      pshmem_ulong_atomic_fetch_nbi,             \
+            unsigned long long*: pshmem_ulonglong_atomic_fetch_nbi,         \
+            float*:       pshmem_float_atomic_fetch_nbi,                    \
+            double*:      pshmem_double_atomic_fetch_nbi,                   \
+            size_t*:      pshmem_size_atomic_fetch_nbi,                     \
+            ptrdiff_t*:   pshmem_ptrdiff_atomic_fetch_nbi)(__VA_ARGS__)
+#endif
+
+
+/* Atomic Nonblocking Compare and Swap */
+OSHMEM_DECLSPEC void pshmem_ctx_int_atomic_compare_swap_nbi(shmem_ctx_t ctx, int *fetch, int *target, int cond, int value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_long_atomic_compare_swap_nbi(shmem_ctx_t ctx, long *fetch, long *target, long cond, long value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_longlong_atomic_compare_swap_nbi(shmem_ctx_t ctx, long long *fetch, long long *target, long long cond, long long value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_uint_atomic_compare_swap_nbi(shmem_ctx_t ctx, unsigned int *fetch, unsigned int *target, unsigned int cond, unsigned int value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_ulong_atomic_compare_swap_nbi(shmem_ctx_t ctx, unsigned long *fetch, unsigned long *target, unsigned long cond, unsigned long value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_ulonglong_atomic_compare_swap_nbi(shmem_ctx_t ctx, unsigned long long *fetch, unsigned long long *target, unsigned long long cond, unsigned long long value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_int32_atomic_compare_swap_nbi(shmem_ctx_t ctx, int32_t *fetch, int32_t *target, int32_t cond, int32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_int64_atomic_compare_swap_nbi(shmem_ctx_t ctx, int64_t *fetch, int64_t *target, int64_t cond, int64_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_uint32_atomic_compare_swap_nbi(shmem_ctx_t ctx, uint32_t *fetch, uint32_t *target, uint32_t cond, uint32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_uint64_atomic_compare_swap_nbi(shmem_ctx_t ctx, uint64_t *fetch, uint64_t *target, uint64_t cond, uint64_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_size_atomic_compare_swap_nbi(shmem_ctx_t ctx, size_t *fetch, size_t *target, size_t cond, size_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_ptrdiff_atomic_compare_swap_nbi(shmem_ctx_t ctx, ptrdiff_t *fetch, ptrdiff_t *target, ptrdiff_t cond, ptrdiff_t value, int pe);
+
+OSHMEM_DECLSPEC void pshmem_int_atomic_compare_swap_nbi(int *fetch, int *target, int cond, int value, int pe);
+OSHMEM_DECLSPEC void pshmem_long_atomic_compare_swap_nbi(long *fetch, long *target, long cond, long value, int pe);
+OSHMEM_DECLSPEC void pshmem_longlong_atomic_compare_swap_nbi(long long *fetch, long long *target, long long cond, long long value, int pe);
+OSHMEM_DECLSPEC void pshmem_uint_atomic_compare_swap_nbi(unsigned int *fetch, unsigned int *target, unsigned int cond, unsigned int value, int pe);
+OSHMEM_DECLSPEC void pshmem_ulong_atomic_compare_swap_nbi(unsigned long *fetch, unsigned long *target, unsigned long cond, unsigned long value, int pe);
+OSHMEM_DECLSPEC void pshmem_ulonglong_atomic_compare_swap_nbi(unsigned long long *fetch, unsigned long long *target, unsigned long long cond, unsigned long long value, int pe);
+OSHMEM_DECLSPEC void pshmem_int32_atomic_compare_swap_nbi(int32_t *fetch, int32_t *target, int32_t cond, int32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_int64_atomic_compare_swap_nbi(int64_t *fetch, int64_t *target, int64_t cond, int64_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_uint32_atomic_compare_swap_nbi(uint32_t *fetch, uint32_t *target, uint32_t cond, uint32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_uint64_atomic_compare_swap_nbi(uint64_t *fetch, uint64_t *target, uint64_t cond, uint64_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_size_atomic_compare_swap_nbi(size_t *fetch, size_t *target, size_t cond, size_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ptrdiff_atomic_compare_swap_nbi(ptrdiff_t *fetch, ptrdiff_t *target, ptrdiff_t cond, ptrdiff_t value, int pe);
+
+#if OSHMEM_HAVE_C11
+#define pshmem_atomic_compare_swap_nbi(...)                                        \
+    _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                                   \
+            shmem_ctx_t:  _Generic((__OSHMEM_VAR_ARG2(__VA_ARGS__)),               \
+                int*:         pshmem_ctx_int_atomic_compare_swap_nbi,              \
+                long*:        pshmem_ctx_long_atomic_compare_swap_nbi,             \
+                long long*:   pshmem_ctx_longlong_atomic_compare_swap_nbi,         \
+                unsigned int*:       pshmem_ctx_uint_atomic_compare_swap_nbi,      \
+                unsigned long*:      pshmem_ctx_ulong_atomic_compare_swap_nbi,     \
+                unsigned long long*: pshmem_ctx_ulonglong_atomic_compare_swap_nbi, \
+                default:      __opshmem_datatype_ignore),                          \
+            int*:        pshmem_int_atomic_compare_swap_nbi,                       \
+            long*:       pshmem_long_atomic_compare_swap_nbi,                      \
+            long long*:  pshmem_longlong_atomic_compare_swap_nbi,                  \
+            unsigned int*:       pshmem_uint_atomic_compare_swap_nbi,              \
+            unsigned long*:      pshmem_ulong_atomic_compare_swap_nbi,             \
+            unsigned long long*: pshmem_ulonglong_atomic_compare_swap_nbi)(__VA_ARGS__)
+#endif
+
+/* Atomic Nonblocking Swap */
+OSHMEM_DECLSPEC void pshmem_ctx_int_atomic_swap_nbi(shmem_ctx_t ctx, int *fetch, int *target, int value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_long_atomic_swap_nbi(shmem_ctx_t ctx, long *fetch, long *target, long value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_longlong_atomic_swap_nbi(shmem_ctx_t ctx, long long *fetch, long long *target, long long value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_uint_atomic_swap_nbi(shmem_ctx_t ctx, unsigned int *fetch, unsigned int *target, unsigned int value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_ulong_atomic_swap_nbi(shmem_ctx_t ctx, unsigned long *fetch, unsigned long *target, unsigned long value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_ulonglong_atomic_swap_nbi(shmem_ctx_t ctx, unsigned long long *fetch, unsigned long long *target, unsigned long long value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_float_atomic_swap_nbi(shmem_ctx_t ctx, float *fetch, float *target, float value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_double_atomic_swap_nbi(shmem_ctx_t ctx, double *fetch, double *target, double value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_int32_atomic_swap_nbi(shmem_ctx_t ctx, int32_t *fetch, int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_int64_atomic_swap_nbi(shmem_ctx_t ctx, int64_t *fetch, int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_uint32_atomic_swap_nbi(shmem_ctx_t ctx, uint32_t *fetch, uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_uint64_atomic_swap_nbi(shmem_ctx_t ctx, uint64_t *fetch, uint64_t *target, uint64_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_size_atomic_swap_nbi(shmem_ctx_t ctx, size_t *fetch, size_t *target, size_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_ptrdiff_atomic_swap_nbi(shmem_ctx_t ctx, ptrdiff_t *fetch, ptrdiff_t *target,  ptrdiff_t value, int pe);
+
+OSHMEM_DECLSPEC void pshmem_int_atomic_swap_nbi(int *fetch, int *target, int value, int pe);
+OSHMEM_DECLSPEC void pshmem_long_atomic_swap_nbi(long *fetch, long *target, long value, int pe);
+OSHMEM_DECLSPEC void pshmem_longlong_atomic_swap_nbi(long long *fetch, long long *target, long long value, int pe);
+OSHMEM_DECLSPEC void pshmem_uint_atomic_swap_nbi(unsigned int *fetch, unsigned int *target, unsigned int value, int pe);
+OSHMEM_DECLSPEC void pshmem_ulong_atomic_swap_nbi(unsigned long *fetch, unsigned long *target, unsigned long value, int pe);
+OSHMEM_DECLSPEC void pshmem_ulonglong_atomic_swap_nbi(unsigned long long *fetch, unsigned long long *target, unsigned long long value, int pe);
+OSHMEM_DECLSPEC void pshmem_float_atomic_swap_nbi(float *fetch, float *target, float value, int pe);
+OSHMEM_DECLSPEC void pshmem_double_atomic_swap_nbi(double *fetch, double *target, double value, int pe);
+OSHMEM_DECLSPEC void pshmem_int32_atomic_swap_nbi(int32_t *fetch, int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_int64_atomic_swap_nbi(int64_t *fetch, int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_uint32_atomic_swap_nbi(uint32_t *fetch, uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_uint64_atomic_swap_nbi(uint64_t *fetch, uint64_t *target, uint64_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_size_atomic_swap_nbi(size_t *fetch, size_t *target, size_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ptrdiff_atomic_swap_nbi(ptrdiff_t *fetch, ptrdiff_t *target, ptrdiff_t value, int pe);
+
+#if OSHMEM_HAVE_C11
+#define pshmem_atomic_swap_nbi(...)                                        \
+    _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                           \
+            shmem_ctx_t:  _Generic((__OSHMEM_VAR_ARG2(__VA_ARGS__)),       \
+                int*:         pshmem_ctx_int_atomic_swap_nbi,              \
+                long*:        pshmem_ctx_long_atomic_swap_nbi,             \
+                long long*:   pshmem_ctx_longlong_atomic_swap_nbi,         \
+                unsigned int*:       pshmem_ctx_uint_atomic_swap_nbi,      \
+                unsigned long*:      pshmem_ctx_ulong_atomic_swap_nbi,     \
+                unsigned long long*: pshmem_ctx_ulonglong_atomic_swap_nbi, \
+                float*:      pshmem_ctx_float_atomic_swap_nbi,             \
+                double*:      pshmem_ctx_double_atomic_swap_nbi,           \
+                default:      __opshmem_datatype_ignore),                  \
+            int*:        pshmem_int_atomic_swap_nbi,                       \
+            long*:       pshmem_long_atomic_swap_nbi,                      \
+            long long*:  pshmem_longlong_atomic_swap_nbi,                  \
+            unsigned int*:       pshmem_uint_atomic_swap_nbi,              \
+            unsigned long*:      pshmem_ulong_atomic_swap_nbi,             \
+            unsigned long long*: pshmem_ulonglong_atomic_swap_nbi,         \
+            float*:      pshmem_float_atomic_swap_nbi,                     \
+            double*:     pshmem_double_atomic_swap_nbi)(__VA_ARGS__)
+#endif
+
+
+
+/* Atomic Nonblocking Fetch and Increment */
+OSHMEM_DECLSPEC void pshmem_ctx_int_atomic_fetch_inc_nbi(shmem_ctx_t ctx, int *fetch, int *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_long_atomic_fetch_inc_nbi(shmem_ctx_t ctx, long *fetch, long *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_longlong_atomic_fetch_inc_nbi(shmem_ctx_t ctx, long long *fetch, long long *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_uint_atomic_fetch_inc_nbi(shmem_ctx_t ctx, unsigned int *fetch, unsigned int *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_ulong_atomic_fetch_inc_nbi(shmem_ctx_t ctx, unsigned long *fetch,  unsigned long *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_ulonglong_atomic_fetch_inc_nbi(shmem_ctx_t ctx, unsigned long long *fetch, unsigned long long *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_int32_atomic_fetch_inc_nbi(shmem_ctx_t ctx, int32_t *fetch, int32_t *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_int64_atomic_fetch_inc_nbi(shmem_ctx_t ctx, int64_t *fetch, int64_t *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_uint32_atomic_fetch_inc_nbi(shmem_ctx_t ctx, uint32_t *fetch, uint32_t *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_uint64_atomic_fetch_inc_nbi(shmem_ctx_t ctx, uint64_t *fetch, uint64_t *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_size_atomic_fetch_inc_nbi(shmem_ctx_t ctx, size_t *fetch, size_t *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_ptrdiff_atomic_fetch_inc_nbi(shmem_ctx_t ctx, ptrdiff_t *fetch, ptrdiff_t *target, int pe);
+
+OSHMEM_DECLSPEC void pshmem_int_atomic_fetch_inc_nbi(int *fetch, int *target, int pe);
+OSHMEM_DECLSPEC void pshmem_long_atomic_fetch_inc_nbi(long *fetch, long *target, int pe);
+OSHMEM_DECLSPEC void pshmem_longlong_atomic_fetch_inc_nbi(long long *fetch, long long *target, int pe);
+OSHMEM_DECLSPEC void pshmem_uint_atomic_fetch_inc_nbi(unsigned int *fetch, unsigned int *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ulong_atomic_fetch_inc_nbi(unsigned long *fetch, unsigned long *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ulonglong_atomic_fetch_inc_nbi(unsigned long long *fetch, unsigned long long *target, int pe);
+OSHMEM_DECLSPEC void pshmem_int32_atomic_fetch_inc_nbi(int32_t *fetch, int32_t *target, int pe);
+OSHMEM_DECLSPEC void pshmem_int64_atomic_fetch_inc_nbi(int64_t *fetch, int64_t *target, int pe);
+OSHMEM_DECLSPEC void pshmem_uint32_atomic_fetch_inc_nbi(uint32_t *fetch, uint32_t *target, int pe);
+OSHMEM_DECLSPEC void pshmem_uint64_atomic_fetch_inc_nbi(uint64_t *fetch, uint64_t *target, int pe);
+OSHMEM_DECLSPEC void pshmem_size_atomic_fetch_inc_nbi(size_t *fetch, size_t *target, int pe);
+OSHMEM_DECLSPEC void pshmem_ptrdiff_atomic_fetch_inc_nbi(ptrdiff_t *fetch, ptrdiff_t *target, int pe);
+
+#if OSHMEM_HAVE_C11
+#define pshmem_atomic_fetch_inc_nbi(...)                                        \
+    _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                                \
+            shmem_ctx_t:  _Generic((__OSHMEM_VAR_ARG2(__VA_ARGS__)),            \
+                int*:         pshmem_ctx_int_atomic_fetch_inc_nbi,              \
+                long*:        pshmem_ctx_long_atomic_fetch_inc_nbi,             \
+                long long*:   pshmem_ctx_longlong_atomic_fetch_inc_nbi,         \
+                unsigned int*:       pshmem_ctx_uint_atomic_fetch_inc_nbi,      \
+                unsigned long*:      pshmem_ctx_ulong_atomic_fetch_inc_nbi,     \
+                unsigned long long*: pshmem_ctx_ulonglong_atomic_fetch_inc_nbi, \
+                default:      __opshmem_datatype_ignore),                       \
+            int*:        pshmem_int_atomic_fetch_inc_nbi,                       \
+            long*:       pshmem_long_atomic_fetch_inc_nbi,                      \
+            long long*:  pshmem_longlong_atomic_fetch_inc_nbi,                  \
+            unsigned int*:       pshmem_uint_atomic_fetch_inc_nbi,              \
+            unsigned long*:      pshmem_ulong_atomic_fetch_inc_nbi,             \
+            unsigned long long*: pshmem_ulonglong_atomic_fetch_inc_nbi)(__VA_ARGS__)
+#endif
+
+
+/* Atomic Nonblocking Fetch and Add */
+OSHMEM_DECLSPEC void pshmem_ctx_int_atomic_fetch_add_nbi(shmem_ctx_t ctx, int *fetch, int *target, int value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_long_atomic_fetch_add_nbi(shmem_ctx_t ctx, long *fetch, long *target, long value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_longlong_atomic_fetch_add_nbi(shmem_ctx_t ctx, long long *fetch, long long *target, long long value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_uint_atomic_fetch_add_nbi(shmem_ctx_t ctx, unsigned int *fetch, unsigned int *target, unsigned int value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_ulong_atomic_fetch_add_nbi(shmem_ctx_t ctx, unsigned long *fetch,  unsigned long *target, unsigned long value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_ulonglong_atomic_fetch_add_nbi(shmem_ctx_t ctx, unsigned long long *fetch, unsigned long long *target, unsigned long long value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_int32_atomic_fetch_add_nbi(shmem_ctx_t ctx, int32_t *fetch, int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_int64_atomic_fetch_add_nbi(shmem_ctx_t ctx, int64_t *fetch, int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_uint32_atomic_fetch_add_nbi(shmem_ctx_t ctx, uint32_t *fetch, uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_uint64_atomic_fetch_add_nbi(shmem_ctx_t ctx, uint64_t *fetch, uint64_t *target, uint64_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_size_atomic_fetch_add_nbi(shmem_ctx_t ctx, size_t *fetch, size_t *target, size_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_ptrdiff_atomic_fetch_add_nbi(shmem_ctx_t ctx, ptrdiff_t *fetch, ptrdiff_t *target, ptrdiff_t value, int pe);
+
+OSHMEM_DECLSPEC void pshmem_int_atomic_fetch_add_nbi(int *fetch, int *target, int value, int pe);
+OSHMEM_DECLSPEC void pshmem_long_atomic_fetch_add_nbi(long *fetch, long *target, long value, int pe);
+OSHMEM_DECLSPEC void pshmem_longlong_atomic_fetch_add_nbi(long long *fetch, long long *target, long long value, int pe);
+OSHMEM_DECLSPEC void pshmem_uint_atomic_fetch_add_nbi(unsigned int *fetch, unsigned int *target, unsigned int value, int pe);
+OSHMEM_DECLSPEC void pshmem_ulong_atomic_fetch_add_nbi(unsigned long *fetch, unsigned long *target, unsigned long value, int pe);
+OSHMEM_DECLSPEC void pshmem_ulonglong_atomic_fetch_add_nbi(unsigned long long *fetch, unsigned long long *target, unsigned long long value, int pe);
+OSHMEM_DECLSPEC void pshmem_int32_atomic_fetch_add_nbi(int32_t *fetch, int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_int64_atomic_fetch_add_nbi(int64_t *fetch, int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_uint32_atomic_fetch_add_nbi(uint32_t *fetch, uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_uint64_atomic_fetch_add_nbi(uint64_t *fetch, uint64_t *target, uint64_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_size_atomic_fetch_add_nbi(size_t *fetch, size_t *target, size_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ptrdiff_atomic_fetch_add_nbi(ptrdiff_t *fetch, ptrdiff_t *target, ptrdiff_t value, int pe);
+
+#if OSHMEM_HAVE_C11
+#define pshmem_atomic_fetch_add_nbi(...)                                        \
+    _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                                \
+            shmem_ctx_t:  _Generic((__OSHMEM_VAR_ARG2(__VA_ARGS__)),            \
+                int*:         pshmem_ctx_int_atomic_fetch_add_nbi,              \
+                long*:        pshmem_ctx_long_atomic_fetch_add_nbi,             \
+                long long*:   pshmem_ctx_longlong_atomic_fetch_add_nbi,         \
+                unsigned int*:       pshmem_ctx_uint_atomic_fetch_add_nbi,      \
+                unsigned long*:      pshmem_ctx_ulong_atomic_fetch_add_nbi,     \
+                unsigned long long*: pshmem_ctx_ulonglong_atomic_fetch_add_nbi, \
+                default:      __opshmem_datatype_ignore),                       \
+            int*:        pshmem_int_atomic_fetch_add_nbi,                       \
+            long*:       pshmem_long_atomic_fetch_add_nbi,                      \
+            long long*:  pshmem_longlong_atomic_fetch_add_nbi,                  \
+            unsigned int*:       pshmem_uint_atomic_fetch_add_nbi,              \
+            unsigned long*:      pshmem_ulong_atomic_fetch_add_nbi,             \
+            unsigned long long*: pshmem_ulonglong_atomic_fetch_add_nbi)(__VA_ARGS__)
+#endif
+
+
+/* Atomic Nonblocking Fetch and And */
+OSHMEM_DECLSPEC void pshmem_ctx_uint_atomic_fetch_and_nbi(shmem_ctx_t ctx, unsigned int *fetch, unsigned int *target, unsigned int value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_ulong_atomic_fetch_and_nbi(shmem_ctx_t ctx, unsigned long *fetch,  unsigned long *target, unsigned long value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_ulonglong_atomic_fetch_and_nbi(shmem_ctx_t ctx, unsigned long long *fetch, unsigned long long *target, unsigned long long value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_int32_atomic_fetch_and_nbi(shmem_ctx_t ctx, int32_t *fetch, int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_int64_atomic_fetch_and_nbi(shmem_ctx_t ctx, int64_t *fetch, int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_uint32_atomic_fetch_and_nbi(shmem_ctx_t ctx, uint32_t *fetch, uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_uint64_atomic_fetch_and_nbi(shmem_ctx_t ctx, uint64_t *fetch, uint64_t *target, uint64_t value, int pe);
+
+OSHMEM_DECLSPEC void pshmem_uint_atomic_fetch_and_nbi(unsigned int *fetch, unsigned int *target, unsigned int value, int pe);
+OSHMEM_DECLSPEC void pshmem_ulong_atomic_fetch_and_nbi(unsigned long *fetch, unsigned long *target, unsigned long value, int pe);
+OSHMEM_DECLSPEC void pshmem_ulonglong_atomic_fetch_and_nbi(unsigned long long *fetch, unsigned long long *target, unsigned long long value, int pe);
+OSHMEM_DECLSPEC void pshmem_int32_atomic_fetch_and_nbi(int32_t *fetch, int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_int64_atomic_fetch_and_nbi(int64_t *fetch, int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_uint32_atomic_fetch_and_nbi(uint32_t *fetch, uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_uint64_atomic_fetch_and_nbi(uint64_t *fetch, uint64_t *target, uint64_t value, int pe);
+
+#if OSHMEM_HAVE_C11
+#define pshmem_atomic_fetch_and_nbi(...)                                        \
+    _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                                \
+            shmem_ctx_t:  _Generic((__OSHMEM_VAR_ARG2(__VA_ARGS__)),            \
+                unsigned int*:       pshmem_ctx_uint_atomic_fetch_and_nbi,      \
+                unsigned long*:      pshmem_ctx_ulong_atomic_fetch_and_nbi,     \
+                unsigned long long*: pshmem_ctx_ulonglong_atomic_fetch_and_nbi, \
+                int32_t*:      pshmem_ctx_int32_atomic_fetch_and_nbi,           \
+                int64_t*:      pshmem_ctx_int64_atomic_fetch_and_nbi,           \
+                default:      __opshmem_datatype_ignore),                       \
+            unsigned int*:       pshmem_uint_atomic_fetch_and_nbi,              \
+            unsigned long*:      pshmem_ulong_atomic_fetch_and_nbi,             \
+            int32_t*:      pshmem_int32_atomic_fetch_and_nbi,                   \
+            int64_t*:      pshmem_int64_atomic_fetch_and_nbi,                   \
+            unsigned long long*: pshmem_ulonglong_atomic_fetch_and_nbi)(__VA_ARGS__)
+#endif
+
+/* Atomic Nonblocking Fetch and OR */
+OSHMEM_DECLSPEC void pshmem_ctx_uint_atomic_fetch_or_nbi(shmem_ctx_t ctx, unsigned int *fetch, unsigned int *target, unsigned int value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_ulong_atomic_fetch_or_nbi(shmem_ctx_t ctx, unsigned long *fetch,  unsigned long *target, unsigned long value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_ulonglong_atomic_fetch_or_nbi(shmem_ctx_t ctx, unsigned long long *fetch, unsigned long long *target, unsigned long long value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_int32_atomic_fetch_or_nbi(shmem_ctx_t ctx, int32_t *fetch, int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_int64_atomic_fetch_or_nbi(shmem_ctx_t ctx, int64_t *fetch, int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_uint32_atomic_fetch_or_nbi(shmem_ctx_t ctx, uint32_t *fetch, uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_uint64_atomic_fetch_or_nbi(shmem_ctx_t ctx, uint64_t *fetch, uint64_t *target, uint64_t value, int pe);
+
+OSHMEM_DECLSPEC void pshmem_uint_atomic_fetch_or_nbi(unsigned int *fetch, unsigned int *target, unsigned int value, int pe);
+OSHMEM_DECLSPEC void pshmem_ulong_atomic_fetch_or_nbi(unsigned long *fetch, unsigned long *target, unsigned long value, int pe);
+OSHMEM_DECLSPEC void pshmem_ulonglong_atomic_fetch_or_nbi(unsigned long long *fetch, unsigned long long *target, unsigned long long value, int pe);
+OSHMEM_DECLSPEC void pshmem_int32_atomic_fetch_or_nbi(int32_t *fetch, int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_int64_atomic_fetch_or_nbi(int64_t *fetch, int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_uint32_atomic_fetch_or_nbi(uint32_t *fetch, uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_uint64_atomic_fetch_or_nbi(uint64_t *fetch, uint64_t *target, uint64_t value, int pe);
+
+#if OSHMEM_HAVE_C11
+#define pshmem_atomic_fetch_or_nbi(...)                                        \
+    _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                               \
+            shmem_ctx_t:  _Generic((__OSHMEM_VAR_ARG2(__VA_ARGS__)),           \
+                unsigned int*:       pshmem_ctx_uint_atomic_fetch_or_nbi,      \
+                unsigned long*:      pshmem_ctx_ulong_atomic_fetch_or_nbi,     \
+                unsigned long long*: pshmem_ctx_ulonglong_atomic_fetch_or_nbi, \
+                int32_t*:      pshmem_ctx_int32_atomic_fetch_or_nbi,           \
+                int64_t*:      pshmem_ctx_int64_atomic_fetch_or_nbi,           \
+                default:      __opshmem_datatype_ignore),                      \
+            unsigned int*:       pshmem_uint_atomic_fetch_or_nbi,              \
+            unsigned long*:      pshmem_ulong_atomic_fetch_or_nbi,             \
+            int32_t*:      pshmem_int32_atomic_fetch_or_nbi,                   \
+            int64_t*:      pshmem_int64_atomic_fetch_or_nbi,                   \
+            unsigned long long*: pshmem_ulonglong_atomic_fetch_or_nbi)(__VA_ARGS__)
+#endif
+
+
+/* Atomic Nonblocking Fetch and XOR */
+OSHMEM_DECLSPEC void pshmem_ctx_uint_atomic_fetch_xor_nbi(shmem_ctx_t ctx, unsigned int *fetch, unsigned int *target, unsigned int value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_ulong_atomic_fetch_xor_nbi(shmem_ctx_t ctx, unsigned long *fetch,  unsigned long *target, unsigned long value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_ulonglong_atomic_fetch_xor_nbi(shmem_ctx_t ctx, unsigned long long *fetch, unsigned long long *target, unsigned long long value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_int32_atomic_fetch_xor_nbi(shmem_ctx_t ctx, int32_t *fetch, int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_int64_atomic_fetch_xor_nbi(shmem_ctx_t ctx, int64_t *fetch, int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_uint32_atomic_fetch_xor_nbi(shmem_ctx_t ctx, uint32_t *fetch, uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_ctx_uint64_atomic_fetch_xor_nbi(shmem_ctx_t ctx, uint64_t *fetch, uint64_t *target, uint64_t value, int pe);
+
+OSHMEM_DECLSPEC void pshmem_uint_atomic_fetch_xor_nbi(unsigned int *fetch, unsigned int *target, unsigned int value, int pe);
+OSHMEM_DECLSPEC void pshmem_ulong_atomic_fetch_xor_nbi(unsigned long *fetch, unsigned long *target, unsigned long value, int pe);
+OSHMEM_DECLSPEC void pshmem_ulonglong_atomic_fetch_xor_nbi(unsigned long long *fetch, unsigned long long *target, unsigned long long value, int pe);
+OSHMEM_DECLSPEC void pshmem_int32_atomic_fetch_xor_nbi(int32_t *fetch, int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_int64_atomic_fetch_xor_nbi(int64_t *fetch, int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_uint32_atomic_fetch_xor_nbi(uint32_t *fetch, uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC void pshmem_uint64_atomic_fetch_xor_nbi(uint64_t *fetch, uint64_t *target, uint64_t value, int pe);
+
+#if OSHMEM_HAVE_C11
+#define pshmem_atomic_fetch_xor_nbi(...)                                        \
+    _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                                \
+            shmem_ctx_t:  _Generic((__OSHMEM_VAR_ARG2(__VA_ARGS__)),            \
+                unsigned int*:       pshmem_ctx_uint_atomic_fetch_xor_nbi,      \
+                unsigned long*:      pshmem_ctx_ulong_atomic_fetch_xor_nbi,     \
+                unsigned long long*: pshmem_ctx_ulonglong_atomic_fetch_xor_nbi, \
+                int32_t*:      pshmem_ctx_int32_atomic_fetch_xor_nbi,           \
+                int64_t*:      pshmem_ctx_int64_atomic_fetch_xor_nbi,           \
+                default:      __opshmem_datatype_ignore),                       \
+            unsigned int*:       pshmem_uint_atomic_fetch_xor_nbi,              \
+            unsigned long*:      pshmem_ulong_atomic_fetch_xor_nbi,             \
+            int32_t*:      pshmem_int32_atomic_fetch_xor_nbi,                   \
+            int64_t*:      pshmem_int64_atomic_fetch_xor_nbi,                   \
+            unsigned long long*: pshmem_ulonglong_atomic_fetch_xor_nbi)(__VA_ARGS__)
+#endif
+
+/*
+ *
+ * Control of profiling
+ *
+ */
+OSHMEM_DECLSPEC void pshmem_pcontrol(int level, ...);
+
+
 /*
  * Lock functions
  */
@@ -1545,10 +2785,174 @@ OSHMEM_DECLSPEC  void pshmem_ptrdiff_wait_until(volatile ptrdiff_t *addr, int cm
         int*:         pshmem_int_wait_until,                 \
         long*:        pshmem_long_wait_until,                \
         long long*:   pshmem_longlong_wait_until,            \
-        unsigned short*:       pshmem_short_wait_until,      \
-        unsigned int*:         pshmem_int_wait_until,        \
-        unsigned long*:        pshmem_long_wait_until,       \
-        unsigned long long*:   pshmem_longlong_wait_until)(addr, cmp, value)
+        unsigned short*:       pshmem_ushort_wait_until,      \
+        unsigned int*:         pshmem_uint_wait_until,        \
+        unsigned long*:        pshmem_ulong_wait_until,       \
+        unsigned long long*:   pshmem_ulonglong_wait_until)(addr, cmp, value)
+#endif
+
+OSHMEM_DECLSPEC  void pshmem_short_wait_until_all(volatile short *ivars, size_t nelems, const int *status, int cmp, short value);
+OSHMEM_DECLSPEC  void pshmem_ushort_wait_until_all(volatile unsigned short *ivars, size_t nelems, const int *status, int cmp, unsigned short value);
+OSHMEM_DECLSPEC  void pshmem_int_wait_until_all(volatile int *ivars, size_t nelems, const int *status, int cmp, int value);
+OSHMEM_DECLSPEC  void pshmem_long_wait_until_all(volatile long *ivars, size_t nelems, const int *status, int cmp, long value);
+OSHMEM_DECLSPEC  void pshmem_longlong_wait_until_all(volatile long long *ivars, size_t nelems, const int *status, int cmp, long long value);
+OSHMEM_DECLSPEC  void pshmem_uint_wait_until_all(volatile unsigned int *ivars, size_t nelems, const int *status, int cmp, unsigned int value);
+OSHMEM_DECLSPEC  void pshmem_ulong_wait_until_all(volatile unsigned long *ivars, size_t nelems, const int *status, int cmp, unsigned long value);
+OSHMEM_DECLSPEC  void pshmem_ulonglong_wait_until_all(volatile unsigned long long *ivars, size_t nelems, const int *status, int cmp, unsigned long long value);
+OSHMEM_DECLSPEC  void pshmem_int32_wait_until_all(volatile int32_t *ivars, size_t nelems, const int *status, int cmp, int32_t value);
+OSHMEM_DECLSPEC  void pshmem_int64_wait_until_all(volatile int64_t *ivars, size_t nelems, const int *status, int cmp, int64_t value);
+OSHMEM_DECLSPEC  void pshmem_uint32_wait_until_all(volatile uint32_t *ivars, size_t nelems, const int *status, int cmp, uint32_t value);
+OSHMEM_DECLSPEC  void pshmem_uint64_wait_until_all(volatile uint64_t *ivars, size_t nelems, const int *status, int cmp, uint64_t value);
+OSHMEM_DECLSPEC  void pshmem_size_wait_until_all(volatile size_t *ivars, size_t nelems, const int *status, int cmp, size_t value);
+OSHMEM_DECLSPEC  void pshmem_ptrdiff_wait_until_all(volatile ptrdiff_t *ivars, size_t nelems, const int *status, int cmp, ptrdiff_t value);
+#if OSHMEM_HAVE_C11
+#define pshmem_wait_until_all(ivars, nelems, status, cmp, value)          \
+    _Generic(&*(ivars),                                                   \
+        short*:                pshmem_short_wait_until_all,               \
+        unsigned short*:       pshmem_ushort_wait_until_all,              \
+        int*:                  pshmem_int_wait_until_all,                 \
+        long*:                 pshmem_long_wait_until_all,                \
+        long long*:            pshmem_longlong_wait_until_all,            \
+        unsigned int*:         pshmem_uint_wait_until_all,                \
+        unsigned long*:        pshmem_ulong_wait_until_all,               \
+        unsigned long long*:   pshmem_ulonglong_wait_until_all)(ivars, nelems, status, cmp, value)
+#endif
+
+OSHMEM_DECLSPEC  size_t pshmem_short_wait_until_any(volatile short *ivars, size_t nelems, const int *status, int cmp, short value);
+OSHMEM_DECLSPEC  size_t pshmem_ushort_wait_until_any(volatile unsigned short *ivars, size_t nelems, const int *status, int cmp, unsigned short value);
+OSHMEM_DECLSPEC  size_t pshmem_int_wait_until_any(volatile int *ivars, size_t nelems, const int *status, int cmp, int value);
+OSHMEM_DECLSPEC  size_t pshmem_long_wait_until_any(volatile long *ivars, size_t nelems, const int *status, int cmp, long value);
+OSHMEM_DECLSPEC  size_t pshmem_longlong_wait_until_any(volatile long long *ivars, size_t nelems, const int *status, int cmp, long long value);
+OSHMEM_DECLSPEC  size_t pshmem_uint_wait_until_any(volatile unsigned int *ivars, size_t nelems, const int *status, int cmp, unsigned int value);
+OSHMEM_DECLSPEC  size_t pshmem_ulong_wait_until_any(volatile unsigned long *ivars, size_t nelems, const int *status, int cmp, unsigned long value);
+OSHMEM_DECLSPEC  size_t pshmem_ulonglong_wait_until_any(volatile unsigned long long *ivars, size_t nelems, const int *status, int cmp, unsigned long long value);
+OSHMEM_DECLSPEC  size_t pshmem_int32_wait_until_any(volatile int32_t *ivars, size_t nelems, const int *status, int cmp, int32_t value);
+OSHMEM_DECLSPEC  size_t pshmem_int64_wait_until_any(volatile int64_t *ivars, size_t nelems, const int *status, int cmp, int64_t value);
+OSHMEM_DECLSPEC  size_t pshmem_uint32_wait_until_any(volatile uint32_t *ivars, size_t nelems, const int *status, int cmp, uint32_t value);
+OSHMEM_DECLSPEC  size_t pshmem_uint64_wait_until_any(volatile uint64_t *ivars, size_t nelems, const int *status, int cmp, uint64_t value);
+OSHMEM_DECLSPEC  size_t pshmem_size_wait_until_any(volatile size_t *ivars, size_t nelems, const int *status, int cmp, size_t value);
+OSHMEM_DECLSPEC  size_t pshmem_ptrdiff_wait_until_any(volatile ptrdiff_t *ivars, size_t nelems, const int *status, int cmp, ptrdiff_t value);
+#if OSHMEM_HAVE_C11
+#define pshmem_wait_until_any(ivars, nelems, status, cmp, value)          \
+    _Generic(&*(ivars),                                                   \
+        short*:                pshmem_short_wait_until_any,               \
+        unsigned short*:       pshmem_ushort_wait_until_any,              \
+        int*:                  pshmem_int_wait_until_any,                 \
+        long*:                 pshmem_long_wait_until_any,                \
+        long long*:            pshmem_longlong_wait_until_any,            \
+        unsigned int*:         pshmem_uint_wait_until_any,                \
+        unsigned long*:        pshmem_ulong_wait_until_any,               \
+        unsigned long long*:   pshmem_ulonglong_wait_until_any)(ivars, nelems, status, cmp, value)
+#endif
+
+OSHMEM_DECLSPEC  size_t pshmem_short_wait_until_some(volatile short *ivars, size_t nelems,  size_t *indices, const int *status, int cmp, short value);
+OSHMEM_DECLSPEC  size_t pshmem_ushort_wait_until_some(volatile unsigned short *ivars, size_t nelems,  size_t *indices, const int *status, int cmp, unsigned short value);
+OSHMEM_DECLSPEC  size_t pshmem_int_wait_until_some(volatile int *ivars, size_t nelems, size_t *indices, const int *status, int cmp, int value);
+OSHMEM_DECLSPEC  size_t pshmem_long_wait_until_some(volatile long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, long value);
+OSHMEM_DECLSPEC  size_t pshmem_longlong_wait_until_some(volatile long long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, long long value);
+OSHMEM_DECLSPEC  size_t pshmem_uint_wait_until_some(volatile unsigned int *ivars, size_t nelems, size_t *indices, const int *status, int cmp, unsigned int value);
+OSHMEM_DECLSPEC  size_t pshmem_ulong_wait_until_some(volatile unsigned long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, unsigned long value);
+OSHMEM_DECLSPEC  size_t pshmem_ulonglong_wait_until_some(volatile unsigned long long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, unsigned long long value);
+OSHMEM_DECLSPEC  size_t pshmem_int32_wait_until_some(volatile int32_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, int32_t value);
+OSHMEM_DECLSPEC  size_t pshmem_int64_wait_until_some(volatile int64_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, int64_t value);
+OSHMEM_DECLSPEC  size_t pshmem_uint32_wait_until_some(volatile uint32_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, uint32_t value);
+OSHMEM_DECLSPEC  size_t pshmem_uint64_wait_until_some(volatile uint64_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, uint64_t value);
+OSHMEM_DECLSPEC  size_t pshmem_size_wait_until_some(volatile size_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, size_t value);
+OSHMEM_DECLSPEC  size_t pshmem_ptrdiff_wait_until_some(volatile ptrdiff_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, ptrdiff_t value);
+#if OSHMEM_HAVE_C11
+#define pshmem_wait_until_some(ivars, nelems, indices, status, cmp, value)          \
+    _Generic(&*(ivars),                                                    \
+        short*:                pshmem_short_wait_until_some,               \
+        unsigned short*:       pshmem_ushort_wait_until_some,              \
+        int*:                  pshmem_int_wait_until_some,                 \
+        long*:                 pshmem_long_wait_until_some,                \
+        long long*:            pshmem_longlong_wait_until_some,            \
+        unsigned int*:         pshmem_uint_wait_until_some,                \
+        unsigned long*:        pshmem_ulong_wait_until_some,               \
+        unsigned long long*:   pshmem_ulonglong_wait_until_some)(ivars, nelems, indices, status, cmp, value)
+#endif
+
+OSHMEM_DECLSPEC  void pshmem_short_wait_until_all_vector(volatile short *ivars, size_t nelems, const int *status, int cmp, short *values);
+OSHMEM_DECLSPEC  void pshmem_ushort_wait_until_all_vector(volatile unsigned short *ivars, size_t nelems, const int *status, int cmp, unsigned short *values);
+OSHMEM_DECLSPEC  void pshmem_int_wait_until_all_vector(volatile int *ivars, size_t nelems, const int *status, int cmp, int *values);
+OSHMEM_DECLSPEC  void pshmem_long_wait_until_all_vector(volatile long *ivars, size_t nelems, const int *status, int cmp, long *values);
+OSHMEM_DECLSPEC  void pshmem_longlong_wait_until_all_vector(volatile long long *ivars, size_t nelems, const int *status, int cmp, long long *values);
+OSHMEM_DECLSPEC  void pshmem_uint_wait_until_all_vector(volatile unsigned int *ivars, size_t nelems, const int *status, int cmp, unsigned int *values);
+OSHMEM_DECLSPEC  void pshmem_ulong_wait_until_all_vector(volatile unsigned long *ivars, size_t nelems, const int *status, int cmp, unsigned long *values);
+OSHMEM_DECLSPEC  void pshmem_ulonglong_wait_until_all_vector(volatile unsigned long long *ivars, size_t nelems, const int *status, int cmp, unsigned long long *values);
+OSHMEM_DECLSPEC  void pshmem_int32_wait_until_all_vector(volatile int32_t *ivars, size_t nelems, const int *status, int cmp, int32_t *values);
+OSHMEM_DECLSPEC  void pshmem_int64_wait_until_all_vector(volatile int64_t *ivars, size_t nelems, const int *status, int cmp, int64_t *values);
+OSHMEM_DECLSPEC  void pshmem_uint32_wait_until_all_vector(volatile uint32_t *ivars, size_t nelems, const int *status, int cmp, uint32_t *values);
+OSHMEM_DECLSPEC  void pshmem_uint64_wait_until_all_vector(volatile uint64_t *ivars, size_t nelems, const int *status, int cmp, uint64_t *values);
+OSHMEM_DECLSPEC  void pshmem_size_wait_until_all_vector(volatile size_t *ivars, size_t nelems, const int *status, int cmp, size_t *values);
+OSHMEM_DECLSPEC  void pshmem_ptrdiff_wait_until_all_vector(volatile ptrdiff_t *ivars, size_t nelems, const int *status, int cmp, ptrdiff_t *values);
+#if OSHMEM_HAVE_C11
+#define pshmem_wait_until_all_vector(ivars, nelems, status, cmp, values)         \
+    _Generic(&*(ivars),                                                          \
+        short*:                pshmem_short_wait_until_all_vector,               \
+        unsigned short*:       pshmem_ushort_wait_until_all_vector,              \
+        int*:                  pshmem_int_wait_until_all_vector,                 \
+        long*:                 pshmem_long_wait_until_all_vector,                \
+        long long*:            pshmem_longlong_wait_until_all_vector,            \
+        unsigned int*:         pshmem_uint_wait_until_all_vector,                \
+        unsigned long*:        pshmem_ulong_wait_until_all_vector,               \
+        unsigned long long*:   pshmem_ulonglong_wait_until_all_vector)(ivars, nelems, status, cmp, values)
+#endif
+
+
+OSHMEM_DECLSPEC  size_t pshmem_short_wait_until_any_vector(volatile short *ivars, size_t nelems, const int *status, int cmp, short *values);
+OSHMEM_DECLSPEC  size_t pshmem_ushort_wait_until_any_vector(volatile unsigned short *ivars, size_t nelems, const int *status, int cmp, unsigned short *values);
+OSHMEM_DECLSPEC  size_t pshmem_int_wait_until_any_vector(volatile int *ivars, size_t nelems, const int *status, int cmp, int *values);
+OSHMEM_DECLSPEC  size_t pshmem_long_wait_until_any_vector(volatile long *ivars, size_t nelems, const int *status, int cmp, long *values);
+OSHMEM_DECLSPEC  size_t pshmem_longlong_wait_until_any_vector(volatile long long *ivars, size_t nelems, const int *status, int cmp, long long *values);
+OSHMEM_DECLSPEC  size_t pshmem_uint_wait_until_any_vector(volatile unsigned int *ivars, size_t nelems, const int *status, int cmp, unsigned int *values);
+OSHMEM_DECLSPEC  size_t pshmem_ulong_wait_until_any_vector(volatile unsigned long *ivars, size_t nelems, const int *status, int cmp, unsigned long *values);
+OSHMEM_DECLSPEC  size_t pshmem_ulonglong_wait_until_any_vector(volatile unsigned long long *ivars, size_t nelems, const int *status, int cmp, unsigned long long *values);
+OSHMEM_DECLSPEC  size_t pshmem_int32_wait_until_any_vector(volatile int32_t *ivars, size_t nelems, const int *status, int cmp, int32_t *values);
+OSHMEM_DECLSPEC  size_t pshmem_int64_wait_until_any_vector(volatile int64_t *ivars, size_t nelems, const int *status, int cmp, int64_t *values);
+OSHMEM_DECLSPEC  size_t pshmem_uint32_wait_until_any_vector(volatile uint32_t *ivars, size_t nelems, const int *status, int cmp, uint32_t *values);
+OSHMEM_DECLSPEC  size_t pshmem_uint64_wait_until_any_vector(volatile uint64_t *ivars, size_t nelems, const int *status, int cmp, uint64_t *values);
+OSHMEM_DECLSPEC  size_t pshmem_size_wait_until_any_vector(volatile size_t *ivars, size_t nelems, const int *status, int cmp, size_t *values);
+OSHMEM_DECLSPEC  size_t pshmem_ptrdiff_wait_until_any_vector(volatile ptrdiff_t *ivars, size_t nelems, const int *status, int cmp, ptrdiff_t *values);
+#if OSHMEM_HAVE_C11
+#define pshmem_wait_until_any_vector(ivars, nelems, status, cmp, values)         \
+    _Generic(&*(ivars),                                                          \
+        short*:                pshmem_short_wait_until_any_vector,               \
+        unsigned short*:       pshmem_ushort_wait_until_any_vector,              \
+        int*:                  pshmem_int_wait_until_any_vector,                 \
+        long*:                 pshmem_long_wait_until_any_vector,                \
+        long long*:            pshmem_longlong_wait_until_any_vector,            \
+        unsigned int*:         pshmem_uint_wait_until_any_vector,                \
+        unsigned long*:        pshmem_ulong_wait_until_any_vector,               \
+        unsigned long long*:   pshmem_ulonglong_wait_until_any_vector)(ivars, nelems, status, cmp, values)
+#endif
+
+
+OSHMEM_DECLSPEC  size_t pshmem_short_wait_until_some_vector(volatile short *ivars, size_t nelems, size_t *indices, const int *status, int cmp, short *values);
+OSHMEM_DECLSPEC  size_t pshmem_ushort_wait_until_some_vector(volatile unsigned short *ivars, size_t nelems, size_t *indices, const int *status, int cmp, unsigned short *values);
+OSHMEM_DECLSPEC  size_t pshmem_int_wait_until_some_vector(volatile int *ivars, size_t nelems, size_t *indices, const int *status, int cmp, int *values);
+OSHMEM_DECLSPEC  size_t pshmem_long_wait_until_some_vector(volatile long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, long *values);
+OSHMEM_DECLSPEC  size_t pshmem_longlong_wait_until_some_vector(volatile long long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, long long *values);
+OSHMEM_DECLSPEC  size_t pshmem_uint_wait_until_some_vector(volatile unsigned int *ivars, size_t nelems, size_t *indices, const int *status, int cmp, unsigned int *values);
+OSHMEM_DECLSPEC  size_t pshmem_ulong_wait_until_some_vector(volatile unsigned long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, unsigned long *values);
+OSHMEM_DECLSPEC  size_t pshmem_ulonglong_wait_until_some_vector(volatile unsigned long long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, unsigned long long *values);
+OSHMEM_DECLSPEC  size_t pshmem_int32_wait_until_some_vector(volatile int32_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, int32_t *values);
+OSHMEM_DECLSPEC  size_t pshmem_int64_wait_until_some_vector(volatile int64_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, int64_t *values);
+OSHMEM_DECLSPEC  size_t pshmem_uint32_wait_until_some_vector(volatile uint32_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, uint32_t *values);
+OSHMEM_DECLSPEC  size_t pshmem_uint64_wait_until_some_vector(volatile uint64_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, uint64_t *values);
+OSHMEM_DECLSPEC  size_t pshmem_size_wait_until_some_vector(volatile size_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, size_t *values);
+OSHMEM_DECLSPEC  size_t pshmem_ptrdiff_wait_until_some_vector(volatile ptrdiff_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, ptrdiff_t *values);
+#if OSHMEM_HAVE_C11
+#define pshmem_wait_until_some_vector(ivars, nelems, indices, status, cmp, values)\
+    _Generic(&*(ivars),                                                           \
+        short*:                pshmem_short_wait_until_some_vector,               \
+        unsigned short*:       pshmem_ushort_wait_until_some_vector,              \
+        int*:                  pshmem_int_wait_until_some_vector,                 \
+        long*:                 pshmem_long_wait_until_some_vector,                \
+        long long*:            pshmem_longlong_wait_until_some_vector,            \
+        unsigned int*:         pshmem_uint_wait_until_some_vector,                \
+        unsigned long*:        pshmem_ulong_wait_until_some_vector,               \
+        unsigned long long*:   pshmem_ulonglong_wait_until_some_vector)(ivars, nelems, indices, status, cmp, values)
 #endif
 
 OSHMEM_DECLSPEC  int pshmem_short_test(volatile short *addr, int cmp, short value);
@@ -1578,18 +2982,189 @@ OSHMEM_DECLSPEC  int pshmem_ptrdiff_test(volatile ptrdiff_t *addr, int cmp, ptrd
         unsigned long long*:   pshmem_longlong_test)(addr, cmp, value)
 #endif
 
+OSHMEM_DECLSPEC  int pshmem_short_test_all(volatile short *ivars, size_t nelems, const int *status, int cmp, short value);
+OSHMEM_DECLSPEC  int pshmem_ushort_test_all(volatile unsigned short *ivars, size_t nelems, const int *status, int cmp, unsigned short value);
+OSHMEM_DECLSPEC  int pshmem_int_test_all(volatile int *ivars, size_t nelems, const int *status, int cmp, int value);
+OSHMEM_DECLSPEC  int pshmem_long_test_all(volatile long *ivars, size_t nelems, const int *status, int cmp, long value);
+OSHMEM_DECLSPEC  int pshmem_longlong_test_all(volatile long long *ivars, size_t nelems, const int *status, int cmp, long long value);
+OSHMEM_DECLSPEC  int pshmem_uint_test_all(volatile unsigned int *ivars, size_t nelems, const int *status, int cmp, unsigned int value);
+OSHMEM_DECLSPEC  int pshmem_ulong_test_all(volatile unsigned long *ivars, size_t nelems, const int *status, int cmp, unsigned long value);
+OSHMEM_DECLSPEC  int pshmem_ulonglong_test_all(volatile unsigned long long *ivars, size_t nelems, const int *status, int cmp, unsigned long long value);
+OSHMEM_DECLSPEC  int pshmem_int32_test_all(volatile int32_t *ivars, size_t nelems, const int *status, int cmp, int32_t value);
+OSHMEM_DECLSPEC  int pshmem_int64_test_all(volatile int64_t *ivars, size_t nelems, const int *status, int cmp, int64_t value);
+OSHMEM_DECLSPEC  int pshmem_uint32_test_all(volatile uint32_t *ivars, size_t nelems, const int *status, int cmp, uint32_t value);
+OSHMEM_DECLSPEC  int pshmem_uint64_test_all(volatile uint64_t *ivars, size_t nelems, const int *status, int cmp, uint64_t value);
+OSHMEM_DECLSPEC  int pshmem_size_test_all(volatile size_t *ivars, size_t nelems, const int *status, int cmp, size_t value);
+OSHMEM_DECLSPEC  int pshmem_ptrdiff_test_all(volatile ptrdiff_t *ivars, size_t nelems, const int *status, int cmp, ptrdiff_t value);
+#if OSHMEM_HAVE_C11
+#define pshmem_test_all(ivars, nelems, status, cmp, value)          \
+    _Generic(&*(ivars),                                             \
+        short*:                pshmem_short_test_all,               \
+        unsigned short*:       pshmem_ushort_test_all,              \
+        int*:                  pshmem_int_test_all,                 \
+        long*:                 pshmem_long_test_all,                \
+        long long*:            pshmem_longlong_test_all,            \
+        unsigned int*:         pshmem_uint_test_all,                \
+        unsigned long*:        pshmem_ulong_test_all,               \
+        unsigned long long*:   pshmem_ulonglong_test_all)(ivars, nelems, status, cmp, value)
+#endif
+
+OSHMEM_DECLSPEC  size_t pshmem_short_test_any(volatile short *ivars, size_t nelems, const int *status, int cmp, short value);
+OSHMEM_DECLSPEC  size_t pshmem_ushort_test_any(volatile unsigned short *ivars, size_t nelems, const int *status, int cmp, unsigned short value);
+OSHMEM_DECLSPEC  size_t pshmem_int_test_any(volatile int *ivars, size_t nelems, const int *status, int cmp, int value);
+OSHMEM_DECLSPEC  size_t pshmem_long_test_any(volatile long *ivars, size_t nelems, const int *status, int cmp, long value);
+OSHMEM_DECLSPEC  size_t pshmem_longlong_test_any(volatile long long *ivars, size_t nelems, const int *status, int cmp, long long value);
+OSHMEM_DECLSPEC  size_t pshmem_uint_test_any(volatile unsigned int *ivars, size_t nelems, const int *status, int cmp, unsigned int value);
+OSHMEM_DECLSPEC  size_t pshmem_ulong_test_any(volatile unsigned long *ivars, size_t nelems, const int *status, int cmp, unsigned long value);
+OSHMEM_DECLSPEC  size_t pshmem_ulonglong_test_any(volatile unsigned long long *ivars, size_t nelems, const int *status, int cmp, unsigned long long value);
+OSHMEM_DECLSPEC  size_t pshmem_int32_test_any(volatile int32_t *ivars, size_t nelems, const int *status, int cmp, int32_t value);
+OSHMEM_DECLSPEC  size_t pshmem_int64_test_any(volatile int64_t *ivars, size_t nelems, const int *status, int cmp, int64_t value);
+OSHMEM_DECLSPEC  size_t pshmem_uint32_test_any(volatile uint32_t *ivars, size_t nelems, const int *status, int cmp, uint32_t value);
+OSHMEM_DECLSPEC  size_t pshmem_uint64_test_any(volatile uint64_t *ivars, size_t nelems, const int *status, int cmp, uint64_t value);
+OSHMEM_DECLSPEC  size_t pshmem_size_test_any(volatile size_t *ivars, size_t nelems, const int *status, int cmp, size_t value);
+OSHMEM_DECLSPEC  size_t pshmem_ptrdiff_test_any(volatile ptrdiff_t *ivars, size_t nelems, const int *status, int cmp, ptrdiff_t value);
+#if OSHMEM_HAVE_C11
+#define pshmem_test_any(ivars, nelems, status, cmp, value)          \
+    _Generic(&*(ivars),                                             \
+        short*:                pshmem_short_test_any,               \
+        unsigned short*:       pshmem_ushort_test_any,              \
+        int*:                  pshmem_int_test_any,                 \
+        long*:                 pshmem_long_test_any,                \
+        long long*:            pshmem_longlong_test_any,            \
+        unsigned int*:         pshmem_uint_test_any,                \
+        unsigned long*:        pshmem_ulong_test_any,               \
+        unsigned long long*:   pshmem_ulonglong_test_any)(ivars, nelems, status, cmp, value)
+#endif
+
+OSHMEM_DECLSPEC  size_t pshmem_short_test_some(volatile short *ivars, size_t nelems, size_t *indices, const int *status, int cmp, short value);
+OSHMEM_DECLSPEC  size_t pshmem_ushort_test_some(volatile unsigned short *ivars, size_t nelems, size_t *indices, const int *status, int cmp, unsigned short value);
+OSHMEM_DECLSPEC  size_t pshmem_int_test_some(volatile int *ivars, size_t nelems, size_t *indices, const int *status, int cmp, int value);
+OSHMEM_DECLSPEC  size_t pshmem_long_test_some(volatile long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, long value);
+OSHMEM_DECLSPEC  size_t pshmem_longlong_test_some(volatile long long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, long long value);
+OSHMEM_DECLSPEC  size_t pshmem_uint_test_some(volatile unsigned int *ivars, size_t nelems, size_t *indices, const int *status, int cmp, unsigned int value);
+OSHMEM_DECLSPEC  size_t pshmem_ulong_test_some(volatile unsigned long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, unsigned long value);
+OSHMEM_DECLSPEC  size_t pshmem_ulonglong_test_some(volatile unsigned long long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, unsigned long long value);
+OSHMEM_DECLSPEC  size_t pshmem_int32_test_some(volatile int32_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, int32_t value);
+OSHMEM_DECLSPEC  size_t pshmem_int64_test_some(volatile int64_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, int64_t value);
+OSHMEM_DECLSPEC  size_t pshmem_uint32_test_some(volatile uint32_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, uint32_t value);
+OSHMEM_DECLSPEC  size_t pshmem_uint64_test_some(volatile uint64_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, uint64_t value);
+OSHMEM_DECLSPEC  size_t pshmem_size_test_some(volatile size_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, size_t value);
+OSHMEM_DECLSPEC  size_t pshmem_ptrdiff_test_some(volatile ptrdiff_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, ptrdiff_t value);
+#if OSHMEM_HAVE_C11
+#define pshmem_test_some(ivars, nelems, indices, status, cmp, value) \
+    _Generic(&*(ivars),                                              \
+        short*:                pshmem_short_test_some,               \
+        unsigned short*:       pshmem_ushort_test_some,              \
+        int*:                  pshmem_int_test_some,                 \
+        long*:                 pshmem_long_test_some,                \
+        long long*:            pshmem_longlong_test_some,            \
+        unsigned int*:         pshmem_uint_test_some,                \
+        unsigned long*:        pshmem_ulong_test_some,               \
+        unsigned long long*:   pshmem_ulonglong_test_some)(ivars, nelems, indices, status, cmp, value)
+#endif
+
+OSHMEM_DECLSPEC  int pshmem_short_test_all_vector(volatile short *ivars, size_t nelems, const int *status, int cmp, short *values);
+OSHMEM_DECLSPEC  int pshmem_ushort_test_all_vector(volatile unsigned short *ivars, size_t nelems, const int *status, int cmp, unsigned short *values);
+OSHMEM_DECLSPEC  int pshmem_int_test_all_vector(volatile int *ivars, size_t nelems, const int *status, int cmp, int *values);
+OSHMEM_DECLSPEC  int pshmem_long_test_all_vector(volatile long *ivars, size_t nelems, const int *status, int cmp, long *values);
+OSHMEM_DECLSPEC  int pshmem_longlong_test_all_vector(volatile long long *ivars, size_t nelems, const int *status, int cmp, long long *values);
+OSHMEM_DECLSPEC  int pshmem_uint_test_all_vector(volatile unsigned int *ivars, size_t nelems, const int *status, int cmp, unsigned int *values);
+OSHMEM_DECLSPEC  int pshmem_ulong_test_all_vector(volatile unsigned long *ivars, size_t nelems, const int *status, int cmp, unsigned long *values);
+OSHMEM_DECLSPEC  int pshmem_ulonglong_test_all_vector(volatile unsigned long long *ivars, size_t nelems, const int *status, int cmp, unsigned long long *values);
+OSHMEM_DECLSPEC  int pshmem_int32_test_all_vector(volatile int32_t *ivars, size_t nelems, const int *status, int cmp, int32_t *values);
+OSHMEM_DECLSPEC  int pshmem_int64_test_all_vector(volatile int64_t *ivars, size_t nelems, const int *status, int cmp, int64_t *values);
+OSHMEM_DECLSPEC  int pshmem_uint32_test_all_vector(volatile uint32_t *ivars, size_t nelems, const int *status, int cmp, uint32_t *values);
+OSHMEM_DECLSPEC  int pshmem_uint64_test_all_vector(volatile uint64_t *ivars, size_t nelems, const int *status, int cmp, uint64_t *values);
+OSHMEM_DECLSPEC  int pshmem_size_test_all_vector(volatile size_t *ivars, size_t nelems, const int *status, int cmp, size_t *values);
+OSHMEM_DECLSPEC  int pshmem_ptrdiff_test_all_vector(volatile ptrdiff_t *ivars, size_t nelems, const int *status, int cmp, ptrdiff_t *values);
+#if OSHMEM_HAVE_C11
+#define pshmem_test_all_vector(ivars, nelems, status, cmp, values)         \
+    _Generic(&*(ivars),                                                    \
+        short*:                pshmem_short_test_all_vector,               \
+        unsigned short*:       pshmem_ushort_test_all_vector,              \
+        int*:                  pshmem_int_test_all_vector,                 \
+        long*:                 pshmem_long_test_all_vector,                \
+        long long*:            pshmem_longlong_test_all_vector,            \
+        unsigned int*:         pshmem_uint_test_all_vector,                \
+        unsigned long*:        pshmem_ulong_test_all_vector,               \
+        unsigned long long*:   pshmem_ulonglong_test_all_vector)(ivars, nelems, status, cmp, values)
+#endif
+
+
+OSHMEM_DECLSPEC  size_t pshmem_short_test_any_vector(volatile short *ivars, size_t nelems, const int *status, int cmp, short *values);
+OSHMEM_DECLSPEC  size_t pshmem_ushort_test_any_vector(volatile unsigned short *ivars, size_t nelems, const int *status, int cmp, unsigned short *values);
+OSHMEM_DECLSPEC  size_t pshmem_int_test_any_vector(volatile int *ivars, size_t nelems, const int *status, int cmp, int *values);
+OSHMEM_DECLSPEC  size_t pshmem_long_test_any_vector(volatile long *ivars, size_t nelems, const int *status, int cmp, long *values);
+OSHMEM_DECLSPEC  size_t pshmem_longlong_test_any_vector(volatile long long *ivars, size_t nelems, const int *status, int cmp, long long *values);
+OSHMEM_DECLSPEC  size_t pshmem_uint_test_any_vector(volatile unsigned int *ivars, size_t nelems, const int *status, int cmp, unsigned int *values);
+OSHMEM_DECLSPEC  size_t pshmem_ulong_test_any_vector(volatile unsigned long *ivars, size_t nelems, const int *status, int cmp, unsigned long *values);
+OSHMEM_DECLSPEC  size_t pshmem_ulonglong_test_any_vector(volatile unsigned long long *ivars, size_t nelems, const int *status, int cmp, unsigned long long *values);
+OSHMEM_DECLSPEC  size_t pshmem_int32_test_any_vector(volatile int32_t *ivars, size_t nelems, const int *status, int cmp, int32_t *values);
+OSHMEM_DECLSPEC  size_t pshmem_int64_test_any_vector(volatile int64_t *ivars, size_t nelems, const int *status, int cmp, int64_t *values);
+OSHMEM_DECLSPEC  size_t pshmem_uint32_test_any_vector(volatile uint32_t *ivars, size_t nelems, const int *status, int cmp, uint32_t *values);
+OSHMEM_DECLSPEC  size_t pshmem_uint64_test_any_vector(volatile uint64_t *ivars, size_t nelems, const int *status, int cmp, uint64_t *values);
+OSHMEM_DECLSPEC  size_t pshmem_size_test_any_vector(volatile size_t *ivars, size_t nelems, const int *status, int cmp, size_t *values);
+OSHMEM_DECLSPEC  size_t pshmem_ptrdiff_test_any_vector(volatile ptrdiff_t *ivars, size_t nelems, const int *status, int cmp, ptrdiff_t *values);
+#if OSHMEM_HAVE_C11
+#define pshmem_test_any_vector(ivars, nelems, status, cmp, values)         \
+    _Generic(&*(ivars),                                                    \
+        short*:                pshmem_short_test_any_vector,               \
+        unsigned short*:       pshmem_ushort_test_any_vector,              \
+        int*:                  pshmem_int_test_any_vector,                 \
+        long*:                 pshmem_long_test_any_vector,                \
+        long long*:            pshmem_longlong_test_any_vector,            \
+        unsigned int*:         pshmem_uint_test_any_vector,                \
+        unsigned long*:        pshmem_ulong_test_any_vector,               \
+        unsigned long long*:   pshmem_ulonglong_test_any_vector)(ivars, nelems, status, cmp, values)
+#endif
+
+
+OSHMEM_DECLSPEC  size_t pshmem_short_test_some_vector(volatile short *ivars, size_t nelems, size_t *indices, const int *status, int cmp, short *values);
+OSHMEM_DECLSPEC  size_t pshmem_ushort_test_some_vector(volatile unsigned short *ivars, size_t nelems, size_t *indices, const int *status, int cmp, unsigned short *values);
+OSHMEM_DECLSPEC  size_t pshmem_int_test_some_vector(volatile int *ivars, size_t nelems, size_t *indices, const int *status, int cmp, int *values);
+OSHMEM_DECLSPEC  size_t pshmem_long_test_some_vector(volatile long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, long *values);
+OSHMEM_DECLSPEC  size_t pshmem_longlong_test_some_vector(volatile long long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, long long *values);
+OSHMEM_DECLSPEC  size_t pshmem_uint_test_some_vector(volatile unsigned int *ivars, size_t nelems, size_t *indices, const int *status, int cmp, unsigned int *values);
+OSHMEM_DECLSPEC  size_t pshmem_ulong_test_some_vector(volatile unsigned long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, unsigned long *values);
+OSHMEM_DECLSPEC  size_t pshmem_ulonglong_test_some_vector(volatile unsigned long long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, unsigned long long *values);
+OSHMEM_DECLSPEC  size_t pshmem_int32_test_some_vector(volatile int32_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, int32_t *values);
+OSHMEM_DECLSPEC  size_t pshmem_int64_test_some_vector(volatile int64_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, int64_t *values);
+OSHMEM_DECLSPEC  size_t pshmem_uint32_test_some_vector(volatile uint32_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, uint32_t *values);
+OSHMEM_DECLSPEC  size_t pshmem_uint64_test_some_vector(volatile uint64_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, uint64_t *values);
+OSHMEM_DECLSPEC  size_t pshmem_size_test_some_vector(volatile size_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, size_t *values);
+OSHMEM_DECLSPEC  size_t pshmem_ptrdiff_test_some_vector(volatile ptrdiff_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, ptrdiff_t *values);
+#if OSHMEM_HAVE_C11
+#define pshmem_test_some_vector(ivars, nelems, indices, status, cmp, values)\
+    _Generic(&*(ivars),                                                     \
+        short*:                pshmem_short_test_some_vector,               \
+        unsigned short*:       pshmem_ushort_test_some_vector,              \
+        int*:                  pshmem_int_test_some_vector,                 \
+        long*:                 pshmem_long_test_some_vector,                \
+        long long*:            pshmem_longlong_test_some_vector,            \
+        unsigned int*:         pshmem_uint_test_some_vector,                \
+        unsigned long*:        pshmem_ulong_test_some_vector,               \
+        unsigned long long*:   pshmem_ulonglong_test_some_vector)(ivars, nelems, indices, status, cmp, values)
+#endif
+
 /*
  * Barrier sync routines
  */
 OSHMEM_DECLSPEC  void pshmem_barrier(int PE_start, int logPE_stride, int PE_size, long *pSync);
 OSHMEM_DECLSPEC  void pshmem_barrier_all(void);
-OSHMEM_DECLSPEC  void pshmem_sync(int PE_start, int logPE_stride, int PE_size, long *pSync);
+OSHMEM_DECLSPEC  void pshmem_sync_deprecated(int PE_start, int logPE_stride, int PE_size, long *pSync);
 OSHMEM_DECLSPEC  void pshmem_sync_all(void);
 OSHMEM_DECLSPEC  void pshmem_fence(void);
 OSHMEM_DECLSPEC  void pshmem_ctx_fence(shmem_ctx_t ctx);
 OSHMEM_DECLSPEC  void pshmem_quiet(void);
 OSHMEM_DECLSPEC  void pshmem_ctx_quiet(shmem_ctx_t ctx);
 
+#if OSHMEM_HAVE_C11
+#define pshmem_sync(...)                                               \
+    _Generic((__OSHMEM_VAR_ARG1(__VA_ARGS__)),                         \
+            shmem_team_t: pshmem_team_sync,                            \
+            int:          pshmem_sync_deprecated)(__VA_ARGS__)
+#endif
+
 /*
  * Collective routines
  */
diff --git a/oshmem/include/pshmemx.h b/oshmem/include/pshmemx.h
index 0b4ffcbd202..5a0f7f5a95c 100644
--- a/oshmem/include/pshmemx.h
+++ b/oshmem/include/pshmemx.h
@@ -16,11 +16,6 @@
 extern "C" {
 #endif
 
-/*
- * Symmetric heap routines
- */
-OSHMEM_DECLSPEC  void* pshmemx_malloc_with_hint(size_t size, long hint);
-
 
 /*
  * Legacy API
diff --git a/oshmem/include/shmem-compat.h b/oshmem/include/shmem-compat.h
index 19a873f7903..9f62c67a420 100644
--- a/oshmem/include/shmem-compat.h
+++ b/oshmem/include/shmem-compat.h
@@ -1,4 +1,4 @@
-/* oshmem/include/shmem-compat.h. This file contains OpenSHMEM lagacy API  */
+/* oshmem/include/shmem-compat.h. This file contains OpenSHMEM legacy API  */
 /*
  * Copyright (c) 2014-2017 Mellanox Technologies, Inc.
  *                         All rights reserved.
diff --git a/oshmem/include/shmem.h.in b/oshmem/include/shmem.h.in
index fe1908c9b49..44d1ecac1e5 100644
--- a/oshmem/include/shmem.h.in
+++ b/oshmem/include/shmem.h.in
@@ -84,7 +84,7 @@ enum {
 /* size of symmetric heap in bytes.
  * Can be qualified with the letter 'K', 'M', 'G' or 'T'
  */
-#define SHMEM_HEAP_SIZE "SHMEM_SYMMETRIC_HEAP_SIZE"
+#define SHMEM_HEAP_SIZE "SHMEM_SYMMETRIC_SIZE"
 
 /*
  * Type of allocator used by symmetric heap
@@ -102,6 +102,16 @@ enum {
 #define SHMEM_CTX_PRIVATE               (1<<0)
 #define SHMEM_CTX_SERIALIZED            (1<<1)
 #define SHMEM_CTX_NOSTORE               (1<<2)
+#define SHMEM_CTX_INVALID               NULL
+
+#define SHMEM_SIGNAL_SET                (1<<0)
+#define SHMEM_SIGNAL_ADD                (1<<1)
+
+#define SHMEM_MALLOC_ATOMICS_REMOTE     (1<<0)
+#define SHMEM_MALLOC_SIGNAL_REMOTE      (1<<1)
+
+#define SHMEM_TEAM_INVALID              NULL
+#define SHMEM_TEAM_NUM_CONTEXTS         (1<<0)
 
 /*
  * Deprecated (but still valid) names
@@ -175,7 +185,7 @@ OSHMEM_DECLSPEC void shmem_info_get_version(int *major, int *minor);
 OSHMEM_DECLSPEC void shmem_info_get_name(char *name);
 
 /*
- * Accessability routines
+ * Accessibility routines
  */
 OSHMEM_DECLSPEC int shmem_pe_accessible(int pe);
 OSHMEM_DECLSPEC int shmem_addr_accessible(const void *addr, int pe);
@@ -187,6 +197,7 @@ OSHMEM_DECLSPEC  void* shmem_malloc(size_t size);
 OSHMEM_DECLSPEC  void* shmem_calloc(size_t count, size_t size);
 OSHMEM_DECLSPEC  void* shmem_align(size_t align, size_t size);
 OSHMEM_DECLSPEC  void* shmem_realloc(void *ptr, size_t size);
+OSHMEM_DECLSPEC  void* shmem_malloc_with_hints(size_t size, long hints);
 OSHMEM_DECLSPEC  void shmem_free(void* ptr);
 
 /*
@@ -207,6 +218,569 @@ extern shmem_ctx_t oshmem_ctx_default;
 OSHMEM_DECLSPEC int shmem_ctx_create(long options, shmem_ctx_t *ctx);
 OSHMEM_DECLSPEC void shmem_ctx_destroy(shmem_ctx_t ctx);
 
+/*
+ * Team management operations
+ */
+typedef struct { int dummy; } * shmem_team_t;
+typedef struct { int num_contexts; } shmem_team_config_t;
+
+extern shmem_team_t oshmem_team_shared;
+extern shmem_team_t oshmem_team_world;
+
+#define SHMEM_TEAM_SHARED oshmem_team_shared
+#define SHMEM_TEAM_WORLD  oshmem_team_world
+
+OSHMEM_DECLSPEC int shmem_team_my_pe(shmem_team_t team);
+OSHMEM_DECLSPEC int shmem_team_n_pes(shmem_team_t team);
+OSHMEM_DECLSPEC int shmem_team_get_config(shmem_team_t team, long config_mask, shmem_team_config_t *config);
+OSHMEM_DECLSPEC int shmem_team_translate_pe(shmem_team_t src_team, int src_pe, shmem_team_t dest_team);
+OSHMEM_DECLSPEC int shmem_team_split_strided(shmem_team_t parent_team, int start, int stride, int size, const shmem_team_config_t *config, long config_mask, shmem_team_t *new_team);
+OSHMEM_DECLSPEC int shmem_team_split_2d(shmem_team_t parent_team, int xrange, const shmem_team_config_t *xaxis_config, long xaxis_mask, shmem_team_t *xaxis_team, const shmem_team_config_t *yaxis_config, long yaxis_mask, shmem_team_t *yaxis_team);
+OSHMEM_DECLSPEC void shmem_team_destroy(shmem_team_t team);
+OSHMEM_DECLSPEC int shmem_ctx_get_team(shmem_ctx_t ctx, shmem_team_t *team);
+OSHMEM_DECLSPEC int shmem_team_create_ctx(shmem_team_t team, long options, shmem_ctx_t *ctx);
+
+/*
+ * Teams-based Collectives
+ */
+
+/* Teams sync */
+OSHMEM_DECLSPEC  void shmem_team_sync(shmem_team_t team);
+
+/* Teams alltoall */
+OSHMEM_DECLSPEC  int shmem_char_alltoall(shmem_team_t team, char *target, const char *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_short_alltoall(shmem_team_t team, short *target, const short *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_int_alltoall(shmem_team_t team, int *target, const int *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_long_alltoall(shmem_team_t team, long *target, const long *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_float_alltoall(shmem_team_t team, float *target, const float *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_double_alltoall(shmem_team_t team, double *target, const double *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_longlong_alltoall(shmem_team_t team, long long *target, const long long *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_schar_alltoall(shmem_team_t team, signed char *target, const signed char *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_uchar_alltoall(shmem_team_t team, unsigned char *target, const unsigned char *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_ushort_alltoall(shmem_team_t team, unsigned short *target, const unsigned short *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_uint_alltoall(shmem_team_t team, unsigned int *target, const unsigned int *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_ulong_alltoall(shmem_team_t team, unsigned long *target, const unsigned long *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_ulonglong_alltoall(shmem_team_t team, unsigned long long *target, const unsigned long long *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_longdouble_alltoall(shmem_team_t team, long double *target, const long double *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_int8_alltoall(shmem_team_t team, int8_t *target, const int8_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_int16_alltoall(shmem_team_t team, int16_t *target, const int16_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_int32_alltoall(shmem_team_t team, int32_t *target, const int32_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_int64_alltoall(shmem_team_t team, int64_t *target, const int64_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_uint8_alltoall(shmem_team_t team, uint8_t *target, const uint8_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_uint16_alltoall(shmem_team_t team, uint16_t *target, const uint16_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_uint32_alltoall(shmem_team_t team, uint32_t *target, const uint32_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_uint64_alltoall(shmem_team_t team, uint64_t *target, const uint64_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_size_alltoall(shmem_team_t team, size_t *target, const size_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_ptrdiff_alltoall(shmem_team_t team, ptrdiff_t *target, const ptrdiff_t *source, size_t nelems);
+#if OSHMEM_HAVE_C11
+#define shmem_alltoall(...)                                             \
+    _Generic(&*(__OSHMEM_VAR_ARG2(__VA_ARGS__)),                        \
+            char*:        shmem_char_alltoall,               	        \
+            short*:       shmem_short_alltoall,              	        \
+            int*:         shmem_int_alltoall,                	        \
+            long*:        shmem_long_alltoall,               	        \
+            long long*:   shmem_longlong_alltoall,           	        \
+            signed char*:        shmem_schar_alltoall,       	        \
+            unsigned char*:      shmem_uchar_alltoall,       	        \
+            unsigned short*:     shmem_ushort_alltoall,      	        \
+            unsigned int*:       shmem_uint_alltoall,        	        \
+            unsigned long*:      shmem_ulong_alltoall,       	        \
+            unsigned long long*: shmem_ulonglong_alltoall,   	        \
+            float*:       shmem_float_alltoall,              	        \
+            double*:      shmem_double_alltoall,             	        \
+            long double*: shmem_longdouble_alltoall,         	        \
+            default:      __oshmem_datatype_ignore)(__VA_ARGS__)
+#endif
+
+OSHMEM_DECLSPEC  int shmem_alltoallmem(shmem_team_t team, void *target, const void *source, size_t nelems);
+
+/* Teams alltoalls */
+OSHMEM_DECLSPEC  int shmem_char_alltoalls(shmem_team_t team, char *target, const char *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_short_alltoalls(shmem_team_t team, short *target, const short *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_int_alltoalls(shmem_team_t team, int *target, const int *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_long_alltoalls(shmem_team_t team, long *target, const long *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_float_alltoalls(shmem_team_t team, float *target, const float *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_double_alltoalls(shmem_team_t team, double *target, const double *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_longlong_alltoalls(shmem_team_t team, long long *target, const long long *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_schar_alltoalls(shmem_team_t team, signed char *target, const signed char *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_uchar_alltoalls(shmem_team_t team, unsigned char *target, const unsigned char *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_ushort_alltoalls(shmem_team_t team, unsigned short *target, const unsigned short *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_uint_alltoalls(shmem_team_t team, unsigned int *target, const unsigned int *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_ulong_alltoalls(shmem_team_t team, unsigned long *target, const unsigned long *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_ulonglong_alltoalls(shmem_team_t team, unsigned long long *target, const unsigned long long *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_longdouble_alltoalls(shmem_team_t team, long double *target, const long double *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_int8_alltoalls(shmem_team_t team, int8_t *target, const int8_t *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_int16_alltoalls(shmem_team_t team, int16_t *target, const int16_t *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_int32_alltoalls(shmem_team_t team, int32_t *target, const int32_t *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_int64_alltoalls(shmem_team_t team, int64_t *target, const int64_t *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_uint8_alltoalls(shmem_team_t team, uint8_t *target, const uint8_t *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_uint16_alltoalls(shmem_team_t team, uint16_t *target, const uint16_t *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_uint32_alltoalls(shmem_team_t team, uint32_t *target, const uint32_t *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_uint64_alltoalls(shmem_team_t team, uint64_t *target, const uint64_t *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_size_alltoalls(shmem_team_t team, size_t *target, const size_t *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_ptrdiff_alltoalls(shmem_team_t team, ptrdiff_t *target, const ptrdiff_t *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+#if OSHMEM_HAVE_C11
+#define shmem_alltoalls(...)                                             \
+    _Generic(&*(__OSHMEM_VAR_ARG2(__VA_ARGS__)),                        \
+            char*:        shmem_char_alltoalls,                    \
+            short*:       shmem_short_alltoalls,                   \
+            int*:         shmem_int_alltoalls,                     \
+            long*:        shmem_long_alltoalls,                    \
+            long long*:   shmem_longlong_alltoalls,                \
+            signed char*:        shmem_schar_alltoalls,            \
+            unsigned char*:      shmem_uchar_alltoalls,            \
+            unsigned short*:     shmem_ushort_alltoalls,           \
+            unsigned int*:       shmem_uint_alltoalls,             \
+            unsigned long*:      shmem_ulong_alltoalls,            \
+            unsigned long long*: shmem_ulonglong_alltoalls,        \
+            float*:       shmem_float_alltoalls,                   \
+            double*:      shmem_double_alltoalls,                  \
+            long double*: shmem_longdouble_alltoalls,              \
+            default:      __oshmem_datatype_ignore)(__VA_ARGS__)
+#endif
+
+OSHMEM_DECLSPEC  int shmem_alltoallsmem(shmem_team_t team, void *target, const void *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems);
+
+
+/* Teams broadcast */
+OSHMEM_DECLSPEC  int shmem_char_broadcast(shmem_team_t team, char *target, const char *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int shmem_short_broadcast(shmem_team_t team, short *target, const short *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int shmem_int_broadcast(shmem_team_t team, int *target, const int *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int shmem_long_broadcast(shmem_team_t team, long *target, const long *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int shmem_float_broadcast(shmem_team_t team, float *target, const float *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int shmem_double_broadcast(shmem_team_t team, double *target, const double *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int shmem_longlong_broadcast(shmem_team_t team, long long *target, const long long *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int shmem_schar_broadcast(shmem_team_t team, signed char *target, const signed char *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int shmem_uchar_broadcast(shmem_team_t team, unsigned char *target, const unsigned char *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int shmem_ushort_broadcast(shmem_team_t team, unsigned short *target, const unsigned short *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int shmem_uint_broadcast(shmem_team_t team, unsigned int *target, const unsigned int *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int shmem_ulong_broadcast(shmem_team_t team, unsigned long *target, const unsigned long *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int shmem_ulonglong_broadcast(shmem_team_t team, unsigned long long *target, const unsigned long long *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int shmem_longdouble_broadcast(shmem_team_t team, long double *target, const long double *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int shmem_int8_broadcast(shmem_team_t team, int8_t *target, const int8_t *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int shmem_int16_broadcast(shmem_team_t team, int16_t *target, const int16_t *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int shmem_int32_broadcast(shmem_team_t team, int32_t *target, const int32_t *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int shmem_int64_broadcast(shmem_team_t team, int64_t *target, const int64_t *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int shmem_uint8_broadcast(shmem_team_t team, uint8_t *target, const uint8_t *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int shmem_uint16_broadcast(shmem_team_t team, uint16_t *target, const uint16_t *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int shmem_uint32_broadcast(shmem_team_t team, uint32_t *target, const uint32_t *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int shmem_uint64_broadcast(shmem_team_t team, uint64_t *target, const uint64_t *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int shmem_size_broadcast(shmem_team_t team, size_t *target, const size_t *source, size_t nelems, int PE_root);
+OSHMEM_DECLSPEC  int shmem_ptrdiff_broadcast(shmem_team_t team, ptrdiff_t *target, const ptrdiff_t *source, size_t nelems, int PE_root);
+#if OSHMEM_HAVE_C11
+#define shmem_broadcast(...)                                       \
+    _Generic(&*(__OSHMEM_VAR_ARG2(__VA_ARGS__)),                   \
+            char*:        shmem_char_broadcast,                    \
+            short*:       shmem_short_broadcast,                   \
+            int*:         shmem_int_broadcast,                     \
+            long*:        shmem_long_broadcast,                    \
+            long long*:   shmem_longlong_broadcast,                \
+            signed char*:        shmem_schar_broadcast,            \
+            unsigned char*:      shmem_uchar_broadcast,            \
+            unsigned short*:     shmem_ushort_broadcast,           \
+            unsigned int*:       shmem_uint_broadcast,             \
+            unsigned long*:      shmem_ulong_broadcast,            \
+            unsigned long long*: shmem_ulonglong_broadcast,        \
+            float*:       shmem_float_broadcast,                   \
+            double*:      shmem_double_broadcast,                  \
+            long double*: shmem_longdouble_broadcast,              \
+            default:      __oshmem_datatype_ignore)(__VA_ARGS__)
+#endif
+
+OSHMEM_DECLSPEC  int shmem_broadcastmem(shmem_team_t team, void *target, const void *source, size_t nelems, int PE_root);
+
+/* Teams collect */
+OSHMEM_DECLSPEC  int shmem_char_collect(shmem_team_t team, char *target, const char *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_short_collect(shmem_team_t team, short *target, const short *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_int_collect(shmem_team_t team, int *target, const int *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_long_collect(shmem_team_t team, long *target, const long *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_float_collect(shmem_team_t team, float *target, const float *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_double_collect(shmem_team_t team, double *target, const double *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_longlong_collect(shmem_team_t team, long long *target, const long long *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_schar_collect(shmem_team_t team, signed char *target, const signed char *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_uchar_collect(shmem_team_t team, unsigned char *target, const unsigned char *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_ushort_collect(shmem_team_t team, unsigned short *target, const unsigned short *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_uint_collect(shmem_team_t team, unsigned int *target, const unsigned int *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_ulong_collect(shmem_team_t team, unsigned long *target, const unsigned long *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_ulonglong_collect(shmem_team_t team, unsigned long long *target, const unsigned long long *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_longdouble_collect(shmem_team_t team, long double *target, const long double *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_int8_collect(shmem_team_t team, int8_t *target, const int8_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_int16_collect(shmem_team_t team, int16_t *target, const int16_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_int32_collect(shmem_team_t team, int32_t *target, const int32_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_int64_collect(shmem_team_t team, int64_t *target, const int64_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_uint8_collect(shmem_team_t team, uint8_t *target, const uint8_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_uint16_collect(shmem_team_t team, uint16_t *target, const uint16_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_uint32_collect(shmem_team_t team, uint32_t *target, const uint32_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_uint64_collect(shmem_team_t team, uint64_t *target, const uint64_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_size_collect(shmem_team_t team, size_t *target, const size_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_ptrdiff_collect(shmem_team_t team, ptrdiff_t *target, const ptrdiff_t *source, size_t nelems);
+#if OSHMEM_HAVE_C11
+#define shmem_collect(...)                                         \
+    _Generic(&*(__OSHMEM_VAR_ARG2(__VA_ARGS__)),                   \
+            char*:        shmem_char_collect,                      \
+            short*:       shmem_short_collect,                     \
+            int*:         shmem_int_collect,                       \
+            long*:        shmem_long_collect,                      \
+            long long*:   shmem_longlong_collect,                  \
+            signed char*:        shmem_schar_collect,              \
+            unsigned char*:      shmem_uchar_collect,              \
+            unsigned short*:     shmem_ushort_collect,             \
+            unsigned int*:       shmem_uint_collect,               \
+            unsigned long*:      shmem_ulong_collect,              \
+            unsigned long long*: shmem_ulonglong_collect,          \
+            float*:       shmem_float_collect,                     \
+            double*:      shmem_double_collect,                    \
+            long double*: shmem_longdouble_collect,                \
+            default:      __oshmem_datatype_ignore)(__VA_ARGS__)
+#endif
+
+
+OSHMEM_DECLSPEC  int shmem_collectmem(shmem_team_t team, void *target, const void *source, size_t nelems);
+
+/* Teams fcollect */
+OSHMEM_DECLSPEC  int shmem_char_fcollect(shmem_team_t team, char *target, const char *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_short_fcollect(shmem_team_t team, short *target, const short *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_int_fcollect(shmem_team_t team, int *target, const int *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_long_fcollect(shmem_team_t team, long *target, const long *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_float_fcollect(shmem_team_t team, float *target, const float *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_double_fcollect(shmem_team_t team, double *target, const double *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_longlong_fcollect(shmem_team_t team, long long *target, const long long *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_schar_fcollect(shmem_team_t team, signed char *target, const signed char *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_uchar_fcollect(shmem_team_t team, unsigned char *target, const unsigned char *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_ushort_fcollect(shmem_team_t team, unsigned short *target, const unsigned short *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_uint_fcollect(shmem_team_t team, unsigned int *target, const unsigned int *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_ulong_fcollect(shmem_team_t team, unsigned long *target, const unsigned long *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_ulonglong_fcollect(shmem_team_t team, unsigned long long *target, const unsigned long long *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_longdouble_fcollect(shmem_team_t team, long double *target, const long double *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_int8_fcollect(shmem_team_t team, int8_t *target, const int8_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_int16_fcollect(shmem_team_t team, int16_t *target, const int16_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_int32_fcollect(shmem_team_t team, int32_t *target, const int32_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_int64_fcollect(shmem_team_t team, int64_t *target, const int64_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_uint8_fcollect(shmem_team_t team, uint8_t *target, const uint8_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_uint16_fcollect(shmem_team_t team, uint16_t *target, const uint16_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_uint32_fcollect(shmem_team_t team, uint32_t *target, const uint32_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_uint64_fcollect(shmem_team_t team, uint64_t *target, const uint64_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_size_fcollect(shmem_team_t team, size_t *target, const size_t *source, size_t nelems);
+OSHMEM_DECLSPEC  int shmem_ptrdiff_fcollect(shmem_team_t team, ptrdiff_t *target, const ptrdiff_t *source, size_t nelems);
+#if OSHMEM_HAVE_C11
+#define shmem_fcollect(...)                                        \
+    _Generic(&*(__OSHMEM_VAR_ARG2(__VA_ARGS__)),                   \
+            char*:        shmem_char_fcollect,                     \
+            short*:       shmem_short_fcollect,                    \
+            int*:         shmem_int_fcollect,                      \
+            long*:        shmem_long_fcollect,                     \
+            long long*:   shmem_longlong_fcollect,                 \
+            signed char*:        shmem_schar_fcollect,             \
+            unsigned char*:      shmem_uchar_fcollect,             \
+            unsigned short*:     shmem_ushort_fcollect,            \
+            unsigned int*:       shmem_uint_fcollect,              \
+            unsigned long*:      shmem_ulong_fcollect,             \
+            unsigned long long*: shmem_ulonglong_fcollect,         \
+            float*:       shmem_float_fcollect,                    \
+            double*:      shmem_double_fcollect,                   \
+            long double*: shmem_longdouble_fcollect,               \
+            default:      __oshmem_datatype_ignore)(__VA_ARGS__)
+#endif
+
+OSHMEM_DECLSPEC  int shmem_fcollectmem(shmem_team_t team, void *target, const void *source, size_t nelems);
+
+
+
+/* Teams reduction: AND */
+OSHMEM_DECLSPEC  int shmem_uchar_and_reduce(shmem_team_t team, unsigned char *target, const unsigned char *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_ushort_and_reduce(shmem_team_t team, unsigned short *target, const unsigned short *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint_and_reduce(shmem_team_t team, unsigned int *target, const unsigned int *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_ulong_and_reduce(shmem_team_t team, unsigned long *target, const unsigned long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_ulonglong_and_reduce(shmem_team_t team, unsigned long long *target, const unsigned long long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int_and_reduce(shmem_team_t team, int *target, const int *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_longlong_and_reduce(shmem_team_t team, long long *target, const long long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int8_and_reduce(shmem_team_t team, int8_t *target, const int8_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int16_and_reduce(shmem_team_t team, int16_t *target, const int16_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int32_and_reduce(shmem_team_t team, int32_t *target, const int32_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int64_and_reduce(shmem_team_t team, int64_t *target, const int64_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint8_and_reduce(shmem_team_t team, uint8_t *target, const uint8_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint16_and_reduce(shmem_team_t team, uint16_t *target, const uint16_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint32_and_reduce(shmem_team_t team, uint32_t *target, const uint32_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint64_and_reduce(shmem_team_t team, uint64_t *target, const uint64_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_size_and_reduce(shmem_team_t team, size_t *target, const size_t *source, size_t nreduce);
+#if OSHMEM_HAVE_C11
+#define shmem_and_reduce(...)                                      \
+    _Generic(&*(__OSHMEM_VAR_ARG2(__VA_ARGS__)),                   \
+            unsigned char*:      shmem_uchar_and_reduce,           \
+            unsigned short*:     shmem_ushort_and_reduce,          \
+            unsigned int*:       shmem_uint_and_reduce,            \
+            unsigned long*:      shmem_ulong_and_reduce,           \
+            unsigned long long*: shmem_ulonglong_and_reduce,       \
+            int8_t*: shmem_int8_and_reduce,                        \
+            int16_t*: shmem_int16_and_reduce,                      \
+            int32_t*: shmem_int32_and_reduce,                      \
+            int64_t*: shmem_int64_and_reduce,                      \
+            long long*: shmem_longlong_and_reduce,                 \
+            default:      __oshmem_datatype_ignore)(__VA_ARGS__)
+#endif
+
+
+/* Teams reduction: OR */
+OSHMEM_DECLSPEC  int shmem_uchar_or_reduce(shmem_team_t team, unsigned char *target, const unsigned char *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_ushort_or_reduce(shmem_team_t team, unsigned short *target, const unsigned short *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint_or_reduce(shmem_team_t team, unsigned int *target, const unsigned int *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_ulong_or_reduce(shmem_team_t team, unsigned long *target, const unsigned long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_ulonglong_or_reduce(shmem_team_t team, unsigned long long *target, const unsigned long long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int_or_reduce(shmem_team_t team, int *target, const int *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_longlong_or_reduce(shmem_team_t team, long long *target, const long long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int8_or_reduce(shmem_team_t team, int8_t *target, const int8_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int16_or_reduce(shmem_team_t team, int16_t *target, const int16_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int32_or_reduce(shmem_team_t team, int32_t *target, const int32_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int64_or_reduce(shmem_team_t team, int64_t *target, const int64_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint8_or_reduce(shmem_team_t team, uint8_t *target, const uint8_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint16_or_reduce(shmem_team_t team, uint16_t *target, const uint16_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint32_or_reduce(shmem_team_t team, uint32_t *target, const uint32_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint64_or_reduce(shmem_team_t team, uint64_t *target, const uint64_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_size_or_reduce(shmem_team_t team, size_t *target, const size_t *source, size_t nreduce);
+#if OSHMEM_HAVE_C11
+#define shmem_or_reduce(...)                                       \
+    _Generic(&*(__OSHMEM_VAR_ARG2(__VA_ARGS__)),                   \
+            unsigned char*:      shmem_uchar_or_reduce,            \
+            unsigned short*:     shmem_ushort_or_reduce,           \
+            unsigned int*:       shmem_uint_or_reduce,             \
+            unsigned long*:      shmem_ulong_or_reduce,            \
+            unsigned long long*: shmem_ulonglong_or_reduce,        \
+            int8_t*: shmem_int8_or_reduce,                         \
+            int16_t*: shmem_int16_or_reduce,                       \
+            int32_t*: shmem_int32_or_reduce,                       \
+            int64_t*: shmem_int64_or_reduce,                       \
+            long long*: shmem_longlong_or_reduce,                  \
+            default:      __oshmem_datatype_ignore)(__VA_ARGS__)
+#endif
+
+
+
+/* Teams reduction: XOR */
+OSHMEM_DECLSPEC  int shmem_uchar_xor_reduce(shmem_team_t team, unsigned char *target, const unsigned char *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_ushort_xor_reduce(shmem_team_t team, unsigned short *target, const unsigned short *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint_xor_reduce(shmem_team_t team, unsigned int *target, const unsigned int *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_ulong_xor_reduce(shmem_team_t team, unsigned long *target, const unsigned long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_ulonglong_xor_reduce(shmem_team_t team, unsigned long long *target, const unsigned long long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int_xor_reduce(shmem_team_t team, int *target, const int *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_longlong_xor_reduce(shmem_team_t team, long long *target, const long long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int8_xor_reduce(shmem_team_t team, int8_t *target, const int8_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int16_xor_reduce(shmem_team_t team, int16_t *target, const int16_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int32_xor_reduce(shmem_team_t team, int32_t *target, const int32_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int64_xor_reduce(shmem_team_t team, int64_t *target, const int64_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint8_xor_reduce(shmem_team_t team, uint8_t *target, const uint8_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint16_xor_reduce(shmem_team_t team, uint16_t *target, const uint16_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint32_xor_reduce(shmem_team_t team, uint32_t *target, const uint32_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint64_xor_reduce(shmem_team_t team, uint64_t *target, const uint64_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_size_xor_reduce(shmem_team_t team, size_t *target, const size_t *source, size_t nreduce);
+#if OSHMEM_HAVE_C11
+#define shmem_xor_reduce(...)                                      \
+    _Generic(&*(__OSHMEM_VAR_ARG2(__VA_ARGS__)),                   \
+            unsigned char*:      shmem_uchar_xor_reduce,           \
+            unsigned short*:     shmem_ushort_xor_reduce,          \
+            unsigned int*:       shmem_uint_xor_reduce,            \
+            unsigned long*:      shmem_ulong_xor_reduce,           \
+            unsigned long long*: shmem_ulonglong_xor_reduce,       \
+            int8_t*: shmem_int8_xor_reduce,                        \
+            int16_t*: shmem_int16_xor_reduce,                      \
+            int32_t*: shmem_int32_xor_reduce,                      \
+            int64_t*: shmem_int64_xor_reduce,                      \
+            long long*: shmem_longlong_xor_reduce,                 \
+            default:      __oshmem_datatype_ignore)(__VA_ARGS__)
+#endif
+
+/* Teams reduction: MAX */
+OSHMEM_DECLSPEC  int shmem_char_max_reduce(shmem_team_t team, char *target, const char *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_short_max_reduce(shmem_team_t team, short *target, const short *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int_max_reduce(shmem_team_t team, int *target, const int *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_long_max_reduce(shmem_team_t team, long *target, const long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_float_max_reduce(shmem_team_t team, float *target, const float *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_double_max_reduce(shmem_team_t team, double *target, const double *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_longlong_max_reduce(shmem_team_t team, long long *target, const long long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_schar_max_reduce(shmem_team_t team, signed char *target, const signed char *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uchar_max_reduce(shmem_team_t team, unsigned char *target, const unsigned char *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_ushort_max_reduce(shmem_team_t team, unsigned short *target, const unsigned short *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint_max_reduce(shmem_team_t team, unsigned int *target, const unsigned int *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_ulong_max_reduce(shmem_team_t team, unsigned long *target, const unsigned long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_ulonglong_max_reduce(shmem_team_t team, unsigned long long *target, const unsigned long long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_longdouble_max_reduce(shmem_team_t team, long double *target, const long double *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int8_max_reduce(shmem_team_t team, int8_t *target, const int8_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int16_max_reduce(shmem_team_t team, int16_t *target, const int16_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int32_max_reduce(shmem_team_t team, int32_t *target, const int32_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int64_max_reduce(shmem_team_t team, int64_t *target, const int64_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint8_max_reduce(shmem_team_t team, uint8_t *target, const uint8_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint16_max_reduce(shmem_team_t team, uint16_t *target, const uint16_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint32_max_reduce(shmem_team_t team, uint32_t *target, const uint32_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint64_max_reduce(shmem_team_t team, uint64_t *target, const uint64_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_size_max_reduce(shmem_team_t team, size_t *target, const size_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_ptrdiff_max_reduce(shmem_team_t team, ptrdiff_t *target, const ptrdiff_t *source, size_t nreduce);
+#if OSHMEM_HAVE_C11
+#define shmem_max_reduce(...)                                      \
+    _Generic(&*(__OSHMEM_VAR_ARG2(__VA_ARGS__)),                   \
+            char*:        shmem_char_max_reduce,                   \
+            short*:       shmem_short_max_reduce,                  \
+            int*:         shmem_int_max_reduce,                    \
+            long*:        shmem_long_max_reduce,                   \
+            long long*:   shmem_longlong_max_reduce,               \
+            signed char*:        shmem_schar_max_reduce,           \
+            unsigned char*:      shmem_uchar_max_reduce,           \
+            unsigned short*:     shmem_ushort_max_reduce,          \
+            unsigned int*:       shmem_uint_max_reduce,            \
+            unsigned long*:      shmem_ulong_max_reduce,           \
+            unsigned long long*: shmem_ulonglong_max_reduce,       \
+            float*:       shmem_float_max_reduce,                  \
+            double*:      shmem_double_max_reduce,                 \
+            long double*: shmem_longdouble_max_reduce,             \
+            default:      __oshmem_datatype_ignore)(__VA_ARGS__)
+#endif
+
+
+/* Teams reduction: MIN */
+OSHMEM_DECLSPEC  int shmem_char_min_reduce(shmem_team_t team, char *target, const char *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_short_min_reduce(shmem_team_t team, short *target, const short *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int_min_reduce(shmem_team_t team, int *target, const int *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_long_min_reduce(shmem_team_t team, long *target, const long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_float_min_reduce(shmem_team_t team, float *target, const float *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_double_min_reduce(shmem_team_t team, double *target, const double *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_longlong_min_reduce(shmem_team_t team, long long *target, const long long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_schar_min_reduce(shmem_team_t team, signed char *target, const signed char *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uchar_min_reduce(shmem_team_t team, unsigned char *target, const unsigned char *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_ushort_min_reduce(shmem_team_t team, unsigned short *target, const unsigned short *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint_min_reduce(shmem_team_t team, unsigned int *target, const unsigned int *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_ulong_min_reduce(shmem_team_t team, unsigned long *target, const unsigned long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_ulonglong_min_reduce(shmem_team_t team, unsigned long long *target, const unsigned long long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_longdouble_min_reduce(shmem_team_t team, long double *target, const long double *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int8_min_reduce(shmem_team_t team, int8_t *target, const int8_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int16_min_reduce(shmem_team_t team, int16_t *target, const int16_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int32_min_reduce(shmem_team_t team, int32_t *target, const int32_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int64_min_reduce(shmem_team_t team, int64_t *target, const int64_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint8_min_reduce(shmem_team_t team, uint8_t *target, const uint8_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint16_min_reduce(shmem_team_t team, uint16_t *target, const uint16_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint32_min_reduce(shmem_team_t team, uint32_t *target, const uint32_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint64_min_reduce(shmem_team_t team, uint64_t *target, const uint64_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_size_min_reduce(shmem_team_t team, size_t *target, const size_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_ptrdiff_min_reduce(shmem_team_t team, ptrdiff_t *target, const ptrdiff_t *source, size_t nreduce);
+#if OSHMEM_HAVE_C11
+#define shmem_min_reduce(...)                                      \
+    _Generic(&*(__OSHMEM_VAR_ARG2(__VA_ARGS__)),                   \
+            char*:        shmem_char_min_reduce,               	\
+            short*:       shmem_short_min_reduce,              	\
+            int*:         shmem_int_min_reduce,                	\
+            long*:        shmem_long_min_reduce,               	\
+            long long*:   shmem_longlong_min_reduce,           	\
+            signed char*:        shmem_schar_min_reduce,       	\
+            unsigned char*:      shmem_uchar_min_reduce,       	\
+            unsigned short*:     shmem_ushort_min_reduce,      	\
+            unsigned int*:       shmem_uint_min_reduce,        	\
+            unsigned long*:      shmem_ulong_min_reduce,       	\
+            unsigned long long*: shmem_ulonglong_min_reduce,   	\
+            float*:       shmem_float_min_reduce,              	\
+            double*:      shmem_double_min_reduce,             	\
+            long double*: shmem_longdouble_min_reduce,         	\
+            default:      __oshmem_datatype_ignore)(__VA_ARGS__)
+#endif
+
+
+
+/* Teams reduction: SUM */
+OSHMEM_DECLSPEC  int shmem_char_sum_reduce(shmem_team_t team, char *target, const char *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_short_sum_reduce(shmem_team_t team, short *target, const short *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int_sum_reduce(shmem_team_t team, int *target, const int *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_long_sum_reduce(shmem_team_t team, long *target, const long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_float_sum_reduce(shmem_team_t team, float *target, const float *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_double_sum_reduce(shmem_team_t team, double *target, const double *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_longlong_sum_reduce(shmem_team_t team, long long *target, const long long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_schar_sum_reduce(shmem_team_t team, signed char *target, const signed char *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uchar_sum_reduce(shmem_team_t team, unsigned char *target, const unsigned char *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_ushort_sum_reduce(shmem_team_t team, unsigned short *target, const unsigned short *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint_sum_reduce(shmem_team_t team, unsigned int *target, const unsigned int *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_ulong_sum_reduce(shmem_team_t team, unsigned long *target, const unsigned long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_ulonglong_sum_reduce(shmem_team_t team, unsigned long long *target, const unsigned long long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_longdouble_sum_reduce(shmem_team_t team, long double *target, const long double *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int8_sum_reduce(shmem_team_t team, int8_t *target, const int8_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int16_sum_reduce(shmem_team_t team, int16_t *target, const int16_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int32_sum_reduce(shmem_team_t team, int32_t *target, const int32_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int64_sum_reduce(shmem_team_t team, int64_t *target, const int64_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint8_sum_reduce(shmem_team_t team, uint8_t *target, const uint8_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint16_sum_reduce(shmem_team_t team, uint16_t *target, const uint16_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint32_sum_reduce(shmem_team_t team, uint32_t *target, const uint32_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint64_sum_reduce(shmem_team_t team, uint64_t *target, const uint64_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_size_sum_reduce(shmem_team_t team, size_t *target, const size_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_ptrdiff_sum_reduce(shmem_team_t team, ptrdiff_t *target, const ptrdiff_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_complexd_sum_reduce(shmem_team_t team, OSHMEM_COMPLEX_TYPE(double) *target, const OSHMEM_COMPLEX_TYPE(double) *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_complexf_sum_reduce(shmem_team_t team, OSHMEM_COMPLEX_TYPE(float) *target, const OSHMEM_COMPLEX_TYPE(float) *source, size_t nreduce);
+#if OSHMEM_HAVE_C11
+#define shmem_sum_reduce(...)                                           \
+    _Generic(&*(__OSHMEM_VAR_ARG2(__VA_ARGS__)),                        \
+            char*:        shmem_char_sum_reduce,               		\
+            short*:       shmem_short_sum_reduce,              		\
+            int*:         shmem_int_sum_reduce,                		\
+            long*:        shmem_long_sum_reduce,               		\
+            long long*:   shmem_longlong_sum_reduce,           		\
+            signed char*:        shmem_schar_sum_reduce,       		\
+            unsigned char*:      shmem_uchar_sum_reduce,       		\
+            unsigned short*:     shmem_ushort_sum_reduce,      		\
+            unsigned int*:       shmem_uint_sum_reduce,        		\
+            unsigned long*:      shmem_ulong_sum_reduce,       		\
+            unsigned long long*: shmem_ulonglong_sum_reduce,   		\
+            float*:       shmem_float_sum_reduce,              		\
+            double*:      shmem_double_sum_reduce,             		\
+            long double*: shmem_longdouble_sum_reduce,         		\
+            OSHMEM_COMPLEX_TYPE(double)*: shmem_complexd_sum_reduce,    \
+            OSHMEM_COMPLEX_TYPE(float)*: shmem_complexf_sum_reduce,     \
+            default:      __oshmem_datatype_ignore)(__VA_ARGS__)
+#endif
+
+
+/* Teams reduction: PROD */
+OSHMEM_DECLSPEC  int shmem_char_prod_reduce(shmem_team_t team, char *target, const char *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_short_prod_reduce(shmem_team_t team, short *target, const short *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int_prod_reduce(shmem_team_t team, int *target, const int *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_long_prod_reduce(shmem_team_t team, long *target, const long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_float_prod_reduce(shmem_team_t team, float *target, const float *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_double_prod_reduce(shmem_team_t team, double *target, const double *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_longlong_prod_reduce(shmem_team_t team, long long *target, const long long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_schar_prod_reduce(shmem_team_t team, signed char *target, const signed char *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uchar_prod_reduce(shmem_team_t team, unsigned char *target, const unsigned char *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_ushort_prod_reduce(shmem_team_t team, unsigned short *target, const unsigned short *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint_prod_reduce(shmem_team_t team, unsigned int *target, const unsigned int *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_ulong_prod_reduce(shmem_team_t team, unsigned long *target, const unsigned long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_ulonglong_prod_reduce(shmem_team_t team, unsigned long long *target, const unsigned long long *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_longdouble_prod_reduce(shmem_team_t team, long double *target, const long double *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int8_prod_reduce(shmem_team_t team, int8_t *target, const int8_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int16_prod_reduce(shmem_team_t team, int16_t *target, const int16_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int32_prod_reduce(shmem_team_t team, int32_t *target, const int32_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_int64_prod_reduce(shmem_team_t team, int64_t *target, const int64_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint8_prod_reduce(shmem_team_t team, uint8_t *target, const uint8_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint16_prod_reduce(shmem_team_t team, uint16_t *target, const uint16_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint32_prod_reduce(shmem_team_t team, uint32_t *target, const uint32_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_uint64_prod_reduce(shmem_team_t team, uint64_t *target, const uint64_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_size_prod_reduce(shmem_team_t team, size_t *target, const size_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_ptrdiff_prod_reduce(shmem_team_t team, ptrdiff_t *target, const ptrdiff_t *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_complexd_prod_reduce(shmem_team_t team, OSHMEM_COMPLEX_TYPE(double) *target, const OSHMEM_COMPLEX_TYPE(double) *source, size_t nreduce);
+OSHMEM_DECLSPEC  int shmem_complexf_prod_reduce(shmem_team_t team, OSHMEM_COMPLEX_TYPE(float) *target, const OSHMEM_COMPLEX_TYPE(float) *source, size_t nreduce);
+#if OSHMEM_HAVE_C11
+#define shmem_prod_reduce(...)                                                  \
+    _Generic(&*(__OSHMEM_VAR_ARG2(__VA_ARGS__)),                                \
+            char*:        shmem_char_prod_reduce,               		\
+            short*:       shmem_short_prod_reduce,              		\
+            int*:         shmem_int_prod_reduce,                		\
+            long*:        shmem_long_prod_reduce,               		\
+            long long*:   shmem_longlong_prod_reduce,           		\
+            signed char*:        shmem_schar_prod_reduce,       		\
+            unsigned char*:      shmem_uchar_prod_reduce,       		\
+            unsigned short*:     shmem_ushort_prod_reduce,      		\
+            unsigned int*:       shmem_uint_prod_reduce,        		\
+            unsigned long*:      shmem_ulong_prod_reduce,       		\
+            unsigned long long*: shmem_ulonglong_prod_reduce,   		\
+            float*:       shmem_float_prod_reduce,              		\
+            double*:      shmem_double_prod_reduce,             		\
+            long double*: shmem_longdouble_prod_reduce,         		\
+            OSHMEM_COMPLEX_TYPE(double)*: shmem_complexd_prod_reduce,           \
+            OSHMEM_COMPLEX_TYPE(float)*: shmem_complexf_prod_reduce,            \
+            default:      __oshmem_datatype_ignore)(__VA_ARGS__)
+#endif
+
+
 /*
  * Elemental put routines
  */
@@ -596,6 +1170,216 @@ OSHMEM_DECLSPEC  void shmem_put64_nbi(void *target, const void *source, size_t l
 OSHMEM_DECLSPEC  void shmem_put128_nbi(void *target, const void *source, size_t len, int pe);
 OSHMEM_DECLSPEC  void shmem_putmem_nbi(void *target, const void *source, size_t len, int pe);
 
+/*
+ * Signaled put routines
+ */
+OSHMEM_DECLSPEC  void shmem_ctx_char_put_signal(shmem_ctx_t ctx, char *dest, const char *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_short_put_signal(shmem_ctx_t ctx, short *dest, const short *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_int_put_signal(shmem_ctx_t ctx, int *dest, const int *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_long_put_signal(shmem_ctx_t ctx, long *dest, const long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_float_put_signal(shmem_ctx_t ctx, float *dest, const float *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_double_put_signal(shmem_ctx_t ctx, double *dest, const double *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_longlong_put_signal(shmem_ctx_t ctx, long long *dest, const long long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_schar_put_signal(shmem_ctx_t ctx, signed char *dest, const signed char *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_uchar_put_signal(shmem_ctx_t ctx, unsigned char *dest, const unsigned char *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_ushort_put_signal(shmem_ctx_t ctx, unsigned short *dest, const unsigned short *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_uint_put_signal(shmem_ctx_t ctx, unsigned int *dest, const unsigned int *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_ulong_put_signal(shmem_ctx_t ctx, unsigned long *dest, const unsigned long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_ulonglong_put_signal(shmem_ctx_t ctx, unsigned long long *dest, const unsigned long long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_longdouble_put_signal(shmem_ctx_t ctx, long double *dest, const long double *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_int8_put_signal(shmem_ctx_t ctx, int8_t *dest, const int8_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_int16_put_signal(shmem_ctx_t ctx, int16_t *dest, const int16_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_int32_put_signal(shmem_ctx_t ctx, int32_t *dest, const int32_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_int64_put_signal(shmem_ctx_t ctx, int64_t *dest, const int64_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_uint8_put_signal(shmem_ctx_t ctx, uint8_t *dest, const uint8_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_uint16_put_signal(shmem_ctx_t ctx, uint16_t *dest, const uint16_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_uint32_put_signal(shmem_ctx_t ctx, uint32_t *dest, const uint32_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_uint64_put_signal(shmem_ctx_t ctx, uint64_t *dest, const uint64_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_size_put_signal(shmem_ctx_t ctx, size_t *dest, const size_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_ptrdiff_put_signal(shmem_ctx_t ctx, ptrdiff_t *dest, const ptrdiff_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+
+OSHMEM_DECLSPEC  void shmem_char_put_signal(char *dest, const char *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_short_put_signal(short *dest, const short *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_int_put_signal(int *dest, const int *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_long_put_signal(long *dest, const long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_float_put_signal(float *dest, const float *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_double_put_signal(double *dest, const double *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_longlong_put_signal(long long *dest, const long long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_schar_put_signal(signed char *dest, const signed char *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_uchar_put_signal(unsigned char *dest, const unsigned char *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ushort_put_signal(unsigned short *dest, const unsigned short *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_uint_put_signal(unsigned int *dest, const unsigned int *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ulong_put_signal(unsigned long *dest, const unsigned long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ulonglong_put_signal(unsigned long long *dest, const unsigned long long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_longdouble_put_signal(long double *dest, const long double *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_int8_put_signal(int8_t *dest, const int8_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_int16_put_signal(int16_t *dest, const int16_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_int32_put_signal(int32_t *dest, const int32_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_int64_put_signal(int64_t *dest, const int64_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_uint8_put_signal(uint8_t *dest, const uint8_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_uint16_put_signal(uint16_t *dest, const uint16_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_uint32_put_signal(uint32_t *dest, const uint32_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_uint64_put_signal(uint64_t *dest, const uint64_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_size_put_signal(size_t *dest, const size_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ptrdiff_put_signal(ptrdiff_t *dest, const ptrdiff_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+
+#if OSHMEM_HAVE_C11
+#define shmem_put_signal(...)                                                \
+    _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                             \
+            shmem_ctx_t:  _Generic((__OSHMEM_VAR_ARG2(__VA_ARGS__)), 	      \
+                char*:       shmem_ctx_char_put_signal,                      \
+                short*:      shmem_ctx_short_put_signal,                     \
+                int*:        shmem_ctx_int_put_signal,                       \
+                long*:       shmem_ctx_long_put_signal,                      \
+                long long*:  shmem_ctx_longlong_put_signal,                  \
+                signed char*:        shmem_ctx_schar_put_signal,             \
+                unsigned char*:      shmem_ctx_uchar_put_signal,             \
+                unsigned short*:     shmem_ctx_ushort_put_signal,            \
+                unsigned int*:       shmem_ctx_uint_put_signal,              \
+                unsigned long*:      shmem_ctx_ulong_put_signal,             \
+                unsigned long long*: shmem_ctx_ulonglong_put_signal,         \
+                float*:       shmem_ctx_float_put_signal,                    \
+                double*:      shmem_ctx_double_put_signal,                   \
+                long double*: shmem_ctx_longdouble_put_signal,               \
+                default:      __oshmem_datatype_ignore),                     \
+            char*:       shmem_char_put_signal,                              \
+            short*:      shmem_short_put_signal,                             \
+            int*:        shmem_int_put_signal,                               \
+            long*:       shmem_long_put_signal,                              \
+            long long*:  shmem_longlong_put_signal,                          \
+            signed char*:        shmem_schar_put_signal,                     \
+            unsigned char*:      shmem_uchar_put_signal,                     \
+            unsigned short*:     shmem_ushort_put_signal,                    \
+            unsigned int*:       shmem_uint_put_signal,                      \
+            unsigned long*:      shmem_ulong_put_signal,                     \
+            unsigned long long*: shmem_ulonglong_put_signal,                 \
+            float*:       shmem_float_put_signal,                            \
+            double*:      shmem_double_put_signal,                           \
+            long double*: shmem_longdouble_put_signal)(__VA_ARGS__)
+#endif
+
+OSHMEM_DECLSPEC void shmem_put8_signal(void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void shmem_put16_signal(void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void shmem_put32_signal(void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void shmem_put64_signal(void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void shmem_put128_signal(void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+
+OSHMEM_DECLSPEC void shmem_ctx_put8_signal(shmem_ctx_t ctx, void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_put16_signal(shmem_ctx_t ctx, void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_put32_signal(shmem_ctx_t ctx, void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_put64_signal(shmem_ctx_t ctx, void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_put128_signal(shmem_ctx_t ctx, void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+
+OSHMEM_DECLSPEC void shmem_putmem_signal(void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_putmem_signal(shmem_ctx_t ctx, void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+
+/*
+ * Nonblocking signaled put routines
+ */
+OSHMEM_DECLSPEC  void shmem_ctx_char_put_signal_nbi(shmem_ctx_t ctx, char *dest, const char *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_short_put_signal_nbi(shmem_ctx_t ctx, short *dest, const short *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_int_put_signal_nbi(shmem_ctx_t ctx, int *dest, const int *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_long_put_signal_nbi(shmem_ctx_t ctx, long *dest, const long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_float_put_signal_nbi(shmem_ctx_t ctx, float *dest, const float *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_double_put_signal_nbi(shmem_ctx_t ctx, double *dest, const double *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_longlong_put_signal_nbi(shmem_ctx_t ctx, long long *dest, const long long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_schar_put_signal_nbi(shmem_ctx_t ctx, signed char *dest, const signed char *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_uchar_put_signal_nbi(shmem_ctx_t ctx, unsigned char *dest, const unsigned char *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_ushort_put_signal_nbi(shmem_ctx_t ctx, unsigned short *dest, const unsigned short *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_uint_put_signal_nbi(shmem_ctx_t ctx, unsigned int *dest, const unsigned int *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_ulong_put_signal_nbi(shmem_ctx_t ctx, unsigned long *dest, const unsigned long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_ulonglong_put_signal_nbi(shmem_ctx_t ctx, unsigned long long *dest, const unsigned long long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_longdouble_put_signal_nbi(shmem_ctx_t ctx, long double *dest, const long double *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_int8_put_signal_nbi(shmem_ctx_t ctx, int8_t *dest, const int8_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_int16_put_signal_nbi(shmem_ctx_t ctx, int16_t *dest, const int16_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_int32_put_signal_nbi(shmem_ctx_t ctx, int32_t *dest, const int32_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_int64_put_signal_nbi(shmem_ctx_t ctx, int64_t *dest, const int64_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_uint8_put_signal_nbi(shmem_ctx_t ctx, uint8_t *dest, const uint8_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_uint16_put_signal_nbi(shmem_ctx_t ctx, uint16_t *dest, const uint16_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_uint32_put_signal_nbi(shmem_ctx_t ctx, uint32_t *dest, const uint32_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_uint64_put_signal_nbi(shmem_ctx_t ctx, uint64_t *dest, const uint64_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_size_put_signal_nbi(shmem_ctx_t ctx, size_t *dest, const size_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ctx_ptrdiff_put_signal_nbi(shmem_ctx_t ctx, ptrdiff_t *dest, const ptrdiff_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+
+OSHMEM_DECLSPEC  void shmem_char_put_signal_nbi(char *dest, const char *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_short_put_signal_nbi(short *dest, const short *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_int_put_signal_nbi(int *dest, const int *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_long_put_signal_nbi(long *dest, const long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_float_put_signal_nbi(float *dest, const float *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_double_put_signal_nbi(double *dest, const double *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_longlong_put_signal_nbi(long long *dest, const long long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_schar_put_signal_nbi(signed char *dest, const signed char *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_uchar_put_signal_nbi(unsigned char *dest, const unsigned char *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ushort_put_signal_nbi(unsigned short *dest, const unsigned short *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_uint_put_signal_nbi(unsigned int *dest, const unsigned int *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ulong_put_signal_nbi(unsigned long *dest, const unsigned long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ulonglong_put_signal_nbi(unsigned long long *dest, const unsigned long long *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_longdouble_put_signal_nbi(long double *dest, const long double *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_int8_put_signal_nbi(int8_t *dest, const int8_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_int16_put_signal_nbi(int16_t *dest, const int16_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_int32_put_signal_nbi(int32_t *dest, const int32_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_int64_put_signal_nbi(int64_t *dest, const int64_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_uint8_put_signal_nbi(uint8_t *dest, const uint8_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_uint16_put_signal_nbi(uint16_t *dest, const uint16_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_uint32_put_signal_nbi(uint32_t *dest, const uint32_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_uint64_put_signal_nbi(uint64_t *dest, const uint64_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_size_put_signal_nbi(size_t *dest, const size_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC  void shmem_ptrdiff_put_signal_nbi(ptrdiff_t *dest, const ptrdiff_t *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+
+#if OSHMEM_HAVE_C11
+#define shmem_put_signal_nbi(...)                                                \
+    _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                                 \
+            shmem_ctx_t:  _Generic((__OSHMEM_VAR_ARG2(__VA_ARGS__)),             \
+                char*:       shmem_ctx_char_put_signal_nbi,                      \
+                short*:      shmem_ctx_short_put_signal_nbi,                     \
+                int*:        shmem_ctx_int_put_signal_nbi,                       \
+                long*:       shmem_ctx_long_put_signal_nbi,                      \
+                long long*:  shmem_ctx_longlong_put_signal_nbi,                  \
+                signed char*:        shmem_ctx_schar_put_signal_nbi,             \
+                unsigned char*:      shmem_ctx_uchar_put_signal_nbi,             \
+                unsigned short*:     shmem_ctx_ushort_put_signal_nbi,            \
+                unsigned int*:       shmem_ctx_uint_put_signal_nbi,              \
+                unsigned long*:      shmem_ctx_ulong_put_signal_nbi,             \
+                unsigned long long*: shmem_ctx_ulonglong_put_signal_nbi,         \
+                float*:       shmem_ctx_float_put_signal_nbi,                    \
+                double*:      shmem_ctx_double_put_signal_nbi,                   \
+                long double*: shmem_ctx_longdouble_put_signal_nbi,               \
+                default:      __oshmem_datatype_ignore),                         \
+            char*:       shmem_char_put_signal_nbi,                              \
+            short*:      shmem_short_put_signal_nbi,                             \
+            int*:        shmem_int_put_signal_nbi,                               \
+            long*:       shmem_long_put_signal_nbi,                              \
+            long long*:  shmem_longlong_put_signal_nbi,                          \
+            signed char*:        shmem_schar_put_signal_nbi,                     \
+            unsigned char*:      shmem_uchar_put_signal_nbi,                     \
+            unsigned short*:     shmem_ushort_put_signal_nbi,                    \
+            unsigned int*:       shmem_uint_put_signal_nbi,                      \
+            unsigned long*:      shmem_ulong_put_signal_nbi,                     \
+            unsigned long long*: shmem_ulonglong_put_signal_nbi,                 \
+            float*:       shmem_float_put_signal_nbi,                            \
+            double*:      shmem_double_put_signal_nbi,                           \
+            long double*: shmem_longdouble_put_signal_nbi)(__VA_ARGS__)
+#endif
+
+OSHMEM_DECLSPEC void shmem_put8_signal_nbi(void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void shmem_put16_signal_nbi(void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void shmem_put32_signal_nbi(void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void shmem_put64_signal_nbi(void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void shmem_put128_signal_nbi(void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+
+OSHMEM_DECLSPEC void shmem_ctx_put8_signal_nbi(shmem_ctx_t ctx, void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_put16_signal_nbi(shmem_ctx_t ctx, void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_put32_signal_nbi(shmem_ctx_t ctx, void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_put64_signal_nbi(shmem_ctx_t ctx, void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_put128_signal_nbi(shmem_ctx_t ctx, void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+
+OSHMEM_DECLSPEC void shmem_putmem_signal_nbi(void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_putmem_signal_nbi(shmem_ctx_t ctx, void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe);
+
+
+OSHMEM_DECLSPEC uint64_t shmem_signal_fetch(const uint64_t *sig_addr);
+
+
 /*
  * Elemental get routines
  */
@@ -996,6 +1780,12 @@ OSHMEM_DECLSPEC unsigned long shmem_ctx_ulong_atomic_swap(shmem_ctx_t ctx, unsig
 OSHMEM_DECLSPEC unsigned long long shmem_ctx_ulonglong_atomic_swap(shmem_ctx_t ctx, unsigned long long *target, unsigned long long value, int pe);
 OSHMEM_DECLSPEC float shmem_ctx_float_atomic_swap(shmem_ctx_t ctx, float *target, float value, int pe);
 OSHMEM_DECLSPEC double shmem_ctx_double_atomic_swap(shmem_ctx_t ctx, double *target, double value, int pe);
+OSHMEM_DECLSPEC int32_t shmem_ctx_int32_atomic_swap(shmem_ctx_t ctx, int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC int64_t shmem_ctx_int64_atomic_swap(shmem_ctx_t ctx, int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC uint32_t shmem_ctx_uint32_atomic_swap(shmem_ctx_t ctx, uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC uint64_t shmem_ctx_uint64_atomic_swap(shmem_ctx_t ctx, uint64_t *target, uint64_t value, int pe);
+OSHMEM_DECLSPEC size_t shmem_ctx_size_atomic_swap(shmem_ctx_t ctx, size_t *target, size_t value, int pe);
+OSHMEM_DECLSPEC ptrdiff_t shmem_ctx_ptrdiff_atomic_swap(shmem_ctx_t ctx, ptrdiff_t *target, ptrdiff_t value, int pe);
 
 OSHMEM_DECLSPEC int shmem_int_atomic_swap(int *target, int value, int pe);
 OSHMEM_DECLSPEC long shmem_long_atomic_swap(long *target, long value, int pe);
@@ -1005,6 +1795,12 @@ OSHMEM_DECLSPEC unsigned long shmem_ulong_atomic_swap(unsigned long *target, uns
 OSHMEM_DECLSPEC unsigned long long shmem_ulonglong_atomic_swap(unsigned long long *target, unsigned long long value, int pe);
 OSHMEM_DECLSPEC float shmem_float_atomic_swap(float *target, float value, int pe);
 OSHMEM_DECLSPEC double shmem_double_atomic_swap(double *target, double value, int pe);
+OSHMEM_DECLSPEC int32_t shmem_int32_atomic_swap(int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC int64_t shmem_int64_atomic_swap(int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC uint32_t shmem_uint32_atomic_swap(uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC uint64_t shmem_uint64_atomic_swap(uint64_t *target, uint64_t value, int pe);
+OSHMEM_DECLSPEC size_t shmem_size_atomic_swap(size_t *target, size_t value, int pe);
+OSHMEM_DECLSPEC ptrdiff_t shmem_ptrdiff_atomic_swap(ptrdiff_t *target, ptrdiff_t value, int pe);
 #if OSHMEM_HAVE_C11
 #define shmem_atomic_swap(...)                                       \
     _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                     \
@@ -1052,6 +1848,12 @@ OSHMEM_DECLSPEC void shmem_ctx_ulong_atomic_set(shmem_ctx_t ctx, unsigned long *
 OSHMEM_DECLSPEC void shmem_ctx_ulonglong_atomic_set(shmem_ctx_t ctx, unsigned long long *target, unsigned long long value, int pe);
 OSHMEM_DECLSPEC void shmem_ctx_float_atomic_set(shmem_ctx_t ctx, float *target, float value, int pe);
 OSHMEM_DECLSPEC void shmem_ctx_double_atomic_set(shmem_ctx_t ctx, double *target, double value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_int32_atomic_set(shmem_ctx_t ctx, int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_int64_atomic_set(shmem_ctx_t ctx, int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_uint32_atomic_set(shmem_ctx_t ctx, uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_uint64_atomic_set(shmem_ctx_t ctx, uint64_t *target, uint64_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_size_atomic_set(shmem_ctx_t ctx, size_t *target, size_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_ptrdiff_atomic_set(shmem_ctx_t ctx, ptrdiff_t *target, ptrdiff_t value, int pe);
 
 OSHMEM_DECLSPEC void shmem_int_atomic_set(int *target, int value, int pe);
 OSHMEM_DECLSPEC void shmem_long_atomic_set(long *target, long value, int pe);
@@ -1061,6 +1863,13 @@ OSHMEM_DECLSPEC void shmem_ulong_atomic_set(unsigned long *target, unsigned long
 OSHMEM_DECLSPEC void shmem_ulonglong_atomic_set(unsigned long long *target, unsigned long long value, int pe);
 OSHMEM_DECLSPEC void shmem_float_atomic_set(float *target, float value, int pe);
 OSHMEM_DECLSPEC void shmem_double_atomic_set(double *target, double value, int pe);
+OSHMEM_DECLSPEC void shmem_int32_atomic_set(int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_int64_atomic_set(int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC void shmem_uint32_atomic_set(uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_uint64_atomic_set(uint64_t *target, uint64_t value, int pe);
+OSHMEM_DECLSPEC void shmem_size_atomic_set(size_t *target, size_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ptrdiff_atomic_set(ptrdiff_t *target, ptrdiff_t value, int pe);
+
 #if OSHMEM_HAVE_C11
 #define shmem_atomic_set(...)                                       \
     _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                    \
@@ -1106,6 +1915,12 @@ OSHMEM_DECLSPEC long long shmem_ctx_longlong_atomic_compare_swap(shmem_ctx_t ctx
 OSHMEM_DECLSPEC unsigned int shmem_ctx_uint_atomic_compare_swap(shmem_ctx_t ctx, unsigned int *target, unsigned int cond, unsigned int value, int pe);
 OSHMEM_DECLSPEC unsigned long shmem_ctx_ulong_atomic_compare_swap(shmem_ctx_t ctx, unsigned long *target, unsigned long cond, unsigned long value, int pe);
 OSHMEM_DECLSPEC unsigned long long shmem_ctx_ulonglong_atomic_compare_swap(shmem_ctx_t ctx, unsigned long long *target, unsigned long long cond, unsigned long long value, int pe);
+OSHMEM_DECLSPEC int32_t shmem_ctx_int32_atomic_compare_swap(shmem_ctx_t ctx, int32_t *target, int32_t cond, int32_t value, int pe);
+OSHMEM_DECLSPEC int64_t shmem_ctx_int64_atomic_compare_swap(shmem_ctx_t ctx, int64_t *target, int64_t cond, int64_t value, int pe);
+OSHMEM_DECLSPEC uint32_t shmem_ctx_uint32_atomic_compare_swap(shmem_ctx_t ctx, uint32_t *target, uint32_t cond, uint32_t value, int pe);
+OSHMEM_DECLSPEC uint64_t shmem_ctx_uint64_atomic_compare_swap(shmem_ctx_t ctx, uint64_t *target, uint64_t cond, uint64_t value, int pe);
+OSHMEM_DECLSPEC size_t shmem_ctx_size_atomic_compare_swap(shmem_ctx_t ctx, size_t *target, size_t cond, size_t value, int pe);
+OSHMEM_DECLSPEC ptrdiff_t shmem_ctx_ptrdiff_atomic_compare_swap(shmem_ctx_t ctx, ptrdiff_t *target, ptrdiff_t cond, ptrdiff_t value, int pe);
 
 OSHMEM_DECLSPEC int shmem_int_atomic_compare_swap(int *target, int cond, int value, int pe);
 OSHMEM_DECLSPEC long shmem_long_atomic_compare_swap(long *target, long cond, long value, int pe);
@@ -1113,6 +1928,12 @@ OSHMEM_DECLSPEC long long shmem_longlong_atomic_compare_swap(long long *target,
 OSHMEM_DECLSPEC unsigned int shmem_uint_atomic_compare_swap(unsigned int *target, unsigned int cond, unsigned int value, int pe);
 OSHMEM_DECLSPEC unsigned long shmem_ulong_atomic_compare_swap(unsigned long *target, unsigned long cond, unsigned long value, int pe);
 OSHMEM_DECLSPEC unsigned long long shmem_ulonglong_atomic_compare_swap(unsigned long long *target, unsigned long long cond, unsigned long long value, int pe);
+OSHMEM_DECLSPEC int32_t shmem_int32_atomic_compare_swap(int32_t *target, int32_t cond, int32_t value, int pe);
+OSHMEM_DECLSPEC int64_t shmem_int64_atomic_compare_swap(int64_t *target, int64_t cond, int64_t value, int pe);
+OSHMEM_DECLSPEC uint32_t shmem_uint32_atomic_compare_swap(uint32_t *target, uint32_t cond, uint32_t value, int pe);
+OSHMEM_DECLSPEC uint64_t shmem_uint64_atomic_compare_swap(uint64_t *target, uint64_t cond, uint64_t value, int pe);
+OSHMEM_DECLSPEC size_t shmem_size_atomic_compare_swap(size_t *target, size_t cond, size_t value, int pe);
+OSHMEM_DECLSPEC ptrdiff_t shmem_ptrdiff_atomic_compare_swap(ptrdiff_t *target, ptrdiff_t cond, ptrdiff_t value, int pe);
 
 #if OSHMEM_HAVE_C11
 #define shmem_atomic_compare_swap(...)                                \
@@ -1152,6 +1973,12 @@ OSHMEM_DECLSPEC long long shmem_ctx_longlong_atomic_fetch_add(shmem_ctx_t ctx, l
 OSHMEM_DECLSPEC unsigned int shmem_ctx_uint_atomic_fetch_add(shmem_ctx_t ctx, unsigned int *target, unsigned int value, int pe);
 OSHMEM_DECLSPEC unsigned long shmem_ctx_ulong_atomic_fetch_add(shmem_ctx_t ctx, unsigned long *target, unsigned long value, int pe);
 OSHMEM_DECLSPEC unsigned long long shmem_ctx_ulonglong_atomic_fetch_add(shmem_ctx_t ctx, unsigned long long *target, unsigned long long value, int pe);
+OSHMEM_DECLSPEC int32_t shmem_ctx_int32_atomic_fetch_add(shmem_ctx_t ctx, int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC int64_t shmem_ctx_int64_atomic_fetch_add(shmem_ctx_t ctx, int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC uint32_t shmem_ctx_uint32_atomic_fetch_add(shmem_ctx_t ctx, uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC uint64_t shmem_ctx_uint64_atomic_fetch_add(shmem_ctx_t ctx, uint64_t *target, uint64_t value, int pe);
+OSHMEM_DECLSPEC size_t shmem_ctx_size_atomic_fetch_add(shmem_ctx_t ctx, size_t *target, size_t value, int pe);
+OSHMEM_DECLSPEC ptrdiff_t shmem_ctx_ptrdiff_atomic_fetch_add(shmem_ctx_t ctx, ptrdiff_t *target, ptrdiff_t value, int pe);
 
 OSHMEM_DECLSPEC int shmem_int_atomic_fetch_add(int *target, int value, int pe);
 OSHMEM_DECLSPEC long shmem_long_atomic_fetch_add(long *target, long value, int pe);
@@ -1159,6 +1986,12 @@ OSHMEM_DECLSPEC long long shmem_longlong_atomic_fetch_add(long long *target, lon
 OSHMEM_DECLSPEC unsigned int shmem_uint_atomic_fetch_add(unsigned int *target, unsigned int value, int pe);
 OSHMEM_DECLSPEC unsigned long shmem_ulong_atomic_fetch_add(unsigned long *target, unsigned long value, int pe);
 OSHMEM_DECLSPEC unsigned long long shmem_ulonglong_atomic_fetch_add(unsigned long long *target, unsigned long long value, int pe);
+OSHMEM_DECLSPEC int32_t shmem_int32_atomic_fetch_add(int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC int64_t shmem_int64_atomic_fetch_add(int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC uint32_t shmem_uint32_atomic_fetch_add(uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC uint64_t shmem_uint64_atomic_fetch_add(uint64_t *target, uint64_t value, int pe);
+OSHMEM_DECLSPEC size_t shmem_size_atomic_fetch_add(size_t *target, size_t value, int pe);
+OSHMEM_DECLSPEC ptrdiff_t shmem_ptrdiff_atomic_fetch_add(ptrdiff_t *target, ptrdiff_t value, int pe);
 #if OSHMEM_HAVE_C11
 #define shmem_atomic_fetch_add(...)                                        \
     _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                           \
@@ -1321,6 +2154,12 @@ OSHMEM_DECLSPEC unsigned long shmem_ctx_ulong_atomic_fetch(shmem_ctx_t ctx, cons
 OSHMEM_DECLSPEC unsigned long long shmem_ctx_ulonglong_atomic_fetch(shmem_ctx_t ctx, const unsigned long long *target, int pe);
 OSHMEM_DECLSPEC float shmem_ctx_float_atomic_fetch(shmem_ctx_t ctx, const float *target, int pe);
 OSHMEM_DECLSPEC double shmem_ctx_double_atomic_fetch(shmem_ctx_t ctx, const double *target, int pe);
+OSHMEM_DECLSPEC int32_t shmem_ctx_int32_atomic_fetch(shmem_ctx_t ctx, const int32_t *target, int pe);
+OSHMEM_DECLSPEC int64_t shmem_ctx_int64_atomic_fetch(shmem_ctx_t ctx, const int64_t *target, int pe);
+OSHMEM_DECLSPEC uint32_t shmem_ctx_uint32_atomic_fetch(shmem_ctx_t ctx, const uint32_t *target, int pe);
+OSHMEM_DECLSPEC uint64_t shmem_ctx_uint64_atomic_fetch(shmem_ctx_t ctx, const uint64_t *target, int pe);
+OSHMEM_DECLSPEC size_t shmem_ctx_size_atomic_fetch(shmem_ctx_t ctx, const size_t *target, int pe);
+OSHMEM_DECLSPEC ptrdiff_t shmem_ctx_ptrdiff_atomic_fetch(shmem_ctx_t ctx, const ptrdiff_t *target, int pe);
 
 OSHMEM_DECLSPEC int shmem_int_atomic_fetch(const int *target, int pe);
 OSHMEM_DECLSPEC long shmem_long_atomic_fetch(const long *target, int pe);
@@ -1330,6 +2169,12 @@ OSHMEM_DECLSPEC unsigned long shmem_ulong_atomic_fetch(const unsigned long *targ
 OSHMEM_DECLSPEC unsigned long long shmem_ulonglong_atomic_fetch(const unsigned long long *target, int pe);
 OSHMEM_DECLSPEC float shmem_float_atomic_fetch(const float *target, int pe);
 OSHMEM_DECLSPEC double shmem_double_atomic_fetch(const double *target, int pe);
+OSHMEM_DECLSPEC int32_t shmem_int32_atomic_fetch(const int32_t *target, int pe);
+OSHMEM_DECLSPEC int64_t shmem_int64_atomic_fetch(const int64_t *target, int pe);
+OSHMEM_DECLSPEC uint32_t shmem_uint32_atomic_fetch(const uint32_t *target, int pe);
+OSHMEM_DECLSPEC uint64_t shmem_uint64_atomic_fetch(const uint64_t *target, int pe);
+OSHMEM_DECLSPEC size_t shmem_size_atomic_fetch(const size_t *target, int pe);
+OSHMEM_DECLSPEC ptrdiff_t shmem_ptrdiff_atomic_fetch(const ptrdiff_t *target, int pe);
 #if OSHMEM_HAVE_C11
 #define shmem_atomic_fetch(...)                                      \
     _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                     \
@@ -1375,6 +2220,12 @@ OSHMEM_DECLSPEC long long shmem_ctx_longlong_atomic_fetch_inc(shmem_ctx_t ctx, l
 OSHMEM_DECLSPEC unsigned int shmem_ctx_uint_atomic_fetch_inc(shmem_ctx_t ctx, unsigned int *target, int pe);
 OSHMEM_DECLSPEC unsigned long shmem_ctx_ulong_atomic_fetch_inc(shmem_ctx_t ctx, unsigned long *target, int pe);
 OSHMEM_DECLSPEC unsigned long long shmem_ctx_ulonglong_atomic_fetch_inc(shmem_ctx_t ctx, unsigned long long *target, int pe);
+OSHMEM_DECLSPEC int32_t shmem_ctx_int32_atomic_fetch_inc(shmem_ctx_t ctx, int32_t *target, int pe);
+OSHMEM_DECLSPEC int64_t shmem_ctx_int64_atomic_fetch_inc(shmem_ctx_t ctx, int64_t *target, int pe);
+OSHMEM_DECLSPEC uint32_t shmem_ctx_uint32_atomic_fetch_inc(shmem_ctx_t ctx, uint32_t *target, int pe);
+OSHMEM_DECLSPEC uint64_t shmem_ctx_uint64_atomic_fetch_inc(shmem_ctx_t ctx, uint64_t *target, int pe);
+OSHMEM_DECLSPEC size_t shmem_ctx_size_atomic_fetch_inc(shmem_ctx_t ctx, size_t *target, int pe);
+OSHMEM_DECLSPEC ptrdiff_t shmem_ctx_ptrdiff_atomic_fetch_inc(shmem_ctx_t ctx, ptrdiff_t *target, int pe);
 
 OSHMEM_DECLSPEC int shmem_int_atomic_fetch_inc(int *target, int pe);
 OSHMEM_DECLSPEC long shmem_long_atomic_fetch_inc(long *target, int pe);
@@ -1382,6 +2233,13 @@ OSHMEM_DECLSPEC long long shmem_longlong_atomic_fetch_inc(long long *target, int
 OSHMEM_DECLSPEC unsigned int shmem_uint_atomic_fetch_inc(unsigned int *target, int pe);
 OSHMEM_DECLSPEC unsigned long shmem_ulong_atomic_fetch_inc(unsigned long *target, int pe);
 OSHMEM_DECLSPEC unsigned long long shmem_ulonglong_atomic_fetch_inc(unsigned long long *target, int pe);
+OSHMEM_DECLSPEC int32_t shmem_int32_atomic_fetch_inc(int32_t *target, int pe);
+OSHMEM_DECLSPEC int64_t shmem_int64_atomic_fetch_inc(int64_t *target, int pe);
+OSHMEM_DECLSPEC uint32_t shmem_uint32_atomic_fetch_inc(uint32_t *target, int pe);
+OSHMEM_DECLSPEC uint64_t shmem_uint64_atomic_fetch_inc(uint64_t *target, int pe);
+OSHMEM_DECLSPEC size_t shmem_size_atomic_fetch_inc(size_t *target, int pe);
+OSHMEM_DECLSPEC ptrdiff_t shmem_ptrdiff_atomic_fetch_inc(ptrdiff_t *target, int pe);
+
 #if OSHMEM_HAVE_C11
 #define shmem_atomic_fetch_inc(...)                                 \
     _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                     \
@@ -1419,6 +2277,12 @@ OSHMEM_DECLSPEC void shmem_ctx_longlong_atomic_add(shmem_ctx_t ctx, long long *t
 OSHMEM_DECLSPEC void shmem_ctx_uint_atomic_add(shmem_ctx_t ctx, unsigned int *target, unsigned int value, int pe);
 OSHMEM_DECLSPEC void shmem_ctx_ulong_atomic_add(shmem_ctx_t ctx, unsigned long *target, unsigned long value, int pe);
 OSHMEM_DECLSPEC void shmem_ctx_ulonglong_atomic_add(shmem_ctx_t ctx, unsigned long long *target, unsigned long long value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_int32_atomic_add(shmem_ctx_t ctx, int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_int64_atomic_add(shmem_ctx_t ctx, int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_uint32_atomic_add(shmem_ctx_t ctx, uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_uint64_atomic_add(shmem_ctx_t ctx, uint64_t *target, uint64_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_size_atomic_add(shmem_ctx_t ctx, size_t *target, size_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_ptrdiff_atomic_add(shmem_ctx_t ctx, ptrdiff_t *target, ptrdiff_t value, int pe);
 
 OSHMEM_DECLSPEC void shmem_int_atomic_add(int *target, int value, int pe);
 OSHMEM_DECLSPEC void shmem_long_atomic_add(long *target, long value, int pe);
@@ -1426,6 +2290,12 @@ OSHMEM_DECLSPEC void shmem_longlong_atomic_add(long long *target, long long valu
 OSHMEM_DECLSPEC void shmem_uint_atomic_add(unsigned int *target, unsigned int value, int pe);
 OSHMEM_DECLSPEC void shmem_ulong_atomic_add(unsigned long *target, unsigned long value, int pe);
 OSHMEM_DECLSPEC void shmem_ulonglong_atomic_add(unsigned long long *target, unsigned long long value, int pe);
+OSHMEM_DECLSPEC void shmem_int32_atomic_add(int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_int64_atomic_add(int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC void shmem_uint32_atomic_add(uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_uint64_atomic_add(uint64_t *target, uint64_t value, int pe);
+OSHMEM_DECLSPEC void shmem_size_atomic_add(size_t *target, size_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ptrdiff_atomic_add(ptrdiff_t *target, ptrdiff_t value, int pe);
 #if OSHMEM_HAVE_C11
 #define shmem_atomic_add(...)                                        \
     _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                     \
@@ -1589,6 +2459,12 @@ OSHMEM_DECLSPEC void shmem_ctx_longlong_atomic_inc(shmem_ctx_t ctx, long long *t
 OSHMEM_DECLSPEC void shmem_ctx_uint_atomic_inc(shmem_ctx_t ctx, unsigned int *target, int pe);
 OSHMEM_DECLSPEC void shmem_ctx_ulong_atomic_inc(shmem_ctx_t ctx, unsigned long *target, int pe);
 OSHMEM_DECLSPEC void shmem_ctx_ulonglong_atomic_inc(shmem_ctx_t ctx, unsigned long long *target, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_int32_atomic_inc(shmem_ctx_t ctx, int32_t *target, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_int64_atomic_inc(shmem_ctx_t ctx, int64_t *target, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_uint32_atomic_inc(shmem_ctx_t ctx, uint32_t *target, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_uint64_atomic_inc(shmem_ctx_t ctx, uint64_t *target, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_size_atomic_inc(shmem_ctx_t ctx, size_t *target, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_ptrdiff_atomic_inc(shmem_ctx_t ctx, ptrdiff_t *target, int pe);
 
 OSHMEM_DECLSPEC void shmem_int_atomic_inc(int *target, int pe);
 OSHMEM_DECLSPEC void shmem_long_atomic_inc(long *target, int pe);
@@ -1596,6 +2472,12 @@ OSHMEM_DECLSPEC void shmem_longlong_atomic_inc(long long *target, int pe);
 OSHMEM_DECLSPEC void shmem_uint_atomic_inc(unsigned int *target, int pe);
 OSHMEM_DECLSPEC void shmem_ulong_atomic_inc(unsigned long *target, int pe);
 OSHMEM_DECLSPEC void shmem_ulonglong_atomic_inc(unsigned long long *target, int pe);
+OSHMEM_DECLSPEC void shmem_int32_atomic_inc(int32_t *target, int pe);
+OSHMEM_DECLSPEC void shmem_int64_atomic_inc(int64_t *target, int pe);
+OSHMEM_DECLSPEC void shmem_uint32_atomic_inc(uint32_t *target, int pe);
+OSHMEM_DECLSPEC void shmem_uint64_atomic_inc(uint64_t *target, int pe);
+OSHMEM_DECLSPEC void shmem_size_atomic_inc(size_t *target, int pe);
+OSHMEM_DECLSPEC void shmem_ptrdiff_atomic_inc(ptrdiff_t *target, int pe);
 #if OSHMEM_HAVE_C11
 #define shmem_atomic_inc(...)                                       \
     _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                    \
@@ -1626,6 +2508,372 @@ OSHMEM_DECLSPEC void shmem_longlong_inc(long long *target, int pe);
             long long*:   shmem_longlong_inc)(dst, pe)
 #endif
 
+
+/*
+ * Nonblocking Atomic Memory Operations
+ */
+
+/* Atomic Nonblocking Fetch */
+OSHMEM_DECLSPEC void shmem_ctx_int_atomic_fetch_nbi(shmem_ctx_t ctx, int *fetch, const int *target, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_long_atomic_fetch_nbi(shmem_ctx_t ctx, long *fetch, const long *target, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_longlong_atomic_fetch_nbi(shmem_ctx_t ctx, long long *fetch, const long long *target, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_uint_atomic_fetch_nbi(shmem_ctx_t ctx, unsigned int *fetch, const unsigned int *target, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_ulong_atomic_fetch_nbi(shmem_ctx_t ctx, unsigned long *fetch,  const unsigned long *target, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_ulonglong_atomic_fetch_nbi(shmem_ctx_t ctx, unsigned long long *fetch, const unsigned long long *target, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_float_atomic_fetch_nbi(shmem_ctx_t ctx, float *fetch, const float *target, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_double_atomic_fetch_nbi(shmem_ctx_t ctx, double *fetch, const double *target, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_int32_atomic_fetch_nbi(shmem_ctx_t ctx, int32_t *fetch, const int32_t *target, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_int64_atomic_fetch_nbi(shmem_ctx_t ctx, int64_t *fetch, const int64_t *target, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_uint32_atomic_fetch_nbi(shmem_ctx_t ctx, uint32_t *fetch, const uint32_t *target, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_uint64_atomic_fetch_nbi(shmem_ctx_t ctx, uint64_t *fetch, const uint64_t *target, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_size_atomic_fetch_nbi(shmem_ctx_t ctx, size_t *fetch, const size_t *target, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_ptrdiff_atomic_fetch_nbi(shmem_ctx_t ctx, ptrdiff_t *fetch, const ptrdiff_t *target, int pe);
+
+OSHMEM_DECLSPEC void shmem_int_atomic_fetch_nbi(int *fetch, const int *target, int pe);
+OSHMEM_DECLSPEC void shmem_long_atomic_fetch_nbi(long *fetch, const long *target, int pe);
+OSHMEM_DECLSPEC void shmem_longlong_atomic_fetch_nbi(long long *fetch, const long long *target, int pe);
+OSHMEM_DECLSPEC void shmem_uint_atomic_fetch_nbi(unsigned int *fetch, const unsigned int *target, int pe);
+OSHMEM_DECLSPEC void shmem_ulong_atomic_fetch_nbi(unsigned long *fetch, const unsigned long *target, int pe);
+OSHMEM_DECLSPEC void shmem_ulonglong_atomic_fetch_nbi(unsigned long long *fetch, const unsigned long long *target, int pe);
+OSHMEM_DECLSPEC void shmem_float_atomic_fetch_nbi(float *fetch, const float *target, int pe);
+OSHMEM_DECLSPEC void shmem_double_atomic_fetch_nbi(double *fetch, const double *target, int pe);
+OSHMEM_DECLSPEC void shmem_int32_atomic_fetch_nbi(int32_t *fetch, const int32_t *target, int pe);
+OSHMEM_DECLSPEC void shmem_int64_atomic_fetch_nbi(int64_t *fetch, const int64_t *target, int pe);
+OSHMEM_DECLSPEC void shmem_uint32_atomic_fetch_nbi(uint32_t *fetch, const uint32_t *target, int pe);
+OSHMEM_DECLSPEC void shmem_uint64_atomic_fetch_nbi(uint64_t *fetch, const uint64_t *target, int pe);
+OSHMEM_DECLSPEC void shmem_size_atomic_fetch_nbi(size_t *fetch, const size_t *target, int pe);
+OSHMEM_DECLSPEC void shmem_ptrdiff_atomic_fetch_nbi(ptrdiff_t *fetch, const ptrdiff_t *target, int pe);
+
+#if OSHMEM_HAVE_C11
+#define shmem_atomic_fetch_nbi(...)                                        \
+    _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                           \
+            shmem_ctx_t:  _Generic((__OSHMEM_VAR_ARG2(__VA_ARGS__)),       \
+                int*:         shmem_ctx_int_atomic_fetch_nbi,              \
+                long*:        shmem_ctx_long_atomic_fetch_nbi,             \
+                long long*:   shmem_ctx_longlong_atomic_fetch_nbi,         \
+                unsigned int*:       shmem_ctx_uint_atomic_fetch_nbi,      \
+                unsigned long*:      shmem_ctx_ulong_atomic_fetch_nbi,     \
+                unsigned long long*: shmem_ctx_ulonglong_atomic_fetch_nbi, \
+                float*:       shmem_ctx_float_atomic_fetch_nbi,            \
+                double*:      shmem_ctx_double_atomic_fetch_nbi,           \
+                default:      __oshmem_datatype_ignore),                   \
+            int*:        shmem_int_atomic_fetch_nbi,                       \
+            long*:       shmem_long_atomic_fetch_nbi,                      \
+            long long*:  shmem_longlong_atomic_fetch_nbi,                  \
+            unsigned int*:       shmem_uint_atomic_fetch_nbi,              \
+            unsigned long*:      shmem_ulong_atomic_fetch_nbi,             \
+            unsigned long long*: shmem_ulonglong_atomic_fetch_nbi,         \
+            float*:       shmem_float_atomic_fetch_nbi,                    \
+            double*:      shmem_double_atomic_fetch_nbi)(__VA_ARGS__)
+#endif
+
+
+/* Atomic Nonblocking Compare and Swap */
+OSHMEM_DECLSPEC void shmem_ctx_int_atomic_compare_swap_nbi(shmem_ctx_t ctx, int *fetch, int *target, int cond, int value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_long_atomic_compare_swap_nbi(shmem_ctx_t ctx, long *fetch, long *target, long cond, long value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_longlong_atomic_compare_swap_nbi(shmem_ctx_t ctx, long long *fetch, long long *target, long long cond, long long value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_uint_atomic_compare_swap_nbi(shmem_ctx_t ctx, unsigned int *fetch, unsigned int *target, unsigned int cond, unsigned int value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_ulong_atomic_compare_swap_nbi(shmem_ctx_t ctx, unsigned long *fetch, unsigned long *target, unsigned long cond, unsigned long value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_ulonglong_atomic_compare_swap_nbi(shmem_ctx_t ctx, unsigned long long *fetch, unsigned long long *target, unsigned long long cond, unsigned long long value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_int32_atomic_compare_swap_nbi(shmem_ctx_t ctx, int32_t *fetch, int32_t *target, int32_t cond, int32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_int64_atomic_compare_swap_nbi(shmem_ctx_t ctx, int64_t *fetch, int64_t *target, int64_t cond, int64_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_uint32_atomic_compare_swap_nbi(shmem_ctx_t ctx, uint32_t *fetch, uint32_t *target, uint32_t cond, uint32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_uint64_atomic_compare_swap_nbi(shmem_ctx_t ctx, uint64_t *fetch, uint64_t *target, uint64_t cond, uint64_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_size_atomic_compare_swap_nbi(shmem_ctx_t ctx, size_t *fetch, size_t *target, size_t cond, size_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_ptrdiff_atomic_compare_swap_nbi(shmem_ctx_t ctx, ptrdiff_t *fetch, ptrdiff_t *target, ptrdiff_t cond, ptrdiff_t value, int pe);
+
+OSHMEM_DECLSPEC void shmem_int_atomic_compare_swap_nbi(int *fetch, int *target, int cond, int value, int pe);
+OSHMEM_DECLSPEC void shmem_long_atomic_compare_swap_nbi(long *fetch, long *target, long cond, long value, int pe);
+OSHMEM_DECLSPEC void shmem_longlong_atomic_compare_swap_nbi(long long *fetch, long long *target, long long cond, long long value, int pe);
+OSHMEM_DECLSPEC void shmem_uint_atomic_compare_swap_nbi(unsigned int *fetch, unsigned int *target, unsigned int cond, unsigned int value, int pe);
+OSHMEM_DECLSPEC void shmem_ulong_atomic_compare_swap_nbi(unsigned long *fetch, unsigned long *target, unsigned long cond, unsigned long value, int pe);
+OSHMEM_DECLSPEC void shmem_ulonglong_atomic_compare_swap_nbi(unsigned long long *fetch, unsigned long long *target, unsigned long long cond, unsigned long long value, int pe);
+OSHMEM_DECLSPEC void shmem_int32_atomic_compare_swap_nbi(int32_t *fetch, int32_t *target, int32_t cond, int32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_int64_atomic_compare_swap_nbi(int64_t *fetch, int64_t *target, int64_t cond, int64_t value, int pe);
+OSHMEM_DECLSPEC void shmem_uint32_atomic_compare_swap_nbi(uint32_t *fetch, uint32_t *target, uint32_t cond, uint32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_uint64_atomic_compare_swap_nbi(uint64_t *fetch, uint64_t *target, uint64_t cond, uint64_t value, int pe);
+OSHMEM_DECLSPEC void shmem_size_atomic_compare_swap_nbi(size_t *fetch, size_t *target, size_t cond, size_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ptrdiff_atomic_compare_swap_nbi(ptrdiff_t *fetch, ptrdiff_t *target, ptrdiff_t cond, ptrdiff_t value, int pe);
+
+#if OSHMEM_HAVE_C11
+#define shmem_atomic_compare_swap_nbi(...)                                        \
+    _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                                  \
+            shmem_ctx_t:  _Generic((__OSHMEM_VAR_ARG2(__VA_ARGS__)),              \
+                int*:         shmem_ctx_int_atomic_compare_swap_nbi,              \
+                long*:        shmem_ctx_long_atomic_compare_swap_nbi,             \
+                long long*:   shmem_ctx_longlong_atomic_compare_swap_nbi,         \
+                unsigned int*:       shmem_ctx_uint_atomic_compare_swap_nbi,      \
+                unsigned long*:      shmem_ctx_ulong_atomic_compare_swap_nbi,     \
+                unsigned long long*: shmem_ctx_ulonglong_atomic_compare_swap_nbi, \
+                default:      __oshmem_datatype_ignore),                          \
+            int*:        shmem_int_atomic_compare_swap_nbi,                       \
+            long*:       shmem_long_atomic_compare_swap_nbi,                      \
+            long long*:  shmem_longlong_atomic_compare_swap_nbi,                  \
+            unsigned int*:       shmem_uint_atomic_compare_swap_nbi,              \
+            unsigned long*:      shmem_ulong_atomic_compare_swap_nbi,             \
+            unsigned long long*: shmem_ulonglong_atomic_compare_swap_nbi)(__VA_ARGS__)
+#endif
+
+/* Atomic Nonblocking Swap */
+OSHMEM_DECLSPEC void shmem_ctx_int_atomic_swap_nbi(shmem_ctx_t ctx, int *fetch, int *target, int value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_long_atomic_swap_nbi(shmem_ctx_t ctx, long *fetch, long *target, long value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_longlong_atomic_swap_nbi(shmem_ctx_t ctx, long long *fetch, long long *target, long long value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_uint_atomic_swap_nbi(shmem_ctx_t ctx, unsigned int *fetch, unsigned int *target, unsigned int value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_ulong_atomic_swap_nbi(shmem_ctx_t ctx, unsigned long *fetch, unsigned long *target, unsigned long value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_ulonglong_atomic_swap_nbi(shmem_ctx_t ctx, unsigned long long *fetch, unsigned long long *target, unsigned long long value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_float_atomic_swap_nbi(shmem_ctx_t ctx, float *fetch, float *target, float value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_double_atomic_swap_nbi(shmem_ctx_t ctx, double *fetch, double *target, double value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_int32_atomic_swap_nbi(shmem_ctx_t ctx, int32_t *fetch, int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_int64_atomic_swap_nbi(shmem_ctx_t ctx, int64_t *fetch, int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_uint32_atomic_swap_nbi(shmem_ctx_t ctx, uint32_t *fetch, uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_uint64_atomic_swap_nbi(shmem_ctx_t ctx, uint64_t *fetch, uint64_t *target, uint64_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_size_atomic_swap_nbi(shmem_ctx_t ctx, size_t *fetch, size_t *target, size_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_ptrdiff_atomic_swap_nbi(shmem_ctx_t ctx, ptrdiff_t *fetch, ptrdiff_t *target,  ptrdiff_t value, int pe);
+
+OSHMEM_DECLSPEC void shmem_int_atomic_swap_nbi(int *fetch, int *target, int value, int pe);
+OSHMEM_DECLSPEC void shmem_long_atomic_swap_nbi(long *fetch, long *target, long value, int pe);
+OSHMEM_DECLSPEC void shmem_longlong_atomic_swap_nbi(long long *fetch, long long *target, long long value, int pe);
+OSHMEM_DECLSPEC void shmem_uint_atomic_swap_nbi(unsigned int *fetch, unsigned int *target, unsigned int value, int pe);
+OSHMEM_DECLSPEC void shmem_ulong_atomic_swap_nbi(unsigned long *fetch, unsigned long *target, unsigned long value, int pe);
+OSHMEM_DECLSPEC void shmem_ulonglong_atomic_swap_nbi(unsigned long long *fetch, unsigned long long *target, unsigned long long value, int pe);
+OSHMEM_DECLSPEC void shmem_float_atomic_swap_nbi(float *fetch, float *target, float value, int pe);
+OSHMEM_DECLSPEC void shmem_double_atomic_swap_nbi(double *fetch, double *target, double value, int pe);
+OSHMEM_DECLSPEC void shmem_int32_atomic_swap_nbi(int32_t *fetch, int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_int64_atomic_swap_nbi(int64_t *fetch, int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC void shmem_uint32_atomic_swap_nbi(uint32_t *fetch, uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_uint64_atomic_swap_nbi(uint64_t *fetch, uint64_t *target, uint64_t value, int pe);
+OSHMEM_DECLSPEC void shmem_size_atomic_swap_nbi(size_t *fetch, size_t *target, size_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ptrdiff_atomic_swap_nbi(ptrdiff_t *fetch, ptrdiff_t *target, ptrdiff_t value, int pe);
+
+#if OSHMEM_HAVE_C11
+#define shmem_atomic_swap_nbi(...)                                        \
+    _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                          \
+            shmem_ctx_t:  _Generic((__OSHMEM_VAR_ARG2(__VA_ARGS__)),      \
+                int*:         shmem_ctx_int_atomic_swap_nbi,              \
+                long*:        shmem_ctx_long_atomic_swap_nbi,             \
+                long long*:   shmem_ctx_longlong_atomic_swap_nbi,         \
+                unsigned int*:       shmem_ctx_uint_atomic_swap_nbi,      \
+                unsigned long*:      shmem_ctx_ulong_atomic_swap_nbi,     \
+                unsigned long long*: shmem_ctx_ulonglong_atomic_swap_nbi, \
+                float*:      shmem_ctx_float_atomic_swap_nbi,             \
+                double*:      shmem_ctx_double_atomic_swap_nbi,           \
+                default:      __oshmem_datatype_ignore),                  \
+            int*:        shmem_int_atomic_swap_nbi,                       \
+            long*:       shmem_long_atomic_swap_nbi,                      \
+            long long*:  shmem_longlong_atomic_swap_nbi,                  \
+            unsigned int*:       shmem_uint_atomic_swap_nbi,              \
+            unsigned long*:      shmem_ulong_atomic_swap_nbi,             \
+            unsigned long long*: shmem_ulonglong_atomic_swap_nbi,         \
+            float*:      shmem_float_atomic_swap_nbi,                     \
+            double*:     shmem_double_atomic_swap_nbi)(__VA_ARGS__)
+#endif
+
+
+
+/* Atomic Nonblocking Fetch and Increment */
+OSHMEM_DECLSPEC void shmem_ctx_int_atomic_fetch_inc_nbi(shmem_ctx_t ctx, int *fetch, int *target, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_long_atomic_fetch_inc_nbi(shmem_ctx_t ctx, long *fetch, long *target, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_longlong_atomic_fetch_inc_nbi(shmem_ctx_t ctx, long long *fetch, long long *target, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_uint_atomic_fetch_inc_nbi(shmem_ctx_t ctx, unsigned int *fetch, unsigned int *target, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_ulong_atomic_fetch_inc_nbi(shmem_ctx_t ctx, unsigned long *fetch,  unsigned long *target, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_ulonglong_atomic_fetch_inc_nbi(shmem_ctx_t ctx, unsigned long long *fetch, unsigned long long *target, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_int32_atomic_fetch_inc_nbi(shmem_ctx_t ctx, int32_t *fetch, int32_t *target, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_int64_atomic_fetch_inc_nbi(shmem_ctx_t ctx, int64_t *fetch, int64_t *target, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_uint32_atomic_fetch_inc_nbi(shmem_ctx_t ctx, uint32_t *fetch, uint32_t *target, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_uint64_atomic_fetch_inc_nbi(shmem_ctx_t ctx, uint64_t *fetch, uint64_t *target, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_size_atomic_fetch_inc_nbi(shmem_ctx_t ctx, size_t *fetch, size_t *target, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_ptrdiff_atomic_fetch_inc_nbi(shmem_ctx_t ctx, ptrdiff_t *fetch, ptrdiff_t *target, int pe);
+
+OSHMEM_DECLSPEC void shmem_int_atomic_fetch_inc_nbi(int *fetch, int *target, int pe);
+OSHMEM_DECLSPEC void shmem_long_atomic_fetch_inc_nbi(long *fetch, long *target, int pe);
+OSHMEM_DECLSPEC void shmem_longlong_atomic_fetch_inc_nbi(long long *fetch, long long *target, int pe);
+OSHMEM_DECLSPEC void shmem_uint_atomic_fetch_inc_nbi(unsigned int *fetch, unsigned int *target, int pe);
+OSHMEM_DECLSPEC void shmem_ulong_atomic_fetch_inc_nbi(unsigned long *fetch, unsigned long *target, int pe);
+OSHMEM_DECLSPEC void shmem_ulonglong_atomic_fetch_inc_nbi(unsigned long long *fetch, unsigned long long *target, int pe);
+OSHMEM_DECLSPEC void shmem_int32_atomic_fetch_inc_nbi(int32_t *fetch, int32_t *target, int pe);
+OSHMEM_DECLSPEC void shmem_int64_atomic_fetch_inc_nbi(int64_t *fetch, int64_t *target, int pe);
+OSHMEM_DECLSPEC void shmem_uint32_atomic_fetch_inc_nbi(uint32_t *fetch, uint32_t *target, int pe);
+OSHMEM_DECLSPEC void shmem_uint64_atomic_fetch_inc_nbi(uint64_t *fetch, uint64_t *target, int pe);
+OSHMEM_DECLSPEC void shmem_size_atomic_fetch_inc_nbi(size_t *fetch, size_t *target, int pe);
+OSHMEM_DECLSPEC void shmem_ptrdiff_atomic_fetch_inc_nbi(ptrdiff_t *fetch, ptrdiff_t *target, int pe);
+
+#if OSHMEM_HAVE_C11
+#define shmem_atomic_fetch_inc_nbi(...)                                        \
+    _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                               \
+            shmem_ctx_t:  _Generic((__OSHMEM_VAR_ARG2(__VA_ARGS__)),           \
+                int*:         shmem_ctx_int_atomic_fetch_inc_nbi,              \
+                long*:        shmem_ctx_long_atomic_fetch_inc_nbi,             \
+                long long*:   shmem_ctx_longlong_atomic_fetch_inc_nbi,         \
+                unsigned int*:       shmem_ctx_uint_atomic_fetch_inc_nbi,      \
+                unsigned long*:      shmem_ctx_ulong_atomic_fetch_inc_nbi,     \
+                unsigned long long*: shmem_ctx_ulonglong_atomic_fetch_inc_nbi, \
+                default:      __oshmem_datatype_ignore),                       \
+            int*:        shmem_int_atomic_fetch_inc_nbi,                       \
+            long*:       shmem_long_atomic_fetch_inc_nbi,                      \
+            long long*:  shmem_longlong_atomic_fetch_inc_nbi,                  \
+            unsigned int*:       shmem_uint_atomic_fetch_inc_nbi,              \
+            unsigned long*:      shmem_ulong_atomic_fetch_inc_nbi,             \
+            unsigned long long*: shmem_ulonglong_atomic_fetch_inc_nbi)(__VA_ARGS__)
+#endif
+
+
+/* Atomic Nonblocking Fetch and Add */
+OSHMEM_DECLSPEC void shmem_ctx_int_atomic_fetch_add_nbi(shmem_ctx_t ctx, int *fetch, int *target, int value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_long_atomic_fetch_add_nbi(shmem_ctx_t ctx, long *fetch, long *target, long value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_longlong_atomic_fetch_add_nbi(shmem_ctx_t ctx, long long *fetch, long long *target, long long value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_uint_atomic_fetch_add_nbi(shmem_ctx_t ctx, unsigned int *fetch, unsigned int *target, unsigned int value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_ulong_atomic_fetch_add_nbi(shmem_ctx_t ctx, unsigned long *fetch,  unsigned long *target, unsigned long value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_ulonglong_atomic_fetch_add_nbi(shmem_ctx_t ctx, unsigned long long *fetch, unsigned long long *target, unsigned long long value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_int32_atomic_fetch_add_nbi(shmem_ctx_t ctx, int32_t *fetch, int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_int64_atomic_fetch_add_nbi(shmem_ctx_t ctx, int64_t *fetch, int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_uint32_atomic_fetch_add_nbi(shmem_ctx_t ctx, uint32_t *fetch, uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_uint64_atomic_fetch_add_nbi(shmem_ctx_t ctx, uint64_t *fetch, uint64_t *target, uint64_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_size_atomic_fetch_add_nbi(shmem_ctx_t ctx, size_t *fetch, size_t *target, size_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_ptrdiff_atomic_fetch_add_nbi(shmem_ctx_t ctx, ptrdiff_t *fetch, ptrdiff_t *target, ptrdiff_t value, int pe);
+
+OSHMEM_DECLSPEC void shmem_int_atomic_fetch_add_nbi(int *fetch, int *target, int value, int pe);
+OSHMEM_DECLSPEC void shmem_long_atomic_fetch_add_nbi(long *fetch, long *target, long value, int pe);
+OSHMEM_DECLSPEC void shmem_longlong_atomic_fetch_add_nbi(long long *fetch, long long *target, long long value, int pe);
+OSHMEM_DECLSPEC void shmem_uint_atomic_fetch_add_nbi(unsigned int *fetch, unsigned int *target, unsigned int value, int pe);
+OSHMEM_DECLSPEC void shmem_ulong_atomic_fetch_add_nbi(unsigned long *fetch, unsigned long *target, unsigned long value, int pe);
+OSHMEM_DECLSPEC void shmem_ulonglong_atomic_fetch_add_nbi(unsigned long long *fetch, unsigned long long *target, unsigned long long value, int pe);
+OSHMEM_DECLSPEC void shmem_int32_atomic_fetch_add_nbi(int32_t *fetch, int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_int64_atomic_fetch_add_nbi(int64_t *fetch, int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC void shmem_uint32_atomic_fetch_add_nbi(uint32_t *fetch, uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_uint64_atomic_fetch_add_nbi(uint64_t *fetch, uint64_t *target, uint64_t value, int pe);
+OSHMEM_DECLSPEC void shmem_size_atomic_fetch_add_nbi(size_t *fetch, size_t *target, size_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ptrdiff_atomic_fetch_add_nbi(ptrdiff_t *fetch, ptrdiff_t *target, ptrdiff_t value, int pe);
+
+#if OSHMEM_HAVE_C11
+#define shmem_atomic_fetch_add_nbi(...)                                        \
+    _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                               \
+            shmem_ctx_t:  _Generic((__OSHMEM_VAR_ARG2(__VA_ARGS__)),           \
+                int*:         shmem_ctx_int_atomic_fetch_add_nbi,              \
+                long*:        shmem_ctx_long_atomic_fetch_add_nbi,             \
+                long long*:   shmem_ctx_longlong_atomic_fetch_add_nbi,         \
+                unsigned int*:       shmem_ctx_uint_atomic_fetch_add_nbi,      \
+                unsigned long*:      shmem_ctx_ulong_atomic_fetch_add_nbi,     \
+                unsigned long long*: shmem_ctx_ulonglong_atomic_fetch_add_nbi, \
+                default:      __oshmem_datatype_ignore),                       \
+            int*:        shmem_int_atomic_fetch_add_nbi,                       \
+            long*:       shmem_long_atomic_fetch_add_nbi,                      \
+            long long*:  shmem_longlong_atomic_fetch_add_nbi,                  \
+            unsigned int*:       shmem_uint_atomic_fetch_add_nbi,              \
+            unsigned long*:      shmem_ulong_atomic_fetch_add_nbi,             \
+            unsigned long long*: shmem_ulonglong_atomic_fetch_add_nbi)(__VA_ARGS__)
+#endif
+
+
+/* Atomic Nonblocking Fetch and And */
+OSHMEM_DECLSPEC void shmem_ctx_uint_atomic_fetch_and_nbi(shmem_ctx_t ctx, unsigned int *fetch, unsigned int *target, unsigned int value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_ulong_atomic_fetch_and_nbi(shmem_ctx_t ctx, unsigned long *fetch,  unsigned long *target, unsigned long value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_ulonglong_atomic_fetch_and_nbi(shmem_ctx_t ctx, unsigned long long *fetch, unsigned long long *target, unsigned long long value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_int32_atomic_fetch_and_nbi(shmem_ctx_t ctx, int32_t *fetch, int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_int64_atomic_fetch_and_nbi(shmem_ctx_t ctx, int64_t *fetch, int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_uint32_atomic_fetch_and_nbi(shmem_ctx_t ctx, uint32_t *fetch, uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_uint64_atomic_fetch_and_nbi(shmem_ctx_t ctx, uint64_t *fetch, uint64_t *target, uint64_t value, int pe);
+
+OSHMEM_DECLSPEC void shmem_uint_atomic_fetch_and_nbi(unsigned int *fetch, unsigned int *target, unsigned int value, int pe);
+OSHMEM_DECLSPEC void shmem_ulong_atomic_fetch_and_nbi(unsigned long *fetch, unsigned long *target, unsigned long value, int pe);
+OSHMEM_DECLSPEC void shmem_ulonglong_atomic_fetch_and_nbi(unsigned long long *fetch, unsigned long long *target, unsigned long long value, int pe);
+OSHMEM_DECLSPEC void shmem_int32_atomic_fetch_and_nbi(int32_t *fetch, int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_int64_atomic_fetch_and_nbi(int64_t *fetch, int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC void shmem_uint32_atomic_fetch_and_nbi(uint32_t *fetch, uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_uint64_atomic_fetch_and_nbi(uint64_t *fetch, uint64_t *target, uint64_t value, int pe);
+
+#if OSHMEM_HAVE_C11
+#define shmem_atomic_fetch_and_nbi(...)                                        \
+    _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                               \
+            shmem_ctx_t:  _Generic((__OSHMEM_VAR_ARG2(__VA_ARGS__)),           \
+                unsigned int*:       shmem_ctx_uint_atomic_fetch_and_nbi,      \
+                unsigned long*:      shmem_ctx_ulong_atomic_fetch_and_nbi,     \
+                unsigned long long*: shmem_ctx_ulonglong_atomic_fetch_and_nbi, \
+                int32_t*: shmem_ctx_int32_atomic_fetch_and_nbi,                \
+                int64_t*: shmem_ctx_int64_atomic_fetch_and_nbi,                \
+                default:      __oshmem_datatype_ignore),                       \
+            unsigned int*:       shmem_uint_atomic_fetch_and_nbi,              \
+            unsigned long*:      shmem_ulong_atomic_fetch_and_nbi,             \
+            int32_t*: shmem_int32_atomic_fetch_and_nbi,                        \
+            int64_t*: shmem_int64_atomic_fetch_and_nbi,                        \
+            unsigned long long*: shmem_ulonglong_atomic_fetch_and_nbi)(__VA_ARGS__)
+#endif
+
+/* Atomic Nonblocking Fetch and OR */
+OSHMEM_DECLSPEC void shmem_ctx_uint_atomic_fetch_or_nbi(shmem_ctx_t ctx, unsigned int *fetch, unsigned int *target, unsigned int value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_ulong_atomic_fetch_or_nbi(shmem_ctx_t ctx, unsigned long *fetch,  unsigned long *target, unsigned long value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_ulonglong_atomic_fetch_or_nbi(shmem_ctx_t ctx, unsigned long long *fetch, unsigned long long *target, unsigned long long value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_int32_atomic_fetch_or_nbi(shmem_ctx_t ctx, int32_t *fetch, int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_int64_atomic_fetch_or_nbi(shmem_ctx_t ctx, int64_t *fetch, int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_uint32_atomic_fetch_or_nbi(shmem_ctx_t ctx, uint32_t *fetch, uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_uint64_atomic_fetch_or_nbi(shmem_ctx_t ctx, uint64_t *fetch, uint64_t *target, uint64_t value, int pe);
+
+OSHMEM_DECLSPEC void shmem_uint_atomic_fetch_or_nbi(unsigned int *fetch, unsigned int *target, unsigned int value, int pe);
+OSHMEM_DECLSPEC void shmem_ulong_atomic_fetch_or_nbi(unsigned long *fetch, unsigned long *target, unsigned long value, int pe);
+OSHMEM_DECLSPEC void shmem_ulonglong_atomic_fetch_or_nbi(unsigned long long *fetch, unsigned long long *target, unsigned long long value, int pe);
+OSHMEM_DECLSPEC void shmem_int32_atomic_fetch_or_nbi(int32_t *fetch, int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_int64_atomic_fetch_or_nbi(int64_t *fetch, int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC void shmem_uint32_atomic_fetch_or_nbi(uint32_t *fetch, uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_uint64_atomic_fetch_or_nbi(uint64_t *fetch, uint64_t *target, uint64_t value, int pe);
+
+#if OSHMEM_HAVE_C11
+#define shmem_atomic_fetch_or_nbi(...)                                        \
+    _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                              \
+            shmem_ctx_t:  _Generic((__OSHMEM_VAR_ARG2(__VA_ARGS__)),          \
+                unsigned int*:       shmem_ctx_uint_atomic_fetch_or_nbi,      \
+                unsigned long*:      shmem_ctx_ulong_atomic_fetch_or_nbi,     \
+                unsigned long long*: shmem_ctx_ulonglong_atomic_fetch_or_nbi, \
+                int32_t*: shmem_ctx_int32_atomic_fetch_or_nbi,                \
+                int64_t*: shmem_ctx_int64_atomic_fetch_or_nbi,                \
+                default:      __oshmem_datatype_ignore),                      \
+            unsigned int*:       shmem_uint_atomic_fetch_or_nbi,              \
+            unsigned long*:      shmem_ulong_atomic_fetch_or_nbi,             \
+            int32_t*: shmem_int32_atomic_fetch_or_nbi,                        \
+            int64_t*: shmem_int64_atomic_fetch_or_nbi,                        \
+            unsigned long long*: shmem_ulonglong_atomic_fetch_or_nbi)(__VA_ARGS__)
+#endif
+
+
+/* Atomic Nonblocking Fetch and XOR */
+OSHMEM_DECLSPEC void shmem_ctx_uint_atomic_fetch_xor_nbi(shmem_ctx_t ctx, unsigned int *fetch, unsigned int *target, unsigned int value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_ulong_atomic_fetch_xor_nbi(shmem_ctx_t ctx, unsigned long *fetch,  unsigned long *target, unsigned long value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_ulonglong_atomic_fetch_xor_nbi(shmem_ctx_t ctx, unsigned long long *fetch, unsigned long long *target, unsigned long long value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_int32_atomic_fetch_xor_nbi(shmem_ctx_t ctx, int32_t *fetch, int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_int64_atomic_fetch_xor_nbi(shmem_ctx_t ctx, int64_t *fetch, int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_uint32_atomic_fetch_xor_nbi(shmem_ctx_t ctx, uint32_t *fetch, uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_ctx_uint64_atomic_fetch_xor_nbi(shmem_ctx_t ctx, uint64_t *fetch, uint64_t *target, uint64_t value, int pe);
+
+OSHMEM_DECLSPEC void shmem_uint_atomic_fetch_xor_nbi(unsigned int *fetch, unsigned int *target, unsigned int value, int pe);
+OSHMEM_DECLSPEC void shmem_ulong_atomic_fetch_xor_nbi(unsigned long *fetch, unsigned long *target, unsigned long value, int pe);
+OSHMEM_DECLSPEC void shmem_ulonglong_atomic_fetch_xor_nbi(unsigned long long *fetch, unsigned long long *target, unsigned long long value, int pe);
+OSHMEM_DECLSPEC void shmem_int32_atomic_fetch_xor_nbi(int32_t *fetch, int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_int64_atomic_fetch_xor_nbi(int64_t *fetch, int64_t *target, int64_t value, int pe);
+OSHMEM_DECLSPEC void shmem_uint32_atomic_fetch_xor_nbi(uint32_t *fetch, uint32_t *target, uint32_t value, int pe);
+OSHMEM_DECLSPEC void shmem_uint64_atomic_fetch_xor_nbi(uint64_t *fetch, uint64_t *target, uint64_t value, int pe);
+
+#if OSHMEM_HAVE_C11
+#define shmem_atomic_fetch_xor_nbi(...)                                        \
+    _Generic(&*(__OSHMEM_VAR_ARG1(__VA_ARGS__)),                               \
+            shmem_ctx_t:  _Generic((__OSHMEM_VAR_ARG2(__VA_ARGS__)),           \
+                unsigned int*:       shmem_ctx_uint_atomic_fetch_xor_nbi,      \
+                unsigned long*:      shmem_ctx_ulong_atomic_fetch_xor_nbi,     \
+                unsigned long long*: shmem_ctx_ulonglong_atomic_fetch_xor_nbi, \
+                int32_t*: shmem_ctx_int32_atomic_fetch_xor_nbi,                \
+                int64_t*: shmem_ctx_int64_atomic_fetch_xor_nbi,                \
+                default:      __oshmem_datatype_ignore),                       \
+            unsigned int*:       shmem_uint_atomic_fetch_xor_nbi,              \
+            unsigned long*:      shmem_ulong_atomic_fetch_xor_nbi,             \
+            int32_t*: shmem_int32_atomic_fetch_xor_nbi,                        \
+            int64_t*: shmem_int64_atomic_fetch_xor_nbi,                        \
+            unsigned long long*: shmem_ulonglong_atomic_fetch_xor_nbi)(__VA_ARGS__)
+#endif
+
+/*
+ *
+ * Control of profiling
+ *
+ */
+OSHMEM_DECLSPEC void shmem_pcontrol(int level, ...);
+
 /*
  * Lock functions
  */
@@ -1669,6 +2917,170 @@ OSHMEM_DECLSPEC  void shmem_ptrdiff_wait_until(volatile ptrdiff_t *addr, int cmp
         unsigned long long*:   shmem_ulonglong_wait_until)(addr, cmp, value)
 #endif
 
+OSHMEM_DECLSPEC  void shmem_short_wait_until_all(volatile short *ivars, size_t nelems, const int *status, int cmp, short value);
+OSHMEM_DECLSPEC  void shmem_ushort_wait_until_all(volatile unsigned short *ivars, size_t nelems, const int *status, int cmp, unsigned short value);
+OSHMEM_DECLSPEC  void shmem_int_wait_until_all(volatile int *ivars, size_t nelems, const int *status, int cmp, int value);
+OSHMEM_DECLSPEC  void shmem_long_wait_until_all(volatile long *ivars, size_t nelems, const int *status, int cmp, long value);
+OSHMEM_DECLSPEC  void shmem_longlong_wait_until_all(volatile long long *ivars, size_t nelems, const int *status, int cmp, long long value);
+OSHMEM_DECLSPEC  void shmem_uint_wait_until_all(volatile unsigned int *ivars, size_t nelems, const int *status, int cmp, unsigned int value);
+OSHMEM_DECLSPEC  void shmem_ulong_wait_until_all(volatile unsigned long *ivars, size_t nelems, const int *status, int cmp, unsigned long value);
+OSHMEM_DECLSPEC  void shmem_ulonglong_wait_until_all(volatile unsigned long long *ivars, size_t nelems, const int *status, int cmp, unsigned long long value);
+OSHMEM_DECLSPEC  void shmem_int32_wait_until_all(volatile int32_t *ivars, size_t nelems, const int *status, int cmp, int32_t value);
+OSHMEM_DECLSPEC  void shmem_int64_wait_until_all(volatile int64_t *ivars, size_t nelems, const int *status, int cmp, int64_t value);
+OSHMEM_DECLSPEC  void shmem_uint32_wait_until_all(volatile uint32_t *ivars, size_t nelems, const int *status, int cmp, uint32_t value);
+OSHMEM_DECLSPEC  void shmem_uint64_wait_until_all(volatile uint64_t *ivars, size_t nelems, const int *status, int cmp, uint64_t value);
+OSHMEM_DECLSPEC  void shmem_size_wait_until_all(volatile size_t *ivars, size_t nelems, const int *status, int cmp, size_t value);
+OSHMEM_DECLSPEC  void shmem_ptrdiff_wait_until_all(volatile ptrdiff_t *ivars, size_t nelems, const int *status, int cmp, ptrdiff_t value);
+#if OSHMEM_HAVE_C11
+#define shmem_wait_until_all(ivars, nelems, status, cmp, value)          \
+    _Generic(&*(ivars),                                                  \
+        short*:                shmem_short_wait_until_all,               \
+        unsigned short*:       shmem_ushort_wait_until_all,              \
+        int*:                  shmem_int_wait_until_all,                 \
+        long*:                 shmem_long_wait_until_all,                \
+        long long*:            shmem_longlong_wait_until_all,            \
+        unsigned int*:         shmem_uint_wait_until_all,                \
+        unsigned long*:        shmem_ulong_wait_until_all,               \
+        unsigned long long*:   shmem_ulonglong_wait_until_all)(ivars, nelems, status, cmp, value)
+#endif
+
+OSHMEM_DECLSPEC  size_t shmem_short_wait_until_any(volatile short *ivars, size_t nelems, const int *status, int cmp, short value);
+OSHMEM_DECLSPEC  size_t shmem_ushort_wait_until_any(volatile unsigned short *ivars, size_t nelems, const int *status, int cmp, unsigned short value);
+OSHMEM_DECLSPEC  size_t shmem_int_wait_until_any(volatile int *ivars, size_t nelems, const int *status, int cmp, int value);
+OSHMEM_DECLSPEC  size_t shmem_long_wait_until_any(volatile long *ivars, size_t nelems, const int *status, int cmp, long value);
+OSHMEM_DECLSPEC  size_t shmem_longlong_wait_until_any(volatile long long *ivars, size_t nelems, const int *status, int cmp, long long value);
+OSHMEM_DECLSPEC  size_t shmem_uint_wait_until_any(volatile unsigned int *ivars, size_t nelems, const int *status, int cmp, unsigned int value);
+OSHMEM_DECLSPEC  size_t shmem_ulong_wait_until_any(volatile unsigned long *ivars, size_t nelems, const int *status, int cmp, unsigned long value);
+OSHMEM_DECLSPEC  size_t shmem_ulonglong_wait_until_any(volatile unsigned long long *ivars, size_t nelems, const int *status, int cmp, unsigned long long value);
+OSHMEM_DECLSPEC  size_t shmem_int32_wait_until_any(volatile int32_t *ivars, size_t nelems, const int *status, int cmp, int32_t value);
+OSHMEM_DECLSPEC  size_t shmem_int64_wait_until_any(volatile int64_t *ivars, size_t nelems, const int *status, int cmp, int64_t value);
+OSHMEM_DECLSPEC  size_t shmem_uint32_wait_until_any(volatile uint32_t *ivars, size_t nelems, const int *status, int cmp, uint32_t value);
+OSHMEM_DECLSPEC  size_t shmem_uint64_wait_until_any(volatile uint64_t *ivars, size_t nelems, const int *status, int cmp, uint64_t value);
+OSHMEM_DECLSPEC  size_t shmem_size_wait_until_any(volatile size_t *ivars, size_t nelems, const int *status, int cmp, size_t value);
+OSHMEM_DECLSPEC  size_t shmem_ptrdiff_wait_until_any(volatile ptrdiff_t *ivars, size_t nelems, const int *status, int cmp, ptrdiff_t value);
+#if OSHMEM_HAVE_C11
+#define shmem_wait_until_any(ivars, nelems, status, cmp, value)          \
+    _Generic(&*(ivars),                                                  \
+        short*:                shmem_short_wait_until_any,               \
+        unsigned short*:       shmem_ushort_wait_until_any,              \
+        int*:                  shmem_int_wait_until_any,                 \
+        long*:                 shmem_long_wait_until_any,                \
+        long long*:            shmem_longlong_wait_until_any,            \
+        unsigned int*:         shmem_uint_wait_until_any,                \
+        unsigned long*:        shmem_ulong_wait_until_any,               \
+        unsigned long long*:   shmem_ulonglong_wait_until_any)(ivars, nelems, status, cmp, value)
+#endif
+
+OSHMEM_DECLSPEC  size_t shmem_short_wait_until_some(volatile short *ivars, size_t nelems,  size_t *indices, const int *status, int cmp, short value);
+OSHMEM_DECLSPEC  size_t shmem_ushort_wait_until_some(volatile unsigned short *ivars, size_t nelems,  size_t *indices, const int *status, int cmp, unsigned short value);
+OSHMEM_DECLSPEC  size_t shmem_int_wait_until_some(volatile int *ivars, size_t nelems, size_t *indices, const int *status, int cmp, int value);
+OSHMEM_DECLSPEC  size_t shmem_long_wait_until_some(volatile long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, long value);
+OSHMEM_DECLSPEC  size_t shmem_longlong_wait_until_some(volatile long long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, long long value);
+OSHMEM_DECLSPEC  size_t shmem_uint_wait_until_some(volatile unsigned int *ivars, size_t nelems, size_t *indices, const int *status, int cmp, unsigned int value);
+OSHMEM_DECLSPEC  size_t shmem_ulong_wait_until_some(volatile unsigned long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, unsigned long value);
+OSHMEM_DECLSPEC  size_t shmem_ulonglong_wait_until_some(volatile unsigned long long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, unsigned long long value);
+OSHMEM_DECLSPEC  size_t shmem_int32_wait_until_some(volatile int32_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, int32_t value);
+OSHMEM_DECLSPEC  size_t shmem_int64_wait_until_some(volatile int64_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, int64_t value);
+OSHMEM_DECLSPEC  size_t shmem_uint32_wait_until_some(volatile uint32_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, uint32_t value);
+OSHMEM_DECLSPEC  size_t shmem_uint64_wait_until_some(volatile uint64_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, uint64_t value);
+OSHMEM_DECLSPEC  size_t shmem_size_wait_until_some(volatile size_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, size_t value);
+OSHMEM_DECLSPEC  size_t shmem_ptrdiff_wait_until_some(volatile ptrdiff_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, ptrdiff_t value);
+#if OSHMEM_HAVE_C11
+#define shmem_wait_until_some(ivars, nelems, indices, status, cmp, value) \
+    _Generic(&*(ivars),                                                   \
+        short*:                shmem_short_wait_until_some,               \
+        unsigned short*:       shmem_ushort_wait_until_some,              \
+        int*:                  shmem_int_wait_until_some,                 \
+        long*:                 shmem_long_wait_until_some,                \
+        long long*:            shmem_longlong_wait_until_some,            \
+        unsigned int*:         shmem_uint_wait_until_some,                \
+        unsigned long*:        shmem_ulong_wait_until_some,               \
+        unsigned long long*:   shmem_ulonglong_wait_until_some)(ivars, nelems, indices, status, cmp, value)
+#endif
+
+OSHMEM_DECLSPEC  void shmem_short_wait_until_all_vector(volatile short *ivars, size_t nelems, const int *status, int cmp, short *values);
+OSHMEM_DECLSPEC  void shmem_ushort_wait_until_all_vector(volatile unsigned short *ivars, size_t nelems, const int *status, int cmp, unsigned short *values);
+OSHMEM_DECLSPEC  void shmem_int_wait_until_all_vector(volatile int *ivars, size_t nelems, const int *status, int cmp, int *values);
+OSHMEM_DECLSPEC  void shmem_long_wait_until_all_vector(volatile long *ivars, size_t nelems, const int *status, int cmp, long *values);
+OSHMEM_DECLSPEC  void shmem_longlong_wait_until_all_vector(volatile long long *ivars, size_t nelems, const int *status, int cmp, long long *values);
+OSHMEM_DECLSPEC  void shmem_uint_wait_until_all_vector(volatile unsigned int *ivars, size_t nelems, const int *status, int cmp, unsigned int *values);
+OSHMEM_DECLSPEC  void shmem_ulong_wait_until_all_vector(volatile unsigned long *ivars, size_t nelems, const int *status, int cmp, unsigned long *values);
+OSHMEM_DECLSPEC  void shmem_ulonglong_wait_until_all_vector(volatile unsigned long long *ivars, size_t nelems, const int *status, int cmp, unsigned long long *values);
+OSHMEM_DECLSPEC  void shmem_int32_wait_until_all_vector(volatile int32_t *ivars, size_t nelems, const int *status, int cmp, int32_t *values);
+OSHMEM_DECLSPEC  void shmem_int64_wait_until_all_vector(volatile int64_t *ivars, size_t nelems, const int *status, int cmp, int64_t *values);
+OSHMEM_DECLSPEC  void shmem_uint32_wait_until_all_vector(volatile uint32_t *ivars, size_t nelems, const int *status, int cmp, uint32_t *values);
+OSHMEM_DECLSPEC  void shmem_uint64_wait_until_all_vector(volatile uint64_t *ivars, size_t nelems, const int *status, int cmp, uint64_t *values);
+OSHMEM_DECLSPEC  void shmem_size_wait_until_all_vector(volatile size_t *ivars, size_t nelems, const int *status, int cmp, size_t *values);
+OSHMEM_DECLSPEC  void shmem_ptrdiff_wait_until_all_vector(volatile ptrdiff_t *ivars, size_t nelems, const int *status, int cmp, ptrdiff_t *values);
+#if OSHMEM_HAVE_C11
+#define shmem_wait_until_all_vector(ivars, nelems, status, cmp, values)         \
+    _Generic(&*(ivars),                                                         \
+        short*:                shmem_short_wait_until_all_vector,               \
+        unsigned short*:       shmem_ushort_wait_until_all_vector,              \
+        int*:                  shmem_int_wait_until_all_vector,                 \
+        long*:                 shmem_long_wait_until_all_vector,                \
+        long long*:            shmem_longlong_wait_until_all_vector,            \
+        unsigned int*:         shmem_uint_wait_until_all_vector,                \
+        unsigned long*:        shmem_ulong_wait_until_all_vector,               \
+        unsigned long long*:   shmem_ulonglong_wait_until_all_vector)(ivars, nelems, status, cmp, values)
+#endif
+
+
+OSHMEM_DECLSPEC  size_t shmem_short_wait_until_any_vector(volatile short *ivars, size_t nelems, const int *status, int cmp, short *values);
+OSHMEM_DECLSPEC  size_t shmem_ushort_wait_until_any_vector(volatile unsigned short *ivars, size_t nelems, const int *status, int cmp, unsigned short *values);
+OSHMEM_DECLSPEC  size_t shmem_int_wait_until_any_vector(volatile int *ivars, size_t nelems, const int *status, int cmp, int *values);
+OSHMEM_DECLSPEC  size_t shmem_long_wait_until_any_vector(volatile long *ivars, size_t nelems, const int *status, int cmp, long *values);
+OSHMEM_DECLSPEC  size_t shmem_longlong_wait_until_any_vector(volatile long long *ivars, size_t nelems, const int *status, int cmp, long long *values);
+OSHMEM_DECLSPEC  size_t shmem_uint_wait_until_any_vector(volatile unsigned int *ivars, size_t nelems, const int *status, int cmp, unsigned int *values);
+OSHMEM_DECLSPEC  size_t shmem_ulong_wait_until_any_vector(volatile unsigned long *ivars, size_t nelems, const int *status, int cmp, unsigned long *values);
+OSHMEM_DECLSPEC  size_t shmem_ulonglong_wait_until_any_vector(volatile unsigned long long *ivars, size_t nelems, const int *status, int cmp, unsigned long long *values);
+OSHMEM_DECLSPEC  size_t shmem_int32_wait_until_any_vector(volatile int32_t *ivars, size_t nelems, const int *status, int cmp, int32_t *values);
+OSHMEM_DECLSPEC  size_t shmem_int64_wait_until_any_vector(volatile int64_t *ivars, size_t nelems, const int *status, int cmp, int64_t *values);
+OSHMEM_DECLSPEC  size_t shmem_uint32_wait_until_any_vector(volatile uint32_t *ivars, size_t nelems, const int *status, int cmp, uint32_t *values);
+OSHMEM_DECLSPEC  size_t shmem_uint64_wait_until_any_vector(volatile uint64_t *ivars, size_t nelems, const int *status, int cmp, uint64_t *values);
+OSHMEM_DECLSPEC  size_t shmem_size_wait_until_any_vector(volatile size_t *ivars, size_t nelems, const int *status, int cmp, size_t *values);
+OSHMEM_DECLSPEC  size_t shmem_ptrdiff_wait_until_any_vector(volatile ptrdiff_t *ivars, size_t nelems, const int *status, int cmp, ptrdiff_t *values);
+#if OSHMEM_HAVE_C11
+#define shmem_wait_until_any_vector(ivars, nelems, status, cmp, values)         \
+    _Generic(&*(ivars),                                                         \
+        short*:                shmem_short_wait_until_any_vector,               \
+        unsigned short*:       shmem_ushort_wait_until_any_vector,              \
+        int*:                  shmem_int_wait_until_any_vector,                 \
+        long*:                 shmem_long_wait_until_any_vector,                \
+        long long*:            shmem_longlong_wait_until_any_vector,            \
+        unsigned int*:         shmem_uint_wait_until_any_vector,                \
+        unsigned long*:        shmem_ulong_wait_until_any_vector,               \
+        unsigned long long*:   shmem_ulonglong_wait_until_any_vector)(ivars, nelems, status, cmp, values)
+#endif
+
+
+OSHMEM_DECLSPEC  size_t shmem_short_wait_until_some_vector(volatile short *ivars, size_t nelems, size_t *indices, const int *status, int cmp, short *values);
+OSHMEM_DECLSPEC  size_t shmem_ushort_wait_until_some_vector(volatile unsigned short *ivars, size_t nelems, size_t *indices, const int *status , int cmp, unsigned short *values);
+OSHMEM_DECLSPEC  size_t shmem_int_wait_until_some_vector(volatile int *ivars, size_t nelems, size_t *indices, const int *status, int cmp, int *values);
+OSHMEM_DECLSPEC  size_t shmem_long_wait_until_some_vector(volatile long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, long *values);
+OSHMEM_DECLSPEC  size_t shmem_longlong_wait_until_some_vector(volatile long long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, long long *values);
+OSHMEM_DECLSPEC  size_t shmem_uint_wait_until_some_vector(volatile unsigned int *ivars, size_t nelems, size_t *indices, const int *status, int cmp, unsigned int *values);
+OSHMEM_DECLSPEC  size_t shmem_ulong_wait_until_some_vector(volatile unsigned long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, unsigned long *values);
+OSHMEM_DECLSPEC  size_t shmem_ulonglong_wait_until_some_vector(volatile unsigned long long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, unsigned long long *values);
+OSHMEM_DECLSPEC  size_t shmem_int32_wait_until_some_vector(volatile int32_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, int32_t *values);
+OSHMEM_DECLSPEC  size_t shmem_int64_wait_until_some_vector(volatile int64_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, int64_t *values);
+OSHMEM_DECLSPEC  size_t shmem_uint32_wait_until_some_vector(volatile uint32_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, uint32_t *values);
+OSHMEM_DECLSPEC  size_t shmem_uint64_wait_until_some_vector(volatile uint64_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, uint64_t *values);
+OSHMEM_DECLSPEC  size_t shmem_size_wait_until_some_vector(volatile size_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, size_t *values);
+OSHMEM_DECLSPEC  size_t shmem_ptrdiff_wait_until_some_vector(volatile ptrdiff_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, ptrdiff_t *values);
+#if OSHMEM_HAVE_C11
+#define shmem_wait_until_some_vector(ivars, nelems, indices, status, cmp, values)\
+    _Generic(&*(ivars),                                                          \
+        short*:                shmem_short_wait_until_some_vector,               \
+        unsigned short*:       shmem_ushort_wait_until_some_vector,              \
+        int*:                  shmem_int_wait_until_some_vector,                 \
+        long*:                 shmem_long_wait_until_some_vector,                \
+        long long*:            shmem_longlong_wait_until_some_vector,            \
+        unsigned int*:         shmem_uint_wait_until_some_vector,                \
+        unsigned long*:        shmem_ulong_wait_until_some_vector,               \
+        unsigned long long*:   shmem_ulonglong_wait_until_some_vector)(ivars, nelems, indices, status, cmp, values)
+#endif
+
 OSHMEM_DECLSPEC  int shmem_short_test(volatile short *addr, int cmp, short value);
 OSHMEM_DECLSPEC  int shmem_int_test(volatile int *addr, int cmp, int value);
 OSHMEM_DECLSPEC  int shmem_long_test(volatile long *addr, int cmp, long value);
@@ -1696,24 +3108,193 @@ OSHMEM_DECLSPEC  int shmem_ptrdiff_test(volatile ptrdiff_t *addr, int cmp, ptrdi
         unsigned long long*:   shmem_ulonglong_test)(addr, cmp, value)
 #endif
 
+OSHMEM_DECLSPEC  int shmem_short_test_all(volatile short *ivars, size_t nelems, const int *status, int cmp, short value);
+OSHMEM_DECLSPEC  int shmem_ushort_test_all(volatile unsigned short *ivars, size_t nelems, const int *status, int cmp, unsigned short value);
+OSHMEM_DECLSPEC  int shmem_int_test_all(volatile int *ivars, size_t nelems, const int *status, int cmp, int value);
+OSHMEM_DECLSPEC  int shmem_long_test_all(volatile long *ivars, size_t nelems, const int *status, int cmp, long value);
+OSHMEM_DECLSPEC  int shmem_longlong_test_all(volatile long long *ivars, size_t nelems, const int *status, int cmp, long long value);
+OSHMEM_DECLSPEC  int shmem_uint_test_all(volatile unsigned int *ivars, size_t nelems, const int *status, int cmp, unsigned int value);
+OSHMEM_DECLSPEC  int shmem_ulong_test_all(volatile unsigned long *ivars, size_t nelems, const int *status, int cmp, unsigned long value);
+OSHMEM_DECLSPEC  int shmem_ulonglong_test_all(volatile unsigned long long *ivars, size_t nelems, const int *status, int cmp, unsigned long long value);
+OSHMEM_DECLSPEC  int shmem_int32_test_all(volatile int32_t *ivars, size_t nelems, const int *status, int cmp, int32_t value);
+OSHMEM_DECLSPEC  int shmem_int64_test_all(volatile int64_t *ivars, size_t nelems, const int *status, int cmp, int64_t value);
+OSHMEM_DECLSPEC  int shmem_uint32_test_all(volatile uint32_t *ivars, size_t nelems, const int *status, int cmp, uint32_t value);
+OSHMEM_DECLSPEC  int shmem_uint64_test_all(volatile uint64_t *ivars, size_t nelems, const int *status, int cmp, uint64_t value);
+OSHMEM_DECLSPEC  int shmem_size_test_all(volatile size_t *ivars, size_t nelems, const int *status, int cmp, size_t value);
+OSHMEM_DECLSPEC  int shmem_ptrdiff_test_all(volatile ptrdiff_t *ivars, size_t nelems, const int *status, int cmp, ptrdiff_t value);
+#if OSHMEM_HAVE_C11
+#define shmem_test_all(ivars, nelems, status, cmp, value)          \
+    _Generic(&*(ivars),                                            \
+        short*:                shmem_short_test_all,               \
+        unsigned short*:       shmem_ushort_test_all,              \
+        int*:                  shmem_int_test_all,                 \
+        long*:                 shmem_long_test_all,                \
+        long long*:            shmem_longlong_test_all,            \
+        unsigned int*:         shmem_uint_test_all,                \
+        unsigned long*:        shmem_ulong_test_all,               \
+        unsigned long long*:   shmem_ulonglong_test_all)(ivars, nelems, status, cmp, value)
+#endif
+
+OSHMEM_DECLSPEC  size_t shmem_short_test_any(volatile short *ivars, size_t nelems, const int *status, int cmp, short value);
+OSHMEM_DECLSPEC  size_t shmem_ushort_test_any(volatile unsigned short *ivars, size_t nelems, const int *status, int cmp, unsigned short value);
+OSHMEM_DECLSPEC  size_t shmem_int_test_any(volatile int *ivars, size_t nelems, const int *status, int cmp, int value);
+OSHMEM_DECLSPEC  size_t shmem_long_test_any(volatile long *ivars, size_t nelems, const int *status, int cmp, long value);
+OSHMEM_DECLSPEC  size_t shmem_longlong_test_any(volatile long long *ivars, size_t nelems, const int *status, int cmp, long long value);
+OSHMEM_DECLSPEC  size_t shmem_uint_test_any(volatile unsigned int *ivars, size_t nelems, const int *status, int cmp, unsigned int value);
+OSHMEM_DECLSPEC  size_t shmem_ulong_test_any(volatile unsigned long *ivars, size_t nelems, const int *status, int cmp, unsigned long value);
+OSHMEM_DECLSPEC  size_t shmem_ulonglong_test_any(volatile unsigned long long *ivars, size_t nelems, const int *status, int cmp, unsigned long long value);
+OSHMEM_DECLSPEC  size_t shmem_int32_test_any(volatile int32_t *ivars, size_t nelems, const int *status, int cmp, int32_t value);
+OSHMEM_DECLSPEC  size_t shmem_int64_test_any(volatile int64_t *ivars, size_t nelems, const int *status, int cmp, int64_t value);
+OSHMEM_DECLSPEC  size_t shmem_uint32_test_any(volatile uint32_t *ivars, size_t nelems, const int *status, int cmp, uint32_t value);
+OSHMEM_DECLSPEC  size_t shmem_uint64_test_any(volatile uint64_t *ivars, size_t nelems, const int *status, int cmp, uint64_t value);
+OSHMEM_DECLSPEC  size_t shmem_size_test_any(volatile size_t *ivars, size_t nelems, const int *status, int cmp, size_t value);
+OSHMEM_DECLSPEC  size_t shmem_ptrdiff_test_any(volatile ptrdiff_t *ivars, size_t nelems, const int *status, int cmp, ptrdiff_t value);
+#if OSHMEM_HAVE_C11
+#define shmem_test_any(ivars, nelems, status, cmp, value)          \
+    _Generic(&*(ivars),                                            \
+        short*:                shmem_short_test_any,               \
+        unsigned short*:       shmem_ushort_test_any,              \
+        int*:                  shmem_int_test_any,                 \
+        long*:                 shmem_long_test_any,                \
+        long long*:            shmem_longlong_test_any,            \
+        unsigned int*:         shmem_uint_test_any,                \
+        unsigned long*:        shmem_ulong_test_any,               \
+        unsigned long long*:   shmem_ulonglong_test_any)(ivars, nelems, status, cmp, value)
+#endif
+
+OSHMEM_DECLSPEC  size_t shmem_short_test_some(volatile short *ivars, size_t nelems, size_t *indices, const int *status, int cmp, short value) ;
+OSHMEM_DECLSPEC  size_t shmem_ushort_test_some(volatile unsigned short *ivars, size_t nelems, size_t *indices, const int *status, int cmp, unsigned short value);
+OSHMEM_DECLSPEC  size_t shmem_int_test_some(volatile int *ivars, size_t nelems, size_t *indices, const int *status, int cmp, int value);
+OSHMEM_DECLSPEC  size_t shmem_long_test_some(volatile long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, long value);
+OSHMEM_DECLSPEC  size_t shmem_longlong_test_some(volatile long long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, long long value);
+OSHMEM_DECLSPEC  size_t shmem_uint_test_some(volatile unsigned int *ivars, size_t nelems, size_t *indices, const int *status, int cmp, unsigned int value);
+OSHMEM_DECLSPEC  size_t shmem_ulong_test_some(volatile unsigned long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, unsigned long value);
+OSHMEM_DECLSPEC  size_t shmem_ulonglong_test_some(volatile unsigned long long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, unsigned long long value);
+OSHMEM_DECLSPEC  size_t shmem_int32_test_some(volatile int32_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, int32_t value);
+OSHMEM_DECLSPEC  size_t shmem_int64_test_some(volatile int64_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, int64_t value);
+OSHMEM_DECLSPEC  size_t shmem_uint32_test_some(volatile uint32_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, uint32_t value);
+OSHMEM_DECLSPEC  size_t shmem_uint64_test_some(volatile uint64_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, uint64_t value);
+OSHMEM_DECLSPEC  size_t shmem_size_test_some(volatile size_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, size_t value);
+OSHMEM_DECLSPEC  size_t shmem_ptrdiff_test_some(volatile ptrdiff_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, ptrdiff_t value);
+#if OSHMEM_HAVE_C11
+#define shmem_test_some(ivars, nelems, indices, status, cmp, value) \
+    _Generic(&*(ivars),                                             \
+        short*:                shmem_short_test_some,               \
+        unsigned short*:       shmem_ushort_test_some,              \
+        int*:                  shmem_int_test_some,                 \
+        long*:                 shmem_long_test_some,                \
+        long long*:            shmem_longlong_test_some,            \
+        unsigned int*:         shmem_uint_test_some,                \
+        unsigned long*:        shmem_ulong_test_some,               \
+        unsigned long long*:   shmem_ulonglong_test_some)(ivars, nelems, indices, status, cmp, value)
+#endif
+
+OSHMEM_DECLSPEC  int shmem_short_test_all_vector(volatile short *ivars, size_t nelems, const int *status, int cmp, short *values);
+OSHMEM_DECLSPEC  int shmem_ushort_test_all_vector(volatile unsigned short *ivars, size_t nelems, const int *status, int cmp, unsigned short *values);
+OSHMEM_DECLSPEC  int shmem_int_test_all_vector(volatile int *ivars, size_t nelems, const int *status, int cmp, int *values);
+OSHMEM_DECLSPEC  int shmem_long_test_all_vector(volatile long *ivars, size_t nelems, const int *status, int cmp, long *values);
+OSHMEM_DECLSPEC  int shmem_longlong_test_all_vector(volatile long long *ivars, size_t nelems, const int *status, int cmp, long long *values);
+OSHMEM_DECLSPEC  int shmem_uint_test_all_vector(volatile unsigned int *ivars, size_t nelems, const int *status, int cmp, unsigned int *values);
+OSHMEM_DECLSPEC  int shmem_ulong_test_all_vector(volatile unsigned long *ivars, size_t nelems, const int *status, int cmp, unsigned long *values);
+OSHMEM_DECLSPEC  int shmem_ulonglong_test_all_vector(volatile unsigned long long *ivars, size_t nelems, const int *status, int cmp, unsigned long long *values);
+OSHMEM_DECLSPEC  int shmem_int32_test_all_vector(volatile int32_t *ivars, size_t nelems, const int *status, int cmp, int32_t *values);
+OSHMEM_DECLSPEC  int shmem_int64_test_all_vector(volatile int64_t *ivars, size_t nelems, const int *status, int cmp, int64_t *values);
+OSHMEM_DECLSPEC  int shmem_uint32_test_all_vector(volatile uint32_t *ivars, size_t nelems, const int *status, int cmp, uint32_t *values);
+OSHMEM_DECLSPEC  int shmem_uint64_test_all_vector(volatile uint64_t *ivars, size_t nelems, const int *status, int cmp, uint64_t *values);
+OSHMEM_DECLSPEC  int shmem_size_test_all_vector(volatile size_t *ivars, size_t nelems, const int *status, int cmp, size_t *values);
+OSHMEM_DECLSPEC  int shmem_ptrdiff_test_all_vector(volatile ptrdiff_t *ivars, size_t nelems, const int *status, int cmp, ptrdiff_t *values);
+#if OSHMEM_HAVE_C11
+#define shmem_test_all_vector(ivars, nelems, status, cmp, values)         \
+    _Generic(&*(ivars),                                                   \
+        short*:                shmem_short_test_all_vector,               \
+        unsigned short*:       shmem_ushort_test_all_vector,              \
+        int*:                  shmem_int_test_all_vector,                 \
+        long*:                 shmem_long_test_all_vector,                \
+        long long*:            shmem_longlong_test_all_vector,            \
+        unsigned int*:         shmem_uint_test_all_vector,                \
+        unsigned long*:        shmem_ulong_test_all_vector,               \
+        unsigned long long*:   shmem_ulonglong_test_all_vector)(ivars, nelems, status, cmp, values)
+#endif
+
+
+OSHMEM_DECLSPEC  size_t shmem_short_test_any_vector(volatile short *ivars, size_t nelems, const int *status, int cmp, short *values);
+OSHMEM_DECLSPEC  size_t shmem_ushort_test_any_vector(volatile unsigned short *ivars, size_t nelems, const int *status, int cmp, unsigned short *values);
+OSHMEM_DECLSPEC  size_t shmem_int_test_any_vector(volatile int *ivars, size_t nelems, const int *status, int cmp, int *values);
+OSHMEM_DECLSPEC  size_t shmem_long_test_any_vector(volatile long *ivars, size_t nelems, const int *status, int cmp, long *values);
+OSHMEM_DECLSPEC  size_t shmem_longlong_test_any_vector(volatile long long *ivars, size_t nelems, const int *status, int cmp, long long *values);
+OSHMEM_DECLSPEC  size_t shmem_uint_test_any_vector(volatile unsigned int *ivars, size_t nelems, const int *status, int cmp, unsigned int *values);
+OSHMEM_DECLSPEC  size_t shmem_ulong_test_any_vector(volatile unsigned long *ivars, size_t nelems, const int *status, int cmp, unsigned long *values);
+OSHMEM_DECLSPEC  size_t shmem_ulonglong_test_any_vector(volatile unsigned long long *ivars, size_t nelems, const int *status, int cmp, unsigned long long *values);
+OSHMEM_DECLSPEC  size_t shmem_int32_test_any_vector(volatile int32_t *ivars, size_t nelems, const int *status, int cmp, int32_t *values);
+OSHMEM_DECLSPEC  size_t shmem_int64_test_any_vector(volatile int64_t *ivars, size_t nelems, const int *status, int cmp, int64_t *values);
+OSHMEM_DECLSPEC  size_t shmem_uint32_test_any_vector(volatile uint32_t *ivars, size_t nelems, const int *status, int cmp, uint32_t *values);
+OSHMEM_DECLSPEC  size_t shmem_uint64_test_any_vector(volatile uint64_t *ivars, size_t nelems, const int *status, int cmp, uint64_t *values);
+OSHMEM_DECLSPEC  size_t shmem_size_test_any_vector(volatile size_t *ivars, size_t nelems, const int *status, int cmp, size_t *values);
+OSHMEM_DECLSPEC  size_t shmem_ptrdiff_test_any_vector(volatile ptrdiff_t *ivars, size_t nelems, const int *status, int cmp, ptrdiff_t *values);
+#if OSHMEM_HAVE_C11
+#define shmem_test_any_vector(ivars, nelems, status, cmp, values)         \
+    _Generic(&*(ivars),                                                   \
+        short*:                shmem_short_test_any_vector,               \
+        unsigned short*:       shmem_ushort_test_any_vector,              \
+        int*:                  shmem_int_test_any_vector,                 \
+        long*:                 shmem_long_test_any_vector,                \
+        long long*:            shmem_longlong_test_any_vector,            \
+        unsigned int*:         shmem_uint_test_any_vector,                \
+        unsigned long*:        shmem_ulong_test_any_vector,               \
+        unsigned long long*:   shmem_ulonglong_test_any_vector)(ivars, nelems, status, cmp, values)
+#endif
+
+
+OSHMEM_DECLSPEC  size_t shmem_short_test_some_vector(volatile short *ivars, size_t nelems, size_t *indices, const int *status, int cmp, short *values);
+OSHMEM_DECLSPEC  size_t shmem_ushort_test_some_vector(volatile unsigned short *ivars, size_t nelems, size_t *indices, const int *status, int cmp, unsigned short *values);
+OSHMEM_DECLSPEC  size_t shmem_int_test_some_vector(volatile int *ivars, size_t nelems, size_t *indices, const int *status, int cmp, int *values);
+OSHMEM_DECLSPEC  size_t shmem_long_test_some_vector(volatile long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, long *values);
+OSHMEM_DECLSPEC  size_t shmem_longlong_test_some_vector(volatile long long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, long long *values);
+OSHMEM_DECLSPEC  size_t shmem_uint_test_some_vector(volatile unsigned int *ivars, size_t nelems, size_t *indices, const int *status, int cmp, unsigned int *values);
+OSHMEM_DECLSPEC  size_t shmem_ulong_test_some_vector(volatile unsigned long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, unsigned long *values);
+OSHMEM_DECLSPEC  size_t shmem_ulonglong_test_some_vector(volatile unsigned long long *ivars, size_t nelems, size_t *indices, const int *status, int cmp, unsigned long long *values);
+OSHMEM_DECLSPEC  size_t shmem_int32_test_some_vector(volatile int32_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, int32_t *values);
+OSHMEM_DECLSPEC  size_t shmem_int64_test_some_vector(volatile int64_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, int64_t *values);
+OSHMEM_DECLSPEC  size_t shmem_uint32_test_some_vector(volatile uint32_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, uint32_t *values);
+OSHMEM_DECLSPEC  size_t shmem_uint64_test_some_vector(volatile uint64_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, uint64_t *values);
+OSHMEM_DECLSPEC  size_t shmem_size_test_some_vector(volatile size_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, size_t *values);
+OSHMEM_DECLSPEC  size_t shmem_ptrdiff_test_some_vector(volatile ptrdiff_t *ivars, size_t nelems, size_t *indices, const int *status, int cmp, ptrdiff_t *values);
+#if OSHMEM_HAVE_C11
+#define shmem_test_some_vector(ivars, nelems, indices, status, cmp, values)    \
+    _Generic(&*(ivars),                                                    	\
+        short*:                shmem_short_test_some_vector,               	\
+        unsigned short*:       shmem_ushort_test_some_vector,              	\
+        int*:                  shmem_int_test_some_vector,                 	\
+        long*:                 shmem_long_test_some_vector,                	\
+        long long*:            shmem_longlong_test_some_vector,            	\
+        unsigned int*:         shmem_uint_test_some_vector,                	\
+        unsigned long*:        shmem_ulong_test_some_vector,               	\
+        unsigned long long*:   shmem_ulonglong_test_some_vector)(ivars, nelems, indices, status, cmp, values)
+#endif
+
 /*
  * Barrier sync routines
  */
 OSHMEM_DECLSPEC  void shmem_barrier(int PE_start, int logPE_stride, int PE_size, long *pSync);
 OSHMEM_DECLSPEC  void shmem_barrier_all(void);
-OSHMEM_DECLSPEC  void shmem_sync(int PE_start, int logPE_stride, int PE_size, long *pSync);
+OSHMEM_DECLSPEC  void shmem_sync_deprecated(int PE_start, int logPE_stride, int PE_size, long *pSync);
 OSHMEM_DECLSPEC  void shmem_sync_all(void);
 OSHMEM_DECLSPEC  void shmem_fence(void);
 OSHMEM_DECLSPEC  void shmem_ctx_fence(shmem_ctx_t ctx);
 OSHMEM_DECLSPEC  void shmem_quiet(void);
 OSHMEM_DECLSPEC  void shmem_ctx_quiet(shmem_ctx_t ctx);
 
+#if OSHMEM_HAVE_C11
+#define shmem_sync(...)                                          \
+    _Generic((__OSHMEM_VAR_ARG1(__VA_ARGS__)),                   \
+            shmem_team_t: shmem_team_sync,                       \
+            int:          shmem_sync_deprecated)(__VA_ARGS__)
+#endif
 /*
  * Collective routines
  */
 OSHMEM_DECLSPEC void shmem_broadcast32(void *target, const void *source, size_t nlong, int PE_root, int PE_start, int logPE_stride, int PE_size, long *pSync);
 OSHMEM_DECLSPEC void shmem_broadcast64(void *target, const void *source, size_t nlong, int PE_root, int PE_start, int logPE_stride, int PE_size, long *pSync);
-OSHMEM_DECLSPEC void shmem_broadcast(void *target, const void *source, size_t nlong, int PE_root, int PE_start, int logPE_stride, int PE_size, long *pSync);
 OSHMEM_DECLSPEC void shmem_collect32(void *target, const void *source, size_t nlong, int PE_start, int logPE_stride, int PE_size, long *pSync);
 OSHMEM_DECLSPEC void shmem_collect64(void *target, const void *source, size_t nlong, int PE_start, int logPE_stride, int PE_size, long *pSync);
 OSHMEM_DECLSPEC void shmem_fcollect32(void *target, const void *source, size_t nlong, int PE_start, int logPE_stride, int PE_size, long *pSync);
diff --git a/oshmem/include/shmemx.h b/oshmem/include/shmemx.h
index f7e7de68295..db4ffb5e81c 100644
--- a/oshmem/include/shmemx.h
+++ b/oshmem/include/shmemx.h
@@ -36,11 +36,6 @@ enum {
  * file. These extensions shall use the shmemx_ prefix for all routine, variable, and constant names.
  */
 
-/*
- * Symmetric heap routines
- */
-OSHMEM_DECLSPEC  void* shmemx_malloc_with_hint(size_t size, long hint);
-
 /*
  * Elemental put routines
  */
diff --git a/oshmem/info/info.c b/oshmem/info/info.c
index b35d137f62d..ac5963d9dcf 100644
--- a/oshmem/info/info.c
+++ b/oshmem/info/info.c
@@ -1,9 +1,9 @@
 /*
- * Copyright (c) 2015      Mellanox Technologies, Inc.
- *                         All rights reserved.
- * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
- * Copyright (c) 2019      Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2015-2020      Mellanox Technologies, Inc.
+ *                              All rights reserved.
+ * Copyright (c) 2018           Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2019           Research Organization for Information Science
+ *                              and Technology (RIST).  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -47,7 +47,7 @@ oshmem_info_t oshmem_shmem_info_env = {
  */
 static int oshmem_info_value_to_bool(char *value, bool *interp);
 static int oshmem_info_value_to_int(char *value, int *interp);
-static int oshmem_info_get_heap_size(char *value, size_t *interp);
+static int oshmem_info_get_heap_size(const char *name, char *value, size_t *interp);
 static int oshmem_info_get_library_version(char *version, int *len);
 
 
@@ -105,19 +105,19 @@ int oshmem_info_init(void)
     }
 
     if (NULL != (cptr = getenv(OSHMEM_ENV_SYMMETRIC_SIZE))) {
-        char *p1 = getenv(SHMEM_HEAP_SIZE);
-        if (p1 && strcmp(cptr, p1)) {
-            SHMEM_API_ERROR("Found conflict between env '%s' and '%s'.\n",
-                          OSHMEM_ENV_SYMMETRIC_SIZE, SHMEM_HEAP_SIZE);
-            ret = OSHMEM_ERR_BAD_PARAM;
-            goto out;
+        char *p1;
+        if (NULL != (p1 = getenv(SHMEM_HEAP_SIZE))) {
+            SHMEM_API_WARNING("SYMMETRIC_HEAP_SIZE and SMA_HEAP_SIZE are both set;"
+             "SYMMETRIC_HEAP_SIZE takes precedence.");
         }
-        ret = oshmem_info_get_heap_size(cptr, &oshmem_shmem_info_env.symmetric_heap_size);
+        ret = oshmem_info_get_heap_size(OSHMEM_ENV_SYMMETRIC_SIZE, cptr,
+                                        &oshmem_shmem_info_env.symmetric_heap_size);
         if (OSHMEM_SUCCESS != ret) {
             goto out;
         }
     } else if (NULL != (cptr = getenv(SHMEM_HEAP_SIZE))) {
-        ret = oshmem_info_get_heap_size(cptr, &oshmem_shmem_info_env.symmetric_heap_size);
+        ret = oshmem_info_get_heap_size(SHMEM_HEAP_SIZE, cptr,
+                                        &oshmem_shmem_info_env.symmetric_heap_size);
         if (OSHMEM_SUCCESS != ret) {
             goto out;
         }
@@ -217,7 +217,7 @@ static int oshmem_info_value_to_int(char *value, int *interp)
     return OSHMEM_SUCCESS;
 }
 
-static int oshmem_info_get_heap_size(char *value, size_t *interp)
+static int oshmem_info_get_heap_size(const char *name, char *value, size_t *interp)
 {
     char *p;
     long long factor = 1;
@@ -259,8 +259,7 @@ static int oshmem_info_get_heap_size(char *value, size_t *interp)
     if (size <= 0) {
         return OSHMEM_ERR_BAD_PARAM;
     } else {
-        opal_setenv(OSHMEM_ENV_SYMMETRIC_SIZE, p, true, &environ);
-        opal_setenv(SHMEM_HEAP_SIZE, p, true, &environ);
+        opal_setenv(name, p, true, &environ);
 /* Probably needless code */
 #if 0
         char *tmp = p;
diff --git a/oshmem/mca/atomic/atomic.h b/oshmem/mca/atomic/atomic.h
index 912f2a71955..d4683ee4afd 100644
--- a/oshmem/mca/atomic/atomic.h
+++ b/oshmem/mca/atomic/atomic.h
@@ -105,6 +105,42 @@ BEGIN_C_DECLS
                            target, value, pe);                             \
     }
 
+#define DO_OSHMEM_TYPE_FOP_NBI(ctx, type_name, type, op, fetch, target, value, pe) do {        \
+        int rc = OSHMEM_SUCCESS;                                                    \
+        size_t size = 0;                                                            \
+        type out_value;                                                             \
+                                                                                    \
+        RUNTIME_CHECK_INIT();                                                       \
+        RUNTIME_CHECK_PE(pe);                                                       \
+        RUNTIME_CHECK_ADDR(target);                                                 \
+                                                                                    \
+        size = sizeof(out_value);                                                   \
+        rc = MCA_ATOMIC_CALL(f##op##_nb(                                            \
+            ctx,                                                                    \
+            fetch,                                                                  \
+            (void*)target,                                                          \
+            (void*)&out_value,                                                      \
+            value,                                                                  \
+            size,                                                                   \
+            pe));                                                                   \
+        RUNTIME_CHECK_RC(rc);                                                       \
+                                                                                    \
+        return ;                                                           	     \
+    } while (0)
+
+#define OSHMEM_TYPE_FOP_NBI(type_name, type, prefix, op)                                \
+    void prefix##_##type_name##_atomic_fetch_##op##_nbi(type *fetch, type *target, type value, int pe) \
+    {                                                                                   \
+        DO_OSHMEM_TYPE_FOP_NBI(oshmem_ctx_default, type_name, type, op,                 \
+                           fetch, target, value, pe);                                   \
+    }
+
+#define OSHMEM_CTX_TYPE_FOP_NBI(type_name, type, prefix, op)                            \
+    void prefix##_ctx_##type_name##_atomic_fetch_##op##_nbi(shmem_ctx_t ctx, type *fetch, type *target, type value, int pe) \
+    {                                                                                   \
+        DO_OSHMEM_TYPE_FOP_NBI(ctx, type_name, type, op,                                \
+                           fetch, target, value, pe);                                   \
+    }
 /* ******************************************************************** */
 
 struct oshmem_op_t;
@@ -144,9 +180,9 @@ struct mca_atomic_base_component_1_0_0_t {
 };
 typedef struct mca_atomic_base_component_1_0_0_t mca_atomic_base_component_1_0_0_t;
 
-/** Per guidence in mca.h, use the unversioned struct name if you just
+/** Per guidance in mca.h, use the unversioned struct name if you just
  want to always keep up with the most recent version of the
- interace. */
+ interface. */
 typedef struct mca_atomic_base_component_1_0_0_t mca_atomic_base_component_t;
 
 /**
@@ -216,12 +252,56 @@ struct mca_atomic_base_module_1_0_0_t {
                         uint64_t value,
                         size_t size,
                         int pe);
+    int (*atomic_fadd_nb)(shmem_ctx_t ctx,
+                       void *fetch,
+                       void *target,
+                       void *prev,
+                       uint64_t value,
+                       size_t size,
+                       int pe);
+    int (*atomic_cswap_nb)(shmem_ctx_t ctx,
+                        void *fetch,
+                        void *target,
+                        uint64_t *prev, /* prev is used internally by wrapper, we may
+                                           always use 64-bit value */
+                        uint64_t cond,
+                        uint64_t value,
+                        size_t size,
+                        int pe);
+    int (*atomic_swap_nb)(shmem_ctx_t ctx,
+                       void *fetch,
+                       void *target,
+                       void *prev,
+                       uint64_t value,
+                       size_t size,
+                       int pe);
+    int (*atomic_fand_nb)(shmem_ctx_t ctx,
+                       void *fetch,
+                       void *target,
+                       void *prev,
+                       uint64_t value,
+                       size_t size,
+                       int pe);
+    int (*atomic_for_nb)(shmem_ctx_t ctx,
+                       void *fetch,
+                      void *target,
+                      void *prev,
+                      uint64_t value,
+                      size_t size,
+                      int pe);
+    int (*atomic_fxor_nb)(shmem_ctx_t ctx,
+                       void *fetch,
+                       void *target,
+                       void *prev,
+                       uint64_t value,
+                       size_t size,
+                       int pe);
 };
 typedef struct mca_atomic_base_module_1_0_0_t mca_atomic_base_module_1_0_0_t;
 
-/** Per guidence in mca.h, use the unversioned struct name if you just
+/** Per guidance in mca.h, use the unversioned struct name if you just
  want to always keep up with the most recent version of the
- interace. */
+ interface. */
 typedef struct mca_atomic_base_module_1_0_0_t mca_atomic_base_module_t;
 OSHMEM_DECLSPEC OBJ_CLASS_DECLARATION(mca_atomic_base_module_t);
 
diff --git a/oshmem/mca/atomic/base/base.h b/oshmem/mca/atomic/base/base.h
index e8fe9833bbb..e69b84dfac5 100644
--- a/oshmem/mca/atomic/base/base.h
+++ b/oshmem/mca/atomic/base/base.h
@@ -40,7 +40,7 @@ OSHMEM_DECLSPEC extern mca_base_framework_t oshmem_atomic_base_framework;
 #define __ATOMIC_FILE__ __FILE__
 #endif
 
-#ifdef OPAL_ENABLE_DEBUG
+#if OPAL_ENABLE_DEBUG
 #define ATOMIC_VERBOSE(level, ...) \
     oshmem_output_verbose(level, oshmem_atomic_base_framework.framework_output, \
         "%s:%d - %s()", __ATOMIC_FILE__, __LINE__, __func__, __VA_ARGS__)
diff --git a/oshmem/mca/atomic/basic/atomic_basic.h b/oshmem/mca/atomic/basic/atomic_basic.h
index b581f855052..5b01bd5450c 100644
--- a/oshmem/mca/atomic/basic/atomic_basic.h
+++ b/oshmem/mca/atomic/basic/atomic_basic.h
@@ -23,7 +23,7 @@ BEGIN_C_DECLS
 
 /* Globally exported variables */
 
-OSHMEM_MODULE_DECLSPEC extern mca_atomic_base_component_1_0_0_t
+OSHMEM_DECLSPEC extern mca_atomic_base_component_1_0_0_t
 mca_atomic_basic_component;
 
 OSHMEM_DECLSPEC void atomic_basic_lock(shmem_ctx_t ctx, int pe);
diff --git a/oshmem/mca/atomic/ucx/Makefile.am b/oshmem/mca/atomic/ucx/Makefile.am
index 8818616d4b8..2a7e78c57ca 100644
--- a/oshmem/mca/atomic/ucx/Makefile.am
+++ b/oshmem/mca/atomic/ucx/Makefile.am
@@ -35,7 +35,7 @@ mcacomponent_LTLIBRARIES = $(component_install)
 mca_atomic_ucx_la_SOURCES = $(ucx_sources)
 mca_atomic_ucx_la_LIBADD = $(top_builddir)/oshmem/liboshmem.la \
 	$(atomic_ucx_LIBS) \
-	$(OPAL_TOP_BUILDDIR)/opal/mca/common/ucx/lib@OPAL_LIB_PREFIX@mca_common_ucx.la
+	$(OPAL_TOP_BUILDDIR)/opal/mca/common/ucx/lib@OPAL_LIB_NAME@mca_common_ucx.la
 mca_atomic_ucx_la_LDFLAGS = -module -avoid-version $(atomic_ucx_LDFLAGS)
 
 noinst_LTLIBRARIES = $(component_noinst)
diff --git a/oshmem/mca/atomic/ucx/atomic_ucx.h b/oshmem/mca/atomic/ucx/atomic_ucx.h
index dd588bdc0b5..b6de75be77e 100644
--- a/oshmem/mca/atomic/ucx/atomic_ucx.h
+++ b/oshmem/mca/atomic/ucx/atomic_ucx.h
@@ -26,7 +26,7 @@ BEGIN_C_DECLS
 
 /* Globally exported variables */
 
-OSHMEM_MODULE_DECLSPEC extern mca_atomic_base_component_1_0_0_t
+OSHMEM_DECLSPEC extern mca_atomic_base_component_1_0_0_t
 mca_atomic_ucx_component;
 
 /* this component works with spml:ucx only */
diff --git a/oshmem/mca/atomic/ucx/atomic_ucx_cswap.c b/oshmem/mca/atomic/ucx/atomic_ucx_cswap.c
index 45b0ce00692..39eec9a4f68 100644
--- a/oshmem/mca/atomic/ucx/atomic_ucx_cswap.c
+++ b/oshmem/mca/atomic/ucx/atomic_ucx_cswap.c
@@ -39,16 +39,19 @@ int mca_atomic_ucx_cswap(shmem_ctx_t ctx,
         .reply_buffer = prev
     };
 #endif
+    assert(NULL != prev);
 
-    if ((8 != size) && (4 != size)) {
+    if (size == 8) {
+        *prev = value;
+    } else if (size == 4) {
+        *(uint32_t*)prev = value;
+    } else {
         ATOMIC_ERROR("[#%d] Type size must be 4 or 8 bytes.", my_pe);
         return OSHMEM_ERROR;
     }
 
-    assert(NULL != prev);
-
-    *prev      = value;
-    ucx_mkey   = mca_spml_ucx_get_mkey(ctx, pe, target, (void *)&rva, mca_spml_self);
+    ucx_mkey   = mca_spml_ucx_ctx_mkey_by_va(ctx, pe, target, (void *)&rva, mca_spml_self);
+    assert(NULL != ucx_mkey);
 #if HAVE_DECL_UCP_ATOMIC_OP_NBX
     status_ptr = ucp_atomic_op_nbx(ucx_ctx->ucp_peers[pe].ucp_conn,
                                    UCP_ATOMIC_OP_CSWAP, &cond, 1, rva,
@@ -71,3 +74,4 @@ int mca_atomic_ucx_cswap(shmem_ctx_t ctx,
                                         "ucp_atomic_fetch_nb");
 #endif
 }
+
diff --git a/oshmem/mca/atomic/ucx/atomic_ucx_module.c b/oshmem/mca/atomic/ucx/atomic_ucx_module.c
index 8a9a4a06311..172626dd0c6 100644
--- a/oshmem/mca/atomic/ucx/atomic_ucx_module.c
+++ b/oshmem/mca/atomic/ucx/atomic_ucx_module.c
@@ -55,36 +55,38 @@ int mca_atomic_ucx_op(shmem_ctx_t ctx,
                       ucp_atomic_post_op_t op)
 #endif
 {
-    ucs_status_t status;
     spml_ucx_mkey_t *ucx_mkey;
     uint64_t rva;
     mca_spml_ucx_ctx_t *ucx_ctx = (mca_spml_ucx_ctx_t *)ctx;
 #if HAVE_DECL_UCP_ATOMIC_OP_NBX
     ucs_status_ptr_t status_ptr;
+#else
+    ucs_status_t status;
 #endif
+    int res;
 
     assert((8 == size) || (4 == size));
 
-    ucx_mkey = mca_spml_ucx_get_mkey(ctx, pe, target, (void *)&rva, mca_spml_self);
-
+    ucx_mkey = mca_spml_ucx_ctx_mkey_by_va(ctx, pe, target, (void *)&rva, mca_spml_self);
+    assert(NULL != ucx_mkey);
 #if HAVE_DECL_UCP_ATOMIC_OP_NBX
     status_ptr = ucp_atomic_op_nbx(ucx_ctx->ucp_peers[pe].ucp_conn,
                                    op, &value, 1, rva, ucx_mkey->rkey,
                                    &mca_spml_ucp_request_params[size >> 3]);
-    if (OPAL_LIKELY(!UCS_PTR_IS_ERR(status_ptr))) {
-        mca_spml_ucx_remote_op_posted(ucx_ctx, pe);
-    }
-    status = UCS_PTR_STATUS(status_ptr);
+    res = opal_common_ucx_wait_request(status_ptr, ucx_ctx->ucp_worker[0],
+                                       "ucp_atomic_op_nbx post");
 #else
     status = ucp_atomic_post(ucx_ctx->ucp_peers[pe].ucp_conn,
                              op, value, size, rva,
                              ucx_mkey->rkey);
+    res = ucx_status_to_oshmem(status);
 #endif
-    if (OPAL_LIKELY(UCS_OK == status)) {
+
+    if (OPAL_LIKELY(OSHMEM_SUCCESS == res)) {
         mca_spml_ucx_remote_op_posted(ucx_ctx, pe);
     }
 
-    return ucx_status_to_oshmem(status);
+    return res;
 }
 
 static inline
@@ -115,7 +117,8 @@ int mca_atomic_ucx_fop(shmem_ctx_t ctx,
 
     assert((8 == size) || (4 == size));
 
-    ucx_mkey = mca_spml_ucx_get_mkey(ctx, pe, target, (void *)&rva, mca_spml_self);
+    ucx_mkey = mca_spml_ucx_ctx_mkey_by_va(ctx, pe, target, (void *)&rva, mca_spml_self);
+    assert(NULL != ucx_mkey);
 #if HAVE_DECL_UCP_ATOMIC_OP_NBX
     status_ptr = ucp_atomic_op_nbx(ucx_ctx->ucp_peers[pe].ucp_conn, op, &value, 1,
                                    rva, ucx_mkey->rkey, &param);
@@ -265,6 +268,77 @@ static int mca_atomic_ucx_swap(shmem_ctx_t ctx,
 #endif
 }
 
+static int mca_atomic_ucx_fadd_nb(shmem_ctx_t ctx,
+                               void *fetch,
+                               void *target,
+                               void *prev,
+                               uint64_t value,
+                               size_t size,
+                               int pe)
+{
+    return OSHMEM_ERR_NOT_IMPLEMENTED;
+}
+
+static int mca_atomic_ucx_fand_nb(shmem_ctx_t ctx,
+                               void *fetch,
+                               void *target,
+                               void *prev,
+                               uint64_t value,
+                               size_t size,
+                               int pe)
+{
+    return OSHMEM_ERR_NOT_IMPLEMENTED;
+}
+
+static int mca_atomic_ucx_for_nb(shmem_ctx_t ctx,
+                               void *fetch,
+                               void *target,
+                               void *prev,
+                               uint64_t value,
+                               size_t size,
+                               int pe)
+{
+    return OSHMEM_ERR_NOT_IMPLEMENTED;
+}
+
+static int mca_atomic_ucx_fxor_nb(shmem_ctx_t ctx,
+                               void *fetch,
+                               void *target,
+                               void *prev,
+                               uint64_t value,
+                               size_t size,
+                               int pe)
+{
+    return OSHMEM_ERR_NOT_IMPLEMENTED;
+}
+
+static int mca_atomic_ucx_swap_nb(shmem_ctx_t ctx,
+                               void *fetch,
+                               void *target,
+                               void *prev,
+                               uint64_t value,
+                               size_t size,
+                               int pe)
+{
+    return OSHMEM_ERR_NOT_IMPLEMENTED;
+}
+
+
+static int mca_atomic_ucx_cswap_nb(shmem_ctx_t ctx,
+                                   void *fetch,
+                                   void *target,
+                                   uint64_t *prev,
+                                   uint64_t cond,
+                                   uint64_t value,
+                                   size_t size,
+                                   int pe)
+{
+    return OSHMEM_ERR_NOT_IMPLEMENTED;
+}
+
+
+
+
 
 mca_atomic_base_module_t *
 mca_atomic_ucx_query(int *priority)
@@ -285,6 +359,12 @@ mca_atomic_ucx_query(int *priority)
         module->super.atomic_fxor  = mca_atomic_ucx_fxor;
         module->super.atomic_swap  = mca_atomic_ucx_swap;
         module->super.atomic_cswap = mca_atomic_ucx_cswap;
+        module->super.atomic_fadd_nb  = mca_atomic_ucx_fadd_nb;
+        module->super.atomic_fand_nb  = mca_atomic_ucx_fand_nb;
+        module->super.atomic_for_nb   = mca_atomic_ucx_for_nb;
+        module->super.atomic_fxor_nb  = mca_atomic_ucx_fxor_nb;
+        module->super.atomic_swap_nb  = mca_atomic_ucx_swap_nb;
+        module->super.atomic_cswap_nb = mca_atomic_ucx_cswap_nb;
         return &(module->super);
     }
 
diff --git a/oshmem/mca/atomic/ucx/configure.m4 b/oshmem/mca/atomic/ucx/configure.m4
index 53b057da3f9..fbad0b54bf6 100644
--- a/oshmem/mca/atomic/ucx/configure.m4
+++ b/oshmem/mca/atomic/ucx/configure.m4
@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2015      Mellanox Technologies, Inc.
  *                         All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -24,7 +25,6 @@ AC_DEFUN([MCA_oshmem_atomic_ucx_CONFIG],[
 
 
     # substitute in the things needed to build ucx 
-    AC_SUBST([atomic_ucx_CFLAGS])
     AC_SUBST([atomic_ucx_CPPFLAGS])
     AC_SUBST([atomic_ucx_LDFLAGS])
     AC_SUBST([atomic_ucx_LIBS])
diff --git a/oshmem/mca/memheap/README.md b/oshmem/mca/memheap/README.md
index b487eb08e50..66aeb4b8e6a 100644
--- a/oshmem/mca/memheap/README.md
+++ b/oshmem/mca/memheap/README.md
@@ -3,7 +3,7 @@
 Copyright (c) 2013      Mellanox Technologies, Inc.
                         All rights reserved
 
-MEMHEAP Infrustructure is responsible for managing the symmetric heap.
+MEMHEAP Infrastructure is responsible for managing the symmetric heap.
 The framework currently has following components: buddy and
 ptmalloc. buddy which uses a buddy allocator in order to manage the
 Memory allocations on the symmetric heap. Ptmalloc is an adaptation of
@@ -11,7 +11,7 @@ ptmalloc3.
 
 Additional components may be added easily to the framework by defining
 the component's and the module's base and extended structures, and
-their funtionalities.
+their functionalities.
 
 The buddy allocator has the following data structures:
 
@@ -54,7 +54,7 @@ The supported activities are:
 
 1. buddy_init (Initialization)
 1. buddy_alloc (Allocates a variable on the symmetric heap)
-1. buddy_free (frees a variable previously allocated on the symetric heap)
+1. buddy_free (frees a variable previously allocated on the symmetric heap)
 1. buddy_finalize (Finalization).
 
 Data members of buddy module:
diff --git a/oshmem/mca/memheap/base/Makefile.am b/oshmem/mca/memheap/base/Makefile.am
index 0244f7ea103..4414fca7939 100644
--- a/oshmem/mca/memheap/base/Makefile.am
+++ b/oshmem/mca/memheap/base/Makefile.am
@@ -1,6 +1,7 @@
 # Copyright (c) 2014      Mellanox Technologies, Inc.
 #                         All rights reserved.
 # Copyright (c) 2016      IBM Corporation.  All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -8,7 +9,7 @@
 # $HEADER$
 #
 
-dist_oshmemdata_DATA += base/help-oshmem-memheap.txt
+EXTRA_DIST = base/help-oshmem-memheap.txt
 
 AM_CPPFLAGS = -DOMPI_LIBMPI_NAME=\"$(OMPI_LIBMPI_NAME)\"
 
diff --git a/oshmem/mca/memheap/base/base.h b/oshmem/mca/memheap/base/base.h
index 64cdb01ca4f..97dac884d2a 100644
--- a/oshmem/mca/memheap/base/base.h
+++ b/oshmem/mca/memheap/base/base.h
@@ -40,8 +40,8 @@ OSHMEM_DECLSPEC int mca_memheap_base_select(void);
 
 extern int mca_memheap_base_already_opened;
 extern int mca_memheap_base_key_exchange;
+extern int mca_memheap_num_segments_warn;
 
-#define MCA_MEMHEAP_MAX_SEGMENTS    32
 #define HEAP_SEG_INDEX              0
 #define MCA_MEMHEAP_SEG_COUNT       2
 
@@ -54,8 +54,9 @@ typedef struct mca_memheap_base_config {
 
 
 typedef struct mca_memheap_map {
-    map_segment_t   mem_segs[MCA_MEMHEAP_MAX_SEGMENTS]; /* TODO: change into pointer array */
+    map_segment_t  *mem_segs;
     int             n_segments;
+    int             capacity;
     int             num_transports;
 } mca_memheap_map_t;
 
@@ -70,6 +71,7 @@ int mca_memheap_base_reg(mca_memheap_map_t *);
 int mca_memheap_base_dereg(mca_memheap_map_t *);
 int memheap_oob_init(mca_memheap_map_t *);
 void memheap_oob_destruct(void);
+map_segment_t *mca_memheap_base_allocate_segment(mca_memheap_map_t *map);
 
 OSHMEM_DECLSPEC int mca_memheap_base_is_symmetric_addr(const void* va);
 OSHMEM_DECLSPEC sshmem_mkey_t *mca_memheap_base_get_mkey(void* va,
@@ -135,7 +137,7 @@ OSHMEM_DECLSPEC extern mca_base_framework_t oshmem_memheap_base_framework;
 #define __SPML_FILE__ __FILE__
 #endif
 
-#ifdef OPAL_ENABLE_DEBUG
+#if OPAL_ENABLE_DEBUG
 #define MEMHEAP_VERBOSE(level, ...) \
     oshmem_output_verbose(level, oshmem_memheap_base_framework.framework_output, \
         "%s:%d - %s()", __SPML_FILE__, __LINE__, __func__, __VA_ARGS__)
@@ -176,13 +178,35 @@ static inline int memheap_is_va_in_segment(void *va, int segno)
     return map_segment_is_va_in(&memheap_find_seg(segno)->super, va);
 }
 
-static inline int memheap_find_segnum(void *va)
+static inline int memheap_find_segnum(void *va, int pe)
 {
     int i;
-
-    for (i = 0; i < mca_memheap_base_map.n_segments; i++) {
-        if (memheap_is_va_in_segment(va, i)) {
-            return i;
+    int my_pe = oshmem_my_proc_id();
+
+    if (pe == my_pe) {
+        /* Find segment number for local segment using va_base
+         * TODO: Merge local and remote segment information in mkeys_cache
+         */
+        for (i = 0; i < mca_memheap_base_map.n_segments; i++) {
+            if (memheap_is_va_in_segment(va, i)) {
+                return i;
+            }
+        }
+    } else {
+        /* Find segment number for remote segments using va_base */
+        for (i = 0; i < mca_memheap_base_map.n_segments; i++) {
+            map_segment_t *seg = memheap_find_seg(i);
+            if (seg) {
+                sshmem_mkey_t **mkeys_cache = seg->mkeys_cache;
+                if (mkeys_cache) {
+                    if (mkeys_cache[pe]) {
+                        if ((va >= mkeys_cache[pe]->va_base) &&
+                            ((char*)va < (char*)mkeys_cache[pe]->va_base + mkeys_cache[pe]->len)) {
+                            return i;
+                        }
+                    }
+                }
+            }
         }
     }
     return MEMHEAP_SEG_INVALID;
@@ -200,22 +224,6 @@ static inline void *map_segment_va2rva(mkey_segment_t *seg, void *va)
     return memheap_va2rva(va, seg->super.va_base, seg->rva_base);
 }
 
-static inline map_base_segment_t *map_segment_find_va(map_base_segment_t *segs,
-                                                      size_t elem_size, void *va)
-{
-    map_base_segment_t *rseg;
-    int i;
-
-    for (i = 0; i < MCA_MEMHEAP_MAX_SEGMENTS; i++) {
-        rseg = (map_base_segment_t *)((char *)segs + elem_size * i);
-        if (OPAL_LIKELY(map_segment_is_va_in(rseg, va))) {
-            return rseg;
-        }
-    }
-
-    return NULL;
-}
-
 void mkey_segment_init(mkey_segment_t *seg, sshmem_mkey_t *mkey, uint32_t segno);
 
 static inline map_segment_t *memheap_find_va(void* va)
diff --git a/oshmem/mca/memheap/base/memheap_base_alloc.c b/oshmem/mca/memheap/base/memheap_base_alloc.c
index 3354a187def..b90415bbf80 100644
--- a/oshmem/mca/memheap/base/memheap_base_alloc.c
+++ b/oshmem/mca/memheap/base/memheap_base_alloc.c
@@ -18,6 +18,7 @@
 #include "oshmem/mca/memheap/memheap.h"
 #include "oshmem/mca/memheap/base/base.h"
 #include "ompi/util/timings.h"
+#include "opal/util/minmax.h"
 
 
 int mca_memheap_base_alloc_init(mca_memheap_map_t *map, size_t size, long hint,
@@ -35,7 +36,12 @@ int mca_memheap_base_alloc_init(mca_memheap_map_t *map, size_t size, long hint,
         assert(HEAP_SEG_INDEX < map->n_segments);
     }
 
-    map_segment_t *s = &map->mem_segs[map->n_segments];
+    map_segment_t *s = mca_memheap_base_allocate_segment(map);
+    if (NULL == s) {
+        MEMHEAP_ERROR("failed to allocate segment");
+        return OSHMEM_ERR_OUT_OF_RESOURCE;
+    }
+
     seg_filename = oshmem_get_unique_file_name(oshmem_my_proc_id());
 
     OPAL_TIMING_ENV_NEXT(timing, "oshmem_get_unique_file_name()");
@@ -72,6 +78,11 @@ void mca_memheap_base_alloc_exit(mca_memheap_map_t *map)
             mca_sshmem_unlink(s);
         }
     }
+
+    free(map->mem_segs);
+    map->n_segments = 0;
+    map->capacity = 0;
+    map->mem_segs = NULL;
 }
 
 int mca_memheap_alloc_with_hint(size_t size, long hint, void** ptr)
@@ -90,3 +101,33 @@ int mca_memheap_alloc_with_hint(size_t size, long hint, void** ptr)
 
     return MCA_MEMHEAP_CALL(alloc(size, ptr));
 }
+
+map_segment_t *mca_memheap_base_allocate_segment(mca_memheap_map_t *map)
+{
+    static int warned = 0;
+    map_segment_t *segments;
+    int capacity;
+
+    assert(map->n_segments <= map->capacity);
+
+    if (!warned && (map->n_segments > mca_memheap_num_segments_warn)) {
+        MEMHEAP_WARN("too many segments are registered: %d. This may cause "
+                     "performance degradation. Pls try adding --mca "
+                     "memheap_base_max_segments <NUMBER> to mpirun/oshrun "
+                     "command line to suppress this message", map->n_segments);
+        warned = 1;
+    }
+
+    if (map->n_segments == map->capacity) {
+        capacity = opal_max(map->capacity * 2, 4);
+        segments = realloc(map->mem_segs, capacity * sizeof(*map->mem_segs));
+        if (segments == NULL) {
+            return NULL;
+        }
+
+        map->capacity = capacity;
+        map->mem_segs = segments;
+    }
+
+    return &map->mem_segs[map->n_segments];
+}
diff --git a/oshmem/mca/memheap/base/memheap_base_frame.c b/oshmem/mca/memheap/base/memheap_base_frame.c
index 23ebf0860db..53a71b27a9e 100644
--- a/oshmem/mca/memheap/base/memheap_base_frame.c
+++ b/oshmem/mca/memheap/base/memheap_base_frame.c
@@ -36,6 +36,7 @@ int mca_memheap_base_key_exchange = 1;
 opal_list_t mca_memheap_base_components_opened = {{0}};
 int mca_memheap_base_already_opened = 0;
 mca_memheap_map_t mca_memheap_base_map = {{{{0}}}};
+int mca_memheap_num_segments_warn = 32;
 
 static int mca_memheap_base_register(mca_base_register_flag_t flags)
 {
@@ -59,6 +60,14 @@ static int mca_memheap_base_register(mca_base_register_flag_t flags)
                           MCA_BASE_VAR_SCOPE_LOCAL,
                           &mca_memheap_base_config.device_nic_mem_seg_size);
 
+    mca_base_var_register("oshmem", "memheap", "base", "max_segments",
+                          "Display a warning if the number of segments of the "
+                          "shared memheap exceeds this value",
+                          MCA_BASE_VAR_TYPE_INT, NULL, 0,
+                          MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
+                          MCA_BASE_VAR_SCOPE_LOCAL,
+                          &mca_memheap_num_segments_warn);
+
     return OSHMEM_SUCCESS;
 }
 
diff --git a/oshmem/mca/memheap/base/memheap_base_mkey.c b/oshmem/mca/memheap/base/memheap_base_mkey.c
index db682498f73..cda9253e980 100644
--- a/oshmem/mca/memheap/base/memheap_base_mkey.c
+++ b/oshmem/mca/memheap/base/memheap_base_mkey.c
@@ -151,9 +151,6 @@ static void unpack_remote_mkeys(shmem_ctx_t ctx, pmix_data_buffer_t *msg, int re
     int32_t n;
     int32_t tr_id;
     int i;
-    ompi_proc_t *proc;
-
-    proc = oshmem_proc_group_find(oshmem_group_all, remote_pe);
     cnt = 1;
     PMIx_Data_unpack(NULL, msg, &n, &cnt, PMIX_UINT32);
     for (i = 0; i < n; i++) {
@@ -168,7 +165,7 @@ static void unpack_remote_mkeys(shmem_ctx_t ctx, pmix_data_buffer_t *msg, int re
         if (0 == memheap_oob.mkeys[tr_id].va_base) {
             cnt = 1;
             PMIx_Data_unpack(NULL, msg, &memheap_oob.mkeys[tr_id].u.key, &cnt, PMIX_UINT64);
-            if (OPAL_PROC_ON_LOCAL_NODE(proc->super.proc_flags)) {
+            if (oshmem_proc_on_local_node(remote_pe)) {
                 memheap_attach_segment(&memheap_oob.mkeys[tr_id], tr_id);
             }
         } else {
@@ -230,11 +227,12 @@ static void do_recv(int source_pe, pmix_data_buffer_t* buffer)
         PMIx_Data_pack(NULL, msg, &msg_type, 1, PMIX_UINT8);
 
         if (OSHMEM_SUCCESS != pack_local_mkeys(msg, source_pe, seg)) {
-            OBJ_RELEASE(msg);
+            PMIX_DATA_BUFFER_RELEASE(msg);
             goto send_fail;
         }
 
         rc = send_buffer(source_pe, msg);
+        PMIX_DATA_BUFFER_RELEASE(msg);
         if (MPI_SUCCESS != rc) {
             MEMHEAP_ERROR("FAILED to send rml message %d", rc);
             OMPI_ERROR_LOG(rc);
@@ -274,6 +272,7 @@ static void do_recv(int source_pe, pmix_data_buffer_t* buffer)
     PMIx_Data_pack(NULL, msg, &msg_type, 1, PMIX_UINT8);
 
     rc = send_buffer(source_pe, msg);
+    PMIX_DATA_BUFFER_RELEASE(msg);
     if (MPI_SUCCESS != rc) {
         MEMHEAP_ERROR("FAILED to send rml message %d", rc);
         OMPI_ERROR_LOG(rc);
@@ -283,7 +282,7 @@ static void do_recv(int source_pe, pmix_data_buffer_t* buffer)
 
 /**
  * simple/fast version of MPI_Test that
- * - only works with persistant request
+ * - only works with persistent request
  * - does not do any progress
  * - can be safely called from within opal_progress()
  */
@@ -446,15 +445,14 @@ void memheap_oob_destruct(void)
 static int send_buffer(int pe, pmix_data_buffer_t *msg)
 {
     void *buffer;
-    int32_t size;
+    size_t size;
     int rc;
 
     PMIX_DATA_BUFFER_UNLOAD(msg, buffer, size);
     rc = PMPI_Send(buffer, size, MPI_BYTE, pe, 0, oshmem_comm_world);
     free(buffer);
-    PMIX_DATA_BUFFER_RELEASE(msg);
 
-    MEMHEAP_VERBOSE(5, "message sent: dst=%d, rc=%d, %d bytes!", pe, rc, size);
+    MEMHEAP_VERBOSE(5, "message sent: dst=%d, rc=%d, %" PRIsize_t " bytes!", pe, rc, size);
     return rc;
 }
 
@@ -495,6 +493,7 @@ static int memheap_oob_get_mkeys(shmem_ctx_t ctx, int pe, uint32_t seg, sshmem_m
     PMIx_Data_pack(NULL, msg, &seg, 1, PMIX_UINT32);
 
     rc = send_buffer(pe, msg);
+    PMIX_DATA_BUFFER_RELEASE(msg);
     if (MPI_SUCCESS != rc) {
         OPAL_THREAD_UNLOCK(&memheap_oob.lck);
         MEMHEAP_ERROR("FAILED to send rml message %d", rc);
@@ -524,7 +523,7 @@ void mca_memheap_modex_recv_all(void)
     pmix_data_buffer_t *msg = NULL;
     void *send_buffer = NULL;
     char *rcv_buffer = NULL;
-    int size;
+    size_t size;
     int *rcv_size = NULL;
     int *rcv_n_transports = NULL;
     int *rcv_offsets = NULL;
@@ -585,7 +584,7 @@ void mca_memheap_modex_recv_all(void)
 
     /* Do allgather */
     PMIX_DATA_BUFFER_UNLOAD(msg, send_buffer, size);
-    MEMHEAP_VERBOSE(1, "local keys packed into %d bytes, %d segments", size, memheap_map->n_segments);
+    MEMHEAP_VERBOSE(1, "local keys packed into %d bytes, %" PRIsize_t " segments", size, memheap_map->n_segments);
 
     OPAL_TIMING_ENV_NEXT(recv_all, "serialize data");
 
@@ -621,7 +620,7 @@ void mca_memheap_modex_recv_all(void)
 
     rcv_buffer = malloc (buffer_size);
     if (NULL == rcv_buffer) {
-        MEMHEAP_ERROR("failed to allocate recieve buffer");
+        MEMHEAP_ERROR("failed to allocate receive buffer");
         rc = OSHMEM_ERR_OUT_OF_RESOURCE;
         goto exit_fatal;
     }
@@ -769,13 +768,8 @@ void mkey_segment_init(mkey_segment_t *seg, sshmem_mkey_t *mkey, uint32_t segno)
 {
     map_segment_t *s;
 
-    if (segno >= MCA_MEMHEAP_MAX_SEGMENTS) {
-        return;
-    }
-
     s = memheap_find_seg(segno);
     assert(NULL != s);
-
     seg->super.va_base = s->super.va_base;
     seg->super.va_end  = s->super.va_end;
     seg->rva_base      = mkey->va_base;
diff --git a/oshmem/mca/memheap/base/memheap_base_register.c b/oshmem/mca/memheap/base/memheap_base_register.c
index ea742b2eb5e..4412d9ac74a 100644
--- a/oshmem/mca/memheap/base/memheap_base_register.c
+++ b/oshmem/mca/memheap/base/memheap_base_register.c
@@ -86,7 +86,7 @@ static int _dereg_segment(map_segment_t *s)
                 continue;
             if (s->mkeys_cache[j]) {
                 if (s->mkeys_cache[j]->len) {
-                    MCA_SPML_CALL(rmkey_free(s->mkeys_cache[j]));
+                    MCA_SPML_CALL(rmkey_free(s->mkeys_cache[j], j));
                     free(s->mkeys_cache[j]->u.data);
                     s->mkeys_cache[j]->len = 0;
                 }
diff --git a/oshmem/mca/memheap/base/memheap_base_select.c b/oshmem/mca/memheap/base/memheap_base_select.c
index c32f799d373..4ea808afa30 100644
--- a/oshmem/mca/memheap/base/memheap_base_select.c
+++ b/oshmem/mca/memheap/base/memheap_base_select.c
@@ -25,6 +25,8 @@
 #include "oshmem/mca/sshmem/base/base.h"
 #include "ompi/util/timings.h"
 
+#include <sys/mman.h>
+
 mca_memheap_base_config_t mca_memheap_base_config = {
     .device_nic_mem_seg_size = 0
 };
@@ -106,6 +108,136 @@ static size_t _memheap_size(void)
     return (size_t) memheap_align(oshmem_shmem_info_env.symmetric_heap_size);
 }
 
+static void *memheap_mmap_get(void *hint, size_t size)
+{
+    void *addr;
+
+    addr = mmap(hint, size,
+                PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    if (addr == MAP_FAILED) {
+        return NULL;
+    }
+
+    return addr;
+}
+
+static int memheap_exchange_base_address(size_t size, void **address)
+{
+    int nprocs = oshmem_num_procs();
+    int need_sync = (*address == NULL);
+    void *base = NULL;
+    void *ptr = NULL;
+    int rc, i;
+    void **bases;
+
+    bases = calloc(nprocs, sizeof(*bases));
+    if (NULL == bases) {
+        return OSHMEM_ERROR;
+    }
+
+    if (oshmem_my_proc_id() == 0) {
+        ptr = memheap_mmap_get(NULL, size);
+        base = ptr;
+    }
+
+    rc = oshmem_shmem_bcast(&base, sizeof(base), 0);
+    if (OSHMEM_SUCCESS != rc) {
+        MEMHEAP_ERROR("Failed to exchange allocated vma for base segment "
+                      "(error %d)", rc);
+        goto out;
+    }
+
+    if (oshmem_my_proc_id() != 0) {
+        ptr = memheap_mmap_get(base, size);
+    }
+
+    MEMHEAP_VERBOSE(100, "#%d: exchange base address: base %p: %s",
+                    oshmem_my_proc_id(), base,
+                    (base == ptr)? "ok" : "unavailable");
+
+    *address = base;
+    if (need_sync) {
+        /* They all succeed or fail to allow fallback */
+        rc = oshmem_shmem_allgather(&ptr, bases, sizeof(ptr));
+        if (OSHMEM_SUCCESS != rc) {
+            MEMHEAP_ERROR("Failed to exchange selected vma for base segment "
+                          "(error %d)", rc);
+            goto out;
+        }
+
+        for (i = 0; i < nprocs; i++) {
+            if ((NULL == bases[i]) || (bases[i] != base)) {
+                *address = NULL;
+                break;
+            }
+        }
+    } else if (ptr != base) {
+        /* Any failure terminates the rank and others start teardown */
+        rc = OSHMEM_ERROR;
+    }
+
+out:
+    if (((OSHMEM_SUCCESS != rc) || (*address == NULL)) && (ptr != NULL)) {
+        (void)munmap(ptr, size);
+    }
+
+    free(bases);
+    return rc;
+}
+
+
+/*
+ * The returned mca_sshmem_base_start_address value is reserved by using
+ * mmap() for the expected size.
+ */
+static int memheap_base_segment_setup(size_t size)
+{
+    int rc;
+
+    if ((mca_sshmem_base_start_address == (void *)UINTPTR_MAX) ||
+        (mca_sshmem_base_start_address == NULL)) {
+        if (UINTPTR_MAX == 0xFFFFFFFF) {
+            /**
+             * if 32 bit we set sshmem_base_start_adress to 0
+             * to let OS allocate segment automatically
+             */
+            mca_sshmem_base_start_address = NULL;
+            return OSHMEM_SUCCESS;
+        }
+
+        rc = memheap_exchange_base_address(size, &mca_sshmem_base_start_address);
+        if (OSHMEM_SUCCESS != rc) {
+            MEMHEAP_ERROR("Failed to setup base segment address (error %d)", rc);
+            return rc;
+        }
+
+        if (NULL != mca_sshmem_base_start_address) {
+            goto done; /* Region is reserved */
+        }
+
+#if defined(__aarch64__)
+        mca_sshmem_base_start_address = (void*)0xAB0000000000;
+#else
+        mca_sshmem_base_start_address = (void*)0xFF000000;
+#endif
+    }
+
+    if (mca_sshmem_base_start_address != memheap_mmap_get(
+                                      mca_sshmem_base_start_address, size)) {
+        MEMHEAP_ERROR("Failed to create segment address %p/%zu",
+                      mca_sshmem_base_start_address, size);
+        return OSHMEM_ERROR;
+    }
+
+done:
+    if (oshmem_my_proc_id() == 0) {
+        MEMHEAP_VERBOSE(10, "Using symmetric segment address %p/%zu",
+                        mca_sshmem_base_start_address, size);
+    }
+
+    return OSHMEM_SUCCESS;
+}
+
 static memheap_context_t* _memheap_create(void)
 {
     int rc = OSHMEM_SUCCESS;
@@ -123,13 +255,18 @@ static memheap_context_t* _memheap_create(void)
 
     OPAL_TIMING_ENV_NEXT(timing, "_memheap_size()");
 
-    /* Inititialize symmetric area */
-    if (OSHMEM_SUCCESS == rc) {
-        rc = mca_memheap_base_alloc_init(&mca_memheap_base_map,
-                                         user_size + MEMHEAP_BASE_PRIVATE_SIZE, 0,
-                                         "regular_mem");
+    /* Locate and reserve symmetric area */
+    rc = memheap_base_segment_setup(user_size + MEMHEAP_BASE_PRIVATE_SIZE);
+    if (OSHMEM_SUCCESS != rc) {
+        MEMHEAP_ERROR("Failed to negotiate base segment addres");
+        return NULL;
     }
 
+    /* Initialize symmetric area */
+    rc = mca_memheap_base_alloc_init(&mca_memheap_base_map,
+                                     user_size + MEMHEAP_BASE_PRIVATE_SIZE, 0,
+                                     "regular_mem");
+
     OPAL_TIMING_ENV_NEXT(timing, "mca_memheap_base_alloc_init()");
 
     /* Initialize atomic symmetric area */
diff --git a/oshmem/mca/memheap/base/memheap_base_static.c b/oshmem/mca/memheap/base/memheap_base_static.c
index 4e97253a9ee..2e9dcb91205 100644
--- a/oshmem/mca/memheap/base/memheap_base_static.c
+++ b/oshmem/mca/memheap/base/memheap_base_static.c
@@ -2,6 +2,7 @@
  * Copyright (c) 2013      Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2016      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2023      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -14,68 +15,115 @@
 #include "oshmem/proc/proc.h"
 #include "oshmem/mca/memheap/memheap.h"
 #include "oshmem/mca/memheap/base/base.h"
+#include "oshmem/mca/sshmem/base/base.h"
 #include "oshmem/util/oshmem_util.h"
+#include "opal/util/minmax.h"
 
 #include <stdio.h>
+#include <limits.h>
+#include <stdlib.h>
+#include <pthread.h>
 
-struct map_segment_desc {
+static int _check_perms(const char *perm);
+static int _check_non_static_segment(const map_segment_t *mem_segs,
+                                     int n_segment,
+                                     const void *start, const void *end);
+static int _check_address(void *start, void **end);
+static int _check_pathname(uint64_t inode, const char *pathname);
+
+int mca_memheap_base_static_init(mca_memheap_map_t *map)
+{
+    /* read and parse segments from /proc/self/maps */
+    int ret = OSHMEM_SUCCESS;
+    int n_segments = map->n_segments;
+    uint64_t total_mem = 0;
     void* start;
     void* end;
     char perms[8];
     uint64_t offset;
     char dev[8];
     uint64_t inode;
-    char pathname[MAXPATHLEN];
-};
+    char pathname[OPAL_PATH_MAX];
+    FILE *fp;
+    char line[1024];
+    map_segment_t *s;
 
-typedef struct memheap_static_context {
-    struct {
-        void* start;
-        void* end;
-    } mem_segs[MCA_MEMHEAP_MAX_SEGMENTS];
-    int n_segments;
-} memheap_static_context_t;
+    assert(map);
+    assert(HEAP_SEG_INDEX < map->n_segments);
 
-static memheap_static_context_t memheap_context;
+    /* FIXME!!! Linux specific code */
+    fp = fopen("/proc/self/maps", "r");
+    if (NULL == fp) {
+        MEMHEAP_ERROR("Failed to open /proc/self/maps");
+        return OSHMEM_ERROR;
+    }
 
-static int _load_segments(void);
-static int _check_perms(struct map_segment_desc *seg);
-static int _check_address(struct map_segment_desc *seg);
-static int _check_pathname(struct map_segment_desc *seg);
+    while (NULL != fgets(line, sizeof(line), fp)) {
+        if (3 > sscanf(line,
+               "%llx-%llx %s %llx %s %llx %s",
+               (unsigned long long *) &start,
+               (unsigned long long *) &end,
+               perms,
+               (unsigned long long *) &offset,
+               dev,
+               (unsigned long long *) &inode,
+               pathname)) {
+            MEMHEAP_ERROR("Failed to sscanf /proc/self/maps output %s", line);
+            ret = OSHMEM_ERROR;
+            goto out;
+        }
 
-int mca_memheap_base_static_init(mca_memheap_map_t *map)
-{
-    /* read and parse segments from /proc/self/maps */
-    int ret = OSHMEM_SUCCESS;
+        if (OSHMEM_ERROR == _check_non_static_segment(
+                                                   map->mem_segs, n_segments,
+                                                   start, end)) {
+            continue;
+        }
 
-    assert(map);
-    assert(HEAP_SEG_INDEX < map->n_segments);
+        if (OSHMEM_ERROR == _check_address(start, &end))
+            continue;
 
-    ret = _load_segments();
+        if (OSHMEM_ERROR == _check_pathname(inode, pathname))
+            continue;
 
-    if (OSHMEM_SUCCESS == ret) {
-        int i;
-        size_t total_mem;
+        if (OSHMEM_ERROR == _check_perms(perms))
+            continue;
 
-        for (i = 0, total_mem = 0; i < memheap_context.n_segments; i++) {
-            map_segment_t *s = &map->mem_segs[map->n_segments];
+        MEMHEAP_VERBOSE(5, "add: %s", line);
 
-            memset(s, 0, sizeof(*s));
-            MAP_SEGMENT_RESET_FLAGS(s);
-            s->seg_id = MAP_SEGMENT_SHM_INVALID;
-            s->super.va_base = memheap_context.mem_segs[i].start;
-            s->super.va_end  = memheap_context.mem_segs[i].end;
+        if ((map->n_segments > 0) &&
+            (start == map->mem_segs[map->n_segments - 1].super.va_end)) {
+            s = &map->mem_segs[map->n_segments - 1];
+            MEMHEAP_VERBOSE(5, "Coalescing segment");
+            s->super.va_end = end;
             s->seg_size = ((uintptr_t)s->super.va_end - (uintptr_t)s->super.va_base);
-            s->type = MAP_SEGMENT_STATIC;
-            map->n_segments++;
+            continue;
+        }
 
-            total_mem += ((uintptr_t)s->super.va_end - (uintptr_t)s->super.va_base);
+        s = mca_memheap_base_allocate_segment(map);
+        if (NULL == s) {
+            MEMHEAP_ERROR("failed to allocate segment");
+            ret = OSHMEM_ERR_OUT_OF_RESOURCE;
+            goto out;
         }
-        MEMHEAP_VERBOSE(1,
-                        "Memheap static memory: %llu byte(s), %d segments",
-                        (unsigned long long)total_mem, map->n_segments);
+
+        memset(s, 0, sizeof(*s));
+        MAP_SEGMENT_RESET_FLAGS(s);
+        s->seg_id        = MAP_SEGMENT_SHM_INVALID;
+        s->super.va_base = start;
+        s->super.va_end  = end;
+        s->seg_size      = ((uintptr_t)s->super.va_end - (uintptr_t)s->super.va_base);
+        s->type          = MAP_SEGMENT_STATIC;
+        map->n_segments++;
+
+        total_mem += ((uintptr_t)s->super.va_end - (uintptr_t)s->super.va_base);
     }
 
+    MEMHEAP_VERBOSE(1,
+                    "Memheap static memory: %llu byte(s), %d segments",
+                    total_mem, map->n_segments);
+
+out:
+    fclose(fp);
     return ret;
 }
 
@@ -84,50 +132,102 @@ void mca_memheap_base_static_exit(mca_memheap_map_t *map)
     assert(map);
 }
 
-static int _check_perms(struct map_segment_desc *seg)
+static int _check_perms(const char *perms)
 {
-    if (!strcmp(seg->perms, "rw-p") || !strcmp(seg->perms, "rwxp"))
+    if (!strcmp(perms, "rw-p") || !strcmp(perms, "rwxp"))
         return OSHMEM_SUCCESS;
 
     return OSHMEM_ERROR;
 }
 
-static int _check_address(struct map_segment_desc *seg)
+static int _check_non_static_segment(const map_segment_t *mem_segs,
+                                     int n_segment,
+                                     const void *start, const void *end)
+{
+    int i;
+
+    for (i = 0; i < n_segment; i++) {
+        if ((start <= mem_segs[i].super.va_base) &&
+            (mem_segs[i].super.va_base < end)) {
+            MEMHEAP_VERBOSE(100,
+                            "non static segment: %p-%p already exists as %p-%p",
+                            start, end, mem_segs[i].super.va_base,
+                            mem_segs[i].super.va_end);
+            return OSHMEM_ERROR;
+        }
+    }
+
+    return OSHMEM_SUCCESS;
+}
+
+static int _check_address(void *start, void **end)
 {
     /* FIXME Linux specific code */
 #ifdef __linux__
     extern unsigned _end;
-    void* data_end = &_end;
+    uintptr_t data_end = (uintptr_t)&_end;
 
     /**
      * SGI shmem only supports globals&static in main program.
      * It does not support them in shared objects or in dlopen()
-     * (Clarified on PGAS 2011 tutorial)
+     * (Clarified on PGAS 2011 tutorial).
      *
-     * So ignored any maps that start higher then process _end
-     * FIXME: make sure we do not register symmetric heap twice
-     * if we decide to allow shared objects
+     * So ignored any maps that start higher then process _end.
      */
-    if ((uintptr_t)seg->start > (uintptr_t)data_end) {
+    if ((uintptr_t)start > data_end) {
         MEMHEAP_VERBOSE(100,
                         "skip segment: data _end < segment start (%p < %p)",
-                        data_end, seg->start);
+                        data_end, start);
         return OSHMEM_ERROR;
     }
 
-    if ((uintptr_t)seg->end > (uintptr_t)data_end) {
+    if ((uintptr_t)*end > data_end) {
         MEMHEAP_VERBOSE(100,
                         "adjust segment: data _end < segment end (%p < %p",
-                        data_end, seg->end);
-         seg->end = data_end;
+                        data_end, *end);
+         *end = (void*)data_end;
     }
 #endif
     return OSHMEM_SUCCESS;
 }
 
-static int _check_pathname(struct map_segment_desc *seg)
+static int _check_pathname(uint64_t inode, const char *pathname)
 {
-    /* Probably we need to check found path but
+    static const char *proc_self_exe = "/proc/self/exe";
+    static int warned = 0;
+    char exe_path[OPAL_PATH_MAX];
+    char module_path[OPAL_PATH_MAX];
+    char *path;
+
+    if (0 == inode) {
+        /* segment is not mapped to file, allow sharing it */
+        return OSHMEM_SUCCESS;
+    }
+
+    path = realpath(proc_self_exe, exe_path);
+    if (NULL == path) {
+        if (0 == warned) {
+            MEMHEAP_VERBOSE(100, "failed to read link %s: %m", proc_self_exe);
+            MEMHEAP_VERBOSE(100, "all segments will be registered");
+            warned = 1;
+        }
+
+        return OSHMEM_SUCCESS;
+    }
+
+    /* for file-mapped segments allow segments from start process only */
+    path = realpath(pathname, module_path);
+    if (NULL == path) {
+        return OSHMEM_ERROR;
+    }
+
+    if (!strncmp(exe_path, module_path, sizeof(exe_path))) {
+        return OSHMEM_SUCCESS;
+    }
+
+    return OSHMEM_ERROR;
+
+    /* Probably we need more accurate path check
      * To press check coverity issue following code is disabled
      */
 #if 0
@@ -168,66 +268,3 @@ static int _check_pathname(struct map_segment_desc *seg)
     return OSHMEM_SUCCESS;
 }
 
-static int _load_segments(void)
-{
-    FILE *fp;
-    char line[1024];
-    struct map_segment_desc seg;
-
-    memheap_context.n_segments = 0;
-    /* FIXME!!! Linux specific code */
-    fp = fopen("/proc/self/maps", "r");
-    if (NULL == fp) {
-        MEMHEAP_ERROR("Failed to open /proc/self/maps");
-        return OSHMEM_ERROR;
-    }
-
-    while (NULL != fgets(line, sizeof(line), fp)) {
-        memset(&seg, 0, sizeof(seg));
-        if (3 > sscanf(line,
-               "%llx-%llx %s %llx %s %llx %s",
-               (unsigned long long *) &seg.start,
-               (unsigned long long *) &seg.end,
-               seg.perms,
-               (unsigned long long *) &seg.offset,
-               seg.dev,
-               (unsigned long long *) &seg.inode,
-               seg.pathname)) {
-            MEMHEAP_ERROR("Failed to sscanf /proc/self/maps output %s", line);
-            fclose(fp);
-            return OSHMEM_ERROR;
-        }
-
-        if (OSHMEM_ERROR == _check_address(&seg))
-            continue;
-
-        if (OSHMEM_ERROR == _check_pathname(&seg))
-            continue;
-
-        if (OSHMEM_ERROR == _check_perms(&seg))
-            continue;
-
-        MEMHEAP_VERBOSE(5, "add: %s", line);
-        if (MCA_MEMHEAP_MAX_SEGMENTS <= memheap_context.n_segments) {
-            MEMHEAP_ERROR("too many segments (max = %d): skip %s",
-                          MCA_MEMHEAP_MAX_SEGMENTS, line);
-            continue;
-        }
-        if (memheap_context.n_segments > 0
-                && seg.start
-                        == memheap_context.mem_segs[memheap_context.n_segments
-                                - 1].end) {
-            MEMHEAP_VERBOSE(5, "Coalescing segment");
-            memheap_context.mem_segs[memheap_context.n_segments - 1].end =
-                    seg.end;
-        } else {
-            memheap_context.mem_segs[memheap_context.n_segments].start =
-                    seg.start;
-            memheap_context.mem_segs[memheap_context.n_segments].end = seg.end;
-            memheap_context.n_segments++;
-        }
-    }
-
-    fclose(fp);
-    return OSHMEM_SUCCESS;
-}
diff --git a/oshmem/mca/memheap/buddy/memheap_buddy_component.c b/oshmem/mca/memheap/buddy/memheap_buddy_component.c
index 2a282e5f950..715f7743374 100644
--- a/oshmem/mca/memheap/buddy/memheap_buddy_component.c
+++ b/oshmem/mca/memheap/buddy/memheap_buddy_component.c
@@ -57,7 +57,7 @@ mca_memheap_buddy_component_query(mca_base_module_t **module, int *priority)
 }
 
 /*
- * This function is automaticaly called from mca_base_components_close.
+ * This function is automatically called from mca_base_components_close.
  * It releases the component's allocated memory.
  */
 int mca_memheap_buddy_component_close()
diff --git a/oshmem/mca/memheap/buddy/memheap_buddy_component.h b/oshmem/mca/memheap/buddy/memheap_buddy_component.h
index 177edcb6bf4..8a4e177381f 100644
--- a/oshmem/mca/memheap/buddy/memheap_buddy_component.h
+++ b/oshmem/mca/memheap/buddy/memheap_buddy_component.h
@@ -19,7 +19,7 @@ BEGIN_C_DECLS
 /*
  * MEMHEAP module functions.
  */
-OSHMEM_MODULE_DECLSPEC extern mca_memheap_base_component_2_0_0_t mca_memheap_buddy_component;
+OSHMEM_DECLSPEC extern mca_memheap_base_component_2_0_0_t mca_memheap_buddy_component;
 
 END_C_DECLS
 
diff --git a/oshmem/mca/memheap/memheap.h b/oshmem/mca/memheap/memheap.h
index 07c4e2f2f05..46fe41980fb 100644
--- a/oshmem/mca/memheap/memheap.h
+++ b/oshmem/mca/memheap/memheap.h
@@ -88,7 +88,7 @@ typedef struct mca_memheap_base_component_2_0_0_t mca_memheap_base_component_t;
  * memheap module descriptor
  */
 struct mca_memheap_base_module_t {
-    mca_memheap_base_component_t                   *memheap_component;  /** Memory Heap Management Componenet */
+    mca_memheap_base_component_t                   *memheap_component;  /** Memory Heap Management Component */
     mca_memheap_base_module_finalize_fn_t           memheap_finalize;
     mca_memheap_base_module_alloc_fn_t              memheap_alloc;
     mca_memheap_base_module_memalign_fn_t           memheap_memalign;
diff --git a/oshmem/mca/memheap/ptmalloc/malloc.c b/oshmem/mca/memheap/ptmalloc/malloc.c
index 3551d8cd610..0b11a6fb327 100644
--- a/oshmem/mca/memheap/ptmalloc/malloc.c
+++ b/oshmem/mca/memheap/ptmalloc/malloc.c
@@ -1263,9 +1263,6 @@ extern void*     sbrk(ptrdiff_t);
 #if USE_LOCKS
 #ifndef WIN32
 #include <pthread.h>
-#if defined (__SVR4) && defined (__sun)  /* solaris */
-#include <thread.h>
-#endif /* solaris */
 #else
 #ifndef _M_AMD64
 /* These are already defined on AMD64 builds */
@@ -1443,7 +1440,7 @@ static FORCEINLINE void* win32direct_mmap(size_t size) {
   return (ptr != 0)? ptr: MFAIL;
 }
 
-/* This function supports releasing coalesed segments */
+/* This function supports releasing coalesced segments */
 static FORCEINLINE int win32munmap(void* ptr, size_t size) {
   MEMORY_BASIC_INFORMATION minfo;
   char* cptr = (char*)ptr;
@@ -1480,7 +1477,7 @@ static FORCEINLINE int win32munmap(void* ptr, size_t size) {
 #define CALL_MORECORE(S)     MFAIL
 #endif /* HAVE_MORECORE */
 
-/* mstate bit set if continguous morecore disabled or failed */
+/* mstate bit set if contiguous morecore disabled or failed */
 #define USE_NONCONTIGUOUS_BIT (4U)
 
 /* segment bit set in create_mspace_with_base */
@@ -1544,15 +1541,11 @@ static FORCEINLINE int pthread_acquire_lock (MLOCK_T *sl) {
         break;
       }
       if ((++spins & SPINS_PER_YIELD) == 0) {
-#if defined (__SVR4) && defined (__sun) /* solaris */
-        thr_yield();
-#else
 #ifdef linux
         sched_yield();
 #else  /* no-op yield on unknown systems */
         ;
 #endif /* linux */
-#endif /* solaris */
       }
     }
   }
@@ -2094,7 +2087,7 @@ nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   work in the same way as fd/bk pointers of small chunks.
 
   Each tree contains a power of 2 sized range of chunk sizes (the
-  smallest is 0x100 <= x < 0x180), which is is divided in half at each
+  smallest is 0x100 <= x < 0x180), which is divided in half at each
   tree level, with the chunks in the smaller half of the range (0x100
   <= x < 0x140 for the top nose) in the left subtree and the larger
   half (0x140 <= x < 0x180) in the right subtree.  This is, of course,
@@ -2284,7 +2277,7 @@ typedef struct malloc_segment* msegmentptr;
 
   Trim support
     Fields holding the amount of unused topmost memory that should trigger
-    timming, and a counter to force periodic scanning to release unused
+    timing, and a counter to force periodic scanning to release unused
     non-topmost segments.
 
   Locking
@@ -2692,7 +2685,7 @@ static size_t traverse_and_check(mstate m);
   http://www.usenix.org/events/lisa03/tech/robertson.html The footer
   of an inuse chunk holds the xor of its mstate and a random seed,
   that is checked upon calls to free() and realloc().  This is
-  (probablistically) unguessable from outside the program, but can be
+  (probabilistically) unguessable from outside the program, but can be
   computed by any code successfully malloc'ing any chunk, so does not
   itself provide protection against code that has already broken
   security through some other means.  Unlike Robertson et al, we
@@ -4539,7 +4532,7 @@ void* dlmalloc(size_t bytes) {
 
 void dlfree(void* mem) {
   /*
-     Consolidate freed chunks with preceeding or succeeding bordering
+     Consolidate freed chunks with preceding or succeeding bordering
      free chunks, if they exist, and then place in a bin.  Intermixed
      with special cases for top, dv, mmapped chunks, and usage errors.
   */
@@ -5430,10 +5423,10 @@ int mspace_mallopt(int param_number, int value) {
         Wolfram Gloger (Gloger@lrz.uni-muenchen.de).
       * Use last_remainder in more cases.
       * Pack bins using idea from  colin@nyx10.cs.du.edu
-      * Use ordered bins instead of best-fit threshhold
+      * Use ordered bins instead of best-fit threshold
       * Eliminate block-local decls to simplify tracing and debugging.
       * Support another case of realloc via move into top
-      * Fix error occuring when initial sbrk_base not word-aligned.
+      * Fix error occurring when initial sbrk_base not word-aligned.
       * Rely on page size for units instead of SBRK_UNIT to
         avoid surprises about sbrk alignment conventions.
       * Add mallinfo, mallopt. Thanks to Raymond Nijssen
diff --git a/oshmem/mca/memheap/ptmalloc/memheap_ptmalloc_component.c b/oshmem/mca/memheap/ptmalloc/memheap_ptmalloc_component.c
index e390a48aa3e..2e13e8cf995 100644
--- a/oshmem/mca/memheap/ptmalloc/memheap_ptmalloc_component.c
+++ b/oshmem/mca/memheap/ptmalloc/memheap_ptmalloc_component.c
@@ -58,7 +58,7 @@ mca_memheap_ptmalloc_component_query(mca_base_module_t **module, int *priority)
 }
 
 /*
- * This function is automaticaly called from mca_base_components_close.
+ * This function is automatically called from mca_base_components_close.
  * It releases the component's allocated memory.
  */
 int mca_memheap_ptmalloc_component_close()
diff --git a/oshmem/mca/memheap/ptmalloc/memheap_ptmalloc_component.h b/oshmem/mca/memheap/ptmalloc/memheap_ptmalloc_component.h
index 0f4a0053dbc..9d607aa32ec 100644
--- a/oshmem/mca/memheap/ptmalloc/memheap_ptmalloc_component.h
+++ b/oshmem/mca/memheap/ptmalloc/memheap_ptmalloc_component.h
@@ -19,7 +19,7 @@ BEGIN_C_DECLS
 /*
  * MEMHEAP module functions.
  */
-OSHMEM_MODULE_DECLSPEC extern mca_memheap_base_component_2_0_0_t mca_memheap_ptmalloc_component;
+OSHMEM_DECLSPEC extern mca_memheap_base_component_2_0_0_t mca_memheap_ptmalloc_component;
 
 END_C_DECLS
 
diff --git a/oshmem/mca/scoll/base/base.h b/oshmem/mca/scoll/base/base.h
index 08af8e197b5..854f8cebfb2 100644
--- a/oshmem/mca/scoll/base/base.h
+++ b/oshmem/mca/scoll/base/base.h
@@ -52,7 +52,7 @@ OSHMEM_DECLSPEC extern mca_base_framework_t oshmem_scoll_base_framework;
 #define __SCOLL_FILE__ __FILE__
 #endif
 
-#ifdef OPAL_ENABLE_DEBUG
+#if OPAL_ENABLE_DEBUG
 #define SCOLL_VERBOSE(level, ...) \
     oshmem_output_verbose(level, oshmem_scoll_base_framework.framework_output, \
        "%s:%d - %s()", __SCOLL_FILE__, __LINE__, __func__, __VA_ARGS__)
diff --git a/oshmem/mca/scoll/basic/scoll_basic.h b/oshmem/mca/scoll/basic/scoll_basic.h
index 066ff6cacdd..73365c62b4e 100644
--- a/oshmem/mca/scoll/basic/scoll_basic.h
+++ b/oshmem/mca/scoll/basic/scoll_basic.h
@@ -32,7 +32,7 @@ BEGIN_C_DECLS
 
 /* Globally exported variables */
 
-OSHMEM_MODULE_DECLSPEC extern mca_scoll_base_component_1_0_0_t
+OSHMEM_DECLSPEC extern mca_scoll_base_component_1_0_0_t
 mca_scoll_basic_component;
 
 extern int mca_scoll_basic_priority_param;
diff --git a/oshmem/mca/scoll/basic/scoll_basic_alltoall.c b/oshmem/mca/scoll/basic/scoll_basic_alltoall.c
index 6a87e85578f..f539e5d0e1f 100644
--- a/oshmem/mca/scoll/basic/scoll_basic_alltoall.c
+++ b/oshmem/mca/scoll/basic/scoll_basic_alltoall.c
@@ -116,7 +116,7 @@ get_dst_pe(struct oshmem_group_t *group, int src_blk_idx, int dst_blk_idx, int *
     (*dst_pe_idx) = (dst_blk_idx + src_blk_idx) % group->proc_count;
 
     /* convert to the global pe */
-    return oshmem_proc_pe(group->proc_array[*dst_pe_idx]);
+    return oshmem_proc_pe_vpid(group, *dst_pe_idx);
 }
 
 static int a2as_alg_simple(struct oshmem_group_t *group,
diff --git a/oshmem/mca/scoll/basic/scoll_basic_barrier.c b/oshmem/mca/scoll/basic/scoll_basic_barrier.c
index e4d0ee23d3d..fd60d10c28c 100644
--- a/oshmem/mca/scoll/basic/scoll_basic_barrier.c
+++ b/oshmem/mca/scoll/basic/scoll_basic_barrier.c
@@ -103,7 +103,7 @@ static int _algorithm_central_counter(struct oshmem_group_t *group,
     int rc = OSHMEM_SUCCESS;
     long value = SHMEM_SYNC_INIT;
     int root_id = 0;
-    int PE_root = oshmem_proc_pe(group->proc_array[root_id]);
+    int PE_root = oshmem_proc_pe_vpid(group, root_id);
     int i = 0;
 
     SCOLL_VERBOSE(12, "[#%d] Barrier algorithm: Central Counter", group->my_pe);
@@ -124,7 +124,7 @@ static int _algorithm_central_counter(struct oshmem_group_t *group,
 
             wait_pe_count = group->proc_count;
             for (i = 0; i < group->proc_count; i++) {
-                wait_pe_array[i] = oshmem_proc_pe(group->proc_array[i]);
+                wait_pe_array[i] = oshmem_proc_pe_vpid(group, i);
             }
             wait_pe_array[root_id] = OSHMEM_PE_INVALID;
             wait_pe_count--;
@@ -151,7 +151,7 @@ static int _algorithm_central_counter(struct oshmem_group_t *group,
             value = SHMEM_SYNC_RUN;
             for (i = 0; (i < group->proc_count) && (rc == OSHMEM_SUCCESS);
                     i++) {
-                pe_cur = oshmem_proc_pe(group->proc_array[i]);
+                pe_cur = oshmem_proc_pe_vpid(group, i);
                 if (pe_cur != PE_root) {
                     rc = MCA_SPML_CALL(put(oshmem_ctx_default, (void*)pSync, sizeof(value), (void*)&value, pe_cur));
                 }
@@ -238,7 +238,7 @@ static int _algorithm_tournament(struct oshmem_group_t *group, long *pSync)
             SCOLL_VERBOSE(14, "[#%d] round = %d wait", group->my_pe, round);
             rc = MCA_SPML_CALL(wait((void*)pSync, SHMEM_CMP_EQ, (void*)&value, SHMEM_LONG));
         } else {
-            peer_pe = oshmem_proc_pe(group->proc_array[peer_id]);
+            peer_pe = oshmem_proc_pe_vpid(group, peer_id);
 
 #if 1 /* It is ugly implementation of compare and swap operation
          Usage of this hack does not give performance improvement but
@@ -284,7 +284,7 @@ static int _algorithm_tournament(struct oshmem_group_t *group, long *pSync)
         for (peer_id = 1;
                 (peer_id < group->proc_count) && (rc == OSHMEM_SUCCESS);
                 peer_id++) {
-            peer_pe = oshmem_proc_pe(group->proc_array[peer_id]);
+            peer_pe = oshmem_proc_pe_vpid(group, peer_id);
             rc = MCA_SPML_CALL(put(oshmem_ctx_default, (void*)pSync, sizeof(value), (void*)&value, peer_pe));
         }
     }
@@ -333,7 +333,7 @@ static int _algorithm_recursive_doubling(struct oshmem_group_t *group,
     if (my_id >= floor2_proc) {
         /* I am in extra group, my partner is node (my_id-y) in basic group */
         peer_id = my_id - floor2_proc;
-        peer_pe = oshmem_proc_pe(group->proc_array[peer_id]);
+        peer_pe = oshmem_proc_pe_vpid(group, peer_id);
 
         SCOLL_VERBOSE(14,
                       "[#%d] is extra and signal to #%d",
@@ -357,7 +357,7 @@ static int _algorithm_recursive_doubling(struct oshmem_group_t *group,
         if ((group->proc_count - floor2_proc) > my_id) {
             /* I am in basic group, my partner is node (my_id+y) in extra group */
             peer_id = my_id + floor2_proc;
-            peer_pe = oshmem_proc_pe(group->proc_array[peer_id]);
+            peer_pe = oshmem_proc_pe_vpid(group, peer_id);
 
             SCOLL_VERBOSE(14,
                           "[#%d] wait a signal from #%d",
@@ -376,8 +376,7 @@ static int _algorithm_recursive_doubling(struct oshmem_group_t *group,
             /* Update exit condition and round counter */
             exit_flag >>= 1;
             round++;
-
-            peer_pe = oshmem_proc_pe(group->proc_array[peer_id]);
+            peer_pe = oshmem_proc_pe_vpid(group, peer_id);
 
 #if 1 /* It is ugly implementation of compare and swap operation
          Usage of this hack does not give performance improvement but
@@ -420,7 +419,7 @@ static int _algorithm_recursive_doubling(struct oshmem_group_t *group,
         if ((group->proc_count - floor2_proc) > my_id) {
             /* I am in basic group, my partner is node (my_id+y) in extra group */
             peer_id = my_id + floor2_proc;
-            peer_pe = oshmem_proc_pe(group->proc_array[peer_id]);
+            peer_pe = oshmem_proc_pe_vpid(group, peer_id);
 
             SCOLL_VERBOSE(14, "[#%d] signals to #%d", group->my_pe, peer_pe);
             value = SHMEM_SYNC_RUN;
@@ -462,8 +461,7 @@ static int _algorithm_dissemination(struct oshmem_group_t *group, long *pSync)
     for (round = 0; (round <= log2_proc) && (rc == OSHMEM_SUCCESS); round++) {
         /* Define a peer to send signal */
         peer_id = (my_id + (1 << round)) % group->proc_count;
-
-        peer_pe = oshmem_proc_pe(group->proc_array[peer_id]);
+        peer_pe = oshmem_proc_pe_vpid(group, peer_id);
 
 #if 1 /* It is ugly implementation of compare and swap operation
          Usage of this hack does not give performance improvement but
@@ -502,7 +500,7 @@ static int _algorithm_basic(struct oshmem_group_t *group, long *pSync)
 {
     int rc = OSHMEM_SUCCESS;
     int root_id = 0;
-    int PE_root = oshmem_proc_pe(group->proc_array[root_id]);
+    int PE_root = oshmem_proc_pe_vpid(group, root_id);
     int i = 0;
 
     SCOLL_VERBOSE(12, "[#%d] Barrier algorithm: Basic", group->my_pe);
@@ -525,7 +523,7 @@ static int _algorithm_basic(struct oshmem_group_t *group, long *pSync)
         int pe_cur = 0;
 
         for (i = 0; (i < group->proc_count) && (rc == OSHMEM_SUCCESS); i++) {
-            pe_cur = oshmem_proc_pe(group->proc_array[i]);
+            pe_cur = oshmem_proc_pe_vpid(group, i);
             if (pe_cur != PE_root) {
                 rc = MCA_SPML_CALL(recv(NULL, 0, pe_cur)); 
             }
@@ -535,7 +533,7 @@ static int _algorithm_basic(struct oshmem_group_t *group, long *pSync)
         }
 
         for (i = 0; (i < group->proc_count) && (rc == OSHMEM_SUCCESS); i++) {
-            pe_cur = oshmem_proc_pe(group->proc_array[i]);
+            pe_cur = oshmem_proc_pe_vpid(group, i);
             if (pe_cur != PE_root) {
                 rc = MCA_SPML_CALL(send(NULL, 0, pe_cur, MCA_SPML_BASE_PUT_STANDARD));
             }
@@ -564,7 +562,7 @@ static int _algorithm_adaptive(struct oshmem_group_t *group, long *pSync)
             if (i == my_id)
                 continue;
 
-            if (!OPAL_PROC_ON_LOCAL_NODE(group->proc_array[i]->super.proc_flags)) {
+            if (!oshmem_proc_on_local_node(i)) {
                 local_peers_only = false;
                 break;
             }
diff --git a/oshmem/mca/scoll/basic/scoll_basic_broadcast.c b/oshmem/mca/scoll/basic/scoll_basic_broadcast.c
index 5ba4686eacc..b07e379ff85 100644
--- a/oshmem/mca/scoll/basic/scoll_basic_broadcast.c
+++ b/oshmem/mca/scoll/basic/scoll_basic_broadcast.c
@@ -144,7 +144,7 @@ static int _algorithm_central_counter(struct oshmem_group_t *group,
                       "[#%d] send data to all PE in the group",
                       group->my_pe);
         for (i = 0; (i < group->proc_count) && (rc == OSHMEM_SUCCESS); i++) {
-            pe_cur = oshmem_proc_pe(group->proc_array[i]);
+            pe_cur = oshmem_proc_pe_vpid(group, i);
             if (pe_cur != PE_root) {
                 SCOLL_VERBOSE(15,
                               "[#%d] send data to #%d",
@@ -233,7 +233,7 @@ static int _algorithm_binomial_tree(struct oshmem_group_t *group,
         if (peer_id < group->proc_count) {
             /* Wait for the child to be ready to receive (pSync must have the initial value) */
             peer_id = (peer_id + root_id) % group->proc_count;
-            peer_pe = oshmem_proc_pe(group->proc_array[peer_id]);
+            peer_pe = oshmem_proc_pe_vpid(group, peer_id);
 
             SCOLL_VERBOSE(14,
                           "[#%d] check remote pe is ready to receive #%d",
diff --git a/oshmem/mca/scoll/basic/scoll_basic_collect.c b/oshmem/mca/scoll/basic/scoll_basic_collect.c
index 5689780d7f1..4631bfeb7b0 100644
--- a/oshmem/mca/scoll/basic/scoll_basic_collect.c
+++ b/oshmem/mca/scoll/basic/scoll_basic_collect.c
@@ -155,7 +155,7 @@ static int _algorithm_f_central_counter(struct oshmem_group_t *group,
 {
     int rc = OSHMEM_SUCCESS;
     int i = 0;
-    int PE_root = oshmem_proc_pe(group->proc_array[0]);
+    int PE_root = oshmem_proc_pe_vpid(group, 0);
 
     SCOLL_VERBOSE(12,
                   "[#%d] Collect algorithm: Central Counter (identical size)",
@@ -174,7 +174,7 @@ static int _algorithm_f_central_counter(struct oshmem_group_t *group,
                       group->my_pe);
         for (i = 0; (i < group->proc_count) && (rc == OSHMEM_SUCCESS); i++) {
             /* Get PE ID of a peer from the group */
-            pe_cur = oshmem_proc_pe(group->proc_array[i]);
+            pe_cur = oshmem_proc_pe_vpid(group, i);
 
             if (pe_cur == group->my_pe)
                 continue;
@@ -221,7 +221,7 @@ static int _algorithm_f_tournament(struct oshmem_group_t *group,
     int my_id = oshmem_proc_group_find_id(group, group->my_pe);
     int peer_id = 0;
     int peer_pe = 0;
-    int PE_root = oshmem_proc_pe(group->proc_array[0]);
+    int PE_root = oshmem_proc_pe_vpid(group, 0);
 
     SCOLL_VERBOSE(12,
                   "[#%d] Collect algorithm: Tournament (identical size)",
@@ -255,7 +255,7 @@ static int _algorithm_f_tournament(struct oshmem_group_t *group,
             SCOLL_VERBOSE(14, "[#%d] round = %d wait", group->my_pe, round);
             rc = MCA_SPML_CALL(wait((void*)pSync, SHMEM_CMP_EQ, (void*)&value, SHMEM_LONG));
         } else {
-            peer_pe = oshmem_proc_pe(group->proc_array[peer_id]);
+            peer_pe = oshmem_proc_pe_vpid(group, peer_id);
 
 #if 1 /* It is ugly implementation of compare and swap operation
          Usage of this hack does not give performance improvement but
@@ -294,7 +294,7 @@ static int _algorithm_f_tournament(struct oshmem_group_t *group,
         for (peer_id = 1;
                 (peer_id < group->proc_count) && (rc == OSHMEM_SUCCESS);
                 peer_id++) {
-            peer_pe = oshmem_proc_pe(group->proc_array[peer_id]);
+            peer_pe = oshmem_proc_pe_vpid(group, peer_id);
             rc = MCA_SPML_CALL(put(oshmem_ctx_default, (void*)pSync, sizeof(value), (void*)&value, peer_pe));
         }
     }
@@ -339,7 +339,7 @@ static int _algorithm_f_ring(struct oshmem_group_t *group,
     SCOLL_VERBOSE(15, "[#%d] pSync[0] = %ld", group->my_pe, pSync[0]);
 
     peer_id = (my_id + 1) % group->proc_count;
-    peer_pe = oshmem_proc_pe(group->proc_array[peer_id]);
+    peer_pe = oshmem_proc_pe_vpid(group, peer_id);
     memcpy((void*) ((unsigned char*) target + my_id * nlong),
            (void *) source,
            nlong);
@@ -420,13 +420,12 @@ static int _algorithm_f_recursive_doubling(struct oshmem_group_t *group,
 
         /* I am in extra group, my partner is node (my_id-y) in basic group */
         peer_id = my_id - floor2_proc;
-        peer_pe = oshmem_proc_pe(group->proc_array[peer_id]);
+        peer_pe = oshmem_proc_pe_vpid(group, peer_id);
 
         for (i = 0; (i < group->proc_count) && (rc == OSHMEM_SUCCESS); i++) {
             if (i == my_id)
                 continue;
-
-            pe_cur = oshmem_proc_pe(group->proc_array[i]);
+            pe_cur = oshmem_proc_pe_vpid(group, i);
 
             SCOLL_VERBOSE(14,
                           "[#%d] is extra send data to #%d",
@@ -450,7 +449,7 @@ static int _algorithm_f_recursive_doubling(struct oshmem_group_t *group,
         if ((group->proc_count - floor2_proc) > my_id) {
             /* I am in basic group, my partner is node (my_id+y) in extra group */
             peer_id = my_id + floor2_proc;
-            peer_pe = oshmem_proc_pe(group->proc_array[peer_id]);
+            peer_pe = oshmem_proc_pe_vpid(group, peer_id);
 
             SCOLL_VERBOSE(14,
                           "[#%d] wait a signal from #%d",
@@ -469,8 +468,7 @@ static int _algorithm_f_recursive_doubling(struct oshmem_group_t *group,
             /* Update exit condition and round counter */
             exit_flag >>= 1;
             round++;
-
-            peer_pe = oshmem_proc_pe(group->proc_array[peer_id]);
+            peer_pe = oshmem_proc_pe_vpid(group, peer_id);
 
 #if 1 /* It is ugly implementation of compare and swap operation
          Usage of this hack does not give performance improvement but
@@ -507,7 +505,7 @@ static int _algorithm_f_recursive_doubling(struct oshmem_group_t *group,
         if ((group->proc_count - floor2_proc) > my_id) {
             /* I am in basic group, my partner is node (my_id+y) in extra group */
             peer_id = my_id + floor2_proc;
-            peer_pe = oshmem_proc_pe(group->proc_array[peer_id]);
+            peer_pe = oshmem_proc_pe_vpid(group, peer_id);
 
             SCOLL_VERBOSE(14,
                           "[#%d] is extra send data to #%d",
@@ -542,7 +540,7 @@ static int _algorithm_central_collector(struct oshmem_group_t *group,
     int rc = OSHMEM_SUCCESS;
     size_t offset = 0;
     int i = 0;
-    int PE_root = oshmem_proc_pe(group->proc_array[0]);
+    int PE_root = oshmem_proc_pe_vpid(group, 0);
 
     SCOLL_VERBOSE(12,
                   "[#%d] Collect algorithm: Central Counter (vary size)",
@@ -573,7 +571,7 @@ static int _algorithm_central_collector(struct oshmem_group_t *group,
                 for (i = 1; (i < group->proc_count) && (rc == OSHMEM_SUCCESS);
                         i++) {
                     if (wait_pe_array[i] == 0) {
-                        pe_cur = oshmem_proc_pe(group->proc_array[i]);
+                        pe_cur = oshmem_proc_pe_vpid(group, i);
                         value = 0;
                         rc = MCA_SPML_CALL(get(oshmem_ctx_default, (void*)pSync, sizeof(value), (void*)&value, pe_cur));
                         if ((rc == OSHMEM_SUCCESS)
@@ -602,7 +600,7 @@ static int _algorithm_central_collector(struct oshmem_group_t *group,
                 }
 
                 /* Get PE ID of a peer from the group */
-                pe_cur = oshmem_proc_pe(group->proc_array[i]);
+                pe_cur = oshmem_proc_pe_vpid(group, i);
 
                 /* Get data from the current peer */
                 rc = MCA_SPML_CALL(get(oshmem_ctx_default, (void *)source, (size_t)wait_pe_array[i], (void*)((unsigned char*)target + offset), pe_cur));
diff --git a/oshmem/mca/scoll/basic/scoll_basic_reduce.c b/oshmem/mca/scoll/basic/scoll_basic_reduce.c
index b8ecb9e7daf..a7837c4d872 100644
--- a/oshmem/mca/scoll/basic/scoll_basic_reduce.c
+++ b/oshmem/mca/scoll/basic/scoll_basic_reduce.c
@@ -186,7 +186,7 @@ static int _algorithm_central_counter(struct oshmem_group_t *group,
 {
     int rc = OSHMEM_SUCCESS;
     int i = 0;
-    int PE_root = oshmem_proc_pe(group->proc_array[0]);
+    int PE_root = oshmem_proc_pe_vpid(group, 0);
 
     SCOLL_VERBOSE(12, "[#%d] Reduce algorithm: Central Counter", group->my_pe);
 
@@ -204,7 +204,7 @@ static int _algorithm_central_counter(struct oshmem_group_t *group,
             for (i = 0; (i < group->proc_count) && (rc == OSHMEM_SUCCESS);
                     i++) {
                 /* Get PE ID of a peer from the group */
-                pe_cur = oshmem_proc_pe(group->proc_array[i]);
+                pe_cur = oshmem_proc_pe_vpid(group, i);
 
                 if (pe_cur == group->my_pe)
                     continue;
@@ -265,7 +265,7 @@ static int _algorithm_tournament(struct oshmem_group_t *group,
     int peer_id = 0;
     int peer_pe = 0;
     void *target_cur = NULL;
-    int PE_root = oshmem_proc_pe(group->proc_array[0]);
+    int PE_root = oshmem_proc_pe_vpid(group, 0);
 
     SCOLL_VERBOSE(12, "[#%d] Reduce algorithm: Tournament", group->my_pe);
     SCOLL_VERBOSE(15, "[#%d] pSync[0] = %ld", group->my_pe, pSync[0]);
@@ -304,7 +304,7 @@ static int _algorithm_tournament(struct oshmem_group_t *group,
                 op->o_func.c_fn(target, target_cur, nlong / op->dt_size);
             }
         } else {
-            peer_pe = oshmem_proc_pe(group->proc_array[peer_id]);
+            peer_pe = oshmem_proc_pe_vpid(group, peer_id);
 
 #if 1 /* It is ugly implementation of compare and swap operation
          Usage of this hack does not give performance improvement but
@@ -345,7 +345,7 @@ static int _algorithm_tournament(struct oshmem_group_t *group,
         for (peer_id = 1;
                 (peer_id < group->proc_count) && (rc == OSHMEM_SUCCESS);
                 peer_id++) {
-            peer_pe = oshmem_proc_pe(group->proc_array[peer_id]);
+            peer_pe = oshmem_proc_pe_vpid(group, peer_id);
             rc = MCA_SPML_CALL(put(oshmem_ctx_default, (void*)pSync, sizeof(value), (void*)&value, peer_pe));
         }
     }
@@ -416,7 +416,7 @@ static int _algorithm_recursive_doubling(struct oshmem_group_t *group,
     if (my_id >= floor2_proc) {
         /* I am in extra group, my partner is node (my_id-y) in basic group */
         peer_id = my_id - floor2_proc;
-        peer_pe = oshmem_proc_pe(group->proc_array[peer_id]);
+        peer_pe = oshmem_proc_pe_vpid(group, peer_id);
 
         /* Special procedure is needed in case target and source are the same */
         if (source == target) {
@@ -448,7 +448,7 @@ static int _algorithm_recursive_doubling(struct oshmem_group_t *group,
         if ((group->proc_count - floor2_proc) > my_id) {
             /* I am in basic group, my partner is node (my_id+y) in extra group */
             peer_id = my_id + floor2_proc;
-            peer_pe = oshmem_proc_pe(group->proc_array[peer_id]);
+            peer_pe = oshmem_proc_pe_vpid(group, peer_id);
 
             /* Special procedure is needed in case target and source are the same */
             if (source == target) {
@@ -481,8 +481,7 @@ static int _algorithm_recursive_doubling(struct oshmem_group_t *group,
             /* Update exit condition and round counter */
             exit_flag >>= 1;
             round++;
-
-            peer_pe = oshmem_proc_pe(group->proc_array[peer_id]);
+            peer_pe = oshmem_proc_pe_vpid(group, peer_id);
 
 #if 1 /* It is ugly implementation of compare and swap operation
          Usage of this hack does not give performance improvement but
@@ -524,7 +523,7 @@ static int _algorithm_recursive_doubling(struct oshmem_group_t *group,
         if ((group->proc_count - floor2_proc) > my_id) {
             /* I am in basic group, my partner is node (my_id+y) in extra group */
             peer_id = my_id + floor2_proc;
-            peer_pe = oshmem_proc_pe(group->proc_array[peer_id]);
+            peer_pe = oshmem_proc_pe_vpid(group, peer_id);
 
             SCOLL_VERBOSE(14,
                           "[#%d] is extra send data to #%d",
@@ -566,7 +565,7 @@ static int _algorithm_linear(struct oshmem_group_t *group,
     rank = group->my_pe;
     size = group->proc_count;
     int root_id = size - 1;
-    int root_pe = oshmem_proc_pe(group->proc_array[root_id]);
+    int root_pe = oshmem_proc_pe_vpid(group, root_id);
 
     SCOLL_VERBOSE(12, "[#%d] Reduce algorithm: Basic", group->my_pe);
 
@@ -592,7 +591,7 @@ static int _algorithm_linear(struct oshmem_group_t *group,
             memcpy(target, (void *) source, nlong);
         } else {
             peer_id = size - 1;
-            peer_pe = oshmem_proc_pe(group->proc_array[peer_id]);
+            peer_pe = oshmem_proc_pe_vpid(group, peer_id);
             rc = MCA_SPML_CALL(recv(target, nlong, peer_pe));
         }
         if (OSHMEM_SUCCESS != rc) {
@@ -609,7 +608,7 @@ static int _algorithm_linear(struct oshmem_group_t *group,
                 inbuf = (char*) source;
             } else {
                 peer_id = i;
-                peer_pe = oshmem_proc_pe(group->proc_array[peer_id]);
+                peer_pe = oshmem_proc_pe_vpid(group, peer_id);
                 rc = MCA_SPML_CALL(recv(pml_buffer, nlong, peer_pe));
                 if (OSHMEM_SUCCESS != rc) {
                     if (NULL != free_buffer) {
@@ -671,7 +670,7 @@ static int _algorithm_log(struct oshmem_group_t *group,
     int peer_id = 0;
     int peer_pe = 0;
     int root_id = 0;
-    int root_pe = oshmem_proc_pe(group->proc_array[root_id]);
+    int root_pe = oshmem_proc_pe_vpid(group, root_id);
     int dim = 0;
 
     /* Initialize */
@@ -719,7 +718,7 @@ static int _algorithm_log(struct oshmem_group_t *group,
         if (vrank & mask) {
             peer_id = vrank & ~mask;
             peer_id = (peer_id + root_id) % size;
-            peer_pe = oshmem_proc_pe(group->proc_array[peer_id]);
+            peer_pe = oshmem_proc_pe_vpid(group, peer_id);
 
             rc = MCA_SPML_CALL(send((void*)snd_buffer, nlong, peer_pe, MCA_SPML_BASE_PUT_STANDARD));
             if (OSHMEM_SUCCESS != rc) {
@@ -738,7 +737,7 @@ static int _algorithm_log(struct oshmem_group_t *group,
                 continue;
             }
             peer_id = (peer_id + root_id) % size;
-            peer_pe = oshmem_proc_pe(group->proc_array[peer_id]);
+            peer_pe = oshmem_proc_pe_vpid(group, peer_id);
 
             /* Most of the time (all except the first one for commutative
              * operations) we receive in the user provided buffer
diff --git a/oshmem/mca/scoll/mpi/Makefile.am b/oshmem/mca/scoll/mpi/Makefile.am
index f8782fccffa..542046a50b6 100644
--- a/oshmem/mca/scoll/mpi/Makefile.am
+++ b/oshmem/mca/scoll/mpi/Makefile.am
@@ -1,6 +1,7 @@
 # Copyright (c) 2013-2015 Mellanox Technologies, Inc.
 #                         All rights reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 #  Additional copyrights may follow
@@ -9,7 +10,7 @@
 #
 #
 
-dist_oshmemdata_DATA = \
+EXTRA_DIST = \
 	help-oshmem-scoll-mpi.txt
 
 scoll_mpi_sources = \
diff --git a/oshmem/mca/scoll/mpi/help-oshmem-scoll-mpi.txt b/oshmem/mca/scoll/mpi/help-oshmem-scoll-mpi.txt
index 90e9432987e..5e58b2176af 100644
--- a/oshmem/mca/scoll/mpi/help-oshmem-scoll-mpi.txt
+++ b/oshmem/mca/scoll/mpi/help-oshmem-scoll-mpi.txt
@@ -9,7 +9,7 @@
 #
 [module_enable:fatal]
 scoll:mpi module reports issue during module enabling phase.
-Try to use scoll:mpi component with anoter one
+Try to use scoll:mpi component with another one
 for example scoll:basic
 
   Error: %s
diff --git a/oshmem/mca/scoll/mpi/scoll_mpi.h b/oshmem/mca/scoll/mpi/scoll_mpi.h
index 4938cb1d6d6..0a6e0e0342f 100644
--- a/oshmem/mca/scoll/mpi/scoll_mpi.h
+++ b/oshmem/mca/scoll/mpi/scoll_mpi.h
@@ -49,7 +49,7 @@ struct mca_scoll_mpi_component_t {
 };
 typedef struct mca_scoll_mpi_component_t mca_scoll_mpi_component_t;
 
-OMPI_MODULE_DECLSPEC extern mca_scoll_mpi_component_t mca_scoll_mpi_component;
+OMPI_DECLSPEC extern mca_scoll_mpi_component_t mca_scoll_mpi_component;
 
 
 /**
diff --git a/oshmem/mca/scoll/mpi/scoll_mpi_debug.h b/oshmem/mca/scoll/mpi/scoll_mpi_debug.h
index eac6008a1de..4009f49d872 100644
--- a/oshmem/mca/scoll/mpi/scoll_mpi_debug.h
+++ b/oshmem/mca/scoll/mpi/scoll_mpi_debug.h
@@ -18,7 +18,7 @@
 #define __MPI_FILE__ __FILE__
 #endif
 
-#ifdef OPAL_ENABLE_DEBUG
+#if OPAL_ENABLE_DEBUG
 #define MPI_COLL_VERBOSE(level, ...) \
     oshmem_output_verbose(level, mca_scoll_mpi_output, "%s:%d - %s() ", \
                         __MPI_FILE__, __LINE__, __FUNCTION__, __VA_ARGS__)
diff --git a/oshmem/mca/scoll/mpi/scoll_mpi_module.c b/oshmem/mca/scoll/mpi/scoll_mpi_module.c
index ca487caa696..a0f15ca0477 100644
--- a/oshmem/mca/scoll/mpi/scoll_mpi_module.c
+++ b/oshmem/mca/scoll/mpi/scoll_mpi_module.c
@@ -145,7 +145,7 @@ mca_scoll_mpi_comm_query(oshmem_group_t *osh_group, int *priority)
 
         /* Fill the map "group_rank-to-world_rank" in order to create a new proc group */
         for (i = 0; i < osh_group->proc_count; i++) {
-            ranks[i] = osh_group->proc_array[i]->super.proc_name.vpid;
+            ranks[i] = oshmem_proc_pe_vpid(osh_group, i);
         }
 
         OPAL_TIMING_ENV_NEXT(comm_query, "build_ranks");
diff --git a/oshmem/mca/scoll/mpi/scoll_mpi_ops.c b/oshmem/mca/scoll/mpi/scoll_mpi_ops.c
index d40bb76a4c6..b28a493aa24 100644
--- a/oshmem/mca/scoll/mpi/scoll_mpi_ops.c
+++ b/oshmem/mca/scoll/mpi/scoll_mpi_ops.c
@@ -115,6 +115,8 @@ int mca_scoll_mpi_collect(struct oshmem_group_t *group,
     int i;
     void *sbuf, *rbuf;
     int *disps, *recvcounts;
+    ompi_count_array_t recvcounts_desc;
+    ompi_disp_array_t disps_desc;
     MPI_COLL_VERBOSE(20,"RUNNING MPI ALLGATHER");
     mpi_module = (mca_scoll_mpi_module_t *) group->g_scoll.scoll_collect_module;
 
@@ -196,8 +198,10 @@ int mca_scoll_mpi_collect(struct oshmem_group_t *group,
             disps[i] = disps[i - 1] + recvcounts[i - 1];
         }
 
-        rc = mpi_module->comm->c_coll->coll_allgatherv(source, nlong, stype, target, recvcounts,
-                                                       disps, rtype, mpi_module->comm,
+        OMPI_COUNT_ARRAY_INIT(&recvcounts_desc, recvcounts);
+        OMPI_DISP_ARRAY_INIT(&disps_desc, disps);
+        rc = mpi_module->comm->c_coll->coll_allgatherv(source, nlong, stype, target, recvcounts_desc,
+                                                       disps_desc, rtype, mpi_module->comm,
                                                        mpi_module->comm->c_coll->coll_allgatherv_module);
 failed_allgather:
         free(recvcounts);
diff --git a/oshmem/mca/scoll/scoll.h b/oshmem/mca/scoll/scoll.h
index 4839e0d9c52..bd8803398b7 100644
--- a/oshmem/mca/scoll/scoll.h
+++ b/oshmem/mca/scoll/scoll.h
@@ -67,9 +67,9 @@ struct mca_scoll_base_component_1_0_0_t {
 };
 typedef struct mca_scoll_base_component_1_0_0_t mca_scoll_base_component_1_0_0_t;
 
-/** Per guidence in mca.h, use the unversioned struct name if you just
+/** Per guidance in mca.h, use the unversioned struct name if you just
  want to always keep up with the most recent version of the
- interace. */
+ interface. */
 typedef struct mca_scoll_base_component_1_0_0_t mca_scoll_base_component_t;
 
 /**
diff --git a/oshmem/mca/scoll/ucc/Makefile.am b/oshmem/mca/scoll/ucc/Makefile.am
new file mode 100644
index 00000000000..04a46e11768
--- /dev/null
+++ b/oshmem/mca/scoll/ucc/Makefile.am
@@ -0,0 +1,47 @@
+# Copyright (c) 2021 Mellanox Technologies, Inc.
+#                         All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
+# $COPYRIGHT$
+#
+#  Additional copyrights may follow
+#
+# $HEADER$
+#
+#
+
+EXTRA_DIST = \
+	help-oshmem-scoll-ucc.txt
+
+AM_CPPFLAGS = $(scoll_ucc_CPPFLAGS)
+
+scoll_ucc_sources =           \
+		scoll_ucc.h           \
+		scoll_ucc_debug.h     \
+		scoll_ucc_dtypes.h    \
+		scoll_ucc_common.h    \
+		scoll_ucc_module.c    \
+		scoll_ucc_component.c \
+		scoll_ucc_barrier.c   \
+		scoll_ucc_broadcast.c \
+		scoll_ucc_reduce.c    \
+		scoll_ucc_collect.c   \
+		scoll_ucc_alltoall.c
+if MCA_BUILD_oshmem_scoll_ucc_DSO
+component_noinst =
+component_install = mca_scoll_ucc.la
+else
+component_noinst = libmca_scoll_ucc.la
+component_install =
+endif
+
+mcacomponentdir = $(pkglibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_scoll_ucc_la_SOURCES = $(scoll_ucc_sources)
+mca_scoll_ucc_la_LIBADD = $(top_builddir)/oshmem/liboshmem.la \
+	$(scoll_ucc_LIBS)
+mca_scoll_ucc_la_LDFLAGS = -module -avoid-version $(scoll_ucc_LDFLAGS)
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_scoll_ucc_la_SOURCES =$(scoll_ucc_sources)
+libmca_scoll_ucc_la_LIBADD = $(scoll_ucc_LIBS)
+libmca_scoll_ucc_la_LDFLAGS = -module -avoid-version $(scoll_ucc_LDFLAGS)
diff --git a/oshmem/mca/scoll/ucc/configure.m4 b/oshmem/mca/scoll/ucc/configure.m4
new file mode 100644
index 00000000000..40f1849c374
--- /dev/null
+++ b/oshmem/mca/scoll/ucc/configure.m4
@@ -0,0 +1,38 @@
+# -*- shell-script -*-
+#
+#
+# Copyright (c) 2021      Mellanox Technologies. All rights reserved.
+# Copyright (c) 2015      Research Organization for Information Science
+#                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+
+# MCA_scoll_ucc_CONFIG([action-if-can-compile],
+#                      [action-if-cant-compile])
+# ------------------------------------------------
+AC_DEFUN([MCA_oshmem_scoll_ucc_CONFIG],[
+    AC_CONFIG_FILES([oshmem/mca/scoll/ucc/Makefile])
+
+    OMPI_CHECK_UCC([scoll_ucc],
+                     [scoll_ucc_happy="yes"],
+                     [scoll_ucc_happy="no"])
+
+    AS_IF([test "$scoll_ucc_happy" = "yes"],
+          [scoll_ucc_WRAPPER_EXTRA_LDFLAGS="$scoll_ucc_LDFLAGS"
+           scoll_ucc_CPPFLAGS="$scoll_ucc_CPPFLAGS"
+           scoll_ucc_WRAPPER_EXTRA_LIBS="$scoll_ucc_LIBS"
+           $1],
+          [$2])
+
+    # substitute in the things needed to build ucc
+    AC_SUBST([scoll_ucc_CPPFLAGS])
+    AC_SUBST([scoll_ucc_LDFLAGS])
+    AC_SUBST([scoll_ucc_LIBS])
+])dnl
+
diff --git a/oshmem/mca/scoll/ucc/help-oshmem-scoll-ucc.txt b/oshmem/mca/scoll/ucc/help-oshmem-scoll-ucc.txt
new file mode 100644
index 00000000000..bf4bbac334f
--- /dev/null
+++ b/oshmem/mca/scoll/ucc/help-oshmem-scoll-ucc.txt
@@ -0,0 +1,16 @@
+#
+# Copyright (c) 2021      Mellanox Technologies, Inc.
+#                         All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+[module_enable:fatal]
+scoll:ucc module reports issue during module enabling phase.
+Try to use scoll:ucc component with another one
+for example scoll:basic
+
+  Error: %s
+#
diff --git a/oshmem/mca/scoll/ucc/scoll_ucc.h b/oshmem/mca/scoll/ucc/scoll_ucc.h
new file mode 100644
index 00000000000..dcf52787cf6
--- /dev/null
+++ b/oshmem/mca/scoll/ucc/scoll_ucc.h
@@ -0,0 +1,131 @@
+/*
+ * Copyright (c) 2021      Mellanox Technologies. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef MCA_SCOLL_UCC_H
+#define MCA_SCOLL_UCC_H
+
+#include "oshmem_config.h"
+
+#include "shmem.h"
+#include "oshmem/mca/mca.h"
+#include "oshmem/mca/scoll/scoll.h"
+#include "oshmem/proc/proc.h"
+
+#include "scoll_ucc_debug.h"
+
+#include <ucc/api/ucc.h>
+
+BEGIN_C_DECLS
+
+#define SCOLL_UCC_CTS (UCC_COLL_TYPE_BARRIER | UCC_COLL_TYPE_BCAST | \
+                       UCC_COLL_TYPE_ALLREDUCE | UCC_COLL_TYPE_ALLGATHER | \
+                       UCC_COLL_TYPE_ALLTOALL)
+
+#define SCOLL_UCC_CTS_STR "barrier,broadcast,reduce,collect,alltoall"
+
+int mca_scoll_ucc_progress(void);
+
+/**
+ * Globally exported structure
+ */
+struct mca_scoll_ucc_component_t {
+    mca_scoll_base_component_1_0_0_t super;
+    int ucc_priority;
+    int ucc_verbose;
+    int ucc_enable;
+    int ucc_np;
+    char * cls;
+    char * cts;
+    int nr_modules;
+    bool libucc_initialized;
+    ucc_context_h ucc_context;
+    ucc_lib_h ucc_lib;
+    ucc_lib_attr_t ucc_lib_attr;
+    ucc_coll_type_t cts_requested;
+};
+typedef struct mca_scoll_ucc_component_t mca_scoll_ucc_component_t;
+
+OMPI_DECLSPEC extern mca_scoll_ucc_component_t mca_scoll_ucc_component;
+
+/**
+ * UCC enabled team
+ */
+struct mca_scoll_ucc_module_t {
+    mca_scoll_base_module_t super;
+
+    oshmem_group_t             *group;
+    ucc_team_h                  ucc_team;
+    long                       *pSync;
+    
+    /* Saved handlers - for fallback */
+    mca_scoll_base_module_reduce_fn_t previous_reduce;
+    mca_scoll_base_module_t *previous_reduce_module;
+    mca_scoll_base_module_broadcast_fn_t previous_broadcast;
+    mca_scoll_base_module_t *previous_broadcast_module;
+    mca_scoll_base_module_barrier_fn_t previous_barrier;
+    mca_scoll_base_module_t *previous_barrier_module;
+    mca_scoll_base_module_collect_fn_t previous_collect;
+    mca_scoll_base_module_t *previous_collect_module;
+    mca_scoll_base_module_alltoall_fn_t previous_alltoall;
+    mca_scoll_base_module_t *previous_alltoall_module;
+};
+typedef struct mca_scoll_ucc_module_t mca_scoll_ucc_module_t;
+
+OBJ_CLASS_DECLARATION(mca_scoll_ucc_module_t);
+
+/* API functions */
+int mca_scoll_ucc_init_query(bool enable_progress_threads, bool enable_mpi_threads);
+
+int mca_scoll_ucc_team_create(mca_scoll_ucc_module_t *ucc_module, 
+                              oshmem_group_t *osh_group);
+
+int mca_scoll_ucc_init_ctx(oshmem_group_t *osh_group);
+
+mca_scoll_base_module_t* mca_scoll_ucc_comm_query(oshmem_group_t *osh_group, int *priority);
+
+int mca_scoll_ucc_barrier(struct oshmem_group_t *group, long *pSync, int alg);
+
+int mca_scoll_ucc_broadcast(struct oshmem_group_t *group,
+                            int PE_root,
+                            void *target,
+                            const void *source,
+                            size_t nlong,
+                            long *pSync,
+                            bool nlong_type,
+                            int alg);
+
+int mca_scoll_ucc_collect(struct oshmem_group_t *group,
+                          void *target,
+                          const void *source,
+                          size_t nlong,
+                          long *pSync,
+                          bool nlong_type,
+                          int alg);
+
+int mca_scoll_ucc_reduce(struct oshmem_group_t *group,
+                         struct oshmem_op_t *op,
+                         void *target,
+                         const void *source,
+                         size_t nlong,
+                         long *pSync,
+                         void *pWrk,
+                         int alg);
+
+int mca_scoll_ucc_alltoall(struct oshmem_group_t *group,
+                           void *target,
+                           const void *source,
+                           ptrdiff_t dst, ptrdiff_t sst,
+                           size_t nelems,
+                           size_t element_size,
+                           long *pSync,
+                           int alg);
+
+END_C_DECLS
+
+#endif
diff --git a/oshmem/mca/scoll/ucc/scoll_ucc_alltoall.c b/oshmem/mca/scoll/ucc/scoll_ucc_alltoall.c
new file mode 100644
index 00000000000..c521f2ffab3
--- /dev/null
+++ b/oshmem/mca/scoll/ucc/scoll_ucc_alltoall.c
@@ -0,0 +1,103 @@
+/**
+  Copyright (c) 2021 Mellanox Technologies. All rights reserved.
+  $COPYRIGHT$
+
+  Additional copyrights may follow
+
+  $HEADER$
+ */
+#include "scoll_ucc.h"
+#include "scoll_ucc_dtypes.h"
+#include "scoll_ucc_common.h"
+
+#include <ucc/api/ucc.h>
+
+static inline ucc_status_t mca_scoll_ucc_alltoall_init(const void *sbuf, void *rbuf, 
+                                                       int count, size_t elem_size,
+                                                       mca_scoll_ucc_module_t *ucc_module,
+                                                       ucc_coll_req_h *req)
+{
+    ucc_datatype_t dt;
+
+    if (elem_size == 8) {
+        dt = UCC_DT_INT64;
+    } else if (elem_size == 4) {
+        dt = UCC_DT_INT32;
+    } else {
+        dt = UCC_DT_INT8;
+    }
+
+    ucc_coll_args_t coll = {
+        .mask = UCC_COLL_ARGS_FIELD_FLAGS | UCC_COLL_ARGS_FIELD_GLOBAL_WORK_BUFFER,
+        .coll_type = UCC_COLL_TYPE_ALLTOALL,
+        .src.info = {
+            .buffer = (void *)sbuf,
+            .count = count * ucc_module->group->proc_count,
+            .datatype = dt,
+            .mem_type = UCC_MEMORY_TYPE_UNKNOWN
+        },
+        .dst.info = {
+            .buffer = rbuf,
+            .count = count * ucc_module->group->proc_count,
+            .datatype = dt,
+            .mem_type = UCC_MEMORY_TYPE_UNKNOWN
+        },
+        .flags = UCC_COLL_ARGS_FLAG_MEM_MAPPED_BUFFERS,
+        .global_work_buffer = ucc_module->pSync,
+    };
+
+    if (NULL == mca_scoll_ucc_component.ucc_context) {
+        if (OSHMEM_ERROR == mca_scoll_ucc_init_ctx(ucc_module->group)) {
+            return OSHMEM_ERROR;
+        }
+    }
+
+    if (NULL == ucc_module->ucc_team) {
+        if (OSHMEM_ERROR == mca_scoll_ucc_team_create(ucc_module, ucc_module->group)) {
+            return OSHMEM_ERROR;
+        }
+    }
+    SCOLL_UCC_REQ_INIT(req, coll, ucc_module);
+    return UCC_OK;
+fallback:
+    return UCC_ERR_NOT_SUPPORTED;
+}
+
+
+int mca_scoll_ucc_alltoall(struct oshmem_group_t *group,
+                           void *target,
+                           const void *source,
+                           ptrdiff_t dst, ptrdiff_t sst,
+                           size_t nelems,
+                           size_t element_size,
+                           long *pSync,
+                           int alg)
+{
+    mca_scoll_ucc_module_t *ucc_module;
+    size_t count;
+    ucc_coll_req_h req;
+    int rc;
+
+    UCC_VERBOSE(3, "running ucc alltoall");
+    ucc_module = (mca_scoll_ucc_module_t *) group->g_scoll.scoll_alltoall_module;
+    count = nelems;
+
+    if ((sst != 1) || (dst != 1)) {
+        goto fallback;
+    }
+    /* Do nothing on zero-length request */
+    if (OPAL_UNLIKELY(!nelems)) {
+        return OSHMEM_SUCCESS;
+    }
+
+    SCOLL_UCC_CHECK(mca_scoll_ucc_alltoall_init(source, target, count, 
+                                                element_size, ucc_module, &req));
+    SCOLL_UCC_CHECK(ucc_collective_post(req));
+    SCOLL_UCC_CHECK(scoll_ucc_req_wait(req));
+    return OSHMEM_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback alltoall");
+    PREVIOUS_SCOLL_FN(ucc_module, alltoall, group, target, source,
+                      dst, sst, nelems, element_size, pSync, alg);
+    return rc;
+}
diff --git a/oshmem/mca/scoll/ucc/scoll_ucc_barrier.c b/oshmem/mca/scoll/ucc/scoll_ucc_barrier.c
new file mode 100644
index 00000000000..6fc88ea4a07
--- /dev/null
+++ b/oshmem/mca/scoll/ucc/scoll_ucc_barrier.c
@@ -0,0 +1,59 @@
+/**
+  Copyright (c) 2021 Mellanox Technologies. All rights reserved.
+  $COPYRIGHT$
+
+  Additional copyrights may follow
+
+  $HEADER$
+ */
+#include "scoll_ucc.h"
+#include "scoll_ucc_dtypes.h"
+#include "scoll_ucc_common.h"
+
+#include <ucc/api/ucc.h>
+
+static inline ucc_status_t mca_scoll_ucc_barrier_init(mca_scoll_ucc_module_t * ucc_module,
+                                                      ucc_coll_req_h * req)
+{
+    ucc_coll_args_t coll = {
+        .mask = 0,
+        .coll_type = UCC_COLL_TYPE_BARRIER
+    };
+
+    if (NULL == mca_scoll_ucc_component.ucc_context) {
+        if (OSHMEM_ERROR == mca_scoll_ucc_init_ctx(ucc_module->group)) {
+            return OSHMEM_ERROR;
+        }
+    }
+
+    if (NULL == ucc_module->ucc_team) {
+        if (OSHMEM_ERROR == mca_scoll_ucc_team_create(ucc_module, ucc_module->group)) {
+            return OSHMEM_ERROR;
+        }
+    }
+
+    SCOLL_UCC_REQ_INIT(req, coll, ucc_module);
+    return UCC_OK;
+fallback:
+    return UCC_ERR_NOT_SUPPORTED;
+}
+
+int mca_scoll_ucc_barrier(struct oshmem_group_t *group, long *pSync, int alg)
+{
+    mca_scoll_ucc_module_t *ucc_module;
+    ucc_coll_req_h req;
+    int            rc;
+
+    UCC_VERBOSE(3, "running ucc barrier");
+    ucc_module = (mca_scoll_ucc_module_t *) group->g_scoll.scoll_barrier_module;
+
+    SCOLL_UCC_CHECK(mca_scoll_ucc_barrier_init(ucc_module, &req));
+    SCOLL_UCC_CHECK(ucc_collective_post(req));
+    SCOLL_UCC_CHECK(scoll_ucc_req_wait(req));
+    return OSHMEM_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback barrier");
+    PREVIOUS_SCOLL_FN(ucc_module, barrier, group,
+                      pSync, alg);
+    return rc;
+}
diff --git a/oshmem/mca/scoll/ucc/scoll_ucc_broadcast.c b/oshmem/mca/scoll/ucc/scoll_ucc_broadcast.c
new file mode 100644
index 00000000000..d3838e2574e
--- /dev/null
+++ b/oshmem/mca/scoll/ucc/scoll_ucc_broadcast.c
@@ -0,0 +1,86 @@
+/**
+  Copyright (c) 2021 Mellanox Technologies. All rights reserved.
+  $COPYRIGHT$
+
+  Additional copyrights may follow
+
+  $HEADER$
+ */
+#include "scoll_ucc.h"
+#include "scoll_ucc_dtypes.h"
+#include "scoll_ucc_common.h"
+
+#include <ucc/api/ucc.h>
+
+static inline ucc_status_t mca_scoll_ucc_broadcast_init(void * buf, int count,
+                                                        int root, 
+                                                        mca_scoll_ucc_module_t * ucc_module,
+                                                        ucc_coll_req_h * req)
+{
+    ucc_coll_args_t coll = {
+        .mask = 0,
+        .coll_type = UCC_COLL_TYPE_BCAST,
+        .root = root,
+        .src.info = {
+            .buffer = buf,
+            .count = count,
+            .datatype = UCC_DT_INT8,
+            .mem_type = UCC_MEMORY_TYPE_UNKNOWN
+        }
+    };
+
+    if (NULL == mca_scoll_ucc_component.ucc_context) {
+        if (OSHMEM_ERROR == mca_scoll_ucc_init_ctx(ucc_module->group)) {
+            return OSHMEM_ERROR;
+        }
+    }
+
+    if (NULL == ucc_module->ucc_team) {
+        if (OSHMEM_ERROR == mca_scoll_ucc_team_create(ucc_module, ucc_module->group)) {
+            return OSHMEM_ERROR;
+        }
+    }
+    SCOLL_UCC_REQ_INIT(req, coll, ucc_module);
+    return UCC_OK;
+fallback:
+    return UCC_ERR_NOT_SUPPORTED;
+}
+
+
+int mca_scoll_ucc_broadcast(struct oshmem_group_t *group,
+                            int PE_root,
+                            void *target,
+                            const void *source,
+                            size_t nlong,
+                            long *pSync,
+                            bool nlong_type,
+                            int alg)
+{
+    mca_scoll_ucc_module_t * ucc_module;
+    void * buf;
+    ucc_coll_req_h req;
+    int rc;
+
+    UCC_VERBOSE(3, "running ucc bcast");
+    ucc_module = (mca_scoll_ucc_module_t *) group->g_scoll.scoll_broadcast_module;
+    if (group->my_pe == PE_root) {
+        buf = (void *) source;
+    } else {
+        buf = target;
+    }
+
+    /* Do nothing on zero-length request */
+    if (OPAL_UNLIKELY(!nlong)) {
+        return OSHMEM_SUCCESS;
+    }
+
+    SCOLL_UCC_CHECK(mca_scoll_ucc_broadcast_init(buf, nlong, PE_root, ucc_module, &req));
+    SCOLL_UCC_CHECK(ucc_collective_post(req));
+    SCOLL_UCC_CHECK(scoll_ucc_req_wait(req));
+    return OSHMEM_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback bcast");
+    PREVIOUS_SCOLL_FN(ucc_module, broadcast, group, PE_root, target, source,
+                      nlong, pSync, nlong_type, alg);
+    return rc;
+}
diff --git a/oshmem/mca/scoll/ucc/scoll_ucc_collect.c b/oshmem/mca/scoll/ucc/scoll_ucc_collect.c
new file mode 100644
index 00000000000..fadcef09d86
--- /dev/null
+++ b/oshmem/mca/scoll/ucc/scoll_ucc_collect.c
@@ -0,0 +1,88 @@
+/**
+  Copyright (c) 2021 Mellanox Technologies. All rights reserved.
+  $COPYRIGHT$
+
+  Additional copyrights may follow
+
+  $HEADER$
+ */
+#include "scoll_ucc.h"
+#include "scoll_ucc_dtypes.h"
+#include "scoll_ucc_common.h"
+
+#include <ucc/api/ucc.h>
+
+static inline ucc_status_t mca_scoll_ucc_collect_init(const void * sbuf, void * rbuf,
+                                                      size_t count,
+                                                      mca_scoll_ucc_module_t * ucc_module,
+                                                      ucc_coll_req_h * req)
+{
+    size_t          gsize = ucc_module->group->proc_count;
+    ucc_coll_args_t coll  = {
+        .mask = 0,
+        .coll_type = UCC_COLL_TYPE_ALLGATHER,
+        .src.info = {
+            .buffer   = (void *) sbuf,
+            .count    = count,
+            .datatype = UCC_DT_INT8,
+            .mem_type = UCC_MEMORY_TYPE_UNKNOWN
+        }, 
+        .dst.info = {
+            .buffer   = rbuf,
+            .count    = count * gsize,
+            .datatype = UCC_DT_INT8,
+            .mem_type = UCC_MEMORY_TYPE_UNKNOWN
+        },
+    };
+
+    if (NULL == mca_scoll_ucc_component.ucc_context) {
+        if (OSHMEM_ERROR == mca_scoll_ucc_init_ctx(ucc_module->group)) {
+            return OSHMEM_ERROR;
+        }
+    }
+
+    if (NULL == ucc_module->ucc_team) {
+        if (OSHMEM_ERROR == mca_scoll_ucc_team_create(ucc_module, ucc_module->group)) {
+            return OSHMEM_ERROR;
+        }
+    }
+    SCOLL_UCC_REQ_INIT(req, coll, ucc_module);
+    return UCC_OK;
+fallback:
+    return UCC_ERR_NOT_SUPPORTED;
+}
+
+int mca_scoll_ucc_collect(struct oshmem_group_t *group,
+                          void *target,
+                          const void *source,
+                          size_t nlong,
+                          long *pSync,
+                          bool nlong_type,
+                          int alg)
+{
+    mca_scoll_ucc_module_t *ucc_module;
+    ucc_coll_req_h req;
+    int rc;
+
+    UCC_VERBOSE(3, "running ucc collect");
+    ucc_module = (mca_scoll_ucc_module_t *) group->g_scoll.scoll_collect_module;
+
+    if (false == nlong_type) {
+        /* vector type of collect */
+        goto fallback;
+    }
+
+    if (OPAL_UNLIKELY(!nlong)) {
+        return OSHMEM_SUCCESS;
+    }
+
+    SCOLL_UCC_CHECK(mca_scoll_ucc_collect_init(source, target, nlong, ucc_module, &req));
+    SCOLL_UCC_CHECK(ucc_collective_post(req));
+    SCOLL_UCC_CHECK(scoll_ucc_req_wait(req));
+    return OSHMEM_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback collect");
+    PREVIOUS_SCOLL_FN(ucc_module, collect, group, target, source,
+                      nlong, pSync, nlong_type, alg);
+    return rc;
+}
diff --git a/oshmem/mca/scoll/ucc/scoll_ucc_common.h b/oshmem/mca/scoll/ucc/scoll_ucc_common.h
new file mode 100644
index 00000000000..0c80ed91765
--- /dev/null
+++ b/oshmem/mca/scoll/ucc/scoll_ucc_common.h
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2021      Mellanox Technologies. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#ifndef MCA_SCOLL_UCC_COMMON_H
+#define MCA_SCOLL_UCC_COMMON_H
+
+#define SCOLL_UCC_CHECK(_call) do { \
+    if (UCC_OK != (_call)) {        \
+        goto fallback;              \
+    }                               \
+} while(0)
+
+#define SCOLL_UCC_REQ_INIT(_req, _coll, _module) do {       \
+    SCOLL_UCC_CHECK(ucc_collective_init(&_coll, _req,       \
+                                       _module->ucc_team)); \
+} while(0)
+
+static inline ucc_status_t scoll_ucc_req_wait(ucc_coll_req_h req)
+{
+    ucc_status_t status;
+    while (UCC_OK != (status = ucc_collective_test(req))) {
+        if (0 > status) {
+            UCC_ERROR("ucc_collective_test failed: %s",
+                      ucc_status_string(status));
+            return status;
+        }
+        ucc_context_progress(mca_scoll_ucc_component.ucc_context);
+        opal_progress();
+    }
+    return ucc_collective_finalize(req);
+}
+
+#endif
+ 
diff --git a/oshmem/mca/scoll/ucc/scoll_ucc_component.c b/oshmem/mca/scoll/ucc/scoll_ucc_component.c
new file mode 100644
index 00000000000..5a7bcc84ff4
--- /dev/null
+++ b/oshmem/mca/scoll/ucc/scoll_ucc_component.c
@@ -0,0 +1,178 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2021 Mellanox Technologies. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include <stdio.h>
+
+#include <dlfcn.h>
+#include <libgen.h>
+
+#include "scoll_ucc.h"
+#include "opal/util/argv.h"
+
+/*
+ * Public string showing the oshmem scoll_ucc component version number
+ */
+const char * mca_scoll_ucc_component_version_string =
+  "OpenSHMEM UCC collective MCA component version " OSHMEM_VERSION;
+
+
+static int ucc_open(void);
+static int ucc_close(void);
+static int ucc_register(void);
+
+int mca_scoll_ucc_output = -1;
+
+mca_scoll_ucc_component_t mca_scoll_ucc_component = {
+
+    /* First, the mca_component_t struct containing meta information
+       about the component */
+    {
+        .scoll_version = {
+            MCA_SCOLL_BASE_VERSION_2_0_0,
+
+            /* Component name and version */
+            .mca_component_name = "ucc",
+            MCA_BASE_MAKE_VERSION(component, OSHMEM_MAJOR_VERSION, OSHMEM_MINOR_VERSION,
+                                  OSHMEM_RELEASE_VERSION),
+
+            /* Component open and close functions */
+            .mca_open_component = ucc_open,
+            .mca_close_component = ucc_close,
+            .mca_register_component_params = ucc_register,
+            .mca_query_component = NULL,
+        },
+        .scoll_data = {
+            /* The component is not checkpoint ready */
+            MCA_BASE_METADATA_PARAM_NONE
+        },
+
+        /* Initialization / querying functions */
+        .scoll_init = mca_scoll_ucc_init_query,
+        .scoll_query = mca_scoll_ucc_comm_query,
+    },
+    75,                 /* priority */
+    0,                  /* verbose level */
+    0,                  /* ucc_enable */
+    2,                  /* ucc_np */
+    "",                 /* cls */
+    SCOLL_UCC_CTS_STR,  /* cts */
+    0,                  /* nr_modules */
+    false,              /* libucc_initialized */
+    NULL                /* ucc_context */
+};
+
+static int ucc_register(void)
+{
+    mca_scoll_ucc_component_t * cm = &mca_scoll_ucc_component;
+    mca_base_component_t * c       = &cm->super.scoll_version;
+
+    mca_base_component_var_register(c, "priority", 
+                                    "Priority of the UCC scoll component",
+                                    MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, 
+                                    OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+                                    &cm->ucc_priority);
+
+    mca_base_component_var_register(c, "verbose", 
+                                    "Verbose level of the UCC scoll component",
+                                    MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, 
+                                    OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+                                    &cm->ucc_verbose);
+
+    mca_base_component_var_register(c, "enable", 
+                                    "[1|0|] Enable/Disable UCC scoll component",
+                                    MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, 
+                                    OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+                                    &cm->ucc_enable);
+
+    mca_base_component_var_register(c, "np", 
+                                    "Minimal Active Set / Team size for UCC scoll component",
+                                    MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, 
+                                    OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+                                    &cm->ucc_np);
+
+    cm->cts = "";
+    mca_base_component_var_register(c, "cls", 
+                                    "Comma separated list of UCC CLS to be used for team creation",
+                                    MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, 
+                                    OPAL_INFO_LVL_6, MCA_BASE_VAR_SCOPE_READONLY,
+                                    &cm->cls);
+    cm->cts = SCOLL_UCC_CTS_STR;
+    mca_base_component_var_register(c, "cts", 
+                                    "Comma separated list of UCC coll types to be enabled",
+                                    MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, 
+                                    OPAL_INFO_LVL_6, MCA_BASE_VAR_SCOPE_READONLY,
+                                    &cm->cts);
+    return OSHMEM_SUCCESS;
+}
+
+static ucc_coll_type_t mca_scoll_ucc_str_to_type(const char *str)
+{
+    if (0 == strcasecmp(str, "barrier")) {
+        return UCC_COLL_TYPE_BARRIER;
+    } else if (0 == strcasecmp(str, "broadcast")) {
+        return UCC_COLL_TYPE_BCAST;
+    } else if (0 == strcasecmp(str, "reduce")) {
+        return UCC_COLL_TYPE_ALLREDUCE;
+    } else if (0 == strcasecmp(str, "alltoall")) {
+        return UCC_COLL_TYPE_ALLTOALL;
+    } else if (0 == strcasecmp(str, "collect")) {
+        return UCC_COLL_TYPE_ALLGATHER;
+    }
+    UCC_ERROR("incorrect value for cts: %s, allowed: %s",
+              str, SCOLL_UCC_CTS_STR);
+    return UCC_COLL_TYPE_LAST;
+}
+
+static void mca_scoll_ucc_init_default_cts(void)
+{
+    mca_scoll_ucc_component_t *cm = &mca_scoll_ucc_component;
+    bool disable;
+    char ** cts;
+    int n_cts, i;
+    char * str;
+    ucc_coll_type_t *ct, c;
+
+    disable              = (cm->cts[0] == '^') ? true : false;
+    cts                  = opal_argv_split(disable ? (cm->cts + 1) : cm->cts, ',');
+    n_cts                = opal_argv_count(cts);
+    cm->cts_requested    = disable ? SCOLL_UCC_CTS : 0;
+    for (i = 0; i < n_cts; i++) {
+        str = cts[i];
+        ct  = &cm->cts_requested;
+        
+        c = mca_scoll_ucc_str_to_type(str);
+        if (UCC_COLL_TYPE_LAST == c) {
+            *ct = SCOLL_UCC_CTS;
+            break;
+        }
+        if (disable) {
+            (*ct) &= ~c;
+        } else {
+            (*ct) |= c;
+        }
+    }
+    opal_argv_free(cts);
+}
+
+static int ucc_open(void)
+{
+    mca_scoll_ucc_component_t *cm;
+    cm = &mca_scoll_ucc_component;
+    cm->libucc_initialized = false;
+
+    mca_scoll_ucc_output = opal_output_open(NULL);
+    opal_output_set_verbosity(mca_scoll_ucc_output, cm->ucc_verbose);
+    mca_scoll_ucc_init_default_cts();
+    return OSHMEM_SUCCESS;
+}
+
+static int ucc_close(void)
+{
+    return OSHMEM_SUCCESS;
+}
diff --git a/oshmem/mca/scoll/ucc/scoll_ucc_debug.h b/oshmem/mca/scoll/ucc/scoll_ucc_debug.h
new file mode 100644
index 00000000000..74c86567f2d
--- /dev/null
+++ b/oshmem/mca/scoll/ucc/scoll_ucc_debug.h
@@ -0,0 +1,35 @@
+/**
+  Copyright (c) 2021 Mellanox Technologies. All rights reserved.
+  $COPYRIGHT$
+
+  Additional copyrights may follow
+
+  $HEADER$
+ */
+
+#ifndef SCOLL_UCC_DEBUG_H
+#define SCOLL_UCC_DEBUG_H
+#include "oshmem_config.h"
+#pragma GCC system_header
+
+#ifdef __BASE_FILE__
+#define __UCC_FILE__ __BASE_FILE__
+#else
+#define __UCC_FILE__ __FILE__
+#endif
+
+#if OPAL_ENABLE_DEBUG
+#define UCC_VERBOSE(level, ...) \
+    oshmem_output_verbose(level, mca_scoll_ucc_output, "%s:%d - %s() ", \
+                        __UCC_FILE__, __LINE__, __FUNCTION__, __VA_ARGS__)
+#else
+#define UCC_VERBOSE(level, ...)
+#endif
+
+#define UCC_ERROR(...) \
+    oshmem_output_verbose(0, mca_scoll_ucc_output, "Error: %s:%d - %s() ", \
+                        __UCC_FILE__, __LINE__, __FUNCTION__, __VA_ARGS__)
+
+extern int mca_scoll_ucc_output;
+
+#endif // SCOLL_UCC_DEBUG_H
diff --git a/oshmem/mca/scoll/ucc/scoll_ucc_dtypes.h b/oshmem/mca/scoll/ucc/scoll_ucc_dtypes.h
new file mode 100644
index 00000000000..fe7fe8f3f5b
--- /dev/null
+++ b/oshmem/mca/scoll/ucc/scoll_ucc_dtypes.h
@@ -0,0 +1,125 @@
+/*
+ * Copyright (c) 2021 Mellanox Technologies. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#ifndef SCOLL_UCC_DTYPES_H
+#define SCOLL_UCC_DTYPES_H
+
+#include "oshmem/op/op.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/op/op.h"
+
+#include <ucc/api/ucc.h>
+
+#define SCOLL_UCC_DT_UNSUPPORTED -1
+#define SCOLL_UCC_OP_UNSUPPORTED -1
+
+static ucc_datatype_t shmem_datatype_to_ucc_dt[OSHMEM_OP_TYPE_NUMBER + 1] = {
+#if SIZEOF_SHORT == 2
+    [OSHMEM_OP_TYPE_SHORT] = UCC_DT_INT16,
+#else
+    [OSHMEM_OP_TYPE_SHORT] = SCOLL_UCC_DT_UNSUPPORTED,
+#endif
+#if SIZEOF_INT == 4
+    [OSHMEM_OP_TYPE_INT] = UCC_DT_INT32,
+#else
+    [OSHMEM_OP_TYPE_INT] = SCOLL_UCC_DT_UNSUPPORTED,
+#endif
+#if SIZEOF_LONG == 8
+    [OSHMEM_OP_TYPE_LONG] = UCC_DT_INT64,
+#else
+    [OSHMEM_OP_TYPE_LONG] = SCOLL_UCC_DT_UNSUPPORTED,
+#endif
+#if SIZEOF_LONG_LONG == 8
+    [OSHMEM_OP_TYPE_LLONG] = UCC_DT_INT64,
+#else
+    [OSHMEM_OP_TYPE_LLONG] = SCOLL_UCC_DT_UNSUPPORTED,
+#endif
+    [OSHMEM_OP_TYPE_INT16_T] = UCC_DT_INT16,
+    [OSHMEM_OP_TYPE_INT32_T] = UCC_DT_INT32,
+    [OSHMEM_OP_TYPE_INT64_T] = UCC_DT_INT64,
+#if SIZEOF_FLOAT == 4
+    [OSHMEM_OP_TYPE_FLOAT] = UCC_DT_FLOAT32,
+#else
+    [OSHMEM_OP_TYPE_FLOAT] = SCOLL_UCC_DT_UNSUPPORTED,
+#endif
+#if SIZEOF_DOUBLE == 8
+    [OSHMEM_OP_TYPE_DOUBLE] = UCC_DT_FLOAT64,
+#else
+    [OSHMEM_OP_TYPE_DOUBLE] = SCOLL_UCC_DT_UNSUPPORTED,
+#endif
+#if UCC_HAVE_COMPLEX_AND_FLOAT128_DT
+    [OSHMEM_OP_TYPE_FREAL16] = UCC_DT_FLOAT128,
+    #if SIZEOF_LONG_DOUBLE == 16
+        [OSHMEM_OP_TYPE_LDOUBLE] = UCC_DT_FLOAT128,
+    #else
+        [OSHMEM_OP_TYPE_LDOUBLE] = SCOLL_UCC_DT_UNSUPPORTED,
+    #endif
+    #if SIZEOF_FLOAT__COMPLEX == 8
+        [OSHMEM_OP_TYPE_FCOMPLEX] = UCC_DT_FLOAT32_COMPLEX,
+    #else
+        [OSHMEM_OP_TYPE_FCOMPLEX] = SCOLL_UCC_DT_UNSUPPORTED,
+    #endif
+    #if SIZEOF_DOUBLE__COMPLEX == 16
+        [OSHMEM_OP_TYPE_DCOMPLEX] = UCC_DT_FLOAT64_COMPLEX,
+    #else
+        [OSHMEM_OP_TYPE_DCOMPLEX] = SCOLL_UCC_DT_UNSUPPORTED,
+    #endif
+#else
+    [OSHMEM_OP_TYPE_FREAL16] = SCOLL_UCC_DT_UNSUPPORTED,
+    [OSHMEM_OP_TYPE_LDOUBLE] = SCOLL_UCC_DT_UNSUPPORTED,
+    [OSHMEM_OP_TYPE_FCOMPLEX] = SCOLL_UCC_DT_UNSUPPORTED,
+    [OSHMEM_OP_TYPE_DCOMPLEX] = SCOLL_UCC_DT_UNSUPPORTED,
+#endif
+    [OSHMEM_OP_TYPE_FINT2] = UCC_DT_INT16,
+    [OSHMEM_OP_TYPE_FINT4] = UCC_DT_INT32,
+    [OSHMEM_OP_TYPE_FINT8] = UCC_DT_INT64,
+    [OSHMEM_OP_TYPE_FREAL4] = UCC_DT_FLOAT32,
+    [OSHMEM_OP_TYPE_FREAL8] = UCC_DT_FLOAT64,
+    [OSHMEM_OP_TYPE_NUMBER] = SCOLL_UCC_DT_UNSUPPORTED
+};
+
+static inline ucc_datatype_t shmem_op_to_ucc_dtype(oshmem_op_t *op)
+{
+    ucc_datatype_t dtype;
+
+    dtype = shmem_datatype_to_ucc_dt[op->dt];
+    if (op->dt == OSHMEM_OP_TYPE_NUMBER) {
+        switch (op->dt_size) {
+            case 8:
+                return UCC_DT_INT64;
+            case 4:
+                return UCC_DT_INT32;
+            case 2:
+                return UCC_DT_INT16;
+            case 1:
+                return UCC_DT_INT8;
+            default:
+                return SCOLL_UCC_DT_UNSUPPORTED;
+        }
+    }
+
+    return dtype;
+}
+
+static ucc_reduction_op_t oshmem_op_to_ucc_op_map[OSHMEM_OP_NUMBER + 1] = {
+   UCC_OP_BAND,                  /* OSHMEM_OP_AND */
+   UCC_OP_BOR,                   /* OSHMEM_OP_OR */
+   UCC_OP_BXOR,                  /* OSHMEM_OP_XOR */
+   UCC_OP_MAX,                   /* OSHMEM_OP_MAX */
+   UCC_OP_MIN,                   /* OSHMEM_OP_MIN */
+   UCC_OP_SUM,                   /* OSHMEM_OP_SUM */
+   UCC_OP_PROD,                  /* OSHMEM_OP_PROD */
+   SCOLL_UCC_OP_UNSUPPORTED,     /* OSHMEM_OP_NUMBER */
+};
+
+static inline ucc_reduction_op_t shmem_op_to_ucc_op(int op)
+{
+    return oshmem_op_to_ucc_op_map[op];
+}
+
+#endif /* SCOLL_UCC_DTYPES_H */
diff --git a/oshmem/mca/scoll/ucc/scoll_ucc_module.c b/oshmem/mca/scoll/ucc/scoll_ucc_module.c
new file mode 100644
index 00000000000..90aa4243463
--- /dev/null
+++ b/oshmem/mca/scoll/ucc/scoll_ucc_module.c
@@ -0,0 +1,501 @@
+/*
+ * Copyright (c) 2021 Mellanox Technologies. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "oshmem_config.h"
+
+#include "opal/util/show_help.h"
+#include "opal/util/timings.h"
+#include "oshmem/proc/proc.h"
+#include "oshmem/runtime/runtime.h"
+#include "oshmem/mca/memheap/base/base.h"
+#include "ompi/mca/coll/base/coll_tags.h"
+#include "ompi/mca/pml/pml.h"
+#include "scoll_ucc.h"
+#include "scoll_ucc_debug.h"
+
+#include <ucc/api/ucc.h>
+
+#define OBJ_RELEASE_IF_NOT_NULL( obj ) if( NULL != (obj) ) OBJ_RELEASE( obj );
+
+int mca_scoll_ucc_init_query(bool enable_progress_threads, bool enable_ucc_threads)
+{
+    return OSHMEM_SUCCESS;
+}
+
+static void mca_scoll_ucc_module_clear(mca_scoll_ucc_module_t *ucc_module)
+{
+    ucc_module->previous_barrier      = NULL;
+    ucc_module->previous_broadcast    = NULL;
+    ucc_module->previous_reduce       = NULL;
+    ucc_module->previous_collect      = NULL;
+    ucc_module->previous_alltoall     = NULL;
+}
+
+static void mca_scoll_ucc_module_construct(mca_scoll_ucc_module_t *ucc_module)
+{
+    mca_scoll_ucc_module_clear(ucc_module);
+}
+
+int mca_scoll_ucc_progress(void)
+{
+    ucc_context_progress(mca_scoll_ucc_component.ucc_context);
+    return OSHMEM_SUCCESS;
+}
+
+static void mca_scoll_ucc_module_destruct(mca_scoll_ucc_module_t *ucc_module)
+{
+    ucc_status_t status;
+    if (ucc_module->ucc_team) {
+        while(UCC_INPROGRESS == (status = ucc_team_destroy(ucc_module->ucc_team))) {}
+        if (status != UCC_OK) {
+            UCC_ERROR("UCC team destroy failed");
+        }
+        MCA_MEMHEAP_CALL(private_free(ucc_module->pSync));
+        --mca_scoll_ucc_component.nr_modules;
+    }
+
+    if (0 == mca_scoll_ucc_component.nr_modules) {
+        if (mca_scoll_ucc_component.libucc_initialized) {
+            if (mca_scoll_ucc_component.ucc_context) {
+                opal_progress_unregister(mca_scoll_ucc_progress);
+                ucc_context_destroy(mca_scoll_ucc_component.ucc_context);
+            }
+            UCC_VERBOSE(1, "finalizing ucc library");
+            ucc_finalize(mca_scoll_ucc_component.ucc_lib);
+            mca_scoll_ucc_component.libucc_initialized = false;
+        }
+    }         
+
+    OBJ_RELEASE_IF_NOT_NULL(ucc_module->previous_alltoall_module);
+    OBJ_RELEASE_IF_NOT_NULL(ucc_module->previous_collect_module);
+    OBJ_RELEASE_IF_NOT_NULL(ucc_module->previous_reduce_module);
+    OBJ_RELEASE_IF_NOT_NULL(ucc_module->previous_broadcast_module);
+    OBJ_RELEASE_IF_NOT_NULL(ucc_module->previous_barrier_module);
+
+    mca_scoll_ucc_module_clear(ucc_module);
+}
+
+#define UCC_SAVE_PREV_SCOLL_API(__api) do {\
+    ucc_module->previous_ ## __api            = osh_group->g_scoll.scoll_ ## __api;\
+    ucc_module->previous_ ## __api ## _module = osh_group->g_scoll.scoll_ ## __api ## _module;\
+    if (!osh_group->g_scoll.scoll_ ## __api || !osh_group->g_scoll.scoll_ ## __api ## _module) {\
+        UCC_VERBOSE(1, "no underlying " # __api"; disqualifying myself");\
+        return OSHMEM_ERROR;\
+    }\
+    OBJ_RETAIN(ucc_module->previous_ ## __api ## _module);\
+} while(0)
+
+static int mca_scoll_ucc_save_coll_handlers(mca_scoll_base_module_t *module, 
+                                            oshmem_group_t *osh_group)
+{
+    mca_scoll_ucc_module_t* ucc_module = (mca_scoll_ucc_module_t*) module;
+    UCC_SAVE_PREV_SCOLL_API(barrier);
+    UCC_SAVE_PREV_SCOLL_API(broadcast);
+    UCC_SAVE_PREV_SCOLL_API(reduce);
+    UCC_SAVE_PREV_SCOLL_API(collect);
+    UCC_SAVE_PREV_SCOLL_API(alltoall);
+    return OSHMEM_SUCCESS;
+}
+
+typedef struct oob_allgather_req 
+{
+    void           *sbuf;
+    void           *rbuf;
+    void           *oob_coll_ctx;
+    size_t          msglen;
+    int             iter;
+    int             index;
+    ompi_request_t *reqs[2];
+} oob_allgather_req_t;
+
+static ucc_status_t oob_allgather_free(void *req)
+{
+    free(req);
+    return UCC_OK;
+}
+
+static ucc_status_t oob_allgather(void *sbuf, void *rbuf, size_t msglen,
+                                  void *oob_coll_ctx, void **req)
+{
+    oob_allgather_req_t *oob_req = malloc(sizeof(*oob_req));
+    oob_req->sbuf                = sbuf;
+    oob_req->rbuf                = rbuf;
+    oob_req->msglen              = msglen;
+    oob_req->oob_coll_ctx        = oob_coll_ctx;
+    oob_req->iter                = 0;
+    oob_req->index               = -1;
+    *req                         = oob_req;
+    return UCC_OK;
+}
+
+static inline ucc_status_t oob_probe_test(oob_allgather_req_t *oob_req) 
+{
+    int probe       = 0;
+    int probe_count = 5;
+    int completed;
+
+    do {
+        ompi_request_test_all(2, oob_req->reqs, &completed, MPI_STATUS_IGNORE);
+        ++probe;
+    } while (!completed && probe < probe_count);
+
+    if (!completed) {
+        return UCC_INPROGRESS;
+    }
+    return UCC_OK;
+}
+
+static int index_cmpfunc(const void * a, const void * b)
+{
+    return (*(int *)a - *(int *)b);
+}
+
+static ucc_status_t oob_allgather_test(void *req)
+{
+    oob_allgather_req_t *oob_req = (oob_allgather_req_t *)req;
+    oshmem_group_t      *group   = (oshmem_group_t *)oob_req->oob_coll_ctx;
+    char                *tmpsend = NULL;
+    char                *tmprecv = NULL;
+    int                 *index   = &oob_req->index;
+    size_t               msglen  = oob_req->msglen;
+    int                 *tmp;
+    unsigned int         rank;
+    int size, sendto, recvfrom, recvdatafrom, senddatafrom;
+
+    rank = group->my_pe;
+    size = group->proc_count;
+    if (-1 == *index) {
+        tmp =
+            bsearch(&rank, group->proc_vpids, size, sizeof(int), index_cmpfunc);
+        *index = ((ptrdiff_t)tmp - (ptrdiff_t)group->proc_vpids) /
+                 sizeof(group->proc_vpids[0]);
+    }
+
+    if (0 == oob_req->iter) {
+        tmprecv = (char *)oob_req->rbuf + (ptrdiff_t)*index * (ptrdiff_t)msglen;
+        memcpy(tmprecv, oob_req->sbuf, msglen);
+    }
+
+    sendto   = (*index + 1) % size;
+    sendto   = group->proc_vpids[sendto];
+    recvfrom = (*index - 1 + size) % size;
+    recvfrom = group->proc_vpids[recvfrom];
+    for (; oob_req->iter < size - 1; oob_req->iter++) {
+        if (oob_req->iter > 0) {
+            if (UCC_INPROGRESS == oob_probe_test(oob_req)) {
+                return UCC_INPROGRESS;
+            }
+        }
+
+        recvdatafrom = (*index - oob_req->iter - 1 + size) % size;
+        senddatafrom = (*index - oob_req->iter + size) % size;
+        tmprecv = (char *) oob_req->rbuf + (ptrdiff_t) recvdatafrom * (ptrdiff_t) msglen;
+        tmpsend = (char *) oob_req->rbuf + (ptrdiff_t) senddatafrom * (ptrdiff_t) msglen;
+        MCA_PML_CALL(isend(tmpsend, msglen, MPI_BYTE, sendto, MCA_COLL_BASE_TAG_UCC,
+                     MCA_PML_BASE_SEND_STANDARD, oshmem_comm_world, &oob_req->reqs[0]));
+        MCA_PML_CALL(irecv(tmprecv, msglen, MPI_BYTE, recvfrom, 
+                     MCA_COLL_BASE_TAG_UCC, oshmem_comm_world, &oob_req->reqs[1]));
+    }
+    return oob_probe_test(oob_req);
+}
+
+static int mca_scoll_ucc_init(oshmem_group_t *osh_group)
+{
+    mca_scoll_ucc_component_t *cm   = &mca_scoll_ucc_component;
+    ucc_lib_config_h           lib_config;
+    ucc_thread_mode_t          tm_requested;
+    ucc_lib_params_t           lib_params;
+
+    tm_requested           = oshmem_mpi_thread_multiple ? UCC_THREAD_MULTIPLE :
+                                                          UCC_THREAD_SINGLE;
+    lib_params.mask        = UCC_LIB_PARAM_FIELD_THREAD_MODE;
+    lib_params.thread_mode = tm_requested;
+
+    if (UCC_OK != ucc_lib_config_read("OSHMEM", NULL, &lib_config)) {
+        UCC_ERROR("UCC lib config read failed");
+        return OSHMEM_ERROR;
+    }
+    if (strlen(cm->cls) > 0) {
+        if (UCC_OK != ucc_lib_config_modify(lib_config, "CLS", cm->cls)) {
+            ucc_lib_config_release(lib_config);
+            UCC_ERROR("failed to modify UCC lib config to set CLS");
+            return OSHMEM_ERROR;
+        }
+    }
+    if (UCC_OK != ucc_init(&lib_params, lib_config, &cm->ucc_lib)) {
+        UCC_ERROR("UCC lib init failed");
+        ucc_lib_config_release(lib_config);
+        cm->ucc_enable = 0;
+        return OSHMEM_ERROR;
+    }
+    ucc_lib_config_release(lib_config);
+
+    cm->ucc_lib_attr.mask = UCC_LIB_ATTR_FIELD_THREAD_MODE |
+                            UCC_LIB_ATTR_FIELD_COLL_TYPES;
+    if (UCC_OK != ucc_lib_get_attr(cm->ucc_lib, &cm->ucc_lib_attr)) {
+        UCC_ERROR("UCC get lib attr failed");
+        goto cleanup_lib;
+    }
+
+    if (cm->ucc_lib_attr.thread_mode < tm_requested) {
+        UCC_ERROR("UCC library doesn't support SHMEM_THREAD_MULTIPLE");
+        goto cleanup_lib;
+    }
+
+    cm->libucc_initialized = true;
+    return OSHMEM_SUCCESS;
+
+cleanup_lib:
+    ucc_finalize(cm->ucc_lib);
+    cm->ucc_enable         = 0;
+    cm->libucc_initialized = false;
+    return OSHMEM_ERROR;
+}
+
+int mca_scoll_ucc_init_ctx(oshmem_group_t *osh_group) 
+{
+    mca_scoll_ucc_component_t *cm   = &mca_scoll_ucc_component;
+    ucc_mem_map_t             *maps = NULL;
+    char                       str_buf[256];
+    ucc_context_config_h       ctx_config;
+    ucc_context_params_t       ctx_params;
+    int                        segment;
+
+    maps = (ucc_mem_map_t *)malloc(sizeof(ucc_mem_map_t) *
+                                   memheap_map->n_segments);
+    if (NULL == maps) {
+        UCC_ERROR("failed to allocate space for UCC memory params");
+    }
+    for (segment = 0; segment < memheap_map->n_segments; segment++) {
+        maps[segment].address = memheap_map->mem_segs[segment].mkeys[0].va_base;
+        maps[segment].len =
+            (ptrdiff_t)memheap_map->mem_segs[segment].super.va_end -
+            (ptrdiff_t)memheap_map->mem_segs[segment].super.va_base;
+    }
+    ctx_params.mask =
+        UCC_CONTEXT_PARAM_FIELD_OOB | UCC_CONTEXT_PARAM_FIELD_MEM_PARAMS;
+    ctx_params.oob.allgather         = oob_allgather;
+    ctx_params.oob.req_test          = oob_allgather_test;
+    ctx_params.oob.req_free          = oob_allgather_free;
+    ctx_params.oob.coll_info         = (void *)oshmem_group_all;
+    ctx_params.oob.n_oob_eps         = oshmem_group_all->proc_count;
+    ctx_params.oob.oob_ep            = oshmem_group_all->my_pe;
+    ctx_params.mem_params.segments   = maps;
+    ctx_params.mem_params.n_segments = memheap_map->n_segments;
+
+    if (UCC_OK != ucc_context_config_read(cm->ucc_lib, NULL, &ctx_config)) {
+        UCC_ERROR("UCC context config read failed");
+        goto cleanup_lib;
+    }
+
+    sprintf(str_buf, "%u", osh_group->proc_count);
+    if (UCC_OK != ucc_context_config_modify(ctx_config, NULL,
+                                            "ESTIMATED_NUM_EPS", str_buf)) {
+        UCC_ERROR("UCC context config modify failed for estimated_num_eps");
+        goto cleanup_lib;
+    }
+
+    sprintf(str_buf, "%u", opal_process_info.num_local_peers + 1);
+    if (UCC_OK != ucc_context_config_modify(ctx_config, NULL,
+                                            "ESTIMATED_NUM_PPN", str_buf)) {
+        UCC_ERROR("UCC context config modify failed for estimated_num_eps");
+        goto cleanup_lib;
+    }
+
+    if (UCC_OK != ucc_context_create(cm->ucc_lib, &ctx_params, ctx_config,
+                                     &cm->ucc_context)) {
+        UCC_ERROR("UCC context create failed");
+        ucc_context_config_release(ctx_config);
+        goto cleanup_lib;
+    }
+    ucc_context_config_release(ctx_config);
+
+    free(maps);
+    opal_progress_register(mca_scoll_ucc_progress);
+    UCC_VERBOSE(1, "initialized ucc context");
+    cm->libucc_initialized = true;
+    return OSHMEM_SUCCESS;
+
+cleanup_lib:
+    if (NULL != maps) {
+        free(maps);
+    }
+    ucc_finalize(cm->ucc_lib);
+    cm->ucc_enable         = 0;
+    cm->libucc_initialized = false;
+    return OSHMEM_ERROR;
+}
+
+int mca_scoll_ucc_team_create(mca_scoll_ucc_module_t *ucc_module,
+                              oshmem_group_t         *osh_group)
+{
+    mca_scoll_ucc_component_t *cm       = &mca_scoll_ucc_component;
+    ucc_status_t               status   = UCC_OK;
+    long                      *pSync    = NULL;
+    int                       *tmp;
+    ucc_ep_map_t               map;
+    int                        index;
+    size_t                     size;
+    ucc_context_attr_t         attr;
+
+    attr.mask = UCC_CONTEXT_ATTR_FIELD_WORK_BUFFER_SIZE;
+    ucc_context_get_attr(cm->ucc_context, &attr);
+    size = attr.global_work_buffer_size;
+    if (size & 0x7) {
+        size += 8 - (size & 0x7);
+    }
+    MCA_MEMHEAP_CALL(private_alloc(size * sizeof(long), (void **)&pSync));
+    memset(pSync, 0, size * sizeof(long));
+
+    map.type            = UCC_EP_MAP_ARRAY;
+    map.ep_num          = osh_group->proc_count;
+    map.array.elem_size = 4;
+    tmp                 = bsearch(&osh_group->my_pe, osh_group->proc_vpids,
+                                  osh_group->proc_count, sizeof(int), index_cmpfunc);
+    index               = ((ptrdiff_t)tmp - (ptrdiff_t)osh_group->proc_vpids) /
+            sizeof(osh_group->proc_vpids[0]);
+    map.array.map                 = (void *)osh_group->proc_vpids;
+    ucc_team_params_t team_params = {
+        .mask = UCC_TEAM_PARAM_FIELD_EP | UCC_TEAM_PARAM_FIELD_EP_RANGE |
+                UCC_TEAM_PARAM_FIELD_OOB | UCC_TEAM_PARAM_FIELD_FLAGS,
+        .oob =
+            {
+                .allgather = oob_allgather,
+                .req_test  = oob_allgather_test,
+                .req_free  = oob_allgather_free,
+                .coll_info = (void *)osh_group,
+                .n_oob_eps = osh_group->proc_count,
+                .oob_ep    = index,
+            },
+        .ep     = index,
+        .ep_map = map,
+        .flags  = UCC_TEAM_FLAG_COLL_WORK_BUFFER,
+    };
+
+    if (UCC_OK != ucc_team_create_post(&cm->ucc_context, 1, &team_params,
+                                       &ucc_module->ucc_team)) {
+        UCC_ERROR("ucc_team_create_post failed");
+        goto err;
+    }
+
+    while (UCC_INPROGRESS ==
+           (status = ucc_team_create_test(ucc_module->ucc_team))) {
+        opal_progress();
+    }
+    if (UCC_OK != status) {
+        UCC_ERROR("ucc_team_create_test failed (%d)", status);
+        goto err;
+    }
+    ucc_module->pSync = pSync;
+    ++cm->nr_modules;
+    return OSHMEM_SUCCESS;
+err:
+    ucc_module->ucc_team = NULL;
+    cm->ucc_enable       = 0;
+    opal_progress_unregister(mca_scoll_ucc_progress);
+    return OSHMEM_ERROR;
+}
+
+/*
+ * Initialize module on the communicator
+ */
+static int mca_scoll_ucc_module_enable(mca_scoll_base_module_t *module,
+                                       oshmem_group_t *osh_group)
+{
+    mca_scoll_ucc_component_t *cm         = &mca_scoll_ucc_component;
+    mca_scoll_ucc_module_t    *ucc_module = (mca_scoll_ucc_module_t *) module;
+
+    ucc_module->ucc_team = NULL;
+    if (OSHMEM_SUCCESS != mca_scoll_ucc_save_coll_handlers(module, osh_group)) {
+        UCC_ERROR("UCC module enable failed");
+        /* There are no modules available */
+        opal_show_help("help-oshmem-scoll-ucc.txt",
+                       "module_enable:fatal", true,
+    	       		   "UCC module enable failed - aborting to prevent inconsistent application state");
+        goto err;
+    }
+    UCC_VERBOSE(1, "ucc enabled");
+    return OSHMEM_SUCCESS;
+err:
+    cm->ucc_enable = 0;
+    opal_progress_unregister(mca_scoll_ucc_progress);
+    return OSHMEM_ERROR;
+}
+
+#define SET_SCOLL_PTR(_module, _COLL, _coll) do {                       \
+    _module->super.scoll_  ## _coll = NULL;                             \
+    if ((mca_scoll_ucc_component.ucc_lib_attr.coll_types &              \
+         UCC_COLL_TYPE_ ## _COLL)) {                                    \
+        if (mca_scoll_ucc_component.cts_requested &                     \
+            UCC_COLL_TYPE_ ## _COLL) {                                  \
+            _module->super.scoll_ ## _coll  = mca_scoll_ucc_  ## _coll; \
+        }                                                               \
+    }                                                                   \
+} while(0)
+
+/*
+ * Invoked when there's a new communicator/group that has been created.
+ * Look at the communicator and decide which set of functions and
+ * priority we want to return.
+ */
+mca_scoll_base_module_t *
+mca_scoll_ucc_comm_query(oshmem_group_t *osh_group, int *priority)
+{
+    mca_scoll_base_module_t   *module;
+    mca_scoll_ucc_module_t    *ucc_module;
+    mca_scoll_ucc_component_t *cm;
+    
+    *priority = 0;
+    cm = &mca_scoll_ucc_component;
+
+    if (!cm->ucc_enable) {
+        return NULL;
+    }
+
+    if ((osh_group->proc_count < 2) || (osh_group->proc_count < cm->ucc_np)) {
+        return NULL;
+    }
+    OPAL_TIMING_ENV_INIT(comm_query);
+
+    if (!cm->libucc_initialized) {
+        if (memheap_map && memheap_map->n_segments > 0) {
+            if (OSHMEM_SUCCESS != mca_scoll_ucc_init(osh_group)) {
+                cm->ucc_enable = 0;
+                return NULL;
+            }
+        }
+    }
+
+    ucc_module = OBJ_NEW(mca_scoll_ucc_module_t);
+    if (!ucc_module) {
+        cm->ucc_enable = 0;
+        return NULL;
+    }
+
+    ucc_module->group = osh_group;
+    ucc_module->super.scoll_module_enable = mca_scoll_ucc_module_enable;
+    *priority = cm->ucc_priority;
+    SET_SCOLL_PTR(ucc_module, BARRIER, barrier);
+    SET_SCOLL_PTR(ucc_module, BCAST, broadcast);
+    SET_SCOLL_PTR(ucc_module, ALLREDUCE, reduce);
+    SET_SCOLL_PTR(ucc_module, ALLGATHER, collect);
+    SET_SCOLL_PTR(ucc_module, ALLTOALL, alltoall);
+
+    module = &ucc_module->super;
+    return module;
+}
+
+
+OBJ_CLASS_INSTANCE(mca_scoll_ucc_module_t,
+        mca_scoll_base_module_t,
+        mca_scoll_ucc_module_construct,
+        mca_scoll_ucc_module_destruct);
+
+
+
diff --git a/oshmem/mca/scoll/ucc/scoll_ucc_reduce.c b/oshmem/mca/scoll/ucc/scoll_ucc_reduce.c
new file mode 100644
index 00000000000..6b99f6ea983
--- /dev/null
+++ b/oshmem/mca/scoll/ucc/scoll_ucc_reduce.c
@@ -0,0 +1,107 @@
+/**
+  Copyright (c) 2021 Mellanox Technologies. All rights reserved.
+  $COPYRIGHT$
+
+  Additional copyrights may follow
+
+  $HEADER$
+ */
+#include "scoll_ucc.h"
+#include "scoll_ucc_dtypes.h"
+#include "scoll_ucc_common.h"
+
+#include <ucc/api/ucc.h>
+
+static inline ucc_status_t mca_scoll_ucc_reduce_init(const void *sbuf, void *rbuf,
+                                                     int count, struct oshmem_op_t * op,
+                                                     mca_scoll_ucc_module_t * ucc_module,
+                                                     ucc_coll_req_h * req)
+{
+    ucc_datatype_t ucc_dt;
+    ucc_reduction_op_t ucc_op;
+
+    ucc_dt = shmem_op_to_ucc_dtype(op);
+    ucc_op = shmem_op_to_ucc_op(op->op);
+
+    if (OPAL_UNLIKELY((ucc_datatype_t) SCOLL_UCC_DT_UNSUPPORTED == ucc_dt)) {
+        UCC_VERBOSE(5, "shmem datatype is not supported: dtype # = %d", 
+            op->dt);
+    }
+    if (OPAL_UNLIKELY((ucc_reduction_op_t) SCOLL_UCC_OP_UNSUPPORTED == ucc_op)) {
+        UCC_VERBOSE(5, "shmem reduction op is not supported: op # = %d", 
+            op->op);
+    }
+
+    ucc_coll_args_t coll = {
+        .mask = 0,
+        .coll_type = UCC_COLL_TYPE_ALLREDUCE,
+        .src.info = {
+            .buffer = (void *)sbuf,
+            .count = count,
+            .datatype = ucc_dt,
+            .mem_type = UCC_MEMORY_TYPE_UNKNOWN
+        },
+        .dst.info = {
+            .buffer = rbuf,
+            .count = count,
+            .datatype = ucc_dt,
+            .mem_type = UCC_MEMORY_TYPE_UNKNOWN
+        },
+        .op = ucc_op,
+    };
+
+    if (sbuf == rbuf) {
+        coll.mask |= UCC_COLL_ARGS_FIELD_FLAGS;
+        coll.flags = UCC_COLL_ARGS_FLAG_IN_PLACE;
+    }
+
+    if (NULL == mca_scoll_ucc_component.ucc_context) {
+        if (OSHMEM_ERROR == mca_scoll_ucc_init_ctx(ucc_module->group)) {
+            return OSHMEM_ERROR;
+        }
+    }
+
+    if (NULL == ucc_module->ucc_team) {
+        if (OSHMEM_ERROR == mca_scoll_ucc_team_create(ucc_module, ucc_module->group)) {
+            return OSHMEM_ERROR;
+        }
+    }
+    SCOLL_UCC_REQ_INIT(req, coll, ucc_module);
+    return UCC_OK;
+fallback:
+    return UCC_ERR_NOT_SUPPORTED;
+}
+
+int mca_scoll_ucc_reduce(struct oshmem_group_t *group,
+                         struct oshmem_op_t *op,
+                         void *target,
+                         const void *source,
+                         size_t nlong,
+                         long *pSync,
+                         void *pWrk,
+                         int alg)
+{
+    mca_scoll_ucc_module_t *ucc_module;
+    size_t count;
+    ucc_coll_req_h req;
+    int rc;
+
+    UCC_VERBOSE(3, "running ucc reduce");
+    ucc_module = (mca_scoll_ucc_module_t *) group->g_scoll.scoll_reduce_module;
+    count = nlong / op->dt_size;
+
+    /* Do nothing on zero-length request */
+    if (OPAL_UNLIKELY(!nlong)) {
+        return OSHMEM_SUCCESS;
+    }
+
+    SCOLL_UCC_CHECK(mca_scoll_ucc_reduce_init(source, target, count, op, ucc_module, &req));
+    SCOLL_UCC_CHECK(ucc_collective_post(req));
+    SCOLL_UCC_CHECK(scoll_ucc_req_wait(req));
+    return OSHMEM_SUCCESS;
+fallback:
+    UCC_VERBOSE(3, "running fallback reduction");
+    PREVIOUS_SCOLL_FN(ucc_module, reduce, group, op, target,
+                      source, nlong, pSync, pWrk, alg);
+    return rc;
+}
diff --git a/oshmem/mca/spml/base/base.h b/oshmem/mca/spml/base/base.h
index 75a4eaec18d..47b66bc15a5 100644
--- a/oshmem/mca/spml/base/base.h
+++ b/oshmem/mca/spml/base/base.h
@@ -78,7 +78,7 @@ OSHMEM_DECLSPEC int mca_spml_base_oob_get_mkeys(shmem_ctx_t ctx,
                                                 sshmem_mkey_t *mkeys);
 
 OSHMEM_DECLSPEC void mca_spml_base_rmkey_unpack(shmem_ctx_t ctx, sshmem_mkey_t *mkey, uint32_t seg, int pe, int tr_id);
-OSHMEM_DECLSPEC void mca_spml_base_rmkey_free(sshmem_mkey_t *mkey);
+OSHMEM_DECLSPEC void mca_spml_base_rmkey_free(sshmem_mkey_t *mkey, int pe);
 OSHMEM_DECLSPEC void *mca_spml_base_rmkey_ptr(const void *dst_addr, sshmem_mkey_t *mkey, int pe);
 
 OSHMEM_DECLSPEC int mca_spml_base_put_nb(void *dst_addr,
@@ -109,7 +109,7 @@ OSHMEM_DECLSPEC extern mca_base_framework_t oshmem_spml_base_framework;
 #define __SPML_FILE__ __FILE__
 #endif
 
-#ifdef OPAL_ENABLE_DEBUG
+#if OPAL_ENABLE_DEBUG
 #define SPML_VERBOSE(level, ...) \
     oshmem_output_verbose(level, oshmem_spml_base_framework.framework_output, \
         "%s:%d - %s()", __SPML_FILE__, __LINE__, __func__, __VA_ARGS__)
diff --git a/oshmem/mca/spml/base/spml_base.c b/oshmem/mca/spml/base/spml_base.c
index 27e7d7a58e8..1362e07ecc5 100644
--- a/oshmem/mca/spml/base/spml_base.c
+++ b/oshmem/mca/spml/base/spml_base.c
@@ -48,184 +48,74 @@
         }                                          \
     } while (_res == 0);
 
+#define SPML_BASE_TEST_CASE(_type, _shmem_type, _addr, _value, _cmp, _out_value) \
+    case _shmem_type: \
+        { \
+            _type typed_value = *(const _type*)_value; \
+            const _type *typed_addr = (const _type*)_addr; \
+            SPML_BASE_DO_CMP((*_out_value), typed_addr , _cmp, typed_value); \
+        } \
+        break;
+
 /**
  * Check on a variable given in addr to see it is not equal to value.
  */
 int mca_spml_base_test(void* addr, int cmp, void* value, int datatype, int *out_value)
 {
-    volatile int *int_addr;
-    volatile long *long_addr;
-    volatile short *short_addr;
-    volatile long long *longlong_addr;
-    volatile int32_t *int32_addr;
-    volatile int64_t *int64_addr;
-
-    int int_value;
-    long long_value;
-    short short_value;
-    long long longlong_value;
-    int32_t int32_value;
-    int64_t int64_value;
-
-    ompi_fortran_integer_t *fint_addr, fint_value;
-    ompi_fortran_integer4_t *fint4_addr, fint4_value;
-    ompi_fortran_integer8_t *fint8_addr, fint8_value;
-
     switch (datatype) {
-
-    /* Int */
-    case SHMEM_INT:
-        int_value = *(int*) value;
-        int_addr = (int*) addr;
-        SPML_BASE_DO_CMP((*out_value), int_addr, cmp, int_value);
-        break;
-
-        /* Short */
-    case SHMEM_SHORT:
-        short_value = *(short*) value;
-        short_addr = (short*) addr;
-        SPML_BASE_DO_CMP((*out_value), short_addr, cmp, short_value);
-        break;
-
-        /* Long */
-    case SHMEM_LONG:
-        long_value = *(long*) value;
-        long_addr = (long*) addr;
-        SPML_BASE_DO_CMP((*out_value), long_addr, cmp, long_value);
-        break;
-
-        /* Long-Long */
-    case SHMEM_LLONG:
-        longlong_value = *(long long*) value;
-        longlong_addr = (long long*) addr;
-        SPML_BASE_DO_CMP((*out_value), longlong_addr, cmp, longlong_value);
-        break;
-
-       /* Int32_t */
-    case SHMEM_INT32_T:
-        int32_value = *(int32_t*) value;
-        int32_addr = (int32_t*) addr;
-        SPML_BASE_DO_CMP((*out_value), int32_addr, cmp, int32_value);
-        break;
-
-       /* Int64_t */
-    case SHMEM_INT64_T:
-        int64_value = *(int64_t*) value;
-        int64_addr = (int64_t*) addr;
-        SPML_BASE_DO_CMP((*out_value), int64_addr, cmp, int64_value);
-        break;
-
-        /*C equivalent of Fortran integer type */
-    case SHMEM_FINT:
-        fint_value = *(ompi_fortran_integer_t *) value;
-        fint_addr = (ompi_fortran_integer_t *) addr;
-        SPML_BASE_DO_CMP((*out_value), fint_addr, cmp, fint_value);
-        break;
-
-        /*C equivalent of Fortran int4 type*/
-    case SHMEM_FINT4:
-        fint4_value = *(ompi_fortran_integer4_t *) value;
-        fint4_addr = (ompi_fortran_integer4_t *) addr;
-        SPML_BASE_DO_CMP((*out_value), fint4_addr, cmp, fint4_value);
-        break;
-
-        /*C equivalent of Fortran int8 type*/
-    case SHMEM_FINT8:
-        fint8_value = *(ompi_fortran_integer8_t *) value;
-        fint8_addr = (ompi_fortran_integer8_t *) addr;
-        SPML_BASE_DO_CMP((*out_value), fint8_addr, cmp, fint8_value);
-        break;
+        SPML_BASE_TEST_CASE(int, SHMEM_INT, addr, value, cmp, out_value);
+        SPML_BASE_TEST_CASE(unsigned int, SHMEM_UINT, addr, value, cmp, out_value);
+        SPML_BASE_TEST_CASE(long, SHMEM_LONG, addr, value, cmp, out_value);
+        SPML_BASE_TEST_CASE(unsigned long, SHMEM_ULONG, addr, value, cmp, out_value);
+        SPML_BASE_TEST_CASE(short, SHMEM_SHORT, addr, value, cmp, out_value);
+        SPML_BASE_TEST_CASE(unsigned short, SHMEM_USHORT, addr, value, cmp, out_value);
+        SPML_BASE_TEST_CASE(long long, SHMEM_LLONG, addr, value, cmp, out_value);
+        SPML_BASE_TEST_CASE(unsigned long long, SHMEM_ULLONG, addr, value, cmp, out_value);
+        SPML_BASE_TEST_CASE(int32_t, SHMEM_INT32_T, addr, value, cmp, out_value);
+        SPML_BASE_TEST_CASE(uint32_t, SHMEM_UINT32_T, addr, value, cmp, out_value);
+        SPML_BASE_TEST_CASE(int64_t, SHMEM_INT64_T, addr, value, cmp, out_value);
+        SPML_BASE_TEST_CASE(uint64_t, SHMEM_UINT64_T, addr, value, cmp, out_value);
+        SPML_BASE_TEST_CASE(size_t, SHMEM_SIZE_T, addr, value, cmp, out_value);
+        SPML_BASE_TEST_CASE(ptrdiff_t, SHMEM_PTRDIFF_T, addr, value, cmp, out_value);
+        SPML_BASE_TEST_CASE(ompi_fortran_integer_t, SHMEM_FINT, addr, value, cmp, out_value);
+        SPML_BASE_TEST_CASE(ompi_fortran_integer4_t, SHMEM_FINT4, addr, value, cmp, out_value);
+        SPML_BASE_TEST_CASE(ompi_fortran_integer8_t, SHMEM_FINT8, addr, value, cmp, out_value);
     }
 
     return OSHMEM_SUCCESS;
 }
 
+#define SPML_BASE_WAIT_CASE(_type, _shmem_type, _addr, _value, _cmp, _res) \
+    case _shmem_type: \
+        { \
+            _type typed_value = *(const _type*)_value; \
+            const _type *typed_addr = (const _type*)_addr; \
+            SPML_BASE_DO_WAIT(_res, typed_addr, _cmp, typed_value); \
+        } \
+        break;
+
 int mca_spml_base_wait(void* addr, int cmp, void* value, int datatype)
 {
-    volatile int *int_addr;
-    volatile long *long_addr;
-    volatile short *short_addr;
-    volatile long long *longlong_addr;
-    volatile int32_t *int32_addr;
-    volatile int64_t *int64_addr;
-
-    int int_value;
-    long long_value;
-    short short_value;
-    long long longlong_value;
-    int32_t int32_value;
-    int64_t int64_value;
-
-    ompi_fortran_integer_t *fint_addr, fint_value;
-    ompi_fortran_integer4_t *fint4_addr, fint4_value;
-    ompi_fortran_integer8_t *fint8_addr, fint8_value;
-
     int res = 0;
 
     switch (datatype) {
-
-    /* Int */
-    case SHMEM_INT:
-        int_value = *(int*) value;
-        int_addr = (int*) addr;
-        SPML_BASE_DO_WAIT(res, int_addr, cmp, int_value);
-        break;
-
-        /* Short */
-    case SHMEM_SHORT:
-        short_value = *(short*) value;
-        short_addr = (short*) addr;
-        SPML_BASE_DO_WAIT(res, short_addr, cmp, short_value);
-        break;
-
-        /* Long */
-    case SHMEM_LONG:
-        long_value = *(long*) value;
-        long_addr = (long*) addr;
-        SPML_BASE_DO_WAIT(res, long_addr, cmp, long_value);
-        break;
-
-        /* Long-Long */
-    case SHMEM_LLONG:
-        longlong_value = *(long long*) value;
-        longlong_addr = (long long*) addr;
-        SPML_BASE_DO_WAIT(res, longlong_addr, cmp, longlong_value);
-        break;
-
-       /* Int32_t */
-    case SHMEM_INT32_T:
-        int32_value = *(int32_t*) value;
-        int32_addr = (int32_t*) addr;
-        SPML_BASE_DO_WAIT(res, int32_addr, cmp, int32_value);
-        break;
-
-       /* Int64_t */
-    case SHMEM_INT64_T:
-        int64_value = *(int64_t*) value;
-        int64_addr = (int64_t*) addr;
-        SPML_BASE_DO_WAIT(res, int64_addr, cmp, int64_value);
-        break;
-
-        /*C equivalent of Fortran integer type */
-    case SHMEM_FINT:
-        fint_value = *(ompi_fortran_integer_t *) value;
-        fint_addr = (ompi_fortran_integer_t *) addr;
-        SPML_BASE_DO_WAIT(res, fint_addr, cmp, fint_value);
-        break;
-
-        /*C equivalent of Fortran int4 type*/
-    case SHMEM_FINT4:
-        fint4_value = *(ompi_fortran_integer4_t *) value;
-        fint4_addr = (ompi_fortran_integer4_t *) addr;
-        SPML_BASE_DO_WAIT(res, fint4_addr, cmp, fint4_value);
-        break;
-
-        /*C equivalent of Fortran int8 type*/
-    case SHMEM_FINT8:
-        fint8_value = *(ompi_fortran_integer8_t *) value;
-        fint8_addr = (ompi_fortran_integer8_t *) addr;
-        SPML_BASE_DO_WAIT(res, fint8_addr, cmp, fint8_value);
-        break;
+        SPML_BASE_WAIT_CASE(int, SHMEM_INT, addr, value, cmp, res);
+        SPML_BASE_WAIT_CASE(unsigned int, SHMEM_UINT, addr, value, cmp, res);
+        SPML_BASE_WAIT_CASE(long, SHMEM_LONG, addr, value, cmp, res);
+        SPML_BASE_WAIT_CASE(unsigned long, SHMEM_ULONG, addr, value, cmp, res);
+        SPML_BASE_WAIT_CASE(short, SHMEM_SHORT, addr, value, cmp, res);
+        SPML_BASE_WAIT_CASE(unsigned short, SHMEM_USHORT, addr, value, cmp, res);
+        SPML_BASE_WAIT_CASE(long long, SHMEM_LLONG, addr, value, cmp, res);
+        SPML_BASE_WAIT_CASE(unsigned long long, SHMEM_ULLONG, addr, value, cmp, res);
+        SPML_BASE_WAIT_CASE(int32_t, SHMEM_INT32_T, addr, value, cmp, res);
+        SPML_BASE_WAIT_CASE(uint32_t, SHMEM_UINT32_T, addr, value, cmp, res);
+        SPML_BASE_WAIT_CASE(int64_t, SHMEM_INT64_T, addr, value, cmp, res);
+        SPML_BASE_WAIT_CASE(uint64_t, SHMEM_UINT64_T, addr, value, cmp, res);
+        SPML_BASE_WAIT_CASE(size_t, SHMEM_SIZE_T, addr, value, cmp, res);
+        SPML_BASE_WAIT_CASE(ptrdiff_t, SHMEM_PTRDIFF_T, addr, value, cmp, res);
+        SPML_BASE_WAIT_CASE(ompi_fortran_integer_t, SHMEM_FINT, addr, value, cmp, res);
+        SPML_BASE_WAIT_CASE(ompi_fortran_integer4_t, SHMEM_FINT4, addr, value, cmp, res);
+        SPML_BASE_WAIT_CASE(ompi_fortran_integer8_t, SHMEM_FINT8, addr, value, cmp, res);
     }
 
     return OSHMEM_SUCCESS;
@@ -256,7 +146,7 @@ void mca_spml_base_rmkey_unpack(shmem_ctx_t ctx, sshmem_mkey_t *mkey, uint32_t s
 {
 }
 
-void mca_spml_base_rmkey_free(sshmem_mkey_t *mkey)
+void mca_spml_base_rmkey_free(sshmem_mkey_t *mkey, int pe)
 {
 }
 
diff --git a/oshmem/mca/spml/base/spml_base_select.c b/oshmem/mca/spml/base/spml_base_select.c
index 22b68ee7422..69f5f112e16 100644
--- a/oshmem/mca/spml/base/spml_base_select.c
+++ b/oshmem/mca/spml/base/spml_base_select.c
@@ -168,7 +168,7 @@ int mca_spml_base_select(bool enable_progress_threads, bool enable_mpi_threads)
 
             if (NULL != om->om_component->spmlm_finalize) {
 
-                /* Blatently ignore the return code (what would we do to
+                /* Blatantly ignore the return code (what would we do to
                  recover, anyway?  This component is going away, so errors
                  don't matter anymore) */
 
diff --git a/oshmem/mca/spml/spml.h b/oshmem/mca/spml/spml.h
index ca62b5f0bd4..4ef7eefdf51 100644
--- a/oshmem/mca/spml/spml.h
+++ b/oshmem/mca/spml/spml.h
@@ -150,7 +150,7 @@ typedef void * (*mca_spml_base_module_mkey_ptr_fn_t)(const void *dst_addr, sshme
  *
  * @param mkey remote mkey
  */
-typedef void (*mca_spml_base_module_mkey_free_fn_t)(sshmem_mkey_t *);
+typedef void (*mca_spml_base_module_mkey_free_fn_t)(sshmem_mkey_t *, int pe);
 
 /**
  * Register (Pinn) a buffer of 'size' bits starting in address addr
@@ -193,9 +193,9 @@ typedef int (*mca_spml_base_module_oob_get_mkeys_fn_t)(shmem_ctx_t ctx, int pe,
  * @return OSHMEM_SUCCESS or failure status.
  *
  */
-typedef int (*mca_spml_base_module_add_procs_fn_t)(ompi_proc_t** procs,
+typedef int (*mca_spml_base_module_add_procs_fn_t)(struct oshmem_group_t* group,
                                                    size_t nprocs);
-typedef int (*mca_spml_base_module_del_procs_fn_t)(ompi_proc_t** procs,
+typedef int (*mca_spml_base_module_del_procs_fn_t)(struct oshmem_group_t* group,
                                                    size_t nprocs);
 
 
@@ -253,6 +253,647 @@ typedef int (*mca_spml_base_module_put_nb_fn_t)(shmem_ctx_t ctx,
                                                 int dst,
                                                 void **handle);
 
+
+
+/**
+ * The put-with-signal routines provide a method for copying data from a
+ * contiguous local data object to a data object on a specified PE and
+ * subsequently updating a remote flag to signal completion.
+ *
+ * @param ctx      A context handle specifying the context on which to perform the
+ *                 operation. When this argument is not provided, the operation is
+ *                 performed on the default context.
+ * @param dst_addr The address in the remote PE of the object being written.
+ * @param size     The number of bytes to be written.
+ * @param src_addr An address on the local PE holdng the value to be written.
+ * @param sig_addr Symmetric address of the signal data object to be updated on the
+ *                 remote PE as a signal.
+ * @param signal   Unsigned 64-bit value that is used for updating the remote sig_addr
+ *                 signal data object.
+ * @param sig_op   Signal operator that represents the type of update to be performed
+ *                 on the remote sig_addr signal data object.
+ * @param pe       PE number of the remote PE.
+ *
+ * @return         OSHMEM_SUCCESS or failure status.
+ */
+
+typedef int (*mca_spml_base_module_put_signal_fn_t)(shmem_ctx_t ctx,
+                                                    void* dst_addr,
+                                                    size_t size,
+                                                    void* src_addr,
+                                                    uint64_t *sig_addr,
+                                                    uint64_t signal,
+                                                    int sig_op,
+                                                    int dst);
+
+
+/**
+ * The nonblocking put-with-signal routines provide a method for copying data
+ * from a contiguous local data object to a data object on a specified PE and
+ * subsequently updating a remote flag to signal completion.
+ *
+ * @param ctx      A context handle specifying the context on which to perform the
+ *                 operation. When this argument is not provided, the operation is
+ *                 performed on the default context.
+ * @param dst_addr The address in the remote PE of the object being written.
+ * @param size     The number of bytes to be written.
+ * @param src_addr An address on the local PE holdng the value to be written.
+ * @param sig_addr Symmetric address of the signal data object to be updated on the
+ *                 remote PE as a signal.
+ * @param signal   Unsigned 64-bit value that is used for updating the remote sig_addr
+ *                 signal data object.
+ * @param sig_op   Signal operator that represents the type of update to be performed
+ *                 on the remote sig_addr signal data object.
+ * @param pe       PE number of the remote PE.
+ *
+ * @return         OSHMEM_SUCCESS or failure status.
+ */
+typedef int (*mca_spml_base_module_put_signal_nb_fn_t) (shmem_ctx_t ctx,
+                                            void* dst_addr,
+                                            size_t size,
+                                            void* src_addr,
+                                            uint64_t *sig_addr,
+                                            uint64_t signal,
+                                            int sig_op,
+                                            int dst);
+
+/*
+ *  Wait on an array of variables on the local PE until all variables
+ *  meet the specified wait condition.
+ *
+ *
+ *  @param  ivars       Symmetric address of an array of remotely accessible
+ *                      data objects. The type of ivars should match that
+ *                      implied in the SYNOPSIS section.
+ *  @param  cmp         A comparison operator that compares elements of ivars with cmp_value.
+ *  @param  cmp_value   The value to be compared with the objects pointed to by ivars.
+ *  @param  nelems      The number of elements in the ivars array.
+ *  @param  status      Local address of an optional mask array of length nelems that indicates
+ *                      which elements in ivars are excluded from the wait set.
+ *  @param  datatype    Type of the objects
+ *
+ *  @return             None
+ */
+typedef void(*mca_spml_base_module_wait_until_all_fn_t)(void *ivars,
+                                                    int cmp,
+                                                    void *cmp_value,
+                                                    size_t nelems,
+                                                    const int *status,
+                                                    int datatype);
+
+/*
+ *  Wait on an array of variables on the local PE until any one variable
+ *  meets the specified wait condition.
+ *
+ *  @param  ivars       Symmetric address of an array of remotely accessible
+ *                      data objects. The type of ivars should match that
+ *                      implied in the SYNOPSIS section.
+ *  @param  cmp         A comparison operator that compares elements of ivars with cmp_value.
+ *  @param  cmp_value   The value to be compared with the objects pointed to by ivars.
+ *  @param  nelems      The number of elements in the ivars array.
+ *  @param  status      Local address of an optional mask array of length nelems that indicates
+ *                      which elements in ivars are excluded from the wait set.
+ *  @param  datatype    Type of the objects
+ *
+ *  @return             Returns the index of an element in the ivars array that satisfies the 
+ *                      wait condition. If the wait set is empty, this routine returns SIZE_MAX.
+ */
+typedef size_t (*mca_spml_base_module_wait_until_any_fn_t)(void *ivars,
+                                                    int cmp,
+                                                    void *cmp_value,
+                                                    size_t nelems,
+                                                    const int *status,
+                                                    int datatype);
+
+
+/*
+ *  Wait on an array of variables on the local PE until at least one variable
+ *  meets the specified wait condition.
+ *
+ *  @param  ivars       Symmetric address of an array of remotely accessible
+ *                      data objects. The type of ivars should match that
+ *                      implied in the SYNOPSIS section.
+ *  @param  cmp         A comparison operator that compares elements of ivars with cmp_value.
+ *  @param  cmp_value   The value to be compared with the objects pointed to by ivars.
+ *  @param  nelems      The number of elements in the ivars array.
+ *  @param  indices     Local address of an array of indices of length at least nelems into
+ *                      ivars that satisfied the wait condition.
+ *  @param  status      Local address of an optional mask array of length nelems that indicates
+ *                      which elements in ivars are excluded from the wait set.
+ *  @param  datatype    Type of the objects
+ *
+ *  @return             Returns the number of indices returned in the indices array. If the wait 
+ *                      set is empty, this routine returns 0.
+ */
+typedef size_t (*mca_spml_base_module_wait_until_some_fn_t)(void *ivars,
+                                                    int cmp,
+                                                    void *cmp_value,
+                                                    size_t nelems,
+                                                    size_t *indices,
+                                                    const int *status,
+                                                    int datatype);
+
+
+/*
+ *  Wait on an array of variables on the local PE until all variables meet the
+ *  specified wait conditions.
+ *
+ *  @param  ivars       Symmetric address of an array of remotely accessible
+ *                      data objects. The type of ivars should match that
+ *                      implied in the SYNOPSIS section.
+ *  @param  cmp         A comparison operator that compares elements of ivars with elements
+ *                      of cmp_values.
+ *  @param  cmp_values  Local address of an array of length nelems containing values to be
+ *                      compared with the respective objects in ivars.
+ *  @param  nelems      The number of elements in the ivars array.
+ *  @param  status      Local address of an optional mask array of length nelems that indicates
+ *                      which elements in ivars are excluded from the wait set.
+ *  @param  datatype    Type of the objects
+ *
+ *  @return             None
+ *                      
+ */
+typedef void (*mca_spml_base_module_wait_until_all_vector_fn_t)(void *ivars,
+                                                    int cmp,
+                                                    void *cmp_values,
+                                                    size_t nelems,
+                                                    const int *status,
+                                                    int datatype);
+
+/*
+ *  Wait on an array of variables on the local PE until any one variable
+ *  meets the specified wait condition.
+ *
+ *  @param  ivars       Symmetric address of an array of remotely accessible
+ *                      data objects. The type of ivars should match that
+ *                      implied in the SYNOPSIS section.
+ *  @param  cmp         A comparison operator that compares elements of ivars with cmp_value.
+ *  @param  cmp_value   The value to be compared with the objects pointed to by ivars.
+ *  @param  nelems      The number of elements in the ivars array.
+ *  @param  status      Local address of an optional mask array of length nelems that indicates
+ *                      which elements in ivars are excluded from the wait set.
+ *  @param  datatype    Type of the objects
+ *
+ *  @return             Returns the index of an element in the ivars array that satisfies the 
+ *                      test condition. If the test set is empty or no conditions in the test 
+ *                      set are satisfied, this routine returns SIZE_MAX.
+ */
+typedef size_t (*mca_spml_base_module_wait_until_any_vector_fn_t)(void *ivars,
+                                                    int cmp,
+                                                    void *cmp_value,
+                                                    size_t nelems,
+                                                    const int *status,
+                                                    int datatype);
+
+/*
+ *  Wait on an array of variables on the local PE until at least one variable meets the
+ *  its specified wait condition.
+ *
+ *  @param  ivars       Symmetric address of an array of remotely accessible
+ *                      data objects. The type of ivars should match that
+ *                      implied in the SYNOPSIS section.
+ *  @param  cmp         A comparison operator that compares elements of ivars with elements
+ *                      of cmp_values.
+ *  @param  cmp_values  Local address of an array of length nelems containing values to be
+ *                      compared with the respective objects in ivars.
+ *  @param  nelems      The number of elements in the ivars array.
+ *  @param  indices     Local address of an array of indices of length at least nelems into ivars
+ *                      that satisfied the wait condition.
+ *  @param  status      Local address of an optional mask array of length nelems that indicates
+ *                      which elements in ivars are excluded from the wait set.
+ *  @param  datatype    Type of the objects
+ *
+ *  @return             Returns the number of indices returned in the indices array. If the test 
+ *                      set is empty, this routine returns 0.
+ */
+typedef size_t (*mca_spml_base_module_wait_until_some_vector_fn_t)(void *ivars,
+                                                    int cmp,
+                                                    void *cmp_value,
+                                                    size_t nelems,
+                                                    size_t *indices,
+                                                    const int *status,
+                                                    int datatype);
+
+
+/*
+ *  Indicate whether all variables within an array of variables on the local PE meet
+ *  a specified test condition.
+ *
+ *
+ *  @param  ivars       Symmetric address of an array of remotely accessible
+ *                      data objects. The type of ivars should match that
+ *                      implied in the SYNOPSIS section.
+ *  @param  cmp         A comparison operator that compares elements of ivars with cmp_value.
+ *  @param  cmp_value   The value to be compared with the objects pointed to by ivars.
+ *  @param  nelems      The number of elements in the ivars array.
+ *  @param  status      Local address of an optional mask array of length nelems that indicates
+ *                      which elements in ivars are excluded from the test set.
+ *  @param  datatype    Type of the objects
+ *
+ *  @return             Returns 1 if all variables in ivars satisfy the test condition or if 
+ *                      nelems is 0, otherwise this routine returns 0.
+ */
+typedef int (*mca_spml_base_module_test_all_fn_t)(void *ivars,
+                                                    int cmp,
+                                                    void *cmp_value,
+                                                    size_t nelems,
+                                                    const int *status,
+                                                    int datatype);
+
+/*
+ *  Indicate whether any one variable within an array of variables on the local PE meets
+ *  a specified test condition.
+ *
+ *  @param  ivars       Symmetric address of an array of remotely accessible
+ *                      data objects. The type of ivars should match that
+ *                      implied in the SYNOPSIS section.
+ *  @param  cmp         A comparison operator that compares elements of ivars with cmp_value.
+ *  @param  cmp_value   The value to be compared with the objects pointed to by ivars.
+ *  @param  nelems      The number of elements in the ivars array.
+ *  @param  status      Local address of an optional mask array of length nelems that indicates
+ *                      which elements in ivars are excluded from the test set.
+ *  @param  datatype    Type of the objects
+ *
+ *  @return             Returns the index of an element in the ivars array that satisfies the 
+ *                      test condition. If the test set is empty or no conditions in the test
+ *                      set are satisfied, this routine returns SIZE_MAX..
+ */
+typedef size_t (*mca_spml_base_module_test_any_fn_t)(void *ivars,
+                                                    int cmp,
+                                                    void *cmp_value,
+                                                    size_t nelems,
+                                                    const int *status,
+                                                    int datatype);
+
+
+/*
+ *  Indicate whether at least one variable within an array of variables on the local PE meets
+ *  a specified test condition.
+ *
+ *  @param  ivars       Symmetric address of an array of remotely accessible
+ *                      data objects. The type of ivars should match that
+ *                      implied in the SYNOPSIS section.
+ *  @param  cmp         A comparison operator that compares elements of ivars with cmp_value.
+ *  @param  cmp_value   The value to be compared with the objects pointed to by ivars.
+ *  @param  nelems      The number of elements in the ivars array.
+ *  @param  indices     Local address of an array of indices of length at least nelems into
+ *                      ivars that satisfied the wait condition.
+ *  @param  status      Local address of an optional mask array of length nelems that indicates
+ *                      which elements in ivars are excluded from the test set.
+ *  @param  datatype    Type of the objects
+ *
+ *  @return             Returns the number of indices returned in the indices array. If the test 
+ *                      set is empty, this routine returns 0.
+ */
+typedef size_t (*mca_spml_base_module_test_some_fn_t)(void *ivars,
+                                                    int cmp,
+                                                    void *cmp_value,
+                                                    size_t nelems,
+                                                    size_t *indices,
+                                                    const int *status,
+                                                    int datatype);
+
+
+/*
+ *  Indicate whether all variables within an array of variables on the local PE meet the
+ *  specified test conditions.
+ *
+ *  @param  ivars       Symmetric address of an array of remotely accessible
+ *                      data objects. The type of ivars should match that
+ *                      implied in the SYNOPSIS section.
+ *  @param  cmp         A comparison operator that compares elements of ivars with elements
+ *                      of cmp_values.
+ *  @param  cmp_values  Local address of an array of length nelems containing values to be
+ *                      compared with the respective objects in ivars.
+ *  @param  nelems      The number of elements in the ivars array.
+ *  @param  status      Local address of an optional mask array of length nelems that indicates
+ *                      which elements in ivars are excluded from the test set.
+ *  @param  datatype    Type of the objects
+ *
+ *  @return             Returns 1 if all variables in ivars satisfy the test conditions or if 
+ *                      nelems is 0, otherwise this routine returns 0.
+ */
+typedef int (*mca_spml_base_module_test_all_vector_fn_t)(void *ivars,
+                                                    int cmp,
+                                                    void *cmp_values,
+                                                    size_t nelems,
+                                                    const int *status,
+                                                    int datatype);
+
+/*
+ *  Indicate whether any one variable within an array of variables on the local PE meets
+ *  its specified test condition.
+ *
+ *  @param  ivars       Symmetric address of an array of remotely accessible
+ *                      data objects. The type of ivars should match that
+ *                      implied in the SYNOPSIS section.
+ *  @param  cmp         A comparison operator that compares elements of ivars with cmp_value.
+ *  @param  cmp_values  Local address of an array of length nelems containing values to be
+ *                      compared with the respective objects in ivars.
+ *  @param  nelems      The number of elements in the ivars array.
+ *  @param  status      Local address of an optional mask array of length nelems that indicates
+ *                      which elements in ivars are excluded from the test set.
+ *  @param  datatype    Type of the objects
+ *
+ *  @return             OSHMEM_SUCCESS or failure status.
+ */
+typedef size_t (*mca_spml_base_module_test_any_vector_fn_t)(void *ivars, int cmp,
+                                                            void *cmp_values,
+                                                            size_t nelems,
+                                                            const int *status,
+                                                            int datatype);
+
+/*
+ *  Indicate whether at least one variable within an array of variables on the local PE meets
+ *  its specified test condition.
+ *
+ *  @param  ivars       Symmetric address of an array of remotely accessible
+ *                      data objects. The type of ivars should match that
+ *                      implied in the SYNOPSIS section.
+ *  @param  cmp         A comparison operator that compares elements of ivars with elements
+ *                      of cmp_values.
+ *  @param  cmp_values  Local address of an array of length nelems containing values to be
+ *                      compared with the respective objects in ivars.
+ *  @param  nelems      The number of elements in the ivars array.
+ *  @param  indices     Local address of an array of indices of length at least nelems into ivars
+ *                      that satisfied the wait condition.
+ *  @param  status      Local address of an optional mask array of length nelems that indicates
+ *                      which elements in ivars are excluded from the test set.
+ *  @param  datatype    Type of the objects
+ *
+ *  @return             OSHMEM_SUCCESS or failure status.
+ */
+typedef size_t (*mca_spml_base_module_test_some_vector_fn_t)(void *ivars,
+                                                             int cmp,
+                                                             void *cmp_values,
+                                                             size_t nelems,
+                                                             size_t *indices,
+                                                             const int *status,
+                                                             int datatype);
+
+/*
+ *  Registers the arrival of a PE at a synchronization point.
+ *  This routine does not return until all other PEs in a given
+ *  OpenSHMEM team or active set arrive at this synchronization point.
+ *
+ *
+ *  @param  team       An OpenSHMEM team handle.
+ *
+ *  @return            OSHMEM_SUCCESS or failure status.
+ *                     Zero on successful local completion. Nonzero otherwise.
+ */
+typedef int (*mca_spml_base_module_team_sync_fn_t)(shmem_team_t team);
+
+
+/*
+ *  Returns the number of the calling PE within a specified team.
+ *
+ *  @param  team       An OpenSHMEM team handle.
+ *
+ *  @return            The number of the calling PE within the specified
+ *                     team, or the value -1 if the team handle compares
+ *                     equal to SHMEM_TEAM_INVALID
+ */
+typedef int (*mca_spml_base_module_team_my_pe_fn_t)(shmem_team_t team);
+
+
+/*
+ *  Returns the number of PEs in a specified team.
+ *
+ *  @param  team       An OpenSHMEM team handle.
+ *
+ *  @return            The number of PEs in the specified team, or the
+ *                     value -1 if the team handle compares equal to
+ *                     SHMEM_TEAM_INVALID.
+ */
+typedef int (*mca_spml_base_module_team_n_pes_fn_t)(shmem_team_t team);
+
+
+
+/*
+ *  Return the configuration parameters of a given team
+ *
+ *  @param  team            An OpenSHMEM team handle.
+ *
+ *  @param  config_mask     The bitwise mask representing the set of
+ *                          configuration parameters to fetch from the
+ *                          given team.
+ *
+ *  @param  config          A pointer to the configuration parameters for the
+ *                          given team.
+ *
+ *
+ *  @return                 OSHMEM_SUCCESS or failure status.
+ *
+ */
+typedef int (*mca_spml_base_module_team_get_config_fn_t)(shmem_team_t team, long
+        config_mask, shmem_team_config_t *config);
+
+/*
+ *  Translate a given PE number from one team to the corresponding PE number in
+ *  another team.
+ *
+ *  @param  src_team    An OpenSHMEM team handle.
+ *  @param  src_pe      A PE number in src_team.
+ *  @param  dest_team   An OpenSHMEM team handle.
+ *
+ *
+ *  @return             The specified PE’s number in the dest_team, or a value
+ *                      of -1 if any team handle arguments are invalid or the
+ *                      src_pe is not in both the source and destination teams.
+ */
+typedef int (*mca_spml_base_module_team_translate_pe_fn_t)(shmem_team_t src_team,
+        int src_pe, shmem_team_t dest_team);
+
+
+
+/*
+ *  Create a new OpenSHMEM team from a subset of the existing parent team PEs,
+ *  where the subset is defined by the PE triplet (start, stride, and size)
+ *  supplied to the routine.
+ *
+ *  @param  parent_team    An OpenSHMEM team handle.
+ *  @param  start          The lowest PE number of the subset of PEs from the parent team
+ *                         that will form the new team.
+ *  @param  stride         The stride between team PE numbers in the parent team that comprise the subset
+ *                         of PEs that will form the new team.
+ *  @param  size           The number of PEs from the parent team in the subset of PEs that
+ *                         will form the new team. size must be a positive integer.
+ *  @param  config         A pointer to the configuration parameters for the new team.
+ *  @param  config_mask    The bitwise mask representing the set of configuration parameters
+ *                         to use from config.
+ *  @param  new_team       An OpenSHMEM team handle. Upon successful creation, it references an OpenSHMEM
+ *                         team that contains the subset of all PEs in the parent team
+ *                         specified by the PE triplet provided.m
+ *
+ *
+ *  @return                OSHMEM_SUCCESS or failure status.
+ *
+ */
+typedef int (*mca_spml_base_module_team_split_strided_fn_t)(shmem_team_t
+        parent_team, int start, int stride, int size, const shmem_team_config_t
+        *config, long config_mask, shmem_team_t *new_team);
+
+
+/*
+ *  Create two new teams by splitting an existing parent team into two subsets
+ *  based on a 2D Cartesian space defined by the xrange argument and a y
+ *  dimension that is derived from xrange and the parent team size.
+ *
+ *  @param  parent_team    An OpenSHMEM team handle.
+ *  @param  xrange         A positive integer representing the number of elements in the first dimension.
+ *  @param  xaxis_config   A pointer to the configuration parameters for the new x-axis team.
+ *  @param  xaxis_mask     The bitwise mask representing the set of configuration parameters to
+ *                         use from xaxis_config.
+ *  @param  xaxis_team     A new PE team handle representing a PE subset consisting of all the
+ *                         PEs that have the same coordinate along the y-axis as the calling PE..
+ *  @param  yaxis_config   A pointer to the configuration parameters for the new y-axis team.
+ *  @param  yaxis_mask     The bitwise mask representing the set of configuration parameters to use
+ *                         from yaxis_config.
+ *  @param  yaxis_team     A new PE team handle representing a PE subset consisting of all the PEs
+ *                         that have the same coordinate along the x-axis as the calling PE.
+ *
+ *  @return                OSHMEM_SUCCESS or failure status.
+ *
+ */
+typedef int (*mca_spml_base_module_team_split_2d_fn_t)(shmem_team_t parent_team,
+        int xrange, const shmem_team_config_t *xaxis_config, long xaxis_mask,
+        shmem_team_t *xaxis_team, const shmem_team_config_t *yaxis_config, long
+        yaxis_mask, shmem_team_t *yaxis_team);
+
+
+/*
+ *  Destroy an existing team.
+ *
+ *  @param  team           An OpenSHMEM team handle.
+ *
+ *  @return                OSHMEM_SUCCESS or failure status.
+ *
+ */
+typedef int (*mca_spml_base_module_team_destroy_fn_t)(shmem_team_t team);
+
+/*
+ *  Retrieve the team associated with the communication context.
+ *
+ *  @param  ctx            A handle to a communication context.
+ *  @param  team           A pointer to a handle to the associated PE team.
+ *
+ *  @return                OSHMEM_SUCCESS or failure status.
+ *
+ */
+typedef int (*mca_spml_base_module_team_get_fn_t)(shmem_ctx_t ctx, shmem_team_t *team);
+
+/*
+ *  Create a communication context from a team.
+ *
+ *  @param  team           An OpenSHMEM team handle.
+ *  @param  options        The set of options requested for the given context.
+ *  @param  ctx            A handle to the newly created context.
+ *
+ *  @return                OSHMEM_SUCCESS or failure status.
+ *
+ */
+typedef int (*mca_spml_base_module_team_create_ctx_fn_t)(shmem_team_t team, long options, shmem_ctx_t *ctx);
+
+/*
+ *  Exchanges a fixed amount of contiguous data blocks between all pairs of
+ *  PEs participating in the collective routine..
+ *
+ *  @param  team       An OpenSHMEM team handle.
+ *  @param  dest       Symmetric address of a data object large enough to
+ *                     receive the combined total of nelems elements from each PE in the active set.
+ *  @param  source     Symmetric address of a data object that contains nelems elements of data
+ *                     for each PE in the active set, ordered according to destination PE.
+ *  @param  nelems     The number of elements to exchange for each PE.
+ *  @param  datatype   Datatype of the elements
+ *
+ *  @return            OSHMEM_SUCCESS or failure status.
+ *
+ */
+typedef int (*mca_spml_base_module_team_alltoall_fn_t)(shmem_team_t team, void
+        *dest, const void *source, size_t nelems, int datatype);
+
+/*
+ *  Exchanges a fixed amount of strided data blocks between all pairs of PEs
+ *  participating in the collective routine.
+ *
+ *  @param  team       An OpenSHMEM team handle.
+ *  @param  dest       Symmetric address of a data object large enough to
+ *                     receive the combined total of nelems elements from each PE in the active set.
+ *  @param  source     Symmetric address of a data object that contains nelems elements of data
+ *                     for each PE in the active set, ordered according to destination PE.
+ *  @param  dst        The stride between consecutive elements of the dest data object. The stride
+ *                     is scaled by the element size. A value of 1 indicates contiguous data.
+ *  @param  sst        The stride between consecutive elements of the source data object. The stride
+ *                     is scaled by the element size. A value of 1 indicates contiguous data
+ *  @param  nelems     The number of elements to exchange for each PE.
+ *  @param  datatype   Datatype of the elements
+ *
+ *  @return            OSHMEM_SUCCESS or failure status.
+ *
+ */
+typedef int (*mca_spml_base_module_team_alltoalls_fn_t)(shmem_team_t team, void
+        *dest, const void *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems,
+        int datatype);
+
+
+/*
+ *  Broadcasts a block of data from one PE to one or more destination PEs.
+ *
+ *  @param  team       An OpenSHMEM team handle.
+ *  @param  dest       Symmetric address of destination data object.
+ *  @param  source     Symmetric address of the source data object.
+ *  @param  nelems     The number of elements in source and dest arrays
+ *  @param  PE_root    Zero-based ordinal of the PE, with respect to the team or
+ *                     active set, from which the data is copied..
+ *  @param  datatype   Datatype of the elements
+ *
+ *  @return            OSHMEM_SUCCESS or failure status.
+ *
+ */
+typedef int (*mca_spml_base_module_team_broadcast_fn_t)(shmem_team_t team, void
+        *dest, const void *source, size_t nelems, int PE_root, int datatype);
+
+
+
+/*
+ *  Concatenates blocks of data from multiple PEs to an array in every PE participating in
+ *  the collective routine.
+ *
+ *  @param  team       An OpenSHMEM team handle.
+ *  @param  dest       Symmetric address of an array large enough to accept the
+ *                     concatenation of the source arrays on all participating PEs.
+ *  @param  source     Symmetric address of the source data object.
+ *  @param  nelems     The number of elements in source array.
+ *  @param  datatype   Datatype of the elements
+ *
+ *  @return            OSHMEM_SUCCESS or failure status.
+ *
+ */
+typedef int (*mca_spml_base_module_team_collect_fn_t)(shmem_team_t team, void
+        *dest, const void *source, size_t nelems, int datatype);
+typedef int (*mca_spml_base_module_team_fcollect_fn_t)(shmem_team_t team, void
+        *dest, const void *source, size_t nelems, int datatype);
+
+/*
+ *  Performs a math reduction across a set of PEs.
+ *
+ *  @param  team       An OpenSHMEM team handle.
+ *  @param  dest       Symmetric address of an array, of length nreduce elements,
+ *                     to receive the result of the reduction routines.
+ *  @param  source     Symmetric address of an array, of length nreduce elements, that
+ *                     contains one element for each separate reduction routine.
+ *  @param  nreduce    The number of elements in the dest and source arrays.
+ *  @param  operation  Operations from list of supported oshmem ops
+ *  @param  datatype   Datatype of the elements
+ *
+ *  @return            OSHMEM_SUCCESS or failure status.
+ *
+ */
+typedef int (*mca_spml_base_module_team_reduce_fn_t)(shmem_team_t team, void
+        *dest, const void *source, size_t nreduce, int operation, int datatype);
+
+
 /**
  * Blocking data transfer from remote PE.
  * Read data from remote PE.
@@ -393,15 +1034,50 @@ struct mca_spml_base_module_1_0_0_t {
 
     mca_spml_base_module_put_fn_t spml_put;
     mca_spml_base_module_put_nb_fn_t spml_put_nb;
+    mca_spml_base_module_put_signal_fn_t spml_put_signal;
+    mca_spml_base_module_put_signal_nb_fn_t spml_put_signal_nb;
+
     mca_spml_base_module_get_fn_t spml_get;
     mca_spml_base_module_get_nb_fn_t spml_get_nb;
 
     mca_spml_base_module_recv_fn_t spml_recv;
     mca_spml_base_module_send_fn_t spml_send;
 
-    mca_spml_base_module_wait_fn_t spml_wait;
-    mca_spml_base_module_wait_nb_fn_t spml_wait_nb;
-    mca_spml_base_module_test_fn_t spml_test;
+    mca_spml_base_module_wait_fn_t                   spml_wait;
+    mca_spml_base_module_wait_nb_fn_t                spml_wait_nb;
+    mca_spml_base_module_wait_until_all_fn_t         spml_wait_until_all;
+    mca_spml_base_module_wait_until_any_fn_t         spml_wait_until_any;
+    mca_spml_base_module_wait_until_some_fn_t        spml_wait_until_some;
+    mca_spml_base_module_wait_until_all_vector_fn_t  spml_wait_until_all_vector;
+    mca_spml_base_module_wait_until_any_vector_fn_t  spml_wait_until_any_vector;
+    mca_spml_base_module_wait_until_some_vector_fn_t spml_wait_until_some_vector;
+
+    mca_spml_base_module_test_fn_t                   spml_test;
+    mca_spml_base_module_test_all_fn_t               spml_test_all;
+    mca_spml_base_module_test_any_fn_t               spml_test_any;
+    mca_spml_base_module_test_some_fn_t              spml_test_some;
+    mca_spml_base_module_test_all_vector_fn_t        spml_test_all_vector;
+    mca_spml_base_module_test_any_vector_fn_t        spml_test_any_vector;
+    mca_spml_base_module_test_some_vector_fn_t       spml_test_some_vector;
+
+    mca_spml_base_module_team_sync_fn_t              spml_team_sync;
+    mca_spml_base_module_team_my_pe_fn_t             spml_team_my_pe;
+    mca_spml_base_module_team_n_pes_fn_t             spml_team_n_pes;
+    mca_spml_base_module_team_get_config_fn_t        spml_team_get_config;
+    mca_spml_base_module_team_translate_pe_fn_t      spml_team_translate_pe;
+    mca_spml_base_module_team_split_strided_fn_t     spml_team_split_strided;
+    mca_spml_base_module_team_split_2d_fn_t          spml_team_split_2d;
+    mca_spml_base_module_team_destroy_fn_t           spml_team_destroy;
+    mca_spml_base_module_team_get_fn_t               spml_team_get;
+    mca_spml_base_module_team_create_ctx_fn_t        spml_team_create_ctx;
+
+    mca_spml_base_module_team_alltoall_fn_t          spml_team_alltoall;
+    mca_spml_base_module_team_alltoalls_fn_t         spml_team_alltoalls;
+    mca_spml_base_module_team_broadcast_fn_t         spml_team_broadcast;
+    mca_spml_base_module_team_collect_fn_t           spml_team_collect;
+    mca_spml_base_module_team_fcollect_fn_t          spml_team_fcollect;
+    mca_spml_base_module_team_reduce_fn_t            spml_team_reduce;
+
     mca_spml_base_module_fence_fn_t spml_fence;
     mca_spml_base_module_quiet_fn_t spml_quiet;
 
diff --git a/oshmem/mca/spml/ucx/Makefile.am b/oshmem/mca/spml/ucx/Makefile.am
index 94a48a58e7a..7d9e9137e5b 100644
--- a/oshmem/mca/spml/ucx/Makefile.am
+++ b/oshmem/mca/spml/ucx/Makefile.am
@@ -35,7 +35,7 @@ mcacomponent_LTLIBRARIES = $(component_install)
 mca_spml_ucx_la_SOURCES = $(ucx_sources)
 mca_spml_ucx_la_LIBADD = $(top_builddir)/oshmem/liboshmem.la \
 	$(spml_ucx_LIBS) \
-	$(OPAL_TOP_BUILDDIR)/opal/mca/common/ucx/lib@OPAL_LIB_PREFIX@mca_common_ucx.la
+	$(OPAL_TOP_BUILDDIR)/opal/mca/common/ucx/lib@OPAL_LIB_NAME@mca_common_ucx.la
 mca_spml_ucx_la_LDFLAGS = -module -avoid-version $(spml_ucx_LDFLAGS)
 
 noinst_LTLIBRARIES = $(component_noinst)
diff --git a/oshmem/mca/spml/ucx/configure.m4 b/oshmem/mca/spml/ucx/configure.m4
index 135d2c98421..efd03fcbcb7 100644
--- a/oshmem/mca/spml/ucx/configure.m4
+++ b/oshmem/mca/spml/ucx/configure.m4
@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2015      Mellanox Technologies, Inc.
  *                         All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -24,7 +25,6 @@ AC_DEFUN([MCA_oshmem_spml_ucx_CONFIG],[
 
 
     # substitute in the things needed to build ucx 
-    AC_SUBST([spml_ucx_CFLAGS])
     AC_SUBST([spml_ucx_CPPFLAGS])
     AC_SUBST([spml_ucx_LDFLAGS])
     AC_SUBST([spml_ucx_LIBS])
diff --git a/oshmem/mca/spml/ucx/spml_ucx.c b/oshmem/mca/spml/ucx/spml_ucx.c
index fba5b2cb806..40b20ff7bff 100644
--- a/oshmem/mca/spml/ucx/spml_ucx.c
+++ b/oshmem/mca/spml/ucx/spml_ucx.c
@@ -10,7 +10,7 @@
  *
  * $HEADER$
  */
-
+ 
 #define _GNU_SOURCE
 #include <stdio.h>
 
@@ -22,6 +22,7 @@
 #include "opal/datatype/opal_convertor.h"
 #include "opal/mca/common/ucx/common_ucx.h"
 #include "opal/util/opal_environ.h"
+#include "opal/util/minmax.h"
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/mca/pml/pml.h"
 
@@ -57,13 +58,12 @@ mca_spml_ucx_t mca_spml_ucx = {
         .spml_ctx_destroy   = mca_spml_ucx_ctx_destroy,
         .spml_put           = mca_spml_ucx_put,
         .spml_put_nb        = mca_spml_ucx_put_nb,
+        .spml_put_signal    = mca_spml_ucx_put_signal,
+        .spml_put_signal_nb = mca_spml_ucx_put_signal_nb,
         .spml_get           = mca_spml_ucx_get,
         .spml_get_nb        = mca_spml_ucx_get_nb,
         .spml_recv          = mca_spml_ucx_recv,
         .spml_send          = mca_spml_ucx_send,
-        .spml_wait          = mca_spml_base_wait,
-        .spml_wait_nb       = mca_spml_base_wait_nb,
-        .spml_test          = mca_spml_base_test,
         .spml_fence         = mca_spml_ucx_fence,
         .spml_quiet         = mca_spml_ucx_quiet,
         .spml_rmkey_unpack  = mca_spml_ucx_rmkey_unpack,
@@ -71,7 +71,38 @@ mca_spml_ucx_t mca_spml_ucx = {
         .spml_rmkey_ptr     = mca_spml_ucx_rmkey_ptr,
         .spml_memuse_hook   = mca_spml_ucx_memuse_hook,
         .spml_put_all_nb    = mca_spml_ucx_put_all_nb,
-        .self               = (void*)&mca_spml_ucx
+        .spml_wait                      = mca_spml_base_wait,
+        .spml_wait_nb                   = mca_spml_base_wait_nb,
+        .spml_wait_until_all            = mca_spml_ucx_wait_until_all,
+        .spml_wait_until_any            = mca_spml_ucx_wait_until_any,
+        .spml_wait_until_some           = mca_spml_ucx_wait_until_some,
+        .spml_wait_until_all_vector     = mca_spml_ucx_wait_until_all_vector,
+        .spml_wait_until_any_vector     = mca_spml_ucx_wait_until_any_vector,
+        .spml_wait_until_some_vector    = mca_spml_ucx_wait_until_some_vector,
+        .spml_test                      = mca_spml_base_test,
+        .spml_test_all          	= mca_spml_ucx_test_all,
+        .spml_test_any          	= mca_spml_ucx_test_any,
+        .spml_test_some         	= mca_spml_ucx_test_some,
+        .spml_test_all_vector   	= mca_spml_ucx_test_all_vector,
+        .spml_test_any_vector   	= mca_spml_ucx_test_any_vector,
+        .spml_test_some_vector  	= mca_spml_ucx_test_some_vector,
+        .spml_team_sync                 = mca_spml_ucx_team_sync,
+        .spml_team_my_pe                = mca_spml_ucx_team_my_pe,
+        .spml_team_n_pes                = mca_spml_ucx_team_n_pes,
+        .spml_team_get_config           = mca_spml_ucx_team_get_config,
+        .spml_team_translate_pe         = mca_spml_ucx_team_translate_pe,
+        .spml_team_split_strided        = mca_spml_ucx_team_split_strided,
+        .spml_team_split_2d             = mca_spml_ucx_team_split_2d,
+        .spml_team_destroy              = mca_spml_ucx_team_destroy,
+        .spml_team_get                  = mca_spml_ucx_team_get,
+        .spml_team_create_ctx           = mca_spml_ucx_team_create_ctx,
+        .spml_team_alltoall             = mca_spml_ucx_team_alltoall,
+        .spml_team_alltoalls            = mca_spml_ucx_team_alltoalls,
+        .spml_team_broadcast            = mca_spml_ucx_team_broadcast,
+        .spml_team_collect              = mca_spml_ucx_team_collect,
+        .spml_team_fcollect             = mca_spml_ucx_team_fcollect,
+        .spml_team_reduce               = mca_spml_ucx_team_reduce,
+        .self                           = (void*)&mca_spml_ucx
     },
 
     .ucp_context            = NULL,
@@ -85,12 +116,181 @@ mca_spml_ucx_ctx_t mca_spml_ucx_ctx_default = {
     .ucp_worker         = NULL,
     .ucp_peers          = NULL,
     .options            = 0,
-    .synchronized_quiet = false
+    .synchronized_quiet = false,
+    .strong_sync        = SPML_UCX_STRONG_ORDERING_NONE
 };
 
-#if HAVE_DECL_UCP_ATOMIC_OP_NBX
+#ifdef HAVE_UCP_REQUEST_PARAM_T
 static ucp_request_param_t mca_spml_ucx_request_param = {0};
+static ucp_request_param_t mca_spml_ucx_request_param_b = {
+    .op_attr_mask = UCP_OP_ATTR_FLAG_FAST_CMPL
+};
+#endif
+
+unsigned
+mca_spml_ucx_mem_map_flags_symmetric_rkey(struct mca_spml_ucx *spml_ucx)
+{
+#if HAVE_DECL_UCP_MEM_MAP_SYMMETRIC_RKEY
+    if (spml_ucx->symmetric_rkey_max_count > 0) {
+        return UCP_MEM_MAP_SYMMETRIC_RKEY;
+    }
+#endif
+
+    return 0;
+}
+
+void mca_spml_ucx_rkey_store_init(mca_spml_ucx_rkey_store_t *store)
+{
+    store->array = NULL;
+    store->count = 0;
+    store->size  = 0;
+}
+
+void mca_spml_ucx_rkey_store_cleanup(mca_spml_ucx_rkey_store_t *store)
+{
+    int i;
+
+    for (i = 0; i < store->count; i++) {
+        if (store->array[i].refcnt != 0) {
+            SPML_UCX_ERROR("rkey store destroy: %d/%d has refcnt %d > 0",
+                           i, store->count, store->array[i].refcnt);
+        }
+
+        ucp_rkey_destroy(store->array[i].rkey);
+    }
+
+    free(store->array);
+}
+
+/**
+ * Find position in sorted array for existing or future entry
+ *
+ * @param[in]  store  Store of the entries
+ * @param[in]  worker Common worker for rkeys used
+ * @param[in]  rkey   Remote key to search for
+ * @param[out] index  Index of entry
+ *
+ * @return
+ *   OSHMEM_ERR_NOT_FOUND: index contains the position where future element
+ *                         should be inserted to keep array sorted
+ *   OSHMEM_SUCCESS      : index contains the position of the element
+ *   Other error         : index is not valid
+ */
+static int mca_spml_ucx_rkey_store_find(const mca_spml_ucx_rkey_store_t *store,
+                                        const ucp_worker_h worker,
+                                        const ucp_rkey_h rkey,
+                                        int *index)
+{
+#if HAVE_DECL_UCP_RKEY_COMPARE
+    ucp_rkey_compare_params_t params;
+    int i, result, m, end;
+    ucs_status_t status;
+
+    for (i = 0, end = store->count; i < end;) {
+        m = (i + end) / 2;
+
+        params.field_mask = 0;
+        status = ucp_rkey_compare(worker, store->array[m].rkey,
+                                  rkey, &params, &result);
+        if (status != UCS_OK) {
+            return OSHMEM_ERROR;
+        } else if (result == 0) {
+            *index = m;
+            return OSHMEM_SUCCESS;
+        } else if (result > 0) {
+            end = m;
+        } else {
+            i = m + 1;
+        }
+    }
+
+    *index = i;
+    return OSHMEM_ERR_NOT_FOUND;
+#else
+    return OSHMEM_ERROR;
 #endif
+}
+
+static void mca_spml_ucx_rkey_store_insert(mca_spml_ucx_rkey_store_t *store,
+                                           int i, ucp_rkey_h rkey)
+{
+    int size;
+    mca_spml_ucx_rkey_t *tmp;
+
+    if (store->count >= mca_spml_ucx.symmetric_rkey_max_count) {
+        return;
+    }
+
+    if (store->count >= store->size) {
+        size = opal_min(opal_max(store->size, 8) * 2,
+                        mca_spml_ucx.symmetric_rkey_max_count);
+        tmp  = realloc(store->array, size * sizeof(*store->array));
+        if (tmp == NULL) {
+            return;
+        }
+
+        store->array = tmp;
+        store->size  = size;
+    }
+
+    memmove(&store->array[i + 1], &store->array[i],
+            (store->count - i) * sizeof(*store->array));
+    store->array[i].rkey   = rkey;
+    store->array[i].refcnt = 1;
+    store->count++;
+    return;
+}
+
+/* Takes ownership of input ucp remote key */
+static ucp_rkey_h mca_spml_ucx_rkey_store_get(mca_spml_ucx_rkey_store_t *store,
+                                              ucp_worker_h worker,
+                                              ucp_rkey_h rkey)
+{
+    int ret, i;
+
+    if (mca_spml_ucx.symmetric_rkey_max_count == 0) {
+        return rkey;
+    }
+
+    ret = mca_spml_ucx_rkey_store_find(store, worker, rkey, &i);
+    if (ret == OSHMEM_SUCCESS) {
+        ucp_rkey_destroy(rkey);
+        store->array[i].refcnt++;
+        return store->array[i].rkey;
+    }
+
+    if (ret == OSHMEM_ERR_NOT_FOUND) {
+        mca_spml_ucx_rkey_store_insert(store, i, rkey);
+    }
+
+    return rkey;
+}
+
+static void mca_spml_ucx_rkey_store_put(mca_spml_ucx_rkey_store_t *store,
+                                        ucp_worker_h worker,
+                                        ucp_rkey_h rkey)
+{
+    mca_spml_ucx_rkey_t *entry;
+    int ret, i;
+
+    ret = mca_spml_ucx_rkey_store_find(store, worker, rkey, &i);
+    if (ret != OSHMEM_SUCCESS) {
+        goto out;
+    }
+
+    entry = &store->array[i];
+    assert(entry->rkey == rkey);
+    if (--entry->refcnt > 0) {
+        return;
+    }
+
+    memmove(&store->array[i], &store->array[i + 1],
+            (store->count - (i + 1)) * sizeof(*store->array));
+    store->count--;
+
+out:
+    ucp_rkey_destroy(rkey);
+}
 
 int mca_spml_ucx_enable(bool enable)
 {
@@ -104,7 +304,154 @@ int mca_spml_ucx_enable(bool enable)
     return OSHMEM_SUCCESS;
 }
 
-int mca_spml_ucx_del_procs(ompi_proc_t** procs, size_t nprocs)
+/* initialize the mkey cache */
+void mca_spml_ucx_peer_mkey_cache_init(mca_spml_ucx_ctx_t *ucx_ctx, int pe)
+{
+    ucx_ctx->ucp_peers[pe].mkeys = NULL;
+    ucx_ctx->ucp_peers[pe].mkeys_cnt = 0;
+}
+
+/* add a new mkey and update the mkeys_cnt */
+int mca_spml_ucx_peer_mkey_cache_add(ucp_peer_t *ucp_peer, int index)
+{
+    /* Allocate an array to hold the pointers to the ucx_cached_mkey */
+    if (index >= (int)ucp_peer->mkeys_cnt){
+        int old_size = ucp_peer->mkeys_cnt;
+        ucp_peer->mkeys_cnt = index + 1;
+        ucp_peer->mkeys = realloc(ucp_peer->mkeys, sizeof(ucp_peer->mkeys[0]) * ucp_peer->mkeys_cnt);
+        if (NULL == ucp_peer->mkeys) {
+            SPML_UCX_ERROR("Failed to obtain new mkey: OOM - failed to expand the descriptor buffer");
+            return OSHMEM_ERR_OUT_OF_RESOURCE;
+        }
+        /* NOTE: release code checks for the rkey != NULL as a sign of used element:
+        Account for the following scenario below by zero'ing the unused elements:
+        |MKEY1|00000|MKEY2|??????|NEW-MKEY|
+        |<--- old_size -->|
+        */
+        memset(ucp_peer->mkeys + old_size, 0, (ucp_peer->mkeys_cnt - old_size) * sizeof(ucp_peer->mkeys[0]));
+    } else {
+        /* Make sure we don't leak memory */
+        assert(NULL == ucp_peer->mkeys[index]);
+    }
+
+    ucp_peer->mkeys[index] = (spml_ucx_cached_mkey_t *) malloc(sizeof(*ucp_peer->mkeys[0]));
+    if (NULL == ucp_peer->mkeys[index]) {
+        SPML_UCX_ERROR("Failed to obtain new ucx_cached_mkey: OOM - failed to expand the descriptor buffer");
+        return OSHMEM_ERR_OUT_OF_RESOURCE;
+    }
+    return OSHMEM_SUCCESS;
+}
+
+/* Release individual mkeys */
+int mca_spml_ucx_peer_mkey_cache_del(ucp_peer_t *ucp_peer, int segno)
+{
+    if (((int)ucp_peer->mkeys_cnt <= segno) || (segno < 0)) {
+        return OSHMEM_ERR_NOT_AVAILABLE;
+    }
+    if (NULL != ucp_peer->mkeys[segno]) {
+        free(ucp_peer->mkeys[segno]);
+        ucp_peer->mkeys[segno] = NULL;
+    }
+    return OSHMEM_SUCCESS;
+}
+
+/* Release the memkey map from a ucp_peer if it has any element in memkey */
+void mca_spml_ucx_peer_mkey_cache_release(ucp_peer_t *ucp_peer)
+{
+    size_t i;
+    if (ucp_peer->mkeys_cnt) {
+        for(i = 0; i < ucp_peer->mkeys_cnt; i++) {
+            assert(NULL == ucp_peer->mkeys[i]);
+        }
+        free(ucp_peer->mkeys);
+        ucp_peer->mkeys = NULL;
+    }
+}
+
+int mca_spml_ucx_ctx_mkey_new(mca_spml_ucx_ctx_t *ucx_ctx, int pe, uint32_t segno, spml_ucx_mkey_t **mkey)
+{
+    ucp_peer_t *ucp_peer;
+    spml_ucx_cached_mkey_t *ucx_cached_mkey;
+    int rc;
+    ucp_peer = &(ucx_ctx->ucp_peers[pe]);
+    rc = mca_spml_ucx_peer_mkey_cache_add(ucp_peer, segno);
+    if (OSHMEM_SUCCESS != rc) {
+        return rc;
+    }
+    rc = mca_spml_ucx_peer_mkey_get(ucp_peer, segno, &ucx_cached_mkey);
+    if (OSHMEM_SUCCESS != rc) {
+        return rc;
+    }
+    *mkey = &(ucx_cached_mkey->key);
+    return OSHMEM_SUCCESS;
+}
+
+int mca_spml_ucx_ctx_mkey_cache(mca_spml_ucx_ctx_t *ucx_ctx, sshmem_mkey_t *mkey, uint32_t segno, int dst_pe)
+{
+    ucp_peer_t *peer;
+    spml_ucx_cached_mkey_t *ucx_cached_mkey;
+    int rc;
+
+    peer = &(ucx_ctx->ucp_peers[dst_pe]);
+    rc = mca_spml_ucx_peer_mkey_get(peer, segno, &ucx_cached_mkey);
+    if (OSHMEM_SUCCESS != rc) {
+        SPML_UCX_ERROR("mca_spml_ucx_peer_mkey_get failed");
+        return rc;
+    }
+    mkey_segment_init(&ucx_cached_mkey->super, mkey, segno);
+    return OSHMEM_SUCCESS;
+}
+
+int mca_spml_ucx_ctx_mkey_add(mca_spml_ucx_ctx_t *ucx_ctx, int pe, uint32_t segno, sshmem_mkey_t *mkey, spml_ucx_mkey_t **ucx_mkey)
+{
+    int rc;
+    ucs_status_t err;
+    ucp_rkey_h rkey;
+
+    rc = mca_spml_ucx_ctx_mkey_new(ucx_ctx, pe, segno, ucx_mkey);
+    if (OSHMEM_SUCCESS != rc) {
+        SPML_UCX_ERROR("mca_spml_ucx_ctx_mkey_new failed");
+        return rc;
+    }
+
+    if (mkey->u.data) {
+        err = ucp_ep_rkey_unpack(ucx_ctx->ucp_peers[pe].ucp_conn, mkey->u.data, &rkey);
+        if (UCS_OK != err) {
+            SPML_UCX_ERROR("failed to unpack rkey: %s", ucs_status_string(err));
+            return OSHMEM_ERROR;
+        }
+
+        if (!oshmem_proc_on_local_node(pe)) {
+            rkey = mca_spml_ucx_rkey_store_get(&ucx_ctx->rkey_store, ucx_ctx->ucp_worker[0], rkey);
+        }
+
+        (*ucx_mkey)->rkey = rkey;
+
+        rc = mca_spml_ucx_ctx_mkey_cache(ucx_ctx, mkey, segno, pe);
+        if (OSHMEM_SUCCESS != rc) {
+            SPML_UCX_ERROR("mca_spml_ucx_ctx_mkey_cache failed");
+            return rc;
+        }
+    }
+    return OSHMEM_SUCCESS;
+}
+
+int mca_spml_ucx_ctx_mkey_del(mca_spml_ucx_ctx_t *ucx_ctx, int pe, uint32_t segno, spml_ucx_mkey_t *ucx_mkey)
+{
+    ucp_peer_t *ucp_peer;
+    int rc;
+    ucp_peer = &(ucx_ctx->ucp_peers[pe]);
+    mca_spml_ucx_rkey_store_put(&ucx_ctx->rkey_store, ucx_ctx->ucp_worker[0], ucx_mkey->rkey);
+    ucx_mkey->rkey = NULL;
+    rc = mca_spml_ucx_peer_mkey_cache_del(ucp_peer, segno);
+    if(OSHMEM_SUCCESS != rc){
+        SPML_UCX_ERROR("mca_spml_ucx_peer_mkey_cache_del failed");
+        return rc;
+    }
+    return OSHMEM_SUCCESS;
+}
+
+int mca_spml_ucx_del_procs(oshmem_group_t* group, size_t nprocs)
 {
     size_t ucp_workers = mca_spml_ucx.ucp_workers;
     opal_common_ucx_del_proc_t *del_procs;
@@ -128,6 +475,8 @@ int mca_spml_ucx_del_procs(ompi_proc_t** procs, size_t nprocs)
 
         /* mark peer as disconnected */
         mca_spml_ucx_ctx_default.ucp_peers[i].ucp_conn = NULL;
+        /* release the cached_ep_mkey buffer */
+        mca_spml_ucx_peer_mkey_cache_release(&(mca_spml_ucx_ctx_default.ucp_peers[i]));
     }
 
     ret = opal_common_ucx_del_procs_nofence(del_procs, nprocs, oshmem_my_proc_id(),
@@ -156,7 +505,7 @@ int mca_spml_ucx_del_procs(ompi_proc_t** procs, size_t nprocs)
     return ret;
 }
 
-/* TODO: move func into common place, use it with rkey exchng too */
+/* TODO: move func into common place, use it with rkey exchange too */
 static int oshmem_shmem_xchng(
         void **local_data, unsigned int *local_size, int nprocs, int ucp_workers,
         void **rdata_p, unsigned int **roffsets_p, unsigned int **rsizes_p)
@@ -248,13 +597,11 @@ static int oshmem_shmem_xchng(
 }
 
 
-static char spml_ucx_transport_ids[1] = { 0 };
-
 int mca_spml_ucx_init_put_op_mask(mca_spml_ucx_ctx_t *ctx, size_t nprocs)
 {
     int res;
 
-    if (mca_spml_ucx.synchronized_quiet) {
+    if (mca_spml_ucx_is_strong_ordering(ctx)) {
         ctx->put_proc_indexes = malloc(nprocs * sizeof(*ctx->put_proc_indexes));
         if (NULL == ctx->put_proc_indexes) {
             return OSHMEM_ERR_OUT_OF_RESOURCE;
@@ -276,7 +623,7 @@ int mca_spml_ucx_init_put_op_mask(mca_spml_ucx_ctx_t *ctx, size_t nprocs)
 
 int mca_spml_ucx_clear_put_op_mask(mca_spml_ucx_ctx_t *ctx)
 {
-    if (mca_spml_ucx.synchronized_quiet && ctx->put_proc_indexes) {
+    if (mca_spml_ucx_is_strong_ordering(ctx) && ctx->put_proc_indexes) {
         OBJ_DESTRUCT(&ctx->put_op_bitmap);
         free(ctx->put_proc_indexes);
     }
@@ -284,7 +631,7 @@ int mca_spml_ucx_clear_put_op_mask(mca_spml_ucx_ctx_t *ctx)
     return OSHMEM_SUCCESS;
 }
 
-int mca_spml_ucx_add_procs(ompi_proc_t** procs, size_t nprocs)
+int mca_spml_ucx_add_procs(oshmem_group_t* group, size_t nprocs)
 {
     int rc                  = OSHMEM_ERROR;
     int my_rank             = oshmem_my_proc_id();
@@ -292,7 +639,7 @@ int mca_spml_ucx_add_procs(ompi_proc_t** procs, size_t nprocs)
     unsigned int *wk_roffs  = NULL;
     unsigned int *wk_rsizes = NULL;
     char *wk_raddrs         = NULL;
-    size_t i, j, w, n;
+    size_t i, w, n;
     ucs_status_t err;
     ucp_address_t **wk_local_addr;
     unsigned int *wk_addr_len;
@@ -359,12 +706,8 @@ int mca_spml_ucx_add_procs(ompi_proc_t** procs, size_t nprocs)
             goto error2;
         }
 
-        OSHMEM_PROC_DATA(procs[i])->num_transports = 1;
-        OSHMEM_PROC_DATA(procs[i])->transport_ids = spml_ucx_transport_ids;
-
-        for (j = 0; j < MCA_MEMHEAP_MAX_SEGMENTS; j++) {
-            mca_spml_ucx_ctx_default.ucp_peers[i].mkeys[j].key.rkey = NULL;
-        }
+        /* Initialize mkeys as NULL for all processes */
+        mca_spml_ucx_peer_mkey_cache_init(&mca_spml_ucx_ctx_default, i);
     }
 
     for (i = 0; i < mca_spml_ucx.ucp_workers; i++) {
@@ -419,15 +762,27 @@ int mca_spml_ucx_add_procs(ompi_proc_t** procs, size_t nprocs)
 
 }
 
-void mca_spml_ucx_rmkey_free(sshmem_mkey_t *mkey)
+void mca_spml_ucx_rmkey_free(sshmem_mkey_t *mkey, int pe)
 {
     spml_ucx_mkey_t   *ucx_mkey;
+    uint32_t segno;
+    int rc;
 
     if (!mkey->spml_context) {
         return;
     }
+    segno = memheap_find_segnum(mkey->va_base, pe);
+    if (MEMHEAP_SEG_INVALID == segno) {
+        SPML_UCX_ERROR("mca_spml_ucx_rmkey_free failed because of invalid "
+            "segment number: %d\n", segno);
+        return;
+    }
+
     ucx_mkey = (spml_ucx_mkey_t *)(mkey->spml_context);
-    ucp_rkey_destroy(ucx_mkey->rkey);
+    rc = mca_spml_ucx_ctx_mkey_del(&mca_spml_ucx_ctx_default, pe, segno, ucx_mkey);
+    if (OSHMEM_SUCCESS != rc) {
+        SPML_UCX_ERROR("mca_spml_ucx_ctx_mkey_del failed\n");
+    }
 }
 
 void *mca_spml_ucx_rmkey_ptr(const void *dst_addr, sshmem_mkey_t *mkey, int pe)
@@ -451,22 +806,16 @@ void mca_spml_ucx_rmkey_unpack(shmem_ctx_t ctx, sshmem_mkey_t *mkey, uint32_t se
 {
     spml_ucx_mkey_t   *ucx_mkey;
     mca_spml_ucx_ctx_t *ucx_ctx = (mca_spml_ucx_ctx_t *)ctx;
-    ucs_status_t err;
-    
-    ucx_mkey = &ucx_ctx->ucp_peers[pe].mkeys[segno].key;
+    int rc;
 
-    err = ucp_ep_rkey_unpack(ucx_ctx->ucp_peers[pe].ucp_conn,
-            mkey->u.data,
-            &ucx_mkey->rkey); 
-    if (UCS_OK != err) {
-        SPML_UCX_ERROR("failed to unpack rkey: %s", ucs_status_string(err));
+    rc = mca_spml_ucx_ctx_mkey_add(ucx_ctx, pe, segno, mkey, &ucx_mkey);
+    if (OSHMEM_SUCCESS != rc) {
+        SPML_UCX_ERROR("mca_spml_ucx_ctx_mkey_cache failed");
         goto error_fatal;
     }
-
     if (ucx_ctx == &mca_spml_ucx_ctx_default) {
         mkey->spml_context = ucx_mkey;
     }
-    mca_spml_ucx_cache_mkey(ucx_ctx, mkey, segno, pe);
     return;
 
 error_fatal:
@@ -480,14 +829,18 @@ void mca_spml_ucx_memuse_hook(void *addr, size_t length)
     spml_ucx_mkey_t *ucx_mkey;
     ucp_mem_advise_params_t params;
     ucs_status_t status;
+    int rc;
 
     if (!(mca_spml_ucx.heap_reg_nb && memheap_is_va_in_segment(addr, HEAP_SEG_INDEX))) {
         return;
     }
 
-    my_pe    = oshmem_my_proc_id();
-    ucx_mkey = &mca_spml_ucx_ctx_default.ucp_peers[my_pe].mkeys[HEAP_SEG_INDEX].key;
-
+    my_pe = oshmem_my_proc_id();
+    rc = mca_spml_ucx_ctx_mkey_by_seg(&mca_spml_ucx_ctx_default, my_pe, HEAP_SEG_INDEX, &ucx_mkey);
+    if (OSHMEM_SUCCESS != rc) {
+        SPML_UCX_ERROR("mca_spml_ucx_ctx_mkey_by_seg failed");
+        return;
+    }
     params.field_mask = UCP_MEM_ADVISE_PARAM_FIELD_ADDRESS |
                         UCP_MEM_ADVISE_PARAM_FIELD_LENGTH |
                         UCP_MEM_ADVISE_PARAM_FIELD_ADVICE;
@@ -510,13 +863,15 @@ sshmem_mkey_t *mca_spml_ucx_register(void* addr,
 {
     sshmem_mkey_t *mkeys;
     ucs_status_t status;
-    spml_ucx_mkey_t   *ucx_mkey;
+    spml_ucx_mkey_t   *ucx_mkey = NULL;
     size_t len;
     ucp_mem_map_params_t mem_map_params;
-    int segno;
+    uint32_t segno;
     map_segment_t *mem_seg;
     unsigned flags;
     int my_pe = oshmem_my_proc_id();
+    int rc;
+    ucp_mem_h mem_h;
 
     *count = 0;
     mkeys = (sshmem_mkey_t *) calloc(1, sizeof(*mkeys));
@@ -524,12 +879,14 @@ sshmem_mkey_t *mca_spml_ucx_register(void* addr,
         return NULL;
     }
 
-    segno   = memheap_find_segnum(addr);
+    segno   = memheap_find_segnum(addr, my_pe);
+    if (MEMHEAP_SEG_INVALID == segno) {
+        SPML_UCX_ERROR("mca_spml_ucx_register failed because of invalid "
+            "segment number: %d\n", segno);
+        goto error_out;
+    }
     mem_seg = memheap_find_seg(segno);
 
-    ucx_mkey = &mca_spml_ucx_ctx_default.ucp_peers[my_pe].mkeys[segno].key;
-    mkeys[0].spml_context = ucx_mkey;
-
     /* if possible use mem handle already created by ucx allocator */
     if (MAP_SEGMENT_ALLOC_UCX != mem_seg->type) {
         flags = 0;
@@ -542,22 +899,23 @@ sshmem_mkey_t *mca_spml_ucx_register(void* addr,
                                     UCP_MEM_MAP_PARAM_FIELD_FLAGS;
         mem_map_params.address    = addr;
         mem_map_params.length     = size;
-        mem_map_params.flags      = flags;
+        mem_map_params.flags      = flags |
+            mca_spml_ucx_mem_map_flags_symmetric_rkey(&mca_spml_ucx);
 
-        status = ucp_mem_map(mca_spml_ucx.ucp_context, &mem_map_params, &ucx_mkey->mem_h);
+        status = ucp_mem_map(mca_spml_ucx.ucp_context, &mem_map_params, &mem_h);
         if (UCS_OK != status) {
             goto error_out;
         }
 
     } else {
         mca_sshmem_ucx_segment_context_t *ctx = mem_seg->context;
-        ucx_mkey->mem_h = ctx->ucp_memh;
+        mem_h  = ctx->ucp_memh;
     }
 
-    status = ucp_rkey_pack(mca_spml_ucx.ucp_context, ucx_mkey->mem_h,
-                           &mkeys[0].u.data, &len);
+    status = ucp_rkey_pack(mca_spml_ucx.ucp_context, mem_h,
+                           &mkeys[SPML_UCX_TRANSP_IDX].u.data, &len);
     if (UCS_OK != status) {
-        goto error_unmap;
+        goto error_out;
     }
     if (len >= 0xffff) {
         SPML_UCX_ERROR("packed rkey is too long: %llu >= %d",
@@ -565,23 +923,22 @@ sshmem_mkey_t *mca_spml_ucx_register(void* addr,
                 0xffff);
         oshmem_shmem_abort(-1);
     }
-
-    status = ucp_ep_rkey_unpack(mca_spml_ucx_ctx_default.ucp_peers[oshmem_group_self->my_pe].ucp_conn,
-                                mkeys[0].u.data,
-                                &ucx_mkey->rkey);
-    if (UCS_OK != status) {
-        SPML_UCX_ERROR("failed to unpack rkey");
+    mkeys[SPML_UCX_TRANSP_IDX].len     = len;
+    mkeys[SPML_UCX_TRANSP_IDX].va_base = addr;
+    *count = SPML_UCX_TRANSP_CNT;
+    rc = mca_spml_ucx_ctx_mkey_add(&mca_spml_ucx_ctx_default, my_pe, segno, &mkeys[SPML_UCX_TRANSP_IDX], &ucx_mkey);
+    if (OSHMEM_SUCCESS != rc) {
+        SPML_UCX_ERROR("mca_spml_ucx_ctx_mkey_cache failed");
         goto error_unmap;
     }
-
-    mkeys[0].len     = len;
-    mkeys[0].va_base = addr;
-    *count = 1;
-    mca_spml_ucx_cache_mkey(&mca_spml_ucx_ctx_default, &mkeys[0], segno, my_pe);
+    ucx_mkey->mem_h = mem_h;
+    mkeys[SPML_UCX_TRANSP_IDX].spml_context = ucx_mkey;
     return mkeys;
 
 error_unmap:
-    ucp_mem_unmap(mca_spml_ucx.ucp_context, ucx_mkey->mem_h);
+    if (NULL != ucx_mkey) {
+        ucp_mem_unmap(mca_spml_ucx.ucp_context, ucx_mkey->mem_h);
+    }
 error_out:
     free(mkeys);
 
@@ -592,29 +949,42 @@ int mca_spml_ucx_deregister(sshmem_mkey_t *mkeys)
 {
     spml_ucx_mkey_t   *ucx_mkey;
     map_segment_t *mem_seg;
+    int my_pe = oshmem_my_proc_id();
+    int rc;
+    uint32_t segno;
 
     MCA_SPML_CALL(quiet(oshmem_ctx_default));
     if (!mkeys)
         return OSHMEM_SUCCESS;
 
-    if (!mkeys[0].spml_context)
+    if (!mkeys[SPML_UCX_TRANSP_IDX].spml_context)
         return OSHMEM_SUCCESS;
 
-    mem_seg  = memheap_find_va(mkeys[0].va_base);
-    ucx_mkey = (spml_ucx_mkey_t*)mkeys[0].spml_context;
-
+    mem_seg  = memheap_find_va(mkeys[SPML_UCX_TRANSP_IDX].va_base);
     if (OPAL_UNLIKELY(NULL == mem_seg)) {
         return OSHMEM_ERROR;
     }
 
+    segno = memheap_find_segnum(mkeys[SPML_UCX_TRANSP_IDX].va_base, my_pe);
+    if (MEMHEAP_SEG_INVALID == segno) {
+        SPML_UCX_ERROR("mca_spml_ucx_deregister failed because of invalid "
+            "segment number: %d\n", segno);
+        return OSHMEM_ERROR;
+    }
+
+    ucx_mkey = (spml_ucx_mkey_t*)mkeys[SPML_UCX_TRANSP_IDX].spml_context;
+
     if (MAP_SEGMENT_ALLOC_UCX != mem_seg->type) {
         ucp_mem_unmap(mca_spml_ucx.ucp_context, ucx_mkey->mem_h);
     }
-    ucp_rkey_destroy(ucx_mkey->rkey);
-    ucx_mkey->rkey = NULL;
 
-    if (0 < mkeys[0].len) {
-        ucp_rkey_buffer_release(mkeys[0].u.data);
+    rc = mca_spml_ucx_ctx_mkey_del(&mca_spml_ucx_ctx_default, my_pe, segno, ucx_mkey);
+    if (OSHMEM_SUCCESS != rc) {
+        SPML_UCX_ERROR("mca_spml_ucx_ctx_mkey_del failed\n");
+        return rc;
+    }
+    if (0 < mkeys[SPML_UCX_TRANSP_IDX].len) {
+        ucp_rkey_buffer_release(mkeys[SPML_UCX_TRANSP_IDX].u.data);
     }
 
     free(mkeys);
@@ -674,10 +1044,14 @@ static int mca_spml_ucx_ctx_create_common(long options, mca_spml_ucx_ctx_t **ucx
     ucx_ctx->ucp_worker = calloc(1, sizeof(ucp_worker_h));
     ucx_ctx->ucp_workers = 1;
     ucx_ctx->synchronized_quiet = mca_spml_ucx_ctx_default.synchronized_quiet;
+    ucx_ctx->strong_sync = mca_spml_ucx_ctx_default.strong_sync;      
 
     params.field_mask  = UCP_WORKER_PARAM_FIELD_THREAD_MODE;
-    if (oshmem_mpi_thread_provided == SHMEM_THREAD_SINGLE || options & SHMEM_CTX_PRIVATE || options & SHMEM_CTX_SERIALIZED) {
+    if (oshmem_mpi_thread_provided == SHMEM_THREAD_SINGLE ||
+        oshmem_mpi_thread_provided == SHMEM_THREAD_FUNNELED || options & SHMEM_CTX_PRIVATE) {
         params.thread_mode = UCS_THREAD_MODE_SINGLE;
+    } else if (oshmem_mpi_thread_provided == SHMEM_THREAD_SERIALIZED || options & SHMEM_CTX_SERIALIZED) {
+        params.thread_mode = UCS_THREAD_MODE_SERIALIZED;
     } else {
         params.thread_mode = UCS_THREAD_MODE_MULTI;
     }
@@ -713,20 +1087,16 @@ static int mca_spml_ucx_ctx_create_common(long options, mca_spml_ucx_ctx_t **ucx
 
         for (j = 0; j < memheap_map->n_segments; j++) {
             mkey = &memheap_map->mem_segs[j].mkeys_cache[i][0];
-            ucx_mkey = &ucx_ctx->ucp_peers[i].mkeys[j].key;
-            if (mkey->u.data) {
-                err = ucp_ep_rkey_unpack(ucx_ctx->ucp_peers[i].ucp_conn,
-                                         mkey->u.data,
-                                         &ucx_mkey->rkey);
-                if (UCS_OK != err) {
-                    SPML_UCX_ERROR("failed to unpack rkey");
-                    goto error2;
-                }
-                mca_spml_ucx_cache_mkey(ucx_ctx, mkey, j, i);
+            rc = mca_spml_ucx_ctx_mkey_add(ucx_ctx, i, j, mkey, &ucx_mkey);
+            if (OSHMEM_SUCCESS != rc) {
+                SPML_UCX_ERROR("mca_spml_ucx_ctx_mkey_add failed");
+                goto error2;
             }
         }
     }
 
+    mca_spml_ucx_rkey_store_init(&ucx_ctx->rkey_store);
+
     *ucx_ctx_p = ucx_ctx;
 
     return OSHMEM_SUCCESS;
@@ -815,8 +1185,9 @@ void mca_spml_ucx_ctx_destroy(shmem_ctx_t ctx)
 
 int mca_spml_ucx_get(shmem_ctx_t ctx, void *src_addr, size_t size, void *dst_addr, int src)
 {
-    void *rva;
-    spml_ucx_mkey_t *ucx_mkey = mca_spml_ucx_get_mkey(ctx, src, src_addr, &rva, &mca_spml_ucx);
+    void *rva = NULL;
+    spml_ucx_mkey_t *ucx_mkey = mca_spml_ucx_ctx_mkey_by_va(ctx, src, src_addr, &rva, &mca_spml_ucx);
+    assert(NULL != ucx_mkey);
     mca_spml_ucx_ctx_t *ucx_ctx = (mca_spml_ucx_ctx_t *)ctx;
 #if (HAVE_DECL_UCP_GET_NBX || HAVE_DECL_UCP_GET_NB)
     ucs_status_ptr_t request;
@@ -826,7 +1197,7 @@ int mca_spml_ucx_get(shmem_ctx_t ctx, void *src_addr, size_t size, void *dst_add
 
 #if HAVE_DECL_UCP_GET_NBX
     request = ucp_get_nbx(ucx_ctx->ucp_peers[src].ucp_conn, dst_addr, size,
-                          (uint64_t)rva, ucx_mkey->rkey, &mca_spml_ucx_request_param);
+                          (uint64_t)rva, ucx_mkey->rkey, &mca_spml_ucx_request_param_b);
     return opal_common_ucx_wait_request(request, ucx_ctx->ucp_worker[0], "ucp_get_nbx");
 #elif HAVE_DECL_UCP_GET_NB
     request = ucp_get_nb(ucx_ctx->ucp_peers[src].ucp_conn, dst_addr, size,
@@ -841,9 +1212,10 @@ int mca_spml_ucx_get(shmem_ctx_t ctx, void *src_addr, size_t size, void *dst_add
 
 int mca_spml_ucx_get_nb(shmem_ctx_t ctx, void *src_addr, size_t size, void *dst_addr, int src, void **handle)
 {
-    void *rva;
+    void *rva = NULL;
     ucs_status_t status;
-    spml_ucx_mkey_t *ucx_mkey = mca_spml_ucx_get_mkey(ctx, src, src_addr, &rva, &mca_spml_ucx);
+    spml_ucx_mkey_t *ucx_mkey = mca_spml_ucx_ctx_mkey_by_va(ctx, src, src_addr, &rva, &mca_spml_ucx);
+    assert(NULL != ucx_mkey);
     mca_spml_ucx_ctx_t *ucx_ctx = (mca_spml_ucx_ctx_t *)ctx;
 #if HAVE_DECL_UCP_GET_NBX
     ucs_status_ptr_t status_ptr;
@@ -868,9 +1240,10 @@ int mca_spml_ucx_get_nb(shmem_ctx_t ctx, void *src_addr, size_t size, void *dst_
 int mca_spml_ucx_get_nb_wprogress(shmem_ctx_t ctx, void *src_addr, size_t size, void *dst_addr, int src, void **handle)
 {
     unsigned int i;
-    void *rva;
+    void *rva = NULL;
     ucs_status_t status;
-    spml_ucx_mkey_t *ucx_mkey = mca_spml_ucx_get_mkey(ctx, src, src_addr, &rva, &mca_spml_ucx);
+    spml_ucx_mkey_t *ucx_mkey = mca_spml_ucx_ctx_mkey_by_va(ctx, src, src_addr, &rva, &mca_spml_ucx);
+    assert(NULL != ucx_mkey);
     mca_spml_ucx_ctx_t *ucx_ctx = (mca_spml_ucx_ctx_t *)ctx;
 #if HAVE_DECL_UCP_GET_NBX
     ucs_status_ptr_t status_ptr;
@@ -905,8 +1278,9 @@ int mca_spml_ucx_get_nb_wprogress(shmem_ctx_t ctx, void *src_addr, size_t size,
 
 int mca_spml_ucx_put(shmem_ctx_t ctx, void* dst_addr, size_t size, void* src_addr, int dst)
 {
-    void *rva;
-    spml_ucx_mkey_t *ucx_mkey = mca_spml_ucx_get_mkey(ctx, dst, dst_addr, &rva, &mca_spml_ucx);
+    void *rva = NULL;
+    spml_ucx_mkey_t *ucx_mkey = mca_spml_ucx_ctx_mkey_by_va(ctx, dst, dst_addr, &rva, &mca_spml_ucx);
+    assert(NULL != ucx_mkey);
     mca_spml_ucx_ctx_t *ucx_ctx = (mca_spml_ucx_ctx_t *)ctx;
     int res;
 #if (HAVE_DECL_UCP_PUT_NBX || HAVE_DECL_UCP_PUT_NB)
@@ -917,7 +1291,7 @@ int mca_spml_ucx_put(shmem_ctx_t ctx, void* dst_addr, size_t size, void* src_add
 
 #if HAVE_DECL_UCP_PUT_NBX
     request = ucp_put_nbx(ucx_ctx->ucp_peers[dst].ucp_conn, src_addr, size,
-                          (uint64_t)rva, ucx_mkey->rkey, &mca_spml_ucx_request_param);
+                          (uint64_t)rva, ucx_mkey->rkey, &mca_spml_ucx_request_param_b);
     res = opal_common_ucx_wait_request(request, ucx_ctx->ucp_worker[0], "ucp_put_nbx");
 #elif HAVE_DECL_UCP_PUT_NB
     request = ucp_put_nb(ucx_ctx->ucp_peers[dst].ucp_conn, src_addr, size,
@@ -938,8 +1312,9 @@ int mca_spml_ucx_put(shmem_ctx_t ctx, void* dst_addr, size_t size, void* src_add
 
 int mca_spml_ucx_put_nb(shmem_ctx_t ctx, void* dst_addr, size_t size, void* src_addr, int dst, void **handle)
 {
-    void *rva;
-    spml_ucx_mkey_t *ucx_mkey = mca_spml_ucx_get_mkey(ctx, dst, dst_addr, &rva, &mca_spml_ucx);
+    void *rva = NULL;
+    spml_ucx_mkey_t *ucx_mkey = mca_spml_ucx_ctx_mkey_by_va(ctx, dst, dst_addr, &rva, &mca_spml_ucx);
+    assert(NULL != ucx_mkey);
     mca_spml_ucx_ctx_t *ucx_ctx = (mca_spml_ucx_ctx_t *)ctx;
     ucs_status_t status;
 #if HAVE_DECL_UCP_PUT_NBX
@@ -970,9 +1345,10 @@ int mca_spml_ucx_put_nb(shmem_ctx_t ctx, void* dst_addr, size_t size, void* src_
 int mca_spml_ucx_put_nb_wprogress(shmem_ctx_t ctx, void* dst_addr, size_t size, void* src_addr, int dst, void **handle)
 {
     unsigned int i;
-    void *rva;
+    void *rva = NULL;
     ucs_status_t status;
-    spml_ucx_mkey_t *ucx_mkey = mca_spml_ucx_get_mkey(ctx, dst, dst_addr, &rva, &mca_spml_ucx);
+    spml_ucx_mkey_t *ucx_mkey = mca_spml_ucx_ctx_mkey_by_va(ctx, dst, dst_addr, &rva, &mca_spml_ucx);
+    assert(NULL != ucx_mkey);
     mca_spml_ucx_ctx_t *ucx_ctx = (mca_spml_ucx_ctx_t *)ctx;
 #if HAVE_DECL_UCP_PUT_NBX
     ucs_status_ptr_t status_ptr;
@@ -1008,14 +1384,81 @@ int mca_spml_ucx_put_nb_wprogress(shmem_ctx_t ctx, void* dst_addr, size_t size,
     return ucx_status_to_oshmem_nb(status);
 }
 
+static int mca_spml_ucx_strong_sync(shmem_ctx_t ctx)
+{
+    mca_spml_ucx_ctx_t *ucx_ctx = (mca_spml_ucx_ctx_t *)ctx;
+    ucs_status_ptr_t request;
+    static int flush_get_data;
+    unsigned i;
+    int ret;
+    int idx;
+#if !(HAVE_DECL_UCP_EP_FLUSH_NBX || HAVE_DECL_UCP_EP_FLUSH_NB)
+    ucs_status_t status;
+#endif
+
+    for (i = 0; i < ucx_ctx->put_proc_count; i++) {
+        idx = ucx_ctx->put_proc_indexes[i];
+
+        switch (ucx_ctx->strong_sync) {
+        case SPML_UCX_STRONG_ORDERING_NONE:
+        case SPML_UCX_STRONG_ORDERING_GETNB:
+            ret = mca_spml_ucx_get_nb(ctx,
+                                      ucx_ctx->ucp_peers[idx].mkeys[SPML_UCX_SERVICE_SEG]->super.super.va_base,
+                                      sizeof(flush_get_data), &flush_get_data, idx, NULL);
+            break;
+        case SPML_UCX_STRONG_ORDERING_GET:
+            ret = mca_spml_ucx_get(ctx,
+                                   ucx_ctx->ucp_peers[idx].mkeys[SPML_UCX_SERVICE_SEG]->super.super.va_base,
+                                   sizeof(flush_get_data), &flush_get_data, idx);
+            break;
+#if HAVE_DECL_UCP_EP_FLUSH_NBX
+        case SPML_UCX_STRONG_ORDERING_FLUSH:
+            request = ucp_ep_flush_nbx(ucx_ctx->ucp_peers[idx].ucp_conn,
+                                       &mca_spml_ucx_request_param_b);
+            ret = opal_common_ucx_wait_request(request, ucx_ctx->ucp_worker[0], "ucp_flush_nbx");
+#elif HAVE_DECL_UCP_EP_FLUSH_NB
+            request = ucp_ep_flush_nb(ucx_ctx->ucp_peers[idx].ucp_conn, 0, opal_common_ucx_empty_complete_cb);
+            ret = opal_common_ucx_wait_request(request, ucx_ctx->ucp_worker[0], "ucp_flush_nb");
+#else
+            status = ucp_ep_flush(ucx_ctx->ucp_peers[idx].ucp_conn);
+            ret = (status == UCS_OK) ? OPAL_SUCCESS : OPAL_ERROR;
+#endif
+            break;
+        default:
+            /* unknown mode */
+            ret = OMPI_SUCCESS;
+            break;
+        }
+
+        if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
+            oshmem_shmem_abort(-1);
+            return ret;
+        }
+
+        opal_bitmap_clear_bit(&ucx_ctx->put_op_bitmap, idx);
+    }
+
+    ucx_ctx->put_proc_count = 0;
+    return OSHMEM_SUCCESS;
+}
+
 int mca_spml_ucx_fence(shmem_ctx_t ctx)
 {
+    mca_spml_ucx_ctx_t *ucx_ctx = (mca_spml_ucx_ctx_t *)ctx;
     ucs_status_t err;
+    int ret;
     unsigned int i = 0;
-    mca_spml_ucx_ctx_t *ucx_ctx = (mca_spml_ucx_ctx_t *)ctx;
 
     opal_atomic_wmb();
 
+    if (ucx_ctx->strong_sync != SPML_UCX_STRONG_ORDERING_NONE) {
+        ret = mca_spml_ucx_strong_sync(ctx);
+        if (ret != OSHMEM_SUCCESS) {
+            oshmem_shmem_abort(-1);
+            return ret;
+        }
+    }
+
     for (i=0; i < ucx_ctx->ucp_workers; i++) {
         if (ucx_ctx->ucp_worker[i] != NULL) {
             err = ucp_worker_fence(ucx_ctx->ucp_worker[i]);
@@ -1031,26 +1474,16 @@ int mca_spml_ucx_fence(shmem_ctx_t ctx)
 
 int mca_spml_ucx_quiet(shmem_ctx_t ctx)
 {
-    int flush_get_data;
     int ret;
     unsigned i;
-    int idx;
     mca_spml_ucx_ctx_t *ucx_ctx = (mca_spml_ucx_ctx_t *)ctx;
 
-    if (mca_spml_ucx.synchronized_quiet) {
-        for (i = 0; i < ucx_ctx->put_proc_count; i++) {
-            idx = ucx_ctx->put_proc_indexes[i];
-            ret = mca_spml_ucx_get_nb(ctx,
-                                      ucx_ctx->ucp_peers[idx].mkeys->super.super.va_base,
-                                      sizeof(flush_get_data), &flush_get_data, idx, NULL);
-            if (OMPI_SUCCESS != ret) {
-                oshmem_shmem_abort(-1);
-                return ret;
-            }
-
-            opal_bitmap_clear_bit(&ucx_ctx->put_op_bitmap, idx);
+    if (ucx_ctx->synchronized_quiet) {
+        ret = mca_spml_ucx_strong_sync(ctx);
+        if (ret != OSHMEM_SUCCESS) {
+            oshmem_shmem_abort(-1);
+            return ret;
         }
-        ucx_ctx->put_proc_count = 0;
     }
 
     opal_atomic_wmb();
@@ -1210,3 +1643,343 @@ int mca_spml_ucx_put_all_nb(void *dest, const void *source, size_t size, long *c
 
     return OSHMEM_SUCCESS;
 }
+
+/* This routine is not implemented */
+int mca_spml_ucx_put_signal(shmem_ctx_t ctx, void* dst_addr, size_t size, void*
+        src_addr, uint64_t *sig_addr, uint64_t signal, int sig_op, int dst)
+{
+    return OSHMEM_ERR_NOT_IMPLEMENTED;
+}
+
+/* This routine is not implemented */
+int mca_spml_ucx_put_signal_nb(shmem_ctx_t ctx, void* dst_addr, size_t size,
+        void* src_addr, uint64_t *sig_addr, uint64_t signal, int sig_op, int
+        dst)
+{
+    return OSHMEM_ERR_NOT_IMPLEMENTED;
+}
+
+/* This routine is not implemented */
+void mca_spml_ucx_wait_until_all(void *ivars, int cmp, void
+        *cmp_value, size_t nelems, const int *status, int datatype)
+{
+    RUNTIME_SHMEM_NOT_IMPLEMENTED_API_ABORT();
+}
+
+/* This routine is not implemented */
+size_t mca_spml_ucx_wait_until_any(void *ivars, int cmp, void
+        *cmp_value, size_t nelems, const int *status, int datatype)
+{
+    RUNTIME_SHMEM_NOT_IMPLEMENTED_API_ABORT_RET_SIZE_T();
+}
+
+/* This routine is not implemented */
+size_t mca_spml_ucx_wait_until_some(void *ivars, int cmp, void
+        *cmp_value, size_t nelems, size_t *indices, const int *status, int
+        datatype)
+{
+    RUNTIME_SHMEM_NOT_IMPLEMENTED_API_ABORT_RET_SIZE_T();
+}
+
+/* This routine is not implemented */ 
+void mca_spml_ucx_wait_until_all_vector(void *ivars, int cmp, void
+        *cmp_values, size_t nelems, const int *status, int datatype)
+{
+    RUNTIME_SHMEM_NOT_IMPLEMENTED_API_ABORT();
+}
+
+/* This routine is not implemented */
+size_t mca_spml_ucx_wait_until_any_vector(void *ivars, int cmp, void
+        *cmp_value, size_t nelems, const int *status, int datatype)
+{
+    RUNTIME_SHMEM_NOT_IMPLEMENTED_API_ABORT_RET_SIZE_T();
+}
+
+/* This routine is not implemented */
+size_t mca_spml_ucx_wait_until_some_vector(void *ivars, int cmp, void
+        *cmp_value, size_t nelems, size_t *indices, const int *status, int
+        datatype)
+{
+    RUNTIME_SHMEM_NOT_IMPLEMENTED_API_ABORT_RET_SIZE_T();
+}
+
+/* This routine is not implemented */
+int mca_spml_ucx_test_all(void *ivars, int cmp, void *cmp_value,
+        size_t nelems, const int *status, int datatype)
+{
+    return OSHMEM_ERR_NOT_IMPLEMENTED;
+}
+
+/* This routine is not implemented */
+size_t mca_spml_ucx_test_any(void *ivars, int cmp, void *cmp_value,
+        size_t nelems, const int *status, int datatype)
+{
+    RUNTIME_SHMEM_NOT_IMPLEMENTED_API_ABORT_RET_SIZE_T();
+}
+
+/* This routine is not implemented */
+size_t mca_spml_ucx_test_some(void *ivars, int cmp, void *cmp_value,
+        size_t nelems, size_t *indices, const int *status, int datatype)
+{
+    RUNTIME_SHMEM_NOT_IMPLEMENTED_API_ABORT_RET_SIZE_T();
+}
+
+/* This routine is not implemented */
+int mca_spml_ucx_test_all_vector(void *ivars, int cmp, void *cmp_values,
+                                 size_t nelems, const int *status,
+                                 int datatype)
+{
+    return OSHMEM_ERR_NOT_IMPLEMENTED;
+}
+
+/* This routine is not implemented */
+size_t mca_spml_ucx_test_any_vector(void *ivars, int cmp,
+                                    void *cmp_values, size_t nelems,
+                                    const int *status, int datatype)
+{
+    RUNTIME_SHMEM_NOT_IMPLEMENTED_API_ABORT_RET_SIZE_T();
+}
+
+/* This routine is not implemented */
+size_t mca_spml_ucx_test_some_vector(void *ivars, int cmp,
+                                     void *cmp_values, size_t nelems,
+                                     size_t *indices, const int *status,
+                                     int datatype)
+{
+    RUNTIME_SHMEM_NOT_IMPLEMENTED_API_ABORT_RET_SIZE_T();
+}
+
+/* This routine is not implemented */
+int mca_spml_ucx_team_sync(shmem_team_t team)
+{
+    return OSHMEM_ERR_NOT_IMPLEMENTED;
+}
+
+int mca_spml_ucx_team_my_pe(shmem_team_t team)
+{
+    mca_spml_ucx_team_t *ucx_team = (mca_spml_ucx_team_t *)team;
+
+    if (team == SHMEM_TEAM_WORLD) {
+        return shmem_my_pe();
+    }
+
+    return ucx_team->my_pe;
+}
+
+int mca_spml_ucx_team_n_pes(shmem_team_t team)
+{
+    mca_spml_ucx_team_t *ucx_team = (mca_spml_ucx_team_t *)team;
+
+    if (team == SHMEM_TEAM_WORLD) {
+        return shmem_n_pes();
+    }
+
+    return ucx_team->n_pes;
+}
+
+int mca_spml_ucx_team_get_config(shmem_team_t team, long config_mask,
+        shmem_team_config_t *config)
+{
+    mca_spml_ucx_team_t *ucx_team = (mca_spml_ucx_team_t *)team;
+    SPML_UCX_VALIDATE_TEAM(team);
+
+    memcpy(config, &ucx_team->config, sizeof(shmem_team_config_t));
+
+    return SHMEM_SUCCESS;
+}
+
+static inline int mca_spml_ucx_is_pe_in_strided_team(int src_pe, int start,
+                                                     int stride, int size)
+{
+    return (src_pe >= start) && (src_pe < start + size * stride)
+           && ((src_pe - start) % stride == 0);
+}
+
+int mca_spml_ucx_team_translate_pe(shmem_team_t src_team, int src_pe,
+                                   shmem_team_t dest_team)
+{
+    mca_spml_ucx_team_t *ucx_src_team  = (mca_spml_ucx_team_t*) src_team;
+    mca_spml_ucx_team_t *ucx_dest_team = (mca_spml_ucx_team_t*) dest_team;
+    int global_pe;
+
+    if ((src_pe == SPML_UCX_PE_NOT_IN_TEAM) || (src_team == dest_team)) {
+        return src_pe;
+    }
+
+    global_pe = ucx_src_team->start + src_pe * ucx_src_team->stride;
+
+    if (dest_team == SHMEM_TEAM_WORLD) {
+        return global_pe;
+    }
+
+    if (!mca_spml_ucx_is_pe_in_strided_team(global_pe, ucx_dest_team->start, ucx_dest_team->stride,
+                                            ucx_dest_team->n_pes)) {
+        return SPML_UCX_PE_NOT_IN_TEAM;
+    }
+
+    return (global_pe - ucx_dest_team->start) / ucx_dest_team->stride;
+}
+
+int mca_spml_ucx_team_split_strided(shmem_team_t parent_team, int start, int
+        stride, int size, const shmem_team_config_t *config, long config_mask,
+        shmem_team_t *new_team)
+{
+    mca_spml_ucx_team_t *ucx_parent_team;
+    mca_spml_ucx_team_t *ucx_new_team;
+    int parent_pe;
+    int parent_start;
+    int parent_stride;
+    int my_pe;
+
+    SPML_UCX_ASSERT(((start + size * stride) <= oshmem_num_procs()) &&
+                    (stride > 0) && (size > 0));
+
+    if (parent_team == SHMEM_TEAM_WORLD) {
+        parent_pe = shmem_my_pe();
+        parent_start = 0;
+        parent_stride = 1;
+    } else {
+        ucx_parent_team = (mca_spml_ucx_team_t*) parent_team;
+        parent_pe = ucx_parent_team->my_pe;
+        parent_start = ucx_parent_team->start;
+        parent_stride = ucx_parent_team->stride;
+    }
+
+    if (mca_spml_ucx_is_pe_in_strided_team(parent_pe, start, stride, size)) {
+        my_pe = (parent_pe - start) / stride;
+    } else {
+        /* not in team, according to spec it should be SHMEM_TEAM_INVALID but its value is NULL which
+        can be also interpreted as 0 (first pe), therefore -1 is used */
+        my_pe = SPML_UCX_PE_NOT_IN_TEAM;
+    }
+
+    /* In order to simplify pe translations start and stride are calculated with respect to
+     * world_team */
+    ucx_new_team = (mca_spml_ucx_team_t *)malloc(sizeof(mca_spml_ucx_team_t));
+    ucx_new_team->start = parent_start + (start * parent_stride);
+    ucx_new_team->stride = parent_stride * stride;
+
+    ucx_new_team->n_pes = size;
+    ucx_new_team->my_pe = my_pe;
+
+    ucx_new_team->config = calloc(1, sizeof(mca_spml_ucx_team_config_t));
+
+    if (config != NULL) {
+        memcpy(&ucx_new_team->config->super, config, sizeof(shmem_team_config_t));
+    }
+
+    ucx_new_team->parent_team = (mca_spml_ucx_team_t*)parent_team;
+
+    *new_team = (shmem_team_t)ucx_new_team;
+
+    return OSHMEM_SUCCESS;
+}
+
+int mca_spml_ucx_team_split_2d(shmem_team_t parent_team, int xrange, const
+        shmem_team_config_t *xaxis_config, long xaxis_mask, shmem_team_t
+        *xaxis_team, const shmem_team_config_t *yaxis_config, long yaxis_mask,
+        shmem_team_t *yaxis_team)
+{
+    mca_spml_ucx_team_t *ucx_parent_team = (mca_spml_ucx_team_t*) parent_team;
+    int parent_n_pes = (parent_team == SHMEM_TEAM_WORLD) ?
+                                        oshmem_num_procs() :
+                                        ucx_parent_team->n_pes;
+    int parent_my_pe = (parent_team == SHMEM_TEAM_WORLD) ?
+                                        shmem_my_pe() :
+                                        ucx_parent_team->my_pe;
+    int yrange = parent_n_pes / xrange;
+    int pe_x = parent_my_pe % xrange;
+    int pe_y = parent_my_pe / xrange;
+    int rc;
+
+    /* Create x-team of my_pe */
+    rc = mca_spml_ucx_team_split_strided(parent_team, pe_y * xrange, 1, xrange,
+                                         xaxis_config, xaxis_mask, xaxis_team);
+
+    if (rc != OSHMEM_SUCCESS) {
+        SPML_UCX_ERROR("mca_spml_ucx_team_split_strided failed (x-axis team creation)");
+        return rc;
+    }
+
+    /* Create y-team of my_pe */
+    rc = mca_spml_ucx_team_split_strided(parent_team, pe_x, xrange, yrange,
+                                         yaxis_config, yaxis_mask, yaxis_team);
+    if (rc != OSHMEM_SUCCESS) {
+        SPML_UCX_ERROR("mca_spml_ucx_team_split_strided failed (y-axis team creation)");
+        goto out_free_xaxis;
+    }
+
+    return OSHMEM_SUCCESS;
+
+out_free_xaxis:
+    mca_spml_ucx_team_destroy(*xaxis_team);
+    return rc;
+}
+
+/* This routine is not implemented */
+int mca_spml_ucx_team_destroy(shmem_team_t team)
+{
+    mca_spml_ucx_team_t *ucx_team = (mca_spml_ucx_team_t *)team;
+
+    SPML_UCX_VALIDATE_TEAM(team);
+
+    free(ucx_team->config);
+    free(team);
+
+    return OSHMEM_SUCCESS;
+}
+
+/* This routine is not implemented */
+int mca_spml_ucx_team_get(shmem_ctx_t ctx, shmem_team_t *team)
+{
+    return OSHMEM_ERR_NOT_IMPLEMENTED;
+}
+
+/* This routine is not implemented */
+int mca_spml_ucx_team_create_ctx(shmem_team_t team, long options, shmem_ctx_t *ctx)
+{
+    return OSHMEM_ERR_NOT_IMPLEMENTED;
+}
+
+/* This routine is not implemented */
+int mca_spml_ucx_team_alltoall(shmem_team_t team, void
+        *dest, const void *source, size_t nelems, int datatype)
+{
+    return OSHMEM_ERR_NOT_IMPLEMENTED;
+}
+
+/* This routine is not implemented */
+int mca_spml_ucx_team_alltoalls(shmem_team_t team, void
+        *dest, const void *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems,
+        int datatype)
+{
+    return OSHMEM_ERR_NOT_IMPLEMENTED;
+}
+
+/* This routine is not implemented */
+int mca_spml_ucx_team_broadcast(shmem_team_t team, void
+        *dest, const void *source, size_t nelems, int PE_root, int datatype)
+{
+    return OSHMEM_ERR_NOT_IMPLEMENTED;
+}
+
+/* This routine is not implemented */
+int mca_spml_ucx_team_collect(shmem_team_t team, void
+        *dest, const void *source, size_t nelems, int datatype)
+{
+    return OSHMEM_ERR_NOT_IMPLEMENTED;
+}
+
+/* This routine is not implemented */
+int mca_spml_ucx_team_fcollect(shmem_team_t team, void
+        *dest, const void *source, size_t nelems, int datatype)
+{
+    return OSHMEM_ERR_NOT_IMPLEMENTED;
+}
+
+/* This routine is not implemented */
+int mca_spml_ucx_team_reduce(shmem_team_t team, void
+        *dest, const void *source, size_t nreduce, int operation, int datatype)
+{
+    return OSHMEM_ERR_NOT_IMPLEMENTED;
+}
diff --git a/oshmem/mca/spml/ucx/spml_ucx.h b/oshmem/mca/spml/ucx/spml_ucx.h
index b2802e7161a..0d1f345e925 100644
--- a/oshmem/mca/spml/ucx/spml_ucx.h
+++ b/oshmem/mca/spml/ucx/spml_ucx.h
@@ -43,7 +43,27 @@ BEGIN_C_DECLS
 
 #define SPML_UCX_ASSERT  MCA_COMMON_UCX_ASSERT
 #define SPML_UCX_ERROR   MCA_COMMON_UCX_ERROR
+#define SPML_UCX_WARN    MCA_COMMON_UCX_WARN
 #define SPML_UCX_VERBOSE MCA_COMMON_UCX_VERBOSE
+#define SPML_UCX_TRANSP_IDX 0
+#define SPML_UCX_TRANSP_CNT 1
+#define SPML_UCX_SERVICE_SEG 0
+#define SPML_UCX_PE_NOT_IN_TEAM -1
+
+#define SPML_UCX_VALIDATE_TEAM(_team)                       \
+    do {                                                    \
+        if (OPAL_UNLIKELY((_team) == SHMEM_TEAM_INVALID)) { \
+            SPML_UCX_ERROR("Invalid team at %s", __func__); \
+            return OSHMEM_ERROR;                            \
+        }                                                   \
+    } while (0)
+
+enum {
+    SPML_UCX_STRONG_ORDERING_NONE  = 0, /* don't use strong ordering */
+    SPML_UCX_STRONG_ORDERING_GETNB = 1, /* use non-blocking read to provide ordering */
+    SPML_UCX_STRONG_ORDERING_GET   = 2, /* use blocking read to provide ordering*/
+    SPML_UCX_STRONG_ORDERING_FLUSH = 3  /* flush EP to provide ordering */
+};
 
 /**
  * UCX SPML module
@@ -62,20 +82,35 @@ typedef struct spml_ucx_cached_mkey spml_ucx_cached_mkey_t;
 
 struct ucp_peer {
     ucp_ep_h                 ucp_conn;
-    spml_ucx_cached_mkey_t   mkeys[MCA_MEMHEAP_MAX_SEGMENTS];
+    spml_ucx_cached_mkey_t **mkeys;
+    size_t                   mkeys_cnt;
 };
 typedef struct ucp_peer ucp_peer_t;
- 
+
+/* An rkey_store entry */
+typedef struct mca_spml_ucx_rkey {
+    ucp_rkey_h rkey;
+    int        refcnt;
+} mca_spml_ucx_rkey_t;
+
+typedef struct mca_spml_ucx_rkey_store {
+    mca_spml_ucx_rkey_t *array;
+    int                  size;
+    int                  count;
+} mca_spml_ucx_rkey_store_t;
+
 struct mca_spml_ucx_ctx {
-    ucp_worker_h            *ucp_worker;
-    ucp_peer_t              *ucp_peers;
-    long                     options;
-    opal_bitmap_t            put_op_bitmap;
-    unsigned long            nb_progress_cnt;
-    unsigned int             ucp_workers;
-    int                     *put_proc_indexes;
-    unsigned                 put_proc_count;
-    bool                     synchronized_quiet;
+    ucp_worker_h             *ucp_worker;
+    ucp_peer_t               *ucp_peers;
+    long                      options;
+    opal_bitmap_t             put_op_bitmap;
+    unsigned long             nb_progress_cnt;
+    unsigned int              ucp_workers;
+    int                      *put_proc_indexes;
+    unsigned                  put_proc_count;
+    bool                      synchronized_quiet;
+    int                       strong_sync;
+    mca_spml_ucx_rkey_store_t rkey_store;
 };
 typedef struct mca_spml_ucx_ctx mca_spml_ucx_ctx_t;
 
@@ -89,6 +124,20 @@ typedef struct mca_spml_ucx_ctx_array {
     mca_spml_ucx_ctx_t       **ctxs;
 } mca_spml_ucx_ctx_array_t;
 
+typedef struct mca_spml_ucx_team_config {
+    shmem_team_config_t super;
+
+} mca_spml_ucx_team_config_t;
+
+typedef struct mca_spml_ucx_team {
+    int                         n_pes;
+    int                         my_pe;
+    int                         stride;
+    int                         start;
+    mca_spml_ucx_team_config_t  *config;
+    struct mca_spml_ucx_team    *parent_team;
+} mca_spml_ucx_team_t;
+
 struct mca_spml_ucx {
     mca_spml_base_module_t   super;
     ucp_context_h            ucp_context;
@@ -110,13 +159,13 @@ struct mca_spml_ucx {
     mca_spml_ucx_ctx_t       *aux_ctx;
     pthread_spinlock_t       async_lock;
     int                      aux_refcnt;
-    bool                     synchronized_quiet;
     unsigned long            nb_progress_thresh_global;
     unsigned long            nb_put_progress_thresh;
     unsigned long            nb_get_progress_thresh;
     unsigned long            nb_ucp_worker_progress;
     unsigned int             ucp_workers;
     unsigned int             ucp_worker_cnt;
+    int                      symmetric_rkey_max_count;
 };
 typedef struct mca_spml_ucx mca_spml_ucx_t;
 
@@ -186,11 +235,11 @@ extern int mca_spml_ucx_deregister(sshmem_mkey_t *mkeys);
 extern void mca_spml_ucx_memuse_hook(void *addr, size_t length);
 
 extern void mca_spml_ucx_rmkey_unpack(shmem_ctx_t ctx, sshmem_mkey_t *mkey, uint32_t segno, int pe, int tr_id);
-extern void mca_spml_ucx_rmkey_free(sshmem_mkey_t *mkey);
+extern void mca_spml_ucx_rmkey_free(sshmem_mkey_t *mkey, int pe);
 extern void *mca_spml_ucx_rmkey_ptr(const void *dst_addr, sshmem_mkey_t *, int pe);
 
-extern int mca_spml_ucx_add_procs(ompi_proc_t** procs, size_t nprocs);
-extern int mca_spml_ucx_del_procs(ompi_proc_t** procs, size_t nprocs);
+extern int mca_spml_ucx_add_procs(oshmem_group_t* group, size_t nprocs);
+extern int mca_spml_ucx_del_procs(oshmem_group_t* group, size_t nprocs);
 extern int mca_spml_ucx_fence(shmem_ctx_t ctx);
 extern int mca_spml_ucx_quiet(shmem_ctx_t ctx);
 extern int spml_ucx_default_progress(void);
@@ -200,6 +249,92 @@ void mca_spml_ucx_async_cb(int fd, short event, void *cbdata);
 
 int mca_spml_ucx_init_put_op_mask(mca_spml_ucx_ctx_t *ctx, size_t nprocs);
 int mca_spml_ucx_clear_put_op_mask(mca_spml_ucx_ctx_t *ctx);
+int mca_spml_ucx_peer_mkey_cache_add(ucp_peer_t *ucp_peer, int index);
+int mca_spml_ucx_peer_mkey_cache_del(ucp_peer_t *ucp_peer, int segno);
+void mca_spml_ucx_peer_mkey_cache_release(ucp_peer_t *ucp_peer);
+void mca_spml_ucx_peer_mkey_cache_init(mca_spml_ucx_ctx_t *ucx_ctx, int pe);
+
+extern int mca_spml_ucx_put_signal(shmem_ctx_t ctx, void* dst_addr, size_t size, void*
+        src_addr, uint64_t *sig_addr, uint64_t signal, int sig_op, int dst);
+
+extern int mca_spml_ucx_put_signal_nb(shmem_ctx_t ctx, void* dst_addr, size_t size,
+        void* src_addr, uint64_t *sig_addr, uint64_t signal, int sig_op, int
+        dst);
+extern void mca_spml_ucx_wait_until_all(void *ivars, int cmp, void
+        *cmp_value, size_t nelems, const int *status, int datatype);
+extern size_t mca_spml_ucx_wait_until_any(void *ivars, int cmp, void
+        *cmp_value, size_t nelems, const int *status, int datatype);
+extern size_t mca_spml_ucx_wait_until_some(void *ivars, int cmp, void
+        *cmp_value, size_t nelems, size_t *indices, const int *status, int
+        datatype);
+extern void mca_spml_ucx_wait_until_all_vector(void *ivars, int cmp, void
+        *cmp_values, size_t nelems, const int *status, int datatype);
+extern size_t mca_spml_ucx_wait_until_any_vector(void *ivars, int cmp, void
+        *cmp_value, size_t nelems, const int *status, int datatype);
+extern size_t mca_spml_ucx_wait_until_some_vector(void *ivars, int cmp, void
+        *cmp_value, size_t nelems, size_t *indices, const int *status, int
+        datatype);
+extern int mca_spml_ucx_test_all(void *ivars, int cmp, void *cmp_value,
+        size_t nelems, const int *status, int datatype);
+extern size_t mca_spml_ucx_test_any(void *ivars, int cmp, void *cmp_value,
+        size_t nelems, const int *status, int datatype);
+extern size_t mca_spml_ucx_test_some(void *ivars, int cmp, void *cmp_value,
+        size_t nelems, size_t *indices, const int *status, int datatype);
+extern int mca_spml_ucx_test_all_vector(void *ivars, int cmp, void
+        *cmp_values, size_t nelems, const int *status, int datatype);
+extern size_t mca_spml_ucx_test_any_vector(void *ivars, int cmp, void *cmp_values, size_t nelems,
+                                           const int *status, int datatype);
+extern size_t mca_spml_ucx_test_some_vector(void *ivars, int cmp, void *cmp_values, size_t nelems,
+                                            size_t *indices, const int *status, int datatype);
+extern int mca_spml_ucx_team_sync(shmem_team_t team);
+extern int mca_spml_ucx_team_my_pe(shmem_team_t team);
+extern int mca_spml_ucx_team_n_pes(shmem_team_t team);
+extern int mca_spml_ucx_team_get_config(shmem_team_t team, long config_mask,
+        shmem_team_config_t *config);
+extern int mca_spml_ucx_team_translate_pe(shmem_team_t src_team, int src_pe,
+        shmem_team_t dest_team);
+extern int mca_spml_ucx_team_split_strided(shmem_team_t parent_team, int start, int
+        stride, int size, const shmem_team_config_t *config, long config_mask,
+        shmem_team_t *new_team);
+extern int mca_spml_ucx_team_split_2d(shmem_team_t parent_team, int xrange, const
+        shmem_team_config_t *xaxis_config, long xaxis_mask, shmem_team_t
+        *xaxis_team, const shmem_team_config_t *yaxis_config, long yaxis_mask,
+        shmem_team_t *yaxis_team);
+extern int mca_spml_ucx_team_destroy(shmem_team_t team);
+extern int mca_spml_ucx_team_get(shmem_ctx_t ctx, shmem_team_t *team);
+extern int mca_spml_ucx_team_create_ctx(shmem_team_t team, long options, shmem_ctx_t *ctx);
+extern int mca_spml_ucx_team_alltoall(shmem_team_t team, void
+        *dest, const void *source, size_t nelems, int datatype);
+extern int mca_spml_ucx_team_alltoalls(shmem_team_t team, void
+        *dest, const void *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems,
+        int datatype);
+extern int mca_spml_ucx_team_broadcast(shmem_team_t team, void
+        *dest, const void *source, size_t nelems, int PE_root, int datatype);
+extern int mca_spml_ucx_team_collect(shmem_team_t team, void
+        *dest, const void *source, size_t nelems, int datatype);
+extern int mca_spml_ucx_team_fcollect(shmem_team_t team, void
+        *dest, const void *source, size_t nelems, int datatype);
+extern int mca_spml_ucx_team_reduce(shmem_team_t team, void
+        *dest, const void *source, size_t nreduce, int operation, int datatype);
+
+extern unsigned
+mca_spml_ucx_mem_map_flags_symmetric_rkey(struct mca_spml_ucx *spml_ucx);
+
+extern void mca_spml_ucx_rkey_store_init(mca_spml_ucx_rkey_store_t *store);
+extern void mca_spml_ucx_rkey_store_cleanup(mca_spml_ucx_rkey_store_t *store);
+
+static inline int
+mca_spml_ucx_peer_mkey_get(ucp_peer_t *ucp_peer, int index, spml_ucx_cached_mkey_t **out_rmkey)
+{
+    *out_rmkey = NULL;
+    if (OPAL_UNLIKELY((index >= (int)ucp_peer->mkeys_cnt) || (0 > index))) {
+        SPML_UCX_ERROR("Failed to get mkey for segment: bad index = %d, cached mkeys count: %zu",
+                       index, ucp_peer->mkeys_cnt);
+        return OSHMEM_ERR_BAD_PARAM;
+    }
+    *out_rmkey = ucp_peer->mkeys[index];
+    return OSHMEM_SUCCESS;
+}
 
 static inline void mca_spml_ucx_aux_lock(void)
 {
@@ -215,26 +350,44 @@ static inline void mca_spml_ucx_aux_unlock(void)
     }
 }
 
-static inline void mca_spml_ucx_cache_mkey(mca_spml_ucx_ctx_t *ucx_ctx,
-                                           sshmem_mkey_t *mkey, uint32_t segno, int dst_pe)
-{
-    ucp_peer_t *peer;
+int mca_spml_ucx_ctx_mkey_new(mca_spml_ucx_ctx_t *ucx_ctx, int pe, uint32_t segno, spml_ucx_mkey_t **mkey);
+int mca_spml_ucx_ctx_mkey_cache(mca_spml_ucx_ctx_t *ucx_ctx, sshmem_mkey_t *mkey, uint32_t segno, int dst_pe);
+int mca_spml_ucx_ctx_mkey_add(mca_spml_ucx_ctx_t *ucx_ctx, int pe, uint32_t segno, sshmem_mkey_t *mkey, spml_ucx_mkey_t **ucx_mkey);
+int mca_spml_ucx_ctx_mkey_del(mca_spml_ucx_ctx_t *ucx_ctx, int pe, uint32_t segno, spml_ucx_mkey_t *ucx_mkey);
 
-    peer = &(ucx_ctx->ucp_peers[dst_pe]);
-    mkey_segment_init(&peer->mkeys[segno].super, mkey, segno);
+static inline int
+mca_spml_ucx_ctx_mkey_by_seg(mca_spml_ucx_ctx_t *ucx_ctx, int pe, uint32_t segno, spml_ucx_mkey_t **mkey)
+{
+    ucp_peer_t *ucp_peer;
+    spml_ucx_cached_mkey_t *ucx_cached_mkey;
+    int rc;
+    ucp_peer = &(ucx_ctx->ucp_peers[pe]);
+    rc = mca_spml_ucx_peer_mkey_get(ucp_peer, segno, &ucx_cached_mkey);
+    if (OSHMEM_SUCCESS != rc) {
+        return rc;
+    }
+    *mkey = &(ucx_cached_mkey->key);
+    return OSHMEM_SUCCESS;
 }
 
 static inline spml_ucx_mkey_t * 
-mca_spml_ucx_get_mkey(shmem_ctx_t ctx, int pe, void *va, void **rva, mca_spml_ucx_t* module)
+mca_spml_ucx_ctx_mkey_by_va(shmem_ctx_t ctx, int pe, void *va, void **rva, mca_spml_ucx_t* module)
 {
-    spml_ucx_cached_mkey_t *mkey;
+    spml_ucx_cached_mkey_t **mkey;
     mca_spml_ucx_ctx_t *ucx_ctx = (mca_spml_ucx_ctx_t *)ctx;
+    size_t i;
 
     mkey = ucx_ctx->ucp_peers[pe].mkeys;
-    mkey = (spml_ucx_cached_mkey_t *)map_segment_find_va(&mkey->super.super, sizeof(*mkey), va);
-    assert(mkey != NULL);
-    *rva = map_segment_va2rva(&mkey->super, va);
-    return &mkey->key;
+    for (i = 0; i < ucx_ctx->ucp_peers[pe].mkeys_cnt; i++) {
+        if (NULL == mkey[i]) {
+            continue;
+        }
+        if (OPAL_LIKELY(map_segment_is_va_in(&mkey[i]->super.super, va))) {
+            *rva = map_segment_va2rva(&mkey[i]->super, va);
+            return &mkey[i]->key;
+        }
+    }
+    return NULL;
 }
 
 static inline int ucx_status_to_oshmem(ucs_status_t status)
@@ -255,9 +408,15 @@ static inline int ucx_status_to_oshmem_nb(ucs_status_t status)
 #endif
 }
 
+static inline int mca_spml_ucx_is_strong_ordering(mca_spml_ucx_ctx_t *ctx)
+{
+    return (ctx->strong_sync != SPML_UCX_STRONG_ORDERING_NONE) ||
+           ctx->synchronized_quiet;
+}
+
 static inline void mca_spml_ucx_remote_op_posted(mca_spml_ucx_ctx_t *ctx, int dst)
 {
-    if (OPAL_UNLIKELY(ctx->synchronized_quiet)) {
+    if (OPAL_UNLIKELY(mca_spml_ucx_is_strong_ordering(ctx))) {
         if (!opal_bitmap_is_set_bit(&ctx->put_op_bitmap, dst)) {
             ctx->put_proc_indexes[ctx->put_proc_count++] = dst;
             opal_bitmap_set_bit(&ctx->put_op_bitmap, dst);
@@ -271,4 +430,3 @@ static inline void mca_spml_ucx_remote_op_posted(mca_spml_ucx_ctx_t *ctx, int ds
 END_C_DECLS
 
 #endif
-
diff --git a/oshmem/mca/spml/ucx/spml_ucx_component.c b/oshmem/mca/spml/ucx/spml_ucx_component.c
index eb40076a706..affc73f3f88 100644
--- a/oshmem/mca/spml/ucx/spml_ucx_component.c
+++ b/oshmem/mca/spml/ucx/spml_ucx_component.c
@@ -153,6 +153,10 @@ static int mca_spml_ucx_component_register(void)
                                      "Enable asynchronous progress thread",
                                      &mca_spml_ucx.async_progress);
 
+    mca_spml_ucx_param_register_int("symmetric_rkey_max_count", 0,
+                                    "Size of the symmetric key store. Non-zero to enable, typical use 5000",
+                                    &mca_spml_ucx.symmetric_rkey_max_count);
+
     mca_spml_ucx_param_register_int("async_tick_usec", 3000,
                                     "Asynchronous progress tick granularity (in usec)",
                                     &mca_spml_ucx.async_tick);
@@ -161,6 +165,11 @@ static int mca_spml_ucx_component_register(void)
                                      "Use synchronized quiet on shmem_quiet or shmem_barrier_all operations",
                                      &mca_spml_ucx_ctx_default.synchronized_quiet);
 
+    mca_spml_ucx_param_register_int("strong_sync", 0,
+                                    "Use strong synchronization on shmem_quiet, shmem_fence or shmem_barrier_all operations: "
+                                    "0 - don't do strong synchronization, 1 - use non blocking get, 2 - use blocking get, 3 - use flush operation",
+                                    &mca_spml_ucx_ctx_default.strong_sync);
+
     mca_spml_ucx_param_register_ulong("nb_progress_thresh_global", 0,
                                     "Number of nb_put or nb_get operations before ucx progress is triggered. Disabled by default (0). Setting this value will override nb_put/get_progress_thresh.",
                                     &mca_spml_ucx.nb_progress_thresh_global);
@@ -239,11 +248,13 @@ void mca_spml_ucx_async_cb(int fd, short event, void *cbdata)
 
 static int mca_spml_ucx_component_open(void)
 {
+    opal_common_ucx_mca_register();
     return OSHMEM_SUCCESS;
 }
 
 static int mca_spml_ucx_component_close(void)
 {
+    opal_common_ucx_mca_deregister();
     return OSHMEM_SUCCESS;
 }
 
@@ -262,8 +273,6 @@ static int spml_ucx_init(void)
         return OSHMEM_ERROR;
     }
 
-    opal_common_ucx_mca_register();
-
     memset(&params, 0, sizeof(params));
     params.field_mask        = UCP_PARAM_FIELD_FEATURES          |
                                UCP_PARAM_FIELD_ESTIMATED_NUM_EPS |
@@ -313,6 +322,8 @@ static int spml_ucx_init(void)
     wkr_params.field_mask  = UCP_WORKER_PARAM_FIELD_THREAD_MODE;
     if (oshmem_mpi_thread_requested == SHMEM_THREAD_MULTIPLE) {
         wkr_params.thread_mode = UCS_THREAD_MODE_MULTI;
+    } else if (oshmem_mpi_thread_requested == SHMEM_THREAD_SERIALIZED) {
+        wkr_params.thread_mode = UCS_THREAD_MODE_SERIALIZED;
     } else {
         wkr_params.thread_mode = UCS_THREAD_MODE_SINGLE;
     }
@@ -327,6 +338,8 @@ static int spml_ucx_init(void)
         mca_spml_ucx_ctx_default.ucp_workers++;
     }
 
+    mca_spml_ucx_rkey_store_init(&mca_spml_ucx_ctx_default.rkey_store);
+
     wrk_attr.field_mask = UCP_WORKER_ATTR_FIELD_THREAD_MODE;
     err = ucp_worker_query(mca_spml_ucx_ctx_default.ucp_worker[0], &wrk_attr);
 
@@ -383,7 +396,14 @@ mca_spml_ucx_component_init(int* priority,
     if (OSHMEM_SUCCESS != spml_ucx_init())
         return NULL ;
 
+    if ((mca_spml_ucx_ctx_default.strong_sync < SPML_UCX_STRONG_ORDERING_NONE) ||
+        (mca_spml_ucx_ctx_default.strong_sync > SPML_UCX_STRONG_ORDERING_FLUSH)) {
+        SPML_UCX_ERROR("incorrect value of strong_sync parameter: %d",
+                       mca_spml_ucx_ctx_default.strong_sync);
+    }
+
     SPML_UCX_VERBOSE(50, "*** ucx initialized ****");
+
     return &mca_spml_ucx.super;
 }
 
@@ -391,13 +411,23 @@ static void _ctx_cleanup(mca_spml_ucx_ctx_t *ctx)
 {
     int i, j, nprocs = oshmem_num_procs();
     opal_common_ucx_del_proc_t *del_procs;
+    spml_ucx_mkey_t   *ucx_mkey;
+    int rc;
 
     del_procs = malloc(sizeof(*del_procs) * nprocs);
 
     for (i = 0; i < nprocs; ++i) {
         for (j = 0; j < memheap_map->n_segments; j++) {
-            if (ctx->ucp_peers[i].mkeys[j].key.rkey != NULL) {
-                ucp_rkey_destroy(ctx->ucp_peers[i].mkeys[j].key.rkey);
+            rc = mca_spml_ucx_ctx_mkey_by_seg(ctx, i, j, &ucx_mkey);
+            if (OSHMEM_SUCCESS != rc) {
+                SPML_UCX_ERROR("mca_spml_ucx_ctx_mkey_by_seg failed");
+            } else {
+                if (ucx_mkey->rkey != NULL) {
+                    rc = mca_spml_ucx_ctx_mkey_del(ctx, i, j, ucx_mkey);
+                    if (OSHMEM_SUCCESS != rc) {
+                        SPML_UCX_ERROR("mca_spml_ucx_ctx_mkey_del failed");
+                    }
+                }
             }
         }
 
@@ -414,9 +444,24 @@ static void _ctx_cleanup(mca_spml_ucx_ctx_t *ctx)
     free(ctx->ucp_peers);
 }
 
+static void mca_spml_ucx_ctx_fini(mca_spml_ucx_ctx_t *ctx)
+{
+    unsigned int i;
+
+    mca_spml_ucx_rkey_store_cleanup(&ctx->rkey_store);
+    for (i = 0; i < ctx->ucp_workers; i++) {
+        ucp_worker_destroy(ctx->ucp_worker[i]);
+    }
+    free(ctx->ucp_worker);
+    if (ctx != &mca_spml_ucx_ctx_default) {
+        free(ctx);
+    }
+}
+
 static int mca_spml_ucx_component_fini(void)
 {
-    int fenced = 0, i;
+    volatile int fenced = 0;
+    int i;
     int ret = OSHMEM_SUCCESS;
 
     opal_progress_unregister(spml_ucx_default_progress);
@@ -448,8 +493,10 @@ static int mca_spml_ucx_component_fini(void)
 
 
     ret = opal_common_ucx_mca_pmix_fence_nb(&fenced);
-    if (OPAL_SUCCESS != ret) {
-        return ret;
+    if (ret != PMIX_SUCCESS) {
+        SPML_UCX_WARN("pmix fence failed: %s", PMIx_Error_string(ret));
+        /* In case of pmix fence failure just continue cleanup */
+        fenced = 1;
     }
 
     while (!fenced) {
@@ -470,36 +517,26 @@ static int mca_spml_ucx_component_fini(void)
         }
     }
 
-    /* delete all workers */
     for (i = 0; i < mca_spml_ucx.active_array.ctxs_count; i++) {
-        ucp_worker_destroy(mca_spml_ucx.active_array.ctxs[i]->ucp_worker[0]);
-        free(mca_spml_ucx.active_array.ctxs[i]->ucp_worker);
-        free(mca_spml_ucx.active_array.ctxs[i]);
+        mca_spml_ucx_ctx_fini(mca_spml_ucx.active_array.ctxs[i]);
     }
 
     for (i = 0; i < mca_spml_ucx.idle_array.ctxs_count; i++) {
-        ucp_worker_destroy(mca_spml_ucx.idle_array.ctxs[i]->ucp_worker[0]);
-        free(mca_spml_ucx.idle_array.ctxs[i]->ucp_worker);
-        free(mca_spml_ucx.idle_array.ctxs[i]);
+        mca_spml_ucx_ctx_fini(mca_spml_ucx.idle_array.ctxs[i]);
     }
 
     if (mca_spml_ucx_ctx_default.ucp_worker) {
-        for (i = 0; i < (signed int)mca_spml_ucx.ucp_workers; i++) {
-            ucp_worker_destroy(mca_spml_ucx_ctx_default.ucp_worker[i]);
-        }
-        free(mca_spml_ucx_ctx_default.ucp_worker);
+        mca_spml_ucx_ctx_fini(&mca_spml_ucx_ctx_default);
     }
 
     if (mca_spml_ucx.aux_ctx != NULL) {
-        ucp_worker_destroy(mca_spml_ucx.aux_ctx->ucp_worker[0]);
-        free(mca_spml_ucx.aux_ctx->ucp_worker);
+        mca_spml_ucx_ctx_fini(mca_spml_ucx.aux_ctx);
     }
 
     mca_spml_ucx.enabled = false;  /* not anymore */
 
     free(mca_spml_ucx.active_array.ctxs);
     free(mca_spml_ucx.idle_array.ctxs);
-    free(mca_spml_ucx.aux_ctx);
 
     SHMEM_MUTEX_DESTROY(mca_spml_ucx.internal_mutex);
     pthread_mutex_destroy(&mca_spml_ucx.ctx_create_mutex);
diff --git a/oshmem/mca/spml/ucx/spml_ucx_component.h b/oshmem/mca/spml/ucx/spml_ucx_component.h
index 9c532ad4776..874dbc5c653 100644
--- a/oshmem/mca/spml/ucx/spml_ucx_component.h
+++ b/oshmem/mca/spml/ucx/spml_ucx_component.h
@@ -19,7 +19,7 @@ BEGIN_C_DECLS
 /*
  * SPML module functions.
  */
-OSHMEM_MODULE_DECLSPEC extern mca_spml_base_component_2_0_0_t mca_spml_ucx_component;
+OSHMEM_DECLSPEC extern mca_spml_base_component_2_0_0_t mca_spml_ucx_component;
 END_C_DECLS
 
 #endif
diff --git a/oshmem/mca/sshmem/base/Makefile.am b/oshmem/mca/sshmem/base/Makefile.am
index 05b4af1afd0..d3b7720874f 100644
--- a/oshmem/mca/sshmem/base/Makefile.am
+++ b/oshmem/mca/sshmem/base/Makefile.am
@@ -1,5 +1,6 @@
 # Copyright (c) 2014      Mellanox Technologies, Inc.
 #                         All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -7,7 +8,7 @@
 # $HEADER$
 #
 
-dist_oshmemdata_DATA = base/help-oshmem-sshmem.txt
+EXTRA_DIST = base/help-oshmem-sshmem.txt
 
 headers += \
         base/base.h
diff --git a/oshmem/mca/sshmem/base/sshmem_base_open.c b/oshmem/mca/sshmem/base/sshmem_base_open.c
index 8f14a37fcb8..2694120e1cf 100644
--- a/oshmem/mca/sshmem/base/sshmem_base_open.c
+++ b/oshmem/mca/sshmem/base/sshmem_base_open.c
@@ -31,15 +31,7 @@
  * globals
  */
 
-/**
- * if 32 bit we set sshmem_base_start_adress to 0
- * to let OS allocate segment automatically
- */
-#if UINTPTR_MAX == 0xFFFFFFFF
-void *mca_sshmem_base_start_address = (void*)0;
-#else
-void* mca_sshmem_base_start_address = (void*)0xFF000000;
-#endif
+void *mca_sshmem_base_start_address = (void *)UINTPTR_MAX;
 
 char * mca_sshmem_base_backing_file_dir = NULL;
 
@@ -57,7 +49,7 @@ mca_sshmem_base_register (mca_base_register_flag_t flags)
                                  "base",
                                  "start_address",
                                  "Specify base address for shared memory region",
-                                 MCA_BASE_VAR_TYPE_UNSIGNED_LONG_LONG,
+                                 MCA_BASE_VAR_TYPE_UNSIGNED_LONG,
                                  NULL,
                                  0,
                                  MCA_BASE_VAR_FLAG_SETTABLE,
diff --git a/oshmem/mca/sshmem/base/sshmem_base_wrappers.c b/oshmem/mca/sshmem/base/sshmem_base_wrappers.c
index 0bc60bfcce8..07761d2bb4b 100644
--- a/oshmem/mca/sshmem/base/sshmem_base_wrappers.c
+++ b/oshmem/mca/sshmem/base/sshmem_base_wrappers.c
@@ -85,5 +85,9 @@ shmem_ds_reset(map_segment_t *ds_buf)
     ds_buf->super.va_end = 0;
     ds_buf->seg_size = 0;
     ds_buf->type = MAP_SEGMENT_UNKNOWN;
+    ds_buf->mkeys_cache = NULL;
+    ds_buf->mkeys = NULL;
+    ds_buf->alloc_hints = 0;
+    ds_buf->allocator = NULL;
 }
 
diff --git a/oshmem/mca/sshmem/mmap/Makefile.am b/oshmem/mca/sshmem/mmap/Makefile.am
index 5529cdd7f08..2b3f7309bb5 100644
--- a/oshmem/mca/sshmem/mmap/Makefile.am
+++ b/oshmem/mca/sshmem/mmap/Makefile.am
@@ -1,6 +1,7 @@
 # Copyright (c) 2014      Mellanox Technologies, Inc.
 #                         All rights reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -8,7 +9,7 @@
 # $HEADER$
 #
 
-dist_oshmemdata_DATA = help-oshmem-sshmem-mmap.txt
+EXTRA_DIST = help-oshmem-sshmem-mmap.txt
 
 sources = \
         sshmem_mmap.h \
diff --git a/oshmem/mca/sshmem/mmap/sshmem_mmap.h b/oshmem/mca/sshmem/mmap/sshmem_mmap.h
index 1335af55a2e..778cd162d58 100644
--- a/oshmem/mca/sshmem/mmap/sshmem_mmap.h
+++ b/oshmem/mca/sshmem/mmap/sshmem_mmap.h
@@ -29,7 +29,7 @@ typedef struct mca_sshmem_mmap_component_t {
     int is_start_addr_fixed;
 } mca_sshmem_mmap_component_t;
 
-OSHMEM_MODULE_DECLSPEC extern mca_sshmem_mmap_component_t
+OSHMEM_DECLSPEC extern mca_sshmem_mmap_component_t
 mca_sshmem_mmap_component;
 
 typedef struct mca_sshmem_mmap_module_t {
diff --git a/oshmem/mca/sshmem/sysv/Makefile.am b/oshmem/mca/sshmem/sysv/Makefile.am
index dd677a8d3c6..71502d48d86 100644
--- a/oshmem/mca/sshmem/sysv/Makefile.am
+++ b/oshmem/mca/sshmem/sysv/Makefile.am
@@ -2,6 +2,7 @@
 #                         All rights reserved.
 # Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -9,7 +10,7 @@
 # $HEADER$
 #
 
-dist_oshmemdata_DATA = help-oshmem-sshmem-sysv.txt
+EXTRA_DIST = help-oshmem-sshmem-sysv.txt
 
 sources = \
         sshmem_sysv.h \
diff --git a/oshmem/mca/sshmem/sysv/sshmem_sysv.h b/oshmem/mca/sshmem/sysv/sshmem_sysv.h
index 7242b5a608f..d6e4a6ce5ab 100644
--- a/oshmem/mca/sshmem/sysv/sshmem_sysv.h
+++ b/oshmem/mca/sshmem/sysv/sshmem_sysv.h
@@ -30,7 +30,7 @@ typedef struct mca_sshmem_sysv_component_t {
     int use_hp;
 } mca_sshmem_sysv_component_t;
 
-OSHMEM_MODULE_DECLSPEC extern mca_sshmem_sysv_component_t
+OSHMEM_DECLSPEC extern mca_sshmem_sysv_component_t
 mca_sshmem_sysv_component;
 
 typedef struct mca_sshmem_sysv_module_t {
@@ -38,7 +38,7 @@ typedef struct mca_sshmem_sysv_module_t {
 } mca_sshmem_sysv_module_t;
 extern mca_sshmem_sysv_module_t mca_sshmem_sysv_module;
 
-OSHMEM_MODULE_DECLSPEC extern size_t sshmem_sysv_gethugepagesize(void);
+OSHMEM_DECLSPEC extern size_t sshmem_sysv_gethugepagesize(void);
 
 END_C_DECLS
 
diff --git a/oshmem/mca/sshmem/sysv/sshmem_sysv_module.c b/oshmem/mca/sshmem/sysv/sshmem_sysv_module.c
index c90c9976358..5dc6d429fff 100644
--- a/oshmem/mca/sshmem/sysv/sshmem_sysv_module.c
+++ b/oshmem/mca/sshmem/sysv/sshmem_sysv_module.c
@@ -170,6 +170,7 @@ segment_create(map_segment_t *ds_buf,
     }
 
     /* Attach to the segment */
+    (void)munmap(mca_sshmem_base_start_address, size);
     addr = shmat(shmid, (void *) mca_sshmem_base_start_address, 0);
     if (addr == (void *) -1L) {
         opal_show_help("help-oshmem-sshmem.txt",
@@ -259,7 +260,7 @@ segment_detach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
 
     if (mca_sshmem_sysv_component.use_hp != 0) {
         /**
-         *  Workaround kernel panic when detaching huge pages from user space simultanously from several processes
+         *  Workaround kernel panic when detaching huge pages from user space simultaneously from several processes
          *  dont detach here instead let kernel do it during process cleanup
          */
         /* shmdt((void *)ds_buf->seg_base_addr); */
diff --git a/oshmem/mca/sshmem/ucx/Makefile.am b/oshmem/mca/sshmem/ucx/Makefile.am
index ce37cd0e906..a7e93e51ec1 100644
--- a/oshmem/mca/sshmem/ucx/Makefile.am
+++ b/oshmem/mca/sshmem/ucx/Makefile.am
@@ -1,6 +1,7 @@
 # Copyright (c) 2014      Mellanox Technologies, Inc.
 #                         All rights reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -8,8 +9,6 @@
 # $HEADER$
 #
 
-#dist_oshmemdata_DATA = help-oshmem-sshmem-mmap.txt
-
 AM_CPPFLAGS = $(sshmem_ucx_CPPFLAGS)
 
 sources = \
@@ -40,4 +39,3 @@ mca_sshmem_ucx_la_LIBADD = $(top_builddir)/oshmem/liboshmem.la \
 noinst_LTLIBRARIES = $(component_noinst)
 libmca_sshmem_ucx_la_SOURCES =$(sources)
 libmca_sshmem_ucx_la_LDFLAGS = -module -avoid-version $(sshmem_ucx_LDFLAGS)
-
diff --git a/oshmem/mca/sshmem/ucx/configure.m4 b/oshmem/mca/sshmem/ucx/configure.m4
index c94bfaf7149..7bb9038c5d0 100644
--- a/oshmem/mca/sshmem/ucx/configure.m4
+++ b/oshmem/mca/sshmem/ucx/configure.m4
@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2017      Mellanox Technologies, Inc.
  *                         All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -27,43 +28,16 @@ AC_DEFUN([MCA_oshmem_sshmem_ucx_CONFIG],[
     save_LIBS="$LIBS"
     save_CPPFLAGS="$CPPFLAGS"
 
-    alloc_dm_LDFLAGS=" -L$ompi_check_ucx_libdir/ucx"
-    alloc_dm_LIBS=" -luct_ib"
     CPPFLAGS+=" $sshmem_ucx_CPPFLAGS"
-    LDFLAGS+=" $sshmem_ucx_LDFLAGS $alloc_dm_LDFLAGS"
-    LIBS+=" $sshmem_ucx_LIBS $alloc_dm_LIBS"
-
-    AC_LANG_PUSH([C])
-    AC_LINK_IFELSE([AC_LANG_PROGRAM(
-          [[
-            #include <ucp/core/ucp_resource.h>
-            #include <uct/ib/base/ib_alloc.h>
-          ]],
-          [[
-            uct_md_h md = ucp_context_find_tl_md((ucp_context_h)NULL, "");
-            (void)uct_ib_md_alloc_device_mem(md, NULL, NULL, 0, "", NULL);
-            uct_ib_md_release_device_mem(NULL);
-          ]])],
-          [
-           AC_MSG_NOTICE([UCX device memory allocation is supported])
-           AC_DEFINE([HAVE_UCX_DEVICE_MEM], [1], [Support for device memory allocation])
-           sshmem_ucx_LIBS+=" $alloc_dm_LIBS"
-           sshmem_ucx_LDFLAGS+=" $alloc_dm_LDFLAGS"
-          ],
-          [
-           AC_MSG_NOTICE([UCX device memory allocation is not supported])
-           AC_DEFINE([HAVE_UCX_DEVICE_MEM], [0], [Support for device memory allocation])
-          ])
-    AC_LANG_POP([C])
+    LDFLAGS+=" $sshmem_ucx_LDFLAGS"
+    LIBS+=" $sshmem_ucx_LIBS"
 
     CPPFLAGS="$save_CPPFLAGS"
     LDFLAGS="$save_LDFLAGS"
     LIBS="$save_LIBS"
 
     # substitute in the things needed to build ucx 
-    AC_SUBST([sshmem_ucx_CFLAGS])
     AC_SUBST([sshmem_ucx_CPPFLAGS])
     AC_SUBST([sshmem_ucx_LDFLAGS])
     AC_SUBST([sshmem_ucx_LIBS])
 ])dnl
-
diff --git a/oshmem/mca/sshmem/ucx/sshmem_ucx.h b/oshmem/mca/sshmem/ucx/sshmem_ucx.h
index fa264b40f42..90d41ac002c 100644
--- a/oshmem/mca/sshmem/ucx/sshmem_ucx.h
+++ b/oshmem/mca/sshmem/ucx/sshmem_ucx.h
@@ -31,11 +31,10 @@ typedef struct mca_sshmem_ucx_component_t {
     int priority;
 } mca_sshmem_ucx_component_t;
 
-OSHMEM_MODULE_DECLSPEC extern mca_sshmem_ucx_component_t
+OSHMEM_DECLSPEC extern mca_sshmem_ucx_component_t
 mca_sshmem_ucx_component;
 
 typedef struct mca_sshmem_ucx_segment_context {
-    void                           *dev_mem;
     sshmem_ucx_shadow_allocator_t  *shadow_allocator;
     ucp_mem_h                       ucp_memh;
 } mca_sshmem_ucx_segment_context_t;
diff --git a/oshmem/mca/sshmem/ucx/sshmem_ucx_module.c b/oshmem/mca/sshmem/ucx/sshmem_ucx_module.c
index fa38d0693a0..6333c904221 100644
--- a/oshmem/mca/sshmem/ucx/sshmem_ucx_module.c
+++ b/oshmem/mca/sshmem/ucx/sshmem_ucx_module.c
@@ -26,13 +26,6 @@
 
 #include "sshmem_ucx.h"
 
-//#include <ucs/sys/math.h>
-
-#if HAVE_UCX_DEVICE_MEM
-#include <ucp/core/ucp_resource.h>
-#include <uct/ib/base/ib_alloc.h>
-#endif
-
 #define ALLOC_ELEM_SIZE sizeof(uint64_t)
 #define min(a,b) ((a) < (b) ? (a) : (b))
 #define max(a,b) ((a) > (b) ? (a) : (b))
@@ -104,7 +97,7 @@ static segment_allocator_t sshmem_ucx_allocator = {
 
 static int
 segment_create_internal(map_segment_t *ds_buf, void *address, size_t size,
-                        unsigned flags, long hint, void *dev_mem)
+                        unsigned flags, ucs_memory_type_t mem_type, int err_level)
 {
     mca_sshmem_ucx_segment_context_t *ctx;
     int rc = OSHMEM_SUCCESS;
@@ -120,15 +113,20 @@ segment_create_internal(map_segment_t *ds_buf, void *address, size_t size,
 
     mem_map_params.field_mask = UCP_MEM_MAP_PARAM_FIELD_ADDRESS |
                                 UCP_MEM_MAP_PARAM_FIELD_LENGTH |
-                                UCP_MEM_MAP_PARAM_FIELD_FLAGS;
+                                UCP_MEM_MAP_PARAM_FIELD_FLAGS  |
+                                UCP_MEM_MAP_PARAM_FIELD_MEMORY_TYPE;
 
-    mem_map_params.address    = address;
-    mem_map_params.length     = size;
-    mem_map_params.flags      = flags;
+    mem_map_params.address     = address;
+    mem_map_params.length      = size;
+    mem_map_params.flags       = flags |
+        mca_spml_ucx_mem_map_flags_symmetric_rkey(spml);
+    mem_map_params.memory_type = mem_type;
 
     status = ucp_mem_map(spml->ucp_context, &mem_map_params, &mem_h);
     if (UCS_OK != status) {
-        SSHMEM_ERROR("ucp_mem_map() failed: %s\n", ucs_status_string(status));
+        SSHMEM_VERBOSE(err_level, "ucp_mem_map(memory_type=%s) failed: %s\n",
+                       ucs_memory_type_names[mem_type], 
+                       ucs_status_string(status));
         rc = OSHMEM_ERROR;
         goto out;
     }
@@ -161,12 +159,7 @@ segment_create_internal(map_segment_t *ds_buf, void *address, size_t size,
     ds_buf->super.va_end  = (void*)((uintptr_t)ds_buf->super.va_base + ds_buf->seg_size);
     ds_buf->context       = ctx;
     ds_buf->type          = MAP_SEGMENT_ALLOC_UCX;
-    ds_buf->alloc_hints   = hint;
     ctx->ucp_memh         = mem_h;
-    ctx->dev_mem          = dev_mem;
-    if (hint) {
-        ds_buf->allocator = &sshmem_ucx_allocator;
-    }
 
 out:
     OPAL_OUTPUT_VERBOSE(
@@ -181,82 +174,36 @@ segment_create_internal(map_segment_t *ds_buf, void *address, size_t size,
     return rc;
 }
 
-#if HAVE_UCX_DEVICE_MEM
-static uct_ib_device_mem_h alloc_device_mem(mca_spml_ucx_t *spml, size_t size,
-                                            void **address_p)
-{
-    uct_ib_device_mem_h dev_mem = NULL;
-    ucs_status_t status;
-    uct_md_h uct_md;
-    void *address;
-    size_t length;
-
-    uct_md = ucp_context_find_tl_md(spml->ucp_context, "mlx5");
-    if (uct_md == NULL) {
-        SSHMEM_VERBOSE(1, "ucp_context_find_tl_md() returned NULL\n");
-        return NULL;
-    }
-
-    /* If found a matching memory domain, allocate device memory on it */
-    length  = size;
-    address = NULL;
-    status = uct_ib_md_alloc_device_mem(uct_md, &length, &address,
-                                        UCT_MD_MEM_ACCESS_ALL, "sshmem_seg",
-                                        &dev_mem);
-    if (status != UCS_OK) {
-        /* If could not allocate device memory - fallback to mmap (since some
-         * PEs in the job may succeed and while others failed */
-        SSHMEM_VERBOSE(1, "uct_ib_md_alloc_dm() failed: %s\n",
-                       ucs_status_string(status));
-        return NULL;
-    }
-
-    SSHMEM_VERBOSE(3, "uct_ib_md_alloc_dm() returned address %p\n", address);
-    *address_p = address;
-    return dev_mem;
-}
-#endif
-
 static int
 segment_create(map_segment_t *ds_buf,
                const char *file_name,
                size_t size, long hint)
 {
     mca_spml_ucx_t *spml = (mca_spml_ucx_t*)mca_spml.self;
-    unsigned flags;
+    unsigned flags       = UCP_MEM_MAP_ALLOCATE;
 
-#if HAVE_UCX_DEVICE_MEM
-    int ret = OSHMEM_ERROR;
     if (hint & SHMEM_HINT_DEVICE_NIC_MEM) {
-        if (size > UINT_MAX) {
-            return OSHMEM_ERR_BAD_PARAM;
+#if HAVE_DECL_UCS_MEMORY_TYPE_RDMA
+        int status = segment_create_internal(ds_buf, NULL, size,
+                                             flags, UCS_MEMORY_TYPE_RDMA, 3);
+        if (status == OSHMEM_SUCCESS) {
+            ds_buf->alloc_hints = hint;
+            ds_buf->allocator   = &sshmem_ucx_allocator;
+            return OSHMEM_SUCCESS;
         }
-
-        void *dev_mem_address;
-        uct_ib_device_mem_h dev_mem = alloc_device_mem(spml, size,
-                                                       &dev_mem_address);
-        if (dev_mem != NULL) {
-            int ret;
-            ret = segment_create_internal(ds_buf, dev_mem_address, size, 0,
-                                          hint, dev_mem);
-            if (ret == OSHMEM_SUCCESS) {
-                return OSHMEM_SUCCESS;
-            } else if (dev_mem != NULL) {
-                uct_ib_md_release_device_mem(dev_mem);
-                /* fallback to regular allocation */
-            }
-        }
-    }
+#else 
+       SSHMEM_VERBOSE(3, "DEVICE_NIC_MEM hint ignored since UCX does not "
+                      "support MEMORY_TYPE_RDMA");
 #endif
+        return OSHMEM_ERR_NOT_IMPLEMENTED;
+    }
 
-    flags = UCP_MEM_MAP_ALLOCATE | (spml->heap_reg_nb ? UCP_MEM_MAP_NONBLOCK : 0);
-    if (hint) {
-        return segment_create_internal(ds_buf, NULL, size, flags, hint, NULL);
-    } else {
-        return segment_create_internal(ds_buf, mca_sshmem_base_start_address,
-                                       size, flags | UCP_MEM_MAP_FIXED, hint,
-                                       NULL);
+    flags |= UCP_MEM_MAP_FIXED;
+    if (spml->heap_reg_nb) {
+        flags |= UCP_MEM_MAP_NONBLOCK;
     }
+    return segment_create_internal(ds_buf, mca_sshmem_base_start_address, size,
+                                   flags, UCS_MEMORY_TYPE_HOST, 0);
 }
 
 static void *
@@ -303,12 +250,6 @@ segment_unlink(map_segment_t *ds_buf)
 
     ucp_mem_unmap(spml->ucp_context, ctx->ucp_memh);
 
-#if HAVE_UCX_DEVICE_MEM
-    if (ctx->dev_mem) {
-        uct_ib_md_release_device_mem(ctx->dev_mem);
-    }
-#endif
-
     ds_buf->context = NULL;
     free(ctx);
 
diff --git a/oshmem/proc/proc.c b/oshmem/proc/proc.c
index 6e3446c9abe..da6fdeaeb99 100644
--- a/oshmem/proc/proc.c
+++ b/oshmem/proc/proc.c
@@ -31,13 +31,53 @@
 
 
 static opal_mutex_t oshmem_proc_lock;
+static opal_bitmap_t _oshmem_local_vpids;       /* Track the vpids in local node */
+int oshmem_proc_init_set_local_vpids()
+{
+    opal_process_name_t wildcard_rank;
+    int ret = OMPI_SUCCESS;
+    char *val = NULL;
+    
+    ret = opal_bitmap_init(&_oshmem_local_vpids, ompi_comm_size(oshmem_comm_world));
+    if (OSHMEM_SUCCESS != ret) {
+        return ret;
+    }
+    /* Add all local peers first */
+    wildcard_rank.jobid = OMPI_PROC_MY_NAME->jobid;
+    wildcard_rank.vpid = OMPI_NAME_WILDCARD->vpid;
+    /* retrieve the local peers */
+    OPAL_MODEX_RECV_VALUE(ret, PMIX_LOCAL_PEERS,
+                          &wildcard_rank, &val, PMIX_STRING);
+
+    if (OPAL_SUCCESS == ret && NULL != val) {
+        char **peers = opal_argv_split(val, ',');
+        int i;
+        free(val);
+        for (i=0; NULL != peers[i]; i++) {
+            ompi_vpid_t local_rank = strtoul(peers[i], NULL, 10);
+            opal_bitmap_set_bit(&_oshmem_local_vpids, local_rank);
+        }
+        opal_argv_free(peers);
+    }
+    return OSHMEM_SUCCESS;
+}
+
+bool oshmem_proc_on_local_node(int pe)
+{
+    return opal_bitmap_is_set_bit(&_oshmem_local_vpids, pe);
+}
 
 int oshmem_proc_init(void)
 {
+    int ret;
     OBJ_CONSTRUCT(&oshmem_proc_lock, opal_mutex_t);
+    OBJ_CONSTRUCT(&_oshmem_local_vpids, opal_bitmap_t);
+
+    ret = oshmem_proc_init_set_local_vpids();
+    if(OSHMEM_SUCCESS != ret) {
+        return ret;
+    }
 
-    /* check oshmem_proc_data_t can fit within ompi_proc_t padding */
-    assert(sizeof(oshmem_proc_data_t) <= OMPI_PROC_PADDING_SIZE);
     /* check ompi_proc_t padding is aligned on a pointer */
     assert(0 == (offsetof(ompi_proc_t, padding) & (sizeof(char *)-1)));
 
@@ -139,6 +179,7 @@ int oshmem_proc_group_finalize(void)
         }
     }
 
+    OBJ_DESTRUCT(&_oshmem_local_vpids);
     OBJ_DESTRUCT(&oshmem_group_array);
 
     oshmem_group_cache_destroy();
@@ -150,8 +191,6 @@ oshmem_group_t* oshmem_proc_group_create(int pe_start, int pe_stride, int pe_siz
     int cur_pe, count_pe;
     int i;
     oshmem_group_t* group = NULL;
-    ompi_proc_t** proc_array = NULL;
-    ompi_proc_t* proc = NULL;
 
     assert(oshmem_proc_local());
 
@@ -171,52 +210,29 @@ oshmem_group_t* oshmem_proc_group_create(int pe_start, int pe_stride, int pe_siz
     OPAL_THREAD_LOCK(&oshmem_proc_lock);
 
     /* allocate an array */
-    proc_array = (ompi_proc_t**) malloc(pe_size * sizeof(ompi_proc_t*));
-    if (NULL == proc_array) {
-        OBJ_RELEASE(group);
+    group->proc_vpids = (opal_vpid_t *) malloc(pe_size * sizeof(group->proc_vpids[0]));
+    if (NULL == group->proc_vpids) {
         OPAL_THREAD_UNLOCK(&oshmem_proc_lock);
-        return NULL ;
+        return NULL;
     }
 
     group->my_pe = oshmem_proc_pe(oshmem_proc_local());
     group->is_member = 0;
     for (i = 0 ; i < ompi_comm_size(oshmem_comm_world) ; i++) {
-        proc = oshmem_proc_find(i);
-        if (NULL == proc) {
-            opal_output(0,
-                    "Error: Can not find proc object for pe = %d", i);
-            free(proc_array);
-            OBJ_RELEASE(group);
-            OPAL_THREAD_UNLOCK(&oshmem_proc_lock);
-            return NULL;
-        }
         if (count_pe >= (int) pe_size) {
             break;
         } else if ((cur_pe >= pe_start)
                 && ((pe_stride == 0)
                     || (((cur_pe - pe_start) % pe_stride) == 0))) {
-            proc_array[count_pe++] = proc;
-            if (oshmem_proc_pe(proc) == group->my_pe)
+            group->proc_vpids[count_pe] = i;
+            count_pe ++;
+            if (i == group->my_pe)
                 group->is_member = 1;
         }
         cur_pe++;
     }
-    group->proc_array = proc_array;
     group->proc_count = (int) count_pe;
     group->ompi_comm = NULL;
-
-    /* Prepare peers list */
-    OBJ_CONSTRUCT(&(group->peer_list), opal_list_t);
-    {
-        opal_namelist_t *peer = NULL;
-
-        for (i = 0; i < group->proc_count; i++) {
-            peer = OBJ_NEW(opal_namelist_t);
-            peer->name.jobid = OSHMEM_PROC_JOBID(group->proc_array[i]);
-            peer->name.vpid = OSHMEM_PROC_VPID(group->proc_array[i]);
-            opal_list_append(&(group->peer_list), &peer->super);
-        }
-    }
     group->id = opal_pointer_array_add(&oshmem_group_array, group);
 
     memset(&group->g_scoll, 0, sizeof(mca_scoll_base_group_scoll_t));
@@ -251,20 +267,8 @@ oshmem_proc_group_destroy_internal(oshmem_group_t* group, int scoll_unselect)
         mca_scoll_base_group_unselect(group);
     }
 
-    /* Destroy proc array */
-    if (group->proc_array) {
-        free(group->proc_array);
-    }
-
-    /* Destroy peer list */
-    {
-        opal_list_item_t *item;
-
-        while (NULL != (item = opal_list_remove_first(&(group->peer_list)))) {
-            /* destruct the item (we constructed it), then free the memory chunk */
-            OBJ_RELEASE(item);
-        }
-        OBJ_DESTRUCT(&(group->peer_list));
+    if (group->proc_vpids) {
+        free(group->proc_vpids);
     }
 
     /* reset the oshmem_group_array entry - make sure that the
diff --git a/oshmem/proc/proc.h b/oshmem/proc/proc.h
index 0660b691231..95e7ff60e06 100644
--- a/oshmem/proc/proc.h
+++ b/oshmem/proc/proc.h
@@ -19,7 +19,9 @@
 #include "oshmem/constants.h"
 
 #include "opal/class/opal_list.h"
+#include "opal/class/opal_bitmap.h"
 #include "opal/util/proc.h"
+#include "opal/util/argv.h"
 #include "opal/mca/hwloc/hwloc-internal.h"
 
 #include "ompi/proc/proc.h"
@@ -37,19 +39,6 @@ struct oshmem_group_t;
 
 #define OSHMEM_PE_INVALID   (-1)
 
-/* This struct will be copied into the padding field of an ompi_proc_t
- * so the size of oshmem_proc_data_t must be less or equal than
- * OMPI_PROC_PADDING_SIZE */
-struct oshmem_proc_data_t {
-    char * transport_ids;
-    int num_transports;
-};
-
-typedef struct oshmem_proc_data_t oshmem_proc_data_t;
-
-#define OSHMEM_PROC_DATA(proc) \
-    ((oshmem_proc_data_t *)(proc)->padding)
-
 /**
  * Group of Open SHMEM processes structure
  *
@@ -61,9 +50,7 @@ struct oshmem_group_t {
     int                         my_pe;
     int                         proc_count;     /**< number of processes in group */
     int                         is_member;   /* true if my_pe is part of the group, participate in collectives */
-    struct ompi_proc_t          **proc_array; /**< list of pointers to ompi_proc_t structures
-                                                   for each process in the group */
-    opal_list_t                 peer_list;
+    opal_vpid_t                 *proc_vpids; /* vpids of each process in group */
 
     /* Collectives module interface and data */
     mca_scoll_base_group_scoll_t g_scoll;
@@ -149,14 +136,21 @@ static inline ompi_proc_t *oshmem_proc_find(int pe)
     return oshmem_proc_for_find(name);
 }
 
+static inline int oshmem_proc_pe_vpid(oshmem_group_t *group, int pe)
+{
+    if (OPAL_LIKELY(pe < group->proc_count)) {
+        return (group->proc_vpids[pe]);
+    } else {
+        return -1;
+    }
+}
+
 static inline int oshmem_proc_pe(ompi_proc_t *proc)
 {
     return (proc ? (int) ((ompi_process_name_t*)&proc->super.proc_name)->vpid : -1);
 }
 
-#define OSHMEM_PROC_JOBID(PROC)    (((ompi_process_name_t*)&((PROC)->super.proc_name))->jobid)
-#define OSHMEM_PROC_VPID(PROC)     (((ompi_process_name_t*)&((PROC)->super.proc_name))->vpid)
-
+bool oshmem_proc_on_local_node(int pe);
 /**
  * Initialize the OSHMEM process predefined groups
  *
@@ -232,40 +226,6 @@ oshmem_proc_group_create_nofail(int pe_start, int pe_stride, int pe_size)
  */
 OSHMEM_DECLSPEC void oshmem_proc_group_destroy(oshmem_group_t* group);
 
-static inline ompi_proc_t *oshmem_proc_group_all(int pe)
-{
-    return oshmem_group_all->proc_array[pe];
-}
-
-static inline ompi_proc_t *oshmem_proc_group_find(oshmem_group_t* group,
-                                                    int pe)
-{
-    int i = 0;
-    ompi_proc_t* proc = NULL;
-
-    if (OPAL_LIKELY(group)) {
-        if (OPAL_LIKELY(group == oshmem_group_all)) {
-            /* To improve performance use direct index. It is feature of oshmem_group_all */
-            proc = group->proc_array[pe];
-        } else {
-            for (i = 0; i < group->proc_count; i++) {
-                if (pe == oshmem_proc_pe(group->proc_array[i])) {
-                    proc = group->proc_array[i];
-                    break;
-                }
-            }
-        }
-    } else {
-        ompi_process_name_t name;
-
-        name.jobid = OMPI_PROC_MY_NAME->jobid;
-        name.vpid = pe;
-        proc = oshmem_proc_for_find(name);
-    }
-
-    return proc;
-}
-
 static inline int oshmem_proc_group_find_id(oshmem_group_t* group, int pe)
 {
     int i = 0;
@@ -273,7 +233,7 @@ static inline int oshmem_proc_group_find_id(oshmem_group_t* group, int pe)
 
     if (group) {
         for (i = 0; i < group->proc_count; i++) {
-            if (pe == oshmem_proc_pe(group->proc_array[i])) {
+            if (pe == oshmem_proc_pe_vpid(group, i)) {
                 id = i;
                 break;
             }
@@ -299,22 +259,6 @@ static inline int oshmem_my_proc_id(void)
     return oshmem_group_self->my_pe;
 }
 
-static inline int oshmem_get_transport_id(int pe)
-{
-    ompi_proc_t *proc;
-
-    proc = oshmem_proc_group_find(oshmem_group_all, pe);
-
-    return (int) OSHMEM_PROC_DATA(proc)->transport_ids[0];
-}
-
-static inline int oshmem_get_transport_count(int pe)
-{
-    ompi_proc_t *proc;
-    proc = oshmem_proc_group_find(oshmem_group_all, pe);
-    return OSHMEM_PROC_DATA(proc)->num_transports;
-}
-
 END_C_DECLS
 
 #endif /* OSHMEM_PROC_PROC_H */
diff --git a/oshmem/request/request.h b/oshmem/request/request.h
index 97c6c2630ee..47e84ba975d 100644
--- a/oshmem/request/request.h
+++ b/oshmem/request/request.h
@@ -83,7 +83,7 @@ typedef int (*oshmem_request_cancel_fn_t)(struct oshmem_request_t* request,
 /*
  * Optional function called when the request is completed from the SHMEM
  * library perspective. This function is not allowed to release any
- * ressources related to the request.
+ * resources related to the request.
  */
 typedef int (*oshmem_request_complete_fn_t)(struct oshmem_request_t* request);
 
@@ -133,7 +133,7 @@ struct oshmem_request_t {
 typedef struct oshmem_request_t oshmem_request_t;
 
 /**
- * Padded struct to maintain back compatibiltiy.
+ * Padded struct to maintain back compatibility.
  * See oshmem/communicator/communicator.h comments with struct oshmem_group_t
  * for full explanation why we chose the following padding construct for predefines.
  */
@@ -172,7 +172,7 @@ typedef struct oshmem_predefined_request_t oshmem_predefined_request_t;
  * This function should be called only from the SHMEM layer. It should
  * never be called from the SPML. It take care of the upper level clean-up.
  * When the user call MPI_Request_free we should release all SHMEM level
- * ressources, so we have to call this function too.
+ * resources, so we have to call this function too.
  */
 #define OSHMEM_REQUEST_FINI(request)                                      \
 do {                                                                    \
@@ -221,7 +221,7 @@ typedef int (*oshmem_request_test_any_fn_t)(size_t count,
  *
  * @param count (IN)      Number of requests
  * @param requests (IN)   Array of requests
- * @param completed (OUT) Flag indicating wether all requests completed.
+ * @param completed (OUT) Flag indicating whether all requests completed.
  * @param statuses (OUT)  Array of completion statuses.
  * @return                OSHMEM_SUCCESS or failure status.
  *
diff --git a/oshmem/runtime/Makefile.am b/oshmem/runtime/Makefile.am
index 301b1e01ae5..f3be80a7d16 100644
--- a/oshmem/runtime/Makefile.am
+++ b/oshmem/runtime/Makefile.am
@@ -1,5 +1,6 @@
 # Copyright (c) 2013      Mellanox Technologies, Inc.
 #                         All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -9,9 +10,7 @@
 
 # This makefile.am does not stand on its own - it is included from oshmem/Makefile.am
 
-if PROJECT_OSHMEM
-dist_oshmemdata_DATA += runtime/help-shmem-runtime.txt
-endif
+EXTRA_DIST += runtime/help-shmem-runtime.txt
 
 headers += \
 	runtime/runtime.h \
@@ -26,4 +25,3 @@ liboshmem_la_SOURCES += \
 	runtime/oshmem_shmem_params.c \
 	runtime/oshmem_shmem_exchange.c \
 	runtime/oshmem_info_support.c
-
diff --git a/oshmem/runtime/oshmem_shmem_exchange.c b/oshmem/runtime/oshmem_shmem_exchange.c
index 730eaef2e46..147340e53c4 100644
--- a/oshmem/runtime/oshmem_shmem_exchange.c
+++ b/oshmem/runtime/oshmem_shmem_exchange.c
@@ -16,6 +16,15 @@
 #include "oshmem/runtime/runtime.h"
 #include "oshmem/runtime/params.h"
 
+int oshmem_shmem_bcast(void *buf, int elem_size, int root)
+{
+    int rc;
+
+    rc = PMPI_Bcast(buf, elem_size, MPI_BYTE, root, oshmem_comm_world);
+
+    return rc;
+}
+
 int oshmem_shmem_allgather(void *send_buf, void *rcv_buf, int elem_size)
 {
     int rc;
diff --git a/oshmem/runtime/oshmem_shmem_finalize.c b/oshmem/runtime/oshmem_shmem_finalize.c
index 7e979b8a62e..1950e20fdd1 100644
--- a/oshmem/runtime/oshmem_shmem_finalize.c
+++ b/oshmem/runtime/oshmem_shmem_finalize.c
@@ -53,7 +53,7 @@
 #include "oshmem/shmem/shmem_lock.h"
 #include "oshmem/runtime/oshmem_shmem_preconnect.h"
 
-extern int oshmem_shmem_globalexit_status;
+extern int oshmem_shmem_inglobalexit;
 
 static int _shmem_finalize(void);
 
@@ -79,7 +79,7 @@ int oshmem_shmem_finalize(void)
     if ((OSHMEM_SUCCESS == ret) &&
         (state >= OMPI_MPI_STATE_INIT_COMPLETED &&
          state < OMPI_MPI_STATE_FINALIZE_PAST_COMM_SELF_DESTRUCT) &&
-        oshmem_shmem_globalexit_status == 0) {
+         (oshmem_shmem_inglobalexit == 0)) {
         PMPI_Comm_free(&oshmem_comm_world);
         ret = ompi_mpi_finalize();
     }
@@ -126,7 +126,7 @@ static int _shmem_finalize(void)
 
     if (OSHMEM_SUCCESS
             != (ret =
-                    MCA_SPML_CALL(del_procs(oshmem_group_all->proc_array, oshmem_group_all->proc_count)))) {
+                    MCA_SPML_CALL(del_procs(oshmem_group_all, oshmem_group_all->proc_count)))) {
         return ret;
     }
 
diff --git a/oshmem/runtime/oshmem_shmem_init.c b/oshmem/runtime/oshmem_shmem_init.c
index 355d0995b05..64af4e18f3e 100644
--- a/oshmem/runtime/oshmem_shmem_init.c
+++ b/oshmem/runtime/oshmem_shmem_init.c
@@ -65,12 +65,6 @@
 #include <sys/mman.h>
 #endif
 
-#if OPAL_CC_USE_PRAGMA_IDENT
-#pragma ident OMPI_IDENT_STRING
-#elif OPAL_CC_USE_IDENT
-#ident OSHMEM_IDENT_STRING
-#endif
-
 /*
  * WHAT: add thread for invoking opal_progress() function
  * WHY:  SHMEM based on current ompi/trunk (by the time of integrating into Open MPI)
@@ -101,6 +95,9 @@ shmem_internal_mutex_t shmem_internal_mutex_alloc = {{0}};
 
 shmem_ctx_t oshmem_ctx_default = NULL;
 
+shmem_team_t oshmem_team_shared = NULL;
+shmem_team_t oshmem_team_world  = NULL;
+
 static int _shmem_init(int argc, char **argv, int requested, int *provided);
 
 #if OSHMEM_OPAL_THREAD_ENABLE
@@ -108,8 +105,8 @@ static void* shmem_opal_thread(void* argc)
 {
 /*
  * WHAT: sleep() invocation
- * WHY:  there occures a segfault sometimes and sleep()
- *       reduces it's possibility
+ * WHY:  there occurs a segfault sometimes and sleep()
+ *       reduces its possibility
  */
     sleep(1);
     while(oshmem_shmem_initialized)
@@ -347,7 +344,7 @@ static int _shmem_init(int argc, char **argv, int requested, int *provided)
     OPAL_TIMING_ENV_NEXT(timing, "MCA_SPML_CALL(enable())");
 
     ret =
-            MCA_SPML_CALL(add_procs(oshmem_group_all->proc_array, oshmem_group_all->proc_count));
+            MCA_SPML_CALL(add_procs(oshmem_group_all, oshmem_group_all->proc_count));
     if (OSHMEM_SUCCESS != ret) {
         error = "SPML add procs failed";
         goto error;
diff --git a/oshmem/runtime/oshmem_shmem_params.c b/oshmem/runtime/oshmem_shmem_params.c
index 24035be24ee..3d68fcb927a 100644
--- a/oshmem/runtime/oshmem_shmem_params.c
+++ b/oshmem/runtime/oshmem_shmem_params.c
@@ -17,9 +17,10 @@
 #include "oshmem/constants.h"
 
 
-int oshmem_shmem_lock_recursive = 0;
-int oshmem_shmem_api_verbose = 0;
-int oshmem_preconnect_all = 0;
+int oshmem_shmem_lock_recursive   = 0;
+int oshmem_shmem_api_verbose      = 0;
+int oshmem_shmem_enable_mcs_locks = 1;
+int oshmem_preconnect_all         = 0;
 
 int oshmem_shmem_register_params(void)
 {
@@ -38,6 +39,19 @@ int oshmem_shmem_register_params(void)
                                  MCA_BASE_VAR_SCOPE_READONLY,
                                  &oshmem_shmem_lock_recursive);
 
+    (void) mca_base_var_register("oshmem",
+                                 "oshmem",
+                                 NULL,
+                                 "enable_mcs_lock",
+                                 "enable mcs locks",
+                                 MCA_BASE_VAR_TYPE_INT,
+                                 NULL,
+                                 1,
+                                 MCA_BASE_VAR_FLAG_SETTABLE,
+                                 OPAL_INFO_LVL_9,
+                                 MCA_BASE_VAR_SCOPE_READONLY,
+                                 &oshmem_shmem_enable_mcs_locks);
+
     (void) mca_base_var_register("oshmem",
                                  "oshmem",
                                  NULL,
diff --git a/oshmem/runtime/params.h b/oshmem/runtime/params.h
index e1a2d8cf1d8..979b1125d08 100644
--- a/oshmem/runtime/params.h
+++ b/oshmem/runtime/params.h
@@ -37,6 +37,13 @@ OSHMEM_DECLSPEC extern int oshmem_shmem_api_verbose;
  */
 OSHMEM_DECLSPEC extern int oshmem_preconnect_all;
 
+
+/**
+ * Whether to force SHMEM processes to use MCS locking
+ * for shmem_locks
+ */
+OSHMEM_DECLSPEC extern int oshmem_shmem_enable_mcs_locks;
+
 END_C_DECLS
 
 #endif /* OSHMEM_RUNTIME_PARAMS_H */
diff --git a/oshmem/runtime/runtime.h b/oshmem/runtime/runtime.h
index 7f83fdc3e0d..02d60a8c137 100644
--- a/oshmem/runtime/runtime.h
+++ b/oshmem/runtime/runtime.h
@@ -82,7 +82,7 @@ OSHMEM_DECLSPEC extern shmem_ctx_t oshmem_ctx_default;
 #define OSHMEM_THREADLEVEL_SERIALIZED_BF 0x00000004
 #define OSHMEM_THREADLEVEL_MULTIPLE_BF   0x00000008
 
-/** In ompi_mpi_init: the lists of Fortran 90 mathing datatypes.
+/** In ompi_mpi_init: the lists of Fortran 90 matching datatypes.
  * We need these lists and hashtables in order to satisfy the new
  * requirements introduced in MPI 2-1 Sect. 10.2.5,
  * MPI_TYPE_CREATE_F90_xxxx, page 295, line 47.
@@ -105,12 +105,12 @@ OSHMEM_DECLSPEC extern const char oshmem_version_string[];
  * @returns OSHMEM_SUCCESS if successful
  * @returns Error code if unsuccessful
  *
- * Intialize all support code needed for SHMEM applications.  This
+ * Initialize all support code needed for SHMEM applications.  This
  * function should only be called by SHMEM applications (including
  * singletons).  If this function is called, ompi_init() and
  * ompi_rte_init() should *not* be called.
  *
- * It is permissable to pass in (0, NULL) for (argc, argv).
+ * It is permissible to pass in (0, NULL) for (argc, argv).
  */
 int oshmem_shmem_init(int argc, char **argv, int requested, int *provided);
 
@@ -128,6 +128,11 @@ int oshmem_shmem_finalize(void);
  */
 OSHMEM_DECLSPEC int oshmem_shmem_abort(int errcode);
 
+/**
+ * Broadcast between all PEs
+ */
+OSHMEM_DECLSPEC int oshmem_shmem_bcast(void *buf, int elem_size, int root);
+
 /**
  * Allgather between all PEs
  */
@@ -151,6 +156,8 @@ OSHMEM_DECLSPEC int oshmem_shmem_register_params(void);
 
 #if OSHMEM_PARAM_CHECK == 1
 
+
+
 #define RUNTIME_CHECK_ERROR(...)                                    \
     do {                                                            \
         fprintf(stderr, "[%s]%s[%s:%d:%s] ",                        \
@@ -160,6 +167,15 @@ OSHMEM_DECLSPEC int oshmem_shmem_register_params(void);
         fprintf(stderr, __VA_ARGS__);                               \
     } while(0);
 
+/* check if this routine is implemented. Can be used for routines that do
+ * not return error code. */
+#define RUNTIME_CHECK_IMPL_RC(x)                                            \
+    if (x <= -1)                                                            \
+    {                                                                       \
+        int _rc = x;                                                        \
+        RUNTIME_CHECK_ERROR("Internal error is appeared rc = %d\n", (_rc)); \
+    }
+
 /**
  * Check if SHMEM API generates internal error return code
  * Note: most API does not return error code
@@ -222,9 +238,22 @@ OSHMEM_DECLSPEC int oshmem_shmem_register_params(void);
 #define RUNTIME_CHECK_ADDR(x)
 #define RUNTIME_CHECK_ADDR_SIZE(x,s)
 #define RUNTIME_CHECK_WITH_MEMHEAP_SIZE(x)
+#define RUNTIME_CHECK_IMPL_RC(x)
 
 #endif  /* OSHMEM_PARAM_CHECK */
 
+#define RUNTIME_SHMEM_NOT_IMPLEMENTED_API_ABORT()                    \
+    do {                                                             \
+        SHMEM_API_ERROR("Called non-implemented API: %s", __func__); \
+        oshmem_shmem_abort(OSHMEM_ERR_NOT_IMPLEMENTED);              \
+    } while (0)
+
+#define RUNTIME_SHMEM_NOT_IMPLEMENTED_API_ABORT_RET_SIZE_T() \
+    do {                                                     \
+        RUNTIME_SHMEM_NOT_IMPLEMENTED_API_ABORT();           \
+        return SIZE_MAX;                                     \
+    } while (0)
+
 END_C_DECLS
 
 #endif /* OSHMEM_SHMEM_RUNTIME_H */
diff --git a/oshmem/shmem/Makefile.am b/oshmem/shmem/Makefile.am
index bf219790643..fab81bddd52 100644
--- a/oshmem/shmem/Makefile.am
+++ b/oshmem/shmem/Makefile.am
@@ -1,5 +1,6 @@
 # Copyright (c) 2013-2015 Mellanox Technologies, Inc.
 #                         All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -7,11 +8,7 @@
 # $HEADER$
 #
 
-EXTRA_DIST =
-
 headers += shmem/shmem_api_logger.h \
            shmem/shmem_lock.h
 
-if PROJECT_OSHMEM
-dist_oshmemdata_DATA += shmem/help-shmem-api.txt
-endif
+EXTRA_DIST += shmem/help-shmem-api.txt
diff --git a/oshmem/shmem/c/Makefile.am b/oshmem/shmem/c/Makefile.am
index 3acd7e400d0..194de248008 100644
--- a/oshmem/shmem/c/Makefile.am
+++ b/oshmem/shmem/c/Makefile.am
@@ -2,6 +2,8 @@
 # Copyright (c) 2013-2016 Mellanox Technologies, Inc.
 #                         All rights reserved
 # Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2021      Amazon.com, Inc. or its affiliates.
+#                         All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -10,12 +12,9 @@
 #
 
 
-if OSHMEM_PROFILING
-    SUBDIRS = profile
-endif
-
 OSHMEM_AUX_SOURCES = \
-	shmem_lock.c
+	shmem_lock.c \
+	shmem_mcs_lock.c
 
 OSHMEM_API_SOURCES = \
 	shmem_init.c \
@@ -70,23 +69,38 @@ OSHMEM_API_SOURCES = \
 	shmem_set_lock.c \
 	shmem_test_lock.c \
 	shmem_global_exit.c \
+	shmem_cswap_nb.c  \
+	shmem_fadd_nb.c \
+	shmem_fand_nb.c \
+	shmem_fetch_nb.c \
+	shmem_finc_nb.c \
+	shmem_for_nb.c \
+	shmem_fxor_nb.c \
+	shmem_pcontrol.c \
+	shmem_put_signal.c \
+	shmem_put_signal_nb.c \
+	shmem_swap_nb.c \
+	shmem_team.c \
+	shmem_wait_ivars.c \
+	shmem_test_ivars.c \
 	shmem_info.c
 
-AM_CPPFLAGS     = -DOSHMEM_PROFILING=0
-
 noinst_LTLIBRARIES =
 if PROJECT_OSHMEM
 # Only build if we're building OSHMEM
 noinst_LTLIBRARIES += liboshmem_c.la
 endif
 
-headers =
+headers = profile-defines.h
 
 # In case when OSHMEM_PROFILING enabled
 # We build api from c/profile folder
-liboshmem_c_la_SOURCES = $(OSHMEM_AUX_SOURCES)
-if ! OSHMEM_PROFILING
-liboshmem_c_la_SOURCES += $(OSHMEM_API_SOURCES)
+liboshmem_c_la_SOURCES = $(OSHMEM_AUX_SOURCES) \
+	$(OSHMEM_API_SOURCES)
+if OSHMEM_PROFILING
+liboshmem_c_la_CPPFLAGS = -DOSHMEM_PROFILING=1
+else
+liboshmem_c_la_CPPFLAGS = -DOSHMEM_PROFILING=0
 endif
 
 if PROJECT_OSHMEM
diff --git a/oshmem/shmem/c/profile-defines.h b/oshmem/shmem/c/profile-defines.h
new file mode 100644
index 00000000000..e6386f1cc73
--- /dev/null
+++ b/oshmem/shmem/c/profile-defines.h
@@ -0,0 +1,2084 @@
+/*
+ * Copyright (c) 2013-2017 Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef OSHMEM_C_PROFILE_DEFINES_H
+#define OSHMEM_C_PROFILE_DEFINES_H
+/*
+ * This file is included in the top directory only if
+ * profiling is required. Once profiling is required,
+ * this file will replace all shmem_* symbols with
+ * pshmem_* symbols
+ */
+
+/*
+ * Initialization routines
+ */
+#define shmem_init                   pshmem_init
+#define shmem_init_thread            pshmem_init_thread
+#define start_pes                    pstart_pes /* shmem-compat.h */
+
+/*
+ * Finalization routines
+ */
+#define shmem_finalize               pshmem_finalize
+#define shmem_global_exit            pshmem_global_exit
+
+/*
+ * Query routines
+ */
+#define shmem_n_pes                  pshmem_n_pes
+#define shmem_query_thread           pshmem_query_thread
+#define shmem_my_pe                  pshmem_my_pe
+#define _num_pes                     p_num_pes /* shmem-compat.h */
+#define _my_pe                       p_my_pe /* shmem-compat.h */
+
+/*
+ * Accessibility routines
+ */
+#define shmem_pe_accessible          pshmem_pe_accessible
+#define shmem_addr_accessible        pshmem_addr_accessible
+
+/*
+ * Symmetric heap routines
+ */
+#define shmem_malloc                 pshmem_malloc
+#define shmem_calloc                 pshmem_calloc
+#define shmem_align                  pshmem_align
+#define shmem_realloc                pshmem_realloc
+#define shmem_malloc_with_hints      pshmem_malloc_with_hints
+#define shmem_free                   pshmem_free
+#define shmalloc                     pshmalloc /* shmem-compat.h */
+#define shmemalign                   pshmemalign /* shmem-compat.h */
+#define shrealloc                    pshrealloc /* shmem-compat.h */
+#define shfree                       pshfree /* shmem-compat.h */
+
+/*
+ * Remote pointer operations
+ */
+#define shmem_ptr                    pshmem_ptr
+
+/*
+ * Communication context operations
+ */
+#define shmem_ctx_create             pshmem_ctx_create
+#define shmem_ctx_destroy            pshmem_ctx_destroy
+
+
+/*
+ * Team management routines
+ */
+#define shmem_team_sync              pshmem_team_sync
+#define shmem_team_my_pe             pshmem_team_my_pe
+#define shmem_team_n_pes             pshmem_team_n_pes
+#define shmem_team_get_config        pshmem_team_get_config
+#define shmem_team_translate_pe      pshmem_team_translate_pe
+#define shmem_team_split_strided     pshmem_team_split_strided
+#define shmem_team_split_2d          pshmem_team_split_2d
+#define shmem_team_destroy           pshmem_team_destroy
+#define shmem_ctx_get_team           pshmem_ctx_get_team
+#define shmem_team_create_ctx        pshmem_team_create_ctx
+
+/*
+ * Teams-based Collectives
+ */
+
+/* Teams alltoall */
+#define shmem_char_alltoall         	pshmem_char_alltoall
+#define shmem_short_alltoall         	pshmem_short_alltoall
+#define shmem_int_alltoall         	    pshmem_int_alltoall
+#define shmem_long_alltoall         	pshmem_long_alltoall
+#define shmem_float_alltoall         	pshmem_float_alltoall
+#define shmem_double_alltoall         	pshmem_double_alltoall
+#define shmem_longlong_alltoall         pshmem_longlong_alltoall
+#define shmem_schar_alltoall         	pshmem_schar_alltoall
+#define shmem_uchar_alltoall         	pshmem_uchar_alltoall
+#define shmem_ushort_alltoall         	pshmem_ushort_alltoall
+#define shmem_uint_alltoall         	pshmem_uint_alltoall
+#define shmem_ulong_alltoall         	pshmem_ulong_alltoall
+#define shmem_ulonglong_alltoall      	pshmem_ulonglong_alltoall
+#define shmem_longdouble_alltoall       pshmem_longdouble_alltoall
+#define shmem_int8_alltoall         	pshmem_int8_alltoall
+#define shmem_int16_alltoall         	pshmem_int16_alltoall
+#define shmem_int32_alltoall         	pshmem_int32_alltoall
+#define shmem_int64_alltoall         	pshmem_int64_alltoall
+#define shmem_uint8_alltoall         	pshmem_uint8_alltoall
+#define shmem_uint16_alltoall         	pshmem_uint16_alltoall
+#define shmem_uint32_alltoall         	pshmem_uint32_alltoall
+#define shmem_uint64_alltoall         	pshmem_uint64_alltoall
+#define shmem_size_alltoall         	pshmem_size_alltoall
+#define shmem_ptrdiff_alltoall         pshmem_ptrdiff_alltoall
+
+#define shmem_alltoallmem               pshmem_alltoallmem
+
+
+/* Teams alltoalls */
+#define shmem_char_alltoalls         	pshmem_char_alltoalls
+#define shmem_short_alltoalls         	pshmem_short_alltoalls
+#define shmem_int_alltoalls            pshmem_int_alltoalls
+#define shmem_long_alltoalls         	pshmem_long_alltoalls
+#define shmem_float_alltoalls         	pshmem_float_alltoalls
+#define shmem_double_alltoalls         pshmem_double_alltoalls
+#define shmem_longlong_alltoalls       pshmem_longlong_alltoalls
+#define shmem_schar_alltoalls         	pshmem_schar_alltoalls
+#define shmem_uchar_alltoalls         	pshmem_uchar_alltoalls
+#define shmem_ushort_alltoalls         pshmem_ushort_alltoalls
+#define shmem_uint_alltoalls         	pshmem_uint_alltoalls
+#define shmem_ulong_alltoalls         	pshmem_ulong_alltoalls
+#define shmem_ulonglong_alltoalls      pshmem_ulonglong_alltoalls
+#define shmem_longdouble_alltoalls     pshmem_longdouble_alltoalls
+#define shmem_int8_alltoalls         	pshmem_int8_alltoalls
+#define shmem_int16_alltoalls         	pshmem_int16_alltoalls
+#define shmem_int32_alltoalls         	pshmem_int32_alltoalls
+#define shmem_int64_alltoalls         	pshmem_int64_alltoalls
+#define shmem_uint8_alltoalls         	pshmem_uint8_alltoalls
+#define shmem_uint16_alltoalls         pshmem_uint16_alltoalls
+#define shmem_uint32_alltoalls         pshmem_uint32_alltoalls
+#define shmem_uint64_alltoalls         pshmem_uint64_alltoalls
+#define shmem_size_alltoalls         	pshmem_size_alltoalls
+#define shmem_ptrdiff_alltoalls        pshmem_ptrdiff_alltoalls
+
+#define shmem_alltoallsmem             pshmem_alltoallsmem
+
+
+/* Teams broadcast */
+#define shmem_char_broadcast         	pshmem_char_broadcast
+#define shmem_short_broadcast         	pshmem_short_broadcast
+#define shmem_int_broadcast        	pshmem_int_broadcast
+#define shmem_long_broadcast         	pshmem_long_broadcast
+#define shmem_float_broadcast         	pshmem_float_broadcast
+#define shmem_double_broadcast         pshmem_double_broadcast
+#define shmem_longlong_broadcast       pshmem_longlong_broadcast
+#define shmem_schar_broadcast         	pshmem_schar_broadcast
+#define shmem_uchar_broadcast         	pshmem_uchar_broadcast
+#define shmem_ushort_broadcast         pshmem_ushort_broadcast
+#define shmem_uint_broadcast         	pshmem_uint_broadcast
+#define shmem_ulong_broadcast         	pshmem_ulong_broadcast
+#define shmem_ulonglong_broadcast      pshmem_ulonglong_broadcast
+#define shmem_longdouble_broadcast     pshmem_longdouble_broadcast
+#define shmem_int8_broadcast         	pshmem_int8_broadcast
+#define shmem_int16_broadcast         	pshmem_int16_broadcast
+#define shmem_int32_broadcast         	pshmem_int32_broadcast
+#define shmem_int64_broadcast         	pshmem_int64_broadcast
+#define shmem_uint8_broadcast         	pshmem_uint8_broadcast
+#define shmem_uint16_broadcast         pshmem_uint16_broadcast
+#define shmem_uint32_broadcast         pshmem_uint32_broadcast
+#define shmem_uint64_broadcast         pshmem_uint64_broadcast
+#define shmem_size_broadcast         	pshmem_size_broadcast
+#define shmem_ptrdiff_broadcast        pshmem_ptrdiff_broadcast
+
+#define shmem_broadcastmem             pshmem_broadcastmem
+
+
+/* Teams collect */
+#define shmem_char_collect          	pshmem_char_collect
+#define shmem_short_collect         	pshmem_short_collect
+#define shmem_int_collect         	pshmem_int_collect
+#define shmem_long_collect          	pshmem_long_collect
+#define shmem_float_collect         	pshmem_float_collect
+#define shmem_double_collect         	pshmem_double_collect
+#define shmem_longlong_collect         pshmem_longlong_collect
+#define shmem_schar_collect         	pshmem_schar_collect
+#define shmem_uchar_collect         	pshmem_uchar_collect
+#define shmem_ushort_collect         	pshmem_ushort_collect
+#define shmem_uint_collect          	pshmem_uint_collect
+#define shmem_ulong_collect         	pshmem_ulong_collect
+#define shmem_ulonglong_collect      	pshmem_ulonglong_collect
+#define shmem_longdouble_collect       pshmem_longdouble_collect
+#define shmem_int8_collect         	pshmem_int8_collect
+#define shmem_int16_collect         	pshmem_int16_collect
+#define shmem_int32_collect         	pshmem_int32_collect
+#define shmem_int64_collect         	pshmem_int64_collect
+#define shmem_uint8_collect         	pshmem_uint8_collect
+#define shmem_uint16_collect         	pshmem_uint16_collect
+#define shmem_uint32_collect         	pshmem_uint32_collect
+#define shmem_uint64_collect         	pshmem_uint64_collect
+#define shmem_size_collect            	pshmem_size_collect
+#define shmem_ptrdiff_collect         	pshmem_ptrdiff_collect
+
+#define shmem_collectmem                pshmem_collectmem
+
+
+/* Teams fcollect */
+#define shmem_char_fcollect         	pshmem_char_fcollect
+#define shmem_short_fcollect         	pshmem_short_fcollect
+#define shmem_int_fcollect         	pshmem_int_fcollect
+#define shmem_long_fcollect         	pshmem_long_fcollect
+#define shmem_float_fcollect         	pshmem_float_fcollect
+#define shmem_double_fcollect         	pshmem_double_fcollect
+#define shmem_longlong_fcollect        pshmem_longlong_fcollect
+#define shmem_schar_fcollect         	pshmem_schar_fcollect
+#define shmem_uchar_fcollect         	pshmem_uchar_fcollect
+#define shmem_ushort_fcollect         	pshmem_ushort_fcollect
+#define shmem_uint_fcollect         	pshmem_uint_fcollect
+#define shmem_ulong_fcollect         	pshmem_ulong_fcollect
+#define shmem_ulonglong_fcollect      	pshmem_ulonglong_fcollect
+#define shmem_longdouble_fcollect      pshmem_longdouble_fcollect
+#define shmem_int8_fcollect         	pshmem_int8_fcollect
+#define shmem_int16_fcollect         	pshmem_int16_fcollect
+#define shmem_int32_fcollect         	pshmem_int32_fcollect
+#define shmem_int64_fcollect         	pshmem_int64_fcollect
+#define shmem_uint8_fcollect         	pshmem_uint8_fcollect
+#define shmem_uint16_fcollect         	pshmem_uint16_fcollect
+#define shmem_uint32_fcollect         	pshmem_uint32_fcollect
+#define shmem_uint64_fcollect         	pshmem_uint64_fcollect
+#define shmem_size_fcollect         	pshmem_size_fcollect
+#define shmem_ptrdiff_fcollect         pshmem_ptrdiff_fcollect
+
+#define shmem_fcollectmem               pshmem_fcollectmem
+
+
+/* Teams reduction: AND */
+#define shmem_uchar_and_reduce          pshmem_uchar_and_reduce
+#define shmem_ushort_and_reduce         pshmem_ushort_and_reduce
+#define shmem_uint_and_reduce           pshmem_uint_and_reduce
+#define shmem_ulong_and_reduce          pshmem_ulong_and_reduce
+#define shmem_ulonglong_and_reduce      pshmem_ulonglong_and_reduce
+#define shmem_int_and_reduce            pshmem_int_and_reduce
+#define shmem_longlong_and_reduce       pshmem_longlong_and_reduce
+#define shmem_int8_and_reduce           pshmem_int8_and_reduce
+#define shmem_int16_and_reduce          pshmem_int16_and_reduce
+#define shmem_int32_and_reduce          pshmem_int32_and_reduce
+#define shmem_int64_and_reduce          pshmem_int64_and_reduce
+#define shmem_uint8_and_reduce          pshmem_uint8_and_reduce
+#define shmem_uint16_and_reduce         pshmem_uint16_and_reduce
+#define shmem_uint32_and_reduce         pshmem_uint32_and_reduce
+#define shmem_uint64_and_reduce         pshmem_uint64_and_reduce
+#define shmem_size_and_reduce           pshmem_size_and_reduce
+
+
+/* Teams reduction: OR */
+#define shmem_uchar_or_reduce          pshmem_uchar_or_reduce
+#define shmem_ushort_or_reduce         pshmem_ushort_or_reduce
+#define shmem_uint_or_reduce           pshmem_uint_or_reduce
+#define shmem_ulong_or_reduce          pshmem_ulong_or_reduce
+#define shmem_ulonglong_or_reduce      pshmem_ulonglong_or_reduce
+#define shmem_int8_or_reduce           pshmem_int8_or_reduce
+#define shmem_int16_or_reduce          pshmem_int16_or_reduce
+#define shmem_int32_or_reduce          pshmem_int32_or_reduce
+#define shmem_int64_or_reduce          pshmem_int64_or_reduce
+#define shmem_uint8_or_reduce          pshmem_uint8_or_reduce
+#define shmem_uint16_or_reduce         pshmem_uint16_or_reduce
+#define shmem_uint32_or_reduce         pshmem_uint32_or_reduce
+#define shmem_uint64_or_reduce         pshmem_uint64_or_reduce
+#define shmem_size_or_reduce           pshmem_size_or_reduce
+
+
+/* Teams reduction: XOR */
+#define shmem_uchar_xor_reduce          pshmem_uchar_xor_reduce
+#define shmem_ushort_xor_reduce         pshmem_ushort_xor_reduce
+#define shmem_uint_xor_reduce           pshmem_uint_xor_reduce
+#define shmem_ulong_xor_reduce          pshmem_ulong_xor_reduce
+#define shmem_ulonglong_xor_reduce      pshmem_ulonglong_xor_reduce
+#define shmem_int8_xor_reduce           pshmem_int8_xor_reduce
+#define shmem_int16_xor_reduce          pshmem_int16_xor_reduce
+#define shmem_int32_xor_reduce          pshmem_int32_xor_reduce
+#define shmem_int64_xor_reduce          pshmem_int64_xor_reduce
+#define shmem_uint8_xor_reduce          pshmem_uint8_xor_reduce
+#define shmem_uint16_xor_reduce         pshmem_uint16_xor_reduce
+#define shmem_uint32_xor_reduce         pshmem_uint32_xor_reduce
+#define shmem_uint64_xor_reduce         pshmem_uint64_xor_reduce
+#define shmem_size_xor_reduce           pshmem_size_xor_reduce
+
+
+/* Teams reduction: MAX */
+#define shmem_char_max_reduce           pshmem_char_max_reduce
+#define shmem_short_max_reduce          pshmem_short_max_reduce
+#define shmem_int_max_reduce            pshmem_int_max_reduce
+#define shmem_long_max_reduce           pshmem_long_max_reduce
+#define shmem_float_max_reduce          pshmem_float_max_reduce
+#define shmem_double_max_reduce         pshmem_double_max_reduce
+#define shmem_longlong_max_reduce       pshmem_longlong_max_reduce
+#define shmem_schar_max_reduce          pshmem_schar_max_reduce
+#define shmem_longdouble_max_reduce     pshmem_longdouble_max_reduce
+#define shmem_ptrdiff_max_reduce        pshmem_ptrdiff_max_reduce
+#define shmem_uchar_max_reduce          pshmem_uchar_max_reduce
+#define shmem_ushort_max_reduce         pshmem_ushort_max_reduce
+#define shmem_uint_max_reduce           pshmem_uint_max_reduce
+#define shmem_ulong_max_reduce          pshmem_ulong_max_reduce
+#define shmem_ulonglong_max_reduce      pshmem_ulonglong_max_reduce
+#define shmem_int8_max_reduce           pshmem_int8_max_reduce
+#define shmem_int16_max_reduce          pshmem_int16_max_reduce
+#define shmem_int32_max_reduce          pshmem_int32_max_reduce
+#define shmem_int64_max_reduce          pshmem_int64_max_reduce
+#define shmem_uint8_max_reduce          pshmem_uint8_max_reduce
+#define shmem_uint16_max_reduce         pshmem_uint16_max_reduce
+#define shmem_uint32_max_reduce         pshmem_uint32_max_reduce
+#define shmem_uint64_max_reduce         pshmem_uint64_max_reduce
+#define shmem_size_max_reduce           pshmem_size_max_reduce
+
+
+/* Teams reduction: MIN */
+#define shmem_char_min_reduce           pshmem_char_min_reduce
+#define shmem_short_min_reduce          pshmem_short_min_reduce
+#define shmem_int_min_reduce            pshmem_int_min_reduce
+#define shmem_long_min_reduce           pshmem_long_min_reduce
+#define shmem_float_min_reduce          pshmem_float_min_reduce
+#define shmem_double_min_reduce         pshmem_double_min_reduce
+#define shmem_longlong_min_reduce       pshmem_longlong_min_reduce
+#define shmem_schar_min_reduce          pshmem_schar_min_reduce
+#define shmem_longdouble_min_reduce     pshmem_longdouble_min_reduce
+#define shmem_ptrdiff_min_reduce        pshmem_ptrdiff_min_reduce
+#define shmem_uchar_min_reduce          pshmem_uchar_min_reduce
+#define shmem_ushort_min_reduce         pshmem_ushort_min_reduce
+#define shmem_uint_min_reduce           pshmem_uint_min_reduce
+#define shmem_ulong_min_reduce          pshmem_ulong_min_reduce
+#define shmem_ulonglong_min_reduce      pshmem_ulonglong_min_reduce
+#define shmem_int8_min_reduce           pshmem_int8_min_reduce
+#define shmem_int16_min_reduce          pshmem_int16_min_reduce
+#define shmem_int32_min_reduce          pshmem_int32_min_reduce
+#define shmem_int64_min_reduce          pshmem_int64_min_reduce
+#define shmem_uint8_min_reduce          pshmem_uint8_min_reduce
+#define shmem_uint16_min_reduce         pshmem_uint16_min_reduce
+#define shmem_uint32_min_reduce         pshmem_uint32_min_reduce
+#define shmem_uint64_min_reduce         pshmem_uint64_min_reduce
+#define shmem_size_min_reduce           pshmem_size_min_reduce
+
+
+/* Teams reduction: SUM */
+#define shmem_char_sum_reduce           pshmem_char_sum_reduce
+#define shmem_short_sum_reduce          pshmem_short_sum_reduce
+#define shmem_int_sum_reduce            pshmem_int_sum_reduce
+#define shmem_long_sum_reduce           pshmem_long_sum_reduce
+#define shmem_float_sum_reduce          pshmem_float_sum_reduce
+#define shmem_double_sum_reduce         pshmem_double_sum_reduce
+#define shmem_longlong_sum_reduce       pshmem_longlong_sum_reduce
+#define shmem_schar_sum_reduce          pshmem_schar_sum_reduce
+#define shmem_longdouble_sum_reduce     pshmem_longdouble_sum_reduce
+#define shmem_ptrdiff_sum_reduce        pshmem_ptrdiff_sum_reduce
+#define shmem_uchar_sum_reduce          pshmem_uchar_sum_reduce
+#define shmem_ushort_sum_reduce         pshmem_ushort_sum_reduce
+#define shmem_uint_sum_reduce           pshmem_uint_sum_reduce
+#define shmem_ulong_sum_reduce          pshmem_ulong_sum_reduce
+#define shmem_ulonglong_sum_reduce      pshmem_ulonglong_sum_reduce
+#define shmem_int8_sum_reduce           pshmem_int8_sum_reduce
+#define shmem_int16_sum_reduce          pshmem_int16_sum_reduce
+#define shmem_int32_sum_reduce          pshmem_int32_sum_reduce
+#define shmem_int64_sum_reduce          pshmem_int64_sum_reduce
+#define shmem_uint8_sum_reduce          pshmem_uint8_sum_reduce
+#define shmem_uint16_sum_reduce         pshmem_uint16_sum_reduce
+#define shmem_uint32_sum_reduce         pshmem_uint32_sum_reduce
+#define shmem_uint64_sum_reduce         pshmem_uint64_sum_reduce
+#define shmem_size_sum_reduce           pshmem_size_sum_reduce
+#define shmem_complexd_sum_reduce       pshmem_complexd_sum_reduce
+#define shmem_complexf_sum_reduce       pshmem_complexf_sum_reduce
+
+
+/* Teams reduction: PROD */
+#define shmem_char_prod_reduce           pshmem_char_prod_reduce
+#define shmem_short_prod_reduce          pshmem_short_prod_reduce
+#define shmem_int_prod_reduce            pshmem_int_prod_reduce
+#define shmem_long_prod_reduce           pshmem_long_prod_reduce
+#define shmem_float_prod_reduce          pshmem_float_prod_reduce
+#define shmem_double_prod_reduce         pshmem_double_prod_reduce
+#define shmem_longlong_prod_reduce       pshmem_longlong_prod_reduce
+#define shmem_schar_prod_reduce          pshmem_schar_prod_reduce
+#define shmem_longdouble_prod_reduce     pshmem_longdouble_prod_reduce
+#define shmem_ptrdiff_prod_reduce        pshmem_ptrdiff_prod_reduce
+#define shmem_uchar_prod_reduce          pshmem_uchar_prod_reduce
+#define shmem_ushort_prod_reduce         pshmem_ushort_prod_reduce
+#define shmem_uint_prod_reduce           pshmem_uint_prod_reduce
+#define shmem_ulong_prod_reduce          pshmem_ulong_prod_reduce
+#define shmem_ulonglong_prod_reduce      pshmem_ulonglong_prod_reduce
+#define shmem_int8_prod_reduce           pshmem_int8_prod_reduce
+#define shmem_int16_prod_reduce          pshmem_int16_prod_reduce
+#define shmem_int32_prod_reduce          pshmem_int32_prod_reduce
+#define shmem_int64_prod_reduce          pshmem_int64_prod_reduce
+#define shmem_uint8_prod_reduce          pshmem_uint8_prod_reduce
+#define shmem_uint16_prod_reduce         pshmem_uint16_prod_reduce
+#define shmem_uint32_prod_reduce         pshmem_uint32_prod_reduce
+#define shmem_uint64_prod_reduce         pshmem_uint64_prod_reduce
+#define shmem_size_prod_reduce           pshmem_size_prod_reduce
+#define shmem_complexd_prod_reduce       pshmem_complexd_prod_reduce
+#define shmem_complexf_prod_reduce       pshmem_complexf_prod_reduce
+
+/*
+ * Elemental put routines
+ */
+#define shmem_ctx_char_p             pshmem_ctx_char_p
+#define shmem_ctx_short_p            pshmem_ctx_short_p
+#define shmem_ctx_int_p              pshmem_ctx_int_p
+#define shmem_ctx_long_p             pshmem_ctx_long_p
+#define shmem_ctx_float_p            pshmem_ctx_float_p
+#define shmem_ctx_double_p           pshmem_ctx_double_p
+#define shmem_ctx_longlong_p         pshmem_ctx_longlong_p
+#define shmem_ctx_schar_p            pshmem_ctx_schar_p
+#define shmem_ctx_uchar_p            pshmem_ctx_uchar_p
+#define shmem_ctx_ushort_p           pshmem_ctx_ushort_p
+#define shmem_ctx_uint_p             pshmem_ctx_uint_p
+#define shmem_ctx_ulong_p            pshmem_ctx_ulong_p
+#define shmem_ctx_ulonglong_p        pshmem_ctx_ulonglong_p
+#define shmem_ctx_longdouble_p       pshmem_ctx_longdouble_p
+#define shmem_ctx_int8_p             pshmem_ctx_int8_p
+#define shmem_ctx_int16_p            pshmem_ctx_int16_p
+#define shmem_ctx_int32_p            pshmem_ctx_int32_p
+#define shmem_ctx_int64_p            pshmem_ctx_int64_p
+#define shmem_ctx_uint8_p            pshmem_ctx_uint8_p
+#define shmem_ctx_uint16_p           pshmem_ctx_uint16_p
+#define shmem_ctx_uint32_p           pshmem_ctx_uint32_p
+#define shmem_ctx_uint64_p           pshmem_ctx_uint64_p
+#define shmem_ctx_size_p             pshmem_ctx_size_p
+#define shmem_ctx_ptrdiff_p          pshmem_ctx_ptrdiff_p
+
+#define shmem_char_p                 pshmem_char_p
+#define shmem_short_p                pshmem_short_p
+#define shmem_int_p                  pshmem_int_p
+#define shmem_long_p                 pshmem_long_p
+#define shmem_float_p                pshmem_float_p
+#define shmem_double_p               pshmem_double_p
+#define shmem_longlong_p             pshmem_longlong_p
+#define shmem_schar_p                pshmem_schar_p
+#define shmem_uchar_p                pshmem_uchar_p
+#define shmem_ushort_p               pshmem_ushort_p
+#define shmem_uint_p                 pshmem_uint_p
+#define shmem_ulong_p                pshmem_ulong_p
+#define shmem_ulonglong_p            pshmem_ulonglong_p
+#define shmem_longdouble_p           pshmem_longdouble_p
+#define shmem_int8_p                 pshmem_int8_p
+#define shmem_int16_p                pshmem_int16_p
+#define shmem_int32_p                pshmem_int32_p
+#define shmem_int64_p                pshmem_int64_p
+#define shmem_uint8_p                pshmem_uint8_p
+#define shmem_uint16_p               pshmem_uint16_p
+#define shmem_uint32_p               pshmem_uint32_p
+#define shmem_uint64_p               pshmem_uint64_p
+#define shmem_size_p                 pshmem_size_p
+#define shmem_ptrdiff_p              pshmem_ptrdiff_p
+
+#define shmemx_int16_p               pshmemx_int16_p
+#define shmemx_int32_p               pshmemx_int32_p
+#define shmemx_int64_p               pshmemx_int64_p
+
+/*
+ * Signaled put routines
+ */
+#define shmem_ctx_char_put_signal             pshmem_ctx_char_put_signal
+#define shmem_ctx_short_put_signal            pshmem_ctx_short_put_signal
+#define shmem_ctx_int_put_signal              pshmem_ctx_int_put_signal
+#define shmem_ctx_long_put_signal             pshmem_ctx_long_put_signal
+#define shmem_ctx_float_put_signal            pshmem_ctx_float_put_signal
+#define shmem_ctx_double_put_signal           pshmem_ctx_double_put_signal
+#define shmem_ctx_longlong_put_signal         pshmem_ctx_longlong_put_signal
+#define shmem_ctx_schar_put_signal            pshmem_ctx_schar_put_signal
+#define shmem_ctx_uchar_put_signal            pshmem_ctx_uchar_put_signal
+#define shmem_ctx_ushort_put_signal           pshmem_ctx_ushort_put_signal
+#define shmem_ctx_uint_put_signal             pshmem_ctx_uint_put_signal
+#define shmem_ctx_ulong_put_signal            pshmem_ctx_ulong_put_signal
+#define shmem_ctx_ulonglong_put_signal        pshmem_ctx_ulonglong_put_signal
+#define shmem_ctx_longdouble_put_signal       pshmem_ctx_longdouble_put_signal
+#define shmem_ctx_int8_put_signal             pshmem_ctx_int8_put_signal
+#define shmem_ctx_int16_put_signal            pshmem_ctx_int16_put_signal
+#define shmem_ctx_int32_put_signal            pshmem_ctx_int32_put_signal
+#define shmem_ctx_int64_put_signal            pshmem_ctx_int64_put_signal
+#define shmem_ctx_uint8_put_signal            pshmem_ctx_uint8_put_signal
+#define shmem_ctx_uint16_put_signal           pshmem_ctx_uint16_put_signal
+#define shmem_ctx_uint32_put_signal           pshmem_ctx_uint32_put_signal
+#define shmem_ctx_uint64_put_signal           pshmem_ctx_uint64_put_signal
+#define shmem_ctx_size_put_signal             pshmem_ctx_size_put_signal
+#define shmem_ctx_ptrdiff_put_signal          pshmem_ctx_ptrdiff_put_signal
+
+#define shmem_char_put_signal                 pshmem_char_put_signal
+#define shmem_short_put_signal                pshmem_short_put_signal
+#define shmem_int_put_signal                  pshmem_int_put_signal
+#define shmem_long_put_signal                 pshmem_long_put_signal
+#define shmem_float_put_signal                pshmem_float_put_signal
+#define shmem_double_put_signal               pshmem_double_put_signal
+#define shmem_longlong_put_signal             pshmem_longlong_put_signal
+#define shmem_schar_put_signal                pshmem_schar_put_signal
+#define shmem_uchar_put_signal                pshmem_uchar_put_signal
+#define shmem_ushort_put_signal               pshmem_ushort_put_signal
+#define shmem_uint_put_signal                 pshmem_uint_put_signal
+#define shmem_ulong_put_signal                pshmem_ulong_put_signal
+#define shmem_ulonglong_put_signal            pshmem_ulonglong_put_signal
+#define shmem_longdouble_put_signal           pshmem_longdouble_put_signal
+#define shmem_int8_put_signal                 pshmem_int8_put_signal
+#define shmem_int16_put_signal                pshmem_int16_put_signal
+#define shmem_int32_put_signal                pshmem_int32_put_signal
+#define shmem_int64_put_signal                pshmem_int64_put_signal
+#define shmem_uint8_put_signal                pshmem_uint8_put_signal
+#define shmem_uint16_put_signal               pshmem_uint16_put_signal
+#define shmem_uint32_put_signal               pshmem_uint32_put_signal
+#define shmem_uint64_put_signal               pshmem_uint64_put_signal
+#define shmem_size_put_signal                 pshmem_size_put_signal
+#define shmem_ptrdiff_put_signal              pshmem_ptrdiff_put_signal
+
+#define shmem_put8_signal                     pshmem_put8_signal
+#define shmem_put16_signal                    pshmem_put16_signal
+#define shmem_put32_signal                    pshmem_put32_signal
+#define shmem_put64_signal                    pshmem_put64_signal
+#define shmem_put128_signal                   pshmem_put128_signal
+
+#define shmem_ctx_put8_signal                 pshmem_ctx_put8_signal
+#define shmem_ctx_put16_signal                pshmem_ctx_put16_signal
+#define shmem_ctx_put32_signal                pshmem_ctx_put32_signal
+#define shmem_ctx_put64_signal                pshmem_ctx_put64_signal
+#define shmem_ctx_put128_signal               pshmem_ctx_put128_signal
+
+#define shmem_putmem_signal                  pshmem_putmem_signal
+#define shmem_ctx_putmem_signal              pshmem_ctx_putmem_signal
+
+/*
+ * Nonblocking signaled put routines
+ */
+#define shmem_ctx_char_put_signal_nbi             pshmem_ctx_char_put_signal_nbi
+#define shmem_ctx_short_put_signal_nbi            pshmem_ctx_short_put_signal_nbi
+#define shmem_ctx_int_put_signal_nbi              pshmem_ctx_int_put_signal_nbi
+#define shmem_ctx_long_put_signal_nbi             pshmem_ctx_long_put_signal_nbi
+#define shmem_ctx_float_put_signal_nbi            pshmem_ctx_float_put_signal_nbi
+#define shmem_ctx_double_put_signal_nbi           pshmem_ctx_double_put_signal_nbi
+#define shmem_ctx_longlong_put_signal_nbi         pshmem_ctx_longlong_put_signal_nbi
+#define shmem_ctx_schar_put_signal_nbi            pshmem_ctx_schar_put_signal_nbi
+#define shmem_ctx_uchar_put_signal_nbi            pshmem_ctx_uchar_put_signal_nbi
+#define shmem_ctx_ushort_put_signal_nbi           pshmem_ctx_ushort_put_signal_nbi
+#define shmem_ctx_uint_put_signal_nbi             pshmem_ctx_uint_put_signal_nbi
+#define shmem_ctx_ulong_put_signal_nbi            pshmem_ctx_ulong_put_signal_nbi
+#define shmem_ctx_ulonglong_put_signal_nbi        pshmem_ctx_ulonglong_put_signal_nbi
+#define shmem_ctx_longdouble_put_signal_nbi       pshmem_ctx_longdouble_put_signal_nbi
+#define shmem_ctx_int8_put_signal_nbi             pshmem_ctx_int8_put_signal_nbi
+#define shmem_ctx_int16_put_signal_nbi            pshmem_ctx_int16_put_signal_nbi
+#define shmem_ctx_int32_put_signal_nbi            pshmem_ctx_int32_put_signal_nbi
+#define shmem_ctx_int64_put_signal_nbi            pshmem_ctx_int64_put_signal_nbi
+#define shmem_ctx_uint8_put_signal_nbi            pshmem_ctx_uint8_put_signal_nbi
+#define shmem_ctx_uint16_put_signal_nbi           pshmem_ctx_uint16_put_signal_nbi
+#define shmem_ctx_uint32_put_signal_nbi           pshmem_ctx_uint32_put_signal_nbi
+#define shmem_ctx_uint64_put_signal_nbi           pshmem_ctx_uint64_put_signal_nbi
+#define shmem_ctx_size_put_signal_nbi             pshmem_ctx_size_put_signal_nbi
+#define shmem_ctx_ptrdiff_put_signal_nbi          pshmem_ctx_ptrdiff_put_signal_nbi
+
+#define shmem_char_put_signal_nbi                 pshmem_char_put_signal_nbi
+#define shmem_short_put_signal_nbi                pshmem_short_put_signal_nbi
+#define shmem_int_put_signal_nbi                  pshmem_int_put_signal_nbi
+#define shmem_long_put_signal_nbi                 pshmem_long_put_signal_nbi
+#define shmem_float_put_signal_nbi                pshmem_float_put_signal_nbi
+#define shmem_double_put_signal_nbi               pshmem_double_put_signal_nbi
+#define shmem_longlong_put_signal_nbi             pshmem_longlong_put_signal_nbi
+#define shmem_schar_put_signal_nbi                pshmem_schar_put_signal_nbi
+#define shmem_uchar_put_signal_nbi                pshmem_uchar_put_signal_nbi
+#define shmem_ushort_put_signal_nbi               pshmem_ushort_put_signal_nbi
+#define shmem_uint_put_signal_nbi                 pshmem_uint_put_signal_nbi
+#define shmem_ulong_put_signal_nbi                pshmem_ulong_put_signal_nbi
+#define shmem_ulonglong_put_signal_nbi            pshmem_ulonglong_put_signal_nbi
+#define shmem_longdouble_put_signal_nbi           pshmem_longdouble_put_signal_nbi
+#define shmem_int8_put_signal_nbi                 pshmem_int8_put_signal_nbi
+#define shmem_int16_put_signal_nbi                pshmem_int16_put_signal_nbi
+#define shmem_int32_put_signal_nbi                pshmem_int32_put_signal_nbi
+#define shmem_int64_put_signal_nbi                pshmem_int64_put_signal_nbi
+#define shmem_uint8_put_signal_nbi                pshmem_uint8_put_signal_nbi
+#define shmem_uint16_put_signal_nbi               pshmem_uint16_put_signal_nbi
+#define shmem_uint32_put_signal_nbi               pshmem_uint32_put_signal_nbi
+#define shmem_uint64_put_signal_nbi               pshmem_uint64_put_signal_nbi
+#define shmem_size_put_signal_nbi                 pshmem_size_put_signal_nbi
+#define shmem_ptrdiff_put_signal_nbi              pshmem_ptrdiff_put_signal_nbi
+
+#define shmem_put8_signal_nbi                     pshmem_put8_signal_nbi
+#define shmem_put16_signal_nbi                    pshmem_put16_signal_nbi
+#define shmem_put32_signal_nbi                    pshmem_put32_signal_nbi
+#define shmem_put64_signal_nbi                    pshmem_put64_signal_nbi
+#define shmem_put128_signal_nbi                   pshmem_put128_signal_nbi
+
+#define shmem_ctx_put8_signal_nbi                 pshmem_ctx_put8_signal_nbi
+#define shmem_ctx_put16_signal_nbi                pshmem_ctx_put16_signal_nbi
+#define shmem_ctx_put32_signal_nbi                pshmem_ctx_put32_signal_nbi
+#define shmem_ctx_put64_signal_nbi                pshmem_ctx_put64_signal_nbi
+#define shmem_ctx_put128_signal_nbi               pshmem_ctx_put128_signal_nbi
+
+#define shmem_putmem_signal_nbi                  pshmem_putmem_signal_nbi
+#define shmem_ctx_putmem_signal_nbi              pshmem_ctx_putmem_signal_nbi
+
+#define shmem_signal_fetch                       pshmem_signal_fetch
+
+/*
+ * Block data put routines
+ */
+#define shmem_ctx_char_put           pshmem_ctx_char_put
+#define shmem_ctx_short_put          pshmem_ctx_short_put
+#define shmem_ctx_int_put            pshmem_ctx_int_put
+#define shmem_ctx_long_put           pshmem_ctx_long_put
+#define shmem_ctx_float_put          pshmem_ctx_float_put
+#define shmem_ctx_double_put         pshmem_ctx_double_put
+#define shmem_ctx_longlong_put       pshmem_ctx_longlong_put
+#define shmem_ctx_schar_put          pshmem_ctx_schar_put
+#define shmem_ctx_uchar_put          pshmem_ctx_uchar_put
+#define shmem_ctx_ushort_put         pshmem_ctx_ushort_put
+#define shmem_ctx_uint_put           pshmem_ctx_uint_put
+#define shmem_ctx_ulong_put          pshmem_ctx_ulong_put
+#define shmem_ctx_ulonglong_put      pshmem_ctx_ulonglong_put
+#define shmem_ctx_longdouble_put     pshmem_ctx_longdouble_put
+#define shmem_ctx_int8_put           pshmem_ctx_int8_put
+#define shmem_ctx_int16_put          pshmem_ctx_int16_put
+#define shmem_ctx_int32_put          pshmem_ctx_int32_put
+#define shmem_ctx_int64_put          pshmem_ctx_int64_put
+#define shmem_ctx_uint8_put          pshmem_ctx_uint8_put
+#define shmem_ctx_uint16_put         pshmem_ctx_uint16_put
+#define shmem_ctx_uint32_put         pshmem_ctx_uint32_put
+#define shmem_ctx_uint64_put         pshmem_ctx_uint64_put
+#define shmem_ctx_size_put           pshmem_ctx_size_put
+#define shmem_ctx_ptrdiff_put        pshmem_ctx_ptrdiff_put
+
+#define shmem_char_put               pshmem_char_put /* shmem-compat.h */
+#define shmem_short_put              pshmem_short_put
+#define shmem_int_put                pshmem_int_put
+#define shmem_long_put               pshmem_long_put
+#define shmem_float_put              pshmem_float_put
+#define shmem_double_put             pshmem_double_put
+#define shmem_longlong_put           pshmem_longlong_put
+#define shmem_schar_put              pshmem_schar_put
+#define shmem_uchar_put              pshmem_uchar_put
+#define shmem_ushort_put             pshmem_ushort_put
+#define shmem_uint_put               pshmem_uint_put
+#define shmem_ulong_put              pshmem_ulong_put
+#define shmem_ulonglong_put          pshmem_ulonglong_put
+#define shmem_longdouble_put         pshmem_longdouble_put
+#define shmem_int8_put               pshmem_int8_put
+#define shmem_int16_put              pshmem_int16_put
+#define shmem_int32_put              pshmem_int32_put
+#define shmem_int64_put              pshmem_int64_put
+#define shmem_uint8_put              pshmem_uint8_put
+#define shmem_uint16_put             pshmem_uint16_put
+#define shmem_uint32_put             pshmem_uint32_put
+#define shmem_uint64_put             pshmem_uint64_put
+#define shmem_size_put               pshmem_size_put
+#define shmem_ptrdiff_put            pshmem_ptrdiff_put
+
+#define shmem_ctx_put8               pshmem_ctx_put8
+#define shmem_ctx_put16              pshmem_ctx_put16
+#define shmem_ctx_put32              pshmem_ctx_put32
+#define shmem_ctx_put64              pshmem_ctx_put64
+#define shmem_ctx_put128             pshmem_ctx_put128
+#define shmem_ctx_putmem             pshmem_ctx_putmem
+
+#define shmem_put8                   pshmem_put8
+#define shmem_put16                  pshmem_put16
+#define shmem_put32                  pshmem_put32
+#define shmem_put64                  pshmem_put64
+#define shmem_put128                 pshmem_put128
+#define shmem_putmem                 pshmem_putmem
+
+
+/*
+ * Strided put routines
+ */
+#define shmem_ctx_char_iput           pshmem_ctx_char_iput
+#define shmem_ctx_short_iput          pshmem_ctx_short_iput
+#define shmem_ctx_int_iput            pshmem_ctx_int_iput
+#define shmem_ctx_long_iput           pshmem_ctx_long_iput
+#define shmem_ctx_float_iput          pshmem_ctx_float_iput
+#define shmem_ctx_double_iput         pshmem_ctx_double_iput
+#define shmem_ctx_longlong_iput       pshmem_ctx_longlong_iput
+#define shmem_ctx_schar_iput          pshmem_ctx_schar_iput
+#define shmem_ctx_uchar_iput          pshmem_ctx_uchar_iput
+#define shmem_ctx_ushort_iput         pshmem_ctx_ushort_iput
+#define shmem_ctx_uint_iput           pshmem_ctx_uint_iput
+#define shmem_ctx_ulong_iput          pshmem_ctx_ulong_iput
+#define shmem_ctx_ulonglong_iput      pshmem_ctx_ulonglong_iput
+#define shmem_ctx_longdouble_iput     pshmem_ctx_longdouble_iput
+#define shmem_ctx_int8_iput           pshmem_ctx_int8_iput
+#define shmem_ctx_int16_iput          pshmem_ctx_int16_iput
+#define shmem_ctx_int32_iput          pshmem_ctx_int32_iput
+#define shmem_ctx_int64_iput          pshmem_ctx_int64_iput
+#define shmem_ctx_uint8_iput          pshmem_ctx_uint8_iput
+#define shmem_ctx_uint16_iput         pshmem_ctx_uint16_iput
+#define shmem_ctx_uint32_iput         pshmem_ctx_uint32_iput
+#define shmem_ctx_uint64_iput         pshmem_ctx_uint64_iput
+#define shmem_ctx_size_iput           pshmem_ctx_size_iput
+#define shmem_ctx_ptrdiff_iput        pshmem_ctx_ptrdiff_iput
+
+#define shmem_char_iput               pshmem_char_iput
+#define shmem_short_iput              pshmem_short_iput
+#define shmem_int_iput                pshmem_int_iput
+#define shmem_long_iput               pshmem_long_iput
+#define shmem_float_iput              pshmem_float_iput
+#define shmem_double_iput             pshmem_double_iput
+#define shmem_longlong_iput           pshmem_longlong_iput
+#define shmem_schar_iput              pshmem_schar_iput
+#define shmem_uchar_iput              pshmem_uchar_iput
+#define shmem_ushort_iput             pshmem_ushort_iput
+#define shmem_uint_iput               pshmem_uint_iput
+#define shmem_ulong_iput              pshmem_ulong_iput
+#define shmem_ulonglong_iput          pshmem_ulonglong_iput
+#define shmem_longdouble_iput         pshmem_longdouble_iput
+#define shmem_int8_iput               pshmem_int8_iput
+#define shmem_int16_iput              pshmem_int16_iput
+#define shmem_int32_iput              pshmem_int32_iput
+#define shmem_int64_iput              pshmem_int64_iput
+#define shmem_uint8_iput              pshmem_uint8_iput
+#define shmem_uint16_iput             pshmem_uint16_iput
+#define shmem_uint32_iput             pshmem_uint32_iput
+#define shmem_uint64_iput             pshmem_uint64_iput
+#define shmem_size_iput               pshmem_size_iput
+#define shmem_ptrdiff_iput            pshmem_ptrdiff_iput
+
+#define shmem_ctx_iput8              pshmem_ctx_iput8
+#define shmem_ctx_iput16             pshmem_ctx_iput16
+#define shmem_ctx_iput32             pshmem_ctx_iput32
+#define shmem_ctx_iput64             pshmem_ctx_iput64
+#define shmem_ctx_iput128            pshmem_ctx_iput128
+
+#define shmem_iput8                  pshmem_iput8
+#define shmem_iput16                 pshmem_iput16
+#define shmem_iput32                 pshmem_iput32
+#define shmem_iput64                 pshmem_iput64
+#define shmem_iput128                pshmem_iput128
+
+/*
+ * Non-block data put routines
+ */
+#define shmem_ctx_char_put_nbi           pshmem_ctx_char_put_nbi
+#define shmem_ctx_short_put_nbi          pshmem_ctx_short_put_nbi
+#define shmem_ctx_int_put_nbi            pshmem_ctx_int_put_nbi
+#define shmem_ctx_long_put_nbi           pshmem_ctx_long_put_nbi
+#define shmem_ctx_float_put_nbi          pshmem_ctx_float_put_nbi
+#define shmem_ctx_double_put_nbi         pshmem_ctx_double_put_nbi
+#define shmem_ctx_longlong_put_nbi       pshmem_ctx_longlong_put_nbi
+#define shmem_ctx_schar_put_nbi          pshmem_ctx_schar_put_nbi
+#define shmem_ctx_uchar_put_nbi          pshmem_ctx_uchar_put_nbi
+#define shmem_ctx_ushort_put_nbi         pshmem_ctx_ushort_put_nbi
+#define shmem_ctx_uint_put_nbi           pshmem_ctx_uint_put_nbi
+#define shmem_ctx_ulong_put_nbi          pshmem_ctx_ulong_put_nbi
+#define shmem_ctx_ulonglong_put_nbi      pshmem_ctx_ulonglong_put_nbi
+#define shmem_ctx_longdouble_put_nbi     pshmem_ctx_longdouble_put_nbi
+#define shmem_ctx_int8_put_nbi           pshmem_ctx_int8_put_nbi
+#define shmem_ctx_int16_put_nbi          pshmem_ctx_int16_put_nbi
+#define shmem_ctx_int32_put_nbi          pshmem_ctx_int32_put_nbi
+#define shmem_ctx_int64_put_nbi          pshmem_ctx_int64_put_nbi
+#define shmem_ctx_uint8_put_nbi          pshmem_ctx_uint8_put_nbi
+#define shmem_ctx_uint16_put_nbi         pshmem_ctx_uint16_put_nbi
+#define shmem_ctx_uint32_put_nbi         pshmem_ctx_uint32_put_nbi
+#define shmem_ctx_uint64_put_nbi         pshmem_ctx_uint64_put_nbi
+#define shmem_ctx_size_put_nbi           pshmem_ctx_size_put_nbi
+#define shmem_ctx_ptrdiff_put_nbi        pshmem_ctx_ptrdiff_put_nbi
+
+#define shmem_char_put_nbi               pshmem_char_put_nbi
+#define shmem_short_put_nbi              pshmem_short_put_nbi
+#define shmem_int_put_nbi                pshmem_int_put_nbi
+#define shmem_long_put_nbi               pshmem_long_put_nbi
+#define shmem_float_put_nbi              pshmem_float_put_nbi
+#define shmem_double_put_nbi             pshmem_double_put_nbi
+#define shmem_longlong_put_nbi           pshmem_longlong_put_nbi
+#define shmem_schar_put_nbi              pshmem_schar_put_nbi
+#define shmem_uchar_put_nbi              pshmem_uchar_put_nbi
+#define shmem_ushort_put_nbi             pshmem_ushort_put_nbi
+#define shmem_uint_put_nbi               pshmem_uint_put_nbi
+#define shmem_ulong_put_nbi              pshmem_ulong_put_nbi
+#define shmem_ulonglong_put_nbi          pshmem_ulonglong_put_nbi
+#define shmem_longdouble_put_nbi         pshmem_longdouble_put_nbi
+#define shmem_int8_put_nbi               pshmem_int8_put_nbi
+#define shmem_int16_put_nbi              pshmem_int16_put_nbi
+#define shmem_int32_put_nbi              pshmem_int32_put_nbi
+#define shmem_int64_put_nbi              pshmem_int64_put_nbi
+#define shmem_uint8_put_nbi              pshmem_uint8_put_nbi
+#define shmem_uint16_put_nbi             pshmem_uint16_put_nbi
+#define shmem_uint32_put_nbi             pshmem_uint32_put_nbi
+#define shmem_uint64_put_nbi             pshmem_uint64_put_nbi
+#define shmem_size_put_nbi               pshmem_size_put_nbi
+#define shmem_ptrdiff_put_nbi            pshmem_ptrdiff_put_nbi
+
+#define shmem_ctx_put8_nbi           pshmem_ctx_put8_nbi
+#define shmem_ctx_put16_nbi          pshmem_ctx_put16_nbi
+#define shmem_ctx_put32_nbi          pshmem_ctx_put32_nbi
+#define shmem_ctx_put64_nbi          pshmem_ctx_put64_nbi
+#define shmem_ctx_put128_nbi         pshmem_ctx_put128_nbi
+#define shmem_ctx_putmem_nbi         pshmem_ctx_putmem_nbi
+
+#define shmem_put8_nbi               pshmem_put8_nbi
+#define shmem_put16_nbi              pshmem_put16_nbi
+#define shmem_put32_nbi              pshmem_put32_nbi
+#define shmem_put64_nbi              pshmem_put64_nbi
+#define shmem_put128_nbi             pshmem_put128_nbi
+#define shmem_putmem_nbi             pshmem_putmem_nbi
+
+/*
+ * Elemental get routines
+ */
+#define shmem_ctx_char_g             pshmem_ctx_char_g
+#define shmem_ctx_short_g            pshmem_ctx_short_g
+#define shmem_ctx_int_g              pshmem_ctx_int_g
+#define shmem_ctx_long_g             pshmem_ctx_long_g
+#define shmem_ctx_float_g            pshmem_ctx_float_g
+#define shmem_ctx_double_g           pshmem_ctx_double_g
+#define shmem_ctx_longlong_g         pshmem_ctx_longlong_g
+#define shmem_ctx_schar_g            pshmem_ctx_schar_g
+#define shmem_ctx_uchar_g            pshmem_ctx_uchar_g
+#define shmem_ctx_ushort_g           pshmem_ctx_ushort_g
+#define shmem_ctx_uint_g             pshmem_ctx_uint_g
+#define shmem_ctx_ulong_g            pshmem_ctx_ulong_g
+#define shmem_ctx_ulonglong_g        pshmem_ctx_ulonglong_g
+#define shmem_ctx_longdouble_g       pshmem_ctx_longdouble_g
+#define shmem_ctx_int8_g             pshmem_ctx_int8_g
+#define shmem_ctx_int16_g            pshmem_ctx_int16_g
+#define shmem_ctx_int32_g            pshmem_ctx_int32_g
+#define shmem_ctx_int64_g            pshmem_ctx_int64_g
+#define shmem_ctx_uint8_g            pshmem_ctx_uint8_g
+#define shmem_ctx_uint16_g           pshmem_ctx_uint16_g
+#define shmem_ctx_uint32_g           pshmem_ctx_uint32_g
+#define shmem_ctx_uint64_g           pshmem_ctx_uint64_g
+#define shmem_ctx_size_g             pshmem_ctx_size_g
+#define shmem_ctx_ptrdiff_g          pshmem_ctx_ptrdiff_g
+
+#define shmem_char_g                 pshmem_char_g
+#define shmem_short_g                pshmem_short_g
+#define shmem_int_g                  pshmem_int_g
+#define shmem_long_g                 pshmem_long_g
+#define shmem_float_g                pshmem_float_g
+#define shmem_double_g               pshmem_double_g
+#define shmem_longlong_g             pshmem_longlong_g
+#define shmem_schar_g                pshmem_schar_g
+#define shmem_uchar_g                pshmem_uchar_g
+#define shmem_ushort_g               pshmem_ushort_g
+#define shmem_uint_g                 pshmem_uint_g
+#define shmem_ulong_g                pshmem_ulong_g
+#define shmem_ulonglong_g            pshmem_ulonglong_g
+#define shmem_longdouble_g           pshmem_longdouble_g
+#define shmem_int8_g                 pshmem_int8_g
+#define shmem_int16_g                pshmem_int16_g
+#define shmem_int32_g                pshmem_int32_g
+#define shmem_int64_g                pshmem_int64_g
+#define shmem_uint8_g                pshmem_uint8_g
+#define shmem_uint16_g               pshmem_uint16_g
+#define shmem_uint32_g               pshmem_uint32_g
+#define shmem_uint64_g               pshmem_uint64_g
+#define shmem_size_g                 pshmem_size_g
+#define shmem_ptrdiff_g              pshmem_ptrdiff_g
+
+#define shmemx_int16_g               pshmemx_int16_g
+#define shmemx_int32_g               pshmemx_int32_g
+#define shmemx_int64_g               pshmemx_int64_g
+
+/*
+ * Block data get routines
+ */
+#define shmem_ctx_char_get           pshmem_ctx_char_get
+#define shmem_ctx_short_get          pshmem_ctx_short_get
+#define shmem_ctx_int_get            pshmem_ctx_int_get
+#define shmem_ctx_long_get           pshmem_ctx_long_get
+#define shmem_ctx_float_get          pshmem_ctx_float_get
+#define shmem_ctx_double_get         pshmem_ctx_double_get
+#define shmem_ctx_longlong_get       pshmem_ctx_longlong_get
+#define shmem_ctx_schar_get          pshmem_ctx_schar_get
+#define shmem_ctx_uchar_get          pshmem_ctx_uchar_get
+#define shmem_ctx_ushort_get         pshmem_ctx_ushort_get
+#define shmem_ctx_uint_get           pshmem_ctx_uint_get
+#define shmem_ctx_ulong_get          pshmem_ctx_ulong_get
+#define shmem_ctx_ulonglong_get      pshmem_ctx_ulonglong_get
+#define shmem_ctx_longdouble_get     pshmem_ctx_longdouble_get
+#define shmem_ctx_int8_get           pshmem_ctx_int8_get
+#define shmem_ctx_int16_get          pshmem_ctx_int16_get
+#define shmem_ctx_int32_get          pshmem_ctx_int32_get
+#define shmem_ctx_int64_get          pshmem_ctx_int64_get
+#define shmem_ctx_uint8_get          pshmem_ctx_uint8_get
+#define shmem_ctx_uint16_get         pshmem_ctx_uint16_get
+#define shmem_ctx_uint32_get         pshmem_ctx_uint32_get
+#define shmem_ctx_uint64_get         pshmem_ctx_uint64_get
+#define shmem_ctx_size_get           pshmem_ctx_size_get
+#define shmem_ctx_ptrdiff_get        pshmem_ctx_ptrdiff_get
+
+#define shmem_char_get               pshmem_char_get /* shmem-compat.h */
+#define shmem_short_get              pshmem_short_get
+#define shmem_int_get                pshmem_int_get
+#define shmem_long_get               pshmem_long_get
+#define shmem_float_get              pshmem_float_get
+#define shmem_double_get             pshmem_double_get
+#define shmem_longlong_get           pshmem_longlong_get
+#define shmem_schar_get              pshmem_schar_get
+#define shmem_uchar_get              pshmem_uchar_get
+#define shmem_ushort_get             pshmem_ushort_get
+#define shmem_uint_get               pshmem_uint_get
+#define shmem_ulong_get              pshmem_ulong_get
+#define shmem_ulonglong_get          pshmem_ulonglong_get
+#define shmem_longdouble_get         pshmem_longdouble_get
+#define shmem_int8_get               pshmem_int8_get
+#define shmem_int16_get              pshmem_int16_get
+#define shmem_int32_get              pshmem_int32_get
+#define shmem_int64_get              pshmem_int64_get
+#define shmem_uint8_get              pshmem_uint8_get
+#define shmem_uint16_get             pshmem_uint16_get
+#define shmem_uint32_get             pshmem_uint32_get
+#define shmem_uint64_get             pshmem_uint64_get
+#define shmem_size_get               pshmem_size_get
+#define shmem_ptrdiff_get            pshmem_ptrdiff_get
+
+#define shmem_ctx_get8               pshmem_ctx_get8
+#define shmem_ctx_get16              pshmem_ctx_get16
+#define shmem_ctx_get32              pshmem_ctx_get32
+#define shmem_ctx_get64              pshmem_ctx_get64
+#define shmem_ctx_get128             pshmem_ctx_get128
+#define shmem_ctx_getmem             pshmem_ctx_getmem
+
+#define shmem_get8                   pshmem_get8
+#define shmem_get16                  pshmem_get16
+#define shmem_get32                  pshmem_get32
+#define shmem_get64                  pshmem_get64
+#define shmem_get128                 pshmem_get128
+#define shmem_getmem                 pshmem_getmem
+
+/*
+ * Strided get routines
+ */
+#define shmem_ctx_char_iget           pshmem_ctx_char_iget
+#define shmem_ctx_short_iget          pshmem_ctx_short_iget
+#define shmem_ctx_int_iget            pshmem_ctx_int_iget
+#define shmem_ctx_long_iget           pshmem_ctx_long_iget
+#define shmem_ctx_float_iget          pshmem_ctx_float_iget
+#define shmem_ctx_double_iget         pshmem_ctx_double_iget
+#define shmem_ctx_longlong_iget       pshmem_ctx_longlong_iget
+#define shmem_ctx_schar_iget          pshmem_ctx_schar_iget
+#define shmem_ctx_uchar_iget          pshmem_ctx_uchar_iget
+#define shmem_ctx_ushort_iget         pshmem_ctx_ushort_iget
+#define shmem_ctx_uint_iget           pshmem_ctx_uint_iget
+#define shmem_ctx_ulong_iget          pshmem_ctx_ulong_iget
+#define shmem_ctx_ulonglong_iget      pshmem_ctx_ulonglong_iget
+#define shmem_ctx_longdouble_iget     pshmem_ctx_longdouble_iget
+#define shmem_ctx_int8_iget           pshmem_ctx_int8_iget
+#define shmem_ctx_int16_iget          pshmem_ctx_int16_iget
+#define shmem_ctx_int32_iget          pshmem_ctx_int32_iget
+#define shmem_ctx_int64_iget          pshmem_ctx_int64_iget
+#define shmem_ctx_uint8_iget          pshmem_ctx_uint8_iget
+#define shmem_ctx_uint16_iget         pshmem_ctx_uint16_iget
+#define shmem_ctx_uint32_iget         pshmem_ctx_uint32_iget
+#define shmem_ctx_uint64_iget         pshmem_ctx_uint64_iget
+#define shmem_ctx_size_iget           pshmem_ctx_size_iget
+#define shmem_ctx_ptrdiff_iget        pshmem_ctx_ptrdiff_iget
+
+#define shmem_char_iget               pshmem_char_iget
+#define shmem_short_iget              pshmem_short_iget
+#define shmem_int_iget                pshmem_int_iget
+#define shmem_long_iget               pshmem_long_iget
+#define shmem_float_iget              pshmem_float_iget
+#define shmem_double_iget             pshmem_double_iget
+#define shmem_longlong_iget           pshmem_longlong_iget
+#define shmem_schar_iget              pshmem_schar_iget
+#define shmem_uchar_iget              pshmem_uchar_iget
+#define shmem_ushort_iget             pshmem_ushort_iget
+#define shmem_uint_iget               pshmem_uint_iget
+#define shmem_ulong_iget              pshmem_ulong_iget
+#define shmem_ulonglong_iget          pshmem_ulonglong_iget
+#define shmem_longdouble_iget         pshmem_longdouble_iget
+#define shmem_int8_iget               pshmem_int8_iget
+#define shmem_int16_iget              pshmem_int16_iget
+#define shmem_int32_iget              pshmem_int32_iget
+#define shmem_int64_iget              pshmem_int64_iget
+#define shmem_uint8_iget              pshmem_uint8_iget
+#define shmem_uint16_iget             pshmem_uint16_iget
+#define shmem_uint32_iget             pshmem_uint32_iget
+#define shmem_uint64_iget             pshmem_uint64_iget
+#define shmem_size_iget               pshmem_size_iget
+#define shmem_ptrdiff_iget            pshmem_ptrdiff_iget
+
+#define shmem_ctx_iget8              pshmem_ctx_iget8
+#define shmem_ctx_iget16             pshmem_ctx_iget16
+#define shmem_ctx_iget32             pshmem_ctx_iget32
+#define shmem_ctx_iget64             pshmem_ctx_iget64
+#define shmem_ctx_iget128            pshmem_ctx_iget128
+
+#define shmem_iget8                  pshmem_iget8
+#define shmem_iget16                 pshmem_iget16
+#define shmem_iget32                 pshmem_iget32
+#define shmem_iget64                 pshmem_iget64
+#define shmem_iget128                pshmem_iget128
+
+/*
+ * Non-block data get routines
+ */
+#define shmem_ctx_char_get_nbi           pshmem_ctx_char_get_nbi
+#define shmem_ctx_short_get_nbi          pshmem_ctx_short_get_nbi
+#define shmem_ctx_int_get_nbi            pshmem_ctx_int_get_nbi
+#define shmem_ctx_long_get_nbi           pshmem_ctx_long_get_nbi
+#define shmem_ctx_float_get_nbi          pshmem_ctx_float_get_nbi
+#define shmem_ctx_double_get_nbi         pshmem_ctx_double_get_nbi
+#define shmem_ctx_longlong_get_nbi       pshmem_ctx_longlong_get_nbi
+#define shmem_ctx_schar_get_nbi          pshmem_ctx_schar_get_nbi
+#define shmem_ctx_uchar_get_nbi          pshmem_ctx_uchar_get_nbi
+#define shmem_ctx_ushort_get_nbi         pshmem_ctx_ushort_get_nbi
+#define shmem_ctx_uint_get_nbi           pshmem_ctx_uint_get_nbi
+#define shmem_ctx_ulong_get_nbi          pshmem_ctx_ulong_get_nbi
+#define shmem_ctx_ulonglong_get_nbi      pshmem_ctx_ulonglong_get_nbi
+#define shmem_ctx_longdouble_get_nbi     pshmem_ctx_longdouble_get_nbi
+#define shmem_ctx_int8_get_nbi           pshmem_ctx_int8_get_nbi
+#define shmem_ctx_int16_get_nbi          pshmem_ctx_int16_get_nbi
+#define shmem_ctx_int32_get_nbi          pshmem_ctx_int32_get_nbi
+#define shmem_ctx_int64_get_nbi          pshmem_ctx_int64_get_nbi
+#define shmem_ctx_uint8_get_nbi          pshmem_ctx_uint8_get_nbi
+#define shmem_ctx_uint16_get_nbi         pshmem_ctx_uint16_get_nbi
+#define shmem_ctx_uint32_get_nbi         pshmem_ctx_uint32_get_nbi
+#define shmem_ctx_uint64_get_nbi         pshmem_ctx_uint64_get_nbi
+#define shmem_ctx_size_get_nbi           pshmem_ctx_size_get_nbi
+#define shmem_ctx_ptrdiff_get_nbi        pshmem_ctx_ptrdiff_get_nbi
+
+#define shmem_char_get_nbi               pshmem_char_get_nbi
+#define shmem_short_get_nbi              pshmem_short_get_nbi
+#define shmem_int_get_nbi                pshmem_int_get_nbi
+#define shmem_long_get_nbi               pshmem_long_get_nbi
+#define shmem_float_get_nbi              pshmem_float_get_nbi
+#define shmem_double_get_nbi             pshmem_double_get_nbi
+#define shmem_longlong_get_nbi           pshmem_longlong_get_nbi
+#define shmem_schar_get_nbi              pshmem_schar_get_nbi
+#define shmem_uchar_get_nbi              pshmem_uchar_get_nbi
+#define shmem_ushort_get_nbi             pshmem_ushort_get_nbi
+#define shmem_uint_get_nbi               pshmem_uint_get_nbi
+#define shmem_ulong_get_nbi              pshmem_ulong_get_nbi
+#define shmem_ulonglong_get_nbi          pshmem_ulonglong_get_nbi
+#define shmem_longdouble_get_nbi         pshmem_longdouble_get_nbi
+#define shmem_int8_get_nbi               pshmem_int8_get_nbi
+#define shmem_int16_get_nbi              pshmem_int16_get_nbi
+#define shmem_int32_get_nbi              pshmem_int32_get_nbi
+#define shmem_int64_get_nbi              pshmem_int64_get_nbi
+#define shmem_uint8_get_nbi              pshmem_uint8_get_nbi
+#define shmem_uint16_get_nbi             pshmem_uint16_get_nbi
+#define shmem_uint32_get_nbi             pshmem_uint32_get_nbi
+#define shmem_uint64_get_nbi             pshmem_uint64_get_nbi
+#define shmem_size_get_nbi               pshmem_size_get_nbi
+#define shmem_ptrdiff_get_nbi            pshmem_ptrdiff_get_nbi
+
+#define shmem_ctx_get8_nbi           pshmem_ctx_get8_nbi
+#define shmem_ctx_get16_nbi          pshmem_ctx_get16_nbi
+#define shmem_ctx_get32_nbi          pshmem_ctx_get32_nbi
+#define shmem_ctx_get64_nbi          pshmem_ctx_get64_nbi
+#define shmem_ctx_get128_nbi         pshmem_ctx_get128_nbi
+#define shmem_ctx_getmem_nbi         pshmem_ctx_getmem_nbi
+
+#define shmem_get8_nbi               pshmem_get8_nbi
+#define shmem_get16_nbi              pshmem_get16_nbi
+#define shmem_get32_nbi              pshmem_get32_nbi
+#define shmem_get64_nbi              pshmem_get64_nbi
+#define shmem_get128_nbi             pshmem_get128_nbi
+#define shmem_getmem_nbi             pshmem_getmem_nbi
+
+/*
+ * Atomic operations
+ */
+/* Atomic swap */
+#define shmem_ctx_double_atomic_swap pshmem_ctx_double_atomic_swap
+#define shmem_ctx_float_atomic_swap  pshmem_ctx_float_atomic_swap
+#define shmem_ctx_int_atomic_swap    pshmem_ctx_int_atomic_swap
+#define shmem_ctx_long_atomic_swap   pshmem_ctx_long_atomic_swap
+#define shmem_ctx_longlong_atomic_swap pshmem_ctx_longlong_atomic_swap
+#define shmem_ctx_uint_atomic_swap   pshmem_ctx_uint_atomic_swap
+#define shmem_ctx_ulong_atomic_swap  pshmem_ctx_ulong_atomic_swap
+#define shmem_ctx_ulonglong_atomic_swap pshmem_ctx_ulonglong_atomic_swap
+#define shmem_ctx_int32_atomic_swap    pshmem_ctx_int32_atomic_swap
+#define shmem_ctx_int64_atomic_swap    pshmem_ctx_int64_atomic_swap
+#define shmem_ctx_uint32_atomic_swap    pshmem_ctx_uint32_atomic_swap
+#define shmem_ctx_uint64_atomic_swap    pshmem_ctx_uint64_atomic_swap
+#define shmem_ctx_size_atomic_swap    pshmem_ctx_size_atomic_swap
+#define shmem_ctx_ptrdiff_atomic_swap    pshmem_ctx_ptrdiff_atomic_swap
+
+#define shmem_double_atomic_swap     pshmem_double_atomic_swap
+#define shmem_float_atomic_swap      pshmem_float_atomic_swap
+#define shmem_int_atomic_swap        pshmem_int_atomic_swap
+#define shmem_long_atomic_swap       pshmem_long_atomic_swap
+#define shmem_longlong_atomic_swap   pshmem_longlong_atomic_swap
+#define shmem_uint_atomic_swap       pshmem_uint_atomic_swap
+#define shmem_ulong_atomic_swap      pshmem_ulong_atomic_swap
+#define shmem_ulonglong_atomic_swap  pshmem_ulonglong_atomic_swap
+#define shmem_int32_atomic_swap    pshmem_int32_atomic_swap
+#define shmem_int64_atomic_swap    pshmem_int64_atomic_swap
+#define shmem_uint32_atomic_swap    pshmem_uint32_atomic_swap
+#define shmem_uint64_atomic_swap    pshmem_uint64_atomic_swap
+#define shmem_size_atomic_swap    pshmem_size_atomic_swap
+#define shmem_ptrdiff_atomic_swap    pshmem_ptrdiff_atomic_swap
+
+#define shmem_double_swap            pshmem_double_swap
+#define shmem_float_swap             pshmem_float_swap
+#define shmem_int_swap               pshmem_int_swap
+#define shmem_long_swap              pshmem_long_swap
+#define shmem_longlong_swap          pshmem_longlong_swap
+
+#define shmemx_int32_swap            pshmemx_int32_swap
+#define shmemx_int64_swap            pshmemx_int64_swap
+
+/* Atomic set */
+#define shmem_ctx_double_atomic_set pshmem_ctx_double_atomic_set
+#define shmem_ctx_float_atomic_set  pshmem_ctx_float_atomic_set
+#define shmem_ctx_int_atomic_set    pshmem_ctx_int_atomic_set
+#define shmem_ctx_long_atomic_set   pshmem_ctx_long_atomic_set
+#define shmem_ctx_longlong_atomic_set pshmem_ctx_longlong_atomic_set
+#define shmem_ctx_uint_atomic_set   pshmem_ctx_uint_atomic_set
+#define shmem_ctx_ulong_atomic_set  pshmem_ctx_ulong_atomic_set
+#define shmem_ctx_ulonglong_atomic_set pshmem_ctx_ulonglong_atomic_set
+#define shmem_ctx_int32_atomic_set  pshmem_ctx_int32_atomic_set
+#define shmem_ctx_int64_atomic_set  pshmem_ctx_int64_atomic_set
+#define shmem_ctx_uint32_atomic_set  pshmem_ctx_uint32_atomic_set
+#define shmem_ctx_uint64_atomic_set  pshmem_ctx_uint64_atomic_set
+#define shmem_ctx_size_atomic_set  pshmem_ctx_size_atomic_set
+#define shmem_ctx_ptrdiff_atomic_set  pshmem_ctx_ptrdiff_atomic_set
+
+#define shmem_double_atomic_set     pshmem_double_atomic_set
+#define shmem_float_atomic_set      pshmem_float_atomic_set
+#define shmem_int_atomic_set        pshmem_int_atomic_set
+#define shmem_long_atomic_set       pshmem_long_atomic_set
+#define shmem_longlong_atomic_set   pshmem_longlong_atomic_set
+#define shmem_uint_atomic_set       pshmem_uint_atomic_set
+#define shmem_ulong_atomic_set      pshmem_ulong_atomic_set
+#define shmem_ulonglong_atomic_set  pshmem_ulonglong_atomic_set
+#define shmem_int32_atomic_set      pshmem_int32_atomic_set
+#define shmem_int64_atomic_set      pshmem_int64_atomic_set
+#define shmem_uint32_atomic_set     pshmem_uint32_atomic_set
+#define shmem_uint64_atomic_set     pshmem_uint64_atomic_set
+#define shmem_size_atomic_set       pshmem_size_atomic_set
+#define shmem_ptrdiff_atomic_set    pshmem_ptrdiff_atomic_set
+
+#define shmem_double_set            pshmem_double_set
+#define shmem_float_set             pshmem_float_set
+#define shmem_int_set               pshmem_int_set
+#define shmem_long_set              pshmem_long_set
+#define shmem_longlong_set          pshmem_longlong_set
+
+#define shmemx_int32_set            pshmemx_int32_set
+#define shmemx_int64_set            pshmemx_int64_set
+
+/* Atomic conditional swap */
+#define shmem_ctx_int_atomic_compare_swap   pshmem_ctx_int_atomic_compare_swap
+#define shmem_ctx_long_atomic_compare_swap  pshmem_ctx_long_atomic_compare_swap
+#define shmem_ctx_longlong_atomic_compare_swap pshmem_ctx_longlong_atomic_compare_swap
+#define shmem_ctx_uint_atomic_compare_swap  pshmem_ctx_uint_atomic_compare_swap
+#define shmem_ctx_ulong_atomic_compare_swap pshmem_ctx_ulong_atomic_compare_swap
+#define shmem_ctx_ulonglong_atomic_compare_swap pshmem_ctx_ulonglong_atomic_compare_swap
+#define shmem_ctx_int32_atomic_compare_swap   pshmem_ctx_int32_atomic_compare_swap
+#define shmem_ctx_int64_atomic_compare_swap   pshmem_ctx_int64_atomic_compare_swap
+#define shmem_ctx_uint32_atomic_compare_swap   pshmem_ctx_uint32_atomic_compare_swap
+#define shmem_ctx_uint64_atomic_compare_swap   pshmem_ctx_uint64_atomic_compare_swap
+#define shmem_ctx_size_atomic_compare_swap   pshmem_ctx_size_atomic_compare_swap
+#define shmem_ctx_ptrdiff_atomic_compare_swap   pshmem_ctx_ptrdiff_atomic_compare_swap
+
+#define shmem_int_atomic_compare_swap       pshmem_int_atomic_compare_swap
+#define shmem_long_atomic_compare_swap      pshmem_long_atomic_compare_swap
+#define shmem_longlong_atomic_compare_swap  pshmem_longlong_atomic_compare_swap
+#define shmem_uint_atomic_compare_swap      pshmem_uint_atomic_compare_swap
+#define shmem_ulong_atomic_compare_swap     pshmem_ulong_atomic_compare_swap
+#define shmem_ulonglong_atomic_compare_swap pshmem_ulonglong_atomic_compare_swap
+#define shmem_int32_atomic_compare_swap   pshmem_int32_atomic_compare_swap
+#define shmem_int64_atomic_compare_swap   pshmem_int64_atomic_compare_swap
+#define shmem_uint32_atomic_compare_swap   pshmem_uint32_atomic_compare_swap
+#define shmem_uint64_atomic_compare_swap   pshmem_uint64_atomic_compare_swap
+#define shmem_size_atomic_compare_swap   pshmem_size_atomic_compare_swap
+#define shmem_ptrdiff_atomic_compare_swap   pshmem_ptrdiff_atomic_compare_swap
+
+#define shmem_int_cswap              pshmem_int_cswap
+#define shmem_long_cswap             pshmem_long_cswap
+#define shmem_longlong_cswap         pshmem_longlong_cswap
+
+#define shmemx_int32_cswap           pshmemx_int32_cswap
+#define shmemx_int64_cswap           pshmemx_int64_cswap
+
+/* Atomic Fetch&Add */
+#define shmem_ctx_int_atomic_fetch_add       pshmem_ctx_int_atomic_fetch_add
+#define shmem_ctx_long_atomic_fetch_add      pshmem_ctx_long_atomic_fetch_add
+#define shmem_ctx_longlong_atomic_fetch_add  pshmem_ctx_longlong_atomic_fetch_add
+#define shmem_ctx_uint_atomic_fetch_add      pshmem_ctx_uint_atomic_fetch_add
+#define shmem_ctx_ulong_atomic_fetch_add     pshmem_ctx_ulong_atomic_fetch_add
+#define shmem_ctx_ulonglong_atomic_fetch_add pshmem_ctx_ulonglong_atomic_fetch_add
+#define shmem_ctx_int32_atomic_fetch_add   pshmem_ctx_int32_atomic_fetch_add
+#define shmem_ctx_int64_atomic_fetch_add   pshmem_ctx_int64_atomic_fetch_add
+#define shmem_ctx_uint32_atomic_fetch_add   pshmem_ctx_uint32_atomic_fetch_add
+#define shmem_ctx_uint64_atomic_fetch_add   pshmem_ctx_uint64_atomic_fetch_add
+#define shmem_ctx_size_atomic_fetch_add   pshmem_ctx_size_atomic_fetch_add
+#define shmem_ctx_ptrdiff_atomic_fetch_add   pshmem_ctx_ptrdiff_atomic_fetch_add
+
+#define shmem_int_atomic_fetch_add           pshmem_int_atomic_fetch_add
+#define shmem_long_atomic_fetch_add          pshmem_long_atomic_fetch_add
+#define shmem_longlong_atomic_fetch_add      pshmem_longlong_atomic_fetch_add
+#define shmem_uint_atomic_fetch_add          pshmem_uint_atomic_fetch_add
+#define shmem_ulong_atomic_fetch_add         pshmem_ulong_atomic_fetch_add
+#define shmem_ulonglong_atomic_fetch_add     pshmem_ulonglong_atomic_fetch_add
+#define shmem_int32_atomic_fetch_add   pshmem_int32_atomic_fetch_add
+#define shmem_int64_atomic_fetch_add   pshmem_int64_atomic_fetch_add
+#define shmem_uint32_atomic_fetch_add   pshmem_uint32_atomic_fetch_add
+#define shmem_uint64_atomic_fetch_add   pshmem_uint64_atomic_fetch_add
+#define shmem_size_atomic_fetch_add   pshmem_size_atomic_fetch_add
+#define shmem_ptrdiff_atomic_fetch_add   pshmem_ptrdiff_atomic_fetch_add
+
+#define shmem_int_fadd                       pshmem_int_fadd
+#define shmem_long_fadd                      pshmem_long_fadd
+#define shmem_longlong_fadd                  pshmem_longlong_fadd
+
+#define shmemx_int32_fadd                    pshmemx_int32_fadd
+#define shmemx_int64_fadd                    pshmemx_int64_fadd
+
+/* Atomic Fetch&And */
+#define shmem_int_atomic_fetch_and        pshmem_int_atomic_fetch_and
+#define shmem_long_atomic_fetch_and       pshmem_long_atomic_fetch_and
+#define shmem_longlong_atomic_fetch_and   pshmem_longlong_atomic_fetch_and
+#define shmem_uint_atomic_fetch_and       pshmem_uint_atomic_fetch_and
+#define shmem_ulong_atomic_fetch_and      pshmem_ulong_atomic_fetch_and
+#define shmem_ulonglong_atomic_fetch_and  pshmem_ulonglong_atomic_fetch_and
+#define shmem_int32_atomic_fetch_and      pshmem_int32_atomic_fetch_and
+#define shmem_int64_atomic_fetch_and      pshmem_int64_atomic_fetch_and
+#define shmem_uint32_atomic_fetch_and     pshmem_uint32_atomic_fetch_and
+#define shmem_uint64_atomic_fetch_and     pshmem_uint64_atomic_fetch_and
+
+#define shmem_ctx_int_atomic_fetch_and    pshmem_ctx_int_atomic_fetch_and
+#define shmem_ctx_long_atomic_fetch_and   pshmem_ctx_long_atomic_fetch_and
+#define shmem_ctx_longlong_atomic_fetch_and pshmem_ctx_longlong_atomic_fetch_and
+#define shmem_ctx_uint_atomic_fetch_and   pshmem_ctx_uint_atomic_fetch_and
+#define shmem_ctx_ulong_atomic_fetch_and  pshmem_ctx_ulong_atomic_fetch_and
+#define shmem_ctx_ulonglong_atomic_fetch_and pshmem_ctx_ulonglong_atomic_fetch_and
+#define shmem_ctx_int32_atomic_fetch_and  pshmem_ctx_int32_atomic_fetch_and
+#define shmem_ctx_int64_atomic_fetch_and  pshmem_ctx_int64_atomic_fetch_and
+#define shmem_ctx_uint32_atomic_fetch_and pshmem_ctx_uint32_atomic_fetch_and
+#define shmem_ctx_uint64_atomic_fetch_and pshmem_ctx_uint64_atomic_fetch_and
+
+#define shmemx_int32_atomic_fetch_and     pshmemx_int32_atomic_fetch_and
+#define shmemx_int64_atomic_fetch_and     pshmemx_int64_atomic_fetch_and
+#define shmemx_uint32_atomic_fetch_and    pshmemx_uint32_atomic_fetch_and
+#define shmemx_uint64_atomic_fetch_and    pshmemx_uint64_atomic_fetch_and
+
+/* Atomic Fetch&Or */
+#define shmem_int_atomic_fetch_or         pshmem_int_atomic_fetch_or
+#define shmem_long_atomic_fetch_or        pshmem_long_atomic_fetch_or
+#define shmem_longlong_atomic_fetch_or    pshmem_longlong_atomic_fetch_or
+#define shmem_uint_atomic_fetch_or        pshmem_uint_atomic_fetch_or
+#define shmem_ulong_atomic_fetch_or       pshmem_ulong_atomic_fetch_or
+#define shmem_ulonglong_atomic_fetch_or   pshmem_ulonglong_atomic_fetch_or
+#define shmem_int32_atomic_fetch_or       pshmem_int32_atomic_fetch_or
+#define shmem_int64_atomic_fetch_or       pshmem_int64_atomic_fetch_or
+#define shmem_uint32_atomic_fetch_or      pshmem_uint32_atomic_fetch_or
+#define shmem_uint64_atomic_fetch_or      pshmem_uint64_atomic_fetch_or
+
+#define shmem_ctx_int_atomic_fetch_or     pshmem_ctx_int_atomic_fetch_or
+#define shmem_ctx_long_atomic_fetch_or    pshmem_ctx_long_atomic_fetch_or
+#define shmem_ctx_longlong_atomic_fetch_or pshmem_ctx_longlong_atomic_fetch_or
+#define shmem_ctx_uint_atomic_fetch_or    pshmem_ctx_uint_atomic_fetch_or
+#define shmem_ctx_ulong_atomic_fetch_or   pshmem_ctx_ulong_atomic_fetch_or
+#define shmem_ctx_ulonglong_atomic_fetch_or pshmem_ctx_ulonglong_atomic_fetch_or
+#define shmem_ctx_int32_atomic_fetch_or   pshmem_ctx_int32_atomic_fetch_or
+#define shmem_ctx_int64_atomic_fetch_or   pshmem_ctx_int64_atomic_fetch_or
+#define shmem_ctx_uint32_atomic_fetch_or  pshmem_ctx_uint32_atomic_fetch_or
+#define shmem_ctx_uint64_atomic_fetch_or  pshmem_ctx_uint64_atomic_fetch_or
+
+#define shmemx_int32_atomic_fetch_or      pshmemx_int32_atomic_fetch_or
+#define shmemx_int64_atomic_fetch_or      pshmemx_int64_atomic_fetch_or
+#define shmemx_uint32_atomic_fetch_or     pshmemx_uint32_atomic_fetch_or
+#define shmemx_uint64_atomic_fetch_or     pshmemx_uint64_atomic_fetch_or
+
+/* Atomic Fetch&Xor */
+#define shmem_int_atomic_fetch_xor        pshmem_int_atomic_fetch_xor
+#define shmem_long_atomic_fetch_xor       pshmem_long_atomic_fetch_xor
+#define shmem_longlong_atomic_fetch_xor   pshmem_longlong_atomic_fetch_xor
+#define shmem_uint_atomic_fetch_xor       pshmem_uint_atomic_fetch_xor
+#define shmem_ulong_atomic_fetch_xor      pshmem_ulong_atomic_fetch_xor
+#define shmem_ulonglong_atomic_fetch_xor  pshmem_ulonglong_atomic_fetch_xor
+#define shmem_int32_atomic_fetch_xor      pshmem_int32_atomic_fetch_xor
+#define shmem_int64_atomic_fetch_xor      pshmem_int64_atomic_fetch_xor
+#define shmem_uint32_atomic_fetch_xor     pshmem_uint32_atomic_fetch_xor
+#define shmem_uint64_atomic_fetch_xor     pshmem_uint64_atomic_fetch_xor
+
+#define shmem_ctx_int_atomic_fetch_xor    pshmem_ctx_int_atomic_fetch_xor
+#define shmem_ctx_long_atomic_fetch_xor   pshmem_ctx_long_atomic_fetch_xor
+#define shmem_ctx_longlong_atomic_fetch_xor pshmem_ctx_longlong_atomic_fetch_xor
+#define shmem_ctx_uint_atomic_fetch_xor   pshmem_ctx_uint_atomic_fetch_xor
+#define shmem_ctx_ulong_atomic_fetch_xor  pshmem_ctx_ulong_atomic_fetch_xor
+#define shmem_ctx_ulonglong_atomic_fetch_xor pshmem_ctx_ulonglong_atomic_fetch_xor
+#define shmem_ctx_int32_atomic_fetch_xor  pshmem_ctx_int32_atomic_fetch_xor
+#define shmem_ctx_int64_atomic_fetch_xor  pshmem_ctx_int64_atomic_fetch_xor
+#define shmem_ctx_uint32_atomic_fetch_xor pshmem_ctx_uint32_atomic_fetch_xor
+#define shmem_ctx_uint64_atomic_fetch_xor pshmem_ctx_uint64_atomic_fetch_xor
+
+#define shmemx_int32_atomic_fetch_xor     pshmemx_int32_atomic_fetch_xor
+#define shmemx_int64_atomic_fetch_xor     pshmemx_int64_atomic_fetch_xor
+#define shmemx_uint32_atomic_fetch_xor    pshmemx_uint32_atomic_fetch_xor
+#define shmemx_uint64_atomic_fetch_xor    pshmemx_uint64_atomic_fetch_xor
+
+/* Atomic Fetch */
+#define shmem_ctx_double_atomic_fetch pshmem_ctx_double_atomic_fetch
+#define shmem_ctx_float_atomic_fetch  pshmem_ctx_float_atomic_fetch
+#define shmem_ctx_int_atomic_fetch    pshmem_ctx_int_atomic_fetch
+#define shmem_ctx_long_atomic_fetch   pshmem_ctx_long_atomic_fetch
+#define shmem_ctx_longlong_atomic_fetch pshmem_ctx_longlong_atomic_fetch
+#define shmem_ctx_uint_atomic_fetch   pshmem_ctx_uint_atomic_fetch
+#define shmem_ctx_ulong_atomic_fetch  pshmem_ctx_ulong_atomic_fetch
+#define shmem_ctx_ulonglong_atomic_fetch pshmem_ctx_ulonglong_atomic_fetch
+#define shmem_ctx_int32_atomic_fetch    pshmem_ctx_int32_atomic_fetch
+#define shmem_ctx_int64_atomic_fetch    pshmem_ctx_int64_atomic_fetch
+#define shmem_ctx_uint32_atomic_fetch    pshmem_ctx_uint32_atomic_fetch
+#define shmem_ctx_uint64_atomic_fetch    pshmem_ctx_uint64_atomic_fetch
+#define shmem_ctx_size_atomic_fetch    pshmem_ctx_size_atomic_fetch
+#define shmem_ctx_ptrdiff_atomic_fetch    pshmem_ctx_ptrdiff_atomic_fetch
+
+#define shmem_double_atomic_fetch     pshmem_double_atomic_fetch
+#define shmem_float_atomic_fetch      pshmem_float_atomic_fetch
+#define shmem_int_atomic_fetch        pshmem_int_atomic_fetch
+#define shmem_long_atomic_fetch       pshmem_long_atomic_fetch
+#define shmem_longlong_atomic_fetch   pshmem_longlong_atomic_fetch
+#define shmem_uint_atomic_fetch       pshmem_uint_atomic_fetch
+#define shmem_ulong_atomic_fetch      pshmem_ulong_atomic_fetch
+#define shmem_ulonglong_atomic_fetch  pshmem_ulonglong_atomic_fetch
+#define shmem_int32_atomic_fetch    pshmem_int32_atomic_fetch
+#define shmem_int64_atomic_fetch    pshmem_int64_atomic_fetch
+#define shmem_uint32_atomic_fetch    pshmem_uint32_atomic_fetch
+#define shmem_uint64_atomic_fetch    pshmem_uint64_atomic_fetch
+#define shmem_size_atomic_fetch    pshmem_size_atomic_fetch
+#define shmem_ptrdiff_atomic_fetch    pshmem_ptrdiff_atomic_fetch
+
+#define shmem_double_fetch            pshmem_double_fetch
+#define shmem_float_fetch             pshmem_float_fetch
+#define shmem_int_fetch               pshmem_int_fetch
+#define shmem_long_fetch              pshmem_long_fetch
+#define shmem_longlong_fetch          pshmem_longlong_fetch
+
+#define shmemx_int32_fetch            pshmemx_int32_fetch
+#define shmemx_int64_fetch            pshmemx_int64_fetch
+
+/* Atomic Fetch&Inc */
+#define shmem_ctx_int_atomic_fetch_inc    pshmem_ctx_int_atomic_fetch_inc
+#define shmem_ctx_long_atomic_fetch_inc   pshmem_ctx_long_atomic_fetch_inc
+#define shmem_ctx_longlong_atomic_fetch_inc pshmem_ctx_longlong_atomic_fetch_inc
+#define shmem_ctx_uint_atomic_fetch_inc    pshmem_ctx_uint_atomic_fetch_inc
+#define shmem_ctx_ulong_atomic_fetch_inc   pshmem_ctx_ulong_atomic_fetch_inc
+#define shmem_ctx_ulonglong_atomic_fetch_inc pshmem_ctx_ulonglong_atomic_fetch_inc
+#define shmem_ctx_int32_atomic_fetch_inc    pshmem_ctx_int32_atomic_fetch_inc
+#define shmem_ctx_int64_atomic_fetch_inc    pshmem_ctx_int64_atomic_fetch_inc
+#define shmem_ctx_uint32_atomic_fetch_inc    pshmem_ctx_uint32_atomic_fetch_inc
+#define shmem_ctx_uint64_atomic_fetch_inc    pshmem_ctx_uint64_atomic_fetch_inc
+#define shmem_ctx_size_atomic_fetch_inc    pshmem_ctx_size_atomic_fetch_inc
+#define shmem_ctx_ptrdiff_atomic_fetch_inc    pshmem_ctx_ptrdiff_atomic_fetch_inc
+
+#define shmem_uint_atomic_fetch_inc        pshmem_uint_atomic_fetch_inc
+#define shmem_ulong_atomic_fetch_inc       pshmem_ulong_atomic_fetch_inc
+#define shmem_ulonglong_atomic_fetch_inc   pshmem_ulonglong_atomic_fetch_inc
+#define shmem_int_atomic_fetch_inc        pshmem_int_atomic_fetch_inc
+#define shmem_long_atomic_fetch_inc       pshmem_long_atomic_fetch_inc
+#define shmem_longlong_atomic_fetch_inc   pshmem_longlong_atomic_fetch_inc
+#define shmem_int32_atomic_fetch_inc    pshmem_int32_atomic_fetch_inc
+#define shmem_int64_atomic_fetch_inc    pshmem_int64_atomic_fetch_inc
+#define shmem_uint32_atomic_fetch_inc    pshmem_uint32_atomic_fetch_inc
+#define shmem_uint64_atomic_fetch_inc    pshmem_uint64_atomic_fetch_inc
+#define shmem_size_atomic_fetch_inc    pshmem_size_atomic_fetch_inc
+#define shmem_ptrdiff_atomic_fetch_inc    pshmem_ptrdiff_atomic_fetch_inc
+
+
+
+#define shmem_int_finc               pshmem_int_finc
+#define shmem_long_finc              pshmem_long_finc
+#define shmem_longlong_finc          pshmem_longlong_finc
+
+#define shmemx_int32_finc            pshmemx_int32_finc
+#define shmemx_int64_finc            pshmemx_int64_finc
+
+/* Atomic Add */
+#define shmem_ctx_int_atomic_add     pshmem_ctx_int_atomic_add
+#define shmem_ctx_long_atomic_add    pshmem_ctx_long_atomic_add
+#define shmem_ctx_longlong_atomic_add pshmem_ctx_longlong_atomic_add
+#define shmem_ctx_uint_atomic_add    pshmem_ctx_uint_atomic_add
+#define shmem_ctx_ulong_atomic_add   pshmem_ctx_ulong_atomic_add
+#define shmem_ctx_ulonglong_atomic_add pshmem_ctx_ulonglong_atomic_add
+#define shmem_ctx_int32_atomic_add     pshmem_ctx_int32_atomic_add
+#define shmem_ctx_int64_atomic_add     pshmem_ctx_int64_atomic_add
+#define shmem_ctx_uint32_atomic_add     pshmem_ctx_uint32_atomic_add
+#define shmem_ctx_uint64_atomic_add     pshmem_ctx_uint64_atomic_add
+#define shmem_ctx_size_atomic_add     pshmem_ctx_size_atomic_add
+#define shmem_ctx_ptrdiff_atomic_add     pshmem_ctx_ptrdiff_atomic_add
+
+#define shmem_int_atomic_add         pshmem_int_atomic_add
+#define shmem_long_atomic_add        pshmem_long_atomic_add
+#define shmem_longlong_atomic_add    pshmem_longlong_atomic_add
+#define shmem_uint_atomic_add        pshmem_uint_atomic_add
+#define shmem_ulong_atomic_add       pshmem_ulong_atomic_add
+#define shmem_ulonglong_atomic_add   pshmem_ulonglong_atomic_add
+#define shmem_int32_atomic_add     pshmem_int32_atomic_add
+#define shmem_int64_atomic_add     pshmem_int64_atomic_add
+#define shmem_uint32_atomic_add     pshmem_uint32_atomic_add
+#define shmem_uint64_atomic_add     pshmem_uint64_atomic_add
+#define shmem_size_atomic_add     pshmem_size_atomic_add
+#define shmem_ptrdiff_atomic_add     pshmem_ptrdiff_atomic_add
+
+#define shmem_int_add                pshmem_int_add
+#define shmem_long_add               pshmem_long_add
+#define shmem_longlong_add           pshmem_longlong_add
+
+#define shmemx_int32_add             pshmemx_int32_add
+#define shmemx_int64_add             pshmemx_int64_add
+
+/* Atomic And */
+#define shmem_int_atomic_and         pshmem_int_atomic_and
+#define shmem_long_atomic_and        pshmem_long_atomic_and
+#define shmem_longlong_atomic_and    pshmem_longlong_atomic_and
+#define shmem_uint_atomic_and        pshmem_uint_atomic_and
+#define shmem_ulong_atomic_and       pshmem_ulong_atomic_and
+#define shmem_ulonglong_atomic_and   pshmem_ulonglong_atomic_and
+#define shmem_int32_atomic_and       pshmem_int32_atomic_and
+#define shmem_int64_atomic_and       pshmem_int64_atomic_and
+#define shmem_uint32_atomic_and      pshmem_uint32_atomic_and
+#define shmem_uint64_atomic_and      pshmem_uint64_atomic_and
+
+#define shmem_ctx_int_atomic_and     pshmem_ctx_int_atomic_and
+#define shmem_ctx_long_atomic_and    pshmem_ctx_long_atomic_and
+#define shmem_ctx_longlong_atomic_and pshmem_ctx_longlong_atomic_and
+#define shmem_ctx_uint_atomic_and    pshmem_ctx_uint_atomic_and
+#define shmem_ctx_ulong_atomic_and   pshmem_ctx_ulong_atomic_and
+#define shmem_ctx_ulonglong_atomic_and pshmem_ctx_ulonglong_atomic_and
+#define shmem_ctx_int32_atomic_and   pshmem_ctx_int32_atomic_and
+#define shmem_ctx_int64_atomic_and   pshmem_ctx_int64_atomic_and
+#define shmem_ctx_uint32_atomic_and  pshmem_ctx_uint32_atomic_and
+#define shmem_ctx_uint64_atomic_and  pshmem_ctx_uint64_atomic_and
+
+#define shmemx_int32_atomic_and      pshmemx_int32_atomic_and
+#define shmemx_int64_atomic_and      pshmemx_int64_atomic_and
+
+#define shmemx_uint32_atomic_and     pshmemx_uint32_atomic_and
+#define shmemx_uint64_atomic_and     pshmemx_uint64_atomic_and
+
+/* Atomic Or */
+#define shmem_int_atomic_or          pshmem_int_atomic_or
+#define shmem_long_atomic_or         pshmem_long_atomic_or
+#define shmem_longlong_atomic_or     pshmem_longlong_atomic_or
+#define shmem_uint_atomic_or         pshmem_uint_atomic_or
+#define shmem_ulong_atomic_or        pshmem_ulong_atomic_or
+#define shmem_ulonglong_atomic_or    pshmem_ulonglong_atomic_or
+#define shmem_int32_atomic_or        pshmem_int32_atomic_or
+#define shmem_int64_atomic_or        pshmem_int64_atomic_or
+#define shmem_uint32_atomic_or       pshmem_uint32_atomic_or
+#define shmem_uint64_atomic_or       pshmem_uint64_atomic_or
+
+#define shmem_ctx_int_atomic_or      pshmem_ctx_int_atomic_or
+#define shmem_ctx_long_atomic_or     pshmem_ctx_long_atomic_or
+#define shmem_ctx_longlong_atomic_or pshmem_ctx_longlong_atomic_or
+#define shmem_ctx_uint_atomic_or     pshmem_ctx_uint_atomic_or
+#define shmem_ctx_ulong_atomic_or    pshmem_ctx_ulong_atomic_or
+#define shmem_ctx_ulonglong_atomic_or pshmem_ctx_ulonglong_atomic_or
+#define shmem_ctx_int32_atomic_or    pshmem_ctx_int32_atomic_or
+#define shmem_ctx_int64_atomic_or    pshmem_ctx_int64_atomic_or
+#define shmem_ctx_uint32_atomic_or   pshmem_ctx_uint32_atomic_or
+#define shmem_ctx_uint64_atomic_or   pshmem_ctx_uint64_atomic_or
+
+#define shmemx_int32_atomic_or       pshmemx_int32_atomic_or
+#define shmemx_int64_atomic_or       pshmemx_int64_atomic_or
+
+#define shmemx_uint32_atomic_or      pshmemx_uint32_atomic_or
+#define shmemx_uint64_atomic_or      pshmemx_uint64_atomic_or
+
+/* Atomic Xor */
+#define shmem_int_atomic_xor         pshmem_int_atomic_xor
+#define shmem_long_atomic_xor        pshmem_long_atomic_xor
+#define shmem_longlong_atomic_xor    pshmem_longlong_atomic_xor
+#define shmem_uint_atomic_xor        pshmem_uint_atomic_xor
+#define shmem_ulong_atomic_xor       pshmem_ulong_atomic_xor
+#define shmem_ulonglong_atomic_xor   pshmem_ulonglong_atomic_xor
+#define shmem_int32_atomic_xor       pshmem_int32_atomic_xor
+#define shmem_int64_atomic_xor       pshmem_int64_atomic_xor
+#define shmem_uint32_atomic_xor      pshmem_uint32_atomic_xor
+#define shmem_uint64_atomic_xor      pshmem_uint64_atomic_xor
+
+#define shmem_ctx_int_atomic_xor     pshmem_ctx_int_atomic_xor
+#define shmem_ctx_long_atomic_xor    pshmem_ctx_long_atomic_xor
+#define shmem_ctx_longlong_atomic_xor pshmem_ctx_longlong_atomic_xor
+#define shmem_ctx_uint_atomic_xor    pshmem_ctx_uint_atomic_xor
+#define shmem_ctx_ulong_atomic_xor   pshmem_ctx_ulong_atomic_xor
+#define shmem_ctx_ulonglong_atomic_xor pshmem_ctx_ulonglong_atomic_xor
+#define shmem_ctx_int32_atomic_xor   pshmem_ctx_int32_atomic_xor
+#define shmem_ctx_int64_atomic_xor   pshmem_ctx_int64_atomic_xor
+#define shmem_ctx_uint32_atomic_xor  pshmem_ctx_uint32_atomic_xor
+#define shmem_ctx_uint64_atomic_xor  pshmem_ctx_uint64_atomic_xor
+
+#define shmemx_int32_atomic_xor      pshmemx_int32_atomic_xor
+#define shmemx_int64_atomic_xor      pshmemx_int64_atomic_xor
+
+#define shmemx_uint32_atomic_xor     pshmemx_uint32_atomic_xor
+#define shmemx_uint64_atomic_xor     pshmemx_uint64_atomic_xor
+
+/* Atomic Inc */
+#define shmem_ctx_int_atomic_inc     pshmem_ctx_int_atomic_inc
+#define shmem_ctx_long_atomic_inc    pshmem_ctx_long_atomic_inc
+#define shmem_ctx_longlong_atomic_inc pshmem_ctx_longlong_atomic_inc
+#define shmem_ctx_uint_atomic_inc    pshmem_ctx_uint_atomic_inc
+#define shmem_ctx_ulong_atomic_inc   pshmem_ctx_ulong_atomic_inc
+#define shmem_ctx_ulonglong_atomic_inc pshmem_ctx_ulonglong_atomic_inc
+#define shmem_ctx_int32_atomic_inc     pshmem_ctx_int32_atomic_inc
+#define shmem_ctx_int64_atomic_inc     pshmem_ctx_int64_atomic_inc
+#define shmem_ctx_uint32_atomic_inc     pshmem_ctx_uint32_atomic_inc
+#define shmem_ctx_uint64_atomic_inc     pshmem_ctx_uint64_atomic_inc
+#define shmem_ctx_size_atomic_inc     pshmem_ctx_size_atomic_inc
+#define shmem_ctx_ptrdiff_atomic_inc     pshmem_ctx_ptrdiff_atomic_inc
+
+#define shmem_int_atomic_inc         pshmem_int_atomic_inc
+#define shmem_long_atomic_inc        pshmem_long_atomic_inc
+#define shmem_longlong_atomic_inc    pshmem_longlong_atomic_inc
+#define shmem_uint_atomic_inc        pshmem_uint_atomic_inc
+#define shmem_ulong_atomic_inc       pshmem_ulong_atomic_inc
+#define shmem_ulonglong_atomic_inc   pshmem_ulonglong_atomic_inc
+#define shmem_int32_atomic_inc     pshmem_int32_atomic_inc
+#define shmem_int64_atomic_inc     pshmem_int64_atomic_inc
+#define shmem_uint32_atomic_inc     pshmem_uint32_atomic_inc
+#define shmem_uint64_atomic_inc     pshmem_uint64_atomic_inc
+#define shmem_size_atomic_inc     pshmem_size_atomic_inc
+#define shmem_ptrdiff_atomic_inc     pshmem_ptrdiff_atomic_inc
+
+#define shmem_int_inc                pshmem_int_inc
+#define shmem_long_inc               pshmem_long_inc
+#define shmem_longlong_inc           pshmem_longlong_inc
+
+#define shmemx_int32_inc             pshmemx_int32_inc
+#define shmemx_int64_inc             pshmemx_int64_inc
+
+/* Nonblocking Atomic Fetch */
+#define shmem_ctx_double_atomic_fetch_nbi           pshmem_ctx_double_atomic_fetch_nbi
+#define shmem_ctx_float_atomic_fetch_nbi            pshmem_ctx_float_atomic_fetch_nbi
+#define shmem_ctx_int_atomic_fetch_nbi              pshmem_ctx_int_atomic_fetch_nbi
+#define shmem_ctx_long_atomic_fetch_nbi             pshmem_ctx_long_atomic_fetch_nbi
+#define shmem_ctx_longlong_atomic_fetch_nbi         pshmem_ctx_longlong_atomic_fetch_nbi
+#define shmem_ctx_uint_atomic_fetch_nbi             pshmem_ctx_uint_atomic_fetch_nbi
+#define shmem_ctx_ulong_atomic_fetch_nbi            pshmem_ctx_ulong_atomic_fetch_nbi
+#define shmem_ctx_ulonglong_atomic_fetch_nbi        pshmem_ctx_ulonglong_atomic_fetch_nbi
+#define shmem_ctx_int32_atomic_fetch_nbi            pshmem_ctx_int32_atomic_fetch_nbi
+#define shmem_ctx_int64_atomic_fetch_nbi            pshmem_ctx_int64_atomic_fetch_nbi
+#define shmem_ctx_uint32_atomic_fetch_nbi           pshmem_ctx_uint32_atomic_fetch_nbi
+#define shmem_ctx_uint64_atomic_fetch_nbi           pshmem_ctx_uint64_atomic_fetch_nbi
+#define shmem_ctx_size_atomic_fetch_nbi             pshmem_ctx_size_atomic_fetch_nbi
+#define shmem_ctx_ptrdiff_atomic_fetch_nbi          pshmem_ctx_ptrdiff_atomic_fetch_nbi
+
+#define shmem_double_atomic_fetch_nbi     		pshmem_double_atomic_fetch_nbi
+#define shmem_float_atomic_fetch_nbi      		pshmem_float_atomic_fetch_nbi
+#define shmem_int_atomic_fetch_nbi        		pshmem_int_atomic_fetch_nbi
+#define shmem_long_atomic_fetch_nbi       		pshmem_long_atomic_fetch_nbi
+#define shmem_longlong_atomic_fetch_nbi   		pshmem_longlong_atomic_fetch_nbi
+#define shmem_uint_atomic_fetch_nbi       		pshmem_uint_atomic_fetch_nbi
+#define shmem_ulong_atomic_fetch_nbi      		pshmem_ulong_atomic_fetch_nbi
+#define shmem_ulonglong_atomic_fetch_nbi  		pshmem_ulonglong_atomic_fetch_nbi
+#define shmem_int32_atomic_fetch_nbi      		pshmem_int32_atomic_fetch_nbi
+#define shmem_int64_atomic_fetch_nbi      		pshmem_int64_atomic_fetch_nbi
+#define shmem_uint32_atomic_fetch_nbi     		pshmem_uint32_atomic_fetch_nbi
+#define shmem_uint64_atomic_fetch_nbi     		pshmem_uint64_atomic_fetch_nbi
+#define shmem_size_atomic_fetch_nbi       		pshmem_size_atomic_fetch_nbi
+#define shmem_ptrdiff_atomic_fetch_nbi    		pshmem_ptrdiff_atomic_fetch_nbi
+
+
+/* Nonblocking Atomic Compare Swap */
+#define shmem_ctx_int_atomic_compare_swap_nbi               pshmem_ctx_int_atomic_compare_swap_nbi
+#define shmem_ctx_long_atomic_compare_swap_nbi              pshmem_ctx_long_atomic_compare_swap_nbi
+#define shmem_ctx_longlong_atomic_compare_swap_nbi          pshmem_ctx_longlong_atomic_compare_swap_nbi
+#define shmem_ctx_uint_atomic_compare_swap_nbi              pshmem_ctx_uint_atomic_compare_swap_nbi
+#define shmem_ctx_ulong_atomic_compare_swap_nbi             pshmem_ctx_ulong_atomic_compare_swap_nbi
+#define shmem_ctx_ulonglong_atomic_compare_swap_nbi         pshmem_ctx_ulonglong_atomic_compare_swap_nbi
+#define shmem_ctx_int32_atomic_compare_swap_nbi             pshmem_ctx_int32_atomic_compare_swap_nbi
+#define shmem_ctx_int64_atomic_compare_swap_nbi             pshmem_ctx_int64_atomic_compare_swap_nbi
+#define shmem_ctx_uint32_atomic_compare_swap_nbi            pshmem_ctx_uint32_atomic_compare_swap_nbi
+#define shmem_ctx_uint64_atomic_compare_swap_nbi            pshmem_ctx_uint64_atomic_compare_swap_nbi
+#define shmem_ctx_size_atomic_compare_swap_nbi              pshmem_ctx_size_atomic_compare_swap_nbi
+#define shmem_ctx_ptrdiff_atomic_compare_swap_nbi           pshmem_ctx_ptrdiff_atomic_compare_swap_nbi
+
+#define shmem_int_atomic_compare_swap_nbi        	pshmem_int_atomic_compare_swap_nbi
+#define shmem_long_atomic_compare_swap_nbi       	pshmem_long_atomic_compare_swap_nbi
+#define shmem_longlong_atomic_compare_swap_nbi   	pshmem_longlong_atomic_compare_swap_nbi
+#define shmem_uint_atomic_compare_swap_nbi       	pshmem_uint_atomic_compare_swap_nbi
+#define shmem_ulong_atomic_compare_swap_nbi      	pshmem_ulong_atomic_compare_swap_nbi
+#define shmem_ulonglong_atomic_compare_swap_nbi        pshmem_ulonglong_atomic_compare_swap_nbi
+#define shmem_int32_atomic_compare_swap_nbi      	pshmem_int32_atomic_compare_swap_nbi
+#define shmem_int64_atomic_compare_swap_nbi      	pshmem_int64_atomic_compare_swap_nbi
+#define shmem_uint32_atomic_compare_swap_nbi     	pshmem_uint32_atomic_compare_swap_nbi
+#define shmem_uint64_atomic_compare_swap_nbi     	pshmem_uint64_atomic_compare_swap_nbi
+#define shmem_size_atomic_compare_swap_nbi       	pshmem_size_atomic_compare_swap_nbi
+#define shmem_ptrdiff_atomic_compare_swap_nbi    	pshmem_ptrdiff_atomic_compare_swap_nbi
+
+
+/* Nonblocking Atomic Swap */
+#define shmem_ctx_double_atomic_swap_nbi            pshmem_ctx_double_atomic_swap_nbi
+#define shmem_ctx_float_atomic_swap_nbi             pshmem_ctx_float_atomic_swap_nbi
+#define shmem_ctx_int_atomic_swap_nbi               pshmem_ctx_int_atomic_swap_nbi
+#define shmem_ctx_long_atomic_swap_nbi              pshmem_ctx_long_atomic_swap_nbi
+#define shmem_ctx_longlong_atomic_swap_nbi          pshmem_ctx_longlong_atomic_swap_nbi
+#define shmem_ctx_uint_atomic_swap_nbi              pshmem_ctx_uint_atomic_swap_nbi
+#define shmem_ctx_ulong_atomic_swap_nbi             pshmem_ctx_ulong_atomic_swap_nbi
+#define shmem_ctx_ulonglong_atomic_swap_nbi         pshmem_ctx_ulonglong_atomic_swap_nbi
+#define shmem_ctx_int32_atomic_swap_nbi             pshmem_ctx_int32_atomic_swap_nbi
+#define shmem_ctx_int64_atomic_swap_nbi             pshmem_ctx_int64_atomic_swap_nbi
+#define shmem_ctx_uint32_atomic_swap_nbi            pshmem_ctx_uint32_atomic_swap_nbi
+#define shmem_ctx_uint64_atomic_swap_nbi            pshmem_ctx_uint64_atomic_swap_nbi
+#define shmem_ctx_size_atomic_swap_nbi              pshmem_ctx_size_atomic_swap_nbi
+#define shmem_ctx_ptrdiff_atomic_swap_nbi           pshmem_ctx_ptrdiff_atomic_swap_nbi
+
+#define shmem_double_atomic_swap_nbi     	        pshmem_double_atomic_swap_nbi
+#define shmem_float_atomic_swap_nbi      	        pshmem_float_atomic_swap_nbi
+#define shmem_int_atomic_swap_nbi        	        pshmem_int_atomic_swap_nbi
+#define shmem_long_atomic_swap_nbi       	        pshmem_long_atomic_swap_nbi
+#define shmem_longlong_atomic_swap_nbi   	        pshmem_longlong_atomic_swap_nbi
+#define shmem_uint_atomic_swap_nbi       		pshmem_uint_atomic_swap_nbi
+#define shmem_ulong_atomic_swap_nbi      		pshmem_ulong_atomic_swap_nbi
+#define shmem_ulonglong_atomic_swap_nbi  		pshmem_ulonglong_atomic_swap_nbi
+#define shmem_int32_atomic_swap_nbi      		pshmem_int32_atomic_swap_nbi
+#define shmem_int64_atomic_swap_nbi      		pshmem_int64_atomic_swap_nbi
+#define shmem_uint32_atomic_swap_nbi     		pshmem_uint32_atomic_swap_nbi
+#define shmem_uint64_atomic_swap_nbi     		pshmem_uint64_atomic_swap_nbi
+#define shmem_size_atomic_swap_nbi       		pshmem_size_atomic_swap_nbi
+#define shmem_ptrdiff_atomic_swap_nbi    		pshmem_ptrdiff_atomic_swap_nbi
+
+
+/* Nonblocking Atomic Fetch and Increment */
+#define shmem_ctx_int_atomic_fetch_inc_nbi               pshmem_ctx_int_atomic_fetch_inc_nbi
+#define shmem_ctx_long_atomic_fetch_inc_nbi              pshmem_ctx_long_atomic_fetch_inc_nbi
+#define shmem_ctx_longlong_atomic_fetch_inc_nbi          pshmem_ctx_longlong_atomic_fetch_inc_nbi
+#define shmem_ctx_uint_atomic_fetch_inc_nbi              pshmem_ctx_uint_atomic_fetch_inc_nbi
+#define shmem_ctx_ulong_atomic_fetch_inc_nbi             pshmem_ctx_ulong_atomic_fetch_inc_nbi
+#define shmem_ctx_ulonglong_atomic_fetch_inc_nbi         pshmem_ctx_ulonglong_atomic_fetch_inc_nbi
+#define shmem_ctx_int32_atomic_fetch_inc_nbi             pshmem_ctx_int32_atomic_fetch_inc_nbi
+#define shmem_ctx_int64_atomic_fetch_inc_nbi             pshmem_ctx_int64_atomic_fetch_inc_nbi
+#define shmem_ctx_uint32_atomic_fetch_inc_nbi            pshmem_ctx_uint32_atomic_fetch_inc_nbi
+#define shmem_ctx_uint64_atomic_fetch_inc_nbi            pshmem_ctx_uint64_atomic_fetch_inc_nbi
+#define shmem_ctx_size_atomic_fetch_inc_nbi              pshmem_ctx_size_atomic_fetch_inc_nbi
+#define shmem_ctx_ptrdiff_atomic_fetch_inc_nbi           pshmem_ctx_ptrdiff_atomic_fetch_inc_nbi
+
+#define shmem_int_atomic_fetch_inc_nbi        			 pshmem_int_atomic_fetch_inc_nbi
+#define shmem_long_atomic_fetch_inc_nbi       			 pshmem_long_atomic_fetch_inc_nbi
+#define shmem_longlong_atomic_fetch_inc_nbi   			 pshmem_longlong_atomic_fetch_inc_nbi
+#define shmem_uint_atomic_fetch_inc_nbi       			 pshmem_uint_atomic_fetch_inc_nbi
+#define shmem_ulong_atomic_fetch_inc_nbi      			 pshmem_ulong_atomic_fetch_inc_nbi
+#define shmem_ulonglong_atomic_fetch_inc_nbi  			 pshmem_ulonglong_atomic_fetch_inc_nbi
+#define shmem_int32_atomic_fetch_inc_nbi      			 pshmem_int32_atomic_fetch_inc_nbi
+#define shmem_int64_atomic_fetch_inc_nbi      			 pshmem_int64_atomic_fetch_inc_nbi
+#define shmem_uint32_atomic_fetch_inc_nbi     			 pshmem_uint32_atomic_fetch_inc_nbi
+#define shmem_uint64_atomic_fetch_inc_nbi     			 pshmem_uint64_atomic_fetch_inc_nbi
+#define shmem_size_atomic_fetch_inc_nbi       			 pshmem_size_atomic_fetch_inc_nbi
+#define shmem_ptrdiff_atomic_fetch_inc_nbi    			 pshmem_ptrdiff_atomic_fetch_inc_nbi
+
+
+/* Nonblocking Atomic Fetch and Add */
+#define shmem_ctx_int_atomic_fetch_add_nbi               pshmem_ctx_int_atomic_fetch_add_nbi
+#define shmem_ctx_long_atomic_fetch_add_nbi              pshmem_ctx_long_atomic_fetch_add_nbi
+#define shmem_ctx_longlong_atomic_fetch_add_nbi          pshmem_ctx_longlong_atomic_fetch_add_nbi
+#define shmem_ctx_uint_atomic_fetch_add_nbi              pshmem_ctx_uint_atomic_fetch_add_nbi
+#define shmem_ctx_ulong_atomic_fetch_add_nbi             pshmem_ctx_ulong_atomic_fetch_add_nbi
+#define shmem_ctx_ulonglong_atomic_fetch_add_nbi         pshmem_ctx_ulonglong_atomic_fetch_add_nbi
+#define shmem_ctx_int32_atomic_fetch_add_nbi             pshmem_ctx_int32_atomic_fetch_add_nbi
+#define shmem_ctx_int64_atomic_fetch_add_nbi             pshmem_ctx_int64_atomic_fetch_add_nbi
+#define shmem_ctx_uint32_atomic_fetch_add_nbi            pshmem_ctx_uint32_atomic_fetch_add_nbi
+#define shmem_ctx_uint64_atomic_fetch_add_nbi            pshmem_ctx_uint64_atomic_fetch_add_nbi
+#define shmem_ctx_size_atomic_fetch_add_nbi              pshmem_ctx_size_atomic_fetch_add_nbi
+#define shmem_ctx_ptrdiff_atomic_fetch_add_nbi           pshmem_ctx_ptrdiff_atomic_fetch_add_nbi
+
+#define shmem_int_atomic_fetch_add_nbi        			 pshmem_int_atomic_fetch_add_nbi
+#define shmem_long_atomic_fetch_add_nbi       			 pshmem_long_atomic_fetch_add_nbi
+#define shmem_longlong_atomic_fetch_add_nbi   			 pshmem_longlong_atomic_fetch_add_nbi
+#define shmem_uint_atomic_fetch_add_nbi       			 pshmem_uint_atomic_fetch_add_nbi
+#define shmem_ulong_atomic_fetch_add_nbi      			 pshmem_ulong_atomic_fetch_add_nbi
+#define shmem_ulonglong_atomic_fetch_add_nbi  			 pshmem_ulonglong_atomic_fetch_add_nbi
+#define shmem_int32_atomic_fetch_add_nbi      			 pshmem_int32_atomic_fetch_add_nbi
+#define shmem_int64_atomic_fetch_add_nbi      			 pshmem_int64_atomic_fetch_add_nbi
+#define shmem_uint32_atomic_fetch_add_nbi     			 pshmem_uint32_atomic_fetch_add_nbi
+#define shmem_uint64_atomic_fetch_add_nbi     			 pshmem_uint64_atomic_fetch_add_nbi
+#define shmem_size_atomic_fetch_add_nbi       			 pshmem_size_atomic_fetch_add_nbi
+#define shmem_ptrdiff_atomic_fetch_add_nbi    			 pshmem_ptrdiff_atomic_fetch_add_nbi
+
+
+/* Nonblocking Atomic Fetch and And */
+#define shmem_ctx_uint_atomic_fetch_and_nbi              pshmem_ctx_uint_atomic_fetch_and_nbi
+#define shmem_ctx_ulong_atomic_fetch_and_nbi             pshmem_ctx_ulong_atomic_fetch_and_nbi
+#define shmem_ctx_ulonglong_atomic_fetch_and_nbi         pshmem_ctx_ulonglong_atomic_fetch_and_nbi
+#define shmem_ctx_int32_atomic_fetch_and_nbi             pshmem_ctx_int32_atomic_fetch_and_nbi
+#define shmem_ctx_int64_atomic_fetch_and_nbi             pshmem_ctx_int64_atomic_fetch_and_nbi
+#define shmem_ctx_uint32_atomic_fetch_and_nbi            pshmem_ctx_uint32_atomic_fetch_and_nbi
+#define shmem_ctx_uint64_atomic_fetch_and_nbi            pshmem_ctx_uint64_atomic_fetch_and_nbi
+
+#define shmem_uint_atomic_fetch_and_nbi       			 pshmem_uint_atomic_fetch_and_nbi
+#define shmem_ulong_atomic_fetch_and_nbi      			 pshmem_ulong_atomic_fetch_and_nbi
+#define shmem_ulonglong_atomic_fetch_and_nbi  			 pshmem_ulonglong_atomic_fetch_and_nbi
+#define shmem_int32_atomic_fetch_and_nbi      			 pshmem_int32_atomic_fetch_and_nbi
+#define shmem_int64_atomic_fetch_and_nbi      			 pshmem_int64_atomic_fetch_and_nbi
+#define shmem_uint32_atomic_fetch_and_nbi     			 pshmem_uint32_atomic_fetch_and_nbi
+#define shmem_uint64_atomic_fetch_and_nbi     			 pshmem_uint64_atomic_fetch_and_nbi
+
+
+/* Nonblocking Atomic Fetch and OR */
+#define shmem_ctx_uint_atomic_fetch_or_nbi              pshmem_ctx_uint_atomic_fetch_or_nbi
+#define shmem_ctx_ulong_atomic_fetch_or_nbi             pshmem_ctx_ulong_atomic_fetch_or_nbi
+#define shmem_ctx_ulonglong_atomic_fetch_or_nbi         pshmem_ctx_ulonglong_atomic_fetch_or_nbi
+#define shmem_ctx_int32_atomic_fetch_or_nbi             pshmem_ctx_int32_atomic_fetch_or_nbi
+#define shmem_ctx_int64_atomic_fetch_or_nbi             pshmem_ctx_int64_atomic_fetch_or_nbi
+#define shmem_ctx_uint32_atomic_fetch_or_nbi            pshmem_ctx_uint32_atomic_fetch_or_nbi
+#define shmem_ctx_uint64_atomic_fetch_or_nbi            pshmem_ctx_uint64_atomic_fetch_or_nbi
+
+#define shmem_uint_atomic_fetch_or_nbi       			pshmem_uint_atomic_fetch_or_nbi
+#define shmem_ulong_atomic_fetch_or_nbi      			pshmem_ulong_atomic_fetch_or_nbi
+#define shmem_ulonglong_atomic_fetch_or_nbi  			pshmem_ulonglong_atomic_fetch_or_nbi
+#define shmem_int32_atomic_fetch_or_nbi      			pshmem_int32_atomic_fetch_or_nbi
+#define shmem_int64_atomic_fetch_or_nbi      			pshmem_int64_atomic_fetch_or_nbi
+#define shmem_uint32_atomic_fetch_or_nbi     			pshmem_uint32_atomic_fetch_or_nbi
+#define shmem_uint64_atomic_fetch_or_nbi     			pshmem_uint64_atomic_fetch_or_nbi
+
+
+/* Nonblocking Atomic Fetch and XOR */
+#define shmem_ctx_uint_atomic_fetch_xor_nbi              pshmem_ctx_uint_atomic_fetch_xor_nbi
+#define shmem_ctx_ulong_atomic_fetch_xor_nbi             pshmem_ctx_ulong_atomic_fetch_xor_nbi
+#define shmem_ctx_ulonglong_atomic_fetch_xor_nbi         pshmem_ctx_ulonglong_atomic_fetch_xor_nbi
+#define shmem_ctx_int32_atomic_fetch_xor_nbi             pshmem_ctx_int32_atomic_fetch_xor_nbi
+#define shmem_ctx_int64_atomic_fetch_xor_nbi             pshmem_ctx_int64_atomic_fetch_xor_nbi
+#define shmem_ctx_uint32_atomic_fetch_xor_nbi            pshmem_ctx_uint32_atomic_fetch_xor_nbi
+#define shmem_ctx_uint64_atomic_fetch_xor_nbi            pshmem_ctx_uint64_atomic_fetch_xor_nbi
+
+#define shmem_uint_atomic_fetch_xor_nbi       			 pshmem_uint_atomic_fetch_xor_nbi
+#define shmem_ulong_atomic_fetch_xor_nbi      			 pshmem_ulong_atomic_fetch_xor_nbi
+#define shmem_ulonglong_atomic_fetch_xor_nbi  			 pshmem_ulonglong_atomic_fetch_xor_nbi
+#define shmem_int32_atomic_fetch_xor_nbi      			 pshmem_int32_atomic_fetch_xor_nbi
+#define shmem_int64_atomic_fetch_xor_nbi      			 pshmem_int64_atomic_fetch_xor_nbi
+#define shmem_uint32_atomic_fetch_xor_nbi     			 pshmem_uint32_atomic_fetch_xor_nbi
+#define shmem_uint64_atomic_fetch_xor_nbi     			 pshmem_uint64_atomic_fetch_xor_nbi
+
+/*
+ * Control of profile
+ */
+
+#define shmem_pcontrol               pshmem_pcontrol
+
+/*
+ * Lock functions
+ */
+#define shmem_set_lock               pshmem_set_lock
+#define shmem_clear_lock             pshmem_clear_lock
+#define shmem_test_lock              pshmem_test_lock
+
+/*
+ * P2P sync routines
+ */
+#define shmem_short_wait             pshmem_short_wait
+#define shmem_int_wait               pshmem_int_wait
+#define shmem_long_wait              pshmem_long_wait
+#define shmem_longlong_wait          pshmem_longlong_wait
+#define shmem_wait                   pshmem_wait
+#define shmemx_int32_wait            pshmemx_int32_wait
+#define shmemx_int64_wait            pshmemx_int64_wait
+
+#define shmem_short_wait_until       pshmem_short_wait_until
+#define shmem_int_wait_until         pshmem_int_wait_until
+#define shmem_long_wait_until        pshmem_long_wait_until
+#define shmem_longlong_wait_until    pshmem_longlong_wait_until
+#define shmem_ushort_wait_until      pshmem_ushort_wait_until
+#define shmem_uint_wait_until        pshmem_uint_wait_until
+#define shmem_ulong_wait_until       pshmem_ulong_wait_until
+#define shmem_ulonglong_wait_until   pshmem_ulonglong_wait_until
+#define shmem_int32_wait_until       pshmem_int32_wait_until
+#define shmem_int64_wait_until       pshmem_int64_wait_until
+#define shmem_uint32_wait_until      pshmem_uint32_wait_until
+#define shmem_uint64_wait_until      pshmem_uint64_wait_until
+#define shmem_size_wait_until        pshmem_size_wait_until
+#define shmem_ptrdiff_wait_until     pshmem_ptrdiff_wait_until
+
+#define shmemx_int32_wait_until      pshmemx_int32_wait_until
+#define shmemx_int64_wait_until      pshmemx_int64_wait_until
+
+#define shmem_short_wait_until_all       pshmem_short_wait_until_all
+#define shmem_ushort_wait_until_all      pshmem_ushort_wait_until_all
+#define shmem_int_wait_until_all         pshmem_int_wait_until_all
+#define shmem_long_wait_until_all        pshmem_long_wait_until_all
+#define shmem_longlong_wait_until_all    pshmem_longlong_wait_until_all
+#define shmem_uint_wait_until_all        pshmem_uint_wait_until_all
+#define shmem_ulong_wait_until_all       pshmem_ulong_wait_until_all
+#define shmem_ulonglong_wait_until_all   pshmem_ulonglong_wait_until_all
+#define shmem_int32_wait_until_all       pshmem_int32_wait_until_all
+#define shmem_int64_wait_until_all       pshmem_int64_wait_until_all
+#define shmem_uint32_wait_until_all      pshmem_uint32_wait_until_all
+#define shmem_uint64_wait_until_all      pshmem_uint64_wait_until_all
+#define shmem_size_wait_until_all        pshmem_size_wait_until_all
+#define shmem_ptrdiff_wait_until_all     pshmem_ptrdiff_wait_until_all
+
+
+#define shmem_short_wait_until_any       pshmem_short_wait_until_any
+#define shmem_ushort_wait_until_any      pshmem_ushort_wait_until_any
+#define shmem_int_wait_until_any         pshmem_int_wait_until_any
+#define shmem_long_wait_until_any        pshmem_long_wait_until_any
+#define shmem_longlong_wait_until_any    pshmem_longlong_wait_until_any
+#define shmem_uint_wait_until_any        pshmem_uint_wait_until_any
+#define shmem_ulong_wait_until_any       pshmem_ulong_wait_until_any
+#define shmem_ulonglong_wait_until_any   pshmem_ulonglong_wait_until_any
+#define shmem_int32_wait_until_any       pshmem_int32_wait_until_any
+#define shmem_int64_wait_until_any       pshmem_int64_wait_until_any
+#define shmem_uint32_wait_until_any      pshmem_uint32_wait_until_any
+#define shmem_uint64_wait_until_any      pshmem_uint64_wait_until_any
+#define shmem_size_wait_until_any        pshmem_size_wait_until_any
+#define shmem_ptrdiff_wait_until_any     pshmem_ptrdiff_wait_until_any
+
+
+#define shmem_short_wait_until_some       pshmem_short_wait_until_some
+#define shmem_ushort_wait_until_some      pshmem_ushort_wait_until_some
+#define shmem_int_wait_until_some         pshmem_int_wait_until_some
+#define shmem_long_wait_until_some        pshmem_long_wait_until_some
+#define shmem_longlong_wait_until_some    pshmem_longlong_wait_until_some
+#define shmem_uint_wait_until_some        pshmem_uint_wait_until_some
+#define shmem_ulong_wait_until_some       pshmem_ulong_wait_until_some
+#define shmem_ulonglong_wait_until_some   pshmem_ulonglong_wait_until_some
+#define shmem_int32_wait_until_some       pshmem_int32_wait_until_some
+#define shmem_int64_wait_until_some       pshmem_int64_wait_until_some
+#define shmem_uint32_wait_until_some      pshmem_uint32_wait_until_some
+#define shmem_uint64_wait_until_some      pshmem_uint64_wait_until_some
+#define shmem_size_wait_until_some        pshmem_size_wait_until_some
+#define shmem_ptrdiff_wait_until_some     pshmem_ptrdiff_wait_until_some
+
+
+#define shmem_short_wait_until_all_vector       pshmem_short_wait_until_all_vector
+#define shmem_ushort_wait_until_all_vector      pshmem_ushort_wait_until_all_vector
+#define shmem_int_wait_until_all_vector         pshmem_int_wait_until_all_vector
+#define shmem_long_wait_until_all_vector        pshmem_long_wait_until_all_vector
+#define shmem_longlong_wait_until_all_vector    pshmem_longlong_wait_until_all_vector
+#define shmem_uint_wait_until_all_vector        pshmem_uint_wait_until_all_vector
+#define shmem_ulong_wait_until_all_vector       pshmem_ulong_wait_until_all_vector
+#define shmem_ulonglong_wait_until_all_vector   pshmem_ulonglong_wait_until_all_vector
+#define shmem_int32_wait_until_all_vector       pshmem_int32_wait_until_all_vector
+#define shmem_int64_wait_until_all_vector       pshmem_int64_wait_until_all_vector
+#define shmem_uint32_wait_until_all_vector      pshmem_uint32_wait_until_all_vector
+#define shmem_uint64_wait_until_all_vector      pshmem_uint64_wait_until_all_vector
+#define shmem_size_wait_until_all_vector        pshmem_size_wait_until_all_vector
+#define shmem_ptrdiff_wait_until_all_vector     pshmem_ptrdiff_wait_until_all_vector
+
+
+#define shmem_short_wait_until_any_vector       pshmem_short_wait_until_any_vector
+#define shmem_ushort_wait_until_any_vector      pshmem_ushort_wait_until_any_vector
+#define shmem_int_wait_until_any_vector         pshmem_int_wait_until_any_vector
+#define shmem_long_wait_until_any_vector        pshmem_long_wait_until_any_vector
+#define shmem_longlong_wait_until_any_vector    pshmem_longlong_wait_until_any_vector
+#define shmem_uint_wait_until_any_vector        pshmem_uint_wait_until_any_vector
+#define shmem_ulong_wait_until_any_vector       pshmem_ulong_wait_until_any_vector
+#define shmem_ulonglong_wait_until_any_vector   pshmem_ulonglong_wait_until_any_vector
+#define shmem_int32_wait_until_any_vector       pshmem_int32_wait_until_any_vector
+#define shmem_int64_wait_until_any_vector       pshmem_int64_wait_until_any_vector
+#define shmem_uint32_wait_until_any_vector      pshmem_uint32_wait_until_any_vector
+#define shmem_uint64_wait_until_any_vector      pshmem_uint64_wait_until_any_vector
+#define shmem_size_wait_until_any_vector        pshmem_size_wait_until_any_vector
+#define shmem_ptrdiff_wait_until_any_vector     pshmem_ptrdiff_wait_until_any_vector
+
+
+#define shmem_short_wait_until_some_vector       pshmem_short_wait_until_some_vector
+#define shmem_ushort_wait_until_some_vector      pshmem_ushort_wait_until_some_vector
+#define shmem_int_wait_until_some_vector         pshmem_int_wait_until_some_vector
+#define shmem_long_wait_until_some_vector        pshmem_long_wait_until_some_vector
+#define shmem_longlong_wait_until_some_vector    pshmem_longlong_wait_until_some_vector
+#define shmem_uint_wait_until_some_vector        pshmem_uint_wait_until_some_vector
+#define shmem_ulong_wait_until_some_vector       pshmem_ulong_wait_until_some_vector
+#define shmem_ulonglong_wait_until_some_vector   pshmem_ulonglong_wait_until_some_vector
+#define shmem_int32_wait_until_some_vector       pshmem_int32_wait_until_some_vector
+#define shmem_int64_wait_until_some_vector       pshmem_int64_wait_until_some_vector
+#define shmem_uint32_wait_until_some_vector      pshmem_uint32_wait_until_some_vector
+#define shmem_uint64_wait_until_some_vector      pshmem_uint64_wait_until_some_vector
+#define shmem_size_wait_until_some_vector        pshmem_size_wait_until_some_vector
+#define shmem_ptrdiff_wait_until_some_vector     pshmem_ptrdiff_wait_until_some_vector
+
+
+#define shmem_short_test             pshmem_short_test
+#define shmem_int_test               pshmem_int_test
+#define shmem_long_test              pshmem_long_test
+#define shmem_longlong_test          pshmem_longlong_test
+#define shmem_ushort_test            pshmem_ushort_test
+#define shmem_uint_test              pshmem_uint_test
+#define shmem_ulong_test             pshmem_ulong_test
+#define shmem_ulonglong_test         pshmem_ulonglong_test
+#define shmem_int32_test             pshmem_int32_test
+#define shmem_int64_test             pshmem_int64_test
+#define shmem_uint32_test            pshmem_uint32_test
+#define shmem_uint64_test            pshmem_uint64_test
+#define shmem_size_test              pshmem_size_test
+#define shmem_ptrdiff_test           pshmem_ptrdiff_test
+
+
+#define shmem_short_test_all       pshmem_short_test_all
+#define shmem_ushort_test_all      pshmem_ushort_test_all
+#define shmem_int_test_all         pshmem_int_test_all
+#define shmem_long_test_all        pshmem_long_test_all
+#define shmem_longlong_test_all    pshmem_longlong_test_all
+#define shmem_uint_test_all        pshmem_uint_test_all
+#define shmem_ulong_test_all       pshmem_ulong_test_all
+#define shmem_ulonglong_test_all   pshmem_ulonglong_test_all
+#define shmem_int32_test_all       pshmem_int32_test_all
+#define shmem_int64_test_all       pshmem_int64_test_all
+#define shmem_uint32_test_all      pshmem_uint32_test_all
+#define shmem_uint64_test_all      pshmem_uint64_test_all
+#define shmem_size_test_all        pshmem_size_test_all
+#define shmem_ptrdiff_test_all     pshmem_ptrdiff_test_all
+
+
+#define shmem_short_test_any       pshmem_short_test_any
+#define shmem_ushort_test_any      pshmem_ushort_test_any
+#define shmem_int_test_any         pshmem_int_test_any
+#define shmem_long_test_any        pshmem_long_test_any
+#define shmem_longlong_test_any    pshmem_longlong_test_any
+#define shmem_uint_test_any        pshmem_uint_test_any
+#define shmem_ulong_test_any       pshmem_ulong_test_any
+#define shmem_ulonglong_test_any   pshmem_ulonglong_test_any
+#define shmem_int32_test_any       pshmem_int32_test_any
+#define shmem_int64_test_any       pshmem_int64_test_any
+#define shmem_uint32_test_any      pshmem_uint32_test_any
+#define shmem_uint64_test_any      pshmem_uint64_test_any
+#define shmem_size_test_any        pshmem_size_test_any
+#define shmem_ptrdiff_test_any     pshmem_ptrdiff_test_any
+
+
+
+#define shmem_short_test_some       pshmem_short_test_some
+#define shmem_ushort_test_some      pshmem_ushort_test_some
+#define shmem_int_test_some         pshmem_int_test_some
+#define shmem_long_test_some        pshmem_long_test_some
+#define shmem_longlong_test_some    pshmem_longlong_test_some
+#define shmem_uint_test_some        pshmem_uint_test_some
+#define shmem_ulong_test_some       pshmem_ulong_test_some
+#define shmem_ulonglong_test_some   pshmem_ulonglong_test_some
+#define shmem_int32_test_some       pshmem_int32_test_some
+#define shmem_int64_test_some       pshmem_int64_test_some
+#define shmem_uint32_test_some      pshmem_uint32_test_some
+#define shmem_uint64_test_some      pshmem_uint64_test_some
+#define shmem_size_test_some        pshmem_size_test_some
+#define shmem_ptrdiff_test_some     pshmem_ptrdiff_test_some
+
+
+#define shmem_short_test_all_vector       pshmem_short_test_all_vector
+#define shmem_ushort_test_all_vector      pshmem_ushort_test_all_vector
+#define shmem_int_test_all_vector         pshmem_int_test_all_vector
+#define shmem_long_test_all_vector        pshmem_long_test_all_vector
+#define shmem_longlong_test_all_vector    pshmem_longlong_test_all_vector
+#define shmem_uint_test_all_vector        pshmem_uint_test_all_vector
+#define shmem_ulong_test_all_vector       pshmem_ulong_test_all_vector
+#define shmem_ulonglong_test_all_vector   pshmem_ulonglong_test_all_vector
+#define shmem_int32_test_all_vector       pshmem_int32_test_all_vector
+#define shmem_int64_test_all_vector       pshmem_int64_test_all_vector
+#define shmem_uint32_test_all_vector      pshmem_uint32_test_all_vector
+#define shmem_uint64_test_all_vector      pshmem_uint64_test_all_vector
+#define shmem_size_test_all_vector        pshmem_size_test_all_vector
+#define shmem_ptrdiff_test_all_vector     pshmem_ptrdiff_test_all_vector
+
+
+#define shmem_short_test_any_vector       pshmem_short_test_any_vector
+#define shmem_ushort_test_any_vector      pshmem_ushort_test_any_vector
+#define shmem_int_test_any_vector         pshmem_int_test_any_vector
+#define shmem_long_test_any_vector        pshmem_long_test_any_vector
+#define shmem_longlong_test_any_vector    pshmem_longlong_test_any_vector
+#define shmem_uint_test_any_vector        pshmem_uint_test_any_vector
+#define shmem_ulong_test_any_vector       pshmem_ulong_test_any_vector
+#define shmem_ulonglong_test_any_vector   pshmem_ulonglong_test_any_vector
+#define shmem_int32_test_any_vector       pshmem_int32_test_any_vector
+#define shmem_int64_test_any_vector       pshmem_int64_test_any_vector
+#define shmem_uint32_test_any_vector      pshmem_uint32_test_any_vector
+#define shmem_uint64_test_any_vector      pshmem_uint64_test_any_vector
+#define shmem_size_test_any_vector        pshmem_size_test_any_vector
+#define shmem_ptrdiff_test_any_vector     pshmem_ptrdiff_test_any_vector
+
+
+#define shmem_short_test_some_vector       pshmem_short_test_some_vector
+#define shmem_ushort_test_some_vector      pshmem_ushort_test_some_vector
+#define shmem_int_test_some_vector         pshmem_int_test_some_vector
+#define shmem_long_test_some_vector        pshmem_long_test_some_vector
+#define shmem_longlong_test_some_vector    pshmem_longlong_test_some_vector
+#define shmem_uint_test_some_vector        pshmem_uint_test_some_vector
+#define shmem_ulong_test_some_vector       pshmem_ulong_test_some_vector
+#define shmem_ulonglong_test_some_vector   pshmem_ulonglong_test_some_vector
+#define shmem_int32_test_some_vector       pshmem_int32_test_some_vector
+#define shmem_int64_test_some_vector       pshmem_int64_test_some_vector
+#define shmem_uint32_test_some_vector      pshmem_uint32_test_some_vector
+#define shmem_uint64_test_some_vector      pshmem_uint64_test_some_vector
+#define shmem_size_test_some_vector        pshmem_size_test_some_vector
+#define shmem_ptrdiff_test_some_vector     pshmem_ptrdiff_test_some_vector
+
+/*
+ * Barrier sync routines
+ */
+#define shmem_barrier                pshmem_barrier
+#define shmem_barrier_all            pshmem_barrier_all
+#define shmem_sync_all               pshmem_sync_all
+#define shmem_sync_deprecated        pshmem_sync_deprecated
+#define shmem_fence                  pshmem_fence
+#define shmem_ctx_fence              pshmem_ctx_fence
+#define shmem_quiet                  pshmem_quiet
+#define shmem_ctx_quiet              pshmem_ctx_quiet
+
+/*
+ * Collective routines
+ */
+#define shmem_broadcast32            pshmem_broadcast32
+#define shmem_broadcast64            pshmem_broadcast64
+#define shmem_collect32              pshmem_collect32
+#define shmem_collect64              pshmem_collect64
+#define shmem_fcollect32             pshmem_fcollect32
+#define shmem_fcollect64             pshmem_fcollect64
+
+/*
+ * Reduction routines
+ */
+#define shmem_short_and_to_all       pshmem_short_and_to_all
+#define shmem_int_and_to_all         pshmem_int_and_to_all
+#define shmem_long_and_to_all        pshmem_long_and_to_all
+#define shmem_longlong_and_to_all    pshmem_longlong_and_to_all
+#define shmemx_int16_and_to_all      pshmemx_int16_and_to_all
+#define shmemx_int32_and_to_all      pshmemx_int32_and_to_all
+#define shmemx_int64_and_to_all      pshmemx_int64_and_to_all
+
+#define shmem_short_or_to_all        pshmem_short_or_to_all
+#define shmem_int_or_to_all          pshmem_int_or_to_all
+#define shmem_long_or_to_all         pshmem_long_or_to_all
+#define shmem_longlong_or_to_all     pshmem_longlong_or_to_all
+#define shmemx_int16_or_to_all       pshmemx_int16_or_to_all
+#define shmemx_int32_or_to_all       pshmemx_int32_or_to_all
+#define shmemx_int64_or_to_all       pshmemx_int64_or_to_all
+
+#define shmem_short_xor_to_all       pshmem_short_xor_to_all
+#define shmem_int_xor_to_all         pshmem_int_xor_to_all
+#define shmem_long_xor_to_all        pshmem_long_xor_to_all
+#define shmem_longlong_xor_to_all    pshmem_longlong_xor_to_all
+#define shmemx_int16_xor_to_all      pshmemx_int16_xor_to_all
+#define shmemx_int32_xor_to_all      pshmemx_int32_xor_to_all
+#define shmemx_int64_xor_to_all      pshmemx_int64_xor_to_all
+
+#define shmem_short_max_to_all       pshmem_short_max_to_all
+#define shmem_int_max_to_all         pshmem_int_max_to_all
+#define shmem_long_max_to_all        pshmem_long_max_to_all
+#define shmem_longlong_max_to_all    pshmem_longlong_max_to_all
+#define shmem_float_max_to_all       pshmem_float_max_to_all
+#define shmem_double_max_to_all      pshmem_double_max_to_all
+#define shmem_longdouble_max_to_all  pshmem_longdouble_max_to_all
+#define shmemx_int16_max_to_all      pshmemx_int16_max_to_all
+#define shmemx_int32_max_to_all      pshmemx_int32_max_to_all
+#define shmemx_int64_max_to_all      pshmemx_int64_max_to_all
+
+#define shmem_short_min_to_all       pshmem_short_min_to_all
+#define shmem_int_min_to_all         pshmem_int_min_to_all
+#define shmem_long_min_to_all        pshmem_long_min_to_all
+#define shmem_longlong_min_to_all    pshmem_longlong_min_to_all
+#define shmem_float_min_to_all       pshmem_float_min_to_all
+#define shmem_double_min_to_all      pshmem_double_min_to_all
+#define shmem_longdouble_min_to_all  pshmem_longdouble_min_to_all
+#define shmemx_int16_min_to_all      pshmemx_int16_min_to_all
+#define shmemx_int32_min_to_all      pshmemx_int32_min_to_all
+#define shmemx_int64_min_to_all      pshmemx_int64_min_to_all
+
+#define shmem_short_sum_to_all       pshmem_short_sum_to_all
+#define shmem_int_sum_to_all         pshmem_int_sum_to_all
+#define shmem_long_sum_to_all        pshmem_long_sum_to_all
+#define shmem_longlong_sum_to_all    pshmem_longlong_sum_to_all
+#define shmem_float_sum_to_all       pshmem_float_sum_to_all
+#define shmem_double_sum_to_all      pshmem_double_sum_to_all
+#define shmem_longdouble_sum_to_all  pshmem_longdouble_sum_to_all
+#define shmem_complexf_sum_to_all    pshmem_complexf_sum_to_all
+#define shmem_complexd_sum_to_all    pshmem_complexd_sum_to_all
+#define shmemx_int16_sum_to_all      pshmemx_int16_sum_to_all
+#define shmemx_int32_sum_to_all      pshmemx_int32_sum_to_all
+#define shmemx_int64_sum_to_all      pshmemx_int64_sum_to_all
+
+#define shmem_short_prod_to_all      pshmem_short_prod_to_all
+#define shmem_int_prod_to_all        pshmem_int_prod_to_all
+#define shmem_long_prod_to_all       pshmem_long_prod_to_all
+#define shmem_longlong_prod_to_all   pshmem_longlong_prod_to_all
+#define shmem_float_prod_to_all      pshmem_float_prod_to_all
+#define shmem_double_prod_to_all     pshmem_double_prod_to_all
+#define shmem_longdouble_prod_to_all pshmem_longdouble_prod_to_all
+#define shmem_complexf_prod_to_all   pshmem_complexf_prod_to_all
+#define shmem_complexd_prod_to_all   pshmem_complexd_prod_to_all
+#define shmemx_int16_prod_to_all     pshmemx_int16_prod_to_all
+#define shmemx_int32_prod_to_all     pshmemx_int32_prod_to_all
+#define shmemx_int64_prod_to_all     pshmemx_int64_prod_to_all
+
+/*
+ * Alltoall routines
+ */
+#define shmem_alltoall32             pshmem_alltoall32
+#define shmem_alltoall64             pshmem_alltoall64
+#define shmem_alltoalls32            pshmem_alltoalls32
+#define shmem_alltoalls64            pshmem_alltoalls64
+
+/*
+ * Platform specific cache management routines
+ */
+#define shmem_udcflush              pshmem_udcflush
+#define shmem_udcflush_line         pshmem_udcflush_line
+#define shmem_set_cache_inv         pshmem_set_cache_inv
+#define shmem_set_cache_line_inv    pshmem_set_cache_line_inv
+#define shmem_clear_cache_inv       pshmem_clear_cache_inv
+#define shmem_clear_cache_line_inv  pshmem_clear_cache_line_inv
+
+#endif /* OSHMEM_C_PROFILE_DEFINES_H */
diff --git a/oshmem/shmem/c/profile/Makefile.am b/oshmem/shmem/c/profile/Makefile.am
deleted file mode 100644
index 717d9fbf2ef..00000000000
--- a/oshmem/shmem/c/profile/Makefile.am
+++ /dev/null
@@ -1,116 +0,0 @@
-#
-# Copyright (c) 2013-2016 Mellanox Technologies, Inc.
-#                         All rights reserved
-# Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-include $(top_srcdir)/Makefile.ompi-rules
-
-#
-# OSHMEM_PROFILING flag is enabled when we want our shmem_* symbols
-# to be replaced by pshmem_*. In other words, this flag decides
-# whether "profile/defines.h" is included or not. "profile/defines.h"
-# replaces all shmem_* symbols with pshmem_* symbols. In this directory
-# we definately need it to be 1.
-#
-AM_CPPFLAGS = -DOSHMEM_PROFILING=1
-
-noinst_LTLIBRARIES =
-if PROJECT_OSHMEM
-# Only build if we're building OSHMEM
-noinst_LTLIBRARIES += liboshmem_c_pshmem.la
-endif
-
-headers = defines.h
-
-
-OSHMEM_API_SOURCES = \
-	pshmem_init.c \
-	pshmem_finalize.c \
-	pshmem_free.c \
-	pshmem_alloc.c \
-	pshmem_realloc.c \
-	pshmem_align.c \
-	pshmem_query.c \
-	pshmem_p.c \
-	pshmem_context.c \
-	pshmem_put.c \
-	pshmem_g.c \
-	pshmem_get.c \
-	pshmem_alltoall.c \
-	pshmem_broadcast.c \
-	pshmem_collect.c \
-	pshmem_ptr.c \
-	pshmem_pe_accessible.c \
-	pshmem_addr_accessible.c \
-	pshmem_barrier.c \
-	pshmem_sync.c \
-	pshmem_fence.c \
-	pshmem_quiet.c \
-	pshmem_wait.c \
-	pshmem_iget.c \
-	pshmem_iput.c \
-	pshmem_get_nb.c \
-	pshmem_put_nb.c \
-	pshmem_udcflush.c \
-	pshmem_udcflush_line.c \
-	pshmem_set_cache_inv.c \
-	pshmem_set_cache_line_inv.c \
-	pshmem_clear_cache_inv.c \
-	pshmem_clear_cache_line_inv.c \
-	pshmem_reduce.c \
-	pshmem_swap.c \
-	pshmem_set.c \
-	pshmem_cswap.c \
-	pshmem_fadd.c \
-	pshmem_fand.c \
-	pshmem_for.c \
-	pshmem_fxor.c \
-	pshmem_fetch.c \
-	pshmem_finc.c \
-	pshmem_add.c \
-	pshmem_and.c \
-	pshmem_or.c \
-	pshmem_xor.c \
-	pshmem_inc.c \
-	pshmem_clear_lock.c \
-	pshmem_set_lock.c \
-	pshmem_test_lock.c \
-	pshmem_global_exit.c \
-	pshmem_info.c
-
-nodist_liboshmem_c_pshmem_la_SOURCES = \
-	$(OSHMEM_API_SOURCES)
-
-#
-# Sym link in the sources from the real OSHMEM directory
-#
-$(nodist_liboshmem_c_pshmem_la_SOURCES):
-	$(OMPI_V_LN_S) if test ! -r $@ ; then \
-		pname=`echo $@ | cut -b '2-'` ; \
-		$(LN_S) $(top_srcdir)/oshmem/shmem/c/$$pname $@ ; \
-	fi
-
-if PROJECT_OSHMEM
-if WANT_INSTALL_HEADERS
-oshmemdir = $(oshmemincludedir)/$(subdir)
-oshmem_HEADERS = $(headers)
-endif
-endif
-
-# These files were created by targets above
-
-MAINTAINERCLEANFILES = $(nodist_liboshmem_c_pshmem_la_SOURCES)
-
-# Don't want these targets in here
-
-tags-recursive:
-tags:
-TAGS:
-GTAGS:
-ID:
diff --git a/oshmem/shmem/c/profile/defines.h b/oshmem/shmem/c/profile/defines.h
deleted file mode 100644
index fa30d783778..00000000000
--- a/oshmem/shmem/c/profile/defines.h
+++ /dev/null
@@ -1,1120 +0,0 @@
-/*
- * Copyright (c) 2013-2017 Mellanox Technologies, Inc.
- *                         All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef OSHMEM_C_PROFILE_DEFINES_H
-#define OSHMEM_C_PROFILE_DEFINES_H
-/*
- * This file is included in the top directory only if
- * profiling is required. Once profiling is required,
- * this file will replace all shmem_* symbols with
- * pshmem_* symbols
- */
-
-/*
- * Initialization routines
- */
-#define shmem_init                   pshmem_init
-#define shmem_init_thread            pshmem_init_thread
-#define start_pes                    pstart_pes /* shmem-compat.h */
-
-/*
- * Finalization routines
- */
-#define shmem_finalize               pshmem_finalize
-#define shmem_global_exit            pshmem_global_exit
-
-/*
- * Query routines
- */
-#define shmem_n_pes                  pshmem_n_pes
-#define shmem_query_thread           pshmem_query_thread
-#define shmem_my_pe                  pshmem_my_pe
-#define _num_pes                     p_num_pes /* shmem-compat.h */
-#define _my_pe                       p_my_pe /* shmem-compat.h */
-
-/*
- * Accessability routines
- */
-#define shmem_pe_accessible          pshmem_pe_accessible
-#define shmem_addr_accessible        pshmem_addr_accessible
-
-/*
- * Symmetric heap routines
- */
-#define shmem_malloc                 pshmem_malloc
-#define shmem_calloc                 pshmem_calloc
-#define shmem_align                  pshmem_align
-#define shmem_realloc                pshmem_realloc
-#define shmem_free                   pshmem_free
-#define shmalloc                     pshmalloc /* shmem-compat.h */
-#define shmemalign                   pshmemalign /* shmem-compat.h */
-#define shrealloc                    pshrealloc /* shmem-compat.h */
-#define shfree                       pshfree /* shmem-compat.h */
-
-#define shmemx_malloc_with_hint      pshmemx_malloc_with_hint
-
-/*
- * Remote pointer operations
- */
-#define shmem_ptr                    pshmem_ptr
-
-/*
- * Communication context operations
- */
-#define shmem_ctx_create             pshmem_ctx_create
-#define shmem_ctx_destroy            pshmem_ctx_destroy
-
-/*
- * Elemental put routines
- */
-#define shmem_ctx_char_p             pshmem_ctx_char_p
-#define shmem_ctx_short_p            pshmem_ctx_short_p
-#define shmem_ctx_int_p              pshmem_ctx_int_p
-#define shmem_ctx_long_p             pshmem_ctx_long_p
-#define shmem_ctx_float_p            pshmem_ctx_float_p
-#define shmem_ctx_double_p           pshmem_ctx_double_p
-#define shmem_ctx_longlong_p         pshmem_ctx_longlong_p
-#define shmem_ctx_schar_p            pshmem_ctx_schar_p
-#define shmem_ctx_uchar_p            pshmem_ctx_uchar_p
-#define shmem_ctx_ushort_p           pshmem_ctx_ushort_p
-#define shmem_ctx_uint_p             pshmem_ctx_uint_p
-#define shmem_ctx_ulong_p            pshmem_ctx_ulong_p
-#define shmem_ctx_ulonglong_p        pshmem_ctx_ulonglong_p
-#define shmem_ctx_longdouble_p       pshmem_ctx_longdouble_p
-#define shmem_ctx_int8_p             pshmem_ctx_int8_p
-#define shmem_ctx_int16_p            pshmem_ctx_int16_p
-#define shmem_ctx_int32_p            pshmem_ctx_int32_p
-#define shmem_ctx_int64_p            pshmem_ctx_int64_p
-#define shmem_ctx_uint8_p            pshmem_ctx_uint8_p
-#define shmem_ctx_uint16_p           pshmem_ctx_uint16_p
-#define shmem_ctx_uint32_p           pshmem_ctx_uint32_p
-#define shmem_ctx_uint64_p           pshmem_ctx_uint64_p
-#define shmem_ctx_size_p             pshmem_ctx_size_p
-#define shmem_ctx_ptrdiff_p          pshmem_ctx_ptrdiff_p
-
-#define shmem_char_p                 pshmem_char_p
-#define shmem_short_p                pshmem_short_p
-#define shmem_int_p                  pshmem_int_p
-#define shmem_long_p                 pshmem_long_p
-#define shmem_float_p                pshmem_float_p
-#define shmem_double_p               pshmem_double_p
-#define shmem_longlong_p             pshmem_longlong_p
-#define shmem_schar_p                pshmem_schar_p
-#define shmem_uchar_p                pshmem_uchar_p
-#define shmem_ushort_p               pshmem_ushort_p
-#define shmem_uint_p                 pshmem_uint_p
-#define shmem_ulong_p                pshmem_ulong_p
-#define shmem_ulonglong_p            pshmem_ulonglong_p
-#define shmem_longdouble_p           pshmem_longdouble_p
-#define shmem_int8_p                 pshmem_int8_p
-#define shmem_int16_p                pshmem_int16_p
-#define shmem_int32_p                pshmem_int32_p
-#define shmem_int64_p                pshmem_int64_p
-#define shmem_uint8_p                pshmem_uint8_p
-#define shmem_uint16_p               pshmem_uint16_p
-#define shmem_uint32_p               pshmem_uint32_p
-#define shmem_uint64_p               pshmem_uint64_p
-#define shmem_size_p                 pshmem_size_p
-#define shmem_ptrdiff_p              pshmem_ptrdiff_p
-
-#define shmemx_int16_p               pshmemx_int16_p
-#define shmemx_int32_p               pshmemx_int32_p
-#define shmemx_int64_p               pshmemx_int64_p
-
-/*
- * Block data put routines
- */
-#define shmem_ctx_char_put           pshmem_ctx_char_put
-#define shmem_ctx_short_put          pshmem_ctx_short_put
-#define shmem_ctx_int_put            pshmem_ctx_int_put
-#define shmem_ctx_long_put           pshmem_ctx_long_put
-#define shmem_ctx_float_put          pshmem_ctx_float_put
-#define shmem_ctx_double_put         pshmem_ctx_double_put
-#define shmem_ctx_longlong_put       pshmem_ctx_longlong_put
-#define shmem_ctx_schar_put          pshmem_ctx_schar_put
-#define shmem_ctx_uchar_put          pshmem_ctx_uchar_put
-#define shmem_ctx_ushort_put         pshmem_ctx_ushort_put
-#define shmem_ctx_uint_put           pshmem_ctx_uint_put
-#define shmem_ctx_ulong_put          pshmem_ctx_ulong_put
-#define shmem_ctx_ulonglong_put      pshmem_ctx_ulonglong_put
-#define shmem_ctx_longdouble_put     pshmem_ctx_longdouble_put
-#define shmem_ctx_int8_put           pshmem_ctx_int8_put
-#define shmem_ctx_int16_put          pshmem_ctx_int16_put
-#define shmem_ctx_int32_put          pshmem_ctx_int32_put
-#define shmem_ctx_int64_put          pshmem_ctx_int64_put
-#define shmem_ctx_uint8_put          pshmem_ctx_uint8_put
-#define shmem_ctx_uint16_put         pshmem_ctx_uint16_put
-#define shmem_ctx_uint32_put         pshmem_ctx_uint32_put
-#define shmem_ctx_uint64_put         pshmem_ctx_uint64_put
-#define shmem_ctx_size_put           pshmem_ctx_size_put
-#define shmem_ctx_ptrdiff_put        pshmem_ctx_ptrdiff_put
-
-#define shmem_char_put               pshmem_char_put /* shmem-compat.h */
-#define shmem_short_put              pshmem_short_put
-#define shmem_int_put                pshmem_int_put
-#define shmem_long_put               pshmem_long_put
-#define shmem_float_put              pshmem_float_put
-#define shmem_double_put             pshmem_double_put
-#define shmem_longlong_put           pshmem_longlong_put
-#define shmem_schar_put              pshmem_schar_put
-#define shmem_uchar_put              pshmem_uchar_put
-#define shmem_ushort_put             pshmem_ushort_put
-#define shmem_uint_put               pshmem_uint_put
-#define shmem_ulong_put              pshmem_ulong_put
-#define shmem_ulonglong_put          pshmem_ulonglong_put
-#define shmem_longdouble_put         pshmem_longdouble_put
-#define shmem_int8_put               pshmem_int8_put
-#define shmem_int16_put              pshmem_int16_put
-#define shmem_int32_put              pshmem_int32_put
-#define shmem_int64_put              pshmem_int64_put
-#define shmem_uint8_put              pshmem_uint8_put
-#define shmem_uint16_put             pshmem_uint16_put
-#define shmem_uint32_put             pshmem_uint32_put
-#define shmem_uint64_put             pshmem_uint64_put
-#define shmem_size_put               pshmem_size_put
-#define shmem_ptrdiff_put            pshmem_ptrdiff_put
-
-#define shmem_ctx_put8               pshmem_ctx_put8
-#define shmem_ctx_put16              pshmem_ctx_put16
-#define shmem_ctx_put32              pshmem_ctx_put32
-#define shmem_ctx_put64              pshmem_ctx_put64
-#define shmem_ctx_put128             pshmem_ctx_put128
-#define shmem_ctx_putmem             pshmem_ctx_putmem
-
-#define shmem_put8                   pshmem_put8
-#define shmem_put16                  pshmem_put16
-#define shmem_put32                  pshmem_put32
-#define shmem_put64                  pshmem_put64
-#define shmem_put128                 pshmem_put128
-#define shmem_putmem                 pshmem_putmem
-
-/*
- * Strided put routines
- */
-#define shmem_ctx_char_iput           pshmem_ctx_char_iput
-#define shmem_ctx_short_iput          pshmem_ctx_short_iput
-#define shmem_ctx_int_iput            pshmem_ctx_int_iput
-#define shmem_ctx_long_iput           pshmem_ctx_long_iput
-#define shmem_ctx_float_iput          pshmem_ctx_float_iput
-#define shmem_ctx_double_iput         pshmem_ctx_double_iput
-#define shmem_ctx_longlong_iput       pshmem_ctx_longlong_iput
-#define shmem_ctx_schar_iput          pshmem_ctx_schar_iput
-#define shmem_ctx_uchar_iput          pshmem_ctx_uchar_iput
-#define shmem_ctx_ushort_iput         pshmem_ctx_ushort_iput
-#define shmem_ctx_uint_iput           pshmem_ctx_uint_iput
-#define shmem_ctx_ulong_iput          pshmem_ctx_ulong_iput
-#define shmem_ctx_ulonglong_iput      pshmem_ctx_ulonglong_iput
-#define shmem_ctx_longdouble_iput     pshmem_ctx_longdouble_iput
-#define shmem_ctx_int8_iput           pshmem_ctx_int8_iput
-#define shmem_ctx_int16_iput          pshmem_ctx_int16_iput
-#define shmem_ctx_int32_iput          pshmem_ctx_int32_iput
-#define shmem_ctx_int64_iput          pshmem_ctx_int64_iput
-#define shmem_ctx_uint8_iput          pshmem_ctx_uint8_iput
-#define shmem_ctx_uint16_iput         pshmem_ctx_uint16_iput
-#define shmem_ctx_uint32_iput         pshmem_ctx_uint32_iput
-#define shmem_ctx_uint64_iput         pshmem_ctx_uint64_iput
-#define shmem_ctx_size_iput           pshmem_ctx_size_iput
-#define shmem_ctx_ptrdiff_iput        pshmem_ctx_ptrdiff_iput
-
-#define shmem_char_iput               pshmem_char_iput
-#define shmem_short_iput              pshmem_short_iput
-#define shmem_int_iput                pshmem_int_iput
-#define shmem_long_iput               pshmem_long_iput
-#define shmem_float_iput              pshmem_float_iput
-#define shmem_double_iput             pshmem_double_iput
-#define shmem_longlong_iput           pshmem_longlong_iput
-#define shmem_schar_iput              pshmem_schar_iput
-#define shmem_uchar_iput              pshmem_uchar_iput
-#define shmem_ushort_iput             pshmem_ushort_iput
-#define shmem_uint_iput               pshmem_uint_iput
-#define shmem_ulong_iput              pshmem_ulong_iput
-#define shmem_ulonglong_iput          pshmem_ulonglong_iput
-#define shmem_longdouble_iput         pshmem_longdouble_iput
-#define shmem_int8_iput               pshmem_int8_iput
-#define shmem_int16_iput              pshmem_int16_iput
-#define shmem_int32_iput              pshmem_int32_iput
-#define shmem_int64_iput              pshmem_int64_iput
-#define shmem_uint8_iput              pshmem_uint8_iput
-#define shmem_uint16_iput             pshmem_uint16_iput
-#define shmem_uint32_iput             pshmem_uint32_iput
-#define shmem_uint64_iput             pshmem_uint64_iput
-#define shmem_size_iput               pshmem_size_iput
-#define shmem_ptrdiff_iput            pshmem_ptrdiff_iput
-
-#define shmem_ctx_iput8              pshmem_ctx_iput8
-#define shmem_ctx_iput16             pshmem_ctx_iput16
-#define shmem_ctx_iput32             pshmem_ctx_iput32
-#define shmem_ctx_iput64             pshmem_ctx_iput64
-#define shmem_ctx_iput128            pshmem_ctx_iput128
-
-#define shmem_iput8                  pshmem_iput8
-#define shmem_iput16                 pshmem_iput16
-#define shmem_iput32                 pshmem_iput32
-#define shmem_iput64                 pshmem_iput64
-#define shmem_iput128                pshmem_iput128
-
-/*
- * Non-block data put routines
- */
-#define shmem_ctx_char_put_nbi           pshmem_ctx_char_put_nbi
-#define shmem_ctx_short_put_nbi          pshmem_ctx_short_put_nbi
-#define shmem_ctx_int_put_nbi            pshmem_ctx_int_put_nbi
-#define shmem_ctx_long_put_nbi           pshmem_ctx_long_put_nbi
-#define shmem_ctx_float_put_nbi          pshmem_ctx_float_put_nbi
-#define shmem_ctx_double_put_nbi         pshmem_ctx_double_put_nbi
-#define shmem_ctx_longlong_put_nbi       pshmem_ctx_longlong_put_nbi
-#define shmem_ctx_schar_put_nbi          pshmem_ctx_schar_put_nbi
-#define shmem_ctx_uchar_put_nbi          pshmem_ctx_uchar_put_nbi
-#define shmem_ctx_ushort_put_nbi         pshmem_ctx_ushort_put_nbi
-#define shmem_ctx_uint_put_nbi           pshmem_ctx_uint_put_nbi
-#define shmem_ctx_ulong_put_nbi          pshmem_ctx_ulong_put_nbi
-#define shmem_ctx_ulonglong_put_nbi      pshmem_ctx_ulonglong_put_nbi
-#define shmem_ctx_longdouble_put_nbi     pshmem_ctx_longdouble_put_nbi
-#define shmem_ctx_int8_put_nbi           pshmem_ctx_int8_put_nbi
-#define shmem_ctx_int16_put_nbi          pshmem_ctx_int16_put_nbi
-#define shmem_ctx_int32_put_nbi          pshmem_ctx_int32_put_nbi
-#define shmem_ctx_int64_put_nbi          pshmem_ctx_int64_put_nbi
-#define shmem_ctx_uint8_put_nbi          pshmem_ctx_uint8_put_nbi
-#define shmem_ctx_uint16_put_nbi         pshmem_ctx_uint16_put_nbi
-#define shmem_ctx_uint32_put_nbi         pshmem_ctx_uint32_put_nbi
-#define shmem_ctx_uint64_put_nbi         pshmem_ctx_uint64_put_nbi
-#define shmem_ctx_size_put_nbi           pshmem_ctx_size_put_nbi
-#define shmem_ctx_ptrdiff_put_nbi        pshmem_ctx_ptrdiff_put_nbi
-
-#define shmem_char_put_nbi               pshmem_char_put_nbi
-#define shmem_short_put_nbi              pshmem_short_put_nbi
-#define shmem_int_put_nbi                pshmem_int_put_nbi
-#define shmem_long_put_nbi               pshmem_long_put_nbi
-#define shmem_float_put_nbi              pshmem_float_put_nbi
-#define shmem_double_put_nbi             pshmem_double_put_nbi
-#define shmem_longlong_put_nbi           pshmem_longlong_put_nbi
-#define shmem_schar_put_nbi              pshmem_schar_put_nbi
-#define shmem_uchar_put_nbi              pshmem_uchar_put_nbi
-#define shmem_ushort_put_nbi             pshmem_ushort_put_nbi
-#define shmem_uint_put_nbi               pshmem_uint_put_nbi
-#define shmem_ulong_put_nbi              pshmem_ulong_put_nbi
-#define shmem_ulonglong_put_nbi          pshmem_ulonglong_put_nbi
-#define shmem_longdouble_put_nbi         pshmem_longdouble_put_nbi
-#define shmem_int8_put_nbi               pshmem_int8_put_nbi
-#define shmem_int16_put_nbi              pshmem_int16_put_nbi
-#define shmem_int32_put_nbi              pshmem_int32_put_nbi
-#define shmem_int64_put_nbi              pshmem_int64_put_nbi
-#define shmem_uint8_put_nbi              pshmem_uint8_put_nbi
-#define shmem_uint16_put_nbi             pshmem_uint16_put_nbi
-#define shmem_uint32_put_nbi             pshmem_uint32_put_nbi
-#define shmem_uint64_put_nbi             pshmem_uint64_put_nbi
-#define shmem_size_put_nbi               pshmem_size_put_nbi
-#define shmem_ptrdiff_put_nbi            pshmem_ptrdiff_put_nbi
-
-#define shmem_ctx_put8_nbi           pshmem_ctx_put8_nbi
-#define shmem_ctx_put16_nbi          pshmem_ctx_put16_nbi
-#define shmem_ctx_put32_nbi          pshmem_ctx_put32_nbi
-#define shmem_ctx_put64_nbi          pshmem_ctx_put64_nbi
-#define shmem_ctx_put128_nbi         pshmem_ctx_put128_nbi
-#define shmem_ctx_putmem_nbi         pshmem_ctx_putmem_nbi
-
-#define shmem_put8_nbi               pshmem_put8_nbi
-#define shmem_put16_nbi              pshmem_put16_nbi
-#define shmem_put32_nbi              pshmem_put32_nbi
-#define shmem_put64_nbi              pshmem_put64_nbi
-#define shmem_put128_nbi             pshmem_put128_nbi
-#define shmem_putmem_nbi             pshmem_putmem_nbi
-
-/*
- * Elemental get routines
- */
-#define shmem_ctx_char_g             pshmem_ctx_char_g
-#define shmem_ctx_short_g            pshmem_ctx_short_g
-#define shmem_ctx_int_g              pshmem_ctx_int_g
-#define shmem_ctx_long_g             pshmem_ctx_long_g
-#define shmem_ctx_float_g            pshmem_ctx_float_g
-#define shmem_ctx_double_g           pshmem_ctx_double_g
-#define shmem_ctx_longlong_g         pshmem_ctx_longlong_g
-#define shmem_ctx_schar_g            pshmem_ctx_schar_g
-#define shmem_ctx_uchar_g            pshmem_ctx_uchar_g
-#define shmem_ctx_ushort_g           pshmem_ctx_ushort_g
-#define shmem_ctx_uint_g             pshmem_ctx_uint_g
-#define shmem_ctx_ulong_g            pshmem_ctx_ulong_g
-#define shmem_ctx_ulonglong_g        pshmem_ctx_ulonglong_g
-#define shmem_ctx_longdouble_g       pshmem_ctx_longdouble_g
-#define shmem_ctx_int8_g             pshmem_ctx_int8_g
-#define shmem_ctx_int16_g            pshmem_ctx_int16_g
-#define shmem_ctx_int32_g            pshmem_ctx_int32_g
-#define shmem_ctx_int64_g            pshmem_ctx_int64_g
-#define shmem_ctx_uint8_g            pshmem_ctx_uint8_g
-#define shmem_ctx_uint16_g           pshmem_ctx_uint16_g
-#define shmem_ctx_uint32_g           pshmem_ctx_uint32_g
-#define shmem_ctx_uint64_g           pshmem_ctx_uint64_g
-#define shmem_ctx_size_g             pshmem_ctx_size_g
-#define shmem_ctx_ptrdiff_g          pshmem_ctx_ptrdiff_g
-
-#define shmem_char_g                 pshmem_char_g
-#define shmem_short_g                pshmem_short_g
-#define shmem_int_g                  pshmem_int_g
-#define shmem_long_g                 pshmem_long_g
-#define shmem_float_g                pshmem_float_g
-#define shmem_double_g               pshmem_double_g
-#define shmem_longlong_g             pshmem_longlong_g
-#define shmem_schar_g                pshmem_schar_g
-#define shmem_uchar_g                pshmem_uchar_g
-#define shmem_ushort_g               pshmem_ushort_g
-#define shmem_uint_g                 pshmem_uint_g
-#define shmem_ulong_g                pshmem_ulong_g
-#define shmem_ulonglong_g            pshmem_ulonglong_g
-#define shmem_longdouble_g           pshmem_longdouble_g
-#define shmem_int8_g                 pshmem_int8_g
-#define shmem_int16_g                pshmem_int16_g
-#define shmem_int32_g                pshmem_int32_g
-#define shmem_int64_g                pshmem_int64_g
-#define shmem_uint8_g                pshmem_uint8_g
-#define shmem_uint16_g               pshmem_uint16_g
-#define shmem_uint32_g               pshmem_uint32_g
-#define shmem_uint64_g               pshmem_uint64_g
-#define shmem_size_g                 pshmem_size_g
-#define shmem_ptrdiff_g              pshmem_ptrdiff_g
-
-#define shmemx_int16_g               pshmemx_int16_g
-#define shmemx_int32_g               pshmemx_int32_g
-#define shmemx_int64_g               pshmemx_int64_g
-
-/*
- * Block data get routines
- */
-#define shmem_ctx_char_get           pshmem_ctx_char_get
-#define shmem_ctx_short_get          pshmem_ctx_short_get
-#define shmem_ctx_int_get            pshmem_ctx_int_get
-#define shmem_ctx_long_get           pshmem_ctx_long_get
-#define shmem_ctx_float_get          pshmem_ctx_float_get
-#define shmem_ctx_double_get         pshmem_ctx_double_get
-#define shmem_ctx_longlong_get       pshmem_ctx_longlong_get
-#define shmem_ctx_schar_get          pshmem_ctx_schar_get
-#define shmem_ctx_uchar_get          pshmem_ctx_uchar_get
-#define shmem_ctx_ushort_get         pshmem_ctx_ushort_get
-#define shmem_ctx_uint_get           pshmem_ctx_uint_get
-#define shmem_ctx_ulong_get          pshmem_ctx_ulong_get
-#define shmem_ctx_ulonglong_get      pshmem_ctx_ulonglong_get
-#define shmem_ctx_longdouble_get     pshmem_ctx_longdouble_get
-#define shmem_ctx_int8_get           pshmem_ctx_int8_get
-#define shmem_ctx_int16_get          pshmem_ctx_int16_get
-#define shmem_ctx_int32_get          pshmem_ctx_int32_get
-#define shmem_ctx_int64_get          pshmem_ctx_int64_get
-#define shmem_ctx_uint8_get          pshmem_ctx_uint8_get
-#define shmem_ctx_uint16_get         pshmem_ctx_uint16_get
-#define shmem_ctx_uint32_get         pshmem_ctx_uint32_get
-#define shmem_ctx_uint64_get         pshmem_ctx_uint64_get
-#define shmem_ctx_size_get           pshmem_ctx_size_get
-#define shmem_ctx_ptrdiff_get        pshmem_ctx_ptrdiff_get
-
-#define shmem_char_get               pshmem_char_get /* shmem-compat.h */
-#define shmem_short_get              pshmem_short_get
-#define shmem_int_get                pshmem_int_get
-#define shmem_long_get               pshmem_long_get
-#define shmem_float_get              pshmem_float_get
-#define shmem_double_get             pshmem_double_get
-#define shmem_longlong_get           pshmem_longlong_get
-#define shmem_schar_get              pshmem_schar_get
-#define shmem_uchar_get              pshmem_uchar_get
-#define shmem_ushort_get             pshmem_ushort_get
-#define shmem_uint_get               pshmem_uint_get
-#define shmem_ulong_get              pshmem_ulong_get
-#define shmem_ulonglong_get          pshmem_ulonglong_get
-#define shmem_longdouble_get         pshmem_longdouble_get
-#define shmem_int8_get               pshmem_int8_get
-#define shmem_int16_get              pshmem_int16_get
-#define shmem_int32_get              pshmem_int32_get
-#define shmem_int64_get              pshmem_int64_get
-#define shmem_uint8_get              pshmem_uint8_get
-#define shmem_uint16_get             pshmem_uint16_get
-#define shmem_uint32_get             pshmem_uint32_get
-#define shmem_uint64_get             pshmem_uint64_get
-#define shmem_size_get               pshmem_size_get
-#define shmem_ptrdiff_get            pshmem_ptrdiff_get
-
-#define shmem_ctx_get8               pshmem_ctx_get8
-#define shmem_ctx_get16              pshmem_ctx_get16
-#define shmem_ctx_get32              pshmem_ctx_get32
-#define shmem_ctx_get64              pshmem_ctx_get64
-#define shmem_ctx_get128             pshmem_ctx_get128
-#define shmem_ctx_getmem             pshmem_ctx_getmem
-
-#define shmem_get8                   pshmem_get8
-#define shmem_get16                  pshmem_get16
-#define shmem_get32                  pshmem_get32
-#define shmem_get64                  pshmem_get64
-#define shmem_get128                 pshmem_get128
-#define shmem_getmem                 pshmem_getmem
-
-/*
- * Strided get routines
- */
-#define shmem_ctx_char_iget           pshmem_ctx_char_iget
-#define shmem_ctx_short_iget          pshmem_ctx_short_iget
-#define shmem_ctx_int_iget            pshmem_ctx_int_iget
-#define shmem_ctx_long_iget           pshmem_ctx_long_iget
-#define shmem_ctx_float_iget          pshmem_ctx_float_iget
-#define shmem_ctx_double_iget         pshmem_ctx_double_iget
-#define shmem_ctx_longlong_iget       pshmem_ctx_longlong_iget
-#define shmem_ctx_schar_iget          pshmem_ctx_schar_iget
-#define shmem_ctx_uchar_iget          pshmem_ctx_uchar_iget
-#define shmem_ctx_ushort_iget         pshmem_ctx_ushort_iget
-#define shmem_ctx_uint_iget           pshmem_ctx_uint_iget
-#define shmem_ctx_ulong_iget          pshmem_ctx_ulong_iget
-#define shmem_ctx_ulonglong_iget      pshmem_ctx_ulonglong_iget
-#define shmem_ctx_longdouble_iget     pshmem_ctx_longdouble_iget
-#define shmem_ctx_int8_iget           pshmem_ctx_int8_iget
-#define shmem_ctx_int16_iget          pshmem_ctx_int16_iget
-#define shmem_ctx_int32_iget          pshmem_ctx_int32_iget
-#define shmem_ctx_int64_iget          pshmem_ctx_int64_iget
-#define shmem_ctx_uint8_iget          pshmem_ctx_uint8_iget
-#define shmem_ctx_uint16_iget         pshmem_ctx_uint16_iget
-#define shmem_ctx_uint32_iget         pshmem_ctx_uint32_iget
-#define shmem_ctx_uint64_iget         pshmem_ctx_uint64_iget
-#define shmem_ctx_size_iget           pshmem_ctx_size_iget
-#define shmem_ctx_ptrdiff_iget        pshmem_ctx_ptrdiff_iget
-
-#define shmem_char_iget               pshmem_char_iget
-#define shmem_short_iget              pshmem_short_iget
-#define shmem_int_iget                pshmem_int_iget
-#define shmem_long_iget               pshmem_long_iget
-#define shmem_float_iget              pshmem_float_iget
-#define shmem_double_iget             pshmem_double_iget
-#define shmem_longlong_iget           pshmem_longlong_iget
-#define shmem_schar_iget              pshmem_schar_iget
-#define shmem_uchar_iget              pshmem_uchar_iget
-#define shmem_ushort_iget             pshmem_ushort_iget
-#define shmem_uint_iget               pshmem_uint_iget
-#define shmem_ulong_iget              pshmem_ulong_iget
-#define shmem_ulonglong_iget          pshmem_ulonglong_iget
-#define shmem_longdouble_iget         pshmem_longdouble_iget
-#define shmem_int8_iget               pshmem_int8_iget
-#define shmem_int16_iget              pshmem_int16_iget
-#define shmem_int32_iget              pshmem_int32_iget
-#define shmem_int64_iget              pshmem_int64_iget
-#define shmem_uint8_iget              pshmem_uint8_iget
-#define shmem_uint16_iget             pshmem_uint16_iget
-#define shmem_uint32_iget             pshmem_uint32_iget
-#define shmem_uint64_iget             pshmem_uint64_iget
-#define shmem_size_iget               pshmem_size_iget
-#define shmem_ptrdiff_iget            pshmem_ptrdiff_iget
-
-#define shmem_ctx_iget8              pshmem_ctx_iget8
-#define shmem_ctx_iget16             pshmem_ctx_iget16
-#define shmem_ctx_iget32             pshmem_ctx_iget32
-#define shmem_ctx_iget64             pshmem_ctx_iget64
-#define shmem_ctx_iget128            pshmem_ctx_iget128
-
-#define shmem_iget8                  pshmem_iget8
-#define shmem_iget16                 pshmem_iget16
-#define shmem_iget32                 pshmem_iget32
-#define shmem_iget64                 pshmem_iget64
-#define shmem_iget128                pshmem_iget128
-
-/*
- * Non-block data get routines
- */
-#define shmem_ctx_char_get_nbi           pshmem_ctx_char_get_nbi
-#define shmem_ctx_short_get_nbi          pshmem_ctx_short_get_nbi
-#define shmem_ctx_int_get_nbi            pshmem_ctx_int_get_nbi
-#define shmem_ctx_long_get_nbi           pshmem_ctx_long_get_nbi
-#define shmem_ctx_float_get_nbi          pshmem_ctx_float_get_nbi
-#define shmem_ctx_double_get_nbi         pshmem_ctx_double_get_nbi
-#define shmem_ctx_longlong_get_nbi       pshmem_ctx_longlong_get_nbi
-#define shmem_ctx_schar_get_nbi          pshmem_ctx_schar_get_nbi
-#define shmem_ctx_uchar_get_nbi          pshmem_ctx_uchar_get_nbi
-#define shmem_ctx_ushort_get_nbi         pshmem_ctx_ushort_get_nbi
-#define shmem_ctx_uint_get_nbi           pshmem_ctx_uint_get_nbi
-#define shmem_ctx_ulong_get_nbi          pshmem_ctx_ulong_get_nbi
-#define shmem_ctx_ulonglong_get_nbi      pshmem_ctx_ulonglong_get_nbi
-#define shmem_ctx_longdouble_get_nbi     pshmem_ctx_longdouble_get_nbi
-#define shmem_ctx_int8_get_nbi           pshmem_ctx_int8_get_nbi
-#define shmem_ctx_int16_get_nbi          pshmem_ctx_int16_get_nbi
-#define shmem_ctx_int32_get_nbi          pshmem_ctx_int32_get_nbi
-#define shmem_ctx_int64_get_nbi          pshmem_ctx_int64_get_nbi
-#define shmem_ctx_uint8_get_nbi          pshmem_ctx_uint8_get_nbi
-#define shmem_ctx_uint16_get_nbi         pshmem_ctx_uint16_get_nbi
-#define shmem_ctx_uint32_get_nbi         pshmem_ctx_uint32_get_nbi
-#define shmem_ctx_uint64_get_nbi         pshmem_ctx_uint64_get_nbi
-#define shmem_ctx_size_get_nbi           pshmem_ctx_size_get_nbi
-#define shmem_ctx_ptrdiff_get_nbi        pshmem_ctx_ptrdiff_get_nbi
-
-#define shmem_char_get_nbi               pshmem_char_get_nbi
-#define shmem_short_get_nbi              pshmem_short_get_nbi
-#define shmem_int_get_nbi                pshmem_int_get_nbi
-#define shmem_long_get_nbi               pshmem_long_get_nbi
-#define shmem_float_get_nbi              pshmem_float_get_nbi
-#define shmem_double_get_nbi             pshmem_double_get_nbi
-#define shmem_longlong_get_nbi           pshmem_longlong_get_nbi
-#define shmem_schar_get_nbi              pshmem_schar_get_nbi
-#define shmem_uchar_get_nbi              pshmem_uchar_get_nbi
-#define shmem_ushort_get_nbi             pshmem_ushort_get_nbi
-#define shmem_uint_get_nbi               pshmem_uint_get_nbi
-#define shmem_ulong_get_nbi              pshmem_ulong_get_nbi
-#define shmem_ulonglong_get_nbi          pshmem_ulonglong_get_nbi
-#define shmem_longdouble_get_nbi         pshmem_longdouble_get_nbi
-#define shmem_int8_get_nbi               pshmem_int8_get_nbi
-#define shmem_int16_get_nbi              pshmem_int16_get_nbi
-#define shmem_int32_get_nbi              pshmem_int32_get_nbi
-#define shmem_int64_get_nbi              pshmem_int64_get_nbi
-#define shmem_uint8_get_nbi              pshmem_uint8_get_nbi
-#define shmem_uint16_get_nbi             pshmem_uint16_get_nbi
-#define shmem_uint32_get_nbi             pshmem_uint32_get_nbi
-#define shmem_uint64_get_nbi             pshmem_uint64_get_nbi
-#define shmem_size_get_nbi               pshmem_size_get_nbi
-#define shmem_ptrdiff_get_nbi            pshmem_ptrdiff_get_nbi
-
-#define shmem_ctx_get8_nbi           pshmem_ctx_get8_nbi
-#define shmem_ctx_get16_nbi          pshmem_ctx_get16_nbi
-#define shmem_ctx_get32_nbi          pshmem_ctx_get32_nbi
-#define shmem_ctx_get64_nbi          pshmem_ctx_get64_nbi
-#define shmem_ctx_get128_nbi         pshmem_ctx_get128_nbi
-#define shmem_ctx_getmem_nbi         pshmem_ctx_getmem_nbi
-
-#define shmem_get8_nbi               pshmem_get8_nbi
-#define shmem_get16_nbi              pshmem_get16_nbi
-#define shmem_get32_nbi              pshmem_get32_nbi
-#define shmem_get64_nbi              pshmem_get64_nbi
-#define shmem_get128_nbi             pshmem_get128_nbi
-#define shmem_getmem_nbi             pshmem_getmem_nbi
-
-/*
- * Atomic operations
- */
-/* Atomic swap */
-#define shmem_ctx_double_atomic_swap pshmem_ctx_double_atomic_swap
-#define shmem_ctx_float_atomic_swap  pshmem_ctx_float_atomic_swap
-#define shmem_ctx_int_atomic_swap    pshmem_ctx_int_atomic_swap
-#define shmem_ctx_long_atomic_swap   pshmem_ctx_long_atomic_swap
-#define shmem_ctx_longlong_atomic_swap pshmem_ctx_longlong_atomic_swap
-#define shmem_ctx_uint_atomic_swap   pshmem_ctx_uint_atomic_swap
-#define shmem_ctx_ulong_atomic_swap  pshmem_ctx_ulong_atomic_swap
-#define shmem_ctx_ulonglong_atomic_swap pshmem_ctx_ulonglong_atomic_swap
-
-#define shmem_double_atomic_swap     pshmem_double_atomic_swap
-#define shmem_float_atomic_swap      pshmem_float_atomic_swap
-#define shmem_int_atomic_swap        pshmem_int_atomic_swap
-#define shmem_long_atomic_swap       pshmem_long_atomic_swap
-#define shmem_longlong_atomic_swap   pshmem_longlong_atomic_swap
-#define shmem_uint_atomic_swap       pshmem_uint_atomic_swap
-#define shmem_ulong_atomic_swap      pshmem_ulong_atomic_swap
-#define shmem_ulonglong_atomic_swap  pshmem_ulonglong_atomic_swap
-
-#define shmem_double_swap            pshmem_double_swap
-#define shmem_float_swap             pshmem_float_swap
-#define shmem_int_swap               pshmem_int_swap
-#define shmem_long_swap              pshmem_long_swap
-#define shmem_longlong_swap          pshmem_longlong_swap
-
-#define shmemx_int32_swap            pshmemx_int32_swap
-#define shmemx_int64_swap            pshmemx_int64_swap
-
-/* Atomic set */
-#define shmem_ctx_double_atomic_set pshmem_ctx_double_atomic_set
-#define shmem_ctx_float_atomic_set  pshmem_ctx_float_atomic_set
-#define shmem_ctx_int_atomic_set    pshmem_ctx_int_atomic_set
-#define shmem_ctx_long_atomic_set   pshmem_ctx_long_atomic_set
-#define shmem_ctx_longlong_atomic_set pshmem_ctx_longlong_atomic_set
-#define shmem_ctx_uint_atomic_set   pshmem_ctx_uint_atomic_set
-#define shmem_ctx_ulong_atomic_set  pshmem_ctx_ulong_atomic_set
-#define shmem_ctx_ulonglong_atomic_set pshmem_ctx_ulonglong_atomic_set
-
-#define shmem_double_atomic_set     pshmem_double_atomic_set
-#define shmem_float_atomic_set      pshmem_float_atomic_set
-#define shmem_int_atomic_set        pshmem_int_atomic_set
-#define shmem_long_atomic_set       pshmem_long_atomic_set
-#define shmem_longlong_atomic_set   pshmem_longlong_atomic_set
-#define shmem_uint_atomic_set       pshmem_uint_atomic_set
-#define shmem_ulong_atomic_set      pshmem_ulong_atomic_set
-#define shmem_ulonglong_atomic_set  pshmem_ulonglong_atomic_set
-
-#define shmem_double_set            pshmem_double_set
-#define shmem_float_set             pshmem_float_set
-#define shmem_int_set               pshmem_int_set
-#define shmem_long_set              pshmem_long_set
-#define shmem_longlong_set          pshmem_longlong_set
-
-#define shmemx_int32_set            pshmemx_int32_set
-#define shmemx_int64_set            pshmemx_int64_set
-
-/* Atomic conditional swap */
-#define shmem_ctx_int_atomic_compare_swap   pshmem_ctx_int_atomic_compare_swap
-#define shmem_ctx_long_atomic_compare_swap  pshmem_ctx_long_atomic_compare_swap
-#define shmem_ctx_longlong_atomic_compare_swap pshmem_ctx_longlong_atomic_compare_swap
-#define shmem_ctx_uint_atomic_compare_swap  pshmem_ctx_uint_atomic_compare_swap
-#define shmem_ctx_ulong_atomic_compare_swap pshmem_ctx_ulong_atomic_compare_swap
-#define shmem_ctx_ulonglong_atomic_compare_swap pshmem_ctx_ulonglong_atomic_compare_swap
-
-#define shmem_int_atomic_compare_swap       pshmem_int_atomic_compare_swap
-#define shmem_long_atomic_compare_swap      pshmem_long_atomic_compare_swap
-#define shmem_longlong_atomic_compare_swap  pshmem_longlong_atomic_compare_swap
-#define shmem_uint_atomic_compare_swap      pshmem_uint_atomic_compare_swap
-#define shmem_ulong_atomic_compare_swap     pshmem_ulong_atomic_compare_swap
-#define shmem_ulonglong_atomic_compare_swap pshmem_ulonglong_atomic_compare_swap
-
-#define shmem_int_cswap              pshmem_int_cswap
-#define shmem_long_cswap             pshmem_long_cswap
-#define shmem_longlong_cswap         pshmem_longlong_cswap
-
-#define shmemx_int32_cswap           pshmemx_int32_cswap
-#define shmemx_int64_cswap           pshmemx_int64_cswap
-
-/* Atomic Fetch&Add */
-#define shmem_ctx_int_atomic_fetch_add       pshmem_ctx_int_atomic_fetch_add
-#define shmem_ctx_long_atomic_fetch_add      pshmem_ctx_long_atomic_fetch_add
-#define shmem_ctx_longlong_atomic_fetch_add  pshmem_ctx_longlong_atomic_fetch_add
-#define shmem_ctx_uint_atomic_fetch_add      pshmem_ctx_uint_atomic_fetch_add
-#define shmem_ctx_ulong_atomic_fetch_add     pshmem_ctx_ulong_atomic_fetch_add
-#define shmem_ctx_ulonglong_atomic_fetch_add pshmem_ctx_ulonglong_atomic_fetch_add
-
-#define shmem_int_atomic_fetch_add           pshmem_int_atomic_fetch_add
-#define shmem_long_atomic_fetch_add          pshmem_long_atomic_fetch_add
-#define shmem_longlong_atomic_fetch_add      pshmem_longlong_atomic_fetch_add
-#define shmem_uint_atomic_fetch_add          pshmem_uint_atomic_fetch_add
-#define shmem_ulong_atomic_fetch_add         pshmem_ulong_atomic_fetch_add
-#define shmem_ulonglong_atomic_fetch_add     pshmem_ulonglong_atomic_fetch_add
-
-#define shmem_int_fadd                       pshmem_int_fadd
-#define shmem_long_fadd                      pshmem_long_fadd
-#define shmem_longlong_fadd                  pshmem_longlong_fadd
-
-#define shmemx_int32_fadd                    pshmemx_int32_fadd
-#define shmemx_int64_fadd                    pshmemx_int64_fadd
-
-/* Atomic Fetch&And */
-#define shmem_int_atomic_fetch_and        pshmem_int_atomic_fetch_and
-#define shmem_long_atomic_fetch_and       pshmem_long_atomic_fetch_and
-#define shmem_longlong_atomic_fetch_and   pshmem_longlong_atomic_fetch_and
-#define shmem_uint_atomic_fetch_and       pshmem_uint_atomic_fetch_and
-#define shmem_ulong_atomic_fetch_and      pshmem_ulong_atomic_fetch_and
-#define shmem_ulonglong_atomic_fetch_and  pshmem_ulonglong_atomic_fetch_and
-#define shmem_int32_atomic_fetch_and      pshmem_int32_atomic_fetch_and
-#define shmem_int64_atomic_fetch_and      pshmem_int64_atomic_fetch_and
-#define shmem_uint32_atomic_fetch_and     pshmem_uint32_atomic_fetch_and
-#define shmem_uint64_atomic_fetch_and     pshmem_uint64_atomic_fetch_and
-
-#define shmem_ctx_int_atomic_fetch_and    pshmem_ctx_int_atomic_fetch_and
-#define shmem_ctx_long_atomic_fetch_and   pshmem_ctx_long_atomic_fetch_and
-#define shmem_ctx_longlong_atomic_fetch_and pshmem_ctx_longlong_atomic_fetch_and
-#define shmem_ctx_uint_atomic_fetch_and   pshmem_ctx_uint_atomic_fetch_and
-#define shmem_ctx_ulong_atomic_fetch_and  pshmem_ctx_ulong_atomic_fetch_and
-#define shmem_ctx_ulonglong_atomic_fetch_and pshmem_ctx_ulonglong_atomic_fetch_and
-#define shmem_ctx_int32_atomic_fetch_and  pshmem_ctx_int32_atomic_fetch_and
-#define shmem_ctx_int64_atomic_fetch_and  pshmem_ctx_int64_atomic_fetch_and
-#define shmem_ctx_uint32_atomic_fetch_and pshmem_ctx_uint32_atomic_fetch_and
-#define shmem_ctx_uint64_atomic_fetch_and pshmem_ctx_uint64_atomic_fetch_and
-
-#define shmemx_int32_atomic_fetch_and     pshmemx_int32_atomic_fetch_and
-#define shmemx_int64_atomic_fetch_and     pshmemx_int64_atomic_fetch_and
-#define shmemx_uint32_atomic_fetch_and    pshmemx_uint32_atomic_fetch_and
-#define shmemx_uint64_atomic_fetch_and    pshmemx_uint64_atomic_fetch_and
-
-/* Atomic Fetch&Or */
-#define shmem_int_atomic_fetch_or         pshmem_int_atomic_fetch_or
-#define shmem_long_atomic_fetch_or        pshmem_long_atomic_fetch_or
-#define shmem_longlong_atomic_fetch_or    pshmem_longlong_atomic_fetch_or
-#define shmem_uint_atomic_fetch_or        pshmem_uint_atomic_fetch_or
-#define shmem_ulong_atomic_fetch_or       pshmem_ulong_atomic_fetch_or
-#define shmem_ulonglong_atomic_fetch_or   pshmem_ulonglong_atomic_fetch_or
-#define shmem_int32_atomic_fetch_or       pshmem_int32_atomic_fetch_or
-#define shmem_int64_atomic_fetch_or       pshmem_int64_atomic_fetch_or
-#define shmem_uint32_atomic_fetch_or      pshmem_uint32_atomic_fetch_or
-#define shmem_uint64_atomic_fetch_or      pshmem_uint64_atomic_fetch_or
-
-#define shmem_ctx_int_atomic_fetch_or     pshmem_ctx_int_atomic_fetch_or
-#define shmem_ctx_long_atomic_fetch_or    pshmem_ctx_long_atomic_fetch_or
-#define shmem_ctx_longlong_atomic_fetch_or pshmem_ctx_longlong_atomic_fetch_or
-#define shmem_ctx_uint_atomic_fetch_or    pshmem_ctx_uint_atomic_fetch_or
-#define shmem_ctx_ulong_atomic_fetch_or   pshmem_ctx_ulong_atomic_fetch_or
-#define shmem_ctx_ulonglong_atomic_fetch_or pshmem_ctx_ulonglong_atomic_fetch_or
-#define shmem_ctx_int32_atomic_fetch_or   pshmem_ctx_int32_atomic_fetch_or
-#define shmem_ctx_int64_atomic_fetch_or   pshmem_ctx_int64_atomic_fetch_or
-#define shmem_ctx_uint32_atomic_fetch_or  pshmem_ctx_uint32_atomic_fetch_or
-#define shmem_ctx_uint64_atomic_fetch_or  pshmem_ctx_uint64_atomic_fetch_or
-
-#define shmemx_int32_atomic_fetch_or      pshmemx_int32_atomic_fetch_or
-#define shmemx_int64_atomic_fetch_or      pshmemx_int64_atomic_fetch_or
-#define shmemx_uint32_atomic_fetch_or     pshmemx_uint32_atomic_fetch_or
-#define shmemx_uint64_atomic_fetch_or     pshmemx_uint64_atomic_fetch_or
-
-/* Atomic Fetch&Xor */
-#define shmem_int_atomic_fetch_xor        pshmem_int_atomic_fetch_xor
-#define shmem_long_atomic_fetch_xor       pshmem_long_atomic_fetch_xor
-#define shmem_longlong_atomic_fetch_xor   pshmem_longlong_atomic_fetch_xor
-#define shmem_uint_atomic_fetch_xor       pshmem_uint_atomic_fetch_xor
-#define shmem_ulong_atomic_fetch_xor      pshmem_ulong_atomic_fetch_xor
-#define shmem_ulonglong_atomic_fetch_xor  pshmem_ulonglong_atomic_fetch_xor
-#define shmem_int32_atomic_fetch_xor      pshmem_int32_atomic_fetch_xor
-#define shmem_int64_atomic_fetch_xor      pshmem_int64_atomic_fetch_xor
-#define shmem_uint32_atomic_fetch_xor     pshmem_uint32_atomic_fetch_xor
-#define shmem_uint64_atomic_fetch_xor     pshmem_uint64_atomic_fetch_xor
-
-#define shmem_ctx_int_atomic_fetch_xor    pshmem_ctx_int_atomic_fetch_xor
-#define shmem_ctx_long_atomic_fetch_xor   pshmem_ctx_long_atomic_fetch_xor
-#define shmem_ctx_longlong_atomic_fetch_xor pshmem_ctx_longlong_atomic_fetch_xor
-#define shmem_ctx_uint_atomic_fetch_xor   pshmem_ctx_uint_atomic_fetch_xor
-#define shmem_ctx_ulong_atomic_fetch_xor  pshmem_ctx_ulong_atomic_fetch_xor
-#define shmem_ctx_ulonglong_atomic_fetch_xor pshmem_ctx_ulonglong_atomic_fetch_xor
-#define shmem_ctx_int32_atomic_fetch_xor  pshmem_ctx_int32_atomic_fetch_xor
-#define shmem_ctx_int64_atomic_fetch_xor  pshmem_ctx_int64_atomic_fetch_xor
-#define shmem_ctx_uint32_atomic_fetch_xor pshmem_ctx_uint32_atomic_fetch_xor
-#define shmem_ctx_uint64_atomic_fetch_xor pshmem_ctx_uint64_atomic_fetch_xor
-
-#define shmemx_int32_atomic_fetch_xor     pshmemx_int32_atomic_fetch_xor
-#define shmemx_int64_atomic_fetch_xor     pshmemx_int64_atomic_fetch_xor
-#define shmemx_uint32_atomic_fetch_xor    pshmemx_uint32_atomic_fetch_xor
-#define shmemx_uint64_atomic_fetch_xor    pshmemx_uint64_atomic_fetch_xor
-
-/* Atomic Fetch */
-#define shmem_ctx_double_atomic_fetch pshmem_ctx_double_atomic_fetch
-#define shmem_ctx_float_atomic_fetch  pshmem_ctx_float_atomic_fetch
-#define shmem_ctx_int_atomic_fetch    pshmem_ctx_int_atomic_fetch
-#define shmem_ctx_long_atomic_fetch   pshmem_ctx_long_atomic_fetch
-#define shmem_ctx_longlong_atomic_fetch pshmem_ctx_longlong_atomic_fetch
-#define shmem_ctx_uint_atomic_fetch   pshmem_ctx_uint_atomic_fetch
-#define shmem_ctx_ulong_atomic_fetch  pshmem_ctx_ulong_atomic_fetch
-#define shmem_ctx_ulonglong_atomic_fetch pshmem_ctx_ulonglong_atomic_fetch
-
-#define shmem_double_atomic_fetch     pshmem_double_atomic_fetch
-#define shmem_float_atomic_fetch      pshmem_float_atomic_fetch
-#define shmem_int_atomic_fetch        pshmem_int_atomic_fetch
-#define shmem_long_atomic_fetch       pshmem_long_atomic_fetch
-#define shmem_longlong_atomic_fetch   pshmem_longlong_atomic_fetch
-#define shmem_uint_atomic_fetch       pshmem_uint_atomic_fetch
-#define shmem_ulong_atomic_fetch      pshmem_ulong_atomic_fetch
-#define shmem_ulonglong_atomic_fetch  pshmem_ulonglong_atomic_fetch
-
-#define shmem_double_fetch            pshmem_double_fetch
-#define shmem_float_fetch             pshmem_float_fetch
-#define shmem_int_fetch               pshmem_int_fetch
-#define shmem_long_fetch              pshmem_long_fetch
-#define shmem_longlong_fetch          pshmem_longlong_fetch
-
-#define shmemx_int32_fetch            pshmemx_int32_fetch
-#define shmemx_int64_fetch            pshmemx_int64_fetch
-
-/* Atomic Fetch&Inc */
-#define shmem_ctx_int_atomic_fetch_inc    pshmem_ctx_int_atomic_fetch_inc
-#define shmem_ctx_long_atomic_fetch_inc   pshmem_ctx_long_atomic_fetch_inc
-#define shmem_ctx_longlong_atomic_fetch_inc pshmem_ctx_longlong_atomic_fetch_inc
-#define shmem_ctx_uint_atomic_fetch_inc    pshmem_ctx_uint_atomic_fetch_inc
-#define shmem_ctx_ulong_atomic_fetch_inc   pshmem_ctx_ulong_atomic_fetch_inc
-#define shmem_ctx_ulonglong_atomic_fetch_inc pshmem_ctx_ulonglong_atomic_fetch_inc
-
-#define shmem_uint_atomic_fetch_inc        pshmem_uint_atomic_fetch_inc
-#define shmem_ulong_atomic_fetch_inc       pshmem_ulong_atomic_fetch_inc
-#define shmem_ulonglong_atomic_fetch_inc   pshmem_ulonglong_atomic_fetch_inc
-#define shmem_int_atomic_fetch_inc        pshmem_int_atomic_fetch_inc
-#define shmem_long_atomic_fetch_inc       pshmem_long_atomic_fetch_inc
-#define shmem_longlong_atomic_fetch_inc   pshmem_longlong_atomic_fetch_inc
-
-#define shmem_int_finc               pshmem_int_finc
-#define shmem_long_finc              pshmem_long_finc
-#define shmem_longlong_finc          pshmem_longlong_finc
-
-#define shmemx_int32_finc            pshmemx_int32_finc
-#define shmemx_int64_finc            pshmemx_int64_finc
-
-/* Atomic Add */
-#define shmem_ctx_int_atomic_add     pshmem_ctx_int_atomic_add
-#define shmem_ctx_long_atomic_add    pshmem_ctx_long_atomic_add
-#define shmem_ctx_longlong_atomic_add pshmem_ctx_longlong_atomic_add
-#define shmem_ctx_uint_atomic_add    pshmem_ctx_uint_atomic_add
-#define shmem_ctx_ulong_atomic_add   pshmem_ctx_ulong_atomic_add
-#define shmem_ctx_ulonglong_atomic_add pshmem_ctx_ulonglong_atomic_add
-
-#define shmem_int_atomic_add         pshmem_int_atomic_add
-#define shmem_long_atomic_add        pshmem_long_atomic_add
-#define shmem_longlong_atomic_add    pshmem_longlong_atomic_add
-#define shmem_uint_atomic_add        pshmem_uint_atomic_add
-#define shmem_ulong_atomic_add       pshmem_ulong_atomic_add
-#define shmem_ulonglong_atomic_add   pshmem_ulonglong_atomic_add
-
-#define shmem_int_add                pshmem_int_add
-#define shmem_long_add               pshmem_long_add
-#define shmem_longlong_add           pshmem_longlong_add
-
-#define shmemx_int32_add             pshmemx_int32_add
-#define shmemx_int64_add             pshmemx_int64_add
-
-/* Atomic And */
-#define shmem_int_atomic_and         pshmem_int_atomic_and
-#define shmem_long_atomic_and        pshmem_long_atomic_and
-#define shmem_longlong_atomic_and    pshmem_longlong_atomic_and
-#define shmem_uint_atomic_and        pshmem_uint_atomic_and
-#define shmem_ulong_atomic_and       pshmem_ulong_atomic_and
-#define shmem_ulonglong_atomic_and   pshmem_ulonglong_atomic_and
-#define shmem_int32_atomic_and       pshmem_int32_atomic_and
-#define shmem_int64_atomic_and       pshmem_int64_atomic_and
-#define shmem_uint32_atomic_and      pshmem_uint32_atomic_and
-#define shmem_uint64_atomic_and      pshmem_uint64_atomic_and
-
-#define shmem_ctx_int_atomic_and     pshmem_ctx_int_atomic_and
-#define shmem_ctx_long_atomic_and    pshmem_ctx_long_atomic_and
-#define shmem_ctx_longlong_atomic_and pshmem_ctx_longlong_atomic_and
-#define shmem_ctx_uint_atomic_and    pshmem_ctx_uint_atomic_and
-#define shmem_ctx_ulong_atomic_and   pshmem_ctx_ulong_atomic_and
-#define shmem_ctx_ulonglong_atomic_and pshmem_ctx_ulonglong_atomic_and
-#define shmem_ctx_int32_atomic_and   pshmem_ctx_int32_atomic_and
-#define shmem_ctx_int64_atomic_and   pshmem_ctx_int64_atomic_and
-#define shmem_ctx_uint32_atomic_and  pshmem_ctx_uint32_atomic_and
-#define shmem_ctx_uint64_atomic_and  pshmem_ctx_uint64_atomic_and
-
-#define shmemx_int32_atomic_and      pshmemx_int32_atomic_and
-#define shmemx_int64_atomic_and      pshmemx_int64_atomic_and
-
-#define shmemx_uint32_atomic_and     pshmemx_uint32_atomic_and
-#define shmemx_uint64_atomic_and     pshmemx_uint64_atomic_and
-
-/* Atomic Or */
-#define shmem_int_atomic_or          pshmem_int_atomic_or
-#define shmem_long_atomic_or         pshmem_long_atomic_or
-#define shmem_longlong_atomic_or     pshmem_longlong_atomic_or
-#define shmem_uint_atomic_or         pshmem_uint_atomic_or
-#define shmem_ulong_atomic_or        pshmem_ulong_atomic_or
-#define shmem_ulonglong_atomic_or    pshmem_ulonglong_atomic_or
-#define shmem_int32_atomic_or        pshmem_int32_atomic_or
-#define shmem_int64_atomic_or        pshmem_int64_atomic_or
-#define shmem_uint32_atomic_or       pshmem_uint32_atomic_or
-#define shmem_uint64_atomic_or       pshmem_uint64_atomic_or
-
-#define shmem_ctx_int_atomic_or      pshmem_ctx_int_atomic_or
-#define shmem_ctx_long_atomic_or     pshmem_ctx_long_atomic_or
-#define shmem_ctx_longlong_atomic_or pshmem_ctx_longlong_atomic_or
-#define shmem_ctx_uint_atomic_or     pshmem_ctx_uint_atomic_or
-#define shmem_ctx_ulong_atomic_or    pshmem_ctx_ulong_atomic_or
-#define shmem_ctx_ulonglong_atomic_or pshmem_ctx_ulonglong_atomic_or
-#define shmem_ctx_int32_atomic_or    pshmem_ctx_int32_atomic_or
-#define shmem_ctx_int64_atomic_or    pshmem_ctx_int64_atomic_or
-#define shmem_ctx_uint32_atomic_or   pshmem_ctx_uint32_atomic_or
-#define shmem_ctx_uint64_atomic_or   pshmem_ctx_uint64_atomic_or
-
-#define shmemx_int32_atomic_or       pshmemx_int32_atomic_or
-#define shmemx_int64_atomic_or       pshmemx_int64_atomic_or
-
-#define shmemx_uint32_atomic_or      pshmemx_uint32_atomic_or
-#define shmemx_uint64_atomic_or      pshmemx_uint64_atomic_or
-
-/* Atomic Xor */
-#define shmem_int_atomic_xor         pshmem_int_atomic_xor
-#define shmem_long_atomic_xor        pshmem_long_atomic_xor
-#define shmem_longlong_atomic_xor    pshmem_longlong_atomic_xor
-#define shmem_uint_atomic_xor        pshmem_uint_atomic_xor
-#define shmem_ulong_atomic_xor       pshmem_ulong_atomic_xor
-#define shmem_ulonglong_atomic_xor   pshmem_ulonglong_atomic_xor
-#define shmem_int32_atomic_xor       pshmem_int32_atomic_xor
-#define shmem_int64_atomic_xor       pshmem_int64_atomic_xor
-#define shmem_uint32_atomic_xor      pshmem_uint32_atomic_xor
-#define shmem_uint64_atomic_xor      pshmem_uint64_atomic_xor
-
-#define shmem_ctx_int_atomic_xor     pshmem_ctx_int_atomic_xor
-#define shmem_ctx_long_atomic_xor    pshmem_ctx_long_atomic_xor
-#define shmem_ctx_longlong_atomic_xor pshmem_ctx_longlong_atomic_xor
-#define shmem_ctx_uint_atomic_xor    pshmem_ctx_uint_atomic_xor
-#define shmem_ctx_ulong_atomic_xor   pshmem_ctx_ulong_atomic_xor
-#define shmem_ctx_ulonglong_atomic_xor pshmem_ctx_ulonglong_atomic_xor
-#define shmem_ctx_int32_atomic_xor   pshmem_ctx_int32_atomic_xor
-#define shmem_ctx_int64_atomic_xor   pshmem_ctx_int64_atomic_xor
-#define shmem_ctx_uint32_atomic_xor  pshmem_ctx_uint32_atomic_xor
-#define shmem_ctx_uint64_atomic_xor  pshmem_ctx_uint64_atomic_xor
-
-#define shmemx_int32_atomic_xor      pshmemx_int32_atomic_xor
-#define shmemx_int64_atomic_xor      pshmemx_int64_atomic_xor
-
-#define shmemx_uint32_atomic_xor     pshmemx_uint32_atomic_xor
-#define shmemx_uint64_atomic_xor     pshmemx_uint64_atomic_xor
-
-/* Atomic Inc */
-#define shmem_ctx_int_atomic_inc     pshmem_ctx_int_atomic_inc
-#define shmem_ctx_long_atomic_inc    pshmem_ctx_long_atomic_inc
-#define shmem_ctx_longlong_atomic_inc pshmem_ctx_longlong_atomic_inc
-#define shmem_ctx_uint_atomic_inc    pshmem_ctx_uint_atomic_inc
-#define shmem_ctx_ulong_atomic_inc   pshmem_ctx_ulong_atomic_inc
-#define shmem_ctx_ulonglong_atomic_inc pshmem_ctx_ulonglong_atomic_inc
-
-#define shmem_int_atomic_inc         pshmem_int_atomic_inc
-#define shmem_long_atomic_inc        pshmem_long_atomic_inc
-#define shmem_longlong_atomic_inc    pshmem_longlong_atomic_inc
-#define shmem_uint_atomic_inc        pshmem_uint_atomic_inc
-#define shmem_ulong_atomic_inc       pshmem_ulong_atomic_inc
-#define shmem_ulonglong_atomic_inc   pshmem_ulonglong_atomic_inc
-
-#define shmem_int_inc                pshmem_int_inc
-#define shmem_long_inc               pshmem_long_inc
-#define shmem_longlong_inc           pshmem_longlong_inc
-
-#define shmemx_int32_inc             pshmemx_int32_inc
-#define shmemx_int64_inc             pshmemx_int64_inc
-
-/*
- * Lock functions
- */
-#define shmem_set_lock               pshmem_set_lock
-#define shmem_clear_lock             pshmem_clear_lock
-#define shmem_test_lock              pshmem_test_lock
-
-/*
- * P2P sync routines
- */
-#define shmem_short_wait             pshmem_short_wait
-#define shmem_int_wait               pshmem_int_wait
-#define shmem_long_wait              pshmem_long_wait
-#define shmem_longlong_wait          pshmem_longlong_wait
-#define shmem_wait                   pshmem_wait
-#define shmemx_int32_wait            pshmemx_int32_wait
-#define shmemx_int64_wait            pshmemx_int64_wait
-
-#define shmem_short_wait_until       pshmem_short_wait_until
-#define shmem_int_wait_until         pshmem_int_wait_until
-#define shmem_long_wait_until        pshmem_long_wait_until
-#define shmem_longlong_wait_until    pshmem_longlong_wait_until
-#define shmem_ushort_wait_until      pshmem_ushort_wait_until
-#define shmem_uint_wait_until        pshmem_uint_wait_until
-#define shmem_ulong_wait_until       pshmem_ulong_wait_until
-#define shmem_ulonglong_wait_until   pshmem_ulonglong_wait_until
-#define shmem_int32_wait_until       pshmem_int32_wait_until
-#define shmem_int64_wait_until       pshmem_int64_wait_until
-#define shmem_uint32_wait_until      pshmem_uint32_wait_until
-#define shmem_uint64_wait_until      pshmem_uint64_wait_until
-#define shmem_size_wait_until        pshmem_size_wait_until
-#define shmem_ptrdiff_wait_until     pshmem_ptrdiff_wait_until
-
-#define shmemx_int32_wait_until      pshmemx_int32_wait_until
-#define shmemx_int64_wait_until      pshmemx_int64_wait_until
-
-#define shmem_short_test             pshmem_short_test
-#define shmem_int_test               pshmem_int_test
-#define shmem_long_test              pshmem_long_test
-#define shmem_longlong_test          pshmem_longlong_test
-#define shmem_ushort_test            pshmem_ushort_test
-#define shmem_uint_test              pshmem_uint_test
-#define shmem_ulong_test             pshmem_ulong_test
-#define shmem_ulonglong_test         pshmem_ulonglong_test
-#define shmem_int32_test             pshmem_int32_test
-#define shmem_int64_test             pshmem_int64_test
-#define shmem_uint32_test            pshmem_uint32_test
-#define shmem_uint64_test            pshmem_uint64_test
-#define shmem_size_test              pshmem_size_test
-#define shmem_ptrdiff_test           pshmem_ptrdiff_test
-
-/*
- * Barrier sync routines
- */
-#define shmem_barrier                pshmem_barrier
-#define shmem_barrier_all            pshmem_barrier_all
-#define shmem_sync                   pshmem_sync
-#define shmem_sync_all               pshmem_sync_all
-#define shmem_fence                  pshmem_fence
-#define shmem_ctx_fence              pshmem_ctx_fence
-#define shmem_quiet                  pshmem_quiet
-#define shmem_ctx_quiet              pshmem_ctx_quiet
-
-/*
- * Collective routines
- */
-#define shmem_broadcast32            pshmem_broadcast32
-#define shmem_broadcast64            pshmem_broadcast64
-#define shmem_collect32              pshmem_collect32
-#define shmem_collect64              pshmem_collect64
-#define shmem_fcollect32             pshmem_fcollect32
-#define shmem_fcollect64             pshmem_fcollect64
-
-/*
- * Reduction routines
- */
-#define shmem_short_and_to_all       pshmem_short_and_to_all
-#define shmem_int_and_to_all         pshmem_int_and_to_all
-#define shmem_long_and_to_all        pshmem_long_and_to_all
-#define shmem_longlong_and_to_all    pshmem_longlong_and_to_all
-#define shmemx_int16_and_to_all      pshmemx_int16_and_to_all
-#define shmemx_int32_and_to_all      pshmemx_int32_and_to_all
-#define shmemx_int64_and_to_all      pshmemx_int64_and_to_all
-
-#define shmem_short_or_to_all        pshmem_short_or_to_all
-#define shmem_int_or_to_all          pshmem_int_or_to_all
-#define shmem_long_or_to_all         pshmem_long_or_to_all
-#define shmem_longlong_or_to_all     pshmem_longlong_or_to_all
-#define shmemx_int16_or_to_all       pshmemx_int16_or_to_all
-#define shmemx_int32_or_to_all       pshmemx_int32_or_to_all
-#define shmemx_int64_or_to_all       pshmemx_int64_or_to_all
-
-#define shmem_short_xor_to_all       pshmem_short_xor_to_all
-#define shmem_int_xor_to_all         pshmem_int_xor_to_all
-#define shmem_long_xor_to_all        pshmem_long_xor_to_all
-#define shmem_longlong_xor_to_all    pshmem_longlong_xor_to_all
-#define shmemx_int16_xor_to_all      pshmemx_int16_xor_to_all
-#define shmemx_int32_xor_to_all      pshmemx_int32_xor_to_all
-#define shmemx_int64_xor_to_all      pshmemx_int64_xor_to_all
-
-#define shmem_short_max_to_all       pshmem_short_max_to_all
-#define shmem_int_max_to_all         pshmem_int_max_to_all
-#define shmem_long_max_to_all        pshmem_long_max_to_all
-#define shmem_longlong_max_to_all    pshmem_longlong_max_to_all
-#define shmem_float_max_to_all       pshmem_float_max_to_all
-#define shmem_double_max_to_all      pshmem_double_max_to_all
-#define shmem_longdouble_max_to_all  pshmem_longdouble_max_to_all
-#define shmemx_int16_max_to_all      pshmemx_int16_max_to_all
-#define shmemx_int32_max_to_all      pshmemx_int32_max_to_all
-#define shmemx_int64_max_to_all      pshmemx_int64_max_to_all
-
-#define shmem_short_min_to_all       pshmem_short_min_to_all
-#define shmem_int_min_to_all         pshmem_int_min_to_all
-#define shmem_long_min_to_all        pshmem_long_min_to_all
-#define shmem_longlong_min_to_all    pshmem_longlong_min_to_all
-#define shmem_float_min_to_all       pshmem_float_min_to_all
-#define shmem_double_min_to_all      pshmem_double_min_to_all
-#define shmem_longdouble_min_to_all  pshmem_longdouble_min_to_all
-#define shmemx_int16_min_to_all      pshmemx_int16_min_to_all
-#define shmemx_int32_min_to_all      pshmemx_int32_min_to_all
-#define shmemx_int64_min_to_all      pshmemx_int64_min_to_all
-
-#define shmem_short_sum_to_all       pshmem_short_sum_to_all
-#define shmem_int_sum_to_all         pshmem_int_sum_to_all
-#define shmem_long_sum_to_all        pshmem_long_sum_to_all
-#define shmem_longlong_sum_to_all    pshmem_longlong_sum_to_all
-#define shmem_float_sum_to_all       pshmem_float_sum_to_all
-#define shmem_double_sum_to_all      pshmem_double_sum_to_all
-#define shmem_longdouble_sum_to_all  pshmem_longdouble_sum_to_all
-#define shmem_complexf_sum_to_all    pshmem_complexf_sum_to_all
-#define shmem_complexd_sum_to_all    pshmem_complexd_sum_to_all
-#define shmemx_int16_sum_to_all      pshmemx_int16_sum_to_all
-#define shmemx_int32_sum_to_all      pshmemx_int32_sum_to_all
-#define shmemx_int64_sum_to_all      pshmemx_int64_sum_to_all
-
-#define shmem_short_prod_to_all      pshmem_short_prod_to_all
-#define shmem_int_prod_to_all        pshmem_int_prod_to_all
-#define shmem_long_prod_to_all       pshmem_long_prod_to_all
-#define shmem_longlong_prod_to_all   pshmem_longlong_prod_to_all
-#define shmem_float_prod_to_all      pshmem_float_prod_to_all
-#define shmem_double_prod_to_all     pshmem_double_prod_to_all
-#define shmem_longdouble_prod_to_all pshmem_longdouble_prod_to_all
-#define shmem_complexf_prod_to_all   pshmem_complexf_prod_to_all
-#define shmem_complexd_prod_to_all   pshmem_complexd_prod_to_all
-#define shmemx_int16_prod_to_all     pshmemx_int16_prod_to_all
-#define shmemx_int32_prod_to_all     pshmemx_int32_prod_to_all
-#define shmemx_int64_prod_to_all     pshmemx_int64_prod_to_all
-
-/*
- * Alltoall routines
- */
-#define shmem_alltoall32             pshmem_alltoall32
-#define shmem_alltoall64             pshmem_alltoall64
-#define shmem_alltoalls32            pshmem_alltoalls32
-#define shmem_alltoalls64            pshmem_alltoalls64
-
-/*
- * Platform specific cache management routines
- */
-#define shmem_udcflush              pshmem_udcflush
-#define shmem_udcflush_line         pshmem_udcflush_line
-#define shmem_set_cache_inv         pshmem_set_cache_inv
-#define shmem_set_cache_line_inv    pshmem_set_cache_line_inv
-#define shmem_clear_cache_inv       pshmem_clear_cache_inv
-#define shmem_clear_cache_line_inv  pshmem_clear_cache_line_inv
-
-#endif /* OSHMEM_C_PROFILE_DEFINES_H */
diff --git a/oshmem/shmem/c/shmem_add.c b/oshmem/shmem/c/shmem_add.c
index 6435496892a..4b78c278f3a 100644
--- a/oshmem/shmem/c/shmem_add.c
+++ b/oshmem/shmem/c/shmem_add.c
@@ -66,6 +66,12 @@
 #pragma weak shmem_ctx_uint_atomic_add = pshmem_ctx_uint_atomic_add
 #pragma weak shmem_ctx_ulong_atomic_add = pshmem_ctx_ulong_atomic_add
 #pragma weak shmem_ctx_ulonglong_atomic_add = pshmem_ctx_ulonglong_atomic_add
+#pragma weak shmem_ctx_int32_atomic_add = pshmem_ctx_int32_atomic_add
+#pragma weak shmem_ctx_int64_atomic_add = pshmem_ctx_int64_atomic_add
+#pragma weak shmem_ctx_uint32_atomic_add = pshmem_ctx_uint32_atomic_add
+#pragma weak shmem_ctx_uint64_atomic_add = pshmem_ctx_uint64_atomic_add
+#pragma weak shmem_ctx_size_atomic_add = pshmem_ctx_size_atomic_add
+#pragma weak shmem_ctx_ptrdiff_atomic_add = pshmem_ctx_ptrdiff_atomic_add
 
 #pragma weak shmem_int_atomic_add = pshmem_int_atomic_add
 #pragma weak shmem_long_atomic_add = pshmem_long_atomic_add
@@ -73,6 +79,12 @@
 #pragma weak shmem_uint_atomic_add = pshmem_uint_atomic_add
 #pragma weak shmem_ulong_atomic_add = pshmem_ulong_atomic_add
 #pragma weak shmem_ulonglong_atomic_add = pshmem_ulonglong_atomic_add
+#pragma weak shmem_int32_atomic_add = pshmem_int32_atomic_add
+#pragma weak shmem_int64_atomic_add = pshmem_int64_atomic_add
+#pragma weak shmem_uint32_atomic_add = pshmem_uint32_atomic_add
+#pragma weak shmem_uint64_atomic_add = pshmem_uint64_atomic_add
+#pragma weak shmem_size_atomic_add = pshmem_size_atomic_add
+#pragma weak shmem_ptrdiff_atomic_add = pshmem_ptrdiff_atomic_add
 
 #pragma weak shmem_int_add = pshmem_int_add
 #pragma weak shmem_long_add = pshmem_long_add
@@ -80,7 +92,7 @@
 
 #pragma weak shmemx_int32_add = pshmemx_int32_add
 #pragma weak shmemx_int64_add = pshmemx_int64_add
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 SHMEM_CTX_TYPE_ATOMIC_ADD(_uint, unsigned int, shmem)
@@ -89,12 +101,24 @@ SHMEM_CTX_TYPE_ATOMIC_ADD(_ulonglong, unsigned long long, shmem)
 SHMEM_CTX_TYPE_ATOMIC_ADD(_int, int, shmem)
 SHMEM_CTX_TYPE_ATOMIC_ADD(_long, long, shmem)
 SHMEM_CTX_TYPE_ATOMIC_ADD(_longlong, long long, shmem)
+SHMEM_CTX_TYPE_ATOMIC_ADD(_int32, int32_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_ADD(_int64, int64_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_ADD(_uint32, uint32_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_ADD(_uint64, uint64_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_ADD(_size, size_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_ADD(_ptrdiff, ptrdiff_t, shmem)
 SHMEM_TYPE_ATOMIC_ADD(_int, int, shmem)
 SHMEM_TYPE_ATOMIC_ADD(_long, long, shmem)
 SHMEM_TYPE_ATOMIC_ADD(_longlong, long long, shmem)
 SHMEM_TYPE_ATOMIC_ADD(_uint, unsigned int, shmem)
 SHMEM_TYPE_ATOMIC_ADD(_ulong, unsigned long, shmem)
 SHMEM_TYPE_ATOMIC_ADD(_ulonglong, unsigned long long, shmem)
+SHMEM_TYPE_ATOMIC_ADD(_int32, int32_t, shmem)
+SHMEM_TYPE_ATOMIC_ADD(_int64, int64_t, shmem)
+SHMEM_TYPE_ATOMIC_ADD(_uint32, uint32_t, shmem)
+SHMEM_TYPE_ATOMIC_ADD(_uint64, uint64_t, shmem)
+SHMEM_TYPE_ATOMIC_ADD(_size, size_t, shmem)
+SHMEM_TYPE_ATOMIC_ADD(_ptrdiff, ptrdiff_t, shmem)
 
 /* deprecated APIs */
 #define SHMEM_TYPE_ADD(type_name, type, prefix)                     \
diff --git a/oshmem/shmem/c/shmem_addr_accessible.c b/oshmem/shmem/c/shmem_addr_accessible.c
index 724318a894f..fbfe1ec2b33 100644
--- a/oshmem/shmem/c/shmem_addr_accessible.c
+++ b/oshmem/shmem/c/shmem_addr_accessible.c
@@ -19,7 +19,7 @@
 #if OSHMEM_PROFILING
 #include "oshmem/include/pshmem.h"
 #pragma weak shmem_addr_accessible = pshmem_addr_accessible
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 int shmem_addr_accessible(const void *addr, int pe)
diff --git a/oshmem/shmem/c/shmem_align.c b/oshmem/shmem/c/shmem_align.c
index 52c7d625396..7f5046412de 100644
--- a/oshmem/shmem/c/shmem_align.c
+++ b/oshmem/shmem/c/shmem_align.c
@@ -21,7 +21,7 @@
 #include "oshmem/include/pshmem.h"
 #pragma weak shmem_align = pshmem_align
 #pragma weak shmemalign = pshmemalign
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 static inline void* _shmemalign(size_t align, size_t size);
diff --git a/oshmem/shmem/c/shmem_alloc.c b/oshmem/shmem/c/shmem_alloc.c
index 92592ce8ca3..c633ed48fd9 100644
--- a/oshmem/shmem/c/shmem_alloc.c
+++ b/oshmem/shmem/c/shmem_alloc.c
@@ -24,8 +24,9 @@
 #pragma weak shmem_malloc            = pshmem_malloc
 #pragma weak shmem_calloc            = pshmem_calloc
 #pragma weak shmalloc                = pshmalloc
-#pragma weak shmemx_malloc_with_hint = pshmemx_malloc_with_hint
-#include "oshmem/shmem/c/profile/defines.h"
+#pragma weak shmem_malloc_with_hints = pshmem_malloc_with_hints
+
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 static inline void* _shmalloc(size_t size);
@@ -38,6 +39,7 @@ void* shmem_malloc(size_t size)
 void* shmem_calloc(size_t count, size_t size)
 {
     size_t req_sz = count * size;
+    if (!req_sz) return NULL;
     void *ptr = _shmalloc(req_sz);
     if (ptr) {
         memset(ptr, 0, req_sz);
@@ -76,7 +78,7 @@ static inline void* _shmalloc(size_t size)
     return pBuff;
 }
 
-void* shmemx_malloc_with_hint(size_t size, long hint)
+void* shmem_malloc_with_hints(size_t size, long hint)
 {
     int rc;
     void* pBuff = NULL;
@@ -105,3 +107,5 @@ void* shmemx_malloc_with_hint(size_t size, long hint)
 #endif
     return pBuff;
 }
+
+
diff --git a/oshmem/shmem/c/shmem_alltoall.c b/oshmem/shmem/c/shmem_alltoall.c
index 8dc8b01312f..75eb592c53c 100644
--- a/oshmem/shmem/c/shmem_alltoall.c
+++ b/oshmem/shmem/c/shmem_alltoall.c
@@ -17,6 +17,7 @@
 #include "oshmem/runtime/runtime.h"
 
 #include "oshmem/mca/scoll/scoll.h"
+#include "oshmem/mca/spml/spml.h"
 
 #include "oshmem/proc/proc.h"
 
@@ -102,10 +103,155 @@ static void _shmem_alltoall(void *target,
 #pragma weak shmem_alltoall64 = pshmem_alltoall64
 #pragma weak shmem_alltoalls32 = pshmem_alltoalls32
 #pragma weak shmem_alltoalls64 = pshmem_alltoalls64
-#include "oshmem/shmem/c/profile/defines.h"
+
+/* Teams alltoall */
+#pragma weak shmem_char_alltoall         		= pshmem_char_alltoall
+#pragma weak shmem_short_alltoall         		= pshmem_short_alltoall
+#pragma weak shmem_int_alltoall         	    	= pshmem_int_alltoall
+#pragma weak shmem_long_alltoall         		= pshmem_long_alltoall
+#pragma weak shmem_float_alltoall         		= pshmem_float_alltoall
+#pragma weak shmem_double_alltoall         		= pshmem_double_alltoall
+#pragma weak shmem_longlong_alltoall         		= pshmem_longlong_alltoall
+#pragma weak shmem_schar_alltoall         		= pshmem_schar_alltoall
+#pragma weak shmem_uchar_alltoall         		= pshmem_uchar_alltoall
+#pragma weak shmem_ushort_alltoall         		= pshmem_ushort_alltoall
+#pragma weak shmem_uint_alltoall         		= pshmem_uint_alltoall
+#pragma weak shmem_ulong_alltoall         		= pshmem_ulong_alltoall
+#pragma weak shmem_ulonglong_alltoall      		= pshmem_ulonglong_alltoall
+#pragma weak shmem_longdouble_alltoall       		= pshmem_longdouble_alltoall
+#pragma weak shmem_int8_alltoall         		= pshmem_int8_alltoall
+#pragma weak shmem_int16_alltoall         		= pshmem_int16_alltoall
+#pragma weak shmem_int32_alltoall         		= pshmem_int32_alltoall
+#pragma weak shmem_int64_alltoall         		= pshmem_int64_alltoall
+#pragma weak shmem_uint8_alltoall         		= pshmem_uint8_alltoall
+#pragma weak shmem_uint16_alltoall         		= pshmem_uint16_alltoall
+#pragma weak shmem_uint32_alltoall         		= pshmem_uint32_alltoall
+#pragma weak shmem_uint64_alltoall         		= pshmem_uint64_alltoall
+#pragma weak shmem_size_alltoall         		= pshmem_size_alltoall
+#pragma weak shmem_ptrdiff_alltoall         		= pshmem_ptrdiff_alltoall
+
+#pragma weak shmem_alltoallmem               		= pshmem_alltoallmem
+
+/* Teams alltoalls */
+#pragma weak shmem_char_alltoalls         		= pshmem_char_alltoalls
+#pragma weak shmem_short_alltoalls         		= pshmem_short_alltoalls
+#pragma weak shmem_int_alltoalls       	    	= pshmem_int_alltoalls
+#pragma weak shmem_long_alltoalls         		= pshmem_long_alltoalls
+#pragma weak shmem_float_alltoalls         		= pshmem_float_alltoalls
+#pragma weak shmem_double_alltoalls         		= pshmem_double_alltoalls
+#pragma weak shmem_longlong_alltoalls        		= pshmem_longlong_alltoalls
+#pragma weak shmem_schar_alltoalls         		= pshmem_schar_alltoalls
+#pragma weak shmem_uchar_alltoalls         		= pshmem_uchar_alltoalls
+#pragma weak shmem_ushort_alltoalls         		= pshmem_ushort_alltoalls
+#pragma weak shmem_uint_alltoalls         		= pshmem_uint_alltoalls
+#pragma weak shmem_ulong_alltoalls         		= pshmem_ulong_alltoalls
+#pragma weak shmem_ulonglong_alltoalls      		= pshmem_ulonglong_alltoalls
+#pragma weak shmem_longdouble_alltoalls      		= pshmem_longdouble_alltoalls
+#pragma weak shmem_int8_alltoalls         		= pshmem_int8_alltoalls
+#pragma weak shmem_int16_alltoalls         		= pshmem_int16_alltoalls
+#pragma weak shmem_int32_alltoalls         		= pshmem_int32_alltoalls
+#pragma weak shmem_int64_alltoalls         		= pshmem_int64_alltoalls
+#pragma weak shmem_uint8_alltoalls         		= pshmem_uint8_alltoalls
+#pragma weak shmem_uint16_alltoalls         		= pshmem_uint16_alltoalls
+#pragma weak shmem_uint32_alltoalls         		= pshmem_uint32_alltoalls
+#pragma weak shmem_uint64_alltoalls         		= pshmem_uint64_alltoalls
+#pragma weak shmem_size_alltoalls         		= pshmem_size_alltoalls
+#pragma weak shmem_ptrdiff_alltoalls        		= pshmem_ptrdiff_alltoalls
+
+#pragma weak shmem_alltoallsmem              		= pshmem_alltoallsmem
+
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 SHMEM_TYPE_ALLTOALL(_alltoall32, sizeof(uint32_t))
 SHMEM_TYPE_ALLTOALL(_alltoall64, sizeof(uint64_t))
 SHMEM_TYPE_ALLTOALLS(_alltoalls32, sizeof(uint32_t))
 SHMEM_TYPE_ALLTOALLS(_alltoalls64, sizeof(uint64_t))
+
+
+
+#define SHMEM_TYPE_TEAM_ALLTOALL(type_name, type, code, postfix)    \
+    int  shmem##type_name##postfix(shmem_team_t team, type *dest, const type *source, size_t nelems)   \
+    {                                                               \
+        int rc = 0;                                                 \
+                                                                    \
+        RUNTIME_CHECK_INIT();                                       \
+                                                                    \
+        rc = MCA_SPML_CALL(team_alltoall(                           \
+            team, (void*)dest, (void*)source, nelems, code));       \
+        RUNTIME_CHECK_RC(rc);                                       \
+                                                                    \
+        return rc;                                                  \
+    }
+
+
+SHMEM_TYPE_TEAM_ALLTOALL(_char, char, SHMEM_CHAR, _alltoall)
+SHMEM_TYPE_TEAM_ALLTOALL(_short, short, SHMEM_SHORT, _alltoall)
+SHMEM_TYPE_TEAM_ALLTOALL(_int, int, SHMEM_INT, _alltoall)
+SHMEM_TYPE_TEAM_ALLTOALL(_long, long, SHMEM_LONG, _alltoall)
+SHMEM_TYPE_TEAM_ALLTOALL(_float, float, SHMEM_FLOAT, _alltoall)
+SHMEM_TYPE_TEAM_ALLTOALL(_double, double, SHMEM_DOUBLE, _alltoall)
+SHMEM_TYPE_TEAM_ALLTOALL(_longlong, long long, SHMEM_LLONG, _alltoall)
+SHMEM_TYPE_TEAM_ALLTOALL(_schar, signed char, SHMEM_SCHAR, _alltoall)
+SHMEM_TYPE_TEAM_ALLTOALL(_uchar, unsigned char, SHMEM_UCHAR, _alltoall)
+SHMEM_TYPE_TEAM_ALLTOALL(_ushort, unsigned short, SHMEM_USHORT, _alltoall)
+SHMEM_TYPE_TEAM_ALLTOALL(_uint, unsigned int, SHMEM_UINT, _alltoall)
+SHMEM_TYPE_TEAM_ALLTOALL(_ulong, unsigned long, SHMEM_ULONG, _alltoall)
+SHMEM_TYPE_TEAM_ALLTOALL(_ulonglong, unsigned long long, SHMEM_ULLONG, _alltoall)
+SHMEM_TYPE_TEAM_ALLTOALL(_longdouble, long double, SHMEM_LDOUBLE, _alltoall)
+SHMEM_TYPE_TEAM_ALLTOALL(_int8, int8_t, SHMEM_INT8_T, _alltoall)
+SHMEM_TYPE_TEAM_ALLTOALL(_int16, int16_t, SHMEM_INT16_T, _alltoall)
+SHMEM_TYPE_TEAM_ALLTOALL(_int32, int32_t, SHMEM_INT32_T, _alltoall)
+SHMEM_TYPE_TEAM_ALLTOALL(_int64, int64_t, SHMEM_INT64_T, _alltoall)
+SHMEM_TYPE_TEAM_ALLTOALL(_uint8, uint8_t, SHMEM_UINT8_T, _alltoall)
+SHMEM_TYPE_TEAM_ALLTOALL(_uint16, uint16_t, SHMEM_UINT16_T, _alltoall)
+SHMEM_TYPE_TEAM_ALLTOALL(_uint32, uint32_t, SHMEM_UINT32_T, _alltoall)
+SHMEM_TYPE_TEAM_ALLTOALL(_uint64, uint64_t, SHMEM_UINT64_T, _alltoall)
+SHMEM_TYPE_TEAM_ALLTOALL(_size, size_t, SHMEM_SIZE_T, _alltoall)
+SHMEM_TYPE_TEAM_ALLTOALL(_ptrdiff, ptrdiff_t, SHMEM_PTRDIFF_T, _alltoall)
+
+SHMEM_TYPE_TEAM_ALLTOALL(, void, SHMEM_BYTE, _alltoallmem)
+
+
+#define SHMEM_TYPE_TEAM_ALLTOALLS(type_name, type, code, postfix)   \
+    int  shmem##type_name##postfix(shmem_team_t team, type *dest, const type *source, ptrdiff_t dst, ptrdiff_t sst,  size_t nelems)   \
+    {                                                               \
+        int rc = 0;                                                 \
+                                                                    \
+        RUNTIME_CHECK_INIT();                                       \
+                                                                    \
+        rc = MCA_SPML_CALL(team_alltoalls(                          \
+            team, (void*)dest, (void*)source,                       \
+                    dst, sst, nelems, code));                       \
+        RUNTIME_CHECK_RC(rc);                                       \
+                                                                    \
+        return rc;                                                  \
+    }
+
+
+SHMEM_TYPE_TEAM_ALLTOALLS(_char, char, SHMEM_CHAR, _alltoalls)
+SHMEM_TYPE_TEAM_ALLTOALLS(_short, short, SHMEM_SHORT, _alltoalls)
+SHMEM_TYPE_TEAM_ALLTOALLS(_int, int, SHMEM_INT, _alltoalls)
+SHMEM_TYPE_TEAM_ALLTOALLS(_long, long, SHMEM_LONG, _alltoalls)
+SHMEM_TYPE_TEAM_ALLTOALLS(_float, float, SHMEM_FLOAT, _alltoalls)
+SHMEM_TYPE_TEAM_ALLTOALLS(_double, double, SHMEM_DOUBLE, _alltoalls)
+SHMEM_TYPE_TEAM_ALLTOALLS(_longlong, long long, SHMEM_LLONG, _alltoalls)
+SHMEM_TYPE_TEAM_ALLTOALLS(_schar, signed char, SHMEM_SCHAR, _alltoalls)
+SHMEM_TYPE_TEAM_ALLTOALLS(_uchar, unsigned char, SHMEM_UCHAR, _alltoalls)
+SHMEM_TYPE_TEAM_ALLTOALLS(_ushort, unsigned short, SHMEM_USHORT, _alltoalls)
+SHMEM_TYPE_TEAM_ALLTOALLS(_uint, unsigned int, SHMEM_UINT, _alltoalls)
+SHMEM_TYPE_TEAM_ALLTOALLS(_ulong, unsigned long, SHMEM_ULONG, _alltoalls)
+SHMEM_TYPE_TEAM_ALLTOALLS(_ulonglong, unsigned long long, SHMEM_ULLONG, _alltoalls)
+SHMEM_TYPE_TEAM_ALLTOALLS(_longdouble, long double, SHMEM_LDOUBLE, _alltoalls)
+SHMEM_TYPE_TEAM_ALLTOALLS(_int8, int8_t, SHMEM_INT8_T, _alltoalls)
+SHMEM_TYPE_TEAM_ALLTOALLS(_int16, int16_t, SHMEM_INT16_T, _alltoalls)
+SHMEM_TYPE_TEAM_ALLTOALLS(_int32, int32_t, SHMEM_INT32_T, _alltoalls)
+SHMEM_TYPE_TEAM_ALLTOALLS(_int64, int64_t, SHMEM_INT64_T, _alltoalls)
+SHMEM_TYPE_TEAM_ALLTOALLS(_uint8, uint8_t, SHMEM_UINT8_T, _alltoalls)
+SHMEM_TYPE_TEAM_ALLTOALLS(_uint16, uint16_t, SHMEM_UINT16_T, _alltoalls)
+SHMEM_TYPE_TEAM_ALLTOALLS(_uint32, uint32_t, SHMEM_UINT32_T, _alltoalls)
+SHMEM_TYPE_TEAM_ALLTOALLS(_uint64, uint64_t, SHMEM_UINT64_T, _alltoalls)
+SHMEM_TYPE_TEAM_ALLTOALLS(_size, size_t, SHMEM_SIZE_T, _alltoalls)
+SHMEM_TYPE_TEAM_ALLTOALLS(_ptrdiff, ptrdiff_t, SHMEM_PTRDIFF_T, _alltoalls)
+
+SHMEM_TYPE_TEAM_ALLTOALLS(, void, SHMEM_BYTE, _alltoallsmem)
diff --git a/oshmem/shmem/c/shmem_and.c b/oshmem/shmem/c/shmem_and.c
index 0f4c5be9d39..eb271adb62e 100644
--- a/oshmem/shmem/c/shmem_and.c
+++ b/oshmem/shmem/c/shmem_and.c
@@ -51,7 +51,7 @@
 #pragma weak shmemx_int64_atomic_and    = pshmemx_int64_atomic_and
 #pragma weak shmemx_uint32_atomic_and   = pshmemx_uint32_atomic_and
 #pragma weak shmemx_uint64_atomic_and   = pshmemx_uint64_atomic_and
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 OSHMEM_TYPE_OP(int, int, shmem, and)
diff --git a/oshmem/shmem/c/shmem_barrier.c b/oshmem/shmem/c/shmem_barrier.c
index 7ce0ddc96f7..8ef2458236b 100644
--- a/oshmem/shmem/c/shmem_barrier.c
+++ b/oshmem/shmem/c/shmem_barrier.c
@@ -24,7 +24,7 @@
 #include "oshmem/include/pshmem.h"
 #pragma weak shmem_barrier = pshmem_barrier
 #pragma weak shmem_barrier_all = pshmem_barrier_all
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 void shmem_barrier(int PE_start, int logPE_stride, int PE_size, long *pSync)
diff --git a/oshmem/shmem/c/shmem_broadcast.c b/oshmem/shmem/c/shmem_broadcast.c
index fea737825da..12b78d4e009 100644
--- a/oshmem/shmem/c/shmem_broadcast.c
+++ b/oshmem/shmem/c/shmem_broadcast.c
@@ -17,6 +17,7 @@
 #include "oshmem/runtime/runtime.h"
 
 #include "oshmem/mca/scoll/scoll.h"
+#include "oshmem/mca/spml/spml.h"
 
 #include "oshmem/proc/proc.h"
 
@@ -69,7 +70,7 @@ static void _shmem_broadcast(void *target,
         }
 
         /* Define actual PE using relative in active set */
-        PE_root = oshmem_proc_pe(group->proc_array[PE_root]);
+        PE_root = oshmem_proc_pe_vpid(group, PE_root);
 
         /* Call collective broadcast operation */
         rc = group->g_scoll.scoll_broadcast(group,
@@ -90,8 +91,82 @@ static void _shmem_broadcast(void *target,
 #include "oshmem/include/pshmem.h"
 #pragma weak shmem_broadcast32 = pshmem_broadcast32
 #pragma weak shmem_broadcast64 = pshmem_broadcast64
-#include "oshmem/shmem/c/profile/defines.h"
+
+/* Teams broadcast */
+#pragma weak shmem_char_broadcast         		= pshmem_char_broadcast
+#pragma weak shmem_short_broadcast         		= pshmem_short_broadcast
+#pragma weak shmem_int_broadcast        	    	= pshmem_int_broadcast
+#pragma weak shmem_long_broadcast         		= pshmem_long_broadcast
+#pragma weak shmem_float_broadcast         		= pshmem_float_broadcast
+#pragma weak shmem_double_broadcast         		= pshmem_double_broadcast
+#pragma weak shmem_longlong_broadcast        		= pshmem_longlong_broadcast
+#pragma weak shmem_schar_broadcast         		= pshmem_schar_broadcast
+#pragma weak shmem_uchar_broadcast         		= pshmem_uchar_broadcast
+#pragma weak shmem_ushort_broadcast         		= pshmem_ushort_broadcast
+#pragma weak shmem_uint_broadcast         		= pshmem_uint_broadcast
+#pragma weak shmem_ulong_broadcast         		= pshmem_ulong_broadcast
+#pragma weak shmem_ulonglong_broadcast      		= pshmem_ulonglong_broadcast
+#pragma weak shmem_longdouble_broadcast      		= pshmem_longdouble_broadcast
+#pragma weak shmem_int8_broadcast         		= pshmem_int8_broadcast
+#pragma weak shmem_int16_broadcast         		= pshmem_int16_broadcast
+#pragma weak shmem_int32_broadcast         		= pshmem_int32_broadcast
+#pragma weak shmem_int64_broadcast         		= pshmem_int64_broadcast
+#pragma weak shmem_uint8_broadcast         		= pshmem_uint8_broadcast
+#pragma weak shmem_uint16_broadcast         		= pshmem_uint16_broadcast
+#pragma weak shmem_uint32_broadcast         		= pshmem_uint32_broadcast
+#pragma weak shmem_uint64_broadcast         		= pshmem_uint64_broadcast
+#pragma weak shmem_size_broadcast         		= pshmem_size_broadcast
+#pragma weak shmem_ptrdiff_broadcast        		= pshmem_ptrdiff_broadcast
+
+#pragma weak shmem_broadcastmem              		= pshmem_broadcastmem
+
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 SHMEM_TYPE_BROADCAST(_broadcast32, sizeof(uint32_t))
 SHMEM_TYPE_BROADCAST(_broadcast64, sizeof(uint64_t))
+
+
+
+#define SHMEM_TYPE_TEAM_BROADCAST(type_name, type, code, postfix)    \
+    int  shmem##type_name##postfix(shmem_team_t team, type *dest, const type *source, size_t nelems, int PE_root)   \
+    {                                                               \
+        int rc = 0;                                                 \
+                                                                    \
+        RUNTIME_CHECK_INIT();                                       \
+                                                                    \
+        rc = MCA_SPML_CALL(team_broadcast(                          \
+            team, (void*)dest, (void*)source,                       \
+                    nelems, PE_root, code));                        \
+        RUNTIME_CHECK_RC(rc);                                       \
+                                                                    \
+        return rc;                                                  \
+    }
+
+
+SHMEM_TYPE_TEAM_BROADCAST(_char, char, SHMEM_CHAR, _broadcast)
+SHMEM_TYPE_TEAM_BROADCAST(_short, short, SHMEM_SHORT, _broadcast)
+SHMEM_TYPE_TEAM_BROADCAST(_int, int, SHMEM_INT, _broadcast)
+SHMEM_TYPE_TEAM_BROADCAST(_long, long, SHMEM_LONG, _broadcast)
+SHMEM_TYPE_TEAM_BROADCAST(_float, float, SHMEM_FLOAT, _broadcast)
+SHMEM_TYPE_TEAM_BROADCAST(_double, double, SHMEM_DOUBLE, _broadcast)
+SHMEM_TYPE_TEAM_BROADCAST(_longlong, long long, SHMEM_LLONG, _broadcast)
+SHMEM_TYPE_TEAM_BROADCAST(_schar, signed char, SHMEM_SCHAR, _broadcast)
+SHMEM_TYPE_TEAM_BROADCAST(_uchar, unsigned char, SHMEM_UCHAR, _broadcast)
+SHMEM_TYPE_TEAM_BROADCAST(_ushort, unsigned short, SHMEM_USHORT, _broadcast)
+SHMEM_TYPE_TEAM_BROADCAST(_uint, unsigned int, SHMEM_UINT, _broadcast)
+SHMEM_TYPE_TEAM_BROADCAST(_ulong, unsigned long, SHMEM_ULONG, _broadcast)
+SHMEM_TYPE_TEAM_BROADCAST(_ulonglong, unsigned long long, SHMEM_ULLONG, _broadcast)
+SHMEM_TYPE_TEAM_BROADCAST(_longdouble, long double, SHMEM_LDOUBLE, _broadcast)
+SHMEM_TYPE_TEAM_BROADCAST(_int8, int8_t, SHMEM_INT8_T, _broadcast)
+SHMEM_TYPE_TEAM_BROADCAST(_int16, int16_t, SHMEM_INT16_T, _broadcast)
+SHMEM_TYPE_TEAM_BROADCAST(_int32, int32_t, SHMEM_INT32_T, _broadcast)
+SHMEM_TYPE_TEAM_BROADCAST(_int64, int64_t, SHMEM_INT64_T, _broadcast)
+SHMEM_TYPE_TEAM_BROADCAST(_uint8, uint8_t, SHMEM_UINT8_T, _broadcast)
+SHMEM_TYPE_TEAM_BROADCAST(_uint16, uint16_t, SHMEM_UINT16_T, _broadcast)
+SHMEM_TYPE_TEAM_BROADCAST(_uint32, uint32_t, SHMEM_UINT32_T, _broadcast)
+SHMEM_TYPE_TEAM_BROADCAST(_uint64, uint64_t, SHMEM_UINT64_T, _broadcast)
+SHMEM_TYPE_TEAM_BROADCAST(_size, size_t, SHMEM_SIZE_T, _broadcast)
+SHMEM_TYPE_TEAM_BROADCAST(_ptrdiff, ptrdiff_t, SHMEM_PTRDIFF_T, _broadcast)
+
+SHMEM_TYPE_TEAM_BROADCAST(, void, SHMEM_BYTE, _broadcastmem)
diff --git a/oshmem/shmem/c/shmem_clear_cache_inv.c b/oshmem/shmem/c/shmem_clear_cache_inv.c
index f9d084b3a61..c80cf7e6feb 100644
--- a/oshmem/shmem/c/shmem_clear_cache_inv.c
+++ b/oshmem/shmem/c/shmem_clear_cache_inv.c
@@ -4,6 +4,8 @@
  * Copyright (c) 2014      Intel, Inc. All rights reserved.
  * Copyright (c) 2017      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -18,17 +20,17 @@
 #include "oshmem/mca/spml/spml.h"
 
 #include "oshmem/shmem/shmem_api_logger.h"
-#include "opal/sys/architecture.h"
+#include "opal/opal_portable_platform.h"
 
 #if OSHMEM_PROFILING
 #include "oshmem/include/pshmem.h"
 #pragma weak shmem_clear_cache_inv = pshmem_clear_cache_inv
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 void shmem_clear_cache_inv(void)
 {
-#if (OPAL_ASSEMBLY_ARCH == OPAL_IA32) || (OPAL_ASSEMBLY_ARCH == OPAL_X86_64)
+#if defined(PLATFORM_ARCH_X86) || defined(PLATFORM_ARCH_X86_64)
     do {SHMEM_API_VERBOSE(10,"shmem_clear_cache_inv is not supported by the current CPU architecture");}while (0);
 #else
     /* another implementation */
diff --git a/oshmem/shmem/c/shmem_clear_cache_line_inv.c b/oshmem/shmem/c/shmem_clear_cache_line_inv.c
index 3ce8af8fc8f..720b2d6ed20 100644
--- a/oshmem/shmem/c/shmem_clear_cache_line_inv.c
+++ b/oshmem/shmem/c/shmem_clear_cache_line_inv.c
@@ -4,6 +4,8 @@
  * Copyright (c) 2014      Intel, Inc. All rights reserved.
  * Copyright (c) 2017      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -18,17 +20,17 @@
 #include "oshmem/mca/spml/spml.h"
 
 #include "oshmem/shmem/shmem_api_logger.h"
-#include "opal/sys/architecture.h"
+#include "opal/opal_portable_platform.h"
 
 #if OSHMEM_PROFILING
 #include "oshmem/include/pshmem.h"
 #pragma weak shmem_clear_cache_line_inv = pshmem_clear_cache_line_inv
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 void shmem_clear_cache_line_inv(void *target)
 {
-#if (OPAL_ASSEMBLY_ARCH == OPAL_IA32) || (OPAL_ASSEMBLY_ARCH == OPAL_X86_64)
+#if defined(PLATFORM_ARCH_X86) || defined(PLATFORM_ARCH_X86_64)
     do {SHMEM_API_VERBOSE(10,"shmem_clear_cache_line_inv is not supported by the current CPU architecture");}while (0);
 #else
     /* another implementation */
diff --git a/oshmem/shmem/c/shmem_clear_lock.c b/oshmem/shmem/c/shmem_clear_lock.c
index 2b731d73477..4c94038d316 100644
--- a/oshmem/shmem/c/shmem_clear_lock.c
+++ b/oshmem/shmem/c/shmem_clear_lock.c
@@ -1,4 +1,6 @@
 /*
+ * Copyright (c) 2023      NVIDIA Corporation.
+ *                         All rights reserved.
  * Copyright (c) 2013-2016 Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2019      Research Organization for Information Science
@@ -18,14 +20,21 @@
 #include "oshmem/shmem/shmem_api_logger.h"
 #include "oshmem/runtime/runtime.h"
 #include "oshmem/shmem/shmem_lock.h"
+#include "oshmem/runtime/params.h"
 
 #if OSHMEM_PROFILING
 #include "oshmem/include/pshmem.h"
 #pragma weak shmem_clear_lock = pshmem_clear_lock
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 void shmem_clear_lock(volatile long *lock)
 {
-    _shmem_clear_lock((void *)lock, sizeof(long));
+    if (oshmem_shmem_enable_mcs_locks) {
+        SHMEM_API_VERBOSE(10, "Clear Lock with MCS Lock implementation");
+        _shmem_mcs_clear_lock((long *)lock);
+    } else {
+        SHMEM_API_VERBOSE(10, "Clear Lock with Ticket Lock implementation");
+        _shmem_clear_lock((void *)lock, sizeof(long));
+    }
 }
diff --git a/oshmem/shmem/c/shmem_collect.c b/oshmem/shmem/c/shmem_collect.c
index 8d14a80ffa1..804e6817ef6 100644
--- a/oshmem/shmem/c/shmem_collect.c
+++ b/oshmem/shmem/c/shmem_collect.c
@@ -17,6 +17,7 @@
 #include "oshmem/runtime/runtime.h"
 
 #include "oshmem/mca/scoll/scoll.h"
+#include "oshmem/mca/spml/spml.h"
 
 #include "oshmem/proc/proc.h"
 
@@ -80,10 +81,162 @@ static void _shmem_collect(void *target,
 #pragma weak shmem_collect64 = pshmem_collect64
 #pragma weak shmem_fcollect32 = pshmem_fcollect32
 #pragma weak shmem_fcollect64 = pshmem_fcollect64
-#include "oshmem/shmem/c/profile/defines.h"
+
+/* Teams collect */
+#pragma weak shmem_char_collect          		= pshmem_char_collect
+#pragma weak shmem_short_collect         		= pshmem_short_collect
+#pragma weak shmem_int_collect         	    	        = pshmem_int_collect
+#pragma weak shmem_long_collect          		= pshmem_long_collect
+#pragma weak shmem_float_collect         		= pshmem_float_collect
+#pragma weak shmem_double_collect         		= pshmem_double_collect
+#pragma weak shmem_longlong_collect          		= pshmem_longlong_collect
+#pragma weak shmem_schar_collect         		= pshmem_schar_collect
+#pragma weak shmem_uchar_collect         		= pshmem_uchar_collect
+#pragma weak shmem_ushort_collect         		= pshmem_ushort_collect
+#pragma weak shmem_uint_collect          		= pshmem_uint_collect
+#pragma weak shmem_ulong_collect         		= pshmem_ulong_collect
+#pragma weak shmem_ulonglong_collect      		= pshmem_ulonglong_collect
+#pragma weak shmem_longdouble_collect        		= pshmem_longdouble_collect
+#pragma weak shmem_int8_collect         	    	= pshmem_int8_collect
+#pragma weak shmem_int16_collect         		= pshmem_int16_collect
+#pragma weak shmem_int32_collect         		= pshmem_int32_collect
+#pragma weak shmem_int64_collect         		= pshmem_int64_collect
+#pragma weak shmem_uint8_collect         		= pshmem_uint8_collect
+#pragma weak shmem_uint16_collect         		= pshmem_uint16_collect
+#pragma weak shmem_uint32_collect         		= pshmem_uint32_collect
+#pragma weak shmem_uint64_collect         		= pshmem_uint64_collect
+#pragma weak shmem_size_collect            		= pshmem_size_collect
+#pragma weak shmem_ptrdiff_collect         		= pshmem_ptrdiff_collect
+
+#pragma weak shmem_collectmem                		= pshmem_collectmem
+
+
+/* Teams fcollect */
+#pragma weak shmem_char_fcollect         		= pshmem_char_fcollect
+#pragma weak shmem_short_fcollect         		= pshmem_short_fcollect
+#pragma weak shmem_int_fcollect         	    	= pshmem_int_fcollect
+#pragma weak shmem_long_fcollect         		= pshmem_long_fcollect
+#pragma weak shmem_float_fcollect         		= pshmem_float_fcollect
+#pragma weak shmem_double_fcollect         		= pshmem_double_fcollect
+#pragma weak shmem_longlong_fcollect         	        = pshmem_longlong_fcollect
+#pragma weak shmem_schar_fcollect         		= pshmem_schar_fcollect
+#pragma weak shmem_uchar_fcollect         		= pshmem_uchar_fcollect
+#pragma weak shmem_ushort_fcollect         		= pshmem_ushort_fcollect
+#pragma weak shmem_uint_fcollect         		= pshmem_uint_fcollect
+#pragma weak shmem_ulong_fcollect         		= pshmem_ulong_fcollect
+#pragma weak shmem_ulonglong_fcollect      		= pshmem_ulonglong_fcollect
+#pragma weak shmem_longdouble_fcollect       		= pshmem_longdouble_fcollect
+#pragma weak shmem_int8_fcollect         		= pshmem_int8_fcollect
+#pragma weak shmem_int16_fcollect         		= pshmem_int16_fcollect
+#pragma weak shmem_int32_fcollect         		= pshmem_int32_fcollect
+#pragma weak shmem_int64_fcollect         		= pshmem_int64_fcollect
+#pragma weak shmem_uint8_fcollect         		= pshmem_uint8_fcollect
+#pragma weak shmem_uint16_fcollect         		= pshmem_uint16_fcollect
+#pragma weak shmem_uint32_fcollect         		= pshmem_uint32_fcollect
+#pragma weak shmem_uint64_fcollect         		= pshmem_uint64_fcollect
+#pragma weak shmem_size_fcollect         		= pshmem_size_fcollect
+#pragma weak shmem_ptrdiff_fcollect         	        = pshmem_ptrdiff_fcollect
+
+#pragma weak shmem_fcollectmem               		= pshmem_fcollectmem
+
+
+
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 SHMEM_TYPE_COLLECT(_collect32, sizeof(uint32_t), false)
 SHMEM_TYPE_COLLECT(_collect64, sizeof(uint64_t), false)
 SHMEM_TYPE_COLLECT(_fcollect32, sizeof(uint32_t), true)
 SHMEM_TYPE_COLLECT(_fcollect64, sizeof(uint64_t), true)
+
+
+#define SHMEM_TYPE_TEAM_COLLECT(type_name, type, code, postfix)     \
+    int  shmem##type_name##postfix(shmem_team_t team, type *dest, const type *source, size_t nelems)   \
+    {                                                               \
+        int rc = 0;                                                 \
+                                                                    \
+        RUNTIME_CHECK_INIT();                                       \
+                                                                    \
+        rc = MCA_SPML_CALL(team_collect(                            \
+            team, (void*)dest, (void*)source,                       \
+                    nelems, code));                                 \
+        RUNTIME_CHECK_RC(rc);                                       \
+                                                                    \
+        return rc;                                                  \
+    }
+
+
+SHMEM_TYPE_TEAM_COLLECT(_char, char, SHMEM_CHAR, _collect)
+SHMEM_TYPE_TEAM_COLLECT(_short, short, SHMEM_SHORT, _collect)
+SHMEM_TYPE_TEAM_COLLECT(_int, int, SHMEM_INT, _collect)
+SHMEM_TYPE_TEAM_COLLECT(_long, long, SHMEM_LONG, _collect)
+SHMEM_TYPE_TEAM_COLLECT(_float, float, SHMEM_FLOAT, _collect)
+SHMEM_TYPE_TEAM_COLLECT(_double, double, SHMEM_DOUBLE, _collect)
+SHMEM_TYPE_TEAM_COLLECT(_longlong, long long, SHMEM_LLONG, _collect)
+SHMEM_TYPE_TEAM_COLLECT(_schar, signed char, SHMEM_SCHAR, _collect)
+SHMEM_TYPE_TEAM_COLLECT(_uchar, unsigned char, SHMEM_UCHAR, _collect)
+SHMEM_TYPE_TEAM_COLLECT(_ushort, unsigned short, SHMEM_USHORT, _collect)
+SHMEM_TYPE_TEAM_COLLECT(_uint, unsigned int, SHMEM_UINT, _collect)
+SHMEM_TYPE_TEAM_COLLECT(_ulong, unsigned long, SHMEM_ULONG, _collect)
+SHMEM_TYPE_TEAM_COLLECT(_ulonglong, unsigned long long, SHMEM_ULLONG, _collect)
+SHMEM_TYPE_TEAM_COLLECT(_longdouble, long double, SHMEM_LDOUBLE, _collect)
+SHMEM_TYPE_TEAM_COLLECT(_int8, int8_t, SHMEM_INT8_T, _collect)
+SHMEM_TYPE_TEAM_COLLECT(_int16, int16_t, SHMEM_INT16_T, _collect)
+SHMEM_TYPE_TEAM_COLLECT(_int32, int32_t, SHMEM_INT32_T, _collect)
+SHMEM_TYPE_TEAM_COLLECT(_int64, int64_t, SHMEM_INT64_T, _collect)
+SHMEM_TYPE_TEAM_COLLECT(_uint8, uint8_t, SHMEM_UINT8_T, _collect)
+SHMEM_TYPE_TEAM_COLLECT(_uint16, uint16_t, SHMEM_UINT16_T, _collect)
+SHMEM_TYPE_TEAM_COLLECT(_uint32, uint32_t, SHMEM_UINT32_T, _collect)
+SHMEM_TYPE_TEAM_COLLECT(_uint64, uint64_t, SHMEM_UINT64_T, _collect)
+SHMEM_TYPE_TEAM_COLLECT(_size, size_t, SHMEM_SIZE_T, _collect)
+SHMEM_TYPE_TEAM_COLLECT(_ptrdiff, ptrdiff_t, SHMEM_PTRDIFF_T, _collect)
+
+SHMEM_TYPE_TEAM_COLLECT(, void, SHMEM_BYTE, _collectmem)
+
+
+
+
+#define SHMEM_TYPE_TEAM_FCOLLECT(type_name, type, code, postfix)    \
+    int  shmem##type_name##postfix(shmem_team_t team, type *dest, const type *source,  size_t nelems)   \
+    {                                                               \
+        int rc = 0;                                                 \
+                                                                    \
+        RUNTIME_CHECK_INIT();                                       \
+                                                                    \
+        rc = MCA_SPML_CALL(team_fcollect(                           \
+            team, (void*)dest, (void*)source,                       \
+                    nelems, code));                                 \
+        RUNTIME_CHECK_RC(rc);                                       \
+                                                                    \
+        return rc;                                                  \
+    }
+
+
+
+SHMEM_TYPE_TEAM_FCOLLECT(_char, char, SHMEM_CHAR, _fcollect)
+SHMEM_TYPE_TEAM_FCOLLECT(_short, short, SHMEM_SHORT, _fcollect)
+SHMEM_TYPE_TEAM_FCOLLECT(_int, int, SHMEM_INT, _fcollect)
+SHMEM_TYPE_TEAM_FCOLLECT(_long, long, SHMEM_LONG, _fcollect)
+SHMEM_TYPE_TEAM_FCOLLECT(_float, float, SHMEM_FLOAT, _fcollect)
+SHMEM_TYPE_TEAM_FCOLLECT(_double, double, SHMEM_DOUBLE, _fcollect)
+SHMEM_TYPE_TEAM_FCOLLECT(_longlong, long long, SHMEM_LLONG, _fcollect)
+SHMEM_TYPE_TEAM_FCOLLECT(_schar, signed char, SHMEM_SCHAR, _fcollect)
+SHMEM_TYPE_TEAM_FCOLLECT(_uchar, unsigned char, SHMEM_UCHAR, _fcollect)
+SHMEM_TYPE_TEAM_FCOLLECT(_ushort, unsigned short, SHMEM_USHORT, _fcollect)
+SHMEM_TYPE_TEAM_FCOLLECT(_uint, unsigned int, SHMEM_UINT, _fcollect)
+SHMEM_TYPE_TEAM_FCOLLECT(_ulong, unsigned long, SHMEM_ULONG, _fcollect)
+SHMEM_TYPE_TEAM_FCOLLECT(_ulonglong, unsigned long long, SHMEM_ULLONG, _fcollect)
+SHMEM_TYPE_TEAM_FCOLLECT(_longdouble, long double, SHMEM_LDOUBLE, _fcollect)
+SHMEM_TYPE_TEAM_FCOLLECT(_int8, int8_t, SHMEM_INT8_T, _fcollect)
+SHMEM_TYPE_TEAM_FCOLLECT(_int16, int16_t, SHMEM_INT16_T, _fcollect)
+SHMEM_TYPE_TEAM_FCOLLECT(_int32, int32_t, SHMEM_INT32_T, _fcollect)
+SHMEM_TYPE_TEAM_FCOLLECT(_int64, int64_t, SHMEM_INT64_T, _fcollect)
+SHMEM_TYPE_TEAM_FCOLLECT(_uint8, uint8_t, SHMEM_UINT8_T, _fcollect)
+SHMEM_TYPE_TEAM_FCOLLECT(_uint16, uint16_t, SHMEM_UINT16_T, _fcollect)
+SHMEM_TYPE_TEAM_FCOLLECT(_uint32, uint32_t, SHMEM_UINT32_T, _fcollect)
+SHMEM_TYPE_TEAM_FCOLLECT(_uint64, uint64_t, SHMEM_UINT64_T, _fcollect)
+SHMEM_TYPE_TEAM_FCOLLECT(_size, size_t, SHMEM_SIZE_T, _fcollect)
+SHMEM_TYPE_TEAM_FCOLLECT(_ptrdiff, ptrdiff_t, SHMEM_PTRDIFF_T, _fcollect)
+
+SHMEM_TYPE_TEAM_FCOLLECT(, void, SHMEM_BYTE, _fcollectmem)
+
diff --git a/oshmem/shmem/c/shmem_context.c b/oshmem/shmem/c/shmem_context.c
index d82736568f0..a73731bb977 100644
--- a/oshmem/shmem/c/shmem_context.c
+++ b/oshmem/shmem/c/shmem_context.c
@@ -27,7 +27,7 @@
 #include "oshmem/include/pshmem.h"
 #pragma weak shmem_ctx_create = pshmem_ctx_create
 #pragma weak shmem_ctx_destroy = pshmem_ctx_destroy
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 int shmem_ctx_create(long options, shmem_ctx_t *ctx)
diff --git a/oshmem/shmem/c/shmem_cswap.c b/oshmem/shmem/c/shmem_cswap.c
index 99e3b1f9420..a80511af253 100644
--- a/oshmem/shmem/c/shmem_cswap.c
+++ b/oshmem/shmem/c/shmem_cswap.c
@@ -71,6 +71,12 @@
 #pragma weak shmem_ctx_int_atomic_compare_swap = pshmem_ctx_int_atomic_compare_swap
 #pragma weak shmem_ctx_long_atomic_compare_swap = pshmem_ctx_long_atomic_compare_swap
 #pragma weak shmem_ctx_longlong_atomic_compare_swap = pshmem_ctx_longlong_atomic_compare_swap
+#pragma weak shmem_ctx_int32_atomic_compare_swap   	= pshmem_ctx_int32_atomic_compare_swap
+#pragma weak shmem_ctx_int64_atomic_compare_swap   	= pshmem_ctx_int64_atomic_compare_swap
+#pragma weak shmem_ctx_uint32_atomic_compare_swap   	= pshmem_ctx_uint32_atomic_compare_swap
+#pragma weak shmem_ctx_uint64_atomic_compare_swap   	= pshmem_ctx_uint64_atomic_compare_swap
+#pragma weak shmem_ctx_size_atomic_compare_swap   	= pshmem_ctx_size_atomic_compare_swap
+#pragma weak shmem_ctx_ptrdiff_atomic_compare_swap   	= pshmem_ctx_ptrdiff_atomic_compare_swap
 
 #pragma weak shmem_int_atomic_compare_swap = pshmem_int_atomic_compare_swap
 #pragma weak shmem_long_atomic_compare_swap = pshmem_long_atomic_compare_swap
@@ -78,6 +84,12 @@
 #pragma weak shmem_uint_atomic_compare_swap = pshmem_uint_atomic_compare_swap
 #pragma weak shmem_ulong_atomic_compare_swap = pshmem_ulong_atomic_compare_swap
 #pragma weak shmem_ulonglong_atomic_compare_swap = pshmem_ulonglong_atomic_compare_swap
+#pragma weak shmem_int32_atomic_compare_swap   	= pshmem_int32_atomic_compare_swap
+#pragma weak shmem_int64_atomic_compare_swap   	= pshmem_int64_atomic_compare_swap
+#pragma weak shmem_uint32_atomic_compare_swap   	= pshmem_uint32_atomic_compare_swap
+#pragma weak shmem_uint64_atomic_compare_swap   	= pshmem_uint64_atomic_compare_swap
+#pragma weak shmem_size_atomic_compare_swap    	= pshmem_size_atomic_compare_swap
+#pragma weak shmem_ptrdiff_atomic_compare_swap   	= pshmem_ptrdiff_atomic_compare_swap
 
 #pragma weak shmem_int_cswap = pshmem_int_cswap
 #pragma weak shmem_long_cswap = pshmem_long_cswap
@@ -85,7 +97,8 @@
 
 #pragma weak shmemx_int32_cswap = pshmemx_int32_cswap
 #pragma weak shmemx_int64_cswap = pshmemx_int64_cswap
-#include "oshmem/shmem/c/profile/defines.h"
+
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 SHMEM_CTX_TYPE_ATOMIC_COMPARE_SWAP(_int, int, shmem)
@@ -94,12 +107,24 @@ SHMEM_CTX_TYPE_ATOMIC_COMPARE_SWAP(_longlong, long long, shmem)
 SHMEM_CTX_TYPE_ATOMIC_COMPARE_SWAP(_uint, unsigned int, shmem)
 SHMEM_CTX_TYPE_ATOMIC_COMPARE_SWAP(_ulong, unsigned long, shmem)
 SHMEM_CTX_TYPE_ATOMIC_COMPARE_SWAP(_ulonglong, unsigned long long, shmem)
+SHMEM_CTX_TYPE_ATOMIC_COMPARE_SWAP(_int32, int32_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_COMPARE_SWAP(_int64, int64_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_COMPARE_SWAP(_uint32, uint32_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_COMPARE_SWAP(_uint64, uint64_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_COMPARE_SWAP(_size, size_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_COMPARE_SWAP(_ptrdiff, ptrdiff_t, shmem)
 SHMEM_TYPE_ATOMIC_COMPARE_SWAP(_int, int, shmem)
 SHMEM_TYPE_ATOMIC_COMPARE_SWAP(_long, long, shmem)
 SHMEM_TYPE_ATOMIC_COMPARE_SWAP(_longlong, long long, shmem)
 SHMEM_TYPE_ATOMIC_COMPARE_SWAP(_uint, unsigned int, shmem)
 SHMEM_TYPE_ATOMIC_COMPARE_SWAP(_ulong, unsigned long, shmem)
 SHMEM_TYPE_ATOMIC_COMPARE_SWAP(_ulonglong, unsigned long long, shmem)
+SHMEM_TYPE_ATOMIC_COMPARE_SWAP(_int32, int32_t, shmem)
+SHMEM_TYPE_ATOMIC_COMPARE_SWAP(_int64, int64_t, shmem)
+SHMEM_TYPE_ATOMIC_COMPARE_SWAP(_uint32, uint32_t, shmem)
+SHMEM_TYPE_ATOMIC_COMPARE_SWAP(_uint64, uint64_t, shmem)
+SHMEM_TYPE_ATOMIC_COMPARE_SWAP(_size, size_t, shmem)
+SHMEM_TYPE_ATOMIC_COMPARE_SWAP(_ptrdiff, ptrdiff_t, shmem)
 
 /* deprecated APIs */
 #define SHMEM_TYPE_CSWAP(type_name, type, prefix)                   \
diff --git a/oshmem/shmem/c/shmem_cswap_nb.c b/oshmem/shmem/c/shmem_cswap_nb.c
new file mode 100644
index 00000000000..fe69bf54302
--- /dev/null
+++ b/oshmem/shmem/c/shmem_cswap_nb.c
@@ -0,0 +1,127 @@
+/*
+ * Copyright (c) 2021      NVIDIA Corporation.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "oshmem_config.h"
+
+#include "oshmem/constants.h"
+#include "oshmem/include/shmem.h"
+#include "oshmem/include/shmemx.h"
+
+#include "oshmem/runtime/runtime.h"
+
+#include "oshmem/mca/atomic/atomic.h"
+
+/*
+ * shmem_cswap performs an nonblocking atomic conditional swap operation.
+ * The conditional swap routines write value to address target on PE pe, and return the previous
+ * contents of target. The replacement must occur only if cond is equal to target;
+ * otherwise target is left unchanged. In either case, the routine must return the initial value
+ * of target. The operation must be completed without the possibility of another process updating
+ * target between the time of the fetch and the update.
+ */
+#define DO_SHMEM_TYPE_ATOMIC_COMPARE_SWAP_NBI(ctx, type, fetch, target, cond, value, pe, out_value) do { \
+        int rc = OSHMEM_SUCCESS;                                    \
+        size_t size = 0;                                            \
+                                                                    \
+        RUNTIME_CHECK_INIT();                                       \
+        RUNTIME_CHECK_PE(pe);                                       \
+        RUNTIME_CHECK_ADDR(target);                                 \
+                                                                    \
+        size = sizeof(value);                                       \
+        rc = MCA_ATOMIC_CALL(cswap_nb(                              \
+            ctx,                                                    \
+            fetch,                                                  \
+            (void*)target,                                          \
+            (uint64_t*)&out_value,                                  \
+            OSHMEM_ATOMIC_PTR_2_INT(&cond, sizeof(cond)),           \
+            OSHMEM_ATOMIC_PTR_2_INT(&value, sizeof(value)),         \
+            size,                                                   \
+            pe));                                                   \
+        RUNTIME_CHECK_RC(rc);                                       \
+    } while (0)
+
+#define SHMEM_CTX_TYPE_ATOMIC_COMPARE_SWAP_NBI(type_name, type, prefix)                  \
+    void prefix##_ctx##type_name##_atomic_compare_swap_nbi(shmem_ctx_t ctx, type *fetch, type *target, type cond, type value, int pe) \
+    {                                                                                    \
+        type out_value;                                                                  \
+        DO_SHMEM_TYPE_ATOMIC_COMPARE_SWAP_NBI(ctx, type, fetch, target, cond, value,     \
+                                   pe, out_value);                                       \
+        return ;                                                                         \
+    }
+
+#define SHMEM_TYPE_ATOMIC_COMPARE_SWAP_NBI(type_name, type, prefix)                      \
+    void prefix##type_name##_atomic_compare_swap_nbi(type *fetch, type *target, type cond, type value, int pe) \
+    {                                                                                    \
+        type out_value;                                                                  \
+        DO_SHMEM_TYPE_ATOMIC_COMPARE_SWAP_NBI(oshmem_ctx_default, type, fetch, target,   \
+                                   cond, value, pe, out_value);                          \
+        return ;                                                                         \
+    }
+
+#if OSHMEM_PROFILING
+#include "oshmem/include/pshmem.h"
+
+/* Nonblocking Atomic Compare Swap */
+#pragma weak shmem_ctx_int_atomic_compare_swap_nbi               = pshmem_ctx_int_atomic_compare_swap_nbi
+#pragma weak shmem_ctx_long_atomic_compare_swap_nbi              = pshmem_ctx_long_atomic_compare_swap_nbi
+#pragma weak shmem_ctx_longlong_atomic_compare_swap_nbi          = pshmem_ctx_longlong_atomic_compare_swap_nbi
+#pragma weak shmem_ctx_uint_atomic_compare_swap_nbi              = pshmem_ctx_uint_atomic_compare_swap_nbi
+#pragma weak shmem_ctx_ulong_atomic_compare_swap_nbi             = pshmem_ctx_ulong_atomic_compare_swap_nbi
+#pragma weak shmem_ctx_ulonglong_atomic_compare_swap_nbi         = pshmem_ctx_ulonglong_atomic_compare_swap_nbi
+#pragma weak shmem_ctx_int32_atomic_compare_swap_nbi             = pshmem_ctx_int32_atomic_compare_swap_nbi
+#pragma weak shmem_ctx_int64_atomic_compare_swap_nbi             = pshmem_ctx_int64_atomic_compare_swap_nbi
+#pragma weak shmem_ctx_uint32_atomic_compare_swap_nbi            = pshmem_ctx_uint32_atomic_compare_swap_nbi
+#pragma weak shmem_ctx_uint64_atomic_compare_swap_nbi            = pshmem_ctx_uint64_atomic_compare_swap_nbi
+#pragma weak shmem_ctx_size_atomic_compare_swap_nbi              = pshmem_ctx_size_atomic_compare_swap_nbi
+#pragma weak shmem_ctx_ptrdiff_atomic_compare_swap_nbi           = pshmem_ctx_ptrdiff_atomic_compare_swap_nbi
+
+#pragma weak shmem_int_atomic_compare_swap_nbi        			 = pshmem_int_atomic_compare_swap_nbi
+#pragma weak shmem_long_atomic_compare_swap_nbi       			 = pshmem_long_atomic_compare_swap_nbi
+#pragma weak shmem_longlong_atomic_compare_swap_nbi   			 = pshmem_longlong_atomic_compare_swap_nbi
+#pragma weak shmem_uint_atomic_compare_swap_nbi       			 = pshmem_uint_atomic_compare_swap_nbi
+#pragma weak shmem_ulong_atomic_compare_swap_nbi      			 = pshmem_ulong_atomic_compare_swap_nbi
+#pragma weak shmem_ulonglong_atomic_compare_swap_nbi  			 = pshmem_ulonglong_atomic_compare_swap_nbi
+#pragma weak shmem_int32_atomic_compare_swap_nbi      			 = pshmem_int32_atomic_compare_swap_nbi
+#pragma weak shmem_int64_atomic_compare_swap_nbi      			 = pshmem_int64_atomic_compare_swap_nbi
+#pragma weak shmem_uint32_atomic_compare_swap_nbi     			 = pshmem_uint32_atomic_compare_swap_nbi
+#pragma weak shmem_uint64_atomic_compare_swap_nbi     			 = pshmem_uint64_atomic_compare_swap_nbi
+#pragma weak shmem_size_atomic_compare_swap_nbi       			 = pshmem_size_atomic_compare_swap_nbi
+#pragma weak shmem_ptrdiff_atomic_compare_swap_nbi    			 = pshmem_ptrdiff_atomic_compare_swap_nbi
+
+
+#include "oshmem/shmem/c/profile-defines.h"
+#endif
+
+SHMEM_CTX_TYPE_ATOMIC_COMPARE_SWAP_NBI(_int, int, shmem)
+SHMEM_CTX_TYPE_ATOMIC_COMPARE_SWAP_NBI(_long, long, shmem)
+SHMEM_CTX_TYPE_ATOMIC_COMPARE_SWAP_NBI(_longlong, long long, shmem)
+SHMEM_CTX_TYPE_ATOMIC_COMPARE_SWAP_NBI(_uint, unsigned int, shmem)
+SHMEM_CTX_TYPE_ATOMIC_COMPARE_SWAP_NBI(_ulong, unsigned long, shmem)
+SHMEM_CTX_TYPE_ATOMIC_COMPARE_SWAP_NBI(_ulonglong, unsigned long long, shmem)
+SHMEM_CTX_TYPE_ATOMIC_COMPARE_SWAP_NBI(_int32, int32_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_COMPARE_SWAP_NBI(_int64, int64_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_COMPARE_SWAP_NBI(_uint32, uint32_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_COMPARE_SWAP_NBI(_uint64, uint64_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_COMPARE_SWAP_NBI(_size, size_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_COMPARE_SWAP_NBI(_ptrdiff, ptrdiff_t, shmem)
+
+SHMEM_TYPE_ATOMIC_COMPARE_SWAP_NBI(_int, int, shmem)
+SHMEM_TYPE_ATOMIC_COMPARE_SWAP_NBI(_long, long, shmem)
+SHMEM_TYPE_ATOMIC_COMPARE_SWAP_NBI(_longlong, long long, shmem)
+SHMEM_TYPE_ATOMIC_COMPARE_SWAP_NBI(_uint, unsigned int, shmem)
+SHMEM_TYPE_ATOMIC_COMPARE_SWAP_NBI(_ulong, unsigned long, shmem)
+SHMEM_TYPE_ATOMIC_COMPARE_SWAP_NBI(_ulonglong, unsigned long long, shmem)
+SHMEM_TYPE_ATOMIC_COMPARE_SWAP_NBI(_int32, int32_t, shmem)
+SHMEM_TYPE_ATOMIC_COMPARE_SWAP_NBI(_int64, int64_t, shmem)
+SHMEM_TYPE_ATOMIC_COMPARE_SWAP_NBI(_uint32, uint32_t, shmem)
+SHMEM_TYPE_ATOMIC_COMPARE_SWAP_NBI(_uint64, uint64_t, shmem)
+SHMEM_TYPE_ATOMIC_COMPARE_SWAP_NBI(_size, size_t, shmem)
+SHMEM_TYPE_ATOMIC_COMPARE_SWAP_NBI(_ptrdiff, ptrdiff_t, shmem)
+
+
diff --git a/oshmem/shmem/c/shmem_fadd.c b/oshmem/shmem/c/shmem_fadd.c
index 33f18973aca..5b06712a570 100644
--- a/oshmem/shmem/c/shmem_fadd.c
+++ b/oshmem/shmem/c/shmem_fadd.c
@@ -70,6 +70,12 @@
 #pragma weak shmem_ctx_uint_atomic_fetch_add = pshmem_ctx_uint_atomic_fetch_add
 #pragma weak shmem_ctx_ulong_atomic_fetch_add = pshmem_ctx_ulong_atomic_fetch_add
 #pragma weak shmem_ctx_ulonglong_atomic_fetch_add = pshmem_ctx_ulonglong_atomic_fetch_add
+#pragma weak shmem_ctx_int32_atomic_fetch_add   	= pshmem_ctx_int32_atomic_fetch_add
+#pragma weak shmem_ctx_int64_atomic_fetch_add   	= pshmem_ctx_int64_atomic_fetch_add
+#pragma weak shmem_ctx_uint32_atomic_fetch_add   	= pshmem_ctx_uint32_atomic_fetch_add
+#pragma weak shmem_ctx_uint64_atomic_fetch_add   	= pshmem_ctx_uint64_atomic_fetch_add
+#pragma weak shmem_ctx_size_atomic_fetch_add   	= pshmem_ctx_size_atomic_fetch_add
+#pragma weak shmem_ctx_ptrdiff_atomic_fetch_add   	= pshmem_ctx_ptrdiff_atomic_fetch_add
 
 #pragma weak shmem_int_atomic_fetch_add = pshmem_int_atomic_fetch_add
 #pragma weak shmem_long_atomic_fetch_add = pshmem_long_atomic_fetch_add
@@ -77,6 +83,12 @@
 #pragma weak shmem_uint_atomic_fetch_add = pshmem_uint_atomic_fetch_add
 #pragma weak shmem_ulong_atomic_fetch_add = pshmem_ulong_atomic_fetch_add
 #pragma weak shmem_ulonglong_atomic_fetch_add = pshmem_ulonglong_atomic_fetch_add
+#pragma weak shmem_int32_atomic_fetch_add   	= pshmem_int32_atomic_fetch_add
+#pragma weak shmem_int64_atomic_fetch_add   	= pshmem_int64_atomic_fetch_add
+#pragma weak shmem_uint32_atomic_fetch_add   	= pshmem_uint32_atomic_fetch_add
+#pragma weak shmem_uint64_atomic_fetch_add   	= pshmem_uint64_atomic_fetch_add
+#pragma weak shmem_size_atomic_fetch_add   	= pshmem_size_atomic_fetch_add
+#pragma weak shmem_ptrdiff_atomic_fetch_add   	= pshmem_ptrdiff_atomic_fetch_add
 
 #pragma weak shmem_int_fadd = pshmem_int_fadd
 #pragma weak shmem_long_fadd = pshmem_long_fadd
@@ -84,7 +96,9 @@
 
 #pragma weak shmemx_int32_fadd = pshmemx_int32_fadd
 #pragma weak shmemx_int64_fadd = pshmemx_int64_fadd
-#include "oshmem/shmem/c/profile/defines.h"
+
+
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 SHMEM_CTX_TYPE_ATOMIC_FETCH_ADD(_int, int, shmem)
@@ -93,12 +107,24 @@ SHMEM_CTX_TYPE_ATOMIC_FETCH_ADD(_longlong, long long, shmem)
 SHMEM_CTX_TYPE_ATOMIC_FETCH_ADD(_uint, unsigned int, shmem)
 SHMEM_CTX_TYPE_ATOMIC_FETCH_ADD(_ulong, unsigned long, shmem)
 SHMEM_CTX_TYPE_ATOMIC_FETCH_ADD(_ulonglong, unsigned long long, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_ADD(_int32, int32_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_ADD(_int64, int64_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_ADD(_uint32, uint32_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_ADD(_uint64, uint64_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_ADD(_size, size_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_ADD(_ptrdiff, ptrdiff_t, shmem)
 SHMEM_TYPE_ATOMIC_FETCH_ADD(_int, int, shmem)
 SHMEM_TYPE_ATOMIC_FETCH_ADD(_long, long, shmem)
 SHMEM_TYPE_ATOMIC_FETCH_ADD(_longlong, long long, shmem)
 SHMEM_TYPE_ATOMIC_FETCH_ADD(_uint, unsigned int, shmem)
 SHMEM_TYPE_ATOMIC_FETCH_ADD(_ulong, unsigned long, shmem)
 SHMEM_TYPE_ATOMIC_FETCH_ADD(_ulonglong, unsigned long long, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_ADD(_int32, int32_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_ADD(_int64, int64_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_ADD(_uint32, uint32_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_ADD(_uint64, uint64_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_ADD(_size, size_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_ADD(_ptrdiff, ptrdiff_t, shmem)
 
 /* deprecated APIs */
 #define SHMEM_TYPE_FADD(type_name, type, prefix)                    \
diff --git a/oshmem/shmem/c/shmem_fadd_nb.c b/oshmem/shmem/c/shmem_fadd_nb.c
new file mode 100644
index 00000000000..0ed6bc0e2f9
--- /dev/null
+++ b/oshmem/shmem/c/shmem_fadd_nb.c
@@ -0,0 +1,122 @@
+/*
+ * Copyright (c) 2021      NVIDIA Corporation.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "oshmem_config.h"
+
+#include "oshmem/constants.h"
+#include "oshmem/include/shmem.h"
+#include "oshmem/include/shmemx.h"
+
+#include "oshmem/runtime/runtime.h"
+
+#include "oshmem/op/op.h"
+#include "oshmem/mca/atomic/atomic.h"
+
+/*
+ * These routines perform a nonblocking fetch-and-add operation.
+ * The fetch and add routines retrieve the value at address target on PE pe, and update
+ * target with the result of incrementing the retrieved value by one. The operation must be
+ * completed without the possibility of another process updating target between the time of
+ * the fetch and the update.
+ */
+#define DO_SHMEM_TYPE_ATOMIC_FETCH_ADD_NBI(ctx, type_name, type, fetch, target, value, pe, out_value) do { \
+        int rc = OSHMEM_SUCCESS;                                    \
+        size_t size;                                                \
+                                                                    \
+        RUNTIME_CHECK_INIT();                                       \
+        RUNTIME_CHECK_PE(pe);                                       \
+        RUNTIME_CHECK_ADDR(target);                                 \
+                                                                    \
+        size = sizeof(out_value);                                   \
+        rc = MCA_ATOMIC_CALL(fadd_nb(                               \
+            ctx,                                                    \
+            fetch,                                                  \
+            (void*)target,                                          \
+            (void*)&out_value,                                      \
+            value,                                                  \
+            size,                                                   \
+            pe));                                                   \
+        RUNTIME_CHECK_RC(rc);                                       \
+    } while (0)
+
+#define SHMEM_CTX_TYPE_ATOMIC_FETCH_ADD_NBI(type_name, type, prefix)            \
+    void prefix##_ctx##type_name##_atomic_fetch_add_nbi(shmem_ctx_t ctx, type *fetch, type *target, type value, int pe) \
+    {                                                                           \
+        type out_value;                                                         \
+        DO_SHMEM_TYPE_ATOMIC_FETCH_ADD_NBI(ctx, type_name, type, fetch, target, \
+                                  value, pe, out_value);                        \
+        return ;                                                                \
+    }
+
+#define SHMEM_TYPE_ATOMIC_FETCH_ADD_NBI(type_name, type, prefix)                \
+    void prefix##type_name##_atomic_fetch_add_nbi(type *fetch, type *target, type value, int pe) \
+    {                                                                           \
+        type out_value;                                                         \
+        DO_SHMEM_TYPE_ATOMIC_FETCH_ADD_NBI(oshmem_ctx_default, type_name,       \
+                                  type, fetch, target, value, pe, out_value);   \
+        return ;                                                                \
+    }
+
+#if OSHMEM_PROFILING
+#include "oshmem/include/pshmem.h"
+
+/* Nonblocking Atomic Fetch and Add */
+#pragma weak shmem_ctx_int_atomic_fetch_add_nbi               = pshmem_ctx_int_atomic_fetch_add_nbi
+#pragma weak shmem_ctx_long_atomic_fetch_add_nbi              = pshmem_ctx_long_atomic_fetch_add_nbi
+#pragma weak shmem_ctx_longlong_atomic_fetch_add_nbi          = pshmem_ctx_longlong_atomic_fetch_add_nbi
+#pragma weak shmem_ctx_uint_atomic_fetch_add_nbi              = pshmem_ctx_uint_atomic_fetch_add_nbi
+#pragma weak shmem_ctx_ulong_atomic_fetch_add_nbi             = pshmem_ctx_ulong_atomic_fetch_add_nbi
+#pragma weak shmem_ctx_ulonglong_atomic_fetch_add_nbi         = pshmem_ctx_ulonglong_atomic_fetch_add_nbi
+#pragma weak shmem_ctx_int32_atomic_fetch_add_nbi             = pshmem_ctx_int32_atomic_fetch_add_nbi
+#pragma weak shmem_ctx_int64_atomic_fetch_add_nbi             = pshmem_ctx_int64_atomic_fetch_add_nbi
+#pragma weak shmem_ctx_uint32_atomic_fetch_add_nbi            = pshmem_ctx_uint32_atomic_fetch_add_nbi
+#pragma weak shmem_ctx_uint64_atomic_fetch_add_nbi            = pshmem_ctx_uint64_atomic_fetch_add_nbi
+#pragma weak shmem_ctx_size_atomic_fetch_add_nbi              = pshmem_ctx_size_atomic_fetch_add_nbi
+#pragma weak shmem_ctx_ptrdiff_atomic_fetch_add_nbi           = pshmem_ctx_ptrdiff_atomic_fetch_add_nbi
+
+#pragma weak shmem_int_atomic_fetch_add_nbi        			  = pshmem_int_atomic_fetch_add_nbi
+#pragma weak shmem_long_atomic_fetch_add_nbi       			  = pshmem_long_atomic_fetch_add_nbi
+#pragma weak shmem_longlong_atomic_fetch_add_nbi   			  = pshmem_longlong_atomic_fetch_add_nbi
+#pragma weak shmem_uint_atomic_fetch_add_nbi       			  = pshmem_uint_atomic_fetch_add_nbi
+#pragma weak shmem_ulong_atomic_fetch_add_nbi      			  = pshmem_ulong_atomic_fetch_add_nbi
+#pragma weak shmem_ulonglong_atomic_fetch_add_nbi  			  = pshmem_ulonglong_atomic_fetch_add_nbi
+#pragma weak shmem_int32_atomic_fetch_add_nbi      			  = pshmem_int32_atomic_fetch_add_nbi
+#pragma weak shmem_int64_atomic_fetch_add_nbi      			  = pshmem_int64_atomic_fetch_add_nbi
+#pragma weak shmem_uint32_atomic_fetch_add_nbi     			  = pshmem_uint32_atomic_fetch_add_nbi
+#pragma weak shmem_uint64_atomic_fetch_add_nbi     			  = pshmem_uint64_atomic_fetch_add_nbi
+#pragma weak shmem_size_atomic_fetch_add_nbi       			  = pshmem_size_atomic_fetch_add_nbi
+#pragma weak shmem_ptrdiff_atomic_fetch_add_nbi    			  = pshmem_ptrdiff_atomic_fetch_add_nbi
+
+#include "oshmem/shmem/c/profile-defines.h"
+#endif
+
+SHMEM_CTX_TYPE_ATOMIC_FETCH_ADD_NBI(_int, int, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_ADD_NBI(_long, long, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_ADD_NBI(_longlong, long long, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_ADD_NBI(_uint, unsigned int, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_ADD_NBI(_ulong, unsigned long, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_ADD_NBI(_ulonglong, unsigned long long, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_ADD_NBI(_int32, int32_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_ADD_NBI(_int64, int64_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_ADD_NBI(_uint32, uint32_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_ADD_NBI(_uint64, uint64_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_ADD_NBI(_size, size_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_ADD_NBI(_ptrdiff, ptrdiff_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_ADD_NBI(_int, int, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_ADD_NBI(_long, long, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_ADD_NBI(_longlong, long long, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_ADD_NBI(_uint, unsigned int, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_ADD_NBI(_ulong, unsigned long, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_ADD_NBI(_ulonglong, unsigned long long, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_ADD_NBI(_int32, int32_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_ADD_NBI(_int64, int64_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_ADD_NBI(_uint32, uint32_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_ADD_NBI(_uint64, uint64_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_ADD_NBI(_size, size_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_ADD_NBI(_ptrdiff, ptrdiff_t, shmem)
diff --git a/oshmem/shmem/c/shmem_fand.c b/oshmem/shmem/c/shmem_fand.c
index 6761844f291..7bbf5952d1e 100644
--- a/oshmem/shmem/c/shmem_fand.c
+++ b/oshmem/shmem/c/shmem_fand.c
@@ -53,7 +53,8 @@
 #pragma weak shmemx_int64_atomic_fetch_and    = pshmemx_int64_atomic_fetch_and
 #pragma weak shmemx_uint32_atomic_fetch_and   = pshmemx_uint32_atomic_fetch_and
 #pragma weak shmemx_uint64_atomic_fetch_and   = pshmemx_uint64_atomic_fetch_and
-#include "oshmem/shmem/c/profile/defines.h"
+
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 OSHMEM_TYPE_FOP(int, int, shmem, and)
diff --git a/oshmem/shmem/c/shmem_fand_nb.c b/oshmem/shmem/c/shmem_fand_nb.c
new file mode 100644
index 00000000000..ded0df46928
--- /dev/null
+++ b/oshmem/shmem/c/shmem_fand_nb.c
@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2021      NVIDIA Corporation.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "oshmem_config.h"
+
+#include "oshmem/constants.h"
+#include "oshmem/include/shmem.h"
+#include "oshmem/include/shmemx.h"
+
+#include "oshmem/runtime/runtime.h"
+
+#include "oshmem/op/op.h"
+#include "oshmem/mca/atomic/atomic.h"
+
+/*
+ * These routines perform an atomic nonblocking fetch-and-and operation.
+ * The fetch and and routines retrieve the value at address target on PE pe, and update
+ * target with the result of 'and' operation value to the retrieved value. The operation
+ * must be completed without the possibility of another process updating target between
+ * the time of the fetch and the update.
+ */
+#if OSHMEM_PROFILING
+#include "oshmem/include/pshmem.h"
+
+/* Nonblocking Atomic Fetch and And */
+#pragma weak shmem_ctx_uint_atomic_fetch_and_nbi              = pshmem_ctx_uint_atomic_fetch_and_nbi
+#pragma weak shmem_ctx_ulong_atomic_fetch_and_nbi             = pshmem_ctx_ulong_atomic_fetch_and_nbi
+#pragma weak shmem_ctx_ulonglong_atomic_fetch_and_nbi         = pshmem_ctx_ulonglong_atomic_fetch_and_nbi
+#pragma weak shmem_ctx_int32_atomic_fetch_and_nbi             = pshmem_ctx_int32_atomic_fetch_and_nbi
+#pragma weak shmem_ctx_int64_atomic_fetch_and_nbi             = pshmem_ctx_int64_atomic_fetch_and_nbi
+#pragma weak shmem_ctx_uint32_atomic_fetch_and_nbi            = pshmem_ctx_uint32_atomic_fetch_and_nbi
+#pragma weak shmem_ctx_uint64_atomic_fetch_and_nbi            = pshmem_ctx_uint64_atomic_fetch_and_nbi
+
+#pragma weak shmem_uint_atomic_fetch_and_nbi       			  = pshmem_uint_atomic_fetch_and_nbi
+#pragma weak shmem_ulong_atomic_fetch_and_nbi      			  = pshmem_ulong_atomic_fetch_and_nbi
+#pragma weak shmem_ulonglong_atomic_fetch_and_nbi  			  = pshmem_ulonglong_atomic_fetch_and_nbi
+#pragma weak shmem_int32_atomic_fetch_and_nbi      			  = pshmem_int32_atomic_fetch_and_nbi
+#pragma weak shmem_int64_atomic_fetch_and_nbi      			  = pshmem_int64_atomic_fetch_and_nbi
+#pragma weak shmem_uint32_atomic_fetch_and_nbi     			  = pshmem_uint32_atomic_fetch_and_nbi
+#pragma weak shmem_uint64_atomic_fetch_and_nbi     			  = pshmem_uint64_atomic_fetch_and_nbi
+
+#include "oshmem/shmem/c/profile-defines.h"
+#endif
+
+OSHMEM_TYPE_FOP_NBI(uint, unsigned int, shmem, and)
+OSHMEM_TYPE_FOP_NBI(ulong, unsigned long, shmem, and)
+OSHMEM_TYPE_FOP_NBI(ulonglong, unsigned long long, shmem, and)
+OSHMEM_TYPE_FOP_NBI(int32, int32_t, shmem, and)
+OSHMEM_TYPE_FOP_NBI(int64, int64_t, shmem, and)
+OSHMEM_TYPE_FOP_NBI(uint32, uint32_t, shmem, and)
+OSHMEM_TYPE_FOP_NBI(uint64, uint64_t, shmem, and)
+
+OSHMEM_CTX_TYPE_FOP_NBI(uint, unsigned int, shmem, and)
+OSHMEM_CTX_TYPE_FOP_NBI(ulong, unsigned long, shmem, and)
+OSHMEM_CTX_TYPE_FOP_NBI(ulonglong, unsigned long long, shmem, and)
+OSHMEM_CTX_TYPE_FOP_NBI(int32, int32_t, shmem, and)
+OSHMEM_CTX_TYPE_FOP_NBI(int64, int64_t, shmem, and)
+OSHMEM_CTX_TYPE_FOP_NBI(uint32, uint32_t, shmem, and)
+OSHMEM_CTX_TYPE_FOP_NBI(uint64, uint64_t, shmem, and)
+
diff --git a/oshmem/shmem/c/shmem_fence.c b/oshmem/shmem/c/shmem_fence.c
index 0a049bee57b..d1fc2b442aa 100644
--- a/oshmem/shmem/c/shmem_fence.c
+++ b/oshmem/shmem/c/shmem_fence.c
@@ -18,7 +18,7 @@
 #include "oshmem/include/pshmem.h"
 #pragma weak shmem_fence = pshmem_fence
 #pragma weak shmem_ctx_fence = pshmem_ctx_fence
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 void shmem_fence(void)
diff --git a/oshmem/shmem/c/shmem_fetch.c b/oshmem/shmem/c/shmem_fetch.c
index fe02d5a16e5..58092eda8c6 100644
--- a/oshmem/shmem/c/shmem_fetch.c
+++ b/oshmem/shmem/c/shmem_fetch.c
@@ -73,6 +73,12 @@
 #pragma weak shmem_ctx_ulonglong_atomic_fetch = pshmem_ctx_ulonglong_atomic_fetch
 #pragma weak shmem_ctx_double_atomic_fetch = pshmem_ctx_double_atomic_fetch
 #pragma weak shmem_ctx_float_atomic_fetch = pshmem_ctx_float_atomic_fetch
+#pragma weak shmem_ctx_int32_atomic_fetch    	= pshmem_ctx_int32_atomic_fetch
+#pragma weak shmem_ctx_int64_atomic_fetch    	= pshmem_ctx_int64_atomic_fetch
+#pragma weak shmem_ctx_uint32_atomic_fetch    	= pshmem_ctx_uint32_atomic_fetch
+#pragma weak shmem_ctx_uint64_atomic_fetch    	= pshmem_ctx_uint64_atomic_fetch
+#pragma weak shmem_ctx_size_atomic_fetch    	= pshmem_ctx_size_atomic_fetch
+#pragma weak shmem_ctx_ptrdiff_atomic_fetch   	= pshmem_ctx_ptrdiff_atomic_fetch
 
 #pragma weak shmem_int_atomic_fetch = pshmem_int_atomic_fetch
 #pragma weak shmem_long_atomic_fetch = pshmem_long_atomic_fetch
@@ -82,6 +88,12 @@
 #pragma weak shmem_ulonglong_atomic_fetch = pshmem_ulonglong_atomic_fetch
 #pragma weak shmem_double_atomic_fetch = pshmem_double_atomic_fetch
 #pragma weak shmem_float_atomic_fetch = pshmem_float_atomic_fetch
+#pragma weak shmem_int32_atomic_fetch    	= pshmem_int32_atomic_fetch
+#pragma weak shmem_int64_atomic_fetch    	= pshmem_int64_atomic_fetch
+#pragma weak shmem_uint32_atomic_fetch    	= pshmem_uint32_atomic_fetch
+#pragma weak shmem_uint64_atomic_fetch    	= pshmem_uint64_atomic_fetch
+#pragma weak shmem_size_atomic_fetch    	= pshmem_size_atomic_fetch
+#pragma weak shmem_ptrdiff_atomic_fetch    	= pshmem_ptrdiff_atomic_fetch
 
 #pragma weak shmem_int_fetch = pshmem_int_fetch
 #pragma weak shmem_long_fetch = pshmem_long_fetch
@@ -91,7 +103,12 @@
 
 #pragma weak shmemx_int32_fetch = pshmemx_int32_fetch
 #pragma weak shmemx_int64_fetch = pshmemx_int64_fetch
-#include "oshmem/shmem/c/profile/defines.h"
+
+#include "oshmem/shmem/c/profile-defines.h"
+
+
+
+
 #endif
 
 SHMEM_CTX_TYPE_ATOMIC_FETCH(_int, int, shmem)
@@ -102,6 +119,12 @@ SHMEM_CTX_TYPE_ATOMIC_FETCH(_ulong, unsigned long, shmem)
 SHMEM_CTX_TYPE_ATOMIC_FETCH(_ulonglong, unsigned long long, shmem)
 SHMEM_CTX_TYPE_ATOMIC_FETCH(_double, double, shmem)
 SHMEM_CTX_TYPE_ATOMIC_FETCH(_float, float, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH(_int32, int32_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH(_int64, int64_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH(_uint32, uint32_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH(_uint64, uint64_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH(_size, size_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH(_ptrdiff, ptrdiff_t, shmem)
 SHMEM_TYPE_ATOMIC_FETCH(_int, int, shmem)
 SHMEM_TYPE_ATOMIC_FETCH(_long, long, shmem)
 SHMEM_TYPE_ATOMIC_FETCH(_longlong, long long, shmem)
@@ -110,6 +133,12 @@ SHMEM_TYPE_ATOMIC_FETCH(_ulong, unsigned long, shmem)
 SHMEM_TYPE_ATOMIC_FETCH(_ulonglong, unsigned long long, shmem)
 SHMEM_TYPE_ATOMIC_FETCH(_double, double, shmem)
 SHMEM_TYPE_ATOMIC_FETCH(_float, float, shmem)
+SHMEM_TYPE_ATOMIC_FETCH(_int32, int32_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH(_int64, int64_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH(_uint32, uint32_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH(_uint64, uint64_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH(_size, size_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH(_ptrdiff, ptrdiff_t, shmem)
 
 /* deprecated APIs */
 #define SHMEM_TYPE_FETCH(type_name, type, prefix)                   \
diff --git a/oshmem/shmem/c/shmem_fetch_nb.c b/oshmem/shmem/c/shmem_fetch_nb.c
new file mode 100644
index 00000000000..9fe1b31be9f
--- /dev/null
+++ b/oshmem/shmem/c/shmem_fetch_nb.c
@@ -0,0 +1,134 @@
+/*
+ * Copyright (c) 2021      NVIDIA Corporation.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "oshmem_config.h"
+
+#include "oshmem/constants.h"
+#include "oshmem/include/shmem.h"
+#include "oshmem/include/shmemx.h"
+
+#include "oshmem/runtime/runtime.h"
+
+#include "oshmem/op/op.h"
+#include "oshmem/mca/atomic/atomic.h"
+
+/*
+ * These routines perform an nonblocking atomic fetch operation.
+ * The fetch routines retrieve the value at address target on PE pe.
+ * The operation must be completed without the possibility of another process
+ * updating target during the fetch.
+ */
+#define DO_SHMEM_TYPE_ATOMIC_FETCH_NBI(ctx, type_name, type, fetch, target, pe, out_value) do { \
+        int rc = OSHMEM_SUCCESS;                                    \
+        size_t size = 0;                                            \
+        type value = 0;                                             \
+                                                                    \
+        RUNTIME_CHECK_INIT();                                       \
+        RUNTIME_CHECK_PE(pe);                                       \
+        RUNTIME_CHECK_ADDR(target);                                 \
+                                                                    \
+        size = sizeof(out_value);                                   \
+        rc = MCA_ATOMIC_CALL(fadd_nb(                               \
+            ctx,                                                    \
+            fetch,                                                  \
+            (void*)target,                                          \
+            (void*)&out_value,                                      \
+            value,                                                  \
+            size,                                                   \
+            pe));                                                   \
+        RUNTIME_CHECK_RC(rc);                                       \
+    } while (0)
+
+#define SHMEM_CTX_TYPE_ATOMIC_FETCH_NBI(type_name, type, prefix)                 \
+    void prefix##_ctx##type_name##_atomic_fetch_nbi(shmem_ctx_t ctx, type *fetch, const type *target, int pe) \
+    {                                                                            \
+        type out_value;                                                          \
+        DO_SHMEM_TYPE_ATOMIC_FETCH_NBI(ctx, type_name, type, fetch, target,      \
+                                   pe, out_value);                               \
+        return ;                                                                 \
+    }
+
+#define SHMEM_TYPE_ATOMIC_FETCH_NBI(type_name, type, prefix)                     \
+    void prefix##type_name##_atomic_fetch_nbi(type *fetch, const type *target, int pe) \
+    {                                                                            \
+        type out_value;                                                          \
+        DO_SHMEM_TYPE_ATOMIC_FETCH_NBI(oshmem_ctx_default, type_name,            \
+                                   type, fetch, target, pe, out_value);          \
+                                                                                 \
+        return ;                                                                 \
+    }
+
+
+#if OSHMEM_PROFILING
+#include "oshmem/include/pshmem.h"
+
+/* Nonblocking Atomic Fetch */
+#pragma weak shmem_ctx_double_atomic_fetch_nbi           = pshmem_ctx_double_atomic_fetch_nbi
+#pragma weak shmem_ctx_float_atomic_fetch_nbi            = pshmem_ctx_float_atomic_fetch_nbi
+#pragma weak shmem_ctx_int_atomic_fetch_nbi              = pshmem_ctx_int_atomic_fetch_nbi
+#pragma weak shmem_ctx_long_atomic_fetch_nbi             = pshmem_ctx_long_atomic_fetch_nbi
+#pragma weak shmem_ctx_longlong_atomic_fetch_nbi         = pshmem_ctx_longlong_atomic_fetch_nbi
+#pragma weak shmem_ctx_uint_atomic_fetch_nbi             = pshmem_ctx_uint_atomic_fetch_nbi
+#pragma weak shmem_ctx_ulong_atomic_fetch_nbi            = pshmem_ctx_ulong_atomic_fetch_nbi
+#pragma weak shmem_ctx_ulonglong_atomic_fetch_nbi        = pshmem_ctx_ulonglong_atomic_fetch_nbi
+#pragma weak shmem_ctx_int32_atomic_fetch_nbi            = pshmem_ctx_int32_atomic_fetch_nbi
+#pragma weak shmem_ctx_int64_atomic_fetch_nbi            = pshmem_ctx_int64_atomic_fetch_nbi
+#pragma weak shmem_ctx_uint32_atomic_fetch_nbi           = pshmem_ctx_uint32_atomic_fetch_nbi
+#pragma weak shmem_ctx_uint64_atomic_fetch_nbi           = pshmem_ctx_uint64_atomic_fetch_nbi
+#pragma weak shmem_ctx_size_atomic_fetch_nbi             = pshmem_ctx_size_atomic_fetch_nbi
+#pragma weak shmem_ctx_ptrdiff_atomic_fetch_nbi          = pshmem_ctx_ptrdiff_atomic_fetch_nbi
+
+#pragma weak shmem_double_atomic_fetch_nbi     			 = pshmem_double_atomic_fetch_nbi
+#pragma weak shmem_float_atomic_fetch_nbi      			 = pshmem_float_atomic_fetch_nbi
+#pragma weak shmem_int_atomic_fetch_nbi        			 = pshmem_int_atomic_fetch_nbi
+#pragma weak shmem_long_atomic_fetch_nbi       			 = pshmem_long_atomic_fetch_nbi
+#pragma weak shmem_longlong_atomic_fetch_nbi   			 = pshmem_longlong_atomic_fetch_nbi
+#pragma weak shmem_uint_atomic_fetch_nbi       			 = pshmem_uint_atomic_fetch_nbi
+#pragma weak shmem_ulong_atomic_fetch_nbi      			 = pshmem_ulong_atomic_fetch_nbi
+#pragma weak shmem_ulonglong_atomic_fetch_nbi  			 = pshmem_ulonglong_atomic_fetch_nbi
+#pragma weak shmem_int32_atomic_fetch_nbi      			 = pshmem_int32_atomic_fetch_nbi
+#pragma weak shmem_int64_atomic_fetch_nbi      			 = pshmem_int64_atomic_fetch_nbi
+#pragma weak shmem_uint32_atomic_fetch_nbi     			 = pshmem_uint32_atomic_fetch_nbi
+#pragma weak shmem_uint64_atomic_fetch_nbi     			 = pshmem_uint64_atomic_fetch_nbi
+#pragma weak shmem_size_atomic_fetch_nbi       			 = pshmem_size_atomic_fetch_nbi
+#pragma weak shmem_ptrdiff_atomic_fetch_nbi    			 = pshmem_ptrdiff_atomic_fetch_nbi
+
+#include "oshmem/shmem/c/profile-defines.h"
+#endif
+
+SHMEM_CTX_TYPE_ATOMIC_FETCH_NBI(_int, int, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_NBI(_long, long, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_NBI(_longlong, long long, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_NBI(_uint, unsigned int, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_NBI(_ulong, unsigned long, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_NBI(_ulonglong, unsigned long long, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_NBI(_double, double, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_NBI(_float, float, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_NBI(_int32, int32_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_NBI(_int64, int64_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_NBI(_uint32, uint32_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_NBI(_uint64, uint64_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_NBI(_size, size_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_NBI(_ptrdiff, ptrdiff_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_NBI(_int, int, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_NBI(_long, long, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_NBI(_longlong, long long, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_NBI(_uint, unsigned int, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_NBI(_ulong, unsigned long, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_NBI(_ulonglong, unsigned long long, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_NBI(_double, double, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_NBI(_float, float, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_NBI(_int32, int32_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_NBI(_int64, int64_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_NBI(_uint32, uint32_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_NBI(_uint64, uint64_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_NBI(_size, size_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_NBI(_ptrdiff, ptrdiff_t, shmem)
+
+
diff --git a/oshmem/shmem/c/shmem_finalize.c b/oshmem/shmem/c/shmem_finalize.c
index e35a7a72e33..4e0a2681948 100644
--- a/oshmem/shmem/c/shmem_finalize.c
+++ b/oshmem/shmem/c/shmem_finalize.c
@@ -17,7 +17,7 @@
 #if OSHMEM_PROFILING
 #include "oshmem/include/pshmem.h"
 #pragma weak shmem_finalize = pshmem_finalize
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 void shmem_finalize(void)
diff --git a/oshmem/shmem/c/shmem_finc.c b/oshmem/shmem/c/shmem_finc.c
index de3ae9b2dd1..913dea8b90a 100644
--- a/oshmem/shmem/c/shmem_finc.c
+++ b/oshmem/shmem/c/shmem_finc.c
@@ -71,6 +71,12 @@
 #pragma weak shmem_ctx_uint_atomic_fetch_inc = pshmem_ctx_uint_atomic_fetch_inc
 #pragma weak shmem_ctx_ulong_atomic_fetch_inc = pshmem_ctx_ulong_atomic_fetch_inc
 #pragma weak shmem_ctx_ulonglong_atomic_fetch_inc = pshmem_ctx_ulonglong_atomic_fetch_inc
+#pragma weak shmem_ctx_int32_atomic_fetch_inc    	= pshmem_ctx_int32_atomic_fetch_inc
+#pragma weak shmem_ctx_int64_atomic_fetch_inc    	= pshmem_ctx_int64_atomic_fetch_inc
+#pragma weak shmem_ctx_uint32_atomic_fetch_inc    	= pshmem_ctx_uint32_atomic_fetch_inc
+#pragma weak shmem_ctx_uint64_atomic_fetch_inc    	= pshmem_ctx_uint64_atomic_fetch_inc
+#pragma weak shmem_ctx_size_atomic_fetch_inc    	= pshmem_ctx_size_atomic_fetch_inc
+#pragma weak shmem_ctx_ptrdiff_atomic_fetch_inc    	= pshmem_ctx_ptrdiff_atomic_fetch_inc
 
 #pragma weak shmem_int_atomic_fetch_inc = pshmem_int_atomic_fetch_inc
 #pragma weak shmem_long_atomic_fetch_inc = pshmem_long_atomic_fetch_inc
@@ -78,6 +84,12 @@
 #pragma weak shmem_uint_atomic_fetch_inc = pshmem_uint_atomic_fetch_inc
 #pragma weak shmem_ulong_atomic_fetch_inc = pshmem_ulong_atomic_fetch_inc
 #pragma weak shmem_ulonglong_atomic_fetch_inc = pshmem_ulonglong_atomic_fetch_inc
+#pragma weak shmem_int32_atomic_fetch_inc    	= pshmem_int32_atomic_fetch_inc
+#pragma weak shmem_int64_atomic_fetch_inc    	= pshmem_int64_atomic_fetch_inc
+#pragma weak shmem_uint32_atomic_fetch_inc    	= pshmem_uint32_atomic_fetch_inc
+#pragma weak shmem_uint64_atomic_fetch_inc    	= pshmem_uint64_atomic_fetch_inc
+#pragma weak shmem_size_atomic_fetch_inc    	= pshmem_size_atomic_fetch_inc
+#pragma weak shmem_ptrdiff_atomic_fetch_inc   	= pshmem_ptrdiff_atomic_fetch_inc
 
 #pragma weak shmem_int_finc = pshmem_int_finc
 #pragma weak shmem_long_finc = pshmem_long_finc
@@ -85,7 +97,8 @@
 
 #pragma weak shmemx_int32_finc = pshmemx_int32_finc
 #pragma weak shmemx_int64_finc = pshmemx_int64_finc
-#include "oshmem/shmem/c/profile/defines.h"
+
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 SHMEM_CTX_TYPE_ATOMIC_FETCH_INC(_int, int, shmem)
@@ -94,12 +107,25 @@ SHMEM_CTX_TYPE_ATOMIC_FETCH_INC(_longlong, long long, shmem)
 SHMEM_CTX_TYPE_ATOMIC_FETCH_INC(_uint, unsigned int, shmem)
 SHMEM_CTX_TYPE_ATOMIC_FETCH_INC(_ulong, unsigned long, shmem)
 SHMEM_CTX_TYPE_ATOMIC_FETCH_INC(_ulonglong, unsigned long long, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_INC(_int32, int32_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_INC(_int64, int64_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_INC(_uint32, uint32_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_INC(_uint64, uint64_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_INC(_size, size_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_INC(_ptrdiff, ptrdiff_t, shmem)
+
 SHMEM_TYPE_ATOMIC_FETCH_INC(_int, int, shmem)
 SHMEM_TYPE_ATOMIC_FETCH_INC(_long, long, shmem)
 SHMEM_TYPE_ATOMIC_FETCH_INC(_longlong, long long, shmem)
 SHMEM_TYPE_ATOMIC_FETCH_INC(_uint, unsigned int, shmem)
 SHMEM_TYPE_ATOMIC_FETCH_INC(_ulong, unsigned long, shmem)
 SHMEM_TYPE_ATOMIC_FETCH_INC(_ulonglong, unsigned long long, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_INC(_int32, int32_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_INC(_int64, int64_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_INC(_uint32, uint32_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_INC(_uint64, uint64_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_INC(_size, size_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_INC(_ptrdiff, ptrdiff_t, shmem)
 
 /* deprecated APIs */
 #define SHMEM_TYPE_FINC(type_name, type, prefix)                    \
diff --git a/oshmem/shmem/c/shmem_finc_nb.c b/oshmem/shmem/c/shmem_finc_nb.c
new file mode 100644
index 00000000000..7a0b4d59923
--- /dev/null
+++ b/oshmem/shmem/c/shmem_finc_nb.c
@@ -0,0 +1,123 @@
+/*
+ * Copyright (c) 2021      NVIDIA Corporation.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "oshmem_config.h"
+
+#include "oshmem/constants.h"
+#include "oshmem/include/shmem.h"
+#include "oshmem/include/shmemx.h"
+
+#include "oshmem/runtime/runtime.h"
+
+#include "oshmem/op/op.h"
+#include "oshmem/mca/atomic/atomic.h"
+
+/*
+ * These routines perform a nonblocking fetch-and-increment operation.
+ * The fetch and increment routines retrieve the value at address target on PE pe, and update
+ * target with the result of incrementing the retrieved value by one. The operation must be
+ * completed without the possibility of another process updating target between the time of
+ * the fetch and the update.
+ */
+#define DO_SHMEM_TYPE_ATOMIC_FETCH_INC_NBI(ctx, type_name, type, fetch, target, pe, out_value) do { \
+        int rc = OSHMEM_SUCCESS;                                    \
+        size_t size;                                                \
+        type value = 1;                                             \
+                                                                    \
+        RUNTIME_CHECK_INIT();                                       \
+        RUNTIME_CHECK_PE(pe);                                       \
+        RUNTIME_CHECK_ADDR(target);                                 \
+                                                                    \
+        size = sizeof(out_value);                                   \
+        rc = MCA_ATOMIC_CALL(fadd_nb(                               \
+            ctx,                                                    \
+            fetch,                                                  \
+            (void*)target,                                          \
+            (void*)&out_value,                                      \
+            value,                                                  \
+            size,                                                   \
+            pe));                                                   \
+        RUNTIME_CHECK_RC(rc);                                       \
+    } while (0)
+
+#define SHMEM_CTX_TYPE_ATOMIC_FETCH_INC_NBI(type_name, type, prefix)            \
+    void prefix##_ctx##type_name##_atomic_fetch_inc_nbi(shmem_ctx_t ctx, type *fetch, type *target, int pe) \
+    {                                                                           \
+        type out_value;                                                         \
+        DO_SHMEM_TYPE_ATOMIC_FETCH_INC_NBI(ctx, type_name, type, fetch, target, \
+                                  pe, out_value);                               \
+        return ;                                                                \
+    }
+
+#define SHMEM_TYPE_ATOMIC_FETCH_INC_NBI(type_name, type, prefix)                \
+    void prefix##type_name##_atomic_fetch_inc_nbi(type *fetch, type *target, int pe) \
+    {                                                                           \
+        type out_value;                                                         \
+        DO_SHMEM_TYPE_ATOMIC_FETCH_INC_NBI(oshmem_ctx_default, type_name,       \
+                                  type, fetch, target, pe, out_value);          \
+        return ;                                                                \
+    }
+
+#if OSHMEM_PROFILING
+#include "oshmem/include/pshmem.h"
+
+/* Nonblocking Atomic Fetch and Increment */
+#pragma weak shmem_ctx_int_atomic_fetch_inc_nbi               = pshmem_ctx_int_atomic_fetch_inc_nbi
+#pragma weak shmem_ctx_long_atomic_fetch_inc_nbi              = pshmem_ctx_long_atomic_fetch_inc_nbi
+#pragma weak shmem_ctx_longlong_atomic_fetch_inc_nbi          = pshmem_ctx_longlong_atomic_fetch_inc_nbi
+#pragma weak shmem_ctx_uint_atomic_fetch_inc_nbi              = pshmem_ctx_uint_atomic_fetch_inc_nbi
+#pragma weak shmem_ctx_ulong_atomic_fetch_inc_nbi             = pshmem_ctx_ulong_atomic_fetch_inc_nbi
+#pragma weak shmem_ctx_ulonglong_atomic_fetch_inc_nbi         = pshmem_ctx_ulonglong_atomic_fetch_inc_nbi
+#pragma weak shmem_ctx_int32_atomic_fetch_inc_nbi             = pshmem_ctx_int32_atomic_fetch_inc_nbi
+#pragma weak shmem_ctx_int64_atomic_fetch_inc_nbi             = pshmem_ctx_int64_atomic_fetch_inc_nbi
+#pragma weak shmem_ctx_uint32_atomic_fetch_inc_nbi            = pshmem_ctx_uint32_atomic_fetch_inc_nbi
+#pragma weak shmem_ctx_uint64_atomic_fetch_inc_nbi            = pshmem_ctx_uint64_atomic_fetch_inc_nbi
+#pragma weak shmem_ctx_size_atomic_fetch_inc_nbi              = pshmem_ctx_size_atomic_fetch_inc_nbi
+#pragma weak shmem_ctx_ptrdiff_atomic_fetch_inc_nbi           = pshmem_ctx_ptrdiff_atomic_fetch_inc_nbi
+
+#pragma weak shmem_int_atomic_fetch_inc_nbi        			  = pshmem_int_atomic_fetch_inc_nbi
+#pragma weak shmem_long_atomic_fetch_inc_nbi       			  = pshmem_long_atomic_fetch_inc_nbi
+#pragma weak shmem_longlong_atomic_fetch_inc_nbi   			  = pshmem_longlong_atomic_fetch_inc_nbi
+#pragma weak shmem_uint_atomic_fetch_inc_nbi       			  = pshmem_uint_atomic_fetch_inc_nbi
+#pragma weak shmem_ulong_atomic_fetch_inc_nbi      			  = pshmem_ulong_atomic_fetch_inc_nbi
+#pragma weak shmem_ulonglong_atomic_fetch_inc_nbi  			  = pshmem_ulonglong_atomic_fetch_inc_nbi
+#pragma weak shmem_int32_atomic_fetch_inc_nbi      			  = pshmem_int32_atomic_fetch_inc_nbi
+#pragma weak shmem_int64_atomic_fetch_inc_nbi      			  = pshmem_int64_atomic_fetch_inc_nbi
+#pragma weak shmem_uint32_atomic_fetch_inc_nbi     			  = pshmem_uint32_atomic_fetch_inc_nbi
+#pragma weak shmem_uint64_atomic_fetch_inc_nbi     			  = pshmem_uint64_atomic_fetch_inc_nbi
+#pragma weak shmem_size_atomic_fetch_inc_nbi       			  = pshmem_size_atomic_fetch_inc_nbi
+#pragma weak shmem_ptrdiff_atomic_fetch_inc_nbi    			  = pshmem_ptrdiff_atomic_fetch_inc_nbi
+
+#include "oshmem/shmem/c/profile-defines.h"
+#endif
+
+SHMEM_CTX_TYPE_ATOMIC_FETCH_INC_NBI(_int, int, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_INC_NBI(_long, long, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_INC_NBI(_longlong, long long, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_INC_NBI(_uint, unsigned int, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_INC_NBI(_ulong, unsigned long, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_INC_NBI(_ulonglong, unsigned long long, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_INC_NBI(_int32, int32_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_INC_NBI(_int64, int64_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_INC_NBI(_uint32, uint32_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_INC_NBI(_uint64, uint64_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_INC_NBI(_size, size_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_FETCH_INC_NBI(_ptrdiff, ptrdiff_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_INC_NBI(_int, int, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_INC_NBI(_long, long, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_INC_NBI(_longlong, long long, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_INC_NBI(_uint, unsigned int, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_INC_NBI(_ulong, unsigned long, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_INC_NBI(_ulonglong, unsigned long long, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_INC_NBI(_int32, int32_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_INC_NBI(_int64, int64_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_INC_NBI(_uint32, uint32_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_INC_NBI(_uint64, uint64_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_INC_NBI(_size, size_t, shmem)
+SHMEM_TYPE_ATOMIC_FETCH_INC_NBI(_ptrdiff, ptrdiff_t, shmem)
diff --git a/oshmem/shmem/c/shmem_for.c b/oshmem/shmem/c/shmem_for.c
index 4d0e732004d..c9289f4985e 100644
--- a/oshmem/shmem/c/shmem_for.c
+++ b/oshmem/shmem/c/shmem_for.c
@@ -53,7 +53,8 @@
 #pragma weak shmemx_int64_atomic_fetch_or    = pshmemx_int64_atomic_fetch_or
 #pragma weak shmemx_uint32_atomic_fetch_or   = pshmemx_uint32_atomic_fetch_or
 #pragma weak shmemx_uint64_atomic_fetch_or   = pshmemx_uint64_atomic_fetch_or
-#include "oshmem/shmem/c/profile/defines.h"
+
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 OSHMEM_TYPE_FOP(int, int, shmem, or)
diff --git a/oshmem/shmem/c/shmem_for_nb.c b/oshmem/shmem/c/shmem_for_nb.c
new file mode 100644
index 00000000000..ddbfeabb21b
--- /dev/null
+++ b/oshmem/shmem/c/shmem_for_nb.c
@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2021      NVIDIA Corporation.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "oshmem_config.h"
+
+#include "oshmem/constants.h"
+#include "oshmem/include/shmem.h"
+#include "oshmem/include/shmemx.h"
+
+#include "oshmem/runtime/runtime.h"
+
+#include "oshmem/op/op.h"
+#include "oshmem/mca/atomic/atomic.h"
+
+/*
+ * These routines perform an atomic nonblocking fetch-and-or operation.
+ * The fetch and or routines retrieve the value at address target on PE pe, and update
+ * target with the result of 'or' operation value to the retrieved value. The operation
+ * must be completed without the possibility of another process updating target between
+ * the time of the fetch and the update.
+ */
+#if OSHMEM_PROFILING
+#include "oshmem/include/pshmem.h"
+
+/* Nonblocking Atomic Fetch and Or */
+#pragma weak shmem_ctx_uint_atomic_fetch_or_nbi              = pshmem_ctx_uint_atomic_fetch_or_nbi
+#pragma weak shmem_ctx_ulong_atomic_fetch_or_nbi             = pshmem_ctx_ulong_atomic_fetch_or_nbi
+#pragma weak shmem_ctx_ulonglong_atomic_fetch_or_nbi         = pshmem_ctx_ulonglong_atomic_fetch_or_nbi
+#pragma weak shmem_ctx_int32_atomic_fetch_or_nbi             = pshmem_ctx_int32_atomic_fetch_or_nbi
+#pragma weak shmem_ctx_int64_atomic_fetch_or_nbi             = pshmem_ctx_int64_atomic_fetch_or_nbi
+#pragma weak shmem_ctx_uint32_atomic_fetch_or_nbi            = pshmem_ctx_uint32_atomic_fetch_or_nbi
+#pragma weak shmem_ctx_uint64_atomic_fetch_or_nbi            = pshmem_ctx_uint64_atomic_fetch_or_nbi
+
+#pragma weak shmem_uint_atomic_fetch_or_nbi       			  = pshmem_uint_atomic_fetch_or_nbi
+#pragma weak shmem_ulong_atomic_fetch_or_nbi      			  = pshmem_ulong_atomic_fetch_or_nbi
+#pragma weak shmem_ulonglong_atomic_fetch_or_nbi  			  = pshmem_ulonglong_atomic_fetch_or_nbi
+#pragma weak shmem_int32_atomic_fetch_or_nbi      			  = pshmem_int32_atomic_fetch_or_nbi
+#pragma weak shmem_int64_atomic_fetch_or_nbi      			  = pshmem_int64_atomic_fetch_or_nbi
+#pragma weak shmem_uint32_atomic_fetch_or_nbi     			  = pshmem_uint32_atomic_fetch_or_nbi
+#pragma weak shmem_uint64_atomic_fetch_or_nbi     			  = pshmem_uint64_atomic_fetch_or_nbi
+
+#include "oshmem/shmem/c/profile-defines.h"
+#endif
+
+OSHMEM_TYPE_FOP_NBI(uint, unsigned int, shmem, or)
+OSHMEM_TYPE_FOP_NBI(ulong, unsigned long, shmem, or)
+OSHMEM_TYPE_FOP_NBI(ulonglong, unsigned long long, shmem, or)
+OSHMEM_TYPE_FOP_NBI(int32, int32_t, shmem, or)
+OSHMEM_TYPE_FOP_NBI(int64, int64_t, shmem, or)
+OSHMEM_TYPE_FOP_NBI(uint32, uint32_t, shmem, or)
+OSHMEM_TYPE_FOP_NBI(uint64, uint64_t, shmem, or)
+
+OSHMEM_CTX_TYPE_FOP_NBI(uint, unsigned int, shmem, or)
+OSHMEM_CTX_TYPE_FOP_NBI(ulong, unsigned long, shmem, or)
+OSHMEM_CTX_TYPE_FOP_NBI(ulonglong, unsigned long long, shmem, or)
+OSHMEM_CTX_TYPE_FOP_NBI(int32, int32_t, shmem, or)
+OSHMEM_CTX_TYPE_FOP_NBI(int64, int64_t, shmem, or)
+OSHMEM_CTX_TYPE_FOP_NBI(uint32, uint32_t, shmem, or)
+OSHMEM_CTX_TYPE_FOP_NBI(uint64, uint64_t, shmem, or)
+
diff --git a/oshmem/shmem/c/shmem_free.c b/oshmem/shmem/c/shmem_free.c
index 22da004589b..a03d8bf5119 100644
--- a/oshmem/shmem/c/shmem_free.c
+++ b/oshmem/shmem/c/shmem_free.c
@@ -24,7 +24,7 @@
 #include "oshmem/include/pshmem.h"
 #pragma weak shmem_free = pshmem_free
 #pragma weak shfree = pshfree
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 static inline void _shfree(void* ptr);
diff --git a/oshmem/shmem/c/shmem_fxor.c b/oshmem/shmem/c/shmem_fxor.c
index 41fe2249c6b..a1d977f6ce0 100644
--- a/oshmem/shmem/c/shmem_fxor.c
+++ b/oshmem/shmem/c/shmem_fxor.c
@@ -53,7 +53,8 @@
 #pragma weak shmemx_int64_atomic_fetch_xor    = pshmemx_int64_atomic_fetch_xor
 #pragma weak shmemx_uint32_atomic_fetch_xor   = pshmemx_uint32_atomic_fetch_xor
 #pragma weak shmemx_uint64_atomic_fetch_xor   = pshmemx_uint64_atomic_fetch_xor
-#include "oshmem/shmem/c/profile/defines.h"
+
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 OSHMEM_TYPE_FOP(int, int, shmem, xor)
diff --git a/oshmem/shmem/c/shmem_fxor_nb.c b/oshmem/shmem/c/shmem_fxor_nb.c
new file mode 100644
index 00000000000..2a40265f734
--- /dev/null
+++ b/oshmem/shmem/c/shmem_fxor_nb.c
@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2021      NVIDIA Corporation.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "oshmem_config.h"
+
+#include "oshmem/constants.h"
+#include "oshmem/include/shmem.h"
+#include "oshmem/include/shmemx.h"
+
+#include "oshmem/runtime/runtime.h"
+
+#include "oshmem/op/op.h"
+#include "oshmem/mca/atomic/atomic.h"
+
+/*
+ * These routines perform an atomic nonblocking fetch-and-xor operation.
+ * The fetch and xor routines retrieve the value at address target on PE pe, and update
+ * target with the result of 'xor' operation value to the retrieved value. The operation
+ * must be completed without the possibility of another process updating target between
+ * the time of the fetch and the update.
+ */
+#if OSHMEM_PROFILING
+#include "oshmem/include/pshmem.h"
+
+/* Nonblocking Atomic Fetch and Xor */
+#pragma weak shmem_ctx_uint_atomic_fetch_xor_nbi              = pshmem_ctx_uint_atomic_fetch_xor_nbi
+#pragma weak shmem_ctx_ulong_atomic_fetch_xor_nbi             = pshmem_ctx_ulong_atomic_fetch_xor_nbi
+#pragma weak shmem_ctx_ulonglong_atomic_fetch_xor_nbi         = pshmem_ctx_ulonglong_atomic_fetch_xor_nbi
+#pragma weak shmem_ctx_int32_atomic_fetch_xor_nbi             = pshmem_ctx_int32_atomic_fetch_xor_nbi
+#pragma weak shmem_ctx_int64_atomic_fetch_xor_nbi             = pshmem_ctx_int64_atomic_fetch_xor_nbi
+#pragma weak shmem_ctx_uint32_atomic_fetch_xor_nbi            = pshmem_ctx_uint32_atomic_fetch_xor_nbi
+#pragma weak shmem_ctx_uint64_atomic_fetch_xor_nbi            = pshmem_ctx_uint64_atomic_fetch_xor_nbi
+
+#pragma weak shmem_uint_atomic_fetch_xor_nbi       			  = pshmem_uint_atomic_fetch_xor_nbi
+#pragma weak shmem_ulong_atomic_fetch_xor_nbi      			  = pshmem_ulong_atomic_fetch_xor_nbi
+#pragma weak shmem_ulonglong_atomic_fetch_xor_nbi  			  = pshmem_ulonglong_atomic_fetch_xor_nbi
+#pragma weak shmem_int32_atomic_fetch_xor_nbi      			  = pshmem_int32_atomic_fetch_xor_nbi
+#pragma weak shmem_int64_atomic_fetch_xor_nbi      			  = pshmem_int64_atomic_fetch_xor_nbi
+#pragma weak shmem_uint32_atomic_fetch_xor_nbi     			  = pshmem_uint32_atomic_fetch_xor_nbi
+#pragma weak shmem_uint64_atomic_fetch_xor_nbi     			  = pshmem_uint64_atomic_fetch_xor_nbi
+
+#include "oshmem/shmem/c/profile-defines.h"
+#endif
+
+OSHMEM_TYPE_FOP_NBI(uint, unsigned int, shmem, xor)
+OSHMEM_TYPE_FOP_NBI(ulong, unsigned long, shmem, xor)
+OSHMEM_TYPE_FOP_NBI(ulonglong, unsigned long long, shmem, xor)
+OSHMEM_TYPE_FOP_NBI(int32, int32_t, shmem, xor)
+OSHMEM_TYPE_FOP_NBI(int64, int64_t, shmem, xor)
+OSHMEM_TYPE_FOP_NBI(uint32, uint32_t, shmem, xor)
+OSHMEM_TYPE_FOP_NBI(uint64, uint64_t, shmem, xor)
+
+OSHMEM_CTX_TYPE_FOP_NBI(uint, unsigned int, shmem, xor)
+OSHMEM_CTX_TYPE_FOP_NBI(ulong, unsigned long, shmem, xor)
+OSHMEM_CTX_TYPE_FOP_NBI(ulonglong, unsigned long long, shmem, xor)
+OSHMEM_CTX_TYPE_FOP_NBI(int32, int32_t, shmem, xor)
+OSHMEM_CTX_TYPE_FOP_NBI(int64, int64_t, shmem, xor)
+OSHMEM_CTX_TYPE_FOP_NBI(uint32, uint32_t, shmem, xor)
+OSHMEM_CTX_TYPE_FOP_NBI(uint64, uint64_t, shmem, xor)
+
diff --git a/oshmem/shmem/c/shmem_g.c b/oshmem/shmem/c/shmem_g.c
index aa96afaba68..816b09e7f63 100644
--- a/oshmem/shmem/c/shmem_g.c
+++ b/oshmem/shmem/c/shmem_g.c
@@ -111,7 +111,7 @@
 #pragma weak shmemx_int16_g = pshmemx_int16_g
 #pragma weak shmemx_int32_g = pshmemx_int32_g
 #pragma weak shmemx_int64_g = pshmemx_int64_g
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 SHMEM_CTX_TYPE_G(_char, char, shmem)
diff --git a/oshmem/shmem/c/shmem_get.c b/oshmem/shmem/c/shmem_get.c
index 452557fdd6a..eac21d5563c 100644
--- a/oshmem/shmem/c/shmem_get.c
+++ b/oshmem/shmem/c/shmem_get.c
@@ -120,7 +120,7 @@
 #pragma weak shmem_get32 = pshmem_get32
 #pragma weak shmem_get64 = pshmem_get64
 #pragma weak shmem_get128 = pshmem_get128
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 SHMEM_CTX_TYPE_GET(_char, char)
diff --git a/oshmem/shmem/c/shmem_get_nb.c b/oshmem/shmem/c/shmem_get_nb.c
index 877b7ac0201..c89f7df587e 100644
--- a/oshmem/shmem/c/shmem_get_nb.c
+++ b/oshmem/shmem/c/shmem_get_nb.c
@@ -120,7 +120,7 @@
 #pragma weak shmem_get64_nbi = pshmem_get64_nbi
 #pragma weak shmem_get128_nbi = pshmem_get128_nbi
 #pragma weak shmem_getmem_nbi = pshmem_getmem_nbi
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 SHMEM_CTX_TYPE_GET_NB(_char, char)
diff --git a/oshmem/shmem/c/shmem_global_exit.c b/oshmem/shmem/c/shmem_global_exit.c
index c71ee62b636..361928266a8 100644
--- a/oshmem/shmem/c/shmem_global_exit.c
+++ b/oshmem/shmem/c/shmem_global_exit.c
@@ -13,11 +13,12 @@
 
 #include "oshmem/include/shmem.h"
 #include "oshmem/runtime/runtime.h"
+#include "oshmem/proc/proc.h"
 
 #if OSHMEM_PROFILING
 #include "oshmem/include/pshmem.h"
 #pragma weak shmem_global_exit = pshmem_global_exit
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 extern int oshmem_shmem_inglobalexit;
diff --git a/oshmem/shmem/c/shmem_iget.c b/oshmem/shmem/c/shmem_iget.c
index eb4d31cf802..679f73f5d67 100644
--- a/oshmem/shmem/c/shmem_iget.c
+++ b/oshmem/shmem/c/shmem_iget.c
@@ -123,7 +123,7 @@
 #pragma weak shmem_iget32 = pshmem_iget32
 #pragma weak shmem_iget64 = pshmem_iget64
 #pragma weak shmem_iget128 = pshmem_iget128
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 SHMEM_CTX_TYPE_IGET(_char, char)
diff --git a/oshmem/shmem/c/shmem_inc.c b/oshmem/shmem/c/shmem_inc.c
index 8c44c0258d1..8389e19e35c 100644
--- a/oshmem/shmem/c/shmem_inc.c
+++ b/oshmem/shmem/c/shmem_inc.c
@@ -66,12 +66,24 @@
 #pragma weak shmem_ctx_uint_atomic_inc = pshmem_ctx_uint_atomic_inc
 #pragma weak shmem_ctx_ulong_atomic_inc = pshmem_ctx_ulong_atomic_inc
 #pragma weak shmem_ctx_ulonglong_atomic_inc = pshmem_ctx_ulonglong_atomic_inc
+#pragma weak shmem_ctx_int32_atomic_inc = pshmem_ctx_int32_atomic_inc
+#pragma weak shmem_ctx_int64_atomic_inc = pshmem_ctx_int64_atomic_inc
+#pragma weak shmem_ctx_uint32_atomic_inc = pshmem_ctx_uint32_atomic_inc
+#pragma weak shmem_ctx_uint64_atomic_inc = pshmem_ctx_uint64_atomic_inc
+#pragma weak shmem_ctx_size_atomic_inc = pshmem_ctx_size_atomic_inc
+#pragma weak shmem_ctx_ptrdiff_atomic_inc = pshmem_ctx_ptrdiff_atomic_inc
 
 #pragma weak shmem_int_atomic_inc = pshmem_int_atomic_inc
 #pragma weak shmem_long_atomic_inc = pshmem_long_atomic_inc
 #pragma weak shmem_longlong_atomic_inc = pshmem_longlong_atomic_inc
 #pragma weak shmem_uint_atomic_inc = pshmem_uint_atomic_inc
 #pragma weak shmem_ulong_atomic_inc = pshmem_ulong_atomic_inc
+#pragma weak shmem_int32_atomic_inc = pshmem_int32_atomic_inc
+#pragma weak shmem_int64_atomic_inc = pshmem_int64_atomic_inc
+#pragma weak shmem_uint32_atomic_inc = pshmem_uint32_atomic_inc
+#pragma weak shmem_uint64_atomic_inc = pshmem_uint64_atomic_inc
+#pragma weak shmem_size_atomic_inc = pshmem_size_atomic_inc
+#pragma weak shmem_ptrdiff_atomic_inc = pshmem_ptrdiff_atomic_inc
 #pragma weak shmem_ulonglong_atomic_inc = pshmem_ulonglong_atomic_inc
 
 #pragma weak shmem_int_inc = pshmem_int_inc
@@ -80,7 +92,7 @@
 
 #pragma weak shmemx_int32_inc = pshmemx_int32_inc
 #pragma weak shmemx_int64_inc = pshmemx_int64_inc
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 SHMEM_CTX_TYPE_ATOMIC_INC(_int, int, shmem)
@@ -89,12 +101,24 @@ SHMEM_CTX_TYPE_ATOMIC_INC(_longlong, long long, shmem)
 SHMEM_CTX_TYPE_ATOMIC_INC(_uint, unsigned int, shmem)
 SHMEM_CTX_TYPE_ATOMIC_INC(_ulong, unsigned long, shmem)
 SHMEM_CTX_TYPE_ATOMIC_INC(_ulonglong, unsigned long long, shmem)
+SHMEM_CTX_TYPE_ATOMIC_INC(_int32, int32_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_INC(_int64, int64_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_INC(_uint32, uint32_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_INC(_uint64, uint64_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_INC(_size, size_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_INC(_ptrdiff, ptrdiff_t, shmem)
 SHMEM_TYPE_ATOMIC_INC(_int, int, shmem)
 SHMEM_TYPE_ATOMIC_INC(_long, long, shmem)
 SHMEM_TYPE_ATOMIC_INC(_longlong, long long, shmem)
 SHMEM_TYPE_ATOMIC_INC(_uint, unsigned int, shmem)
 SHMEM_TYPE_ATOMIC_INC(_ulong, unsigned long, shmem)
 SHMEM_TYPE_ATOMIC_INC(_ulonglong, unsigned long long, shmem)
+SHMEM_TYPE_ATOMIC_INC(_int32, int32_t, shmem)
+SHMEM_TYPE_ATOMIC_INC(_int64, int64_t, shmem)
+SHMEM_TYPE_ATOMIC_INC(_uint32, uint32_t, shmem)
+SHMEM_TYPE_ATOMIC_INC(_uint64, uint64_t, shmem)
+SHMEM_TYPE_ATOMIC_INC(_size, size_t, shmem)
+SHMEM_TYPE_ATOMIC_INC(_ptrdiff, ptrdiff_t, shmem)
 
 #define SHMEM_TYPE_INC(type_name, type, prefix)                     \
     void prefix##type_name##_inc(type *target, int pe)              \
diff --git a/oshmem/shmem/c/shmem_init.c b/oshmem/shmem/c/shmem_init.c
index 0b8330d8525..16e8731b2f1 100644
--- a/oshmem/shmem/c/shmem_init.c
+++ b/oshmem/shmem/c/shmem_init.c
@@ -27,7 +27,7 @@
 #pragma weak shmem_init = pshmem_init
 #pragma weak shmem_init_thread = pshmem_init_thread
 #pragma weak start_pes = pstart_pes
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 extern int oshmem_shmem_globalexit_status;
diff --git a/oshmem/shmem/c/shmem_iput.c b/oshmem/shmem/c/shmem_iput.c
index 0e83b6c8495..08f93f79ea1 100644
--- a/oshmem/shmem/c/shmem_iput.c
+++ b/oshmem/shmem/c/shmem_iput.c
@@ -124,7 +124,7 @@
 #pragma weak shmem_iput32 = pshmem_iput32
 #pragma weak shmem_iput64 = pshmem_iput64
 #pragma weak shmem_iput128 = pshmem_iput128
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 SHMEM_CTX_TYPE_IPUT(_char, char)
diff --git a/oshmem/shmem/c/shmem_mcs_lock.c b/oshmem/shmem/c/shmem_mcs_lock.c
new file mode 100644
index 00000000000..3d7e97ee7b4
--- /dev/null
+++ b/oshmem/shmem/c/shmem_mcs_lock.c
@@ -0,0 +1,239 @@
+/*
+ * Copyright (c) 2023      NVIDIA Corporation.
+ *                         All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "oshmem_config.h"
+
+#include "oshmem/constants.h"
+#include "oshmem/include/shmem.h"
+#include "oshmem/runtime/params.h"
+#include "oshmem/runtime/runtime.h"
+#include <stdlib.h>
+#include <memory.h>
+
+#include "oshmem/shmem/shmem_api_logger.h"
+#include "oshmem/shmem/shmem_lock.h"
+#include "oshmem/mca/memheap/memheap.h"
+#include "oshmem/mca/memheap/base/base.h"
+#include "oshmem/mca/atomic/atomic.h"
+
+#define OPAL_BITWISE_SIZEOF_LONG (SIZEOF_LONG * 8)
+
+
+/** Use basic MCS distributed lock algorithm for lock */
+struct shmem_mcs_lock {
+    /** has meaning only on MCSQ_TAIL OWNER */
+    int tail;
+    /** It has meaning on all PEs */
+    /** The next pointer is a combination of the PE ID and wait signal */
+    int next;
+};
+typedef struct shmem_mcs_lock shmem_mcs_lock_t;
+
+#define SHMEM_MCSL_TAIL_OWNER(lock_ptr)\
+    (((uintptr_t)(lock_ptr) / sizeof(long)) % shmem_n_pes())
+
+#define SHMEM_MCSL_NEXT_MASK            0x7FFFFFFFU
+#define SHMEM_MCSL_SIGNAL_MASK          0x80000000U /** Wait signal mask */
+#define SHMEM_MCSL_NEXT(lock_val)       ((lock_val) & SHMEM_MCSL_NEXT_MASK)
+/** Improve readability */
+#define SHMEM_MCSL_GET_PE(tail_val)     ((tail_val) & SHMEM_MCSL_NEXT_MASK)
+#define SHMEM_MCSL_SIGNAL(lock_val)     ((lock_val) & SHMEM_MCSL_SIGNAL_MASK)
+#define SHMEM_MCSL_SET_SIGNAL(lock_val) ((lock_val) | SHMEM_MCSL_SIGNAL_MASK)
+
+void
+_shmem_mcs_set_lock(long *lockp)
+{
+    shmem_mcs_lock_t *lock = (shmem_mcs_lock_t *) lockp;
+    int mcs_tail_owner     = SHMEM_MCSL_TAIL_OWNER(lock);
+    int new_tail_req       = 0;
+    int *tail              = &(lock->tail);
+    int *next              = &(lock->next);
+    int my_pe              = shmem_my_pe();
+    int curr               = 0;
+    int out_value          = 0;
+    int prev_tail          = 0;
+    int prev_tailpe        = 0;
+    int tval               = 0;
+    int tmp_val            = 0;
+    int retv               = 0;
+    uint64_t value_tmp     = 0;
+
+    RUNTIME_CHECK_INIT();
+    /**
+     *  Initializing next pointer to next mask
+     *  Done atomically to avoid races as NEXT pointer
+     *  can be modified by other PEs while acquiring or
+     *  releasing it.
+     */
+    /**
+     * Can make this to be shmem_atomic_set to be safe
+     * in non-cc architectures
+     * has an impact on performance
+     */
+    value_tmp = SHMEM_MCSL_NEXT_MASK;
+    out_value = SHMEM_MCSL_NEXT_MASK;
+    retv = MCA_ATOMIC_CALL(swap(oshmem_ctx_default, (void*)next,
+                                (void*)&out_value, value_tmp,
+                                sizeof(int), my_pe));
+    RUNTIME_CHECK_RC(retv);
+    MCA_SPML_CALL(quiet(oshmem_ctx_default));
+
+    /** Signal for setting lock */
+    new_tail_req = SHMEM_MCSL_SET_SIGNAL(my_pe);
+    /**
+     * Swap and make me the new tail and update in tail owner
+     * Get the previous tail PE.
+     */
+    retv = MCA_ATOMIC_CALL(swap(oshmem_ctx_default, (void *)tail,
+                                (void*)&prev_tail,
+                                OSHMEM_ATOMIC_PTR_2_INT(&new_tail_req,
+                                                        sizeof(new_tail_req)),
+                                sizeof(int), mcs_tail_owner));
+    RUNTIME_CHECK_RC(retv);
+
+    prev_tailpe = SHMEM_MCSL_GET_PE(prev_tail);
+    if (SHMEM_MCSL_SIGNAL(prev_tail)) {
+        /**
+         * Someone else has got the lock before this PE
+         * Adding this PE to the previous tail PE's Next pointer
+         * Substract the SIGNAL Bit to avoid changing it.
+         */
+        tmp_val = my_pe - SHMEM_MCSL_NEXT_MASK;
+        retv = MCA_ATOMIC_CALL(add(oshmem_ctx_default, (void*)next, tmp_val,
+                                   sizeof(int), prev_tailpe));
+        RUNTIME_CHECK_RC(retv);
+        /**
+         * This value to be changed eventually by predecessor
+         * when its lock is released.
+         * Need to be done atomically to avoid any races where
+         * next pointer is modified by another PE acquiring or
+         * releasing this.
+         */
+        retv = MCA_ATOMIC_CALL(add(oshmem_ctx_default, (void *)next,
+                                   SHMEM_MCSL_SIGNAL_MASK, sizeof(int),
+                                   my_pe));
+        RUNTIME_CHECK_RC(retv);
+        MCA_SPML_CALL(quiet(oshmem_ctx_default));
+        /**  Wait for predecessor release lock to this PE signal to false. */
+        retv = MCA_ATOMIC_CALL(fadd(oshmem_ctx_default, (void*)next,
+                                    (void*)&curr, tval, sizeof(int), my_pe));
+        RUNTIME_CHECK_RC(retv);
+
+        while (SHMEM_MCSL_SIGNAL(curr)) {
+            retv = MCA_SPML_CALL(wait((void*)next, SHMEM_CMP_NE,
+                                      (void*)&curr, SHMEM_INT));
+            RUNTIME_CHECK_RC(retv);
+            retv = MCA_ATOMIC_CALL(fadd(oshmem_ctx_default, (void*)next,
+                                        (void*)&curr, tval, sizeof(int),
+                                        my_pe));
+            RUNTIME_CHECK_RC(retv);
+        }
+    }
+/** else.. this pe has got the lock as no one else had it */
+}
+
+void
+_shmem_mcs_clear_lock(long *lockp)
+{
+    shmem_mcs_lock_t *lock = (shmem_mcs_lock_t *) lockp;
+    int mcs_tail_owner     = SHMEM_MCSL_TAIL_OWNER(lock);
+    int *tail              = &(lock->tail);
+    int *next              = &(lock->next);
+    int my_pe              = shmem_my_pe();
+    int next_value         = 0;
+    int swap_cond          = 0;
+    int prev_value         = 0;
+    int tval               = 0;
+    int val_tmp            = 0;
+    int nmask              = 0;
+    int a_val              = 0;
+    int retv               = 0;
+
+    /**
+     * Can make atomic fetch to be safe in non-cc architectures
+     * Has impact on performance
+     */
+    retv = MCA_ATOMIC_CALL(fadd(oshmem_ctx_default, (void*)next,
+                                (void*)&next_value, tval, sizeof(int),
+                                my_pe));
+    RUNTIME_CHECK_RC(retv);
+    MCA_SPML_CALL(quiet(oshmem_ctx_default));
+
+    if (next_value == SHMEM_MCSL_NEXT_MASK) {
+        swap_cond = SHMEM_MCSL_SET_SIGNAL(my_pe);
+        retv = MCA_ATOMIC_CALL(cswap(oshmem_ctx_default,
+                                     (void *)tail, (uint64_t *)&(prev_value),
+                                     OSHMEM_ATOMIC_PTR_2_INT(&swap_cond,
+                                     sizeof(swap_cond)),
+                                     OSHMEM_ATOMIC_PTR_2_INT(&val_tmp,
+                                     sizeof(val_tmp)), sizeof(int),
+                                     mcs_tail_owner));
+        RUNTIME_CHECK_RC(retv);
+
+        /** I am the tail.. and lock is released */
+        if (prev_value == swap_cond) {
+            return;
+        }
+        /**
+         * I am not the tail, another PE maybe racing to acquire lock,
+         * let them complete setting themselves as our next
+         */
+        nmask = SHMEM_MCSL_NEXT_MASK;
+        while(next_value == nmask) {
+            retv = MCA_SPML_CALL(wait((void*)next, SHMEM_CMP_NE,
+                                      (void*)&nmask, SHMEM_INT));
+            RUNTIME_CHECK_RC(retv);
+            retv = MCA_ATOMIC_CALL(fadd(oshmem_ctx_default, (void*)next,
+                                        (void*)&next_value, tval,
+                                        sizeof(int), my_pe));
+            RUNTIME_CHECK_RC(retv);
+        }
+    }
+    /** There is a successor release lock to the successor */
+    a_val = SHMEM_MCSL_SIGNAL_MASK;
+    retv = MCA_ATOMIC_CALL(add(oshmem_ctx_default,
+                               (void *)next, a_val, sizeof(a_val),
+                               SHMEM_MCSL_NEXT(next_value)));
+    RUNTIME_CHECK_RC(retv);
+    MCA_SPML_CALL(quiet(oshmem_ctx_default));
+}
+
+int
+_shmem_mcs_test_lock(long *lockp)
+{
+    shmem_mcs_lock_t *lock = (shmem_mcs_lock_t *) lockp;
+    int mcs_tail_owner     = SHMEM_MCSL_TAIL_OWNER(lock);
+    int new_tail_req       = 0;
+    int prev_tail          = 0;
+    int tmp_cond           = 0;
+    int *tail              = &(lock->tail);
+    int *next              = &(lock->next);
+    int my_pe              = shmem_my_pe();
+    int retv               = 0;
+
+    /** Initializing next pointer to next mask */
+    *next = SHMEM_MCSL_NEXT_MASK;
+
+    /** Signal for setting lock */
+    new_tail_req = SHMEM_MCSL_SET_SIGNAL(my_pe);
+
+    /** Check if previously cleared before swapping */
+    retv = MCA_ATOMIC_CALL(cswap(oshmem_ctx_default,
+                                 (void *)tail, (uint64_t *)&(prev_tail),
+                                 OSHMEM_ATOMIC_PTR_2_INT(&tmp_cond,
+                                                         sizeof(tmp_cond)),
+                                 OSHMEM_ATOMIC_PTR_2_INT(&new_tail_req,
+                                                         sizeof(new_tail_req)),
+                                 sizeof(int), mcs_tail_owner));
+    RUNTIME_CHECK_RC(retv);
+
+    return (0 != prev_tail);
+}
diff --git a/oshmem/shmem/c/shmem_or.c b/oshmem/shmem/c/shmem_or.c
index 7bdbb59ad34..3d440cc7328 100644
--- a/oshmem/shmem/c/shmem_or.c
+++ b/oshmem/shmem/c/shmem_or.c
@@ -51,7 +51,7 @@
 #pragma weak shmemx_int64_atomic_or    = pshmemx_int64_atomic_or
 #pragma weak shmemx_uint32_atomic_or   = pshmemx_uint32_atomic_or
 #pragma weak shmemx_uint64_atomic_or   = pshmemx_uint64_atomic_or
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 OSHMEM_TYPE_OP(int, int, shmem, or)
diff --git a/oshmem/shmem/c/shmem_p.c b/oshmem/shmem/c/shmem_p.c
index 974333a4361..ae34bcefd32 100644
--- a/oshmem/shmem/c/shmem_p.c
+++ b/oshmem/shmem/c/shmem_p.c
@@ -112,7 +112,8 @@
 #pragma weak shmemx_int16_p = pshmemx_int16_p
 #pragma weak shmemx_int32_p = pshmemx_int32_p
 #pragma weak shmemx_int64_p = pshmemx_int64_p
-#include "oshmem/shmem/c/profile/defines.h"
+
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 SHMEM_CTX_TYPE_P(_char, char, shmem)
diff --git a/oshmem/shmem/c/shmem_pcontrol.c b/oshmem/shmem/c/shmem_pcontrol.c
new file mode 100644
index 00000000000..b87a2829ced
--- /dev/null
+++ b/oshmem/shmem/c/shmem_pcontrol.c
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2021      NVIDIA Corporation.
+ *                         All rights reserved.
+ * Copyright (c) 2019      Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "oshmem_config.h"
+
+#include "oshmem/constants.h"
+#include "oshmem/include/shmem.h"
+
+#include "oshmem/runtime/runtime.h"
+
+#include "oshmem/mca/scoll/scoll.h"
+
+#include "oshmem/proc/proc.h"
+
+
+#if OSHMEM_PROFILING
+#include "oshmem/include/pshmem.h"
+
+#pragma weak shmem_pcontrol    = pshmem_pcontrol
+#include "oshmem/shmem/c/profile-defines.h"
+#endif
+
+
+
+
+void shmem_pcontrol(int level, ...)
+{
+    return ;
+}
+
diff --git a/oshmem/shmem/c/shmem_pe_accessible.c b/oshmem/shmem/c/shmem_pe_accessible.c
index 3e6923a0bc8..923d781df03 100644
--- a/oshmem/shmem/c/shmem_pe_accessible.c
+++ b/oshmem/shmem/c/shmem_pe_accessible.c
@@ -22,7 +22,7 @@
 #if OSHMEM_PROFILING
 #include "oshmem/include/pshmem.h"
 #pragma weak shmem_pe_accessible = pshmem_pe_accessible
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 int shmem_pe_accessible(int pe)
diff --git a/oshmem/shmem/c/shmem_ptr.c b/oshmem/shmem/c/shmem_ptr.c
index 7bfb6d014f6..f69a83766da 100644
--- a/oshmem/shmem/c/shmem_ptr.c
+++ b/oshmem/shmem/c/shmem_ptr.c
@@ -26,12 +26,11 @@
 #if OSHMEM_PROFILING
 #include "oshmem/include/pshmem.h"
 #pragma weak shmem_ptr = pshmem_ptr
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 void *shmem_ptr(const void *dst_addr, int pe)
 {
-    ompi_proc_t *proc;
     sshmem_mkey_t *mkey;
     int i;
     void *rva;
@@ -46,8 +45,7 @@ void *shmem_ptr(const void *dst_addr, int pe)
     }
 
     /* The memory must be on the local node */
-    proc = oshmem_proc_group_find(oshmem_group_all, pe);
-    if (!OPAL_PROC_ON_LOCAL_NODE(proc->super.proc_flags)) {
+    if (!oshmem_proc_on_local_node(pe)) {
         return NULL;
     }
 
diff --git a/oshmem/shmem/c/shmem_put.c b/oshmem/shmem/c/shmem_put.c
index 98b6f24c7b8..16306d6d97e 100644
--- a/oshmem/shmem/c/shmem_put.c
+++ b/oshmem/shmem/c/shmem_put.c
@@ -123,7 +123,7 @@
 #pragma weak shmem_put32 = pshmem_put32
 #pragma weak shmem_put64 = pshmem_put64
 #pragma weak shmem_put128 = pshmem_put128
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 SHMEM_CTX_TYPE_PUT(_char, char)
diff --git a/oshmem/shmem/c/shmem_put_nb.c b/oshmem/shmem/c/shmem_put_nb.c
index bf63130e236..89e4bf18240 100644
--- a/oshmem/shmem/c/shmem_put_nb.c
+++ b/oshmem/shmem/c/shmem_put_nb.c
@@ -127,7 +127,7 @@
 #pragma weak shmem_ctx_put64_nbi = pshmem_ctx_put64_nbi
 #pragma weak shmem_ctx_put128_nbi = pshmem_ctx_put128_nbi
 #pragma weak shmem_ctx_putmem_nbi = pshmem_ctx_putmem_nbi
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 SHMEM_CTX_TYPE_PUT_NB(_char, char)
diff --git a/oshmem/shmem/c/shmem_put_signal.c b/oshmem/shmem/c/shmem_put_signal.c
new file mode 100644
index 00000000000..4c50dc10ede
--- /dev/null
+++ b/oshmem/shmem/c/shmem_put_signal.c
@@ -0,0 +1,230 @@
+/*
+ * Copyright (c) 2021      NVIDIA Corporation.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "oshmem_config.h"
+
+#include "oshmem/constants.h"
+#include "oshmem/include/shmem.h"
+
+#include "oshmem/runtime/runtime.h"
+
+#include "oshmem/mca/spml/spml.h"
+
+#define DO_SHMEM_TYPE_PUT_SIGNAL(ctx, type, dest, source, nelems, sig_addr, signal, sig_op, pe) do {  \
+        int rc = OSHMEM_SUCCESS;                                    \
+        size_t size = 0;                                            \
+                                                                    \
+        RUNTIME_CHECK_INIT();                                       \
+        RUNTIME_CHECK_PE(pe);                                       \
+        RUNTIME_CHECK_ADDR(dest);                                   \
+                                                                    \
+        size = nelems * sizeof(type);                               \
+        rc = MCA_SPML_CALL(put_signal(                              \
+            ctx,                                                    \
+            (void*)dest,                                            \
+            size,                                                   \
+            (void*)source,                                          \
+            sig_addr, signal, sig_op, pe));                         \
+        RUNTIME_CHECK_RC(rc);                                       \
+    } while (0)
+
+#define SHMEM_CTX_TYPE_PUT_SIGNAL(type_name, type)                                                    \
+    void shmem_ctx##type_name##_put_signal(shmem_ctx_t ctx, type *dest, const type *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe)\
+    {                                                                                                 \
+        DO_SHMEM_TYPE_PUT_SIGNAL(ctx, type, dest, source, nelems, sig_addr, signal, sig_op, pe);      \
+        return ;                                                                                      \
+    }
+
+#define SHMEM_TYPE_PUT_SIGNAL(type_name, type)                            \
+    void shmem##type_name##_put_signal(type *dest, const type *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe)\
+    {                                                                     \
+        DO_SHMEM_TYPE_PUT_SIGNAL(oshmem_ctx_default, type, dest,          \
+                          source, nelems, sig_addr, signal, sig_op, pe);  \
+        return ;                                                          \
+    }
+
+#if OSHMEM_PROFILING
+#include "oshmem/include/pshmem.h"
+#pragma weak shmem_ctx_char_put_signal             = pshmem_ctx_char_put_signal
+#pragma weak shmem_ctx_short_put_signal            = pshmem_ctx_short_put_signal
+#pragma weak shmem_ctx_int_put_signal              = pshmem_ctx_int_put_signal
+#pragma weak shmem_ctx_long_put_signal             = pshmem_ctx_long_put_signal
+#pragma weak shmem_ctx_float_put_signal            = pshmem_ctx_float_put_signal
+#pragma weak shmem_ctx_double_put_signal           = pshmem_ctx_double_put_signal
+#pragma weak shmem_ctx_longlong_put_signal         = pshmem_ctx_longlong_put_signal
+#pragma weak shmem_ctx_schar_put_signal            = pshmem_ctx_schar_put_signal
+#pragma weak shmem_ctx_uchar_put_signal            = pshmem_ctx_uchar_put_signal
+#pragma weak shmem_ctx_ushort_put_signal           = pshmem_ctx_ushort_put_signal
+#pragma weak shmem_ctx_uint_put_signal             = pshmem_ctx_uint_put_signal
+#pragma weak shmem_ctx_ulong_put_signal            = pshmem_ctx_ulong_put_signal
+#pragma weak shmem_ctx_ulonglong_put_signal        = pshmem_ctx_ulonglong_put_signal
+#pragma weak shmem_ctx_longdouble_put_signal       = pshmem_ctx_longdouble_put_signal
+#pragma weak shmem_ctx_int8_put_signal             = pshmem_ctx_int8_put_signal
+#pragma weak shmem_ctx_int16_put_signal            = pshmem_ctx_int16_put_signal
+#pragma weak shmem_ctx_int32_put_signal            = pshmem_ctx_int32_put_signal
+#pragma weak shmem_ctx_int64_put_signal            = pshmem_ctx_int64_put_signal
+#pragma weak shmem_ctx_uint8_put_signal            = pshmem_ctx_uint8_put_signal
+#pragma weak shmem_ctx_uint16_put_signal           = pshmem_ctx_uint16_put_signal
+#pragma weak shmem_ctx_uint32_put_signal           = pshmem_ctx_uint32_put_signal
+#pragma weak shmem_ctx_uint64_put_signal           = pshmem_ctx_uint64_put_signal
+#pragma weak shmem_ctx_size_put_signal             = pshmem_ctx_size_put_signal
+#pragma weak shmem_ctx_ptrdiff_put_signal          = pshmem_ctx_ptrdiff_put_signal
+
+#pragma weak shmem_char_put_signal                 = pshmem_char_put_signal
+#pragma weak shmem_short_put_signal                = pshmem_short_put_signal
+#pragma weak shmem_int_put_signal                  = pshmem_int_put_signal
+#pragma weak shmem_long_put_signal                 = pshmem_long_put_signal
+#pragma weak shmem_float_put_signal                = pshmem_float_put_signal
+#pragma weak shmem_double_put_signal               = pshmem_double_put_signal
+#pragma weak shmem_longlong_put_signal             = pshmem_longlong_put_signal
+#pragma weak shmem_schar_put_signal                = pshmem_schar_put_signal
+#pragma weak shmem_uchar_put_signal                = pshmem_uchar_put_signal
+#pragma weak shmem_ushort_put_signal               = pshmem_ushort_put_signal
+#pragma weak shmem_uint_put_signal                 = pshmem_uint_put_signal
+#pragma weak shmem_ulong_put_signal                = pshmem_ulong_put_signal
+#pragma weak shmem_ulonglong_put_signal            = pshmem_ulonglong_put_signal
+#pragma weak shmem_longdouble_put_signal           = pshmem_longdouble_put_signal
+#pragma weak shmem_int8_put_signal                 = pshmem_int8_put_signal
+#pragma weak shmem_int16_put_signal                = pshmem_int16_put_signal
+#pragma weak shmem_int32_put_signal                = pshmem_int32_put_signal
+#pragma weak shmem_int64_put_signal                = pshmem_int64_put_signal
+#pragma weak shmem_uint8_put_signal                = pshmem_uint8_put_signal
+#pragma weak shmem_uint16_put_signal               = pshmem_uint16_put_signal
+#pragma weak shmem_uint32_put_signal               = pshmem_uint32_put_signal
+#pragma weak shmem_uint64_put_signal               = pshmem_uint64_put_signal
+#pragma weak shmem_size_put_signal                 = pshmem_size_put_signal
+#pragma weak shmem_ptrdiff_put_signal              = pshmem_ptrdiff_put_signal
+
+#pragma weak shmem_put8_signal                     = pshmem_put8_signal
+#pragma weak shmem_put16_signal                    = pshmem_put16_signal
+#pragma weak shmem_put32_signal                    = pshmem_put32_signal
+#pragma weak shmem_put64_signal                    = pshmem_put64_signal
+#pragma weak shmem_put128_signal                   = pshmem_put128_signal
+
+#pragma weak shmem_ctx_put8_signal                 = pshmem_ctx_put8_signal
+#pragma weak shmem_ctx_put16_signal                = pshmem_ctx_put16_signal
+#pragma weak shmem_ctx_put32_signal                = pshmem_ctx_put32_signal
+#pragma weak shmem_ctx_put64_signal                = pshmem_ctx_put64_signal
+#pragma weak shmem_ctx_put128_signal               = pshmem_ctx_put128_signal
+
+#pragma weak shmem_putmem_signal                  = pshmem_putmem_signal
+#pragma weak shmem_ctx_putmem_signal              = pshmem_ctx_putmem_signal
+
+
+#pragma weak shmem_signal_fetch                   = pshmem_signal_fetch
+
+
+
+
+#include "oshmem/shmem/c/profile-defines.h"
+#endif
+
+SHMEM_CTX_TYPE_PUT_SIGNAL(_char, char)
+SHMEM_CTX_TYPE_PUT_SIGNAL(_short, short)
+SHMEM_CTX_TYPE_PUT_SIGNAL(_int, int)
+SHMEM_CTX_TYPE_PUT_SIGNAL(_long, long)
+SHMEM_CTX_TYPE_PUT_SIGNAL(_longlong, long long)
+SHMEM_CTX_TYPE_PUT_SIGNAL(_schar, signed char)
+SHMEM_CTX_TYPE_PUT_SIGNAL(_uchar, unsigned char)
+SHMEM_CTX_TYPE_PUT_SIGNAL(_ushort, unsigned short)
+SHMEM_CTX_TYPE_PUT_SIGNAL(_uint, unsigned int)
+SHMEM_CTX_TYPE_PUT_SIGNAL(_ulong, unsigned long)
+SHMEM_CTX_TYPE_PUT_SIGNAL(_ulonglong, unsigned long long)
+SHMEM_CTX_TYPE_PUT_SIGNAL(_float, float)
+SHMEM_CTX_TYPE_PUT_SIGNAL(_double, double)
+SHMEM_CTX_TYPE_PUT_SIGNAL(_longdouble, long double)
+SHMEM_CTX_TYPE_PUT_SIGNAL(_int8, int8_t)
+SHMEM_CTX_TYPE_PUT_SIGNAL(_int16, int16_t)
+SHMEM_CTX_TYPE_PUT_SIGNAL(_int32, int32_t)
+SHMEM_CTX_TYPE_PUT_SIGNAL(_int64, int64_t)
+SHMEM_CTX_TYPE_PUT_SIGNAL(_uint8, uint8_t)
+SHMEM_CTX_TYPE_PUT_SIGNAL(_uint16, uint16_t)
+SHMEM_CTX_TYPE_PUT_SIGNAL(_uint32, uint32_t)
+SHMEM_CTX_TYPE_PUT_SIGNAL(_uint64, uint64_t)
+SHMEM_CTX_TYPE_PUT_SIGNAL(_size, size_t)
+SHMEM_CTX_TYPE_PUT_SIGNAL(_ptrdiff, ptrdiff_t)
+
+SHMEM_TYPE_PUT_SIGNAL(_char, char)
+SHMEM_TYPE_PUT_SIGNAL(_short, short)
+SHMEM_TYPE_PUT_SIGNAL(_int, int)
+SHMEM_TYPE_PUT_SIGNAL(_long, long)
+SHMEM_TYPE_PUT_SIGNAL(_longlong, long long)
+SHMEM_TYPE_PUT_SIGNAL(_schar, signed char)
+SHMEM_TYPE_PUT_SIGNAL(_uchar, unsigned char)
+SHMEM_TYPE_PUT_SIGNAL(_ushort, unsigned short)
+SHMEM_TYPE_PUT_SIGNAL(_uint, unsigned int)
+SHMEM_TYPE_PUT_SIGNAL(_ulong, unsigned long)
+SHMEM_TYPE_PUT_SIGNAL(_ulonglong, unsigned long long)
+SHMEM_TYPE_PUT_SIGNAL(_float, float)
+SHMEM_TYPE_PUT_SIGNAL(_double, double)
+SHMEM_TYPE_PUT_SIGNAL(_longdouble, long double)
+SHMEM_TYPE_PUT_SIGNAL(_int8, int8_t)
+SHMEM_TYPE_PUT_SIGNAL(_int16, int16_t)
+SHMEM_TYPE_PUT_SIGNAL(_int32, int32_t)
+SHMEM_TYPE_PUT_SIGNAL(_int64, int64_t)
+SHMEM_TYPE_PUT_SIGNAL(_uint8, uint8_t)
+SHMEM_TYPE_PUT_SIGNAL(_uint16, uint16_t)
+SHMEM_TYPE_PUT_SIGNAL(_uint32, uint32_t)
+SHMEM_TYPE_PUT_SIGNAL(_uint64, uint64_t)
+SHMEM_TYPE_PUT_SIGNAL(_size, size_t)
+SHMEM_TYPE_PUT_SIGNAL(_ptrdiff, ptrdiff_t)
+
+#define DO_SHMEM_PUTMEM_SIGNAL(ctx, dest, source, element_size, nelems, sig_addr, signal, sig_op, pe) do { \
+        int rc = OSHMEM_SUCCESS;                                    \
+        size_t size = 0;                                            \
+                                                                    \
+        RUNTIME_CHECK_INIT();                                       \
+        RUNTIME_CHECK_PE(pe);                                       \
+        RUNTIME_CHECK_ADDR(dest);                                   \
+                                                                    \
+        size = nelems * element_size;                               \
+        rc = MCA_SPML_CALL(put_signal(                              \
+            ctx,                                                    \
+            (void*)dest,                                            \
+            size,                                                   \
+            (void*)source,                                          \
+            sig_addr, signal, sig_op, pe));                         \
+        RUNTIME_CHECK_RC(rc);                                       \
+    } while (0)
+
+#define SHMEM_CTX_TYPE_PUTMEM_SIGNAL(name, element_size, prefix)               \
+    void prefix##_ctx##name(shmem_ctx_t ctx, void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe) \
+    {                                                                          \
+        DO_SHMEM_PUTMEM_SIGNAL(ctx, dest, source,                              \
+                        element_size, nelems, sig_addr, signal, sig_op, pe);   \
+        return ;                                                               \
+    }
+
+#define SHMEM_TYPE_PUTMEM_SIGNAL(name, element_size, prefix)                           \
+    void prefix##name(void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe) \
+    {                                                                                  \
+        DO_SHMEM_PUTMEM_SIGNAL(oshmem_ctx_default, dest,                               \
+                        source, element_size, nelems, sig_addr, signal, sig_op, pe);   \
+        return ;                                                                       \
+    }
+
+SHMEM_CTX_TYPE_PUTMEM_SIGNAL(_putmem_signal, 1, shmem)
+SHMEM_CTX_TYPE_PUTMEM_SIGNAL(_put8_signal,  1, shmem)
+SHMEM_CTX_TYPE_PUTMEM_SIGNAL(_put16_signal, 2, shmem)
+SHMEM_CTX_TYPE_PUTMEM_SIGNAL(_put32_signal, 4, shmem)
+SHMEM_CTX_TYPE_PUTMEM_SIGNAL(_put64_signal, 8, shmem)
+SHMEM_CTX_TYPE_PUTMEM_SIGNAL(_put128_signal, 16, shmem)
+SHMEM_TYPE_PUTMEM_SIGNAL(_putmem_signal, 1, shmem)
+SHMEM_TYPE_PUTMEM_SIGNAL(_put8_signal,  1, shmem)
+SHMEM_TYPE_PUTMEM_SIGNAL(_put16_signal, 2, shmem)
+SHMEM_TYPE_PUTMEM_SIGNAL(_put32_signal, 4, shmem)
+SHMEM_TYPE_PUTMEM_SIGNAL(_put64_signal, 8, shmem)
+SHMEM_TYPE_PUTMEM_SIGNAL(_put128_signal, 16, shmem)
+
+
+uint64_t shmem_signal_fetch(const uint64_t *sig_addr)
+{
+    return OSHMEM_ERR_NOT_IMPLEMENTED;
+}
+
diff --git a/oshmem/shmem/c/shmem_put_signal_nb.c b/oshmem/shmem/c/shmem_put_signal_nb.c
new file mode 100644
index 00000000000..3b39e250fcd
--- /dev/null
+++ b/oshmem/shmem/c/shmem_put_signal_nb.c
@@ -0,0 +1,220 @@
+/*
+ * Copyright (c) 2021      NVIDIA Corporation.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "oshmem_config.h"
+
+#include "oshmem/constants.h"
+#include "oshmem/include/shmem.h"
+
+#include "oshmem/runtime/runtime.h"
+
+#include "oshmem/mca/spml/spml.h"
+
+#define DO_SHMEM_TYPE_PUT_SIGNAL_NBI(ctx, type, dest, source, nelems, sig_addr, signal, sig_op, pe) do {  \
+        int rc = OSHMEM_SUCCESS;                                    \
+        size_t size = 0;                                            \
+                                                                    \
+        RUNTIME_CHECK_INIT();                                       \
+        RUNTIME_CHECK_PE(pe);                                       \
+        RUNTIME_CHECK_ADDR(dest);                                   \
+                                                                    \
+        size = nelems * sizeof(type);                               \
+        rc = MCA_SPML_CALL(put_signal_nb(                           \
+            ctx,                                                    \
+            (void*)dest,                                            \
+            size,                                                   \
+            (void*)source,                                          \
+            sig_addr, signal, sig_op, pe));                         \
+        RUNTIME_CHECK_RC(rc);                                       \
+    } while (0)
+
+#define SHMEM_CTX_TYPE_PUT_SIGNAL_NBI(type_name, type)                                                 \
+    void shmem_ctx##type_name##_put_signal_nbi(shmem_ctx_t ctx, type *dest, const type *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe)\
+    {                                                                                                  \
+        DO_SHMEM_TYPE_PUT_SIGNAL_NBI(ctx, type, dest, source, nelems, sig_addr, signal, sig_op, pe);   \
+        return ;                                                                                       \
+    }
+
+#define SHMEM_TYPE_PUT_SIGNAL_NBI(type_name, type)                                                     \
+    void shmem##type_name##_put_signal_nbi(type *dest, const type *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe)\
+    {                                                                                                  \
+        DO_SHMEM_TYPE_PUT_SIGNAL_NBI(oshmem_ctx_default, type, dest,                                   \
+                          source, nelems, sig_addr, signal, sig_op, pe);                               \
+        return ;                                                                                       \
+    }
+
+#if OSHMEM_PROFILING
+#include "oshmem/include/pshmem.h"
+
+#pragma weak shmem_ctx_char_put_signal_nbi             = pshmem_ctx_char_put_signal_nbi
+#pragma weak shmem_ctx_short_put_signal_nbi            = pshmem_ctx_short_put_signal_nbi
+#pragma weak shmem_ctx_int_put_signal_nbi              = pshmem_ctx_int_put_signal_nbi
+#pragma weak shmem_ctx_long_put_signal_nbi             = pshmem_ctx_long_put_signal_nbi
+#pragma weak shmem_ctx_float_put_signal_nbi            = pshmem_ctx_float_put_signal_nbi
+#pragma weak shmem_ctx_double_put_signal_nbi           = pshmem_ctx_double_put_signal_nbi
+#pragma weak shmem_ctx_longlong_put_signal_nbi         = pshmem_ctx_longlong_put_signal_nbi
+#pragma weak shmem_ctx_schar_put_signal_nbi            = pshmem_ctx_schar_put_signal_nbi
+#pragma weak shmem_ctx_uchar_put_signal_nbi            = pshmem_ctx_uchar_put_signal_nbi
+#pragma weak shmem_ctx_ushort_put_signal_nbi           = pshmem_ctx_ushort_put_signal_nbi
+#pragma weak shmem_ctx_uint_put_signal_nbi             = pshmem_ctx_uint_put_signal_nbi
+#pragma weak shmem_ctx_ulong_put_signal_nbi            = pshmem_ctx_ulong_put_signal_nbi
+#pragma weak shmem_ctx_ulonglong_put_signal_nbi        = pshmem_ctx_ulonglong_put_signal_nbi
+#pragma weak shmem_ctx_longdouble_put_signal_nbi       = pshmem_ctx_longdouble_put_signal_nbi
+#pragma weak shmem_ctx_int8_put_signal_nbi             = pshmem_ctx_int8_put_signal_nbi
+#pragma weak shmem_ctx_int16_put_signal_nbi            = pshmem_ctx_int16_put_signal_nbi
+#pragma weak shmem_ctx_int32_put_signal_nbi            = pshmem_ctx_int32_put_signal_nbi
+#pragma weak shmem_ctx_int64_put_signal_nbi            = pshmem_ctx_int64_put_signal_nbi
+#pragma weak shmem_ctx_uint8_put_signal_nbi            = pshmem_ctx_uint8_put_signal_nbi
+#pragma weak shmem_ctx_uint16_put_signal_nbi           = pshmem_ctx_uint16_put_signal_nbi
+#pragma weak shmem_ctx_uint32_put_signal_nbi           = pshmem_ctx_uint32_put_signal_nbi
+#pragma weak shmem_ctx_uint64_put_signal_nbi           = pshmem_ctx_uint64_put_signal_nbi
+#pragma weak shmem_ctx_size_put_signal_nbi             = pshmem_ctx_size_put_signal_nbi
+#pragma weak shmem_ctx_ptrdiff_put_signal_nbi          = pshmem_ctx_ptrdiff_put_signal_nbi
+
+#pragma weak shmem_char_put_signal_nbi                 = pshmem_char_put_signal_nbi
+#pragma weak shmem_short_put_signal_nbi                = pshmem_short_put_signal_nbi
+#pragma weak shmem_int_put_signal_nbi                  = pshmem_int_put_signal_nbi
+#pragma weak shmem_long_put_signal_nbi                 = pshmem_long_put_signal_nbi
+#pragma weak shmem_float_put_signal_nbi                = pshmem_float_put_signal_nbi
+#pragma weak shmem_double_put_signal_nbi               = pshmem_double_put_signal_nbi
+#pragma weak shmem_longlong_put_signal_nbi             = pshmem_longlong_put_signal_nbi
+#pragma weak shmem_schar_put_signal_nbi                = pshmem_schar_put_signal_nbi
+#pragma weak shmem_uchar_put_signal_nbi                = pshmem_uchar_put_signal_nbi
+#pragma weak shmem_ushort_put_signal_nbi               = pshmem_ushort_put_signal_nbi
+#pragma weak shmem_uint_put_signal_nbi                 = pshmem_uint_put_signal_nbi
+#pragma weak shmem_ulong_put_signal_nbi                = pshmem_ulong_put_signal_nbi
+#pragma weak shmem_ulonglong_put_signal_nbi            = pshmem_ulonglong_put_signal_nbi
+#pragma weak shmem_longdouble_put_signal_nbi           = pshmem_longdouble_put_signal_nbi
+#pragma weak shmem_int8_put_signal_nbi                 = pshmem_int8_put_signal_nbi
+#pragma weak shmem_int16_put_signal_nbi                = pshmem_int16_put_signal_nbi
+#pragma weak shmem_int32_put_signal_nbi                = pshmem_int32_put_signal_nbi
+#pragma weak shmem_int64_put_signal_nbi                = pshmem_int64_put_signal_nbi
+#pragma weak shmem_uint8_put_signal_nbi                = pshmem_uint8_put_signal_nbi
+#pragma weak shmem_uint16_put_signal_nbi               = pshmem_uint16_put_signal_nbi
+#pragma weak shmem_uint32_put_signal_nbi               = pshmem_uint32_put_signal_nbi
+#pragma weak shmem_uint64_put_signal_nbi               = pshmem_uint64_put_signal_nbi
+#pragma weak shmem_size_put_signal_nbi                 = pshmem_size_put_signal_nbi
+#pragma weak shmem_ptrdiff_put_signal_nbi              = pshmem_ptrdiff_put_signal_nbi
+
+#pragma weak shmem_put8_signal_nbi                     = pshmem_put8_signal_nbi
+#pragma weak shmem_put16_signal_nbi                    = pshmem_put16_signal_nbi
+#pragma weak shmem_put32_signal_nbi                    = pshmem_put32_signal_nbi
+#pragma weak shmem_put64_signal_nbi                    = pshmem_put64_signal_nbi
+#pragma weak shmem_put128_signal_nbi                   = pshmem_put128_signal_nbi
+
+#pragma weak shmem_ctx_put8_signal_nbi                 = pshmem_ctx_put8_signal_nbi
+#pragma weak shmem_ctx_put16_signal_nbi                = pshmem_ctx_put16_signal_nbi
+#pragma weak shmem_ctx_put32_signal_nbi                = pshmem_ctx_put32_signal_nbi
+#pragma weak shmem_ctx_put64_signal_nbi                = pshmem_ctx_put64_signal_nbi
+#pragma weak shmem_ctx_put128_signal_nbi               = pshmem_ctx_put128_signal_nbi
+
+#pragma weak shmem_putmem_signal_nbi                  = pshmem_putmem_signal_nbi
+#pragma weak shmem_ctx_putmem_signal_nbi              = pshmem_ctx_putmem_signal_nbi
+
+
+#include "oshmem/shmem/c/profile-defines.h"
+#endif
+
+SHMEM_CTX_TYPE_PUT_SIGNAL_NBI(_char, char)
+SHMEM_CTX_TYPE_PUT_SIGNAL_NBI(_short, short)
+SHMEM_CTX_TYPE_PUT_SIGNAL_NBI(_int, int)
+SHMEM_CTX_TYPE_PUT_SIGNAL_NBI(_long, long)
+SHMEM_CTX_TYPE_PUT_SIGNAL_NBI(_longlong, long long)
+SHMEM_CTX_TYPE_PUT_SIGNAL_NBI(_schar, signed char)
+SHMEM_CTX_TYPE_PUT_SIGNAL_NBI(_uchar, unsigned char)
+SHMEM_CTX_TYPE_PUT_SIGNAL_NBI(_ushort, unsigned short)
+SHMEM_CTX_TYPE_PUT_SIGNAL_NBI(_uint, unsigned int)
+SHMEM_CTX_TYPE_PUT_SIGNAL_NBI(_ulong, unsigned long)
+SHMEM_CTX_TYPE_PUT_SIGNAL_NBI(_ulonglong, unsigned long long)
+SHMEM_CTX_TYPE_PUT_SIGNAL_NBI(_float, float)
+SHMEM_CTX_TYPE_PUT_SIGNAL_NBI(_double, double)
+SHMEM_CTX_TYPE_PUT_SIGNAL_NBI(_longdouble, long double)
+SHMEM_CTX_TYPE_PUT_SIGNAL_NBI(_int8, int8_t)
+SHMEM_CTX_TYPE_PUT_SIGNAL_NBI(_int16, int16_t)
+SHMEM_CTX_TYPE_PUT_SIGNAL_NBI(_int32, int32_t)
+SHMEM_CTX_TYPE_PUT_SIGNAL_NBI(_int64, int64_t)
+SHMEM_CTX_TYPE_PUT_SIGNAL_NBI(_uint8, uint8_t)
+SHMEM_CTX_TYPE_PUT_SIGNAL_NBI(_uint16, uint16_t)
+SHMEM_CTX_TYPE_PUT_SIGNAL_NBI(_uint32, uint32_t)
+SHMEM_CTX_TYPE_PUT_SIGNAL_NBI(_uint64, uint64_t)
+SHMEM_CTX_TYPE_PUT_SIGNAL_NBI(_size, size_t)
+SHMEM_CTX_TYPE_PUT_SIGNAL_NBI(_ptrdiff, ptrdiff_t)
+
+SHMEM_TYPE_PUT_SIGNAL_NBI(_char, char)
+SHMEM_TYPE_PUT_SIGNAL_NBI(_short, short)
+SHMEM_TYPE_PUT_SIGNAL_NBI(_int, int)
+SHMEM_TYPE_PUT_SIGNAL_NBI(_long, long)
+SHMEM_TYPE_PUT_SIGNAL_NBI(_longlong, long long)
+SHMEM_TYPE_PUT_SIGNAL_NBI(_schar, signed char)
+SHMEM_TYPE_PUT_SIGNAL_NBI(_uchar, unsigned char)
+SHMEM_TYPE_PUT_SIGNAL_NBI(_ushort, unsigned short)
+SHMEM_TYPE_PUT_SIGNAL_NBI(_uint, unsigned int)
+SHMEM_TYPE_PUT_SIGNAL_NBI(_ulong, unsigned long)
+SHMEM_TYPE_PUT_SIGNAL_NBI(_ulonglong, unsigned long long)
+SHMEM_TYPE_PUT_SIGNAL_NBI(_float, float)
+SHMEM_TYPE_PUT_SIGNAL_NBI(_double, double)
+SHMEM_TYPE_PUT_SIGNAL_NBI(_longdouble, long double)
+SHMEM_TYPE_PUT_SIGNAL_NBI(_int8, int8_t)
+SHMEM_TYPE_PUT_SIGNAL_NBI(_int16, int16_t)
+SHMEM_TYPE_PUT_SIGNAL_NBI(_int32, int32_t)
+SHMEM_TYPE_PUT_SIGNAL_NBI(_int64, int64_t)
+SHMEM_TYPE_PUT_SIGNAL_NBI(_uint8, uint8_t)
+SHMEM_TYPE_PUT_SIGNAL_NBI(_uint16, uint16_t)
+SHMEM_TYPE_PUT_SIGNAL_NBI(_uint32, uint32_t)
+SHMEM_TYPE_PUT_SIGNAL_NBI(_uint64, uint64_t)
+SHMEM_TYPE_PUT_SIGNAL_NBI(_size, size_t)
+SHMEM_TYPE_PUT_SIGNAL_NBI(_ptrdiff, ptrdiff_t)
+
+#define DO_SHMEM_PUTMEM_SIGNAL_NBI(ctx, dest, source, element_size, nelems, sig_addr, signal, sig_op, pe) do { \
+        int rc = OSHMEM_SUCCESS;                                    \
+        size_t size = 0;                                            \
+                                                                    \
+        RUNTIME_CHECK_INIT();                                       \
+        RUNTIME_CHECK_PE(pe);                                       \
+        RUNTIME_CHECK_ADDR(dest);                                   \
+                                                                    \
+        size = nelems * element_size;                               \
+        rc = MCA_SPML_CALL(put_signal_nb(                           \
+            ctx,                                                    \
+            (void*)dest,                                            \
+            size,                                                   \
+            (void*)source,                                          \
+            sig_addr, signal, sig_op, pe));                         \
+        RUNTIME_CHECK_RC(rc);                                       \
+    } while (0)
+
+#define SHMEM_CTX_TYPE_PUTMEM_SIGNAL_NBI(name, element_size, prefix)           \
+    void prefix##_ctx##name(shmem_ctx_t ctx, void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe) \
+    {                                                                          \
+        DO_SHMEM_PUTMEM_SIGNAL_NBI(ctx, dest, source,                          \
+                        element_size, nelems, sig_addr, signal, sig_op, pe);   \
+        return ;                                                               \
+    }
+
+#define SHMEM_TYPE_PUTMEM_SIGNAL_NBI(name, element_size, prefix)                       \
+    void prefix##name(void *dest, const void *source, size_t nelems, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe) \
+    {                                                                                  \
+        DO_SHMEM_PUTMEM_SIGNAL_NBI(oshmem_ctx_default, dest,                           \
+                        source, element_size, nelems, sig_addr, signal, sig_op, pe);   \
+        return ;                                                                       \
+    }
+
+SHMEM_CTX_TYPE_PUTMEM_SIGNAL_NBI(_putmem_signal_nbi, 1, shmem)
+SHMEM_CTX_TYPE_PUTMEM_SIGNAL_NBI(_put8_signal_nbi,  1, shmem)
+SHMEM_CTX_TYPE_PUTMEM_SIGNAL_NBI(_put16_signal_nbi, 2, shmem)
+SHMEM_CTX_TYPE_PUTMEM_SIGNAL_NBI(_put32_signal_nbi, 4, shmem)
+SHMEM_CTX_TYPE_PUTMEM_SIGNAL_NBI(_put64_signal_nbi, 8, shmem)
+SHMEM_CTX_TYPE_PUTMEM_SIGNAL_NBI(_put128_signal_nbi, 16, shmem)
+SHMEM_TYPE_PUTMEM_SIGNAL_NBI(_putmem_signal_nbi, 1, shmem)
+SHMEM_TYPE_PUTMEM_SIGNAL_NBI(_put8_signal_nbi,  1, shmem)
+SHMEM_TYPE_PUTMEM_SIGNAL_NBI(_put16_signal_nbi, 2, shmem)
+SHMEM_TYPE_PUTMEM_SIGNAL_NBI(_put32_signal_nbi, 4, shmem)
+SHMEM_TYPE_PUTMEM_SIGNAL_NBI(_put64_signal_nbi, 8, shmem)
+SHMEM_TYPE_PUTMEM_SIGNAL_NBI(_put128_signal_nbi, 16, shmem)
+
diff --git a/oshmem/shmem/c/shmem_query.c b/oshmem/shmem/c/shmem_query.c
index fd3f1771d94..68809619ac3 100644
--- a/oshmem/shmem/c/shmem_query.c
+++ b/oshmem/shmem/c/shmem_query.c
@@ -23,7 +23,7 @@
 #pragma weak shmem_query_thread = pshmem_query_thread
 #pragma weak _num_pes = p_num_pes
 #pragma weak _my_pe = p_my_pe
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 int _num_pes(void)
diff --git a/oshmem/shmem/c/shmem_quiet.c b/oshmem/shmem/c/shmem_quiet.c
index e27b273d30b..75d983f7d7b 100644
--- a/oshmem/shmem/c/shmem_quiet.c
+++ b/oshmem/shmem/c/shmem_quiet.c
@@ -18,7 +18,7 @@
 #include "oshmem/include/pshmem.h"
 #pragma weak shmem_quiet = pshmem_quiet
 #pragma weak shmem_ctx_quiet = pshmem_ctx_quiet
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 void shmem_quiet(void)
diff --git a/oshmem/shmem/c/shmem_realloc.c b/oshmem/shmem/c/shmem_realloc.c
index a37ae7e6c4b..c637e05c7a9 100644
--- a/oshmem/shmem/c/shmem_realloc.c
+++ b/oshmem/shmem/c/shmem_realloc.c
@@ -24,7 +24,7 @@
 #include "oshmem/include/pshmem.h"
 #pragma weak shmem_realloc = pshmem_realloc
 #pragma weak shrealloc = pshrealloc
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 static inline void* _shrealloc(void *ptr, size_t size);
diff --git a/oshmem/shmem/c/shmem_reduce.c b/oshmem/shmem/c/shmem_reduce.c
index 02c47023ce7..0c4adc1fbdf 100644
--- a/oshmem/shmem/c/shmem_reduce.c
+++ b/oshmem/shmem/c/shmem_reduce.c
@@ -16,6 +16,7 @@
 #include "oshmem/runtime/runtime.h"
 
 #include "oshmem/mca/scoll/scoll.h"
+#include "oshmem/mca/spml/spml.h"
 #include "oshmem/proc/proc.h"
 #include "oshmem/op/op.h"
 
@@ -137,7 +138,173 @@
 #pragma weak shmemx_int16_prod_to_all   =  pshmemx_int16_prod_to_all
 #pragma weak shmemx_int32_prod_to_all   =  pshmemx_int32_prod_to_all
 #pragma weak shmemx_int64_prod_to_all   =  pshmemx_int64_prod_to_all
-#include "oshmem/shmem/c/profile/defines.h"
+
+/* Teams reduction: AND */
+#pragma weak shmem_uchar_and_reduce          		= pshmem_uchar_and_reduce
+#pragma weak shmem_ushort_and_reduce         		= pshmem_ushort_and_reduce
+#pragma weak shmem_uint_and_reduce           		= pshmem_uint_and_reduce
+#pragma weak shmem_ulong_and_reduce          		= pshmem_ulong_and_reduce
+#pragma weak shmem_ulonglong_and_reduce      		= pshmem_ulonglong_and_reduce
+#pragma weak shmem_int_and_reduce                  = pshmem_int_and_reduce
+#pragma weak shmem_longlong_and_reduce             = pshmem_longlong_and_reduce
+#pragma weak shmem_int8_and_reduce           		= pshmem_int8_and_reduce
+#pragma weak shmem_int16_and_reduce          		= pshmem_int16_and_reduce
+#pragma weak shmem_int32_and_reduce          		= pshmem_int32_and_reduce
+#pragma weak shmem_int64_and_reduce          		= pshmem_int64_and_reduce
+#pragma weak shmem_uint8_and_reduce          		= pshmem_uint8_and_reduce
+#pragma weak shmem_uint16_and_reduce         		= pshmem_uint16_and_reduce
+#pragma weak shmem_uint32_and_reduce         		= pshmem_uint32_and_reduce
+#pragma weak shmem_uint64_and_reduce         		= pshmem_uint64_and_reduce
+#pragma weak shmem_size_and_reduce           		= pshmem_size_and_reduce
+
+/* Teams reduction: OR */
+#pragma weak shmem_uchar_or_reduce          		= pshmem_uchar_or_reduce
+#pragma weak shmem_ushort_or_reduce         		= pshmem_ushort_or_reduce
+#pragma weak shmem_uint_or_reduce           		= pshmem_uint_or_reduce
+#pragma weak shmem_ulong_or_reduce          		= pshmem_ulong_or_reduce
+#pragma weak shmem_ulonglong_or_reduce      		= pshmem_ulonglong_or_reduce
+#pragma weak shmem_int8_or_reduce           		= pshmem_int8_or_reduce
+#pragma weak shmem_int16_or_reduce          		= pshmem_int16_or_reduce
+#pragma weak shmem_int32_or_reduce          		= pshmem_int32_or_reduce
+#pragma weak shmem_int64_or_reduce          		= pshmem_int64_or_reduce
+#pragma weak shmem_uint8_or_reduce          		= pshmem_uint8_or_reduce
+#pragma weak shmem_uint16_or_reduce         		= pshmem_uint16_or_reduce
+#pragma weak shmem_uint32_or_reduce         		= pshmem_uint32_or_reduce
+#pragma weak shmem_uint64_or_reduce         		= pshmem_uint64_or_reduce
+#pragma weak shmem_size_or_reduce           		= pshmem_size_or_reduce
+
+
+/* Teams reduction: XOR */
+#pragma weak shmem_uchar_xor_reduce          		= pshmem_uchar_xor_reduce
+#pragma weak shmem_ushort_xor_reduce         		= pshmem_ushort_xor_reduce
+#pragma weak shmem_uint_xor_reduce           		= pshmem_uint_xor_reduce
+#pragma weak shmem_ulong_xor_reduce          		= pshmem_ulong_xor_reduce
+#pragma weak shmem_ulonglong_xor_reduce      		= pshmem_ulonglong_xor_reduce
+#pragma weak shmem_int8_xor_reduce           		= pshmem_int8_xor_reduce
+#pragma weak shmem_int16_xor_reduce          		= pshmem_int16_xor_reduce
+#pragma weak shmem_int32_xor_reduce          		= pshmem_int32_xor_reduce
+#pragma weak shmem_int64_xor_reduce          		= pshmem_int64_xor_reduce
+#pragma weak shmem_uint8_xor_reduce          		= pshmem_uint8_xor_reduce
+#pragma weak shmem_uint16_xor_reduce         		= pshmem_uint16_xor_reduce
+#pragma weak shmem_uint32_xor_reduce         		= pshmem_uint32_xor_reduce
+#pragma weak shmem_uint64_xor_reduce         		= pshmem_uint64_xor_reduce
+#pragma weak shmem_size_xor_reduce           		= pshmem_size_xor_reduce
+
+
+/* Teams reduction: MAX */
+#pragma weak shmem_char_max_reduce           		= pshmem_char_max_reduce
+#pragma weak shmem_short_max_reduce          		= pshmem_short_max_reduce
+#pragma weak shmem_int_max_reduce            		= pshmem_int_max_reduce
+#pragma weak shmem_long_max_reduce           		= pshmem_long_max_reduce
+#pragma weak shmem_float_max_reduce          		= pshmem_float_max_reduce
+#pragma weak shmem_double_max_reduce         		= pshmem_double_max_reduce
+#pragma weak shmem_longlong_max_reduce       		= pshmem_longlong_max_reduce
+#pragma weak shmem_schar_max_reduce          		= pshmem_schar_max_reduce
+#pragma weak shmem_longdouble_max_reduce     		= pshmem_longdouble_max_reduce
+#pragma weak shmem_ptrdiff_max_reduce        		= pshmem_ptrdiff_max_reduce
+#pragma weak shmem_uchar_max_reduce          		= pshmem_uchar_max_reduce
+#pragma weak shmem_ushort_max_reduce         		= pshmem_ushort_max_reduce
+#pragma weak shmem_uint_max_reduce           		= pshmem_uint_max_reduce
+#pragma weak shmem_ulong_max_reduce          		= pshmem_ulong_max_reduce
+#pragma weak shmem_ulonglong_max_reduce      		= pshmem_ulonglong_max_reduce
+#pragma weak shmem_int8_max_reduce           		= pshmem_int8_max_reduce
+#pragma weak shmem_int16_max_reduce          		= pshmem_int16_max_reduce
+#pragma weak shmem_int32_max_reduce          		= pshmem_int32_max_reduce
+#pragma weak shmem_int64_max_reduce          		= pshmem_int64_max_reduce
+#pragma weak shmem_uint8_max_reduce          		= pshmem_uint8_max_reduce
+#pragma weak shmem_uint16_max_reduce         		= pshmem_uint16_max_reduce
+#pragma weak shmem_uint32_max_reduce         		= pshmem_uint32_max_reduce
+#pragma weak shmem_uint64_max_reduce         		= pshmem_uint64_max_reduce
+#pragma weak shmem_size_max_reduce           		= pshmem_size_max_reduce
+
+
+/* Teams reduction: MIN */
+#pragma weak shmem_char_min_reduce           		= pshmem_char_min_reduce
+#pragma weak shmem_short_min_reduce          		= pshmem_short_min_reduce
+#pragma weak shmem_int_min_reduce            		= pshmem_int_min_reduce
+#pragma weak shmem_long_min_reduce           		= pshmem_long_min_reduce
+#pragma weak shmem_float_min_reduce          		= pshmem_float_min_reduce
+#pragma weak shmem_double_min_reduce         		= pshmem_double_min_reduce
+#pragma weak shmem_longlong_min_reduce       		= pshmem_longlong_min_reduce
+#pragma weak shmem_schar_min_reduce          		= pshmem_schar_min_reduce
+#pragma weak shmem_longdouble_min_reduce     		= pshmem_longdouble_min_reduce
+#pragma weak shmem_ptrdiff_min_reduce        		= pshmem_ptrdiff_min_reduce
+#pragma weak shmem_uchar_min_reduce          		= pshmem_uchar_min_reduce
+#pragma weak shmem_ushort_min_reduce         		= pshmem_ushort_min_reduce
+#pragma weak shmem_uint_min_reduce           		= pshmem_uint_min_reduce
+#pragma weak shmem_ulong_min_reduce          		= pshmem_ulong_min_reduce
+#pragma weak shmem_ulonglong_min_reduce      		= pshmem_ulonglong_min_reduce
+#pragma weak shmem_int8_min_reduce           		= pshmem_int8_min_reduce
+#pragma weak shmem_int16_min_reduce          		= pshmem_int16_min_reduce
+#pragma weak shmem_int32_min_reduce          		= pshmem_int32_min_reduce
+#pragma weak shmem_int64_min_reduce          		= pshmem_int64_min_reduce
+#pragma weak shmem_uint8_min_reduce          		= pshmem_uint8_min_reduce
+#pragma weak shmem_uint16_min_reduce         		= pshmem_uint16_min_reduce
+#pragma weak shmem_uint32_min_reduce         		= pshmem_uint32_min_reduce
+#pragma weak shmem_uint64_min_reduce         		= pshmem_uint64_min_reduce
+#pragma weak shmem_size_min_reduce           		= pshmem_size_min_reduce
+
+
+/* Teams reduction: SUM */
+#pragma weak shmem_char_sum_reduce           		= pshmem_char_sum_reduce
+#pragma weak shmem_short_sum_reduce          		= pshmem_short_sum_reduce
+#pragma weak shmem_int_sum_reduce            		= pshmem_int_sum_reduce
+#pragma weak shmem_long_sum_reduce           		= pshmem_long_sum_reduce
+#pragma weak shmem_float_sum_reduce          		= pshmem_float_sum_reduce
+#pragma weak shmem_double_sum_reduce         		= pshmem_double_sum_reduce
+#pragma weak shmem_longlong_sum_reduce       		= pshmem_longlong_sum_reduce
+#pragma weak shmem_schar_sum_reduce          		= pshmem_schar_sum_reduce
+#pragma weak shmem_longdouble_sum_reduce     		= pshmem_longdouble_sum_reduce
+#pragma weak shmem_ptrdiff_sum_reduce        		= pshmem_ptrdiff_sum_reduce
+#pragma weak shmem_uchar_sum_reduce          		= pshmem_uchar_sum_reduce
+#pragma weak shmem_ushort_sum_reduce         		= pshmem_ushort_sum_reduce
+#pragma weak shmem_uint_sum_reduce           		= pshmem_uint_sum_reduce
+#pragma weak shmem_ulong_sum_reduce          		= pshmem_ulong_sum_reduce
+#pragma weak shmem_ulonglong_sum_reduce      		= pshmem_ulonglong_sum_reduce
+#pragma weak shmem_int8_sum_reduce           		= pshmem_int8_sum_reduce
+#pragma weak shmem_int16_sum_reduce          		= pshmem_int16_sum_reduce
+#pragma weak shmem_int32_sum_reduce          		= pshmem_int32_sum_reduce
+#pragma weak shmem_int64_sum_reduce          		= pshmem_int64_sum_reduce
+#pragma weak shmem_uint8_sum_reduce          		= pshmem_uint8_sum_reduce
+#pragma weak shmem_uint16_sum_reduce         		= pshmem_uint16_sum_reduce
+#pragma weak shmem_uint32_sum_reduce         		= pshmem_uint32_sum_reduce
+#pragma weak shmem_uint64_sum_reduce         		= pshmem_uint64_sum_reduce
+#pragma weak shmem_size_sum_reduce           		= pshmem_size_sum_reduce
+#pragma weak shmem_complexd_sum_reduce       		= pshmem_complexd_sum_reduce
+#pragma weak shmem_complexf_sum_reduce       		= pshmem_complexf_sum_reduce
+
+
+/* Teams reduction: PROD */
+#pragma weak shmem_char_prod_reduce           		= pshmem_char_prod_reduce
+#pragma weak shmem_short_prod_reduce          		= pshmem_short_prod_reduce
+#pragma weak shmem_int_prod_reduce            		= pshmem_int_prod_reduce
+#pragma weak shmem_long_prod_reduce           		= pshmem_long_prod_reduce
+#pragma weak shmem_float_prod_reduce          		= pshmem_float_prod_reduce
+#pragma weak shmem_double_prod_reduce         		= pshmem_double_prod_reduce
+#pragma weak shmem_longlong_prod_reduce       		= pshmem_longlong_prod_reduce
+#pragma weak shmem_schar_prod_reduce          		= pshmem_schar_prod_reduce
+#pragma weak shmem_longdouble_prod_reduce     		= pshmem_longdouble_prod_reduce
+#pragma weak shmem_ptrdiff_prod_reduce        		= pshmem_ptrdiff_prod_reduce
+#pragma weak shmem_uchar_prod_reduce          		= pshmem_uchar_prod_reduce
+#pragma weak shmem_ushort_prod_reduce         		= pshmem_ushort_prod_reduce
+#pragma weak shmem_uint_prod_reduce           		= pshmem_uint_prod_reduce
+#pragma weak shmem_ulong_prod_reduce          		= pshmem_ulong_prod_reduce
+#pragma weak shmem_ulonglong_prod_reduce      		= pshmem_ulonglong_prod_reduce
+#pragma weak shmem_int8_prod_reduce           		= pshmem_int8_prod_reduce
+#pragma weak shmem_int16_prod_reduce          		= pshmem_int16_prod_reduce
+#pragma weak shmem_int32_prod_reduce          		= pshmem_int32_prod_reduce
+#pragma weak shmem_int64_prod_reduce          		= pshmem_int64_prod_reduce
+#pragma weak shmem_uint8_prod_reduce          		= pshmem_uint8_prod_reduce
+#pragma weak shmem_uint16_prod_reduce         		= pshmem_uint16_prod_reduce
+#pragma weak shmem_uint32_prod_reduce         		= pshmem_uint32_prod_reduce
+#pragma weak shmem_uint64_prod_reduce         		= pshmem_uint64_prod_reduce
+#pragma weak shmem_size_prod_reduce           		= pshmem_size_prod_reduce
+#pragma weak shmem_complexd_prod_reduce       		= pshmem_complexd_prod_reduce
+#pragma weak shmem_complexf_prod_reduce       		= pshmem_complexf_prod_reduce
+
+
+
+#include "oshmem/shmem/c/profile-defines.h"
 #endif /* OSHMEM_PROFILING */
 
 SHMEM_TYPE_REDUCE_OP(and, _short, short, shmem)
@@ -211,3 +378,184 @@ SHMEM_TYPE_REDUCE_OP(prod, _complexd, double complex, shmem)
 SHMEM_TYPE_REDUCE_OP(prod, _int16, int16_t, shmemx)
 SHMEM_TYPE_REDUCE_OP(prod, _int32, int32_t, shmemx)
 SHMEM_TYPE_REDUCE_OP(prod, _int64, int64_t, shmemx)
+
+
+#define SHMEM_TYPE_TEAM_REDUCE_OP(_op, type_name, type, op_code, code)                      \
+    int shmem##type_name##_##_op##_reduce( shmem_team_t team, type *dest, const type *source, size_t nreduce)                                 \
+{                                                                                           \
+    int rc = OSHMEM_SUCCESS;                                                                \
+                                                                                            \
+    RUNTIME_CHECK_INIT();                                                                   \
+    RUNTIME_CHECK_ADDR_SIZE(dest, nreduce);                                                 \
+    RUNTIME_CHECK_ADDR_SIZE(source, nreduce);                                               \
+                                                                                            \
+    {                                                                                       \
+                                                                                            \
+        /* Call collective reduce operation */                                              \
+        rc = MCA_SPML_CALL(team_reduce(                                                     \
+            team, (void*)dest, (void*)source, nreduce, op_code, code));                     \
+                                                                                            \
+    }                                                                                       \
+    RUNTIME_CHECK_RC(rc);                                                                   \
+                                                                                            \
+    return rc;                                                                              \
+}
+
+SHMEM_TYPE_TEAM_REDUCE_OP(and, _uchar, unsigned char, OSHMEM_OP_AND, SHMEM_UCHAR)
+SHMEM_TYPE_TEAM_REDUCE_OP(and, _ushort, unsigned short, OSHMEM_OP_AND, SHMEM_USHORT)
+SHMEM_TYPE_TEAM_REDUCE_OP(and, _uint, unsigned int, OSHMEM_OP_AND, SHMEM_UINT)
+SHMEM_TYPE_TEAM_REDUCE_OP(and, _ulong, unsigned long, OSHMEM_OP_AND, SHMEM_ULONG)
+SHMEM_TYPE_TEAM_REDUCE_OP(and, _ulonglong, unsigned long long, OSHMEM_OP_AND, SHMEM_ULLONG)
+SHMEM_TYPE_TEAM_REDUCE_OP(and, _int, int, OSHMEM_OP_AND, SHMEM_INT)
+SHMEM_TYPE_TEAM_REDUCE_OP(and, _longlong, long long, OSHMEM_OP_AND, SHMEM_LLONG)
+SHMEM_TYPE_TEAM_REDUCE_OP(and, _int8, int8_t, OSHMEM_OP_AND, SHMEM_INT8_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(and, _int16, int16_t, OSHMEM_OP_AND, SHMEM_INT16_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(and, _int32, int32_t, OSHMEM_OP_AND, SHMEM_INT32_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(and, _int64, int64_t, OSHMEM_OP_AND, SHMEM_INT64_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(and, _uint8, uint8_t, OSHMEM_OP_AND, SHMEM_UINT8_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(and, _uint16, uint16_t, OSHMEM_OP_AND, SHMEM_UINT16_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(and, _uint32, uint32_t, OSHMEM_OP_AND, SHMEM_UINT32_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(and, _uint64, uint64_t, OSHMEM_OP_AND, SHMEM_UINT64_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(and, _size, size_t, OSHMEM_OP_AND, SHMEM_SIZE_T)
+
+SHMEM_TYPE_TEAM_REDUCE_OP(or, _uchar, unsigned char, OSHMEM_OP_OR, SHMEM_UCHAR)
+SHMEM_TYPE_TEAM_REDUCE_OP(or, _ushort, unsigned short, OSHMEM_OP_OR, SHMEM_USHORT)
+SHMEM_TYPE_TEAM_REDUCE_OP(or, _uint, unsigned int, OSHMEM_OP_OR, SHMEM_UINT)
+SHMEM_TYPE_TEAM_REDUCE_OP(or, _ulong, unsigned long, OSHMEM_OP_OR, SHMEM_ULONG)
+SHMEM_TYPE_TEAM_REDUCE_OP(or, _ulonglong, unsigned long long, OSHMEM_OP_OR, SHMEM_ULLONG)
+SHMEM_TYPE_TEAM_REDUCE_OP(or, _int, int, OSHMEM_OP_OR, SHMEM_INT)
+SHMEM_TYPE_TEAM_REDUCE_OP(or, _longlong, long long, OSHMEM_OP_OR, SHMEM_LLONG)
+SHMEM_TYPE_TEAM_REDUCE_OP(or, _int8, int8_t, OSHMEM_OP_OR, SHMEM_INT8_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(or, _int16, int16_t, OSHMEM_OP_OR, SHMEM_INT16_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(or, _int32, int32_t, OSHMEM_OP_OR, SHMEM_INT32_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(or, _int64, int64_t, OSHMEM_OP_OR, SHMEM_INT64_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(or, _uint8, uint8_t, OSHMEM_OP_OR, SHMEM_UINT8_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(or, _uint16, uint16_t, OSHMEM_OP_OR, SHMEM_UINT16_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(or, _uint32, uint32_t, OSHMEM_OP_OR, SHMEM_UINT32_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(or, _uint64, uint64_t, OSHMEM_OP_OR, SHMEM_UINT64_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(or, _size, size_t, OSHMEM_OP_OR, SHMEM_SIZE_T)
+
+SHMEM_TYPE_TEAM_REDUCE_OP(xor, _uchar, unsigned char, OSHMEM_OP_XOR, SHMEM_UCHAR)
+SHMEM_TYPE_TEAM_REDUCE_OP(xor, _ushort, unsigned short, OSHMEM_OP_XOR, SHMEM_USHORT)
+SHMEM_TYPE_TEAM_REDUCE_OP(xor, _uint, unsigned int, OSHMEM_OP_XOR, SHMEM_UINT)
+SHMEM_TYPE_TEAM_REDUCE_OP(xor, _ulong, unsigned long, OSHMEM_OP_XOR, SHMEM_ULONG)
+SHMEM_TYPE_TEAM_REDUCE_OP(xor, _ulonglong, unsigned long long, OSHMEM_OP_XOR, SHMEM_ULLONG)
+SHMEM_TYPE_TEAM_REDUCE_OP(xor, _int, int, OSHMEM_OP_XOR, SHMEM_INT)
+SHMEM_TYPE_TEAM_REDUCE_OP(xor, _longlong, long long, OSHMEM_OP_XOR, SHMEM_LLONG)
+SHMEM_TYPE_TEAM_REDUCE_OP(xor, _int8, int8_t, OSHMEM_OP_XOR, SHMEM_INT8_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(xor, _int16, int16_t, OSHMEM_OP_XOR, SHMEM_INT16_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(xor, _int32, int32_t, OSHMEM_OP_XOR, SHMEM_INT32_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(xor, _int64, int64_t, OSHMEM_OP_XOR, SHMEM_INT64_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(xor, _uint8, uint8_t, OSHMEM_OP_XOR, SHMEM_UINT8_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(xor, _uint16, uint16_t, OSHMEM_OP_XOR, SHMEM_UINT16_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(xor, _uint32, uint32_t, OSHMEM_OP_XOR, SHMEM_UINT32_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(xor, _uint64, uint64_t, OSHMEM_OP_XOR, SHMEM_UINT64_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(xor, _size, size_t, OSHMEM_OP_XOR, SHMEM_SIZE_T)
+
+
+
+SHMEM_TYPE_TEAM_REDUCE_OP(max, _uchar, unsigned char, OSHMEM_OP_MAX, SHMEM_UCHAR)
+SHMEM_TYPE_TEAM_REDUCE_OP(max, _ushort, unsigned short, OSHMEM_OP_MAX, SHMEM_USHORT)
+SHMEM_TYPE_TEAM_REDUCE_OP(max, _uint, unsigned int, OSHMEM_OP_MAX, SHMEM_UINT)
+SHMEM_TYPE_TEAM_REDUCE_OP(max, _ulong, unsigned long, OSHMEM_OP_MAX, SHMEM_ULONG)
+SHMEM_TYPE_TEAM_REDUCE_OP(max, _ulonglong, unsigned long long, OSHMEM_OP_MAX, SHMEM_ULLONG)
+SHMEM_TYPE_TEAM_REDUCE_OP(max, _int8, int8_t, OSHMEM_OP_MAX, SHMEM_INT8_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(max, _int16, int16_t, OSHMEM_OP_MAX, SHMEM_INT16_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(max, _int32, int32_t, OSHMEM_OP_MAX, SHMEM_INT32_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(max, _int64, int64_t, OSHMEM_OP_MAX, SHMEM_INT64_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(max, _uint8, uint8_t, OSHMEM_OP_MAX, SHMEM_UINT8_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(max, _uint16, uint16_t, OSHMEM_OP_MAX, SHMEM_UINT16_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(max, _uint32, uint32_t, OSHMEM_OP_MAX, SHMEM_UINT32_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(max, _uint64, uint64_t, OSHMEM_OP_MAX, SHMEM_UINT64_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(max, _size, size_t, OSHMEM_OP_MAX, SHMEM_SIZE_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(max, _char, char, OSHMEM_OP_MAX, SHMEM_CHAR)
+SHMEM_TYPE_TEAM_REDUCE_OP(max, _short, short, OSHMEM_OP_MAX, SHMEM_SHORT)
+SHMEM_TYPE_TEAM_REDUCE_OP(max, _int, int, OSHMEM_OP_MAX, SHMEM_INT)
+SHMEM_TYPE_TEAM_REDUCE_OP(max, _long, long, OSHMEM_OP_MAX, SHMEM_LONG)
+SHMEM_TYPE_TEAM_REDUCE_OP(max, _float, float, OSHMEM_OP_MAX, SHMEM_FLOAT)
+SHMEM_TYPE_TEAM_REDUCE_OP(max, _double, double, OSHMEM_OP_MAX, SHMEM_DOUBLE)
+SHMEM_TYPE_TEAM_REDUCE_OP(max, _longlong, long long, OSHMEM_OP_MAX, SHMEM_LLONG)
+SHMEM_TYPE_TEAM_REDUCE_OP(max, _schar, signed char, OSHMEM_OP_MAX, SHMEM_SCHAR)
+SHMEM_TYPE_TEAM_REDUCE_OP(max, _longdouble, long double, OSHMEM_OP_MAX, SHMEM_LDOUBLE)
+SHMEM_TYPE_TEAM_REDUCE_OP(max, _ptrdiff, ptrdiff_t, OSHMEM_OP_MAX, SHMEM_PTRDIFF_T)
+
+
+SHMEM_TYPE_TEAM_REDUCE_OP(min, _uchar, unsigned char, OSHMEM_OP_MIN, SHMEM_UCHAR)
+SHMEM_TYPE_TEAM_REDUCE_OP(min, _ushort, unsigned short, OSHMEM_OP_MIN, SHMEM_USHORT)
+SHMEM_TYPE_TEAM_REDUCE_OP(min, _uint, unsigned int, OSHMEM_OP_MIN, SHMEM_UINT)
+SHMEM_TYPE_TEAM_REDUCE_OP(min, _ulong, unsigned long, OSHMEM_OP_MIN, SHMEM_ULONG)
+SHMEM_TYPE_TEAM_REDUCE_OP(min, _ulonglong, unsigned long long, OSHMEM_OP_MIN, SHMEM_ULLONG)
+SHMEM_TYPE_TEAM_REDUCE_OP(min, _int8, int8_t, OSHMEM_OP_MIN, SHMEM_INT8_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(min, _int16, int16_t, OSHMEM_OP_MIN, SHMEM_INT16_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(min, _int32, int32_t, OSHMEM_OP_MIN, SHMEM_INT32_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(min, _int64, int64_t, OSHMEM_OP_MIN, SHMEM_INT64_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(min, _uint8, uint8_t, OSHMEM_OP_MIN, SHMEM_UINT8_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(min, _uint16, uint16_t, OSHMEM_OP_MIN, SHMEM_UINT16_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(min, _uint32, uint32_t, OSHMEM_OP_MIN, SHMEM_UINT32_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(min, _uint64, uint64_t, OSHMEM_OP_MIN, SHMEM_UINT64_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(min, _size, size_t, OSHMEM_OP_MIN, SHMEM_SIZE_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(min, _char, char, OSHMEM_OP_MIN, SHMEM_CHAR)
+SHMEM_TYPE_TEAM_REDUCE_OP(min, _short, short, OSHMEM_OP_MIN, SHMEM_SHORT)
+SHMEM_TYPE_TEAM_REDUCE_OP(min, _int, int, OSHMEM_OP_MIN, SHMEM_INT)
+SHMEM_TYPE_TEAM_REDUCE_OP(min, _long, long, OSHMEM_OP_MIN, SHMEM_LONG)
+SHMEM_TYPE_TEAM_REDUCE_OP(min, _float, float, OSHMEM_OP_MIN, SHMEM_FLOAT)
+SHMEM_TYPE_TEAM_REDUCE_OP(min, _double, double, OSHMEM_OP_MIN, SHMEM_DOUBLE)
+SHMEM_TYPE_TEAM_REDUCE_OP(min, _longlong, long long, OSHMEM_OP_MIN, SHMEM_LLONG)
+SHMEM_TYPE_TEAM_REDUCE_OP(min, _schar, signed char, OSHMEM_OP_MIN, SHMEM_SCHAR)
+SHMEM_TYPE_TEAM_REDUCE_OP(min, _longdouble, long double, OSHMEM_OP_MIN, SHMEM_LDOUBLE)
+SHMEM_TYPE_TEAM_REDUCE_OP(min, _ptrdiff, ptrdiff_t, OSHMEM_OP_MIN, SHMEM_PTRDIFF_T)
+
+
+SHMEM_TYPE_TEAM_REDUCE_OP(sum, _uchar, unsigned char, OSHMEM_OP_SUM, SHMEM_UCHAR)
+SHMEM_TYPE_TEAM_REDUCE_OP(sum, _ushort, unsigned short, OSHMEM_OP_SUM, SHMEM_USHORT)
+SHMEM_TYPE_TEAM_REDUCE_OP(sum, _uint, unsigned int, OSHMEM_OP_SUM, SHMEM_UINT)
+SHMEM_TYPE_TEAM_REDUCE_OP(sum, _ulong, unsigned long, OSHMEM_OP_SUM, SHMEM_ULONG)
+SHMEM_TYPE_TEAM_REDUCE_OP(sum, _ulonglong, unsigned long long, OSHMEM_OP_SUM, SHMEM_ULLONG)
+SHMEM_TYPE_TEAM_REDUCE_OP(sum, _int8, int8_t, OSHMEM_OP_SUM, SHMEM_INT8_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(sum, _int16, int16_t, OSHMEM_OP_SUM, SHMEM_INT16_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(sum, _int32, int32_t, OSHMEM_OP_SUM, SHMEM_INT32_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(sum, _int64, int64_t, OSHMEM_OP_SUM, SHMEM_INT64_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(sum, _uint8, uint8_t, OSHMEM_OP_SUM, SHMEM_UINT8_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(sum, _uint16, uint16_t, OSHMEM_OP_SUM, SHMEM_UINT16_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(sum, _uint32, uint32_t, OSHMEM_OP_SUM, SHMEM_UINT32_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(sum, _uint64, uint64_t, OSHMEM_OP_SUM, SHMEM_UINT64_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(sum, _size, size_t, OSHMEM_OP_SUM, SHMEM_SIZE_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(sum, _char, char, OSHMEM_OP_SUM, SHMEM_CHAR)
+SHMEM_TYPE_TEAM_REDUCE_OP(sum, _short, short, OSHMEM_OP_SUM, SHMEM_SHORT)
+SHMEM_TYPE_TEAM_REDUCE_OP(sum, _int, int, OSHMEM_OP_SUM, SHMEM_INT)
+SHMEM_TYPE_TEAM_REDUCE_OP(sum, _long, long, OSHMEM_OP_SUM, SHMEM_LONG)
+SHMEM_TYPE_TEAM_REDUCE_OP(sum, _float, float, OSHMEM_OP_SUM, SHMEM_FLOAT)
+SHMEM_TYPE_TEAM_REDUCE_OP(sum, _double, double, OSHMEM_OP_SUM, SHMEM_DOUBLE)
+SHMEM_TYPE_TEAM_REDUCE_OP(sum, _longlong, long long, OSHMEM_OP_SUM, SHMEM_LLONG)
+SHMEM_TYPE_TEAM_REDUCE_OP(sum, _schar, signed char, OSHMEM_OP_SUM, SHMEM_SCHAR)
+SHMEM_TYPE_TEAM_REDUCE_OP(sum, _longdouble, long double, OSHMEM_OP_SUM, SHMEM_LDOUBLE)
+SHMEM_TYPE_TEAM_REDUCE_OP(sum, _ptrdiff, ptrdiff_t, OSHMEM_OP_SUM, SHMEM_PTRDIFF_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(sum, _complexd, double complex, OSHMEM_OP_SUM, SHMEM_COMPLEXD)
+SHMEM_TYPE_TEAM_REDUCE_OP(sum, _complexf, float complex, OSHMEM_OP_SUM, SHMEM_COMPLEXF)
+
+SHMEM_TYPE_TEAM_REDUCE_OP(prod, _uchar, unsigned char, OSHMEM_OP_PROD, SHMEM_UCHAR)
+SHMEM_TYPE_TEAM_REDUCE_OP(prod, _ushort, unsigned short, OSHMEM_OP_PROD, SHMEM_USHORT)
+SHMEM_TYPE_TEAM_REDUCE_OP(prod, _uint, unsigned int, OSHMEM_OP_PROD, SHMEM_UINT)
+SHMEM_TYPE_TEAM_REDUCE_OP(prod, _ulong, unsigned long, OSHMEM_OP_PROD, SHMEM_ULONG)
+SHMEM_TYPE_TEAM_REDUCE_OP(prod, _ulonglong, unsigned long long, OSHMEM_OP_PROD, SHMEM_ULLONG)
+SHMEM_TYPE_TEAM_REDUCE_OP(prod, _int8, int8_t, OSHMEM_OP_PROD, SHMEM_INT8_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(prod, _int16, int16_t, OSHMEM_OP_PROD, SHMEM_INT16_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(prod, _int32, int32_t, OSHMEM_OP_PROD, SHMEM_INT32_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(prod, _int64, int64_t, OSHMEM_OP_PROD, SHMEM_INT64_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(prod, _uint8, uint8_t, OSHMEM_OP_PROD, SHMEM_UINT8_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(prod, _uint16, uint16_t, OSHMEM_OP_PROD, SHMEM_UINT16_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(prod, _uint32, uint32_t, OSHMEM_OP_PROD, SHMEM_UINT32_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(prod, _uint64, uint64_t, OSHMEM_OP_PROD, SHMEM_UINT64_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(prod, _size, size_t, OSHMEM_OP_PROD, SHMEM_SIZE_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(prod, _char, char, OSHMEM_OP_PROD, SHMEM_CHAR)
+SHMEM_TYPE_TEAM_REDUCE_OP(prod, _short, short, OSHMEM_OP_PROD, SHMEM_SHORT)
+SHMEM_TYPE_TEAM_REDUCE_OP(prod, _int, int, OSHMEM_OP_PROD, SHMEM_INT)
+SHMEM_TYPE_TEAM_REDUCE_OP(prod, _long, long, OSHMEM_OP_PROD, SHMEM_LONG)
+SHMEM_TYPE_TEAM_REDUCE_OP(prod, _float, float, OSHMEM_OP_PROD, SHMEM_FLOAT)
+SHMEM_TYPE_TEAM_REDUCE_OP(prod, _double, double, OSHMEM_OP_PROD, SHMEM_DOUBLE)
+SHMEM_TYPE_TEAM_REDUCE_OP(prod, _longlong, long long, OSHMEM_OP_PROD, SHMEM_LLONG)
+SHMEM_TYPE_TEAM_REDUCE_OP(prod, _schar, signed char, OSHMEM_OP_PROD, SHMEM_SCHAR)
+SHMEM_TYPE_TEAM_REDUCE_OP(prod, _longdouble, long double, OSHMEM_OP_PROD, SHMEM_LDOUBLE)
+SHMEM_TYPE_TEAM_REDUCE_OP(prod, _ptrdiff, ptrdiff_t, OSHMEM_OP_PROD, SHMEM_PTRDIFF_T)
+SHMEM_TYPE_TEAM_REDUCE_OP(prod, _complexd, double complex, OSHMEM_OP_PROD, SHMEM_COMPLEXD)
+SHMEM_TYPE_TEAM_REDUCE_OP(prod, _complexf, float complex, OSHMEM_OP_PROD, SHMEM_COMPLEXF)
diff --git a/oshmem/shmem/c/shmem_set.c b/oshmem/shmem/c/shmem_set.c
index 30fef842cb3..268ba8221bd 100644
--- a/oshmem/shmem/c/shmem_set.c
+++ b/oshmem/shmem/c/shmem_set.c
@@ -68,6 +68,12 @@
 #pragma weak shmem_ctx_ulonglong_atomic_set = pshmem_ctx_ulonglong_atomic_set
 #pragma weak shmem_ctx_float_atomic_set = pshmem_ctx_float_atomic_set
 #pragma weak shmem_ctx_double_atomic_set = pshmem_ctx_double_atomic_set
+#pragma weak shmem_ctx_int32_atomic_set  	= pshmem_ctx_int32_atomic_set
+#pragma weak shmem_ctx_int64_atomic_set  	= pshmem_ctx_int64_atomic_set
+#pragma weak shmem_ctx_uint32_atomic_set  	= pshmem_ctx_uint32_atomic_set
+#pragma weak shmem_ctx_uint64_atomic_set  	= pshmem_ctx_uint64_atomic_set
+#pragma weak shmem_ctx_size_atomic_set  	= pshmem_ctx_size_atomic_set
+#pragma weak shmem_ctx_ptrdiff_atomic_set  	= pshmem_ctx_ptrdiff_atomic_set
 
 #pragma weak shmem_int_atomic_set = pshmem_int_atomic_set
 #pragma weak shmem_long_atomic_set = pshmem_long_atomic_set
@@ -77,6 +83,12 @@
 #pragma weak shmem_ulonglong_atomic_set = pshmem_ulonglong_atomic_set
 #pragma weak shmem_float_atomic_set = pshmem_float_atomic_set
 #pragma weak shmem_double_atomic_set = pshmem_double_atomic_set
+#pragma weak shmem_int32_atomic_set      	= pshmem_int32_atomic_set
+#pragma weak shmem_int64_atomic_set      	= pshmem_int64_atomic_set
+#pragma weak shmem_uint32_atomic_set     	= pshmem_uint32_atomic_set
+#pragma weak shmem_uint64_atomic_set     	= pshmem_uint64_atomic_set
+#pragma weak shmem_size_atomic_set       	= pshmem_size_atomic_set
+#pragma weak shmem_ptrdiff_atomic_set    	= pshmem_ptrdiff_atomic_set
 
 #pragma weak shmem_int_set = pshmem_int_set
 #pragma weak shmem_long_set = pshmem_long_set
@@ -86,7 +98,7 @@
 
 #pragma weak shmemx_int32_set = pshmemx_int32_set
 #pragma weak shmemx_int64_set = pshmemx_int64_set
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 SHMEM_CTX_TYPE_ATOMIC_SET(_int, int, shmem)
@@ -97,6 +109,12 @@ SHMEM_CTX_TYPE_ATOMIC_SET(_ulong, unsigned long, shmem)
 SHMEM_CTX_TYPE_ATOMIC_SET(_ulonglong, unsigned long long, shmem)
 SHMEM_CTX_TYPE_ATOMIC_SET(_float, float, shmem)
 SHMEM_CTX_TYPE_ATOMIC_SET(_double, double, shmem)
+SHMEM_CTX_TYPE_ATOMIC_SET(_int32, int32_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_SET(_int64, int64_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_SET(_uint32, uint32_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_SET(_uint64, uint64_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_SET(_size, size_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_SET(_ptrdiff, ptrdiff_t, shmem)
 SHMEM_TYPE_ATOMIC_SET(_int, int, shmem)
 SHMEM_TYPE_ATOMIC_SET(_long, long, shmem)
 SHMEM_TYPE_ATOMIC_SET(_longlong, long long, shmem)
@@ -105,6 +123,12 @@ SHMEM_TYPE_ATOMIC_SET(_ulong, unsigned long, shmem)
 SHMEM_TYPE_ATOMIC_SET(_ulonglong, unsigned long long, shmem)
 SHMEM_TYPE_ATOMIC_SET(_float, float, shmem)
 SHMEM_TYPE_ATOMIC_SET(_double, double, shmem)
+SHMEM_TYPE_ATOMIC_SET(_int32, int32_t, shmem)
+SHMEM_TYPE_ATOMIC_SET(_int64, int64_t, shmem)
+SHMEM_TYPE_ATOMIC_SET(_uint32, uint32_t, shmem)
+SHMEM_TYPE_ATOMIC_SET(_uint64, uint64_t, shmem)
+SHMEM_TYPE_ATOMIC_SET(_size, size_t, shmem)
+SHMEM_TYPE_ATOMIC_SET(_ptrdiff, ptrdiff_t, shmem)
 
 /* deprecated APIs */
 #define SHMEM_TYPE_SET(type_name, type, prefix)                     \
diff --git a/oshmem/shmem/c/shmem_set_cache_inv.c b/oshmem/shmem/c/shmem_set_cache_inv.c
index 33f80f53c16..8c3e00f13fb 100644
--- a/oshmem/shmem/c/shmem_set_cache_inv.c
+++ b/oshmem/shmem/c/shmem_set_cache_inv.c
@@ -4,6 +4,8 @@
  * Copyright (c) 2014      Intel, Inc. All rights reserved.
  * Copyright (c) 2017      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -18,17 +20,17 @@
 #include "oshmem/mca/spml/spml.h"
 
 #include "oshmem/shmem/shmem_api_logger.h"
-#include "opal/sys/architecture.h"
+#include "opal/opal_portable_platform.h"
 
 #if OSHMEM_PROFILING
 #include "oshmem/include/pshmem.h"
 #pragma weak shmem_set_cache_inv = pshmem_set_cache_inv
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 void shmem_set_cache_inv(void)
 {
-#if (OPAL_ASSEMBLY_ARCH == OPAL_IA32) || (OPAL_ASSEMBLY_ARCH == OPAL_X86_64)
+#if defined(PLATFORM_ARCH_X86) || defined(PLATFORM_ARCH_X86_64)
     do {SHMEM_API_VERBOSE(10,"shmem_set_cache_inv is not supported by the current CPU architecture");}while (0);
 #else
     /* another implementation */
diff --git a/oshmem/shmem/c/shmem_set_cache_line_inv.c b/oshmem/shmem/c/shmem_set_cache_line_inv.c
index 894782f8a9b..74c612ac6e9 100644
--- a/oshmem/shmem/c/shmem_set_cache_line_inv.c
+++ b/oshmem/shmem/c/shmem_set_cache_line_inv.c
@@ -4,6 +4,8 @@
  * Copyright (c) 2014      Intel, Inc. All rights reserved.
  * Copyright (c) 2017      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -18,17 +20,17 @@
 #include "oshmem/mca/spml/spml.h"
 
 #include "oshmem/shmem/shmem_api_logger.h"
-#include "opal/sys/architecture.h"
+#include "opal/opal_portable_platform.h"
 
 #if OSHMEM_PROFILING
 #include "oshmem/include/pshmem.h"
 #pragma weak shmem_set_cache_line_inv = pshmem_set_cache_line_inv
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 void shmem_set_cache_line_inv(void *target)
 {
-#if (OPAL_ASSEMBLY_ARCH == OPAL_IA32) || (OPAL_ASSEMBLY_ARCH == OPAL_X86_64)
+#if defined(PLATFORM_ARCH_X86) || defined(PLATFORM_ARCH_X86_64)
     do {SHMEM_API_VERBOSE(10,"shmem_set_cache_line_inv is not supported by the current CPU architecture");}while (0);
 #else
     /* another implementation */
diff --git a/oshmem/shmem/c/shmem_set_lock.c b/oshmem/shmem/c/shmem_set_lock.c
index a4864ffe441..90cc9bb706e 100644
--- a/oshmem/shmem/c/shmem_set_lock.c
+++ b/oshmem/shmem/c/shmem_set_lock.c
@@ -1,4 +1,6 @@
 /*
+ * Copyright (c) 2023      NVIDIA Corporation.
+ *                         All rights reserved.
  * Copyright (c) 2013-2016 Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2019      Research Organization for Information Science
@@ -18,14 +20,21 @@
 #include "oshmem/shmem/shmem_api_logger.h"
 #include "oshmem/runtime/runtime.h"
 #include "oshmem/shmem/shmem_lock.h"
+#include "oshmem/runtime/params.h"
 
 #if OSHMEM_PROFILING
 #include "oshmem/include/pshmem.h"
 #pragma weak shmem_set_lock = pshmem_set_lock
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 void shmem_set_lock(volatile long *lock)
 {
-    _shmem_set_lock((void *)lock, sizeof(long));
+    if (oshmem_shmem_enable_mcs_locks) {
+        SHMEM_API_VERBOSE(10, "Set Lock with MCS Lock implementation");
+        _shmem_mcs_set_lock((long *)lock);
+    } else {
+        SHMEM_API_VERBOSE(10, "Set Lock with Ticket Lock implementation");
+        _shmem_set_lock((void *)lock, sizeof(long));
+    }
 }
diff --git a/oshmem/shmem/c/shmem_swap.c b/oshmem/shmem/c/shmem_swap.c
index b6586d0934d..d118b22d5e8 100644
--- a/oshmem/shmem/c/shmem_swap.c
+++ b/oshmem/shmem/c/shmem_swap.c
@@ -70,6 +70,12 @@
 #pragma weak shmem_ctx_ulonglong_atomic_swap = pshmem_ctx_ulonglong_atomic_swap
 #pragma weak shmem_ctx_float_atomic_swap = pshmem_ctx_float_atomic_swap
 #pragma weak shmem_ctx_double_atomic_swap = pshmem_ctx_double_atomic_swap
+#pragma weak shmem_ctx_int32_atomic_swap = pshmem_ctx_int32_atomic_swap
+#pragma weak shmem_ctx_int64_atomic_swap = pshmem_ctx_int64_atomic_swap
+#pragma weak shmem_ctx_uint32_atomic_swap = pshmem_ctx_uint32_atomic_swap
+#pragma weak shmem_ctx_uint64_atomic_swap = pshmem_ctx_uint64_atomic_swap
+#pragma weak shmem_ctx_size_atomic_swap = pshmem_ctx_size_atomic_swap
+#pragma weak shmem_ctx_ptrdiff_atomic_swap = pshmem_ctx_ptrdiff_atomic_swap
 
 #pragma weak shmem_int_atomic_swap = pshmem_int_atomic_swap
 #pragma weak shmem_long_atomic_swap = pshmem_long_atomic_swap
@@ -79,6 +85,12 @@
 #pragma weak shmem_ulonglong_atomic_swap = pshmem_ulonglong_atomic_swap
 #pragma weak shmem_float_atomic_swap = pshmem_float_atomic_swap
 #pragma weak shmem_double_atomic_swap = pshmem_double_atomic_swap
+#pragma weak shmem_int32_atomic_swap = pshmem_int32_atomic_swap
+#pragma weak shmem_int64_atomic_swap = pshmem_int64_atomic_swap
+#pragma weak shmem_uint32_atomic_swap = pshmem_uint32_atomic_swap
+#pragma weak shmem_uint64_atomic_swap = pshmem_uint64_atomic_swap
+#pragma weak shmem_size_atomic_swap = pshmem_size_atomic_swap
+#pragma weak shmem_ptrdiff_atomic_swap = pshmem_ptrdiff_atomic_swap
 
 #pragma weak shmem_int_swap = pshmem_int_swap
 #pragma weak shmem_long_swap = pshmem_long_swap
@@ -88,7 +100,8 @@
 
 #pragma weak shmemx_int32_swap = pshmemx_int32_swap
 #pragma weak shmemx_int64_swap = pshmemx_int64_swap
-#include "oshmem/shmem/c/profile/defines.h"
+
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 SHMEM_CTX_TYPE_ATOMIC_SWAP(_int, int, shmem)
@@ -99,6 +112,12 @@ SHMEM_CTX_TYPE_ATOMIC_SWAP(_ulong, unsigned long, shmem)
 SHMEM_CTX_TYPE_ATOMIC_SWAP(_ulonglong, unsigned long long, shmem)
 SHMEM_CTX_TYPE_ATOMIC_SWAP(_float, float, shmem)
 SHMEM_CTX_TYPE_ATOMIC_SWAP(_double, double, shmem)
+SHMEM_CTX_TYPE_ATOMIC_SWAP(_int32, int32_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_SWAP(_int64, int64_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_SWAP(_uint32, uint32_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_SWAP(_uint64, uint64_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_SWAP(_size, size_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_SWAP(_ptrdiff, ptrdiff_t, shmem)
 SHMEM_TYPE_ATOMIC_SWAP(_int, int, shmem)
 SHMEM_TYPE_ATOMIC_SWAP(_long, long, shmem)
 SHMEM_TYPE_ATOMIC_SWAP(_longlong, long long, shmem)
@@ -107,6 +126,12 @@ SHMEM_TYPE_ATOMIC_SWAP(_ulong, unsigned long, shmem)
 SHMEM_TYPE_ATOMIC_SWAP(_ulonglong, unsigned long long, shmem)
 SHMEM_TYPE_ATOMIC_SWAP(_float, float, shmem)
 SHMEM_TYPE_ATOMIC_SWAP(_double, double, shmem)
+SHMEM_TYPE_ATOMIC_SWAP(_int32, int32_t, shmem)
+SHMEM_TYPE_ATOMIC_SWAP(_int64, int64_t, shmem)
+SHMEM_TYPE_ATOMIC_SWAP(_uint32, uint32_t, shmem)
+SHMEM_TYPE_ATOMIC_SWAP(_uint64, uint64_t, shmem)
+SHMEM_TYPE_ATOMIC_SWAP(_size, size_t, shmem)
+SHMEM_TYPE_ATOMIC_SWAP(_ptrdiff, ptrdiff_t, shmem)
 
 /* deprecated APIs */
 #define SHMEM_TYPE_SWAP(type_name, type, prefix)                    \
diff --git a/oshmem/shmem/c/shmem_swap_nb.c b/oshmem/shmem/c/shmem_swap_nb.c
new file mode 100644
index 00000000000..a8d560ba6b6
--- /dev/null
+++ b/oshmem/shmem/c/shmem_swap_nb.c
@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2021      NVIDIA Corporation.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "oshmem_config.h"
+
+#include "oshmem/constants.h"
+#include "oshmem/include/shmem.h"
+#include "oshmem/include/shmemx.h"
+
+#include "oshmem/runtime/runtime.h"
+
+#include "oshmem/mca/atomic/atomic.h"
+
+/*
+ * It performs an atomic nonblocking swap operation.
+ * The atomic swap routines write value to address target on PE pe, and return the previous
+ * contents of target. The operation must be completed without the possibility of another
+ * process updating target between the time of the fetch and the update.
+ */
+#define DO_SHMEM_TYPE_ATOMIC_SWAP_NBI(ctx, type, fetch, target, value, pe, out_value) do { \
+        int rc = OSHMEM_SUCCESS;                                    \
+        size_t size = 0;                                            \
+                                                                    \
+        RUNTIME_CHECK_INIT();                                       \
+        RUNTIME_CHECK_PE(pe);                                       \
+        RUNTIME_CHECK_ADDR(target);                                 \
+                                                                    \
+        size = sizeof(out_value);                                   \
+        rc = MCA_ATOMIC_CALL(swap_nb(                               \
+            ctx,                                                    \
+            fetch,                                                  \
+            (void*)target,                                          \
+            (void*)&out_value,                                      \
+            OSHMEM_ATOMIC_PTR_2_INT(&value, sizeof(value)),         \
+            size,                                                   \
+            pe));                                                   \
+        RUNTIME_CHECK_RC(rc);                                       \
+    } while (0)
+
+#define SHMEM_CTX_TYPE_ATOMIC_SWAP_NBI(type_name, type, prefix)                 \
+    void prefix##_ctx##type_name##_atomic_swap_nbi(shmem_ctx_t ctx, type *fetch, type *target, type value, int pe) \
+    {                                                                           \
+        type out_value;                                                         \
+        DO_SHMEM_TYPE_ATOMIC_SWAP_NBI(ctx, type, fetch, target, value, pe,      \
+                                  out_value);                                   \
+        return ;                                                                \
+    }
+
+#define SHMEM_TYPE_ATOMIC_SWAP_NBI(type_name, type, prefix)                     \
+    void prefix##type_name##_atomic_swap_nbi(type *fetch, type *target, type value, int pe)\
+    {                                                                           \
+        type out_value;                                                         \
+        DO_SHMEM_TYPE_ATOMIC_SWAP_NBI(oshmem_ctx_default, type, fetch, target,  \
+                                  value, pe, out_value);                        \
+        return ;                                                                \
+    }
+
+#if OSHMEM_PROFILING
+#include "oshmem/include/pshmem.h"
+
+/* Nonblocking Atomic Swap */
+#pragma weak shmem_ctx_double_atomic_swap_nbi            = pshmem_ctx_double_atomic_swap_nbi
+#pragma weak shmem_ctx_float_atomic_swap_nbi             = pshmem_ctx_float_atomic_swap_nbi
+#pragma weak shmem_ctx_int_atomic_swap_nbi               = pshmem_ctx_int_atomic_swap_nbi
+#pragma weak shmem_ctx_long_atomic_swap_nbi              = pshmem_ctx_long_atomic_swap_nbi
+#pragma weak shmem_ctx_longlong_atomic_swap_nbi          = pshmem_ctx_longlong_atomic_swap_nbi
+#pragma weak shmem_ctx_uint_atomic_swap_nbi              = pshmem_ctx_uint_atomic_swap_nbi
+#pragma weak shmem_ctx_ulong_atomic_swap_nbi             = pshmem_ctx_ulong_atomic_swap_nbi
+#pragma weak shmem_ctx_ulonglong_atomic_swap_nbi         = pshmem_ctx_ulonglong_atomic_swap_nbi
+#pragma weak shmem_ctx_int32_atomic_swap_nbi             = pshmem_ctx_int32_atomic_swap_nbi
+#pragma weak shmem_ctx_int64_atomic_swap_nbi             = pshmem_ctx_int64_atomic_swap_nbi
+#pragma weak shmem_ctx_uint32_atomic_swap_nbi            = pshmem_ctx_uint32_atomic_swap_nbi
+#pragma weak shmem_ctx_uint64_atomic_swap_nbi            = pshmem_ctx_uint64_atomic_swap_nbi
+#pragma weak shmem_ctx_size_atomic_swap_nbi              = pshmem_ctx_size_atomic_swap_nbi
+#pragma weak shmem_ctx_ptrdiff_atomic_swap_nbi           = pshmem_ctx_ptrdiff_atomic_swap_nbi
+
+#pragma weak shmem_double_atomic_swap_nbi     			 = pshmem_double_atomic_swap_nbi
+#pragma weak shmem_float_atomic_swap_nbi      			 = pshmem_float_atomic_swap_nbi
+#pragma weak shmem_int_atomic_swap_nbi        			 = pshmem_int_atomic_swap_nbi
+#pragma weak shmem_long_atomic_swap_nbi       			 = pshmem_long_atomic_swap_nbi
+#pragma weak shmem_longlong_atomic_swap_nbi   			 = pshmem_longlong_atomic_swap_nbi
+#pragma weak shmem_uint_atomic_swap_nbi       			 = pshmem_uint_atomic_swap_nbi
+#pragma weak shmem_ulong_atomic_swap_nbi      			 = pshmem_ulong_atomic_swap_nbi
+#pragma weak shmem_ulonglong_atomic_swap_nbi  			 = pshmem_ulonglong_atomic_swap_nbi
+#pragma weak shmem_int32_atomic_swap_nbi      			 = pshmem_int32_atomic_swap_nbi
+#pragma weak shmem_int64_atomic_swap_nbi      			 = pshmem_int64_atomic_swap_nbi
+#pragma weak shmem_uint32_atomic_swap_nbi     			 = pshmem_uint32_atomic_swap_nbi
+#pragma weak shmem_uint64_atomic_swap_nbi     			 = pshmem_uint64_atomic_swap_nbi
+#pragma weak shmem_size_atomic_swap_nbi       			 = pshmem_size_atomic_swap_nbi
+#pragma weak shmem_ptrdiff_atomic_swap_nbi    			 = pshmem_ptrdiff_atomic_swap_nbi
+
+#include "oshmem/shmem/c/profile-defines.h"
+#endif
+
+SHMEM_CTX_TYPE_ATOMIC_SWAP_NBI(_int, int, shmem)
+SHMEM_CTX_TYPE_ATOMIC_SWAP_NBI(_long, long, shmem)
+SHMEM_CTX_TYPE_ATOMIC_SWAP_NBI(_longlong, long long, shmem)
+SHMEM_CTX_TYPE_ATOMIC_SWAP_NBI(_uint, unsigned int, shmem)
+SHMEM_CTX_TYPE_ATOMIC_SWAP_NBI(_ulong, unsigned long, shmem)
+SHMEM_CTX_TYPE_ATOMIC_SWAP_NBI(_ulonglong, unsigned long long, shmem)
+SHMEM_CTX_TYPE_ATOMIC_SWAP_NBI(_float, float, shmem)
+SHMEM_CTX_TYPE_ATOMIC_SWAP_NBI(_double, double, shmem)
+SHMEM_CTX_TYPE_ATOMIC_SWAP_NBI(_int32, int32_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_SWAP_NBI(_int64, int64_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_SWAP_NBI(_uint32, uint32_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_SWAP_NBI(_uint64, uint64_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_SWAP_NBI(_size, size_t, shmem)
+SHMEM_CTX_TYPE_ATOMIC_SWAP_NBI(_ptrdiff, ptrdiff_t, shmem)
+SHMEM_TYPE_ATOMIC_SWAP_NBI(_int, int, shmem)
+SHMEM_TYPE_ATOMIC_SWAP_NBI(_long, long, shmem)
+SHMEM_TYPE_ATOMIC_SWAP_NBI(_longlong, long long, shmem)
+SHMEM_TYPE_ATOMIC_SWAP_NBI(_uint, unsigned int, shmem)
+SHMEM_TYPE_ATOMIC_SWAP_NBI(_ulong, unsigned long, shmem)
+SHMEM_TYPE_ATOMIC_SWAP_NBI(_ulonglong, unsigned long long, shmem)
+SHMEM_TYPE_ATOMIC_SWAP_NBI(_float, float, shmem)
+SHMEM_TYPE_ATOMIC_SWAP_NBI(_double, double, shmem)
+SHMEM_TYPE_ATOMIC_SWAP_NBI(_int32, int32_t, shmem)
+SHMEM_TYPE_ATOMIC_SWAP_NBI(_int64, int64_t, shmem)
+SHMEM_TYPE_ATOMIC_SWAP_NBI(_uint32, uint32_t, shmem)
+SHMEM_TYPE_ATOMIC_SWAP_NBI(_uint64, uint64_t, shmem)
+SHMEM_TYPE_ATOMIC_SWAP_NBI(_size, size_t, shmem)
+SHMEM_TYPE_ATOMIC_SWAP_NBI(_ptrdiff, ptrdiff_t, shmem)
diff --git a/oshmem/shmem/c/shmem_sync.c b/oshmem/shmem/c/shmem_sync.c
index 9f2b983aa19..1cd7266fc3a 100644
--- a/oshmem/shmem/c/shmem_sync.c
+++ b/oshmem/shmem/c/shmem_sync.c
@@ -22,12 +22,12 @@
 
 #if OSHMEM_PROFILING
 #include "oshmem/include/pshmem.h"
-#pragma weak shmem_sync = pshmem_sync
 #pragma weak shmem_sync_all = pshmem_sync_all
-#include "oshmem/shmem/c/profile/defines.h"
+#pragma weak shmem_sync_deprecated = pshmem_sync_deprecated
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
-void shmem_sync(int PE_start, int logPE_stride, int PE_size, long *pSync)
+void shmem_sync_deprecated(int PE_start, int logPE_stride, int PE_size, long *pSync)
 {
     int rc;
     oshmem_group_t* group;
diff --git a/oshmem/shmem/c/shmem_team.c b/oshmem/shmem/c/shmem_team.c
new file mode 100644
index 00000000000..0a4aacfa48d
--- /dev/null
+++ b/oshmem/shmem/c/shmem_team.c
@@ -0,0 +1,153 @@
+/*
+ * Copyright (c) 2021      NVIDIA Corporation.
+ *                         All rights reserved.
+ * Copyright (c) 2019      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "oshmem_config.h"
+
+#include "oshmem/constants.h"
+#include "oshmem/include/shmem.h"
+#include "oshmem/include/shmemx.h"
+
+#include "oshmem/runtime/runtime.h"
+
+#include "oshmem/mca/spml/spml.h"
+
+#if OSHMEM_PROFILING
+#include "oshmem/include/pshmem.h"
+/*
+ * Team management routines
+ */
+#pragma weak shmem_team_sync             		= pshmem_team_sync
+#pragma weak shmem_team_my_pe             		= pshmem_team_my_pe
+#pragma weak shmem_team_n_pes             		= pshmem_team_n_pes
+#pragma weak shmem_team_get_config        		= pshmem_team_get_config
+#pragma weak shmem_team_translate_pe      		= pshmem_team_translate_pe
+#pragma weak shmem_team_split_strided     		= pshmem_team_split_strided
+#pragma weak shmem_team_split_2d          		= pshmem_team_split_2d
+#pragma weak shmem_team_destroy           		= pshmem_team_destroy
+#pragma weak shmem_ctx_get_team           		= pshmem_ctx_get_team
+#pragma weak shmem_team_create_ctx                      = pshmem_team_create_ctx
+
+#include "oshmem/shmem/c/profile-defines.h"
+#endif
+
+void shmem_team_sync(shmem_team_t team)
+{
+    int rc = 0;
+
+    RUNTIME_CHECK_INIT();
+
+    rc = MCA_SPML_CALL(team_sync(team));
+    RUNTIME_CHECK_IMPL_RC(rc);
+
+    return ;
+}
+
+int shmem_team_my_pe(shmem_team_t team)
+{
+    RUNTIME_CHECK_INIT();
+
+    return MCA_SPML_CALL(team_my_pe(team));
+}
+
+int shmem_team_n_pes(shmem_team_t team)
+{
+    int rc = 0;
+
+    RUNTIME_CHECK_INIT();
+
+    rc = MCA_SPML_CALL(team_n_pes(team));
+    RUNTIME_CHECK_IMPL_RC(rc);
+
+    return rc;
+}
+int shmem_team_get_config(shmem_team_t team, long config_mask, shmem_team_config_t *config)
+{
+    int rc = 0;
+
+    RUNTIME_CHECK_INIT();
+
+    rc = MCA_SPML_CALL(team_get_config(team, config_mask, config));
+    RUNTIME_CHECK_RC(rc);
+
+    return rc;
+}
+int shmem_team_translate_pe(shmem_team_t src_team, int src_pe, shmem_team_t dest_team)
+{
+    RUNTIME_CHECK_INIT();
+
+    return MCA_SPML_CALL(team_translate_pe(src_team, src_pe, dest_team));
+}
+
+int shmem_team_split_strided (shmem_team_t parent_team, int start, int stride,
+        int size, const shmem_team_config_t *config, long config_mask,
+        shmem_team_t *new_team)
+{
+    int rc = 0;
+
+    RUNTIME_CHECK_INIT();
+
+    rc = MCA_SPML_CALL(team_split_strided(parent_team, start, stride, size,
+                config, config_mask, new_team));
+    RUNTIME_CHECK_RC(rc);
+
+    return rc;
+}
+
+int shmem_team_split_2d (shmem_team_t parent_team, int xrange, const
+        shmem_team_config_t *xaxis_config, long xaxis_mask, shmem_team_t
+        *xaxis_team, const shmem_team_config_t *yaxis_config, long yaxis_mask,
+        shmem_team_t *yaxis_team)
+{
+    int rc = 0;
+
+    RUNTIME_CHECK_INIT();
+
+    rc = MCA_SPML_CALL(team_split_2d(parent_team, xrange, xaxis_config,
+                xaxis_mask, xaxis_team, yaxis_config, yaxis_mask, yaxis_team));
+    RUNTIME_CHECK_RC(rc);
+
+    return rc;
+}
+
+void shmem_team_destroy(shmem_team_t team)
+{
+    int rc = 0;
+
+    RUNTIME_CHECK_INIT();
+
+    rc = MCA_SPML_CALL(team_destroy(team));
+    RUNTIME_CHECK_RC(rc);
+
+    return ;
+}
+
+int shmem_ctx_get_team(shmem_ctx_t ctx, shmem_team_t *team)
+{
+    int rc = 0;
+
+    RUNTIME_CHECK_INIT();
+
+    rc = MCA_SPML_CALL(team_get(ctx, team));
+    RUNTIME_CHECK_RC(rc);
+
+    return rc;
+}
+
+int shmem_team_create_ctx(shmem_team_t team, long options, shmem_ctx_t *ctx)
+{
+    int rc = 0;
+
+    RUNTIME_CHECK_INIT();
+
+    rc = MCA_SPML_CALL(team_create_ctx(team, options, ctx));
+    RUNTIME_CHECK_RC(rc);
+
+    return rc;
+}
diff --git a/oshmem/shmem/c/shmem_test_ivars.c b/oshmem/shmem/c/shmem_test_ivars.c
new file mode 100644
index 00000000000..c631f825e3b
--- /dev/null
+++ b/oshmem/shmem/c/shmem_test_ivars.c
@@ -0,0 +1,306 @@
+/*
+ * Copyright (c) 2021      NVIDIA Corporation.
+ *                         All rights reserved.
+ * Copyright (c) 2019      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "oshmem_config.h"
+
+#include "oshmem/constants.h"
+#include "oshmem/include/shmem.h"
+#include "oshmem/include/shmemx.h"
+
+#include "oshmem/runtime/runtime.h"
+
+#include "oshmem/mca/spml/spml.h"
+
+
+#if OSHMEM_PROFILING
+#include "oshmem/include/pshmem.h"
+
+#pragma weak shmem_short_test_all       = pshmem_short_test_all
+#pragma weak shmem_ushort_test_all      = pshmem_ushort_test_all
+#pragma weak shmem_int_test_all         = pshmem_int_test_all
+#pragma weak shmem_long_test_all        = pshmem_long_test_all
+#pragma weak shmem_longlong_test_all    = pshmem_longlong_test_all
+#pragma weak shmem_uint_test_all        = pshmem_uint_test_all
+#pragma weak shmem_ulong_test_all       = pshmem_ulong_test_all
+#pragma weak shmem_ulonglong_test_all   = pshmem_ulonglong_test_all
+#pragma weak shmem_int32_test_all       = pshmem_int32_test_all
+#pragma weak shmem_int64_test_all       = pshmem_int64_test_all
+#pragma weak shmem_uint32_test_all      = pshmem_uint32_test_all
+#pragma weak shmem_uint64_test_all      = pshmem_uint64_test_all
+#pragma weak shmem_size_test_all        = pshmem_size_test_all
+#pragma weak shmem_ptrdiff_test_all     = pshmem_ptrdiff_test_all
+
+
+#pragma weak shmem_short_test_any       = pshmem_short_test_any
+#pragma weak shmem_ushort_test_any      = pshmem_ushort_test_any
+#pragma weak shmem_int_test_any         = pshmem_int_test_any
+#pragma weak shmem_long_test_any        = pshmem_long_test_any
+#pragma weak shmem_longlong_test_any    = pshmem_longlong_test_any
+#pragma weak shmem_uint_test_any        = pshmem_uint_test_any
+#pragma weak shmem_ulong_test_any       = pshmem_ulong_test_any
+#pragma weak shmem_ulonglong_test_any   = pshmem_ulonglong_test_any
+#pragma weak shmem_int32_test_any       = pshmem_int32_test_any
+#pragma weak shmem_int64_test_any       = pshmem_int64_test_any
+#pragma weak shmem_uint32_test_any      = pshmem_uint32_test_any
+#pragma weak shmem_uint64_test_any      = pshmem_uint64_test_any
+#pragma weak shmem_size_test_any        = pshmem_size_test_any
+#pragma weak shmem_ptrdiff_test_any     = pshmem_ptrdiff_test_any
+
+#pragma weak shmem_short_test_some       = pshmem_short_test_some
+#pragma weak shmem_ushort_test_some      = pshmem_ushort_test_some
+#pragma weak shmem_int_test_some         = pshmem_int_test_some
+#pragma weak shmem_long_test_some        = pshmem_long_test_some
+#pragma weak shmem_longlong_test_some    = pshmem_longlong_test_some
+#pragma weak shmem_uint_test_some        = pshmem_uint_test_some
+#pragma weak shmem_ulong_test_some       = pshmem_ulong_test_some
+#pragma weak shmem_ulonglong_test_some   = pshmem_ulonglong_test_some
+#pragma weak shmem_int32_test_some       = pshmem_int32_test_some
+#pragma weak shmem_int64_test_some       = pshmem_int64_test_some
+#pragma weak shmem_uint32_test_some      = pshmem_uint32_test_some
+#pragma weak shmem_uint64_test_some      = pshmem_uint64_test_some
+#pragma weak shmem_size_test_some        = pshmem_size_test_some
+#pragma weak shmem_ptrdiff_test_some     = pshmem_ptrdiff_test_some
+
+
+
+#pragma weak shmem_short_test_all_vector       = pshmem_short_test_all_vector
+#pragma weak shmem_ushort_test_all_vector      = pshmem_ushort_test_all_vector
+#pragma weak shmem_int_test_all_vector         = pshmem_int_test_all_vector
+#pragma weak shmem_long_test_all_vector        = pshmem_long_test_all_vector
+#pragma weak shmem_longlong_test_all_vector    = pshmem_longlong_test_all_vector
+#pragma weak shmem_uint_test_all_vector        = pshmem_uint_test_all_vector
+#pragma weak shmem_ulong_test_all_vector       = pshmem_ulong_test_all_vector
+#pragma weak shmem_ulonglong_test_all_vector   = pshmem_ulonglong_test_all_vector
+#pragma weak shmem_int32_test_all_vector       = pshmem_int32_test_all_vector
+#pragma weak shmem_int64_test_all_vector       = pshmem_int64_test_all_vector
+#pragma weak shmem_uint32_test_all_vector      = pshmem_uint32_test_all_vector
+#pragma weak shmem_uint64_test_all_vector      = pshmem_uint64_test_all_vector
+#pragma weak shmem_size_test_all_vector        = pshmem_size_test_all_vector
+#pragma weak shmem_ptrdiff_test_all_vector     = pshmem_ptrdiff_test_all_vector
+
+
+#pragma weak shmem_short_test_any_vector       = pshmem_short_test_any_vector
+#pragma weak shmem_ushort_test_any_vector      = pshmem_ushort_test_any_vector
+#pragma weak shmem_int_test_any_vector         = pshmem_int_test_any_vector
+#pragma weak shmem_long_test_any_vector        = pshmem_long_test_any_vector
+#pragma weak shmem_longlong_test_any_vector    = pshmem_longlong_test_any_vector
+#pragma weak shmem_uint_test_any_vector        = pshmem_uint_test_any_vector
+#pragma weak shmem_ulong_test_any_vector       = pshmem_ulong_test_any_vector
+#pragma weak shmem_ulonglong_test_any_vector   = pshmem_ulonglong_test_any_vector
+#pragma weak shmem_int32_test_any_vector       = pshmem_int32_test_any_vector
+#pragma weak shmem_int64_test_any_vector       = pshmem_int64_test_any_vector
+#pragma weak shmem_uint32_test_any_vector      = pshmem_uint32_test_any_vector
+#pragma weak shmem_uint64_test_any_vector      = pshmem_uint64_test_any_vector
+#pragma weak shmem_size_test_any_vector        = pshmem_size_test_any_vector
+#pragma weak shmem_ptrdiff_test_any_vector     = pshmem_ptrdiff_test_any_vector
+
+
+#pragma weak shmem_short_test_some_vector       = pshmem_short_test_some_vector
+#pragma weak shmem_ushort_test_some_vector      = pshmem_ushort_test_some_vector
+#pragma weak shmem_int_test_some_vector         = pshmem_int_test_some_vector
+#pragma weak shmem_long_test_some_vector        = pshmem_long_test_some_vector
+#pragma weak shmem_longlong_test_some_vector    = pshmem_longlong_test_some_vector
+#pragma weak shmem_uint_test_some_vector        = pshmem_uint_test_some_vector
+#pragma weak shmem_ulong_test_some_vector       = pshmem_ulong_test_some_vector
+#pragma weak shmem_ulonglong_test_some_vector   = pshmem_ulonglong_test_some_vector
+#pragma weak shmem_int32_test_some_vector       = pshmem_int32_test_some_vector
+#pragma weak shmem_int64_test_some_vector       = pshmem_int64_test_some_vector
+#pragma weak shmem_uint32_test_some_vector      = pshmem_uint32_test_some_vector
+#pragma weak shmem_uint64_test_some_vector      = pshmem_uint64_test_some_vector
+#pragma weak shmem_size_test_some_vector        = pshmem_size_test_some_vector
+#pragma weak shmem_ptrdiff_test_some_vector     = pshmem_ptrdiff_test_some_vector
+
+
+#include "oshmem/shmem/c/profile-defines.h"
+#endif
+
+#define SHMEM_TYPE_TEST_ALL(type_name, type, code, prefix)          \
+    int prefix##type_name##_test_all(volatile type *ivars, size_t nelems, const int *status, int cmp, type value)   \
+    {                                                               \
+        int rc;                                                     \
+                                                                    \
+        RUNTIME_CHECK_INIT();                                       \
+                                                                    \
+        rc = MCA_SPML_CALL(test_all(                                \
+            (void*)ivars,                                           \
+            cmp,                                                    \
+            (void*)&value,                                          \
+            nelems, status, code));                                 \
+        RUNTIME_CHECK_IMPL_RC(rc);                                  \
+                                                                    \
+        return rc;                                                  \
+    }
+
+#define SHMEM_TYPE_TEST_ANY(type_name, type, code, prefix)          \
+    size_t prefix##type_name##_test_any(volatile type *ivars, size_t nelems, const int *status, int cmp, type value)   \
+    {                                                               \
+        RUNTIME_CHECK_INIT();                                       \
+                                                                    \
+        return MCA_SPML_CALL(test_any(                              \
+                                      (void*)ivars,                 \
+                                      cmp,                          \
+                                      (void*)&value,                \
+                                      nelems, status, code));       \
+    }
+
+
+#define SHMEM_TYPE_TEST_SOME(type_name, type, code, prefix)         \
+    size_t  prefix##type_name##_test_some(volatile type *ivars, size_t nelems, size_t *indices, const int *status, int cmp, type value)   \
+    {                                                               \
+        RUNTIME_CHECK_INIT();                                       \
+                                                                    \
+        return MCA_SPML_CALL(test_some(                             \
+            (void*)ivars,                                           \
+            cmp,                                                    \
+            (void*)&value,                                          \
+            nelems, indices, status, code));                        \
+    }
+
+#define SHMEM_TYPE_TEST_ANY_VECTOR(type_name, type, code, prefix)   \
+    size_t prefix##type_name##_test_any_vector(volatile type *ivars, size_t nelems, const int *status, int cmp, type *values)   \
+    {                                                               \
+        RUNTIME_CHECK_INIT();                                       \
+                                                                    \
+        return MCA_SPML_CALL(test_any_vector(                       \
+            (void*)ivars,                                           \
+            cmp,                                                    \
+            (void*)values,                                          \
+            nelems, status, code));                                 \
+    }
+
+#define SHMEM_TYPE_TEST_SOME_VECTOR(type_name, type, code, prefix)  \
+    size_t prefix##type_name##_test_some_vector(volatile type *ivars, size_t nelems, size_t *indices, const int *status, int cmp, type *values)   \
+    {                                                               \
+        RUNTIME_CHECK_INIT();                                       \
+                                                                    \
+        return MCA_SPML_CALL(test_some_vector(                      \
+            (void*)ivars,                                           \
+            cmp,                                                    \
+            (void*)values,                                          \
+            nelems, indices, status, code));                        \
+    }
+
+
+#define SHMEM_TYPE_TEST_ALL_VECTOR(type_name, type, code, prefix)   \
+   int prefix##type_name##_test_all_vector(volatile type *ivars, size_t nelems, const int *status, int cmp, type *values)   \
+    {                                                               \
+        int rc;                                                     \
+                                                                    \
+        RUNTIME_CHECK_INIT();                                       \
+                                                                    \
+        rc = MCA_SPML_CALL(test_all_vector(                         \
+            (void*)ivars,                                           \
+            cmp,                                                    \
+            (void*)values,                                          \
+            nelems, status, code));                                 \
+        RUNTIME_CHECK_IMPL_RC(rc);                                  \
+                                                                    \
+        return rc;                                                  \
+    }
+
+
+
+SHMEM_TYPE_TEST_ALL(_short,  short, SHMEM_SHORT, shmem)
+SHMEM_TYPE_TEST_ALL(_ushort,  unsigned short, SHMEM_USHORT, shmem)
+SHMEM_TYPE_TEST_ALL(_int,  int, SHMEM_INT, shmem)
+SHMEM_TYPE_TEST_ALL(_long,  long, SHMEM_LONG, shmem)
+SHMEM_TYPE_TEST_ALL(_longlong,  long long, SHMEM_LLONG, shmem)
+SHMEM_TYPE_TEST_ALL(_uint,  unsigned int, SHMEM_UINT, shmem)
+SHMEM_TYPE_TEST_ALL(_ulong,  unsigned long, SHMEM_ULONG, shmem)
+SHMEM_TYPE_TEST_ALL(_ulonglong,  unsigned long long, SHMEM_ULLONG, shmem)
+SHMEM_TYPE_TEST_ALL(_int32,  int32_t, SHMEM_INT32_T, shmem)
+SHMEM_TYPE_TEST_ALL(_int64,  int64_t, SHMEM_INT64_T, shmem)
+SHMEM_TYPE_TEST_ALL(_uint32,  uint32_t, SHMEM_UINT32_T, shmem)
+SHMEM_TYPE_TEST_ALL(_uint64,  uint64_t, SHMEM_UINT64_T, shmem)
+SHMEM_TYPE_TEST_ALL(_size,  size_t, SHMEM_SIZE_T, shmem)
+SHMEM_TYPE_TEST_ALL(_ptrdiff,  ptrdiff_t, SHMEM_PTRDIFF_T, shmem)
+
+SHMEM_TYPE_TEST_ANY(_short,  short, SHMEM_SHORT, shmem)
+SHMEM_TYPE_TEST_ANY(_ushort,  unsigned short, SHMEM_USHORT, shmem)
+SHMEM_TYPE_TEST_ANY(_int,  int, SHMEM_INT, shmem)
+SHMEM_TYPE_TEST_ANY(_long,  long, SHMEM_LONG, shmem)
+SHMEM_TYPE_TEST_ANY(_longlong,  long long, SHMEM_LLONG, shmem)
+SHMEM_TYPE_TEST_ANY(_uint,  unsigned int, SHMEM_UINT, shmem)
+SHMEM_TYPE_TEST_ANY(_ulong,  unsigned long, SHMEM_ULONG, shmem)
+SHMEM_TYPE_TEST_ANY(_ulonglong,  unsigned long long, SHMEM_ULLONG, shmem)
+SHMEM_TYPE_TEST_ANY(_int32,  int32_t, SHMEM_INT32_T, shmem)
+SHMEM_TYPE_TEST_ANY(_int64,  int64_t, SHMEM_INT64_T, shmem)
+SHMEM_TYPE_TEST_ANY(_uint32,  uint32_t, SHMEM_UINT32_T, shmem)
+SHMEM_TYPE_TEST_ANY(_uint64,  uint64_t, SHMEM_UINT64_T, shmem)
+SHMEM_TYPE_TEST_ANY(_size,  size_t, SHMEM_SIZE_T, shmem)
+SHMEM_TYPE_TEST_ANY(_ptrdiff,  ptrdiff_t, SHMEM_PTRDIFF_T, shmem)
+
+
+SHMEM_TYPE_TEST_SOME(_short,  short, SHMEM_SHORT, shmem)
+SHMEM_TYPE_TEST_SOME(_ushort,  unsigned short, SHMEM_USHORT, shmem)
+SHMEM_TYPE_TEST_SOME(_int,  int, SHMEM_INT, shmem)
+SHMEM_TYPE_TEST_SOME(_long,  long, SHMEM_LONG, shmem)
+SHMEM_TYPE_TEST_SOME(_longlong,  long long, SHMEM_LLONG, shmem)
+SHMEM_TYPE_TEST_SOME(_uint,  unsigned int, SHMEM_UINT, shmem)
+SHMEM_TYPE_TEST_SOME(_ulong,  unsigned long, SHMEM_ULONG, shmem)
+SHMEM_TYPE_TEST_SOME(_ulonglong,  unsigned long long, SHMEM_ULLONG, shmem)
+SHMEM_TYPE_TEST_SOME(_int32,  int32_t, SHMEM_INT32_T, shmem)
+SHMEM_TYPE_TEST_SOME(_int64,  int64_t, SHMEM_INT64_T, shmem)
+SHMEM_TYPE_TEST_SOME(_uint32,  uint32_t, SHMEM_UINT32_T, shmem)
+SHMEM_TYPE_TEST_SOME(_uint64,  uint64_t, SHMEM_UINT64_T, shmem)
+SHMEM_TYPE_TEST_SOME(_size,  size_t, SHMEM_SIZE_T, shmem)
+SHMEM_TYPE_TEST_SOME(_ptrdiff,  ptrdiff_t, SHMEM_PTRDIFF_T, shmem)
+
+
+SHMEM_TYPE_TEST_ALL_VECTOR(_short,  short, SHMEM_SHORT, shmem)
+SHMEM_TYPE_TEST_ALL_VECTOR(_ushort,  unsigned short, SHMEM_USHORT, shmem)
+SHMEM_TYPE_TEST_ALL_VECTOR(_int,  int, SHMEM_INT, shmem)
+SHMEM_TYPE_TEST_ALL_VECTOR(_long,  long, SHMEM_LONG, shmem)
+SHMEM_TYPE_TEST_ALL_VECTOR(_longlong,  long long, SHMEM_LLONG, shmem)
+SHMEM_TYPE_TEST_ALL_VECTOR(_uint,  unsigned int, SHMEM_UINT, shmem)
+SHMEM_TYPE_TEST_ALL_VECTOR(_ulong,  unsigned long, SHMEM_ULONG, shmem)
+SHMEM_TYPE_TEST_ALL_VECTOR(_ulonglong,  unsigned long long, SHMEM_ULLONG, shmem)
+SHMEM_TYPE_TEST_ALL_VECTOR(_int32,  int32_t, SHMEM_INT32_T, shmem)
+SHMEM_TYPE_TEST_ALL_VECTOR(_int64,  int64_t, SHMEM_INT64_T, shmem)
+SHMEM_TYPE_TEST_ALL_VECTOR(_uint32,  uint32_t, SHMEM_UINT32_T, shmem)
+SHMEM_TYPE_TEST_ALL_VECTOR(_uint64,  uint64_t, SHMEM_UINT64_T, shmem)
+SHMEM_TYPE_TEST_ALL_VECTOR(_size,  size_t, SHMEM_SIZE_T, shmem)
+SHMEM_TYPE_TEST_ALL_VECTOR(_ptrdiff,  ptrdiff_t, SHMEM_PTRDIFF_T, shmem)
+
+
+
+SHMEM_TYPE_TEST_ANY_VECTOR(_short,  short, SHMEM_SHORT, shmem)
+SHMEM_TYPE_TEST_ANY_VECTOR(_ushort,  unsigned short, SHMEM_USHORT, shmem)
+SHMEM_TYPE_TEST_ANY_VECTOR(_int,  int, SHMEM_INT, shmem)
+SHMEM_TYPE_TEST_ANY_VECTOR(_long,  long, SHMEM_LONG, shmem)
+SHMEM_TYPE_TEST_ANY_VECTOR(_longlong,  long long, SHMEM_LLONG, shmem)
+SHMEM_TYPE_TEST_ANY_VECTOR(_uint,  unsigned int, SHMEM_UINT, shmem)
+SHMEM_TYPE_TEST_ANY_VECTOR(_ulong,  unsigned long, SHMEM_ULONG, shmem)
+SHMEM_TYPE_TEST_ANY_VECTOR(_ulonglong,  unsigned long long, SHMEM_ULLONG, shmem)
+SHMEM_TYPE_TEST_ANY_VECTOR(_int32,  int32_t, SHMEM_INT32_T, shmem)
+SHMEM_TYPE_TEST_ANY_VECTOR(_int64,  int64_t, SHMEM_INT64_T, shmem)
+SHMEM_TYPE_TEST_ANY_VECTOR(_uint32,  uint32_t, SHMEM_UINT32_T, shmem)
+SHMEM_TYPE_TEST_ANY_VECTOR(_uint64,  uint64_t, SHMEM_UINT64_T, shmem)
+SHMEM_TYPE_TEST_ANY_VECTOR(_size,  size_t, SHMEM_SIZE_T, shmem)
+SHMEM_TYPE_TEST_ANY_VECTOR(_ptrdiff,  ptrdiff_t, SHMEM_PTRDIFF_T, shmem)
+
+
+
+SHMEM_TYPE_TEST_SOME_VECTOR(_short,  short, SHMEM_SHORT, shmem)
+SHMEM_TYPE_TEST_SOME_VECTOR(_ushort,  unsigned short, SHMEM_USHORT, shmem)
+SHMEM_TYPE_TEST_SOME_VECTOR(_int,  int, SHMEM_INT, shmem)
+SHMEM_TYPE_TEST_SOME_VECTOR(_long,  long, SHMEM_LONG, shmem)
+SHMEM_TYPE_TEST_SOME_VECTOR(_longlong,  long long, SHMEM_LLONG, shmem)
+SHMEM_TYPE_TEST_SOME_VECTOR(_uint,  unsigned int, SHMEM_UINT, shmem)
+SHMEM_TYPE_TEST_SOME_VECTOR(_ulong,  unsigned long, SHMEM_ULONG, shmem)
+SHMEM_TYPE_TEST_SOME_VECTOR(_ulonglong,  unsigned long long, SHMEM_ULLONG, shmem)
+SHMEM_TYPE_TEST_SOME_VECTOR(_int32,  int32_t, SHMEM_INT32_T, shmem)
+SHMEM_TYPE_TEST_SOME_VECTOR(_int64,  int64_t, SHMEM_INT64_T, shmem)
+SHMEM_TYPE_TEST_SOME_VECTOR(_uint32,  uint32_t, SHMEM_UINT32_T, shmem)
+SHMEM_TYPE_TEST_SOME_VECTOR(_uint64,  uint64_t, SHMEM_UINT64_T, shmem)
+SHMEM_TYPE_TEST_SOME_VECTOR(_size,  size_t, SHMEM_SIZE_T, shmem)
+SHMEM_TYPE_TEST_SOME_VECTOR(_ptrdiff,  ptrdiff_t, SHMEM_PTRDIFF_T, shmem)
+
+
diff --git a/oshmem/shmem/c/shmem_test_lock.c b/oshmem/shmem/c/shmem_test_lock.c
index 0d587da3969..0cae5576f5f 100644
--- a/oshmem/shmem/c/shmem_test_lock.c
+++ b/oshmem/shmem/c/shmem_test_lock.c
@@ -1,4 +1,6 @@
 /*
+ * Copyright (c) 2023      NVIDIA Corporation.
+ *                         All rights reserved.
  * Copyright (c) 2013-2016 Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2019      Research Organization for Information Science
@@ -18,15 +20,22 @@
 #include "oshmem/include/shmem.h"
 #include "oshmem/shmem/shmem_api_logger.h"
 #include "oshmem/runtime/runtime.h"
+#include "oshmem/runtime/params.h"
 #include "oshmem/shmem/shmem_lock.h"
 
 #if OSHMEM_PROFILING
 #include "oshmem/include/pshmem.h"
 #pragma weak shmem_test_lock = pshmem_test_lock
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 int shmem_test_lock(volatile long *lock)
 {
-    return _shmem_test_lock((void *)lock, sizeof(long));
+    if (oshmem_shmem_enable_mcs_locks) {
+        SHMEM_API_VERBOSE(10, "Test lock using MCS Lock implementation");
+        return _shmem_mcs_test_lock((long *)lock);
+    } else {
+        SHMEM_API_VERBOSE(10, "Test_lock using Ticket Lock implementation");
+        return _shmem_test_lock((void *)lock, sizeof(long));
+    }
 }
diff --git a/oshmem/shmem/c/shmem_udcflush.c b/oshmem/shmem/c/shmem_udcflush.c
index eb0fa227434..a60a7f54411 100644
--- a/oshmem/shmem/c/shmem_udcflush.c
+++ b/oshmem/shmem/c/shmem_udcflush.c
@@ -4,6 +4,8 @@
  * Copyright (c) 2014      Intel, Inc. All rights reserved.
  * Copyright (c) 2017      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -17,17 +19,17 @@
 
 #include "oshmem/mca/spml/spml.h"
 #include "oshmem/shmem/shmem_api_logger.h"
-#include "opal/sys/architecture.h"
+#include "opal/opal_portable_platform.h"
 
 #if OSHMEM_PROFILING
 #include "oshmem/include/pshmem.h"
 #pragma weak shmem_udcflush = pshmem_udcflush
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 void shmem_udcflush(void)
 {
-#if (OPAL_ASSEMBLY_ARCH == OPAL_IA32) || (OPAL_ASSEMBLY_ARCH == OPAL_X86_64)
+#if defined(PLATFORM_ARCH_X86) || defined(PLATFORM_ARCH_X86_64)
     do {SHMEM_API_VERBOSE(10,"shmem_udcflush is not supported by the current CPU architecture");}while (0);
 #else
     /* another implementation */
diff --git a/oshmem/shmem/c/shmem_udcflush_line.c b/oshmem/shmem/c/shmem_udcflush_line.c
index e6302e93db8..7603a93bd77 100644
--- a/oshmem/shmem/c/shmem_udcflush_line.c
+++ b/oshmem/shmem/c/shmem_udcflush_line.c
@@ -4,6 +4,8 @@
  * Copyright (c) 2014      Intel, Inc. All rights reserved.
  * Copyright (c) 2017      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -16,17 +18,17 @@
 #include "oshmem/include/shmem.h"
 
 #include "oshmem/shmem/shmem_api_logger.h"
-#include "opal/sys/architecture.h"
+#include "opal/opal_portable_platform.h"
 
 #if OSHMEM_PROFILING
 #include "oshmem/include/pshmem.h"
 #pragma weak shmem_udcflush_line = pshmem_udcflush_line
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 void shmem_udcflush_line(void *target)
 {
-#if (OPAL_ASSEMBLY_ARCH == OPAL_IA32) || (OPAL_ASSEMBLY_ARCH == OPAL_X86_64)
+#if defined(PLATFORM_ARCH_X86) || defined(PLATFORM_ARCH_X86_64)
     do {SHMEM_API_VERBOSE(10,"shmem_udcflush_line is not supported by the current CPU architecture");}while (0);
 #else
     /* another implementation */
diff --git a/oshmem/shmem/c/shmem_wait.c b/oshmem/shmem/c/shmem_wait.c
index 32d0f53c4ba..fc2e74d3dcc 100644
--- a/oshmem/shmem/c/shmem_wait.c
+++ b/oshmem/shmem/c/shmem_wait.c
@@ -68,21 +68,23 @@
 #pragma weak shmemx_int32_wait_until = pshmemx_int32_wait_until
 #pragma weak shmemx_int64_wait_until = pshmemx_int64_wait_until
 
-#pragma weak shmem_short_test = pshmem_short_test
-#pragma weak shmem_int_test = pshmem_int_test
-#pragma weak shmem_long_test = pshmem_long_test
-#pragma weak shmem_longlong_test = pshmem_longlong_test
-#pragma weak shmem_ushort_test = pshmem_ushort_test
-#pragma weak shmem_uint_test = pshmem_uint_test
-#pragma weak shmem_ulong_test = pshmem_ulong_test
-#pragma weak shmem_ulonglong_test = pshmem_ulonglong_test
-#pragma weak shmem_int32_test = pshmem_int32_test
-#pragma weak shmem_int64_test = pshmem_int64_test
-#pragma weak shmem_uint32_test = pshmem_uint32_test
-#pragma weak shmem_uint64_test = pshmem_uint64_test
-#pragma weak shmem_size_test = pshmem_size_test
-#pragma weak shmem_ptrdiff_test = pshmem_ptrdiff_test
-#include "oshmem/shmem/c/profile/defines.h"
+#pragma weak shmem_short_test             = pshmem_short_test
+#pragma weak shmem_int_test               = pshmem_int_test
+#pragma weak shmem_long_test              = pshmem_long_test
+#pragma weak shmem_longlong_test          = pshmem_longlong_test
+#pragma weak shmem_ushort_test            = pshmem_ushort_test
+#pragma weak shmem_uint_test              = pshmem_uint_test
+#pragma weak shmem_ulong_test             = pshmem_ulong_test
+#pragma weak shmem_ulonglong_test         = pshmem_ulonglong_test
+#pragma weak shmem_int32_test             = pshmem_int32_test
+#pragma weak shmem_int64_test             = pshmem_int64_test
+#pragma weak shmem_uint32_test            = pshmem_uint32_test
+#pragma weak shmem_uint64_test            = pshmem_uint64_test
+#pragma weak shmem_size_test              = pshmem_size_test
+#pragma weak shmem_ptrdiff_test           = pshmem_ptrdiff_test
+
+
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 SHMEM_TYPE_WAIT(, volatile long, SHMEM_LONG, shmem)
@@ -114,16 +116,16 @@ SHMEM_TYPE_WAIT_UNTIL(_short, volatile short, SHMEM_SHORT, shmem)
 SHMEM_TYPE_WAIT_UNTIL(_int, volatile int, SHMEM_INT, shmem)
 SHMEM_TYPE_WAIT_UNTIL(_long, volatile long, SHMEM_LONG, shmem)
 SHMEM_TYPE_WAIT_UNTIL(_longlong, volatile long long, SHMEM_LLONG, shmem)
-SHMEM_TYPE_WAIT_UNTIL(_ushort, volatile unsigned short, SHMEM_SHORT, shmem)
-SHMEM_TYPE_WAIT_UNTIL(_uint, volatile unsigned int, SHMEM_INT, shmem)
-SHMEM_TYPE_WAIT_UNTIL(_ulong, volatile unsigned long, SHMEM_LONG, shmem)
-SHMEM_TYPE_WAIT_UNTIL(_ulonglong, volatile unsigned long long, SHMEM_LLONG, shmem)
+SHMEM_TYPE_WAIT_UNTIL(_ushort, volatile unsigned short, SHMEM_USHORT, shmem)
+SHMEM_TYPE_WAIT_UNTIL(_uint, volatile unsigned int, SHMEM_UINT, shmem)
+SHMEM_TYPE_WAIT_UNTIL(_ulong, volatile unsigned long, SHMEM_ULONG, shmem)
+SHMEM_TYPE_WAIT_UNTIL(_ulonglong, volatile unsigned long long, SHMEM_ULLONG, shmem)
 SHMEM_TYPE_WAIT_UNTIL(_int32, volatile int32_t, SHMEM_INT32_T, shmem)
 SHMEM_TYPE_WAIT_UNTIL(_int64, volatile int64_t, SHMEM_INT64_T, shmem)
-SHMEM_TYPE_WAIT_UNTIL(_uint32, volatile uint32_t, SHMEM_INT32_T, shmem)
-SHMEM_TYPE_WAIT_UNTIL(_uint64, volatile uint64_t, SHMEM_INT64_T, shmem)
-SHMEM_TYPE_WAIT_UNTIL(_size, volatile size_t, SHMEM_LLONG, shmem)
-SHMEM_TYPE_WAIT_UNTIL(_ptrdiff, volatile ptrdiff_t, SHMEM_LLONG, shmem)
+SHMEM_TYPE_WAIT_UNTIL(_uint32, volatile uint32_t, SHMEM_UINT32_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL(_uint64, volatile uint64_t, SHMEM_UINT64_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL(_size, volatile size_t, SHMEM_SIZE_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL(_ptrdiff, volatile ptrdiff_t, SHMEM_PTRDIFF_T, shmem)
 
 SHMEM_TYPE_WAIT_UNTIL(_int32, int32_t, SHMEM_INT32_T, shmemx)
 SHMEM_TYPE_WAIT_UNTIL(_int64, int64_t, SHMEM_INT64_T, shmemx)
@@ -150,13 +152,13 @@ SHMEM_TYPE_TEST(_short, volatile short, SHMEM_SHORT, shmem)
 SHMEM_TYPE_TEST(_int, volatile int, SHMEM_INT, shmem)
 SHMEM_TYPE_TEST(_long, volatile long, SHMEM_LONG, shmem)
 SHMEM_TYPE_TEST(_longlong, volatile long long, SHMEM_LLONG, shmem)
-SHMEM_TYPE_TEST(_ushort, volatile unsigned short, SHMEM_SHORT, shmem)
-SHMEM_TYPE_TEST(_uint, volatile unsigned int, SHMEM_INT, shmem)
-SHMEM_TYPE_TEST(_ulong, volatile unsigned long, SHMEM_LONG, shmem)
-SHMEM_TYPE_TEST(_ulonglong, volatile unsigned long long, SHMEM_LLONG, shmem)
+SHMEM_TYPE_TEST(_ushort, volatile unsigned short, SHMEM_USHORT, shmem)
+SHMEM_TYPE_TEST(_uint, volatile unsigned int, SHMEM_UINT, shmem)
+SHMEM_TYPE_TEST(_ulong, volatile unsigned long, SHMEM_ULONG, shmem)
+SHMEM_TYPE_TEST(_ulonglong, volatile unsigned long long, SHMEM_ULLONG, shmem)
 SHMEM_TYPE_TEST(_int32, volatile int32_t, SHMEM_INT32_T, shmem)
 SHMEM_TYPE_TEST(_int64, volatile int64_t, SHMEM_INT64_T, shmem)
-SHMEM_TYPE_TEST(_uint32, volatile uint32_t, SHMEM_INT32_T, shmem)
-SHMEM_TYPE_TEST(_uint64, volatile uint64_t, SHMEM_INT64_T, shmem)
-SHMEM_TYPE_TEST(_size, volatile size_t, SHMEM_LLONG, shmem)
-SHMEM_TYPE_TEST(_ptrdiff, volatile ptrdiff_t, SHMEM_LLONG, shmem)
+SHMEM_TYPE_TEST(_uint32, volatile uint32_t, SHMEM_UINT32_T, shmem)
+SHMEM_TYPE_TEST(_uint64, volatile uint64_t, SHMEM_UINT64_T, shmem)
+SHMEM_TYPE_TEST(_size, volatile size_t, SHMEM_SIZE_T, shmem)
+SHMEM_TYPE_TEST(_ptrdiff, volatile ptrdiff_t, SHMEM_PTRDIFF_T, shmem)
diff --git a/oshmem/shmem/c/shmem_wait_ivars.c b/oshmem/shmem/c/shmem_wait_ivars.c
new file mode 100644
index 00000000000..895c45da4f3
--- /dev/null
+++ b/oshmem/shmem/c/shmem_wait_ivars.c
@@ -0,0 +1,295 @@
+/*
+ * Copyright (c) 2021      NVIDIA Corporation.
+ *                         All rights reserved.
+ * Copyright (c) 2019      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "oshmem_config.h"
+
+#include "oshmem/constants.h"
+#include "oshmem/include/shmem.h"
+#include "oshmem/include/shmemx.h"
+
+#include "oshmem/runtime/runtime.h"
+
+#include "oshmem/mca/spml/spml.h"
+
+
+#if OSHMEM_PROFILING
+#include "oshmem/include/pshmem.h"
+
+#pragma weak shmem_short_wait_until_all       = pshmem_short_wait_until_all
+#pragma weak shmem_ushort_wait_until_all      = pshmem_ushort_wait_until_all
+#pragma weak shmem_int_wait_until_all         = pshmem_int_wait_until_all
+#pragma weak shmem_long_wait_until_all        = pshmem_long_wait_until_all
+#pragma weak shmem_longlong_wait_until_all    = pshmem_longlong_wait_until_all
+#pragma weak shmem_uint_wait_until_all        = pshmem_uint_wait_until_all
+#pragma weak shmem_ulong_wait_until_all       = pshmem_ulong_wait_until_all
+#pragma weak shmem_ulonglong_wait_until_all   = pshmem_ulonglong_wait_until_all
+#pragma weak shmem_int32_wait_until_all       = pshmem_int32_wait_until_all
+#pragma weak shmem_int64_wait_until_all       = pshmem_int64_wait_until_all
+#pragma weak shmem_uint32_wait_until_all      = pshmem_uint32_wait_until_all
+#pragma weak shmem_uint64_wait_until_all      = pshmem_uint64_wait_until_all
+#pragma weak shmem_size_wait_until_all        = pshmem_size_wait_until_all
+#pragma weak shmem_ptrdiff_wait_until_all     = pshmem_ptrdiff_wait_until_all
+
+
+#pragma weak shmem_short_wait_until_any       = pshmem_short_wait_until_any
+#pragma weak shmem_ushort_wait_until_any      = pshmem_ushort_wait_until_any
+#pragma weak shmem_int_wait_until_any         = pshmem_int_wait_until_any
+#pragma weak shmem_long_wait_until_any        = pshmem_long_wait_until_any
+#pragma weak shmem_longlong_wait_until_any    = pshmem_longlong_wait_until_any
+#pragma weak shmem_uint_wait_until_any        = pshmem_uint_wait_until_any
+#pragma weak shmem_ulong_wait_until_any       = pshmem_ulong_wait_until_any
+#pragma weak shmem_ulonglong_wait_until_any   = pshmem_ulonglong_wait_until_any
+#pragma weak shmem_int32_wait_until_any       = pshmem_int32_wait_until_any
+#pragma weak shmem_int64_wait_until_any       = pshmem_int64_wait_until_any
+#pragma weak shmem_uint32_wait_until_any      = pshmem_uint32_wait_until_any
+#pragma weak shmem_uint64_wait_until_any      = pshmem_uint64_wait_until_any
+#pragma weak shmem_size_wait_until_any        = pshmem_size_wait_until_any
+#pragma weak shmem_ptrdiff_wait_until_any     = pshmem_ptrdiff_wait_until_any
+
+
+#pragma weak shmem_short_wait_until_some       = pshmem_short_wait_until_some
+#pragma weak shmem_ushort_wait_until_some      = pshmem_ushort_wait_until_some
+#pragma weak shmem_int_wait_until_some         = pshmem_int_wait_until_some
+#pragma weak shmem_long_wait_until_some        = pshmem_long_wait_until_some
+#pragma weak shmem_longlong_wait_until_some    = pshmem_longlong_wait_until_some
+#pragma weak shmem_uint_wait_until_some        = pshmem_uint_wait_until_some
+#pragma weak shmem_ulong_wait_until_some       = pshmem_ulong_wait_until_some
+#pragma weak shmem_ulonglong_wait_until_some   = pshmem_ulonglong_wait_until_some
+#pragma weak shmem_int32_wait_until_some       = pshmem_int32_wait_until_some
+#pragma weak shmem_int64_wait_until_some       = pshmem_int64_wait_until_some
+#pragma weak shmem_uint32_wait_until_some      = pshmem_uint32_wait_until_some
+#pragma weak shmem_uint64_wait_until_some      = pshmem_uint64_wait_until_some
+#pragma weak shmem_size_wait_until_some        = pshmem_size_wait_until_some
+#pragma weak shmem_ptrdiff_wait_until_some     = pshmem_ptrdiff_wait_until_some
+
+
+#pragma weak shmem_short_wait_until_all_vector       = pshmem_short_wait_until_all_vector
+#pragma weak shmem_ushort_wait_until_all_vector      = pshmem_ushort_wait_until_all_vector
+#pragma weak shmem_int_wait_until_all_vector         = pshmem_int_wait_until_all_vector
+#pragma weak shmem_long_wait_until_all_vector        = pshmem_long_wait_until_all_vector
+#pragma weak shmem_longlong_wait_until_all_vector    = pshmem_longlong_wait_until_all_vector
+#pragma weak shmem_uint_wait_until_all_vector        = pshmem_uint_wait_until_all_vector
+#pragma weak shmem_ulong_wait_until_all_vector       = pshmem_ulong_wait_until_all_vector
+#pragma weak shmem_ulonglong_wait_until_all_vector   = pshmem_ulonglong_wait_until_all_vector
+#pragma weak shmem_int32_wait_until_all_vector       = pshmem_int32_wait_until_all_vector
+#pragma weak shmem_int64_wait_until_all_vector       = pshmem_int64_wait_until_all_vector
+#pragma weak shmem_uint32_wait_until_all_vector      = pshmem_uint32_wait_until_all_vector
+#pragma weak shmem_uint64_wait_until_all_vector      = pshmem_uint64_wait_until_all_vector
+#pragma weak shmem_size_wait_until_all_vector        = pshmem_size_wait_until_all_vector
+#pragma weak shmem_ptrdiff_wait_until_all_vector     = pshmem_ptrdiff_wait_until_all_vector
+
+
+#pragma weak shmem_short_wait_until_any_vector       = pshmem_short_wait_until_any_vector
+#pragma weak shmem_ushort_wait_until_any_vector      = pshmem_ushort_wait_until_any_vector
+#pragma weak shmem_int_wait_until_any_vector         = pshmem_int_wait_until_any_vector
+#pragma weak shmem_long_wait_until_any_vector        = pshmem_long_wait_until_any_vector
+#pragma weak shmem_longlong_wait_until_any_vector    = pshmem_longlong_wait_until_any_vector
+#pragma weak shmem_uint_wait_until_any_vector        = pshmem_uint_wait_until_any_vector
+#pragma weak shmem_ulong_wait_until_any_vector       = pshmem_ulong_wait_until_any_vector
+#pragma weak shmem_ulonglong_wait_until_any_vector   = pshmem_ulonglong_wait_until_any_vector
+#pragma weak shmem_int32_wait_until_any_vector       = pshmem_int32_wait_until_any_vector
+#pragma weak shmem_int64_wait_until_any_vector       = pshmem_int64_wait_until_any_vector
+#pragma weak shmem_uint32_wait_until_any_vector      = pshmem_uint32_wait_until_any_vector
+#pragma weak shmem_uint64_wait_until_any_vector      = pshmem_uint64_wait_until_any_vector
+#pragma weak shmem_size_wait_until_any_vector        = pshmem_size_wait_until_any_vector
+#pragma weak shmem_ptrdiff_wait_until_any_vector     = pshmem_ptrdiff_wait_until_any_vector
+
+
+#pragma weak shmem_short_wait_until_some_vector       = pshmem_short_wait_until_some_vector
+#pragma weak shmem_ushort_wait_until_some_vector      = pshmem_ushort_wait_until_some_vector
+#pragma weak shmem_int_wait_until_some_vector         = pshmem_int_wait_until_some_vector
+#pragma weak shmem_long_wait_until_some_vector        = pshmem_long_wait_until_some_vector
+#pragma weak shmem_longlong_wait_until_some_vector    = pshmem_longlong_wait_until_some_vector
+#pragma weak shmem_uint_wait_until_some_vector        = pshmem_uint_wait_until_some_vector
+#pragma weak shmem_ulong_wait_until_some_vector       = pshmem_ulong_wait_until_some_vector
+#pragma weak shmem_ulonglong_wait_until_some_vector   = pshmem_ulonglong_wait_until_some_vector
+#pragma weak shmem_int32_wait_until_some_vector       = pshmem_int32_wait_until_some_vector
+#pragma weak shmem_int64_wait_until_some_vector       = pshmem_int64_wait_until_some_vector
+#pragma weak shmem_uint32_wait_until_some_vector      = pshmem_uint32_wait_until_some_vector
+#pragma weak shmem_uint64_wait_until_some_vector      = pshmem_uint64_wait_until_some_vector
+#pragma weak shmem_size_wait_until_some_vector        = pshmem_size_wait_until_some_vector
+#pragma weak shmem_ptrdiff_wait_until_some_vector     = pshmem_ptrdiff_wait_until_some_vector
+
+
+#include "oshmem/shmem/c/profile-defines.h"
+#endif
+
+#define SHMEM_TYPE_WAIT_UNTIL_ALL(type_name, type, code, prefix)    \
+    void prefix##type_name##_wait_until_all(volatile type *ivars, size_t nelems, const int *status, int cmp, type value)   \
+    {                                                               \
+        RUNTIME_CHECK_INIT();                                       \
+                                                                    \
+        MCA_SPML_CALL(wait_until_all(                               \
+            (void*)ivars,                                           \
+            cmp,                                                    \
+            (void*)&value,                                          \
+            nelems, status, code));                                 \
+    }
+
+
+#define SHMEM_TYPE_WAIT_UNTIL_ANY(type_name, type, code, prefix)    \
+    size_t prefix##type_name##_wait_until_any(volatile type *ivars, size_t nelems, const int *status, int cmp, type value)   \
+    {                                                               \
+        RUNTIME_CHECK_INIT();                                       \
+                                                                    \
+        return MCA_SPML_CALL(wait_until_any(                        \
+            (void*)ivars,                                           \
+            cmp,                                                    \
+            (void*)&value,                                          \
+            nelems, status, code));                                 \
+    }
+
+
+#define SHMEM_TYPE_WAIT_UNTIL_SOME(type_name, type, code, prefix)   \
+    size_t  prefix##type_name##_wait_until_some(volatile type *ivars, size_t nelems, size_t *indices, const int *status, int cmp, type value)   \
+    {                                                               \
+        RUNTIME_CHECK_INIT();                                       \
+                                                                    \
+        return MCA_SPML_CALL(wait_until_some(                       \
+            (void*)ivars,                                           \
+            cmp,                                                    \
+            (void*)&value,                                          \
+            nelems, indices, status, code));                        \
+    }
+
+#define SHMEM_TYPE_WAIT_UNTIL_ANY_VECTOR(type_name, type, code, prefix)    \
+    size_t prefix##type_name##_wait_until_any_vector(volatile type *ivars, size_t nelems, const int *status, int cmp, type *values)   \
+    {                                                               \
+        RUNTIME_CHECK_INIT();                                       \
+                                                                    \
+        return MCA_SPML_CALL(wait_until_any_vector(                 \
+            (void*)ivars,                                           \
+            cmp,                                                    \
+            (void*)values,                                          \
+            nelems, status, code));                                 \
+    }
+
+#define SHMEM_TYPE_WAIT_UNTIL_SOME_VECTOR(type_name, type, code, prefix)    \
+    size_t prefix##type_name##_wait_until_some_vector(volatile type *ivars, size_t nelems, size_t *indices, const int *status, int cmp, type *values)   \
+    {                                                               \
+        RUNTIME_CHECK_INIT();                                       \
+                                                                    \
+        return MCA_SPML_CALL(wait_until_some_vector(                \
+            (void*)ivars,                                           \
+            cmp,                                                    \
+            (void*)values,                                          \
+            nelems, indices, status, code));                        \
+    }
+
+
+#define SHMEM_TYPE_WAIT_UNTIL_ALL_VECTOR(type_name, type, code, prefix)    \
+    void prefix##type_name##_wait_until_all_vector(volatile type *ivars, size_t nelems, const int *status, int cmp, type *values)   \
+    {                                                               \
+        MCA_SPML_CALL(wait_until_all_vector(                        \
+            (void*)ivars,                                           \
+            cmp,                                                    \
+            (void*)values,                                          \
+            nelems, status, code));                                 \
+    }
+
+
+
+SHMEM_TYPE_WAIT_UNTIL_ALL(_short,  short, SHMEM_SHORT, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ALL(_ushort,  unsigned short, SHMEM_USHORT, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ALL(_int,  int, SHMEM_INT, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ALL(_long,  long, SHMEM_LONG, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ALL(_longlong,  long long, SHMEM_LLONG, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ALL(_uint,  unsigned int, SHMEM_UINT, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ALL(_ulong,  unsigned long, SHMEM_ULONG, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ALL(_ulonglong,  unsigned long long, SHMEM_ULLONG, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ALL(_int32,  int32_t, SHMEM_INT32_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ALL(_int64,  int64_t, SHMEM_INT64_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ALL(_uint32,  uint32_t, SHMEM_UINT32_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ALL(_uint64,  uint64_t, SHMEM_UINT64_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ALL(_size,  size_t, SHMEM_SIZE_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ALL(_ptrdiff,  ptrdiff_t, SHMEM_PTRDIFF_T, shmem)
+
+SHMEM_TYPE_WAIT_UNTIL_ANY(_short,  short, SHMEM_SHORT, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ANY(_ushort,  unsigned short, SHMEM_USHORT, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ANY(_int,  int, SHMEM_INT, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ANY(_long,  long, SHMEM_LONG, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ANY(_longlong,  long long, SHMEM_LLONG, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ANY(_uint,  unsigned int, SHMEM_UINT, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ANY(_ulong,  unsigned long, SHMEM_ULONG, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ANY(_ulonglong,  unsigned long long, SHMEM_ULLONG, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ANY(_int32,  int32_t, SHMEM_INT32_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ANY(_int64,  int64_t, SHMEM_INT64_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ANY(_uint32,  uint32_t, SHMEM_UINT32_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ANY(_uint64,  uint64_t, SHMEM_UINT64_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ANY(_size,  size_t, SHMEM_SIZE_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ANY(_ptrdiff,  ptrdiff_t, SHMEM_PTRDIFF_T, shmem)
+
+
+SHMEM_TYPE_WAIT_UNTIL_SOME(_short,  short, SHMEM_SHORT, shmem)
+SHMEM_TYPE_WAIT_UNTIL_SOME(_ushort,  unsigned short, SHMEM_USHORT, shmem)
+SHMEM_TYPE_WAIT_UNTIL_SOME(_int,  int, SHMEM_INT, shmem)
+SHMEM_TYPE_WAIT_UNTIL_SOME(_long,  long, SHMEM_LONG, shmem)
+SHMEM_TYPE_WAIT_UNTIL_SOME(_longlong,  long long, SHMEM_LLONG, shmem)
+SHMEM_TYPE_WAIT_UNTIL_SOME(_uint,  unsigned int, SHMEM_UINT, shmem)
+SHMEM_TYPE_WAIT_UNTIL_SOME(_ulong,  unsigned long, SHMEM_ULONG, shmem)
+SHMEM_TYPE_WAIT_UNTIL_SOME(_ulonglong,  unsigned long long, SHMEM_ULLONG, shmem)
+SHMEM_TYPE_WAIT_UNTIL_SOME(_int32,  int32_t, SHMEM_INT32_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL_SOME(_int64,  int64_t, SHMEM_INT64_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL_SOME(_uint32,  uint32_t, SHMEM_UINT32_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL_SOME(_uint64,  uint64_t, SHMEM_UINT64_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL_SOME(_size,  size_t, SHMEM_SIZE_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL_SOME(_ptrdiff,  ptrdiff_t, SHMEM_PTRDIFF_T, shmem)
+
+
+SHMEM_TYPE_WAIT_UNTIL_ALL_VECTOR(_short,  short, SHMEM_SHORT, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ALL_VECTOR(_ushort,  unsigned short, SHMEM_USHORT, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ALL_VECTOR(_int,  int, SHMEM_INT, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ALL_VECTOR(_long,  long, SHMEM_LONG, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ALL_VECTOR(_longlong,  long long, SHMEM_LLONG, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ALL_VECTOR(_uint,  unsigned int, SHMEM_UINT, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ALL_VECTOR(_ulong,  unsigned long, SHMEM_ULONG, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ALL_VECTOR(_ulonglong,  unsigned long long, SHMEM_ULLONG, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ALL_VECTOR(_int32,  int32_t, SHMEM_INT32_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ALL_VECTOR(_int64,  int64_t, SHMEM_INT64_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ALL_VECTOR(_uint32,  uint32_t, SHMEM_UINT32_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ALL_VECTOR(_uint64,  uint64_t, SHMEM_UINT64_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ALL_VECTOR(_size,  size_t, SHMEM_SIZE_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ALL_VECTOR(_ptrdiff,  ptrdiff_t, SHMEM_PTRDIFF_T, shmem)
+
+
+
+SHMEM_TYPE_WAIT_UNTIL_ANY_VECTOR(_short,  short, SHMEM_SHORT, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ANY_VECTOR(_ushort,  unsigned short, SHMEM_USHORT, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ANY_VECTOR(_int,  int, SHMEM_INT, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ANY_VECTOR(_long,  long, SHMEM_LONG, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ANY_VECTOR(_longlong,  long long, SHMEM_LLONG, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ANY_VECTOR(_uint,  unsigned int, SHMEM_UINT, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ANY_VECTOR(_ulong,  unsigned long, SHMEM_ULONG, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ANY_VECTOR(_ulonglong,  unsigned long long, SHMEM_ULLONG, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ANY_VECTOR(_int32,  int32_t, SHMEM_INT32_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ANY_VECTOR(_int64,  int64_t, SHMEM_INT64_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ANY_VECTOR(_uint32,  uint32_t, SHMEM_UINT32_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ANY_VECTOR(_uint64,  uint64_t, SHMEM_UINT64_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ANY_VECTOR(_size,  size_t, SHMEM_SIZE_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL_ANY_VECTOR(_ptrdiff,  ptrdiff_t, SHMEM_PTRDIFF_T, shmem)
+
+
+
+SHMEM_TYPE_WAIT_UNTIL_SOME_VECTOR(_short,  short, SHMEM_SHORT, shmem)
+SHMEM_TYPE_WAIT_UNTIL_SOME_VECTOR(_ushort,  unsigned short, SHMEM_USHORT, shmem)
+SHMEM_TYPE_WAIT_UNTIL_SOME_VECTOR(_int,  int, SHMEM_INT, shmem)
+SHMEM_TYPE_WAIT_UNTIL_SOME_VECTOR(_long,  long, SHMEM_LONG, shmem)
+SHMEM_TYPE_WAIT_UNTIL_SOME_VECTOR(_longlong,  long long, SHMEM_LLONG, shmem)
+SHMEM_TYPE_WAIT_UNTIL_SOME_VECTOR(_uint,  unsigned int, SHMEM_UINT, shmem)
+SHMEM_TYPE_WAIT_UNTIL_SOME_VECTOR(_ulong,  unsigned long, SHMEM_ULONG, shmem)
+SHMEM_TYPE_WAIT_UNTIL_SOME_VECTOR(_ulonglong,  unsigned long long, SHMEM_ULLONG, shmem)
+SHMEM_TYPE_WAIT_UNTIL_SOME_VECTOR(_int32,  int32_t, SHMEM_INT32_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL_SOME_VECTOR(_int64,  int64_t, SHMEM_INT64_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL_SOME_VECTOR(_uint32,  uint32_t, SHMEM_UINT32_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL_SOME_VECTOR(_uint64,  uint64_t, SHMEM_UINT64_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL_SOME_VECTOR(_size,  size_t, SHMEM_SIZE_T, shmem)
+SHMEM_TYPE_WAIT_UNTIL_SOME_VECTOR(_ptrdiff,  ptrdiff_t, SHMEM_PTRDIFF_T, shmem)
+
+
diff --git a/oshmem/shmem/c/shmem_xor.c b/oshmem/shmem/c/shmem_xor.c
index b2209ce6648..26a22083e50 100644
--- a/oshmem/shmem/c/shmem_xor.c
+++ b/oshmem/shmem/c/shmem_xor.c
@@ -51,7 +51,7 @@
 #pragma weak shmemx_int64_atomic_xor    = pshmemx_int64_atomic_xor
 #pragma weak shmemx_uint32_atomic_xor   = pshmemx_uint32_atomic_xor
 #pragma weak shmemx_uint64_atomic_xor   = pshmemx_uint64_atomic_xor
-#include "oshmem/shmem/c/profile/defines.h"
+#include "oshmem/shmem/c/profile-defines.h"
 #endif
 
 OSHMEM_TYPE_OP(int, int, shmem, xor)
diff --git a/oshmem/shmem/fortran/Makefile.am b/oshmem/shmem/fortran/Makefile.am
index 25610758f9f..83dcc0b996f 100644
--- a/oshmem/shmem/fortran/Makefile.am
+++ b/oshmem/shmem/fortran/Makefile.am
@@ -4,6 +4,8 @@
 # Copyright (c) 2013-2014 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2014      Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2021      Amazon.com, Inc. or its affiliates.
+#                         All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -11,30 +13,28 @@
 # $HEADER$
 #
 
-# This is guaranteed to be false if we're not building OSHMEM at all
-#
-if OSHMEM_BUILD_FORTRAN_BINDINGS
-if OSHMEM_PROFILING
-    SUBDIRS = profile
-endif
-endif
-
 if OSHMEM_BUILD_FORTRAN_BINDINGS
-oshmem_fortran_lib       = liboshmem_fortran.la
+# note that liboshmem_fortran_profile.la is built regardless of
+# whether or not the profiling layer is built.  It holds all the api
+# functions that might have a profiling version.
+oshmem_fortran_libs       = liboshmem_fortran.la liboshmem_fortran_profile.la
 else
-oshmem_fortran_lib       =
+oshmem_fortran_libs       =
 endif
 
-noinst_LTLIBRARIES = $(oshmem_fortran_lib)
+noinst_LTLIBRARIES = $(oshmem_fortran_libs)
 
 headers = prototypes_shmem.h \
+          prototypes_pshmem.h \
           bindings.h \
+          pbindings.h \
+          profile-defines.h \
           shmem_fortran_pointer.h
 
 liboshmem_fortran_la_SOURCES = shmem_finalize_f.c
+liboshmem_fortran_la_LIBADD = liboshmem_fortran_profile.la
 
-if ! OSHMEM_PROFILING
-liboshmem_fortran_la_SOURCES += \
+liboshmem_fortran_profile_la_SOURCES = \
                     shmem_init_f.c \
                     shmem_global_exit_f.c \
                     shmem_n_pes_f.c \
@@ -138,7 +138,10 @@ liboshmem_fortran_la_SOURCES += \
                     shmem_fence_f.c \
                     shmem_info_f.c
 
-AM_CPPFLAGS     = -DOSHMEM_PROFILING=0
+if OSHMEM_PROFILING
+liboshmem_fortran_profile_la_CPPFLAGS = -DOSHMEM_PROFILING=1
+else
+liboshmem_fortran_profile_la_CPPFLAGS = -DOSHMEM_PROFILING=0
 endif
 
 if PROJECT_OSHMEM
diff --git a/oshmem/shmem/fortran/profile/pbindings.h b/oshmem/shmem/fortran/pbindings.h
similarity index 100%
rename from oshmem/shmem/fortran/profile/pbindings.h
rename to oshmem/shmem/fortran/pbindings.h
diff --git a/oshmem/shmem/fortran/profile/defines.h b/oshmem/shmem/fortran/profile-defines.h
similarity index 100%
rename from oshmem/shmem/fortran/profile/defines.h
rename to oshmem/shmem/fortran/profile-defines.h
diff --git a/oshmem/shmem/fortran/profile/Makefile.am b/oshmem/shmem/fortran/profile/Makefile.am
deleted file mode 100644
index 546d56eac65..00000000000
--- a/oshmem/shmem/fortran/profile/Makefile.am
+++ /dev/null
@@ -1,159 +0,0 @@
-#
-# Copyright (c) 2013      Mellanox Technologies, Inc.
-#                         All rights reserved
-# Copyright (c) 2013-2014 Cisco Systems, Inc.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-include $(top_srcdir)/Makefile.ompi-rules
-
-AM_CPPFLAGS     = -DOSHMEM_PROFILING=1
-
-# This is guaranteed to be false if we're not building OSHMEM at all
-if OSHMEM_BUILD_FORTRAN_BINDINGS
-pshmem_fortran_lib       = liboshmem_fortran_pshmem.la
-else
-pshmem_fortran_lib       =
-endif
-
-noinst_LTLIBRARIES = $(pshmem_fortran_lib)
-
-headers = prototypes_pshmem.h \
-	pbindings.h \
-	defines.h
-
-nodist_liboshmem_fortran_pshmem_la_SOURCES = \
-	pshmem_init_f.c \
-	pshmem_global_exit_f.c \
-	pshmem_n_pes_f.c \
-	pshmem_my_pe_f.c \
-	pshmem_barrier_all_f.c \
-	pshpalloc_f.c \
-	pshpdeallc_f.c \
-	pshpclmove_f.c \
-	pshmem_ptr_f.c \
-	pshmem_pe_accessible_f.c \
-	pshmem_addr_accessible_f.c \
-	pshmem_character_put_f.c \
-	pshmem_double_put_f.c \
-	pshmem_complex_put_f.c \
-	pshmem_logical_put_f.c \
-	pshmem_integer_put_f.c \
-	pshmem_real_put_f.c \
-	pshmem_put4_f.c \
-	pshmem_put8_f.c \
-	pshmem_put32_f.c \
-	pshmem_put64_f.c \
-	pshmem_put128_f.c \
-	pshmem_putmem_f.c \
-	pshmem_complex_iput_f.c \
-	pshmem_double_iput_f.c \
-	pshmem_integer_iput_f.c \
-	pshmem_iput128_f.c \
-	pshmem_iput32_f.c \
-	pshmem_iput4_f.c \
-	pshmem_iput64_f.c \
-	pshmem_iput8_f.c \
-	pshmem_logical_iput_f.c \
-	pshmem_real_iput_f.c \
-	pshmem_put_nb_f.c \
-	pshmem_character_get_f.c \
-	pshmem_complex_get_f.c \
-	pshmem_double_get_f.c \
-	pshmem_get128_f.c \
-	pshmem_get32_f.c \
-	pshmem_get4_f.c \
-	pshmem_get64_f.c \
-	pshmem_get8_f.c \
-	pshmem_getmem_f.c \
-	pshmem_integer_get_f.c \
-	pshmem_logical_get_f.c \
-	pshmem_real_get_f.c \
-	pshmem_complex_iget_f.c \
-	pshmem_double_iget_f.c \
-	pshmem_iget128_f.c \
-	pshmem_iget32_f.c \
-	pshmem_iget4_f.c \
-	pshmem_iget64_f.c \
-	pshmem_iget8_f.c \
-	pshmem_integer_iget_f.c \
-	pshmem_logical_iget_f.c \
-	pshmem_real_iget_f.c \
-	pshmem_get_nb_f.c \
-	pshmem_swap_f.c \
-	pshmem_int4_swap_f.c \
-	pshmem_int8_swap_f.c \
-	pshmem_real4_swap_f.c \
-	pshmem_real8_swap_f.c \
-	pshmem_int4_set_f.c \
-	pshmem_int8_set_f.c \
-	pshmem_real4_set_f.c \
-	pshmem_real8_set_f.c \
-	pshmem_int4_cswap_f.c \
-	pshmem_int8_cswap_f.c \
-	pshmem_int4_fadd_f.c \
-	pshmem_int8_fadd_f.c \
-	pshmem_int4_fetch_f.c \
-	pshmem_int8_fetch_f.c \
-	pshmem_real4_fetch_f.c \
-	pshmem_real8_fetch_f.c \
-	pshmem_int4_finc_f.c \
-	pshmem_int8_finc_f.c \
-	pshmem_int4_add_f.c \
-	pshmem_int8_add_f.c \
-	pshmem_int4_wait_f.c \
-	pshmem_int8_wait_f.c \
-	pshmem_wait_f.c \
-	pshmem_int4_wait_until_f.c \
-	pshmem_int8_wait_until_f.c \
-	pshmem_wait_until_f.c \
-	pshmem_barrier_f.c \
-	pshmem_and_to_all_f.c \
-	pshmem_or_to_all_f.c \
-	pshmem_xor_to_all_f.c \
-	pshmem_max_to_all_f.c \
-	pshmem_min_to_all_f.c \
-	pshmem_sum_to_all_f.c \
-	pshmem_prod_to_all_f.c \
-	pshmem_collect_f.c \
-	pshmem_broadcast_f.c \
-	pshmem_alltoall_f.c \
-	pshmem_lock_f.c \
-	pshmem_cache_f.c \
-	pshmem_int4_inc_f.c \
-	pshmem_int8_inc_f.c \
-	pshmem_quiet_f.c \
-	pshmem_fence_f.c \
-	pshmem_info_f.c
-
-#
-# Sym link in the sources from the real OSHMEM directory
-#
-$(nodist_liboshmem_fortran_pshmem_la_SOURCES):
-	$(OMPI_V_LN_S) if test ! -r $@ ; then \
-		pname=`echo $@ | cut -b '2-'` ; \
-		$(LN_S) $(top_srcdir)/oshmem/shmem/fortran/$$pname $@ ; \
-	fi
-
-if PROJECT_OSHMEM
-if WANT_INSTALL_HEADERS
-oshmemdir = $(oshmemincludedir)/$(subdir)
-oshmem_HEADERS = $(headers)
-endif
-endif
-
-# These files were created by targets above
-
-MAINTAINERCLEANFILES = $(nodist_liboshmem_fortran_pshmem_la_SOURCES)
-
-# Don't want these targets in here
-
-tags-recursive:
-tags:
-TAGS:
-GTAGS:
-ID:
diff --git a/oshmem/shmem/fortran/profile/prototypes_pshmem.h b/oshmem/shmem/fortran/prototypes_pshmem.h
similarity index 100%
rename from oshmem/shmem/fortran/profile/prototypes_pshmem.h
rename to oshmem/shmem/fortran/prototypes_pshmem.h
diff --git a/oshmem/shmem/fortran/shmem_addr_accessible_f.c b/oshmem/shmem/fortran/shmem_addr_accessible_f.c
index d4aac33ecf3..dae10d869c5 100644
--- a/oshmem/shmem/fortran/shmem_addr_accessible_f.c
+++ b/oshmem/shmem/fortran/shmem_addr_accessible_f.c
@@ -16,9 +16,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_ADDR_ACCESSIBLE, shmem_addr_accessible)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_FUNCTION (MPI_Fint,
diff --git a/oshmem/shmem/fortran/shmem_alltoall_f.c b/oshmem/shmem/fortran/shmem_alltoall_f.c
index 6845edcf3f0..46c07257f43 100644
--- a/oshmem/shmem/fortran/shmem_alltoall_f.c
+++ b/oshmem/shmem/fortran/shmem_alltoall_f.c
@@ -18,12 +18,12 @@
 #include "oshmem/op/op.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_ALLTOALL32, shmem_alltoall32)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_ALLTOALL64, shmem_alltoall64)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_ALLTOALLS32, shmem_alltoalls32)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_ALLTOALLS64, shmem_alltoalls64)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_and_to_all_f.c b/oshmem/shmem/fortran/shmem_and_to_all_f.c
index d653360dbf8..51099e8a473 100644
--- a/oshmem/shmem/fortran/shmem_and_to_all_f.c
+++ b/oshmem/shmem/fortran/shmem_and_to_all_f.c
@@ -18,11 +18,11 @@
 #include "oshmem/op/op.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT2_AND_TO_ALL, shmem_int2_and_to_all)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT4_AND_TO_ALL, shmem_int4_and_to_all)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT8_AND_TO_ALL, shmem_int8_and_to_all)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_barrier_all_f.c b/oshmem/shmem/fortran/shmem_barrier_all_f.c
index 1e63510ed27..9e2e160efa1 100644
--- a/oshmem/shmem/fortran/shmem_barrier_all_f.c
+++ b/oshmem/shmem/fortran/shmem_barrier_all_f.c
@@ -14,9 +14,9 @@
 #include "oshmem/include/shmem.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_BARRIER_ALL, shmem_barrier_all)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_barrier_f.c b/oshmem/shmem/fortran/shmem_barrier_f.c
index d7e4e4459c4..946109e5a73 100644
--- a/oshmem/shmem/fortran/shmem_barrier_f.c
+++ b/oshmem/shmem/fortran/shmem_barrier_f.c
@@ -14,9 +14,9 @@
 #include "oshmem/include/shmem.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_BARRIER, shmem_barrier)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_broadcast_f.c b/oshmem/shmem/fortran/shmem_broadcast_f.c
index af2a9b7185c..d36e37aa38b 100644
--- a/oshmem/shmem/fortran/shmem_broadcast_f.c
+++ b/oshmem/shmem/fortran/shmem_broadcast_f.c
@@ -18,12 +18,12 @@
 #include "oshmem/op/op.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_BROADCAST4, shmem_broadcast4)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_BROADCAST8, shmem_broadcast8)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_BROADCAST32, shmem_broadcast32)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_BROADCAST64, shmem_broadcast64)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
@@ -85,7 +85,7 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
         }\
         \
         /* Define actual PE using relative in active set */\
-        rel_PE_root = oshmem_proc_pe(group->proc_array[OMPI_FINT_2_INT(*PE_root)]);\
+        rel_PE_root = oshmem_proc_pe_vpid(group, OMPI_FINT_2_INT(*PE_root));\
         \
         /* Call collective broadcast operation */\
         rc = group->g_scoll.scoll_broadcast( group, \
diff --git a/oshmem/shmem/fortran/shmem_cache_f.c b/oshmem/shmem/fortran/shmem_cache_f.c
index a171c526c6e..5fc2643e1ab 100644
--- a/oshmem/shmem/fortran/shmem_cache_f.c
+++ b/oshmem/shmem/fortran/shmem_cache_f.c
@@ -16,14 +16,14 @@
 #include "ompi/datatype/ompi_datatype.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_SET_CACHE_INV, shmem_set_cache_inv)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_SET_CACHE_LINE_INV, shmem_set_cache_line_inv)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_CLEAR_CACHE_INV, shmem_clear_cache_inv)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_CLEAR_CACHE_LINE_INV, shmem_clear_cache_line_inv)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_UDCFLUSH, shmem_udcflush)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_UDCFLUSH_LINE, shmem_udcflush_line)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_character_get_f.c b/oshmem/shmem/fortran/shmem_character_get_f.c
index 7932e66af57..9e6322bbd4d 100644
--- a/oshmem/shmem/fortran/shmem_character_get_f.c
+++ b/oshmem/shmem/fortran/shmem_character_get_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_CHARACTER_GET, shmem_character_get)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_character_put_f.c b/oshmem/shmem/fortran/shmem_character_put_f.c
index babada9c499..13e5405dacf 100644
--- a/oshmem/shmem/fortran/shmem_character_put_f.c
+++ b/oshmem/shmem/fortran/shmem_character_put_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_CHARACTER_PUT, shmem_character_put)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_collect_f.c b/oshmem/shmem/fortran/shmem_collect_f.c
index d990a6c902f..b05f8b56764 100644
--- a/oshmem/shmem/fortran/shmem_collect_f.c
+++ b/oshmem/shmem/fortran/shmem_collect_f.c
@@ -18,7 +18,7 @@
 #include "oshmem/op/op.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_COLLECT4, shmem_collect4)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_COLLECT8, shmem_collect8)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_COLLECT32, shmem_collect32)
@@ -27,7 +27,7 @@ SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_FCOLLECT4, shmem_fcollect4)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_FCOLLECT8, shmem_fcollect8)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_FCOLLECT32, shmem_fcollect32)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_FCOLLECT64, shmem_fcollect64)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_complex_get_f.c b/oshmem/shmem/fortran/shmem_complex_get_f.c
index 21465b5293d..fc6fb643722 100644
--- a/oshmem/shmem/fortran/shmem_complex_get_f.c
+++ b/oshmem/shmem/fortran/shmem_complex_get_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_COMPLEX_GET, shmem_complex_get)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_complex_iget_f.c b/oshmem/shmem/fortran/shmem_complex_iget_f.c
index 51e5a59dd27..9a5ef31a69e 100644
--- a/oshmem/shmem/fortran/shmem_complex_iget_f.c
+++ b/oshmem/shmem/fortran/shmem_complex_iget_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_COMPLEX_IGET, shmem_complex_iget)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_complex_iput_f.c b/oshmem/shmem/fortran/shmem_complex_iput_f.c
index 00b96ff71f7..ab610aabb02 100644
--- a/oshmem/shmem/fortran/shmem_complex_iput_f.c
+++ b/oshmem/shmem/fortran/shmem_complex_iput_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_COMPLEX_IPUT, shmem_complex_iput)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_complex_put_f.c b/oshmem/shmem/fortran/shmem_complex_put_f.c
index 1cff50e1fcc..d414bd177fa 100644
--- a/oshmem/shmem/fortran/shmem_complex_put_f.c
+++ b/oshmem/shmem/fortran/shmem_complex_put_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_COMPLEX_PUT, shmem_complex_put)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_double_get_f.c b/oshmem/shmem/fortran/shmem_double_get_f.c
index 0870f55823b..e507af7504d 100644
--- a/oshmem/shmem/fortran/shmem_double_get_f.c
+++ b/oshmem/shmem/fortran/shmem_double_get_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_DOUBLE_GET, shmem_double_get)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_double_iget_f.c b/oshmem/shmem/fortran/shmem_double_iget_f.c
index a5da3afdbca..53c6e6f0789 100644
--- a/oshmem/shmem/fortran/shmem_double_iget_f.c
+++ b/oshmem/shmem/fortran/shmem_double_iget_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_DOUBLE_IGET, shmem_double_iget)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_double_iput_f.c b/oshmem/shmem/fortran/shmem_double_iput_f.c
index ad3fc1d1f36..878137d7f10 100644
--- a/oshmem/shmem/fortran/shmem_double_iput_f.c
+++ b/oshmem/shmem/fortran/shmem_double_iput_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_DOUBLE_IPUT, shmem_double_iput)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_double_put_f.c b/oshmem/shmem/fortran/shmem_double_put_f.c
index 9ac5efd020b..a201e3df337 100644
--- a/oshmem/shmem/fortran/shmem_double_put_f.c
+++ b/oshmem/shmem/fortran/shmem_double_put_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_DOUBLE_PUT, shmem_double_put)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_fence_f.c b/oshmem/shmem/fortran/shmem_fence_f.c
index c1fdf9303a8..8413600d5b6 100644
--- a/oshmem/shmem/fortran/shmem_fence_f.c
+++ b/oshmem/shmem/fortran/shmem_fence_f.c
@@ -15,9 +15,9 @@
 #include "oshmem/mca/spml/spml.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_FENCE, shmem_fence)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_get128_f.c b/oshmem/shmem/fortran/shmem_get128_f.c
index f6c5e45af74..e396fd65fde 100644
--- a/oshmem/shmem/fortran/shmem_get128_f.c
+++ b/oshmem/shmem/fortran/shmem_get128_f.c
@@ -18,9 +18,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_GET128, shmem_get128)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_get32_f.c b/oshmem/shmem/fortran/shmem_get32_f.c
index 5f73e8d8c17..302aa100a90 100644
--- a/oshmem/shmem/fortran/shmem_get32_f.c
+++ b/oshmem/shmem/fortran/shmem_get32_f.c
@@ -18,9 +18,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_GET32, shmem_get32)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_get4_f.c b/oshmem/shmem/fortran/shmem_get4_f.c
index 20da34ca8fc..b9ff6d5756f 100644
--- a/oshmem/shmem/fortran/shmem_get4_f.c
+++ b/oshmem/shmem/fortran/shmem_get4_f.c
@@ -18,9 +18,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_GET4, shmem_get4)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_get64_f.c b/oshmem/shmem/fortran/shmem_get64_f.c
index f585cfaa99f..c923ae040ed 100644
--- a/oshmem/shmem/fortran/shmem_get64_f.c
+++ b/oshmem/shmem/fortran/shmem_get64_f.c
@@ -18,9 +18,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_GET64, shmem_get64)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_get8_f.c b/oshmem/shmem/fortran/shmem_get8_f.c
index 8ba75bcecd8..d5b3ea64f01 100644
--- a/oshmem/shmem/fortran/shmem_get8_f.c
+++ b/oshmem/shmem/fortran/shmem_get8_f.c
@@ -18,9 +18,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_GET8, shmem_get8)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_get_nb_f.c b/oshmem/shmem/fortran/shmem_get_nb_f.c
index f5af43eca74..9c1a47cddf4 100644
--- a/oshmem/shmem/fortran/shmem_get_nb_f.c
+++ b/oshmem/shmem/fortran/shmem_get_nb_f.c
@@ -18,7 +18,7 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_GETMEM_NBI, shmem_getmem_nbi)
 
@@ -35,7 +35,7 @@ SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_GET32_NBI, shmem_get32_nbi)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_GET64_NBI, shmem_get64_nbi)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_GET128_NBI, shmem_get128_nbi)
 
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_getmem_f.c b/oshmem/shmem/fortran/shmem_getmem_f.c
index dae32735982..eec5f05e8a5 100644
--- a/oshmem/shmem/fortran/shmem_getmem_f.c
+++ b/oshmem/shmem/fortran/shmem_getmem_f.c
@@ -18,9 +18,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_GETMEM, shmem_getmem)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_global_exit_f.c b/oshmem/shmem/fortran/shmem_global_exit_f.c
index 72ec888be59..a210d979eea 100644
--- a/oshmem/shmem/fortran/shmem_global_exit_f.c
+++ b/oshmem/shmem/fortran/shmem_global_exit_f.c
@@ -14,9 +14,9 @@
 #include "oshmem/include/shmem.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_GLOBAL_EXIT, shmem_global_exit)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_iget128_f.c b/oshmem/shmem/fortran/shmem_iget128_f.c
index 0a2737ae4cf..d371f84b11d 100644
--- a/oshmem/shmem/fortran/shmem_iget128_f.c
+++ b/oshmem/shmem/fortran/shmem_iget128_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_IGET128, shmem_iget128)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_iget32_f.c b/oshmem/shmem/fortran/shmem_iget32_f.c
index e1b8395ed25..17e90e9f677 100644
--- a/oshmem/shmem/fortran/shmem_iget32_f.c
+++ b/oshmem/shmem/fortran/shmem_iget32_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_IGET32, shmem_iget32)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_iget4_f.c b/oshmem/shmem/fortran/shmem_iget4_f.c
index 868d8a9b72f..56727817399 100644
--- a/oshmem/shmem/fortran/shmem_iget4_f.c
+++ b/oshmem/shmem/fortran/shmem_iget4_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_IGET4, shmem_iget4)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_iget64_f.c b/oshmem/shmem/fortran/shmem_iget64_f.c
index fafab50c50f..2f5f9405465 100644
--- a/oshmem/shmem/fortran/shmem_iget64_f.c
+++ b/oshmem/shmem/fortran/shmem_iget64_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_IGET64, shmem_iget64)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_iget8_f.c b/oshmem/shmem/fortran/shmem_iget8_f.c
index 2e9d76733a9..03b24d70025 100644
--- a/oshmem/shmem/fortran/shmem_iget8_f.c
+++ b/oshmem/shmem/fortran/shmem_iget8_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_IGET8, shmem_iget8)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_init_f.c b/oshmem/shmem/fortran/shmem_init_f.c
index 2300284c8d4..c4531de8329 100644
--- a/oshmem/shmem/fortran/shmem_init_f.c
+++ b/oshmem/shmem/fortran/shmem_init_f.c
@@ -14,10 +14,10 @@
 #include "oshmem/include/shmem.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INIT, shmem_init)
 SHMEM_GENERATE_WEAK_BINDINGS(START_PES, start_pes)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_int4_add_f.c b/oshmem/shmem/fortran/shmem_int4_add_f.c
index 8a985a41cf2..6ff106601d6 100644
--- a/oshmem/shmem/fortran/shmem_int4_add_f.c
+++ b/oshmem/shmem/fortran/shmem_int4_add_f.c
@@ -20,9 +20,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT4_ADD, shmem_int4_add)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_int4_cswap_f.c b/oshmem/shmem/fortran/shmem_int4_cswap_f.c
index a1e5fbfe924..18d5ff6396b 100644
--- a/oshmem/shmem/fortran/shmem_int4_cswap_f.c
+++ b/oshmem/shmem/fortran/shmem_int4_cswap_f.c
@@ -22,9 +22,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT4_CSWAP, shmem_int4_cswap)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_FUNCTION (ompi_fortran_integer4_t,
diff --git a/oshmem/shmem/fortran/shmem_int4_fadd_f.c b/oshmem/shmem/fortran/shmem_int4_fadd_f.c
index e00d33f2f7f..03b37f9cd27 100644
--- a/oshmem/shmem/fortran/shmem_int4_fadd_f.c
+++ b/oshmem/shmem/fortran/shmem_int4_fadd_f.c
@@ -20,9 +20,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT4_FADD, shmem_int4_fadd)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_FUNCTION (ompi_fortran_integer4_t,
diff --git a/oshmem/shmem/fortran/shmem_int4_fetch_f.c b/oshmem/shmem/fortran/shmem_int4_fetch_f.c
index 4e920f5b2a3..9de05b7a0f1 100644
--- a/oshmem/shmem/fortran/shmem_int4_fetch_f.c
+++ b/oshmem/shmem/fortran/shmem_int4_fetch_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT4_FETCH, shmem_int4_fetch)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_FUNCTION (ompi_fortran_integer4_t,
diff --git a/oshmem/shmem/fortran/shmem_int4_finc_f.c b/oshmem/shmem/fortran/shmem_int4_finc_f.c
index acc53355b28..cefcf506f44 100644
--- a/oshmem/shmem/fortran/shmem_int4_finc_f.c
+++ b/oshmem/shmem/fortran/shmem_int4_finc_f.c
@@ -20,9 +20,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT4_FINC, shmem_int4_finc)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_FUNCTION (ompi_fortran_integer4_t,
diff --git a/oshmem/shmem/fortran/shmem_int4_inc_f.c b/oshmem/shmem/fortran/shmem_int4_inc_f.c
index ebf039f1d1d..0c3fe128c95 100644
--- a/oshmem/shmem/fortran/shmem_int4_inc_f.c
+++ b/oshmem/shmem/fortran/shmem_int4_inc_f.c
@@ -20,9 +20,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT4_INC, shmem_int4_inc)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_int4_set_f.c b/oshmem/shmem/fortran/shmem_int4_set_f.c
index 5ef02b5967c..f76cbc80cc3 100644
--- a/oshmem/shmem/fortran/shmem_int4_set_f.c
+++ b/oshmem/shmem/fortran/shmem_int4_set_f.c
@@ -18,9 +18,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT4_SET, shmem_int4_set)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_int4_swap_f.c b/oshmem/shmem/fortran/shmem_int4_swap_f.c
index f0de74e1540..d608961e835 100644
--- a/oshmem/shmem/fortran/shmem_int4_swap_f.c
+++ b/oshmem/shmem/fortran/shmem_int4_swap_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT4_SWAP, shmem_int4_swap)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_FUNCTION (ompi_fortran_integer4_t,
diff --git a/oshmem/shmem/fortran/shmem_int4_wait_f.c b/oshmem/shmem/fortran/shmem_int4_wait_f.c
index 42f766580f8..cdab0ed62ad 100644
--- a/oshmem/shmem/fortran/shmem_int4_wait_f.c
+++ b/oshmem/shmem/fortran/shmem_int4_wait_f.c
@@ -17,9 +17,9 @@
 #include "ompi/datatype/ompi_datatype.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT4_WAIT, shmem_int4_wait)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_int4_wait_until_f.c b/oshmem/shmem/fortran/shmem_int4_wait_until_f.c
index c2411315a33..10a6aa9ec57 100644
--- a/oshmem/shmem/fortran/shmem_int4_wait_until_f.c
+++ b/oshmem/shmem/fortran/shmem_int4_wait_until_f.c
@@ -17,9 +17,9 @@
 #include "ompi/datatype/ompi_datatype.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT4_WAIT_UNTIL, shmem_int4_wait_until)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_int8_add_f.c b/oshmem/shmem/fortran/shmem_int8_add_f.c
index 67dfbef61b1..b238e454f8c 100644
--- a/oshmem/shmem/fortran/shmem_int8_add_f.c
+++ b/oshmem/shmem/fortran/shmem_int8_add_f.c
@@ -20,9 +20,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT8_ADD, shmem_int8_add)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_int8_cswap_f.c b/oshmem/shmem/fortran/shmem_int8_cswap_f.c
index e4e4637e1c1..d0ec61c495f 100644
--- a/oshmem/shmem/fortran/shmem_int8_cswap_f.c
+++ b/oshmem/shmem/fortran/shmem_int8_cswap_f.c
@@ -21,9 +21,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT8_CSWAP, shmem_int8_cswap)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_FUNCTION (ompi_fortran_integer8_t,
diff --git a/oshmem/shmem/fortran/shmem_int8_fadd_f.c b/oshmem/shmem/fortran/shmem_int8_fadd_f.c
index 9c62a499a1d..e942b55eaf0 100644
--- a/oshmem/shmem/fortran/shmem_int8_fadd_f.c
+++ b/oshmem/shmem/fortran/shmem_int8_fadd_f.c
@@ -20,9 +20,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT8_FADD, shmem_int8_fadd)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_FUNCTION (ompi_fortran_integer8_t,
diff --git a/oshmem/shmem/fortran/shmem_int8_fetch_f.c b/oshmem/shmem/fortran/shmem_int8_fetch_f.c
index 4795e27838b..0aee9fcc128 100644
--- a/oshmem/shmem/fortran/shmem_int8_fetch_f.c
+++ b/oshmem/shmem/fortran/shmem_int8_fetch_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT8_FETCH, shmem_int8_fetch)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_FUNCTION (ompi_fortran_integer8_t,
diff --git a/oshmem/shmem/fortran/shmem_int8_finc_f.c b/oshmem/shmem/fortran/shmem_int8_finc_f.c
index 5cc8f94bfac..99f7d23af92 100644
--- a/oshmem/shmem/fortran/shmem_int8_finc_f.c
+++ b/oshmem/shmem/fortran/shmem_int8_finc_f.c
@@ -20,9 +20,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT8_FINC, shmem_int8_finc)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_FUNCTION (ompi_fortran_integer8_t,
diff --git a/oshmem/shmem/fortran/shmem_int8_inc_f.c b/oshmem/shmem/fortran/shmem_int8_inc_f.c
index 984a62c086c..1b7af91fee0 100644
--- a/oshmem/shmem/fortran/shmem_int8_inc_f.c
+++ b/oshmem/shmem/fortran/shmem_int8_inc_f.c
@@ -20,9 +20,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT8_INC, shmem_int8_inc)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_int8_set_f.c b/oshmem/shmem/fortran/shmem_int8_set_f.c
index 182680c306f..676df463a81 100644
--- a/oshmem/shmem/fortran/shmem_int8_set_f.c
+++ b/oshmem/shmem/fortran/shmem_int8_set_f.c
@@ -18,9 +18,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT8_SET, shmem_int8_set)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_int8_swap_f.c b/oshmem/shmem/fortran/shmem_int8_swap_f.c
index 36dd76f67d1..b03cd4153b8 100644
--- a/oshmem/shmem/fortran/shmem_int8_swap_f.c
+++ b/oshmem/shmem/fortran/shmem_int8_swap_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT8_SWAP, shmem_int8_swap)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_FUNCTION (ompi_fortran_integer8_t,
diff --git a/oshmem/shmem/fortran/shmem_int8_wait_f.c b/oshmem/shmem/fortran/shmem_int8_wait_f.c
index be9fcf7d094..021ed612ce8 100644
--- a/oshmem/shmem/fortran/shmem_int8_wait_f.c
+++ b/oshmem/shmem/fortran/shmem_int8_wait_f.c
@@ -17,9 +17,9 @@
 #include "ompi/datatype/ompi_datatype.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT8_WAIT, shmem_int8_wait)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_int8_wait_until_f.c b/oshmem/shmem/fortran/shmem_int8_wait_until_f.c
index a40f4b15359..65b523fa632 100644
--- a/oshmem/shmem/fortran/shmem_int8_wait_until_f.c
+++ b/oshmem/shmem/fortran/shmem_int8_wait_until_f.c
@@ -17,9 +17,9 @@
 #include "ompi/datatype/ompi_datatype.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT8_WAIT_UNTIL, shmem_int8_wait_until)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_integer_get_f.c b/oshmem/shmem/fortran/shmem_integer_get_f.c
index 1442fe43021..b88efcce2f6 100644
--- a/oshmem/shmem/fortran/shmem_integer_get_f.c
+++ b/oshmem/shmem/fortran/shmem_integer_get_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INTEGER_GET, shmem_integer_get)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_integer_iget_f.c b/oshmem/shmem/fortran/shmem_integer_iget_f.c
index 66350fde108..08b59fcbb79 100644
--- a/oshmem/shmem/fortran/shmem_integer_iget_f.c
+++ b/oshmem/shmem/fortran/shmem_integer_iget_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INTEGER_IGET, shmem_integer_iget)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_integer_iput_f.c b/oshmem/shmem/fortran/shmem_integer_iput_f.c
index 393bc053870..730a5e99890 100644
--- a/oshmem/shmem/fortran/shmem_integer_iput_f.c
+++ b/oshmem/shmem/fortran/shmem_integer_iput_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INTEGER_IPUT, shmem_integer_iput)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_integer_put_f.c b/oshmem/shmem/fortran/shmem_integer_put_f.c
index 1505d22edff..bea4371ddc5 100644
--- a/oshmem/shmem/fortran/shmem_integer_put_f.c
+++ b/oshmem/shmem/fortran/shmem_integer_put_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INTEGER_PUT, shmem_integer_put)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_iput128_f.c b/oshmem/shmem/fortran/shmem_iput128_f.c
index 4428bf7af64..1f7105f4896 100644
--- a/oshmem/shmem/fortran/shmem_iput128_f.c
+++ b/oshmem/shmem/fortran/shmem_iput128_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_IPUT128, shmem_iput128)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_iput32_f.c b/oshmem/shmem/fortran/shmem_iput32_f.c
index 3d57fcc691c..6167a5085f4 100644
--- a/oshmem/shmem/fortran/shmem_iput32_f.c
+++ b/oshmem/shmem/fortran/shmem_iput32_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_IPUT32, shmem_iput32)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_iput4_f.c b/oshmem/shmem/fortran/shmem_iput4_f.c
index 4e89c33bdb3..aef7a6670a6 100644
--- a/oshmem/shmem/fortran/shmem_iput4_f.c
+++ b/oshmem/shmem/fortran/shmem_iput4_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_IPUT4, shmem_iput4)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_iput64_f.c b/oshmem/shmem/fortran/shmem_iput64_f.c
index 0f4fb3e6d0e..35a4392bdce 100644
--- a/oshmem/shmem/fortran/shmem_iput64_f.c
+++ b/oshmem/shmem/fortran/shmem_iput64_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_IPUT64, shmem_iput64)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_iput8_f.c b/oshmem/shmem/fortran/shmem_iput8_f.c
index 0c5e0b2fcaa..434488b399a 100644
--- a/oshmem/shmem/fortran/shmem_iput8_f.c
+++ b/oshmem/shmem/fortran/shmem_iput8_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_IPUT8, shmem_iput8)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_lock_f.c b/oshmem/shmem/fortran/shmem_lock_f.c
index ce5bd48b4fa..563535f48be 100644
--- a/oshmem/shmem/fortran/shmem_lock_f.c
+++ b/oshmem/shmem/fortran/shmem_lock_f.c
@@ -17,11 +17,11 @@
 #include "oshmem/shmem/shmem_lock.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_SET_LOCK, shmem_set_lock)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_CLEAR_LOCK, shmem_clear_lock)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_TEST_LOCK, shmem_test_lock)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_logical_get_f.c b/oshmem/shmem/fortran/shmem_logical_get_f.c
index 68b0188f1ae..ed9cf4327e2 100644
--- a/oshmem/shmem/fortran/shmem_logical_get_f.c
+++ b/oshmem/shmem/fortran/shmem_logical_get_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_LOGICAL_GET, shmem_logical_get)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_logical_iget_f.c b/oshmem/shmem/fortran/shmem_logical_iget_f.c
index 9f6e54ae9ac..e3ab0b79dc5 100644
--- a/oshmem/shmem/fortran/shmem_logical_iget_f.c
+++ b/oshmem/shmem/fortran/shmem_logical_iget_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_LOGICAL_IGET, shmem_logical_iget)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_logical_iput_f.c b/oshmem/shmem/fortran/shmem_logical_iput_f.c
index f0f704fbc20..dc22fcd3133 100644
--- a/oshmem/shmem/fortran/shmem_logical_iput_f.c
+++ b/oshmem/shmem/fortran/shmem_logical_iput_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_LOGICAL_IPUT, shmem_logical_iput)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_logical_put_f.c b/oshmem/shmem/fortran/shmem_logical_put_f.c
index d1c2cf39b15..b41ac40a020 100644
--- a/oshmem/shmem/fortran/shmem_logical_put_f.c
+++ b/oshmem/shmem/fortran/shmem_logical_put_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_LOGICAL_PUT, shmem_logical_put)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_max_to_all_f.c b/oshmem/shmem/fortran/shmem_max_to_all_f.c
index 8b3b4465317..9c3d2c9df73 100644
--- a/oshmem/shmem/fortran/shmem_max_to_all_f.c
+++ b/oshmem/shmem/fortran/shmem_max_to_all_f.c
@@ -18,14 +18,14 @@
 #include "oshmem/op/op.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT2_MAX_TO_ALL, shmem_int2_max_to_all)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT4_MAX_TO_ALL, shmem_int4_max_to_all)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT8_MAX_TO_ALL, shmem_int8_max_to_all)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_REAL4_MAX_TO_ALL, shmem_real4_max_to_all)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_REAL8_MAX_TO_ALL, shmem_real8_max_to_all)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_REAL16_MAX_TO_ALL, shmem_real16_max_to_all)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_min_to_all_f.c b/oshmem/shmem/fortran/shmem_min_to_all_f.c
index 22201286f8e..9f73d80f663 100644
--- a/oshmem/shmem/fortran/shmem_min_to_all_f.c
+++ b/oshmem/shmem/fortran/shmem_min_to_all_f.c
@@ -18,14 +18,14 @@
 #include "oshmem/op/op.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT2_MIN_TO_ALL, shmem_int2_min_to_all)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT4_MIN_TO_ALL, shmem_int4_min_to_all)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT8_MIN_TO_ALL, shmem_int8_min_to_all)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_REAL4_MIN_TO_ALL, shmem_real4_min_to_all)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_REAL8_MIN_TO_ALL, shmem_real8_min_to_all)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_REAL16_MIN_TO_ALL, shmem_real16_min_to_all)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_my_pe_f.c b/oshmem/shmem/fortran/shmem_my_pe_f.c
index f8823896bb9..d7c844233df 100644
--- a/oshmem/shmem/fortran/shmem_my_pe_f.c
+++ b/oshmem/shmem/fortran/shmem_my_pe_f.c
@@ -14,11 +14,11 @@
 #include "oshmem/include/shmem.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_MY_PE, shmem_my_pe)
 SHMEM_GENERATE_WEAK_BINDINGS(MY_PE, my_pe)
 #pragma weak _my_pe_ = p_my_pe_
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_FUNCTION (MPI_Fint,
diff --git a/oshmem/shmem/fortran/shmem_n_pes_f.c b/oshmem/shmem/fortran/shmem_n_pes_f.c
index 3647625055e..ae39d46903a 100644
--- a/oshmem/shmem/fortran/shmem_n_pes_f.c
+++ b/oshmem/shmem/fortran/shmem_n_pes_f.c
@@ -14,10 +14,10 @@
 #include "oshmem/include/shmem.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_N_PES, shmem_n_pes)
 SHMEM_GENERATE_WEAK_BINDINGS(NUM_PES, num_pes)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_FUNCTION (MPI_Fint,
diff --git a/oshmem/shmem/fortran/shmem_or_to_all_f.c b/oshmem/shmem/fortran/shmem_or_to_all_f.c
index 7bca154b606..f800d3a8062 100644
--- a/oshmem/shmem/fortran/shmem_or_to_all_f.c
+++ b/oshmem/shmem/fortran/shmem_or_to_all_f.c
@@ -18,11 +18,11 @@
 #include "oshmem/op/op.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT2_OR_TO_ALL, shmem_int2_or_to_all)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT4_OR_TO_ALL, shmem_int4_or_to_all)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT8_OR_TO_ALL, shmem_int8_or_to_all)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_pe_accessible_f.c b/oshmem/shmem/fortran/shmem_pe_accessible_f.c
index 191633897d5..cc90e1c01f9 100644
--- a/oshmem/shmem/fortran/shmem_pe_accessible_f.c
+++ b/oshmem/shmem/fortran/shmem_pe_accessible_f.c
@@ -16,9 +16,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_PE_ACCESSIBLE, shmem_pe_accessible)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_FUNCTION (ompi_fortran_logical_t,
diff --git a/oshmem/shmem/fortran/shmem_prod_to_all_f.c b/oshmem/shmem/fortran/shmem_prod_to_all_f.c
index c093bb00573..e1732dbf2cb 100644
--- a/oshmem/shmem/fortran/shmem_prod_to_all_f.c
+++ b/oshmem/shmem/fortran/shmem_prod_to_all_f.c
@@ -18,7 +18,7 @@
 #include "oshmem/op/op.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT2_PROD_TO_ALL, shmem_int2_prod_to_all)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT4_PROD_TO_ALL, shmem_int4_prod_to_all)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT8_PROD_TO_ALL, shmem_int8_prod_to_all)
@@ -27,7 +27,7 @@ SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_COMP8_PROD_TO_ALL, shmem_comp8_prod_to_all)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_REAL4_PROD_TO_ALL, shmem_real4_prod_to_all)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_REAL8_PROD_TO_ALL, shmem_real8_prod_to_all)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_REAL16_PROD_TO_ALL, shmem_real16_prod_to_all)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_ptr_f.c b/oshmem/shmem/fortran/shmem_ptr_f.c
index 74df2655695..78669cfa53d 100644
--- a/oshmem/shmem/fortran/shmem_ptr_f.c
+++ b/oshmem/shmem/fortran/shmem_ptr_f.c
@@ -16,9 +16,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_PTR, shmem_ptr)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_FUNCTION (FORTRAN_POINTER_T *,
diff --git a/oshmem/shmem/fortran/shmem_put128_f.c b/oshmem/shmem/fortran/shmem_put128_f.c
index cabb0ded118..a77e82f16a9 100644
--- a/oshmem/shmem/fortran/shmem_put128_f.c
+++ b/oshmem/shmem/fortran/shmem_put128_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_PUT128, shmem_put128)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_put32_f.c b/oshmem/shmem/fortran/shmem_put32_f.c
index 2f2c2510ef1..f7e5add52fb 100644
--- a/oshmem/shmem/fortran/shmem_put32_f.c
+++ b/oshmem/shmem/fortran/shmem_put32_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_PUT32, shmem_put32)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_put4_f.c b/oshmem/shmem/fortran/shmem_put4_f.c
index 6fba537602a..7546a7905e5 100644
--- a/oshmem/shmem/fortran/shmem_put4_f.c
+++ b/oshmem/shmem/fortran/shmem_put4_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_PUT4, shmem_put4)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_put64_f.c b/oshmem/shmem/fortran/shmem_put64_f.c
index caad85c91a1..dd28c09dc80 100644
--- a/oshmem/shmem/fortran/shmem_put64_f.c
+++ b/oshmem/shmem/fortran/shmem_put64_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_PUT64, shmem_put64)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_put8_f.c b/oshmem/shmem/fortran/shmem_put8_f.c
index ba484ecaf19..b9a0b1a7b04 100644
--- a/oshmem/shmem/fortran/shmem_put8_f.c
+++ b/oshmem/shmem/fortran/shmem_put8_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_PUT8, shmem_put8)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_put_nb_f.c b/oshmem/shmem/fortran/shmem_put_nb_f.c
index b1cabcbfa18..fb43c6b1e07 100644
--- a/oshmem/shmem/fortran/shmem_put_nb_f.c
+++ b/oshmem/shmem/fortran/shmem_put_nb_f.c
@@ -18,7 +18,7 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_PUTMEM_NBI, shmem_putmem_nbi)
 
@@ -35,7 +35,7 @@ SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_PUT32_NBI, shmem_put32_nbi)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_PUT64_NBI, shmem_put64_nbi)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_PUT128_NBI, shmem_put128_nbi)
 
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_putmem_f.c b/oshmem/shmem/fortran/shmem_putmem_f.c
index 663540fea27..be93d6eec50 100644
--- a/oshmem/shmem/fortran/shmem_putmem_f.c
+++ b/oshmem/shmem/fortran/shmem_putmem_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_PUTMEM, shmem_putmem)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_quiet_f.c b/oshmem/shmem/fortran/shmem_quiet_f.c
index 24c03e6ae15..ee42c72890e 100644
--- a/oshmem/shmem/fortran/shmem_quiet_f.c
+++ b/oshmem/shmem/fortran/shmem_quiet_f.c
@@ -15,9 +15,9 @@
 #include "oshmem/mca/spml/spml.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_QUIET, shmem_quiet)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_real4_fetch_f.c b/oshmem/shmem/fortran/shmem_real4_fetch_f.c
index 66c3b84b9c6..56d5330c477 100644
--- a/oshmem/shmem/fortran/shmem_real4_fetch_f.c
+++ b/oshmem/shmem/fortran/shmem_real4_fetch_f.c
@@ -18,9 +18,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_REAL4_FETCH, shmem_real4_fetch)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_FUNCTION (ompi_fortran_real4_t,
diff --git a/oshmem/shmem/fortran/shmem_real4_set_f.c b/oshmem/shmem/fortran/shmem_real4_set_f.c
index 1a4e222cc2f..e61e584afb4 100644
--- a/oshmem/shmem/fortran/shmem_real4_set_f.c
+++ b/oshmem/shmem/fortran/shmem_real4_set_f.c
@@ -18,9 +18,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_REAL4_SET, shmem_real4_set)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_real4_swap_f.c b/oshmem/shmem/fortran/shmem_real4_swap_f.c
index 57f21d55ba1..4ac9407f5f5 100644
--- a/oshmem/shmem/fortran/shmem_real4_swap_f.c
+++ b/oshmem/shmem/fortran/shmem_real4_swap_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_REAL4_SWAP, shmem_real4_swap)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_FUNCTION (ompi_fortran_real4_t,
diff --git a/oshmem/shmem/fortran/shmem_real8_fetch_f.c b/oshmem/shmem/fortran/shmem_real8_fetch_f.c
index d7a2381df6a..37e0fc09d5a 100644
--- a/oshmem/shmem/fortran/shmem_real8_fetch_f.c
+++ b/oshmem/shmem/fortran/shmem_real8_fetch_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_REAL8_FETCH, shmem_real8_fetch)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_FUNCTION (ompi_fortran_real8_t,
diff --git a/oshmem/shmem/fortran/shmem_real8_set_f.c b/oshmem/shmem/fortran/shmem_real8_set_f.c
index 7c7384e43af..7d29dac429b 100644
--- a/oshmem/shmem/fortran/shmem_real8_set_f.c
+++ b/oshmem/shmem/fortran/shmem_real8_set_f.c
@@ -18,9 +18,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_REAL8_SET, shmem_real8_set)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_real8_swap_f.c b/oshmem/shmem/fortran/shmem_real8_swap_f.c
index e08822c2f82..4d084283629 100644
--- a/oshmem/shmem/fortran/shmem_real8_swap_f.c
+++ b/oshmem/shmem/fortran/shmem_real8_swap_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_REAL8_SWAP, shmem_real8_swap)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_FUNCTION (ompi_fortran_real8_t,
diff --git a/oshmem/shmem/fortran/shmem_real_get_f.c b/oshmem/shmem/fortran/shmem_real_get_f.c
index ef0c337ccd6..8d2370a0d8a 100644
--- a/oshmem/shmem/fortran/shmem_real_get_f.c
+++ b/oshmem/shmem/fortran/shmem_real_get_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_REAL_GET, shmem_real_get)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_real_iget_f.c b/oshmem/shmem/fortran/shmem_real_iget_f.c
index dd6d4567b23..f1a54d2dc22 100644
--- a/oshmem/shmem/fortran/shmem_real_iget_f.c
+++ b/oshmem/shmem/fortran/shmem_real_iget_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_REAL_IGET, shmem_real_iget)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_real_iput_f.c b/oshmem/shmem/fortran/shmem_real_iput_f.c
index 7383dc74e3b..13f1a56397b 100644
--- a/oshmem/shmem/fortran/shmem_real_iput_f.c
+++ b/oshmem/shmem/fortran/shmem_real_iput_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_REAL_IPUT, shmem_real_iput)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_real_put_f.c b/oshmem/shmem/fortran/shmem_real_put_f.c
index 3ef8582069b..26e2baff19c 100644
--- a/oshmem/shmem/fortran/shmem_real_put_f.c
+++ b/oshmem/shmem/fortran/shmem_real_put_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_REAL_PUT, shmem_real_put)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_sum_to_all_f.c b/oshmem/shmem/fortran/shmem_sum_to_all_f.c
index ca48f484407..aca69a06b76 100644
--- a/oshmem/shmem/fortran/shmem_sum_to_all_f.c
+++ b/oshmem/shmem/fortran/shmem_sum_to_all_f.c
@@ -18,7 +18,7 @@
 #include "oshmem/op/op.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT2_SUM_TO_ALL, shmem_int2_sum_to_all)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT4_SUM_TO_ALL, shmem_int4_sum_to_all)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT8_SUM_TO_ALL, shmem_int8_sum_to_all)
@@ -27,7 +27,7 @@ SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_COMP8_SUM_TO_ALL, shmem_comp8_sum_to_all)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_REAL4_SUM_TO_ALL, shmem_real4_sum_to_all)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_REAL8_SUM_TO_ALL, shmem_real8_sum_to_all)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_REAL16_SUM_TO_ALL, shmem_real16_sum_to_all)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_swap_f.c b/oshmem/shmem/fortran/shmem_swap_f.c
index 4378d72908c..aa03abcbfa4 100644
--- a/oshmem/shmem/fortran/shmem_swap_f.c
+++ b/oshmem/shmem/fortran/shmem_swap_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_SWAP, shmem_swap)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_FUNCTION (MPI_Fint,
diff --git a/oshmem/shmem/fortran/shmem_wait_f.c b/oshmem/shmem/fortran/shmem_wait_f.c
index 5cd12f5487f..e188a35b2ce 100644
--- a/oshmem/shmem/fortran/shmem_wait_f.c
+++ b/oshmem/shmem/fortran/shmem_wait_f.c
@@ -17,9 +17,9 @@
 #include "ompi/datatype/ompi_datatype.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_WAIT, shmem_wait)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_wait_until_f.c b/oshmem/shmem/fortran/shmem_wait_until_f.c
index c3f47c2946d..574475ce4ee 100644
--- a/oshmem/shmem/fortran/shmem_wait_until_f.c
+++ b/oshmem/shmem/fortran/shmem_wait_until_f.c
@@ -17,9 +17,9 @@
 #include "ompi/datatype/ompi_datatype.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_WAIT_UNTIL, shmem_wait_until)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shmem_xor_to_all_f.c b/oshmem/shmem/fortran/shmem_xor_to_all_f.c
index f85d62b92b0..25dd043a2da 100644
--- a/oshmem/shmem/fortran/shmem_xor_to_all_f.c
+++ b/oshmem/shmem/fortran/shmem_xor_to_all_f.c
@@ -18,13 +18,13 @@
 #include "oshmem/op/op.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT2_XOR_TO_ALL, shmem_int2_xor_to_all)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT4_XOR_TO_ALL, shmem_int4_xor_to_all)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_INT8_XOR_TO_ALL, shmem_int8_xor_to_all)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_COMP4_XOR_TO_ALL, shmem_comp4_xor_to_all)
 SHMEM_GENERATE_WEAK_BINDINGS(SHMEM_COMP8_XOR_TO_ALL, shmem_comp8_xor_to_all)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shpalloc_f.c b/oshmem/shmem/fortran/shpalloc_f.c
index 424ae85fa0c..b2104712978 100644
--- a/oshmem/shmem/fortran/shpalloc_f.c
+++ b/oshmem/shmem/fortran/shpalloc_f.c
@@ -19,9 +19,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHPALLOC, shpalloc)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shpclmove_f.c b/oshmem/shmem/fortran/shpclmove_f.c
index 10266a2aba9..c39168fce62 100644
--- a/oshmem/shmem/fortran/shpclmove_f.c
+++ b/oshmem/shmem/fortran/shpclmove_f.c
@@ -17,9 +17,9 @@
 #include "oshmem/runtime/runtime.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHPCLMOVE, shpclmove)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/fortran/shpdeallc_f.c b/oshmem/shmem/fortran/shpdeallc_f.c
index fd529a6d8c8..9eef7901965 100644
--- a/oshmem/shmem/fortran/shpdeallc_f.c
+++ b/oshmem/shmem/fortran/shpdeallc_f.c
@@ -16,9 +16,9 @@
 #include "stdio.h"
 
 #if OSHMEM_PROFILING
-#include "oshmem/shmem/fortran/profile/pbindings.h"
+#include "oshmem/shmem/fortran/pbindings.h"
 SHMEM_GENERATE_WEAK_BINDINGS(SHPDEALLC, shpdeallc)
-#include "oshmem/shmem/fortran/profile/defines.h"
+#include "oshmem/shmem/fortran/profile-defines.h"
 #endif
 
 SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
diff --git a/oshmem/shmem/man/man3/Makefile.extra b/oshmem/shmem/man/man3/Makefile.extra
deleted file mode 100644
index c47ba2c2fb7..00000000000
--- a/oshmem/shmem/man/man3/Makefile.extra
+++ /dev/null
@@ -1,247 +0,0 @@
-# -*- makefile -*-
-# Copyright (c) 2015      Mellanox Technologies, Inc.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-include $(top_srcdir)/Makefile.ompi-rules
-
-shmem_legacy_api_man_pages = \
-        shmem/man/man3/start_pes.3 \
-        shmem/man/man3/_num_pes.3 \
-        shmem/man/man3/_my_pe.3 \
-        shmem/man/man3/shmalloc.3 \
-        shmem/man/man3/shmemalign.3 \
-        shmem/man/man3/shrealloc.3 \
-        shmem/man/man3/shfree.3
-
-shmem_api_man_pages = \
-        shmem/man/man3/intro_shmem.3 \
-        shmem/man/man3/OpenSHMEM.3 \
-        shmem/man/man3/shmem_init.3 \
-        shmem/man/man3/shmem_finalize.3 \
-        shmem/man/man3/shmem_n_pes.3 \
-        shmem/man/man3/shmem_my_pe.3 \
-        shmem/man/man3/shmem_pe_accessible.3 \
-        shmem/man/man3/shmem_addr_accessible.3 \
-        shmem/man/man3/shmem_malloc.3 \
-        shmem/man/man3/shmem_align.3 \
-        shmem/man/man3/shmem_realloc.3 \
-        shmem/man/man3/shmem_free.3 \
-        shmem/man/man3/shmem_ptr.3 \
-        shmem/man/man3/shmem_char_p.3 \
-        shmem/man/man3/shmem_short_p.3 \
-        shmem/man/man3/shmem_int_p.3 \
-        shmem/man/man3/shmem_long_p.3 \
-        shmem/man/man3/shmem_float_p.3 \
-        shmem/man/man3/shmem_double_p.3 \
-        shmem/man/man3/shmem_longlong_p.3 \
-        shmem/man/man3/shmem_longdouble_p.3 \
-        shmem/man/man3/shmem_char_put.3 \
-        shmem/man/man3/shmem_short_put.3 \
-        shmem/man/man3/shmem_int_put.3 \
-        shmem/man/man3/shmem_long_put.3 \
-        shmem/man/man3/shmem_float_put.3 \
-        shmem/man/man3/shmem_double_put.3 \
-        shmem/man/man3/shmem_longlong_put.3 \
-        shmem/man/man3/shmem_longdouble_put.3 \
-        shmem/man/man3/shmem_put32.3 \
-        shmem/man/man3/shmem_put64.3 \
-        shmem/man/man3/shmem_put128.3 \
-        shmem/man/man3/shmem_putmem.3 \
-        shmem/man/man3/shmem_short_iput.3 \
-        shmem/man/man3/shmem_int_iput.3 \
-        shmem/man/man3/shmem_long_iput.3 \
-        shmem/man/man3/shmem_float_iput.3 \
-        shmem/man/man3/shmem_double_iput.3 \
-        shmem/man/man3/shmem_longlong_iput.3 \
-        shmem/man/man3/shmem_longdouble_iput.3 \
-        shmem/man/man3/shmem_iput32.3 \
-        shmem/man/man3/shmem_iput64.3 \
-        shmem/man/man3/shmem_iput128.3 \
-        shmem/man/man3/shmem_char_g.3 \
-        shmem/man/man3/shmem_short_g.3 \
-        shmem/man/man3/shmem_int_g.3 \
-        shmem/man/man3/shmem_long_g.3 \
-        shmem/man/man3/shmem_float_g.3 \
-        shmem/man/man3/shmem_double_g.3 \
-        shmem/man/man3/shmem_longlong_g.3 \
-        shmem/man/man3/shmem_longdouble_g.3 \
-        shmem/man/man3/shmem_char_get.3 \
-        shmem/man/man3/shmem_short_get.3 \
-        shmem/man/man3/shmem_int_get.3 \
-        shmem/man/man3/shmem_long_get.3 \
-        shmem/man/man3/shmem_float_get.3 \
-        shmem/man/man3/shmem_double_get.3 \
-        shmem/man/man3/shmem_longlong_get.3 \
-        shmem/man/man3/shmem_longdouble_get.3 \
-        shmem/man/man3/shmem_get32.3 \
-        shmem/man/man3/shmem_get64.3 \
-        shmem/man/man3/shmem_get128.3 \
-        shmem/man/man3/shmem_getmem.3 \
-        shmem/man/man3/shmem_short_iget.3 \
-        shmem/man/man3/shmem_int_iget.3 \
-        shmem/man/man3/shmem_long_iget.3 \
-        shmem/man/man3/shmem_float_iget.3 \
-        shmem/man/man3/shmem_double_iget.3 \
-        shmem/man/man3/shmem_longlong_iget.3 \
-        shmem/man/man3/shmem_longdouble_iget.3 \
-        shmem/man/man3/shmem_iget32.3 \
-        shmem/man/man3/shmem_iget64.3 \
-        shmem/man/man3/shmem_iget128.3 \
-        shmem/man/man3/shmem_swap.3 \
-        shmem/man/man3/shmem_int_swap.3 \
-        shmem/man/man3/shmem_long_swap.3 \
-        shmem/man/man3/shmem_longlong_swap.3 \
-        shmem/man/man3/shmem_float_swap.3 \
-        shmem/man/man3/shmem_double_swap.3 \
-\
-        shmem/man/man3/shmem_double_set.3 \
-        shmem/man/man3/shmem_float_set.3 \
-        shmem/man/man3/shmem_int_set.3 \
-        shmem/man/man3/shmem_longlong_set.3 \
-        shmem/man/man3/shmem_long_set.3 \
-\
-        shmem/man/man3/shmem_int_cswap.3 \
-        shmem/man/man3/shmem_long_cswap.3 \
-        shmem/man/man3/shmem_longlong_cswap.3 \
-        shmem/man/man3/shmem_int_fadd.3 \
-        shmem/man/man3/shmem_long_fadd.3 \
-        shmem/man/man3/shmem_longlong_fadd.3 \
-\
-        shmem/man/man3/shmem_double_fetch.3 \
-        shmem/man/man3/shmem_float_fetch.3 \
-        shmem/man/man3/shmem_int_fetch.3 \
-        shmem/man/man3/shmem_long_fetch.3 \
-        shmem/man/man3/shmem_longlong_fetch.3 \
-\
-        shmem/man/man3/shmem_int_finc.3 \
-        shmem/man/man3/shmem_long_finc.3 \
-        shmem/man/man3/shmem_longlong_finc.3 \
-        shmem/man/man3/shmem_int_add.3 \
-        shmem/man/man3/shmem_long_add.3 \
-        shmem/man/man3/shmem_longlong_add.3 \
-        shmem/man/man3/shmem_int_inc.3 \
-        shmem/man/man3/shmem_long_inc.3 \
-        shmem/man/man3/shmem_longlong_inc.3 \
-        shmem/man/man3/shmem_set_lock.3 \
-        shmem/man/man3/shmem_clear_lock.3 \
-        shmem/man/man3/shmem_test_lock.3 \
-        shmem/man/man3/shmem_wait.3 \
-        shmem/man/man3/shmem_short_wait.3 \
-        shmem/man/man3/shmem_int_wait.3 \
-        shmem/man/man3/shmem_long_wait.3 \
-        shmem/man/man3/shmem_longlong_wait.3 \
-        shmem/man/man3/shmem_wait_until.3 \
-        shmem/man/man3/shmem_short_wait_until.3 \
-        shmem/man/man3/shmem_int_wait_until.3 \
-        shmem/man/man3/shmem_long_wait_until.3 \
-        shmem/man/man3/shmem_longlong_wait_until.3 \
-        shmem/man/man3/shmem_barrier.3 \
-        shmem/man/man3/shmem_barrier_all.3 \
-        shmem/man/man3/shmem_fence.3 \
-        shmem/man/man3/shmem_quiet.3 \
-        shmem/man/man3/shmem_broadcast32.3 \
-        shmem/man/man3/shmem_broadcast64.3 \
-        shmem/man/man3/shmem_alltoall32.3 \
-        shmem/man/man3/shmem_alltoall64.3 \
-        shmem/man/man3/shmem_alltoalls32.3 \
-        shmem/man/man3/shmem_alltoalls64.3 \
-        shmem/man/man3/shmem_collect32.3 \
-        shmem/man/man3/shmem_collect64.3 \
-        shmem/man/man3/shmem_fcollect32.3 \
-        shmem/man/man3/shmem_fcollect64.3 \
-        shmem/man/man3/shmem_short_and_to_all.3 \
-        shmem/man/man3/shmem_int_and_to_all.3 \
-        shmem/man/man3/shmem_long_and_to_all.3 \
-        shmem/man/man3/shmem_longlong_and_to_all.3 \
-        shmem/man/man3/shmem_short_or_to_all.3 \
-        shmem/man/man3/shmem_int_or_to_all.3 \
-        shmem/man/man3/shmem_long_or_to_all.3 \
-        shmem/man/man3/shmem_longlong_or_to_all.3 \
-        shmem/man/man3/shmem_short_xor_to_all.3 \
-        shmem/man/man3/shmem_int_xor_to_all.3 \
-        shmem/man/man3/shmem_long_xor_to_all.3 \
-        shmem/man/man3/shmem_longlong_xor_to_all.3 \
-        shmem/man/man3/shmem_short_max_to_all.3 \
-        shmem/man/man3/shmem_int_max_to_all.3 \
-        shmem/man/man3/shmem_long_max_to_all.3 \
-        shmem/man/man3/shmem_longlong_max_to_all.3 \
-        shmem/man/man3/shmem_float_max_to_all.3 \
-        shmem/man/man3/shmem_double_max_to_all.3 \
-        shmem/man/man3/shmem_longdouble_max_to_all.3 \
-        shmem/man/man3/shmem_short_min_to_all.3 \
-        shmem/man/man3/shmem_int_min_to_all.3 \
-        shmem/man/man3/shmem_long_min_to_all.3 \
-        shmem/man/man3/shmem_longlong_min_to_all.3 \
-        shmem/man/man3/shmem_float_min_to_all.3 \
-        shmem/man/man3/shmem_double_min_to_all.3 \
-        shmem/man/man3/shmem_longdouble_min_to_all.3 \
-        shmem/man/man3/shmem_short_sum_to_all.3 \
-        shmem/man/man3/shmem_int_sum_to_all.3 \
-        shmem/man/man3/shmem_long_sum_to_all.3 \
-        shmem/man/man3/shmem_longlong_sum_to_all.3 \
-        shmem/man/man3/shmem_float_sum_to_all.3 \
-        shmem/man/man3/shmem_double_sum_to_all.3 \
-        shmem/man/man3/shmem_complexf_sum_to_all.3 \
-        shmem/man/man3/shmem_complexd_sum_to_all.3 \
-        shmem/man/man3/shmem_short_prod_to_all.3 \
-        shmem/man/man3/shmem_int_prod_to_all.3 \
-        shmem/man/man3/shmem_long_prod_to_all.3 \
-        shmem/man/man3/shmem_longlong_prod_to_all.3 \
-        shmem/man/man3/shmem_float_prod_to_all.3 \
-        shmem/man/man3/shmem_double_prod_to_all.3 \
-        shmem/man/man3/shmem_longdouble_prod_to_all.3 \
-        shmem/man/man3/shmem_complexf_prod_to_all.3 \
-        shmem/man/man3/shmem_complexd_prod_to_all.3 \
-        shmem/man/man3/shmem_udcflush.3 \
-        shmem/man/man3/shmem_udcflush_line.3 \
-        shmem/man/man3/shmem_set_cache_inv.3 \
-        shmem/man/man3/shmem_set_cache_line_inv.3 \
-        shmem/man/man3/shmem_clear_cache_inv.3 \
-        shmem/man/man3/shmem_clear_cache_line_inv.3 \
-        shmem/man/man3/shmem_info_get_name.3 \
-        shmem/man/man3/shmem_info_get_version.3 \
-        shmem/man/man3/shmem_global_exit.3 \
-\
-        shmem/man/man3/shmem_getmem_nbi.3 \
-        shmem/man/man3/shmem_char_get_nbi.3 \
-        shmem/man/man3/shmem_short_get_nbi.3 \
-        shmem/man/man3/shmem_int_get_nbi.3 \
-        shmem/man/man3/shmem_long_get_nbi.3 \
-        shmem/man/man3/shmem_longlong_get_nbi.3 \
-        shmem/man/man3/shmem_float_get_nbi.3 \
-        shmem/man/man3/shmem_double_get_nbi.3 \
-        shmem/man/man3/shmem_longdouble_get_nbi.3 \
-        shmem/man/man3/shmem_get8_nbi.3 \
-        shmem/man/man3/shmem_get16_nbi.3 \
-        shmem/man/man3/shmem_get32_nbi.3 \
-        shmem/man/man3/shmem_get64_nbi.3 \
-        shmem/man/man3/shmem_get128_nbi.3 \
-\
-        shmem/man/man3/shmem_putmem_nbi.3 \
-        shmem/man/man3/shmem_char_put_nbi.3 \
-        shmem/man/man3/shmem_short_put_nbi.3 \
-        shmem/man/man3/shmem_int_put_nbi.3 \
-        shmem/man/man3/shmem_long_put_nbi.3 \
-        shmem/man/man3/shmem_longlong_put_nbi.3 \
-        shmem/man/man3/shmem_float_put_nbi.3 \
-        shmem/man/man3/shmem_double_put_nbi.3 \
-        shmem/man/man3/shmem_longdouble_put_nbi.3 \
-        shmem/man/man3/shmem_put8_nbi.3 \
-        shmem/man/man3/shmem_put16_nbi.3 \
-        shmem/man/man3/shmem_put32_nbi.3 \
-        shmem/man/man3/shmem_put64_nbi.3 \
-        shmem/man/man3/shmem_put128_nbi.3
-
-
-if PROJECT_OSHMEM
-nodist_man_MANS += $(shmem_api_man_pages)
-nodist_man_MANS += $(shmem_legacy_api_man_pages)
-endif  # PROJECT_OSHMEM
-EXTRA_DIST += $(shmem_api_man_pages:.3=.3in)
-EXTRA_DIST += $(shmem_legacy_api_man_pages:.3=.3in)
-
diff --git a/oshmem/shmem/man/man3/OpenSHMEM.3in b/oshmem/shmem/man/man3/OpenSHMEM.3in
deleted file mode 100644
index 97469c502b1..00000000000
--- a/oshmem/shmem/man/man3/OpenSHMEM.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/intro_shmem.3
diff --git a/oshmem/shmem/man/man3/_my_pe.3in b/oshmem/shmem/man/man3/_my_pe.3in
deleted file mode 100644
index 228477044bc..00000000000
--- a/oshmem/shmem/man/man3/_my_pe.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_my_pe.3
diff --git a/oshmem/shmem/man/man3/_num_pes.3in b/oshmem/shmem/man/man3/_num_pes.3in
deleted file mode 100644
index ce155633cae..00000000000
--- a/oshmem/shmem/man/man3/_num_pes.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_n_pes.3
diff --git a/oshmem/shmem/man/man3/intro_shmem.3in b/oshmem/shmem/man/man3/intro_shmem.3in
deleted file mode 100644
index 95b6c977ff5..00000000000
--- a/oshmem/shmem/man/man3/intro_shmem.3in
+++ /dev/null
@@ -1,1311 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "INTRO\\_SHMEM" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-intro_shmem \- Introduction to the OpenSHMEM programming model
-.PP
-.SH DESCRIPTION
-
-The SHMEM programming model consists of library routines that provide low\-latency,
-high\-bandwidth communication for use in highly parallelized scalable programs. The
-routines in the OpenSHMEM application programming interface (API) provide a programming
-model for exchanging data between cooperating parallel processes. The resulting programs
-are similar in style to Message Passing Interface (MPI) programs. The SHMEM API can
-be used either alone or in combination with MPI routines in the same parallel program.
-.PP
-An OpenSHMEM program is SPMD (single program, multiple data) in style. The SHMEM
-processes, called processing elements or PEs, all start at the same time and they all run the
-same program. Usually the PEs perform computation on their own subdomains of the larger
-problem and periodically communicate with other PEs to exchange information on
-which the next computation phase depends.
-.PP
-The OpenSHMEM routines minimize the overhead associated with data transfer requests,
-maximize bandwidth and minimize data latency. Data latency is the period of time that
-starts when a PE initiates a transfer of data and ends when a PE can use the data.
-OpenSHMEM routines support remote data transfer through put operations, which transfer
-data to a different PE, get operations, which transfer data from a different PE, and remote
-pointers, which allow direct references to data objects owned by another PE. Other
-operations supported are collective broadcast and reduction, barrier synchronization, and
-atomic memory operations. An atomic memory operation is an atomic read\-and\-update
-operation, such as a fetch\-and\-increment, on a remote or local data object.
-.PP
-.SH OPENSHMEM ROUTINES
-
-This section lists the significant OpenSHMEM message\-passing routines.
-.TP
-PE queries
-.PP
-.RS
-.TP
-.B *
-C/C++ only:
-.RS
-.PP
-.RS
-.RE
-.TP
-.B *
-\fI_num_pes\fP(3)
-.TP
-.B *
-\fI_my_pe\fP(3)
-.RE
-.RS
-.PP
-.RE
-.TP
-.B *
-Fortran only:
-.RS
-.PP
-.RS
-.RE
-.TP
-.B *
-\fINUM_PES\fP(3)
-.TP
-.B *
-\fIMY_PE\fP(3)
-.RE
-.RS
-.PP
-.RE
-.RE
-.PP
-.RE
-.TP
-Elemental data put routines
-.PP
-.RS
-.TP
-.B *
-C/C++ only:
-.RS
-.PP
-.RS
-.RE
-.TP
-.B *
-\fIshmem_double_p\fP(3)
-.TP
-.B *
-\fIshmem_float_p\fP(3)
-.TP
-.B *
-\fIshmem_int_p\fP(3)
-.TP
-.B *
-\fIshmem_long_p\fP(3)
-.TP
-.B *
-\fIshmem_short_p.\fP(3)
-.RE
-.RS
-.PP
-.RE
-.RE
-.PP
-.RE
-.TP
-Block data put routines
-.PP
-.RS
-.TP
-.B *
-C/C++ and Fortran:
-.RS
-.PP
-.RS
-.RE
-.TP
-.B *
-\fIshmem_put32\fP(3)
-.TP
-.B *
-\fIshmem_put64\fP(3)
-.TP
-.B *
-\fIshmem_put128\fP(3)
-.RE
-.RS
-.PP
-.RE
-.TP
-.B *
-C/C++ only:
-.RS
-.PP
-.RS
-.RE
-.TP
-.B *
-\fIshmem_double_put\fP(3)
-.TP
-.B *
-\fIshmem_float_put\fP(3)
-.TP
-.B *
-\fIshmem_int_put\fP(3)
-.TP
-.B *
-\fIshmem_long_put\fP(3)
-.TP
-.B *
-\fIshmem_short_put.\fP(3)
-.RE
-.RS
-.PP
-.RE
-.TP
-.B *
-Fortran only:
-.RS
-.PP
-.RS
-.RE
-.TP
-.B *
-\fIshmem_complex_put\fP(3)
-.TP
-.B *
-\fIshmem_integer_put\fP(3)
-.TP
-.B *
-\fIshmem_logical_put\fP(3)
-.TP
-.B *
-\fIshmem_real_put\fP(3)
-.RE
-.RS
-.PP
-.RE
-.RE
-.PP
-.RE
-.TP
-Elemental data get routines
-.PP
-.RS
-.TP
-.B *
-C/C++ only:
-.RS
-.PP
-.RS
-.RE
-.TP
-.B *
-\fIshmem_double_g\fP(3)
-.TP
-.B *
-\fIshmem_float_g\fP(3)
-.TP
-.B *
-\fIshmem_int_g\fP(3)
-.TP
-.B *
-\fIshmem_long_g\fP(3)
-.TP
-.B *
-\fIshmem_short_g\fP(3)
-.RE
-.RS
-.PP
-.RE
-.RE
-.PP
-.RE
-.TP
-Block data get routines
-.RS
-.TP
-.B *
-C/C++ and Fortran:
-.RS
-.PP
-.RS
-.RE
-.TP
-.B *
-\fIshmem_get32\fP(3)
-.TP
-.B *
-\fIshmem_get64\fP(3)
-.TP
-.B *
-\fIshmem_get128\fP(3)
-.RE
-.RS
-.PP
-.RE
-.TP
-.B *
-C/C++ only:
-.RS
-.PP
-.RS
-.RE
-.TP
-.B *
-\fIshmem_double_get\fP(3)
-.TP
-.B *
-\fIshmem_float_get\fP(3)
-.TP
-.B *
-\fIshmem_int_get\fP(3)
-.TP
-.B *
-\fIshmem_long_get\fP(3)
-.TP
-.B *
-\fIshmem_short_get\fP(3)
-.RE
-.RS
-.PP
-.RE
-.TP
-.B *
-Fortran only:
-.RS
-.PP
-.RS
-.RE
-.TP
-.B *
-\fIshmem_complex_get\fP(3)
-.TP
-.B *
-\fIshmem_integer_get\fP(3)
-.TP
-.B *
-\fIshmem_logical_get\fP(3)
-.TP
-.B *
-\fIshmem_real_get\fP(3)
-.RE
-.RS
-.PP
-.RE
-.RE
-.PP
-.RE
-.TP
-Strided put routines
-.RS
-.TP
-.B *
-C/C++ and Fortran:
-.RS
-.PP
-.RS
-.RE
-.TP
-.B *
-\fIshmem_iput32\fP(3)
-.TP
-.B *
-\fIshmem_iput64\fP(3)
-.TP
-.B *
-\fIshmem_iput128\fP(3)
-.RE
-.RS
-.PP
-.RE
-.TP
-.B *
-C/C++ only:
-.RS
-.PP
-.RS
-.RE
-.TP
-.B *
-\fIshmem_double_iput\fP(3)
-.TP
-.B *
-\fIshmem_float_iput\fP(3)
-.TP
-.B *
-\fIshmem_int_iput\fP(3)
-.TP
-.B *
-\fIshmem_long_iput\fP(3)
-.TP
-.B *
-\fIshmem_short_iput\fP(3)
-.RE
-.RS
-.PP
-.RE
-.TP
-.B *
-Fortran only:
-.RS
-.PP
-.RS
-.RE
-.TP
-.B *
-\fIshmem_complex_iput\fP(3)
-.TP
-.B *
-\fIshmem_integer_iput\fP(3)
-.TP
-.B *
-\fIshmem_logical_iput\fP(3)
-.TP
-.B *
-\fIshmem_real_iput\fP(3)
-.RE
-.RS
-.PP
-.RE
-.RE
-.PP
-.RE
-.TP
-Strided get routines
-.PP
-.RS
-.TP
-.B *
-C/C++ and Fortran:
-.RS
-.PP
-.RS
-.RE
-.TP
-.B *
-\fIshmem_iget32\fP(3)
-.TP
-.B *
-\fIshmem_iget64\fP(3)
-.TP
-.B *
-\fIshmem_iget128\fP(3)
-.RE
-.RS
-.PP
-.RE
-.TP
-.B *
-C/C++ only:
-.RS
-.PP
-.RS
-.RE
-.TP
-.B *
-\fIshmem_double_iget\fP(3)
-.TP
-.B *
-\fIshmem_float_iget\fP(3)
-.TP
-.B *
-\fIshmem_int_iget\fP(3)
-.TP
-.B *
-\fIshmem_long_iget\fP(3)
-.TP
-.B *
-\fIshmem_short_iget\fP(3)
-.RE
-.RS
-.PP
-.RE
-.TP
-.B *
-Fortran only:
-.RS
-.PP
-.RS
-.RE
-.TP
-.B *
-\fIshmem_complex_iget\fP(3)
-.TP
-.B *
-\fIshmem_integer_iget\fP(3)
-.TP
-.B *
-\fIshmem_logical_iget\fP(3)
-.TP
-.B *
-\fIshmem_real_iget\fP(3)
-.RE
-.RS
-.PP
-.RE
-.RE
-.PP
-.RE
-.TP
-Point\-to\-point synchronization routines
-.RS
-.TP
-.B *
-C/C++ only:
-.RS
-.PP
-.RS
-.RE
-.TP
-.B *
-\fIshmem_int_wait\fP(3)
-.TP
-.B *
-\fIshmem_int_wait_until\fP(3)
-.TP
-.B *
-\fIshmem_long_wait\fP(3)
-.TP
-.B *
-\fIshmem_long_wait_until\fP(3)
-.TP
-.B *
-\fIshmem_longlong_wait\fP(3)
-.TP
-.B *
-\fIshmem_longlong_wait_until\fP(3)
-.TP
-.B *
-\fIshmem_short_wait\fP(3)
-.TP
-.B *
-\fIshmem_short_wait_until\fP(3)
-.RE
-.RS
-.PP
-.RE
-.TP
-.B *
-Fortran:
-.RS
-.PP
-.RS
-.RE
-.TP
-.B *
-\fIshmem_int4_wait\fP(3)
-.TP
-.B *
-\fIshmem_int4_wait_until\fP(3)
-.TP
-.B *
-\fIshmem_int8_wait\fP(3)
-.TP
-.B *
-\fIshmem_int8_wait_until\fP(3)
-.RE
-.RS
-.PP
-.RE
-.RE
-.PP
-.RE
-.TP
-Barrier synchronization routines
-.PP
-.RS
-.TP
-.B *
-C/C++ and Fortran:
-.RS
-.PP
-.RS
-.RE
-.TP
-.B *
-\fIshmem_barrier_all\fP(3)
-.TP
-.B *
-\fIshmem_barrier\fP(3)
-.RE
-.RS
-.PP
-.RE
-.RE
-.PP
-.RE
-.TP
-Atomic memory fetch\-and\-operate (fetch\-op) routines
-.RS
-.TP
-.B *
-C/C++ and Fortran:
-.RS
-.TP
-.B *
-shmem_swap
-.RE
-.RS
-.PP
-.RE
-.RE
-.PP
-.RE
-.TP
-Reduction routines
-.RS
-.TP
-.B *
-C/C++ only:
-.RS
-.TP
-.B *
-\fIshmem_int_and_to_all\fP(3)
-.TP
-.B *
-\fIshmem_long_and_to_all\fP(3)
-.TP
-.B *
-\fIshmem_longlong_and_to_all\fP(3)
-.TP
-.B *
-\fIshmem_short_and_to_all\fP(3)
-.TP
-.B *
-\fIshmem_double_max_to_all\fP(3)
-.TP
-.B *
-\fIshmem_float_max_to_all\fP(3)
-.TP
-.B *
-\fIshmem_int_max_to_all\fP(3)
-.TP
-.B *
-\fIshmem_long_max_to_all\fP(3)
-.TP
-.B *
-\fIshmem_longlong_max_to_all\fP(3)
-.TP
-.B *
-\fIshmem_short_max_to_all\fP(3)
-.TP
-.B *
-\fIshmem_double_min_to_all\fP(3)
-.TP
-.B *
-\fIshmem_float_min_to_all\fP(3)
-.TP
-.B *
-\fIshmem_int_min_to_all\fP(3)
-.TP
-.B *
-\fIshmem_long_min_to_all\fP(3)
-.TP
-.B *
-\fIshmem_longlong_min_to_all\fP(3)
-.TP
-.B *
-\fIshmem_short_min_to_all\fP(3)
-.TP
-.B *
-\fIshmem_double_sum_to_all\fP(3)
-.TP
-.B *
-\fIshmem_float_sum_to_all\fP(3)
-.TP
-.B *
-\fIshmem_int_sum_to_all\fP(3)
-.TP
-.B *
-\fIshmem_long_sum_to_all\fP(3)
-.TP
-.B *
-\fIshmem_longlong_sum_to_all\fP(3)
-.TP
-.B *
-\fIshmem_short_sum_to_all\fP(3)
-.TP
-.B *
-\fIshmem_double_prod_to_all\fP(3)
-.TP
-.B *
-\fIshmem_float_prod_to_all\fP(3)
-.TP
-.B *
-\fIshmem_int_prod_to_all\fP(3)
-.TP
-.B *
-\fIshmem_long_prod_to_all\fP(3)
-.TP
-.B *
-\fIshmem_longlong_prod_to_all\fP(3)
-.TP
-.B *
-\fIshmem_short_prod_to_all\fP(3)
-.TP
-.B *
-\fIshmem_int_or_to_all\fP(3)
-.TP
-.B *
-\fIshmem_long_or_to_all\fP(3)
-.TP
-.B *
-\fIshmem_longlong_or_to_all\fP(3)
-.TP
-.B *
-\fIshmem_short_or_to_all\fP(3)
-.TP
-.B *
-\fIshmem_int_xor_to_all\fP(3)
-.TP
-.B *
-\fIshmem_long_xor_to_all\fP(3)
-.TP
-.B *
-\fIshmem_longlong_xor_to_all\fP(3)
-.TP
-.B *
-\fIshmem_short_xor_to_all\fP(3)
-.RE
-.RS
-.PP
-.RE
-.TP
-.B *
-Fortran only:
-.RS
-.TP
-.B *
-\fIshmem_int4_and_to_all\fP(3)
-.TP
-.B *
-\fIshmem_int8_and_to_all\fP(3)
-.TP
-.B *
-\fIshmem_real4_max_to_all\fP(3)
-.TP
-.B *
-\fIshmem_real8_max_to_all\fP(3)
-.TP
-.B *
-\fIshmem_int4_max_to_all\fP(3)
-.TP
-.B *
-\fIshmem_int8_max_to_all\fP(3)
-.TP
-.B *
-\fIshmem_real4_min_to_all\fP(3)
-.TP
-.B *
-\fIshmem_real8_min_to_all\fP(3)
-.TP
-.B *
-\fIshmem_int4_min_to_all\fP(3)
-.TP
-.B *
-\fIshmem_int8_min_to_all\fP(3)
-.TP
-.B *
-\fIshmem_real4_sum_to_all\fP(3)
-.TP
-.B *
-\fIshmem_real8_sum_to_all\fP(3)
-.TP
-.B *
-\fIshmem_int4_sum_to_all\fP(3)
-.TP
-.B *
-\fIshmem_int8_sum_to_all\fP(3)
-.TP
-.B *
-\fIshmem_real4_prod_to_all\fP(3)
-.TP
-.B *
-\fIshmem_real8_prod_to_all\fP(3)
-.TP
-.B *
-\fIshmem_int4_prod_to_all\fP(3)
-.TP
-.B *
-\fIshmem_int8_prod_to_all\fP(3)
-.TP
-.B *
-\fIshmem_int4_or_to_all\fP(3)
-.TP
-.B *
-\fIshmem_int8_or_to_all\fP(3)
-.TP
-.B *
-\fIshmem_int4_xor_to_all\fP(3)
-.TP
-.B *
-\fIshmem_int8_xor_to_all\fP(3)
-.RE
-.RS
-.PP
-.RE
-.RE
-.PP
-.RE
-.TP
-Broadcast routines
-.PP
-.RS
-.TP
-.B *
-C/C++ and Fortran:
-.RS
-.PP
-.RS
-.RE
-.TP
-.B *
-\fIshmem_broadcast32\fP(3)
-.TP
-.B *
-\fIshmem_broadcast64\fP(3)
-.RE
-.RS
-.PP
-.RE
-.RE
-.PP
-.RE
-.TP
-Cache management routines
-.PP
-.RS
-.TP
-.B *
-C/C++ and Fortran:
-.RS
-.PP
-.RS
-.RE
-.TP
-.B *
-\fIshmem_udcflush\fP(3)
-.TP
-.B *
-\fIshmem_udcflush_line\fP(3)
-.RE
-.RS
-.PP
-.RE
-.RE
-.PP
-.RE
-.TP
-Byte\-granularity block put routines
-.PP
-.RS
-.TP
-.B *
-C/C++ and Fortran
-.RS
-.PP
-.RS
-.RE
-.TP
-.B *
-\fIshmem_putmem\fP(3)
-.TP
-.B *
-\fIshmem_getmem\fP(3)
-.RE
-.RS
-.PP
-.RE
-.TP
-.B *
-Fortran only:
-.RS
-.PP
-.RS
-.RE
-.TP
-.B *
-\fIshmem_character_put\fP(3)
-.TP
-.B *
-\fIshmem_character_get\fP(3)
-.RE
-.RS
-.PP
-.RE
-.RE
-.PP
-.RE
-.TP
-Collect routines
-.RS
-.TP
-.B *
-C/C++ and Fortran:
-.RS
-.PP
-.RS
-.RE
-.TP
-.B *
-\fIshmem_collect32\fP(3)
-.TP
-.B *
-\fIshmem_collect64\fP(3)
-.TP
-.B *
-\fIshmem_fcollect32\fP(3)
-.TP
-.B *
-\fIshmem_fcollect64\fP(3)
-.RE
-.RS
-.PP
-.RE
-.RE
-.PP
-.RE
-.TP
-Atomic memory fetch\-and\-operate (fetch\-op) routines
-.RS
-.TP
-.B *
-C/C++ only:
-.RS
-.TP
-.B *
-\fIshmem_double_swap\fP(3)
-.TP
-.B *
-\fIshmem_float_swap\fP(3)
-.TP
-.B *
-\fIshmem_int_cswap\fP(3)
-.TP
-.B *
-\fIshmem_int_fadd\fP(3)
-.TP
-.B *
-\fIshmem_int_finc\fP(3)
-.TP
-.B *
-\fIshmem_int_swap\fP(3)
-.TP
-.B *
-\fIshmem_long_cswap\fP(3)
-.TP
-.B *
-\fIshmem_long_fadd\fP(3)
-.TP
-.B *
-\fIshmem_long_finc\fP(3)
-.TP
-.B *
-\fIshmem_long_swap\fP(3)
-.TP
-.B *
-\fIshmem_longlong_cswap\fP(3)
-.TP
-.B *
-\fIshmem_longlong_fadd\fP(3)
-.TP
-.B *
-\fIshmem_longlong_finc\fP(3)
-.TP
-.B *
-\fIshmem_longlong_swap\fP(3)
-.RE
-.RS
-.PP
-.RE
-.TP
-.B *
-Fortran only:
-.RS
-.TP
-.B *
-\fIshmem_int4_cswap\fP(3)
-.TP
-.B *
-\fIshmem_int4_fadd\fP(3)
-.TP
-.B *
-\fIshmem_int4_finc\fP(3)
-.TP
-.B *
-\fIshmem_int4_swap\fP(3)
-.TP
-.B *
-\fIshmem_int8_swap\fP(3)
-.TP
-.B *
-\fIshmem_real4_swap\fP(3)
-.TP
-.B *
-\fIshmem_real8_swap\fP(3)
-.TP
-.B *
-\fIshmem_int8_cswap\fP(3)
-.RE
-.RS
-.PP
-.RE
-.RE
-.PP
-.RE
-.TP
-Atomic memory operation routines
-.RS
-.TP
-.B *
-Fortran only:
-.RS
-.PP
-.RS
-.RE
-.TP
-.B *
-\fIshmem_int4_add\fP(3)
-.TP
-.B *
-\fIshmem_int4_inc\fP(3)
-.RE
-.RS
-.PP
-.RE
-.RE
-.PP
-.RE
-.TP
-Remote memory pointer function
-.RS
-.TP
-.B *
-C/C++ and Fortran:
-.RS
-.PP
-.RS
-.RE
-.TP
-.B *
-\fIshmem_ptr\fP(3)
-.RE
-.RS
-.PP
-.RE
-.RE
-.PP
-.RE
-.TP
-Reduction routines
-.RS
-.TP
-.B *
-C/C++ only:
-.RS
-.TP
-.B *
-\fIshmem_longdouble_max_to_all\fP(3)
-.TP
-.B *
-\fIshmem_longdouble_min_to_all\fP(3)
-.TP
-.B *
-\fIshmem_longdouble_prod_to_all\fP(3)
-.TP
-.B *
-\fIshmem_longdouble_sum_to_all\fP(3)
-.RE
-.RS
-.PP
-.RE
-.TP
-.B *
-Fortran only:
-.RS
-.PP
-.RS
-.RE
-.TP
-.B *
-\fIshmem_real16_max_to_all\fP(3)
-.TP
-.B *
-\fIshmem_real16_min_to_all\fP(3)
-.TP
-.B *
-\fIshmem_real16_prod_to_all\fP(3)
-.TP
-.B *
-\fIshmem_real16_sum_to_all\fP(3)
-.RE
-.RS
-.PP
-.RE
-.RE
-.PP
-.RE
-.TP
-Accessibility query routines
-.RS
-.TP
-.B *
-C/C++ and Fortran:
-.RS
-.TP
-.B *
-\fIshmem_pe_accessible\fP(3)
-.TP
-.B *
-\fIshmem_addr_accessible\fP(3)
-.RE
-.RS
-.PP
-.RE
-.RE
-.PP
-.RE
-.TP
-Symmetric Data Objects
-.PP
-Consistent with the SPMD nature of the OpenSHMEM programming model is the
-concept of symmetric data objects. These are arrays or variables that
-exist with the same size, type, and relative address on all PEs.
-Another term for symmetric data objects is "remotely accessible data objects".
-In the interface definitions for OpenSHMEM data transfer routines, one or more of the
-parameters are typically required to be symmetric or remotely accessible.
-.PP
-The following kinds of data objects are symmetric:
-.RS
-.TP
-.B *
-Fortran data objects in common blocks or with the SAVE attribute. These data
-objects must not be defined in a dynamic shared object (DSO).
-.TP
-.B *
-Non\-stack C and C++ variables. These data objects must not be defined in a DSO.
-.TP
-.B *
-Fortran arrays allocated with \fIshpalloc\fP(3F)
-.TP
-.B *
-C and C++ data allocated by \fIshmalloc\fP(3C)
-.RE
-.RS
-.PP
-.RE
-.TP
-Collective Routines
-Some SHMEM routines, for example, \fIshmem_broadcast\fP(3)
-and
-\fIshmem_float_sum_to_all\fP(3),
-are classified as collective routines
-because they distribute work across a set of PEs.
-They must be called concurrently by all PEs in the active set defined by the PE_start,
-logPE_stride, PE_size argument triplet. The following man pages describe the OpenSHMEM
-collective routines:
-.RS
-.TP
-.B *
-\fIshmem_and\fP(3)
-.TP
-.B *
-\fIshmem_barrier\fP(3)
-.TP
-.B *
-\fIshmem_broadcast\fP(3)
-.TP
-.B *
-\fIshmem_collect\fP(3)
-.TP
-.B *
-\fIshmem_max\fP(3)
-.TP
-.B *
-\fIshmem_min\fP(3)
-.TP
-.B *
-\fIshmem_or\fP(3)
-.TP
-.B *
-\fIshmem_prod\fP(3)
-.TP
-.B *
-\fIshmem_sum\fP(3)
-.TP
-.B *
-\fIshmem_xor\fP(3)
-.RE
-.RS
-.PP
-.RE
-.PP
-.SH USING THE SYMMETRIC WORK ARRAY, PSYNC
-
-Multiple pSync arrays are often needed if a particular PE calls as OpenSHMEM collective
-routine twice without intervening barrier synchronization. Problems would occur if some PEs
-in the active set for call 2 arrive at call 2 before processing of call 1 is complete by all PEs in
-the call 1 active set. You can use \fIshmem_barrier\fP(3)
-or \fIshmem_barrier_all\fP(3)
-to perform a barrier synchronization between consecutive calls to OpenSHMEM collective
-routines.
-.PP
-There are two special cases:
-.RE
-.TP
-.B *
-The \fIshmem_barrier\fP(3) routine allows the same pSync array to be used on
-consecutive calls as long as the active PE set does not change.
-.TP
-.B *
-If the same collective routine is called multiple times with the same active set, the
-calls may alternate between two pSync arrays. The SHMEM routines guarantee that a
-first call is completely finished by all PEs by the time processing of a third call begins
-on any PE.
-.PP
-Because the SHMEM routines restore pSync to its original contents, multiple calls that
-use the same pSync array do not require that pSync be reinitialized after the first call.
-.PP
-.SH SHMEM ENVIRONMENT VARIABLES
-
-This section lists the significant SHMEM environment variables.
-.TP
-.B *
-\fBSMA_VERSION\fP print the library version at start\-up.
-.TP
-.B *
-\fBSMA_INFO\fP print helpful text about all these environment variables.
-.TP
-.B *
-\fBSMA_SYMMETRIC_SIZE\fP number of bytes to allocate for the symmetric heap.
-.TP
-.B *
-\fBSMA_DEBUG\fP enable debugging messages.
-.PP
-The first call to SHMEM must be \fIstart_pes\fP(3)\&.
-This routines initialize the SHMEM runtime.
-.PP
-Calling any other SHMEM routines beforehand has undefined behavior. Multiple calls
-to this routine is not allowed.
-.PP
-.SH COMPILING AND RUNNING OPENSHMEM PROGRAMS
-
-The OpenSHMEM specification is silent regarding how OpenSHMEM programs are compiled,
-linked and run. This section shows some examples of how wrapper programs could be utilized
-to compile and launch applications. The commands are styled after wrapper programs
-found in many MPI implementations.
-.PP
-The following sample command line demonstrates running an OpenSHMEM Program using a wrapper script (\fBoshrun\fP
-in this case):
-.PP
-.TP
-.B *
-C/C++:
-.Vb
-oshcc c_program.c
-.Ve
-.TP
-.B *
-FORTRAN:
-.Vb
-oshfort fortran_program.f
-.Ve
-.PP
-The following sample command line demonstrates running an OpenSHMEM Program assuming that the library provides a wrapper script for such purpose
-(named \fBoshrun\fP
-for this example):
-.PP
-.Vb
-oshrun \-np 32 ./a.out
-.Ve
-.PP
-.SH EXAMPLES
-
-\fBExample 1\fP:
-The following Fortran OpenSHMEM program directs all PEs to sum
-simultaneously the numbers in the VALUES variable across all PEs:
-.Vb
-PROGRAM REDUCTION
-  REAL VALUES, SUM
-  COMMON /C/ VALUES
-  REAL WORK
-
-  CALL START_PES(0)
-  VALUES = MY_PE()
-  CALL SHMEM_BARRIER_ALL ! Synchronize all PEs
-  SUM = 0.0
-  DO I = 0, NUM_PES()\-1
-    CALL SHMEM_REAL_GET(WORK, VALUES, 1, I) ! Get next value
-    SUM = SUM + WORK                ! Sum it
-  ENDDO
-  PRINT *, 'PE ', MY_PE(), ' COMPUTED SUM=', SUM
-  CALL SHMEM_BARRIER_ALL
-END
-.Ve
-\fBExample 2\fP:
-The following C OpenSHMEM program transfers an array of 10 longs from
-PE 0 to PE 1:
-.Vb
-#include <mpp/shmem.h>
-
-main() {
-  long source[10] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
-  static long target[10];
-
-  shmem_init();
-  if (shmem_my_pe() == 0) {
-    /* put 10 elements into target on PE 1 */
-    shmem_long_put(target, source, 10, 1);
-  }
-  shmem_barrier_all(); /* sync sender and receiver */
-  if (shmem_my_pe() == 1)
-    printf("target[0] on PE %d is %d\\n", shmem_my_pe(), target[0]);
-}
-.Ve
-.SH SEE ALSO
-
-The following man pages also contain information on OpenSHMEM routines. See the
-specific man pages for implementation information.
-.PP
-\fIshmem_add\fP(3),
-\fIshmem_and\fP(3),
-\fIshmem_barrier\fP(3),
-\fIshmem_barrier_all\fP(3),
-\fIshmem_broadcast\fP(3),
-\fIshmem_cache\fP(3),
-\fIshmem_collect\fP(3),
-\fIshmem_cswap\fP(3),
-\fIshmem_fadd\fP(3),
-\fIshmem_fence\fP(3),
-\fIshmem_finc\fP(3),
-\fIshmem_get\fP(3),
-\fIshmem_iget\fP(3),
-\fIshmem_inc\fP(3),
-\fIshmem_iput\fP(3),
-\fIshmem_lock\fP(3),
-\fIshmem_max\fP(3),
-\fIshmem_min\fP(3),
-\fIshmem_my_pe\fP(3),
-\fIshmem_or\fP(3),
-\fIshmem_prod\fP(3),
-\fIshmem_put\fP(3),
-\fIshmem_quiet\fP(3),
-\fIshmem_short_g\fP(3),
-\fIshmem_short_p\fP(3),
-\fIshmem_sum\fP(3),
-\fIshmem_swap\fP(3),
-\fIshmem_wait\fP(3),
-\fIshmem_xor\fP(3),
-\fIshmem_pe_accessible\fP(3),
-\fIshmem_addr_accessible\fP(3),
-\fIshmem_init\fP(3),
-\fIshmem_malloc\fP(3C),
-\fIshmem_my_pe\fP(3I),
-\fIshmem_n_pes\fP(3I)
diff --git a/oshmem/shmem/man/man3/shfree.3in b/oshmem/shmem/man/man3/shfree.3in
deleted file mode 100644
index 6cdb8014e9e..00000000000
--- a/oshmem/shmem/man/man3/shfree.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_malloc.3
diff --git a/oshmem/shmem/man/man3/shmalloc.3in b/oshmem/shmem/man/man3/shmalloc.3in
deleted file mode 100644
index 5255598fa98..00000000000
--- a/oshmem/shmem/man/man3/shmalloc.3in
+++ /dev/null
@@ -1,105 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMALLOC" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmalloc\fP(3),
-\fIshfree\fP(3),
-\fIshmemalign\fP(3),
-\fIshrealloc\fP(3)
-\- Symmetric heap memory management functions.
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-void *shmalloc(size_t size);
-
-void shfree(void *ptr);
-
-void *shrealloc(void *ptr, size_t size);
-
-void *shmemalign(size_t alignment, size_t size);
-
-extern long malloc_error;
-.Ve
-.SH DESCRIPTION
-
-The \fBshmalloc\fP
-function returns a pointer to a block of at least size bytes
-suitably aligned for any use. This space is allocated from the symmetric heap (in contrast
-to \fImalloc\fP(3C),
-which allocates from the private heap).
-.PP
-The \fBshmemalign\fP
-function allocates a block in the symmetric heap that has a
-byte alignment specified by the alignment argument.
-.PP
-The \fBshfree\fP
-function causes the block to which ptr points to, to be deallocated,
-that is, made available for further allocation. If ptr is a null pointer, no action
-occurs; otherwise, if the argument does not match a pointer earlier returned by a symmetric
-heap function, or if the space has already been deallocated, malloc_error is set to indicate the
-error, and shfree returns.
-.PP
-The \fBshrealloc\fP
-function changes the size of the block to which ptr points to, to the
-size (in bytes) specified by size.
-.PP
-The contents of the block are unchanged up to the lesser of the new and old sizes. If the new
-size is larger, the value of the newly allocated portion of the block is indeterminate. If ptr is a
-null pointer, the shrealloc function behaves like the shmalloc function for the specified size. If
-size is 0 and ptr is not a null pointer, the block to which it points to is freed. Otherwise, if ptr
-does not match a pointer earlier returned by a symmetric heap function, or if the space has
-already been deallocated, the malloc_error variable is set to indicate the error, and shrealloc
-returns a null pointer. If the space cannot be allocated, the block to which ptr points to is
-unchanged.
-.PP
-The shmalloc, shfree, and shrealloc functions are provided so that multiple PEs in an
-application can allocate symmetric, remotely accessible memory blocks. These memory
-blocks can then be used with (shmem) communication routines. Each of these functions call
-the \fIshmem_barrier_all\fP(3)
-function before returning; this ensures that all PEs
-participate in the memory allocation, and that the memory on other PEs can be used as soon
-as the local PE returns.
-.PP
-The user is responsible for calling these functions with identical argument(s) on all PEs; if
-differing size arguments are used, subsequent calls may not return the same symmetric heap
-address on all PEs.
-.PP
-.SH NOTES
-
-The total size of the symmetric heap is determined at job startup. One can adjust the size of
-the heap using the SHMEM_SYMMETRIC_HEAP_SIZE environment variable. See the
-\fIintro_shmem\fP(3)
-man page for futher details.
-The shmalloc, shfree, and shrealloc functions differ from the private heap allocation functions
-in that all PEs in an application must call them (a barrier is used to ensure this).
-.PP
-.SH RETURN VALUES
-
-The \fBshmalloc\fP
-function returns a pointer to the allocated space (which should
-be identical on all PEs); otherwise, it returns a null pointer (with malloc_error set).
-The \fBshfree\fP
-function returns no value.
-The \fBshrealloc\fP
-function returns a pointer to the allocated space (which
-may have moved); otherwise, it returns a null pointer (with malloc_error set).
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3),
-\fImy_pe\fP(3I),
-\fIstart_pes\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_addr_accessible.3in b/oshmem/shmem/man/man3/shmem_addr_accessible.3in
deleted file mode 100644
index d04b6d5cea1..00000000000
--- a/oshmem/shmem/man/man3/shmem_addr_accessible.3in
+++ /dev/null
@@ -1,56 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_ADDR\\_ACCESSIBLE" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-shmem_addr_accessible \- Indicates if an address is accessible via OpenSHMEM operations
-from the specified remote PE.
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-int shmem_addr_accessible(const void *addr, int pe);
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-LOGICAL LOG, SHMEM_ADDR_ACCESSIBLE
-INTEGER pe
-
-LOG = SHMEM_ADDR_ACCESSIBLE(addr, pe)
-.Ve
-.SH DESCRIPTION
-
-shmem_addr_accessible is a query function that indicates whether a local address is
-accessible via SHMEM operations from the specified remote PE.
-.PP
-This function verifies that the remote PE is accessible via SHMEM data transfer functions from
-the local PE, and that the specified address is in a symmetric data segment with respect to the
-remote PE.
-.PP
-.SH RETURN VALUES
-
-C: The return value is 1 if addr is a symmetric data object and accessible via SHMEM
-operations from the specified remote PE; otherwise, it is 0.
-.PP
-Fortran: The return value is \&.TRUE. if addr is a symmetric data object and accessible via
-SHMEM operations from the specified remote PE; otherwise, it is \&.FALSE..
-.PP
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3),
-\fIshmem_pe_accessible\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_align.3in b/oshmem/shmem/man/man3/shmem_align.3in
deleted file mode 100644
index 6cdb8014e9e..00000000000
--- a/oshmem/shmem/man/man3/shmem_align.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_malloc.3
diff --git a/oshmem/shmem/man/man3/shmem_alltoall32.3in b/oshmem/shmem/man/man3/shmem_alltoall32.3in
deleted file mode 100644
index c2f08d65847..00000000000
--- a/oshmem/shmem/man/man3/shmem_alltoall32.3in
+++ /dev/null
@@ -1,226 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2016      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_ALLTOALL" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmem_alltoall32\fP(3),
-\fIshmem_alltoall64\fP(3),
-\fIshmem_alltoalls32\fP(3),
-\fIshmem_alltoalls64\fP(3)
-\- collective routine where each PE exchanges a fixed amount of data with all
-other PEs in the Active set
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-void shmem_alltoall32(void *target, const void *source,
-  size_t nelems, int PE_start, int logPE_stride,
-  int PE_size, long *pSync);
-
-void shmem_alltoall64(void *target, const void *source,
-  size_t nelems, int PE_start, int logPE_stride,
-  int PE_size, long *pSync);
-
-void shmem_alltoalls32(void *target, const void *source,
-  ptrdiff_t dst, ptrdiff_t sst,
-  size_t nelems, int PE_start, int logPE_stride,
-  int PE_size, long *pSync);
-
-void shmem_alltoalls64(void *target, const void *source,
-  ptrdiff_t dst, ptrdiff_t sst,
-  size_t nelems, int PE_start, int logPE_stride,
-  int PE_size, long *pSync);
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER dst, sst, nelems, PE_root, PE_start, logPE_stride, PE_size
-INTEGER pSync(SHMEM_ALLTOALL_SYNC_SIZE)
-
-CALL SHMEM_ALLTOALL32(target, source, nelems,
-& PE_start, logPE_stride, PE_size, fIpSync)
-
-CALL SHMEM_ALLTOALL64(target, source, nelems,
-& PE_start, logPE_stride, PE_size, pSync)
-
-CALL SHMEM_ALLTOALLS32(target, source, dst, sst, nelems,
-& PE_start, logPE_stride, PE_size, pSync)
-
-CALL SHMEM_ALLTOALLS64(target, source, dst, sst, nelems,
-& PE_start, logPE_stride, PE_size, pSync)
-.Ve
-.SH DESCRIPTION
-
-.PP
-The shmem_alltoalls routines are collective routines. Each PE in the Active set exchanges nelems strided
-data elements of size 32 bits (for shmem_alltoalls32) or 64 bits (for shmem_alltoalls64) with all other PEs
-in the set. Both strides, dst and sst, must be greater than or equal to 1. The sst*jth block sent from PE i to
-PE j is placed in the dst*ith block of the dest data object on PE j.
-As with all OpenSHMEM collective routines, these routines assume that only PEs in the Active set call the
-routine. If a PE not in the Active set calls an OpenSHMEM collective routine, undefined behavior results.
-The values of arguments dst, sst, nelems, PE_start, logPE_stride, and PE_size must be equal on all PEs in
-the Active set. The same dest and source data objects, and the same pSync work array must be passed to all
-PEs in the Active set.
-Before any PE calls to a shmem_alltoalls routine, the following conditions must exist (synchronization via
-a barrier or some other method is often needed to ensure this): The pSync array on all PEs in the Active set
-is not still in use from a prior call to a shmem_alltoalls routine. The dest data object on all PEs in the
-Active set is ready to accept the shmem_alltoalls data.
-Upon return from a shmem_alltoalls routine, the following is true for the local PE: Its dest symmetric data
-object is completely updated and the data has been copied out of the source data object. The values in the
-pSync array are restored to the original values.
-.PP
-The arguments are as follows:
-.TP
-A symmetric data object with one of the following data types:
-.RS
-.TP
-\fBshmem_alltoall32\fP: Any noncharacter type that
-has an element size of 32 bits. No Fortran derived types or C/C++ structures are allowed.
-.TP
-\fBshmem_alltoall64\fP: Any noncharacter type that has an element size
-of 64 bits. No Fortran derived types or C/C++ structures are allowed.
-.RE
-.RS
-.PP
-.RE
-target
-A symmetric data object large enough to receive the combined total of
-nelems elements from each PE in the Active set.
-.TP
-source
-A symmetric data object that contains nelems elements of data for each
-PE in the Active set, ordered according to destination PE.
-.TP
-dst
-The stride between consecutive elements of the dest data object. The
-stride is scaled by the element size. A value of 1 indicates contiguous
-data. dst must be of type ptrdiff_t. If you are using Fortran, it must be
-a default integer value.
-.TP
-sst
-The stride between consecutive elements of the source data object. The
-stride is scaled by the element size. A value of 1 indicates contiguous
-data. sst must be of type ptrdiff_t. If you are using Fortran, it must be a
-default integer value.
-.TP
-nelems
-The number of elements to exchange for each PE. nelems must be of
-type size_t for C/C++. If you are using Fortran, it must be a default
-integer value
-.TP
-PE_start
-The lowest virtual PE number of the active set of PEs. PE_start must be of
-type integer. If you are using Fortran, it must be a default integer value.
-.TP
-logPE_stride
-The log (base 2) of the stride between consecutive virtual PE numbers in
-the active set. log_PE_stride must be of type integer. If you are using Fortran, it must be a
-default integer value.
-.TP
-PE_size
-The number of PEs in the active set. PE_size must be of type integer. If you
-are using Fortran, it must be a default integer value.
-.PP
-.TP
-pSync
-A symmetric work array. In C/C++, pSync must be of type long and size
-_SHMEM_ALLTOALL_SYNC_SIZE.
-In Fortran, pSync must be of type integer and size SHMEM_ALLTOALL_SYNC_SIZE. Every
-element of this array must be initialized with the value _SHMEM_SYNC_VALUE (in C/C++)
-or SHMEM_SYNC_VALUE (in Fortran) before any of the PEs in the active set enter
-shmem_barrier().
-.PP
-Upon return from a shmem_alltoalls routine, the following is true for the local PE: Its dest
-symmetric data object is completely updated and the data has been copied out of the source 
-data object. The values in the pSync array are restored to the original values.
-.PP
-The values of arguments PE_root, PE_start, logPE_stride, and PE_size must be equal on
-all PEs in the active set. The same target and source data objects and the same pSync work
-array must be passed to all PEs in the active set.
-.PP
-Before any PE calls a alltoall routine, you must ensure that the following conditions exist
-(synchronization via a barrier or some other method is often needed to ensure this): The
-pSync array on all PEs in the active set is not still in use from a prior call to a alltoall
-routine. The target array on all PEs in the active set is ready to accept the alltoall data.
-.SH NOTES
-
-The terms collective and symmetric are defined in \fIintro_shmem\fP(3)\&.
-.PP
-All SHMEM alltoall routines restore pSync to its original contents. Multiple calls to SHMEM
-routines that use the same pSync array do not require that pSync be reinitialized after the
-first call.
-.PP
-You must ensure the that the pSync array is not being updated by any PE in the active set
-while any of the PEs participates in processing of a SHMEM broadcast routine. Be careful to
-avoid these situations: If the pSync array is initialized at run time, some type of
-synchronization is needed to ensure that all PEs in the working set have initialized pSync
-before any of them enter a SHMEM routine called with the pSync synchronization array. A
-pSync array may be reused on a subsequent SHMEM broadcast routine only if none of the PEs
-in the active set are still processing a prior SHMEM alltoall routine call that used the same
-pSync array. In general, this can be ensured only by doing some type of synchronization.
-However, in the special case of SHMEM routines being called with the same active set, you
-can allocate two pSync arrays and alternate between them on successive calls.
-.PP
-.SH EXAMPLES
-
-.PP
-C/C++ example:
-.Vb
-#include <shmem.h>
-#include <stdio.h>
-long pSync[SHMEM_ALLTOALL_SYNC_SIZE];
-int main(void)
-{
-int64_t *source, *dest;
-int i, count, pe;
-shmem_init();
-count = 2;
-dest = (int64_t*) shmem_malloc(count * shmem_n_pes() * sizeof(int64_t));
-source = (int64_t*) shmem_malloc(count * shmem_n_pes() * sizeof(int64_t));
-/* assign source values */
-for (pe=0; pe <shmem_n_pes(); pe++){
-for (i=0; i<count; i++){
-source[(pe*count)+i] = shmem_my_pe() + pe;
-dest[(pe*count)+i] = 9999;
-}
-}
-for (i=0; i< SHMEM_ALLTOALLS_SYNC_SIZE; i++) {
-pSync[i] = SHMEM_SYNC_VALUE;
-}
-/* wait for all PEs to initialize pSync */
-shmem_barrier_all();
-/* alltoalls on all PES */
-shmem_alltoalls64(dest, source, 1, 1, count, 0, 0, shmem_n_pes(), pSync);
-/* verify results */
-for (pe=0; pe<shmem_n_pes(); pe++) {
-for (i=0; i<count; i++){
-if (dest[(pe*count)+i] != shmem_my_pe() + pe) {
-printf("[%d] ERROR: dest[%d]=%ld, should be %d\n",
-shmem_my_pe(),(pe*count)+i,dest[(pe*count)+i],
-shmem_n_pes() + pe);
-}
-}
-}
-shmem_barrier_all();
-shmem_free(dest);
-shmem_free(source);
-shmem_finalize();
-return 0;
-}
-.PP
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_alltoall64.3in b/oshmem/shmem/man/man3/shmem_alltoall64.3in
deleted file mode 100644
index 9874ecc234d..00000000000
--- a/oshmem/shmem/man/man3/shmem_alltoall64.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_alltoall32.3
diff --git a/oshmem/shmem/man/man3/shmem_alltoalls32.3in b/oshmem/shmem/man/man3/shmem_alltoalls32.3in
deleted file mode 100644
index 9874ecc234d..00000000000
--- a/oshmem/shmem/man/man3/shmem_alltoalls32.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_alltoall32.3
diff --git a/oshmem/shmem/man/man3/shmem_alltoalls64.3in b/oshmem/shmem/man/man3/shmem_alltoalls64.3in
deleted file mode 100644
index 9874ecc234d..00000000000
--- a/oshmem/shmem/man/man3/shmem_alltoalls64.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_alltoall32.3
diff --git a/oshmem/shmem/man/man3/shmem_barrier.3in b/oshmem/shmem/man/man3/shmem_barrier.3in
deleted file mode 100644
index e9e9722a04a..00000000000
--- a/oshmem/shmem/man/man3/shmem_barrier.3in
+++ /dev/null
@@ -1,112 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_BARRIER" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-shmem_barrier \- Performs a barrier operation on a subset of processing elements (PEs).
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-void shmem_barrier(int PE_start, int logPE_stride, int PE_size,
-  long *pSync);
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER PE_start, logPE_stride, PE_size
-INTEGER pSync(SHMEM_BARRIER_SYNC_SIZE)
-
-CALL SHMEM_BARRIER(PE_start, logPE_stride, PE_size, pSync)
-.Ve
-.SH DESCRIPTION
-
-The shmem_barrier routine does not return until the subset of PEs specified by
-\fBPE_start\fP,
-\fBlogPE_stride\fP
-and \fBPE_size\fP,
-has entered this routine at the
-same point of the execution path.
-.PP
-As with all SHMEM collective routines, each of these routines assumes that only PEs in the
-active set call the routine. If a PE not in the active set calls a SHMEM collective routine,
-undefined behavior results.
-.PP
-The arguments are as follows:
-.TP
-PE_start
-The lowest virtual PE number of the active set of PEs. PE_start must be of
-type integer. If you are using Fortran, it must be a default integer value.
-.TP
-logPE_stride
-The log (base 2) of the stride between consecutive virtual PE numbers in
-the active set. logPE_stride must be of type integer. If you are using Fortran, it must be a
-default integer value.
-.TP
-PE_size
-The number of PEs in the active set. PE_size must be of type integer. If you
-are using Fortran, it must be a default integer value.
-.TP
-pSync
-A symmetric work array. In C/C++, pSync must be of type int and size
-_SHMEM_BARRIER_SYNC_SIZE. In Fortran, pSync must be of type integer and size
-SHMEM_BARRIER_SYNC_SIZE. If you are using Fortran, it must be a default integer type.
-Every element of this array must be initialized to 0 before any of the PEs in the active set enter
-shmem_barrier the first time.
-.PP
-The values of arguments PE_start, logPE_stride, and PE_size must be equal on all PEs in the
-active set. The same work array must be passed in pSync to all PEs in the active set.
-.PP
-shmem_barrier ensures that all previously issued local stores and previously issued remote
-memory updates done by any of the PEs in the active set (by using SHMEM calls, for
-example \fIshmem_put\fP(3))
-are complete before returning.
-.PP
-The same pSync array may be reused on consecutive calls to shmem_barrier if the same
-active PE set is used.
-.PP
-.SH NOTES
-
-The term symmetric is defined in \fIintro_shmem\fP(3)\&.
-.PP
-If the pSync array is initialized at run time, be sure to use some type of synchronization, for
-example, a call to \fIshmem_barrier_all\fP(3),
-before calling shmem_barrier for the first
-time.
-.PP
-If the active set does not change, shmem_barrier can be called repeatedly with the same
-pSync array. No additional synchronization beyond that implied by shmem_barrier itself is
-necessary in this case.
-.PP
-.SH EXAMPLES
-
-C/C++ example:
-.Vb
-shmem_barrier(PE_start, logPE_stride, size, pSync);
-.Ve
-Fortran example:
-.Vb
-INTEGER PSYNC(SHMEM_BARRIER_SYNC_SIZE)
-INTEGER PE_START, LOGPE_STRIDE, PE_SIZE, PSYNC
-DATA PSYNC /SHMEM_BARRIER_SYNC_SIZE*0/
-
-CALL SHMEM_BARRIER(PE_START, LOGPE_STRIDE, PE_SIZE, PSYNC)
-.Ve
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3),
-\fIshmem_barrier_all\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_barrier_all.3in b/oshmem/shmem/man/man3/shmem_barrier_all.3in
deleted file mode 100644
index a530b63e7ae..00000000000
--- a/oshmem/shmem/man/man3/shmem_barrier_all.3in
+++ /dev/null
@@ -1,59 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_BARRIER\\_ALL" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-shmem_barrier_all \- Suspends the execution of the calling PE until all other PEs issue a call
-to this particular shmem_barrier_all() statement.
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-void shmem_barrier_all(void);
-.Ve
-Fortran:
-.Vb
-include 'mpp/shmem.h'
-
-CALL SHMEM_BARRIER_ALL
-.Ve
-.SH DESCRIPTION
-
-The shmem_barrier_all routine does not return until all other PEs have entered this routine
-at the same point of the execution path.
-.PP
-Prior to synchronizing with other PEs, shmem_barrier_all ensures completion of all
-previously issued local memory stores and remote memory updates issued via SHMEM
-functions such as \fIshmem_put32\fP(3)\&.
-.PP
-.SH EXAMPLES
-
-.Vb
-setup_data()
-{
-  if (shmem_my_pe() == 0) {
-    setup();
-  }
-
-  /* All PEs wait for PE 0 to complete setup().  */
-  shmem_barrier_all();
-}
-.Ve
-.PP
-.SH SEE ALSO
-
-\fIshmem_barrier\fP(3),
-\fIshmem_init\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_broadcast32.3in b/oshmem/shmem/man/man3/shmem_broadcast32.3in
deleted file mode 100644
index abb38e7952e..00000000000
--- a/oshmem/shmem/man/man3/shmem_broadcast32.3in
+++ /dev/null
@@ -1,186 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_BROADCAST" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmem_broadcast4\fP(3),
-\fIshmem_broadcast8\fP(3),
-\fIshmem_broadcast32\fP(3),
-\fIshmem_broadcast64\fP(3)
-\- Copy a data object from a designated PE to a target
-location on all other PEs of the active set.
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-void shmem_broadcast32(void *target, const void *source,
-  size_t nelems, int PE_root, int PE_start, int logPE_stride,
-  int PE_size, long *pSync);
-
-void shmem_broadcast64(void *target, const void *source,
-  size_t nelems, int PE_root, int PE_start, int logPE_stride,
-  int PE_size, long *pSync);
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER nelems, PE_root, PE_start, logPE_stride, PE_size
-INTEGER pSync(SHMEM_BCAST_SYNC_SIZE)
-
-CALL SHMEM_BROADCAST4(target, source, nelems, PE_root,
-& PE_start, logPE_stride, PE_size, fIpSync)
-
-CALL SHMEM_BROADCAST8(target, source, nelems, PE_root,
-& PE_start, logPE_stride, PE_size, pSync)
-
-CALL SHMEM_BROADCAST32(target, source, nelems,
-& PE_root, PE_start, logPE_stride, PE_size, pSync)
-
-CALL SHMEM_BROADCAST64(target, source, nelems,
-& PE_root, PE_start, logPE_stride, PE_size, pSync)
-.Ve
-.SH DESCRIPTION
-
-The broadcast routines write the data at address source of the PE specified by
-\fBPE_root\fP
-to address \fBtarget\fP
-on all other PEs in the active set. The active set of
-PEs is defined by the triplet \fBPE_start\fP,
-\fBlogPE_stride\fP
-and \fBPE_size\fP\&.
-The data is not copied to the target address on the PE specified by \fBPE_root\fP\&.
-Before returning, the broadcast routines ensure that the elements of the pSync array are
-restored to their initial values.
-.PP
-As with all SHMEM collective routines, each of these routines assumes that only PEs in the
-active set call the routine. If a PE not in the active set calls a SHMEM collective routine,
-undefined behavior results.
-.PP
-The arguments are as follows:
-.TP
-target
-A symmetric data object with one of the following data types:
-.RS
-.TP
-\fBshmem_broadcast8, shmem_broadcast64\fP: Any noncharacter type that
-has an element size of 64 bits. No Fortran derived types or C/C++ structures are allowed.
-.TP
-\fBshmem_broadcast32\fP: Any noncharacter type that has an element size
-of 32 bits. No Fortran derived types or C/C++ structures are allowed.
-.TP
-\fBshmem_broadcast4\fP: Any noncharacter type that has an element size
-of 32 bits.
-.RE
-.RS
-.PP
-.RE
-.TP
-source
-A symmetric data object that can be of any data type that is permissible for the
-target argument.
-.TP
-nelems
-The number of elements in source. For shmem_broadcast32 and
-shmem_broadcast4, this is the number of 32\-bit halfwords. nelems must be of type integer.
-If you are using Fortran, it must be a default integer value.
-.TP
-PE_root
-Zero\-based ordinal of the PE, with respect to the active set, from which the
-data is copied. Must be greater than or equal to 0 and less than PE_size. PE_root must be of
-type integer. If you are using Fortran, it must be a default integer value.
-.TP
-PE_start
-The lowest virtual PE number of the active set of PEs. PE_start must be of
-type integer. If you are using Fortran, it must be a default integer value.
-.TP
-logPE_stride
-The log (base 2) of the stride between consecutive virtual PE numbers in
-the active set. log_PE_stride must be of type integer. If you are using Fortran, it must be a
-default integer value.
-.TP
-PE_size
-The number of PEs in the active set. PE_size must be of type integer. If you
-are using Fortran, it must be a default integer value.
-.PP
-.TP
-pSync
-A symmetric work array. In C/C++, pSync must be of type long and size
-_SHMEM_BCAST_SYNC_SIZE.
-In Fortran, pSync must be of type integer and size SHMEM_BCAST_SYNC_SIZE. Every
-element of this array must be initialized with the value _SHMEM_SYNC_VALUE (in C/C++)
-or SHMEM_SYNC_VALUE (in Fortran) before any of the PEs in the active set enter
-shmem_barrier().
-.PP
-The values of arguments PE_root, PE_start, logPE_stride, and PE_size must be equal on
-all PEs in the active set. The same target and source data objects and the same pSync work
-array must be passed to all PEs in the active set.
-.PP
-Before any PE calls a broadcast routine, you must ensure that the following conditions exist
-(synchronization via a barrier or some other method is often needed to ensure this): The
-pSync array on all PEs in the active set is not still in use from a prior call to a broadcast
-routine. The target array on all PEs in the active set is ready to accept the broadcast data.
-.PP
-Upon return from a broadcast routine, the following are true for the local PE: If the current PE
-is not the root PE, the target data object is updated. The values in the pSync array are
-restored to the original values.
-.SH NOTES
-
-The terms collective and symmetric are defined in \fIintro_shmem\fP(3)\&.
-.PP
-All SHMEM broadcast routines restore pSync to its original contents. Multiple calls to SHMEM
-routines that use the same pSync array do not require that pSync be reinitialized after the
-first call.
-.PP
-You must ensure the that the pSync array is not being updated by any PE in the active set
-while any of the PEs participates in processing of a SHMEM broadcast routine. Be careful to
-avoid these situations: If the pSync array is initialized at run time, some type of
-synchronization is needed to ensure that all PEs in the working set have initialized pSync
-before any of them enter a SHMEM routine called with the pSync synchronization array. A
-pSync array may be reused on a subsequent SHMEM broadcast routine only if none of the PEs
-in the active set are still processing a prior SHMEM broadcast routine call that used the same
-pSync array. In general, this can be ensured only by doing some type of synchronization.
-However, in the special case of SHMEM routines being called with the same active set, you
-can allocate two pSync arrays and alternate between them on successive calls.
-.PP
-.SH EXAMPLES
-
-In the following examples, the call to shmem_broadcast64 copies source on PE 4 to target
-on PEs 5, 6, and 7.
-.PP
-C/C++ example:
-.Vb
-for (i=0; i < _SHMEM_BCAST_SYNC_SIZE; i++) {
-  pSync[i] = _SHMEM_SYNC_VALUE;
-}
-shmem_barrier_all(); /* Wait for all PEs to initialize pSync */
-shmem_broadcast64(target, source, nelems, 0, 4, 0, 4, pSync);
-.Ve
-Fortran example:
-.Vb
-INTEGER PSYNC(SHMEM_BCAST_SYNC_SIZE)
-INTEGER TARGET, SOURCE, NELEMS, PE_ROOT, PE_START,
-& LOGPE_STRIDE, PE_SIZE, PSYNC
-COMMON /COM/ TARGET, SOURCE
-DATA PSYNC /SHMEM_BCAST_SYNC_SIZE*SHMEM_SYNC_VALUE/
-
-CALL SHMEM_BROADCAST64(TARGET, SOURCE, NELEMS, 0, 4, 0, 4,
-& PSYNC)
-.Ve
-.PP
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_broadcast64.3in b/oshmem/shmem/man/man3/shmem_broadcast64.3in
deleted file mode 100644
index c9b171b335c..00000000000
--- a/oshmem/shmem/man/man3/shmem_broadcast64.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_broadcast32.3
diff --git a/oshmem/shmem/man/man3/shmem_char_g.3in b/oshmem/shmem/man/man3/shmem_char_g.3in
deleted file mode 100644
index 512444a1638..00000000000
--- a/oshmem/shmem/man/man3/shmem_char_g.3in
+++ /dev/null
@@ -1,64 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_CHAR\\_G" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmem_char_g\fP(3),
-\fIshmem_float_g\fP(3),
-\fIshmem_int_g\fP(3),
-\fIshmem_long_g\fP(3),
-\fIshmem_short_g\fP(3),
-\fIshmem_longlong_g\fP(3),
-\fIshmem_longdouble_g\fP(3)
-\- These routines provide a low latency mechanism to read basic types (char, short, int, float, double, long, long long, long double) from symmetric data objects on remote PEs.
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-
-char shmem_char_g(const char *addr, int pe);
-
-short shmem_short_g(const short *addr, int pe);
-
-int shmem_int_g(const int *addr, int pe);
-
-long shmem_long_g(const long *addr, int pe);
-
-long shmem_longlong_g(const long long *addr, int pe);
-
-float shmem_float_g(const float *addr, int pe);
-
-double shmem_double_g(const double *addr, int pe);
-
-long shmem_longdouble_g(const long double *addr, int pe);
-
-.Ve
-.SH DESCRIPTION
-
-These routines provide a very low latency get capability for single elements of most basic types.
-.PP
-The arguments are as follows:
-.TP
-addr
-The remotely accessible array element or scalar data object which will receive the
-data on the remote PE.
-.TP
-pe
-The number of the remote PE.
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3),
-\fIshmem_get\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_char_get.3in b/oshmem/shmem/man/man3/shmem_char_get.3in
deleted file mode 100644
index 80910049200..00000000000
--- a/oshmem/shmem/man/man3/shmem_char_get.3in
+++ /dev/null
@@ -1,207 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_GET" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmem_character_get\fP(3),
-\fIshmem_complex_get\fP(3),
-\fIshmem_double_get\fP(3),
-\fIshmem_float_get\fP(3),
-\fIshmem_get4\fP(3),
-\fIshmem_get8\fP(3),
-\fIshmem_get32\fP(3),
-\fIshmem_get64\fP(3),
-\fIshmem_get128\fP(3),
-\fIshmem_getmem\fP(3),
-\fIshmem_int_get\fP(3),
-\fIshmem_integer_get\fP(3),
-\fIshmem_logical_get\fP(3),
-\fIshmem_long_get\fP(3),
-\fIshmem_longdouble_get\fP(3),
-\fIshmem_longlong_get\fP(3),
-\fIshmem_real_get\fP(3),
-\fIshmem_short_get\fP(3)
-\- Transfers data from a specified processing element (PE).
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-void shmem_get32(void *target, const void *source,
-  size_t len, int pe);
-
-void shmem_get64(void *target, const void *source,
-  size_t len, int pe);
-
-void shmem_get128(void *target, const void *source,
-  size_t len, int pe);
-
-void shmem_getmem(void *target, const void *source,
-  size_t len, int pe);
-
-void shmem_int_get(int *target, const int *source,
-  size_t len, int pe);
-
-void shmem_double_get(double *target, const double *source,
-  size_t len, int pe);
-
-void shmem_float_get(float *target, const float *source,
-  size_t len, int pe);
-
-void shmem_long_get(long *target, const long *source,
-  size_t len, int pe);
-
-void shmem_longdouble_get(long double *target,
-  const long double *source, size_t len, int pe);
-
-void shmem_longlong_get(long long *target,
-  const long long *source, size_t len, int pe);
-
-void shmem_short_get(short *target,
-  const short *source, size_t len, int pe);
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER len, pe
-
-CALL SHMEM_CHARACTER_GET(target, source, len, pe)
-
-CALL SHMEM_COMPLEX_GET(target, source, len, pe)
-
-CALL SHMEM_DOUBLE_GET(target, source, len, pe)
-
-CALL SHMEM_GET4(target, source, len, pe)
-
-CALL SHMEM_GET8(target, source, len, pe)
-
-CALL SHMEM_GET32(target, source, len, pe)
-
-CALL SHMEM_GET64(target, source, len, pe)
-
-CALL SHMEM_GET128(target, source, len, pe)
-
-CALL SHMEM_GETMEM(target, source, len, pe)
-
-CALL SHMEM_INTEGER_GET(target, source, len, pe)
-
-CALL SHMEM_LOGICAL_GET(target, source, len, pe)
-
-CALL SHMEM_REAL_GET(target, source, len, pe)
-.Ve
-.SH DESCRIPTION
-
-The shmem_get routines transfer \fBnelems\fP
-elements of the data object at address \fBsource\fP
-on the remote PE \fBpe\fP,
-to the data object at address \fBtarget\fP
-on the local PE. These routines
-return after the data has been copied to address \fBtarget\fP
-on the local PE.
-.PP
-The arguments are as follows:
-.TP
-target
-Local data object to be updated.
-.TP
-source
-Data object on the PE identified by pe that contains the data to be copied. This
-data object must be remotely accessible.
-.TP
-len
-Number of elements in the target and source arrays. len must be of type integer. If
-you are using Fortran, it must be a constant, variable, or array element of default
-integer type.
-.TP
-pe
-PE number of the remote PE. pe must be of type integer. If you are using Fortran, it
-must be a constant, variable, or array element of default integer type.
-.PP
-The target and source data objects must conform to typing constraints, which are as follows:
-.TP
-\fBshmem_getmem\fP: Fortran: Any noncharacter type. C: Any data type. len is
-scaled in bytes.
-.TP
-\fBshmem_get4, shmem_get32\fP: Any noncharacter type that has a storage size
-equal to 32 bits.
-.TP
-{shmem_get8, shmem_get64}: Any noncharacter type that has a storage size equal to
-64 bits.
-.TP
-\fBshmem_get128\fP: Any noncharacter type that has a storage size equal to 128
-bits.
-.TP
-\fBshmem_short_get\fP: Elements of type short.
-.TP
-\fBshmem_int_get\fP: Elements of type int.
-.TP
-\fBshmem_long_get\fP: Elements of type long.
-.TP
-\fBshmem_longlong_get\fP: Elements of type long long.
-.TP
-\fBshmem_float_get\fP: Elements of type float.
-.TP
-\fBshmem_double_get\fP: Elements of type double.
-.TP
-\fBshmem_longdouble_get\fP: Elements of type long double.
-.TP
-\fBSHMEM_CHARACTER_GET\fP: Elements of type character. len is the number of
-characters to transfer. The actual character lengths of the source and target variables are
-ignored.
-.TP
-\fBSHMEM_COMPLEX_GET\fP: Elements of type complex of default size.
-.TP
-\fBSHMEM_DOUBLE_GET\fP: (Fortran) Elements of type double precision.
-.TP
-\fBSHMEM_INTEGER_GET\fP: Elements of type integer.
-.TP
-\fBSHMEM_LOGICAL_GET\fP: Elements of type logical.
-.TP
-\fBSHMEM_REAL_GET\fP: Elements of type real.
-.PP
-If you are using Fortran, data types must be of default size. For example, a real variable must
-be declared as REAL, REAL*4, or REAL(KIND=4).
-.SH NOTES
-
-See \fIintro_shmem\fP(3)
-for a definition of the term remotely accessible.
-.SH EXAMPLES
-
-Consider this simple example for Fortran.
-.Vb
-PROGRAM REDUCTION
-  REAL VALUES, SUM
-  COMMON /C/ VALUES
-  REAL WORK
-
-  CALL START_PES(0) ! ALLOW ANY NUMBER OF PES
-  VALUES = MY_PE() ! INITIALIZE IT TO SOMETHING
-  CALL SHMEM_BARRIER_ALL
-  SUM = 0.0
-  DO I = 0,NUM_PES()\-1
-    CALL SHMEM_REAL_GET(WORK, VALUES, 1, I)
-    SUM = SUM + WORK
-  ENDDO
-  PRINT *, 'PE ', MY_PE(), ' COMPUTED SUM=', SUM
-  CALL SHMEM_BARRIER_ALL
-END
-.Ve
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3),
-\fIshmem_put\fP(3),
-\fIshmem_iget\fP(3),
-\fIshmem_quiet\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_char_get_nbi.3in b/oshmem/shmem/man/man3/shmem_char_get_nbi.3in
deleted file mode 100644
index 50449e90a6d..00000000000
--- a/oshmem/shmem/man/man3/shmem_char_get_nbi.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_getmem_nbi.3
diff --git a/oshmem/shmem/man/man3/shmem_char_p.3in b/oshmem/shmem/man/man3/shmem_char_p.3in
deleted file mode 100644
index 3d122b2d9cc..00000000000
--- a/oshmem/shmem/man/man3/shmem_char_p.3in
+++ /dev/null
@@ -1,73 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_CHAR\\_P" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmem_char_p\fP(3),
-\fIshmem_float_p\fP(3),
-\fIshmem_int_p\fP(3),
-\fIshmem_long_p\fP(3),
-\fIshmem_short_p\fP(3),
-\fIshmem_longlong_p\fP(3),
-\fIshmem_longdouble_p\fP(3)
-\- These routines provide a low latency mechanism to write basic types (char, short, int, float, double, long, long long, long double) to symmetric data objects on remote PEs.
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-
-void shmem_char_p(char *addr, char value, int pe);
-
-void shmem_short_p(short *addr, short value, int pe);
-
-void shmem_int_p(int *addr, int value, int pe);
-
-void shmem_long_p(long *addr, long value, int pe);
-
-void shmem_longlong_p(long long *addr, long long value, int pe);
-
-void shmem_float_p(float *addr, float value, int pe);
-
-void shmem_double_p(double *addr, double value, int pe);
-
-void shmem_longdouble_p(long double *addr, long double value, int pe);
-
-.Ve
-.SH DESCRIPTION
-
-These routines provide a very low latency put capability for single elements of most basic types.
-.PP
-The arguments are as follows:
-.TP
-addr
-The remotely accessible array element or scalar data object which will receive the
-data on the remote PE.
-.TP
-value
-The value to be transferred to addr on the remote PE.
-.TP
-pe
-The number of the remote PE.
-.PP
-As with \fIshmem_put\fP(3),
-these functions start the remote transfer and may return before
-the data is delivered to the remote PE. Use \fIshmem_quiet\fP(3)
-to force completion of all
-remote PUT transfers.
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3),
-\fIshmem_put\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_char_put.3in b/oshmem/shmem/man/man3/shmem_char_put.3in
deleted file mode 100644
index fca93b5ff90..00000000000
--- a/oshmem/shmem/man/man3/shmem_char_put.3in
+++ /dev/null
@@ -1,214 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_PUT" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmem_character_put\fP(3),
-\fIshmem_complex_put\fP(3),
-\fIshmem_double_put\fP(3),
-\fIshmem_float_put\fP(3),
-\fIshmem_int_put\fP(3),
-\fIshmem_integer_put\fP(3),
-\fIshmem_logical_put\fP(3),
-\fIshmem_long_put\fP(3),
-\fIshmem_longdouble_put\fP(3),
-\fIshmem_longlong_put\fP(3),
-\fIshmem_put4\fP(3),
-\fIshmem_put8\fP(3),
-\fIshmem_put32\fP(3),
-\fIshmem_put64\fP(3),
-\fIshmem_put128\fP(3),
-\fIshmem_putmem\fP(3),
-\fIshmem_real_put\fP(3),
-\fIshmem_short_put\fP(3)
-\- Transfers data to a specified
-processing element (PE)
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-void shmem_double_put(double *target, const double *source,
-  size_t len, int pe);
-
-void shmem_float_put(float *target, const float *source,
-  size_t len, int pe);
-
-void shmem_int_put(int *target, const int *source, size_t len,
-  int pe);
-
-void shmem_long_put(long *target, const long *source,
-  size_t len, int pe);
-
-void shmem_longdouble_put(long double *target,
-  const long double *source, size_t len, int pe);
-
-void shmem_longlong_put(long long *target,
-  const long long *source, size_t len, int pe);
-
-void shmem_put32(void *target, const void *source, size_t len,
-  int pe);
-
-void shmem_put64(void *target, const void *source, size_t len,
-  int pe);
-
-void shmem_put128(void *target, const void *source, size_t len,
-  int pe);
-
-void shmem_putmem(void *target, const void *source, size_t len,
-  int pe);
-
-void shmem_short_put(short *target, const short *source,
-  size_t len, int pe);
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER len, pe
-
-CALL SHMEM_CHARACTER_PUT(target, source, len, pe)
-
-CALL SHMEM_COMPLEX_PUT(target, source, len, pe)
-
-CALL SHMEM_DOUBLE_PUT(target, source, len, pe)
-
-CALL SHMEM_INTEGER_PUT(target, source, len, pe)
-
-CALL SHMEM_LOGICAL_PUT(target, source, len, pe)
-
-CALL SHMEM_PUT(target, source, len, pe)
-
-CALL SHMEM_PUT4(target, source, len, pe)
-
-CALL SHMEM_PUT8(target, source, len, pe)
-
-CALL SHMEM_PUT32(target, source, len, pe)
-
-CALL SHMEM_PUT64(target, source, len, pe)
-
-CALL SHMEM_PUT128(target, source, len, pe)
-
-CALL SHMEM_PUTMEM(target, source, len, pe)
-
-CALL SHMEM_REAL_PUT(target, source, len, pe)
-.Ve
-.SH DESCRIPTION
-
-These routines transfer \fBnelems\fP
-elements of the data object at address
-\fBsource\fP
-on the calling PE, to the data object at address \fBtarget\fP
-on the remote
-PE \fBpe\fP\&.
-These routines start the remote transfer and may return before the data is
-delivered to the remote PE.
-.PP
-The delivery of data into the data object on the destination PE from different put calls may
-occur in any order. Because of this, two successive put operations may deliver data out of
-order unless a call to \fIshmem_fence\fP(3)
-is introduced between the two calls.
-.PP
-The arguments are as follows:
-.TP
-target
-Data object to be updated on the remote PE. This data object must be remotely
-accessible.
-.TP
-source
-Data object containing the data to be copied.
-.TP
-len
-Number of elements in the target and source arrays. len must be of type integer. If
-you are using Fortran, it must be a constant, variable, or array element of default integer
-type.
-.TP
-pe
-PE number of the remote PE. pe must be of type integer. If you are using Fortran, it
-must be a constant, variable, or array element of default integer type.
-.PP
-The target and source data objects must conform to certain typing constraints, which are as
-follows:
-.TP
-\fBshmem_putmem\fP: Fortran: Any noncharacter type. C: Any data type. len is scaled in
-bytes.
-.TP
-\fBshmem_put4, shmem_put32:\fP Any noncharacter type that has a storage size
-equal to 32 bits.
-.TP
-\fBshmem_put8, shmem_put64:\fP Any noncharacter type that has a storage size
-equal to 64 bits.
-.TP
-\fBshmem_put128:\fP Any noncharacter type that has a storage size equal to 128
-bits.
-.TP
-\fBshmem_short_put:\fP Elements of type short.
-.TP
-\fBshmem_int_put:\fP Elements of type int.
-.TP
-\fBshmem_long_put:\fP Elements of type long.
-.TP
-\fBshmem_longlong_put:\fP Elements of type long long.
-.TP
-\fBshmem_float_put:\fP Elements of type float.
-.TP
-\fBshmem_double_put:\fP Elements of type double.
-.TP
-\fBshmem_longdouble_put:\fP Elements of type long double.
-.TP
-\fBSHMEM_CHARACTER_PUT:\fP Elements of type character. len is the number of
-characters to transfer. The actual character lengths of the source and target variables are
-ignored.
-.TP
-\fBSHMEM_COMPLEX_PUT:\fP Elements of type complex of default size.
-.TP
-\fBSHMEM_DOUBLE_PUT:\fP (Fortran) Elements of type double precision.
-.TP
-\fBSHMEM_INTEGER_PUT:\fP Elements of type integer.
-.TP
-\fBSHMEM_LOGICAL_PUT:\fP Elements of type logical.
-.TP
-\fBSHMEM_REAL_PUT:\fP Elements of type real.
-If you are using Fortran, data types must be of default size. For example, a real variable must
-be declared as REAL, REAL*4, or REAL(KIND=4).
-.PP
-.SH EXAMPLES
-
-The following shmem_put example is for C/C++ programs:
-.Vb
-#include <stdio.h>
-#include <mpp/shmem.h>
-
-main()
-{
-  long source[10] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
-  static long target[10];
-  shmem_init();
-
-  if (shmem_my_pe() == 0) {
-    /* put 10 words into target on PE 1 */
-    shmem_long_put(target, source, 10, 1);
-  }
-  shmem_barrier_all();  /* sync sender and receiver */
-  if (shmem_my_pe() == 1)
-    shmem_udcflush();  /* not required on Altix systems */
-  printf("target[0] on PE %d is %d\\n", shmem_my_pe(), target[0]);
-}
-.Ve
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3),
-\fIshmem_iput\fP(3),
-\fIshmem_quiet\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_char_put_nbi.3in b/oshmem/shmem/man/man3/shmem_char_put_nbi.3in
deleted file mode 100644
index fb4ad1413b0..00000000000
--- a/oshmem/shmem/man/man3/shmem_char_put_nbi.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_putmem_nbi.3
diff --git a/oshmem/shmem/man/man3/shmem_clear_cache_inv.3in b/oshmem/shmem/man/man3/shmem_clear_cache_inv.3in
deleted file mode 100644
index 4a6a361ef97..00000000000
--- a/oshmem/shmem/man/man3/shmem_clear_cache_inv.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_udcflush.3
diff --git a/oshmem/shmem/man/man3/shmem_clear_cache_line_inv.3in b/oshmem/shmem/man/man3/shmem_clear_cache_line_inv.3in
deleted file mode 100644
index 4a6a361ef97..00000000000
--- a/oshmem/shmem/man/man3/shmem_clear_cache_line_inv.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_udcflush.3
diff --git a/oshmem/shmem/man/man3/shmem_clear_lock.3in b/oshmem/shmem/man/man3/shmem_clear_lock.3in
deleted file mode 100644
index 9a1ee8ffc2e..00000000000
--- a/oshmem/shmem/man/man3/shmem_clear_lock.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_set_lock.3
diff --git a/oshmem/shmem/man/man3/shmem_collect32.3in b/oshmem/shmem/man/man3/shmem_collect32.3in
deleted file mode 100644
index bce6dc5aa00..00000000000
--- a/oshmem/shmem/man/man3/shmem_collect32.3in
+++ /dev/null
@@ -1,197 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_COLLECT" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmem_collect4\fP(3),
-\fIshmem_collect8\fP(3),
-\fIshmem_collect32\fP(3),
-\fIshmem_collect64\fP(3),
-\fIshmem_fcollect\fP(3),
-\fIshmem_fcollect4\fP(3),
-\fIshmem_fcollect8\fP(3),
-\fIshmem_fcollect32\fP(3),
-\fIshmem_fcollect64\fP(3)
-\- Concatenates blocks of data from multiple processing elements (PEs) to an array in every PE
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-void shmem_collect32(void *target, const void *source,
-  size_t nelems, int PE_start, int logPE_stride, int PE_size,
-  long *pSync);
-
-void shmem_collect64(void *target, const void *source,
-  size_t nelems, int PE_start, int logPE_stride, int PE_size,
-  long *pSync);
-
-void shmem_fcollect32(void *target, const void *source,
-  size_t nelems, int PE_start, int logPE_stride, int PE_size,
-  long *pSync);
-
-void shmem_fcollect64(void *target, const void *source,
-  size_t nelems, int PE_start, int logPE_stride, int PE_size,
-  long *pSync);
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER nelems
-INTEGER PE_start, logPE_stride, PE_size
-INTEGER pSync(SHMEM_COLLECT_SYNC_SIZE)
-
-CALL SHMEM_COLLECT4(target, source, nelems, PE_start,
-& logPE_stride, PE_size, pSync)
-
-CALL SHMEM_COLLECT8(target, source, nelems, PE_start,
-& logPE_stride, PE_size, pSync)
-
-CALL SHMEM_FCOLLECT4(target, source, nelems, PE_start,
-& logPE_stride, PE_size, pSync)
-
-CALL SHMEM_FCOLLECT8(target, source, nelems, PE_start,
-& logPE_stride, PE_size, pSync)
-.Ve
-.SH DESCRIPTION
-
-The shared memory (SHMEM) collect and fcollect routines concatenate nelems 64\-bit or 32\-bit
-data items from the source array into the target array, over the set of PEs defined by
-PE_start, log2PE_stride, and PE_size, in processor number order. The resultant target array
-contains the contribution from PE PE_start first, then the contribution from PE PE_start +
-PE_stride second, and so on. The collected result is written to the target array for all PEs in
-the active set.
-.PP
-The fcollect routines require that nelems be the same value in all participating PEs, while the
-collect routines allow nelems to vary from PE to PE.
-.PP
-The resulting target array is as follows:
-.Vb
-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-
-   source(1..nelems)
-       from PE (PE_start + 0 * (2**logPE_stride))
-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-
-   source(1..nelems)
-       from PE (PE_start + 1 * (2**logPE_stride))
-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-
-   ...
-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-
-   source(1..nelems) from
-       PE (PE_start + (PE_size \- 1) * (2**logPE_stride))
-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-
-.Ve
-.PP
-As with all SHMEM collective routines, each of these routines assumes that only PEs in
-the active set call the routine. If a PE not in the active set calls a SHMEM collective routine,
-undefined behavior results.
-.PP
-The arguments are as follows:
-.TP
-target
-A symmetric array. The target argument must be large enough to accept the concatenation of the source arrays on all PEs. The data types are
-as follows:
-.RS
-.TP
-[shmem_collect8, shmem_collect64, shmem_fcollect8, and
-shmem_fcollect64] any data type with an element size of 64 bits. Fortran derived types,
-Fortran character type, and C/C++ structures are not permitted.
-.TP
-[shmem_collect4, shmem_collect32, shmem_fcollect4, and
-shmem_fcollect32] any data type with an element size of 32 bits. Fortran derived types,
-Fortran character type, and C/C++ structures are not permitted.
-.RE
-.RS
-.PP
-.RE
-.TP
-source
-A symmetric data object that can be of any type permissible for the target
-argument.
-.TP
-nelems
-The number of elements in the source array. nelems must be of type integer. If
-you are using Fortran, it must be a default integer value.
-.TP
-PE_start
-The lowest virtual PE number of the active set of PEs. PE_start must be of
-type integer. If you are using Fortran, it must be a default integer value.
-.TP
-logPE_stride
-The log (base 2) of the stride between consecutive virtual PE numbers in
-the active set. logPE_stride must be of type integer. If you are using Fortran, it must be a
-default integer value.
-.TP
-PE_size
-The number of PEs in the active set. PE_size must be of type integer. If you
-are using Fortran, it must be a default integer value.
-.TP
-pSync
-A symmetric work array. In C/C++, pSync must be of type int and size
-_SHMEM_COLLECT_SYNC_SIZE. In Fortran, pSync must be of type integer and size
-SHMEM_COLLECT_SYNC_SIZE. If you are using Fortran, it must be a default integer value.
-Every element of this array must be initialized with the value _SHMEM_SYNC_VALUE in
-C/C++ or SHMEM_SYNC_VALUE in Fortran before any of the PEs in the active set enter
-shmem_barrier().
-.PP
-The values of arguments PE_start, logPE_stride, and PE_size must be equal on all PEs in
-the active set. The same target and source arrays and the same pSync work array must be
-passed to all PEs in the active set.
-.PP
-Upon return from a collective routine, the following are true for the local PE: The target array
-is updated. The values in the pSync array are restored to the original values.
-.SH NOTES
-
-The terms collective and symmetric are defined in \fIintro_shmem\fP(3)\&.
-All SHMEM collective routines reset the values in pSync before they return, so a particular
-pSync buffer need only be initialized the first time it is used.
-.PP
-You must ensure that the pSync array is not being updated on any PE in the active set while
-any of the PEs participate in processing of a SHMEM collective routine. Be careful to
-avoid these situations: If the pSync array is initialized at run time, some type of
-synchronization is needed to ensure that all PEs in the working set have initialized
-pSync before any of them enter a SHMEM routine called with the pSync synchronization array.
-A pSync array can be reused on a subsequent SHMEM collective routine only if none
-of the PEs in the active set are still processing a prior SHMEM collective routine call that used
-the same pSync array. In general, this may be ensured only by doing some type of
-synchronization. However, in the special case of SHMEM routines being called with the same
-active set, you can allocate two pSync arrays and alternate between them on
-successive calls.
-.PP
-The collective routines operate on active PE sets that have a non\-power\-of\-two PE_size
-with some performance degradation. They operate with no performance degradation
-when nelems is a non\-power\-of\-two value.
-.SH EXAMPLES
-
-C/C++:
-.Vb
-for (i=0; i < _SHMEM_COLLECT_SYNC_SIZE; i++) {
-  pSync[i] = _SHMEM_SYNC_VALUE;
-}
-shmem_barrier_all(); /* Wait for all PEs to initialize pSync */
-shmem_collect32(target, source, 64, pe_start, logPE_stride,
-   pe_size, pSync);
-.Ve
-Fortran:
-.Vb
-INTEGER PSYNC(SHMEM_COLLECT_SYNC_SIZE)
-DATA PSYNC /SHMEM_COLLECT_SYNC_SIZE*SHMEM_SYNC_VALUE/
-
-CALL SHMEM_COLLECT4(TARGET, SOURCE, 64, PE_START,
-& LOGPE_STRIDE, PE_SIZE, PSYNC)
-.Ve
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_collect64.3in b/oshmem/shmem/man/man3/shmem_collect64.3in
deleted file mode 100644
index 42d0ca9595c..00000000000
--- a/oshmem/shmem/man/man3/shmem_collect64.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_collect32.3
diff --git a/oshmem/shmem/man/man3/shmem_complexd_prod_to_all.3in b/oshmem/shmem/man/man3/shmem_complexd_prod_to_all.3in
deleted file mode 100644
index 39b196d0820..00000000000
--- a/oshmem/shmem/man/man3/shmem_complexd_prod_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_prod_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_complexd_sum_to_all.3in b/oshmem/shmem/man/man3/shmem_complexd_sum_to_all.3in
deleted file mode 100644
index f75a4948417..00000000000
--- a/oshmem/shmem/man/man3/shmem_complexd_sum_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_sum_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_complexf_prod_to_all.3in b/oshmem/shmem/man/man3/shmem_complexf_prod_to_all.3in
deleted file mode 100644
index 39b196d0820..00000000000
--- a/oshmem/shmem/man/man3/shmem_complexf_prod_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_prod_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_complexf_sum_to_all.3in b/oshmem/shmem/man/man3/shmem_complexf_sum_to_all.3in
deleted file mode 100644
index f75a4948417..00000000000
--- a/oshmem/shmem/man/man3/shmem_complexf_sum_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_sum_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_double_fetch.3in b/oshmem/shmem/man/man3/shmem_double_fetch.3in
deleted file mode 100644
index 7213c75e538..00000000000
--- a/oshmem/shmem/man/man3/shmem_double_fetch.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_int_fetch.3
diff --git a/oshmem/shmem/man/man3/shmem_double_g.3in b/oshmem/shmem/man/man3/shmem_double_g.3in
deleted file mode 100644
index cd9ba27b1bc..00000000000
--- a/oshmem/shmem/man/man3/shmem_double_g.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_g.3
diff --git a/oshmem/shmem/man/man3/shmem_double_get.3in b/oshmem/shmem/man/man3/shmem_double_get.3in
deleted file mode 100644
index 7cecf74d7ad..00000000000
--- a/oshmem/shmem/man/man3/shmem_double_get.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_get.3
diff --git a/oshmem/shmem/man/man3/shmem_double_get_nbi.3in b/oshmem/shmem/man/man3/shmem_double_get_nbi.3in
deleted file mode 100644
index 50449e90a6d..00000000000
--- a/oshmem/shmem/man/man3/shmem_double_get_nbi.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_getmem_nbi.3
diff --git a/oshmem/shmem/man/man3/shmem_double_iget.3in b/oshmem/shmem/man/man3/shmem_double_iget.3in
deleted file mode 100644
index df8770591af..00000000000
--- a/oshmem/shmem/man/man3/shmem_double_iget.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_iget.3
diff --git a/oshmem/shmem/man/man3/shmem_double_iput.3in b/oshmem/shmem/man/man3/shmem_double_iput.3in
deleted file mode 100644
index 2fdf7a97e3d..00000000000
--- a/oshmem/shmem/man/man3/shmem_double_iput.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_iput.3
diff --git a/oshmem/shmem/man/man3/shmem_double_max_to_all.3in b/oshmem/shmem/man/man3/shmem_double_max_to_all.3in
deleted file mode 100644
index 3e41c814889..00000000000
--- a/oshmem/shmem/man/man3/shmem_double_max_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_max_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_double_min_to_all.3in b/oshmem/shmem/man/man3/shmem_double_min_to_all.3in
deleted file mode 100644
index d688221529b..00000000000
--- a/oshmem/shmem/man/man3/shmem_double_min_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_min_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_double_p.3in b/oshmem/shmem/man/man3/shmem_double_p.3in
deleted file mode 100644
index 42b9fd0e81f..00000000000
--- a/oshmem/shmem/man/man3/shmem_double_p.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_p.3
diff --git a/oshmem/shmem/man/man3/shmem_double_prod_to_all.3in b/oshmem/shmem/man/man3/shmem_double_prod_to_all.3in
deleted file mode 100644
index 39b196d0820..00000000000
--- a/oshmem/shmem/man/man3/shmem_double_prod_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_prod_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_double_put.3in b/oshmem/shmem/man/man3/shmem_double_put.3in
deleted file mode 100644
index 9c7b2e25452..00000000000
--- a/oshmem/shmem/man/man3/shmem_double_put.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_put.3
diff --git a/oshmem/shmem/man/man3/shmem_double_put_nbi.3in b/oshmem/shmem/man/man3/shmem_double_put_nbi.3in
deleted file mode 100644
index fb4ad1413b0..00000000000
--- a/oshmem/shmem/man/man3/shmem_double_put_nbi.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_putmem_nbi.3
diff --git a/oshmem/shmem/man/man3/shmem_double_set.3in b/oshmem/shmem/man/man3/shmem_double_set.3in
deleted file mode 100644
index a02bb7ea5ea..00000000000
--- a/oshmem/shmem/man/man3/shmem_double_set.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_int_set.3
diff --git a/oshmem/shmem/man/man3/shmem_double_sum_to_all.3in b/oshmem/shmem/man/man3/shmem_double_sum_to_all.3in
deleted file mode 100644
index f75a4948417..00000000000
--- a/oshmem/shmem/man/man3/shmem_double_sum_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_sum_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_double_swap.3in b/oshmem/shmem/man/man3/shmem_double_swap.3in
deleted file mode 100644
index 31324f2c2e3..00000000000
--- a/oshmem/shmem/man/man3/shmem_double_swap.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_swap.3
diff --git a/oshmem/shmem/man/man3/shmem_fcollect32.3in b/oshmem/shmem/man/man3/shmem_fcollect32.3in
deleted file mode 100644
index 42d0ca9595c..00000000000
--- a/oshmem/shmem/man/man3/shmem_fcollect32.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_collect32.3
diff --git a/oshmem/shmem/man/man3/shmem_fcollect64.3in b/oshmem/shmem/man/man3/shmem_fcollect64.3in
deleted file mode 100644
index 42d0ca9595c..00000000000
--- a/oshmem/shmem/man/man3/shmem_fcollect64.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_collect32.3
diff --git a/oshmem/shmem/man/man3/shmem_fence.3in b/oshmem/shmem/man/man3/shmem_fence.3in
deleted file mode 100644
index 12e97a55dc7..00000000000
--- a/oshmem/shmem/man/man3/shmem_fence.3in
+++ /dev/null
@@ -1,54 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_FENCE" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-shmem_fence \- Provides a separate ordering on the sequence of puts issued by this PE to each destination
-PE.
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-void shmem_fence(void);
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-CALL SHMEM_FENCE
-.Ve
-.SH DESCRIPTION
-
-The \fBshmem_fence()\fP
-routine provides an ordering on the put operations issued by the calling
-PE prior to the call to \fBshmem_fence()\fP
-relative to the put operations issued by the
-calling PE following the call to \fBshmem_fence()\fP\&.
-It guarantees that all such prior put operations
-issued to a particular destination PE are fully written to the symmetric memory of
-that destination PE, before any such following put operations to that same destination PE
-are written to the symmetric memory of that destination PE.
-Note that the ordering is provided separately on the sequences of puts from the calling PE to
-each distinct destination PE. The \fBshmem_quiet()\fP
-routine should be used instead if ordering
-of puts is required when multiple destination PEs are involved.
-.SH NOTES
-
-The shmem_quiet function should be called if ordering of puts is desired when multiple remote
-PEs are involved.
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_finalize.3in b/oshmem/shmem/man/man3/shmem_finalize.3in
deleted file mode 100644
index d0acafdf6ea..00000000000
--- a/oshmem/shmem/man/man3/shmem_finalize.3in
+++ /dev/null
@@ -1,47 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_FINALIZE" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-shmem_finalize
-\- A collective operation that releases resources used by the OpenSHMEM library.
-This only terminates the Open-SHMEM portion of a program, not the entire program.
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-void shmem_finalize(void);
-.Ve
-Fortran:
-.Vb
-include 'mpp/shmem.fh'
-CALL SHMEM_FINALIZE
-.Ve
-.SH DESCRIPTION
-
-shmem_finalize
-is a collective operation that ends the OpenSHMEM portion of a program previously initialized
-by shmem_init and releases resources used by the OpenSHMEM library. This collective operation requires
-all PEs to participate in the call. There is an implicit global barrier in shmem_finalize so that pending
-communication is completed, and no resources can be released until all PEs have entered shmem_finalize.
-shmem_finalize must be the last OpenSHMEM library call encountered in the OpenSHMEM portion of
-a program. A call to shmem_finalize will release any resources initialized by a corresponding call to
-shmem_init. All processes and threads that represent the PEs will still exist after the call to shmem_finalize
-returns, but they will no longer have access to any resources that have been released.
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3),
-\fIshmem_my_pe\fP(3),
-\fIshmem_init\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_float_fetch.3in b/oshmem/shmem/man/man3/shmem_float_fetch.3in
deleted file mode 100644
index 7213c75e538..00000000000
--- a/oshmem/shmem/man/man3/shmem_float_fetch.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_int_fetch.3
diff --git a/oshmem/shmem/man/man3/shmem_float_g.3in b/oshmem/shmem/man/man3/shmem_float_g.3in
deleted file mode 100644
index cd9ba27b1bc..00000000000
--- a/oshmem/shmem/man/man3/shmem_float_g.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_g.3
diff --git a/oshmem/shmem/man/man3/shmem_float_get.3in b/oshmem/shmem/man/man3/shmem_float_get.3in
deleted file mode 100644
index 7cecf74d7ad..00000000000
--- a/oshmem/shmem/man/man3/shmem_float_get.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_get.3
diff --git a/oshmem/shmem/man/man3/shmem_float_get_nbi.3in b/oshmem/shmem/man/man3/shmem_float_get_nbi.3in
deleted file mode 100644
index 50449e90a6d..00000000000
--- a/oshmem/shmem/man/man3/shmem_float_get_nbi.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_getmem_nbi.3
diff --git a/oshmem/shmem/man/man3/shmem_float_iget.3in b/oshmem/shmem/man/man3/shmem_float_iget.3in
deleted file mode 100644
index df8770591af..00000000000
--- a/oshmem/shmem/man/man3/shmem_float_iget.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_iget.3
diff --git a/oshmem/shmem/man/man3/shmem_float_iput.3in b/oshmem/shmem/man/man3/shmem_float_iput.3in
deleted file mode 100644
index 2fdf7a97e3d..00000000000
--- a/oshmem/shmem/man/man3/shmem_float_iput.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_iput.3
diff --git a/oshmem/shmem/man/man3/shmem_float_max_to_all.3in b/oshmem/shmem/man/man3/shmem_float_max_to_all.3in
deleted file mode 100644
index 3e41c814889..00000000000
--- a/oshmem/shmem/man/man3/shmem_float_max_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_max_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_float_min_to_all.3in b/oshmem/shmem/man/man3/shmem_float_min_to_all.3in
deleted file mode 100644
index d688221529b..00000000000
--- a/oshmem/shmem/man/man3/shmem_float_min_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_min_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_float_p.3in b/oshmem/shmem/man/man3/shmem_float_p.3in
deleted file mode 100644
index 42b9fd0e81f..00000000000
--- a/oshmem/shmem/man/man3/shmem_float_p.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_p.3
diff --git a/oshmem/shmem/man/man3/shmem_float_prod_to_all.3in b/oshmem/shmem/man/man3/shmem_float_prod_to_all.3in
deleted file mode 100644
index 39b196d0820..00000000000
--- a/oshmem/shmem/man/man3/shmem_float_prod_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_prod_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_float_put.3in b/oshmem/shmem/man/man3/shmem_float_put.3in
deleted file mode 100644
index 9c7b2e25452..00000000000
--- a/oshmem/shmem/man/man3/shmem_float_put.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_put.3
diff --git a/oshmem/shmem/man/man3/shmem_float_put_nbi.3in b/oshmem/shmem/man/man3/shmem_float_put_nbi.3in
deleted file mode 100644
index fb4ad1413b0..00000000000
--- a/oshmem/shmem/man/man3/shmem_float_put_nbi.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_putmem_nbi.3
diff --git a/oshmem/shmem/man/man3/shmem_float_set.3in b/oshmem/shmem/man/man3/shmem_float_set.3in
deleted file mode 100644
index a02bb7ea5ea..00000000000
--- a/oshmem/shmem/man/man3/shmem_float_set.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_int_set.3
diff --git a/oshmem/shmem/man/man3/shmem_float_sum_to_all.3in b/oshmem/shmem/man/man3/shmem_float_sum_to_all.3in
deleted file mode 100644
index f75a4948417..00000000000
--- a/oshmem/shmem/man/man3/shmem_float_sum_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_sum_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_float_swap.3in b/oshmem/shmem/man/man3/shmem_float_swap.3in
deleted file mode 100644
index 31324f2c2e3..00000000000
--- a/oshmem/shmem/man/man3/shmem_float_swap.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_swap.3
diff --git a/oshmem/shmem/man/man3/shmem_free.3in b/oshmem/shmem/man/man3/shmem_free.3in
deleted file mode 100644
index 6cdb8014e9e..00000000000
--- a/oshmem/shmem/man/man3/shmem_free.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_malloc.3
diff --git a/oshmem/shmem/man/man3/shmem_get128.3in b/oshmem/shmem/man/man3/shmem_get128.3in
deleted file mode 100644
index 7cecf74d7ad..00000000000
--- a/oshmem/shmem/man/man3/shmem_get128.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_get.3
diff --git a/oshmem/shmem/man/man3/shmem_get128_nbi.3in b/oshmem/shmem/man/man3/shmem_get128_nbi.3in
deleted file mode 100644
index 50449e90a6d..00000000000
--- a/oshmem/shmem/man/man3/shmem_get128_nbi.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_getmem_nbi.3
diff --git a/oshmem/shmem/man/man3/shmem_get16_nbi.3in b/oshmem/shmem/man/man3/shmem_get16_nbi.3in
deleted file mode 100644
index 50449e90a6d..00000000000
--- a/oshmem/shmem/man/man3/shmem_get16_nbi.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_getmem_nbi.3
diff --git a/oshmem/shmem/man/man3/shmem_get32.3in b/oshmem/shmem/man/man3/shmem_get32.3in
deleted file mode 100644
index 7cecf74d7ad..00000000000
--- a/oshmem/shmem/man/man3/shmem_get32.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_get.3
diff --git a/oshmem/shmem/man/man3/shmem_get32_nbi.3in b/oshmem/shmem/man/man3/shmem_get32_nbi.3in
deleted file mode 100644
index 50449e90a6d..00000000000
--- a/oshmem/shmem/man/man3/shmem_get32_nbi.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_getmem_nbi.3
diff --git a/oshmem/shmem/man/man3/shmem_get64.3in b/oshmem/shmem/man/man3/shmem_get64.3in
deleted file mode 100644
index 7cecf74d7ad..00000000000
--- a/oshmem/shmem/man/man3/shmem_get64.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_get.3
diff --git a/oshmem/shmem/man/man3/shmem_get64_nbi.3in b/oshmem/shmem/man/man3/shmem_get64_nbi.3in
deleted file mode 100644
index 50449e90a6d..00000000000
--- a/oshmem/shmem/man/man3/shmem_get64_nbi.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_getmem_nbi.3
diff --git a/oshmem/shmem/man/man3/shmem_get8_nbi.3in b/oshmem/shmem/man/man3/shmem_get8_nbi.3in
deleted file mode 100644
index 50449e90a6d..00000000000
--- a/oshmem/shmem/man/man3/shmem_get8_nbi.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_getmem_nbi.3
diff --git a/oshmem/shmem/man/man3/shmem_getmem.3in b/oshmem/shmem/man/man3/shmem_getmem.3in
deleted file mode 100644
index 7cecf74d7ad..00000000000
--- a/oshmem/shmem/man/man3/shmem_getmem.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_get.3
diff --git a/oshmem/shmem/man/man3/shmem_getmem_nbi.3in b/oshmem/shmem/man/man3/shmem_getmem_nbi.3in
deleted file mode 100644
index 702e1db3de6..00000000000
--- a/oshmem/shmem/man/man3/shmem_getmem_nbi.3in
+++ /dev/null
@@ -1,168 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2016      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_GET\\_NBI" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmem_getmem_nbi\fP(3),
-\fIshmem_char_get_nbi\fP(3),
-\fIshmem_short_get_nbi\fP(3),
-\fIshmem_int_get_nbi\fP(3),
-\fIshmem_long_get_nbi\fP(3),
-\fIshmem_longlong_get_nbi\fP(3),
-\fIshmem_float_get_nbi\fP(3),
-\fIshmem_double_get_nbi\fP(3),
-\fIshmem_longdouble_get_nbi\fP(3),
-\fIshmem_get8_nbi\fP(3),
-\fIshmem_get16_nbi\fP(3),
-\fIshmem_get32_nbi\fP(3),
-\fIshmem_get64_nbi\fP(3),
-\fIshmem_get128_nbi\fP(3),
-\- The nonblocking get routines provide a method for copying data from a contiguous remote data object on the specified PE to the local data object.
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-void shmem_getmem_nbi(void *dest, const void *source,
-  size_t nelems, int pe);
-
-void shmem_char_get(char *dest, const char *source,
-  size_t nelems, int pe);
-
-void shmem_short_get(short *dest, const short *source,
-  size_t nelems, int pe);
-
-void shmem_int_get(int *dest, const int *source,
-  size_t nelems, int pe);
-
-void shmem_long_get(long *dest, const long *source,
-  size_t nelems, int pe);
-
-void shmem_longlong_get(long long *dest, const long long *source,
-  size_t nelems, int pe);
-
-void shmem_float_get(float *dest, const float *source,
-  size_t nelems, int pe);
-
-void shmem_double_get(double *dest, const double *source,
-  size_t nelems, int pe);
-
-void shmem_longdouble_get(long double *dest, const long double *source,
-  size_t nelems, int pe);
-
-void shmem_get8(void *dest, const void *source,
-  size_t nelems, int pe);
-
-void shmem_get16(void *dest, const void *source,
-  size_t nelems, int pe);
-
-void shmem_get32(void *dest, const void *source,
-  size_t nelems, int pe);
-
-void shmem_get64(void *dest, const void *source,
-  size_t nelems, int pe);
-
-void shmem_get128(void *dest, const void *source,
-  size_t nelems, int pe);
-
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER nelems, pe
-
-CALL SHMEM_GETMEM_NBI(dest, source, nelems, pe)
-
-CALL SHMEM_CHARACTER_GET_NBI(dest, source, nelems, pe)
-
-CALL SHMEM_COMPLEX_GET_NBI(dest, source, nelems, pe)
-
-CALL SHMEM_DOUBLE_GET_NBI(dest, source, nelems, pe)
-
-CALL SHMEM_INTEGER_GET_NBI(dest, source, nelems, pe)
-
-CALL SHMEM_LOGICAL_GET_NBI(dest, source, nelems, pe)
-
-CALL SHMEM_REAL_GET_NBI(dest, source, nelems, pe)
-
-CALL SHMEM_GET4_NBI(dest, source, nelems, pe)
-
-CALL SHMEM_GET8_NBI(dest, source, nelems, pe)
-
-CALL SHMEM_GET32_NBI(dest, source, nelems, pe)
-
-CALL SHMEM_GET64_NBI(dest, source, nelems, pe)
-
-CALL SHMEM_GET128_NBI(dest, source, nelems, pe)
-
-.Ve
-.SH DESCRIPTION
-
-The get routines provide a method for copying a contiguous symmetric data
-object from a different PE to a contiguous data object on the local PE.
-The routines return after posting the operation. The operation is
-considered complete after a subsequent call to shmem_quiet. At the completion
-of shmem_quiet, the data has been delivered to the dest array on the local PE.
-.PP
-The arguments are as follows:
-.TP
-dest
-Local data object to be updated.
-.TP
-source
-Data object on the PE identified by pe that contains the data to be copied. This
-data object must be remotely accessible.
-.TP
-nelems
-Number of elements in the target and source arrays. len must be of type integer. If
-you are using Fortran, it must be a constant, variable, or array element of default
-integer type.
-.TP
-pe
-PE number of the remote PE. pe must be of type integer. If you are using Fortran, it
-must be a constant, variable, or array element of default integer type.
-.PP
-If you are using Fortran, data types must be of default size. For example, a real variable must
-be declared as REAL, REAL*4, or REAL(KIND=4).
-.SH NOTES
-
-See \fIintro_shmem\fP(3)
-for a definition of the term remotely accessible.
-.SH EXAMPLES
-
-Consider this simple example for Fortran.
-.Vb
-PROGRAM REDUCTION
-  REAL VALUES, SUM
-  COMMON /C/ VALUES
-  REAL WORK
-
-  CALL START_PES(0) ! ALLOW ANY NUMBER OF PES
-  VALUES = MY_PE() ! INITIALIZE IT TO SOMETHING
-  CALL SHMEM_BARRIER_ALL
-  SUM = 0.0
-  DO I = 0,NUM_PES()\-1
-    CALL SHMEM_REAL_GET_NBI(WORK, VALUES, 1, I)
-    CALL SHMEM_QUIET                ! wait for delivery
-    SUM = SUM + WORK
-  ENDDO
-  PRINT *, 'PE ', MY_PE(), ' COMPUTED SUM=', SUM
-  CALL SHMEM_BARRIER_ALL
-END
-.Ve
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3),
-\fIshmem_quiet\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_global_exit.3in b/oshmem/shmem/man/man3/shmem_global_exit.3in
deleted file mode 100644
index 718fc63b9b5..00000000000
--- a/oshmem/shmem/man/man3/shmem_global_exit.3in
+++ /dev/null
@@ -1,47 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_GLOBAL\\_EXIT" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-shmem_global_exit
-\- A routine that allows any PE to force termination of an entire program.
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-void shmem_global_exit(int status);
-.Ve
-Fortran:
-.Vb
-include 'mpp/shmem.fh'
-INTEGER STATUS
-CALL SHMEM_GLOBAL_EXIT(status)
-.Ve
-.SH DESCRIPTION
-
-shmem_global_exit()
-shmem_global_exit is a non-collective routine that allows any one PE to force termination of an Open-
-SHMEM program for all PEs, passing an exit status to the execution environment. This routine terminates
-the entire program, not just the OpenSHMEM portion. When any PE calls shmem_global_exit, it results in
-the immediate notification to all PEs to terminate. shmem_global_exit flushes I/O and releases resources
-in accordance with C/C++/Fortran language requirements for normal program termination. If more than
-one PE calls shmem_global_exit, then the exit status returned to the environment shall be one of the values
-passed to shmem_global_exit as the status argument. There is no return to the caller of shmem_global_exit;
-control is returned from the OpenSHMEM program to the execution environment for all PEs.
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3),
-\fIshmem_my_pe\fP(3),
-\fIshmem_init\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_iget128.3in b/oshmem/shmem/man/man3/shmem_iget128.3in
deleted file mode 100644
index df8770591af..00000000000
--- a/oshmem/shmem/man/man3/shmem_iget128.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_iget.3
diff --git a/oshmem/shmem/man/man3/shmem_iget32.3in b/oshmem/shmem/man/man3/shmem_iget32.3in
deleted file mode 100644
index df8770591af..00000000000
--- a/oshmem/shmem/man/man3/shmem_iget32.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_iget.3
diff --git a/oshmem/shmem/man/man3/shmem_iget64.3in b/oshmem/shmem/man/man3/shmem_iget64.3in
deleted file mode 100644
index df8770591af..00000000000
--- a/oshmem/shmem/man/man3/shmem_iget64.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_iget.3
diff --git a/oshmem/shmem/man/man3/shmem_info_get_name.3in b/oshmem/shmem/man/man3/shmem_info_get_name.3in
deleted file mode 100644
index 36fd1da352d..00000000000
--- a/oshmem/shmem/man/man3/shmem_info_get_name.3in
+++ /dev/null
@@ -1,48 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_INFO\\_GET\\_NAME" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-shmem_info_get_name
-\- This routine returns the vendor defined character string.
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-void shmem_info_get_name(char *name);
-.Ve
-Fortran:
-.Vb
-include 'mpp/shmem.fh'
-SHMEM_INFO_GET_NAME(NAME)
-CHARACTER *(*)NAME
-.Ve
-.SH DESCRIPTION
-
-shmem_info_get_name()
-This routine returns the vendor defined character string of size defined by the constant SHMEM_MAX_NAME_LEN.
-The program calling this function prepares the memory of size SHMEM_MAX_NAME_LEN, and the implementation
-copies the string of size at most SHMEM_MAX_NAME_LEN. In C, the string is terminated
-by a null character. In Fortran, the string of size less than SHMEM_MAX_NAME_LEN is padded with
-blank characters up to size SHMEM_MAX_NAME_LEN. The implementation copying a string of size
-greater than SHMEM_MAX_NAME_LEN results in an undefined behavior. Multiple invocations of the
-routine in an OpenSHMEM program always return the same string. For a given library implementation,
-the major and minor version returned by these calls is consistent with the compile-time constants defined
-in its shmem.h.
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3),
-\fIshmem_my_pe\fP(3),
-\fIshmem_init\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_info_get_version.3in b/oshmem/shmem/man/man3/shmem_info_get_version.3in
deleted file mode 100644
index 7313c37567f..00000000000
--- a/oshmem/shmem/man/man3/shmem_info_get_version.3in
+++ /dev/null
@@ -1,43 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_INFO\\_GET\\_VERSION" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-shmem_info_get_version
-\- Returns the major and minor version of the library implementation.
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-void shmem_info_get_version(int *major, int *minor);
-.Ve
-Fortran:
-.Vb
-include 'mpp/shmem.fh'
-SHMEM_INFO_GET_VERSION(MAJOR, MINOR)
-INTEGER MAJOR, MINOR
-.Ve
-.SH DESCRIPTION
-
-shmem_info_get_version()
-This routine returns the major and minor version of the OpenSHMEM standard in use. For a given library
-implementation, the major and minor version returned by these calls is consistent with the compile-time
-constants, SHMEM_MAJOR_VERSION and SHMEM_MINOR_VERSION, defined in its shmem.h. The
-valid major version value is 1, and the valid minor version value is 2.
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3),
-\fIshmem_my_pe\fP(3),
-\fIshmem_init\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_init.3in b/oshmem/shmem/man/man3/shmem_init.3in
deleted file mode 100644
index 00d49c63182..00000000000
--- a/oshmem/shmem/man/man3/shmem_init.3in
+++ /dev/null
@@ -1,84 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_INIT" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-shmem_init, start_pes
-\- Allocates a block of memory from the symmetric heap.
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-void shmem_init(void);
-void start_pes(int npes);
-.Ve
-Fortran:
-.Vb
-CALL SHMEM_INIT()
-CALL START_PES(npes)
-.Ve
-.SH DESCRIPTION
-
-The start_pes routine should be the first statement in a SHMEM parallel program.
-.PP
-The start_pes routine accepts the following argument:
-.TP
-npes
-Unused. Should be set to 0.
-.PP
-This routine initializes the SHMEM API, therefore it must be called before calling any
-other SHMEM routine.
-This routine is responsible inter alia for setting up the symmetric heap on the calling PE, and
-the creation of the virtual PE numbers. Upon successful return from this routine, the calling PE
-will be able to communicate with and transfer data to other PEs.
-.PP
-Multiple calls to this function are not allowed.
-.PP
-For an overview of programming with SHMEM communication routines, example SHMEM
-programs, and instructions for compiling SHMEM programs, see the \fIintro_shmem\fP(3)
-man page.
-.SH EXAMPLES
-
-This is a simple program that calls \fIshmem_integer_put\fP(3):
-.Vb
-PROGRAM PUT
-  INCLUDE "mpp/shmem.fh"
-
-  INTEGER TARG, SRC, RECEIVER, BAR
-  COMMON /T/ TARG
-  PARAMETER (RECEIVER=1)
-
-  CALL SHMEM_INIT()
-  IF (MY_PE() .EQ. 0) THEN
-    SRC = 33
-    CALL SHMEM_INTEGER_PUT(TARG, SRC, 1, RECEIVER)
-  ENDIF
-  CALL SHMEM_BARRIER_ALL ! SYNCHRONIZES SENDER AND RECEIVER
-  IF (MY_PE() .EQ. RECEIVER) THEN
-    PRINT *,'PE ', MY_PE(),' TARG=',TARG,' (expect 33)'
-  ENDIF
-END
-.Ve
-.SH NOTES
-
-If the start_pes call is not the first statement in a program, unexpected results may occur on
-some architectures.
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3),
-\fIshmem_barrier\fP(3),
-\fIshmem_barrier_all\fP(3),
-\fIshmem_put\fP(3),
-\fImy_pe\fP(3I),
-\fIshmem_n_pes\fP(3I)
diff --git a/oshmem/shmem/man/man3/shmem_int_add.3in b/oshmem/shmem/man/man3/shmem_int_add.3in
deleted file mode 100644
index ff4728492a3..00000000000
--- a/oshmem/shmem/man/man3/shmem_int_add.3in
+++ /dev/null
@@ -1,76 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_ADD" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmem_int_add\fP(3),
-\fIshmem_int4_add\fP(3),
-\fIshmem_int8_add\fP(3),
-\fIshmem_long_add\fP(3),
-\fIshmem_longlong_add\fP(3)
-\- Performs an atomic add
-operation.
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-void shmem_int_add(int *target, int value, int pe);
-void shmem_long_add(long *target, long value, int pe);
-void shmem_longlong_add(long long *target, long long value,
-  int pe);
-.Ve
-Fortran:
-.Vb
-include 'mpp/shmem.h'
-
-INTEGER pe
-
-CALL SHMEM_INT4_ADD(target, value, pe)
-CALL SHMEM_INT8_ADD(target, value, pe)
-.Ve
-.SH DESCRIPTION
-
-The atomic add routines add \fBvalue\fP
-to the data at address \fBtarget\fP
-on PE
-\fBpe\fP\&.
-The operation completes without the possibility of another process updating
-target between the time of the fetch and the update.
-.PP
-The arguments are as follows:
-.TP
-target
-The remotely accessible integer data object to be updated on the remote PE. If
-you are using C/C++, the type of target should match that implied in the SYNOPSIS section. If
-you are using the Fortran compiler, it must be of type integer with an element size of 4 bytes
-for SHMEM_INT4_ADD and 8 bytes for SHMEM_INT8_ADD.
-.TP
-value
-The value to be atomically added to target. If you are using C/C++, the type of
-value should match that implied in the SYNOPSIS section. If you are using Fortran, it must be
-of type integer with an element size of target.
-.TP
-pe
-An integer that indicates the PE number upon which target is to be updated. If you
-are using Fortran, it must be a default integer value.
-.PP
-.SH NOTES
-
-The term remotely accessible is defined in \fIintro_shmem\fP(3)\&.
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3),
-\fIshmem_cache\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_int_and_to_all.3in b/oshmem/shmem/man/man3/shmem_int_and_to_all.3in
deleted file mode 100644
index 0a769b7c8a8..00000000000
--- a/oshmem/shmem/man/man3/shmem_int_and_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_and_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_int_cswap.3in b/oshmem/shmem/man/man3/shmem_int_cswap.3in
deleted file mode 100644
index 0f28e7712e2..00000000000
--- a/oshmem/shmem/man/man3/shmem_int_cswap.3in
+++ /dev/null
@@ -1,127 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_CSWAP" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmem_int_cswap\fP(3),
-\fIshmem_int4_cswap\fP(3),
-\fIshmem_int8_cswap\fP(3),
-\fIshmem_long_cswap\fP(3),
-\fIshmem_longlong_cswap\fP(3)
-\- Performs an atomic conditional swap to a remote data object
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-int shmem_int_cswap(int *target, int cond, int value, int pe);
-
-long shmem_long_cswap(long *target, long cond, long value,
-  int pe);
-
-long long shmem_longlong_cswap(longlong *target,
-  longlong cond, longlong value, int pe);
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER pe
-
-INTEGER(KIND=4) SHMEM_INT4_CSWAP
-ires = SHMEM_INT4_CSWAP(target, cond, value, pe)
-
-INTEGER(KIND=8) SHMEM_INT8_CSWAP
-ires = SHMEM_INT8_CSWAP(target, cond, value, pe)
-.Ve
-.SH DESCRIPTION
-
-The conditional swap routines conditionally update a target data object on an arbitrary
-processing element (PE) and return the prior contents of the data object in one atomic
-operation.
-.PP
-The arguments are as follows:
-.TP
-target
-The remotely accessible integer data object to be updated on the remote PE. If
-you are using C/C++, the type of target should match that implied in the SYNOPSIS section. If
-you are using the Fortran compiler, it must be of type integer with an element size of 4 bytes
-for SHMEM_INT4_ADD and 8 bytes for SHMEM_INT8_ADD.
-.TP
-value
-The value to be atomically added to target. If you are using C/C++, the type of
-value should match that implied in the SYNOPSIS section. If you are using Fortran, it must be
-of type integer with an element size of target.
-.TP
-pe
-An integer that indicates the PE number upon which target is to be updated. If you
-are using Fortran, it must be a default integer value.
-.TP
-target
-The remotely accessible integer data object to be updated on the remote PE. If
-you are using C/C++, the data type of target should match that implied in the SYNOPSIS
-section. If you are using Fortran, it must be of the following type:
-.RS
-.TP
-\fBSHMEM_INT4_CSWAP\fP: 4\-byte integer
-.TP
-\fBSHMEM_INT8_CSWAP\fP: 8\-byte integer
-.RE
-.RS
-.PP
-.RE
-.TP
-cond
-cond is compared to the remote target value. If cond and the remote target are
-equal, then value is swapped into the remote target. Otherwise, the remote target is
-unchanged. In either case, the old value of the remote target is returned as the function return
-value. cond must be of the same data type as target.
-.TP
-value
-The value to be atomically written to the remote PE. value must be the same data
-type as target.
-.TP
-pe
-An integer that indicates the PE number upon which target is to be updated. If you
-are using Fortran, it must be a default integer value.
-.PP
-.SH NOTES
-
-The term remotely accessible is defined in \fIintro_shmem\fP(3)\&.
-.SH RETURN VALUES
-
-The contents that had been in the target data object on the remote PE prior to the conditional
-swap.
-.SH EXAMPLES
-
-The following call ensures that the first PE to execute the conditional swap will successfully
-write its PE number to race_winner on PE 0.
-.Vb
-main()
-{
-  static int race_winner = \-1;
-  int oldval;
-
-  shmem_init();
-  oldval = shmem_int_cswap(&race_winner, \-1, shmem_my_pe(), 0);
-  if (oldval == \-1)
-    printf("pe %d was first\\n",shmem_my_pe());
-}
-.Ve
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3),
-\fIshmem_cache\fP(3),
-\fIshmem_swap\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_int_fadd.3in b/oshmem/shmem/man/man3/shmem_int_fadd.3in
deleted file mode 100644
index a1eb7269240..00000000000
--- a/oshmem/shmem/man/man3/shmem_int_fadd.3in
+++ /dev/null
@@ -1,79 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_FADD" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmem_int4_fadd\fP(3),
-\fIshmem_int8_fadd\fP(3),
-\fIshmem_int_fadd\fP(3),
-\fIshmem_long_fadd\fP(3),
-\fIshmem_longlong_fadd\fP(3)
-\- Performs an atomic fetch\-and\-add operation on a remote data object
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-int shmem_int_fadd(int *target, int value, int pe);
-
-long shmem_long_fadd(long *target, long value, int pe);
-
-long long shmem_longlong_fadd(long long *target, longlong value,
-  int pe);
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER pe
-
-INTEGER(KIND=4) SHMEM_INT4_FADD, ires, target, value
-ires = SHMEM_INT4_FADD(target, value, pe)
-
-INTEGER(KIND=8) SHMEM_INT8_FADD, ires, target, value
-ires = SHMEM_INT8_FADD(target, value, pe)
-.Ve
-.SH DESCRIPTION
-
-shmem_fadd functions perform an atomic fetch\-and\-add operation. An atomic
-fetch\-and\-add operation fetches the old target and adds value to target without the
-possibility of another process updating target between the time of the fetch and the update.
-These routines add value to target on Processing Element (PE) pe and return the previous
-contents of target as an atomic operation.
-.PP
-The arguments are as follows:
-.TP
-target
-The remotely accessible integer data object to be updated on the remote PE. The
-type of target should match that implied in the SYNOPSIS section.
-.TP
-value
-The value to be atomically added to target. The type of value should match that
-implied in the SYNOPSIS section.
-.TP
-pe
-An integer that indicates the PE number on which target is to be updated. If you are
-using Fortran, it must be a default integer value.
-.PP
-.SH NOTES
-
-The term remotely accessible is defined in \fIintro_shmem\fP(3)\&.
-.SH RETURN VALUES
-
-The contents that had been at the target address on the remote PE prior to the atomic addition
-operation.
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_int_fetch.3in b/oshmem/shmem/man/man3/shmem_int_fetch.3in
deleted file mode 100644
index e11e131eb69..00000000000
--- a/oshmem/shmem/man/man3/shmem_int_fetch.3in
+++ /dev/null
@@ -1,83 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2016      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_FETCH" "3" "Unreleased developer copy" "gitclone" "Open MPI"
-.SH NAME
-
-\fIshmem_int4_fetch\fP(3),
-\fIshmem_int8_fetch\fP(3),
-\fIshmem_int_fetch\fP(3),
-\fIshmem_long_fetch\fP(3),
-\fIshmem_longlong_fetch\fP(3)
-\fIshmem_double_fetch\fP(3)
-\fIshmem_float_fetch\fP(3)
-\- Atomically fetches the value of a remote data object
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-int shmem_int_fetch(int *target, int pe);
-
-long shmem_long_fetch(long *target, int pe);
-
-long long shmem_longlong_fetch(long long *target, int pe);
-
-double shmem_double_fetch(long long *target, int pe);
-
-float  shmem_float_fetch(float *target, int pe);
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER pe
-
-INTEGER(KIND=4) SHMEM_INT4_FETCH, ires, target
-ires = SHMEM_INT4_FETCH(target, pe)
-
-INTEGER(KIND=8) SHMEM_INT8_FETCH, ires, target
-ires = SHMEM_INT8_FETCH(target, pe)
-
-
-REAL(KIND=4) SHMEM_INT4_FETCH, ires, target
-ires = SHMEM_REAL4_FETCH(target, pe)
-
-REAL(KIND=8) SHMEM_INT8_FETCH, ires, target
-ires = SHMEM_REAL8_FETCH(target, pe)
-
-.Ve
-.SH DESCRIPTION
-
-The shmem_fetch functions perform an atomic fetch operation. They return the contents of the
-\fBtarget\fP as an atomic operation.
-
-.PP
-The arguments are as follows:
-.TP
-target
-The remotely accessible data object to be fetched from the remote PE.
-.TP
-pe
-An integer that indicates the PE number from which \fItarget\fP is to be fetched. If you are
-using Fortran, it must be a default integer value.
-.PP
-
-.SH RETURN VALUES
-The contents at the \fItarget\fP address on the remote PE. 
-The data type of the return value is the same as the the
-type of the remote data object.
-
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_int_finc.3in b/oshmem/shmem/man/man3/shmem_int_finc.3in
deleted file mode 100644
index 0f1808693e9..00000000000
--- a/oshmem/shmem/man/man3/shmem_int_finc.3in
+++ /dev/null
@@ -1,76 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_FINC" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmem_int4_finc\fP(3),
-\fIshmem_int8_finc\fP(3),
-\fIshmem_int_finc\fP(3),
-\fIshmem_long_finc\fP(3),
-\fIshmem_longlong_finc\fP(3)
-\- Performs an atomic fetch\-and\-increment operation on a remote data object
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-int shmem_int_finc(int *target, int pe);
-
-long shmem_long_finc(long *target, int pe);
-
-long long shmem_longlong_finc(long long *target, int pe);
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER pe
-INTEGER(KIND=4) SHMEM_INT4_FINC, target4
-INTEGER(KIND=8) SHMEM_INT8_FINC, target8
-
-ires4 = SHMEM_INT4_FINC(target4, pe)
-
-ires8 = SHMEM_INT8_FINC(target8, pe)
-.Ve
-.SH DESCRIPTION
-
-The fetch and increment routines retrieve the value at address \fBtarget\fP
-on PE
-\fBpe\fP,
-and update \fBtarget\fP
-with the result of incrementing the retrieved value by
-one. The operation must be completed without the possibility of another process updating
-\fBtarget\fP
-between the time of the fetch and the update.
-.PP
-The arguments are as follows:
-.TP
-target
-The remotely accessible integer data object to be updated on the remote PE. The
-type of target should match that implied in the SYNOPSIS section.
-.TP
-pe
-An integer that indicates the PE number upon which target is to be updated. If you
-are using Fortran, it must be a default integer value.
-.PP
-.SH NOTES
-
-The term remotely accessible is defined in \fIintro_shmem\fP(3)\&.
-.SH RETURN VALUES
-
-The contents that had been at the target address on the remote PE prior to the increment.
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_int_g.3in b/oshmem/shmem/man/man3/shmem_int_g.3in
deleted file mode 100644
index cd9ba27b1bc..00000000000
--- a/oshmem/shmem/man/man3/shmem_int_g.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_g.3
diff --git a/oshmem/shmem/man/man3/shmem_int_get.3in b/oshmem/shmem/man/man3/shmem_int_get.3in
deleted file mode 100644
index 7cecf74d7ad..00000000000
--- a/oshmem/shmem/man/man3/shmem_int_get.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_get.3
diff --git a/oshmem/shmem/man/man3/shmem_int_get_nbi.3in b/oshmem/shmem/man/man3/shmem_int_get_nbi.3in
deleted file mode 100644
index 50449e90a6d..00000000000
--- a/oshmem/shmem/man/man3/shmem_int_get_nbi.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_getmem_nbi.3
diff --git a/oshmem/shmem/man/man3/shmem_int_iget.3in b/oshmem/shmem/man/man3/shmem_int_iget.3in
deleted file mode 100644
index df8770591af..00000000000
--- a/oshmem/shmem/man/man3/shmem_int_iget.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_iget.3
diff --git a/oshmem/shmem/man/man3/shmem_int_inc.3in b/oshmem/shmem/man/man3/shmem_int_inc.3in
deleted file mode 100644
index 0a879d766d7..00000000000
--- a/oshmem/shmem/man/man3/shmem_int_inc.3in
+++ /dev/null
@@ -1,73 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_INC" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmem_int4_inc\fP(3),
-\fIshmem_int8_inc\fP(3),
-\fIshmem_int_inc\fP(3),
-\fIshmem_long_inc\fP(3),
-\fIshmem_longlong_inc\fP(3)
-\- These routines perform an atomic increment operation on a remote data object.
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-int shmem_int_inc(int *target, int pe);
-
-long shmem_long_inc(long *target, int pe);
-
-long long shmem_longlong_inc(long long *target, int pe);
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER pe
-INTEGER(KIND=4) SHMEM_INT4_INC, target4
-INTEGER(KIND=8) SHMEM_INT8_INC, target8
-
-ires4 = SHMEM_INT4_INC(target4, pe)
-
-ires8 = SHMEM_INT8_INC(target8, pe)
-.Ve
-.SH DESCRIPTION
-
-The atomic increment routines replace the value of \fBtarget\fP
-with its value incremented by
-one. The operation must be completed without the possibility of another process updating
-\fBtarget\fP
-between the time of the fetch and the update.
-.PP
-The arguments are as follows:
-.TP
-target
-The remotely accessible integer data object to be updated on the remote PE. The
-type of target should match that implied in the SYNOPSIS section.
-.TP
-pe
-An integer that indicates the PE number upon which target is to be updated. If you
-are using Fortran, it must be a default integer value.
-.PP
-.SH NOTES
-
-The term remotely accessible is defined in \fIintro_shmem\fP(3)\&.
-.SH RETURN VALUES
-
-None.
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_int_iput.3in b/oshmem/shmem/man/man3/shmem_int_iput.3in
deleted file mode 100644
index 2fdf7a97e3d..00000000000
--- a/oshmem/shmem/man/man3/shmem_int_iput.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_iput.3
diff --git a/oshmem/shmem/man/man3/shmem_int_max_to_all.3in b/oshmem/shmem/man/man3/shmem_int_max_to_all.3in
deleted file mode 100644
index 3e41c814889..00000000000
--- a/oshmem/shmem/man/man3/shmem_int_max_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_max_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_int_min_to_all.3in b/oshmem/shmem/man/man3/shmem_int_min_to_all.3in
deleted file mode 100644
index d688221529b..00000000000
--- a/oshmem/shmem/man/man3/shmem_int_min_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_min_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_int_or_to_all.3in b/oshmem/shmem/man/man3/shmem_int_or_to_all.3in
deleted file mode 100644
index 17b3fe22e3c..00000000000
--- a/oshmem/shmem/man/man3/shmem_int_or_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_or_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_int_p.3in b/oshmem/shmem/man/man3/shmem_int_p.3in
deleted file mode 100644
index 42b9fd0e81f..00000000000
--- a/oshmem/shmem/man/man3/shmem_int_p.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_p.3
diff --git a/oshmem/shmem/man/man3/shmem_int_prod_to_all.3in b/oshmem/shmem/man/man3/shmem_int_prod_to_all.3in
deleted file mode 100644
index 39b196d0820..00000000000
--- a/oshmem/shmem/man/man3/shmem_int_prod_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_prod_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_int_put.3in b/oshmem/shmem/man/man3/shmem_int_put.3in
deleted file mode 100644
index 9c7b2e25452..00000000000
--- a/oshmem/shmem/man/man3/shmem_int_put.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_put.3
diff --git a/oshmem/shmem/man/man3/shmem_int_put_nbi.3in b/oshmem/shmem/man/man3/shmem_int_put_nbi.3in
deleted file mode 100644
index fb4ad1413b0..00000000000
--- a/oshmem/shmem/man/man3/shmem_int_put_nbi.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_putmem_nbi.3
diff --git a/oshmem/shmem/man/man3/shmem_int_set.3in b/oshmem/shmem/man/man3/shmem_int_set.3in
deleted file mode 100644
index 7a68b2eee35..00000000000
--- a/oshmem/shmem/man/man3/shmem_int_set.3in
+++ /dev/null
@@ -1,77 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2016      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_SET" "3" "Unreleased developer copy" "gitclone" "Open MPI"
-.SH NAME
-
-\fIshmem_double_set\fP(3),
-\fIshmem_float_set\fP(3),
-\fIshmem_int_set\fP(3),
-\fIshmem_long_set\fP(3),
-\fIshmem_longlong_set\fP(3)
-\fIshmem_int4_set\fP(3),
-\fIshmem_int8_set\fP(3),
-\fIshmem_real4_set\fP(3),
-\fIshmem_real8_set\fP(3),
-\- Atomically sets the value of a remote data object
-
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-void shmem_double_set(double *target, double value, int pe);
-
-void shmem_float_set(float *target, float value, int pe);
-
-void shmem_int_set(int *target, int value, int pe);
-
-void shmem_long_set(long *target, long value, int pe);
-
-void shmem_longlong_set(long long *target, long long value, int pe);
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER pe
-
-CALL SHMEM_INT4_SET(target, value, pe)
-CALL SHMEM_INT8_SET(target, value, pe)
-CALL SHMEM_REAL4_SET(target, value, pe)
-CALL SHMEM_REAL8_SET(target, value, pe)
-
-.Ve
-.SH DESCRIPTION
-
-The set routines write the \fBvalue\fP into the address \fBtarget\fP on \fBpe\fP as an atomic operation.
-.PP
-The arguments are as follows:
-.TP
-target
-The remotely accessible data object to be set on the remote PE. 
-.TP
-value
-The value to be atomically written to the remote PE. 
-.TP
-pe
-An integer that indicates the PE number upon which target is to be updated. If you
-are using Fortran, it must be a default integer value.
-.PP
-.SH RETURN VALUES
-NONE
-
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3)
-
diff --git a/oshmem/shmem/man/man3/shmem_int_sum_to_all.3in b/oshmem/shmem/man/man3/shmem_int_sum_to_all.3in
deleted file mode 100644
index f75a4948417..00000000000
--- a/oshmem/shmem/man/man3/shmem_int_sum_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_sum_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_int_swap.3in b/oshmem/shmem/man/man3/shmem_int_swap.3in
deleted file mode 100644
index 31324f2c2e3..00000000000
--- a/oshmem/shmem/man/man3/shmem_int_swap.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_swap.3
diff --git a/oshmem/shmem/man/man3/shmem_int_wait.3in b/oshmem/shmem/man/man3/shmem_int_wait.3in
deleted file mode 100644
index 03267ffbc55..00000000000
--- a/oshmem/shmem/man/man3/shmem_int_wait.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_wait.3
diff --git a/oshmem/shmem/man/man3/shmem_int_wait_until.3in b/oshmem/shmem/man/man3/shmem_int_wait_until.3in
deleted file mode 100644
index 03267ffbc55..00000000000
--- a/oshmem/shmem/man/man3/shmem_int_wait_until.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_wait.3
diff --git a/oshmem/shmem/man/man3/shmem_int_xor_to_all.3in b/oshmem/shmem/man/man3/shmem_int_xor_to_all.3in
deleted file mode 100644
index bda77ecf1d1..00000000000
--- a/oshmem/shmem/man/man3/shmem_int_xor_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_xor_all.3
diff --git a/oshmem/shmem/man/man3/shmem_iput128.3in b/oshmem/shmem/man/man3/shmem_iput128.3in
deleted file mode 100644
index 2fdf7a97e3d..00000000000
--- a/oshmem/shmem/man/man3/shmem_iput128.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_iput.3
diff --git a/oshmem/shmem/man/man3/shmem_iput32.3in b/oshmem/shmem/man/man3/shmem_iput32.3in
deleted file mode 100644
index 2fdf7a97e3d..00000000000
--- a/oshmem/shmem/man/man3/shmem_iput32.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_iput.3
diff --git a/oshmem/shmem/man/man3/shmem_iput64.3in b/oshmem/shmem/man/man3/shmem_iput64.3in
deleted file mode 100644
index 2fdf7a97e3d..00000000000
--- a/oshmem/shmem/man/man3/shmem_iput64.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_iput.3
diff --git a/oshmem/shmem/man/man3/shmem_long_add.3in b/oshmem/shmem/man/man3/shmem_long_add.3in
deleted file mode 100644
index 8606d1aa608..00000000000
--- a/oshmem/shmem/man/man3/shmem_long_add.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_int_add.3
diff --git a/oshmem/shmem/man/man3/shmem_long_and_to_all.3in b/oshmem/shmem/man/man3/shmem_long_and_to_all.3in
deleted file mode 100644
index 0a769b7c8a8..00000000000
--- a/oshmem/shmem/man/man3/shmem_long_and_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_and_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_long_cswap.3in b/oshmem/shmem/man/man3/shmem_long_cswap.3in
deleted file mode 100644
index cf2a78fe882..00000000000
--- a/oshmem/shmem/man/man3/shmem_long_cswap.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_int_cswap.3
diff --git a/oshmem/shmem/man/man3/shmem_long_fadd.3in b/oshmem/shmem/man/man3/shmem_long_fadd.3in
deleted file mode 100644
index d3bc2141cad..00000000000
--- a/oshmem/shmem/man/man3/shmem_long_fadd.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_int_fadd.3
diff --git a/oshmem/shmem/man/man3/shmem_long_fetch.3in b/oshmem/shmem/man/man3/shmem_long_fetch.3in
deleted file mode 100644
index 7213c75e538..00000000000
--- a/oshmem/shmem/man/man3/shmem_long_fetch.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_int_fetch.3
diff --git a/oshmem/shmem/man/man3/shmem_long_finc.3in b/oshmem/shmem/man/man3/shmem_long_finc.3in
deleted file mode 100644
index fc800da8ff8..00000000000
--- a/oshmem/shmem/man/man3/shmem_long_finc.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_int_finc.3
diff --git a/oshmem/shmem/man/man3/shmem_long_g.3in b/oshmem/shmem/man/man3/shmem_long_g.3in
deleted file mode 100644
index cd9ba27b1bc..00000000000
--- a/oshmem/shmem/man/man3/shmem_long_g.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_g.3
diff --git a/oshmem/shmem/man/man3/shmem_long_get.3in b/oshmem/shmem/man/man3/shmem_long_get.3in
deleted file mode 100644
index 7cecf74d7ad..00000000000
--- a/oshmem/shmem/man/man3/shmem_long_get.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_get.3
diff --git a/oshmem/shmem/man/man3/shmem_long_get_nbi.3in b/oshmem/shmem/man/man3/shmem_long_get_nbi.3in
deleted file mode 100644
index 50449e90a6d..00000000000
--- a/oshmem/shmem/man/man3/shmem_long_get_nbi.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_getmem_nbi.3
diff --git a/oshmem/shmem/man/man3/shmem_long_iget.3in b/oshmem/shmem/man/man3/shmem_long_iget.3in
deleted file mode 100644
index df8770591af..00000000000
--- a/oshmem/shmem/man/man3/shmem_long_iget.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_iget.3
diff --git a/oshmem/shmem/man/man3/shmem_long_inc.3in b/oshmem/shmem/man/man3/shmem_long_inc.3in
deleted file mode 100644
index 96ae7ddd34c..00000000000
--- a/oshmem/shmem/man/man3/shmem_long_inc.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_int_inc.3
diff --git a/oshmem/shmem/man/man3/shmem_long_iput.3in b/oshmem/shmem/man/man3/shmem_long_iput.3in
deleted file mode 100644
index 2fdf7a97e3d..00000000000
--- a/oshmem/shmem/man/man3/shmem_long_iput.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_iput.3
diff --git a/oshmem/shmem/man/man3/shmem_long_max_to_all.3in b/oshmem/shmem/man/man3/shmem_long_max_to_all.3in
deleted file mode 100644
index 3e41c814889..00000000000
--- a/oshmem/shmem/man/man3/shmem_long_max_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_max_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_long_min_to_all.3in b/oshmem/shmem/man/man3/shmem_long_min_to_all.3in
deleted file mode 100644
index d688221529b..00000000000
--- a/oshmem/shmem/man/man3/shmem_long_min_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_min_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_long_or_to_all.3in b/oshmem/shmem/man/man3/shmem_long_or_to_all.3in
deleted file mode 100644
index 17b3fe22e3c..00000000000
--- a/oshmem/shmem/man/man3/shmem_long_or_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_or_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_long_p.3in b/oshmem/shmem/man/man3/shmem_long_p.3in
deleted file mode 100644
index 42b9fd0e81f..00000000000
--- a/oshmem/shmem/man/man3/shmem_long_p.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_p.3
diff --git a/oshmem/shmem/man/man3/shmem_long_prod_to_all.3in b/oshmem/shmem/man/man3/shmem_long_prod_to_all.3in
deleted file mode 100644
index 39b196d0820..00000000000
--- a/oshmem/shmem/man/man3/shmem_long_prod_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_prod_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_long_put.3in b/oshmem/shmem/man/man3/shmem_long_put.3in
deleted file mode 100644
index 9c7b2e25452..00000000000
--- a/oshmem/shmem/man/man3/shmem_long_put.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_put.3
diff --git a/oshmem/shmem/man/man3/shmem_long_put_nbi.3in b/oshmem/shmem/man/man3/shmem_long_put_nbi.3in
deleted file mode 100644
index fb4ad1413b0..00000000000
--- a/oshmem/shmem/man/man3/shmem_long_put_nbi.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_putmem_nbi.3
diff --git a/oshmem/shmem/man/man3/shmem_long_set.3in b/oshmem/shmem/man/man3/shmem_long_set.3in
deleted file mode 100644
index a02bb7ea5ea..00000000000
--- a/oshmem/shmem/man/man3/shmem_long_set.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_int_set.3
diff --git a/oshmem/shmem/man/man3/shmem_long_sum_to_all.3in b/oshmem/shmem/man/man3/shmem_long_sum_to_all.3in
deleted file mode 100644
index f75a4948417..00000000000
--- a/oshmem/shmem/man/man3/shmem_long_sum_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_sum_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_long_swap.3in b/oshmem/shmem/man/man3/shmem_long_swap.3in
deleted file mode 100644
index 31324f2c2e3..00000000000
--- a/oshmem/shmem/man/man3/shmem_long_swap.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_swap.3
diff --git a/oshmem/shmem/man/man3/shmem_long_wait.3in b/oshmem/shmem/man/man3/shmem_long_wait.3in
deleted file mode 100644
index 03267ffbc55..00000000000
--- a/oshmem/shmem/man/man3/shmem_long_wait.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_wait.3
diff --git a/oshmem/shmem/man/man3/shmem_long_wait_until.3in b/oshmem/shmem/man/man3/shmem_long_wait_until.3in
deleted file mode 100644
index 03267ffbc55..00000000000
--- a/oshmem/shmem/man/man3/shmem_long_wait_until.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_wait.3
diff --git a/oshmem/shmem/man/man3/shmem_long_xor_to_all.3in b/oshmem/shmem/man/man3/shmem_long_xor_to_all.3in
deleted file mode 100644
index bda77ecf1d1..00000000000
--- a/oshmem/shmem/man/man3/shmem_long_xor_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_xor_all.3
diff --git a/oshmem/shmem/man/man3/shmem_longdouble_g.3in b/oshmem/shmem/man/man3/shmem_longdouble_g.3in
deleted file mode 100644
index cd9ba27b1bc..00000000000
--- a/oshmem/shmem/man/man3/shmem_longdouble_g.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_g.3
diff --git a/oshmem/shmem/man/man3/shmem_longdouble_get.3in b/oshmem/shmem/man/man3/shmem_longdouble_get.3in
deleted file mode 100644
index 7cecf74d7ad..00000000000
--- a/oshmem/shmem/man/man3/shmem_longdouble_get.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_get.3
diff --git a/oshmem/shmem/man/man3/shmem_longdouble_get_nbi.3in b/oshmem/shmem/man/man3/shmem_longdouble_get_nbi.3in
deleted file mode 100644
index 50449e90a6d..00000000000
--- a/oshmem/shmem/man/man3/shmem_longdouble_get_nbi.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_getmem_nbi.3
diff --git a/oshmem/shmem/man/man3/shmem_longdouble_iget.3in b/oshmem/shmem/man/man3/shmem_longdouble_iget.3in
deleted file mode 100644
index df8770591af..00000000000
--- a/oshmem/shmem/man/man3/shmem_longdouble_iget.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_iget.3
diff --git a/oshmem/shmem/man/man3/shmem_longdouble_iput.3in b/oshmem/shmem/man/man3/shmem_longdouble_iput.3in
deleted file mode 100644
index 2fdf7a97e3d..00000000000
--- a/oshmem/shmem/man/man3/shmem_longdouble_iput.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_iput.3
diff --git a/oshmem/shmem/man/man3/shmem_longdouble_max_to_all.3in b/oshmem/shmem/man/man3/shmem_longdouble_max_to_all.3in
deleted file mode 100644
index 3e41c814889..00000000000
--- a/oshmem/shmem/man/man3/shmem_longdouble_max_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_max_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_longdouble_min_to_all.3in b/oshmem/shmem/man/man3/shmem_longdouble_min_to_all.3in
deleted file mode 100644
index d688221529b..00000000000
--- a/oshmem/shmem/man/man3/shmem_longdouble_min_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_min_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_longdouble_p.3in b/oshmem/shmem/man/man3/shmem_longdouble_p.3in
deleted file mode 100644
index 42b9fd0e81f..00000000000
--- a/oshmem/shmem/man/man3/shmem_longdouble_p.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_p.3
diff --git a/oshmem/shmem/man/man3/shmem_longdouble_prod_to_all.3in b/oshmem/shmem/man/man3/shmem_longdouble_prod_to_all.3in
deleted file mode 100644
index 39b196d0820..00000000000
--- a/oshmem/shmem/man/man3/shmem_longdouble_prod_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_prod_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_longdouble_put.3in b/oshmem/shmem/man/man3/shmem_longdouble_put.3in
deleted file mode 100644
index 9c7b2e25452..00000000000
--- a/oshmem/shmem/man/man3/shmem_longdouble_put.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_put.3
diff --git a/oshmem/shmem/man/man3/shmem_longdouble_put_nbi.3in b/oshmem/shmem/man/man3/shmem_longdouble_put_nbi.3in
deleted file mode 100644
index fb4ad1413b0..00000000000
--- a/oshmem/shmem/man/man3/shmem_longdouble_put_nbi.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_putmem_nbi.3
diff --git a/oshmem/shmem/man/man3/shmem_longdouble_sum_to_all.3in b/oshmem/shmem/man/man3/shmem_longdouble_sum_to_all.3in
deleted file mode 100644
index f75a4948417..00000000000
--- a/oshmem/shmem/man/man3/shmem_longdouble_sum_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_sum_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_longlong_add.3in b/oshmem/shmem/man/man3/shmem_longlong_add.3in
deleted file mode 100644
index 8606d1aa608..00000000000
--- a/oshmem/shmem/man/man3/shmem_longlong_add.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_int_add.3
diff --git a/oshmem/shmem/man/man3/shmem_longlong_and_to_all.3in b/oshmem/shmem/man/man3/shmem_longlong_and_to_all.3in
deleted file mode 100644
index 0a769b7c8a8..00000000000
--- a/oshmem/shmem/man/man3/shmem_longlong_and_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_and_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_longlong_cswap.3in b/oshmem/shmem/man/man3/shmem_longlong_cswap.3in
deleted file mode 100644
index cf2a78fe882..00000000000
--- a/oshmem/shmem/man/man3/shmem_longlong_cswap.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_int_cswap.3
diff --git a/oshmem/shmem/man/man3/shmem_longlong_fadd.3in b/oshmem/shmem/man/man3/shmem_longlong_fadd.3in
deleted file mode 100644
index d3bc2141cad..00000000000
--- a/oshmem/shmem/man/man3/shmem_longlong_fadd.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_int_fadd.3
diff --git a/oshmem/shmem/man/man3/shmem_longlong_fetch.3in b/oshmem/shmem/man/man3/shmem_longlong_fetch.3in
deleted file mode 100644
index 7213c75e538..00000000000
--- a/oshmem/shmem/man/man3/shmem_longlong_fetch.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_int_fetch.3
diff --git a/oshmem/shmem/man/man3/shmem_longlong_finc.3in b/oshmem/shmem/man/man3/shmem_longlong_finc.3in
deleted file mode 100644
index fc800da8ff8..00000000000
--- a/oshmem/shmem/man/man3/shmem_longlong_finc.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_int_finc.3
diff --git a/oshmem/shmem/man/man3/shmem_longlong_g.3in b/oshmem/shmem/man/man3/shmem_longlong_g.3in
deleted file mode 100644
index cd9ba27b1bc..00000000000
--- a/oshmem/shmem/man/man3/shmem_longlong_g.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_g.3
diff --git a/oshmem/shmem/man/man3/shmem_longlong_get.3in b/oshmem/shmem/man/man3/shmem_longlong_get.3in
deleted file mode 100644
index 7cecf74d7ad..00000000000
--- a/oshmem/shmem/man/man3/shmem_longlong_get.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_get.3
diff --git a/oshmem/shmem/man/man3/shmem_longlong_get_nbi.3in b/oshmem/shmem/man/man3/shmem_longlong_get_nbi.3in
deleted file mode 100644
index 50449e90a6d..00000000000
--- a/oshmem/shmem/man/man3/shmem_longlong_get_nbi.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_getmem_nbi.3
diff --git a/oshmem/shmem/man/man3/shmem_longlong_iget.3in b/oshmem/shmem/man/man3/shmem_longlong_iget.3in
deleted file mode 100644
index df8770591af..00000000000
--- a/oshmem/shmem/man/man3/shmem_longlong_iget.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_iget.3
diff --git a/oshmem/shmem/man/man3/shmem_longlong_inc.3in b/oshmem/shmem/man/man3/shmem_longlong_inc.3in
deleted file mode 100644
index 96ae7ddd34c..00000000000
--- a/oshmem/shmem/man/man3/shmem_longlong_inc.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_int_inc.3
diff --git a/oshmem/shmem/man/man3/shmem_longlong_iput.3in b/oshmem/shmem/man/man3/shmem_longlong_iput.3in
deleted file mode 100644
index 2fdf7a97e3d..00000000000
--- a/oshmem/shmem/man/man3/shmem_longlong_iput.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_iput.3
diff --git a/oshmem/shmem/man/man3/shmem_longlong_max_to_all.3in b/oshmem/shmem/man/man3/shmem_longlong_max_to_all.3in
deleted file mode 100644
index 3e41c814889..00000000000
--- a/oshmem/shmem/man/man3/shmem_longlong_max_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_max_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_longlong_min_to_all.3in b/oshmem/shmem/man/man3/shmem_longlong_min_to_all.3in
deleted file mode 100644
index d688221529b..00000000000
--- a/oshmem/shmem/man/man3/shmem_longlong_min_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_min_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_longlong_or_to_all.3in b/oshmem/shmem/man/man3/shmem_longlong_or_to_all.3in
deleted file mode 100644
index 17b3fe22e3c..00000000000
--- a/oshmem/shmem/man/man3/shmem_longlong_or_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_or_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_longlong_p.3in b/oshmem/shmem/man/man3/shmem_longlong_p.3in
deleted file mode 100644
index 42b9fd0e81f..00000000000
--- a/oshmem/shmem/man/man3/shmem_longlong_p.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_p.3
diff --git a/oshmem/shmem/man/man3/shmem_longlong_prod_to_all.3in b/oshmem/shmem/man/man3/shmem_longlong_prod_to_all.3in
deleted file mode 100644
index 39b196d0820..00000000000
--- a/oshmem/shmem/man/man3/shmem_longlong_prod_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_prod_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_longlong_put.3in b/oshmem/shmem/man/man3/shmem_longlong_put.3in
deleted file mode 100644
index 9c7b2e25452..00000000000
--- a/oshmem/shmem/man/man3/shmem_longlong_put.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_put.3
diff --git a/oshmem/shmem/man/man3/shmem_longlong_put_nbi.3in b/oshmem/shmem/man/man3/shmem_longlong_put_nbi.3in
deleted file mode 100644
index fb4ad1413b0..00000000000
--- a/oshmem/shmem/man/man3/shmem_longlong_put_nbi.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_putmem_nbi.3
diff --git a/oshmem/shmem/man/man3/shmem_longlong_set.3in b/oshmem/shmem/man/man3/shmem_longlong_set.3in
deleted file mode 100644
index a02bb7ea5ea..00000000000
--- a/oshmem/shmem/man/man3/shmem_longlong_set.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_int_set.3
diff --git a/oshmem/shmem/man/man3/shmem_longlong_sum_to_all.3in b/oshmem/shmem/man/man3/shmem_longlong_sum_to_all.3in
deleted file mode 100644
index f75a4948417..00000000000
--- a/oshmem/shmem/man/man3/shmem_longlong_sum_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_sum_to_all.3
diff --git a/oshmem/shmem/man/man3/shmem_longlong_swap.3in b/oshmem/shmem/man/man3/shmem_longlong_swap.3in
deleted file mode 100644
index 31324f2c2e3..00000000000
--- a/oshmem/shmem/man/man3/shmem_longlong_swap.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_swap.3
diff --git a/oshmem/shmem/man/man3/shmem_longlong_wait.3in b/oshmem/shmem/man/man3/shmem_longlong_wait.3in
deleted file mode 100644
index 03267ffbc55..00000000000
--- a/oshmem/shmem/man/man3/shmem_longlong_wait.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_wait.3
diff --git a/oshmem/shmem/man/man3/shmem_longlong_wait_until.3in b/oshmem/shmem/man/man3/shmem_longlong_wait_until.3in
deleted file mode 100644
index 03267ffbc55..00000000000
--- a/oshmem/shmem/man/man3/shmem_longlong_wait_until.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_wait.3
diff --git a/oshmem/shmem/man/man3/shmem_longlong_xor_to_all.3in b/oshmem/shmem/man/man3/shmem_longlong_xor_to_all.3in
deleted file mode 100644
index bda77ecf1d1..00000000000
--- a/oshmem/shmem/man/man3/shmem_longlong_xor_to_all.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_short_xor_all.3
diff --git a/oshmem/shmem/man/man3/shmem_malloc.3in b/oshmem/shmem/man/man3/shmem_malloc.3in
deleted file mode 100644
index c7491e563e9..00000000000
--- a/oshmem/shmem/man/man3/shmem_malloc.3in
+++ /dev/null
@@ -1,113 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMALLOC" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmem_malloc\fP(3),
-\fIshmem_free\fP(3),
-\fIshmem_align\fP(3),
-\fIshmem_realloc\fP(3)
-\fIshmalloc\fP(3),
-\fIshfree\fP(3),
-\fIshmemalign\fP(3),
-\fIshrealloc\fP(3)
-\- Symmetric heap memory management functions.
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-void *shmem_malloc(size_t size);
-void *shmalloc(size_t size);
-
-void shmem_free(void *ptr);
-void shfree(void *ptr);
-
-void *shmem_realloc(void *ptr, size_t size);
-void *shrealloc(void *ptr, size_t size);
-
-void *shmem_align(size_t alignment, size_t size);
-void *shmemalign(size_t alignment, size_t size);
-
-extern long malloc_error;
-.Ve
-.SH DESCRIPTION
-
-The \fBshmem_malloc\fP
-function returns a pointer to a block of at least size bytes
-suitably aligned for any use. This space is allocated from the symmetric heap (in contrast
-to \fImalloc\fP(3C),
-which allocates from the private heap).
-.PP
-The \fBshmem_align\fP
-function allocates a block in the symmetric heap that has a
-byte alignment specified by the alignment argument.
-.PP
-The \fBshmem_free\fP
-function causes the block to which ptr points to, to be deallocated,
-that is, made available for further allocation. If ptr is a null pointer, no action
-occurs; otherwise, if the argument does not match a pointer earlier returned by a symmetric
-heap function, or if the space has already been deallocated, malloc_error is set to indicate the
-error, and shfree returns.
-.PP
-The \fBshmem_realloc\fP
-function changes the size of the block to which ptr points to, to the
-size (in bytes) specified by size.
-.PP
-The contents of the block are unchanged up to the lesser of the new and old sizes. If the new
-size is larger, the value of the newly allocated portion of the block is indeterminate. If ptr is a
-null pointer, the shrealloc function behaves like the shmalloc function for the specified size. If
-size is 0 and ptr is not a null pointer, the block to which it points to is freed. Otherwise, if ptr
-does not match a pointer earlier returned by a symmetric heap function, or if the space has
-already been deallocated, the malloc_error variable is set to indicate the error, and shrealloc
-returns a null pointer. If the space cannot be allocated, the block to which ptr points to is
-unchanged.
-.PP
-The shmem_malloc, shmem_free, and shmem_realloc functions are provided so that multiple PEs in an
-application can allocate symmetric, remotely accessible memory blocks. These memory
-blocks can then be used with (shmem) communication routines. Each of these functions call
-the \fIshmem_barrier_all\fP(3)
-function before returning; this ensures that all PEs
-participate in the memory allocation, and that the memory on other PEs can be used as soon
-as the local PE returns.
-.PP
-The user is responsible for calling these functions with identical argument(s) on all PEs; if
-differing size arguments are used, subsequent calls may not return the same symmetric heap
-address on all PEs.
-.PP
-.SH NOTES
-
-The total size of the symmetric heap is determined at job startup. One can adjust the size of
-the heap using the SHMEM_SYMMETRIC_HEAP_SIZE environment variable. See the
-\fIintro_shmem\fP(3)
-man page for futher details.
-The shmem_malloc, shmem_free, and shmem_realloc functions differ from the private heap allocation functions
-in that all PEs in an application must call them (a barrier is used to ensure this).
-.PP
-.SH RETURN VALUES
-
-The \fBshmem_malloc\fP
-function returns a pointer to the allocated space (which should
-be identical on all PEs); otherwise, it returns a null pointer (with malloc_error set).
-The \fBshmem_free\fP
-function returns no value.
-The \fBshmem_realloc\fP
-function returns a pointer to the allocated space (which
-may have moved); otherwise, it returns a null pointer (with malloc_error set).
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3),
-\fIshmem_my_pe\fP(3I),
-\fIshmem_init\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_my_pe.3in b/oshmem/shmem/man/man3/shmem_my_pe.3in
deleted file mode 100644
index cd92e13a688..00000000000
--- a/oshmem/shmem/man/man3/shmem_my_pe.3in
+++ /dev/null
@@ -1,41 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_MY\\_PE" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-shmem_my_pe, my_pe, _my_pe \- Returns the virtual PE number of the calling PE.
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-int shmem_my_pe (void);
-int my_pe (void);
-.Ve
-Fortran:
-.Vb
-include 'mpp/shmem.fh'
-I = SHMEM_MY_PE ()
-I = MY_PE ()
-.Ve
-.SH DESCRIPTION
-
-my_pe() or shmem_my_pe() return the processing element (PE) number of the calling PE. It accepts no
-arguments. The result is an integer between 0 and npes \- 1, where npes is the total
-number of PEs executing the current program.
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3),
-\fIshmem_n_pes\fP(3),
-\fIshmem_init\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_n_pes.3in b/oshmem/shmem/man/man3/shmem_n_pes.3in
deleted file mode 100644
index 83d7c309125..00000000000
--- a/oshmem/shmem/man/man3/shmem_n_pes.3in
+++ /dev/null
@@ -1,39 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_N\\_PES" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-num_pes, _num_pes, shmem_n_pes \- Returns the number of processing elements (PEs) used to run the application.
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-int _num_pes (void);
-int shmem_n_pes (void);
-.Ve
-Fortran:
-.Vb
-include 'mpp/shmem.fh'
-I = NUM_PES ()
-I = SHMEM_N_PES ()
-.Ve
-.SH DESCRIPTION
-
-num_pes() or shmem_n_pes() return the total number of PEs running in an application.
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3),
-\fIshmem_my_pe\fP(3),
-\fIshmem_init\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_pe_accessible.3in b/oshmem/shmem/man/man3/shmem_pe_accessible.3in
deleted file mode 100644
index 66b38b9224b..00000000000
--- a/oshmem/shmem/man/man3/shmem_pe_accessible.3in
+++ /dev/null
@@ -1,54 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_PE\\_ACCESSIBLE" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-shmem_pe_accessible \- Determines whether a processing element (PE) is accessible via
-SHMEM data transfer operations.
-.SH SYNOPSIS
-
-C:
-.Vb
-#include <mpp/shmem.h>
-
-int shmem_pe_accessible(int pe);
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-LOGICAL LOG, SHMEM_PE_ACCESSIBLE
-INTEGER pe
-
-LOG = SHMEM_PE_ACCESSIBLE(pe)
-.Ve
-.SH DESCRIPTION
-
-shmem_pe_accessible returns a value that indicates whether the calling PE is able to perform
-OpenSHMEM communication operations with the remote PE.
-.SH RETURN VALUES
-
-.TP
-C/C++
-The return value is 1 if the specified PE is a valid remote PE for SHMEM functions;
-otherwise,it is 0.
-.TP
-Fortran
-The return value is \&.TRUE. if the specified PE is a valid remote PE for SHMEM
-functions; otherwise, it is \&.FALSE..
-.PP
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3),
-\fIshmem_addr_accessible\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_ptr.3in b/oshmem/shmem/man/man3/shmem_ptr.3in
deleted file mode 100644
index 6c6240ef110..00000000000
--- a/oshmem/shmem/man/man3/shmem_ptr.3in
+++ /dev/null
@@ -1,129 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_PTR" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmem_ptr\fP(3)
-\- Returns a pointer to a data object on a specified processing element
-(PE).
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-void *shmem_ptr(const void *target, int pe);
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-POINTER (PTR, POINTEE)
-INTEGER pe
-
-PTR = SHMEM_PTR(target, pe)
-.Ve
-.SH DESCRIPTION
-
-The shmem_ptr routine returns an address that can be used to directly reference
-\fBtarget\fP
-on the remote PE \fBpe\fP\&.
-With this address we can perform ordinary loads
-and stores to the remote address.
-.PP
-When a sequence of loads (gets) and stores (puts) to a data object on a remote PE does not
-match the access pattern provided in a SHMEM data transfer routine like
-\fIshmem_put32\fP(3)
-or \fIshmem_real_iget\fP(3),
-the shmem_ptr function can
-provide an efficient means to accomplish the communication.
-.PP
-The arguments are as follows:
-.TP
-target
-The symmetric data object to be referenced.
-.TP
-pe
-An integer that indicates the PE number on which target is to be accessed. If you
-are using Fortran, it must be a default integer value.
-.PP
-.SH EXAMPLES
-
-This Fortran program calls shmem_ptr and then PE 0 writes to the BIGD array on PE 1:
-.Vb
-PROGRAM REMOTEWRITE
-  INCLUDE 'mpp/shmem.fh'
-
-  INTEGER BIGD(100)
-  SAVE BIGD
-  INTEGER POINTEE(*)
-
-  POINTER (PTR,POINTEE)
-  CALL START_PES(0)
-  IF (MY_PE() .EQ. 0) THEN
-                             ! initialize PE 1's BIGD array
-    PTR = SHMEM_PTR(BIGD, 1) ! get address of PE 1's BIGD
-                             ! array
-    DO I=1,100
-      POINTEE(I) = I
-    ENDDO
-  ENDIF
-  CALL SHMEM_BARRIER_ALL
-  IF (MY_PE() .EQ. 1) THEN
-    PRINT *, 'BIGD on PE 1 is: '
-    PRINT *, BIGD
-  ENDIF
-END
-.Ve
-This is the equivalent program written in C:
-.Vb
-#include <mpp/shmem.h>
-main()
-{
-  static int bigd[100];
-  int *ptr;
-  int i;
-
-  shmem_init();
-  if (shmem_my_pe() == 0) {
-  /* initialize PE 1's bigd array */
-    ptr = shmem_ptr(bigd, 1);
-    for (i=0; i<100; i++)
-      *ptr++ = i+1;
-  }
-  shmem_barrier_all();
-  if (shmem_my_pe() == 1) {
-    printf("bigd on PE 1 is:\\n");
-    for (i=0; i<100; i++)
-      printf(" %d\\n",bigd[i]);
-    printf("\\n");
-  }
-}
-.Ve
-.SH NOTES
-
-The shmem_ptr function is available only on systems where ordinary memory loads and
-stores are used to implement SHMEM put and get operations.
-.PP
-.SH RETURN VALUES
-
-shmem_ptr returns a pointer to the data object on the specified remote PE. If target is not
-remotely accessible, a NULL pointer is returned.
-.PP
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3),
-\fIshmem_put\fP(3),
-\fIshmem_get\fP(3)
-.PP
diff --git a/oshmem/shmem/man/man3/shmem_put128.3in b/oshmem/shmem/man/man3/shmem_put128.3in
deleted file mode 100644
index 9c7b2e25452..00000000000
--- a/oshmem/shmem/man/man3/shmem_put128.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_put.3
diff --git a/oshmem/shmem/man/man3/shmem_put128_nbi.3in b/oshmem/shmem/man/man3/shmem_put128_nbi.3in
deleted file mode 100644
index fb4ad1413b0..00000000000
--- a/oshmem/shmem/man/man3/shmem_put128_nbi.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_putmem_nbi.3
diff --git a/oshmem/shmem/man/man3/shmem_put16_nbi.3in b/oshmem/shmem/man/man3/shmem_put16_nbi.3in
deleted file mode 100644
index fb4ad1413b0..00000000000
--- a/oshmem/shmem/man/man3/shmem_put16_nbi.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_putmem_nbi.3
diff --git a/oshmem/shmem/man/man3/shmem_put32.3in b/oshmem/shmem/man/man3/shmem_put32.3in
deleted file mode 100644
index 9c7b2e25452..00000000000
--- a/oshmem/shmem/man/man3/shmem_put32.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_put.3
diff --git a/oshmem/shmem/man/man3/shmem_put32_nbi.3in b/oshmem/shmem/man/man3/shmem_put32_nbi.3in
deleted file mode 100644
index fb4ad1413b0..00000000000
--- a/oshmem/shmem/man/man3/shmem_put32_nbi.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_putmem_nbi.3
diff --git a/oshmem/shmem/man/man3/shmem_put64.3in b/oshmem/shmem/man/man3/shmem_put64.3in
deleted file mode 100644
index 9c7b2e25452..00000000000
--- a/oshmem/shmem/man/man3/shmem_put64.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_put.3
diff --git a/oshmem/shmem/man/man3/shmem_put64_nbi.3in b/oshmem/shmem/man/man3/shmem_put64_nbi.3in
deleted file mode 100644
index fb4ad1413b0..00000000000
--- a/oshmem/shmem/man/man3/shmem_put64_nbi.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_putmem_nbi.3
diff --git a/oshmem/shmem/man/man3/shmem_put8_nbi.3in b/oshmem/shmem/man/man3/shmem_put8_nbi.3in
deleted file mode 100644
index fb4ad1413b0..00000000000
--- a/oshmem/shmem/man/man3/shmem_put8_nbi.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_putmem_nbi.3
diff --git a/oshmem/shmem/man/man3/shmem_putmem.3in b/oshmem/shmem/man/man3/shmem_putmem.3in
deleted file mode 100644
index 9c7b2e25452..00000000000
--- a/oshmem/shmem/man/man3/shmem_putmem.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_put.3
diff --git a/oshmem/shmem/man/man3/shmem_putmem_nbi.3in b/oshmem/shmem/man/man3/shmem_putmem_nbi.3in
deleted file mode 100644
index f987e670c7d..00000000000
--- a/oshmem/shmem/man/man3/shmem_putmem_nbi.3in
+++ /dev/null
@@ -1,171 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2016      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_PUT\\_NBI" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmem_putmem_nbi\fP(3),
-\fIshmem_char_put_nbi\fP(3),
-\fIshmem_short_put_nbi\fP(3),
-\fIshmem_int_put_nbi\fP(3),
-\fIshmem_long_put_nbi\fP(3),
-\fIshmem_longlong_put_nbi\fP(3),
-\fIshmem_float_put_nbi\fP(3),
-\fIshmem_double_put_nbi\fP(3),
-\fIshmem_longdouble_put_nbi\fP(3),
-\fIshmem_put8_nbi\fP(3),
-\fIshmem_put16_nbi\fP(3),
-\fIshmem_put32_nbi\fP(3),
-\fIshmem_put64_nbi\fP(3),
-\fIshmem_put128_nbi\fP(3),
-\- The nonblocking put routines provide a method for copying data from a contiguous local data object to a data object on a specified PE.
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-void shmem_putmem_nbi(void *dest, const void *source,
-  size_t nelems, int pe);
-
-void shmem_char_put(char *dest, const char *source,
-  size_t nelems, int pe);
-
-void shmem_short_put(short *dest, const short *source,
-  size_t nelems, int pe);
-
-void shmem_int_put(int *dest, const int *source,
-  size_t nelems, int pe);
-
-void shmem_long_put(long *dest, const long *source,
-  size_t nelems, int pe);
-
-void shmem_longlong_put(long long *dest, const long long *source,
-  size_t nelems, int pe);
-
-void shmem_float_put(float *dest, const float *source,
-  size_t nelems, int pe);
-
-void shmem_double_put(double *dest, const double *source,
-  size_t nelems, int pe);
-
-void shmem_longdouble_put(long double *dest, const long double *source,
-  size_t nelems, int pe);
-
-void shmem_put8(void *dest, const void *source,
-  size_t nelems, int pe);
-
-void shmem_put16(void *dest, const void *source,
-  size_t nelems, int pe);
-
-void shmem_put32(void *dest, const void *source,
-  size_t nelems, int pe);
-
-void shmem_put64(void *dest, const void *source,
-  size_t nelems, int pe);
-
-void shmem_put128(void *dest, const void *source,
-  size_t nelems, int pe);
-
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER nelems, pe
-
-CALL SHMEM_PUTMEM_NBI(dest, source, nelems, pe)
-
-CALL SHMEM_CHARACTER_PUT_NBI(dest, source, nelems, pe)
-
-CALL SHMEM_COMPLEX_PUT_NBI(dest, source, nelems, pe)
-
-CALL SHMEM_DOUBLE_PUT_NBI(dest, source, nelems, pe)
-
-CALL SHMEM_INTEGER_PUT_NBI(dest, source, nelems, pe)
-
-CALL SHMEM_LOGICAL_PUT_NBI(dest, source, nelems, pe)
-
-CALL SHMEM_REAL_PUT_NBI(dest, source, nelems, pe)
-
-CALL SHMEM_PUT4_NBI(dest, source, nelems, pe)
-
-CALL SHMEM_PUT8_NBI(dest, source, nelems, pe)
-
-CALL SHMEM_PUT32_NBI(dest, source, nelems, pe)
-
-CALL SHMEM_PUT64_NBI(dest, source, nelems, pe)
-
-CALL SHMEM_PUT128_NBI(dest, source, nelems, pe)
-
-.Ve
-.SH DESCRIPTION
-
-The routines return after posting the operation. The operation is considered
-complete after a subsequent call to shmem_quiet. At the completion of shmem_quiet,
-the data has been copied into the dest array on the destination PE.
-The delivery of data words into the data object on the destination PE may occur
-in any order. Furthermore, two successive put routines may deliver data out of
-order unless a call to shmem_fence is introduced between the two calls.
-.PP
-The arguments are as follows:
-.TP
-dest
-Data object to be updated on the remote PE. This data object must be
-remotely accessible.
-.TP
-source
-Data object containing the data to be copied.
-.TP
-nelems
-Number of elements in the dest and source arrays. nelems must be
-of type size_t for C. If you are using Fortran, it must be a constant,
-variable, or array element of default integer type.
-.TP
-pe
-PE number of the remote PE. pe must be of type integer. If you are using Fortran, it
-must be a constant, variable, or array element of default integer type.
-.PP
-If you are using Fortran, data types must be of default size. For example, a real variable must
-be declared as REAL, REAL*4, or REAL(KIND=4).
-.SH NOTES
-
-See \fIintro_shmem\fP(3)
-for a definition of the term remotely accessible.
-.SH EXAMPLES
-
-Consider this simple example for Fortran.
-.Vb
-#include <stdio.h>
-#include <mpp/shmem.h>
-
-main()
-{
-  long source[10] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
-  static long target[10];
-  shmem_init();
-
-  if (shmem_my_pe() == 0) {
-    /* put 10 words into target on PE 1 */
-    shmem_long_put_nbi(target, source, 10, 1);
-    shmem_quiet();
-  }
-  shmem_barrier_all();  /* sync sender and receiver */
-  if (shmem_my_pe() == 1)
-    shmem_udcflush();  /* not required on Altix systems */
-  printf("target[0] on PE %d is %d\\n", shmem_my_pe(), target[0]);
-}
-.Ve
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3),
-\fIshmem_quiet\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_quiet.3in b/oshmem/shmem/man/man3/shmem_quiet.3in
deleted file mode 100644
index cd92ae17944..00000000000
--- a/oshmem/shmem/man/man3/shmem_quiet.3in
+++ /dev/null
@@ -1,84 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_QUIET" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmem_quiet\fP(3)
-\- Waits for completion of all outstanding remote writes issued by a
-processing element (PE).
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-void shmem_quiet(void);
-.Ve
-Fortran:
-.Vb
-CALL SHMEM_QUIET
-.Ve
-.SH DESCRIPTION
-
-shmem_quiet ensures ordering of put (remote write) operations. All put operations issued to
-any processing element (PE) prior to the call to shmem_quiet are guaranteed to be visible to
-all other PEs no later than any subsequent memory load or store, remote put or get, or
-synchronization operations that follow the call to shmem_quiet.
-.SH NOTES
-
-shmem_quiet is most useful as a way of ensuring ordering of delivery of several put
-operations. For example, you might use shmem_quiet to await delivery of a block of data
-before issuing another put, which sets a completion flag on another PE.
-.br
-shmem_quiet is not usually needed if \fIshmem_barrier_all\fP(3)
-or
-\fIshmem_barrier\fP(3)
-are called. The barrier routines all wait for the completion of
-outstanding remote writes (puts).
-.SH EXAMPLES
-
-.Vb
-PROGRAM COMPFLAG
-  INCLUDE "mpp/shmem.fh"
-
-  INTEGER FLAG_VAR, ARRAY(100), RECEIVER, SENDER
-  COMMON/FLAG/FLAG_VAR
-  COMMON/DATA/ARRAY
-  INTRINSIC MY_PE
-
-  FLAG_VAR = 0
-  CALL SHMEM_BARRIER_ALL ! wait for FLAG_VAR to be initialized
-  SENDER = 0                        ! PE 0 sends the data
-  RECEIVER = 1                      ! PE 1 receives the data
-
-  IF (MY_PE() .EQ. 0) THEN
-    ARRAY = 33
-    CALL SHMEM_PUT(ARRAY, ARRAY, 100, RECEIVER) ! start sending data
-    CALL SHMEM_QUIET                ! wait for delivery
-    CALL SHMEM_PUT(FLAG_VAR, 1, 1, RECEIVER) ! send completion flag
-  ELSE IF (MY_PE() .EQ. RECEIVER) THEN
-    CALL SHMEM_UDCFLUSH
-    CALL SHMEM_WAIT(FLAG_VAR, 0)
-    PRINT *,ARRAY                       ! ARRAY has been delivered
-  ENDIF
-END
-.Ve
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3),
-\fIshmem_barrier\fP(3),
-\fIshmem_barrier_all\fP(3),
-\fIshmem_fence\fP(3),
-\fIshmem_put\fP(3),
-\fIshmem_wait\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_realloc.3in b/oshmem/shmem/man/man3/shmem_realloc.3in
deleted file mode 100644
index 6cdb8014e9e..00000000000
--- a/oshmem/shmem/man/man3/shmem_realloc.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_malloc.3
diff --git a/oshmem/shmem/man/man3/shmem_set_cache_inv.3in b/oshmem/shmem/man/man3/shmem_set_cache_inv.3in
deleted file mode 100644
index 4a6a361ef97..00000000000
--- a/oshmem/shmem/man/man3/shmem_set_cache_inv.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_udcflush.3
diff --git a/oshmem/shmem/man/man3/shmem_set_cache_line_inv.3in b/oshmem/shmem/man/man3/shmem_set_cache_line_inv.3in
deleted file mode 100644
index 4a6a361ef97..00000000000
--- a/oshmem/shmem/man/man3/shmem_set_cache_line_inv.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_udcflush.3
diff --git a/oshmem/shmem/man/man3/shmem_set_lock.3in b/oshmem/shmem/man/man3/shmem_set_lock.3in
deleted file mode 100644
index d3b57361ea3..00000000000
--- a/oshmem/shmem/man/man3/shmem_set_lock.3in
+++ /dev/null
@@ -1,78 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_LOCK" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmem_set_lock\fP(3),
-\fIshmem_clear_lock\fP(3),
-\fIshmem_test_lock\fP(3)
-\- Releases, locks, and tests a mutual exclusion memory lock.
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-void shmem_clear_lock(volatile long *lock);
-
-void shmem_set_lock(volatile long *lock);
-
-int shmem_test_lock(volatile long *lock);
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER lock, SHMEM_TEST_LOCK
-
-CALL SHMEM_CLEAR_LOCK(lock)
-
-CALL SHMEM_SET_LOCK(lock)
-
-I = SHMEM_TEST_LOCK(lock)
-.Ve
-.SH DESCRIPTION
-
-The shmem_set_lock routine sets a mutual exclusion lock after waiting for the lock to be
-freed by any other PE currently holding the lock. Waiting PEs are assured of getting the lock
-in a first\-come, first\-served manner.
-.PP
-The shmem_clear_lock routine releases a lock previously set by shmem_set_lock after
-ensuring that all local and remote stores initiated in the critical region are complete.
-.PP
-The shmem_test_lock function sets a mutual exclusion lock only if it is currently cleared.
-By using this function, a PE can avoid blocking on a set lock. If the lock is currently set, the
-function returns without waiting.
-These routines are appropriate for protecting a critical region from simultaneous update by
-multiple PEs.
-They accept the following arguments:
-.TP
-lock
-A symmetric data object that is a scalar variable or an array of length 1. This
-data object must be set to 0 on all processing elements (PEs) prior to the first use. lock must
-be of type integer. If you are using Fortran, it must be of default kind.
-.PP
-.SH NOTES
-
-The term symmetric data object is defined on \fIintro_shmem\fP(3)\&.
-.PP
-.SH RETURN VALUES
-
-The shmem_test_lock function returns 0 if the lock was originally cleared and this call
-was able to set the lock. A value of 1 is returned if the lock had been set and the call returned
-without waiting to set the lock.
-.PP
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_short_and_to_all.3in b/oshmem/shmem/man/man3/shmem_short_and_to_all.3in
deleted file mode 100644
index 065ee57fdc3..00000000000
--- a/oshmem/shmem/man/man3/shmem_short_and_to_all.3in
+++ /dev/null
@@ -1,206 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2014-2016 Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_AND" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmem_int_and_to_all\fP(3),
-\fIshmem_int4_and_to_all\fP(3),
-\fIshmem_int8_and_to_all\fP(3),
-\fIshmem_long_and_to_all\fP(3),
-\fIshmem_longlong_and_to_all\fP(3),
-\fIshmem_short_and_to_all\fP(3)
-\- Performs a bitwise AND operation on symmetric
-arrays over the active set of PEs.
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-void shmem_int_and_to_all(int *target, const int *source,
-  int nreduce, int PE_start, int logPE_stride, int PE_size,
-  int *pWrk, long *pSync);
-
-void shmem_long_and_to_all(long *target, const long *source,
-  int nreduce, int PE_start, int  logPE_stride, int PE_size,
-  long *pWrk, long *pSync);
-
-void shmem_longlong_and_to_all(long long *target,
-  const long long *source, int nreduce, int PE_start, int logPE_stride,
-  int PE_size, long long *pWrk, long *pSync);
-
-void shmem_short_and_to_all(short *target, const short *source,
-  int nreduce, int PE_start, int logPE_stride, int PE_size,
-  short *pWrk, long *pSync);
-
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER pSync(SHMEM_REDUCE_SYNC_SIZE)
-INTEGER nreduce, PE_start, logPE_stride, PE_size
-
-CALL SHMEM_INT4_AND_TO_ALL(target, source, nreduce,
-& PE_start, logPE_stride, PE_size, pWrk, pSync)
-
-CALL SHMEM_INT8_AND_TO_ALL(target, source, nreduce,
-& PE_start, logPE_stride, PE_size, pWrk, pSync)
-.Ve
-.PP
-.SH DESCRIPTION
-
-The shared memory (SHMEM) reduction routines compute one or more reductions across
-symmetric arrays on multiple virtual PEs. A reduction performs an associative binary
-operation across a set of values. For a list of other SHMEM reduction routines, see
-\fIintro_shmem\fP(3)\&.
-.PP
-The nreduce argument determines the number of separate reductions to perform. The source
-array on all PEs in the active set provides one element for each reduction. The results of the
-reductions are placed in the target array on all PEs in the active set. The active set is defined
-by the PE_start, logPE_stride, PE_size triplet.
-.PP
-The source and target arrays may be the same array, but they may not be overlapping arrays.
-As with all SHMEM collective routines, each of these routines assumes that only PEs in the
-active set call the routine. If a PE not in the active set calls a SHMEM collective routine,
-undefined behavior results.
-.PP
-The arguments are as follows:
-.TP
-target
-A symmetric array, of length nreduce elements, to receive the result of the
-reduction operations. The data type of target varies with the version of the reduction routine
-being called. When calling from C/C++, refer to the SYNOPSIS section for data type
-information. When calling from Fortran, the target data types are as follows:
-.RS
-.TP
-\fBshmem_int8_and_to_all\fP: Integer, with an element size of 8 bytes
-.TP
-\fBshmem_int4_and_to_all\fP: Integer, with an element size of 4 bytes
-.RE
-.RS
-.PP
-.RE
-.TP
-source
-A symmetric array, of length nreduce elements, that contains one element for
-each separate reduction operation. The source argument must have the same data type as
-target.
-.TP
-nreduce
-The number of elements in the target and source arrays. nreduce must be of
-type integer. If you are using Fortran, it must be a default integer value.
-.TP
-PE_start
-The lowest virtual PE number of the active set of PEs. PE_start must be of
-type integer. If you are using Fortran, it must be a default integer value.
-.TP
-logPE_stride
-The log (base 2) of the stride between consecutive virtual PE numbers in
-the active set. logPE_stride must be of type integer. If you are using Fortran, it must be a
-default integer value.
-.TP
-PE_size
-The number of PEs in the active set. PE_size must be of type integer. If you
-are using Fortran, it must be a default integer value.
-.TP
-pWrk
-A symmetric work array. The pWrk argument must have the same data type as
-target. In C/C++, this contains max(nreduce/2 + 1,
-_SHMEM_REDUCE_MIN_WRKDATA_SIZE) elements. In Fortran, this contains
-max(nreduce/2 + 1, SHMEM_REDUCE_MIN_WRKDATA_SIZE) elements.
-.TP
-pSync
-A symmetric work array. In C/C++, pSync must be of type long and size
-_SHMEM_REDUCE_SYNC_SIZE. In Fortran, pSync must be of type integer and size
-SHMEM_REDUCE_SYNC_SIZE. If you are using Fortran, it must be a default integer value.
-Every element of this array must be initialized with the value _SHMEM_SYNC_VALUE (in
-C/C++) or SHMEM_SYNC_VALUE (in Fortran) before any of the PEs in the active set enter
-the reduction routine.
-.PP
-The values of arguments nreduce, PE_start, logPE_stride, and PE_size must be equal on all
-PEs in the active set. The same target and source arrays, and the same pWrk and pSync work
-arrays, must be passed to all PEs in the active set.
-.PP
-Before any PE calls a reduction routine, you must ensure that the following conditions exist
-(synchronization via a barrier or some other method is often needed to ensure this): The
-pWrk and pSync arrays on all PEs in the active set are not still in use from a prior call to a
-collective SHMEM routine. The target array on all PEs in the active set is ready to accept the
-results of the reduction.
-.PP
-Upon return from a reduction routine, the following are true for the local PE: The target array
-is updated. The values in the pSync array are restored to the original values.
-.PP
-.SH NOTES
-
-The terms collective, symmetric, and cache aligned are defined in \fIintro_shmem\fP(3)\&.
-All SHMEM reduction routines reset the values in pSync before they return, so a particular
-pSync buffer need only be initialized the first time it is used.
-.PP
-You must ensure that the pSync array is not being updated on any PE in the active set while
-any of the PEs participate in processing of a SHMEM reduction routine. Be careful to avoid the
-following situations: If the pSync array is initialized at run time, some type of
-synchronization is needed to ensure that all PEs in the working set have initialized pSync
-before any of them enter a SHMEM routine called with the pSync synchronization array. A
-pSync or pWrk array can be reused in a subsequent reduction routine call only if none of the
-PEs in the active set are still processing a prior reduction routine call that used the same
-pSync or pWrk arrays. In general, this can be assured only by doing some type of
-synchronization. However, in the special case of reduction routines being called with the
-same active set, you can allocate two pSync and pWrk arrays and alternate between them on
-successive calls.
-.PP
-.SH EXAMPLES
-
-\fBExample 1\fP:
-This Fortran example statically initializes the pSync array and finds the logical AND of the integer variable FOO across all even PEs.
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER PSYNC(SHMEM_REDUCE_SYNC_SIZE)
-DATA PSYNC /SHMEM_REDUCE_SYNC_SIZE*SHMEM_SYNC_VALUE/
-PARAMETER (NR=1)
-REAL PWRK(MAX(NR/2+1, SHMEM_REDUCE_MIN_WRKDATA_SIZE))
-INTEGER FOO, FOOAND
-COMMON /COM/ FOO, FOOAND, PWRK
-INTRINSIC MY_PE
-
-IF ( MOD(MY_PE(),2) .EQ. 0) THEN
-  CALL SHMEM_INT8_AND_TO_ALL(FOOAND, FOO, NR, 0, 1, N$PES/2,
-  & PWRK, PSYNC)
-  PRINT *, 'Result on PE ', MY_PE(), ' is ', FOOAND
-ENDIF
-.Ve
-\fBExample 2\fP:
-Consider the following C call:
-.Vb
-shmem_int_and_to_all( target, source, 3, 0, 0, 8, pwrk, psync );
-.Ve
-The preceding call is more efficient, but semantically equivalent to, the combination of the
-following calls:
-.Vb
-shmem_int_and_to_all(&(target[0]), &(source[0]), 1, 0, 0, 8,
-  pwrk1, psync1);
-
-shmem_int_and_to_all(&(target[1]), &(source[1]), 1, 0, 0, 8,
-  pwrk2, psync2);
-
-shmem_int_and_to_all(&(target[2]), &(source[2]), 1, 0, 0, 8,
-  pwrk1, psync1);
-.Ve
-Note that two sets of pWrk and pSync arrays are used alternately because no
-synchronization is done between calls.
-.SH SEE ALSO
-
-\fIf90\fP(1),
-\fIintro_shmem\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_short_g.3in b/oshmem/shmem/man/man3/shmem_short_g.3in
deleted file mode 100644
index cd9ba27b1bc..00000000000
--- a/oshmem/shmem/man/man3/shmem_short_g.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_g.3
diff --git a/oshmem/shmem/man/man3/shmem_short_get.3in b/oshmem/shmem/man/man3/shmem_short_get.3in
deleted file mode 100644
index 7cecf74d7ad..00000000000
--- a/oshmem/shmem/man/man3/shmem_short_get.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_get.3
diff --git a/oshmem/shmem/man/man3/shmem_short_get_nbi.3in b/oshmem/shmem/man/man3/shmem_short_get_nbi.3in
deleted file mode 100644
index 50449e90a6d..00000000000
--- a/oshmem/shmem/man/man3/shmem_short_get_nbi.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_getmem_nbi.3
diff --git a/oshmem/shmem/man/man3/shmem_short_iget.3in b/oshmem/shmem/man/man3/shmem_short_iget.3in
deleted file mode 100644
index 5538cd707b7..00000000000
--- a/oshmem/shmem/man/man3/shmem_short_iget.3in
+++ /dev/null
@@ -1,217 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_IGET" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmem_complex_iget\fP(3),
-\fIshmem_double_iget\fP(3),
-\fIshmem_float_iget\fP(3),
-\fIshmem_iget4\fP(3),
-\fIshmem_iget8\fP(3),
-\fIshmem_iget32\fP(3),
-\fIshmem_iget64\fP(3),
-\fIshmem_iget128\fP(3),
-\fIshmem_int_iget\fP(3),
-\fIshmem_integer_iget\fP(3),
-\fIshmem_logical_iget\fP(3),
-\fIshmem_long_iget\fP(3),
-\fIshmem_longdouble_iget\fP(3),
-\fIshmem_longlong_iget\fP(3),
-\fIshmem_real_iget\fP(3),
-\fIshmem_short_iget\fP(3)
-\- Transfers strided data from a specified processing element (PE)
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-void shmem_iget32(void *target, const void *source,
-  ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe);
-
-void shmem_iget64(void *target, const void *source,
-  ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe);
-
-void shmem_iget128(void *target, const void *source,
-  ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe);
-
-void shmem_int_iget(int *target, const int *source,
-  ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe);
-
-void shmem_double_iget(double *target, const double *source,
-  ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe);
-
-void shmem_float_iget(float *target, const float *source,
-  ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe);
-
-void shmem_long_iget(long *target, const long *source,
-  ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe);
-
-void shmem_longdouble_iget(long double *target,
-  const long double *source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
-
-void shmem_longlong_iget(long long *target,
-  const long long *source, ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe);
-
-void shmem_short_iget(short *target,
-  const short *source, ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe);
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER tst, sst, len, pe
-
-CALL SHMEM_COMPLEX_IGET(target, source, tst, sst, len,
-& pe)
-
-CALL SHMEM_DOUBLE_IGET(target, source, tst, sst, len,
-& pe)
-
-CALL SHMEM_IGET4(target, source, tst, sst, len, pe)
-
-CALL SHMEM_IGET8(target, source, tst, sst, len, pe)
-
-CALL SHMEM_IGET32(target, source, tst, sst, len, pe)
-
-CALL SHMEM_IGET64(target, source, tst, sst, len, pe)
-
-CALL SHMEM_IGET128(target, source, tst, sst, len, pe)
-
-CALL SHMEM_INTEGER_IGET(target, source, tst, sst, len,
-& pe)
-
-CALL SHMEM_LOGICAL_IGET(target, source, tst, sst, len,
-& pe)
-
-CALL SHMEM_REAL_IGET(target, source, tst, sst, len, pe)
-.Ve
-.SH DESCRIPTION
-
-The strided get routines retrieve array data available at address source on remote PE (pe).
-The elements of the \fBsource\fP
-array are separated by a stride \fBsst\fP\&.
-Once the data is received,
-it is stored at the local memory address \fBtarget\fP,
-separated by stride \fBtst\fP\&.
-The routines return
-when the data has been copied into the local \fBtarget\fP
-array.
-.PP
-The arguments are as follows:
-.TP
-target
-Array to be updated on the local PE.
-.TP
-source
-Array containing the data to be copied on the remote PE.
-.TP
-tst
-The stride between consecutive elements of the target array. The stride is scaled by
-the element size of the target array. A value of 1 indicates contiguous data. tst must be of
-type integer. If you are calling from Fortran, it must be a default integer value.
-.TP
-sst
-The stride between consecutive elements of the source array. The stride is scaled
-by the element size of the source array. A value of 1 indicates contiguous data. sst must be
-of type integer. If you are calling from Fortran, it must be a default integer value.
-.TP
-len
-Number of elements in the target and source arrays. len must be of type integer. If
-you are using Fortran, it must be a constant, variable, or array element of default integer
-type.
-.TP
-pe
-PE number of the remote PE. pe must be of type integer. If you are using Fortran, it
-must be a constant, variable, or array element of default integer type.
-.PP
-The target and source data objects must conform to typing constraints, which are as
-follows:
-.TP
-\fBshmem_iget32, shmem_iget4\fP: Any noncharacter type that has a storage size
-equal to 32 bits.
-.TP
-\fBshmem_iget64, shmem_iget8\fP: Any noncharacter type that has a storage size
-equal to 64 bits.
-.TP
-\fBshmem_iget128\fP: Any noncharacter type that has a storage size equal to
-128 bits.
-.TP
-\fBshmem_short_iget\fP: Elements of type short.
-.TP
-\fBshmem_int_iget\fP: Elements of type int.
-.TP
-\fBshmem_long_iget\fP: Elements of type long.
-.TP
-\fBshmem_longlong_iget\fP: Elements of type long long.
-.TP
-\fBshmem_float_iget\fP: Elements of type float.
-.TP
-\fBshmem_double_iget\fP: Elements of type double.
-.TP
-\fBshmem_longdouble_iget\fP: Elements of type long double.
-.TP
-\fBSHMEM_COMPLEX_IGET\fP: Elements of type complex of default size.
-.TP
-\fBSHMEM_DOUBLE_IGET\fP: (Fortran) Elements of type double precision.
-.TP
-\fBSHMEM_INTEGER_IGET\fP: Elements of type integer.
-.TP
-\fBSHMEM_LOGICAL_IGET\fP: Elements of type logical.
-.TP
-\fBSHMEM_REAL_IGET\fP: Elements of type real.
-.TP
-\fBshmem_longdouble_iget\fP: Elements of type long double.
-.TP
-\fBSHMEM_COMPLEX_IGET\fP: Elements of type complex of default size.
-.TP
-\fBSHMEM_DOUBLE_IGET\fP: (Fortran) Elements of type double precision.
-.TP
-\fBSHMEM_INTEGER_IGET\fP: Elements of type integer.
-.TP
-\fBSHMEM_LOGICAL_IGET\fP: Elements of type logical.
-.TP
-\fBSHMEM_REAL_IGET\fP: Elements of type real.
-.PP
-If you are using Fortran, data types must be of default size. For example, a real variable must
-be declared as REAL, REAL*4, or REAL(KIND=4).
-.PP
-.SH NOTES
-
-See \fIintro_shmem\fP(3)
-for a definition of the term remotely accessible.
-.PP
-.SH EXAMPLES
-
-The following simple example uses shmem_logical_iget in a Fortran program. Compile
-this example with the \-lsma compiler option.
-.Vb
-PROGRAM STRIDELOGICAL
-  LOGICAL SOURCE(10), TARGET(5)
-  SAVE SOURCE ! SAVE MAKES IT REMOTELY ACCESSIBLE
-  DATA SOURCE /.T.,.F.,.T.,.F.,.T.,.F.,.T.,.F.,.T.,.F./
-  DATA TARGET / 5*.F. /
-
-  CALL START_PES(2)
-  IF (MY_PE() .EQ. 0) THEN
-    CALL SHMEM_LOGICAL_IGET(TARGET, SOURCE, 1, 2, 5, 1)
-    PRINT*,'TARGET AFTER SHMEM_LOGICAL_IGET:',TARGET
-  ENDIF
-  CALL SHMEM_BARRIER_ALL
-END
-.Ve
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3),
-\fIshmem_get\fP(3),
-\fIshmem_quiet\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_short_iput.3in b/oshmem/shmem/man/man3/shmem_short_iput.3in
deleted file mode 100644
index 103bf2034a0..00000000000
--- a/oshmem/shmem/man/man3/shmem_short_iput.3in
+++ /dev/null
@@ -1,220 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_IPUT" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmem_complex_iput\fP(3),
-\fIshmem_double_iput\fP(3),
-\fIshmem_float_iput\fP(3),
-\fIshmem_int_iput\fP(3),
-\fIshmem_integer_iput\fP(3),
-\fIshmem_iput4\fP(3),
-\fIshmem_iput8\fP(3),
-\fIshmem_iput32\fP(3),
-\fIshmem_iput64\fP(3),
-\fIshmem_iput128\fP(3),
-\fIshmem_logical_iput\fP(3),
-\fIshmem_long_iput\fP(3),
-\fIshmem_longdouble_iput\fP(3),
-\fIshmem_longlong_iput\fP(3),
-\fIshmem_real_iput\fP(3),
-\fIshmem_short_iput\fP(3)
-\- Transfer strided data to a specified processing element (PE).
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-void shmem_double_iput(double *target, const double *source,
-  ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe);
-
-void shmem_float_iput(float *target, const float *source,
-  ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe);
-
-void shmem_int_iput(int *target, const int *source,
-  ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe);
-
-void shmem_iput32(void *target, const void *source,
-  ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe);
-
-void shmem_iput64(void *target, const void *source,
-  ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe);
-
-void shmem_iput128(void *target, const void *source,
-  ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe);
-
-void shmem_long_iput(long *target, const long *source,
-  ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe);
-
-void shmem_longdouble_iput(long double *target,
-  const long double *source, ptrdiff_t tst, ptrdiff_t sst,
-  size_t len, int pe);
-
-void shmem_longlong_iput(long long *target,
-  const long long *source, ptrdiff_t tst, ptrdiff_t sst,
-  size_t len, int pe);
-
-void shmem_short_iput(short *target, const short *source,
-  ptrdiff_t tst, ptrdiff_t sst, size_t len, int pe);
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER tst, sst, len, pe
-
-CALL SHMEM_COMPLEX_IPUT(target, source, tst, sst, len,
-& pe)
-
-CALL SHMEM_DOUBLE_IPUT(target, source, tst, sst, len,
-& pe)
-
-CALL SHMEM_INTEGER_IPUT(target, source, tst, sst, len,
-& pe)
-
-CALL SHMEM_IPUT4(target, source, tst, sst, len, pe)
-
-CALL SHMEM_IPUT8(target, source, tst, sst, len, pe)
-
-CALL SHMEM_IPUT32(target, source, tst, sst, len, pe)
-
-CALL SHMEM_IPUT64(target, source, tst, sst, len, pe)
-
-CALL SHMEM_IPUT128(target, source, tst, sst, len, pe)
-
-CALL SHMEM_LOGICAL_IPUT(target, source, tst, sst, len,
-& pe)
-
-CALL SHMEM_REAL_IPUT(target, source, tst, sst, len, pe)
-.Ve
-.PP
-.SH DESCRIPTION
-
-The shmem_iput routines read the elements of a local array (\fBsource\fP)
-and write them
-to a remote array (\fBtarget\fP)
-on the PE indicated by \fBpe\fP\&.
-These routines return
-when the data has been copied out of the source array on the local PE but not necessarily
-before the data has been delivered to the remote data object.
-.PP
-The arguments are as follows:
-.TP
-target
-Array to be updated on the remote PE. This data object must be remotely
-accessible.
-.TP
-source
-Array containing the data to be copied.
-.TP
-tst
-The stride between consecutive elements of the target array. The stride is scaled by
-the element size of the target array. A value of 1 indicates contiguous data. tst must be of
-type integer. If you are using Fortran, it must be a default integer value.
-.TP
-sst
-The stride between consecutive elements of the source array. The stride is scaled
-by the element size of the source array. A value of 1 indicates contiguous data. sst must be
-of type integer. If you are using Fortran, it must be a default integer value.
-.TP
-len
-Number of elements in the target and source arrays. len must be of type integer. If
-you are using Fortran, it must be a constant, variable, or array element of default integer
-type.
-.TP
-pe
-PE number of the remote PE. pe must be of type integer. If you are using Fortran, it
-must be a constant, variable, or array element of default integer type.
-.PP
-The target and source data objects must conform to typing constraints, which are as follows:
-.PP
-.TP
-\fBshmem_iput32, shmem_iput4\fP: Any noncharacter type that has a storage size equal
-to 32 bits.
-.TP
-\fBshmem_iput64, shmem_iput8\fP: Any noncharacter type that has a storage size equal
-to 64 bits.
-.TP
-\fBshmem_iput128\fP: Any noncharacter type that has a storage size equal to 128 bits.
-.TP
-\fBshmem_short_iput\fP: Elements of type short.
-.TP
-\fBshmem_int_iput\fP: Elements of type int.
-.TP
-\fBshmem_long_iput\fP: Elements of type long.
-.TP
-\fBshmem_longlong_iput\fP: Elements of type long long.
-.TP
-\fBshmem_float_iput\fP: Elements of type float.
-.TP
-\fBshmem_double_iput\fP: Elements of type double.
-.TP
-\fBshmem_longdouble_iput\fP: Elements of type long double.
-.TP
-\fBSHMEM_COMPLEX_IPUT\fP: Elements of type complex of default size.
-.TP
-\fBSHMEM_DOUBLE_IPUT\fP: (Fortran) Elements of type double precision.
-.TP
-\fBSHMEM_INTEGER_IPUT\fP: Elements of type integer.
-.TP
-\fBSHMEM_LOGICAL_IPUT\fP: Elements of type logical.
-.TP
-\fBSHMEM_REAL_IPUT\fP: Elements of type real.
-.TP
-\fBSHMEM_LOGICAL_IPUT\fP: Elements of type logical.
-.TP
-\fBSHMEM_REAL_IPUT\fP: Elements of type real.
-.PP
-If you are using Fortran, data types must be of default size. For example, a real variable must
-be declared as REAL, REAL*4 or REAL(KIND=4).
-.PP
-.SH NOTES
-
-See \fIintro_shmem\fP(3)
-for a definition of the term remotely accessible.
-.PP
-.SH EXAMPLES
-
-Consider the following simple shmem_long_iput example for C/C++ programs.
-.Vb
-#include <mpp/shmem.h>
-
-main()
-{
-  short source[10] = { 1, 2, 3, 4, 5,
-  6, 7, 8, 9, 10 };
-  static short target[10];
-
-  shmem_init();
-  if (shmem_my_pe() == 0) {
-    /* put 10 words into target on PE 1 */
-    shmem_short_iput(target, source, 1, 2, 5, 1);
-  }
-  shmem_barrier_all(); /* sync sender and receiver */
-  if (shmem_my_pe() == 1) {
-    shmem_udcflush(); /* not required on IRIX systems */
-    printf("target on PE %d is %d %d %d %d %d0, shmem_my_pe(),
-    (int)target[0], (int)target[1], (int)target[2],
-    (int)target[3], (int)target[4] );
-  }
-  shmem_barrier_all(); /* sync before exiting */
-}
-.Ve
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3),
-\fIshmem_iget\fP(3),
-\fIshmem_put\fP(3),
-\fIshmem_quiet\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_short_max_to_all.3in b/oshmem/shmem/man/man3/shmem_short_max_to_all.3in
deleted file mode 100644
index a522c3e7c2e..00000000000
--- a/oshmem/shmem/man/man3/shmem_short_max_to_all.3in
+++ /dev/null
@@ -1,238 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2014-2016 Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_MAX" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmem_double_max_to_all\fP(3),
-\fIshmem_float_max_to_all\fP(3),
-\fIshmem_int_max_to_all\fP(3),
-\fIshmem_int4_max_to_all\fP(3),
-\fIshmem_int8_max_to_all\fP(3),
-\fIshmem_long_max_to_all\fP(3),
-\fIshmem_longdouble_max_to_all\fP(3),
-\fIshmem_longlong_max_to_all\fP(3),
-\fIshmem_real4_max_to_all\fP(3),
-\fIshmem_real8_max_to_all\fP(3),
-\fIshmem_real16_max_to_all\fP(3),
-\fIshmem_short_max_to_all\fP(3)
-\- Performs a maximum function reduction across a set of processing elements (PEs).
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-void shmem_double_max_to_all(double *target, const double *source,
-  int nreduce, int PE_start, int logPE_stride, int PE_size,
-  double *pWrk, long *pSync);
-
-void shmem_float_max_to_all(float *target, const float *source,
-  int nreduce, int PE_start, int logPE_stride, int PE_size,
-  float *pWrk, long *pSync);
-
-void shmem_int_max_to_all(int *target, const int *source,
-  int nreduce, int PE_start, int logPE_stride, int PE_size,
-  int *pWrk, long *pSync);
-
-void shmem_long_max_to_all(long *target, const long *source,
-  int nreduce, int PE_start, int logPE_stride, int PE_size,
-  long *pWrk, long *pSync);
-
-void shmem_longdouble_max_to_all(long double *target,
-  const long double *source, int nreduce, int PE_start,
-  int logPE_stride, int PE_size, long double *pWrk, long *pSync);
-
-void shmem_longlong_max_to_all(long long *target,
-  const long long *source, int nreduce,  int PE_start,
-  int logPE_stride, int PE_size, long long *pWrk, long *pSync);
-
- void shmem_short_max_to_all(short *target, const short *source,
-  int nreduce, int PE_start, int logPE_stride, int PE_size,
-  short *pWrk, long *pSync);
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER pSync(SHMEM_REDUCE_SYNC_SIZE)
-
-INTEGER nreduce, PE_start, logPE_stride, PE_size
-
-CALL SHMEM_INT4_MAX_TO_ALL(target, source, nreduce,
-& PE_start, logPE_stride, PE_size, pWrk, pSync)
-
-CALL SHMEM_INT8_MAX_TO_ALL(target, source, nreduce,
-& PE_start, logPE_stride, PE_size, pWrk, pSync)
-
-CALL SHMEM_REAL4_MAX_TO_ALL(target, source, nreduce,
-& PE_start, logPE_stride, PE_size, pWrk, pSync)
-
-CALL SHMEM_REAL8_MAX_TO_ALL(target, source, nreduce,
-& PE_start, logPE_stride, PE_size, pWrk, pSync)
-
-CALL SHMEM_REAL16_MAX_TO_ALL(target, source, nreduce,
-& PE_start, logPE_stride, PE_size, pWrk, pSync)
-.Ve
-.SH DESCRIPTION
-
-The shared memory (SHMEM) reduction routines compute one or more reductions across
-symmetric arrays on multiple virtual PEs. A reduction performs an associative binary
-operation across a set of values. For a list of other SHMEM reduction routines, see
-\fIintro_shmem\fP(3)\&.
-.PP
-As with all SHMEM collective routines, each of these routines assumes that only PEs in the
-active set call the routine. If a PE not in the active set calls a SHMEM collective routine,
-undefined behavior results.
-.PP
-The nreduce argument determines the number of separate reductions to perform. The source
-array on all PEs in the active set provides one element for each reduction. The results of the
-reductions are placed in the target array on all PEs in the active set. The active set is defined
-by the PE_start, logPE_stride, PE_size triplet.
-.PP
-The source and target arrays may be the same array, but they may not be overlapping arrays.
-.PP
-The arguments are as follows:
-.TP
-target
-A symmetric array of length nreduce elements to receive the results of the
-reduction operations. The data type of target varies with the version of the reduction routine
-being called. When calling from C, refer to the SYNOPSIS section for data type information.
-.PP
-When calling from Fortran, the target data types are as follows:
-.RS
-.TP
-\fBshmem_comp8_max_to_all\fP: Complex, with an element size equal to two
-8\-byte real values.
-.TP
-\fBshmem_int4_max_to_all\fP: Integer, with an element size of 4 bytes.
-.TP
-\fBshmem_int8_max_to_all\fP: Integer, with an element size of 8 bytes.
-.TP
-\fBshmem_real4_max_to_all\fP: Real, with an element size of 4 bytes.
-.TP
-\fBshmem_real16_max_to_all\fP: Real, with an element size of 16 bytes.
-.RE
-.RS
-.PP
-.RE
-.TP
-source
-A symmetric array of length nreduce elements that contains one element for
-each separate reduction operation. The source argument must have the same data type as
-target.
-.TP
-nreduce
-The number of elements in the target and source arrays. nreduce must be of
-type integer. If you are using Fortran, it must be a default integer value.
-.TP
-PE_start
-The lowest virtual PE number of the active set of PEs. PE_start must be of
-type integer. If you are using Fortran, it must be a default integer value.
-.TP
-logPE_stride
-The log (base 2) of the stride between consecutive virtual PE numbers in
-the active set. logPE_stride must be of type integer. If you are using Fortran, it must be a
-default integer value.
-.TP
-PE_size
-The number of PEs in the active set. PE_size must be of type integer. If you
-are using Fortran, it must be a default integer value.
-.TP
-pWrk
-A symmetric work array. The pWrk argument must have the same data type as
-target. In C/C++, this contains max(nreduce/2 + 1,
-_SHMEM_REDUCE_MIN_WRKDATA_SIZE) elements. In Fortran, this contains
-max(nreduce/2 + 1, SHMEM_REDUCE_MIN_WRKDATA_SIZE) elements.
-.TP
-pSync
-A symmetric work array. In C/C++, pSync is of type long and size
-_SHMEM_REDUCE_SYNC_SIZE. In Fortran, pSync is of type integer and size
-SHMEM_REDUCE_SYNC_SIZE. If you are using Fortran, it must be a default integer value.
-Every element of this array must be initialized with the value _SHMEM_SYNC_VALUE (in
-C/C++) or SHMEM_SYNC_VALUE (in Fortran) before any of the PEs in the active set enter
-the reduction routine.
-.PP
-The values of arguments nreduce, PE_start, logPE_stride, and PE_size must be equal on all
-PEs in the active set. The same target and source arrays, and the same pWrk and pSync work
-arrays, must be passed to all PEs in the active set.
-.PP
-Before any PE calls a reduction routine, you must ensure that the following conditions exist
-(synchronization via a barrier or some other method is often needed to ensure this): The
-pWrk and pSync arrays on all PEs in the active set are not still in use from a prior call to a
-collective SHMEM routine. The target array on all PEs in the active set is ready to accept the
-results of the reduction.
-.PP
-Upon return from a reduction routine, the following are true for the local PE: The target array
-is updated. The values in the pSync array are restored to the original values.
-.PP
-.SH NOTES
-
-The terms collective, symmetric, and cache aligned are defined in \fIintro_shmem\fP(3)\&.
-All SHMEM reduction routines reset the values in pSync before they return, so a particular
-pSync buffer need only be initialized the first time it is used.
-.PP
-You must ensure that the pSync array is not being updated on any PE in the active set while
-any of the PEs participate in processing of a SHMEM reduction routine. Be careful of the
-following situations: If the pSync array is initialized at run time, some type of
-synchronization is needed to ensure that all PEs in the working set have initialized pSync
-before any of them enter a SHMEM routine called with the pSync synchronization array. A
-pSync or pWrk array can be reused in a subsequent reduction routine call only if none
-of the PEs in the active set are still processing a prior reduction routine call that used the
-same pSync or pWrk arrays.
-.PP
-In general, this can be assured only by doing some type of synchronization. However, in the
-special case of reduction routines being called with the same active set, you can allocate two
-pSync and pWrk arrays and alternate between them on successive calls.
-.PP
-.SH EXAMPLES
-
-\fBExample 1:\fP
-This Fortran example statically initializes the pSync array and finds the
-maximum value of real variable FOO across all even PEs.
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER PSYNC(SHMEM_REDUCE_SYNC_SIZE)
-DATA PSYNC /SHMEM_REDUCE_SYNC_SIZE*SHMEM_SYNC_VALUE/
-PARAMETER (NR=1)
-REAL FOO, FOOMAX, PWRK(MAX(NR/2+1,SHMEM_REDUCE_MIN_WRKDATA_SIZE))
-COMMON /COM/ FOO, FOOMAX, PWRK
-INTRINSIC MY_PE
-
-IF ( MOD(MY_PE(),2) .EQ. 0) THEN
-  CALL SHMEM_REAL8_MAX_TO_ALL(FOOMAX, FOO, NR, 0, 1, N$PES/2,
-  & PWRK, PSYNC)
-  PRINT *, 'Result on PE ', MY_PE(), ' is ', FOOMAX
-ENDIF
-.Ve
-\fBExample 2:\fP
-Consider the following C/C++ call:
-.Vb
-shmem_int_max_to_all( target, source, 3, 0, 0, 8, pwrk, psync );
-.Ve
-The preceding call is more efficient, but semantically equivalent to, the combination of the
-following calls:
-.Vb
-shmem_int_max_to_all(&(target[0]), &(source[0]), 1, 0, 0, 8,
-  pwrk1, psync1);
-shmem_int_max_to_all(&(target[1]), &(source[1]), 1, 0, 0, 8,
-  pwrk2, psync2);
-shmem_int_max_to_all(&(target[2]), &(source[2]), 1, 0, 0, 8,
-  pwrk1, psync1);
-.Ve
-Note that two sets of pWrk and pSync arrays are used alternately because no synchronization
-is done between calls.
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_short_min_to_all.3in b/oshmem/shmem/man/man3/shmem_short_min_to_all.3in
deleted file mode 100644
index 6b0e3c4a0c5..00000000000
--- a/oshmem/shmem/man/man3/shmem_short_min_to_all.3in
+++ /dev/null
@@ -1,234 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2014-2016 Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_MIN" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmem_double_min_to_all\fP(3),
-\fIshmem_float_min_to_all\fP(3),
-\fIshmem_int_min_to_all\fP(3),
-\fIshmem_int4_min_to_all\fP(3),
-\fIshmem_int8_min_to_all\fP(3),
-\fIshmem_long_min_to_all\fP(3),
-\fIshmem_longdouble_min_to_all\fP(3),
-\fIshmem_longlong_min_to_all\fP(3),
-\fIshmem_real4_min_to_all\fP(3),
-\fIshmem_real8_min_to_all\fP(3),
-\fIshmem_real16_min_to_all\fP(3),
-\fIshmem_short_min_to_all\fP(3)
-\- Performs a minimum function reduction across a set of processing elements (PEs)
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-void shmem_double_min_to_all(double *target, const double *source,
-  int nreduce, int PE_start, int logPE_stride, int PE_size,
-  double *pWrk, long *pSync);
-
-void shmem_float_min_to_all(float *target, const float *source,
-  int nreduce, int PE_start, int logPE_stride, int PE_size,
-  float *pWrk, long *pSync);
-
-void shmem_int_min_to_all(int *target, const int *source,
-  int nreduce, int PE_start, int logPE_stride, int PE_size,
-  int *pWrk, long *pSync);
-
-void shmem_long_min_to_all(long *target, const long *source,
-  int nreduce, int PE_start, int logPE_stride, int PE_size,
-  long *pWrk, long *pSync);
-
-void shmem_longdouble_min_to_all(long double *target,
-  const long double *source, int nreduce, int PE_start,
-  int logPE_stride, int PE_size, long double *pWrk,
-  long *pSync);
-
-void shmem_longlong_min_to_all(long long *target,
-  const long long *source, int nreduce, int PE_start, int logPE_stride,
-  int PE_size, long long *pWrk, long *pSync);
-
-void shmem_short_min_to_all(short *target, const short *source,
-  int nreduce, int PE_start, int logPE_stride, int PE_size,
-  short *pWrk, long *pSync);
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER pSync(SHMEM_REDUCE_SYNC_SIZE)
-INTEGER nreduce, PE_start, logPE_stride, PE_size
-
-CALL SHMEM_INT4_MIN_TO_ALL(target, source, nreduce, PE_start,
-& logPE_stride, PE_size, pWrk, pSync)
-
-CALL SHMEM_INT8_MIN_TO_ALL(target, source, nreduce, PE_start,
-& logPE_stride, PE_size, pWrk, pSync)
-
-CALL SHMEM_REAL4_MIN_TO_ALL(target, source, nreduce, PE_start,
-& logPE_stride, PE_size, pWrk, pSync)
-
-CALL SHMEM_REAL8_MIN_TO_ALL(target, source, nreduce, PE_start,
-& logPE_stride, PE_size, pWrk, pSync)
-
-CALL SHMEM_REAL16_MIN_TO_ALL(target, source, nreduce, PE_start,
-& logPE_stride, PE_size, pWrk, pSync)
-.Ve
-.SH DESCRIPTION
-
-The shared memory (SHMEM) reduction routines compute one or more reductions across
-symmetric arrays on multiple virtual PEs. A reduction performs an associative binary
-operation across a set of values. For a list of other SHMEM reduction routines, see
-\fIintro_shmem\fP(3)\&.
-.PP
-As with all SHMEM collective routines, each of these routines assumes that only PEs in the
-active set call the routine. If a PE not in the active set calls a SHMEM collective routine,
-undefined behavior results.
-.PP
-The nreduce argument determines the number of separate reductions to perform. The source
-array on all PEs in the active set provides one element for each reduction. The results of the
-reductions are placed in the target array on all PEs in the active set. The active set is defined
-by the PE_start, logPE_stride, PE_size triplet.
-.PP
-The source and target arrays may be the same array, but they may not be overlapping arrays.
-.PP
-The arguments are as follows:
-.TP
-target
-A symmetric array of length nreduce elements to receive the results of the
-reduction operations. The data type of target varies with the version of the reduction routine
-being called. When calling from C/C++, refer to the SYNOPSIS section for data type
-information. When calling from Fortran, the target data types are as follows:
-.RS
-.TP
-\fBshmem_int4_min_to_all\fP: Integer, with an element size of 4 bytes
-.TP
-\fBshmem_int8_min_to_all\fP: Integer, with an element size of 8 bytes
-.TP
-\fBshmem_real4_min_to_all\fP: Real, with an element size of 4 bytes
-.TP
-\fBshmem_real8_min_to_all\fP: Real, with an element size of 8 bytes
-.TP
-\fBshmem_real16_min_to_all\fP: Real, with an element size of 16 bytes
-.TP
-\fBsource A symmetric array\fP: of length nreduce elements, that contains one
-element for each separate reduction operation. The source argument must have the same
-data type as target.
-.RE
-.RS
-.PP
-.RE
-.TP
-nreduce
-The number of elements in the target and source arrays. nreduce must be of
-type integer. If you are using Fortran, it must be a default integer value.
-.TP
-PE_start
-The lowest virtual PE number of the active set of PEs. PE_start must be of
-type integer. If you are using Fortran, it must be a default integer value.
-.TP
-logPE_stride
-The log (base 2) of the stride between consecutive virtual PE numbers in
-the active set. logPE_stride must be of type integer. If you are using Fortran, it must be a
-default integer value.
-.TP
-PE_size
-The number of PEs in the active set. PE_size must be of type integer. If you
-are using Fortran, it must be a default integer value.
-.TP
-pWrk
-A symmetric work array. The pWrk argument must have the same data type as
-target. In C/C++, this contains max(nreduce/2 + 1,
-_SHMEM_REDUCE_MIN_WRKDATA_SIZE) elements. In Fortran, this contains
-max(nreduce/2 + 1, SHMEM_REDUCE_MIN_WRKDATA_SIZE) elements.
-.TP
-pSync
-A symmetric work array. In C/C++, pSync is of type long and size
-_SHMEM_REDUCE_SYNC_SIZE. In Fortran, pSync is of type integer and size
-SHMEM_REDUCE_SYNC_SIZE. If you are using Fortran, it must be a default integer value.
-Every element of this array must be initialized with the value _SHMEM_SYNC_VALUE (in
-C/C++) or SHMEM_SYNC_VALUE (in Fortran) before any of the PEs in the active set enter
-the reduction routine.
-.PP
-The values of arguments nreduce, PE_start, logPE_stride, and PE_size must be equal on all
-PEs in the active set. The same target and source arrays, and the same pWrk and pSync work
-arrays, must be passed to all PEs in the active set.
-.PP
-Before any PE calls a reduction routine, you must ensure that the following conditions exist
-(synchronization via a barrier or some other method is often needed to ensure this): The
-pWrk and pSync arrays on all PEs in the active set are not still in use from a prior call to a
-collective SHMEM routine. The target array on all PEs in the active set is ready to accept the
-results of the reduction.
-.PP
-Upon return from a reduction routine, the following are true for the local PE: The target array
-is updated. The values in the pSync array are restored to the original values.
-.PP
-.SH NOTES
-
-The terms collective, symmetric, and cache aligned are defined in \fIintro_shmem\fP(3)\&.
-All SHMEM reduction routines reset the values in pSync before they return, so a particular
-pSync buffer need only be initialized the first time it is used.
-.PP
-You must ensure that the pSync array is not being updated on any PE in the active set while
-any of the PEs participate in processing of a SHMEM reduction routine. Be careful of the
-following situations: If the pSync array is initialized at run time, some type of
-synchronization is needed to ensure that all PEs in the working set have initialized pSync
-before any of them enter a SHMEM routine called with the pSync synchronization array. A
-pSync or pWrk array can be reused in a subsequent reduction routine call only if none
-of the PEs in the active set are still processing a prior reduction routine call that used the
-same pSync or pWrk arrays. In general, this can be assured only by doing some type of
-synchronization. However, in the special case of reduction routines being called with the
-same active set, you can allocate two pSync and pWrk arrays and alternate between them on
-successive calls.
-.PP
-.SH EXAMPLES
-
-\fBExample 1:\fP
-This Fortran example statically initializes the pSync array and finds the
-minimum value of real variable FOO across all the even PEs.
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER PSYNC(SHMEM_REDUCE_SYNC_SIZE)
-DATA PSYNC /SHMEM_REDUCE_SYNC_SIZE*SHMEM_SYNC_VALUE/
-PARAMETER (NR=1)
-REAL FOO, FOOMIN, PWRK(MAX(NR/2+1,SHMEM_REDUCE_MIN_WRKDATA_SIZE))
-COMMON /COM/ FOO, FOOMIN, PWRK
-INTRINSIC MY_PE
-
-IF ( MOD(MY_PE(),2) .EQ. 0) THEN
-  CALL SHMEM_REAL8_MIN_TO_ALL(FOOMIN, FOO, NR, 0, 1, N$PES/2,
-  & PWRK, PSYNC)
-  PRINT *, 'Result on PE ', MY_PE(), ' is ', FOOMIN
-ENDIF
-.Ve
-\fBExample 2:\fP
-Consider the following C/C++ call:
-.Vb
-shmem_int_min_to_all( target, source, 3, 0, 0, 8, pwrk, psync );
-.Ve
-The preceding call is more efficient, but semantically equivalent to, the combination of the
-following calls:
-.Vb
-shmem_int_min_to_all(&(target[0]), &(source[0]), 1, 0, 0, 8,
-  pwrk1, psync1);
-shmem_int_min_to_all(&(target[1]), &(source[1]), 1, 0, 0, 8,
-  pwrk2, psync2);
-shmem_int_min_to_all(&(target[2]), &(source[2]), 1, 0, 0, 8,
-  pwrk1, psync1);
-.Ve
-Note that two sets of pWrk and pSync arrays are used alternately because no synchronization
-is done between calls.
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_short_or_to_all.3in b/oshmem/shmem/man/man3/shmem_short_or_to_all.3in
deleted file mode 100644
index bcb5a2c0c53..00000000000
--- a/oshmem/shmem/man/man3/shmem_short_or_to_all.3in
+++ /dev/null
@@ -1,202 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2014-2016 Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_OR" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmem_int_or_to_all\fP(3),
-\fIshmem_int4_or_to_all\fP(3),
-\fIshmem_int8_or_to_all\fP(3),
-\fIshmem_long_or_to_all\fP(3),
-\fIshmem_longlong_or_to_all\fP(3),
-\fIshmem_short_or_to_all\fP(3)
-\- Performs a bitwise OR function reduction across a set of processing elements (PEs)
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-void shmem_int_or_to_all(int *target, const int *source,
-  int nreduce, int PE_start, int logPE_stride, int PE_size,
-  int *pWrk, long *pSync);
-
-void shmem_long_or_to_all(long *target, const long *source,
-  int nreduce, int PE_start, int logPE_stride, int PE_size,
-  long *pWrk, long *pSync);
-
-void shmem_longlong_or_to_all(long long *target,
-  const long long *source, int nreduce, int PE_start, int logPE_stride,
-  int PE_size, long long *pWrk, long *pSync);
-
-void shmem_short_or_to_all(short *target, const short *source,
-  int nreduce, int PE_start, int logPE_stride, int PE_size,
-  short *pWrk, long *pSync);
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER pSync(SHMEM_REDUCE_SYNC_SIZE)
-INTEGER nreduce, PE_start, logPE_stride, PE_size
-
-CALL SHMEM_INT4_OR_TO_ALL(target, source, nreduce, PE_start,
-& logPE_stride, PE_size, pWrk, pSync)
-
-CALL SHMEM_INT8_OR_TO_ALL(target, source, nreduce, PE_start,
-& logPE_stride, PE_size, pWrk, pSync)
-.Ve
-.SH DESCRIPTION
-
-The shared memory (SHMEM) reduction routines compute one or more reductions across
-symmetric arrays on multiple virtual PEs. A reduction performs an associative binary
-operation across a set of values. For a list of other SHMEM reduction routines, see
-intro_shmem(3).
-.PP
-As with all SHMEM collective routines, each of these routines assumes that only PEs in the
-active set call the routine. If a PE not in the active set calls a SHMEM collective routine,
-undefined behavior results.
-.PP
-The nreduce argument determines the number of separate reductions to perform. The source
-array on all PEs in the active set provides one element for each reduction. The results of the
-reductions are placed in the target array on all PEs in the active set. The active set is defined
-by the PE_start, logPE_stride, PE_size triplet.
-.PP
-The source and target arrays may be the same array, but they may not be overlapping arrays.
-.PP
-The arguments are as follows:
-.TP
-target
-A symmetric array of length nreduce elements to receive the results of the
-reduction operations. The data type of target varies with the version of the reduction routine
-being called. When calling from C/C++, refer to the SYNOPSIS section for data type
-information. When calling from Fortran, the target data types are as follows:
-.RS
-.TP
-\fBshmem_int8_or_to_all\fP Integer, with an element size of 8 bytes.
-.TP
-\fBshmem_int4_or_to_all\fP Integer, with an element size of 4 bytes.
-.RE
-.RS
-.PP
-.RE
-.TP
-source
-A symmetric array, of length nreduce elements, that contains one element for
-each separate reduction operation. The source argument must have the same data type as
-target.
-.TP
-nreduce
-The number of elements in the target and source arrays. nreduce must be of
-type integer. If you are using Fortran, it must be a default integer value.
-.TP
-PE_start
-The lowest virtual PE number of the active set of PEs. PE_start must be of
-type integer. If you are using Fortran, it must be a default integer value.
-.TP
-logPE_stride
-The log (base 2) of the stride between consecutive virtual PE numbers in
-the active set. logPE_stride must be of type integer. If you are using Fortran, it must be a
-default integer value.
-.TP
-PE_size
-The number of PEs in the active set. PE_size must be of type integer. If you
-are using Fortran, it must be a default integer value.
-.TP
-pWrk
-A symmetric work array. The pWrk argument must have the same data type as
-target. In C/C++, this contains max(nreduce/2 + 1,
-_SHMEM_REDUCE_MIN_WRKDATA_SIZE) elements. In Fortran, this contains
-max(nreduce/2 + 1, SHMEM_REDUCE_MIN_WRKDATA_SIZE) elements.
-.TP
-pSync
-A symmetric work array. In C/C++, pSync is of type long and size
-_SHMEM_REDUCE_SYNC_SIZE. In Fortran, pSync is of type integer and size
-SHMEM_REDUCE_SYNC_SIZE. If you are using Fortran, it must be a default integer value.
-Every element of this array must be initialized with the value _SHMEM_SYNC_VALUE (in
-C/C++) or SHMEM_SYNC_VALUE (in Fortran) before any of the PEs in the active set enter
-the reduction routine.
-.PP
-The values of arguments nreduce, PE_start, logPE_stride, and PE_size must be equal on
-all PEs in the active set. The same target and source arrays, and the same pWrk and pSync
-work arrays, must be passed to all PEs in the active set.
-.PP
-Before any PE calls a reduction routine, you must ensure that the following conditions exist
-(synchronization via a barrier or some other method is often needed to ensure this): The
-pWrk and pSync arrays on all PEs in the active set are not still in use from a prior call to a
-collective SHMEM routine. The target array on all PEs in the active set is ready to accept the
-results of the reduction.
-.PP
-Upon return from a reduction routine, the following are true: The target array is updated. The
-values in the pSync array are restored to the original values.
-.PP
-.SH NOTES
-
-The terms collective, symmetric, and cache aligned are defined in \fIintro_shmem\fP(3)\&.
-All SHMEM reduction routines reset the values in pSync before they return, so a particular
-pSync buffer need only be initialized the first time it is used.
-.PP
-You must ensure that the pSync array is not being updated on any PE in the active set while
-any of the PEs participate in processing of a SHMEM reduction routine. Be careful to avoid
-these situations: If the pSync array is initialized at run time, some type of synchronization is
-needed to ensure that all PEs in the working set have initialized pSync before any of them
-enter a SHMEM routine called with the pSync synchronization array. A pSync or pWrk array
-can be reused in a subsequent reduction routine call only if none of the PEs in the active set
-are still processing a prior reduction routine call that used the same pSync or pWrk arrays. In
-general, this can be assured only by doing some type of synchronization. However, in the
-special case of reduction routines being called with the same active set, you can allocate two
-pSync and pWrk arrays and alternate between them on successive calls.
-.PP
-.SH EXAMPLES
-
-\fBExample 1:\fP
-This Fortran example statically initializes the pSync array and finds the
-logical OR of the integer variable FOO across all even PEs.
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER PSYNC(SHMEM_REDUCE_SYNC_SIZE)
-DATA PSYNC /SHMEM_REDUCE_SYNC_SIZE*SHMEM_SYNC_VALUE/
-PARAMETER (NR=1)
-REAL PWRK(MAX(NR/2+1,SHMEM_REDUCE_MIN_WRKDATA_SIZE))
-INTEGER FOO, FOOOR
-COMMON /COM/ FOO, FOOOR, PWRK
-INTRINSIC MY_PE
-
-IF ( MOD(MY_PE(),2) .EQ. 0) THEN
-  CALL SHMEM_INT8_OR_TO_ALL(FOOOR, FOO, NR, 0, 1, N$PES/2,
-  & PWRK, PSYNC)
-  PRINT *,'Result on PE ',MY_PE(),' is ',FOOOR
-ENDIF
-.Ve
-\fBExample 2:\fP
-Consider the following C/C++ call:
-.Vb
-shmem_int_or_to_all( target, source, 3, 0, 0, 8, pwrk, psync );
-.Ve
-The preceding call is more efficient, but semantically equivalent to, the combination of the
-following calls:
-.Vb
-shmem_int_or_to_all(&(target[0]), &(source[0]), 1, 0, 0, 8,
-  pwrk1, psync1);
-shmem_int_or_to_all(&(target[1]), &(source[1]), 1, 0, 0, 8,
-  pwrk2, psync2);
-shmem_int_or_to_all(&(target[2]), &(source[2]), 1, 0, 0, 8,
-  pwrk1, psync1);
-.Ve
-Note that two sets of pWrk and pSync arrays are used alternately because no synchronization
-is done between calls.
-.PP
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_short_p.3in b/oshmem/shmem/man/man3/shmem_short_p.3in
deleted file mode 100644
index 42b9fd0e81f..00000000000
--- a/oshmem/shmem/man/man3/shmem_short_p.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_p.3
diff --git a/oshmem/shmem/man/man3/shmem_short_prod_to_all.3in b/oshmem/shmem/man/man3/shmem_short_prod_to_all.3in
deleted file mode 100644
index b03b3dc18e1..00000000000
--- a/oshmem/shmem/man/man3/shmem_short_prod_to_all.3in
+++ /dev/null
@@ -1,259 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2014-2016 Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_PROD" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmem_comp4_prod_to_all\fP(3),
-\fIshmem_comp8_prod_to_all\fP(3),
-\fIshmem_complexd_prod_to_all\fP(3),
-\fIshmem_complexf_prod_to_all\fP(3),
-\fIshmem_double_prod_to_all\fP(3),
-\fIshmem_float_prod_to_all\fP(3),
-\fIshmem_int_prod_to_all\fP(3),
-\fIshmem_int4_prod_to_all\fP(3),
-\fIshmem_int8_prod_to_all\fP(3),
-\fIshmem_long_prod_to_all\fP(3),
-\fIshmem_longdouble_prod_to_all\fP(3),
-\fIshmem_longlong_prod_to_all\fP(3),
-\fIshmem_real8_prod_to_all\fP(3),
-\fIshmem_real16_prod_to_all\fP(3),
-\fIshmem_real4_prod_to_all\fP(3),
-\fIshmem_short_prod_to_all\fP(3)
-\- Performs
-a product reduction across a set of processing elements (PEs)
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-void shmem_complexd_prod_to_all(double complex *target,
-  const double complex *source, int nreduce, int PE_start,
-  int logPE_stride, int PE_size, double complex *pWrk,
-  long *pSync);
-
-void shmem_complexf_prod_to_all(float complex *target,
-  const float complex *source, int nreduce, int PE_start,
-  int logPE_stride, int PE_size, float complex *pWrk,
-  long *pSync);
-
-void shmem_double_prod_to_all(double *target, const double *source,
-  int nreduce, int PE_start, int logPE_stride, int PE_size,
-  double *pWrk, long *pSync);
-
-void shmem_float_prod_to_all(float *target, const float *source,
-  int nreduce, int PE_start, int logPE_stride, int PE_size,
-  float *pWrk, long *pSync);
-
-void shmem_int_prod_to_all(int *target, const int *source,
-  int nreduce, int PE_start, int logPE_stride, int PE_size,
-  int *pWrk, long *pSync);
-
-void shmem_long_prod_to_all(long *target, const long *source,
-  int nreduce, int PE_start, int logPE_stride, int PE_size,
-  long *pWrk, long *pSync);
-
-void shmem_longdouble_prod_to_all(long double *target,
-  const long double *source, int nreduce, int PE_start,
-  int logPE_stride, int PE_size, long double *pWrk,
-  long *pSync);
-
-void shmem_longlong_prod_to_all(long long *target,
-  const long long *source, int nreduce, int PE_start,
-  int logPE_stride, int PE_size, long long *pWrk,
-  long *pSync);
-
-void shmem_short_prod_to_all(short *target, const short *source,
-  int nreduce, int PE_start, int logPE_stride, int PE_size,
-  short *pWrk, long *pSync);
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER pSync(SHMEM_REDUCE_SYNC_SIZE)
-INTEGER nreduce, PE_start, logPE_stride, PE_size
-
-CALL SHMEM_COMP4_PROD_TO_ALL(target, source, nreduce, PE_start,
-& logPE_stride, PE_size, pWrk, pSync)
-
-CALL SHMEM_COMP8_PROD_TO_ALL(target, source, nreduce, PE_start,
-& logPE_stride, PE_size, pWrk, pSync)
-
-CALL SHMEM_INT4_PROD_TO_ALL(target, source, nreduce, PE_start,
-& logPE_stride, PE_size, pWrk, pSync)
-
-CALL SHMEM_INT8_PROD_TO_ALL(target, source, nreduce, PE_start,
-& logPE_stride, PE_size, pWrk, pSync)
-
-CALL SHMEM_REAL4_PROD_TO_ALL(target, source, nreduce, PE_start,
-& logPE_stride, PE_size, pWrk, pSync)
-
-CALL SHMEM_REAL8_PROD_TO_ALL(target, source, nreduce, PE_start,
-& logPE_stride, PE_size, pWrk, pSync)
-
-CALL SHMEM_REAL16_PROD_TO_ALL(target, source, nreduce, PE_start,
-& logPE_stride, PE_size, pWrk, pSync)
-.Ve
-.SH DESCRIPTION
-
-The shared memory (SHMEM) reduction routines compute one or more reductions across
-symmetric arrays on multiple virtual PEs. A reduction performs an associative binary
-operation across a set of values. For a list of other SHMEM reduction routines, see
-\fIintro_shmem\fP(3)\&.
-.PP
-As with all SHMEM collective routines, each of these routines assumes that only PEs in the
-active set call the routine. If a PE not in the active set calls a SHMEM collective routine,
-undefined behavior results.
-.PP
-The nreduce argument determines the number of separate reductions to perform. The source
-array on all PEs in the active set provides one element for each reduction. The results of the
-reductions are placed in the target array on all PEs in the active set. The active set is defined
-by the PE_start, logPE_stride, PE_size triplet.
-.PP
-The source and target arrays may be the same array, but they may not be overlapping arrays.
-.PP
-The arguments are as follows:
-.TP
-target
-A symmetric array of length nreduce elements to receive the results of the
-reduction operations. The data type of target varies with the version of the reduction routine
-being called and the language used. When calling from C/C++, refer to the SYNOPSIS section
-for data type information. When calling from Fortran, the target data types are as follows:
-.RS
-.TP
-\fBshmem_comp4_prod_to_all\fP: Complex, with an element size equal to two
-4\-byte real values.
-.TP
-\fBshmem_comp8_prod_to_all\fP: Complex, with an element size equal to two
-8\-byte real values.
-.TP
-\fBshmem_int4_prod_to_all\fP: Integer, with an element size of 4 bytes
-.TP
-\fBshmem_int8_prod_to_all\fP: Integer, with an element size of 8 bytes
-.TP
-\fBshmem_real4_prod_to_all\fP: Real, with an element size of 4 bytes
-.TP
-\fBshmem_real8_prod_to_all\fP: Real, with an element size of 8 bytes
-.TP
-\fBshmem_real16_prod_to_all\fP: Real, with an element size of 16 bytes
-.RE
-.RS
-.PP
-.RE
-.TP
-source
-A symmetric array, of length nreduce elements, that contains one element for
-each separate reduction operation. The source argument must have the same data type as
-target.
-.TP
-nreduce
-The number of elements in the target and source arrays. nreduce must be of
-type integer. If you are using Fortran, it must be a default integer value.
-.TP
-PE_start
-The lowest virtual PE number of the active set of PEs. PE_start must be of
-type integer. If you are using Fortran, it must be a default integer value.
-.TP
-logPE_stride
-The log (base 2) of the stride between consecutive virtual PE numbers in
-the active set. logPE_stride must be of type integer. If you are using Fortran, it must be a
-default integer value.
-.TP
-PE_size
-The number of PEs in the active set. PE_size must be of type integer. If you
-are using Fortran, it must be a default integer value.
-.TP
-pWrk
-A symmetric work array. The pWrk argument must have the same data type as
-target. In C/C++, this contains max(nreduce/2 + 1,
-_SHMEM_REDUCE_MIN_WRKDATA_SIZE) elements. In Fortran, this contains
-max(nreduce/2 + 1, SHMEM_REDUCE_MIN_WRKDATA_SIZE) elements.
-.TP
-pSync
-A symmetric work array. In C/C++, pSync is of type long and size
-_SHMEM_REDUCE_SYNC_SIZE. In Fortran, pSync is of type integer and size
-SHMEM_REDUCE_SYNC_SIZE. If you are using Fortran, it must be a default integer value.
-Before any of the PEs in the active set enter the reduction routine, every element of this array
-must be initialized with the value _SHMEM_SYNC_VALUE (in C/C++) or
-SHMEM_SYNC_VALUE (in Fortran).
-.PP
-The values of arguments nreduce, PE_start, logPE_stride, and PE_size must be equal on all
-PEs in the active set. The same target and source arrays, and the same pWrk and pSync work
-arrays, must be passed to all PEs in the active set. Before any PE calls a reduction routine, you
-must ensure that the following conditions exist (synchronization via a barrier or some
-other method is often needed to ensure this): The pWrk and pSync arrays on all PEs in the
-active set are not still in use from a prior call to a collective SHMEM routine. The target array
-on all PEs in the active set is ready to accept the results of the reduction.
-.PP
-Upon return from a reduction routine, the following are true for the local PE: The target array
-is updated. The values in the pSync array are restored to the original values.
-.SH NOTES
-
-The terms collective, symmetric, and cache aligned are defined in \fIintro_shmem\fP(3)\&.
-All SHMEM reduction routines reset the values in pSync before they return, so a particular
-pSync buffer need only be initialized the first time it is used.
-.PP
-You must ensure that the pSync array is not being updated on any PE in the active set while
-any of the PEs participate in processing of a SHMEM reduction routine. Be careful of the
-following situations: If the pSync array is initialized at run time, some type of
-synchronization is needed to ensure that all PEs in the working set have initialized pSync
-before any of them enter a SHMEM routine called with the pSync synchronization array. A
-pSync or pWrk array can be reused in a subsequent reduction routine call only if none of the
-PEs in the active set are still processing a prior reduction routine call that used the same
-pSync or pWrk arrays. In general, this can be assured only by doing some type of
-synchronization. However, in the special case of reduction routines being called with the
-same active set, you can allocate two pSync and pWrk arrays and alternate between them on
-successive calls.
-.SH EXAMPLES
-
-\fBExample 1:\fP
-This Fortran example statically initializes the pSync array and finds the
-product of the real variable FOO across all the even PEs.
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER PSYNC(SHMEM_REDUCE_SYNC_SIZE)
-DATA PSYNC /SHMEM_REDUCE_SYNC_SIZE*SHMEM_SYNC_VALUE/
-PARAMETER (NR=1)
-REAL FOO, FOOPROD, PWRK(MAX(NR/2+1,SHMEM_REDUCE_MIN_WRKDATA_SIZE))
-COMMON /COM/ FOO, FOOPROD, PWRK
-INTRINSIC MY_PE
-
-IF ( MOD(MY_PE(),2) .EQ. 0) THEN
-  CALL SHMEM_COMP8_PROD_TO_ALL(FOOPROD, FOO, NR, 0, 1, N$PES/2,
-  & PWRK, PSYNC)
-  PRINT *, 'Result on PE ', MY_PE(), ' is ', FOOPROD
-ENDIF
-.Ve
-\fBExample 2:\fP
-Consider the following C/C++ call:
-.Vb
-shmem_short_prod_to_all(target, source, 3, 0, 0, 8, pwrk, psync);
-.Ve
-The preceding call is more efficient, but semantically equivalent to, the combination of the
-following calls:
-.Vb
-shmem_short_prod_to_all(&(target[0]), &(source[0]), 1, 0, 0, 8,
-  pwrk1, psync1);
-shmem_short_prod_to_all(&(target[1]), &(source[1]), 1, 0, 0, 8,
-  pwrk2, psync2);
-shmem_short_prod_to_all(&(target[2]), &(source[2]), 1, 0, 0, 8,
-  pwrk1, psync1);
-.Ve
-Note that two sets of pWrk and pSync arrays are used alternately because no synchronization
-is done between calls.
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_short_put.3in b/oshmem/shmem/man/man3/shmem_short_put.3in
deleted file mode 100644
index 9c7b2e25452..00000000000
--- a/oshmem/shmem/man/man3/shmem_short_put.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_char_put.3
diff --git a/oshmem/shmem/man/man3/shmem_short_put_nbi.3in b/oshmem/shmem/man/man3/shmem_short_put_nbi.3in
deleted file mode 100644
index fb4ad1413b0..00000000000
--- a/oshmem/shmem/man/man3/shmem_short_put_nbi.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_putmem_nbi.3
diff --git a/oshmem/shmem/man/man3/shmem_short_sum_to_all.3in b/oshmem/shmem/man/man3/shmem_short_sum_to_all.3in
deleted file mode 100644
index d94c72c1c67..00000000000
--- a/oshmem/shmem/man/man3/shmem_short_sum_to_all.3in
+++ /dev/null
@@ -1,281 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2014-2016 Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_SUM" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmem_comp4_sum_to_all\fP(3),
-\fIshmem_comp8_sum_to_all\fP(3),
-\fIshmem_complexd_sum_to_all\fP(3),
-\fIshmem_complexf_sum_to_all\fP(3),
-\fIshmem_double_sum_to_all\fP(3),
-\fIshmem_float_sum_to_all\fP(3),
-\fIshmem_int_sum_to_all\fP(3),
-\fIshmem_int4_sum_to_all\fP(3),
-\fIshmem_int8_sum_to_all\fP(3),
-\fIshmem_long_sum_to_all\fP(3),
-\fIshmem_longdouble_sum_to_all\fP(3),
-\fIshmem_longlong_sum_to_all\fP(3),
-\fIshmem_real4_sum_to_all\fP(3),
-\fIshmem_real8_sum_to_all\fP(3),
-\fIshmem_real16_sum_to_all\fP(3),
-\fIshmem_short_sum_to_all\fP(3)
-\- Performs
-a sum reduction across a set of processing elements (PEs)
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-void shmem_complexd_sum_to_all(double complex *target,
-  const double complex *source, int nreduce, int PE_start,
-  int logPE_stride, int PE_size, double complex *pWrk,
-  long *pSync);
-
-void shmem_complexf_sum_to_all(float complex *target,
-  const float complex *source, int nreduce, int PE_start,
-  int logPE_stride, int PE_size, float complex *pWrk,
-  long *pSync);
-
-void shmem_double_sum_to_all(double *target,
-  const double *source, int nreduce, int PE_start, int logPE_stride,
-  int PE_size, double *pWrk, long *pSync);
-
-void shmem_float_sum_to_all(float *target, const float *source,
-  int nreduce, int PE_start, int logPE_stride, int PE_size,
-  float *pWrk, long *pSync);
-
-void shmem_int_sum_to_all(int *target, const int *source,
-  int nreduce, int PE_start, int logPE_stride, int PE_size,
-  int *pWrk, long *pSync);
-
-void shmem_long_sum_to_all(long *target, const long *source,
-  int nreduce, int PE_start, int logPE_stride, int PE_size,
-  long *pWrk, long *pSync);
-
-void shmem_longdouble_sum_to_all(long double *target,
-  const long double *source, int nreduce, int PE_start, int
-  logPE_stride, int PE_size, long double *pWrk, long *pSync);
-
-void shmem_longlong_sum_to_all(long long *target,
-  const long long *source, int nreduce, int PE_start,
-  int logPE_stride, int PE_size, long long *pWrk,
-  long *pSync);
-
-void shmem_short_sum_to_all(short *target, const short *source,
-  int nreduce, int PE_start, int logPE_stride, int PE_size,
-  short *pWrk, long *pSync);
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER pSync(SHMEM_REDUCE_SYNC_SIZE)
-INTEGER nreduce, PE_start, logPE_stride, PE_size
-
-CALL SHMEM_COMP4_SUM_TO_ALL(target, source, nreduce,
-& PE_start, logPE_stride, PE_size, pWrk, pSync)
-
-CALL SHMEM_COMP8_SUM_TO_ALL(target, source, nreduce,
-& PE_start, logPE_stride, PE_size, pWrk, pSync)
-
-CALL SHMEM_INT4_SUM_TO_ALL(target, source, nreduce,
-& PE_start, logPE_stride, PE_size, pWrk, pSync)
-
-CALL SHMEM_INT8_SUM_TO_ALL(target, source, nreduce,
-& PE_start, logPE_stride, PE_size, pWrk, pSync)
-
-CALL SHMEM_REAL4_SUM_TO_ALL(target, source, nreduce,
-& PE_start, logPE_stride, PE_size, pWrk, pSync)
-
-CALL SHMEM_REAL8_SUM_TO_ALL(target, source, nreduce,
-& PE_start, logPE_stride, PE_size, pWrk, pSync)
-
-CALL SHMEM_REAL16_SUM_TO_ALL(target, source, nreduce,
-& PE_start, logPE_stride, PE_size, pWrk, pSync)
-.Ve
-.SH DESCRIPTION
-
-The shared memory (SHMEM) reduction routines compute one or more reductions across
-symmetric arrays on multiple virtual PEs. A reduction performs an associative binary
-operation across a set of values. For a list of other SHMEM reduction routines, see
-\fIintro_shmem\fP(3)\&.
-.PP
-As with all SHMEM collective routines, each of these routines assumes that only PEs in the
-active set call the routine. If a PE not in the active set calls a SHMEM collective routine,
-undefined behavior results.
-.PP
-The nreduce argument determines the number of separate reductions to perform. The source
-array on all PEs in the active set provides one element for each reduction. The results of the
-reductions are placed in the target array on all PEs in the active set. The active set is defined
-by the PE_start, logPE_stride, PE_size triplet.
-.PP
-The source and target arrays may be the same array, but they may not be overlapping arrays.
-.PP
-The arguments are as follows:
-.TP
-target
-The remotely accessible integer data object to be updated on the remote PE. If
-you are using C/C++, the type of target should match that implied in the SYNOPSIS section.
-If you are using the Fortran compiler, it must be of type integer with an element size of 4
-bytes for SHMEM_INT4_ADD and 8 bytes for SHMEM_INT8_ADD.
-.TP
-value
-The value to be atomically added to target. If you are using C/C++, the type of
-value should match that implied in the SYNOPSIS section. If you are using Fortran, it must be
-of type integer with an element size of target.
-.TP
-pe
-An integer that indicates the PE number upon which target is to be updated. If you
-are using Fortran, it must be a default integer value.
-.TP
-target
-A symmetric array of length nreduce elements to receive the results of the
-reduction operations.
-.br
-The data type of target varies with the version of the reduction routine being called and the
-language used. When calling from C/C++, refer to the SYNOPSIS section for data type
-information. When calling from Fortran, the target data types are as follows:
-.RS
-.TP
-\fBshmem_comp4_sum_to_all:\fP COMPLEX(KIND=4).
-.TP
-\fBshmem_comp8_sum_to_all:\fP Complex. If you are using Fortran, it must be
-a default complex value.
-.TP
-\fBshmem_int4_sum_to_all:\fP INTEGER(KIND=4).
-.TP
-\fBshmem_int8_sum_to_all:\fP Integer. If you are using Fortran, it must be a
-default integer value.
-.TP
-\fBshmem_real4_sum_to_all:\fP REAL(KIND=4).
-.TP
-\fBshmem_real8_sum_to_all:\fP Real. If you are using Fortran, it must be a
-default real value.
-.TP
-\fBshmem_real16_sum_to_all:\fP Real. If you are using Fortran, it must be a
-default real value.
-.RE
-.RS
-.PP
-.RE
-.TP
-source
-A symmetric array, of length nreduce elements, that contains one element for
-each separate reduction operation. The source argument must have the same data type as
-target.
-.TP
-nreduce
-The number of elements in the target and source arrays. nreduce must be of
-type integer. If you are using Fortran, it must be a default integer value.
-.TP
-PE_start
-The lowest virtual PE number of the active set of PEs. PE_start must be of
-type integer. If you are using Fortran, it must be a default integer value.
-.TP
-logPE_stride
-The log (base 2) of the stride between consecutive virtual PE numbers in
-the active set. logPE_stride must be of type integer. If you are using Fortran, it must be a
-default integer value.
-.TP
-PE_size
-The number of PEs in the active set. PE_size must be of type integer. If you
-are using Fortran, it must be a default integer value.
-.TP
-pWrk
-A symmetric work array. The pWrk argument must have the same data type as
-target. In C/C++, this contains max(nreduce/2 + 1,
-_SHMEM_REDUCE_MIN_WRKDATA_SIZE) elements. In Fortran, this contains
-max(nreduce/2 + 1, SHMEM_REDUCE_MIN_WRKDATA_SIZE) elements.
-.TP
-pSync
-A symmetric work array. In C/C++, pSync is of type long and size
-_SHMEM_REDUCE_SYNC_SIZE. In Fortran, pSync is of type integer and size
-SHMEM_REDUCE_SYNC_SIZE. It must be a default integer value. Every element of this array
-must be initialized with the value _SHMEM_SYNC_VALUE (in C/C++) or
-SHMEM_SYNC_VALUE (in Fortran) before any of the PEs in the active set enter the reduction
-routine.
-.PP
-The values of arguments nreduce, PE_start, logPE_stride, and PE_size must be equal on all
-PEs in the active set. The same target and source arrays, and the same pWrk and pSync work
-arrays, must be passed to all PEs in the active set.
-.PP
-Before any PE calls a reduction routine, you must ensure that the following conditions exist
-(synchronization via a barrier or some other method is often needed to ensure this): The
-pWrk and pSync arrays on all PEs in the active set are not still in use from a prior call to a
-collective SHMEM routine. The target array on all PEs in the active set is ready to accept the
-results of the reduction.
-.PP
-Upon return from a reduction routine, the following are true for the local PE: The target array
-is updated. The values in the pSync array are restored to the original values.
-.SH NOTES
-
-The terms collective, symmetric, and cache aligned are defined in \fIintro_shmem\fP(3)\&.
-.PP
-All SHMEM reduction routines reset the values in pSync before they return, so a particular
-pSync buffer need only be initialized the first time it is used.
-.PP
-You must ensure that the pSync array is not being updated on any PE in the active set while
-any of the PEs participate in processing of a SHMEM reduction routine. Be careful of the
-following situations: If the pSync array is initialized at run time, some type of
-synchronization is needed to ensure that all PEs in the working set have initialized pSync
-before any of them enter a SHMEM routine called with the pSync synchronization array. A
-pSync or pWrk array can be reused in a subsequent reduction routine call only if none
-of the PEs in the active set are still processing a prior reduction routine call that used the
-same pSync or pWrk arrays. In general, this can be assured only by doing some
-type of synchronization. However, in the special case of reduction routines being called with
-the same active set, you can allocate two pSync and pWrk arrays and alternate between them
-on successive calls.
-.SH EXAMPLES
-
-\fBExample 1:\fP
-This Fortran example statically initializes the pSync array and finds the
-sum of the real variable FOO across all even PEs.
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER PSYNC(SHMEM_REDUCE_SYNC_SIZE)
-DATA PSYNC /SHMEM_REDUCE_SYNC_SIZE*SHMEM_SYNC_VALUE/
-PARAMETER (NR=1)
-REAL FOO, FOOSUM, PWRK(MAX(NR/2+1,SHMEM_REDUCE_MIN_WRKDATA_SIZE))
-COMMON /COM/ FOO, FOOSUM, PWRK
-INTRINSIC MY_PE
-
-IF ( MOD(MY_PE(),2) .EQ. 0) THEN
-  CALL SHMEM_INT4_SUM_TO_ALL(FOOSUM, FOO, NR, 0, 1, N$PES/2,
-  & PWRK, PSYNC)
-  PRINT *, 'Result on PE ', MY_PE(), ' is ', FOOSUM
-ENDIF
-.Ve
-\fBExample 2:\fP
-Consider the following C/C++ call:
-.Vb
-shmem_int_sum_to_all( target, source, 3, 0, 0, 8, pwrk, psync );
-.Ve
-The preceding call is more efficient, but semantically equivalent to, the combination of the
-following calls:
-.Vb
-shmem_int_sum_to_all(&(target[0]), &(source[0]), 1, 0, 0, 8,
-  pwrk1, psync1);
-shmem_int_sum_to_all(&(target[1]), &(source[1]), 1, 0, 0, 8,
-  pwrk2, psync2);
-shmem_int_sum_to_all(&(target[2]), &(source[2]), 1, 0, 0, 8,
-  pwrk1, psync1);
-
-Note that two sets of pWrk and pSync arrays are used alternately because no
-synchronization is done between calls.
-.Ve
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_short_wait.3in b/oshmem/shmem/man/man3/shmem_short_wait.3in
deleted file mode 100644
index 03267ffbc55..00000000000
--- a/oshmem/shmem/man/man3/shmem_short_wait.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_wait.3
diff --git a/oshmem/shmem/man/man3/shmem_short_wait_until.3in b/oshmem/shmem/man/man3/shmem_short_wait_until.3in
deleted file mode 100644
index 03267ffbc55..00000000000
--- a/oshmem/shmem/man/man3/shmem_short_wait_until.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_wait.3
diff --git a/oshmem/shmem/man/man3/shmem_short_xor_to_all.3in b/oshmem/shmem/man/man3/shmem_short_xor_to_all.3in
deleted file mode 100644
index be88b2c4d2e..00000000000
--- a/oshmem/shmem/man/man3/shmem_short_xor_to_all.3in
+++ /dev/null
@@ -1,215 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2014-2016 Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_XOR" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmem_comp4_xor_to_all\fP(3),
-\fIshmem_int_xor_to_all\fP(3),
-\fIshmem_int4_xor_to_all\fP(3),
-\fIshmem_int8_xor_to_all\fP(3),
-\fIshmem_long_xor_to_all\fP(3),
-\fIshmem_longlong_xor_to_all\fP(3),
-\fIshmem_short_xor_to_all\fP(3)
-\- Performs a bitwise XOR operation on symmetric
-arrays over the active set of PEs.
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-void shmem_int_xor_to_all(int *target, const int *source,
-  int nreduce, int PE_start, int logPE_stride, int PE_size,
-  int *pWrk, long *pSync);
-
-void shmem_long_xor_to_all(long *target, const long *source,
-  int nreduce, int PE_start, int logPE_stride, int PE_size,
-  long *pWrk, long *pSync);
-
-void shmem_longlong_xor_to_all(long long *target,
-  const long long *source, int nreduce, int PE_start, int logPE_stride,
-  int PE_size, long long *pWrk, long *pSync);
-
-void shmem_short_xor_to_all(short *target, const short *source,
-  int nreduce, int PE_start, int logPE_stride, int PE_size,
-  short *pWrk, long *pSync);
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER pSync(SHMEM_REDUCE_SYNC_SIZE)
-INTEGER nreduce, PE_start, logPE_stride, PE_size
-
-CALL SHMEM_COMP4_XOR_TO_ALL(target, source, nreduce,
-& PE_start, logPE_stride, PE_size, pWrk, pSync)
-
-CALL SHMEM_INT4_XOR_TO_ALL(target, source, nreduce,
-& PE_start, logPE_stride, PE_size, pWrk, pSync)
-
-CALL SHMEM_INT8_XOR_TO_ALL(target, source, nreduce,
-& PE_start, logPE_stride, PE_size, pWrk, pSync)
-.Ve
-.SH DESCRIPTION
-
-The shared memory (SHMEM) reduction routines compute one or more reductions across
-symmetric arrays on multiple virtual PEs. A reduction performs an associative binary
-operation across a set of values. For a list of other SHMEM reduction routines, see
-\fIintro_shmem\fP(3)\&.
-.PP
-As with all SHMEM collective routines, each of these routines assumes that only PEs in the
-active set call the routine. If a PE not in the active set calls a SHMEM collective routine,
-undefined behavior results.
-.PP
-The nreduce argument determines the number of separate reductions to perform. The source
-array on all PEs in the active set provides one element for each reduction. The results of the
-reductions are placed in the target array on all PEs in the active set. The active set is defined
-by the PE_start, logPE_stride, PE_size triplet.
-.PP
-The source and target arrays may be the same array, but they may not be overlapping arrays.
-.PP
-The arguments are as follows:
-.TP
-target
-A symmetric array of length nreduce elements to receive the results of the
-reduction operations.
-The data type of target varies with the version of the reduction routine being called and the
-language used. When calling from C/C++, refer to the SYNOPSIS section for data type
-information. When calling from Fortran, the target data types are as follows:
-.RS
-.TP
-\fBshmem_comp8_xor_to_all:\fP Complex, with an element size equal to two 8\-
-byte real values
-.TP
-\fBshmem_comp4_xor_to_all:\fP Complex, with an element size equal to two 4\-
-byte real values
-.TP
-\fBshmem_int8_xor_to_all:\fP Integer, with an element size of 8 bytes
-.TP
-\fBshmem_int4_xor_to_all:\fP Integer, with an element size of 4 bytes
-.TP
-\fBshmem_real8_xor_to_all:\fP Real, with an element size of 8 bytes
-.TP
-\fBshmem_real4_xor_to_all:\fP Real, with an element size of 4 bytes
-.RE
-.RS
-.PP
-.RE
-.TP
-source
-A symmetric array, of length nreduce elements, that contains one element for
-each separate reduction operation. The source argument must have the same data type as
-target.
-.TP
-nreduce
-The number of elements in the target and source arrays. nreduce must be of
-type integer. If you are using Fortran, it must be a default integer value.
-.TP
-PE_start
-The lowest virtual PE number of the active set of PEs. PE_start must be of
-type integer. If you are using Fortran, it must be a default integer value.
-.TP
-logPE_stride
-The log (base 2) of the stride between consecutive virtual PE numbers in
-the active set. logPE_stride must be of type integer. If you are using Fortran, it must be a
-default integer value.
-.TP
-PE_size
-The number of PEs in the active set. PE_size must be of type integer. If you
-are using Fortran, it must be a default integer value.
-.TP
-pWrk
-A symmetric work array. The pWrk argument must have the same data type as
-target. In C/C++, this contains max(nreduce/2 + 1,
-_SHMEM_REDUCE_MIN_WRKDATA_SIZE) elements. In Fortran, this contains
-max(nreduce/2 + 1, SHMEM_REDUCE_MIN_WRKDATA_SIZE) elements.
-.TP
-pSync
-A symmetric work array. In C/C++, pSync is of type long and size
-_SHMEM_REDUCE_SYNC_SIZE. In Fortran, pSync is of type integer and size
-SHMEM_REDUCE_SYNC_SIZE. If you are using Fortran, it must be a default integer value.
-Every element of this array must be initialized with the value _SHMEM_SYNC_VALUE (in
-C/C++) or SHMEM_SYNC_VALUE (in Fortran) before any of the PEs in the active set enter
-the reduction routine.
-.PP
-The values of arguments nreduce, PE_start, logPE_stride, and PE_size must be equal on all
-PEs in the active set. The same target and source arrays, and the same pWrk and pSync
-work arrays, must be passed to all PEs in the active set.
-.PP
-Before any PE calls a reduction routine, you must ensure that the following conditions exist
-(synchronization via a barrier or some other method is often needed to ensure this): The
-pWrk and pSync arrays on all PEs in the active set are not still in use from a prior call to a
-collective SHMEM routine. The target array on all PEs in the active set is ready to accept the
-results of the reduction.
-.PP
-Upon return from a reduction routine, the following are true for the local PE: The target array
-is updated. The values in the pSync array are restored to the original values.
-.SH NOTES
-
-The terms collective, symmetric, and cache aligned are defined in \fIintro_shmem\fP(3)\&.
-All SHMEM reduction routines reset the values in pSync before they return, so a particular
-pSync buffer need only be initialized the first time it is used.
-.PP
-You must ensure that the pSync array is not being updated on any PE in the active set while
-any of the PEs participate in processing of a SHMEM reduction routine. Be careful of the
-following situations: If the pSync array is initialized at run time, some type of
-synchronization is needed to ensure that all PEs in the working set have initialized pSync
-before any of them enter a SHMEM routine called with the pSync synchronization array. A
-pSync or pWrk array can be reused in a subsequent reduction routine call only if none of the
-PEs in the active set are still processing a prior reduction routine call that used the same
-pSync or pWrk arrays. In general, this can be assured only by doing some type of
-synchronization. However, in the special case of reduction routines being called with the
-same active set, you can allocate two pSync and pWrk arrays and alternate between them on
-successive calls.
-.SH EXAMPLES
-
-\fBExample 1:\fP
-This Fortran example statically initializes the pSync array and computes
-the exclusive OR of variable FOO across all even PEs.
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER PSYNC(SHMEM_REDUCE_SYNC_SIZE)
-DATA PSYNC /SHMEM_REDUCE_SYNC_SIZE*SHMEM_SYNC_VALUE/
-PARAMETER (NR=1)
-REAL FOO, FOOXOR, PWRK(MAX(NR/2+1,SHMEM_REDUCE_MIN_WRKDATA_SIZE))
-COMMON /COM/ FOO, FOOXOR, PWRK
-INTRINSIC MY_PE
-
-IF ( MOD(MY_PE(),2) .EQ. 0) THEN
-  CALL SHMEM_REAL8_XOR_TO_ALL(FOOXOR, FOO, NR, 0, 1, N$PES/2,
-  & PWRK, PSYNC)
-  PRINT *, 'Result on PE ', MY_PE(), ' is ', FOOXOR
-ENDIF
-.Ve
-\fBExample 2:\fP
-Consider the following C/C++ call:
-.Vb
-shmem_short_xor_to_all( target, source, 3, 0, 0, 8, pwrk, psync );
-.Ve
-The preceding call is more efficient, but semantically equivalent to, the combination of the
-following calls:
-.Vb
-shmem_short_xor_to_all(&(target[0]), &(source[0]), 1, 0, 0, 8,
-  pwrk1, psync1);
-shmem_short_xor_to_all(&(target[1]), &(source[1]), 1, 0, 0, 8,
-  pwrk2, psync2);
-shmem_short_xor_to_all(&(target[2]), &(source[2]), 1, 0, 0, 8,
-  pwrk1, psync1);
-.Ve
-Note that two sets of pWrk and pSync arrays are used alternately because no synchronization
-is done between calls.
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_swap.3in b/oshmem/shmem/man/man3/shmem_swap.3in
deleted file mode 100644
index bc8daafe0ed..00000000000
--- a/oshmem/shmem/man/man3/shmem_swap.3in
+++ /dev/null
@@ -1,115 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_SWAP" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmem_double_swap\fP(3),
-\fIshmem_float_swap\fP(3),
-\fIshmem_int_swap\fP(3),
-\fIshmem_long_swap\fP(3),
-\fIshmem_swap\fP(3),
-\fIshmem_int4_swap\fP(3),
-\fIshmem_int8_swap\fP(3),
-\fIshmem_real4_swap\fP(3),
-\fIshmem_real8_swap\fP(3),
-\fIshmem_longlong_swap\fP(3)
-\- Performs an atomic swap to a remote data object
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-double shmem_double_swap(double *target, double value,
-  int pe);
-
-float shmem_float_swap(float *target, float value, int pe);
-
-int shmem_int_swap(int *target, int value, int pe);
-
-long shmem_long_swap(long *target, long value, int pe);
-
-long long shmem_longlong_swap(long long *target,
-  long long value, int pe);
-
-long shmem_swap(long *target, long value, int pe);
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-INTEGER pe
-
-INTEGER SHMEM_SWAP
-ires = SHMEM_SWAP(target, value, pe)
-
-INTEGER(KIND=4) SHMEM_INT4_SWAP
-ires = SHMEM_INT4_SWAP(target, value, pe)
-
-INTEGER(KIND=8) SHMEM_INT8_SWAP
-ires = SHMEM_INT8_SWAP(target, value, pe)
-
-REAL(KIND=4) SHMEM_REAL4_SWAP
-res = SHMEM_REAL4_SWAP(target, value, pe)
-
-REAL(KIND=8) SHMEM_REAL8_SWAP
-res = SHMEM_REAL8_SWAP(target, value, pe)
-.Ve
-.SH DESCRIPTION
-
-The atomic swap routines write \fBvalue\fP
-to address target on PE \fBpe\fP,
-and return
-the previous contents of \fBtarget\fP
-in one atomic operation.
-.PP
-The arguments are as follows:
-.TP
-target
-The remotely accessible integer data object to be updated on the remote PE. If
-you are using C/C++, the type of target should match that implied in the SYNOPSIS section. If
-you are using Fortran, it must be of the following type:
-.RS
-.TP
-\fBSHMEM_SWAP:\fP Integer of default kind
-.TP
-\fBSHMEM_INT4_SWAP:\fP 4\-byte integer
-.TP
-\fBSHMEM_INT8_SWAP:\fP 8\-byte integer
-.TP
-\fBSHMEM_REAL4_SWAP:\fP 4\-byte real
-.TP
-\fBSHMEM_REAL8_SWAP:\fP 8\-byte real
-.RE
-.RS
-.PP
-.RE
-.TP
-value
-Value to be atomically written to the remote PE. value is the same type as target.
-.TP
-pe
-An integer that indicates the PE number on which target is to be updated. If you are
-using Fortran, it must be a default integer value.
-.PP
-.SH NOTES
-
-The term remotely accessible is defined in \fIintro_shmem\fP(3)\&.
-.SH RETURN VALUES
-
-The contents that had been at the target address on the remote PE prior to the swap is
-returned.
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_test_lock.3in b/oshmem/shmem/man/man3/shmem_test_lock.3in
deleted file mode 100644
index 9a1ee8ffc2e..00000000000
--- a/oshmem/shmem/man/man3/shmem_test_lock.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_set_lock.3
diff --git a/oshmem/shmem/man/man3/shmem_udcflush.3in b/oshmem/shmem/man/man3/shmem_udcflush.3in
deleted file mode 100644
index 3edeb74023d..00000000000
--- a/oshmem/shmem/man/man3/shmem_udcflush.3in
+++ /dev/null
@@ -1,94 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_CACHE" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmem_clear_cache_inv\fP(3),
-\fIshmem_set_cache_inv\fP(3),
-\fIshmem_set_cache_line_inv\fP(3),
-\fIshmem_udcflush\fP(3),
-\fIshmem_udcflush_line\fP(3)
-\- Controls data cache utilities
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-void shmem_clear_cache_inv(void);
-void shmem_clear_cache_line_inv(void *target);
-void shmem_set_cache_inv(void);
-void shmem_set_cache_line_inv(void *target);
-void shmem_udcflush(void);
-void shmem_udcflush_line(void *target);
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-CALL SHMEM_CLEAR_CACHE_INV
-CALL SHMEM_CLEAR_CACHE_LINE_INV(target)
-CALL SHMEM_SET_CACHE_INV
-CALL SHMEM_SET_CACHE_LINE_INV(target)
-
-CALL SHMEM_UDCFLUSH
-CALL SHMEM_UDCFLUSH_LINE(target)
-.Ve
-.SH DESCRIPTION
-
-The following argument is passed to the cache line control routines:
-.TP
-target
-A data object that is local to the processing element (PE). target can be of
-any noncharacter type. If you are using Fortran, it can be of any kind.
-.PP
-\fBshmem_clear_cache_inv\fP
-disables automatic cache coherency mode previously
-enabled by shmem_set_cache_inv or shmem_set_cache_line_inv.
-.PP
-\fBshmem_clear_cache_line_inv\fP
-disables automatic cache coherency mode for the
-cache line associated with the address of \fBtarget\fP
-only.
-.PP
-\fBshmem_set_cache_inv\fP
-enables the OpenSHMEM API to automatically decide the
-best strategy for cache coherency.
-.PP
-\fBshmem_set_cache_line_inv\fP
-enables automatic cache coherency mode for the
-cache line associated with the address of \fBtarget\fP
-only.
-.PP
-\fBshmem_clear_cache_inv\fP
-disables automatic cache coherency mode previously
-enabled by shmem_set_cache_inv or shmem_set_cache_line_inv.
-.PP
-\fBshmem_udcflush\fP
-makes the entire user data cache coherent.
-.PP
-\fBshmem_udcflush_line\fP
-makes coherent the cache line that corresponds with
-the address specified by target.
-.PP
-.SH NOTES
-
-These routines have been retained for improved backward compatability with legacy
-architectures.
-.PP
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3),
-\fIshmem_put\fP(3),
-\fIshmem_swap\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_udcflush_line.3in b/oshmem/shmem/man/man3/shmem_udcflush_line.3in
deleted file mode 100644
index 4a6a361ef97..00000000000
--- a/oshmem/shmem/man/man3/shmem_udcflush_line.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_udcflush.3
diff --git a/oshmem/shmem/man/man3/shmem_wait.3in b/oshmem/shmem/man/man3/shmem_wait.3in
deleted file mode 100644
index c664ae4c1ac..00000000000
--- a/oshmem/shmem/man/man3/shmem_wait.3in
+++ /dev/null
@@ -1,205 +0,0 @@
-.\" -*- nroff -*-
-.\" Copyright (c) 2015      University of Houston.  All rights reserved.
-.\" Copyright (c) 2015      Mellanox Technologies, Inc.
-.\" $COPYRIGHT$
-.de Vb
-.ft CW
-.nf
-..
-.de Ve
-.ft R
-
-.fi
-..
-.TH "SHMEM\\_WAIT" "3" "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
-.SH NAME
-
-\fIshmem_int_wait\fP(3),
-\fIshmem_int_wait\fP(3)_until,
-\fIshmem_int4_wait\fP(3),
-\fIshmem_int4_wait\fP(3)_until,
-\fIshmem_int8_wait\fP(3),
-\fIshmem_int8_wait\fP(3)_until,
-\fIshmem_long_wait\fP(3),
-\fIshmem_long_wait\fP(3)_until,
-\fIshmem_longlong_wait\fP(3),
-\fIshmem_longlong_wait\fP(3)_until,
-\fIshmem_short_wait\fP(3),
-\fIshmem_short_wait\fP(3)_until,
-\fIshmem_wait\fP(3),
-\fIshmem_wait\fP(3)_until
-\- Waits for a variable on the local processing element (PE) to change
-.SH SYNOPSIS
-
-C or C++:
-.Vb
-#include <mpp/shmem.h>
-
-void shmem_int_wait(volatile int *var, int value);
-
-void shmem_int_wait_until(volatile int *var, int cond, int value);
-
-void shmem_long_wait(volatile long *var, long value);
-
-void shmem_long_wait_until(volatile long *var, int cond, long value);
-
-void shmem_longlong_wait(volatile long long *var, long long value);
-
-void shmem_longlong_wait_until(volatile long long *var, int cond,
-  long long value);
-
-void shmem_short_wait(volatile short *var, short value);
-
-void shmem_short_wait_until(volatile short *var, int cond,
-  short value);
-
-void shmem_wait(volatile long *ivar, long cmp_value);
-
-void shmem_wait_until(volatile long *ivar, int cmp, long value);
-.Ve
-Fortran:
-.Vb
-INCLUDE "mpp/shmem.fh"
-
-CALL SHMEM_INT4_WAIT(ivar, cmp_value)
-
-CALL SHMEM_INT4_WAIT_UNTIL(ivar, cmp, cmp_value)
-
-CALL SHMEM_INT8_WAIT(ivar, cmp_value)
-
-CALL SHMEM_INT8_WAIT_UNTIL(ivar, cmp, cmp_value)
-
-CALL SHMEM_WAIT(ivar, cmp_value)
-
-CALL SHMEM_WAIT_UNTIL(ivar, cmp, cmp_value)
-.Ve
-.SH DESCRIPTION
-
-shmem_wait and shmem_wait_until wait for \fBivar\fP
-to be changed by a remote write
-or atomic swap issued by a different processor. These routines can be used for point\-to\-
-point directed synchronization. A call to shmem_wait does not return until some other
-processor writes a value, not equal to cmp_value, into \fBivar\fP
-on the waiting
-processor. A call to shmem_wait_until does not return until some other processor changes
-\fBivar\fP
-to satisfy the condition implied by cmp and cmp_value. This mechanism is
-useful when a processor needs to tell another processor that it has completed some action.
-.PP
-The arguments are as follows:
-.TP
-target
-The remotely accessible integer data object to be updated on the remote PE. If
-you are using C/C++, the type of target should match that implied in the SYNOPSIS section.
-If you are using the Fortran compiler, it must be of type integer with an element size of 4
-bytes for SHMEM_INT4_ADD and 8 bytes for SHMEM_INT8_ADD.
-.TP
-value
-The value to be atomically added to target. If you are using C/C++, the type of
-value should match that implied in the SYNOPSIS section. If you are using Fortran, it must be
-of type integer with an element size of target.
-.TP
-pe
-An integer that indicates the PE number upon which target is to be updated. If you
-are using Fortran, it must be a default integer value.
-.TP
-ivar
-A remotely accessible integer variable that is being updated by another PE. If you
-are using C/C++, the type of ivar should match that implied in the SYNOPSIS section. If you
-are using Fortran, ivar must be a specific sized integer type according to
-the function being called, as follows:
-.RS
-.TP
-\fBshmem_wait, shmem_wait_until:\fP default INTEGER
-.TP
-\fBshmem_int4_wait, shmem_int4_wait_until:\fP INTEGER*4
-.TP
-\fBshmem_int8_wait, shmem_int8_wait_until:\fP INTEGER*8
-.RE
-.RS
-.PP
-.RE
-.TP
-cmp
-The compare operator that compares ivar with cmp_value. cmp must be of type
-integer. If you are using Fortran, it must be of default kind. If you are using C/C++, the type
-of cmp should match that implied in the SYNOPSIS section. The following cmp values are
-supported:
-.RS
-.TP
-SHMEM_CMP_EQ
-Equal
-.TP
-SHMEM_CMP_NE
-Not equal
-.TP
-SHMEM_CMP_GT
-Greater than
-.TP
-SHMEM_CMP_LE
-Less than or equal to
-.TP
-SHMEM_CMP_LT
-Less than
-.TP
-SHMEM_CMP_GE
-Greater than or equal to
-.RE
-.RS
-.PP
-.RE
-.TP
-cmp_value
-cmp_value must be of type integer. If you are using C/C++, the type of
-cmp_value should match thatimplied in the SYNOPSIS section. If you are using Fortran,
-cmp_value must be an integer of the same size and kind as ivar.
-The shmem_wait routines return when ivar is no longer equal to cmp_value.
-The shmem_wait_until routines return when the compare condition is true. The compare
-condition is defined by the ivar argument compared with the cmp_value using the
-comparison operator, cmp.
-.PP
-.SH EXAMPLES
-
-\fBExample 1:\fP
-The following call returns when variable ivar is not equal to 100:
-.Vb
-INTEGER*8 IVAR
-
-CALL SHMEM_INT8_WAIT(IVAR, INT8(100))
-.Ve
-\fBExample 2:\fP
-The following call to SHMEM_INT8_WAIT_UNTIL is equivalent to the
-call to SHMEM_INT8_WAIT in example 1:
-.Vb
-INTEGER*8 IVAR
-
-CALL SHMEM_INT8_WAIT_UNTIL(IVAR, SHMEM_CMP_NE, INT8(100))
-.Ve
-\fBExample 3:\fP
-The following C/C++ call waits until the sign bit in ivar is set by a
-transfer from a remote PE:
-.Vb
-int ivar;
-
-shmem_int_wait_until(&ivar, SHMEM_CMP_LT, 0);
-.Ve
-\fBExample 4:\fP
-The following Fortran example is in the context of a subroutine:
-.Vb
-SUBROUTINE EXAMPLE()
-  INTEGER FLAG_VAR
-  COMMON/FLAG/FLAG_VAR
-  . . .
-  FLAG_VAR = FLAG_VALUE ! initialize the event variable
-  . . .
-  IF (FLAG_VAR .EQ. FLAG_VALUE) THEN
-    CALL SHMEM_WAIT(FLAG_VAR, FLAG_VALUE)
-  ENDIF
-  FLAG_VAR = FLAG_VALUE ! reset the event variable for next time
-  . . .
-END
-.Ve
-.SH SEE ALSO
-
-\fIintro_shmem\fP(3),
-\fIshmem_put\fP(3)
diff --git a/oshmem/shmem/man/man3/shmem_wait_until.3in b/oshmem/shmem/man/man3/shmem_wait_until.3in
deleted file mode 100644
index 03267ffbc55..00000000000
--- a/oshmem/shmem/man/man3/shmem_wait_until.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_wait.3
diff --git a/oshmem/shmem/man/man3/shmemalign.3in b/oshmem/shmem/man/man3/shmemalign.3in
deleted file mode 100644
index 6cdb8014e9e..00000000000
--- a/oshmem/shmem/man/man3/shmemalign.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_malloc.3
diff --git a/oshmem/shmem/man/man3/shrealloc.3in b/oshmem/shmem/man/man3/shrealloc.3in
deleted file mode 100644
index 6cdb8014e9e..00000000000
--- a/oshmem/shmem/man/man3/shrealloc.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_malloc.3
diff --git a/oshmem/shmem/man/man3/start_pes.3in b/oshmem/shmem/man/man3/start_pes.3in
deleted file mode 100644
index c50c47089ee..00000000000
--- a/oshmem/shmem/man/man3/start_pes.3in
+++ /dev/null
@@ -1 +0,0 @@
-.so man3/shmem_init.3
diff --git a/oshmem/shmem/shmem_api_logger.h b/oshmem/shmem/shmem_api_logger.h
index af7df50fe73..6daec140823 100644
--- a/oshmem/shmem/shmem_api_logger.h
+++ b/oshmem/shmem/shmem_api_logger.h
@@ -1,7 +1,7 @@
 /*
- * Copyright (c) 2013      Mellanox Technologies, Inc.
- *                         All rights reserved.
- * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013-2020      Mellanox Technologies, Inc.
+ *                              All rights reserved.
+ * Copyright (c) 2015           Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -27,7 +27,7 @@ OSHMEM_DECLSPEC extern int shmem_api_logger_output;
 #define __SPML_FILE__ __FILE__
 #endif
 
-#ifdef OPAL_ENABLE_DEBUG
+#if OPAL_ENABLE_DEBUG
 #define SHMEM_API_VERBOSE(level, ...) \
     oshmem_output_verbose(level, shmem_api_logger_output, \
         "%s:%d - %s()", __SPML_FILE__, __LINE__, __func__, __VA_ARGS__)
@@ -39,4 +39,9 @@ OSHMEM_DECLSPEC extern int shmem_api_logger_output;
     oshmem_output(shmem_api_logger_output, \
         "Error: %s:%d - %s()", __SPML_FILE__, __LINE__, __func__, __VA_ARGS__)
 
+#define SHMEM_API_WARNING(...) \
+    oshmem_output(shmem_api_logger_output, \
+        "Warning: %s:%d - %s()", __SPML_FILE__, __LINE__, __func__, __VA_ARGS__)
+
+
 #endif /*SHMEM_API_LOGGER_H*/
diff --git a/oshmem/shmem/shmem_lock.h b/oshmem/shmem/shmem_lock.h
index c338339c529..de138f45ff9 100644
--- a/oshmem/shmem/shmem_lock.h
+++ b/oshmem/shmem/shmem_lock.h
@@ -22,5 +22,8 @@ void _shmem_set_lock(void *lock, int lock_size);
 int _shmem_test_lock(void *lock, int lock_size);
 void _shmem_clear_lock(void *lock, int lock_size);
 
+void _shmem_mcs_set_lock(long *lock);
+void _shmem_mcs_clear_lock(long *lock);
+int  _shmem_mcs_test_lock(long *lock);
 
 #endif /*SHMEM_LOCK_H*/
diff --git a/oshmem/tools/oshmem_info/Makefile.am b/oshmem/tools/oshmem_info/Makefile.am
index 54335fe7cbf..e569d7538c0 100644
--- a/oshmem/tools/oshmem_info/Makefile.am
+++ b/oshmem/tools/oshmem_info/Makefile.am
@@ -1,11 +1,12 @@
 #
 # Copyright (c) 2014      Mellanox Technologies, Inc.
 #                         All rights reserved.
-# Copyright (c) 2014-2020 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2014-2022 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2016      IBM Corporation.  All rights reserved.
 # Copyright (c) 2017      Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
 # Copyright (c) 2019      Intel, Inc.  All rights reserved.
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -44,26 +45,15 @@ endif
 
 include $(top_srcdir)/Makefile.ompi-rules
 
-nodist_man_MANS =
 bin_PROGRAMS =
 
-man_pages = oshmem_info.1
-
-EXTRA_DIST = $(man_pages:.1=.1in)
-
 if PROJECT_OSHMEM
-# Only build/install the binary and man pages if we're building oshmem
+# Only build/install the binary pages if we're building oshmem
 bin_PROGRAMS += oshmem_info
-nodist_man_MANS += $(man_pages)
-dist_ompidata_DATA = \
-     help-oshmem-info.txt
 endif
 
-
-# Ensure that the man pages are rebuilt if the opal_config.h file
-# changes; a "good enough" way to know if configure was run again (and
-# therefore the release date or version may have changed)
-$(nodist_man_MANS): $(top_builddir)/opal/include/opal_config.h
+EXTRA_DIST = \
+     help-oshmem-info.txt
 
 oshmem_info_SOURCES = \
         oshmem_info.h \
@@ -72,10 +62,4 @@ oshmem_info_SOURCES = \
 
 oshmem_info_LDADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la
 oshmem_info_LDADD += $(top_builddir)/oshmem/liboshmem.la
-oshmem_info_LDADD += $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
-
-clean-local:
-	test -z "$(OMPI_CXX_TEMPLATE_REPOSITORY)" || rm -rf $(OMPI_CXX_TEMPLATE_REPOSITORY)
-
-distclean-local:
-	rm -f $(man_pages)
+oshmem_info_LDADD += $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
diff --git a/oshmem/tools/oshmem_info/param.c b/oshmem/tools/oshmem_info/param.c
index bcc85693742..601ceb167f7 100644
--- a/oshmem/tools/oshmem_info/param.c
+++ b/oshmem/tools/oshmem_info/param.c
@@ -40,7 +40,7 @@
 #include "opal/runtime/opal_info_support.h"
 
 #include "ompi/tools/ompi_info/ompi_info.h"
-#include "ompi/include/mpi_portable_platform.h"
+#include "opal/opal_portable_platform.h"
 
 #include "oshmem/tools/oshmem_info/oshmem_info.h"
 
@@ -147,9 +147,9 @@ void oshmem_info_do_config(bool want_all)
     opal_info_out("C compiler absolute", "compiler:c:absolute",
                   OPAL_CC_ABSOLUTE);
     opal_info_out("C compiler family name", "compiler:c:familyname",
-                  _STRINGIFY(OPAL_BUILD_PLATFORM_COMPILER_FAMILYNAME));
+                  PLATFORM_STRINGIFY(PLATFORM_COMPILER_FAMILYNAME));
     opal_info_out("C compiler version", "compiler:c:version",
-                  _STRINGIFY(OPAL_BUILD_PLATFORM_COMPILER_VERSION_STR));
+                  PLATFORM_COMPILER_VERSION_STR);
 
     if (want_all) {
         opal_info_out_int("C char size", "compiler:c:sizeof:char", sizeof(char));
@@ -334,16 +334,16 @@ void oshmem_info_do_config(bool want_all)
         opal_info_out("Build LDFLAGS", "option:build:ldflags", OMPI_BUILD_LDFLAGS);
         opal_info_out("Build LIBS", "option:build:libs", OMPI_BUILD_LIBS);
 
-        opal_info_out("Wrapper extra CFLAGS", "option:wrapper:extra_cflags",
-                      WRAPPER_EXTRA_CFLAGS);
-        opal_info_out("Wrapper extra CXXFLAGS", "option:wrapper:extra_cxxflags",
-                      WRAPPER_EXTRA_CXXFLAGS);
-        opal_info_out("Wrapper extra FCFLAGS", "option:wrapper:extra_fcflags",
-                      WRAPPER_EXTRA_FCFLAGS);
-        opal_info_out("Wrapper extra LDFLAGS", "option:wrapper:extra_ldflags",
-                      WRAPPER_EXTRA_LDFLAGS);
-        opal_info_out("Wrapper extra LIBS", "option:wrapper:extra_libs",
-                      WRAPPER_EXTRA_LIBS);
+        opal_info_out("Wrapper CFLAGS", "option:wrapper:cflags",
+                      OSHMEM_WRAPPER_CFLAGS);
+        opal_info_out("Wrapper CXXFLAGS", "option:wrapper:cxxflags",
+                      OSHMEM_WRAPPER_CXXFLAGS);
+        opal_info_out("Wrapper FCFLAGS", "option:wrapper:fcflags",
+                      OSHMEM_WRAPPER_FCFLAGS);
+        opal_info_out("Wrapper LDFLAGS", "option:wrapper:ldflags",
+                      OSHMEM_WRAPPER_LDFLAGS);
+        opal_info_out("Wrapper LIBS", "option:wrapper:libs",
+                      OSHMEM_WRAPPER_LIBS);
     }
 
     opal_info_out("Internal debug support", "option:debug", debug);
diff --git a/oshmem/tools/wrappers/Makefile.am b/oshmem/tools/wrappers/Makefile.am
index dd97c5a9faf..f026266eb6e 100644
--- a/oshmem/tools/wrappers/Makefile.am
+++ b/oshmem/tools/wrappers/Makefile.am
@@ -12,10 +12,10 @@
 
 include $(top_srcdir)/Makefile.ompi-rules
 
-man_pages = oshcc.1 shmemcc.1 oshc++.1 shmemc++.1 oshcxx.1 shmemcxx.1 oshfort.1 shmemfort.1
-
 if PROJECT_OSHMEM
-man_MANS = $(man_pages)
+
+pkgconfigdir = $(libdir)/pkgconfig
+pkgconfig_DATA = oshmem.pc oshmem-c.pc oshmem-cxx.pc oshmem-fort.pc
 
 nodist_oshmemdata_DATA =  \
 	shmemcc-wrapper-data.txt \
@@ -81,23 +81,11 @@ install-data-cs: mk_pkgdatadir
 
 uninstall-local-cs:
 	rm -f $(DESTDIR)$(bindir)/shmemCC$(EXEEXT) \
-	$(DESTDIR)$(mandir)/man1/shmemCC.1 \
 	$(DESTDIR)$(pkgdatadir)/shmemCC-wrapper-data.txt
 	rm -f $(DESTDIR)$(bindir)/oshCC$(EXEEXT) \
-	$(DESTDIR)$(mandir)/man1/oshCC.1 \
 	$(DESTDIR)$(pkgdatadir)/oshCC-wrapper-data.txt
 
-oshCC.1: $(top_builddir)/opal/tools/wrappers/generic_wrapper.1
-	rm -f oshCC.1
-	sed -e 's/#COMMAND#/oshCC/g' -e 's/#PROJECT#/Open SHMEM/g' -e 's/#PROJECT_SHORT#/OSHMEM/g' -e 's/#LANGUAGE#/C++/g' < $(top_builddir)/opal/tools/wrappers/generic_wrapper.1 > oshCC.1
-
-shmemCC.1: $(top_builddir)/opal/tools/wrappers/generic_wrapper.1
-	rm -f shmemCC.1
-	sed -e 's/#COMMAND#/shmemCC/g' -e 's/#PROJECT#/Open SHMEM/g' -e 's/#PROJECT_SHORT#/OSHMEM/g' -e 's/#LANGUAGE#/C++/g' < $(top_builddir)/opal/tools/wrappers/generic_wrapper.1 > shmemCC.1
-
 if CASE_SENSITIVE_FS
-man_MANS += oshCC.1 shmemCC.1
-
 targets_install_exec += install-exec-cs
 
 targets_install_data += install-data-cs
@@ -109,48 +97,4 @@ install-exec-hook: $(targets_install_exec)
 install-data-hook: $(targets_install_data)
 uninstall-local: $(targets_uninstall_local)
 
-
-########################################################
-#
-# Man page generation / handling
-#
-########################################################
-distclean-local:
-	rm -f $(man_MANS)
-
-$(top_builddir)/opal/tools/wrappers/generic_wrapper.1:
-	(cd $(top_builddir)/opal/tools/wrappers && $(MAKE) $(AM_MAKEFLAGS) generic_wrapper.1)
-
-oshcc.1: $(top_builddir)/opal/tools/wrappers/generic_wrapper.1
-	rm -f oshcc.1
-	sed -e 's/#COMMAND#/oshcc/g' -e 's/#PROJECT#/Open SHMEM/g' -e 's/#PROJECT_SHORT#/OSHMEM/g' -e 's/#LANGUAGE#/C/g' < $(top_builddir)/opal/tools/wrappers/generic_wrapper.1 > oshcc.1
-
-shmemcc.1: $(top_builddir)/opal/tools/wrappers/generic_wrapper.1
-	rm -f shmemcc.1
-	sed -e 's/#COMMAND#/shmemcc/g' -e 's/#PROJECT#/Open SHMEM/g' -e 's/#PROJECT_SHORT#/OSHMEM/g' -e 's/#LANGUAGE#/C/g' < $(top_builddir)/opal/tools/wrappers/generic_wrapper.1 > shmemcc.1
-
-oshc++.1: $(top_builddir)/opal/tools/wrappers/generic_wrapper.1
-	rm -f oshc++.1
-	sed -e 's/#COMMAND#/oshc++/g' -e 's/#PROJECT#/Open SHMEM/g' -e 's/#PROJECT_SHORT#/OSHMEM/g' -e 's/#LANGUAGE#/C++/g' < $(top_builddir)/opal/tools/wrappers/generic_wrapper.1 > oshc++.1
-
-shmemc++.1: $(top_builddir)/opal/tools/wrappers/generic_wrapper.1
-	rm -f shmemc++.1
-	sed -e 's/#COMMAND#/shmemc++/g' -e 's/#PROJECT#/Open SHMEM/g' -e 's/#PROJECT_SHORT#/OSHMEM/g' -e 's/#LANGUAGE#/C++/g' < $(top_builddir)/opal/tools/wrappers/generic_wrapper.1 > shmemc++.1
-
-oshcxx.1: $(top_builddir)/opal/tools/wrappers/generic_wrapper.1
-	rm -f oshcxx.1
-	sed -e 's/#COMMAND#/oshcxx/g' -e 's/#PROJECT#/Open SHMEM/g' -e 's/#PROJECT_SHORT#/OSHMEM/g' -e 's/#LANGUAGE#/C++/g' < $(top_builddir)/opal/tools/wrappers/generic_wrapper.1 > oshcxx.1
-
-shmemcxx.1: $(top_builddir)/opal/tools/wrappers/generic_wrapper.1
-	rm -f shmemcxx.1
-	sed -e 's/#COMMAND#/shmemcxx/g' -e 's/#PROJECT#/Open SHMEM/g' -e 's/#PROJECT_SHORT#/OSHMEM/g' -e 's/#LANGUAGE#/C++/g' < $(top_builddir)/opal/tools/wrappers/generic_wrapper.1 > shmemcxx.1
-
-oshfort.1: $(top_builddir)/opal/tools/wrappers/generic_wrapper.1
-	rm -f oshfort.1
-	sed -e 's/#COMMAND#/oshfort/g' -e 's/#PROJECT#/Open SHMEM/g' -e 's/#PROJECT_SHORT#/OSHMEM/g' -e 's/#LANGUAGE#/Fortran/g' < $(top_builddir)/opal/tools/wrappers/generic_wrapper.1 > oshfort.1
-
-shmemfort.1: $(top_builddir)/opal/tools/wrappers/generic_wrapper.1
-	rm -f shmemfort.1
-	sed -e 's/#COMMAND#/shmemfort/g' -e 's/#PROJECT#/Open SHMEM/g' -e 's/#PROJECT_SHORT#/OSHMEM/g' -e 's/#LANGUAGE#/Fortran/g' < $(top_builddir)/opal/tools/wrappers/generic_wrapper.1 > shmemfort.1
-
 endif # PROJECT_OSHMEM
diff --git a/oshmem/tools/wrappers/oshmem-c.pc.in b/oshmem/tools/wrappers/oshmem-c.pc.in
new file mode 100644
index 00000000000..a600e3ff2a9
--- /dev/null
+++ b/oshmem/tools/wrappers/oshmem-c.pc.in
@@ -0,0 +1,21 @@
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2016      IBM Corporation.  All rights reserved.
+# Copyright (c) 2018      Research Organization for Information Science
+#                         and Technology (RIST).  All rights reserved.
+#
+Name: Open SHMEM
+Description: Open MPI based Open SHMEM implementation
+Version: @OSHMEM_VERSION@
+URL: https://www.open-mpi.org/
+#
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+includedir=@includedir@
+libdir=@libdir@
+pkgincludedir=@opalincludedir@
+
+Libs: @OSHMEM_PC_LIBS@
+Libs.private: @OSHMEM_PC_LIBS_PRIVATE@
+Cflags: @OSHMEM_PC_CFLAGS@
+Requires: @OSHMEM_PC_MODULES@
+Requires.private: @OSHMEM_PC_MODULES_PRIVATE@
diff --git a/oshmem/tools/wrappers/oshmem-cxx.pc.in b/oshmem/tools/wrappers/oshmem-cxx.pc.in
new file mode 100644
index 00000000000..8c2ff28c682
--- /dev/null
+++ b/oshmem/tools/wrappers/oshmem-cxx.pc.in
@@ -0,0 +1,21 @@
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2016      IBM Corporation.  All rights reserved.
+# Copyright (c) 2018      Research Organization for Information Science
+#                         and Technology (RIST).  All rights reserved.
+#
+Name: Open SHMEM
+Description: Open MPI based Open SHMEM implementation
+Version: @OSHMEM_VERSION@
+URL: https://www.open-mpi.org/
+#
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+includedir=@includedir@
+libdir=@libdir@
+pkgincludedir=@opalincludedir@
+
+Libs: @OMPI_PC_LIBS@
+Libs.private: @OMPI_PC_LIBS_PRIVATE@
+Cflags: @OMPI_PC_CFLAGS@
+Requires: @OMPI_PC_MODULES@
+Requires.private: @OMPI_PC_MODULES_PRIVATE@
diff --git a/oshmem/tools/wrappers/oshmem-fort.pc.in b/oshmem/tools/wrappers/oshmem-fort.pc.in
new file mode 100644
index 00000000000..bf1da22b0e5
--- /dev/null
+++ b/oshmem/tools/wrappers/oshmem-fort.pc.in
@@ -0,0 +1,21 @@
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2016      IBM Corporation.  All rights reserved.
+# Copyright (c) 2018      Research Organization for Information Science
+#                         and Technology (RIST).  All rights reserved.
+#
+Name: Open SHMEM
+Description: Open MPI based Open SHMEM implementation
+Version: @OSHMEM_VERSION@
+URL: https://www.open-mpi.org/
+#
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+includedir=@includedir@
+libdir=@libdir@
+pkgincludedir=@opalincludedir@
+
+Libs: @OSHMEM_PC_FC_LIBS@
+Libs.private: @OSHMEM_PC_FC_LIBS_PRIVATE@
+Cflags: @OSHMEM_PC_FC_CFLAGS@
+Requires: @OSHMEM_PC_MODULES@
+Requires.private: @OSHMEM_PC_MODULES_PRIVATE@
diff --git a/oshmem/tools/wrappers/oshmem.pc.in b/oshmem/tools/wrappers/oshmem.pc.in
new file mode 100644
index 00000000000..a600e3ff2a9
--- /dev/null
+++ b/oshmem/tools/wrappers/oshmem.pc.in
@@ -0,0 +1,21 @@
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2016      IBM Corporation.  All rights reserved.
+# Copyright (c) 2018      Research Organization for Information Science
+#                         and Technology (RIST).  All rights reserved.
+#
+Name: Open SHMEM
+Description: Open MPI based Open SHMEM implementation
+Version: @OSHMEM_VERSION@
+URL: https://www.open-mpi.org/
+#
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+includedir=@includedir@
+libdir=@libdir@
+pkgincludedir=@opalincludedir@
+
+Libs: @OSHMEM_PC_LIBS@
+Libs.private: @OSHMEM_PC_LIBS_PRIVATE@
+Cflags: @OSHMEM_PC_CFLAGS@
+Requires: @OSHMEM_PC_MODULES@
+Requires.private: @OSHMEM_PC_MODULES_PRIVATE@
diff --git a/oshmem/tools/wrappers/shmemc++-wrapper-data.txt.in b/oshmem/tools/wrappers/shmemc++-wrapper-data.txt.in
index ee0da2a9e6b..8763387105a 100644
--- a/oshmem/tools/wrappers/shmemc++-wrapper-data.txt.in
+++ b/oshmem/tools/wrappers/shmemc++-wrapper-data.txt.in
@@ -2,6 +2,7 @@
 #                         All rights reserved.
 # Copyright (c) 2014-2015 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2020      Intel, Inc.  All rights reserved.
+# Copyright (c) 2016-2021 IBM Corporation. All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -22,15 +23,12 @@ language=C++
 compiler_env=CXX
 compiler_flags_env=CXXFLAGS
 compiler=@CXX@
-preprocessor_flags=@OMPI_WRAPPER_EXTRA_CPPFLAGS@
-compiler_flags=@OMPI_WRAPPER_EXTRA_CXXFLAGS@
-linker_flags=@OMPI_WRAPPER_EXTRA_LDFLAGS@
-# Note that per https://svn.open-mpi.org/trac/ompi/ticket/3422, we
-# intentionally only link in the SHMEM and MPI libraries (OPAL,
-# etc. are pulled in implicitly) because we intend SHMEM/MPI
-# applications to only use the SHMEM and MPI APIs.
-libs=-loshmem -lmpi
-libs_static=-loshmem -lmpi -l@OPAL_LIB_PREFIX@open-pal @OMPI_WRAPPER_EXTRA_LIBS@
+preprocessor_flags=@OSHMEM_WRAPPER_CPPFLAGS@
+compiler_flags=@OSHMEM_WRAPPER_CXXFLAGS@
+linker_flags=@OSHMEM_WRAPPER_LDFLAGS@
+linker_flags_static=@OSHMEM_WRAPPER_LDFLAGS_STATIC@
+libs=@OSHMEM_WRAPPER_LIBS@
+libs_static=@OSHMEM_WRAPPER_LIBS_STATIC@
 dyn_lib_file=liboshmem.@OPAL_DYN_LIB_SUFFIX@
 static_lib_file=liboshmem.a
 required_file=
diff --git a/oshmem/tools/wrappers/shmemcc-wrapper-data.txt.in b/oshmem/tools/wrappers/shmemcc-wrapper-data.txt.in
index 7078910d2a3..39aa40a5407 100644
--- a/oshmem/tools/wrappers/shmemcc-wrapper-data.txt.in
+++ b/oshmem/tools/wrappers/shmemcc-wrapper-data.txt.in
@@ -23,15 +23,12 @@ language=C
 compiler_env=CC
 compiler_flags_env=CFLAGS
 compiler=@CC@
-preprocessor_flags=@OMPI_WRAPPER_EXTRA_CPPFLAGS@
-compiler_flags=@OMPI_WRAPPER_EXTRA_CFLAGS@
-linker_flags=@OMPI_WRAPPER_EXTRA_LDFLAGS@
-# Note that per https://svn.open-mpi.org/trac/ompi/ticket/3422, we
-# intentionally only link in the SHMEM and MPI libraries (OPAL,
-# etc. are pulled in implicitly) because we intend SHMEM/MPI
-# applications to only use the SHMEM and MPI APIs.
-libs=-loshmem -l@OMPI_LIBMPI_NAME@
-libs_static=-loshmem -l@OMPI_LIBMPI_NAME@ -l@OPAL_LIB_PREFIX@open-pal @OMPI_WRAPPER_EXTRA_LIBS@
+preprocessor_flags=@OSHMEM_WRAPPER_CPPFLAGS@
+compiler_flags=@OSHMEM_WRAPPER_CFLAGS@
+linker_flags=@OSHMEM_WRAPPER_LDFLAGS@
+linker_flags_static=@OSHMEM_WRAPPER_LDFLAGS_STATIC@
+libs=@OSHMEM_WRAPPER_LIBS@
+libs_static=@OSHMEM_WRAPPER_LIBS_STATIC@
 dyn_lib_file=liboshmem.@OPAL_DYN_LIB_SUFFIX@
 static_lib_file=liboshmem.a
 required_file=
diff --git a/oshmem/tools/wrappers/shmemfort-wrapper-data.txt.in b/oshmem/tools/wrappers/shmemfort-wrapper-data.txt.in
index ac3c239a402..009cce31600 100644
--- a/oshmem/tools/wrappers/shmemfort-wrapper-data.txt.in
+++ b/oshmem/tools/wrappers/shmemfort-wrapper-data.txt.in
@@ -24,14 +24,11 @@ compiler_env=FC
 compiler_flags_env=FCFLAGS
 compiler=@FC@
 preprocessor_flags=
-compiler_flags=@OMPI_WRAPPER_EXTRA_FCFLAGS@
-linker_flags=@OMPI_WRAPPER_EXTRA_FC_LDFLAGS@
-# Note that per https://svn.open-mpi.org/trac/ompi/ticket/3422, we
-# intentionally only link in the SHMEM and MPI libraries (OPAL,
-# etc. are pulled in implicitly) because we intend SHMEM/MPI
-# applications to only use the SHMEM and MPI APIs.
-libs=-loshmem -l@OMPI_LIBMPI_NAME@_mpifh -l@OMPI_LIBMPI_NAME@
-libs_static=-loshmem -l@OMPI_LIBMPI_NAME@_mpifh -l@OMPI_LIBMPI_NAME@ -l@OPAL_LIB_PREFIX@open-pal @OMPI_WRAPPER_EXTRA_LIBS@
+compiler_flags=@OSHMEM_WRAPPER_FCFLAGS@
+linker_flags=@OSHMEM_WRAPPER_FC_LDFLAGS@
+linker_flags_static=@OSHMEM_WRAPPER_FC_LDFLAGS_STATIC@
+libs=@OSHMEM_WRAPPER_LIBS@
+libs_static=@OSHMEM_WRAPPER_LIBS_STATIC@
 dyn_lib_file=liboshmem.@OPAL_DYN_LIB_SUFFIX@
 static_lib_file=liboshmem.a
 required_file=@OMPI_WRAPPER_FORTRAN_REQUIRED_FILE@
diff --git a/oshmem/util/oshmem_util.h b/oshmem/util/oshmem_util.h
index c65c4212f6a..02aa5fa65c1 100644
--- a/oshmem/util/oshmem_util.h
+++ b/oshmem/util/oshmem_util.h
@@ -30,7 +30,7 @@ void oshmem_output_verbose(int level, int output_id, const char* prefix,
     const char* file, int line, const char* function, const char* format, ...);
 
 /*
- * Temporary wrapper which ingores output verbosity level
+ * Temporary wrapper which ignores output verbosity level
  * to ensure error messages are seeing by user
  */
 void oshmem_output(int output_id, const char* prefix, const char* file,
diff --git a/test/asm/Makefile.am b/test/asm/Makefile.am
index d8f0e4ccc3b..53359a4beaf 100644
--- a/test/asm/Makefile.am
+++ b/test/asm/Makefile.am
@@ -90,5 +90,5 @@ maintainer-clean-local:
 	atomic_math_noinline.c \
 	atomic_cmpset_noinline.c
 
-distclean:
+distclean-local:
 	rm -rf *.dSYM .deps .libs *.log *.o *.trs $(check_PROGRAMS) Makefile
diff --git a/test/asm/atomic_barrier.c b/test/asm/atomic_barrier.c
index 3542ba74cf4..264951b45dd 100644
--- a/test/asm/atomic_barrier.c
+++ b/test/asm/atomic_barrier.c
@@ -10,6 +10,8 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -22,11 +24,8 @@
 
 #include "opal/sys/atomic.h"
 
-int
-main(int argc, char *argv[])
+int main(int argc, char *argv[])
 {
-#if OPAL_HAVE_ATOMIC_MEM_BARRIER
-
     /* there really isn't a great way to test that the barriers
        actually barrier, but at least make sure they don't kill the
        machine.*/
@@ -36,8 +35,4 @@ main(int argc, char *argv[])
     opal_atomic_wmb();
 
     return 0;
-#else
-    return 77;
-#endif
 }
-
diff --git a/test/asm/atomic_cmpset.c b/test/asm/atomic_cmpset.c
index 9fe964125a7..9f6d84588da 100644
--- a/test/asm/atomic_cmpset.c
+++ b/test/asm/atomic_cmpset.c
@@ -30,7 +30,7 @@
 
 #include <assert.h>
 #ifdef HAVE_PTHREAD_H
-#include <pthread.h>
+#    include <pthread.h>
 #endif
 #include <stdarg.h>
 #include <stdio.h>
@@ -39,7 +39,6 @@
 
 #include "opal/sys/atomic.h"
 
-
 /* default options */
 int nreps = 100;
 int nthreads = 2;
@@ -50,12 +49,10 @@ opal_atomic_int32_t val32 = 0;
 int32_t old32 = 0;
 int32_t new32 = 0;
 
-#if OPAL_HAVE_ATOMIC_MATH_64
 opal_atomic_int64_t vol64 = 0;
 opal_atomic_int64_t val64 = 0;
 int64_t old64 = 0;
 int64_t new64 = 0;
-#endif
 
 #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128
 opal_atomic_int128_t vol128;
@@ -73,7 +70,6 @@ opal_atomic_intptr_t volptr = 0;
 intptr_t oldptr = 0;
 intptr_t newptr = 0;
 
-
 static void *thread_main(void *arg)
 {
     int rank = (int) (unsigned long) arg;
@@ -83,10 +79,8 @@ static void *thread_main(void *arg)
 
     for (i = 0; i < nreps; i++) {
         opal_atomic_add_fetch_32(&val32, 5);
-#if OPAL_HAVE_ATOMIC_MATH_64
         opal_atomic_add_fetch_64(&val64, 5);
-#endif
-        opal_atomic_add (&valint, 5);
+        opal_atomic_add(&valint, 5);
     }
 
     return (void *) (unsigned long) (rank + 1000);
@@ -103,168 +97,128 @@ int main(int argc, char *argv[])
     }
     nthreads = atoi(argv[1]);
 
-
     /* first test single-threaded functionality */
 
     /* -- cmpset 32-bit tests -- */
 
     vol32 = 42, old32 = 42, new32 = 50;
-    assert(opal_atomic_compare_exchange_strong_32 (&vol32, &old32, new32) == true);
+    assert(opal_atomic_compare_exchange_strong_32(&vol32, &old32, new32) == true);
     opal_atomic_rmb();
     assert(vol32 == new32);
     assert(old32 == 42);
 
     vol32 = 42, old32 = 420, new32 = 50;
-    assert(opal_atomic_compare_exchange_strong_32 (&vol32, &old32, new32) ==  false);
+    assert(opal_atomic_compare_exchange_strong_32(&vol32, &old32, new32) == false);
     opal_atomic_rmb();
     assert(vol32 == 42);
     assert(old32 == 42);
 
     vol32 = 42, old32 = 42, new32 = 50;
-    assert(opal_atomic_compare_exchange_strong_32 (&vol32, &old32, new32) == true);
+    assert(opal_atomic_compare_exchange_strong_32(&vol32, &old32, new32) == true);
     assert(vol32 == new32);
     assert(old32 == 42);
 
     vol32 = 42, old32 = 420, new32 = 50;
-    assert(opal_atomic_compare_exchange_strong_acq_32 (&vol32, &old32, new32) == false);
+    assert(opal_atomic_compare_exchange_strong_acq_32(&vol32, &old32, new32) == false);
     assert(vol32 == 42);
     assert(old32 == 42);
 
     vol32 = 42, old32 = 42, new32 = 50;
-    assert(opal_atomic_compare_exchange_strong_rel_32 (&vol32, &old32, new32) ==  true);
+    assert(opal_atomic_compare_exchange_strong_rel_32(&vol32, &old32, new32) == true);
     opal_atomic_rmb();
     assert(vol32 == new32);
     assert(old32 == 42);
 
     vol32 = 42, old32 = 420, new32 = 50;
-    assert(opal_atomic_compare_exchange_strong_rel_32 (&vol32, &old32, new32) == false);
+    assert(opal_atomic_compare_exchange_strong_rel_32(&vol32, &old32, new32) == false);
     opal_atomic_rmb();
     assert(vol32 == 42);
     assert(old32 == 42);
 
     /* -- cmpset 64-bit tests -- */
 
-#if OPAL_HAVE_ATOMIC_MATH_64
     vol64 = 42, old64 = 42, new64 = 50;
-    assert(opal_atomic_compare_exchange_strong_64 (&vol64, &old64, new64) == true);
+    assert(opal_atomic_compare_exchange_strong_64(&vol64, &old64, new64) == true);
     opal_atomic_rmb();
     assert(new64 == vol64);
     assert(old64 == 42);
 
     vol64 = 42, old64 = 420, new64 = 50;
-    assert(opal_atomic_compare_exchange_strong_64 (&vol64, &old64, new64) == false);
+    assert(opal_atomic_compare_exchange_strong_64(&vol64, &old64, new64) == false);
     opal_atomic_rmb();
     assert(vol64 == 42);
     assert(old64 == 42);
 
     vol64 = 42, old64 = 42, new64 = 50;
-    assert(opal_atomic_compare_exchange_strong_acq_64 (&vol64, &old64, new64) == true);
+    assert(opal_atomic_compare_exchange_strong_acq_64(&vol64, &old64, new64) == true);
     assert(vol64 == new64);
     assert(old64 == 42);
 
     vol64 = 42, old64 = 420, new64 = 50;
-    assert(opal_atomic_compare_exchange_strong_acq_64 (&vol64, &old64, new64) == false);
+    assert(opal_atomic_compare_exchange_strong_acq_64(&vol64, &old64, new64) == false);
     assert(vol64 == 42);
     assert(old64 == 42);
 
     vol64 = 42, old64 = 42, new64 = 50;
-    assert(opal_atomic_compare_exchange_strong_rel_64 (&vol64, &old64, new64) == true);
+    assert(opal_atomic_compare_exchange_strong_rel_64(&vol64, &old64, new64) == true);
     opal_atomic_rmb();
     assert(vol64 == new64);
     assert(old64 == 42);
 
     vol64 = 42, old64 = 420, new64 = 50;
-    assert(opal_atomic_compare_exchange_strong_rel_64 (&vol64, &old64, new64) == false);
+    assert(opal_atomic_compare_exchange_strong_rel_64(&vol64, &old64, new64) == false);
     opal_atomic_rmb();
     assert(vol64 == 42);
     assert(old64 == 42);
-#endif
 
     /* -- cmpset 128-bit tests -- */
 
 #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128
     vol128 = 42, old128 = 42, new128 = 50;
-    assert(opal_atomic_compare_exchange_strong_128 (&vol128, &old128, new128) == true);
+    assert(opal_atomic_compare_exchange_strong_128(&vol128, &old128, new128) == true);
     opal_atomic_rmb();
     assert(new128 == vol128);
     assert(old128 == 42);
 
     vol128 = 42, old128 = 420, new128 = 50;
-    assert(opal_atomic_compare_exchange_strong_128 (&vol128, &old128, new128) == false);
+    assert(opal_atomic_compare_exchange_strong_128(&vol128, &old128, new128) == false);
     opal_atomic_rmb();
     assert(vol128 == 42);
     assert(old128 == 42);
 #endif
 
-    /* -- cmpset int tests -- */
-
-    volint = 42, oldint = 42, newint = 50;
-    assert(opal_atomic_compare_exchange_strong (&volint, &oldint, newint) == true);
-    opal_atomic_rmb();
-    assert(volint == newint);
-    assert(oldint == 42);
-
-    volint = 42, oldint = 420, newint = 50;
-    assert(opal_atomic_compare_exchange_strong (&volint, &oldint, newint) == false);
-    opal_atomic_rmb();
-    assert(volint == 42);
-    assert(oldint == 42);
-
-    volint = 42, oldint = 42, newint = 50;
-    assert(opal_atomic_compare_exchange_strong_acq (&volint, &oldint, newint) == true);
-    assert(volint == newint);
-    assert(oldint == 42);
-
-    volint = 42, oldint = 420, newint = 50;
-    assert(opal_atomic_compare_exchange_strong_acq (&volint, &oldint, newint) == false);
-    assert(volint == 42);
-    assert(oldint == 42);
-
-    volint = 42, oldint = 42, newint = 50;
-    assert(opal_atomic_compare_exchange_strong_rel (&volint, &oldint, newint) == true);
-    opal_atomic_rmb();
-    assert(volint == newint);
-    assert(oldint == 42);
-
-    volint = 42, oldint = 420, newint = 50;
-    assert(opal_atomic_compare_exchange_strong_rel (&volint, &oldint, newint) == false);
-    opal_atomic_rmb();
-    assert(volint == 42);
-    assert(oldint == 42);
-
-
     /* -- cmpset ptr tests -- */
 
     volptr = 42, oldptr = 42, newptr = 50;
-    assert(opal_atomic_compare_exchange_strong_ptr (&volptr, &oldptr, newptr) == true);
+    assert(opal_atomic_compare_exchange_strong_ptr(&volptr, &oldptr, newptr) == true);
     opal_atomic_rmb();
     assert(volptr == newptr);
     assert(oldptr == 42);
 
     volptr = 42, oldptr = 420, newptr = 50;
-    assert(opal_atomic_compare_exchange_strong_ptr (&volptr, &oldptr, newptr) == false);
+    assert(opal_atomic_compare_exchange_strong_ptr(&volptr, &oldptr, newptr) == false);
     opal_atomic_rmb();
     assert(volptr == 42);
     assert(oldptr == 42);
 
     volptr = 42, oldptr = 42, newptr = 50;
-    assert(opal_atomic_compare_exchange_strong_acq_ptr (&volptr, &oldptr, newptr) == true);
+    assert(opal_atomic_compare_exchange_strong_acq_ptr(&volptr, &oldptr, newptr) == true);
     assert(volptr == newptr);
     assert(oldptr == 42);
 
     volptr = 42, oldptr = 420, newptr = 50;
-    assert(opal_atomic_compare_exchange_strong_acq_ptr (&volptr, &oldptr, newptr) == false);
+    assert(opal_atomic_compare_exchange_strong_acq_ptr(&volptr, &oldptr, newptr) == false);
     assert(volptr == 42);
     assert(oldptr == 42);
 
     volptr = 42, oldptr = 42, newptr = 50;
-    assert(opal_atomic_compare_exchange_strong_rel_ptr (&volptr, &oldptr, newptr) == true);
+    assert(opal_atomic_compare_exchange_strong_rel_ptr(&volptr, &oldptr, newptr) == true);
     opal_atomic_rmb();
     assert(volptr == newptr);
     assert(oldptr == 42);
 
     volptr = 42, oldptr = 420, newptr = 50;
-    assert(opal_atomic_compare_exchange_strong_rel_ptr (&volptr, &oldptr, newptr) == false);
+    assert(opal_atomic_compare_exchange_strong_rel_ptr(&volptr, &oldptr, newptr) == false);
     opal_atomic_rmb();
     assert(volptr == 42);
     assert(oldptr == 42);
@@ -277,26 +231,21 @@ int main(int argc, char *argv[])
     assert((42 + 5) == val32);
 
     /* -- add_64 tests -- */
-#if OPAL_HAVE_ATOMIC_MATH_64
     val64 = 42;
     assert(opal_atomic_add_fetch_64(&val64, 5) == (42 + 5));
     opal_atomic_rmb();
     assert((42 + 5) == val64);
-#endif
     /* -- add_int tests -- */
 
     valint = 42;
-    opal_atomic_add (&valint, 5);
+    opal_atomic_add(&valint, 5);
     opal_atomic_rmb();
     assert((42 + 5) == valint);
 
-
     /* threaded tests */
 
     val32 = 0;
-#if OPAL_HAVE_ATOMIC_MATH_64
     val64 = 0ul;
-#endif
     valint = 0;
 
     /* -- create the thread set -- */
@@ -326,10 +275,8 @@ int main(int argc, char *argv[])
 
     opal_atomic_rmb();
     assert((5 * nthreads * nreps) == val32);
-#if OPAL_HAVE_ATOMIC_MATH_64
     opal_atomic_rmb();
-    assert((5 * nthreads * nreps) ==  val64);
-#endif
+    assert((5 * nthreads * nreps) == val64);
     opal_atomic_rmb();
     assert((5 * nthreads * nreps) == valint);
 
diff --git a/test/asm/atomic_math.c b/test/asm/atomic_math.c
index 8629ad9780f..864d7a2a4d7 100644
--- a/test/asm/atomic_math.c
+++ b/test/asm/atomic_math.c
@@ -23,40 +23,34 @@
 #include "opal_config.h"
 
 #ifdef HAVE_PTHREAD_H
-#include <pthread.h>
+#    include <pthread.h>
 #endif
-#include <stdlib.h>
 #include <stdio.h>
+#include <stdlib.h>
 
 #include "opal/sys/atomic.h"
 
 #define TEST_REPS 500
 
 opal_atomic_int32_t val32 = 0;
-#if OPAL_HAVE_ATOMIC_MATH_64
 opal_atomic_int64_t val64 = 0;
-#endif
 opal_atomic_int_t valint = 0;
 
-static void* atomic_math_test(void* arg)
+static void *atomic_math_test(void *arg)
 {
-    int count = *((int*) arg);
+    int count = *((int *) arg);
     int i;
 
-    for (i = 0 ; i < count ; ++i) {
-        (void)opal_atomic_add_fetch_32(&val32, 5);
-#if OPAL_HAVE_ATOMIC_MATH_64
-        (void)opal_atomic_add_fetch_64(&val64, 6);
-#endif
-        opal_atomic_add (&valint, 4);
+    for (i = 0; i < count; ++i) {
+        (void) opal_atomic_add_fetch_32(&val32, 5);
+        (void) opal_atomic_add_fetch_64(&val64, 6);
+        opal_atomic_add(&valint, 4);
     }
 
     return NULL;
 }
 
-
-static int
-atomic_math_test_th(int count, int thr_count)
+static int atomic_math_test_th(int count, int thr_count)
 {
     int value;
     pthread_t *th;
@@ -75,8 +69,7 @@ atomic_math_test_th(int count, int thr_count)
        disappearing before the threads start. */
     value = count;
     for (tid = 0; tid < thr_count; tid++) {
-        if (pthread_create(&th[tid], NULL, atomic_math_test,
-                           (void*) &value) != 0) {
+        if (pthread_create(&th[tid], NULL, atomic_math_test, (void *) &value) != 0) {
             perror("pthread_create");
             exit(EXIT_FAILURE);
         }
@@ -96,14 +89,10 @@ atomic_math_test_th(int count, int thr_count)
     return 0;
 }
 
-
-int
-main(int argc, char *argv[])
+int main(int argc, char *argv[])
 {
     int32_t test32;
-#if OPAL_HAVE_ATOMIC_MATH_64
     int64_t test64;
-#endif
     int ret = 77;
     int num_threads = 1;
 
@@ -113,151 +102,175 @@ main(int argc, char *argv[])
     }
     num_threads = atoi(argv[1]);
 
-    test32 = opal_atomic_add_fetch_32 (&val32, 17);
+    test32 = opal_atomic_add_fetch_32(&val32, 17);
     if (test32 != 17 || val32 != 17) {
-        fprintf (stderr, "error in opal_atomic_add_fetch_32. expected (17, 17), got (%d, %d)\n", test32, val32);
+        fprintf(stderr, "error in opal_atomic_add_fetch_32. expected (17, 17), got (%d, %d)\n",
+                test32, val32);
         exit(EXIT_FAILURE);
     }
 
-    test32 = opal_atomic_fetch_add_32 (&val32, 13);
+    test32 = opal_atomic_fetch_add_32(&val32, 13);
     if (test32 != 17 || val32 != 30) {
-        fprintf (stderr, "error in opal_atomic_fetch_add_32. expected (17, 30), got (%d, %d)\n", test32, val32);
+        fprintf(stderr, "error in opal_atomic_fetch_add_32. expected (17, 30), got (%d, %d)\n",
+                test32, val32);
         exit(EXIT_FAILURE);
     }
 
-
-
-    test32 = opal_atomic_and_fetch_32 (&val32, 0x18);
+    test32 = opal_atomic_and_fetch_32(&val32, 0x18);
     if (test32 != 24 || val32 != 24) {
-        fprintf (stderr, "error in opal_atomic_and_fetch_32. expected (24, 24), got (%d, %d)\n", test32, val32);
+        fprintf(stderr, "error in opal_atomic_and_fetch_32. expected (24, 24), got (%d, %d)\n",
+                test32, val32);
         exit(EXIT_FAILURE);
     }
 
-    test32 = opal_atomic_fetch_and_32 (&val32, 0x10);
+    test32 = opal_atomic_fetch_and_32(&val32, 0x10);
     if (test32 != 24 || val32 != 16) {
-        fprintf (stderr, "error in opal_atomic_fetch_and_32. expected (24, 16), got (%d, %d)\n", test32, val32);
+        fprintf(stderr, "error in opal_atomic_fetch_and_32. expected (24, 16), got (%d, %d)\n",
+                test32, val32);
         exit(EXIT_FAILURE);
     }
 
-
-
-    test32 = opal_atomic_or_fetch_32 (&val32, 0x03);
+    test32 = opal_atomic_or_fetch_32(&val32, 0x03);
     if (test32 != 19 || val32 != 19) {
-        fprintf (stderr, "error in opal_atomic_or_fetch_32. expected (19, 19), got (%d, %d)\n", test32, val32);
+        fprintf(stderr, "error in opal_atomic_or_fetch_32. expected (19, 19), got (%d, %d)\n",
+                test32, val32);
         exit(EXIT_FAILURE);
     }
 
-    test32 = opal_atomic_fetch_or_32 (&val32, 0x04);
+    test32 = opal_atomic_fetch_or_32(&val32, 0x04);
     if (test32 != 19 || val32 != 23) {
-        fprintf (stderr, "error in opal_atomic_fetch_or_32. expected (19, 23), got (%d, %d)\n", test32, val32);
+        fprintf(stderr, "error in opal_atomic_fetch_or_32. expected (19, 23), got (%d, %d)\n",
+                test32, val32);
         exit(EXIT_FAILURE);
     }
 
-
-    test32 = opal_atomic_xor_fetch_32 (&val32, 0x03);
+    test32 = opal_atomic_xor_fetch_32(&val32, 0x03);
     if (test32 != 20 || val32 != 20) {
-        fprintf (stderr, "error in opal_atomic_xor_fetch_32. expected (20, 20), got (%d, %d)\n", test32, val32);
+        fprintf(stderr, "error in opal_atomic_xor_fetch_32. expected (20, 20), got (%d, %d)\n",
+                test32, val32);
         exit(EXIT_FAILURE);
     }
 
-    test32 = opal_atomic_fetch_xor_32 (&val32, 0x05);
+    test32 = opal_atomic_fetch_xor_32(&val32, 0x05);
     if (test32 != 20 || val32 != 17) {
-        fprintf (stderr, "error in opal_atomic_fetch_xor_32. expected (20, 17), got (%d, %d)\n", test32, val32);
+        fprintf(stderr, "error in opal_atomic_fetch_xor_32. expected (20, 17), got (%d, %d)\n",
+                test32, val32);
         exit(EXIT_FAILURE);
     }
 
-
-
-    test32 = opal_atomic_sub_fetch_32 (&val32, 14);
+    test32 = opal_atomic_sub_fetch_32(&val32, 14);
     if (test32 != 3 || val32 != 3) {
-        fprintf (stderr, "error in opal_atomic_sub_fetch_32. expected (3, 3), got (%d, %d)\n", test32, val32);
+        fprintf(stderr, "error in opal_atomic_sub_fetch_32. expected (3, 3), got (%d, %d)\n",
+                test32, val32);
         exit(EXIT_FAILURE);
     }
 
-    test32 = opal_atomic_fetch_xor_32 (&val32, 3);
+    test32 = opal_atomic_fetch_xor_32(&val32, 3);
     if (test32 != 3 || val32 != 0) {
-        fprintf (stderr, "error in opal_atomic_fetch_sub_32. expected (3, 0), got (%d, %d)\n", test32, val32);
+        fprintf(stderr, "error in opal_atomic_fetch_sub_32. expected (3, 0), got (%d, %d)\n",
+                test32, val32);
         exit(EXIT_FAILURE);
     }
 
-#if OPAL_HAVE_ATOMIC_MATH_64
-    test64 = opal_atomic_add_fetch_64 (&val64, 17);
+    test64 = opal_atomic_add_fetch_64(&val64, 17);
     if (test64 != 17 || val64 != 17) {
-        fprintf (stderr, "error in opal_atomic_add_fetch_64. expected (17, 17), got (%" PRId64 ", %" PRId64 ")\n", test64, val64);
+        fprintf(stderr,
+                "error in opal_atomic_add_fetch_64. expected (17, 17), got (%" PRId64 ", %" PRId64
+                ")\n",
+                test64, val64);
         exit(EXIT_FAILURE);
     }
 
-    test64 = opal_atomic_fetch_add_64 (&val64, 13);
+    test64 = opal_atomic_fetch_add_64(&val64, 13);
     if (test64 != 17 || val64 != 30) {
-        fprintf (stderr, "error in opal_atomic_fetch_add_64. expected (17, 30), got (%" PRId64 ", %" PRId64 ")\n", test64, val64);
+        fprintf(stderr,
+                "error in opal_atomic_fetch_add_64. expected (17, 30), got (%" PRId64 ", %" PRId64
+                ")\n",
+                test64, val64);
         exit(EXIT_FAILURE);
     }
 
-
-
-    test64 = opal_atomic_and_fetch_64 (&val64, 0x18);
+    test64 = opal_atomic_and_fetch_64(&val64, 0x18);
     if (test64 != 24 || val64 != 24) {
-        fprintf (stderr, "error in opal_atomic_and_fetch_64. expected (24, 24), got (%" PRId64 ", %" PRId64 ")\n", test64, val64);
+        fprintf(stderr,
+                "error in opal_atomic_and_fetch_64. expected (24, 24), got (%" PRId64 ", %" PRId64
+                ")\n",
+                test64, val64);
         exit(EXIT_FAILURE);
     }
 
-    test64 = opal_atomic_fetch_and_64 (&val64, 0x10);
+    test64 = opal_atomic_fetch_and_64(&val64, 0x10);
     if (test64 != 24 || val64 != 16) {
-        fprintf (stderr, "error in opal_atomic_fetch_and_64. expected (24, 16), got (%" PRId64 ", %" PRId64 ")\n", test64, val64);
+        fprintf(stderr,
+                "error in opal_atomic_fetch_and_64. expected (24, 16), got (%" PRId64 ", %" PRId64
+                ")\n",
+                test64, val64);
         exit(EXIT_FAILURE);
     }
 
-
-
-    test64 = opal_atomic_or_fetch_64 (&val64, 0x03);
+    test64 = opal_atomic_or_fetch_64(&val64, 0x03);
     if (test64 != 19 || val64 != 19) {
-        fprintf (stderr, "error in opal_atomic_or_fetch_64. expected (19, 19), got (%" PRId64 ", %" PRId64 ")\n", test64, val64);
+        fprintf(stderr,
+                "error in opal_atomic_or_fetch_64. expected (19, 19), got (%" PRId64 ", %" PRId64
+                ")\n",
+                test64, val64);
         exit(EXIT_FAILURE);
     }
 
-    test64 = opal_atomic_fetch_or_64 (&val64, 0x04);
+    test64 = opal_atomic_fetch_or_64(&val64, 0x04);
     if (test64 != 19 || val64 != 23) {
-        fprintf (stderr, "error in opal_atomic_fetch_or_64. expected (19, 23), got (%" PRId64 ", %" PRId64 ")\n", test64, val64);
+        fprintf(stderr,
+                "error in opal_atomic_fetch_or_64. expected (19, 23), got (%" PRId64 ", %" PRId64
+                ")\n",
+                test64, val64);
         exit(EXIT_FAILURE);
     }
 
-
-    test64 = opal_atomic_xor_fetch_64 (&val64, 0x03);
+    test64 = opal_atomic_xor_fetch_64(&val64, 0x03);
     if (test64 != 20 || val64 != 20) {
-        fprintf (stderr, "error in opal_atomic_xor_fetch_64. expected (20, 20), got (%" PRId64 ", %" PRId64 ")\n", test64, val64);
+        fprintf(stderr,
+                "error in opal_atomic_xor_fetch_64. expected (20, 20), got (%" PRId64 ", %" PRId64
+                ")\n",
+                test64, val64);
         exit(EXIT_FAILURE);
     }
 
-    test64 = opal_atomic_fetch_xor_64 (&val64, 0x05);
+    test64 = opal_atomic_fetch_xor_64(&val64, 0x05);
     if (test64 != 20 || val64 != 17) {
-        fprintf (stderr, "error in opal_atomic_fetch_xor_64. expected (20, 17), got (%" PRId64 ", %" PRId64 ")\n", test64, val64);
+        fprintf(stderr,
+                "error in opal_atomic_fetch_xor_64. expected (20, 17), got (%" PRId64 ", %" PRId64
+                ")\n",
+                test64, val64);
         exit(EXIT_FAILURE);
     }
 
-
-
-    test64 = opal_atomic_sub_fetch_64 (&val64, 14);
+    test64 = opal_atomic_sub_fetch_64(&val64, 14);
     if (test64 != 3 || val64 != 3) {
-        fprintf (stderr, "error in opal_atomic_sub_fetch_64. expected (3, 3), got (%" PRId64 ", %" PRId64 ")\n", test64, val64);
+        fprintf(stderr,
+                "error in opal_atomic_sub_fetch_64. expected (3, 3), got (%" PRId64 ", %" PRId64
+                ")\n",
+                test64, val64);
         exit(EXIT_FAILURE);
     }
 
-    test64 = opal_atomic_fetch_xor_64 (&val64, 3);
+    test64 = opal_atomic_fetch_xor_64(&val64, 3);
     if (test64 != 3 || val64 != 0) {
-        fprintf (stderr, "error in opal_atomic_fetch_sub_64. expected (3, 0), got (%" PRId64 ", %" PRId64 ")\n", test64, val64);
+        fprintf(stderr,
+                "error in opal_atomic_fetch_sub_64. expected (3, 0), got (%" PRId64 ", %" PRId64
+                ")\n",
+                test64, val64);
         exit(EXIT_FAILURE);
     }
-#endif
 
     ret = atomic_math_test_th(TEST_REPS, num_threads);
-    if (ret == 77) return ret;
+    if (ret == 77)
+        return ret;
     opal_atomic_mb();
     if (val32 != TEST_REPS * num_threads * 5) {
         printf("opal_atomic_add_fetch32 failed.  Expected %d, got %d.\n",
                TEST_REPS * num_threads * 5, val32);
         ret = 1;
     }
-#if OPAL_HAVE_ATOMIC_MATH_64
     if (val64 != TEST_REPS * num_threads * 6) {
         /* Safe to case to (int) here because we know it's going to be
            a small value */
@@ -265,9 +278,6 @@ main(int argc, char *argv[])
                TEST_REPS * num_threads * 6, (int) val64);
         ret = 1;
     }
-#else
-    printf("      * skipping 64 bit tests\n");
-#endif
     if (valint != TEST_REPS * num_threads * 4) {
         printf("opal_atomic_add_fetch32 failed.  Expected %d, got %d.\n",
                TEST_REPS * num_threads * 4, valint);
diff --git a/test/asm/atomic_spinlock.c b/test/asm/atomic_spinlock.c
index a6c68b134d0..f918f3637b0 100644
--- a/test/asm/atomic_spinlock.c
+++ b/test/asm/atomic_spinlock.c
@@ -23,10 +23,10 @@
 #include "opal_config.h"
 
 #ifdef HAVE_PTHREAD_H
-#include <pthread.h>
+#    include <pthread.h>
 #endif
-#include <stdlib.h>
 #include <stdio.h>
+#include <stdlib.h>
 
 #include "opal/sys/atomic.h"
 
@@ -42,25 +42,29 @@ struct start_info {
 
 static int atomic_spinlock_test(opal_atomic_lock_t *lock, int count, int id);
 
-static void* atomic_spinlock_start(void* arg)
+static void *atomic_spinlock_start(void *arg)
 {
-    struct start_info *data = (struct start_info*) arg;
+    struct start_info *data = (struct start_info *) arg;
 
-    return (void*) (unsigned long) atomic_spinlock_test(data->lock, data->count,
-                                        data->tid);
+    return (void *) (unsigned long) atomic_spinlock_test(data->lock, data->count, data->tid);
 }
 
-static int
-atomic_spinlock_test_th(opal_atomic_lock_t *lock, int count, int id, int thr_count)
+static int atomic_spinlock_test_th(opal_atomic_lock_t *lock, int count, int id, int thr_count)
 {
     pthread_t *th;
     int tid, ret = 0;
     struct start_info *data;
 
     th = (pthread_t *) malloc(thr_count * sizeof(pthread_t));
-    if (!th) { perror("malloc"); exit(EXIT_FAILURE); }
+    if (!th) {
+        perror("malloc");
+        exit(EXIT_FAILURE);
+    }
     data = (struct start_info *) malloc(thr_count * sizeof(struct start_info));
-    if (!th) { perror("malloc"); exit(EXIT_FAILURE); }
+    if (!th) {
+        perror("malloc");
+        exit(EXIT_FAILURE);
+    }
 
     for (tid = 0; tid < thr_count; tid++) {
         data[tid].tid = tid;
@@ -90,28 +94,30 @@ atomic_spinlock_test_th(opal_atomic_lock_t *lock, int count, int id, int thr_cou
     return ret;
 }
 
-
-static int
-atomic_spinlock_test(opal_atomic_lock_t *lock, int count, int id)
+static int atomic_spinlock_test(opal_atomic_lock_t *lock, int count, int id)
 {
     int i;
 
-    for (i = 0 ; i < count ; ++i) {
+    for (i = 0; i < count; ++i) {
         opal_atomic_lock(lock);
-        if (atomic_verbose) { printf("id %03d has the lock (lock)\n", id); }
+        if (atomic_verbose) {
+            printf("id %03d has the lock (lock)\n", id);
+        }
         opal_atomic_unlock(lock);
 
-        while (opal_atomic_trylock(lock)) { ; }
-        if (atomic_verbose) { printf("id %03d has the lock (trylock)\n", id); }
+        while (opal_atomic_trylock(lock)) {
+            ;
+        }
+        if (atomic_verbose) {
+            printf("id %03d has the lock (trylock)\n", id);
+        }
         opal_atomic_unlock(lock);
     }
 
     return 0;
 }
 
-
-int
-main(int argc, char *argv[])
+int main(int argc, char *argv[])
 {
     int ret = 77;
     opal_atomic_lock_t lock;
diff --git a/test/asm/run_tests b/test/asm/run_tests
old mode 100644
new mode 100755
diff --git a/test/carto/carto-file b/test/carto/carto-file
index 46e6c8ef323..aeacf6965c1 100644
--- a/test/carto/carto-file
+++ b/test/carto/carto-file
@@ -21,7 +21,7 @@
 #
 #
 # Connection declaration  From node   To node:weight   To node:weight   ......
-# (Reserve word)          (declered   (declered        (declered
+# (Reserve word)          (declared   (declared        (declared
 #                          above)      above)           above)
 #===============================================================================================
   CONNECTION              slot0       mem0:0           slot1:1           slot2:1 mthca0:1 eth0:1
diff --git a/test/carto/carto_test.c b/test/carto/carto_test.c
index 2ff81d6d86c..2c81e978c32 100644
--- a/test/carto/carto_test.c
+++ b/test/carto/carto_test.c
@@ -23,16 +23,14 @@
 #define false 0
 #define true 1
 
-
-#include <mpi.h>
-#include <stdio.h>
-#include "../../opal/mca/carto/carto.h"
+#include "../../opal/class/opal_graph.h"
 #include "../../opal/mca/carto/base/base.h"
+#include "../../opal/mca/carto/carto.h"
 #include "../../opal/util/output.h"
-#include "../../opal/class/opal_graph.h"
+#include <mpi.h>
+#include <stdio.h>
 
-int
-main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int rank, size, distance, distance_array_size, i;
     opal_carto_graph_t *graph;
@@ -44,37 +42,38 @@ main(int argc, char* argv[])
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &size);
 
-
     printf("Hello, world, I am %d of %d\n", rank, size);
     if (0 == rank) {
         /**
          *
          */
 
-        opal_output(0," \n\nget_host_graph Full\n");
-        opal_carto_base_get_host_graph(&graph,NULL);
+        opal_output(0, " \n\nget_host_graph Full\n");
+        opal_carto_base_get_host_graph(&graph, NULL);
         opal_graph_print(graph);
         slot0 = opal_carto_base_find_node(graph, "slot0");
         if (NULL == slot0) {
-            opal_output(0,"couldn't find slot0 in the graph exiting\n");
+            opal_output(0, "couldn't find slot0 in the graph exiting\n");
             opal_carto_base_free_graph(graph);
             return -1;
         }
         end_node = opal_carto_base_find_node(graph, "slot3");
         if (NULL == end_node) {
-            opal_output(0,"couldn't find mthca1 in the graph exiting\n");
+            opal_output(0, "couldn't find mthca1 in the graph exiting\n");
             opal_carto_base_free_graph(graph);
             return -1;
         }
         distance = opal_carto_base_spf(graph, slot0, end_node);
-        opal_output(0,"\nThe distance between slot0 and slot3 is %d\n",distance);
+        opal_output(0, "\nThe distance between slot0 and slot3 is %d\n", distance);
         distance_array = OBJ_NEW(opal_value_array_t);
         opal_value_array_init(distance_array, sizeof(opal_carto_node_distance_t));
         opal_value_array_reserve(distance_array, 50);
-        distance_array_size = opal_carto_base_get_nodes_distance(graph, slot0, NULL, distance_array);
-        for (i=0; i < distance_array_size; i++) {
+        distance_array_size = opal_carto_base_get_nodes_distance(graph, slot0, NULL,
+                                                                 distance_array);
+        for (i = 0; i < distance_array_size; i++) {
             node_distance = opal_value_array_get_item(distance_array, i);
-            opal_output(0,"Node %s distance from slot0 is %d\n",node_distance->node->node_name, node_distance->node_distance);
+            opal_output(0, "Node %s distance from slot0 is %d\n", node_distance->node->node_name,
+                        node_distance->node_distance);
         }
         OBJ_RELEASE(distance_array);
         opal_carto_base_free_graph(graph);
@@ -82,30 +81,32 @@ main(int argc, char* argv[])
          *
          */
 
-        opal_output(0," \n\nget_host_graph Infiniband\n");
-        opal_carto_base_get_host_graph(&graph,"Infiniband");
+        opal_output(0, " \n\nget_host_graph Infiniband\n");
+        opal_carto_base_get_host_graph(&graph, "Infiniband");
         opal_graph_print(graph);
         slot0 = opal_carto_base_find_node(graph, "slot0");
         if (NULL == slot0) {
-            opal_output(0,"couldn't find slot0 in the graph exiting\n");
+            opal_output(0, "couldn't find slot0 in the graph exiting\n");
             opal_carto_base_free_graph(graph);
             return -1;
         }
         end_node = opal_carto_base_find_node(graph, "mthca1");
         if (NULL == end_node) {
-            opal_output(0,"couldn't find mthca1 in the graph exiting\n");
+            opal_output(0, "couldn't find mthca1 in the graph exiting\n");
             opal_carto_base_free_graph(graph);
             return -1;
         }
         distance = opal_carto_base_spf(graph, slot0, end_node);
-        opal_output(0,"\nThe distance between slot0 and mthca1 is %d\n",distance);
+        opal_output(0, "\nThe distance between slot0 and mthca1 is %d\n", distance);
         distance_array = OBJ_NEW(opal_value_array_t);
         opal_value_array_init(distance_array, sizeof(opal_carto_node_distance_t));
         opal_value_array_reserve(distance_array, 50);
-        distance_array_size = opal_carto_base_get_nodes_distance(graph, slot0, "Infiniband", distance_array);
-        for (i=0; i < distance_array_size; i++) {
+        distance_array_size = opal_carto_base_get_nodes_distance(graph, slot0, "Infiniband",
+                                                                 distance_array);
+        for (i = 0; i < distance_array_size; i++) {
             node_distance = opal_value_array_get_item(distance_array, i);
-            opal_output(0,"Node %s distance from slot0 is %d\n",node_distance->node->node_name, node_distance->node_distance);
+            opal_output(0, "Node %s distance from slot0 is %d\n", node_distance->node->node_name,
+                        node_distance->node_distance);
         }
         OBJ_RELEASE(distance_array);
         opal_carto_base_free_graph(graph);
@@ -113,30 +114,32 @@ main(int argc, char* argv[])
          *
          */
 
-        opal_output(0," \n\nget_host_graph Ethernet\n");
-        opal_carto_base_get_host_graph(&graph,"Ethernet");
+        opal_output(0, " \n\nget_host_graph Ethernet\n");
+        opal_carto_base_get_host_graph(&graph, "Ethernet");
         opal_graph_print(graph);
         slot0 = opal_carto_base_find_node(graph, "slot0");
         if (NULL == slot0) {
-            opal_output(0,"couldn't find slot0 in the graph exiting\n");
+            opal_output(0, "couldn't find slot0 in the graph exiting\n");
             opal_carto_base_free_graph(graph);
             return -1;
         }
         end_node = opal_carto_base_find_node(graph, "eth1");
         if (NULL == end_node) {
-            opal_output(0,"couldn't find mthca1 in the graph exiting\n");
+            opal_output(0, "couldn't find mthca1 in the graph exiting\n");
             opal_carto_base_free_graph(graph);
             return -1;
         }
         distance = opal_carto_base_spf(graph, slot0, end_node);
-        opal_output(0,"\nThe distance between slot0 and eth1 is %d\n",distance);
+        opal_output(0, "\nThe distance between slot0 and eth1 is %d\n", distance);
         distance_array = OBJ_NEW(opal_value_array_t);
         opal_value_array_init(distance_array, sizeof(opal_carto_node_distance_t));
         opal_value_array_reserve(distance_array, 50);
-        distance_array_size = opal_carto_base_get_nodes_distance(graph, slot0, "Ethernet", distance_array);
-        for (i=0; i < distance_array_size; i++) {
+        distance_array_size = opal_carto_base_get_nodes_distance(graph, slot0, "Ethernet",
+                                                                 distance_array);
+        for (i = 0; i < distance_array_size; i++) {
             node_distance = opal_value_array_get_item(distance_array, i);
-            opal_output(0,"Node %s distance from slot0 is %d\n",node_distance->node->node_name, node_distance->node_distance);
+            opal_output(0, "Node %s distance from slot0 is %d\n", node_distance->node->node_name,
+                        node_distance->node_distance);
         }
         OBJ_RELEASE(distance_array);
         opal_carto_base_free_graph(graph);
@@ -144,39 +147,38 @@ main(int argc, char* argv[])
          *
          */
 
-        opal_output(0," \n\nget_host_graph Memory\n");
-        opal_carto_base_get_host_graph(&graph,"Memory");
+        opal_output(0, " \n\nget_host_graph Memory\n");
+        opal_carto_base_get_host_graph(&graph, "Memory");
         opal_graph_print(graph);
         slot0 = opal_carto_base_find_node(graph, "slot0");
         if (NULL == slot0) {
-            opal_output(0,"couldn't find slot0 in the graph exiting\n");
+            opal_output(0, "couldn't find slot0 in the graph exiting\n");
             opal_carto_base_free_graph(graph);
             return -1;
         }
         end_node = opal_carto_base_find_node(graph, "mem3");
         if (NULL == end_node) {
-            opal_output(0,"couldn't find mthca1 in the graph exiting\n");
+            opal_output(0, "couldn't find mthca1 in the graph exiting\n");
             opal_carto_base_free_graph(graph);
             return -1;
         }
         distance = opal_carto_base_spf(graph, slot0, end_node);
-        opal_output(0,"\nThe distance between slot0 and mem3 is %d\n",distance);
+        opal_output(0, "\nThe distance between slot0 and mem3 is %d\n", distance);
         distance_array = OBJ_NEW(opal_value_array_t);
         opal_value_array_init(distance_array, sizeof(opal_carto_node_distance_t));
         opal_value_array_reserve(distance_array, 50);
-        distance_array_size = opal_carto_base_get_nodes_distance(graph, slot0, "Memory", distance_array);
-        for (i=0; i < distance_array_size; i++) {
+        distance_array_size = opal_carto_base_get_nodes_distance(graph, slot0, "Memory",
+                                                                 distance_array);
+        for (i = 0; i < distance_array_size; i++) {
             node_distance = opal_value_array_get_item(distance_array, i);
-            opal_output(0,"Node %s distance from slot0 is %d\n",node_distance->node->node_name, node_distance->node_distance);
+            opal_output(0, "Node %s distance from slot0 is %d\n", node_distance->node->node_name,
+                        node_distance->node_distance);
         }
         OBJ_RELEASE(distance_array);
-       opal_carto_base_free_graph(graph);
-
+        opal_carto_base_free_graph(graph);
     }
     MPI_Barrier(MPI_COMM_WORLD);
     MPI_Finalize();
 
     return 0;
-
 }
-
diff --git a/test/class/Makefile.am b/test/class/Makefile.am
index 4702b11c5d1..1e39017584b 100644
--- a/test/class/Makefile.am
+++ b/test/class/Makefile.am
@@ -35,67 +35,74 @@ check_PROGRAMS = \
 	opal_value_array \
 	opal_pointer_array \
 	opal_lifo \
-	opal_fifo
+	opal_fifo \
+	opal_cstring
 
 TESTS = $(check_PROGRAMS)
 
 opal_bitmap_SOURCES = opal_bitmap.c
 opal_bitmap_LDADD = \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
         $(top_builddir)/test/support/libsupport.a
 opal_bitmap_DEPENDENCIES = $(opal_bitmap_LDADD)
 
 opal_list_SOURCES = opal_list.c
 opal_list_LDADD = \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
         $(top_builddir)/test/support/libsupport.a
 opal_list_DEPENDENCIES = $(opal_list_LDADD)
 
 opal_hash_table_SOURCES = opal_hash_table.c
 opal_hash_table_LDADD = \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
         $(top_builddir)/test/support/libsupport.a
 opal_hash_table_DEPENDENCIES = $(opal_hash_table_LDADD)
 
 opal_proc_table_SOURCES = opal_proc_table.c
 opal_proc_table_LDADD = \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
         $(top_builddir)/test/support/libsupport.a
 opal_proc_table_DEPENDENCIES = $(opal_proc_table_LDADD)
 
 opal_pointer_array_SOURCES = opal_pointer_array.c
 opal_pointer_array_LDADD = \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
         $(top_builddir)/test/support/libsupport.a
 opal_pointer_array_DEPENDENCIES = $(opal_pointer_array_LDADD)
 
 opal_value_array_SOURCES = opal_value_array.c
 opal_value_array_LDADD = \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
         $(top_builddir)/test/support/libsupport.a
 opal_value_array_DEPENDENCIES = $(opal_value_array_LDADD)
 
 ompi_rb_tree_SOURCES = ompi_rb_tree.c
 ompi_rb_tree_LDADD = \
         $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
 	$(top_builddir)/test/support/libsupport.a
 ompi_rb_tree_DEPENDENCIES = $(ompi_rb_tree_LDADD)
 
 opal_lifo_SOURCES = opal_lifo.c
 opal_lifo_LDADD = \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
 	$(top_builddir)/test/support/libsupport.a
 opal_lifo_DEPENDENCIES = $(opal_lifo_LDADD)
 
 opal_fifo_SOURCES = opal_fifo.c
 opal_fifo_LDADD = \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
 	$(top_builddir)/test/support/libsupport.a
 opal_fifo_DEPENDENCIES = $(opal_fifo_LDADD)
 
+opal_cstring_SOURCES = opal_cstring.c
+opal_cstring_LDADD = \
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
+	$(top_builddir)/test/support/libsupport.a
+opal_cstring_DEPENDENCIES = $(opal_cstring_LDADD)
+
 clean-local:
 	rm -f opal_bitmap_test_out.txt opal_hash_table_test_out.txt opal_proc_table_test_out.txt
 
-distclean:
+distclean-local:
 	rm -rf *.dSYM .deps .libs *.log *.txt *.o *.trs $(check_PROGRAMS) Makefile
diff --git a/test/class/ompi_rb_tree.c b/test/class/ompi_rb_tree.c
index a13c0e055ec..a09cdbacf55 100644
--- a/test/class/ompi_rb_tree.c
+++ b/test/class/ompi_rb_tree.c
@@ -23,67 +23,60 @@
 #include "opal_config.h"
 #include <stdint.h>
 #ifdef HAVE_SYS_TIME_H
-#include <sys/time.h>
+#    include <sys/time.h>
 #else
-#include <sys/_time.h>
+#    include <sys/_time.h>
 #endif
-#include <string.h>
-#include "support.h"
 #include "opal/class/opal_rb_tree.h"
 #include "opal/mca/mpool/base/base.h"
+#include "support.h"
+#include <string.h>
 
 #define NUM_KEYS 10000
-#define SEED  1
-int keys[] = {
-    0, 1, 2, 3, 4, 5, 6, 7
-};
+#define SEED     1
+int keys[] = {0, 1, 2, 3, 4, 5, 6, 7};
 
-int values[] = {
-    10, 11, 12, 13, 14, 15, 16, 17
-};
+int values[] = {10, 11, 12, 13, 14, 15, 16, 17};
 
-int comp_fn(void * ele1, void * ele2);
+int comp_fn(void *ele1, void *ele2);
 void test1(void);
-int comp_key(void* key1, void* key2);
+int comp_key(void *key1, void *key2);
 void test_keys(void);
 
-int comp_fn(void * ele1, void * ele2)
+int comp_fn(void *ele1, void *ele2)
 {
-    if(*((int *) ele1) > *((int *) ele2)) {
-        return(1);
+    if (*((int *) ele1) > *((int *) ele2)) {
+        return (1);
     }
-    if(*((int *) ele1) < *((int *) ele2)) {
-        return(-1);
+    if (*((int *) ele1) < *((int *) ele2)) {
+        return (-1);
     }
-    return(0);
+    return (0);
 }
 
-struct my_key_t{
+struct my_key_t {
     void *base;
     void *bound;
-}; typedef struct my_key_t my_key_t;
+};
+typedef struct my_key_t my_key_t;
 
-struct my_val_t{
-    my_key_t* key;
+struct my_val_t {
+    my_key_t *key;
     int val;
-}; typedef struct my_val_t my_val_t;
-
+};
+typedef struct my_val_t my_val_t;
 
-int comp_key(void* key1, void* key2) {
-    if( ((my_key_t*) key1)->base <
-        ((my_key_t*) key2)->base) {
+int comp_key(void *key1, void *key2)
+{
+    if (((my_key_t *) key1)->base < ((my_key_t *) key2)->base) {
         return -1;
-    }
-    else if ( ((my_key_t*) key1)->base >
-              ((my_key_t*) key2)->bound) {
+    } else if (((my_key_t *) key1)->base > ((my_key_t *) key2)->bound) {
         return 1;
-    }
-    else {
+    } else {
         return 0;
     }
 }
 
-
 void test_keys(void)
 {
     opal_rb_tree_t tree;
@@ -98,40 +91,37 @@ void test_keys(void)
     OBJ_CONSTRUCT(&tree, opal_rb_tree_t);
     rc = opal_rb_tree_init(&tree, comp_key);
     srand(SEED);
-    for(i = 0; i < NUM_KEYS; i++) {
+    for (i = 0; i < NUM_KEYS; i++) {
         cur_key = &(keys[i]);
         cur_val = &(vals[i]);
         cur_val->key = cur_key;
         cur_val->val = i;
         tmp = (long) rand();
-        cur_key->base = (void*) tmp;
+        cur_key->base = (void *) tmp;
         tmp += (long) rand();
-        cur_key->bound = (void*) tmp;
+        cur_key->bound = (void *) tmp;
         rc = opal_rb_tree_insert(&tree, cur_key, cur_val);
-        if(OPAL_SUCCESS != rc) {
+        if (OPAL_SUCCESS != rc) {
             test_failure("error inserting element in the tree");
         }
     }
-    for(i = 0; i < NUM_KEYS; i+=2) {
+    for (i = 0; i < NUM_KEYS; i += 2) {
         cur_key = &(keys[i]);
         rc = opal_rb_tree_delete(&tree, cur_key);
-        if(OPAL_SUCCESS != rc) {
+        if (OPAL_SUCCESS != rc) {
             test_failure("error deleting element in the tree");
         }
     }
-    for(i = 1; i < NUM_KEYS; i+=2) {
+    for (i = 1; i < NUM_KEYS; i += 2) {
         cur_key = &(keys[i]);
-        cur_val = (my_val_t*) opal_rb_tree_find(&tree, cur_key);
-        if(cur_val == NULL) {
+        cur_val = (my_val_t *) opal_rb_tree_find(&tree, cur_key);
+        if (cur_val == NULL) {
             test_failure("lookup returned NULL item");
-        }
-        else if(cur_val->val != i && (cur_val->key->base > cur_key->base ||
-                                      cur_val->key->bound < cur_key->base)) {
-            sprintf(buf, "lookup returned invalid item, returned %d, extected %d",
-                    cur_val->val, i);
+        } else if (cur_val->val != i
+                   && (cur_val->key->base > cur_key->base || cur_val->key->bound < cur_key->base)) {
+            sprintf(buf, "lookup returned invalid item, returned %d, extected %d", cur_val->val, i);
             test_failure(buf);
         }
-
     }
 }
 
@@ -139,121 +129,121 @@ void test1(void)
 {
     opal_rb_tree_t tree;
     int rc;
-    void * result;
+    void *result;
 
     OBJ_CONSTRUCT(&tree, opal_rb_tree_t);
     rc = opal_rb_tree_init(&tree, comp_fn);
-    if(!test_verify_int(OPAL_SUCCESS, rc)) {
+    if (!test_verify_int(OPAL_SUCCESS, rc)) {
         test_failure("failed to properly initialize the tree");
     }
 
     rc = opal_rb_tree_insert(&tree, &keys[0], &values[0]);
-    if(!test_verify_int(OPAL_SUCCESS, rc)) {
+    if (!test_verify_int(OPAL_SUCCESS, rc)) {
         test_failure("failed to properly insert a new node");
     }
     result = opal_rb_tree_find(&tree, &keys[0]);
-    if(NULL == result) {
+    if (NULL == result) {
         test_failure("lookup returned null!");
     }
-    if(!test_verify_int(values[0], *((int *) result))) {
+    if (!test_verify_int(values[0], *((int *) result))) {
         test_failure("failed to properly insert a new node");
     }
 
     rc = opal_rb_tree_insert(&tree, &keys[1], &values[1]);
-    if(!test_verify_int(OPAL_SUCCESS, rc)) {
+    if (!test_verify_int(OPAL_SUCCESS, rc)) {
         test_failure("failed to properly insert a new node");
     }
     result = opal_rb_tree_find(&tree, &keys[1]);
-    if(NULL == result) {
+    if (NULL == result) {
         test_failure("lookup returned null!");
     }
-    if(!test_verify_int(values[1], *((int *) result))) {
+    if (!test_verify_int(values[1], *((int *) result))) {
         test_failure("failed to properly insert a new node");
     }
 
     rc = opal_rb_tree_insert(&tree, &keys[2], &values[2]);
-    if(!test_verify_int(OPAL_SUCCESS, rc)) {
+    if (!test_verify_int(OPAL_SUCCESS, rc)) {
         test_failure("failed to properly insert a new node");
     }
     result = opal_rb_tree_find(&tree, &keys[2]);
-    if(NULL == result) {
+    if (NULL == result) {
         test_failure("lookup returned null!");
     }
-    if(!test_verify_int(values[2], *((int *) result))) {
+    if (!test_verify_int(values[2], *((int *) result))) {
         test_failure("failed to properly insert a new node");
     }
 
     rc = opal_rb_tree_insert(&tree, &keys[3], &values[3]);
-    if(!test_verify_int(OPAL_SUCCESS, rc)) {
+    if (!test_verify_int(OPAL_SUCCESS, rc)) {
         test_failure("failed to properly insert a new node");
     }
     result = opal_rb_tree_find(&tree, &keys[3]);
-    if(NULL == result) {
+    if (NULL == result) {
         test_failure("lookup returned null!");
     }
-    if(!test_verify_int(values[3], *((int *) result))) {
+    if (!test_verify_int(values[3], *((int *) result))) {
         test_failure("failed to properly insert a new node");
     }
 
     rc = opal_rb_tree_insert(&tree, &keys[4], &values[4]);
-    if(!test_verify_int(OPAL_SUCCESS, rc)) {
+    if (!test_verify_int(OPAL_SUCCESS, rc)) {
         test_failure("failed to properly insert a new node");
     }
     result = opal_rb_tree_find(&tree, &keys[4]);
-    if(NULL == result) {
+    if (NULL == result) {
         test_failure("lookup returned null!");
     }
-    if(!test_verify_int(values[4], *((int *) result))) {
+    if (!test_verify_int(values[4], *((int *) result))) {
         test_failure("failed to properly insert a new node");
     }
 
     rc = opal_rb_tree_insert(&tree, &keys[5], &values[5]);
-    if(!test_verify_int(OPAL_SUCCESS, rc)) {
+    if (!test_verify_int(OPAL_SUCCESS, rc)) {
         test_failure("failed to properly insert a new node");
     }
     result = opal_rb_tree_find(&tree, &keys[5]);
-    if(NULL == result) {
+    if (NULL == result) {
         test_failure("lookup returned null!");
     }
-    if(!test_verify_int(values[5], *((int *) result))) {
+    if (!test_verify_int(values[5], *((int *) result))) {
         test_failure("failed to properly insert a new node");
     }
 
     rc = opal_rb_tree_insert(&tree, &keys[6], &values[6]);
-    if(!test_verify_int(OPAL_SUCCESS, rc)) {
+    if (!test_verify_int(OPAL_SUCCESS, rc)) {
         test_failure("failed to properly insert a new node");
     }
     result = opal_rb_tree_find(&tree, &keys[6]);
-    if(NULL == result) {
+    if (NULL == result) {
         test_failure("lookup returned null!");
     }
-    if(!test_verify_int(values[6], *((int *) result))) {
+    if (!test_verify_int(values[6], *((int *) result))) {
         test_failure("failed to properly insert a new node");
     }
 
     rc = opal_rb_tree_insert(&tree, &keys[7], &values[7]);
-    if(!test_verify_int(OPAL_SUCCESS, rc)) {
+    if (!test_verify_int(OPAL_SUCCESS, rc)) {
         test_failure("failed to properly insert a new node");
     }
     result = opal_rb_tree_find(&tree, &keys[7]);
-    if(NULL == result) {
+    if (NULL == result) {
         test_failure("lookup returned null!");
     }
-    if(!test_verify_int(values[7], *((int *) result))) {
+    if (!test_verify_int(values[7], *((int *) result))) {
         test_failure("failed to properly insert a new node");
     }
 
     rc = opal_rb_tree_size(&tree);
-    if(!test_verify_int(8, rc)) {
+    if (!test_verify_int(8, rc)) {
         test_failure("failed to properly insert a new node");
     }
 
     rc = opal_rb_tree_delete(&tree, &keys[0]);
-    if(!test_verify_int(OPAL_SUCCESS, rc)) {
+    if (!test_verify_int(OPAL_SUCCESS, rc)) {
         test_failure("failed to properly delete a node");
     }
     result = opal_rb_tree_find(&tree, &keys[0]);
-    if(NULL != result) {
+    if (NULL != result) {
         test_failure("lookup returned a value instead of null!");
     } else {
         test_success();
@@ -263,7 +253,7 @@ void test1(void)
 }
 
 /* the following test is based on memory lookups in the mpool */
-int mem_node_compare(void * key1, void * key2);
+int mem_node_compare(void *key1, void *key2);
 void test2(void);
 
 /* the maximum number of memory pools a piece of memory can be registered with */
@@ -272,34 +262,27 @@ void test2(void);
 /* the number of memory segments to allocate */
 #define NUM_ALLOCATIONS 500
 
-struct opal_test_rb_key_t
-{
-    void * bottom;          /* the bottom of the memory range */
-    void * top;             /* the top of the memory range */
+struct opal_test_rb_key_t {
+    void *bottom; /* the bottom of the memory range */
+    void *top;    /* the top of the memory range */
 };
 typedef struct opal_test_rb_key_t opal_test_rb_key_t;
 
-struct opal_test_rb_value_t
-{
+struct opal_test_rb_value_t {
     opal_free_list_item_t super; /* the parent class */
-    opal_test_rb_key_t key; /* the key which holds the memory pointers */
-    mca_mpool_base_module_t* registered_mpools[MAX_REGISTRATIONS];
-                            /* the mpools the memory is registered with */
+    opal_test_rb_key_t key;      /* the key which holds the memory pointers */
+    mca_mpool_base_module_t *registered_mpools[MAX_REGISTRATIONS];
+    /* the mpools the memory is registered with */
 };
 typedef struct opal_test_rb_value_t opal_test_rb_value_t;
 
 OBJ_CLASS_INSTANCE(opal_test_rb_value_t, opal_free_list_item_t, NULL, NULL);
 
-int mem_node_compare(void * key1, void * key2)
+int mem_node_compare(void *key1, void *key2)
 {
-    if(((opal_test_rb_key_t *) key1)->bottom <
-       ((opal_test_rb_key_t *) key2)->bottom)
-    {
+    if (((opal_test_rb_key_t *) key1)->bottom < ((opal_test_rb_key_t *) key2)->bottom) {
         return -1;
-    }
-    else if(((opal_test_rb_key_t *) key1)->bottom >
-            ((opal_test_rb_key_t *) key2)->top)
-    {
+    } else if (((opal_test_rb_key_t *) key1)->bottom > ((opal_test_rb_key_t *) key2)->top) {
         return 1;
     }
     return 0;
@@ -308,73 +291,60 @@ int mem_node_compare(void * key1, void * key2)
 void test2(void)
 {
     opal_free_list_t key_list;
-    opal_free_list_item_t * new_value;
+    opal_free_list_item_t *new_value;
     opal_rb_tree_t tree;
     int rc, i, size;
-    void * result, * lookup;
-    void * mem[NUM_ALLOCATIONS];
-    opal_free_list_item_t * key_array[NUM_ALLOCATIONS];
+    void *result, *lookup;
+    void *mem[NUM_ALLOCATIONS];
+    opal_free_list_item_t *key_array[NUM_ALLOCATIONS];
     struct timeval start, end;
 
     OBJ_CONSTRUCT(&key_list, opal_free_list_t);
-    opal_free_list_init (&key_list, sizeof(opal_test_rb_value_t),
-            opal_cache_line_size,
-            OBJ_CLASS(opal_test_rb_value_t),
-            0,opal_cache_line_size,
-            0, -1 , 128, NULL, 0, NULL, NULL, NULL);
+    opal_free_list_init(&key_list, sizeof(opal_test_rb_value_t), opal_cache_line_size,
+                        OBJ_CLASS(opal_test_rb_value_t), 0, opal_cache_line_size, 0, -1, 128, NULL,
+                        0, NULL, NULL, NULL);
 
     OBJ_CONSTRUCT(&tree, opal_rb_tree_t);
     rc = opal_rb_tree_init(&tree, mem_node_compare);
-    if(!test_verify_int(OPAL_SUCCESS, rc)) {
+    if (!test_verify_int(OPAL_SUCCESS, rc)) {
         test_failure("failed to properly initialize the tree");
     }
 
     size = 1;
-    for(i = 0; i < NUM_ALLOCATIONS; i++)
-    {
+    for (i = 0; i < NUM_ALLOCATIONS; i++) {
         mem[i] = malloc(size);
-        if(NULL == mem[i])
-        {
+        if (NULL == mem[i]) {
             test_failure("system out of memory");
             return;
         }
-        new_value = opal_free_list_get (&key_list);
-        if(NULL == new_value)
-        {
+        new_value = opal_free_list_get(&key_list);
+        if (NULL == new_value) {
             test_failure("failed to get memory from free list");
         }
         key_array[i] = new_value;
         ((opal_test_rb_value_t *) new_value)->key.bottom = mem[i];
-        ((opal_test_rb_value_t *) new_value)->key.top =
-                                            (void *) ((size_t) mem[i] + size - 1);
-        ((opal_test_rb_value_t *) new_value)->registered_mpools[0] = (void *)(intptr_t) i;
-        rc = opal_rb_tree_insert(&tree, &((opal_test_rb_value_t *)new_value)->key,
-                        new_value);
-        if(OPAL_SUCCESS != rc)
-        {
+        ((opal_test_rb_value_t *) new_value)->key.top = (void *) ((size_t) mem[i] + size - 1);
+        ((opal_test_rb_value_t *) new_value)->registered_mpools[0] = (void *) (intptr_t) i;
+        rc = opal_rb_tree_insert(&tree, &((opal_test_rb_value_t *) new_value)->key, new_value);
+        if (OPAL_SUCCESS != rc) {
             test_failure("failed to properly insert a new node");
         }
         size += 1;
     }
 
     gettimeofday(&start, NULL);
-    for(i = 0; i < NUM_ALLOCATIONS; i++)
-    {
+    for (i = 0; i < NUM_ALLOCATIONS; i++) {
         lookup = (void *) ((size_t) mem[i] + i);
         result = opal_rb_tree_find(&tree, &lookup);
-        if(NULL == result)
-        {
+        if (NULL == result) {
             test_failure("lookup returned null!");
-        } else if(i != ((int)(intptr_t) ((opal_test_rb_value_t *) result)->registered_mpools[0]))
-        {
+        } else if (i != ((int) (intptr_t)((opal_test_rb_value_t *) result)->registered_mpools[0])) {
             test_failure("lookup returned wrong node!");
         }
         result = opal_rb_tree_find(&tree, &lookup);
-        if(NULL == result)
-        {
+        if (NULL == result) {
             test_failure("lookup returned null!");
-        } else if(i != ((int)(intptr_t) ((opal_test_rb_value_t *) result)->registered_mpools[0]))
-        {
+        } else if (i != ((int) (intptr_t)((opal_test_rb_value_t *) result)->registered_mpools[0])) {
             test_failure("lookup returned wrong node!");
         }
     }
@@ -383,18 +353,16 @@ void test2(void)
 
 #if 0
     i = (end.tv_sec - start.tv_sec) * 1000000 + (end.tv_usec - start.tv_usec);
-    printf("In a %d node tree, %d lookups took %f microseonds each\n",
+    printf("In a %d node tree, %d lookups took %f microseconds each\n",
             NUM_ALLOCATIONS, NUM_ALLOCATIONS * 2,
             (float) i / (float) (NUM_ALLOCATIONS * 2));
 #endif
 
-    for(i = 0; i < NUM_ALLOCATIONS; i++)
-    {
-        if(NULL != mem[i])
-        {
+    for (i = 0; i < NUM_ALLOCATIONS; i++) {
+        if (NULL != mem[i]) {
             free(mem[i]);
         }
-        opal_free_list_return (&(key_list), key_array[i]);
+        opal_free_list_return(&(key_list), key_array[i]);
     }
 
     OBJ_DESTRUCT(&tree);
diff --git a/test/class/opal_bitmap.c b/test/class/opal_bitmap.c
index 58778410645..b878a97dcb6 100644
--- a/test/class/opal_bitmap.c
+++ b/test/class/opal_bitmap.c
@@ -4,19 +4,19 @@
 
 #include "opal_config.h"
 
-#include <stdio.h>
 #include "support.h"
+#include <stdio.h>
 
 #include "opal/class/opal_bitmap.h"
 #include "opal/constants.h"
 
-#define BSIZE 26
+#define BSIZE            26
 #define OPAL_INVALID_BIT -1
-#define ERR_CODE -2
+#define ERR_CODE         -2
 
-#define PRINT_VALID_ERR \
+#define PRINT_VALID_ERR                                        \
     fprintf(error_out, "================================ \n"); \
-    fprintf(error_out, "This is suppossed to throw error \n"); \
+    fprintf(error_out, "This is supposed to throw error  \n"); \
     fprintf(error_out, "================================ \n")
 
 static void test_bitmap_set(opal_bitmap_t *bm);
@@ -26,7 +26,6 @@ static void test_bitmap_clear_all(opal_bitmap_t *bm);
 static void test_bitmap_set_all(opal_bitmap_t *bm);
 static void test_bitmap_find_and_set(opal_bitmap_t *bm);
 
-
 static int set_bit(opal_bitmap_t *bm, int bit);
 static int clear_bit(opal_bitmap_t *bm, int bit);
 static int is_set_bit(opal_bitmap_t *bm, int bit);
@@ -39,7 +38,7 @@ static int find_and_set(opal_bitmap_t *bm, int bit);
 static void print_bitmap(opal_bitmap_t *bm);
 #endif
 
-static FILE *error_out=NULL;
+static FILE *error_out = NULL;
 
 int main(int argc, char *argv[])
 {
@@ -53,8 +52,9 @@ int main(int argc, char *argv[])
 #ifdef STANDALONE
     error_out = stderr;
 #else
-    error_out = fopen( "./opal_bitmap_test_out.txt", "w" );
-    if( error_out == NULL ) error_out = stderr;
+    error_out = fopen("./opal_bitmap_test_out.txt", "w");
+    if (error_out == NULL)
+        error_out = stderr;
 #endif
 
     /* Initialize bitmap  */
@@ -62,17 +62,17 @@ int main(int argc, char *argv[])
     PRINT_VALID_ERR;
     err = opal_bitmap_init(NULL, 2);
     if (err == OPAL_ERR_BAD_PARAM)
-	fprintf(error_out, "ERROR: Initialization of bitmap failed\n\n");
+        fprintf(error_out, "ERROR: Initialization of bitmap failed\n\n");
 
     PRINT_VALID_ERR;
     err = opal_bitmap_init(&bm, -1);
     if (err == OPAL_ERR_BAD_PARAM)
-	fprintf(error_out, "ERROR: Initialization of bitmap failed \n\n");
+        fprintf(error_out, "ERROR: Initialization of bitmap failed \n\n");
 
     err = opal_bitmap_init(&bm, BSIZE);
     if (0 > err) {
-	fprintf(error_out, "Error in bitmap create -- aborting \n");
-	exit(-1);
+        fprintf(error_out, "Error in bitmap create -- aborting \n");
+        exit(-1);
     }
 
     fprintf(error_out, "\nTesting bitmap set... \n");
@@ -103,10 +103,9 @@ int main(int argc, char *argv[])
     return 0;
 }
 
-
-
-void test_bitmap_set(opal_bitmap_t *bm) {
-    int result=0;
+void test_bitmap_set(opal_bitmap_t *bm)
+{
+    int result = 0;
 
     /* start of bitmap and boundaries */
     set_bit(bm, 0);
@@ -127,12 +126,12 @@ void test_bitmap_set(opal_bitmap_t *bm) {
     /* invalid bit */
     PRINT_VALID_ERR;
     result = set_bit(bm, -1);
-    TEST_AND_REPORT(result, ERR_CODE,"opal_bitmap_set_bit");
+    TEST_AND_REPORT(result, ERR_CODE, "opal_bitmap_set_bit");
 }
 
-
-void test_bitmap_clear(opal_bitmap_t *bm) {
-    int result=0;
+void test_bitmap_clear(opal_bitmap_t *bm)
+{
+    int result = 0;
 
     /* Valid set bits  */
     clear_bit(bm, 29);
@@ -144,16 +143,15 @@ void test_bitmap_clear(opal_bitmap_t *bm) {
     /* invalid bit */
     PRINT_VALID_ERR;
     result = clear_bit(bm, -1);
-    TEST_AND_REPORT(result, ERR_CODE,"opal_bitmap_clear_bit");
+    TEST_AND_REPORT(result, ERR_CODE, "opal_bitmap_clear_bit");
     PRINT_VALID_ERR;
     result = clear_bit(bm, 142);
-    TEST_AND_REPORT(result, ERR_CODE,"opal_bitmap_clear_bit");
+    TEST_AND_REPORT(result, ERR_CODE, "opal_bitmap_clear_bit");
 }
 
-
 void test_bitmap_is_set(opal_bitmap_t *bm)
 {
-    int result=0;
+    int result = 0;
 
     /* First set some bits */
     test_bitmap_set(bm);
@@ -163,18 +161,17 @@ void test_bitmap_is_set(opal_bitmap_t *bm)
     is_set_bit(bm, 32);
 
     result = is_set_bit(bm, 1122);
-    TEST_AND_REPORT(result,0,"opal_bitmap_is_set_bit");
+    TEST_AND_REPORT(result, 0, "opal_bitmap_is_set_bit");
     is_set_bit(bm, -33);
-    TEST_AND_REPORT(result,0,"opal_bitmap_is_set_bit");
+    TEST_AND_REPORT(result, 0, "opal_bitmap_is_set_bit");
     is_set_bit(bm, -1);
-    TEST_AND_REPORT(result,0,"opal_bitmap_is_set_bit");
+    TEST_AND_REPORT(result, 0, "opal_bitmap_is_set_bit");
 }
 
-
 void test_bitmap_find_and_set(opal_bitmap_t *bm)
 {
     int bsize;
-    int result=0;
+    int result = 0;
 
     opal_bitmap_clear_all_bits(bm);
     result = find_and_set(bm, 0);
@@ -213,7 +210,6 @@ void test_bitmap_clear_all(opal_bitmap_t *bm)
     TEST_AND_REPORT(result, 0, " error in opal_bitmap_clear_all_bits");
 }
 
-
 void test_bitmap_set_all(opal_bitmap_t *bm)
 {
     int result = set_all(bm);
@@ -224,32 +220,30 @@ int set_bit(opal_bitmap_t *bm, int bit)
 {
     int err = opal_bitmap_set_bit(bm, bit);
     if (err != 0 || !opal_bitmap_is_set_bit(bm, bit)) {
-	    fprintf(error_out, "ERROR: set_bit for bit = %d\n\n", bit);
-	    return ERR_CODE;
-	}
+        fprintf(error_out, "ERROR: set_bit for bit = %d\n\n", bit);
+        return ERR_CODE;
+    }
     return 0;
 }
 
-
 int clear_bit(opal_bitmap_t *bm, int bit)
 {
     int err = opal_bitmap_clear_bit(bm, bit);
     if ((err != 0) || opal_bitmap_is_set_bit(bm, bit)) {
-	fprintf(error_out, "ERROR: clear_bit for bit = %d \n\n", bit);
-	return ERR_CODE;
+        fprintf(error_out, "ERROR: clear_bit for bit = %d \n\n", bit);
+        return ERR_CODE;
     }
 
     return 0;
 }
 
-
 int is_set_bit(opal_bitmap_t *bm, int bit)
 {
     bool result = opal_bitmap_is_set_bit(bm, bit);
 
     if (result) {
         if (bit < 0) {
-            fprintf(error_out, "ERROR: is_set_bit for bit = %d \n\n",bit);
+            fprintf(error_out, "ERROR: is_set_bit for bit = %d \n\n", bit);
             return ERR_CODE;
         }
         return 0;
@@ -257,7 +251,7 @@ int is_set_bit(opal_bitmap_t *bm, int bit)
 
     if (!result) {
         if (0 <= bit && bit <= bm->array_size && !opal_bitmap_is_set_bit(bm, bit)) {
-            fprintf(error_out, "ERROR: is_set_bit for bit = %d \n\n",bit);
+            fprintf(error_out, "ERROR: is_set_bit for bit = %d \n\n", bit);
             return ERR_CODE;
         }
         return 0;
@@ -266,7 +260,6 @@ int is_set_bit(opal_bitmap_t *bm, int bit)
     return 0;
 }
 
-
 int find_and_set(opal_bitmap_t *bm, int bit)
 {
     int ret, pos;
@@ -274,18 +267,17 @@ int find_and_set(opal_bitmap_t *bm, int bit)
        level stub, this function will be called in sequence to test */
 
     ret = opal_bitmap_find_and_set_first_unset_bit(bm, &pos);
-    if (ret != OPAL_SUCCESS) return ret;
+    if (ret != OPAL_SUCCESS)
+        return ret;
 
     if (pos != bit) {
-	fprintf(error_out, "ERROR: find_and_set: expected to find_and_set %d\n\n",
-		bit);
-	return ERR_CODE;
+        fprintf(error_out, "ERROR: find_and_set: expected to find_and_set %d\n\n", bit);
+        return ERR_CODE;
     }
 
     return 0;
 }
 
-
 int clear_all(opal_bitmap_t *bm)
 {
     int i;
@@ -293,30 +285,27 @@ int clear_all(opal_bitmap_t *bm)
         return ERR_CODE;
     }
     for (i = 0; i < bm->array_size; ++i)
-	if (bm->bitmap[i] != 0) {
-	    fprintf(error_out, "ERROR: clear_all for bitmap array entry %d\n\n",
-		    i);
-	    return ERR_CODE;
-	}
+        if (bm->bitmap[i] != 0) {
+            fprintf(error_out, "ERROR: clear_all for bitmap array entry %d\n\n", i);
+            return ERR_CODE;
+        }
     return 0;
 }
 
-
 int set_all(opal_bitmap_t *bm)
 {
-   int i;
-   if (OPAL_SUCCESS != opal_bitmap_set_all_bits(bm)) {
-       return ERR_CODE;
-   }
-   for (i = 0; i < bm->array_size; ++i)
-       if (bm->bitmap[i] != 0xffffffffffffffffUL) {
-	   fprintf(error_out, "ERROR: set_all for bitmap arry entry %d\n\n", i);
-	   return ERR_CODE;
-       }
-   return 0;
+    int i;
+    if (OPAL_SUCCESS != opal_bitmap_set_all_bits(bm)) {
+        return ERR_CODE;
+    }
+    for (i = 0; i < bm->array_size; ++i)
+        if (bm->bitmap[i] != 0xffffffffffffffffUL) {
+            fprintf(error_out, "ERROR: set_all for bitmap array entry %d\n\n", i);
+            return ERR_CODE;
+        }
+    return 0;
 }
 
-
 #if WANT_PRINT_BITMAP
 void print_bitmap(opal_bitmap_t *bm)
 {
@@ -325,8 +314,7 @@ void print_bitmap(opal_bitmap_t *bm)
 
     int i;
     for (i = 0; i < bm->array_size; ++i) {
-	fprintf(error_out, "---\n bitmap[%d] = %x \n---\n\n", i,
-		(bm->bitmap[i] & 0xff));
+        fprintf(error_out, "---\n bitmap[%d] = %x \n---\n\n", i, (bm->bitmap[i] & 0xff));
     }
     fprintf(error_out, "========================= \n");
     return;
diff --git a/test/class/opal_cstring.c b/test/class/opal_cstring.c
new file mode 100644
index 00000000000..d59c3573f48
--- /dev/null
+++ b/test/class/opal_cstring.c
@@ -0,0 +1,247 @@
+/*
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "opal_config.h"
+
+#include <assert.h>
+
+#include "opal/class/opal_cstring.h"
+#include <stdio.h>
+#include <stdlib.h>
+
+#define CHECK_INT(actual, expected)                   \
+    if (expected != actual) {                         \
+        printf("%s:%d: expected %d, got %d\n",        \
+               __FILE__, __LINE__, expected, actual); \
+        exit(1);                                      \
+    }
+
+#define CHECK_BOOL(actual, expected)                  \
+    if (expected != actual) {                         \
+        printf("%s:%d: expected %s, got %s\n",        \
+               __FILE__, __LINE__,                    \
+               expected ? "true" : "false",           \
+               actual ? "true" : "false");            \
+        exit(1);                                      \
+    }
+
+int main(int argc, char *argv[])
+{
+    int ret, int_val;
+    bool bool_val;
+    opal_cstring_t *cstr;
+
+    /*
+     * bool tests
+     */
+    cstr = opal_cstring_create("true");
+    ret = opal_cstring_to_bool(cstr, &bool_val);
+    CHECK_INT(ret, OPAL_SUCCESS);
+    CHECK_BOOL(bool_val, true);
+    OBJ_RELEASE(cstr);
+
+    cstr = opal_cstring_create("yes");
+    ret = opal_cstring_to_bool(cstr, &bool_val);
+    CHECK_INT(ret, OPAL_SUCCESS);
+    CHECK_BOOL(bool_val, true);
+    OBJ_RELEASE(cstr);
+
+    cstr = opal_cstring_create("false");
+    ret = opal_cstring_to_bool(cstr, &bool_val);
+    CHECK_INT(ret, OPAL_SUCCESS);
+    CHECK_BOOL(bool_val, false);
+    OBJ_RELEASE(cstr);
+
+    cstr = opal_cstring_create("no");
+    ret = opal_cstring_to_bool(cstr, &bool_val);
+    CHECK_INT(ret, OPAL_SUCCESS);
+    CHECK_BOOL(bool_val, false);
+    OBJ_RELEASE(cstr);
+
+    cstr = opal_cstring_create("1");
+    ret = opal_cstring_to_bool(cstr, &bool_val);
+    CHECK_INT(ret, OPAL_SUCCESS);
+    CHECK_BOOL(bool_val, true);
+    OBJ_RELEASE(cstr);
+
+    cstr = opal_cstring_create("0");
+    ret = opal_cstring_to_bool(cstr, &bool_val);
+    CHECK_INT(ret, OPAL_SUCCESS);
+    CHECK_BOOL(bool_val, false);
+    OBJ_RELEASE(cstr);
+
+    cstr = opal_cstring_create("1.0");
+    ret = opal_cstring_to_bool(cstr, &bool_val);
+    CHECK_INT(ret, OPAL_SUCCESS);
+    CHECK_BOOL(bool_val, true);
+    OBJ_RELEASE(cstr);
+
+    cstr = opal_cstring_create("0.0");
+    ret = opal_cstring_to_bool(cstr, &bool_val);
+    CHECK_INT(ret, OPAL_SUCCESS);
+    CHECK_BOOL(bool_val, false);
+    OBJ_RELEASE(cstr);
+
+    cstr = opal_cstring_create("foobar");
+    ret = opal_cstring_to_bool(cstr, &bool_val);
+    CHECK_INT(ret, OPAL_ERR_BAD_PARAM);
+    OBJ_RELEASE(cstr);
+
+    cstr = opal_cstring_create(NULL);
+    ret = opal_cstring_to_bool(cstr, &bool_val);
+    CHECK_INT(ret, OPAL_ERR_BAD_PARAM);
+    OBJ_RELEASE(cstr);
+
+
+    /*
+     * bool string tests
+     */
+    bool_val = opal_str_to_bool("true");
+    CHECK_BOOL(bool_val, true);
+
+    bool_val = opal_str_to_bool("yes");
+    CHECK_BOOL(bool_val, true);
+
+    bool_val = opal_str_to_bool("false");
+    CHECK_BOOL(bool_val, false);
+
+    bool_val = opal_str_to_bool("no");
+    CHECK_BOOL(bool_val, false);
+
+    bool_val = opal_str_to_bool("1");
+    CHECK_BOOL(bool_val, true);
+
+    bool_val = opal_str_to_bool("0");
+    CHECK_BOOL(bool_val, false);
+
+    bool_val = opal_str_to_bool("1.0");
+    CHECK_BOOL(bool_val, true);
+
+    bool_val = opal_str_to_bool("0.0");
+    CHECK_BOOL(bool_val, false);
+
+    bool_val = opal_str_to_bool("foobar");
+    CHECK_BOOL(bool_val, false);
+
+    bool_val = opal_str_to_bool(NULL);
+    CHECK_BOOL(bool_val, false);
+
+    /*
+     * integer tests
+     */
+    cstr = opal_cstring_create("1");
+    ret = opal_cstring_to_int(cstr, &int_val);
+    CHECK_INT(ret, OPAL_SUCCESS);
+    CHECK_INT(int_val, 1);
+    OBJ_RELEASE(cstr);
+
+    cstr = opal_cstring_create("0");
+    ret = opal_cstring_to_int(cstr, &int_val);
+    CHECK_INT(ret, OPAL_SUCCESS);
+    CHECK_INT(int_val, 0);
+    OBJ_RELEASE(cstr);
+
+    /* test intentional overflow cases */
+    if (sizeof(long) > sizeof(int)) {
+        long input = (long)INT_MAX + 1L;
+        char *input_str;
+
+        ret = asprintf(&input_str, "%ld", input);
+        if (ret < 0) {
+            printf("%s:%d: asprintf()", __FILE__, __LINE__);
+            exit(1);
+        }
+
+        cstr = opal_cstring_create(input_str);
+        ret = opal_cstring_to_int(cstr, &int_val);
+        CHECK_INT(ret, OPAL_ERR_BAD_PARAM);
+        OBJ_RELEASE(cstr);
+        free(input_str);
+    }
+
+    /* yes, this may be redundant from above, but always try both
+       cases for simplicity */
+    if (sizeof(long long) > sizeof(int)) {
+        long long input = (long long)INT_MAX + 1L;
+        char *input_str;
+
+        ret = asprintf(&input_str, "%lld", input);
+        if (ret < 0) {
+            printf("%s:%d: asprintf()", __FILE__, __LINE__);
+            exit(1);
+        }
+
+        cstr = opal_cstring_create(input_str);
+        ret = opal_cstring_to_int(cstr, &int_val);
+        CHECK_INT(ret, OPAL_ERR_BAD_PARAM);
+        OBJ_RELEASE(cstr);
+        free(input_str);
+    }
+
+    {
+        long input = (long)LONG_MAX;
+        char *input_str;
+
+        ret = asprintf(&input_str, "%ld0", input);
+        if (ret < 0) {
+            printf("%s:%d: asprintf()", __FILE__, __LINE__);
+            exit(1);
+        }
+
+        cstr = opal_cstring_create(input_str);
+        ret = opal_cstring_to_int(cstr, &int_val);
+        CHECK_INT(ret, OPAL_ERR_BAD_PARAM);
+        OBJ_RELEASE(cstr);
+        free(input_str);
+    }
+
+
+    cstr = opal_cstring_create("1.0");
+    ret = opal_cstring_to_int(cstr, &int_val);
+    CHECK_INT(ret, OPAL_ERR_BAD_PARAM);
+    OBJ_RELEASE(cstr);
+
+    cstr = opal_cstring_create("0.0");
+    ret = opal_cstring_to_int(cstr, &int_val);
+    CHECK_INT(ret, OPAL_ERR_BAD_PARAM);
+    OBJ_RELEASE(cstr);
+
+    cstr = opal_cstring_create("foobar");
+    ret = opal_cstring_to_int(cstr, &int_val);
+    CHECK_INT(ret, OPAL_ERR_BAD_PARAM);
+    OBJ_RELEASE(cstr);
+
+    cstr = opal_cstring_create(NULL);
+    ret = opal_cstring_to_int(cstr, &int_val);
+    CHECK_INT(ret, OPAL_ERR_BAD_PARAM);
+    OBJ_RELEASE(cstr);
+
+    cstr = opal_cstring_create("true");
+    ret = opal_cstring_to_int(cstr, &int_val);
+    CHECK_INT(ret, OPAL_ERR_BAD_PARAM);
+    OBJ_RELEASE(cstr);
+
+    cstr = opal_cstring_create("yes");
+    ret = opal_cstring_to_int(cstr, &int_val);
+    CHECK_INT(ret, OPAL_ERR_BAD_PARAM);
+    OBJ_RELEASE(cstr);
+
+    cstr = opal_cstring_create("false");
+    ret = opal_cstring_to_int(cstr, &int_val);
+    CHECK_INT(ret, OPAL_ERR_BAD_PARAM);
+    OBJ_RELEASE(cstr);
+
+    cstr = opal_cstring_create("no");
+    ret = opal_cstring_to_int(cstr, &int_val);
+    CHECK_INT(ret, OPAL_ERR_BAD_PARAM);
+    OBJ_RELEASE(cstr);
+
+    return 0;
+}
diff --git a/test/class/opal_fifo.c b/test/class/opal_fifo.c
index 9a2ea797866..67a497c658f 100644
--- a/test/class/opal_fifo.c
+++ b/test/class/opal_fifo.c
@@ -13,110 +13,116 @@
 #include "opal_config.h"
 #include <assert.h>
 
-#include "support.h"
 #include "opal/class/opal_fifo.h"
-#include "opal/runtime/opal.h"
 #include "opal/constants.h"
 #include "opal/mca/threads/threads.h"
+#include "opal/runtime/opal.h"
+#include "support.h"
 
-#include <stdlib.h>
-#include <stdio.h>
 #include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
 
 #include <sys/time.h>
 
 #define OPAL_FIFO_TEST_THREAD_COUNT 8
-#define ITERATIONS 1000000
-#define ITEM_COUNT 100
-#define ITEMS_PER_LOOP 30
+#define ITERATIONS                  1000000
+#define ITEM_COUNT                  100
+#define ITEMS_PER_LOOP              30
 
 #if !defined(timersub)
-#define timersub(a, b, r) \
-    do {                  \
-        (r)->tv_sec = (a)->tv_sec - (b)->tv_sec;        \
-        if ((a)->tv_usec < (b)->tv_usec) {              \
-            (r)->tv_sec--;                              \
-            (a)->tv_usec += 1000000;                    \
-        }                                               \
-        (r)->tv_usec = (a)->tv_usec - (b)->tv_usec;     \
-    } while (0)
+#    define timersub(a, b, r)                           \
+        do {                                            \
+            (r)->tv_sec = (a)->tv_sec - (b)->tv_sec;    \
+            if ((a)->tv_usec < (b)->tv_usec) {          \
+                (r)->tv_sec--;                          \
+                (a)->tv_usec += 1000000;                \
+            }                                           \
+            (r)->tv_usec = (a)->tv_usec - (b)->tv_usec; \
+        } while (0)
 #endif
 
-static void *thread_test (opal_object_t *arg) {
+static void *thread_test(opal_object_t *arg)
+{
     opal_thread_t *t = (opal_thread_t *) arg;
     opal_fifo_t *fifo = (opal_fifo_t *) t->t_arg;
     opal_list_item_t *item;
     struct timeval start, stop, total;
     double timing;
 
-    gettimeofday (&start, NULL);
-    for (int i = 0 ; i < ITERATIONS ; ++i) {
-        item = opal_fifo_pop_atomic (fifo);
+    gettimeofday(&start, NULL);
+    for (int i = 0; i < ITERATIONS; ++i) {
+        item = opal_fifo_pop_atomic(fifo);
         if (NULL != item) {
-            (void) opal_fifo_push_atomic (fifo, item);
+            (void) opal_fifo_push_atomic(fifo, item);
         }
     }
-    gettimeofday (&stop, NULL);
+    gettimeofday(&stop, NULL);
 
     timersub(&stop, &start, &total);
 
     timing = ((double) total.tv_sec + (double) total.tv_usec * 1e-6) / (double) ITERATIONS;
 
-    printf ("Atomics thread finished. Time: %d s %d us %d nsec/poppush\n", (int) total.tv_sec,
-            (int)total.tv_usec, (int)(timing / 1e-9));
+    printf("Atomics thread finished. Time: %d s %d us %d nsec/poppush\n", (int) total.tv_sec,
+           (int) total.tv_usec, (int) (timing / 1e-9));
 
     return NULL;
 }
 
-static void *thread_test_exhaust (opal_object_t *arg) {
-  opal_thread_t *t = (opal_thread_t *) arg;
-  opal_fifo_t *fifo = (opal_fifo_t *) t->t_arg;
-  opal_list_item_t *items[ITEMS_PER_LOOP];
-  struct timeval start, stop, total;
-  int item_count = 0;
-  double timing;
-
-  gettimeofday (&start, NULL);
-
-  for (int i = 0 ; i < ITERATIONS ; i += ITEMS_PER_LOOP) {
-    for (int j = 0 ; j < ITEMS_PER_LOOP ; ++j) {
-      items[j] = opal_fifo_pop_atomic (fifo);
-      if (items[j]) {
-	++item_count;
-      }
-    }
+static void *thread_test_exhaust(opal_object_t *arg)
+{
+    opal_thread_t *t = (opal_thread_t *) arg;
+    opal_fifo_t *fifo = (opal_fifo_t *) t->t_arg;
+    opal_list_item_t *items[ITEMS_PER_LOOP];
+    struct timeval start, stop, total;
+    int item_count = 0;
+    double timing;
+
+    gettimeofday(&start, NULL);
+
+    for (int i = 0; i < ITERATIONS; i += ITEMS_PER_LOOP) {
+        for (int j = 0; j < ITEMS_PER_LOOP; ++j) {
+            items[j] = opal_fifo_pop_atomic(fifo);
+            if (items[j]) {
+                ++item_count;
+            }
+        }
 
-    for (int j = 0 ; j < ITEMS_PER_LOOP ; ++j) {
-      if (items[j]) {
-	(void) opal_fifo_push_atomic (fifo, items[j]);
-      }
+        for (int j = 0; j < ITEMS_PER_LOOP; ++j) {
+            if (items[j]) {
+                (void) opal_fifo_push_atomic(fifo, items[j]);
+            }
+        }
     }
-  }
 
-  gettimeofday (&stop, NULL);
+    gettimeofday(&stop, NULL);
 
-  timersub(&stop, &start, &total);
+    timersub(&stop, &start, &total);
 
-  timing = ((double) total.tv_sec + (double) total.tv_usec * 1e-6) / (double) item_count;
+    timing = ((double) total.tv_sec + (double) total.tv_usec * 1e-6) / (double) item_count;
 
-  fprintf (stderr, "Exhaustive atomics thread finished. Popped %d items. Time: %d s %d us %d nsec/poppush\n", item_count,
-	   (int) total.tv_sec, (int)total.tv_usec, (int)(timing / 1e-9));
+    fprintf(
+        stderr,
+        "Exhaustive atomics thread finished. Popped %d items. Time: %d s %d us %d nsec/poppush\n",
+        item_count, (int) total.tv_sec, (int) total.tv_usec, (int) (timing / 1e-9));
 
-  return NULL;
+    return NULL;
 }
 
-static bool check_fifo_consistency (opal_fifo_t *fifo, int expected_count)
+static bool check_fifo_consistency(opal_fifo_t *fifo, int expected_count)
 {
-    opal_list_item_t * item;
+    opal_list_item_t *item;
     int count;
 
-    for (count = 0, item = (opal_list_item_t *) fifo->opal_fifo_head.data.item ; item != &fifo->opal_fifo_ghost ;
-         item = opal_list_get_next(item), count++);
+    for (count = 0, item = (opal_list_item_t *) fifo->opal_fifo_head.data.item;
+         item != &fifo->opal_fifo_ghost; item = opal_list_get_next(item), count++)
+        ;
 
     return count == expected_count;
 }
 
-int main (int argc, char *argv[]) {
+int main(int argc, char *argv[])
+{
     opal_thread_t threads[OPAL_FIFO_TEST_THREAD_COUNT];
     opal_list_item_t *item, *prev, *item2;
     struct timeval start, stop, total;
@@ -125,11 +131,11 @@ int main (int argc, char *argv[]) {
     double timing;
     int rc;
 
-    rc = opal_init_util (&argc, &argv);
+    rc = opal_init_util(&argc, &argv);
     test_verify_int(OPAL_SUCCESS, rc);
     if (OPAL_SUCCESS != rc) {
         test_finalize();
-        exit (1);
+        exit(1);
     }
 
     test_init("opal_fifo_t");
@@ -137,123 +143,127 @@ int main (int argc, char *argv[]) {
     OBJ_CONSTRUCT(&fifo, opal_fifo_t);
 
     item = OBJ_NEW(opal_list_item_t);
-    prev = opal_fifo_push_st (&fifo, item);
+    prev = opal_fifo_push_st(&fifo, item);
     if (&fifo.opal_fifo_ghost == prev) {
-        test_success ();
+        test_success();
     } else {
-        test_failure (" opal_fifo_push_st on empty fifo");
+        test_failure(" opal_fifo_push_st on empty fifo");
     }
 
-    item2 = opal_fifo_pop_st (&fifo);
+    item2 = opal_fifo_pop_st(&fifo);
     if (item == item2) {
-        test_success ();
+        test_success();
     } else {
-        test_failure (" opal_fifo_pop_st");
+        test_failure(" opal_fifo_pop_st");
     }
 
     OBJ_RELEASE(item);
 
-    for (int i = 0 ; i < ITEM_COUNT ; ++i) {
+    for (int i = 0; i < ITEM_COUNT; ++i) {
         item = OBJ_NEW(opal_list_item_t);
         item->item_free = 0;
-        opal_fifo_push_st (&fifo, item);
+        opal_fifo_push_st(&fifo, item);
     }
 
-    if (check_fifo_consistency (&fifo, ITEM_COUNT)) {
-        test_success ();
+    if (check_fifo_consistency(&fifo, ITEM_COUNT)) {
+        test_success();
     } else {
-        test_failure (" opal_fifo_push_st(multiple items)");
+        test_failure(" opal_fifo_push_st(multiple items)");
     }
 
-    gettimeofday (&start, NULL);
-    for (int i = 0 ; i < ITERATIONS ; ++i) {
-        item = opal_fifo_pop_st (&fifo);
-        (void) opal_fifo_push_st (&fifo, item);
+    gettimeofday(&start, NULL);
+    for (int i = 0; i < ITERATIONS; ++i) {
+        item = opal_fifo_pop_st(&fifo);
+        (void) opal_fifo_push_st(&fifo, item);
     }
-    gettimeofday (&stop, NULL);
+    gettimeofday(&stop, NULL);
 
     timersub(&stop, &start, &total);
 
     timing = ((double) total.tv_sec + (double) total.tv_usec * 1e-6) / (double) ITERATIONS;
 
-    if (check_fifo_consistency (&fifo, ITEM_COUNT)) {
-        test_success ();
+    if (check_fifo_consistency(&fifo, ITEM_COUNT)) {
+        test_success();
     } else {
-        test_failure (" fifo push/pop");
+        test_failure(" fifo push/pop");
     }
 
-    printf ("Single thread test. Time: %d s %d us %d nsec/poppush\n", (int) total.tv_sec,
-            (int)total.tv_usec, (int)(timing / 1e-9));
+    printf("Single thread test. Time: %d s %d us %d nsec/poppush\n", (int) total.tv_sec,
+           (int) total.tv_usec, (int) (timing / 1e-9));
 
     threads[0].t_arg = &fifo;
-    thread_test ((opal_object_t *) &threads[0]);
+    thread_test((opal_object_t *) &threads[0]);
 
-    if (check_fifo_consistency (&fifo, ITEM_COUNT)) {
-        test_success ();
+    if (check_fifo_consistency(&fifo, ITEM_COUNT)) {
+        test_success();
     } else {
-        test_failure (" fifo push/pop single-threaded with atomics");
+        test_failure(" fifo push/pop single-threaded with atomics");
     }
 
-    gettimeofday (&start, NULL);
-    for (int i = 0 ; i < OPAL_FIFO_TEST_THREAD_COUNT ; ++i) {
+    gettimeofday(&start, NULL);
+    for (int i = 0; i < OPAL_FIFO_TEST_THREAD_COUNT; ++i) {
         OBJ_CONSTRUCT(&threads[i], opal_thread_t);
         threads[i].t_run = thread_test;
         threads[i].t_arg = &fifo;
-        opal_thread_start (threads + i);
+        opal_thread_start(threads + i);
     }
 
-    for (int i = 0 ; i < OPAL_FIFO_TEST_THREAD_COUNT ; ++i) {
+    for (int i = 0; i < OPAL_FIFO_TEST_THREAD_COUNT; ++i) {
         void *ret;
 
-        opal_thread_join (threads + i, &ret);
+        opal_thread_join(threads + i, &ret);
     }
-    gettimeofday (&stop, NULL);
+    gettimeofday(&stop, NULL);
 
     timersub(&stop, &start, &total);
 
-    timing = ((double) total.tv_sec + (double) total.tv_usec * 1e-6) / (double) (ITERATIONS * OPAL_FIFO_TEST_THREAD_COUNT);
+    timing = ((double) total.tv_sec + (double) total.tv_usec * 1e-6)
+             / (double) (ITERATIONS * OPAL_FIFO_TEST_THREAD_COUNT);
 
-    if (check_fifo_consistency (&fifo, ITEM_COUNT)) {
-        test_success ();
+    if (check_fifo_consistency(&fifo, ITEM_COUNT)) {
+        test_success();
     } else {
-        test_failure (" fifo push/pop multi-threaded with atomics");
+        test_failure(" fifo push/pop multi-threaded with atomics");
     }
 
-    printf ("All threads finished. Thread count: %d Time: %d s %d us %d nsec/poppush\n",
-            OPAL_FIFO_TEST_THREAD_COUNT, (int) total.tv_sec, (int)total.tv_usec, (int)(timing / 1e-9));
-
+    printf("All threads finished. Thread count: %d Time: %d s %d us %d nsec/poppush\n",
+           OPAL_FIFO_TEST_THREAD_COUNT, (int) total.tv_sec, (int) total.tv_usec,
+           (int) (timing / 1e-9));
 
-    gettimeofday (&start, NULL);
-    for (int i = 0 ; i < OPAL_FIFO_TEST_THREAD_COUNT ; ++i) {
+    gettimeofday(&start, NULL);
+    for (int i = 0; i < OPAL_FIFO_TEST_THREAD_COUNT; ++i) {
         OBJ_CONSTRUCT(&threads[i], opal_thread_t);
         threads[i].t_run = thread_test_exhaust;
         threads[i].t_arg = &fifo;
-        opal_thread_start (threads + i);
+        opal_thread_start(threads + i);
     }
 
-    for (int i = 0 ; i < OPAL_FIFO_TEST_THREAD_COUNT ; ++i) {
+    for (int i = 0; i < OPAL_FIFO_TEST_THREAD_COUNT; ++i) {
         void *ret;
 
-        opal_thread_join (threads + i, &ret);
+        opal_thread_join(threads + i, &ret);
     }
-    gettimeofday (&stop, NULL);
+    gettimeofday(&stop, NULL);
 
     timersub(&stop, &start, &total);
 
-    timing = ((double) total.tv_sec + (double) total.tv_usec * 1e-6) / (double) (ITERATIONS * OPAL_FIFO_TEST_THREAD_COUNT);
+    timing = ((double) total.tv_sec + (double) total.tv_usec * 1e-6)
+             / (double) (ITERATIONS * OPAL_FIFO_TEST_THREAD_COUNT);
 
-    if (check_fifo_consistency (&fifo, ITEM_COUNT)) {
-        test_success ();
+    if (check_fifo_consistency(&fifo, ITEM_COUNT)) {
+        test_success();
     } else {
-        test_failure (" fifo push/pop multi-threaded with atomics when there are insufficient items");
+        test_failure(
+            " fifo push/pop multi-threaded with atomics when there are insufficient items");
     }
 
-    printf ("All threads finished. Thread count: %d Time: %d s %d us %d nsec/poppush\n",
-            OPAL_FIFO_TEST_THREAD_COUNT, (int) total.tv_sec, (int)total.tv_usec, (int)(timing / 1e-9));
+    printf("All threads finished. Thread count: %d Time: %d s %d us %d nsec/poppush\n",
+           OPAL_FIFO_TEST_THREAD_COUNT, (int) total.tv_sec, (int) total.tv_usec,
+           (int) (timing / 1e-9));
 
     success = true;
-    for (int i = 0 ; i < ITEM_COUNT ; ++i) {
-        item = opal_fifo_pop_st (&fifo);
+    for (int i = 0; i < ITEM_COUNT; ++i) {
+        item = opal_fifo_pop_st(&fifo);
         if (NULL == item) {
             success = false;
             break;
@@ -262,14 +272,14 @@ int main (int argc, char *argv[]) {
     }
 
     if (success) {
-        test_success ();
+        test_success();
     } else {
-        test_failure (" fifo pop all items");
+        test_failure(" fifo pop all items");
     }
 
     OBJ_DESTRUCT(&fifo);
 
-    opal_finalize_util ();
+    opal_finalize_util();
 
-    return test_finalize ();
+    return test_finalize();
 }
diff --git a/test/class/opal_hash_table.c b/test/class/opal_hash_table.c
index e4f0e215f70..4ae405accc7 100644
--- a/test/class/opal_hash_table.c
+++ b/test/class/opal_hash_table.c
@@ -22,47 +22,31 @@
  */
 
 #include "opal_config.h"
-#include <stdint.h>
-#include <string.h>
-#include "support.h"
-#include "opal/class/opal_object.h"
 #include "opal/class/opal_hash_table.h"
-#include "opal/runtime/opal.h"
+#include "opal/class/opal_object.h"
 #include "opal/constants.h"
+#include "opal/runtime/opal.h"
+#include "support.h"
+#include <stdint.h>
+#include <string.h>
 
-static FILE *error_out=NULL;
-
-char *num_keys[] = {
-    "1234", "1234",
-    "5678", "5678",
-    "12450", "12450",
-    "45623", "45623",
-    NULL
-};
-
+static FILE *error_out = NULL;
 
-char *str_keys[] = {
-    "foo", "bar",
-    "2", "this cow jumped over the moon",
-    "this is another key", "this is another value",
-    "key key", "value value",
-    NULL
-};
+char *num_keys[] = {"1234", "1234", "5678", "5678", "12450", "12450", "45623", "45623", NULL};
 
+char *str_keys[] = {"foo",
+                    "bar",
+                    "2",
+                    "this cow jumped over the moon",
+                    "this is another key",
+                    "this is another value",
+                    "key key",
+                    "value value",
+                    NULL};
 
-char *perm_keys[] = {
-    "abcdef", "abcdef",
-    "bcdefa", "bcdefa",
-    "cdefab", "cdefab",
-    "defabc", "defabc",
-    "efabcd", "efabcd",
-    "fabcde", "fabcde",
-    "badcfe", "badcfe",
-    "badcef", "badcef",
-    "abdcfe", "abdcfe",
-    "bcdaef", "bcdaef",
-    NULL
-};
+char *perm_keys[] = {"abcdef", "abcdef", "bcdefa", "bcdefa", "cdefab", "cdefab", "defabc",
+                     "defabc", "efabcd", "efabcd", "fabcde", "fabcde", "badcfe", "badcfe",
+                     "badcef", "badcef", "abdcfe", "abdcfe", "bcdaef", "bcdaef", NULL};
 
 /*
  * This data specifically knows about the April'2014 version of hash tables.
@@ -72,13 +56,15 @@ char *perm_keys[] = {
  * Then... it removes a key and checks again (via traversal)
  * and removes another key and re-checks.
  */
-static char* remove_keys[] = {
-    "1", "A", "2", "B", "4", "D", "6", "F", "10", "J", NULL, /* insert as-is: ...AB.D.F...J... */
-    "2", "b", "4", "d", "5", "e", "3", "c", NULL, /* insert with capacity-offset: ...ABbDdFec.J... */
-    "ABbDdFecJ",		/* traversal expectation */
-    "4", "ABbdeFcJ",		/* remove D (...ABbdeFc..J...) then expected traversal */
-    "2", "AbcdeFJ",		/* remove B (...AbcdeF...J...) then expected traversal */
-    NULL			/* end removals and expectations */
+static char *remove_keys[] = {
+    "1",         "A",        "2", "B", "4", "D", "6", "F", "10", "J",
+    NULL, /* insert as-is: ...AB.D.F...J... */
+    "2",         "b",        "4", "d", "5", "e", "3", "c", NULL, /* insert with capacity-offset:
+                                                                    ...ABbDdFec.J... */
+    "ABbDdFecJ",                                                 /* traversal expectation */
+    "4",         "ABbdeFcJ", /* remove D (...ABbdeFc..J...) then expected traversal */
+    "2",         "AbcdeFJ",  /* remove B (...AbcdeF...J...) then expected traversal */
+    NULL                     /* end removals and expectations */
 };
 
 typedef union {
@@ -88,87 +74,83 @@ typedef union {
 
 static void validate_table(opal_hash_table_t *table, char *keys[], int is_numeric_keys)
 {
-    int         j, ret;
+    int j, ret;
     value_t value;
 
-    for ( j = 0; keys[j]; j += 2) {
-        if ( 1 == is_numeric_keys ) {
-            ret = opal_hash_table_get_value_uint32(table, atoi(keys[j]),
-                                                   (void**) &value.uvalue);
+    for (j = 0; keys[j]; j += 2) {
+        if (1 == is_numeric_keys) {
+            ret = opal_hash_table_get_value_uint32(table, atoi(keys[j]), (void **) &value.uvalue);
             if (OPAL_SUCCESS != ret) {
                 test_failure("opal_hash_table_get_value_uint32 failed");
             }
         } else {
-            ret = opal_hash_table_get_value_ptr(table, keys[j],
-                                                strlen(keys[j]),
-                                                &value.vvalue);
+            ret = opal_hash_table_get_value_ptr(table, keys[j], strlen(keys[j]), &value.vvalue);
             if (OPAL_SUCCESS != ret) {
                 test_failure("opal_hash_table_get_value_ptr failed");
             }
         }
-        test_verify_str(keys[j+1], value.vvalue);
+        test_verify_str(keys[j + 1], value.vvalue);
     }
-    test_verify_int(j/2, opal_hash_table_get_size(table));
+    test_verify_int(j / 2, opal_hash_table_get_size(table));
 }
 
-static void
-validate_remove_traversal(opal_hash_table_t * table, const char * expected_chars)
+static void validate_remove_traversal(opal_hash_table_t *table, const char *expected_chars)
 {
     /* all values are single-character strings */
     /* expected_chars are those single characters as a string */
-    const int debug = 0;	/* turn this on if you want to see the details */
+    const int debug = 0; /* turn this on if you want to see the details */
     int rc, problems = 0;
-    const char * expected_scanner = expected_chars;
+    const char *expected_scanner = expected_chars;
     uint32_t key;
-    void * raw_value;
-    void * node;
+    void *raw_value;
+    void *node;
     if (debug) {
-	fprintf(stderr, "debug: expecting '%s' capacity is %d\n",
-		expected_chars, (int) table->ht_capacity);
+        fprintf(stderr, "debug: expecting '%s' capacity is %d\n", expected_chars,
+                (int) table->ht_capacity);
     }
     for (rc = opal_hash_table_get_first_key_uint32(table, &key, &raw_value, &node);
-	 OPAL_SUCCESS == rc;
-	 rc = opal_hash_table_get_next_key_uint32(table, &key, &raw_value, node, &node)) {
-	const char * value = (const char *) raw_value;
-	char expected, actual;
-	if (debug) {
-	    fprintf(stderr, "key %d (probe at %d) value '%s' excpected_scanner '%s'\n",
-		    key, (int) (key%table->ht_capacity), value, expected_scanner);
-	}
-	if (1 != strlen(value)) {
-	    fprintf(stderr, "key %d's value '%s' is not a one-character string\n", key, value);
-	    problems += 1;
-	    continue;		/* might as well be completely noisy */
-	}
-	if ('\0' == *expected_scanner) {
-	    fprintf(stderr, "Found key %d value '%s' but not expected!\n", key, value);
-	    problems += 1;
-	    continue;
-	}
-	expected = *expected_scanner++;
-	actual = *value;
-	if (actual != expected) {
-	    fprintf(stderr, "Expected '%c' but got '%c'\n", expected, actual);
-	    problems += 1;
-	    continue;
-	}
+         OPAL_SUCCESS == rc;
+         rc = opal_hash_table_get_next_key_uint32(table, &key, &raw_value, node, &node)) {
+        const char *value = (const char *) raw_value;
+        char expected, actual;
+        if (debug) {
+            fprintf(stderr, "key %d (probe at %d) value '%s' excpected_scanner '%s'\n", key,
+                    (int) (key % table->ht_capacity), value, expected_scanner);
+        }
+        if (1 != strlen(value)) {
+            fprintf(stderr, "key %d's value '%s' is not a one-character string\n", key, value);
+            problems += 1;
+            continue; /* might as well be completely noisy */
+        }
+        if ('\0' == *expected_scanner) {
+            fprintf(stderr, "Found key %d value '%s' but not expected!\n", key, value);
+            problems += 1;
+            continue;
+        }
+        expected = *expected_scanner++;
+        actual = *value;
+        if (actual != expected) {
+            fprintf(stderr, "Expected '%c' but got '%c'\n", expected, actual);
+            problems += 1;
+            continue;
+        }
     }
     /* final checks */
     if (OPAL_ERROR != rc) {
-	fprintf(stderr, "table traversal did not end in OPAL_ERROR?!?\n");
-	problems += 1;
+        fprintf(stderr, "table traversal did not end in OPAL_ERROR?!?\n");
+        problems += 1;
     }
     if ('\0' != *expected_scanner) {
-	fprintf(stderr, "Still expecting more key/values: '%s'\n", expected_scanner);
-	problems += 1;
+        fprintf(stderr, "Still expecting more key/values: '%s'\n", expected_scanner);
+        problems += 1;
     }
 
     /* resolution */
     if (problems > 0) {
-	fflush(stderr);
-	test_failure("validate_remove_traversal");
+        fflush(stderr);
+        test_failure("validate_remove_traversal");
     } else {
-	test_success();
+        test_success();
     }
 }
 
@@ -176,9 +158,8 @@ static void test_htable(opal_hash_table_t *table)
 {
     int j;
     fprintf(error_out, "\nTesting integer keys...\n");
-    for ( j = 0; num_keys[j]; j += 2)
-    {
-        opal_hash_table_set_value_uint32(table, atoi(num_keys[j]), num_keys[j+1]);
+    for (j = 0; num_keys[j]; j += 2) {
+        opal_hash_table_set_value_uint32(table, atoi(num_keys[j]), num_keys[j + 1]);
     }
     validate_table(table, num_keys, 1);
 
@@ -187,9 +168,8 @@ static void test_htable(opal_hash_table_t *table)
     test_verify_int(0, opal_hash_table_get_size(table));
 
     fprintf(error_out, "\nTesting string keys...\n");
-    for ( j = 0; str_keys[j]; j += 2)
-    {
-        opal_hash_table_set_value_ptr(table, str_keys[j], strlen(str_keys[j]), str_keys[j+1]);
+    for (j = 0; str_keys[j]; j += 2) {
+        opal_hash_table_set_value_ptr(table, str_keys[j], strlen(str_keys[j]), str_keys[j + 1]);
     }
     validate_table(table, str_keys, 0);
 
@@ -200,9 +180,8 @@ static void test_htable(opal_hash_table_t *table)
     fprintf(error_out, "\nTesting collision resolution...\n");
     /* All of the string keys in keys array should
         have the same hash value. */
-    for ( j = 0; perm_keys[j]; j += 2)
-    {
-        opal_hash_table_set_value_ptr(table, perm_keys[j], strlen(perm_keys[j]), perm_keys[j+1]);
+    for (j = 0; perm_keys[j]; j += 2) {
+        opal_hash_table_set_value_ptr(table, perm_keys[j], strlen(perm_keys[j]), perm_keys[j + 1]);
     }
 
     validate_table(table, perm_keys, 0);
@@ -213,18 +192,18 @@ static void test_htable(opal_hash_table_t *table)
 
     fprintf(error_out, "\nTesting removal and traversal...\n");
     j = 0;
-    char * str;
+    char *str;
     while (NULL != (str = remove_keys[j++])) {
-	opal_hash_table_set_value_uint32(table, atoi(str), remove_keys[j++]);
+        opal_hash_table_set_value_uint32(table, atoi(str), remove_keys[j++]);
     }
     while (NULL != (str = remove_keys[j++])) {
-	/* generate collisions */
-	opal_hash_table_set_value_uint32(table, atoi(str) + table->ht_capacity, remove_keys[j++]);
+        /* generate collisions */
+        opal_hash_table_set_value_uint32(table, atoi(str) + table->ht_capacity, remove_keys[j++]);
     }
     validate_remove_traversal(table, remove_keys[j++]);
     while (NULL != (str = remove_keys[j++])) {
-	opal_hash_table_remove_value_uint32(table, atoi(str));
-	validate_remove_traversal(table, remove_keys[j++]);
+        opal_hash_table_remove_value_uint32(table, atoi(str));
+        validate_remove_traversal(table, remove_keys[j++]);
     }
 
     /* remove all values for next test */
@@ -234,14 +213,12 @@ static void test_htable(opal_hash_table_t *table)
     fprintf(error_out, "\n\n");
 }
 
-
 static void test_dynamic(void)
 {
-    opal_hash_table_t     *table;
+    opal_hash_table_t *table;
 
     table = OBJ_NEW(opal_hash_table_t);
-    if ( NULL == table )
-    {
+    if (NULL == table) {
         fprintf(error_out, "Error: Unable to create hash table.\n");
         exit(-1);
     }
@@ -252,10 +229,9 @@ static void test_dynamic(void)
     OBJ_RELEASE(table);
 }
 
-
 static void test_static(void)
 {
-    opal_hash_table_t     table;
+    opal_hash_table_t table;
 
     OBJ_CONSTRUCT(&table, opal_hash_table_t);
     opal_hash_table_init(&table, 128);
@@ -266,7 +242,6 @@ static void test_static(void)
     OBJ_DESTRUCT(&table);
 }
 
-
 int main(int argc, char **argv)
 {
     int rc;
@@ -283,17 +258,18 @@ int main(int argc, char **argv)
 #ifdef STANDALONE
     error_out = stderr;
 #else
-    error_out = fopen( "./opal_hash_table_test_out.txt", "w" );
-    if( error_out == NULL ) error_out = stderr;
+    error_out = fopen("./opal_hash_table_test_out.txt", "w");
+    if (error_out == NULL)
+        error_out = stderr;
 #endif
 
     test_dynamic();
     test_static();
 #ifndef STANDALONE
-    fclose( error_out );
+    fclose(error_out);
 #endif
 
-    opal_finalize_util ();
+    opal_finalize_util();
 
     return test_finalize();
 }
diff --git a/test/class/opal_lifo.c b/test/class/opal_lifo.c
index 3f4d9b7d92d..ab67d6e4aeb 100644
--- a/test/class/opal_lifo.c
+++ b/test/class/opal_lifo.c
@@ -12,71 +12,74 @@
 #include "opal_config.h"
 #include <assert.h>
 
-#include "support.h"
 #include "opal/class/opal_lifo.h"
-#include "opal/runtime/opal.h"
 #include "opal/constants.h"
 #include "opal/mca/threads/threads.h"
+#include "opal/runtime/opal.h"
+#include "support.h"
 
-#include <stdlib.h>
-#include <stdio.h>
 #include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
 #include <sys/time.h>
 
 #define OPAL_LIFO_TEST_THREAD_COUNT 8
-#define ITERATIONS 1000000
-#define ITEM_COUNT 100
+#define ITERATIONS                  1000000
+#define ITEM_COUNT                  100
 
 #if !defined(timersub)
-#define timersub(a, b, r) \
-    do {                  \
-        (r)->tv_sec = (a)->tv_sec - (b)->tv_sec;        \
-        if ((a)->tv_usec < (b)->tv_usec) {              \
-            (r)->tv_sec--;                              \
-            (a)->tv_usec += 1000000;                    \
-        }                                               \
-        (r)->tv_usec = (a)->tv_usec - (b)->tv_usec;     \
-    } while (0)
+#    define timersub(a, b, r)                           \
+        do {                                            \
+            (r)->tv_sec = (a)->tv_sec - (b)->tv_sec;    \
+            if ((a)->tv_usec < (b)->tv_usec) {          \
+                (r)->tv_sec--;                          \
+                (a)->tv_usec += 1000000;                \
+            }                                           \
+            (r)->tv_usec = (a)->tv_usec - (b)->tv_usec; \
+        } while (0)
 #endif
 
-static void *thread_test (opal_object_t *arg) {
+static void *thread_test(opal_object_t *arg)
+{
     opal_thread_t *t = (opal_thread_t *) arg;
     opal_lifo_t *lifo = (opal_lifo_t *) t->t_arg;
     opal_list_item_t *item;
     struct timeval start, stop, total;
     double timing;
 
-    gettimeofday (&start, NULL);
-    for (int i = 0 ; i < ITERATIONS ; ++i) {
-        item = opal_lifo_pop_atomic (lifo);
+    gettimeofday(&start, NULL);
+    for (int i = 0; i < ITERATIONS; ++i) {
+        item = opal_lifo_pop_atomic(lifo);
         if (NULL != item) {
-            (void) opal_lifo_push_atomic (lifo, item);
+            (void) opal_lifo_push_atomic(lifo, item);
         }
     }
-    gettimeofday (&stop, NULL);
+    gettimeofday(&stop, NULL);
 
     timersub(&stop, &start, &total);
 
     timing = ((double) total.tv_sec + (double) total.tv_usec * 1e-6) / (double) ITERATIONS;
 
-    printf ("Atomics thread finished. Time: %d s %d us %d nsec/poppush\n", (int) total.tv_sec,
-            (int)total.tv_usec, (int)(timing / 1e-9));
+    printf("Atomics thread finished. Time: %d s %d us %d nsec/poppush\n", (int) total.tv_sec,
+           (int) total.tv_usec, (int) (timing / 1e-9));
 
     return NULL;
 }
 
-static bool check_lifo_consistency (opal_lifo_t *lifo, int expected_count)
+static bool check_lifo_consistency(opal_lifo_t *lifo, int expected_count)
 {
     opal_list_item_t *item;
     int count;
 
-    for (count = 0, item = (opal_list_item_t *) lifo->opal_lifo_head.data.item ; item != &lifo->opal_lifo_ghost ;
-         item = opal_list_get_next(item), count++);
+    for (count = 0, item = (opal_list_item_t *) lifo->opal_lifo_head.data.item;
+         item != &lifo->opal_lifo_ghost; item = opal_list_get_next(item), count++)
+        ;
 
     return count == expected_count;
 }
 
-int main (int argc, char *argv[]) {
+int main(int argc, char *argv[])
+{
     opal_thread_t threads[OPAL_LIFO_TEST_THREAD_COUNT];
     opal_list_item_t *item, *prev, *item2;
     struct timeval start, stop, total;
@@ -85,11 +88,11 @@ int main (int argc, char *argv[]) {
     double timing;
     int rc;
 
-    rc = opal_init_util (&argc, &argv);
+    rc = opal_init_util(&argc, &argv);
     test_verify_int(OPAL_SUCCESS, rc);
     if (OPAL_SUCCESS != rc) {
         test_finalize();
-        exit (1);
+        exit(1);
     }
 
     test_init("opal_lifo_t");
@@ -97,94 +100,96 @@ int main (int argc, char *argv[]) {
     OBJ_CONSTRUCT(&lifo, opal_lifo_t);
 
     item = OBJ_NEW(opal_list_item_t);
-    prev = opal_lifo_push_st (&lifo, item);
+    prev = opal_lifo_push_st(&lifo, item);
     if (&lifo.opal_lifo_ghost == prev) {
-        test_success ();
+        test_success();
     } else {
-        test_failure (" opal_lifo_push_st on empty lifo");
+        test_failure(" opal_lifo_push_st on empty lifo");
     }
 
-    item2 = opal_lifo_pop_st (&lifo);
+    item2 = opal_lifo_pop_st(&lifo);
     if (item == item2) {
-        test_success ();
+        test_success();
     } else {
-        test_failure (" opal_lifo_pop_st");
+        test_failure(" opal_lifo_pop_st");
     }
 
     OBJ_RELEASE(item);
 
-    for (int i = 0 ; i < ITEM_COUNT ; ++i) {
+    for (int i = 0; i < ITEM_COUNT; ++i) {
         item = OBJ_NEW(opal_list_item_t);
         item->item_free = 0;
-        opal_lifo_push_st (&lifo, item);
+        opal_lifo_push_st(&lifo, item);
     }
 
-    if (check_lifo_consistency (&lifo, ITEM_COUNT)) {
-        test_success ();
+    if (check_lifo_consistency(&lifo, ITEM_COUNT)) {
+        test_success();
     } else {
-        test_failure (" opal_lifo_push_st(multiple items)");
+        test_failure(" opal_lifo_push_st(multiple items)");
     }
 
-    gettimeofday (&start, NULL);
-    for (int i = 0 ; i < ITERATIONS ; ++i) {
-        item = opal_lifo_pop_st (&lifo);
-        (void) opal_lifo_push_st (&lifo, item);
+    gettimeofday(&start, NULL);
+    for (int i = 0; i < ITERATIONS; ++i) {
+        item = opal_lifo_pop_st(&lifo);
+        (void) opal_lifo_push_st(&lifo, item);
     }
-    gettimeofday (&stop, NULL);
+    gettimeofday(&stop, NULL);
 
     timersub(&stop, &start, &total);
 
     timing = ((double) total.tv_sec + (double) total.tv_usec * 1e-6) / (double) ITERATIONS;
 
-    if (check_lifo_consistency (&lifo, ITEM_COUNT)) {
-        test_success ();
+    if (check_lifo_consistency(&lifo, ITEM_COUNT)) {
+        test_success();
     } else {
-        test_failure (" lifo push/pop");
+        test_failure(" lifo push/pop");
     }
 
-    printf ("Single thread test. Time: %d s %d us %d nsec/poppush\n", (int) total.tv_sec,
-            (int)total.tv_usec, (int)(timing / 1e-9));
+    printf("Single thread test. Time: %d s %d us %d nsec/poppush\n", (int) total.tv_sec,
+           (int) total.tv_usec, (int) (timing / 1e-9));
 
     threads[0].t_arg = &lifo;
-    thread_test ((opal_object_t *) &threads[0]);
+    thread_test((opal_object_t *) &threads[0]);
 
-    if (check_lifo_consistency (&lifo, ITEM_COUNT)) {
-        test_success ();
+    if (check_lifo_consistency(&lifo, ITEM_COUNT)) {
+        test_success();
     } else {
-        test_failure (" lifo push/pop single-threaded with atomics");
+        test_failure(" lifo push/pop single-threaded with atomics");
     }
 
-    gettimeofday (&start, NULL);
-    for (int i = 0 ; i < OPAL_LIFO_TEST_THREAD_COUNT ; ++i) {
+    gettimeofday(&start, NULL);
+    for (int i = 0; i < OPAL_LIFO_TEST_THREAD_COUNT; ++i) {
         OBJ_CONSTRUCT(&threads[i], opal_thread_t);
         threads[i].t_run = thread_test;
         threads[i].t_arg = &lifo;
-        opal_thread_start (threads + i);
+        opal_thread_start(threads + i);
     }
 
-    for (int i = 0 ; i < OPAL_LIFO_TEST_THREAD_COUNT ; ++i) {
+    for (int i = 0; i < OPAL_LIFO_TEST_THREAD_COUNT; ++i) {
         void *ret;
 
-        opal_thread_join (threads + i, &ret);
+        opal_thread_join(threads + i, &ret);
     }
-    gettimeofday (&stop, NULL);
+    gettimeofday(&stop, NULL);
 
     timersub(&stop, &start, &total);
 
-    timing = ((double) total.tv_sec + (double) total.tv_usec * 1e-6) / (double) (ITERATIONS * OPAL_LIFO_TEST_THREAD_COUNT);
+    timing = ((double) total.tv_sec + (double) total.tv_usec * 1e-6)
+             / (double) (ITERATIONS * OPAL_LIFO_TEST_THREAD_COUNT);
 
-    if (check_lifo_consistency (&lifo, ITEM_COUNT)) {
-        test_success ();
+    if (check_lifo_consistency(&lifo, ITEM_COUNT)) {
+        test_success();
     } else {
-        test_failure (" lifo push/pop multi-threaded with atomics");
+        test_failure(" lifo push/pop multi-threaded with atomics");
     }
 
-    printf ("All threads finished. Thread count: %d Time: %d s %d us %d nsec/poppush\n",
-            OPAL_LIFO_TEST_THREAD_COUNT, (int) total.tv_sec, (int)total.tv_usec, (int)(timing / 1e-9));
+    printf("All threads finished. Thread count: %d Time: %d s %d us %d nsec/poppush\n",
+           OPAL_LIFO_TEST_THREAD_COUNT, (int) total.tv_sec, (int) total.tv_usec,
+           (int) (timing / 1e-9));
 
     success = true;
-    for (int i = 0 ; i < ITEM_COUNT ; ++i) {
-        item = opal_lifo_pop_st (&lifo);
+    for (int i = 0; i < ITEM_COUNT; ++i) {
+        item = opal_lifo_pop_st(&lifo);
         if (NULL == item) {
             success = false;
             break;
@@ -193,14 +198,14 @@ int main (int argc, char *argv[]) {
     }
 
     if (success) {
-        test_success ();
+        test_success();
     } else {
-        test_failure (" list pop all items");
+        test_failure(" list pop all items");
     }
 
     OBJ_DESTRUCT(&lifo);
 
-    opal_finalize_util ();
+    opal_finalize_util();
 
-    return test_finalize ();
+    return test_finalize();
 }
diff --git a/test/class/opal_list.c b/test/class/opal_list.c
index e8e171c0797..714ab2714de 100644
--- a/test/class/opal_list.c
+++ b/test/class/opal_list.c
@@ -20,10 +20,10 @@
 #include "opal_config.h"
 #include <assert.h>
 
-#include "support.h"
 #include "opal/class/opal_list.h"
-#include "opal/runtime/opal.h"
 #include "opal/constants.h"
+#include "opal/runtime/opal.h"
+#include "support.h"
 
 /*
  * Data type used for testing
@@ -35,15 +35,13 @@ typedef struct test_data {
     size_t data;
 } test_data_t;
 
-OBJ_CLASS_INSTANCE(test_data_t,
-                   opal_list_item_t,
-                   NULL, NULL);
+OBJ_CLASS_INSTANCE(test_data_t, opal_list_item_t, NULL, NULL);
 
 int main(int argc, char **argv)
 {
     /* local variables */
     opal_list_t list, x;
-    size_t indx,i,list_size, tmp_size_1, tmp_size_2,size_elements;
+    size_t indx, i, list_size, tmp_size_1, tmp_size_2, size_elements;
     int error_cnt, rc;
     test_data_t *elements, *ele;
     opal_list_item_t *item;
@@ -62,8 +60,8 @@ int main(int argc, char **argv)
     OBJ_CONSTRUCT(&x, opal_list_t);
 
     /* check length of list */
-    list_size=opal_list_get_size(&list);
-    if( 0 == list_size ) {
+    list_size = opal_list_get_size(&list);
+    if (0 == list_size) {
         test_success();
     } else {
         test_failure(" opal_list_get_size");
@@ -77,20 +75,20 @@ int main(int argc, char **argv)
     }
 
     /* create test elements */
-    size_elements=4;
-    elements=(test_data_t *)malloc(sizeof(test_data_t)*size_elements);
+    size_elements = 4;
+    elements = (test_data_t *) malloc(sizeof(test_data_t) * size_elements);
     assert(elements);
-    for(i=0 ; i < size_elements ; i++) {
+    for (i = 0; i < size_elements; i++) {
         OBJ_CONSTRUCT(elements + i, test_data_t);
-        (elements+i)->data=i;
+        (elements + i)->data = i;
     }
 
     /* populate list */
-    for(i=0 ; i < size_elements ; i++) {
-        opal_list_append(&list,(opal_list_item_t *)(elements+i));
+    for (i = 0; i < size_elements; i++) {
+        opal_list_append(&list, (opal_list_item_t *) (elements + i));
     }
-    list_size=opal_list_get_size(&list);
-    if( list_size == size_elements ) {
+    list_size = opal_list_get_size(&list);
+    if (list_size == size_elements) {
         test_success();
     } else {
         test_failure(" populating list");
@@ -104,222 +102,186 @@ int main(int argc, char **argv)
     }
 
     /* check that list is ordered as expected */
-    i=0;
-    error_cnt=0;
-    for(ele = (test_data_t *) opal_list_get_first(&list);
-            ele != (test_data_t *) opal_list_get_end(&list);
-            ele = (test_data_t *) ((opal_list_item_t *)ele)->opal_list_next) {
-        if( ele->data != i )
+    i = 0;
+    error_cnt = 0;
+    for (ele = (test_data_t *) opal_list_get_first(&list);
+         ele != (test_data_t *) opal_list_get_end(&list);
+         ele = (test_data_t *) ((opal_list_item_t *) ele)->opal_list_next) {
+        if (ele->data != i)
             error_cnt++;
         i++;
     }
-    if( 0 == error_cnt ) {
+    if (0 == error_cnt) {
         test_success();
     } else {
         test_failure(" error in list order ");
     }
 
     /* check opal_list_get_first */
-    ele = (test_data_t *)NULL;
+    ele = (test_data_t *) NULL;
     ele = (test_data_t *) opal_list_get_first(&list);
     assert(ele);
-    if( 0 == ele->data ) {
+    if (0 == ele->data) {
         test_success();
     } else {
         test_failure(" error in opal_list_get_first");
     }
-    i=0;
-    for(ele = (test_data_t *) opal_list_get_first(&list);
-            ele != (test_data_t *) opal_list_get_end(&list);
-            ele = (test_data_t *) ((opal_list_item_t *)ele)->opal_list_next) {
+    i = 0;
+    for (ele = (test_data_t *) opal_list_get_first(&list);
+         ele != (test_data_t *) opal_list_get_end(&list);
+         ele = (test_data_t *) ((opal_list_item_t *) ele)->opal_list_next) {
         i++;
     }
-    if( size_elements == i ) {
+    if (size_elements == i) {
         test_success();
     } else {
         test_failure(" error in opal_list_get_first - list size changed ");
     }
 
     /* check opal_list_get_last */
-    ele = (test_data_t *)NULL;
+    ele = (test_data_t *) NULL;
     ele = (test_data_t *) opal_list_get_last(&list);
     assert(ele);
-    if( (size_elements-1) == ele->data ) {
+    if ((size_elements - 1) == ele->data) {
         test_success();
     } else {
         test_failure(" error in opal_list_get_last");
     }
-    i=0;
-    for(ele = (test_data_t *) opal_list_get_first(&list);
-            ele != (test_data_t *) opal_list_get_end(&list);
-            ele = (test_data_t *) ((opal_list_item_t *)ele)->opal_list_next) {
+    i = 0;
+    for (ele = (test_data_t *) opal_list_get_first(&list);
+         ele != (test_data_t *) opal_list_get_end(&list);
+         ele = (test_data_t *) ((opal_list_item_t *) ele)->opal_list_next) {
         i++;
     }
-    if( size_elements == i ) {
+    if (size_elements == i) {
         test_success();
     } else {
         test_failure(" error in opal_list_get_first - list size changed ");
     }
 
     /* check opal_list_remove_first */
-    ele = (test_data_t *)NULL;
+    ele = (test_data_t *) NULL;
     ele = (test_data_t *) opal_list_remove_first(&list);
     assert(ele);
-    if( 0 == ele->data ) {
+    if (0 == ele->data) {
         test_success();
     } else {
         test_failure(" error in opal_list_remove_first");
     }
-    i=0;
-    for(ele = (test_data_t *) opal_list_get_first(&list);
-            ele != (test_data_t *) opal_list_get_end(&list);
-            ele = (test_data_t *) ((opal_list_item_t *)ele)->opal_list_next) {
+    i = 0;
+    for (ele = (test_data_t *) opal_list_get_first(&list);
+         ele != (test_data_t *) opal_list_get_end(&list);
+         ele = (test_data_t *) ((opal_list_item_t *) ele)->opal_list_next) {
         i++;
     }
-    if( (size_elements-1) == i ) {
+    if ((size_elements - 1) == i) {
         test_success();
     } else {
         test_failure(" error in opal_list_remove_first - list size changed ");
     }
 
     /* test opal_list_prepend */
-    opal_list_prepend(&list,(opal_list_item_t *)elements);
-    ele = (test_data_t *)NULL;
+    opal_list_prepend(&list, (opal_list_item_t *) elements);
+    ele = (test_data_t *) NULL;
     ele = (test_data_t *) opal_list_get_first(&list);
     assert(ele);
-    if( 0 == ele->data ) {
+    if (0 == ele->data) {
         test_success();
     } else {
         test_failure(" error in opal_list_prepend");
     }
-    i=0;
-    for(ele = (test_data_t *) opal_list_get_first(&list);
-            ele != (test_data_t *) opal_list_get_end(&list);
-            ele = (test_data_t *) ((opal_list_item_t *)ele)->opal_list_next) {
+    i = 0;
+    for (ele = (test_data_t *) opal_list_get_first(&list);
+         ele != (test_data_t *) opal_list_get_end(&list);
+         ele = (test_data_t *) ((opal_list_item_t *) ele)->opal_list_next) {
         i++;
     }
-    if( size_elements == i ) {
+    if (size_elements == i) {
         test_success();
     } else {
         test_failure(" error in opal_list_prepend - list size changed ");
     }
 
     /* check opal_list_remove_last */
-    ele = (test_data_t *)NULL;
+    ele = (test_data_t *) NULL;
     ele = (test_data_t *) opal_list_remove_last(&list);
     assert(ele);
-    if( (size_elements-1) == ele->data ) {
+    if ((size_elements - 1) == ele->data) {
         test_success();
     } else {
         test_failure(" error in opal_list_remove_last");
     }
-    i=0;
-    for(ele = (test_data_t *) opal_list_get_first(&list);
-            ele != (test_data_t *) opal_list_get_end(&list);
-            ele = (test_data_t *) ((opal_list_item_t *)ele)->opal_list_next) {
+    i = 0;
+    for (ele = (test_data_t *) opal_list_get_first(&list);
+         ele != (test_data_t *) opal_list_get_end(&list);
+         ele = (test_data_t *) ((opal_list_item_t *) ele)->opal_list_next) {
         i++;
     }
-    if( (size_elements-1) == i ) {
+    if ((size_elements - 1) == i) {
         test_success();
     } else {
         test_failure(" error in opal_list_remove_last - list size changed ");
     }
 
     /* test opal_list_append */
-    opal_list_append(&list,(opal_list_item_t *)(elements+size_elements-1));
-    ele = (test_data_t *)NULL;
+    opal_list_append(&list, (opal_list_item_t *) (elements + size_elements - 1));
+    ele = (test_data_t *) NULL;
     ele = (test_data_t *) opal_list_get_last(&list);
     assert(ele);
-    if( (size_elements-1) == ele->data ) {
+    if ((size_elements - 1) == ele->data) {
         test_success();
     } else {
         test_failure(" error in opal_list_append");
     }
-    i=0;
-    for(ele = (test_data_t *) opal_list_get_first(&list);
-            ele != (test_data_t *) opal_list_get_end(&list);
-            ele = (test_data_t *) ((opal_list_item_t *)ele)->opal_list_next) {
+    i = 0;
+    for (ele = (test_data_t *) opal_list_get_first(&list);
+         ele != (test_data_t *) opal_list_get_end(&list);
+         ele = (test_data_t *) ((opal_list_item_t *) ele)->opal_list_next) {
         i++;
     }
-    if( size_elements == i ) {
+    if (size_elements == i) {
         test_success();
     } else {
         test_failure(" error in opal_list_append - list size changed ");
     }
 
     /* remove element from list */
-    indx=size_elements/2;
-    if( 0 == indx )
-        indx=1;
+    indx = size_elements / 2;
+    if (0 == indx)
+        indx = 1;
     assert(2 <= size_elements);
-    ele = (test_data_t *)NULL;
-    ele = (test_data_t *)
-        opal_list_remove_item(&list,(opal_list_item_t *)(elements+indx));
+    ele = (test_data_t *) NULL;
+    ele = (test_data_t *) opal_list_remove_item(&list, (opal_list_item_t *) (elements + indx));
     assert(ele);
-    if( (indx-1) == ele->data ) {
+    if ((indx - 1) == ele->data) {
         test_success();
     } else {
         test_failure(" error in opal_list_remove - previous");
     }
-    ele=(test_data_t *)(((opal_list_item_t *)ele)->opal_list_next);
-    if( (indx+1) == ele->data ) {
+    ele = (test_data_t *) (((opal_list_item_t *) ele)->opal_list_next);
+    if ((indx + 1) == ele->data) {
         test_success();
     } else {
         test_failure(" error in opal_list_remove - next");
     }
-    i=0;
-    for(ele = (test_data_t *) opal_list_get_first(&list);
-            ele != (test_data_t *) opal_list_get_end(&list);
-            ele = (test_data_t *) ((opal_list_item_t *)ele)->opal_list_next) {
+    i = 0;
+    for (ele = (test_data_t *) opal_list_get_first(&list);
+         ele != (test_data_t *) opal_list_get_end(&list);
+         ele = (test_data_t *) ((opal_list_item_t *) ele)->opal_list_next) {
         i++;
     }
-    if( (size_elements-1) == i ) {
+    if ((size_elements - 1) == i) {
         test_success();
     } else {
         test_failure(" error in opal_list_remove - list size changed incorrectly");
     }
 
-    /* test the insert function */
-    i=opal_list_insert(&list,(opal_list_item_t *)(elements+indx),indx);
-    if( 1 == i ) {
-        test_success();
-    } else {
-        test_failure(" error in opal_list_remove_item \n");
-    }
-
-    i=0;
-    for(ele = (test_data_t *) opal_list_get_first(&list);
-            ele != (test_data_t *) opal_list_get_end(&list);
-            ele = (test_data_t *) ((opal_list_item_t *)ele)->opal_list_next) {
-        i++;
-    }
-    if( size_elements == i ) {
-        test_success();
-    } else {
-        test_failure(" error in opal_list_insert - incorrect list length");
-    }
-    i=0;
-    error_cnt=0;
-    for(ele = (test_data_t *) opal_list_get_first(&list);
-            ele != (test_data_t *) opal_list_get_end(&list);
-            ele = (test_data_t *) ((opal_list_item_t *)ele)->opal_list_next) {
-        if( ele->data != i )
-            error_cnt++;
-        i++;
-    }
-    if( 0 == error_cnt ) {
-        test_success();
-    } else {
-        test_failure(" error in list order - opal_list_remove_item ");
-    }
-
     /* test the splice and join functions  */
     list_size = opal_list_get_size(&list);
-    for (i = 0, item = opal_list_get_first(&list) ;
-         i < list_size / 2 ; ++i, item = opal_list_get_next(item)) {
+    for (i = 0, item = opal_list_get_first(&list); i < list_size / 2;
+         ++i, item = opal_list_get_next(item)) {
     }
-    opal_list_splice(&x, opal_list_get_end(&x),
-                     &list, item, opal_list_get_end(&list));
+    opal_list_splice(&x, opal_list_get_end(&x), &list, item, opal_list_get_end(&list));
     tmp_size_1 = opal_list_get_size(&list);
     tmp_size_2 = opal_list_get_size(&x);
     if (tmp_size_1 != i) {
@@ -341,9 +303,10 @@ int main(int argc, char **argv)
         test_success();
     }
 
-    if (NULL != elements) free(elements);
+    if (NULL != elements)
+        free(elements);
 
-    opal_finalize_util ();
+    opal_finalize_util();
 
     return test_finalize();
 }
diff --git a/test/class/opal_pointer_array.c b/test/class/opal_pointer_array.c
index 5c7bec18607..b0ce90b2a53 100644
--- a/test/class/opal_pointer_array.c
+++ b/test/class/opal_pointer_array.c
@@ -24,25 +24,26 @@
  */
 
 #include "opal_config.h"
-#include <stdlib.h>
-#include <stdio.h>
 #include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
 
-#include "support.h"
 #include "opal/class/opal_pointer_array.h"
+#include "support.h"
 
 typedef union {
     int ivalue;
     char *cvalue;
 } value_t;
 
-static void test(bool thread_usage){
+static void test(bool thread_usage)
+{
 
     /* local variables */
     opal_pointer_array_t *array;
     value_t *test_data;
-    int len_test_data,i,test_len_in_array,error_cnt;
+    int len_test_data, i, test_len_in_array, error_cnt;
     int ele_index;
     int error_code;
     value_t value;
@@ -50,39 +51,39 @@ static void test(bool thread_usage){
     /* initialize thread levels */
     opal_set_using_threads(thread_usage);
 
-    array=OBJ_NEW(opal_pointer_array_t);
+    array = OBJ_NEW(opal_pointer_array_t);
     assert(array);
 
-    len_test_data=5;
-    test_data=malloc(sizeof(value_t)*len_test_data);
+    len_test_data = 5;
+    test_data = malloc(sizeof(value_t) * len_test_data);
     assert(test_data);
 
-    for(i=0 ; i < len_test_data ; i++ ) {
-        test_data[i].ivalue = (i+1);
+    for (i = 0; i < len_test_data; i++) {
+        test_data[i].ivalue = (i + 1);
     }
 
     /* add data to table */
-    test_len_in_array=3;
-    assert(len_test_data>=test_len_in_array);
-    for(i=0 ; i < test_len_in_array ; i++ ) {
-        opal_pointer_array_add(array,test_data[i].cvalue);
+    test_len_in_array = 3;
+    assert(len_test_data >= test_len_in_array);
+    for (i = 0; i < test_len_in_array; i++) {
+        opal_pointer_array_add(array, test_data[i].cvalue);
     }
     /* check to see that test_len_in_array are in array */
-    if( (array->size - array->number_free) == test_len_in_array) {
+    if ((array->size - array->number_free) == test_len_in_array) {
         test_success();
     } else {
         test_failure("check on number of elements in array");
     }
 
     /* check order of data */
-    error_cnt=0;
-    for(i=0 ; i < test_len_in_array ; i++ ) {
+    error_cnt = 0;
+    for (i = 0; i < test_len_in_array; i++) {
         value.cvalue = array->addr[i];
-        if( (i+1) != value.ivalue ) {
+        if ((i + 1) != value.ivalue) {
             error_cnt++;
         }
     }
-    if( 0 == error_cnt ) {
+    if (0 == error_cnt) {
         test_success();
     } else {
         test_failure(" data check ");
@@ -90,19 +91,19 @@ static void test(bool thread_usage){
 
     /* free 2nd element and make sure that value is reset correctly,
      *   and that the lowest_index is also reset correctly */
-    ele_index=1;
-    error_code=opal_pointer_array_set_item(array,ele_index,NULL);
-    if( 0 == error_code ) {
+    ele_index = 1;
+    error_code = opal_pointer_array_set_item(array, ele_index, NULL);
+    if (0 == error_code) {
         test_success();
     } else {
         test_failure(" opal_pointer_array_set_item ");
     }
-    if( NULL == array->addr[ele_index]){
+    if (NULL == array->addr[ele_index]) {
         test_success();
     } else {
         test_failure(" set pointer value");
     }
-    if( ele_index == array->lowest_free ) {
+    if (ele_index == array->lowest_free) {
         test_success();
     } else {
         test_failure(" lowest free ");
@@ -110,28 +111,28 @@ static void test(bool thread_usage){
 
     /* test opal_pointer_array_get_item */
     opal_pointer_array_remove_all(array);
-    error_cnt=0;
-    for(i=0 ; i < array->size ; i++ ) {
+    error_cnt = 0;
+    for (i = 0; i < array->size; i++) {
         value.ivalue = i + 2;
-        ele_index=opal_pointer_array_add(array, value.cvalue);
-        if( i != ele_index ) {
+        ele_index = opal_pointer_array_add(array, value.cvalue);
+        if (i != ele_index) {
             error_cnt++;
         }
     }
-    if( 0 == error_cnt ) {
+    if (0 == error_cnt) {
         test_success();
     } else {
         test_failure(" opal_pointer_array_add 2nd ");
     }
 
-    error_cnt=0;
-    for(i=0 ; i < array->size ; i++ ) {
-        value.cvalue = opal_pointer_array_get_item(array,i);
-        if( (i+2) != value.ivalue ) {
+    error_cnt = 0;
+    for (i = 0; i < array->size; i++) {
+        value.cvalue = opal_pointer_array_get_item(array, i);
+        if ((i + 2) != value.ivalue) {
             error_cnt++;
         }
     }
-    if( 0 == error_cnt ) {
+    if (0 == error_cnt) {
         test_success();
     } else {
         test_failure(" data check - 2nd ");
@@ -140,23 +141,23 @@ static void test(bool thread_usage){
     OBJ_RELEASE(array);
     assert(NULL == array);
 
-    array=OBJ_NEW(opal_pointer_array_t);
+    array = OBJ_NEW(opal_pointer_array_t);
     assert(array);
     opal_pointer_array_init(array, 0, 4, 2);
-    for( i = 0; i < 4; i++ ) {
+    for (i = 0; i < 4; i++) {
         value.ivalue = i + 1;
-        if( 0 > opal_pointer_array_add( array, value.cvalue ) ) {
+        if (0 > opal_pointer_array_add(array, value.cvalue)) {
             test_failure("Add/Remove: failure during initial data_add ");
         }
     }
-    for( i = i-1; i >= 0; i-- ) {
-        if( i % 2 )
-            if( 0 != opal_pointer_array_set_item(array, i, NULL) )
+    for (i = i - 1; i >= 0; i--) {
+        if (i % 2)
+            if (0 != opal_pointer_array_set_item(array, i, NULL))
                 test_failure("Add/Remove: failure during item removal ");
     }
-    for( i = 0; i < 4; i++ ) {
-        if( !opal_pointer_array_add( array, (void*)(uintptr_t)(i+1) ) ) {
-            if( i != 2 ) {
+    for (i = 0; i < 4; i++) {
+        if (!opal_pointer_array_add(array, (void *) (uintptr_t)(i + 1))) {
+            if (i != 2) {
                 test_failure("Add/Remove: failure during the readd ");
                 break;
             }
@@ -169,7 +170,6 @@ static void test(bool thread_usage){
     free(test_data);
 }
 
-
 int main(int argc, char **argv)
 {
     test_init("opal_pointer_array");
diff --git a/test/class/opal_proc_table.c b/test/class/opal_proc_table.c
index 1720f988487..c6ab08b75d8 100644
--- a/test/class/opal_proc_table.c
+++ b/test/class/opal_proc_table.c
@@ -24,47 +24,32 @@
  */
 
 #include "opal_config.h"
-#include <stdint.h>
-#include <string.h>
-#include "support.h"
-#include "opal/class/opal_object.h"
 #include "opal/class/opal_hash_table.h"
-#include "opal/runtime/opal.h"
+#include "opal/class/opal_object.h"
 #include "opal/constants.h"
+#include "opal/runtime/opal.h"
+#include "support.h"
+#include <stdint.h>
+#include <string.h>
 
-static FILE *error_out=NULL;
-
-char *num_keys[] = {
-    "0", "1234", "1234",
-    "0", "5678", "5678",
-    "1", "12450", "12450",
-    "1", "45623", "45623",
-    NULL
-};
-
+static FILE *error_out = NULL;
 
-char *str_keys[] = {
-    "foo", "bar",
-    "2", "this cow jumped over the moon",
-    "this is another key", "this is another value",
-    "key key", "value value",
-    NULL
-};
+char *num_keys[] = {"0",     "1234",  "1234", "0",     "5678",  "5678", "1",
+                    "12450", "12450", "1",    "45623", "45623", NULL};
 
+char *str_keys[] = {"foo",
+                    "bar",
+                    "2",
+                    "this cow jumped over the moon",
+                    "this is another key",
+                    "this is another value",
+                    "key key",
+                    "value value",
+                    NULL};
 
-char *perm_keys[] = {
-    "abcdef", "abcdef",
-    "bcdefa", "bcdefa",
-    "cdefab", "cdefab",
-    "defabc", "defabc",
-    "efabcd", "efabcd",
-    "fabcde", "fabcde",
-    "badcfe", "badcfe",
-    "badcef", "badcef",
-    "abdcfe", "abdcfe",
-    "bcdaef", "bcdaef",
-    NULL
-};
+char *perm_keys[] = {"abcdef", "abcdef", "bcdefa", "bcdefa", "cdefab", "cdefab", "defabc",
+                     "defabc", "efabcd", "efabcd", "fabcde", "fabcde", "badcfe", "badcfe",
+                     "badcef", "badcef", "abdcfe", "abdcfe", "bcdaef", "bcdaef", NULL};
 
 #if 0
 /*
@@ -92,19 +77,18 @@ typedef union {
 
 static void validate_table(opal_proc_table_t *table, char *keys[])
 {
-    int         j, ret;
+    int j, ret;
     value_t value;
 
-    for ( j = 0; keys[j]; j += 3) {
+    for (j = 0; keys[j]; j += 3) {
         opal_process_name_t key;
         key.jobid = atoi(keys[j]);
-        key.vpid = atoi(keys[j+1]);
-        ret = opal_proc_table_get_value(table, key,
-                                        (void**) &value.uvalue);
+        key.vpid = atoi(keys[j + 1]);
+        ret = opal_proc_table_get_value(table, key, (void **) &value.uvalue);
         if (OPAL_SUCCESS != ret) {
             test_failure("opal_proc_table_get_value failed");
         }
-        test_verify_str(keys[j+2], value.vvalue);
+        test_verify_str(keys[j + 2], value.vvalue);
     }
 }
 
@@ -175,44 +159,41 @@ static void test_ptable(opal_proc_table_t *table)
 {
     int j;
     char *n1, *n2;
-    char * v;
+    char *v;
     int rc;
     opal_process_name_t key;
     fprintf(error_out, "\nTesting integer keys...\n");
-    for ( j = 0; num_keys[j]; j += 3)
-    {
+    for (j = 0; num_keys[j]; j += 3) {
         opal_process_name_t key;
         key.jobid = atoi(num_keys[j]);
-        key.vpid = atoi(num_keys[j+1]);
-        opal_proc_table_set_value(table, key, num_keys[j+2]);
+        key.vpid = atoi(num_keys[j + 1]);
+        opal_proc_table_set_value(table, key, num_keys[j + 2]);
     }
     validate_table(table, num_keys);
-    rc = opal_proc_table_get_first_key(table, &key, (void **)&v, (void **)&n1, (void **)&n2);
+    rc = opal_proc_table_get_first_key(table, &key, (void **) &v, (void **) &n1, (void **) &n2);
     if (OPAL_SUCCESS != rc) {
         fprintf(error_out, "*** FAILED opal_proc_table_get_first_key ***\n");
-        return ;
+        return;
     }
-    for (j=0; num_keys[j]; j+=3) {
+    for (j = 0; num_keys[j]; j += 3) {
         if (OPAL_SUCCESS != rc) {
-            fprintf(error_out, "*** FAILED opal_proc_table_get_next_key (%d) ***\n", j/3);
+            fprintf(error_out, "*** FAILED opal_proc_table_get_next_key (%d) ***\n", j / 3);
         }
 
-        if (key.jobid != (opal_jobid_t)atoi(num_keys[j]) ||
-            key.vpid != (opal_vpid_t)atoi(num_keys[j+1]) ||
-            0 != strcmp(num_keys[j+2], v)) {
-                fprintf(error_out, "*** FAILED at %d, expected ((%s,%s),%s) got ((%d,%d),%s)\n", j/3,
-                        num_keys[j], num_keys[j+1], num_keys[j+2],
-                        key.jobid, key.vpid, v);
+        if (key.jobid != (opal_jobid_t) atoi(num_keys[j])
+            || key.vpid != (opal_vpid_t) atoi(num_keys[j + 1]) || 0 != strcmp(num_keys[j + 2], v)) {
+            fprintf(error_out, "*** FAILED at %d, expected ((%s,%s),%s) got ((%d,%d),%s)\n", j / 3,
+                    num_keys[j], num_keys[j + 1], num_keys[j + 2], key.jobid, key.vpid, v);
         }
 
-        rc = opal_proc_table_get_next_key(table, &key, (void **)&v, n1, (void **)&n1, n2, (void **)&n2);
+        rc = opal_proc_table_get_next_key(table, &key, (void **) &v, n1, (void **) &n1, n2,
+                                          (void **) &n2);
     }
     if (OPAL_SUCCESS == rc) {
-        fprintf(error_out, "*** DID NOT FAIL last opal_proc_table_get_next_key %d\n", j/3);
+        fprintf(error_out, "*** DID NOT FAIL last opal_proc_table_get_next_key %d\n", j / 3);
         return;
     }
 
-
 #if 0
     /* remove all values for next test */
     opal_proc_table_remove_all(table);
@@ -242,14 +223,12 @@ static void test_ptable(opal_proc_table_t *table)
     fprintf(error_out, "\n\n");
 }
 
-
 static void test_dynamic(void)
 {
-    opal_proc_table_t     *table;
+    opal_proc_table_t *table;
 
     table = OBJ_NEW(opal_proc_table_t);
-    if ( NULL == table )
-    {
+    if (NULL == table) {
         fprintf(error_out, "Error: Unable to create hash table.\n");
         exit(-1);
     }
@@ -260,10 +239,9 @@ static void test_dynamic(void)
     OBJ_RELEASE(table);
 }
 
-
 static void test_static(void)
 {
-    opal_proc_table_t     table;
+    opal_proc_table_t table;
 
     OBJ_CONSTRUCT(&table, opal_proc_table_t);
     opal_proc_table_init(&table, 8, 128);
@@ -274,7 +252,6 @@ static void test_static(void)
     OBJ_DESTRUCT(&table);
 }
 
-
 int main(int argc, char **argv)
 {
     int rc;
@@ -291,17 +268,18 @@ int main(int argc, char **argv)
 #ifdef STANDALONE
     error_out = stderr;
 #else
-    error_out = fopen( "./opal_proc_table_test_out.txt", "w" );
-    if( error_out == NULL ) error_out = stderr;
+    error_out = fopen("./opal_proc_table_test_out.txt", "w");
+    if (error_out == NULL)
+        error_out = stderr;
 #endif
 
     test_dynamic();
     test_static();
 #ifndef STANDALONE
-    fclose( error_out );
+    fclose(error_out);
 #endif
 
-    opal_finalize_util ();
+    opal_finalize_util();
 
     return test_finalize();
 }
diff --git a/test/class/opal_value_array.c b/test/class/opal_value_array.c
index be09ce4ca77..2145e2fabc4 100644
--- a/test/class/opal_value_array.c
+++ b/test/class/opal_value_array.c
@@ -22,18 +22,17 @@
  */
 
 #include "opal_config.h"
-#include <stdlib.h>
-#include <stdio.h>
 #include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
 
-#include "support.h"
 #include "opal/class/opal_value_array.h"
 #include "opal/runtime/opal.h"
+#include "support.h"
 
 #define NUM_ITEMS 10
 
-
 int main(int argc, char **argv)
 {
     uint64_t i, val;
@@ -55,13 +54,13 @@ int main(int argc, char **argv)
     test_verify_int(0, opal_value_array_get_size(&array));
 
     /* add several items to the array */
-    for(i=0; i < NUM_ITEMS; i++) {
+    for (i = 0; i < NUM_ITEMS; i++) {
         opal_value_array_append_item(&array, &i);
     }
     test_verify_int(NUM_ITEMS, opal_value_array_get_size(&array));
 
     /* verify contents */
-    for(i=0; i < NUM_ITEMS; i++) {
+    for (i = 0; i < NUM_ITEMS; i++) {
         val = OPAL_VALUE_ARRAY_GET_ITEM(&array, uint64_t, i);
         if (val != i) {
             test_failure("Comparison failure");
@@ -80,12 +79,12 @@ int main(int argc, char **argv)
 
     /* initialize array */
     count = 0;
-    for(i=0; i < NUM_ITEMS; i++) {
+    for (i = 0; i < NUM_ITEMS; i++) {
         OPAL_VALUE_ARRAY_SET_ITEM(&array, uint64_t, i, count++);
     }
 
     /* grow it */
-    for(i=0; i < NUM_ITEMS; i++) {
+    for (i = 0; i < NUM_ITEMS; i++) {
         opal_value_array_append_item(&array, &count);
         count++;
     }
@@ -93,7 +92,7 @@ int main(int argc, char **argv)
     test_verify_int(count, opal_value_array_get_size(&array));
 
     /* validate contents */
-    for(i=0; i < count; i++) {
+    for (i = 0; i < count; i++) {
         test_verify_int(i, OPAL_VALUE_ARRAY_GET_ITEM(&array, uint64_t, i));
     }
 
@@ -101,12 +100,12 @@ int main(int argc, char **argv)
     opal_value_array_remove_item(&array, NUM_ITEMS);
 
     /* check size */
-    test_verify_int(count-1, opal_value_array_get_size(&array));
+    test_verify_int(count - 1, opal_value_array_get_size(&array));
 
     /* validate contents */
-    for(i=0; i < count-1; i++) {
-        if(i >= NUM_ITEMS) {
-            test_verify_int(i+1, OPAL_VALUE_ARRAY_GET_ITEM(&array, uint64_t, i));
+    for (i = 0; i < count - 1; i++) {
+        if (i >= NUM_ITEMS) {
+            test_verify_int(i + 1, OPAL_VALUE_ARRAY_GET_ITEM(&array, uint64_t, i));
         } else {
             test_verify_int(i, OPAL_VALUE_ARRAY_GET_ITEM(&array, uint64_t, i));
         }
@@ -114,7 +113,7 @@ int main(int argc, char **argv)
 
     OBJ_DESTRUCT(&array);
 
-    opal_finalize_util ();
+    opal_finalize_util();
 
     return test_finalize();
 }
diff --git a/test/datatype/Makefile.am b/test/datatype/Makefile.am
index 3da8fdffb73..3d6fd3289b5 100644
--- a/test/datatype/Makefile.am
+++ b/test/datatype/Makefile.am
@@ -15,7 +15,7 @@
 #
 
 if PROJECT_OMPI
-    MPI_TESTS = checksum position position_noncontig ddt_test ddt_raw ddt_raw2 unpack_ooo ddt_pack external32 large_data
+    MPI_TESTS = checksum position position_noncontig ddt_test ddt_raw ddt_raw2 unpack_ooo ddt_pack external32 large_data partial
     MPI_CHECKS = to_self reduce_local
 endif
 TESTS = opal_datatype_test unpack_hetero $(MPI_TESTS)
@@ -26,49 +26,49 @@ unpack_ooo_SOURCES = unpack_ooo.c ddt_lib.c ddt_lib.h
 unpack_ooo_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS)
 unpack_ooo_LDADD = \
         $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 
 ddt_test_SOURCES = ddt_test.c ddt_lib.c ddt_lib.h
 ddt_test_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS)
 ddt_test_LDADD = \
         $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 
 ddt_raw_SOURCES = ddt_raw.c ddt_lib.c ddt_lib.h
 ddt_raw_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS)
 ddt_raw_LDADD = \
         $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 
 ddt_raw2_SOURCES = ddt_raw2.c ddt_lib.c ddt_lib.h
 ddt_raw2_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS)
 ddt_raw2_LDADD = \
         $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 
 ddt_pack_SOURCES = ddt_pack.c
 ddt_pack_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS)
 ddt_pack_LDADD = \
         $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 
 checksum_SOURCES = checksum.c
 checksum_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS)
 checksum_LDADD = \
         $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 
 position_SOURCES = position.c
 position_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS)
 position_LDADD = \
         $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 
 position_noncontig_SOURCES = position_noncontig.c
 position_noncontig_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS)
 position_noncontig_LDADD = \
         $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 
 to_self_SOURCES = to_self.c
 to_self_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS)
@@ -78,29 +78,35 @@ large_data_SOURCES = large_data.c
 large_data_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS)
 large_data_LDADD = \
         $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 
 opal_datatype_test_SOURCES = opal_datatype_test.c opal_ddt_lib.c opal_ddt_lib.h
 opal_datatype_test_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS)
 opal_datatype_test_LDADD = \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 
 external32_SOURCES = external32.c
 external32_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS)
 external32_LDADD = \
         $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 
 unpack_hetero_SOURCES = unpack_hetero.c
 unpack_hetero_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS)
 unpack_hetero_LDADD = \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 
 reduce_local_SOURCES = reduce_local.c
 reduce_local_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS)
 reduce_local_LDADD = \
         $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 
-distclean:
+partial_SOURCES = partial.c
+partial_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS)
+partial_LDADD = \
+        $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
+
+distclean-local:
 	rm -rf *.dSYM .deps .libs *.log *.o *.trs $(check_PROGRAMS) Makefile
diff --git a/test/datatype/check_op.sh b/test/datatype/check_op.sh
index 820e938eaf5..7110acf100c 100755
--- a/test/datatype/check_op.sh
+++ b/test/datatype/check_op.sh
@@ -5,10 +5,10 @@ set -u
 echo "ompi version with AVX512 -- Usage: arg1: count of elements, args2: 'i'|'u'|'f'|'d' : datatype: signed, unsigned, float, double. args3 size of type. args4 operation"
 mpirun="mpirun --mca pml ob1 --mca btl vader,self"
 # For SVE-architecture
-# echo "$mpirun -mca op_sve_hardware_available 0 -mca op_avx_hardware_available 0 -np 1 Reduce_local_float 1048576  i 8 max"
+# echo "$mpirun -mca op_sve_hardware_available 0 -mca op_avx_hardware_available 0 -n 1 Reduce_local_float 1048576  i 8 max"
 
 # For X86_64 architectures
-# echo "$mpirun -mca op_avx_support 0 -np 1 Reduce_local_float 1048576 i 8 max"
+# echo "$mpirun -mca op_avx_support 0 -n 1 Reduce_local_float 1048576 i 8 max"
 
 Orange="\033[0;33m"
 Blue="\033[0;34m"
@@ -27,7 +27,7 @@ for op in max min sum prod band bor bxor; do
         for size in 0 1 7 15 31 63 127 130; do
             foo=$((1024 * 1024 + $size))
             echo -e "Test $Yellow __mm512 instruction for loop $NC Total_num_bits = $foo * $type_size "
-            cmd="$mpirun -np 1 reduce_local -l $foo -u $foo -t i -s $type_size -o $op"
+            cmd="$mpirun -n 1 reduce_local -l $foo -u $foo -t i -s $type_size -o $op"
             if test $verbose -eq 1 ; then echo $cmd; fi
             eval $cmd
         done
@@ -46,7 +46,7 @@ for op in max min sum prod band bor bxor; do
         for size in 0 1 7 15 31 63 127 130; do
             foo=$((1024 * 1024 + $size))
             echo -e "Test $Yellow __mm512 instruction for loop $NC Total_num_bits = $foo * $type_size"
-            cmd="$mpirun -np 1 reduce_local -l $foo -u $foo -t u -s $type_size -o $op"
+            cmd="$mpirun -n 1 reduce_local -l $foo -u $foo -t u -s $type_size -o $op"
             if test $verbose -eq 1 ; then echo $cmd; fi
             eval $cmd
         done
@@ -61,7 +61,7 @@ for op in max min sum prod; do
     for size in 1024 127 130; do
         foo=$((1024 * 1024 + $size))
         echo -e "Test $Yellow __mm512 instruction for loop $NC Total_num_bits = $foo * 32"
-        cmd="$mpirun -np 1 reduce_local -l $foo -u $foo -t f -s 32 -o $op"
+        cmd="$mpirun -n 1 reduce_local -l $foo -u $foo -t f -s 32 -o $op"
         if test $verbose -eq 1 ; then echo $cmd; fi
         eval $cmd
     done
@@ -73,7 +73,7 @@ for op in max min sum prod; do
     for size in 1024 127 130; do
         foo=$((1024 * 1024 + $size))
         echo -e "Test $Yellow __mm512 instruction for loop $NC Total_num_bits = $foo * 64"
-        cmd="$mpirun -np 1 reduce_local -l $foo -u $foo -t d -s 64 -o $op"
+        cmd="$mpirun -n 1 reduce_local -l $foo -u $foo -t d -s 64 -o $op"
         if test $verbose -eq 1 ; then echo $cmd; fi
         eval $cmd
     done
diff --git a/test/datatype/checksum.c b/test/datatype/checksum.c
index f470c920de8..39c47c2f730 100644
--- a/test/datatype/checksum.c
+++ b/test/datatype/checksum.c
@@ -13,8 +13,8 @@
  */
 
 #include "ompi_config.h"
-#include "opal/datatype/opal_convertor.h"
 #include "ompi/datatype/ompi_datatype.h"
+#include "opal/datatype/opal_convertor.h"
 #include "opal/datatype/opal_datatype_checksum.h"
 #include "opal/runtime/opal.h"
 
@@ -29,34 +29,34 @@ typedef struct {
     int useless;
 } my_data_t;
 
-int main( int argc, char* argv[] )
+int main(int argc, char *argv[])
 {
     MPI_Datatype sparse;
     int *array, *packed;
-    my_data_t* sparse_array;
+    my_data_t *sparse_array;
     int i;
     uint32_t iov_count;
     size_t max_data;
     uint32_t pack_checksum, contiguous_checksum, sparse_checksum, manual_checksum;
     struct iovec iov[2];
-    opal_convertor_t* convertor;
+    opal_convertor_t *convertor;
 
-    opal_init_util (NULL, NULL);
+    opal_init(NULL, NULL);
     ompi_datatype_init();
-    srandom( (int)time(NULL) );
+    srandom((int) time(NULL));
     /*srandomdev();*/
 
-    ompi_datatype_create_vector( SIZE, 1, 2, MPI_INT, &sparse );
-    ompi_datatype_commit( &sparse );
+    ompi_datatype_create_vector(SIZE, 1, 2, MPI_INT, &sparse);
+    ompi_datatype_commit(&sparse);
 
-    sparse_array = (my_data_t*)malloc( sizeof(my_data_t) * SIZE );
-    array = (int*)malloc( sizeof(int) * SIZE );
-    packed = (int*)malloc( sizeof(int) * SIZE );
+    sparse_array = (my_data_t *) malloc(sizeof(my_data_t) * SIZE);
+    array = (int *) malloc(sizeof(int) * SIZE);
+    packed = (int *) malloc(sizeof(int) * SIZE);
 
     /**
      * Initialize the sparse data using the index.
      */
-    for( i = 0; i < SIZE; i++ ) {
+    for (i = 0; i < SIZE; i++) {
         sparse_array[i].useful = random();
         sparse_array[i].useless = 0;
     }
@@ -65,16 +65,16 @@ int main( int argc, char* argv[] )
      * Pack the sparse data into the packed array. This simulate the first step
      * of the buffered operation.
      */
-    convertor = opal_convertor_create( opal_local_arch, 0 );
-    opal_convertor_personalize( convertor, CONVERTOR_WITH_CHECKSUM, NULL );
-    opal_convertor_prepare_for_send( convertor, &(sparse->super), SIZE, sparse_array );
+    convertor = opal_convertor_create(opal_local_arch, 0);
+    opal_convertor_personalize(convertor, CONVERTOR_WITH_CHECKSUM, NULL);
+    opal_convertor_prepare_for_send(convertor, &(sparse->super), SIZE, sparse_array);
 
     iov[0].iov_base = packed;
     iov[0].iov_len = sizeof(int) * SIZE;
     max_data = iov[0].iov_len;
 
     iov_count = 1;
-    opal_convertor_pack( convertor, iov, &iov_count, &max_data );
+    opal_convertor_pack(convertor, iov, &iov_count, &max_data);
     pack_checksum = convertor->checksum;
 
     OBJ_RELEASE(convertor);
@@ -83,16 +83,16 @@ int main( int argc, char* argv[] )
      * Now move the data from the packed array into the fragment to
      * be sent over the network (still simulation).
      */
-    convertor = opal_convertor_create( opal_local_arch, 0 );
-    opal_convertor_personalize( convertor, CONVERTOR_WITH_CHECKSUM, NULL );
-    opal_convertor_prepare_for_send( convertor, &(ompi_mpi_int.dt.super), SIZE, packed );
+    convertor = opal_convertor_create(opal_local_arch, 0);
+    opal_convertor_personalize(convertor, CONVERTOR_WITH_CHECKSUM, NULL);
+    opal_convertor_prepare_for_send(convertor, &(ompi_mpi_int.dt.super), SIZE, packed);
 
     iov[0].iov_base = array;
     iov[0].iov_len = sizeof(int) * SIZE;
     max_data = iov[0].iov_len;
 
     iov_count = 1;
-    opal_convertor_pack( convertor, iov, &iov_count, &max_data );
+    opal_convertor_pack(convertor, iov, &iov_count, &max_data);
     contiguous_checksum = convertor->checksum;
 
     OBJ_RELEASE(convertor);
@@ -102,18 +102,18 @@ int main( int argc, char* argv[] )
      * the network and now we unpack it in the user memory using 2
      * separate iovec.
      */
-    convertor = opal_convertor_create( opal_local_arch, 0 );
-    opal_convertor_personalize( convertor, CONVERTOR_WITH_CHECKSUM, NULL );
-    opal_convertor_prepare_for_recv( convertor, &(sparse->super), SIZE, sparse_array );
+    convertor = opal_convertor_create(opal_local_arch, 0);
+    opal_convertor_personalize(convertor, CONVERTOR_WITH_CHECKSUM, NULL);
+    opal_convertor_prepare_for_recv(convertor, &(sparse->super), SIZE, sparse_array);
 
     max_data = sizeof(int) * SIZE;
     iov[0].iov_base = array;
     iov[0].iov_len = max_data / 2;
-    iov[1].iov_base = (char*)array + iov[0].iov_len;
+    iov[1].iov_base = (char *) array + iov[0].iov_len;
     iov[1].iov_len = max_data - iov[0].iov_len;
 
     iov_count = 2;
-    opal_convertor_unpack( convertor, iov, &iov_count, &max_data );
+    opal_convertor_unpack(convertor, iov, &iov_count, &max_data);
     sparse_checksum = convertor->checksum;
 
     OBJ_RELEASE(convertor);
@@ -126,15 +126,14 @@ int main( int argc, char* argv[] )
     /**
      * The 3 checksum have to match.
      */
-    printf( "contiguous checksum %x\n", contiguous_checksum );
-    printf( "packed checksum     %x\n", pack_checksum );
-    printf( "sparse checksum     %x\n", sparse_checksum );
-    if( (sparse_checksum != contiguous_checksum) ||
-        (pack_checksum != sparse_checksum) ) {
-        printf( "ERROR!!! the checksum algorithm does not work as expected\n" );
+    printf("contiguous checksum %x\n", contiguous_checksum);
+    printf("packed checksum     %x\n", pack_checksum);
+    printf("sparse checksum     %x\n", sparse_checksum);
+    if ((sparse_checksum != contiguous_checksum) || (pack_checksum != sparse_checksum)) {
+        printf("ERROR!!! the checksum algorithm does not work as expected\n");
         return 1;
     }
-    printf( "COOL the 3 checksum match\n" );
+    printf("COOL the 3 checksum match\n");
 
     /**
      * Now that the packed buffer contain the data we want, let's try to call
@@ -143,17 +142,16 @@ int main( int argc, char* argv[] )
     {
         uint32_t ui1 = 0;
         size_t ui2 = 0;
-        manual_checksum = OPAL_CSUM_PARTIAL( packed, sizeof(int) * SIZE, &ui1, &ui2 );
+        manual_checksum = OPAL_CSUM_PARTIAL(packed, sizeof(int) * SIZE, &ui1, &ui2);
     }
-    printf( "manual checksum     %x\n", manual_checksum );
+    printf("manual checksum     %x\n", manual_checksum);
 
     free(sparse_array);
     free(array);
     free(packed);
 
     /* clean-ups all data allocations */
-    ompi_datatype_finalize();
-    opal_finalize_util ();
+    opal_finalize_util();
 
     return 0;
 }
diff --git a/test/datatype/ddt_lib.c b/test/datatype/ddt_lib.c
index ffdee2b57f3..0a0e9dc7e01 100644
--- a/test/datatype/ddt_lib.c
+++ b/test/datatype/ddt_lib.c
@@ -24,17 +24,18 @@
 #include "ompi_config.h"
 #include "ddt_lib.h"
 #include "ompi/constants.h"
-#include <time.h>
 #include <stdlib.h>
+#include <time.h>
 #ifdef HAVE_SYS_TIME_H
-#include <sys/time.h>
+#    include <sys/time.h>
 #endif
 #include <stdio.h>
 #include <string.h>
 
 #define TIMER_DATA_TYPE struct timeval
-#define GET_TIME(TV)   gettimeofday( &(TV), NULL )
-#define ELAPSED_TIME(TSTART, TEND)  (((TEND).tv_sec - (TSTART).tv_sec) * 1000000 + ((TEND).tv_usec - (TSTART).tv_usec))
+#define GET_TIME(TV)    gettimeofday(&(TV), NULL)
+#define ELAPSED_TIME(TSTART, TEND) \
+    (((TEND).tv_sec - (TSTART).tv_sec) * 1000000 + ((TEND).tv_usec - (TSTART).tv_usec))
 
 #define DUMP_DATA_AFTER_COMMIT 0x00000001
 #define CHECK_PACK_UNPACK      0x00000002
@@ -44,168 +45,165 @@ uint32_t outputFlags = CHECK_PACK_UNPACK | VALIDATE_DATA;
 /**
  * Cache cleanup.
  */
-#define CACHE_SIZE (4*1024*1024)
-void cache_trash( void )
+#define CACHE_SIZE (4 * 1024 * 1024)
+void cache_trash(void)
 {
-    char* buffer;
+    char *buffer;
 
-    buffer = (char*)malloc( sizeof(char) * CACHE_SIZE );
-    memset( buffer, 1, CACHE_SIZE );
-    memset( buffer, 0xff, CACHE_SIZE );
-    free( buffer );
+    buffer = (char *) malloc(sizeof(char) * CACHE_SIZE);
+    memset(buffer, 1, CACHE_SIZE);
+    memset(buffer, 0xff, CACHE_SIZE);
+    free(buffer);
 }
 
 /**
  * Data-type functions.
  */
-ompi_datatype_t* create_inversed_vector( const ompi_datatype_t* type, int length )
+ompi_datatype_t *create_inversed_vector(const ompi_datatype_t *type, int length)
 {
-   ompi_datatype_t* type1;
+    ompi_datatype_t *type1;
 
-   ompi_datatype_create_vector( length, 1, 2, type, &type1 );
+    ompi_datatype_create_vector(length, 1, 2, type, &type1);
 
-   ompi_datatype_commit( &type1 );
-   return type1;
+    ompi_datatype_commit(&type1);
+    return type1;
 }
 
-void print_double_mat( unsigned int N, double* mat )
+void print_double_mat(unsigned int N, double *mat)
 {
-   unsigned int i, j;
-   double* pMat;
-
-   for( i = 0; i < N; i++ ) {
-      printf( "(%4d) :", i * N * (int)sizeof(double) );
-      pMat = mat + i * N;
-      for( j = 0; j < N; j++ ) {
-         printf( "%5.1f ", *pMat );
-         pMat++;
-      }
-      printf( "\n" );
-   }
+    unsigned int i, j;
+    double *pMat;
+
+    for (i = 0; i < N; i++) {
+        printf("(%4d) :", i * N * (int) sizeof(double));
+        pMat = mat + i * N;
+        for (j = 0; j < N; j++) {
+            printf("%5.1f ", *pMat);
+            pMat++;
+        }
+        printf("\n");
+    }
 }
 
-int init_random_upper_matrix( unsigned int N, double* mat )
+int init_random_upper_matrix(unsigned int N, double *mat)
 {
     unsigned int i, j;
 
-    srand( time(NULL) );
-    for( i = 0; i < N; i++ ) {
+    srand(time(NULL));
+    for (i = 0; i < N; i++) {
         mat += i;
-        for( j = i; j < N; j++ ) {
-            *mat = (double)random();
+        for (j = i; j < N; j++) {
+            *mat = (double) random();
             mat++;
         }
     }
     return OMPI_SUCCESS;
 }
 
-int check_diag_matrix( unsigned int N, double* mat1, double* mat2 )
+int check_diag_matrix(unsigned int N, double *mat1, double *mat2)
 {
-   unsigned int i, j;
-
-   for( i = 0; i < N; i++ ) {
-      mat1 += i;
-      mat2 += i;
-      for( j = i; j < N; j++ ) {
-         if( *mat1 != *mat2 ) {
-            printf( "error in position (%d, %d) expect %f and find %f\n",
-                    i, j, *mat1, *mat2 );
-            printf( "hex %lx != %lx\n", *(long*)mat1, *(long*)mat2 );
-            return OMPI_ERROR;
-         }
-         mat1++; mat2++;
-      }
-   }
-   return OMPI_SUCCESS;
+    unsigned int i, j;
+
+    for (i = 0; i < N; i++) {
+        mat1 += i;
+        mat2 += i;
+        for (j = i; j < N; j++) {
+            if (*mat1 != *mat2) {
+                printf("error in position (%d, %d) expect %f and find %f\n", i, j, *mat1, *mat2);
+                printf("hex %lx != %lx\n", *(long *) mat1, *(long *) mat2);
+                return OMPI_ERROR;
+            }
+            mat1++;
+            mat2++;
+        }
+    }
+    return OMPI_SUCCESS;
 }
 
-ompi_datatype_t* upper_matrix( unsigned int mat_size )
+ompi_datatype_t *upper_matrix(unsigned int mat_size)
 {
     int *disp, *blocklen;
     unsigned int i;
-    ompi_datatype_t* upper;
+    ompi_datatype_t *upper;
 
-    disp = (int*)malloc( sizeof(int) * mat_size );
-    blocklen = (int*)malloc( sizeof(int) * mat_size );
+    disp = (int *) malloc(sizeof(int) * mat_size);
+    blocklen = (int *) malloc(sizeof(int) * mat_size);
 
-    for( i = 0; i < mat_size; i++ ) {
+    for (i = 0; i < mat_size; i++) {
         disp[i] = i * mat_size + i;
         blocklen[i] = mat_size - i;
     }
 
-    ompi_datatype_create_indexed( mat_size, blocklen, disp, &ompi_mpi_double.dt,
-                             &upper );
-    ompi_datatype_commit( &upper );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        ompi_datatype_dump( upper );
+    ompi_datatype_create_indexed(mat_size, blocklen, disp, &ompi_mpi_double.dt, &upper);
+    ompi_datatype_commit(&upper);
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        ompi_datatype_dump(upper);
     }
-    free( disp );
-    free( blocklen );
+    free(disp);
+    free(blocklen);
     return upper;
 }
 
-ompi_datatype_t* lower_matrix( unsigned int mat_size )
+ompi_datatype_t *lower_matrix(unsigned int mat_size)
 {
     int *disp, *blocklen;
     unsigned int i;
-    ompi_datatype_t* upper;
+    ompi_datatype_t *upper;
 
-    disp = (int*)malloc( sizeof(int) * mat_size );
-    blocklen = (int*)malloc( sizeof(int) * mat_size );
+    disp = (int *) malloc(sizeof(int) * mat_size);
+    blocklen = (int *) malloc(sizeof(int) * mat_size);
 
-    for( i = 0; i < mat_size; i++ ) {
+    for (i = 0; i < mat_size; i++) {
         disp[i] = i * mat_size;
         blocklen[i] = i;
     }
 
-    ompi_datatype_create_indexed( mat_size, blocklen, disp, &ompi_mpi_double.dt,
-                             &upper );
-    free( disp );
-    free( blocklen );
+    ompi_datatype_create_indexed(mat_size, blocklen, disp, &ompi_mpi_double.dt, &upper);
+    free(disp);
+    free(blocklen);
     return upper;
 }
 
-ompi_datatype_t* test_matrix_borders( unsigned int size, unsigned int width )
+ompi_datatype_t *test_matrix_borders(unsigned int size, unsigned int width)
 {
-   ompi_datatype_t *pdt, *pdt_line;
-   int disp[2];
-   int blocklen[2];
-
-   disp[0] = 0;
-   blocklen[0] = width;
-   disp[1] = (size - width) * sizeof(double);
-   blocklen[1] = width;
-
-   ompi_datatype_create_indexed( 2, blocklen, disp, &ompi_mpi_double.dt,
-                            &pdt_line );
-   ompi_datatype_create_contiguous( size, pdt_line, &pdt );
-   OBJ_RELEASE( pdt_line ); /*assert( pdt_line == NULL );*/
-   return pdt;
+    ompi_datatype_t *pdt, *pdt_line;
+    int disp[2];
+    int blocklen[2];
+
+    disp[0] = 0;
+    blocklen[0] = width;
+    disp[1] = (size - width) * sizeof(double);
+    blocklen[1] = width;
+
+    ompi_datatype_create_indexed(2, blocklen, disp, &ompi_mpi_double.dt, &pdt_line);
+    ompi_datatype_create_contiguous(size, pdt_line, &pdt);
+    OBJ_RELEASE(pdt_line); /*assert( pdt_line == NULL );*/
+    return pdt;
 }
 
-ompi_datatype_t* test_contiguous( void )
+ompi_datatype_t *test_contiguous(void)
 {
     ompi_datatype_t *pdt, *pdt1, *pdt2;
 
-    printf( "test contiguous (alignment)\n" );
+    printf("test contiguous (alignment)\n");
     ompi_datatype_create_contiguous(0, &ompi_mpi_datatype_null.dt, &pdt1);
-    ompi_datatype_add( pdt1, &ompi_mpi_double.dt, 1, 0, -1 );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        ompi_datatype_dump( pdt1 );
+    ompi_datatype_add(pdt1, &ompi_mpi_double.dt, 1, 0, -1);
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        ompi_datatype_dump(pdt1);
     }
-    ompi_datatype_add( pdt1, &ompi_mpi_char.dt, 1, 8, -1 );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        ompi_datatype_dump( pdt1 );
+    ompi_datatype_add(pdt1, &ompi_mpi_char.dt, 1, 8, -1);
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        ompi_datatype_dump(pdt1);
     }
-    ompi_datatype_create_contiguous( 4, pdt1, &pdt2 );
-    OBJ_RELEASE( pdt1 ); /*assert( pdt1 == NULL );*/
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        ompi_datatype_dump( pdt2 );
+    ompi_datatype_create_contiguous(4, pdt1, &pdt2);
+    OBJ_RELEASE(pdt1); /*assert( pdt1 == NULL );*/
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        ompi_datatype_dump(pdt2);
     }
-    ompi_datatype_create_contiguous( 2, pdt2, &pdt );
-    OBJ_RELEASE( pdt2 ); /*assert( pdt2 == NULL );*/
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        ompi_datatype_dump( pdt );
+    ompi_datatype_create_contiguous(2, pdt2, &pdt);
+    OBJ_RELEASE(pdt2); /*assert( pdt2 == NULL );*/
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        ompi_datatype_dump(pdt);
     }
     return pdt;
 }
@@ -215,33 +213,33 @@ typedef struct __struct_char_double {
     double d;
 } char_double_t;
 
-ompi_datatype_t* test_struct_char_double( void )
+ompi_datatype_t *test_struct_char_double(void)
 {
     char_double_t data;
     int lengths[] = {1, 1};
     MPI_Aint displ[] = {0, 0};
     ompi_datatype_t *pdt;
-    ompi_datatype_t* types[] = { &ompi_mpi_char.dt, &ompi_mpi_double.dt};
+    ompi_datatype_t *types[] = {&ompi_mpi_char.dt, &ompi_mpi_double.dt};
 
-    displ[0] = (char*)&(data.c) - (char*)&(data);
-    displ[1] = (char*)&(data.d) - (char*)&(data);
+    displ[0] = (char *) &(data.c) - (char *) &(data);
+    displ[1] = (char *) &(data.d) - (char *) &(data);
 
-    ompi_datatype_create_struct( 2, lengths, displ, types, &pdt );
-    ompi_datatype_commit( &pdt );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        ompi_datatype_dump( pdt );
+    ompi_datatype_create_struct(2, lengths, displ, types, &pdt);
+    ompi_datatype_commit(&pdt);
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        ompi_datatype_dump(pdt);
     }
     return pdt;
 }
 
-ompi_datatype_t* test_create_twice_two_doubles( void )
+ompi_datatype_t *test_create_twice_two_doubles(void)
 {
-    ompi_datatype_t* pdt;
+    ompi_datatype_t *pdt;
 
-    ompi_datatype_create_vector( 2, 2, 5, &ompi_mpi_double.dt, &pdt );
-    ompi_datatype_commit( &pdt );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        ompi_datatype_dump( pdt );
+    ompi_datatype_create_vector(2, 2, 5, &ompi_mpi_double.dt, &pdt);
+    ompi_datatype_commit(&pdt);
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        ompi_datatype_dump(pdt);
     }
     return pdt;
 }
@@ -270,69 +268,69 @@ ompi_datatype_t* test_create_twice_two_doubles( void )
   --C-----D*-[ C ][INT]        MPI_INT count 2 disp 0xa24 (2596) extent 4
   --C-----D*-[ C ][INT]        MPI_INT count 1 disp 0xa80 (2688) extent 4
 */
-static int blacs_length[] = { 13, 13, 13, 13, 13, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 };
-static int blacs_indices[] = { 1144/4, 1232/4, 1320/4, 1408/4, 1496/4, 1584/4, 1676/4, 1768/4,
-                               1860/4, 1952/4, 2044/4, 2136/4, 2228/4, 2320/4, 2412/4, 2504/4,
-                               2596/4, 2688/4 };
-ompi_datatype_t* test_create_blacs_type( void )
+static int blacs_length[] = {13, 13, 13, 13, 13, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1};
+static int blacs_indices[] = {1144 / 4, 1232 / 4, 1320 / 4, 1408 / 4, 1496 / 4, 1584 / 4,
+                              1676 / 4, 1768 / 4, 1860 / 4, 1952 / 4, 2044 / 4, 2136 / 4,
+                              2228 / 4, 2320 / 4, 2412 / 4, 2504 / 4, 2596 / 4, 2688 / 4};
+ompi_datatype_t *test_create_blacs_type(void)
 {
     ompi_datatype_t *pdt;
 
-    ompi_datatype_create_indexed( 18, blacs_length, blacs_indices, &ompi_mpi_int.dt, &pdt );
-    ompi_datatype_commit( &pdt );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        ompi_datatype_dump( pdt );
+    ompi_datatype_create_indexed(18, blacs_length, blacs_indices, &ompi_mpi_int.dt, &pdt);
+    ompi_datatype_commit(&pdt);
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        ompi_datatype_dump(pdt);
     }
     return pdt;
 }
 
-ompi_datatype_t* test_create_blacs_type1( const ompi_datatype_t* base_type )
+ompi_datatype_t *test_create_blacs_type1(const ompi_datatype_t *base_type)
 {
     ompi_datatype_t *pdt;
 
-    ompi_datatype_create_vector( 7, 1, 3, base_type, &pdt );
-    ompi_datatype_commit( &pdt );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        ompi_datatype_dump( pdt );
+    ompi_datatype_create_vector(7, 1, 3, base_type, &pdt);
+    ompi_datatype_commit(&pdt);
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        ompi_datatype_dump(pdt);
     }
     return pdt;
 }
 
-ompi_datatype_t* test_create_blacs_type2( const ompi_datatype_t* base_type )
+ompi_datatype_t *test_create_blacs_type2(const ompi_datatype_t *base_type)
 {
     ompi_datatype_t *pdt;
 
-    ompi_datatype_create_vector( 7, 1, 2, base_type, &pdt );
-    ompi_datatype_commit( &pdt );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        ompi_datatype_dump( pdt );
+    ompi_datatype_create_vector(7, 1, 2, base_type, &pdt);
+    ompi_datatype_commit(&pdt);
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        ompi_datatype_dump(pdt);
     }
     return pdt;
 }
 
-ompi_datatype_t* test_struct( void )
+ompi_datatype_t *test_struct(void)
 {
-    ompi_datatype_t* types[] = { &ompi_mpi_float.dt  /* ompi_datatype_basicDatatypes[DT_FLOAT] */,
-                                 NULL,
-                                 &ompi_mpi_char.dt  /* ompi_datatype_basicDatatypes[DT_CHAR] */ };
-    int lengths[] = { 2, 1, 3 };
-    MPI_Aint disp[] = { 0, 16, 26 };
-    ompi_datatype_t* pdt, *pdt1;
-
-    printf( "test struct\n" );
+    ompi_datatype_t *types[] = {&ompi_mpi_float.dt /* ompi_datatype_basicDatatypes[DT_FLOAT] */,
+                                NULL,
+                                &ompi_mpi_char.dt /* ompi_datatype_basicDatatypes[DT_CHAR] */};
+    int lengths[] = {2, 1, 3};
+    MPI_Aint disp[] = {0, 16, 26};
+    ompi_datatype_t *pdt, *pdt1;
+
+    printf("test struct\n");
     ompi_datatype_create_contiguous(0, &ompi_mpi_datatype_null.dt, &pdt1);
-    ompi_datatype_add( pdt1, &ompi_mpi_double.dt, 1, 0, -1 );
-    ompi_datatype_add( pdt1, &ompi_mpi_char.dt, 1, 8, -1 );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        ompi_datatype_dump( pdt1 );
+    ompi_datatype_add(pdt1, &ompi_mpi_double.dt, 1, 0, -1);
+    ompi_datatype_add(pdt1, &ompi_mpi_char.dt, 1, 8, -1);
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        ompi_datatype_dump(pdt1);
     }
 
     types[1] = pdt1;
 
-    ompi_datatype_create_struct( 3, lengths, disp, types, &pdt );
-    OBJ_RELEASE( pdt1 ); /*assert( pdt1 == NULL );*/
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        ompi_datatype_dump( pdt );
+    ompi_datatype_create_struct(3, lengths, disp, types, &pdt);
+    OBJ_RELEASE(pdt1); /*assert( pdt1 == NULL );*/
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        ompi_datatype_dump(pdt);
     }
     return pdt;
 }
@@ -344,7 +342,7 @@ struct structure {
     double transfered_2;
 };
 
-ompi_datatype_t* create_struct_constant_gap_resized_ddt( ompi_datatype_t* type )
+ompi_datatype_t *create_struct_constant_gap_resized_ddt(ompi_datatype_t *type)
 {
     struct structure data[1];
     ompi_datatype_t *struct_type, *temp_type;
@@ -361,9 +359,10 @@ ompi_datatype_t* create_struct_constant_gap_resized_ddt( ompi_datatype_t* type )
     ompi_datatype_create_struct(2, blocklens, disps, types, &temp_type);
     ompi_datatype_create_resized(temp_type, 0, sizeof(data[0]), &struct_type);
     ompi_datatype_commit(&struct_type);
-    OBJ_RELEASE(temp_type); assert( temp_type == NULL );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        ompi_datatype_dump( struct_type );
+    OBJ_RELEASE(temp_type);
+    assert(temp_type == NULL);
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        ompi_datatype_dump(struct_type);
     }
 
     return struct_type;
@@ -384,74 +383,75 @@ typedef struct {
 #define SSTRANGE_CNT 10
 #define USE_RESIZED
 
-ompi_datatype_t* create_strange_dt( void )
+ompi_datatype_t *create_strange_dt(void)
 {
     sdata_intern v[2];
     MPI_Aint displ[3];
-    ompi_datatype_t* types[3] = { &ompi_mpi_int.dt };
+    ompi_datatype_t *types[3] = {&ompi_mpi_int.dt};
     sstrange t[2];
     int pBlock[3] = {1, 10, 1}, dispi[3];
     ompi_datatype_t *pdt, *pdt1, *pdt2, *pdtTemp;
 
-    dispi[0] = (int)((char*)&(v[0].i1) - (char*)&(v[0]));  /* 0 */
-    dispi[1] = (int)(((char*)(&(v[0].i2)) - (char*)&(v[0])) / sizeof(int));  /* 2 */
-    ompi_datatype_create_indexed_block( 2, 1, dispi, &ompi_mpi_int.dt, &pdtTemp );
+    dispi[0] = (int) ((char *) &(v[0].i1) - (char *) &(v[0]));                   /* 0 */
+    dispi[1] = (int) (((char *) (&(v[0].i2)) - (char *) &(v[0])) / sizeof(int)); /* 2 */
+    ompi_datatype_create_indexed_block(2, 1, dispi, &ompi_mpi_int.dt, &pdtTemp);
 #ifdef USE_RESIZED
     /* optional */
     displ[0] = 0;
-    displ[1] = (char*)&(v[1]) - (char*)&(v[0]);
-    ompi_datatype_create_resized( pdtTemp, displ[0], displ[1], &pdt1 );
-    OBJ_RELEASE( pdtTemp ); assert( pdtTemp == NULL );
+    displ[1] = (char *) &(v[1]) - (char *) &(v[0]);
+    ompi_datatype_create_resized(pdtTemp, displ[0], displ[1], &pdt1);
+    OBJ_RELEASE(pdtTemp);
+    assert(pdtTemp == NULL);
 #else
     pdt1 = pdtTemp;
-#endif  /* USE_RESIZED */
+#endif /* USE_RESIZED */
 
     types[1] = pdt1;
     types[2] = &ompi_mpi_int.dt;
     displ[0] = 0;
-    displ[1] = (long)((char*)&(t[0].v[0]) - (char*)&(t[0]));
-    displ[2] = (long)((char*)&(t[0].last) - (char*)&(t[0]));
-    ompi_datatype_create_struct( 3, pBlock, displ, types, &pdtTemp );
+    displ[1] = (long) ((char *) &(t[0].v[0]) - (char *) &(t[0]));
+    displ[2] = (long) ((char *) &(t[0].last) - (char *) &(t[0]));
+    ompi_datatype_create_struct(3, pBlock, displ, types, &pdtTemp);
 #ifdef USE_RESIZED
     /* optional */
-    displ[1] = (char*)&(t[1]) - (char*)&(t[0]);
-    ompi_datatype_create_resized( pdtTemp, displ[0], displ[1], &pdt2 );
-    OBJ_RELEASE( pdtTemp ); assert( pdtTemp == NULL );
+    displ[1] = (char *) &(t[1]) - (char *) &(t[0]);
+    ompi_datatype_create_resized(pdtTemp, displ[0], displ[1], &pdt2);
+    OBJ_RELEASE(pdtTemp);
+    assert(pdtTemp == NULL);
 #else
     pdt2 = pdtTemp;
-#endif  /* USE_RESIZED */
+#endif /* USE_RESIZED */
 
-    ompi_datatype_create_contiguous( SSTRANGE_CNT, pdt2, &pdt );
+    ompi_datatype_create_contiguous(SSTRANGE_CNT, pdt2, &pdt);
 
-    OBJ_RELEASE( pdt1 );
-    OBJ_RELEASE( pdt2 );
-    printf( "\nStrange datatype BEFORE COMMIT\n" );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        ompi_datatype_dump( pdt );
+    OBJ_RELEASE(pdt1);
+    OBJ_RELEASE(pdt2);
+    printf("\nStrange datatype BEFORE COMMIT\n");
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        ompi_datatype_dump(pdt);
     }
-    ompi_datatype_commit( &pdt );
-    printf( "\nStrange datatype AFTER COMMIT\n" );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        ompi_datatype_dump( pdt );
+    ompi_datatype_commit(&pdt);
+    printf("\nStrange datatype AFTER COMMIT\n");
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        ompi_datatype_dump(pdt);
     }
     return pdt;
 }
 
-ompi_datatype_t* create_contiguous_type( const ompi_datatype_t* data, int count )
+ompi_datatype_t *create_contiguous_type(const ompi_datatype_t *data, int count)
 {
-    ompi_datatype_t* contiguous;
+    ompi_datatype_t *contiguous;
 
-    ompi_datatype_create_contiguous( count, data, &contiguous );
-    ompi_datatype_commit( &contiguous );
+    ompi_datatype_create_contiguous(count, data, &contiguous);
+    ompi_datatype_commit(&contiguous);
     return contiguous;
 }
 
-ompi_datatype_t* create_vector_type( const ompi_datatype_t* data, int count, int length, int stride )
+ompi_datatype_t *create_vector_type(const ompi_datatype_t *data, int count, int length, int stride)
 {
-    ompi_datatype_t* vector;
+    ompi_datatype_t *vector;
 
-    ompi_datatype_create_vector( count, length, stride, data, &vector );
-    ompi_datatype_commit( &vector );
+    ompi_datatype_create_vector(count, length, stride, data, &vector);
+    ompi_datatype_commit(&vector);
     return vector;
 }
-
diff --git a/test/datatype/ddt_lib.h b/test/datatype/ddt_lib.h
index fc4bb811882..78b11855aba 100644
--- a/test/datatype/ddt_lib.h
+++ b/test/datatype/ddt_lib.h
@@ -22,16 +22,17 @@
 
 #include "ompi_config.h"
 #include "ompi/datatype/ompi_datatype.h"
-#include <time.h>
 #include <stdlib.h>
+#include <time.h>
 #ifdef HAVE_SYS_TIME_H
-#include <sys/time.h>
+#    include <sys/time.h>
 #endif
 #include <stdio.h>
 
 #define TIMER_DATA_TYPE struct timeval
-#define GET_TIME(TV)   gettimeofday( &(TV), NULL )
-#define ELAPSED_TIME(TSTART, TEND)  (((TEND).tv_sec - (TSTART).tv_sec) * 1000000 + ((TEND).tv_usec - (TSTART).tv_usec))
+#define GET_TIME(TV)    gettimeofday(&(TV), NULL)
+#define ELAPSED_TIME(TSTART, TEND) \
+    (((TEND).tv_sec - (TSTART).tv_sec) * 1000000 + ((TEND).tv_usec - (TSTART).tv_usec))
 
 #define DUMP_DATA_AFTER_COMMIT 0x00000001
 #define CHECK_PACK_UNPACK      0x00000002
@@ -41,22 +42,22 @@ extern uint32_t outputFlags;
 /**
  * Cache cleanup.
  */
-extern void cache_trash( void );
+extern void cache_trash(void);
 
 /**
  * Data-type functions.
  */
-ompi_datatype_t* create_inversed_vector( const ompi_datatype_t* type, int length );
+ompi_datatype_t *create_inversed_vector(const ompi_datatype_t *type, int length);
 
-extern void print_double_mat( unsigned int N, double* mat );
-extern int init_random_upper_matrix( unsigned int N, double* mat );
-extern int check_diag_matrix( unsigned int N, double* mat1, double* mat2 );
-extern ompi_datatype_t* upper_matrix( unsigned int mat_size );
-extern ompi_datatype_t* lower_matrix( unsigned int mat_size );
-extern ompi_datatype_t* test_matrix_borders( unsigned int size, unsigned int width );
-extern ompi_datatype_t* test_contiguous( void );
-extern ompi_datatype_t* test_struct_char_double( void );
-extern ompi_datatype_t* test_create_twice_two_doubles( void );
+extern void print_double_mat(unsigned int N, double *mat);
+extern int init_random_upper_matrix(unsigned int N, double *mat);
+extern int check_diag_matrix(unsigned int N, double *mat1, double *mat2);
+extern ompi_datatype_t *upper_matrix(unsigned int mat_size);
+extern ompi_datatype_t *lower_matrix(unsigned int mat_size);
+extern ompi_datatype_t *test_matrix_borders(unsigned int size, unsigned int width);
+extern ompi_datatype_t *test_contiguous(void);
+extern ompi_datatype_t *test_struct_char_double(void);
+extern ompi_datatype_t *test_create_twice_two_doubles(void);
 
 /*
   Datatype 0x832cf28 size 0 align 1 id 0 length 4 used 0
@@ -82,13 +83,12 @@ extern ompi_datatype_t* test_create_twice_two_doubles( void );
   --C-----D*-[ C ][INT]        MPI_INT count 2 disp 0xa24 (2596) extent 4
   --C-----D*-[ C ][INT]        MPI_INT count 1 disp 0xa80 (2688) extent 4
 */
-extern ompi_datatype_t* test_create_blacs_type( void );
-extern ompi_datatype_t* test_create_blacs_type1( const ompi_datatype_t* base_type );
-extern ompi_datatype_t* test_create_blacs_type2( const ompi_datatype_t* base_type );
-extern ompi_datatype_t* test_struct( void );
-extern ompi_datatype_t* create_strange_dt( void );
-extern ompi_datatype_t* create_contiguous_type( const ompi_datatype_t* data, int count );
-extern ompi_datatype_t* create_vector_type( const ompi_datatype_t* data, int count,
-                                            int length, int stride );
-extern ompi_datatype_t* create_struct_constant_gap_resized_ddt( ompi_datatype_t* type );
-
+extern ompi_datatype_t *test_create_blacs_type(void);
+extern ompi_datatype_t *test_create_blacs_type1(const ompi_datatype_t *base_type);
+extern ompi_datatype_t *test_create_blacs_type2(const ompi_datatype_t *base_type);
+extern ompi_datatype_t *test_struct(void);
+extern ompi_datatype_t *create_strange_dt(void);
+extern ompi_datatype_t *create_contiguous_type(const ompi_datatype_t *data, int count);
+extern ompi_datatype_t *create_vector_type(const ompi_datatype_t *data, int count, int length,
+                                           int stride);
+extern ompi_datatype_t *create_struct_constant_gap_resized_ddt(ompi_datatype_t *type);
diff --git a/test/datatype/ddt_pack.c b/test/datatype/ddt_pack.c
index 60d5fd278d1..bdb6bc462ea 100644
--- a/test/datatype/ddt_pack.c
+++ b/test/datatype/ddt_pack.c
@@ -25,8 +25,8 @@
 
 #include "ompi_config.h"
 #include "ompi/datatype/ompi_datatype.h"
-#include "opal/datatype/opal_convertor.h"
 #include "ompi/proc/proc.h"
+#include "opal/datatype/opal_convertor.h"
 #include "opal/runtime/opal.h"
 
 #include <stdlib.h>
@@ -34,40 +34,43 @@
 
 #include <poll.h>
 
-static int get_extents(ompi_datatype_t * type, ptrdiff_t *lb, ptrdiff_t *extent, ptrdiff_t *true_lb, ptrdiff_t *true_extent) {
+static int get_extents(ompi_datatype_t *type, ptrdiff_t *lb, ptrdiff_t *extent, ptrdiff_t *true_lb,
+                       ptrdiff_t *true_extent)
+{
     int ret;
 
     ret = ompi_datatype_get_extent(type, lb, extent);
-    if (MPI_SUCCESS != ret) return ret;
+    if (MPI_SUCCESS != ret)
+        return ret;
     ret = ompi_datatype_get_true_extent(type, true_lb, true_extent);
-    if (MPI_SUCCESS != ret) return ret;
+    if (MPI_SUCCESS != ret)
+        return ret;
 
     return 0;
 }
 
-int
-main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     size_t packed_ddt_len;
     const void *packed_ddt;
     void *payload, *ptr;
     struct ompi_datatype_t *unpacked_dt;
     int ret = 0;
-    int         blen[4];
-    ptrdiff_t    disp[4];
+    int blen[4];
+    ptrdiff_t disp[4];
     ompi_datatype_t *newType, *types[4], *struct_type, *vec_type, *dup_type;
-    ptrdiff_t    old_lb, old_extent, old_true_lb, old_true_extent;
-    ptrdiff_t    lb, extent, true_lb, true_extent;
+    ptrdiff_t old_lb, old_extent, old_true_lb, old_true_extent;
+    ptrdiff_t lb, extent, true_lb, true_extent;
 
     /* make ompi_proc_local () work ... */
     struct ompi_proc_t dummy_proc;
     ompi_proc_local_proc = &dummy_proc;
 
-
     int _dbg = 0;
-    while (_dbg) poll(NULL, 0, 1);
+    while (_dbg)
+        poll(NULL, 0, 1);
 
-    opal_init_util (NULL, NULL);
+    opal_init(NULL, NULL);
     ompi_datatype_init();
 
     /**
@@ -82,11 +85,11 @@ main(int argc, char* argv[])
     packed_ddt_len = ompi_datatype_pack_description_length(&ompi_mpi_int.dt);
     ptr = payload = malloc(packed_ddt_len);
     ret = ompi_datatype_get_pack_description(&ompi_mpi_int.dt, &packed_ddt);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
 
     memcpy(payload, packed_ddt, packed_ddt_len);
-    unpacked_dt = ompi_datatype_create_from_packed_description(&payload,
-                                                               ompi_proc_local());
+    unpacked_dt = ompi_datatype_create_from_packed_description(&payload, ompi_proc_local());
     free(ptr);
     if (unpacked_dt == &ompi_mpi_int32_t.dt) {
         printf("\tPASSED\n");
@@ -103,30 +106,44 @@ main(int argc, char* argv[])
      */
 
     printf("---> Simple test using a struct and few predefined datatype (4 * MPI_INT).\n");
-    blen[0] = 1;         blen[1] = 2;        blen[2] = 3;        blen[3] = 4;
-    disp[0] = 0;         disp[1] = 4;        disp[2] = 8;        disp[3] = 12;
-    types[0] = &ompi_mpi_int.dt; types[1] = &ompi_mpi_int.dt; types[2] = &ompi_mpi_int.dt; types[3] = &ompi_mpi_int.dt;
-    ret = ompi_datatype_create_struct( 4, blen, disp, types, &struct_type );
-    if (ret != 0) goto cleanup;
+    blen[0] = 1;
+    blen[1] = 2;
+    blen[2] = 3;
+    blen[3] = 4;
+    disp[0] = 0;
+    disp[1] = 4;
+    disp[2] = 8;
+    disp[3] = 12;
+    types[0] = &ompi_mpi_int.dt;
+    types[1] = &ompi_mpi_int.dt;
+    types[2] = &ompi_mpi_int.dt;
+    types[3] = &ompi_mpi_int.dt;
+    ret = ompi_datatype_create_struct(4, blen, disp, types, &struct_type);
+    if (ret != 0)
+        goto cleanup;
 
     {
         int count = 4;
-        const int* a_i[2] = {&count, blen};
-        ret = ompi_datatype_set_args( struct_type, count + 1, a_i, count, disp,
-                                      count, types, MPI_COMBINER_STRUCT);
-        if (ret != 0) goto cleanup;
+        const int *a_i[2] = {&count, blen};
+        ret = ompi_datatype_set_args(struct_type, count + 1, a_i, count, disp, count, types,
+                                     MPI_COMBINER_STRUCT);
+        if (ret != 0)
+            goto cleanup;
     }
 
     ret = ompi_datatype_commit(&struct_type);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
 
     ret = get_extents(struct_type, &old_lb, &old_extent, &old_true_lb, &old_true_extent);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
 
     packed_ddt_len = ompi_datatype_pack_description_length(struct_type);
     ptr = payload = malloc(packed_ddt_len);
     ret = ompi_datatype_get_pack_description(struct_type, &packed_ddt);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
 
     memcpy(payload, packed_ddt, packed_ddt_len);
 
@@ -138,10 +155,11 @@ main(int argc, char* argv[])
         goto cleanup;
     } else {
         ret = get_extents(unpacked_dt, &lb, &extent, &true_lb, &true_extent);
-        if (ret != 0) goto cleanup;
+        if (ret != 0)
+            goto cleanup;
 
-        if (old_lb != lb || old_extent != extent ||
-            old_true_lb != true_lb || old_true_extent != extent) {
+        if (old_lb != lb || old_extent != extent || old_true_lb != true_lb
+            || old_true_extent != extent) {
             printf("\tFAILED: datatypes don't match\n");
             ret = 1;
             goto cleanup;
@@ -149,10 +167,12 @@ main(int argc, char* argv[])
         printf("\tPASSED\n");
     }
     ret = ompi_datatype_destroy(&struct_type);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
 
     ret = ompi_datatype_destroy(&unpacked_dt);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
 
     /**
      *
@@ -163,33 +183,37 @@ main(int argc, char* argv[])
     printf("---> Less Basic test with MPI_Type_vector\n");
 
     ret = ompi_datatype_create_vector(2, 1, 1, &ompi_mpi_int.dt, &vec_type);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
 
     {
         int count = 2;
         int blocklength = 1;
         int stride = 1;
-        const int* a_i[3] = {&count, &blocklength, &stride};
-        ompi_datatype_t * type = &ompi_mpi_int.dt;
-        ret = ompi_datatype_set_args(vec_type, 3, a_i, 0, NULL, 1, &type, MPI_COMBINER_VECTOR );
-        if (ret != 0) goto cleanup;
+        const int *a_i[3] = {&count, &blocklength, &stride};
+        ompi_datatype_t *type = &ompi_mpi_int.dt;
+        ret = ompi_datatype_set_args(vec_type, 3, a_i, 0, NULL, 1, &type, MPI_COMBINER_VECTOR);
+        if (ret != 0)
+            goto cleanup;
     }
 
     ret = ompi_datatype_commit(&vec_type);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
 
     ret = get_extents(vec_type, &old_lb, &old_extent, &old_true_lb, &old_true_extent);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
 
     packed_ddt_len = ompi_datatype_pack_description_length(vec_type);
     ptr = payload = malloc(packed_ddt_len);
     ret = ompi_datatype_get_pack_description(vec_type, &packed_ddt);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
 
     memcpy(payload, packed_ddt, packed_ddt_len);
 
-    unpacked_dt = ompi_datatype_create_from_packed_description(&payload,
-                                                          ompi_proc_local());
+    unpacked_dt = ompi_datatype_create_from_packed_description(&payload, ompi_proc_local());
     free(ptr);
     if (unpacked_dt == NULL) {
         printf("\tFAILED: could not unpack datatype\n");
@@ -197,10 +221,11 @@ main(int argc, char* argv[])
         goto cleanup;
     } else {
         ret = get_extents(unpacked_dt, &lb, &extent, &true_lb, &true_extent);
-        if (ret != 0) goto cleanup;
+        if (ret != 0)
+            goto cleanup;
 
-        if (old_lb != lb || old_extent != extent ||
-            old_true_lb != true_lb || old_true_extent != extent) {
+        if (old_lb != lb || old_extent != extent || old_true_lb != true_lb
+            || old_true_extent != extent) {
             printf("\tFAILED: datatypes don't match\n");
             ret = 1;
             goto cleanup;
@@ -208,10 +233,12 @@ main(int argc, char* argv[])
         printf("\tPASSED\n");
     }
     ret = ompi_datatype_destroy(&vec_type);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
 
     ret = ompi_datatype_destroy(&unpacked_dt);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
 
     /**
      *
@@ -222,36 +249,40 @@ main(int argc, char* argv[])
     printf("---> Test with MPI_Type_create_indexed_block\n");
 
     blen[0] = 0;
-    blen[1] = 20*sizeof(double);
+    blen[1] = 20 * sizeof(double);
 
     ret = ompi_datatype_create_indexed_block(2, 10, blen, &ompi_mpi_double.dt, &newType);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
 
     {
         int count = 2;
         int blocklength = 10;
-        const int* a_i[3] = {&count, &blocklength, blen};
-        ompi_datatype_t * oldtype = &ompi_mpi_double.dt;
-        ompi_datatype_set_args( newType, 2 + count, a_i, 0, NULL, 1, &oldtype,
-                                MPI_COMBINER_INDEXED_BLOCK );
-        if (ret != 0) goto cleanup;
+        const int *a_i[3] = {&count, &blocklength, blen};
+        ompi_datatype_t *oldtype = &ompi_mpi_double.dt;
+        ompi_datatype_set_args(newType, 2 + count, a_i, 0, NULL, 1, &oldtype,
+                               MPI_COMBINER_INDEXED_BLOCK);
+        if (ret != 0)
+            goto cleanup;
     }
 
     ret = ompi_datatype_commit(&newType);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
 
     ret = get_extents(newType, &old_lb, &old_extent, &old_true_lb, &old_true_extent);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
 
     packed_ddt_len = ompi_datatype_pack_description_length(newType);
 
     ptr = payload = malloc(packed_ddt_len);
     ret = ompi_datatype_get_pack_description(newType, &packed_ddt);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
 
     memcpy(payload, packed_ddt, packed_ddt_len);
-    unpacked_dt = ompi_datatype_create_from_packed_description(&payload,
-                                                               ompi_proc_local());
+    unpacked_dt = ompi_datatype_create_from_packed_description(&payload, ompi_proc_local());
     free(ptr);
     if (unpacked_dt == NULL) {
         printf("\tFAILED: could not unpack datatype\n");
@@ -259,10 +290,11 @@ main(int argc, char* argv[])
         goto cleanup;
     } else {
         ret = get_extents(unpacked_dt, &lb, &extent, &true_lb, &true_extent);
-        if (ret != 0) goto cleanup;
+        if (ret != 0)
+            goto cleanup;
 
-        if (old_lb != lb || old_extent != extent ||
-            old_true_lb != true_lb || old_true_extent != extent) {
+        if (old_lb != lb || old_extent != extent || old_true_lb != true_lb
+            || old_true_extent != extent) {
             printf("\tFAILED: datatypes don't match\n");
             ret = 1;
             goto cleanup;
@@ -270,10 +302,12 @@ main(int argc, char* argv[])
         printf("\tPASSED\n");
     }
     ret = ompi_datatype_destroy(&newType);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
 
     ret = ompi_datatype_destroy(&unpacked_dt);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
 
     /**
      *
@@ -286,33 +320,37 @@ main(int argc, char* argv[])
     blen[0] = 10;
     blen[1] = 10;
     disp[0] = 0;
-    disp[1] = 20*sizeof(double);
+    disp[1] = 20 * sizeof(double);
 
     ret = ompi_datatype_create_hindexed(2, blen, disp, &ompi_mpi_double.dt, &newType);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
 
     {
         int count = 2;
-        const int* a_i[2] = {&count, blen};
-        ompi_datatype_t * oldtype = &ompi_mpi_double.dt;
-        ret = ompi_datatype_set_args( newType, count + 1, a_i, count, disp,
-                                      1, &oldtype, MPI_COMBINER_HINDEXED );
-        if (ret != 0) goto cleanup;
+        const int *a_i[2] = {&count, blen};
+        ompi_datatype_t *oldtype = &ompi_mpi_double.dt;
+        ret = ompi_datatype_set_args(newType, count + 1, a_i, count, disp, 1, &oldtype,
+                                     MPI_COMBINER_HINDEXED);
+        if (ret != 0)
+            goto cleanup;
     }
 
     ret = ompi_datatype_commit(&newType);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
 
     ret = get_extents(newType, &old_lb, &old_extent, &old_true_lb, &old_true_extent);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
 
     packed_ddt_len = ompi_datatype_pack_description_length(newType);
     ptr = payload = malloc(packed_ddt_len);
     ret = ompi_datatype_get_pack_description(newType, &packed_ddt);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
     memcpy(payload, packed_ddt, packed_ddt_len);
-    unpacked_dt = ompi_datatype_create_from_packed_description(&payload,
-                                                          ompi_proc_local());
+    unpacked_dt = ompi_datatype_create_from_packed_description(&payload, ompi_proc_local());
     free(ptr);
     if (unpacked_dt == NULL) {
         printf("\tFAILED: could not unpack datatype\n");
@@ -320,10 +358,11 @@ main(int argc, char* argv[])
         goto cleanup;
     } else {
         ret = get_extents(unpacked_dt, &lb, &extent, &true_lb, &true_extent);
-        if (ret != 0) goto cleanup;
+        if (ret != 0)
+            goto cleanup;
 
-        if (old_lb != lb || old_extent != extent ||
-            old_true_lb != true_lb || old_true_extent != extent) {
+        if (old_lb != lb || old_extent != extent || old_true_lb != true_lb
+            || old_true_extent != extent) {
             printf("\tFAILED: datatypes don't match\n");
             ret = 1;
             goto cleanup;
@@ -331,9 +370,10 @@ main(int argc, char* argv[])
         printf("\tPASSED\n");
     }
     ret = ompi_datatype_destroy(&newType);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
 
-    newType = unpacked_dt;  /* save it for later */
+    newType = unpacked_dt; /* save it for later */
 
     /**
      *
@@ -348,34 +388,39 @@ main(int argc, char* argv[])
     disp[1] = 64;
     types[0] = &ompi_mpi_int.dt;
     types[1] = newType;
-    ret = ompi_datatype_create_struct( 2, blen, disp, types, &struct_type );
-    if (ret != 0) goto cleanup;
+    ret = ompi_datatype_create_struct(2, blen, disp, types, &struct_type);
+    if (ret != 0)
+        goto cleanup;
 
     {
         int count = 2;
-        const int* a_i[2] = {&count, blen};
-        ret = ompi_datatype_set_args( struct_type, count + 1, a_i, count, disp,
-                                      count, types, MPI_COMBINER_STRUCT );
-        if (ret != 0) goto cleanup;
+        const int *a_i[2] = {&count, blen};
+        ret = ompi_datatype_set_args(struct_type, count + 1, a_i, count, disp, count, types,
+                                     MPI_COMBINER_STRUCT);
+        if (ret != 0)
+            goto cleanup;
     }
 
     ret = ompi_datatype_commit(&struct_type);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
 
     ret = ompi_datatype_destroy(&newType);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
 
     ret = get_extents(struct_type, &old_lb, &old_extent, &old_true_lb, &old_true_extent);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
 
     packed_ddt_len = ompi_datatype_pack_description_length(struct_type);
     ptr = payload = malloc(packed_ddt_len);
     ret = ompi_datatype_get_pack_description(struct_type, &packed_ddt);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
     memcpy(payload, packed_ddt, packed_ddt_len);
 
-    unpacked_dt = ompi_datatype_create_from_packed_description(&payload,
-                                                          ompi_proc_local());
+    unpacked_dt = ompi_datatype_create_from_packed_description(&payload, ompi_proc_local());
     free(ptr);
     if (unpacked_dt == NULL) {
         printf("\tFAILED: could not unpack datatype\n");
@@ -383,10 +428,11 @@ main(int argc, char* argv[])
         goto cleanup;
     } else {
         ret = get_extents(unpacked_dt, &lb, &extent, &true_lb, &true_extent);
-        if (ret != 0) goto cleanup;
+        if (ret != 0)
+            goto cleanup;
 
-        if (old_lb != lb || old_extent != extent ||
-            old_true_lb != true_lb || old_true_extent != extent) {
+        if (old_lb != lb || old_extent != extent || old_true_lb != true_lb
+            || old_true_extent != extent) {
             printf("\tFAILED: datatypes don't match\n");
             ret = 1;
             goto cleanup;
@@ -394,10 +440,12 @@ main(int argc, char* argv[])
         printf("\tPASSED\n");
     }
     ret = ompi_datatype_destroy(&struct_type);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
 
     ret = ompi_datatype_destroy(&unpacked_dt);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
 
     /**
      *
@@ -407,20 +455,23 @@ main(int argc, char* argv[])
     printf("---> Basic test with dup'ed MPI_INT\n");
 
     ret = get_extents(&ompi_mpi_int.dt, &old_lb, &old_extent, &old_true_lb, &old_true_extent);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
     ret = ompi_datatype_duplicate(&ompi_mpi_int.dt, &dup_type);
-    if (ret != 0) goto cleanup;
-    ompi_datatype_t * type = &ompi_mpi_int.dt;
+    if (ret != 0)
+        goto cleanup;
+    ompi_datatype_t *type = &ompi_mpi_int.dt;
     ret = ompi_datatype_set_args(dup_type, 0, NULL, 0, NULL, 1, &type, MPI_COMBINER_DUP);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
     packed_ddt_len = ompi_datatype_pack_description_length(dup_type);
     ptr = payload = malloc(packed_ddt_len);
     ret = ompi_datatype_get_pack_description(dup_type, &packed_ddt);
-    if (ret != 0) goto cleanup;
+    if (ret != 0)
+        goto cleanup;
 
     memcpy(payload, packed_ddt, packed_ddt_len);
-    unpacked_dt = ompi_datatype_create_from_packed_description(&payload,
-                                                               ompi_proc_local());
+    unpacked_dt = ompi_datatype_create_from_packed_description(&payload, ompi_proc_local());
     free(ptr);
     if (unpacked_dt == NULL) {
         printf("\tFAILED: could not unpack datatype\n");
@@ -428,10 +479,11 @@ main(int argc, char* argv[])
         goto cleanup;
     } else {
         ret = get_extents(unpacked_dt, &lb, &extent, &true_lb, &true_extent);
-        if (ret != 0) goto cleanup;
+        if (ret != 0)
+            goto cleanup;
 
-        if (old_lb != lb || old_extent != extent ||
-            old_true_lb != true_lb || old_true_extent != extent) {
+        if (old_lb != lb || old_extent != extent || old_true_lb != true_lb
+            || old_true_extent != extent) {
             printf("\tFAILED: datatypes don't match\n");
             ret = 1;
             goto cleanup;
@@ -447,9 +499,8 @@ main(int argc, char* argv[])
     }
     ompi_datatype_destroy(&dup_type);
 
- cleanup:
-    ompi_datatype_finalize();
-    opal_finalize_util ();
+cleanup:
+    opal_finalize_util();
 
     return ret;
 }
diff --git a/test/datatype/ddt_raw.c b/test/datatype/ddt_raw.c
index 00da61f0a27..f4847be3b2b 100644
--- a/test/datatype/ddt_raw.c
+++ b/test/datatype/ddt_raw.c
@@ -27,30 +27,33 @@
 #include "opal/datatype/opal_convertor.h"
 #include "opal/runtime/opal.h"
 
-#include <time.h>
 #include <stdlib.h>
+#include <time.h>
 #ifdef HAVE_SYS_TIME_H
-#include <sys/time.h>
+#    include <sys/time.h>
 #endif
 #include <stdio.h>
 
 /* Compile with:
-mpicc -DHAVE_CONFIG_H -I. -I../../include -I../../../ompi-trunk/include  -I../.. -I../../include -I../../../ompi-trunk/opal -I../../../ompi-trunk/orte -I../../../ompi-trunk/ompi -g ddt_test.c -o ddt_test
+mpicc -DHAVE_CONFIG_H -I. -I../../include -I../../../ompi-trunk/include  -I../.. -I../../include
+-I../../../ompi-trunk/opal -I../../../ompi-trunk/orte -I../../../ompi-trunk/ompi -g ddt_test.c -o
+ddt_test
 */
 
 #define TIMER_DATA_TYPE struct timeval
-#define GET_TIME(TV)   gettimeofday( &(TV), NULL )
-#define ELAPSED_TIME(TSTART, TEND)  (((TEND).tv_sec - (TSTART).tv_sec) * 1000000 + ((TEND).tv_usec - (TSTART).tv_usec))
+#define GET_TIME(TV)    gettimeofday(&(TV), NULL)
+#define ELAPSED_TIME(TSTART, TEND) \
+    (((TEND).tv_sec - (TSTART).tv_sec) * 1000000 + ((TEND).tv_usec - (TSTART).tv_usec))
 
 #define DUMP_DATA_AFTER_COMMIT 0x00000001
 #define CHECK_PACK_UNPACK      0x00000002
 
 uint32_t remote_arch = 0xffffffff;
 
-static int test_upper( unsigned int length )
+static int test_upper(unsigned int length)
 {
     ompi_datatype_t *pdt;
-    opal_convertor_t * pConv;
+    opal_convertor_t *pConv;
     int rc = OMPI_SUCCESS;
     unsigned int i, iov_count, total_length;
     size_t max_data;
@@ -58,33 +61,34 @@ static int test_upper( unsigned int length )
     TIMER_DATA_TYPE start, end;
     long total_time;
 
-    printf( "test upper matrix\n" );
-    pdt = upper_matrix( length );
+    printf("test upper matrix\n");
+    pdt = upper_matrix(length);
     /*dt_dump( pdt );*/
 
-    total_length = length * (length + 1) * ( sizeof(double) / 2);
+    total_length = length * (length + 1) * (sizeof(double) / 2);
 
-    pConv = opal_convertor_create( remote_arch, 0 );
-    if( OMPI_SUCCESS != opal_convertor_prepare_for_send( pConv, &(pdt->super), 1, NULL ) ) {
-        printf( "Cannot attach the datatype to a convertor\n" );
+    pConv = opal_convertor_create(remote_arch, 0);
+    if (OMPI_SUCCESS != opal_convertor_prepare_for_send(pConv, &(pdt->super), 1, NULL)) {
+        printf("Cannot attach the datatype to a convertor\n");
         return OMPI_ERROR;
     }
 
-    GET_TIME( start );
-    for( i = total_length; i > 0; ) {
+    GET_TIME(start);
+    for (i = total_length; i > 0;) {
         iov_count = 5;
         max_data = 0;
-        opal_convertor_raw( pConv, iov, &iov_count, &max_data );
+        opal_convertor_raw(pConv, iov, &iov_count, &max_data);
         i -= max_data;
     }
-    GET_TIME( end );
-    total_time = ELAPSED_TIME( start, end );
-    printf( "complete raw in %ld microsec\n", total_time );
+    GET_TIME(end);
+    total_time = ELAPSED_TIME(start, end);
+    printf("complete raw in %ld microsec\n", total_time);
 
     /* test the automatic destruction pf the data */
-    ompi_datatype_destroy( &pdt ); assert( pdt == NULL );
+    ompi_datatype_destroy(&pdt);
+    assert(pdt == NULL);
 
-    OBJ_RELEASE( pConv );
+    OBJ_RELEASE(pConv);
     return rc;
 }
 
@@ -97,26 +101,26 @@ static int test_upper( unsigned int length )
  *  - and one using 2 convertors created from different datatypes.
  *
  */
-static int local_copy_ddt_raw( ompi_datatype_t* pdt, int count, int iov_num )
+static int local_copy_ddt_raw(ompi_datatype_t *pdt, int count, int iov_num)
 {
-    struct iovec* iov;
-    opal_convertor_t* convertor;
+    struct iovec *iov;
+    opal_convertor_t *convertor;
     TIMER_DATA_TYPE start, end;
     long total_time;
     uint32_t iov_count = iov_num;
     size_t max_data = 0, remaining_length;
 
-    iov = (struct iovec*)malloc(iov_num * sizeof(struct iovec));
+    iov = (struct iovec *) malloc(iov_num * sizeof(struct iovec));
 
-    convertor = opal_convertor_create( remote_arch, 0 );
-    if( OMPI_SUCCESS != opal_convertor_prepare_for_send( convertor, &(pdt->super), count, NULL ) ) {
-        printf( "Cannot attach the datatype to a convertor\n" );
+    convertor = opal_convertor_create(remote_arch, 0);
+    if (OMPI_SUCCESS != opal_convertor_prepare_for_send(convertor, &(pdt->super), count, NULL)) {
+        printf("Cannot attach the datatype to a convertor\n");
         return OMPI_ERROR;
     }
 
     remaining_length = count * pdt->super.size;
-    GET_TIME( start );
-    while( 0 == opal_convertor_raw(convertor, iov, &iov_count, &max_data) ) {
+    GET_TIME(start);
+    while (0 == opal_convertor_raw(convertor, iov, &iov_count, &max_data)) {
 #if 0
         printf( "New raw extraction (iov_count = %d, max_data = %zu)\n",
                 iov_count, max_data );
@@ -128,13 +132,13 @@ static int local_copy_ddt_raw( ompi_datatype_t* pdt, int count, int iov_num )
         iov_count = iov_num;
     }
     remaining_length -= max_data;
-    GET_TIME( end );
-    total_time = ELAPSED_TIME( start, end );
-    printf( "raw extraction in %ld microsec\n", total_time );
-    OBJ_RELEASE( convertor );
-    if( remaining_length != 0 ) {
-        printf( "Not all raw description was been extracted (%lu bytes missing)\n",
-                (unsigned long) remaining_length );
+    GET_TIME(end);
+    total_time = ELAPSED_TIME(start, end);
+    printf("raw extraction in %ld microsec\n", total_time);
+    OBJ_RELEASE(convertor);
+    if (remaining_length != 0) {
+        printf("Not all raw description was been extracted (%lu bytes missing)\n",
+               (unsigned long) remaining_length);
     }
     free(iov);
     return OMPI_SUCCESS;
@@ -151,175 +155,194 @@ static int local_copy_ddt_raw( ompi_datatype_t* pdt, int count, int iov_num )
  * modifications on the datatype engine should first pass all the tests from this file,
  * before going into other tests.
  */
-int main( int argc, char* argv[] )
+int main(int argc, char *argv[])
 {
     ompi_datatype_t *pdt, *pdt1, *pdt2, *pdt3;
     int rc, length = 500, iov_num = 5;
 
-    opal_init_util (NULL, NULL);
+    opal_init(NULL, NULL);
     ompi_datatype_init();
 
     /**
      * By default simulate homogeneous architectures.
      */
     remote_arch = opal_local_arch;
-    printf( "\n\n#\n * TEST INVERSED VECTOR\n #\n\n" );
-    pdt = create_inversed_vector( &ompi_mpi_int.dt, 10 );
-    if( outputFlags & CHECK_PACK_UNPACK ) {
+    printf("\n\n#\n * TEST INVERSED VECTOR\n #\n\n");
+    pdt = create_inversed_vector(&ompi_mpi_int.dt, 10);
+    if (outputFlags & CHECK_PACK_UNPACK) {
         local_copy_ddt_raw(pdt, 100, iov_num);
     }
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
-    printf( "\n\n#\n * TEST STRANGE DATATYPE\n #\n\n" );
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
+    printf("\n\n#\n * TEST STRANGE DATATYPE\n #\n\n");
     pdt = create_strange_dt();
-    if( outputFlags & CHECK_PACK_UNPACK ) {
+    if (outputFlags & CHECK_PACK_UNPACK) {
         local_copy_ddt_raw(pdt, 1, iov_num);
     }
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
 
-    printf( "\n\n#\n * TEST UPPER TRIANGULAR MATRIX (size 100)\n #\n\n" );
+    printf("\n\n#\n * TEST UPPER TRIANGULAR MATRIX (size 100)\n #\n\n");
     pdt = upper_matrix(100);
-    if( outputFlags & CHECK_PACK_UNPACK ) {
+    if (outputFlags & CHECK_PACK_UNPACK) {
         local_copy_ddt_raw(pdt, 1, iov_num);
     }
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
 
-    printf( "\n\n#\n * TEST UPPER MATRIX\n #\n\n" );
-    rc = test_upper( length );
-    if( rc == 0 )
-        printf( "decode [PASSED]\n" );
+    printf("\n\n#\n * TEST UPPER MATRIX\n #\n\n");
+    rc = test_upper(length);
+    if (rc == 0)
+        printf("decode [PASSED]\n");
     else
-        printf( "decode [NOT PASSED]\n" );
+        printf("decode [NOT PASSED]\n");
 
-    printf( "\n\n#\n * TEST MATRIX BORDERS\n #\n\n" );
-    pdt = test_matrix_borders( length, 100 );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        ompi_datatype_dump( pdt );
+    printf("\n\n#\n * TEST MATRIX BORDERS\n #\n\n");
+    pdt = test_matrix_borders(length, 100);
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        ompi_datatype_dump(pdt);
     }
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
 
-    printf( "\n\n#\n * TEST CONTIGUOUS\n #\n\n" );
+    printf("\n\n#\n * TEST CONTIGUOUS\n #\n\n");
     pdt = test_contiguous();
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
-    printf( "\n\n#\n * TEST STRUCT\n #\n\n" );
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
+    printf("\n\n#\n * TEST STRUCT\n #\n\n");
     pdt = test_struct();
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
 
     ompi_datatype_create_contiguous(0, &ompi_mpi_datatype_null.dt, &pdt1);
     ompi_datatype_create_contiguous(0, &ompi_mpi_datatype_null.dt, &pdt2);
     ompi_datatype_create_contiguous(0, &ompi_mpi_datatype_null.dt, &pdt3);
 
-    ompi_datatype_add( pdt3, &ompi_mpi_int.dt, 10, 0, -1 );
-    ompi_datatype_add( pdt3, &ompi_mpi_float.dt, 5, 10 * sizeof(int), -1 );
+    ompi_datatype_add(pdt3, &ompi_mpi_int.dt, 10, 0, -1);
+    ompi_datatype_add(pdt3, &ompi_mpi_float.dt, 5, 10 * sizeof(int), -1);
 
-    ompi_datatype_add( pdt2, &ompi_mpi_float.dt, 1, 0, -1 );
-    ompi_datatype_add( pdt2, pdt3, 3, sizeof(int) * 1, -1 );
+    ompi_datatype_add(pdt2, &ompi_mpi_float.dt, 1, 0, -1);
+    ompi_datatype_add(pdt2, pdt3, 3, sizeof(int) * 1, -1);
 
-    ompi_datatype_add( pdt1, &ompi_mpi_long_long_int.dt, 5, 0, -1 );
-    ompi_datatype_add( pdt1, &ompi_mpi_long_double.dt, 2, sizeof(long long) * 5, -1 );
+    ompi_datatype_add(pdt1, &ompi_mpi_long_long_int.dt, 5, 0, -1);
+    ompi_datatype_add(pdt1, &ompi_mpi_long_double.dt, 2, sizeof(long long) * 5, -1);
 
-    printf( ">>--------------------------------------------<<\n" );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        ompi_datatype_dump( pdt1 );
+    printf(">>--------------------------------------------<<\n");
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        ompi_datatype_dump(pdt1);
     }
-    printf( ">>--------------------------------------------<<\n" );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        ompi_datatype_dump( pdt2 );
+    printf(">>--------------------------------------------<<\n");
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        ompi_datatype_dump(pdt2);
     }
-    printf( ">>--------------------------------------------<<\n" );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        ompi_datatype_dump( pdt3 );
+    printf(">>--------------------------------------------<<\n");
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        ompi_datatype_dump(pdt3);
     }
 
-    OBJ_RELEASE( pdt1 ); assert( pdt1 == NULL );
-    OBJ_RELEASE( pdt2 ); assert( pdt2 == NULL );
-    OBJ_RELEASE( pdt3 ); assert( pdt3 == NULL );
+    OBJ_RELEASE(pdt1);
+    assert(pdt1 == NULL);
+    OBJ_RELEASE(pdt2);
+    assert(pdt2 == NULL);
+    OBJ_RELEASE(pdt3);
+    assert(pdt3 == NULL);
 
-    printf( ">>--------------------------------------------<<\n" );
-    printf( " Contiguous datatype (MPI_DOUBLE)\n" );
+    printf(">>--------------------------------------------<<\n");
+    printf(" Contiguous datatype (MPI_DOUBLE)\n");
     pdt = MPI_DOUBLE;
-    if( outputFlags & CHECK_PACK_UNPACK ) {
+    if (outputFlags & CHECK_PACK_UNPACK) {
         local_copy_ddt_raw(pdt, 4500, iov_num);
     }
-    printf( ">>--------------------------------------------<<\n" );
+    printf(">>--------------------------------------------<<\n");
 
-    printf( ">>--------------------------------------------<<\n" );
-    if( outputFlags & CHECK_PACK_UNPACK ) {
-        printf( "Contiguous multiple datatype (4500*1)\n" );
-        pdt = create_contiguous_type( MPI_DOUBLE, 4500 );
+    printf(">>--------------------------------------------<<\n");
+    if (outputFlags & CHECK_PACK_UNPACK) {
+        printf("Contiguous multiple datatype (4500*1)\n");
+        pdt = create_contiguous_type(MPI_DOUBLE, 4500);
         local_copy_ddt_raw(pdt, 1, iov_num);
-        OBJ_RELEASE( pdt ); assert( pdt == NULL );
-        printf( "Contiguous multiple datatype (450*10)\n" );
-        pdt = create_contiguous_type( MPI_DOUBLE, 450 );
+        OBJ_RELEASE(pdt);
+        assert(pdt == NULL);
+        printf("Contiguous multiple datatype (450*10)\n");
+        pdt = create_contiguous_type(MPI_DOUBLE, 450);
         local_copy_ddt_raw(pdt, 10, iov_num);
-        OBJ_RELEASE( pdt ); assert( pdt == NULL );
-        printf( "Contiguous multiple datatype (45*100)\n" );
-        pdt = create_contiguous_type( MPI_DOUBLE, 45 );
+        OBJ_RELEASE(pdt);
+        assert(pdt == NULL);
+        printf("Contiguous multiple datatype (45*100)\n");
+        pdt = create_contiguous_type(MPI_DOUBLE, 45);
         local_copy_ddt_raw(pdt, 100, iov_num);
-        OBJ_RELEASE( pdt ); assert( pdt == NULL );
-        printf( "Contiguous multiple datatype (100*45)\n" );
-        pdt = create_contiguous_type( MPI_DOUBLE, 100 );
+        OBJ_RELEASE(pdt);
+        assert(pdt == NULL);
+        printf("Contiguous multiple datatype (100*45)\n");
+        pdt = create_contiguous_type(MPI_DOUBLE, 100);
         local_copy_ddt_raw(pdt, 45, iov_num);
-        OBJ_RELEASE( pdt ); assert( pdt == NULL );
-        printf( "Contiguous multiple datatype (10*450)\n" );
-        pdt = create_contiguous_type( MPI_DOUBLE, 10 );
+        OBJ_RELEASE(pdt);
+        assert(pdt == NULL);
+        printf("Contiguous multiple datatype (10*450)\n");
+        pdt = create_contiguous_type(MPI_DOUBLE, 10);
         local_copy_ddt_raw(pdt, 450, iov_num);
-        OBJ_RELEASE( pdt ); assert( pdt == NULL );
-        printf( "Contiguous multiple datatype (1*4500)\n" );
-        pdt = create_contiguous_type( MPI_DOUBLE, 1 );
+        OBJ_RELEASE(pdt);
+        assert(pdt == NULL);
+        printf("Contiguous multiple datatype (1*4500)\n");
+        pdt = create_contiguous_type(MPI_DOUBLE, 1);
         local_copy_ddt_raw(pdt, 4500, iov_num);
-        OBJ_RELEASE( pdt ); assert( pdt == NULL );
+        OBJ_RELEASE(pdt);
+        assert(pdt == NULL);
     }
-    printf( ">>--------------------------------------------<<\n" );
-    printf( ">>--------------------------------------------<<\n" );
-    printf( "Vector datatype (450 times 10 double stride 11)\n" );
-    pdt = create_vector_type( MPI_DOUBLE, 450, 10, 11 );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        ompi_datatype_dump( pdt );
+    printf(">>--------------------------------------------<<\n");
+    printf(">>--------------------------------------------<<\n");
+    printf("Vector datatype (450 times 10 double stride 11)\n");
+    pdt = create_vector_type(MPI_DOUBLE, 450, 10, 11);
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        ompi_datatype_dump(pdt);
     }
-    if( outputFlags & CHECK_PACK_UNPACK ) {
+    if (outputFlags & CHECK_PACK_UNPACK) {
         local_copy_ddt_raw(pdt, 1, iov_num);
     }
-    printf( ">>--------------------------------------------<<\n" );
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
+    printf(">>--------------------------------------------<<\n");
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
 
-    printf( ">>--------------------------------------------<<\n" );
+    printf(">>--------------------------------------------<<\n");
     pdt = test_struct_char_double();
-    if( outputFlags & CHECK_PACK_UNPACK ) {
+    if (outputFlags & CHECK_PACK_UNPACK) {
         local_copy_ddt_raw(pdt, 4500, iov_num);
     }
-    printf( ">>--------------------------------------------<<\n" );
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
+    printf(">>--------------------------------------------<<\n");
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
 
-    printf( ">>--------------------------------------------<<\n" );
+    printf(">>--------------------------------------------<<\n");
     pdt = test_create_twice_two_doubles();
-    if( outputFlags & CHECK_PACK_UNPACK ) {
+    if (outputFlags & CHECK_PACK_UNPACK) {
         local_copy_ddt_raw(pdt, 4500, iov_num);
     }
-    printf( ">>--------------------------------------------<<\n" );
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
+    printf(">>--------------------------------------------<<\n");
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
 
-    printf( ">>--------------------------------------------<<\n" );
+    printf(">>--------------------------------------------<<\n");
     pdt = test_create_blacs_type();
-    if( outputFlags & CHECK_PACK_UNPACK ) {
-        if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-            ompi_datatype_dump( pdt );
+    if (outputFlags & CHECK_PACK_UNPACK) {
+        if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+            ompi_datatype_dump(pdt);
         }
         local_copy_ddt_raw(pdt, 4500, iov_num);
     }
-    printf( ">>--------------------------------------------<<\n" );
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
-
-    printf( ">>--------------------------------------------<<\n" );
-    pdt1 = test_create_blacs_type1( &ompi_mpi_int.dt );
-    if( outputFlags & CHECK_PACK_UNPACK ) {
-        local_copy_ddt_raw( pdt1, 1, iov_num );
+    printf(">>--------------------------------------------<<\n");
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
+
+    printf(">>--------------------------------------------<<\n");
+    pdt1 = test_create_blacs_type1(&ompi_mpi_int.dt);
+    if (outputFlags & CHECK_PACK_UNPACK) {
+        local_copy_ddt_raw(pdt1, 1, iov_num);
     }
-    printf( ">>--------------------------------------------<<\n" );
-    OBJ_RELEASE( pdt1 ); assert( pdt1 == NULL );
+    printf(">>--------------------------------------------<<\n");
+    OBJ_RELEASE(pdt1);
+    assert(pdt1 == NULL);
 
     /* clean-ups all data allocations */
-    ompi_datatype_finalize();
-    opal_finalize_util ();
+    opal_finalize_util();
 
     return OMPI_SUCCESS;
 }
diff --git a/test/datatype/ddt_raw2.c b/test/datatype/ddt_raw2.c
index 7e91a323f7a..b7fcf6c62e9 100644
--- a/test/datatype/ddt_raw2.c
+++ b/test/datatype/ddt_raw2.c
@@ -18,39 +18,33 @@
 #include "opal/datatype/opal_datatype_internal.h"
 #include "opal/runtime/opal.h"
 
-#include <time.h>
 #include <stdlib.h>
+#include <time.h>
 #ifdef HAVE_SYS_TIME_H
-#include <sys/time.h>
+#    include <sys/time.h>
 #endif
 #include <stdio.h>
 
-
-static int
-mca_common_ompio_decode_datatype ( ompi_datatype_t *datatype,
-                                   int count,
-                                   struct iovec **iov,
-                                   uint32_t *iovec_count,
-                                   int increment)
+static int mca_common_ompio_decode_datatype(ompi_datatype_t *datatype, int count,
+                                            struct iovec **iov, uint32_t *iovec_count,
+                                            int increment)
 {
     opal_convertor_t *convertor;
     size_t remaining_length = 0;
     uint32_t i;
     uint32_t temp_count;
-    struct iovec *temp_iov=NULL;
+    struct iovec *temp_iov = NULL;
     size_t temp_data;
 
-    convertor = opal_convertor_create( opal_local_arch, 0 );
+    convertor = opal_convertor_create(opal_local_arch, 0);
 
-    if (OMPI_SUCCESS != opal_convertor_prepare_for_send (convertor,
-                                                         &(datatype->super),
-                                                         count,
-                                                         NULL)) {
-        opal_output (1, "Cannot attach the datatype to a convertor\n");
+    if (OMPI_SUCCESS
+        != opal_convertor_prepare_for_send(convertor, &(datatype->super), count, NULL)) {
+        opal_output(1, "Cannot attach the datatype to a convertor\n");
         return OMPI_ERROR;
     }
 
-    if ( 0 == datatype->super.size ) {
+    if (0 == datatype->super.size) {
         *iovec_count = 0;
         *iov = NULL;
         return OMPI_SUCCESS;
@@ -59,247 +53,155 @@ mca_common_ompio_decode_datatype ( ompi_datatype_t *datatype,
     remaining_length = count * datatype->super.size;
 
     temp_count = increment;
-    temp_iov = (struct iovec*)malloc(temp_count * sizeof(struct iovec));
+    temp_iov = (struct iovec *) malloc(temp_count * sizeof(struct iovec));
     if (NULL == temp_iov) {
-        opal_output (1, "OUT OF MEMORY\n");
+        opal_output(1, "OUT OF MEMORY\n");
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
 
-    while (0 == opal_convertor_raw(convertor, temp_iov,
-                                   &temp_count, &temp_data)) {
+    while (0 == opal_convertor_raw(convertor, temp_iov, &temp_count, &temp_data)) {
         *iovec_count = *iovec_count + temp_count;
-        *iov = (struct iovec *) realloc (*iov, *iovec_count * sizeof(struct iovec));
+        *iov = (struct iovec *) realloc(*iov, *iovec_count * sizeof(struct iovec));
         if (NULL == *iov) {
             opal_output(1, "OUT OF MEMORY\n");
             free(temp_iov);
             return OMPI_ERR_OUT_OF_RESOURCE;
         }
-        for (i = 0 ; i < temp_count ; i++) {
-            (*iov)[i+(*iovec_count-temp_count)].iov_base = temp_iov[i].iov_base;
-            (*iov)[i+(*iovec_count-temp_count)].iov_len = temp_iov[i].iov_len;
+        for (i = 0; i < temp_count; i++) {
+            (*iov)[i + (*iovec_count - temp_count)].iov_base = temp_iov[i].iov_base;
+            (*iov)[i + (*iovec_count - temp_count)].iov_len = temp_iov[i].iov_len;
         }
 
         remaining_length -= temp_data;
         temp_count = increment;
     }
     *iovec_count = *iovec_count + temp_count;
-    if ( temp_count > 0 ) {
-        *iov = (struct iovec *) realloc (*iov, *iovec_count * sizeof(struct iovec));
+    if (temp_count > 0) {
+        *iov = (struct iovec *) realloc(*iov, *iovec_count * sizeof(struct iovec));
         if (NULL == *iov) {
             opal_output(1, "OUT OF MEMORY\n");
             free(temp_iov);
             return OMPI_ERR_OUT_OF_RESOURCE;
         }
     }
-    for (i=0 ; i<temp_count ; i++) {
-        (*iov)[i+(*iovec_count-temp_count)].iov_base = temp_iov[i].iov_base;
-        (*iov)[i+(*iovec_count-temp_count)].iov_len = temp_iov[i].iov_len;
+    for (i = 0; i < temp_count; i++) {
+        (*iov)[i + (*iovec_count - temp_count)].iov_base = temp_iov[i].iov_base;
+        (*iov)[i + (*iovec_count - temp_count)].iov_len = temp_iov[i].iov_len;
     }
 
     remaining_length -= temp_data;
 
     if (remaining_length != 0) {
-        printf( "Not all raw description was been extracted (%lu bytes missing)\n",
-                (unsigned long) remaining_length );
+        printf("Not all raw description was been extracted (%lu bytes missing)\n",
+               (unsigned long) remaining_length);
     }
 
-    free (temp_iov);
+    free(temp_iov);
 
     return OMPI_SUCCESS;
 }
 
-int main (int argc, char *argv[]) {
+int main(int argc, char *argv[])
+{
     dt_elem_desc_t descs[185] = {
-        { .loop = { { 16, 0}, 2, 4, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 12, 1, 4} },
-        { .end_loop = { {16, 1}, 2, -1,  12, 4} },
-        { .elem = { { 310, 9 }, 1, 1352, 1, 64} },
-        { .loop = { { 16, 0}, 2, 3, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 8, 1, 1408} },
-        { .end_loop = { {16, 1}, 2, -1,  8, 1424} },
-        { .loop = { { 16, 0}, 2, 4, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 12, 1, 10932} },
-        { .end_loop = { {16, 1}, 2, -1,  12, 10932} },
-        { .elem = { { 310, 9 }, 1, 640, 1, 10992} },
-        { .elem = { { 310, 9 }, 1, 712, 1, 14248} },
-        { .loop = { { 16, 0}, 2, 3, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 8, 1, 14952} },
-        { .end_loop = { {16, 1}, 2, -1,  8, 14968} },
-        { .loop = { { 16, 0}, 2, 4, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 12, 1, 21860} },
-        { .end_loop = { {16, 1}, 2, -1,  12, 21860} },
-        { .elem = { { 310, 9 }, 1, 1352, 1, 21920} },
-        { .loop = { { 16, 0}, 2, 3, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 8, 1, 23264} },
-        { .end_loop = { {16, 1}, 2, -1,  8, 23280} },
-        { .loop = { { 16, 0}, 2, 4, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 12, 1, 30172} },
-        { .end_loop = { {16, 1}, 2, -1,  12, 30172} },
-        { .elem = { { 310, 9 }, 1, 192, 1, 30232} },
-        { .elem = { { 310, 9 }, 1, 1160, 1, 33040} },
-        { .loop = { { 16, 0}, 2, 3, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 8, 1, 34192} },
-        { .end_loop = { {16, 1}, 2, -1,  8, 34208} },
-        { .loop = { { 16, 0}, 2, 4, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 12, 1, 41100} },
-        { .end_loop = { {16, 1}, 2, -1,  12, 41100} },
-        { .elem = { { 310, 9 }, 1, 1352, 1, 41160} },
-        { .loop = { { 16, 0}, 2, 3, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 8, 1, 42504} },
-        { .end_loop = { {16, 1}, 2, -1,  8, 42520} },
-        { .loop = { { 16, 0}, 2, 4, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 12, 1, 52028} },
-        { .end_loop = { {16, 1}, 2, -1,  12, 52028} },
-        { .elem = { { 310, 9 }, 1, 1352, 1, 52088} },
-        { .loop = { { 16, 0}, 2, 3, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 8, 1, 53432} },
-        { .end_loop = { {16, 1}, 2, -1,  8, 53448} },
-        { .loop = { { 16, 0}, 2, 4, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 12, 1, 60340} },
-        { .end_loop = { {16, 1}, 2, -1,  12, 60340} },
-        { .elem = { { 310, 9 }, 1, 1344, 1, 60400} },
-        { .loop = { { 16, 0}, 2, 4, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 8, 1, 64360} },
-        { .end_loop = { {16, 1}, 2, -1,  8, 64360} },
-        { .loop = { { 16, 0}, 2, 4, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 12, 1, 71268} },
-        { .end_loop = { {16, 1}, 2, -1,  12, 71268} },
-        { .elem = { { 310, 9 }, 1, 1352, 1, 71328} },
-        { .loop = { { 16, 0}, 2, 3, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 8, 1, 72672} },
-        { .end_loop = { {16, 1}, 2, -1,  8, 72688} },
-        { .loop = { { 16, 0}, 2, 4, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 12, 1, 79580} },
-        { .end_loop = { {16, 1}, 2, -1,  12, 79580} },
-        { .elem = { { 310, 9 }, 1, 896, 1, 79640} },
-        { .elem = { { 310, 9 }, 1, 456, 1, 83152} },
-        { .loop = { { 16, 0}, 2, 3, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 8, 1, 83600} },
-        { .end_loop = { {16, 1}, 2, -1,  8, 83616} },
-        { .loop = { { 16, 0}, 2, 4, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 12, 1, 90508} },
-        { .end_loop = { {16, 1}, 2, -1,  12, 90508} },
-        { .elem = { { 310, 9 }, 1, 1352, 1, 90568} },
-        { .loop = { { 16, 0}, 2, 3, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 8, 1, 91912} },
-        { .end_loop = { {16, 1}, 2, -1,  8, 91928} },
-        { .loop = { { 16, 0}, 2, 4, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 12, 1, 98820} },
-        { .end_loop = { {16, 1}, 2, -1,  12, 98820} },
-        { .elem = { { 310, 9 }, 1, 448, 1, 98880} },
-        { .elem = { { 310, 9 }, 1, 904, 1, 101944} },
-        { .loop = { { 16, 0}, 2, 3, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 8, 1, 102840} },
-        { .end_loop = { {16, 1}, 2, -1,  8, 102856} },
-        { .loop = { { 16, 0}, 2, 4, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 12, 1, 109748} },
-        { .end_loop = { {16, 1}, 2, -1,  12, 109748} },
-        { .elem = { { 310, 9 }, 1, 1352, 1, 109808} },
-        { .loop = { { 16, 0}, 2, 3, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 8, 1, 111152} },
-        { .end_loop = { {16, 1}, 2, -1,  8, 111168} },
-        { .loop = { { 16, 0}, 2, 4, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 12, 1, 118060} },
-        { .end_loop = { {16, 1}, 2, -1,  12, 118060} },
-        { .elem = { { 310, 9 }, 1, 1352, 1, 120736} },
-        { .loop = { { 16, 0}, 2, 3, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 8, 1, 122080} },
-        { .end_loop = { {16, 1}, 2, -1,  8, 122096} },
-        { .loop = { { 16, 0}, 2, 4, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 12, 1, 128988} },
-        { .end_loop = { {16, 1}, 2, -1,  12, 128988} },
-        { .elem = { { 310, 9 }, 1, 1352, 1, 129048} },
-        { .loop = { { 16, 0}, 2, 3, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 8, 1, 130392} },
-        { .end_loop = { {16, 1}, 2, -1,  8, 130408} },
-        { .loop = { { 16, 0}, 2, 4, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 12, 1, 139916} },
-        { .end_loop = { {16, 1}, 2, -1,  12, 139916} },
-        { .elem = { { 310, 9 }, 1, 1352, 1, 139976} },
-        { .loop = { { 16, 0}, 2, 3, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 8, 1, 141320} },
-        { .end_loop = { {16, 1}, 2, -1,  8, 141336} },
-        { .loop = { { 16, 0}, 2, 4, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 12, 1, 148228} },
-        { .end_loop = { {16, 1}, 2, -1,  12, 148228} },
-        { .elem = { { 310, 9 }, 1, 1152, 1, 148288} },
-        { .elem = { { 310, 9 }, 1, 200, 1, 152056} },
-        { .loop = { { 16, 0}, 2, 3, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 8, 1, 152248} },
-        { .end_loop = { {16, 1}, 2, -1,  8, 152264} },
-        { .loop = { { 16, 0}, 2, 4, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 12, 1, 159156} },
-        { .end_loop = { {16, 1}, 2, -1,  12, 159156} },
-        { .elem = { { 310, 9 }, 1, 1352, 1, 159216} },
-        { .loop = { { 16, 0}, 2, 3, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 8, 1, 160560} },
-        { .end_loop = { {16, 1}, 2, -1,  8, 160576} },
-        { .loop = { { 16, 0}, 2, 4, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 12, 1, 167468} },
-        { .end_loop = { {16, 1}, 2, -1,  12, 167468} },
-        { .elem = { { 310, 9 }, 1, 704, 1, 167528} },
-        { .elem = { { 310, 9 }, 1, 648, 1, 170848} },
-        { .loop = { { 16, 0}, 2, 3, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 8, 1, 171488} },
-        { .end_loop = { {16, 1}, 2, -1,  8, 171504} },
-        { .loop = { { 16, 0}, 2, 4, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 12, 1, 178396} },
-        { .end_loop = { {16, 1}, 2, -1,  12, 178396} },
-        { .elem = { { 310, 9 }, 1, 1352, 1, 178456} },
-        { .loop = { { 16, 0}, 2, 3, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 8, 1, 179800} },
-        { .end_loop = { {16, 1}, 2, -1,  8, 179816} },
-        { .loop = { { 16, 0}, 2, 4, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 12, 1, 186708} },
-        { .end_loop = { {16, 1}, 2, -1,  12, 186708} },
-        { .elem = { { 310, 9 }, 1, 256, 1, 186768} },
-        { .elem = { { 310, 9 }, 1, 1096, 1, 189640} },
-        { .loop = { { 16, 0}, 2, 3, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 8, 1, 190728} },
-        { .end_loop = { {16, 1}, 2, -1,  8, 190744} },
-        { .loop = { { 16, 0}, 2, 4, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 12, 1, 197636} },
-        { .end_loop = { {16, 1}, 2, -1,  12, 197636} },
-        { .elem = { { 310, 9 }, 1, 1352, 1, 197696} },
-        { .loop = { { 16, 0}, 2, 3, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 8, 1, 199040} },
-        { .end_loop = { {16, 1}, 2, -1,  8, 199056} },
-        { .loop = { { 16, 0}, 2, 4, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 12, 1, 208564} },
-        { .end_loop = { {16, 1}, 2, -1,  12, 208564} },
-        { .elem = { { 310, 9 }, 1, 1352, 1, 208624} },
-        { .loop = { { 16, 0}, 2, 3, -1, 16} },
-        { .elem = { { 310, 9 }, 1, 8, 1, 209968} },
-        { .end_loop = { {16, 1}, 2, -1,  8, 209984} },
-        { .elem = { { 310, 9 }, 1, 12, 1, 216876} },
-        { .elem = { { 310, 9 }, 1, 16, 1, 216936} },
-        { .elem = { { 310, 9 }, 1, 16, 1, 217000} },
-        { .elem = { { 310, 9 }, 1, 16, 1, 217064} },
-        { .elem = { { 310, 9 }, 1, 16, 1, 217128} },
-        { .elem = { { 310, 9 }, 1, 16, 1, 217192} },
-        { .elem = { { 310, 9 }, 1, 16, 1, 217256} },
-        { .elem = { { 310, 9 }, 1, 16, 1, 217320} },
-        { .elem = { { 310, 9 }, 1, 16, 1, 217384} },
-        { .elem = { { 310, 9 }, 1, 16, 1, 217448} },
-        { .elem = { { 310, 9 }, 1, 16, 1, 217512} },
-        { .elem = { { 310, 9 }, 1, 16, 1, 217576} },
-        { .elem = { { 310, 9 }, 1, 16, 1, 217640} },
-        { .elem = { { 310, 9 }, 1, 16, 1, 217704} },
-        { .elem = { { 310, 9 }, 1, 16, 1, 217768} },
-        { .elem = { { 310, 9 }, 1, 16, 1, 217832} },
-        { .elem = { { 310, 9 }, 1, 16, 1, 217896} },
-        { .elem = { { 310, 9 }, 1, 16, 1, 217960} },
-        { .elem = { { 310, 9 }, 1, 16, 1, 218024} },
-        { .elem = { { 310, 9 }, 1, 16, 1, 218088} },
-        { .elem = { { 310, 9 }, 1, 16, 1, 218152} },
-        { .elem = { { 310, 9 }, 1, 16, 1, 218216} },
-        { .elem = { { 310, 9 }, 1, 8, 1, 218280} },
-        { .end_loop = { {0, 1}, 184, -1,  31684, 4} },
+        {.loop = {{16, 0}, 2, 4, -1, 16}},          {.elem = {{310, 9}, 1, 12, 1, 4}},
+        {.end_loop = {{16, 1}, 2, -1, 12, 4}},      {.elem = {{310, 9}, 1, 1352, 1, 64}},
+        {.loop = {{16, 0}, 2, 3, -1, 16}},          {.elem = {{310, 9}, 1, 8, 1, 1408}},
+        {.end_loop = {{16, 1}, 2, -1, 8, 1424}},    {.loop = {{16, 0}, 2, 4, -1, 16}},
+        {.elem = {{310, 9}, 1, 12, 1, 10932}},      {.end_loop = {{16, 1}, 2, -1, 12, 10932}},
+        {.elem = {{310, 9}, 1, 640, 1, 10992}},     {.elem = {{310, 9}, 1, 712, 1, 14248}},
+        {.loop = {{16, 0}, 2, 3, -1, 16}},          {.elem = {{310, 9}, 1, 8, 1, 14952}},
+        {.end_loop = {{16, 1}, 2, -1, 8, 14968}},   {.loop = {{16, 0}, 2, 4, -1, 16}},
+        {.elem = {{310, 9}, 1, 12, 1, 21860}},      {.end_loop = {{16, 1}, 2, -1, 12, 21860}},
+        {.elem = {{310, 9}, 1, 1352, 1, 21920}},    {.loop = {{16, 0}, 2, 3, -1, 16}},
+        {.elem = {{310, 9}, 1, 8, 1, 23264}},       {.end_loop = {{16, 1}, 2, -1, 8, 23280}},
+        {.loop = {{16, 0}, 2, 4, -1, 16}},          {.elem = {{310, 9}, 1, 12, 1, 30172}},
+        {.end_loop = {{16, 1}, 2, -1, 12, 30172}},  {.elem = {{310, 9}, 1, 192, 1, 30232}},
+        {.elem = {{310, 9}, 1, 1160, 1, 33040}},    {.loop = {{16, 0}, 2, 3, -1, 16}},
+        {.elem = {{310, 9}, 1, 8, 1, 34192}},       {.end_loop = {{16, 1}, 2, -1, 8, 34208}},
+        {.loop = {{16, 0}, 2, 4, -1, 16}},          {.elem = {{310, 9}, 1, 12, 1, 41100}},
+        {.end_loop = {{16, 1}, 2, -1, 12, 41100}},  {.elem = {{310, 9}, 1, 1352, 1, 41160}},
+        {.loop = {{16, 0}, 2, 3, -1, 16}},          {.elem = {{310, 9}, 1, 8, 1, 42504}},
+        {.end_loop = {{16, 1}, 2, -1, 8, 42520}},   {.loop = {{16, 0}, 2, 4, -1, 16}},
+        {.elem = {{310, 9}, 1, 12, 1, 52028}},      {.end_loop = {{16, 1}, 2, -1, 12, 52028}},
+        {.elem = {{310, 9}, 1, 1352, 1, 52088}},    {.loop = {{16, 0}, 2, 3, -1, 16}},
+        {.elem = {{310, 9}, 1, 8, 1, 53432}},       {.end_loop = {{16, 1}, 2, -1, 8, 53448}},
+        {.loop = {{16, 0}, 2, 4, -1, 16}},          {.elem = {{310, 9}, 1, 12, 1, 60340}},
+        {.end_loop = {{16, 1}, 2, -1, 12, 60340}},  {.elem = {{310, 9}, 1, 1344, 1, 60400}},
+        {.loop = {{16, 0}, 2, 4, -1, 16}},          {.elem = {{310, 9}, 1, 8, 1, 64360}},
+        {.end_loop = {{16, 1}, 2, -1, 8, 64360}},   {.loop = {{16, 0}, 2, 4, -1, 16}},
+        {.elem = {{310, 9}, 1, 12, 1, 71268}},      {.end_loop = {{16, 1}, 2, -1, 12, 71268}},
+        {.elem = {{310, 9}, 1, 1352, 1, 71328}},    {.loop = {{16, 0}, 2, 3, -1, 16}},
+        {.elem = {{310, 9}, 1, 8, 1, 72672}},       {.end_loop = {{16, 1}, 2, -1, 8, 72688}},
+        {.loop = {{16, 0}, 2, 4, -1, 16}},          {.elem = {{310, 9}, 1, 12, 1, 79580}},
+        {.end_loop = {{16, 1}, 2, -1, 12, 79580}},  {.elem = {{310, 9}, 1, 896, 1, 79640}},
+        {.elem = {{310, 9}, 1, 456, 1, 83152}},     {.loop = {{16, 0}, 2, 3, -1, 16}},
+        {.elem = {{310, 9}, 1, 8, 1, 83600}},       {.end_loop = {{16, 1}, 2, -1, 8, 83616}},
+        {.loop = {{16, 0}, 2, 4, -1, 16}},          {.elem = {{310, 9}, 1, 12, 1, 90508}},
+        {.end_loop = {{16, 1}, 2, -1, 12, 90508}},  {.elem = {{310, 9}, 1, 1352, 1, 90568}},
+        {.loop = {{16, 0}, 2, 3, -1, 16}},          {.elem = {{310, 9}, 1, 8, 1, 91912}},
+        {.end_loop = {{16, 1}, 2, -1, 8, 91928}},   {.loop = {{16, 0}, 2, 4, -1, 16}},
+        {.elem = {{310, 9}, 1, 12, 1, 98820}},      {.end_loop = {{16, 1}, 2, -1, 12, 98820}},
+        {.elem = {{310, 9}, 1, 448, 1, 98880}},     {.elem = {{310, 9}, 1, 904, 1, 101944}},
+        {.loop = {{16, 0}, 2, 3, -1, 16}},          {.elem = {{310, 9}, 1, 8, 1, 102840}},
+        {.end_loop = {{16, 1}, 2, -1, 8, 102856}},  {.loop = {{16, 0}, 2, 4, -1, 16}},
+        {.elem = {{310, 9}, 1, 12, 1, 109748}},     {.end_loop = {{16, 1}, 2, -1, 12, 109748}},
+        {.elem = {{310, 9}, 1, 1352, 1, 109808}},   {.loop = {{16, 0}, 2, 3, -1, 16}},
+        {.elem = {{310, 9}, 1, 8, 1, 111152}},      {.end_loop = {{16, 1}, 2, -1, 8, 111168}},
+        {.loop = {{16, 0}, 2, 4, -1, 16}},          {.elem = {{310, 9}, 1, 12, 1, 118060}},
+        {.end_loop = {{16, 1}, 2, -1, 12, 118060}}, {.elem = {{310, 9}, 1, 1352, 1, 120736}},
+        {.loop = {{16, 0}, 2, 3, -1, 16}},          {.elem = {{310, 9}, 1, 8, 1, 122080}},
+        {.end_loop = {{16, 1}, 2, -1, 8, 122096}},  {.loop = {{16, 0}, 2, 4, -1, 16}},
+        {.elem = {{310, 9}, 1, 12, 1, 128988}},     {.end_loop = {{16, 1}, 2, -1, 12, 128988}},
+        {.elem = {{310, 9}, 1, 1352, 1, 129048}},   {.loop = {{16, 0}, 2, 3, -1, 16}},
+        {.elem = {{310, 9}, 1, 8, 1, 130392}},      {.end_loop = {{16, 1}, 2, -1, 8, 130408}},
+        {.loop = {{16, 0}, 2, 4, -1, 16}},          {.elem = {{310, 9}, 1, 12, 1, 139916}},
+        {.end_loop = {{16, 1}, 2, -1, 12, 139916}}, {.elem = {{310, 9}, 1, 1352, 1, 139976}},
+        {.loop = {{16, 0}, 2, 3, -1, 16}},          {.elem = {{310, 9}, 1, 8, 1, 141320}},
+        {.end_loop = {{16, 1}, 2, -1, 8, 141336}},  {.loop = {{16, 0}, 2, 4, -1, 16}},
+        {.elem = {{310, 9}, 1, 12, 1, 148228}},     {.end_loop = {{16, 1}, 2, -1, 12, 148228}},
+        {.elem = {{310, 9}, 1, 1152, 1, 148288}},   {.elem = {{310, 9}, 1, 200, 1, 152056}},
+        {.loop = {{16, 0}, 2, 3, -1, 16}},          {.elem = {{310, 9}, 1, 8, 1, 152248}},
+        {.end_loop = {{16, 1}, 2, -1, 8, 152264}},  {.loop = {{16, 0}, 2, 4, -1, 16}},
+        {.elem = {{310, 9}, 1, 12, 1, 159156}},     {.end_loop = {{16, 1}, 2, -1, 12, 159156}},
+        {.elem = {{310, 9}, 1, 1352, 1, 159216}},   {.loop = {{16, 0}, 2, 3, -1, 16}},
+        {.elem = {{310, 9}, 1, 8, 1, 160560}},      {.end_loop = {{16, 1}, 2, -1, 8, 160576}},
+        {.loop = {{16, 0}, 2, 4, -1, 16}},          {.elem = {{310, 9}, 1, 12, 1, 167468}},
+        {.end_loop = {{16, 1}, 2, -1, 12, 167468}}, {.elem = {{310, 9}, 1, 704, 1, 167528}},
+        {.elem = {{310, 9}, 1, 648, 1, 170848}},    {.loop = {{16, 0}, 2, 3, -1, 16}},
+        {.elem = {{310, 9}, 1, 8, 1, 171488}},      {.end_loop = {{16, 1}, 2, -1, 8, 171504}},
+        {.loop = {{16, 0}, 2, 4, -1, 16}},          {.elem = {{310, 9}, 1, 12, 1, 178396}},
+        {.end_loop = {{16, 1}, 2, -1, 12, 178396}}, {.elem = {{310, 9}, 1, 1352, 1, 178456}},
+        {.loop = {{16, 0}, 2, 3, -1, 16}},          {.elem = {{310, 9}, 1, 8, 1, 179800}},
+        {.end_loop = {{16, 1}, 2, -1, 8, 179816}},  {.loop = {{16, 0}, 2, 4, -1, 16}},
+        {.elem = {{310, 9}, 1, 12, 1, 186708}},     {.end_loop = {{16, 1}, 2, -1, 12, 186708}},
+        {.elem = {{310, 9}, 1, 256, 1, 186768}},    {.elem = {{310, 9}, 1, 1096, 1, 189640}},
+        {.loop = {{16, 0}, 2, 3, -1, 16}},          {.elem = {{310, 9}, 1, 8, 1, 190728}},
+        {.end_loop = {{16, 1}, 2, -1, 8, 190744}},  {.loop = {{16, 0}, 2, 4, -1, 16}},
+        {.elem = {{310, 9}, 1, 12, 1, 197636}},     {.end_loop = {{16, 1}, 2, -1, 12, 197636}},
+        {.elem = {{310, 9}, 1, 1352, 1, 197696}},   {.loop = {{16, 0}, 2, 3, -1, 16}},
+        {.elem = {{310, 9}, 1, 8, 1, 199040}},      {.end_loop = {{16, 1}, 2, -1, 8, 199056}},
+        {.loop = {{16, 0}, 2, 4, -1, 16}},          {.elem = {{310, 9}, 1, 12, 1, 208564}},
+        {.end_loop = {{16, 1}, 2, -1, 12, 208564}}, {.elem = {{310, 9}, 1, 1352, 1, 208624}},
+        {.loop = {{16, 0}, 2, 3, -1, 16}},          {.elem = {{310, 9}, 1, 8, 1, 209968}},
+        {.end_loop = {{16, 1}, 2, -1, 8, 209984}},  {.elem = {{310, 9}, 1, 12, 1, 216876}},
+        {.elem = {{310, 9}, 1, 16, 1, 216936}},     {.elem = {{310, 9}, 1, 16, 1, 217000}},
+        {.elem = {{310, 9}, 1, 16, 1, 217064}},     {.elem = {{310, 9}, 1, 16, 1, 217128}},
+        {.elem = {{310, 9}, 1, 16, 1, 217192}},     {.elem = {{310, 9}, 1, 16, 1, 217256}},
+        {.elem = {{310, 9}, 1, 16, 1, 217320}},     {.elem = {{310, 9}, 1, 16, 1, 217384}},
+        {.elem = {{310, 9}, 1, 16, 1, 217448}},     {.elem = {{310, 9}, 1, 16, 1, 217512}},
+        {.elem = {{310, 9}, 1, 16, 1, 217576}},     {.elem = {{310, 9}, 1, 16, 1, 217640}},
+        {.elem = {{310, 9}, 1, 16, 1, 217704}},     {.elem = {{310, 9}, 1, 16, 1, 217768}},
+        {.elem = {{310, 9}, 1, 16, 1, 217832}},     {.elem = {{310, 9}, 1, 16, 1, 217896}},
+        {.elem = {{310, 9}, 1, 16, 1, 217960}},     {.elem = {{310, 9}, 1, 16, 1, 218024}},
+        {.elem = {{310, 9}, 1, 16, 1, 218088}},     {.elem = {{310, 9}, 1, 16, 1, 218152}},
+        {.elem = {{310, 9}, 1, 16, 1, 218216}},     {.elem = {{310, 9}, 1, 8, 1, 218280}},
+        {.end_loop = {{0, 1}, 184, -1, 31684, 4}},
     };
-    opal_init_util(NULL, NULL);
+    opal_init(NULL, NULL);
     ompi_datatype_init();
 
-    ompi_datatype_t * datatype = OBJ_NEW(ompi_datatype_t);
+    ompi_datatype_t *datatype = OBJ_NEW(ompi_datatype_t);
 
     datatype->super.flags = 3332;
     datatype->super.id = 0;
@@ -321,20 +223,20 @@ int main (int argc, char *argv[]) {
 
     /* Get the entire raw description of the datatype in a single call */
     uint32_t iovec_count_300 = 0;
-    struct iovec * iov_300 = NULL;
-    mca_common_ompio_decode_datatype ( datatype, 1, &iov_300, &iovec_count_300, 300);
+    struct iovec *iov_300 = NULL;
+    mca_common_ompio_decode_datatype(datatype, 1, &iov_300, &iovec_count_300, 300);
     /* Get the raw description of the datatype 10 elements at the time. This stresses some
      * of the execution paths in the convertor raw.
      */
     uint32_t iovec_count_10 = 0;
-    struct iovec * iov_10 = NULL;
-    mca_common_ompio_decode_datatype ( datatype, 1, &iov_10, &iovec_count_10, 10);
+    struct iovec *iov_10 = NULL;
+    mca_common_ompio_decode_datatype(datatype, 1, &iov_10, &iovec_count_10, 10);
     /* Get the raw description of the datatype one element at the time. This stresses all
      * execution paths in the convertor raw.
      */
     uint32_t iovec_count_1 = 0;
-    struct iovec * iov_1 = NULL;
-    mca_common_ompio_decode_datatype ( datatype, 1, &iov_1, &iovec_count_1, 1);
+    struct iovec *iov_1 = NULL;
+    mca_common_ompio_decode_datatype(datatype, 1, &iov_1, &iovec_count_1, 1);
 
     assert(iovec_count_300 == iovec_count_10);
     assert(iovec_count_300 == iovec_count_1);
@@ -349,4 +251,3 @@ int main (int argc, char *argv[]) {
 
     return 0;
 }
-
diff --git a/test/datatype/ddt_test.c b/test/datatype/ddt_test.c
index aa2de4954ae..d8c879e3cfc 100644
--- a/test/datatype/ddt_test.c
+++ b/test/datatype/ddt_test.c
@@ -20,34 +20,37 @@
 
 #include "ompi_config.h"
 #include "ddt_lib.h"
-#include "opal/runtime/opal.h"
 #include "opal/datatype/opal_convertor.h"
-#include <time.h>
+#include "opal/runtime/opal.h"
 #include <stdlib.h>
+#include <time.h>
 #ifdef HAVE_SYS_TIME_H
-#include <sys/time.h>
+#    include <sys/time.h>
 #endif
 #include <stdio.h>
 #include <string.h>
 
 /* Compile with:
-mpicc -DHAVE_CONFIG_H -I. -I../../include -I../../../ompi-trunk/include  -I../.. -I../../include -I../../../ompi-trunk/opal -I../../../ompi-trunk/orte -I../../../ompi-trunk/ompi -g ddt_test.c -o ddt_test
+mpicc -DHAVE_CONFIG_H -I. -I../../include -I../../../ompi-trunk/include  -I../.. -I../../include
+-I../../../ompi-trunk/opal -I../../../ompi-trunk/orte -I../../../ompi-trunk/ompi -g ddt_test.c -o
+ddt_test
 */
 
 #define TIMER_DATA_TYPE struct timeval
-#define GET_TIME(TV)   gettimeofday( &(TV), NULL )
-#define ELAPSED_TIME(TSTART, TEND)  (((TEND).tv_sec - (TSTART).tv_sec) * 1000000 + ((TEND).tv_usec - (TSTART).tv_usec))
+#define GET_TIME(TV)    gettimeofday(&(TV), NULL)
+#define ELAPSED_TIME(TSTART, TEND) \
+    (((TEND).tv_sec - (TSTART).tv_sec) * 1000000 + ((TEND).tv_usec - (TSTART).tv_usec))
 
 #define DUMP_DATA_AFTER_COMMIT 0x00000001
 #define CHECK_PACK_UNPACK      0x00000002
 
 uint32_t remote_arch = 0xffffffff;
 
-static int test_upper( unsigned int length )
+static int test_upper(unsigned int length)
 {
     double *mat1, *mat2, *inbuf;
     ompi_datatype_t *pdt;
-    opal_convertor_t * pConv;
+    opal_convertor_t *pConv;
     char *ptr;
     int rc;
     unsigned int i, j, iov_count, split_chunk, total_length;
@@ -56,60 +59,62 @@ static int test_upper( unsigned int length )
     TIMER_DATA_TYPE start, end;
     long total_time;
 
-    printf( "test upper matrix\n" );
-    pdt = upper_matrix( length );
+    printf("test upper matrix\n");
+    pdt = upper_matrix(length);
     /*dt_dump( pdt );*/
 
-    mat1 = malloc( length * length * sizeof(double) );
-    init_random_upper_matrix( length, mat1 );
-    mat2 = calloc( length * length, sizeof(double) );
+    mat1 = malloc(length * length * sizeof(double));
+    init_random_upper_matrix(length, mat1);
+    mat2 = calloc(length * length, sizeof(double));
 
-    total_length = length * (length + 1) * ( sizeof(double) / 2);
-    inbuf = (double*)malloc( total_length );
-    ptr = (char*)inbuf;
+    total_length = length * (length + 1) * (sizeof(double) / 2);
+    inbuf = (double *) malloc(total_length);
+    ptr = (char *) inbuf;
     /* copy upper matrix in the array simulating the input buffer */
-    for( i = 0; i < length; i++ ) {
+    for (i = 0; i < length; i++) {
         uint32_t pos = i * length + i;
-        for( j = i; j < length; j++, pos++ ) {
+        for (j = i; j < length; j++, pos++) {
             *inbuf = mat1[pos];
             inbuf++;
         }
     }
-    inbuf = (double*)ptr;
-    pConv = opal_convertor_create( remote_arch, 0 );
-    if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( pConv, &(pdt->super), 1, mat2 ) ) {
-        printf( "Cannot attach the datatype to a convertor\n" );
+    inbuf = (double *) ptr;
+    pConv = opal_convertor_create(remote_arch, 0);
+    if (OPAL_SUCCESS != opal_convertor_prepare_for_recv(pConv, &(pdt->super), 1, mat2)) {
+        printf("Cannot attach the datatype to a convertor\n");
         return OMPI_ERROR;
     }
 
-    GET_TIME( start );
+    GET_TIME(start);
     split_chunk = (length + 1) * sizeof(double);
     /*    split_chunk = (total_length + 1) * sizeof(double); */
-    for( i = total_length; i > 0; ) {
-        if( i <= split_chunk ) {  /* equal test just to be able to set a breakpoint */
+    for (i = total_length; i > 0;) {
+        if (i <= split_chunk) { /* equal test just to be able to set a breakpoint */
             split_chunk = i;
         }
         a.iov_base = ptr;
         a.iov_len = split_chunk;
         iov_count = 1;
         max_data = split_chunk;
-        opal_convertor_unpack( pConv, &a, &iov_count, &max_data );
+        opal_convertor_unpack(pConv, &a, &iov_count, &max_data);
         ptr += max_data;
         i -= max_data;
-        if( mat2[0] != inbuf[0] ) assert(0);
+        if (mat2[0] != inbuf[0])
+            assert(0);
     }
-    GET_TIME( end );
-    total_time = ELAPSED_TIME( start, end );
-    printf( "complete unpacking in %ld microsec\n", total_time );
-    free( inbuf );
-    rc = check_diag_matrix( length, mat1, mat2 );
-    free( mat1 );
-    free( mat2 );
+    GET_TIME(end);
+    total_time = ELAPSED_TIME(start, end);
+    printf("complete unpacking in %ld microsec\n", total_time);
+    free(inbuf);
+    rc = check_diag_matrix(length, mat1, mat2);
+    free(mat1);
+    free(mat2);
 
     /* test the automatic destruction pf the data */
-    ompi_datatype_destroy( &pdt ); assert( pdt == NULL );
+    ompi_datatype_destroy(&pdt);
+    assert(pdt == NULL);
 
-    OBJ_RELEASE( pConv );
+    OBJ_RELEASE(pConv);
     return rc;
 }
 
@@ -118,16 +123,17 @@ static int test_upper( unsigned int length )
  * is not an easy task. Define a function to centralize the complexity in a
  * single location.
  */
-static size_t compute_buffer_length(ompi_datatype_t* pdt, int count)
+static size_t compute_buffer_length(ompi_datatype_t *pdt, int count)
 {
     MPI_Aint extent, lb, true_extent, true_lb;
     size_t length;
 
     ompi_datatype_get_extent(pdt, &lb, &extent);
-    ompi_datatype_get_true_extent(pdt, &true_lb, &true_extent); (void)true_lb;
+    ompi_datatype_get_true_extent(pdt, &true_lb, &true_extent);
+    (void) true_lb;
     length = true_lb + true_extent + (count - 1) * extent;
 
-    return  length;
+    return length;
 }
 
 /**
@@ -139,7 +145,7 @@ static size_t compute_buffer_length(ompi_datatype_t* pdt, int count)
  *  - and one using 2 convertors created from different data-types.
  *
  */
-static int local_copy_ddt_count( ompi_datatype_t* pdt, int count )
+static int local_copy_ddt_count(ompi_datatype_t *pdt, int count)
 {
     void *pdst, *psrc;
     TIMER_DATA_TYPE start, end;
@@ -151,30 +157,29 @@ static int local_copy_ddt_count( ompi_datatype_t* pdt, int count )
     pdst = malloc(length);
     psrc = malloc(length);
 
-    for( size_t i = 0; i < length; i++ )
-	((char*)psrc)[i] = i % 128 + 32;
+    for (size_t i = 0; i < length; i++)
+        ((char *) psrc)[i] = i % 128 + 32;
     memset(pdst, 0, length);
 
-    cache_trash();  /* make sure the cache is useless */
+    cache_trash(); /* make sure the cache is useless */
 
-    GET_TIME( start );
-    if( OMPI_SUCCESS != ompi_datatype_copy_content_same_ddt( pdt, count, pdst, psrc ) ) {
-        printf( "Unable to copy the datatype in the function local_copy_ddt_count."
-                " Is the datatype committed ?\n" );
+    GET_TIME(start);
+    if (OMPI_SUCCESS != ompi_datatype_copy_content_same_ddt(pdt, count, pdst, psrc)) {
+        printf("Unable to copy the datatype in the function local_copy_ddt_count."
+               " Is the datatype committed ?\n");
     }
-    GET_TIME( end );
-    total_time = ELAPSED_TIME( start, end );
-    printf( "direct local copy in %ld microsec\n", total_time );
+    GET_TIME(end);
+    total_time = ELAPSED_TIME(start, end);
+    printf("direct local copy in %ld microsec\n", total_time);
     free(pdst);
     free(psrc);
 
     return OMPI_SUCCESS;
 }
 
-static int
-local_copy_with_convertor_2datatypes( ompi_datatype_t* send_type, int send_count,
-                                      ompi_datatype_t* recv_type, int recv_count,
-                                      int chunk )
+static int local_copy_with_convertor_2datatypes(ompi_datatype_t *send_type, int send_count,
+                                                ompi_datatype_t *recv_type, int recv_count,
+                                                int chunk)
 {
     void *pdst = NULL, *psrc = NULL, *ptemp = NULL;
     opal_convertor_t *send_convertor = NULL, *recv_convertor = NULL;
@@ -188,35 +193,37 @@ local_copy_with_convertor_2datatypes( ompi_datatype_t* send_type, int send_count
 
     rlength = compute_buffer_length(recv_type, recv_count);
     slength = compute_buffer_length(send_type, send_count);
-    pdst  = malloc( rlength );
-    psrc  = malloc( slength );
-    ptemp = malloc( chunk );
+    pdst = malloc(rlength);
+    psrc = malloc(slength);
+    ptemp = malloc(chunk);
 
     /* initialize the buffers to prevent valgrind from complaining */
-    for( size_t i = 0; i < slength; i++ )
-            ((char*)psrc)[i] = i % 128 + 32;
+    for (size_t i = 0; i < slength; i++)
+        ((char *) psrc)[i] = i % 128 + 32;
     memset(pdst, 0, rlength);
 
-    send_convertor = opal_convertor_create( remote_arch, 0 );
-    if( OPAL_SUCCESS != opal_convertor_prepare_for_send( send_convertor, &(send_type->super), send_count, psrc ) ) {
-        printf( "Unable to create the send convertor. Is the datatype committed ?\n" );
+    send_convertor = opal_convertor_create(remote_arch, 0);
+    if (OPAL_SUCCESS
+        != opal_convertor_prepare_for_send(send_convertor, &(send_type->super), send_count, psrc)) {
+        printf("Unable to create the send convertor. Is the datatype committed ?\n");
         goto clean_and_return;
     }
-    recv_convertor = opal_convertor_create( remote_arch, 0 );
-    if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( recv_convertor, &(recv_type->super), recv_count, pdst ) ) {
-        printf( "Unable to create the recv convertor. Is the datatype committed ?\n" );
+    recv_convertor = opal_convertor_create(remote_arch, 0);
+    if (OPAL_SUCCESS
+        != opal_convertor_prepare_for_recv(recv_convertor, &(recv_type->super), recv_count, pdst)) {
+        printf("Unable to create the recv convertor. Is the datatype committed ?\n");
         goto clean_and_return;
     }
 
-    cache_trash();  /* make sure the cache is useless */
+    cache_trash(); /* make sure the cache is useless */
 
-    GET_TIME( start );
-    while( (done1 & done2) != 1 ) {
+    GET_TIME(start);
+    while ((done1 & done2) != 1) {
         /* They are supposed to finish in exactly the same time. */
-        if( done1 | done2 ) {
-            printf( "WRONG !!! the send is %s but the receive is %s in local_copy_with_convertor_2datatypes\n",
-                    (done1 ? "finish" : "not finish"),
-                    (done2 ? "finish" : "not finish") );
+        if (done1 | done2) {
+            printf("WRONG !!! the send is %s but the receive is %s in "
+                   "local_copy_with_convertor_2datatypes\n",
+                   (done1 ? "finish" : "not finish"), (done2 ? "finish" : "not finish"));
         }
 
         max_data = chunk;
@@ -224,38 +231,43 @@ local_copy_with_convertor_2datatypes( ompi_datatype_t* send_type, int send_count
         iov.iov_base = ptemp;
         iov.iov_len = chunk;
 
-        if( done1 == 0 ) {
-            done1 = opal_convertor_pack( send_convertor, &iov, &iov_count, &max_data );
+        if (done1 == 0) {
+            done1 = opal_convertor_pack(send_convertor, &iov, &iov_count, &max_data);
         }
 
-        if( done2 == 0 ) {
-            GET_TIME( unpack_start );
-            done2 = opal_convertor_unpack( recv_convertor, &iov, &iov_count, &max_data );
-            GET_TIME( unpack_end );
-            unpack_time += ELAPSED_TIME( unpack_start, unpack_end );
+        if (done2 == 0) {
+            GET_TIME(unpack_start);
+            done2 = opal_convertor_unpack(recv_convertor, &iov, &iov_count, &max_data);
+            GET_TIME(unpack_end);
+            unpack_time += ELAPSED_TIME(unpack_start, unpack_end);
         }
 
         length += max_data;
     }
-    GET_TIME( end );
-    total_time = ELAPSED_TIME( start, end );
-    printf( "copying different data-types using convertors in %ld microsec\n", total_time );
-    printf( "\t unpack in %ld microsec [pack in %ld microsec]\n", unpack_time,
-            total_time - unpack_time );
- clean_and_return:
-    if( send_convertor != NULL ) {
-        OBJ_RELEASE( send_convertor ); assert( send_convertor == NULL );
+    GET_TIME(end);
+    total_time = ELAPSED_TIME(start, end);
+    printf("copying different data-types using convertors in %ld microsec\n", total_time);
+    printf("\t unpack in %ld microsec [pack in %ld microsec]\n", unpack_time,
+           total_time - unpack_time);
+clean_and_return:
+    if (send_convertor != NULL) {
+        OBJ_RELEASE(send_convertor);
+        assert(send_convertor == NULL);
     }
-    if( recv_convertor != NULL ) {
-        OBJ_RELEASE( recv_convertor ); assert( recv_convertor == NULL );
+    if (recv_convertor != NULL) {
+        OBJ_RELEASE(recv_convertor);
+        assert(recv_convertor == NULL);
     }
-    if( NULL != pdst ) free( pdst );
-    if( NULL != psrc ) free( psrc );
-    if( NULL != ptemp ) free( ptemp );
+    if (NULL != pdst)
+        free(pdst);
+    if (NULL != psrc)
+        free(psrc);
+    if (NULL != ptemp)
+        free(ptemp);
     return OMPI_SUCCESS;
 }
 
-static int local_copy_with_convertor( ompi_datatype_t* pdt, int count, int chunk )
+static int local_copy_with_convertor(ompi_datatype_t *pdt, int count, int chunk)
 {
     void *pdst = NULL, *psrc = NULL, *ptemp = NULL;
     opal_convertor_t *send_convertor = NULL, *recv_convertor = NULL;
@@ -268,34 +280,36 @@ static int local_copy_with_convertor( ompi_datatype_t* pdt, int count, int chunk
 
     max_data = compute_buffer_length(pdt, count);
 
-    pdst  = malloc(max_data);
-    psrc  = malloc(max_data);
+    pdst = malloc(max_data);
+    psrc = malloc(max_data);
     ptemp = malloc(chunk);
 
-    for( int i = 0; i < length; ((char*)psrc)[i] = i % 128 + 32, i++ );
-    memset( pdst, 0, length );
+    for (int i = 0; i < length; ((char *) psrc)[i] = i % 128 + 32, i++)
+        ;
+    memset(pdst, 0, length);
 
-    send_convertor = opal_convertor_create( remote_arch, 0 );
-    if( OPAL_SUCCESS != opal_convertor_prepare_for_send( send_convertor, &(pdt->super), count, psrc ) ) {
-        printf( "Unable to create the send convertor. Is the datatype committed ?\n" );
+    send_convertor = opal_convertor_create(remote_arch, 0);
+    if (OPAL_SUCCESS
+        != opal_convertor_prepare_for_send(send_convertor, &(pdt->super), count, psrc)) {
+        printf("Unable to create the send convertor. Is the datatype committed ?\n");
         goto clean_and_return;
     }
 
-    recv_convertor = opal_convertor_create( remote_arch, 0 );
-    if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( recv_convertor, &(pdt->super), count, pdst ) ) {
-        printf( "Unable to create the recv convertor. Is the datatype committed ?\n" );
+    recv_convertor = opal_convertor_create(remote_arch, 0);
+    if (OPAL_SUCCESS
+        != opal_convertor_prepare_for_recv(recv_convertor, &(pdt->super), count, pdst)) {
+        printf("Unable to create the recv convertor. Is the datatype committed ?\n");
         goto clean_and_return;
     }
 
-    cache_trash();  /* make sure the cache is useless */
+    cache_trash(); /* make sure the cache is useless */
 
-    GET_TIME( start );
-    while( (done1 & done2) != 1 ) {
+    GET_TIME(start);
+    while ((done1 & done2) != 1) {
         /* They are supposed to finish in exactly the same time. */
-        if( done1 | done2 ) {
-            printf( "WRONG !!! the send is %s but the receive is %s in local_copy_with_convertor\n",
-                    (done1 ? "finish" : "not finish"),
-                    (done2 ? "finish" : "not finish") );
+        if (done1 | done2) {
+            printf("WRONG !!! the send is %s but the receive is %s in local_copy_with_convertor\n",
+                   (done1 ? "finish" : "not finish"), (done2 ? "finish" : "not finish"));
         }
 
         max_data = chunk;
@@ -303,31 +317,36 @@ static int local_copy_with_convertor( ompi_datatype_t* pdt, int count, int chunk
         iov.iov_base = ptemp;
         iov.iov_len = chunk;
 
-        if( done1 == 0 ) {
-            done1 = opal_convertor_pack( send_convertor, &iov, &iov_count, &max_data );
+        if (done1 == 0) {
+            done1 = opal_convertor_pack(send_convertor, &iov, &iov_count, &max_data);
         }
 
-        if( done2 == 0 ) {
-            GET_TIME( unpack_start );
-            done2 = opal_convertor_unpack( recv_convertor, &iov, &iov_count, &max_data );
-            GET_TIME( unpack_end );
-            unpack_time += ELAPSED_TIME( unpack_start, unpack_end );
+        if (done2 == 0) {
+            GET_TIME(unpack_start);
+            done2 = opal_convertor_unpack(recv_convertor, &iov, &iov_count, &max_data);
+            GET_TIME(unpack_end);
+            unpack_time += ELAPSED_TIME(unpack_start, unpack_end);
         }
 
         length += max_data;
     }
-    GET_TIME( end );
-    total_time = ELAPSED_TIME( start, end );
-    printf( "copying same data-type using convertors in %ld microsec\n", total_time );
-    printf( "\t unpack in %ld microsec [pack in %ld microsec]\n", unpack_time,
-            total_time - unpack_time );
- clean_and_return:
-    if( NULL != send_convertor ) OBJ_RELEASE( send_convertor );
-    if( NULL != recv_convertor ) OBJ_RELEASE( recv_convertor );
-
-    if( NULL != pdst ) free( pdst );
-    if( NULL != psrc ) free( psrc );
-    if( NULL != ptemp ) free( ptemp );
+    GET_TIME(end);
+    total_time = ELAPSED_TIME(start, end);
+    printf("copying same data-type using convertors in %ld microsec\n", total_time);
+    printf("\t unpack in %ld microsec [pack in %ld microsec]\n", unpack_time,
+           total_time - unpack_time);
+clean_and_return:
+    if (NULL != send_convertor)
+        OBJ_RELEASE(send_convertor);
+    if (NULL != recv_convertor)
+        OBJ_RELEASE(recv_convertor);
+
+    if (NULL != pdst)
+        free(pdst);
+    if (NULL != psrc)
+        free(psrc);
+    if (NULL != ptemp)
+        free(ptemp);
     return OMPI_SUCCESS;
 }
 
@@ -338,208 +357,229 @@ static int local_copy_with_convertor( ompi_datatype_t* pdt, int count, int chunk
  * on the data-type engine should first pass all the tests from this file, before going into other
  * tests.
  */
-int main( int argc, char* argv[] )
+int main(int argc, char *argv[])
 {
     ompi_datatype_t *pdt, *pdt1, *pdt2, *pdt3;
     int rc, length = 500;
 
-    opal_init_util(&argc, &argv);
+    opal_init(&argc, &argv);
     ompi_datatype_init();
 
     /**
      * By default simulate homogeneous architectures.
      */
     remote_arch = opal_local_arch;
-    printf( "\n\n#\n * TEST INVERSED VECTOR\n #\n\n" );
-    pdt = create_inversed_vector( &ompi_mpi_int.dt, 10 );
-    if( outputFlags & CHECK_PACK_UNPACK ) {
+    printf("\n\n#\n * TEST INVERSED VECTOR\n #\n\n");
+    pdt = create_inversed_vector(&ompi_mpi_int.dt, 10);
+    if (outputFlags & CHECK_PACK_UNPACK) {
         local_copy_ddt_count(pdt, 100);
         local_copy_with_convertor(pdt, 100, 956);
     }
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
-    printf( "\n\n#\n * TEST STRANGE DATATYPE\n #\n\n" );
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
+    printf("\n\n#\n * TEST STRANGE DATATYPE\n #\n\n");
     pdt = create_strange_dt();
-    if( outputFlags & CHECK_PACK_UNPACK ) {
+    if (outputFlags & CHECK_PACK_UNPACK) {
         local_copy_ddt_count(pdt, 1);
         local_copy_with_convertor(pdt, 1, 956);
     }
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
 
-    printf( "\n\n#\n * TEST UPPER TRIANGULAR MATRIX (size 100)\n #\n\n" );
+    printf("\n\n#\n * TEST UPPER TRIANGULAR MATRIX (size 100)\n #\n\n");
     pdt = upper_matrix(100);
-    if( outputFlags & CHECK_PACK_UNPACK ) {
+    if (outputFlags & CHECK_PACK_UNPACK) {
         local_copy_ddt_count(pdt, 1);
         local_copy_with_convertor(pdt, 1, 48);
     }
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
 
-    printf( "\n\n#\n * TEST UPPER MATRIX\n #\n\n" );
-    rc = test_upper( length );
-    if( rc == 0 )
-        printf( "decode [PASSED]\n" );
+    printf("\n\n#\n * TEST UPPER MATRIX\n #\n\n");
+    rc = test_upper(length);
+    if (rc == 0)
+        printf("decode [PASSED]\n");
     else
-        printf( "decode [NOT PASSED]\n" );
+        printf("decode [NOT PASSED]\n");
 
-    printf( "\n\n#\n * TEST MATRIX BORDERS\n #\n\n" );
-    pdt = test_matrix_borders( length, 100 );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        ompi_datatype_dump( pdt );
+    printf("\n\n#\n * TEST MATRIX BORDERS\n #\n\n");
+    pdt = test_matrix_borders(length, 100);
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        ompi_datatype_dump(pdt);
     }
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
 
-    printf( "\n\n#\n * TEST CONTIGUOUS\n #\n\n" );
+    printf("\n\n#\n * TEST CONTIGUOUS\n #\n\n");
     pdt = test_contiguous();
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
-    printf( "\n\n#\n * TEST STRUCT\n #\n\n" );
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
+    printf("\n\n#\n * TEST STRUCT\n #\n\n");
     pdt = test_struct();
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
 
     ompi_datatype_create_contiguous(0, &ompi_mpi_datatype_null.dt, &pdt1);
     ompi_datatype_create_contiguous(0, &ompi_mpi_datatype_null.dt, &pdt2);
     ompi_datatype_create_contiguous(0, &ompi_mpi_datatype_null.dt, &pdt3);
 
-    ompi_datatype_add( pdt3, &ompi_mpi_int.dt, 10, 0, -1 );
-    ompi_datatype_add( pdt3, &ompi_mpi_float.dt, 5, 10 * sizeof(int), -1 );
+    ompi_datatype_add(pdt3, &ompi_mpi_int.dt, 10, 0, -1);
+    ompi_datatype_add(pdt3, &ompi_mpi_float.dt, 5, 10 * sizeof(int), -1);
 
-    ompi_datatype_add( pdt2, &ompi_mpi_float.dt, 1, 0, -1 );
-    ompi_datatype_add( pdt2, pdt3, 3, sizeof(int) * 1, -1 );
+    ompi_datatype_add(pdt2, &ompi_mpi_float.dt, 1, 0, -1);
+    ompi_datatype_add(pdt2, pdt3, 3, sizeof(int) * 1, -1);
 
-    ompi_datatype_add( pdt1, &ompi_mpi_long_long_int.dt, 5, 0, -1 );
-    ompi_datatype_add( pdt1, &ompi_mpi_long_double.dt, 2, sizeof(long long) * 5, -1 );
+    ompi_datatype_add(pdt1, &ompi_mpi_long_long_int.dt, 5, 0, -1);
+    ompi_datatype_add(pdt1, &ompi_mpi_long_double.dt, 2, sizeof(long long) * 5, -1);
 
-    printf( ">>--------------------------------------------<<\n" );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        ompi_datatype_dump( pdt1 );
+    printf(">>--------------------------------------------<<\n");
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        ompi_datatype_dump(pdt1);
     }
-    printf( ">>--------------------------------------------<<\n" );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        ompi_datatype_dump( pdt2 );
+    printf(">>--------------------------------------------<<\n");
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        ompi_datatype_dump(pdt2);
     }
-    printf( ">>--------------------------------------------<<\n" );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        ompi_datatype_dump( pdt3 );
+    printf(">>--------------------------------------------<<\n");
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        ompi_datatype_dump(pdt3);
     }
 
-    OBJ_RELEASE( pdt1 ); assert( pdt1 == NULL );
-    OBJ_RELEASE( pdt2 ); assert( pdt2 == NULL );
-    OBJ_RELEASE( pdt3 ); assert( pdt3 == NULL );
+    OBJ_RELEASE(pdt1);
+    assert(pdt1 == NULL);
+    OBJ_RELEASE(pdt2);
+    assert(pdt2 == NULL);
+    OBJ_RELEASE(pdt3);
+    assert(pdt3 == NULL);
 
-    printf( ">>--------------------------------------------<<\n" );
-    printf( " Contiguous data-type (MPI_DOUBLE)\n" );
+    printf(">>--------------------------------------------<<\n");
+    printf(" Contiguous data-type (MPI_DOUBLE)\n");
     pdt = MPI_DOUBLE;
-    if( outputFlags & CHECK_PACK_UNPACK ) {
+    if (outputFlags & CHECK_PACK_UNPACK) {
         local_copy_ddt_count(pdt, 4500);
-        local_copy_with_convertor( pdt, 4500, 12 );
-        local_copy_with_convertor_2datatypes( pdt, 4500, pdt, 4500, 12 );
+        local_copy_with_convertor(pdt, 4500, 12);
+        local_copy_with_convertor_2datatypes(pdt, 4500, pdt, 4500, 12);
     }
-    printf( ">>--------------------------------------------<<\n" );
+    printf(">>--------------------------------------------<<\n");
 
-    printf( ">>--------------------------------------------<<\n" );
-    if( outputFlags & CHECK_PACK_UNPACK ) {
-        printf( "Contiguous multiple data-type (4500*1)\n" );
-        pdt = create_contiguous_type( MPI_DOUBLE, 4500 );
+    printf(">>--------------------------------------------<<\n");
+    if (outputFlags & CHECK_PACK_UNPACK) {
+        printf("Contiguous multiple data-type (4500*1)\n");
+        pdt = create_contiguous_type(MPI_DOUBLE, 4500);
         local_copy_ddt_count(pdt, 1);
-        local_copy_with_convertor( pdt, 1, 12 );
-        local_copy_with_convertor_2datatypes( pdt, 1, pdt, 1, 12 );
-        OBJ_RELEASE( pdt ); assert( pdt == NULL );
-        printf( "Contiguous multiple data-type (450*10)\n" );
-        pdt = create_contiguous_type( MPI_DOUBLE, 450 );
+        local_copy_with_convertor(pdt, 1, 12);
+        local_copy_with_convertor_2datatypes(pdt, 1, pdt, 1, 12);
+        OBJ_RELEASE(pdt);
+        assert(pdt == NULL);
+        printf("Contiguous multiple data-type (450*10)\n");
+        pdt = create_contiguous_type(MPI_DOUBLE, 450);
         local_copy_ddt_count(pdt, 10);
-        local_copy_with_convertor( pdt, 10, 12 );
-        local_copy_with_convertor_2datatypes( pdt, 10, pdt, 10, 12 );
-        OBJ_RELEASE( pdt ); assert( pdt == NULL );
-        printf( "Contiguous multiple data-type (45*100)\n" );
-        pdt = create_contiguous_type( MPI_DOUBLE, 45 );
+        local_copy_with_convertor(pdt, 10, 12);
+        local_copy_with_convertor_2datatypes(pdt, 10, pdt, 10, 12);
+        OBJ_RELEASE(pdt);
+        assert(pdt == NULL);
+        printf("Contiguous multiple data-type (45*100)\n");
+        pdt = create_contiguous_type(MPI_DOUBLE, 45);
         local_copy_ddt_count(pdt, 100);
-        local_copy_with_convertor( pdt, 100, 12 );
-        local_copy_with_convertor_2datatypes( pdt, 100, pdt, 100, 12 );
-        OBJ_RELEASE( pdt ); assert( pdt == NULL );
-        printf( "Contiguous multiple data-type (100*45)\n" );
-        pdt = create_contiguous_type( MPI_DOUBLE, 100 );
+        local_copy_with_convertor(pdt, 100, 12);
+        local_copy_with_convertor_2datatypes(pdt, 100, pdt, 100, 12);
+        OBJ_RELEASE(pdt);
+        assert(pdt == NULL);
+        printf("Contiguous multiple data-type (100*45)\n");
+        pdt = create_contiguous_type(MPI_DOUBLE, 100);
         local_copy_ddt_count(pdt, 45);
-        local_copy_with_convertor( pdt, 45, 12 );
-        local_copy_with_convertor_2datatypes( pdt, 45, pdt, 45, 12 );
-        OBJ_RELEASE( pdt ); assert( pdt == NULL );
-        printf( "Contiguous multiple data-type (10*450)\n" );
-        pdt = create_contiguous_type( MPI_DOUBLE, 10 );
+        local_copy_with_convertor(pdt, 45, 12);
+        local_copy_with_convertor_2datatypes(pdt, 45, pdt, 45, 12);
+        OBJ_RELEASE(pdt);
+        assert(pdt == NULL);
+        printf("Contiguous multiple data-type (10*450)\n");
+        pdt = create_contiguous_type(MPI_DOUBLE, 10);
         local_copy_ddt_count(pdt, 450);
-        local_copy_with_convertor( pdt, 450, 12 );
-        local_copy_with_convertor_2datatypes( pdt, 450, pdt, 450, 12 );
-        OBJ_RELEASE( pdt ); assert( pdt == NULL );
-        printf( "Contiguous multiple data-type (1*4500)\n" );
-        pdt = create_contiguous_type( MPI_DOUBLE, 1 );
+        local_copy_with_convertor(pdt, 450, 12);
+        local_copy_with_convertor_2datatypes(pdt, 450, pdt, 450, 12);
+        OBJ_RELEASE(pdt);
+        assert(pdt == NULL);
+        printf("Contiguous multiple data-type (1*4500)\n");
+        pdt = create_contiguous_type(MPI_DOUBLE, 1);
         local_copy_ddt_count(pdt, 4500);
-        local_copy_with_convertor( pdt, 4500, 12 );
-        local_copy_with_convertor_2datatypes( pdt, 4500, pdt, 4500, 12 );
-        OBJ_RELEASE( pdt ); assert( pdt == NULL );
+        local_copy_with_convertor(pdt, 4500, 12);
+        local_copy_with_convertor_2datatypes(pdt, 4500, pdt, 4500, 12);
+        OBJ_RELEASE(pdt);
+        assert(pdt == NULL);
     }
-    printf( ">>--------------------------------------------<<\n" );
-    printf( ">>--------------------------------------------<<\n" );
-    printf( "Vector data-type (450 times 10 double stride 11)\n" );
-    pdt = create_vector_type( MPI_DOUBLE, 450, 10, 11 );
-    ompi_datatype_dump( pdt );
-    if( outputFlags & CHECK_PACK_UNPACK ) {
+    printf(">>--------------------------------------------<<\n");
+    printf(">>--------------------------------------------<<\n");
+    printf("Vector data-type (450 times 10 double stride 11)\n");
+    pdt = create_vector_type(MPI_DOUBLE, 450, 10, 11);
+    ompi_datatype_dump(pdt);
+    if (outputFlags & CHECK_PACK_UNPACK) {
         local_copy_ddt_count(pdt, 1);
-        local_copy_with_convertor( pdt, 1, 12 );
-        local_copy_with_convertor_2datatypes( pdt, 1, pdt, 1, 12 );
-        local_copy_with_convertor( pdt, 1, 82 );
-        local_copy_with_convertor_2datatypes( pdt, 1, pdt, 1, 82 );
-        local_copy_with_convertor( pdt, 1, 6000 );
-        local_copy_with_convertor_2datatypes( pdt, 1, pdt, 1, 6000 );
-        local_copy_with_convertor( pdt, 1, 36000 );
-        local_copy_with_convertor_2datatypes( pdt, 1, pdt, 1, 36000 );
+        local_copy_with_convertor(pdt, 1, 12);
+        local_copy_with_convertor_2datatypes(pdt, 1, pdt, 1, 12);
+        local_copy_with_convertor(pdt, 1, 82);
+        local_copy_with_convertor_2datatypes(pdt, 1, pdt, 1, 82);
+        local_copy_with_convertor(pdt, 1, 6000);
+        local_copy_with_convertor_2datatypes(pdt, 1, pdt, 1, 6000);
+        local_copy_with_convertor(pdt, 1, 36000);
+        local_copy_with_convertor_2datatypes(pdt, 1, pdt, 1, 36000);
     }
-    printf( ">>--------------------------------------------<<\n" );
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
+    printf(">>--------------------------------------------<<\n");
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
 
-    printf( ">>--------------------------------------------<<\n" );
+    printf(">>--------------------------------------------<<\n");
     pdt = test_struct_char_double();
-    if( outputFlags & CHECK_PACK_UNPACK ) {
+    if (outputFlags & CHECK_PACK_UNPACK) {
         local_copy_ddt_count(pdt, 4500);
-        local_copy_with_convertor( pdt, 4500, 12 );
-        local_copy_with_convertor_2datatypes( pdt, 4500, pdt, 4500, 12 );
+        local_copy_with_convertor(pdt, 4500, 12);
+        local_copy_with_convertor_2datatypes(pdt, 4500, pdt, 4500, 12);
     }
-    printf( ">>--------------------------------------------<<\n" );
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
+    printf(">>--------------------------------------------<<\n");
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
 
-    printf( ">>--------------------------------------------<<\n" );
+    printf(">>--------------------------------------------<<\n");
     pdt = test_create_twice_two_doubles();
-    if( outputFlags & CHECK_PACK_UNPACK ) {
+    if (outputFlags & CHECK_PACK_UNPACK) {
         local_copy_ddt_count(pdt, 4500);
-        local_copy_with_convertor( pdt, 4500, 12 );
-        local_copy_with_convertor_2datatypes( pdt, 4500, pdt, 4500, 12 );
+        local_copy_with_convertor(pdt, 4500, 12);
+        local_copy_with_convertor_2datatypes(pdt, 4500, pdt, 4500, 12);
     }
-    printf( ">>--------------------------------------------<<\n" );
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
+    printf(">>--------------------------------------------<<\n");
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
 
-    printf( ">>--------------------------------------------<<\n" );
+    printf(">>--------------------------------------------<<\n");
     pdt = test_create_blacs_type();
-    if( outputFlags & CHECK_PACK_UNPACK ) {
-        ompi_datatype_dump( pdt );
+    if (outputFlags & CHECK_PACK_UNPACK) {
+        ompi_datatype_dump(pdt);
         local_copy_ddt_count(pdt, 2);
         local_copy_ddt_count(pdt, 4500);
-        local_copy_with_convertor( pdt, 4500, 956 );
-        local_copy_with_convertor_2datatypes( pdt, 4500, pdt, 4500, 956 );
-        local_copy_with_convertor( pdt, 4500, 16*1024 );
-        local_copy_with_convertor_2datatypes( pdt, 4500, pdt, 4500, 16*1024 );
-        local_copy_with_convertor( pdt, 4500, 64*1024 );
-        local_copy_with_convertor_2datatypes( pdt, 4500, pdt, 4500, 64*1024 );
+        local_copy_with_convertor(pdt, 4500, 956);
+        local_copy_with_convertor_2datatypes(pdt, 4500, pdt, 4500, 956);
+        local_copy_with_convertor(pdt, 4500, 16 * 1024);
+        local_copy_with_convertor_2datatypes(pdt, 4500, pdt, 4500, 16 * 1024);
+        local_copy_with_convertor(pdt, 4500, 64 * 1024);
+        local_copy_with_convertor_2datatypes(pdt, 4500, pdt, 4500, 64 * 1024);
     }
-    printf( ">>--------------------------------------------<<\n" );
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
-
-    printf( ">>--------------------------------------------<<\n" );
-    pdt1 = test_create_blacs_type1( &ompi_mpi_int.dt );
-    pdt2 = test_create_blacs_type2( &ompi_mpi_int.dt );
-    if( outputFlags & CHECK_PACK_UNPACK ) {
-        local_copy_with_convertor_2datatypes( pdt1, 1, pdt2, 1, 100 );
+    printf(">>--------------------------------------------<<\n");
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
+
+    printf(">>--------------------------------------------<<\n");
+    pdt1 = test_create_blacs_type1(&ompi_mpi_int.dt);
+    pdt2 = test_create_blacs_type2(&ompi_mpi_int.dt);
+    if (outputFlags & CHECK_PACK_UNPACK) {
+        local_copy_with_convertor_2datatypes(pdt1, 1, pdt2, 1, 100);
     }
-    printf( ">>--------------------------------------------<<\n" );
-    OBJ_RELEASE( pdt1 ); assert( pdt1 == NULL );
-    OBJ_RELEASE( pdt2 ); assert( pdt2 == NULL );
+    printf(">>--------------------------------------------<<\n");
+    OBJ_RELEASE(pdt1);
+    assert(pdt1 == NULL);
+    OBJ_RELEASE(pdt2);
+    assert(pdt2 == NULL);
 
     /* clean-ups all data allocations */
-    ompi_datatype_finalize();
+    opal_finalize_util();
 
     return OMPI_SUCCESS;
 }
diff --git a/test/datatype/external32.c b/test/datatype/external32.c
index d09938510ba..9ef83afac57 100644
--- a/test/datatype/external32.c
+++ b/test/datatype/external32.c
@@ -15,56 +15,74 @@
 
 #include "ompi_config.h"
 #include "ompi/datatype/ompi_datatype.h"
-#include "opal/runtime/opal.h"
 #include "opal/datatype/opal_convertor.h"
 #include "opal/datatype/opal_datatype_internal.h"
+#include "opal/runtime/opal.h"
 #include <arpa/inet.h>
 
 static int verbose = 0;
- 
-typedef int (*checker_t)(void*, void*, ompi_datatype_t *, int, void*);
 
-int check_contiguous( void* send_buffer, void* packed,
-                      ompi_datatype_t * datatype, int count, void* arg );
+typedef int (*checker_t)(void *, void *, ompi_datatype_t *, int, void *);
 
-int check_vector( void* send_buffer, void* packed,
-                  ompi_datatype_t * datatype, int count, void* arg );
+int check_contiguous(void *send_buffer, void *packed, ompi_datatype_t *datatype, int count,
+                     void *arg);
 
-static int pack_unpack_datatype( void* send_data, ompi_datatype_t *datatype, int count,
-                                 void* recv_data, checker_t validator, void *validator_arg );
+int check_vector(void *send_buffer, void *packed, ompi_datatype_t *datatype, int count, void *arg);
 
-static void dump_hex(void* what, size_t length);
+static int pack_unpack_datatype(void *send_data, ompi_datatype_t *datatype, int count,
+                                void *recv_data, checker_t validator, void *validator_arg);
 
-static void dump_hex(void* what, size_t length)
+static void dump_hex(const char *msg, const void *vbuf, int nbytes, int start_from, int stop_at,
+                     int vals_per_line)
 {
-    size_t i;
-    for( i = 0; i < length; i++ ) {
-        printf("%02x", (unsigned int)(((unsigned char*)what)[i]));
+    const char *buf = (const char *) vbuf;
+
+    if (-1 == stop_at)
+        stop_at = nbytes;
+
+    for (int i = (start_from / vals_per_line) * vals_per_line; i < nbytes; ++i) {
+        if (i >= stop_at)
+            return;
+        if (0 == (i % vals_per_line)) {
+            if (NULL == msg)
+                printf("\n");
+            else
+                printf("\n%s", msg);
+        } else {
+            if (i % 4 == 0) {
+                printf("  ");
+            }
+        }
+        printf(" ");
+        if (i < start_from)
+            printf("  ");
+        else
+            printf("%02x", *((unsigned char *) (buf + i)));
     }
 }
 
-int check_contiguous( void* send_buffer, void* packed,
-                      ompi_datatype_t* datatype, int count, void* arg )
+int check_contiguous(void *send_buffer, void *packed, ompi_datatype_t *datatype, int count,
+                     void *arg)
 {
     int i;
 
-    if( (datatype == &ompi_mpi_int.dt) || (datatype == &ompi_mpi_int32_t.dt) ) {
+    if ((datatype == &ompi_mpi_int.dt) || (datatype == &ompi_mpi_int32_t.dt)) {
         uint32_t val;
-        for( i = 0 ; i < count; i++ ) {
-            val = htonl(((uint32_t*)send_buffer)[i]);
-            if( val != ((uint32_t*)packed)[i] ) {
-                printf("Error at position %d expected %x found %x (type %s)\n",
-                       i, ((uint32_t*)packed)[i], ((uint32_t*)send_buffer)[i], datatype->name);
+        for (i = 0; i < count; i++) {
+            val = htonl(((uint32_t *) send_buffer)[i]);
+            if (val != ((uint32_t *) packed)[i]) {
+                printf("Error at position %d expected %x found %x (type %s)\n", i,
+                       ((uint32_t *) packed)[i], ((uint32_t *) send_buffer)[i], datatype->name);
                 return -1;
             }
         }
-    } else if( (datatype == &ompi_mpi_short.dt) || (datatype == &ompi_mpi_int16_t.dt) ) {
+    } else if ((datatype == &ompi_mpi_short.dt) || (datatype == &ompi_mpi_int16_t.dt)) {
         uint16_t val;
-        for( i = 0 ; i < count; i++ ) {
-            val = htons(((uint16_t*)send_buffer)[i]);
-            if( val != ((uint16_t*)packed)[i] ) {
-                printf("Error at position %d expected %x found %x (type %s)\n",
-                       i, ((uint16_t*)packed)[i], ((uint16_t*)send_buffer)[i], datatype->name);
+        for (i = 0; i < count; i++) {
+            val = htons(((uint16_t *) send_buffer)[i]);
+            if (val != ((uint16_t *) packed)[i]) {
+                printf("Error at position %d expected %x found %x (type %s)\n", i,
+                       ((uint16_t *) packed)[i], ((uint16_t *) send_buffer)[i], datatype->name);
                 return -1;
             }
         }
@@ -75,29 +93,28 @@ int check_contiguous( void* send_buffer, void* packed,
     return 0;
 }
 
-int check_vector( void* send_buffer, void* packed,
-                  ompi_datatype_t* datatype, int count, void* arg )
+int check_vector(void *send_buffer, void *packed, ompi_datatype_t *datatype, int count, void *arg)
 {
     int i;
-    ompi_datatype_t *origtype = (ompi_datatype_t *)arg;
+    ompi_datatype_t *origtype = (ompi_datatype_t *) arg;
 
-    if( (origtype == &ompi_mpi_int.dt) || (origtype == &ompi_mpi_int32_t.dt) ) {
+    if ((origtype == &ompi_mpi_int.dt) || (origtype == &ompi_mpi_int32_t.dt)) {
         uint32_t val;
-        for( i = 0 ; i < count; i++ ) {
-            val = htonl(((uint32_t*)send_buffer)[2*i]);
-            if( val != ((uint32_t*)packed)[i] ) {
-                printf("Error at position %d expected %x found %x (type %s)\n",
-                       i, ((uint32_t*)packed)[i], ((uint32_t*)send_buffer)[2*i], datatype->name);
+        for (i = 0; i < count; i++) {
+            val = htonl(((uint32_t *) send_buffer)[2 * i]);
+            if (val != ((uint32_t *) packed)[i]) {
+                printf("Error at position %d expected %x found %x (type %s)\n", i,
+                       ((uint32_t *) packed)[i], ((uint32_t *) send_buffer)[2 * i], datatype->name);
                 return -1;
             }
         }
-    } else if( (origtype == &ompi_mpi_short.dt) || (origtype == &ompi_mpi_int16_t.dt) ) {
+    } else if ((origtype == &ompi_mpi_short.dt) || (origtype == &ompi_mpi_int16_t.dt)) {
         uint16_t val;
-        for( i = 0 ; i < count; i++ ) {
-            val = htons(((uint16_t*)send_buffer)[2*i]);
-            if( val != ((uint16_t*)packed)[i] ) {
-                printf("Error at position %d expected %x found %x (type %s)\n",
-                       i, ((uint16_t*)packed)[i], ((uint16_t*)send_buffer)[2*i], datatype->name);
+        for (i = 0; i < count; i++) {
+            val = htons(((uint16_t *) send_buffer)[2 * i]);
+            if (val != ((uint16_t *) packed)[i]) {
+                printf("Error at position %d expected %x found %x (type %s)\n", i,
+                       ((uint16_t *) packed)[i], ((uint16_t *) send_buffer)[2 * i], datatype->name);
                 return -1;
             }
         }
@@ -108,44 +125,50 @@ int check_vector( void* send_buffer, void* packed,
     return 0;
 }
 
-static int pack_unpack_datatype( void* send_data, ompi_datatype_t *datatype, int count,
-                                 void* recv_data,
-                                 checker_t validator, void* validator_arg)
+static int pack_unpack_datatype(void *send_data, ompi_datatype_t *datatype, int count,
+                                void *recv_data, checker_t validator, void *validator_arg)
 {
     MPI_Aint position = 0, buffer_size;
-    void* buffer;
+    void *buffer;
     int error;
 
-    error = ompi_datatype_pack_external_size("external32",
-                                             count, datatype, &buffer_size);
-    if( MPI_SUCCESS != error ) goto return_error_code;
+    error = ompi_datatype_pack_external_size("external32", count, datatype, &buffer_size);
+    if (MPI_SUCCESS != error)
+        goto return_error_code;
 
-    buffer = (void*)malloc(buffer_size);
-    if( NULL == buffer ) { error = MPI_ERR_UNKNOWN; goto return_error_code; }
+    buffer = (void *) malloc(buffer_size);
+    if (NULL == buffer) {
+        error = MPI_ERR_UNKNOWN;
+        goto return_error_code;
+    }
 
-    error = ompi_datatype_pack_external("external32", (void*)send_data, count, datatype,
-                                        buffer, buffer_size, &position);
-    if( MPI_SUCCESS != error ) goto return_error_code;
-    if( 0 != validator(send_data, buffer, datatype, count, validator_arg) ) {
+    error = ompi_datatype_pack_external("external32", (void *) send_data, count, datatype, buffer,
+                                        buffer_size, &position);
+    if (MPI_SUCCESS != error)
+        goto return_error_code;
+    if (0 != validator(send_data, buffer, datatype, count, validator_arg)) {
         printf("Error during pack external. Bailing out\n");
         return -1;
     }
 
-    printf("packed %ld bytes into a %ld bytes buffer ", position, buffer_size); dump_hex(buffer, position); printf("\n");
+    printf("packed %ld bytes into a %ld bytes buffer ", position, buffer_size);
+    dump_hex(NULL, buffer, position, 0, -1, 24);
+    printf("\n");
 
     position = 0;
-    error = ompi_datatype_unpack_external("external32", buffer, buffer_size, &position,
-                                          recv_data, count, datatype);
-    if( MPI_SUCCESS != error ) goto return_error_code;
+    error = ompi_datatype_unpack_external("external32", buffer, buffer_size, &position, recv_data,
+                                          count, datatype);
+    if (MPI_SUCCESS != error)
+        goto return_error_code;
     free(buffer);
 
- return_error_code:
+return_error_code:
     return (error == MPI_SUCCESS ? 0 : -1);
 }
 
 int main(int argc, char *argv[])
 {
-    opal_init_util(&argc, &argv);
+    opal_init(&argc, &argv);
     ompi_datatype_init();
 
     /* Simple contiguous data: MPI_INT32_T */
@@ -153,17 +176,21 @@ int main(int argc, char *argv[])
         int32_t send_data[2] = {1234, 5678};
         int32_t recv_data[2] = {-1, -1};
 
-        if( verbose ) {
+        if (verbose) {
             printf("send data %08x %08x \n", send_data[0], send_data[1]);
-            printf("data "); dump_hex(&send_data, sizeof(int32_t) * 2); printf("\n");
+            printf("data ");
+            dump_hex(NULL, &send_data, sizeof(int32_t) * 2, 0, -1, 24);
+            printf("\n");
         }
-        (void)pack_unpack_datatype( send_data, &ompi_mpi_int32_t.dt, 2,
-                                    recv_data, check_contiguous, (void*)&ompi_mpi_int32_t.dt );
-        if( verbose ) {
-            printf("recv "); dump_hex(&recv_data, sizeof(int32_t) * 2); printf("\n");
+        (void) pack_unpack_datatype(send_data, &ompi_mpi_int32_t.dt, 2, recv_data, check_contiguous,
+                                    (void *) &ompi_mpi_int32_t.dt);
+        if (verbose) {
+            printf("recv ");
+            dump_hex(NULL, &recv_data, sizeof(int32_t) * 2, 0, -1, 24);
+            printf("\n");
             printf("recv data %08x %08x \n", recv_data[0], recv_data[1]);
         }
-        if( (send_data[0] != recv_data[0]) || (send_data[1] != recv_data[1]) ) {
+        if ((send_data[0] != recv_data[0]) || (send_data[1] != recv_data[1])) {
             printf("Error during external32 pack/unack for contiguous types (MPI_INT32_T)\n");
             exit(-1);
         }
@@ -173,17 +200,21 @@ int main(int argc, char *argv[])
         int16_t send_data[2] = {1234, 5678};
         int16_t recv_data[2] = {-1, -1};
 
-        if( verbose ) {
+        if (verbose) {
             printf("send data %08x %08x \n", send_data[0], send_data[1]);
-            printf("data "); dump_hex(&send_data, sizeof(int16_t) * 2); printf("\n");
+            printf("data ");
+            dump_hex(NULL, &send_data, sizeof(int16_t) * 2, 0, -1, 24);
+            printf("\n");
         }
-        (void)pack_unpack_datatype( send_data, &ompi_mpi_int16_t.dt, 2,
-                                    recv_data, check_contiguous, (void*)&ompi_mpi_int16_t.dt );
-        if( verbose ) {
-            printf("recv "); dump_hex(&recv_data, sizeof(int16_t) * 2); printf("\n");
+        (void) pack_unpack_datatype(send_data, &ompi_mpi_int16_t.dt, 2, recv_data, check_contiguous,
+                                    (void *) &ompi_mpi_int16_t.dt);
+        if (verbose) {
+            printf("recv ");
+            dump_hex(NULL, &recv_data, sizeof(int16_t) * 2, 0, -1, 24);
+            printf("\n");
             printf("recv data %08x %08x \n", recv_data[0], recv_data[1]);
         }
-        if( (send_data[0] != recv_data[0]) || (send_data[1] != recv_data[1]) ) {
+        if ((send_data[0] != recv_data[0]) || (send_data[1] != recv_data[1])) {
             printf("Error during external32 pack/unack for contiguous types\n");
             exit(-1);
         }
@@ -192,37 +223,44 @@ int main(int argc, char *argv[])
     /* Vector datatype */
     printf("\n\nVector datatype\n\n");
     {
-        int count=2, blocklength=1, stride=2;
+        int count = 2, blocklength = 1, stride = 2;
         int send_data[3] = {1234, 0, 5678};
         int recv_data[3] = {-1, -1, -1};
         ompi_datatype_t *ddt;
 
-        ompi_datatype_create_vector ( count, blocklength, stride, &ompi_mpi_int.dt, &ddt );
+        ompi_datatype_create_vector(count, blocklength, stride, &ompi_mpi_int.dt, &ddt);
         {
-            const int* a_i[3] = {&count, &blocklength, &stride};
+            const int *a_i[3] = {&count, &blocklength, &stride};
             ompi_datatype_t *type = &ompi_mpi_int.dt;
 
-            ompi_datatype_set_args( ddt, 3, a_i, 0, NULL, 1, &type, MPI_COMBINER_VECTOR );
+            ompi_datatype_set_args(ddt, 3, a_i, 0, NULL, 1, &type, MPI_COMBINER_VECTOR);
         }
         ompi_datatype_commit(&ddt);
 
-        if( verbose ) {
+        if (verbose) {
             printf("send data %08x %x08x %08x \n", send_data[0], send_data[1], send_data[2]);
-            printf("data "); dump_hex(&send_data, sizeof(int32_t) * 3); printf("\n");
+            printf("data ");
+            dump_hex(NULL, &send_data, sizeof(int32_t) * 3, 0, -1, 24);
+            printf("\n");
         }
-        (void)pack_unpack_datatype( send_data, ddt, 1, recv_data, check_vector, (void*)&ompi_mpi_int32_t.dt );
-        if( verbose ) {
-            printf("recv "); dump_hex(&recv_data, sizeof(int32_t) * 3); printf("\n");
+        (void) pack_unpack_datatype(send_data, ddt, 1, recv_data, check_vector,
+                                    (void *) &ompi_mpi_int32_t.dt);
+        if (verbose) {
+            printf("recv ");
+            dump_hex(NULL, &recv_data, sizeof(int32_t) * 3, 0, -1, 24);
+            printf("\n");
             printf("recv data %08x %08x %08x \n", recv_data[0], recv_data[1], recv_data[2]);
         }
         ompi_datatype_destroy(&ddt);
-        if( (send_data[0] != recv_data[0]) || (send_data[2] != recv_data[2]) ) {
+        if ((send_data[0] != recv_data[0]) || (send_data[2] != recv_data[2])) {
             printf("Error during external32 pack/unack for vector types (MPI_INT32_T)\n");
+            printf("[0]: %d ? %d  |  [2]: %d ? %d  ([1]: %d ? %d)\n", send_data[0], recv_data[0],
+                   send_data[2], recv_data[2], send_data[1], recv_data[1]);
             exit(-1);
         }
     }
 
-    ompi_datatype_finalize();
+    opal_finalize_util();
 
     return 0;
 }
diff --git a/test/datatype/large_data.c b/test/datatype/large_data.c
index 1c031b36629..5558d6a6455 100644
--- a/test/datatype/large_data.c
+++ b/test/datatype/large_data.c
@@ -16,75 +16,73 @@
  */
 
 #include <mpi.h>
-#include <stdio.h>
 #include <stddef.h>
+#include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
 #include "ompi_config.h"
 #include "ompi/datatype/ompi_datatype.h"
-#include "opal/runtime/opal.h"
 #include "opal/datatype/opal_convertor.h"
 #include "opal/datatype/opal_datatype_internal.h"
+#include "opal/runtime/opal.h"
 
 #define MAX_IOVEC 10
-#define MAX_CHUNK (1024*1024*1024)  /* 1GB */
+#define MAX_CHUNK (1024 * 1024 * 1024) /* 1GB */
 
 static int verbose = 0;
 
-static size_t
-count_length_via_convertor_raw(char* msg,
-                               MPI_Datatype dtype, int count)
+static size_t count_length_via_convertor_raw(char *msg, MPI_Datatype dtype, int count)
 {
-    opal_convertor_t* pconv;
+    opal_convertor_t *pconv;
     struct iovec iov[MAX_IOVEC];
     uint32_t iov_count = MAX_IOVEC, i;
     size_t length = MAX_CHUNK, packed_iovec = 0, packed = 0;
 
-    pconv = opal_convertor_create( opal_local_arch, 0 );
-    opal_convertor_prepare_for_send(pconv, (const struct opal_datatype_t *)dtype, 1, NULL);
-    while( 0 == opal_convertor_raw(pconv, iov, &iov_count, &length) ) {
-        if( verbose ) {
-            printf("iov_count = %d packed_iovec = %"PRIsize_t" length = %"PRIsize_t"\n",
+    pconv = opal_convertor_create(opal_local_arch, 0);
+    opal_convertor_prepare_for_send(pconv, (const struct opal_datatype_t *) dtype, 1, NULL);
+    while (0 == opal_convertor_raw(pconv, iov, &iov_count, &length)) {
+        if (verbose) {
+            printf("iov_count = %d packed_iovec = %" PRIsize_t " length = %" PRIsize_t "\n",
                    iov_count, packed_iovec, length);
         }
         packed += length;
-        for( i = 0; i < iov_count; i++ ) {
+        for (i = 0; i < iov_count; i++) {
             packed_iovec += iov[i].iov_len;
-            if( verbose ) {
-                printf("[%s] add %"PRIsize_t" bytes -> so far %"PRIsize_t" bytes\n",
-                       msg, iov[i].iov_len, packed_iovec);
+            if (verbose) {
+                printf("[%s] add %" PRIsize_t " bytes -> so far %" PRIsize_t " bytes\n", msg,
+                       iov[i].iov_len, packed_iovec);
             }
         }
-        if( packed != packed_iovec ) {
-            printf( "[%s] Raw data amount diverges %"PRIsize_t" != %"PRIsize_t"\n",
-                    msg, packed, packed_iovec);
+        if (packed != packed_iovec) {
+            printf("[%s] Raw data amount diverges %" PRIsize_t " != %" PRIsize_t "\n", msg, packed,
+                   packed_iovec);
             exit(-1);
         }
-        iov_count = MAX_IOVEC;  /* number of available iov */
+        iov_count = MAX_IOVEC; /* number of available iov */
         length = MAX_CHUNK;
     }
-    if( verbose ) {
-        printf("iov_count = %d packed_iovec = %"PRIsize_t" length = %"PRIsize_t"\n",
-               iov_count, packed_iovec, length);
+    if (verbose) {
+        printf("iov_count = %d packed_iovec = %" PRIsize_t " length = %" PRIsize_t "\n", iov_count,
+               packed_iovec, length);
     }
     packed += length;
-    for( i = 0; i < iov_count; i++ ) {
+    for (i = 0; i < iov_count; i++) {
         packed_iovec += iov[i].iov_len;
-        if( verbose ) {
-            printf("[%s] add %"PRIsize_t" bytes -> so far %"PRIsize_t" bytes\n",
-                   msg, iov[i].iov_len, packed_iovec);
+        if (verbose) {
+            printf("[%s] add %" PRIsize_t " bytes -> so far %" PRIsize_t " bytes\n", msg,
+                   iov[i].iov_len, packed_iovec);
         }
     }
-    if( packed != packed_iovec ) {
-        printf( "[%s] Raw data amount diverges %"PRIsize_t" != %"PRIsize_t"\n",
-                msg, packed, packed_iovec);
+    if (packed != packed_iovec) {
+        printf("[%s] Raw data amount diverges %" PRIsize_t " != %" PRIsize_t "\n", msg, packed,
+               packed_iovec);
         exit(-1);
     }
     return packed_iovec;
 }
 
-int main(int argc, char * argv[])
+int main(int argc, char *argv[])
 {
 
     int const per_process = 192;
@@ -92,17 +90,17 @@ int main(int argc, char * argv[])
     int blocklen, stride, count;
 
     int scounts[2] = {per_process, per_process};
-    int sdispls[2] = {3*per_process, 0*per_process};
+    int sdispls[2] = {3 * per_process, 0 * per_process};
     int rcounts[2] = {per_process, per_process};
-    int rdispls[2] = {1*per_process, 2*per_process};
+    int rdispls[2] = {1 * per_process, 2 * per_process};
 
     MPI_Datatype ddt, stype, rtype;
     size_t length, packed;
 
-    opal_init_util(&argc, &argv);
+    opal_init(&argc, &argv);
     ompi_datatype_init();
 
-    ompi_datatype_create_contiguous( per_type, MPI_FLOAT, &ddt);
+    ompi_datatype_create_contiguous(per_type, MPI_FLOAT, &ddt);
 
     /*
      * Large sparse datatype: indexed contiguous
@@ -112,8 +110,9 @@ int main(int argc, char * argv[])
 
     packed = count_length_via_convertor_raw("1. INDEX", stype, 1);
     opal_datatype_type_size(&stype->super, &length);
-    if( length != packed ) {
-        printf("Mismatched length of packed data to datatype size (%"PRIsize_t" != %"PRIsize_t")\n",
+    if (length != packed) {
+        printf("Mismatched length of packed data to datatype size (%" PRIsize_t " != %" PRIsize_t
+               ")\n",
                packed, length);
         exit(-2);
     }
@@ -127,8 +126,9 @@ int main(int argc, char * argv[])
 
     packed = count_length_via_convertor_raw("2. INDEX", rtype, 1);
     opal_datatype_type_size(&rtype->super, &length);
-    if( length != packed ) {
-        printf("Mismatched length of packed data to datatype size (%"PRIsize_t" != %"PRIsize_t")\n",
+    if (length != packed) {
+        printf("Mismatched length of packed data to datatype size (%" PRIsize_t " != %" PRIsize_t
+               ")\n",
                packed, length);
         exit(-2);
     }
@@ -145,8 +145,9 @@ int main(int argc, char * argv[])
 
     packed = count_length_via_convertor_raw("3. VECTOR", ddt, 1);
     opal_datatype_type_size(&ddt->super, &length);
-    if( length != packed ) {
-        printf("Mismatched length of packed data to datatype size (%"PRIsize_t" != %"PRIsize_t")\n",
+    if (length != packed) {
+        printf("Mismatched length of packed data to datatype size (%" PRIsize_t " != %" PRIsize_t
+               ")\n",
                packed, length);
         exit(-2);
     }
@@ -162,8 +163,9 @@ int main(int argc, char * argv[])
 
     packed = count_length_via_convertor_raw("4. CONTIG", ddt, 1);
     opal_datatype_type_size(&ddt->super, &length);
-    if( length != packed ) {
-        printf("Mismatched length of packed data to datatype size (%"PRIsize_t" != %"PRIsize_t")\n",
+    if (length != packed) {
+        printf("Mismatched length of packed data to datatype size (%" PRIsize_t " != %" PRIsize_t
+               ")\n",
                packed, length);
         exit(-2);
     }
diff --git a/test/datatype/opal_datatype_test.c b/test/datatype/opal_datatype_test.c
index ab5c9484fa3..06f3dd12695 100644
--- a/test/datatype/opal_datatype_test.c
+++ b/test/datatype/opal_datatype_test.c
@@ -24,26 +24,28 @@
  */
 
 #include "opal_config.h"
-#include "opal_ddt_lib.h"
-#include "opal/runtime/opal.h"
+#include "opal/datatype/opal_convertor.h"
 #include "opal/datatype/opal_datatype.h"
 #include "opal/datatype/opal_datatype_internal.h"
-#include "opal/datatype/opal_convertor.h"
-#include <time.h>
+#include "opal/runtime/opal.h"
+#include "opal_ddt_lib.h"
 #include <stdlib.h>
+#include <time.h>
 #ifdef HAVE_SYS_TIME_H
-#include <sys/time.h>
+#    include <sys/time.h>
 #endif
 #include <stdio.h>
 #include <string.h>
 
 /* Compile with:
-gcc -DHAVE_CONFIG_H -I. -I../../include -I../.. -I../../include -I../../../ompi-trunk/opal -I../../../ompi-trunk/orte -g opal_datatype_test.c -o opal_datatype_test
+gcc -DHAVE_CONFIG_H -I. -I../../include -I../.. -I../../include -I../../../ompi-trunk/opal
+-I../../../ompi-trunk/orte -g opal_datatype_test.c -o opal_datatype_test
 */
 
 #define TIMER_DATA_TYPE struct timeval
-#define GET_TIME(TV)   gettimeofday( &(TV), NULL )
-#define ELAPSED_TIME(TSTART, TEND)  (((TEND).tv_sec - (TSTART).tv_sec) * 1000000 + ((TEND).tv_usec - (TSTART).tv_usec))
+#define GET_TIME(TV)    gettimeofday(&(TV), NULL)
+#define ELAPSED_TIME(TSTART, TEND) \
+    (((TEND).tv_sec - (TSTART).tv_sec) * 1000000 + ((TEND).tv_usec - (TSTART).tv_usec))
 
 uint32_t remote_arch = 0xffffffff;
 
@@ -51,21 +53,21 @@ uint32_t remote_arch = 0xffffffff;
  * Generic function computing the amount of memory to be allocated to fit
  * a contiguous memory layout of count times pdt.
  */
-static size_t compute_memory_size( opal_datatype_t const * const pdt, int count )
+static size_t compute_memory_size(opal_datatype_t const *const pdt, int count)
 {
     ptrdiff_t extent, true_lb, true_extent;
 
-    opal_datatype_type_extent( pdt, &extent );
-    opal_datatype_get_true_extent( pdt, &true_lb, &true_extent );
+    opal_datatype_type_extent(pdt, &extent);
+    opal_datatype_get_true_extent(pdt, &true_lb, &true_extent);
 
-    return (size_t)(true_lb + true_extent + (count-1) * extent);
+    return (size_t)(true_lb + true_extent + (count - 1) * extent);
 }
 
-static int test_upper( unsigned int length )
+static int test_upper(unsigned int length)
 {
     double *mat1, *mat2, *inbuf;
     opal_datatype_t *pdt;
-    opal_convertor_t * pConv;
+    opal_convertor_t *pConv;
     char *ptr;
     int rc;
     unsigned int i, j, iov_count, split_chunk, total_length;
@@ -75,64 +77,64 @@ static int test_upper( unsigned int length )
     long total_time;
 
     /*printf( "test upper matrix\n" );*/
-    pdt = upper_matrix( length );
+    pdt = upper_matrix(length);
     /*opal_datatype_dump( pdt );*/
 
-    mat1 = malloc( length * length * sizeof(double) );
-    init_random_upper_matrix( length, mat1 );
-    mat2 = calloc( length * length, sizeof(double) );
+    mat1 = malloc(length * length * sizeof(double));
+    init_random_upper_matrix(length, mat1);
+    mat2 = calloc(length * length, sizeof(double));
 
-    total_length = length * (length + 1) * ( sizeof(double) / 2);
-    inbuf = (double*)malloc( total_length );
-    ptr = (char*)inbuf;
+    total_length = length * (length + 1) * (sizeof(double) / 2);
+    inbuf = (double *) malloc(total_length);
+    ptr = (char *) inbuf;
     /* copy upper matrix in the array simulating the input buffer */
-    for( i = 0; i < length; i++ ) {
+    for (i = 0; i < length; i++) {
         uint32_t pos = i * length + i;
-        for( j = i; j < length; j++, pos++ ) {
+        for (j = i; j < length; j++, pos++) {
             *inbuf = mat1[pos];
             inbuf++;
         }
     }
-    inbuf = (double*)ptr;
-    pConv = opal_convertor_create( remote_arch, 0 );
-    if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( pConv, pdt, 1, mat2 ) ) {
-        printf( "Cannot attach the datatype to a convertor\n" );
+    inbuf = (double *) ptr;
+    pConv = opal_convertor_create(remote_arch, 0);
+    if (OPAL_SUCCESS != opal_convertor_prepare_for_recv(pConv, pdt, 1, mat2)) {
+        printf("Cannot attach the datatype to a convertor\n");
         return OPAL_ERROR;
     }
 
-    GET_TIME( start );
+    GET_TIME(start);
     split_chunk = (length + 1) * sizeof(double);
     /*    split_chunk = (total_length + 1) * sizeof(double); */
-    for( i = total_length; i > 0; ) {
-        if( i <= split_chunk ) {  /* equal test just to be able to set a breakpoint */
+    for (i = total_length; i > 0;) {
+        if (i <= split_chunk) { /* equal test just to be able to set a breakpoint */
             split_chunk = i;
         }
         a.iov_base = ptr;
         a.iov_len = split_chunk;
         iov_count = 1;
         max_data = split_chunk;
-        opal_convertor_unpack( pConv, &a, &iov_count, &max_data );
+        opal_convertor_unpack(pConv, &a, &iov_count, &max_data);
         ptr += max_data;
         i -= max_data;
-        if( mat2[0] != inbuf[0] ) assert(0);
+        if (mat2[0] != inbuf[0])
+            assert(0);
     }
-    GET_TIME( end );
-    total_time = ELAPSED_TIME( start, end );
-    printf( "complete unpacking in %ld microsec\n", total_time );
-    free( inbuf );
-    rc = check_diag_matrix( length, mat1, mat2 );
-    free( mat1 );
-    free( mat2 );
+    GET_TIME(end);
+    total_time = ELAPSED_TIME(start, end);
+    printf("complete unpacking in %ld microsec\n", total_time);
+    free(inbuf);
+    rc = check_diag_matrix(length, mat1, mat2);
+    free(mat1);
+    free(mat2);
 
     /* test the automatic destruction pf the data */
-    opal_datatype_destroy( &pdt );
-    assert( pdt == NULL );
+    opal_datatype_destroy(&pdt);
+    assert(pdt == NULL);
 
-    OBJ_RELEASE( pConv );
+    OBJ_RELEASE(pConv);
     return rc;
 }
 
-
 /**
  * Conversion function. They deal with data-types in 3 ways, always making local copies.
  * In order to allow performance testings, there are 3 functions:
@@ -142,7 +144,7 @@ static int test_upper( unsigned int length )
  *  - and one using 2 convertors created from different data-types.
  *
  */
-static int local_copy_ddt_count( opal_datatype_t const * const pdt, int count )
+static int local_copy_ddt_count(opal_datatype_t const *const pdt, int count)
 {
     ptrdiff_t lb, extent;
     size_t malloced_size;
@@ -153,57 +155,59 @@ static int local_copy_ddt_count( opal_datatype_t const * const pdt, int count )
     int errors = 0;
 
     malloced_size = compute_memory_size(pdt, count);
-    opal_datatype_get_extent( pdt, &lb, &extent );
+    opal_datatype_get_extent(pdt, &lb, &extent);
 
-    odst = (char*)malloc( malloced_size );
-    osrc = (char*)malloc( malloced_size );
+    odst = (char *) malloc(malloced_size);
+    osrc = (char *) malloc(malloced_size);
 
     {
-        for( size_t i = 0; i < malloced_size; i++ ) osrc[i] = i % 128 + 32;
+        for (size_t i = 0; i < malloced_size; i++)
+            osrc[i] = i % 128 + 32;
         memcpy(odst, osrc, malloced_size);
     }
     pdst = odst - lb;
     psrc = osrc - lb;
 
-    cache_trash();  /* make sure the cache is useless */
-
-    GET_TIME( start );
-    if( OPAL_SUCCESS != opal_datatype_copy_content_same_ddt( pdt, count, pdst, psrc ) ) {
-        printf( "Unable to copy the datatype in the function local_copy_ddt_count."
-                " Is the datatype committed ?\n" );
-    }
-    GET_TIME( end );
-    total_time = ELAPSED_TIME( start, end );
-    printf( "direct local copy in %ld microsec\n", total_time );
-    if(outputFlags & VALIDATE_DATA) {
-        for( size_t i = 0; i < malloced_size; i++ ) {
-            if( odst[i] != osrc[i] ) {
-                printf("error at position %lu (%d != %d)\n",
-                       (unsigned long)i, (int)(odst[i]), (int)(osrc[i]));
+    cache_trash(); /* make sure the cache is useless */
+
+    GET_TIME(start);
+    if (OPAL_SUCCESS != opal_datatype_copy_content_same_ddt(pdt, count, pdst, psrc)) {
+        printf("Unable to copy the datatype in the function local_copy_ddt_count."
+               " Is the datatype committed ?\n");
+    }
+    GET_TIME(end);
+    total_time = ELAPSED_TIME(start, end);
+    printf("direct local copy in %ld microsec\n", total_time);
+    if (outputFlags & VALIDATE_DATA) {
+        for (size_t i = 0; i < malloced_size; i++) {
+            if (odst[i] != osrc[i]) {
+                printf("error at position %lu (%d != %d)\n", (unsigned long) i, (int) (odst[i]),
+                       (int) (osrc[i]));
                 errors++;
-                if(outputFlags & QUIT_ON_FIRST_ERROR) {
+                if (outputFlags & QUIT_ON_FIRST_ERROR) {
                     opal_datatype_dump(pdt);
-                    assert(0); exit(-1);
+                    assert(0);
+                    exit(-1);
                 }
             }
         }
-        if( 0 == errors ) {
+        if (0 == errors) {
             printf("Validation check successfully passed\n");
         } else {
             printf("Found %d errors. Giving up!\n", errors);
             exit(-1);
         }
     }
-    free( odst );
-    free( osrc );
+    free(odst);
+    free(osrc);
 
     return (0 == errors ? OPAL_SUCCESS : errors);
 }
 
-static int
-local_copy_with_convertor_2datatypes( opal_datatype_t const * const send_type, int send_count,
-                                      opal_datatype_t const * const recv_type, int recv_count,
-                                      int chunk )
+static int local_copy_with_convertor_2datatypes(opal_datatype_t const *const send_type,
+                                                int send_count,
+                                                opal_datatype_t const *const recv_type,
+                                                int recv_count, int chunk)
 {
     ptrdiff_t send_lb, send_extent, recv_lb, recv_extent;
     void *pdst = NULL, *psrc = NULL, *ptemp = NULL;
@@ -211,7 +215,8 @@ local_copy_with_convertor_2datatypes( opal_datatype_t const * const send_type, i
     opal_convertor_t *send_convertor = NULL, *recv_convertor = NULL;
     struct iovec iov;
     uint32_t iov_count;
-    size_t max_data, length = 0, send_malloced_size, recv_malloced_size;;
+    size_t max_data, length = 0, send_malloced_size, recv_malloced_size;
+    ;
     int32_t done1 = 0, done2 = 0;
     TIMER_DATA_TYPE start, end, unpack_start, unpack_end;
     long total_time, unpack_time = 0;
@@ -219,42 +224,44 @@ local_copy_with_convertor_2datatypes( opal_datatype_t const * const send_type, i
     send_malloced_size = compute_memory_size(send_type, send_count);
     recv_malloced_size = compute_memory_size(recv_type, recv_count);
 
-    opal_datatype_get_extent( send_type, &send_lb, &send_extent );
-    opal_datatype_get_extent( recv_type, &recv_lb, &recv_extent );
+    opal_datatype_get_extent(send_type, &send_lb, &send_extent);
+    opal_datatype_get_extent(recv_type, &recv_lb, &recv_extent);
 
-    odst = (char*)malloc( recv_malloced_size );
-    osrc = (char*)malloc( send_malloced_size );
-    ptemp = malloc( chunk );
+    odst = (char *) malloc(recv_malloced_size);
+    osrc = (char *) malloc(send_malloced_size);
+    ptemp = malloc(chunk);
 
     /* fill up the receiver with ZEROS */
     {
-        for( size_t i = 0; i < send_malloced_size; i++ )
+        for (size_t i = 0; i < send_malloced_size; i++)
             osrc[i] = i % 128 + 32;
     }
-    memset( odst, 0, recv_malloced_size );
-    pdst  = odst - recv_lb;
-    psrc  = osrc - send_lb;
+    memset(odst, 0, recv_malloced_size);
+    pdst = odst - recv_lb;
+    psrc = osrc - send_lb;
 
-    send_convertor = opal_convertor_create( remote_arch, 0 );
-    if( OPAL_SUCCESS != opal_convertor_prepare_for_send( send_convertor, send_type, send_count, psrc ) ) {
-        printf( "Unable to create the send convertor. Is the datatype committed ?\n" );
+    send_convertor = opal_convertor_create(remote_arch, 0);
+    if (OPAL_SUCCESS
+        != opal_convertor_prepare_for_send(send_convertor, send_type, send_count, psrc)) {
+        printf("Unable to create the send convertor. Is the datatype committed ?\n");
         goto clean_and_return;
     }
-    recv_convertor = opal_convertor_create( remote_arch, 0 );
-    if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( recv_convertor, recv_type, recv_count, pdst ) ) {
-        printf( "Unable to create the recv convertor. Is the datatype committed ?\n" );
+    recv_convertor = opal_convertor_create(remote_arch, 0);
+    if (OPAL_SUCCESS
+        != opal_convertor_prepare_for_recv(recv_convertor, recv_type, recv_count, pdst)) {
+        printf("Unable to create the recv convertor. Is the datatype committed ?\n");
         goto clean_and_return;
     }
 
-    cache_trash();  /* make sure the cache is useless */
+    cache_trash(); /* make sure the cache is useless */
 
-    GET_TIME( start );
-    while( (done1 & done2) != 1 ) {
+    GET_TIME(start);
+    while ((done1 & done2) != 1) {
         /* They are supposed to finish in exactly the same time. */
-        if( done1 | done2 ) {
-            printf( "WRONG !!! the send is %s but the receive is %s in local_copy_with_convertor_2datatypes\n",
-                    (done1 ? "finish" : "not finish"),
-                    (done2 ? "finish" : "not finish") );
+        if (done1 | done2) {
+            printf("WRONG !!! the send is %s but the receive is %s in "
+                   "local_copy_with_convertor_2datatypes\n",
+                   (done1 ? "finish" : "not finish"), (done2 ? "finish" : "not finish"));
         }
 
         max_data = chunk;
@@ -262,20 +269,20 @@ local_copy_with_convertor_2datatypes( opal_datatype_t const * const send_type, i
         iov.iov_base = ptemp;
         iov.iov_len = chunk;
 
-        if( done1 == 0 ) {
-            done1 = opal_convertor_pack( send_convertor, &iov, &iov_count, &max_data );
+        if (done1 == 0) {
+            done1 = opal_convertor_pack(send_convertor, &iov, &iov_count, &max_data);
         }
 
-        if( done2 == 0 ) {
-            GET_TIME( unpack_start );
-            done2 = opal_convertor_unpack( recv_convertor, &iov, &iov_count, &max_data );
-            GET_TIME( unpack_end );
-            unpack_time += ELAPSED_TIME( unpack_start, unpack_end );
+        if (done2 == 0) {
+            GET_TIME(unpack_start);
+            done2 = opal_convertor_unpack(recv_convertor, &iov, &iov_count, &max_data);
+            GET_TIME(unpack_end);
+            unpack_time += ELAPSED_TIME(unpack_start, unpack_end);
         }
 
         length += max_data;
 
-        if( outputFlags & RESET_CONVERTORS ) {
+        if (outputFlags & RESET_CONVERTORS) {
             size_t pos = 0;
             opal_convertor_set_position(send_convertor, &pos);
             pos = length;
@@ -289,25 +296,30 @@ local_copy_with_convertor_2datatypes( opal_datatype_t const * const send_type, i
             assert(pos == length);
         }
     }
-    GET_TIME( end );
-    total_time = ELAPSED_TIME( start, end );
-    printf( "copying different data-types using convertors in %ld microsec\n", total_time );
-    printf( "\t unpack in %ld microsec [pack in %ld microsec]\n", unpack_time,
-            total_time - unpack_time );
- clean_and_return:
-    if( send_convertor != NULL ) {
-        OBJ_RELEASE( send_convertor ); assert( send_convertor == NULL );
-    }
-    if( recv_convertor != NULL ) {
-        OBJ_RELEASE( recv_convertor ); assert( recv_convertor == NULL );
-    }
-    if( NULL != odst ) free( odst );
-    if( NULL != osrc ) free( osrc );
-    if( NULL != ptemp ) free( ptemp );
+    GET_TIME(end);
+    total_time = ELAPSED_TIME(start, end);
+    printf("copying different data-types using convertors in %ld microsec\n", total_time);
+    printf("\t unpack in %ld microsec [pack in %ld microsec]\n", unpack_time,
+           total_time - unpack_time);
+clean_and_return:
+    if (send_convertor != NULL) {
+        OBJ_RELEASE(send_convertor);
+        assert(send_convertor == NULL);
+    }
+    if (recv_convertor != NULL) {
+        OBJ_RELEASE(recv_convertor);
+        assert(recv_convertor == NULL);
+    }
+    if (NULL != odst)
+        free(odst);
+    if (NULL != osrc)
+        free(osrc);
+    if (NULL != ptemp)
+        free(ptemp);
     return OPAL_SUCCESS;
 }
 
-static int local_copy_with_convertor( opal_datatype_t const * const pdt, int count, int chunk )
+static int local_copy_with_convertor(opal_datatype_t const *const pdt, int count, int chunk)
 {
     ptrdiff_t lb, extent;
     void *pdst = NULL, *psrc = NULL, *ptemp = NULL;
@@ -321,40 +333,40 @@ static int local_copy_with_convertor( opal_datatype_t const * const pdt, int cou
     long total_time, unpack_time = 0;
 
     malloced_size = compute_memory_size(pdt, count);
-    opal_datatype_get_extent( pdt, &lb, &extent );
+    opal_datatype_get_extent(pdt, &lb, &extent);
 
-    odst = (char*)malloc( malloced_size );
-    osrc = (char*)malloc( malloced_size );
-    ptemp = malloc( chunk );
+    odst = (char *) malloc(malloced_size);
+    osrc = (char *) malloc(malloced_size);
+    ptemp = malloc(chunk);
 
     {
-        for( size_t i = 0; i < malloced_size; osrc[i] = i % 128 + 32, i++ );
+        for (size_t i = 0; i < malloced_size; osrc[i] = i % 128 + 32, i++)
+            ;
         memcpy(odst, osrc, malloced_size);
     }
-    pdst  = odst - lb;
-    psrc  = osrc - lb;
+    pdst = odst - lb;
+    psrc = osrc - lb;
 
-    send_convertor = opal_convertor_create( remote_arch, 0 );
-    if( OPAL_SUCCESS != opal_convertor_prepare_for_send( send_convertor, pdt, count, psrc ) ) {
-        printf( "Unable to create the send convertor. Is the datatype committed ?\n" );
+    send_convertor = opal_convertor_create(remote_arch, 0);
+    if (OPAL_SUCCESS != opal_convertor_prepare_for_send(send_convertor, pdt, count, psrc)) {
+        printf("Unable to create the send convertor. Is the datatype committed ?\n");
         goto clean_and_return;
     }
 
-    recv_convertor = opal_convertor_create( remote_arch, 0 );
-    if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( recv_convertor, pdt, count, pdst ) ) {
-        printf( "Unable to create the recv convertor. Is the datatype committed ?\n" );
+    recv_convertor = opal_convertor_create(remote_arch, 0);
+    if (OPAL_SUCCESS != opal_convertor_prepare_for_recv(recv_convertor, pdt, count, pdst)) {
+        printf("Unable to create the recv convertor. Is the datatype committed ?\n");
         goto clean_and_return;
     }
 
-    cache_trash();  /* make sure the cache is useless */
+    cache_trash(); /* make sure the cache is useless */
 
-    GET_TIME( start );
-    while( (done1 & done2) != 1 ) {
+    GET_TIME(start);
+    while ((done1 & done2) != 1) {
         /* They are supposed to finish in exactly the same time. */
-        if( done1 | done2 ) {
-            printf( "WRONG !!! the send is %s but the receive is %s in local_copy_with_convertor\n",
-                    (done1 ? "finish" : "not finish"),
-                    (done2 ? "finish" : "not finish") );
+        if (done1 | done2) {
+            printf("WRONG !!! the send is %s but the receive is %s in local_copy_with_convertor\n",
+                   (done1 ? "finish" : "not finish"), (done2 ? "finish" : "not finish"));
         }
 
         max_data = chunk;
@@ -362,131 +374,187 @@ static int local_copy_with_convertor( opal_datatype_t const * const pdt, int cou
         iov.iov_base = ptemp;
         iov.iov_len = chunk;
 
-        if( done1 == 0 ) {
-            done1 = opal_convertor_pack( send_convertor, &iov, &iov_count, &max_data );
+        if (done1 == 0) {
+            done1 = opal_convertor_pack(send_convertor, &iov, &iov_count, &max_data);
         }
 
-        if( done2 == 0 ) {
-            GET_TIME( unpack_start );
-            done2 = opal_convertor_unpack( recv_convertor, &iov, &iov_count, &max_data );
-            GET_TIME( unpack_end );
-            unpack_time += ELAPSED_TIME( unpack_start, unpack_end );
+        if (done2 == 0) {
+            GET_TIME(unpack_start);
+            done2 = opal_convertor_unpack(recv_convertor, &iov, &iov_count, &max_data);
+            GET_TIME(unpack_end);
+            unpack_time += ELAPSED_TIME(unpack_start, unpack_end);
         }
 
         length += max_data;
-        if( outputFlags & RESET_CONVERTORS ) {
-            struct dt_stack_t stack[1+send_convertor->stack_pos];
+        if (outputFlags & RESET_CONVERTORS) {
+            struct dt_stack_t stack[1 + send_convertor->stack_pos];
             int i, stack_pos = send_convertor->stack_pos;
             size_t pos;
 
-            if( 0 == done1 ) {
-                memcpy(stack, send_convertor->pStack, (1+send_convertor->stack_pos) * sizeof(struct dt_stack_t));
+            if (0 == done1) {
+                memcpy(stack, send_convertor->pStack,
+                       (1 + send_convertor->stack_pos) * sizeof(struct dt_stack_t));
                 pos = 0;
                 opal_convertor_set_position(send_convertor, &pos);
                 pos = length;
                 opal_convertor_set_position(send_convertor, &pos);
                 assert(pos == length);
-                for(i = 0; i <= stack_pos; i++ ) {
-                    if( stack[i].index != send_convertor->pStack[i].index )
-                        {errors = 1; printf("send stack[%d].index differs (orig %d != new %d) (completed %lu/%lu)\n",
-                                            i, stack[i].index, send_convertor->pStack[i].index,
-                                            length, pdt->size * count);}
-                    if( stack[i].count != send_convertor->pStack[i].count ) {
-                        if( stack[i].type == send_convertor->pStack[i].type ) {
-                            {errors = 1; printf("send stack[%d].count differs (orig %lu != new %lu) (completed %lu/%lu)\n",
-                                                    i, stack[i].count, send_convertor->pStack[i].count,
-                                                    length, pdt->size * count);}
+                for (i = 0; i <= stack_pos; i++) {
+                    if (stack[i].index != send_convertor->pStack[i].index) {
+                        errors = 1;
+                        printf("send stack[%d].index differs (orig %d != new %d) (completed "
+                               "%lu/%lu)\n",
+                               i, stack[i].index, send_convertor->pStack[i].index, length,
+                               pdt->size * count);
+                    }
+                    if (stack[i].count != send_convertor->pStack[i].count) {
+                        if (stack[i].type == send_convertor->pStack[i].type) {
+                            {
+                                errors = 1;
+                                printf("send stack[%d].count differs (orig %lu != new %lu) "
+                                       "(completed %lu/%lu)\n",
+                                       i, stack[i].count, send_convertor->pStack[i].count, length,
+                                       pdt->size * count);
+                            }
                         } else {
-                            if( (OPAL_DATATYPE_MAX_PREDEFINED <= stack[i].type) || (OPAL_DATATYPE_MAX_PREDEFINED <= send_convertor->pStack[i].type) )
-                                {errors = 1; printf("send stack[%d].type wrong (orig %d != new %d) (completed %lu/%lu)\n",
-                                                    i, (int)stack[i].type, (int)send_convertor->pStack[i].type,
-                                                    length, pdt->size * count);}
-                            else if( (stack[i].count * opal_datatype_basicDatatypes[stack[i].type]->size) !=
-                                     (send_convertor->pStack[i].count * opal_datatype_basicDatatypes[send_convertor->pStack[i].type]->size) )
-                                {errors = 1; printf("send stack[%d].type*count differs (orig (%d,%lu) != new (%d, %lu)) (completed %lu/%lu)\n",
-                                                    i, (int)stack[i].type, stack[i].count,
-                                                    (int)send_convertor->pStack[i].type, send_convertor->pStack[i].count,
-                                                    length, pdt->size * count);}
+                            if ((OPAL_DATATYPE_MAX_PREDEFINED <= stack[i].type)
+                                || (OPAL_DATATYPE_MAX_PREDEFINED
+                                    <= send_convertor->pStack[i].type)) {
+                                errors = 1;
+                                printf("send stack[%d].type wrong (orig %d != new %d) (completed "
+                                       "%lu/%lu)\n",
+                                       i, (int) stack[i].type, (int) send_convertor->pStack[i].type,
+                                       length, pdt->size * count);
+                            } else if ((stack[i].count
+                                        * opal_datatype_basicDatatypes[stack[i].type]->size)
+                                       != (send_convertor->pStack[i].count
+                                           * opal_datatype_basicDatatypes[send_convertor->pStack[i]
+                                                                              .type]
+                                                 ->size)) {
+                                errors = 1;
+                                printf("send stack[%d].type*count differs (orig (%d,%lu) != new "
+                                       "(%d, %lu)) (completed %lu/%lu)\n",
+                                       i, (int) stack[i].type, stack[i].count,
+                                       (int) send_convertor->pStack[i].type,
+                                       send_convertor->pStack[i].count, length, pdt->size * count);
+                            }
                         }
                     }
-                    if( stack[i].disp != send_convertor->pStack[i].disp )
-                        {errors = 1; printf("send stack[%d].disp differs (orig %p != new %p) (completed %lu/%lu)\n",
-                                            i, (void*)stack[i].disp, (void*)send_convertor->pStack[i].disp,
-                                            length, pdt->size * count);}
-                    if(0 != errors) {assert(0); exit(-1);}
+                    if (stack[i].disp != send_convertor->pStack[i].disp) {
+                        errors = 1;
+                        printf(
+                            "send stack[%d].disp differs (orig %p != new %p) (completed %lu/%lu)\n",
+                            i, (void *) stack[i].disp, (void *) send_convertor->pStack[i].disp,
+                            length, pdt->size * count);
+                    }
+                    if (0 != errors) {
+                        assert(0);
+                        exit(-1);
+                    }
                 }
             }
-            if( 0 == done2 ) {
-                memcpy(stack, recv_convertor->pStack, (1+recv_convertor->stack_pos) * sizeof(struct dt_stack_t));
+            if (0 == done2) {
+                memcpy(stack, recv_convertor->pStack,
+                       (1 + recv_convertor->stack_pos) * sizeof(struct dt_stack_t));
                 pos = 0;
                 opal_convertor_set_position(recv_convertor, &pos);
                 pos = length;
                 opal_convertor_set_position(recv_convertor, &pos);
                 assert(pos == length);
-                for(i = 0; i <= stack_pos; i++ ) {
-                    if( stack[i].index != recv_convertor->pStack[i].index )
-                        {errors = 1; printf("recv stack[%d].index differs (orig %d != new %d) (completed %lu/%lu)\n",
-                                            i, stack[i].index, recv_convertor->pStack[i].index,
-                                            length, pdt->size * count);}
-                    if( stack[i].count != recv_convertor->pStack[i].count ) {
-                        if( stack[i].type == recv_convertor->pStack[i].type ) {
-                            {errors = 1; printf("recv stack[%d].count differs (orig %lu != new %lu) (completed %lu/%lu)\n",
-                                                    i, stack[i].count, recv_convertor->pStack[i].count,
-                                                    length, pdt->size * count);}
+                for (i = 0; i <= stack_pos; i++) {
+                    if (stack[i].index != recv_convertor->pStack[i].index) {
+                        errors = 1;
+                        printf("recv stack[%d].index differs (orig %d != new %d) (completed "
+                               "%lu/%lu)\n",
+                               i, stack[i].index, recv_convertor->pStack[i].index, length,
+                               pdt->size * count);
+                    }
+                    if (stack[i].count != recv_convertor->pStack[i].count) {
+                        if (stack[i].type == recv_convertor->pStack[i].type) {
+                            {
+                                errors = 1;
+                                printf("recv stack[%d].count differs (orig %lu != new %lu) "
+                                       "(completed %lu/%lu)\n",
+                                       i, stack[i].count, recv_convertor->pStack[i].count, length,
+                                       pdt->size * count);
+                            }
                         } else {
-                            if( (OPAL_DATATYPE_MAX_PREDEFINED <= stack[i].type) || (OPAL_DATATYPE_MAX_PREDEFINED <= recv_convertor->pStack[i].type) )
-                                {errors = 1; printf("recv stack[%d].type wrong (orig %d != new %d) (completed %lu/%lu)\n",
-                                                    i, (int)stack[i].type, (int)recv_convertor->pStack[i].type,
-                                                    length, pdt->size * count);}
-                            else if( (stack[i].count * opal_datatype_basicDatatypes[stack[i].type]->size) !=
-                                     (recv_convertor->pStack[i].count * opal_datatype_basicDatatypes[recv_convertor->pStack[i].type]->size) )
-                                {errors = 1; printf("recv stack[%d].type*count differs (orig (%d,%lu) != new (%d, %lu)) (completed %lu/%lu)\n",
-                                                    i, (int)stack[i].type, stack[i].count,
-                                                    (int)recv_convertor->pStack[i].type, recv_convertor->pStack[i].count,
-                                                    length, pdt->size * count);}
+                            if ((OPAL_DATATYPE_MAX_PREDEFINED <= stack[i].type)
+                                || (OPAL_DATATYPE_MAX_PREDEFINED
+                                    <= recv_convertor->pStack[i].type)) {
+                                errors = 1;
+                                printf("recv stack[%d].type wrong (orig %d != new %d) (completed "
+                                       "%lu/%lu)\n",
+                                       i, (int) stack[i].type, (int) recv_convertor->pStack[i].type,
+                                       length, pdt->size * count);
+                            } else if ((stack[i].count
+                                        * opal_datatype_basicDatatypes[stack[i].type]->size)
+                                       != (recv_convertor->pStack[i].count
+                                           * opal_datatype_basicDatatypes[recv_convertor->pStack[i]
+                                                                              .type]
+                                                 ->size)) {
+                                errors = 1;
+                                printf("recv stack[%d].type*count differs (orig (%d,%lu) != new "
+                                       "(%d, %lu)) (completed %lu/%lu)\n",
+                                       i, (int) stack[i].type, stack[i].count,
+                                       (int) recv_convertor->pStack[i].type,
+                                       recv_convertor->pStack[i].count, length, pdt->size * count);
+                            }
                         }
                     }
-                    if( stack[i].disp != recv_convertor->pStack[i].disp )
-                        {errors = 1; printf("recv stack[%d].disp differs (orig %p != new %p) (completed %lu/%lu)\n",
-                                            i, (void*)stack[i].disp, (void*)recv_convertor->pStack[i].disp,
-                                            length, pdt->size * count);}
-                    if(0 != errors) {assert(0); exit(-1);}
+                    if (stack[i].disp != recv_convertor->pStack[i].disp) {
+                        errors = 1;
+                        printf(
+                            "recv stack[%d].disp differs (orig %p != new %p) (completed %lu/%lu)\n",
+                            i, (void *) stack[i].disp, (void *) recv_convertor->pStack[i].disp,
+                            length, pdt->size * count);
+                    }
+                    if (0 != errors) {
+                        assert(0);
+                        exit(-1);
+                    }
                 }
             }
         }
     }
-    GET_TIME( end );
-    total_time = ELAPSED_TIME( start, end );
-    printf( "copying same data-type using convertors in %ld microsec\n", total_time );
-    printf( "\t unpack in %ld microsec [pack in %ld microsec]\n", unpack_time,
-            total_time - unpack_time );
-
-    if(outputFlags & VALIDATE_DATA) {
-        for( size_t i = errors = 0; i < malloced_size; i++ ) {
-            if( odst[i] != osrc[i] ) {
-                printf("error at position %lu (%d != %d)\n",
-                       (unsigned long)i, (int)(odst[i]), (int)(osrc[i]));
+    GET_TIME(end);
+    total_time = ELAPSED_TIME(start, end);
+    printf("copying same data-type using convertors in %ld microsec\n", total_time);
+    printf("\t unpack in %ld microsec [pack in %ld microsec]\n", unpack_time,
+           total_time - unpack_time);
+
+    if (outputFlags & VALIDATE_DATA) {
+        for (size_t i = errors = 0; i < malloced_size; i++) {
+            if (odst[i] != osrc[i]) {
+                printf("error at position %lu (%d != %d)\n", (unsigned long) i, (int) (odst[i]),
+                       (int) (osrc[i]));
                 errors++;
-                if(outputFlags & QUIT_ON_FIRST_ERROR) {
+                if (outputFlags & QUIT_ON_FIRST_ERROR) {
                     opal_datatype_dump(pdt);
-                    assert(0); exit(-1);
+                    assert(0);
+                    exit(-1);
                 }
             }
         }
-        if( 0 == errors ) {
+        if (0 == errors) {
             printf("Validation check successfully passed\n");
         } else {
             printf("Found %d errors. Giving up!\n", errors);
             exit(-1);
         }
     }
- clean_and_return:
-    if( NULL != send_convertor ) OBJ_RELEASE( send_convertor );
-    if( NULL != recv_convertor ) OBJ_RELEASE( recv_convertor );
-
-    if( NULL != odst ) free( odst );
-    if( NULL != osrc ) free( osrc );
-    if( NULL != ptemp ) free( ptemp );
+clean_and_return:
+    if (NULL != send_convertor)
+        OBJ_RELEASE(send_convertor);
+    if (NULL != recv_convertor)
+        OBJ_RELEASE(recv_convertor);
+
+    if (NULL != odst)
+        free(odst);
+    if (NULL != osrc)
+        free(osrc);
+    if (NULL != ptemp)
+        free(ptemp);
     return (0 == errors ? OPAL_SUCCESS : errors);
 }
 
@@ -497,226 +565,247 @@ static int local_copy_with_convertor( opal_datatype_t const * const pdt, int cou
  * on the data-type engine should first pass all the tests from this file, before going into other
  * tests.
  */
-int main( int argc, char* argv[] )
+int main(int argc, char *argv[])
 {
     opal_datatype_t *pdt, *pdt1, *pdt2, *pdt3;
     int rc, length = 500;
 
-    opal_init_util (NULL, NULL);
+    opal_init(NULL, NULL);
 
     /**
      * By default simulate homogeneous architectures.
      */
     remote_arch = opal_local_arch;
-    printf( "\n\n#\n * TEST CREATE CONTIGUOUS\n#\n\n" );
-    pdt = create_contiguous_type( &opal_datatype_int1, 10 );
-    if( outputFlags & CHECK_PACK_UNPACK ) {
+    printf("\n\n#\n * TEST CREATE CONTIGUOUS\n#\n\n");
+    pdt = create_contiguous_type(&opal_datatype_int1, 10);
+    if (outputFlags & CHECK_PACK_UNPACK) {
         local_copy_ddt_count(pdt, 100);
         local_copy_with_convertor(pdt, 100, 956);
     }
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
 
-    printf( "\n\n#\n * TEST STRANGE DATATYPE\n#\n\n" );
+    printf("\n\n#\n * TEST STRANGE DATATYPE\n#\n\n");
     pdt = create_strange_dt();
-    if( outputFlags & CHECK_PACK_UNPACK ) {
+    if (outputFlags & CHECK_PACK_UNPACK) {
         local_copy_ddt_count(pdt, 1);
         local_copy_with_convertor(pdt, 1, 956);
     }
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
 
-    printf( "\n\n#\n * TEST UPPER TRIANGULAR MATRIX (size 100)\n#\n\n" );
+    printf("\n\n#\n * TEST UPPER TRIANGULAR MATRIX (size 100)\n#\n\n");
     pdt = upper_matrix(100);
-    if( outputFlags & CHECK_PACK_UNPACK ) {
+    if (outputFlags & CHECK_PACK_UNPACK) {
         local_copy_ddt_count(pdt, 1);
         local_copy_with_convertor(pdt, 1, 48);
     }
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
 
-    printf( "\n\n#\n * TEST UPPER MATRIX\n#\n\n" );
-    rc = test_upper( length );
-    if( rc == 0 )
-        printf( "decode [PASSED]\n" );
+    printf("\n\n#\n * TEST UPPER MATRIX\n#\n\n");
+    rc = test_upper(length);
+    if (rc == 0)
+        printf("decode [PASSED]\n");
     else
-        printf( "decode [NOT PASSED]\n" );
+        printf("decode [NOT PASSED]\n");
 
-    printf( "\n\n#\n * TEST MATRIX BORDERS\n#\n\n" );
-    pdt = test_matrix_borders( length, 100 );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        opal_datatype_dump( pdt );
+    printf("\n\n#\n * TEST MATRIX BORDERS\n#\n\n");
+    pdt = test_matrix_borders(length, 100);
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        opal_datatype_dump(pdt);
     }
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
-
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
 
-    printf( "\n\n#\n * TEST CONTIGUOUS\n#\n\n" );
+    printf("\n\n#\n * TEST CONTIGUOUS\n#\n\n");
     pdt = test_contiguous();
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
-    printf( "\n\n#\n * TEST STRUCT\n#\n\n" );
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
+    printf("\n\n#\n * TEST STRUCT\n#\n\n");
     pdt = test_struct();
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
 
     opal_datatype_create_contiguous(0, &opal_datatype_empty, &pdt1);
     opal_datatype_create_contiguous(0, &opal_datatype_empty, &pdt2);
     opal_datatype_create_contiguous(0, &opal_datatype_empty, &pdt3);
 
-    opal_datatype_add( pdt3, &opal_datatype_int4, 10, 0, -1 );
-    opal_datatype_add( pdt3, &opal_datatype_float4, 5, 10 * sizeof(int), -1 );
+    opal_datatype_add(pdt3, &opal_datatype_int4, 10, 0, -1);
+    opal_datatype_add(pdt3, &opal_datatype_float4, 5, 10 * sizeof(int), -1);
 
-    opal_datatype_add( pdt2, &opal_datatype_float4, 1, 0, -1 );
-    opal_datatype_add( pdt2, pdt3, 3, sizeof(int) * 1, -1 );
+    opal_datatype_add(pdt2, &opal_datatype_float4, 1, 0, -1);
+    opal_datatype_add(pdt2, pdt3, 3, sizeof(int) * 1, -1);
 
-    opal_datatype_add( pdt1, &opal_datatype_int8, 5, 0, -1 );
-    opal_datatype_add( pdt1, &opal_datatype_float16, 2, sizeof(long long) * 5, -1 );
+    opal_datatype_add(pdt1, &opal_datatype_int8, 5, 0, -1);
+    opal_datatype_add(pdt1, &opal_datatype_float16, 2, sizeof(long long) * 5, -1);
 
-    printf( ">>--------------------------------------------<<\n" );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        opal_datatype_dump( pdt1 );
+    printf(">>--------------------------------------------<<\n");
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        opal_datatype_dump(pdt1);
     }
-    printf( ">>--------------------------------------------<<\n" );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        opal_datatype_dump( pdt2 );
+    printf(">>--------------------------------------------<<\n");
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        opal_datatype_dump(pdt2);
     }
-    printf( ">>--------------------------------------------<<\n" );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        opal_datatype_dump( pdt3 );
+    printf(">>--------------------------------------------<<\n");
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        opal_datatype_dump(pdt3);
     }
 
-    OBJ_RELEASE( pdt1 ); assert( pdt1 == NULL );
-    OBJ_RELEASE( pdt2 ); assert( pdt2 == NULL );
-    OBJ_RELEASE( pdt3 ); assert( pdt3 == NULL );
+    OBJ_RELEASE(pdt1);
+    assert(pdt1 == NULL);
+    OBJ_RELEASE(pdt2);
+    assert(pdt2 == NULL);
+    OBJ_RELEASE(pdt3);
+    assert(pdt3 == NULL);
 
-    printf( ">>--------------------------------------------<<\n" );
-    printf( " Contiguous data-type (opal_datatype_float8)\n" );
-    if( outputFlags & CHECK_PACK_UNPACK ) {
-        opal_datatype_t const * const ddt = &opal_datatype_float8;
-        local_copy_ddt_count( ddt, 4500);
-        local_copy_with_convertor( ddt, 4500, 12 );
-        local_copy_with_convertor_2datatypes( ddt, 4500, ddt, 4500, 12 );
+    printf(">>--------------------------------------------<<\n");
+    printf(" Contiguous data-type (opal_datatype_float8)\n");
+    if (outputFlags & CHECK_PACK_UNPACK) {
+        opal_datatype_t const *const ddt = &opal_datatype_float8;
+        local_copy_ddt_count(ddt, 4500);
+        local_copy_with_convertor(ddt, 4500, 12);
+        local_copy_with_convertor_2datatypes(ddt, 4500, ddt, 4500, 12);
     }
-    printf( ">>--------------------------------------------<<\n" );
+    printf(">>--------------------------------------------<<\n");
 
-    printf( ">>--------------------------------------------<<\n" );
-    if( outputFlags & CHECK_PACK_UNPACK ) {
-        printf( "Contiguous multiple data-type (4500*1)\n" );
-        pdt = create_contiguous_type( &opal_datatype_float8, 4500 );
+    printf(">>--------------------------------------------<<\n");
+    if (outputFlags & CHECK_PACK_UNPACK) {
+        printf("Contiguous multiple data-type (4500*1)\n");
+        pdt = create_contiguous_type(&opal_datatype_float8, 4500);
         local_copy_ddt_count(pdt, 1);
-        local_copy_with_convertor( pdt, 1, 12 );
-        local_copy_with_convertor_2datatypes( pdt, 1, pdt, 1, 12 );
-        OBJ_RELEASE( pdt ); assert( pdt == NULL );
-        printf( "Contiguous multiple data-type (450*10)\n" );
-        pdt = create_contiguous_type( &opal_datatype_float8, 450 );
+        local_copy_with_convertor(pdt, 1, 12);
+        local_copy_with_convertor_2datatypes(pdt, 1, pdt, 1, 12);
+        OBJ_RELEASE(pdt);
+        assert(pdt == NULL);
+        printf("Contiguous multiple data-type (450*10)\n");
+        pdt = create_contiguous_type(&opal_datatype_float8, 450);
         local_copy_ddt_count(pdt, 10);
-        local_copy_with_convertor( pdt, 10, 12 );
-        local_copy_with_convertor_2datatypes( pdt, 10, pdt, 10, 12 );
-        OBJ_RELEASE( pdt ); assert( pdt == NULL );
-        printf( "Contiguous multiple data-type (45*100)\n" );
-        pdt = create_contiguous_type( &opal_datatype_float8, 45 );
+        local_copy_with_convertor(pdt, 10, 12);
+        local_copy_with_convertor_2datatypes(pdt, 10, pdt, 10, 12);
+        OBJ_RELEASE(pdt);
+        assert(pdt == NULL);
+        printf("Contiguous multiple data-type (45*100)\n");
+        pdt = create_contiguous_type(&opal_datatype_float8, 45);
         local_copy_ddt_count(pdt, 100);
-        local_copy_with_convertor( pdt, 100, 12 );
-        local_copy_with_convertor_2datatypes( pdt, 100, pdt, 100, 12 );
-        OBJ_RELEASE( pdt ); assert( pdt == NULL );
-        printf( "Contiguous multiple data-type (100*45)\n" );
-        pdt = create_contiguous_type( &opal_datatype_float8, 100 );
+        local_copy_with_convertor(pdt, 100, 12);
+        local_copy_with_convertor_2datatypes(pdt, 100, pdt, 100, 12);
+        OBJ_RELEASE(pdt);
+        assert(pdt == NULL);
+        printf("Contiguous multiple data-type (100*45)\n");
+        pdt = create_contiguous_type(&opal_datatype_float8, 100);
         local_copy_ddt_count(pdt, 45);
-        local_copy_with_convertor( pdt, 45, 12 );
-        local_copy_with_convertor_2datatypes( pdt, 45, pdt, 45, 12 );
-        OBJ_RELEASE( pdt ); assert( pdt == NULL );
-        printf( "Contiguous multiple data-type (10*450)\n" );
-        pdt = create_contiguous_type( &opal_datatype_float8, 10 );
+        local_copy_with_convertor(pdt, 45, 12);
+        local_copy_with_convertor_2datatypes(pdt, 45, pdt, 45, 12);
+        OBJ_RELEASE(pdt);
+        assert(pdt == NULL);
+        printf("Contiguous multiple data-type (10*450)\n");
+        pdt = create_contiguous_type(&opal_datatype_float8, 10);
         local_copy_ddt_count(pdt, 450);
-        local_copy_with_convertor( pdt, 450, 12 );
-        local_copy_with_convertor_2datatypes( pdt, 450, pdt, 450, 12 );
-        OBJ_RELEASE( pdt ); assert( pdt == NULL );
-        printf( "Contiguous multiple data-type (1*4500)\n" );
-        pdt = create_contiguous_type( &opal_datatype_float8, 1 );
+        local_copy_with_convertor(pdt, 450, 12);
+        local_copy_with_convertor_2datatypes(pdt, 450, pdt, 450, 12);
+        OBJ_RELEASE(pdt);
+        assert(pdt == NULL);
+        printf("Contiguous multiple data-type (1*4500)\n");
+        pdt = create_contiguous_type(&opal_datatype_float8, 1);
         local_copy_ddt_count(pdt, 4500);
-        local_copy_with_convertor( pdt, 4500, 12 );
-        local_copy_with_convertor_2datatypes( pdt, 4500, pdt, 4500, 12 );
-        OBJ_RELEASE( pdt ); assert( pdt == NULL );
-    }
-    printf( ">>--------------------------------------------<<\n" );
-    printf( ">>--------------------------------------------<<\n" );
-    printf( "Vector data-type (450 times 10 double stride 11)\n" );
-    pdt = create_vector_type( &opal_datatype_float8, 450, 10, 11 );
-    opal_datatype_dump( pdt );
-    if( outputFlags & CHECK_PACK_UNPACK ) {
+        local_copy_with_convertor(pdt, 4500, 12);
+        local_copy_with_convertor_2datatypes(pdt, 4500, pdt, 4500, 12);
+        OBJ_RELEASE(pdt);
+        assert(pdt == NULL);
+    }
+    printf(">>--------------------------------------------<<\n");
+    printf(">>--------------------------------------------<<\n");
+    printf("Vector data-type (450 times 10 double stride 11)\n");
+    pdt = create_vector_type(&opal_datatype_float8, 450, 10, 11);
+    opal_datatype_dump(pdt);
+    if (outputFlags & CHECK_PACK_UNPACK) {
         local_copy_ddt_count(pdt, 1);
-        local_copy_with_convertor( pdt, 1, 12 );
-        local_copy_with_convertor_2datatypes( pdt, 1, pdt, 1, 12 );
-        local_copy_with_convertor( pdt, 1, 82 );
-        local_copy_with_convertor_2datatypes( pdt, 1, pdt, 1, 82 );
-        local_copy_with_convertor( pdt, 1, 6000 );
-        local_copy_with_convertor_2datatypes( pdt, 1, pdt, 1, 6000 );
-        local_copy_with_convertor( pdt, 1, 36000 );
-        local_copy_with_convertor_2datatypes( pdt, 1, pdt, 1, 36000 );
-    }
-    printf( ">>--------------------------------------------<<\n" );
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
-
-    printf( ">>--------------------------------------------<<\n" );
-    printf( "Struct data-type resized (double unused followed by 2 used doubles)\n" );
-    pdt = create_struct_constant_gap_resized_ddt( &opal_datatype_float8 );
-    opal_datatype_dump( pdt );
-    if( outputFlags & CHECK_PACK_UNPACK ) {
+        local_copy_with_convertor(pdt, 1, 12);
+        local_copy_with_convertor_2datatypes(pdt, 1, pdt, 1, 12);
+        local_copy_with_convertor(pdt, 1, 82);
+        local_copy_with_convertor_2datatypes(pdt, 1, pdt, 1, 82);
+        local_copy_with_convertor(pdt, 1, 6000);
+        local_copy_with_convertor_2datatypes(pdt, 1, pdt, 1, 6000);
+        local_copy_with_convertor(pdt, 1, 36000);
+        local_copy_with_convertor_2datatypes(pdt, 1, pdt, 1, 36000);
+    }
+    printf(">>--------------------------------------------<<\n");
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
+
+    printf(">>--------------------------------------------<<\n");
+    printf("Struct data-type resized (double unused followed by 2 used doubles)\n");
+    pdt = create_struct_constant_gap_resized_ddt(&opal_datatype_float8);
+    opal_datatype_dump(pdt);
+    if (outputFlags & CHECK_PACK_UNPACK) {
         local_copy_ddt_count(pdt, 1);
-        local_copy_with_convertor( pdt, 100, 11 );
-        local_copy_with_convertor_2datatypes( pdt, 100, pdt, 100, 11 );
-        local_copy_with_convertor( pdt, 100, 82 );
-        local_copy_with_convertor_2datatypes( pdt, 100, pdt, 100, 81 );
-        local_copy_with_convertor( pdt, 1500, 6000 );
-        local_copy_with_convertor_2datatypes( pdt, 1500, pdt, 1500, 666 );
-        local_copy_with_convertor( pdt, 10000, 36000 );
-        local_copy_with_convertor_2datatypes( pdt, 10000, pdt, 10000, 1111 );
-    }
-    printf( ">>--------------------------------------------<<\n" );
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
-
-    printf( ">>--------------------------------------------<<\n" );
+        local_copy_with_convertor(pdt, 100, 11);
+        local_copy_with_convertor_2datatypes(pdt, 100, pdt, 100, 11);
+        local_copy_with_convertor(pdt, 100, 82);
+        local_copy_with_convertor_2datatypes(pdt, 100, pdt, 100, 81);
+        local_copy_with_convertor(pdt, 1500, 6000);
+        local_copy_with_convertor_2datatypes(pdt, 1500, pdt, 1500, 666);
+        local_copy_with_convertor(pdt, 10000, 36000);
+        local_copy_with_convertor_2datatypes(pdt, 10000, pdt, 10000, 1111);
+    }
+    printf(">>--------------------------------------------<<\n");
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
+
+    printf(">>--------------------------------------------<<\n");
     pdt = test_struct_char_double();
-    if( outputFlags & CHECK_PACK_UNPACK ) {
+    if (outputFlags & CHECK_PACK_UNPACK) {
         local_copy_ddt_count(pdt, 4500);
-        local_copy_with_convertor( pdt, 4500, 12 );
-        local_copy_with_convertor_2datatypes( pdt, 4500, pdt, 4500, 12 );
+        local_copy_with_convertor(pdt, 4500, 12);
+        local_copy_with_convertor_2datatypes(pdt, 4500, pdt, 4500, 12);
     }
-    printf( ">>--------------------------------------------<<\n" );
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
+    printf(">>--------------------------------------------<<\n");
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
 
-    printf( ">>--------------------------------------------<<\n" );
+    printf(">>--------------------------------------------<<\n");
     pdt = test_create_twice_two_doubles();
-    if( outputFlags & CHECK_PACK_UNPACK ) {
+    if (outputFlags & CHECK_PACK_UNPACK) {
         local_copy_ddt_count(pdt, 4500);
-        local_copy_with_convertor( pdt, 4500, 12 );
-        local_copy_with_convertor_2datatypes( pdt, 4500, pdt, 4500, 12 );
+        local_copy_with_convertor(pdt, 4500, 12);
+        local_copy_with_convertor_2datatypes(pdt, 4500, pdt, 4500, 12);
     }
-    printf( ">>--------------------------------------------<<\n" );
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
+    printf(">>--------------------------------------------<<\n");
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
 
-    printf( ">>--------------------------------------------<<\n" );
+    printf(">>--------------------------------------------<<\n");
     pdt = test_create_blacs_type();
-    if( outputFlags & CHECK_PACK_UNPACK ) {
-        opal_datatype_dump( pdt );
+    if (outputFlags & CHECK_PACK_UNPACK) {
+        opal_datatype_dump(pdt);
         local_copy_ddt_count(pdt, 4500);
-        local_copy_with_convertor( pdt, 4500, 956 );
-        local_copy_with_convertor_2datatypes( pdt, 4500, pdt, 4500, 956 );
-        local_copy_with_convertor( pdt, 4500, 16*1024 );
-        local_copy_with_convertor_2datatypes( pdt, 4500, pdt, 4500, 16*1024 );
-        local_copy_with_convertor( pdt, 4500, 64*1024 );
-        local_copy_with_convertor_2datatypes( pdt, 4500, pdt, 4500, 64*1024 );
-    }
-    printf( ">>--------------------------------------------<<\n" );
-    OBJ_RELEASE( pdt ); assert( pdt == NULL );
-
-    printf( ">>--------------------------------------------<<\n" );
-    pdt1 = test_create_blacs_type1( &opal_datatype_int4 );
-    pdt2 = test_create_blacs_type2( &opal_datatype_int4 );
-    if( outputFlags & CHECK_PACK_UNPACK ) {
-        local_copy_with_convertor_2datatypes( pdt1, 1, pdt2, 1, 100 );
-    }
-    printf( ">>--------------------------------------------<<\n" );
-    OBJ_RELEASE( pdt1 ); assert( pdt1 == NULL );
-    OBJ_RELEASE( pdt2 ); assert( pdt2 == NULL );
+        local_copy_with_convertor(pdt, 4500, 956);
+        local_copy_with_convertor_2datatypes(pdt, 4500, pdt, 4500, 956);
+        local_copy_with_convertor(pdt, 4500, 16 * 1024);
+        local_copy_with_convertor_2datatypes(pdt, 4500, pdt, 4500, 16 * 1024);
+        local_copy_with_convertor(pdt, 4500, 64 * 1024);
+        local_copy_with_convertor_2datatypes(pdt, 4500, pdt, 4500, 64 * 1024);
+    }
+    printf(">>--------------------------------------------<<\n");
+    OBJ_RELEASE(pdt);
+    assert(pdt == NULL);
+
+    printf(">>--------------------------------------------<<\n");
+    pdt1 = test_create_blacs_type1(&opal_datatype_int4);
+    pdt2 = test_create_blacs_type2(&opal_datatype_int4);
+    if (outputFlags & CHECK_PACK_UNPACK) {
+        local_copy_with_convertor_2datatypes(pdt1, 1, pdt2, 1, 100);
+    }
+    printf(">>--------------------------------------------<<\n");
+    OBJ_RELEASE(pdt1);
+    assert(pdt1 == NULL);
+    OBJ_RELEASE(pdt2);
+    assert(pdt2 == NULL);
 
     /* clean-ups all data allocations */
-    opal_finalize_util ();
+    opal_finalize_util();
 
     return OPAL_SUCCESS;
 }
diff --git a/test/datatype/opal_ddt_lib.c b/test/datatype/opal_ddt_lib.c
index 605d7deedb6..eb2ae848940 100644
--- a/test/datatype/opal_ddt_lib.c
+++ b/test/datatype/opal_ddt_lib.c
@@ -24,49 +24,50 @@
 #include "opal/constants.h"
 #include "opal/datatype/opal_datatype.h"
 
-
 uint32_t outputFlags = VALIDATE_DATA | CHECK_PACK_UNPACK | RESET_CONVERTORS | QUIT_ON_FIRST_ERROR;
 
-static int32_t opal_datatype_create_indexed( int count, const int* pBlockLength, const int* pDisp,
-                                             const opal_datatype_t* oldType, opal_datatype_t** newType );
-static int32_t opal_datatype_create_hindexed( int count, const int* pBlockLength, const ptrdiff_t* pDisp,
-                                              const opal_datatype_t* oldType, opal_datatype_t** newType );
-static int32_t opal_datatype_create_struct( int count, const int* pBlockLength,
-                                            const ptrdiff_t* pDisp,
-                                            opal_datatype_t** pTypes, opal_datatype_t** newType );
-static int32_t opal_datatype_create_vector( int count, int bLength, int stride,
-                                            const opal_datatype_t* oldType, opal_datatype_t** newType );
-static int32_t opal_datatype_create_hvector( int count, int bLength, ptrdiff_t stride,
-                                             const opal_datatype_t* oldType, opal_datatype_t** newType );
-
+static int32_t opal_datatype_create_indexed(int count, const int *pBlockLength, const int *pDisp,
+                                            const opal_datatype_t *oldType,
+                                            opal_datatype_t **newType);
+static int32_t opal_datatype_create_hindexed(int count, const int *pBlockLength,
+                                             const ptrdiff_t *pDisp, const opal_datatype_t *oldType,
+                                             opal_datatype_t **newType);
+static int32_t opal_datatype_create_struct(int count, const int *pBlockLength,
+                                           const ptrdiff_t *pDisp, opal_datatype_t **pTypes,
+                                           opal_datatype_t **newType);
+static int32_t opal_datatype_create_vector(int count, int bLength, int stride,
+                                           const opal_datatype_t *oldType,
+                                           opal_datatype_t **newType);
+static int32_t opal_datatype_create_hvector(int count, int bLength, ptrdiff_t stride,
+                                            const opal_datatype_t *oldType,
+                                            opal_datatype_t **newType);
 
 /**
  * Cache cleanup.
  */
-#define CACHE_SIZE (4*1024*1024)
-void cache_trash( void )
+#define CACHE_SIZE (4 * 1024 * 1024)
+void cache_trash(void)
 {
-    char* buffer;
+    char *buffer;
 
-    buffer = (char*)malloc( sizeof(char) * CACHE_SIZE );
-    memset( buffer, 1, CACHE_SIZE );
-    memset( buffer, 0xff, CACHE_SIZE );
-    free( buffer );
+    buffer = (char *) malloc(sizeof(char) * CACHE_SIZE);
+    memset(buffer, 1, CACHE_SIZE);
+    memset(buffer, 0xff, CACHE_SIZE);
+    free(buffer);
 }
 
-opal_datatype_t* test_create_twice_two_doubles( void )
+opal_datatype_t *test_create_twice_two_doubles(void)
 {
-    opal_datatype_t* pdt;
+    opal_datatype_t *pdt;
 
-    opal_datatype_create_vector( 2, 2, 5, &opal_datatype_float8, &pdt );
-    opal_datatype_commit( pdt );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        opal_datatype_dump( pdt );
+    opal_datatype_create_vector(2, 2, 5, &opal_datatype_float8, &pdt);
+    opal_datatype_commit(pdt);
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        opal_datatype_dump(pdt);
     }
     return pdt;
 }
 
-
 /*
   Datatype 0x832cf28 size 0 align 1 id 0 length 4 used 0
   true_lb 0 true_ub 0 (true_extent 0) lb 0 ub 0 (extent 0)
@@ -91,70 +92,68 @@ opal_datatype_t* test_create_twice_two_doubles( void )
   --C-----D*-[ C ][INT]        MPI_INT count 2 disp 0xa24 (2596) extent 4
   --C-----D*-[ C ][INT]        MPI_INT count 1 disp 0xa80 (2688) extent 4
 */
-static int blacs_length[] = { 13, 13, 13, 13, 13, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 };
-static int blacs_indices[] = { 1144/4, 1232/4, 1320/4, 1408/4, 1496/4, 1584/4, 1676/4, 1768/4,
-                               1860/4, 1952/4, 2044/4, 2136/4, 2228/4, 2320/4, 2412/4, 2504/4,
-                               2596/4, 2688/4 };
-opal_datatype_t* test_create_blacs_type( void )
+static int blacs_length[] = {13, 13, 13, 13, 13, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1};
+static int blacs_indices[] = {1144 / 4, 1232 / 4, 1320 / 4, 1408 / 4, 1496 / 4, 1584 / 4,
+                              1676 / 4, 1768 / 4, 1860 / 4, 1952 / 4, 2044 / 4, 2136 / 4,
+                              2228 / 4, 2320 / 4, 2412 / 4, 2504 / 4, 2596 / 4, 2688 / 4};
+opal_datatype_t *test_create_blacs_type(void)
 {
     opal_datatype_t *pdt;
 
-    opal_datatype_create_indexed( 18, blacs_length, blacs_indices, &opal_datatype_int4, &pdt );
-    opal_datatype_commit( pdt );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        opal_datatype_dump( pdt );
+    opal_datatype_create_indexed(18, blacs_length, blacs_indices, &opal_datatype_int4, &pdt);
+    opal_datatype_commit(pdt);
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        opal_datatype_dump(pdt);
     }
     return pdt;
 }
 
-opal_datatype_t* test_create_blacs_type1( opal_datatype_t const * const base_type )
+opal_datatype_t *test_create_blacs_type1(opal_datatype_t const *const base_type)
 {
     opal_datatype_t *pdt;
 
-    opal_datatype_create_vector( 7, 1, 3, base_type, &pdt );
-    opal_datatype_commit( pdt );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        opal_datatype_dump( pdt );
+    opal_datatype_create_vector(7, 1, 3, base_type, &pdt);
+    opal_datatype_commit(pdt);
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        opal_datatype_dump(pdt);
     }
     return pdt;
 }
 
-opal_datatype_t* test_create_blacs_type2( opal_datatype_t const * const base_type )
+opal_datatype_t *test_create_blacs_type2(opal_datatype_t const *const base_type)
 {
     opal_datatype_t *pdt;
 
-    opal_datatype_create_vector( 7, 1, 2, base_type, &pdt );
-    opal_datatype_commit( pdt );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        opal_datatype_dump( pdt );
+    opal_datatype_create_vector(7, 1, 2, base_type, &pdt);
+    opal_datatype_commit(pdt);
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        opal_datatype_dump(pdt);
     }
     return pdt;
 }
 
-
-opal_datatype_t* test_struct( void )
+opal_datatype_t *test_struct(void)
 {
-    opal_datatype_t* types[] = { (opal_datatype_t*)&opal_datatype_float4,
-                                 NULL,
-                                 (opal_datatype_t*)&opal_datatype_int1 };
-    int lengths[] = { 2, 1, 3 };
-    ptrdiff_t disp[] = { 0, 16, 26 };
-    opal_datatype_t* pdt, *pdt1;
-
-    printf( "test struct\n" );
+    opal_datatype_t *types[] = {(opal_datatype_t *) &opal_datatype_float4, NULL,
+                                (opal_datatype_t *) &opal_datatype_int1};
+    int lengths[] = {2, 1, 3};
+    ptrdiff_t disp[] = {0, 16, 26};
+    opal_datatype_t *pdt, *pdt1;
+
+    printf("test struct\n");
     opal_datatype_create_contiguous(0, &opal_datatype_empty, &pdt1);
-    opal_datatype_add( pdt1, &opal_datatype_float8, 1, 0, -1 );
-    opal_datatype_add( pdt1, &opal_datatype_int1, 1, 8, -1 );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        opal_datatype_dump( pdt1 );
+    opal_datatype_add(pdt1, &opal_datatype_float8, 1, 0, -1);
+    opal_datatype_add(pdt1, &opal_datatype_int1, 1, 8, -1);
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        opal_datatype_dump(pdt1);
     }
 
     types[1] = pdt1;
 
-    opal_datatype_create_struct( 3, lengths, disp, types, &pdt );
-    OBJ_RELEASE( pdt1 ); /*assert( pdt1 == NULL );*/
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        opal_datatype_dump( pdt );
+    opal_datatype_create_struct(3, lengths, disp, types, &pdt);
+    OBJ_RELEASE(pdt1); /*assert( pdt1 == NULL );*/
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        opal_datatype_dump(pdt);
     }
     return pdt;
 }
@@ -164,22 +163,22 @@ typedef struct __struct_char_double {
     double d;
 } char_double_t;
 
-opal_datatype_t* test_struct_char_double( void )
+opal_datatype_t *test_struct_char_double(void)
 {
     char_double_t data;
     int lengths[] = {1, 1};
     ptrdiff_t displ[] = {0, 0};
     opal_datatype_t *pdt;
-    opal_datatype_t* types[] = { (opal_datatype_t*)&opal_datatype_int1,
-                                 (opal_datatype_t*)&opal_datatype_float8};
+    opal_datatype_t *types[] = {(opal_datatype_t *) &opal_datatype_int1,
+                                (opal_datatype_t *) &opal_datatype_float8};
 
-    displ[0] = (char*)&(data.c) - (char*)&(data);
-    displ[1] = (char*)&(data.d) - (char*)&(data);
+    displ[0] = (char *) &(data.c) - (char *) &(data);
+    displ[1] = (char *) &(data.d) - (char *) &(data);
 
-    opal_datatype_create_struct( 2, lengths, displ, types, &pdt );
-    opal_datatype_commit( pdt );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        opal_datatype_dump( pdt );
+    opal_datatype_create_struct(2, lengths, displ, types, &pdt);
+    opal_datatype_commit(pdt);
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        opal_datatype_dump(pdt);
     }
     return pdt;
 }
@@ -199,57 +198,56 @@ typedef struct {
 #define SSTRANGE_CNT 10
 #define USE_RESIZED
 
-opal_datatype_t* create_strange_dt( void )
+opal_datatype_t *create_strange_dt(void)
 {
     sdata_intern v[2];
     ptrdiff_t displ[3];
     opal_datatype_t *pdt, *pdt1;
 
     opal_datatype_create_contiguous(0, &opal_datatype_empty, &pdt1);
-    opal_datatype_add( pdt1, &opal_datatype_float8, 1, 0, -1 );
-    opal_datatype_add( pdt1, &opal_datatype_int1, 1, 8, -1 );
+    opal_datatype_add(pdt1, &opal_datatype_float8, 1, 0, -1);
+    opal_datatype_add(pdt1, &opal_datatype_int1, 1, 8, -1);
 
 #ifdef USE_RESIZED
     /* optional */
     displ[0] = 0;
-    displ[1] = (char*)&(v[1]) - (char*)&(v[0]);
-    opal_datatype_resize( pdt1, displ[0], displ[1]);
-#endif  /* USE_RESIZED */
+    displ[1] = (char *) &(v[1]) - (char *) &(v[0]);
+    opal_datatype_resize(pdt1, displ[0], displ[1]);
+#endif /* USE_RESIZED */
 
-    opal_datatype_create_contiguous( SSTRANGE_CNT, pdt1, &pdt );
+    opal_datatype_create_contiguous(SSTRANGE_CNT, pdt1, &pdt);
 
-    OBJ_RELEASE( pdt1 );
-    printf( "\nStrange datatype BEFORE COMMIT\n" );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        opal_datatype_dump( pdt );
+    OBJ_RELEASE(pdt1);
+    printf("\nStrange datatype BEFORE COMMIT\n");
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        opal_datatype_dump(pdt);
     }
 
-    opal_datatype_commit( pdt );
-    printf( "\nStrange datatype AFTER COMMIT\n" );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        opal_datatype_dump( pdt );
+    opal_datatype_commit(pdt);
+    printf("\nStrange datatype AFTER COMMIT\n");
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        opal_datatype_dump(pdt);
     }
     return pdt;
 }
 
-opal_datatype_t* create_vector_type( const opal_datatype_t* data, int count, int length, int stride )
+opal_datatype_t *create_vector_type(const opal_datatype_t *data, int count, int length, int stride)
 {
-    opal_datatype_t* vector;
+    opal_datatype_t *vector;
 
-    opal_datatype_create_vector( count, length, stride, data, &vector );
-    opal_datatype_commit( vector );
+    opal_datatype_create_vector(count, length, stride, data, &vector);
+    opal_datatype_commit(vector);
     return vector;
 }
 
-
-opal_datatype_t* create_contiguous_type( const opal_datatype_t* type, int length )
+opal_datatype_t *create_contiguous_type(const opal_datatype_t *type, int length)
 {
-   opal_datatype_t* newtype;
+    opal_datatype_t *newtype;
 
-   opal_datatype_create_contiguous( length, type, &newtype );
-   opal_datatype_commit( newtype );
+    opal_datatype_create_contiguous(length, type, &newtype);
+    opal_datatype_commit(newtype);
 
-   return newtype;
+    return newtype;
 }
 
 /* Create a non-contiguous resized datatype */
@@ -259,13 +257,13 @@ struct structure {
     double transfered_2;
 };
 
-opal_datatype_t* create_struct_constant_gap_resized_ddt( const opal_datatype_t* type )
+opal_datatype_t *create_struct_constant_gap_resized_ddt(const opal_datatype_t *type)
 {
     opal_datatype_t *struct_type;
 
     opal_datatype_create_contiguous(0, &opal_datatype_empty, &struct_type);
-    opal_datatype_add( struct_type, &opal_datatype_float8, 1,  8, -1 );
-    opal_datatype_add( struct_type, &opal_datatype_float8, 1, 16, -1 );
+    opal_datatype_add(struct_type, &opal_datatype_float8, 1, 8, -1);
+    opal_datatype_add(struct_type, &opal_datatype_float8, 1, 16, -1);
 
     opal_datatype_resize(struct_type, 0, sizeof(struct structure));
     opal_datatype_commit(struct_type);
@@ -273,126 +271,127 @@ opal_datatype_t* create_struct_constant_gap_resized_ddt( const opal_datatype_t*
     return struct_type;
 }
 
-
 /*****************************************************************************/
 /* Copied Function to get test to work */
 /*****************************************************************************/
-static int32_t opal_datatype_create_indexed( int count, const int* pBlockLength, const int* pDisp,
-                                             const opal_datatype_t* oldType, opal_datatype_t** newType )
+static int32_t opal_datatype_create_indexed(int count, const int *pBlockLength, const int *pDisp,
+                                            const opal_datatype_t *oldType,
+                                            opal_datatype_t **newType)
 {
-    opal_datatype_t* pdt;
+    opal_datatype_t *pdt;
     int i, dLength, endat, disp;
     ptrdiff_t extent;
 
-    if( 0 == count ) {
-        *newType = opal_datatype_create( 0 );
-        opal_datatype_add( *newType, &opal_datatype_empty, 0, 0, 0);
+    if (0 == count) {
+        *newType = opal_datatype_create(0);
+        opal_datatype_add(*newType, &opal_datatype_empty, 0, 0, 0);
         return OPAL_SUCCESS;
     }
 
     disp = pDisp[0];
     dLength = pBlockLength[0];
     endat = disp + dLength;
-    opal_datatype_type_extent( oldType, &extent );
-    if( 1 >= count ) {
-        pdt = opal_datatype_create( oldType->desc.used + 2 );
+    opal_datatype_type_extent(oldType, &extent);
+    if (1 >= count) {
+        pdt = opal_datatype_create(oldType->desc.used + 2);
         /* multiply by count to make it zero if count is zero */
-        opal_datatype_add( pdt, oldType, count * dLength, disp * extent, extent );
+        opal_datatype_add(pdt, oldType, count * dLength, disp * extent, extent);
     } else {
-        pdt = opal_datatype_create( count * (2 + oldType->desc.used) );
-        for( i = 1; i < count; i++ ) {
-            if( endat == pDisp[i] ) {
+        pdt = opal_datatype_create(count * (2 + oldType->desc.used));
+        for (i = 1; i < count; i++) {
+            if (endat == pDisp[i]) {
                 /* contiguous with the previsious */
                 dLength += pBlockLength[i];
                 endat += pBlockLength[i];
             } else {
-                opal_datatype_add( pdt, oldType, dLength, disp * extent, extent );
+                opal_datatype_add(pdt, oldType, dLength, disp * extent, extent);
                 disp = pDisp[i];
                 dLength = pBlockLength[i];
                 endat = disp + pBlockLength[i];
             }
         }
-        opal_datatype_add( pdt, oldType, dLength, disp * extent, extent );
+        opal_datatype_add(pdt, oldType, dLength, disp * extent, extent);
     }
 
     *newType = pdt;
     return OPAL_SUCCESS;
 }
 
-static int32_t opal_datatype_create_hindexed( int count, const int* pBlockLength, const ptrdiff_t* pDisp,
-                                              const opal_datatype_t* oldType, opal_datatype_t** newType )
+static int32_t opal_datatype_create_hindexed(int count, const int *pBlockLength,
+                                             const ptrdiff_t *pDisp, const opal_datatype_t *oldType,
+                                             opal_datatype_t **newType)
 {
-    opal_datatype_t* pdt;
+    opal_datatype_t *pdt;
     int i, dLength;
     ptrdiff_t extent, disp, endat;
 
-    if( 0 == count ) {
-        *newType = opal_datatype_create( 0 );
-        opal_datatype_add( *newType, &opal_datatype_empty, 0, 0, 0);
+    if (0 == count) {
+        *newType = opal_datatype_create(0);
+        opal_datatype_add(*newType, &opal_datatype_empty, 0, 0, 0);
         return OPAL_SUCCESS;
     }
 
-    opal_datatype_type_extent( oldType, &extent );
+    opal_datatype_type_extent(oldType, &extent);
 
-    pdt = opal_datatype_create( count * (2 + oldType->desc.used) );
+    pdt = opal_datatype_create(count * (2 + oldType->desc.used));
     disp = pDisp[0];
     dLength = pBlockLength[0];
     endat = disp + dLength * extent;
-    if( 1 >= count ) {
-        pdt = opal_datatype_create( oldType->desc.used + 2 );
+    if (1 >= count) {
+        pdt = opal_datatype_create(oldType->desc.used + 2);
         /* multiply by count to make it zero if count is zero */
-        opal_datatype_add( pdt, oldType, count * dLength, disp, extent );
+        opal_datatype_add(pdt, oldType, count * dLength, disp, extent);
     } else {
-        for( i = 1; i < count; i++ ) {
-            if( endat == pDisp[i] ) {
+        for (i = 1; i < count; i++) {
+            if (endat == pDisp[i]) {
                 /* contiguous with the previsious */
                 dLength += pBlockLength[i];
                 endat += pBlockLength[i] * extent;
             } else {
-                opal_datatype_add( pdt, oldType, dLength, disp, extent );
+                opal_datatype_add(pdt, oldType, dLength, disp, extent);
                 disp = pDisp[i];
                 dLength = pBlockLength[i];
                 endat = disp + pBlockLength[i] * extent;
             }
         }
-        opal_datatype_add( pdt, oldType, dLength, disp, extent );
+        opal_datatype_add(pdt, oldType, dLength, disp, extent);
     }
     *newType = pdt;
     return OPAL_SUCCESS;
 }
 
-
-static int32_t opal_datatype_create_struct( int count, const int* pBlockLength, const ptrdiff_t* pDisp,
-                                            opal_datatype_t** pTypes, opal_datatype_t** newType )
+static int32_t opal_datatype_create_struct(int count, const int *pBlockLength,
+                                           const ptrdiff_t *pDisp, opal_datatype_t **pTypes,
+                                           opal_datatype_t **newType)
 {
     int i;
-    ptrdiff_t disp = 0, endto, lastExtent, lastDisp;
-    int lastBlock;
+    ptrdiff_t disp = 0, endto, lastExtent, lastDisp, lastBlock;
     opal_datatype_t *pdt, *lastType;
 
-    if( 0 == count ) {
-        *newType = opal_datatype_create( 0 );
-        opal_datatype_add( *newType, &opal_datatype_empty, 0, 0, 0);
+    if (0 == count) {
+        *newType = opal_datatype_create(0);
+        opal_datatype_add(*newType, &opal_datatype_empty, 0, 0, 0);
         return OPAL_SUCCESS;
     }
 
     /* if we compute the total number of elements before we can
      * avoid increasing the size of the desc array often.
      */
-    lastType = (opal_datatype_t*)pTypes[0];
+    lastType = (opal_datatype_t *) pTypes[0];
     lastBlock = pBlockLength[0];
     lastExtent = lastType->ub - lastType->lb;
     lastDisp = pDisp[0];
     endto = pDisp[0] + lastExtent * lastBlock;
 
-    for( i = 1; i < count; i++ ) {
-        if( (pTypes[i] == lastType) && (pDisp[i] == endto) ) {
+    for (i = 1; i < count; i++) {
+        if ((pTypes[i] == lastType) && (pDisp[i] == endto)) {
             lastBlock += pBlockLength[i];
             endto = lastDisp + lastBlock * lastExtent;
         } else {
             disp += lastType->desc.used;
-            if( lastBlock > 1 ) disp += 2;
-            lastType = (opal_datatype_t*)pTypes[i];
+            if (lastBlock > 1)
+                disp += 2;
+            lastType = (opal_datatype_t *) pTypes[i];
             lastExtent = lastType->ub - lastType->lb;
             lastBlock = pBlockLength[i];
             lastDisp = pDisp[i];
@@ -400,94 +399,94 @@ static int32_t opal_datatype_create_struct( int count, const int* pBlockLength,
         }
     }
     disp += lastType->desc.used;
-    if( lastBlock != 1 ) disp += 2;
+    if (lastBlock != 1)
+        disp += 2;
 
-    lastType = (opal_datatype_t*)pTypes[0];
+    lastType = (opal_datatype_t *) pTypes[0];
     lastBlock = pBlockLength[0];
     lastExtent = lastType->ub - lastType->lb;
     lastDisp = pDisp[0];
     endto = pDisp[0] + lastExtent * lastBlock;
 
-    pdt = opal_datatype_create( (int32_t)disp );
+    pdt = opal_datatype_create((int32_t) disp);
 
     /* Do again the same loop but now add the elements */
-    for( i = 1; i < count; i++ ) {
-        if( (pTypes[i] == lastType) && (pDisp[i] == endto) ) {
+    for (i = 1; i < count; i++) {
+        if ((pTypes[i] == lastType) && (pDisp[i] == endto)) {
             lastBlock += pBlockLength[i];
             endto = lastDisp + lastBlock * lastExtent;
         } else {
-            opal_datatype_add( pdt, lastType, lastBlock, lastDisp, lastExtent );
-            lastType = (opal_datatype_t*)pTypes[i];
+            opal_datatype_add(pdt, lastType, lastBlock, lastDisp, lastExtent);
+            lastType = (opal_datatype_t *) pTypes[i];
             lastExtent = lastType->ub - lastType->lb;
             lastBlock = pBlockLength[i];
             lastDisp = pDisp[i];
             endto = lastDisp + lastExtent * lastBlock;
         }
     }
-    opal_datatype_add( pdt, lastType, lastBlock, lastDisp, lastExtent );
+    opal_datatype_add(pdt, lastType, lastBlock, lastDisp, lastExtent);
 
-     *newType = pdt;
+    *newType = pdt;
     return OPAL_SUCCESS;
 }
 
-
-static int32_t opal_datatype_create_vector( int count, int bLength, int stride,
-                                            const opal_datatype_t* oldType, opal_datatype_t** newType )
+static int32_t opal_datatype_create_vector(int count, int bLength, int stride,
+                                           const opal_datatype_t *oldType,
+                                           opal_datatype_t **newType)
 {
     opal_datatype_t *pTempData, *pData;
     ptrdiff_t extent = oldType->ub - oldType->lb;
 
-
-    if( 0 == count ) {
-        *newType = opal_datatype_create( 0 );
-        opal_datatype_add( *newType, &opal_datatype_empty, 0, 0, 0);
+    if (0 == count) {
+        *newType = opal_datatype_create(0);
+        opal_datatype_add(*newType, &opal_datatype_empty, 0, 0, 0);
         return OPAL_SUCCESS;
     }
 
-    pData = opal_datatype_create( oldType->desc.used + 2 );
-    if( (bLength == stride) || (1 == count) ) {  /* the elements are contiguous */
-        opal_datatype_add( pData, oldType, count * bLength, 0, extent );
+    pData = opal_datatype_create(oldType->desc.used + 2);
+    if ((bLength == stride) || (1 == count)) { /* the elements are contiguous */
+        opal_datatype_add(pData, oldType, count * bLength, 0, extent);
     } else {
-        if( 1 == bLength ) {
-            opal_datatype_add( pData, oldType, count, 0, extent * stride );
+        if (1 == bLength) {
+            opal_datatype_add(pData, oldType, count, 0, extent * stride);
         } else {
-            opal_datatype_add( pData, oldType, bLength, 0, extent );
+            opal_datatype_add(pData, oldType, bLength, 0, extent);
             pTempData = pData;
-            pData = opal_datatype_create( oldType->desc.used + 2 + 2 );
-            opal_datatype_add( pData, pTempData, count, 0, extent * stride );
-            OBJ_RELEASE( pTempData );
+            pData = opal_datatype_create(oldType->desc.used + 2 + 2);
+            opal_datatype_add(pData, pTempData, count, 0, extent * stride);
+            OBJ_RELEASE(pTempData);
         }
     }
     *newType = pData;
     return OPAL_SUCCESS;
 }
 
-
-static int32_t opal_datatype_create_hvector( int count, int bLength, ptrdiff_t stride,
-                                             const opal_datatype_t* oldType, opal_datatype_t** newType )
+static int32_t opal_datatype_create_hvector(int count, int bLength, ptrdiff_t stride,
+                                            const opal_datatype_t *oldType,
+                                            opal_datatype_t **newType)
 {
     opal_datatype_t *pTempData, *pData;
     ptrdiff_t extent = oldType->ub - oldType->lb;
 
-    if( 0 == count ) {
-        *newType = opal_datatype_create( 0 );
-        opal_datatype_add( *newType, &opal_datatype_empty, 0, 0, 0);
+    if (0 == count) {
+        *newType = opal_datatype_create(0);
+        opal_datatype_add(*newType, &opal_datatype_empty, 0, 0, 0);
         return OPAL_SUCCESS;
     }
 
-    pTempData = opal_datatype_create( oldType->desc.used + 2 );
-    if( ((extent * bLength) == stride) || (1 == count) ) {  /* contiguous */
+    pTempData = opal_datatype_create(oldType->desc.used + 2);
+    if (((extent * bLength) == stride) || (1 == count)) { /* contiguous */
         pData = pTempData;
-        opal_datatype_add( pData, oldType, count * bLength, 0, extent );
+        opal_datatype_add(pData, oldType, count * bLength, 0, extent);
     } else {
-        if( 1 == bLength ) {
+        if (1 == bLength) {
             pData = pTempData;
-            opal_datatype_add( pData, oldType, count, 0, stride );
+            opal_datatype_add(pData, oldType, count, 0, stride);
         } else {
-            opal_datatype_add( pTempData, oldType, bLength, 0, extent );
-            pData = opal_datatype_create( oldType->desc.used + 2 + 2 );
-            opal_datatype_add( pData, pTempData, count, 0, stride );
-            OBJ_RELEASE( pTempData );
+            opal_datatype_add(pTempData, oldType, bLength, 0, extent);
+            pData = opal_datatype_create(oldType->desc.used + 2 + 2);
+            opal_datatype_add(pData, pTempData, count, 0, stride);
+            OBJ_RELEASE(pTempData);
         }
     }
     *newType = pData;
@@ -495,143 +494,132 @@ static int32_t opal_datatype_create_hvector( int count, int bLength, ptrdiff_t s
 }
 
 /*****************************************************************************/
-int init_random_upper_matrix( unsigned int N, double* mat )
+int init_random_upper_matrix(unsigned int N, double *mat)
 {
     unsigned int i, j;
 
-    srand( time(NULL) );
-    for( i = 0; i < N; i++ ) {
+    srand(time(NULL));
+    for (i = 0; i < N; i++) {
         mat += i;
-        for( j = i; j < N; j++ ) {
-            *mat = (double)random();
+        for (j = i; j < N; j++) {
+            *mat = (double) random();
             mat++;
         }
     }
     return OPAL_SUCCESS;
 }
 
-int check_diag_matrix( unsigned int N, double* mat1, double* mat2 )
+int check_diag_matrix(unsigned int N, double *mat1, double *mat2)
 {
-   unsigned int i, j;
-
-   for( i = 0; i < N; i++ ) {
-      mat1 += i;
-      mat2 += i;
-      for( j = i; j < N; j++ ) {
-         if( *mat1 != *mat2 ) {
-            printf( "error in position (%d, %d) expect %f and find %f\n",
-                    i, j, *mat1, *mat2 );
-            printf( "hex %lx != %lx\n", *(long*)mat1, *(long*)mat2 );
-            return OPAL_ERROR;
-         }
-         mat1++; mat2++;
-      }
-   }
-   return OPAL_SUCCESS;
-}
+    unsigned int i, j;
 
+    for (i = 0; i < N; i++) {
+        mat1 += i;
+        mat2 += i;
+        for (j = i; j < N; j++) {
+            if (*mat1 != *mat2) {
+                printf("error in position (%d, %d) expect %f and find %f\n", i, j, *mat1, *mat2);
+                printf("hex %lx != %lx\n", *(long *) mat1, *(long *) mat2);
+                return OPAL_ERROR;
+            }
+            mat1++;
+            mat2++;
+        }
+    }
+    return OPAL_SUCCESS;
+}
 
-opal_datatype_t* upper_matrix( unsigned int mat_size )
+opal_datatype_t *upper_matrix(unsigned int mat_size)
 {
     int *disp, *blocklen;
     unsigned int i;
-    opal_datatype_t* upper;
+    opal_datatype_t *upper;
 
-    disp = (int*)malloc( sizeof(int) * mat_size );
-    blocklen = (int*)malloc( sizeof(int) * mat_size );
+    disp = (int *) malloc(sizeof(int) * mat_size);
+    blocklen = (int *) malloc(sizeof(int) * mat_size);
 
-    for( i = 0; i < mat_size; i++ ) {
+    for (i = 0; i < mat_size; i++) {
         disp[i] = i * mat_size + i;
         blocklen[i] = mat_size - i;
     }
 
 #if SIZEOF_DOUBLE == 4
-    opal_datatype_create_indexed( mat_size, blocklen, disp,
-                                  &opal_datatype_float4,
-                                  &upper );
+    opal_datatype_create_indexed(mat_size, blocklen, disp, &opal_datatype_float4, &upper);
 #else
-    opal_datatype_create_indexed( mat_size, blocklen, disp,
-                                  &opal_datatype_float8,
-                                  &upper );
+    opal_datatype_create_indexed(mat_size, blocklen, disp, &opal_datatype_float8, &upper);
 #endif
-    opal_datatype_commit( upper );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        opal_datatype_dump( upper );
+    opal_datatype_commit(upper);
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        opal_datatype_dump(upper);
     }
-    free( disp );
-    free( blocklen );
+    free(disp);
+    free(blocklen);
     return upper;
 }
 
-opal_datatype_t* lower_matrix( unsigned int mat_size )
+opal_datatype_t *lower_matrix(unsigned int mat_size)
 {
     int *disp, *blocklen;
     unsigned int i;
-    opal_datatype_t* upper;
+    opal_datatype_t *upper;
 
-    disp = (int*)malloc( sizeof(int) * mat_size );
-    blocklen = (int*)malloc( sizeof(int) * mat_size );
+    disp = (int *) malloc(sizeof(int) * mat_size);
+    blocklen = (int *) malloc(sizeof(int) * mat_size);
 
-    for( i = 0; i < mat_size; i++ ) {
+    for (i = 0; i < mat_size; i++) {
         disp[i] = i * mat_size;
         blocklen[i] = i;
     }
 #if SIZEOF_DOUBLE == 4
-    opal_datatype_create_indexed( mat_size, blocklen, disp, &opal_datatype_float4,
-                                  &upper );
+    opal_datatype_create_indexed(mat_size, blocklen, disp, &opal_datatype_float4, &upper);
 #else
-    opal_datatype_create_indexed( mat_size, blocklen, disp, &opal_datatype_float8,
-                                  &upper );
+    opal_datatype_create_indexed(mat_size, blocklen, disp, &opal_datatype_float8, &upper);
 #endif
-    free( disp );
-    free( blocklen );
+    free(disp);
+    free(blocklen);
     return upper;
 }
 
-opal_datatype_t* test_matrix_borders( unsigned int size, unsigned int width )
+opal_datatype_t *test_matrix_borders(unsigned int size, unsigned int width)
 {
-   opal_datatype_t *pdt, *pdt_line;
-   int disp[2];
-   int blocklen[2];
-
-   disp[0] = 0;
-   blocklen[0] = width;
-   disp[1] = (size - width) * sizeof(double);
-   blocklen[1] = width;
-
-   opal_datatype_create_indexed( 2, blocklen, disp, &opal_datatype_float8,
-                                 &pdt_line );
-   opal_datatype_create_contiguous( size, pdt_line, &pdt );
-   OBJ_RELEASE( pdt_line ); /*assert( pdt_line == NULL );*/
-   return pdt;
+    opal_datatype_t *pdt, *pdt_line;
+    int disp[2];
+    int blocklen[2];
+
+    disp[0] = 0;
+    blocklen[0] = width;
+    disp[1] = (size - width) * sizeof(double);
+    blocklen[1] = width;
+
+    opal_datatype_create_indexed(2, blocklen, disp, &opal_datatype_float8, &pdt_line);
+    opal_datatype_create_contiguous(size, pdt_line, &pdt);
+    OBJ_RELEASE(pdt_line); /*assert( pdt_line == NULL );*/
+    return pdt;
 }
 
-
-opal_datatype_t* test_contiguous( void )
+opal_datatype_t *test_contiguous(void)
 {
     opal_datatype_t *pdt, *pdt1, *pdt2;
 
-    printf( "test contiguous (alignment)\n" );
+    printf("test contiguous (alignment)\n");
     opal_datatype_create_contiguous(0, &opal_datatype_empty, &pdt1);
-    opal_datatype_add( pdt1, &opal_datatype_float8, 1, 0, -1 );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        opal_datatype_dump( pdt1 );
+    opal_datatype_add(pdt1, &opal_datatype_float8, 1, 0, -1);
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        opal_datatype_dump(pdt1);
     }
-    opal_datatype_add( pdt1, &opal_datatype_int1, 1, 8, -1 );
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        opal_datatype_dump( pdt1 );
+    opal_datatype_add(pdt1, &opal_datatype_int1, 1, 8, -1);
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        opal_datatype_dump(pdt1);
     }
-    opal_datatype_create_contiguous( 4, pdt1, &pdt2 );
-    OBJ_RELEASE( pdt1 ); /*assert( pdt1 == NULL );*/
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        opal_datatype_dump( pdt2 );
+    opal_datatype_create_contiguous(4, pdt1, &pdt2);
+    OBJ_RELEASE(pdt1); /*assert( pdt1 == NULL );*/
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        opal_datatype_dump(pdt2);
     }
-    opal_datatype_create_contiguous( 2, pdt2, &pdt );
-    OBJ_RELEASE( pdt2 ); /*assert( pdt2 == NULL );*/
-    if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
-        opal_datatype_dump( pdt );
+    opal_datatype_create_contiguous(2, pdt2, &pdt);
+    OBJ_RELEASE(pdt2); /*assert( pdt2 == NULL );*/
+    if (outputFlags & DUMP_DATA_AFTER_COMMIT) {
+        opal_datatype_dump(pdt);
     }
     return pdt;
 }
-
-
diff --git a/test/datatype/opal_ddt_lib.h b/test/datatype/opal_ddt_lib.h
index ca7e84ab5b8..f32437825ac 100644
--- a/test/datatype/opal_ddt_lib.h
+++ b/test/datatype/opal_ddt_lib.h
@@ -18,10 +18,10 @@
 
 #include "opal/datatype/opal_datatype.h"
 
-
 #define TIMER_DATA_TYPE struct timeval
-#define GET_TIME(TV)   gettimeofday( &(TV), NULL )
-#define ELAPSED_TIME(TSTART, TEND)  (((TEND).tv_sec - (TSTART).tv_sec) * 1000000 + ((TEND).tv_usec - (TSTART).tv_usec))
+#define GET_TIME(TV)    gettimeofday(&(TV), NULL)
+#define ELAPSED_TIME(TSTART, TEND) \
+    (((TEND).tv_sec - (TSTART).tv_sec) * 1000000 + ((TEND).tv_usec - (TSTART).tv_usec))
 
 #define DUMP_DATA_AFTER_COMMIT 0x00000001
 #define CHECK_PACK_UNPACK      0x00000002
@@ -31,23 +31,24 @@
 
 extern uint32_t outputFlags;
 
-extern void cache_trash( void );
-extern opal_datatype_t* create_contiguous_type( opal_datatype_t const * const type, int length );
-extern opal_datatype_t* create_vector_type( opal_datatype_t const* const data, int count, int length, int stride );
-extern opal_datatype_t* create_strange_dt( void );
-extern opal_datatype_t* upper_matrix( unsigned int mat_size );
-extern opal_datatype_t* lower_matrix( unsigned int mat_size );
-extern opal_datatype_t* test_matrix_borders( unsigned int size, unsigned int width );
-extern int init_random_upper_matrix( unsigned int N, double* mat );
-extern int check_diag_matrix( unsigned int N, double* mat1, double* mat2 );
-extern opal_datatype_t* test_contiguous( void );
-extern opal_datatype_t* test_struct_char_double( void );
-extern opal_datatype_t* test_create_twice_two_doubles( void );
-extern opal_datatype_t* test_struct( void );
-extern opal_datatype_t* test_create_blacs_type( void );
-extern opal_datatype_t* test_create_blacs_type1( opal_datatype_t const * const base_type );
-extern opal_datatype_t* test_create_blacs_type2( opal_datatype_t const * const base_type );
-
-extern opal_datatype_t* create_struct_constant_gap_resized_ddt( const opal_datatype_t* type );
+extern void cache_trash(void);
+extern opal_datatype_t *create_contiguous_type(opal_datatype_t const *const type, int length);
+extern opal_datatype_t *create_vector_type(opal_datatype_t const *const data, int count, int length,
+                                           int stride);
+extern opal_datatype_t *create_strange_dt(void);
+extern opal_datatype_t *upper_matrix(unsigned int mat_size);
+extern opal_datatype_t *lower_matrix(unsigned int mat_size);
+extern opal_datatype_t *test_matrix_borders(unsigned int size, unsigned int width);
+extern int init_random_upper_matrix(unsigned int N, double *mat);
+extern int check_diag_matrix(unsigned int N, double *mat1, double *mat2);
+extern opal_datatype_t *test_contiguous(void);
+extern opal_datatype_t *test_struct_char_double(void);
+extern opal_datatype_t *test_create_twice_two_doubles(void);
+extern opal_datatype_t *test_struct(void);
+extern opal_datatype_t *test_create_blacs_type(void);
+extern opal_datatype_t *test_create_blacs_type1(opal_datatype_t const *const base_type);
+extern opal_datatype_t *test_create_blacs_type2(opal_datatype_t const *const base_type);
+
+extern opal_datatype_t *create_struct_constant_gap_resized_ddt(const opal_datatype_t *type);
 
 #endif /* TEST_OPAL_DDT_LIB_H */
diff --git a/test/datatype/partial.c b/test/datatype/partial.c
new file mode 100644
index 00000000000..9721bf5388a
--- /dev/null
+++ b/test/datatype/partial.c
@@ -0,0 +1,181 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2018-2020 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2018      Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2022      Cisco Systems, Inc.  All rights reserved
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "opal/datatype/opal_convertor.h"
+#include "opal/datatype/opal_datatype_checksum.h"
+#include "opal/runtime/opal.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+
+#define TYPE_COUNT  3
+#define TYPE_BLEN   2
+#define TYPE_STRIDE 4
+
+#define CONT_COUNT 2
+
+#define COUNT 3
+
+#define CHUNK ((TYPE_BLEN * 8) * 2 - 4)
+
+/**
+ * Print how many elements on both sides of ptr.
+ */
+static void show_neighborhood(double *ptr, int how_many, bool show_hex)
+{
+    int i;
+
+    printf("%12p: ", (void *) ptr);
+    for (i = -how_many; i < how_many; i++) {
+        if (0 == i) {
+            printf(" <%g> ", ptr[i]);
+        } else {
+            printf("  %g  ", ptr[i]);
+        }
+    }
+    if (show_hex) {
+        char *cptr = (char *) ptr;
+        printf("\n            : ");
+        for (i = -how_many; i < how_many; i++) {
+            if (0 == i)
+                printf(" <");
+            for (size_t j = 0; j < sizeof(double); j++) {
+                printf("%02x", cptr[i * sizeof(double) + j]);
+            }
+            if (0 == i)
+                printf("> ");
+            else
+                printf(" ");
+        }
+    }
+    printf("\n\n");
+}
+
+/**
+ * -------G---[---][---]    OPAL_LOOP_S 19 times the next 2 elements extent 18432
+ * -cC---P-DB-[---][---]    OPAL_FLOAT8 count 72 disp 0x80 (128) blen 16 extent 256 (size 9216)
+ * -------G---[---][---]    OPAL_LOOP_E prev 2 elements first elem displacement 128 size of data
+ * 9216
+ * -------G---[---][---]    OPAL_LOOP_E prev 3 elements first elem displacement 128 size of data
+ * 175104
+ */
+
+int main(int argc, char *argv[])
+{
+    ompi_datatype_t *vector;
+    ompi_datatype_t *base;
+    uint32_t iov_count;
+    size_t max_data, size, length;
+    struct iovec iov[2];
+    opal_convertor_t *convertor;
+    ptrdiff_t extent, base_extent;
+    double *array, *packed;
+    char *bpacked;
+    int i, j;
+
+    opal_init(NULL, NULL);
+    ompi_datatype_init();
+
+    ompi_datatype_create_vector(TYPE_COUNT, TYPE_BLEN, TYPE_STRIDE, MPI_DOUBLE, &base);
+    ompi_datatype_create_contiguous(CONT_COUNT, base, &vector);
+
+    opal_datatype_commit(&vector->super);
+
+    ompi_datatype_dump(vector);
+
+    opal_datatype_type_size(&vector->super, &size);
+    opal_datatype_type_extent(&vector->super, &extent);
+    opal_datatype_type_extent(&base->super, &base_extent);
+
+    array = (double *) malloc(extent * COUNT);
+    packed = (double *) malloc(size * COUNT);
+    bpacked = (char *) packed;
+
+    /**
+     * Initialize the sparse data using the index.
+     */
+    for (i = 0; i < (TYPE_BLEN * TYPE_COUNT * CONT_COUNT * COUNT); i++) {
+        packed[i] = (double) (i % TYPE_BLEN);
+    }
+    memset(array, extent * COUNT, TYPE_BLEN + 1);
+
+    /**
+     * Pack the sparse data into the packed array. This simulate the first step
+     * of the buffered operation.
+     */
+    convertor = opal_convertor_create(opal_local_arch, 0);
+    opal_convertor_prepare_for_recv(convertor, &vector->super, COUNT, array);
+
+    for (length = 0; length < (size * COUNT);) {
+        iov[0].iov_base = bpacked + length;
+        iov[0].iov_len = CHUNK;
+        max_data = iov[0].iov_len;
+
+        iov_count = 1;
+        opal_convertor_unpack(convertor, iov, &iov_count, &max_data);
+        length += max_data;
+
+        int idx = 0;
+        size_t checked = 0;
+        for (int m = 0; m < COUNT; m++) {
+            char *mptr = (char *) array + m * extent;
+            for (int k = 0; k < CONT_COUNT; k++) {
+                char *kptr = mptr + k * base_extent;
+                for (j = 0; j < TYPE_COUNT; j++) {
+                    double *jarray = (double *) kptr + j * TYPE_STRIDE;
+                    for (i = 0; i < TYPE_BLEN; i++) {
+                        checked += sizeof(double);
+                        if (checked > length)
+                            goto next_iteration;
+                        if (jarray[i] != (double) (idx % TYPE_BLEN)) {
+                            fprintf(
+                                stderr,
+                                "\n\n\nError during check for the %d element, length %" PRIsize_t
+                                " (chunk %d)\n",
+                                idx, length, CHUNK);
+                            fprintf(stderr,
+                                    "Error at position %d [%d:%d:%d:%d] found %g expected %g\n\n\n",
+                                    idx, m, k, j, i, jarray[i], (double) (idx % TYPE_BLEN));
+                            show_neighborhood(jarray + i, 4, true);
+                            exit(-1);
+                        }
+                        idx++;
+                    }
+                }
+            }
+        }
+    next_iteration:
+        /* nothing special to do here, just move to the next conversion */
+        continue;
+    }
+
+    OBJ_RELEASE(convertor);
+
+    /**
+     * The datatype is not useful anymore
+     */
+    OBJ_RELEASE(vector);
+
+    free(array);
+    free(packed);
+
+    /* clean-ups all data allocations */
+    opal_finalize_util();
+
+    return 0;
+}
diff --git a/test/datatype/position.c b/test/datatype/position.c
index c92adebcbaa..94809e07a10 100644
--- a/test/datatype/position.c
+++ b/test/datatype/position.c
@@ -18,10 +18,10 @@
 #include <stdio.h>
 #include <string.h>
 
-#include "opal/datatype/opal_convertor.h"
 #include "ompi/datatype/ompi_datatype.h"
-#include "opal/util/output.h"
+#include "opal/datatype/opal_convertor.h"
 #include "opal/runtime/opal.h"
+#include "opal/util/output.h"
 
 /**
  * The purpose of this test is to simulate the multi-network packing and
@@ -36,63 +36,60 @@ static int data_count = 2048;
 typedef struct {
     size_t position;
     size_t size;
-    void* buffer;
+    void *buffer;
 } ddt_segment_t;
 
-static int
-create_segments( ompi_datatype_t* datatype, int count,
-                 size_t segment_length,
-                 ddt_segment_t** segments, int* seg_count )
+static int create_segments(ompi_datatype_t *datatype, int count, size_t segment_length,
+                           ddt_segment_t **segments, int *seg_count)
 {
     size_t data_size, total_length, position;
-    opal_convertor_t* convertor;
+    opal_convertor_t *convertor;
     int i;
-    ddt_segment_t* segment;
+    ddt_segment_t *segment;
 
-    ompi_datatype_type_size( datatype, &data_size );
+    ompi_datatype_type_size(datatype, &data_size);
     data_size *= count;
     *seg_count = data_size / segment_length;
-    if( ((*seg_count) * segment_length) != data_size )
+    if (((*seg_count) * segment_length) != data_size)
         *seg_count += 1;
- allocate_segments:
-    *segments = (ddt_segment_t*)malloc( (*seg_count) * sizeof(ddt_segment_t) );
+allocate_segments:
+    *segments = (ddt_segment_t *) malloc((*seg_count) * sizeof(ddt_segment_t));
 
-    convertor = opal_convertor_create( opal_local_arch, 0 );
-    opal_convertor_prepare_for_send( convertor, &(datatype->super), count, NULL );
+    convertor = opal_convertor_create(opal_local_arch, 0);
+    opal_convertor_prepare_for_send(convertor, &(datatype->super), count, NULL);
 
     position = 0;
     total_length = 0;
-    for( i = 0; i < (*seg_count); i++ ) {
+    for (i = 0; i < (*seg_count); i++) {
         segment = &((*segments)[i]);
         segment->buffer = malloc(segment_length);
         segment->position = position;
 
         /* Find the end of the segment */
         position += segment_length;
-        opal_convertor_set_position( convertor, &position );
+        opal_convertor_set_position(convertor, &position);
         segment->size = position - segment->position;
         total_length += segment->size;
     }
     OBJ_RELEASE(convertor);
-    if( total_length != data_size ) {
-        for( i = 0; i < (*seg_count); i++ ) {
+    if (total_length != data_size) {
+        for (i = 0; i < (*seg_count); i++) {
             segment = &((*segments)[i]);
             free(segment->buffer);
         }
-        free( *segments );
+        free(*segments);
         (*seg_count) += 1;
         goto allocate_segments;
     }
     return 0;
 }
 
-static int
-shuffle_segments( ddt_segment_t* segments, int seg_count )
+static int shuffle_segments(ddt_segment_t *segments, int seg_count)
 {
     ddt_segment_t temporary;
     int i;
 
-    for( i = 0; i < (seg_count/2); i += 2 ) {
+    for (i = 0; i < (seg_count / 2); i += 2) {
         temporary = segments[i];
         segments[i] = segments[seg_count - i - 1];
         segments[seg_count - i - 1] = temporary;
@@ -100,81 +97,75 @@ shuffle_segments( ddt_segment_t* segments, int seg_count )
     return 0;
 }
 
-static int
-pack_segments( ompi_datatype_t* datatype, int count,
-               size_t segment_size,
-               ddt_segment_t* segments, int seg_count,
-               void* buffer )
+static int pack_segments(ompi_datatype_t *datatype, int count, size_t segment_size,
+                         ddt_segment_t *segments, int seg_count, void *buffer)
 {
     size_t max_size, position;
-    opal_convertor_t* convertor;
+    opal_convertor_t *convertor;
     struct iovec iov;
     int i;
     uint32_t iov_count;
 
-    convertor = opal_convertor_create( opal_local_arch, 0 );
-    opal_convertor_prepare_for_send( convertor, &(datatype->super), count, buffer );
+    convertor = opal_convertor_create(opal_local_arch, 0);
+    opal_convertor_prepare_for_send(convertor, &(datatype->super), count, buffer);
 
-    for( i = 0; i < seg_count; i++ ) {
-        iov.iov_len  = segments[i].size;
+    for (i = 0; i < seg_count; i++) {
+        iov.iov_len = segments[i].size;
         iov.iov_base = segments[i].buffer;
         max_size = iov.iov_len;
         position = segments[i].position;
-        opal_convertor_set_position( convertor, &position );
-        if( position != segments[i].position ) {
-            opal_output( 0, "Setting position failed (%lu != %lu)\n",
-                         (unsigned long)segments[i].position, (unsigned long)position );
+        opal_convertor_set_position(convertor, &position);
+        if (position != segments[i].position) {
+            opal_output(0, "Setting position failed (%lu != %lu)\n",
+                        (unsigned long) segments[i].position, (unsigned long) position);
             break;
         }
 
         iov_count = 1;
-        opal_convertor_pack( convertor, &iov, &iov_count, &max_size );
-        if( max_size != segments[i].size ) {
-            opal_output( 0, "Amount of packed data do not match (%lu != %lu)\n",
-                         (unsigned long)max_size, (unsigned long)segments[i].size );
-            opal_output( 0, "Segment %d position %lu size %lu\n", i,
-                         (unsigned long)segments[i].position, segments[i].size );
+        opal_convertor_pack(convertor, &iov, &iov_count, &max_size);
+        if (max_size != segments[i].size) {
+            opal_output(0, "Amount of packed data do not match (%lu != %lu)\n",
+                        (unsigned long) max_size, (unsigned long) segments[i].size);
+            opal_output(0, "Segment %d position %lu size %lu\n", i,
+                        (unsigned long) segments[i].position, segments[i].size);
         }
     }
     OBJ_RELEASE(convertor);
     return i;
 }
 
-static int
-unpack_segments( ompi_datatype_t* datatype, int count,
-                 size_t segment_size,
-                 ddt_segment_t* segments, int seg_count,
-                 void* buffer )
+static int unpack_segments(ompi_datatype_t *datatype, int count, size_t segment_size,
+                           ddt_segment_t *segments, int seg_count, void *buffer)
 {
-    opal_convertor_t* convertor;
+    opal_convertor_t *convertor;
     size_t max_size, position;
     int i;
     uint32_t iov_count;
     struct iovec iov;
 
-    convertor = opal_convertor_create( opal_local_arch, 0 );
-    opal_convertor_prepare_for_recv( convertor, &(datatype->super), count, buffer );
+    convertor = opal_convertor_create(opal_local_arch, 0);
+    opal_convertor_prepare_for_recv(convertor, &(datatype->super), count, buffer);
 
-    for( i = 0; i < seg_count; i++ ) {
+    for (i = 0; i < seg_count; i++) {
         iov.iov_len = segments[i].size;
         iov.iov_base = segments[i].buffer;
         max_size = iov.iov_len;
 
         position = segments[i].position;
-        opal_convertor_set_position( convertor, &position );
-        if( position != segments[i].position ) {
-            opal_output( 0, "Setting position failed (%lu != %lu)\n",
-                         (unsigned long)segments[i].position, (unsigned long)position );
+        opal_convertor_set_position(convertor, &position);
+        if (position != segments[i].position) {
+            opal_output(0, "Setting position failed (%lu != %lu)\n",
+                        (unsigned long) segments[i].position, (unsigned long) position);
             break;
         }
 
         iov_count = 1;
-        opal_convertor_unpack( convertor, &iov, &iov_count, &max_size );
-        if( max_size != segments[i].size ) {
-            opal_output( 0, "Amount of unpacked data do not match (%lu != %lu)\n",
-                         (unsigned long)max_size, (unsigned long)segments[i].size );
-            opal_output( 0, "Segment %d position %lu size %lu\n", i,
-                         (unsigned long)segments[i].position, segments[i].size );
+        opal_convertor_unpack(convertor, &iov, &iov_count, &max_size);
+        if (max_size != segments[i].size) {
+            opal_output(0, "Amount of unpacked data do not match (%lu != %lu)\n",
+                        (unsigned long) max_size, (unsigned long) segments[i].size);
+            opal_output(0, "Segment %d position %lu size %lu\n", i,
+                        (unsigned long) segments[i].position, segments[i].size);
         }
     }
     OBJ_RELEASE(convertor);
@@ -183,7 +174,7 @@ unpack_segments( ompi_datatype_t* datatype, int count,
 
 typedef struct {
     long double ld;
-    int         i;
+    int i;
 } ddt_ldi_t;
 
 #if 0
@@ -202,86 +193,81 @@ static void dump_ldi( ddt_ldi_t* buffer, int start_pos, int end_pos )
 #if (OPAL_ENABLE_DEBUG == 1) && (OPAL_C_HAVE_VISIBILITY == 0)
 extern bool opal_ddt_unpack_debug;
 extern bool opal_ddt_pack_debug;
-extern bool opal_ddt_position_debug ;
-#endif  /* OPAL_ENABLE_DEBUG */
+extern bool opal_ddt_position_debug;
+#endif /* OPAL_ENABLE_DEBUG */
 
-static char* bytes_dump( void* src, size_t cnt )
+static char *bytes_dump(void *src, size_t cnt)
 {
     static char text[1024];
     int index, i;
 
-    index = sprintf( text, "0x" );
-    for( i = 0; i < (int)cnt; i++ )
-        index += sprintf( text + index, "%x", (int)(((char*)src)[i]) );
+    index = sprintf(text, "0x");
+    for (i = 0; i < (int) cnt; i++)
+        index += sprintf(text + index, "%x", (int) (((char *) src)[i]));
     *(text + index) = '\0';
     return text;
 }
 
-int main( int argc, char* argv[] )
+int main(int argc, char *argv[])
 {
-    ddt_segment_t* segments;
+    ddt_segment_t *segments;
     ddt_ldi_t *send_buffer, *recv_buffer;
     int i, seg_count, errors;
     int show_only_first_error = 1;
-    ompi_datatype_t* datatype = MPI_LONG_DOUBLE_INT;
+    ompi_datatype_t *datatype = MPI_LONG_DOUBLE_INT;
 
-    send_buffer = malloc( sizeof(ddt_ldi_t) * data_count );
-    recv_buffer = malloc( sizeof(ddt_ldi_t) * data_count );
+    send_buffer = malloc(sizeof(ddt_ldi_t) * data_count);
+    recv_buffer = malloc(sizeof(ddt_ldi_t) * data_count);
 
-    for( i = 0; i < data_count; i++ ) {
-        send_buffer[i].ld = (long double)i + (long double)i / 100000.0;
-        send_buffer[i].i  = i;
+    for (i = 0; i < data_count; i++) {
+        send_buffer[i].ld = (long double) i + (long double) i / 100000.0;
+        send_buffer[i].i = i;
     }
-    memcpy(recv_buffer, send_buffer, sizeof(ddt_ldi_t) * data_count );
+    memcpy(recv_buffer, send_buffer, sizeof(ddt_ldi_t) * data_count);
 
-    opal_init_util (NULL, NULL);
+    opal_init(NULL, NULL);
     ompi_datatype_init();
 
 #if (OPAL_ENABLE_DEBUG == 1) && (OPAL_C_HAVE_VISIBILITY == 0)
-    opal_ddt_unpack_debug   = false;
-    opal_ddt_pack_debug     = false;
+    opal_ddt_unpack_debug = false;
+    opal_ddt_pack_debug = false;
     opal_ddt_position_debug = false;
-#endif  /* OPAL_ENABLE_DEBUG */
+#endif /* OPAL_ENABLE_DEBUG */
 
-    create_segments( datatype, data_count, fragment_size,
-                     &segments, &seg_count );
+    create_segments(datatype, data_count, fragment_size, &segments, &seg_count);
 
     /* shuffle the segments */
-    shuffle_segments( segments, seg_count );
+    shuffle_segments(segments, seg_count);
 
     /* pack the data */
-    pack_segments( datatype, data_count, fragment_size, segments, seg_count,
-                   send_buffer );
+    pack_segments(datatype, data_count, fragment_size, segments, seg_count, send_buffer);
 
     /* unpack the data back in the user space (recv buffer) */
-    unpack_segments( datatype, data_count, fragment_size, segments, seg_count,
-                     recv_buffer );
+    unpack_segments(datatype, data_count, fragment_size, segments, seg_count, recv_buffer);
 
     /* And now check the data */
-    for( errors = i = 0; i < data_count; i++ ) {
+    for (errors = i = 0; i < data_count; i++) {
         /*if( !bytes_equal(&send_buffer[i].ld, &recv_buffer[i].ld, sizeof(long double)) ||*/
-        if( (send_buffer[i].ld != recv_buffer[i].ld) ||
-            (send_buffer[i].i != recv_buffer[i].i) ) {
-            if( (show_only_first_error && (0 == errors)) ||
-                !show_only_first_error ) {
-                printf( "error at %4d [*(%s,%d)\n"
-                        "             != (%s,%d)\n", i,
-                        bytes_dump( &send_buffer[i].ld, sizeof(long double)), send_buffer[i].i,
-                        bytes_dump( &recv_buffer[i].ld, sizeof(long double)), recv_buffer[i].i );
+        if ((send_buffer[i].ld != recv_buffer[i].ld) || (send_buffer[i].i != recv_buffer[i].i)) {
+            if ((show_only_first_error && (0 == errors)) || !show_only_first_error) {
+                printf("error at %4d [*(%s,%d)\n"
+                       "             != (%s,%d)\n",
+                       i, bytes_dump(&send_buffer[i].ld, sizeof(long double)), send_buffer[i].i,
+                       bytes_dump(&recv_buffer[i].ld, sizeof(long double)), recv_buffer[i].i);
             }
             errors++;
         }
     }
-    printf( "Found %d errors\n", errors );
-    free(send_buffer); free(recv_buffer);
+    printf("Found %d errors\n", errors);
+    free(send_buffer);
+    free(recv_buffer);
 
-    for( i = 0; i < seg_count; i++ ) {
-        free( segments[i].buffer );
+    for (i = 0; i < seg_count; i++) {
+        free(segments[i].buffer);
     }
     free(segments);
 
-    ompi_datatype_finalize();
-    opal_finalize_util ();
+    opal_finalize_util();
 
     return (0 == errors ? 0 : -1);
 }
diff --git a/test/datatype/position_noncontig.c b/test/datatype/position_noncontig.c
index 65a7a16e114..5d52dbb0243 100644
--- a/test/datatype/position_noncontig.c
+++ b/test/datatype/position_noncontig.c
@@ -18,10 +18,10 @@
 #include <stdio.h>
 #include <string.h>
 
-#include "opal/datatype/opal_convertor.h"
 #include "ompi/datatype/ompi_datatype.h"
-#include "opal/util/output.h"
+#include "opal/datatype/opal_convertor.h"
 #include "opal/runtime/opal.h"
+#include "opal/util/output.h"
 
 /**
  * The purpose of this test is to simulate the multi-network packing and
@@ -35,63 +35,60 @@ static int fragment_size = 113;
 typedef struct {
     size_t position;
     size_t size;
-    void* buffer;
+    void *buffer;
 } ddt_segment_t;
 
-static int
-create_segments( ompi_datatype_t* datatype, int count,
-                 size_t segment_length,
-                 ddt_segment_t** segments, int* seg_count )
+static int create_segments(ompi_datatype_t *datatype, int count, size_t segment_length,
+                           ddt_segment_t **segments, int *seg_count)
 {
     size_t data_size, total_length, position;
-    opal_convertor_t* convertor;
+    opal_convertor_t *convertor;
     int i;
-    ddt_segment_t* segment;
+    ddt_segment_t *segment;
 
-    ompi_datatype_type_size( datatype, &data_size );
+    ompi_datatype_type_size(datatype, &data_size);
     data_size *= count;
     *seg_count = data_size / segment_length;
-    if( ((*seg_count) * segment_length) != data_size )
+    if (((*seg_count) * segment_length) != data_size)
         *seg_count += 1;
- allocate_segments:
-    *segments = (ddt_segment_t*)malloc( (*seg_count) * sizeof(ddt_segment_t) );
+allocate_segments:
+    *segments = (ddt_segment_t *) malloc((*seg_count) * sizeof(ddt_segment_t));
 
-    convertor = opal_convertor_create( opal_local_arch, 0 );
-    opal_convertor_prepare_for_send( convertor, &(datatype->super), count, NULL );
+    convertor = opal_convertor_create(opal_local_arch, 0);
+    opal_convertor_prepare_for_send(convertor, &(datatype->super), count, NULL);
 
     position = 0;
     total_length = 0;
-    for( i = 0; i < (*seg_count); i++ ) {
+    for (i = 0; i < (*seg_count); i++) {
         segment = &((*segments)[i]);
         segment->buffer = malloc(segment_length);
         segment->position = position;
 
         /* Find the end of the segment */
         position += segment_length;
-        opal_convertor_set_position( convertor, &position );
+        opal_convertor_set_position(convertor, &position);
         segment->size = position - segment->position;
         total_length += segment->size;
     }
     OBJ_RELEASE(convertor);
-    if( total_length != data_size ) {
-        for( i = 0; i < (*seg_count); i++ ) {
+    if (total_length != data_size) {
+        for (i = 0; i < (*seg_count); i++) {
             segment = &((*segments)[i]);
             free(segment->buffer);
         }
-        free( *segments );
+        free(*segments);
         (*seg_count) += 1;
         goto allocate_segments;
     }
     return 0;
 }
 
-static int
-shuffle_segments( ddt_segment_t* segments, int seg_count )
+static int shuffle_segments(ddt_segment_t *segments, int seg_count)
 {
     ddt_segment_t temporary;
     int i;
 
-    for( i = 0; i < (seg_count/2); i += 2 ) {
+    for (i = 0; i < (seg_count / 2); i += 2) {
         temporary = segments[i];
         segments[i] = segments[seg_count - i - 1];
         segments[seg_count - i - 1] = temporary;
@@ -99,81 +96,75 @@ shuffle_segments( ddt_segment_t* segments, int seg_count )
     return 0;
 }
 
-static int
-pack_segments( ompi_datatype_t* datatype, int count,
-               size_t segment_size,
-               ddt_segment_t* segments, int seg_count,
-               void* buffer )
+static int pack_segments(ompi_datatype_t *datatype, int count, size_t segment_size,
+                         ddt_segment_t *segments, int seg_count, void *buffer)
 {
     size_t max_size, position;
-    opal_convertor_t* convertor;
+    opal_convertor_t *convertor;
     struct iovec iov;
     int i;
     uint32_t iov_count;
 
-    convertor = opal_convertor_create( opal_local_arch, 0 );
-    opal_convertor_prepare_for_send( convertor, &(datatype->super), count, buffer );
+    convertor = opal_convertor_create(opal_local_arch, 0);
+    opal_convertor_prepare_for_send(convertor, &(datatype->super), count, buffer);
 
-    for( i = 0; i < seg_count; i++ ) {
-        iov.iov_len  = segments[i].size;
+    for (i = 0; i < seg_count; i++) {
+        iov.iov_len = segments[i].size;
         iov.iov_base = segments[i].buffer;
         max_size = iov.iov_len;
         position = segments[i].position;
-        opal_convertor_set_position( convertor, &position );
-        if( position != segments[i].position ) {
-            opal_output( 0, "Setting position failed (%lu != %lu)\n",
-                         (unsigned long)segments[i].position, (unsigned long)position );
+        opal_convertor_set_position(convertor, &position);
+        if (position != segments[i].position) {
+            opal_output(0, "Setting position failed (%lu != %lu)\n",
+                        (unsigned long) segments[i].position, (unsigned long) position);
             break;
         }
 
         iov_count = 1;
-        opal_convertor_pack( convertor, &iov, &iov_count, &max_size );
-        if( max_size != segments[i].size ) {
-            opal_output( 0, "Amount of packed data do not match (%lu != %lu)\n",
-                         (unsigned long)max_size, (unsigned long)segments[i].size );
-            opal_output( 0, "Segment %d position %lu size %lu\n", i,
-                         (unsigned long)segments[i].position, segments[i].size );
+        opal_convertor_pack(convertor, &iov, &iov_count, &max_size);
+        if (max_size != segments[i].size) {
+            opal_output(0, "Amount of packed data do not match (%lu != %lu)\n",
+                        (unsigned long) max_size, (unsigned long) segments[i].size);
+            opal_output(0, "Segment %d position %lu size %lu\n", i,
+                        (unsigned long) segments[i].position, segments[i].size);
         }
     }
     OBJ_RELEASE(convertor);
     return i;
 }
 
-static int
-unpack_segments( ompi_datatype_t* datatype, int count,
-                 size_t segment_size,
-                 ddt_segment_t* segments, int seg_count,
-                 void* buffer )
+static int unpack_segments(ompi_datatype_t *datatype, int count, size_t segment_size,
+                           ddt_segment_t *segments, int seg_count, void *buffer)
 {
-    opal_convertor_t* convertor;
+    opal_convertor_t *convertor;
     size_t max_size, position;
     int i;
     uint32_t iov_count;
     struct iovec iov;
 
-    convertor = opal_convertor_create( opal_local_arch, 0 );
-    opal_convertor_prepare_for_recv( convertor, &(datatype->super), count, buffer );
+    convertor = opal_convertor_create(opal_local_arch, 0);
+    opal_convertor_prepare_for_recv(convertor, &(datatype->super), count, buffer);
 
-    for( i = 0; i < seg_count; i++ ) {
+    for (i = 0; i < seg_count; i++) {
         iov.iov_len = segments[i].size;
         iov.iov_base = segments[i].buffer;
         max_size = iov.iov_len;
 
         position = segments[i].position;
-        opal_convertor_set_position( convertor, &position );
-        if( position != segments[i].position ) {
-            opal_output( 0, "Setting position failed (%lu != %lu)\n",
-                         (unsigned long)segments[i].position, (unsigned long)position );
+        opal_convertor_set_position(convertor, &position);
+        if (position != segments[i].position) {
+            opal_output(0, "Setting position failed (%lu != %lu)\n",
+                        (unsigned long) segments[i].position, (unsigned long) position);
             break;
         }
 
         iov_count = 1;
-        opal_convertor_unpack( convertor, &iov, &iov_count, &max_size );
-        if( max_size != segments[i].size ) {
-            opal_output( 0, "Amount of unpacked data do not match (%lu != %lu)\n",
-                         (unsigned long)max_size, (unsigned long)segments[i].size );
-            opal_output( 0, "Segment %d position %lu size %lu\n", i,
-                         (unsigned long)segments[i].position, segments[i].size );
+        opal_convertor_unpack(convertor, &iov, &iov_count, &max_size);
+        if (max_size != segments[i].size) {
+            opal_output(0, "Amount of unpacked data do not match (%lu != %lu)\n",
+                        (unsigned long) max_size, (unsigned long) segments[i].size);
+            opal_output(0, "Segment %d position %lu size %lu\n", i,
+                        (unsigned long) segments[i].position, segments[i].size);
         }
     }
     OBJ_RELEASE(convertor);
@@ -183,72 +174,68 @@ unpack_segments( ompi_datatype_t* datatype, int count,
 #if (OPAL_ENABLE_DEBUG == 1) && (OPAL_C_HAVE_VISIBILITY == 0)
 extern bool opal_ddt_unpack_debug;
 extern bool opal_ddt_pack_debug;
-extern bool opal_ddt_position_debug ;
-#endif  /* OPAL_ENABLE_DEBUG */
+extern bool opal_ddt_position_debug;
+#endif /* OPAL_ENABLE_DEBUG */
 
-int main( int argc, char* argv[] )
+int main(int argc, char *argv[])
 {
-    ddt_segment_t* segments;
+    ddt_segment_t *segments;
     int *send_buffer, *recv_buffer;
     int i, seg_count, errors;
     int show_only_first_error = 1;
-    ompi_datatype_t* datatype = MPI_DATATYPE_NULL;
+    ompi_datatype_t *datatype = MPI_DATATYPE_NULL;
 
 #define NELT (300)
-    send_buffer = malloc(NELT*sizeof(int));
-    recv_buffer = malloc(NELT*sizeof(int));
+    send_buffer = malloc(NELT * sizeof(int));
+    recv_buffer = malloc(NELT * sizeof(int));
     for (i = 0; i < NELT; ++i) {
         send_buffer[i] = i;
         recv_buffer[i] = 0xdeadbeef;
     }
 
-    opal_init_util (NULL, NULL);
+    opal_init(NULL, NULL);
     ompi_datatype_init();
 
-    ompi_datatype_create_vector(NELT/2, 1, 2, MPI_INT, &datatype);
+    ompi_datatype_create_vector(NELT / 2, 1, 2, MPI_INT, &datatype);
     ompi_datatype_commit(&datatype);
 
 #if (OPAL_ENABLE_DEBUG == 1) && (OPAL_C_HAVE_VISIBILITY == 0)
-    opal_ddt_unpack_debug   = false;
-    opal_ddt_pack_debug     = false;
+    opal_ddt_unpack_debug = false;
+    opal_ddt_pack_debug = false;
     opal_ddt_position_debug = false;
-#endif  /* OPAL_ENABLE_DEBUG */
+#endif /* OPAL_ENABLE_DEBUG */
 
-    create_segments( datatype, 1, fragment_size,
-                     &segments, &seg_count );
+    create_segments(datatype, 1, fragment_size, &segments, &seg_count);
 
     /* shuffle the segments */
-    shuffle_segments( segments, seg_count );
+    shuffle_segments(segments, seg_count);
 
     /* pack the data */
-    pack_segments( datatype, 1, fragment_size, segments, seg_count,
-                   send_buffer );
+    pack_segments(datatype, 1, fragment_size, segments, seg_count, send_buffer);
 
     /* unpack the data back in the user space (recv buffer) */
-    unpack_segments( datatype, 1, fragment_size, segments, seg_count,
-                     recv_buffer );
+    unpack_segments(datatype, 1, fragment_size, segments, seg_count, recv_buffer);
 
     /* And now check the data */
-    for( errors = i = 0; i < NELT; i++ ) {
-        int expected = ((i % 2) ? (int)0xdeadbeef : i);
+    for (errors = i = 0; i < NELT; i++) {
+        int expected = ((i % 2) ? (int) 0xdeadbeef : i);
         if (recv_buffer[i] != expected) {
-            if( (show_only_first_error && (0 == errors)) ||
-                !show_only_first_error ) {
+            if ((show_only_first_error && (0 == errors)) || !show_only_first_error) {
                 printf("error at index %4d: 0x%08x != 0x%08x\n", i, recv_buffer[i], expected);
             }
             errors++;
         }
     }
-    printf( "Found %d errors\n", errors );
-    free(send_buffer); free(recv_buffer);
+    printf("Found %d errors\n", errors);
+    free(send_buffer);
+    free(recv_buffer);
 
-    for( i = 0; i < seg_count; i++ ) {
-        free( segments[i].buffer );
+    for (i = 0; i < seg_count; i++) {
+        free(segments[i].buffer);
     }
     free(segments);
 
-    ompi_datatype_finalize();
-    opal_finalize_util ();
+    opal_finalize_util();
 
     return (0 == errors ? 0 : -1);
 }
diff --git a/test/datatype/reduce_local.c b/test/datatype/reduce_local.c
index f227439b714..9f1c06d4ba0 100644
--- a/test/datatype/reduce_local.c
+++ b/test/datatype/reduce_local.c
@@ -5,6 +5,7 @@
  *                         reserved.
  * Copyright (c) 2020      Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2024      Stony Brook University.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -12,103 +13,106 @@
  * $HEADER$
  */
 
+/* needed for strsep() */
+#define _DEFAULT_SOURCE
+#define _BSD_SOURCE
+
+/* needed for posix_memalign() and getopt() */
+#define _POSIX_C_SOURCE 200809L
+
+#include "ompi_config.h"
+#include <stdbool.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <sys/time.h>
-#include <stdbool.h>
-#include <stdint.h>
 #include <unistd.h>
 
 #include "mpi.h"
 #include "ompi/communicator/communicator.h"
-#include "ompi/runtime/mpiruntime.h"
 #include "ompi/datatype/ompi_datatype.h"
+#include "ompi/runtime/mpiruntime.h"
 
 typedef struct op_name_s {
-    char* name;
-    char* mpi_op_name;
+    char *name;
+    char *mpi_op_name;
     MPI_Op op;
 } op_name_t;
-static op_name_t array_of_ops [] = {
-    { "max", "MPI_MAX", MPI_MAX },
-    { "min", "MPI_MIN", MPI_MIN },
-    { "sum", "MPI_SUM", MPI_SUM },
-    { "prod", "MPI_PROD", MPI_PROD },
-    { "land", "MPI_LAND", MPI_LAND },
-    { "band", "MPI_BAND", MPI_BAND },
-    { "lor", "MPI_LOR", MPI_LOR },
-    { "bor", "MPI_BOR", MPI_BOR },
-    { "lxor", "MPI_LXOR", MPI_LXOR },
-    { "bxor", "MPI_BXOR", MPI_BXOR },
-    { "replace", "MPI_REPLACE", MPI_REPLACE },
-    { NULL, "MPI_OP_NULL", MPI_OP_NULL }
+static op_name_t array_of_ops[] = {
+    {"max", "MPI_MAX", MPI_MAX},
+    {"min", "MPI_MIN", MPI_MIN},
+    {"sum", "MPI_SUM", MPI_SUM},
+    {"prod", "MPI_PROD", MPI_PROD},
+    {"land", "MPI_LAND", MPI_LAND},
+    {"band", "MPI_BAND", MPI_BAND},
+    {"lor", "MPI_LOR", MPI_LOR},
+    {"bor", "MPI_BOR", MPI_BOR},
+    {"lxor", "MPI_LXOR", MPI_LXOR},
+    {"bxor", "MPI_BXOR", MPI_BXOR},
+    {"replace", "MPI_REPLACE", MPI_REPLACE},
+    {NULL, "MPI_OP_NULL", MPI_OP_NULL},
 };
-static int do_ops[12] = { -1, };  /* index of the ops to do. Size +1 larger than the array_of_ops */
+static int do_ops[12] = {
+    -1,
+}; /* index of the ops to do. Size +1 larger than the array_of_ops */
 static int verbose = 0;
 static int total_errors = 0;
 
-#define max(a,b) \
-   ({ __typeof__ (a) _a = (a); \
-       __typeof__ (b) _b = (b); \
-     _a > _b ? _a : _b; })
+#define max(a, b) (a) > (b) ? (a) : (b)
 
-#define min(a,b) \
-   ({ __typeof__ (a) _a = (a); \
-       __typeof__ (b) _b = (b); \
-     _a < _b ? _a : _b; })
+#define min(a, b) (a) < (b) ? (a) : (b)
 
-static void print_status(char* op, char* type, int type_size,
-                         int count, int max_shift, double *duration, int repeats,
-                         int correct )
+static void print_status(char *op, char *type, int type_size, int count, int max_shift,
+                         double *duration, int repeats, int correct)
 {
-    if(correct) {
-        printf("%-10s %s %-10d%s ", op, type, type_size, (verbose ? " [\033[1;32msuccess\033[0m]" : ""));
+    if (correct) {
+        printf("%-10s %s %-10d%s ", op, type, type_size,
+               (verbose ? " [\033[1;32msuccess\033[0m]" : ""));
     } else {
         printf("%-10s %s [\033[1;31mfail\033[0m]", op, type);
         total_errors++;
     }
-    if( 1 == max_shift ) {
+    if (1 == max_shift) {
         printf(" count  %-10d  time (seconds) %.8f seconds\n", count, duration[0] / repeats);
     } else {
         printf(" count  %-10d  time (seconds / shifts) ", count);
-        for( int i = 0; i < max_shift; i++ ) {
-            printf("%.8f ", duration[i] / repeats );
+        for (int i = 0; i < max_shift; i++) {
+            printf("%.8f ", duration[i] / repeats);
         }
         printf("\n");
     }
 }
 
 static int do_ops_built = 0;
-static int
-build_do_ops( char* optarg, int* do_ops)
+static int build_do_ops(char *optarg, int *do_ops)
 {
     int i;
-    if( 0 == strcmp(optarg, "all") ) {
-        for( i = 0; NULL != array_of_ops[i].name; i++ ) {
+    if (0 == strcmp(optarg, "all")) {
+        for (i = 0; NULL != array_of_ops[i].name; i++) {
             do_ops[i] = i;
         }
-        do_ops[i] = -1;  /* stop */
+        do_ops[i] = -1; /* stop */
     } else {
         int n, idx = 0;
-        char* token, *arg = optarg;
+        char *token, *arg = optarg;
         while ((token = strsep(&arg, ",")) != NULL) {
-            for( i = 0; NULL != array_of_ops[i].name; i++ ) {  /* find the op */
-                if( 0 == strcmp(array_of_ops[i].name, token) ) {
+            for (i = 0; NULL != array_of_ops[i].name; i++) { /* find the op */
+                if (0 == strcmp(array_of_ops[i].name, token)) {
                     /* check if the op was not already selected */
-                    for(n = 0; n < idx; n++ ) {
-                        if( i == do_ops[n] ) {
+                    for (n = 0; n < idx; n++) {
+                        if (i == do_ops[n]) {
                             break;
                         }
                     }
-                    if( n >= idx ) {
+                    if (n >= idx) {
                         do_ops[idx++] = i;
-                        do_ops[idx]   = -1;
+                        do_ops[idx] = -1;
                     }
                     break;
                 }
             }
-            if( NULL == array_of_ops[i].name ) {
+            if (NULL == array_of_ops[i].name) {
                 fprintf(stderr, "Unknown op %s. Ignored.\n", token);
             }
         }
@@ -117,7 +121,7 @@ build_do_ops( char* optarg, int* do_ops)
     return 0;
 }
 
-
+/* clang-format off */
 #define MPI_OP_TEST(OPNAME, MPIOP, MPITYPE, TYPE, INBUF, INOUT_BUF, CHECK_BUF, COUNT, TYPE_PREFIX) \
 do { \
     const TYPE *_p1 = ((TYPE*)(INBUF)), *_p3 = ((TYPE*)(CHECK_BUF)); \
@@ -156,7 +160,7 @@ do { \
         for(int _r = repeats; _r > 0; _r--) { \
             memcpy(_p2, _p3, sizeof(TYPE) * (COUNT)); \
             tstart = MPI_Wtime(); \
-            MPI_Reduce_local(_p1+_k, _p2+_k, (COUNT), (MPITYPE), (MPIOP)); \
+            MPI_Reduce_local(_p1+_k, _p2+_k, (COUNT)-_k, (MPITYPE), (MPIOP)); \
             tend = MPI_Wtime(); \
             duration[_k] += (tend - tstart); \
             if( check ) { \
@@ -174,6 +178,7 @@ do { \
     } \
     goto check_and_continue; \
 } while (0)
+/* clang-format on */
 
 int main(int argc, char **argv)
 {
@@ -184,28 +189,28 @@ int main(int argc, char **argv)
     double *duration, tstart, tend;
     bool check = true;
     char type[5] = "uifd", *op = "sum", *mpi_type;
-    int lower = 1, upper = 1000000, skip_op_type;
+    int lower = 1, upper = 16*1024*1024, skip_op_type;
     MPI_Op mpi_op;
 
-    while( -1 != (c = getopt(argc, argv, "l:u:r:t:o:i:s:n:1:2:vfh")) ) {
-        switch(c) {
+    while (-1 != (c = getopt(argc, argv, "l:u:r:t:o:i:s:n:1:2:vfh"))) {
+        switch (c) {
         case 'l':
             lower = atoi(optarg);
-            if( lower <= 0 ) {
+            if (lower <= 0) {
                 fprintf(stderr, "The lower number of elements must be positive\n");
                 exit(-1);
             }
             break;
         case 'u':
             upper = atoi(optarg);
-            if( lower <= 0 ) {
+            if (lower <= 0) {
                 fprintf(stderr, "The upper number of elements must be positive\n");
                 exit(-1);
             }
             break;
         case 'i':
             max_shift = atoi(optarg);
-            if( max_shift <= 0 ) {
+            if (max_shift <= 0) {
                 fprintf(stderr, "The max shift must be positive\n");
                 exit(-1);
             }
@@ -218,27 +223,30 @@ int main(int argc, char **argv)
             break;
         case 'r':
             repeats = atoi(optarg);
-            if( repeats <= 0 ) {
+            if (repeats <= 0) {
                 fprintf(stderr, "The number of repetitions (%d) must be positive\n", repeats);
                 exit(-1);
             }
             break;
         case 't':
-            for( i = 0; i < (int)strlen(optarg); i++ ) {
-                if( ! (('i' == optarg[i]) || ('u' == optarg[i]) ||
-                       ('f' == optarg[i]) || ('d' == optarg[i])) ) {
-                    fprintf(stderr, "type must be i (signed int), u (unsigned int), f (float) or d (double)\n");
+            for (i = 0; i < (int) strlen(optarg); i++) {
+                if (!(('i' == optarg[i]) || ('u' == optarg[i]) || ('f' == optarg[i])
+                      || ('d' == optarg[i]))) {
+                    fprintf(
+                        stderr,
+                        "type must be i (signed int), u (unsigned int), f (float) or d (double)\n");
                     exit(-1);
                 }
             }
             strncpy(type, optarg, 4);
             break;
         case 'o':
-            build_do_ops( optarg, do_ops);
+            build_do_ops(optarg, do_ops);
             break;
         case 's':
             type_size = atoi(optarg);
-            if( ! ((8 == type_size) || (16 == type_size) || (32 == type_size) || (64 == type_size)) ) {
+            if (!((8 == type_size) || (16 == type_size) || (32 == type_size)
+                  || (64 == type_size))) {
                 fprintf(stderr, "type_size must be 8, 16, 32 or 64. %d is an invalid value\n",
                         type_size);
                 exit(-1);
@@ -246,20 +254,21 @@ int main(int argc, char **argv)
             break;
         case '1':
             op1_alignment = atoi(optarg);
-            if( op1_alignment < 0 ) {
+            if (op1_alignment < 0) {
                 fprintf(stderr, "alignment for the first operand must be positive\n");
                 exit(-1);
             }
             break;
         case '2':
             res_alignment = atoi(optarg);
-            if( res_alignment < 0 ) {
+            if (res_alignment < 0) {
                 fprintf(stderr, "alignment for the result must be positive\n");
                 exit(-1);
             }
             break;
         case 'h':
-            fprintf(stdout, "%s options are:\n"
+            fprintf(stdout,
+                    "%s options are:\n"
                     " -l <number> : lower number of elements\n"
                     " -u <number> : upper number of elements\n"
                     " -s <type_size> : 8, 16, 32 or 64 bits elements\n"
@@ -271,476 +280,435 @@ int main(int argc, char **argv)
                     " -1 <number> : (mis)alignment in elements for the first op\n"
                     " -2 <number> : (mis)alignment in elements for the result\n"
                     " -v: increase the verbosity level\n"
-                    " -h: this help message\n", argv[0]);
+                    " -f: turn off correctness checks\n"
+                    " -h: this help message\n",
+                    argv[0]);
             exit(0);
         }
     }
 
-    if( !do_ops_built ) {  /* not yet done, take the default */
-            build_do_ops( "all", do_ops);
+    if (!do_ops_built) { /* not yet done, take the default */
+        build_do_ops("all", do_ops);
     }
-    posix_memalign( &in_buf,          64, (upper + op1_alignment) * sizeof(double));
-    posix_memalign( &inout_buf,       64, (upper + res_alignment) * sizeof(double));
-    posix_memalign( &inout_check_buf, 64, upper * sizeof(double));
-    duration = (double*)malloc(max_shift * sizeof(double));
+    posix_memalign(&in_buf, 64, (upper + op1_alignment) * sizeof(double));
+    posix_memalign(&inout_buf, 64, (upper + res_alignment) * sizeof(double));
+    posix_memalign(&inout_check_buf, 64, upper * sizeof(double));
+    duration = (double *) malloc(max_shift * sizeof(double));
 
     ompi_mpi_init(argc, argv, MPI_THREAD_SERIALIZED, &provided, false);
 
-    rank = ompi_comm_rank(MPI_COMM_WORLD); (void)rank;
-    size = ompi_comm_size(MPI_COMM_WORLD); (void)size;
+    rank = ompi_comm_rank(MPI_COMM_WORLD);
+    (void) rank;
+    size = ompi_comm_size(MPI_COMM_WORLD);
+    (void) size;
 
-    for(uint32_t type_idx = 0; type_idx < strlen(type); type_idx++ ) {
-        for(uint32_t op_idx = 0; do_ops[op_idx] >= 0; op_idx++ ) {
-            op     = array_of_ops[do_ops[op_idx]].name;
+    for (uint32_t type_idx = 0; type_idx < strlen(type); type_idx++) {
+        for (uint32_t op_idx = 0; do_ops[op_idx] >= 0; op_idx++) {
+            op = array_of_ops[do_ops[op_idx]].name;
             mpi_op = array_of_ops[do_ops[op_idx]].op;
             skip_op_type = 1;
 
-            for( count = lower; count <= upper; count += count ) {
+            for (count = lower; count <= upper; count += count) {
                 mpi_type = NULL;
                 correctness = 1;
-                if('i' == type[type_idx]) {
-                    if( 8 == type_size ) {
-                        int8_t *in_int8 = (int8_t*)((char*)in_buf + op1_alignment * sizeof(int8_t)),
-                            *inout_int8 = (int8_t*)((char*)inout_buf + res_alignment * sizeof(int8_t)),
-                            *inout_int8_for_check = (int8_t*)inout_check_buf;
-                        for( i = 0; i < count; i++ ) {
+                if ('i' == type[type_idx]) {
+                    if (8 == type_size) {
+                        int8_t *in_int8 = (int8_t *) ((char *) in_buf
+                                                      + op1_alignment * sizeof(int8_t)),
+                               *inout_int8 = (int8_t *) ((char *) inout_buf
+                                                         + res_alignment * sizeof(int8_t)),
+                               *inout_int8_for_check = (int8_t *) inout_check_buf;
+                        for (i = 0; i < count; i++) {
                             in_int8[i] = 5;
                             inout_int8[i] = inout_int8_for_check[i] = -3;
                         }
                         mpi_type = "MPI_INT8_T";
 
-                        if( 0 == strcmp(op, "sum") ) {
-                            MPI_OP_TEST( +, mpi_op, MPI_INT8_T, int8_t,
-                                         in_int8, inout_int8, inout_int8_for_check,
-                                         count, PRId8);
-                        }
-                        if( 0 == strcmp(op, "bor") ) {
-                            MPI_OP_TEST( |, mpi_op, MPI_INT8_T, int8_t,
-                                         in_int8, inout_int8, inout_int8_for_check,
-                                         count, PRId8);
-                        }
-                        if( 0 == strcmp(op, "bxor") ) {
-                            MPI_OP_TEST( ^, mpi_op, MPI_INT8_T, int8_t,
-                                         in_int8, inout_int8, inout_int8_for_check,
-                                         count, PRId8);
-                        }
-                        if( 0 == strcmp(op, "prod") ) {
-                            MPI_OP_TEST( *, mpi_op, MPI_INT8_T, int8_t,
-                                         in_int8, inout_int8, inout_int8_for_check,
-                                         count, PRId8);
-                        }
-                        if( 0 == strcmp(op, "band") ) {
-                            MPI_OP_TEST( &, mpi_op, MPI_INT8_T, int8_t,
-                                         in_int8, inout_int8, inout_int8_for_check,
-                                         count, PRId8);
-                        }
-                        if( 0 == strcmp(op, "max") ) {
-                            MPI_OP_MINMAX_TEST(max, mpi_op,  MPI_INT8_T, int8_t,
-                                               in_int8, inout_int8, inout_int8_for_check,
-                                               count, PRId8);
-                        }
-                        if( 0 == strcmp(op, "min") ) {  //intentionly reversed in and out
-                            MPI_OP_MINMAX_TEST(min, mpi_op,  MPI_INT8_T, int8_t,
-                                               in_int8, inout_int8, inout_int8_for_check,
-                                               count, PRId8);
+                        if (0 == strcmp(op, "sum")) {
+                            MPI_OP_TEST(+, mpi_op, MPI_INT8_T, int8_t, in_int8, inout_int8,
+                                        inout_int8_for_check, count, PRId8);
+                        }
+                        if (0 == strcmp(op, "bor")) {
+                            MPI_OP_TEST(|, mpi_op, MPI_INT8_T, int8_t, in_int8, inout_int8,
+                                        inout_int8_for_check, count, PRId8);
+                        }
+                        if (0 == strcmp(op, "bxor")) {
+                            MPI_OP_TEST(^, mpi_op, MPI_INT8_T, int8_t, in_int8, inout_int8,
+                                        inout_int8_for_check, count, PRId8);
+                        }
+                        if (0 == strcmp(op, "prod")) {
+                            MPI_OP_TEST(*, mpi_op, MPI_INT8_T, int8_t, in_int8, inout_int8,
+                                        inout_int8_for_check, count, PRId8);
+                        }
+                        if (0 == strcmp(op, "band")) {
+                            MPI_OP_TEST(&, mpi_op, MPI_INT8_T, int8_t, in_int8, inout_int8,
+                                        inout_int8_for_check, count, PRId8);
+                        }
+                        if (0 == strcmp(op, "max")) {
+                            MPI_OP_MINMAX_TEST(max, mpi_op, MPI_INT8_T, int8_t, in_int8, inout_int8,
+                                               inout_int8_for_check, count, PRId8);
+                        }
+                        if (0 == strcmp(op, "min")) { // intentionally reversed in and out
+                            MPI_OP_MINMAX_TEST(min, mpi_op, MPI_INT8_T, int8_t, in_int8, inout_int8,
+                                               inout_int8_for_check, count, PRId8);
                         }
                     }
-                    if( 16 == type_size ) {
-                        int16_t *in_int16 = (int16_t*)((char*)in_buf + op1_alignment * sizeof(int16_t)),
-                            *inout_int16 = (int16_t*)((char*)inout_buf + res_alignment * sizeof(int16_t)),
-                            *inout_int16_for_check = (int16_t*)inout_check_buf;
-                        for( i = 0; i < count; i++ ) {
+                    if (16 == type_size) {
+                        int16_t *in_int16 = (int16_t *) ((char *) in_buf
+                                                         + op1_alignment * sizeof(int16_t)),
+                                *inout_int16 = (int16_t *) ((char *) inout_buf
+                                                            + res_alignment * sizeof(int16_t)),
+                                *inout_int16_for_check = (int16_t *) inout_check_buf;
+                        for (i = 0; i < count; i++) {
                             in_int16[i] = 5;
                             inout_int16[i] = inout_int16_for_check[i] = -3;
                         }
                         mpi_type = "MPI_INT16_T";
 
-                        if( 0 == strcmp(op, "sum") ) {
-                            MPI_OP_TEST( +, mpi_op, MPI_INT16_T, int16_t,
-                                         in_int16, inout_int16, inout_int16_for_check,
-                                         count, PRId16);
-                        }
-                        if( 0 == strcmp(op, "bor") ) {
-                            MPI_OP_TEST( |, mpi_op, MPI_INT16_T, int16_t,
-                                         in_int16, inout_int16, inout_int16_for_check,
-                                         count, PRId16);
-                        }
-                        if( 0 == strcmp(op, "bxor") ) {
-                            MPI_OP_TEST( ^, mpi_op, MPI_INT16_T, int16_t,
-                                         in_int16, inout_int16, inout_int16_for_check,
-                                         count, PRId16);
-                        }
-                        if( 0 == strcmp(op, "prod") ) {
-                            MPI_OP_TEST( *, mpi_op, MPI_INT16_T, int16_t,
-                                         in_int16, inout_int16, inout_int16_for_check,
-                                         count, PRId16);
-                        }
-                        if( 0 == strcmp(op, "band") ) {
-                            MPI_OP_TEST( &, mpi_op, MPI_INT16_T, int16_t,
-                                         in_int16, inout_int16, inout_int16_for_check,
-                                         count, PRId16);
-                        }
-                        if( 0 == strcmp(op, "max") ) {
-                            MPI_OP_MINMAX_TEST(max, mpi_op,  MPI_INT16_T, int16_t,
-                                               in_int16, inout_int16, inout_int16_for_check,
-                                               count, PRId16);
-                        }
-                        if( 0 == strcmp(op, "min") ) {  //intentionly reversed in and out
-                            MPI_OP_MINMAX_TEST(min, mpi_op,  MPI_INT16_T, int16_t,
-                                               in_int16, inout_int16, inout_int16_for_check,
-                                               count, PRId16);
+                        if (0 == strcmp(op, "sum")) {
+                            MPI_OP_TEST(+, mpi_op, MPI_INT16_T, int16_t, in_int16, inout_int16,
+                                        inout_int16_for_check, count, PRId16);
+                        }
+                        if (0 == strcmp(op, "bor")) {
+                            MPI_OP_TEST(|, mpi_op, MPI_INT16_T, int16_t, in_int16, inout_int16,
+                                        inout_int16_for_check, count, PRId16);
+                        }
+                        if (0 == strcmp(op, "bxor")) {
+                            MPI_OP_TEST(^, mpi_op, MPI_INT16_T, int16_t, in_int16, inout_int16,
+                                        inout_int16_for_check, count, PRId16);
+                        }
+                        if (0 == strcmp(op, "prod")) {
+                            MPI_OP_TEST(*, mpi_op, MPI_INT16_T, int16_t, in_int16, inout_int16,
+                                        inout_int16_for_check, count, PRId16);
+                        }
+                        if (0 == strcmp(op, "band")) {
+                            MPI_OP_TEST(&, mpi_op, MPI_INT16_T, int16_t, in_int16, inout_int16,
+                                        inout_int16_for_check, count, PRId16);
+                        }
+                        if (0 == strcmp(op, "max")) {
+                            MPI_OP_MINMAX_TEST(max, mpi_op, MPI_INT16_T, int16_t, in_int16,
+                                               inout_int16, inout_int16_for_check, count, PRId16);
+                        }
+                        if (0 == strcmp(op, "min")) { // intentionally reversed in and out
+                            MPI_OP_MINMAX_TEST(min, mpi_op, MPI_INT16_T, int16_t, in_int16,
+                                               inout_int16, inout_int16_for_check, count, PRId16);
                         }
                     }
-                    if( 32 == type_size ) {
-                        int32_t *in_int32 = (int32_t*)((char*)in_buf + op1_alignment * sizeof(int32_t)),
-                            *inout_int32 = (int32_t*)((char*)inout_buf + res_alignment * sizeof(int32_t)),
-                            *inout_int32_for_check = (int32_t*)inout_check_buf;
-                        for( i = 0; i < count; i++ ) {
+                    if (32 == type_size) {
+                        int32_t *in_int32 = (int32_t *) ((char *) in_buf
+                                                         + op1_alignment * sizeof(int32_t)),
+                                *inout_int32 = (int32_t *) ((char *) inout_buf
+                                                            + res_alignment * sizeof(int32_t)),
+                                *inout_int32_for_check = (int32_t *) inout_check_buf;
+                        for (i = 0; i < count; i++) {
                             in_int32[i] = 5;
                             inout_int32[i] = inout_int32_for_check[i] = 3;
                         }
                         mpi_type = "MPI_INT32_T";
 
-                        if( 0 == strcmp(op, "sum") ) {
-                            MPI_OP_TEST( +, mpi_op, MPI_INT32_T, int32_t,
-                                         in_int32, inout_int32, inout_int32_for_check,
-                                         count, PRId32);
-                        }
-                        if( 0 == strcmp(op, "bor") ) {
-                            MPI_OP_TEST( |, mpi_op, MPI_INT32_T, int32_t,
-                                         in_int32, inout_int32, inout_int32_for_check,
-                                         count, PRId32);
-                        }
-                        if( 0 == strcmp(op, "bxor") ) {
-                            MPI_OP_TEST( ^, mpi_op, MPI_INT32_T, int32_t,
-                                         in_int32, inout_int32, inout_int32_for_check,
-                                         count, PRId32);
-                        }
-                        if( 0 == strcmp(op, "prod") ) {
-                            MPI_OP_TEST( *, mpi_op, MPI_INT32_T, int32_t,
-                                         in_int32, inout_int32, inout_int32_for_check,
-                                         count, PRId32);
-                        }
-                        if( 0 == strcmp(op, "band") ) {
-                            MPI_OP_TEST( &, mpi_op, MPI_INT32_T, int32_t,
-                                         in_int32, inout_int32, inout_int32_for_check,
-                                         count, PRId32);
-                        }
-                        if( 0 == strcmp(op, "max") ) {
-                            MPI_OP_MINMAX_TEST(max, mpi_op,  MPI_INT32_T, int32_t,
-                                               in_int32, inout_int32, inout_int32_for_check,
-                                               count, PRId32);
-                        }
-                        if( 0 == strcmp(op, "min") ) {  //intentionly reversed in and out
-                            MPI_OP_MINMAX_TEST(min, mpi_op,  MPI_INT32_T, int32_t,
-                                               in_int32, inout_int32, inout_int32_for_check,
-                                               count, PRId32);
+                        if (0 == strcmp(op, "sum")) {
+                            MPI_OP_TEST(+, mpi_op, MPI_INT32_T, int32_t, in_int32, inout_int32,
+                                        inout_int32_for_check, count, PRId32);
+                        }
+                        if (0 == strcmp(op, "bor")) {
+                            MPI_OP_TEST(|, mpi_op, MPI_INT32_T, int32_t, in_int32, inout_int32,
+                                        inout_int32_for_check, count, PRId32);
+                        }
+                        if (0 == strcmp(op, "bxor")) {
+                            MPI_OP_TEST(^, mpi_op, MPI_INT32_T, int32_t, in_int32, inout_int32,
+                                        inout_int32_for_check, count, PRId32);
+                        }
+                        if (0 == strcmp(op, "prod")) {
+                            MPI_OP_TEST(*, mpi_op, MPI_INT32_T, int32_t, in_int32, inout_int32,
+                                        inout_int32_for_check, count, PRId32);
+                        }
+                        if (0 == strcmp(op, "band")) {
+                            MPI_OP_TEST(&, mpi_op, MPI_INT32_T, int32_t, in_int32, inout_int32,
+                                        inout_int32_for_check, count, PRId32);
+                        }
+                        if (0 == strcmp(op, "max")) {
+                            MPI_OP_MINMAX_TEST(max, mpi_op, MPI_INT32_T, int32_t, in_int32,
+                                               inout_int32, inout_int32_for_check, count, PRId32);
+                        }
+                        if (0 == strcmp(op, "min")) { // intentionally reversed in and out
+                            MPI_OP_MINMAX_TEST(min, mpi_op, MPI_INT32_T, int32_t, in_int32,
+                                               inout_int32, inout_int32_for_check, count, PRId32);
                         }
                     }
-                    if( 64 == type_size ) {
-                        int64_t *in_int64 = (int64_t*)((char*)in_buf + op1_alignment * sizeof(int64_t)),
-                            *inout_int64 = (int64_t*)((char*)inout_buf + res_alignment * sizeof(int64_t)),
-                            *inout_int64_for_check = (int64_t*)inout_check_buf;
-                        for( i = 0; i < count; i++ ) {
+                    if (64 == type_size) {
+                        int64_t *in_int64 = (int64_t *) ((char *) in_buf
+                                                         + op1_alignment * sizeof(int64_t)),
+                                *inout_int64 = (int64_t *) ((char *) inout_buf
+                                                            + res_alignment * sizeof(int64_t)),
+                                *inout_int64_for_check = (int64_t *) inout_check_buf;
+                        for (i = 0; i < count; i++) {
                             in_int64[i] = 5;
                             inout_int64[i] = inout_int64_for_check[i] = 3;
                         }
                         mpi_type = "MPI_INT64_T";
 
-                        if( 0 == strcmp(op, "sum") ) {
-                            MPI_OP_TEST( +, mpi_op, MPI_INT64_T, int64_t,
-                                         in_int64, inout_int64, inout_int64_for_check,
-                                         count, PRId64);
-                        }
-                        if( 0 == strcmp(op, "bor") ) {
-                            MPI_OP_TEST( |, mpi_op, MPI_INT64_T, int64_t,
-                                         in_int64, inout_int64, inout_int64_for_check,
-                                         count, PRId64);
-                        }
-                        if( 0 == strcmp(op, "bxor") ) {
-                            MPI_OP_TEST( ^, mpi_op, MPI_INT64_T, int64_t,
-                                         in_int64, inout_int64, inout_int64_for_check,
-                                         count, PRId64);
-                        }
-                        if( 0 == strcmp(op, "prod") ) {
-                            MPI_OP_TEST( *, mpi_op, MPI_INT64_T, int64_t,
-                                         in_int64, inout_int64, inout_int64_for_check,
-                                         count, PRId64);
-                        }
-                        if( 0 == strcmp(op, "band") ) {
-                            MPI_OP_TEST( &, mpi_op, MPI_INT64_T, int64_t,
-                                         in_int64, inout_int64, inout_int64_for_check,
-                                         count, PRId64);
-                        }
-                        if( 0 == strcmp(op, "max") ) {
-                            MPI_OP_MINMAX_TEST(max, mpi_op,  MPI_INT64_T, int64_t,
-                                               in_int64, inout_int64, inout_int64_for_check,
-                                               count, PRId64);
-                        }
-                        if( 0 == strcmp(op, "min") ) {  //intentionly reversed in and out
-                            MPI_OP_MINMAX_TEST(min, mpi_op,  MPI_INT64_T, int64_t,
-                                               in_int64, inout_int64, inout_int64_for_check,
-                                               count, PRId64);
+                        if (0 == strcmp(op, "sum")) {
+                            MPI_OP_TEST(+, mpi_op, MPI_INT64_T, int64_t, in_int64, inout_int64,
+                                        inout_int64_for_check, count, PRId64);
+                        }
+                        if (0 == strcmp(op, "bor")) {
+                            MPI_OP_TEST(|, mpi_op, MPI_INT64_T, int64_t, in_int64, inout_int64,
+                                        inout_int64_for_check, count, PRId64);
+                        }
+                        if (0 == strcmp(op, "bxor")) {
+                            MPI_OP_TEST(^, mpi_op, MPI_INT64_T, int64_t, in_int64, inout_int64,
+                                        inout_int64_for_check, count, PRId64);
+                        }
+                        if (0 == strcmp(op, "prod")) {
+                            MPI_OP_TEST(*, mpi_op, MPI_INT64_T, int64_t, in_int64, inout_int64,
+                                        inout_int64_for_check, count, PRId64);
+                        }
+                        if (0 == strcmp(op, "band")) {
+                            MPI_OP_TEST(&, mpi_op, MPI_INT64_T, int64_t, in_int64, inout_int64,
+                                        inout_int64_for_check, count, PRId64);
+                        }
+                        if (0 == strcmp(op, "max")) {
+                            MPI_OP_MINMAX_TEST(max, mpi_op, MPI_INT64_T, int64_t, in_int64,
+                                               inout_int64, inout_int64_for_check, count, PRId64);
+                        }
+                        if (0 == strcmp(op, "min")) { // intentionally reversed in and out
+                            MPI_OP_MINMAX_TEST(min, mpi_op, MPI_INT64_T, int64_t, in_int64,
+                                               inout_int64, inout_int64_for_check, count, PRId64);
                         }
                     }
                 }
 
-                if( 'u' == type[type_idx] ) {
-                    if( 8 == type_size ) {
-                        uint8_t *in_uint8 = (uint8_t*)((char*)in_buf + op1_alignment * sizeof(uint8_t)),
-                            *inout_uint8 = (uint8_t*)((char*)inout_buf + res_alignment * sizeof(uint8_t)),
-                            *inout_uint8_for_check = (uint8_t*)inout_check_buf;
-                        for( i = 0; i < count; i++ ) {
+                if ('u' == type[type_idx]) {
+                    if (8 == type_size) {
+                        uint8_t *in_uint8 = (uint8_t *) ((char *) in_buf
+                                                         + op1_alignment * sizeof(uint8_t)),
+                                *inout_uint8 = (uint8_t *) ((char *) inout_buf
+                                                            + res_alignment * sizeof(uint8_t)),
+                                *inout_uint8_for_check = (uint8_t *) inout_check_buf;
+                        for (i = 0; i < count; i++) {
                             in_uint8[i] = 5;
                             inout_uint8[i] = inout_uint8_for_check[i] = 2;
                         }
                         mpi_type = "MPI_UINT8_T";
 
-                        if( 0 == strcmp(op, "sum") ) {
-                            MPI_OP_TEST( +, mpi_op, MPI_UINT8_T, uint8_t,
-                                         in_uint8, inout_uint8, inout_uint8_for_check,
-                                         count, PRIu8);
-                        }
-                        if( 0 == strcmp(op, "bor") ) {
-                            MPI_OP_TEST( |, mpi_op, MPI_UINT8_T, uint8_t,
-                                         in_uint8, inout_uint8, inout_uint8_for_check,
-                                         count, PRIu8);
-                        }
-                        if( 0 == strcmp(op, "bxor") ) {
-                            MPI_OP_TEST( ^, mpi_op, MPI_UINT8_T, uint8_t,
-                                         in_uint8, inout_uint8, inout_uint8_for_check,
-                                         count, PRIu8);
-                        }
-                        if( 0 == strcmp(op, "prod") ) {
-                            MPI_OP_TEST( *, mpi_op, MPI_UINT8_T, uint8_t,
-                                         in_uint8, inout_uint8, inout_uint8_for_check,
-                                         count, PRIu8);
-                        }
-                        if( 0 == strcmp(op, "band") ) {
-                            MPI_OP_TEST( &, mpi_op, MPI_UINT8_T, uint8_t,
-                                         in_uint8, inout_uint8, inout_uint8_for_check,
-                                         count, PRIu8);
-                        }
-                        if( 0 == strcmp(op, "max") ) {
-                            MPI_OP_MINMAX_TEST(max, mpi_op,  MPI_UINT8_T, uint8_t,
-                                               in_uint8, inout_uint8, inout_uint8_for_check,
-                                               count, PRIu8);
-                        }
-                        if( 0 == strcmp(op, "min") ) {  //intentionly reversed in and out
-                            MPI_OP_MINMAX_TEST(min, mpi_op,  MPI_UINT8_T, uint8_t,
-                                               in_uint8, inout_uint8, inout_uint8_for_check,
-                                               count, PRIu8);
+                        if (0 == strcmp(op, "sum")) {
+                            MPI_OP_TEST(+, mpi_op, MPI_UINT8_T, uint8_t, in_uint8, inout_uint8,
+                                        inout_uint8_for_check, count, PRIu8);
+                        }
+                        if (0 == strcmp(op, "bor")) {
+                            MPI_OP_TEST(|, mpi_op, MPI_UINT8_T, uint8_t, in_uint8, inout_uint8,
+                                        inout_uint8_for_check, count, PRIu8);
+                        }
+                        if (0 == strcmp(op, "bxor")) {
+                            MPI_OP_TEST(^, mpi_op, MPI_UINT8_T, uint8_t, in_uint8, inout_uint8,
+                                        inout_uint8_for_check, count, PRIu8);
+                        }
+                        if (0 == strcmp(op, "prod")) {
+                            MPI_OP_TEST(*, mpi_op, MPI_UINT8_T, uint8_t, in_uint8, inout_uint8,
+                                        inout_uint8_for_check, count, PRIu8);
+                        }
+                        if (0 == strcmp(op, "band")) {
+                            MPI_OP_TEST(&, mpi_op, MPI_UINT8_T, uint8_t, in_uint8, inout_uint8,
+                                        inout_uint8_for_check, count, PRIu8);
+                        }
+                        if (0 == strcmp(op, "max")) {
+                            MPI_OP_MINMAX_TEST(max, mpi_op, MPI_UINT8_T, uint8_t, in_uint8,
+                                               inout_uint8, inout_uint8_for_check, count, PRIu8);
+                        }
+                        if (0 == strcmp(op, "min")) { // intentionally reversed in and out
+                            MPI_OP_MINMAX_TEST(min, mpi_op, MPI_UINT8_T, uint8_t, in_uint8,
+                                               inout_uint8, inout_uint8_for_check, count, PRIu8);
                         }
                     }
-                    if( 16 == type_size ) {
-                        uint16_t *in_uint16 = (uint16_t*)((char*)in_buf + op1_alignment * sizeof(uint16_t)),
-                            *inout_uint16 = (uint16_t*)((char*)inout_buf + res_alignment * sizeof(uint16_t)),
-                            *inout_uint16_for_check = (uint16_t*)inout_check_buf;
-                        for( i = 0; i < count; i++ ) {
+                    if (16 == type_size) {
+                        uint16_t *in_uint16 = (uint16_t *) ((char *) in_buf
+                                                            + op1_alignment * sizeof(uint16_t)),
+                                 *inout_uint16 = (uint16_t *) ((char *) inout_buf
+                                                               + res_alignment * sizeof(uint16_t)),
+                                 *inout_uint16_for_check = (uint16_t *) inout_check_buf;
+                        for (i = 0; i < count; i++) {
                             in_uint16[i] = 5;
                             inout_uint16[i] = inout_uint16_for_check[i] = 1234;
                         }
                         mpi_type = "MPI_UINT16_T";
 
-                        if( 0 == strcmp(op, "sum") ) {
-                            MPI_OP_TEST( +, mpi_op, MPI_UINT16_T, uint16_t,
-                                         in_uint16, inout_uint16, inout_uint16_for_check,
-                                         count, PRIu16);
-                        }
-                        if( 0 == strcmp(op, "bor") ) {
-                            MPI_OP_TEST( |, mpi_op, MPI_UINT16_T, uint16_t,
-                                         in_uint16, inout_uint16, inout_uint16_for_check,
-                                         count, PRIu16);
-                        }
-                        if( 0 == strcmp(op, "bxor") ) {
-                            MPI_OP_TEST( ^, mpi_op, MPI_UINT16_T, uint16_t,
-                                         in_uint16, inout_uint16, inout_uint16_for_check,
-                                         count, PRIu16);
-                        }
-                        if( 0 == strcmp(op, "prod") ) {
-                            MPI_OP_TEST( *, mpi_op, MPI_UINT16_T, uint16_t,
-                                         in_uint16, inout_uint16, inout_uint16_for_check,
-                                         count, PRIu16);
-                        }
-                        if( 0 == strcmp(op, "band") ) {
-                            MPI_OP_TEST( &, mpi_op, MPI_UINT16_T, uint16_t,
-                                         in_uint16, inout_uint16, inout_uint16_for_check,
-                                         count, PRIu16);
-                        }
-                        if( 0 == strcmp(op, "max") ) {
-                            MPI_OP_MINMAX_TEST(max, mpi_op,  MPI_UINT16_T, uint16_t,
-                                               in_uint16, inout_uint16, inout_uint16_for_check,
-                                               count, PRIu16);
-                        }
-                        if( 0 == strcmp(op, "min") ) {  //intentionly reversed in and out
-                            MPI_OP_MINMAX_TEST(min, mpi_op,  MPI_UINT16_T, uint16_t,
-                                               in_uint16, inout_uint16, inout_uint16_for_check,
-                                               count, PRIu16);
+                        if (0 == strcmp(op, "sum")) {
+                            MPI_OP_TEST(+, mpi_op, MPI_UINT16_T, uint16_t, in_uint16, inout_uint16,
+                                        inout_uint16_for_check, count, PRIu16);
+                        }
+                        if (0 == strcmp(op, "bor")) {
+                            MPI_OP_TEST(|, mpi_op, MPI_UINT16_T, uint16_t, in_uint16, inout_uint16,
+                                        inout_uint16_for_check, count, PRIu16);
+                        }
+                        if (0 == strcmp(op, "bxor")) {
+                            MPI_OP_TEST(^, mpi_op, MPI_UINT16_T, uint16_t, in_uint16, inout_uint16,
+                                        inout_uint16_for_check, count, PRIu16);
+                        }
+                        if (0 == strcmp(op, "prod")) {
+                            MPI_OP_TEST(*, mpi_op, MPI_UINT16_T, uint16_t, in_uint16, inout_uint16,
+                                        inout_uint16_for_check, count, PRIu16);
+                        }
+                        if (0 == strcmp(op, "band")) {
+                            MPI_OP_TEST(&, mpi_op, MPI_UINT16_T, uint16_t, in_uint16, inout_uint16,
+                                        inout_uint16_for_check, count, PRIu16);
+                        }
+                        if (0 == strcmp(op, "max")) {
+                            MPI_OP_MINMAX_TEST(max, mpi_op, MPI_UINT16_T, uint16_t, in_uint16,
+                                               inout_uint16, inout_uint16_for_check, count, PRIu16);
+                        }
+                        if (0 == strcmp(op, "min")) { // intentionally reversed in and out
+                            MPI_OP_MINMAX_TEST(min, mpi_op, MPI_UINT16_T, uint16_t, in_uint16,
+                                               inout_uint16, inout_uint16_for_check, count, PRIu16);
                         }
                     }
-                    if( 32 == type_size ) {
-                        uint32_t *in_uint32 = (uint32_t*)((char*)in_buf + op1_alignment * sizeof(uint32_t)),
-                            *inout_uint32 = (uint32_t*)((char*)inout_buf + res_alignment * sizeof(uint32_t)),
-                            *inout_uint32_for_check = (uint32_t*)inout_check_buf;
-                        for( i = 0; i < count; i++ ) {
+                    if (32 == type_size) {
+                        uint32_t *in_uint32 = (uint32_t *) ((char *) in_buf
+                                                            + op1_alignment * sizeof(uint32_t)),
+                                 *inout_uint32 = (uint32_t *) ((char *) inout_buf
+                                                               + res_alignment * sizeof(uint32_t)),
+                                 *inout_uint32_for_check = (uint32_t *) inout_check_buf;
+                        for (i = 0; i < count; i++) {
                             in_uint32[i] = 5;
                             inout_uint32[i] = inout_uint32_for_check[i] = 3;
                         }
                         mpi_type = "MPI_UINT32_T";
 
-                        if( 0 == strcmp(op, "sum") ) {
-                            MPI_OP_TEST( +, mpi_op, MPI_UINT32_T, uint32_t,
-                                         in_uint32, inout_uint32, inout_uint32_for_check,
-                                         count, PRIu32);
-                        }
-                        if( 0 == strcmp(op, "bor") ) {
-                            MPI_OP_TEST( |, mpi_op, MPI_UINT32_T, uint32_t,
-                                         in_uint32, inout_uint32, inout_uint32_for_check,
-                                         count, PRIu32);
-                        }
-                        if( 0 == strcmp(op, "bxor") ) {
-                            MPI_OP_TEST( ^, mpi_op, MPI_UINT32_T, uint32_t,
-                                         in_uint32, inout_uint32, inout_uint32_for_check,
-                                         count, PRIu32);
-                        }
-                        if( 0 == strcmp(op, "prod") ) {
-                            MPI_OP_TEST( *, mpi_op, MPI_UINT32_T, uint32_t,
-                                         in_uint32, inout_uint32, inout_uint32_for_check,
-                                         count, PRIu32);
-                        }
-                        if( 0 == strcmp(op, "band") ) {
-                            MPI_OP_TEST( &, mpi_op, MPI_UINT32_T, uint32_t,
-                                         in_uint32, inout_uint32, inout_uint32_for_check,
-                                         count, PRIu32);
-                        }
-                        if( 0 == strcmp(op, "max") ) {
-                            MPI_OP_MINMAX_TEST(max, mpi_op,  MPI_UINT32_T, uint32_t,
-                                               in_uint32, inout_uint32, inout_uint32_for_check,
-                                               count, PRIu32);
-                        }
-                        if( 0 == strcmp(op, "min") ) {  //intentionly reversed in and out
-                            MPI_OP_MINMAX_TEST(min, mpi_op,  MPI_UINT32_T, uint32_t,
-                                               in_uint32, inout_uint32, inout_uint32_for_check,
-                                               count, PRIu32);
+                        if (0 == strcmp(op, "sum")) {
+                            MPI_OP_TEST(+, mpi_op, MPI_UINT32_T, uint32_t, in_uint32, inout_uint32,
+                                        inout_uint32_for_check, count, PRIu32);
+                        }
+                        if (0 == strcmp(op, "bor")) {
+                            MPI_OP_TEST(|, mpi_op, MPI_UINT32_T, uint32_t, in_uint32, inout_uint32,
+                                        inout_uint32_for_check, count, PRIu32);
+                        }
+                        if (0 == strcmp(op, "bxor")) {
+                            MPI_OP_TEST(^, mpi_op, MPI_UINT32_T, uint32_t, in_uint32, inout_uint32,
+                                        inout_uint32_for_check, count, PRIu32);
+                        }
+                        if (0 == strcmp(op, "prod")) {
+                            MPI_OP_TEST(*, mpi_op, MPI_UINT32_T, uint32_t, in_uint32, inout_uint32,
+                                        inout_uint32_for_check, count, PRIu32);
+                        }
+                        if (0 == strcmp(op, "band")) {
+                            MPI_OP_TEST(&, mpi_op, MPI_UINT32_T, uint32_t, in_uint32, inout_uint32,
+                                        inout_uint32_for_check, count, PRIu32);
+                        }
+                        if (0 == strcmp(op, "max")) {
+                            MPI_OP_MINMAX_TEST(max, mpi_op, MPI_UINT32_T, uint32_t, in_uint32,
+                                               inout_uint32, inout_uint32_for_check, count, PRIu32);
+                        }
+                        if (0 == strcmp(op, "min")) { // intentionally reversed in and out
+                            MPI_OP_MINMAX_TEST(min, mpi_op, MPI_UINT32_T, uint32_t, in_uint32,
+                                               inout_uint32, inout_uint32_for_check, count, PRIu32);
                         }
                     }
-                    if( 64 == type_size ) {
-                        uint64_t *in_uint64 = (uint64_t*)((char*)in_buf + op1_alignment * sizeof(uint64_t)),
-                              *inout_uint64 = (uint64_t*)((char*)inout_buf + res_alignment * sizeof(uint64_t)),
-                            *inout_uint64_for_check = (uint64_t*)inout_check_buf;
-                        for( i = 0; i < count; i++ ) {
+                    if (64 == type_size) {
+                        uint64_t *in_uint64 = (uint64_t *) ((char *) in_buf
+                                                            + op1_alignment * sizeof(uint64_t)),
+                                 *inout_uint64 = (uint64_t *) ((char *) inout_buf
+                                                               + res_alignment * sizeof(uint64_t)),
+                                 *inout_uint64_for_check = (uint64_t *) inout_check_buf;
+                        for (i = 0; i < count; i++) {
                             in_uint64[i] = 5;
                             inout_uint64[i] = inout_uint64_for_check[i] = 32433;
                         }
                         mpi_type = "MPI_UINT64_T";
 
-                        if( 0 == strcmp(op, "sum") ) {
-                            MPI_OP_TEST( +, mpi_op, MPI_UINT64_T, uint64_t,
-                                         in_uint64, inout_uint64, inout_uint64_for_check,
-                                         count, PRIu64);
-                        }
-                        if( 0 == strcmp(op, "bor") ) {
-                            MPI_OP_TEST( |, mpi_op, MPI_UINT64_T, uint64_t,
-                                         in_uint64, inout_uint64, inout_uint64_for_check,
-                                         count, PRIu64);
-                        }
-                        if( 0 == strcmp(op, "bxor") ) {
-                            MPI_OP_TEST( ^, mpi_op, MPI_UINT64_T, uint64_t,
-                                         in_uint64, inout_uint64, inout_uint64_for_check,
-                                         count, PRIu64);
-                        }
-                        if( 0 == strcmp(op, "prod") ) {
-                            MPI_OP_TEST( *, mpi_op, MPI_UINT64_T, uint64_t,
-                                         in_uint64, inout_uint64, inout_uint64_for_check,
-                                         count, PRIu64);
-                        }
-                        if( 0 == strcmp(op, "band") ) {
-                            MPI_OP_TEST( &, mpi_op, MPI_UINT64_T, uint64_t,
-                                         in_uint64, inout_uint64, inout_uint64_for_check,
-                                         count, PRIu64);
-                        }
-                        if( 0 == strcmp(op, "max") ) {
-                            MPI_OP_MINMAX_TEST(max, mpi_op,  MPI_UINT64_T, uint64_t,
-                                               in_uint64, inout_uint64, inout_uint64_for_check,
-                                               count, PRIu64);
-                        }
-                        if( 0 == strcmp(op, "min") ) {
-                            MPI_OP_MINMAX_TEST(min, mpi_op,  MPI_UINT64_T, uint64_t,
-                                               in_uint64, inout_uint64, inout_uint64_for_check,
-                                               count, PRIu64);
+                        if (0 == strcmp(op, "sum")) {
+                            MPI_OP_TEST(+, mpi_op, MPI_UINT64_T, uint64_t, in_uint64, inout_uint64,
+                                        inout_uint64_for_check, count, PRIu64);
+                        }
+                        if (0 == strcmp(op, "bor")) {
+                            MPI_OP_TEST(|, mpi_op, MPI_UINT64_T, uint64_t, in_uint64, inout_uint64,
+                                        inout_uint64_for_check, count, PRIu64);
+                        }
+                        if (0 == strcmp(op, "bxor")) {
+                            MPI_OP_TEST(^, mpi_op, MPI_UINT64_T, uint64_t, in_uint64, inout_uint64,
+                                        inout_uint64_for_check, count, PRIu64);
+                        }
+                        if (0 == strcmp(op, "prod")) {
+                            MPI_OP_TEST(*, mpi_op, MPI_UINT64_T, uint64_t, in_uint64, inout_uint64,
+                                        inout_uint64_for_check, count, PRIu64);
+                        }
+                        if (0 == strcmp(op, "band")) {
+                            MPI_OP_TEST(&, mpi_op, MPI_UINT64_T, uint64_t, in_uint64, inout_uint64,
+                                        inout_uint64_for_check, count, PRIu64);
+                        }
+                        if (0 == strcmp(op, "max")) {
+                            MPI_OP_MINMAX_TEST(max, mpi_op, MPI_UINT64_T, uint64_t, in_uint64,
+                                               inout_uint64, inout_uint64_for_check, count, PRIu64);
+                        }
+                        if (0 == strcmp(op, "min")) {
+                            MPI_OP_MINMAX_TEST(min, mpi_op, MPI_UINT64_T, uint64_t, in_uint64,
+                                               inout_uint64, inout_uint64_for_check, count, PRIu64);
                         }
                     }
                 }
 
-                if( 'f' == type[type_idx] ) {
-                    float *in_float = (float*)((char*)in_buf + op1_alignment * sizeof(float)),
-                        *inout_float = (float*)((char*)inout_buf + res_alignment * sizeof(float)),
-                        *inout_float_for_check = (float*)inout_check_buf;
-                    for( i = 0; i < count; i++ ) {
-                        in_float[i] = 1000.0+1;
-                        inout_float[i] = inout_float_for_check[i] = 100.0+2;
+                if ('f' == type[type_idx]) {
+                    float *in_float = (float *) ((char *) in_buf + op1_alignment * sizeof(float)),
+                          *inout_float = (float *) ((char *) inout_buf
+                                                    + res_alignment * sizeof(float)),
+                          *inout_float_for_check = (float *) inout_check_buf;
+                    for (i = 0; i < count; i++) {
+                        in_float[i] = 1000.0 + 1;
+                        inout_float[i] = inout_float_for_check[i] = 100.0 + 2;
                     }
                     mpi_type = "MPI_FLOAT";
 
-                    if( 0 == strcmp(op, "sum") ) {
-                        MPI_OP_TEST( +, mpi_op, MPI_FLOAT, float,
-                                     in_float, inout_float, inout_float_for_check,
-                                     count, "f");
+                    if (0 == strcmp(op, "sum")) {
+                        MPI_OP_TEST(+, mpi_op, MPI_FLOAT, float, in_float, inout_float,
+                                    inout_float_for_check, count, "f");
                     }
-                    if( 0 == strcmp(op, "prod") ) {
-                        MPI_OP_TEST( *, mpi_op, MPI_FLOAT, float,
-                                     in_float, inout_float, inout_float_for_check,
-                                     count, "f");
+                    if (0 == strcmp(op, "prod")) {
+                        MPI_OP_TEST(*, mpi_op, MPI_FLOAT, float, in_float, inout_float,
+                                    inout_float_for_check, count, "f");
                     }
-                    if( 0 == strcmp(op, "max") ) {
-                        MPI_OP_MINMAX_TEST(max, mpi_op,  MPI_FLOAT, float,
-                                           in_float, inout_float, inout_float_for_check,
-                                           count, "f");
+                    if (0 == strcmp(op, "max")) {
+                        MPI_OP_MINMAX_TEST(max, mpi_op, MPI_FLOAT, float, in_float, inout_float,
+                                           inout_float_for_check, count, "f");
                     }
-                    if( 0 == strcmp(op, "min") ) {
-                        MPI_OP_MINMAX_TEST(min, mpi_op,  MPI_FLOAT, float,
-                                           in_float, inout_float, inout_float_for_check,
-                                           count, "f");
+                    if (0 == strcmp(op, "min")) {
+                        MPI_OP_MINMAX_TEST(min, mpi_op, MPI_FLOAT, float, in_float, inout_float,
+                                           inout_float_for_check, count, "f");
                     }
                 }
 
-                if( 'd' == type[type_idx] ) {
-                    double *in_double = (double*)((char*)in_buf + op1_alignment * sizeof(double)),
-                        *inout_double = (double*)((char*)inout_buf + res_alignment * sizeof(double)),
-                        *inout_double_for_check = (double*)inout_check_buf;
-                    for( i = 0; i < count; i++ ) {
-                        in_double[i] = 10.0+1;
-                        inout_double[i] = inout_double_for_check[i] = 1.0+2;
+                if ('d' == type[type_idx]) {
+                    double *in_double = (double *) ((char *) in_buf
+                                                    + op1_alignment * sizeof(double)),
+                           *inout_double = (double *) ((char *) inout_buf
+                                                       + res_alignment * sizeof(double)),
+                           *inout_double_for_check = (double *) inout_check_buf;
+                    for (i = 0; i < count; i++) {
+                        in_double[i] = 10.0 + 1;
+                        inout_double[i] = inout_double_for_check[i] = 1.0 + 2;
                     }
                     mpi_type = "MPI_DOUBLE";
 
-                    if( 0 == strcmp(op, "sum") ) {
-                        MPI_OP_TEST( +, mpi_op, MPI_DOUBLE, double,
-                                     in_double, inout_double, inout_double_for_check,
-                                     count, "g");
+                    if (0 == strcmp(op, "sum")) {
+                        MPI_OP_TEST(+, mpi_op, MPI_DOUBLE, double, in_double, inout_double,
+                                    inout_double_for_check, count, "g");
                     }
-                    if( 0 == strcmp(op, "prod") ) {
-                        MPI_OP_TEST( *, mpi_op, MPI_DOUBLE, double,
-                                     in_double, inout_double, inout_double_for_check,
-                                     count, "f");
+                    if (0 == strcmp(op, "prod")) {
+                        MPI_OP_TEST(*, mpi_op, MPI_DOUBLE, double, in_double, inout_double,
+                                    inout_double_for_check, count, "f");
                     }
-                    if( 0 == strcmp(op, "max") ) {
-                        MPI_OP_MINMAX_TEST(max, mpi_op,  MPI_DOUBLE, double,
-                                           in_double, inout_double, inout_double_for_check,
-                                           count, "f");
+                    if (0 == strcmp(op, "max")) {
+                        MPI_OP_MINMAX_TEST(max, mpi_op, MPI_DOUBLE, double, in_double, inout_double,
+                                           inout_double_for_check, count, "f");
                     }
-                    if( 0 == strcmp(op, "min") ) {
-                        MPI_OP_MINMAX_TEST(min, mpi_op,  MPI_DOUBLE, double,
-                                           in_double, inout_double, inout_double_for_check,
-                                           count, "f");
+                    if (0 == strcmp(op, "min")) {
+                        MPI_OP_MINMAX_TEST(min, mpi_op, MPI_DOUBLE, double, in_double, inout_double,
+                                           inout_double_for_check, count, "f");
                     }
                 }
-        check_and_continue:
-                if( !skip_op_type )
-                    print_status(array_of_ops[do_ops[op_idx]].mpi_op_name,
-                                 mpi_type, type_size, count, max_shift, duration, repeats, correctness);
+            check_and_continue:
+                if (!skip_op_type)
+                    print_status(array_of_ops[do_ops[op_idx]].mpi_op_name, mpi_type, type_size,
+                                 count, max_shift, duration, repeats, correctness);
             }
-            if( !skip_op_type )
+            if (!skip_op_type)
                 printf("\n");
         }
     }
@@ -752,4 +720,3 @@ int main(int argc, char **argv)
 
     return (0 == total_errors) ? 0 : -1;
 }
-
diff --git a/test/datatype/to_self.c b/test/datatype/to_self.c
index 8b614759a97..e7158bb7561 100644
--- a/test/datatype/to_self.c
+++ b/test/datatype/to_self.c
@@ -12,35 +12,32 @@
 
 #include "mpi.h"
 
+#include <math.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
-#include <math.h>
 
 #if 0 && OPEN_MPI
 extern void ompi_datatype_dump( MPI_Datatype ddt );
-#define MPI_DDT_DUMP(ddt) ompi_datatype_dump( (ddt) )
+#    define MPI_DDT_DUMP(ddt) ompi_datatype_dump((ddt))
 #else
-#define MPI_DDT_DUMP(ddt)
-#endif  /* OPEN_MPI */
+#    define MPI_DDT_DUMP(ddt)
+#endif /* OPEN_MPI */
 
-static MPI_Datatype
-create_merged_contig_with_gaps(int count)  /* count of the basic datatype */
+static MPI_Datatype create_merged_contig_with_gaps(int count) /* count of the basic datatype */
 {
     int array_of_blocklengths[] = {1, 1, 1};
     MPI_Aint array_of_displacements[] = {0, 8, 16};
     MPI_Datatype array_of_types[] = {MPI_DOUBLE, MPI_LONG, MPI_CHAR};
     MPI_Datatype type;
 
-    MPI_Type_create_struct(3, array_of_blocklengths,
-                           array_of_displacements, array_of_types,
-                           &type);
-    if( 1 < count ) {
+    MPI_Type_create_struct(3, array_of_blocklengths, array_of_displacements, array_of_types, &type);
+    if (1 < count) {
         MPI_Datatype temp = type;
         MPI_Type_contiguous(count, temp, &type);
     }
     MPI_Type_commit(&type);
-    MPI_DDT_DUMP( type );
+    MPI_DDT_DUMP(type);
     return type;
 }
 
@@ -51,10 +48,10 @@ struct structure {
     double transfered_2;
 };
 
-static MPI_Datatype
-create_struct_constant_gap_resized_ddt( int number,  /* IGNORED: number of repetitions */
-                                        int contig_size,  /* IGNORED: number of elements in a contiguous chunk */
-                                        int gap_size )    /* IGNORED: number of elements in a gap */
+static MPI_Datatype create_struct_constant_gap_resized_ddt(
+    int number,      /* IGNORED: number of repetitions */
+    int contig_size, /* IGNORED: number of elements in a contiguous chunk */
+    int gap_size)    /* IGNORED: number of elements in a gap */
 {
     struct structure data[1];
     MPI_Datatype struct_type, temp_type;
@@ -72,111 +69,109 @@ create_struct_constant_gap_resized_ddt( int number,  /* IGNORED: number of repet
     MPI_Type_create_resized(temp_type, 0, sizeof(data[0]), &struct_type);
     MPI_Type_commit(&struct_type);
     MPI_Type_free(&temp_type);
-    MPI_DDT_DUMP( struct_type );
+    MPI_DDT_DUMP(struct_type);
 
     return struct_type;
 }
 
 /* Create a datatype similar to the one use by HPL */
 static MPI_Datatype
-create_indexed_constant_gap_ddt( int number,  /* number of repetitions */
-                                 int contig_size,  /* number of elements in a contiguous chunk */
-                                 int gap_size )    /* number of elements in a gap */
+create_indexed_constant_gap_ddt(int number,      /* number of repetitions */
+                                int contig_size, /* number of elements in a contiguous chunk */
+                                int gap_size)    /* number of elements in a gap */
 {
     MPI_Datatype dt, *types;
     int i, *bLength;
-    MPI_Aint* displ;
+    MPI_Aint *displ;
 
-    types = (MPI_Datatype*)malloc( sizeof(MPI_Datatype) * number );
-    bLength = (int*)malloc( sizeof(int) * number );
-    displ = (MPI_Aint*)malloc( sizeof(MPI_Aint) * number );
+    types = (MPI_Datatype *) malloc(sizeof(MPI_Datatype) * number);
+    bLength = (int *) malloc(sizeof(int) * number);
+    displ = (MPI_Aint *) malloc(sizeof(MPI_Aint) * number);
 
     types[0] = MPI_DOUBLE;
     bLength[0] = contig_size;
     displ[0] = 0;
-    for( i = 1; i < number; i++ ) {
+    for (i = 1; i < number; i++) {
         types[i] = MPI_DOUBLE;
         bLength[i] = contig_size;
-        displ[i] = displ[i-1] + sizeof(double) * (contig_size + gap_size);
+        displ[i] = displ[i - 1] + sizeof(double) * (contig_size + gap_size);
     }
-    MPI_Type_create_struct( number, bLength, displ, types, &dt );
-    MPI_DDT_DUMP( dt );
+    MPI_Type_create_struct(number, bLength, displ, types, &dt);
+    MPI_DDT_DUMP(dt);
     free(types);
     free(bLength);
     free(displ);
-    MPI_Type_commit( &dt );
+    MPI_Type_commit(&dt);
     return dt;
 }
 
-static MPI_Datatype
-create_optimized_indexed_constant_gap_ddt( int number,  /* number of repetitions */
-                                           int contig_size,  /* number of elements in a contiguous chunk */
-                                           int gap_size )    /* number of elements in a gap */
+static MPI_Datatype create_optimized_indexed_constant_gap_ddt(
+    int number,      /* number of repetitions */
+    int contig_size, /* number of elements in a contiguous chunk */
+    int gap_size)    /* number of elements in a gap */
 {
     MPI_Datatype dt;
 
-    MPI_Type_vector( number, contig_size, (contig_size + gap_size), MPI_DOUBLE, &dt );
-    MPI_Type_commit( &dt );
-    MPI_DDT_DUMP( dt );
+    MPI_Type_vector(number, contig_size, (contig_size + gap_size), MPI_DOUBLE, &dt);
+    MPI_Type_commit(&dt);
+    MPI_DDT_DUMP(dt);
     return dt;
 }
 
 typedef struct {
-   int i[2];
-   float f;
+    int i[2];
+    float f;
 } internal_struct;
 typedef struct {
-   int v1;
-   int gap1;
-   internal_struct is[3];
+    int v1;
+    int gap1;
+    internal_struct is[3];
 } ddt_gap;
 
-static MPI_Datatype
-create_indexed_gap_ddt( void )
+static MPI_Datatype create_indexed_gap_ddt(void)
 {
     ddt_gap dt[2];
     MPI_Datatype dt1, dt2, dt3;
-    int bLength[2] = { 2, 1 };
-    MPI_Datatype types[2] = { MPI_INT, MPI_FLOAT };
+    int bLength[2] = {2, 1};
+    MPI_Datatype types[2] = {MPI_INT, MPI_FLOAT};
     MPI_Aint displ[2];
 
-    MPI_Get_address( &(dt[0].is[0].i[0]), &(displ[0]) );
-    MPI_Get_address( &(dt[0].is[0].f), &(displ[1]) );
+    MPI_Get_address(&(dt[0].is[0].i[0]), &(displ[0]));
+    MPI_Get_address(&(dt[0].is[0].f), &(displ[1]));
     displ[1] -= displ[0];
     displ[0] -= displ[0];
-    MPI_Type_create_struct( 2, bLength, displ, types, &dt1 );
+    MPI_Type_create_struct(2, bLength, displ, types, &dt1);
     /*MPI_DDT_DUMP( dt1 );*/
-    MPI_Type_contiguous( 3, dt1, &dt2 );
+    MPI_Type_contiguous(3, dt1, &dt2);
     /*MPI_DDT_DUMP( dt2 );*/
     bLength[0] = 1;
     bLength[1] = 1;
-    MPI_Get_address( &(dt[0].v1), &(displ[0]) );
-    MPI_Get_address( &(dt[0].is[0]), &(displ[1]) );
+    MPI_Get_address(&(dt[0].v1), &(displ[0]));
+    MPI_Get_address(&(dt[0].is[0]), &(displ[1]));
     displ[1] -= displ[0];
     displ[0] -= displ[0];
     types[0] = MPI_INT;
     types[1] = dt2;
-    MPI_Type_create_struct( 2, bLength, displ, types, &dt3 );
+    MPI_Type_create_struct(2, bLength, displ, types, &dt3);
     /*MPI_DDT_DUMP( dt3 );*/
-    MPI_Type_free( &dt1 );
-    MPI_Type_free( &dt2 );
-    MPI_Type_contiguous( 10, dt3, &dt1 );
-    MPI_DDT_DUMP( dt1 );
-    MPI_Type_free( &dt3 );
-    MPI_Type_commit( &dt1 );
+    MPI_Type_free(&dt1);
+    MPI_Type_free(&dt2);
+    MPI_Type_contiguous(10, dt3, &dt1);
+    MPI_DDT_DUMP(dt1);
+    MPI_Type_free(&dt3);
+    MPI_Type_commit(&dt1);
     return dt1;
 }
 
-static MPI_Datatype
-create_indexed_gap_optimized_ddt( void )
+static MPI_Datatype create_indexed_gap_optimized_ddt(void)
 {
     MPI_Datatype dt1, dt2, dt3;
     int bLength[3];
     MPI_Datatype types[3];
     MPI_Aint displ[3];
 
-    MPI_Type_contiguous( 40, MPI_BYTE, &dt1 );
-    MPI_Type_create_resized( dt1, 0, 44, &dt2 );
+    MPI_Type_contiguous(40, MPI_BYTE, &dt1);
+    MPI_Type_create_resized(dt1, 0, 44, &dt2);
 
     bLength[0] = 4;
     bLength[1] = 9;
@@ -190,16 +185,15 @@ create_indexed_gap_optimized_ddt( void )
     displ[1] = 8;
     displ[2] = 44 * 9 + 8;
 
-    MPI_Type_create_struct( 3, bLength, displ, types, &dt3 );
+    MPI_Type_create_struct(3, bLength, displ, types, &dt3);
 
-    MPI_Type_free( &dt1 );
-    MPI_Type_free( &dt2 );
-    MPI_DDT_DUMP( dt3 );
-    MPI_Type_commit( &dt3 );
+    MPI_Type_free(&dt1);
+    MPI_Type_free(&dt2);
+    MPI_DDT_DUMP(dt3);
+    MPI_Type_commit(&dt3);
     return dt3;
 }
 
-
 /********************************************************************
  *******************************************************************/
 
@@ -210,370 +204,365 @@ create_indexed_gap_optimized_ddt( void )
 #define DO_STRUCT_CONSTANT_GAP_RESIZED    0x00000010
 #define DO_STRUCT_MERGED_WITH_GAP_RESIZED 0x00000020
 
-#define DO_PACK                         0x01000000
-#define DO_UNPACK                       0x02000000
-#define DO_ISEND_RECV                   0x04000000
-#define DO_ISEND_IRECV                  0x08000000
-#define DO_IRECV_SEND                   0x10000000
-#define DO_IRECV_ISEND                  0x20000000
+#define DO_PACK        0x01000000
+#define DO_UNPACK      0x02000000
+#define DO_ISEND_RECV  0x04000000
+#define DO_ISEND_IRECV 0x08000000
+#define DO_IRECV_SEND  0x10000000
+#define DO_IRECV_ISEND 0x20000000
 
-#define MIN_LENGTH   1024
-#define MAX_LENGTH   (1024*1024)
+#define MIN_LENGTH 1024
+#define MAX_LENGTH (1024 * 1024)
 
-static int cycles  = 100;
-static int trials  = 20;
+static int cycles = 100;
+static int trials = 20;
 static int warmups = 2;
 
-static void print_result( int length, int trials, double* timers )
+static void print_result(int length, int trials, double *timers)
 {
     double bandwidth, clock_prec, temp;
     double min_time, max_time, average, std_dev = 0.0;
     double ordered[trials];
     int t, pos, quartile_start, quartile_end;
 
-    for( t = 0; t < trials; ordered[t] = timers[t], t++ );
-    for( t = 0; t < trials-1; t++ ) {
+    for (t = 0; t < trials; ordered[t] = timers[t], t++)
+        ;
+    for (t = 0; t < trials - 1; t++) {
         temp = ordered[t];
         pos = t;
-        for( int i = t+1; i < trials; i++ ) {
-            if( temp > ordered[i] ) {
+        for (int i = t + 1; i < trials; i++) {
+            if (temp > ordered[i]) {
                 temp = ordered[i];
                 pos = i;
             }
         }
-        if( pos != t ) {
+        if (pos != t) {
             temp = ordered[t];
             ordered[t] = ordered[pos];
             ordered[pos] = temp;
         }
     }
     quartile_start = trials - (3 * trials) / 4;
-    quartile_end   = trials - (1 * trials) / 4;
+    quartile_end = trials - (1 * trials) / 4;
     clock_prec = MPI_Wtick();
     min_time = ordered[quartile_start];
     max_time = ordered[quartile_start];
     average = ordered[quartile_start];
-    for( t = quartile_start + 1; t < quartile_end; t++ ) {
-        if( min_time > ordered[t] ) min_time = ordered[t];
-        if( max_time < ordered[t] ) max_time = ordered[t];
+    for (t = quartile_start + 1; t < quartile_end; t++) {
+        if (min_time > ordered[t])
+            min_time = ordered[t];
+        if (max_time < ordered[t])
+            max_time = ordered[t];
         average += ordered[t];
     }
     average /= (quartile_end - quartile_start);
-    for( t = quartile_start; t < quartile_end; t++ ) {
+    for (t = quartile_start; t < quartile_end; t++) {
         std_dev += (ordered[t] - average) * (ordered[t] - average);
     }
-    std_dev = sqrt( std_dev/(quartile_end - quartile_start) );
-    
+    std_dev = sqrt(std_dev / (quartile_end - quartile_start));
+
     bandwidth = (length * clock_prec) / (1024.0 * 1024.0) / (average * clock_prec);
-    printf( "%8d\t%15g\t%10.4f MB/s [min %10g max %10g std %2.2f%%]\n", length, average, bandwidth,
-            min_time, max_time, (100.0 * std_dev) / average );
+    printf("%8d\t%15g\t%10.4f MB/s [min %10g max %10g std %2.2f%%]\n", length, average, bandwidth,
+           min_time, max_time, (100.0 * std_dev) / average);
 }
 
-static int pack( int cycles,
-                 MPI_Datatype sdt, int scount, void* sbuf,
-                 void* packed_buf )
+static int pack(int cycles, MPI_Datatype sdt, int scount, void *sbuf, void *packed_buf)
 {
     int position, myself, c, t, outsize;
     double timers[trials];
 
-    MPI_Type_size( sdt, &outsize );
+    MPI_Type_size(sdt, &outsize);
     outsize *= scount;
 
-    MPI_Comm_rank( MPI_COMM_WORLD, &myself );
+    MPI_Comm_rank(MPI_COMM_WORLD, &myself);
 
-    for( t = 0; t < warmups; t++ ) {
-        for( c = 0; c < cycles; c++ ) {
+    for (t = 0; t < warmups; t++) {
+        for (c = 0; c < cycles; c++) {
             position = 0;
             MPI_Pack(sbuf, scount, sdt, packed_buf, outsize, &position, MPI_COMM_WORLD);
         }
     }
-    
-    for( t = 0; t < trials; t++ ) {
+
+    for (t = 0; t < trials; t++) {
         timers[t] = MPI_Wtime();
-        for( c = 0; c < cycles; c++ ) {
+        for (c = 0; c < cycles; c++) {
             position = 0;
             MPI_Pack(sbuf, scount, sdt, packed_buf, outsize, &position, MPI_COMM_WORLD);
         }
         timers[t] = (MPI_Wtime() - timers[t]) / cycles;
     }
-    print_result( outsize, trials, timers );
+    print_result(outsize, trials, timers);
     return 0;
 }
 
-static int unpack( int cycles,
-                   void* packed_buf,
-                   MPI_Datatype rdt, int rcount, void* rbuf )
+static int unpack(int cycles, void *packed_buf, MPI_Datatype rdt, int rcount, void *rbuf)
 {
     int position, myself, c, t, insize;
     double timers[trials];
 
-    MPI_Type_size( rdt, &insize );
+    MPI_Type_size(rdt, &insize);
     insize *= rcount;
 
-    MPI_Comm_rank( MPI_COMM_WORLD, &myself );
+    MPI_Comm_rank(MPI_COMM_WORLD, &myself);
 
-    for( t = 0; t < warmups; t++ ) {
-        for( c = 0; c < cycles; c++ ) {
+    for (t = 0; t < warmups; t++) {
+        for (c = 0; c < cycles; c++) {
             position = 0;
             MPI_Unpack(packed_buf, insize, &position, rbuf, rcount, rdt, MPI_COMM_WORLD);
         }
     }
 
-    for( t = 0; t < trials; t++ ) {
+    for (t = 0; t < trials; t++) {
         timers[t] = MPI_Wtime();
-        for( c = 0; c < cycles; c++ ) {
+        for (c = 0; c < cycles; c++) {
             position = 0;
             MPI_Unpack(packed_buf, insize, &position, rbuf, rcount, rdt, MPI_COMM_WORLD);
         }
         timers[t] = (MPI_Wtime() - timers[t]) / cycles;
     }
-    print_result( insize, trials, timers );
+    print_result(insize, trials, timers);
     return 0;
 }
 
-static int isend_recv( int cycles,
-                       MPI_Datatype sdt, int scount, void* sbuf,
-                       MPI_Datatype rdt, int rcount, void* rbuf )
+static int isend_recv(int cycles, MPI_Datatype sdt, int scount, void *sbuf, MPI_Datatype rdt,
+                      int rcount, void *rbuf)
 {
     int myself, tag = 0, c, t, slength, rlength;
     MPI_Status status;
     MPI_Request req;
     double timers[trials];
 
-    MPI_Type_size( sdt, &slength );
+    MPI_Type_size(sdt, &slength);
     slength *= scount;
-    MPI_Type_size( rdt, &rlength );
+    MPI_Type_size(rdt, &rlength);
     rlength *= rcount;
 
-    MPI_Comm_rank( MPI_COMM_WORLD, &myself );
+    MPI_Comm_rank(MPI_COMM_WORLD, &myself);
 
-    for( t = 0; t < trials; t++ ) {
+    for (t = 0; t < trials; t++) {
         timers[t] = MPI_Wtime();
-        for( c = 0; c < cycles; c++ ) {
-            MPI_Isend( sbuf, scount, sdt, myself, tag, MPI_COMM_WORLD, &req );
-            MPI_Recv( rbuf, rcount, rdt, myself, tag, MPI_COMM_WORLD, &status );
-            MPI_Wait( &req, &status );
+        for (c = 0; c < cycles; c++) {
+            MPI_Isend(sbuf, scount, sdt, myself, tag, MPI_COMM_WORLD, &req);
+            MPI_Recv(rbuf, rcount, rdt, myself, tag, MPI_COMM_WORLD, &status);
+            MPI_Wait(&req, &status);
         }
         timers[t] = (MPI_Wtime() - timers[t]) / cycles;
     }
-    print_result( rlength, trials, timers );
+    print_result(rlength, trials, timers);
     return 0;
 }
 
-static int irecv_send( int cycles,
-                       MPI_Datatype sdt, int scount, void* sbuf,
-                       MPI_Datatype rdt, int rcount, void* rbuf )
+static int irecv_send(int cycles, MPI_Datatype sdt, int scount, void *sbuf, MPI_Datatype rdt,
+                      int rcount, void *rbuf)
 {
     int myself, tag = 0, c, t, slength, rlength;
     MPI_Request req;
     MPI_Status status;
     double timers[trials];
 
-    MPI_Type_size( sdt, &slength );
+    MPI_Type_size(sdt, &slength);
     slength *= scount;
-    MPI_Type_size( rdt, &rlength );
+    MPI_Type_size(rdt, &rlength);
     rlength *= rcount;
 
-    MPI_Comm_rank( MPI_COMM_WORLD, &myself );
+    MPI_Comm_rank(MPI_COMM_WORLD, &myself);
 
-    for( t = 0; t < trials; t++ ) {
+    for (t = 0; t < trials; t++) {
         timers[t] = MPI_Wtime();
-        for( c = 0; c < cycles; c++ ) {
-            MPI_Irecv( rbuf, rcount, rdt, myself, tag, MPI_COMM_WORLD, &req );
-            MPI_Send( sbuf, scount, sdt, myself, tag, MPI_COMM_WORLD );
-            MPI_Wait( &req, &status );
+        for (c = 0; c < cycles; c++) {
+            MPI_Irecv(rbuf, rcount, rdt, myself, tag, MPI_COMM_WORLD, &req);
+            MPI_Send(sbuf, scount, sdt, myself, tag, MPI_COMM_WORLD);
+            MPI_Wait(&req, &status);
         }
         timers[t] = (MPI_Wtime() - timers[t]) / cycles;
     }
-    print_result( rlength, trials, timers );
+    print_result(rlength, trials, timers);
     return 0;
 }
 
-static int isend_irecv_wait( int cycles,
-                             MPI_Datatype sdt, int scount, void* sbuf,
-                             MPI_Datatype rdt, int rcount, void* rbuf )
+static int isend_irecv_wait(int cycles, MPI_Datatype sdt, int scount, void *sbuf, MPI_Datatype rdt,
+                            int rcount, void *rbuf)
 {
     int myself, tag = 0, c, t, slength, rlength;
     MPI_Request requests[2];
     MPI_Status statuses[2];
     double timers[trials];
 
-    MPI_Type_size( sdt, &slength );
+    MPI_Type_size(sdt, &slength);
     slength *= scount;
-    MPI_Type_size( rdt, &rlength );
+    MPI_Type_size(rdt, &rlength);
     rlength *= rcount;
 
-    MPI_Comm_rank( MPI_COMM_WORLD, &myself );
+    MPI_Comm_rank(MPI_COMM_WORLD, &myself);
 
-    for( t = 0; t < trials; t++ ) {
+    for (t = 0; t < trials; t++) {
         timers[t] = MPI_Wtime();
-        for( c = 0; c < cycles; c++ ) {
-            MPI_Isend( sbuf, scount, sdt, myself, tag, MPI_COMM_WORLD, &requests[0] );
-            MPI_Irecv( rbuf, rcount, rdt, myself, tag, MPI_COMM_WORLD, &requests[1] );
-            MPI_Waitall( 2, requests, statuses );
+        for (c = 0; c < cycles; c++) {
+            MPI_Isend(sbuf, scount, sdt, myself, tag, MPI_COMM_WORLD, &requests[0]);
+            MPI_Irecv(rbuf, rcount, rdt, myself, tag, MPI_COMM_WORLD, &requests[1]);
+            MPI_Waitall(2, requests, statuses);
         }
         timers[t] = (MPI_Wtime() - timers[t]) / cycles;
     }
-    print_result( rlength, trials, timers );
+    print_result(rlength, trials, timers);
     return 0;
 }
 
-static int irecv_isend_wait( int cycles,
-                             MPI_Datatype sdt, int scount, void* sbuf,
-                             MPI_Datatype rdt, int rcount, void* rbuf )
+static int irecv_isend_wait(int cycles, MPI_Datatype sdt, int scount, void *sbuf, MPI_Datatype rdt,
+                            int rcount, void *rbuf)
 {
     int myself, tag = 0, c, t, slength, rlength;
     MPI_Request requests[2];
     MPI_Status statuses[2];
     double timers[trials];
 
-    MPI_Type_size( sdt, &slength );
+    MPI_Type_size(sdt, &slength);
     slength *= scount;
-    MPI_Type_size( rdt, &rlength );
+    MPI_Type_size(rdt, &rlength);
     rlength *= rcount;
 
-    MPI_Comm_rank( MPI_COMM_WORLD, &myself );
+    MPI_Comm_rank(MPI_COMM_WORLD, &myself);
 
-    for( t = 0; t < trials; t++ ) {
+    for (t = 0; t < trials; t++) {
         timers[t] = MPI_Wtime();
-        for( c = 0; c < cycles; c++ ) {
-            MPI_Irecv( rbuf, rcount, rdt, myself, tag, MPI_COMM_WORLD, &requests[0] );
-            MPI_Isend( sbuf, scount, sdt, myself, tag, MPI_COMM_WORLD, &requests[1] );
-            MPI_Waitall( 2, requests, statuses );
+        for (c = 0; c < cycles; c++) {
+            MPI_Irecv(rbuf, rcount, rdt, myself, tag, MPI_COMM_WORLD, &requests[0]);
+            MPI_Isend(sbuf, scount, sdt, myself, tag, MPI_COMM_WORLD, &requests[1]);
+            MPI_Waitall(2, requests, statuses);
         }
         timers[t] = (MPI_Wtime() - timers[t]) / cycles;
     }
-    print_result( rlength, trials, timers);
+    print_result(rlength, trials, timers);
     return 0;
 }
 
-static int do_test_for_ddt( int doop, MPI_Datatype sddt, MPI_Datatype rddt, int length )
+static int do_test_for_ddt(int doop, MPI_Datatype sddt, MPI_Datatype rddt, int length)
 {
     MPI_Aint lb, extent;
     char *sbuf, *rbuf;
     int i;
 
-    MPI_Type_get_extent( sddt, &lb, &extent );
-    sbuf = (char*)malloc( length );
-    rbuf = (char*)malloc( length );
-    if( doop & DO_PACK ) {
+    MPI_Type_get_extent(sddt, &lb, &extent);
+    sbuf = (char *) malloc(length);
+    rbuf = (char *) malloc(length);
+    if (doop & DO_PACK) {
         printf("# Pack (max length %d)\n", length);
-        for( i = 1; i <= (length/extent); i *= 2 ) {
-            pack( cycles, sddt, i, sbuf, rbuf );
+        for (i = 1; i <= (length / extent); i *= 2) {
+            pack(cycles, sddt, i, sbuf, rbuf);
         }
     }
 
-    if( doop & DO_UNPACK ) {
+    if (doop & DO_UNPACK) {
         printf("# Unpack (length %d)\n", length);
-        for( i = 1; i <= (length/extent); i *= 2 ) {
-            unpack( cycles, sbuf, rddt, i, rbuf );
+        for (i = 1; i <= (length / extent); i *= 2) {
+            unpack(cycles, sbuf, rddt, i, rbuf);
         }
     }
 
-    if( doop & DO_ISEND_RECV ) {
-        printf( "# Isend recv (length %d)\n", length );
-        for( i = 1; i <= (length/extent); i *= 2 ) {
-            isend_recv( cycles, sddt, i, sbuf, rddt, i, rbuf );
+    if (doop & DO_ISEND_RECV) {
+        printf("# Isend recv (length %d)\n", length);
+        for (i = 1; i <= (length / extent); i *= 2) {
+            isend_recv(cycles, sddt, i, sbuf, rddt, i, rbuf);
         }
     }
 
-    if( doop & DO_ISEND_IRECV ) {
-        printf( "# Isend Irecv Wait (length %d)\n", length );
-        for( i = 1; i <= (length/extent); i *= 2 ) {
-            isend_irecv_wait( cycles, sddt, i, sbuf, rddt, i, rbuf );
+    if (doop & DO_ISEND_IRECV) {
+        printf("# Isend Irecv Wait (length %d)\n", length);
+        for (i = 1; i <= (length / extent); i *= 2) {
+            isend_irecv_wait(cycles, sddt, i, sbuf, rddt, i, rbuf);
         }
     }
 
-    if( doop & DO_IRECV_SEND ) {
-        printf( "# Irecv send (length %d)\n", length );
-        for( i = 1; i <= (length/extent); i *= 2 ) {
-            irecv_send( cycles, sddt, i, sbuf, rddt, i, rbuf );
+    if (doop & DO_IRECV_SEND) {
+        printf("# Irecv send (length %d)\n", length);
+        for (i = 1; i <= (length / extent); i *= 2) {
+            irecv_send(cycles, sddt, i, sbuf, rddt, i, rbuf);
         }
     }
 
-    if( doop & DO_IRECV_SEND ) {
-        printf( "# Irecv Isend Wait (length %d)\n", length );
-        for( i = 1; i <= (length/extent); i *= 2 ) {
-            irecv_isend_wait( cycles, sddt, i, sbuf, rddt, i, rbuf );
+    if (doop & DO_IRECV_SEND) {
+        printf("# Irecv Isend Wait (length %d)\n", length);
+        for (i = 1; i <= (length / extent); i *= 2) {
+            irecv_isend_wait(cycles, sddt, i, sbuf, rddt, i, rbuf);
         }
     }
-    free( sbuf );
-    free( rbuf );
+    free(sbuf);
+    free(rbuf);
     return 0;
 }
 
-int main( int argc, char* argv[] )
+int main(int argc, char *argv[])
 {
-    int run_tests = 0xffff;  /* do all datatype tests by default */
+    int run_tests = 0xffff; /* do all datatype tests by default */
     int rank, size;
     MPI_Datatype ddt;
 
     run_tests |= DO_PACK | DO_UNPACK;
-    
-    MPI_Init (&argc, &argv);
 
-    MPI_Comm_rank (MPI_COMM_WORLD, &rank);
-    MPI_Comm_size (MPI_COMM_WORLD, &size);
+    MPI_Init(&argc, &argv);
+
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    MPI_Comm_size(MPI_COMM_WORLD, &size);
 
-    if( rank != 0 ) {
+    if (rank != 0) {
         MPI_Finalize();
         exit(0);
     }
 
-    if( run_tests & DO_CONTIG ) {
-        printf( "\ncontiguous datatype\n\n" );
-        do_test_for_ddt( run_tests, MPI_INT, MPI_INT, MAX_LENGTH );
+    if (run_tests & DO_CONTIG) {
+        printf("\ncontiguous datatype\n\n");
+        do_test_for_ddt(run_tests, MPI_INT, MPI_INT, MAX_LENGTH);
     }
 
-    if( run_tests & DO_INDEXED_GAP ) {
-        printf( "\nindexed gap\n\n" );
+    if (run_tests & DO_INDEXED_GAP) {
+        printf("\nindexed gap\n\n");
         ddt = create_indexed_gap_ddt();
-        MPI_DDT_DUMP( ddt );
-        do_test_for_ddt( run_tests, ddt, ddt, MAX_LENGTH );
-        MPI_Type_free( &ddt );
+        MPI_DDT_DUMP(ddt);
+        do_test_for_ddt(run_tests, ddt, ddt, MAX_LENGTH);
+        MPI_Type_free(&ddt);
     }
 
-    if( run_tests & DO_OPTIMIZED_INDEXED_GAP ) {
-        printf( "\noptimized indexed gap\n\n" );
+    if (run_tests & DO_OPTIMIZED_INDEXED_GAP) {
+        printf("\noptimized indexed gap\n\n");
         ddt = create_indexed_gap_optimized_ddt();
-        MPI_DDT_DUMP( ddt );
-        do_test_for_ddt( run_tests, ddt, ddt, MAX_LENGTH );
-        MPI_Type_free( &ddt );
+        MPI_DDT_DUMP(ddt);
+        do_test_for_ddt(run_tests, ddt, ddt, MAX_LENGTH);
+        MPI_Type_free(&ddt);
     }
 
-    if( run_tests & DO_CONSTANT_GAP ) {
-        printf( "\nconstant indexed gap\n\n" );
-        ddt = create_indexed_constant_gap_ddt( 80, 100, 1 );
-        MPI_DDT_DUMP( ddt );
-        do_test_for_ddt( run_tests, ddt, ddt, MAX_LENGTH );
-        MPI_Type_free( &ddt );
+    if (run_tests & DO_CONSTANT_GAP) {
+        printf("\nconstant indexed gap\n\n");
+        ddt = create_indexed_constant_gap_ddt(80, 100, 1);
+        MPI_DDT_DUMP(ddt);
+        do_test_for_ddt(run_tests, ddt, ddt, MAX_LENGTH);
+        MPI_Type_free(&ddt);
     }
 
-    if( run_tests & DO_CONSTANT_GAP ) {
-        printf( "\noptimized constant indexed gap\n\n" );
-        ddt = create_optimized_indexed_constant_gap_ddt( 80, 100, 1 );
-        MPI_DDT_DUMP( ddt );
-        do_test_for_ddt( run_tests, ddt, ddt, MAX_LENGTH );
-        MPI_Type_free( &ddt );
+    if (run_tests & DO_CONSTANT_GAP) {
+        printf("\noptimized constant indexed gap\n\n");
+        ddt = create_optimized_indexed_constant_gap_ddt(80, 100, 1);
+        MPI_DDT_DUMP(ddt);
+        do_test_for_ddt(run_tests, ddt, ddt, MAX_LENGTH);
+        MPI_Type_free(&ddt);
     }
 
-    if( run_tests & DO_STRUCT_CONSTANT_GAP_RESIZED ) {
-        printf( "\nstruct constant gap resized\n\n" );
-        ddt = create_struct_constant_gap_resized_ddt( 0 /* unused */, 0 /* unused */, 0 /* unused */ );
-        MPI_DDT_DUMP( ddt );
-        do_test_for_ddt( run_tests, ddt, ddt, MAX_LENGTH );
-        MPI_Type_free( &ddt );
+    if (run_tests & DO_STRUCT_CONSTANT_GAP_RESIZED) {
+        printf("\nstruct constant gap resized\n\n");
+        ddt = create_struct_constant_gap_resized_ddt(0 /* unused */, 0 /* unused */,
+                                                     0 /* unused */);
+        MPI_DDT_DUMP(ddt);
+        do_test_for_ddt(run_tests, ddt, ddt, MAX_LENGTH);
+        MPI_Type_free(&ddt);
     }
 
-    if( run_tests & DO_STRUCT_MERGED_WITH_GAP_RESIZED ) {
-        printf( "\nstruct constant gap resized\n\n" );
-        ddt = create_merged_contig_with_gaps( 1 );
-        MPI_DDT_DUMP( ddt );
-        do_test_for_ddt( run_tests, ddt, ddt, MAX_LENGTH );
-        MPI_Type_free( &ddt );
+    if (run_tests & DO_STRUCT_MERGED_WITH_GAP_RESIZED) {
+        printf("\nstruct constant gap resized\n\n");
+        ddt = create_merged_contig_with_gaps(1);
+        MPI_DDT_DUMP(ddt);
+        do_test_for_ddt(run_tests, ddt, ddt, MAX_LENGTH);
+        MPI_Type_free(&ddt);
     }
 
-    MPI_Finalize ();
+    MPI_Finalize();
     exit(0);
 }
-
diff --git a/test/datatype/unpack_hetero.c b/test/datatype/unpack_hetero.c
index 49e31b006dd..731d7fb62c8 100644
--- a/test/datatype/unpack_hetero.c
+++ b/test/datatype/unpack_hetero.c
@@ -2,6 +2,7 @@
 /*
  * Copyright (c) 2014-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -10,22 +11,23 @@
  */
 
 #include "opal_config.h"
-#include "opal/runtime/opal.h"
+#include "opal/datatype/opal_convertor.h"
 #include "opal/datatype/opal_datatype.h"
 #include "opal/datatype/opal_datatype_internal.h"
-#include "opal/datatype/opal_convertor.h"
 #include "opal/datatype/opal_datatype_prototypes.h"
+#include "opal/runtime/opal.h"
 #include "opal/util/arch.h"
-#include <time.h>
 #include <stdlib.h>
+#include <time.h>
 #ifdef HAVE_SYS_TIME_H
-#include <sys/time.h>
+#    include <sys/time.h>
 #endif
 #include <stdio.h>
 #include <string.h>
 
 /* Compile with:
-gcc -DHAVE_CONFIG_H -I. -I../../include -I../.. -I../../include -I../../../ompi-trunk/opal -I../../../ompi-trunk/orte -g opal_datatype_test.c -o opal_datatype_test
+gcc -DHAVE_CONFIG_H -I. -I../../include -I../.. -I../../include -I../../../ompi-trunk/opal
+-I../../../ompi-trunk/orte -g opal_datatype_test.c -o opal_datatype_test
 */
 
 uint32_t remote_arch = 0xffffffff;
@@ -37,63 +39,65 @@ uint32_t remote_arch = 0xffffffff;
  * on the data-type engine should first pass all the tests from this file, before going into other
  * tests.
  */
-int main( int argc, char* argv[] )
+int main(int argc, char *argv[])
 {
-    opal_init_util (NULL, NULL);
+    opal_init(NULL, NULL);
 
     /**
      * By default simulate homogeneous architectures.
      */
     remote_arch = opal_local_arch ^ OPAL_ARCH_ISBIGENDIAN;
 
-    opal_convertor_t * pConv;
+    opal_convertor_t *pConv;
     int sbuf[2], rbuf[2];
     size_t max_data;
     struct iovec a;
     uint32_t iov_count;
 
-    sbuf[0] = 0x01000000; sbuf[1] = 0x02000000;
+    sbuf[0] = 0x01000000;
+    sbuf[1] = 0x02000000;
 
-    printf( "\n\n#\n * TEST UNPACKING 1 int out of 1\n#\n\n" );
+    printf("\n\n#\n * TEST UNPACKING 1 int out of 1\n#\n\n");
 
-    pConv = opal_convertor_create( remote_arch, 0 );
-    rbuf[0] = -1; rbuf[1] = -1;
-    if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( pConv, &opal_datatype_int4, 1, rbuf ) ) {
-        printf( "Cannot attach the datatype to a convertor\n" );
+    pConv = opal_convertor_create(remote_arch, 0);
+    rbuf[0] = -1;
+    rbuf[1] = -1;
+    if (OPAL_SUCCESS != opal_convertor_prepare_for_recv(pConv, &opal_datatype_int4, 1, rbuf)) {
+        printf("Cannot attach the datatype to a convertor\n");
         return OPAL_ERROR;
     }
-    
+
     a.iov_base = sbuf;
     a.iov_len = 4;
     iov_count = 1;
     max_data = 4;
-    opal_unpack_general( pConv, &a, &iov_count, &max_data );
+    opal_unpack_general(pConv, &a, &iov_count, &max_data);
 
     assert(1 == rbuf[0]);
     assert(-1 == rbuf[1]);
     OBJ_RELEASE(pConv);
 
-    printf( "\n\n#\n * TEST UNPACKING 1 int out of 2\n#\n\n" );
-    pConv = opal_convertor_create( remote_arch, 0 );
-    rbuf[0] = -1; rbuf[1] = -1;
-    if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( pConv, &opal_datatype_int4, 2, rbuf ) ) {
-        printf( "Cannot attach the datatype to a convertor\n" );
+    printf("\n\n#\n * TEST UNPACKING 1 int out of 2\n#\n\n");
+    pConv = opal_convertor_create(remote_arch, 0);
+    rbuf[0] = -1;
+    rbuf[1] = -1;
+    if (OPAL_SUCCESS != opal_convertor_prepare_for_recv(pConv, &opal_datatype_int4, 2, rbuf)) {
+        printf("Cannot attach the datatype to a convertor\n");
         return OPAL_ERROR;
     }
-    
 
     a.iov_base = sbuf;
     a.iov_len = 4;
     iov_count = 1;
     max_data = 4;
-    opal_unpack_general( pConv, &a, &iov_count, &max_data );
+    opal_unpack_general(pConv, &a, &iov_count, &max_data);
 
     assert(1 == rbuf[0]);
     assert(-1 == rbuf[1]);
     OBJ_RELEASE(pConv);
 
     /* clean-ups all data allocations */
-    opal_finalize_util ();
+    opal_finalize();
 
     return OPAL_SUCCESS;
 }
diff --git a/test/datatype/unpack_ooo.c b/test/datatype/unpack_ooo.c
index 58ef8a95774..7fdfe790916 100644
--- a/test/datatype/unpack_ooo.c
+++ b/test/datatype/unpack_ooo.c
@@ -15,18 +15,19 @@
 
 #include "ompi_config.h"
 #include "ompi/datatype/ompi_datatype.h"
-#include "opal/runtime/opal.h"
 #include "opal/datatype/opal_convertor.h"
 #include "opal/datatype/opal_datatype_internal.h"
-#include <time.h>
-#include <stdlib.h>
+#include "opal/runtime/opal.h"
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
+#include <time.h>
 #include <unistd.h>
 
 #define N 331
 
 uint32_t remote_arch = 0xffffffff;
+bool report_all_errors = true;
 
 struct foo_t {
     int i[3];
@@ -38,53 +39,58 @@ struct pfoo_t {
     double d[2];
 } pfoo = {0}, *pbar = NULL;
 
-static void print_hex(void* ptr, int count, int space)
+static void print_hex(void *ptr, int count, char *epilog, char *prolog)
 {
-    for( int i = 0; i < count; i++ ) {
-        fprintf(stderr, "%02x", (unsigned int)(((unsigned char*)ptr)[i]));
+    if (NULL != epilog)
+        fprintf(stderr, "%s", epilog);
+    for (int i = 0; i < count; i++) {
+        fprintf(stderr, "%02x", (unsigned int) (((unsigned char *) ptr)[i]));
     }
-    if(space) fprintf(stderr, " ");
+    if (NULL != prolog)
+        fprintf(stderr, "%s", prolog);
 }
 
-static void print_bar_pbar(struct foo_t* bar, struct pfoo_t* pbar)
+static void print_bar_pbar(struct foo_t *_bar, struct pfoo_t *_pbar)
 {
-    print_hex(&bar->i[0], sizeof(int), 1);
-    print_hex(&bar->i[2], sizeof(int), 1);
-    print_hex(&bar->d[0], sizeof(double), 1);
-    print_hex(&bar->d[2], sizeof(double), 1);
-    fprintf(stderr, "\n");
-    print_hex(&pbar->i[0], sizeof(int), 1);
-    print_hex(&pbar->i[1], sizeof(int), 1);
-    print_hex(&pbar->d[0], sizeof(double), 1);
-    print_hex(&pbar->d[1], sizeof(double), 1);
-    fprintf(stderr, "\n");
+    print_hex(&_bar->i[0], sizeof(int), NULL, " ");
+    print_hex(&_bar->i[1], sizeof(int), "[", "] ");
+    print_hex(&_bar->i[2], sizeof(int), NULL, " ");
+    print_hex(&_bar->d[0], sizeof(double), NULL, " ");
+    print_hex(&_bar->d[1], sizeof(double), "[", "] ");
+    print_hex(&_bar->d[2], sizeof(double), NULL, "\n");
+
+    print_hex(&_pbar->i[0], sizeof(int), NULL, "            ");
+    print_hex(&_pbar->i[1], sizeof(int), NULL, " ");
+    print_hex(&_pbar->d[0], sizeof(double), NULL, "                    ");
+    print_hex(&_pbar->d[1], sizeof(double), NULL, "\n");
 }
 
-static void print_stack(opal_convertor_t* conv)
+static void print_stack(opal_convertor_t *conv)
 {
-    printf("Stack pos %d [converted %" PRIsize_t "/%" PRIsize_t "]\n",
-           conv->stack_pos, conv->bConverted, conv->local_size);
-    for( uint32_t i = 0; i <= conv->stack_pos; i++ ) {
-        printf( "[%u] index %d, type %s count %" PRIsize_t " disp %p\n",
-                i, conv->pStack[i].index, opal_datatype_basicDatatypes[conv->pStack[i].type]->name,
-                conv->pStack[i].count, (void*)conv->pStack[i].disp);
+    printf("Stack pos %d [converted %" PRIsize_t "/%" PRIsize_t "]\n", conv->stack_pos,
+           conv->bConverted, conv->local_size);
+    for (uint32_t i = 0; i <= conv->stack_pos; i++) {
+        printf("[%u] index %d, type %s count %" PRIsize_t " disp %p\n", i, conv->pStack[i].index,
+               opal_datatype_basicDatatypes[conv->pStack[i].type]->name, conv->pStack[i].count,
+               (void *) conv->pStack[i].disp);
     }
     printf("\n");
 }
 
-static int testcase(ompi_datatype_t * newtype, size_t arr[10][2]) {
+static int testcase(ompi_datatype_t *newtype, size_t arr[][2])
+{
     int i, j, errors = 0;
     struct iovec a;
     unsigned int iov_count;
     size_t max_data;
     size_t pos;
-    opal_convertor_t * pConv;
+    opal_convertor_t *pConv;
 
     for (j = 0; j < N; ++j) {
-        pbar[j].i[0] = 123+j;
-        pbar[j].i[1] = 789+j;
-        pbar[j].d[0] = 123.456+j;
-        pbar[j].d[1] = 789.123+j;
+        pbar[j].i[0] = 123 + j;
+        pbar[j].i[1] = 789 + j;
+        pbar[j].d[0] = 123.456 + j;
+        pbar[j].d[1] = 789.123 + j;
         memset(&bar[j].i[0], 0xFF, sizeof(int));
         memset(&bar[j].i[2], 0xFF, sizeof(int));
         bar[j].i[1] = 0;
@@ -93,23 +99,23 @@ static int testcase(ompi_datatype_t * newtype, size_t arr[10][2]) {
         bar[j].d[1] = 0.0;
     }
 
-    pConv = opal_convertor_create( remote_arch, 0 );
-    if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( pConv, &(newtype->super), N, bar ) ) {
-        printf( "Cannot attach the datatype to a convertor\n" );
+    pConv = opal_convertor_create(remote_arch, 0);
+    if (OPAL_SUCCESS != opal_convertor_prepare_for_recv(pConv, &(newtype->super), N, bar)) {
+        printf("Cannot attach the datatype to a convertor\n");
         return OMPI_ERROR;
     }
 
-    for (i=0; arr[i][0] != 0; i++) {
+    for (i = 0; 0 != arr[i][0]; i++) {
         /* add some garbage before and after the source data */
-        a.iov_base = malloc(arr[i][0]+2048);
+        a.iov_base = malloc(arr[i][0] + 2048);
         if (NULL == a.iov_base) {
             printf("cannot malloc iov_base\n");
             return 1;
         }
         memset(a.iov_base, 0xAA, 1024);
-        memcpy((char*)a.iov_base+1024, (char *)pbar + arr[i][1], arr[i][0]);
-        memset((char*)a.iov_base+1024+arr[i][0], 0xAA, 1024);
-        a.iov_base = (char*)a.iov_base + 1024;
+        memcpy((char *) a.iov_base + 1024, (char *) pbar + arr[i][1], arr[i][0]);
+        memset((char *) a.iov_base + 1024 + arr[i][0], 0xAA, 1024);
+        a.iov_base = (char *) a.iov_base + 1024;
         a.iov_len = arr[i][0];
         iov_count = 1;
         max_data = a.iov_len;
@@ -117,50 +123,66 @@ static int testcase(ompi_datatype_t * newtype, size_t arr[10][2]) {
         opal_convertor_set_position(pConv, &pos);
         print_stack(pConv);
         assert(arr[i][1] == pos);
-        opal_convertor_unpack( pConv, &a, &iov_count, &max_data );
-        a.iov_base = (char*)a.iov_base - 1024;
+        opal_convertor_unpack(pConv, &a, &iov_count, &max_data);
+        a.iov_base = (char *) a.iov_base - 1024;
         free(a.iov_base);
     }
 
     for (j = 0; j < N; ++j) {
-        if (bar[j].i[0] != pbar[j].i[0] ||
-            bar[j].i[1] != 0 ||
-            bar[j].i[2] != pbar[j].i[1] ||
-            bar[j].d[0] != pbar[j].d[0] ||
-            bar[j].d[1] != 0.0 ||
-            bar[j].d[2] != pbar[j].d[1]) {
-            if(0 == errors) {
-                (void)opal_datatype_dump(&newtype->super);
-                fprintf(stderr, "ERROR ! position=%d/%d, ptr = %p"
+        if (bar[j].i[0] != pbar[j].i[0] || bar[j].i[1] != 0 || bar[j].i[2] != pbar[j].i[1]
+            || bar[j].d[0] != pbar[j].d[0] || bar[j].d[1] != 0.0 || bar[j].d[2] != pbar[j].d[1]) {
+            if (0 == errors || report_all_errors) {
+                ptrdiff_t displ;
+                char *error_location = "in gaps";
+                if (bar[j].i[0] != pbar[j].i[0]) {
+                    displ = (char *) &bar[j].i[0] - (char *) &bar[0];
+                    error_location = "i[0]";
+                } else if (bar[j].i[2] != pbar[j].i[1]) {
+                    displ = (char *) &bar[j].i[1] - (char *) &bar[0];
+                    error_location = "i[2]";
+                } else if (bar[j].d[0] != pbar[j].d[0]) {
+                    displ = (char *) &bar[j].d[0] - (char *) &bar[0];
+                    error_location = "d[0]";
+                } else if (bar[j].d[2] != pbar[j].d[1]) {
+                    displ = (char *) &bar[j].d[1] - (char *) &bar[0];
+                    error_location = "d[2]";
+                } else {
+                    displ = (char *) &bar[j] - (char *) &bar[0];
+                }
+                for (i = 0; 0 != arr[i][0]; i++) {
+                    if ((displ >= arr[i][1]) && (displ <= (arr[i][1] + arr[i][0]))) {
+                        fprintf(stderr,
+                                "Problem encountered %li bytes into the %d unpack [%" PRIsize_t
+                                ":%" PRIsize_t "]\n",
+                                displ - arr[i][1], i, arr[i][1], arr[i][0]);
+                        break;
+                    }
+                }
+
+                (void) opal_datatype_dump(&newtype->super);
+                fprintf(stderr,
+                        "ERROR ! struct %d/%d in field %s, ptr = %p"
                         " got (%d,%d,%d,%g,%g,%g) expected (%d,%d,%d,%g,%g,%g)\n",
-                        j, N, (void*)&bar[j],
-                        bar[j].i[0],
-                        bar[j].i[1],
-                        bar[j].i[2],
-                        bar[j].d[0],
-                        bar[j].d[1],
-                        bar[j].d[2],
-                        pbar[j].i[0],
-                        0,
-                        pbar[j].i[1],
-                        pbar[j].d[0],
-                        0.0,
-                        pbar[j].d[1]);
+                        j, N, error_location, (void *) &bar[j], bar[j].i[0], bar[j].i[1],
+                        bar[j].i[2], bar[j].d[0], bar[j].d[1], bar[j].d[2], pbar[j].i[0], 0,
+                        pbar[j].i[1], pbar[j].d[0], 0.0, pbar[j].d[1]);
                 print_bar_pbar(&bar[j], &pbar[j]);
+                if (report_all_errors)
+                    fprintf(stderr, "\n\n");
             }
             errors++;
         }
     }
-    OBJ_RELEASE( pConv );
+    OBJ_RELEASE(pConv);
     return errors;
 }
 
 static int unpack_ooo(void)
 {
-    ompi_datatype_t * t1;
-    ompi_datatype_t * t2;
-    ompi_datatype_t * type[2];
-    ompi_datatype_t * newtype;
+    ompi_datatype_t *t1;
+    ompi_datatype_t *t2;
+    ompi_datatype_t *type[2];
+    ompi_datatype_t *newtype;
     MPI_Aint disp[2];
     int len[2], rc;
 
@@ -169,7 +191,7 @@ static int unpack_ooo(void)
         fprintf(stderr, "could not create vector t1\n");
         return 1;
     }
-    rc = ompi_datatype_commit (&t1);
+    rc = ompi_datatype_commit(&t1);
     if (OMPI_SUCCESS != rc) {
         fprintf(stderr, "could not commit vector t1\n");
         return 1;
@@ -180,77 +202,61 @@ static int unpack_ooo(void)
         fprintf(stderr, "could not create vector t2\n");
         return 1;
     }
-    rc = ompi_datatype_commit (&t2);
+    rc = ompi_datatype_commit(&t2);
     if (OMPI_SUCCESS != rc) {
         fprintf(stderr, "could not commit vector t2\n");
         return 1;
     }
 
-/*
- * btl=0x7f7823672580 bytes_received=992 data_offset=0
- * btl=0x7f7823260420 bytes_received=1325 data_offset=992
- * btl=0x7f7823672580 bytes_received=992 data_offset=2317
- * btl=0x7f7823672580 bytes_received=992 data_offset=3309
- * btl=0x7f7823672580 bytes_received=992 data_offset=4301
- * btl=0x7f7823672580 bytes_received=992 data_offset=5293
- * btl=0x7f7823672580 bytes_received=992 data_offset=6285
- * btl=0x7f7823672580 bytes_received=667 data_offset=7277
- */
+    /*
+     * btl=0x7f7823672580 bytes_received=992 data_offset=0
+     * btl=0x7f7823260420 bytes_received=1325 data_offset=992
+     * btl=0x7f7823672580 bytes_received=992 data_offset=2317
+     * btl=0x7f7823672580 bytes_received=992 data_offset=3309
+     * btl=0x7f7823672580 bytes_received=992 data_offset=4301
+     * btl=0x7f7823672580 bytes_received=992 data_offset=5293
+     * btl=0x7f7823672580 bytes_received=992 data_offset=6285
+     * btl=0x7f7823672580 bytes_received=667 data_offset=7277
+     */
     size_t test1[9][2] = {
         {992, 0},
-        {1325, 992},
-        {992, 2317},
-        {992, 3309},
-        {992, 4301},
-        {992, 5293},
-        {992, 6285},
-        {667, 7277},
+        {1325, 0 + 992},
+        {992, 992 + 1325 /* = 2317 */},
+        {992, 2317 + 992 /* = 3309 */},
+        {992, 3309 + 992 /* = 4301 */},
+        {992, 4301 + 992 /* = 5293 */},
+        {992, 5293 + 992 /* = 6285 */},
+        {667, 6285 + 992 /* = 7277 */},
         {0, -1},
     };
 
-/*
- * btl=0x7f80bc545580 bytes_received=992 data_offset=0
- * btl=0x7f80bc545580 bytes_received=992 data_offset=2317
- * btl=0x7f80bc545580 bytes_received=992 data_offset=3309
- * btl=0x7f80bc545580 bytes_received=992 data_offset=4301
- * btl=0x7f80bc545580 bytes_received=992 data_offset=5293
- * btl=0x7f80bc545580 bytes_received=992 data_offset=6285
- * btl=0x7f80bc133420 bytes_received=1325 data_offset=992
- * btl=0x7f80bc545580 bytes_received=667 data_offset=7277
- */
+    /*
+     * btl=0x7f80bc545580 bytes_received=992 data_offset=0
+     * btl=0x7f80bc545580 bytes_received=992 data_offset=2317
+     * btl=0x7f80bc545580 bytes_received=992 data_offset=3309
+     * btl=0x7f80bc545580 bytes_received=992 data_offset=4301
+     * btl=0x7f80bc545580 bytes_received=992 data_offset=5293
+     * btl=0x7f80bc545580 bytes_received=992 data_offset=6285
+     * btl=0x7f80bc133420 bytes_received=1325 data_offset=992
+     * btl=0x7f80bc545580 bytes_received=667 data_offset=7277
+     */
     size_t test2[9][2] = {
-        {992, 0},
-        {992, 2317},
-        {992, 3309},
-        {992, 4301},
-        {992, 5293},
-        {992, 6285},
-        {1325, 992},
-        {667, 7277},
-        {0, -1},
+        {992, 0},    {992, 2317}, {992, 3309}, {992, 4301}, {992, 5293},
+        {992, 6285}, {1325, 992}, {667, 7277}, {0, -1},
     };
 
-/* trimmed version of test2 */
+    /* trimmed version of test2 */
     size_t test3[9][2] = {
-        {992, 0},
-        {4960, 2317},
-        {1325, 992},
-        {667, 7277},
-        {0, -1},
+        {992, 0}, {4960, 2317}, {1325, 992}, {667, 7277}, {0, -1},
     };
 
-/* an other test case */
+    /* an other test case */
     size_t test4[9][2] = {
-        {992, 0},
-        {992, 2976},
-        {992, 1984},
-        {992, 992},
-        {3976, 3968},
-        {0, -1},
+        {992, 0}, {992, 2976}, {992, 1984}, {992, 992}, {3976, 3968}, {0, -1},
     };
 
-    disp[0] = (long)(&foo.i[0]) - (long)&foo;
-    disp[1] = (long)(&foo.d[0]) - (long)&foo;
+    disp[0] = (long) (&foo.i[0]) - (long) &foo;
+    disp[1] = (long) (&foo.d[0]) - (long) &foo;
 
     type[0] = t1;
     type[1] = t2;
@@ -263,67 +269,67 @@ static int unpack_ooo(void)
         fprintf(stderr, "could not create struct\n");
         return 1;
     }
-    rc = ompi_datatype_commit (&newtype);
+    rc = ompi_datatype_commit(&newtype);
     if (OMPI_SUCCESS != rc) {
         fprintf(stderr, "could not create struct\n");
         return 1;
     }
 
-    pbar = (struct pfoo_t *)malloc (N * sizeof(struct pfoo_t));
+    pbar = (struct pfoo_t *) malloc(N * sizeof(struct pfoo_t));
     if (NULL == pbar) {
         fprintf(stderr, "could not malloc pbar\n");
         return 1;
     }
-    bar = (struct foo_t *)malloc (N * sizeof(struct foo_t));
+    bar = (struct foo_t *) malloc(N * sizeof(struct foo_t));
     if (NULL == bar) {
         fprintf(stderr, "could not malloc bar\n");
         return 1;
     }
 
     if (0 != testcase(newtype, test1)) {
-        printf ("test1 failed\n");
+        printf("test1 failed\n");
         return 2;
     }
 
     if (0 != testcase(newtype, test2)) {
-        printf ("test2 failed\n");
+        printf("test2 failed\n");
         return 2;
     }
 
     if (0 != testcase(newtype, test3)) {
-        printf ("test3 failed\n");
+        printf("test3 failed\n");
         return 2;
     }
 
     if (0 != testcase(newtype, test4)) {
-        printf ("test4 failed\n");
+        printf("test4 failed\n");
         return 2;
     }
 
-
     /* test the automatic destruction pf the data */
-    ompi_datatype_destroy( &newtype ); assert( newtype == NULL );
+    ompi_datatype_destroy(&newtype);
+    assert(newtype == NULL);
 
     return rc;
 }
 
-int main( int argc, char* argv[] )
+int main(int argc, char *argv[])
 {
     int rc;
 
-    opal_init_util(&argc, &argv);
+    opal_init(&argc, &argv);
     ompi_datatype_init();
     /**
      * By default simulate homogeneous architectures.
      */
     remote_arch = opal_local_arch;
 
-    printf( "\n\n#\n * TEST UNPACK OUT OF ORDER\n #\n\n" );
+    printf("\n\n#\n * TEST UNPACK OUT OF ORDER\n #\n\n");
     rc = unpack_ooo();
-    if( rc == 0 ) {
-        printf( "unpack out of order [PASSED]\n" );
+    if (rc == 0) {
+        printf("unpack out of order [PASSED]\n");
         return 0;
     }
-    printf( "unpack out of order [NOT PASSED]\n" );
+    printf("unpack out of order [NOT PASSED]\n");
     return -1;
 }
diff --git a/test/event/Makefile.am b/test/event/Makefile.am
index 03917bba70a..ffba4a0f224 100644
--- a/test/event/Makefile.am
+++ b/test/event/Makefile.am
@@ -29,20 +29,20 @@ TESTS = $(check_PROGRAMS)
 signal_test_SOURCES = signal-test.c
 signal_test_LDFLAGS = $(OPAL_PKG_CONFIG_LDFLAGS)
 signal_test_LDADD = \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 signal_test_DEPENDENCIES = $(signal_test_LDADD)
 
 time_test_SOURCES = time-test.c
 time_test_LDFLAGS = $(OPAL_PKG_CONFIG_LDFLAGS)
 time_test_LDADD = \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 time_test_DEPENDENCIES = $(time_test_LDADD)
 
 event_test_SOURCES = event-test.c
 event_test_LDFLAGS = $(OPAL_PKG_CONFIG_LDFLAGS)
 event_test_LDADD = \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 event_test_DEPENDENCIES = $(event_test_LDADD)
 
-distclean:
+distclean-local:
 	rm -rf *.dSYM .deps .libs *.log *.o *.trs $(check_PROGRAMS) Makefile
diff --git a/test/event/event-test.c b/test/event/event-test.c
index 5701dfe2ab7..672f767123d 100644
--- a/test/event/event-test.c
+++ b/test/event/event-test.c
@@ -4,142 +4,138 @@
  */
 
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 #include <sys/stat.h>
 #ifndef WIN32
-#ifdef HAVE_SYS_QUEUE_H
-#include <sys/queue.h>
-#endif
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_SYS_TIME_H
-#include <sys/time.h>
-#endif
+#    ifdef HAVE_SYS_QUEUE_H
+#        include <sys/queue.h>
+#    endif
+#    ifdef HAVE_UNISTD_H
+#        include <unistd.h>
+#    endif
+#    ifdef HAVE_SYS_TIME_H
+#        include <sys/time.h>
+#    endif
 #else
-#include <windows.h>
+#    include <windows.h>
 #endif
+#include <errno.h>
 #include <fcntl.h>
-#include <stdlib.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
-#include <errno.h>
 #include <unistd.h>
 
-#include "opal/util/event.h"
 #include "opal/runtime/opal.h"
+#include "opal/util/event.h"
 
-static void
-fifo_read(int fd, short event, void *arg)
+static void fifo_read(int fd, short event, void *arg)
 {
-	char buf[255];
-	int len;
-	opal_event_t *ev = arg;
+    char buf[255];
+    int len;
+    opal_event_t *ev = arg;
 #ifdef WIN32
-	DWORD dwBytesRead;
+    DWORD dwBytesRead;
 #endif
 
-	/* Reschedule this event */
-	opal_event.add(ev, NULL);
+    /* Reschedule this event */
+    opal_event.add(ev, NULL);
 
-	fprintf(stderr, "fifo_read called with fd: %d, event: %d, arg: %p\n",
-		fd, event, arg);
+    fprintf(stderr, "fifo_read called with fd: %d, event: %d, arg: %p\n", fd, event, arg);
 #ifdef WIN32
-	len = ReadFile((HANDLE)fd, buf, sizeof(buf) - 1, &dwBytesRead, NULL);
+    len = ReadFile((HANDLE) fd, buf, sizeof(buf) - 1, &dwBytesRead, NULL);
 
-	/* Check for end of file. */
-	if(len && dwBytesRead == 0) {
-		fprintf(stderr, "End Of File");
-		opal_event.del(ev);
-		return;
-	}
+    /* Check for end of file. */
+    if (len && dwBytesRead == 0) {
+        fprintf(stderr, "End Of File");
+        opal_event.del(ev);
+        return;
+    }
 
-	buf[dwBytesRead + 1] = '\0';
+    buf[dwBytesRead + 1] = '\0';
 #else
-	len = read(fd, buf, sizeof(buf) - 1);
+    len = read(fd, buf, sizeof(buf) - 1);
 
-	if (len == -1) {
-		perror("read");
-		return;
-	} else if (len == 0) {
-		fprintf(stderr, "Connection closed\n");
-		return;
-	}
+    if (len == -1) {
+        perror("read");
+        return;
+    } else if (len == 0) {
+        fprintf(stderr, "Connection closed\n");
+        return;
+    }
 
-	buf[len] = '\0';
+    buf[len] = '\0';
 #endif
-	fprintf(stdout, "Read: %s\n", buf);
+    fprintf(stdout, "Read: %s\n", buf);
 }
 
-int
-main (int argc, char **argv)
+int main(int argc, char **argv)
 {
-	opal_event_t evfifo;
+    opal_event_t evfifo;
 #ifdef WIN32
-	HANDLE socket;
-	// Open a file.
-	socket = CreateFile("test.txt",     // open File
-			GENERIC_READ,                 // open for reading
-			0,                            // do not share
-			NULL,                         // no security
-			OPEN_EXISTING,                // existing file only
-			FILE_ATTRIBUTE_NORMAL,        // normal file
-			NULL);                        // no attr. template
-
-	if(socket == INVALID_HANDLE_VALUE)
-		return 1;
+    HANDLE socket;
+    // Open a file.
+    socket = CreateFile("test.txt",            // open File
+                        GENERIC_READ,          // open for reading
+                        0,                     // do not share
+                        NULL,                  // no security
+                        OPEN_EXISTING,         // existing file only
+                        FILE_ATTRIBUTE_NORMAL, // normal file
+                        NULL);                 // no attr. template
+
+    if (socket == INVALID_HANDLE_VALUE)
+        return 1;
 
 #else
-	struct stat st;
-	char *fifo = "event.fifo";
-	int socket;
-
-	if (lstat (fifo, &st) == 0) {
-		if ((st.st_mode & S_IFMT) == S_IFREG) {
-			errno = EEXIST;
-			perror("lstat");
-			exit (1);
-		}
-	}
-
-	unlink (fifo);
-	if (mkfifo (fifo, 0600) == -1) {
-		perror("mkfifo");
-		exit (1);
-	}
-
-	/* Linux pipes are broken, we need O_RDWR instead of O_RDONLY */
-#ifdef __linux
-	socket = open (fifo, O_RDWR | O_NONBLOCK, 0);
-#else
-	socket = open (fifo, O_RDONLY | O_NONBLOCK, 0);
+    struct stat st;
+    char *fifo = "event.fifo";
+    int socket;
+
+    if (lstat(fifo, &st) == 0) {
+        if ((st.st_mode & S_IFMT) == S_IFREG) {
+            errno = EEXIST;
+            perror("lstat");
+            exit(1);
+        }
+    }
+
+    unlink(fifo);
+    if (mkfifo(fifo, 0600) == -1) {
+        perror("mkfifo");
+        exit(1);
+    }
+
+    /* Linux pipes are broken, we need O_RDWR instead of O_RDONLY */
+#    ifdef __linux
+    socket = open(fifo, O_RDWR | O_NONBLOCK, 0);
+#    else
+    socket = open(fifo, O_RDONLY | O_NONBLOCK, 0);
+#    endif
+
+    if (socket == -1) {
+        perror("open");
+        exit(1);
+    }
+
+    fprintf(stderr, "Write data to %s\n", fifo);
 #endif
+    /* Initialize the event library */
+    opal_init();
 
-	if (socket == -1) {
-		perror("open");
-		exit (1);
-	}
-
-	fprintf(stderr, "Write data to %s\n", fifo);
-#endif
-	/* Initialize the event library */
-	opal_init();
-
-	/* Initialize one event */
+    /* Initialize one event */
 #ifdef WIN32
-	opal_event.set(opal_event_base, &evfifo, (int)socket, OPAL_EV_READ, fifo_read, &evfifo);
+    opal_event.set(opal_event_base, &evfifo, (int) socket, OPAL_EV_READ, fifo_read, &evfifo);
 #else
-	opal_event.set(opal_event_base, &evfifo, socket, OPAL_EV_READ, fifo_read, &evfifo);
+    opal_event.set(opal_event_base, &evfifo, socket, OPAL_EV_READ, fifo_read, &evfifo);
 #endif
 
-	/* Add it to the active events, without a timeout */
-	opal_event.add(&evfifo, NULL);
+    /* Add it to the active events, without a timeout */
+    opal_event.add(&evfifo, NULL);
 
-	opal_event.dispatch(opal_event_base);
+    opal_event.dispatch(opal_event_base);
 #ifdef WIN32
-	CloseHandle(socket);
+    CloseHandle(socket);
 #endif
-	return (0);
+    return (0);
 }
-
diff --git a/test/event/signal-test.c b/test/event/signal-test.c
index e16be7e9919..a3ced9a09ff 100644
--- a/test/event/signal-test.c
+++ b/test/event/signal-test.c
@@ -4,66 +4,63 @@
  */
 
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 #include <sys/stat.h>
 #ifndef WIN32
-#ifdef HAVE_SYS_QUEUE_H
-#include <sys/queue.h>
-#endif
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_SYS_TIME_H
-#include <sys/time.h>
-#endif
+#    ifdef HAVE_SYS_QUEUE_H
+#        include <sys/queue.h>
+#    endif
+#    ifdef HAVE_UNISTD_H
+#        include <unistd.h>
+#    endif
+#    ifdef HAVE_SYS_TIME_H
+#        include <sys/time.h>
+#    endif
 #else
-#include <windows.h>
+#    include <windows.h>
 #endif
-#include <signal.h>
+#include <errno.h>
 #include <fcntl.h>
-#include <stdlib.h>
+#include <signal.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
-#include <errno.h>
 
-#include "opal/util/event.h"
 #include "opal/runtime/opal.h"
+#include "opal/util/event.h"
 
 int called = 0;
 
-void
-signal_cb(int fd, short event, void *arg)
+void signal_cb(int fd, short event, void *arg)
 {
-	opal_event_t *signal = arg;
+    opal_event_t *signal = arg;
 
-	printf("%s: got signal %d\n", __func__, OPAL_EVENT_SIGNAL(signal));
+    printf("%s: got signal %d\n", __func__, OPAL_EVENT_SIGNAL(signal));
 
-	if (called >= 2)
-		opal_event.del(signal);
+    if (called >= 2)
+        opal_event.del(signal);
 
-	called++;
+    called++;
 }
 
-int
-main (int argc, char **argv)
+int main(int argc, char **argv)
 {
     opal_event_t signal_int, signal_term;
 
-	/* Initialize the event library */
-	opal_init();
+    /* Initialize the event library */
+    opal_init();
 
-	/* Initialize one event */
-	opal_event.set(opal_event_base, &signal_term, SIGUSR1, OPAL_EV_SIGNAL|OPAL_EV_PERSIST, signal_cb,
-	    &signal_term);
-	opal_event.set(opal_event_base, &signal_int, SIGUSR2, OPAL_EV_SIGNAL|OPAL_EV_PERSIST, signal_cb,
-	    &signal_int);
+    /* Initialize one event */
+    opal_event.set(opal_event_base, &signal_term, SIGUSR1, OPAL_EV_SIGNAL | OPAL_EV_PERSIST,
+                   signal_cb, &signal_term);
+    opal_event.set(opal_event_base, &signal_int, SIGUSR2, OPAL_EV_SIGNAL | OPAL_EV_PERSIST,
+                   signal_cb, &signal_int);
 
-	opal_event.add(&signal_int, NULL);
-	opal_event.add(&signal_term, NULL);
+    opal_event.add(&signal_int, NULL);
+    opal_event.add(&signal_term, NULL);
 
-	opal_event.dispatch(opal_event_base);
+    opal_event.dispatch(opal_event_base);
 
-	return (0);
+    return (0);
 }
-
diff --git a/test/event/time-test.c b/test/event/time-test.c
index 9f8086ca7a4..e12ef98b259 100644
--- a/test/event/time-test.c
+++ b/test/event/time-test.c
@@ -4,68 +4,64 @@
  */
 
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 #include <sys/stat.h>
 #ifndef WIN32
-#ifdef HAVE_SYS_QUEUE_H
-#include <sys/queue.h>
-#endif
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
+#    ifdef HAVE_SYS_QUEUE_H
+#        include <sys/queue.h>
+#    endif
+#    ifdef HAVE_UNISTD_H
+#        include <unistd.h>
+#    endif
 #else
-#include <time.h>
+#    include <time.h>
 #endif
 #ifdef HAVE_SYS_TIME_H
-#include <sys/time.h>
+#    include <sys/time.h>
 #endif
+#include <errno.h>
 #include <fcntl.h>
-#include <stdlib.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
-#include <errno.h>
 
 #include "opal/event/event.h"
 
 int lasttime;
 
-void
-timeout_cb(int fd, short event, void *arg)
+void timeout_cb(int fd, short event, void *arg)
 {
-	struct timeval tv;
-	opal_event_t *timeout = arg;
-	int newtime = time(NULL);
+    struct timeval tv;
+    opal_event_t *timeout = arg;
+    int newtime = time(NULL);
 
-	printf("%s: called at %d: %d\n", __func__, newtime,
-	    newtime - lasttime);
-	lasttime = newtime;
+    printf("%s: called at %d: %d\n", __func__, newtime, newtime - lasttime);
+    lasttime = newtime;
 
-	timerclear(&tv);
-	tv.tv_sec = 2;
-	opal_event_add(timeout, &tv);
+    timerclear(&tv);
+    tv.tv_sec = 2;
+    opal_event_add(timeout, &tv);
 }
 
-int
-main (int argc, char **argv)
+int main(int argc, char **argv)
 {
-	opal_event_t timeout;
-	struct timeval tv;
+    opal_event_t timeout;
+    struct timeval tv;
 
-	/* Initialize the event library */
-	opal_event_init();
+    /* Initialize the event library */
+    opal_event_init();
 
-	/* Initialize one event */
-	opal_evtimer_set(&timeout, timeout_cb, &timeout);
+    /* Initialize one event */
+    opal_evtimer_set(&timeout, timeout_cb, &timeout);
 
-	timerclear(&tv);
-	tv.tv_sec = 2;
-	opal_event_add(&timeout, &tv);
+    timerclear(&tv);
+    tv.tv_sec = 2;
+    opal_event_add(&timeout, &tv);
 
-	lasttime = time(NULL);
+    lasttime = time(NULL);
 
-	opal_event_dispatch();
+    opal_event_dispatch();
 
-	return (0);
+    return (0);
 }
-
diff --git a/test/memchecker/Makefile.am b/test/memchecker/Makefile.am
index 39334d7a379..b2cfd5575ad 100644
--- a/test/memchecker/Makefile.am
+++ b/test/memchecker/Makefile.am
@@ -31,34 +31,34 @@ TESTS = \
 access_buffer_test_SOURCES = access_buffer_test.c
 access_buffer_test_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS)
 access_buffer_test_LDADD = \
-        $(top_builddir)/ompi/lib@OPAL_LIB_PREFIX@mpi.la
+        $(top_builddir)/ompi/lib@OPAL_LIB_NAME@mpi.la
 access_buffer_test_DEPENDENCIES = $(access_buffer_test_LDADD)
 
 irecv_init_check_SOURCES = irecv_init_check.c
 irecv_init_check_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS)
 irecv_init_check_LDADD = \
-        $(top_builddir)/ompi/lib@OPAL_LIB_PREFIX@mpi.la
+        $(top_builddir)/ompi/lib@OPAL_LIB_NAME@mpi.la
 irecv_init_check_DEPENDENCIES = $(irecv_init_check_LDADD)
 
 
 irecv_uninit_check_SOURCES = irecv_uninit_check.c
 irecv_uninit_check_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS)
 irecv_uninit_check_LDADD = \
-        $(top_builddir)/ompi/lib@OPAL_LIB_PREFIX@mpi.la
+        $(top_builddir)/ompi/lib@OPAL_LIB_NAME@mpi.la
 irecv_uninit_check_DEPENDENCIES = $(irecv_uninit_check_LDADD)
 
 non_blocking_send_test_SOURCES = non_blocking_send_test.c
 non_blocking_send_test_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS)
 non_blocking_send_test_LDADD = \
-        $(top_builddir)/ompi/lib@OPAL_LIB_PREFIX@mpi.la
+        $(top_builddir)/ompi/lib@OPAL_LIB_NAME@mpi.la
 non_blocking_send_test_DEPENDENCIES = $(non_blocking_send_test_LDADD)
 
 
 non_blocking_recv_test_SOURCES = non_blocking_recv_test.c
 non_blocking_recv_test_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS)
 non_blocking_recv_test_LDADD = \
-        $(top_builddir)/ompi/lib@OPAL_LIB_PREFIX@mpi.la
+        $(top_builddir)/ompi/lib@OPAL_LIB_NAME@mpi.la
 non_blocking_recv_test_DEPENDENCIES = $(non_blocking_recv_test_LDADD)
 
-distclean:
+distclean-local:
 	rm -rf *.dSYM .deps *.log *.o *.trs $(check_PROGRAMS) Makefile
diff --git a/test/memchecker/irecv_init_check.c b/test/memchecker/irecv_init_check.c
index 0fa7ecb2a4e..7bc0ee8cd9b 100644
--- a/test/memchecker/irecv_init_check.c
+++ b/test/memchecker/irecv_init_check.c
@@ -8,26 +8,26 @@
  * $HEADER$
  */
 
-#include <stdio.h>
 #include "mpi.h"
+#include <stdio.h>
 
-int main( int argc, char *argv[] )
+int main(int argc, char *argv[])
 {
     int rank, size;
-    int send_value[10],recv_value[10];
+    int send_value[10], recv_value[10];
     int i;
     MPI_Status status;
     MPI_Request send_request;
     MPI_Request recv_request;
 
-    MPI_Init( &argc, &argv );
+    MPI_Init(&argc, &argv);
 
-    MPI_Comm_rank( MPI_COMM_WORLD, &rank );
-    MPI_Comm_size( MPI_COMM_WORLD, &size );
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    MPI_Comm_size(MPI_COMM_WORLD, &size);
 
-    for(i=0; i < 10; i++) {
+    for (i = 0; i < 10; i++) {
         send_value[i] = i;
-        printf("\n%d:%d",i,send_value[i]);
+        printf("\n%d:%d", i, send_value[i]);
     }
 
     /*
@@ -37,8 +37,8 @@ int main( int argc, char *argv[] )
     recv_value[9] = 100;
 
     if (size != 2) {
-        fprintf (stderr, "Error: Need 2 processes\n");
-        MPI_Finalize ();
+        fprintf(stderr, "Error: Need 2 processes\n");
+        MPI_Finalize();
     }
 
     /*
@@ -46,20 +46,18 @@ int main( int argc, char *argv[] )
      * Perfectly valid, but the receiver (in strict mode?) should not
      * depend on the data.
      */
-    MPI_Isend (&send_value, 9, MPI_INT,
-               (rank + 1) % size, 4711, MPI_COMM_WORLD, &send_request);
-    MPI_Irecv (&recv_value, 10, MPI_INT,
-               (rank + size - 1) % size, 4711, MPI_COMM_WORLD, &recv_request);
+    MPI_Isend(&send_value, 9, MPI_INT, (rank + 1) % size, 4711, MPI_COMM_WORLD, &send_request);
+    MPI_Irecv(&recv_value, 10, MPI_INT, (rank + size - 1) % size, 4711, MPI_COMM_WORLD,
+              &recv_request);
 
-    MPI_Wait (&send_request, &status);
-    MPI_Wait (&recv_request, &status);
+    MPI_Wait(&send_request, &status);
+    MPI_Wait(&recv_request, &status);
 
     /*
      * In strict mode, this should return an error.
      */
     printf("\nError in strict mode: buf[9]:%d", recv_value[9]);
 
-    MPI_Finalize ();
+    MPI_Finalize();
     return 0;
 }
-
diff --git a/test/memchecker/irecv_uninit_check.c b/test/memchecker/irecv_uninit_check.c
index 8dc411c7663..451fa083c56 100644
--- a/test/memchecker/irecv_uninit_check.c
+++ b/test/memchecker/irecv_uninit_check.c
@@ -8,27 +8,26 @@
  * $HEADER$
  */
 
-#include <stdio.h>
 #include "mpi.h"
+#include <stdio.h>
 
-int main( int argc, char *argv[] )
+int main(int argc, char *argv[])
 {
     int rank, size;
-    int send_value[10],recv_value[10];
+    int send_value[10], recv_value[10];
     int i;
     MPI_Status status;
     MPI_Request send_request;
     MPI_Request recv_request;
 
-    MPI_Init( &argc, &argv );
+    MPI_Init(&argc, &argv);
 
-    MPI_Comm_rank( MPI_COMM_WORLD, &rank );
-    MPI_Comm_size( MPI_COMM_WORLD, &size );
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    MPI_Comm_size(MPI_COMM_WORLD, &size);
 
-    for(i=0;i<10;i++)
-    {
+    for (i = 0; i < 10; i++) {
         send_value[i] = i;
-        printf("\n%d:%d",i,send_value[i]);
+        printf("\n%d:%d", i, send_value[i]);
     }
 
     /*
@@ -37,8 +36,8 @@ int main( int argc, char *argv[] )
      */
 
     if (size != 2) {
-        fprintf (stderr, "Error: Need 2 processes\n");
-        MPI_Finalize ();
+        fprintf(stderr, "Error: Need 2 processes\n");
+        MPI_Finalize();
     }
 
     /*
@@ -46,19 +45,18 @@ int main( int argc, char *argv[] )
      * Perfectly valid, but the receiver (in strict mode?) should not
      * depend on the data.
      */
-    MPI_Isend (&send_value, 9, MPI_INT,
-			   (rank + 1) % size, 4711, MPI_COMM_WORLD, &send_request);
-    MPI_Irecv (&recv_value, 10, MPI_INT,
-			   (rank + size - 1) % size, 4711, MPI_COMM_WORLD, &recv_request);
+    MPI_Isend(&send_value, 9, MPI_INT, (rank + 1) % size, 4711, MPI_COMM_WORLD, &send_request);
+    MPI_Irecv(&recv_value, 10, MPI_INT, (rank + size - 1) % size, 4711, MPI_COMM_WORLD,
+              &recv_request);
 
-    MPI_Wait (&send_request, &status);
-    MPI_Wait (&recv_request, &status);
+    MPI_Wait(&send_request, &status);
+    MPI_Wait(&recv_request, &status);
 
     /*
      * This should return an error in any mode.
      */
-    printf("\nError: buf[9]:%d",recv_value[9]);
+    printf("\nError: buf[9]:%d", recv_value[9]);
 
-    MPI_Finalize ();
+    MPI_Finalize();
     return 0;
 }
diff --git a/test/memchecker/non_blocking_recv_test.c b/test/memchecker/non_blocking_recv_test.c
index 793959f8a84..505c0acd33c 100644
--- a/test/memchecker/non_blocking_recv_test.c
+++ b/test/memchecker/non_blocking_recv_test.c
@@ -8,52 +8,51 @@
  * $HEADER$
  */
 
-#include <stdio.h>
 #include "mpi.h"
+#include <stdio.h>
 
-int main( int argc, char *argv[] )
+int main(int argc, char *argv[])
 {
     int rank, size;
-    int send_value,recv_value;
+    int send_value, recv_value;
 
     MPI_Status status;
     MPI_Request send_request;
     MPI_Request recv_request;
 
-    MPI_Init( &argc, &argv );
+    MPI_Init(&argc, &argv);
 
-    MPI_Comm_rank( MPI_COMM_WORLD, &rank );
-    MPI_Comm_size( MPI_COMM_WORLD, &size );
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    MPI_Comm_size(MPI_COMM_WORLD, &size);
 
     send_value = 10;
     recv_value = 0;
 
     if (size != 2) {
-        fprintf (stderr, "Error: Need 2 processes\n");
-        MPI_Finalize ();
+        fprintf(stderr, "Error: Need 2 processes\n");
+        MPI_Finalize();
     }
 
-    MPI_Isend (&send_value, 1, MPI_INT,
-               (rank + 1) % size, 4711, MPI_COMM_WORLD, &send_request);
-    MPI_Irecv (&recv_value, 1, MPI_INT,
-               (rank + size - 1) % size, 4711, MPI_COMM_WORLD, &recv_request);
+    MPI_Isend(&send_value, 1, MPI_INT, (rank + 1) % size, 4711, MPI_COMM_WORLD, &send_request);
+    MPI_Irecv(&recv_value, 1, MPI_INT, (rank + size - 1) % size, 4711, MPI_COMM_WORLD,
+              &recv_request);
 
     /*
      * Reading or writing of the receive buffer before
      * non blocking send finished will cause an error.
      */
-    printf("\nsent: %d\n",recv_value);
+    printf("\nsent: %d\n", recv_value);
     recv_value = 12;
 
-    MPI_Wait (&send_request, &status);
-    MPI_Wait (&recv_request, &status);
+    MPI_Wait(&send_request, &status);
+    MPI_Wait(&recv_request, &status);
     /*
      * Using of receive buffer after non blocking send
      * is finished will be ok.
      */
-    printf("\nsent: %d\n",recv_value);
+    printf("\nsent: %d\n", recv_value);
     recv_value = 16;
 
-    MPI_Finalize ();
+    MPI_Finalize();
     return 0;
 }
diff --git a/test/memchecker/non_blocking_send_test.c b/test/memchecker/non_blocking_send_test.c
index 0a6b25d20ad..09eb4b299c6 100644
--- a/test/memchecker/non_blocking_send_test.c
+++ b/test/memchecker/non_blocking_send_test.c
@@ -8,54 +8,52 @@
  * $HEADER$
  */
 
-#include <stdio.h>
 #include "mpi.h"
+#include <stdio.h>
 
-int main( int argc, char *argv[] )
+int main(int argc, char *argv[])
 {
     int rank, size;
-    int send_value,recv_value;
+    int send_value, recv_value;
 
     MPI_Status status;
     MPI_Request send_request;
     MPI_Request recv_request;
 
-    MPI_Init( &argc, &argv );
+    MPI_Init(&argc, &argv);
 
-    MPI_Comm_rank( MPI_COMM_WORLD, &rank );
-    MPI_Comm_size( MPI_COMM_WORLD, &size );
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    MPI_Comm_size(MPI_COMM_WORLD, &size);
 
     send_value = 10;
     recv_value = 0;
 
     if (size != 2) {
-        fprintf (stderr, "Error: Need 2 processes\n");
-        MPI_Finalize ();
+        fprintf(stderr, "Error: Need 2 processes\n");
+        MPI_Finalize();
     }
 
-    MPI_Isend (&send_value, 1, MPI_INT,
-               (rank + 1) % size, 4711, MPI_COMM_WORLD, &send_request);
-    MPI_Irecv (&recv_value, 1, MPI_INT,
-               (rank + size - 1) % size, 4711, MPI_COMM_WORLD, &recv_request);
+    MPI_Isend(&send_value, 1, MPI_INT, (rank + 1) % size, 4711, MPI_COMM_WORLD, &send_request);
+    MPI_Irecv(&recv_value, 1, MPI_INT, (rank + size - 1) % size, 4711, MPI_COMM_WORLD,
+              &recv_request);
 
     /*
      * Reading or writing of the send buffer before
      * non blocking send is finished will cause an error.
      */
-    printf("\nsent: %d\n",send_value);
+    printf("\nsent: %d\n", send_value);
     send_value = 12;
 
-    MPI_Wait (&send_request, &status);
-    MPI_Wait (&recv_request, &status);
+    MPI_Wait(&send_request, &status);
+    MPI_Wait(&recv_request, &status);
 
     /*
      * Using of send buffer after non blocking send
      * is finished will be ok.
      */
-    printf("\nsent: %d\n",send_value);
+    printf("\nsent: %d\n", send_value);
     send_value = 16;
 
-    MPI_Finalize ();
+    MPI_Finalize();
     return 0;
 }
-
diff --git a/test/monitoring/Makefile.am b/test/monitoring/Makefile.am
index b95d59ed4dd..ed7b257d8e7 100644
--- a/test/monitoring/Makefile.am
+++ b/test/monitoring/Makefile.am
@@ -22,28 +22,28 @@ if PROJECT_OMPI
     monitoring_test_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS)
     monitoring_test_LDADD = \
         $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
     test_pvar_access_SOURCES = test_pvar_access.c
     test_pvar_access_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS)
     test_pvar_access_LDADD = \
 	$(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
-	$(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+	$(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
     test_overhead_SOURCES = test_overhead.c
     test_overhead_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS)
     test_overhead_LDADD = \
 	$(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
-	$(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+	$(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
     check_monitoring_SOURCES = check_monitoring.c
     check_monitoring_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS)
     check_monitoring_LDADD = \
 	$(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
-	$(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+	$(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
     example_reduce_count_SOURCES = example_reduce_count.c
     example_reduce_count_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS)
     example_reduce_count_LDADD = \
 	$(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
-	$(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+	$(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 endif # PROJECT_OMPI
 
-distclean:
+distclean-local:
 	rm -rf *.dSYM .deps .libs *.la *.lo monitoring_test test_pvar_access test_overhead check_monitoring example_reduce_count prof *.log *.o *.trs Makefile
diff --git a/test/monitoring/check_monitoring.c b/test/monitoring/check_monitoring.c
index ae7e73eb8d3..395501cdc2a 100644
--- a/test/monitoring/check_monitoring.c
+++ b/test/monitoring/check_monitoring.c
@@ -3,6 +3,7 @@
  * Copyright (c) 2017      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2024      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -15,122 +16,132 @@
 
   To be run as:
 
-  mpirun -np 4 --mca pml_monitoring_enable 2 ./check_monitoring
+  mpirun -n 4 --mca pml_monitoring_enable 2 ./check_monitoring
 */
 
 #include <mpi.h>
-#include <stdlib.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
 
-#define PVAR_GENERATE_VARIABLES(pvar_prefix, pvar_name, pvar_class)     \
-    /* Variables */                                                     \
-    static MPI_T_pvar_handle pvar_prefix ## _handle;                    \
-    static const char pvar_prefix ## _pvar_name[] = pvar_name;          \
-    static int pvar_prefix ## _pvar_idx;                                \
-    /* Functions */                                                     \
-    static inline int pvar_prefix ## _start(MPI_T_pvar_session session) \
-    {                                                                   \
-        int MPIT_result;                                                \
-        MPIT_result = MPI_T_pvar_start(session, pvar_prefix ## _handle); \
-        if( MPI_SUCCESS != MPIT_result ) {                              \
-            fprintf(stderr, "Failed to start handle on \"%s\" pvar, check that you have " \
-                    "enabled the monitoring component.\n", pvar_prefix ## _pvar_name); \
-            MPI_Abort(MPI_COMM_WORLD, MPIT_result);                     \
-        }                                                               \
-        return MPIT_result;                                             \
-    }                                                                   \
-    static inline int pvar_prefix ## _init(MPI_T_pvar_session session)  \
-    {                                                                   \
-        int MPIT_result;                                                \
-        MPI_Comm comm = MPI_COMM_WORLD;                                 \
-        /* Get index */                                                 \
-        MPIT_result = MPI_T_pvar_get_index(pvar_prefix ## _pvar_name,   \
-                                           pvar_class,                  \
-                                           &(pvar_prefix ## _pvar_idx)); \
-        if( MPI_SUCCESS != MPIT_result ) {                              \
-            fprintf(stderr, "Cannot find monitoring MPI_Tool \"%s\" pvar, check that you have " \
-                    "enabled the monitoring component.\n", pvar_prefix ## _pvar_name); \
-            MPI_Abort(MPI_COMM_WORLD, MPIT_result);                     \
-            return MPIT_result;                                         \
-        }                                                               \
-        /* Allocate handle */                                           \
-        /* Allocating a new PVAR in a session will reset the counters */ \
-        int count;                                                      \
-        MPIT_result = MPI_T_pvar_handle_alloc(session, pvar_prefix ## _pvar_idx, \
-                                              &comm, &(pvar_prefix ## _handle), \
-                                              &count);                  \
-        if( MPI_SUCCESS != MPIT_result ) {                              \
-            fprintf(stderr, "Failed to allocate handle on \"%s\" pvar, check that you have " \
-                    "enabled the monitoring component.\n", pvar_prefix ## _pvar_name); \
-            MPI_Abort(MPI_COMM_WORLD, MPIT_result);                     \
-            return MPIT_result;                                         \
-        }                                                               \
-        /* Start PVAR */                                                \
-        return pvar_prefix ## _start(session);                          \
-    }                                                                   \
-    static inline int pvar_prefix ## _stop(MPI_T_pvar_session session)  \
-    {                                                                   \
-        int MPIT_result;                                                \
-        MPIT_result = MPI_T_pvar_stop(session, pvar_prefix ## _handle); \
-        if( MPI_SUCCESS != MPIT_result ) {                              \
-            fprintf(stderr, "Failed to stop handle on \"%s\" pvar, check that you have " \
-                    "enabled the monitoring component.\n", pvar_prefix ## _pvar_name); \
-            MPI_Abort(MPI_COMM_WORLD, MPIT_result);                     \
-        }                                                               \
-        return MPIT_result;                                             \
-    }                                                                   \
-    static inline int pvar_prefix ## _finalize(MPI_T_pvar_session session) \
-    {                                                                   \
-        int MPIT_result;                                                \
-        /* Stop PVAR */                                                 \
-        MPIT_result = pvar_prefix ## _stop(session);                    \
-        /* Free handle */						\
-        MPIT_result = MPI_T_pvar_handle_free(session, &(pvar_prefix ## _handle)); \
-        if( MPI_SUCCESS != MPIT_result ) {				\
-            fprintf(stderr, "Failed to allocate handle on \"%s\" pvar, check that you have " \
-                    "enabled the monitoring component.\n", pvar_prefix ## _pvar_name); \
-            MPI_Abort(MPI_COMM_WORLD, MPIT_result);			\
-            return MPIT_result;                                         \
-        }								\
-        return MPIT_result;						\
-    }                                                                   \
-    static inline int pvar_prefix ## _read(MPI_T_pvar_session session, void*values) \
-    {                                                                   \
-        int MPIT_result;                                                \
-        /* Stop pvar */                                                 \
-        MPIT_result = pvar_prefix ## _stop(session);                    \
-        /* Read values */						\
-        MPIT_result = MPI_T_pvar_read(session, pvar_prefix ## _handle, values); \
-        if( MPI_SUCCESS != MPIT_result ) {				\
-            fprintf(stderr, "Failed to read handle on \"%s\" pvar, check that you have " \
-                    "enabled the monitoring component.\n", pvar_prefix ## _pvar_name); \
-            MPI_Abort(MPI_COMM_WORLD, MPIT_result);			\
-        }								\
-        /* Start and return */                                          \
-        return pvar_prefix ## _start(session);                          \
+#define PVAR_GENERATE_VARIABLES(pvar_prefix, pvar_name, pvar_class)                     \
+    /* Variables */                                                                     \
+    static MPI_T_pvar_handle pvar_prefix##_handle;                                      \
+    static const char pvar_prefix##_pvar_name[] = pvar_name;                            \
+    static int pvar_prefix##_pvar_idx;                                                  \
+    /* Functions */                                                                     \
+    static inline int pvar_prefix##_start(MPI_T_pvar_session session)                   \
+    {                                                                                   \
+        int MPIT_result;                                                                \
+        MPIT_result = MPI_T_pvar_start(session, pvar_prefix##_handle);                  \
+        if (MPI_SUCCESS != MPIT_result) {                                               \
+            fprintf(stderr,                                                             \
+                    "Failed to start handle on \"%s\" pvar, check that you have "       \
+                    "enabled the monitoring component.\n",                              \
+                    pvar_prefix##_pvar_name);                                           \
+            MPI_Abort(MPI_COMM_WORLD, MPIT_result);                                     \
+        }                                                                               \
+        return MPIT_result;                                                             \
+    }                                                                                   \
+    static inline int pvar_prefix##_init(MPI_T_pvar_session session)                    \
+    {                                                                                   \
+        int MPIT_result;                                                                \
+        MPI_Comm comm = MPI_COMM_WORLD;                                                 \
+        /* Get index */                                                                 \
+        MPIT_result = MPI_T_pvar_get_index(pvar_prefix##_pvar_name, pvar_class,         \
+                                           &(pvar_prefix##_pvar_idx));                  \
+        if (MPI_SUCCESS != MPIT_result) {                                               \
+            fprintf(stderr,                                                             \
+                    "Cannot find monitoring MPI_Tool \"%s\" pvar, check that you have " \
+                    "enabled the monitoring component.\n",                              \
+                    pvar_prefix##_pvar_name);                                           \
+            MPI_Abort(MPI_COMM_WORLD, MPIT_result);                                     \
+            return MPIT_result;                                                         \
+        }                                                                               \
+        /* Allocate handle */                                                           \
+        /* Allocating a new PVAR in a session will reset the counters */                \
+        int count;                                                                      \
+        MPIT_result = MPI_T_pvar_handle_alloc(session, pvar_prefix##_pvar_idx, &comm,   \
+                                              &(pvar_prefix##_handle), &count);         \
+        if (MPI_SUCCESS != MPIT_result) {                                               \
+            fprintf(stderr,                                                             \
+                    "Failed to allocate handle on \"%s\" pvar, check that you have "    \
+                    "enabled the monitoring component.\n",                              \
+                    pvar_prefix##_pvar_name);                                           \
+            MPI_Abort(MPI_COMM_WORLD, MPIT_result);                                     \
+            return MPIT_result;                                                         \
+        }                                                                               \
+        /* Start PVAR */                                                                \
+        return pvar_prefix##_start(session);                                            \
+    }                                                                                   \
+    static inline int pvar_prefix##_stop(MPI_T_pvar_session session)                    \
+    {                                                                                   \
+        int MPIT_result;                                                                \
+        MPIT_result = MPI_T_pvar_stop(session, pvar_prefix##_handle);                   \
+        if (MPI_SUCCESS != MPIT_result) {                                               \
+            fprintf(stderr,                                                             \
+                    "Failed to stop handle on \"%s\" pvar, check that you have "        \
+                    "enabled the monitoring component.\n",                              \
+                    pvar_prefix##_pvar_name);                                           \
+            MPI_Abort(MPI_COMM_WORLD, MPIT_result);                                     \
+        }                                                                               \
+        return MPIT_result;                                                             \
+    }                                                                                   \
+    static inline int pvar_prefix##_finalize(MPI_T_pvar_session session)                \
+    {                                                                                   \
+        int MPIT_result;                                                                \
+        /* Stop PVAR */                                                                 \
+        MPIT_result = pvar_prefix##_stop(session);                                      \
+        /* Free handle */                                                               \
+        MPIT_result = MPI_T_pvar_handle_free(session, &(pvar_prefix##_handle));         \
+        if (MPI_SUCCESS != MPIT_result) {                                               \
+            fprintf(stderr,                                                             \
+                    "Failed to allocate handle on \"%s\" pvar, check that you have "    \
+                    "enabled the monitoring component.\n",                              \
+                    pvar_prefix##_pvar_name);                                           \
+            MPI_Abort(MPI_COMM_WORLD, MPIT_result);                                     \
+            return MPIT_result;                                                         \
+        }                                                                               \
+        return MPIT_result;                                                             \
+    }                                                                                   \
+    static inline int pvar_prefix##_read(MPI_T_pvar_session session, void *values)      \
+    {                                                                                   \
+        int MPIT_result;                                                                \
+        /* Stop pvar */                                                                 \
+        MPIT_result = pvar_prefix##_stop(session);                                      \
+        /* Read values */                                                               \
+        MPIT_result = MPI_T_pvar_read(session, pvar_prefix##_handle, values);           \
+        if (MPI_SUCCESS != MPIT_result) {                                               \
+            fprintf(stderr,                                                             \
+                    "Failed to read handle on \"%s\" pvar, check that you have "        \
+                    "enabled the monitoring component.\n",                              \
+                    pvar_prefix##_pvar_name);                                           \
+            MPI_Abort(MPI_COMM_WORLD, MPIT_result);                                     \
+        }                                                                               \
+        /* Start and return */                                                          \
+        return pvar_prefix##_start(session);                                            \
     }
 
-#define GENERATE_CS(prefix, pvar_name_prefix, pvar_class_c, pvar_class_s) \
-    PVAR_GENERATE_VARIABLES(prefix ## _count, pvar_name_prefix "_count", pvar_class_c) \
-    PVAR_GENERATE_VARIABLES(prefix ## _size, pvar_name_prefix "_size", pvar_class_s) \
-    static inline int pvar_ ## prefix ## _init(MPI_T_pvar_session session) \
-    {                                                                   \
-        prefix ## _count_init(session);                                 \
-        return prefix ## _size_init(session);                           \
-    }                                                                   \
-    static inline int pvar_ ## prefix ## _finalize(MPI_T_pvar_session session) \
-    {                                                                   \
-        prefix ## _count_finalize(session);                             \
-        return prefix ## _size_finalize(session);                       \
-    }                                                                   \
-    static inline void pvar_ ## prefix ## _read(MPI_T_pvar_session session, \
-                                                size_t*cvalues, size_t*svalues) \
-    {                                                                   \
-        /* Read count values */                                         \
-        prefix ## _count_read(session, cvalues);                        \
-        /* Read size values */                                          \
-        prefix ## _size_read(session, svalues);                         \
+#define GENERATE_CS(prefix, pvar_name_prefix, pvar_class_c, pvar_class_s)                \
+    PVAR_GENERATE_VARIABLES(prefix##_count, pvar_name_prefix "_count", pvar_class_c)     \
+    PVAR_GENERATE_VARIABLES(prefix##_size, pvar_name_prefix "_size", pvar_class_s)       \
+    static inline int pvar_##prefix##_init(MPI_T_pvar_session session)                   \
+    {                                                                                    \
+        prefix##_count_init(session);                                                    \
+        return prefix##_size_init(session);                                              \
+    }                                                                                    \
+    static inline int pvar_##prefix##_finalize(MPI_T_pvar_session session)               \
+    {                                                                                    \
+        prefix##_count_finalize(session);                                                \
+        return prefix##_size_finalize(session);                                          \
+    }                                                                                    \
+    static inline void pvar_##prefix##_read(MPI_T_pvar_session session, size_t *cvalues, \
+                                            size_t *svalues)                             \
+    {                                                                                    \
+        /* Read count values */                                                          \
+        prefix##_count_read(session, cvalues);                                           \
+        /* Read size values */                                                           \
+        prefix##_size_read(session, svalues);                                            \
     }
 
 GENERATE_CS(pml, "pml_monitoring_messages", MPI_T_PVAR_CLASS_SIZE, MPI_T_PVAR_CLASS_SIZE)
@@ -143,12 +154,12 @@ GENERATE_CS(a2a, "coll_monitoring_a2a", MPI_T_PVAR_CLASS_COUNTER, MPI_T_PVAR_CLA
 
 static size_t *old_cvalues, *old_svalues;
 
-static inline void pvar_all_init(MPI_T_pvar_session*session, int world_size)
+static inline void pvar_all_init(MPI_T_pvar_session *session, int world_size)
 {
     int MPIT_result, provided;
     MPIT_result = MPI_T_init_thread(MPI_THREAD_SINGLE, &provided);
     if (MPIT_result != MPI_SUCCESS) {
-        fprintf(stderr, "Failed to initialiaze MPI_Tools sub-system.\n");
+        fprintf(stderr, "Failed to initialize MPI_Tools sub-system.\n");
         MPI_Abort(MPI_COMM_WORLD, MPIT_result);
     }
     MPIT_result = MPI_T_pvar_session_create(session);
@@ -167,7 +178,7 @@ static inline void pvar_all_init(MPI_T_pvar_session*session, int world_size)
     pvar_a2a_init(*session);
 }
 
-static inline void pvar_all_finalize(MPI_T_pvar_session*session)
+static inline void pvar_all_finalize(MPI_T_pvar_session *session)
 {
     int MPIT_result;
     pvar_pml_finalize(*session);
@@ -183,7 +194,7 @@ static inline void pvar_all_finalize(MPI_T_pvar_session*session)
         printf("Failed to close a session for PVARs.\n");
         MPI_Abort(MPI_COMM_WORLD, MPIT_result);
     }
-    (void)MPI_T_finalize();
+    (void) MPI_T_finalize();
 }
 
 static inline int pvar_pml_check(MPI_T_pvar_session session, int world_size, int world_rank)
@@ -194,29 +205,30 @@ static inline int pvar_pml_check(MPI_T_pvar_session session, int world_size, int
     svalues = cvalues + world_size;
     /* Get values */
     pvar_pml_read(session, cvalues, svalues);
-    for( i = 0; i < world_size && MPI_SUCCESS == ret; ++i ) {
+    for (i = 0; i < world_size && MPI_SUCCESS == ret; ++i) {
         /* Check count values */
-        if( i == world_rank && (cvalues[i] - old_cvalues[i]) != (size_t) 0 ) {
+        if (i == world_rank && (cvalues[i] - old_cvalues[i]) != (size_t) 0) {
             fprintf(stderr, "Error in %s: count_values[%d]=%zu, and should be equal to %zu.\n",
                     __func__, i, cvalues[i] - old_cvalues[i], (size_t) 0);
             ret = -1;
-        } else if ( i != world_rank && (cvalues[i] - old_cvalues[i]) < (size_t) world_size ) {
-            fprintf(stderr, "Error in %s: count_values[%d]=%zu, and should be >= %zu.\n",
-                    __func__, i, cvalues[i] - old_cvalues[i], (size_t) world_size);
+        } else if (i != world_rank && (cvalues[i] - old_cvalues[i]) < (size_t) world_size) {
+            fprintf(stderr, "Error in %s: count_values[%d]=%zu, and should be >= %zu.\n", __func__,
+                    i, cvalues[i] - old_cvalues[i], (size_t) world_size);
             ret = -1;
         }
         /* Check size values */
-        if( i == world_rank && (svalues[i] - old_svalues[i]) != (size_t) 0 ) {
+        if (i == world_rank && (svalues[i] - old_svalues[i]) != (size_t) 0) {
             fprintf(stderr, "Error in %s: size_values[%d]=%zu, and should be equal to %zu.\n",
                     __func__, i, svalues[i] - old_svalues[i], (size_t) 0);
             ret = -1;
-        } else if ( i != world_rank && (svalues[i] - old_svalues[i]) < (size_t) (world_size * 13 * sizeof(char)) ) {
-            fprintf(stderr, "Error in %s: size_values[%d]=%zu, and should be >= %zu.\n",
-                    __func__, i, svalues[i] - old_svalues[i], (size_t) (world_size * 13 * sizeof(char)));
+        } else if (i != world_rank
+                   && (svalues[i] - old_svalues[i]) < (size_t)(world_size * 13 * sizeof(char))) {
+            fprintf(stderr, "Error in %s: size_values[%d]=%zu, and should be >= %zu.\n", __func__,
+                    i, svalues[i] - old_svalues[i], (size_t)(world_size * 13 * sizeof(char)));
             ret = -1;
         }
     }
-    if( MPI_SUCCESS == ret ) {
+    if (MPI_SUCCESS == ret) {
         fprintf(stdout, "Check PML...[ OK ]\n");
     } else {
         fprintf(stdout, "Check PML...[FAIL]\n");
@@ -238,37 +250,37 @@ static inline int pvar_osc_check(MPI_T_pvar_session session, int world_size, int
     memset(cvalues, 0, 2 * world_size * sizeof(size_t));
     /* Check OSC sent values */
     pvar_osc_s_read(session, cvalues, svalues);
-    for( i = 0; i < world_size && MPI_SUCCESS == ret; ++i ) {
+    for (i = 0; i < world_size && MPI_SUCCESS == ret; ++i) {
         /* Check count values */
-        if( cvalues[i] < (size_t) world_size ) {
-            fprintf(stderr, "Error in %s: count_values[%d]=%zu, and should be >= %zu.\n",
-                    __func__, i, cvalues[i], (size_t) world_size);
+        if (cvalues[i] < (size_t) world_size) {
+            fprintf(stderr, "Error in %s: count_values[%d]=%zu, and should be >= %zu.\n", __func__,
+                    i, cvalues[i], (size_t) world_size);
             ret = -1;
         }
         /* Check size values */
-        if( svalues[i] < (size_t) (world_size * 13 * sizeof(char)) ) {
-            fprintf(stderr, "Error in %s: size_values[%d]=%zu, and should be >= %zu.\n",
-                    __func__, i, svalues[i], (size_t) (world_size * 13 * sizeof(char)));
+        if (svalues[i] < (size_t)(world_size * 13 * sizeof(char))) {
+            fprintf(stderr, "Error in %s: size_values[%d]=%zu, and should be >= %zu.\n", __func__,
+                    i, svalues[i], (size_t)(world_size * 13 * sizeof(char)));
             ret = -1;
         }
     }
     /* Check OSC received values */
     pvar_osc_r_read(session, cvalues, svalues);
-    for( i = 0; i < world_size && MPI_SUCCESS == ret; ++i ) {
+    for (i = 0; i < world_size && MPI_SUCCESS == ret; ++i) {
         /* Check count values */
-        if( cvalues[i] < (size_t) world_size ) {
-            fprintf(stderr, "Error in %s: count_values[%d]=%zu, and should be >= %zu.\n",
-                    __func__, i, cvalues[i], (size_t) world_size);
+        if (cvalues[i] < (size_t) world_size) {
+            fprintf(stderr, "Error in %s: count_values[%d]=%zu, and should be >= %zu.\n", __func__,
+                    i, cvalues[i], (size_t) world_size);
             ret = -1;
         }
         /* Check size values */
-        if( svalues[i] < (size_t) (world_size * 13 * sizeof(char)) ) {
-            fprintf(stderr, "Error in %s: size_values[%d]=%zu, and should be >= %zu.\n",
-                    __func__, i, svalues[i], (size_t) (world_size * 13 * sizeof(char)));
+        if (svalues[i] < (size_t)(world_size * 13 * sizeof(char))) {
+            fprintf(stderr, "Error in %s: size_values[%d]=%zu, and should be >= %zu.\n", __func__,
+                    i, svalues[i], (size_t)(world_size * 13 * sizeof(char)));
             ret = -1;
         }
     }
-    if( MPI_SUCCESS == ret ) {
+    if (MPI_SUCCESS == ret) {
         fprintf(stdout, "Check OSC...[ OK ]\n");
     } else {
         fprintf(stdout, "Check OSC...[FAIL]\n");
@@ -280,7 +292,8 @@ static inline int pvar_osc_check(MPI_T_pvar_session session, int world_size, int
     return ret;
 }
 
-static inline int pvar_coll_check(MPI_T_pvar_session session, int world_size, int world_rank) {
+static inline int pvar_coll_check(MPI_T_pvar_session session, int world_size, int world_rank)
+{
     int i, ret = MPI_SUCCESS;
     size_t count, size;
     size_t *cvalues, *svalues;
@@ -288,67 +301,69 @@ static inline int pvar_coll_check(MPI_T_pvar_session session, int world_size, in
     svalues = cvalues + world_size;
     /* Get COLL values */
     pvar_coll_read(session, cvalues, svalues);
-    for( i = 0; i < world_size && MPI_SUCCESS == ret; ++i ) {
+    for (i = 0; i < world_size && MPI_SUCCESS == ret; ++i) {
         /* Check count values */
-        if( i == world_rank && cvalues[i] != (size_t) 0 ) {
+        if (i == world_rank && cvalues[i] != (size_t) 0) {
             fprintf(stderr, "Error in %s: count_values[%d]=%zu, and should be equal to %zu.\n",
                     __func__, i, cvalues[i], (size_t) 0);
             ret = -1;
-        } else if ( i != world_rank && cvalues[i] < (size_t) (world_size + 1) * 4 ) {
-            fprintf(stderr, "Error in %s: count_values[%d]=%zu, and should be >= %zu.\n",
-                    __func__, i, cvalues[i], (size_t) (world_size + 1) * 4);
+        } else if (i != world_rank && cvalues[i] < (size_t)(world_size + 1) * 4) {
+            fprintf(stderr, "Error in %s: count_values[%d]=%zu, and should be >= %zu.\n", __func__,
+                    i, cvalues[i], (size_t)(world_size + 1) * 4);
             ret = -1;
         }
         /* Check size values */
-        if( i == world_rank && svalues[i] != (size_t) 0 ) {
+        if (i == world_rank && svalues[i] != (size_t) 0) {
             fprintf(stderr, "Error in %s: size_values[%d]=%zu, and should be equal to %zu.\n",
                     __func__, i, svalues[i], (size_t) 0);
             ret = -1;
-        } else if ( i != world_rank && svalues[i] < (size_t) (world_size * (2 * 13 * sizeof(char) + sizeof(int)) + 13 * 3 * sizeof(char) + sizeof(int)) ) {
-            fprintf(stderr, "Error in %s: size_values[%d]=%zu, and should be >= %zu.\n",
-                    __func__, i, svalues[i], (size_t) (world_size * (2 * 13 * sizeof(char) + sizeof(int)) + 13 * 3 * sizeof(char) + sizeof(int)));
+        } else if (i != world_rank
+                   && svalues[i] < (size_t)(world_size * (2 * 13 * sizeof(char) + sizeof(int))
+                                            + 13 * 3 * sizeof(char) + sizeof(int))) {
+            fprintf(stderr, "Error in %s: size_values[%d]=%zu, and should be >= %zu.\n", __func__,
+                    i, svalues[i],
+                    (size_t)(world_size * (2 * 13 * sizeof(char) + sizeof(int))
+                             + 13 * 3 * sizeof(char) + sizeof(int)));
             ret = -1;
         }
     }
     /* Check One-to-all COLL values */
     pvar_o2a_read(session, &count, &size);
-    if( count < (size_t) 2 ) {
-        fprintf(stderr, "Error in %s: count_o2a=%zu, and should be >= %zu.\n",
-                __func__, count, (size_t) 2);
+    if (count < (size_t) 2) {
+        fprintf(stderr, "Error in %s: count_o2a=%zu, and should be >= %zu.\n", __func__, count,
+                (size_t) 2);
         ret = -1;
     }
-    if( size < (size_t) ((world_size - 1) * 13 * 2 * sizeof(char)) ) {
-        fprintf(stderr, "Error in %s: size_o2a=%zu, and should be >= %zu.\n",
-                __func__, size, (size_t) ((world_size - 1) * 13 * 2 * sizeof(char)));
+    if (size < (size_t)((world_size - 1) * 13 * 2 * sizeof(char))) {
+        fprintf(stderr, "Error in %s: size_o2a=%zu, and should be >= %zu.\n", __func__, size,
+                (size_t)((world_size - 1) * 13 * 2 * sizeof(char)));
         ret = -1;
     }
     /* Check All-to-one COLL values */
     pvar_a2o_read(session, &count, &size);
-    if( count < (size_t) 2 ) {
-        fprintf(stderr, "Error in %s: count_a2o=%zu, and should be >= %zu.\n",
-                __func__, count, (size_t) 2);
+    if (count < (size_t) 2) {
+        fprintf(stderr, "Error in %s: count_a2o=%zu, and should be >= %zu.\n", __func__, count,
+                (size_t) 2);
         ret = -1;
     }
-    if( size < (size_t) ((world_size - 1) * (13 * sizeof(char) + sizeof(int))) ) {
-        fprintf(stderr, "Error in %s: size_a2o=%zu, and should be >= %zu.\n",
-                __func__, size,
-                (size_t) ((world_size - 1) * (13 * sizeof(char) + sizeof(int))));
+    if (size < (size_t)((world_size - 1) * (13 * sizeof(char) + sizeof(int)))) {
+        fprintf(stderr, "Error in %s: size_a2o=%zu, and should be >= %zu.\n", __func__, size,
+                (size_t)((world_size - 1) * (13 * sizeof(char) + sizeof(int))));
         ret = -1;
     }
     /* Check All-to-all COLL values */
     pvar_a2a_read(session, &count, &size);
-    if( count < (size_t) (world_size * 4) ) {
-        fprintf(stderr, "Error in %s: count_a2a=%zu, and should be >= %zu.\n",
-                __func__, count, (size_t) (world_size * 4));
+    if (count < (size_t)(world_size * 4)) {
+        fprintf(stderr, "Error in %s: count_a2a=%zu, and should be >= %zu.\n", __func__, count,
+                (size_t)(world_size * 4));
         ret = -1;
     }
-    if( size < (size_t) (world_size * (world_size - 1) * (2 * 13 * sizeof(char) + sizeof(int))) ) {
-        fprintf(stderr, "Error in %s: size_a2a=%zu, and should be >= %zu.\n",
-                __func__, size,
-                (size_t) (world_size * (world_size - 1) * (2 * 13 * sizeof(char) + sizeof(int))));
+    if (size < (size_t)(world_size * (world_size - 1) * (2 * 13 * sizeof(char) + sizeof(int)))) {
+        fprintf(stderr, "Error in %s: size_a2a=%zu, and should be >= %zu.\n", __func__, size,
+                (size_t)(world_size * (world_size - 1) * (2 * 13 * sizeof(char) + sizeof(int))));
         ret = -1;
     }
-    if( MPI_SUCCESS == ret ) {
+    if (MPI_SUCCESS == ret) {
         fprintf(stdout, "Check COLL...[ OK ]\n");
     } else {
         fprintf(stdout, "Check COLL...[FAIL]\n");
@@ -360,7 +375,7 @@ static inline int pvar_coll_check(MPI_T_pvar_session session, int world_size, in
     return ret;
 }
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int size, i, n, to, from, world_rank;
     MPI_T_pvar_session session;
@@ -371,7 +386,7 @@ int main(int argc, char* argv[])
     MPI_Init(NULL, NULL);
     MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
     MPI_Comm_size(MPI_COMM_WORLD, &size);
-    
+
     pvar_all_init(&session, size);
 
     /* first phase: exchange size times data with everyone in
@@ -379,52 +394,60 @@ int main(int argc, char* argv[])
        first in order to ease the prediction of messages exchanged of
        each kind.
     */
-    char*coll_buff = malloc(2 * size * 13 * sizeof(char));
-    char*coll_recv_buff = coll_buff + size * 13;
+    char *coll_buff = malloc(2 * size * 13 * sizeof(char));
+    char *coll_recv_buff = coll_buff + size * 13;
     int sum_ranks;
-    for( n = 0; n < size; ++n ) {
+    for (n = 0; n < size; ++n) {
         /* Allgather */
         memset(coll_buff, 0, size * 13 * sizeof(char));
         MPI_Allgather(s1, 13, MPI_CHAR, coll_buff, 13, MPI_CHAR, MPI_COMM_WORLD);
-        for( i = 0; i < size; ++i ) {
-            if( strncmp(s1, &coll_buff[i * 13], 13) ) {
-                fprintf(stderr, "Error in Allgather check: received \"%s\" instead of "
-                        "\"hello world!\" from %d.\n", &coll_buff[i * 13], i);
+        for (i = 0; i < size; ++i) {
+            if (strncmp(s1, &coll_buff[i * 13], 13)) {
+                fprintf(stderr,
+                        "Error in Allgather check: received \"%s\" instead of "
+                        "\"hello world!\" from %d.\n",
+                        &coll_buff[i * 13], i);
                 MPI_Abort(MPI_COMM_WORLD, -1);
             }
         }
         /* Scatter */
         MPI_Scatter(coll_buff, 13, MPI_CHAR, s2, 13, MPI_CHAR, n, MPI_COMM_WORLD);
-        if( strncmp(s1, s2, 13) ) {
-            fprintf(stderr, "Error in Scatter check: received \"%s\" instead of "
-                    "\"hello world!\" from %d.\n", s2, n);
+        if (strncmp(s1, s2, 13)) {
+            fprintf(stderr,
+                    "Error in Scatter check: received \"%s\" instead of "
+                    "\"hello world!\" from %d.\n",
+                    s2, n);
             MPI_Abort(MPI_COMM_WORLD, -1);
         }
         /* Allreduce */
         MPI_Allreduce(&world_rank, &sum_ranks, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
-        if( sum_ranks != ((size - 1) * size / 2) ) {
-            fprintf(stderr, "Error in Allreduce check: sum_ranks=%d instead of %d.\n",
-                    sum_ranks, (size - 1) * size / 2);
+        if (sum_ranks != ((size - 1) * size / 2)) {
+            fprintf(stderr, "Error in Allreduce check: sum_ranks=%d instead of %d.\n", sum_ranks,
+                    (size - 1) * size / 2);
             MPI_Abort(MPI_COMM_WORLD, -1);
         }
         /* Alltoall */
         memset(coll_recv_buff, 0, size * 13 * sizeof(char));
         MPI_Alltoall(coll_buff, 13, MPI_CHAR, coll_recv_buff, 13, MPI_CHAR, MPI_COMM_WORLD);
-        for( i = 0; i < size; ++i ) {
-            if( strncmp(s1, &coll_recv_buff[i * 13], 13) ) {
-                fprintf(stderr, "Error in Alltoall check: received \"%s\" instead of "
-                        "\"hello world!\" from %d.\n", &coll_recv_buff[i * 13], i);
+        for (i = 0; i < size; ++i) {
+            if (strncmp(s1, &coll_recv_buff[i * 13], 13)) {
+                fprintf(stderr,
+                        "Error in Alltoall check: received \"%s\" instead of "
+                        "\"hello world!\" from %d.\n",
+                        &coll_recv_buff[i * 13], i);
                 MPI_Abort(MPI_COMM_WORLD, -1);
             }
         }
         /* Bcast */
-        if( n == world_rank ) {
+        if (n == world_rank) {
             MPI_Bcast(s1, 13, MPI_CHAR, n, MPI_COMM_WORLD);
         } else {
             MPI_Bcast(s2, 13, MPI_CHAR, n, MPI_COMM_WORLD);
-            if( strncmp(s1, s2, 13) ) {
-                fprintf(stderr, "Error in Bcast check: received \"%s\" instead of "
-                        "\"hello world!\" from %d.\n", s2, n);
+            if (strncmp(s1, s2, 13)) {
+                fprintf(stderr,
+                        "Error in Bcast check: received \"%s\" instead of "
+                        "\"hello world!\" from %d.\n",
+                        s2, n);
                 MPI_Abort(MPI_COMM_WORLD, -1);
             }
         }
@@ -433,82 +456,85 @@ int main(int argc, char* argv[])
         /* Gather */
         memset(coll_buff, 0, size * 13 * sizeof(char));
         MPI_Gather(s1, 13, MPI_CHAR, coll_buff, 13, MPI_CHAR, n, MPI_COMM_WORLD);
-        if( n == world_rank ) {
-            for( i = 0; i < size; ++i ) {
-                if( strncmp(s1, &coll_buff[i * 13], 13) ) {
-                    fprintf(stderr, "Error in Gather check: received \"%s\" instead of "
-                            "\"hello world!\" from %d.\n", &coll_buff[i * 13], i);
+        if (n == world_rank) {
+            for (i = 0; i < size; ++i) {
+                if (strncmp(s1, &coll_buff[i * 13], 13)) {
+                    fprintf(stderr,
+                            "Error in Gather check: received \"%s\" instead of "
+                            "\"hello world!\" from %d.\n",
+                            &coll_buff[i * 13], i);
                     MPI_Abort(MPI_COMM_WORLD, -1);
                 }
             }
         }
         /* Reduce */
         MPI_Reduce(&world_rank, &sum_ranks, 1, MPI_INT, MPI_SUM, n, MPI_COMM_WORLD);
-        if( n == world_rank ) {
-            if( sum_ranks != ((size - 1) * size / 2) ) {
-                fprintf(stderr, "Error in Reduce check: sum_ranks=%d instead of %d.\n",
-                        sum_ranks, (size - 1) * size / 2);
+        if (n == world_rank) {
+            if (sum_ranks != ((size - 1) * size / 2)) {
+                fprintf(stderr, "Error in Reduce check: sum_ranks=%d instead of %d.\n", sum_ranks,
+                        (size - 1) * size / 2);
                 MPI_Abort(MPI_COMM_WORLD, -1);
             }
-        }        
+        }
     }
     free(coll_buff);
-    if( -1 == pvar_coll_check(session, size, world_rank) ) MPI_Abort(MPI_COMM_WORLD, -1);
+    if (-1 == pvar_coll_check(session, size, world_rank))
+        MPI_Abort(MPI_COMM_WORLD, -1);
 
     /* second phase: exchange size times data with everyone except self
-       in MPI_COMM_WORLD with Send/Recv */    
-    for( n = 0; n < size; ++n ) {
-        for( i = 0; i < size - 1; ++i ) {
-            to = (world_rank+1+i)%size;
-            from = (world_rank+size-1-i)%size;
-            if(world_rank < to){
+       in MPI_COMM_WORLD with Send/Recv */
+    for (n = 0; n < size; ++n) {
+        for (i = 0; i < size - 1; ++i) {
+            to = (world_rank + 1 + i) % size;
+            from = (world_rank + size - 1 - i) % size;
+            if (world_rank < to) {
                 MPI_Send(s1, 13, MPI_CHAR, to, world_rank, MPI_COMM_WORLD);
                 MPI_Recv(s2, 13, MPI_CHAR, from, from, MPI_COMM_WORLD, &status);
             } else {
                 MPI_Recv(s2, 13, MPI_CHAR, from, from, MPI_COMM_WORLD, &status);
                 MPI_Send(s1, 13, MPI_CHAR, to, world_rank, MPI_COMM_WORLD);
             }
-            if( strncmp(s2, "hello world!", 13) ) {
-                fprintf(stderr, "Error in PML check: s2=\"%s\" instead of \"hello world!\".\n",
-                        s2);
+            if (strncmp(s2, "hello world!", 13)) {
+                fprintf(stderr, "Error in PML check: s2=\"%s\" instead of \"hello world!\".\n", s2);
                 MPI_Abort(MPI_COMM_WORLD, -1);
             }
         }
-    }    
-    if( -1 == pvar_pml_check(session, size, world_rank) ) MPI_Abort(MPI_COMM_WORLD, -1);
+    }
+    if (-1 == pvar_pml_check(session, size, world_rank))
+        MPI_Abort(MPI_COMM_WORLD, -1);
 
     /* third phase: exchange size times data with everyone, including self, in
-       MPI_COMM_WORLD with RMA opertations */
+       MPI_COMM_WORLD with RMA operations */
     char win_buff[20];
     MPI_Win win;
     MPI_Win_create(win_buff, 20, sizeof(char), MPI_INFO_NULL, MPI_COMM_WORLD, &win);
-    for( n = 0; n < size; ++n ) {
-        for( i = 0; i < size; ++i ) {
+    for (n = 0; n < size; ++n) {
+        for (i = 0; i < size; ++i) {
             MPI_Win_lock(MPI_LOCK_EXCLUSIVE, i, 0, win);
             MPI_Put(s1, 13, MPI_CHAR, i, 0, 13, MPI_CHAR, win);
             MPI_Win_unlock(i, win);
         }
         MPI_Win_lock(MPI_LOCK_EXCLUSIVE, world_rank, 0, win);
-        if( strncmp(win_buff, "hello world!", 13) ) {
+        if (strncmp(win_buff, "hello world!", 13)) {
             fprintf(stderr, "Error in OSC check: win_buff=\"%s\" instead of \"hello world!\".\n",
                     win_buff);
             MPI_Abort(MPI_COMM_WORLD, -1);
         }
         MPI_Win_unlock(world_rank, win);
-        for( i = 0; i < size; ++i ) {
+        for (i = 0; i < size; ++i) {
             MPI_Win_lock(MPI_LOCK_EXCLUSIVE, i, 0, win);
             MPI_Get(s2, 13, MPI_CHAR, i, 0, 13, MPI_CHAR, win);
             MPI_Win_unlock(i, win);
-            if( strncmp(s2, "hello world!", 13) ) {
-                fprintf(stderr, "Error in OSC check: s2=\"%s\" instead of \"hello world!\".\n",
-                        s2);
+            if (strncmp(s2, "hello world!", 13)) {
+                fprintf(stderr, "Error in OSC check: s2=\"%s\" instead of \"hello world!\".\n", s2);
                 MPI_Abort(MPI_COMM_WORLD, -1);
             }
         }
     }
     MPI_Win_free(&win);
-    if( -1 == pvar_osc_check(session, size, world_rank) ) MPI_Abort(MPI_COMM_WORLD, -1);
-    
+    if (-1 == pvar_osc_check(session, size, world_rank))
+        MPI_Abort(MPI_COMM_WORLD, -1);
+
     pvar_all_finalize(&session);
 
     MPI_Finalize();
diff --git a/test/monitoring/example_reduce_count.c b/test/monitoring/example_reduce_count.c
index 12319db5388..cb5ede304bb 100644
--- a/test/monitoring/example_reduce_count.c
+++ b/test/monitoring/example_reduce_count.c
@@ -7,22 +7,22 @@
  * $HEADER$
  */
 
-#include <stdlib.h>
-#include <stdio.h>
 #include <mpi.h>
+#include <stdio.h>
+#include <stdlib.h>
 
 static MPI_T_pvar_handle count_handle;
 static const char count_pvar_name[] = "pml_monitoring_messages_count";
 static int count_pvar_idx;
 
-int main(int argc, char**argv)
+int main(int argc, char **argv)
 {
     int rank, size, n, to, from, tagno, MPIT_result, provided, count;
     MPI_T_pvar_session session;
     MPI_Status status;
     MPI_Request request;
     MPI_Comm comm = MPI_COMM_WORLD;
-    size_t*counts;
+    size_t *counts;
 
     n = -1;
     MPI_Init(&argc, &argv);
@@ -34,95 +34,100 @@ int main(int argc, char**argv)
 
     MPIT_result = MPI_T_init_thread(MPI_THREAD_SINGLE, &provided);
     if (MPIT_result != MPI_SUCCESS)
-	MPI_Abort(MPI_COMM_WORLD, MPIT_result);
+        MPI_Abort(MPI_COMM_WORLD, MPIT_result);
 
     MPIT_result = MPI_T_pvar_get_index(count_pvar_name, MPI_T_PVAR_CLASS_SIZE, &count_pvar_idx);
     if (MPIT_result != MPI_SUCCESS) {
-	printf("cannot find monitoring MPI_T \"%s\" pvar, check that you have monitoring pml\n",
-	       count_pvar_name);
-	MPI_Abort(MPI_COMM_WORLD, MPIT_result);
+        printf("cannot find monitoring MPI_T \"%s\" pvar, check that you have monitoring pml\n",
+               count_pvar_name);
+        MPI_Abort(MPI_COMM_WORLD, MPIT_result);
     }
 
     MPIT_result = MPI_T_pvar_session_create(&session);
     if (MPIT_result != MPI_SUCCESS) {
-	printf("cannot create a session for \"%s\" pvar\n", count_pvar_name);
-	MPI_Abort(MPI_COMM_WORLD, MPIT_result);
+        printf("cannot create a session for \"%s\" pvar\n", count_pvar_name);
+        MPI_Abort(MPI_COMM_WORLD, MPIT_result);
     }
 
     /* Allocating a new PVAR in a session will reset the counters */
-    MPIT_result = MPI_T_pvar_handle_alloc(session, count_pvar_idx,
-					  &comm, &count_handle, &count);
+    MPIT_result = MPI_T_pvar_handle_alloc(session, count_pvar_idx, &comm, &count_handle, &count);
     if (MPIT_result != MPI_SUCCESS) {
-	printf("failed to allocate handle on \"%s\" pvar, check that you have monitoring pml\n",
-	       count_pvar_name);
-	MPI_Abort(MPI_COMM_WORLD, MPIT_result);
+        printf("failed to allocate handle on \"%s\" pvar, check that you have monitoring pml\n",
+               count_pvar_name);
+        MPI_Abort(MPI_COMM_WORLD, MPIT_result);
     }
 
-    counts = (size_t*)malloc(count * sizeof(size_t));
+    counts = (size_t *) malloc(count * sizeof(size_t));
 
     MPIT_result = MPI_T_pvar_start(session, count_handle);
     if (MPIT_result != MPI_SUCCESS) {
-	printf("failed to start handle on \"%s\" pvar, check that you have monitoring pml\n",
-	       count_pvar_name);
-	MPI_Abort(MPI_COMM_WORLD, MPIT_result);
+        printf("failed to start handle on \"%s\" pvar, check that you have monitoring pml\n",
+               count_pvar_name);
+        MPI_Abort(MPI_COMM_WORLD, MPIT_result);
     }
 
     /* Token Ring communications */
     if (rank == 0) {
         n = 25;
-        MPI_Isend(&n,1,MPI_INT,to,tagno,MPI_COMM_WORLD,&request);
+        MPI_Isend(&n, 1, MPI_INT, to, tagno, MPI_COMM_WORLD, &request);
     }
     while (1) {
         MPI_Irecv(&n, 1, MPI_INT, from, tagno, MPI_COMM_WORLD, &request);
         MPI_Wait(&request, &status);
-        if (rank == 0) {n--;tagno++;}
+        if (rank == 0) {
+            n--;
+            tagno++;
+        }
         MPI_Isend(&n, 1, MPI_INT, to, tagno, MPI_COMM_WORLD, &request);
-        if (rank != 0) {n--;tagno++;}
-        if (n<0){
+        if (rank != 0) {
+            n--;
+            tagno++;
+        }
+        if (n < 0) {
             break;
         }
     }
 
     MPIT_result = MPI_T_pvar_read(session, count_handle, counts);
     if (MPIT_result != MPI_SUCCESS) {
-	printf("failed to read handle on \"%s\" pvar, check that you have monitoring pml\n",
-	       count_pvar_name);
-	MPI_Abort(MPI_COMM_WORLD, MPIT_result);
+        printf("failed to read handle on \"%s\" pvar, check that you have monitoring pml\n",
+               count_pvar_name);
+        MPI_Abort(MPI_COMM_WORLD, MPIT_result);
     }
 
     /*** REDUCE ***/
     MPI_Allreduce(MPI_IN_PLACE, counts, count, MPI_UNSIGNED_LONG, MPI_MAX, MPI_COMM_WORLD);
 
-    if(0 == rank) {
-	for(n = 0; n < count; ++n)
-	    printf("%zu%s", counts[n], n < count - 1 ? ", " : "\n");
+    if (0 == rank) {
+        for (n = 0; n < count; ++n)
+            printf("%zu%s", counts[n], n < count - 1 ? ", " : "\n");
     }
 
     free(counts);
 
     MPIT_result = MPI_T_pvar_stop(session, count_handle);
     if (MPIT_result != MPI_SUCCESS) {
-	printf("failed to stop handle on \"%s\" pvar, check that you have monitoring pml\n",
-	       count_pvar_name);
-	MPI_Abort(MPI_COMM_WORLD, MPIT_result);
+        printf("failed to stop handle on \"%s\" pvar, check that you have monitoring pml\n",
+               count_pvar_name);
+        MPI_Abort(MPI_COMM_WORLD, MPIT_result);
     }
 
     MPIT_result = MPI_T_pvar_handle_free(session, &count_handle);
     if (MPIT_result != MPI_SUCCESS) {
-	printf("failed to free handle on \"%s\" pvar, check that you have monitoring pml\n",
-	       count_pvar_name);
-	MPI_Abort(MPI_COMM_WORLD, MPIT_result);
+        printf("failed to free handle on \"%s\" pvar, check that you have monitoring pml\n",
+               count_pvar_name);
+        MPI_Abort(MPI_COMM_WORLD, MPIT_result);
     }
 
     MPIT_result = MPI_T_pvar_session_free(&session);
     if (MPIT_result != MPI_SUCCESS) {
-	printf("cannot close a session for \"%s\" pvar\n", count_pvar_name);
-	MPI_Abort(MPI_COMM_WORLD, MPIT_result);
+        printf("cannot close a session for \"%s\" pvar\n", count_pvar_name);
+        MPI_Abort(MPI_COMM_WORLD, MPIT_result);
     }
 
-    (void)MPI_T_finalize();
+    (void) MPI_T_finalize();
 
     MPI_Finalize();
-    
+
     return EXIT_SUCCESS;
 }
diff --git a/test/monitoring/monitoring_test.c b/test/monitoring/monitoring_test.c
index 9ae77abc493..c3afa2f5abc 100644
--- a/test/monitoring/monitoring_test.c
+++ b/test/monitoring/monitoring_test.c
@@ -24,7 +24,7 @@ To disable the RMA operations testing, add "--without-rma" as an application par
 
 To be run as (without using MPI_Tool):
 
-mpirun -np 4 --mca pml_monitoring_enable 2 --mca pml_monitoring_enable_output 3 --mca pml_monitoring_filename prof/output ./monitoring_test
+mpirun -n 4 --mca pml_monitoring_enable 2 --mca pml_monitoring_enable_output 3 --mca pml_monitoring_filename prof/output ./monitoring_test
 
 with the results being, as an example:
 output.1.prof
@@ -61,31 +61,30 @@ A2A     1       0 bytes 0 msgs sent
 
 */
 
-
 #include "mpi.h"
 #include <stdio.h>
 #include <string.h>
 
 static MPI_T_pvar_handle flush_handle;
 static const char flush_pvar_name[] = "pml_monitoring_flush";
-static const void*nullbuf = NULL;
+static const void *nullbuf = NULL;
 static int flush_pvar_idx;
 static int with_mpit = 0;
-static int with_rma  = 1;
+static int with_rma = 1;
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int rank, size, n, to, from, tagno, MPIT_result, provided, count, world_rank;
     MPI_T_pvar_session session;
     MPI_Comm newcomm;
     MPI_Comm comm = MPI_COMM_WORLD;
     char filename[1024];
-    
-    for ( int arg_it = 1; argc > 1 && arg_it < argc; ++arg_it ) {
-        if( 0 == strcmp(argv[arg_it], "--with-mpit") ) {
+
+    for (int arg_it = 1; argc > 1 && arg_it < argc; ++arg_it) {
+        if (0 == strcmp(argv[arg_it], "--with-mpit")) {
             with_mpit = 1;
             printf("enable MPIT support\n");
-        } else if( 0 == strcmp(argv[arg_it], "--without-rma") ) {
+        } else if (0 == strcmp(argv[arg_it], "--without-rma")) {
             with_rma = 0;
             printf("disable RMA testing\n");
         }
@@ -101,12 +100,13 @@ int main(int argc, char* argv[])
     from = (rank - 1) % size;
     tagno = 201;
 
-    if( with_mpit ) {
+    if (with_mpit) {
         MPIT_result = MPI_T_init_thread(MPI_THREAD_SINGLE, &provided);
         if (MPIT_result != MPI_SUCCESS)
             MPI_Abort(MPI_COMM_WORLD, MPIT_result);
 
-        MPIT_result = MPI_T_pvar_get_index(flush_pvar_name, MPI_T_PVAR_CLASS_GENERIC, &flush_pvar_idx);
+        MPIT_result = MPI_T_pvar_get_index(flush_pvar_name, MPI_T_PVAR_CLASS_GENERIC,
+                                           &flush_pvar_idx);
         if (MPIT_result != MPI_SUCCESS) {
             printf("cannot find monitoring MPI_T \"%s\" pvar, check that you have monitoring pml\n",
                    flush_pvar_name);
@@ -120,8 +120,8 @@ int main(int argc, char* argv[])
         }
 
         /* Allocating a new PVAR in a session will reset the counters */
-        MPIT_result = MPI_T_pvar_handle_alloc(session, flush_pvar_idx,
-                                              &comm, &flush_handle, &count);
+        MPIT_result = MPI_T_pvar_handle_alloc(session, flush_pvar_idx, &comm, &flush_handle,
+                                              &count);
         if (MPIT_result != MPI_SUCCESS) {
             printf("failed to allocate handle on \"%s\" pvar, check that you have monitoring pml\n",
                    flush_pvar_name);
@@ -138,19 +138,25 @@ int main(int argc, char* argv[])
 
     if (rank == 0) {
         n = 25;
-        MPI_Send(&n,1,MPI_INT,to,tagno,MPI_COMM_WORLD);
+        MPI_Send(&n, 1, MPI_INT, to, tagno, MPI_COMM_WORLD);
     }
     while (1) {
         MPI_Recv(&n, 1, MPI_INT, from, tagno, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
-        if (rank == 0) {n--;tagno++;}
+        if (rank == 0) {
+            n--;
+            tagno++;
+        }
         MPI_Send(&n, 1, MPI_INT, to, tagno, MPI_COMM_WORLD);
-        if (rank != 0) {n--;tagno++;}
-        if (n<0){
+        if (rank != 0) {
+            n--;
+            tagno++;
+        }
+        if (n < 0) {
             break;
         }
     }
 
-    if( with_mpit ) {
+    if (with_mpit) {
         /* Build one file per processes
               Every thing that has been monitored by each
               process since the last flush will be output in filename */
@@ -162,9 +168,9 @@ int main(int argc, char* argv[])
         */
         sprintf(filename, "prof/phase_1");
 
-        if( MPI_SUCCESS != MPI_T_pvar_write(session, flush_handle, filename) ) {
-            fprintf(stderr, "Process %d cannot save monitoring in %s.%d.prof\n",
-                    world_rank, filename, world_rank);
+        if (MPI_SUCCESS != MPI_T_pvar_write(session, flush_handle, filename)) {
+            fprintf(stderr, "Process %d cannot save monitoring in %s.%d.prof\n", world_rank,
+                    filename, world_rank);
         }
         /* Force the writing of the monitoring data */
         MPIT_result = MPI_T_pvar_stop(session, flush_handle);
@@ -183,7 +189,7 @@ int main(int argc, char* argv[])
         /* Don't set a filename. If we stop the session before setting it, then no output file
          * will be generated.
          */
-        if( MPI_SUCCESS != MPI_T_pvar_write(session, flush_handle, (void*)&nullbuf) ) {
+        if (MPI_SUCCESS != MPI_T_pvar_write(session, flush_handle, (void *) &nullbuf)) {
             fprintf(stderr, "Process %d cannot save monitoring in %s\n", world_rank, filename);
         }
     }
@@ -193,25 +199,31 @@ int main(int argc, char* argv[])
       even ranks will circulate a token
       while odd ranks will perform a all_to_all
     */
-    MPI_Comm_split(MPI_COMM_WORLD, rank%2, rank, &newcomm);
+    MPI_Comm_split(MPI_COMM_WORLD, rank % 2, rank, &newcomm);
 
-    if(rank%2){ /*odd ranks (in COMM_WORD) circulate a token*/
+    if (rank % 2) { /*odd ranks (in COMM_WORD) circulate a token*/
         MPI_Comm_rank(newcomm, &rank);
         MPI_Comm_size(newcomm, &size);
-        if( size > 1 ) {
+        if (size > 1) {
             to = (rank + 1) % size;
             from = (rank - 1) % size;
             tagno = 201;
-            if (rank == 0){
+            if (rank == 0) {
                 n = 50;
                 MPI_Send(&n, 1, MPI_INT, to, tagno, newcomm);
             }
-            while (1){
+            while (1) {
                 MPI_Recv(&n, 1, MPI_INT, from, tagno, newcomm, MPI_STATUS_IGNORE);
-                if (rank == 0) {n--; tagno++;}
+                if (rank == 0) {
+                    n--;
+                    tagno++;
+                }
                 MPI_Send(&n, 1, MPI_INT, to, tagno, newcomm);
-                if (rank != 0) {n--; tagno++;}
-                if (n<0){
+                if (rank != 0) {
+                    n--;
+                    tagno++;
+                }
+                if (n < 0) {
                     break;
                 }
             }
@@ -222,13 +234,13 @@ int main(int argc, char* argv[])
         MPI_Comm newcomm2;
         MPI_Comm_rank(newcomm, &rank);
         MPI_Comm_size(newcomm, &size);
-        MPI_Alltoall(send_buff, 10240/size, MPI_INT, recv_buff, 10240/size, MPI_INT, newcomm);
-        MPI_Comm_split(newcomm, rank%2, rank, &newcomm2);
+        MPI_Alltoall(send_buff, 10240 / size, MPI_INT, recv_buff, 10240 / size, MPI_INT, newcomm);
+        MPI_Comm_split(newcomm, rank % 2, rank, &newcomm2);
         MPI_Barrier(newcomm2);
         MPI_Comm_free(&newcomm2);
     }
 
-    if( with_mpit ) {
+    if (with_mpit) {
         /* Build one file per processes
               Every thing that has been monitored by each
               process since the last flush will be output in filename */
@@ -240,9 +252,9 @@ int main(int argc, char* argv[])
                                                   */
         sprintf(filename, "prof/phase_2");
 
-        if( MPI_SUCCESS != MPI_T_pvar_write(session, flush_handle, filename) ) {
-            fprintf(stderr, "Process %d cannot save monitoring in %s.%d.prof\n",
-                    world_rank, filename, world_rank);
+        if (MPI_SUCCESS != MPI_T_pvar_write(session, flush_handle, filename)) {
+            fprintf(stderr, "Process %d cannot save monitoring in %s.%d.prof\n", world_rank,
+                    filename, world_rank);
         }
 
         /* Force the writing of the monitoring data */
@@ -262,90 +274,93 @@ int main(int argc, char* argv[])
         /* Don't set a filename. If we stop the session before setting it, then no output
          * will be generated.
          */
-        if( MPI_SUCCESS != MPI_T_pvar_write(session, flush_handle, (void*)&nullbuf ) ) {
+        if (MPI_SUCCESS != MPI_T_pvar_write(session, flush_handle, (void *) &nullbuf)) {
             fprintf(stderr, "Process %d cannot save monitoring in %s\n", world_rank, filename);
         }
     }
 
-    if( with_rma ) {
-      MPI_Win win;
-      int rs_buff[10240];
-      int win_buff[10240];
-      MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-      MPI_Comm_size(MPI_COMM_WORLD, &size);
-      to = (rank + 1) % size;
-      from = (rank + size - 1) % size;
-      for( int v = 0; v < 10240; ++v )
-        rs_buff[v] = win_buff[v] = rank;
-
-      MPI_Win_create(win_buff, 10240 * sizeof(int), sizeof(int), MPI_INFO_NULL, MPI_COMM_WORLD, &win);
-      MPI_Win_fence(MPI_MODE_NOPRECEDE, win);
-      if( rank%2 ) {
-        MPI_Win_fence(MPI_MODE_NOSTORE | MPI_MODE_NOPUT, win);
-        MPI_Get(rs_buff, 10240, MPI_INT, from, 0, 10240, MPI_INT, win);
-      } else {
-        MPI_Put(rs_buff, 10240, MPI_INT, to, 0, 10240, MPI_INT, win);
-        MPI_Win_fence(MPI_MODE_NOSTORE | MPI_MODE_NOPUT, win);
-      }
-      MPI_Win_fence(MPI_MODE_NOSUCCEED, win);
-
-      for( int v = 0; v < 10240; ++v )
-        if( rs_buff[v] != win_buff[v] && ((rank%2 && rs_buff[v] != from) || (!(rank%2) && rs_buff[v] != rank)) ) {
-	  printf("Error on checking exchanged values: %s_buff[%d] == %d instead of %d\n",
-		 rank%2 ? "rs" : "win", v, rs_buff[v], rank%2 ? from : rank);
-	  MPI_Abort(MPI_COMM_WORLD, -1);
+    if (with_rma) {
+        MPI_Win win;
+        int rs_buff[10240];
+        int win_buff[10240];
+        MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+        MPI_Comm_size(MPI_COMM_WORLD, &size);
+        to = (rank + 1) % size;
+        from = (rank + size - 1) % size;
+        for (int v = 0; v < 10240; ++v)
+            rs_buff[v] = win_buff[v] = rank;
+
+        MPI_Win_create(win_buff, 10240 * sizeof(int), sizeof(int), MPI_INFO_NULL, MPI_COMM_WORLD,
+                       &win);
+        MPI_Win_fence(MPI_MODE_NOPRECEDE, win);
+        if (rank % 2) {
+            MPI_Win_fence(MPI_MODE_NOSTORE | MPI_MODE_NOPUT, win);
+            MPI_Get(rs_buff, 10240, MPI_INT, from, 0, 10240, MPI_INT, win);
+        } else {
+            MPI_Put(rs_buff, 10240, MPI_INT, to, 0, 10240, MPI_INT, win);
+            MPI_Win_fence(MPI_MODE_NOSTORE | MPI_MODE_NOPUT, win);
         }
+        MPI_Win_fence(MPI_MODE_NOSUCCEED, win);
+
+        for (int v = 0; v < 10240; ++v)
+            if (rs_buff[v] != win_buff[v]
+                && ((rank % 2 && rs_buff[v] != from) || (!(rank % 2) && rs_buff[v] != rank))) {
+                printf("Error on checking exchanged values: %s_buff[%d] == %d instead of %d\n",
+                       rank % 2 ? "rs" : "win", v, rs_buff[v], rank % 2 ? from : rank);
+                MPI_Abort(MPI_COMM_WORLD, -1);
+            }
 
-      MPI_Group world_group, newcomm_group, distant_group;
-      MPI_Comm_group(MPI_COMM_WORLD, &world_group);
-      MPI_Comm_group(newcomm, &newcomm_group);
-      MPI_Group_difference(world_group, newcomm_group, &distant_group);
-      if( rank%2 ) {
-        MPI_Win_post(distant_group, 0, win);
-        MPI_Win_wait(win);
-        /* Check received values */
-        for( int v = 0; v < 10240; ++v )
-	  if( from != win_buff[v] ) {
-	    printf("Error on checking exchanged values: win_buff[%d] == %d instead of %d\n",
-		   v, win_buff[v], from);
-	    MPI_Abort(MPI_COMM_WORLD, -1);
-	  }
-      } else {
-        MPI_Win_start(distant_group, 0, win);
-        MPI_Put(rs_buff, 10240, MPI_INT, to, 0, 10240, MPI_INT, win);
-        MPI_Win_complete(win);
-      }
-      MPI_Group_free(&world_group);
-      MPI_Group_free(&newcomm_group);
-      MPI_Group_free(&distant_group);
-      MPI_Barrier(MPI_COMM_WORLD);
-
-      for( int v = 0; v < 10240; ++v ) rs_buff[v] = rank;
-
-      MPI_Win_lock(MPI_LOCK_EXCLUSIVE, to, 0, win);
-      MPI_Put(rs_buff, 10240, MPI_INT, to, 0, 10240, MPI_INT, win);
-      MPI_Win_unlock(to, win);
-
-      MPI_Barrier(MPI_COMM_WORLD);
-
-      /* Check received values */
-      for( int v = 0; v < 10240; ++v )
-        if( from != win_buff[v] ) {
-	  printf("Error on checking exchanged values: win_buff[%d] == %d instead of %d\n",
-		 v, win_buff[v], from);
-	  MPI_Abort(MPI_COMM_WORLD, -1);
+        MPI_Group world_group, newcomm_group, distant_group;
+        MPI_Comm_group(MPI_COMM_WORLD, &world_group);
+        MPI_Comm_group(newcomm, &newcomm_group);
+        MPI_Group_difference(world_group, newcomm_group, &distant_group);
+        if (rank % 2) {
+            MPI_Win_post(distant_group, 0, win);
+            MPI_Win_wait(win);
+            /* Check received values */
+            for (int v = 0; v < 10240; ++v)
+                if (from != win_buff[v]) {
+                    printf("Error on checking exchanged values: win_buff[%d] == %d instead of %d\n",
+                           v, win_buff[v], from);
+                    MPI_Abort(MPI_COMM_WORLD, -1);
+                }
+        } else {
+            MPI_Win_start(distant_group, 0, win);
+            MPI_Put(rs_buff, 10240, MPI_INT, to, 0, 10240, MPI_INT, win);
+            MPI_Win_complete(win);
         }
+        MPI_Group_free(&world_group);
+        MPI_Group_free(&newcomm_group);
+        MPI_Group_free(&distant_group);
+        MPI_Barrier(MPI_COMM_WORLD);
+
+        for (int v = 0; v < 10240; ++v)
+            rs_buff[v] = rank;
+
+        MPI_Win_lock(MPI_LOCK_EXCLUSIVE, to, 0, win);
+        MPI_Put(rs_buff, 10240, MPI_INT, to, 0, 10240, MPI_INT, win);
+        MPI_Win_unlock(to, win);
+
+        MPI_Barrier(MPI_COMM_WORLD);
+
+        /* Check received values */
+        for (int v = 0; v < 10240; ++v)
+            if (from != win_buff[v]) {
+                printf("Error on checking exchanged values: win_buff[%d] == %d instead of %d\n", v,
+                       win_buff[v], from);
+                MPI_Abort(MPI_COMM_WORLD, -1);
+            }
 
-      MPI_Win_free(&win);
+        MPI_Win_free(&win);
     }
 
-    if( with_mpit ) {
+    if (with_mpit) {
         /* the filename for flushing monitoring now uses 3 as phase number! */
         sprintf(filename, "prof/phase_3");
 
-        if( MPI_SUCCESS != MPI_T_pvar_write(session, flush_handle, filename) ) {
-            fprintf(stderr, "Process %d cannot save monitoring in %s.%d.prof\n",
-                    world_rank, filename, world_rank);
+        if (MPI_SUCCESS != MPI_T_pvar_write(session, flush_handle, filename)) {
+            fprintf(stderr, "Process %d cannot save monitoring in %s.%d.prof\n", world_rank,
+                    filename, world_rank);
         }
 
         MPIT_result = MPI_T_pvar_stop(session, flush_handle);
@@ -368,7 +383,7 @@ int main(int argc, char* argv[])
             MPI_Abort(MPI_COMM_WORLD, MPIT_result);
         }
 
-        (void)MPI_T_finalize();
+        (void) MPI_T_finalize();
     }
 
     MPI_Comm_free(&newcomm);
diff --git a/test/monitoring/test_overhead.c b/test/monitoring/test_overhead.c
index 5356761334a..72d6f41e6ae 100644
--- a/test/monitoring/test_overhead.c
+++ b/test/monitoring/test_overhead.c
@@ -20,17 +20,17 @@
 
 */
 
-#include <stdlib.h>
+#include "mpi.h"
 #include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
 #include <sys/time.h>
 #include <time.h>
-#include <string.h>
-#include "mpi.h"
 
-#define NB_ITER 1000
+#define NB_ITER      1000
 #define FULL_NB_ITER (size_world * NB_ITER)
-#define MAX_SIZE (1024 * 1024 * 1.4)
-#define NB_OPS 6
+#define MAX_SIZE     (1024 * 1024 * 1.4)
+#define NB_OPS       6
 
 static int rank_world = -1;
 static int size_world = 0;
@@ -39,27 +39,27 @@ static int from = -1;
 static MPI_Win win = MPI_WIN_NULL;
 
 /* Sorting results */
-static int comp_double(const void*_a, const void*_b)
+static int comp_double(const void *_a, const void *_b)
 {
-    const double*a = _a;
-    const double*b = _b;
-    if(*a < *b)
+    const double *a = _a;
+    const double *b = _b;
+    if (*a < *b)
         return -1;
-    else if(*a > *b)
+    else if (*a > *b)
         return 1;
     else
         return 0;
 }
 
 /* Timing */
-static inline void get_tick(struct timespec*t)
+static inline void get_tick(struct timespec *t)
 {
 #if defined(__bg__)
-#  define CLOCK_TYPE CLOCK_REALTIME
+#    define CLOCK_TYPE CLOCK_REALTIME
 #elif defined(CLOCK_MONOTONIC_RAW)
-#  define CLOCK_TYPE CLOCK_MONOTONIC_RAW
+#    define CLOCK_TYPE CLOCK_MONOTONIC_RAW
 #elif defined(CLOCK_MONOTONIC)
-#  define CLOCK_TYPE CLOCK_MONOTONIC
+#    define CLOCK_TYPE CLOCK_MONOTONIC
 #endif
 #if defined(CLOCK_TYPE)
     clock_gettime(CLOCK_TYPE, t);
@@ -70,24 +70,26 @@ static inline void get_tick(struct timespec*t)
     t->tv_nsec = tv.tv_usec * 1000;
 #endif
 }
-static inline double timing_delay(const struct timespec*const t1, const struct timespec*const t2)
+static inline double timing_delay(const struct timespec *const t1, const struct timespec *const t2)
 {
-    const double delay = 1000000.0 * (t2->tv_sec - t1->tv_sec) + (t2->tv_nsec - t1->tv_nsec) / 1000.0;
+    const double delay = 1000000.0 * (t2->tv_sec - t1->tv_sec)
+                         + (t2->tv_nsec - t1->tv_nsec) / 1000.0;
     return delay;
 }
 
 /* Operations */
-static inline void op_send(double*res, void*sbuf, int size, int tagno, void*rbuf) {
+static inline void op_send(double *res, void *sbuf, int size, int tagno, void *rbuf)
+{
     MPI_Request request;
     struct timespec start, end;
 
     /* Post to be sure no unexpected message will be generated */
     MPI_Irecv(rbuf, size, MPI_BYTE, from, tagno, MPI_COMM_WORLD, &request);
-            
+
     /* Token ring to synchronize */
     /* We message the sender to make him know we are ready to
        receive (even for non-eager mode sending) */
-    if( 0 == rank_world ) {
+    if (0 == rank_world) {
         MPI_Send(NULL, 0, MPI_BYTE, from, 100, MPI_COMM_WORLD);
         MPI_Recv(NULL, 0, MPI_BYTE, to, 100, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
     } else {
@@ -104,13 +106,14 @@ static inline void op_send(double*res, void*sbuf, int size, int tagno, void*rbuf
     *res = timing_delay(&start, &end);
 }
 
-static inline void op_send_pingpong(double*res, void*sbuf, int size, int tagno, void*rbuf) {
+static inline void op_send_pingpong(double *res, void *sbuf, int size, int tagno, void *rbuf)
+{
     struct timespec start, end;
 
     MPI_Barrier(MPI_COMM_WORLD);
 
     /* do monitored operation */
-    if(rank_world % 2) { /* Odd ranks : Recv - Send */
+    if (rank_world % 2) { /* Odd ranks : Recv - Send */
         MPI_Recv(rbuf, size, MPI_BYTE, from, tagno, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
         MPI_Send(sbuf, size, MPI_BYTE, from, tagno, MPI_COMM_WORLD);
         MPI_Barrier(MPI_COMM_WORLD);
@@ -131,7 +134,8 @@ static inline void op_send_pingpong(double*res, void*sbuf, int size, int tagno,
     *res = timing_delay(&start, &end) / 2;
 }
 
-static inline void op_coll(double*res, void*buff, int size, int tagno, void*rbuf) {
+static inline void op_coll(double *res, void *buff, int size, int tagno, void *rbuf)
+{
     struct timespec start, end;
     MPI_Barrier(MPI_COMM_WORLD);
 
@@ -143,7 +147,8 @@ static inline void op_coll(double*res, void*buff, int size, int tagno, void*rbuf
     *res = timing_delay(&start, &end);
 }
 
-static inline void op_a2a(double*res, void*sbuf, int size, int tagno, void*rbuf) {
+static inline void op_a2a(double *res, void *sbuf, int size, int tagno, void *rbuf)
+{
     struct timespec start, end;
     MPI_Barrier(MPI_COMM_WORLD);
 
@@ -155,7 +160,8 @@ static inline void op_a2a(double*res, void*sbuf, int size, int tagno, void*rbuf)
     *res = timing_delay(&start, &end);
 }
 
-static inline void op_put(double*res, void*sbuf, int size, int tagno, void*rbuf) {
+static inline void op_put(double *res, void *sbuf, int size, int tagno, void *rbuf)
+{
     struct timespec start, end;
 
     MPI_Win_lock(MPI_LOCK_EXCLUSIVE, to, 0, win);
@@ -169,7 +175,8 @@ static inline void op_put(double*res, void*sbuf, int size, int tagno, void*rbuf)
     *res = timing_delay(&start, &end);
 }
 
-static inline void op_get(double*res, void*rbuf, int size, int tagno, void*sbuf) {
+static inline void op_get(double *res, void *rbuf, int size, int tagno, void *sbuf)
+{
     struct timespec start, end;
 
     MPI_Win_lock(MPI_LOCK_SHARED, to, 0, win);
@@ -183,34 +190,35 @@ static inline void op_get(double*res, void*rbuf, int size, int tagno, void*sbuf)
     *res = timing_delay(&start, &end);
 }
 
-static inline void do_bench(int size, char*sbuf, double*results,
-                            void(*op)(double*, void*, int, int, void*)) {
+static inline void do_bench(int size, char *sbuf, double *results,
+                            void (*op)(double *, void *, int, int, void *))
+{
     int iter;
     int tagno = 201;
-    char*rbuf = sbuf ? sbuf + size : NULL;
+    char *rbuf = sbuf ? sbuf + size : NULL;
 
-    if(op == op_put || op == op_get){
-	win = MPI_WIN_NULL;
-	MPI_Win_create(rbuf, size, 1, MPI_INFO_NULL, MPI_COMM_WORLD, &win);
+    if (op == op_put || op == op_get) {
+        win = MPI_WIN_NULL;
+        MPI_Win_create(rbuf, size, 1, MPI_INFO_NULL, MPI_COMM_WORLD, &win);
     }
 
-    for( iter = 0; iter < NB_ITER; ++iter ) {
+    for (iter = 0; iter < NB_ITER; ++iter) {
         op(&results[iter], sbuf, size, tagno, rbuf);
         MPI_Barrier(MPI_COMM_WORLD);
     }
 
-    if(op == op_put || op == op_get){
-	MPI_Win_free(&win);
-	win = MPI_WIN_NULL;
+    if (op == op_put || op == op_get) {
+        MPI_Win_free(&win);
+        win = MPI_WIN_NULL;
     }
 }
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int size, iter, nop;
-    char*sbuf = NULL;
+    char *sbuf = NULL;
     double results[NB_ITER];
-    void(*op)(double*, void*, int, int, void*);
+    void (*op)(double *, void *, int, int, void *);
     char name[255];
     MPI_Init(&argc, &argv);
     MPI_Comm_rank(MPI_COMM_WORLD, &rank_world);
@@ -220,8 +228,8 @@ int main(int argc, char* argv[])
 
     double full_res[FULL_NB_ITER];
 
-    for( nop = 0; nop < NB_OPS; ++nop ) {
-        switch(nop) {
+    for (nop = 0; nop < NB_OPS; ++nop) {
+        switch (nop) {
         case 0:
             op = op_send;
             sprintf(name, "MPI_Send");
@@ -248,40 +256,45 @@ int main(int argc, char* argv[])
             break;
         }
 
-        if( 0 == rank_world )
-            printf("# %s%%%d\n# size  \t|  latency \t| 10^6 B/s \t| MB/s   \t| median  \t| q1     \t| q3     \t| d1     \t| d9     \t| avg    \t| max\n", name, size_world);
+        if (0 == rank_world)
+            printf("# %s%%%d\n# size  \t|  latency \t| 10^6 B/s \t| MB/s   \t| median  \t| q1     "
+                   "\t| q3     \t| d1     \t| d9     \t| avg    \t| max\n",
+                   name, size_world);
 
-        for(size = 0; size < MAX_SIZE; size = ((int)(size * 1.4) > size) ? (size * 1.4) : (size + 1)) {
+        for (size = 0; size < MAX_SIZE;
+             size = ((int) (size * 1.4) > size) ? (size * 1.4) : (size + 1)) {
             /* Init buffers */
-            if( 0 != size ) {
-                sbuf = (char *)realloc(sbuf, (size_world + 1) * size); /* sbuf + alltoall recv buf */
-            }        
+            if (0 != size) {
+                sbuf = (char *) realloc(sbuf,
+                                        (size_world + 1) * size); /* sbuf + alltoall recv buf */
+            }
 
             do_bench(size, sbuf, results, op);
 
-            MPI_Gather(results, NB_ITER, MPI_DOUBLE, full_res, NB_ITER, MPI_DOUBLE, 0, MPI_COMM_WORLD);
+            MPI_Gather(results, NB_ITER, MPI_DOUBLE, full_res, NB_ITER, MPI_DOUBLE, 0,
+                       MPI_COMM_WORLD);
 
-            if( 0 == rank_world ) {
+            if (0 == rank_world) {
                 qsort(full_res, FULL_NB_ITER, sizeof(double), &comp_double);
                 const double min_lat = full_res[0];
                 const double max_lat = full_res[FULL_NB_ITER - 1];
                 const double med_lat = full_res[(FULL_NB_ITER - 1) / 2];
-                const double q1_lat  = full_res[(FULL_NB_ITER - 1) / 4];
-                const double q3_lat  = full_res[ 3 * (FULL_NB_ITER - 1) / 4];
-                const double d1_lat  = full_res[(FULL_NB_ITER - 1) / 10];
-                const double d9_lat  = full_res[ 9 * (FULL_NB_ITER - 1) / 10];
+                const double q1_lat = full_res[(FULL_NB_ITER - 1) / 4];
+                const double q3_lat = full_res[3 * (FULL_NB_ITER - 1) / 4];
+                const double d1_lat = full_res[(FULL_NB_ITER - 1) / 10];
+                const double d9_lat = full_res[9 * (FULL_NB_ITER - 1) / 10];
                 double avg_lat = 0.0;
-                for( iter = 0; iter < FULL_NB_ITER; iter++ ){
+                for (iter = 0; iter < FULL_NB_ITER; iter++) {
                     avg_lat += full_res[iter];
                 }
                 avg_lat /= FULL_NB_ITER;
                 const double bw_million_byte = size / min_lat;
-                const double bw_mbyte        = bw_million_byte / 1.048576;
+                const double bw_mbyte = bw_million_byte / 1.048576;
 
-                printf("%9lld\t%9.3lf\t%9.3f\t%9.3f\t%9.3lf\t%9.3lf\t%9.3lf\t%9.3lf\t%9.3lf\t%9.3lf\t%9.3lf",
-                       (long long)size, min_lat, bw_million_byte, bw_mbyte,
-                       med_lat, q1_lat, q3_lat, d1_lat, d9_lat,
-                       avg_lat, max_lat);
+                printf("%9lld\t%9.3lf\t%9.3f\t%9.3f\t%9.3lf\t%9.3lf\t%9.3lf\t%9.3lf\t%9.3lf\t%9."
+                       "3lf\t%9.3lf",
+                       (long long) size, min_lat, bw_million_byte, bw_mbyte, med_lat, q1_lat,
+                       q3_lat, d1_lat, d9_lat, avg_lat, max_lat);
                 printf("\n");
             }
         }
diff --git a/test/monitoring/test_overhead.sh b/test/monitoring/test_overhead.sh
index 96814ba0bad..f6a3cd2b8c1 100755
--- a/test/monitoring/test_overhead.sh
+++ b/test/monitoring/test_overhead.sh
@@ -18,7 +18,7 @@
 # results in order to create heatmaps. To work properly, this scripts
 # needs sqlite3, sed, awk and gnuplot. It also needs the rights to
 # write/create directories in the working path. Temporary files can be
-# found in $resdir/.tmp. They are cleaned between two executions fo
+# found in $resdir/.tmp. They are cleaned between two executions for
 # this script.
 #
 # This file create as an output one heatmap per operation
diff --git a/test/monitoring/test_pvar_access.c b/test/monitoring/test_pvar_access.c
index d035ea388e1..6412e9cc42d 100644
--- a/test/monitoring/test_pvar_access.c
+++ b/test/monitoring/test_pvar_access.c
@@ -20,7 +20,7 @@ Contact the authors for questions.
 
 To be run as:
 
-mpirun -np 4 --mca pml_monitoring_enable 2 ./test_pvar_access
+mpirun -n 4 --mca pml_monitoring_enable 2 ./test_pvar_access
 
 Then, the output should be:
 Flushing phase 1:
@@ -41,9 +41,9 @@ I       3       2       20 bytes        4 msgs sent
 
 */
 
-#include <stdlib.h>
-#include <stdio.h>
 #include <mpi.h>
+#include <stdio.h>
+#include <stdlib.h>
 
 static MPI_T_pvar_handle count_handle;
 static MPI_T_pvar_handle msize_handle;
@@ -52,16 +52,16 @@ static const char msize_pvar_name[] = "pml_monitoring_messages_size";
 static int count_pvar_idx, msize_pvar_idx;
 static int world_rank, world_size;
 
-static void print_vars(int rank, int size, size_t* msg_count, size_t*msg_size)
+static void print_vars(int rank, int size, size_t *msg_count, size_t *msg_size)
 {
     int i;
-    for(i = 0; i < size; ++i) {
-        if(0 != msg_size[i])
+    for (i = 0; i < size; ++i) {
+        if (0 != msg_size[i])
             printf("I\t%d\t%d\t%zu bytes\t%zu msgs sent\n", rank, i, msg_size[i], msg_count[i]);
     }
 }
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int rank, size, n, to, from, tagno, MPIT_result, provided, count;
     MPI_T_pvar_session session;
@@ -69,8 +69,8 @@ int main(int argc, char* argv[])
     MPI_Comm newcomm;
     MPI_Comm comm = MPI_COMM_WORLD;
     MPI_Request request;
-    size_t*msg_count_p1, *msg_size_p1;
-    size_t*msg_count_p2, *msg_size_p2;
+    size_t *msg_count_p1, *msg_size_p1;
+    size_t *msg_count_p2, *msg_size_p2;
 
     /* first phase : make a token circulated in MPI_COMM_WORLD */
     n = -1;
@@ -86,7 +86,7 @@ int main(int argc, char* argv[])
     MPIT_result = MPI_T_init_thread(MPI_THREAD_SINGLE, &provided);
     if (MPIT_result != MPI_SUCCESS)
         MPI_Abort(MPI_COMM_WORLD, MPIT_result);
-    
+
     /* Retrieve the pvar indices */
     MPIT_result = MPI_T_pvar_get_index(count_pvar_name, MPI_T_PVAR_CLASS_SIZE, &count_pvar_idx);
     if (MPIT_result != MPI_SUCCESS) {
@@ -104,21 +104,19 @@ int main(int argc, char* argv[])
     /* Get session for pvar binding */
     MPIT_result = MPI_T_pvar_session_create(&session);
     if (MPIT_result != MPI_SUCCESS) {
-        printf("cannot create a session for \"%s\" and \"%s\" pvars\n",
-               count_pvar_name, msize_pvar_name);
+        printf("cannot create a session for \"%s\" and \"%s\" pvars\n", count_pvar_name,
+               msize_pvar_name);
         MPI_Abort(MPI_COMM_WORLD, MPIT_result);
     }
 
     /* Allocating a new PVAR in a session will reset the counters */
-    MPIT_result = MPI_T_pvar_handle_alloc(session, count_pvar_idx,
-                                          &comm, &count_handle, &count);
+    MPIT_result = MPI_T_pvar_handle_alloc(session, count_pvar_idx, &comm, &count_handle, &count);
     if (MPIT_result != MPI_SUCCESS) {
         printf("failed to allocate handle on \"%s\" pvar, check that you have monitoring pml\n",
                count_pvar_name);
         MPI_Abort(MPI_COMM_WORLD, MPIT_result);
     }
-    MPIT_result = MPI_T_pvar_handle_alloc(session, msize_pvar_idx,
-                                          &comm, &msize_handle, &count);
+    MPIT_result = MPI_T_pvar_handle_alloc(session, msize_pvar_idx, &comm, &msize_handle, &count);
     if (MPIT_result != MPI_SUCCESS) {
         printf("failed to allocate handle on \"%s\" pvar, check that you have monitoring pml\n",
                msize_pvar_name);
@@ -128,8 +126,8 @@ int main(int argc, char* argv[])
     /* Allocate arrays to retrieve results */
     msg_count_p1 = calloc(count * 4, sizeof(size_t));
     msg_size_p1 = &msg_count_p1[count];
-    msg_count_p2 = &msg_count_p1[2*count];
-    msg_size_p2 = &msg_count_p1[3*count];
+    msg_count_p2 = &msg_count_p1[2 * count];
+    msg_size_p2 = &msg_count_p1[3 * count];
 
     /* Start pvar */
     MPIT_result = MPI_T_pvar_start(session, count_handle);
@@ -144,18 +142,24 @@ int main(int argc, char* argv[])
                msize_pvar_name);
         MPI_Abort(MPI_COMM_WORLD, MPIT_result);
     }
-    
+
     if (rank == 0) {
         n = 25;
-        MPI_Isend(&n,1,MPI_INT,to,tagno,MPI_COMM_WORLD,&request);
+        MPI_Isend(&n, 1, MPI_INT, to, tagno, MPI_COMM_WORLD, &request);
     }
     while (1) {
         MPI_Irecv(&n, 1, MPI_INT, from, tagno, MPI_COMM_WORLD, &request);
         MPI_Wait(&request, &status);
-        if (rank == 0) {n--;tagno++;}
+        if (rank == 0) {
+            n--;
+            tagno++;
+        }
         MPI_Isend(&n, 1, MPI_INT, to, tagno, MPI_COMM_WORLD, &request);
-        if (rank != 0) {n--;tagno++;}
-        if (n<0){
+        if (rank != 0) {
+            n--;
+            tagno++;
+        }
+        if (n < 0) {
             break;
         }
     }
@@ -166,7 +170,7 @@ int main(int argc, char* argv[])
         printf("failed to stop handle on \"%s\" pvar, check that you have monitoring pml\n",
                count_pvar_name);
         MPI_Abort(MPI_COMM_WORLD, MPIT_result);
-    }    
+    }
     MPIT_result = MPI_T_pvar_stop(session, msize_handle);
     if (MPIT_result != MPI_SUCCESS) {
         printf("failed to stop handle on \"%s\" pvar, check that you have monitoring pml\n",
@@ -188,7 +192,7 @@ int main(int argc, char* argv[])
     }
 
     /* Circulate a token to proper display the results */
-    if(0 == world_rank) {
+    if (0 == world_rank) {
         printf("Flushing phase 1:\n");
         print_vars(world_rank, world_size, msg_count_p1, msg_size_p1);
         MPI_Send(NULL, 0, MPI_BYTE, (world_rank + 1) % world_size, 300, MPI_COMM_WORLD);
@@ -212,7 +216,7 @@ int main(int argc, char* argv[])
                msize_pvar_name);
         MPI_Abort(MPI_COMM_WORLD, MPIT_result);
     }
-    
+
     /*
       Second phase. Work with different communicators.
       even ranks will circulate a token
@@ -230,26 +234,32 @@ int main(int argc, char* argv[])
                msize_pvar_name);
         MPI_Abort(MPI_COMM_WORLD, MPIT_result);
     }
-    
-    MPI_Comm_split(MPI_COMM_WORLD, rank%2, rank, &newcomm);
 
-    if(rank%2){ /*even ranks (in COMM_WORD) circulate a token*/
+    MPI_Comm_split(MPI_COMM_WORLD, rank % 2, rank, &newcomm);
+
+    if (rank % 2) { /*even ranks (in COMM_WORD) circulate a token*/
         MPI_Comm_rank(newcomm, &rank);
         MPI_Comm_size(newcomm, &size);
-        if( size > 1 ) {
+        if (size > 1) {
             to = (rank + 1) % size;
             from = (rank - 1) % size;
             tagno = 201;
-            if (rank == 0){
+            if (rank == 0) {
                 n = 50;
                 MPI_Send(&n, 1, MPI_INT, to, tagno, newcomm);
             }
-            while (1){
+            while (1) {
                 MPI_Recv(&n, 1, MPI_INT, from, tagno, newcomm, &status);
-                if (rank == 0) {n--; tagno++;}
+                if (rank == 0) {
+                    n--;
+                    tagno++;
+                }
                 MPI_Send(&n, 1, MPI_INT, to, tagno, newcomm);
-                if (rank != 0) {n--; tagno++;}
-                if (n<0){
+                if (rank != 0) {
+                    n--;
+                    tagno++;
+                }
+                if (n < 0) {
                     break;
                 }
             }
@@ -259,8 +269,8 @@ int main(int argc, char* argv[])
         int recv_buff[10240];
         MPI_Comm_rank(newcomm, &rank);
         MPI_Comm_size(newcomm, &size);
-        MPI_Alltoall(send_buff, 10240/size, MPI_INT, recv_buff, 10240/size, MPI_INT, newcomm);
-        MPI_Comm_split(newcomm, rank%2, rank, &newcomm);
+        MPI_Alltoall(send_buff, 10240 / size, MPI_INT, recv_buff, 10240 / size, MPI_INT, newcomm);
+        MPI_Comm_split(newcomm, rank % 2, rank, &newcomm);
         MPI_Barrier(newcomm);
     }
 
@@ -278,13 +288,13 @@ int main(int argc, char* argv[])
     }
 
     /* Taking only in account the second phase */
-    for(int i = 0; i < size; ++i) {
+    for (int i = 0; i < size; ++i) {
         msg_count_p2[i] -= msg_count_p1[i];
         msg_size_p2[i] -= msg_size_p1[i];
     }
 
     /* Circulate a token to proper display the results */
-    if(0 == world_rank) {
+    if (0 == world_rank) {
         printf("Flushing phase 2:\n");
         print_vars(world_rank, world_size, msg_count_p2, msg_size_p2);
         MPI_Send(NULL, 0, MPI_BYTE, (world_rank + 1) % world_size, 300, MPI_COMM_WORLD);
@@ -310,15 +320,15 @@ int main(int argc, char* argv[])
 
     MPIT_result = MPI_T_pvar_session_free(&session);
     if (MPIT_result != MPI_SUCCESS) {
-        printf("cannot close a session for \"%s\" and \"%s\" pvars\n",
-               count_pvar_name, msize_pvar_name);
+        printf("cannot close a session for \"%s\" and \"%s\" pvars\n", count_pvar_name,
+               msize_pvar_name);
         MPI_Abort(MPI_COMM_WORLD, MPIT_result);
     }
 
-    (void)MPI_T_finalize();
+    (void) MPI_T_finalize();
 
     free(msg_count_p1);
-    
+
     MPI_Finalize();
     return EXIT_SUCCESS;
 }
diff --git a/test/mpi/environment/Makefile.am b/test/mpi/environment/Makefile.am
index 885c15b09d6..820856a3d51 100644
--- a/test/mpi/environment/Makefile.am
+++ b/test/mpi/environment/Makefile.am
@@ -29,5 +29,5 @@ chello_LDADD = \
 chello_DEPENDENCIES = $(chello_LDADD)
 
 
-distclean:
+distclean-local:
 	rm -rf *.dSYM .deps *.log *.o *.trs $(noinst_PROGRAMS) Makefile
diff --git a/test/mpi/environment/chello.c b/test/mpi/environment/chello.c
index 453a5a1bf93..6f436673b3e 100644
--- a/test/mpi/environment/chello.c
+++ b/test/mpi/environment/chello.c
@@ -22,14 +22,13 @@
 #include <mpi.h>
 #include <stdio.h>
 
-int
-main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
-  MPI_Init(&argc, &argv);
+    MPI_Init(&argc, &argv);
 
-  printf("Hello, World\n");
+    printf("Hello, World\n");
 
-  MPI_Finalize();
+    MPI_Finalize();
 
-  return 0;
+    return 0;
 }
diff --git a/test/mpool/Makefile.am b/test/mpool/Makefile.am
index 620ed8c9a56..11ad7e19e76 100644
--- a/test/mpool/Makefile.am
+++ b/test/mpool/Makefile.am
@@ -14,8 +14,8 @@ check_PROGRAMS = $(TESTS) $(MPI_CHECKS)
 mpool_memkind_SOURCES = mpool_memkind.c
 
 LDFLAGS = $(OPAL_PKG_CONFIG_LDFLAGS)
-LDADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+LDADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 
-distclean:
+distclean-local:
 	rm -rf *.dSYM .deps .libs *.log *.o *.trs $(check_PROGRAMS) Makefile
 
diff --git a/test/mpool/mpool_memkind.c b/test/mpool/mpool_memkind.c
index d33b0da9ef6..d3a3e775ddf 100644
--- a/test/mpool/mpool_memkind.c
+++ b/test/mpool/mpool_memkind.c
@@ -23,41 +23,28 @@
  * only do this test if we have built with memkind support
  */
 
+#include "opal_config.h"
 #include <stdio.h>
-#include <string.h>
 #include <stdlib.h>
-#include "opal_config.h"
+#include <string.h>
 #ifdef HAVE_MEMKIND_H
-#include "opal/constants.h"
-#include "opal/align.h"
-#include "opal/mca/mpool/mpool.h"
-#include "opal/include/opal/frameworks.h"
-#include "opal/runtime/opal.h"
-
-#define SIZE (2 * 1024 * 1024)
-
-const char *memory_types[] = {
-  "memkind_default",
-  "memkind_hbw",
-   NULL
-};
-
-const char *memory_policy[] = {
-   "mempolicy_bind_local",
-   "mempolicy_bind_all",
-   "mempolicy_perferred_local",
-   "mempolicy_interleave_local",
-   "mempolicy_interleave_all",
-   NULL
-};
-
-const char *memory_kind_bits[] = {
-   "memkind_mask_page_size_4KB",
-   "memkind_mask_page_size_2MB",
-   NULL
-};
-
-int main (int argc, char* argv[])
+#    include "opal/align.h"
+#    include "opal/constants.h"
+#    include "opal/include/opal/frameworks.h"
+#    include "opal/mca/mpool/mpool.h"
+#    include "opal/runtime/opal.h"
+
+#    define SIZE (2 * 1024 * 1024)
+
+const char *memory_types[] = {"memkind_default", "memkind_hbw", NULL};
+
+const char *memory_policy[] = {"mempolicy_bind_local",      "mempolicy_bind_all",
+                               "mempolicy_perferred_local", "mempolicy_interleave_local",
+                               "mempolicy_interleave_all",  NULL};
+
+const char *memory_kind_bits[] = {"memkind_mask_page_size_4KB", "memkind_mask_page_size_2MB", NULL};
+
+int main(int argc, char *argv[])
 {
     int ret = 0;
     void *ptr = NULL;
@@ -70,7 +57,7 @@ int main (int argc, char* argv[])
 
     opal_init_util(&argc, &argv);
 
-    if (opal_frameworks == NULL){
+    if (opal_frameworks == NULL) {
         error = "opal frameworks is NULL";
         goto error;
     }
@@ -100,7 +87,7 @@ int main (int argc, char* argv[])
         goto error;
     }
 
-    if (0 != ((uintptr_t)ptr % OPAL_ALIGN_MIN)) {
+    if (0 != ((uintptr_t) ptr % OPAL_ALIGN_MIN)) {
         error = "improper memory alignment detected";
         goto error;
     }
@@ -109,16 +96,16 @@ int main (int argc, char* argv[])
      * now try policies
      */
 
-    mp_ptr = (char **)memory_policy;
+    mp_ptr = (char **) memory_policy;
     while (NULL != *mp_ptr) {
 
-        mt_ptr = (char **)memory_types;
+        mt_ptr = (char **) memory_types;
         while (NULL != *mt_ptr) {
 
-            mk_ptr = (char **)memory_kind_bits;
+            mk_ptr = (char **) memory_kind_bits;
             while (NULL != *mk_ptr) {
-                snprintf(hints, sizeof(hints), "%s,policy=%s,type=%s,kind=%s", 
-                         mpool_hints, *mp_ptr, *mt_ptr, *mk_ptr);
+                snprintf(hints, sizeof(hints), "%s,policy=%s,type=%s,kind=%s", mpool_hints, *mp_ptr,
+                         *mt_ptr, *mk_ptr);
                 ptr = mca_mpool_base_alloc(SIZE, NULL, hints);
                 if (NULL == ptr) {
                     error = "mca_mpool_base_alloc() failed";
@@ -130,7 +117,7 @@ int main (int argc, char* argv[])
                     goto error;
                 }
 
-                if (0 != ((uintptr_t)ptr % OPAL_ALIGN_MIN)) {
+                if (0 != ((uintptr_t) ptr % OPAL_ALIGN_MIN)) {
                     error = "improper memory alignment detected";
                     goto error;
                 }
@@ -164,7 +151,7 @@ int main (int argc, char* argv[])
     return ret;
 }
 #else
-int main (int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     return 77;
 }
diff --git a/test/runtime/Makefile.am b/test/runtime/Makefile.am
index 7c194b5f6e8..676bfc21de9 100644
--- a/test/runtime/Makefile.am
+++ b/test/runtime/Makefile.am
@@ -52,9 +52,9 @@ orte_init_finalize_DEPENDENCIES = $(orte_init_finalize_LDADD)
 opal_init_finalize_SOURCES = \
     opal_init_finalize.c
 opal_init_finalize_LDADD = \
-	$(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
+	$(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
 	$(top_builddir)/test/support/libsupport.a
 opal_init_finalize_DEPENDENCIES = $(opal_init_finalize_LDADD)
 
-distclean:
+distclean-local:
 	rm -rf *.dSYM .deps *.log *.o *.trs $(check_PROGRAMS) Makefile
diff --git a/test/runtime/opal_init_finalize.c b/test/runtime/opal_init_finalize.c
index 42b4a179272..bf07bc577be 100644
--- a/test/runtime/opal_init_finalize.c
+++ b/test/runtime/opal_init_finalize.c
@@ -18,11 +18,10 @@
 
 #include "ompi_config.h"
 
-#include "opal/runtime/opal.h"
 #include "opal/constants.h"
+#include "opal/runtime/opal.h"
 
-int
-main(int argc, char *argv[])
+int main(int argc, char *argv[])
 {
     int ret;
 
diff --git a/test/runtime/orte_init_finalize.c b/test/runtime/orte_init_finalize.c
index b134f0f4221..da9c2d3498d 100644
--- a/test/runtime/orte_init_finalize.c
+++ b/test/runtime/orte_init_finalize.c
@@ -19,20 +19,19 @@
 #include "orte_config.h"
 #include "orte/orte_constants.h"
 
-
 #include <stdio.h>
-#include <string.h>
 #include <stdlib.h>
+#include <string.h>
 #ifdef HAVE_SYS_PARAM_H
-#include <sys/param.h>
+#    include <sys/param.h>
 #endif
 #ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
+#    include <netinet/in.h>
 #endif
 
 #include "orte/runtime/runtime.h"
 
-int main (int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int rc;
 
diff --git a/test/runtime/sigchld.c b/test/runtime/sigchld.c
index d958a26de12..ea060793294 100644
--- a/test/runtime/sigchld.c
+++ b/test/runtime/sigchld.c
@@ -18,15 +18,15 @@
 
 #include "ompi_config.h"
 #include "mpi.h"
-#include "orte/runtime/orte_wait.h"
 #include "opal/runtime/opal_progress.h"
+#include "orte/runtime/orte_wait.h"
 #include "runtime/runtime.h"
 
 #ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+#    include <sys/types.h>
 #endif
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 #include <stdio.h>
 #include <stdlib.h>
@@ -39,7 +39,6 @@ static void callback(pid_t pid, int status, void *data)
     count--;
 }
 
-
 int main(int argc, char *argv[])
 {
     pid_t pid, ret;
diff --git a/test/runtime/start_shut.c b/test/runtime/start_shut.c
index c7004bcd77c..6eb6e8171bf 100644
--- a/test/runtime/start_shut.c
+++ b/test/runtime/start_shut.c
@@ -19,25 +19,24 @@
 #include "orte_config.h"
 #include "orte/orte_constants.h"
 
-
 #include <stdio.h>
-#include <string.h>
 #include <stdlib.h>
+#include <string.h>
 #ifdef HAVE_SYS_PARAM_H
-#include <sys/param.h>
+#    include <sys/param.h>
 #endif
 #ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
+#    include <netinet/in.h>
 #endif
 
-#include "support.h"
 #include "orte/runtime/runtime.h"
+#include "support.h"
 
 #define NUM_ITERS 3
 
 FILE *test_out;
 
-int main (int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int rc, i;
 
@@ -50,14 +49,16 @@ int main (int argc, char* argv[])
         exit(1);
     }
 
-    for (i=0; i < NUM_ITERS; i++) {
+    for (i = 0; i < NUM_ITERS; i++) {
         if (ORTE_SUCCESS != (rc = orte_finalize())) {
-            fprintf(test_out, "iter %d: couldn't complete orte system finalize - error %d\n", i, rc);
+            fprintf(test_out, "iter %d: couldn't complete orte system finalize - error %d\n", i,
+                    rc);
             exit(1);
         }
         fprintf(test_out, "\tfinalize successful\n");
         if (ORTE_SUCCESS != (rc = orte_init(true))) {
-            fprintf(test_out, "iter %d: couldn't complete orte system init - error code %d\n", i, rc);
+            fprintf(test_out, "iter %d: couldn't complete orte system init - error code %d\n", i,
+                    rc);
             exit(1);
         }
     }
diff --git a/test/simple/abort.c b/test/simple/abort.c
index 65f17fddfd5..f67e77f5f23 100644
--- a/test/simple/abort.c
+++ b/test/simple/abort.c
@@ -11,7 +11,7 @@
 
 #include "mpi.h"
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int rank, size;
     int errcode;
@@ -29,7 +29,7 @@ int main(int argc, char* argv[])
     printf("Hello, World, I am %d of %d\n", rank, size);
 
     if (1 == size) {
-            MPI_Abort(MPI_COMM_WORLD, errcode);
+        MPI_Abort(MPI_COMM_WORLD, errcode);
     } else {
         if (1 == rank) {
             MPI_Abort(MPI_COMM_WORLD, errcode);
diff --git a/test/simple/accept.c b/test/simple/accept.c
index 3462c7500d0..c68866f7665 100644
--- a/test/simple/accept.c
+++ b/test/simple/accept.c
@@ -5,10 +5,10 @@
  * Test of connect/accept - the accept (server) side
  */
 
-#include <stdio.h>
 #include "mpi.h"
+#include <stdio.h>
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int rank, size;
     MPI_Info info;
diff --git a/test/simple/add_host.c b/test/simple/add_host.c
index d0a812bca8d..e59c4e498d9 100644
--- a/test/simple/add_host.c
+++ b/test/simple/add_host.c
@@ -1,29 +1,27 @@
 #include <stdio.h>
 #include <stdlib.h>
+#include <sys/param.h>
 #include <sys/types.h>
 #include <unistd.h>
-#include <sys/param.h>
-
-#include "opal/runtime/opal.h"
 
 #include <mpi.h>
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int msg, rc;
     MPI_Comm parent, child;
     int rank, size;
-    const char *hostname;
+    char hostname[1024];
     pid_t pid;
-    char *env_rank,*env_nspace;
+    char *env_rank, *env_nspace;
     MPI_Info info;
 
     env_rank = getenv("PMIX_RANK");
     env_nspace = getenv("PMIX_NAMESPACE");
     pid = getpid();
-    hostname = opal_gethostname();
+    gethostname(hostname, 1024);
 
-    printf("[%s:%s pid %ld] starting up on node %s!\n", env_nspace, env_rank, (long)pid, hostname);
+    printf("[%s:%s pid %ld] starting up on node %s!\n", env_nspace, env_rank, (long) pid, hostname);
 
     MPI_Init(NULL, NULL);
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
@@ -32,12 +30,20 @@ int main(int argc, char* argv[])
     MPI_Comm_get_parent(&parent);
     /* If we get COMM_NULL back, then we're the parent */
     if (MPI_COMM_NULL == parent) {
+        if (argc < 3) {
+            printf("%s requires two arguments (%d)- the name of the first host to add, and the "
+                   "name of the second host to add\n",
+                   argv[0], argc);
+            exit(1);
+        }
+
         pid = getpid();
-        printf("Parent [pid %ld] about to spawn!\n", (long)pid);
+        printf("Parent [pid %ld] about to spawn!\n", (long) pid);
         MPI_Info_create(&info);
-        MPI_Info_set(info, "add-host", "rhc002:24");
-        if (MPI_SUCCESS != (rc = MPI_Comm_spawn(argv[0], MPI_ARGV_NULL, 3, info,
-                                                0, MPI_COMM_WORLD, &child, MPI_ERRCODES_IGNORE))) {
+        MPI_Info_set(info, "PMIX_ADD_HOST", argv[1]);
+        if (MPI_SUCCESS
+            != (rc = MPI_Comm_spawn(argv[0], MPI_ARGV_NULL, 3, info, 0, MPI_COMM_WORLD, &child,
+                                    MPI_ERRCODES_IGNORE))) {
             printf("Child failed to spawn\n");
             return rc;
         }
@@ -49,10 +55,13 @@ int main(int argc, char* argv[])
         }
         MPI_Comm_disconnect(&child);
         printf("Parent disconnected\n");
+        sleep(2);
+        printf("\n\n\n");
         /* do it again */
-        MPI_Info_set(info, "add-host", "rhc003:24");
-        if (MPI_SUCCESS != (rc = MPI_Comm_spawn(argv[0], MPI_ARGV_NULL, 3, info,
-                                                0, MPI_COMM_WORLD, &child, MPI_ERRCODES_IGNORE))) {
+        MPI_Info_set(info, "PMIX_ADD_HOST", argv[2]);
+        if (MPI_SUCCESS
+            != (rc = MPI_Comm_spawn(argv[0], MPI_ARGV_NULL, 3, info, 0, MPI_COMM_WORLD, &child,
+                                    MPI_ERRCODES_IGNORE))) {
             printf("Child failed to spawn\n");
             return rc;
         }
@@ -70,7 +79,7 @@ int main(int argc, char* argv[])
         MPI_Comm_rank(MPI_COMM_WORLD, &rank);
         MPI_Comm_size(MPI_COMM_WORLD, &size);
         pid = getpid();
-        printf("Hello from the child %d of %d on host %s pid %ld\n", rank, 3, hostname, (long)pid);
+        printf("Hello from the child %d of %d on host %s pid %ld\n", rank, 3, hostname, (long) pid);
         if (0 == rank) {
             MPI_Recv(&msg, 1, MPI_INT, 0, 1, parent, MPI_STATUS_IGNORE);
             printf("Child %d received msg: %d\n", rank, msg);
diff --git a/test/simple/attach.c b/test/simple/attach.c
index adbe00e2deb..589736f986b 100644
--- a/test/simple/attach.c
+++ b/test/simple/attach.c
@@ -5,14 +5,14 @@
  * The most basic of MPI applications
  */
 
+#include <fcntl.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <sys/types.h>
 #include <sys/stat.h>
-#include <fcntl.h>
+#include <sys/types.h>
 #include <unistd.h>
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     unsigned char fifo_cmd = 1;
     int fd;
diff --git a/test/simple/bad_arg.c b/test/simple/bad_arg.c
index 282b97794a5..616f100bbd4 100644
--- a/test/simple/bad_arg.c
+++ b/test/simple/bad_arg.c
@@ -9,22 +9,22 @@
  * $HEADER$
  */
 
+#include "mpi.h"
 #include <stdio.h>
 #include <unistd.h>
-#include "mpi.h"
 
-void errhandler(MPI_Comm* comm, int* err) {
+void errhandler(MPI_Comm *comm, int *err)
+{
     int rank, len = 0;
     char errstr[MPI_MAX_ERROR_STRING] = {0};
     char cname[MPI_MAX_OBJECT_NAME] = {0};
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Error_string(*err, errstr, &len);
     MPI_Comm_get_name(*comm, cname, &len);
-    fprintf(stderr, "Rank %02d: The error handler for %s has been invoked with error %d: %s\n\n", rank, cname, *err, errstr);
+    fprintf(stderr, "Rank %02d: The error handler for %s has been invoked with error %d: %s\n\n",
+            rank, cname, *err, errstr);
 }
 
-
-
 int main(int argc, char *argv[])
 {
     int rank, size, i = 5;
@@ -38,34 +38,37 @@ int main(int argc, char *argv[])
     MPI_Comm_set_errhandler(MPI_COMM_WORLD, errh);
     MPI_Comm_set_errhandler(MPI_COMM_SELF, errh);
 
-    if(0 == rank) {
-        fprintf(stderr,
-"This test will have rank 1 pass an invalid argument to a local MPI call.\n"
-"An error handler has been set, so:\n"
-"  * A high quality implementation should refrain from calling MPI_ERRORS_ARE_FATAL.\n"
-"  * An MPI-4 compliant implementation should trigger the MPI_COMM_SELF error handler.\n"
-"  * An MPI-3 compliant implementation should trigger the MPI_COMM_WORLD error handler.\n\n");
+    if (0 == rank) {
+        fprintf(
+            stderr,
+            "This test will have rank 1 pass an invalid argument to a local MPI call.\n"
+            "An error handler has been set, so:\n"
+            "  * A high quality implementation should refrain from calling MPI_ERRORS_ARE_FATAL.\n"
+            "  * An MPI-4 compliant implementation should trigger the MPI_COMM_SELF error "
+            "handler.\n"
+            "  * An MPI-3 compliant implementation should trigger the MPI_COMM_WORLD error "
+            "handler.\n\n");
     }
 
     sleep(1);
 
-    if(1 == rank) {
+    if (1 == rank) {
         MPI_Type_set_name(MPI_DATATYPE_NULL, "bad type argument");
-    }
-    else {
+    } else {
         MPI_Type_set_name(MPI_INT, "good type argument");
     }
     /* The reported error was not 'important' so we expect MPI to remain
      * operative */
 
     /* progress for some time to see if some error handler gets called */
-    while(i-- > 0) {
+    while (i-- > 0) {
         MPI_Sendrecv_replace(&size, 1, MPI_INT, 0, 0, 0, 0, MPI_COMM_SELF, MPI_STATUS_IGNORE);
         sleep(1);
-        if(0 == rank) fprintf(stderr, "\rWaiting for %2ds", i);
+        if (0 == rank)
+            fprintf(stderr, "\rWaiting for %2ds", i);
     }
-    if(0 == rank) fprintf(stderr, "\n");
-
+    if (0 == rank)
+        fprintf(stderr, "\n");
 
     MPI_Barrier(MPI_COMM_WORLD);
     fprintf(stderr, "Rank %02d: I have survived till the end of the test.\n", rank);
@@ -73,4 +76,3 @@ int main(int argc, char *argv[])
     MPI_Finalize();
     return 0;
 }
-
diff --git a/test/simple/bad_exit.c b/test/simple/bad_exit.c
index c319a0ae154..55536714e39 100644
--- a/test/simple/bad_exit.c
+++ b/test/simple/bad_exit.c
@@ -1,25 +1,24 @@
+#include <mpi.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
-#include <mpi.h>
 
 #define RANK_DEATH 1
 
 int main(int argc, char **argv)
 {
-   int rank;
-   MPI_Init(&argc,&argv);
-   MPI_Comm_rank(MPI_COMM_WORLD,&rank);
+    int rank;
+    MPI_Init(&argc, &argv);
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
 
-   sleep(2);
-   if (rank==RANK_DEATH) {
-       printf("Rank %d exiting without calling finalize...\n", rank);
-       exit(1);
-   }
-   sleep(2);
-   printf("Rank %d calling MPI_Finalize\n", rank);
-   MPI_Finalize();
-   printf("Rank %d exiting\n", rank);
-   return 0;
+    sleep(2);
+    if (rank == RANK_DEATH) {
+        printf("Rank %d exiting without calling finalize...\n", rank);
+        exit(1);
+    }
+    sleep(2);
+    printf("Rank %d calling MPI_Finalize\n", rank);
+    MPI_Finalize();
+    printf("Rank %d exiting\n", rank);
+    return 0;
 }
-
diff --git a/test/simple/badcoll.c b/test/simple/badcoll.c
index 0b4bfdc22a0..a6210e6cd40 100644
--- a/test/simple/badcoll.c
+++ b/test/simple/badcoll.c
@@ -1,6 +1,6 @@
+#include "mpi.h"
 #include <stdio.h>
 #include <unistd.h>
-#include "mpi.h"
 
 const int count = 1234;
 int buffer[1234] = {0};
@@ -13,8 +13,7 @@ int main(int argc, char *argv[])
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &size);
 
-
-    for (i=0; i < 1000; i++) {
+    for (i = 0; i < 1000; i++) {
         fprintf(stderr, "%d: Executing Bcast #%d\n", rank, i);
         MPI_Bcast(buffer, count, MPI_INT, 0, MPI_COMM_WORLD);
         if (0 != rank) {
@@ -25,4 +24,3 @@ int main(int argc, char *argv[])
     MPI_Finalize();
     return 0;
 }
-
diff --git a/test/simple/bcast_loop.c b/test/simple/bcast_loop.c
index 39f7c49b2df..70dd286c813 100644
--- a/test/simple/bcast_loop.c
+++ b/test/simple/bcast_loop.c
@@ -1,7 +1,7 @@
 #include <mpi.h>
 #include <stdio.h>
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int myid, nprocs, tag;
     int i, m, nt;
@@ -33,4 +33,3 @@ int main(int argc, char* argv[])
 
     return 0;
 }
-
diff --git a/test/simple/binding.c b/test/simple/binding.c
index 36456616e14..eaa4ce3d5c7 100644
--- a/test/simple/binding.c
+++ b/test/simple/binding.c
@@ -6,17 +6,17 @@
  */
 
 #define _GNU_SOURCE
+#include "mpi.h"
+#include "opal/mca/hwloc/base/base.h"
+#include "opal/runtime/opal.h"
+#include <sched.h>
 #include <stdio.h>
 #include <sys/types.h>
 #include <unistd.h>
-#include <sched.h>
-#include "opal/mca/hwloc/base/base.h"
-#include "opal/runtime/opal.h"
-#include "mpi.h"
 
 #include "orte/util/proc_info.h"
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int rank, size, rc;
     hwloc_cpuset_t cpus;
@@ -39,20 +39,20 @@ int main(int argc, char* argv[])
     }
 
     printf("[%s;%d] Hello, World, I am %d of %d [%d local peers]: get_cpubind: %d bitmap %s\n",
-           hostname, (int)getpid(), rank, size, orte_process_info.num_local_peers, rc,
+           hostname, (int) getpid(), rank, size, orte_process_info.num_local_peers, rc,
            (NULL == bindings) ? "NULL" : bindings);
 
     nrcpus = sysconf(_SC_NPROCESSORS_ONLN);
     mask = CPU_ALLOC(nrcpus);
     csize = CPU_ALLOC_SIZE(nrcpus);
     CPU_ZERO_S(csize, mask);
-    if ( sched_getaffinity(0, csize, mask) == -1 ) {
+    if (sched_getaffinity(0, csize, mask) == -1) {
         perror("sched_getaffinity");
     } else {
-        for ( c = 0; c < nrcpus; c++ ) {
-                if ( CPU_ISSET_S(c, csize, mask) ) {
-                        printf("[%s:%d] CPU %d is set\n", hostname, (int)getpid(), c);
-                }
+        for (c = 0; c < nrcpus; c++) {
+            if (CPU_ISSET_S(c, csize, mask)) {
+                printf("[%s:%d] CPU %d is set\n", hostname, (int) getpid(), c);
+            }
         }
     }
 
diff --git a/test/simple/client.c b/test/simple/client.c
index c79ebc9def8..daf3d831a06 100644
--- a/test/simple/client.c
+++ b/test/simple/client.c
@@ -1,37 +1,38 @@
+#include "mpi.h"
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include "mpi.h"
 
 #define MAX_DATA 100
 
-int main( int argc, char **argv )
+int main(int argc, char **argv)
 {
- MPI_Comm server;
- double buf[MAX_DATA];
- char port_name[MPI_MAX_PORT_NAME];
- int done = 0, tag, n, CNT=0;
-
- MPI_Init( &argc, &argv );
- strcpy(port_name, argv[1] );  /* assume server's name is cmd-line arg */
-
- MPI_Comm_connect( port_name, MPI_INFO_NULL, 0, MPI_COMM_WORLD, &server );
-
- n = MAX_DATA;
-
- while (!done)
-   {
-     tag = 2; /* Action to perform */
-     if ( CNT == 5 ) { tag = 0; done = 1; }
-     fprintf(stderr, "Client sending message %d\n", CNT);
-     MPI_Send( buf, n, MPI_DOUBLE, 0, tag, server );
-     CNT++;
-     /* etc */
-   }
-
- MPI_Comm_disconnect( &server );
- MPI_Finalize();
-
- return 0;
+    MPI_Comm server;
+    double buf[MAX_DATA];
+    char port_name[MPI_MAX_PORT_NAME];
+    int done = 0, tag, n, CNT = 0;
+
+    MPI_Init(&argc, &argv);
+    strcpy(port_name, argv[1]); /* assume server's name is cmd-line arg */
+
+    MPI_Comm_connect(port_name, MPI_INFO_NULL, 0, MPI_COMM_WORLD, &server);
+
+    n = MAX_DATA;
+
+    while (!done) {
+        tag = 2; /* Action to perform */
+        if (CNT == 5) {
+            tag = 0;
+            done = 1;
+        }
+        fprintf(stderr, "Client sending message %d\n", CNT);
+        MPI_Send(buf, n, MPI_DOUBLE, 0, tag, server);
+        CNT++;
+        /* etc */
+    }
+
+    MPI_Comm_disconnect(&server);
+    MPI_Finalize();
+
+    return 0;
 }
-
diff --git a/test/simple/comm_abort.c b/test/simple/comm_abort.c
index 7acb8fad4b4..d42a721b6ae 100644
--- a/test/simple/comm_abort.c
+++ b/test/simple/comm_abort.c
@@ -11,14 +11,15 @@
  * Test aborting communicators
  */
 
+#include "mpi.h"
 #include <stdio.h>
 #include <unistd.h>
-#include "mpi.h"
 
-#define print1(format...) if(0 == rank) printf(format)
+#define print1(format...) \
+    if (0 == rank)        \
+    printf(format)
 
-
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int rank, size, more;
     double start, now;
@@ -28,72 +29,76 @@ int main(int argc, char* argv[])
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &size);
 
-    if(0 == rank && size%2) {
+    if (0 == rank && size % 2) {
         fprintf(stderr, "This test requires an even number of processes\n\n");
         MPI_Abort(MPI_COMM_WORLD, size);
     }
 
     /* Setup: split our world in a set of 2-processes islands */
-    MPI_Comm_split(MPI_COMM_WORLD, rank/2, rank, &comm_pair_fatal);
+    MPI_Comm_split(MPI_COMM_WORLD, rank / 2, rank, &comm_pair_fatal);
     MPI_Comm_set_errhandler(comm_pair_fatal, MPI_ERRORS_ARE_FATAL);
-    MPI_Comm_split(MPI_COMM_WORLD, rank/2, rank, &comm_pair_return);
+    MPI_Comm_split(MPI_COMM_WORLD, rank / 2, rank, &comm_pair_return);
     MPI_Comm_set_errhandler(comm_pair_return, MPI_ERRORS_RETURN);
-    MPI_Comm_split(MPI_COMM_WORLD, rank/2, rank, &comm_pair_abort);
-    /* If this code fails to compile, the MPI implementation is not compliant 
+    MPI_Comm_split(MPI_COMM_WORLD, rank / 2, rank, &comm_pair_abort);
+    /* If this code fails to compile, the MPI implementation is not compliant
      * with MPI-4 (TODO: add ref to chapter/line when MPI-4 published). */
     MPI_Comm_set_errhandler(comm_pair_abort, MPI_ERRORS_ABORT);
     MPI_Barrier(MPI_COMM_WORLD);
 
-    print1(
-"This program will test partial abort functionality (communicator scoped abort).\n"
-"   Each test will perform a loop of communication on a subcommunicator for about\n"
-"   1 second between printouts, and then, a 1 second cooldown.\n");
+    print1("This program will test partial abort functionality (communicator scoped abort).\n"
+           "   Each test will perform a loop of communication on a subcommunicator for about\n"
+           "   1 second between printouts, and then, a 1 second cooldown.\n");
 
     print1("\n\n"
-"Test1: MPI_Abort(MPI_COMM_SELF) aborts only one process?\n"
-"       In a high quality implementation, all ranks except %d\n"
-"       should report their presence.\n", 1);
-    if(rank == 1) {
+           "Test1: MPI_Abort(MPI_COMM_SELF) aborts only one process?\n"
+           "       In a high quality implementation, all ranks except %d\n"
+           "       should report their presence.\n",
+           1);
+    if (rank == 1) {
         MPI_Abort(MPI_COMM_SELF, 1);
     }
     /* Spin on communication for 1 second to let time for Abort to have an
      * effect, if any. */
-    more = 1; start = MPI_Wtime();
+    more = 1;
+    start = MPI_Wtime();
     do {
         now = MPI_Wtime();
-        if(now - start > 1.) more = 0;
-        if(rank > 1) /* don't reduce on aborted pairs */
+        if (now - start > 1.)
+            more = 0;
+        if (rank > 1) /* don't reduce on aborted pairs */
             MPI_Allreduce(MPI_IN_PLACE, &more, 1, MPI_INT, MPI_MIN, comm_pair_fatal);
-    } while(more);
+    } while (more);
     printf("  This is rank %d: still kickin after %d MPI_Abort'ed self\n", rank, 1);
 
     sleep(1);
     print1("===============================================================\n");
 
     print1("\n\n"
-"Test2: MPI_Abort(comm) aborts all processes in comm?\n"
-"       In a high quality implementation, all ranks except %d--%d\n"
-"       should report their presence.\n", 1, 3);
-    if(rank == 3) {
+           "Test2: MPI_Abort(comm) aborts all processes in comm?\n"
+           "       In a high quality implementation, all ranks except %d--%d\n"
+           "       should report their presence.\n",
+           1, 3);
+    if (rank == 3) {
         MPI_Abort(comm_pair_return, 2);
     }
     /* Spin on communication for 1 second to let time for Abort to have an
      * effect, if any. */
-    more = 1; start = MPI_Wtime();
+    more = 1;
+    start = MPI_Wtime();
     do {
         now = MPI_Wtime();
-        if(now - start > 1.) more = 0;
-        if(rank > 3) /* don't reduce on aborted pairs */
+        if (now - start > 1.)
+            more = 0;
+        if (rank > 3) /* don't reduce on aborted pairs */
             MPI_Allreduce(MPI_IN_PLACE, &more, 1, MPI_INT, MPI_MIN, comm_pair_fatal);
-    } while(more);
+    } while (more);
     printf("  This is rank %d: still kickin after %d aborted comm pair %d-%d\n", rank, 3, 2, 3);
 
     /* This process should have aborted, give it an opportunity to do so if no
      * async progress: message to self to spin MPI progress. */
-    if(rank == 2) {
-        MPI_Sendrecv(&start, 1, MPI_DOUBLE, 0, 0,
-                     &now, 1, MPI_DOUBLE, 0, 0, 
-                     MPI_COMM_SELF, MPI_STATUS_IGNORE);
+    if (rank == 2) {
+        MPI_Sendrecv(&start, 1, MPI_DOUBLE, 0, 0, &now, 1, MPI_DOUBLE, 0, 0, MPI_COMM_SELF,
+                     MPI_STATUS_IGNORE);
         printf("  This is rank %d: ERROR: I SHOULD HAVE ABORTED!\n", 2);
     }
 
@@ -101,70 +106,75 @@ int main(int argc, char* argv[])
     print1("===============================================================\n");
 
     print1("\n\n"
-"Test3: MPI_ERRORS_ABORT aborts all processes in comm?\n"
-"       In a high quality implementation, all ranks except %d--%d\n"
-"       should report their presence.\n", 1, 5);
-    if(rank == 5) {
+           "Test3: MPI_ERRORS_ABORT aborts all processes in comm?\n"
+           "       In a high quality implementation, all ranks except %d--%d\n"
+           "       should report their presence.\n",
+           1, 5);
+    if (rank == 5) {
         MPI_Comm_call_errhandler(comm_pair_abort, 3);
     }
     /* Spin on communication for 1 second to let time for Abort to have an
      * effect, if any. */
-    more = 1; start = MPI_Wtime();
+    more = 1;
+    start = MPI_Wtime();
     do {
         now = MPI_Wtime();
-        if(now - start > 1.) more = 0;
-        if(rank > 5) /* don't reduce on aborted pairs */
+        if (now - start > 1.)
+            more = 0;
+        if (rank > 5) /* don't reduce on aborted pairs */
             MPI_Allreduce(MPI_IN_PLACE, &more, 1, MPI_INT, MPI_MIN, comm_pair_fatal);
-    } while(more);
+    } while (more);
     printf("  This is rank %d: still kickin after %d aborted comm pair %d-%d\n", rank, 5, 4, 5);
 
     /* This process should have aborted, give it an opportunity to do so if no
      * async progress: message to self to spin MPI progress. */
-    if(rank == 4) {
-        MPI_Sendrecv(&start, 1, MPI_DOUBLE, 0, 0,
-                     &now, 1, MPI_DOUBLE, 0, 0, 
-                     MPI_COMM_SELF, MPI_STATUS_IGNORE);
+    if (rank == 4) {
+        MPI_Sendrecv(&start, 1, MPI_DOUBLE, 0, 0, &now, 1, MPI_DOUBLE, 0, 0, MPI_COMM_SELF,
+                     MPI_STATUS_IGNORE);
         printf("  This is rank %d: ERROR: I SHOULD HAVE ABORTED!\n", 4);
     }
 
     sleep(1);
     print1("===============================================================\n");
 
-    print1("\n\n"\
-"Test4: Communicating with an aborted process %d returns a good error code?\n"
-"       In a high quality implementation, rank %d should print an error string;\n"
-"       In a higher quality implementation the error should be of class\n"
-"       MPI_ERR_PROC_ABORTED.\n", 1, 0);
-    if(rank == 0) {
+    print1("\n\n"
+           "Test4: Communicating with an aborted process %d returns a good error code?\n"
+           "       In a high quality implementation, rank %d should print an error string;\n"
+           "       In a higher quality implementation the error should be of class\n"
+           "       MPI_ERR_PROC_ABORTED.\n",
+           1, 0);
+    if (rank == 0) {
         int err, class, slen;
         char str[MPI_MAX_ERROR_STRING];
         /* remember, 1 aborted in test1 */
         MPI_Error_class(err, &class);
         MPI_Error_string(err, str, &slen);
         err = MPI_Recv(&more, 1, MPI_INT, 1, 0, comm_pair_return, MPI_STATUS_IGNORE);
-        printf("  This is rank %d: Recv(from=%d) returned code=%d: class=%d: %s\n", 0, 1, err, class, str);
+        printf("  This is rank %d: Recv(from=%d) returned code=%d: class=%d: %s\n", 0, 1, err,
+               class, str);
     }
 
     sleep(1);
     print1("===============================================================\n");
 
     print1("\n\n"
-"Test5: MPI_ERRORS_ARE_FATAL aborts all processes?\n");
-    if(rank == 0) {
+           "Test5: MPI_ERRORS_ARE_FATAL aborts all processes?\n");
+    if (rank == 0) {
         MPI_Comm_call_errhandler(comm_pair_fatal, 5);
     }
     /* Spin on communication for 1 second to let time for Abort to have an
      * effect, if any. */
-    more = 1; start = MPI_Wtime();
+    more = 1;
+    start = MPI_Wtime();
     do {
         now = MPI_Wtime();
-        if(now - start > 1.) more = 0;
-        if(rank > 5) /* don't reduce on aborted pairs */
+        if (now - start > 1.)
+            more = 0;
+        if (rank > 5) /* don't reduce on aborted pairs */
             MPI_Allreduce(MPI_IN_PLACE, &more, 1, MPI_INT, MPI_MIN, comm_pair_fatal);
-    } while(more);
-    MPI_Sendrecv(&start, 1, MPI_DOUBLE, 0, 0,
-                 &now, 1, MPI_DOUBLE, 0, 0, 
-                 MPI_COMM_SELF, MPI_STATUS_IGNORE);
+    } while (more);
+    MPI_Sendrecv(&start, 1, MPI_DOUBLE, 0, 0, &now, 1, MPI_DOUBLE, 0, 0, MPI_COMM_SELF,
+                 MPI_STATUS_IGNORE);
     printf("  This is rank %d: ERROR: I SHOULD HAVE ABORTED!\n", rank);
 
     /* Should never get there */
diff --git a/test/simple/concurrent_spawn.c b/test/simple/concurrent_spawn.c
index 3c2eefbc7d6..25cfd92caf9 100644
--- a/test/simple/concurrent_spawn.c
+++ b/test/simple/concurrent_spawn.c
@@ -4,22 +4,22 @@
 #include "opal/runtime/opal.h"
 
 #include <stdio.h>
+#include <stdlib.h>
 #include <sys/types.h>
 #include <unistd.h>
-#include <stdlib.h>
 
 #include <mpi.h>
 
 #define NUM_CHILDREN 5
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int msg;
     MPI_Comm parent, children[NUM_CHILDREN];
     int rank, size, i;
     const char *hostname;
     pid_t pid;
-    char *child_argv[2] = { "", NULL };
+    char *child_argv[2] = {"", NULL};
 
     MPI_Init(NULL, NULL);
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
@@ -32,11 +32,9 @@ int main(int argc, char* argv[])
         /* First, spawn all the children.  Give them an argv
            identifying which child they are */
         for (i = 0; i < NUM_CHILDREN; ++i) {
-            printf("Parent [pid %ld] about to spawn child #%d\n",
-                   (long)pid, i);
+            printf("Parent [pid %ld] about to spawn child #%d\n", (long) pid, i);
             opal_asprintf(&(child_argv[0]), "%d", i);
-            MPI_Comm_spawn(argv[0], child_argv, 1, MPI_INFO_NULL,
-                           0, MPI_COMM_WORLD, &children[i],
+            MPI_Comm_spawn(argv[0], child_argv, 1, MPI_INFO_NULL, 0, MPI_COMM_WORLD, &children[i],
                            MPI_ERRCODES_IGNORE);
             printf("Parent done with spawn of child %d\n", i);
         }
@@ -60,19 +58,18 @@ int main(int argc, char* argv[])
     else {
         hostname = opal_gethostname();
         if (argc == 1) {
-            printf("ERROR: child did not receive exepcted argv!\n");
+            printf("ERROR: child did not receive expected argv!\n");
             i = -1;
         } else {
             i = atoi(argv[1]);
         }
         pid = getpid();
-        printf("Hello from the child %d on host %s pid %ld\n", i, hostname, (long)pid);
+        printf("Hello from the child %d on host %s pid %ld\n", i, hostname, (long) pid);
         if (0 == rank) {
             MPI_Recv(&msg, 1, MPI_INT, 0, 1, parent, MPI_STATUS_IGNORE);
             printf("Child %d received msg: %d\n", i, msg);
             if (i != msg) {
-                printf("ERROR: Child %d got wrong message (got %d, expected %d)\n",
-                       i, msg, i);
+                printf("ERROR: Child %d got wrong message (got %d, expected %d)\n", i, msg, i);
             }
         }
         MPI_Comm_disconnect(&parent);
diff --git a/test/simple/connect.c b/test/simple/connect.c
index 2de897eb40f..ed9bb47bc47 100644
--- a/test/simple/connect.c
+++ b/test/simple/connect.c
@@ -5,10 +5,10 @@
  * Test of connect/accept - the accept (server) side
  */
 
-#include <stdio.h>
 #include "mpi.h"
+#include <stdio.h>
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int rank, size;
     MPI_Comm server;
diff --git a/test/simple/crisscross.c b/test/simple/crisscross.c
index 993b15423fc..0f2f544ebb0 100644
--- a/test/simple/crisscross.c
+++ b/test/simple/crisscross.c
@@ -2,272 +2,252 @@
   cc -o crisscross crisscross.c -lmpi
 */
 
+#include "mpi.h"
 #include <stdio.h>
 #include <stdlib.h>
-#include "mpi.h"
-#include <unistd.h>
 #include <string.h>
 #include <time.h>
+#include <unistd.h>
 
 #define MAX_RR_NAME 7
 
 int main(int argc, char *argv[])
 {
- MPI_Status     status;               /* MPI status                          */
- int            mpierr;               /* MPI function return code            */
- int            rank;                 /* Process rank within MPI_COMM_WORLD  */
- int            nproc;                /* Total number of MPI processes       */
- int            tag0=41;              /* MPI message tag                     */
- int            tag1=42;              /* MPI message tag                     */
- int            tag2=43;              /* MPI message tag                     */
- int            warmup=1;             /* MPI warmup loops                    */
-
- char           process_name[MPI_MAX_PROCESSOR_NAME + 1];
- char           partner_name[MPI_MAX_PROCESSOR_NAME + 1];
-
- char           rr_blank[] = {"       "};
- char           rr_empty[] = {"???????"};
-
- int            n_bytes=128*1024*1024;
- int            n_loops=2;
- unsigned char* send_buff;
- unsigned char* recv_buff;
-
- int            i,j,k,m,count,mismatch;
-
- double         et1,et2,mbs;
- double         avg_mbs=0, sum_avg_mbs=0;
- int            xfers=0, sum_xfers=0;
- double         max_mbs=-1.0,min_mbs=999999.9;
- double         r_max_mbs,r_min_mbs;
-
- time_t         curtime;
- struct tm     *loctime;
-
- if ( argc > 2 )
-   {
-    n_loops = atoi(argv[2]);
-    n_loops = n_loops < 1 ? 10 : n_loops;
-   }
- if ( argc > 1 )
-   {
-    n_bytes = atoi(argv[1]);
-    n_bytes = n_bytes < 1 ? 32768 : n_bytes;
-   }
-
- send_buff = (unsigned char *) valloc(n_bytes);
- recv_buff = (unsigned char *) valloc(n_bytes);
-
- for ( i=0; i<n_bytes; i++ )
-   {
-    send_buff[i] = i%128;
-   }
-
- mpierr = MPI_Init(&argc, &argv);
- if (mpierr != MPI_SUCCESS)
-   {
-    fprintf(stderr, "MPI Error %d (MPI_Init)\n",mpierr);
-    fflush(stderr);
-    MPI_Abort(MPI_COMM_WORLD, -1);
-   }
-
- MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN);
-
- mpierr = MPI_Comm_rank(MPI_COMM_WORLD, &rank);
- if (mpierr != MPI_SUCCESS || rank < 0)
-   {
-    fprintf(stderr, "MPI Error %d (MPI_Comm_rank)\n",mpierr);
-    fflush(stderr);
-    MPI_Abort(MPI_COMM_WORLD, -1);
-   }
-
- if ( rank == 0 )
-   {
-    curtime = time (NULL);
-    loctime = localtime (&curtime);
-    printf("\n   %s\n",asctime (loctime));
-   }
-
- mpierr = MPI_Comm_size(MPI_COMM_WORLD, &nproc);
- if (mpierr != MPI_SUCCESS || nproc < 1 || nproc <= rank)
-   {
-    fprintf(stderr, "MPI Error %d (MPI_Comm_size) [%d]\n",mpierr, rank);
-    fflush(stderr);
-    MPI_Abort(MPI_COMM_WORLD, -1);
-   }
-
- mpierr = MPI_Get_processor_name(process_name, &count);
- if (mpierr != MPI_SUCCESS)
-   {
-    fprintf(stderr,"MPI Error %d (MPI_Get_processor_name) [%d]\n", mpierr, rank);
-    sprintf(process_name, "%s", rr_empty);
-   }
- else
-   {
-    if (count < MAX_RR_NAME) strncat(&process_name[count],rr_blank,MAX_RR_NAME-count);
-    process_name[MAX_RR_NAME] = '\0';
-   }
-
- for ( i=0; i<nproc; i++ )
-   {
-    mpierr = MPI_Barrier(MPI_COMM_WORLD);
-    if (mpierr != MPI_SUCCESS)
-      {
-       fprintf(stderr, "MPI Error %d (MPI_Barrier) [%d]\n", mpierr, rank);
-       fflush(stderr);
-       MPI_Abort(MPI_COMM_WORLD, -1);
-      }
-    for ( j=0; j<nproc; j++ )
-      {
-       if ( i != j )
-         {
-          if (rank == j)
-            {
-             mpierr = MPI_Sendrecv(process_name, MPI_MAX_PROCESSOR_NAME + 1, MPI_CHAR, i, tag0,
-                                   partner_name, MPI_MAX_PROCESSOR_NAME + 1, MPI_CHAR, i, tag0, MPI_COMM_WORLD, &status);
-             if (mpierr != MPI_SUCCESS)
-               {
-                fprintf(stderr,"MPI Error %d (MPI_Sendrecv) %s [%d,%d]\n",mpierr,process_name,rank,i);
-                fflush(stderr);
-                MPI_Abort(MPI_COMM_WORLD, -1);
-               }
-             for ( k=0; k<n_bytes; k++ )
-               {
-                recv_buff[k] = 0x80;
-               }
-            }
-          if ( rank == i )
-            {
-             mpierr = MPI_Sendrecv(process_name, MPI_MAX_PROCESSOR_NAME + 1, MPI_CHAR, j, tag0,
-                                   partner_name, MPI_MAX_PROCESSOR_NAME + 1, MPI_CHAR, j, tag0, MPI_COMM_WORLD, &status);
-             if (mpierr != MPI_SUCCESS)
-               {
-                fprintf(stderr,"MPI Error %d (MPI_Sendrecv) %s [%d,%d]\n",mpierr,process_name,i,j);
-                fflush(stderr);
-                MPI_Abort(MPI_COMM_WORLD, -1);
-               }
-            }
-          for ( k=0; k<n_loops+warmup; k++ )
-            {
-             if ( rank == i )
-               {
-                if (k == warmup) et1 = MPI_Wtime();
-                mpierr = MPI_Send(send_buff, n_bytes, MPI_BYTE, j, tag1, MPI_COMM_WORLD);
-                if (mpierr != MPI_SUCCESS)
-                  {
-                   fprintf(stderr,"MPI Error %d (MPI_Send) %s [4%d] --> %s [4%d]\n",mpierr,process_name,i,partner_name,j);
-                   fflush(stderr);
-                   MPI_Abort(MPI_COMM_WORLD, -1);
-                  }
-               }
-             if ( rank == j )
-               {
-                mpierr = MPI_Recv(recv_buff, n_bytes, MPI_BYTE, i, tag1, MPI_COMM_WORLD, &status);
-                if (mpierr != MPI_SUCCESS)
-                  {
-                   fprintf(stderr,"MPI Error %d (MPI_Recv) %s [4%d] <-- %s [4%d]\n",mpierr,process_name,j,partner_name,i);
-                   fflush(stderr);
-                   MPI_Abort(MPI_COMM_WORLD, -1);
-                  }
-                if (k == n_loops+warmup-1)  et2 = MPI_Wtime();
-               }
-            }
-          if ( rank == i )
-            {
-             mpierr = MPI_Send(&et1, 1, MPI_DOUBLE, j, tag1, MPI_COMM_WORLD);
-             if (mpierr != MPI_SUCCESS)
-               {
-                fprintf(stderr,"MPI Error %d (MPI_Send) %s [4%d] --> %s [4%d]\n",mpierr,process_name,i,partner_name,j);
-                fflush(stderr);
-                MPI_Abort(MPI_COMM_WORLD, -1);
-               }
-            }
-          if ( rank == j )
-            {
-             mpierr = MPI_Recv(&et1, 1, MPI_DOUBLE, i, tag1, MPI_COMM_WORLD, &status);
-             if (mpierr != MPI_SUCCESS)
-               {
-                fprintf(stderr,"MPI Error %d (MPI_Recv) %s [4%d] <-- %s [4%d]\n",mpierr,process_name,j,partner_name,i);
-                fflush(stderr);
-                MPI_Abort(MPI_COMM_WORLD, -1);
-               }
-             mbs = ((double)n_loops*n_bytes)/(1000000.0*(et2-et1));
-             if (mbs < 50.0)
-               {
-                printf("   %s [%4d]   =====>>   %s [%4d]  %9.1f mbs     SLOW!\n",partner_name,i,process_name,j,mbs);
-               }
-             else
-               {
-                printf("   %s [%4d]   =====>>   %s [%4d]  %9.1f mbs\n",partner_name,i,process_name,j,mbs);
-               }
-
-             min_mbs = (mbs < min_mbs) ? mbs:min_mbs;
-             max_mbs = (mbs > max_mbs) ? mbs:max_mbs;
-
-             avg_mbs += mbs;
-             xfers++;
-             mismatch = 0;
-             for ( k=0; k<n_bytes; k++ )
-               {
-                if ( recv_buff[k] != k%128 ) mismatch++;
-               }
-             if ( mismatch ) printf("                                                                  WARNING! %d data mismatches\n",mismatch);
-             fflush(stdout);
+    MPI_Status status; /* MPI status                          */
+    int mpierr;        /* MPI function return code            */
+    int rank;          /* Process rank within MPI_COMM_WORLD  */
+    int nproc;         /* Total number of MPI processes       */
+    int tag0 = 41;     /* MPI message tag                     */
+    int tag1 = 42;     /* MPI message tag                     */
+    int tag2 = 43;     /* MPI message tag                     */
+    int warmup = 1;    /* MPI warmup loops                    */
+
+    char process_name[MPI_MAX_PROCESSOR_NAME + 1];
+    char partner_name[MPI_MAX_PROCESSOR_NAME + 1];
+
+    char rr_blank[] = {"       "};
+    char rr_empty[] = {"???????"};
+
+    int n_bytes = 128 * 1024 * 1024;
+    int n_loops = 2;
+    unsigned char *send_buff;
+    unsigned char *recv_buff;
+
+    int i, j, k, m, count, mismatch;
+
+    double et1, et2, mbs;
+    double avg_mbs = 0, sum_avg_mbs = 0;
+    int xfers = 0, sum_xfers = 0;
+    double max_mbs = -1.0, min_mbs = 999999.9;
+    double r_max_mbs, r_min_mbs;
+
+    time_t curtime;
+    struct tm *loctime;
+
+    if (argc > 2) {
+        n_loops = atoi(argv[2]);
+        n_loops = n_loops < 1 ? 10 : n_loops;
+    }
+    if (argc > 1) {
+        n_bytes = atoi(argv[1]);
+        n_bytes = n_bytes < 1 ? 32768 : n_bytes;
+    }
+
+    send_buff = (unsigned char *) valloc(n_bytes);
+    recv_buff = (unsigned char *) valloc(n_bytes);
+
+    for (i = 0; i < n_bytes; i++) {
+        send_buff[i] = i % 128;
+    }
+
+    mpierr = MPI_Init(&argc, &argv);
+    if (mpierr != MPI_SUCCESS) {
+        fprintf(stderr, "MPI Error %d (MPI_Init)\n", mpierr);
+        fflush(stderr);
+        MPI_Abort(MPI_COMM_WORLD, -1);
+    }
+
+    MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN);
+
+    mpierr = MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    if (mpierr != MPI_SUCCESS || rank < 0) {
+        fprintf(stderr, "MPI Error %d (MPI_Comm_rank)\n", mpierr);
+        fflush(stderr);
+        MPI_Abort(MPI_COMM_WORLD, -1);
+    }
+
+    if (rank == 0) {
+        curtime = time(NULL);
+        loctime = localtime(&curtime);
+        printf("\n   %s\n", asctime(loctime));
+    }
+
+    mpierr = MPI_Comm_size(MPI_COMM_WORLD, &nproc);
+    if (mpierr != MPI_SUCCESS || nproc < 1 || nproc <= rank) {
+        fprintf(stderr, "MPI Error %d (MPI_Comm_size) [%d]\n", mpierr, rank);
+        fflush(stderr);
+        MPI_Abort(MPI_COMM_WORLD, -1);
+    }
+
+    mpierr = MPI_Get_processor_name(process_name, &count);
+    if (mpierr != MPI_SUCCESS) {
+        fprintf(stderr, "MPI Error %d (MPI_Get_processor_name) [%d]\n", mpierr, rank);
+        sprintf(process_name, "%s", rr_empty);
+    } else {
+        if (count < MAX_RR_NAME)
+            strncat(&process_name[count], rr_blank, MAX_RR_NAME - count);
+        process_name[MAX_RR_NAME] = '\0';
+    }
+
+    for (i = 0; i < nproc; i++) {
+        mpierr = MPI_Barrier(MPI_COMM_WORLD);
+        if (mpierr != MPI_SUCCESS) {
+            fprintf(stderr, "MPI Error %d (MPI_Barrier) [%d]\n", mpierr, rank);
+            fflush(stderr);
+            MPI_Abort(MPI_COMM_WORLD, -1);
+        }
+        for (j = 0; j < nproc; j++) {
+            if (i != j) {
+                if (rank == j) {
+                    mpierr = MPI_Sendrecv(process_name, MPI_MAX_PROCESSOR_NAME + 1, MPI_CHAR, i,
+                                          tag0, partner_name, MPI_MAX_PROCESSOR_NAME + 1, MPI_CHAR,
+                                          i, tag0, MPI_COMM_WORLD, &status);
+                    if (mpierr != MPI_SUCCESS) {
+                        fprintf(stderr, "MPI Error %d (MPI_Sendrecv) %s [%d,%d]\n", mpierr,
+                                process_name, rank, i);
+                        fflush(stderr);
+                        MPI_Abort(MPI_COMM_WORLD, -1);
+                    }
+                    for (k = 0; k < n_bytes; k++) {
+                        recv_buff[k] = 0x80;
+                    }
+                }
+                if (rank == i) {
+                    mpierr = MPI_Sendrecv(process_name, MPI_MAX_PROCESSOR_NAME + 1, MPI_CHAR, j,
+                                          tag0, partner_name, MPI_MAX_PROCESSOR_NAME + 1, MPI_CHAR,
+                                          j, tag0, MPI_COMM_WORLD, &status);
+                    if (mpierr != MPI_SUCCESS) {
+                        fprintf(stderr, "MPI Error %d (MPI_Sendrecv) %s [%d,%d]\n", mpierr,
+                                process_name, i, j);
+                        fflush(stderr);
+                        MPI_Abort(MPI_COMM_WORLD, -1);
+                    }
+                }
+                for (k = 0; k < n_loops + warmup; k++) {
+                    if (rank == i) {
+                        if (k == warmup)
+                            et1 = MPI_Wtime();
+                        mpierr = MPI_Send(send_buff, n_bytes, MPI_BYTE, j, tag1, MPI_COMM_WORLD);
+                        if (mpierr != MPI_SUCCESS) {
+                            fprintf(stderr, "MPI Error %d (MPI_Send) %s [4%d] --> %s [4%d]\n",
+                                    mpierr, process_name, i, partner_name, j);
+                            fflush(stderr);
+                            MPI_Abort(MPI_COMM_WORLD, -1);
+                        }
+                    }
+                    if (rank == j) {
+                        mpierr = MPI_Recv(recv_buff, n_bytes, MPI_BYTE, i, tag1, MPI_COMM_WORLD,
+                                          &status);
+                        if (mpierr != MPI_SUCCESS) {
+                            fprintf(stderr, "MPI Error %d (MPI_Recv) %s [4%d] <-- %s [4%d]\n",
+                                    mpierr, process_name, j, partner_name, i);
+                            fflush(stderr);
+                            MPI_Abort(MPI_COMM_WORLD, -1);
+                        }
+                        if (k == n_loops + warmup - 1)
+                            et2 = MPI_Wtime();
+                    }
+                }
+                if (rank == i) {
+                    mpierr = MPI_Send(&et1, 1, MPI_DOUBLE, j, tag1, MPI_COMM_WORLD);
+                    if (mpierr != MPI_SUCCESS) {
+                        fprintf(stderr, "MPI Error %d (MPI_Send) %s [4%d] --> %s [4%d]\n", mpierr,
+                                process_name, i, partner_name, j);
+                        fflush(stderr);
+                        MPI_Abort(MPI_COMM_WORLD, -1);
+                    }
+                }
+                if (rank == j) {
+                    mpierr = MPI_Recv(&et1, 1, MPI_DOUBLE, i, tag1, MPI_COMM_WORLD, &status);
+                    if (mpierr != MPI_SUCCESS) {
+                        fprintf(stderr, "MPI Error %d (MPI_Recv) %s [4%d] <-- %s [4%d]\n", mpierr,
+                                process_name, j, partner_name, i);
+                        fflush(stderr);
+                        MPI_Abort(MPI_COMM_WORLD, -1);
+                    }
+                    mbs = ((double) n_loops * n_bytes) / (1000000.0 * (et2 - et1));
+                    if (mbs < 50.0) {
+                        printf("   %s [%4d]   =====>>   %s [%4d]  %9.1f mbs     SLOW!\n",
+                               partner_name, i, process_name, j, mbs);
+                    } else {
+                        printf("   %s [%4d]   =====>>   %s [%4d]  %9.1f mbs\n", partner_name, i,
+                               process_name, j, mbs);
+                    }
+
+                    min_mbs = (mbs < min_mbs) ? mbs : min_mbs;
+                    max_mbs = (mbs > max_mbs) ? mbs : max_mbs;
+
+                    avg_mbs += mbs;
+                    xfers++;
+                    mismatch = 0;
+                    for (k = 0; k < n_bytes; k++) {
+                        if (recv_buff[k] != k % 128)
+                            mismatch++;
+                    }
+                    if (mismatch)
+                        printf("                                                                  "
+                               "WARNING! %d data mismatches\n",
+                               mismatch);
+                    fflush(stdout);
+                }
             }
-         }
-      }
-   }
-
- mpierr = MPI_Reduce(&xfers, &sum_xfers, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
- if (mpierr != MPI_SUCCESS)
-   {
-    fprintf(stderr,"MPI Error %d (MPI_Reduce) %s [%d]\n",mpierr,process_name,rank);
-    fflush(stderr);
-    MPI_Abort(MPI_COMM_WORLD, -1);
-   }
-
- mpierr = MPI_Reduce(&avg_mbs, &sum_avg_mbs, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
- if (mpierr != MPI_SUCCESS)
-   {
-    fprintf(stderr,"MPI Error %d (MPI_Reduce) %s [%d]\n",mpierr,process_name,rank);
-    fflush(stderr);
-    MPI_Abort(MPI_COMM_WORLD, -1);
-   }
-
- mpierr = MPI_Reduce(&min_mbs, &r_min_mbs, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
- if (mpierr != MPI_SUCCESS)
-   {
-    fprintf(stderr,"MPI Error %d (MPI_Reduce) %s [%d]\n",mpierr,process_name,rank);
-    fflush(stderr);
-    MPI_Abort(MPI_COMM_WORLD, -1);
-   }
+        }
+    }
+
+    mpierr = MPI_Reduce(&xfers, &sum_xfers, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
+    if (mpierr != MPI_SUCCESS) {
+        fprintf(stderr, "MPI Error %d (MPI_Reduce) %s [%d]\n", mpierr, process_name, rank);
+        fflush(stderr);
+        MPI_Abort(MPI_COMM_WORLD, -1);
+    }
+
+    mpierr = MPI_Reduce(&avg_mbs, &sum_avg_mbs, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+    if (mpierr != MPI_SUCCESS) {
+        fprintf(stderr, "MPI Error %d (MPI_Reduce) %s [%d]\n", mpierr, process_name, rank);
+        fflush(stderr);
+        MPI_Abort(MPI_COMM_WORLD, -1);
+    }
+
+    mpierr = MPI_Reduce(&min_mbs, &r_min_mbs, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
+    if (mpierr != MPI_SUCCESS) {
+        fprintf(stderr, "MPI Error %d (MPI_Reduce) %s [%d]\n", mpierr, process_name, rank);
+        fflush(stderr);
+        MPI_Abort(MPI_COMM_WORLD, -1);
+    }
+
+    mpierr = MPI_Reduce(&max_mbs, &r_max_mbs, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
+    if (mpierr != MPI_SUCCESS) {
+        fprintf(stderr, "MPI Error %d (MPI_Reduce) %s [%d]\n", mpierr, process_name, rank);
+        fflush(stderr);
+        MPI_Abort(MPI_COMM_WORLD, -1);
+    }
+
+    mpierr = MPI_Finalize();
+    if (mpierr != MPI_SUCCESS) {
+        fprintf(stderr, "MPI Error %d (MPI_Finalize) %s [%d]\n", mpierr, process_name, rank);
+        fflush(stderr);
+        MPI_Abort(MPI_COMM_WORLD, -1);
+    }
 
- mpierr = MPI_Reduce(&max_mbs, &r_max_mbs, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
- if (mpierr != MPI_SUCCESS)
-   {
-    fprintf(stderr,"MPI Error %d (MPI_Reduce) %s [%d]\n",mpierr,process_name,rank);
-    fflush(stderr);
-    MPI_Abort(MPI_COMM_WORLD, -1);
-   }
-
- mpierr = MPI_Finalize();
- if (mpierr != MPI_SUCCESS)
-   {
-    fprintf(stderr,"MPI Error %d (MPI_Finalize) %s [%d]\n",mpierr,process_name,rank);
-    fflush(stderr);
-    MPI_Abort(MPI_COMM_WORLD, -1);
-   }
-
- fflush(stdout);
-
- if ( rank == 0 )
-   {
-    mbs = sum_avg_mbs/sum_xfers;
-    printf("\n     average tranfer rate for %d transfers: %9.1f mbs\n",sum_xfers, mbs);
-    printf("     minimum tranfer rate for %d transfers: %9.1f mbs\n",sum_xfers, r_min_mbs);
-    printf("     maximum tranfer rate for %d transfers: %9.1f mbs\n",sum_xfers, r_max_mbs);
     fflush(stdout);
-   }
 
- return 0;
+    if (rank == 0) {
+        mbs = sum_avg_mbs / sum_xfers;
+        printf("\n     average transfer rate for %d transfers: %9.1f mbs\n", sum_xfers, mbs);
+        printf("     minimum transfer rate for %d transfers: %9.1f mbs\n", sum_xfers, r_min_mbs);
+        printf("     maximum transfer rate for %d transfers: %9.1f mbs\n", sum_xfers, r_max_mbs);
+        fflush(stdout);
+    }
+
+    return 0;
 }
diff --git a/test/simple/debugger.c b/test/simple/debugger.c
index 3cecd4b58df..0bb5119bd85 100644
--- a/test/simple/debugger.c
+++ b/test/simple/debugger.c
@@ -9,22 +9,23 @@
 #include <stdio.h>
 #include <unistd.h>
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
 
-    int i, rc, j=0;
+    int i, rc, j = 0;
     double pi;
     pid_t pid;
 
     pid = getpid();
 
-    printf("spin: Pid %ld\n", (long)pid);
+    printf("spin: Pid %ld\n", (long) pid);
 
     i = 0;
     while (0 == j) {
         i++;
         pi = i / 3.14159256;
-        if (i > 100) i = 0;
+        if (i > 100)
+            i = 0;
     }
 
     return 0;
diff --git a/test/simple/delayed_abort.c b/test/simple/delayed_abort.c
index 572b2a30f40..41f9a5d3244 100644
--- a/test/simple/delayed_abort.c
+++ b/test/simple/delayed_abort.c
@@ -9,11 +9,11 @@
 
 #include "opal/runtime/opal.h"
 
+#include "mpi.h"
 #include <stdio.h>
 #include <unistd.h>
-#include "mpi.h"
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int rank, size;
     const char *hostname;
@@ -25,7 +25,7 @@ int main(int argc, char* argv[])
     hostname = opal_gethostname();
     printf("%s: I am %d of %d. pid=%d\n", hostname, rank, size, getpid());
 
-    if (rank%3 == 0) {
+    if (rank % 3 == 0) {
         printf("%s: rank %d aborts\n", hostname, rank);
         if (rank == 3) {
             printf("%s: rank %d is going to sleep\n", hostname, rank);
diff --git a/test/simple/early_abort.c b/test/simple/early_abort.c
index a3aed65dcd7..5398fa4a7b8 100644
--- a/test/simple/early_abort.c
+++ b/test/simple/early_abort.c
@@ -5,13 +5,13 @@
  * The most basic of MPI applications
  */
 
+#include "mpi.h"
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include "mpi.h"
+#include <unistd.h>
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int rank, size;
     char *rk;
diff --git a/test/simple/hello.c b/test/simple/hello.c
index f1bd7fd3e2f..0c8bf34c2e0 100644
--- a/test/simple/hello.c
+++ b/test/simple/hello.c
@@ -5,11 +5,10 @@
  * The most basic of MPI applications
  */
 
-#include <stdio.h>
 #include "mpi.h"
+#include <stdio.h>
 
-
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int rank, size;
 
diff --git a/test/simple/hello_barrier.c b/test/simple/hello_barrier.c
index de8bdf7ce19..eba0697428c 100644
--- a/test/simple/hello_barrier.c
+++ b/test/simple/hello_barrier.c
@@ -5,10 +5,10 @@
  * The most basic of MPI applications
  */
 
-#include <stdio.h>
 #include "mpi.h"
+#include <stdio.h>
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int rank, size;
 
diff --git a/test/simple/hello_nodename.c b/test/simple/hello_nodename.c
index b3c87e977a2..6528e4c8f34 100644
--- a/test/simple/hello_nodename.c
+++ b/test/simple/hello_nodename.c
@@ -12,12 +12,12 @@
 #include "opal/runtime/opal.h"
 
 #include <stdio.h>
-#include <unistd.h>
 #include <stdlib.h>
+#include <unistd.h>
 
 #include "mpi.h"
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int rank, size;
     const char *hostname;
@@ -36,17 +36,18 @@ int main(int argc, char* argv[])
     if (NULL == appnum) {
         opal_asprintf(&appstr, "UNDEFINED");
     } else {
-        opal_asprintf(&appstr, "%d", *(int*)appnum);
+        opal_asprintf(&appstr, "%d", *(int *) appnum);
     }
     MPI_Comm_get_attr(MPI_COMM_WORLD, MPI_UNIVERSE_SIZE, &univ_size, &flag);
     if (NULL == univ_size) {
         opal_asprintf(&unistr, "UNDEFINED");
     } else {
-        opal_asprintf(&unistr, "%d", *(int*)univ_size);
+        opal_asprintf(&unistr, "%d", *(int *) univ_size);
     }
 
     hostname = opal_gethostname();
-    printf("Hello, World, I am %d of %d on host %s from app number %s universe size %s universe envar %s\n",
+    printf("Hello, World, I am %d of %d on host %s from app number %s universe size %s universe "
+           "envar %s\n",
            rank, size, hostname, appstr, unistr, (NULL == envar) ? "NULL" : envar);
 
     MPI_Finalize();
diff --git a/test/simple/hello_output.c b/test/simple/hello_output.c
index 950e31eb240..ee384d9d883 100644
--- a/test/simple/hello_output.c
+++ b/test/simple/hello_output.c
@@ -6,12 +6,11 @@
  */
 #include "opal_config.h"
 
-#include <stdio.h>
 #include "mpi.h"
 #include "opal/util/output.h"
+#include <stdio.h>
 
-
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int rank, size;
     int stream, stream2;
diff --git a/test/simple/hello_show_help.c b/test/simple/hello_show_help.c
index d133c8b6cff..53dd4663f1e 100644
--- a/test/simple/hello_show_help.c
+++ b/test/simple/hello_show_help.c
@@ -7,14 +7,14 @@
 
 #include "opal_config.h"
 
-#include <stdio.h>
-#include <unistd.h>
 #include "mpi.h"
 #include "opal/util/output.h"
+#include <stdio.h>
+#include <unistd.h>
 
-#include "orte/util/show_help.h"
+#include "opal/util/show_help.h"
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int rank, size;
     int stream;
@@ -24,28 +24,37 @@ int main(int argc, char* argv[])
     MPI_Comm_size(MPI_COMM_WORLD, &size);
 
     if (0 == rank) {
-        opal_output(0, "============================================================================");
-        opal_output(0, "This test ensures that the aggregation functionality of the orte_show_help\nsystem is working properly.  It outputs a bogus warning about orte_init(),\nand contains sleep statements to ensure that the timer is firiing properly\nin the HNP and aggregates messages properly.  The total sleep time is\n(3 * num_procs).  You should see:\n\n - aggregation messages from the HNP every five seconds or so\n - a total of (2 * num_procs) messages");
-        opal_output(0, "============================================================================");
+        opal_output(0,
+                    "============================================================================");
+        opal_output(
+            0, "This test ensures that the aggregation functionality of the opal_show_help\nsystem "
+               "is working properly.  It outputs a bogus warning about opal_init(),\nand contains "
+               "sleep statements to ensure that the timer is firing properly\nin the HNP and "
+               "aggregates messages properly.  The total sleep time is\n(3 * num_procs).  You "
+               "should see:\n\n - aggregation messages from the HNP every five seconds or so\n - a "
+               "total of (2 * num_procs) messages");
+        opal_output(0,
+                    "============================================================================");
     }
     MPI_Barrier(MPI_COMM_WORLD);
 
-    orte_show_help("help-orte-runtime.txt",
-                   "orte_init:startup:internal-failure", true,
-                   "Nothing", "ORTE_EVERYTHING_IS_PEACHY", "42");
+    opal_show_help("help-opal-runtime.txt", "opal_init:startup:internal-failure",
+                   true, "Nothing", 42);
     sleep(rank * 3);
 
-    orte_show_help("help-orte-runtime.txt",
-                   "orte_init:startup:internal-failure", true,
-                   "Duplicate orte_show_help detection",
-                   "ORTE_SHOW_HELP_DUPLICATE_FAILED", "99999");
+    opal_show_help("help-opal-runtime.txt", "opal_init:startup:internal-failure", true,
+                   true, "Duplicate opal_show_help detection", 42);
 
     MPI_Barrier(MPI_COMM_WORLD);
 
     if (0 == rank) {
-        opal_output(0, "============================================================================");
-        opal_output(0, "The test is now complete.  Please verify that the HNP output all the required\nmessages (you may see 1 or 2 more messages from the HNP after this message).");
-        opal_output(0, "============================================================================");
+        opal_output(0,
+                    "============================================================================");
+        opal_output(0, "The test is now complete.  Please verify that the HNP output all the "
+                       "required\nmessages (you may see 1 or 2 more messages from the HNP after "
+                       "this message).");
+        opal_output(0,
+                    "============================================================================");
     }
     MPI_Finalize();
 
diff --git a/test/simple/info_spawn.c b/test/simple/info_spawn.c
index b75efc1d341..1003938bc06 100644
--- a/test/simple/info_spawn.c
+++ b/test/simple/info_spawn.c
@@ -3,14 +3,14 @@
 #include "opal/runtime/opal.h"
 
 #include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
 #include <sys/types.h>
 #include <unistd.h>
-#include <string.h>
-#include <stdlib.h>
 
 #include <mpi.h>
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int msg, rc, i;
     MPI_Comm parent, child;
@@ -34,9 +34,9 @@ int main(int argc, char* argv[])
         }
 
         pid = getpid();
-        printf("Parent [pid %ld] about to spawn!\n", (long)pid);
+        printf("Parent [pid %ld] about to spawn!\n", (long) pid);
         MPI_Info_create(&info);
-        for (i=1; i < argc; i++) {
+        for (i = 1; i < argc; i++) {
             tmp = strdup(argv[i]);
             keyval = strchr(tmp, ':');
             if (NULL == keyval) {
@@ -49,8 +49,9 @@ int main(int argc, char* argv[])
             MPI_Info_set(info, tmp, keyval);
             free(tmp);
         }
-        if (MPI_SUCCESS != (rc = MPI_Comm_spawn(argv[0], MPI_ARGV_NULL, 3, info,
-                                                0, MPI_COMM_WORLD, &child, MPI_ERRCODES_IGNORE))) {
+        if (MPI_SUCCESS
+            != (rc = MPI_Comm_spawn(argv[0], MPI_ARGV_NULL, 3, info, 0, MPI_COMM_WORLD, &child,
+                                    MPI_ERRCODES_IGNORE))) {
             printf("Child failed to spawn\n");
             return rc;
         }
@@ -69,7 +70,7 @@ int main(int argc, char* argv[])
         MPI_Comm_size(MPI_COMM_WORLD, &size);
         hostname = opal_gethostname();
         pid = getpid();
-        printf("Hello from the child %d of %d on host %s pid %ld\n", rank, 3, hostname, (long)pid);
+        printf("Hello from the child %d of %d on host %s pid %ld\n", rank, 3, hostname, (long) pid);
         if (0 == rank) {
             MPI_Recv(&msg, 1, MPI_INT, 0, 1, parent, MPI_STATUS_IGNORE);
             printf("Child %d received msg: %d\n", rank, msg);
diff --git a/test/simple/init-exit77.c b/test/simple/init-exit77.c
index 81c2b37f4d2..b0fb0f2e7e3 100644
--- a/test/simple/init-exit77.c
+++ b/test/simple/init-exit77.c
@@ -1,13 +1,14 @@
+#include "mpi.h"
 #include <stdio.h>
 #include <stdlib.h>
-#include "mpi.h"
 
-int main(int argc, char **argv) {
-   int debugme = 1;
+int main(int argc, char **argv)
+{
+    int debugme = 1;
 
-   MPI_Init(&argc, &argv);
-   printf("init...\n");
-   fflush(0);
-   MPI_Finalize();
-   exit(77);
+    MPI_Init(&argc, &argv);
+    printf("init...\n");
+    fflush(0);
+    MPI_Finalize();
+    exit(77);
 }
diff --git a/test/simple/initial_errh.c b/test/simple/initial_errh.c
index 27adbd6ce08..430437d8546 100644
--- a/test/simple/initial_errh.c
+++ b/test/simple/initial_errh.c
@@ -9,26 +9,30 @@
  * $HEADER$
  */
 
+#include "mpi.h"
 #include <stdio.h>
 #include <string.h>
 #include <unistd.h>
-#include "mpi.h"
 
-#define print1(format...) if(0 == rank) printf(format)
+#define print1(format...) \
+    if (0 == rank)        \
+    printf(format)
 
 int main_child(int argc, char *argv[]);
 
 int main(int argc, char *argv[])
 {
-    int rank=MPI_PROC_NULL, rc;
+    int rank = MPI_PROC_NULL, rc;
     /* info_env and error handlers */
-    char init_errh_info[MPI_MAX_INFO_VAL+1]; int flag;
+    char init_errh_info[MPI_MAX_INFO_VAL + 1];
+    int flag;
     MPI_Errhandler errh;
     /* error ops */
-    int eclass=MPI_SUCCESS;
-    char estr[MPI_MAX_ERROR_STRING]="NOT UPDATED"; int slen;
+    int eclass = MPI_SUCCESS;
+    char estr[MPI_MAX_ERROR_STRING] = "NOT UPDATED";
+    int slen;
     /* spawn params */
-    char* spawn_argv[3];
+    char *spawn_argv[3];
     MPI_Info spawn_info;
     int spawn_err[2] = {MPI_SUCCESS};
     MPI_Comm icomm = MPI_COMM_NULL;
@@ -36,7 +40,7 @@ int main(int argc, char *argv[])
     /* We will verify pre-init behavior in a spawnee to avoid aborting early in
      * implementations with only partial support.
      */
-    if(argc > 1 && 0 == strcmp(argv[1], "preinit-error")) {
+    if (argc > 1 && 0 == strcmp(argv[1], "preinit-error")) {
         return main_child(argc, argv);
     }
 
@@ -45,42 +49,43 @@ int main(int argc, char *argv[])
     MPI_Init(&argc, &argv);
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
 
-    print1(
-"# This test checks compliance with MPI-4 initial error handler.\n"
-"# This test assumes that the command line parameter '-initial-errhandler mpi_errors_return'\n"
-"# is passed to 'mpiexec', in which case, a compliant implementation will:\n"
-"#   * Set the MPI_INFO_ENV key to the requested error handler.\n"
-"#   * The requested handler is set on the predefined communicators MPI_COMM_SELF, MPI_COMM_WORLD,\n"
-"#     and the communicator returned from MPI_COMM_GET_PARENT.\n"
-"# In a high quality implementation:\n"
-"#   * Errors reported from calls during, before, and after MPI_INIT and MPI_FINALIZE also invoke the\n"
-"#     initial error handler.\n"
-"#   * MPI_ERROR_STRING and MPI_ERROR_CLASS provide useful information before/after MPI_INIT and\n"
-"#     MPI_FINALIZE respectively.\n\n");
+    print1("# This test checks compliance with MPI-4 initial error handler.\n"
+           "# This test assumes that the command line parameter '-initial-errhandler "
+           "mpi_errors_return'\n"
+           "# is passed to 'mpiexec', in which case, a compliant implementation will:\n"
+           "#   * Set the MPI_INFO_ENV key to the requested error handler.\n"
+           "#   * The requested handler is set on the predefined communicators MPI_COMM_SELF, "
+           "MPI_COMM_WORLD,\n"
+           "#     and the communicator returned from MPI_COMM_GET_PARENT.\n"
+           "# In a high quality implementation:\n"
+           "#   * Errors reported from calls during, before, and after MPI_INIT and MPI_FINALIZE "
+           "also invoke the\n"
+           "#     initial error handler.\n"
+           "#   * MPI_ERROR_STRING and MPI_ERROR_CLASS provide useful information before/after "
+           "MPI_INIT and\n"
+           "#     MPI_FINALIZE respectively.\n\n");
 
     print1("MPI_INFO_ENV for key 'mpi_initial_errhandler'\n");
     MPI_Info_get(MPI_INFO_ENV, "mpi_initial_errhandler", MPI_MAX_INFO_VAL, init_errh_info, &flag);
-    if(flag) {
-        print1("  MPI-4 COMPLIANT:\tMPI_INFO_ENV value set for key 'mpi_initial_errhandler' = %s\n\n", init_errh_info);
-    }
-    else {
-        print1("  NOT MPI-4 COMPLIANT:\tMPI_INFO_ENV has no value set for key 'mpi_initial_errhandler'\n\n");
+    if (flag) {
+        print1(
+            "  MPI-4 COMPLIANT:\tMPI_INFO_ENV value set for key 'mpi_initial_errhandler' = %s\n\n",
+            init_errh_info);
+    } else {
+        print1("  NOT MPI-4 COMPLIANT:\tMPI_INFO_ENV has no value set for key "
+               "'mpi_initial_errhandler'\n\n");
     }
 
     print1("MPI_COMM_GET_ERRHANDLER:\n");
     MPI_Comm_get_errhandler(MPI_COMM_SELF, &errh);
-    if(MPI_ERRORS_RETURN == errh) {
+    if (MPI_ERRORS_RETURN == errh) {
         print1("  MPI-4 COMPLIANT:\tMPI_COMM_SELF error handler set to MPI_ERRORS_RETURN.\n\n");
-    }
-    else
-    if(MPI_ERRORS_ABORT == errh) {
+    } else if (MPI_ERRORS_ABORT == errh) {
         print1("  UNEXPECTED:\tMPI_COMM_SELF error handler set to MPI_ERRORS_ABORT.\n\n");
-    }
-    else
-    if(MPI_ERRORS_ARE_FATAL == errh) {
-        print1("  NOT MPI-4 COMPLIANT:\tMPI_COMM_SELF error handler set to MPI_ERRORS_ARE_FATAL.\n\n");
-    }
-    else {
+    } else if (MPI_ERRORS_ARE_FATAL == errh) {
+        print1(
+            "  NOT MPI-4 COMPLIANT:\tMPI_COMM_SELF error handler set to MPI_ERRORS_ARE_FATAL.\n\n");
+    } else {
         print1("  UNEXPECTED:\tMPI_COMM_SELF error handler is not one of the predefined ones.\n\n");
     }
 
@@ -101,29 +106,34 @@ int main(int argc, char *argv[])
 
     /* set error handler to fatal before FINALIZE */
     rc = MPI_Comm_set_errhandler(MPI_COMM_SELF, MPI_ERRORS_ARE_FATAL);
-    if(MPI_SUCCESS != rc) {
+    if (MPI_SUCCESS != rc) {
         MPI_Error_string(rc, estr, &slen);
-        fprintf(stderr, "  UNEXPECTED: An error occured during MPI_COMM_SETERRHANDLER(SELF) rc=%d: %s\n", rc, estr);
+        fprintf(stderr,
+                "  UNEXPECTED: An error occurred during MPI_COMM_SETERRHANDLER(SELF) rc=%d: %s\n",
+                rc, estr);
         return rc;
     }
     /* FINALIZE should force reversion to the initial errhandler, so we need to
      * check again (though we did not insert errors so all should go smooth). */
     rc = MPI_Finalize();
-    if(MPI_SUCCESS != rc) {
+    if (MPI_SUCCESS != rc) {
         MPI_Error_string(rc, estr, &slen);
-        fprintf(stderr, "  UNEXPECTED: An error occured during MPI_FINALIZE rc=%d: %s\n", rc, estr);
+        fprintf(stderr, "  UNEXPECTED: An error occurred during MPI_FINALIZE rc=%d: %s\n", rc, estr);
         return rc;
     }
 
     printf("Post-finalize MPI_ERROR_STRING call:\n");
     rc = MPI_Error_string(MPI_ERR_WIN, estr, &slen);
-    if(MPI_SUCCESS != rc) {
-        fprintf(stderr, "  NOT MPI-4 COMPLIANT:\tpost-finalize MPI_ERROR_STRING returned %d (expected MPI_SUCCESS)\n", rc);
-    }
-    else if(0 == strcmp(estr, "NOT UPDATED")) {
-        fprintf(stderr, "  NOT MPI-4 COMPLIANT:\tpost-finalize MPI_ERROR_STRING did not set a valid string.\n");
-    }
-    else {
+    if (MPI_SUCCESS != rc) {
+        fprintf(stderr,
+                "  NOT MPI-4 COMPLIANT:\tpost-finalize MPI_ERROR_STRING returned %d (expected "
+                "MPI_SUCCESS)\n",
+                rc);
+    } else if (0 == strcmp(estr, "NOT UPDATED")) {
+        fprintf(
+            stderr,
+            "  NOT MPI-4 COMPLIANT:\tpost-finalize MPI_ERROR_STRING did not set a valid string.\n");
+    } else {
         /* We can't further check if the error string makes sense; In any
          * case, any string is compliant, even low-quality non-informative
          * generic strings. So we just print it. */
@@ -132,35 +142,42 @@ int main(int argc, char *argv[])
     return 0;
 }
 
-int main_child(int argc, char *argv[]) {
-    int rank=0, rc;
-    MPI_Comm icomm=MPI_COMM_NULL;
-    int eclass=MPI_SUCCESS;
-    char estr[MPI_MAX_ERROR_STRING]="NOT UPDATED"; int slen;
+int main_child(int argc, char *argv[])
+{
+    int rank = 0, rc;
+    MPI_Comm icomm = MPI_COMM_NULL;
+    int eclass = MPI_SUCCESS;
+    char estr[MPI_MAX_ERROR_STRING] = "NOT UPDATED";
+    int slen;
 
     /* ERROR_CLASS and ERROR_STRING are callable before MPI_INIT */
 
     printf("Pre-init MPI_ERROR_CLASS call:\n");
     rc = MPI_Error_class(MPI_ERR_WIN, &eclass);
-    if(MPI_SUCCESS != rc) {
-        fprintf(stderr, "  NOT MPI-4 COMPLIANT:\tpre-init MPI_ERROR_CLASS returned %d (expected MPI_SUCCESS)\n", rc);
-    }
-    else if(MPI_ERR_WIN != eclass) {
-        fprintf(stderr, "  NOT MPI-4 COMPLIANT:\tpre-init MPI_ERROR_CLASS set eclass=%d (expected %d)\n", eclass, MPI_ERR_WIN);
-    }
-    else {
+    if (MPI_SUCCESS != rc) {
+        fprintf(
+            stderr,
+            "  NOT MPI-4 COMPLIANT:\tpre-init MPI_ERROR_CLASS returned %d (expected MPI_SUCCESS)\n",
+            rc);
+    } else if (MPI_ERR_WIN != eclass) {
+        fprintf(stderr,
+                "  NOT MPI-4 COMPLIANT:\tpre-init MPI_ERROR_CLASS set eclass=%d (expected %d)\n",
+                eclass, MPI_ERR_WIN);
+    } else {
         printf("  MPI-4 COMPLIANT:\tPre-init MPI_ERROR_CLASS\n");
     }
 
     print1("Pre-init MPI_ERROR_STRING call:\n");
     rc = MPI_Error_string(MPI_ERR_WIN, estr, &slen);
-    if(MPI_SUCCESS != rc) {
-        fprintf(stderr, "  NOT MPI-4 COMPLIANT:\tpre-init MPI_ERROR_STRING returned %d (expected MPI_SUCCESS)\n", rc);
-    }
-    else if(0 == strcmp(estr, "NOT UPDATED")) {
-        fprintf(stderr, "  NOT MPI-4 COMPLIANT:\tpre-init MPI_ERROR_STRING did not set a valid string.\n");
-    }
-    else {
+    if (MPI_SUCCESS != rc) {
+        fprintf(stderr,
+                "  NOT MPI-4 COMPLIANT:\tpre-init MPI_ERROR_STRING returned %d (expected "
+                "MPI_SUCCESS)\n",
+                rc);
+    } else if (0 == strcmp(estr, "NOT UPDATED")) {
+        fprintf(stderr,
+                "  NOT MPI-4 COMPLIANT:\tpre-init MPI_ERROR_STRING did not set a valid string.\n");
+    } else {
         /* We can't further check if the error string makes sense; In any
          * case, any string is compliant, even low-quality non-informative
          * generic strings. So we just print it. */
@@ -168,30 +185,35 @@ int main_child(int argc, char *argv[]) {
     }
 
     printf("Pre-init error in a call: compliant if it does not abort\n");
-    rc = MPI_Error_class(MPI_ERR_LASTCODE+1, &eclass);
+    rc = MPI_Error_class(MPI_ERR_LASTCODE + 1, &eclass);
     eclass = rc;
     rc = MPI_Error_string(eclass, estr, &slen);
-    if(MPI_SUCCESS != rc) {
-        printf("  MPI-4 COMPLIANT:\tPre-init MPI_ERROR_CLASS with erroneous arguments returned (LOW QUALITY: error code=%d caused error %d in MPI_ERROR_STRING).\n", eclass, rc);
-    }
-    else {
-        printf("  MPI-4 COMPLIANT:\tPre-init MPI_ERROR_STRING for non-existing code returned %d: %s\n", eclass, estr);
+    if (MPI_SUCCESS != rc) {
+        printf("  MPI-4 COMPLIANT:\tPre-init MPI_ERROR_CLASS with erroneous arguments returned "
+               "(LOW QUALITY: error code=%d caused error %d in MPI_ERROR_STRING).\n",
+               eclass, rc);
+    } else {
+        printf(
+            "  MPI-4 COMPLIANT:\tPre-init MPI_ERROR_STRING for non-existing code returned %d: %s\n",
+            eclass, estr);
     }
 
     printf("Initializing MPI and setting error handlers on predefined communicators.\n");
     rc = MPI_Init(&argc, &argv);
-    if(MPI_SUCCESS != rc) {
+    if (MPI_SUCCESS != rc) {
         MPI_Error_string(rc, estr, &slen);
-        fprintf(stderr, "  UNEXPECTED: An error occured during MPI_INIT rc=%d: %s\n", rc, estr);
+        fprintf(stderr, "  UNEXPECTED: An error occurred during MPI_INIT rc=%d: %s\n", rc, estr);
         return rc;
     }
 
     /* sync-up with parent */
     MPI_Comm_get_parent(&icomm);
     rc = MPI_Comm_set_errhandler(icomm, MPI_ERRORS_ARE_FATAL);
-    if(MPI_SUCCESS != rc) {
+    if (MPI_SUCCESS != rc) {
         MPI_Error_string(rc, estr, &slen);
-        fprintf(stderr, "  UNEXPECTED: An error occured during MPI_COMM_SETERRHANDLER(PARENT) rc=%d: %s\n", rc, estr);
+        fprintf(stderr,
+                "  UNEXPECTED: An error occurred during MPI_COMM_SETERRHANDLER(PARENT) rc=%d: %s\n",
+                rc, estr);
         return rc;
     }
     MPI_Barrier(icomm);
diff --git a/test/simple/intercomm1.c b/test/simple/intercomm1.c
index 8c01d3aef29..8be182e652e 100644
--- a/test/simple/intercomm1.c
+++ b/test/simple/intercomm1.c
@@ -1,25 +1,23 @@
-#include <stdio.h>
 #include "mpi.h"
+#include <stdio.h>
 
-
-int main( int argc, char *argv[] )
+int main(int argc, char *argv[])
 {
     MPI_Status status;
-    MPI_Comm comm,scomm;
-    int rank, size, color, errs=0;
-    MPI_Init( 0, 0 );
-    MPI_Comm_rank( MPI_COMM_WORLD, &rank );
+    MPI_Comm comm, scomm;
+    int rank, size, color, errs = 0;
+    MPI_Init(0, 0);
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     color = rank % 2;
     printf("%d Calling split\n", rank);
-    MPI_Comm_split( MPI_COMM_WORLD, color, rank, &scomm );
+    MPI_Comm_split(MPI_COMM_WORLD, color, rank, &scomm);
     printf("%d Calling Intercomm_create\n", rank);
-    MPI_Intercomm_create( scomm, 0, MPI_COMM_WORLD, 1-color, 1, &comm);
-    printf("%d Completet\n", rank);
-    MPI_Comm_rank( comm, &rank );
-    MPI_Comm_remote_size( comm, &size );
+    MPI_Intercomm_create(scomm, 0, MPI_COMM_WORLD, 1 - color, 1, &comm);
+    printf("%d Complete\n", rank);
+    MPI_Comm_rank(comm, &rank);
+    MPI_Comm_remote_size(comm, &size);
     MPI_Comm_free(&scomm);
     MPI_Comm_free(&comm);
     MPI_Finalize();
     return errs;
 }
-
diff --git a/test/simple/intercomm_create.c b/test/simple/intercomm_create.c
index 8e76958a212..8a8c39d1b39 100644
--- a/test/simple/intercomm_create.c
+++ b/test/simple/intercomm_create.c
@@ -1,8 +1,8 @@
 #define _GNU_SOURCE
+#include <stdarg.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <stdarg.h>
 #include <unistd.h>
 
 #include <mpi.h>
@@ -10,39 +10,38 @@
 #define NB_SPAWN 4
 
 static void do_parent(char *argv[], int rank, int count);
-static void do_target(char* argv[], MPI_Comm parent);
+static void do_target(char *argv[], MPI_Comm parent);
 
 static char *cmd_argv1 = "b";
 static char *cmd_argv2 = "c";
 static char *whoami = "a";
 static int tag = 201;
 
-void ompitest_warning( char* filename, int lineno, const char* fmt, ... )
+void ompitest_warning(char *filename, int lineno, const char *fmt, ...)
 {
-    char* buf = NULL;
+    char *buf = NULL;
     va_list va_list;
 
     va_start(va_list, fmt);
-    vasprintf( &buf, fmt, va_list );
+    vasprintf(&buf, fmt, va_list);
     va_end(va_list);
-    printf( "*warning* %s:%d %s\n", filename, lineno, buf );
+    printf("*warning* %s:%d %s\n", filename, lineno, buf);
     free(buf);
 }
 
-void ompitest_error( char* filename, int lineno, const char* fmt, ... )
+void ompitest_error(char *filename, int lineno, const char *fmt, ...)
 {
-    char* buf = NULL;
+    char *buf = NULL;
     va_list va_list;
 
     va_start(va_list, fmt);
-    vasprintf( &buf, fmt, va_list );
+    vasprintf(&buf, fmt, va_list);
     va_end(va_list);
-    printf( "*error* %s:%d %s\n", filename, lineno, buf );
+    printf("*error* %s:%d %s\n", filename, lineno, buf);
     free(buf);
 }
 
-int
-main(int argc, char *argv[])
+int main(int argc, char *argv[])
 {
     int rank, size;
     MPI_Comm parent;
@@ -71,58 +70,55 @@ main(int argc, char *argv[])
     return 0;
 }
 
-static int
-spawn_and_merge( char* argv[], char* arg, int count,
-                 MPI_Comm* inter, MPI_Comm* intra )
+static int spawn_and_merge(char *argv[], char *arg, int count, MPI_Comm *inter, MPI_Comm *intra)
 {
     int *errcode, err, i;
     char *spawn_argv[2];
 
     errcode = malloc(sizeof(int) * count);
     if (errcode == NULL)
-        ompitest_error(__FILE__, __LINE__, "Doh!  Rank %d was not able to allocate enough memory.  MPI test aborted!\n", 0);
+        ompitest_error(__FILE__, __LINE__,
+                       "Doh!  Rank %d was not able to allocate enough memory.  MPI test aborted!\n",
+                       0);
     memset(errcode, -1, count);
     /*MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN);*/
 
     spawn_argv[0] = arg;
     spawn_argv[1] = NULL;
-    err = MPI_Comm_spawn(argv[0], spawn_argv, count, MPI_INFO_NULL, 0,
-                         MPI_COMM_WORLD, inter, errcode);
+    err = MPI_Comm_spawn(argv[0], spawn_argv, count, MPI_INFO_NULL, 0, MPI_COMM_WORLD, inter,
+                         errcode);
     for (i = 0; i < count; i++)
         if (errcode[i] != MPI_SUCCESS)
-            ompitest_error(__FILE__, __LINE__,
-                           "ERROR: MPI_Comm_spawn returned errcode[%d] = %d\n",
+            ompitest_error(__FILE__, __LINE__, "ERROR: MPI_Comm_spawn returned errcode[%d] = %d\n",
                            i, errcode[i]);
     if (err != MPI_SUCCESS)
-        ompitest_error(__FILE__, __LINE__,
-                       "ERROR: MPI_Comm_spawn returned errcode = %d\n", err);
-    err = MPI_Intercomm_merge( *inter, 0, intra );
+        ompitest_error(__FILE__, __LINE__, "ERROR: MPI_Comm_spawn returned errcode = %d\n", err);
+    err = MPI_Intercomm_merge(*inter, 0, intra);
     free(errcode);
     return err;
 }
 
-static void
-do_parent(char *argv[], int rank, int count)
+static void do_parent(char *argv[], int rank, int count)
 {
     MPI_Comm ab_inter, ab_intra, ac_inter, ac_intra, ab_c_inter, abc_intra;
     int err;
 
-    err = spawn_and_merge( argv, cmd_argv1, count, &ab_inter, &ab_intra );
-    err = spawn_and_merge( argv, cmd_argv2, count, &ac_inter, &ac_intra );
+    err = spawn_and_merge(argv, cmd_argv1, count, &ab_inter, &ab_intra);
+    err = spawn_and_merge(argv, cmd_argv2, count, &ac_inter, &ac_intra);
 
-    printf( "%s: MPI_Intercomm_create( ab_intra, 0, ac_intra, %d, %d, &inter) (%d)\n",
-            whoami, count, tag, err );
-    err = MPI_Intercomm_create( ab_intra, 0, ac_intra, count, tag, &ab_c_inter );
-    printf( "%s: intercomm_create (%d)\n", whoami, err );
+    printf("%s: MPI_Intercomm_create( ab_intra, 0, ac_intra, %d, %d, &inter) (%d)\n", whoami, count,
+           tag, err);
+    err = MPI_Intercomm_create(ab_intra, 0, ac_intra, count, tag, &ab_c_inter);
+    printf("%s: intercomm_create (%d)\n", whoami, err);
 
-    printf( "%s: barrier on inter-comm - before\n", whoami );
+    printf("%s: barrier on inter-comm - before\n", whoami);
     err = MPI_Barrier(ab_c_inter);
-    printf( "%s: barrier on inter-comm - after\n", whoami );
+    printf("%s: barrier on inter-comm - after\n", whoami);
 
     err = MPI_Intercomm_merge(ab_c_inter, 0, &abc_intra);
-    printf( "%s: intercomm_merge(%d) (%d) [rank %d]\n", whoami, 0, err, rank );
+    printf("%s: intercomm_merge(%d) (%d) [rank %d]\n", whoami, 0, err, rank);
     err = MPI_Barrier(abc_intra);
-    printf( "%s: barrier (%d)\n", whoami, err );
+    printf("%s: barrier (%d)\n", whoami, err);
 
     MPI_Comm_free(&abc_intra);
     MPI_Comm_free(&ab_c_inter);
@@ -133,38 +129,39 @@ do_parent(char *argv[], int rank, int count)
     MPI_Comm_disconnect(&ac_inter);
 }
 
-
-static void
-do_target(char* argv[], MPI_Comm parent)
+static void do_target(char *argv[], MPI_Comm parent)
 {
     int rank, first = 0, err;
     MPI_Comm intra, inter, merge1;
 
-    if( 0 == strcmp(argv[1], cmd_argv1) ) first = 1;
+    if (0 == strcmp(argv[1], cmd_argv1))
+        first = 1;
 
     /*MPI_Comm_set_errhandler(parent, MPI_ERRORS_RETURN);*/
 
-    err = MPI_Intercomm_merge( parent, 1, &intra );
+    err = MPI_Intercomm_merge(parent, 1, &intra);
     MPI_Comm_rank(intra, &rank);
 
-    if( first ) {
-        printf( "%s: MPI_Intercomm_create( intra, 0, intra, MPI_COMM_NULL, %d, &inter) [rank %d]\n", whoami, tag, rank );
-        err = MPI_Intercomm_create( intra, 0, MPI_COMM_NULL, 0, tag, &inter);
-        printf( "%s: intercomm_create (%d)\n", whoami, err );
+    if (first) {
+        printf("%s: MPI_Intercomm_create( intra, 0, intra, MPI_COMM_NULL, %d, &inter) [rank %d]\n",
+               whoami, tag, rank);
+        err = MPI_Intercomm_create(intra, 0, MPI_COMM_NULL, 0, tag, &inter);
+        printf("%s: intercomm_create (%d)\n", whoami, err);
     } else {
-        printf( "%s: MPI_Intercomm_create( MPI_COMM_WORLD, 0, intra, 0, %d, &inter) [rank %d]\n", whoami, tag, rank );
-        err = MPI_Intercomm_create( MPI_COMM_WORLD, 0, intra, 0, tag, &inter);
-        printf( "%s: intercomm_create (%d)\n", whoami, err );
+        printf("%s: MPI_Intercomm_create( MPI_COMM_WORLD, 0, intra, 0, %d, &inter) [rank %d]\n",
+               whoami, tag, rank);
+        err = MPI_Intercomm_create(MPI_COMM_WORLD, 0, intra, 0, tag, &inter);
+        printf("%s: intercomm_create (%d)\n", whoami, err);
     }
-    printf( "%s: barrier on inter-comm - before\n", whoami );
+    printf("%s: barrier on inter-comm - before\n", whoami);
     err = MPI_Barrier(inter);
-    printf( "%s: barrier on inter-comm - after\n", whoami );
+    printf("%s: barrier on inter-comm - after\n", whoami);
 
-    err = MPI_Intercomm_merge( inter, 0, &merge1 );
+    err = MPI_Intercomm_merge(inter, 0, &merge1);
     MPI_Comm_rank(merge1, &rank);
-    printf( "%s: intercomm_merge(%d) (%d) [rank %d]\n", whoami, first, err, rank );
+    printf("%s: intercomm_merge(%d) (%d) [rank %d]\n", whoami, first, err, rank);
     err = MPI_Barrier(merge1);
-    printf( "%s: barrier (%d)\n", whoami, err );
+    printf("%s: barrier (%d)\n", whoami, err);
 
     MPI_Comm_free(&merge1);
     MPI_Comm_free(&inter);
diff --git a/test/simple/interlib.c b/test/simple/interlib.c
index fefec2b2385..470eb3e3138 100644
--- a/test/simple/interlib.c
+++ b/test/simple/interlib.c
@@ -5,31 +5,27 @@
  * The most basic of MPI applications
  */
 
-#include <stdio.h>
 #include <pthread.h>
+#include <stdio.h>
 
-#include "opal/mca/hwloc/base/base.h"
 #include "mpi.h"
+#include "opal/mca/hwloc/base/base.h"
 
-#include "orte/util/proc_info.h"
 #include "opal/mca/pmix/base/base.h"
+#include "orte/util/proc_info.h"
 
 static size_t interlibhandler_id = SIZE_MAX;
 static opal_pmix_lock_t thread_complete;
 
-static void model_registration_callback(int status,
-                                        size_t errhandler_ref,
-                                        void *cbdata)
+static void model_registration_callback(int status, size_t errhandler_ref, void *cbdata)
 {
-    opal_pmix_lock_t *lock = (opal_pmix_lock_t*)cbdata;
+    opal_pmix_lock_t *lock = (opal_pmix_lock_t *) cbdata;
 
     interlibhandler_id = errhandler_ref;
     OPAL_PMIX_WAKEUP_THREAD(lock);
 }
-static void model_callback(int status,
-                           const opal_process_name_t *source,
-                           opal_list_t *info, opal_list_t *results,
-                           opal_pmix_notification_complete_fn_t cbfunc,
+static void model_callback(int status, const opal_process_name_t *source, opal_list_t *info,
+                           opal_list_t *results, opal_pmix_notification_complete_fn_t cbfunc,
                            void *cbdata)
 {
     opal_value_t *val;
@@ -37,9 +33,9 @@ static void model_callback(int status,
     /* we can ignore our own callback as we obviously
      * know that we are OpenMP */
     if (NULL != info) {
-        OPAL_LIST_FOREACH(val, info, opal_value_t) {
-            if (0 == strcmp(val->key, OPAL_PMIX_PROGRAMMING_MODEL) &&
-                0 == strcmp(val->data.string, "OpenMP")) {
+        OPAL_LIST_FOREACH (val, info, opal_value_t) {
+            if (0 == strcmp(val->key, OPAL_PMIX_PROGRAMMING_MODEL)
+                && 0 == strcmp(val->data.string, "OpenMP")) {
                 goto cback;
             }
             if (PMIX_STRING == val->type) {
@@ -49,7 +45,7 @@ static void model_callback(int status,
     }
     /* otherwise, do something clever here */
 
-  cback:
+cback:
     /* we must NOT tell the event handler state machine that we
      * are the last step as that will prevent it from notifying
      * anyone else that might be listening for declarations */
@@ -61,20 +57,17 @@ static void model_callback(int status,
 
 static void opcbfunc(int status, void *cbdata)
 {
-    opal_pmix_lock_t *lock = (opal_pmix_lock_t*)cbdata;
+    opal_pmix_lock_t *lock = (opal_pmix_lock_t *) cbdata;
     OPAL_PMIX_WAKEUP_THREAD(lock);
 }
 
-static void infocb(int status,
-                   opal_list_t *info,
-                   void *cbdata,
-                   opal_pmix_release_cbfunc_t release_fn,
-                   void *release_cbdata)
+static void infocb(int status, opal_list_t *info, void *cbdata,
+                   opal_pmix_release_cbfunc_t release_fn, void *release_cbdata)
 {
-    opal_pmix_lock_t *lock = (opal_pmix_lock_t*)cbdata;
+    opal_pmix_lock_t *lock = (opal_pmix_lock_t *) cbdata;
     opal_value_t *kv;
 
-    OPAL_LIST_FOREACH(kv, info, opal_value_t) {
+    OPAL_LIST_FOREACH (kv, info, opal_value_t) {
         opal_output(0, "QUERY DATA KEY: %s VALUE %s", kv->key, kv->data.string);
     }
     if (NULL != release_fn) {
@@ -130,10 +123,8 @@ static void *mylib(void *ptr)
         /* it is, so let's just use the event notification
          * API to let everyone know we are here */
         OPAL_PMIX_CONSTRUCT_LOCK(&lock);
-        ret = opal_pmix.notify_event(OPAL_ERR_MODEL_DECLARED,
-                                     &orte_process_info.my_name,
-                                     OPAL_PMIX_RANGE_PROC_LOCAL, &info,
-                                     opcbfunc, &lock);
+        ret = opal_pmix.notify_event(OPAL_ERR_MODEL_DECLARED, &orte_process_info.my_name,
+                                     OPAL_PMIX_RANGE_PROC_LOCAL, &info, opcbfunc, &lock);
         OPAL_PMIX_WAIT_THREAD(&lock);
         OPAL_PMIX_DESTRUCT_LOCK(&lock);
         OPAL_LIST_DESTRUCT(&info);
@@ -156,7 +147,7 @@ static void *mylib(void *ptr)
     /* specify the event code */
     OBJ_CONSTRUCT(&info, opal_list_t);
     kv = OBJ_NEW(opal_value_t);
-    kv->key = strdup("status");   // the key here is irrelevant
+    kv->key = strdup("status"); // the key here is irrelevant
     kv->type = OPAL_INT;
     kv->data.integer = OPAL_ERR_MODEL_DECLARED;
     opal_list_append(&info, &kv->super);
@@ -165,9 +156,8 @@ static void *mylib(void *ptr)
      * the event stipulates its range as proc_local. We rely
      * on that here */
     OPAL_PMIX_CONSTRUCT_LOCK(&lock);
-    opal_pmix.register_evhandler(&info, &directives, model_callback,
-                                 model_registration_callback,
-                                 (void*)&lock);
+    opal_pmix.register_evhandler(&info, &directives, model_callback, model_registration_callback,
+                                 (void *) &lock);
     OPAL_PMIX_WAIT_THREAD(&lock);
     OPAL_PMIX_DESTRUCT_LOCK(&lock);
     OPAL_LIST_DESTRUCT(&info);
@@ -189,8 +179,7 @@ static void *mylib(void *ptr)
     OPAL_LIST_DESTRUCT(&info);
 
     /* Get something */
-    opal_pmix.get(&orte_process_info.my_name,
-                  "WASSUP", NULL, &kv);
+    opal_pmix.get(&orte_process_info.my_name, "WASSUP", NULL, &kv);
     if (NULL == kv) {
         fprintf(stderr, "ERROR GETTING WASSUP\n");
     } else {
@@ -209,13 +198,13 @@ static void *mylib(void *ptr)
     kv = OBJ_NEW(opal_value_t);
     kv->key = strdup(OPAL_PMIX_WAIT);
     kv->type = OPAL_INT;
-    kv->data.integer = 0;  // wait for all
+    kv->data.integer = 0; // wait for all
     opal_list_append(&directives, &kv->super);
 
     if (OPAL_SUCCESS != opal_pmix.lookup(&info, &directives)) {
         fprintf(stderr, "LOOKUP FAILED\n");
     } else {
-        pdata = (opal_pmix_pdata_t*)opal_list_get_first(&info);
+        pdata = (opal_pmix_pdata_t *) opal_list_get_first(&info);
         fprintf(stderr, "LOOKUP RETURNED %s\n", pdata->value.data.string);
     }
     OPAL_LIST_DESTRUCT(&info);
@@ -230,7 +219,7 @@ static void *mylib(void *ptr)
     return NULL;
 }
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int rank, size, rc;
     hwloc_cpuset_t cpus;
@@ -260,7 +249,7 @@ int main(int argc, char* argv[])
         goto done;
     }
 
-    printf("[%lu] Rank %d: getting topology\n", (unsigned long)pid, rank);
+    printf("[%lu] Rank %d: getting topology\n", (unsigned long) pid, rank);
     fflush(stdout);
     if (OPAL_SUCCESS == opal_hwloc_base_get_topology()) {
         cpus = hwloc_bitmap_alloc();
@@ -268,9 +257,8 @@ int main(int argc, char* argv[])
         hwloc_bitmap_list_asprintf(&bindings, cpus);
     }
 
-    printf("Hello, World, I am %d of %d [%d local peers]: get_cpubind: %d bitmap %s\n",
-           rank, size, orte_process_info.num_local_peers, rc,
-           (NULL == bindings) ? "NULL" : bindings);
+    printf("Hello, World, I am %d of %d [%d local peers]: get_cpubind: %d bitmap %s\n", rank, size,
+           orte_process_info.num_local_peers, rc, (NULL == bindings) ? "NULL" : bindings);
 
     /* publish something */
     OBJ_CONSTRUCT(&list, opal_list_t);
@@ -287,7 +275,7 @@ int main(int argc, char* argv[])
         fprintf(stderr, "Error joining thread\n");
     }
 
-  done:
+done:
     MPI_Finalize();
     return 0;
 }
diff --git a/test/simple/iof.c b/test/simple/iof.c
index 4ec85b88616..88d2b6d0fe9 100644
--- a/test/simple/iof.c
+++ b/test/simple/iof.c
@@ -1,11 +1,11 @@
-#include <stdlib.h>
+#include <mpi.h>
+#include <stdbool.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
-#include <stdbool.h>
 #include <unistd.h>
-#include <mpi.h>
 
-#define ORTE_IOF_BASE_MSG_MAX   2048
+#define ORTE_IOF_BASE_MSG_MAX 2048
 
 int main(int argc, char *argv[])
 {
@@ -45,7 +45,7 @@ int main(int argc, char *argv[])
             fprintf(stderr, "Rank %d: recvd blob %d\n", rank, pos);
             ++pos;
             done = true;
-            for (i=0; i < ORTE_IOF_BASE_MSG_MAX; i++) {
+            for (i = 0; i < ORTE_IOF_BASE_MSG_MAX; i++) {
                 if (0 != msg[i]) {
                     done = false;
                     break;
diff --git a/test/simple/loop_child.c b/test/simple/loop_child.c
index 7a5000abbfe..caeaac89210 100644
--- a/test/simple/loop_child.c
+++ b/test/simple/loop_child.c
@@ -1,13 +1,12 @@
-#include <string.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <unistd.h>
+#include "mpi.h"
 #include <pthread.h>
 #include <semaphore.h>
-#include "mpi.h"
-
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
 
-int main( int argc, char **argv )
+int main(int argc, char **argv)
 {
     MPI_Comm parent;
     MPI_Comm merged;
@@ -25,6 +24,6 @@ int main( int argc, char **argv )
     MPI_Comm_free(&merged);
     MPI_Comm_disconnect(&parent);
     MPI_Finalize();
-    printf("Child %d: exiting\n", (int)getpid());
+    printf("Child %d: exiting\n", (int) getpid());
     return 0;
 }
diff --git a/test/simple/loop_spawn.c b/test/simple/loop_spawn.c
index 1d5f19a1205..57596233046 100644
--- a/test/simple/loop_spawn.c
+++ b/test/simple/loop_spawn.c
@@ -1,14 +1,13 @@
 /*file .c : spawned  the file Exe*/
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
 #include "mpi.h"
+#include <errno.h>
 #include <pthread.h>
 #include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
 #include <sys/time.h>
-#include <errno.h>
-#define     EXE_TEST             "./loop_child"
-
+#include <unistd.h>
+#define EXE_TEST "./loop_child"
 
 int main(int argc, char **argv)
 {
@@ -26,23 +25,21 @@ int main(int argc, char **argv)
     printf("parent*******************************\n");
     printf("parent: Launching MPI*\n");
 
-    MPI_Init( &argc, &argv);
+    MPI_Init(&argc, &argv);
 
     for (iter = 0; iter < itermax; ++iter) {
-        MPI_Comm_spawn(EXE_TEST, NULL, 1, MPI_INFO_NULL,
-                       0, MPI_COMM_WORLD, &comm, &err);
+        MPI_Comm_spawn(EXE_TEST, NULL, 1, MPI_INFO_NULL, 0, MPI_COMM_WORLD, &comm, &err);
         printf("parent: MPI_Comm_spawn #%d return : %d\n", iter, err);
 
         MPI_Intercomm_merge(comm, 0, &merged);
         MPI_Comm_rank(merged, &rank);
         MPI_Comm_size(merged, &size);
-        printf("parent: MPI_Comm_spawn #%d rank %d, size %d\n",
-               iter, rank, size);
+        printf("parent: MPI_Comm_spawn #%d rank %d, size %d\n", iter, rank, size);
         MPI_Comm_free(&merged);
         MPI_Comm_disconnect(&comm);
     }
 
     MPI_Finalize();
-    printf("parent: End .\n" );
+    printf("parent: End .\n");
     return 0;
 }
diff --git a/test/simple/mpi_barrier.c b/test/simple/mpi_barrier.c
index 9eeef50b687..315258aec6d 100644
--- a/test/simple/mpi_barrier.c
+++ b/test/simple/mpi_barrier.c
@@ -5,10 +5,10 @@
  * The most basic of MPI applications
  */
 
-#include <stdio.h>
 #include "mpi.h"
+#include <stdio.h>
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     MPI_Init(&argc, &argv);
     MPI_Barrier(MPI_COMM_WORLD);
diff --git a/test/simple/mpi_info.c b/test/simple/mpi_info.c
index 6973c5e9080..186503e85cc 100644
--- a/test/simple/mpi_info.c
+++ b/test/simple/mpi_info.c
@@ -5,36 +5,27 @@
  * The most basic of MPI applications
  */
 
-#include <stdio.h>
 #include "mpi.h"
+#include <stdio.h>
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     char value[MPI_MAX_INFO_VAL];
     int flag;
-    char *keys[] = {
-        "command",
-        "argv",
-        "maxprocs",
-        "soft",
-        "host",
-        "arch",
-        "wdir",
-        "thread_level",
-        "ompi_num_apps",
-        "ompi_first_rank",
-        "ompi_np",
-        "ompi_positioned_file_dir"
-    };
+    char *keys[] = {"command",       "argv",
+                    "maxprocs",      "soft",
+                    "host",          "arch",
+                    "wdir",          "thread_level",
+                    "ompi_num_apps", "ompi_first_rank",
+                    "ompi_np",       "ompi_positioned_file_dir"};
     int i, nk;
 
     MPI_Init(&argc, &argv);
 
-    nk = sizeof(keys) / sizeof(char*);
+    nk = sizeof(keys) / sizeof(char *);
 
-    for (i=0; i < nk; i++) {
-        MPI_Info_get(MPI_INFO_ENV, keys[i], MPI_MAX_INFO_VAL,
-                     value, &flag);
+    for (i = 0; i < nk; i++) {
+        MPI_Info_get(MPI_INFO_ENV, keys[i], MPI_MAX_INFO_VAL, value, &flag);
         fprintf(stderr, "%s: %s\n", keys[i], (flag) ? value : "Not found");
     }
 
diff --git a/test/simple/mpi_no_op.c b/test/simple/mpi_no_op.c
index 1d382add87d..a38c1b7bf3e 100644
--- a/test/simple/mpi_no_op.c
+++ b/test/simple/mpi_no_op.c
@@ -5,10 +5,10 @@
  * The most basic of MPI applications
  */
 
-#include <stdio.h>
 #include "mpi.h"
+#include <stdio.h>
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     MPI_Init(&argc, &argv);
 
diff --git a/test/simple/mpi_spin.c b/test/simple/mpi_spin.c
index 3bd0d2069b9..a868f61f2e9 100644
--- a/test/simple/mpi_spin.c
+++ b/test/simple/mpi_spin.c
@@ -6,21 +6,19 @@
  * abnormal program termination
  */
 
-#include <stdio.h>
 #include "mpi.h"
+#include <stdio.h>
 
 #include "opal/dss/dss.h"
 #include "opal/mca/pmix/pmix.h"
 #include "opal/util/output.h"
-#include "orte/util/name_fns.h"
 #include "orte/constants.h"
+#include "orte/util/name_fns.h"
 
 static volatile bool register_active = false;
 
-static void _event_fn(int status,
-                      const opal_process_name_t *source,
-                      opal_list_t *info, opal_list_t *results,
-                      opal_pmix_notification_complete_fn_t cbfunc,
+static void _event_fn(int status, const opal_process_name_t *source, opal_list_t *info,
+                      opal_list_t *results, opal_pmix_notification_complete_fn_t cbfunc,
                       void *cbdata)
 {
     opal_value_t *kv;
@@ -29,7 +27,7 @@ static void _event_fn(int status,
     /* the name of the terminating proc should be on the info list */
     proc.jobid = ORTE_JOBID_INVALID;
     proc.vpid = ORTE_VPID_INVALID;
-    OPAL_LIST_FOREACH(kv, info, opal_value_t) {
+    OPAL_LIST_FOREACH (kv, info, opal_value_t) {
         if (0 == strcmp(kv->key, OPAL_PMIX_EVENT_AFFECTED_PROC)) {
             proc.jobid = kv->data.name.jobid;
             proc.vpid = kv->data.name.vpid;
@@ -37,8 +35,7 @@ static void _event_fn(int status,
         }
     }
 
-    opal_output(0, "NOTIFIED OF TERMINATION OF PROC %s",
-                ORTE_NAME_PRINT(&proc));
+    opal_output(0, "NOTIFIED OF TERMINATION OF PROC %s", ORTE_NAME_PRINT(&proc));
 
     /* must let the notifier know we are done */
     if (NULL != cbfunc) {
@@ -46,18 +43,15 @@ static void _event_fn(int status,
     }
 }
 
-static void _register_fn(int status,
-                         size_t evhandler_ref,
-                         void *cbdata)
+static void _register_fn(int status, size_t evhandler_ref, void *cbdata)
 {
-    opal_list_t *codes = (opal_list_t*)cbdata;
+    opal_list_t *codes = (opal_list_t *) cbdata;
 
     OPAL_LIST_RELEASE(codes);
     register_active = false;
 }
 
-
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
 
     int i;
@@ -82,7 +76,8 @@ int main(int argc, char* argv[])
     while (1) {
         i++;
         pi = i / 3.14159256;
-        if (i > 100) i = 0;
+        if (i > 100)
+            i = 0;
     }
 
     MPI_Finalize();
diff --git a/test/simple/multi_abort.c b/test/simple/multi_abort.c
index e76b0d0de0e..8849cfd7f4e 100644
--- a/test/simple/multi_abort.c
+++ b/test/simple/multi_abort.c
@@ -5,10 +5,10 @@
  * The most basic of MPI applications
  */
 
-#include <stdio.h>
 #include "mpi.h"
+#include <stdio.h>
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int rank, size;
 
@@ -18,7 +18,8 @@ int main(int argc, char* argv[])
 
     printf("Hello, World, I am %d of %d\n", rank, size);
 
-    if (0 != rank) MPI_Abort(MPI_COMM_WORLD, rank);
+    if (0 != rank)
+        MPI_Abort(MPI_COMM_WORLD, rank);
 
     MPI_Finalize();
     return 0;
diff --git a/test/simple/no-disconnect.c b/test/simple/no-disconnect.c
index db54de80f00..d493a1bd946 100644
--- a/test/simple/no-disconnect.c
+++ b/test/simple/no-disconnect.c
@@ -9,12 +9,12 @@
    as of Open MPI 1.7.  However, we hope this feature will
    work in 2.0. with the new state machine based orte. */
 
-#include <unistd.h>
-#include <stdlib.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
-#include <time.h>
 #include <sys/utsname.h>
+#include <time.h>
+#include <unistd.h>
 
 #include <mpi.h>
 
@@ -31,14 +31,15 @@ const int max_depth = 4;
  *     MPI_Barrier with  nice_barrier
  */
 
-
-static int nice_send(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm) {
+static int nice_send(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm)
+{
     /* Assume a standard (presumably short/eager) send suffices. */
     return MPI_Send(buf, count, datatype, dest, tag, comm);
 }
 
-
-static int nice_recv(void *buf, int count, MPI_Datatype datatype, int source, int tag, MPI_Comm comm, MPI_Status *status) {
+static int nice_recv(void *buf, int count, MPI_Datatype datatype, int source, int tag,
+                     MPI_Comm comm, MPI_Status *status)
+{
     MPI_Request req;
     int flag;
     struct timespec dt;
@@ -51,53 +52,52 @@ static int nice_recv(void *buf, int count, MPI_Datatype datatype, int source, in
      * something less than a millisecond should be sufficient.
      * Excessive sleep times (e.g., 1 second) would degrade performance.
      */
-    dt.tv_sec    =       0;
-    dt.tv_nsec   =  100000;
+    dt.tv_sec = 0;
+    dt.tv_nsec = 100000;
 
     MPI_Irecv(buf, count, datatype, source, tag, comm, &req);
 
     MPI_Test(&req, &flag, status);
-    while ( ! flag ) {
+    while (!flag) {
         nanosleep(&dt, NULL);
         MPI_Test(&req, &flag, status);
     }
     return MPI_SUCCESS;
 }
 
-
-static void nice_barrier(MPI_Comm comm) {
+static void nice_barrier(MPI_Comm comm)
+{
     int me, np, jump, buf = -1;
 
-    MPI_Comm_rank(comm,&me);
-    MPI_Comm_size(comm,&np);
+    MPI_Comm_rank(comm, &me);
+    MPI_Comm_size(comm, &np);
 
     /* fan in */
-    for ( jump = 1; jump < np; jump <<= 1 ) {
-        if ( ( me & jump ) != 0 ) {
+    for (jump = 1; jump < np; jump <<= 1) {
+        if ((me & jump) != 0) {
             nice_send(&buf, 1, MPI_INT, me - jump, 343, comm);
             break;
-        } else if ( me + jump < np ) {
+        } else if (me + jump < np) {
             nice_recv(&buf, 1, MPI_INT, me + jump, 343, comm, MPI_STATUS_IGNORE);
         }
     }
 
     /* fan out */
-    if ( 0 != me ) {
+    if (0 != me) {
         nice_recv(&buf, 1, MPI_INT, me - jump, 344, comm, MPI_STATUS_IGNORE);
     }
     jump >>= 1;
-    for ( ; jump > 0; jump >>= 1 ) {
-        if ( me + jump < np ) {
+    for (; jump > 0; jump >>= 1) {
+        if (me + jump < np) {
             nice_send(&buf, 1, MPI_INT, me + jump, 344, comm);
         }
     }
 }
 
-
-int main (int argc, char **argv)
+int main(int argc, char **argv)
 {
-    char bufs   [NCHARS];               /* send buffer  */
-    char bufr[2][NCHARS];               /* recv buffers */
+    char bufs[NCHARS];    /* send buffer  */
+    char bufr[2][NCHARS]; /* recv buffers */
     MPI_Comm parent;
     int level = 0, participate = 1;
     struct utsname buf;
@@ -108,13 +108,14 @@ int main (int argc, char **argv)
         printf("Skipping, because the orte cannot handle concurrent MPI_Comm_spawns\n");
         return 77;
     } else {
-        printf("Verify that this test is truly working because conncurrent MPI_Comm_spawns"
+        printf("Verify that this test is truly working because concurrent MPI_Comm_spawns"
                " has not worked before.\n");
     }
 #endif
 
     uname(&buf);
-    printf("I AM pid %d with level %d on %s\n", getpid(), (argc < 2)?0:atoi(argv[1]), buf.nodename);
+    printf("I AM pid %d with level %d on %s\n", getpid(), (argc < 2) ? 0 : atoi(argv[1]),
+           buf.nodename);
 
     MPI_Init(&argc, &argv);
     MPI_Comm_get_parent(&parent);
@@ -122,8 +123,8 @@ int main (int argc, char **argv)
     if (MPI_COMM_NULL != parent) {
         /* spawned processes get stuff from parent */
         level = atoi(argv[1]);
-        MPI_Recv(&bufr[0], sizeof(char)*NCHARS, MPI_CHAR, MPI_ANY_SOURCE,
-                 MPI_ANY_TAG, parent, MPI_STATUS_IGNORE);
+        MPI_Recv(&bufr[0], sizeof(char) * NCHARS, MPI_CHAR, MPI_ANY_SOURCE, MPI_ANY_TAG, parent,
+                 MPI_STATUS_IGNORE);
         printf("Parent sent: %s\n", bufr[0]);
     } else {
 
@@ -142,60 +143,62 @@ int main (int argc, char **argv)
 
         int me, np;
 
-        MPI_Comm_size(MPI_COMM_WORLD,&np);
-        MPI_Comm_rank(MPI_COMM_WORLD,&me);
+        MPI_Comm_size(MPI_COMM_WORLD, &np);
+        MPI_Comm_rank(MPI_COMM_WORLD, &me);
 
-        if ( np > 4 ) {
+        if (np > 4) {
             /* turn off all but every 4th process */
-            if ( ( me & 3 ) != 0 ) participate = 0;
-        } else
-        if ( np > 2 ) {
+            if ((me & 3) != 0)
+                participate = 0;
+        } else if (np > 2) {
             /* turn off all but every 2nd process */
-            if ( ( me & 1 ) != 0 ) participate = 0;
+            if ((me & 1) != 0)
+                participate = 0;
         }
     }
 
     /* all spawned processes and selected "root" processes participate */
-    if ( participate ) {
+    if (participate) {
         printf("level = %d\n", level);
 
         /* prepare send buffer */
-        sprintf(bufs,"level %d (pid:%d)", level, getpid());
+        sprintf(bufs, "level %d (pid:%d)", level, getpid());
 
         /* spawn */
         if (level < max_depth) {
             int i, nspawn = 2, errcodes[1];
             MPI_Request req[2];
-            MPI_Comm   comm[2];
+            MPI_Comm comm[2];
             char argv1[NCHARS];
             char *args[2];
 
             /* level 0 spawns only one process to mimic the original test */
-            if ( level == 0 ) nspawn = 1;
+            if (level == 0)
+                nspawn = 1;
 
             /* prepare command line arguments */
-            snprintf(argv1, sizeof(argv1), "%d", level+1);
+            snprintf(argv1, sizeof(argv1), "%d", level + 1);
             args[0] = argv1;
             args[1] = NULL;
 
             /* spawn, with a message sent to and received from each child */
-            for ( i = 0; i < nspawn; i++ ) {
-                MPI_Comm_spawn(argv[0], args, 1, MPI_INFO_NULL, 0, MPI_COMM_SELF,
-                               &comm[i], errcodes);
-                MPI_Send(&bufs, sizeof(char)*NCHARS, MPI_CHAR, 0, 100, comm[i]);
-                MPI_Irecv(&bufr[i], sizeof(char)*NCHARS, MPI_CHAR, MPI_ANY_SOURCE,
-                          MPI_ANY_TAG, comm[i], &req[i]);
+            for (i = 0; i < nspawn; i++) {
+                MPI_Comm_spawn(argv[0], args, 1, MPI_INFO_NULL, 0, MPI_COMM_SELF, &comm[i],
+                               errcodes);
+                MPI_Send(&bufs, sizeof(char) * NCHARS, MPI_CHAR, 0, 100, comm[i]);
+                MPI_Irecv(&bufr[i], sizeof(char) * NCHARS, MPI_CHAR, MPI_ANY_SOURCE, MPI_ANY_TAG,
+                          comm[i], &req[i]);
             }
 
             /* wait for messages from children and print them */
             MPI_Waitall(nspawn, req, MPI_STATUSES_IGNORE);
-            for ( i = 0; i < nspawn; i++ )
+            for (i = 0; i < nspawn; i++)
                 printf("Child %d sent: %s\n", i, bufr[i]);
         }
 
         /* send message back to parent */
         if (MPI_COMM_NULL != parent) {
-            MPI_Send(&bufs, sizeof(char)*NCHARS, MPI_CHAR, 0, 100, parent);
+            MPI_Send(&bufs, sizeof(char) * NCHARS, MPI_CHAR, 0, 100, parent);
         }
     }
 
diff --git a/test/simple/nonzero.c b/test/simple/nonzero.c
index 4b7ff266bfe..20e2393960b 100644
--- a/test/simple/nonzero.c
+++ b/test/simple/nonzero.c
@@ -1,16 +1,16 @@
+#include <mpi.h>
 #include <stdlib.h>
 #include <unistd.h>
-#include <mpi.h>
 
 int main(int argc, char **argv)
 {
     int rank;
 
-    if(argc < 2) {
+    if (argc < 2) {
         return 0;
     }
     MPI_Init(&argc, &argv);
-    MPI_Comm_rank(MPI_COMM_WORLD,&rank);
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
 
     int i = atoi(argv[1]);
 
diff --git a/test/simple/parallel_r64.c b/test/simple/parallel_r64.c
index c7354b4f4cf..7ca5219417f 100644
--- a/test/simple/parallel_r64.c
+++ b/test/simple/parallel_r64.c
@@ -5,218 +5,208 @@
 #include <stdio.h>
 #include <string.h>
 
-#define D      3             /* dimensions */
+#define D 3 /* dimensions */
 
-#define X   1024             /* global x grid size */
-#define Y   1024             /* global y grid size */
-#define Z   1024             /* global z grid size */
+#define X 1024 /* global x grid size */
+#define Y 1024 /* global y grid size */
+#define Z 1024 /* global z grid size */
 
-#define nx   256             /* local x grid size */
-#define ny   256             /* local y grid size */
-#define nz   256             /* local z grid size */
+#define nx 256 /* local x grid size */
+#define ny 256 /* local y grid size */
+#define nz 256 /* local z grid size */
 
-#define ng (nx*ny*nz)        /* local grid (cube) size */
+#define ng (nx * ny * nz) /* local grid (cube) size */
 
-#define npx    4             /* number of PE's in x direction */
-#define npy    4             /* number of PE's in y direction */
-#define npz    4             /* number of PE's in z direction */
+#define npx 4 /* number of PE's in x direction */
+#define npy 4 /* number of PE's in y direction */
+#define npz 4 /* number of PE's in z direction */
 
-#define np (npx*npy*npz)  /* total PE count */
+#define np (npx * npy * npz) /* total PE count */
 
 #define LOOP 1
 
 #define MAX_RR_NAME 7
 
-int
-main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
-  int  i, rank, npes, bug=0;
-  int buf[ng];
-  MPI_File     thefile;
-  MPI_Status   status;
-  MPI_Datatype filetype;
-  MPI_Comm     new_comm;
-  MPI_Offset   offset=0;
-  MPI_Info     info=MPI_INFO_NULL;
-  int gsize[D],distrib[D],dargs[D],psize[D];
-  int dims[D],periods[D],reorder;
-  double t1,t2,mbs;
-  double to1,to2,tc1,tc2;
-  double et,eto,etc;
-  double max_mbs,min_mbs,avg_mbs;
-  double max_et,min_et,avg_et;
-  double max_eto,min_eto,avg_eto;
-  double max_etc,min_etc,avg_etc;
-  char process_name[MPI_MAX_PROCESSOR_NAME + 1];
-  char rr_blank[] = {"       "};
-  char rr_empty[] = {"???????"};
-  int  count;
-
-  MPI_Init(&argc, &argv);
-  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-  MPI_Comm_size(MPI_COMM_WORLD, &npes);
-  if ( rank == 0 )
-    {
-     if ( argc < 2 )
-       {
-        printf(" ERROR: no filename given\n");
-        bug++;
-       }
-     if ( npes == np )
-       {
-        printf(" file name: %s\n",argv[1]);
-        printf(" total number of PE's: %3d\n",np);
-        printf(" number of PE's in x direction: %3d\n",npx);
-        printf(" number of PE's in y direction: %3d\n",npy);
-        printf(" number of PE's in z direction: %3d\n",npz);
-        printf(" global grid size: %dx%dx%d 4 byte integers (total %lu)\n",X,Y,Z,(unsigned long)X*Y*Z);
-        printf("  local grid size: %dx%dx%d 4 byte integers (total %d)\n",nx,ny,nz,ng);
-       }
-     else
-       {
-        printf(" ERROR: total number of PE's must be %d\n",np);
-        printf("        actual number of PE's was %d\n",npes);
-        bug++;
-       }
-     if ( bug )
-       {
-        MPI_Abort(MPI_COMM_WORLD,-1);
-       }
+    int i, rank, npes, bug = 0;
+    int buf[ng];
+    MPI_File thefile;
+    MPI_Status status;
+    MPI_Datatype filetype;
+    MPI_Comm new_comm;
+    MPI_Offset offset = 0;
+    MPI_Info info = MPI_INFO_NULL;
+    int gsize[D], distrib[D], dargs[D], psize[D];
+    int dims[D], periods[D], reorder;
+    double t1, t2, mbs;
+    double to1, to2, tc1, tc2;
+    double et, eto, etc;
+    double max_mbs, min_mbs, avg_mbs;
+    double max_et, min_et, avg_et;
+    double max_eto, min_eto, avg_eto;
+    double max_etc, min_etc, avg_etc;
+    char process_name[MPI_MAX_PROCESSOR_NAME + 1];
+    char rr_blank[] = {"       "};
+    char rr_empty[] = {"???????"};
+    int count;
+
+    MPI_Init(&argc, &argv);
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    MPI_Comm_size(MPI_COMM_WORLD, &npes);
+    if (rank == 0) {
+        if (argc < 2) {
+            printf(" ERROR: no filename given\n");
+            bug++;
+        }
+        if (npes == np) {
+            printf(" file name: %s\n", argv[1]);
+            printf(" total number of PE's: %3d\n", np);
+            printf(" number of PE's in x direction: %3d\n", npx);
+            printf(" number of PE's in y direction: %3d\n", npy);
+            printf(" number of PE's in z direction: %3d\n", npz);
+            printf(" global grid size: %dx%dx%d 4 byte integers (total %lu)\n", X, Y, Z,
+                   (unsigned long) X * Y * Z);
+            printf("  local grid size: %dx%dx%d 4 byte integers (total %d)\n", nx, ny, nz, ng);
+        } else {
+            printf(" ERROR: total number of PE's must be %d\n", np);
+            printf("        actual number of PE's was %d\n", npes);
+            bug++;
+        }
+        if (bug) {
+            MPI_Abort(MPI_COMM_WORLD, -1);
+        }
+    }
+    if (MPI_Get_processor_name(process_name, &count) != MPI_SUCCESS) {
+        sprintf(process_name, "%s", rr_empty);
+    } else {
+        if (count < MAX_RR_NAME)
+            strncat(&process_name[count], rr_blank, MAX_RR_NAME - count);
+        process_name[MAX_RR_NAME] = '\0';
     }
- if ( MPI_Get_processor_name(process_name, &count) != MPI_SUCCESS)
-   {
-    sprintf(process_name, "%s", rr_empty);
-   }
- else
-   {
-    if (count < MAX_RR_NAME) strncat(&process_name[count],rr_blank,MAX_RR_NAME-count);
-    process_name[MAX_RR_NAME] = '\0';
-   }
 
-  MPI_Info_create(&info);
+    MPI_Info_create(&info);
 
-/* allow multiple writers to write to the file concurrently */
+    /* allow multiple writers to write to the file concurrently */
 
-/*MPI_Info_set(info,"panfs_concurrent_write","1");*/
+    /*MPI_Info_set(info,"panfs_concurrent_write","1");*/
 
-/* use data aggregation */
+    /* use data aggregation */
 
-/*MPI_Info_set(info,"romio_cb_write","enable"); */
-/*MPI_Info_set(info,"romio_cb_write","disable");*/
-/*MPI_Info_set(info,"romio_cb_read","enable"); */
-/*MPI_Info_set(info,"romio_cb_read","disable");*/
+    /*MPI_Info_set(info,"romio_cb_write","enable"); */
+    /*MPI_Info_set(info,"romio_cb_write","disable");*/
+    /*MPI_Info_set(info,"romio_cb_read","enable"); */
+    /*MPI_Info_set(info,"romio_cb_read","disable");*/
 
-/* use one aggregator/writer per node */
+    /* use one aggregator/writer per node */
 
-/*MPI_Info_set(info,"cb_config_list","*:1");*/
+    /*MPI_Info_set(info,"cb_config_list","*:1");*/
 
-/* aggregators/writers per allocation: use this or the above (both work) */
+    /* aggregators/writers per allocation: use this or the above (both work) */
 
-/*i = ((npes-1)/8) + 1;
-  sprintf(awpa,"%d",i);
-  MPI_Info_set (info,"cb_nodes",awpa);*/
+    /*i = ((npes-1)/8) + 1;
+      sprintf(awpa,"%d",i);
+      MPI_Info_set (info,"cb_nodes",awpa);*/
 
-  for ( i=0; i<D; i++ )
-    {
-     periods[i] = 1;  /* true */
+    for (i = 0; i < D; i++) {
+        periods[i] = 1; /* true */
     }
 
-  reorder = 1;        /* true */
+    reorder = 1; /* true */
 
-  dims[0] = npx;
-  dims[1] = npy;
-  dims[2] = npz;
+    dims[0] = npx;
+    dims[1] = npy;
+    dims[2] = npz;
 
-  MPI_Cart_create(MPI_COMM_WORLD, D, dims, periods, reorder, &new_comm);
+    MPI_Cart_create(MPI_COMM_WORLD, D, dims, periods, reorder, &new_comm);
 
-  for ( i=0; i<D; i++ )
-    {
-     distrib[i] = MPI_DISTRIBUTE_BLOCK;
-     dargs[i]   = MPI_DISTRIBUTE_DFLT_DARG;
-/*   psize[i]   = 0; */
+    for (i = 0; i < D; i++) {
+        distrib[i] = MPI_DISTRIBUTE_BLOCK;
+        dargs[i] = MPI_DISTRIBUTE_DFLT_DARG;
+        /*   psize[i]   = 0; */
     }
 
-  gsize[0] = X;
-  gsize[1] = Y;
-  gsize[2] = Z;
+    gsize[0] = X;
+    gsize[1] = Y;
+    gsize[2] = Z;
 
-  psize[0] = npx;
-  psize[1] = npy;
-  psize[2] = npz;
+    psize[0] = npx;
+    psize[1] = npy;
+    psize[2] = npz;
 
-/*
-  MPI_Dims_create(npes, D, psize);
+    /*
+      MPI_Dims_create(npes, D, psize);
 
-  printf("psize %d %d %d\n",psize[0],psize[1],psize[2]);
-*/
+      printf("psize %d %d %d\n",psize[0],psize[1],psize[2]);
+    */
 
-  MPI_Type_create_darray(npes, rank, D, gsize, distrib, dargs, psize, MPI_ORDER_FORTRAN, MPI_INT, &filetype);
+    MPI_Type_create_darray(npes, rank, D, gsize, distrib, dargs, psize, MPI_ORDER_FORTRAN, MPI_INT,
+                           &filetype);
 
-  MPI_Type_commit(&filetype);
+    MPI_Type_commit(&filetype);
 
-  to1 = MPI_Wtime();
-  MPI_File_open(new_comm, argv[1], MPI_MODE_RDONLY, info, &thefile);
-  to2 = MPI_Wtime();
+    to1 = MPI_Wtime();
+    MPI_File_open(new_comm, argv[1], MPI_MODE_RDONLY, info, &thefile);
+    to2 = MPI_Wtime();
 
-  MPI_File_set_view(thefile, offset, MPI_INT, filetype, "native", MPI_INFO_NULL);
+    MPI_File_set_view(thefile, offset, MPI_INT, filetype, "native", MPI_INFO_NULL);
 
-  t1 = MPI_Wtime();
-  for ( i=0; i<LOOP; i++ )
-    {
-     MPI_File_read_all(thefile, buf, ng, MPI_INT, &status);
+    t1 = MPI_Wtime();
+    for (i = 0; i < LOOP; i++) {
+        MPI_File_read_all(thefile, buf, ng, MPI_INT, &status);
     }
-  t2 = MPI_Wtime();
+    t2 = MPI_Wtime();
 
-/*MPI_File_sync(thefile); */
+    /*MPI_File_sync(thefile); */
 
-  tc1 = MPI_Wtime();
-  MPI_File_close(&thefile);
-  tc2 = MPI_Wtime();
+    tc1 = MPI_Wtime();
+    MPI_File_close(&thefile);
+    tc2 = MPI_Wtime();
 
-  et  = (t2  - t1)/LOOP;
-  eto = (to2 - to1)/LOOP;
-  etc = (tc2 - tc1)/LOOP;
+    et = (t2 - t1) / LOOP;
+    eto = (to2 - to1) / LOOP;
+    etc = (tc2 - tc1) / LOOP;
 
-  mbs = (((double)(LOOP*X*Y*Z)*sizeof(int)))/(1000000.0*(t2-t1));
+    mbs = (((double) (LOOP * X * Y * Z) * sizeof(int))) / (1000000.0 * (t2 - t1));
 
-/*printf(" %s[%3d]    ET  %5.2f  %6.2f  %6.2f     %5.1f mbs       Data %9d %9d \n", process_name, rank, t1, t2, t2-t1, mbs, buf[0], buf[ng-1]);*/
+    /*printf(" %s[%3d]    ET  %5.2f  %6.2f  %6.2f     %5.1f mbs       Data %9d %9d \n",
+     * process_name, rank, t1, t2, t2-t1, mbs, buf[0], buf[ng-1]);*/
 
-  MPI_Barrier(MPI_COMM_WORLD);
+    MPI_Barrier(MPI_COMM_WORLD);
 
-  MPI_Reduce(&mbs, &avg_mbs, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
-  MPI_Reduce(&mbs, &min_mbs, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
-  MPI_Reduce(&mbs, &max_mbs, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&mbs, &avg_mbs, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&mbs, &min_mbs, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&mbs, &max_mbs, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
 
-  MPI_Reduce(&et, &avg_et, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
-  MPI_Reduce(&et, &min_et, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
-  MPI_Reduce(&et, &max_et, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&et, &avg_et, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&et, &min_et, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&et, &max_et, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
 
-  MPI_Reduce(&eto, &avg_eto, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
-  MPI_Reduce(&eto, &min_eto, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
-  MPI_Reduce(&eto, &max_eto, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&eto, &avg_eto, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&eto, &min_eto, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&eto, &max_eto, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
 
-  MPI_Reduce(&etc, &avg_etc, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
-  MPI_Reduce(&etc, &min_etc, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
-  MPI_Reduce(&etc, &max_etc, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&etc, &avg_etc, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&etc, &min_etc, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&etc, &max_etc, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
 
-  fflush(stdout);
+    fflush(stdout);
 
-  if ( rank == 0 )
-    {
-     mbs = avg_mbs/npes;
-     printf("\n     average read rate: %9.1f mbs\n", mbs);
-     printf("     minimum read rate: %9.1f mbs\n", min_mbs);
-     printf("     maximum read rate: %9.1f mbs\n\n", max_mbs);
-     avg_eto = avg_eto/npes;
-     avg_et  = avg_et/npes;
-     avg_etc = avg_etc/npes;
-     printf("     open time:  %9.3f min %9.3f avg %9.3f max\n",min_eto,avg_eto,max_eto);
-     printf("     read time:  %9.3f min %9.3f avg %9.3f max\n",min_et,avg_et,max_et);
-     printf("     close time: %9.3f min %9.3f avg %9.3f max\n\n",min_etc,avg_etc,max_etc);
-     fflush(stdout);
+    if (rank == 0) {
+        mbs = avg_mbs / npes;
+        printf("\n     average read rate: %9.1f mbs\n", mbs);
+        printf("     minimum read rate: %9.1f mbs\n", min_mbs);
+        printf("     maximum read rate: %9.1f mbs\n\n", max_mbs);
+        avg_eto = avg_eto / npes;
+        avg_et = avg_et / npes;
+        avg_etc = avg_etc / npes;
+        printf("     open time:  %9.3f min %9.3f avg %9.3f max\n", min_eto, avg_eto, max_eto);
+        printf("     read time:  %9.3f min %9.3f avg %9.3f max\n", min_et, avg_et, max_et);
+        printf("     close time: %9.3f min %9.3f avg %9.3f max\n\n", min_etc, avg_etc, max_etc);
+        fflush(stdout);
     }
 
-  MPI_Finalize();
+    MPI_Finalize();
 
-  return 0;
+    return 0;
 }
diff --git a/test/simple/parallel_r8.c b/test/simple/parallel_r8.c
index ad3f5f7ad11..b8541e17ed5 100644
--- a/test/simple/parallel_r8.c
+++ b/test/simple/parallel_r8.c
@@ -5,218 +5,208 @@
 #include <stdio.h>
 #include <string.h>
 
-#define D      3             /* dimensions */
+#define D 3 /* dimensions */
 
-#define X   256             /* global x grid size */
-#define Y   256             /* global y grid size */
-#define Z   256             /* global z grid size */
+#define X 256 /* global x grid size */
+#define Y 256 /* global y grid size */
+#define Z 256 /* global z grid size */
 
-#define nx   128             /* local x grid size */
-#define ny   128             /* local y grid size */
-#define nz   128             /* local z grid size */
+#define nx 128 /* local x grid size */
+#define ny 128 /* local y grid size */
+#define nz 128 /* local z grid size */
 
-#define ng (nx*ny*nz)        /* local grid (cube) size */
+#define ng (nx * ny * nz) /* local grid (cube) size */
 
-#define npx    2             /* number of PE's in x direction */
-#define npy    2             /* number of PE's in y direction */
-#define npz    2             /* number of PE's in z direction */
+#define npx 2 /* number of PE's in x direction */
+#define npy 2 /* number of PE's in y direction */
+#define npz 2 /* number of PE's in z direction */
 
-#define np (npx*npy*npz)  /* total PE count */
+#define np (npx * npy * npz) /* total PE count */
 
 #define LOOP 1
 
 #define MAX_RR_NAME 7
 
-int
-main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
-  int  i, rank, npes, bug=0;
-  int buf[ng];
-  MPI_File     thefile;
-  MPI_Status   status;
-  MPI_Datatype filetype;
-  MPI_Comm     new_comm;
-  MPI_Offset   offset=0;
-  MPI_Info     info=MPI_INFO_NULL;
-  int gsize[D],distrib[D],dargs[D],psize[D];
-  int dims[D],periods[D],reorder;
-  double t1,t2,mbs;
-  double to1,to2,tc1,tc2;
-  double et,eto,etc;
-  double max_mbs,min_mbs,avg_mbs;
-  double max_et,min_et,avg_et;
-  double max_eto,min_eto,avg_eto;
-  double max_etc,min_etc,avg_etc;
-  char process_name[MPI_MAX_PROCESSOR_NAME + 1];
-  char rr_blank[] = {"       "};
-  char rr_empty[] = {"???????"};
-  int  count;
-
-  MPI_Init(&argc, &argv);
-  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-  MPI_Comm_size(MPI_COMM_WORLD, &npes);
-  if ( rank == 0 )
-    {
-     if ( argc < 2 )
-       {
-        printf(" ERROR: no filename given\n");
-        bug++;
-       }
-     if ( npes == np )
-       {
-        printf(" file name: %s\n",argv[1]);
-        printf(" total number of PE's: %3d\n",np);
-        printf(" number of PE's in x direction: %3d\n",npx);
-        printf(" number of PE's in y direction: %3d\n",npy);
-        printf(" number of PE's in z direction: %3d\n",npz);
-        printf(" global grid size: %dx%dx%d 4 byte integers (total %lu)\n",X,Y,Z,(unsigned long)X*Y*Z);
-        printf("  local grid size: %dx%dx%d 4 byte integers (total %d)\n",nx,ny,nz,ng);
-       }
-     else
-       {
-        printf(" ERROR: total number of PE's must be %d\n",np);
-        printf("        actual number of PE's was %d\n",npes);
-        bug++;
-       }
-     if ( bug )
-       {
-        MPI_Abort(MPI_COMM_WORLD,-1);
-       }
+    int i, rank, npes, bug = 0;
+    int buf[ng];
+    MPI_File thefile;
+    MPI_Status status;
+    MPI_Datatype filetype;
+    MPI_Comm new_comm;
+    MPI_Offset offset = 0;
+    MPI_Info info = MPI_INFO_NULL;
+    int gsize[D], distrib[D], dargs[D], psize[D];
+    int dims[D], periods[D], reorder;
+    double t1, t2, mbs;
+    double to1, to2, tc1, tc2;
+    double et, eto, etc;
+    double max_mbs, min_mbs, avg_mbs;
+    double max_et, min_et, avg_et;
+    double max_eto, min_eto, avg_eto;
+    double max_etc, min_etc, avg_etc;
+    char process_name[MPI_MAX_PROCESSOR_NAME + 1];
+    char rr_blank[] = {"       "};
+    char rr_empty[] = {"???????"};
+    int count;
+
+    MPI_Init(&argc, &argv);
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    MPI_Comm_size(MPI_COMM_WORLD, &npes);
+    if (rank == 0) {
+        if (argc < 2) {
+            printf(" ERROR: no filename given\n");
+            bug++;
+        }
+        if (npes == np) {
+            printf(" file name: %s\n", argv[1]);
+            printf(" total number of PE's: %3d\n", np);
+            printf(" number of PE's in x direction: %3d\n", npx);
+            printf(" number of PE's in y direction: %3d\n", npy);
+            printf(" number of PE's in z direction: %3d\n", npz);
+            printf(" global grid size: %dx%dx%d 4 byte integers (total %lu)\n", X, Y, Z,
+                   (unsigned long) X * Y * Z);
+            printf("  local grid size: %dx%dx%d 4 byte integers (total %d)\n", nx, ny, nz, ng);
+        } else {
+            printf(" ERROR: total number of PE's must be %d\n", np);
+            printf("        actual number of PE's was %d\n", npes);
+            bug++;
+        }
+        if (bug) {
+            MPI_Abort(MPI_COMM_WORLD, -1);
+        }
+    }
+    if (MPI_Get_processor_name(process_name, &count) != MPI_SUCCESS) {
+        sprintf(process_name, "%s", rr_empty);
+    } else {
+        if (count < MAX_RR_NAME)
+            strncat(&process_name[count], rr_blank, MAX_RR_NAME - count);
+        process_name[MAX_RR_NAME] = '\0';
     }
- if ( MPI_Get_processor_name(process_name, &count) != MPI_SUCCESS)
-   {
-    sprintf(process_name, "%s", rr_empty);
-   }
- else
-   {
-    if (count < MAX_RR_NAME) strncat(&process_name[count],rr_blank,MAX_RR_NAME-count);
-    process_name[MAX_RR_NAME] = '\0';
-   }
 
-  MPI_Info_create(&info);
+    MPI_Info_create(&info);
 
-/* allow multiple writers to write to the file concurrently */
+    /* allow multiple writers to write to the file concurrently */
 
-/*MPI_Info_set(info,"panfs_concurrent_write","1");*/
+    /*MPI_Info_set(info,"panfs_concurrent_write","1");*/
 
-/* use data aggregation */
+    /* use data aggregation */
 
-/*MPI_Info_set(info,"romio_cb_write","enable"); */
-/*MPI_Info_set(info,"romio_cb_write","disable");*/
-/*MPI_Info_set(info,"romio_cb_read","enable"); */
-/*MPI_Info_set(info,"romio_cb_read","disable");*/
+    /*MPI_Info_set(info,"romio_cb_write","enable"); */
+    /*MPI_Info_set(info,"romio_cb_write","disable");*/
+    /*MPI_Info_set(info,"romio_cb_read","enable"); */
+    /*MPI_Info_set(info,"romio_cb_read","disable");*/
 
-/* use one aggregator/writer per node */
+    /* use one aggregator/writer per node */
 
-/*MPI_Info_set(info,"cb_config_list","*:1");*/
+    /*MPI_Info_set(info,"cb_config_list","*:1");*/
 
-/* aggregators/writers per allocation: use this or the above (both work) */
+    /* aggregators/writers per allocation: use this or the above (both work) */
 
-/*i = ((npes-1)/8) + 1;
-  sprintf(awpa,"%d",i);
-  MPI_Info_set (info,"cb_nodes",awpa);*/
+    /*i = ((npes-1)/8) + 1;
+      sprintf(awpa,"%d",i);
+      MPI_Info_set (info,"cb_nodes",awpa);*/
 
-  for ( i=0; i<D; i++ )
-    {
-     periods[i] = 1;  /* true */
+    for (i = 0; i < D; i++) {
+        periods[i] = 1; /* true */
     }
 
-  reorder = 1;        /* true */
+    reorder = 1; /* true */
 
-  dims[0] = npx;
-  dims[1] = npy;
-  dims[2] = npz;
+    dims[0] = npx;
+    dims[1] = npy;
+    dims[2] = npz;
 
-  MPI_Cart_create(MPI_COMM_WORLD, D, dims, periods, reorder, &new_comm);
+    MPI_Cart_create(MPI_COMM_WORLD, D, dims, periods, reorder, &new_comm);
 
-  for ( i=0; i<D; i++ )
-    {
-     distrib[i] = MPI_DISTRIBUTE_BLOCK;
-     dargs[i]   = MPI_DISTRIBUTE_DFLT_DARG;
-/*   psize[i]   = 0; */
+    for (i = 0; i < D; i++) {
+        distrib[i] = MPI_DISTRIBUTE_BLOCK;
+        dargs[i] = MPI_DISTRIBUTE_DFLT_DARG;
+        /*   psize[i]   = 0; */
     }
 
-  gsize[0] = X;
-  gsize[1] = Y;
-  gsize[2] = Z;
+    gsize[0] = X;
+    gsize[1] = Y;
+    gsize[2] = Z;
 
-  psize[0] = npx;
-  psize[1] = npy;
-  psize[2] = npz;
+    psize[0] = npx;
+    psize[1] = npy;
+    psize[2] = npz;
 
-/*
-  MPI_Dims_create(npes, D, psize);
+    /*
+      MPI_Dims_create(npes, D, psize);
 
-  printf("psize %d %d %d\n",psize[0],psize[1],psize[2]);
-*/
+      printf("psize %d %d %d\n",psize[0],psize[1],psize[2]);
+    */
 
-  MPI_Type_create_darray(npes, rank, D, gsize, distrib, dargs, psize, MPI_ORDER_FORTRAN, MPI_INT, &filetype);
+    MPI_Type_create_darray(npes, rank, D, gsize, distrib, dargs, psize, MPI_ORDER_FORTRAN, MPI_INT,
+                           &filetype);
 
-  MPI_Type_commit(&filetype);
+    MPI_Type_commit(&filetype);
 
-  to1 = MPI_Wtime();
-  MPI_File_open(new_comm, argv[1], MPI_MODE_RDONLY, info, &thefile);
-  to2 = MPI_Wtime();
+    to1 = MPI_Wtime();
+    MPI_File_open(new_comm, argv[1], MPI_MODE_RDONLY, info, &thefile);
+    to2 = MPI_Wtime();
 
-  MPI_File_set_view(thefile, offset, MPI_INT, filetype, "native", MPI_INFO_NULL);
+    MPI_File_set_view(thefile, offset, MPI_INT, filetype, "native", MPI_INFO_NULL);
 
-  t1 = MPI_Wtime();
-  for ( i=0; i<LOOP; i++ )
-    {
-     MPI_File_read_all(thefile, buf, ng, MPI_INT, &status);
+    t1 = MPI_Wtime();
+    for (i = 0; i < LOOP; i++) {
+        MPI_File_read_all(thefile, buf, ng, MPI_INT, &status);
     }
-  t2 = MPI_Wtime();
+    t2 = MPI_Wtime();
 
-/*MPI_File_sync(thefile); */
+    /*MPI_File_sync(thefile); */
 
-  tc1 = MPI_Wtime();
-  MPI_File_close(&thefile);
-  tc2 = MPI_Wtime();
+    tc1 = MPI_Wtime();
+    MPI_File_close(&thefile);
+    tc2 = MPI_Wtime();
 
-  et  = (t2  - t1)/LOOP;
-  eto = (to2 - to1)/LOOP;
-  etc = (tc2 - tc1)/LOOP;
+    et = (t2 - t1) / LOOP;
+    eto = (to2 - to1) / LOOP;
+    etc = (tc2 - tc1) / LOOP;
 
-  mbs = (((double)(LOOP*X*Y*Z)*sizeof(int)))/(1000000.0*(t2-t1));
+    mbs = (((double) (LOOP * X * Y * Z) * sizeof(int))) / (1000000.0 * (t2 - t1));
 
-/*printf(" %s[%3d]    ET  %5.2f  %6.2f  %6.2f     %5.1f mbs       Data %9d %9d \n", process_name, rank, t1, t2, t2-t1, mbs, buf[0], buf[ng-1]);*/
+    /*printf(" %s[%3d]    ET  %5.2f  %6.2f  %6.2f     %5.1f mbs       Data %9d %9d \n",
+     * process_name, rank, t1, t2, t2-t1, mbs, buf[0], buf[ng-1]);*/
 
-  MPI_Barrier(MPI_COMM_WORLD);
+    MPI_Barrier(MPI_COMM_WORLD);
 
-  MPI_Reduce(&mbs, &avg_mbs, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
-  MPI_Reduce(&mbs, &min_mbs, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
-  MPI_Reduce(&mbs, &max_mbs, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&mbs, &avg_mbs, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&mbs, &min_mbs, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&mbs, &max_mbs, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
 
-  MPI_Reduce(&et, &avg_et, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
-  MPI_Reduce(&et, &min_et, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
-  MPI_Reduce(&et, &max_et, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&et, &avg_et, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&et, &min_et, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&et, &max_et, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
 
-  MPI_Reduce(&eto, &avg_eto, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
-  MPI_Reduce(&eto, &min_eto, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
-  MPI_Reduce(&eto, &max_eto, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&eto, &avg_eto, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&eto, &min_eto, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&eto, &max_eto, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
 
-  MPI_Reduce(&etc, &avg_etc, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
-  MPI_Reduce(&etc, &min_etc, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
-  MPI_Reduce(&etc, &max_etc, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&etc, &avg_etc, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&etc, &min_etc, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&etc, &max_etc, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
 
-  fflush(stdout);
+    fflush(stdout);
 
-  if ( rank == 0 )
-    {
-     mbs = avg_mbs/npes;
-     printf("\n     average read rate: %9.1f mbs\n", mbs);
-     printf("     minimum read rate: %9.1f mbs\n", min_mbs);
-     printf("     maximum read rate: %9.1f mbs\n\n", max_mbs);
-     avg_eto = avg_eto/npes;
-     avg_et  = avg_et/npes;
-     avg_etc = avg_etc/npes;
-     printf("     open time:  %9.3f min %9.3f avg %9.3f max\n",min_eto,avg_eto,max_eto);
-     printf("     read time:  %9.3f min %9.3f avg %9.3f max\n",min_et,avg_et,max_et);
-     printf("     close time: %9.3f min %9.3f avg %9.3f max\n\n",min_etc,avg_etc,max_etc);
-     fflush(stdout);
+    if (rank == 0) {
+        mbs = avg_mbs / npes;
+        printf("\n     average read rate: %9.1f mbs\n", mbs);
+        printf("     minimum read rate: %9.1f mbs\n", min_mbs);
+        printf("     maximum read rate: %9.1f mbs\n\n", max_mbs);
+        avg_eto = avg_eto / npes;
+        avg_et = avg_et / npes;
+        avg_etc = avg_etc / npes;
+        printf("     open time:  %9.3f min %9.3f avg %9.3f max\n", min_eto, avg_eto, max_eto);
+        printf("     read time:  %9.3f min %9.3f avg %9.3f max\n", min_et, avg_et, max_et);
+        printf("     close time: %9.3f min %9.3f avg %9.3f max\n\n", min_etc, avg_etc, max_etc);
+        fflush(stdout);
     }
 
-  MPI_Finalize();
+    MPI_Finalize();
 
-  return 0;
+    return 0;
 }
diff --git a/test/simple/parallel_w64.c b/test/simple/parallel_w64.c
index 57f0f878bf4..fdb9acdd29c 100644
--- a/test/simple/parallel_w64.c
+++ b/test/simple/parallel_w64.c
@@ -5,224 +5,215 @@
 #include <stdio.h>
 #include <string.h>
 
-#define D      3             /* dimensions */
+#define D 3 /* dimensions */
 
-#define X   1024             /* global x grid size */
-#define Y   1024             /* global y grid size */
-#define Z   1024             /* global z grid size */
+#define X 1024 /* global x grid size */
+#define Y 1024 /* global y grid size */
+#define Z 1024 /* global z grid size */
 
-#define nx   256             /* local x grid size */
-#define ny   256             /* local y grid size */
-#define nz   256             /* local z grid size */
+#define nx 256 /* local x grid size */
+#define ny 256 /* local y grid size */
+#define nz 256 /* local z grid size */
 
-#define ng (nx*ny*nz)        /* local grid (cube) size */
+#define ng (nx * ny * nz) /* local grid (cube) size */
 
-#define npx    4             /* number of PE's in x direction */
-#define npy    4             /* number of PE's in y direction */
-#define npz    4             /* number of PE's in z direction */
+#define npx 4 /* number of PE's in x direction */
+#define npy 4 /* number of PE's in y direction */
+#define npz 4 /* number of PE's in z direction */
 
-#define np (npx*npy*npz)  /* total PE count */
+#define np (npx * npy * npz) /* total PE count */
 
 #define LOOP 1
 
 #define MAX_RR_NAME 7
 
-int
-main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
-  int  i, rank, npes, bug=0;
-  int buf[ng];
-  MPI_File     thefile;
-  MPI_Status   status;
-  MPI_Datatype filetype;
-  MPI_Comm     new_comm;
-  MPI_Offset   offset=0;
-  MPI_Info     info=MPI_INFO_NULL;
-  int gsize[D],distrib[D],dargs[D],psize[D];
-  int dims[D],periods[D],reorder;
-  double t1,t2,mbs;
-  double to1,to2,tc1,tc2;
-  double et,eto,etc;
-  double max_mbs,min_mbs,avg_mbs;
-  double max_et,min_et,avg_et;
-  double max_eto,min_eto,avg_eto;
-  double max_etc,min_etc,avg_etc;
-  char process_name[MPI_MAX_PROCESSOR_NAME + 1];
-  char rr_blank[] = {"       "};
-  char rr_empty[] = {"???????"};
-  int  count;
-
-  MPI_Init(&argc, &argv);
-  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-  MPI_Comm_size(MPI_COMM_WORLD, &npes);
-  if ( rank == 0 )
-    {
-     if ( argc < 2 )
-       {
-        printf(" ERROR: no filename given\n");
-        bug++;
-       }
-     if ( npes == np )
-       {
-        printf(" file name: %s\n",argv[1]);
-        printf(" total number of PE's: %3d\n",np);
-        printf(" number of PE's in x direction: %4d\n",npx);
-        printf(" number of PE's in y direction: %4d\n",npy);
-        printf(" number of PE's in z direction: %4d\n",npz);
-        printf(" global grid size: %dx%dx%d 4 byte integers (total %lu)\n",X,Y,Z,(unsigned long)X*Y*Z);
-        printf("  local grid size: %dx%dx%d 4 byte integers (total %d)\n",nx,ny,nz,ng);
-       }
-     else
-       {
-        printf(" ERROR: total number of PE's must be %d\n",np);
-        printf("        actual number of PE's was %d\n",npes);
-        bug++;
-       }
-     if ( bug )
-       {
-        MPI_Abort(MPI_COMM_WORLD,-1);
-       }
+    int i, rank, npes, bug = 0;
+    int buf[ng];
+    MPI_File thefile;
+    MPI_Status status;
+    MPI_Datatype filetype;
+    MPI_Comm new_comm;
+    MPI_Offset offset = 0;
+    MPI_Info info = MPI_INFO_NULL;
+    int gsize[D], distrib[D], dargs[D], psize[D];
+    int dims[D], periods[D], reorder;
+    double t1, t2, mbs;
+    double to1, to2, tc1, tc2;
+    double et, eto, etc;
+    double max_mbs, min_mbs, avg_mbs;
+    double max_et, min_et, avg_et;
+    double max_eto, min_eto, avg_eto;
+    double max_etc, min_etc, avg_etc;
+    char process_name[MPI_MAX_PROCESSOR_NAME + 1];
+    char rr_blank[] = {"       "};
+    char rr_empty[] = {"???????"};
+    int count;
+
+    MPI_Init(&argc, &argv);
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    MPI_Comm_size(MPI_COMM_WORLD, &npes);
+    if (rank == 0) {
+        if (argc < 2) {
+            printf(" ERROR: no filename given\n");
+            bug++;
+        }
+        if (npes == np) {
+            printf(" file name: %s\n", argv[1]);
+            printf(" total number of PE's: %3d\n", np);
+            printf(" number of PE's in x direction: %4d\n", npx);
+            printf(" number of PE's in y direction: %4d\n", npy);
+            printf(" number of PE's in z direction: %4d\n", npz);
+            printf(" global grid size: %dx%dx%d 4 byte integers (total %lu)\n", X, Y, Z,
+                   (unsigned long) X * Y * Z);
+            printf("  local grid size: %dx%dx%d 4 byte integers (total %d)\n", nx, ny, nz, ng);
+        } else {
+            printf(" ERROR: total number of PE's must be %d\n", np);
+            printf("        actual number of PE's was %d\n", npes);
+            bug++;
+        }
+        if (bug) {
+            MPI_Abort(MPI_COMM_WORLD, -1);
+        }
+    }
+    if (MPI_Get_processor_name(process_name, &count) != MPI_SUCCESS) {
+        sprintf(process_name, "%s", rr_empty);
+    } else {
+        if (count < MAX_RR_NAME)
+            strncat(&process_name[count], rr_blank, MAX_RR_NAME - count);
+        process_name[MAX_RR_NAME] = '\0';
     }
- if ( MPI_Get_processor_name(process_name, &count) != MPI_SUCCESS)
-   {
-       sprintf(process_name, "%s", rr_empty);
-   }
- else
-   {
-    if (count < MAX_RR_NAME) strncat(&process_name[count],rr_blank,MAX_RR_NAME-count);
-    process_name[MAX_RR_NAME] = '\0';
-   }
-
-  MPI_Barrier(MPI_COMM_WORLD);
 
-  MPI_Info_create(&info);
+    MPI_Barrier(MPI_COMM_WORLD);
 
-/* allow multiple writers to write to the file concurrently */
+    MPI_Info_create(&info);
 
-/*MPI_Info_set(info,"panfs_concurrent_write","1");*/
+    /* allow multiple writers to write to the file concurrently */
 
-/* use data aggregation */
+    /*MPI_Info_set(info,"panfs_concurrent_write","1");*/
 
-/*MPI_Info_set(info,"romio_cb_write","enable"); */
-/*MPI_Info_set(info,"romio_cb_write","disable");*/
-/*MPI_Info_set(info,"romio_cb_read","enable"); */
-/*MPI_Info_set(info,"romio_cb_read","disable");*/
+    /* use data aggregation */
 
-/* use one aggregator/writer per node */
+    /*MPI_Info_set(info,"romio_cb_write","enable"); */
+    /*MPI_Info_set(info,"romio_cb_write","disable");*/
+    /*MPI_Info_set(info,"romio_cb_read","enable"); */
+    /*MPI_Info_set(info,"romio_cb_read","disable");*/
 
-/*MPI_Info_set(info,"cb_config_list","*:1");*/
+    /* use one aggregator/writer per node */
 
-/* aggregators/writers per allocation: use this or the above (both work) */
+    /*MPI_Info_set(info,"cb_config_list","*:1");*/
 
-/*i = ((npes-1)/8) + 1;
-  sprintf(awpa,"%d",i);
-  MPI_Info_set (info,"cb_nodes",awpa);*/
+    /* aggregators/writers per allocation: use this or the above (both work) */
 
+    /*i = ((npes-1)/8) + 1;
+      sprintf(awpa,"%d",i);
+      MPI_Info_set (info,"cb_nodes",awpa);*/
 
-  for ( i=0; i<ng; i++ ) buf[i] = rank*10000 + (i+1)%1024;
+    for (i = 0; i < ng; i++)
+        buf[i] = rank * 10000 + (i + 1) % 1024;
 
-  for ( i=0; i<D; i++ )
-    {
-     periods[i] = 1;  /* true */
+    for (i = 0; i < D; i++) {
+        periods[i] = 1; /* true */
     }
 
-  reorder = 1;        /* true */
+    reorder = 1; /* true */
 
-  dims[0] = npx;
-  dims[1] = npy;
-  dims[2] = npz;
+    dims[0] = npx;
+    dims[1] = npy;
+    dims[2] = npz;
 
-  MPI_Cart_create(MPI_COMM_WORLD, D, dims, periods, reorder, &new_comm);
+    MPI_Cart_create(MPI_COMM_WORLD, D, dims, periods, reorder, &new_comm);
 
-  for ( i=0; i<D; i++ )
-    {
-     distrib[i] = MPI_DISTRIBUTE_BLOCK;
-     dargs[i]   = MPI_DISTRIBUTE_DFLT_DARG;
-/*   psize[i]   = 0; */
+    for (i = 0; i < D; i++) {
+        distrib[i] = MPI_DISTRIBUTE_BLOCK;
+        dargs[i] = MPI_DISTRIBUTE_DFLT_DARG;
+        /*   psize[i]   = 0; */
     }
 
-  gsize[0] = X;
-  gsize[1] = Y;
-  gsize[2] = Z;
+    gsize[0] = X;
+    gsize[1] = Y;
+    gsize[2] = Z;
 
-  psize[0] = npx;
-  psize[1] = npy;
-  psize[2] = npz;
+    psize[0] = npx;
+    psize[1] = npy;
+    psize[2] = npz;
 
-/*
-  MPI_Dims_create(npes, D, psize);
+    /*
+      MPI_Dims_create(npes, D, psize);
 
-  printf("psize %d %d %d\n",psize[0],psize[1],psize[2]);
-*/
+      printf("psize %d %d %d\n",psize[0],psize[1],psize[2]);
+    */
 
-  MPI_Type_create_darray(npes, rank, D, gsize, distrib, dargs, psize, MPI_ORDER_FORTRAN, MPI_INT, &filetype);
-/*MPI_Type_create_darray(npes, rank, D, gsize, distrib, dargs, psize, MPI_ORDER_C, MPI_INT, &filetype);              don't do this */
+    MPI_Type_create_darray(npes, rank, D, gsize, distrib, dargs, psize, MPI_ORDER_FORTRAN, MPI_INT,
+                           &filetype);
+    /*MPI_Type_create_darray(npes, rank, D, gsize, distrib, dargs, psize, MPI_ORDER_C, MPI_INT,
+     * &filetype);              don't do this */
 
-  MPI_Type_commit(&filetype);
+    MPI_Type_commit(&filetype);
 
-  to1 = MPI_Wtime();
-  MPI_File_open(new_comm, argv[1], MPI_MODE_WRONLY | MPI_MODE_CREATE, info, &thefile);
-  to2 = MPI_Wtime();
+    to1 = MPI_Wtime();
+    MPI_File_open(new_comm, argv[1], MPI_MODE_WRONLY | MPI_MODE_CREATE, info, &thefile);
+    to2 = MPI_Wtime();
 
-  MPI_File_set_size(thefile, offset);
+    MPI_File_set_size(thefile, offset);
 
-  MPI_File_set_view(thefile, offset, MPI_INT, filetype, "native", MPI_INFO_NULL);
+    MPI_File_set_view(thefile, offset, MPI_INT, filetype, "native", MPI_INFO_NULL);
 
-  t1 = MPI_Wtime();
-  for ( i=0; i<LOOP; i++)
-    {
-     MPI_File_write_all(thefile, buf, ng, MPI_INT, &status);
+    t1 = MPI_Wtime();
+    for (i = 0; i < LOOP; i++) {
+        MPI_File_write_all(thefile, buf, ng, MPI_INT, &status);
     }
-  t2 = MPI_Wtime();
+    t2 = MPI_Wtime();
 
-  tc1 = MPI_Wtime();
-  MPI_File_close(&thefile);
-  tc2 = MPI_Wtime();
+    tc1 = MPI_Wtime();
+    MPI_File_close(&thefile);
+    tc2 = MPI_Wtime();
 
-  et  = (t2  - t1)/LOOP;
-  eto = (to2 - to1)/LOOP;
-  etc = (tc2 - tc1)/LOOP;
+    et = (t2 - t1) / LOOP;
+    eto = (to2 - to1) / LOOP;
+    etc = (tc2 - tc1) / LOOP;
 
-  mbs = (((double)(LOOP*X*Y*Z)*sizeof(int)))/(1000000.0*(t2-t1));
+    mbs = (((double) (LOOP * X * Y * Z) * sizeof(int))) / (1000000.0 * (t2 - t1));
 
-/*printf(" %s[%3d]    ET  %8.2f  %8.2f  %8.2f         %8.1f mbs\n", process_name, rank, t1, t2, t2-t1, mbs);*/
+    /*printf(" %s[%3d]    ET  %8.2f  %8.2f  %8.2f         %8.1f mbs\n", process_name, rank, t1, t2,
+     * t2-t1, mbs);*/
 
-  MPI_Barrier(MPI_COMM_WORLD);
+    MPI_Barrier(MPI_COMM_WORLD);
 
-  MPI_Reduce(&mbs, &avg_mbs, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
-  MPI_Reduce(&mbs, &min_mbs, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
-  MPI_Reduce(&mbs, &max_mbs, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&mbs, &avg_mbs, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&mbs, &min_mbs, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&mbs, &max_mbs, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
 
-  MPI_Reduce(&et, &avg_et, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
-  MPI_Reduce(&et, &min_et, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
-  MPI_Reduce(&et, &max_et, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&et, &avg_et, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&et, &min_et, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&et, &max_et, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
 
-  MPI_Reduce(&eto, &avg_eto, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
-  MPI_Reduce(&eto, &min_eto, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
-  MPI_Reduce(&eto, &max_eto, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&eto, &avg_eto, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&eto, &min_eto, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&eto, &max_eto, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
 
-  MPI_Reduce(&etc, &avg_etc, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
-  MPI_Reduce(&etc, &min_etc, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
-  MPI_Reduce(&etc, &max_etc, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&etc, &avg_etc, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&etc, &min_etc, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&etc, &max_etc, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
 
-  fflush(stdout);
+    fflush(stdout);
 
-  if ( rank == 0 )
-    {
-     mbs = avg_mbs/npes;
-     printf("\n     average write rate: %9.1f mbs\n", mbs);
-     printf("     minimum write rate: %9.1f mbs\n", min_mbs);
-     printf("     maximum write rate: %9.1f mbs\n\n", max_mbs);
-     avg_eto = avg_eto/npes;
-     avg_et  = avg_et/npes;
-     avg_etc = avg_etc/npes;
-     printf("     open time:  %9.3f min %9.3f avg %9.3f max\n",min_eto,avg_eto,max_eto);
-     printf("     write time: %9.3f min %9.3f avg %9.3f max\n",min_et,avg_et,max_et);
-     printf("     close time: %9.3f min %9.3f avg %9.3f max\n\n",min_etc,avg_etc,max_etc);
-     fflush(stdout);
+    if (rank == 0) {
+        mbs = avg_mbs / npes;
+        printf("\n     average write rate: %9.1f mbs\n", mbs);
+        printf("     minimum write rate: %9.1f mbs\n", min_mbs);
+        printf("     maximum write rate: %9.1f mbs\n\n", max_mbs);
+        avg_eto = avg_eto / npes;
+        avg_et = avg_et / npes;
+        avg_etc = avg_etc / npes;
+        printf("     open time:  %9.3f min %9.3f avg %9.3f max\n", min_eto, avg_eto, max_eto);
+        printf("     write time: %9.3f min %9.3f avg %9.3f max\n", min_et, avg_et, max_et);
+        printf("     close time: %9.3f min %9.3f avg %9.3f max\n\n", min_etc, avg_etc, max_etc);
+        fflush(stdout);
     }
 
-  MPI_Finalize();
+    MPI_Finalize();
 
-  return 0;
+    return 0;
 }
diff --git a/test/simple/parallel_w8.c b/test/simple/parallel_w8.c
index 49fc00a10d1..87eb41f6968 100644
--- a/test/simple/parallel_w8.c
+++ b/test/simple/parallel_w8.c
@@ -4,91 +4,83 @@
 #include <stdio.h>
 #include <string.h>
 
-#define D      3             /* dimensions */
+#define D 3 /* dimensions */
 
-#define X   256             /* global x grid size */
-#define Y   256             /* global y grid size */
-#define Z   256             /* global z grid size */
+#define X 256 /* global x grid size */
+#define Y 256 /* global y grid size */
+#define Z 256 /* global z grid size */
 
-#define nx   128             /* local x grid size */
-#define ny   128             /* local y grid size */
-#define nz   128             /* local z grid size */
+#define nx 128 /* local x grid size */
+#define ny 128 /* local y grid size */
+#define nz 128 /* local z grid size */
 
-#define ng (nx*ny*nz)        /* local grid (cube) size */
+#define ng (nx * ny * nz) /* local grid (cube) size */
 
-#define npx    2             /* number of PE's in x direction */
-#define npy    2             /* number of PE's in y direction */
-#define npz    2             /* number of PE's in z direction */
+#define npx 2 /* number of PE's in x direction */
+#define npy 2 /* number of PE's in y direction */
+#define npz 2 /* number of PE's in z direction */
 
-#define np (npx*npy*npz)  /* total PE count */
+#define np (npx * npy * npz) /* total PE count */
 
 #define LOOP 1
 
 #define MAX_RR_NAME 7
 
-int
-main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
-    int  i, rank, npes, bug=0;
+    int i, rank, npes, bug = 0;
     int buf[ng];
-    MPI_File     thefile;
-    MPI_Status   status;
+    MPI_File thefile;
+    MPI_Status status;
     MPI_Datatype filetype;
-    MPI_Comm     new_comm;
-    MPI_Offset   offset=0;
-    MPI_Info     info=MPI_INFO_NULL;
-    int gsize[D],distrib[D],dargs[D],psize[D];
-    int dims[D],periods[D],reorder;
-    double t1,t2,mbs;
-    double to1,to2,tc1,tc2;
-    double et,eto,etc;
-    double max_mbs,min_mbs,avg_mbs;
-    double max_et,min_et,avg_et;
-    double max_eto,min_eto,avg_eto;
-    double max_etc,min_etc,avg_etc;
+    MPI_Comm new_comm;
+    MPI_Offset offset = 0;
+    MPI_Info info = MPI_INFO_NULL;
+    int gsize[D], distrib[D], dargs[D], psize[D];
+    int dims[D], periods[D], reorder;
+    double t1, t2, mbs;
+    double to1, to2, tc1, tc2;
+    double et, eto, etc;
+    double max_mbs, min_mbs, avg_mbs;
+    double max_et, min_et, avg_et;
+    double max_eto, min_eto, avg_eto;
+    double max_etc, min_etc, avg_etc;
     char process_name[MPI_MAX_PROCESSOR_NAME + 1];
     char rr_blank[] = {"       "};
     char rr_empty[] = {"???????"};
-    int  count;
+    int count;
 
     MPI_Init(&argc, &argv);
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &npes);
-    if ( rank == 0 )
-    {
-        if ( argc < 2 )
-        {
+    if (rank == 0) {
+        if (argc < 2) {
             printf(" ERROR: no filename given\n");
             bug++;
         }
-        if ( npes == np )
-        {
-            printf(" file name: %s\n",argv[1]);
-            printf(" total number of PE's: %3d\n",np);
-            printf(" number of PE's in x direction: %4d\n",npx);
-            printf(" number of PE's in y direction: %4d\n",npy);
-            printf(" number of PE's in z direction: %4d\n",npz);
-            printf(" global grid size: %dx%dx%d 4 byte integers (total %lu)\n",X,Y,Z,(unsigned long)X*Y*Z);
-            printf("  local grid size: %dx%dx%d 4 byte integers (total %d)\n",nx,ny,nz,ng);
-        }
-        else
-        {
-            printf(" ERROR: total number of PE's must be %d\n",np);
-            printf("        actual number of PE's was %d\n",npes);
+        if (npes == np) {
+            printf(" file name: %s\n", argv[1]);
+            printf(" total number of PE's: %3d\n", np);
+            printf(" number of PE's in x direction: %4d\n", npx);
+            printf(" number of PE's in y direction: %4d\n", npy);
+            printf(" number of PE's in z direction: %4d\n", npz);
+            printf(" global grid size: %dx%dx%d 4 byte integers (total %lu)\n", X, Y, Z,
+                   (unsigned long) X * Y * Z);
+            printf("  local grid size: %dx%dx%d 4 byte integers (total %d)\n", nx, ny, nz, ng);
+        } else {
+            printf(" ERROR: total number of PE's must be %d\n", np);
+            printf("        actual number of PE's was %d\n", npes);
             bug++;
         }
-        if ( bug )
-        {
-            MPI_Abort(MPI_COMM_WORLD,-1);
+        if (bug) {
+            MPI_Abort(MPI_COMM_WORLD, -1);
         }
     }
-    if ( MPI_Get_processor_name(process_name, &count) != MPI_SUCCESS)
-    {
+    if (MPI_Get_processor_name(process_name, &count) != MPI_SUCCESS) {
         sprintf(process_name, "%s", rr_empty);
-    }
-    else
-    {
-        if (count < MAX_RR_NAME) strncat(&process_name[count],rr_blank,MAX_RR_NAME-count);
+    } else {
+        if (count < MAX_RR_NAME)
+            strncat(&process_name[count], rr_blank, MAX_RR_NAME - count);
         process_name[MAX_RR_NAME] = '\0';
     }
 
@@ -117,15 +109,14 @@ main(int argc, char* argv[])
      sprintf(awpa,"%d",i);
      MPI_Info_set (info,"cb_nodes",awpa);*/
 
+    for (i = 0; i < ng; i++)
+        buf[i] = rank * 10000 + (i + 1) % 1024;
 
-    for ( i=0; i<ng; i++ ) buf[i] = rank*10000 + (i+1)%1024;
-
-    for ( i=0; i<D; i++ )
-    {
-        periods[i] = 1;  /* true */
+    for (i = 0; i < D; i++) {
+        periods[i] = 1; /* true */
     }
 
-    reorder = 1;        /* true */
+    reorder = 1; /* true */
 
     dims[0] = npx;
     dims[1] = npy;
@@ -133,10 +124,9 @@ main(int argc, char* argv[])
 
     MPI_Cart_create(MPI_COMM_WORLD, D, dims, periods, reorder, &new_comm);
 
-    for ( i=0; i<D; i++ )
-    {
+    for (i = 0; i < D; i++) {
         distrib[i] = MPI_DISTRIBUTE_BLOCK;
-        dargs[i]   = MPI_DISTRIBUTE_DFLT_DARG;
+        dargs[i] = MPI_DISTRIBUTE_DFLT_DARG;
         /*   psize[i]   = 0; */
     }
 
@@ -154,8 +144,10 @@ main(int argc, char* argv[])
      printf("psize %d %d %d\n",psize[0],psize[1],psize[2]);
      */
 
-    MPI_Type_create_darray(npes, rank, D, gsize, distrib, dargs, psize, MPI_ORDER_FORTRAN, MPI_INT, &filetype);
-    /*MPI_Type_create_darray(npes, rank, D, gsize, distrib, dargs, psize, MPI_ORDER_C, MPI_INT, &filetype);              don't do this */
+    MPI_Type_create_darray(npes, rank, D, gsize, distrib, dargs, psize, MPI_ORDER_FORTRAN, MPI_INT,
+                           &filetype);
+    /*MPI_Type_create_darray(npes, rank, D, gsize, distrib, dargs, psize, MPI_ORDER_C, MPI_INT,
+     * &filetype);              don't do this */
 
     MPI_Type_commit(&filetype);
 
@@ -168,8 +160,7 @@ main(int argc, char* argv[])
     MPI_File_set_view(thefile, offset, MPI_INT, filetype, "native", MPI_INFO_NULL);
 
     t1 = MPI_Wtime();
-    for ( i=0; i<LOOP; i++)
-    {
+    for (i = 0; i < LOOP; i++) {
         MPI_File_write_all(thefile, buf, ng, MPI_INT, &status);
     }
     t2 = MPI_Wtime();
@@ -178,13 +169,14 @@ main(int argc, char* argv[])
     MPI_File_close(&thefile);
     tc2 = MPI_Wtime();
 
-    et  = (t2  - t1)/LOOP;
-    eto = (to2 - to1)/LOOP;
-    etc = (tc2 - tc1)/LOOP;
+    et = (t2 - t1) / LOOP;
+    eto = (to2 - to1) / LOOP;
+    etc = (tc2 - tc1) / LOOP;
 
-    mbs = (((double)(LOOP*X*Y*Z)*sizeof(int)))/(1000000.0*(t2-t1));
+    mbs = (((double) (LOOP * X * Y * Z) * sizeof(int))) / (1000000.0 * (t2 - t1));
 
-    /*printf(" %s[%3d]    ET  %8.2f  %8.2f  %8.2f         %8.1f mbs\n", process_name, rank, t1, t2, t2-t1, mbs);*/
+    /*printf(" %s[%3d]    ET  %8.2f  %8.2f  %8.2f         %8.1f mbs\n", process_name, rank, t1, t2,
+     * t2-t1, mbs);*/
 
     MPI_Barrier(MPI_COMM_WORLD);
 
@@ -206,18 +198,17 @@ main(int argc, char* argv[])
 
     fflush(stdout);
 
-    if ( rank == 0 )
-    {
-        mbs = avg_mbs/npes;
+    if (rank == 0) {
+        mbs = avg_mbs / npes;
         printf("\n     average write rate: %9.1f mbs\n", mbs);
         printf("     minimum write rate: %9.1f mbs\n", min_mbs);
         printf("     maximum write rate: %9.1f mbs\n\n", max_mbs);
-        avg_eto = avg_eto/npes;
-        avg_et  = avg_et/npes;
-        avg_etc = avg_etc/npes;
-        printf("     open time:  %9.3f min %9.3f avg %9.3f max\n",min_eto,avg_eto,max_eto);
-        printf("     write time: %9.3f min %9.3f avg %9.3f max\n",min_et,avg_et,max_et);
-        printf("     close time: %9.3f min %9.3f avg %9.3f max\n\n",min_etc,avg_etc,max_etc);
+        avg_eto = avg_eto / npes;
+        avg_et = avg_et / npes;
+        avg_etc = avg_etc / npes;
+        printf("     open time:  %9.3f min %9.3f avg %9.3f max\n", min_eto, avg_eto, max_eto);
+        printf("     write time: %9.3f min %9.3f avg %9.3f max\n", min_et, avg_et, max_et);
+        printf("     close time: %9.3f min %9.3f avg %9.3f max\n\n", min_etc, avg_etc, max_etc);
         fflush(stdout);
     }
 
diff --git a/test/simple/pinterlib.c b/test/simple/pinterlib.c
index 40d7f8846a7..f6e15190225 100644
--- a/test/simple/pinterlib.c
+++ b/test/simple/pinterlib.c
@@ -5,10 +5,10 @@
  * The most basic of MPI applications
  */
 
-#include <stdio.h>
-#include <pthread.h>
 #include "mpi.h"
 #include "pmix.h"
+#include <pthread.h>
+#include <stdio.h>
 
 typedef struct {
     pthread_mutex_t mutex;
@@ -17,77 +17,72 @@ typedef struct {
     pmix_status_t status;
 } mylock_t;
 
-#define MY_CONSTRUCT_LOCK(l)                     \
-    do {                                            \
-        pthread_mutex_init(&(l)->mutex, NULL);      \
-        pthread_cond_init(&(l)->cond, NULL);        \
-        (l)->active = true;                         \
-        (l)->status = PMIX_SUCCESS;                 \
-    } while(0)
+#define MY_CONSTRUCT_LOCK(l)                   \
+    do {                                       \
+        pthread_mutex_init(&(l)->mutex, NULL); \
+        pthread_cond_init(&(l)->cond, NULL);   \
+        (l)->active = true;                    \
+        (l)->status = PMIX_SUCCESS;            \
+    } while (0)
 
-#define MY_DESTRUCT_LOCK(l)              \
+#define MY_DESTRUCT_LOCK(l)                 \
     do {                                    \
         pthread_mutex_destroy(&(l)->mutex); \
         pthread_cond_destroy(&(l)->cond);   \
-    } while(0)
-
-#define MY_WAIT_THREAD(lck)                                      \
-    do {                                                            \
-        pthread_mutex_lock(&(lck)->mutex);                          \
-        while ((lck)->active) {                                     \
-            pthread_cond_wait(&(lck)->cond, &(lck)->mutex);         \
-        }                                                           \
-        pthread_mutex_unlock(&(lck)->mutex);                        \
-    } while(0)
-
-#define MY_WAKEUP_THREAD(lck)                        \
-    do {                                                \
-        pthread_mutex_lock(&(lck)->mutex);              \
-        (lck)->active = false;                          \
-        pthread_cond_broadcast(&(lck)->cond);           \
-        pthread_mutex_unlock(&(lck)->mutex);            \
-    } while(0)
-
+    } while (0)
+
+#define MY_WAIT_THREAD(lck)                                 \
+    do {                                                    \
+        pthread_mutex_lock(&(lck)->mutex);                  \
+        while ((lck)->active) {                             \
+            pthread_cond_wait(&(lck)->cond, &(lck)->mutex); \
+        }                                                   \
+        pthread_mutex_unlock(&(lck)->mutex);                \
+    } while (0)
+
+#define MY_WAKEUP_THREAD(lck)                 \
+    do {                                      \
+        pthread_mutex_lock(&(lck)->mutex);    \
+        (lck)->active = false;                \
+        pthread_cond_broadcast(&(lck)->cond); \
+        pthread_mutex_unlock(&(lck)->mutex);  \
+    } while (0)
 
 static size_t interlibhandler_id = SIZE_MAX;
 static mylock_t thread_complete;
 static pmix_proc_t myproc;
 
-static void model_registration_callback(pmix_status_t status,
-                                        size_t errhandler_ref,
-                                        void *cbdata)
+static void model_registration_callback(pmix_status_t status, size_t errhandler_ref, void *cbdata)
 {
-    mylock_t *lock = (mylock_t*)cbdata;
+    mylock_t *lock = (mylock_t *) cbdata;
 
     interlibhandler_id = errhandler_ref;
     MY_WAKEUP_THREAD(lock);
 }
-static void model_callback(size_t evhdlr_registration_id,
-                           pmix_status_t status,
-                           const pmix_proc_t *source,
-                           pmix_info_t info[], size_t ninfo,
+static void model_callback(size_t evhdlr_registration_id, pmix_status_t status,
+                           const pmix_proc_t *source, pmix_info_t info[], size_t ninfo,
                            pmix_info_t *results, size_t nresults,
-                           pmix_event_notification_cbfunc_fn_t cbfunc,
-                           void *cbdata)
+                           pmix_event_notification_cbfunc_fn_t cbfunc, void *cbdata)
 {
     size_t n;
 
     /* we can ignore our own callback as we obviously
      * know that we are OpenMP */
     if (NULL != info) {
-        for (n=0; n < ninfo; n++) {
-            if (0 == strcmp(info[n].key, PMIX_PROGRAMMING_MODEL) &&
-                0 == strcmp(info[n].value.data.string, "OpenMP")) {
+        for (n = 0; n < ninfo; n++) {
+            if (0 == strcmp(info[n].key, PMIX_PROGRAMMING_MODEL)
+                && 0 == strcmp(info[n].value.data.string, "OpenMP")) {
                 goto cback;
             }
             if (PMIX_STRING == info[n].value.type) {
-                fprintf(stderr, "Thread Model Callback Key: %s Val %s\n", info[n].key, info[n].value.data.string);
+                fprintf(stderr, "Thread Model Callback Key: %s Val %s\n", info[n].key,
+                        info[n].value.data.string);
             }
         }
     }
     /* otherwise, do something clever here */
 
-  cback:
+cback:
     /* we must NOT tell the event handler state machine that we
      * are the last step as that will prevent it from notifying
      * anyone else that might be listening for declarations */
@@ -99,20 +94,17 @@ static void model_callback(size_t evhdlr_registration_id,
 
 static void opcbfunc(pmix_status_t status, void *cbdata)
 {
-    mylock_t *lock = (mylock_t*)cbdata;
+    mylock_t *lock = (mylock_t *) cbdata;
     MY_WAKEUP_THREAD(lock);
 }
 
-static void infocb(pmix_status_t status,
-                   pmix_info_t *info, size_t ninfo,
-                   void *cbdata,
-                   pmix_release_cbfunc_t release_fn,
-                   void *release_cbdata)
+static void infocb(pmix_status_t status, pmix_info_t *info, size_t ninfo, void *cbdata,
+                   pmix_release_cbfunc_t release_fn, void *release_cbdata)
 {
-    mylock_t *lock = (mylock_t*)cbdata;
+    mylock_t *lock = (mylock_t *) cbdata;
     size_t n;
 
-    for (n=0; n < ninfo; n++) {
+    for (n = 0; n < ninfo; n++) {
         fprintf(stderr, "QUERY DATA KEY: %s VALUE %s\n", info[n].key, info[n].value.data.string);
     }
     if (NULL != release_fn) {
@@ -155,10 +147,8 @@ static void *mylib(void *ptr)
         /* it is, so let's just use the event notification
          * API to let everyone know we are here */
         MY_CONSTRUCT_LOCK(&lock);
-        ret = PMIx_Notify_event(code, &myproc,
-                                PMIX_RANGE_PROC_LOCAL,
-                                info, 5,
-                                opcbfunc, (void*)&lock);
+        ret = PMIx_Notify_event(code, &myproc, PMIX_RANGE_PROC_LOCAL, info, 5, opcbfunc,
+                                (void *) &lock);
         MY_WAIT_THREAD(&lock);
         MY_DESTRUCT_LOCK(&lock);
     } else {
@@ -178,10 +168,8 @@ static void *mylib(void *ptr)
      * the event stipulates its range as proc_local. We rely
      * on that here */
     MY_CONSTRUCT_LOCK(&lock);
-    PMIx_Register_event_handler(&code, 1, directives, 1,
-                                model_callback,
-                                model_registration_callback,
-                                (void*)&lock);
+    PMIx_Register_event_handler(&code, 1, directives, 1, model_callback,
+                                model_registration_callback, (void *) &lock);
     MY_WAIT_THREAD(&lock);
     MY_DESTRUCT_LOCK(&lock);
     PMIX_INFO_FREE(directives, 1);
@@ -235,7 +223,7 @@ static void *mylib(void *ptr)
     return NULL;
 }
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int rank, size, rc;
     pid_t pid;
@@ -295,7 +283,7 @@ int main(int argc, char* argv[])
         fprintf(stderr, "Error joining thread\n");
     }
 
-  done:
+done:
     MPI_Finalize();
     return 0;
 }
diff --git a/test/simple/pmix.c b/test/simple/pmix.c
index 3ee3f2fcce2..178950b2c57 100644
--- a/test/simple/pmix.c
+++ b/test/simple/pmix.c
@@ -10,29 +10,28 @@
  */
 /*
  * To compile test:
- * mpicc -I$src_dir -I$src_dir/opal/include -I$src_dir/orte/include -I$src_dir/ompi/include -DOMPI_BUILDING=1 pmix.c -o pmix
- * To run test:
- * mpirun -np 2 <any mca params> ./pmix
- * Test should print "Passed" in case of success and print pmix time intervals at process with rank 0.
+ * mpicc -I$src_dir -I$src_dir/opal/include -I$src_dir/orte/include -I$src_dir/ompi/include
+ * -DOMPI_BUILDING=1 pmix.c -o pmix To run test: mpirun -n 2 <any mca params> ./pmix Test should
+ * print "Passed" in case of success and print pmix time intervals at process with rank 0.
  * */
-#include <stdio.h>
-#include <unistd.h>
 #include <mpi.h>
-#include <time.h>
+#include <stdio.h>
 #include <sys/time.h>
+#include <time.h>
+#include <unistd.h>
 
+#include "ompi/proc/proc.h"
 #include "opal/class/opal_list.h"
 #include "opal/mca/pmix/pmix.h"
-#include "ompi/proc/proc.h"
 
-#define DO_FINALIZE(rc,flag,format,args...) \
-    do {                                    \
-        if (flag) {                         \
-            fprintf(stderr, format, args);  \
-        }                                   \
-        MPI_Finalize();                     \
-        return rc;                          \
-    } while(0);
+#define DO_FINALIZE(rc, flag, format, args...) \
+    do {                                       \
+        if (flag) {                            \
+            fprintf(stderr, format, args);     \
+        }                                      \
+        MPI_Finalize();                        \
+        return rc;                             \
+    } while (0);
 
 static int my_rank;
 static volatile bool waiting = true;
@@ -44,10 +43,8 @@ static double get_timestamp(void)
     return ((tv.tv_sec) + (tv.tv_usec) * 1.0e-6);
 }
 
-static void evhandler(int status,
-                      const opal_process_name_t *source,
-                      opal_list_t *info, opal_list_t *results,
-                      opal_pmix_notification_complete_fn_t cbfunc,
+static void evhandler(int status, const opal_process_name_t *source, opal_list_t *info,
+                      opal_list_t *results, opal_pmix_notification_complete_fn_t cbfunc,
                       void *cbdata)
 {
     fprintf(stderr, "%d: received notification status %d\n", my_rank, status);
@@ -57,7 +54,7 @@ static void evhandler(int status,
     waiting = false;
 }
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int rc;
     int recv_data;
@@ -90,12 +87,12 @@ int main(int argc, char* argv[])
     opal_pmix.commit();
     opal_pmix.fence(NULL, 1);
     t3 = get_timestamp();
-    procs = ompi_proc_world ( &numprocs );
+    procs = ompi_proc_world(&numprocs);
     ptr = &recv_data;
     t4 = get_timestamp();
-    for ( i = 0; i < numprocs; i++ ) {
+    for (i = 0; i < numprocs; i++) {
         thisproc = procs[i];
-        OPAL_MODEX_RECV_VALUE(rc, "MY_RANK", &thisproc->super.proc_name, (void**)&ptr, OPAL_INT);
+        OPAL_MODEX_RECV_VALUE(rc, "MY_RANK", &thisproc->super.proc_name, (void **) &ptr, OPAL_INT);
         /* check return status and received data */
         if (OPAL_SUCCESS != rc || i != recv_data) {
             rc = OPAL_ERROR;
@@ -109,13 +106,13 @@ int main(int argc, char* argv[])
     t6 = get_timestamp();
 
     fprintf(stderr, "[%d] Test passed.\n", my_rank);
-    fprintf(stderr, "[%d] \"MODEX_SEND\" %f\n", my_rank, t1-t0);
-    fprintf(stderr, "[%d] \"FENCE\" %f\n", my_rank, t3-t2);
-    fprintf(stderr, "[%d] \"MODEX_RECV\" %f\n", my_rank, t5-t4);
-    fprintf(stderr, "[%d] \"BARRIER\" %f\n", my_rank, t6-t5);
-    fprintf(stderr, "[%d] \"TOTAL\" %f\n", my_rank, t6-t0);
+    fprintf(stderr, "[%d] \"MODEX_SEND\" %f\n", my_rank, t1 - t0);
+    fprintf(stderr, "[%d] \"FENCE\" %f\n", my_rank, t3 - t2);
+    fprintf(stderr, "[%d] \"MODEX_RECV\" %f\n", my_rank, t5 - t4);
+    fprintf(stderr, "[%d] \"BARRIER\" %f\n", my_rank, t6 - t5);
+    fprintf(stderr, "[%d] \"TOTAL\" %f\n", my_rank, t6 - t0);
 
-    fprintf(stderr, "[%d] Pid %d waiting for notification\n", my_rank, (int)getpid());
+    fprintf(stderr, "[%d] Pid %d waiting for notification\n", my_rank, (int) getpid());
 
     /* now wait for notification of someone failing */
     tp.tv_sec = 0;
diff --git a/test/simple/pubsub.c b/test/simple/pubsub.c
index f5ce1e966d8..792eb28d3b1 100644
--- a/test/simple/pubsub.c
+++ b/test/simple/pubsub.c
@@ -5,17 +5,17 @@
  * The most basic of MPI applications
  */
 
-#include <stdio.h>
+#include "mpi.h"
 #include <stdbool.h>
+#include <stdio.h>
 #include <string.h>
-#include "mpi.h"
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int rank, size;
     MPI_Info info, srch;
     char port[MPI_MAX_PORT_NAME];
-    bool local=false;
+    bool local = false;
 
     if (1 < argc) {
         if (0 == strcmp("local", argv[1])) {
diff --git a/test/simple/reduce-hang.c b/test/simple/reduce-hang.c
index 3a1e70c1ac8..20cb6c479ef 100644
--- a/test/simple/reduce-hang.c
+++ b/test/simple/reduce-hang.c
@@ -8,8 +8,8 @@ int main(int argc, char *argv[])
     int i = 8, j;
     int self;
     int do_barrier = 0;
-int k;
-double pi;
+    int k;
+    double pi;
 
     if (getenv("DO_BARRIER")) {
         do_barrier = 1;
diff --git a/test/simple/ring.c b/test/simple/ring.c
index f452fd466b1..a86a3234402 100644
--- a/test/simple/ring.c
+++ b/test/simple/ring.c
@@ -1,9 +1,9 @@
-#include <stdlib.h>
-#include <stdio.h>
 #include <mpi.h>
+#include <stdio.h>
+#include <stdlib.h>
 
-#define SIZE 20
-#define POS 10
+#define SIZE          20
+#define POS           10
 #define INITIAL_VALUE 10
 
 int main(int argc, char *argv[])
@@ -32,14 +32,12 @@ int main(int argc, char *argv[])
 
     if (0 == rank) {
         send_array[pos] = INITIAL_VALUE;
-        MPI_Send(send_array, array_size, MPI_INT, next, tag,
-                 MPI_COMM_WORLD);
+        MPI_Send(send_array, array_size, MPI_INT, next, tag, MPI_COMM_WORLD);
     }
 
     while (1) {
         recv_array[pos] = -1;
-        MPI_Recv(recv_array, array_size, MPI_INT, prev, tag,
-                 MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+        MPI_Recv(recv_array, array_size, MPI_INT, prev, tag, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
         send_array[pos] = recv_array[pos];
         if (rank == 0) {
             --send_array[pos];
@@ -51,8 +49,7 @@ int main(int argc, char *argv[])
     }
 
     if (rank == 0) {
-        MPI_Recv(recv_array, array_size, MPI_INT, prev, tag,
-                 MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+        MPI_Recv(recv_array, array_size, MPI_INT, prev, tag, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
     }
 
     fprintf(stderr, "Rank %d has completed ring\n", rank);
diff --git a/test/simple/segv.c b/test/simple/segv.c
index cde8badb931..4e816e9f269 100644
--- a/test/simple/segv.c
+++ b/test/simple/segv.c
@@ -5,14 +5,14 @@
  * The most basic of MPI applications
  */
 
+#include "mpi.h"
 #include <stdio.h>
 #include <unistd.h>
-#include "mpi.h"
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int rank, size;
-    char *foo=0;
+    char *foo = 0;
 
     MPI_Init(&argc, &argv);
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
diff --git a/test/simple/sendrecv_blaster.c b/test/simple/sendrecv_blaster.c
index e60ea468fe7..0143054c9b3 100644
--- a/test/simple/sendrecv_blaster.c
+++ b/test/simple/sendrecv_blaster.c
@@ -3,55 +3,56 @@
 
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include <time.h>
 #include <sys/time.h>
+#include <time.h>
+#include <unistd.h>
 
 #include "mpi.h"
 
 int main(int argc, char *argv[])
 {
-    MPI_Status     status;               /* MPI status                          */
-    int            mpierr;               /* MPI function return code            */
-    int            rank;                 /* Process rank within MPI_COMM_WORLD  */
-    int            size;
-    int            dest, src;
-    int            tag0=41;              /* MPI message tag                     */
-
-    int            inject;
-    int            report;
-    int            iterations;
-    int            n_bytes;
-    unsigned char* send_buff;
-    unsigned char* recv_buff;
-    char*          tmp;
-
-    int            i, j, count;
+    MPI_Status status; /* MPI status                          */
+    int mpierr;        /* MPI function return code            */
+    int rank;          /* Process rank within MPI_COMM_WORLD  */
+    int size;
+    int dest, src;
+    int tag0 = 41; /* MPI message tag                     */
+
+    int inject;
+    int report;
+    int iterations;
+    int n_bytes;
+    unsigned char *send_buff;
+    unsigned char *recv_buff;
+    char *tmp;
+
+    int i, j, count;
 
     float fraction, randval;
     struct timeval tp;
 
     if (1 < argc) {
-        if (0 == strncmp(argv[1], "-h", 2) ||
-            0 == strncmp(argv[1], "--h", 3)) {
-            printf("Usage: mpirun --options-- ./sendrecv_blaster <options> where options are:\n"
-                   "\tpattern=[self | pair | ring] where\n"
-                   "\t\tself => sendrecv with self\n"
-                   "\t\tpair => sendrecv with a complementary partner [0 <-> N-1, 1 <-> N-2...]\n"
-                   "\t\tring [default] => sendrecv around a ring [0 recvs from N-1 and sends to 1]\n"
-                   "\tsize=[value < 0 => max message size in kbytes, value > 0 => max message size in Mbytes (default=1MByte)]\n"
-                   "\tinject=[value = #iterations before injecting MPI_Sendrecv to self (default: never)]\n"
-                   "\treport=[value = #iterations/reporting point (default: 1000)\n"
-                   "\titerations=[value = #iterations before stopping (default: 1000000)\n");
+        if (0 == strncmp(argv[1], "-h", 2) || 0 == strncmp(argv[1], "--h", 3)) {
+            printf(
+                "Usage: mpirun --options-- ./sendrecv_blaster <options> where options are:\n"
+                "\tpattern=[self | pair | ring] where\n"
+                "\t\tself => sendrecv with self\n"
+                "\t\tpair => sendrecv with a complementary partner [0 <-> N-1, 1 <-> N-2...]\n"
+                "\t\tring [default] => sendrecv around a ring [0 recvs from N-1 and sends to 1]\n"
+                "\tsize=[value < 0 => max message size in kbytes, value > 0 => max message size in "
+                "Mbytes (default=1MByte)]\n"
+                "\tinject=[value = #iterations before injecting MPI_Sendrecv to self (default: "
+                "never)]\n"
+                "\treport=[value = #iterations/reporting point (default: 1000)\n"
+                "\titerations=[value = #iterations before stopping (default: 1000000)\n");
             return 0;
         }
     }
 
     mpierr = MPI_Init(&argc, &argv);
-    if (mpierr != MPI_SUCCESS)
-    {
-        fprintf(stderr, "MPI Error %d (MPI_Init)\n",mpierr);
+    if (mpierr != MPI_SUCCESS) {
+        fprintf(stderr, "MPI Error %d (MPI_Init)\n", mpierr);
         fflush(stderr);
         MPI_Abort(MPI_COMM_WORLD, -1);
     }
@@ -59,23 +60,21 @@ int main(int argc, char *argv[])
     MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN);
 
     mpierr = MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-    if (mpierr != MPI_SUCCESS || rank < 0)
-    {
-        fprintf(stderr, "MPI Error %d (MPI_Comm_rank)\n",mpierr);
+    if (mpierr != MPI_SUCCESS || rank < 0) {
+        fprintf(stderr, "MPI Error %d (MPI_Comm_rank)\n", mpierr);
         fflush(stderr);
         MPI_Abort(MPI_COMM_WORLD, -1);
     }
 
     mpierr = MPI_Comm_size(MPI_COMM_WORLD, &size);
-    if (mpierr != MPI_SUCCESS || size < 0)
-    {
-        fprintf(stderr, "MPI Error %d (MPI_Comm_size)\n",mpierr);
+    if (mpierr != MPI_SUCCESS || size < 0) {
+        fprintf(stderr, "MPI Error %d (MPI_Comm_size)\n", mpierr);
         fflush(stderr);
         MPI_Abort(MPI_COMM_WORLD, -1);
     }
 
     /* setup defaults in lieu of args */
-    n_bytes = 1024*1024;
+    n_bytes = 1024 * 1024;
     inject = -1;
     report = 1000;
     iterations = 1000000;
@@ -85,11 +84,11 @@ int main(int argc, char *argv[])
         src = size - 1;
     }
     dest = rank + 1;
-    if (dest > size-1) {
+    if (dest > size - 1) {
         dest = 0;
     }
 
-    for (i=1; i < argc; i++) {
+    for (i = 1; i < argc; i++) {
         fprintf(stderr, "got %s\n", argv[i]);
         if (0 == strncmp(argv[i], "pattern", strlen("pattern"))) {
             tmp = strchr(argv[i], '=');
@@ -100,7 +99,7 @@ int main(int argc, char *argv[])
                 dest = rank;
             } else if (0 == strcmp(tmp, "pair")) {
                 /* do it pair-wise */
-                src = (size-1) - rank;
+                src = (size - 1) - rank;
                 dest = src;
             } else {
                 /* do a ring */
@@ -109,7 +108,7 @@ int main(int argc, char *argv[])
                     src = size - 1;
                 }
                 dest = rank + 1;
-                if (dest > size-1) {
+                if (dest > size - 1) {
                     dest = 0;
                 }
             }
@@ -120,7 +119,7 @@ int main(int argc, char *argv[])
             if (n_bytes < 0) {
                 n_bytes = -1 * n_bytes * 1024;
             } else {
-                n_bytes = n_bytes * 1024*1024;
+                n_bytes = n_bytes * 1024 * 1024;
             }
         } else if (0 == strncmp(argv[i], "inject", strlen("inject"))) {
             tmp = strchr(argv[i], '=');
@@ -141,41 +140,39 @@ int main(int argc, char *argv[])
     recv_buff = (unsigned char *) valloc(n_bytes);
 
     /* seed the random number generator */
-    gettimeofday (&tp, NULL);
-    srand (tp.tv_usec);
+    gettimeofday(&tp, NULL);
+    srand(tp.tv_usec);
 
-    for ( i=0; i<n_bytes; i++ )
-    {
-        send_buff[i] = i%128;
+    for (i = 0; i < n_bytes; i++) {
+        send_buff[i] = i % 128;
     }
 
     fprintf(stderr, "Rank %d: recving from src %d sending to dest %d with max buff size %dKbytes\n",
-            rank, src, dest, n_bytes/1024);
+            rank, src, dest, n_bytes / 1024);
 
-    i=0;
-    while (i < iterations)
-    {
+    i = 0;
+    while (i < iterations) {
         randval = rand();
-        fraction = randval/RAND_MAX;
+        fraction = randval / RAND_MAX;
         count = fraction * n_bytes;
-        mpierr = MPI_Sendrecv(send_buff, count, MPI_CHAR, dest, tag0,
-                              recv_buff, n_bytes, MPI_CHAR, src, tag0, MPI_COMM_WORLD, &status);
-        if (mpierr != MPI_SUCCESS)
-        {
-            fprintf(stderr,"MPI Error %d (MPI_Sendrecv) [%d,%d] at iteration %d\n",mpierr,src,dest,i);
+        mpierr = MPI_Sendrecv(send_buff, count, MPI_CHAR, dest, tag0, recv_buff, n_bytes, MPI_CHAR,
+                              src, tag0, MPI_COMM_WORLD, &status);
+        if (mpierr != MPI_SUCCESS) {
+            fprintf(stderr, "MPI Error %d (MPI_Sendrecv) [%d,%d] at iteration %d\n", mpierr, src,
+                    dest, i);
             fflush(stderr);
             MPI_Abort(MPI_COMM_WORLD, -1);
         }
         i++;
         if (0 == (i % report)) {
-            fprintf(stderr, "Rank %d has completed %dk iterations\n", rank, i/1000);
+            fprintf(stderr, "Rank %d has completed %dk iterations\n", rank, i / 1000);
         }
         if (0 < inject && 0 == (i % inject)) {
-            mpierr = MPI_Sendrecv(send_buff, count, MPI_CHAR, rank, tag0,
-                                  recv_buff, n_bytes, MPI_CHAR, rank, tag0, MPI_COMM_WORLD, &status);
-            if (mpierr != MPI_SUCCESS)
-            {
-                fprintf(stderr,"MPI Error %d (MPI_Sendrecv) [%d,%d] at iteration %d\n",mpierr,rank,rank,i);
+            mpierr = MPI_Sendrecv(send_buff, count, MPI_CHAR, rank, tag0, recv_buff, n_bytes,
+                                  MPI_CHAR, rank, tag0, MPI_COMM_WORLD, &status);
+            if (mpierr != MPI_SUCCESS) {
+                fprintf(stderr, "MPI Error %d (MPI_Sendrecv) [%d,%d] at iteration %d\n", mpierr,
+                        rank, rank, i);
                 fflush(stderr);
                 MPI_Abort(MPI_COMM_WORLD, -1);
             } else {
diff --git a/test/simple/server.c b/test/simple/server.c
index c5f8ec16b41..60e9a6ff109 100644
--- a/test/simple/server.c
+++ b/test/simple/server.c
@@ -1,55 +1,50 @@
+#include "mpi.h"
+#include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <errno.h>
-#include "mpi.h"
 
 #define MAX_DATA 100
 
-int main( int argc, char **argv )
+int main(int argc, char **argv)
 {
- MPI_Comm client;
- MPI_Status status;
- char port_name[MPI_MAX_PORT_NAME];
- double buf[MAX_DATA];
- int size, again;
+    MPI_Comm client;
+    MPI_Status status;
+    char port_name[MPI_MAX_PORT_NAME];
+    double buf[MAX_DATA];
+    int size, again;
 
- MPI_Init( &argc, &argv );
- MPI_Comm_size(MPI_COMM_WORLD, &size);
- if (size != 1) {
-    fprintf(stderr, "Server too big - need only 1 rank\n");
-    exit(1);
- }
- MPI_Open_port(MPI_INFO_NULL, port_name);
- printf("server available at %s\n",port_name);
+    MPI_Init(&argc, &argv);
+    MPI_Comm_size(MPI_COMM_WORLD, &size);
+    if (size != 1) {
+        fprintf(stderr, "Server too big - need only 1 rank\n");
+        exit(1);
+    }
+    MPI_Open_port(MPI_INFO_NULL, port_name);
+    printf("server available at %s\n", port_name);
 
- while (1)
-   {
-     MPI_Comm_accept( port_name, MPI_INFO_NULL, 0, MPI_COMM_WORLD, &client );
-     again = 1;
+    while (1) {
+        MPI_Comm_accept(port_name, MPI_INFO_NULL, 0, MPI_COMM_WORLD, &client);
+        again = 1;
 
-     while (again)
-       {
-         fprintf(stderr, "Server loop %d\n", again);
-         MPI_Recv( buf, MAX_DATA, MPI_DOUBLE, MPI_ANY_SOURCE,
-MPI_ANY_TAG, client, &status );
+        while (again) {
+            fprintf(stderr, "Server loop %d\n", again);
+            MPI_Recv(buf, MAX_DATA, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG, client, &status);
 
-         switch (status.MPI_TAG)
-           {
-           case 0:
-             fprintf(stderr, "Server recvd terminate cmd\n");
-             MPI_Comm_disconnect( &client );
-             MPI_Close_port(port_name);
-             MPI_Finalize();
-             return 0;
-           case 2: /* do something */
-             fprintf( stderr, "Do something ...\n" );
-             break;
-           default:
-             /* Unexpected message type */
-             MPI_Abort( MPI_COMM_WORLD, 1 );
-           }
-           ++again;
-       }
-   }
+            switch (status.MPI_TAG) {
+            case 0:
+                fprintf(stderr, "Server recvd terminate cmd\n");
+                MPI_Comm_disconnect(&client);
+                MPI_Close_port(port_name);
+                MPI_Finalize();
+                return 0;
+            case 2: /* do something */
+                fprintf(stderr, "Do something ...\n");
+                break;
+            default:
+                /* Unexpected message type */
+                MPI_Abort(MPI_COMM_WORLD, 1);
+            }
+            ++again;
+        }
+    }
 }
-
diff --git a/test/simple/simple_spawn.c b/test/simple/simple_spawn.c
index c8d1093ba94..d71f579ee9f 100644
--- a/test/simple/simple_spawn.c
+++ b/test/simple/simple_spawn.c
@@ -1,27 +1,26 @@
 #include <stdio.h>
 #include <stdlib.h>
+#include <sys/param.h>
 #include <sys/types.h>
 #include <unistd.h>
-#include <sys/param.h>
-
 
 #include <mpi.h>
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int msg, rc;
     MPI_Comm parent, child;
     int rank, size;
     char hostname[1024];
     pid_t pid;
-    char *env_rank,*env_nspace;
+    char *env_rank, *env_nspace;
 
     env_rank = getenv("PMIX_RANK");
     env_nspace = getenv("PMIX_NAMESPACE");
     pid = getpid();
     gethostname(hostname, 1024);
 
-    printf("[%s:%s pid %ld] starting up on node %s!\n", env_nspace, env_rank, (long)pid, hostname);
+    printf("[%s:%s pid %ld] starting up on node %s!\n", env_nspace, env_rank, (long) pid, hostname);
 
     MPI_Init(NULL, NULL);
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
@@ -31,9 +30,9 @@ int main(int argc, char* argv[])
     /* If we get COMM_NULL back, then we're the parent */
     if (MPI_COMM_NULL == parent) {
         pid = getpid();
-        printf("Parent [pid %ld] about to spawn!\n", (long)pid);
-        rc = MPI_Comm_spawn(argv[0], MPI_ARGV_NULL, 3, MPI_INFO_NULL,
-                            0, MPI_COMM_WORLD, &child, MPI_ERRCODES_IGNORE);
+        printf("Parent [pid %ld] about to spawn!\n", (long) pid);
+        rc = MPI_Comm_spawn(argv[0], MPI_ARGV_NULL, 3, MPI_INFO_NULL, 0, MPI_COMM_WORLD, &child,
+                            MPI_ERRCODES_IGNORE);
         if (MPI_SUCCESS != rc) {
             printf("Child failed to spawn\n");
             return rc;
@@ -52,7 +51,7 @@ int main(int argc, char* argv[])
         MPI_Comm_rank(MPI_COMM_WORLD, &rank);
         MPI_Comm_size(MPI_COMM_WORLD, &size);
         pid = getpid();
-        printf("Hello from the child %d of %d on host %s pid %ld\n", rank, 3, hostname, (long)pid);
+        printf("Hello from the child %d of %d on host %s pid %ld\n", rank, 3, hostname, (long) pid);
         if (0 == rank) {
             MPI_Recv(&msg, 1, MPI_INT, 0, 1, parent, MPI_STATUS_IGNORE);
             printf("Child %d received msg: %d\n", rank, msg);
diff --git a/test/simple/singleton_client_server.c b/test/simple/singleton_client_server.c
index 796e30bd5b7..08e68e5cc31 100644
--- a/test/simple/singleton_client_server.c
+++ b/test/simple/singleton_client_server.c
@@ -2,12 +2,12 @@
 
 #include "opal/runtime/opal.h"
 
+#include <errno.h>
+#include <mpi.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <errno.h>
 #include <unistd.h>
-#include <mpi.h>
 
 /*
 
@@ -35,178 +35,164 @@
 
 #define TAG 0
 
-#define CHK(code) do                            \
-  {                                             \
-    int retval = code ;                         \
-    if (retval != MPI_SUCCESS)                  \
-    {                                           \
-      fprintf(stderr, "Error: " #code "\n") ;   \
-      exit(1) ;                                 \
-    }                                           \
-  } while(0)
+#define CHK(code)                                  \
+    do {                                           \
+        int retval = code;                         \
+        if (retval != MPI_SUCCESS) {               \
+            fprintf(stderr, "Error: " #code "\n"); \
+            exit(1);                               \
+        }                                          \
+    } while (0)
 
 int main(int argc, char *argv[])
 {
-  const char *hostname ;
-  char buff[255] ;
-
-  int role ;
-  int num_clients ;
-  int size, rank ;
-  int temp_errno ;
-
-  FILE *fp ;
-  char server_port_name[MPI_MAX_PORT_NAME] ;
-
-  MPI_Comm intercomm, intracomm ;
-  MPI_Status status ;
-  int msg_count ;
-  int i ;
-
-  /* sanity check the args */
-  if(argc != 3)
-  {
-    fprintf(stderr, "usage %s <num clients> <1:server | 0:client>\n", argv[0]) ;
-    exit(1) ;
-  }
-
-  num_clients = atoi(argv[1]) ;
-  role = atoi(argv[2]) ;
-
-  if (num_clients <= 0 || (role != 0 && role != 1))
-  {
-    fprintf(stderr, "usage %s <num clients> <1:server | 0:client>\n", argv[0]) ;
-    exit(1) ;
-  }
-
-  /* initialize MPI  */
-  CHK(MPI_Init(&argc, &argv)) ;
-
-  /* get the node name */
-  /* The opal_gethostname() function is just a wrapper that returns a global
-     variable value that is set earlier, so we don't check for errors here */
-  hostname = opal_gethostname();
-
-  /* server */
-  if(role == 1)
-  {
-    printf("SERVER: on node '%s'\n", hostname) ;
-
-    /* open port to establish connections */
-    CHK(MPI_Open_port(MPI_INFO_NULL, server_port_name)) ;
-
-    printf("SERVER: opened port=%s\n", server_port_name) ;
-
-    /* store the port name */
-    fp = fopen("server_port_name.txt", "w") ;
-    if(fp == NULL)
-    {
-      fprintf(stderr, "fopen failed: %s\n", strerror(errno)) ;
-      exit(1) ;
+    const char *hostname;
+    char buff[255];
+
+    int role;
+    int num_clients;
+    int size, rank;
+    int temp_errno;
+
+    FILE *fp;
+    char server_port_name[MPI_MAX_PORT_NAME];
+
+    MPI_Comm intercomm, intracomm;
+    MPI_Status status;
+    int msg_count;
+    int i;
+
+    /* sanity check the args */
+    if (argc != 3) {
+        fprintf(stderr, "usage %s <num clients> <1:server | 0:client>\n", argv[0]);
+        exit(1);
     }
-    fprintf(fp, "%s", server_port_name) ;
-    fclose(fp) ;
 
-    /* the server accepts connections from all the clients */
-    for(i = 0 ; i < num_clients ; i++ )
-    {
-      /* accept connections at this port */
-      CHK(MPI_Comm_accept(server_port_name, MPI_INFO_NULL, 0,
-                          i == 0 ? MPI_COMM_WORLD : intracomm,
-                          &intercomm)) ;
+    num_clients = atoi(argv[1]);
+    role = atoi(argv[2]);
+
+    if (num_clients <= 0 || (role != 0 && role != 1)) {
+        fprintf(stderr, "usage %s <num clients> <1:server | 0:client>\n", argv[0]);
+        exit(1);
+    }
 
-      printf("SERVER: accepted connection from client %d\n", i+1) ;
+    /* initialize MPI  */
+    CHK(MPI_Init(&argc, &argv));
 
-      /* merge, to form one intra communicator */
-      CHK(MPI_Intercomm_merge(intercomm, 0, &intracomm)) ;
+    /* get the node name */
+    /* The opal_gethostname() function is just a wrapper that returns a global
+       variable value that is set earlier, so we don't check for errors here */
+    hostname = opal_gethostname();
 
-      printf("SERVER: merged with client %d\n", i+1) ;
+    /* server */
+    if (role == 1) {
+        printf("SERVER: on node '%s'\n", hostname);
 
-      CHK(MPI_Comm_size(intracomm, &size)) ;
-      CHK(MPI_Comm_rank(intracomm, &rank)) ;
+        /* open port to establish connections */
+        CHK(MPI_Open_port(MPI_INFO_NULL, server_port_name));
 
-      printf("SERVER: after merging with client %d: size=%d rank=%d\n", i+1, size, rank) ;
-    }
-  } /* end server */
-
-  /* client */
-  if(role == 0)
-  {
-    printf("CLIENT: on node '%s'\n", hostname) ;
-
-    fp = fopen("server_port_name.txt", "r") ;
-    if(fp == NULL)
-    {
-      fprintf(stderr, "fopen failed: %s\n", strerror(errno)) ;
-      exit(1) ;
-    }
-    fscanf(fp, "%s", server_port_name) ;
-    fclose(fp) ;
+        printf("SERVER: opened port=%s\n", server_port_name);
 
-    printf("CLIENT: attempting to connect to server on port=%s\n", server_port_name) ;
+        /* store the port name */
+        fp = fopen("server_port_name.txt", "w");
+        if (fp == NULL) {
+            fprintf(stderr, "fopen failed: %s\n", strerror(errno));
+            exit(1);
+        }
+        fprintf(fp, "%s", server_port_name);
+        fclose(fp);
 
-    /* connect to the server */
-    CHK(MPI_Comm_connect (server_port_name, MPI_INFO_NULL, 0, MPI_COMM_WORLD, &intercomm)) ;
+        /* the server accepts connections from all the clients */
+        for (i = 0; i < num_clients; i++) {
+            /* accept connections at this port */
+            CHK(MPI_Comm_accept(server_port_name, MPI_INFO_NULL, 0,
+                                i == 0 ? MPI_COMM_WORLD : intracomm, &intercomm));
 
-    printf("CLIENT: connected to server on port\n") ;
+            printf("SERVER: accepted connection from client %d\n", i + 1);
 
-    /* merge the server and client to one intra communicator */
-    CHK(MPI_Intercomm_merge(intercomm, 1, &intracomm)) ;
+            /* merge, to form one intra communicator */
+            CHK(MPI_Intercomm_merge(intercomm, 0, &intracomm));
 
-    printf("CLIENT: merged with existing intracomm\n") ;
+            printf("SERVER: merged with client %d\n", i + 1);
 
-    CHK(MPI_Comm_size(intracomm, &size)) ;
-    CHK(MPI_Comm_rank(intracomm, &rank)) ;
+            CHK(MPI_Comm_size(intracomm, &size));
+            CHK(MPI_Comm_rank(intracomm, &rank));
 
-    printf("CLIENT: after merging, new comm: size=%d rank=%d\n", size, rank) ;
+            printf("SERVER: after merging with client %d: size=%d rank=%d\n", i + 1, size, rank);
+        }
+    } /* end server */
 
-    for (i = rank ; i < num_clients ; i++)
-    {
-      /* client performs a collective accept */
-      CHK(MPI_Comm_accept(server_port_name, MPI_INFO_NULL, 0, intracomm, &intercomm)) ;
+    /* client */
+    if (role == 0) {
+        printf("CLIENT: on node '%s'\n", hostname);
 
-      printf("CLIENT: connected to server on port\n") ;
+        fp = fopen("server_port_name.txt", "r");
+        if (fp == NULL) {
+            fprintf(stderr, "fopen failed: %s\n", strerror(errno));
+            exit(1);
+        }
+        fscanf(fp, "%s", server_port_name);
+        fclose(fp);
 
-      /* merge the two intra comms back to one communicator */
-      CHK(MPI_Intercomm_merge(intercomm, 0, &intracomm)) ;
+        printf("CLIENT: attempting to connect to server on port=%s\n", server_port_name);
 
-      printf("CLIENT: merged with existing members\n") ;
+        /* connect to the server */
+        CHK(MPI_Comm_connect(server_port_name, MPI_INFO_NULL, 0, MPI_COMM_WORLD, &intercomm));
 
-      CHK(MPI_Comm_size(intracomm, &size)) ;
-      CHK(MPI_Comm_rank(intracomm, &rank)) ;
+        printf("CLIENT: connected to server on port\n");
 
-      printf("CLIENT: new size after merging with existing members: size=%d rank=%d\n", size, rank) ;
-    }
+        /* merge the server and client to one intra communicator */
+        CHK(MPI_Intercomm_merge(intercomm, 1, &intracomm));
 
-  } /* end client */
+        printf("CLIENT: merged with existing intracomm\n");
 
-  CHK(MPI_Comm_size(intracomm, &size)) ;
-  CHK(MPI_Comm_rank(intracomm, &rank)) ;
+        CHK(MPI_Comm_size(intracomm, &size));
+        CHK(MPI_Comm_rank(intracomm, &rank));
 
-  printf("After fusion: size=%d rank=%d\n", size, rank) ;
+        printf("CLIENT: after merging, new comm: size=%d rank=%d\n", size, rank);
 
-  if(rank == 0)
-  {
-    msg_count = num_clients ;
+        for (i = rank; i < num_clients; i++) {
+            /* client performs a collective accept */
+            CHK(MPI_Comm_accept(server_port_name, MPI_INFO_NULL, 0, intracomm, &intercomm));
 
-    while(msg_count)
-    {
-      CHK(MPI_Recv(buff, 255, MPI_CHAR, MPI_ANY_SOURCE,
-                   MPI_ANY_TAG, intracomm, &status)) ;
+            printf("CLIENT: connected to server on port\n");
 
-      printf("Received hello msg from '%s'\n", buff) ;
-      msg_count-- ;
-    }
-  }
-  else
-  {
-    /* all ranks > 0 */
+            /* merge the two intra comms back to one communicator */
+            CHK(MPI_Intercomm_merge(intercomm, 0, &intracomm));
 
-    CHK(MPI_Send(hostname, strlen(hostname) + 1, MPI_CHAR, 0, TAG, intracomm)) ;
-  }
+            printf("CLIENT: merged with existing members\n");
+
+            CHK(MPI_Comm_size(intracomm, &size));
+            CHK(MPI_Comm_rank(intracomm, &rank));
+
+            printf("CLIENT: new size after merging with existing members: size=%d rank=%d\n", size,
+                   rank);
+        }
+
+    } /* end client */
+
+    CHK(MPI_Comm_size(intracomm, &size));
+    CHK(MPI_Comm_rank(intracomm, &rank));
+
+    printf("After fusion: size=%d rank=%d\n", size, rank);
+
+    if (rank == 0) {
+        msg_count = num_clients;
+
+        while (msg_count) {
+            CHK(MPI_Recv(buff, 255, MPI_CHAR, MPI_ANY_SOURCE, MPI_ANY_TAG, intracomm, &status));
+
+            printf("Received hello msg from '%s'\n", buff);
+            msg_count--;
+        }
+    } else {
+        /* all ranks > 0 */
+
+        CHK(MPI_Send(hostname, strlen(hostname) + 1, MPI_CHAR, 0, TAG, intracomm));
+    }
 
-  CHK(MPI_Finalize()) ;
+    CHK(MPI_Finalize());
 
-  fprintf(stderr, "Rank %d is exiting\n", rank);
-  return 0 ;
+    fprintf(stderr, "Rank %d is exiting\n", rank);
+    return 0;
 }
diff --git a/test/simple/sio.c b/test/simple/sio.c
index bddd5320298..e500bb8c203 100644
--- a/test/simple/sio.c
+++ b/test/simple/sio.c
@@ -5,16 +5,16 @@
  * The most basic of MPI applications
  */
 
-#include <stdio.h>
 #include "mpi.h"
+#include <stdio.h>
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int i;
 
     MPI_Init(&argc, &argv);
 
-    for (i=0; i < 100; i++) {
+    for (i = 0; i < 100; i++) {
         printf("some output from mpitest to test the xml problem: %d\n", i);
     }
 
diff --git a/test/simple/slave.c b/test/simple/slave.c
index 54561c6d576..a00add41cfa 100644
--- a/test/simple/slave.c
+++ b/test/simple/slave.c
@@ -5,17 +5,17 @@
  * The most basic of MPI applications
  */
 
+#include "mpi.h"
 #include <stdio.h>
 #include <unistd.h>
-#include "mpi.h"
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int rank, size;
     MPI_Comm parent;
     int msg;
 
-    printf("Slave [pid %ld] starting up!\n", (long)getpid());
+    printf("Slave [pid %ld] starting up!\n", (long) getpid());
 
     MPI_Init(&argc, &argv);
 
diff --git a/test/simple/spawn-problem/ch_rec.c b/test/simple/spawn-problem/ch_rec.c
index 982c1d91e7c..0ffa446146b 100644
--- a/test/simple/spawn-problem/ch_rec.c
+++ b/test/simple/spawn-problem/ch_rec.c
@@ -1,52 +1,50 @@
-#include <stdlib.h>
+#include <mpi.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
-#include <mpi.h>
 
-int main (int argc, char **argv){
-	char buff[30];
-	MPI_Status st;
-	MPI_Comm comm[2], parent;
-	MPI_Request req[2];
-	int errcodes[1];
-	int level;
-	int x = 6, i, j;
+int main(int argc, char **argv)
+{
+    char buff[30];
+    MPI_Status st;
+    MPI_Comm comm[2], parent;
+    MPI_Request req[2];
+    int errcodes[1];
+    int level;
+    int x = 6, i, j;
 
-	MPI_Init(&argc, &argv);
-	MPI_Comm_get_parent(&parent);
-	argv++;
-	level = atoi(argv[0]);
-	printf("level = %d\n",level);
+    MPI_Init(&argc, &argv);
+    MPI_Comm_get_parent(&parent);
+    argv++;
+    level = atoi(argv[0]);
+    printf("level = %d\n", level);
 
-	MPI_Recv(&buff, sizeof(char)*30, MPI_CHAR, MPI_ANY_SOURCE,
-						MPI_ANY_TAG, parent, &st);
-  printf("Parent sent: %s\n", buff);
+    MPI_Recv(&buff, sizeof(char) * 30, MPI_CHAR, MPI_ANY_SOURCE, MPI_ANY_TAG, parent, &st);
+    printf("Parent sent: %s\n", buff);
 
-	if(level < x){
-		sprintf(argv[0], "%d", level+1);
-		MPI_Comm_spawn("ch_rec", argv, 1, MPI_INFO_NULL, 0, MPI_COMM_SELF,
-										&comm[0], errcodes);
-		sprintf(buff,"level %d (pid:%d)", level, getpid());
-		MPI_Send(&buff, sizeof(char)*30, MPI_CHAR, 0, 100, comm[0]);
-		MPI_Irecv(&buff, sizeof(char)*30, MPI_CHAR, MPI_ANY_SOURCE,
-							MPI_ANY_TAG, comm[0], &req[0]);
+    if (level < x) {
+        sprintf(argv[0], "%d", level + 1);
+        MPI_Comm_spawn("ch_rec", argv, 1, MPI_INFO_NULL, 0, MPI_COMM_SELF, &comm[0], errcodes);
+        sprintf(buff, "level %d (pid:%d)", level, getpid());
+        MPI_Send(&buff, sizeof(char) * 30, MPI_CHAR, 0, 100, comm[0]);
+        MPI_Irecv(&buff, sizeof(char) * 30, MPI_CHAR, MPI_ANY_SOURCE, MPI_ANY_TAG, comm[0],
+                  &req[0]);
 
-		//sleep(2);
-		sprintf(argv[0], "%d", (level+1));
-		MPI_Comm_spawn("ch_rec", argv, 1, MPI_INFO_NULL, 0, MPI_COMM_SELF,
-										&comm[1], errcodes);
-		sprintf(buff,"level %d (pid:%d)", level, getpid());
-		MPI_Send(&buff, sizeof(char)*30, MPI_CHAR, 0, 100, comm[1]);
-		MPI_Irecv(&buff, sizeof(char)*30, MPI_CHAR, MPI_ANY_SOURCE,
-						  MPI_ANY_TAG, comm[1], &req[1]);
+        // sleep(2);
+        sprintf(argv[0], "%d", (level + 1));
+        MPI_Comm_spawn("ch_rec", argv, 1, MPI_INFO_NULL, 0, MPI_COMM_SELF, &comm[1], errcodes);
+        sprintf(buff, "level %d (pid:%d)", level, getpid());
+        MPI_Send(&buff, sizeof(char) * 30, MPI_CHAR, 0, 100, comm[1]);
+        MPI_Irecv(&buff, sizeof(char) * 30, MPI_CHAR, MPI_ANY_SOURCE, MPI_ANY_TAG, comm[1],
+                  &req[1]);
 
-		for(i=0; i<2; i++){
-			MPI_Waitany(2, req, &j, MPI_STATUS_IGNORE);
-			printf("Child %d sent: %s\n", j, buff);
-		}
-	}
-	sprintf(buff,"level %d (pid:%d)", level, getpid());
-	MPI_Send(&buff, sizeof(char)*30, MPI_CHAR, 0, 100, parent);
-	MPI_Finalize();
-	return 0;
+        for (i = 0; i < 2; i++) {
+            MPI_Waitany(2, req, &j, MPI_STATUS_IGNORE);
+            printf("Child %d sent: %s\n", j, buff);
+        }
+    }
+    sprintf(buff, "level %d (pid:%d)", level, getpid());
+    MPI_Send(&buff, sizeof(char) * 30, MPI_CHAR, 0, 100, parent);
+    MPI_Finalize();
+    return 0;
 }
diff --git a/test/simple/spawn-problem/start.c b/test/simple/spawn-problem/start.c
index 5a281ab5bb9..897786d22ad 100644
--- a/test/simple/spawn-problem/start.c
+++ b/test/simple/spawn-problem/start.c
@@ -1,28 +1,27 @@
-#include <stdlib.h>
+#include <mpi.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
-#include <mpi.h>
 
-int main (int argc, char **argv){
-	char buff[30];
-	MPI_Status st;
-	MPI_Comm comm;
-	int errcodes[1];
+int main(int argc, char **argv)
+{
+    char buff[30];
+    MPI_Status st;
+    MPI_Comm comm;
+    int errcodes[1];
 
-	MPI_Init(&argc, &argv);
-	int level = 0;
-	printf("level %d\n", level);
+    MPI_Init(&argc, &argv);
+    int level = 0;
+    printf("level %d\n", level);
 
-	sprintf(argv[0], "%d", level+1);
-	MPI_Comm_spawn("ch_rec", argv, 1, MPI_INFO_NULL, 0, MPI_COMM_SELF,
-									&comm, errcodes);
-	sprintf(buff,"level %d (pid:%d)", level, getpid());
-	MPI_Send(&buff, sizeof(char)*30, MPI_CHAR, 0, 100, comm);
+    sprintf(argv[0], "%d", level + 1);
+    MPI_Comm_spawn("ch_rec", argv, 1, MPI_INFO_NULL, 0, MPI_COMM_SELF, &comm, errcodes);
+    sprintf(buff, "level %d (pid:%d)", level, getpid());
+    MPI_Send(&buff, sizeof(char) * 30, MPI_CHAR, 0, 100, comm);
 
-	MPI_Recv(&buff, sizeof(char)*30, MPI_CHAR, MPI_ANY_SOURCE,
-						MPI_ANY_TAG, comm, &st);
-	printf("Child sent: %s\n", buff);
+    MPI_Recv(&buff, sizeof(char) * 30, MPI_CHAR, MPI_ANY_SOURCE, MPI_ANY_TAG, comm, &st);
+    printf("Child sent: %s\n", buff);
 
-	MPI_Finalize();
-	return 0;
+    MPI_Finalize();
+    return 0;
 }
diff --git a/test/simple/spawn_multiple.c b/test/simple/spawn_multiple.c
index 7607a59983e..f586e9c19e3 100644
--- a/test/simple/spawn_multiple.c
+++ b/test/simple/spawn_multiple.c
@@ -8,7 +8,7 @@
 
 #include <mpi.h>
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int msg;
     MPI_Comm parent, child;
@@ -17,11 +17,11 @@ int main(int argc, char* argv[])
     pid_t pid;
     int i;
     char *cmds[2];
-    char *argv0[] = { "foo", NULL };
-    char *argv1[] = { "bar", NULL };
+    char *argv0[] = {"foo", NULL};
+    char *argv1[] = {"bar", NULL};
     char **spawn_argv[2];
-    int maxprocs[] = { 1, 1 };
-    MPI_Info info[] = { MPI_INFO_NULL, MPI_INFO_NULL };
+    int maxprocs[] = {1, 1};
+    MPI_Info info[] = {MPI_INFO_NULL, MPI_INFO_NULL};
 
     cmds[1] = cmds[0] = argv[0];
     spawn_argv[0] = argv0;
@@ -34,10 +34,9 @@ int main(int argc, char* argv[])
     /* If we get COMM_NULL back, then we're the parent */
     if (MPI_COMM_NULL == parent) {
         pid = getpid();
-        printf("Parent [pid %ld] about to spawn!\n", (long)pid);
-        MPI_Comm_spawn_multiple(2, cmds, spawn_argv, maxprocs,
-                                info, 0, MPI_COMM_WORLD,
-                                &child, MPI_ERRCODES_IGNORE);
+        printf("Parent [pid %ld] about to spawn!\n", (long) pid);
+        MPI_Comm_spawn_multiple(2, cmds, spawn_argv, maxprocs, info, 0, MPI_COMM_WORLD, &child,
+                                MPI_ERRCODES_IGNORE);
         printf("Parent done with spawn\n");
         if (0 == rank) {
             msg = 38;
@@ -54,7 +53,8 @@ int main(int argc, char* argv[])
         MPI_Comm_size(MPI_COMM_WORLD, &size);
         hostname = opal_gethostname();
         pid = getpid();
-        printf("Hello from the child %d of %d on host %s pid %ld: argv[1] = %s\n", rank, size, hostname, (long)pid, argv[1]);
+        printf("Hello from the child %d of %d on host %s pid %ld: argv[1] = %s\n", rank, size,
+               hostname, (long) pid, argv[1]);
         MPI_Recv(&msg, 1, MPI_INT, 0, 1, parent, MPI_STATUS_IGNORE);
         printf("Child %d received msg: %d\n", rank, msg);
         MPI_Comm_disconnect(&parent);
diff --git a/test/simple/spawn_tree.c b/test/simple/spawn_tree.c
index 123d7d3063b..5dd512e175b 100644
--- a/test/simple/spawn_tree.c
+++ b/test/simple/spawn_tree.c
@@ -2,14 +2,15 @@
 
 #include "opal/runtime/opal.h"
 
+#include <math.h>
+#include <mpi.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <math.h>
 #include <unistd.h>
-#include <mpi.h>
 
-int main(int argc, char ** argv){
+int main(int argc, char **argv)
+{
 
     int i;
     int rank, size, child_rank;
@@ -18,7 +19,6 @@ int main(int argc, char ** argv){
     int erro;
     int level, curr_level;
 
-
     if (argc < 2) {
         fprintf(stderr, "Usage: spawn_tree <#levels>\n");
         exit(1);
@@ -27,47 +27,41 @@ int main(int argc, char ** argv){
 
     MPI_Init(&argc, &argv);
 
-
     MPI_Comm_get_parent(&parent);
 
-    if(parent == MPI_COMM_NULL){
-        rank=0;
-    }
-    else{
+    if (parent == MPI_COMM_NULL) {
+        rank = 0;
+    } else {
         MPI_Recv(&rank, 1, MPI_INT, 0, 0, parent, MPI_STATUS_IGNORE);
     }
 
-    curr_level = (int) log2(rank+1);
+    curr_level = (int) log2(rank + 1);
 
     printf(" --> rank: %d and curr_level: %d\n", rank, curr_level);
 
     // Node propagation
-    if(curr_level < level){
+    if (curr_level < level) {
         // 2^(curr_level+1) - 1 + 2*(rank - 2^curr_level - 1) = 2*rank + 1
-        child_rank = 2*rank + 1;
+        child_rank = 2 * rank + 1;
         printf("(%d) Before create rank %d\n", rank, child_rank);
-        MPI_Comm_spawn(argv[0], &argv[1], 1, MPI_INFO_NULL, 0,
-                       MPI_COMM_SELF, &intercomm1, &erro);
+        MPI_Comm_spawn(argv[0], &argv[1], 1, MPI_INFO_NULL, 0, MPI_COMM_SELF, &intercomm1, &erro);
         printf("(%d) After create rank %d\n", rank, child_rank);
 
         MPI_Send(&child_rank, 1, MPI_INT, 0, 0, intercomm1);
 
-        //sleep(1);
+        // sleep(1);
 
         child_rank = child_rank + 1;
         printf("(%d) Before create rank %d\n", rank, child_rank);
-        MPI_Comm_spawn(argv[0], &argv[1], 1, MPI_INFO_NULL, 0,
-                       MPI_COMM_SELF, &intercomm2, &erro);
+        MPI_Comm_spawn(argv[0], &argv[1], 1, MPI_INFO_NULL, 0, MPI_COMM_SELF, &intercomm2, &erro);
         printf("(%d) After create rank %d\n", rank, child_rank);
 
         MPI_Send(&child_rank, 1, MPI_INT, 0, 0, intercomm2);
-
     }
 
     nomehost = opal_gethostname();
     printf("(%d) in %s\n", rank, nomehost);
 
     MPI_Finalize();
-    return(0);
-
+    return (0);
 }
diff --git a/test/simple/thread_init.c b/test/simple/thread_init.c
index 6021c4674e9..d610f4d2b40 100644
--- a/test/simple/thread_init.c
+++ b/test/simple/thread_init.c
@@ -1,11 +1,11 @@
 #include <mpi.h>
 #include <stdio.h>
-int main(int argc, const char* argv[]) {
- int provided = -1;
- printf("Calling MPI_Init_thread...\n");
- MPI_Init_thread(NULL, NULL, MPI_THREAD_MULTIPLE, &provided);
- printf("MPI_Init_thread returned, provided = %d\n", provided);
- MPI_Finalize();
- return 0;
+int main(int argc, const char *argv[])
+{
+    int provided = -1;
+    printf("Calling MPI_Init_thread...\n");
+    MPI_Init_thread(NULL, NULL, MPI_THREAD_MULTIPLE, &provided);
+    printf("MPI_Init_thread returned, provided = %d\n", provided);
+    MPI_Finalize();
+    return 0;
 }
-
diff --git a/test/simple/xlib.c b/test/simple/xlib.c
index e75a874fab0..aa146794c8b 100644
--- a/test/simple/xlib.c
+++ b/test/simple/xlib.c
@@ -1,10 +1,10 @@
-#include <stdlib.h>
-#include <stdio.h>
 #include <mpi.h>
 #include <pmix.h>
+#include <stdio.h>
+#include <stdlib.h>
 
-#define SIZE 20
-#define POS 10
+#define SIZE          20
+#define POS           10
 #define INITIAL_VALUE 10
 
 static pmix_proc_t myproc;
@@ -13,13 +13,10 @@ static pmix_proc_t myproc;
  * when registering for general events - i.e.,, the default
  * handler. We don't technically need to register one, but it
  * is usually good practice to catch any events that occur */
-static void notification_fn(size_t evhdlr_registration_id,
-                            pmix_status_t status,
-                            const pmix_proc_t *source,
-                            pmix_info_t info[], size_t ninfo,
+static void notification_fn(size_t evhdlr_registration_id, pmix_status_t status,
+                            const pmix_proc_t *source, pmix_info_t info[], size_t ninfo,
                             pmix_info_t results[], size_t nresults,
-                            pmix_event_notification_cbfunc_fn_t cbfunc,
-                            void *cbdata)
+                            pmix_event_notification_cbfunc_fn_t cbfunc, void *cbdata)
 {
     /* this example doesn't do anything with default events */
     fprintf(stderr, "Default event handler called with status %s\n", PMIx_Error_string(status));
@@ -36,20 +33,18 @@ static void notification_fn(size_t evhdlr_registration_id,
  * to declare a use-specific notification callback point. In this case,
  * we are asking to know whenever a programming model library is
  * instantiated */
-static void model_callback(size_t evhdlr_registration_id,
-                           pmix_status_t status,
-                           const pmix_proc_t *source,
-                           pmix_info_t info[], size_t ninfo,
+static void model_callback(size_t evhdlr_registration_id, pmix_status_t status,
+                           const pmix_proc_t *source, pmix_info_t info[], size_t ninfo,
                            pmix_info_t results[], size_t nresults,
-                           pmix_event_notification_cbfunc_fn_t cbfunc,
-                           void *cbdata)
+                           pmix_event_notification_cbfunc_fn_t cbfunc, void *cbdata)
 {
     size_t n;
 
-    fprintf(stderr, "Model event handler called with status %d(%s)\n", status, PMIx_Error_string(status));
+    fprintf(stderr, "Model event handler called with status %d(%s)\n", status,
+            PMIx_Error_string(status));
 
     /* check to see what model declared itself */
-    for (n=0; n < ninfo; n++) {
+    for (n = 0; n < ninfo; n++) {
         if (PMIX_STRING == info[n].value.type) {
             fprintf(stderr, "\t%s:\t%s\n", info[n].key, info[n].value.data.string);
         }
@@ -70,15 +65,13 @@ static void model_callback(size_t evhdlr_registration_id,
  * to the registered event. The index is used later on to deregister
  * an event handler - if we don't explicitly deregister it, then the
  * PMIx server will do so when it see us exit */
-static void model_registration_callback(pmix_status_t status,
-                                        size_t evhandler_ref,
-                                        void *cbdata)
+static void model_registration_callback(pmix_status_t status, size_t evhandler_ref, void *cbdata)
 {
-    volatile int *active = (volatile int*)cbdata;
+    volatile int *active = (volatile int *) cbdata;
 
     if (PMIX_SUCCESS != status) {
         fprintf(stderr, "Client %s:%d EVENT HANDLER REGISTRATION FAILED WITH STATUS %d, ref=%lu\n",
-                   myproc.nspace, myproc.rank, status, (unsigned long)evhandler_ref);
+                myproc.nspace, myproc.rank, status, (unsigned long) evhandler_ref);
     }
     *active = status;
 }
@@ -96,7 +89,6 @@ int main(int argc, char *argv[])
     pmix_status_t rc;
     volatile int active;
 
-
     if (1 < argc) {
         fprintf(stderr, "Declaring ourselves\n");
         /* declare ourselves as a non-MPI library prior to MPI_Init */
@@ -117,8 +109,8 @@ int main(int argc, char *argv[])
         ninfo = 1;
         PMIX_INFO_CREATE(info, ninfo);
         PMIX_INFO_LOAD(&info[0], PMIX_EVENT_HDLR_NAME, "APP-MODEL", PMIX_STRING);
-        PMIx_Register_event_handler(&code, 1, info, ninfo,
-                                    model_callback, model_registration_callback, (void*)&active);
+        PMIx_Register_event_handler(&code, 1, info, ninfo, model_callback,
+                                    model_registration_callback, (void *) &active);
         while (-1 == active) {
             usleep(10);
         }
@@ -140,8 +132,8 @@ int main(int argc, char *argv[])
         PMIX_INFO_CREATE(info, ninfo);
         PMIX_INFO_LOAD(&info[0], PMIX_EVENT_HDLR_NAME, "APP-MODEL", PMIX_STRING);
 
-        PMIx_Register_event_handler(&code, 1, info, ninfo,
-                                    model_callback, model_registration_callback, (void*)&active);
+        PMIx_Register_event_handler(&code, 1, info, ninfo, model_callback,
+                                    model_registration_callback, (void *) &active);
         while (-1 == active) {
             usleep(10);
         }
@@ -165,8 +157,7 @@ int main(int argc, char *argv[])
 
     if (0 == rank) {
         send_array[pos] = INITIAL_VALUE;
-        MPI_Send(send_array, array_size, MPI_INT, next, tag,
-                 MPI_COMM_WORLD);
+        MPI_Send(send_array, array_size, MPI_INT, next, tag, MPI_COMM_WORLD);
     }
 
     /* if we didn't already do it, declare another model now */
@@ -189,8 +180,7 @@ int main(int argc, char *argv[])
 
     while (1) {
         recv_array[pos] = -1;
-        MPI_Recv(recv_array, array_size, MPI_INT, prev, tag,
-                 MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+        MPI_Recv(recv_array, array_size, MPI_INT, prev, tag, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
         send_array[pos] = recv_array[pos];
         if (rank == 0) {
             --send_array[pos];
@@ -202,8 +192,7 @@ int main(int argc, char *argv[])
     }
 
     if (rank == 0) {
-        MPI_Recv(recv_array, array_size, MPI_INT, prev, tag,
-                 MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+        MPI_Recv(recv_array, array_size, MPI_INT, prev, tag, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
     }
 
     fprintf(stderr, "Rank %d has completed ring\n", rank);
diff --git a/test/simple/ziaprobe.c b/test/simple/ziaprobe.c
index c9bea293747..38da892294d 100644
--- a/test/simple/ziaprobe.c
+++ b/test/simple/ziaprobe.c
@@ -10,17 +10,17 @@
  * $HEADER$
  *
  */
-#include <stdio.h>
 #include <stdbool.h>
-#include <sys/types.h>
-#include <unistd.h>
+#include <stdio.h>
 #include <stdlib.h>
-#include <time.h>
 #include <sys/time.h>
+#include <sys/types.h>
+#include <time.h>
+#include <unistd.h>
 
 #include <mpi.h>
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int msg;
     int rank, size, my_twin;
@@ -54,8 +54,10 @@ int main(int argc, char* argv[])
     /* this program requires that the size be an integer multiple of ppn */
     if (0 != (size % ppn)) {
         if (0 == rank) {
-            fprintf(stderr, "The number of procs must be an integer multiple of the ppn\n"
-                    "Given: num_procs %d ppn %d\n", size, ppn);
+            fprintf(stderr,
+                    "The number of procs must be an integer multiple of the ppn\n"
+                    "Given: num_procs %d ppn %d\n",
+                    size, ppn);
             MPI_Abort(MPI_COMM_WORLD, 1);
         } else {
             goto cleanup;
@@ -84,7 +86,7 @@ int main(int argc, char* argv[])
     /* first, determine if my node is odd or even */
     my_node = rank / ppn;
 
-     if (0 != (my_node % 2)) {
+    if (0 != (my_node % 2)) {
         /* compute my twin's rank - as I am an odd numbered node, my
          * twin will be on the node below me. Thus, its rank will be
          * my rank - ppn
@@ -145,8 +147,7 @@ int main(int argc, char* argv[])
             MPI_Abort(MPI_COMM_WORLD, 1);
         }
     }
-    MPI_Gather(&my_timestamp, 2, MPI_LONG,
-               timestamps, 2, MPI_LONG, 0, MPI_COMM_WORLD);
+    MPI_Gather(&my_timestamp, 2, MPI_LONG, timestamps, 2, MPI_LONG, 0, MPI_COMM_WORLD);
     if (0 == rank) {
         /* The "timestamps" array will now have everyone's timestamp
          (i.e., rank 0's timestamp will be in pos 0 & 1,, rank 1's timestamp
@@ -155,17 +156,16 @@ int main(int argc, char* argv[])
         maxsec = start_sec;
         maxusec = start_usec;
         maxrank = -1;
-        for (i=0; i < 2*size; i+=2) {
+        for (i = 0; i < 2 * size; i += 2) {
             if (timestamps[i] < maxsec) {
                 continue;
             }
-            if (timestamps[i] == maxsec &&
-                timestamps[i+1] < maxusec) {
+            if (timestamps[i] == maxsec && timestamps[i + 1] < maxusec) {
                 continue;
             }
             maxsec = timestamps[i];
-            maxusec = timestamps[i+1];
-            maxrank = i/2;
+            maxusec = timestamps[i + 1];
+            maxrank = i / 2;
         }
         free(timestamps);
         /* subtract starting time to get time in microsecs for test */
@@ -181,9 +181,9 @@ int main(int argc, char* argv[])
         minutes = seconds / 60l;
         seconds = seconds % 60l;
         if (0 == minutes && 0 == seconds) {
-            fsecs = ((float)(maxsec)*1000000.0 + (float)maxusec) / 1000.0;
-            fprintf(stderr, "Time test was completed in %8.2f millisecs\nSlowest rank: %d\n",
-                    fsecs, maxrank);
+            fsecs = ((float) (maxsec) *1000000.0 + (float) maxusec) / 1000.0;
+            fprintf(stderr, "Time test was completed in %8.2f millisecs\nSlowest rank: %d\n", fsecs,
+                    maxrank);
         } else {
             fprintf(stderr, "Time test was completed in %3lu:%02lu min:sec\nSlowest rank: %d\n",
                     minutes, seconds, maxrank);
diff --git a/test/simple/ziatest.c b/test/simple/ziatest.c
index 63c6cb722fa..a0c94ef6987 100644
--- a/test/simple/ziatest.c
+++ b/test/simple/ziatest.c
@@ -11,17 +11,17 @@
  * $HEADER$
  *
  */
-#include <stdio.h>
 #include <stdbool.h>
-#include <sys/types.h>
-#include <unistd.h>
+#include <stdio.h>
 #include <stdlib.h>
-#include <time.h>
 #include <sys/time.h>
+#include <sys/types.h>
+#include <time.h>
+#include <unistd.h>
 
 #include <mpi.h>
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int nppn;
     struct timeval tv;
@@ -41,8 +41,8 @@ int main(int argc, char* argv[])
     gettimeofday(&tv, NULL);
 
     /* form the command */
-    opal_asprintf(&cmd, "mpirun -npernode %d ./ziaprobe %ld %d",
-             nppn, (long) tv.tv_sec, tv.tv_usec);
+    opal_asprintf(&cmd, "mpirun -npernode %d ./ziaprobe %ld %d", nppn, (long) tv.tv_sec,
+                  tv.tv_usec);
 
     /* execute it */
     system(cmd);
diff --git a/test/spc/Makefile.am b/test/spc/Makefile.am
index 3d060b9d418..4652e3c4440 100644
--- a/test/spc/Makefile.am
+++ b/test/spc/Makefile.am
@@ -17,8 +17,8 @@ if PROJECT_OMPI
     spc_test_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS)
     spc_test_LDADD = \
         $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 endif # PROJECT_OMPI
 
-distclean:
+distclean-local:
 	rm -rf *.dSYM .deps .libs *.la *.lo spc_test prof *.log *.o *.trs Makefile
diff --git a/test/spc/spc_test.c b/test/spc/spc_test.c
index 290b58eedae..4f1ad84523b 100644
--- a/test/spc/spc_test.c
+++ b/test/spc/spc_test.c
@@ -20,16 +20,16 @@ static void message_exchange(int num_messages, int message_size)
 {
     int i, rank;
     /* Use calloc to initialize data to 0's */
-    char *data = (char*)calloc(message_size, sizeof(char));
+    char *data = (char *) calloc(message_size, sizeof(char));
     MPI_Status status;
 
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
 
-    if(rank == 0) {
-        for(i = 0; i < num_messages; i++)
+    if (rank == 0) {
+        for (i = 0; i < num_messages; i++)
             MPI_Send(data, message_size, MPI_BYTE, 1, 123, MPI_COMM_WORLD);
-    } else if(rank == 1) {
-        for(i = 0; i < num_messages; i++)
+    } else if (rank == 1) {
+        for (i = 0; i < num_messages; i++)
             MPI_Recv(data, message_size, MPI_BYTE, 0, 123, MPI_COMM_WORLD, &status);
     }
 
@@ -38,52 +38,53 @@ static void message_exchange(int num_messages, int message_size)
 
 int main(int argc, char **argv)
 {
-    int i, rank, size, provided, num, name_len, desc_len, verbosity, bind, var_class, readonly, continuous, atomic, count, index, MPI_result;
+    int i, rank, size, provided, num, name_len, desc_len, verbosity, bind, var_class, readonly,
+        continuous, atomic, count, index, MPI_result;
     MPI_Datatype datatype;
     MPI_T_enum enumtype;
     char name[256], description[256];
 
     /* Counter names to be read by ranks 0 and 1 */
-    char *counter_names[] = { "runtime_spc_OMPI_SPC_BYTES_SENT_USER",
-                              "runtime_spc_OMPI_SPC_BYTES_RECEIVED_USER" };
+    char *counter_names[] = {"runtime_spc_OMPI_SPC_BYTES_SENT_USER",
+                             "runtime_spc_OMPI_SPC_BYTES_RECEIVED_USER"};
 
     MPI_Init(NULL, NULL);
     MPI_result = MPI_T_init_thread(MPI_THREAD_SINGLE, &provided);
-    if(MPI_result != MPI_SUCCESS) {
+    if (MPI_result != MPI_SUCCESS) {
         fprintf(stderr, "Failed to initialize MPI_T thread.\n");
         MPI_Abort(MPI_COMM_WORLD, MPI_result);
     }
 
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &size);
-    if(size != 2) {
+    if (size != 2) {
         fprintf(stderr, "ERROR: This test should be run with two MPI processes.\n");
         MPI_Abort(MPI_COMM_WORLD, -1);
     }
 
-    /* Determine the MPI_T pvar indices for the OMPI_BYTES_SENT/RECIEVED_USER SPCs */
+    /* Determine the MPI_T pvar indices for the OMPI_BYTES_SENT/RECEIVED_USER SPCs */
     index = -1;
     MPI_result = MPI_T_pvar_get_num(&num);
-    if(MPI_result != MPI_SUCCESS) {
+    if (MPI_result != MPI_SUCCESS) {
         fprintf(stderr, "Failed to get the number of pvars.\n");
         MPI_Abort(MPI_COMM_WORLD, MPI_result);
     }
 
-    for(i = 0; i < num; i++) {
+    for (i = 0; i < num; i++) {
         name_len = desc_len = 256;
-        MPI_result = MPI_T_pvar_get_info(i, name, &name_len, &verbosity,
-                                         &var_class, &datatype, &enumtype, description, &desc_len, &bind,
-                                         &readonly, &continuous, &atomic);
+        MPI_result = MPI_T_pvar_get_info(i, name, &name_len, &verbosity, &var_class, &datatype,
+                                         &enumtype, description, &desc_len, &bind, &readonly,
+                                         &continuous, &atomic);
         if (MPI_result == MPI_T_ERR_INVALID) {
             // skip invalidated MPI_T pvars
             continue;
         }
-        if(MPI_result != MPI_SUCCESS || MPI_result == MPI_T_ERR_PVAR_NO_STARTSTOP) {
+        if (MPI_result != MPI_SUCCESS || MPI_result == MPI_T_ERR_PVAR_NO_STARTSTOP) {
             fprintf(stderr, "Failed to get pvar info.\n");
             MPI_Abort(MPI_COMM_WORLD, MPI_result);
         }
 
-        if(strcmp(name, counter_names[rank]) == 0) {
+        if (strcmp(name, counter_names[rank]) == 0) {
             index = i;
             printf("[%d] %s -> %s\n", rank, name, description);
             break;
@@ -91,7 +92,7 @@ int main(int argc, char **argv)
     }
 
     /* Make sure we found the counters */
-    if(index == -1) {
+    if (index == -1) {
         fprintf(stderr, "ERROR: Couldn't find the appropriate SPC counter in the MPI_T pvars.\n");
         MPI_Abort(MPI_COMM_WORLD, -1);
     }
@@ -102,45 +103,47 @@ int main(int argc, char **argv)
     MPI_T_pvar_handle handle;
     /* Create the MPI_T sessions/handles for the counters and start the counters */
     MPI_result = MPI_T_pvar_session_create(&session);
-    if(MPI_result != MPI_SUCCESS) {
+    if (MPI_result != MPI_SUCCESS) {
         fprintf(stderr, "Failed to create MPI_T pvar session.\n");
         MPI_Abort(MPI_COMM_WORLD, MPI_result);
     }
 
     MPI_result = MPI_T_pvar_handle_alloc(session, index, NULL, &handle, &count);
-    if(MPI_result != MPI_SUCCESS) {
+    if (MPI_result != MPI_SUCCESS) {
         fprintf(stderr, "Failed to allocate the pvar handle.\n");
         MPI_Abort(MPI_COMM_WORLD, MPI_result);
     }
 
     MPI_result = MPI_T_pvar_start(session, handle);
-    if(MPI_result != MPI_SUCCESS) {
-        if(MPI_result != MPI_T_ERR_PVAR_NO_STARTSTOP) {
+    if (MPI_result != MPI_SUCCESS) {
+        if (MPI_result != MPI_T_ERR_PVAR_NO_STARTSTOP) {
             fprintf(stderr, "Failed to start the pvar session.\n");
             MPI_Abort(MPI_COMM_WORLD, MPI_result);
         }
     }
 
     int message_size = 1, expected_bytes = 0;
-    while(message_size <= MAX_SIZE) {
+    while (message_size <= MAX_SIZE) {
         expected_bytes += message_size;
         message_exchange(1, message_size);
         message_size *= 10;
     }
 
     MPI_result = MPI_T_pvar_read(session, handle, &value);
-    if(MPI_result != MPI_SUCCESS) {
+    if (MPI_result != MPI_SUCCESS) {
         fprintf(stderr, "Failed to read the pvar.\n");
         MPI_Abort(MPI_COMM_WORLD, MPI_result);
     }
 
     /* Print the counter values in order by rank */
-    for(i = 0; i < 2; i++) {
-        if(i == rank) {
+    for (i = 0; i < 2; i++) {
+        if (i == rank) {
             printf("[%d] Value Read: %lld\n", rank, value);
             fflush(stdout);
-            if(value != expected_bytes){
-                fprintf(stderr, "The counter value is inaccurate!  It is '%lld'.  It should be '%d'\n", value, expected_bytes);
+            if (value != expected_bytes) {
+                fprintf(stderr,
+                        "The counter value is inaccurate!  It is '%lld'.  It should be '%d'\n",
+                        value, expected_bytes);
                 MPI_Abort(MPI_COMM_WORLD, MPI_ERR_OTHER);
             }
         }
@@ -148,27 +151,27 @@ int main(int argc, char **argv)
     }
     /* Stop the MPI_T session, free the handle, and then free the session */
     MPI_result = MPI_T_pvar_stop(session, handle);
-    if(MPI_result != MPI_SUCCESS) {
-        if(MPI_result != MPI_T_ERR_PVAR_NO_STARTSTOP) {
+    if (MPI_result != MPI_SUCCESS) {
+        if (MPI_result != MPI_T_ERR_PVAR_NO_STARTSTOP) {
             fprintf(stderr, "Failed to stop the pvar session.\n");
             MPI_Abort(MPI_COMM_WORLD, MPI_result);
         }
     }
 
     MPI_result = MPI_T_pvar_handle_free(session, &handle);
-    if(MPI_result != MPI_SUCCESS) {
+    if (MPI_result != MPI_SUCCESS) {
         fprintf(stderr, "Failed to free the pvar handle.\n");
         MPI_Abort(MPI_COMM_WORLD, MPI_result);
     }
 
     MPI_result = MPI_T_pvar_session_free(&session);
-    if(MPI_result != MPI_SUCCESS) {
+    if (MPI_result != MPI_SUCCESS) {
         fprintf(stderr, "Failed to free the pvar session.\n");
         MPI_Abort(MPI_COMM_WORLD, MPI_result);
     }
 
     MPI_result = MPI_T_finalize();
-    if(MPI_result != MPI_SUCCESS) {
+    if (MPI_result != MPI_SUCCESS) {
         fprintf(stderr, "Failed to finalize MPI_T.\n");
         MPI_Abort(MPI_COMM_WORLD, MPI_result);
     }
diff --git a/test/support/Makefile.am b/test/support/Makefile.am
index 85f3d3f8a1e..e8eadefc969 100644
--- a/test/support/Makefile.am
+++ b/test/support/Makefile.am
@@ -32,5 +32,5 @@ libsupport_a_SOURCES = \
         support.c \
         support.h
 
-distclean:
+distclean-local:
 	rm -rf *.dSYM .deps *.log *.o *.trs $(check_LIBRARIES) Makefile
diff --git a/test/support/support.c b/test/support/support.c
index e7411e06fdb..cbf9fca6d94 100644
--- a/test/support/support.c
+++ b/test/support/support.c
@@ -10,6 +10,8 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2024      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -18,13 +20,35 @@
  */
 
 #include "opal_config.h"
-#include <stdlib.h>
-#include <string.h>
 #include <assert.h>
 #include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
 
 #include "support.h"
 
+#define test_verify_number(T, specifier)                                           \
+    int test_verify_##T(T expected_result, T test_result)                          \
+    {                                                                              \
+        int return_value;                                                          \
+        return_value = 1;                                                          \
+        if (expected_result != test_result) {                                      \
+            test_failure("Comparison failure");                                    \
+            fprintf(stderr, " Expected result: " specifier "\n", expected_result); \
+            fprintf(stderr, " Test result: " specifier "\n", test_result);         \
+            fflush(stderr);                                                        \
+            return_value = 0;                                                      \
+        } else {                                                                   \
+            test_success();                                                        \
+        }                                                                          \
+        return return_value;                                                       \
+    }
+
+test_verify_number(int, "%d")
+test_verify_number(int64_t, "%ld")
+test_verify_number(size_t, "%lu")
+test_verify_number(double, "%lf")
+
 /**
  * A testing support library to provide uniform reporting output
  */
@@ -52,17 +76,14 @@ void test_init(const char *a)
     opal_n_failures = 0;
 
     return;
-
 }
 
-
 void test_success(void)
 {
     opal_n_tests++;
     opal_n_success++;
 }
 
-
 void test_failure(const char *a)
 {
     opal_n_tests++;
@@ -74,7 +95,6 @@ void test_failure(const char *a)
     fflush(stderr);
 }
 
-
 int test_verify_str(const char *expected_result, const char *test_result)
 {
     size_t len_expect, len_result;
@@ -84,8 +104,7 @@ int test_verify_str(const char *expected_result, const char *test_result)
     len_expect = expected_result ? strlen(expected_result) : 0;
     len_result = test_result ? strlen(test_result) : 0;
 
-    if ((!(len_expect == len_result)) ||
-        (0 != strcmp(expected_result, test_result))) {
+    if ((!(len_expect == len_result)) || (0 != strcmp(expected_result, test_result))) {
         test_failure("Comparison failure");
         fprintf(stderr, " Expected result: %s\n", expected_result);
         fprintf(stderr, " Test result: %s\n", test_result);
@@ -98,26 +117,6 @@ int test_verify_str(const char *expected_result, const char *test_result)
     return return_value;
 }
 
-
-int test_verify_int(int expected_result, int test_result)
-{
-    int return_value;
-
-    return_value = 1;
-    if (expected_result != test_result) {
-        test_failure("Comparison failure");
-        fprintf(stderr, " Expected result: %d\n", expected_result);
-        fprintf(stderr, " Test result: %d\n", test_result);
-        fflush(stderr);
-        return_value = 0;
-    } else {
-        test_success();
-    }
-
-    return return_value;
-}
-
-
 int test_finalize(void)
 {
     int return_value;
@@ -125,13 +124,12 @@ int test_finalize(void)
     return_value = 0;
 
     if (opal_n_tests == opal_n_success) {
-        fprintf(stderr, "SUPPORT: OMPI Test Passed: %s: (%d tests)\n",
-                opal_description, opal_n_tests);
+        fprintf(stderr, "SUPPORT: OMPI Test Passed: %s: (%d tests)\n", opal_description,
+                opal_n_tests);
         fflush(stderr);
     } else {
-        fprintf(stderr,
-                "SUPPORT: OMPI Test failed: %s (%d of %d failed)\n",
-                opal_description, opal_n_failures, opal_n_tests);
+        fprintf(stderr, "SUPPORT: OMPI Test failed: %s (%d of %d failed)\n", opal_description,
+                opal_n_failures, opal_n_tests);
         fflush(stderr);
         return_value = 1;
     }
@@ -142,14 +140,12 @@ int test_finalize(void)
     return return_value;
 }
 
-
 /* note this is for additional output that does NOT go to STDERR but STDOUT */
-void test_comment (const char* userstr)
+void test_comment(const char *userstr)
 {
-	fprintf(stdout, "%s:%s\n", opal_description, userstr);
+    fprintf(stdout, "%s:%s\n", opal_description, userstr);
 }
 
-
 void test_fail_stop(const char *msg, int status)
 {
     test_failure(msg);
diff --git a/test/support/support.h b/test/support/support.h
index c5995aa11dd..a7249ff535b 100644
--- a/test/support/support.h
+++ b/test/support/support.h
@@ -9,6 +9,8 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2024      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -21,37 +23,39 @@
 
 #include <stdio.h>
 #include <stdlib.h>
+#include <stdint.h>
 
-#define TEST_AND_REPORT(res, exp_res, str)  \
-    if( res == exp_res ) test_success(); \
-    else test_failure(str);
+#define TEST_AND_REPORT(res, exp_res, str) \
+    if (res == exp_res)                    \
+        test_success();                    \
+    else                                   \
+        test_failure(str);
 
 void test_init(const char *a);
 void test_success(void);
 void test_failure(const char *a);
 int test_verify_str(const char *expected_result, const char *test_result);
 int test_verify_int(int expected_result, int test_result);
+int test_verify_int64_t(int64_t expected_result, int64_t test_result);
+int test_verify_size_t(size_t expected_result, size_t test_result);
+int test_verify_double(double expected_result, double test_result);
 int test_finalize(void);
-void test_comment (const char* userstr);
+void test_comment(const char *userstr);
 void test_fail_stop(const char *msg, int status);
 
-
-
 /*
  * test_verify: Non-fatal assertion macro.
  */
 
-#define test_verify(MESSAGE, EXPR)                               \
-    do {                                                         \
-        if (!(EXPR)) {                                           \
-            char s[256];                                         \
-            sprintf(s, "%s:%d: %s: %s\n",                        \
-                    __FILE__, __LINE__, MESSAGE, # EXPR);        \
-            test_failure(s);                                     \
-        } else {                                                 \
-            test_success();                                      \
-        }                                                        \
+#define test_verify(MESSAGE, EXPR)                                             \
+    do {                                                                       \
+        if (!(EXPR)) {                                                         \
+            char s[256];                                                       \
+            sprintf(s, "%s:%d: %s: %s\n", __FILE__, __LINE__, MESSAGE, #EXPR); \
+            test_failure(s);                                                   \
+        } else {                                                               \
+            test_success();                                                    \
+        }                                                                      \
     } while (0)
 
 #endif /* OMPI_SUPPORT_H */
-
diff --git a/test/threads/Makefile.am b/test/threads/Makefile.am
index 024b4f3ae0c..b8d1923006d 100644
--- a/test/threads/Makefile.am
+++ b/test/threads/Makefile.am
@@ -10,7 +10,8 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2016 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2016      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2021      IBM Corporation.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -24,23 +25,28 @@ AM_LDFLAGS = -lpthread
 
 check_PROGRAMS = \
 	opal_thread \
-	opal_condition
+	opal_condition \
+	opal_atomic_thread_bench
 
-# JMS possibly to be re-added when #1232 is fixed
-#TESTS = $(check_PROGRAMS)
-TESTS =
+TESTS = $(check_PROGRAMS)
 
 opal_thread_SOURCES = opal_thread.c
 opal_thread_LDADD = \
         $(top_builddir)/test/support/libsupport.a \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 opal_thread_DEPENDENCIES = $(opal_thread_LDADD)
 
 opal_condition_SOURCES = opal_condition.c
 opal_condition_LDADD = \
         $(top_builddir)/test/support/libsupport.a \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
 opal_condition_DEPENDENCIES = $(opal_condition_LDADD)
 
-distclean:
+opal_atomic_thread_bench_SOURCES = opal_atomic_thread_bench.c
+opal_atomic_thread_bench_LDADD = \
+        $(top_builddir)/test/support/libsupport.a \
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
+opal_atomic_thread_bench_DEPENDENCIES = $(opal_atomic_thread_bench_LDADD)
+
+distclean-local:
 	rm -rf *.dSYM .deps .libs *.log *.o *.trs $(check_PROGRAMS) Makefile
diff --git a/test/threads/opal_atomic_thread_bench.c b/test/threads/opal_atomic_thread_bench.c
new file mode 100644
index 00000000000..279fb0ee8b6
--- /dev/null
+++ b/test/threads/opal_atomic_thread_bench.c
@@ -0,0 +1,299 @@
+/*
+ * Copyright (c) 2021 IBM Corporation.  All rights reserved.
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "opal_config.h"
+
+#include <stdio.h>
+#include <time.h>
+
+#include "opal/constants.h"
+#include "opal/mca/threads/condition.h"
+#include "opal/mca/threads/threads.h"
+#include "opal/runtime/opal.h"
+#include "opal/sys/atomic.h"
+#include "support.h"
+
+#define OPAL_TEST_THREAD_COUNT 8
+#define ITERATIONS             1000000
+#define ITEM_COUNT             100
+
+#ifdef __APPLE__
+/* pthread barrier implementation copied from stack overflow.
+ * Source:
+ * https://stackoverflow.com/questions/3640853/performance-test-sem-t-v-s-dispatch-semaphore-t-and-pthread-once-t-v-s-dispatch
+ */
+typedef int pthread_barrierattr_t;
+struct pthread_barrier_t {
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+    int count;
+    int tripCount;
+};
+
+typedef struct pthread_barrier_t pthread_barrier_t;
+
+int pthread_barrier_init(pthread_barrier_t *barrier, const pthread_barrierattr_t *attr,
+                         unsigned int count)
+{
+    if (count == 0) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (pthread_mutex_init(&barrier->mutex, 0) < 0) {
+        return -1;
+    }
+    if (pthread_cond_init(&barrier->cond, 0) < 0) {
+        pthread_mutex_destroy(&barrier->mutex);
+        return -1;
+    }
+    barrier->tripCount = count;
+    barrier->count = 0;
+
+    return 0;
+}
+
+int pthread_barrier_destroy(pthread_barrier_t *barrier)
+{
+    pthread_cond_destroy(&barrier->cond);
+    pthread_mutex_destroy(&barrier->mutex);
+    return 0;
+}
+
+int pthread_barrier_wait(pthread_barrier_t *barrier)
+{
+    pthread_mutex_lock(&barrier->mutex);
+    ++(barrier->count);
+    if (barrier->count >= barrier->tripCount) {
+        barrier->count = 0;
+        pthread_cond_broadcast(&barrier->cond);
+        pthread_mutex_unlock(&barrier->mutex);
+        return 1;
+    }
+
+    pthread_cond_wait(&barrier->cond, &(barrier->mutex));
+    pthread_mutex_unlock(&barrier->mutex);
+    return 0;
+}
+#endif
+
+static opal_atomic_int64_t var_64 = 0;
+static opal_atomic_int32_t var_32 = 0;
+static pthread_barrier_t barrier;
+
+#if !defined(timersub)
+#    define timersub(a, b, r)                           \
+        do {                                            \
+            (r)->tv_sec = (a)->tv_sec - (b)->tv_sec;    \
+            if ((a)->tv_usec < (b)->tv_usec) {          \
+                (r)->tv_sec--;                          \
+                (a)->tv_usec += 1000000;                \
+            }                                           \
+            (r)->tv_usec = (a)->tv_usec - (b)->tv_usec; \
+        } while (0)
+#endif
+
+#if !defined(OPAL_TEST_DONE)
+#    define OPAL_TEST_DONE(func, val)                                               \
+        {                                                                           \
+            gettimeofday(&stop, NULL);                                              \
+            timersub(&stop, &start, &total);                                        \
+            timing = ((double) total.tv_sec + (double) total.tv_usec * 1e-6)        \
+                     / (double) ITERATIONS;                                         \
+            printf("%s() thread finished. Time: %d s %d us %d nsec/per\n", func,    \
+                   (int) total.tv_sec, (int) total.tv_usec, (int) (timing / 1e-9)); \
+            memset(&stop, 0, sizeof(struct timeval));                               \
+            memset(&start, 0, sizeof(struct timeval));                              \
+            memset(&total, 0, sizeof(struct timeval));                              \
+            /* printf("%ld\n", val); */                                             \
+            fflush(stdout);                                                         \
+            pthread_barrier_wait(&barrier);                                         \
+        }
+#endif
+
+#if !defined(OPAL_RESET_VAR)
+#    define OPAL_RESET_VAR(var)             \
+        {                                   \
+            var = 0;                        \
+            pthread_barrier_wait(&barrier); \
+        }
+#endif
+
+static void *thread_test(void *arg)
+{
+    struct timeval start, stop, total;
+    double timing;
+
+    gettimeofday(&start, NULL);
+    for (int64_t i = 0; i < ITERATIONS; ++i) {
+        opal_atomic_compare_exchange_strong_64(&var_64, &i, i + 1);
+    }
+    OPAL_TEST_DONE("opal_atomic_compare_exchange_strong_64", var_64);
+
+    OPAL_RESET_VAR(var_64);
+
+    gettimeofday(&start, NULL);
+    for (int64_t i = 0; i < ITERATIONS; ++i) {
+        opal_atomic_compare_exchange_strong_rel_64(&var_64, &i, i + 1);
+    }
+    OPAL_TEST_DONE("opal_atomic_compare_exchange_strong_rel_64", var_64);
+
+    OPAL_RESET_VAR(var_64);
+
+    gettimeofday(&start, NULL);
+    for (int64_t i = 0; i < ITERATIONS; ++i) {
+        opal_atomic_compare_exchange_strong_acq_64(&var_64, &i, i + 1);
+    }
+    OPAL_TEST_DONE("opal_atomic_compare_exchange_strong_acq_64", var_64);
+
+    OPAL_RESET_VAR(var_64);
+
+    gettimeofday(&start, NULL);
+    for (int64_t i = 0; i < ITERATIONS; ++i) {
+        opal_atomic_fetch_add_64(&var_64, 1);
+    }
+    OPAL_TEST_DONE("opal_atomic_fetch_add_64", var_64);
+
+    OPAL_RESET_VAR(var_64);
+
+    gettimeofday(&start, NULL);
+    for (int64_t i = 0; i < ITERATIONS; ++i) {
+        opal_atomic_fetch_sub_64(&var_64, 1);
+    }
+    OPAL_TEST_DONE("opal_atomic_fetch_sub_64", var_64);
+
+    OPAL_RESET_VAR(var_64);
+
+    gettimeofday(&start, NULL);
+    for (int64_t i = 0; i < ITERATIONS; ++i) {
+        opal_atomic_fetch_xor_64(&var_64, i);
+    }
+    OPAL_TEST_DONE("opal_atomic_fetch_xor_64", var_64);
+
+    OPAL_RESET_VAR(var_64);
+
+    gettimeofday(&start, NULL);
+    for (int64_t i = 0; i < ITERATIONS; ++i) {
+        opal_atomic_swap_64(&var_64, i);
+    }
+    OPAL_TEST_DONE("opal_atomic_swap_64", var_64);
+
+    OPAL_RESET_VAR(var_64);
+
+#if OPAL_HAVE_ATOMIC_LLSC_64
+    gettimeofday(&start, NULL);
+    for (int64_t i = 0; i < ITERATIONS; ++i) {
+        int ret;
+        opal_atomic_sc_64(&var_64, i, ret);
+    }
+    OPAL_TEST_DONE("opal_atomic_sc_64", var_64);
+
+    OPAL_RESET_VAR(var_64);
+
+    gettimeofday(&start, NULL);
+    for (int64_t i = 0; i < ITERATIONS; ++i) {
+        int ret;
+        opal_atomic_sc_64(&var_64, i, ret);
+    }
+    OPAL_TEST_DONE("opal_atomic_ll_64", var_64);
+
+    OPAL_RESET_VAR(var_64);
+#endif
+
+    gettimeofday(&start, NULL);
+    for (int32_t i = 0; i < ITERATIONS; ++i) {
+        opal_atomic_compare_exchange_strong_32(&var_32, &i, i + 1);
+    }
+    OPAL_TEST_DONE("opal_atomic_compare_exchange_strong_32", var_32);
+
+    OPAL_RESET_VAR(var_32);
+
+    gettimeofday(&start, NULL);
+    for (int32_t i = 0; i < ITERATIONS; ++i) {
+        opal_atomic_compare_exchange_strong_rel_32(&var_32, &i, i + 1);
+    }
+    OPAL_TEST_DONE("opal_atomic_compare_exchange_strong_rel_32", var_32);
+
+    OPAL_RESET_VAR(var_32);
+
+    gettimeofday(&start, NULL);
+    for (int32_t i = 0; i < ITERATIONS; ++i) {
+        opal_atomic_compare_exchange_strong_acq_32(&var_32, &i, i + 1);
+    }
+    OPAL_TEST_DONE("opal_atomic_compare_exchange_strong_acq_32", var_32);
+
+    OPAL_RESET_VAR(var_32);
+
+    gettimeofday(&start, NULL);
+    for (int32_t i = 0; i < ITERATIONS; ++i) {
+        opal_atomic_fetch_add_32(&var_32, 1);
+    }
+    OPAL_TEST_DONE("opal_atomic_fetch_add_32", var_32);
+
+    OPAL_RESET_VAR(var_32);
+
+    gettimeofday(&start, NULL);
+    for (int32_t i = 0; i < ITERATIONS; ++i) {
+        opal_atomic_fetch_sub_32(&var_32, 1);
+    }
+    OPAL_TEST_DONE("opal_atomic_fetch_sub_32", var_32);
+
+    OPAL_RESET_VAR(var_32);
+
+    gettimeofday(&start, NULL);
+    for (int32_t i = 0; i < ITERATIONS; ++i) {
+        opal_atomic_fetch_xor_32(&var_32, i);
+    }
+    OPAL_TEST_DONE("opal_atomic_fetch_xor_32", var_32);
+
+    OPAL_RESET_VAR(var_32);
+
+    gettimeofday(&start, NULL);
+    for (int32_t i = 0; i < ITERATIONS; ++i) {
+        opal_atomic_swap_32(&var_32, i);
+    }
+    OPAL_TEST_DONE("opal_atomic_swap_32", var_32);
+
+    OPAL_RESET_VAR(var_32);
+
+#if OPAL_HAVE_ATOMIC_LLSC_32
+    gettimeofday(&start, NULL);
+    for (int32_t i = 0; i < ITERATIONS; ++i) {
+        int ret;
+        opal_atomic_sc_32(&var_32, i, ret);
+    }
+    OPAL_TEST_DONE("opal_atomic_sc_32", var_32);
+
+    OPAL_RESET_VAR(var_32);
+
+    gettimeofday(&start, NULL);
+    for (int32_t i = 0; i < ITERATIONS; ++i) {
+        int ret;
+        opal_atomic_sc_32(&var_32, i, ret);
+    }
+    OPAL_TEST_DONE("opal_atomic_ll_32", var_32);
+
+    OPAL_RESET_VAR(var_32);
+#endif
+
+    return NULL;
+}
+
+int main(void)
+{
+
+    pthread_barrier_init(&barrier, NULL, OPAL_TEST_THREAD_COUNT);
+
+    pthread_t ts[OPAL_TEST_THREAD_COUNT];
+    for (int i = 0; i < OPAL_TEST_THREAD_COUNT; i++) {
+        pthread_create(&ts[i], NULL, &thread_test, NULL);
+    }
+
+    for (int i = 0; i < OPAL_TEST_THREAD_COUNT; i++) {
+        pthread_join(ts[i], NULL);
+    }
+    return 0;
+}
diff --git a/test/threads/opal_condition.c b/test/threads/opal_condition.c
index cacb3acc5f1..78970632f10 100644
--- a/test/threads/opal_condition.c
+++ b/test/threads/opal_condition.c
@@ -22,12 +22,12 @@
 #include <stdio.h>
 #include <time.h>
 
-#include "support.h"
-#include "opal/runtime/opal.h"
 #include "opal/constants.h"
-#include "opal/mca/threads/threads.h"
 #include "opal/mca/threads/condition.h"
+#include "opal/mca/threads/threads.h"
+#include "opal/runtime/opal.h"
 #include "opal/sys/atomic.h"
+#include "support.h"
 
 static opal_mutex_t mutex;
 static opal_condition_t thr1_cond;
@@ -36,50 +36,55 @@ static opal_condition_t thr2_cond;
 static volatile int thr1_count = 0;
 static volatile int thr2_count = 0;
 
-
 #define TEST_COUNT 100000
 
-
-static void* thr1_run(opal_object_t* obj)
+static void *thr1_run(opal_object_t *obj)
 {
-    int i;
     clock_t c1, c2;
-    opal_mutex_lock(&mutex);
     c1 = clock();
-    for(i=0; i<TEST_COUNT; i++) {
-        opal_condition_wait(&thr1_cond, &mutex);
-        opal_condition_signal(&thr2_cond);
+    opal_mutex_lock(&mutex);
+    while (TEST_COUNT != thr1_count) {
         thr1_count++;
+        opal_condition_signal(&thr2_cond);
+        opal_condition_wait(&thr1_cond, &mutex);
     }
-    c2 = clock();
+
+    // Whoever gets here first needs to alert the other
+    // thread for their last iteration.
+    opal_condition_signal(&thr2_cond);
     opal_mutex_unlock(&mutex);
-    fprintf(stderr, "thr1: time per iteration: %ld usec\n", (long)((c2 - c1) / TEST_COUNT));
+    c2 = clock();
+    fprintf(stderr, "thr1: time per iteration: %ld uses\n", (long)((c2 - c1) / TEST_COUNT));
     return NULL;
 }
 
-static void* thr2_run(opal_object_t* obj)
+static void *thr2_run(opal_object_t *obj)
 {
-    int i;
     clock_t c1, c2;
-    opal_mutex_lock(&mutex);
     c1 = clock();
-    for(i=0; i<TEST_COUNT; i++) {
+    opal_mutex_lock(&mutex);
+    while(TEST_COUNT != thr2_count) {
+        thr2_count++;
         opal_condition_signal(&thr1_cond);
         opal_condition_wait(&thr2_cond, &mutex);
-        thr2_count++;
     }
-    c2 = clock();
+
+    // Whoever gets here first needs to alert the other
+    // thread for the last iteration.
+    opal_condition_signal(&thr1_cond);
     opal_mutex_unlock(&mutex);
+
+    c2 = clock();
     fprintf(stderr, "thr2: time per iteration: %ld usec\n", (long)((c2 - c1) / TEST_COUNT));
     return NULL;
 }
 
 
-int main(int argc, char** argv)
+int main(int argc, char **argv)
 {
     int rc;
-    opal_thread_t* thr1;
-    opal_thread_t* thr2;
+    opal_thread_t *thr1;
+    opal_thread_t *thr2;
 
     test_init("opal_condition_t");
 
diff --git a/test/threads/opal_thread.c b/test/threads/opal_thread.c
index f7f66130d8d..c58d8e7876e 100644
--- a/test/threads/opal_thread.c
+++ b/test/threads/opal_thread.c
@@ -13,30 +13,28 @@
 
 #include <stdio.h>
 
-#include "support.h"
 #include "opal/constants.h"
 #include "opal/mca/threads/threads.h"
 #include "opal/sys/atomic.h"
-
+#include "support.h"
 
 /* Only have the body of this test if we have thread support */
 
 static opal_atomic_int_t count = 0;
 
-
-static void* thr1_run(opal_object_t* obj)
+static void *thr1_run(opal_object_t *obj)
 {
-    opal_atomic_add (&count, 1);
+    opal_atomic_add(&count, 1);
     return NULL;
 }
 
-static void* thr2_run(opal_object_t* obj)
+static void *thr2_run(opal_object_t *obj)
 {
-    opal_atomic_add (&count, 2);
+    opal_atomic_add(&count, 2);
     return NULL;
 }
 
-int main(int argc, char** argv)
+int main(int argc, char **argv)
 {
     int rc;
     opal_thread_t thr1;
diff --git a/test/util/Makefile.am b/test/util/Makefile.am
index 1b3757a27ee..99c1a4b224a 100644
--- a/test/util/Makefile.am
+++ b/test/util/Makefile.am
@@ -15,6 +15,8 @@
 # Copyright (c) 2018      Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
 # Copyright (c) 2018      Intel, Inc.  All rights reserved.
+# Copyright (c) 2024      Amazon.com, Inc. or its affiliates.
+#                         All Rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -37,16 +39,18 @@ AM_CPPFLAGS = -I$(top_srcdir)/test/support
 
 
 check_PROGRAMS = \
+	bipartite_graph \
 	opal_bit_ops \
 	opal_path_nfs \
-	bipartite_graph
+        opal_json \
+        opal_sha256
 
 TESTS = \
 	$(check_PROGRAMS)
 
 #ompi_numtostr_SOURCES = ompi_numtostr.c
 #ompi_numtostr_LDADD = \
-#        $(top_builddir)/opal/libopen-pal.la \
+#        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
 #        $(top_builddir)/test/support/libsupport.a
 #ompi_numtostr_DEPENDENCIES = $(ompi_numtostr_LDADD)
 
@@ -57,43 +61,43 @@ TESTS = \
 
 #opal_if_SOURCES = opal_if.c
 #opal_if_LDADD = \
-#        $(top_builddir)/opal/libopen-pal.la \
+#        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
 #        $(top_builddir)/test/support/libsupport.a
 #opal_if_DEPENDENCIES = $(opal_if_LDADD)
 
 #opal_argv_SOURCES = opal_argv.c
 #opal_argv_LDADD = \
-#        $(top_builddir)/opal/libopen-pal.la \
+#        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
 #        $(top_builddir)/test/support/libsupport.a
 #opal_argv_DEPENDENCIES = $(opal_argv_LDADD)
 
 #opal_basename_SOURCES = opal_basename.c
 #opal_basename_LDADD = \
-#        $(top_builddir)/opal/libopen-pal.la \
+#        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
 #        $(top_builddir)/test/support/libsupport.a
 #opal_basename_DEPENDENCIES = $(opal_basename_LDADD)
 
 opal_bit_ops_SOURCES = opal_bit_ops.c
 opal_bit_ops_LDADD = \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
         $(top_builddir)/test/support/libsupport.a
 opal_bit_ops_DEPENDENCIES = $(opal_path_nfs_LDADD)
 
 opal_path_nfs_SOURCES = opal_path_nfs.c
 opal_path_nfs_LDADD = \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
         $(top_builddir)/test/support/libsupport.a
 opal_path_nfs_DEPENDENCIES = $(opal_path_nfs_LDADD)
 
 #opal_os_path_SOURCES = opal_os_path.c
 #opal_os_path_LDADD = \
-#        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
+#        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
 #        $(top_builddir)/test/support/libsupport.a
 #opal_os_path_DEPENDENCIES = $(opal_os_path_LDADD)
 
 #opal_timer_SOURCES = opal_timer.c
 #opal_timer_LDADD = \
-#        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
+#        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
 #        $(top_builddir)/test/support/libsupport.a
 #opal_timer_DEPENDENCIES = $(opal_timer_LDADD)
 
@@ -106,7 +110,7 @@ opal_path_nfs_DEPENDENCIES = $(opal_path_nfs_LDADD)
 
 #opal_os_create_dirpath_SOURCES = opal_os_create_dirpath.c
 #opal_os_create_dirpath_LDADD = \
-#        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
+#        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
 #        $(top_builddir)/test/support/libsupport.a
 #opal_os_create_dirpath_DEPENDENCIES = $(opal_os_create_dirpath_LDADD)
 
@@ -124,13 +128,25 @@ opal_path_nfs_DEPENDENCIES = $(opal_path_nfs_LDADD)
 
 bipartite_graph_SOURCES = bipartite_graph.c
 bipartite_graph_LDADD = \
-        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
         $(top_builddir)/test/support/libsupport.a
 bipartite_graph_DEPENDENCIES = $(bipartite_graph_LDADD)
 
+opal_json_SOURCES = opal_json.c
+opal_json_LDADD = \
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
+        $(top_builddir)/test/support/libsupport.a
+opal_json_DEPENDENCIES = $(opal_json_LDADD)
+
+opal_sha256_SOURCES = opal_sha256.c
+opal_sha256_LDADD = \
+        $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
+        $(top_builddir)/test/support/libsupport.a
+opal_sha256_DEPENDENCIES = $(opal_sha256_LDADD)
+
 clean-local:
 	rm -f test_session_dir_out test-file opal_path_nfs.out
 
-distclean:
+distclean-local:
 	rm -rf *.dSYM .deps .libs *.out *.log *.o *.trs $(check_PROGRAMS) Makefile
 
diff --git a/test/util/bipartite_graph.c b/test/util/bipartite_graph.c
index e5dd8710ead..adf1174be64 100644
--- a/test/util/bipartite_graph.c
+++ b/test/util/bipartite_graph.c
@@ -12,62 +12,59 @@
 #include <stdlib.h>
 #include <sys/time.h>
 
-#include "opal/constants.h"
 #include "opal/class/opal_list.h"
 #include "opal/class/opal_pointer_array.h"
+#include "opal/constants.h"
 #include "opal/util/bipartite_graph.h"
 #include "opal/util/bipartite_graph_internal.h"
 
-#  define test_out(...) fprintf(stderr, __VA_ARGS__)
-#  define check(a)                                                           \
+#define test_out(...) fprintf(stderr, __VA_ARGS__)
+#define check(a)                                                             \
     do {                                                                     \
         if (!(a)) {                                                          \
             test_out("%s:%d: check failed, '%s'\n", __func__, __LINE__, #a); \
-            return 1;                                              \
+            return 1;                                                        \
         }                                                                    \
     } while (0)
-#  define check_str_eq(a,b)                                     \
-    do {                                                        \
-        const char *a_ = (a);                                   \
-        const char *b_ = (b);                                   \
-        if (0 != strcmp(a_,b_)) {                               \
-            test_out("%s:%d: check failed, \"%s\" != \"%s\"\n", \
-                     __func__, __LINE__, a_, b_);               \
-            return 1;                                 \
-        }                                                       \
+#define check_str_eq(a, b)                                                                   \
+    do {                                                                                     \
+        const char *a_ = (a);                                                                \
+        const char *b_ = (b);                                                                \
+        if (0 != strcmp(a_, b_)) {                                                           \
+            test_out("%s:%d: check failed, \"%s\" != \"%s\"\n", __func__, __LINE__, a_, b_); \
+            return 1;                                                                        \
+        }                                                                                    \
     } while (0)
-#  define check_int_eq(got, expected)                                   \
-    do {                                                                \
-        if ((got) != (expected)) {                                      \
-            test_out("%s:%d: check failed, \"%s\" != \"%s\", got %d\n", \
-                     __func__, __LINE__, #got, #expected, (got));       \
-            return 1;                                         \
-        }                                                               \
+#define check_int_eq(got, expected)                                                               \
+    do {                                                                                          \
+        if ((got) != (expected)) {                                                                \
+            test_out("%s:%d: check failed, \"%s\" != \"%s\", got %d\n", __func__, __LINE__, #got, \
+                     #expected, (got));                                                           \
+            return 1;                                                                             \
+        }                                                                                         \
     } while (0)
 /* just use check_int_eq for now, no public error code to string routine
  * exists (opal_err2str is static) */
-#  define check_err_code(got, expected)                                 \
-    check_int_eq(got, expected)
-#  define check_msg(a, msg)                                \
-    do {                                                   \
-        if (!(a)) {                                        \
-            test_out("%s:%d: check failed, \"%s\" (%s)\n", \
-                     __func__, __LINE__, #a, (msg));       \
-            return 1;                            \
-        }                                                  \
+#define check_err_code(got, expected) check_int_eq(got, expected)
+#define check_msg(a, msg)                                                                  \
+    do {                                                                                   \
+        if (!(a)) {                                                                        \
+            test_out("%s:%d: check failed, \"%s\" (%s)\n", __func__, __LINE__, #a, (msg)); \
+            return 1;                                                                      \
+        }                                                                                  \
     } while (0)
 
-#define check_graph_is_consistent(g)                                         \
-    do {                                                                     \
+#define check_graph_is_consistent(g)                                                \
+    do {                                                                            \
         check(opal_bp_graph_order(g) <= opal_pointer_array_get_size(&g->vertices)); \
         check(g->source_idx >= -1 || g->source_idx < opal_bp_graph_order(g));       \
         check(g->sink_idx >= -1 || g->sink_idx < opal_bp_graph_order(g));           \
     } while (0)
 
-#define check_has_in_out_degree(g, u, expected_indegree, expected_outdegree)   \
-    do {                                                                       \
-        check_int_eq(opal_bp_graph_indegree(g, (u)), expected_indegree);   \
-        check_int_eq(opal_bp_graph_outdegree(g, (u)), expected_outdegree); \
+#define check_has_in_out_degree(g, u, expected_indegree, expected_outdegree) \
+    do {                                                                     \
+        check_int_eq(opal_bp_graph_indegree(g, (u)), expected_indegree);     \
+        check_int_eq(opal_bp_graph_outdegree(g, (u)), expected_outdegree);   \
     } while (0)
 
 /* Check the given path for sanity and that it does not have a cycle.  Uses
@@ -81,7 +78,7 @@
             check(pred[i_] < n);                   \
         }                                          \
         i_ = (sink);                               \
-        j_ = pred[(sink)];                       \
+        j_ = pred[(sink)];                         \
         while (i_ != -1 && j_ != -1) {             \
             check_msg(i_ != j_, "CYCLE DETECTED"); \
             i_ = pred[i_];                         \
@@ -109,23 +106,19 @@ static void e_cleanup(void *e_data)
  * list returned by opal_bp_graph_solve_bipartite_assignment */
 static int cmp_int_pair(const void *a, const void *b)
 {
-    int *ia = (int *)a;
-    int *ib = (int *)b;
+    int *ia = (int *) a;
+    int *ib = (int *) b;
 
     if (ia[0] < ib[0]) {
         return -1;
-    }
-    else if (ia[0] > ib[0]) {
+    } else if (ia[0] > ib[0]) {
         return 1;
-    }
-    else { /* ia[0] == ib[0] */
+    } else { /* ia[0] == ib[0] */
         if (ia[1] < ib[1]) {
             return -1;
-        }
-        else if (ia[1] > ib[1]) {
+        } else if (ia[1] > ib[1]) {
             return 1;
-        }
-        else {
+        } else {
             return 0;
         }
     }
@@ -139,7 +132,7 @@ static double gettime(void)
     struct timeval tv;
     gettimeofday(&tv, NULL);
     wtime = tv.tv_sec;
-    wtime += (double)tv.tv_usec / 1000000.0;
+    wtime += (double) tv.tv_usec / 1000000.0;
 
     return wtime;
 }
@@ -195,7 +188,7 @@ static int test_graph_create(void *ctx)
     check(opal_bp_graph_order(g) == 5);
     check_graph_is_consistent(g);
     err = opal_bp_graph_add_edge(g, /*u=*/0, /*v=*/3, /*cost=*/1,
-                                     /*capacity=*/2, &user_data);
+                                 /*capacity=*/2, &user_data);
     check_graph_is_consistent(g);
     check(v_cleanup_count == 0);
     check(e_cleanup_count == 0);
@@ -234,11 +227,11 @@ static int test_graph_clone(void *ctx)
 
     /* and two edges */
     err = opal_bp_graph_add_edge(g, /*u=*/0, /*v=*/3, /*cost=*/1,
-                                     /*capacity=*/2, &user_data);
+                                 /*capacity=*/2, &user_data);
     check_err_code(err, OPAL_SUCCESS);
     check_graph_is_consistent(g);
     err = opal_bp_graph_add_edge(g, /*u=*/3, /*v=*/1, /*cost=*/2,
-                                     /*capacity=*/100, &user_data);
+                                 /*capacity=*/100, &user_data);
     check_err_code(err, OPAL_SUCCESS);
     check_graph_is_consistent(g);
 
@@ -283,17 +276,17 @@ static int test_graph_accessors(void *ctx)
     check(opal_bp_graph_order(g) == 4);
 
     err = opal_bp_graph_add_edge(g, /*u=*/0, /*v=*/2, /*cost=*/2,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     err = opal_bp_graph_add_edge(g, /*u=*/0, /*v=*/1, /*cost=*/2,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
 
-    check(opal_bp_graph_indegree(g,  0) == 0);
+    check(opal_bp_graph_indegree(g, 0) == 0);
     check(opal_bp_graph_outdegree(g, 0) == 2);
-    check(opal_bp_graph_indegree(g,  1) == 1);
+    check(opal_bp_graph_indegree(g, 1) == 1);
     check(opal_bp_graph_outdegree(g, 1) == 0);
-    check(opal_bp_graph_indegree(g,  2) == 1);
+    check(opal_bp_graph_indegree(g, 2) == 1);
     check(opal_bp_graph_outdegree(g, 2) == 0);
-    check(opal_bp_graph_indegree(g,  3) == 0);
+    check(opal_bp_graph_indegree(g, 3) == 0);
     check(opal_bp_graph_outdegree(g, 3) == 0);
 
     err = opal_bp_graph_free(g);
@@ -327,25 +320,22 @@ static int test_graph_assignment_solver(void *ctx)
     }
 
     err = opal_bp_graph_add_edge(g, /*u=*/0, /*v=*/2, /*cost=*/10,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     err = opal_bp_graph_add_edge(g, /*u=*/1, /*v=*/3, /*cost=*/2,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
 
     me = NULL;
-    err = opal_bp_graph_solve_bipartite_assignment(g,
-                                                    &nme,
-                                                    &me);
+    err = opal_bp_graph_solve_bipartite_assignment(g, &nme, &me);
     check_err_code(err, OPAL_SUCCESS);
     check_int_eq(nme, 2);
     check(me != NULL);
-    qsort(me, nme, 2*sizeof(int), &cmp_int_pair);
+    qsort(me, nme, 2 * sizeof(int), &cmp_int_pair);
     check(me[0] == 0 && me[1] == 2);
     check(me[2] == 1 && me[3] == 3);
 
     err = opal_bp_graph_free(g);
     check_err_code(err, OPAL_SUCCESS);
 
-
     /* TEST CASE: left side has more vertices than the right side
      *
      * 0 --> 3
@@ -362,23 +352,21 @@ static int test_graph_assignment_solver(void *ctx)
     }
 
     err = opal_bp_graph_add_edge(g, /*u=*/0, /*v=*/3, /*cost=*/10,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
     err = opal_bp_graph_add_edge(g, /*u=*/1, /*v=*/4, /*cost=*/2,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
     err = opal_bp_graph_add_edge(g, /*u=*/2, /*v=*/4, /*cost=*/1,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
 
     me = NULL;
-    err = opal_bp_graph_solve_bipartite_assignment(g,
-                                                    &nme,
-                                                    &me);
+    err = opal_bp_graph_solve_bipartite_assignment(g, &nme, &me);
     check_err_code(err, OPAL_SUCCESS);
     check_int_eq(nme, 2);
     check(me != NULL);
-    qsort(me, nme, 2*sizeof(int), &cmp_int_pair);
+    qsort(me, nme, 2 * sizeof(int), &cmp_int_pair);
     check(me[0] == 0 && me[1] == 3);
     check(me[2] == 2 && me[3] == 4);
     free(me);
@@ -386,7 +374,6 @@ static int test_graph_assignment_solver(void *ctx)
     err = opal_bp_graph_free(g);
     check_err_code(err, OPAL_SUCCESS);
 
-
     /* test Christian's case:
      * 0 --> 2
      * 0 --> 3
@@ -404,23 +391,21 @@ static int test_graph_assignment_solver(void *ctx)
     }
 
     err = opal_bp_graph_add_edge(g, /*u=*/0, /*v=*/2, /*cost=*/10,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
     err = opal_bp_graph_add_edge(g, /*u=*/0, /*v=*/3, /*cost=*/1,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
     err = opal_bp_graph_add_edge(g, /*u=*/1, /*v=*/3, /*cost=*/5,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
 
     me = NULL;
-    err = opal_bp_graph_solve_bipartite_assignment(g,
-                                                    &nme,
-                                                    &me);
+    err = opal_bp_graph_solve_bipartite_assignment(g, &nme, &me);
     check_err_code(err, OPAL_SUCCESS);
     check_int_eq(nme, 2);
     check(me != NULL);
-    qsort(me, nme, 2*sizeof(int), &cmp_int_pair);
+    qsort(me, nme, 2 * sizeof(int), &cmp_int_pair);
     check(me[0] == 0 && me[1] == 2);
     check(me[2] == 1 && me[3] == 3);
     free(me);
@@ -445,23 +430,21 @@ static int test_graph_assignment_solver(void *ctx)
     }
 
     err = opal_bp_graph_add_edge(g, /*u=*/0, /*v=*/2, /*cost=*/10,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
     err = opal_bp_graph_add_edge(g, /*u=*/1, /*v=*/2, /*cost=*/1,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
     err = opal_bp_graph_add_edge(g, /*u=*/1, /*v=*/3, /*cost=*/5,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
 
     me = NULL;
-    err = opal_bp_graph_solve_bipartite_assignment(g,
-                                                    &nme,
-                                                    &me);
+    err = opal_bp_graph_solve_bipartite_assignment(g, &nme, &me);
     check_err_code(err, OPAL_SUCCESS);
     check_int_eq(nme, 2);
     check(me != NULL);
-    qsort(me, nme, 2*sizeof(int), &cmp_int_pair);
+    qsort(me, nme, 2 * sizeof(int), &cmp_int_pair);
     check(me[0] == 0 && me[1] == 2);
     check(me[2] == 1 && me[3] == 3);
     free(me);
@@ -486,23 +469,21 @@ static int test_graph_assignment_solver(void *ctx)
     }
 
     err = opal_bp_graph_add_edge(g, /*u=*/0, /*v=*/2, /*cost=*/-1,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
     err = opal_bp_graph_add_edge(g, /*u=*/0, /*v=*/3, /*cost=*/-10,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
     err = opal_bp_graph_add_edge(g, /*u=*/1, /*v=*/3, /*cost=*/-5,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
 
     me = NULL;
-    err = opal_bp_graph_solve_bipartite_assignment(g,
-                                                    &nme,
-                                                    &me);
+    err = opal_bp_graph_solve_bipartite_assignment(g, &nme, &me);
     check_err_code(err, OPAL_SUCCESS);
     check_int_eq(nme, 2);
     check(me != NULL);
-    qsort(me, nme, 2*sizeof(int), &cmp_int_pair);
+    qsort(me, nme, 2 * sizeof(int), &cmp_int_pair);
     check(me[0] == 0 && me[1] == 2);
     check(me[2] == 1 && me[3] == 3);
     free(me);
@@ -510,7 +491,6 @@ static int test_graph_assignment_solver(void *ctx)
     err = opal_bp_graph_free(g);
     check_err_code(err, OPAL_SUCCESS);
 
-
     /* TEST CASE: add some disconnected vertices
      * 0 --> 2
      * 0 --> 3
@@ -529,23 +509,21 @@ static int test_graph_assignment_solver(void *ctx)
     }
 
     err = opal_bp_graph_add_edge(g, /*u=*/0, /*v=*/2, /*cost=*/-1,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
     err = opal_bp_graph_add_edge(g, /*u=*/0, /*v=*/3, /*cost=*/-10,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
     err = opal_bp_graph_add_edge(g, /*u=*/1, /*v=*/3, /*cost=*/-5,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
 
     me = NULL;
-    err = opal_bp_graph_solve_bipartite_assignment(g,
-                                                    &nme,
-                                                    &me);
+    err = opal_bp_graph_solve_bipartite_assignment(g, &nme, &me);
     check_err_code(err, OPAL_SUCCESS);
     check_int_eq(nme, 2);
     check(me != NULL);
-    qsort(me, nme, 2*sizeof(int), &cmp_int_pair);
+    qsort(me, nme, 2 * sizeof(int), &cmp_int_pair);
     check(me[0] == 0 && me[1] == 2);
     check(me[2] == 1 && me[3] == 3);
     free(me);
@@ -571,26 +549,24 @@ static int test_graph_assignment_solver(void *ctx)
     }
 
     err = opal_bp_graph_add_edge(g, /*u=*/0, /*v=*/2, /*cost=*/-4294967296,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
     err = opal_bp_graph_add_edge(g, /*u=*/1, /*v=*/2, /*cost=*/-4294967296,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
     err = opal_bp_graph_add_edge(g, /*u=*/0, /*v=*/3, /*cost=*/-4294967296,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
     err = opal_bp_graph_add_edge(g, /*u=*/1, /*v=*/3, /*cost=*/-4294967296,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
 
     me = NULL;
-    err = opal_bp_graph_solve_bipartite_assignment(g,
-                                                    &nme,
-                                                    &me);
+    err = opal_bp_graph_solve_bipartite_assignment(g, &nme, &me);
     check_err_code(err, OPAL_SUCCESS);
     check_int_eq(nme, 2);
     check(me != NULL);
-    qsort(me, nme, 2*sizeof(int), &cmp_int_pair);
+    qsort(me, nme, 2 * sizeof(int), &cmp_int_pair);
     if (me[1] == 2) {
         check(me[0] == 0 && me[1] == 2);
         check(me[2] == 1 && me[3] == 3);
@@ -603,7 +579,6 @@ static int test_graph_assignment_solver(void *ctx)
     err = opal_bp_graph_free(g);
     check_err_code(err, OPAL_SUCCESS);
 
-
     /* TEST CASE: check that simple cases are solved correctly
      *
      * 0 --> 2
@@ -619,24 +594,21 @@ static int test_graph_assignment_solver(void *ctx)
     }
 
     err = opal_bp_graph_add_edge(g, /*u=*/0, /*v=*/2, /*cost=*/-100,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     err = opal_bp_graph_add_edge(g, /*u=*/1, /*v=*/2, /*cost=*/-100,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
 
     me = NULL;
-    err = opal_bp_graph_solve_bipartite_assignment(g,
-                                                    &nme,
-                                                    &me);
+    err = opal_bp_graph_solve_bipartite_assignment(g, &nme, &me);
     check_err_code(err, OPAL_SUCCESS);
     check_int_eq(nme, 1);
     check(me != NULL);
-    qsort(me, nme, 2*sizeof(int), &cmp_int_pair);
+    qsort(me, nme, 2 * sizeof(int), &cmp_int_pair);
     check((me[0] == 0 || me[0] == 1) && me[1] == 2);
 
     err = opal_bp_graph_free(g);
     check_err_code(err, OPAL_SUCCESS);
 
-
     /* TEST CASE: performance sanity check
      *
      * Construct this graph and ensure that it doesn't take too long on a large
@@ -658,23 +630,21 @@ static int test_graph_assignment_solver(void *ctx)
         }
 
         err = opal_bp_graph_add_edge(g, /*u=*/0, /*v=*/3, /*cost=*/10,
-                                        /*capacity=*/1, NULL);
+                                     /*capacity=*/1, NULL);
         check_err_code(err, OPAL_SUCCESS);
         err = opal_bp_graph_add_edge(g, /*u=*/1, /*v=*/4, /*cost=*/2,
-                                        /*capacity=*/1, NULL);
+                                     /*capacity=*/1, NULL);
         check_err_code(err, OPAL_SUCCESS);
         err = opal_bp_graph_add_edge(g, /*u=*/2, /*v=*/4, /*cost=*/1,
-                                        /*capacity=*/1, NULL);
+                                     /*capacity=*/1, NULL);
         check_err_code(err, OPAL_SUCCESS);
 
         me = NULL;
-        err = opal_bp_graph_solve_bipartite_assignment(g,
-                                                        &nme,
-                                                        &me);
+        err = opal_bp_graph_solve_bipartite_assignment(g, &nme, &me);
         check_err_code(err, OPAL_SUCCESS);
         check_int_eq(nme, 2);
         check(me != NULL);
-        qsort(me, nme, 2*sizeof(int), &cmp_int_pair);
+        qsort(me, nme, 2 * sizeof(int), &cmp_int_pair);
         check(me[0] == 0 && me[1] == 3);
         check(me[2] == 2 && me[3] == 4);
         free(me);
@@ -720,25 +690,25 @@ static int test_graph_bellman_ford(void *ctx)
     }
 
     err = opal_bp_graph_add_edge(g, /*u=*/0, /*v=*/2, /*cost=*/10,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
     err = opal_bp_graph_add_edge(g, /*u=*/1, /*v=*/3, /*cost=*/2,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
     err = opal_bp_graph_add_edge(g, /*u=*/4, /*v=*/0, /*cost=*/0,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
     err = opal_bp_graph_add_edge(g, /*u=*/4, /*v=*/1, /*cost=*/0,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
     err = opal_bp_graph_add_edge(g, /*u=*/2, /*v=*/5, /*cost=*/0,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
     err = opal_bp_graph_add_edge(g, /*u=*/3, /*v=*/5, /*cost=*/0,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
 
-    pred = malloc(6*sizeof(*pred));
+    pred = malloc(6 * sizeof(*pred));
     check(pred != NULL);
     path_found = opal_bp_graph_bellman_ford(g, /*source=*/4, /*target=*/5, pred);
     check(path_found);
@@ -751,7 +721,6 @@ static int test_graph_bellman_ford(void *ctx)
     err = opal_bp_graph_free(g);
     check_err_code(err, OPAL_SUCCESS);
 
-
     /* TEST CASE: left side has more vertices than the right side, then
      * convert to a flow network
      *
@@ -769,19 +738,19 @@ static int test_graph_bellman_ford(void *ctx)
     }
 
     err = opal_bp_graph_add_edge(g, /*u=*/0, /*v=*/3, /*cost=*/10,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
     err = opal_bp_graph_add_edge(g, /*u=*/1, /*v=*/4, /*cost=*/2,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
     err = opal_bp_graph_add_edge(g, /*u=*/2, /*v=*/4, /*cost=*/1,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
 
     err = opal_bp_graph_bipartite_to_flow(g);
     check_err_code(err, OPAL_SUCCESS);
 
-    pred = malloc(7*sizeof(*pred));
+    pred = malloc(7 * sizeof(*pred));
     check(pred != NULL);
     path_found = opal_bp_graph_bellman_ford(g, /*source=*/5, /*target=*/6, pred);
     check(path_found);
@@ -812,20 +781,20 @@ static int test_graph_bellman_ford(void *ctx)
         check_err_code(err, OPAL_SUCCESS);
     }
 
-    err = opal_bp_graph_add_edge(g, /*u=*/0, /*v=*/3, /*cost=*/INT32_MAX+10LL,
-                                     /*capacity=*/1, NULL);
+    err = opal_bp_graph_add_edge(g, /*u=*/0, /*v=*/3, /*cost=*/INT32_MAX + 10LL,
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
-    err = opal_bp_graph_add_edge(g, /*u=*/1, /*v=*/4, /*cost=*/INT32_MAX+2LL,
-                                     /*capacity=*/1, NULL);
+    err = opal_bp_graph_add_edge(g, /*u=*/1, /*v=*/4, /*cost=*/INT32_MAX + 2LL,
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
-    err = opal_bp_graph_add_edge(g, /*u=*/2, /*v=*/4, /*cost=*/INT32_MAX+1LL,
-                                     /*capacity=*/1, NULL);
+    err = opal_bp_graph_add_edge(g, /*u=*/2, /*v=*/4, /*cost=*/INT32_MAX + 1LL,
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
 
     err = opal_bp_graph_bipartite_to_flow(g);
     check_err_code(err, OPAL_SUCCESS);
 
-    pred = malloc(7*sizeof(*pred));
+    pred = malloc(7 * sizeof(*pred));
     check(pred != NULL);
     path_found = opal_bp_graph_bellman_ford(g, /*source=*/5, /*target=*/6, pred);
     check(path_found);
@@ -858,19 +827,19 @@ static int test_graph_bellman_ford(void *ctx)
     }
 
     err = opal_bp_graph_add_edge(g, /*u=*/0, /*v=*/3, /*cost=*/-1,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
     err = opal_bp_graph_add_edge(g, /*u=*/1, /*v=*/4, /*cost=*/-2,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
     err = opal_bp_graph_add_edge(g, /*u=*/2, /*v=*/4, /*cost=*/-10,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
 
     err = opal_bp_graph_bipartite_to_flow(g);
     check_err_code(err, OPAL_SUCCESS);
 
-    pred = malloc(7*sizeof(*pred));
+    pred = malloc(7 * sizeof(*pred));
     check(pred != NULL);
     path_found = opal_bp_graph_bellman_ford(g, /*source=*/5, /*target=*/6, pred);
     check(path_found);
@@ -911,13 +880,13 @@ static int test_graph_flow_conversion(void *ctx)
     }
 
     err = opal_bp_graph_add_edge(g, /*u=*/0, /*v=*/3, /*cost=*/10,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
     err = opal_bp_graph_add_edge(g, /*u=*/1, /*v=*/4, /*cost=*/2,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
     err = opal_bp_graph_add_edge(g, /*u=*/2, /*v=*/4, /*cost=*/1,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
 
     check_int_eq(opal_bp_graph_order(g), 5);
@@ -943,7 +912,6 @@ static int test_graph_flow_conversion(void *ctx)
     err = opal_bp_graph_free(g);
     check_err_code(err, OPAL_SUCCESS);
 
-
     /* TEST CASE: empty graph
      *
      * there's no reason that the code should bother to support this, it's not
@@ -973,7 +941,7 @@ static int test_graph_param_checking(void *ctx)
 
     /* try with no vertices */
     err = opal_bp_graph_add_edge(g, /*u=*/3, /*v=*/5, /*cost=*/0,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_ERR_BAD_PARAM);
 
     for (i = 0; i < 6; ++i) {
@@ -983,34 +951,34 @@ static int test_graph_param_checking(void *ctx)
 
     /* try u out of range */
     err = opal_bp_graph_add_edge(g, /*u=*/9, /*v=*/5, /*cost=*/0,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_ERR_BAD_PARAM);
     err = opal_bp_graph_add_edge(g, /*u=*/6, /*v=*/5, /*cost=*/0,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_ERR_BAD_PARAM);
 
     /* try v out of range */
     err = opal_bp_graph_add_edge(g, /*u=*/2, /*v=*/8, /*cost=*/0,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_ERR_BAD_PARAM);
     err = opal_bp_graph_add_edge(g, /*u=*/2, /*v=*/6, /*cost=*/0,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_ERR_BAD_PARAM);
 
     /* try adding an edge that already exists */
     err = opal_bp_graph_add_edge(g, /*u=*/2, /*v=*/4, /*cost=*/0,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
     err = opal_bp_graph_add_edge(g, /*u=*/2, /*v=*/4, /*cost=*/0,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_EXISTS);
 
     /* try an edge with an out of range cost */
     err = opal_bp_graph_add_edge(g, /*u=*/2, /*v=*/3, /*cost=*/INT64_MAX,
-                                     /*capacity=*/1, NULL);
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_ERR_BAD_PARAM);
-    err = opal_bp_graph_add_edge(g, /*u=*/2, /*v=*/3, /*cost=*/INT64_MAX-1,
-                                     /*capacity=*/1, NULL);
+    err = opal_bp_graph_add_edge(g, /*u=*/2, /*v=*/3, /*cost=*/INT64_MAX - 1,
+                                 /*capacity=*/1, NULL);
     check_err_code(err, OPAL_SUCCESS);
 
     err = opal_bp_graph_free(g);
@@ -1040,7 +1008,8 @@ static int test_graph_helper_macros(void *ctx)
     /* TEST CASE: make sure that an empty path does not cause any edges to be
      * visited */
     RESET_ARRAYS(6, pred, visited);
-    FOREACH_UV_ON_PATH(pred, 3, 5, u, v) {
+    FOREACH_UV_ON_PATH(pred, 3, 5, u, v)
+    {
         visited[u][v] = true;
     }
     for (u = 0; u < 6; ++u) {
@@ -1054,17 +1023,15 @@ static int test_graph_helper_macros(void *ctx)
     pred[5] = 2;
     pred[2] = 1;
     pred[1] = 3;
-    FOREACH_UV_ON_PATH(pred, 3, 5, u, v) {
+    FOREACH_UV_ON_PATH(pred, 3, 5, u, v)
+    {
         visited[u][v] = true;
     }
     for (u = 0; u < 6; ++u) {
         for (v = 0; v < 6; ++v) {
-            if ((u == 2 && v == 5) ||
-                (u == 1 && v == 2) ||
-                (u == 3 && v == 1)) {
+            if ((u == 2 && v == 5) || (u == 1 && v == 2) || (u == 3 && v == 1)) {
                 check(visited[u][v] == true);
-            }
-            else {
+            } else {
                 check(visited[u][v] == false);
             }
         }
@@ -1074,24 +1041,34 @@ static int test_graph_helper_macros(void *ctx)
 
     /* not technically a macro, but make sure that the pair comparison function
      * isn't broken (because it was in an earlier revision...) */
-    pair1[0] = 0; pair1[1] = 1;
-    pair2[0] = 0; pair2[1] = 1;
+    pair1[0] = 0;
+    pair1[1] = 1;
+    pair2[0] = 0;
+    pair2[1] = 1;
     check(cmp_int_pair(&pair1[0], &pair2[0]) == 0);
 
-    pair1[0] = 1; pair1[1] = 1;
-    pair2[0] = 0; pair2[1] = 1;
+    pair1[0] = 1;
+    pair1[1] = 1;
+    pair2[0] = 0;
+    pair2[1] = 1;
     check(cmp_int_pair(pair1, pair2) > 0);
 
-    pair1[0] = 0; pair1[1] = 1;
-    pair2[0] = 1; pair2[1] = 1;
+    pair1[0] = 0;
+    pair1[1] = 1;
+    pair2[0] = 1;
+    pair2[1] = 1;
     check(cmp_int_pair(pair1, pair2) < 0);
 
-    pair1[0] = 1; pair1[1] = 0;
-    pair2[0] = 1; pair2[1] = 1;
+    pair1[0] = 1;
+    pair1[1] = 0;
+    pair2[0] = 1;
+    pair2[1] = 1;
     check(cmp_int_pair(pair1, pair2) < 0);
 
-    pair1[0] = 1; pair1[1] = 1;
-    pair2[0] = 1; pair2[1] = 0;
+    pair1[0] = 1;
+    pair1[1] = 1;
+    pair2[0] = 1;
+    pair2[1] = 0;
     check(cmp_int_pair(pair1, pair2) > 0);
 
     return 0;
diff --git a/test/util/ompi_numtostr.c b/test/util/ompi_numtostr.c
index 9a45b6a4708..16861cb91f9 100644
--- a/test/util/ompi_numtostr.c
+++ b/test/util/ompi_numtostr.c
@@ -1,16 +1,15 @@
 #include "ompi_config.h"
 #include "opal/util/numtostr.h"
 
-#include <string.h>
 #include <stdio.h>
+#include <string.h>
 
 #include "support.h"
 
-int
-main(int argc, char *argv[])
+int main(int argc, char *argv[])
 {
-    char * tst;
-    char * expected;
+    char *tst;
+    char *expected;
 
     test_init("ompi_numtostr_t");
 
@@ -18,10 +17,9 @@ main(int argc, char *argv[])
     expected = malloc(sizeof(long) * 8);
     snprintf(expected, sizeof(long) * 8, "%d", 10);
     if (strcmp(tst, expected) != 0) {
-      test_failure("opal_ltostr test failed");
-    }
-    else {
-      test_success();
+        test_failure("opal_ltostr test failed");
+    } else {
+        test_success();
     }
 
     free(tst);
@@ -31,10 +29,9 @@ main(int argc, char *argv[])
     expected = malloc(sizeof(long) * 8);
     snprintf(expected, sizeof(long) * 8, "%f", 5.32);
     if (strcmp(tst, expected) != 0) {
-      test_failure("opal_dtostr test failed");
-    }
-    else {
-      test_success();
+        test_failure("opal_dtostr test failed");
+    } else {
+        test_success();
     }
 
     test_finalize();
diff --git a/test/util/opal_argv.c b/test/util/opal_argv.c
index 63bb3413daa..7335d46cb1b 100644
--- a/test/util/opal_argv.c
+++ b/test/util/opal_argv.c
@@ -18,12 +18,12 @@
 
 #include "ompi_config.h"
 #include <stdio.h>
-#include <string.h>
 #include <stdlib.h>
+#include <string.h>
 
-#include "support.h"
 #include "opal/constants.h"
 #include "opal/util/argv.h"
+#include "support.h"
 
 static bool test1(void);
 static bool test2(void);
@@ -36,330 +36,342 @@ static bool test8(void);
 static bool test9(void);
 static bool test10(void);
 
-
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
 
-  test_init("opal_argv_t");
-
-  if( test1() ) test_success();
-  else test_failure("test1 argv test failed");
-
-  if( test2() ) test_success();
-  else test_failure("test2 argv test failed");
-
-  if( test3() ) test_success();
-  else test_failure("test3 argv test failed");
-
-  if( test4() ) test_success();
-  else test_failure("test4 argv test failed");
+    test_init("opal_argv_t");
+
+    if (test1())
+        test_success();
+    else
+        test_failure("test1 argv test failed");
+
+    if (test2())
+        test_success();
+    else
+        test_failure("test2 argv test failed");
+
+    if (test3())
+        test_success();
+    else
+        test_failure("test3 argv test failed");
+
+    if (test4())
+        test_success();
+    else
+        test_failure("test4 argv test failed");
+
+    if (test5())
+        test_success();
+    else
+        test_failure("test5 argv test failed");
+
+    if (test6())
+        test_success();
+    else
+        test_failure("test6 argv test failed");
+
+    if (test7())
+        test_success();
+    else
+        test_failure("test7 argv test failed");
+
+    if (test8())
+        test_success();
+    else
+        test_failure("test8 argv test failed");
+
+    if (test9()) {
+        test_success();
+    } else {
+        test_failure("test9 argv test failed");
+    }
 
-  if( test5() ) test_success();
-  else test_failure("test5 argv test failed");
+    if (test10()) {
+        test_success();
+    } else {
+        test_failure("test10 argv test failed");
+    }
 
-  if( test6() ) test_success();
-  else test_failure("test6 argv test failed");
+    /* All done */
+    return test_finalize();
+}
 
-  if( test7() ) test_success();
-  else test_failure("test7 argv test failed");
+static bool test1(void)
+{
+    char *a[] = {"argv1", "argv2", "argv3", NULL};
+    char **argv = NULL;
+    int i, j, argc = 28;
+
+    /* Test basic functionality.  Start with a NULL argv and add the
+       contents of the a array.
+
+       Set argc to be an initial bogus number -- opal_argv_add() should
+       reset it back to zero after the first iteration.
+
+       After adding the a[i], ensure that argv[0 - (i-1)] are the same
+       as a[0 - (i-1)].
+
+       Also ensure that a[i + 1] == NULL and that argc is the proper
+       value. */
+
+    for (i = 0; a[i] != NULL; ++i) {
+        if (opal_argv_append(&argc, &argv, a[i]) != OPAL_SUCCESS) {
+            return false;
+        }
+        for (j = 0; j <= i; ++j) {
+            if (strcmp(argv[j], a[j]) != 0) {
+                return false;
+            }
+        }
+        if (NULL != argv[i + 1]) {
+            return false;
+        }
+        if (argc != i + 1) {
+            return false;
+        }
+    }
+    opal_argv_free(argv);
 
-  if( test8() ) test_success();
-  else test_failure("test8 argv test failed");
+    return true;
+}
 
-  if (test9()) {
-      test_success();
-  } else {
-      test_failure("test9 argv test failed");
-  }
+static bool test2(void)
+{
+    int i, j;
+    char **argv = NULL;
+    int argc;
+    char *a[] = {"aaa", "bbb", "ccc", NULL};
+    char *b[4];
+
+    /* Similar test to above, but ensure that opal_argv_add is actually
+       *copying* the string by value, not by reference.  So copy the a
+       array into b, and then opal_argv_add() from b.  After that,
+       scribble in the first character of the b[] string that we just
+       added, and compare all entries in a to argv -- they should be
+       identical (even though b is now corrupted). */
+
+    for (i = 0; a[i] != NULL; ++i) {
+        b[i] = strdup(a[i]);
+    }
+    b[i] = NULL;
+
+    for (i = 0; b[i] != NULL; ++i) {
+        if (opal_argv_append(&argc, &argv, b[i]) != OPAL_SUCCESS) {
+            return false;
+        }
+        ++b[i][0];
+        for (j = 0; j <= i; ++j) {
+            if (strcmp(argv[j], a[j]) != 0) {
+                return false;
+            }
+        }
+        if (NULL != argv[i + 1]) {
+            return false;
+        }
+        if (argc != i + 1) {
+            return false;
+        }
+    }
 
-  if (test10()) {
-      test_success();
-  } else {
-      test_failure("test10 argv test failed");
-  }
+    opal_argv_free(argv);
+    for (i = 0; b[i] != NULL; ++i) {
+        free(b[i]);
+    }
 
-  /* All done */
-  return test_finalize();
+    return true;
 }
 
-
-static bool test1(void)
+static bool test3(void)
 {
-  char *a[] = { "argv1", "argv2", "argv3", NULL };
-  char **argv = NULL;
-  int i, j, argc = 28;
-
-  /* Test basic functionality.  Start with a NULL argv and add the
-     contents of the a array.
+    int i;
+    int argc;
+    char **argv = NULL;
+    char *a[] = {"aaa", "bbb", "ccc", NULL};
+    char *b[4];
 
-     Set argc to be an initiall bogus number -- opal_argv_add() should
-     reset it back to zero after the first iteration.
+    /* Try to free a null argv -- should be harmless (we'll seg fault if
+       it's not!) */
 
-     After adding the a[i], ensure that argv[0 - (i-1)] are the same
-     as a[0 - (i-1)].
+    opal_argv_free(argv);
 
-     Also ensure that a[i + 1] == NULL and that argc is the proper
-     value. */
+    /* Now add some stuff and try to free it.  We'll seg fault if
+       anything goes wrong.  a is on the stack, so if it mistakenly
+       tries to free it, we should get a seg fault. */
 
-  for (i = 0; a[i] != NULL; ++i) {
-    if (opal_argv_append(&argc, &argv, a[i]) != OPAL_SUCCESS) {
-      return false;
-    }
-    for (j = 0; j <= i; ++j) {
-      if (strcmp(argv[j], a[j]) != 0) {
-        return false;
-      }
-    }
-    if (NULL != argv[i + 1]) {
-      return false;
-    }
-    if (argc != i + 1) {
-      return false;
+    for (i = 0; a[i] != NULL; ++i) {
+        if (opal_argv_append(&argc, &argv, a[i]) != OPAL_SUCCESS) {
+            return false;
+        }
     }
-  }
-  opal_argv_free(argv);
-
-  return true;
-}
+    opal_argv_free(argv);
 
+    /* Do the same thing but guarantee that the copied array was from
+       the heap and was freed before we call opal_argv_free(). */
 
-static bool test2(void)
-{
-  int i, j;
-  char **argv = NULL;
-  int argc;
-  char *a[] = { "aaa", "bbb", "ccc", NULL };
-  char *b[4];
-
-  /* Similar test to above, but ensure that opal_argv_add is actually
-     *copying* the string by value, not by reference.  So copy the a
-     array into b, and then opal_argv_add() from b.  After that,
-     scribble in the first character of the b[] string that we just
-     added, and compare all entries in a to argv -- they should be
-     identical (even though b is now corrupted). */
-
-  for (i = 0; a[i] != NULL; ++i) {
-    b[i] = strdup(a[i]);
-  }
-  b[i] = NULL;
-
-  for (i = 0; b[i] != NULL; ++i) {
-    if (opal_argv_append(&argc, &argv, b[i]) != OPAL_SUCCESS) {
-      return false;
-    }
-    ++b[i][0];
-    for (j = 0; j <= i; ++j) {
-      if (strcmp(argv[j], a[j]) != 0) {
-        return false;
-      }
+    argc = 0;
+    argv = NULL;
+    for (i = 0; a[i] != NULL; ++i) {
+        b[i] = strdup(a[i]);
     }
-    if (NULL != argv[i + 1]) {
-      return false;
+    b[i] = NULL;
+    for (i = 0; b[i] != NULL; ++i) {
+        if (opal_argv_append(&argc, &argv, b[i]) != OPAL_SUCCESS) {
+            return false;
+        }
     }
-    if (argc != i + 1) {
-      return false;
+    for (i = 0; b[i] != NULL; ++i) {
+        free(b[i]);
     }
-  }
-
-  opal_argv_free(argv);
-  for (i = 0; b[i] != NULL; ++i) {
-    free(b[i]);
-  }
-
-  return true;
-}
+    opal_argv_free(argv);
 
+    /* All done */
 
-static bool test3(void)
-{
-  int i;
-  int argc;
-  char **argv = NULL;
-  char *a[] = { "aaa", "bbb", "ccc", NULL };
-  char *b[4];
-
-  /* Try to free a null argv -- should be harmless (we'll seg fault if
-     it's not!) */
-
-  opal_argv_free(argv);
-
-  /* Now add some stuff and try to free it.  We'll seg fault if
-     anything goes wrong.  a is on the stack, so if it mistakenly
-     tries to free it, we should get a seg fault. */
-
-  for (i = 0; a[i] != NULL; ++i) {
-    if (opal_argv_append(&argc, &argv, a[i]) != OPAL_SUCCESS) {
-      return false;
-    }
-  }
-  opal_argv_free(argv);
-
-  /* Do the same thing but guarantee that the copied array was from
-     the heap and was freed before we call opal_argv_free(). */
-
-  argc = 0;
-  argv = NULL;
-  for (i = 0; a[i] != NULL; ++i) {
-    b[i] = strdup(a[i]);
-  }
-  b[i] = NULL;
-  for (i = 0; b[i] != NULL; ++i) {
-    if (opal_argv_append(&argc, &argv, b[i]) != OPAL_SUCCESS) {
-      return false;
-    }
-  }
-  for (i = 0; b[i] != NULL; ++i) {
-    free(b[i]);
-  }
-  opal_argv_free(argv);
-
-  /* All done */
-
-  return true;
+    return true;
 }
 
-
 static bool test4(void)
 {
-  size_t i, count;
-  char *a = strdup("the quick  brown fox jumped over  the lazy  dog a_really_long_argument_to_force_a_long_copy_zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz");
-  char **b;
-  char *start;
-
-  /* split a string into an argv, and compare it against the original
-     string.  Add double spaces into the string; opal_argv_split()
-     should skip them. */
-
-  b = opal_argv_split(a, ' ');
-
-  for (count = i = 1; i < strlen(a); ++i) {
-    if (a[i] != ' ' && a[i - 1] == ' ') {
-      ++count;
-    }
-  }
-  for (i = 0; b[i] != NULL; ++i) {
-    continue;
-  }
-  if (i != count) {
-    return false;
-  }
-
-  /* now do the same thing and compare each token in b */
-
-  for (start = a, count = i = 0; i < strlen(a); ++i) {
-    if (a[i] == ' ' && a[i - 1] != ' ') {
-      a[i] = '\0';
-      if (strcmp(start, b[count]) != 0) {
+    size_t i, count;
+    char *a = strdup("the quick  brown fox jumped over  the lazy  dog "
+                     "a_really_long_argument_to_force_a_long_copy_"
+                     "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"
+                     "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz");
+    char **b;
+    char *start;
+
+    /* split a string into an argv, and compare it against the original
+       string.  Add double spaces into the string; opal_argv_split()
+       should skip them. */
+
+    b = opal_argv_split(a, ' ');
+
+    for (count = i = 1; i < strlen(a); ++i) {
+        if (a[i] != ' ' && a[i - 1] == ' ') {
+            ++count;
+        }
+    }
+    for (i = 0; b[i] != NULL; ++i) {
+        continue;
+    }
+    if (i != count) {
         return false;
-      }
-      ++count;
-      a[i] = ' ';
     }
-    if (a[i] == ' ' && a[i + 1] != ' ') {
-      start = a + i + 1;
+
+    /* now do the same thing and compare each token in b */
+
+    for (start = a, count = i = 0; i < strlen(a); ++i) {
+        if (a[i] == ' ' && a[i - 1] != ' ') {
+            a[i] = '\0';
+            if (strcmp(start, b[count]) != 0) {
+                return false;
+            }
+            ++count;
+            a[i] = ' ';
+        }
+        if (a[i] == ' ' && a[i + 1] != ' ') {
+            start = a + i + 1;
+        }
+    }
+    if (strcmp(start, b[count]) != 0) {
+        return false;
     }
-  }
-  if (strcmp(start, b[count]) != 0) {
-    return false;
-  }
 
-  /* all done */
+    /* all done */
 
-  opal_argv_free(b);
-  free(a);
-  return true;
+    opal_argv_free(b);
+    free(a);
+    return true;
 }
 
-
 static bool test5(void)
 {
-  char *a[] = { "aaa", "bbb", "ccc", NULL };
+    char *a[] = {"aaa", "bbb", "ccc", NULL};
 
-  return (opal_argv_count(NULL) == 0 && opal_argv_count(a) == 3);
+    return (opal_argv_count(NULL) == 0 && opal_argv_count(a) == 3);
 }
 
-
 static bool test6(void)
 {
-  char *a = "the quick brown fox jumped over the lazy dog a_really_long_argument_to_force_a_long_copy_zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz";
-  char **b;
-  char *c;
+    char *a = "the quick brown fox jumped over the lazy dog "
+              "a_really_long_argument_to_force_a_long_copy_"
+              "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"
+              "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz";
+    char **b;
+    char *c;
 
-  /* split the string above and then join it -- the joined version
-     should be just like the original */
+    /* split the string above and then join it -- the joined version
+       should be just like the original */
 
-  b = opal_argv_split(a, ' ');
-  c = opal_argv_join(b, ' ');
+    b = opal_argv_split(a, ' ');
+    c = opal_argv_join(b, ' ');
 
-  if (strcmp(a, c) != 0) {
-    return false;
-  }
+    if (strcmp(a, c) != 0) {
+        return false;
+    }
 
-  /* All done */
+    /* All done */
 
-  free(c);
-  opal_argv_free(b);
-  return true;
+    free(c);
+    opal_argv_free(b);
+    return true;
 }
 
-
 static bool test7(void)
 {
-  char *a[] = { "a", "b", "c", NULL };
-  size_t a_len = (1 + 1 + sizeof(char *)) * 3 + sizeof(char **);
+    char *a[] = {"a", "b", "c", NULL};
+    size_t a_len = (1 + 1 + sizeof(char *)) * 3 + sizeof(char **);
 
-  /* check a NULL pointer first -- should return 0 */
+    /* check a NULL pointer first -- should return 0 */
 
-  if (opal_argv_len(NULL) != (size_t) 0) {
-    return false;
-  }
+    if (opal_argv_len(NULL) != (size_t) 0) {
+        return false;
+    }
 
-  /* now check a real string */
-  /* size should be (sizeof(char **) + (sizeof(char) + sizeof('\0') +
-     sizeof(char*)) * 3) */
+    /* now check a real string */
+    /* size should be (sizeof(char **) + (sizeof(char) + sizeof('\0') +
+       sizeof(char*)) * 3) */
 
-  if (opal_argv_len(a) != a_len) {
-    return false;
-  }
+    if (opal_argv_len(a) != a_len) {
+        return false;
+    }
 
-  /* All done */
+    /* All done */
 
-  return true;
+    return true;
 }
 
-
 static bool test8(void)
 {
-  char *a[] = { "aaa", "bbbbbbbb", "cccccccccc", NULL };
-  int i;
-  char **b;
+    char *a[] = {"aaa", "bbbbbbbb", "cccccccccc", NULL};
+    int i;
+    char **b;
 
-  /* bozo case */
+    /* bozo case */
 
-  if (NULL != opal_argv_copy(NULL)) {
-    return false;
-  }
+    if (NULL != opal_argv_copy(NULL)) {
+        return false;
+    }
 
-  /* dup the a array and compare it (array length, contents, etc.) */
+    /* dup the a array and compare it (array length, contents, etc.) */
 
-  b = opal_argv_copy(a);
+    b = opal_argv_copy(a);
 
-  if (opal_argv_count(a) != opal_argv_count(b)) {
-    return false;
-  }
-  for (i = 0; a[i] != NULL; ++i) {
-    if (0 != strcmp(a[i], b[i])) {
-      return false;
+    if (opal_argv_count(a) != opal_argv_count(b)) {
+        return false;
+    }
+    for (i = 0; a[i] != NULL; ++i) {
+        if (0 != strcmp(a[i], b[i])) {
+            return false;
+        }
     }
-  }
 
-  /* All done */
+    /* All done */
 
-  opal_argv_free(b);
-  return true;
+    opal_argv_free(b);
+    return true;
 }
 
-
 static bool test9(void)
 {
     char **a = NULL;
@@ -374,8 +386,7 @@ static bool test9(void)
     a = NULL;
     argc = 0;
     opal_argv_append(&argc, &a, "foo");
-    if (OPAL_SUCCESS != opal_argv_delete(&argc, &a, 7, 1) ||
-        1 != opal_argv_count(a)) {
+    if (OPAL_SUCCESS != opal_argv_delete(&argc, &a, 7, 1) || 1 != opal_argv_count(a)) {
         return false;
     }
     opal_argv_free(a);
@@ -383,8 +394,7 @@ static bool test9(void)
     a = NULL;
     argc = 0;
     opal_argv_append(&argc, &a, "foo");
-    if (OPAL_SUCCESS != opal_argv_delete(&argc, &a, 0, 0) ||
-        1 != opal_argv_count(a)) {
+    if (OPAL_SUCCESS != opal_argv_delete(&argc, &a, 0, 0) || 1 != opal_argv_count(a)) {
         return false;
     }
     opal_argv_free(a);
@@ -400,13 +410,9 @@ static bool test9(void)
     opal_argv_append(&argc, &a, "d");
     opal_argv_append(&argc, &a, "e");
     opal_argv_append(&argc, &a, "f");
-    if (OPAL_SUCCESS != opal_argv_delete(&argc, &a, 0, 1) ||
-        5 != opal_argv_count(a) ||
-        0 != strcmp(a[0], "b") ||
-        0 != strcmp(a[1], "c") ||
-        0 != strcmp(a[2], "d") ||
-        0 != strcmp(a[3], "e") ||
-        0 != strcmp(a[4], "f")) {
+    if (OPAL_SUCCESS != opal_argv_delete(&argc, &a, 0, 1) || 5 != opal_argv_count(a)
+        || 0 != strcmp(a[0], "b") || 0 != strcmp(a[1], "c") || 0 != strcmp(a[2], "d")
+        || 0 != strcmp(a[3], "e") || 0 != strcmp(a[4], "f")) {
         return false;
     }
     opal_argv_free(a);
@@ -421,12 +427,9 @@ static bool test9(void)
     opal_argv_append(&argc, &a, "d");
     opal_argv_append(&argc, &a, "e");
     opal_argv_append(&argc, &a, "f");
-    if (OPAL_SUCCESS != opal_argv_delete(&argc, &a, 0, 2) ||
-        4 != opal_argv_count(a) ||
-        0 != strcmp(a[0], "c") ||
-        0 != strcmp(a[1], "d") ||
-        0 != strcmp(a[2], "e") ||
-        0 != strcmp(a[3], "f")) {
+    if (OPAL_SUCCESS != opal_argv_delete(&argc, &a, 0, 2) || 4 != opal_argv_count(a)
+        || 0 != strcmp(a[0], "c") || 0 != strcmp(a[1], "d") || 0 != strcmp(a[2], "e")
+        || 0 != strcmp(a[3], "f")) {
         return false;
     }
     opal_argv_free(a);
@@ -441,13 +444,9 @@ static bool test9(void)
     opal_argv_append(&argc, &a, "d");
     opal_argv_append(&argc, &a, "e");
     opal_argv_append(&argc, &a, "f");
-    if (OPAL_SUCCESS != opal_argv_delete(&argc, &a, 1, 1) ||
-        5 != opal_argv_count(a) ||
-        0 != strcmp(a[0], "a") ||
-        0 != strcmp(a[1], "c") ||
-        0 != strcmp(a[2], "d") ||
-        0 != strcmp(a[3], "e") ||
-        0 != strcmp(a[4], "f")) {
+    if (OPAL_SUCCESS != opal_argv_delete(&argc, &a, 1, 1) || 5 != opal_argv_count(a)
+        || 0 != strcmp(a[0], "a") || 0 != strcmp(a[1], "c") || 0 != strcmp(a[2], "d")
+        || 0 != strcmp(a[3], "e") || 0 != strcmp(a[4], "f")) {
         return false;
     }
     opal_argv_free(a);
@@ -462,12 +461,9 @@ static bool test9(void)
     opal_argv_append(&argc, &a, "d");
     opal_argv_append(&argc, &a, "e");
     opal_argv_append(&argc, &a, "f");
-    if (OPAL_SUCCESS != opal_argv_delete(&argc, &a, 1, 2) ||
-        4 != opal_argv_count(a) ||
-        0 != strcmp(a[0], "a") ||
-        0 != strcmp(a[1], "d") ||
-        0 != strcmp(a[2], "e") ||
-        0 != strcmp(a[3], "f")) {
+    if (OPAL_SUCCESS != opal_argv_delete(&argc, &a, 1, 2) || 4 != opal_argv_count(a)
+        || 0 != strcmp(a[0], "a") || 0 != strcmp(a[1], "d") || 0 != strcmp(a[2], "e")
+        || 0 != strcmp(a[3], "f")) {
         return false;
     }
     opal_argv_free(a);
@@ -478,8 +474,7 @@ static bool test9(void)
     argc = 0;
     opal_argv_append(&argc, &a, "a");
     opal_argv_append(&argc, &a, "b");
-    if (OPAL_SUCCESS != opal_argv_delete(&argc, &a, 0, 99) ||
-        0 != opal_argv_count(a)) {
+    if (OPAL_SUCCESS != opal_argv_delete(&argc, &a, 0, 99) || 0 != opal_argv_count(a)) {
         return false;
     }
     opal_argv_free(a);
@@ -490,9 +485,8 @@ static bool test9(void)
     argc = 0;
     opal_argv_append(&argc, &a, "a");
     opal_argv_append(&argc, &a, "b");
-    if (OPAL_SUCCESS != opal_argv_delete(&argc, &a, 1, 99) ||
-        1 != opal_argv_count(a) ||
-        0 != strcmp(a[0], "a")) {
+    if (OPAL_SUCCESS != opal_argv_delete(&argc, &a, 1, 99) || 1 != opal_argv_count(a)
+        || 0 != strcmp(a[0], "a")) {
         return false;
     }
     opal_argv_free(a);
@@ -502,7 +496,6 @@ static bool test9(void)
     return true;
 }
 
-
 static bool test10(void)
 {
     char **orig;
@@ -541,14 +534,10 @@ static bool test10(void)
     opal_argv_append(&o, &orig, "orig a");
     opal_argv_append(&o, &orig, "orig b");
     opal_argv_append(&o, &orig, "orig c");
-    if (OPAL_SUCCESS != opal_argv_insert(&orig, 99, insert) ||
-        6 != opal_argv_count(orig) ||
-        0 != strcmp(orig[0], "orig a") ||
-        0 != strcmp(orig[1], "orig b") ||
-        0 != strcmp(orig[2], "orig c") ||
-        0 != strcmp(orig[3], "insert a") ||
-        0 != strcmp(orig[4], "insert b") ||
-        0 != strcmp(orig[5], "insert c")) {
+    if (OPAL_SUCCESS != opal_argv_insert(&orig, 99, insert) || 6 != opal_argv_count(orig)
+        || 0 != strcmp(orig[0], "orig a") || 0 != strcmp(orig[1], "orig b")
+        || 0 != strcmp(orig[2], "orig c") || 0 != strcmp(orig[3], "insert a")
+        || 0 != strcmp(orig[4], "insert b") || 0 != strcmp(orig[5], "insert c")) {
         return false;
     }
     opal_argv_free(orig);
@@ -566,14 +555,10 @@ static bool test10(void)
     opal_argv_append(&o, &orig, "orig a");
     opal_argv_append(&o, &orig, "orig b");
     opal_argv_append(&o, &orig, "orig c");
-    if (OPAL_SUCCESS != opal_argv_insert(&orig, 0, insert) ||
-        6 != opal_argv_count(orig) ||
-        0 != strcmp(orig[3], "orig a") ||
-        0 != strcmp(orig[4], "orig b") ||
-        0 != strcmp(orig[5], "orig c") ||
-        0 != strcmp(orig[0], "insert a") ||
-        0 != strcmp(orig[1], "insert b") ||
-        0 != strcmp(orig[2], "insert c")) {
+    if (OPAL_SUCCESS != opal_argv_insert(&orig, 0, insert) || 6 != opal_argv_count(orig)
+        || 0 != strcmp(orig[3], "orig a") || 0 != strcmp(orig[4], "orig b")
+        || 0 != strcmp(orig[5], "orig c") || 0 != strcmp(orig[0], "insert a")
+        || 0 != strcmp(orig[1], "insert b") || 0 != strcmp(orig[2], "insert c")) {
         return false;
     }
     opal_argv_free(orig);
@@ -591,14 +576,10 @@ static bool test10(void)
     opal_argv_append(&o, &orig, "orig a");
     opal_argv_append(&o, &orig, "orig b");
     opal_argv_append(&o, &orig, "orig c");
-    if (OPAL_SUCCESS != opal_argv_insert(&orig, 1, insert) ||
-        6 != opal_argv_count(orig) ||
-        0 != strcmp(orig[0], "orig a") ||
-        0 != strcmp(orig[4], "orig b") ||
-        0 != strcmp(orig[5], "orig c") ||
-        0 != strcmp(orig[1], "insert a") ||
-        0 != strcmp(orig[2], "insert b") ||
-        0 != strcmp(orig[3], "insert c")) {
+    if (OPAL_SUCCESS != opal_argv_insert(&orig, 1, insert) || 6 != opal_argv_count(orig)
+        || 0 != strcmp(orig[0], "orig a") || 0 != strcmp(orig[4], "orig b")
+        || 0 != strcmp(orig[5], "orig c") || 0 != strcmp(orig[1], "insert a")
+        || 0 != strcmp(orig[2], "insert b") || 0 != strcmp(orig[3], "insert c")) {
         return false;
     }
     opal_argv_free(orig);
@@ -619,17 +600,12 @@ static bool test10(void)
     opal_argv_append(&o, &orig, "orig d");
     opal_argv_append(&o, &orig, "orig e");
     opal_argv_append(&o, &orig, "orig f");
-    if (OPAL_SUCCESS != opal_argv_insert(&orig, 1, insert) ||
-        9 != opal_argv_count(orig) ||
-        0 != strcmp(orig[0], "orig a") ||
-        0 != strcmp(orig[4], "orig b") ||
-        0 != strcmp(orig[5], "orig c") ||
-        0 != strcmp(orig[6], "orig d") ||
-        0 != strcmp(orig[7], "orig e") ||
-        0 != strcmp(orig[8], "orig f") ||
-        0 != strcmp(orig[1], "insert a") ||
-        0 != strcmp(orig[2], "insert b") ||
-        0 != strcmp(orig[3], "insert c")) {
+    if (OPAL_SUCCESS != opal_argv_insert(&orig, 1, insert) || 9 != opal_argv_count(orig)
+        || 0 != strcmp(orig[0], "orig a") || 0 != strcmp(orig[4], "orig b")
+        || 0 != strcmp(orig[5], "orig c") || 0 != strcmp(orig[6], "orig d")
+        || 0 != strcmp(orig[7], "orig e") || 0 != strcmp(orig[8], "orig f")
+        || 0 != strcmp(orig[1], "insert a") || 0 != strcmp(orig[2], "insert b")
+        || 0 != strcmp(orig[3], "insert c")) {
         return false;
     }
     opal_argv_free(orig);
diff --git a/test/util/opal_basename.c b/test/util/opal_basename.c
index e76a656448a..65de549ccca 100644
--- a/test/util/opal_basename.c
+++ b/test/util/opal_basename.c
@@ -20,37 +20,34 @@
 #include "ompi_config.h"
 
 #include <stdio.h>
-#include <string.h>
 #include <stdlib.h>
+#include <string.h>
 
-#include "support.h"
 #include "opal/util/basename.h"
 #include "opal/util/printf.h"
+#include "support.h"
 
+static void test(const char *in, const char *out);
 
-static void test(const char* in, const char* out);
-
-
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
-  test_init("opal_basename()");
+    test_init("opal_basename()");
 
-  test("foo.txt", "foo.txt");
-  test("/foo/bar/baz", "baz");
-  test("/yow.c", "yow.c");
-  test("/", "/");
+    test("foo.txt", "foo.txt");
+    test("/foo/bar/baz", "baz");
+    test("/yow.c", "yow.c");
+    test("/", "/");
 
-  test("foo.txt/", "foo.txt");
-  test("/foo/bar/baz/", "baz");
-  test("/yow.c/", "yow.c");
-  test("//", "/");
+    test("foo.txt/", "foo.txt");
+    test("/foo/bar/baz/", "baz");
+    test("/yow.c/", "yow.c");
+    test("//", "/");
 
-  /* All done */
-  return test_finalize();
+    /* All done */
+    return test_finalize();
 }
 
-
-void test(const char* in, const char* out)
+void test(const char *in, const char *out)
 {
     char *msg;
     char *ret = opal_basename(in);
@@ -58,8 +55,7 @@ void test(const char* in, const char* out)
     if (0 == strcmp(ret, out)) {
         test_success();
     } else {
-        opal_asprintf(&msg, "Mismatch: input \"%s\", expected \"%s\", got \"%s\"\n",
-                 in, out, ret);
+        opal_asprintf(&msg, "Mismatch: input \"%s\", expected \"%s\", got \"%s\"\n", in, out, ret);
         test_failure(msg);
         free(msg);
     }
@@ -67,5 +63,3 @@ void test(const char* in, const char* out)
         free(ret);
     }
 }
-
-
diff --git a/test/util/opal_bit_ops.c b/test/util/opal_bit_ops.c
index b915b2b242a..17618d8bd25 100644
--- a/test/util/opal_bit_ops.c
+++ b/test/util/opal_bit_ops.c
@@ -23,10 +23,10 @@
 #include <stdio.h>
 #include <string.h>
 
-#include "support.h"
 #include "opal/util/bit_ops.h"
 #include "opal/util/output.h"
 #include "opal/util/printf.h"
+#include "support.h"
 
 /*
 #define DEBUG
@@ -37,38 +37,52 @@ static int test_cube_dim(int value);
 static int test_next_poweroftwo(int value);
 static int test_next_poweroftwo_inclusive(int value);
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int i;
-    int vals[] = {0, 1, 2, 3, 4, 5, 127, 128, 129, (1 << 29) -1, (1 << 29), (1 << 29) +1, (1 << 30) -1, (1 << 30) /* And NOT (1 << 30) +1 */};
+    int vals[] = {
+        0,
+        1,
+        2,
+        3,
+        4,
+        5,
+        127,
+        128,
+        129,
+        (1 << 29) - 1,
+        (1 << 29),
+        (1 << 29) + 1,
+        (1 << 30) - 1,
+        (1 << 30), /* And NOT (1 << 30) +1 */
+    };
     test_init("opal_bit_ops()");
 
 #ifdef DEBUG
-    printf ("Test usage: ./opal_bit_ops [VALUES]\n");
+    printf("Test usage: ./opal_bit_ops [VALUES]\n");
 #endif
 
     if (1 < argc) {
         for (i = 1; i < argc; i++) {
             int value;
-            value = atoi (argv[i]);
-            printf ("Testing %d. argument test_next_poweroftwo(%d): %s\n",
-                    i, value, test_next_poweroftwo(value) ? "correct" : "wrong");
+            value = atoi(argv[i]);
+            printf("Testing %d. argument test_next_poweroftwo(%d): %s\n", i, value,
+                   test_next_poweroftwo(value) ? "correct" : "wrong");
         }
     }
 
-    for (i = 0; i < (int)(sizeof(vals)/sizeof(vals[0])); i++) {
-        test_hibit (vals[i], 8 * sizeof(int) -2);
-        test_hibit (vals[i], 3);
-        test_cube_dim (vals[i]);
-        test_next_poweroftwo (vals[i]);
-        test_next_poweroftwo_inclusive (vals[i]);
+    for (i = 0; i < (int) (sizeof(vals) / sizeof(vals[0])); i++) {
+        test_hibit(vals[i], 8 * sizeof(int) - 2);
+        test_hibit(vals[i], 3);
+        test_cube_dim(vals[i]);
+        test_next_poweroftwo(vals[i]);
+        test_next_poweroftwo_inclusive(vals[i]);
     }
 
     /* All done */
     return test_finalize();
 }
 
-
 /* REFERENCE FUNCTION */
 static int hibit(int value, int start)
 {
@@ -89,33 +103,32 @@ static int hibit(int value, int start)
 static int test_hibit(int value, int start)
 {
     int out;
-    int bit = hibit (value, start);
+    int bit = hibit(value, start);
 
 #ifdef DEBUG
-    printf ("test_hibit(): value:%d expect:%d\n",
-            value, bit);
+    printf("test_hibit(): value:%d expect:%d\n", value, bit);
 #endif
 
-    if (bit == (out = opal_hibit (value, start))) {
+    if (bit == (out = opal_hibit(value, start))) {
         test_success();
         return 1;
     } else {
-        char * msg;
+        char *msg;
         opal_asprintf(&msg, "Mismatch for hibit (w/ start:%d): value:%d, expected:%d got:%d\n",
-                 start, value, bit, out);
+                      start, value, bit, out);
         test_failure(msg);
         free(msg);
     }
     return 0;
 }
 
-
 /* REFERENCE FUNCTION */
 static int cube_dim(int value)
 {
     int dim, size;
 
-    for (dim = 0, size = 1; size < value; ++dim, size <<= 1);
+    for (dim = 0, size = 1; size < value; ++dim, size <<= 1)
+        ;
 
     return dim;
 }
@@ -123,69 +136,64 @@ static int cube_dim(int value)
 static int test_cube_dim(int value)
 {
     int out;
-    int dim = cube_dim (value);
+    int dim = cube_dim(value);
 
 #ifdef DEBUG
-    printf ("test_cube_dim(): value:%d expect:%d\n",
-            value, dim);
+    printf("test_cube_dim(): value:%d expect:%d\n", value, dim);
 #endif
 
-    if (dim == (out = opal_cube_dim (value))) {
+    if (dim == (out = opal_cube_dim(value))) {
         test_success();
         return 1;
     } else {
-        char * msg;
-        opal_asprintf(&msg, "Mismatch for cube_dim: value:%d, expected:%d got:%d\n",
-                 value, dim, out);
+        char *msg;
+        opal_asprintf(&msg, "Mismatch for cube_dim: value:%d, expected:%d got:%d\n", value, dim,
+                      out);
         test_failure(msg);
         free(msg);
     }
     return 0;
 }
 
-
 /* REFERENCE FUNCTION */
 static int next_poweroftwo(int value)
 {
     int power2;
 
-    for (power2 = 1; value; value >>=1, power2 <<=1) /* empty */;
+    for (power2 = 1; value; value >>= 1, power2 <<= 1) /* empty */
+        ;
 
     return power2;
 }
 
-
 static int test_next_poweroftwo(int value)
 {
     int out;
-    int power2 = next_poweroftwo (value);
+    int power2 = next_poweroftwo(value);
 
 #ifdef DEBUG
-    printf ("test_next_poweroftwo(): value:%d expect:%d\n",
-            value, power2);
+    printf("test_next_poweroftwo(): value:%d expect:%d\n", value, power2);
 #endif
 
-    if (power2 == (out = opal_next_poweroftwo (value))) {
+    if (power2 == (out = opal_next_poweroftwo(value))) {
         test_success();
         return 1;
     } else {
-        char * msg;
-        opal_asprintf(&msg, "Mismatch for power-of-two: value:%d, expected:%d got:%d\n",
-                 value, power2, out);
+        char *msg;
+        opal_asprintf(&msg, "Mismatch for power-of-two: value:%d, expected:%d got:%d\n", value,
+                      power2, out);
         test_failure(msg);
         free(msg);
     }
     return 0;
 }
 
-
-
 /* REFERENCE FUNCTION */
 static int next_poweroftwo_inclusive(int value)
 {
     int power2 = 1;
 
-    while ( power2 < value )
+    while (power2 < value)
         power2 <<= 1;
 
     return power2;
@@ -194,27 +202,22 @@ static int next_poweroftwo_inclusive(int value)
 static int test_next_poweroftwo_inclusive(int value)
 {
     int out;
-    int power2 = next_poweroftwo_inclusive (value);
+    int power2 = next_poweroftwo_inclusive(value);
 
 #ifdef DEBUG
-    printf ("test_next_poweroftwo(): value:%d expect:%d\n",
-            value, power2);
+    printf("test_next_poweroftwo(): value:%d expect:%d\n", value, power2);
 #endif
 
-    if (power2 == (out = opal_next_poweroftwo_inclusive (value))) {
+    if (power2 == (out = opal_next_poweroftwo_inclusive(value))) {
         test_success();
         return 1;
     } else {
-        char * msg;
+        char *msg;
         opal_asprintf(&msg, "Mismatch for power-of-two-inclusive: value:%d, expected:%d got:%d\n",
-                 value, power2, out);
+                      value, power2, out);
         test_failure(msg);
         free(msg);
     }
 
     return 0;
 }
-
-
-
-
diff --git a/test/util/opal_error.c b/test/util/opal_error.c
index f815dd9f0bb..c54743c165a 100644
--- a/test/util/opal_error.c
+++ b/test/util/opal_error.c
@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2024      Jeffrey M. Squyres.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -19,43 +20,44 @@
 #include "ompi_config.h"
 
 #include <stdio.h>
-#include <string.h>
 #include <stdlib.h>
+#include <string.h>
 #ifdef HAVE_SYS_PARAM_H
-#include <sys/param.h>
+#    include <sys/param.h>
 #endif
 #ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
+#    include <netinet/in.h>
 #endif
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 #ifdef HAVE_NETDB_H
-#include <netdb.h>
+#    include <netdb.h>
 #endif
 #include <errno.h>
 
-#include "opal/util/error.h"
 #include "opal/constants.h"
 #include "opal/runtime/opal.h"
+#include "opal/util/error.h"
 #include "orte/constants.h"
 
-int
-main(int argc, char *argv[])
+int main(int argc, char *argv[])
 {
     int i;
-    int errors[] = { OPAL_SUCCESS,
-                     OPAL_ERROR,
-                     OPAL_ERR_OUT_OF_RESOURCE,
-                     OPAL_ERR_NOT_FOUND,
-                     OPAL_ERR_BAD_PARAM,
-                     OPAL_ERR_MAX + 10, /* bad value */
-                     1 }; /* sentinal */
+    int errors[] = {
+        OPAL_SUCCESS,
+        OPAL_ERROR,
+        OPAL_ERR_OUT_OF_RESOURCE,
+        OPAL_ERR_NOT_FOUND,
+        OPAL_ERR_BAD_PARAM,
+        OPAL_ERR_MAX + 10, /* bad value */
+        1,                 /* sentinel */
+    };
     char buf[1024];
 
     opal_init(&argc, &argv);
 
-    for (i = 0 ; errors[i] <= 0 ; ++i) {
+    for (i = 0; errors[i] <= 0; ++i) {
         printf("--> error code: %d\n", errors[i]);
         opal_perror(errors[i], "perror test");
         printf("strerror test: %s\n", opal_strerror(errors[i]));
diff --git a/test/util/opal_if.c b/test/util/opal_if.c
index 7bb6feec78f..20d5def9ca2 100644
--- a/test/util/opal_if.c
+++ b/test/util/opal_if.c
@@ -21,29 +21,27 @@
 #include "ompi_config.h"
 
 #include <stdio.h>
-#include <string.h>
 #include <stdlib.h>
+#include <string.h>
 #ifdef HAVE_SYS_PARAM_H
-#include <sys/param.h>
+#    include <sys/param.h>
 #endif
 #ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
+#    include <netinet/in.h>
 #endif
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 #ifdef HAVE_NETDB_H
-#include <netdb.h>
+#    include <netdb.h>
 #endif
 
-#include "support.h"
+#include "opal/constants.h"
 #include "opal/runtime/opal.h"
 #include "opal/util/if.h"
-#include "opal/constants.h"
-
+#include "support.h"
 
-static bool
-test_ifaddrtoname(char *addr)
+static bool test_ifaddrtoname(char *addr)
 {
     int ret;
     char addrname[100];
@@ -58,8 +56,7 @@ test_ifaddrtoname(char *addr)
     }
 }
 
-int
-main(int argc, char *argv[])
+int main(int argc, char *argv[])
 {
     const char *hostname;
 
diff --git a/test/util/opal_json.c b/test/util/opal_json.c
new file mode 100644
index 00000000000..ed914dfdf12
--- /dev/null
+++ b/test/util/opal_json.c
@@ -0,0 +1,272 @@
+/*
+ * Copyright (c) 2024-2025 Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "opal_config.h"
+#include "opal/util/json/opal_json.h"
+#include "opal/runtime/opal.h"
+#include "support.h"
+#include <stdio.h>
+#include <string.h>
+
+enum INPUT_TYPE {
+    JSON_STRING,
+    JSON_FILE,
+    INPUT_TYPE_COUNT,
+};
+
+#define FREE_JSON(obj)          \
+    do {                        \
+        opal_json_free(&(obj)); \
+        obj = NULL;             \
+    } while (0)
+
+static int load_json(const char *string, enum INPUT_TYPE input_type, const opal_json_t **json)
+{
+    FILE *fp = NULL;
+    char filename[] = "XXXXXXXXXX";
+    int ret = 0, fd = -1;
+
+    switch (input_type) {
+    case JSON_STRING:
+        /* Load the input string */
+        ret = opal_json_load(string, strlen(string), json);
+        break;
+    case JSON_FILE:
+        /* Write the input string to a temporary file */
+        fd = mkstemp(filename);
+        if (fd < 0) {
+            test_failure("Failed to create JSON file");
+            ret = 1;
+            break;
+        }
+        fp = fdopen(fd, "w");
+        if (fp == NULL) {
+            test_failure("Failed to open JSON file");
+            ret = 1;
+            break;
+        }
+        fputs(string, fp);
+        fclose(fp);
+        close(fd);
+        /* Load the input string from the temporary file */
+        ret = opal_json_load_file(filename, json, 1);
+        /* Remember to delete the file */
+        (void) remove(filename);
+        break;
+    default:
+        test_failure("Unknown input type!");
+        ret = 1;
+    }
+    return ret;
+}
+
+static void test_json_integer_val(const opal_json_t *json, int64_t expected_value)
+{
+    int ret = 0;
+    int64_t val;
+    ret = opal_json_read_integer(json, &val);
+    if (ret) {
+        test_failure("Failed to read an integer value");
+    } else if (test_verify_int64_t(expected_value, val)) {
+        test_success();
+    } else {
+        test_failure("Got a wrong integer value");
+    }
+}
+
+static void test_json_double_val(const opal_json_t *json, double expected_value)
+{
+    int ret = 0;
+    double val;
+    ret = opal_json_read_double(json, &val);
+    if (ret) {
+        test_failure("Failed to read a double value");
+    } else if (test_verify_double(expected_value, val)) {
+        test_success();
+    } else {
+        test_failure("Got a wrong double value");
+    }
+}
+
+static void test_json_bool_val(const opal_json_t *json, bool expected_value)
+{
+    int ret = 0;
+    bool val;
+    ret = opal_json_read_bool(json, &val);
+    if (ret) {
+        test_failure("Failed to read a boolean value");
+    } else if (test_verify_int(expected_value, val)) {
+        test_success();
+    } else {
+        test_failure("Got a wrong boolean value");
+    }
+}
+
+static void test_json_string_val(const opal_json_t *json, const char *expected_value)
+{
+    int ret;
+    size_t len;
+    char *val = NULL;
+    ret = opal_json_read_string(json, (const char **) &val, &len);
+    if (ret) {
+        test_failure("Failed to read a string value");
+    } else if (test_verify_str(expected_value, val) && test_verify_size_t(strlen(val), len)) {
+        test_success();
+    } else {
+        test_failure("Got a wrong string value");
+    }
+}
+
+static void test_valid_json(void)
+{
+    int ret = 0;
+    size_t size;
+    const opal_json_t *json = NULL, *a = NULL, *b = NULL, *b0 = NULL, *b1 = NULL, *c = NULL,
+                      *d = NULL, *dummy = NULL;
+    const char *found_key;
+
+    /**
+     * Human readable form:
+     * {
+     *   "a": 1,
+     *   "b": [
+     *     2,
+     *     true
+     *   ],
+     *   "c": "this is a string",
+     *   "d": 3.456
+     * }
+     */
+    static const char *s
+        = "{\"a\": 1, \"b\": [2, true], \"c\": \"this is a string\", \"d\": 3.456}";
+
+    for (int input_type = JSON_STRING; input_type < INPUT_TYPE_COUNT; ++input_type) {
+        ret = load_json(s, input_type, &json);
+        if (ret) {
+            test_failure("Failed to load JSON");
+            continue;
+        } else {
+            test_success();
+        }
+
+        ret = opal_json_get_container_size(json, &size);
+        if (ret) {
+            test_failure("Failed to get object size");
+        } else {
+            test_verify_size_t(4, size);
+        }
+
+        ret = opal_json_get_key(json, "a", &a);
+        if (ret) {
+            test_failure("Failed to find a valid key");
+        } else {
+            test_json_integer_val(a, 1);
+        }
+
+        ret = opal_json_get_key(json, "b", &b);
+        if (ret) {
+            test_failure("Failed to find a valid key");
+        } else {
+            ret = opal_json_get_container_size(b, &size);
+            if (ret) {
+                test_failure("Failed to get array length");
+            } else {
+                test_verify_size_t(2, size);
+            }
+
+            ret = opal_json_get_index(b, 0, &b0);
+            if (ret) {
+                test_failure("Failed to get value at an index");
+            } else {
+                test_json_integer_val(b0, 2);
+            }
+
+            ret = opal_json_get_index(b, 1, &b1);
+            if (ret) {
+                test_failure("Failed to get value at an index");
+            } else {
+                test_json_bool_val(b1, true);
+            }
+        }
+
+        ret = opal_json_get_key(json, "c", &c);
+        if (ret) {
+            test_failure("Failed to find a valid key");
+        } else {
+            test_json_string_val(c, "this is a string");
+        }
+
+        ret = opal_json_get_key(json, "d", &d);
+        if (ret) {
+            test_failure("Failed to find a valid key");
+        } else {
+            test_json_double_val(d, 3.456);
+        }
+
+        ret = opal_json_get_key_by_index(json, 3, &found_key, &d);
+        if (ret) {
+            test_failure("opal_json_get_key_by_index failed when called with a valid index");
+        } else {
+            if ( 0 != strcmp(found_key, "d") ) {
+                test_failure("opal_json_get_key_by_index returned the wrong key value");
+            }
+            test_json_double_val(d, 3.456);
+        }
+
+        ret = opal_json_get_key_by_index(json, 4, &found_key, &dummy);
+        if (ret == OPAL_SUCCESS) {
+            test_failure("opal_json_get_key_by_index returned OPAL_SUCCESS when passed invalid index.");
+        }
+
+        /* JSON objects can be released in any order */
+        FREE_JSON(a);
+        FREE_JSON(b);
+        FREE_JSON(b0);
+        FREE_JSON(b1);
+        FREE_JSON(c);
+        FREE_JSON(json);
+    }
+}
+
+static void test_invalid_json_string(void)
+{
+    int ret = 0;
+    static const int cnt = 3;
+    char *test_cases[cnt];
+
+    test_cases[0] = "1,2,3";
+    test_cases[1] = "[1,2,3,]";
+    test_cases[2] = "{a: 1}";
+
+    for (int i = 0; i < 1; ++i) {
+        const opal_json_t *json = NULL;
+        char *test_case = test_cases[i];
+        for (int input_type = JSON_STRING; input_type < INPUT_TYPE_COUNT; ++input_type) {
+            ret = load_json(test_case, input_type, &json);
+            if (ret) {
+                test_success();
+            } else {
+                test_failure("Failed to return error for an invalid JSON string");
+            }
+            FREE_JSON(json);
+        }
+    }
+}
+
+int main(int argc, char **argv)
+{
+
+    opal_init(&argc, &argv);
+    test_init("opal_util_json");
+    test_valid_json();
+    test_invalid_json_string();
+    opal_finalize();
+    return test_finalize();
+}
diff --git a/test/util/opal_os_create_dirpath.c b/test/util/opal_os_create_dirpath.c
index 394e724dddc..ac9a26fe5a4 100644
--- a/test/util/opal_os_create_dirpath.c
+++ b/test/util/opal_os_create_dirpath.c
@@ -19,32 +19,31 @@
 #include "orte_config.h"
 
 #include <stdio.h>
-#include <string.h>
 #include <stdlib.h>
+#include <string.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
+#    include <unistd.h>
 #endif
 #ifdef HAVE_SYS_PARAM_H
-#include <sys/param.h>
+#    include <sys/param.h>
 #endif
 #include <sys/stat.h>
 
-#include "support.h"
-#include "opal/runtime/opal.h"
 #include "opal/constants.h"
-#include "opal/util/os_path.h"
+#include "opal/runtime/opal.h"
 #include "opal/util/os_dirpath.h"
+#include "opal/util/os_path.h"
+#include "support.h"
 
 #define PATH_SEP "/"
 
 static const char *path_sep = PATH_SEP;
 
-static bool test1(void);   /* trivial test */
-static bool test2(void);   /* test existing path, both with and without correct mode */
-static bool test3(void);   /* test making a directory tree */
+static bool test1(void); /* trivial test */
+static bool test2(void); /* test existing path, both with and without correct mode */
+static bool test3(void); /* test making a directory tree */
 
-
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     opal_init(&argc, &argv);
 
@@ -54,23 +53,20 @@ int main(int argc, char* argv[])
 
     if (test1()) {
         test_success();
-    }
-    else {
-      test_failure("opal_os_create_dirpath test1 failed");
+    } else {
+        test_failure("opal_os_create_dirpath test1 failed");
     }
 
     if (test2()) {
         test_success();
-    }
-    else {
-      test_failure("opal_os_create_dirpath test2 failed");
+    } else {
+        test_failure("opal_os_create_dirpath test2 failed");
     }
 
     if (test3()) {
         test_success();
-    }
-    else {
-      test_failure("opal_os_create_dirpath test3 failed");
+    } else {
+        test_failure("opal_os_create_dirpath test3 failed");
     }
 
     test_finalize();
@@ -79,19 +75,17 @@ int main(int argc, char* argv[])
     return 0;
 }
 
-
 static bool test1(void)
 {
 
     /* Test trivial functionality. Program should return OPAL_ERROR when called with NULL path. */
 
     if (OPAL_SUCCESS == opal_os_dirpath_create(NULL, S_IRWXU))
-            return(false);
+        return false;
 
     return true;
 }
 
-
 static bool test2(void)
 {
     char *tmp;
@@ -99,30 +93,30 @@ static bool test2(void)
 
     if (NULL == path_sep) {
         printf("test2 cannot be run\n");
-        return(false);
+        return false;
     }
     tmp = opal_os_path(true, "tmp", NULL);
     if (0 != mkdir(tmp, S_IRWXU)) {
         printf("test2 could not be run - directory could not be made\n");
-        return(false);
+        return false;
     }
 
     if (OPAL_SUCCESS != opal_os_dirpath_create(tmp, S_IRWXU)) {
         rmdir(tmp);
-        return(false);
+        return false;
     }
 
     chmod(tmp, S_IRUSR);
 
     if (OPAL_SUCCESS != opal_os_dirpath_create(tmp, S_IRWXU)) {
         rmdir(tmp);
-        return(false);
+        return false;
     }
 
     stat(tmp, &buf);
     if (S_IRWXU != (S_IRWXU & buf.st_mode)) {
         rmdir(tmp);
-        return(false);
+        return false;
     }
 
     rmdir(tmp);
@@ -131,16 +125,15 @@ static bool test2(void)
     return true;
 }
 
-
 static bool test3(void)
 {
     char *out;
     struct stat buf;
-    char *a[] = { "aaa", "bbb", "ccc", NULL };
+    char *a[] = {"aaa", "bbb", "ccc", NULL};
 
     if (NULL == path_sep) {
         printf("test3 cannot be run\n");
-        return(false);
+        return false;
     }
 
     out = opal_os_path(true, a[0], a[1], a[2], NULL);
@@ -154,10 +147,10 @@ static bool test3(void)
         out = opal_os_path(true, a[0], NULL);
         if (0 == stat(out, &buf))
             rmdir(out);
-        return(false);
+        return false;
     }
 
     free(out);
 
-    return(true);
+    return true;
 }
diff --git a/test/util/opal_os_path.c b/test/util/opal_os_path.c
index 67a5a169751..b5556401b60 100644
--- a/test/util/opal_os_path.c
+++ b/test/util/opal_os_path.c
@@ -18,10 +18,10 @@
 
 #include "orte_config.h"
 #include <stdio.h>
-#include <string.h>
 #include <stdlib.h>
+#include <string.h>
 #ifdef HAVE_SYS_PARAM_H
-#include <sys/param.h>
+#    include <sys/param.h>
 #endif
 
 #include "opal/runtime/opal.h"
@@ -32,13 +32,12 @@
 
 static const char *path_sep = PATH_SEP;
 
-static bool test1(void);   /* trivial answer test */
-static bool test2(void);   /* relative path test */
-static bool test3(void);   /* absolute path test */
-static bool test4(void);   /* missing path separator test */
-
+static bool test1(void); /* trivial answer test */
+static bool test2(void); /* relative path test */
+static bool test3(void); /* absolute path test */
+static bool test4(void); /* missing path separator test */
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     opal_init(&argc, &argv);
 
@@ -46,30 +45,26 @@ int main(int argc, char* argv[])
 
     if (test1()) {
         test_success();
-    }
-    else {
-      test_failure("opal_os_path_t test1 failed");
+    } else {
+        test_failure("opal_os_path_t test1 failed");
     }
 
     if (test2()) {
         test_success();
-    }
-    else {
-      test_failure("opal_os_path_t test2 failed");
+    } else {
+        test_failure("opal_os_path_t test2 failed");
     }
 
     if (test3()) {
         test_success();
-    }
-    else {
-      test_failure("opal_os_path_t test3 failed");
+    } else {
+        test_failure("opal_os_path_t test3 failed");
     }
 
     if (test4()) {
         test_success();
-    }
-    else {
-      test_failure("opal_os_path_t test4 failed");
+    } else {
+        test_failure("opal_os_path_t test4 failed");
     }
 
     opal_finalize();
@@ -78,40 +73,38 @@ int main(int argc, char* argv[])
     return 0;
 }
 
-
 static bool test1(void)
 {
     char *out, answer[100];
 
     /* Test trivial functionality. Program should return ".[separator]" when called in relative
      * mode, and the separator character when called in absolute mode. */
-    if (NULL != (out = opal_os_path(true,NULL))) {
+    if (NULL != (out = opal_os_path(true, NULL))) {
         answer[0] = '\0';
         strcat(answer, ".");
         strcat(answer, path_sep);
         if (0 != strcmp(answer, out))
-            return(false);
+            return false;
         free(out);
     }
-    if (NULL != (out = opal_os_path(false,NULL))) {
+    if (NULL != (out = opal_os_path(false, NULL))) {
         if (0 != strcmp(path_sep, out))
-            return(false);
+            return false;
         free(out);
     }
 
     return true;
 }
 
-
 static bool test2(void)
 {
     char out[1024];
     char *tmp;
-    char *a[] = { "aaa", "bbb", "ccc", NULL };
+    char *a[] = {"aaa", "bbb", "ccc", NULL};
 
     if (NULL == path_sep) {
         printf("test2 cannot be run\n");
-        return(false);
+        return false;
     }
 
     /* Construct a relative path name and see if it comes back correctly. Check multiple depths. */
@@ -122,36 +115,35 @@ static bool test2(void)
 
     tmp = opal_os_path(true, a[0], NULL);
     if (0 != strcmp(out, tmp))
-        return(false);
+        return false;
     free(tmp);
 
     strcat(out, path_sep);
     strcat(out, a[1]);
     tmp = opal_os_path(true, a[0], a[1], NULL);
     if (0 != strcmp(out, tmp))
-        return(false);
+        return false;
     free(tmp);
 
     strcat(out, path_sep);
     strcat(out, a[2]);
     tmp = opal_os_path(true, a[0], a[1], a[2], NULL);
     if (0 != strcmp(out, tmp))
-        return(false);
+        return false;
     free(tmp);
 
     return true;
 }
 
-
 static bool test3(void)
 {
     char out[1024];
     char *tmp;
-    char *a[] = { "aaa", "bbb", "ccc", NULL };
+    char *a[] = {"aaa", "bbb", "ccc", NULL};
 
     if (NULL == path_sep) {
         printf("test3 cannot be run\n");
-        return(false);
+        return false;
     }
 
     /* Same as prior test, only with absolute path name */
@@ -160,21 +152,21 @@ static bool test3(void)
     strcat(out, a[0]);
     tmp = opal_os_path(false, a[0], NULL);
     if (0 != strcmp(out, tmp))
-        return(false);
+        return false;
     free(tmp);
 
     strcat(out, path_sep);
     strcat(out, a[1]);
     tmp = opal_os_path(false, a[0], a[1], NULL);
     if (0 != strcmp(out, tmp))
-        return(false);
+        return false;
     free(tmp);
 
     strcat(out, path_sep);
     strcat(out, a[2]);
     tmp = opal_os_path(false, a[0], a[1], a[2], NULL);
     if (0 != strcmp(out, tmp))
-        return(false);
+        return false;
     free(tmp);
 
     return true;
@@ -182,20 +174,20 @@ static bool test3(void)
 
 static bool test4(void)
 {
-    char a[MAXPATHLEN + 10];
+    char a[OPAL_PATH_MAX + 10];
     int i;
 
     if (NULL == path_sep) {
         printf("test4 cannot be run\n");
-        return(false);
+        return false;
     }
 
-    for (i=0; i< MAXPATHLEN+5; i++) {
+    for (i = 0; i < OPAL_PATH_MAX + 5; i++) {
         a[i] = 'a';
     }
     a[i] = '\0';
     if (NULL != opal_os_path(false, a, NULL)) {
-        return(false);
+        return false;
     }
-    return (true);
+    return true;
 }
diff --git a/test/util/opal_path_nfs.c b/test/util/opal_path_nfs.c
index 5e31e829903..adce1d406b3 100644
--- a/test/util/opal_path_nfs.c
+++ b/test/util/opal_path_nfs.c
@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2010      Oak Ridge National Laboratory.
  *                         All rights reserved.
- * Copyright (c) 2010-2017 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2010-2022 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2010      IBM Corporation.  All rights reserved.
  * Copyright (c) 2014      Los Alamos National Security, LLC. All rights
  *                         reserved.
@@ -28,26 +28,26 @@
 
 #include "opal_config.h"
 
+#include <dirent.h>
 #include <stdio.h>
-#include <string.h>
 #include <stdlib.h>
-#include <dirent.h>
+#include <string.h>
 
 #include <sys/param.h>
 #ifdef HAVE_SYS_MOUNT_H
-#include <sys/mount.h>
+#    include <sys/mount.h>
 #endif
 #ifdef HAVE_SYS_STATFS_H
-#include <sys/statfs.h>
+#    include <sys/statfs.h>
 #endif
 #ifdef HAVE_SYS_VFS_H
-#include <sys/vfs.h>
+#    include <sys/vfs.h>
 #endif
 
-#include "support.h"
-#include "opal/util/path.h"
 #include "opal/util/output.h"
+#include "opal/util/path.h"
 #include "opal/util/printf.h"
+#include "support.h"
 
 #define DEBUG
 
@@ -65,15 +65,15 @@
  */
 
 #if !defined(__linux__)
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     if (argc > 1) {
         int i;
 
         printf("Interactive opal_path_nfs() test:\n");
         for (i = 1; i < argc; i++) {
-            printf ("Is dir[%d]:%s one of the detected network file systems? %s\n",
-                    i, argv[i], opal_path_nfs (argv[i], NULL) ? "Yes": "No");
+            printf("Is dir[%d]:%s one of the detected network file systems? %s\n", i, argv[i],
+                   opal_path_nfs(argv[i], NULL) ? "Yes" : "No");
         }
 
         return 0;
@@ -85,70 +85,67 @@ int main(int argc, char* argv[])
 
 #else /* __linux__ */
 
-static void test(char* file, bool expect);
-static void get_mounts (int * num_dirs, char ** dirs[], bool ** nfs);
+static void test(char *file, bool expect);
+static void get_mounts(int *num_dirs, char **dirs[], bool **nfs);
 
-
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     int num_dirs;
-    char ** dirs;
-    bool * nfs;
+    char **dirs;
+    bool *nfs;
 
     test_init("opal_path_nfs()");
-#ifdef DEBUG
-    printf ("Test usage: ./opal_path_nfs [DIR]\n");
-    printf ("On Linux interprets output from mount(8) to check for nfs and verify opal_path_nfs()\n");
-    printf ("Additionally, you may specify multiple DIR on the cmd-line, of which you the output\n");
-#endif
+#    ifdef DEBUG
+    printf("Test usage: ./opal_path_nfs [DIR]\n");
+    printf(
+        "On Linux interprets output from mount(8) to check for nfs and verify opal_path_nfs()\n");
+    printf("Additionally, you may specify multiple DIR on the cmd-line, of which you the output\n");
+#    endif
 
     if (1 < argc) {
         int i;
         for (i = 1; i < argc; i++)
-            printf ("Is dir[%d]:%s one of the detected network file systems? %s\n",
-                    i, argv[i], opal_path_nfs (argv[i], NULL) ? "Yes": "No");
+            printf("Is dir[%d]:%s one of the detected network file systems? %s\n", i, argv[i],
+                   opal_path_nfs(argv[i], NULL) ? "Yes" : "No");
     }
 
-    get_mounts (&num_dirs, &dirs, &nfs);
+    get_mounts(&num_dirs, &dirs, &nfs);
     while (num_dirs--) {
-        test (dirs[num_dirs], nfs[num_dirs]);
+        test(dirs[num_dirs], nfs[num_dirs]);
     }
 
     /* All done */
     return test_finalize();
 }
 
-
-void test(char* file, bool expect)
+void test(char *file, bool expect)
 {
-#ifdef DEBUG
-    printf ("test(): file:%s bool:%d\n",
-            file, expect);
-#endif
-    if (expect == opal_path_nfs (file, NULL)) {
+#    ifdef DEBUG
+    printf("test(): file:%s bool:%d\n", file, expect);
+#    endif
+    if (expect == opal_path_nfs(file, NULL)) {
         test_success();
     } else {
-        char * msg;
-        opal_asprintf(&msg, "Mismatch: input \"%s\", expected:%d got:%d\n",
-                 file, expect, !expect);
+        char *msg;
+        opal_asprintf(&msg, "Mismatch: input \"%s\", expected:%d got:%d\n", file, expect, !expect);
         test_failure(msg);
         free(msg);
     }
 }
 
-void get_mounts (int * num_dirs, char ** dirs[], bool * nfs[])
+void get_mounts(int *num_dirs, char **dirs[], bool *nfs[])
 {
-#define SIZE 1024
-    char * cmd = "mount | cut -f3,5 -d' ' > opal_path_nfs.out";
+#    define SIZE 1024
+    char *cmd = "mount | cut -f3,5 -d' ' > opal_path_nfs.out";
     int rc;
     int i;
-    FILE * file;
-    char ** dirs_tmp;
-    bool * nfs_tmp;
+    FILE *file;
+    char **dirs_tmp;
+    bool *nfs_tmp;
     char buffer[SIZE];
     struct statfs mystatfs;
 
-    rc = system (cmd);
+    rc = system(cmd);
 
     if (-1 == rc) {
         *num_dirs = 0;
@@ -164,7 +161,7 @@ void get_mounts (int * num_dirs, char ** dirs[], bool * nfs[])
        have an array large enough. */
     file = fopen("opal_path_nfs.out", "r");
     int count = 0;
-    while (NULL != fgets (buffer, SIZE, file)) {
+    while (NULL != fgets(buffer, SIZE, file)) {
         ++count;
     }
     printf("Found %d mounts\n", count);
@@ -172,22 +169,22 @@ void get_mounts (int * num_dirs, char ** dirs[], bool * nfs[])
     // Add one more so we can have a NULL entry at the end
     ++count;
 
-    dirs_tmp = (char**) calloc (count, sizeof(char*));
-    nfs_tmp = (bool*) calloc (count, sizeof(bool));
+    dirs_tmp = (char **) calloc(count, sizeof(char *));
+    nfs_tmp = (bool *) calloc(count, sizeof(bool));
 
     i = 0;
     rc = 4711;
     rewind(file);
     // i should never be more than count, but be safe anyway.
-    while (i < count && NULL != fgets (buffer, SIZE, file)) {
+    while (i < count && NULL != fgets(buffer, SIZE, file)) {
         int mount_known;
         char fs[MAXNAMLEN];
 
         if (!dirs_tmp[i]) {
-            dirs_tmp[i] = malloc (MAXNAMLEN);
+            dirs_tmp[i] = malloc(MAXNAMLEN);
         }
 
-        if (2 != (rc = sscanf (buffer, "%s %s\n", dirs_tmp[i], fs))) {
+        if (2 != (rc = sscanf(buffer, "%s %s\n", dirs_tmp[i], fs))) {
             goto out;
         }
 
@@ -196,7 +193,7 @@ void get_mounts (int * num_dirs, char ** dirs[], bool * nfs[])
          * Cannot distinguish it from NFS in opal_path_nfs, therefore just
          * disregard it, as it is NOT an parallel filesystem...
          */
-        if (0 == strcasecmp (fs, "rpc_pipefs")) {
+        if (0 == strcasecmp(fs, "rpc_pipefs")) {
             continue;
         }
 
@@ -207,13 +204,13 @@ void get_mounts (int * num_dirs, char ** dirs[], bool * nfs[])
         }
 
         /* If we get an fs type of "none", skip it (e.g.,
-           http://www.open-mpi.org/community/lists/devel/2012/09/11493.php) */
+           https://www.open-mpi.org/community/lists/devel/2012/09/11493.php) */
         if (0 == strcasecmp(fs, "none")) {
             continue;
         }
 
         /* If we can not stat the fs, skip it */
-        if (statfs (dirs_tmp[i], &mystatfs)) {
+        if (statfs(dirs_tmp[i], &mystatfs)) {
             continue;
         }
 
@@ -223,31 +220,26 @@ void get_mounts (int * num_dirs, char ** dirs[], bool * nfs[])
          */
         for (mount_known = 0; mount_known < i; mount_known++) {
             /* If we know this mount-point, then exit early */
-            if (0 == strcasecmp (dirs_tmp[mount_known], dirs_tmp[i])) {
-#ifdef DEBUG
-                printf ("get_mounts: already know dir[%d]:%s\n",
-                        mount_known, dirs_tmp[mount_known]);
-#endif
+            if (0 == strcasecmp(dirs_tmp[mount_known], dirs_tmp[i])) {
+#    ifdef DEBUG
+                printf("get_mounts: already know dir[%d]:%s\n", mount_known, dirs_tmp[mount_known]);
+#    endif
                 break;
             }
         }
 
         nfs_tmp[mount_known] = false;
-        if (0 == strcasecmp (fs, "nfs") ||
-            0 == strcasecmp (fs, "nfs4") ||
-            0 == strcasecmp (fs, "lustre") ||
-            0 == strcasecmp (fs, "panfs") ||
-            0 == strcasecmp (fs, "gpfs"))
+        if (0 == strcasecmp(fs, "nfs") || 0 == strcasecmp(fs, "nfs4")
+            || 0 == strcasecmp(fs, "lustre") || 0 == strcasecmp(fs, "panfs")
+            || 0 == strcasecmp(fs, "gpfs"))
             nfs_tmp[mount_known] = true;
-#ifdef DEBUG
-        printf ("get_mounts: dirs[%d]:%s fs:%s nfs:%s\n",
-                mount_known, dirs_tmp[mount_known],
-                fs, nfs_tmp[mount_known] ? "Yes" : "No");
-#endif
+#    ifdef DEBUG
+        printf("get_mounts: dirs[%d]:%s fs:%s nfs:%s\n", mount_known, dirs_tmp[mount_known], fs,
+               nfs_tmp[mount_known] ? "Yes" : "No");
+#    endif
 
         if (mount_known >= i)
             i++;
-
     }
 out:
     *num_dirs = i;
diff --git a/test/util/opal_sha256.c b/test/util/opal_sha256.c
new file mode 100644
index 00000000000..b8862ea3b00
--- /dev/null
+++ b/test/util/opal_sha256.c
@@ -0,0 +1,78 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2022      Triad National Security, LLC. All rights
+ *                         reserved.
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/*********************************************************************
+* Filename:   sha256.c
+* Author:     Brad Conte (brad AT bradconte.com)
+* Copyright:
+* Disclaimer: This code is presented "as is" without any guarantees.
+* Details:    Performs known-answer tests on the corresponding SHA1
+              implementation. These tests do not encompass the full
+              range of available test vectors, however, if the tests
+              pass it is very, very likely that the code is correct
+              and was compiled properly. This code also serves as
+              example usage of the functions.
+*********************************************************************/
+
+/*************************** HEADER FILES ***************************/
+#include <stdio.h>
+#include <memory.h>
+#include <string.h>
+#include "support.h"
+#include "opal/util/sha256.h"
+
+/*********************** FUNCTION DEFINITIONS ***********************/
+static int sha256_test(void)
+{
+    BYTE text1[] = {"abc"};
+    BYTE text2[] = {"abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"};
+    BYTE text3[] = {"aaaaaaaaaa"};
+    BYTE hash1[OPAL_SHA256_BLOCK_SIZE] = {0xba,0x78,0x16,0xbf,0x8f,0x01,0xcf,0xea,0x41,0x41,0x40,0xde,0x5d,0xae,0x22,0x23,
+                                     0xb0,0x03,0x61,0xa3,0x96,0x17,0x7a,0x9c,0xb4,0x10,0xff,0x61,0xf2,0x00,0x15,0xad};
+    BYTE hash2[OPAL_SHA256_BLOCK_SIZE] = {0x24,0x8d,0x6a,0x61,0xd2,0x06,0x38,0xb8,0xe5,0xc0,0x26,0x93,0x0c,0x3e,0x60,0x39,
+                                     0xa3,0x3c,0xe4,0x59,0x64,0xff,0x21,0x67,0xf6,0xec,0xed,0xd4,0x19,0xdb,0x06,0xc1};
+    BYTE hash3[OPAL_SHA256_BLOCK_SIZE] = {0xcd,0xc7,0x6e,0x5c,0x99,0x14,0xfb,0x92,0x81,0xa1,0xc7,0xe2,0x84,0xd7,0x3e,0x67,
+                                     0xf1,0x80,0x9a,0x48,0xa4,0x97,0x20,0x0e,0x04,0x6d,0x39,0xcc,0xc7,0x11,0x2c,0xd0};
+    BYTE buf[OPAL_SHA256_BLOCK_SIZE];
+    opal_sha256_ctx ctx;
+    int idx;
+    int pass = 1;
+
+    opal_sha256_init(&ctx);
+    opal_sha256_update(&ctx, text1, strlen((const char *)text1));
+    opal_sha256_final(&ctx, buf);
+    pass = pass && !memcmp(hash1, buf, OPAL_SHA256_BLOCK_SIZE);
+
+    opal_sha256_init(&ctx);
+    opal_sha256_update(&ctx, text2, strlen((const char *)text2));
+    opal_sha256_final(&ctx, buf);
+    pass = pass && !memcmp(hash2, buf, OPAL_SHA256_BLOCK_SIZE);
+
+    opal_sha256_init(&ctx);
+    for (idx = 0; idx < 100000; ++idx)
+       opal_sha256_update(&ctx, text3, strlen((const char *)text3));
+    opal_sha256_final(&ctx, buf);
+    pass = pass && !memcmp(hash3, buf, OPAL_SHA256_BLOCK_SIZE);
+
+    return pass;
+}
+
+int main(void)
+{
+    test_init("sha256 test");
+
+    if (sha256_test()) {
+        test_success();
+    } else {
+        test_failure("sh256 test failed");
+    }
+
+    return test_finalize();
+}
diff --git a/test/util/opal_timer.c b/test/util/opal_timer.c
index 8873577453a..d277861b215 100644
--- a/test/util/opal_timer.c
+++ b/test/util/opal_timer.c
@@ -20,11 +20,10 @@
 
 #include <unistd.h>
 
-#include "opal/runtime/opal.h"
 #include "opal/mca/timer/base/base.h"
+#include "opal/runtime/opal.h"
 
-int
-main(int argc, char *argv[])
+int main(int argc, char *argv[])
 {
     opal_timer_t start, end, diff;
 
@@ -39,8 +38,7 @@ main(int argc, char *argv[])
     sleep(1);
     end = opal_timer_base_get_cycles();
     diff = end - start;
-    printf("    Slept approximately %llu cycles, or %llu us\n",
-           (unsigned long long) diff,
+    printf("    Slept approximately %llu cycles, or %llu us\n", (unsigned long long) diff,
            (unsigned long long) ((diff * 1000000) / opal_timer_base_get_freq()));
 #else
     printf("--> cycle count not supported\n");
diff --git a/test/util/orte_session_dir.c b/test/util/orte_session_dir.c
index 74269898661..fb9152b5d57 100644
--- a/test/util/orte_session_dir.c
+++ b/test/util/orte_session_dir.c
@@ -19,116 +19,105 @@
 #include "orte_config.h"
 
 #include <stdio.h>
-#include <string.h>
 #include <stdlib.h>
+#include <string.h>
 #ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif  /* HAVE_UNISTD_H */
+#    include <unistd.h>
+#endif /* HAVE_UNISTD_H */
 #ifdef HAVE_SYS_PARAM_H
-#include <sys/param.h>
-#endif  /* HAVE_SYS_PARAM_H */
+#    include <sys/param.h>
+#endif /* HAVE_SYS_PARAM_H */
 #include <sys/stat.h>
 
-#include "support.h"
+#include "opal/util/os_path.h"
 #include "orte/constants.h"
 #include "orte/util/proc_info.h"
-#include "opal/util/os_path.h"
 #include "orte/util/session_dir.h"
-#include "orte/util/proc_info.h"
-
+#include "support.h"
 
-static bool test1(void);   /* given prefix, both one that works and one that fails */
-static bool test2(void);   /* no prefix given, ORTE_PREFIX_ENV set, one good and one bad */
-static bool test3(void);   /* no prefix given, TMPDIR set, one good and one bad */
-static bool test4(void);   /* no prefix given, TMP set, one good and one bad */
-static bool test5(void);   /* no prefix given, HOME set, one good and one bad */
-static bool test6(void);   /* no prefix given, nothing set, one good and one bad */
-static bool test7(void);   /* remove session directory tree */
-static bool test8(void);   /* attempt to remove tree when subdirs present */
+static bool test1(void); /* given prefix, both one that works and one that fails */
+static bool test2(void); /* no prefix given, ORTE_PREFIX_ENV set, one good and one bad */
+static bool test3(void); /* no prefix given, TMPDIR set, one good and one bad */
+static bool test4(void); /* no prefix given, TMP set, one good and one bad */
+static bool test5(void); /* no prefix given, HOME set, one good and one bad */
+static bool test6(void); /* no prefix given, nothing set, one good and one bad */
+static bool test7(void); /* remove session directory tree */
+static bool test8(void); /* attempt to remove tree when subdirs present */
 
 void clear_proc_info(void);
 
-static FILE *test_out=NULL;
+static FILE *test_out = NULL;
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     orte_proc_info(); /* initialize proc info structure */
-    orte_process_info.my_name = (orte_process_name_t*)malloc(sizeof(orte_process_name_t));
+    orte_process_info.my_name = (orte_process_name_t *) malloc(sizeof(orte_process_name_t));
     orte_process_info.my_name->cellid = 0;
     orte_process_info.my_name->jobid = 0;
     orte_process_info.my_name->vpid = 0;
 
     test_init("orte_session_dir_t");
-    test_out = fopen( "test_session_dir_out", "w+" );
-    if( test_out == NULL ) {
-      test_failure("test_session_dir couldn't open test file failed");
-      test_finalize();
-      exit(1);
+    test_out = fopen("test_session_dir_out", "w+");
+    if (test_out == NULL) {
+        test_failure("test_session_dir couldn't open test file failed");
+        test_finalize();
+        exit(1);
     }
 
-
     fprintf(test_out, "running test1\n");
     if (test1()) {
         test_success();
-    }
-    else {
-      test_failure("orte_session_dir_t test1 failed");
+    } else {
+        test_failure("orte_session_dir_t test1 failed");
     }
 
     fprintf(test_out, "running test2\n");
     if (test2()) {
         test_success();
-    }
-    else {
-      test_failure("orte_session_dir_t test2 failed");
+    } else {
+        test_failure("orte_session_dir_t test2 failed");
     }
 
     fprintf(test_out, "running test3\n");
     if (test3()) {
         test_success();
-    }
-    else {
-      test_failure("orte_session_dir_t test3 failed");
+    } else {
+        test_failure("orte_session_dir_t test3 failed");
     }
 
     fprintf(test_out, "running test4\n");
     if (test4()) {
         test_success();
-    }
-    else {
-      test_failure("orte_session_dir_t test4 failed");
+    } else {
+        test_failure("orte_session_dir_t test4 failed");
     }
 
     fprintf(test_out, "running test5\n");
     if (test5()) {
         test_success();
-    }
-    else {
-      test_failure("orte_session_dir_t test5 failed");
+    } else {
+        test_failure("orte_session_dir_t test5 failed");
     }
 
     fprintf(test_out, "running test6\n");
     if (test6()) {
         test_success();
-    }
-    else {
-      test_failure("orte_session_dir_t test6 failed");
+    } else {
+        test_failure("orte_session_dir_t test6 failed");
     }
 
     fprintf(test_out, "running test7\n");
     if (test7()) {
         test_success();
-    }
-    else {
-      test_failure("orte_session_dir_t test7 failed");
+    } else {
+        test_failure("orte_session_dir_t test7 failed");
     }
 
     fprintf(test_out, "running test8\n");
     if (test8()) {
         test_success();
-    }
-    else {
-      test_failure("orte_session_dir_t test8 failed");
+    } else {
+        test_failure("orte_session_dir_t test8 failed");
     }
 
     fprintf(test_out, "completed all tests\n");
@@ -142,7 +131,6 @@ int main(int argc, char* argv[])
     return 0;
 }
 
-
 static bool test1(void)
 {
     /* Test proper action when given a prefix */
@@ -155,16 +143,16 @@ static bool test1(void)
 
     prefix = opal_os_path(false, "tmp", NULL);
     if (ORTE_SUCCESS != orte_session_dir(true, prefix, NULL, NULL, "test-universe", NULL, NULL)) {
-	fprintf(test_out, "test1 - couldn't create specified path\n");
+        fprintf(test_out, "test1 - couldn't create specified path\n");
         free(prefix);
-        return(false);
+        return false;
     }
     /* see if it can access an existing path */
 
     if (ORTE_SUCCESS != orte_session_dir(false, prefix, NULL, NULL, "test-universe", NULL, NULL)) {
-	fprintf(test_out, "test1 - couldn't access existing path\n");
+        fprintf(test_out, "test1 - couldn't access existing path\n");
         free(prefix);
-        return(false);
+        return false;
     }
 
     orte_session_dir_finalize(orte_process_info.my_name);
@@ -174,7 +162,6 @@ static bool test1(void)
     return true;
 }
 
-
 static bool test2(void)
 {
     clear_proc_info();
@@ -184,19 +171,17 @@ static bool test2(void)
     setenv("OMPI_PREFIX_ENV", "/tmp/trythis", 1);
 
     if (ORTE_SUCCESS != orte_session_dir(true, NULL, NULL, NULL, "test-universe", NULL, NULL)) {
-	unsetenv("OMPI_PREFIX_ENV");
-        return(false);
+        unsetenv("OMPI_PREFIX_ENV");
+        return false;
     }
 
     orte_session_dir_finalize(orte_process_info.my_name);
 
     unsetenv("OMPI_PREFIX_ENV");
 
-    return(true);
-
+    return true;
 }
 
-
 static bool test3(void)
 {
     /* use the TMPDIR enviro variable */
@@ -205,18 +190,17 @@ static bool test3(void)
     setenv("TMPDIR", "/tmp/trythis", 1);
 
     if (ORTE_SUCCESS != orte_session_dir(true, NULL, NULL, NULL, "test-universe", NULL, NULL)) {
-	unsetenv("TMPDIR");
-        return(false);
+        unsetenv("TMPDIR");
+        return false;
     }
 
     orte_session_dir_finalize(orte_process_info.my_name);
 
     unsetenv("TMPDIR");
 
-    return(true);
+    return true;
 }
 
-
 static bool test4(void)
 {
     /* use the TMP enviro variable */
@@ -226,18 +210,17 @@ static bool test4(void)
     setenv("TMP", "/tmp/trythis", 1);
 
     if (ORTE_SUCCESS != orte_session_dir(true, NULL, NULL, NULL, "test-universe", NULL, NULL)) {
-	unsetenv("TMP");
-        return(false);
+        unsetenv("TMP");
+        return false;
     }
 
     orte_session_dir_finalize(orte_process_info.my_name);
 
     unsetenv("TMP");
 
-    return(true);
+    return true;
 }
 
-
 static bool test5(void)
 {
     /* use the HOME enviro variable */
@@ -247,34 +230,33 @@ static bool test5(void)
     setenv("HOME", "/tmp/trythis", 1);
 
     if (ORTE_SUCCESS != orte_session_dir(true, NULL, NULL, NULL, "test-universe", NULL, NULL)) {
-	unsetenv("HOME");
-        return(false);
+        unsetenv("HOME");
+        return false;
     }
 
     orte_session_dir_finalize(orte_process_info.my_name);
 
     unsetenv("HOME");
 
-    return(true);
+    return true;
 }
 
-
 static bool test6(void)
 {
 
     clear_proc_info();
 
     /* no enviro variables set, no prefix given
-    * Program should turn to default of /tmp (where "/" is whatever
-    * top-level directory is appropriate for given system)
-    */
+     * Program should turn to default of /tmp (where "/" is whatever
+     * top-level directory is appropriate for given system)
+     */
     if (ORTE_SUCCESS != orte_session_dir(true, NULL, NULL, NULL, "test-universe", NULL, NULL)) {
-        return(false);
+        return false;
     }
 
     orte_session_dir_finalize(orte_process_info.my_name);
 
-    return(true);
+    return true;
 }
 
 static bool test7(void)
@@ -286,14 +268,14 @@ static bool test7(void)
     clear_proc_info();
 
     /* create test proc session directory tree */
-    if (ORTE_SUCCESS != orte_session_dir(true, NULL, "localhost", NULL, "test-universe", "test-job", "test-proc")) {
-	return(false);
+    if (ORTE_SUCCESS
+        != orte_session_dir(true, NULL, "localhost", NULL, "test-universe", "test-job",
+                            "test-proc")) {
+        return false;
     }
 
-    fprintf(test_out, "removing directories: %s\n\t%s\n\t%s\n",
-	    orte_process_info.proc_session_dir,
-	    orte_process_info.job_session_dir,
-	    orte_process_info.universe_session_dir);
+    fprintf(test_out, "removing directories: %s\n\t%s\n\t%s\n", orte_process_info.proc_session_dir,
+            orte_process_info.job_session_dir, orte_process_info.universe_session_dir);
 
     /* create some files */
 
@@ -313,10 +295,11 @@ static bool test7(void)
     fclose(fp);
 
     if (ORTE_SUCCESS != orte_session_dir_finalize(orte_process_info.my_name)) {
-	return(false);
+        return false;
     }
 
-    for (i=0; i < 3; i++) unlink(filenm[i]);
+    for (i = 0; i < 3; i++)
+        unlink(filenm[i]);
     orte_session_dir_finalize(orte_process_info.my_name);
 
     return true;
@@ -331,14 +314,14 @@ static bool test8(void)
     clear_proc_info();
 
     /* create test proc session directory tree */
-    if (ORTE_SUCCESS != orte_session_dir(true, NULL, "localhost", NULL, "test-universe2", "test-job2", "test-proc2")) {
-	return(false);
+    if (ORTE_SUCCESS
+        != orte_session_dir(true, NULL, "localhost", NULL, "test-universe2", "test-job2",
+                            "test-proc2")) {
+        return false;
     }
 
-    fprintf(test_out, "removing directories: %s\n\t%s\n\t%s\n",
-	    orte_process_info.proc_session_dir,
-	    orte_process_info.job_session_dir,
-	    orte_process_info.universe_session_dir);
+    fprintf(test_out, "removing directories: %s\n\t%s\n\t%s\n", orte_process_info.proc_session_dir,
+            orte_process_info.job_session_dir, orte_process_info.universe_session_dir);
 
     /* create some files */
 
@@ -357,12 +340,12 @@ static bool test8(void)
     fprintf(fp, "ss");
     fclose(fp);
 
-
     if (ORTE_SUCCESS != orte_session_dir_finalize(orte_process_info.my_name)) {
-	   return(false);
+        return false;
     }
 
-    for (i=0; i < 3; i++) unlink(filenm[i]);
+    for (i = 0; i < 3; i++)
+        unlink(filenm[i]);
     orte_session_dir_finalize(orte_process_info.my_name);
 
     return true;
@@ -375,5 +358,4 @@ void clear_proc_info(void)
     orte_process_info.universe_session_dir = NULL;
     orte_process_info.job_session_dir = NULL;
     orte_process_info.proc_session_dir = NULL;
-
 }
diff --git a/test/util/orte_universe_setup_file_io.c b/test/util/orte_universe_setup_file_io.c
index a25a082121a..223d0a29f38 100644
--- a/test/util/orte_universe_setup_file_io.c
+++ b/test/util/orte_universe_setup_file_io.c
@@ -20,22 +20,22 @@
 #include "orte/constants.h"
 
 #include <stdio.h>
-#include <string.h>
 #include <stdlib.h>
+#include <string.h>
 #ifdef HAVE_SYS_PARAM_H
-#include <sys/param.h>
+#    include <sys/param.h>
 #endif
 
 #include "orte/util/univ_info.h"
 #include "orte/util/universe_setup_file_io.h"
 #include "support.h"
 
-static bool test1(void);   /* verify it returns info */
-static bool test2(void);   /* test second time through */
+static bool test1(void); /* verify it returns info */
+static bool test2(void); /* test second time through */
 
 orte_universe_t orte_universe_info;
 
-int main(int argc, char* argv[])
+int main(int argc, char *argv[])
 {
     test_init("universe_setup_file_io");
 
@@ -52,23 +52,20 @@ int main(int argc, char* argv[])
 
     if (test1()) {
         test_success();
-    }
-    else {
-      test_failure("universe_setup_file_io write failed");
+    } else {
+        test_failure("universe_setup_file_io write failed");
     }
 
     if (test2()) {
         test_success();
-    }
-    else {
-      test_failure("universe_setup_file_io read failed");
+    } else {
+        test_failure("universe_setup_file_io read failed");
     }
 
     test_finalize();
     return 0;
 }
 
-
 static bool test1(void)
 {
     int rc;
@@ -83,7 +80,6 @@ static bool test1(void)
     return true;
 }
 
-
 static bool test2(void)
 {
     int rc;
@@ -96,15 +92,15 @@ static bool test2(void)
         return false;
     }
 
-    if (0 != strcmp(orte_universe_info.name, univ.name) ||
-        0 != strcmp(orte_universe_info.host, univ.host) ||
-        0 != strcmp(orte_universe_info.uid, univ.uid) ||
-        orte_universe_info.persistence != univ.persistence ||
-        0 != strcmp(orte_universe_info.scope, univ.scope) ||
-        orte_universe_info.console != univ.console ||
-        0 != strcmp(orte_universe_info.seed_uri, univ.seed_uri) ||
-        orte_universe_info.console_connected != univ.console_connected ||
-        orte_universe_info.scriptfile != univ.scriptfile) {
+    if (0 != strcmp(orte_universe_info.name, univ.name)
+        || 0 != strcmp(orte_universe_info.host, univ.host)
+        || 0 != strcmp(orte_universe_info.uid, univ.uid)
+        || orte_universe_info.persistence != univ.persistence
+        || 0 != strcmp(orte_universe_info.scope, univ.scope)
+        || orte_universe_info.console != univ.console
+        || 0 != strcmp(orte_universe_info.seed_uri, univ.seed_uri)
+        || orte_universe_info.console_connected != univ.console_connected
+        || orte_universe_info.scriptfile != univ.scriptfile) {
         fprintf(stderr, "universe_setup_file_io: read mismatch\n");
         return false;
     }